2023-09-23T20:21:36.053 INFO:root:teuthology version: 0.0.1.dev194+g54e62bc 2023-09-23T20:21:36.053 DEBUG:teuthology.run:Teuthology command: teuthology --verbose --name teuthology-2023-09-23_14:23:10-upgrade:quincy-x-reef-distro-default-smithi --owner scheduled_teuthology@teuthology --archive /home/teuthworker/archive/teuthology-2023-09-23_14:23:10-upgrade:quincy-x-reef-distro-default-smithi/7401835 --description upgrade:quincy-x/stress-split/{0-distro/rhel_8.6_container_tools_rhel8 0-roles 1-start 2-first-half-tasks/rbd-cls 3-stress-tasks/{radosbench rbd-cls rbd-import-export rbd_api readwrite snaps-few-objects} 4-second-half-tasks/radosbench mon_election/connectivity} -- /home/teuthworker/archive/teuthology-2023-09-23_14:23:10-upgrade:quincy-x-reef-distro-default-smithi/7401835/orig.config.yaml 2023-09-23T20:21:36.111 DEBUG:teuthology.report:Pushing job info to https://paddles.front.sepia.ceph.com/ 2023-09-23T20:21:36.204 INFO:teuthology.run:Config: archive_path: /home/teuthworker/archive/teuthology-2023-09-23_14:23:10-upgrade:quincy-x-reef-distro-default-smithi/7401835 branch: reef description: upgrade:quincy-x/stress-split/{0-distro/rhel_8.6_container_tools_rhel8 0-roles 1-start 2-first-half-tasks/rbd-cls 3-stress-tasks/{radosbench rbd-cls rbd-import-export rbd_api readwrite snaps-few-objects} 4-second-half-tasks/radosbench mon_election/connectivity} email: ceph-qa@ceph.io first-half-sequence: - cephadm.shell: env: - sha1 mon.a: - ceph config set mgr mgr/cephadm/daemon_cache_timeout 60 - ceph config set global log_to_journald false --force - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 - ceph orch ps - echo wait for minority of mons to upgrade - while ! ceph mon versions | grep $sha1 ; do sleep 2 ; done - ceph orch ps - ceph orch upgrade pause - sleep 60 - ceph orch upgrade resume - echo wait for majority of mons to upgrade - 'while ! ceph mon versions | grep $sha1 | egrep '': [23]'' ; do sleep 2 ; done' - ceph orch ps - ceph orch upgrade pause - sleep 60 - ceph orch upgrade resume - echo wait for all mons to upgrade - 'while ! ceph mon versions | grep $sha1 | grep '': 3'' ; do sleep 2 ; done' - ceph orch ps - ceph orch upgrade pause - sleep 60 - ceph orch upgrade resume - echo wait for half of osds to upgrade - 'while ! ceph osd versions | grep $sha1 | egrep '': [45678]''; do sleep 2 ; done' - ceph orch upgrade pause - ceph orch ps - ceph orch ps - ceph versions first-half-tasks: - workunit: branch: quincy clients: client.0: - cls/test_cls_rbd.sh - print: '**** done cls/test_cls_rbd.sh 5-workload' first_in_suite: false job_id: '7401835' kernel: kdb: true sha1: distro last_in_suite: false machine_type: smithi meta: - desc: 'run basic cls tests for rbd ' - desc: 'run randomized correctness test for rados operations generate write load with rados bench ' - desc: 'run basic cls tests for rbd ' - desc: 'run basic import/export cli tests for rbd ' - desc: 'librbd C and C++ api tests ' - desc: 'randomized correctness test for rados operations on a replicated pool, using only reads, writes, and deletes ' - desc: 'randomized correctness test for rados operations on a replicated pool with snapshot operations ' - desc: 'run randomized correctness test for rados operations generate write load with rados bench ' name: teuthology-2023-09-23_14:23:10-upgrade:quincy-x-reef-distro-default-smithi no_nested_subset: false nuke-on-error: true openstack: - volumes: count: 4 size: 10 os_type: rhel os_version: '8.6' overrides: admin_socket: branch: reef ceph: conf: global: mon election default strategy: 3 mgr: debug mgr: 20 debug ms: 1 mon: debug mon: 20 debug ms: 1 debug paxos: 20 osd: debug ms: 1 debug osd: 20 osd shutdown pgref assert: true create_rbd_pool: true flavor: default log-ignorelist: - \(MDS_ALL_DOWN\) - \(MDS_UP_LESS_THAN_MAX\) sha1: bb0daeff97f020f686596fca6eba20c310e26fb7 ceph-deploy: conf: client: log file: /var/log/ceph/ceph-$name.$pid.log mon: {} install: ceph: flavor: default sha1: bb0daeff97f020f686596fca6eba20c310e26fb7 selinux: whitelist: - scontext=system_u:system_r:logrotate_t:s0 workunit: branch: reef sha1: bb0daeff97f020f686596fca6eba20c310e26fb7 owner: scheduled_teuthology@teuthology priority: 100 repo: https://github.com/ceph/ceph.git roles: - - mon.a - mon.c - mgr.y - osd.0 - osd.1 - osd.2 - osd.3 - client.0 - node-exporter.a - alertmanager.a - - mon.b - mgr.x - osd.4 - osd.5 - osd.6 - osd.7 - client.1 - prometheus.a - grafana.a - node-exporter.b second-half-sequence: sequential: - cephadm.shell: env: - sha1 mon.a: - ceph orch upgrade resume - sleep 60 - echo wait for upgrade to complete - while ceph orch upgrade status | jq '.in_progress' | grep true ; do ceph orch ps ; ceph versions ; sleep 30 ; done - echo upgrade complete - ceph orch ps - ceph versions - ceph versions | jq -e '.overall | length == 1' - ceph versions | jq -e '.overall | keys' | grep $sha1 second-half-tasks: - full_sequential: - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - print: '**** done end radosbench.yaml' seed: 4997 sha1: bb0daeff97f020f686596fca6eba20c310e26fb7 sleep_before_teardown: 0 stress-tasks: - thrashosds: aggressive_pg_num_changes: false chance_force_recovery: 0 chance_pgnum_grow: 1 chance_pgpnum_fix: 1 chance_thrash_cluster_full: 0 chance_thrash_pg_upmap: 0 chance_thrash_pg_upmap_items: 0 disable_objectstore_tool_tests: true timeout: 1200 - full_sequential: - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - print: '**** done end radosbench.yaml' - workunit: branch: quincy clients: client.0: - cls/test_cls_rbd.sh - print: '**** done cls/test_cls_rbd.sh 5-workload' - workunit: branch: quincy clients: client.0: - rbd/import_export.sh env: RBD_CREATE_ARGS: --new-format - print: '**** done rbd/import_export.sh 5-workload' - workunit: branch: quincy clients: client.0: - rbd/test_librbd.sh env: RBD_FEATURES: '61' - print: '**** done rbd/test_librbd.sh 7-workload' - full_sequential: - rados: clients: - client.0 objects: 500 op_weights: delete: 10 read: 45 write: 45 ops: 4000 write_append_excl: false - print: '**** done rados/readwrite 5-workload' - full_sequential: - rados: clients: - client.0 objects: 50 op_weights: delete: 50 read: 100 rollback: 50 snap_create: 50 snap_remove: 50 write: 100 ops: 4000 write_append_excl: false - print: '**** done rados/snaps-few-objects 5-workload' suite: upgrade:quincy-x suite_branch: reef suite_path: /home/teuthworker/src/github.com_ceph_ceph_bb0daeff97f020f686596fca6eba20c310e26fb7/qa suite_relpath: qa suite_repo: https://github.com/ceph/ceph.git suite_sha1: bb0daeff97f020f686596fca6eba20c310e26fb7 targets: smithi026.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBLVvN3+LmzPZ4ixiwyCkS3KZLZ9R6MOPm+kPtaJbl8fExpzbk8TudXCcJWHBL7DGPWFvye2jT6sei9eiX0EUQYs= smithi070.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBLVvN3+LmzPZ4ixiwyCkS3KZLZ9R6MOPm+kPtaJbl8fExpzbk8TudXCcJWHBL7DGPWFvye2jT6sei9eiX0EUQYs= tasks: - pexec: all: - sudo cp /etc/containers/registries.conf /etc/containers/registries.conf.backup - sudo dnf -y module reset container-tools - sudo dnf -y module install container-tools:rhel8 --allowerasing --nobest - sudo cp /etc/containers/registries.conf.backup /etc/containers/registries.conf - install: branch: quincy exclude_packages: - ceph-volume - cephadm: cephadm_branch: quincy cephadm_git_url: https://github.com/ceph/ceph conf: osd: osd_class_default_list: '*' osd_class_load_list: '*' image: quay.ceph.io/ceph-ci/ceph:quincy - cephadm.shell: mon.a: - ceph fs volume create foo - ceph config set mon mon_warn_on_insecure_global_id_reclaim false --force - ceph config set mon mon_warn_on_insecure_global_id_reclaim_allowed false --force - ceph.healthy: null - print: '**** upgrading first half of cluster, with stress ****' - parallel: - first-half-tasks - first-half-sequence - print: '**** done upgrading first half of cluster ****' - ceph.healthy: null - print: '**** applying stress + thrashing to mixed-version cluster ****' - parallel: - stress-tasks - ceph.healthy: null - print: '**** finishing upgrade ****' - parallel: - second-half-tasks - second-half-sequence - ceph.healthy: null teuthology: fragments_dropped: [] meta: {} postmerge: [] teuthology_branch: main teuthology_sha1: 54e62bcbac4e53d9685e08328b790d3b20d71cae timestamp: 2023-09-23_14:23:10 tube: smithi user: teuthology verbose: true worker_log: /home/teuthworker/archive/worker_logs/dispatcher.smithi.2830 2023-09-23T20:21:36.205 INFO:teuthology.run:suite_path is set to /home/teuthworker/src/github.com_ceph_ceph_bb0daeff97f020f686596fca6eba20c310e26fb7/qa; will attempt to use it 2023-09-23T20:21:36.205 INFO:teuthology.run:Found tasks at /home/teuthworker/src/github.com_ceph_ceph_bb0daeff97f020f686596fca6eba20c310e26fb7/qa/tasks 2023-09-23T20:21:36.205 INFO:teuthology.run_tasks:Running task internal.check_packages... 2023-09-23T20:21:36.207 INFO:teuthology.task.internal:Checking packages... 2023-09-23T20:21:36.227 INFO:teuthology.task.internal:Checking packages for os_type 'rhel', flavor 'default' and ceph hash 'bb0daeff97f020f686596fca6eba20c310e26fb7' 2023-09-23T20:21:36.228 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2023-09-23T20:21:36.228 INFO:teuthology.packaging:ref: None 2023-09-23T20:21:36.228 INFO:teuthology.packaging:tag: None 2023-09-23T20:21:36.228 INFO:teuthology.packaging:branch: reef 2023-09-23T20:21:36.228 INFO:teuthology.packaging:sha1: bb0daeff97f020f686596fca6eba20c310e26fb7 2023-09-23T20:21:36.228 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F8%2Fx86_64&ref=reef 2023-09-23T20:21:36.458 INFO:teuthology.task.internal:Found packages for ceph version 18.2.0-460.gbb0daeff 2023-09-23T20:21:36.459 INFO:teuthology.run_tasks:Running task internal.buildpackages_prep... 2023-09-23T20:21:36.465 INFO:teuthology.task.internal:no buildpackages task found 2023-09-23T20:21:36.465 INFO:teuthology.run_tasks:Running task internal.save_config... 2023-09-23T20:21:36.470 INFO:teuthology.task.internal:Saving configuration 2023-09-23T20:21:36.491 INFO:teuthology.run_tasks:Running task internal.check_lock... 2023-09-23T20:21:36.498 INFO:teuthology.task.internal.check_lock:Checking locks... 2023-09-23T20:21:36.521 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi026.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2023-09-23_14:23:10-upgrade:quincy-x-reef-distro-default-smithi/7401835', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'rhel', 'os_version': '8.6', 'arch': 'x86_64', 'locked': True, 'locked_since': '2023-09-23 20:17:05.031747', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBLVvN3+LmzPZ4ixiwyCkS3KZLZ9R6MOPm+kPtaJbl8fExpzbk8TudXCcJWHBL7DGPWFvye2jT6sei9eiX0EUQYs='} 2023-09-23T20:21:36.541 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi070.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2023-09-23_14:23:10-upgrade:quincy-x-reef-distro-default-smithi/7401835', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'rhel', 'os_version': '8.6', 'arch': 'x86_64', 'locked': True, 'locked_since': '2023-09-23 20:17:05.034040', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBLVvN3+LmzPZ4ixiwyCkS3KZLZ9R6MOPm+kPtaJbl8fExpzbk8TudXCcJWHBL7DGPWFvye2jT6sei9eiX0EUQYs='} 2023-09-23T20:21:36.542 INFO:teuthology.run_tasks:Running task internal.add_remotes... 2023-09-23T20:21:36.548 INFO:teuthology.task.internal:roles: ubuntu@smithi026.front.sepia.ceph.com - ['mon.a', 'mon.c', 'mgr.y', 'osd.0', 'osd.1', 'osd.2', 'osd.3', 'client.0', 'node-exporter.a', 'alertmanager.a'] 2023-09-23T20:21:36.548 INFO:teuthology.task.internal:roles: ubuntu@smithi070.front.sepia.ceph.com - ['mon.b', 'mgr.x', 'osd.4', 'osd.5', 'osd.6', 'osd.7', 'client.1', 'prometheus.a', 'grafana.a', 'node-exporter.b'] 2023-09-23T20:21:36.548 INFO:teuthology.run_tasks:Running task console_log... 2023-09-23T20:21:36.621 DEBUG:teuthology.exit:Installing handler: Handler(exiter=, func=.kill_console_loggers at 0x7f78b15d8af0>, signals=[15]) 2023-09-23T20:21:36.622 INFO:teuthology.run_tasks:Running task internal.connect... 2023-09-23T20:21:36.627 INFO:teuthology.task.internal:Opening connections... 2023-09-23T20:21:36.627 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi026.front.sepia.ceph.com 2023-09-23T20:21:36.629 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi026.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2023-09-23T20:21:36.696 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi070.front.sepia.ceph.com 2023-09-23T20:21:36.697 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi070.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2023-09-23T20:21:36.762 INFO:teuthology.run_tasks:Running task internal.push_inventory... 2023-09-23T20:21:36.769 DEBUG:teuthology.orchestra.run.smithi026:> uname -m 2023-09-23T20:21:36.790 INFO:teuthology.orchestra.run.smithi026.stdout:x86_64 2023-09-23T20:21:36.790 DEBUG:teuthology.orchestra.run.smithi026:> cat /etc/os-release 2023-09-23T20:21:36.849 INFO:teuthology.orchestra.run.smithi026.stdout:NAME="Red Hat Enterprise Linux" 2023-09-23T20:21:36.850 INFO:teuthology.orchestra.run.smithi026.stdout:VERSION="8.6 (Ootpa)" 2023-09-23T20:21:36.850 INFO:teuthology.orchestra.run.smithi026.stdout:ID="rhel" 2023-09-23T20:21:36.850 INFO:teuthology.orchestra.run.smithi026.stdout:ID_LIKE="fedora" 2023-09-23T20:21:36.850 INFO:teuthology.orchestra.run.smithi026.stdout:VERSION_ID="8.6" 2023-09-23T20:21:36.850 INFO:teuthology.orchestra.run.smithi026.stdout:PLATFORM_ID="platform:el8" 2023-09-23T20:21:36.850 INFO:teuthology.orchestra.run.smithi026.stdout:PRETTY_NAME="Red Hat Enterprise Linux 8.6 (Ootpa)" 2023-09-23T20:21:36.850 INFO:teuthology.orchestra.run.smithi026.stdout:ANSI_COLOR="0;31" 2023-09-23T20:21:36.850 INFO:teuthology.orchestra.run.smithi026.stdout:CPE_NAME="cpe:/o:redhat:enterprise_linux:8::baseos" 2023-09-23T20:21:36.850 INFO:teuthology.orchestra.run.smithi026.stdout:HOME_URL="https://www.redhat.com/" 2023-09-23T20:21:36.850 INFO:teuthology.orchestra.run.smithi026.stdout:DOCUMENTATION_URL="https://access.redhat.com/documentation/red_hat_enterprise_linux/8/" 2023-09-23T20:21:36.851 INFO:teuthology.orchestra.run.smithi026.stdout:BUG_REPORT_URL="https://bugzilla.redhat.com/" 2023-09-23T20:21:36.851 INFO:teuthology.orchestra.run.smithi026.stdout: 2023-09-23T20:21:36.851 INFO:teuthology.orchestra.run.smithi026.stdout:REDHAT_BUGZILLA_PRODUCT="Red Hat Enterprise Linux 8" 2023-09-23T20:21:36.851 INFO:teuthology.orchestra.run.smithi026.stdout:REDHAT_BUGZILLA_PRODUCT_VERSION=8.6 2023-09-23T20:21:36.851 INFO:teuthology.orchestra.run.smithi026.stdout:REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux" 2023-09-23T20:21:36.851 INFO:teuthology.orchestra.run.smithi026.stdout:REDHAT_SUPPORT_PRODUCT_VERSION="8.6" 2023-09-23T20:21:36.852 INFO:teuthology.lock.ops:Updating smithi026.front.sepia.ceph.com on lock server 2023-09-23T20:21:36.873 DEBUG:teuthology.orchestra.run.smithi070:> uname -m 2023-09-23T20:21:36.890 INFO:teuthology.orchestra.run.smithi070.stdout:x86_64 2023-09-23T20:21:36.890 DEBUG:teuthology.orchestra.run.smithi070:> cat /etc/os-release 2023-09-23T20:21:36.945 INFO:teuthology.orchestra.run.smithi070.stdout:NAME="Red Hat Enterprise Linux" 2023-09-23T20:21:36.946 INFO:teuthology.orchestra.run.smithi070.stdout:VERSION="8.6 (Ootpa)" 2023-09-23T20:21:36.946 INFO:teuthology.orchestra.run.smithi070.stdout:ID="rhel" 2023-09-23T20:21:36.946 INFO:teuthology.orchestra.run.smithi070.stdout:ID_LIKE="fedora" 2023-09-23T20:21:36.946 INFO:teuthology.orchestra.run.smithi070.stdout:VERSION_ID="8.6" 2023-09-23T20:21:36.946 INFO:teuthology.orchestra.run.smithi070.stdout:PLATFORM_ID="platform:el8" 2023-09-23T20:21:36.946 INFO:teuthology.orchestra.run.smithi070.stdout:PRETTY_NAME="Red Hat Enterprise Linux 8.6 (Ootpa)" 2023-09-23T20:21:36.946 INFO:teuthology.orchestra.run.smithi070.stdout:ANSI_COLOR="0;31" 2023-09-23T20:21:36.946 INFO:teuthology.orchestra.run.smithi070.stdout:CPE_NAME="cpe:/o:redhat:enterprise_linux:8::baseos" 2023-09-23T20:21:36.946 INFO:teuthology.orchestra.run.smithi070.stdout:HOME_URL="https://www.redhat.com/" 2023-09-23T20:21:36.946 INFO:teuthology.orchestra.run.smithi070.stdout:DOCUMENTATION_URL="https://access.redhat.com/documentation/red_hat_enterprise_linux/8/" 2023-09-23T20:21:36.947 INFO:teuthology.orchestra.run.smithi070.stdout:BUG_REPORT_URL="https://bugzilla.redhat.com/" 2023-09-23T20:21:36.947 INFO:teuthology.orchestra.run.smithi070.stdout: 2023-09-23T20:21:36.947 INFO:teuthology.orchestra.run.smithi070.stdout:REDHAT_BUGZILLA_PRODUCT="Red Hat Enterprise Linux 8" 2023-09-23T20:21:36.947 INFO:teuthology.orchestra.run.smithi070.stdout:REDHAT_BUGZILLA_PRODUCT_VERSION=8.6 2023-09-23T20:21:36.947 INFO:teuthology.orchestra.run.smithi070.stdout:REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux" 2023-09-23T20:21:36.947 INFO:teuthology.orchestra.run.smithi070.stdout:REDHAT_SUPPORT_PRODUCT_VERSION="8.6" 2023-09-23T20:21:36.947 INFO:teuthology.lock.ops:Updating smithi070.front.sepia.ceph.com on lock server 2023-09-23T20:21:36.973 INFO:teuthology.run_tasks:Running task internal.serialize_remote_roles... 2023-09-23T20:21:36.982 INFO:teuthology.run_tasks:Running task internal.check_conflict... 2023-09-23T20:21:36.988 INFO:teuthology.task.internal:Checking for old test directory... 2023-09-23T20:21:36.988 DEBUG:teuthology.orchestra.run.smithi026:> test '!' -e /home/ubuntu/cephtest 2023-09-23T20:21:36.990 DEBUG:teuthology.orchestra.run.smithi070:> test '!' -e /home/ubuntu/cephtest 2023-09-23T20:21:37.007 INFO:teuthology.run_tasks:Running task internal.check_ceph_data... 2023-09-23T20:21:37.013 INFO:teuthology.task.internal:Checking for non-empty /var/lib/ceph... 2023-09-23T20:21:37.014 DEBUG:teuthology.orchestra.run.smithi026:> test -z $(ls -A /var/lib/ceph) 2023-09-23T20:21:37.050 DEBUG:teuthology.orchestra.run.smithi070:> test -z $(ls -A /var/lib/ceph) 2023-09-23T20:21:37.073 INFO:teuthology.run_tasks:Running task internal.vm_setup... 2023-09-23T20:21:37.131 INFO:teuthology.run_tasks:Running task kernel... 2023-09-23T20:21:37.144 INFO:teuthology.task.kernel:normalize config orig: {'kdb': True, 'sha1': 'distro'} 2023-09-23T20:21:37.145 INFO:teuthology.task.kernel:config {'mon.a': {'kdb': True, 'sha1': 'distro'}, 'mon.b': {'kdb': True, 'sha1': 'distro'}}, timeout 300 2023-09-23T20:21:37.145 DEBUG:teuthology.orchestra.run.smithi026:> test -f /run/.containerenv -o -f /.dockerenv 2023-09-23T20:21:37.145 DEBUG:teuthology.orchestra.run.smithi070:> test -f /run/.containerenv -o -f /.dockerenv 2023-09-23T20:21:37.162 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:21:37.163 DEBUG:teuthology.orchestra.run.smithi070:> uname -r 2023-09-23T20:21:37.164 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:21:37.164 DEBUG:teuthology.orchestra.run.smithi026:> uname -r 2023-09-23T20:21:37.219 INFO:teuthology.orchestra.run.smithi070.stdout:4.18.0-372.9.1.el8.x86_64 2023-09-23T20:21:37.220 INFO:teuthology.task.kernel:Running kernel on smithi070: 4.18.0-372.9.1.el8.x86_64 2023-09-23T20:21:37.220 DEBUG:teuthology.orchestra.run.smithi070:> sudo yum install -y kernel 2023-09-23T20:21:37.224 INFO:teuthology.orchestra.run.smithi026.stdout:4.18.0-372.9.1.el8.x86_64 2023-09-23T20:21:37.224 INFO:teuthology.task.kernel:Running kernel on smithi026: 4.18.0-372.9.1.el8.x86_64 2023-09-23T20:21:37.224 DEBUG:teuthology.orchestra.run.smithi026:> sudo yum install -y kernel 2023-09-23T20:21:38.457 INFO:teuthology.orchestra.run.smithi026.stdout:Updating Subscription Management repositories. 2023-09-23T20:21:38.458 INFO:teuthology.orchestra.run.smithi026.stdout:Unable to read consumer identity 2023-09-23T20:21:38.520 INFO:teuthology.orchestra.run.smithi070.stdout:Updating Subscription Management repositories. 2023-09-23T20:21:38.520 INFO:teuthology.orchestra.run.smithi070.stdout:Unable to read consumer identity 2023-09-23T20:21:39.984 INFO:teuthology.orchestra.run.smithi026.stdout:Copr repo for python3-asyncssh owned by ceph 15 kB/s | 3.5 kB 00:00 2023-09-23T20:21:40.019 INFO:teuthology.orchestra.run.smithi070.stdout:Copr repo for python3-asyncssh owned by ceph 15 kB/s | 3.5 kB 00:00 2023-09-23T20:21:41.001 INFO:teuthology.orchestra.run.smithi026.stdout:Extra Packages for Enterprise Linux 16 MB/s | 16 MB 00:00 2023-09-23T20:21:41.345 INFO:teuthology.orchestra.run.smithi070.stdout:Extra Packages for Enterprise Linux 12 MB/s | 16 MB 00:01 2023-09-23T20:21:44.734 INFO:teuthology.orchestra.run.smithi026.stdout:lab-extras 283 kB/s | 24 kB 00:00 2023-09-23T20:21:44.837 INFO:teuthology.orchestra.run.smithi070.stdout:lab-extras 413 kB/s | 24 kB 00:00 2023-09-23T20:21:45.497 INFO:teuthology.orchestra.run.smithi026.stdout:Package kernel-4.18.0-372.9.1.el8.x86_64 is already installed. 2023-09-23T20:21:45.506 INFO:teuthology.orchestra.run.smithi070.stdout:Package kernel-4.18.0-372.9.1.el8.x86_64 is already installed. 2023-09-23T20:21:45.555 INFO:teuthology.orchestra.run.smithi070.stdout:Dependencies resolved. 2023-09-23T20:21:45.556 INFO:teuthology.orchestra.run.smithi070.stdout:Nothing to do. 2023-09-23T20:21:45.556 INFO:teuthology.orchestra.run.smithi070.stdout:Complete! 2023-09-23T20:21:45.561 INFO:teuthology.orchestra.run.smithi026.stdout:Dependencies resolved. 2023-09-23T20:21:45.562 INFO:teuthology.orchestra.run.smithi026.stdout:Nothing to do. 2023-09-23T20:21:45.563 INFO:teuthology.orchestra.run.smithi026.stdout:Complete! 2023-09-23T20:21:45.678 DEBUG:teuthology.orchestra.run.smithi026:> echo no | sudo yum reinstall kernel || true 2023-09-23T20:21:45.680 DEBUG:teuthology.orchestra.run.smithi070:> echo no | sudo yum reinstall kernel || true 2023-09-23T20:21:46.030 INFO:teuthology.orchestra.run.smithi026.stdout:Updating Subscription Management repositories. 2023-09-23T20:21:46.031 INFO:teuthology.orchestra.run.smithi026.stdout:Unable to read consumer identity 2023-09-23T20:21:46.051 INFO:teuthology.orchestra.run.smithi070.stdout:Updating Subscription Management repositories. 2023-09-23T20:21:46.051 INFO:teuthology.orchestra.run.smithi070.stdout:Unable to read consumer identity 2023-09-23T20:21:46.314 INFO:teuthology.orchestra.run.smithi026.stdout:Last metadata expiration check: 0:00:02 ago on Sat 23 Sep 2023 08:21:44 PM UTC. 2023-09-23T20:21:46.318 INFO:teuthology.orchestra.run.smithi070.stdout:Last metadata expiration check: 0:00:02 ago on Sat 23 Sep 2023 08:21:44 PM UTC. 2023-09-23T20:21:46.359 INFO:teuthology.orchestra.run.smithi026.stdout:Installed package kernel-4.18.0-372.9.1.el8.x86_64 (from anaconda) not available. 2023-09-23T20:21:46.361 INFO:teuthology.orchestra.run.smithi070.stdout:Installed package kernel-4.18.0-372.9.1.el8.x86_64 (from anaconda) not available. 2023-09-23T20:21:46.397 INFO:teuthology.orchestra.run.smithi070.stderr:Error: No packages marked for reinstall. 2023-09-23T20:21:46.400 INFO:teuthology.orchestra.run.smithi026.stderr:Error: No packages marked for reinstall. 2023-09-23T20:21:46.442 DEBUG:teuthology.orchestra.run.smithi070:> sudo yum reinstall -y kernel || true 2023-09-23T20:21:46.447 DEBUG:teuthology.orchestra.run.smithi026:> sudo yum reinstall -y kernel || true 2023-09-23T20:21:46.770 INFO:teuthology.orchestra.run.smithi070.stdout:Updating Subscription Management repositories. 2023-09-23T20:21:46.770 INFO:teuthology.orchestra.run.smithi070.stdout:Unable to read consumer identity 2023-09-23T20:21:46.797 INFO:teuthology.orchestra.run.smithi026.stdout:Updating Subscription Management repositories. 2023-09-23T20:21:46.797 INFO:teuthology.orchestra.run.smithi026.stdout:Unable to read consumer identity 2023-09-23T20:21:47.030 INFO:teuthology.orchestra.run.smithi070.stdout:Last metadata expiration check: 0:00:02 ago on Sat 23 Sep 2023 08:21:44 PM UTC. 2023-09-23T20:21:47.070 INFO:teuthology.orchestra.run.smithi070.stdout:Installed package kernel-4.18.0-372.9.1.el8.x86_64 (from anaconda) not available. 2023-09-23T20:21:47.074 INFO:teuthology.orchestra.run.smithi026.stdout:Last metadata expiration check: 0:00:03 ago on Sat 23 Sep 2023 08:21:44 PM UTC. 2023-09-23T20:21:47.104 INFO:teuthology.orchestra.run.smithi070.stderr:Error: No packages marked for reinstall. 2023-09-23T20:21:47.119 INFO:teuthology.orchestra.run.smithi026.stdout:Installed package kernel-4.18.0-372.9.1.el8.x86_64 (from anaconda) not available. 2023-09-23T20:21:47.154 DEBUG:teuthology.orchestra.run.smithi070:> rpm -q kernel | sort -rV | head -n 1 2023-09-23T20:21:47.169 INFO:teuthology.orchestra.run.smithi026.stderr:Error: No packages marked for reinstall. 2023-09-23T20:21:47.221 DEBUG:teuthology.orchestra.run.smithi026:> rpm -q kernel | sort -rV | head -n 1 2023-09-23T20:21:47.233 INFO:teuthology.orchestra.run.smithi070.stdout:kernel-4.18.0-372.9.1.el8.x86_64 2023-09-23T20:21:47.234 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 4.18.0-372.9.1.el8.x86_64 2023-09-23T20:21:47.234 INFO:teuthology.task.kernel:Newest distro kernel installed and running 2023-09-23T20:21:47.234 INFO:teuthology.task.kernel:Enabling kdb on mon.b... 2023-09-23T20:21:47.234 DEBUG:teuthology.orchestra.run.smithi070:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2023-09-23T20:21:47.261 INFO:teuthology.orchestra.run.smithi070.stdout:ttyS1 2023-09-23T20:21:47.283 DEBUG:teuthology.parallel:result is None 2023-09-23T20:21:47.304 INFO:teuthology.orchestra.run.smithi026.stdout:kernel-4.18.0-372.9.1.el8.x86_64 2023-09-23T20:21:47.304 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 4.18.0-372.9.1.el8.x86_64 2023-09-23T20:21:47.304 INFO:teuthology.task.kernel:Newest distro kernel installed and running 2023-09-23T20:21:47.305 INFO:teuthology.task.kernel:Enabling kdb on mon.a... 2023-09-23T20:21:47.305 DEBUG:teuthology.orchestra.run.smithi026:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2023-09-23T20:21:47.331 INFO:teuthology.orchestra.run.smithi026.stdout:ttyS1 2023-09-23T20:21:47.343 DEBUG:teuthology.parallel:result is None 2023-09-23T20:21:47.343 INFO:teuthology.run_tasks:Running task internal.base... 2023-09-23T20:21:47.350 INFO:teuthology.task.internal:Creating test directory... 2023-09-23T20:21:47.350 DEBUG:teuthology.orchestra.run.smithi026:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2023-09-23T20:21:47.385 DEBUG:teuthology.orchestra.run.smithi070:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2023-09-23T20:21:47.402 INFO:teuthology.run_tasks:Running task internal.archive_upload... 2023-09-23T20:21:47.409 INFO:teuthology.run_tasks:Running task internal.archive... 2023-09-23T20:21:47.415 INFO:teuthology.task.internal:Creating archive directory... 2023-09-23T20:21:47.415 DEBUG:teuthology.orchestra.run.smithi026:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2023-09-23T20:21:47.442 DEBUG:teuthology.orchestra.run.smithi070:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2023-09-23T20:21:47.492 INFO:teuthology.run_tasks:Running task internal.coredump... 2023-09-23T20:21:47.499 INFO:teuthology.task.internal:Enabling coredump saving... 2023-09-23T20:21:47.499 DEBUG:teuthology.orchestra.run.smithi026:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2023-09-23T20:21:47.501 DEBUG:teuthology.orchestra.run.smithi070:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2023-09-23T20:21:47.535 INFO:teuthology.orchestra.run.smithi026.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2023-09-23T20:21:47.539 INFO:teuthology.orchestra.run.smithi070.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2023-09-23T20:21:47.546 INFO:teuthology.orchestra.run.smithi026.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2023-09-23T20:21:47.549 INFO:teuthology.orchestra.run.smithi070.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2023-09-23T20:21:47.550 INFO:teuthology.run_tasks:Running task internal.sudo... 2023-09-23T20:21:47.557 INFO:teuthology.task.internal:Configuring sudo... 2023-09-23T20:21:47.557 DEBUG:teuthology.orchestra.run.smithi026:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2023-09-23T20:21:47.590 DEBUG:teuthology.orchestra.run.smithi070:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2023-09-23T20:21:47.617 INFO:teuthology.run_tasks:Running task internal.syslog... 2023-09-23T20:21:47.625 INFO:teuthology.task.internal.syslog:Starting syslog monitoring... 2023-09-23T20:21:47.625 DEBUG:teuthology.orchestra.run.smithi026:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2023-09-23T20:21:47.657 DEBUG:teuthology.orchestra.run.smithi070:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2023-09-23T20:21:47.673 DEBUG:teuthology.orchestra.run.smithi026:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2023-09-23T20:21:47.756 DEBUG:teuthology.orchestra.run.smithi026:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/kern.log 2023-09-23T20:21:47.803 DEBUG:teuthology.orchestra.run.smithi026:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2023-09-23T20:21:47.884 DEBUG:teuthology.orchestra.run.smithi026:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/misc.log 2023-09-23T20:21:47.929 DEBUG:teuthology.orchestra.run.smithi026:> set -ex 2023-09-23T20:21:47.930 DEBUG:teuthology.orchestra.run.smithi026:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2023-09-23T20:21:47.997 DEBUG:teuthology.orchestra.run.smithi070:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2023-09-23T20:21:48.040 DEBUG:teuthology.orchestra.run.smithi070:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/kern.log 2023-09-23T20:21:48.088 DEBUG:teuthology.orchestra.run.smithi070:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2023-09-23T20:21:48.173 DEBUG:teuthology.orchestra.run.smithi070:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/misc.log 2023-09-23T20:21:48.213 DEBUG:teuthology.orchestra.run.smithi070:> set -ex 2023-09-23T20:21:48.213 DEBUG:teuthology.orchestra.run.smithi070:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2023-09-23T20:21:48.292 DEBUG:teuthology.orchestra.run.smithi026:> sudo service rsyslog restart 2023-09-23T20:21:48.294 DEBUG:teuthology.orchestra.run.smithi070:> sudo service rsyslog restart 2023-09-23T20:21:48.346 INFO:teuthology.orchestra.run.smithi026.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2023-09-23T20:21:48.387 INFO:teuthology.orchestra.run.smithi070.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2023-09-23T20:21:48.727 INFO:teuthology.run_tasks:Running task internal.timer... 2023-09-23T20:21:48.734 INFO:teuthology.task.internal:Starting timer... 2023-09-23T20:21:48.735 INFO:teuthology.run_tasks:Running task pcp... 2023-09-23T20:21:48.743 INFO:teuthology.run_tasks:Running task selinux... 2023-09-23T20:21:48.751 DEBUG:teuthology.task:Applying overrides for task selinux: {'whitelist': ['scontext=system_u:system_r:logrotate_t:s0']} 2023-09-23T20:21:48.751 DEBUG:teuthology.orchestra.run.smithi026:> sudo service auditd rotate 2023-09-23T20:21:48.803 INFO:teuthology.orchestra.run.smithi026.stdout:Rotating logs: 2023-09-23T20:21:48.805 DEBUG:teuthology.orchestra.run.smithi070:> sudo service auditd rotate 2023-09-23T20:21:48.859 INFO:teuthology.orchestra.run.smithi070.stdout:Rotating logs: 2023-09-23T20:21:48.861 DEBUG:teuthology.task.selinux:Getting current SELinux state 2023-09-23T20:21:48.861 DEBUG:teuthology.orchestra.run.smithi026:> /usr/sbin/getenforce 2023-09-23T20:21:48.881 INFO:teuthology.orchestra.run.smithi026.stdout:Permissive 2023-09-23T20:21:48.881 DEBUG:teuthology.orchestra.run.smithi070:> /usr/sbin/getenforce 2023-09-23T20:21:48.902 INFO:teuthology.orchestra.run.smithi070.stdout:Permissive 2023-09-23T20:21:48.903 DEBUG:teuthology.task.selinux:Existing SELinux modes: {'ubuntu@smithi026.front.sepia.ceph.com': 'permissive', 'ubuntu@smithi070.front.sepia.ceph.com': 'permissive'} 2023-09-23T20:21:48.903 DEBUG:teuthology.orchestra.run.smithi026:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 2023-09-23T20:21:48.950 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:21:48.951 DEBUG:teuthology.orchestra.run.smithi070:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 2023-09-23T20:21:48.976 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:21:48.977 INFO:teuthology.task.selinux:Putting SELinux into permissive mode 2023-09-23T20:21:48.977 DEBUG:teuthology.orchestra.run.smithi026:> sudo /usr/sbin/setenforce permissive 2023-09-23T20:21:49.138 DEBUG:teuthology.orchestra.run.smithi070:> sudo /usr/sbin/setenforce permissive 2023-09-23T20:21:49.165 INFO:teuthology.run_tasks:Running task ansible.cephlab... 2023-09-23T20:21:49.175 DEBUG:teuthology.repo_utils:Setting repo remote to https://git.ceph.com/ceph-cm-ansible.git 2023-09-23T20:21:49.179 INFO:teuthology.repo_utils:Fetching main from origin 2023-09-23T20:21:49.317 INFO:teuthology.repo_utils:Resetting repo at /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main to origin/main 2023-09-23T20:21:49.331 INFO:teuthology.task.ansible:Playbook: [{'import_playbook': 'ansible_managed.yml'}, {'import_playbook': 'teuthology.yml'}, {'hosts': 'testnodes', 'tasks': [{'set_fact': {'ran_from_cephlab_playbook': True}}]}, {'import_playbook': 'testnodes.yml'}, {'import_playbook': 'container-host.yml'}, {'import_playbook': 'cobbler.yml'}, {'import_playbook': 'paddles.yml'}, {'import_playbook': 'pulpito.yml'}, {'hosts': 'testnodes', 'become': True, 'tasks': [{'name': 'Touch /ceph-qa-ready', 'file': {'path': '/ceph-qa-ready', 'state': 'touch'}, 'when': 'ran_from_cephlab_playbook|bool'}]}] 2023-09-23T20:21:49.332 DEBUG:teuthology.task.ansible:Running ansible-playbook -v --extra-vars '{"ansible_ssh_user": "ubuntu"}' -i /etc/ansible/hosts --limit smithi026.front.sepia.ceph.com,smithi070.front.sepia.ceph.com /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main/cephlab.yml 2023-09-23T20:30:39.041 DEBUG:teuthology.task.ansible:Reconnecting to [Remote(name='ubuntu@smithi026.front.sepia.ceph.com'), Remote(name='ubuntu@smithi070.front.sepia.ceph.com')] 2023-09-23T20:30:39.045 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi026.front.sepia.ceph.com' 2023-09-23T20:30:39.046 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi026.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2023-09-23T20:30:39.114 DEBUG:teuthology.orchestra.run.smithi026:> true 2023-09-23T20:30:39.184 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi026.front.sepia.ceph.com' 2023-09-23T20:30:39.185 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi070.front.sepia.ceph.com' 2023-09-23T20:30:39.185 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi070.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2023-09-23T20:30:39.253 DEBUG:teuthology.orchestra.run.smithi070:> true 2023-09-23T20:30:39.320 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi070.front.sepia.ceph.com' 2023-09-23T20:30:39.320 INFO:teuthology.run_tasks:Running task clock... 2023-09-23T20:30:39.330 INFO:teuthology.task.clock:Syncing clocks and checking initial clock skew... 2023-09-23T20:30:39.331 INFO:teuthology.orchestra.run:Running command with timeout 360 2023-09-23T20:30:39.331 DEBUG:teuthology.orchestra.run.smithi026:> sudo systemctl stop ntp.service || sudo systemctl stop ntpd.service || sudo systemctl stop chronyd.service ; sudo ntpd -gq || sudo chronyc makestep ; sudo systemctl start ntp.service || sudo systemctl start ntpd.service || sudo systemctl start chronyd.service ; PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2023-09-23T20:30:39.333 INFO:teuthology.orchestra.run:Running command with timeout 360 2023-09-23T20:30:39.333 DEBUG:teuthology.orchestra.run.smithi070:> sudo systemctl stop ntp.service || sudo systemctl stop ntpd.service || sudo systemctl stop chronyd.service ; sudo ntpd -gq || sudo chronyc makestep ; sudo systemctl start ntp.service || sudo systemctl start ntpd.service || sudo systemctl start chronyd.service ; PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2023-09-23T20:30:39.364 INFO:teuthology.orchestra.run.smithi026.stderr:Failed to stop ntp.service: Unit ntp.service not loaded. 2023-09-23T20:30:39.380 INFO:teuthology.orchestra.run.smithi026.stderr:Failed to stop ntpd.service: Unit ntpd.service not loaded. 2023-09-23T20:30:39.391 INFO:teuthology.orchestra.run.smithi070.stderr:Failed to stop ntp.service: Unit ntp.service not loaded. 2023-09-23T20:30:39.406 INFO:teuthology.orchestra.run.smithi070.stderr:Failed to stop ntpd.service: Unit ntpd.service not loaded. 2023-09-23T20:30:39.444 INFO:teuthology.orchestra.run.smithi026.stderr:sudo: ntpd: command not found 2023-09-23T20:30:39.456 INFO:teuthology.orchestra.run.smithi026.stdout:506 Cannot talk to daemon 2023-09-23T20:30:39.471 INFO:teuthology.orchestra.run.smithi026.stderr:Failed to start ntp.service: Unit ntp.service not found. 2023-09-23T20:30:39.475 INFO:teuthology.orchestra.run.smithi070.stderr:sudo: ntpd: command not found 2023-09-23T20:30:39.486 INFO:teuthology.orchestra.run.smithi026.stderr:Failed to start ntpd.service: Unit ntpd.service not found. 2023-09-23T20:30:39.486 INFO:teuthology.orchestra.run.smithi070.stdout:506 Cannot talk to daemon 2023-09-23T20:30:39.500 INFO:teuthology.orchestra.run.smithi070.stderr:Failed to start ntp.service: Unit ntp.service not found. 2023-09-23T20:30:39.514 INFO:teuthology.orchestra.run.smithi070.stderr:Failed to start ntpd.service: Unit ntpd.service not found. 2023-09-23T20:30:39.557 INFO:teuthology.orchestra.run.smithi026.stderr:bash: ntpq: command not found 2023-09-23T20:30:39.561 INFO:teuthology.orchestra.run.smithi026.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2023-09-23T20:30:39.561 INFO:teuthology.orchestra.run.smithi026.stdout:=============================================================================== 2023-09-23T20:30:39.561 INFO:teuthology.orchestra.run.smithi026.stdout:^? hv01.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2023-09-23T20:30:39.561 INFO:teuthology.orchestra.run.smithi026.stdout:^? hv02.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2023-09-23T20:30:39.561 INFO:teuthology.orchestra.run.smithi026.stdout:^? hv03.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2023-09-23T20:30:39.561 INFO:teuthology.orchestra.run.smithi026.stdout:^? hv04.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2023-09-23T20:30:39.583 INFO:teuthology.orchestra.run.smithi070.stderr:bash: ntpq: command not found 2023-09-23T20:30:39.587 INFO:teuthology.orchestra.run.smithi070.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2023-09-23T20:30:39.587 INFO:teuthology.orchestra.run.smithi070.stdout:=============================================================================== 2023-09-23T20:30:39.587 INFO:teuthology.orchestra.run.smithi070.stdout:^? hv01.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2023-09-23T20:30:39.587 INFO:teuthology.orchestra.run.smithi070.stdout:^? hv02.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2023-09-23T20:30:39.587 INFO:teuthology.orchestra.run.smithi070.stdout:^? hv03.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2023-09-23T20:30:39.587 INFO:teuthology.orchestra.run.smithi070.stdout:^? hv04.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2023-09-23T20:30:39.588 INFO:teuthology.run_tasks:Running task pexec... 2023-09-23T20:30:39.596 INFO:teuthology.task.pexec:Executing custom commands... 2023-09-23T20:30:39.597 INFO:teuthology.task.pexec:Running commands on host ubuntu@smithi026.front.sepia.ceph.com 2023-09-23T20:30:39.597 DEBUG:teuthology.orchestra.run.smithi026:> TESTDIR=/home/ubuntu/cephtest bash -s 2023-09-23T20:30:39.597 INFO:teuthology.task.pexec:Running commands on host ubuntu@smithi070.front.sepia.ceph.com 2023-09-23T20:30:39.597 DEBUG:teuthology.orchestra.run.smithi070:> TESTDIR=/home/ubuntu/cephtest bash -s 2023-09-23T20:30:39.995 INFO:teuthology.orchestra.run.smithi026.stdout:Updating Subscription Management repositories. 2023-09-23T20:30:40.000 INFO:teuthology.orchestra.run.smithi070.stdout:Updating Subscription Management repositories. 2023-09-23T20:30:40.896 INFO:teuthology.orchestra.run.smithi070.stdout:Last metadata expiration check: 0:01:38 ago on Sat 23 Sep 2023 08:29:02 PM UTC. 2023-09-23T20:30:40.941 INFO:teuthology.orchestra.run.smithi026.stdout:Last metadata expiration check: 0:01:29 ago on Sat 23 Sep 2023 08:29:11 PM UTC. 2023-09-23T20:30:42.607 INFO:teuthology.orchestra.run.smithi070.stdout:Dependencies resolved. 2023-09-23T20:30:42.608 INFO:teuthology.orchestra.run.smithi070.stdout:================================================================================ 2023-09-23T20:30:42.608 INFO:teuthology.orchestra.run.smithi070.stdout: Package Architecture Version Repository Size 2023-09-23T20:30:42.608 INFO:teuthology.orchestra.run.smithi070.stdout:================================================================================ 2023-09-23T20:30:42.608 INFO:teuthology.orchestra.run.smithi070.stdout:Resetting modules: 2023-09-23T20:30:42.608 INFO:teuthology.orchestra.run.smithi070.stdout: container-tools 2023-09-23T20:30:42.608 INFO:teuthology.orchestra.run.smithi070.stdout: 2023-09-23T20:30:42.608 INFO:teuthology.orchestra.run.smithi070.stdout:Transaction Summary 2023-09-23T20:30:42.608 INFO:teuthology.orchestra.run.smithi070.stdout:================================================================================ 2023-09-23T20:30:42.608 INFO:teuthology.orchestra.run.smithi070.stdout: 2023-09-23T20:30:42.915 INFO:teuthology.orchestra.run.smithi026.stdout:Dependencies resolved. 2023-09-23T20:30:42.916 INFO:teuthology.orchestra.run.smithi026.stdout:================================================================================ 2023-09-23T20:30:42.916 INFO:teuthology.orchestra.run.smithi026.stdout: Package Architecture Version Repository Size 2023-09-23T20:30:42.916 INFO:teuthology.orchestra.run.smithi026.stdout:================================================================================ 2023-09-23T20:30:42.916 INFO:teuthology.orchestra.run.smithi026.stdout:Resetting modules: 2023-09-23T20:30:42.916 INFO:teuthology.orchestra.run.smithi026.stdout: container-tools 2023-09-23T20:30:42.916 INFO:teuthology.orchestra.run.smithi026.stdout: 2023-09-23T20:30:42.916 INFO:teuthology.orchestra.run.smithi026.stdout:Transaction Summary 2023-09-23T20:30:42.916 INFO:teuthology.orchestra.run.smithi026.stdout:================================================================================ 2023-09-23T20:30:42.916 INFO:teuthology.orchestra.run.smithi026.stdout: 2023-09-23T20:30:43.558 INFO:teuthology.orchestra.run.smithi070.stdout:Last metadata expiration check: 0:01:41 ago on Sat 23 Sep 2023 08:29:02 PM UTC. 2023-09-23T20:30:43.938 INFO:teuthology.orchestra.run.smithi026.stdout:Last metadata expiration check: 0:01:32 ago on Sat 23 Sep 2023 08:29:11 PM UTC. 2023-09-23T20:30:45.887 INFO:teuthology.orchestra.run.smithi070.stdout:Complete! 2023-09-23T20:30:46.332 INFO:teuthology.orchestra.run.smithi070.stdout:Updating Subscription Management repositories. 2023-09-23T20:30:46.390 INFO:teuthology.orchestra.run.smithi026.stdout:Complete! 2023-09-23T20:30:46.854 INFO:teuthology.orchestra.run.smithi026.stdout:Updating Subscription Management repositories. 2023-09-23T20:30:47.177 INFO:teuthology.orchestra.run.smithi070.stdout:Last metadata expiration check: 0:01:44 ago on Sat 23 Sep 2023 08:29:02 PM UTC. 2023-09-23T20:30:47.696 INFO:teuthology.orchestra.run.smithi026.stdout:Last metadata expiration check: 0:01:36 ago on Sat 23 Sep 2023 08:29:11 PM UTC. 2023-09-23T20:30:48.879 INFO:teuthology.orchestra.run.smithi070.stdout:Dependencies resolved. 2023-09-23T20:30:48.970 INFO:teuthology.orchestra.run.smithi070.stdout:============================================================================================================ 2023-09-23T20:30:48.970 INFO:teuthology.orchestra.run.smithi070.stdout: Package Arch Version Repository Size 2023-09-23T20:30:48.970 INFO:teuthology.orchestra.run.smithi070.stdout:============================================================================================================ 2023-09-23T20:30:48.970 INFO:teuthology.orchestra.run.smithi070.stdout:Installing group/module packages: 2023-09-23T20:30:48.970 INFO:teuthology.orchestra.run.smithi070.stdout: buildah x86_64 1:1.26.2-1.module+el8.6.0+16771+28dfca77 rhel-8-for-x86_64-appstream-rpms 8.0 M 2023-09-23T20:30:48.971 INFO:teuthology.orchestra.run.smithi070.stdout: cockpit-podman noarch 49.1-1.module+el8.6.0+16771+28dfca77 rhel-8-for-x86_64-appstream-rpms 524 k 2023-09-23T20:30:48.971 INFO:teuthology.orchestra.run.smithi070.stdout: crun x86_64 1.5-1.module+el8.6.0+16771+28dfca77 rhel-8-for-x86_64-appstream-rpms 213 k 2023-09-23T20:30:48.971 INFO:teuthology.orchestra.run.smithi070.stdout: python3-podman noarch 4.0.1-1.module+el8.6.0+16837+85b82211 rhel-8-for-x86_64-appstream-rpms 149 k 2023-09-23T20:30:48.971 INFO:teuthology.orchestra.run.smithi070.stdout: skopeo x86_64 2:1.9.1-1.module+el8.6.0+16771+28dfca77 rhel-8-for-x86_64-appstream-rpms 7.0 M 2023-09-23T20:30:48.971 INFO:teuthology.orchestra.run.smithi070.stdout: toolbox x86_64 0.0.99.3-0.6.module+el8.6.0+16771+28dfca77 rhel-8-for-x86_64-appstream-rpms 2.2 M 2023-09-23T20:30:48.971 INFO:teuthology.orchestra.run.smithi070.stdout: udica noarch 0.2.6-4.module+el8.6.0+16771+28dfca77 rhel-8-for-x86_64-appstream-rpms 49 k 2023-09-23T20:30:48.971 INFO:teuthology.orchestra.run.smithi070.stdout:Installing dependencies: 2023-09-23T20:30:48.971 INFO:teuthology.orchestra.run.smithi070.stdout: python3-pytoml noarch 0.1.14-5.git7dea353.el8 rhel-8-for-x86_64-appstream-rpms 25 k 2023-09-23T20:30:48.971 INFO:teuthology.orchestra.run.smithi070.stdout: python3-pyxdg noarch 0.25-16.el8 rhel-8-for-x86_64-appstream-rpms 94 k 2023-09-23T20:30:48.971 INFO:teuthology.orchestra.run.smithi070.stdout: yajl x86_64 2.1.0-10.el8 rhel-8-for-x86_64-appstream-rpms 41 k 2023-09-23T20:30:48.971 INFO:teuthology.orchestra.run.smithi070.stdout:Installing module profiles: 2023-09-23T20:30:48.971 INFO:teuthology.orchestra.run.smithi070.stdout: container-tools/common 2023-09-23T20:30:48.971 INFO:teuthology.orchestra.run.smithi070.stdout: 2023-09-23T20:30:48.972 INFO:teuthology.orchestra.run.smithi070.stdout:Enabling module streams: 2023-09-23T20:30:48.972 INFO:teuthology.orchestra.run.smithi070.stdout: container-tools rhel8 2023-09-23T20:30:48.972 INFO:teuthology.orchestra.run.smithi070.stdout: 2023-09-23T20:30:48.972 INFO:teuthology.orchestra.run.smithi070.stdout:Transaction Summary 2023-09-23T20:30:48.972 INFO:teuthology.orchestra.run.smithi070.stdout:============================================================================================================ 2023-09-23T20:30:48.972 INFO:teuthology.orchestra.run.smithi070.stdout:Install 10 Packages 2023-09-23T20:30:48.972 INFO:teuthology.orchestra.run.smithi070.stdout: 2023-09-23T20:30:48.973 INFO:teuthology.orchestra.run.smithi070.stdout:Total download size: 18 M 2023-09-23T20:30:48.973 INFO:teuthology.orchestra.run.smithi070.stdout:Installed size: 65 M 2023-09-23T20:30:48.973 INFO:teuthology.orchestra.run.smithi070.stdout:Downloading Packages: 2023-09-23T20:30:49.045 INFO:teuthology.orchestra.run.smithi070.stdout:(1/10): yajl-2.1.0-10.el8.x86_64.rpm 560 kB/s | 41 kB 00:00 2023-09-23T20:30:49.070 INFO:teuthology.orchestra.run.smithi070.stdout:(2/10): python3-pytoml-0.1.14-5.git7dea353.el8. 253 kB/s | 25 kB 00:00 2023-09-23T20:30:49.097 INFO:teuthology.orchestra.run.smithi070.stdout:(3/10): udica-0.2.6-4.module+el8.6.0+16771+28df 982 kB/s | 49 kB 00:00 2023-09-23T20:30:49.121 INFO:teuthology.orchestra.run.smithi070.stdout:(4/10): python3-pyxdg-0.25-16.el8.noarch.rpm 631 kB/s | 94 kB 00:00 2023-09-23T20:30:49.155 INFO:teuthology.orchestra.run.smithi070.stdout:(5/10): cockpit-podman-49.1-1.module+el8.6.0+16 6.1 MB/s | 524 kB 00:00 2023-09-23T20:30:49.180 INFO:teuthology.orchestra.run.smithi070.stdout:(6/10): python3-podman-4.0.1-1.module+el8.6.0+1 2.5 MB/s | 149 kB 00:00 2023-09-23T20:30:49.205 INFO:teuthology.orchestra.run.smithi070.stdout:(7/10): crun-1.5-1.module+el8.6.0+16771+28dfca7 4.1 MB/s | 213 kB 00:00 2023-09-23T20:30:49.305 INFO:teuthology.orchestra.run.smithi070.stdout:(8/10): toolbox-0.0.99.3-0.6.module+el8.6.0+167 22 MB/s | 2.2 MB 00:00 2023-09-23T20:30:49.406 INFO:teuthology.orchestra.run.smithi070.stdout:(9/10): skopeo-1.9.1-1.module+el8.6.0+16771+28d 31 MB/s | 7.0 MB 00:00 2023-09-23T20:30:49.506 INFO:teuthology.orchestra.run.smithi070.stdout:(10/10): buildah-1.26.2-1.module+el8.6.0+16771+ 19 MB/s | 8.0 MB 00:00 2023-09-23T20:30:49.506 INFO:teuthology.orchestra.run.smithi070.stdout:-------------------------------------------------------------------------------- 2023-09-23T20:30:49.506 INFO:teuthology.orchestra.run.smithi070.stdout:Total 34 MB/s | 18 MB 00:00 2023-09-23T20:30:49.629 INFO:teuthology.orchestra.run.smithi026.stdout:Dependencies resolved. 2023-09-23T20:30:49.706 INFO:teuthology.orchestra.run.smithi070.stdout:Running transaction check 2023-09-23T20:30:49.724 INFO:teuthology.orchestra.run.smithi070.stdout:Transaction check succeeded. 2023-09-23T20:30:49.724 INFO:teuthology.orchestra.run.smithi070.stdout:Running transaction test 2023-09-23T20:30:49.729 INFO:teuthology.orchestra.run.smithi026.stdout:============================================================================================================ 2023-09-23T20:30:49.729 INFO:teuthology.orchestra.run.smithi026.stdout: Package Arch Version Repository Size 2023-09-23T20:30:49.729 INFO:teuthology.orchestra.run.smithi026.stdout:============================================================================================================ 2023-09-23T20:30:49.729 INFO:teuthology.orchestra.run.smithi026.stdout:Installing group/module packages: 2023-09-23T20:30:49.729 INFO:teuthology.orchestra.run.smithi026.stdout: buildah x86_64 1:1.26.2-1.module+el8.6.0+16771+28dfca77 rhel-8-for-x86_64-appstream-rpms 8.0 M 2023-09-23T20:30:49.729 INFO:teuthology.orchestra.run.smithi026.stdout: cockpit-podman noarch 49.1-1.module+el8.6.0+16771+28dfca77 rhel-8-for-x86_64-appstream-rpms 524 k 2023-09-23T20:30:49.729 INFO:teuthology.orchestra.run.smithi026.stdout: crun x86_64 1.5-1.module+el8.6.0+16771+28dfca77 rhel-8-for-x86_64-appstream-rpms 213 k 2023-09-23T20:30:49.729 INFO:teuthology.orchestra.run.smithi026.stdout: python3-podman noarch 4.0.1-1.module+el8.6.0+16837+85b82211 rhel-8-for-x86_64-appstream-rpms 149 k 2023-09-23T20:30:49.729 INFO:teuthology.orchestra.run.smithi026.stdout: skopeo x86_64 2:1.9.1-1.module+el8.6.0+16771+28dfca77 rhel-8-for-x86_64-appstream-rpms 7.0 M 2023-09-23T20:30:49.730 INFO:teuthology.orchestra.run.smithi026.stdout: toolbox x86_64 0.0.99.3-0.6.module+el8.6.0+16771+28dfca77 rhel-8-for-x86_64-appstream-rpms 2.2 M 2023-09-23T20:30:49.730 INFO:teuthology.orchestra.run.smithi026.stdout: udica noarch 0.2.6-4.module+el8.6.0+16771+28dfca77 rhel-8-for-x86_64-appstream-rpms 49 k 2023-09-23T20:30:49.730 INFO:teuthology.orchestra.run.smithi026.stdout:Installing dependencies: 2023-09-23T20:30:49.730 INFO:teuthology.orchestra.run.smithi026.stdout: python3-pytoml noarch 0.1.14-5.git7dea353.el8 rhel-8-for-x86_64-appstream-rpms 25 k 2023-09-23T20:30:49.730 INFO:teuthology.orchestra.run.smithi026.stdout: python3-pyxdg noarch 0.25-16.el8 rhel-8-for-x86_64-appstream-rpms 94 k 2023-09-23T20:30:49.730 INFO:teuthology.orchestra.run.smithi026.stdout: yajl x86_64 2.1.0-10.el8 rhel-8-for-x86_64-appstream-rpms 41 k 2023-09-23T20:30:49.730 INFO:teuthology.orchestra.run.smithi026.stdout:Installing module profiles: 2023-09-23T20:30:49.730 INFO:teuthology.orchestra.run.smithi026.stdout: container-tools/common 2023-09-23T20:30:49.730 INFO:teuthology.orchestra.run.smithi026.stdout: 2023-09-23T20:30:49.730 INFO:teuthology.orchestra.run.smithi026.stdout:Enabling module streams: 2023-09-23T20:30:49.730 INFO:teuthology.orchestra.run.smithi026.stdout: container-tools rhel8 2023-09-23T20:30:49.730 INFO:teuthology.orchestra.run.smithi026.stdout: 2023-09-23T20:30:49.731 INFO:teuthology.orchestra.run.smithi026.stdout:Transaction Summary 2023-09-23T20:30:49.731 INFO:teuthology.orchestra.run.smithi026.stdout:============================================================================================================ 2023-09-23T20:30:49.731 INFO:teuthology.orchestra.run.smithi026.stdout:Install 10 Packages 2023-09-23T20:30:49.731 INFO:teuthology.orchestra.run.smithi026.stdout: 2023-09-23T20:30:49.731 INFO:teuthology.orchestra.run.smithi026.stdout:Total download size: 18 M 2023-09-23T20:30:49.731 INFO:teuthology.orchestra.run.smithi026.stdout:Installed size: 65 M 2023-09-23T20:30:49.731 INFO:teuthology.orchestra.run.smithi026.stdout:Downloading Packages: 2023-09-23T20:30:49.807 INFO:teuthology.orchestra.run.smithi026.stdout:(1/10): python3-pyxdg-0.25-16.el8.noarch.rpm 1.2 MB/s | 94 kB 00:00 2023-09-23T20:30:49.832 INFO:teuthology.orchestra.run.smithi026.stdout:(2/10): python3-pytoml-0.1.14-5.git7dea353.el8. 247 kB/s | 25 kB 00:00 2023-09-23T20:30:49.858 INFO:teuthology.orchestra.run.smithi026.stdout:(3/10): yajl-2.1.0-10.el8.x86_64.rpm 325 kB/s | 41 kB 00:00 2023-09-23T20:30:49.883 INFO:teuthology.orchestra.run.smithi026.stdout:(4/10): udica-0.2.6-4.module+el8.6.0+16771+28df 653 kB/s | 49 kB 00:00 2023-09-23T20:30:49.906 INFO:teuthology.orchestra.run.smithi070.stdout:Transaction test succeeded. 2023-09-23T20:30:49.910 INFO:teuthology.orchestra.run.smithi070.stdout:Running transaction 2023-09-23T20:30:49.916 INFO:teuthology.orchestra.run.smithi026.stdout:(5/10): cockpit-podman-49.1-1.module+el8.6.0+16 6.1 MB/s | 524 kB 00:00 2023-09-23T20:30:49.942 INFO:teuthology.orchestra.run.smithi026.stdout:(6/10): python3-podman-4.0.1-1.module+el8.6.0+1 2.5 MB/s | 149 kB 00:00 2023-09-23T20:30:49.967 INFO:teuthology.orchestra.run.smithi026.stdout:(7/10): crun-1.5-1.module+el8.6.0+16771+28dfca7 4.1 MB/s | 213 kB 00:00 2023-09-23T20:30:50.068 INFO:teuthology.orchestra.run.smithi026.stdout:(8/10): toolbox-0.0.99.3-0.6.module+el8.6.0+167 22 MB/s | 2.2 MB 00:00 2023-09-23T20:30:50.193 INFO:teuthology.orchestra.run.smithi026.stdout:(9/10): skopeo-1.9.1-1.module+el8.6.0+16771+28d 28 MB/s | 7.0 MB 00:00 2023-09-23T20:30:50.284 INFO:teuthology.orchestra.run.smithi026.stdout:(10/10): buildah-1.26.2-1.module+el8.6.0+16771+ 19 MB/s | 8.0 MB 00:00 2023-09-23T20:30:50.285 INFO:teuthology.orchestra.run.smithi026.stdout:-------------------------------------------------------------------------------- 2023-09-23T20:30:50.285 INFO:teuthology.orchestra.run.smithi026.stdout:Total 33 MB/s | 18 MB 00:00 2023-09-23T20:30:50.423 INFO:teuthology.orchestra.run.smithi070.stdout: Preparing : 1/1 2023-09-23T20:30:50.494 INFO:teuthology.orchestra.run.smithi026.stdout:Running transaction check 2023-09-23T20:30:50.515 INFO:teuthology.orchestra.run.smithi026.stdout:Transaction check succeeded. 2023-09-23T20:30:50.515 INFO:teuthology.orchestra.run.smithi026.stdout:Running transaction test 2023-09-23T20:30:50.628 INFO:teuthology.orchestra.run.smithi070.stdout: Installing : python3-pytoml-0.1.14-5.git7dea353.el8.noarch 1/10 2023-09-23T20:30:50.710 INFO:teuthology.orchestra.run.smithi026.stdout:Transaction test succeeded. 2023-09-23T20:30:50.714 INFO:teuthology.orchestra.run.smithi026.stdout:Running transaction 2023-09-23T20:30:50.787 INFO:teuthology.orchestra.run.smithi070.stdout: Installing : yajl-2.1.0-10.el8.x86_64 2/10 2023-09-23T20:30:50.982 INFO:teuthology.orchestra.run.smithi070.stdout: Installing : python3-pyxdg-0.25-16.el8.noarch 3/10 2023-09-23T20:30:51.191 INFO:teuthology.orchestra.run.smithi070.stdout: Installing : python3-podman-4.0.1-1.module+el8.6.0+16837+85b822 4/10 2023-09-23T20:30:51.247 INFO:teuthology.orchestra.run.smithi026.stdout: Preparing : 1/1 2023-09-23T20:30:51.428 INFO:teuthology.orchestra.run.smithi026.stdout: Installing : python3-pytoml-0.1.14-5.git7dea353.el8.noarch 1/10 2023-09-23T20:30:51.557 INFO:teuthology.orchestra.run.smithi070.stdout: Installing : crun-1.5-1.module+el8.6.0+16771+28dfca77.x86_64 5/10 2023-09-23T20:30:51.601 INFO:teuthology.orchestra.run.smithi026.stdout: Installing : yajl-2.1.0-10.el8.x86_64 2/10 2023-09-23T20:30:51.797 INFO:teuthology.orchestra.run.smithi026.stdout: Installing : python3-pyxdg-0.25-16.el8.noarch 3/10 2023-09-23T20:30:51.999 INFO:teuthology.orchestra.run.smithi026.stdout: Installing : python3-podman-4.0.1-1.module+el8.6.0+16837+85b822 4/10 2023-09-23T20:30:52.319 INFO:teuthology.orchestra.run.smithi070.stdout: Installing : toolbox-0.0.99.3-0.6.module+el8.6.0+16771+28dfca77 6/10 2023-09-23T20:30:52.357 INFO:teuthology.orchestra.run.smithi026.stdout: Installing : crun-1.5-1.module+el8.6.0+16771+28dfca77.x86_64 5/10 2023-09-23T20:30:53.146 INFO:teuthology.orchestra.run.smithi026.stdout: Installing : toolbox-0.0.99.3-0.6.module+el8.6.0+16771+28dfca77 6/10 2023-09-23T20:30:53.209 INFO:teuthology.orchestra.run.smithi070.stdout: Installing : skopeo-2:1.9.1-1.module+el8.6.0+16771+28dfca77.x86 7/10 2023-09-23T20:30:53.429 INFO:teuthology.orchestra.run.smithi070.stdout: Installing : buildah-1:1.26.2-1.module+el8.6.0+16771+28dfca77.x 8/10 2023-09-23T20:30:53.586 INFO:teuthology.orchestra.run.smithi070.stdout: Installing : cockpit-podman-49.1-1.module+el8.6.0+16771+28dfca7 9/10 2023-09-23T20:30:53.746 INFO:teuthology.orchestra.run.smithi070.stdout: Installing : udica-0.2.6-4.module+el8.6.0+16771+28dfca77.noarch 10/10 2023-09-23T20:30:54.007 INFO:teuthology.orchestra.run.smithi070.stdout: Running scriptlet: udica-0.2.6-4.module+el8.6.0+16771+28dfca77.noarch 10/10 2023-09-23T20:30:54.007 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : python3-pyxdg-0.25-16.el8.noarch 1/10 2023-09-23T20:30:54.007 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : yajl-2.1.0-10.el8.x86_64 2/10 2023-09-23T20:30:54.007 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : python3-pytoml-0.1.14-5.git7dea353.el8.noarch 3/10 2023-09-23T20:30:54.007 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : udica-0.2.6-4.module+el8.6.0+16771+28dfca77.noarch 4/10 2023-09-23T20:30:54.007 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : cockpit-podman-49.1-1.module+el8.6.0+16771+28dfca7 5/10 2023-09-23T20:30:54.007 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : buildah-1:1.26.2-1.module+el8.6.0+16771+28dfca77.x 6/10 2023-09-23T20:30:54.007 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : python3-podman-4.0.1-1.module+el8.6.0+16837+85b822 7/10 2023-09-23T20:30:54.007 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : crun-1.5-1.module+el8.6.0+16771+28dfca77.x86_64 8/10 2023-09-23T20:30:54.008 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : skopeo-2:1.9.1-1.module+el8.6.0+16771+28dfca77.x86 9/10 2023-09-23T20:30:54.111 INFO:teuthology.orchestra.run.smithi026.stdout: Installing : skopeo-2:1.9.1-1.module+el8.6.0+16771+28dfca77.x86 7/10 2023-09-23T20:30:54.361 INFO:teuthology.orchestra.run.smithi026.stdout: Installing : buildah-1:1.26.2-1.module+el8.6.0+16771+28dfca77.x 8/10 2023-09-23T20:30:54.532 INFO:teuthology.orchestra.run.smithi026.stdout: Installing : cockpit-podman-49.1-1.module+el8.6.0+16771+28dfca7 9/10 2023-09-23T20:30:54.700 INFO:teuthology.orchestra.run.smithi026.stdout: Installing : udica-0.2.6-4.module+el8.6.0+16771+28dfca77.noarch 10/10 2023-09-23T20:30:54.969 INFO:teuthology.orchestra.run.smithi026.stdout: Running scriptlet: udica-0.2.6-4.module+el8.6.0+16771+28dfca77.noarch 10/10 2023-09-23T20:30:54.970 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : python3-pyxdg-0.25-16.el8.noarch 1/10 2023-09-23T20:30:54.970 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : yajl-2.1.0-10.el8.x86_64 2/10 2023-09-23T20:30:54.970 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : python3-pytoml-0.1.14-5.git7dea353.el8.noarch 3/10 2023-09-23T20:30:54.970 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : udica-0.2.6-4.module+el8.6.0+16771+28dfca77.noarch 4/10 2023-09-23T20:30:54.970 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : cockpit-podman-49.1-1.module+el8.6.0+16771+28dfca7 5/10 2023-09-23T20:30:54.970 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : buildah-1:1.26.2-1.module+el8.6.0+16771+28dfca77.x 6/10 2023-09-23T20:30:54.970 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : python3-podman-4.0.1-1.module+el8.6.0+16837+85b822 7/10 2023-09-23T20:30:54.970 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : crun-1.5-1.module+el8.6.0+16771+28dfca77.x86_64 8/10 2023-09-23T20:30:54.970 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : skopeo-2:1.9.1-1.module+el8.6.0+16771+28dfca77.x86 9/10 2023-09-23T20:30:55.074 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : toolbox-0.0.99.3-0.6.module+el8.6.0+16771+28dfca77 10/10 2023-09-23T20:30:55.075 INFO:teuthology.orchestra.run.smithi070.stdout:Last metadata expiration check: 0:01:52 ago on Sat 23 Sep 2023 08:29:02 PM UTC. 2023-09-23T20:30:56.125 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : toolbox-0.0.99.3-0.6.module+el8.6.0+16771+28dfca77 10/10 2023-09-23T20:30:56.125 INFO:teuthology.orchestra.run.smithi026.stdout:Last metadata expiration check: 0:01:44 ago on Sat 23 Sep 2023 08:29:11 PM UTC. 2023-09-23T20:30:57.696 INFO:teuthology.orchestra.run.smithi070.stdout:Installed products updated. 2023-09-23T20:30:57.895 INFO:teuthology.orchestra.run.smithi070.stdout: 2023-09-23T20:30:57.895 INFO:teuthology.orchestra.run.smithi070.stdout:Installed: 2023-09-23T20:30:57.895 INFO:teuthology.orchestra.run.smithi070.stdout: buildah-1:1.26.2-1.module+el8.6.0+16771+28dfca77.x86_64 2023-09-23T20:30:57.895 INFO:teuthology.orchestra.run.smithi070.stdout: cockpit-podman-49.1-1.module+el8.6.0+16771+28dfca77.noarch 2023-09-23T20:30:57.895 INFO:teuthology.orchestra.run.smithi070.stdout: crun-1.5-1.module+el8.6.0+16771+28dfca77.x86_64 2023-09-23T20:30:57.895 INFO:teuthology.orchestra.run.smithi070.stdout: python3-podman-4.0.1-1.module+el8.6.0+16837+85b82211.noarch 2023-09-23T20:30:57.896 INFO:teuthology.orchestra.run.smithi070.stdout: python3-pytoml-0.1.14-5.git7dea353.el8.noarch 2023-09-23T20:30:57.896 INFO:teuthology.orchestra.run.smithi070.stdout: python3-pyxdg-0.25-16.el8.noarch 2023-09-23T20:30:57.896 INFO:teuthology.orchestra.run.smithi070.stdout: skopeo-2:1.9.1-1.module+el8.6.0+16771+28dfca77.x86_64 2023-09-23T20:30:57.896 INFO:teuthology.orchestra.run.smithi070.stdout: toolbox-0.0.99.3-0.6.module+el8.6.0+16771+28dfca77.x86_64 2023-09-23T20:30:57.896 INFO:teuthology.orchestra.run.smithi070.stdout: udica-0.2.6-4.module+el8.6.0+16771+28dfca77.noarch 2023-09-23T20:30:57.896 INFO:teuthology.orchestra.run.smithi070.stdout: yajl-2.1.0-10.el8.x86_64 2023-09-23T20:30:57.896 INFO:teuthology.orchestra.run.smithi070.stdout: 2023-09-23T20:30:57.896 INFO:teuthology.orchestra.run.smithi070.stdout:Complete! 2023-09-23T20:30:58.038 DEBUG:teuthology.parallel:result is None 2023-09-23T20:30:58.801 INFO:teuthology.orchestra.run.smithi026.stdout:Installed products updated. 2023-09-23T20:30:59.013 INFO:teuthology.orchestra.run.smithi026.stdout: 2023-09-23T20:30:59.013 INFO:teuthology.orchestra.run.smithi026.stdout:Installed: 2023-09-23T20:30:59.013 INFO:teuthology.orchestra.run.smithi026.stdout: buildah-1:1.26.2-1.module+el8.6.0+16771+28dfca77.x86_64 2023-09-23T20:30:59.013 INFO:teuthology.orchestra.run.smithi026.stdout: cockpit-podman-49.1-1.module+el8.6.0+16771+28dfca77.noarch 2023-09-23T20:30:59.013 INFO:teuthology.orchestra.run.smithi026.stdout: crun-1.5-1.module+el8.6.0+16771+28dfca77.x86_64 2023-09-23T20:30:59.013 INFO:teuthology.orchestra.run.smithi026.stdout: python3-podman-4.0.1-1.module+el8.6.0+16837+85b82211.noarch 2023-09-23T20:30:59.013 INFO:teuthology.orchestra.run.smithi026.stdout: python3-pytoml-0.1.14-5.git7dea353.el8.noarch 2023-09-23T20:30:59.013 INFO:teuthology.orchestra.run.smithi026.stdout: python3-pyxdg-0.25-16.el8.noarch 2023-09-23T20:30:59.013 INFO:teuthology.orchestra.run.smithi026.stdout: skopeo-2:1.9.1-1.module+el8.6.0+16771+28dfca77.x86_64 2023-09-23T20:30:59.014 INFO:teuthology.orchestra.run.smithi026.stdout: toolbox-0.0.99.3-0.6.module+el8.6.0+16771+28dfca77.x86_64 2023-09-23T20:30:59.014 INFO:teuthology.orchestra.run.smithi026.stdout: udica-0.2.6-4.module+el8.6.0+16771+28dfca77.noarch 2023-09-23T20:30:59.014 INFO:teuthology.orchestra.run.smithi026.stdout: yajl-2.1.0-10.el8.x86_64 2023-09-23T20:30:59.014 INFO:teuthology.orchestra.run.smithi026.stdout: 2023-09-23T20:30:59.014 INFO:teuthology.orchestra.run.smithi026.stdout:Complete! 2023-09-23T20:30:59.155 DEBUG:teuthology.parallel:result is None 2023-09-23T20:30:59.155 INFO:teuthology.run_tasks:Running task install... 2023-09-23T20:30:59.164 DEBUG:teuthology.task.install:project ceph 2023-09-23T20:30:59.164 DEBUG:teuthology.task.install:INSTALL overrides: {'ceph': {'flavor': 'default', 'sha1': 'bb0daeff97f020f686596fca6eba20c310e26fb7'}} 2023-09-23T20:30:59.164 DEBUG:teuthology.task.install:config {'branch': 'quincy', 'exclude_packages': ['ceph-volume'], 'flavor': 'default', 'sha1': 'bb0daeff97f020f686596fca6eba20c310e26fb7'} 2023-09-23T20:30:59.164 INFO:teuthology.task.install:Using flavor: default 2023-09-23T20:30:59.170 DEBUG:teuthology.task.install:Package list is: {'deb': ['ceph', 'cephadm', 'ceph-mds', 'ceph-mgr', 'ceph-common', 'ceph-fuse', 'ceph-test', 'radosgw', 'python3-rados', 'python3-rgw', 'python3-cephfs', 'python3-rbd', 'libcephfs2', 'libcephfs-dev', 'librados2', 'librbd1', 'rbd-fuse'], 'rpm': ['ceph-radosgw', 'ceph-test', 'ceph', 'ceph-base', 'cephadm', 'ceph-immutable-object-cache', 'ceph-mgr', 'ceph-mgr-dashboard', 'ceph-mgr-diskprediction-local', 'ceph-mgr-rook', 'ceph-mgr-cephadm', 'ceph-fuse', 'librados-devel', 'libcephfs2', 'libcephfs-devel', 'librados2', 'librbd1', 'python3-rados', 'python3-rgw', 'python3-cephfs', 'python3-rbd', 'rbd-fuse', 'rbd-mirror', 'rbd-nbd']} 2023-09-23T20:30:59.170 INFO:teuthology.task.install:extra packages: [] 2023-09-23T20:30:59.170 DEBUG:teuthology.task.install.rpm:_update_package_list_and_install: config is {'branch': 'quincy', 'cleanup': None, 'debuginfo': None, 'downgrade_packages': [], 'exclude_packages': ['ceph-volume'], 'extra_packages': [], 'extra_system_packages': [], 'extras': None, 'enable_coprs': [], 'flavor': 'default', 'install_ceph_packages': True, 'packages': {}, 'project': 'ceph', 'repos_only': False, 'sha1': 'bb0daeff97f020f686596fca6eba20c310e26fb7', 'tag': None, 'wait_for_package': False} 2023-09-23T20:30:59.171 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2023-09-23T20:30:59.171 INFO:teuthology.packaging:ref: None 2023-09-23T20:30:59.171 INFO:teuthology.packaging:tag: None 2023-09-23T20:30:59.171 INFO:teuthology.packaging:branch: quincy 2023-09-23T20:30:59.171 INFO:teuthology.packaging:sha1: bb0daeff97f020f686596fca6eba20c310e26fb7 2023-09-23T20:30:59.171 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F8%2Fx86_64&ref=quincy 2023-09-23T20:30:59.173 DEBUG:teuthology.task.install.rpm:_update_package_list_and_install: config is {'branch': 'quincy', 'cleanup': None, 'debuginfo': None, 'downgrade_packages': [], 'exclude_packages': ['ceph-volume'], 'extra_packages': [], 'extra_system_packages': [], 'extras': None, 'enable_coprs': [], 'flavor': 'default', 'install_ceph_packages': True, 'packages': {}, 'project': 'ceph', 'repos_only': False, 'sha1': 'bb0daeff97f020f686596fca6eba20c310e26fb7', 'tag': None, 'wait_for_package': False} 2023-09-23T20:30:59.173 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2023-09-23T20:30:59.173 INFO:teuthology.packaging:ref: None 2023-09-23T20:30:59.173 INFO:teuthology.packaging:tag: None 2023-09-23T20:30:59.173 INFO:teuthology.packaging:branch: quincy 2023-09-23T20:30:59.173 INFO:teuthology.packaging:sha1: bb0daeff97f020f686596fca6eba20c310e26fb7 2023-09-23T20:30:59.173 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F8%2Fx86_64&ref=quincy 2023-09-23T20:30:59.397 INFO:teuthology.task.install.rpm:Pulling from https://3.chacra.ceph.com/r/ceph/quincy/61431f6ba2752649b506867ec856b4a6236f90dd/centos/8/flavors/default/ 2023-09-23T20:30:59.398 INFO:teuthology.task.install.rpm:Package version is 17.2.6-1123.g61431f6b 2023-09-23T20:30:59.510 INFO:teuthology.packaging:Writing yum repo: [ceph] name=ceph packages for $basearch baseurl=https://3.chacra.ceph.com/r/ceph/quincy/61431f6ba2752649b506867ec856b4a6236f90dd/centos/8/flavors/default/$basearch enabled=1 gpgcheck=0 type=rpm-md [ceph-noarch] name=ceph noarch packages baseurl=https://3.chacra.ceph.com/r/ceph/quincy/61431f6ba2752649b506867ec856b4a6236f90dd/centos/8/flavors/default/noarch enabled=1 gpgcheck=0 type=rpm-md [ceph-source] name=ceph source packages baseurl=https://3.chacra.ceph.com/r/ceph/quincy/61431f6ba2752649b506867ec856b4a6236f90dd/centos/8/flavors/default/SRPMS enabled=1 gpgcheck=0 type=rpm-md 2023-09-23T20:30:59.510 DEBUG:teuthology.orchestra.run.smithi026:> set -ex 2023-09-23T20:30:59.510 DEBUG:teuthology.orchestra.run.smithi026:> sudo dd of=/etc/yum.repos.d/ceph.repo 2023-09-23T20:30:59.538 INFO:teuthology.task.install.rpm:Installing packages: ceph-radosgw, ceph-test, ceph, ceph-base, cephadm, ceph-immutable-object-cache, ceph-mgr, ceph-mgr-dashboard, ceph-mgr-diskprediction-local, ceph-mgr-rook, ceph-mgr-cephadm, ceph-fuse, librados-devel, libcephfs2, libcephfs-devel, librados2, librbd1, python3-rados, python3-rgw, python3-cephfs, python3-rbd, rbd-fuse, rbd-mirror, rbd-nbd on remote rpm x86_64 2023-09-23T20:30:59.539 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2023-09-23T20:30:59.539 INFO:teuthology.packaging:ref: None 2023-09-23T20:30:59.539 INFO:teuthology.packaging:tag: None 2023-09-23T20:30:59.539 INFO:teuthology.packaging:branch: quincy 2023-09-23T20:30:59.539 INFO:teuthology.packaging:sha1: bb0daeff97f020f686596fca6eba20c310e26fb7 2023-09-23T20:30:59.539 DEBUG:teuthology.orchestra.run.smithi026:> if test -f /etc/yum.repos.d/ceph.repo ; then sudo sed -i -e ':a;N;$!ba;s/enabled=1\ngpg/enabled=1\npriority=1\ngpg/g' -e 's;ref/[a-zA-Z0-9_-]*/;ref/quincy/;g' /etc/yum.repos.d/ceph.repo ; fi 2023-09-23T20:30:59.575 INFO:teuthology.task.install.rpm:Pulling from https://3.chacra.ceph.com/r/ceph/quincy/61431f6ba2752649b506867ec856b4a6236f90dd/centos/8/flavors/default/ 2023-09-23T20:30:59.575 INFO:teuthology.task.install.rpm:Package version is 17.2.6-1123.g61431f6b 2023-09-23T20:30:59.608 DEBUG:teuthology.orchestra.run.smithi026:> sudo touch -a /etc/yum/pluginconf.d/priorities.conf ; test -e /etc/yum/pluginconf.d/priorities.conf.orig || sudo cp -af /etc/yum/pluginconf.d/priorities.conf /etc/yum/pluginconf.d/priorities.conf.orig 2023-09-23T20:30:59.686 INFO:teuthology.packaging:Writing yum repo: [ceph] name=ceph packages for $basearch baseurl=https://3.chacra.ceph.com/r/ceph/quincy/61431f6ba2752649b506867ec856b4a6236f90dd/centos/8/flavors/default/$basearch enabled=1 gpgcheck=0 type=rpm-md [ceph-noarch] name=ceph noarch packages baseurl=https://3.chacra.ceph.com/r/ceph/quincy/61431f6ba2752649b506867ec856b4a6236f90dd/centos/8/flavors/default/noarch enabled=1 gpgcheck=0 type=rpm-md [ceph-source] name=ceph source packages baseurl=https://3.chacra.ceph.com/r/ceph/quincy/61431f6ba2752649b506867ec856b4a6236f90dd/centos/8/flavors/default/SRPMS enabled=1 gpgcheck=0 type=rpm-md 2023-09-23T20:30:59.687 DEBUG:teuthology.orchestra.run.smithi070:> set -ex 2023-09-23T20:30:59.687 DEBUG:teuthology.orchestra.run.smithi070:> sudo dd of=/etc/yum.repos.d/ceph.repo 2023-09-23T20:30:59.688 DEBUG:teuthology.orchestra.run.smithi026:> grep check_obsoletes /etc/yum/pluginconf.d/priorities.conf && sudo sed -i 's/check_obsoletes.*0/check_obsoletes = 1/g' /etc/yum/pluginconf.d/priorities.conf || echo 'check_obsoletes = 1' | sudo tee -a /etc/yum/pluginconf.d/priorities.conf 2023-09-23T20:30:59.720 INFO:teuthology.task.install.rpm:Installing packages: ceph-radosgw, ceph-test, ceph, ceph-base, cephadm, ceph-immutable-object-cache, ceph-mgr, ceph-mgr-dashboard, ceph-mgr-diskprediction-local, ceph-mgr-rook, ceph-mgr-cephadm, ceph-fuse, librados-devel, libcephfs2, libcephfs-devel, librados2, librbd1, python3-rados, python3-rgw, python3-cephfs, python3-rbd, rbd-fuse, rbd-mirror, rbd-nbd on remote rpm x86_64 2023-09-23T20:30:59.720 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2023-09-23T20:30:59.721 INFO:teuthology.packaging:ref: None 2023-09-23T20:30:59.721 INFO:teuthology.packaging:tag: None 2023-09-23T20:30:59.721 INFO:teuthology.packaging:branch: quincy 2023-09-23T20:30:59.721 INFO:teuthology.packaging:sha1: bb0daeff97f020f686596fca6eba20c310e26fb7 2023-09-23T20:30:59.721 DEBUG:teuthology.orchestra.run.smithi070:> if test -f /etc/yum.repos.d/ceph.repo ; then sudo sed -i -e ':a;N;$!ba;s/enabled=1\ngpg/enabled=1\npriority=1\ngpg/g' -e 's;ref/[a-zA-Z0-9_-]*/;ref/quincy/;g' /etc/yum.repos.d/ceph.repo ; fi 2023-09-23T20:30:59.755 INFO:teuthology.orchestra.run.smithi026.stdout:check_obsoletes = 1 2023-09-23T20:30:59.756 DEBUG:teuthology.orchestra.run.smithi026:> sudo yum clean all 2023-09-23T20:30:59.793 DEBUG:teuthology.orchestra.run.smithi070:> sudo touch -a /etc/yum/pluginconf.d/priorities.conf ; test -e /etc/yum/pluginconf.d/priorities.conf.orig || sudo cp -af /etc/yum/pluginconf.d/priorities.conf /etc/yum/pluginconf.d/priorities.conf.orig 2023-09-23T20:30:59.875 DEBUG:teuthology.orchestra.run.smithi070:> grep check_obsoletes /etc/yum/pluginconf.d/priorities.conf && sudo sed -i 's/check_obsoletes.*0/check_obsoletes = 1/g' /etc/yum/pluginconf.d/priorities.conf || echo 'check_obsoletes = 1' | sudo tee -a /etc/yum/pluginconf.d/priorities.conf 2023-09-23T20:30:59.943 INFO:teuthology.orchestra.run.smithi070.stdout:check_obsoletes = 1 2023-09-23T20:30:59.945 DEBUG:teuthology.orchestra.run.smithi070:> sudo yum clean all 2023-09-23T20:31:00.147 INFO:teuthology.orchestra.run.smithi026.stdout:Updating Subscription Management repositories. 2023-09-23T20:31:00.318 INFO:teuthology.orchestra.run.smithi070.stdout:Updating Subscription Management repositories. 2023-09-23T20:31:00.434 INFO:teuthology.orchestra.run.smithi026.stdout:44 files removed 2023-09-23T20:31:00.482 DEBUG:teuthology.orchestra.run.smithi026:> sudo yum -y install ceph-radosgw 2023-09-23T20:31:00.599 INFO:teuthology.orchestra.run.smithi070.stdout:44 files removed 2023-09-23T20:31:00.644 DEBUG:teuthology.orchestra.run.smithi070:> sudo yum -y install ceph-radosgw 2023-09-23T20:31:00.837 INFO:teuthology.orchestra.run.smithi026.stdout:Updating Subscription Management repositories. 2023-09-23T20:31:00.972 INFO:teuthology.orchestra.run.smithi070.stdout:Updating Subscription Management repositories. 2023-09-23T20:31:01.706 INFO:teuthology.orchestra.run.smithi026.stdout:ceph packages for x86_64 174 kB/s | 78 kB 00:00 2023-09-23T20:31:01.857 INFO:teuthology.orchestra.run.smithi070.stdout:ceph packages for x86_64 165 kB/s | 78 kB 00:00 2023-09-23T20:31:02.182 INFO:teuthology.orchestra.run.smithi026.stdout:ceph noarch packages 21 kB/s | 9.1 kB 00:00 2023-09-23T20:31:02.232 INFO:teuthology.orchestra.run.smithi070.stdout:ceph noarch packages 27 kB/s | 9.1 kB 00:00 2023-09-23T20:31:02.533 INFO:teuthology.orchestra.run.smithi026.stdout:ceph source packages 4.4 kB/s | 1.5 kB 00:00 2023-09-23T20:31:02.683 INFO:teuthology.orchestra.run.smithi070.stdout:ceph source packages 3.4 kB/s | 1.5 kB 00:00 2023-09-23T20:31:02.742 INFO:teuthology.orchestra.run.smithi026.stdout:Copr repo for python3-asyncssh owned by ceph 19 kB/s | 3.5 kB 00:00 2023-09-23T20:31:02.867 INFO:teuthology.orchestra.run.smithi070.stdout:Copr repo for python3-asyncssh owned by ceph 21 kB/s | 3.5 kB 00:00 2023-09-23T20:31:04.801 INFO:teuthology.orchestra.run.smithi070.stdout:Extra Packages for Enterprise Linux 8.3 MB/s | 16 MB 00:01 2023-09-23T20:31:05.192 INFO:teuthology.orchestra.run.smithi026.stdout:Extra Packages for Enterprise Linux 6.5 MB/s | 16 MB 00:02 2023-09-23T20:31:08.367 INFO:teuthology.orchestra.run.smithi070.stdout:lab-extras 428 kB/s | 24 kB 00:00 2023-09-23T20:31:08.934 INFO:teuthology.orchestra.run.smithi026.stdout:lab-extras 530 kB/s | 24 kB 00:00 2023-09-23T20:31:09.302 INFO:teuthology.orchestra.run.smithi070.stdout:Red Hat Enterprise Linux 8 for x86_64 - BaseOS 61 MB/s | 53 MB 00:00 2023-09-23T20:31:09.919 INFO:teuthology.orchestra.run.smithi026.stdout:Red Hat Enterprise Linux 8 for x86_64 - BaseOS 58 MB/s | 53 MB 00:00 2023-09-23T20:31:17.378 INFO:teuthology.orchestra.run.smithi070.stdout:Red Hat Enterprise Linux 8 for x86_64 - AppStre 56 MB/s | 47 MB 00:00 2023-09-23T20:31:18.686 INFO:teuthology.orchestra.run.smithi026.stdout:Red Hat Enterprise Linux 8 for x86_64 - AppStre 56 MB/s | 47 MB 00:00 2023-09-23T20:31:28.369 INFO:teuthology.orchestra.run.smithi070.stdout:Dependencies resolved. 2023-09-23T20:31:28.371 INFO:teuthology.orchestra.run.smithi070.stdout:==================================================================================================== 2023-09-23T20:31:28.371 INFO:teuthology.orchestra.run.smithi070.stdout: Package Arch Version Repository Size 2023-09-23T20:31:28.372 INFO:teuthology.orchestra.run.smithi070.stdout:==================================================================================================== 2023-09-23T20:31:28.372 INFO:teuthology.orchestra.run.smithi070.stdout:Installing: 2023-09-23T20:31:28.372 INFO:teuthology.orchestra.run.smithi070.stdout: ceph-radosgw x86_64 2:17.2.6-1123.g61431f6b.el8 ceph 17 M 2023-09-23T20:31:28.372 INFO:teuthology.orchestra.run.smithi070.stdout:Upgrading: 2023-09-23T20:31:28.372 INFO:teuthology.orchestra.run.smithi070.stdout: librados2 x86_64 2:17.2.6-1123.g61431f6b.el8 ceph 3.5 M 2023-09-23T20:31:28.372 INFO:teuthology.orchestra.run.smithi070.stdout: librbd1 x86_64 2:17.2.6-1123.g61431f6b.el8 ceph 3.6 M 2023-09-23T20:31:28.372 INFO:teuthology.orchestra.run.smithi070.stdout:Installing dependencies: 2023-09-23T20:31:28.372 INFO:teuthology.orchestra.run.smithi070.stdout: ceph-base x86_64 2:17.2.6-1123.g61431f6b.el8 ceph 5.2 M 2023-09-23T20:31:28.372 INFO:teuthology.orchestra.run.smithi070.stdout: ceph-common x86_64 2:17.2.6-1123.g61431f6b.el8 ceph 23 M 2023-09-23T20:31:28.372 INFO:teuthology.orchestra.run.smithi070.stdout: ceph-selinux x86_64 2:17.2.6-1123.g61431f6b.el8 ceph 24 k 2023-09-23T20:31:28.372 INFO:teuthology.orchestra.run.smithi070.stdout: gperftools-libs x86_64 1:2.7-9.el8 epel 306 k 2023-09-23T20:31:28.372 INFO:teuthology.orchestra.run.smithi070.stdout: libcephfs2 x86_64 2:17.2.6-1123.g61431f6b.el8 ceph 796 k 2023-09-23T20:31:28.373 INFO:teuthology.orchestra.run.smithi070.stdout: liboath x86_64 2.6.2-3.el8 epel 59 k 2023-09-23T20:31:28.373 INFO:teuthology.orchestra.run.smithi070.stdout: librabbitmq x86_64 0.9.0-3.el8 rhel-8-for-x86_64-baseos-rpms 47 k 2023-09-23T20:31:28.373 INFO:teuthology.orchestra.run.smithi070.stdout: libradosstriper1 x86_64 2:17.2.6-1123.g61431f6b.el8 ceph 506 k 2023-09-23T20:31:28.373 INFO:teuthology.orchestra.run.smithi070.stdout: librdkafka x86_64 0.11.4-3.el8 rhel-8-for-x86_64-appstream-rpms 354 k 2023-09-23T20:31:28.373 INFO:teuthology.orchestra.run.smithi070.stdout: librgw2 x86_64 2:17.2.6-1123.g61431f6b.el8 ceph 5.5 M 2023-09-23T20:31:28.373 INFO:teuthology.orchestra.run.smithi070.stdout: libunwind x86_64 1.3.1-3.el8 epel 75 k 2023-09-23T20:31:28.373 INFO:teuthology.orchestra.run.smithi070.stdout: lttng-ust x86_64 2.8.1-11.el8 rhel-8-for-x86_64-appstream-rpms 259 k 2023-09-23T20:31:28.373 INFO:teuthology.orchestra.run.smithi070.stdout: python3-ceph-argparse x86_64 2:17.2.6-1123.g61431f6b.el8 ceph 45 k 2023-09-23T20:31:28.373 INFO:teuthology.orchestra.run.smithi070.stdout: python3-ceph-common x86_64 2:17.2.6-1123.g61431f6b.el8 ceph 115 k 2023-09-23T20:31:28.373 INFO:teuthology.orchestra.run.smithi070.stdout: python3-cephfs x86_64 2:17.2.6-1123.g61431f6b.el8 ceph 190 k 2023-09-23T20:31:28.373 INFO:teuthology.orchestra.run.smithi070.stdout: python3-prettytable noarch 0.7.2-14.el8 rhel-8-for-x86_64-appstream-rpms 44 k 2023-09-23T20:31:28.374 INFO:teuthology.orchestra.run.smithi070.stdout: python3-rados x86_64 2:17.2.6-1123.g61431f6b.el8 ceph 371 k 2023-09-23T20:31:28.374 INFO:teuthology.orchestra.run.smithi070.stdout: python3-rbd x86_64 2:17.2.6-1123.g61431f6b.el8 ceph 358 k 2023-09-23T20:31:28.374 INFO:teuthology.orchestra.run.smithi070.stdout: python3-rgw x86_64 2:17.2.6-1123.g61431f6b.el8 ceph 106 k 2023-09-23T20:31:28.374 INFO:teuthology.orchestra.run.smithi070.stdout: thrift x86_64 0.13.0-2.el8 epel 1.7 M 2023-09-23T20:31:28.374 INFO:teuthology.orchestra.run.smithi070.stdout: 2023-09-23T20:31:28.374 INFO:teuthology.orchestra.run.smithi070.stdout:Transaction Summary 2023-09-23T20:31:28.375 INFO:teuthology.orchestra.run.smithi070.stdout:==================================================================================================== 2023-09-23T20:31:28.375 INFO:teuthology.orchestra.run.smithi070.stdout:Install 21 Packages 2023-09-23T20:31:28.375 INFO:teuthology.orchestra.run.smithi070.stdout:Upgrade 2 Packages 2023-09-23T20:31:28.375 INFO:teuthology.orchestra.run.smithi070.stdout: 2023-09-23T20:31:28.376 INFO:teuthology.orchestra.run.smithi070.stdout:Total download size: 63 M 2023-09-23T20:31:28.376 INFO:teuthology.orchestra.run.smithi070.stdout:Downloading Packages: 2023-09-23T20:31:29.486 INFO:teuthology.orchestra.run.smithi070.stdout:(1/23): ceph-base-17.2.6-1123.g61431f6b.el8.x86 5.7 MB/s | 5.2 MB 00:00 2023-09-23T20:31:29.569 INFO:teuthology.orchestra.run.smithi070.stdout:(2/23): ceph-selinux-17.2.6-1123.g61431f6b.el8. 293 kB/s | 24 kB 00:00 2023-09-23T20:31:29.694 INFO:teuthology.orchestra.run.smithi070.stdout:(3/23): libcephfs2-17.2.6-1123.g61431f6b.el8.x8 6.2 MB/s | 796 kB 00:00 2023-09-23T20:31:29.820 INFO:teuthology.orchestra.run.smithi070.stdout:(4/23): libradosstriper1-17.2.6-1123.g61431f6b. 4.0 MB/s | 506 kB 00:00 2023-09-23T20:31:30.540 INFO:teuthology.orchestra.run.smithi026.stdout:Dependencies resolved. 2023-09-23T20:31:30.543 INFO:teuthology.orchestra.run.smithi026.stdout:==================================================================================================== 2023-09-23T20:31:30.543 INFO:teuthology.orchestra.run.smithi026.stdout: Package Arch Version Repository Size 2023-09-23T20:31:30.543 INFO:teuthology.orchestra.run.smithi026.stdout:==================================================================================================== 2023-09-23T20:31:30.543 INFO:teuthology.orchestra.run.smithi026.stdout:Installing: 2023-09-23T20:31:30.544 INFO:teuthology.orchestra.run.smithi026.stdout: ceph-radosgw x86_64 2:17.2.6-1123.g61431f6b.el8 ceph 17 M 2023-09-23T20:31:30.544 INFO:teuthology.orchestra.run.smithi026.stdout:Upgrading: 2023-09-23T20:31:30.544 INFO:teuthology.orchestra.run.smithi026.stdout: librados2 x86_64 2:17.2.6-1123.g61431f6b.el8 ceph 3.5 M 2023-09-23T20:31:30.544 INFO:teuthology.orchestra.run.smithi026.stdout: librbd1 x86_64 2:17.2.6-1123.g61431f6b.el8 ceph 3.6 M 2023-09-23T20:31:30.544 INFO:teuthology.orchestra.run.smithi026.stdout:Installing dependencies: 2023-09-23T20:31:30.544 INFO:teuthology.orchestra.run.smithi026.stdout: ceph-base x86_64 2:17.2.6-1123.g61431f6b.el8 ceph 5.2 M 2023-09-23T20:31:30.544 INFO:teuthology.orchestra.run.smithi026.stdout: ceph-common x86_64 2:17.2.6-1123.g61431f6b.el8 ceph 23 M 2023-09-23T20:31:30.544 INFO:teuthology.orchestra.run.smithi026.stdout: ceph-selinux x86_64 2:17.2.6-1123.g61431f6b.el8 ceph 24 k 2023-09-23T20:31:30.544 INFO:teuthology.orchestra.run.smithi026.stdout: gperftools-libs x86_64 1:2.7-9.el8 epel 306 k 2023-09-23T20:31:30.544 INFO:teuthology.orchestra.run.smithi026.stdout: libcephfs2 x86_64 2:17.2.6-1123.g61431f6b.el8 ceph 796 k 2023-09-23T20:31:30.544 INFO:teuthology.orchestra.run.smithi026.stdout: liboath x86_64 2.6.2-3.el8 epel 59 k 2023-09-23T20:31:30.544 INFO:teuthology.orchestra.run.smithi026.stdout: librabbitmq x86_64 0.9.0-3.el8 rhel-8-for-x86_64-baseos-rpms 47 k 2023-09-23T20:31:30.545 INFO:teuthology.orchestra.run.smithi026.stdout: libradosstriper1 x86_64 2:17.2.6-1123.g61431f6b.el8 ceph 506 k 2023-09-23T20:31:30.545 INFO:teuthology.orchestra.run.smithi026.stdout: librdkafka x86_64 0.11.4-3.el8 rhel-8-for-x86_64-appstream-rpms 354 k 2023-09-23T20:31:30.545 INFO:teuthology.orchestra.run.smithi026.stdout: librgw2 x86_64 2:17.2.6-1123.g61431f6b.el8 ceph 5.5 M 2023-09-23T20:31:30.545 INFO:teuthology.orchestra.run.smithi026.stdout: libunwind x86_64 1.3.1-3.el8 epel 75 k 2023-09-23T20:31:30.545 INFO:teuthology.orchestra.run.smithi026.stdout: lttng-ust x86_64 2.8.1-11.el8 rhel-8-for-x86_64-appstream-rpms 259 k 2023-09-23T20:31:30.545 INFO:teuthology.orchestra.run.smithi026.stdout: python3-ceph-argparse x86_64 2:17.2.6-1123.g61431f6b.el8 ceph 45 k 2023-09-23T20:31:30.545 INFO:teuthology.orchestra.run.smithi026.stdout: python3-ceph-common x86_64 2:17.2.6-1123.g61431f6b.el8 ceph 115 k 2023-09-23T20:31:30.545 INFO:teuthology.orchestra.run.smithi026.stdout: python3-cephfs x86_64 2:17.2.6-1123.g61431f6b.el8 ceph 190 k 2023-09-23T20:31:30.545 INFO:teuthology.orchestra.run.smithi026.stdout: python3-prettytable noarch 0.7.2-14.el8 rhel-8-for-x86_64-appstream-rpms 44 k 2023-09-23T20:31:30.545 INFO:teuthology.orchestra.run.smithi026.stdout: python3-rados x86_64 2:17.2.6-1123.g61431f6b.el8 ceph 371 k 2023-09-23T20:31:30.545 INFO:teuthology.orchestra.run.smithi026.stdout: python3-rbd x86_64 2:17.2.6-1123.g61431f6b.el8 ceph 358 k 2023-09-23T20:31:30.545 INFO:teuthology.orchestra.run.smithi026.stdout: python3-rgw x86_64 2:17.2.6-1123.g61431f6b.el8 ceph 106 k 2023-09-23T20:31:30.546 INFO:teuthology.orchestra.run.smithi026.stdout: thrift x86_64 0.13.0-2.el8 epel 1.7 M 2023-09-23T20:31:30.546 INFO:teuthology.orchestra.run.smithi026.stdout: 2023-09-23T20:31:30.546 INFO:teuthology.orchestra.run.smithi026.stdout:Transaction Summary 2023-09-23T20:31:30.546 INFO:teuthology.orchestra.run.smithi026.stdout:==================================================================================================== 2023-09-23T20:31:30.546 INFO:teuthology.orchestra.run.smithi026.stdout:Install 21 Packages 2023-09-23T20:31:30.546 INFO:teuthology.orchestra.run.smithi026.stdout:Upgrade 2 Packages 2023-09-23T20:31:30.546 INFO:teuthology.orchestra.run.smithi026.stdout: 2023-09-23T20:31:30.546 INFO:teuthology.orchestra.run.smithi026.stdout:Total download size: 63 M 2023-09-23T20:31:30.547 INFO:teuthology.orchestra.run.smithi026.stdout:Downloading Packages: 2023-09-23T20:31:30.587 INFO:teuthology.orchestra.run.smithi070.stdout:(5/23): librgw2-17.2.6-1123.g61431f6b.el8.x86_6 7.2 MB/s | 5.5 MB 00:00 2023-09-23T20:31:30.712 INFO:teuthology.orchestra.run.smithi070.stdout:(6/23): python3-ceph-argparse-17.2.6-1123.g6143 357 kB/s | 45 kB 00:00 2023-09-23T20:31:30.913 INFO:teuthology.orchestra.run.smithi070.stdout:(7/23): ceph-radosgw-17.2.6-1123.g61431f6b.el8. 7.2 MB/s | 17 MB 00:02 2023-09-23T20:31:30.938 INFO:teuthology.orchestra.run.smithi070.stdout:(8/23): python3-ceph-common-17.2.6-1123.g61431f 512 kB/s | 115 kB 00:00 2023-09-23T20:31:31.155 INFO:teuthology.orchestra.run.smithi070.stdout:(9/23): ceph-common-17.2.6-1123.g61431f6b.el8.x 9.1 MB/s | 23 MB 00:02 2023-09-23T20:31:31.180 INFO:teuthology.orchestra.run.smithi070.stdout:(10/23): python3-cephfs-17.2.6-1123.g61431f6b.e 712 kB/s | 190 kB 00:00 2023-09-23T20:31:31.205 INFO:teuthology.orchestra.run.smithi070.stdout:(11/23): python3-rados-17.2.6-1123.g61431f6b.el 1.4 MB/s | 371 kB 00:00 2023-09-23T20:31:31.266 INFO:teuthology.orchestra.run.smithi070.stdout:(12/23): python3-rbd-17.2.6-1123.g61431f6b.el8. 3.2 MB/s | 358 kB 00:00 2023-09-23T20:31:31.301 INFO:teuthology.orchestra.run.smithi070.stdout:(13/23): python3-rgw-17.2.6-1123.g61431f6b.el8. 909 kB/s | 106 kB 00:00 2023-09-23T20:31:31.582 INFO:teuthology.orchestra.run.smithi070.stdout:(14/23): liboath-2.6.2-3.el8.x86_64.rpm 188 kB/s | 59 kB 00:00 2023-09-23T20:31:31.606 INFO:teuthology.orchestra.run.smithi070.stdout:(15/23): libunwind-1.3.1-3.el8.x86_64.rpm 243 kB/s | 75 kB 00:00 2023-09-23T20:31:31.657 INFO:teuthology.orchestra.run.smithi070.stdout:(16/23): librabbitmq-0.9.0-3.el8.x86_64.rpm 938 kB/s | 47 kB 00:00 2023-09-23T20:31:31.682 INFO:teuthology.orchestra.run.smithi070.stdout:(17/23): gperftools-libs-2.7-9.el8.x86_64.rpm 643 kB/s | 306 kB 00:00 2023-09-23T20:31:31.708 INFO:teuthology.orchestra.run.smithi070.stdout:(18/23): lttng-ust-2.8.1-11.el8.x86_64.rpm 5.0 MB/s | 259 kB 00:00 2023-09-23T20:31:31.712 INFO:teuthology.orchestra.run.smithi026.stdout:(1/23): ceph-base-17.2.6-1123.g61431f6b.el8.x86 5.3 MB/s | 5.2 MB 00:00 2023-09-23T20:31:31.749 INFO:teuthology.orchestra.run.smithi070.stdout:(19/23): python3-prettytable-0.7.2-14.el8.noarc 657 kB/s | 44 kB 00:00 2023-09-23T20:31:31.774 INFO:teuthology.orchestra.run.smithi070.stdout:(20/23): librdkafka-0.11.4-3.el8.x86_64.rpm 5.6 MB/s | 354 kB 00:00 2023-09-23T20:31:31.806 INFO:teuthology.orchestra.run.smithi026.stdout:(2/23): ceph-selinux-17.2.6-1123.g61431f6b.el8. 266 kB/s | 24 kB 00:00 2023-09-23T20:31:31.954 INFO:teuthology.orchestra.run.smithi026.stdout:(3/23): libcephfs2-17.2.6-1123.g61431f6b.el8.x8 5.2 MB/s | 796 kB 00:00 2023-09-23T20:31:32.096 INFO:teuthology.orchestra.run.smithi026.stdout:(4/23): libradosstriper1-17.2.6-1123.g61431f6b. 3.5 MB/s | 506 kB 00:00 2023-09-23T20:31:32.483 INFO:teuthology.orchestra.run.smithi070.stdout:(21/23): librbd1-17.2.6-1123.g61431f6b.el8.x86_ 5.1 MB/s | 3.6 MB 00:00 2023-09-23T20:31:32.905 INFO:teuthology.orchestra.run.smithi026.stdout:(5/23): librgw2-17.2.6-1123.g61431f6b.el8.x86_6 6.9 MB/s | 5.5 MB 00:00 2023-09-23T20:31:32.924 INFO:teuthology.orchestra.run.smithi070.stdout:(22/23): thrift-0.13.0-2.el8.x86_64.rpm 1.3 MB/s | 1.7 MB 00:01 2023-09-23T20:31:32.971 INFO:teuthology.orchestra.run.smithi026.stdout:(6/23): python3-ceph-argparse-17.2.6-1123.g6143 671 kB/s | 45 kB 00:00 2023-09-23T20:31:33.033 INFO:teuthology.orchestra.run.smithi070.stdout:(23/23): librados2-17.2.6-1123.g61431f6b.el8.x8 2.7 MB/s | 3.5 MB 00:01 2023-09-23T20:31:33.038 INFO:teuthology.orchestra.run.smithi070.stdout:-------------------------------------------------------------------------------- 2023-09-23T20:31:33.038 INFO:teuthology.orchestra.run.smithi070.stdout:Total 14 MB/s | 63 MB 00:04 2023-09-23T20:31:33.088 INFO:teuthology.orchestra.run.smithi070.stdout:Running transaction check 2023-09-23T20:31:33.133 INFO:teuthology.orchestra.run.smithi070.stdout:Transaction check succeeded. 2023-09-23T20:31:33.133 INFO:teuthology.orchestra.run.smithi070.stdout:Running transaction test 2023-09-23T20:31:33.155 INFO:teuthology.orchestra.run.smithi026.stdout:(7/23): ceph-radosgw-17.2.6-1123.g61431f6b.el8. 6.9 MB/s | 17 MB 00:02 2023-09-23T20:31:33.180 INFO:teuthology.orchestra.run.smithi026.stdout:(8/23): python3-ceph-common-17.2.6-1123.g61431f 553 kB/s | 115 kB 00:00 2023-09-23T20:31:33.231 INFO:teuthology.orchestra.run.smithi026.stdout:(9/23): python3-cephfs-17.2.6-1123.g61431f6b.el 2.5 MB/s | 190 kB 00:00 2023-09-23T20:31:33.264 INFO:teuthology.orchestra.run.smithi026.stdout:(10/23): python3-rados-17.2.6-1123.g61431f6b.el 4.3 MB/s | 371 kB 00:00 2023-09-23T20:31:33.306 INFO:teuthology.orchestra.run.smithi026.stdout:(11/23): python3-rbd-17.2.6-1123.g61431f6b.el8. 4.7 MB/s | 358 kB 00:00 2023-09-23T20:31:33.348 INFO:teuthology.orchestra.run.smithi026.stdout:(12/23): python3-rgw-17.2.6-1123.g61431f6b.el8. 1.2 MB/s | 106 kB 00:00 2023-09-23T20:31:33.483 INFO:teuthology.orchestra.run.smithi070.stdout:Transaction test succeeded. 2023-09-23T20:31:33.487 INFO:teuthology.orchestra.run.smithi070.stdout:Running transaction 2023-09-23T20:31:33.490 INFO:teuthology.orchestra.run.smithi026.stdout:(13/23): liboath-2.6.2-3.el8.x86_64.rpm 415 kB/s | 59 kB 00:00 2023-09-23T20:31:33.565 INFO:teuthology.orchestra.run.smithi026.stdout:(14/23): libunwind-1.3.1-3.el8.x86_64.rpm 1.0 MB/s | 75 kB 00:00 2023-09-23T20:31:33.591 INFO:teuthology.orchestra.run.smithi026.stdout:(15/23): gperftools-libs-2.7-9.el8.x86_64.rpm 1.1 MB/s | 306 kB 00:00 2023-09-23T20:31:33.649 INFO:teuthology.orchestra.run.smithi026.stdout:(16/23): librabbitmq-0.9.0-3.el8.x86_64.rpm 806 kB/s | 47 kB 00:00 2023-09-23T20:31:33.891 INFO:teuthology.orchestra.run.smithi026.stdout:(17/23): ceph-common-17.2.6-1123.g61431f6b.el8. 7.4 MB/s | 23 MB 00:03 2023-09-23T20:31:33.934 INFO:teuthology.orchestra.run.smithi026.stdout:(18/23): lttng-ust-2.8.1-11.el8.x86_64.rpm 913 kB/s | 259 kB 00:00 2023-09-23T20:31:33.984 INFO:teuthology.orchestra.run.smithi026.stdout:(19/23): python3-prettytable-0.7.2-14.el8.noarc 479 kB/s | 44 kB 00:00 2023-09-23T20:31:34.017 INFO:teuthology.orchestra.run.smithi026.stdout:(20/23): librdkafka-0.11.4-3.el8.x86_64.rpm 4.6 MB/s | 354 kB 00:00 2023-09-23T20:31:34.343 INFO:teuthology.orchestra.run.smithi026.stdout:(21/23): librados2-17.2.6-1123.g61431f6b.el8.x8 9.7 MB/s | 3.5 MB 00:00 2023-09-23T20:31:34.410 INFO:teuthology.orchestra.run.smithi026.stdout:(22/23): librbd1-17.2.6-1123.g61431f6b.el8.x86_ 9.1 MB/s | 3.6 MB 00:00 2023-09-23T20:31:34.535 INFO:teuthology.orchestra.run.smithi026.stdout:(23/23): thrift-0.13.0-2.el8.x86_64.rpm 1.8 MB/s | 1.7 MB 00:00 2023-09-23T20:31:34.537 INFO:teuthology.orchestra.run.smithi026.stdout:-------------------------------------------------------------------------------- 2023-09-23T20:31:34.537 INFO:teuthology.orchestra.run.smithi026.stdout:Total 16 MB/s | 63 MB 00:03 2023-09-23T20:31:34.553 INFO:teuthology.orchestra.run.smithi070.stdout: Preparing : 1/1 2023-09-23T20:31:34.582 INFO:teuthology.orchestra.run.smithi026.stdout:Running transaction check 2023-09-23T20:31:34.630 INFO:teuthology.orchestra.run.smithi026.stdout:Transaction check succeeded. 2023-09-23T20:31:34.631 INFO:teuthology.orchestra.run.smithi026.stdout:Running transaction test 2023-09-23T20:31:34.740 INFO:teuthology.orchestra.run.smithi070.stdout: Installing : lttng-ust-2.8.1-11.el8.x86_64 1/25 2023-09-23T20:31:34.998 INFO:teuthology.orchestra.run.smithi026.stdout:Transaction test succeeded. 2023-09-23T20:31:35.002 INFO:teuthology.orchestra.run.smithi026.stdout:Running transaction 2023-09-23T20:31:35.122 INFO:teuthology.orchestra.run.smithi070.stdout: Running scriptlet: lttng-ust-2.8.1-11.el8.x86_64 1/25 2023-09-23T20:31:35.298 INFO:teuthology.orchestra.run.smithi070.stdout: Upgrading : librados2-2:17.2.6-1123.g61431f6b.el8.x86_64 2/25 2023-09-23T20:31:35.400 INFO:teuthology.orchestra.run.smithi070.stdout: Running scriptlet: librados2-2:17.2.6-1123.g61431f6b.el8.x86_64 2/25 2023-09-23T20:31:35.596 INFO:teuthology.orchestra.run.smithi070.stdout: Installing : python3-rados-2:17.2.6-1123.g61431f6b.el8.x86_64 3/25 2023-09-23T20:31:35.872 INFO:teuthology.orchestra.run.smithi070.stdout: Installing : liboath-2.6.2-3.el8.x86_64 4/25 2023-09-23T20:31:36.056 INFO:teuthology.orchestra.run.smithi070.stdout: Installing : libcephfs2-2:17.2.6-1123.g61431f6b.el8.x86_64 5/25 2023-09-23T20:31:36.074 INFO:teuthology.orchestra.run.smithi026.stdout: Preparing : 1/1 2023-09-23T20:31:36.295 INFO:teuthology.orchestra.run.smithi026.stdout: Installing : lttng-ust-2.8.1-11.el8.x86_64 1/25 2023-09-23T20:31:36.454 INFO:teuthology.orchestra.run.smithi070.stdout: Running scriptlet: libcephfs2-2:17.2.6-1123.g61431f6b.el8.x86_64 5/25 2023-09-23T20:31:36.686 INFO:teuthology.orchestra.run.smithi026.stdout: Running scriptlet: lttng-ust-2.8.1-11.el8.x86_64 1/25 2023-09-23T20:31:36.698 INFO:teuthology.orchestra.run.smithi070.stdout: Upgrading : librbd1-2:17.2.6-1123.g61431f6b.el8.x86_64 6/25 2023-09-23T20:31:36.797 INFO:teuthology.orchestra.run.smithi070.stdout: Running scriptlet: librbd1-2:17.2.6-1123.g61431f6b.el8.x86_64 6/25 2023-09-23T20:31:36.870 INFO:teuthology.orchestra.run.smithi026.stdout: Upgrading : librados2-2:17.2.6-1123.g61431f6b.el8.x86_64 2/25 2023-09-23T20:31:36.954 INFO:teuthology.orchestra.run.smithi026.stdout: Running scriptlet: librados2-2:17.2.6-1123.g61431f6b.el8.x86_64 2/25 2023-09-23T20:31:36.982 INFO:teuthology.orchestra.run.smithi070.stdout: Installing : librdkafka-0.11.4-3.el8.x86_64 7/25 2023-09-23T20:31:37.048 INFO:teuthology.orchestra.run.smithi070.stdout: Running scriptlet: librdkafka-0.11.4-3.el8.x86_64 7/25 2023-09-23T20:31:37.168 INFO:teuthology.orchestra.run.smithi026.stdout: Installing : python3-rados-2:17.2.6-1123.g61431f6b.el8.x86_64 3/25 2023-09-23T20:31:37.363 INFO:teuthology.orchestra.run.smithi070.stdout: Installing : librabbitmq-0.9.0-3.el8.x86_64 8/25 2023-09-23T20:31:37.408 INFO:teuthology.orchestra.run.smithi026.stdout: Installing : liboath-2.6.2-3.el8.x86_64 4/25 2023-09-23T20:31:37.754 INFO:teuthology.orchestra.run.smithi026.stdout: Installing : libcephfs2-2:17.2.6-1123.g61431f6b.el8.x86_64 5/25 2023-09-23T20:31:38.023 INFO:teuthology.orchestra.run.smithi070.stdout: Installing : thrift-0.13.0-2.el8.x86_64 9/25 2023-09-23T20:31:38.177 INFO:teuthology.orchestra.run.smithi026.stdout: Running scriptlet: libcephfs2-2:17.2.6-1123.g61431f6b.el8.x86_64 5/25 2023-09-23T20:31:38.206 INFO:teuthology.orchestra.run.smithi070.stdout: Installing : librgw2-2:17.2.6-1123.g61431f6b.el8.x86_64 10/25 2023-09-23T20:31:38.269 INFO:teuthology.orchestra.run.smithi070.stdout: Running scriptlet: librgw2-2:17.2.6-1123.g61431f6b.el8.x86_64 10/25 2023-09-23T20:31:38.378 INFO:teuthology.orchestra.run.smithi026.stdout: Upgrading : librbd1-2:17.2.6-1123.g61431f6b.el8.x86_64 6/25 2023-09-23T20:31:38.400 INFO:teuthology.orchestra.run.smithi070.stdout: Installing : python3-ceph-argparse-2:17.2.6-1123.g61431f6b.el8. 11/25 2023-09-23T20:31:38.486 INFO:teuthology.orchestra.run.smithi026.stdout: Running scriptlet: librbd1-2:17.2.6-1123.g61431f6b.el8.x86_64 6/25 2023-09-23T20:31:38.558 INFO:teuthology.orchestra.run.smithi070.stdout: Installing : python3-cephfs-2:17.2.6-1123.g61431f6b.el8.x86_64 12/25 2023-09-23T20:31:38.703 INFO:teuthology.orchestra.run.smithi026.stdout: Installing : librdkafka-0.11.4-3.el8.x86_64 7/25 2023-09-23T20:31:38.758 INFO:teuthology.orchestra.run.smithi070.stdout: Installing : python3-rgw-2:17.2.6-1123.g61431f6b.el8.x86_64 13/25 2023-09-23T20:31:38.759 INFO:teuthology.orchestra.run.smithi026.stdout: Running scriptlet: librdkafka-0.11.4-3.el8.x86_64 7/25 2023-09-23T20:31:38.943 INFO:teuthology.orchestra.run.smithi070.stdout: Installing : python3-rbd-2:17.2.6-1123.g61431f6b.el8.x86_64 14/25 2023-09-23T20:31:39.074 INFO:teuthology.orchestra.run.smithi026.stdout: Installing : librabbitmq-0.9.0-3.el8.x86_64 8/25 2023-09-23T20:31:39.131 INFO:teuthology.orchestra.run.smithi070.stdout: Installing : libradosstriper1-2:17.2.6-1123.g61431f6b.el8.x86_6 15/25 2023-09-23T20:31:39.199 INFO:teuthology.orchestra.run.smithi070.stdout: Running scriptlet: libradosstriper1-2:17.2.6-1123.g61431f6b.el8.x86_6 15/25 2023-09-23T20:31:39.340 INFO:teuthology.orchestra.run.smithi070.stdout: Installing : python3-prettytable-0.7.2-14.el8.noarch 16/25 2023-09-23T20:31:39.505 INFO:teuthology.orchestra.run.smithi070.stdout: Installing : libunwind-1.3.1-3.el8.x86_64 17/25 2023-09-23T20:31:39.730 INFO:teuthology.orchestra.run.smithi070.stdout: Installing : gperftools-libs-1:2.7-9.el8.x86_64 18/25 2023-09-23T20:31:39.765 INFO:teuthology.orchestra.run.smithi026.stdout: Installing : thrift-0.13.0-2.el8.x86_64 9/25 2023-09-23T20:31:39.903 INFO:teuthology.orchestra.run.smithi026.stdout: Installing : librgw2-2:17.2.6-1123.g61431f6b.el8.x86_64 10/25 2023-09-23T20:31:39.914 INFO:teuthology.orchestra.run.smithi070.stdout: Installing : python3-ceph-common-2:17.2.6-1123.g61431f6b.el8.x8 19/25 2023-09-23T20:31:39.972 INFO:teuthology.orchestra.run.smithi026.stdout: Running scriptlet: librgw2-2:17.2.6-1123.g61431f6b.el8.x86_64 10/25 2023-09-23T20:31:40.140 INFO:teuthology.orchestra.run.smithi026.stdout: Installing : python3-ceph-argparse-2:17.2.6-1123.g61431f6b.el8. 11/25 2023-09-23T20:31:40.306 INFO:teuthology.orchestra.run.smithi026.stdout: Installing : python3-cephfs-2:17.2.6-1123.g61431f6b.el8.x86_64 12/25 2023-09-23T20:31:40.515 INFO:teuthology.orchestra.run.smithi026.stdout: Installing : python3-rgw-2:17.2.6-1123.g61431f6b.el8.x86_64 13/25 2023-09-23T20:31:40.709 INFO:teuthology.orchestra.run.smithi026.stdout: Installing : python3-rbd-2:17.2.6-1123.g61431f6b.el8.x86_64 14/25 2023-09-23T20:31:40.886 INFO:teuthology.orchestra.run.smithi026.stdout: Installing : libradosstriper1-2:17.2.6-1123.g61431f6b.el8.x86_6 15/25 2023-09-23T20:31:40.952 INFO:teuthology.orchestra.run.smithi026.stdout: Running scriptlet: libradosstriper1-2:17.2.6-1123.g61431f6b.el8.x86_6 15/25 2023-09-23T20:31:41.096 INFO:teuthology.orchestra.run.smithi026.stdout: Installing : python3-prettytable-0.7.2-14.el8.noarch 16/25 2023-09-23T20:31:41.281 INFO:teuthology.orchestra.run.smithi026.stdout: Installing : libunwind-1.3.1-3.el8.x86_64 17/25 2023-09-23T20:31:41.525 INFO:teuthology.orchestra.run.smithi026.stdout: Installing : gperftools-libs-1:2.7-9.el8.x86_64 18/25 2023-09-23T20:31:41.712 INFO:teuthology.orchestra.run.smithi026.stdout: Installing : python3-ceph-common-2:17.2.6-1123.g61431f6b.el8.x8 19/25 2023-09-23T20:31:43.700 INFO:teuthology.orchestra.run.smithi070.stdout: Running scriptlet: ceph-common-2:17.2.6-1123.g61431f6b.el8.x86_64 20/25 2023-09-23T20:31:43.913 INFO:teuthology.orchestra.run.smithi070.stdout: Installing : ceph-common-2:17.2.6-1123.g61431f6b.el8.x86_64 20/25 2023-09-23T20:31:44.484 INFO:teuthology.orchestra.run.smithi070.stdout: Running scriptlet: ceph-common-2:17.2.6-1123.g61431f6b.el8.x86_64 20/25 2023-09-23T20:31:44.714 INFO:teuthology.orchestra.run.smithi070.stdout: Installing : ceph-base-2:17.2.6-1123.g61431f6b.el8.x86_64 21/25 2023-09-23T20:31:44.780 INFO:teuthology.orchestra.run.smithi070.stdout: Running scriptlet: ceph-base-2:17.2.6-1123.g61431f6b.el8.x86_64 21/25 2023-09-23T20:31:44.946 INFO:teuthology.orchestra.run.smithi070.stdout: Installing : ceph-selinux-2:17.2.6-1123.g61431f6b.el8.x86_64 22/25 2023-09-23T20:31:45.841 INFO:teuthology.orchestra.run.smithi026.stdout: Running scriptlet: ceph-common-2:17.2.6-1123.g61431f6b.el8.x86_64 20/25 2023-09-23T20:31:46.061 INFO:teuthology.orchestra.run.smithi026.stdout: Installing : ceph-common-2:17.2.6-1123.g61431f6b.el8.x86_64 20/25 2023-09-23T20:31:46.674 INFO:teuthology.orchestra.run.smithi026.stdout: Running scriptlet: ceph-common-2:17.2.6-1123.g61431f6b.el8.x86_64 20/25 2023-09-23T20:31:46.894 INFO:teuthology.orchestra.run.smithi026.stdout: Installing : ceph-base-2:17.2.6-1123.g61431f6b.el8.x86_64 21/25 2023-09-23T20:31:46.966 INFO:teuthology.orchestra.run.smithi026.stdout: Running scriptlet: ceph-base-2:17.2.6-1123.g61431f6b.el8.x86_64 21/25 2023-09-23T20:31:47.143 INFO:teuthology.orchestra.run.smithi026.stdout: Installing : ceph-selinux-2:17.2.6-1123.g61431f6b.el8.x86_64 22/25 2023-09-23T20:32:03.260 INFO:teuthology.orchestra.run.smithi070.stdout: Running scriptlet: ceph-selinux-2:17.2.6-1123.g61431f6b.el8.x86_64 22/25 2023-09-23T20:32:03.260 INFO:teuthology.orchestra.run.smithi070.stdout:skipping the directory /sys 2023-09-23T20:32:03.260 INFO:teuthology.orchestra.run.smithi070.stdout:skipping the directory /proc 2023-09-23T20:32:03.261 INFO:teuthology.orchestra.run.smithi070.stdout:skipping the directory /dev 2023-09-23T20:32:03.261 INFO:teuthology.orchestra.run.smithi070.stdout:skipping the directory /run 2023-09-23T20:32:03.261 INFO:teuthology.orchestra.run.smithi070.stdout:skipping the directory /mnt 2023-09-23T20:32:03.261 INFO:teuthology.orchestra.run.smithi070.stdout:skipping the directory /var/tmp 2023-09-23T20:32:03.261 INFO:teuthology.orchestra.run.smithi070.stdout:skipping the directory /home 2023-09-23T20:32:03.261 INFO:teuthology.orchestra.run.smithi070.stdout:skipping the directory /tmp 2023-09-23T20:32:03.261 INFO:teuthology.orchestra.run.smithi070.stdout:skipping the directory /dev 2023-09-23T20:32:03.261 INFO:teuthology.orchestra.run.smithi070.stdout: 2023-09-23T20:32:04.918 INFO:teuthology.orchestra.run.smithi070.stdout: Installing : ceph-radosgw-2:17.2.6-1123.g61431f6b.el8.x86_64 23/25 2023-09-23T20:32:05.004 INFO:teuthology.orchestra.run.smithi070.stdout: Running scriptlet: ceph-radosgw-2:17.2.6-1123.g61431f6b.el8.x86_64 23/25 2023-09-23T20:32:05.004 INFO:teuthology.orchestra.run.smithi070.stdout: Cleanup : librbd1-1:12.2.7-9.el8.x86_64 24/25 2023-09-23T20:32:05.194 INFO:teuthology.orchestra.run.smithi070.stdout: Running scriptlet: librbd1-1:12.2.7-9.el8.x86_64 24/25 2023-09-23T20:32:05.194 INFO:teuthology.orchestra.run.smithi070.stdout: Cleanup : librados2-1:12.2.7-9.el8.x86_64 25/25 2023-09-23T20:32:05.827 INFO:teuthology.orchestra.run.smithi070.stdout: Running scriptlet: librados2-1:12.2.7-9.el8.x86_64 25/25 2023-09-23T20:32:05.828 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : ceph-base-2:17.2.6-1123.g61431f6b.el8.x86_64 1/25 2023-09-23T20:32:05.828 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : ceph-common-2:17.2.6-1123.g61431f6b.el8.x86_64 2/25 2023-09-23T20:32:05.828 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : ceph-radosgw-2:17.2.6-1123.g61431f6b.el8.x86_64 3/25 2023-09-23T20:32:05.828 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : ceph-selinux-2:17.2.6-1123.g61431f6b.el8.x86_64 4/25 2023-09-23T20:32:05.828 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : libcephfs2-2:17.2.6-1123.g61431f6b.el8.x86_64 5/25 2023-09-23T20:32:05.828 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : libradosstriper1-2:17.2.6-1123.g61431f6b.el8.x86_6 6/25 2023-09-23T20:32:05.828 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : librgw2-2:17.2.6-1123.g61431f6b.el8.x86_64 7/25 2023-09-23T20:32:05.828 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : python3-ceph-argparse-2:17.2.6-1123.g61431f6b.el8. 8/25 2023-09-23T20:32:05.828 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : python3-ceph-common-2:17.2.6-1123.g61431f6b.el8.x8 9/25 2023-09-23T20:32:05.828 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : python3-cephfs-2:17.2.6-1123.g61431f6b.el8.x86_64 10/25 2023-09-23T20:32:05.829 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : python3-rados-2:17.2.6-1123.g61431f6b.el8.x86_64 11/25 2023-09-23T20:32:05.829 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : python3-rbd-2:17.2.6-1123.g61431f6b.el8.x86_64 12/25 2023-09-23T20:32:05.829 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : python3-rgw-2:17.2.6-1123.g61431f6b.el8.x86_64 13/25 2023-09-23T20:32:05.829 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : gperftools-libs-1:2.7-9.el8.x86_64 14/25 2023-09-23T20:32:05.829 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : liboath-2.6.2-3.el8.x86_64 15/25 2023-09-23T20:32:05.829 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : libunwind-1.3.1-3.el8.x86_64 16/25 2023-09-23T20:32:05.829 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : thrift-0.13.0-2.el8.x86_64 17/25 2023-09-23T20:32:05.829 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : librabbitmq-0.9.0-3.el8.x86_64 18/25 2023-09-23T20:32:05.829 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : lttng-ust-2.8.1-11.el8.x86_64 19/25 2023-09-23T20:32:05.829 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : python3-prettytable-0.7.2-14.el8.noarch 20/25 2023-09-23T20:32:05.829 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : librdkafka-0.11.4-3.el8.x86_64 21/25 2023-09-23T20:32:05.829 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : librados2-2:17.2.6-1123.g61431f6b.el8.x86_64 22/25 2023-09-23T20:32:05.829 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : librados2-1:12.2.7-9.el8.x86_64 23/25 2023-09-23T20:32:05.830 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : librbd1-2:17.2.6-1123.g61431f6b.el8.x86_64 24/25 2023-09-23T20:32:06.710 INFO:teuthology.orchestra.run.smithi026.stdout: Running scriptlet: ceph-selinux-2:17.2.6-1123.g61431f6b.el8.x86_64 22/25 2023-09-23T20:32:06.710 INFO:teuthology.orchestra.run.smithi026.stdout:skipping the directory /sys 2023-09-23T20:32:06.710 INFO:teuthology.orchestra.run.smithi026.stdout:skipping the directory /proc 2023-09-23T20:32:06.710 INFO:teuthology.orchestra.run.smithi026.stdout:skipping the directory /dev 2023-09-23T20:32:06.710 INFO:teuthology.orchestra.run.smithi026.stdout:skipping the directory /run 2023-09-23T20:32:06.710 INFO:teuthology.orchestra.run.smithi026.stdout:skipping the directory /mnt 2023-09-23T20:32:06.710 INFO:teuthology.orchestra.run.smithi026.stdout:skipping the directory /var/tmp 2023-09-23T20:32:06.711 INFO:teuthology.orchestra.run.smithi026.stdout:skipping the directory /home 2023-09-23T20:32:06.711 INFO:teuthology.orchestra.run.smithi026.stdout:skipping the directory /tmp 2023-09-23T20:32:06.711 INFO:teuthology.orchestra.run.smithi026.stdout:skipping the directory /dev 2023-09-23T20:32:06.711 INFO:teuthology.orchestra.run.smithi026.stdout: 2023-09-23T20:32:07.165 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : librbd1-1:12.2.7-9.el8.x86_64 25/25 2023-09-23T20:32:07.165 INFO:teuthology.orchestra.run.smithi070.stdout:Last metadata expiration check: 0:00:51 ago on Sat 23 Sep 2023 08:31:16 PM UTC. 2023-09-23T20:32:08.491 INFO:teuthology.orchestra.run.smithi026.stdout: Installing : ceph-radosgw-2:17.2.6-1123.g61431f6b.el8.x86_64 23/25 2023-09-23T20:32:08.575 INFO:teuthology.orchestra.run.smithi026.stdout: Running scriptlet: ceph-radosgw-2:17.2.6-1123.g61431f6b.el8.x86_64 23/25 2023-09-23T20:32:08.575 INFO:teuthology.orchestra.run.smithi026.stdout: Cleanup : librbd1-1:12.2.7-9.el8.x86_64 24/25 2023-09-23T20:32:08.884 INFO:teuthology.orchestra.run.smithi026.stdout: Running scriptlet: librbd1-1:12.2.7-9.el8.x86_64 24/25 2023-09-23T20:32:08.884 INFO:teuthology.orchestra.run.smithi026.stdout: Cleanup : librados2-1:12.2.7-9.el8.x86_64 25/25 2023-09-23T20:32:09.524 INFO:teuthology.orchestra.run.smithi026.stdout: Running scriptlet: librados2-1:12.2.7-9.el8.x86_64 25/25 2023-09-23T20:32:09.524 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : ceph-base-2:17.2.6-1123.g61431f6b.el8.x86_64 1/25 2023-09-23T20:32:09.524 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : ceph-common-2:17.2.6-1123.g61431f6b.el8.x86_64 2/25 2023-09-23T20:32:09.524 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : ceph-radosgw-2:17.2.6-1123.g61431f6b.el8.x86_64 3/25 2023-09-23T20:32:09.524 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : ceph-selinux-2:17.2.6-1123.g61431f6b.el8.x86_64 4/25 2023-09-23T20:32:09.524 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : libcephfs2-2:17.2.6-1123.g61431f6b.el8.x86_64 5/25 2023-09-23T20:32:09.525 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : libradosstriper1-2:17.2.6-1123.g61431f6b.el8.x86_6 6/25 2023-09-23T20:32:09.525 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : librgw2-2:17.2.6-1123.g61431f6b.el8.x86_64 7/25 2023-09-23T20:32:09.525 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : python3-ceph-argparse-2:17.2.6-1123.g61431f6b.el8. 8/25 2023-09-23T20:32:09.525 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : python3-ceph-common-2:17.2.6-1123.g61431f6b.el8.x8 9/25 2023-09-23T20:32:09.525 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : python3-cephfs-2:17.2.6-1123.g61431f6b.el8.x86_64 10/25 2023-09-23T20:32:09.525 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : python3-rados-2:17.2.6-1123.g61431f6b.el8.x86_64 11/25 2023-09-23T20:32:09.525 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : python3-rbd-2:17.2.6-1123.g61431f6b.el8.x86_64 12/25 2023-09-23T20:32:09.525 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : python3-rgw-2:17.2.6-1123.g61431f6b.el8.x86_64 13/25 2023-09-23T20:32:09.525 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : gperftools-libs-1:2.7-9.el8.x86_64 14/25 2023-09-23T20:32:09.525 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : liboath-2.6.2-3.el8.x86_64 15/25 2023-09-23T20:32:09.525 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : libunwind-1.3.1-3.el8.x86_64 16/25 2023-09-23T20:32:09.525 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : thrift-0.13.0-2.el8.x86_64 17/25 2023-09-23T20:32:09.526 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : librabbitmq-0.9.0-3.el8.x86_64 18/25 2023-09-23T20:32:09.526 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : lttng-ust-2.8.1-11.el8.x86_64 19/25 2023-09-23T20:32:09.526 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : python3-prettytable-0.7.2-14.el8.noarch 20/25 2023-09-23T20:32:09.526 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : librdkafka-0.11.4-3.el8.x86_64 21/25 2023-09-23T20:32:09.526 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : librados2-2:17.2.6-1123.g61431f6b.el8.x86_64 22/25 2023-09-23T20:32:09.526 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : librados2-1:12.2.7-9.el8.x86_64 23/25 2023-09-23T20:32:09.526 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : librbd1-2:17.2.6-1123.g61431f6b.el8.x86_64 24/25 2023-09-23T20:32:09.604 INFO:teuthology.orchestra.run.smithi070.stdout:Installed products updated. 2023-09-23T20:32:09.793 INFO:teuthology.orchestra.run.smithi070.stdout: 2023-09-23T20:32:09.793 INFO:teuthology.orchestra.run.smithi070.stdout:Upgraded: 2023-09-23T20:32:09.793 INFO:teuthology.orchestra.run.smithi070.stdout: librados2-2:17.2.6-1123.g61431f6b.el8.x86_64 2023-09-23T20:32:09.793 INFO:teuthology.orchestra.run.smithi070.stdout: librbd1-2:17.2.6-1123.g61431f6b.el8.x86_64 2023-09-23T20:32:09.793 INFO:teuthology.orchestra.run.smithi070.stdout:Installed: 2023-09-23T20:32:09.793 INFO:teuthology.orchestra.run.smithi070.stdout: ceph-base-2:17.2.6-1123.g61431f6b.el8.x86_64 2023-09-23T20:32:09.793 INFO:teuthology.orchestra.run.smithi070.stdout: ceph-common-2:17.2.6-1123.g61431f6b.el8.x86_64 2023-09-23T20:32:09.794 INFO:teuthology.orchestra.run.smithi070.stdout: ceph-radosgw-2:17.2.6-1123.g61431f6b.el8.x86_64 2023-09-23T20:32:09.794 INFO:teuthology.orchestra.run.smithi070.stdout: ceph-selinux-2:17.2.6-1123.g61431f6b.el8.x86_64 2023-09-23T20:32:09.794 INFO:teuthology.orchestra.run.smithi070.stdout: gperftools-libs-1:2.7-9.el8.x86_64 2023-09-23T20:32:09.794 INFO:teuthology.orchestra.run.smithi070.stdout: libcephfs2-2:17.2.6-1123.g61431f6b.el8.x86_64 2023-09-23T20:32:09.794 INFO:teuthology.orchestra.run.smithi070.stdout: liboath-2.6.2-3.el8.x86_64 2023-09-23T20:32:09.794 INFO:teuthology.orchestra.run.smithi070.stdout: librabbitmq-0.9.0-3.el8.x86_64 2023-09-23T20:32:09.794 INFO:teuthology.orchestra.run.smithi070.stdout: libradosstriper1-2:17.2.6-1123.g61431f6b.el8.x86_64 2023-09-23T20:32:09.794 INFO:teuthology.orchestra.run.smithi070.stdout: librdkafka-0.11.4-3.el8.x86_64 2023-09-23T20:32:09.794 INFO:teuthology.orchestra.run.smithi070.stdout: librgw2-2:17.2.6-1123.g61431f6b.el8.x86_64 2023-09-23T20:32:09.794 INFO:teuthology.orchestra.run.smithi070.stdout: libunwind-1.3.1-3.el8.x86_64 2023-09-23T20:32:09.794 INFO:teuthology.orchestra.run.smithi070.stdout: lttng-ust-2.8.1-11.el8.x86_64 2023-09-23T20:32:09.795 INFO:teuthology.orchestra.run.smithi070.stdout: python3-ceph-argparse-2:17.2.6-1123.g61431f6b.el8.x86_64 2023-09-23T20:32:09.795 INFO:teuthology.orchestra.run.smithi070.stdout: python3-ceph-common-2:17.2.6-1123.g61431f6b.el8.x86_64 2023-09-23T20:32:09.795 INFO:teuthology.orchestra.run.smithi070.stdout: python3-cephfs-2:17.2.6-1123.g61431f6b.el8.x86_64 2023-09-23T20:32:09.795 INFO:teuthology.orchestra.run.smithi070.stdout: python3-prettytable-0.7.2-14.el8.noarch 2023-09-23T20:32:09.795 INFO:teuthology.orchestra.run.smithi070.stdout: python3-rados-2:17.2.6-1123.g61431f6b.el8.x86_64 2023-09-23T20:32:09.795 INFO:teuthology.orchestra.run.smithi070.stdout: python3-rbd-2:17.2.6-1123.g61431f6b.el8.x86_64 2023-09-23T20:32:09.795 INFO:teuthology.orchestra.run.smithi070.stdout: python3-rgw-2:17.2.6-1123.g61431f6b.el8.x86_64 2023-09-23T20:32:09.795 INFO:teuthology.orchestra.run.smithi070.stdout: thrift-0.13.0-2.el8.x86_64 2023-09-23T20:32:09.795 INFO:teuthology.orchestra.run.smithi070.stdout: 2023-09-23T20:32:09.795 INFO:teuthology.orchestra.run.smithi070.stdout:Complete! 2023-09-23T20:32:09.963 DEBUG:teuthology.orchestra.run.smithi070:> sudo yum -y install ceph-test 2023-09-23T20:32:10.310 INFO:teuthology.orchestra.run.smithi070.stdout:Updating Subscription Management repositories. 2023-09-23T20:32:10.889 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : librbd1-1:12.2.7-9.el8.x86_64 25/25 2023-09-23T20:32:10.889 INFO:teuthology.orchestra.run.smithi026.stdout:Last metadata expiration check: 0:00:52 ago on Sat 23 Sep 2023 08:31:18 PM UTC. 2023-09-23T20:32:11.133 INFO:teuthology.orchestra.run.smithi070.stdout:Last metadata expiration check: 0:00:55 ago on Sat 23 Sep 2023 08:31:16 PM UTC. 2023-09-23T20:32:12.503 INFO:teuthology.orchestra.run.smithi070.stdout:Dependencies resolved. 2023-09-23T20:32:12.504 INFO:teuthology.orchestra.run.smithi070.stdout:============================================================================================ 2023-09-23T20:32:12.505 INFO:teuthology.orchestra.run.smithi070.stdout: Package Arch Version Repository Size 2023-09-23T20:32:12.505 INFO:teuthology.orchestra.run.smithi070.stdout:============================================================================================ 2023-09-23T20:32:12.505 INFO:teuthology.orchestra.run.smithi070.stdout:Installing: 2023-09-23T20:32:12.505 INFO:teuthology.orchestra.run.smithi070.stdout: ceph-test x86_64 2:17.2.6-1123.g61431f6b.el8 ceph 52 M 2023-09-23T20:32:12.505 INFO:teuthology.orchestra.run.smithi070.stdout:Installing dependencies: 2023-09-23T20:32:12.505 INFO:teuthology.orchestra.run.smithi070.stdout: jq x86_64 1.6-3.el8 rhel-8-for-x86_64-appstream-rpms 202 k 2023-09-23T20:32:12.505 INFO:teuthology.orchestra.run.smithi070.stdout: libcephsqlite x86_64 2:17.2.6-1123.g61431f6b.el8 ceph 174 k 2023-09-23T20:32:12.505 INFO:teuthology.orchestra.run.smithi070.stdout: oniguruma x86_64 6.8.2-2.el8 rhel-8-for-x86_64-appstream-rpms 187 k 2023-09-23T20:32:12.505 INFO:teuthology.orchestra.run.smithi070.stdout: socat x86_64 1.7.4.1-1.el8 rhel-8-for-x86_64-appstream-rpms 323 k 2023-09-23T20:32:12.505 INFO:teuthology.orchestra.run.smithi070.stdout: xmlstarlet x86_64 1.6.1-20.el8 lab-extras 70 k 2023-09-23T20:32:12.505 INFO:teuthology.orchestra.run.smithi070.stdout: 2023-09-23T20:32:12.505 INFO:teuthology.orchestra.run.smithi070.stdout:Transaction Summary 2023-09-23T20:32:12.506 INFO:teuthology.orchestra.run.smithi070.stdout:============================================================================================ 2023-09-23T20:32:12.506 INFO:teuthology.orchestra.run.smithi070.stdout:Install 6 Packages 2023-09-23T20:32:12.506 INFO:teuthology.orchestra.run.smithi070.stdout: 2023-09-23T20:32:12.506 INFO:teuthology.orchestra.run.smithi070.stdout:Total download size: 53 M 2023-09-23T20:32:12.506 INFO:teuthology.orchestra.run.smithi070.stdout:Installed size: 233 M 2023-09-23T20:32:12.506 INFO:teuthology.orchestra.run.smithi070.stdout:Downloading Packages: 2023-09-23T20:32:12.545 INFO:teuthology.orchestra.run.smithi070.stdout:(1/6): xmlstarlet-1.6.1-20.el8.x86_64.rpm 1.8 MB/s | 70 kB 00:00 2023-09-23T20:32:12.621 INFO:teuthology.orchestra.run.smithi070.stdout:(2/6): socat-1.7.4.1-1.el8.x86_64.rpm 4.7 MB/s | 323 kB 00:00 2023-09-23T20:32:12.662 INFO:teuthology.orchestra.run.smithi070.stdout:(3/6): oniguruma-6.8.2-2.el8.x86_64.rpm 4.4 MB/s | 187 kB 00:00 2023-09-23T20:32:12.706 INFO:teuthology.orchestra.run.smithi070.stdout:(4/6): jq-1.6-3.el8.x86_64.rpm 4.5 MB/s | 202 kB 00:00 2023-09-23T20:32:12.748 INFO:teuthology.orchestra.run.smithi070.stdout:(5/6): libcephsqlite-17.2.6-1123.g61431f6b.el8. 722 kB/s | 174 kB 00:00 2023-09-23T20:32:13.413 INFO:teuthology.orchestra.run.smithi026.stdout:Installed products updated. 2023-09-23T20:32:13.622 INFO:teuthology.orchestra.run.smithi026.stdout: 2023-09-23T20:32:13.622 INFO:teuthology.orchestra.run.smithi026.stdout:Upgraded: 2023-09-23T20:32:13.622 INFO:teuthology.orchestra.run.smithi026.stdout: librados2-2:17.2.6-1123.g61431f6b.el8.x86_64 2023-09-23T20:32:13.622 INFO:teuthology.orchestra.run.smithi026.stdout: librbd1-2:17.2.6-1123.g61431f6b.el8.x86_64 2023-09-23T20:32:13.622 INFO:teuthology.orchestra.run.smithi026.stdout:Installed: 2023-09-23T20:32:13.622 INFO:teuthology.orchestra.run.smithi026.stdout: ceph-base-2:17.2.6-1123.g61431f6b.el8.x86_64 2023-09-23T20:32:13.622 INFO:teuthology.orchestra.run.smithi026.stdout: ceph-common-2:17.2.6-1123.g61431f6b.el8.x86_64 2023-09-23T20:32:13.622 INFO:teuthology.orchestra.run.smithi026.stdout: ceph-radosgw-2:17.2.6-1123.g61431f6b.el8.x86_64 2023-09-23T20:32:13.623 INFO:teuthology.orchestra.run.smithi026.stdout: ceph-selinux-2:17.2.6-1123.g61431f6b.el8.x86_64 2023-09-23T20:32:13.623 INFO:teuthology.orchestra.run.smithi026.stdout: gperftools-libs-1:2.7-9.el8.x86_64 2023-09-23T20:32:13.623 INFO:teuthology.orchestra.run.smithi026.stdout: libcephfs2-2:17.2.6-1123.g61431f6b.el8.x86_64 2023-09-23T20:32:13.623 INFO:teuthology.orchestra.run.smithi026.stdout: liboath-2.6.2-3.el8.x86_64 2023-09-23T20:32:13.623 INFO:teuthology.orchestra.run.smithi026.stdout: librabbitmq-0.9.0-3.el8.x86_64 2023-09-23T20:32:13.623 INFO:teuthology.orchestra.run.smithi026.stdout: libradosstriper1-2:17.2.6-1123.g61431f6b.el8.x86_64 2023-09-23T20:32:13.623 INFO:teuthology.orchestra.run.smithi026.stdout: librdkafka-0.11.4-3.el8.x86_64 2023-09-23T20:32:13.623 INFO:teuthology.orchestra.run.smithi026.stdout: librgw2-2:17.2.6-1123.g61431f6b.el8.x86_64 2023-09-23T20:32:13.623 INFO:teuthology.orchestra.run.smithi026.stdout: libunwind-1.3.1-3.el8.x86_64 2023-09-23T20:32:13.623 INFO:teuthology.orchestra.run.smithi026.stdout: lttng-ust-2.8.1-11.el8.x86_64 2023-09-23T20:32:13.623 INFO:teuthology.orchestra.run.smithi026.stdout: python3-ceph-argparse-2:17.2.6-1123.g61431f6b.el8.x86_64 2023-09-23T20:32:13.623 INFO:teuthology.orchestra.run.smithi026.stdout: python3-ceph-common-2:17.2.6-1123.g61431f6b.el8.x86_64 2023-09-23T20:32:13.623 INFO:teuthology.orchestra.run.smithi026.stdout: python3-cephfs-2:17.2.6-1123.g61431f6b.el8.x86_64 2023-09-23T20:32:13.624 INFO:teuthology.orchestra.run.smithi026.stdout: python3-prettytable-0.7.2-14.el8.noarch 2023-09-23T20:32:13.624 INFO:teuthology.orchestra.run.smithi026.stdout: python3-rados-2:17.2.6-1123.g61431f6b.el8.x86_64 2023-09-23T20:32:13.624 INFO:teuthology.orchestra.run.smithi026.stdout: python3-rbd-2:17.2.6-1123.g61431f6b.el8.x86_64 2023-09-23T20:32:13.624 INFO:teuthology.orchestra.run.smithi026.stdout: python3-rgw-2:17.2.6-1123.g61431f6b.el8.x86_64 2023-09-23T20:32:13.624 INFO:teuthology.orchestra.run.smithi026.stdout: thrift-0.13.0-2.el8.x86_64 2023-09-23T20:32:13.624 INFO:teuthology.orchestra.run.smithi026.stdout: 2023-09-23T20:32:13.624 INFO:teuthology.orchestra.run.smithi026.stdout:Complete! 2023-09-23T20:32:13.801 DEBUG:teuthology.orchestra.run.smithi026:> sudo yum -y install ceph-test 2023-09-23T20:32:14.171 INFO:teuthology.orchestra.run.smithi026.stdout:Updating Subscription Management repositories. 2023-09-23T20:32:15.052 INFO:teuthology.orchestra.run.smithi026.stdout:Last metadata expiration check: 0:00:57 ago on Sat 23 Sep 2023 08:31:18 PM UTC. 2023-09-23T20:32:15.248 INFO:teuthology.orchestra.run.smithi070.stdout:(6/6): ceph-test-17.2.6-1123.g61431f6b.el8.x86_ 19 MB/s | 52 MB 00:02 2023-09-23T20:32:15.250 INFO:teuthology.orchestra.run.smithi070.stdout:-------------------------------------------------------------------------------- 2023-09-23T20:32:15.250 INFO:teuthology.orchestra.run.smithi070.stdout:Total 19 MB/s | 53 MB 00:02 2023-09-23T20:32:15.286 INFO:teuthology.orchestra.run.smithi070.stdout:Running transaction check 2023-09-23T20:32:15.306 INFO:teuthology.orchestra.run.smithi070.stdout:Transaction check succeeded. 2023-09-23T20:32:15.306 INFO:teuthology.orchestra.run.smithi070.stdout:Running transaction test 2023-09-23T20:32:15.579 INFO:teuthology.orchestra.run.smithi070.stdout:Transaction test succeeded. 2023-09-23T20:32:15.583 INFO:teuthology.orchestra.run.smithi070.stdout:Running transaction 2023-09-23T20:32:16.050 INFO:teuthology.orchestra.run.smithi070.stdout: Preparing : 1/1 2023-09-23T20:32:16.222 INFO:teuthology.orchestra.run.smithi070.stdout: Installing : oniguruma-6.8.2-2.el8.x86_64 1/6 2023-09-23T20:32:16.300 INFO:teuthology.orchestra.run.smithi070.stdout: Running scriptlet: oniguruma-6.8.2-2.el8.x86_64 1/6 2023-09-23T20:32:16.476 INFO:teuthology.orchestra.run.smithi070.stdout: Installing : jq-1.6-3.el8.x86_64 2/6 2023-09-23T20:32:16.605 INFO:teuthology.orchestra.run.smithi026.stdout:Dependencies resolved. 2023-09-23T20:32:16.606 INFO:teuthology.orchestra.run.smithi026.stdout:============================================================================================ 2023-09-23T20:32:16.607 INFO:teuthology.orchestra.run.smithi026.stdout: Package Arch Version Repository Size 2023-09-23T20:32:16.607 INFO:teuthology.orchestra.run.smithi026.stdout:============================================================================================ 2023-09-23T20:32:16.607 INFO:teuthology.orchestra.run.smithi026.stdout:Installing: 2023-09-23T20:32:16.607 INFO:teuthology.orchestra.run.smithi026.stdout: ceph-test x86_64 2:17.2.6-1123.g61431f6b.el8 ceph 52 M 2023-09-23T20:32:16.607 INFO:teuthology.orchestra.run.smithi026.stdout:Installing dependencies: 2023-09-23T20:32:16.607 INFO:teuthology.orchestra.run.smithi026.stdout: jq x86_64 1.6-3.el8 rhel-8-for-x86_64-appstream-rpms 202 k 2023-09-23T20:32:16.607 INFO:teuthology.orchestra.run.smithi026.stdout: libcephsqlite x86_64 2:17.2.6-1123.g61431f6b.el8 ceph 174 k 2023-09-23T20:32:16.607 INFO:teuthology.orchestra.run.smithi026.stdout: oniguruma x86_64 6.8.2-2.el8 rhel-8-for-x86_64-appstream-rpms 187 k 2023-09-23T20:32:16.607 INFO:teuthology.orchestra.run.smithi026.stdout: socat x86_64 1.7.4.1-1.el8 rhel-8-for-x86_64-appstream-rpms 323 k 2023-09-23T20:32:16.607 INFO:teuthology.orchestra.run.smithi026.stdout: xmlstarlet x86_64 1.6.1-20.el8 lab-extras 70 k 2023-09-23T20:32:16.607 INFO:teuthology.orchestra.run.smithi026.stdout: 2023-09-23T20:32:16.607 INFO:teuthology.orchestra.run.smithi026.stdout:Transaction Summary 2023-09-23T20:32:16.608 INFO:teuthology.orchestra.run.smithi026.stdout:============================================================================================ 2023-09-23T20:32:16.608 INFO:teuthology.orchestra.run.smithi026.stdout:Install 6 Packages 2023-09-23T20:32:16.608 INFO:teuthology.orchestra.run.smithi026.stdout: 2023-09-23T20:32:16.608 INFO:teuthology.orchestra.run.smithi026.stdout:Total download size: 53 M 2023-09-23T20:32:16.608 INFO:teuthology.orchestra.run.smithi026.stdout:Installed size: 233 M 2023-09-23T20:32:16.608 INFO:teuthology.orchestra.run.smithi026.stdout:Downloading Packages: 2023-09-23T20:32:16.622 INFO:teuthology.orchestra.run.smithi070.stdout: Installing : socat-1.7.4.1-1.el8.x86_64 3/6 2023-09-23T20:32:16.641 INFO:teuthology.orchestra.run.smithi026.stdout:(1/6): xmlstarlet-1.6.1-20.el8.x86_64.rpm 2.1 MB/s | 70 kB 00:00 2023-09-23T20:32:16.708 INFO:teuthology.orchestra.run.smithi026.stdout:(2/6): socat-1.7.4.1-1.el8.x86_64.rpm 5.1 MB/s | 323 kB 00:00 2023-09-23T20:32:16.746 INFO:teuthology.orchestra.run.smithi070.stdout: Installing : xmlstarlet-1.6.1-20.el8.x86_64 4/6 2023-09-23T20:32:16.750 INFO:teuthology.orchestra.run.smithi026.stdout:(3/6): oniguruma-6.8.2-2.el8.x86_64.rpm 4.5 MB/s | 187 kB 00:00 2023-09-23T20:32:16.792 INFO:teuthology.orchestra.run.smithi026.stdout:(4/6): jq-1.6-3.el8.x86_64.rpm 4.7 MB/s | 202 kB 00:00 2023-09-23T20:32:16.834 INFO:teuthology.orchestra.run.smithi026.stdout:(5/6): libcephsqlite-17.2.6-1123.g61431f6b.el8. 777 kB/s | 174 kB 00:00 2023-09-23T20:32:16.889 INFO:teuthology.orchestra.run.smithi070.stdout: Installing : libcephsqlite-2:17.2.6-1123.g61431f6b.el8.x86_64 5/6 2023-09-23T20:32:19.875 INFO:teuthology.orchestra.run.smithi026.stdout:(6/6): ceph-test-17.2.6-1123.g61431f6b.el8.x86_ 16 MB/s | 52 MB 00:03 2023-09-23T20:32:19.877 INFO:teuthology.orchestra.run.smithi026.stdout:-------------------------------------------------------------------------------- 2023-09-23T20:32:19.877 INFO:teuthology.orchestra.run.smithi026.stdout:Total 16 MB/s | 53 MB 00:03 2023-09-23T20:32:19.915 INFO:teuthology.orchestra.run.smithi026.stdout:Running transaction check 2023-09-23T20:32:19.937 INFO:teuthology.orchestra.run.smithi026.stdout:Transaction check succeeded. 2023-09-23T20:32:19.938 INFO:teuthology.orchestra.run.smithi026.stdout:Running transaction test 2023-09-23T20:32:20.226 INFO:teuthology.orchestra.run.smithi026.stdout:Transaction test succeeded. 2023-09-23T20:32:20.231 INFO:teuthology.orchestra.run.smithi026.stdout:Running transaction 2023-09-23T20:32:20.706 INFO:teuthology.orchestra.run.smithi026.stdout: Preparing : 1/1 2023-09-23T20:32:20.838 INFO:teuthology.orchestra.run.smithi026.stdout: Installing : oniguruma-6.8.2-2.el8.x86_64 1/6 2023-09-23T20:32:20.904 INFO:teuthology.orchestra.run.smithi026.stdout: Running scriptlet: oniguruma-6.8.2-2.el8.x86_64 1/6 2023-09-23T20:32:21.094 INFO:teuthology.orchestra.run.smithi026.stdout: Installing : jq-1.6-3.el8.x86_64 2/6 2023-09-23T20:32:21.247 INFO:teuthology.orchestra.run.smithi026.stdout: Installing : socat-1.7.4.1-1.el8.x86_64 3/6 2023-09-23T20:32:21.413 INFO:teuthology.orchestra.run.smithi026.stdout: Installing : xmlstarlet-1.6.1-20.el8.x86_64 4/6 2023-09-23T20:32:21.555 INFO:teuthology.orchestra.run.smithi026.stdout: Installing : libcephsqlite-2:17.2.6-1123.g61431f6b.el8.x86_64 5/6 2023-09-23T20:32:21.807 INFO:teuthology.orchestra.run.smithi070.stdout: Running scriptlet: libcephsqlite-2:17.2.6-1123.g61431f6b.el8.x86_64 5/6 2023-09-23T20:32:22.051 INFO:teuthology.orchestra.run.smithi070.stdout: Installing : ceph-test-2:17.2.6-1123.g61431f6b.el8.x86_64 6/6 2023-09-23T20:32:22.270 INFO:teuthology.orchestra.run.smithi070.stdout: Running scriptlet: ceph-test-2:17.2.6-1123.g61431f6b.el8.x86_64 6/6 2023-09-23T20:32:22.270 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : ceph-test-2:17.2.6-1123.g61431f6b.el8.x86_64 1/6 2023-09-23T20:32:22.270 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : libcephsqlite-2:17.2.6-1123.g61431f6b.el8.x86_64 2/6 2023-09-23T20:32:22.270 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : xmlstarlet-1.6.1-20.el8.x86_64 3/6 2023-09-23T20:32:22.271 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : socat-1.7.4.1-1.el8.x86_64 4/6 2023-09-23T20:32:22.271 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : oniguruma-6.8.2-2.el8.x86_64 5/6 2023-09-23T20:32:23.322 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : jq-1.6-3.el8.x86_64 6/6 2023-09-23T20:32:23.322 INFO:teuthology.orchestra.run.smithi070.stdout:Last metadata expiration check: 0:01:07 ago on Sat 23 Sep 2023 08:31:16 PM UTC. 2023-09-23T20:32:25.654 INFO:teuthology.orchestra.run.smithi070.stdout:Installed products updated. 2023-09-23T20:32:25.834 INFO:teuthology.orchestra.run.smithi070.stdout: 2023-09-23T20:32:25.834 INFO:teuthology.orchestra.run.smithi070.stdout:Installed: 2023-09-23T20:32:25.834 INFO:teuthology.orchestra.run.smithi070.stdout: ceph-test-2:17.2.6-1123.g61431f6b.el8.x86_64 2023-09-23T20:32:25.834 INFO:teuthology.orchestra.run.smithi070.stdout: jq-1.6-3.el8.x86_64 2023-09-23T20:32:25.834 INFO:teuthology.orchestra.run.smithi070.stdout: libcephsqlite-2:17.2.6-1123.g61431f6b.el8.x86_64 2023-09-23T20:32:25.835 INFO:teuthology.orchestra.run.smithi070.stdout: oniguruma-6.8.2-2.el8.x86_64 2023-09-23T20:32:25.835 INFO:teuthology.orchestra.run.smithi070.stdout: socat-1.7.4.1-1.el8.x86_64 2023-09-23T20:32:25.835 INFO:teuthology.orchestra.run.smithi070.stdout: xmlstarlet-1.6.1-20.el8.x86_64 2023-09-23T20:32:25.835 INFO:teuthology.orchestra.run.smithi070.stdout: 2023-09-23T20:32:25.835 INFO:teuthology.orchestra.run.smithi070.stdout:Complete! 2023-09-23T20:32:25.979 DEBUG:teuthology.orchestra.run.smithi070:> sudo yum -y install ceph 2023-09-23T20:32:26.313 INFO:teuthology.orchestra.run.smithi070.stdout:Updating Subscription Management repositories. 2023-09-23T20:32:26.825 INFO:teuthology.orchestra.run.smithi026.stdout: Running scriptlet: libcephsqlite-2:17.2.6-1123.g61431f6b.el8.x86_64 5/6 2023-09-23T20:32:27.035 INFO:teuthology.orchestra.run.smithi026.stdout: Installing : ceph-test-2:17.2.6-1123.g61431f6b.el8.x86_64 6/6 2023-09-23T20:32:27.099 INFO:teuthology.orchestra.run.smithi070.stdout:Last metadata expiration check: 0:01:11 ago on Sat 23 Sep 2023 08:31:16 PM UTC. 2023-09-23T20:32:27.274 INFO:teuthology.orchestra.run.smithi026.stdout: Running scriptlet: ceph-test-2:17.2.6-1123.g61431f6b.el8.x86_64 6/6 2023-09-23T20:32:27.274 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : ceph-test-2:17.2.6-1123.g61431f6b.el8.x86_64 1/6 2023-09-23T20:32:27.274 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : libcephsqlite-2:17.2.6-1123.g61431f6b.el8.x86_64 2/6 2023-09-23T20:32:27.274 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : xmlstarlet-1.6.1-20.el8.x86_64 3/6 2023-09-23T20:32:27.274 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : socat-1.7.4.1-1.el8.x86_64 4/6 2023-09-23T20:32:27.275 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : oniguruma-6.8.2-2.el8.x86_64 5/6 2023-09-23T20:32:28.373 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : jq-1.6-3.el8.x86_64 6/6 2023-09-23T20:32:28.373 INFO:teuthology.orchestra.run.smithi026.stdout:Last metadata expiration check: 0:01:10 ago on Sat 23 Sep 2023 08:31:18 PM UTC. 2023-09-23T20:32:28.480 INFO:teuthology.orchestra.run.smithi070.stdout:Dependencies resolved. 2023-09-23T20:32:28.483 INFO:teuthology.orchestra.run.smithi070.stdout:======================================================================================================= 2023-09-23T20:32:28.483 INFO:teuthology.orchestra.run.smithi070.stdout: Package Arch Version Repository Size 2023-09-23T20:32:28.484 INFO:teuthology.orchestra.run.smithi070.stdout:======================================================================================================= 2023-09-23T20:32:28.484 INFO:teuthology.orchestra.run.smithi070.stdout:Installing: 2023-09-23T20:32:28.484 INFO:teuthology.orchestra.run.smithi070.stdout: ceph x86_64 2:17.2.6-1123.g61431f6b.el8 ceph 6.4 k 2023-09-23T20:32:28.484 INFO:teuthology.orchestra.run.smithi070.stdout:Installing dependencies: 2023-09-23T20:32:28.484 INFO:teuthology.orchestra.run.smithi070.stdout: ceph-mds x86_64 2:17.2.6-1123.g61431f6b.el8 ceph 2.3 M 2023-09-23T20:32:28.484 INFO:teuthology.orchestra.run.smithi070.stdout: ceph-mgr x86_64 2:17.2.6-1123.g61431f6b.el8 ceph 1.6 M 2023-09-23T20:32:28.484 INFO:teuthology.orchestra.run.smithi070.stdout: ceph-mgr-modules-core noarch 2:17.2.6-1123.g61431f6b.el8 ceph-noarch 238 k 2023-09-23T20:32:28.484 INFO:teuthology.orchestra.run.smithi070.stdout: ceph-mon x86_64 2:17.2.6-1123.g61431f6b.el8 ceph 3.9 M 2023-09-23T20:32:28.484 INFO:teuthology.orchestra.run.smithi070.stdout: ceph-osd x86_64 2:17.2.6-1123.g61431f6b.el8 ceph 17 M 2023-09-23T20:32:28.484 INFO:teuthology.orchestra.run.smithi070.stdout: python3-bcrypt x86_64 3.1.6-2.el8.1 epel 44 k 2023-09-23T20:32:28.484 INFO:teuthology.orchestra.run.smithi070.stdout: python3-beautifulsoup4 noarch 4.6.3-2.el8.1 epel 185 k 2023-09-23T20:32:28.485 INFO:teuthology.orchestra.run.smithi070.stdout: python3-cffi x86_64 1.11.5-5.el8 rhel-8-for-x86_64-baseos-rpms 238 k 2023-09-23T20:32:28.485 INFO:teuthology.orchestra.run.smithi070.stdout: python3-cheroot noarch 8.5.2-1.el8 epel 173 k 2023-09-23T20:32:28.485 INFO:teuthology.orchestra.run.smithi070.stdout: python3-cherrypy noarch 18.4.0-1.el8 epel 384 k 2023-09-23T20:32:28.485 INFO:teuthology.orchestra.run.smithi070.stdout: python3-cryptography x86_64 3.2.1-5.el8 rhel-8-for-x86_64-baseos-rpms 559 k 2023-09-23T20:32:28.485 INFO:teuthology.orchestra.run.smithi070.stdout: python3-jaraco noarch 6.2-6.el8 epel 11 k 2023-09-23T20:32:28.485 INFO:teuthology.orchestra.run.smithi070.stdout: python3-jaraco-functools noarch 2.0-4.el8 epel 18 k 2023-09-23T20:32:28.485 INFO:teuthology.orchestra.run.smithi070.stdout: python3-logutils noarch 0.3.5-11.el8 epel 49 k 2023-09-23T20:32:28.485 INFO:teuthology.orchestra.run.smithi070.stdout: python3-mako noarch 1.0.6-13.el8 rhel-8-for-x86_64-appstream-rpms 157 k 2023-09-23T20:32:28.485 INFO:teuthology.orchestra.run.smithi070.stdout: python3-markupsafe x86_64 0.23-19.el8 rhel-8-for-x86_64-appstream-rpms 39 k 2023-09-23T20:32:28.485 INFO:teuthology.orchestra.run.smithi070.stdout: python3-more-itertools noarch 7.2.0-3.el8 epel 59 k 2023-09-23T20:32:28.485 INFO:teuthology.orchestra.run.smithi070.stdout: python3-pecan noarch 1.3.2-9.el8 epel 283 k 2023-09-23T20:32:28.486 INFO:teuthology.orchestra.run.smithi070.stdout: python3-portend noarch 2.6-1.el8 epel 16 k 2023-09-23T20:32:28.486 INFO:teuthology.orchestra.run.smithi070.stdout: python3-pyOpenSSL noarch 19.0.0-1.el8 rhel-8-for-x86_64-appstream-rpms 103 k 2023-09-23T20:32:28.486 INFO:teuthology.orchestra.run.smithi070.stdout: python3-pycparser noarch 2.14-14.el8 rhel-8-for-x86_64-baseos-rpms 109 k 2023-09-23T20:32:28.486 INFO:teuthology.orchestra.run.smithi070.stdout: python3-pytz noarch 2017.2-9.el8 rhel-8-for-x86_64-appstream-rpms 54 k 2023-09-23T20:32:28.486 INFO:teuthology.orchestra.run.smithi070.stdout: python3-simplegeneric noarch 0.8.1-17.el8 epel 19 k 2023-09-23T20:32:28.486 INFO:teuthology.orchestra.run.smithi070.stdout: python3-singledispatch noarch 3.4.0.3-18.el8 epel 24 k 2023-09-23T20:32:28.486 INFO:teuthology.orchestra.run.smithi070.stdout: python3-tempora noarch 1.14.1-5.el8 epel 29 k 2023-09-23T20:32:28.486 INFO:teuthology.orchestra.run.smithi070.stdout: python3-trustme noarch 0.6.0-4.el8 epel 27 k 2023-09-23T20:32:28.486 INFO:teuthology.orchestra.run.smithi070.stdout: python3-waitress noarch 1.4.3-1.el8 epel 255 k 2023-09-23T20:32:28.486 INFO:teuthology.orchestra.run.smithi070.stdout: python3-webob noarch 1.8.5-1.el8.1 epel 251 k 2023-09-23T20:32:28.486 INFO:teuthology.orchestra.run.smithi070.stdout: python3-webtest noarch 2.0.33-1.el8 epel 85 k 2023-09-23T20:32:28.486 INFO:teuthology.orchestra.run.smithi070.stdout: python3-werkzeug noarch 0.12.2-4.el8 rhel-8-for-x86_64-appstream-rpms 457 k 2023-09-23T20:32:28.487 INFO:teuthology.orchestra.run.smithi070.stdout: python3-zc-lockfile noarch 2.0-2.el8 epel 23 k 2023-09-23T20:32:28.487 INFO:teuthology.orchestra.run.smithi070.stdout: 2023-09-23T20:32:28.487 INFO:teuthology.orchestra.run.smithi070.stdout:Transaction Summary 2023-09-23T20:32:28.487 INFO:teuthology.orchestra.run.smithi070.stdout:======================================================================================================= 2023-09-23T20:32:28.487 INFO:teuthology.orchestra.run.smithi070.stdout:Install 32 Packages 2023-09-23T20:32:28.487 INFO:teuthology.orchestra.run.smithi070.stdout: 2023-09-23T20:32:28.487 INFO:teuthology.orchestra.run.smithi070.stdout:Total download size: 28 M 2023-09-23T20:32:28.488 INFO:teuthology.orchestra.run.smithi070.stdout:Installed size: 110 M 2023-09-23T20:32:28.488 INFO:teuthology.orchestra.run.smithi070.stdout:Downloading Packages: 2023-09-23T20:32:28.997 INFO:teuthology.orchestra.run.smithi070.stdout:(1/32): ceph-17.2.6-1123.g61431f6b.el8.x86_64.r 36 kB/s | 6.4 kB 00:00 2023-09-23T20:32:29.114 INFO:teuthology.orchestra.run.smithi070.stdout:(2/32): ceph-mgr-17.2.6-1123.g61431f6b.el8.x86_ 5.4 MB/s | 1.6 MB 00:00 2023-09-23T20:32:29.640 INFO:teuthology.orchestra.run.smithi070.stdout:(3/32): ceph-mds-17.2.6-1123.g61431f6b.el8.x86_ 2.8 MB/s | 2.3 MB 00:00 2023-09-23T20:32:29.748 INFO:teuthology.orchestra.run.smithi070.stdout:(4/32): ceph-mgr-modules-core-17.2.6-1123.g6143 2.2 MB/s | 238 kB 00:00 2023-09-23T20:32:29.999 INFO:teuthology.orchestra.run.smithi070.stdout:(5/32): ceph-osd-17.2.6-1123.g61431f6b.el8.x86_ 19 MB/s | 17 MB 00:00 2023-09-23T20:32:30.024 INFO:teuthology.orchestra.run.smithi070.stdout:(6/32): python3-bcrypt-3.1.6-2.el8.1.x86_64.rpm 158 kB/s | 44 kB 00:00 2023-09-23T20:32:30.091 INFO:teuthology.orchestra.run.smithi070.stdout:(7/32): python3-cheroot-8.5.2-1.el8.noarch.rpm 2.7 MB/s | 173 kB 00:00 2023-09-23T20:32:30.158 INFO:teuthology.orchestra.run.smithi070.stdout:(8/32): python3-cherrypy-18.4.0-1.el8.noarch.rp 5.6 MB/s | 384 kB 00:00 2023-09-23T20:32:30.183 INFO:teuthology.orchestra.run.smithi070.stdout:(9/32): python3-beautifulsoup4-4.6.3-2.el8.1.no 1.0 MB/s | 185 kB 00:00 2023-09-23T20:32:30.208 INFO:teuthology.orchestra.run.smithi070.stdout:(10/32): python3-jaraco-6.2-6.el8.noarch.rpm 222 kB/s | 11 kB 00:00 2023-09-23T20:32:30.234 INFO:teuthology.orchestra.run.smithi070.stdout:(11/32): python3-jaraco-functools-2.0-4.el8.noa 349 kB/s | 18 kB 00:00 2023-09-23T20:32:30.265 INFO:teuthology.orchestra.run.smithi070.stdout:(12/32): python3-logutils-0.3.5-11.el8.noarch.r 967 kB/s | 49 kB 00:00 2023-09-23T20:32:30.285 INFO:teuthology.orchestra.run.smithi070.stdout:(13/32): python3-more-itertools-7.2.0-3.el8.noa 1.1 MB/s | 59 kB 00:00 2023-09-23T20:32:30.310 INFO:teuthology.orchestra.run.smithi070.stdout:(14/32): python3-pecan-1.3.2-9.el8.noarch.rpm 5.5 MB/s | 283 kB 00:00 2023-09-23T20:32:30.360 INFO:teuthology.orchestra.run.smithi070.stdout:(15/32): python3-portend-2.6-1.el8.noarch.rpm 207 kB/s | 16 kB 00:00 2023-09-23T20:32:30.388 INFO:teuthology.orchestra.run.smithi070.stdout:(16/32): python3-simplegeneric-0.8.1-17.el8.noa 249 kB/s | 19 kB 00:00 2023-09-23T20:32:30.416 INFO:teuthology.orchestra.run.smithi070.stdout:(17/32): python3-singledispatch-3.4.0.3-18.el8. 475 kB/s | 24 kB 00:00 2023-09-23T20:32:30.436 INFO:teuthology.orchestra.run.smithi070.stdout:(18/32): python3-tempora-1.14.1-5.el8.noarch.rp 582 kB/s | 29 kB 00:00 2023-09-23T20:32:30.462 INFO:teuthology.orchestra.run.smithi070.stdout:(19/32): python3-trustme-0.6.0-4.el8.noarch.rpm 549 kB/s | 27 kB 00:00 2023-09-23T20:32:30.487 INFO:teuthology.orchestra.run.smithi070.stdout:(20/32): python3-waitress-1.4.3-1.el8.noarch.rp 5.0 MB/s | 255 kB 00:00 2023-09-23T20:32:30.557 INFO:teuthology.orchestra.run.smithi070.stdout:(21/32): ceph-mon-17.2.6-1123.g61431f6b.el8.x86 2.5 MB/s | 3.9 MB 00:01 2023-09-23T20:32:30.584 INFO:teuthology.orchestra.run.smithi070.stdout:(22/32): python3-webob-1.8.5-1.el8.1.noarch.rpm 2.1 MB/s | 251 kB 00:00 2023-09-23T20:32:30.607 INFO:teuthology.orchestra.run.smithi070.stdout:(23/32): python3-webtest-2.0.33-1.el8.noarch.rp 728 kB/s | 85 kB 00:00 2023-09-23T20:32:30.663 INFO:teuthology.orchestra.run.smithi070.stdout:(24/32): python3-zc-lockfile-2.0-2.el8.noarch.r 211 kB/s | 23 kB 00:00 2023-09-23T20:32:30.688 INFO:teuthology.orchestra.run.smithi070.stdout:(25/32): python3-pycparser-2.14-14.el8.noarch.r 1.0 MB/s | 109 kB 00:00 2023-09-23T20:32:30.722 INFO:teuthology.orchestra.run.smithi070.stdout:(26/32): python3-cryptography-3.2.1-5.el8.x86_6 4.7 MB/s | 559 kB 00:00 2023-09-23T20:32:30.747 INFO:teuthology.orchestra.run.smithi070.stdout:(27/32): python3-pytz-2017.2-9.el8.noarch.rpm 1.0 MB/s | 54 kB 00:00 2023-09-23T20:32:30.772 INFO:teuthology.orchestra.run.smithi070.stdout:(28/32): python3-markupsafe-0.23-19.el8.x86_64. 775 kB/s | 39 kB 00:00 2023-09-23T20:32:30.797 INFO:teuthology.orchestra.run.smithi070.stdout:(29/32): python3-cffi-1.11.5-5.el8.x86_64.rpm 1.7 MB/s | 238 kB 00:00 2023-09-23T20:32:30.823 INFO:teuthology.orchestra.run.smithi070.stdout:(30/32): python3-mako-1.0.6-13.el8.noarch.rpm 2.0 MB/s | 157 kB 00:00 2023-09-23T20:32:30.849 INFO:teuthology.orchestra.run.smithi070.stdout:(31/32): python3-pyOpenSSL-19.0.0-1.el8.noarch. 2.0 MB/s | 103 kB 00:00 2023-09-23T20:32:30.870 INFO:teuthology.orchestra.run.smithi026.stdout:Installed products updated. 2023-09-23T20:32:30.873 INFO:teuthology.orchestra.run.smithi070.stdout:(32/32): python3-werkzeug-0.12.2-4.el8.noarch.r 4.5 MB/s | 457 kB 00:00 2023-09-23T20:32:30.877 INFO:teuthology.orchestra.run.smithi070.stdout:-------------------------------------------------------------------------------- 2023-09-23T20:32:30.877 INFO:teuthology.orchestra.run.smithi070.stdout:Total 12 MB/s | 28 MB 00:02 2023-09-23T20:32:30.966 INFO:teuthology.orchestra.run.smithi070.stdout:Running transaction check 2023-09-23T20:32:31.003 INFO:teuthology.orchestra.run.smithi070.stdout:Transaction check succeeded. 2023-09-23T20:32:31.003 INFO:teuthology.orchestra.run.smithi070.stdout:Running transaction test 2023-09-23T20:32:31.072 INFO:teuthology.orchestra.run.smithi026.stdout: 2023-09-23T20:32:31.072 INFO:teuthology.orchestra.run.smithi026.stdout:Installed: 2023-09-23T20:32:31.072 INFO:teuthology.orchestra.run.smithi026.stdout: ceph-test-2:17.2.6-1123.g61431f6b.el8.x86_64 2023-09-23T20:32:31.072 INFO:teuthology.orchestra.run.smithi026.stdout: jq-1.6-3.el8.x86_64 2023-09-23T20:32:31.072 INFO:teuthology.orchestra.run.smithi026.stdout: libcephsqlite-2:17.2.6-1123.g61431f6b.el8.x86_64 2023-09-23T20:32:31.072 INFO:teuthology.orchestra.run.smithi026.stdout: oniguruma-6.8.2-2.el8.x86_64 2023-09-23T20:32:31.072 INFO:teuthology.orchestra.run.smithi026.stdout: socat-1.7.4.1-1.el8.x86_64 2023-09-23T20:32:31.073 INFO:teuthology.orchestra.run.smithi026.stdout: xmlstarlet-1.6.1-20.el8.x86_64 2023-09-23T20:32:31.073 INFO:teuthology.orchestra.run.smithi026.stdout: 2023-09-23T20:32:31.073 INFO:teuthology.orchestra.run.smithi026.stdout:Complete! 2023-09-23T20:32:31.200 DEBUG:teuthology.orchestra.run.smithi026:> sudo yum -y install ceph 2023-09-23T20:32:31.207 INFO:teuthology.orchestra.run.smithi070.stdout:Transaction test succeeded. 2023-09-23T20:32:31.212 INFO:teuthology.orchestra.run.smithi070.stdout:Running transaction 2023-09-23T20:32:31.558 INFO:teuthology.orchestra.run.smithi026.stdout:Updating Subscription Management repositories. 2023-09-23T20:32:32.305 INFO:teuthology.orchestra.run.smithi070.stdout: Preparing : 1/1 2023-09-23T20:32:32.405 INFO:teuthology.orchestra.run.smithi026.stdout:Last metadata expiration check: 0:01:14 ago on Sat 23 Sep 2023 08:31:18 PM UTC. 2023-09-23T20:32:32.498 INFO:teuthology.orchestra.run.smithi070.stdout: Installing : python3-more-itertools-7.2.0-3.el8.noarch 1/32 2023-09-23T20:32:32.760 INFO:teuthology.orchestra.run.smithi070.stdout: Installing : python3-webob-1.8.5-1.el8.1.noarch 2/32 2023-09-23T20:32:33.064 INFO:teuthology.orchestra.run.smithi070.stdout: Installing : python3-werkzeug-0.12.2-4.el8.noarch 3/32 2023-09-23T20:32:33.287 INFO:teuthology.orchestra.run.smithi070.stdout: Installing : python3-markupsafe-0.23-19.el8.x86_64 4/32 2023-09-23T20:32:33.500 INFO:teuthology.orchestra.run.smithi070.stdout: Installing : python3-mako-1.0.6-13.el8.noarch 5/32 2023-09-23T20:32:33.700 INFO:teuthology.orchestra.run.smithi070.stdout: Installing : python3-pytz-2017.2-9.el8.noarch 6/32 2023-09-23T20:32:33.930 INFO:teuthology.orchestra.run.smithi070.stdout: Installing : python3-pycparser-2.14-14.el8.noarch 7/32 2023-09-23T20:32:33.987 INFO:teuthology.orchestra.run.smithi026.stdout:Dependencies resolved. 2023-09-23T20:32:33.990 INFO:teuthology.orchestra.run.smithi026.stdout:======================================================================================================= 2023-09-23T20:32:33.990 INFO:teuthology.orchestra.run.smithi026.stdout: Package Arch Version Repository Size 2023-09-23T20:32:33.990 INFO:teuthology.orchestra.run.smithi026.stdout:======================================================================================================= 2023-09-23T20:32:33.990 INFO:teuthology.orchestra.run.smithi026.stdout:Installing: 2023-09-23T20:32:33.991 INFO:teuthology.orchestra.run.smithi026.stdout: ceph x86_64 2:17.2.6-1123.g61431f6b.el8 ceph 6.4 k 2023-09-23T20:32:33.991 INFO:teuthology.orchestra.run.smithi026.stdout:Installing dependencies: 2023-09-23T20:32:33.991 INFO:teuthology.orchestra.run.smithi026.stdout: ceph-mds x86_64 2:17.2.6-1123.g61431f6b.el8 ceph 2.3 M 2023-09-23T20:32:33.991 INFO:teuthology.orchestra.run.smithi026.stdout: ceph-mgr x86_64 2:17.2.6-1123.g61431f6b.el8 ceph 1.6 M 2023-09-23T20:32:33.991 INFO:teuthology.orchestra.run.smithi026.stdout: ceph-mgr-modules-core noarch 2:17.2.6-1123.g61431f6b.el8 ceph-noarch 238 k 2023-09-23T20:32:33.991 INFO:teuthology.orchestra.run.smithi026.stdout: ceph-mon x86_64 2:17.2.6-1123.g61431f6b.el8 ceph 3.9 M 2023-09-23T20:32:33.991 INFO:teuthology.orchestra.run.smithi026.stdout: ceph-osd x86_64 2:17.2.6-1123.g61431f6b.el8 ceph 17 M 2023-09-23T20:32:33.991 INFO:teuthology.orchestra.run.smithi026.stdout: python3-bcrypt x86_64 3.1.6-2.el8.1 epel 44 k 2023-09-23T20:32:33.991 INFO:teuthology.orchestra.run.smithi026.stdout: python3-beautifulsoup4 noarch 4.6.3-2.el8.1 epel 185 k 2023-09-23T20:32:33.991 INFO:teuthology.orchestra.run.smithi026.stdout: python3-cffi x86_64 1.11.5-5.el8 rhel-8-for-x86_64-baseos-rpms 238 k 2023-09-23T20:32:33.991 INFO:teuthology.orchestra.run.smithi026.stdout: python3-cheroot noarch 8.5.2-1.el8 epel 173 k 2023-09-23T20:32:33.991 INFO:teuthology.orchestra.run.smithi026.stdout: python3-cherrypy noarch 18.4.0-1.el8 epel 384 k 2023-09-23T20:32:33.992 INFO:teuthology.orchestra.run.smithi026.stdout: python3-cryptography x86_64 3.2.1-5.el8 rhel-8-for-x86_64-baseos-rpms 559 k 2023-09-23T20:32:33.992 INFO:teuthology.orchestra.run.smithi026.stdout: python3-jaraco noarch 6.2-6.el8 epel 11 k 2023-09-23T20:32:33.992 INFO:teuthology.orchestra.run.smithi026.stdout: python3-jaraco-functools noarch 2.0-4.el8 epel 18 k 2023-09-23T20:32:33.992 INFO:teuthology.orchestra.run.smithi026.stdout: python3-logutils noarch 0.3.5-11.el8 epel 49 k 2023-09-23T20:32:33.992 INFO:teuthology.orchestra.run.smithi026.stdout: python3-mako noarch 1.0.6-13.el8 rhel-8-for-x86_64-appstream-rpms 157 k 2023-09-23T20:32:33.992 INFO:teuthology.orchestra.run.smithi026.stdout: python3-markupsafe x86_64 0.23-19.el8 rhel-8-for-x86_64-appstream-rpms 39 k 2023-09-23T20:32:33.992 INFO:teuthology.orchestra.run.smithi026.stdout: python3-more-itertools noarch 7.2.0-3.el8 epel 59 k 2023-09-23T20:32:33.992 INFO:teuthology.orchestra.run.smithi026.stdout: python3-pecan noarch 1.3.2-9.el8 epel 283 k 2023-09-23T20:32:33.992 INFO:teuthology.orchestra.run.smithi026.stdout: python3-portend noarch 2.6-1.el8 epel 16 k 2023-09-23T20:32:33.992 INFO:teuthology.orchestra.run.smithi026.stdout: python3-pyOpenSSL noarch 19.0.0-1.el8 rhel-8-for-x86_64-appstream-rpms 103 k 2023-09-23T20:32:33.992 INFO:teuthology.orchestra.run.smithi026.stdout: python3-pycparser noarch 2.14-14.el8 rhel-8-for-x86_64-baseos-rpms 109 k 2023-09-23T20:32:33.992 INFO:teuthology.orchestra.run.smithi026.stdout: python3-pytz noarch 2017.2-9.el8 rhel-8-for-x86_64-appstream-rpms 54 k 2023-09-23T20:32:33.993 INFO:teuthology.orchestra.run.smithi026.stdout: python3-simplegeneric noarch 0.8.1-17.el8 epel 19 k 2023-09-23T20:32:33.993 INFO:teuthology.orchestra.run.smithi026.stdout: python3-singledispatch noarch 3.4.0.3-18.el8 epel 24 k 2023-09-23T20:32:33.993 INFO:teuthology.orchestra.run.smithi026.stdout: python3-tempora noarch 1.14.1-5.el8 epel 29 k 2023-09-23T20:32:33.993 INFO:teuthology.orchestra.run.smithi026.stdout: python3-trustme noarch 0.6.0-4.el8 epel 27 k 2023-09-23T20:32:33.993 INFO:teuthology.orchestra.run.smithi026.stdout: python3-waitress noarch 1.4.3-1.el8 epel 255 k 2023-09-23T20:32:33.993 INFO:teuthology.orchestra.run.smithi026.stdout: python3-webob noarch 1.8.5-1.el8.1 epel 251 k 2023-09-23T20:32:33.993 INFO:teuthology.orchestra.run.smithi026.stdout: python3-webtest noarch 2.0.33-1.el8 epel 85 k 2023-09-23T20:32:33.993 INFO:teuthology.orchestra.run.smithi026.stdout: python3-werkzeug noarch 0.12.2-4.el8 rhel-8-for-x86_64-appstream-rpms 457 k 2023-09-23T20:32:33.993 INFO:teuthology.orchestra.run.smithi026.stdout: python3-zc-lockfile noarch 2.0-2.el8 epel 23 k 2023-09-23T20:32:33.993 INFO:teuthology.orchestra.run.smithi026.stdout: 2023-09-23T20:32:33.993 INFO:teuthology.orchestra.run.smithi026.stdout:Transaction Summary 2023-09-23T20:32:33.993 INFO:teuthology.orchestra.run.smithi026.stdout:======================================================================================================= 2023-09-23T20:32:33.994 INFO:teuthology.orchestra.run.smithi026.stdout:Install 32 Packages 2023-09-23T20:32:33.994 INFO:teuthology.orchestra.run.smithi026.stdout: 2023-09-23T20:32:33.994 INFO:teuthology.orchestra.run.smithi026.stdout:Total download size: 28 M 2023-09-23T20:32:33.994 INFO:teuthology.orchestra.run.smithi026.stdout:Installed size: 110 M 2023-09-23T20:32:33.994 INFO:teuthology.orchestra.run.smithi026.stdout:Downloading Packages: 2023-09-23T20:32:34.133 INFO:teuthology.orchestra.run.smithi026.stdout:(1/32): ceph-17.2.6-1123.g61431f6b.el8.x86_64.r 59 kB/s | 6.4 kB 00:00 2023-09-23T20:32:34.275 INFO:teuthology.orchestra.run.smithi070.stdout: Installing : python3-cffi-1.11.5-5.el8.x86_64 8/32 2023-09-23T20:32:34.550 INFO:teuthology.orchestra.run.smithi026.stdout:(2/32): ceph-mon-17.2.6-1123.g61431f6b.el8.x86_ 9.4 MB/s | 3.9 MB 00:00 2023-09-23T20:32:34.584 INFO:teuthology.orchestra.run.smithi026.stdout:(3/32): ceph-mgr-17.2.6-1123.g61431f6b.el8.x86_ 2.8 MB/s | 1.6 MB 00:00 2023-09-23T20:32:34.604 INFO:teuthology.orchestra.run.smithi070.stdout: Installing : python3-cryptography-3.2.1-5.el8.x86_64 9/32 2023-09-23T20:32:34.634 INFO:teuthology.orchestra.run.smithi026.stdout:(4/32): ceph-mds-17.2.6-1123.g61431f6b.el8.x86_ 3.8 MB/s | 2.3 MB 00:00 2023-09-23T20:32:34.660 INFO:teuthology.orchestra.run.smithi026.stdout:(5/32): ceph-mgr-modules-core-17.2.6-1123.g6143 3.1 MB/s | 238 kB 00:00 2023-09-23T20:32:34.742 INFO:teuthology.orchestra.run.smithi070.stdout: Installing : python3-pyOpenSSL-19.0.0-1.el8.noarch 10/32 2023-09-23T20:32:34.914 INFO:teuthology.orchestra.run.smithi070.stdout: Installing : python3-trustme-0.6.0-4.el8.noarch 11/32 2023-09-23T20:32:34.968 INFO:teuthology.orchestra.run.smithi026.stdout:(6/32): python3-bcrypt-3.1.6-2.el8.1.x86_64.rpm 131 kB/s | 44 kB 00:00 2023-09-23T20:32:35.119 INFO:teuthology.orchestra.run.smithi026.stdout:(7/32): python3-beautifulsoup4-4.6.3-2.el8.1.no 404 kB/s | 185 kB 00:00 2023-09-23T20:32:35.126 INFO:teuthology.orchestra.run.smithi070.stdout: Installing : python3-bcrypt-3.1.6-2.el8.1.x86_64 12/32 2023-09-23T20:32:35.335 INFO:teuthology.orchestra.run.smithi070.stdout: Installing : python3-zc-lockfile-2.0-2.el8.noarch 13/32 2023-09-23T20:32:35.352 INFO:teuthology.orchestra.run.smithi026.stdout:(8/32): ceph-osd-17.2.6-1123.g61431f6b.el8.x86_ 21 MB/s | 17 MB 00:00 2023-09-23T20:32:35.378 INFO:teuthology.orchestra.run.smithi026.stdout:(9/32): python3-cheroot-8.5.2-1.el8.noarch.rpm 422 kB/s | 173 kB 00:00 2023-09-23T20:32:35.403 INFO:teuthology.orchestra.run.smithi026.stdout:(10/32): python3-cherrypy-18.4.0-1.el8.noarch.r 1.3 MB/s | 384 kB 00:00 2023-09-23T20:32:35.503 INFO:teuthology.orchestra.run.smithi026.stdout:(11/32): python3-jaraco-functools-2.0-4.el8.noa 140 kB/s | 18 kB 00:00 2023-09-23T20:32:35.529 INFO:teuthology.orchestra.run.smithi026.stdout:(12/32): python3-logutils-0.3.5-11.el8.noarch.r 389 kB/s | 49 kB 00:00 2023-09-23T20:32:35.533 INFO:teuthology.orchestra.run.smithi070.stdout: Installing : python3-waitress-1.4.3-1.el8.noarch 14/32 2023-09-23T20:32:35.554 INFO:teuthology.orchestra.run.smithi026.stdout:(13/32): python3-jaraco-6.2-6.el8.noarch.rpm 55 kB/s | 11 kB 00:00 2023-09-23T20:32:35.621 INFO:teuthology.orchestra.run.smithi026.stdout:(14/32): python3-more-itertools-7.2.0-3.el8.noa 501 kB/s | 59 kB 00:00 2023-09-23T20:32:35.663 INFO:teuthology.orchestra.run.smithi026.stdout:(15/32): python3-pecan-1.3.2-9.el8.noarch.rpm 2.1 MB/s | 283 kB 00:00 2023-09-23T20:32:35.688 INFO:teuthology.orchestra.run.smithi026.stdout:(16/32): python3-portend-2.6-1.el8.noarch.rpm 116 kB/s | 16 kB 00:00 2023-09-23T20:32:35.689 INFO:teuthology.orchestra.run.smithi070.stdout: Installing : python3-singledispatch-3.4.0.3-18.el8.noarch 15/32 2023-09-23T20:32:35.730 INFO:teuthology.orchestra.run.smithi026.stdout:(17/32): python3-simplegeneric-0.8.1-17.el8.noa 172 kB/s | 19 kB 00:00 2023-09-23T20:32:35.772 INFO:teuthology.orchestra.run.smithi026.stdout:(18/32): python3-singledispatch-3.4.0.3-18.el8. 220 kB/s | 24 kB 00:00 2023-09-23T20:32:35.830 INFO:teuthology.orchestra.run.smithi026.stdout:(19/32): python3-trustme-0.6.0-4.el8.noarch.rpm 274 kB/s | 27 kB 00:00 2023-09-23T20:32:35.856 INFO:teuthology.orchestra.run.smithi026.stdout:(20/32): python3-tempora-1.14.1-5.el8.noarch.rp 175 kB/s | 29 kB 00:00 2023-09-23T20:32:35.869 INFO:teuthology.orchestra.run.smithi070.stdout: Installing : python3-simplegeneric-0.8.1-17.el8.noarch 16/32 2023-09-23T20:32:35.946 INFO:teuthology.orchestra.run.smithi026.stdout:(21/32): python3-waitress-1.4.3-1.el8.noarch.rp 1.4 MB/s | 255 kB 00:00 2023-09-23T20:32:36.006 INFO:teuthology.orchestra.run.smithi026.stdout:(22/32): python3-webob-1.8.5-1.el8.1.noarch.rpm 1.4 MB/s | 251 kB 00:00 2023-09-23T20:32:36.029 INFO:teuthology.orchestra.run.smithi070.stdout: Installing : python3-logutils-0.3.5-11.el8.noarch 17/32 2023-09-23T20:32:36.056 INFO:teuthology.orchestra.run.smithi026.stdout:(23/32): python3-pycparser-2.14-14.el8.noarch.r 2.1 MB/s | 109 kB 00:00 2023-09-23T20:32:36.082 INFO:teuthology.orchestra.run.smithi026.stdout:(24/32): python3-zc-lockfile-2.0-2.el8.noarch.r 169 kB/s | 23 kB 00:00 2023-09-23T20:32:36.107 INFO:teuthology.orchestra.run.smithi026.stdout:(25/32): python3-webtest-2.0.33-1.el8.noarch.rp 340 kB/s | 85 kB 00:00 2023-09-23T20:32:36.141 INFO:teuthology.orchestra.run.smithi026.stdout:(26/32): python3-cryptography-3.2.1-5.el8.x86_6 6.5 MB/s | 559 kB 00:00 2023-09-23T20:32:36.182 INFO:teuthology.orchestra.run.smithi026.stdout:(27/32): python3-cffi-1.11.5-5.el8.x86_64.rpm 2.3 MB/s | 238 kB 00:00 2023-09-23T20:32:36.190 INFO:teuthology.orchestra.run.smithi070.stdout: Installing : python3-jaraco-6.2-6.el8.noarch 18/32 2023-09-23T20:32:36.208 INFO:teuthology.orchestra.run.smithi026.stdout:(28/32): python3-markupsafe-0.23-19.el8.x86_64. 774 kB/s | 39 kB 00:00 2023-09-23T20:32:36.233 INFO:teuthology.orchestra.run.smithi026.stdout:(29/32): python3-pytz-2017.2-9.el8.noarch.rpm 432 kB/s | 54 kB 00:00 2023-09-23T20:32:36.258 INFO:teuthology.orchestra.run.smithi026.stdout:(30/32): python3-mako-1.0.6-13.el8.noarch.rpm 2.0 MB/s | 157 kB 00:00 2023-09-23T20:32:36.283 INFO:teuthology.orchestra.run.smithi026.stdout:(31/32): python3-pyOpenSSL-19.0.0-1.el8.noarch. 2.0 MB/s | 103 kB 00:00 2023-09-23T20:32:36.309 INFO:teuthology.orchestra.run.smithi026.stdout:(32/32): python3-werkzeug-0.12.2-4.el8.noarch.r 4.4 MB/s | 457 kB 00:00 2023-09-23T20:32:36.310 INFO:teuthology.orchestra.run.smithi026.stdout:-------------------------------------------------------------------------------- 2023-09-23T20:32:36.310 INFO:teuthology.orchestra.run.smithi026.stdout:Total 12 MB/s | 28 MB 00:02 2023-09-23T20:32:36.396 INFO:teuthology.orchestra.run.smithi070.stdout: Installing : python3-jaraco-functools-2.0-4.el8.noarch 19/32 2023-09-23T20:32:36.405 INFO:teuthology.orchestra.run.smithi026.stdout:Running transaction check 2023-09-23T20:32:36.446 INFO:teuthology.orchestra.run.smithi026.stdout:Transaction check succeeded. 2023-09-23T20:32:36.446 INFO:teuthology.orchestra.run.smithi026.stdout:Running transaction test 2023-09-23T20:32:36.604 INFO:teuthology.orchestra.run.smithi070.stdout: Installing : python3-cheroot-8.5.2-1.el8.noarch 20/32 2023-09-23T20:32:36.664 INFO:teuthology.orchestra.run.smithi026.stdout:Transaction test succeeded. 2023-09-23T20:32:36.669 INFO:teuthology.orchestra.run.smithi026.stdout:Running transaction 2023-09-23T20:32:36.782 INFO:teuthology.orchestra.run.smithi070.stdout: Installing : python3-tempora-1.14.1-5.el8.noarch 21/32 2023-09-23T20:32:37.037 INFO:teuthology.orchestra.run.smithi070.stdout: Installing : python3-portend-2.6-1.el8.noarch 22/32 2023-09-23T20:32:37.281 INFO:teuthology.orchestra.run.smithi070.stdout: Installing : python3-cherrypy-18.4.0-1.el8.noarch 23/32 2023-09-23T20:32:37.468 INFO:teuthology.orchestra.run.smithi070.stdout: Installing : python3-beautifulsoup4-4.6.3-2.el8.1.noarch 24/32 2023-09-23T20:32:37.761 INFO:teuthology.orchestra.run.smithi070.stdout: Installing : python3-webtest-2.0.33-1.el8.noarch 25/32 2023-09-23T20:32:37.784 INFO:teuthology.orchestra.run.smithi026.stdout: Preparing : 1/1 2023-09-23T20:32:37.996 INFO:teuthology.orchestra.run.smithi026.stdout: Installing : python3-more-itertools-7.2.0-3.el8.noarch 1/32 2023-09-23T20:32:38.259 INFO:teuthology.orchestra.run.smithi026.stdout: Installing : python3-webob-1.8.5-1.el8.1.noarch 2/32 2023-09-23T20:32:38.378 INFO:teuthology.orchestra.run.smithi070.stdout: Installing : python3-pecan-1.3.2-9.el8.noarch 26/32 2023-09-23T20:32:38.457 INFO:teuthology.orchestra.run.smithi026.stdout: Installing : python3-werkzeug-0.12.2-4.el8.noarch 3/32 2023-09-23T20:32:38.676 INFO:teuthology.orchestra.run.smithi026.stdout: Installing : python3-markupsafe-0.23-19.el8.x86_64 4/32 2023-09-23T20:32:38.884 INFO:teuthology.orchestra.run.smithi026.stdout: Installing : python3-mako-1.0.6-13.el8.noarch 5/32 2023-09-23T20:32:38.967 INFO:teuthology.orchestra.run.smithi070.stdout: Installing : ceph-mgr-modules-core-2:17.2.6-1123.g61431f6b.el8. 27/32 2023-09-23T20:32:39.084 INFO:teuthology.orchestra.run.smithi026.stdout: Installing : python3-pytz-2017.2-9.el8.noarch 6/32 2023-09-23T20:32:39.102 INFO:teuthology.orchestra.run.smithi070.stdout: Installing : ceph-mgr-2:17.2.6-1123.g61431f6b.el8.x86_64 28/32 2023-09-23T20:32:39.283 INFO:teuthology.orchestra.run.smithi026.stdout: Installing : python3-pycparser-2.14-14.el8.noarch 7/32 2023-09-23T20:32:39.616 INFO:teuthology.orchestra.run.smithi026.stdout: Installing : python3-cffi-1.11.5-5.el8.x86_64 8/32 2023-09-23T20:32:39.890 INFO:teuthology.orchestra.run.smithi026.stdout: Installing : python3-cryptography-3.2.1-5.el8.x86_64 9/32 2023-09-23T20:32:40.052 INFO:teuthology.orchestra.run.smithi026.stdout: Installing : python3-pyOpenSSL-19.0.0-1.el8.noarch 10/32 2023-09-23T20:32:40.220 INFO:teuthology.orchestra.run.smithi026.stdout: Installing : python3-trustme-0.6.0-4.el8.noarch 11/32 2023-09-23T20:32:40.453 INFO:teuthology.orchestra.run.smithi026.stdout: Installing : python3-bcrypt-3.1.6-2.el8.1.x86_64 12/32 2023-09-23T20:32:40.575 INFO:teuthology.orchestra.run.smithi070.stdout: Running scriptlet: ceph-mgr-2:17.2.6-1123.g61431f6b.el8.x86_64 28/32 2023-09-23T20:32:40.712 INFO:teuthology.orchestra.run.smithi026.stdout: Installing : python3-zc-lockfile-2.0-2.el8.noarch 13/32 2023-09-23T20:32:40.760 INFO:teuthology.orchestra.run.smithi070.stdout: Installing : ceph-osd-2:17.2.6-1123.g61431f6b.el8.x86_64 29/32 2023-09-23T20:32:40.909 INFO:teuthology.orchestra.run.smithi026.stdout: Installing : python3-waitress-1.4.3-1.el8.noarch 14/32 2023-09-23T20:32:41.048 INFO:teuthology.orchestra.run.smithi026.stdout: Installing : python3-singledispatch-3.4.0.3-18.el8.noarch 15/32 2023-09-23T20:32:41.151 INFO:teuthology.orchestra.run.smithi070.stdout: Running scriptlet: ceph-osd-2:17.2.6-1123.g61431f6b.el8.x86_64 29/32 2023-09-23T20:32:41.239 INFO:teuthology.orchestra.run.smithi026.stdout: Installing : python3-simplegeneric-0.8.1-17.el8.noarch 16/32 2023-09-23T20:32:41.275 INFO:teuthology.orchestra.run.smithi070.stdout: Installing : ceph-mon-2:17.2.6-1123.g61431f6b.el8.x86_64 30/32 2023-09-23T20:32:41.422 INFO:teuthology.orchestra.run.smithi026.stdout: Installing : python3-logutils-0.3.5-11.el8.noarch 17/32 2023-09-23T20:32:41.491 INFO:teuthology.orchestra.run.smithi070.stdout: Running scriptlet: ceph-mon-2:17.2.6-1123.g61431f6b.el8.x86_64 30/32 2023-09-23T20:32:41.617 INFO:teuthology.orchestra.run.smithi070.stdout: Installing : ceph-mds-2:17.2.6-1123.g61431f6b.el8.x86_64 31/32 2023-09-23T20:32:41.634 INFO:teuthology.orchestra.run.smithi070.stdout: Running scriptlet: ceph-mds-2:17.2.6-1123.g61431f6b.el8.x86_64 31/32 2023-09-23T20:32:41.748 INFO:teuthology.orchestra.run.smithi070.stdout: Installing : ceph-2:17.2.6-1123.g61431f6b.el8.x86_64 32/32 2023-09-23T20:32:42.096 INFO:teuthology.orchestra.run.smithi070.stdout: Running scriptlet: ceph-2:17.2.6-1123.g61431f6b.el8.x86_64 32/32 2023-09-23T20:32:42.096 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : ceph-2:17.2.6-1123.g61431f6b.el8.x86_64 1/32 2023-09-23T20:32:42.096 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : ceph-mds-2:17.2.6-1123.g61431f6b.el8.x86_64 2/32 2023-09-23T20:32:42.096 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : ceph-mgr-2:17.2.6-1123.g61431f6b.el8.x86_64 3/32 2023-09-23T20:32:42.096 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : ceph-mon-2:17.2.6-1123.g61431f6b.el8.x86_64 4/32 2023-09-23T20:32:42.096 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : ceph-osd-2:17.2.6-1123.g61431f6b.el8.x86_64 5/32 2023-09-23T20:32:42.096 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : ceph-mgr-modules-core-2:17.2.6-1123.g61431f6b.el8. 6/32 2023-09-23T20:32:42.097 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : python3-bcrypt-3.1.6-2.el8.1.x86_64 7/32 2023-09-23T20:32:42.097 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : python3-beautifulsoup4-4.6.3-2.el8.1.noarch 8/32 2023-09-23T20:32:42.097 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : python3-cheroot-8.5.2-1.el8.noarch 9/32 2023-09-23T20:32:42.097 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : python3-cherrypy-18.4.0-1.el8.noarch 10/32 2023-09-23T20:32:42.097 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : python3-jaraco-6.2-6.el8.noarch 11/32 2023-09-23T20:32:42.097 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : python3-jaraco-functools-2.0-4.el8.noarch 12/32 2023-09-23T20:32:42.097 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : python3-logutils-0.3.5-11.el8.noarch 13/32 2023-09-23T20:32:42.097 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : python3-more-itertools-7.2.0-3.el8.noarch 14/32 2023-09-23T20:32:42.097 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : python3-pecan-1.3.2-9.el8.noarch 15/32 2023-09-23T20:32:42.097 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : python3-portend-2.6-1.el8.noarch 16/32 2023-09-23T20:32:42.097 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : python3-simplegeneric-0.8.1-17.el8.noarch 17/32 2023-09-23T20:32:42.097 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : python3-singledispatch-3.4.0.3-18.el8.noarch 18/32 2023-09-23T20:32:42.098 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : python3-tempora-1.14.1-5.el8.noarch 19/32 2023-09-23T20:32:42.098 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : python3-trustme-0.6.0-4.el8.noarch 20/32 2023-09-23T20:32:42.098 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : python3-waitress-1.4.3-1.el8.noarch 21/32 2023-09-23T20:32:42.098 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : python3-webob-1.8.5-1.el8.1.noarch 22/32 2023-09-23T20:32:42.098 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : python3-webtest-2.0.33-1.el8.noarch 23/32 2023-09-23T20:32:42.098 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : python3-zc-lockfile-2.0-2.el8.noarch 24/32 2023-09-23T20:32:42.098 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : python3-pycparser-2.14-14.el8.noarch 25/32 2023-09-23T20:32:42.098 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : python3-cryptography-3.2.1-5.el8.x86_64 26/32 2023-09-23T20:32:42.098 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : python3-cffi-1.11.5-5.el8.x86_64 27/32 2023-09-23T20:32:42.098 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : python3-pytz-2017.2-9.el8.noarch 28/32 2023-09-23T20:32:42.098 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : python3-markupsafe-0.23-19.el8.x86_64 29/32 2023-09-23T20:32:42.099 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : python3-mako-1.0.6-13.el8.noarch 30/32 2023-09-23T20:32:42.099 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : python3-werkzeug-0.12.2-4.el8.noarch 31/32 2023-09-23T20:32:42.150 INFO:teuthology.orchestra.run.smithi026.stdout: Installing : python3-jaraco-6.2-6.el8.noarch 18/32 2023-09-23T20:32:42.342 INFO:teuthology.orchestra.run.smithi026.stdout: Installing : python3-jaraco-functools-2.0-4.el8.noarch 19/32 2023-09-23T20:32:42.531 INFO:teuthology.orchestra.run.smithi026.stdout: Installing : python3-cheroot-8.5.2-1.el8.noarch 20/32 2023-09-23T20:32:42.699 INFO:teuthology.orchestra.run.smithi026.stdout: Installing : python3-tempora-1.14.1-5.el8.noarch 21/32 2023-09-23T20:32:43.012 INFO:teuthology.orchestra.run.smithi026.stdout: Installing : python3-portend-2.6-1.el8.noarch 22/32 2023-09-23T20:32:43.252 INFO:teuthology.orchestra.run.smithi026.stdout: Installing : python3-cherrypy-18.4.0-1.el8.noarch 23/32 2023-09-23T20:32:43.463 INFO:teuthology.orchestra.run.smithi026.stdout: Installing : python3-beautifulsoup4-4.6.3-2.el8.1.noarch 24/32 2023-09-23T20:32:43.493 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : python3-pyOpenSSL-19.0.0-1.el8.noarch 32/32 2023-09-23T20:32:43.493 INFO:teuthology.orchestra.run.smithi070.stdout:Last metadata expiration check: 0:01:27 ago on Sat 23 Sep 2023 08:31:16 PM UTC. 2023-09-23T20:32:43.762 INFO:teuthology.orchestra.run.smithi026.stdout: Installing : python3-webtest-2.0.33-1.el8.noarch 25/32 2023-09-23T20:32:44.073 INFO:teuthology.orchestra.run.smithi026.stdout: Installing : python3-pecan-1.3.2-9.el8.noarch 26/32 2023-09-23T20:32:44.416 INFO:teuthology.orchestra.run.smithi026.stdout: Installing : ceph-mgr-modules-core-2:17.2.6-1123.g61431f6b.el8. 27/32 2023-09-23T20:32:44.602 INFO:teuthology.orchestra.run.smithi026.stdout: Installing : ceph-mgr-2:17.2.6-1123.g61431f6b.el8.x86_64 28/32 2023-09-23T20:32:45.843 INFO:teuthology.orchestra.run.smithi070.stdout:Installed products updated. 2023-09-23T20:32:46.035 INFO:teuthology.orchestra.run.smithi070.stdout: 2023-09-23T20:32:46.036 INFO:teuthology.orchestra.run.smithi070.stdout:Installed: 2023-09-23T20:32:46.036 INFO:teuthology.orchestra.run.smithi070.stdout: ceph-2:17.2.6-1123.g61431f6b.el8.x86_64 2023-09-23T20:32:46.036 INFO:teuthology.orchestra.run.smithi070.stdout: ceph-mds-2:17.2.6-1123.g61431f6b.el8.x86_64 2023-09-23T20:32:46.036 INFO:teuthology.orchestra.run.smithi070.stdout: ceph-mgr-2:17.2.6-1123.g61431f6b.el8.x86_64 2023-09-23T20:32:46.036 INFO:teuthology.orchestra.run.smithi070.stdout: ceph-mgr-modules-core-2:17.2.6-1123.g61431f6b.el8.noarch 2023-09-23T20:32:46.036 INFO:teuthology.orchestra.run.smithi070.stdout: ceph-mon-2:17.2.6-1123.g61431f6b.el8.x86_64 2023-09-23T20:32:46.036 INFO:teuthology.orchestra.run.smithi070.stdout: ceph-osd-2:17.2.6-1123.g61431f6b.el8.x86_64 2023-09-23T20:32:46.036 INFO:teuthology.orchestra.run.smithi070.stdout: python3-bcrypt-3.1.6-2.el8.1.x86_64 2023-09-23T20:32:46.036 INFO:teuthology.orchestra.run.smithi070.stdout: python3-beautifulsoup4-4.6.3-2.el8.1.noarch 2023-09-23T20:32:46.036 INFO:teuthology.orchestra.run.smithi070.stdout: python3-cffi-1.11.5-5.el8.x86_64 2023-09-23T20:32:46.036 INFO:teuthology.orchestra.run.smithi070.stdout: python3-cheroot-8.5.2-1.el8.noarch 2023-09-23T20:32:46.036 INFO:teuthology.orchestra.run.smithi070.stdout: python3-cherrypy-18.4.0-1.el8.noarch 2023-09-23T20:32:46.037 INFO:teuthology.orchestra.run.smithi070.stdout: python3-cryptography-3.2.1-5.el8.x86_64 2023-09-23T20:32:46.037 INFO:teuthology.orchestra.run.smithi070.stdout: python3-jaraco-6.2-6.el8.noarch 2023-09-23T20:32:46.037 INFO:teuthology.orchestra.run.smithi070.stdout: python3-jaraco-functools-2.0-4.el8.noarch 2023-09-23T20:32:46.037 INFO:teuthology.orchestra.run.smithi070.stdout: python3-logutils-0.3.5-11.el8.noarch 2023-09-23T20:32:46.037 INFO:teuthology.orchestra.run.smithi070.stdout: python3-mako-1.0.6-13.el8.noarch 2023-09-23T20:32:46.037 INFO:teuthology.orchestra.run.smithi070.stdout: python3-markupsafe-0.23-19.el8.x86_64 2023-09-23T20:32:46.037 INFO:teuthology.orchestra.run.smithi070.stdout: python3-more-itertools-7.2.0-3.el8.noarch 2023-09-23T20:32:46.037 INFO:teuthology.orchestra.run.smithi070.stdout: python3-pecan-1.3.2-9.el8.noarch 2023-09-23T20:32:46.037 INFO:teuthology.orchestra.run.smithi070.stdout: python3-portend-2.6-1.el8.noarch 2023-09-23T20:32:46.037 INFO:teuthology.orchestra.run.smithi070.stdout: python3-pyOpenSSL-19.0.0-1.el8.noarch 2023-09-23T20:32:46.037 INFO:teuthology.orchestra.run.smithi070.stdout: python3-pycparser-2.14-14.el8.noarch 2023-09-23T20:32:46.037 INFO:teuthology.orchestra.run.smithi070.stdout: python3-pytz-2017.2-9.el8.noarch 2023-09-23T20:32:46.038 INFO:teuthology.orchestra.run.smithi070.stdout: python3-simplegeneric-0.8.1-17.el8.noarch 2023-09-23T20:32:46.038 INFO:teuthology.orchestra.run.smithi070.stdout: python3-singledispatch-3.4.0.3-18.el8.noarch 2023-09-23T20:32:46.038 INFO:teuthology.orchestra.run.smithi070.stdout: python3-tempora-1.14.1-5.el8.noarch 2023-09-23T20:32:46.038 INFO:teuthology.orchestra.run.smithi070.stdout: python3-trustme-0.6.0-4.el8.noarch 2023-09-23T20:32:46.038 INFO:teuthology.orchestra.run.smithi070.stdout: python3-waitress-1.4.3-1.el8.noarch 2023-09-23T20:32:46.038 INFO:teuthology.orchestra.run.smithi070.stdout: python3-webob-1.8.5-1.el8.1.noarch 2023-09-23T20:32:46.038 INFO:teuthology.orchestra.run.smithi070.stdout: python3-webtest-2.0.33-1.el8.noarch 2023-09-23T20:32:46.038 INFO:teuthology.orchestra.run.smithi070.stdout: python3-werkzeug-0.12.2-4.el8.noarch 2023-09-23T20:32:46.038 INFO:teuthology.orchestra.run.smithi070.stdout: python3-zc-lockfile-2.0-2.el8.noarch 2023-09-23T20:32:46.038 INFO:teuthology.orchestra.run.smithi070.stdout: 2023-09-23T20:32:46.038 INFO:teuthology.orchestra.run.smithi070.stdout:Complete! 2023-09-23T20:32:46.175 DEBUG:teuthology.orchestra.run.smithi070:> sudo yum -y install ceph-base 2023-09-23T20:32:46.182 INFO:teuthology.orchestra.run.smithi026.stdout: Running scriptlet: ceph-mgr-2:17.2.6-1123.g61431f6b.el8.x86_64 28/32 2023-09-23T20:32:46.385 INFO:teuthology.orchestra.run.smithi026.stdout: Installing : ceph-osd-2:17.2.6-1123.g61431f6b.el8.x86_64 29/32 2023-09-23T20:32:46.524 INFO:teuthology.orchestra.run.smithi070.stdout:Updating Subscription Management repositories. 2023-09-23T20:32:46.807 INFO:teuthology.orchestra.run.smithi026.stdout: Running scriptlet: ceph-osd-2:17.2.6-1123.g61431f6b.el8.x86_64 29/32 2023-09-23T20:32:46.959 INFO:teuthology.orchestra.run.smithi026.stdout: Installing : ceph-mon-2:17.2.6-1123.g61431f6b.el8.x86_64 30/32 2023-09-23T20:32:47.188 INFO:teuthology.orchestra.run.smithi026.stdout: Running scriptlet: ceph-mon-2:17.2.6-1123.g61431f6b.el8.x86_64 30/32 2023-09-23T20:32:47.387 INFO:teuthology.orchestra.run.smithi070.stdout:Last metadata expiration check: 0:01:31 ago on Sat 23 Sep 2023 08:31:16 PM UTC. 2023-09-23T20:32:47.609 INFO:teuthology.orchestra.run.smithi026.stdout: Installing : ceph-mds-2:17.2.6-1123.g61431f6b.el8.x86_64 31/32 2023-09-23T20:32:47.629 INFO:teuthology.orchestra.run.smithi026.stdout: Running scriptlet: ceph-mds-2:17.2.6-1123.g61431f6b.el8.x86_64 31/32 2023-09-23T20:32:47.766 INFO:teuthology.orchestra.run.smithi026.stdout: Installing : ceph-2:17.2.6-1123.g61431f6b.el8.x86_64 32/32 2023-09-23T20:32:48.132 INFO:teuthology.orchestra.run.smithi026.stdout: Running scriptlet: ceph-2:17.2.6-1123.g61431f6b.el8.x86_64 32/32 2023-09-23T20:32:48.132 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : ceph-2:17.2.6-1123.g61431f6b.el8.x86_64 1/32 2023-09-23T20:32:48.132 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : ceph-mds-2:17.2.6-1123.g61431f6b.el8.x86_64 2/32 2023-09-23T20:32:48.132 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : ceph-mgr-2:17.2.6-1123.g61431f6b.el8.x86_64 3/32 2023-09-23T20:32:48.132 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : ceph-mon-2:17.2.6-1123.g61431f6b.el8.x86_64 4/32 2023-09-23T20:32:48.133 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : ceph-osd-2:17.2.6-1123.g61431f6b.el8.x86_64 5/32 2023-09-23T20:32:48.133 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : ceph-mgr-modules-core-2:17.2.6-1123.g61431f6b.el8. 6/32 2023-09-23T20:32:48.133 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : python3-bcrypt-3.1.6-2.el8.1.x86_64 7/32 2023-09-23T20:32:48.133 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : python3-beautifulsoup4-4.6.3-2.el8.1.noarch 8/32 2023-09-23T20:32:48.133 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : python3-cheroot-8.5.2-1.el8.noarch 9/32 2023-09-23T20:32:48.133 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : python3-cherrypy-18.4.0-1.el8.noarch 10/32 2023-09-23T20:32:48.133 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : python3-jaraco-6.2-6.el8.noarch 11/32 2023-09-23T20:32:48.133 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : python3-jaraco-functools-2.0-4.el8.noarch 12/32 2023-09-23T20:32:48.133 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : python3-logutils-0.3.5-11.el8.noarch 13/32 2023-09-23T20:32:48.133 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : python3-more-itertools-7.2.0-3.el8.noarch 14/32 2023-09-23T20:32:48.133 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : python3-pecan-1.3.2-9.el8.noarch 15/32 2023-09-23T20:32:48.133 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : python3-portend-2.6-1.el8.noarch 16/32 2023-09-23T20:32:48.134 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : python3-simplegeneric-0.8.1-17.el8.noarch 17/32 2023-09-23T20:32:48.134 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : python3-singledispatch-3.4.0.3-18.el8.noarch 18/32 2023-09-23T20:32:48.134 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : python3-tempora-1.14.1-5.el8.noarch 19/32 2023-09-23T20:32:48.134 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : python3-trustme-0.6.0-4.el8.noarch 20/32 2023-09-23T20:32:48.134 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : python3-waitress-1.4.3-1.el8.noarch 21/32 2023-09-23T20:32:48.134 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : python3-webob-1.8.5-1.el8.1.noarch 22/32 2023-09-23T20:32:48.134 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : python3-webtest-2.0.33-1.el8.noarch 23/32 2023-09-23T20:32:48.134 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : python3-zc-lockfile-2.0-2.el8.noarch 24/32 2023-09-23T20:32:48.134 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : python3-pycparser-2.14-14.el8.noarch 25/32 2023-09-23T20:32:48.134 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : python3-cryptography-3.2.1-5.el8.x86_64 26/32 2023-09-23T20:32:48.134 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : python3-cffi-1.11.5-5.el8.x86_64 27/32 2023-09-23T20:32:48.134 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : python3-pytz-2017.2-9.el8.noarch 28/32 2023-09-23T20:32:48.135 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : python3-markupsafe-0.23-19.el8.x86_64 29/32 2023-09-23T20:32:48.135 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : python3-mako-1.0.6-13.el8.noarch 30/32 2023-09-23T20:32:48.135 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : python3-werkzeug-0.12.2-4.el8.noarch 31/32 2023-09-23T20:32:48.747 INFO:teuthology.orchestra.run.smithi070.stdout:Package ceph-base-2:17.2.6-1123.g61431f6b.el8.x86_64 is already installed. 2023-09-23T20:32:48.856 INFO:teuthology.orchestra.run.smithi070.stdout:Dependencies resolved. 2023-09-23T20:32:48.857 INFO:teuthology.orchestra.run.smithi070.stdout:Nothing to do. 2023-09-23T20:32:48.858 INFO:teuthology.orchestra.run.smithi070.stdout:Complete! 2023-09-23T20:32:49.298 DEBUG:teuthology.orchestra.run.smithi070:> sudo yum -y install cephadm 2023-09-23T20:32:49.642 INFO:teuthology.orchestra.run.smithi070.stdout:Updating Subscription Management repositories. 2023-09-23T20:32:49.713 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : python3-pyOpenSSL-19.0.0-1.el8.noarch 32/32 2023-09-23T20:32:49.713 INFO:teuthology.orchestra.run.smithi026.stdout:Last metadata expiration check: 0:01:31 ago on Sat 23 Sep 2023 08:31:18 PM UTC. 2023-09-23T20:32:50.458 INFO:teuthology.orchestra.run.smithi070.stdout:Last metadata expiration check: 0:01:34 ago on Sat 23 Sep 2023 08:31:16 PM UTC. 2023-09-23T20:32:51.916 INFO:teuthology.orchestra.run.smithi070.stdout:Dependencies resolved. 2023-09-23T20:32:51.916 INFO:teuthology.orchestra.run.smithi070.stdout:================================================================================ 2023-09-23T20:32:51.916 INFO:teuthology.orchestra.run.smithi070.stdout: Package Arch Version Repository Size 2023-09-23T20:32:51.916 INFO:teuthology.orchestra.run.smithi070.stdout:================================================================================ 2023-09-23T20:32:51.916 INFO:teuthology.orchestra.run.smithi070.stdout:Installing: 2023-09-23T20:32:51.917 INFO:teuthology.orchestra.run.smithi070.stdout: cephadm noarch 2:17.2.6-1123.g61431f6b.el8 ceph-noarch 80 k 2023-09-23T20:32:51.917 INFO:teuthology.orchestra.run.smithi070.stdout: 2023-09-23T20:32:51.917 INFO:teuthology.orchestra.run.smithi070.stdout:Transaction Summary 2023-09-23T20:32:51.917 INFO:teuthology.orchestra.run.smithi070.stdout:================================================================================ 2023-09-23T20:32:51.917 INFO:teuthology.orchestra.run.smithi070.stdout:Install 1 Package 2023-09-23T20:32:51.917 INFO:teuthology.orchestra.run.smithi070.stdout: 2023-09-23T20:32:51.917 INFO:teuthology.orchestra.run.smithi070.stdout:Total download size: 80 k 2023-09-23T20:32:51.918 INFO:teuthology.orchestra.run.smithi070.stdout:Installed size: 363 k 2023-09-23T20:32:51.918 INFO:teuthology.orchestra.run.smithi070.stdout:Downloading Packages: 2023-09-23T20:32:52.093 INFO:teuthology.orchestra.run.smithi070.stdout:cephadm-17.2.6-1123.g61431f6b.el8.noarch.rpm 456 kB/s | 80 kB 00:00 2023-09-23T20:32:52.094 INFO:teuthology.orchestra.run.smithi070.stdout:-------------------------------------------------------------------------------- 2023-09-23T20:32:52.094 INFO:teuthology.orchestra.run.smithi070.stdout:Total 453 kB/s | 80 kB 00:00 2023-09-23T20:32:52.094 INFO:teuthology.orchestra.run.smithi070.stdout:Running transaction check 2023-09-23T20:32:52.102 INFO:teuthology.orchestra.run.smithi070.stdout:Transaction check succeeded. 2023-09-23T20:32:52.102 INFO:teuthology.orchestra.run.smithi070.stdout:Running transaction test 2023-09-23T20:32:52.104 INFO:teuthology.orchestra.run.smithi070.stdout:Transaction test succeeded. 2023-09-23T20:32:52.108 INFO:teuthology.orchestra.run.smithi070.stdout:Running transaction 2023-09-23T20:32:52.155 INFO:teuthology.orchestra.run.smithi070.stdout: Preparing : 1/1 2023-09-23T20:32:52.301 INFO:teuthology.orchestra.run.smithi026.stdout:Installed products updated. 2023-09-23T20:32:52.507 INFO:teuthology.orchestra.run.smithi026.stdout: 2023-09-23T20:32:52.507 INFO:teuthology.orchestra.run.smithi026.stdout:Installed: 2023-09-23T20:32:52.507 INFO:teuthology.orchestra.run.smithi026.stdout: ceph-2:17.2.6-1123.g61431f6b.el8.x86_64 2023-09-23T20:32:52.507 INFO:teuthology.orchestra.run.smithi026.stdout: ceph-mds-2:17.2.6-1123.g61431f6b.el8.x86_64 2023-09-23T20:32:52.507 INFO:teuthology.orchestra.run.smithi026.stdout: ceph-mgr-2:17.2.6-1123.g61431f6b.el8.x86_64 2023-09-23T20:32:52.507 INFO:teuthology.orchestra.run.smithi026.stdout: ceph-mgr-modules-core-2:17.2.6-1123.g61431f6b.el8.noarch 2023-09-23T20:32:52.508 INFO:teuthology.orchestra.run.smithi026.stdout: ceph-mon-2:17.2.6-1123.g61431f6b.el8.x86_64 2023-09-23T20:32:52.508 INFO:teuthology.orchestra.run.smithi026.stdout: ceph-osd-2:17.2.6-1123.g61431f6b.el8.x86_64 2023-09-23T20:32:52.508 INFO:teuthology.orchestra.run.smithi026.stdout: python3-bcrypt-3.1.6-2.el8.1.x86_64 2023-09-23T20:32:52.508 INFO:teuthology.orchestra.run.smithi026.stdout: python3-beautifulsoup4-4.6.3-2.el8.1.noarch 2023-09-23T20:32:52.508 INFO:teuthology.orchestra.run.smithi026.stdout: python3-cffi-1.11.5-5.el8.x86_64 2023-09-23T20:32:52.508 INFO:teuthology.orchestra.run.smithi026.stdout: python3-cheroot-8.5.2-1.el8.noarch 2023-09-23T20:32:52.508 INFO:teuthology.orchestra.run.smithi026.stdout: python3-cherrypy-18.4.0-1.el8.noarch 2023-09-23T20:32:52.508 INFO:teuthology.orchestra.run.smithi026.stdout: python3-cryptography-3.2.1-5.el8.x86_64 2023-09-23T20:32:52.508 INFO:teuthology.orchestra.run.smithi026.stdout: python3-jaraco-6.2-6.el8.noarch 2023-09-23T20:32:52.508 INFO:teuthology.orchestra.run.smithi026.stdout: python3-jaraco-functools-2.0-4.el8.noarch 2023-09-23T20:32:52.508 INFO:teuthology.orchestra.run.smithi026.stdout: python3-logutils-0.3.5-11.el8.noarch 2023-09-23T20:32:52.508 INFO:teuthology.orchestra.run.smithi026.stdout: python3-mako-1.0.6-13.el8.noarch 2023-09-23T20:32:52.509 INFO:teuthology.orchestra.run.smithi026.stdout: python3-markupsafe-0.23-19.el8.x86_64 2023-09-23T20:32:52.509 INFO:teuthology.orchestra.run.smithi026.stdout: python3-more-itertools-7.2.0-3.el8.noarch 2023-09-23T20:32:52.509 INFO:teuthology.orchestra.run.smithi026.stdout: python3-pecan-1.3.2-9.el8.noarch 2023-09-23T20:32:52.509 INFO:teuthology.orchestra.run.smithi026.stdout: python3-portend-2.6-1.el8.noarch 2023-09-23T20:32:52.509 INFO:teuthology.orchestra.run.smithi026.stdout: python3-pyOpenSSL-19.0.0-1.el8.noarch 2023-09-23T20:32:52.509 INFO:teuthology.orchestra.run.smithi026.stdout: python3-pycparser-2.14-14.el8.noarch 2023-09-23T20:32:52.509 INFO:teuthology.orchestra.run.smithi026.stdout: python3-pytz-2017.2-9.el8.noarch 2023-09-23T20:32:52.509 INFO:teuthology.orchestra.run.smithi026.stdout: python3-simplegeneric-0.8.1-17.el8.noarch 2023-09-23T20:32:52.509 INFO:teuthology.orchestra.run.smithi026.stdout: python3-singledispatch-3.4.0.3-18.el8.noarch 2023-09-23T20:32:52.509 INFO:teuthology.orchestra.run.smithi026.stdout: python3-tempora-1.14.1-5.el8.noarch 2023-09-23T20:32:52.509 INFO:teuthology.orchestra.run.smithi026.stdout: python3-trustme-0.6.0-4.el8.noarch 2023-09-23T20:32:52.509 INFO:teuthology.orchestra.run.smithi026.stdout: python3-waitress-1.4.3-1.el8.noarch 2023-09-23T20:32:52.509 INFO:teuthology.orchestra.run.smithi026.stdout: python3-webob-1.8.5-1.el8.1.noarch 2023-09-23T20:32:52.510 INFO:teuthology.orchestra.run.smithi026.stdout: python3-webtest-2.0.33-1.el8.noarch 2023-09-23T20:32:52.510 INFO:teuthology.orchestra.run.smithi026.stdout: python3-werkzeug-0.12.2-4.el8.noarch 2023-09-23T20:32:52.510 INFO:teuthology.orchestra.run.smithi026.stdout: python3-zc-lockfile-2.0-2.el8.noarch 2023-09-23T20:32:52.510 INFO:teuthology.orchestra.run.smithi026.stdout: 2023-09-23T20:32:52.510 INFO:teuthology.orchestra.run.smithi026.stdout:Complete! 2023-09-23T20:32:52.737 DEBUG:teuthology.orchestra.run.smithi026:> sudo yum -y install ceph-base 2023-09-23T20:32:53.107 INFO:teuthology.orchestra.run.smithi026.stdout:Updating Subscription Management repositories. 2023-09-23T20:32:53.836 INFO:teuthology.orchestra.run.smithi070.stdout: Running scriptlet: cephadm-2:17.2.6-1123.g61431f6b.el8.noarch 1/1 2023-09-23T20:32:53.927 INFO:teuthology.orchestra.run.smithi026.stdout:Last metadata expiration check: 0:01:35 ago on Sat 23 Sep 2023 08:31:18 PM UTC. 2023-09-23T20:32:54.001 INFO:teuthology.orchestra.run.smithi070.stdout: Installing : cephadm-2:17.2.6-1123.g61431f6b.el8.noarch 1/1 2023-09-23T20:32:54.166 INFO:teuthology.orchestra.run.smithi070.stdout: Running scriptlet: cephadm-2:17.2.6-1123.g61431f6b.el8.noarch 1/1 2023-09-23T20:32:55.244 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : cephadm-2:17.2.6-1123.g61431f6b.el8.noarch 1/1 2023-09-23T20:32:55.245 INFO:teuthology.orchestra.run.smithi070.stdout:Last metadata expiration check: 0:01:39 ago on Sat 23 Sep 2023 08:31:16 PM UTC. 2023-09-23T20:32:55.348 INFO:teuthology.orchestra.run.smithi026.stdout:Package ceph-base-2:17.2.6-1123.g61431f6b.el8.x86_64 is already installed. 2023-09-23T20:32:55.473 INFO:teuthology.orchestra.run.smithi026.stdout:Dependencies resolved. 2023-09-23T20:32:55.474 INFO:teuthology.orchestra.run.smithi026.stdout:Nothing to do. 2023-09-23T20:32:55.474 INFO:teuthology.orchestra.run.smithi026.stdout:Complete! 2023-09-23T20:32:55.572 DEBUG:teuthology.orchestra.run.smithi026:> sudo yum -y install cephadm 2023-09-23T20:32:55.937 INFO:teuthology.orchestra.run.smithi026.stdout:Updating Subscription Management repositories. 2023-09-23T20:32:56.870 INFO:teuthology.orchestra.run.smithi026.stdout:Last metadata expiration check: 0:01:38 ago on Sat 23 Sep 2023 08:31:18 PM UTC. 2023-09-23T20:32:57.665 INFO:teuthology.orchestra.run.smithi070.stdout:Installed products updated. 2023-09-23T20:32:57.847 INFO:teuthology.orchestra.run.smithi070.stdout: 2023-09-23T20:32:57.848 INFO:teuthology.orchestra.run.smithi070.stdout:Installed: 2023-09-23T20:32:57.848 INFO:teuthology.orchestra.run.smithi070.stdout: cephadm-2:17.2.6-1123.g61431f6b.el8.noarch 2023-09-23T20:32:57.848 INFO:teuthology.orchestra.run.smithi070.stdout: 2023-09-23T20:32:57.848 INFO:teuthology.orchestra.run.smithi070.stdout:Complete! 2023-09-23T20:32:57.973 DEBUG:teuthology.orchestra.run.smithi070:> sudo yum -y install ceph-immutable-object-cache 2023-09-23T20:32:58.328 INFO:teuthology.orchestra.run.smithi070.stdout:Updating Subscription Management repositories. 2023-09-23T20:32:58.444 INFO:teuthology.orchestra.run.smithi026.stdout:Dependencies resolved. 2023-09-23T20:32:58.445 INFO:teuthology.orchestra.run.smithi026.stdout:================================================================================ 2023-09-23T20:32:58.445 INFO:teuthology.orchestra.run.smithi026.stdout: Package Arch Version Repository Size 2023-09-23T20:32:58.445 INFO:teuthology.orchestra.run.smithi026.stdout:================================================================================ 2023-09-23T20:32:58.445 INFO:teuthology.orchestra.run.smithi026.stdout:Installing: 2023-09-23T20:32:58.445 INFO:teuthology.orchestra.run.smithi026.stdout: cephadm noarch 2:17.2.6-1123.g61431f6b.el8 ceph-noarch 80 k 2023-09-23T20:32:58.445 INFO:teuthology.orchestra.run.smithi026.stdout: 2023-09-23T20:32:58.445 INFO:teuthology.orchestra.run.smithi026.stdout:Transaction Summary 2023-09-23T20:32:58.445 INFO:teuthology.orchestra.run.smithi026.stdout:================================================================================ 2023-09-23T20:32:58.445 INFO:teuthology.orchestra.run.smithi026.stdout:Install 1 Package 2023-09-23T20:32:58.445 INFO:teuthology.orchestra.run.smithi026.stdout: 2023-09-23T20:32:58.446 INFO:teuthology.orchestra.run.smithi026.stdout:Total download size: 80 k 2023-09-23T20:32:58.446 INFO:teuthology.orchestra.run.smithi026.stdout:Installed size: 363 k 2023-09-23T20:32:58.446 INFO:teuthology.orchestra.run.smithi026.stdout:Downloading Packages: 2023-09-23T20:32:59.007 INFO:teuthology.orchestra.run.smithi026.stdout:cephadm-17.2.6-1123.g61431f6b.el8.noarch.rpm 143 kB/s | 80 kB 00:00 2023-09-23T20:32:59.007 INFO:teuthology.orchestra.run.smithi026.stdout:-------------------------------------------------------------------------------- 2023-09-23T20:32:59.008 INFO:teuthology.orchestra.run.smithi026.stdout:Total 142 kB/s | 80 kB 00:00 2023-09-23T20:32:59.008 INFO:teuthology.orchestra.run.smithi026.stdout:Running transaction check 2023-09-23T20:32:59.018 INFO:teuthology.orchestra.run.smithi026.stdout:Transaction check succeeded. 2023-09-23T20:32:59.018 INFO:teuthology.orchestra.run.smithi026.stdout:Running transaction test 2023-09-23T20:32:59.020 INFO:teuthology.orchestra.run.smithi026.stdout:Transaction test succeeded. 2023-09-23T20:32:59.025 INFO:teuthology.orchestra.run.smithi026.stdout:Running transaction 2023-09-23T20:32:59.157 INFO:teuthology.orchestra.run.smithi070.stdout:Last metadata expiration check: 0:01:43 ago on Sat 23 Sep 2023 08:31:16 PM UTC. 2023-09-23T20:32:59.482 INFO:teuthology.orchestra.run.smithi026.stdout: Preparing : 1/1 2023-09-23T20:33:00.537 INFO:teuthology.orchestra.run.smithi070.stdout:Dependencies resolved. 2023-09-23T20:33:00.537 INFO:teuthology.orchestra.run.smithi070.stdout:================================================================================ 2023-09-23T20:33:00.537 INFO:teuthology.orchestra.run.smithi070.stdout: Package Arch Version Repo Size 2023-09-23T20:33:00.537 INFO:teuthology.orchestra.run.smithi070.stdout:================================================================================ 2023-09-23T20:33:00.537 INFO:teuthology.orchestra.run.smithi070.stdout:Installing: 2023-09-23T20:33:00.537 INFO:teuthology.orchestra.run.smithi070.stdout: ceph-immutable-object-cache x86_64 2:17.2.6-1123.g61431f6b.el8 ceph 161 k 2023-09-23T20:33:00.538 INFO:teuthology.orchestra.run.smithi070.stdout: 2023-09-23T20:33:00.538 INFO:teuthology.orchestra.run.smithi070.stdout:Transaction Summary 2023-09-23T20:33:00.538 INFO:teuthology.orchestra.run.smithi070.stdout:================================================================================ 2023-09-23T20:33:00.538 INFO:teuthology.orchestra.run.smithi070.stdout:Install 1 Package 2023-09-23T20:33:00.538 INFO:teuthology.orchestra.run.smithi070.stdout: 2023-09-23T20:33:00.538 INFO:teuthology.orchestra.run.smithi070.stdout:Total download size: 161 k 2023-09-23T20:33:00.539 INFO:teuthology.orchestra.run.smithi070.stdout:Installed size: 490 k 2023-09-23T20:33:00.539 INFO:teuthology.orchestra.run.smithi070.stdout:Downloading Packages: 2023-09-23T20:33:00.832 INFO:teuthology.orchestra.run.smithi070.stdout:ceph-immutable-object-cache-17.2.6-1123.g61431f 550 kB/s | 161 kB 00:00 2023-09-23T20:33:00.833 INFO:teuthology.orchestra.run.smithi070.stdout:-------------------------------------------------------------------------------- 2023-09-23T20:33:00.833 INFO:teuthology.orchestra.run.smithi070.stdout:Total 547 kB/s | 161 kB 00:00 2023-09-23T20:33:00.833 INFO:teuthology.orchestra.run.smithi070.stdout:Running transaction check 2023-09-23T20:33:00.843 INFO:teuthology.orchestra.run.smithi070.stdout:Transaction check succeeded. 2023-09-23T20:33:00.844 INFO:teuthology.orchestra.run.smithi070.stdout:Running transaction test 2023-09-23T20:33:00.896 INFO:teuthology.orchestra.run.smithi070.stdout:Transaction test succeeded. 2023-09-23T20:33:00.900 INFO:teuthology.orchestra.run.smithi070.stdout:Running transaction 2023-09-23T20:33:01.013 INFO:teuthology.orchestra.run.smithi070.stdout: Preparing : 1/1 2023-09-23T20:33:01.163 INFO:teuthology.orchestra.run.smithi070.stdout: Installing : ceph-immutable-object-cache-2:17.2.6-1123.g61431f6b. 1/1 2023-09-23T20:33:01.332 INFO:teuthology.orchestra.run.smithi026.stdout: Running scriptlet: cephadm-2:17.2.6-1123.g61431f6b.el8.noarch 1/1 2023-09-23T20:33:01.513 INFO:teuthology.orchestra.run.smithi070.stdout: Running scriptlet: ceph-immutable-object-cache-2:17.2.6-1123.g61431f6b. 1/1 2023-09-23T20:33:01.522 INFO:teuthology.orchestra.run.smithi026.stdout: Installing : cephadm-2:17.2.6-1123.g61431f6b.el8.noarch 1/1 2023-09-23T20:33:01.696 INFO:teuthology.orchestra.run.smithi026.stdout: Running scriptlet: cephadm-2:17.2.6-1123.g61431f6b.el8.noarch 1/1 2023-09-23T20:33:02.541 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : ceph-immutable-object-cache-2:17.2.6-1123.g61431f6b. 1/1 2023-09-23T20:33:02.541 INFO:teuthology.orchestra.run.smithi070.stdout:Last metadata expiration check: 0:01:46 ago on Sat 23 Sep 2023 08:31:16 PM UTC. 2023-09-23T20:33:02.787 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : cephadm-2:17.2.6-1123.g61431f6b.el8.noarch 1/1 2023-09-23T20:33:02.787 INFO:teuthology.orchestra.run.smithi026.stdout:Last metadata expiration check: 0:01:44 ago on Sat 23 Sep 2023 08:31:18 PM UTC. 2023-09-23T20:33:05.154 INFO:teuthology.orchestra.run.smithi070.stdout:Installed products updated. 2023-09-23T20:33:05.337 INFO:teuthology.orchestra.run.smithi070.stdout: 2023-09-23T20:33:05.338 INFO:teuthology.orchestra.run.smithi070.stdout:Installed: 2023-09-23T20:33:05.338 INFO:teuthology.orchestra.run.smithi070.stdout: ceph-immutable-object-cache-2:17.2.6-1123.g61431f6b.el8.x86_64 2023-09-23T20:33:05.338 INFO:teuthology.orchestra.run.smithi070.stdout: 2023-09-23T20:33:05.338 INFO:teuthology.orchestra.run.smithi070.stdout:Complete! 2023-09-23T20:33:05.463 DEBUG:teuthology.orchestra.run.smithi070:> sudo yum -y install ceph-mgr 2023-09-23T20:33:05.667 INFO:teuthology.orchestra.run.smithi026.stdout:Installed products updated. 2023-09-23T20:33:05.808 INFO:teuthology.orchestra.run.smithi070.stdout:Updating Subscription Management repositories. 2023-09-23T20:33:05.867 INFO:teuthology.orchestra.run.smithi026.stdout: 2023-09-23T20:33:05.868 INFO:teuthology.orchestra.run.smithi026.stdout:Installed: 2023-09-23T20:33:05.868 INFO:teuthology.orchestra.run.smithi026.stdout: cephadm-2:17.2.6-1123.g61431f6b.el8.noarch 2023-09-23T20:33:05.868 INFO:teuthology.orchestra.run.smithi026.stdout: 2023-09-23T20:33:05.868 INFO:teuthology.orchestra.run.smithi026.stdout:Complete! 2023-09-23T20:33:06.109 DEBUG:teuthology.orchestra.run.smithi026:> sudo yum -y install ceph-immutable-object-cache 2023-09-23T20:33:06.491 INFO:teuthology.orchestra.run.smithi026.stdout:Updating Subscription Management repositories. 2023-09-23T20:33:06.618 INFO:teuthology.orchestra.run.smithi070.stdout:Last metadata expiration check: 0:01:50 ago on Sat 23 Sep 2023 08:31:16 PM UTC. 2023-09-23T20:33:07.336 INFO:teuthology.orchestra.run.smithi026.stdout:Last metadata expiration check: 0:01:49 ago on Sat 23 Sep 2023 08:31:18 PM UTC. 2023-09-23T20:33:07.891 INFO:teuthology.orchestra.run.smithi070.stdout:Package ceph-mgr-2:17.2.6-1123.g61431f6b.el8.x86_64 is already installed. 2023-09-23T20:33:08.001 INFO:teuthology.orchestra.run.smithi070.stdout:Dependencies resolved. 2023-09-23T20:33:08.002 INFO:teuthology.orchestra.run.smithi070.stdout:Nothing to do. 2023-09-23T20:33:08.002 INFO:teuthology.orchestra.run.smithi070.stdout:Complete! 2023-09-23T20:33:08.110 DEBUG:teuthology.orchestra.run.smithi070:> sudo yum -y install ceph-mgr-dashboard 2023-09-23T20:33:08.451 INFO:teuthology.orchestra.run.smithi070.stdout:Updating Subscription Management repositories. 2023-09-23T20:33:08.896 INFO:teuthology.orchestra.run.smithi026.stdout:Dependencies resolved. 2023-09-23T20:33:08.896 INFO:teuthology.orchestra.run.smithi026.stdout:================================================================================ 2023-09-23T20:33:08.897 INFO:teuthology.orchestra.run.smithi026.stdout: Package Arch Version Repo Size 2023-09-23T20:33:08.897 INFO:teuthology.orchestra.run.smithi026.stdout:================================================================================ 2023-09-23T20:33:08.897 INFO:teuthology.orchestra.run.smithi026.stdout:Installing: 2023-09-23T20:33:08.897 INFO:teuthology.orchestra.run.smithi026.stdout: ceph-immutable-object-cache x86_64 2:17.2.6-1123.g61431f6b.el8 ceph 161 k 2023-09-23T20:33:08.897 INFO:teuthology.orchestra.run.smithi026.stdout: 2023-09-23T20:33:08.897 INFO:teuthology.orchestra.run.smithi026.stdout:Transaction Summary 2023-09-23T20:33:08.897 INFO:teuthology.orchestra.run.smithi026.stdout:================================================================================ 2023-09-23T20:33:08.897 INFO:teuthology.orchestra.run.smithi026.stdout:Install 1 Package 2023-09-23T20:33:08.897 INFO:teuthology.orchestra.run.smithi026.stdout: 2023-09-23T20:33:08.898 INFO:teuthology.orchestra.run.smithi026.stdout:Total download size: 161 k 2023-09-23T20:33:08.898 INFO:teuthology.orchestra.run.smithi026.stdout:Installed size: 490 k 2023-09-23T20:33:08.898 INFO:teuthology.orchestra.run.smithi026.stdout:Downloading Packages: 2023-09-23T20:33:09.187 INFO:teuthology.orchestra.run.smithi026.stdout:ceph-immutable-object-cache-17.2.6-1123.g61431f 559 kB/s | 161 kB 00:00 2023-09-23T20:33:09.187 INFO:teuthology.orchestra.run.smithi026.stdout:-------------------------------------------------------------------------------- 2023-09-23T20:33:09.187 INFO:teuthology.orchestra.run.smithi026.stdout:Total 556 kB/s | 161 kB 00:00 2023-09-23T20:33:09.188 INFO:teuthology.orchestra.run.smithi026.stdout:Running transaction check 2023-09-23T20:33:09.199 INFO:teuthology.orchestra.run.smithi026.stdout:Transaction check succeeded. 2023-09-23T20:33:09.200 INFO:teuthology.orchestra.run.smithi026.stdout:Running transaction test 2023-09-23T20:33:09.256 INFO:teuthology.orchestra.run.smithi026.stdout:Transaction test succeeded. 2023-09-23T20:33:09.260 INFO:teuthology.orchestra.run.smithi026.stdout:Running transaction 2023-09-23T20:33:09.277 INFO:teuthology.orchestra.run.smithi070.stdout:Last metadata expiration check: 0:01:53 ago on Sat 23 Sep 2023 08:31:16 PM UTC. 2023-09-23T20:33:09.375 INFO:teuthology.orchestra.run.smithi026.stdout: Preparing : 1/1 2023-09-23T20:33:09.573 INFO:teuthology.orchestra.run.smithi026.stdout: Installing : ceph-immutable-object-cache-2:17.2.6-1123.g61431f6b. 1/1 2023-09-23T20:33:10.064 INFO:teuthology.orchestra.run.smithi026.stdout: Running scriptlet: ceph-immutable-object-cache-2:17.2.6-1123.g61431f6b. 1/1 2023-09-23T20:33:10.649 INFO:teuthology.orchestra.run.smithi070.stdout:Dependencies resolved. 2023-09-23T20:33:10.650 INFO:teuthology.orchestra.run.smithi070.stdout:=================================================================================================== 2023-09-23T20:33:10.650 INFO:teuthology.orchestra.run.smithi070.stdout: Package Arch Version Repository Size 2023-09-23T20:33:10.650 INFO:teuthology.orchestra.run.smithi070.stdout:=================================================================================================== 2023-09-23T20:33:10.651 INFO:teuthology.orchestra.run.smithi070.stdout:Installing: 2023-09-23T20:33:10.651 INFO:teuthology.orchestra.run.smithi070.stdout: ceph-mgr-dashboard noarch 2:17.2.6-1123.g61431f6b.el8 ceph-noarch 1.7 M 2023-09-23T20:33:10.651 INFO:teuthology.orchestra.run.smithi070.stdout:Installing dependencies: 2023-09-23T20:33:10.651 INFO:teuthology.orchestra.run.smithi070.stdout: ceph-grafana-dashboards noarch 2:17.2.6-1123.g61431f6b.el8 ceph-noarch 23 k 2023-09-23T20:33:10.651 INFO:teuthology.orchestra.run.smithi070.stdout: ceph-prometheus-alerts noarch 2:17.2.6-1123.g61431f6b.el8 ceph-noarch 14 k 2023-09-23T20:33:10.651 INFO:teuthology.orchestra.run.smithi070.stdout: python3-jwt noarch 1.6.1-2.el8 rhel-8-for-x86_64-baseos-rpms 43 k 2023-09-23T20:33:10.651 INFO:teuthology.orchestra.run.smithi070.stdout: python3-repoze-lru noarch 0.7-6.el8 epel 34 k 2023-09-23T20:33:10.651 INFO:teuthology.orchestra.run.smithi070.stdout: python3-routes noarch 2.4.1-12.el8 epel 196 k 2023-09-23T20:33:10.651 INFO:teuthology.orchestra.run.smithi070.stdout: 2023-09-23T20:33:10.652 INFO:teuthology.orchestra.run.smithi070.stdout:Transaction Summary 2023-09-23T20:33:10.652 INFO:teuthology.orchestra.run.smithi070.stdout:=================================================================================================== 2023-09-23T20:33:10.652 INFO:teuthology.orchestra.run.smithi070.stdout:Install 6 Packages 2023-09-23T20:33:10.652 INFO:teuthology.orchestra.run.smithi070.stdout: 2023-09-23T20:33:10.652 INFO:teuthology.orchestra.run.smithi070.stdout:Total download size: 2.0 M 2023-09-23T20:33:10.653 INFO:teuthology.orchestra.run.smithi070.stdout:Installed size: 8.0 M 2023-09-23T20:33:10.653 INFO:teuthology.orchestra.run.smithi070.stdout:Downloading Packages: 2023-09-23T20:33:11.120 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : ceph-immutable-object-cache-2:17.2.6-1123.g61431f6b. 1/1 2023-09-23T20:33:11.120 INFO:teuthology.orchestra.run.smithi026.stdout:Last metadata expiration check: 0:01:53 ago on Sat 23 Sep 2023 08:31:18 PM UTC. 2023-09-23T20:33:11.194 INFO:teuthology.orchestra.run.smithi070.stdout:(1/6): ceph-grafana-dashboards-17.2.6-1123.g614 102 kB/s | 23 kB 00:00 2023-09-23T20:33:11.219 INFO:teuthology.orchestra.run.smithi070.stdout:(2/6): ceph-prometheus-alerts-17.2.6-1123.g6143 58 kB/s | 14 kB 00:00 2023-09-23T20:33:11.378 INFO:teuthology.orchestra.run.smithi070.stdout:(3/6): python3-repoze-lru-0.7-6.el8.noarch.rpm 183 kB/s | 34 kB 00:00 2023-09-23T20:33:11.461 INFO:teuthology.orchestra.run.smithi070.stdout:(4/6): ceph-mgr-dashboard-17.2.6-1123.g61431f6b 3.4 MB/s | 1.7 MB 00:00 2023-09-23T20:33:11.486 INFO:teuthology.orchestra.run.smithi070.stdout:(5/6): python3-routes-2.4.1-12.el8.noarch.rpm 735 kB/s | 196 kB 00:00 2023-09-23T20:33:11.511 INFO:teuthology.orchestra.run.smithi070.stdout:(6/6): python3-jwt-1.6.1-2.el8.noarch.rpm 319 kB/s | 43 kB 00:00 2023-09-23T20:33:11.513 INFO:teuthology.orchestra.run.smithi070.stdout:-------------------------------------------------------------------------------- 2023-09-23T20:33:11.513 INFO:teuthology.orchestra.run.smithi070.stdout:Total 2.3 MB/s | 2.0 MB 00:00 2023-09-23T20:33:11.525 INFO:teuthology.orchestra.run.smithi070.stdout:Running transaction check 2023-09-23T20:33:11.534 INFO:teuthology.orchestra.run.smithi070.stdout:Transaction check succeeded. 2023-09-23T20:33:11.535 INFO:teuthology.orchestra.run.smithi070.stdout:Running transaction test 2023-09-23T20:33:11.592 INFO:teuthology.orchestra.run.smithi070.stdout:Transaction test succeeded. 2023-09-23T20:33:11.596 INFO:teuthology.orchestra.run.smithi070.stdout:Running transaction 2023-09-23T20:33:11.870 INFO:teuthology.orchestra.run.smithi070.stdout: Preparing : 1/1 2023-09-23T20:33:12.812 INFO:teuthology.orchestra.run.smithi070.stdout: Installing : python3-jwt-1.6.1-2.el8.noarch 1/6 2023-09-23T20:33:12.984 INFO:teuthology.orchestra.run.smithi070.stdout: Installing : python3-repoze-lru-0.7-6.el8.noarch 2/6 2023-09-23T20:33:13.137 INFO:teuthology.orchestra.run.smithi070.stdout: Installing : python3-routes-2.4.1-12.el8.noarch 3/6 2023-09-23T20:33:13.292 INFO:teuthology.orchestra.run.smithi070.stdout: Installing : ceph-prometheus-alerts-2:17.2.6-1123.g61431f6b.el8.n 4/6 2023-09-23T20:33:13.623 INFO:teuthology.orchestra.run.smithi070.stdout: Installing : ceph-grafana-dashboards-2:17.2.6-1123.g61431f6b.el8. 5/6 2023-09-23T20:33:13.697 INFO:teuthology.orchestra.run.smithi026.stdout:Installed products updated. 2023-09-23T20:33:13.828 INFO:teuthology.orchestra.run.smithi070.stdout: Installing : ceph-mgr-dashboard-2:17.2.6-1123.g61431f6b.el8.noarc 6/6 2023-09-23T20:33:13.900 INFO:teuthology.orchestra.run.smithi026.stdout: 2023-09-23T20:33:13.900 INFO:teuthology.orchestra.run.smithi026.stdout:Installed: 2023-09-23T20:33:13.900 INFO:teuthology.orchestra.run.smithi026.stdout: ceph-immutable-object-cache-2:17.2.6-1123.g61431f6b.el8.x86_64 2023-09-23T20:33:13.901 INFO:teuthology.orchestra.run.smithi026.stdout: 2023-09-23T20:33:13.901 INFO:teuthology.orchestra.run.smithi026.stdout:Complete! 2023-09-23T20:33:14.029 DEBUG:teuthology.orchestra.run.smithi026:> sudo yum -y install ceph-mgr 2023-09-23T20:33:14.041 INFO:teuthology.orchestra.run.smithi070.stdout: Running scriptlet: ceph-mgr-dashboard-2:17.2.6-1123.g61431f6b.el8.noarc 6/6 2023-09-23T20:33:14.041 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : ceph-grafana-dashboards-2:17.2.6-1123.g61431f6b.el8. 1/6 2023-09-23T20:33:14.041 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : ceph-mgr-dashboard-2:17.2.6-1123.g61431f6b.el8.noarc 2/6 2023-09-23T20:33:14.041 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : ceph-prometheus-alerts-2:17.2.6-1123.g61431f6b.el8.n 3/6 2023-09-23T20:33:14.042 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : python3-repoze-lru-0.7-6.el8.noarch 4/6 2023-09-23T20:33:14.042 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : python3-routes-2.4.1-12.el8.noarch 5/6 2023-09-23T20:33:14.398 INFO:teuthology.orchestra.run.smithi026.stdout:Updating Subscription Management repositories. 2023-09-23T20:33:15.071 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : python3-jwt-1.6.1-2.el8.noarch 6/6 2023-09-23T20:33:15.072 INFO:teuthology.orchestra.run.smithi070.stdout:Last metadata expiration check: 0:01:59 ago on Sat 23 Sep 2023 08:31:16 PM UTC. 2023-09-23T20:33:15.231 INFO:teuthology.orchestra.run.smithi026.stdout:Last metadata expiration check: 0:01:57 ago on Sat 23 Sep 2023 08:31:18 PM UTC. 2023-09-23T20:33:16.668 INFO:teuthology.orchestra.run.smithi026.stdout:Package ceph-mgr-2:17.2.6-1123.g61431f6b.el8.x86_64 is already installed. 2023-09-23T20:33:16.789 INFO:teuthology.orchestra.run.smithi026.stdout:Dependencies resolved. 2023-09-23T20:33:16.791 INFO:teuthology.orchestra.run.smithi026.stdout:Nothing to do. 2023-09-23T20:33:16.791 INFO:teuthology.orchestra.run.smithi026.stdout:Complete! 2023-09-23T20:33:16.906 DEBUG:teuthology.orchestra.run.smithi026:> sudo yum -y install ceph-mgr-dashboard 2023-09-23T20:33:17.272 INFO:teuthology.orchestra.run.smithi026.stdout:Updating Subscription Management repositories. 2023-09-23T20:33:17.582 INFO:teuthology.orchestra.run.smithi070.stdout:Installed products updated. 2023-09-23T20:33:17.767 INFO:teuthology.orchestra.run.smithi070.stdout: 2023-09-23T20:33:17.768 INFO:teuthology.orchestra.run.smithi070.stdout:Installed: 2023-09-23T20:33:17.768 INFO:teuthology.orchestra.run.smithi070.stdout: ceph-grafana-dashboards-2:17.2.6-1123.g61431f6b.el8.noarch 2023-09-23T20:33:17.768 INFO:teuthology.orchestra.run.smithi070.stdout: ceph-mgr-dashboard-2:17.2.6-1123.g61431f6b.el8.noarch 2023-09-23T20:33:17.768 INFO:teuthology.orchestra.run.smithi070.stdout: ceph-prometheus-alerts-2:17.2.6-1123.g61431f6b.el8.noarch 2023-09-23T20:33:17.768 INFO:teuthology.orchestra.run.smithi070.stdout: python3-jwt-1.6.1-2.el8.noarch 2023-09-23T20:33:17.768 INFO:teuthology.orchestra.run.smithi070.stdout: python3-repoze-lru-0.7-6.el8.noarch 2023-09-23T20:33:17.768 INFO:teuthology.orchestra.run.smithi070.stdout: python3-routes-2.4.1-12.el8.noarch 2023-09-23T20:33:17.768 INFO:teuthology.orchestra.run.smithi070.stdout: 2023-09-23T20:33:17.768 INFO:teuthology.orchestra.run.smithi070.stdout:Complete! 2023-09-23T20:33:17.896 DEBUG:teuthology.orchestra.run.smithi070:> sudo yum -y install ceph-mgr-diskprediction-local 2023-09-23T20:33:18.212 INFO:teuthology.orchestra.run.smithi026.stdout:Last metadata expiration check: 0:02:00 ago on Sat 23 Sep 2023 08:31:18 PM UTC. 2023-09-23T20:33:18.243 INFO:teuthology.orchestra.run.smithi070.stdout:Updating Subscription Management repositories. 2023-09-23T20:33:19.085 INFO:teuthology.orchestra.run.smithi070.stdout:Last metadata expiration check: 0:02:03 ago on Sat 23 Sep 2023 08:31:16 PM UTC. 2023-09-23T20:33:19.758 INFO:teuthology.orchestra.run.smithi026.stdout:Dependencies resolved. 2023-09-23T20:33:19.759 INFO:teuthology.orchestra.run.smithi026.stdout:=================================================================================================== 2023-09-23T20:33:19.759 INFO:teuthology.orchestra.run.smithi026.stdout: Package Arch Version Repository Size 2023-09-23T20:33:19.759 INFO:teuthology.orchestra.run.smithi026.stdout:=================================================================================================== 2023-09-23T20:33:19.759 INFO:teuthology.orchestra.run.smithi026.stdout:Installing: 2023-09-23T20:33:19.760 INFO:teuthology.orchestra.run.smithi026.stdout: ceph-mgr-dashboard noarch 2:17.2.6-1123.g61431f6b.el8 ceph-noarch 1.7 M 2023-09-23T20:33:19.760 INFO:teuthology.orchestra.run.smithi026.stdout:Installing dependencies: 2023-09-23T20:33:19.760 INFO:teuthology.orchestra.run.smithi026.stdout: ceph-grafana-dashboards noarch 2:17.2.6-1123.g61431f6b.el8 ceph-noarch 23 k 2023-09-23T20:33:19.760 INFO:teuthology.orchestra.run.smithi026.stdout: ceph-prometheus-alerts noarch 2:17.2.6-1123.g61431f6b.el8 ceph-noarch 14 k 2023-09-23T20:33:19.760 INFO:teuthology.orchestra.run.smithi026.stdout: python3-jwt noarch 1.6.1-2.el8 rhel-8-for-x86_64-baseos-rpms 43 k 2023-09-23T20:33:19.760 INFO:teuthology.orchestra.run.smithi026.stdout: python3-repoze-lru noarch 0.7-6.el8 epel 34 k 2023-09-23T20:33:19.760 INFO:teuthology.orchestra.run.smithi026.stdout: python3-routes noarch 2.4.1-12.el8 epel 196 k 2023-09-23T20:33:19.760 INFO:teuthology.orchestra.run.smithi026.stdout: 2023-09-23T20:33:19.760 INFO:teuthology.orchestra.run.smithi026.stdout:Transaction Summary 2023-09-23T20:33:19.760 INFO:teuthology.orchestra.run.smithi026.stdout:=================================================================================================== 2023-09-23T20:33:19.760 INFO:teuthology.orchestra.run.smithi026.stdout:Install 6 Packages 2023-09-23T20:33:19.761 INFO:teuthology.orchestra.run.smithi026.stdout: 2023-09-23T20:33:19.761 INFO:teuthology.orchestra.run.smithi026.stdout:Total download size: 2.0 M 2023-09-23T20:33:19.761 INFO:teuthology.orchestra.run.smithi026.stdout:Installed size: 8.0 M 2023-09-23T20:33:19.761 INFO:teuthology.orchestra.run.smithi026.stdout:Downloading Packages: 2023-09-23T20:33:20.059 INFO:teuthology.orchestra.run.smithi026.stdout:(1/6): ceph-grafana-dashboards-17.2.6-1123.g614 169 kB/s | 23 kB 00:00 2023-09-23T20:33:20.084 INFO:teuthology.orchestra.run.smithi026.stdout:(2/6): ceph-prometheus-alerts-17.2.6-1123.g6143 90 kB/s | 14 kB 00:00 2023-09-23T20:33:20.318 INFO:teuthology.orchestra.run.smithi026.stdout:(3/6): python3-repoze-lru-0.7-6.el8.noarch.rpm 130 kB/s | 34 kB 00:00 2023-09-23T20:33:20.385 INFO:teuthology.orchestra.run.smithi026.stdout:(4/6): python3-jwt-1.6.1-2.el8.noarch.rpm 641 kB/s | 43 kB 00:00 2023-09-23T20:33:20.427 INFO:teuthology.orchestra.run.smithi026.stdout:(5/6): ceph-mgr-dashboard-17.2.6-1123.g61431f6b 3.4 MB/s | 1.7 MB 00:00 2023-09-23T20:33:20.468 INFO:teuthology.orchestra.run.smithi026.stdout:(6/6): python3-routes-2.4.1-12.el8.noarch.rpm 512 kB/s | 196 kB 00:00 2023-09-23T20:33:20.472 INFO:teuthology.orchestra.run.smithi026.stdout:-------------------------------------------------------------------------------- 2023-09-23T20:33:20.472 INFO:teuthology.orchestra.run.smithi026.stdout:Total 2.8 MB/s | 2.0 MB 00:00 2023-09-23T20:33:20.484 INFO:teuthology.orchestra.run.smithi026.stdout:Running transaction check 2023-09-23T20:33:20.494 INFO:teuthology.orchestra.run.smithi026.stdout:Transaction check succeeded. 2023-09-23T20:33:20.494 INFO:teuthology.orchestra.run.smithi026.stdout:Running transaction test 2023-09-23T20:33:20.503 INFO:teuthology.orchestra.run.smithi070.stdout:Dependencies resolved. 2023-09-23T20:33:20.504 INFO:teuthology.orchestra.run.smithi070.stdout:======================================================================================================================= 2023-09-23T20:33:20.504 INFO:teuthology.orchestra.run.smithi070.stdout: Package Arch Version Repository Size 2023-09-23T20:33:20.504 INFO:teuthology.orchestra.run.smithi070.stdout:======================================================================================================================= 2023-09-23T20:33:20.505 INFO:teuthology.orchestra.run.smithi070.stdout:Installing: 2023-09-23T20:33:20.505 INFO:teuthology.orchestra.run.smithi070.stdout: ceph-mgr-diskprediction-local noarch 2:17.2.6-1123.g61431f6b.el8 ceph-noarch 7.4 M 2023-09-23T20:33:20.505 INFO:teuthology.orchestra.run.smithi070.stdout:Installing dependencies: 2023-09-23T20:33:20.505 INFO:teuthology.orchestra.run.smithi070.stdout: libgfortran x86_64 8.5.0-10.1.el8_6 rhel-8-for-x86_64-baseos-rpms 644 k 2023-09-23T20:33:20.505 INFO:teuthology.orchestra.run.smithi070.stdout: libquadmath x86_64 8.5.0-10.1.el8_6 rhel-8-for-x86_64-baseos-rpms 171 k 2023-09-23T20:33:20.505 INFO:teuthology.orchestra.run.smithi070.stdout: openblas x86_64 0.3.15-3.el8 rhel-8-for-x86_64-appstream-rpms 4.7 M 2023-09-23T20:33:20.505 INFO:teuthology.orchestra.run.smithi070.stdout: openblas-threads x86_64 0.3.15-3.el8 rhel-8-for-x86_64-appstream-rpms 4.9 M 2023-09-23T20:33:20.505 INFO:teuthology.orchestra.run.smithi070.stdout: python3-numpy x86_64 1:1.14.3-10.el8 rhel-8-for-x86_64-appstream-rpms 3.7 M 2023-09-23T20:33:20.505 INFO:teuthology.orchestra.run.smithi070.stdout: python3-numpy-f2py x86_64 1:1.14.3-10.el8 rhel-8-for-x86_64-appstream-rpms 225 k 2023-09-23T20:33:20.505 INFO:teuthology.orchestra.run.smithi070.stdout: python3-scipy x86_64 1.0.0-21.module+el8.5.0+10916+41bd434d rhel-8-for-x86_64-appstream-rpms 14 M 2023-09-23T20:33:20.505 INFO:teuthology.orchestra.run.smithi070.stdout: 2023-09-23T20:33:20.505 INFO:teuthology.orchestra.run.smithi070.stdout:Transaction Summary 2023-09-23T20:33:20.506 INFO:teuthology.orchestra.run.smithi070.stdout:======================================================================================================================= 2023-09-23T20:33:20.506 INFO:teuthology.orchestra.run.smithi070.stdout:Install 8 Packages 2023-09-23T20:33:20.506 INFO:teuthology.orchestra.run.smithi070.stdout: 2023-09-23T20:33:20.506 INFO:teuthology.orchestra.run.smithi070.stdout:Total download size: 35 M 2023-09-23T20:33:20.506 INFO:teuthology.orchestra.run.smithi070.stdout:Installed size: 210 M 2023-09-23T20:33:20.506 INFO:teuthology.orchestra.run.smithi070.stdout:Downloading Packages: 2023-09-23T20:33:20.558 INFO:teuthology.orchestra.run.smithi026.stdout:Transaction test succeeded. 2023-09-23T20:33:20.562 INFO:teuthology.orchestra.run.smithi026.stdout:Running transaction 2023-09-23T20:33:20.626 INFO:teuthology.orchestra.run.smithi070.stdout:(1/8): libquadmath-8.5.0-10.1.el8_6.x86_64.rpm 1.4 MB/s | 171 kB 00:00 2023-09-23T20:33:20.660 INFO:teuthology.orchestra.run.smithi070.stdout:(2/8): libgfortran-8.5.0-10.1.el8_6.x86_64.rpm 4.1 MB/s | 644 kB 00:00 2023-09-23T20:33:20.777 INFO:teuthology.orchestra.run.smithi070.stdout:(3/8): python3-numpy-1.14.3-10.el8.x86_64.rpm 32 MB/s | 3.7 MB 00:00 2023-09-23T20:33:20.818 INFO:teuthology.orchestra.run.smithi070.stdout:(4/8): python3-numpy-f2py-1.14.3-10.el8.x86_64. 5.3 MB/s | 225 kB 00:00 2023-09-23T20:33:20.843 INFO:teuthology.orchestra.run.smithi026.stdout: Preparing : 1/1 2023-09-23T20:33:21.019 INFO:teuthology.orchestra.run.smithi070.stdout:(5/8): python3-scipy-1.0.0-21.module+el8.5.0+10 35 MB/s | 14 MB 00:00 2023-09-23T20:33:21.114 INFO:teuthology.orchestra.run.smithi070.stdout:(6/8): openblas-threads-0.3.15-3.el8.x86_64.rpm 17 MB/s | 4.9 MB 00:00 2023-09-23T20:33:21.211 INFO:teuthology.orchestra.run.smithi070.stdout:(7/8): openblas-0.3.15-3.el8.x86_64.rpm 25 MB/s | 4.7 MB 00:00 2023-09-23T20:33:21.295 INFO:teuthology.orchestra.run.smithi070.stdout:(8/8): ceph-mgr-diskprediction-local-17.2.6-112 9.4 MB/s | 7.4 MB 00:00 2023-09-23T20:33:21.296 INFO:teuthology.orchestra.run.smithi070.stdout:-------------------------------------------------------------------------------- 2023-09-23T20:33:21.296 INFO:teuthology.orchestra.run.smithi070.stdout:Total 45 MB/s | 35 MB 00:00 2023-09-23T20:33:21.534 INFO:teuthology.orchestra.run.smithi070.stdout:Running transaction check 2023-09-23T20:33:21.549 INFO:teuthology.orchestra.run.smithi026.stdout: Installing : python3-jwt-1.6.1-2.el8.noarch 1/6 2023-09-23T20:33:21.552 INFO:teuthology.orchestra.run.smithi070.stdout:Transaction check succeeded. 2023-09-23T20:33:21.552 INFO:teuthology.orchestra.run.smithi070.stdout:Running transaction test 2023-09-23T20:33:21.724 INFO:teuthology.orchestra.run.smithi026.stdout: Installing : python3-repoze-lru-0.7-6.el8.noarch 2/6 2023-09-23T20:33:21.845 INFO:teuthology.orchestra.run.smithi070.stdout:Transaction test succeeded. 2023-09-23T20:33:21.849 INFO:teuthology.orchestra.run.smithi070.stdout:Running transaction 2023-09-23T20:33:21.915 INFO:teuthology.orchestra.run.smithi026.stdout: Installing : python3-routes-2.4.1-12.el8.noarch 3/6 2023-09-23T20:33:22.045 INFO:teuthology.orchestra.run.smithi026.stdout: Installing : ceph-prometheus-alerts-2:17.2.6-1123.g61431f6b.el8.n 4/6 2023-09-23T20:33:22.402 INFO:teuthology.orchestra.run.smithi070.stdout: Preparing : 1/1 2023-09-23T20:33:22.418 INFO:teuthology.orchestra.run.smithi026.stdout: Installing : ceph-grafana-dashboards-2:17.2.6-1123.g61431f6b.el8. 5/6 2023-09-23T20:33:22.557 INFO:teuthology.orchestra.run.smithi070.stdout: Installing : libquadmath-8.5.0-10.1.el8_6.x86_64 1/8 2023-09-23T20:33:22.609 INFO:teuthology.orchestra.run.smithi026.stdout: Installing : ceph-mgr-dashboard-2:17.2.6-1123.g61431f6b.el8.noarc 6/6 2023-09-23T20:33:22.703 INFO:teuthology.orchestra.run.smithi070.stdout: Running scriptlet: libquadmath-8.5.0-10.1.el8_6.x86_64 1/8 2023-09-23T20:33:22.832 INFO:teuthology.orchestra.run.smithi070.stdout: Installing : libgfortran-8.5.0-10.1.el8_6.x86_64 2/8 2023-09-23T20:33:22.861 INFO:teuthology.orchestra.run.smithi026.stdout: Running scriptlet: ceph-mgr-dashboard-2:17.2.6-1123.g61431f6b.el8.noarc 6/6 2023-09-23T20:33:22.861 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : ceph-grafana-dashboards-2:17.2.6-1123.g61431f6b.el8. 1/6 2023-09-23T20:33:22.861 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : ceph-mgr-dashboard-2:17.2.6-1123.g61431f6b.el8.noarc 2/6 2023-09-23T20:33:22.862 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : ceph-prometheus-alerts-2:17.2.6-1123.g61431f6b.el8.n 3/6 2023-09-23T20:33:22.862 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : python3-repoze-lru-0.7-6.el8.noarch 4/6 2023-09-23T20:33:22.862 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : python3-routes-2.4.1-12.el8.noarch 5/6 2023-09-23T20:33:23.452 INFO:teuthology.orchestra.run.smithi070.stdout: Running scriptlet: libgfortran-8.5.0-10.1.el8_6.x86_64 2/8 2023-09-23T20:33:23.607 INFO:teuthology.orchestra.run.smithi070.stdout: Installing : openblas-threads-0.3.15-3.el8.x86_64 3/8 2023-09-23T20:33:23.935 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : python3-jwt-1.6.1-2.el8.noarch 6/6 2023-09-23T20:33:23.935 INFO:teuthology.orchestra.run.smithi026.stdout:Last metadata expiration check: 0:02:05 ago on Sat 23 Sep 2023 08:31:18 PM UTC. 2023-09-23T20:33:24.211 INFO:teuthology.orchestra.run.smithi070.stdout: Running scriptlet: openblas-threads-0.3.15-3.el8.x86_64 3/8 2023-09-23T20:33:24.365 INFO:teuthology.orchestra.run.smithi070.stdout: Installing : openblas-0.3.15-3.el8.x86_64 4/8 2023-09-23T20:33:25.053 INFO:teuthology.orchestra.run.smithi070.stdout: Running scriptlet: openblas-0.3.15-3.el8.x86_64 4/8 2023-09-23T20:33:25.367 INFO:teuthology.orchestra.run.smithi070.stdout: Installing : python3-numpy-1:1.14.3-10.el8.x86_64 5/8 2023-09-23T20:33:25.582 INFO:teuthology.orchestra.run.smithi070.stdout: Installing : python3-numpy-f2py-1:1.14.3-10.el8.x86_64 6/8 2023-09-23T20:33:26.452 INFO:teuthology.orchestra.run.smithi026.stdout:Installed products updated. 2023-09-23T20:33:26.650 INFO:teuthology.orchestra.run.smithi026.stdout: 2023-09-23T20:33:26.651 INFO:teuthology.orchestra.run.smithi026.stdout:Installed: 2023-09-23T20:33:26.651 INFO:teuthology.orchestra.run.smithi026.stdout: ceph-grafana-dashboards-2:17.2.6-1123.g61431f6b.el8.noarch 2023-09-23T20:33:26.651 INFO:teuthology.orchestra.run.smithi026.stdout: ceph-mgr-dashboard-2:17.2.6-1123.g61431f6b.el8.noarch 2023-09-23T20:33:26.651 INFO:teuthology.orchestra.run.smithi026.stdout: ceph-prometheus-alerts-2:17.2.6-1123.g61431f6b.el8.noarch 2023-09-23T20:33:26.651 INFO:teuthology.orchestra.run.smithi026.stdout: python3-jwt-1.6.1-2.el8.noarch 2023-09-23T20:33:26.651 INFO:teuthology.orchestra.run.smithi026.stdout: python3-repoze-lru-0.7-6.el8.noarch 2023-09-23T20:33:26.651 INFO:teuthology.orchestra.run.smithi026.stdout: python3-routes-2.4.1-12.el8.noarch 2023-09-23T20:33:26.651 INFO:teuthology.orchestra.run.smithi026.stdout: 2023-09-23T20:33:26.651 INFO:teuthology.orchestra.run.smithi026.stdout:Complete! 2023-09-23T20:33:26.776 DEBUG:teuthology.orchestra.run.smithi026:> sudo yum -y install ceph-mgr-diskprediction-local 2023-09-23T20:33:27.146 INFO:teuthology.orchestra.run.smithi026.stdout:Updating Subscription Management repositories. 2023-09-23T20:33:27.318 INFO:teuthology.orchestra.run.smithi070.stdout: Running scriptlet: python3-numpy-f2py-1:1.14.3-10.el8.x86_64 6/8 2023-09-23T20:33:27.981 INFO:teuthology.orchestra.run.smithi026.stdout:Last metadata expiration check: 0:02:09 ago on Sat 23 Sep 2023 08:31:18 PM UTC. 2023-09-23T20:33:28.546 INFO:teuthology.orchestra.run.smithi070.stdout: Installing : python3-scipy-1.0.0-21.module+el8.5.0+10916+41bd434d 7/8 2023-09-23T20:33:28.889 INFO:teuthology.orchestra.run.smithi070.stdout: Installing : ceph-mgr-diskprediction-local-2:17.2.6-1123.g61431f6 8/8 2023-09-23T20:33:29.155 INFO:teuthology.orchestra.run.smithi070.stdout: Running scriptlet: ceph-mgr-diskprediction-local-2:17.2.6-1123.g61431f6 8/8 2023-09-23T20:33:29.156 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : ceph-mgr-diskprediction-local-2:17.2.6-1123.g61431f6 1/8 2023-09-23T20:33:29.156 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : libgfortran-8.5.0-10.1.el8_6.x86_64 2/8 2023-09-23T20:33:29.156 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : libquadmath-8.5.0-10.1.el8_6.x86_64 3/8 2023-09-23T20:33:29.156 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : python3-scipy-1.0.0-21.module+el8.5.0+10916+41bd434d 4/8 2023-09-23T20:33:29.156 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : python3-numpy-1:1.14.3-10.el8.x86_64 5/8 2023-09-23T20:33:29.156 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : python3-numpy-f2py-1:1.14.3-10.el8.x86_64 6/8 2023-09-23T20:33:29.156 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : openblas-threads-0.3.15-3.el8.x86_64 7/8 2023-09-23T20:33:29.550 INFO:teuthology.orchestra.run.smithi026.stdout:Dependencies resolved. 2023-09-23T20:33:29.551 INFO:teuthology.orchestra.run.smithi026.stdout:======================================================================================================================= 2023-09-23T20:33:29.552 INFO:teuthology.orchestra.run.smithi026.stdout: Package Arch Version Repository Size 2023-09-23T20:33:29.552 INFO:teuthology.orchestra.run.smithi026.stdout:======================================================================================================================= 2023-09-23T20:33:29.552 INFO:teuthology.orchestra.run.smithi026.stdout:Installing: 2023-09-23T20:33:29.552 INFO:teuthology.orchestra.run.smithi026.stdout: ceph-mgr-diskprediction-local noarch 2:17.2.6-1123.g61431f6b.el8 ceph-noarch 7.4 M 2023-09-23T20:33:29.552 INFO:teuthology.orchestra.run.smithi026.stdout:Installing dependencies: 2023-09-23T20:33:29.552 INFO:teuthology.orchestra.run.smithi026.stdout: libgfortran x86_64 8.5.0-10.1.el8_6 rhel-8-for-x86_64-baseos-rpms 644 k 2023-09-23T20:33:29.552 INFO:teuthology.orchestra.run.smithi026.stdout: libquadmath x86_64 8.5.0-10.1.el8_6 rhel-8-for-x86_64-baseos-rpms 171 k 2023-09-23T20:33:29.552 INFO:teuthology.orchestra.run.smithi026.stdout: openblas x86_64 0.3.15-3.el8 rhel-8-for-x86_64-appstream-rpms 4.7 M 2023-09-23T20:33:29.552 INFO:teuthology.orchestra.run.smithi026.stdout: openblas-threads x86_64 0.3.15-3.el8 rhel-8-for-x86_64-appstream-rpms 4.9 M 2023-09-23T20:33:29.552 INFO:teuthology.orchestra.run.smithi026.stdout: python3-numpy x86_64 1:1.14.3-10.el8 rhel-8-for-x86_64-appstream-rpms 3.7 M 2023-09-23T20:33:29.553 INFO:teuthology.orchestra.run.smithi026.stdout: python3-numpy-f2py x86_64 1:1.14.3-10.el8 rhel-8-for-x86_64-appstream-rpms 225 k 2023-09-23T20:33:29.553 INFO:teuthology.orchestra.run.smithi026.stdout: python3-scipy x86_64 1.0.0-21.module+el8.5.0+10916+41bd434d rhel-8-for-x86_64-appstream-rpms 14 M 2023-09-23T20:33:29.553 INFO:teuthology.orchestra.run.smithi026.stdout: 2023-09-23T20:33:29.553 INFO:teuthology.orchestra.run.smithi026.stdout:Transaction Summary 2023-09-23T20:33:29.553 INFO:teuthology.orchestra.run.smithi026.stdout:======================================================================================================================= 2023-09-23T20:33:29.553 INFO:teuthology.orchestra.run.smithi026.stdout:Install 8 Packages 2023-09-23T20:33:29.553 INFO:teuthology.orchestra.run.smithi026.stdout: 2023-09-23T20:33:29.554 INFO:teuthology.orchestra.run.smithi026.stdout:Total download size: 35 M 2023-09-23T20:33:29.554 INFO:teuthology.orchestra.run.smithi026.stdout:Installed size: 210 M 2023-09-23T20:33:29.554 INFO:teuthology.orchestra.run.smithi026.stdout:Downloading Packages: 2023-09-23T20:33:29.633 INFO:teuthology.orchestra.run.smithi026.stdout:(1/8): libgfortran-8.5.0-10.1.el8_6.x86_64.rpm 8.0 MB/s | 644 kB 00:00 2023-09-23T20:33:29.659 INFO:teuthology.orchestra.run.smithi026.stdout:(2/8): libquadmath-8.5.0-10.1.el8_6.x86_64.rpm 1.6 MB/s | 171 kB 00:00 2023-09-23T20:33:29.775 INFO:teuthology.orchestra.run.smithi026.stdout:(3/8): python3-numpy-1.14.3-10.el8.x86_64.rpm 32 MB/s | 3.7 MB 00:00 2023-09-23T20:33:29.817 INFO:teuthology.orchestra.run.smithi026.stdout:(4/8): python3-numpy-f2py-1.14.3-10.el8.x86_64. 5.3 MB/s | 225 kB 00:00 2023-09-23T20:33:29.976 INFO:teuthology.orchestra.run.smithi026.stdout:(5/8): openblas-threads-0.3.15-3.el8.x86_64.rpm 31 MB/s | 4.9 MB 00:00 2023-09-23T20:33:30.134 INFO:teuthology.orchestra.run.smithi026.stdout:(6/8): python3-scipy-1.0.0-21.module+el8.5.0+10 27 MB/s | 14 MB 00:00 2023-09-23T20:33:30.229 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : openblas-0.3.15-3.el8.x86_64 8/8 2023-09-23T20:33:30.229 INFO:teuthology.orchestra.run.smithi070.stdout:Last metadata expiration check: 0:02:14 ago on Sat 23 Sep 2023 08:31:16 PM UTC. 2023-09-23T20:33:30.234 INFO:teuthology.orchestra.run.smithi026.stdout:(7/8): openblas-0.3.15-3.el8.x86_64.rpm 18 MB/s | 4.7 MB 00:00 2023-09-23T20:33:30.393 INFO:teuthology.orchestra.run.smithi026.stdout:(8/8): ceph-mgr-diskprediction-local-17.2.6-112 8.8 MB/s | 7.4 MB 00:00 2023-09-23T20:33:30.394 INFO:teuthology.orchestra.run.smithi026.stdout:-------------------------------------------------------------------------------- 2023-09-23T20:33:30.394 INFO:teuthology.orchestra.run.smithi026.stdout:Total 42 MB/s | 35 MB 00:00 2023-09-23T20:33:30.641 INFO:teuthology.orchestra.run.smithi026.stdout:Running transaction check 2023-09-23T20:33:30.660 INFO:teuthology.orchestra.run.smithi026.stdout:Transaction check succeeded. 2023-09-23T20:33:30.660 INFO:teuthology.orchestra.run.smithi026.stdout:Running transaction test 2023-09-23T20:33:30.973 INFO:teuthology.orchestra.run.smithi026.stdout:Transaction test succeeded. 2023-09-23T20:33:30.977 INFO:teuthology.orchestra.run.smithi026.stdout:Running transaction 2023-09-23T20:33:31.563 INFO:teuthology.orchestra.run.smithi026.stdout: Preparing : 1/1 2023-09-23T20:33:32.210 INFO:teuthology.orchestra.run.smithi026.stdout: Installing : libquadmath-8.5.0-10.1.el8_6.x86_64 1/8 2023-09-23T20:33:32.346 INFO:teuthology.orchestra.run.smithi026.stdout: Running scriptlet: libquadmath-8.5.0-10.1.el8_6.x86_64 1/8 2023-09-23T20:33:32.485 INFO:teuthology.orchestra.run.smithi026.stdout: Installing : libgfortran-8.5.0-10.1.el8_6.x86_64 2/8 2023-09-23T20:33:32.736 INFO:teuthology.orchestra.run.smithi070.stdout:Installed products updated. 2023-09-23T20:33:32.923 INFO:teuthology.orchestra.run.smithi070.stdout: 2023-09-23T20:33:32.923 INFO:teuthology.orchestra.run.smithi070.stdout:Installed: 2023-09-23T20:33:32.923 INFO:teuthology.orchestra.run.smithi070.stdout: ceph-mgr-diskprediction-local-2:17.2.6-1123.g61431f6b.el8.noarch 2023-09-23T20:33:32.923 INFO:teuthology.orchestra.run.smithi070.stdout: libgfortran-8.5.0-10.1.el8_6.x86_64 2023-09-23T20:33:32.923 INFO:teuthology.orchestra.run.smithi070.stdout: libquadmath-8.5.0-10.1.el8_6.x86_64 2023-09-23T20:33:32.923 INFO:teuthology.orchestra.run.smithi070.stdout: openblas-0.3.15-3.el8.x86_64 2023-09-23T20:33:32.923 INFO:teuthology.orchestra.run.smithi070.stdout: openblas-threads-0.3.15-3.el8.x86_64 2023-09-23T20:33:32.923 INFO:teuthology.orchestra.run.smithi070.stdout: python3-numpy-1:1.14.3-10.el8.x86_64 2023-09-23T20:33:32.923 INFO:teuthology.orchestra.run.smithi070.stdout: python3-numpy-f2py-1:1.14.3-10.el8.x86_64 2023-09-23T20:33:32.923 INFO:teuthology.orchestra.run.smithi070.stdout: python3-scipy-1.0.0-21.module+el8.5.0+10916+41bd434d.x86_64 2023-09-23T20:33:32.924 INFO:teuthology.orchestra.run.smithi070.stdout: 2023-09-23T20:33:32.924 INFO:teuthology.orchestra.run.smithi070.stdout:Complete! 2023-09-23T20:33:33.054 DEBUG:teuthology.orchestra.run.smithi070:> sudo yum -y install ceph-mgr-rook 2023-09-23T20:33:33.122 INFO:teuthology.orchestra.run.smithi026.stdout: Running scriptlet: libgfortran-8.5.0-10.1.el8_6.x86_64 2/8 2023-09-23T20:33:33.285 INFO:teuthology.orchestra.run.smithi026.stdout: Installing : openblas-threads-0.3.15-3.el8.x86_64 3/8 2023-09-23T20:33:33.399 INFO:teuthology.orchestra.run.smithi070.stdout:Updating Subscription Management repositories. 2023-09-23T20:33:33.903 INFO:teuthology.orchestra.run.smithi026.stdout: Running scriptlet: openblas-threads-0.3.15-3.el8.x86_64 3/8 2023-09-23T20:33:34.076 INFO:teuthology.orchestra.run.smithi026.stdout: Installing : openblas-0.3.15-3.el8.x86_64 4/8 2023-09-23T20:33:34.212 INFO:teuthology.orchestra.run.smithi070.stdout:Last metadata expiration check: 0:02:18 ago on Sat 23 Sep 2023 08:31:16 PM UTC. 2023-09-23T20:33:34.816 INFO:teuthology.orchestra.run.smithi026.stdout: Running scriptlet: openblas-0.3.15-3.el8.x86_64 4/8 2023-09-23T20:33:35.261 INFO:teuthology.orchestra.run.smithi026.stdout: Installing : python3-numpy-1:1.14.3-10.el8.x86_64 5/8 2023-09-23T20:33:35.460 INFO:teuthology.orchestra.run.smithi026.stdout: Installing : python3-numpy-f2py-1:1.14.3-10.el8.x86_64 6/8 2023-09-23T20:33:35.604 INFO:teuthology.orchestra.run.smithi070.stdout:Dependencies resolved. 2023-09-23T20:33:35.605 INFO:teuthology.orchestra.run.smithi070.stdout:======================================================================================================== 2023-09-23T20:33:35.605 INFO:teuthology.orchestra.run.smithi070.stdout: Package Arch Version Repository Size 2023-09-23T20:33:35.605 INFO:teuthology.orchestra.run.smithi070.stdout:======================================================================================================== 2023-09-23T20:33:35.606 INFO:teuthology.orchestra.run.smithi070.stdout:Installing: 2023-09-23T20:33:35.606 INFO:teuthology.orchestra.run.smithi070.stdout: ceph-mgr-rook noarch 2:17.2.6-1123.g61431f6b.el8 ceph-noarch 47 k 2023-09-23T20:33:35.606 INFO:teuthology.orchestra.run.smithi070.stdout:Installing dependencies: 2023-09-23T20:33:35.606 INFO:teuthology.orchestra.run.smithi070.stdout: python3-cachetools noarch 3.1.1-4.el8 epel 33 k 2023-09-23T20:33:35.606 INFO:teuthology.orchestra.run.smithi070.stdout: python3-certifi noarch 2018.10.15-7.el8 epel 16 k 2023-09-23T20:33:35.606 INFO:teuthology.orchestra.run.smithi070.stdout: python3-google-auth noarch 1:1.1.1-10.el8 epel 81 k 2023-09-23T20:33:35.606 INFO:teuthology.orchestra.run.smithi070.stdout: python3-jsonpatch noarch 1.21-2.el8 rhel-8-for-x86_64-appstream-rpms 27 k 2023-09-23T20:33:35.606 INFO:teuthology.orchestra.run.smithi070.stdout: python3-jsonpointer noarch 1.10-11.el8 rhel-8-for-x86_64-appstream-rpms 20 k 2023-09-23T20:33:35.606 INFO:teuthology.orchestra.run.smithi070.stdout: python3-kubernetes noarch 1:11.0.0-6.el8 epel 1.2 M 2023-09-23T20:33:35.607 INFO:teuthology.orchestra.run.smithi070.stdout: python3-oauthlib noarch 2.1.0-1.el8 rhel-8-for-x86_64-baseos-rpms 155 k 2023-09-23T20:33:35.607 INFO:teuthology.orchestra.run.smithi070.stdout: python3-pyasn1 noarch 0.3.7-6.el8 rhel-8-for-x86_64-appstream-rpms 126 k 2023-09-23T20:33:35.607 INFO:teuthology.orchestra.run.smithi070.stdout: python3-pyasn1-modules noarch 0.3.7-6.el8 rhel-8-for-x86_64-appstream-rpms 110 k 2023-09-23T20:33:35.607 INFO:teuthology.orchestra.run.smithi070.stdout: python3-requests-oauthlib noarch 1.0.0-1.el8 rhel-8-for-x86_64-baseos-rpms 43 k 2023-09-23T20:33:35.607 INFO:teuthology.orchestra.run.smithi070.stdout: python3-rsa noarch 4.9-2.el8 epel 64 k 2023-09-23T20:33:35.607 INFO:teuthology.orchestra.run.smithi070.stdout: python3-websocket-client noarch 0.56.0-5.el8 epel 61 k 2023-09-23T20:33:35.607 INFO:teuthology.orchestra.run.smithi070.stdout: 2023-09-23T20:33:35.607 INFO:teuthology.orchestra.run.smithi070.stdout:Transaction Summary 2023-09-23T20:33:35.607 INFO:teuthology.orchestra.run.smithi070.stdout:======================================================================================================== 2023-09-23T20:33:35.607 INFO:teuthology.orchestra.run.smithi070.stdout:Install 13 Packages 2023-09-23T20:33:35.607 INFO:teuthology.orchestra.run.smithi070.stdout: 2023-09-23T20:33:35.608 INFO:teuthology.orchestra.run.smithi070.stdout:Total download size: 1.9 M 2023-09-23T20:33:35.608 INFO:teuthology.orchestra.run.smithi070.stdout:Installed size: 25 M 2023-09-23T20:33:35.608 INFO:teuthology.orchestra.run.smithi070.stdout:Downloading Packages: 2023-09-23T20:33:35.916 INFO:teuthology.orchestra.run.smithi070.stdout:(1/13): python3-certifi-2018.10.15-7.el8.noarch 72 kB/s | 16 kB 00:00 2023-09-23T20:33:35.932 INFO:teuthology.orchestra.run.smithi070.stdout:(2/13): ceph-mgr-rook-17.2.6-1123.g61431f6b.el8 194 kB/s | 47 kB 00:00 2023-09-23T20:33:35.949 INFO:teuthology.orchestra.run.smithi070.stdout:(3/13): python3-cachetools-3.1.1-4.el8.noarch.r 127 kB/s | 33 kB 00:00 2023-09-23T20:33:36.083 INFO:teuthology.orchestra.run.smithi070.stdout:(4/13): python3-google-auth-1.1.1-10.el8.noarch 486 kB/s | 81 kB 00:00 2023-09-23T20:33:36.175 INFO:teuthology.orchestra.run.smithi070.stdout:(5/13): python3-websocket-client-0.56.0-5.el8.n 665 kB/s | 61 kB 00:00 2023-09-23T20:33:36.217 INFO:teuthology.orchestra.run.smithi070.stdout:(6/13): python3-rsa-4.9-2.el8.noarch.rpm 240 kB/s | 64 kB 00:00 2023-09-23T20:33:36.234 INFO:teuthology.orchestra.run.smithi070.stdout:(7/13): python3-requests-oauthlib-1.0.0-1.el8.n 729 kB/s | 43 kB 00:00 2023-09-23T20:33:36.276 INFO:teuthology.orchestra.run.smithi070.stdout:(8/13): python3-kubernetes-11.0.0-6.el8.noarch. 3.4 MB/s | 1.2 MB 00:00 2023-09-23T20:33:36.301 INFO:teuthology.orchestra.run.smithi070.stdout:(9/13): python3-pyasn1-modules-0.3.7-6.el8.noar 1.8 MB/s | 110 kB 00:00 2023-09-23T20:33:36.335 INFO:teuthology.orchestra.run.smithi070.stdout:(10/13): python3-oauthlib-2.1.0-1.el8.noarch.rp 1.3 MB/s | 155 kB 00:00 2023-09-23T20:33:36.360 INFO:teuthology.orchestra.run.smithi070.stdout:(11/13): python3-jsonpatch-1.21-2.el8.noarch.rp 316 kB/s | 27 kB 00:00 2023-09-23T20:33:36.385 INFO:teuthology.orchestra.run.smithi070.stdout:(12/13): python3-jsonpointer-1.10-11.el8.noarch 398 kB/s | 20 kB 00:00 2023-09-23T20:33:36.410 INFO:teuthology.orchestra.run.smithi070.stdout:(13/13): python3-pyasn1-0.3.7-6.el8.noarch.rpm 1.2 MB/s | 126 kB 00:00 2023-09-23T20:33:36.413 INFO:teuthology.orchestra.run.smithi070.stdout:-------------------------------------------------------------------------------- 2023-09-23T20:33:36.413 INFO:teuthology.orchestra.run.smithi070.stdout:Total 2.4 MB/s | 1.9 MB 00:00 2023-09-23T20:33:36.470 INFO:teuthology.orchestra.run.smithi070.stdout:Running transaction check 2023-09-23T20:33:36.485 INFO:teuthology.orchestra.run.smithi070.stdout:Transaction check succeeded. 2023-09-23T20:33:36.485 INFO:teuthology.orchestra.run.smithi070.stdout:Running transaction test 2023-09-23T20:33:36.556 INFO:teuthology.orchestra.run.smithi070.stdout:Transaction test succeeded. 2023-09-23T20:33:36.560 INFO:teuthology.orchestra.run.smithi070.stdout:Running transaction 2023-09-23T20:33:37.075 INFO:teuthology.orchestra.run.smithi070.stdout: Preparing : 1/1 2023-09-23T20:33:37.270 INFO:teuthology.orchestra.run.smithi070.stdout: Installing : python3-pyasn1-0.3.7-6.el8.noarch 1/13 2023-09-23T20:33:37.328 INFO:teuthology.orchestra.run.smithi026.stdout: Running scriptlet: python3-numpy-f2py-1:1.14.3-10.el8.x86_64 6/8 2023-09-23T20:33:37.506 INFO:teuthology.orchestra.run.smithi070.stdout: Installing : python3-rsa-4.9-2.el8.noarch 2/13 2023-09-23T20:33:37.660 INFO:teuthology.orchestra.run.smithi070.stdout: Installing : python3-pyasn1-modules-0.3.7-6.el8.noarch 3/13 2023-09-23T20:33:37.811 INFO:teuthology.orchestra.run.smithi070.stdout: Installing : python3-jsonpointer-1.10-11.el8.noarch 4/13 2023-09-23T20:33:38.014 INFO:teuthology.orchestra.run.smithi070.stdout: Installing : python3-jsonpatch-1.21-2.el8.noarch 5/13 2023-09-23T20:33:38.266 INFO:teuthology.orchestra.run.smithi070.stdout: Installing : python3-oauthlib-2.1.0-1.el8.noarch 6/13 2023-09-23T20:33:38.456 INFO:teuthology.orchestra.run.smithi070.stdout: Installing : python3-requests-oauthlib-1.0.0-1.el8.noarch 7/13 2023-09-23T20:33:38.606 INFO:teuthology.orchestra.run.smithi026.stdout: Installing : python3-scipy-1.0.0-21.module+el8.5.0+10916+41bd434d 7/8 2023-09-23T20:33:38.652 INFO:teuthology.orchestra.run.smithi070.stdout: Installing : python3-websocket-client-0.56.0-5.el8.noarch 8/13 2023-09-23T20:33:38.751 INFO:teuthology.orchestra.run.smithi026.stdout: Installing : ceph-mgr-diskprediction-local-2:17.2.6-1123.g61431f6 8/8 2023-09-23T20:33:38.809 INFO:teuthology.orchestra.run.smithi070.stdout: Installing : python3-certifi-2018.10.15-7.el8.noarch 9/13 2023-09-23T20:33:39.008 INFO:teuthology.orchestra.run.smithi026.stdout: Running scriptlet: ceph-mgr-diskprediction-local-2:17.2.6-1123.g61431f6 8/8 2023-09-23T20:33:39.008 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : ceph-mgr-diskprediction-local-2:17.2.6-1123.g61431f6 1/8 2023-09-23T20:33:39.008 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : libgfortran-8.5.0-10.1.el8_6.x86_64 2/8 2023-09-23T20:33:39.009 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : libquadmath-8.5.0-10.1.el8_6.x86_64 3/8 2023-09-23T20:33:39.009 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : python3-scipy-1.0.0-21.module+el8.5.0+10916+41bd434d 4/8 2023-09-23T20:33:39.009 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : python3-numpy-1:1.14.3-10.el8.x86_64 5/8 2023-09-23T20:33:39.009 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : python3-numpy-f2py-1:1.14.3-10.el8.x86_64 6/8 2023-09-23T20:33:39.009 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : openblas-threads-0.3.15-3.el8.x86_64 7/8 2023-09-23T20:33:39.009 INFO:teuthology.orchestra.run.smithi070.stdout: Installing : python3-cachetools-3.1.1-4.el8.noarch 10/13 2023-09-23T20:33:40.072 INFO:teuthology.orchestra.run.smithi070.stdout: Installing : python3-google-auth-1:1.1.1-10.el8.noarch 11/13 2023-09-23T20:33:40.107 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : openblas-0.3.15-3.el8.x86_64 8/8 2023-09-23T20:33:40.107 INFO:teuthology.orchestra.run.smithi026.stdout:Last metadata expiration check: 0:02:22 ago on Sat 23 Sep 2023 08:31:18 PM UTC. 2023-09-23T20:33:40.295 INFO:teuthology.orchestra.run.smithi070.stdout: Installing : python3-kubernetes-1:11.0.0-6.el8.noarch 12/13 2023-09-23T20:33:40.453 INFO:teuthology.orchestra.run.smithi070.stdout: Installing : ceph-mgr-rook-2:17.2.6-1123.g61431f6b.el8.noarch 13/13 2023-09-23T20:33:40.688 INFO:teuthology.orchestra.run.smithi070.stdout: Running scriptlet: ceph-mgr-rook-2:17.2.6-1123.g61431f6b.el8.noarch 13/13 2023-09-23T20:33:40.688 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : ceph-mgr-rook-2:17.2.6-1123.g61431f6b.el8.noarch 1/13 2023-09-23T20:33:40.688 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : python3-cachetools-3.1.1-4.el8.noarch 2/13 2023-09-23T20:33:40.688 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : python3-certifi-2018.10.15-7.el8.noarch 3/13 2023-09-23T20:33:40.688 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : python3-google-auth-1:1.1.1-10.el8.noarch 4/13 2023-09-23T20:33:40.688 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : python3-kubernetes-1:11.0.0-6.el8.noarch 5/13 2023-09-23T20:33:40.688 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : python3-rsa-4.9-2.el8.noarch 6/13 2023-09-23T20:33:40.688 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : python3-websocket-client-0.56.0-5.el8.noarch 7/13 2023-09-23T20:33:40.688 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : python3-requests-oauthlib-1.0.0-1.el8.noarch 8/13 2023-09-23T20:33:40.688 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : python3-oauthlib-2.1.0-1.el8.noarch 9/13 2023-09-23T20:33:40.689 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : python3-pyasn1-modules-0.3.7-6.el8.noarch 10/13 2023-09-23T20:33:40.689 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : python3-jsonpatch-1.21-2.el8.noarch 11/13 2023-09-23T20:33:40.689 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : python3-pyasn1-0.3.7-6.el8.noarch 12/13 2023-09-23T20:33:41.777 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : python3-jsonpointer-1.10-11.el8.noarch 13/13 2023-09-23T20:33:41.777 INFO:teuthology.orchestra.run.smithi070.stdout:Last metadata expiration check: 0:02:25 ago on Sat 23 Sep 2023 08:31:16 PM UTC. 2023-09-23T20:33:42.826 INFO:teuthology.orchestra.run.smithi026.stdout:Installed products updated. 2023-09-23T20:33:43.034 INFO:teuthology.orchestra.run.smithi026.stdout: 2023-09-23T20:33:43.034 INFO:teuthology.orchestra.run.smithi026.stdout:Installed: 2023-09-23T20:33:43.034 INFO:teuthology.orchestra.run.smithi026.stdout: ceph-mgr-diskprediction-local-2:17.2.6-1123.g61431f6b.el8.noarch 2023-09-23T20:33:43.034 INFO:teuthology.orchestra.run.smithi026.stdout: libgfortran-8.5.0-10.1.el8_6.x86_64 2023-09-23T20:33:43.034 INFO:teuthology.orchestra.run.smithi026.stdout: libquadmath-8.5.0-10.1.el8_6.x86_64 2023-09-23T20:33:43.034 INFO:teuthology.orchestra.run.smithi026.stdout: openblas-0.3.15-3.el8.x86_64 2023-09-23T20:33:43.034 INFO:teuthology.orchestra.run.smithi026.stdout: openblas-threads-0.3.15-3.el8.x86_64 2023-09-23T20:33:43.035 INFO:teuthology.orchestra.run.smithi026.stdout: python3-numpy-1:1.14.3-10.el8.x86_64 2023-09-23T20:33:43.035 INFO:teuthology.orchestra.run.smithi026.stdout: python3-numpy-f2py-1:1.14.3-10.el8.x86_64 2023-09-23T20:33:43.035 INFO:teuthology.orchestra.run.smithi026.stdout: python3-scipy-1.0.0-21.module+el8.5.0+10916+41bd434d.x86_64 2023-09-23T20:33:43.035 INFO:teuthology.orchestra.run.smithi026.stdout: 2023-09-23T20:33:43.035 INFO:teuthology.orchestra.run.smithi026.stdout:Complete! 2023-09-23T20:33:43.167 DEBUG:teuthology.orchestra.run.smithi026:> sudo yum -y install ceph-mgr-rook 2023-09-23T20:33:43.540 INFO:teuthology.orchestra.run.smithi026.stdout:Updating Subscription Management repositories. 2023-09-23T20:33:44.157 INFO:teuthology.orchestra.run.smithi070.stdout:Installed products updated. 2023-09-23T20:33:44.338 INFO:teuthology.orchestra.run.smithi070.stdout: 2023-09-23T20:33:44.339 INFO:teuthology.orchestra.run.smithi070.stdout:Installed: 2023-09-23T20:33:44.339 INFO:teuthology.orchestra.run.smithi070.stdout: ceph-mgr-rook-2:17.2.6-1123.g61431f6b.el8.noarch 2023-09-23T20:33:44.339 INFO:teuthology.orchestra.run.smithi070.stdout: python3-cachetools-3.1.1-4.el8.noarch 2023-09-23T20:33:44.339 INFO:teuthology.orchestra.run.smithi070.stdout: python3-certifi-2018.10.15-7.el8.noarch 2023-09-23T20:33:44.339 INFO:teuthology.orchestra.run.smithi070.stdout: python3-google-auth-1:1.1.1-10.el8.noarch 2023-09-23T20:33:44.339 INFO:teuthology.orchestra.run.smithi070.stdout: python3-jsonpatch-1.21-2.el8.noarch 2023-09-23T20:33:44.339 INFO:teuthology.orchestra.run.smithi070.stdout: python3-jsonpointer-1.10-11.el8.noarch 2023-09-23T20:33:44.339 INFO:teuthology.orchestra.run.smithi070.stdout: python3-kubernetes-1:11.0.0-6.el8.noarch 2023-09-23T20:33:44.339 INFO:teuthology.orchestra.run.smithi070.stdout: python3-oauthlib-2.1.0-1.el8.noarch 2023-09-23T20:33:44.339 INFO:teuthology.orchestra.run.smithi070.stdout: python3-pyasn1-0.3.7-6.el8.noarch 2023-09-23T20:33:44.339 INFO:teuthology.orchestra.run.smithi070.stdout: python3-pyasn1-modules-0.3.7-6.el8.noarch 2023-09-23T20:33:44.339 INFO:teuthology.orchestra.run.smithi070.stdout: python3-requests-oauthlib-1.0.0-1.el8.noarch 2023-09-23T20:33:44.340 INFO:teuthology.orchestra.run.smithi070.stdout: python3-rsa-4.9-2.el8.noarch 2023-09-23T20:33:44.340 INFO:teuthology.orchestra.run.smithi070.stdout: python3-websocket-client-0.56.0-5.el8.noarch 2023-09-23T20:33:44.340 INFO:teuthology.orchestra.run.smithi070.stdout: 2023-09-23T20:33:44.340 INFO:teuthology.orchestra.run.smithi070.stdout:Complete! 2023-09-23T20:33:44.436 INFO:teuthology.orchestra.run.smithi026.stdout:Last metadata expiration check: 0:02:26 ago on Sat 23 Sep 2023 08:31:18 PM UTC. 2023-09-23T20:33:44.464 DEBUG:teuthology.orchestra.run.smithi070:> sudo yum -y install ceph-mgr-cephadm 2023-09-23T20:33:44.815 INFO:teuthology.orchestra.run.smithi070.stdout:Updating Subscription Management repositories. 2023-09-23T20:33:45.620 INFO:teuthology.orchestra.run.smithi070.stdout:Last metadata expiration check: 0:02:29 ago on Sat 23 Sep 2023 08:31:16 PM UTC. 2023-09-23T20:33:45.989 INFO:teuthology.orchestra.run.smithi026.stdout:Dependencies resolved. 2023-09-23T20:33:45.990 INFO:teuthology.orchestra.run.smithi026.stdout:======================================================================================================== 2023-09-23T20:33:45.990 INFO:teuthology.orchestra.run.smithi026.stdout: Package Arch Version Repository Size 2023-09-23T20:33:45.991 INFO:teuthology.orchestra.run.smithi026.stdout:======================================================================================================== 2023-09-23T20:33:45.991 INFO:teuthology.orchestra.run.smithi026.stdout:Installing: 2023-09-23T20:33:45.991 INFO:teuthology.orchestra.run.smithi026.stdout: ceph-mgr-rook noarch 2:17.2.6-1123.g61431f6b.el8 ceph-noarch 47 k 2023-09-23T20:33:45.991 INFO:teuthology.orchestra.run.smithi026.stdout:Installing dependencies: 2023-09-23T20:33:45.991 INFO:teuthology.orchestra.run.smithi026.stdout: python3-cachetools noarch 3.1.1-4.el8 epel 33 k 2023-09-23T20:33:45.991 INFO:teuthology.orchestra.run.smithi026.stdout: python3-certifi noarch 2018.10.15-7.el8 epel 16 k 2023-09-23T20:33:45.991 INFO:teuthology.orchestra.run.smithi026.stdout: python3-google-auth noarch 1:1.1.1-10.el8 epel 81 k 2023-09-23T20:33:45.991 INFO:teuthology.orchestra.run.smithi026.stdout: python3-jsonpatch noarch 1.21-2.el8 rhel-8-for-x86_64-appstream-rpms 27 k 2023-09-23T20:33:45.991 INFO:teuthology.orchestra.run.smithi026.stdout: python3-jsonpointer noarch 1.10-11.el8 rhel-8-for-x86_64-appstream-rpms 20 k 2023-09-23T20:33:45.991 INFO:teuthology.orchestra.run.smithi026.stdout: python3-kubernetes noarch 1:11.0.0-6.el8 epel 1.2 M 2023-09-23T20:33:45.991 INFO:teuthology.orchestra.run.smithi026.stdout: python3-oauthlib noarch 2.1.0-1.el8 rhel-8-for-x86_64-baseos-rpms 155 k 2023-09-23T20:33:45.991 INFO:teuthology.orchestra.run.smithi026.stdout: python3-pyasn1 noarch 0.3.7-6.el8 rhel-8-for-x86_64-appstream-rpms 126 k 2023-09-23T20:33:45.992 INFO:teuthology.orchestra.run.smithi026.stdout: python3-pyasn1-modules noarch 0.3.7-6.el8 rhel-8-for-x86_64-appstream-rpms 110 k 2023-09-23T20:33:45.992 INFO:teuthology.orchestra.run.smithi026.stdout: python3-requests-oauthlib noarch 1.0.0-1.el8 rhel-8-for-x86_64-baseos-rpms 43 k 2023-09-23T20:33:45.992 INFO:teuthology.orchestra.run.smithi026.stdout: python3-rsa noarch 4.9-2.el8 epel 64 k 2023-09-23T20:33:45.992 INFO:teuthology.orchestra.run.smithi026.stdout: python3-websocket-client noarch 0.56.0-5.el8 epel 61 k 2023-09-23T20:33:45.992 INFO:teuthology.orchestra.run.smithi026.stdout: 2023-09-23T20:33:45.992 INFO:teuthology.orchestra.run.smithi026.stdout:Transaction Summary 2023-09-23T20:33:45.992 INFO:teuthology.orchestra.run.smithi026.stdout:======================================================================================================== 2023-09-23T20:33:45.992 INFO:teuthology.orchestra.run.smithi026.stdout:Install 13 Packages 2023-09-23T20:33:45.992 INFO:teuthology.orchestra.run.smithi026.stdout: 2023-09-23T20:33:45.993 INFO:teuthology.orchestra.run.smithi026.stdout:Total download size: 1.9 M 2023-09-23T20:33:45.993 INFO:teuthology.orchestra.run.smithi026.stdout:Installed size: 25 M 2023-09-23T20:33:45.993 INFO:teuthology.orchestra.run.smithi026.stdout:Downloading Packages: 2023-09-23T20:33:46.298 INFO:teuthology.orchestra.run.smithi026.stdout:(1/13): python3-certifi-2018.10.15-7.el8.noarch 69 kB/s | 16 kB 00:00 2023-09-23T20:33:46.324 INFO:teuthology.orchestra.run.smithi026.stdout:(2/13): ceph-mgr-rook-17.2.6-1123.g61431f6b.el8 180 kB/s | 47 kB 00:00 2023-09-23T20:33:46.349 INFO:teuthology.orchestra.run.smithi026.stdout:(3/13): python3-cachetools-3.1.1-4.el8.noarch.r 114 kB/s | 33 kB 00:00 2023-09-23T20:33:46.457 INFO:teuthology.orchestra.run.smithi026.stdout:(4/13): python3-google-auth-1.1.1-10.el8.noarch 511 kB/s | 81 kB 00:00 2023-09-23T20:33:46.558 INFO:teuthology.orchestra.run.smithi026.stdout:(5/13): python3-websocket-client-0.56.0-5.el8.n 616 kB/s | 61 kB 00:00 2023-09-23T20:33:46.654 INFO:teuthology.orchestra.run.smithi026.stdout:(6/13): python3-requests-oauthlib-1.0.0-1.el8.n 445 kB/s | 43 kB 00:00 2023-09-23T20:33:46.676 INFO:teuthology.orchestra.run.smithi026.stdout:(7/13): python3-rsa-4.9-2.el8.noarch.rpm 197 kB/s | 64 kB 00:00 2023-09-23T20:33:46.693 INFO:teuthology.orchestra.run.smithi026.stdout:(8/13): python3-oauthlib-2.1.0-1.el8.noarch.rpm 4.0 MB/s | 155 kB 00:00 2023-09-23T20:33:46.734 INFO:teuthology.orchestra.run.smithi026.stdout:(9/13): python3-kubernetes-11.0.0-6.el8.noarch. 2.8 MB/s | 1.2 MB 00:00 2023-09-23T20:33:46.761 INFO:teuthology.orchestra.run.smithi026.stdout:(10/13): python3-jsonpatch-1.21-2.el8.noarch.rp 454 kB/s | 27 kB 00:00 2023-09-23T20:33:46.793 INFO:teuthology.orchestra.run.smithi026.stdout:(11/13): python3-pyasn1-modules-0.3.7-6.el8.noa 936 kB/s | 110 kB 00:00 2023-09-23T20:33:46.820 INFO:teuthology.orchestra.run.smithi026.stdout:(12/13): python3-jsonpointer-1.10-11.el8.noarch 399 kB/s | 20 kB 00:00 2023-09-23T20:33:46.861 INFO:teuthology.orchestra.run.smithi026.stdout:(13/13): python3-pyasn1-0.3.7-6.el8.noarch.rpm 1.0 MB/s | 126 kB 00:00 2023-09-23T20:33:46.863 INFO:teuthology.orchestra.run.smithi026.stdout:-------------------------------------------------------------------------------- 2023-09-23T20:33:46.863 INFO:teuthology.orchestra.run.smithi026.stdout:Total 2.2 MB/s | 1.9 MB 00:00 2023-09-23T20:33:46.929 INFO:teuthology.orchestra.run.smithi026.stdout:Running transaction check 2023-09-23T20:33:46.947 INFO:teuthology.orchestra.run.smithi026.stdout:Transaction check succeeded. 2023-09-23T20:33:46.947 INFO:teuthology.orchestra.run.smithi026.stdout:Running transaction test 2023-09-23T20:33:47.018 INFO:teuthology.orchestra.run.smithi070.stdout:Dependencies resolved. 2023-09-23T20:33:47.018 INFO:teuthology.orchestra.run.smithi070.stdout:=============================================================================================== 2023-09-23T20:33:47.019 INFO:teuthology.orchestra.run.smithi070.stdout: Package Arch Version Repository Size 2023-09-23T20:33:47.019 INFO:teuthology.orchestra.run.smithi070.stdout:=============================================================================================== 2023-09-23T20:33:47.019 INFO:teuthology.orchestra.run.smithi070.stdout:Installing: 2023-09-23T20:33:47.019 INFO:teuthology.orchestra.run.smithi070.stdout: ceph-mgr-cephadm noarch 2:17.2.6-1123.g61431f6b.el8 ceph-noarch 121 k 2023-09-23T20:33:47.019 INFO:teuthology.orchestra.run.smithi070.stdout:Installing dependencies: 2023-09-23T20:33:47.019 INFO:teuthology.orchestra.run.smithi070.stdout: python3-asyncssh noarch 2.7.0-2.el8 epel 476 k 2023-09-23T20:33:47.019 INFO:teuthology.orchestra.run.smithi070.stdout: python3-babel noarch 2.5.1-7.el8 rhel-8-for-x86_64-appstream-rpms 4.8 M 2023-09-23T20:33:47.019 INFO:teuthology.orchestra.run.smithi070.stdout: python3-jinja2 noarch 2.10.1-3.el8 rhel-8-for-x86_64-appstream-rpms 538 k 2023-09-23T20:33:47.019 INFO:teuthology.orchestra.run.smithi070.stdout: python3-natsort noarch 7.1.1-2.el8 epel 60 k 2023-09-23T20:33:47.019 INFO:teuthology.orchestra.run.smithi070.stdout: 2023-09-23T20:33:47.019 INFO:teuthology.orchestra.run.smithi070.stdout:Transaction Summary 2023-09-23T20:33:47.019 INFO:teuthology.orchestra.run.smithi070.stdout:=============================================================================================== 2023-09-23T20:33:47.020 INFO:teuthology.orchestra.run.smithi070.stdout:Install 5 Packages 2023-09-23T20:33:47.020 INFO:teuthology.orchestra.run.smithi070.stdout: 2023-09-23T20:33:47.020 INFO:teuthology.orchestra.run.smithi070.stdout:Total download size: 6.0 M 2023-09-23T20:33:47.020 INFO:teuthology.orchestra.run.smithi070.stdout:Installed size: 26 M 2023-09-23T20:33:47.020 INFO:teuthology.orchestra.run.smithi070.stdout:Downloading Packages: 2023-09-23T20:33:47.024 INFO:teuthology.orchestra.run.smithi026.stdout:Transaction test succeeded. 2023-09-23T20:33:47.028 INFO:teuthology.orchestra.run.smithi026.stdout:Running transaction 2023-09-23T20:33:47.293 INFO:teuthology.orchestra.run.smithi070.stdout:(1/5): ceph-mgr-cephadm-17.2.6-1123.g61431f6b.e 604 kB/s | 121 kB 00:00 2023-09-23T20:33:47.341 INFO:teuthology.orchestra.run.smithi070.stdout:(2/5): python3-natsort-7.1.1-2.el8.noarch.rpm 242 kB/s | 60 kB 00:00 2023-09-23T20:33:47.410 INFO:teuthology.orchestra.run.smithi070.stdout:(3/5): python3-jinja2-2.10.1-3.el8.noarch.rpm 4.5 MB/s | 538 kB 00:00 2023-09-23T20:33:47.443 INFO:teuthology.orchestra.run.smithi070.stdout:(4/5): python3-asyncssh-2.7.0-2.el8.noarch.rpm 1.3 MB/s | 476 kB 00:00 2023-09-23T20:33:47.527 INFO:teuthology.orchestra.run.smithi070.stdout:(5/5): python3-babel-2.5.1-7.el8.noarch.rpm 27 MB/s | 4.8 MB 00:00 2023-09-23T20:33:47.529 INFO:teuthology.orchestra.run.smithi070.stdout:-------------------------------------------------------------------------------- 2023-09-23T20:33:47.529 INFO:teuthology.orchestra.run.smithi070.stdout:Total 12 MB/s | 6.0 MB 00:00 2023-09-23T20:33:47.563 INFO:teuthology.orchestra.run.smithi026.stdout: Preparing : 1/1 2023-09-23T20:33:47.582 INFO:teuthology.orchestra.run.smithi070.stdout:Running transaction check 2023-09-23T20:33:47.593 INFO:teuthology.orchestra.run.smithi070.stdout:Transaction check succeeded. 2023-09-23T20:33:47.593 INFO:teuthology.orchestra.run.smithi070.stdout:Running transaction test 2023-09-23T20:33:47.671 INFO:teuthology.orchestra.run.smithi070.stdout:Transaction test succeeded. 2023-09-23T20:33:47.675 INFO:teuthology.orchestra.run.smithi070.stdout:Running transaction 2023-09-23T20:33:47.744 INFO:teuthology.orchestra.run.smithi026.stdout: Installing : python3-pyasn1-0.3.7-6.el8.noarch 1/13 2023-09-23T20:33:47.955 INFO:teuthology.orchestra.run.smithi026.stdout: Installing : python3-rsa-4.9-2.el8.noarch 2/13 2023-09-23T20:33:48.115 INFO:teuthology.orchestra.run.smithi026.stdout: Installing : python3-pyasn1-modules-0.3.7-6.el8.noarch 3/13 2023-09-23T20:33:48.299 INFO:teuthology.orchestra.run.smithi026.stdout: Installing : python3-jsonpointer-1.10-11.el8.noarch 4/13 2023-09-23T20:33:48.517 INFO:teuthology.orchestra.run.smithi026.stdout: Installing : python3-jsonpatch-1.21-2.el8.noarch 5/13 2023-09-23T20:33:48.616 INFO:teuthology.orchestra.run.smithi070.stdout: Preparing : 1/1 2023-09-23T20:33:48.739 INFO:teuthology.orchestra.run.smithi026.stdout: Installing : python3-oauthlib-2.1.0-1.el8.noarch 6/13 2023-09-23T20:33:48.929 INFO:teuthology.orchestra.run.smithi026.stdout: Installing : python3-requests-oauthlib-1.0.0-1.el8.noarch 7/13 2023-09-23T20:33:48.947 INFO:teuthology.orchestra.run.smithi070.stdout: Installing : python3-babel-2.5.1-7.el8.noarch 1/5 2023-09-23T20:33:49.157 INFO:teuthology.orchestra.run.smithi026.stdout: Installing : python3-websocket-client-0.56.0-5.el8.noarch 8/13 2023-09-23T20:33:49.186 INFO:teuthology.orchestra.run.smithi070.stdout: Installing : python3-jinja2-2.10.1-3.el8.noarch 2/5 2023-09-23T20:33:49.341 INFO:teuthology.orchestra.run.smithi026.stdout: Installing : python3-certifi-2018.10.15-7.el8.noarch 9/13 2023-09-23T20:33:49.514 INFO:teuthology.orchestra.run.smithi070.stdout: Installing : python3-natsort-7.1.1-2.el8.noarch 3/5 2023-09-23T20:33:49.584 INFO:teuthology.orchestra.run.smithi026.stdout: Installing : python3-cachetools-3.1.1-4.el8.noarch 10/13 2023-09-23T20:33:49.769 INFO:teuthology.orchestra.run.smithi070.stdout: Installing : python3-asyncssh-2.7.0-2.el8.noarch 4/5 2023-09-23T20:33:49.959 INFO:teuthology.orchestra.run.smithi070.stdout: Installing : ceph-mgr-cephadm-2:17.2.6-1123.g61431f6b.el8.noarch 5/5 2023-09-23T20:33:50.178 INFO:teuthology.orchestra.run.smithi070.stdout: Running scriptlet: ceph-mgr-cephadm-2:17.2.6-1123.g61431f6b.el8.noarch 5/5 2023-09-23T20:33:50.178 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : ceph-mgr-cephadm-2:17.2.6-1123.g61431f6b.el8.noarch 1/5 2023-09-23T20:33:50.178 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : python3-asyncssh-2.7.0-2.el8.noarch 2/5 2023-09-23T20:33:50.178 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : python3-natsort-7.1.1-2.el8.noarch 3/5 2023-09-23T20:33:50.178 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : python3-jinja2-2.10.1-3.el8.noarch 4/5 2023-09-23T20:33:50.740 INFO:teuthology.orchestra.run.smithi026.stdout: Installing : python3-google-auth-1:1.1.1-10.el8.noarch 11/13 2023-09-23T20:33:51.009 INFO:teuthology.orchestra.run.smithi026.stdout: Installing : python3-kubernetes-1:11.0.0-6.el8.noarch 12/13 2023-09-23T20:33:51.174 INFO:teuthology.orchestra.run.smithi026.stdout: Installing : ceph-mgr-rook-2:17.2.6-1123.g61431f6b.el8.noarch 13/13 2023-09-23T20:33:51.199 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : python3-babel-2.5.1-7.el8.noarch 5/5 2023-09-23T20:33:51.199 INFO:teuthology.orchestra.run.smithi070.stdout:Last metadata expiration check: 0:02:35 ago on Sat 23 Sep 2023 08:31:16 PM UTC. 2023-09-23T20:33:51.462 INFO:teuthology.orchestra.run.smithi026.stdout: Running scriptlet: ceph-mgr-rook-2:17.2.6-1123.g61431f6b.el8.noarch 13/13 2023-09-23T20:33:51.462 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : ceph-mgr-rook-2:17.2.6-1123.g61431f6b.el8.noarch 1/13 2023-09-23T20:33:51.463 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : python3-cachetools-3.1.1-4.el8.noarch 2/13 2023-09-23T20:33:51.463 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : python3-certifi-2018.10.15-7.el8.noarch 3/13 2023-09-23T20:33:51.463 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : python3-google-auth-1:1.1.1-10.el8.noarch 4/13 2023-09-23T20:33:51.463 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : python3-kubernetes-1:11.0.0-6.el8.noarch 5/13 2023-09-23T20:33:51.463 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : python3-rsa-4.9-2.el8.noarch 6/13 2023-09-23T20:33:51.463 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : python3-websocket-client-0.56.0-5.el8.noarch 7/13 2023-09-23T20:33:51.463 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : python3-requests-oauthlib-1.0.0-1.el8.noarch 8/13 2023-09-23T20:33:51.463 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : python3-oauthlib-2.1.0-1.el8.noarch 9/13 2023-09-23T20:33:51.463 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : python3-pyasn1-modules-0.3.7-6.el8.noarch 10/13 2023-09-23T20:33:51.463 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : python3-jsonpatch-1.21-2.el8.noarch 11/13 2023-09-23T20:33:51.465 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : python3-pyasn1-0.3.7-6.el8.noarch 12/13 2023-09-23T20:33:52.597 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : python3-jsonpointer-1.10-11.el8.noarch 13/13 2023-09-23T20:33:52.597 INFO:teuthology.orchestra.run.smithi026.stdout:Last metadata expiration check: 0:02:34 ago on Sat 23 Sep 2023 08:31:18 PM UTC. 2023-09-23T20:33:53.550 INFO:teuthology.orchestra.run.smithi070.stdout:Installed products updated. 2023-09-23T20:33:53.737 INFO:teuthology.orchestra.run.smithi070.stdout: 2023-09-23T20:33:53.737 INFO:teuthology.orchestra.run.smithi070.stdout:Installed: 2023-09-23T20:33:53.738 INFO:teuthology.orchestra.run.smithi070.stdout: ceph-mgr-cephadm-2:17.2.6-1123.g61431f6b.el8.noarch 2023-09-23T20:33:53.738 INFO:teuthology.orchestra.run.smithi070.stdout: python3-asyncssh-2.7.0-2.el8.noarch 2023-09-23T20:33:53.738 INFO:teuthology.orchestra.run.smithi070.stdout: python3-babel-2.5.1-7.el8.noarch 2023-09-23T20:33:53.738 INFO:teuthology.orchestra.run.smithi070.stdout: python3-jinja2-2.10.1-3.el8.noarch 2023-09-23T20:33:53.738 INFO:teuthology.orchestra.run.smithi070.stdout: python3-natsort-7.1.1-2.el8.noarch 2023-09-23T20:33:53.738 INFO:teuthology.orchestra.run.smithi070.stdout: 2023-09-23T20:33:53.738 INFO:teuthology.orchestra.run.smithi070.stdout:Complete! 2023-09-23T20:33:53.869 DEBUG:teuthology.orchestra.run.smithi070:> sudo yum -y install ceph-fuse 2023-09-23T20:33:54.223 INFO:teuthology.orchestra.run.smithi070.stdout:Updating Subscription Management repositories. 2023-09-23T20:33:55.051 INFO:teuthology.orchestra.run.smithi070.stdout:Last metadata expiration check: 0:02:39 ago on Sat 23 Sep 2023 08:31:16 PM UTC. 2023-09-23T20:33:55.121 INFO:teuthology.orchestra.run.smithi026.stdout:Installed products updated. 2023-09-23T20:33:55.325 INFO:teuthology.orchestra.run.smithi026.stdout: 2023-09-23T20:33:55.325 INFO:teuthology.orchestra.run.smithi026.stdout:Installed: 2023-09-23T20:33:55.326 INFO:teuthology.orchestra.run.smithi026.stdout: ceph-mgr-rook-2:17.2.6-1123.g61431f6b.el8.noarch 2023-09-23T20:33:55.326 INFO:teuthology.orchestra.run.smithi026.stdout: python3-cachetools-3.1.1-4.el8.noarch 2023-09-23T20:33:55.326 INFO:teuthology.orchestra.run.smithi026.stdout: python3-certifi-2018.10.15-7.el8.noarch 2023-09-23T20:33:55.326 INFO:teuthology.orchestra.run.smithi026.stdout: python3-google-auth-1:1.1.1-10.el8.noarch 2023-09-23T20:33:55.326 INFO:teuthology.orchestra.run.smithi026.stdout: python3-jsonpatch-1.21-2.el8.noarch 2023-09-23T20:33:55.326 INFO:teuthology.orchestra.run.smithi026.stdout: python3-jsonpointer-1.10-11.el8.noarch 2023-09-23T20:33:55.326 INFO:teuthology.orchestra.run.smithi026.stdout: python3-kubernetes-1:11.0.0-6.el8.noarch 2023-09-23T20:33:55.326 INFO:teuthology.orchestra.run.smithi026.stdout: python3-oauthlib-2.1.0-1.el8.noarch 2023-09-23T20:33:55.326 INFO:teuthology.orchestra.run.smithi026.stdout: python3-pyasn1-0.3.7-6.el8.noarch 2023-09-23T20:33:55.326 INFO:teuthology.orchestra.run.smithi026.stdout: python3-pyasn1-modules-0.3.7-6.el8.noarch 2023-09-23T20:33:55.326 INFO:teuthology.orchestra.run.smithi026.stdout: python3-requests-oauthlib-1.0.0-1.el8.noarch 2023-09-23T20:33:55.326 INFO:teuthology.orchestra.run.smithi026.stdout: python3-rsa-4.9-2.el8.noarch 2023-09-23T20:33:55.327 INFO:teuthology.orchestra.run.smithi026.stdout: python3-websocket-client-0.56.0-5.el8.noarch 2023-09-23T20:33:55.327 INFO:teuthology.orchestra.run.smithi026.stdout: 2023-09-23T20:33:55.327 INFO:teuthology.orchestra.run.smithi026.stdout:Complete! 2023-09-23T20:33:55.464 DEBUG:teuthology.orchestra.run.smithi026:> sudo yum -y install ceph-mgr-cephadm 2023-09-23T20:33:55.839 INFO:teuthology.orchestra.run.smithi026.stdout:Updating Subscription Management repositories. 2023-09-23T20:33:56.504 INFO:teuthology.orchestra.run.smithi070.stdout:Dependencies resolved. 2023-09-23T20:33:56.505 INFO:teuthology.orchestra.run.smithi070.stdout:================================================================================ 2023-09-23T20:33:56.505 INFO:teuthology.orchestra.run.smithi070.stdout: Package Arch Version Repository Size 2023-09-23T20:33:56.505 INFO:teuthology.orchestra.run.smithi070.stdout:================================================================================ 2023-09-23T20:33:56.505 INFO:teuthology.orchestra.run.smithi070.stdout:Installing: 2023-09-23T20:33:56.505 INFO:teuthology.orchestra.run.smithi070.stdout: ceph-fuse 2023-09-23T20:33:56.505 INFO:teuthology.orchestra.run.smithi070.stdout: x86_64 2:17.2.6-1123.g61431f6b.el8 ceph 874 k 2023-09-23T20:33:56.505 INFO:teuthology.orchestra.run.smithi070.stdout:Installing dependencies: 2023-09-23T20:33:56.505 INFO:teuthology.orchestra.run.smithi070.stdout: fuse x86_64 2.9.7-15.el8 rhel-8-for-x86_64-baseos-rpms 83 k 2023-09-23T20:33:56.505 INFO:teuthology.orchestra.run.smithi070.stdout: 2023-09-23T20:33:56.505 INFO:teuthology.orchestra.run.smithi070.stdout:Transaction Summary 2023-09-23T20:33:56.505 INFO:teuthology.orchestra.run.smithi070.stdout:================================================================================ 2023-09-23T20:33:56.506 INFO:teuthology.orchestra.run.smithi070.stdout:Install 2 Packages 2023-09-23T20:33:56.506 INFO:teuthology.orchestra.run.smithi070.stdout: 2023-09-23T20:33:56.506 INFO:teuthology.orchestra.run.smithi070.stdout:Total download size: 957 k 2023-09-23T20:33:56.506 INFO:teuthology.orchestra.run.smithi070.stdout:Installed size: 3.0 M 2023-09-23T20:33:56.506 INFO:teuthology.orchestra.run.smithi070.stdout:Downloading Packages: 2023-09-23T20:33:56.576 INFO:teuthology.orchestra.run.smithi070.stdout:(1/2): fuse-2.9.7-15.el8.x86_64.rpm 1.2 MB/s | 83 kB 00:00 2023-09-23T20:33:56.691 INFO:teuthology.orchestra.run.smithi026.stdout:Last metadata expiration check: 0:02:38 ago on Sat 23 Sep 2023 08:31:18 PM UTC. 2023-09-23T20:33:56.777 INFO:teuthology.orchestra.run.smithi070.stdout:(2/2): ceph-fuse-17.2.6-1123.g61431f6b.el8.x86_ 3.2 MB/s | 874 kB 00:00 2023-09-23T20:33:56.778 INFO:teuthology.orchestra.run.smithi070.stdout:-------------------------------------------------------------------------------- 2023-09-23T20:33:56.778 INFO:teuthology.orchestra.run.smithi070.stdout:Total 3.4 MB/s | 957 kB 00:00 2023-09-23T20:33:56.788 INFO:teuthology.orchestra.run.smithi070.stdout:Running transaction check 2023-09-23T20:33:56.809 INFO:teuthology.orchestra.run.smithi070.stdout:Transaction check succeeded. 2023-09-23T20:33:56.809 INFO:teuthology.orchestra.run.smithi070.stdout:Running transaction test 2023-09-23T20:33:56.883 INFO:teuthology.orchestra.run.smithi070.stdout:Transaction test succeeded. 2023-09-23T20:33:56.886 INFO:teuthology.orchestra.run.smithi070.stdout:Running transaction 2023-09-23T20:33:57.036 INFO:teuthology.orchestra.run.smithi070.stdout: Preparing : 1/1 2023-09-23T20:33:57.317 INFO:teuthology.orchestra.run.smithi070.stdout: Installing : fuse-2.9.7-15.el8.x86_64 1/2 2023-09-23T20:33:57.473 INFO:teuthology.orchestra.run.smithi070.stdout: Installing : ceph-fuse-2:17.2.6-1123.g61431f6b.el8.x86_64 2/2 2023-09-23T20:33:57.786 INFO:teuthology.orchestra.run.smithi070.stdout: Running scriptlet: ceph-fuse-2:17.2.6-1123.g61431f6b.el8.x86_64 2/2 2023-09-23T20:33:57.787 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : ceph-fuse-2:17.2.6-1123.g61431f6b.el8.x86_64 1/2 2023-09-23T20:33:58.264 INFO:teuthology.orchestra.run.smithi026.stdout:Dependencies resolved. 2023-09-23T20:33:58.265 INFO:teuthology.orchestra.run.smithi026.stdout:=============================================================================================== 2023-09-23T20:33:58.265 INFO:teuthology.orchestra.run.smithi026.stdout: Package Arch Version Repository Size 2023-09-23T20:33:58.265 INFO:teuthology.orchestra.run.smithi026.stdout:=============================================================================================== 2023-09-23T20:33:58.265 INFO:teuthology.orchestra.run.smithi026.stdout:Installing: 2023-09-23T20:33:58.265 INFO:teuthology.orchestra.run.smithi026.stdout: ceph-mgr-cephadm noarch 2:17.2.6-1123.g61431f6b.el8 ceph-noarch 121 k 2023-09-23T20:33:58.266 INFO:teuthology.orchestra.run.smithi026.stdout:Installing dependencies: 2023-09-23T20:33:58.266 INFO:teuthology.orchestra.run.smithi026.stdout: python3-asyncssh noarch 2.7.0-2.el8 epel 476 k 2023-09-23T20:33:58.266 INFO:teuthology.orchestra.run.smithi026.stdout: python3-babel noarch 2.5.1-7.el8 rhel-8-for-x86_64-appstream-rpms 4.8 M 2023-09-23T20:33:58.266 INFO:teuthology.orchestra.run.smithi026.stdout: python3-jinja2 noarch 2.10.1-3.el8 rhel-8-for-x86_64-appstream-rpms 538 k 2023-09-23T20:33:58.266 INFO:teuthology.orchestra.run.smithi026.stdout: python3-natsort noarch 7.1.1-2.el8 epel 60 k 2023-09-23T20:33:58.266 INFO:teuthology.orchestra.run.smithi026.stdout: 2023-09-23T20:33:58.266 INFO:teuthology.orchestra.run.smithi026.stdout:Transaction Summary 2023-09-23T20:33:58.266 INFO:teuthology.orchestra.run.smithi026.stdout:=============================================================================================== 2023-09-23T20:33:58.266 INFO:teuthology.orchestra.run.smithi026.stdout:Install 5 Packages 2023-09-23T20:33:58.266 INFO:teuthology.orchestra.run.smithi026.stdout: 2023-09-23T20:33:58.267 INFO:teuthology.orchestra.run.smithi026.stdout:Total download size: 6.0 M 2023-09-23T20:33:58.267 INFO:teuthology.orchestra.run.smithi026.stdout:Installed size: 26 M 2023-09-23T20:33:58.267 INFO:teuthology.orchestra.run.smithi026.stdout:Downloading Packages: 2023-09-23T20:33:58.518 INFO:teuthology.orchestra.run.smithi026.stdout:(1/5): python3-natsort-7.1.1-2.el8.noarch.rpm 654 kB/s | 60 kB 00:00 2023-09-23T20:33:58.543 INFO:teuthology.orchestra.run.smithi026.stdout:(2/5): python3-asyncssh-2.7.0-2.el8.noarch.rpm 4.0 MB/s | 476 kB 00:00 2023-09-23T20:33:58.602 INFO:teuthology.orchestra.run.smithi026.stdout:(3/5): python3-jinja2-2.10.1-3.el8.noarch.rpm 6.3 MB/s | 538 kB 00:00 2023-09-23T20:33:58.710 INFO:teuthology.orchestra.run.smithi026.stdout:(4/5): python3-babel-2.5.1-7.el8.noarch.rpm 30 MB/s | 4.8 MB 00:00 2023-09-23T20:33:58.735 INFO:teuthology.orchestra.run.smithi026.stdout:(5/5): ceph-mgr-cephadm-17.2.6-1123.g61431f6b.e 390 kB/s | 121 kB 00:00 2023-09-23T20:33:58.736 INFO:teuthology.orchestra.run.smithi026.stdout:-------------------------------------------------------------------------------- 2023-09-23T20:33:58.736 INFO:teuthology.orchestra.run.smithi026.stdout:Total 13 MB/s | 6.0 MB 00:00 2023-09-23T20:33:58.790 INFO:teuthology.orchestra.run.smithi026.stdout:Running transaction check 2023-09-23T20:33:58.802 INFO:teuthology.orchestra.run.smithi026.stdout:Transaction check succeeded. 2023-09-23T20:33:58.802 INFO:teuthology.orchestra.run.smithi026.stdout:Running transaction test 2023-09-23T20:33:58.844 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : fuse-2.9.7-15.el8.x86_64 2/2 2023-09-23T20:33:58.844 INFO:teuthology.orchestra.run.smithi070.stdout:Last metadata expiration check: 0:02:42 ago on Sat 23 Sep 2023 08:31:16 PM UTC. 2023-09-23T20:33:58.886 INFO:teuthology.orchestra.run.smithi026.stdout:Transaction test succeeded. 2023-09-23T20:33:58.890 INFO:teuthology.orchestra.run.smithi026.stdout:Running transaction 2023-09-23T20:33:59.910 INFO:teuthology.orchestra.run.smithi026.stdout: Preparing : 1/1 2023-09-23T20:34:00.259 INFO:teuthology.orchestra.run.smithi026.stdout: Installing : python3-babel-2.5.1-7.el8.noarch 1/5 2023-09-23T20:34:00.492 INFO:teuthology.orchestra.run.smithi026.stdout: Installing : python3-jinja2-2.10.1-3.el8.noarch 2/5 2023-09-23T20:34:00.827 INFO:teuthology.orchestra.run.smithi026.stdout: Installing : python3-natsort-7.1.1-2.el8.noarch 3/5 2023-09-23T20:34:01.093 INFO:teuthology.orchestra.run.smithi026.stdout: Installing : python3-asyncssh-2.7.0-2.el8.noarch 4/5 2023-09-23T20:34:01.372 INFO:teuthology.orchestra.run.smithi026.stdout: Installing : ceph-mgr-cephadm-2:17.2.6-1123.g61431f6b.el8.noarch 5/5 2023-09-23T20:34:01.396 INFO:teuthology.orchestra.run.smithi070.stdout:Installed products updated. 2023-09-23T20:34:01.576 INFO:teuthology.orchestra.run.smithi070.stdout: 2023-09-23T20:34:01.576 INFO:teuthology.orchestra.run.smithi070.stdout:Installed: 2023-09-23T20:34:01.576 INFO:teuthology.orchestra.run.smithi070.stdout: ceph-fuse-2:17.2.6-1123.g61431f6b.el8.x86_64 fuse-2.9.7-15.el8.x86_64 2023-09-23T20:34:01.576 INFO:teuthology.orchestra.run.smithi070.stdout: 2023-09-23T20:34:01.576 INFO:teuthology.orchestra.run.smithi070.stdout:Complete! 2023-09-23T20:34:01.624 INFO:teuthology.orchestra.run.smithi026.stdout: Running scriptlet: ceph-mgr-cephadm-2:17.2.6-1123.g61431f6b.el8.noarch 5/5 2023-09-23T20:34:01.624 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : ceph-mgr-cephadm-2:17.2.6-1123.g61431f6b.el8.noarch 1/5 2023-09-23T20:34:01.624 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : python3-asyncssh-2.7.0-2.el8.noarch 2/5 2023-09-23T20:34:01.624 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : python3-natsort-7.1.1-2.el8.noarch 3/5 2023-09-23T20:34:01.624 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : python3-jinja2-2.10.1-3.el8.noarch 4/5 2023-09-23T20:34:01.700 DEBUG:teuthology.orchestra.run.smithi070:> sudo yum -y install librados-devel 2023-09-23T20:34:02.055 INFO:teuthology.orchestra.run.smithi070.stdout:Updating Subscription Management repositories. 2023-09-23T20:34:02.678 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : python3-babel-2.5.1-7.el8.noarch 5/5 2023-09-23T20:34:02.678 INFO:teuthology.orchestra.run.smithi026.stdout:Last metadata expiration check: 0:02:44 ago on Sat 23 Sep 2023 08:31:18 PM UTC. 2023-09-23T20:34:02.920 INFO:teuthology.orchestra.run.smithi070.stdout:Last metadata expiration check: 0:02:46 ago on Sat 23 Sep 2023 08:31:16 PM UTC. 2023-09-23T20:34:04.303 INFO:teuthology.orchestra.run.smithi070.stdout:Dependencies resolved. 2023-09-23T20:34:04.309 INFO:teuthology.orchestra.run.smithi070.stdout:================================================================================ 2023-09-23T20:34:04.310 INFO:teuthology.orchestra.run.smithi070.stdout: Package Arch Version Repo Size 2023-09-23T20:34:04.310 INFO:teuthology.orchestra.run.smithi070.stdout:================================================================================ 2023-09-23T20:34:04.310 INFO:teuthology.orchestra.run.smithi070.stdout:Installing: 2023-09-23T20:34:04.310 INFO:teuthology.orchestra.run.smithi070.stdout: librados-devel x86_64 2:17.2.6-1123.g61431f6b.el8 ceph 130 k 2023-09-23T20:34:04.310 INFO:teuthology.orchestra.run.smithi070.stdout: 2023-09-23T20:34:04.310 INFO:teuthology.orchestra.run.smithi070.stdout:Transaction Summary 2023-09-23T20:34:04.310 INFO:teuthology.orchestra.run.smithi070.stdout:================================================================================ 2023-09-23T20:34:04.311 INFO:teuthology.orchestra.run.smithi070.stdout:Install 1 Package 2023-09-23T20:34:04.311 INFO:teuthology.orchestra.run.smithi070.stdout: 2023-09-23T20:34:04.311 INFO:teuthology.orchestra.run.smithi070.stdout:Total download size: 130 k 2023-09-23T20:34:04.311 INFO:teuthology.orchestra.run.smithi070.stdout:Installed size: 464 k 2023-09-23T20:34:04.311 INFO:teuthology.orchestra.run.smithi070.stdout:Downloading Packages: 2023-09-23T20:34:04.484 INFO:teuthology.orchestra.run.smithi070.stdout:librados-devel-17.2.6-1123.g61431f6b.el8.x86_64 731 kB/s | 130 kB 00:00 2023-09-23T20:34:04.485 INFO:teuthology.orchestra.run.smithi070.stdout:-------------------------------------------------------------------------------- 2023-09-23T20:34:04.485 INFO:teuthology.orchestra.run.smithi070.stdout:Total 726 kB/s | 130 kB 00:00 2023-09-23T20:34:04.486 INFO:teuthology.orchestra.run.smithi070.stdout:Running transaction check 2023-09-23T20:34:04.490 INFO:teuthology.orchestra.run.smithi070.stdout:Transaction check succeeded. 2023-09-23T20:34:04.490 INFO:teuthology.orchestra.run.smithi070.stdout:Running transaction test 2023-09-23T20:34:04.543 INFO:teuthology.orchestra.run.smithi070.stdout:Transaction test succeeded. 2023-09-23T20:34:04.547 INFO:teuthology.orchestra.run.smithi070.stdout:Running transaction 2023-09-23T20:34:04.664 INFO:teuthology.orchestra.run.smithi070.stdout: Preparing : 1/1 2023-09-23T20:34:04.812 INFO:teuthology.orchestra.run.smithi070.stdout: Installing : librados-devel-2:17.2.6-1123.g61431f6b.el8.x86_64 1/1 2023-09-23T20:34:05.067 INFO:teuthology.orchestra.run.smithi070.stdout: Running scriptlet: librados-devel-2:17.2.6-1123.g61431f6b.el8.x86_64 1/1 2023-09-23T20:34:05.469 INFO:teuthology.orchestra.run.smithi026.stdout:Installed products updated. 2023-09-23T20:34:05.671 INFO:teuthology.orchestra.run.smithi026.stdout: 2023-09-23T20:34:05.671 INFO:teuthology.orchestra.run.smithi026.stdout:Installed: 2023-09-23T20:34:05.672 INFO:teuthology.orchestra.run.smithi026.stdout: ceph-mgr-cephadm-2:17.2.6-1123.g61431f6b.el8.noarch 2023-09-23T20:34:05.672 INFO:teuthology.orchestra.run.smithi026.stdout: python3-asyncssh-2.7.0-2.el8.noarch 2023-09-23T20:34:05.673 INFO:teuthology.orchestra.run.smithi026.stdout: python3-babel-2.5.1-7.el8.noarch 2023-09-23T20:34:05.673 INFO:teuthology.orchestra.run.smithi026.stdout: python3-jinja2-2.10.1-3.el8.noarch 2023-09-23T20:34:05.673 INFO:teuthology.orchestra.run.smithi026.stdout: python3-natsort-7.1.1-2.el8.noarch 2023-09-23T20:34:05.674 INFO:teuthology.orchestra.run.smithi026.stdout: 2023-09-23T20:34:05.674 INFO:teuthology.orchestra.run.smithi026.stdout:Complete! 2023-09-23T20:34:05.806 DEBUG:teuthology.orchestra.run.smithi026:> sudo yum -y install ceph-fuse 2023-09-23T20:34:06.099 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : librados-devel-2:17.2.6-1123.g61431f6b.el8.x86_64 1/1 2023-09-23T20:34:06.099 INFO:teuthology.orchestra.run.smithi070.stdout:Last metadata expiration check: 0:02:50 ago on Sat 23 Sep 2023 08:31:16 PM UTC. 2023-09-23T20:34:06.182 INFO:teuthology.orchestra.run.smithi026.stdout:Updating Subscription Management repositories. 2023-09-23T20:34:07.078 INFO:teuthology.orchestra.run.smithi026.stdout:Last metadata expiration check: 0:02:49 ago on Sat 23 Sep 2023 08:31:18 PM UTC. 2023-09-23T20:34:08.632 INFO:teuthology.orchestra.run.smithi026.stdout:Dependencies resolved. 2023-09-23T20:34:08.633 INFO:teuthology.orchestra.run.smithi026.stdout:================================================================================ 2023-09-23T20:34:08.633 INFO:teuthology.orchestra.run.smithi026.stdout: Package Arch Version Repository Size 2023-09-23T20:34:08.633 INFO:teuthology.orchestra.run.smithi026.stdout:================================================================================ 2023-09-23T20:34:08.633 INFO:teuthology.orchestra.run.smithi026.stdout:Installing: 2023-09-23T20:34:08.633 INFO:teuthology.orchestra.run.smithi026.stdout: ceph-fuse 2023-09-23T20:34:08.633 INFO:teuthology.orchestra.run.smithi026.stdout: x86_64 2:17.2.6-1123.g61431f6b.el8 ceph 874 k 2023-09-23T20:34:08.633 INFO:teuthology.orchestra.run.smithi026.stdout:Installing dependencies: 2023-09-23T20:34:08.634 INFO:teuthology.orchestra.run.smithi026.stdout: fuse x86_64 2.9.7-15.el8 rhel-8-for-x86_64-baseos-rpms 83 k 2023-09-23T20:34:08.634 INFO:teuthology.orchestra.run.smithi026.stdout: 2023-09-23T20:34:08.634 INFO:teuthology.orchestra.run.smithi026.stdout:Transaction Summary 2023-09-23T20:34:08.634 INFO:teuthology.orchestra.run.smithi026.stdout:================================================================================ 2023-09-23T20:34:08.634 INFO:teuthology.orchestra.run.smithi026.stdout:Install 2 Packages 2023-09-23T20:34:08.634 INFO:teuthology.orchestra.run.smithi026.stdout: 2023-09-23T20:34:08.634 INFO:teuthology.orchestra.run.smithi026.stdout:Total download size: 957 k 2023-09-23T20:34:08.634 INFO:teuthology.orchestra.run.smithi026.stdout:Installed size: 3.0 M 2023-09-23T20:34:08.634 INFO:teuthology.orchestra.run.smithi026.stdout:Downloading Packages: 2023-09-23T20:34:08.644 INFO:teuthology.orchestra.run.smithi070.stdout:Installed products updated. 2023-09-23T20:34:08.710 INFO:teuthology.orchestra.run.smithi026.stdout:(1/2): fuse-2.9.7-15.el8.x86_64.rpm 1.1 MB/s | 83 kB 00:00 2023-09-23T20:34:08.829 INFO:teuthology.orchestra.run.smithi070.stdout: 2023-09-23T20:34:08.830 INFO:teuthology.orchestra.run.smithi070.stdout:Installed: 2023-09-23T20:34:08.830 INFO:teuthology.orchestra.run.smithi070.stdout: librados-devel-2:17.2.6-1123.g61431f6b.el8.x86_64 2023-09-23T20:34:08.830 INFO:teuthology.orchestra.run.smithi070.stdout: 2023-09-23T20:34:08.830 INFO:teuthology.orchestra.run.smithi070.stdout:Complete! 2023-09-23T20:34:08.963 DEBUG:teuthology.orchestra.run.smithi070:> sudo yum -y install libcephfs2 2023-09-23T20:34:09.206 INFO:teuthology.orchestra.run.smithi026.stdout:(2/2): ceph-fuse-17.2.6-1123.g61431f6b.el8.x86_ 1.5 MB/s | 874 kB 00:00 2023-09-23T20:34:09.207 INFO:teuthology.orchestra.run.smithi026.stdout:-------------------------------------------------------------------------------- 2023-09-23T20:34:09.207 INFO:teuthology.orchestra.run.smithi026.stdout:Total 1.6 MB/s | 957 kB 00:00 2023-09-23T20:34:09.220 INFO:teuthology.orchestra.run.smithi026.stdout:Running transaction check 2023-09-23T20:34:09.244 INFO:teuthology.orchestra.run.smithi026.stdout:Transaction check succeeded. 2023-09-23T20:34:09.244 INFO:teuthology.orchestra.run.smithi026.stdout:Running transaction test 2023-09-23T20:34:09.322 INFO:teuthology.orchestra.run.smithi070.stdout:Updating Subscription Management repositories. 2023-09-23T20:34:09.325 INFO:teuthology.orchestra.run.smithi026.stdout:Transaction test succeeded. 2023-09-23T20:34:09.330 INFO:teuthology.orchestra.run.smithi026.stdout:Running transaction 2023-09-23T20:34:10.013 INFO:teuthology.orchestra.run.smithi026.stdout: Preparing : 1/1 2023-09-23T20:34:10.111 INFO:teuthology.orchestra.run.smithi070.stdout:Last metadata expiration check: 0:02:54 ago on Sat 23 Sep 2023 08:31:16 PM UTC. 2023-09-23T20:34:10.821 INFO:teuthology.orchestra.run.smithi026.stdout: Installing : fuse-2.9.7-15.el8.x86_64 1/2 2023-09-23T20:34:10.995 INFO:teuthology.orchestra.run.smithi026.stdout: Installing : ceph-fuse-2:17.2.6-1123.g61431f6b.el8.x86_64 2/2 2023-09-23T20:34:11.377 INFO:teuthology.orchestra.run.smithi070.stdout:Package libcephfs2-2:17.2.6-1123.g61431f6b.el8.x86_64 is already installed. 2023-09-23T20:34:11.380 INFO:teuthology.orchestra.run.smithi026.stdout: Running scriptlet: ceph-fuse-2:17.2.6-1123.g61431f6b.el8.x86_64 2/2 2023-09-23T20:34:11.380 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : ceph-fuse-2:17.2.6-1123.g61431f6b.el8.x86_64 1/2 2023-09-23T20:34:11.487 INFO:teuthology.orchestra.run.smithi070.stdout:Dependencies resolved. 2023-09-23T20:34:11.488 INFO:teuthology.orchestra.run.smithi070.stdout:Nothing to do. 2023-09-23T20:34:11.489 INFO:teuthology.orchestra.run.smithi070.stdout:Complete! 2023-09-23T20:34:11.633 DEBUG:teuthology.orchestra.run.smithi070:> sudo yum -y install libcephfs-devel 2023-09-23T20:34:11.986 INFO:teuthology.orchestra.run.smithi070.stdout:Updating Subscription Management repositories. 2023-09-23T20:34:12.471 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : fuse-2.9.7-15.el8.x86_64 2/2 2023-09-23T20:34:12.472 INFO:teuthology.orchestra.run.smithi026.stdout:Last metadata expiration check: 0:02:54 ago on Sat 23 Sep 2023 08:31:18 PM UTC. 2023-09-23T20:34:12.857 INFO:teuthology.orchestra.run.smithi070.stdout:Last metadata expiration check: 0:02:56 ago on Sat 23 Sep 2023 08:31:16 PM UTC. 2023-09-23T20:34:14.237 INFO:teuthology.orchestra.run.smithi070.stdout:Dependencies resolved. 2023-09-23T20:34:14.238 INFO:teuthology.orchestra.run.smithi070.stdout:================================================================================ 2023-09-23T20:34:14.238 INFO:teuthology.orchestra.run.smithi070.stdout: Package Arch Version Repo Size 2023-09-23T20:34:14.238 INFO:teuthology.orchestra.run.smithi070.stdout:================================================================================ 2023-09-23T20:34:14.238 INFO:teuthology.orchestra.run.smithi070.stdout:Installing: 2023-09-23T20:34:14.238 INFO:teuthology.orchestra.run.smithi070.stdout: libcephfs-devel x86_64 2:17.2.6-1123.g61431f6b.el8 ceph 25 k 2023-09-23T20:34:14.238 INFO:teuthology.orchestra.run.smithi070.stdout: 2023-09-23T20:34:14.238 INFO:teuthology.orchestra.run.smithi070.stdout:Transaction Summary 2023-09-23T20:34:14.239 INFO:teuthology.orchestra.run.smithi070.stdout:================================================================================ 2023-09-23T20:34:14.239 INFO:teuthology.orchestra.run.smithi070.stdout:Install 1 Package 2023-09-23T20:34:14.239 INFO:teuthology.orchestra.run.smithi070.stdout: 2023-09-23T20:34:14.239 INFO:teuthology.orchestra.run.smithi070.stdout:Total download size: 25 k 2023-09-23T20:34:14.239 INFO:teuthology.orchestra.run.smithi070.stdout:Installed size: 109 k 2023-09-23T20:34:14.239 INFO:teuthology.orchestra.run.smithi070.stdout:Downloading Packages: 2023-09-23T20:34:14.390 INFO:teuthology.orchestra.run.smithi070.stdout:libcephfs-devel-17.2.6-1123.g61431f6b.el8.x86_6 167 kB/s | 25 kB 00:00 2023-09-23T20:34:14.390 INFO:teuthology.orchestra.run.smithi070.stdout:-------------------------------------------------------------------------------- 2023-09-23T20:34:14.390 INFO:teuthology.orchestra.run.smithi070.stdout:Total 166 kB/s | 25 kB 00:00 2023-09-23T20:34:14.390 INFO:teuthology.orchestra.run.smithi070.stdout:Running transaction check 2023-09-23T20:34:14.392 INFO:teuthology.orchestra.run.smithi070.stdout:Transaction check succeeded. 2023-09-23T20:34:14.392 INFO:teuthology.orchestra.run.smithi070.stdout:Running transaction test 2023-09-23T20:34:14.394 INFO:teuthology.orchestra.run.smithi070.stdout:Transaction test succeeded. 2023-09-23T20:34:14.398 INFO:teuthology.orchestra.run.smithi070.stdout:Running transaction 2023-09-23T20:34:14.448 INFO:teuthology.orchestra.run.smithi070.stdout: Preparing : 1/1 2023-09-23T20:34:14.626 INFO:teuthology.orchestra.run.smithi070.stdout: Installing : libcephfs-devel-2:17.2.6-1123.g61431f6b.el8.x86_64 1/1 2023-09-23T20:34:14.839 INFO:teuthology.orchestra.run.smithi070.stdout: Running scriptlet: libcephfs-devel-2:17.2.6-1123.g61431f6b.el8.x86_64 1/1 2023-09-23T20:34:15.329 INFO:teuthology.orchestra.run.smithi026.stdout:Installed products updated. 2023-09-23T20:34:15.528 INFO:teuthology.orchestra.run.smithi026.stdout: 2023-09-23T20:34:15.529 INFO:teuthology.orchestra.run.smithi026.stdout:Installed: 2023-09-23T20:34:15.529 INFO:teuthology.orchestra.run.smithi026.stdout: ceph-fuse-2:17.2.6-1123.g61431f6b.el8.x86_64 fuse-2.9.7-15.el8.x86_64 2023-09-23T20:34:15.529 INFO:teuthology.orchestra.run.smithi026.stdout: 2023-09-23T20:34:15.529 INFO:teuthology.orchestra.run.smithi026.stdout:Complete! 2023-09-23T20:34:15.649 DEBUG:teuthology.orchestra.run.smithi026:> sudo yum -y install librados-devel 2023-09-23T20:34:15.826 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : libcephfs-devel-2:17.2.6-1123.g61431f6b.el8.x86_64 1/1 2023-09-23T20:34:15.826 INFO:teuthology.orchestra.run.smithi070.stdout:Last metadata expiration check: 0:02:59 ago on Sat 23 Sep 2023 08:31:16 PM UTC. 2023-09-23T20:34:16.031 INFO:teuthology.orchestra.run.smithi026.stdout:Updating Subscription Management repositories. 2023-09-23T20:34:16.915 INFO:teuthology.orchestra.run.smithi026.stdout:Last metadata expiration check: 0:02:58 ago on Sat 23 Sep 2023 08:31:18 PM UTC. 2023-09-23T20:34:18.256 INFO:teuthology.orchestra.run.smithi070.stdout:Installed products updated. 2023-09-23T20:34:18.439 INFO:teuthology.orchestra.run.smithi070.stdout: 2023-09-23T20:34:18.440 INFO:teuthology.orchestra.run.smithi070.stdout:Installed: 2023-09-23T20:34:18.440 INFO:teuthology.orchestra.run.smithi070.stdout: libcephfs-devel-2:17.2.6-1123.g61431f6b.el8.x86_64 2023-09-23T20:34:18.440 INFO:teuthology.orchestra.run.smithi070.stdout: 2023-09-23T20:34:18.440 INFO:teuthology.orchestra.run.smithi070.stdout:Complete! 2023-09-23T20:34:18.499 INFO:teuthology.orchestra.run.smithi026.stdout:Dependencies resolved. 2023-09-23T20:34:18.500 INFO:teuthology.orchestra.run.smithi026.stdout:================================================================================ 2023-09-23T20:34:18.500 INFO:teuthology.orchestra.run.smithi026.stdout: Package Arch Version Repo Size 2023-09-23T20:34:18.500 INFO:teuthology.orchestra.run.smithi026.stdout:================================================================================ 2023-09-23T20:34:18.500 INFO:teuthology.orchestra.run.smithi026.stdout:Installing: 2023-09-23T20:34:18.500 INFO:teuthology.orchestra.run.smithi026.stdout: librados-devel x86_64 2:17.2.6-1123.g61431f6b.el8 ceph 130 k 2023-09-23T20:34:18.500 INFO:teuthology.orchestra.run.smithi026.stdout: 2023-09-23T20:34:18.500 INFO:teuthology.orchestra.run.smithi026.stdout:Transaction Summary 2023-09-23T20:34:18.500 INFO:teuthology.orchestra.run.smithi026.stdout:================================================================================ 2023-09-23T20:34:18.500 INFO:teuthology.orchestra.run.smithi026.stdout:Install 1 Package 2023-09-23T20:34:18.500 INFO:teuthology.orchestra.run.smithi026.stdout: 2023-09-23T20:34:18.501 INFO:teuthology.orchestra.run.smithi026.stdout:Total download size: 130 k 2023-09-23T20:34:18.501 INFO:teuthology.orchestra.run.smithi026.stdout:Installed size: 464 k 2023-09-23T20:34:18.501 INFO:teuthology.orchestra.run.smithi026.stdout:Downloading Packages: 2023-09-23T20:34:18.572 DEBUG:teuthology.orchestra.run.smithi070:> sudo yum -y install librados2 2023-09-23T20:34:18.685 INFO:teuthology.orchestra.run.smithi026.stdout:librados-devel-17.2.6-1123.g61431f6b.el8.x86_64 704 kB/s | 130 kB 00:00 2023-09-23T20:34:18.686 INFO:teuthology.orchestra.run.smithi026.stdout:-------------------------------------------------------------------------------- 2023-09-23T20:34:18.686 INFO:teuthology.orchestra.run.smithi026.stdout:Total 699 kB/s | 130 kB 00:00 2023-09-23T20:34:18.686 INFO:teuthology.orchestra.run.smithi026.stdout:Running transaction check 2023-09-23T20:34:18.693 INFO:teuthology.orchestra.run.smithi026.stdout:Transaction check succeeded. 2023-09-23T20:34:18.693 INFO:teuthology.orchestra.run.smithi026.stdout:Running transaction test 2023-09-23T20:34:18.752 INFO:teuthology.orchestra.run.smithi026.stdout:Transaction test succeeded. 2023-09-23T20:34:18.755 INFO:teuthology.orchestra.run.smithi026.stdout:Running transaction 2023-09-23T20:34:18.883 INFO:teuthology.orchestra.run.smithi026.stdout: Preparing : 1/1 2023-09-23T20:34:18.926 INFO:teuthology.orchestra.run.smithi070.stdout:Updating Subscription Management repositories. 2023-09-23T20:34:19.059 INFO:teuthology.orchestra.run.smithi026.stdout: Installing : librados-devel-2:17.2.6-1123.g61431f6b.el8.x86_64 1/1 2023-09-23T20:34:19.290 INFO:teuthology.orchestra.run.smithi026.stdout: Running scriptlet: librados-devel-2:17.2.6-1123.g61431f6b.el8.x86_64 1/1 2023-09-23T20:34:19.745 INFO:teuthology.orchestra.run.smithi070.stdout:Last metadata expiration check: 0:03:03 ago on Sat 23 Sep 2023 08:31:16 PM UTC. 2023-09-23T20:34:20.389 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : librados-devel-2:17.2.6-1123.g61431f6b.el8.x86_64 1/1 2023-09-23T20:34:20.390 INFO:teuthology.orchestra.run.smithi026.stdout:Last metadata expiration check: 0:03:02 ago on Sat 23 Sep 2023 08:31:18 PM UTC. 2023-09-23T20:34:21.007 INFO:teuthology.orchestra.run.smithi070.stdout:Package librados2-2:17.2.6-1123.g61431f6b.el8.x86_64 is already installed. 2023-09-23T20:34:21.116 INFO:teuthology.orchestra.run.smithi070.stdout:Dependencies resolved. 2023-09-23T20:34:21.117 INFO:teuthology.orchestra.run.smithi070.stdout:Nothing to do. 2023-09-23T20:34:21.117 INFO:teuthology.orchestra.run.smithi070.stdout:Complete! 2023-09-23T20:34:21.236 DEBUG:teuthology.orchestra.run.smithi070:> sudo yum -y install librbd1 2023-09-23T20:34:21.586 INFO:teuthology.orchestra.run.smithi070.stdout:Updating Subscription Management repositories. 2023-09-23T20:34:22.454 INFO:teuthology.orchestra.run.smithi070.stdout:Last metadata expiration check: 0:03:06 ago on Sat 23 Sep 2023 08:31:16 PM UTC. 2023-09-23T20:34:23.137 INFO:teuthology.orchestra.run.smithi026.stdout:Installed products updated. 2023-09-23T20:34:23.342 INFO:teuthology.orchestra.run.smithi026.stdout: 2023-09-23T20:34:23.342 INFO:teuthology.orchestra.run.smithi026.stdout:Installed: 2023-09-23T20:34:23.342 INFO:teuthology.orchestra.run.smithi026.stdout: librados-devel-2:17.2.6-1123.g61431f6b.el8.x86_64 2023-09-23T20:34:23.342 INFO:teuthology.orchestra.run.smithi026.stdout: 2023-09-23T20:34:23.342 INFO:teuthology.orchestra.run.smithi026.stdout:Complete! 2023-09-23T20:34:23.469 DEBUG:teuthology.orchestra.run.smithi026:> sudo yum -y install libcephfs2 2023-09-23T20:34:23.726 INFO:teuthology.orchestra.run.smithi070.stdout:Package librbd1-2:17.2.6-1123.g61431f6b.el8.x86_64 is already installed. 2023-09-23T20:34:23.835 INFO:teuthology.orchestra.run.smithi070.stdout:Dependencies resolved. 2023-09-23T20:34:23.837 INFO:teuthology.orchestra.run.smithi070.stdout:Nothing to do. 2023-09-23T20:34:23.837 INFO:teuthology.orchestra.run.smithi070.stdout:Complete! 2023-09-23T20:34:23.866 INFO:teuthology.orchestra.run.smithi026.stdout:Updating Subscription Management repositories. 2023-09-23T20:34:23.954 DEBUG:teuthology.orchestra.run.smithi070:> sudo yum -y install python3-rados 2023-09-23T20:34:24.307 INFO:teuthology.orchestra.run.smithi070.stdout:Updating Subscription Management repositories. 2023-09-23T20:34:24.809 INFO:teuthology.orchestra.run.smithi026.stdout:Last metadata expiration check: 0:03:06 ago on Sat 23 Sep 2023 08:31:18 PM UTC. 2023-09-23T20:34:25.175 INFO:teuthology.orchestra.run.smithi070.stdout:Last metadata expiration check: 0:03:09 ago on Sat 23 Sep 2023 08:31:16 PM UTC. 2023-09-23T20:34:26.247 INFO:teuthology.orchestra.run.smithi026.stdout:Package libcephfs2-2:17.2.6-1123.g61431f6b.el8.x86_64 is already installed. 2023-09-23T20:34:26.370 INFO:teuthology.orchestra.run.smithi026.stdout:Dependencies resolved. 2023-09-23T20:34:26.371 INFO:teuthology.orchestra.run.smithi026.stdout:Nothing to do. 2023-09-23T20:34:26.371 INFO:teuthology.orchestra.run.smithi026.stdout:Complete! 2023-09-23T20:34:26.438 INFO:teuthology.orchestra.run.smithi070.stdout:Package python3-rados-2:17.2.6-1123.g61431f6b.el8.x86_64 is already installed. 2023-09-23T20:34:26.499 DEBUG:teuthology.orchestra.run.smithi026:> sudo yum -y install libcephfs-devel 2023-09-23T20:34:26.547 INFO:teuthology.orchestra.run.smithi070.stdout:Dependencies resolved. 2023-09-23T20:34:26.548 INFO:teuthology.orchestra.run.smithi070.stdout:Nothing to do. 2023-09-23T20:34:26.548 INFO:teuthology.orchestra.run.smithi070.stdout:Complete! 2023-09-23T20:34:26.668 DEBUG:teuthology.orchestra.run.smithi070:> sudo yum -y install python3-rgw 2023-09-23T20:34:26.875 INFO:teuthology.orchestra.run.smithi026.stdout:Updating Subscription Management repositories. 2023-09-23T20:34:27.021 INFO:teuthology.orchestra.run.smithi070.stdout:Updating Subscription Management repositories. 2023-09-23T20:34:27.758 INFO:teuthology.orchestra.run.smithi026.stdout:Last metadata expiration check: 0:03:09 ago on Sat 23 Sep 2023 08:31:18 PM UTC. 2023-09-23T20:34:27.882 INFO:teuthology.orchestra.run.smithi070.stdout:Last metadata expiration check: 0:03:11 ago on Sat 23 Sep 2023 08:31:16 PM UTC. 2023-09-23T20:34:29.146 INFO:teuthology.orchestra.run.smithi070.stdout:Package python3-rgw-2:17.2.6-1123.g61431f6b.el8.x86_64 is already installed. 2023-09-23T20:34:29.254 INFO:teuthology.orchestra.run.smithi070.stdout:Dependencies resolved. 2023-09-23T20:34:29.255 INFO:teuthology.orchestra.run.smithi070.stdout:Nothing to do. 2023-09-23T20:34:29.255 INFO:teuthology.orchestra.run.smithi070.stdout:Complete! 2023-09-23T20:34:29.310 INFO:teuthology.orchestra.run.smithi026.stdout:Dependencies resolved. 2023-09-23T20:34:29.311 INFO:teuthology.orchestra.run.smithi026.stdout:================================================================================ 2023-09-23T20:34:29.311 INFO:teuthology.orchestra.run.smithi026.stdout: Package Arch Version Repo Size 2023-09-23T20:34:29.311 INFO:teuthology.orchestra.run.smithi026.stdout:================================================================================ 2023-09-23T20:34:29.311 INFO:teuthology.orchestra.run.smithi026.stdout:Installing: 2023-09-23T20:34:29.311 INFO:teuthology.orchestra.run.smithi026.stdout: libcephfs-devel x86_64 2:17.2.6-1123.g61431f6b.el8 ceph 25 k 2023-09-23T20:34:29.311 INFO:teuthology.orchestra.run.smithi026.stdout: 2023-09-23T20:34:29.311 INFO:teuthology.orchestra.run.smithi026.stdout:Transaction Summary 2023-09-23T20:34:29.311 INFO:teuthology.orchestra.run.smithi026.stdout:================================================================================ 2023-09-23T20:34:29.311 INFO:teuthology.orchestra.run.smithi026.stdout:Install 1 Package 2023-09-23T20:34:29.311 INFO:teuthology.orchestra.run.smithi026.stdout: 2023-09-23T20:34:29.312 INFO:teuthology.orchestra.run.smithi026.stdout:Total download size: 25 k 2023-09-23T20:34:29.312 INFO:teuthology.orchestra.run.smithi026.stdout:Installed size: 109 k 2023-09-23T20:34:29.312 INFO:teuthology.orchestra.run.smithi026.stdout:Downloading Packages: 2023-09-23T20:34:29.360 DEBUG:teuthology.orchestra.run.smithi070:> sudo yum -y install python3-cephfs 2023-09-23T20:34:29.483 INFO:teuthology.orchestra.run.smithi026.stdout:libcephfs-devel-17.2.6-1123.g61431f6b.el8.x86_6 145 kB/s | 25 kB 00:00 2023-09-23T20:34:29.483 INFO:teuthology.orchestra.run.smithi026.stdout:-------------------------------------------------------------------------------- 2023-09-23T20:34:29.484 INFO:teuthology.orchestra.run.smithi026.stdout:Total 144 kB/s | 25 kB 00:00 2023-09-23T20:34:29.484 INFO:teuthology.orchestra.run.smithi026.stdout:Running transaction check 2023-09-23T20:34:29.488 INFO:teuthology.orchestra.run.smithi026.stdout:Transaction check succeeded. 2023-09-23T20:34:29.488 INFO:teuthology.orchestra.run.smithi026.stdout:Running transaction test 2023-09-23T20:34:29.490 INFO:teuthology.orchestra.run.smithi026.stdout:Transaction test succeeded. 2023-09-23T20:34:29.494 INFO:teuthology.orchestra.run.smithi026.stdout:Running transaction 2023-09-23T20:34:29.543 INFO:teuthology.orchestra.run.smithi026.stdout: Preparing : 1/1 2023-09-23T20:34:29.699 INFO:teuthology.orchestra.run.smithi026.stdout: Installing : libcephfs-devel-2:17.2.6-1123.g61431f6b.el8.x86_64 1/1 2023-09-23T20:34:29.713 INFO:teuthology.orchestra.run.smithi070.stdout:Updating Subscription Management repositories. 2023-09-23T20:34:29.913 INFO:teuthology.orchestra.run.smithi026.stdout: Running scriptlet: libcephfs-devel-2:17.2.6-1123.g61431f6b.el8.x86_64 1/1 2023-09-23T20:34:30.534 INFO:teuthology.orchestra.run.smithi070.stdout:Last metadata expiration check: 0:03:14 ago on Sat 23 Sep 2023 08:31:16 PM UTC. 2023-09-23T20:34:30.935 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : libcephfs-devel-2:17.2.6-1123.g61431f6b.el8.x86_64 1/1 2023-09-23T20:34:30.935 INFO:teuthology.orchestra.run.smithi026.stdout:Last metadata expiration check: 0:03:12 ago on Sat 23 Sep 2023 08:31:18 PM UTC. 2023-09-23T20:34:31.814 INFO:teuthology.orchestra.run.smithi070.stdout:Package python3-cephfs-2:17.2.6-1123.g61431f6b.el8.x86_64 is already installed. 2023-09-23T20:34:31.920 INFO:teuthology.orchestra.run.smithi070.stdout:Dependencies resolved. 2023-09-23T20:34:31.921 INFO:teuthology.orchestra.run.smithi070.stdout:Nothing to do. 2023-09-23T20:34:31.921 INFO:teuthology.orchestra.run.smithi070.stdout:Complete! 2023-09-23T20:34:32.033 DEBUG:teuthology.orchestra.run.smithi070:> sudo yum -y install python3-rbd 2023-09-23T20:34:32.385 INFO:teuthology.orchestra.run.smithi070.stdout:Updating Subscription Management repositories. 2023-09-23T20:34:33.268 INFO:teuthology.orchestra.run.smithi070.stdout:Last metadata expiration check: 0:03:17 ago on Sat 23 Sep 2023 08:31:16 PM UTC. 2023-09-23T20:34:33.456 INFO:teuthology.orchestra.run.smithi026.stdout:Installed products updated. 2023-09-23T20:34:33.657 INFO:teuthology.orchestra.run.smithi026.stdout: 2023-09-23T20:34:33.657 INFO:teuthology.orchestra.run.smithi026.stdout:Installed: 2023-09-23T20:34:33.657 INFO:teuthology.orchestra.run.smithi026.stdout: libcephfs-devel-2:17.2.6-1123.g61431f6b.el8.x86_64 2023-09-23T20:34:33.657 INFO:teuthology.orchestra.run.smithi026.stdout: 2023-09-23T20:34:33.658 INFO:teuthology.orchestra.run.smithi026.stdout:Complete! 2023-09-23T20:34:33.781 DEBUG:teuthology.orchestra.run.smithi026:> sudo yum -y install librados2 2023-09-23T20:34:34.158 INFO:teuthology.orchestra.run.smithi026.stdout:Updating Subscription Management repositories. 2023-09-23T20:34:34.534 INFO:teuthology.orchestra.run.smithi070.stdout:Package python3-rbd-2:17.2.6-1123.g61431f6b.el8.x86_64 is already installed. 2023-09-23T20:34:34.643 INFO:teuthology.orchestra.run.smithi070.stdout:Dependencies resolved. 2023-09-23T20:34:34.644 INFO:teuthology.orchestra.run.smithi070.stdout:Nothing to do. 2023-09-23T20:34:34.644 INFO:teuthology.orchestra.run.smithi070.stdout:Complete! 2023-09-23T20:34:34.750 DEBUG:teuthology.orchestra.run.smithi070:> sudo yum -y install rbd-fuse 2023-09-23T20:34:35.003 INFO:teuthology.orchestra.run.smithi026.stdout:Last metadata expiration check: 0:03:17 ago on Sat 23 Sep 2023 08:31:18 PM UTC. 2023-09-23T20:34:35.103 INFO:teuthology.orchestra.run.smithi070.stdout:Updating Subscription Management repositories. 2023-09-23T20:34:35.887 INFO:teuthology.orchestra.run.smithi070.stdout:Last metadata expiration check: 0:03:19 ago on Sat 23 Sep 2023 08:31:16 PM UTC. 2023-09-23T20:34:36.430 INFO:teuthology.orchestra.run.smithi026.stdout:Package librados2-2:17.2.6-1123.g61431f6b.el8.x86_64 is already installed. 2023-09-23T20:34:36.548 INFO:teuthology.orchestra.run.smithi026.stdout:Dependencies resolved. 2023-09-23T20:34:36.549 INFO:teuthology.orchestra.run.smithi026.stdout:Nothing to do. 2023-09-23T20:34:36.549 INFO:teuthology.orchestra.run.smithi026.stdout:Complete! 2023-09-23T20:34:36.663 DEBUG:teuthology.orchestra.run.smithi026:> sudo yum -y install librbd1 2023-09-23T20:34:37.042 INFO:teuthology.orchestra.run.smithi026.stdout:Updating Subscription Management repositories. 2023-09-23T20:34:37.341 INFO:teuthology.orchestra.run.smithi070.stdout:Dependencies resolved. 2023-09-23T20:34:37.341 INFO:teuthology.orchestra.run.smithi070.stdout:================================================================================ 2023-09-23T20:34:37.341 INFO:teuthology.orchestra.run.smithi070.stdout: Package Architecture Version Repository Size 2023-09-23T20:34:37.341 INFO:teuthology.orchestra.run.smithi070.stdout:================================================================================ 2023-09-23T20:34:37.342 INFO:teuthology.orchestra.run.smithi070.stdout:Installing: 2023-09-23T20:34:37.342 INFO:teuthology.orchestra.run.smithi070.stdout: rbd-fuse x86_64 2:17.2.6-1123.g61431f6b.el8 ceph 90 k 2023-09-23T20:34:37.342 INFO:teuthology.orchestra.run.smithi070.stdout: 2023-09-23T20:34:37.342 INFO:teuthology.orchestra.run.smithi070.stdout:Transaction Summary 2023-09-23T20:34:37.342 INFO:teuthology.orchestra.run.smithi070.stdout:================================================================================ 2023-09-23T20:34:37.342 INFO:teuthology.orchestra.run.smithi070.stdout:Install 1 Package 2023-09-23T20:34:37.342 INFO:teuthology.orchestra.run.smithi070.stdout: 2023-09-23T20:34:37.343 INFO:teuthology.orchestra.run.smithi070.stdout:Total download size: 90 k 2023-09-23T20:34:37.343 INFO:teuthology.orchestra.run.smithi070.stdout:Installed size: 264 k 2023-09-23T20:34:37.343 INFO:teuthology.orchestra.run.smithi070.stdout:Downloading Packages: 2023-09-23T20:34:37.598 INFO:teuthology.orchestra.run.smithi070.stdout:rbd-fuse-17.2.6-1123.g61431f6b.el8.x86_64.rpm 354 kB/s | 90 kB 00:00 2023-09-23T20:34:37.598 INFO:teuthology.orchestra.run.smithi070.stdout:-------------------------------------------------------------------------------- 2023-09-23T20:34:37.598 INFO:teuthology.orchestra.run.smithi070.stdout:Total 352 kB/s | 90 kB 00:00 2023-09-23T20:34:37.599 INFO:teuthology.orchestra.run.smithi070.stdout:Running transaction check 2023-09-23T20:34:37.609 INFO:teuthology.orchestra.run.smithi070.stdout:Transaction check succeeded. 2023-09-23T20:34:37.609 INFO:teuthology.orchestra.run.smithi070.stdout:Running transaction test 2023-09-23T20:34:37.662 INFO:teuthology.orchestra.run.smithi070.stdout:Transaction test succeeded. 2023-09-23T20:34:37.665 INFO:teuthology.orchestra.run.smithi070.stdout:Running transaction 2023-09-23T20:34:37.775 INFO:teuthology.orchestra.run.smithi070.stdout: Preparing : 1/1 2023-09-23T20:34:37.888 INFO:teuthology.orchestra.run.smithi026.stdout:Last metadata expiration check: 0:03:19 ago on Sat 23 Sep 2023 08:31:18 PM UTC. 2023-09-23T20:34:38.179 INFO:teuthology.orchestra.run.smithi070.stdout: Installing : rbd-fuse-2:17.2.6-1123.g61431f6b.el8.x86_64 1/1 2023-09-23T20:34:38.410 INFO:teuthology.orchestra.run.smithi070.stdout: Running scriptlet: rbd-fuse-2:17.2.6-1123.g61431f6b.el8.x86_64 1/1 2023-09-23T20:34:39.315 INFO:teuthology.orchestra.run.smithi026.stdout:Package librbd1-2:17.2.6-1123.g61431f6b.el8.x86_64 is already installed. 2023-09-23T20:34:39.434 INFO:teuthology.orchestra.run.smithi026.stdout:Dependencies resolved. 2023-09-23T20:34:39.436 INFO:teuthology.orchestra.run.smithi026.stdout:Nothing to do. 2023-09-23T20:34:39.436 INFO:teuthology.orchestra.run.smithi026.stdout:Complete! 2023-09-23T20:34:39.443 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : rbd-fuse-2:17.2.6-1123.g61431f6b.el8.x86_64 1/1 2023-09-23T20:34:39.444 INFO:teuthology.orchestra.run.smithi070.stdout:Last metadata expiration check: 0:03:23 ago on Sat 23 Sep 2023 08:31:16 PM UTC. 2023-09-23T20:34:39.545 DEBUG:teuthology.orchestra.run.smithi026:> sudo yum -y install python3-rados 2023-09-23T20:34:39.921 INFO:teuthology.orchestra.run.smithi026.stdout:Updating Subscription Management repositories. 2023-09-23T20:34:40.786 INFO:teuthology.orchestra.run.smithi026.stdout:Last metadata expiration check: 0:03:22 ago on Sat 23 Sep 2023 08:31:18 PM UTC. 2023-09-23T20:34:42.073 INFO:teuthology.orchestra.run.smithi070.stdout:Installed products updated. 2023-09-23T20:34:42.239 INFO:teuthology.orchestra.run.smithi026.stdout:Package python3-rados-2:17.2.6-1123.g61431f6b.el8.x86_64 is already installed. 2023-09-23T20:34:42.257 INFO:teuthology.orchestra.run.smithi070.stdout: 2023-09-23T20:34:42.257 INFO:teuthology.orchestra.run.smithi070.stdout:Installed: 2023-09-23T20:34:42.257 INFO:teuthology.orchestra.run.smithi070.stdout: rbd-fuse-2:17.2.6-1123.g61431f6b.el8.x86_64 2023-09-23T20:34:42.257 INFO:teuthology.orchestra.run.smithi070.stdout: 2023-09-23T20:34:42.257 INFO:teuthology.orchestra.run.smithi070.stdout:Complete! 2023-09-23T20:34:42.360 INFO:teuthology.orchestra.run.smithi026.stdout:Dependencies resolved. 2023-09-23T20:34:42.361 INFO:teuthology.orchestra.run.smithi026.stdout:Nothing to do. 2023-09-23T20:34:42.361 INFO:teuthology.orchestra.run.smithi026.stdout:Complete! 2023-09-23T20:34:42.393 DEBUG:teuthology.orchestra.run.smithi070:> sudo yum -y install rbd-mirror 2023-09-23T20:34:42.490 DEBUG:teuthology.orchestra.run.smithi026:> sudo yum -y install python3-rgw 2023-09-23T20:34:42.754 INFO:teuthology.orchestra.run.smithi070.stdout:Updating Subscription Management repositories. 2023-09-23T20:34:42.872 INFO:teuthology.orchestra.run.smithi026.stdout:Updating Subscription Management repositories. 2023-09-23T20:34:43.579 INFO:teuthology.orchestra.run.smithi070.stdout:Last metadata expiration check: 0:03:27 ago on Sat 23 Sep 2023 08:31:16 PM UTC. 2023-09-23T20:34:43.720 INFO:teuthology.orchestra.run.smithi026.stdout:Last metadata expiration check: 0:03:25 ago on Sat 23 Sep 2023 08:31:18 PM UTC. 2023-09-23T20:34:44.981 INFO:teuthology.orchestra.run.smithi070.stdout:Dependencies resolved. 2023-09-23T20:34:44.981 INFO:teuthology.orchestra.run.smithi070.stdout:================================================================================ 2023-09-23T20:34:44.982 INFO:teuthology.orchestra.run.smithi070.stdout: Package Arch Version Repo Size 2023-09-23T20:34:44.982 INFO:teuthology.orchestra.run.smithi070.stdout:================================================================================ 2023-09-23T20:34:44.982 INFO:teuthology.orchestra.run.smithi070.stdout:Installing: 2023-09-23T20:34:44.982 INFO:teuthology.orchestra.run.smithi070.stdout: rbd-mirror x86_64 2:17.2.6-1123.g61431f6b.el8 ceph 3.5 M 2023-09-23T20:34:44.982 INFO:teuthology.orchestra.run.smithi070.stdout: 2023-09-23T20:34:44.982 INFO:teuthology.orchestra.run.smithi070.stdout:Transaction Summary 2023-09-23T20:34:44.982 INFO:teuthology.orchestra.run.smithi070.stdout:================================================================================ 2023-09-23T20:34:44.982 INFO:teuthology.orchestra.run.smithi070.stdout:Install 1 Package 2023-09-23T20:34:44.982 INFO:teuthology.orchestra.run.smithi070.stdout: 2023-09-23T20:34:44.983 INFO:teuthology.orchestra.run.smithi070.stdout:Total download size: 3.5 M 2023-09-23T20:34:44.983 INFO:teuthology.orchestra.run.smithi070.stdout:Installed size: 16 M 2023-09-23T20:34:44.983 INFO:teuthology.orchestra.run.smithi070.stdout:Downloading Packages: 2023-09-23T20:34:45.154 INFO:teuthology.orchestra.run.smithi026.stdout:Package python3-rgw-2:17.2.6-1123.g61431f6b.el8.x86_64 is already installed. 2023-09-23T20:34:45.277 INFO:teuthology.orchestra.run.smithi026.stdout:Dependencies resolved. 2023-09-23T20:34:45.278 INFO:teuthology.orchestra.run.smithi026.stdout:Nothing to do. 2023-09-23T20:34:45.278 INFO:teuthology.orchestra.run.smithi026.stdout:Complete! 2023-09-23T20:34:45.383 DEBUG:teuthology.orchestra.run.smithi026:> sudo yum -y install python3-cephfs 2023-09-23T20:34:45.583 INFO:teuthology.orchestra.run.smithi070.stdout:rbd-mirror-17.2.6-1123.g61431f6b.el8.x86_64.rpm 5.8 MB/s | 3.5 MB 00:00 2023-09-23T20:34:45.584 INFO:teuthology.orchestra.run.smithi070.stdout:-------------------------------------------------------------------------------- 2023-09-23T20:34:45.584 INFO:teuthology.orchestra.run.smithi070.stdout:Total 5.7 MB/s | 3.5 MB 00:00 2023-09-23T20:34:45.585 INFO:teuthology.orchestra.run.smithi070.stdout:Running transaction check 2023-09-23T20:34:45.596 INFO:teuthology.orchestra.run.smithi070.stdout:Transaction check succeeded. 2023-09-23T20:34:45.596 INFO:teuthology.orchestra.run.smithi070.stdout:Running transaction test 2023-09-23T20:34:45.663 INFO:teuthology.orchestra.run.smithi070.stdout:Transaction test succeeded. 2023-09-23T20:34:45.667 INFO:teuthology.orchestra.run.smithi070.stdout:Running transaction 2023-09-23T20:34:45.760 INFO:teuthology.orchestra.run.smithi026.stdout:Updating Subscription Management repositories. 2023-09-23T20:34:46.104 INFO:teuthology.orchestra.run.smithi070.stdout: Preparing : 1/1 2023-09-23T20:34:46.289 INFO:teuthology.orchestra.run.smithi070.stdout: Installing : rbd-mirror-2:17.2.6-1123.g61431f6b.el8.x86_64 1/1 2023-09-23T20:34:46.615 INFO:teuthology.orchestra.run.smithi026.stdout:Last metadata expiration check: 0:03:28 ago on Sat 23 Sep 2023 08:31:18 PM UTC. 2023-09-23T20:34:46.659 INFO:teuthology.orchestra.run.smithi070.stdout: Running scriptlet: rbd-mirror-2:17.2.6-1123.g61431f6b.el8.x86_64 1/1 2023-09-23T20:34:47.719 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : rbd-mirror-2:17.2.6-1123.g61431f6b.el8.x86_64 1/1 2023-09-23T20:34:47.719 INFO:teuthology.orchestra.run.smithi070.stdout:Last metadata expiration check: 0:03:31 ago on Sat 23 Sep 2023 08:31:16 PM UTC. 2023-09-23T20:34:48.038 INFO:teuthology.orchestra.run.smithi026.stdout:Package python3-cephfs-2:17.2.6-1123.g61431f6b.el8.x86_64 is already installed. 2023-09-23T20:34:48.157 INFO:teuthology.orchestra.run.smithi026.stdout:Dependencies resolved. 2023-09-23T20:34:48.158 INFO:teuthology.orchestra.run.smithi026.stdout:Nothing to do. 2023-09-23T20:34:48.158 INFO:teuthology.orchestra.run.smithi026.stdout:Complete! 2023-09-23T20:34:48.271 DEBUG:teuthology.orchestra.run.smithi026:> sudo yum -y install python3-rbd 2023-09-23T20:34:48.647 INFO:teuthology.orchestra.run.smithi026.stdout:Updating Subscription Management repositories. 2023-09-23T20:34:49.500 INFO:teuthology.orchestra.run.smithi026.stdout:Last metadata expiration check: 0:03:31 ago on Sat 23 Sep 2023 08:31:18 PM UTC. 2023-09-23T20:34:50.070 INFO:teuthology.orchestra.run.smithi070.stdout:Installed products updated. 2023-09-23T20:34:50.255 INFO:teuthology.orchestra.run.smithi070.stdout: 2023-09-23T20:34:50.255 INFO:teuthology.orchestra.run.smithi070.stdout:Installed: 2023-09-23T20:34:50.255 INFO:teuthology.orchestra.run.smithi070.stdout: rbd-mirror-2:17.2.6-1123.g61431f6b.el8.x86_64 2023-09-23T20:34:50.255 INFO:teuthology.orchestra.run.smithi070.stdout: 2023-09-23T20:34:50.255 INFO:teuthology.orchestra.run.smithi070.stdout:Complete! 2023-09-23T20:34:50.395 DEBUG:teuthology.orchestra.run.smithi070:> sudo yum -y install rbd-nbd 2023-09-23T20:34:50.749 INFO:teuthology.orchestra.run.smithi070.stdout:Updating Subscription Management repositories. 2023-09-23T20:34:50.914 INFO:teuthology.orchestra.run.smithi026.stdout:Package python3-rbd-2:17.2.6-1123.g61431f6b.el8.x86_64 is already installed. 2023-09-23T20:34:51.034 INFO:teuthology.orchestra.run.smithi026.stdout:Dependencies resolved. 2023-09-23T20:34:51.035 INFO:teuthology.orchestra.run.smithi026.stdout:Nothing to do. 2023-09-23T20:34:51.035 INFO:teuthology.orchestra.run.smithi026.stdout:Complete! 2023-09-23T20:34:51.140 DEBUG:teuthology.orchestra.run.smithi026:> sudo yum -y install rbd-fuse 2023-09-23T20:34:51.519 INFO:teuthology.orchestra.run.smithi026.stdout:Updating Subscription Management repositories. 2023-09-23T20:34:51.545 INFO:teuthology.orchestra.run.smithi070.stdout:Last metadata expiration check: 0:03:35 ago on Sat 23 Sep 2023 08:31:16 PM UTC. 2023-09-23T20:34:52.422 INFO:teuthology.orchestra.run.smithi026.stdout:Last metadata expiration check: 0:03:34 ago on Sat 23 Sep 2023 08:31:18 PM UTC. 2023-09-23T20:34:52.918 INFO:teuthology.orchestra.run.smithi070.stdout:Dependencies resolved. 2023-09-23T20:34:52.919 INFO:teuthology.orchestra.run.smithi070.stdout:================================================================================ 2023-09-23T20:34:52.919 INFO:teuthology.orchestra.run.smithi070.stdout: Package Architecture Version Repository Size 2023-09-23T20:34:52.919 INFO:teuthology.orchestra.run.smithi070.stdout:================================================================================ 2023-09-23T20:34:52.919 INFO:teuthology.orchestra.run.smithi070.stdout:Installing: 2023-09-23T20:34:52.919 INFO:teuthology.orchestra.run.smithi070.stdout: rbd-nbd x86_64 2:17.2.6-1123.g61431f6b.el8 ceph 173 k 2023-09-23T20:34:52.919 INFO:teuthology.orchestra.run.smithi070.stdout: 2023-09-23T20:34:52.919 INFO:teuthology.orchestra.run.smithi070.stdout:Transaction Summary 2023-09-23T20:34:52.920 INFO:teuthology.orchestra.run.smithi070.stdout:================================================================================ 2023-09-23T20:34:52.920 INFO:teuthology.orchestra.run.smithi070.stdout:Install 1 Package 2023-09-23T20:34:52.920 INFO:teuthology.orchestra.run.smithi070.stdout: 2023-09-23T20:34:52.920 INFO:teuthology.orchestra.run.smithi070.stdout:Total download size: 173 k 2023-09-23T20:34:52.920 INFO:teuthology.orchestra.run.smithi070.stdout:Installed size: 552 k 2023-09-23T20:34:52.920 INFO:teuthology.orchestra.run.smithi070.stdout:Downloading Packages: 2023-09-23T20:34:53.228 INFO:teuthology.orchestra.run.smithi070.stdout:rbd-nbd-17.2.6-1123.g61431f6b.el8.x86_64.rpm 563 kB/s | 173 kB 00:00 2023-09-23T20:34:53.228 INFO:teuthology.orchestra.run.smithi070.stdout:-------------------------------------------------------------------------------- 2023-09-23T20:34:53.228 INFO:teuthology.orchestra.run.smithi070.stdout:Total 561 kB/s | 173 kB 00:00 2023-09-23T20:34:53.229 INFO:teuthology.orchestra.run.smithi070.stdout:Running transaction check 2023-09-23T20:34:53.239 INFO:teuthology.orchestra.run.smithi070.stdout:Transaction check succeeded. 2023-09-23T20:34:53.239 INFO:teuthology.orchestra.run.smithi070.stdout:Running transaction test 2023-09-23T20:34:53.295 INFO:teuthology.orchestra.run.smithi070.stdout:Transaction test succeeded. 2023-09-23T20:34:53.298 INFO:teuthology.orchestra.run.smithi070.stdout:Running transaction 2023-09-23T20:34:53.414 INFO:teuthology.orchestra.run.smithi070.stdout: Preparing : 1/1 2023-09-23T20:34:53.576 INFO:teuthology.orchestra.run.smithi070.stdout: Installing : rbd-nbd-2:17.2.6-1123.g61431f6b.el8.x86_64 1/1 2023-09-23T20:34:53.789 INFO:teuthology.orchestra.run.smithi070.stdout: Running scriptlet: rbd-nbd-2:17.2.6-1123.g61431f6b.el8.x86_64 1/1 2023-09-23T20:34:53.951 INFO:teuthology.orchestra.run.smithi026.stdout:Dependencies resolved. 2023-09-23T20:34:53.952 INFO:teuthology.orchestra.run.smithi026.stdout:================================================================================ 2023-09-23T20:34:53.952 INFO:teuthology.orchestra.run.smithi026.stdout: Package Architecture Version Repository Size 2023-09-23T20:34:53.952 INFO:teuthology.orchestra.run.smithi026.stdout:================================================================================ 2023-09-23T20:34:53.952 INFO:teuthology.orchestra.run.smithi026.stdout:Installing: 2023-09-23T20:34:53.952 INFO:teuthology.orchestra.run.smithi026.stdout: rbd-fuse x86_64 2:17.2.6-1123.g61431f6b.el8 ceph 90 k 2023-09-23T20:34:53.952 INFO:teuthology.orchestra.run.smithi026.stdout: 2023-09-23T20:34:53.952 INFO:teuthology.orchestra.run.smithi026.stdout:Transaction Summary 2023-09-23T20:34:53.952 INFO:teuthology.orchestra.run.smithi026.stdout:================================================================================ 2023-09-23T20:34:53.952 INFO:teuthology.orchestra.run.smithi026.stdout:Install 1 Package 2023-09-23T20:34:53.952 INFO:teuthology.orchestra.run.smithi026.stdout: 2023-09-23T20:34:53.953 INFO:teuthology.orchestra.run.smithi026.stdout:Total download size: 90 k 2023-09-23T20:34:53.953 INFO:teuthology.orchestra.run.smithi026.stdout:Installed size: 264 k 2023-09-23T20:34:53.953 INFO:teuthology.orchestra.run.smithi026.stdout:Downloading Packages: 2023-09-23T20:34:54.128 INFO:teuthology.orchestra.run.smithi026.stdout:rbd-fuse-17.2.6-1123.g61431f6b.el8.x86_64.rpm 516 kB/s | 90 kB 00:00 2023-09-23T20:34:54.128 INFO:teuthology.orchestra.run.smithi026.stdout:-------------------------------------------------------------------------------- 2023-09-23T20:34:54.128 INFO:teuthology.orchestra.run.smithi026.stdout:Total 513 kB/s | 90 kB 00:00 2023-09-23T20:34:54.129 INFO:teuthology.orchestra.run.smithi026.stdout:Running transaction check 2023-09-23T20:34:54.139 INFO:teuthology.orchestra.run.smithi026.stdout:Transaction check succeeded. 2023-09-23T20:34:54.139 INFO:teuthology.orchestra.run.smithi026.stdout:Running transaction test 2023-09-23T20:34:54.198 INFO:teuthology.orchestra.run.smithi026.stdout:Transaction test succeeded. 2023-09-23T20:34:54.202 INFO:teuthology.orchestra.run.smithi026.stdout:Running transaction 2023-09-23T20:34:54.318 INFO:teuthology.orchestra.run.smithi026.stdout: Preparing : 1/1 2023-09-23T20:34:54.453 INFO:teuthology.orchestra.run.smithi026.stdout: Installing : rbd-fuse-2:17.2.6-1123.g61431f6b.el8.x86_64 1/1 2023-09-23T20:34:54.681 INFO:teuthology.orchestra.run.smithi026.stdout: Running scriptlet: rbd-fuse-2:17.2.6-1123.g61431f6b.el8.x86_64 1/1 2023-09-23T20:34:54.842 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : rbd-nbd-2:17.2.6-1123.g61431f6b.el8.x86_64 1/1 2023-09-23T20:34:54.843 INFO:teuthology.orchestra.run.smithi070.stdout:Last metadata expiration check: 0:03:38 ago on Sat 23 Sep 2023 08:31:16 PM UTC. 2023-09-23T20:34:55.768 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : rbd-fuse-2:17.2.6-1123.g61431f6b.el8.x86_64 1/1 2023-09-23T20:34:55.768 INFO:teuthology.orchestra.run.smithi026.stdout:Last metadata expiration check: 0:03:37 ago on Sat 23 Sep 2023 08:31:18 PM UTC. 2023-09-23T20:34:57.275 INFO:teuthology.orchestra.run.smithi070.stdout:Installed products updated. 2023-09-23T20:34:57.457 INFO:teuthology.orchestra.run.smithi070.stdout: 2023-09-23T20:34:57.457 INFO:teuthology.orchestra.run.smithi070.stdout:Installed: 2023-09-23T20:34:57.457 INFO:teuthology.orchestra.run.smithi070.stdout: rbd-nbd-2:17.2.6-1123.g61431f6b.el8.x86_64 2023-09-23T20:34:57.457 INFO:teuthology.orchestra.run.smithi070.stdout: 2023-09-23T20:34:57.457 INFO:teuthology.orchestra.run.smithi070.stdout:Complete! 2023-09-23T20:34:57.584 DEBUG:teuthology.parallel:result is None 2023-09-23T20:34:58.304 INFO:teuthology.orchestra.run.smithi026.stdout:Installed products updated. 2023-09-23T20:34:58.502 INFO:teuthology.orchestra.run.smithi026.stdout: 2023-09-23T20:34:58.503 INFO:teuthology.orchestra.run.smithi026.stdout:Installed: 2023-09-23T20:34:58.503 INFO:teuthology.orchestra.run.smithi026.stdout: rbd-fuse-2:17.2.6-1123.g61431f6b.el8.x86_64 2023-09-23T20:34:58.503 INFO:teuthology.orchestra.run.smithi026.stdout: 2023-09-23T20:34:58.503 INFO:teuthology.orchestra.run.smithi026.stdout:Complete! 2023-09-23T20:34:58.628 DEBUG:teuthology.orchestra.run.smithi026:> sudo yum -y install rbd-mirror 2023-09-23T20:34:59.020 INFO:teuthology.orchestra.run.smithi026.stdout:Updating Subscription Management repositories. 2023-09-23T20:34:59.849 INFO:teuthology.orchestra.run.smithi026.stdout:Last metadata expiration check: 0:03:41 ago on Sat 23 Sep 2023 08:31:18 PM UTC. 2023-09-23T20:35:01.405 INFO:teuthology.orchestra.run.smithi026.stdout:Dependencies resolved. 2023-09-23T20:35:01.405 INFO:teuthology.orchestra.run.smithi026.stdout:================================================================================ 2023-09-23T20:35:01.405 INFO:teuthology.orchestra.run.smithi026.stdout: Package Arch Version Repo Size 2023-09-23T20:35:01.405 INFO:teuthology.orchestra.run.smithi026.stdout:================================================================================ 2023-09-23T20:35:01.405 INFO:teuthology.orchestra.run.smithi026.stdout:Installing: 2023-09-23T20:35:01.406 INFO:teuthology.orchestra.run.smithi026.stdout: rbd-mirror x86_64 2:17.2.6-1123.g61431f6b.el8 ceph 3.5 M 2023-09-23T20:35:01.406 INFO:teuthology.orchestra.run.smithi026.stdout: 2023-09-23T20:35:01.406 INFO:teuthology.orchestra.run.smithi026.stdout:Transaction Summary 2023-09-23T20:35:01.407 INFO:teuthology.orchestra.run.smithi026.stdout:================================================================================ 2023-09-23T20:35:01.407 INFO:teuthology.orchestra.run.smithi026.stdout:Install 1 Package 2023-09-23T20:35:01.407 INFO:teuthology.orchestra.run.smithi026.stdout: 2023-09-23T20:35:01.407 INFO:teuthology.orchestra.run.smithi026.stdout:Total download size: 3.5 M 2023-09-23T20:35:01.408 INFO:teuthology.orchestra.run.smithi026.stdout:Installed size: 16 M 2023-09-23T20:35:01.408 INFO:teuthology.orchestra.run.smithi026.stdout:Downloading Packages: 2023-09-23T20:35:01.839 INFO:teuthology.orchestra.run.smithi026.stdout:rbd-mirror-17.2.6-1123.g61431f6b.el8.x86_64.rpm 8.0 MB/s | 3.5 MB 00:00 2023-09-23T20:35:01.840 INFO:teuthology.orchestra.run.smithi026.stdout:-------------------------------------------------------------------------------- 2023-09-23T20:35:01.840 INFO:teuthology.orchestra.run.smithi026.stdout:Total 8.0 MB/s | 3.5 MB 00:00 2023-09-23T20:35:01.840 INFO:teuthology.orchestra.run.smithi026.stdout:Running transaction check 2023-09-23T20:35:01.853 INFO:teuthology.orchestra.run.smithi026.stdout:Transaction check succeeded. 2023-09-23T20:35:01.854 INFO:teuthology.orchestra.run.smithi026.stdout:Running transaction test 2023-09-23T20:35:01.926 INFO:teuthology.orchestra.run.smithi026.stdout:Transaction test succeeded. 2023-09-23T20:35:01.930 INFO:teuthology.orchestra.run.smithi026.stdout:Running transaction 2023-09-23T20:35:02.403 INFO:teuthology.orchestra.run.smithi026.stdout: Preparing : 1/1 2023-09-23T20:35:02.621 INFO:teuthology.orchestra.run.smithi026.stdout: Installing : rbd-mirror-2:17.2.6-1123.g61431f6b.el8.x86_64 1/1 2023-09-23T20:35:02.988 INFO:teuthology.orchestra.run.smithi026.stdout: Running scriptlet: rbd-mirror-2:17.2.6-1123.g61431f6b.el8.x86_64 1/1 2023-09-23T20:35:04.051 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : rbd-mirror-2:17.2.6-1123.g61431f6b.el8.x86_64 1/1 2023-09-23T20:35:04.051 INFO:teuthology.orchestra.run.smithi026.stdout:Last metadata expiration check: 0:03:46 ago on Sat 23 Sep 2023 08:31:18 PM UTC. 2023-09-23T20:35:06.585 INFO:teuthology.orchestra.run.smithi026.stdout:Installed products updated. 2023-09-23T20:35:06.788 INFO:teuthology.orchestra.run.smithi026.stdout: 2023-09-23T20:35:06.788 INFO:teuthology.orchestra.run.smithi026.stdout:Installed: 2023-09-23T20:35:06.788 INFO:teuthology.orchestra.run.smithi026.stdout: rbd-mirror-2:17.2.6-1123.g61431f6b.el8.x86_64 2023-09-23T20:35:06.789 INFO:teuthology.orchestra.run.smithi026.stdout: 2023-09-23T20:35:06.789 INFO:teuthology.orchestra.run.smithi026.stdout:Complete! 2023-09-23T20:35:06.920 DEBUG:teuthology.orchestra.run.smithi026:> sudo yum -y install rbd-nbd 2023-09-23T20:35:07.301 INFO:teuthology.orchestra.run.smithi026.stdout:Updating Subscription Management repositories. 2023-09-23T20:35:08.143 INFO:teuthology.orchestra.run.smithi026.stdout:Last metadata expiration check: 0:03:50 ago on Sat 23 Sep 2023 08:31:18 PM UTC. 2023-09-23T20:35:09.698 INFO:teuthology.orchestra.run.smithi026.stdout:Dependencies resolved. 2023-09-23T20:35:09.698 INFO:teuthology.orchestra.run.smithi026.stdout:================================================================================ 2023-09-23T20:35:09.699 INFO:teuthology.orchestra.run.smithi026.stdout: Package Architecture Version Repository Size 2023-09-23T20:35:09.699 INFO:teuthology.orchestra.run.smithi026.stdout:================================================================================ 2023-09-23T20:35:09.699 INFO:teuthology.orchestra.run.smithi026.stdout:Installing: 2023-09-23T20:35:09.699 INFO:teuthology.orchestra.run.smithi026.stdout: rbd-nbd x86_64 2:17.2.6-1123.g61431f6b.el8 ceph 173 k 2023-09-23T20:35:09.699 INFO:teuthology.orchestra.run.smithi026.stdout: 2023-09-23T20:35:09.699 INFO:teuthology.orchestra.run.smithi026.stdout:Transaction Summary 2023-09-23T20:35:09.699 INFO:teuthology.orchestra.run.smithi026.stdout:================================================================================ 2023-09-23T20:35:09.699 INFO:teuthology.orchestra.run.smithi026.stdout:Install 1 Package 2023-09-23T20:35:09.699 INFO:teuthology.orchestra.run.smithi026.stdout: 2023-09-23T20:35:09.700 INFO:teuthology.orchestra.run.smithi026.stdout:Total download size: 173 k 2023-09-23T20:35:09.700 INFO:teuthology.orchestra.run.smithi026.stdout:Installed size: 552 k 2023-09-23T20:35:09.700 INFO:teuthology.orchestra.run.smithi026.stdout:Downloading Packages: 2023-09-23T20:35:09.992 INFO:teuthology.orchestra.run.smithi026.stdout:rbd-nbd-17.2.6-1123.g61431f6b.el8.x86_64.rpm 592 kB/s | 173 kB 00:00 2023-09-23T20:35:09.993 INFO:teuthology.orchestra.run.smithi026.stdout:-------------------------------------------------------------------------------- 2023-09-23T20:35:09.993 INFO:teuthology.orchestra.run.smithi026.stdout:Total 589 kB/s | 173 kB 00:00 2023-09-23T20:35:09.993 INFO:teuthology.orchestra.run.smithi026.stdout:Running transaction check 2023-09-23T20:35:10.005 INFO:teuthology.orchestra.run.smithi026.stdout:Transaction check succeeded. 2023-09-23T20:35:10.005 INFO:teuthology.orchestra.run.smithi026.stdout:Running transaction test 2023-09-23T20:35:10.065 INFO:teuthology.orchestra.run.smithi026.stdout:Transaction test succeeded. 2023-09-23T20:35:10.069 INFO:teuthology.orchestra.run.smithi026.stdout:Running transaction 2023-09-23T20:35:10.192 INFO:teuthology.orchestra.run.smithi026.stdout: Preparing : 1/1 2023-09-23T20:35:10.409 INFO:teuthology.orchestra.run.smithi026.stdout: Installing : rbd-nbd-2:17.2.6-1123.g61431f6b.el8.x86_64 1/1 2023-09-23T20:35:10.656 INFO:teuthology.orchestra.run.smithi026.stdout: Running scriptlet: rbd-nbd-2:17.2.6-1123.g61431f6b.el8.x86_64 1/1 2023-09-23T20:35:11.727 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : rbd-nbd-2:17.2.6-1123.g61431f6b.el8.x86_64 1/1 2023-09-23T20:35:11.728 INFO:teuthology.orchestra.run.smithi026.stdout:Last metadata expiration check: 0:03:53 ago on Sat 23 Sep 2023 08:31:18 PM UTC. 2023-09-23T20:35:14.242 INFO:teuthology.orchestra.run.smithi026.stdout:Installed products updated. 2023-09-23T20:35:14.444 INFO:teuthology.orchestra.run.smithi026.stdout: 2023-09-23T20:35:14.444 INFO:teuthology.orchestra.run.smithi026.stdout:Installed: 2023-09-23T20:35:14.444 INFO:teuthology.orchestra.run.smithi026.stdout: rbd-nbd-2:17.2.6-1123.g61431f6b.el8.x86_64 2023-09-23T20:35:14.444 INFO:teuthology.orchestra.run.smithi026.stdout: 2023-09-23T20:35:14.444 INFO:teuthology.orchestra.run.smithi026.stdout:Complete! 2023-09-23T20:35:14.567 DEBUG:teuthology.parallel:result is None 2023-09-23T20:35:14.568 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2023-09-23T20:35:14.568 INFO:teuthology.packaging:ref: None 2023-09-23T20:35:14.568 INFO:teuthology.packaging:tag: None 2023-09-23T20:35:14.568 INFO:teuthology.packaging:branch: quincy 2023-09-23T20:35:14.568 INFO:teuthology.packaging:sha1: bb0daeff97f020f686596fca6eba20c310e26fb7 2023-09-23T20:35:14.568 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F8%2Fx86_64&ref=quincy 2023-09-23T20:35:14.826 DEBUG:teuthology.orchestra.run.smithi026:> rpm -q ceph --qf '%{VERSION}-%{RELEASE}' 2023-09-23T20:35:14.849 INFO:teuthology.orchestra.run.smithi026.stdout:17.2.6-1123.g61431f6b.el8 2023-09-23T20:35:14.850 INFO:teuthology.packaging:The installed version of ceph is 17.2.6-1123.g61431f6b.el8 2023-09-23T20:35:14.850 INFO:teuthology.task.install:The correct ceph version 17.2.6-1123.g61431f6b is installed. 2023-09-23T20:35:14.851 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2023-09-23T20:35:14.851 INFO:teuthology.packaging:ref: None 2023-09-23T20:35:14.851 INFO:teuthology.packaging:tag: None 2023-09-23T20:35:14.851 INFO:teuthology.packaging:branch: quincy 2023-09-23T20:35:14.851 INFO:teuthology.packaging:sha1: bb0daeff97f020f686596fca6eba20c310e26fb7 2023-09-23T20:35:14.851 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F8%2Fx86_64&ref=quincy 2023-09-23T20:35:15.083 DEBUG:teuthology.orchestra.run.smithi070:> rpm -q ceph --qf '%{VERSION}-%{RELEASE}' 2023-09-23T20:35:15.109 INFO:teuthology.orchestra.run.smithi070.stdout:17.2.6-1123.g61431f6b.el8 2023-09-23T20:35:15.109 INFO:teuthology.packaging:The installed version of ceph is 17.2.6-1123.g61431f6b.el8 2023-09-23T20:35:15.110 INFO:teuthology.task.install:The correct ceph version 17.2.6-1123.g61431f6b is installed. 2023-09-23T20:35:15.110 INFO:teuthology.task.install.util:Shipping valgrind.supp... 2023-09-23T20:35:15.110 DEBUG:teuthology.orchestra.run.smithi026:> set -ex 2023-09-23T20:35:15.110 DEBUG:teuthology.orchestra.run.smithi026:> sudo dd of=/home/ubuntu/cephtest/valgrind.supp 2023-09-23T20:35:15.140 DEBUG:teuthology.orchestra.run.smithi070:> set -ex 2023-09-23T20:35:15.140 DEBUG:teuthology.orchestra.run.smithi070:> sudo dd of=/home/ubuntu/cephtest/valgrind.supp 2023-09-23T20:35:15.182 INFO:teuthology.task.install.util:Shipping 'daemon-helper'... 2023-09-23T20:35:15.182 DEBUG:teuthology.orchestra.run.smithi026:> set -ex 2023-09-23T20:35:15.182 DEBUG:teuthology.orchestra.run.smithi026:> sudo dd of=/usr/bin/daemon-helper 2023-09-23T20:35:15.211 DEBUG:teuthology.orchestra.run.smithi026:> sudo chmod a=rx -- /usr/bin/daemon-helper 2023-09-23T20:35:15.279 DEBUG:teuthology.orchestra.run.smithi070:> set -ex 2023-09-23T20:35:15.279 DEBUG:teuthology.orchestra.run.smithi070:> sudo dd of=/usr/bin/daemon-helper 2023-09-23T20:35:15.309 DEBUG:teuthology.orchestra.run.smithi070:> sudo chmod a=rx -- /usr/bin/daemon-helper 2023-09-23T20:35:15.377 INFO:teuthology.task.install.util:Shipping 'adjust-ulimits'... 2023-09-23T20:35:15.377 DEBUG:teuthology.orchestra.run.smithi026:> set -ex 2023-09-23T20:35:15.377 DEBUG:teuthology.orchestra.run.smithi026:> sudo dd of=/usr/bin/adjust-ulimits 2023-09-23T20:35:15.405 DEBUG:teuthology.orchestra.run.smithi026:> sudo chmod a=rx -- /usr/bin/adjust-ulimits 2023-09-23T20:35:15.474 DEBUG:teuthology.orchestra.run.smithi070:> set -ex 2023-09-23T20:35:15.474 DEBUG:teuthology.orchestra.run.smithi070:> sudo dd of=/usr/bin/adjust-ulimits 2023-09-23T20:35:15.503 DEBUG:teuthology.orchestra.run.smithi070:> sudo chmod a=rx -- /usr/bin/adjust-ulimits 2023-09-23T20:35:15.572 INFO:teuthology.task.install.util:Shipping 'stdin-killer'... 2023-09-23T20:35:15.572 DEBUG:teuthology.orchestra.run.smithi026:> set -ex 2023-09-23T20:35:15.572 DEBUG:teuthology.orchestra.run.smithi026:> sudo dd of=/usr/bin/stdin-killer 2023-09-23T20:35:15.599 DEBUG:teuthology.orchestra.run.smithi026:> sudo chmod a=rx -- /usr/bin/stdin-killer 2023-09-23T20:35:15.666 DEBUG:teuthology.orchestra.run.smithi070:> set -ex 2023-09-23T20:35:15.667 DEBUG:teuthology.orchestra.run.smithi070:> sudo dd of=/usr/bin/stdin-killer 2023-09-23T20:35:15.694 DEBUG:teuthology.orchestra.run.smithi070:> sudo chmod a=rx -- /usr/bin/stdin-killer 2023-09-23T20:35:15.761 INFO:teuthology.run_tasks:Running task cephadm... 2023-09-23T20:35:15.837 INFO:tasks.cephadm:Config: {'cephadm_branch': 'quincy', 'cephadm_git_url': 'https://github.com/ceph/ceph', 'conf': {'osd': {'osd_class_default_list': '*', 'osd_class_load_list': '*', 'debug ms': 1, 'debug osd': 20, 'osd shutdown pgref assert': True}, 'global': {'mon election default strategy': 3}, 'mgr': {'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20}}, 'image': 'quay.ceph.io/ceph-ci/ceph:quincy', 'create_rbd_pool': True, 'flavor': 'default', 'log-ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)'], 'sha1': 'bb0daeff97f020f686596fca6eba20c310e26fb7'} 2023-09-23T20:35:15.837 INFO:tasks.cephadm:Cluster image is quay.ceph.io/ceph-ci/ceph:quincy 2023-09-23T20:35:15.838 INFO:tasks.cephadm:Cluster fsid is b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 2023-09-23T20:35:15.838 INFO:tasks.cephadm:Choosing monitor IPs and ports... 2023-09-23T20:35:15.838 INFO:tasks.cephadm:Monitor IPs: {'mon.a': '172.21.15.26', 'mon.c': '[v2:172.21.15.26:3301,v1:172.21.15.26:6790]', 'mon.b': '172.21.15.70'} 2023-09-23T20:35:15.838 INFO:tasks.cephadm:First mon is mon.a on smithi026 2023-09-23T20:35:15.838 INFO:tasks.cephadm:First mgr is y 2023-09-23T20:35:15.838 INFO:tasks.cephadm:Normalizing hostnames... 2023-09-23T20:35:15.839 DEBUG:teuthology.orchestra.run.smithi026:> sudo hostname $(hostname -s) 2023-09-23T20:35:15.867 DEBUG:teuthology.orchestra.run.smithi070:> sudo hostname $(hostname -s) 2023-09-23T20:35:15.895 INFO:tasks.cephadm:Downloading cephadm (repo https://github.com/ceph/ceph ref quincy)... 2023-09-23T20:35:15.895 DEBUG:teuthology.orchestra.run.smithi026:> curl --silent https://raw.githubusercontent.com/ceph/ceph/quincy/src/cephadm/cephadm > /home/ubuntu/cephtest/cephadm && ls -l /home/ubuntu/cephtest/cephadm 2023-09-23T20:35:16.127 INFO:teuthology.orchestra.run.smithi026.stdout:-rw-rw-r--. 1 ubuntu ubuntu 366215 Sep 23 20:35 /home/ubuntu/cephtest/cephadm 2023-09-23T20:35:16.127 DEBUG:teuthology.orchestra.run.smithi070:> curl --silent https://raw.githubusercontent.com/ceph/ceph/quincy/src/cephadm/cephadm > /home/ubuntu/cephtest/cephadm && ls -l /home/ubuntu/cephtest/cephadm 2023-09-23T20:35:16.275 INFO:teuthology.orchestra.run.smithi070.stdout:-rw-rw-r--. 1 ubuntu ubuntu 366215 Sep 23 20:35 /home/ubuntu/cephtest/cephadm 2023-09-23T20:35:16.275 DEBUG:teuthology.orchestra.run.smithi026:> test -s /home/ubuntu/cephtest/cephadm && test $(stat -c%s /home/ubuntu/cephtest/cephadm) -gt 1000 && chmod +x /home/ubuntu/cephtest/cephadm 2023-09-23T20:35:16.294 DEBUG:teuthology.orchestra.run.smithi070:> test -s /home/ubuntu/cephtest/cephadm && test $(stat -c%s /home/ubuntu/cephtest/cephadm) -gt 1000 && chmod +x /home/ubuntu/cephtest/cephadm 2023-09-23T20:35:16.323 INFO:tasks.cephadm:Pulling image quay.ceph.io/ceph-ci/ceph:quincy on all hosts... 2023-09-23T20:35:16.323 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy pull 2023-09-23T20:35:16.336 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy pull 2023-09-23T20:35:17.776 INFO:teuthology.orchestra.run.smithi070.stderr:Pulling container image quay.ceph.io/ceph-ci/ceph:quincy... 2023-09-23T20:35:17.838 INFO:teuthology.orchestra.run.smithi026.stderr:Pulling container image quay.ceph.io/ceph-ci/ceph:quincy... 2023-09-23T20:35:46.661 INFO:teuthology.orchestra.run.smithi070.stdout:{ 2023-09-23T20:35:46.661 INFO:teuthology.orchestra.run.smithi070.stdout: "ceph_version": "ceph version 17.2.6-1123-g61431f6b (61431f6ba2752649b506867ec856b4a6236f90dd) quincy (stable)", 2023-09-23T20:35:46.661 INFO:teuthology.orchestra.run.smithi070.stdout: "image_id": "0db14aa010946f310e3ede6ffd96448192b98ccc76c262396fe9cf91887d0d64", 2023-09-23T20:35:46.662 INFO:teuthology.orchestra.run.smithi070.stdout: "repo_digests": [ 2023-09-23T20:35:46.662 INFO:teuthology.orchestra.run.smithi070.stdout: "quay.ceph.io/ceph-ci/ceph@sha256:1d807b57f44e899861bfb3500d17f81ef0ce7a664575560e306bb83d8fe31c44" 2023-09-23T20:35:46.662 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T20:35:46.662 INFO:teuthology.orchestra.run.smithi070.stdout:} 2023-09-23T20:35:51.054 INFO:teuthology.orchestra.run.smithi026.stdout:{ 2023-09-23T20:35:51.055 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph_version": "ceph version 17.2.6-1123-g61431f6b (61431f6ba2752649b506867ec856b4a6236f90dd) quincy (stable)", 2023-09-23T20:35:51.055 INFO:teuthology.orchestra.run.smithi026.stdout: "image_id": "0db14aa010946f310e3ede6ffd96448192b98ccc76c262396fe9cf91887d0d64", 2023-09-23T20:35:51.055 INFO:teuthology.orchestra.run.smithi026.stdout: "repo_digests": [ 2023-09-23T20:35:51.055 INFO:teuthology.orchestra.run.smithi026.stdout: "quay.ceph.io/ceph-ci/ceph@sha256:1d807b57f44e899861bfb3500d17f81ef0ce7a664575560e306bb83d8fe31c44" 2023-09-23T20:35:51.055 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T20:35:51.055 INFO:teuthology.orchestra.run.smithi026.stdout:} 2023-09-23T20:35:51.075 DEBUG:teuthology.orchestra.run.smithi026:> sudo mkdir -p /etc/ceph 2023-09-23T20:35:51.106 DEBUG:teuthology.orchestra.run.smithi070:> sudo mkdir -p /etc/ceph 2023-09-23T20:35:51.144 DEBUG:teuthology.orchestra.run.smithi026:> sudo chmod 777 /etc/ceph 2023-09-23T20:35:51.175 DEBUG:teuthology.orchestra.run.smithi070:> sudo chmod 777 /etc/ceph 2023-09-23T20:35:51.216 INFO:tasks.cephadm:Writing seed config... 2023-09-23T20:35:51.217 INFO:tasks.cephadm: override: [osd] osd_class_default_list = * 2023-09-23T20:35:51.218 INFO:tasks.cephadm: override: [osd] osd_class_load_list = * 2023-09-23T20:35:51.218 INFO:tasks.cephadm: override: [osd] debug ms = 1 2023-09-23T20:35:51.218 INFO:tasks.cephadm: override: [osd] debug osd = 20 2023-09-23T20:35:51.218 INFO:tasks.cephadm: override: [osd] osd shutdown pgref assert = True 2023-09-23T20:35:51.218 INFO:tasks.cephadm: override: [global] mon election default strategy = 3 2023-09-23T20:35:51.218 INFO:tasks.cephadm: override: [mgr] debug mgr = 20 2023-09-23T20:35:51.219 INFO:tasks.cephadm: override: [mgr] debug ms = 1 2023-09-23T20:35:51.219 INFO:tasks.cephadm: override: [mon] debug mon = 20 2023-09-23T20:35:51.219 INFO:tasks.cephadm: override: [mon] debug ms = 1 2023-09-23T20:35:51.219 INFO:tasks.cephadm: override: [mon] debug paxos = 20 2023-09-23T20:35:51.220 DEBUG:teuthology.orchestra.run.smithi026:> set -ex 2023-09-23T20:35:51.220 DEBUG:teuthology.orchestra.run.smithi026:> dd of=/home/ubuntu/cephtest/seed.ceph.conf 2023-09-23T20:35:51.238 DEBUG:tasks.cephadm:Final config: [global] # make logging friendly to teuthology log_to_file = true log_to_stderr = false log to journald = false mon cluster log file level = debug mon clock drift allowed = 1.000 # replicate across OSDs, not hosts osd crush chooseleaf type = 0 #osd pool default size = 2 osd pool default erasure code profile = plugin=jerasure technique=reed_sol_van k=2 m=1 crush-failure-domain=osd # enable some debugging auth debug = true ms die on old message = true ms die on bug = true debug asserts on shutdown = true # adjust warnings mon max pg per osd = 10000# >= luminous mon pg warn max object skew = 0 mon osd allow primary affinity = true mon osd allow pg remap = true mon warn on legacy crush tunables = false mon warn on crush straw calc version zero = false mon warn on no sortbitwise = false mon warn on osd down out interval zero = false mon warn on too few osds = false mon_warn_on_pool_pg_num_not_power_of_two = false # disable pg_autoscaler by default for new pools osd_pool_default_pg_autoscale_mode = off # tests delete pools mon allow pool delete = true fsid = b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 mon election default strategy = 3 [osd] osd scrub load threshold = 5.0 osd scrub max interval = 600 osd mclock profile = high_recovery_ops osd recover clone overlap = true osd recovery max chunk = 1048576 osd deep scrub update digest min age = 30 osd map max advance = 10 osd memory target autotune = true # debugging osd debug shutdown = true osd debug op order = true osd debug verify stray on activate = true osd debug pg log writeout = true osd debug verify cached snaps = true osd debug verify missing on start = true osd debug misdirected ops = true osd op queue = debug_random osd op queue cut off = debug_random osd shutdown pgref assert = True bdev debug aio = true osd sloppy crc = true osd_class_default_list = * osd_class_load_list = * debug ms = 1 debug osd = 20 [mgr] mon reweight min pgs per osd = 4 mon reweight min bytes per osd = 10 mgr/telemetry/nag = false debug mgr = 20 debug ms = 1 [mon] mon data avail warn = 5 mon mgr mkfs grace = 240 mon reweight min pgs per osd = 4 mon osd reporter subtree level = osd mon osd prime pg temp = true mon reweight min bytes per osd = 10 # rotate auth tickets quickly to exercise renewal paths auth mon ticket ttl = 660# 11m auth service ticket ttl = 240# 4m # don't complain about global id reclaim mon_warn_on_insecure_global_id_reclaim = false mon_warn_on_insecure_global_id_reclaim_allowed = false debug mon = 20 debug ms = 1 debug paxos = 20 [client.rgw] rgw cache enabled = true rgw enable ops log = true rgw enable usage log = true 2023-09-23T20:35:51.239 DEBUG:teuthology.orchestra.run.smithi026:mon.a> sudo journalctl -f -n 0 -u ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7@mon.a.service 2023-09-23T20:35:51.282 DEBUG:teuthology.orchestra.run.smithi026:mgr.y> sudo journalctl -f -n 0 -u ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7@mgr.y.service 2023-09-23T20:35:51.310 INFO:journalctl@ceph.mon.a.smithi026.stdout:-- Logs begin at Sat 2023-09-23 20:20:42 UTC. -- 2023-09-23T20:35:51.311 INFO:tasks.cephadm:Bootstrapping... 2023-09-23T20:35:51.311 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy -v bootstrap --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 --config /home/ubuntu/cephtest/seed.ceph.conf --output-config /etc/ceph/ceph.conf --output-keyring /etc/ceph/ceph.client.admin.keyring --output-pub-ssh-key /home/ubuntu/cephtest/ceph.pub --mon-id a --mgr-id y --orphan-initial-daemons --skip-monitoring-stack --mon-ip 172.21.15.26 --skip-admin-label && sudo chmod +r /etc/ceph/ceph.client.admin.keyring 2023-09-23T20:35:51.339 INFO:journalctl@ceph.mgr.y.smithi026.stdout:-- Logs begin at Sat 2023-09-23 20:20:42 UTC. -- 2023-09-23T20:35:51.522 INFO:teuthology.orchestra.run.smithi026.stdout:-------------------------------------------------------------------------------- 2023-09-23T20:35:51.522 INFO:teuthology.orchestra.run.smithi026.stdout:cephadm ['--image', 'quay.ceph.io/ceph-ci/ceph:quincy', '-v', 'bootstrap', '--fsid', 'b3f8e4ec-5a50-11ee-8db4-212e2dc638e7', '--config', '/home/ubuntu/cephtest/seed.ceph.conf', '--output-config', '/etc/ceph/ceph.conf', '--output-keyring', '/etc/ceph/ceph.client.admin.keyring', '--output-pub-ssh-key', '/home/ubuntu/cephtest/ceph.pub', '--mon-id', 'a', '--mgr-id', 'y', '--orphan-initial-daemons', '--skip-monitoring-stack', '--mon-ip', '172.21.15.26', '--skip-admin-label'] 2023-09-23T20:35:51.671 INFO:teuthology.orchestra.run.smithi026.stdout:/bin/podman: stdout 4.1.1 2023-09-23T20:35:51.671 INFO:teuthology.orchestra.run.smithi026.stderr:Specifying an fsid for your cluster offers no advantages and may increase the likelihood of fsid conflicts. 2023-09-23T20:35:51.672 INFO:teuthology.orchestra.run.smithi026.stdout:Verifying podman|docker is present... 2023-09-23T20:35:51.828 INFO:teuthology.orchestra.run.smithi026.stdout:/bin/podman: stdout 4.1.1 2023-09-23T20:35:51.828 INFO:teuthology.orchestra.run.smithi026.stdout:Verifying lvm2 is present... 2023-09-23T20:35:51.828 INFO:teuthology.orchestra.run.smithi026.stdout:Verifying time synchronization is in place... 2023-09-23T20:35:51.835 INFO:teuthology.orchestra.run.smithi026.stdout:Non-zero exit code 1 from systemctl is-enabled chrony.service 2023-09-23T20:35:51.836 INFO:teuthology.orchestra.run.smithi026.stdout:systemctl: stderr Failed to get unit file state for chrony.service: No such file or directory 2023-09-23T20:35:51.842 INFO:teuthology.orchestra.run.smithi026.stdout:Non-zero exit code 3 from systemctl is-active chrony.service 2023-09-23T20:35:51.842 INFO:teuthology.orchestra.run.smithi026.stdout:systemctl: stdout inactive 2023-09-23T20:35:51.848 INFO:teuthology.orchestra.run.smithi026.stdout:systemctl: stdout enabled 2023-09-23T20:35:51.854 INFO:teuthology.orchestra.run.smithi026.stdout:systemctl: stdout active 2023-09-23T20:35:51.854 INFO:teuthology.orchestra.run.smithi026.stdout:Unit chronyd.service is enabled and running 2023-09-23T20:35:51.854 INFO:teuthology.orchestra.run.smithi026.stdout:Repeating the final host check... 2023-09-23T20:35:51.996 INFO:teuthology.orchestra.run.smithi026.stdout:/bin/podman: stdout 4.1.1 2023-09-23T20:35:51.996 INFO:teuthology.orchestra.run.smithi026.stdout:podman (/bin/podman) version 4.1.1 is present 2023-09-23T20:35:51.996 INFO:teuthology.orchestra.run.smithi026.stdout:systemctl is present 2023-09-23T20:35:51.997 INFO:teuthology.orchestra.run.smithi026.stdout:lvcreate is present 2023-09-23T20:35:52.003 INFO:teuthology.orchestra.run.smithi026.stdout:Non-zero exit code 1 from systemctl is-enabled chrony.service 2023-09-23T20:35:52.003 INFO:teuthology.orchestra.run.smithi026.stdout:systemctl: stderr Failed to get unit file state for chrony.service: No such file or directory 2023-09-23T20:35:52.009 INFO:teuthology.orchestra.run.smithi026.stdout:Non-zero exit code 3 from systemctl is-active chrony.service 2023-09-23T20:35:52.009 INFO:teuthology.orchestra.run.smithi026.stdout:systemctl: stdout inactive 2023-09-23T20:35:52.015 INFO:teuthology.orchestra.run.smithi026.stdout:systemctl: stdout enabled 2023-09-23T20:35:52.021 INFO:teuthology.orchestra.run.smithi026.stdout:systemctl: stdout active 2023-09-23T20:35:52.021 INFO:teuthology.orchestra.run.smithi026.stdout:Unit chronyd.service is enabled and running 2023-09-23T20:35:52.021 INFO:teuthology.orchestra.run.smithi026.stdout:Host looks OK 2023-09-23T20:35:52.021 INFO:teuthology.orchestra.run.smithi026.stdout:Cluster fsid: b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 2023-09-23T20:35:52.022 INFO:teuthology.orchestra.run.smithi026.stdout:Acquiring lock 140522473863656 on /run/cephadm/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7.lock 2023-09-23T20:35:52.022 INFO:teuthology.orchestra.run.smithi026.stdout:Lock 140522473863656 acquired on /run/cephadm/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7.lock 2023-09-23T20:35:52.022 INFO:teuthology.orchestra.run.smithi026.stdout:Verifying IP 172.21.15.26 port 3300 ... 2023-09-23T20:35:52.022 INFO:teuthology.orchestra.run.smithi026.stdout:Verifying IP 172.21.15.26 port 6789 ... 2023-09-23T20:35:52.022 INFO:teuthology.orchestra.run.smithi026.stdout:Base mon IP(s) is [172.21.15.26:3300, 172.21.15.26:6789], mon addrv is [v2:172.21.15.26:3300,v1:172.21.15.26:6789] 2023-09-23T20:35:52.026 INFO:teuthology.orchestra.run.smithi026.stdout:/sbin/ip: stdout default via 172.21.15.254 dev ens1f0 proto dhcp metric 100 2023-09-23T20:35:52.026 INFO:teuthology.orchestra.run.smithi026.stdout:/sbin/ip: stdout 172.21.0.0/20 dev ens1f0 proto kernel scope link src 172.21.15.26 metric 100 2023-09-23T20:35:52.028 INFO:teuthology.orchestra.run.smithi026.stdout:/sbin/ip: stdout ::1 dev lo proto kernel metric 256 pref medium 2023-09-23T20:35:52.028 INFO:teuthology.orchestra.run.smithi026.stdout:/sbin/ip: stdout fe80::/64 dev ens1f0 proto kernel metric 256 pref medium 2023-09-23T20:35:52.029 INFO:teuthology.orchestra.run.smithi026.stdout:/sbin/ip: stdout default via fe80::327c:5e00:6487:71e0 dev ens1f0 proto ra metric 1024 expires 1769sec hoplimit 64 pref medium 2023-09-23T20:35:52.031 INFO:teuthology.orchestra.run.smithi026.stdout:/sbin/ip: stdout 1: lo: mtu 65536 state UNKNOWN qlen 1000 2023-09-23T20:35:52.031 INFO:teuthology.orchestra.run.smithi026.stdout:/sbin/ip: stdout inet6 ::1/128 scope host 2023-09-23T20:35:52.031 INFO:teuthology.orchestra.run.smithi026.stdout:/sbin/ip: stdout valid_lft forever preferred_lft forever 2023-09-23T20:35:52.031 INFO:teuthology.orchestra.run.smithi026.stdout:/sbin/ip: stdout 4: ens1f0: mtu 1500 state UP qlen 1000 2023-09-23T20:35:52.032 INFO:teuthology.orchestra.run.smithi026.stdout:/sbin/ip: stdout inet6 fe80::ae1f:6bff:fea5:cac0/64 scope link 2023-09-23T20:35:52.032 INFO:teuthology.orchestra.run.smithi026.stdout:/sbin/ip: stdout valid_lft forever preferred_lft forever 2023-09-23T20:35:52.032 INFO:teuthology.orchestra.run.smithi026.stdout:Mon IP `172.21.15.26` is in CIDR network `172.21.0.0/20` 2023-09-23T20:35:52.032 INFO:teuthology.orchestra.run.smithi026.stdout:Mon IP `172.21.15.26` is in CIDR network `172.21.0.0/20` 2023-09-23T20:35:52.032 INFO:teuthology.orchestra.run.smithi026.stdout:Inferred mon public CIDR from local network configuration ['172.21.0.0/20', '172.21.0.0/20'] 2023-09-23T20:35:52.033 INFO:teuthology.orchestra.run.smithi026.stdout:Internal network (--cluster-network) has not been provided, OSD replication will default to the public_network 2023-09-23T20:35:52.034 INFO:teuthology.orchestra.run.smithi026.stdout:Pulling container image quay.ceph.io/ceph-ci/ceph:quincy... 2023-09-23T20:35:52.484 INFO:teuthology.orchestra.run.smithi026.stdout:/bin/podman: stdout 0db14aa010946f310e3ede6ffd96448192b98ccc76c262396fe9cf91887d0d64 2023-09-23T20:35:52.484 INFO:teuthology.orchestra.run.smithi026.stdout:/bin/podman: stderr Trying to pull quay.ceph.io/ceph-ci/ceph:quincy... 2023-09-23T20:35:52.484 INFO:teuthology.orchestra.run.smithi026.stdout:/bin/podman: stderr Getting image source signatures 2023-09-23T20:35:52.485 INFO:teuthology.orchestra.run.smithi026.stdout:/bin/podman: stderr Copying blob sha256:af7d028f70e220a466755c6119effe51eebe359d9c00eb6b4883db3fabc6becd 2023-09-23T20:35:52.485 INFO:teuthology.orchestra.run.smithi026.stdout:/bin/podman: stderr Copying blob sha256:3f4fd5c3c800c6da9ef94691df203a05a5bbbf32c483d09d620ec06eef42ea2b 2023-09-23T20:35:52.485 INFO:teuthology.orchestra.run.smithi026.stdout:/bin/podman: stderr Copying config sha256:0db14aa010946f310e3ede6ffd96448192b98ccc76c262396fe9cf91887d0d64 2023-09-23T20:35:52.485 INFO:teuthology.orchestra.run.smithi026.stdout:/bin/podman: stderr Writing manifest to image destination 2023-09-23T20:35:52.485 INFO:teuthology.orchestra.run.smithi026.stdout:/bin/podman: stderr Storing signatures 2023-09-23T20:35:53.338 INFO:teuthology.orchestra.run.smithi026.stdout:ceph: stdout ceph version 17.2.6-1123-g61431f6b (61431f6ba2752649b506867ec856b4a6236f90dd) quincy (stable) 2023-09-23T20:35:53.339 INFO:teuthology.orchestra.run.smithi026.stdout:Ceph version: ceph version 17.2.6-1123-g61431f6b (61431f6ba2752649b506867ec856b4a6236f90dd) quincy (stable) 2023-09-23T20:35:53.339 INFO:teuthology.orchestra.run.smithi026.stdout:Extracting ceph user uid/gid from container image... 2023-09-23T20:35:54.101 INFO:teuthology.orchestra.run.smithi026.stdout:stat: stdout 167 167 2023-09-23T20:35:54.101 INFO:teuthology.orchestra.run.smithi026.stdout:Creating initial keys... 2023-09-23T20:35:54.851 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph-authtool: stdout AQAqTA9laqv2GxAAU0CjA0pAAoPbm6WsLeB7hQ== 2023-09-23T20:35:55.602 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph-authtool: stdout AQArTA9ljU6QCxAA2BgYbzRUmahp4t8rRFaO3g== 2023-09-23T20:35:56.347 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph-authtool: stdout AQArTA9l4cK0OhAAakCfNSearwilJAlfUI+RCw== 2023-09-23T20:35:56.348 INFO:teuthology.orchestra.run.smithi026.stdout:Creating initial monmap... 2023-09-23T20:35:57.102 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/monmaptool: stdout /usr/bin/monmaptool: monmap file /tmp/monmap 2023-09-23T20:35:57.102 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/monmaptool: stdout setting min_mon_release = octopus 2023-09-23T20:35:57.102 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/monmaptool: stdout /usr/bin/monmaptool: set fsid to b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 2023-09-23T20:35:57.102 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/monmaptool: stdout /usr/bin/monmaptool: writing epoch 0 to /tmp/monmap (1 monitors) 2023-09-23T20:35:57.102 INFO:teuthology.orchestra.run.smithi026.stdout:monmaptool for a [v2:172.21.15.26:3300,v1:172.21.15.26:6789] on /usr/bin/monmaptool: monmap file /tmp/monmap 2023-09-23T20:35:57.102 INFO:teuthology.orchestra.run.smithi026.stdout:setting min_mon_release = octopus 2023-09-23T20:35:57.102 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/monmaptool: set fsid to b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 2023-09-23T20:35:57.102 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/monmaptool: writing epoch 0 to /tmp/monmap (1 monitors) 2023-09-23T20:35:57.103 INFO:teuthology.orchestra.run.smithi026.stdout: 2023-09-23T20:35:57.103 INFO:teuthology.orchestra.run.smithi026.stdout:Creating mon... 2023-09-23T20:35:57.854 INFO:teuthology.orchestra.run.smithi026.stdout:create mon.a on 2023-09-23T20:35:57.952 INFO:teuthology.orchestra.run.smithi026.stdout:systemctl: stderr Removed /etc/systemd/system/multi-user.target.wants/ceph.target. 2023-09-23T20:35:58.045 INFO:teuthology.orchestra.run.smithi026.stdout:systemctl: stderr Created symlink /etc/systemd/system/multi-user.target.wants/ceph.target → /etc/systemd/system/ceph.target. 2023-09-23T20:35:58.147 INFO:teuthology.orchestra.run.smithi026.stdout:systemctl: stderr Created symlink /etc/systemd/system/multi-user.target.wants/ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7.target → /etc/systemd/system/ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7.target. 2023-09-23T20:35:58.147 INFO:teuthology.orchestra.run.smithi026.stdout:systemctl: stderr Created symlink /etc/systemd/system/ceph.target.wants/ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7.target → /etc/systemd/system/ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7.target. 2023-09-23T20:35:58.280 INFO:teuthology.orchestra.run.smithi026.stdout:Non-zero exit code 1 from systemctl reset-failed ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7@mon.a 2023-09-23T20:35:58.280 INFO:teuthology.orchestra.run.smithi026.stdout:systemctl: stderr Failed to reset failed state of unit ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7@mon.a.service: Unit ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7@mon.a.service not loaded. 2023-09-23T20:35:58.373 INFO:teuthology.orchestra.run.smithi026.stdout:systemctl: stderr Created symlink /etc/systemd/system/ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7.target.wants/ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7@mon.a.service → /etc/systemd/system/ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7@.service. 2023-09-23T20:35:58.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:35:58 smithi026 systemd[1]: Starting Ceph mon.a for b3f8e4ec-5a50-11ee-8db4-212e2dc638e7... 2023-09-23T20:35:58.970 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:35:58 smithi026 podman[95301]: 2023-09-23T20:35:58.970 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:35:58 smithi026 bash[95301]: 4ec1be3e0e8ab175e0f9387c5cb8269f2a646b44421134562079c040a19d2b5d 2023-09-23T20:35:58.970 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:35:58 smithi026 ceph-mon[95341]: mkfs b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 2023-09-23T20:35:58.970 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:35:58 smithi026 ceph-mon[95341]: mon.a is new leader, mons a in quorum (ranks 0) 2023-09-23T20:35:58.981 INFO:teuthology.orchestra.run.smithi026.stdout:Non-zero exit code 1 from systemctl is-enabled firewalld.service 2023-09-23T20:35:58.981 INFO:teuthology.orchestra.run.smithi026.stdout:systemctl: stdout disabled 2023-09-23T20:35:58.989 INFO:teuthology.orchestra.run.smithi026.stdout:Non-zero exit code 3 from systemctl is-active firewalld.service 2023-09-23T20:35:58.989 INFO:teuthology.orchestra.run.smithi026.stdout:systemctl: stdout inactive 2023-09-23T20:35:58.990 INFO:teuthology.orchestra.run.smithi026.stdout:firewalld.service is not enabled 2023-09-23T20:35:58.990 INFO:teuthology.orchestra.run.smithi026.stdout:Not possible to enable service . firewalld.service is not available 2023-09-23T20:35:58.990 INFO:teuthology.orchestra.run.smithi026.stdout:Waiting for mon to start... 2023-09-23T20:35:58.990 INFO:teuthology.orchestra.run.smithi026.stdout:Waiting for mon... 2023-09-23T20:35:59.234 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:35:58 smithi026 systemd[1]: Started Ceph mon.a for b3f8e4ec-5a50-11ee-8db4-212e2dc638e7. 2023-09-23T20:35:59.936 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout cluster: 2023-09-23T20:35:59.938 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout id: b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 2023-09-23T20:35:59.938 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout health: HEALTH_OK 2023-09-23T20:35:59.938 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout 2023-09-23T20:35:59.938 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout services: 2023-09-23T20:35:59.939 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout mon: 1 daemons, quorum a (age 0.710653s) 2023-09-23T20:35:59.939 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout mgr: no daemons active 2023-09-23T20:35:59.939 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout osd: 0 osds: 0 up, 0 in 2023-09-23T20:35:59.939 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout 2023-09-23T20:35:59.939 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout data: 2023-09-23T20:35:59.939 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout pools: 0 pools, 0 pgs 2023-09-23T20:35:59.939 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout objects: 0 objects, 0 B 2023-09-23T20:35:59.939 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout usage: 0 B used, 0 B / 0 B avail 2023-09-23T20:35:59.939 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout pgs: 2023-09-23T20:35:59.939 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout 2023-09-23T20:35:59.939 INFO:teuthology.orchestra.run.smithi026.stdout:mon is available 2023-09-23T20:35:59.940 INFO:teuthology.orchestra.run.smithi026.stdout:Assimilating anything we can from ceph.conf... 2023-09-23T20:36:00.183 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:35:59 smithi026 ceph-mon[95341]: mon.a is new leader, mons a in quorum (ranks 0) 2023-09-23T20:36:00.183 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:35:59 smithi026 ceph-mon[95341]: monmap e1: 1 mons at {a=[v2:172.21.15.26:3300/0,v1:172.21.15.26:6789/0]} removed_ranks: {} 2023-09-23T20:36:00.183 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:35:59 smithi026 ceph-mon[95341]: fsmap 2023-09-23T20:36:00.183 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:35:59 smithi026 ceph-mon[95341]: osdmap e1: 0 total, 0 up, 0 in 2023-09-23T20:36:00.183 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:35:59 smithi026 ceph-mon[95341]: mgrmap e1: no daemons active 2023-09-23T20:36:00.183 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:35:59 smithi026 ceph-mon[95341]: from='client.? 172.21.15.26:0/1345654009' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2023-09-23T20:36:00.982 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout 2023-09-23T20:36:01.049 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout [global] 2023-09-23T20:36:01.049 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout fsid = b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 2023-09-23T20:36:01.049 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout mon_host = [v2:172.21.15.26:3300,v1:172.21.15.26:6789] 2023-09-23T20:36:01.049 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout mon_osd_allow_pg_remap = true 2023-09-23T20:36:01.049 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout mon_osd_allow_primary_affinity = true 2023-09-23T20:36:01.050 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout mon_warn_on_no_sortbitwise = false 2023-09-23T20:36:01.050 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout osd_crush_chooseleaf_type = 0 2023-09-23T20:36:01.050 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout 2023-09-23T20:36:01.050 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout [mgr] 2023-09-23T20:36:01.051 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout mgr/telemetry/nag = false 2023-09-23T20:36:01.051 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout 2023-09-23T20:36:01.051 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout [osd] 2023-09-23T20:36:01.051 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout osd_map_max_advance = 10 2023-09-23T20:36:01.051 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout osd_sloppy_crc = true 2023-09-23T20:36:01.051 INFO:teuthology.orchestra.run.smithi026.stdout:Generating new minimal ceph.conf... 2023-09-23T20:36:01.185 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:36:00 smithi026 ceph-mon[95341]: from='client.? 172.21.15.26:0/3312235273' entity='client.admin' cmd=[{"prefix": "config assimilate-conf"}]: dispatch 2023-09-23T20:36:01.186 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:36:00 smithi026 ceph-mon[95341]: from='client.? 172.21.15.26:0/3312235273' entity='client.admin' cmd='[{"prefix": "config assimilate-conf"}]': finished 2023-09-23T20:36:01.923 INFO:teuthology.orchestra.run.smithi026.stdout:Restarting the monitor... 2023-09-23T20:36:02.272 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:36:01 smithi026 ceph-mon[95341]: from='client.? 172.21.15.26:0/1359614538' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:36:02.272 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:36:01 smithi026 systemd[1]: Stopping Ceph mon.a for b3f8e4ec-5a50-11ee-8db4-212e2dc638e7... 2023-09-23T20:36:02.566 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:36:02 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mon-a[95317]: 2023-09-23T20:36:02.270+0000 7fb8894d9700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-mon -n mon.a -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2023-09-23T20:36:02.566 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:36:02 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mon-a[95317]: 2023-09-23T20:36:02.270+0000 7fb8894d9700 -1 mon.a@0(leader) e1 *** Got Signal Terminated *** 2023-09-23T20:36:02.825 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:36:02 smithi026 bash[95676]: ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mon-a 2023-09-23T20:36:03.272 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:36:03 smithi026 systemd[1]: ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7@mon.a.service: Succeeded. 2023-09-23T20:36:03.272 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:36:03 smithi026 systemd[1]: Stopped Ceph mon.a for b3f8e4ec-5a50-11ee-8db4-212e2dc638e7. 2023-09-23T20:36:03.272 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:36:03 smithi026 systemd[1]: Starting Ceph mon.a for b3f8e4ec-5a50-11ee-8db4-212e2dc638e7... 2023-09-23T20:36:03.496 INFO:teuthology.orchestra.run.smithi026.stdout:Setting mon public_network to 172.21.0.0/20 2023-09-23T20:36:03.543 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:36:03 smithi026 podman[95799]: 2023-09-23T20:36:03.544 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:36:03 smithi026 ceph-mon[95839]: set uid:gid to 167:167 (ceph:ceph) 2023-09-23T20:36:03.544 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:36:03 smithi026 ceph-mon[95839]: ceph version 17.2.6-1123-g61431f6b (61431f6ba2752649b506867ec856b4a6236f90dd) quincy (stable), process ceph-mon, pid 7 2023-09-23T20:36:03.544 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:36:03 smithi026 ceph-mon[95839]: pidfile_write: ignore empty --pid-file 2023-09-23T20:36:03.544 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:36:03 smithi026 bash[95799]: 24cf5254b0b85b8282cfbfe98d1f46d6dbb8e647fdac2aa68e71c36fade36c2b 2023-09-23T20:36:03.544 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:36:03 smithi026 ceph-mon[95839]: load: jerasure load: lrc 2023-09-23T20:36:03.544 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:36:03 smithi026 ceph-mon[95839]: rocksdb: RocksDB version: 6.15.5 2023-09-23T20:36:03.544 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:36:03 smithi026 ceph-mon[95839]: rocksdb: Git sha rocksdb_build_git_sha:@0@ 2023-09-23T20:36:03.544 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:36:03 smithi026 ceph-mon[95839]: rocksdb: Compile date Sep 22 2023 2023-09-23T20:36:03.544 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:36:03 smithi026 ceph-mon[95839]: rocksdb: DB SUMMARY 2023-09-23T20:36:03.544 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:36:03 smithi026 ceph-mon[95839]: rocksdb: DB Session ID: OJ5WVOBFAHVKDVWUVE1M 2023-09-23T20:36:03.544 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:36:03 smithi026 ceph-mon[95839]: rocksdb: CURRENT file: CURRENT 2023-09-23T20:36:03.545 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:36:03 smithi026 ceph-mon[95839]: rocksdb: IDENTITY file: IDENTITY 2023-09-23T20:36:03.545 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:36:03 smithi026 ceph-mon[95839]: rocksdb: MANIFEST file: MANIFEST-000009 size: 131 Bytes 2023-09-23T20:36:03.545 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:36:03 smithi026 ceph-mon[95839]: rocksdb: SST files in /var/lib/ceph/mon/ceph-a/store.db dir, Total Num: 1, files: 000008.sst 2023-09-23T20:36:03.545 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:36:03 smithi026 ceph-mon[95839]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-a/store.db: 000010.log size: 83589 ; 2023-09-23T20:36:03.545 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:36:03 smithi026 ceph-mon[95839]: rocksdb: Options.error_if_exists: 0 2023-09-23T20:36:03.545 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:36:03 smithi026 ceph-mon[95839]: rocksdb: Options.create_if_missing: 0 2023-09-23T20:36:03.545 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:36:03 smithi026 ceph-mon[95839]: rocksdb: Options.paranoid_checks: 1 2023-09-23T20:36:03.545 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:36:03 smithi026 ceph-mon[95839]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2023-09-23T20:36:03.545 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:36:03 smithi026 ceph-mon[95839]: rocksdb: Options.env: 0x55ff70ec2a80 2023-09-23T20:36:03.545 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:36:03 smithi026 ceph-mon[95839]: rocksdb: Options.fs: Posix File System 2023-09-23T20:36:03.545 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:36:03 smithi026 ceph-mon[95839]: rocksdb: Options.info_log: 0x55ff72cefd80 2023-09-23T20:36:03.545 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:36:03 smithi026 ceph-mon[95839]: rocksdb: Options.max_file_opening_threads: 16 2023-09-23T20:36:03.546 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:36:03 smithi026 ceph-mon[95839]: rocksdb: Options.statistics: (nil) 2023-09-23T20:36:03.546 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:36:03 smithi026 ceph-mon[95839]: rocksdb: Options.use_fsync: 0 2023-09-23T20:36:03.546 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:36:03 smithi026 ceph-mon[95839]: rocksdb: Options.max_log_file_size: 0 2023-09-23T20:36:03.546 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:36:03 smithi026 ceph-mon[95839]: rocksdb: Options.max_manifest_file_size: 1073741824 2023-09-23T20:36:03.546 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:36:03 smithi026 ceph-mon[95839]: rocksdb: Options.log_file_time_to_roll: 0 2023-09-23T20:36:03.546 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:36:03 smithi026 ceph-mon[95839]: rocksdb: Options.keep_log_file_num: 1000 2023-09-23T20:36:03.546 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:36:03 smithi026 ceph-mon[95839]: rocksdb: Options.recycle_log_file_num: 0 2023-09-23T20:36:03.546 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:36:03 smithi026 ceph-mon[95839]: rocksdb: Options.allow_fallocate: 1 2023-09-23T20:36:03.546 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:36:03 smithi026 ceph-mon[95839]: rocksdb: Options.allow_mmap_reads: 0 2023-09-23T20:36:03.546 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:36:03 smithi026 ceph-mon[95839]: rocksdb: Options.allow_mmap_writes: 0 2023-09-23T20:36:03.546 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:36:03 smithi026 ceph-mon[95839]: rocksdb: Options.use_direct_reads: 0 2023-09-23T20:36:03.546 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:36:03 smithi026 ceph-mon[95839]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2023-09-23T20:36:03.547 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:36:03 smithi026 ceph-mon[95839]: rocksdb: Options.create_missing_column_families: 0 2023-09-23T20:36:03.547 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:36:03 smithi026 ceph-mon[95839]: rocksdb: Options.db_log_dir: 2023-09-23T20:36:03.547 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:36:03 smithi026 ceph-mon[95839]: rocksdb: Options.wal_dir: /var/lib/ceph/mon/ceph-a/store.db 2023-09-23T20:36:03.547 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:36:03 smithi026 ceph-mon[95839]: rocksdb: Options.table_cache_numshardbits: 6 2023-09-23T20:36:03.547 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:36:03 smithi026 ceph-mon[95839]: rocksdb: Options.WAL_ttl_seconds: 0 2023-09-23T20:36:03.547 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:36:03 smithi026 ceph-mon[95839]: rocksdb: Options.WAL_size_limit_MB: 0 2023-09-23T20:36:03.547 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:36:03 smithi026 ceph-mon[95839]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2023-09-23T20:36:03.547 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:36:03 smithi026 ceph-mon[95839]: rocksdb: Options.manifest_preallocation_size: 4194304 2023-09-23T20:36:03.547 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:36:03 smithi026 ceph-mon[95839]: rocksdb: Options.is_fd_close_on_exec: 1 2023-09-23T20:36:03.547 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:36:03 smithi026 ceph-mon[95839]: rocksdb: Options.advise_random_on_open: 1 2023-09-23T20:36:03.547 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:36:03 smithi026 ceph-mon[95839]: rocksdb: Options.db_write_buffer_size: 0 2023-09-23T20:36:03.547 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:36:03 smithi026 ceph-mon[95839]: rocksdb: Options.write_buffer_manager: 0x55ff739e2960 2023-09-23T20:36:03.547 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:36:03 smithi026 ceph-mon[95839]: rocksdb: Options.access_hint_on_compaction_start: 1 2023-09-23T20:36:03.548 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:36:03 smithi026 ceph-mon[95839]: rocksdb: Options.new_table_reader_for_compaction_inputs: 0 2023-09-23T20:36:03.548 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:36:03 smithi026 ceph-mon[95839]: rocksdb: Options.random_access_max_buffer_size: 1048576 2023-09-23T20:36:03.548 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:36:03 smithi026 ceph-mon[95839]: rocksdb: Options.use_adaptive_mutex: 0 2023-09-23T20:36:03.548 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:36:03 smithi026 ceph-mon[95839]: rocksdb: Options.rate_limiter: (nil) 2023-09-23T20:36:03.548 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:36:03 smithi026 ceph-mon[95839]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2023-09-23T20:36:03.548 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:36:03 smithi026 ceph-mon[95839]: rocksdb: Options.wal_recovery_mode: 2 2023-09-23T20:36:03.548 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:36:03 smithi026 ceph-mon[95839]: rocksdb: Options.enable_thread_tracking: 0 2023-09-23T20:36:03.548 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:36:03 smithi026 ceph-mon[95839]: rocksdb: Options.enable_pipelined_write: 0 2023-09-23T20:36:03.548 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:36:03 smithi026 ceph-mon[95839]: rocksdb: Options.unordered_write: 0 2023-09-23T20:36:03.548 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:36:03 smithi026 ceph-mon[95839]: rocksdb: Options.allow_concurrent_memtable_write: 1 2023-09-23T20:36:03.548 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:36:03 smithi026 ceph-mon[95839]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2023-09-23T20:36:03.548 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:36:03 smithi026 ceph-mon[95839]: rocksdb: Options.write_thread_max_yield_usec: 100 2023-09-23T20:36:03.548 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:36:03 smithi026 ceph-mon[95839]: rocksdb: Options.write_thread_slow_yield_usec: 3 2023-09-23T20:36:03.549 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:36:03 smithi026 ceph-mon[95839]: rocksdb: Options.row_cache: None 2023-09-23T20:36:03.549 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:36:03 smithi026 ceph-mon[95839]: rocksdb: Options.wal_filter: None 2023-09-23T20:36:03.549 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:36:03 smithi026 ceph-mon[95839]: rocksdb: Options.avoid_flush_during_recovery: 0 2023-09-23T20:36:03.549 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:36:03 smithi026 ceph-mon[95839]: rocksdb: Options.allow_ingest_behind: 0 2023-09-23T20:36:03.549 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:36:03 smithi026 ceph-mon[95839]: rocksdb: Options.preserve_deletes: 0 2023-09-23T20:36:03.549 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:36:03 smithi026 ceph-mon[95839]: rocksdb: Options.two_write_queues: 0 2023-09-23T20:36:03.549 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:36:03 smithi026 ceph-mon[95839]: rocksdb: Options.manual_wal_flush: 0 2023-09-23T20:36:03.549 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:36:03 smithi026 ceph-mon[95839]: rocksdb: Options.atomic_flush: 0 2023-09-23T20:36:03.549 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:36:03 smithi026 ceph-mon[95839]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2023-09-23T20:36:03.549 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:36:03 smithi026 ceph-mon[95839]: rocksdb: Options.persist_stats_to_disk: 0 2023-09-23T20:36:03.549 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:36:03 smithi026 ceph-mon[95839]: rocksdb: Options.write_dbid_to_manifest: 0 2023-09-23T20:36:03.549 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:36:03 smithi026 ceph-mon[95839]: rocksdb: Options.log_readahead_size: 0 2023-09-23T20:36:03.549 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:36:03 smithi026 ceph-mon[95839]: rocksdb: Options.file_checksum_gen_factory: Unknown 2023-09-23T20:36:03.550 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:36:03 smithi026 ceph-mon[95839]: rocksdb: Options.best_efforts_recovery: 0 2023-09-23T20:36:03.550 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:36:03 smithi026 ceph-mon[95839]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2023-09-23T20:36:03.550 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:36:03 smithi026 ceph-mon[95839]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2023-09-23T20:36:03.550 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:36:03 smithi026 ceph-mon[95839]: rocksdb: Options.allow_data_in_errors: 0 2023-09-23T20:36:03.550 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:36:03 smithi026 ceph-mon[95839]: rocksdb: Options.db_host_id: __hostname__ 2023-09-23T20:36:03.551 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:36:03 smithi026 ceph-mon[95839]: rocksdb: Options.max_background_jobs: 2 2023-09-23T20:36:03.551 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:36:03 smithi026 ceph-mon[95839]: rocksdb: Options.max_background_compactions: -1 2023-09-23T20:36:03.551 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:36:03 smithi026 ceph-mon[95839]: rocksdb: Options.max_subcompactions: 1 2023-09-23T20:36:03.551 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:36:03 smithi026 ceph-mon[95839]: rocksdb: Options.avoid_flush_during_shutdown: 0 2023-09-23T20:36:03.551 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:36:03 smithi026 ceph-mon[95839]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2023-09-23T20:36:03.551 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:36:03 smithi026 ceph-mon[95839]: rocksdb: Options.delayed_write_rate : 16777216 2023-09-23T20:36:03.551 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:36:03 smithi026 ceph-mon[95839]: rocksdb: Options.max_total_wal_size: 0 2023-09-23T20:36:03.551 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:36:03 smithi026 ceph-mon[95839]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2023-09-23T20:36:03.552 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:36:03 smithi026 ceph-mon[95839]: rocksdb: Options.stats_dump_period_sec: 600 2023-09-23T20:36:03.552 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:36:03 smithi026 ceph-mon[95839]: rocksdb: Options.stats_persist_period_sec: 600 2023-09-23T20:36:03.552 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:36:03 smithi026 ceph-mon[95839]: rocksdb: Options.stats_history_buffer_size: 1048576 2023-09-23T20:36:03.552 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:36:03 smithi026 ceph-mon[95839]: rocksdb: Options.max_open_files: -1 2023-09-23T20:36:03.552 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:36:03 smithi026 ceph-mon[95839]: rocksdb: Options.bytes_per_sync: 0 2023-09-23T20:36:03.552 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:36:03 smithi026 ceph-mon[95839]: rocksdb: Options.wal_bytes_per_sync: 0 2023-09-23T20:36:03.552 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:36:03 smithi026 ceph-mon[95839]: rocksdb: Options.strict_bytes_per_sync: 0 2023-09-23T20:36:03.552 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:36:03 smithi026 ceph-mon[95839]: rocksdb: Options.compaction_readahead_size: 0 2023-09-23T20:36:03.552 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:36:03 smithi026 ceph-mon[95839]: rocksdb: Options.max_background_flushes: -1 2023-09-23T20:36:03.552 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:36:03 smithi026 ceph-mon[95839]: rocksdb: Compression algorithms supported: 2023-09-23T20:36:03.552 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:36:03 smithi026 ceph-mon[95839]: rocksdb: kZSTDNotFinalCompression supported: 0 2023-09-23T20:36:03.552 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:36:03 smithi026 ceph-mon[95839]: rocksdb: kZSTD supported: 0 2023-09-23T20:36:03.553 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:36:03 smithi026 ceph-mon[95839]: rocksdb: kXpressCompression supported: 0 2023-09-23T20:36:03.553 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:36:03 smithi026 ceph-mon[95839]: rocksdb: kLZ4HCCompression supported: 1 2023-09-23T20:36:03.553 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:36:03 smithi026 ceph-mon[95839]: rocksdb: kZlibCompression supported: 1 2023-09-23T20:36:03.553 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:36:03 smithi026 ceph-mon[95839]: rocksdb: kSnappyCompression supported: 1 2023-09-23T20:36:03.553 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:36:03 smithi026 ceph-mon[95839]: rocksdb: kLZ4Compression supported: 1 2023-09-23T20:36:03.553 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:36:03 smithi026 ceph-mon[95839]: rocksdb: kBZip2Compression supported: 0 2023-09-23T20:36:03.553 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:36:03 smithi026 ceph-mon[95839]: rocksdb: Fast CRC32 supported: Supported on x86 2023-09-23T20:36:03.553 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:36:03 smithi026 ceph-mon[95839]: rocksdb: [db/version_set.cc:4724] Recovering from manifest file: /var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000009 2023-09-23T20:36:03.553 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:36:03 smithi026 ceph-mon[95839]: rocksdb: [db/column_family.cc:595] --------------- Options for column family [default]: 2023-09-23T20:36:03.553 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:36:03 smithi026 ceph-mon[95839]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2023-09-23T20:36:03.553 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:36:03 smithi026 ceph-mon[95839]: rocksdb: Options.merge_operator: 2023-09-23T20:36:03.553 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:36:03 smithi026 ceph-mon[95839]: rocksdb: Options.compaction_filter: None 2023-09-23T20:36:03.553 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:36:03 smithi026 ceph-mon[95839]: rocksdb: Options.compaction_filter_factory: None 2023-09-23T20:36:03.554 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:36:03 smithi026 ceph-mon[95839]: rocksdb: Options.sst_partitioner_factory: None 2023-09-23T20:36:03.554 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:36:03 smithi026 ceph-mon[95839]: rocksdb: Options.memtable_factory: SkipListFactory 2023-09-23T20:36:03.554 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:36:03 smithi026 ceph-mon[95839]: rocksdb: Options.table_factory: BlockBasedTable 2023-09-23T20:36:03.554 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:36:03 smithi026 ceph-mon[95839]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x55ff72cc2fa0) 2023-09-23T20:36:03.554 INFO:journalctl@ceph.mon.a.smithi026.stdout: cache_index_and_filter_blocks: 1 2023-09-23T20:36:03.554 INFO:journalctl@ceph.mon.a.smithi026.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2023-09-23T20:36:03.554 INFO:journalctl@ceph.mon.a.smithi026.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2023-09-23T20:36:03.554 INFO:journalctl@ceph.mon.a.smithi026.stdout: pin_top_level_index_and_filter: 1 2023-09-23T20:36:03.554 INFO:journalctl@ceph.mon.a.smithi026.stdout: index_type: 0 2023-09-23T20:36:03.554 INFO:journalctl@ceph.mon.a.smithi026.stdout: data_block_index_type: 0 2023-09-23T20:36:03.554 INFO:journalctl@ceph.mon.a.smithi026.stdout: index_shortening: 1 2023-09-23T20:36:03.554 INFO:journalctl@ceph.mon.a.smithi026.stdout: data_block_hash_table_util_ratio: 0.750000 2023-09-23T20:36:03.554 INFO:journalctl@ceph.mon.a.smithi026.stdout: hash_index_allow_collision: 1 2023-09-23T20:36:03.555 INFO:journalctl@ceph.mon.a.smithi026.stdout: checksum: 1 2023-09-23T20:36:03.555 INFO:journalctl@ceph.mon.a.smithi026.stdout: no_block_cache: 0 2023-09-23T20:36:03.555 INFO:journalctl@ceph.mon.a.smithi026.stdout: block_cache: 0x55ff72cf5090 2023-09-23T20:36:03.555 INFO:journalctl@ceph.mon.a.smithi026.stdout: block_cache_name: BinnedLRUCache 2023-09-23T20:36:03.555 INFO:journalctl@ceph.mon.a.smithi026.stdout: block_cache_options: 2023-09-23T20:36:03.555 INFO:journalctl@ceph.mon.a.smithi026.stdout: capacity : 536870912 2023-09-23T20:36:03.555 INFO:journalctl@ceph.mon.a.smithi026.stdout: num_shard_bits : 4 2023-09-23T20:36:03.555 INFO:journalctl@ceph.mon.a.smithi026.stdout: strict_capacity_limit : 0 2023-09-23T20:36:03.555 INFO:journalctl@ceph.mon.a.smithi026.stdout: high_pri_pool_ratio: 0.000 2023-09-23T20:36:03.555 INFO:journalctl@ceph.mon.a.smithi026.stdout: block_cache_compressed: (nil) 2023-09-23T20:36:03.555 INFO:journalctl@ceph.mon.a.smithi026.stdout: persistent_cache: (nil) 2023-09-23T20:36:03.555 INFO:journalctl@ceph.mon.a.smithi026.stdout: block_size: 4096 2023-09-23T20:36:03.555 INFO:journalctl@ceph.mon.a.smithi026.stdout: block_size_deviation: 10 2023-09-23T20:36:03.556 INFO:journalctl@ceph.mon.a.smithi026.stdout: block_restart_interval: 16 2023-09-23T20:36:03.556 INFO:journalctl@ceph.mon.a.smithi026.stdout: index_block_restart_interval: 1 2023-09-23T20:36:03.556 INFO:journalctl@ceph.mon.a.smithi026.stdout: metadata_block_size: 4096 2023-09-23T20:36:03.556 INFO:journalctl@ceph.mon.a.smithi026.stdout: partition_filters: 0 2023-09-23T20:36:03.556 INFO:journalctl@ceph.mon.a.smithi026.stdout: use_delta_encoding: 1 2023-09-23T20:36:03.556 INFO:journalctl@ceph.mon.a.smithi026.stdout: filter_policy: rocksdb.BuiltinBloomFilter 2023-09-23T20:36:03.556 INFO:journalctl@ceph.mon.a.smithi026.stdout: whole_key_filtering: 1 2023-09-23T20:36:03.556 INFO:journalctl@ceph.mon.a.smithi026.stdout: verify_compression: 0 2023-09-23T20:36:03.556 INFO:journalctl@ceph.mon.a.smithi026.stdout: read_amp_bytes_per_bit: 0 2023-09-23T20:36:03.556 INFO:journalctl@ceph.mon.a.smithi026.stdout: format_version: 4 2023-09-23T20:36:03.556 INFO:journalctl@ceph.mon.a.smithi026.stdout: enable_index_compression: 1 2023-09-23T20:36:03.556 INFO:journalctl@ceph.mon.a.smithi026.stdout: block_align: 0 2023-09-23T20:36:03.557 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:36:03 smithi026 ceph-mon[95839]: rocksdb: Options.write_buffer_size: 33554432 2023-09-23T20:36:03.557 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:36:03 smithi026 ceph-mon[95839]: rocksdb: Options.max_write_buffer_number: 2 2023-09-23T20:36:03.557 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:36:03 smithi026 ceph-mon[95839]: rocksdb: Options.compression: NoCompression 2023-09-23T20:36:03.557 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:36:03 smithi026 ceph-mon[95839]: rocksdb: Options.bottommost_compression: Disabled 2023-09-23T20:36:03.557 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:36:03 smithi026 ceph-mon[95839]: rocksdb: Options.prefix_extractor: nullptr 2023-09-23T20:36:03.557 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:36:03 smithi026 ceph-mon[95839]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2023-09-23T20:36:03.557 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:36:03 smithi026 ceph-mon[95839]: rocksdb: Options.num_levels: 7 2023-09-23T20:36:03.557 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:36:03 smithi026 ceph-mon[95839]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2023-09-23T20:36:03.557 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:36:03 smithi026 ceph-mon[95839]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2023-09-23T20:36:03.557 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:36:03 smithi026 ceph-mon[95839]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2023-09-23T20:36:03.557 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:36:03 smithi026 ceph-mon[95839]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2023-09-23T20:36:03.557 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:36:03 smithi026 ceph-mon[95839]: rocksdb: Options.bottommost_compression_opts.level: 32767 2023-09-23T20:36:03.557 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:36:03 smithi026 ceph-mon[95839]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2023-09-23T20:36:03.557 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:36:03 smithi026 ceph-mon[95839]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2023-09-23T20:36:03.558 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:36:03 smithi026 ceph-mon[95839]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2023-09-23T20:36:03.558 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:36:03 smithi026 ceph-mon[95839]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2023-09-23T20:36:03.558 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:36:03 smithi026 ceph-mon[95839]: rocksdb: Options.bottommost_compression_opts.enabled: false 2023-09-23T20:36:03.558 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:36:03 smithi026 ceph-mon[95839]: rocksdb: Options.compression_opts.window_bits: -14 2023-09-23T20:36:03.558 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:36:03 smithi026 ceph-mon[95839]: rocksdb: Options.compression_opts.level: 32767 2023-09-23T20:36:03.558 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:36:03 smithi026 ceph-mon[95839]: rocksdb: Options.compression_opts.strategy: 0 2023-09-23T20:36:03.558 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:36:03 smithi026 ceph-mon[95839]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2023-09-23T20:36:03.558 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:36:03 smithi026 ceph-mon[95839]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2023-09-23T20:36:03.558 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:36:03 smithi026 ceph-mon[95839]: rocksdb: Options.compression_opts.parallel_threads: 1 2023-09-23T20:36:03.558 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:36:03 smithi026 ceph-mon[95839]: rocksdb: Options.compression_opts.enabled: false 2023-09-23T20:36:03.558 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:36:03 smithi026 ceph-mon[95839]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2023-09-23T20:36:03.558 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:36:03 smithi026 ceph-mon[95839]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2023-09-23T20:36:03.559 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:36:03 smithi026 ceph-mon[95839]: rocksdb: Options.level0_stop_writes_trigger: 36 2023-09-23T20:36:03.559 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:36:03 smithi026 ceph-mon[95839]: rocksdb: Options.target_file_size_base: 67108864 2023-09-23T20:36:03.559 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:36:03 smithi026 ceph-mon[95839]: rocksdb: Options.target_file_size_multiplier: 1 2023-09-23T20:36:03.559 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:36:03 smithi026 ceph-mon[95839]: rocksdb: Options.max_bytes_for_level_base: 268435456 2023-09-23T20:36:03.559 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:36:03 smithi026 ceph-mon[95839]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2023-09-23T20:36:03.559 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:36:03 smithi026 ceph-mon[95839]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2023-09-23T20:36:03.559 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:36:03 smithi026 ceph-mon[95839]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2023-09-23T20:36:03.559 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:36:03 smithi026 ceph-mon[95839]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2023-09-23T20:36:03.559 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:36:03 smithi026 ceph-mon[95839]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2023-09-23T20:36:03.559 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:36:03 smithi026 ceph-mon[95839]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2023-09-23T20:36:03.559 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:36:03 smithi026 ceph-mon[95839]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2023-09-23T20:36:03.559 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:36:03 smithi026 ceph-mon[95839]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2023-09-23T20:36:03.560 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:36:03 smithi026 ceph-mon[95839]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2023-09-23T20:36:03.560 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:36:03 smithi026 ceph-mon[95839]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2023-09-23T20:36:03.560 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:36:03 smithi026 ceph-mon[95839]: rocksdb: Options.max_compaction_bytes: 1677721600 2023-09-23T20:36:03.560 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:36:03 smithi026 ceph-mon[95839]: rocksdb: Options.arena_block_size: 4194304 2023-09-23T20:36:03.560 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:36:03 smithi026 ceph-mon[95839]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2023-09-23T20:36:03.560 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:36:03 smithi026 ceph-mon[95839]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2023-09-23T20:36:03.560 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:36:03 smithi026 ceph-mon[95839]: rocksdb: Options.rate_limit_delay_max_milliseconds: 100 2023-09-23T20:36:03.560 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:36:03 smithi026 ceph-mon[95839]: rocksdb: Options.disable_auto_compactions: 0 2023-09-23T20:36:03.560 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:36:03 smithi026 ceph-mon[95839]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2023-09-23T20:36:03.560 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:36:03 smithi026 ceph-mon[95839]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2023-09-23T20:36:03.560 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:36:03 smithi026 ceph-mon[95839]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2023-09-23T20:36:03.560 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:36:03 smithi026 ceph-mon[95839]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2023-09-23T20:36:03.560 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:36:03 smithi026 ceph-mon[95839]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2023-09-23T20:36:03.560 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:36:03 smithi026 ceph-mon[95839]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2023-09-23T20:36:03.561 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:36:03 smithi026 ceph-mon[95839]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2023-09-23T20:36:03.561 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:36:03 smithi026 ceph-mon[95839]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2023-09-23T20:36:03.561 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:36:03 smithi026 ceph-mon[95839]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2023-09-23T20:36:03.561 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:36:03 smithi026 ceph-mon[95839]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2023-09-23T20:36:03.561 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:36:03 smithi026 ceph-mon[95839]: rocksdb: Options.table_properties_collectors: 2023-09-23T20:36:03.561 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:36:03 smithi026 ceph-mon[95839]: rocksdb: Options.inplace_update_support: 0 2023-09-23T20:36:03.561 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:36:03 smithi026 ceph-mon[95839]: rocksdb: Options.inplace_update_num_locks: 10000 2023-09-23T20:36:03.561 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:36:03 smithi026 ceph-mon[95839]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2023-09-23T20:36:03.561 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:36:03 smithi026 ceph-mon[95839]: rocksdb: Options.memtable_whole_key_filtering: 0 2023-09-23T20:36:03.561 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:36:03 smithi026 ceph-mon[95839]: rocksdb: Options.memtable_huge_page_size: 0 2023-09-23T20:36:03.561 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:36:03 smithi026 ceph-mon[95839]: rocksdb: Options.bloom_locality: 0 2023-09-23T20:36:03.561 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:36:03 smithi026 ceph-mon[95839]: rocksdb: Options.max_successive_merges: 0 2023-09-23T20:36:03.561 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:36:03 smithi026 ceph-mon[95839]: rocksdb: Options.optimize_filters_for_hits: 0 2023-09-23T20:36:03.562 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:36:03 smithi026 ceph-mon[95839]: rocksdb: Options.paranoid_file_checks: 0 2023-09-23T20:36:03.562 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:36:03 smithi026 ceph-mon[95839]: rocksdb: Options.force_consistency_checks: 1 2023-09-23T20:36:03.562 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:36:03 smithi026 ceph-mon[95839]: rocksdb: Options.report_bg_io_stats: 0 2023-09-23T20:36:03.562 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:36:03 smithi026 ceph-mon[95839]: rocksdb: Options.ttl: 2592000 2023-09-23T20:36:03.562 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:36:03 smithi026 ceph-mon[95839]: rocksdb: Options.periodic_compaction_seconds: 0 2023-09-23T20:36:03.562 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:36:03 smithi026 ceph-mon[95839]: rocksdb: Options.enable_blob_files: false 2023-09-23T20:36:03.562 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:36:03 smithi026 ceph-mon[95839]: rocksdb: Options.min_blob_size: 0 2023-09-23T20:36:03.562 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:36:03 smithi026 ceph-mon[95839]: rocksdb: Options.blob_file_size: 268435456 2023-09-23T20:36:03.562 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:36:03 smithi026 ceph-mon[95839]: rocksdb: Options.blob_compression_type: NoCompression 2023-09-23T20:36:03.562 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:36:03 smithi026 ceph-mon[95839]: rocksdb: Options.enable_blob_garbage_collection: false 2023-09-23T20:36:03.562 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:36:03 smithi026 ceph-mon[95839]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2023-09-23T20:36:03.562 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:36:03 smithi026 ceph-mon[95839]: rocksdb: [db/version_set.cc:4764] Recovered from manifest file:/var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000009 succeeded,manifest_file_number is 9, next_file_number is 11, last_sequence is 5, log_number is 5,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2023-09-23T20:36:03.562 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:36:03 smithi026 ceph-mon[95839]: rocksdb: [db/version_set.cc:4779] Column family [default] (ID 0), log number is 5 2023-09-23T20:36:03.563 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:36:03 smithi026 ceph-mon[95839]: rocksdb: [db/version_set.cc:4082] Creating manifest 13 2023-09-23T20:36:03.563 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:36:03 smithi026 ceph-mon[95839]: rocksdb: EVENT_LOG_v1 {"time_micros": 1695501363473146, "job": 1, "event": "recovery_started", "wal_files": [10]} 2023-09-23T20:36:03.563 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:36:03 smithi026 ceph-mon[95839]: rocksdb: [db/db_impl/db_impl_open.cc:845] Recovering log #10 mode 2 2023-09-23T20:36:03.563 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:36:03 smithi026 ceph-mon[95839]: rocksdb: [table/block_based/filter_policy.cc:991] Using legacy Bloom filter with high (20) bits/key. Dramatic filter space and/or accuracy improvement is available with format_version>=5. 2023-09-23T20:36:03.563 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:36:03 smithi026 ceph-mon[95839]: rocksdb: EVENT_LOG_v1 {"time_micros": 1695501363473866, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 14, "file_size": 80136, "file_checksum": "", "file_checksum_func_name": "Unknown", "table_properties": {"data_size": 78404, "index_size": 220, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 581, "raw_key_size": 10136, "raw_average_key_size": 47, "raw_value_size": 72612, "raw_average_value_size": 340, "num_data_blocks": 10, "num_entries": 213, "num_deletions": 3, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "rocksdb.BuiltinBloomFilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; ", "creation_time": 1695501363, "oldest_key_time": 0, "file_creation_time": 0, "db_id": "9fa03219-303e-487f-a76c-0f910ed72ae6", "db_session_id": "OJ5WVOBFAHVKDVWUVE1M"}} 2023-09-23T20:36:03.563 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:36:03 smithi026 ceph-mon[95839]: rocksdb: [db/version_set.cc:4082] Creating manifest 15 2023-09-23T20:36:03.563 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:36:03 smithi026 ceph-mon[95839]: rocksdb: EVENT_LOG_v1 {"time_micros": 1695501363474200, "job": 1, "event": "recovery_finished"} 2023-09-23T20:36:03.563 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:36:03 smithi026 ceph-mon[95839]: rocksdb: [file/delete_scheduler.cc:69] Deleted file /var/lib/ceph/mon/ceph-a/store.db/000010.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2023-09-23T20:36:03.563 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:36:03 smithi026 ceph-mon[95839]: rocksdb: [db/db_impl/db_impl_open.cc:1700] SstFileManager instance 0x55ff72c89180 2023-09-23T20:36:03.563 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:36:03 smithi026 ceph-mon[95839]: rocksdb: DB pointer 0x55ff72d5e000 2023-09-23T20:36:03.563 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:36:03 smithi026 ceph-mon[95839]: rocksdb: [db/db_impl/db_impl.cc:901] ------- DUMPING STATS ------- 2023-09-23T20:36:03.563 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:36:03 smithi026 ceph-mon[95839]: rocksdb: [db/db_impl/db_impl.cc:903] 2023-09-23T20:36:03.564 INFO:journalctl@ceph.mon.a.smithi026.stdout: ** DB Stats ** 2023-09-23T20:36:03.564 INFO:journalctl@ceph.mon.a.smithi026.stdout: Uptime(secs): 0.0 total, 0.0 interval 2023-09-23T20:36:03.564 INFO:journalctl@ceph.mon.a.smithi026.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2023-09-23T20:36:03.564 INFO:journalctl@ceph.mon.a.smithi026.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2023-09-23T20:36:03.564 INFO:journalctl@ceph.mon.a.smithi026.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2023-09-23T20:36:03.564 INFO:journalctl@ceph.mon.a.smithi026.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2023-09-23T20:36:03.564 INFO:journalctl@ceph.mon.a.smithi026.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 MB, 0.00 MB/s 2023-09-23T20:36:03.564 INFO:journalctl@ceph.mon.a.smithi026.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2023-09-23T20:36:03.564 INFO:journalctl@ceph.mon.a.smithi026.stdout: 2023-09-23T20:36:03.564 INFO:journalctl@ceph.mon.a.smithi026.stdout: ** Compaction Stats [default] ** 2023-09-23T20:36:03.564 INFO:journalctl@ceph.mon.a.smithi026.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-09-23T20:36:03.564 INFO:journalctl@ceph.mon.a.smithi026.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-09-23T20:36:03.564 INFO:journalctl@ceph.mon.a.smithi026.stdout: L0 2/0 80.02 KB 0.5 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 172.5 0.00 0.00 1 0.000 0 0 2023-09-23T20:36:03.565 INFO:journalctl@ceph.mon.a.smithi026.stdout: Sum 2/0 80.02 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 172.5 0.00 0.00 1 0.000 0 0 2023-09-23T20:36:03.565 INFO:journalctl@ceph.mon.a.smithi026.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 172.5 0.00 0.00 1 0.000 0 0 2023-09-23T20:36:03.565 INFO:journalctl@ceph.mon.a.smithi026.stdout: 2023-09-23T20:36:03.565 INFO:journalctl@ceph.mon.a.smithi026.stdout: ** Compaction Stats [default] ** 2023-09-23T20:36:03.565 INFO:journalctl@ceph.mon.a.smithi026.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-09-23T20:36:03.565 INFO:journalctl@ceph.mon.a.smithi026.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-09-23T20:36:03.565 INFO:journalctl@ceph.mon.a.smithi026.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 172.5 0.00 0.00 1 0.000 0 0 2023-09-23T20:36:03.565 INFO:journalctl@ceph.mon.a.smithi026.stdout: Uptime(secs): 0.0 total, 0.0 interval 2023-09-23T20:36:03.565 INFO:journalctl@ceph.mon.a.smithi026.stdout: Flush(GB): cumulative 0.000, interval 0.000 2023-09-23T20:36:03.565 INFO:journalctl@ceph.mon.a.smithi026.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2023-09-23T20:36:03.565 INFO:journalctl@ceph.mon.a.smithi026.stdout: AddFile(Total Files): cumulative 0, interval 0 2023-09-23T20:36:03.565 INFO:journalctl@ceph.mon.a.smithi026.stdout: AddFile(L0 Files): cumulative 0, interval 0 2023-09-23T20:36:03.565 INFO:journalctl@ceph.mon.a.smithi026.stdout: AddFile(Keys): cumulative 0, interval 0 2023-09-23T20:36:03.566 INFO:journalctl@ceph.mon.a.smithi026.stdout: Cumulative compaction: 0.00 GB write, 25.36 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-09-23T20:36:03.566 INFO:journalctl@ceph.mon.a.smithi026.stdout: Interval compaction: 0.00 GB write, 25.36 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-09-23T20:36:03.566 INFO:journalctl@ceph.mon.a.smithi026.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2023-09-23T20:36:03.566 INFO:journalctl@ceph.mon.a.smithi026.stdout: 2023-09-23T20:36:03.566 INFO:journalctl@ceph.mon.a.smithi026.stdout: ** File Read Latency Histogram By Level [default] ** 2023-09-23T20:36:03.566 INFO:journalctl@ceph.mon.a.smithi026.stdout: 2023-09-23T20:36:03.566 INFO:journalctl@ceph.mon.a.smithi026.stdout: ** Compaction Stats [default] ** 2023-09-23T20:36:03.566 INFO:journalctl@ceph.mon.a.smithi026.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-09-23T20:36:03.566 INFO:journalctl@ceph.mon.a.smithi026.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-09-23T20:36:03.566 INFO:journalctl@ceph.mon.a.smithi026.stdout: L0 2/0 80.02 KB 0.5 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 172.5 0.00 0.00 1 0.000 0 0 2023-09-23T20:36:03.566 INFO:journalctl@ceph.mon.a.smithi026.stdout: Sum 2/0 80.02 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 172.5 0.00 0.00 1 0.000 0 0 2023-09-23T20:36:03.566 INFO:journalctl@ceph.mon.a.smithi026.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2023-09-23T20:36:03.566 INFO:journalctl@ceph.mon.a.smithi026.stdout: 2023-09-23T20:36:03.567 INFO:journalctl@ceph.mon.a.smithi026.stdout: ** Compaction Stats [default] ** 2023-09-23T20:36:03.567 INFO:journalctl@ceph.mon.a.smithi026.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-09-23T20:36:03.567 INFO:journalctl@ceph.mon.a.smithi026.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-09-23T20:36:03.567 INFO:journalctl@ceph.mon.a.smithi026.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 172.5 0.00 0.00 1 0.000 0 0 2023-09-23T20:36:03.567 INFO:journalctl@ceph.mon.a.smithi026.stdout: Uptime(secs): 0.0 total, 0.0 interval 2023-09-23T20:36:03.567 INFO:journalctl@ceph.mon.a.smithi026.stdout: Flush(GB): cumulative 0.000, interval 0.000 2023-09-23T20:36:03.567 INFO:journalctl@ceph.mon.a.smithi026.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2023-09-23T20:36:03.567 INFO:journalctl@ceph.mon.a.smithi026.stdout: AddFile(Total Files): cumulative 0, interval 0 2023-09-23T20:36:03.567 INFO:journalctl@ceph.mon.a.smithi026.stdout: AddFile(L0 Files): cumulative 0, interval 0 2023-09-23T20:36:03.567 INFO:journalctl@ceph.mon.a.smithi026.stdout: AddFile(Keys): cumulative 0, interval 0 2023-09-23T20:36:03.567 INFO:journalctl@ceph.mon.a.smithi026.stdout: Cumulative compaction: 0.00 GB write, 25.11 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-09-23T20:36:03.567 INFO:journalctl@ceph.mon.a.smithi026.stdout: Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-09-23T20:36:03.568 INFO:journalctl@ceph.mon.a.smithi026.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2023-09-23T20:36:03.568 INFO:journalctl@ceph.mon.a.smithi026.stdout: 2023-09-23T20:36:03.568 INFO:journalctl@ceph.mon.a.smithi026.stdout: ** File Read Latency Histogram By Level [default] ** 2023-09-23T20:36:03.568 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:36:03 smithi026 ceph-mon[95839]: starting mon.a rank 0 at public addrs [v2:172.21.15.26:3300/0,v1:172.21.15.26:6789/0] at bind addrs [v2:172.21.15.26:3300/0,v1:172.21.15.26:6789/0] mon_data /var/lib/ceph/mon/ceph-a fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 2023-09-23T20:36:03.568 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:36:03 smithi026 ceph-mon[95839]: mon.a@-1(???) e1 preinit fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 2023-09-23T20:36:03.568 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:36:03 smithi026 ceph-mon[95839]: mon.a@-1(???).mds e1 new map 2023-09-23T20:36:03.568 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:36:03 smithi026 ceph-mon[95839]: mon.a@-1(???).mds e1 print_map 2023-09-23T20:36:03.568 INFO:journalctl@ceph.mon.a.smithi026.stdout: e1 2023-09-23T20:36:03.568 INFO:journalctl@ceph.mon.a.smithi026.stdout: enable_multiple, ever_enabled_multiple: 1,1 2023-09-23T20:36:03.568 INFO:journalctl@ceph.mon.a.smithi026.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2023-09-23T20:36:03.568 INFO:journalctl@ceph.mon.a.smithi026.stdout: legacy client fscid: -1 2023-09-23T20:36:03.568 INFO:journalctl@ceph.mon.a.smithi026.stdout: 2023-09-23T20:36:03.568 INFO:journalctl@ceph.mon.a.smithi026.stdout: No filesystems configured 2023-09-23T20:36:03.569 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:36:03 smithi026 ceph-mon[95839]: mon.a@-1(???).osd e1 crush map has features 3314932999778484224, adjusting msgr requires 2023-09-23T20:36:03.569 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:36:03 smithi026 ceph-mon[95839]: mon.a@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2023-09-23T20:36:03.569 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:36:03 smithi026 ceph-mon[95839]: mon.a@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2023-09-23T20:36:03.569 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:36:03 smithi026 ceph-mon[95839]: mon.a@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2023-09-23T20:36:03.569 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:36:03 smithi026 ceph-mon[95839]: mon.a@-1(???).paxosservice(auth 1..2) refresh upgraded, format 0 -> 3 2023-09-23T20:36:03.569 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:36:03 smithi026 systemd[1]: Started Ceph mon.a for b3f8e4ec-5a50-11ee-8db4-212e2dc638e7. 2023-09-23T20:36:03.569 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:36:03 smithi026 ceph-mon[95839]: mon.a is new leader, mons a in quorum (ranks 0) 2023-09-23T20:36:03.569 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:36:03 smithi026 ceph-mon[95839]: monmap e1: 1 mons at {a=[v2:172.21.15.26:3300/0,v1:172.21.15.26:6789/0]} removed_ranks: {} 2023-09-23T20:36:03.569 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:36:03 smithi026 ceph-mon[95839]: fsmap 2023-09-23T20:36:03.569 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:36:03 smithi026 ceph-mon[95839]: osdmap e1: 0 total, 0 up, 0 in 2023-09-23T20:36:03.569 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:36:03 smithi026 ceph-mon[95839]: mgrmap e1: no daemons active 2023-09-23T20:36:04.543 INFO:teuthology.orchestra.run.smithi026.stdout:Wrote config to /etc/ceph/ceph.conf 2023-09-23T20:36:04.544 INFO:teuthology.orchestra.run.smithi026.stdout:Wrote keyring to /etc/ceph/ceph.client.admin.keyring 2023-09-23T20:36:04.544 INFO:teuthology.orchestra.run.smithi026.stdout:Creating mgr... 2023-09-23T20:36:04.544 INFO:teuthology.orchestra.run.smithi026.stdout:Verifying port 9283 ... 2023-09-23T20:36:04.676 INFO:teuthology.orchestra.run.smithi026.stdout:Non-zero exit code 1 from systemctl reset-failed ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7@mgr.y 2023-09-23T20:36:04.676 INFO:teuthology.orchestra.run.smithi026.stdout:systemctl: stderr Failed to reset failed state of unit ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7@mgr.y.service: Unit ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7@mgr.y.service not loaded. 2023-09-23T20:36:04.773 INFO:teuthology.orchestra.run.smithi026.stdout:systemctl: stderr Created symlink /etc/systemd/system/ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7.target.wants/ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7@mgr.y.service → /etc/systemd/system/ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7@.service. 2023-09-23T20:36:04.882 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:36:04 smithi026 systemd[1]: Starting Ceph mgr.y for b3f8e4ec-5a50-11ee-8db4-212e2dc638e7... 2023-09-23T20:36:05.371 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:36:05 smithi026 ceph-mon[95839]: from='client.? 172.21.15.26:0/1312367927' entity='client.admin' 2023-09-23T20:36:05.371 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:36:05 smithi026 podman[96070]: 2023-09-23T20:36:05.371 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:36:05 smithi026 bash[96070]: e022dce09ba0e68b67ae311757575a1f54e402a2729cf29f8b46ddca67f502bb 2023-09-23T20:36:05.385 INFO:teuthology.orchestra.run.smithi026.stdout:Non-zero exit code 1 from systemctl is-enabled firewalld.service 2023-09-23T20:36:05.385 INFO:teuthology.orchestra.run.smithi026.stdout:systemctl: stdout disabled 2023-09-23T20:36:05.394 INFO:teuthology.orchestra.run.smithi026.stdout:Non-zero exit code 3 from systemctl is-active firewalld.service 2023-09-23T20:36:05.394 INFO:teuthology.orchestra.run.smithi026.stdout:systemctl: stdout inactive 2023-09-23T20:36:05.394 INFO:teuthology.orchestra.run.smithi026.stdout:firewalld.service is not enabled 2023-09-23T20:36:05.394 INFO:teuthology.orchestra.run.smithi026.stdout:Not possible to enable service . firewalld.service is not available 2023-09-23T20:36:05.402 INFO:teuthology.orchestra.run.smithi026.stdout:Non-zero exit code 1 from systemctl is-enabled firewalld.service 2023-09-23T20:36:05.403 INFO:teuthology.orchestra.run.smithi026.stdout:systemctl: stdout disabled 2023-09-23T20:36:05.410 INFO:teuthology.orchestra.run.smithi026.stdout:Non-zero exit code 3 from systemctl is-active firewalld.service 2023-09-23T20:36:05.410 INFO:teuthology.orchestra.run.smithi026.stdout:systemctl: stdout inactive 2023-09-23T20:36:05.411 INFO:teuthology.orchestra.run.smithi026.stdout:firewalld.service is not enabled 2023-09-23T20:36:05.411 INFO:teuthology.orchestra.run.smithi026.stdout:Not possible to open ports <[9283]>. firewalld.service is not available 2023-09-23T20:36:05.411 INFO:teuthology.orchestra.run.smithi026.stdout:Waiting for mgr to start... 2023-09-23T20:36:05.411 INFO:teuthology.orchestra.run.smithi026.stdout:Waiting for mgr... 2023-09-23T20:36:05.632 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:36:05 smithi026 systemd[1]: Started Ceph mgr.y for b3f8e4ec-5a50-11ee-8db4-212e2dc638e7. 2023-09-23T20:36:05.632 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:36:05 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[96086]: 2023-09-23T20:36:05.452+0000 7f93b7a881c0 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2023-09-23T20:36:05.632 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:36:05 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[96086]: 2023-09-23T20:36:05.546+0000 7f93b7a881c0 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-09-23T20:36:05.981 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:36:05 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[96086]: 2023-09-23T20:36:05.656+0000 7f93b7a881c0 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-09-23T20:36:05.982 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:36:05 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[96086]: 2023-09-23T20:36:05.769+0000 7f93b7a881c0 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-09-23T20:36:05.982 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:36:05 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[96086]: 2023-09-23T20:36:05.976+0000 7f93b7a881c0 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-09-23T20:36:06.253 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:36:06 smithi026 ceph-mon[95839]: from='client.? 172.21.15.26:0/2140532230' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2023-09-23T20:36:06.464 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout 2023-09-23T20:36:06.464 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout { 2023-09-23T20:36:06.464 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "fsid": "b3f8e4ec-5a50-11ee-8db4-212e2dc638e7", 2023-09-23T20:36:06.464 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "health": { 2023-09-23T20:36:06.464 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "status": "HEALTH_OK", 2023-09-23T20:36:06.464 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "checks": {}, 2023-09-23T20:36:06.464 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "mutes": [] 2023-09-23T20:36:06.465 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout }, 2023-09-23T20:36:06.465 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "election_epoch": 5, 2023-09-23T20:36:06.465 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "quorum": [ 2023-09-23T20:36:06.465 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout 0 2023-09-23T20:36:06.465 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout ], 2023-09-23T20:36:06.465 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "quorum_names": [ 2023-09-23T20:36:06.465 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "a" 2023-09-23T20:36:06.465 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout ], 2023-09-23T20:36:06.465 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "quorum_age": 2, 2023-09-23T20:36:06.465 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "monmap": { 2023-09-23T20:36:06.465 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "epoch": 1, 2023-09-23T20:36:06.465 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "min_mon_release_name": "quincy", 2023-09-23T20:36:06.466 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "num_mons": 1 2023-09-23T20:36:06.466 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout }, 2023-09-23T20:36:06.466 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "osdmap": { 2023-09-23T20:36:06.466 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "epoch": 1, 2023-09-23T20:36:06.466 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "num_osds": 0, 2023-09-23T20:36:06.466 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "num_up_osds": 0, 2023-09-23T20:36:06.466 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "osd_up_since": 0, 2023-09-23T20:36:06.466 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "num_in_osds": 0, 2023-09-23T20:36:06.466 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "osd_in_since": 0, 2023-09-23T20:36:06.466 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "num_remapped_pgs": 0 2023-09-23T20:36:06.467 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout }, 2023-09-23T20:36:06.467 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "pgmap": { 2023-09-23T20:36:06.467 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "pgs_by_state": [], 2023-09-23T20:36:06.467 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "num_pgs": 0, 2023-09-23T20:36:06.467 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "num_pools": 0, 2023-09-23T20:36:06.467 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "num_objects": 0, 2023-09-23T20:36:06.467 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "data_bytes": 0, 2023-09-23T20:36:06.467 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "bytes_used": 0, 2023-09-23T20:36:06.467 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "bytes_avail": 0, 2023-09-23T20:36:06.467 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "bytes_total": 0 2023-09-23T20:36:06.467 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout }, 2023-09-23T20:36:06.467 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "fsmap": { 2023-09-23T20:36:06.468 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "epoch": 1, 2023-09-23T20:36:06.468 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "by_rank": [], 2023-09-23T20:36:06.468 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "up:standby": 0 2023-09-23T20:36:06.468 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout }, 2023-09-23T20:36:06.468 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "mgrmap": { 2023-09-23T20:36:06.468 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "available": false, 2023-09-23T20:36:06.468 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "num_standbys": 0, 2023-09-23T20:36:06.468 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "modules": [ 2023-09-23T20:36:06.468 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "iostat", 2023-09-23T20:36:06.468 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "nfs", 2023-09-23T20:36:06.468 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "restful" 2023-09-23T20:36:06.469 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout ], 2023-09-23T20:36:06.469 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "services": {} 2023-09-23T20:36:06.469 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout }, 2023-09-23T20:36:06.469 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "servicemap": { 2023-09-23T20:36:06.469 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "epoch": 1, 2023-09-23T20:36:06.469 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "modified": "2023-09-23T20:35:58.920166+0000", 2023-09-23T20:36:06.469 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "services": {} 2023-09-23T20:36:06.469 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout }, 2023-09-23T20:36:06.469 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "progress_events": {} 2023-09-23T20:36:06.469 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout } 2023-09-23T20:36:06.469 INFO:teuthology.orchestra.run.smithi026.stdout:mgr not available, waiting (1/15)... 2023-09-23T20:36:06.515 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:36:06 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[96086]: 2023-09-23T20:36:06.383+0000 7f93b7a881c0 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-09-23T20:36:06.516 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:36:06 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[96086]: 2023-09-23T20:36:06.490+0000 7f93b7a881c0 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-09-23T20:36:06.883 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:36:06 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[96086]: 2023-09-23T20:36:06.639+0000 7f93b7a881c0 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-09-23T20:36:06.883 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:36:06 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[96086]: 2023-09-23T20:36:06.751+0000 7f93b7a881c0 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-09-23T20:36:07.632 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:36:07 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[96086]: 2023-09-23T20:36:07.256+0000 7f93b7a881c0 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-09-23T20:36:08.381 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:36:07 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[96086]: 2023-09-23T20:36:07.948+0000 7f93b7a881c0 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-09-23T20:36:08.382 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:36:08 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[96086]: 2023-09-23T20:36:08.138+0000 7f93b7a881c0 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-09-23T20:36:09.381 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:36:09 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[96086]: 2023-09-23T20:36:09.117+0000 7f93b7a881c0 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-09-23T20:36:09.382 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:36:09 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[96086]: 2023-09-23T20:36:09.229+0000 7f93b7a881c0 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2023-09-23T20:36:09.509 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout 2023-09-23T20:36:09.509 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout { 2023-09-23T20:36:09.509 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "fsid": "b3f8e4ec-5a50-11ee-8db4-212e2dc638e7", 2023-09-23T20:36:09.509 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "health": { 2023-09-23T20:36:09.509 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "status": "HEALTH_OK", 2023-09-23T20:36:09.509 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "checks": {}, 2023-09-23T20:36:09.509 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "mutes": [] 2023-09-23T20:36:09.509 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout }, 2023-09-23T20:36:09.510 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "election_epoch": 5, 2023-09-23T20:36:09.510 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "quorum": [ 2023-09-23T20:36:09.510 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout 0 2023-09-23T20:36:09.510 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout ], 2023-09-23T20:36:09.510 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "quorum_names": [ 2023-09-23T20:36:09.510 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "a" 2023-09-23T20:36:09.510 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout ], 2023-09-23T20:36:09.510 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "quorum_age": 5, 2023-09-23T20:36:09.510 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "monmap": { 2023-09-23T20:36:09.510 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "epoch": 1, 2023-09-23T20:36:09.510 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "min_mon_release_name": "quincy", 2023-09-23T20:36:09.510 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "num_mons": 1 2023-09-23T20:36:09.511 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout }, 2023-09-23T20:36:09.511 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "osdmap": { 2023-09-23T20:36:09.511 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "epoch": 1, 2023-09-23T20:36:09.511 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "num_osds": 0, 2023-09-23T20:36:09.511 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "num_up_osds": 0, 2023-09-23T20:36:09.511 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "osd_up_since": 0, 2023-09-23T20:36:09.515 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "num_in_osds": 0, 2023-09-23T20:36:09.515 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "osd_in_since": 0, 2023-09-23T20:36:09.515 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "num_remapped_pgs": 0 2023-09-23T20:36:09.515 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout }, 2023-09-23T20:36:09.515 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "pgmap": { 2023-09-23T20:36:09.515 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "pgs_by_state": [], 2023-09-23T20:36:09.515 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "num_pgs": 0, 2023-09-23T20:36:09.515 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "num_pools": 0, 2023-09-23T20:36:09.515 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "num_objects": 0, 2023-09-23T20:36:09.515 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "data_bytes": 0, 2023-09-23T20:36:09.515 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "bytes_used": 0, 2023-09-23T20:36:09.515 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "bytes_avail": 0, 2023-09-23T20:36:09.516 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "bytes_total": 0 2023-09-23T20:36:09.517 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout }, 2023-09-23T20:36:09.517 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "fsmap": { 2023-09-23T20:36:09.517 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "epoch": 1, 2023-09-23T20:36:09.517 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "by_rank": [], 2023-09-23T20:36:09.517 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "up:standby": 0 2023-09-23T20:36:09.517 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout }, 2023-09-23T20:36:09.517 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "mgrmap": { 2023-09-23T20:36:09.517 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "available": false, 2023-09-23T20:36:09.518 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "num_standbys": 0, 2023-09-23T20:36:09.518 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "modules": [ 2023-09-23T20:36:09.518 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "iostat", 2023-09-23T20:36:09.518 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "nfs", 2023-09-23T20:36:09.518 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "restful" 2023-09-23T20:36:09.518 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout ], 2023-09-23T20:36:09.518 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "services": {} 2023-09-23T20:36:09.518 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout }, 2023-09-23T20:36:09.518 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "servicemap": { 2023-09-23T20:36:09.518 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "epoch": 1, 2023-09-23T20:36:09.519 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "modified": "2023-09-23T20:35:58.920166+0000", 2023-09-23T20:36:09.519 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "services": {} 2023-09-23T20:36:09.519 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout }, 2023-09-23T20:36:09.519 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "progress_events": {} 2023-09-23T20:36:09.519 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout } 2023-09-23T20:36:09.519 INFO:teuthology.orchestra.run.smithi026.stdout:mgr not available, waiting (2/15)... 2023-09-23T20:36:09.787 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:36:09 smithi026 ceph-mon[95839]: from='client.? 172.21.15.26:0/357241718' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2023-09-23T20:36:09.787 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:36:09 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[96086]: 2023-09-23T20:36:09.444+0000 7f93b7a881c0 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-09-23T20:36:10.132 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:36:09 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[96086]: 2023-09-23T20:36:09.785+0000 7f93b7a881c0 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2023-09-23T20:36:10.132 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:36:09 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[96086]: 2023-09-23T20:36:09.888+0000 7f93b7a881c0 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-09-23T20:36:10.534 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:36:10 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[96086]: 2023-09-23T20:36:10.216+0000 7f93b7a881c0 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-09-23T20:36:10.881 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:36:10 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[96086]: 2023-09-23T20:36:10.533+0000 7f93b7a881c0 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-09-23T20:36:11.632 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:36:11 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[96086]: 2023-09-23T20:36:11.355+0000 7f93b7a881c0 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-09-23T20:36:11.632 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:36:11 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[96086]: 2023-09-23T20:36:11.511+0000 7f93b7a881c0 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-09-23T20:36:12.131 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:36:11 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[96086]: 2023-09-23T20:36:11.753+0000 7f93b7a881c0 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-09-23T20:36:12.132 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:36:11 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[96086]: 2023-09-23T20:36:11.889+0000 7f93b7a881c0 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-09-23T20:36:12.465 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:36:12 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[96086]: 2023-09-23T20:36:12.158+0000 7f93b7a881c0 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-09-23T20:36:13.631 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:36:13 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[96086]: 2023-09-23T20:36:13.228+0000 7f93b7a881c0 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-09-23T20:36:13.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:36:13 smithi026 ceph-mon[95839]: from='client.? 172.21.15.26:0/915030281' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2023-09-23T20:36:13.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:36:13 smithi026 ceph-mon[95839]: Activating manager daemon y 2023-09-23T20:36:13.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:36:13 smithi026 ceph-mon[95839]: mgrmap e2: y(active, starting, since 0.00312419s) 2023-09-23T20:36:13.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:36:13 smithi026 ceph-mon[95839]: from='mgr.14100 172.21.15.26:0/193456583' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-09-23T20:36:13.633 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:36:13 smithi026 ceph-mon[95839]: from='mgr.14100 172.21.15.26:0/193456583' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-09-23T20:36:13.633 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:36:13 smithi026 ceph-mon[95839]: from='mgr.14100 172.21.15.26:0/193456583' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-09-23T20:36:13.633 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:36:13 smithi026 ceph-mon[95839]: from='mgr.14100 172.21.15.26:0/193456583' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-23T20:36:13.633 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:36:13 smithi026 ceph-mon[95839]: from='mgr.14100 172.21.15.26:0/193456583' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-09-23T20:36:13.633 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:36:13 smithi026 ceph-mon[95839]: Manager daemon y is now available 2023-09-23T20:36:14.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:36:14 smithi026 ceph-mon[95839]: from='mgr.14100 172.21.15.26:0/193456583' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-09-23T20:36:14.383 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:36:14 smithi026 ceph-mon[95839]: from='mgr.14100 172.21.15.26:0/193456583' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-09-23T20:36:14.383 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:36:14 smithi026 ceph-mon[95839]: from='mgr.14100 172.21.15.26:0/193456583' entity='mgr.y' 2023-09-23T20:36:14.383 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:36:14 smithi026 ceph-mon[95839]: from='mgr.14100 172.21.15.26:0/193456583' entity='mgr.y' 2023-09-23T20:36:14.383 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:36:14 smithi026 ceph-mon[95839]: from='mgr.14100 172.21.15.26:0/193456583' entity='mgr.y' 2023-09-23T20:36:14.383 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:36:14 smithi026 ceph-mon[95839]: mgrmap e3: y(active, since 1.007s) 2023-09-23T20:36:15.870 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout 2023-09-23T20:36:15.870 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout { 2023-09-23T20:36:15.870 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "fsid": "b3f8e4ec-5a50-11ee-8db4-212e2dc638e7", 2023-09-23T20:36:15.870 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "health": { 2023-09-23T20:36:15.870 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "status": "HEALTH_OK", 2023-09-23T20:36:15.870 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "checks": {}, 2023-09-23T20:36:15.871 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "mutes": [] 2023-09-23T20:36:15.871 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout }, 2023-09-23T20:36:15.871 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "election_epoch": 5, 2023-09-23T20:36:15.871 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "quorum": [ 2023-09-23T20:36:15.871 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout 0 2023-09-23T20:36:15.871 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout ], 2023-09-23T20:36:15.871 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "quorum_names": [ 2023-09-23T20:36:15.871 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "a" 2023-09-23T20:36:15.871 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout ], 2023-09-23T20:36:15.871 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "quorum_age": 9, 2023-09-23T20:36:15.871 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "monmap": { 2023-09-23T20:36:15.871 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "epoch": 1, 2023-09-23T20:36:15.872 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "min_mon_release_name": "quincy", 2023-09-23T20:36:15.872 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "num_mons": 1 2023-09-23T20:36:15.872 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout }, 2023-09-23T20:36:15.872 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "osdmap": { 2023-09-23T20:36:15.872 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "epoch": 1, 2023-09-23T20:36:15.872 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "num_osds": 0, 2023-09-23T20:36:15.872 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "num_up_osds": 0, 2023-09-23T20:36:15.872 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "osd_up_since": 0, 2023-09-23T20:36:15.872 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "num_in_osds": 0, 2023-09-23T20:36:15.872 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "osd_in_since": 0, 2023-09-23T20:36:15.872 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "num_remapped_pgs": 0 2023-09-23T20:36:15.872 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout }, 2023-09-23T20:36:15.872 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "pgmap": { 2023-09-23T20:36:15.873 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "pgs_by_state": [], 2023-09-23T20:36:15.873 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "num_pgs": 0, 2023-09-23T20:36:15.873 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "num_pools": 0, 2023-09-23T20:36:15.873 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "num_objects": 0, 2023-09-23T20:36:15.873 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "data_bytes": 0, 2023-09-23T20:36:15.873 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "bytes_used": 0, 2023-09-23T20:36:15.873 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "bytes_avail": 0, 2023-09-23T20:36:15.874 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "bytes_total": 0 2023-09-23T20:36:15.874 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout }, 2023-09-23T20:36:15.874 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "fsmap": { 2023-09-23T20:36:15.874 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "epoch": 1, 2023-09-23T20:36:15.874 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "by_rank": [], 2023-09-23T20:36:15.874 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "up:standby": 0 2023-09-23T20:36:15.874 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout }, 2023-09-23T20:36:15.874 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "mgrmap": { 2023-09-23T20:36:15.874 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "available": false, 2023-09-23T20:36:15.874 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "num_standbys": 0, 2023-09-23T20:36:15.874 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "modules": [ 2023-09-23T20:36:15.874 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "iostat", 2023-09-23T20:36:15.875 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "nfs", 2023-09-23T20:36:15.876 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "restful" 2023-09-23T20:36:15.876 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout ], 2023-09-23T20:36:15.876 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "services": {} 2023-09-23T20:36:15.876 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout }, 2023-09-23T20:36:15.876 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "servicemap": { 2023-09-23T20:36:15.876 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "epoch": 1, 2023-09-23T20:36:15.876 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "modified": "2023-09-23T20:35:58.920166+0000", 2023-09-23T20:36:15.876 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "services": {} 2023-09-23T20:36:15.877 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout }, 2023-09-23T20:36:15.877 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "progress_events": {} 2023-09-23T20:36:15.877 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout } 2023-09-23T20:36:15.877 INFO:teuthology.orchestra.run.smithi026.stdout:mgr not available, waiting (3/15)... 2023-09-23T20:36:16.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:36:16 smithi026 ceph-mon[95839]: mgrmap e4: y(active, since 2s) 2023-09-23T20:36:20.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:36:20 smithi026 ceph-mon[95839]: from='client.? 172.21.15.26:0/3515426255' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2023-09-23T20:36:20.925 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout 2023-09-23T20:36:20.925 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout { 2023-09-23T20:36:20.925 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "fsid": "b3f8e4ec-5a50-11ee-8db4-212e2dc638e7", 2023-09-23T20:36:20.926 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "health": { 2023-09-23T20:36:20.926 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "status": "HEALTH_OK", 2023-09-23T20:36:20.926 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "checks": {}, 2023-09-23T20:36:20.926 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "mutes": [] 2023-09-23T20:36:20.926 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout }, 2023-09-23T20:36:20.926 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "election_epoch": 5, 2023-09-23T20:36:20.926 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "quorum": [ 2023-09-23T20:36:20.926 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout 0 2023-09-23T20:36:20.926 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout ], 2023-09-23T20:36:20.926 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "quorum_names": [ 2023-09-23T20:36:20.926 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "a" 2023-09-23T20:36:20.927 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout ], 2023-09-23T20:36:20.927 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "quorum_age": 16, 2023-09-23T20:36:20.927 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "monmap": { 2023-09-23T20:36:20.927 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "epoch": 1, 2023-09-23T20:36:20.927 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "min_mon_release_name": "quincy", 2023-09-23T20:36:20.927 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "num_mons": 1 2023-09-23T20:36:20.927 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout }, 2023-09-23T20:36:20.927 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "osdmap": { 2023-09-23T20:36:20.927 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "epoch": 1, 2023-09-23T20:36:20.927 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "num_osds": 0, 2023-09-23T20:36:20.927 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "num_up_osds": 0, 2023-09-23T20:36:20.933 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "osd_up_since": 0, 2023-09-23T20:36:20.933 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "num_in_osds": 0, 2023-09-23T20:36:20.933 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "osd_in_since": 0, 2023-09-23T20:36:20.933 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "num_remapped_pgs": 0 2023-09-23T20:36:20.933 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout }, 2023-09-23T20:36:20.933 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "pgmap": { 2023-09-23T20:36:20.934 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "pgs_by_state": [], 2023-09-23T20:36:20.934 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "num_pgs": 0, 2023-09-23T20:36:20.934 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "num_pools": 0, 2023-09-23T20:36:20.934 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "num_objects": 0, 2023-09-23T20:36:20.934 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "data_bytes": 0, 2023-09-23T20:36:20.934 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "bytes_used": 0, 2023-09-23T20:36:20.934 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "bytes_avail": 0, 2023-09-23T20:36:20.934 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "bytes_total": 0 2023-09-23T20:36:20.934 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout }, 2023-09-23T20:36:20.934 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "fsmap": { 2023-09-23T20:36:20.935 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "epoch": 1, 2023-09-23T20:36:20.935 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "by_rank": [], 2023-09-23T20:36:20.935 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "up:standby": 0 2023-09-23T20:36:20.935 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout }, 2023-09-23T20:36:20.935 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "mgrmap": { 2023-09-23T20:36:20.935 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "available": true, 2023-09-23T20:36:20.935 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "num_standbys": 0, 2023-09-23T20:36:20.935 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "modules": [ 2023-09-23T20:36:20.935 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "iostat", 2023-09-23T20:36:20.935 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "nfs", 2023-09-23T20:36:20.935 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "restful" 2023-09-23T20:36:20.935 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout ], 2023-09-23T20:36:20.935 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "services": {} 2023-09-23T20:36:20.936 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout }, 2023-09-23T20:36:20.936 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "servicemap": { 2023-09-23T20:36:20.936 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "epoch": 1, 2023-09-23T20:36:20.936 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "modified": "2023-09-23T20:35:58.920166+0000", 2023-09-23T20:36:20.936 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "services": {} 2023-09-23T20:36:20.936 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout }, 2023-09-23T20:36:20.936 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "progress_events": {} 2023-09-23T20:36:20.936 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout } 2023-09-23T20:36:20.936 INFO:teuthology.orchestra.run.smithi026.stdout:mgr is available 2023-09-23T20:36:22.130 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:36:21 smithi026 ceph-mon[95839]: from='client.? 172.21.15.26:0/1952827572' entity='client.admin' cmd=[{"prefix": "config assimilate-conf"}]: dispatch 2023-09-23T20:36:22.171 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout 2023-09-23T20:36:22.171 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout [global] 2023-09-23T20:36:22.171 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout fsid = b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 2023-09-23T20:36:22.172 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout mon_osd_allow_pg_remap = true 2023-09-23T20:36:22.172 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout mon_osd_allow_primary_affinity = true 2023-09-23T20:36:22.172 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout mon_warn_on_no_sortbitwise = false 2023-09-23T20:36:22.172 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout osd_crush_chooseleaf_type = 0 2023-09-23T20:36:22.172 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout 2023-09-23T20:36:22.172 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout [mgr] 2023-09-23T20:36:22.172 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout mgr/telemetry/nag = false 2023-09-23T20:36:22.172 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout 2023-09-23T20:36:22.172 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout [osd] 2023-09-23T20:36:22.172 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout osd_map_max_advance = 10 2023-09-23T20:36:22.172 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout osd_sloppy_crc = true 2023-09-23T20:36:22.172 INFO:teuthology.orchestra.run.smithi026.stdout:Enabling cephadm module... 2023-09-23T20:36:23.432 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:36:23 smithi026 ceph-mon[95839]: from='client.? 172.21.15.26:0/3669344999' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "cephadm"}]: dispatch 2023-09-23T20:36:23.434 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:36:23 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[96086]: ignoring --setuser ceph since I am not root 2023-09-23T20:36:23.434 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:36:23 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[96086]: ignoring --setgroup ceph since I am not root 2023-09-23T20:36:23.434 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:36:23 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[96086]: 2023-09-23T20:36:23.338+0000 7f3bf2a151c0 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2023-09-23T20:36:23.724 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:36:23 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[96086]: 2023-09-23T20:36:23.431+0000 7f3bf2a151c0 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-09-23T20:36:23.724 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:36:23 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[96086]: 2023-09-23T20:36:23.544+0000 7f3bf2a151c0 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-09-23T20:36:23.724 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:36:23 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[96086]: 2023-09-23T20:36:23.657+0000 7f3bf2a151c0 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-09-23T20:36:24.131 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:36:23 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[96086]: 2023-09-23T20:36:23.864+0000 7f3bf2a151c0 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-09-23T20:36:24.531 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:36:24 smithi026 ceph-mon[95839]: from='client.? 172.21.15.26:0/3669344999' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "cephadm"}]': finished 2023-09-23T20:36:24.531 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:36:24 smithi026 ceph-mon[95839]: mgrmap e5: y(active, since 9s) 2023-09-23T20:36:24.531 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:36:24 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[96086]: 2023-09-23T20:36:24.275+0000 7f3bf2a151c0 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-09-23T20:36:24.531 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:36:24 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[96086]: 2023-09-23T20:36:24.380+0000 7f3bf2a151c0 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-09-23T20:36:24.882 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:36:24 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[96086]: 2023-09-23T20:36:24.529+0000 7f3bf2a151c0 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-09-23T20:36:24.882 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:36:24 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[96086]: 2023-09-23T20:36:24.647+0000 7f3bf2a151c0 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-09-23T20:36:24.961 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout { 2023-09-23T20:36:24.961 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "epoch": 5, 2023-09-23T20:36:24.961 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "available": true, 2023-09-23T20:36:24.961 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "active_name": "y", 2023-09-23T20:36:24.962 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "num_standby": 0 2023-09-23T20:36:24.962 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout } 2023-09-23T20:36:24.962 INFO:teuthology.orchestra.run.smithi026.stdout:Waiting for the mgr to restart... 2023-09-23T20:36:24.962 INFO:teuthology.orchestra.run.smithi026.stdout:Waiting for mgr epoch 5... 2023-09-23T20:36:25.381 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:36:25 smithi026 ceph-mon[95839]: from='client.? 172.21.15.26:0/1038559831' entity='client.admin' cmd=[{"prefix": "mgr stat"}]: dispatch 2023-09-23T20:36:25.382 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:36:25 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[96086]: 2023-09-23T20:36:25.159+0000 7f3bf2a151c0 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-09-23T20:36:26.131 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:36:25 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[96086]: 2023-09-23T20:36:25.855+0000 7f3bf2a151c0 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-09-23T20:36:26.132 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:36:26 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[96086]: 2023-09-23T20:36:26.044+0000 7f3bf2a151c0 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-09-23T20:36:27.338 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:36:27 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[96086]: 2023-09-23T20:36:27.014+0000 7f3bf2a151c0 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-09-23T20:36:27.338 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:36:27 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[96086]: 2023-09-23T20:36:27.123+0000 7f3bf2a151c0 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2023-09-23T20:36:28.131 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:36:27 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[96086]: 2023-09-23T20:36:27.338+0000 7f3bf2a151c0 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-09-23T20:36:28.132 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:36:27 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[96086]: 2023-09-23T20:36:27.680+0000 7f3bf2a151c0 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2023-09-23T20:36:28.132 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:36:27 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[96086]: 2023-09-23T20:36:27.783+0000 7f3bf2a151c0 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-09-23T20:36:28.456 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:36:28 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[96086]: 2023-09-23T20:36:28.136+0000 7f3bf2a151c0 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-09-23T20:36:28.882 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:36:28 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[96086]: 2023-09-23T20:36:28.455+0000 7f3bf2a151c0 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-09-23T20:36:29.632 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:36:29 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[96086]: 2023-09-23T20:36:29.307+0000 7f3bf2a151c0 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-09-23T20:36:29.632 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:36:29 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[96086]: 2023-09-23T20:36:29.462+0000 7f3bf2a151c0 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-09-23T20:36:30.109 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:36:29 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[96086]: 2023-09-23T20:36:29.701+0000 7f3bf2a151c0 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-09-23T20:36:30.109 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:36:29 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[96086]: 2023-09-23T20:36:29.838+0000 7f3bf2a151c0 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-09-23T20:36:30.382 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:36:30 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[96086]: 2023-09-23T20:36:30.107+0000 7f3bf2a151c0 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-09-23T20:36:31.632 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:36:31 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[96086]: 2023-09-23T20:36:31.145+0000 7f3bf2a151c0 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-09-23T20:36:31.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:36:31 smithi026 ceph-mon[95839]: Active manager daemon y restarted 2023-09-23T20:36:31.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:36:31 smithi026 ceph-mon[95839]: Activating manager daemon y 2023-09-23T20:36:31.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:36:31 smithi026 ceph-mon[95839]: osdmap e2: 0 total, 0 up, 0 in 2023-09-23T20:36:31.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:36:31 smithi026 ceph-mon[95839]: mgrmap e6: y(active, starting, since 0.00316267s) 2023-09-23T20:36:31.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:36:31 smithi026 ceph-mon[95839]: from='mgr.14120 172.21.15.26:0/4153932774' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-23T20:36:31.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:36:31 smithi026 ceph-mon[95839]: from='mgr.14120 172.21.15.26:0/4153932774' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-09-23T20:36:31.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:36:31 smithi026 ceph-mon[95839]: from='mgr.14120 172.21.15.26:0/4153932774' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-09-23T20:36:31.633 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:36:31 smithi026 ceph-mon[95839]: from='mgr.14120 172.21.15.26:0/4153932774' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-09-23T20:36:31.633 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:36:31 smithi026 ceph-mon[95839]: from='mgr.14120 172.21.15.26:0/4153932774' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-09-23T20:36:31.633 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:36:31 smithi026 ceph-mon[95839]: Manager daemon y is now available 2023-09-23T20:36:31.633 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:36:31 smithi026 ceph-mon[95839]: Found migration_current of "None". Setting to last migration. 2023-09-23T20:36:31.633 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:36:31 smithi026 ceph-mon[95839]: from='mgr.14120 172.21.15.26:0/4153932774' entity='mgr.y' 2023-09-23T20:36:31.633 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:36:31 smithi026 ceph-mon[95839]: from='mgr.14120 172.21.15.26:0/4153932774' entity='mgr.y' 2023-09-23T20:36:31.633 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:36:31 smithi026 ceph-mon[95839]: from='mgr.14120 172.21.15.26:0/4153932774' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:36:31.633 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:36:31 smithi026 ceph-mon[95839]: from='mgr.14120 172.21.15.26:0/4153932774' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:36:32.591 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:36:32 smithi026 ceph-mon[95839]: from='mgr.14120 172.21.15.26:0/4153932774' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:36:32.591 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:36:32 smithi026 ceph-mon[95839]: from='mgr.14120 172.21.15.26:0/4153932774' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-09-23T20:36:32.591 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:36:32 smithi026 ceph-mon[95839]: from='mgr.14120 172.21.15.26:0/4153932774' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-09-23T20:36:32.591 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:36:32 smithi026 ceph-mon[95839]: from='client.14124 -' entity='client.admin' cmd=[{"prefix": "get_command_descriptions"}]: dispatch 2023-09-23T20:36:32.591 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:36:32 smithi026 ceph-mon[95839]: mgrmap e7: y(active, since 1.0055s) 2023-09-23T20:36:32.591 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:36:32 smithi026 ceph-mon[95839]: from='client.14124 -' entity='client.admin' cmd=[{"prefix": "mgr_status"}]: dispatch 2023-09-23T20:36:32.616 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout { 2023-09-23T20:36:32.616 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "mgrmap_epoch": 7, 2023-09-23T20:36:32.617 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "initialized": true 2023-09-23T20:36:32.617 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout } 2023-09-23T20:36:32.617 INFO:teuthology.orchestra.run.smithi026.stdout:mgr epoch 5 is available 2023-09-23T20:36:32.617 INFO:teuthology.orchestra.run.smithi026.stdout:Setting orchestrator backend to cephadm... 2023-09-23T20:36:34.631 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:36:34 smithi026 ceph-mon[95839]: mgrmap e8: y(active, since 2s) 2023-09-23T20:36:34.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:36:34 smithi026 ceph-mon[95839]: [23/Sep/2023:20:36:33] ENGINE Bus STARTING 2023-09-23T20:36:34.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:36:34 smithi026 ceph-mon[95839]: [23/Sep/2023:20:36:33] ENGINE Serving on https://172.21.15.26:7150 2023-09-23T20:36:34.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:36:34 smithi026 ceph-mon[95839]: [23/Sep/2023:20:36:33] ENGINE Bus STARTED 2023-09-23T20:36:34.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:36:34 smithi026 ceph-mon[95839]: from='mgr.14120 172.21.15.26:0/4153932774' entity='mgr.y' 2023-09-23T20:36:34.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:36:34 smithi026 ceph-mon[95839]: from='mgr.14120 172.21.15.26:0/4153932774' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:36:34.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:36:34 smithi026 ceph-mon[95839]: from='client.14132 -' entity='client.admin' cmd=[{"prefix": "orch set backend", "module_name": "cephadm", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T20:36:34.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:36:34 smithi026 ceph-mon[95839]: from='mgr.14120 172.21.15.26:0/4153932774' entity='mgr.y' 2023-09-23T20:36:34.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:36:34 smithi026 ceph-mon[95839]: from='mgr.14120 172.21.15.26:0/4153932774' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:36:35.213 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout value unchanged 2023-09-23T20:36:35.213 INFO:teuthology.orchestra.run.smithi026.stdout:Generating ssh key... 2023-09-23T20:36:35.413 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:36:35 smithi026 ceph-mon[95839]: from='client.14134 -' entity='client.admin' cmd=[{"prefix": "cephadm set-user", "user": "root", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T20:36:36.490 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:36:36 smithi026 ceph-mon[95839]: from='client.14136 -' entity='client.admin' cmd=[{"prefix": "cephadm generate-key", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T20:36:36.490 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:36:36 smithi026 ceph-mon[95839]: Generating ssh key... 2023-09-23T20:36:36.490 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:36:36 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[96086]: Generating public/private rsa key pair. 2023-09-23T20:36:36.490 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:36:36 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[96086]: Your identification has been saved in /tmp/tmpaq_b7k5w/key. 2023-09-23T20:36:36.490 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:36:36 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[96086]: Your public key has been saved in /tmp/tmpaq_b7k5w/key.pub. 2023-09-23T20:36:36.490 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:36:36 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[96086]: The key fingerprint is: 2023-09-23T20:36:36.490 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:36:36 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[96086]: SHA256:UW8h1kwXBPmQ1E+NY9gSSyRV7RYn3mw0IHPT7T3lafk ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 2023-09-23T20:36:36.491 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:36:36 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[96086]: The key's randomart image is: 2023-09-23T20:36:36.491 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:36:36 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[96086]: +---[RSA 3072]----+ 2023-09-23T20:36:36.491 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:36:36 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[96086]: | =XX#B+o| 2023-09-23T20:36:36.491 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:36:36 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[96086]: | o =%+O+O| 2023-09-23T20:36:36.491 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:36:36 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[96086]: | . +* %O| 2023-09-23T20:36:36.491 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:36:36 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[96086]: | . . o=@| 2023-09-23T20:36:36.491 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:36:36 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[96086]: | S .oo| 2023-09-23T20:36:36.491 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:36:36 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[96086]: | E| 2023-09-23T20:36:36.491 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:36:36 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[96086]: | | 2023-09-23T20:36:36.491 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:36:36 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[96086]: | | 2023-09-23T20:36:36.491 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:36:36 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[96086]: | | 2023-09-23T20:36:36.491 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:36:36 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[96086]: +----[SHA256]-----+ 2023-09-23T20:36:37.754 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:36:37 smithi026 ceph-mon[95839]: from='mgr.14120 172.21.15.26:0/4153932774' entity='mgr.y' 2023-09-23T20:36:37.755 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:36:37 smithi026 ceph-mon[95839]: from='mgr.14120 172.21.15.26:0/4153932774' entity='mgr.y' 2023-09-23T20:36:38.098 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCmmtLIqZnPaHVdnBGnaMLZrpfKWPZJsm47f/yb58qaBtA9j8R3cpuQW7r1R2jsGYcBkXdFSGny2v/RTmJgkjGZz+DdDmUBz0sjq2L2n0o8gaTpQ3DRPDE7rc0Jm4gTL7mlh6ttRq0ZX16YP0329T0vfyuzxy82ZdTdfGm43Ia3SoypKhaRrfIumbMPuBXoPtKsRAb/ZaEIq911RAoG9ESq5ScwKFieKwzILFUE3CIXofSVfYlu2H2Y1ncSLuIgmqsZWIoz+JUVbPW6v/EWViVaFbu9o9Rl/DSOXdFP/RTCPM0yGC0szgRS+4ArwHC49RE5fAKtbNKx8RgN/uuWspov864eFFaNjCE8sja7ScJxqSZOlgj44paT+W1fdZ7xsRM+1DWxNkRXL6yZ83VH1Y1sfINRFaGDIhfemjfhtucFt0ypiV8YuLFxXGJaim2V1R8E+mz/tx2Vm4mGCkeId+PO9F5EB3ONpv7QS2zuhjMxjwMUm0wKGpPqM5t8sxzCY4k= ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 2023-09-23T20:36:38.098 INFO:teuthology.orchestra.run.smithi026.stdout:Wrote public SSH key to /home/ubuntu/cephtest/ceph.pub 2023-09-23T20:36:38.098 INFO:teuthology.orchestra.run.smithi026.stdout:Adding key to root@localhost authorized_keys... 2023-09-23T20:36:38.100 INFO:teuthology.orchestra.run.smithi026.stdout:Adding host smithi026... 2023-09-23T20:36:38.881 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:36:38 smithi026 ceph-mon[95839]: from='client.14138 -' entity='client.admin' cmd=[{"prefix": "cephadm get-pub-key", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T20:36:39.741 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:36:39 smithi026 ceph-mon[95839]: from='client.14140 -' entity='client.admin' cmd=[{"prefix": "orch host add", "hostname": "smithi026", "addr": "172.21.15.26", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T20:36:40.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:36:40 smithi026 ceph-mon[95839]: Deploying cephadm binary to smithi026 2023-09-23T20:36:41.348 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout Added host 'smithi026' with addr '172.21.15.26' 2023-09-23T20:36:41.349 INFO:teuthology.orchestra.run.smithi026.stdout:Deploying unmanaged mon service... 2023-09-23T20:36:42.270 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:36:41 smithi026 ceph-mon[95839]: from='mgr.14120 172.21.15.26:0/4153932774' entity='mgr.y' 2023-09-23T20:36:42.270 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:36:41 smithi026 ceph-mon[95839]: Added host smithi026 2023-09-23T20:36:42.270 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:36:41 smithi026 ceph-mon[95839]: from='mgr.14120 172.21.15.26:0/4153932774' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:36:42.692 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout Scheduled mon update... 2023-09-23T20:36:42.692 INFO:teuthology.orchestra.run.smithi026.stdout:Deploying unmanaged mgr service... 2023-09-23T20:36:43.381 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:36:43 smithi026 ceph-mon[95839]: from='client.14142 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "unmanaged": true, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T20:36:43.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:36:43 smithi026 ceph-mon[95839]: Saving service mon spec with placement count:5 2023-09-23T20:36:43.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:36:43 smithi026 ceph-mon[95839]: from='mgr.14120 172.21.15.26:0/4153932774' entity='mgr.y' 2023-09-23T20:36:43.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:36:43 smithi026 ceph-mon[95839]: from='mgr.14120 172.21.15.26:0/4153932774' entity='mgr.y' 2023-09-23T20:36:44.236 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout Scheduled mgr update... 2023-09-23T20:36:44.881 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:36:44 smithi026 ceph-mon[95839]: from='mgr.14120 172.21.15.26:0/4153932774' entity='mgr.y' 2023-09-23T20:36:44.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:36:44 smithi026 ceph-mon[95839]: from='client.14144 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "unmanaged": true, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T20:36:44.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:36:44 smithi026 ceph-mon[95839]: Saving service mgr spec with placement count:2 2023-09-23T20:36:44.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:36:44 smithi026 ceph-mon[95839]: from='mgr.14120 172.21.15.26:0/4153932774' entity='mgr.y' 2023-09-23T20:36:46.381 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:36:46 smithi026 ceph-mon[95839]: from='client.? 172.21.15.26:0/261878887' entity='client.admin' 2023-09-23T20:36:46.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:36:46 smithi026 ceph-mon[95839]: from='mgr.14120 172.21.15.26:0/4153932774' entity='mgr.y' 2023-09-23T20:36:46.822 INFO:teuthology.orchestra.run.smithi026.stdout:Enabling the dashboard module... 2023-09-23T20:36:47.869 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:36:47 smithi026 ceph-mon[95839]: from='client.? 172.21.15.26:0/2005920106' entity='client.admin' 2023-09-23T20:36:47.869 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:36:47 smithi026 ceph-mon[95839]: from='mgr.14120 172.21.15.26:0/4153932774' entity='mgr.y' 2023-09-23T20:36:48.838 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:36:48 smithi026 ceph-mon[95839]: from='client.? 172.21.15.26:0/2418898918' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "dashboard"}]: dispatch 2023-09-23T20:36:48.838 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:36:48 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[96086]: ignoring --setuser ceph since I am not root 2023-09-23T20:36:48.838 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:36:48 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[96086]: ignoring --setgroup ceph since I am not root 2023-09-23T20:36:48.839 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:36:48 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[96086]: 2023-09-23T20:36:48.743+0000 7ffb4cb051c0 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2023-09-23T20:36:49.132 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:36:48 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[96086]: 2023-09-23T20:36:48.838+0000 7ffb4cb051c0 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-09-23T20:36:49.132 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:36:48 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[96086]: 2023-09-23T20:36:48.944+0000 7ffb4cb051c0 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-09-23T20:36:49.132 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:36:49 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[96086]: 2023-09-23T20:36:49.070+0000 7ffb4cb051c0 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-09-23T20:36:49.537 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:36:49 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[96086]: 2023-09-23T20:36:49.281+0000 7ffb4cb051c0 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-09-23T20:36:49.881 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:36:49 smithi026 ceph-mon[95839]: from='client.? 172.21.15.26:0/2418898918' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "dashboard"}]': finished 2023-09-23T20:36:49.948 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:36:49 smithi026 ceph-mon[95839]: mgrmap e9: y(active, since 17s) 2023-09-23T20:36:49.948 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:36:49 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[96086]: 2023-09-23T20:36:49.681+0000 7ffb4cb051c0 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-09-23T20:36:49.948 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:36:49 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[96086]: 2023-09-23T20:36:49.787+0000 7ffb4cb051c0 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-09-23T20:36:50.224 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout { 2023-09-23T20:36:50.225 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "epoch": 9, 2023-09-23T20:36:50.225 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "available": true, 2023-09-23T20:36:50.225 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "active_name": "y", 2023-09-23T20:36:50.225 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "num_standby": 0 2023-09-23T20:36:50.225 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout } 2023-09-23T20:36:50.225 INFO:teuthology.orchestra.run.smithi026.stdout:Waiting for the mgr to restart... 2023-09-23T20:36:50.225 INFO:teuthology.orchestra.run.smithi026.stdout:Waiting for mgr epoch 9... 2023-09-23T20:36:50.381 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:36:49 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[96086]: 2023-09-23T20:36:49.936+0000 7ffb4cb051c0 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-09-23T20:36:50.382 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:36:50 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[96086]: 2023-09-23T20:36:50.062+0000 7ffb4cb051c0 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-09-23T20:36:50.881 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:36:50 smithi026 ceph-mon[95839]: from='client.? 172.21.15.26:0/3095898494' entity='client.admin' cmd=[{"prefix": "mgr stat"}]: dispatch 2023-09-23T20:36:50.882 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:36:50 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[96086]: 2023-09-23T20:36:50.575+0000 7ffb4cb051c0 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-09-23T20:36:51.632 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:36:51 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[96086]: 2023-09-23T20:36:51.263+0000 7ffb4cb051c0 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-09-23T20:36:51.632 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:36:51 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[96086]: 2023-09-23T20:36:51.453+0000 7ffb4cb051c0 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-09-23T20:36:52.748 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:36:52 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[96086]: 2023-09-23T20:36:52.423+0000 7ffb4cb051c0 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-09-23T20:36:52.748 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:36:52 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[96086]: 2023-09-23T20:36:52.534+0000 7ffb4cb051c0 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2023-09-23T20:36:52.748 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:36:52 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[96086]: 2023-09-23T20:36:52.748+0000 7ffb4cb051c0 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-09-23T20:36:53.381 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:36:53 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[96086]: 2023-09-23T20:36:53.090+0000 7ffb4cb051c0 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2023-09-23T20:36:53.382 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:36:53 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[96086]: 2023-09-23T20:36:53.193+0000 7ffb4cb051c0 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-09-23T20:36:53.841 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:36:53 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[96086]: 2023-09-23T20:36:53.521+0000 7ffb4cb051c0 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-09-23T20:36:55.057 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:36:53 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[96086]: 2023-09-23T20:36:53.840+0000 7ffb4cb051c0 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-09-23T20:36:55.057 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:36:54 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[96086]: 2023-09-23T20:36:54.660+0000 7ffb4cb051c0 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-09-23T20:36:55.057 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:36:54 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[96086]: 2023-09-23T20:36:54.817+0000 7ffb4cb051c0 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-09-23T20:36:55.057 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:36:55 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[96086]: 2023-09-23T20:36:55.056+0000 7ffb4cb051c0 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-09-23T20:36:55.381 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:36:55 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[96086]: 2023-09-23T20:36:55.192+0000 7ffb4cb051c0 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-09-23T20:36:55.881 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:36:55 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[96086]: 2023-09-23T20:36:55.459+0000 7ffb4cb051c0 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-09-23T20:36:56.882 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:36:56 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[96086]: 2023-09-23T20:36:56.485+0000 7ffb4cb051c0 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-09-23T20:36:56.883 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:36:56 smithi026 ceph-mon[95839]: Active manager daemon y restarted 2023-09-23T20:36:56.883 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:36:56 smithi026 ceph-mon[95839]: Activating manager daemon y 2023-09-23T20:36:56.883 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:36:56 smithi026 ceph-mon[95839]: osdmap e3: 0 total, 0 up, 0 in 2023-09-23T20:36:56.883 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:36:56 smithi026 ceph-mon[95839]: mgrmap e10: y(active, starting, since 0.0030877s) 2023-09-23T20:36:56.883 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:36:56 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-23T20:36:56.883 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:36:56 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-09-23T20:36:56.883 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:36:56 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-09-23T20:36:56.883 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:36:56 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-09-23T20:36:56.883 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:36:56 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-09-23T20:36:56.883 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:36:56 smithi026 ceph-mon[95839]: Manager daemon y is now available 2023-09-23T20:36:56.884 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:36:56 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:36:56.884 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:36:56 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:36:57.787 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:36:57 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-09-23T20:36:57.787 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:36:57 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-09-23T20:36:57.788 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:36:57 smithi026 ceph-mon[95839]: mgrmap e11: y(active, since 1.00714s) 2023-09-23T20:36:58.285 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout { 2023-09-23T20:36:58.285 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "mgrmap_epoch": 11, 2023-09-23T20:36:58.286 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "initialized": true 2023-09-23T20:36:58.286 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout } 2023-09-23T20:36:58.286 INFO:teuthology.orchestra.run.smithi026.stdout:mgr epoch 9 is available 2023-09-23T20:36:58.286 INFO:teuthology.orchestra.run.smithi026.stdout:Generating a dashboard self-signed certificate... 2023-09-23T20:36:58.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:36:58 smithi026 ceph-mon[95839]: from='client.14156 -' entity='client.admin' cmd=[{"prefix": "get_command_descriptions"}]: dispatch 2023-09-23T20:36:58.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:36:58 smithi026 ceph-mon[95839]: from='client.14156 -' entity='client.admin' cmd=[{"prefix": "mgr_status"}]: dispatch 2023-09-23T20:36:58.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:36:58 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:36:59.632 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout Self-signed certificate created 2023-09-23T20:36:59.632 INFO:teuthology.orchestra.run.smithi026.stdout:Creating initial admin user... 2023-09-23T20:36:59.844 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:36:59 smithi026 ceph-mon[95839]: [23/Sep/2023:20:36:57] ENGINE Bus STARTING 2023-09-23T20:36:59.844 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:36:59 smithi026 ceph-mon[95839]: [23/Sep/2023:20:36:57] ENGINE Serving on https://172.21.15.26:7150 2023-09-23T20:36:59.844 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:36:59 smithi026 ceph-mon[95839]: [23/Sep/2023:20:36:57] ENGINE Bus STARTED 2023-09-23T20:36:59.844 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:36:59 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:36:59.844 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:36:59 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:36:59.844 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:36:59 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:36:59.844 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:36:59 smithi026 ceph-mon[95839]: mgrmap e12: y(active, since 2s) 2023-09-23T20:37:00.822 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:37:00 smithi026 ceph-mon[95839]: from='client.14164 -' entity='client.admin' cmd=[{"prefix": "dashboard create-self-signed-cert", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T20:37:00.822 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:37:00 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:37:01.479 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout {"username": "admin", "password": "$2b$12$u/wH9xKMKNqe0VUXz8MdSOuNHmGaH6vnItYqM1k4fmg30UdQwkTky", "roles": ["administrator"], "name": null, "email": null, "lastUpdate": 1695501420, "enabled": true, "pwdExpirationDate": null, "pwdUpdateRequired": true} 2023-09-23T20:37:01.479 INFO:teuthology.orchestra.run.smithi026.stdout:Fetching dashboard port number... 2023-09-23T20:37:01.881 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:37:01 smithi026 ceph-mon[95839]: from='client.14166 -' entity='client.admin' cmd=[{"prefix": "dashboard ac-user-create", "username": "admin", "rolename": "administrator", "force_password": true, "pwd_update_required": true, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T20:37:01.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:37:01 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:37:02.726 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout 8443 2023-09-23T20:37:02.735 INFO:teuthology.orchestra.run.smithi026.stdout:Non-zero exit code 1 from systemctl is-enabled firewalld.service 2023-09-23T20:37:02.735 INFO:teuthology.orchestra.run.smithi026.stdout:systemctl: stdout disabled 2023-09-23T20:37:02.743 INFO:teuthology.orchestra.run.smithi026.stdout:Non-zero exit code 3 from systemctl is-active firewalld.service 2023-09-23T20:37:02.743 INFO:teuthology.orchestra.run.smithi026.stdout:systemctl: stdout inactive 2023-09-23T20:37:02.743 INFO:teuthology.orchestra.run.smithi026.stdout:firewalld.service is not enabled 2023-09-23T20:37:02.743 INFO:teuthology.orchestra.run.smithi026.stdout:Not possible to open ports <[8443]>. firewalld.service is not available 2023-09-23T20:37:02.745 INFO:teuthology.orchestra.run.smithi026.stdout:Ceph Dashboard is now available at: 2023-09-23T20:37:02.745 INFO:teuthology.orchestra.run.smithi026.stdout: 2023-09-23T20:37:02.745 INFO:teuthology.orchestra.run.smithi026.stdout: URL: https://smithi026:8443/ 2023-09-23T20:37:02.745 INFO:teuthology.orchestra.run.smithi026.stdout: User: admin 2023-09-23T20:37:02.745 INFO:teuthology.orchestra.run.smithi026.stdout: Password: 8oej5vq8vt 2023-09-23T20:37:02.745 INFO:teuthology.orchestra.run.smithi026.stdout: 2023-09-23T20:37:02.745 INFO:teuthology.orchestra.run.smithi026.stdout:Saving cluster configuration to /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/config directory 2023-09-23T20:37:02.746 INFO:teuthology.orchestra.run.smithi026.stdout:Enabling autotune for osd_memory_target 2023-09-23T20:37:02.759 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:37:02 smithi026 ceph-mon[95839]: from='client.? 172.21.15.26:0/4223701015' entity='client.admin' cmd=[{"prefix": "config get", "who": "mgr", "key": "mgr/dashboard/ssl_server_port"}]: dispatch 2023-09-23T20:37:03.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:37:03 smithi026 ceph-mon[95839]: mgrmap e13: y(active, since 6s) 2023-09-23T20:37:05.429 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stderr set mgr/dashboard/cluster/status 2023-09-23T20:37:05.429 INFO:teuthology.orchestra.run.smithi026.stdout:You can access the Ceph CLI as following in case of multi-cluster or non-default config: 2023-09-23T20:37:05.429 INFO:teuthology.orchestra.run.smithi026.stdout: 2023-09-23T20:37:05.429 INFO:teuthology.orchestra.run.smithi026.stdout: sudo /home/ubuntu/cephtest/cephadm shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring 2023-09-23T20:37:05.429 INFO:teuthology.orchestra.run.smithi026.stdout: 2023-09-23T20:37:05.429 INFO:teuthology.orchestra.run.smithi026.stdout:Or, if you are only running a single cluster on this host: 2023-09-23T20:37:05.429 INFO:teuthology.orchestra.run.smithi026.stdout: 2023-09-23T20:37:05.429 INFO:teuthology.orchestra.run.smithi026.stdout: sudo /home/ubuntu/cephtest/cephadm shell 2023-09-23T20:37:05.429 INFO:teuthology.orchestra.run.smithi026.stdout: 2023-09-23T20:37:05.430 INFO:teuthology.orchestra.run.smithi026.stdout:Please consider enabling telemetry to help improve Ceph: 2023-09-23T20:37:05.430 INFO:teuthology.orchestra.run.smithi026.stdout: 2023-09-23T20:37:05.430 INFO:teuthology.orchestra.run.smithi026.stdout: ceph telemetry on 2023-09-23T20:37:05.430 INFO:teuthology.orchestra.run.smithi026.stdout: 2023-09-23T20:37:05.430 INFO:teuthology.orchestra.run.smithi026.stdout:For more information see: 2023-09-23T20:37:05.430 INFO:teuthology.orchestra.run.smithi026.stdout: 2023-09-23T20:37:05.430 INFO:teuthology.orchestra.run.smithi026.stdout: https://docs.ceph.com/docs/master/mgr/telemetry/ 2023-09-23T20:37:05.430 INFO:teuthology.orchestra.run.smithi026.stdout: 2023-09-23T20:37:05.430 INFO:teuthology.orchestra.run.smithi026.stdout:Bootstrap complete. 2023-09-23T20:37:05.465 INFO:tasks.cephadm:Fetching config... 2023-09-23T20:37:05.465 DEBUG:teuthology.orchestra.run.smithi026:> set -ex 2023-09-23T20:37:05.465 DEBUG:teuthology.orchestra.run.smithi026:> dd if=/etc/ceph/ceph.conf of=/dev/stdout 2023-09-23T20:37:05.482 INFO:tasks.cephadm:Fetching client.admin keyring... 2023-09-23T20:37:05.482 DEBUG:teuthology.orchestra.run.smithi026:> set -ex 2023-09-23T20:37:05.482 DEBUG:teuthology.orchestra.run.smithi026:> dd if=/etc/ceph/ceph.client.admin.keyring of=/dev/stdout 2023-09-23T20:37:05.541 INFO:tasks.cephadm:Fetching mon keyring... 2023-09-23T20:37:05.541 DEBUG:teuthology.orchestra.run.smithi026:> set -ex 2023-09-23T20:37:05.541 DEBUG:teuthology.orchestra.run.smithi026:> sudo dd if=/var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/keyring of=/dev/stdout 2023-09-23T20:37:05.609 INFO:tasks.cephadm:Fetching pub ssh key... 2023-09-23T20:37:05.610 DEBUG:teuthology.orchestra.run.smithi026:> set -ex 2023-09-23T20:37:05.610 DEBUG:teuthology.orchestra.run.smithi026:> dd if=/home/ubuntu/cephtest/ceph.pub of=/dev/stdout 2023-09-23T20:37:05.667 INFO:tasks.cephadm:Installing pub ssh key for root users... 2023-09-23T20:37:05.668 DEBUG:teuthology.orchestra.run.smithi026:> sudo install -d -m 0700 /root/.ssh && echo 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCmmtLIqZnPaHVdnBGnaMLZrpfKWPZJsm47f/yb58qaBtA9j8R3cpuQW7r1R2jsGYcBkXdFSGny2v/RTmJgkjGZz+DdDmUBz0sjq2L2n0o8gaTpQ3DRPDE7rc0Jm4gTL7mlh6ttRq0ZX16YP0329T0vfyuzxy82ZdTdfGm43Ia3SoypKhaRrfIumbMPuBXoPtKsRAb/ZaEIq911RAoG9ESq5ScwKFieKwzILFUE3CIXofSVfYlu2H2Y1ncSLuIgmqsZWIoz+JUVbPW6v/EWViVaFbu9o9Rl/DSOXdFP/RTCPM0yGC0szgRS+4ArwHC49RE5fAKtbNKx8RgN/uuWspov864eFFaNjCE8sja7ScJxqSZOlgj44paT+W1fdZ7xsRM+1DWxNkRXL6yZ83VH1Y1sfINRFaGDIhfemjfhtucFt0ypiV8YuLFxXGJaim2V1R8E+mz/tx2Vm4mGCkeId+PO9F5EB3ONpv7QS2zuhjMxjwMUm0wKGpPqM5t8sxzCY4k= ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7' | sudo tee -a /root/.ssh/authorized_keys && sudo chmod 0600 /root/.ssh/authorized_keys 2023-09-23T20:37:05.746 INFO:teuthology.orchestra.run.smithi026.stdout:ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCmmtLIqZnPaHVdnBGnaMLZrpfKWPZJsm47f/yb58qaBtA9j8R3cpuQW7r1R2jsGYcBkXdFSGny2v/RTmJgkjGZz+DdDmUBz0sjq2L2n0o8gaTpQ3DRPDE7rc0Jm4gTL7mlh6ttRq0ZX16YP0329T0vfyuzxy82ZdTdfGm43Ia3SoypKhaRrfIumbMPuBXoPtKsRAb/ZaEIq911RAoG9ESq5ScwKFieKwzILFUE3CIXofSVfYlu2H2Y1ncSLuIgmqsZWIoz+JUVbPW6v/EWViVaFbu9o9Rl/DSOXdFP/RTCPM0yGC0szgRS+4ArwHC49RE5fAKtbNKx8RgN/uuWspov864eFFaNjCE8sja7ScJxqSZOlgj44paT+W1fdZ7xsRM+1DWxNkRXL6yZ83VH1Y1sfINRFaGDIhfemjfhtucFt0ypiV8YuLFxXGJaim2V1R8E+mz/tx2Vm4mGCkeId+PO9F5EB3ONpv7QS2zuhjMxjwMUm0wKGpPqM5t8sxzCY4k= ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 2023-09-23T20:37:05.758 DEBUG:teuthology.orchestra.run.smithi070:> sudo install -d -m 0700 /root/.ssh && echo 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCmmtLIqZnPaHVdnBGnaMLZrpfKWPZJsm47f/yb58qaBtA9j8R3cpuQW7r1R2jsGYcBkXdFSGny2v/RTmJgkjGZz+DdDmUBz0sjq2L2n0o8gaTpQ3DRPDE7rc0Jm4gTL7mlh6ttRq0ZX16YP0329T0vfyuzxy82ZdTdfGm43Ia3SoypKhaRrfIumbMPuBXoPtKsRAb/ZaEIq911RAoG9ESq5ScwKFieKwzILFUE3CIXofSVfYlu2H2Y1ncSLuIgmqsZWIoz+JUVbPW6v/EWViVaFbu9o9Rl/DSOXdFP/RTCPM0yGC0szgRS+4ArwHC49RE5fAKtbNKx8RgN/uuWspov864eFFaNjCE8sja7ScJxqSZOlgj44paT+W1fdZ7xsRM+1DWxNkRXL6yZ83VH1Y1sfINRFaGDIhfemjfhtucFt0ypiV8YuLFxXGJaim2V1R8E+mz/tx2Vm4mGCkeId+PO9F5EB3ONpv7QS2zuhjMxjwMUm0wKGpPqM5t8sxzCY4k= ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7' | sudo tee -a /root/.ssh/authorized_keys && sudo chmod 0600 /root/.ssh/authorized_keys 2023-09-23T20:37:05.806 INFO:teuthology.orchestra.run.smithi070.stdout:ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCmmtLIqZnPaHVdnBGnaMLZrpfKWPZJsm47f/yb58qaBtA9j8R3cpuQW7r1R2jsGYcBkXdFSGny2v/RTmJgkjGZz+DdDmUBz0sjq2L2n0o8gaTpQ3DRPDE7rc0Jm4gTL7mlh6ttRq0ZX16YP0329T0vfyuzxy82ZdTdfGm43Ia3SoypKhaRrfIumbMPuBXoPtKsRAb/ZaEIq911RAoG9ESq5ScwKFieKwzILFUE3CIXofSVfYlu2H2Y1ncSLuIgmqsZWIoz+JUVbPW6v/EWViVaFbu9o9Rl/DSOXdFP/RTCPM0yGC0szgRS+4ArwHC49RE5fAKtbNKx8RgN/uuWspov864eFFaNjCE8sja7ScJxqSZOlgj44paT+W1fdZ7xsRM+1DWxNkRXL6yZ83VH1Y1sfINRFaGDIhfemjfhtucFt0ypiV8YuLFxXGJaim2V1R8E+mz/tx2Vm4mGCkeId+PO9F5EB3ONpv7QS2zuhjMxjwMUm0wKGpPqM5t8sxzCY4k= ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 2023-09-23T20:37:05.819 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph config set mgr mgr/cephadm/allow_ptrace true 2023-09-23T20:37:06.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:37:06 smithi026 ceph-mon[95839]: from='client.? 172.21.15.26:0/3745722695' entity='client.admin' 2023-09-23T20:37:06.434 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:37:08.550 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:37:08 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:37:08.550 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:37:08 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:37:08.550 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:37:08 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi026", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:37:08.550 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:37:08 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:37:08.550 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:37:08 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:37:08.550 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:37:08 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:37:08.600 INFO:tasks.cephadm:Distributing conf and client.admin keyring to all hosts + 0755 2023-09-23T20:37:08.600 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph orch client-keyring set client.admin '*' --mode 0755 2023-09-23T20:37:09.276 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:37:09.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:37:09 smithi026 ceph-mon[95839]: from='client.? 172.21.15.26:0/703920001' entity='client.admin' 2023-09-23T20:37:09.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:37:09 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:37:09.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:37:09 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:37:11.239 INFO:tasks.cephadm:Writing (initial) conf and keyring to smithi070 2023-09-23T20:37:11.239 DEBUG:teuthology.orchestra.run.smithi070:> set -ex 2023-09-23T20:37:11.239 DEBUG:teuthology.orchestra.run.smithi070:> dd of=/etc/ceph/ceph.conf 2023-09-23T20:37:11.259 DEBUG:teuthology.orchestra.run.smithi070:> set -ex 2023-09-23T20:37:11.259 DEBUG:teuthology.orchestra.run.smithi070:> dd of=/etc/ceph/ceph.client.admin.keyring 2023-09-23T20:37:11.319 INFO:tasks.cephadm:Adding host smithi070 to orchestrator... 2023-09-23T20:37:11.320 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph orch host add smithi070 2023-09-23T20:37:11.934 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:37:12.116 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:37:11 smithi026 ceph-mon[95839]: from='client.14176 -' entity='client.admin' cmd=[{"prefix": "orch client-keyring set", "entity": "client.admin", "placement": "*", "mode": "0755", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T20:37:12.117 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:37:11 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:37:12.117 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:37:11 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:37:12.117 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:37:11 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:37:12.117 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:37:11 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:37:12.117 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:37:11 smithi026 ceph-mon[95839]: Updating smithi026:/etc/ceph/ceph.conf 2023-09-23T20:37:12.117 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:37:11 smithi026 ceph-mon[95839]: Updating smithi026:/var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-23T20:37:12.117 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:37:11 smithi026 ceph-mon[95839]: Updating smithi026:/etc/ceph/ceph.client.admin.keyring 2023-09-23T20:37:12.117 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:37:11 smithi026 ceph-mon[95839]: Updating smithi026:/var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/config/ceph.client.admin.keyring 2023-09-23T20:37:12.117 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:37:11 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:37:12.117 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:37:11 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:37:12.118 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:37:11 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:37:14.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:37:14 smithi026 ceph-mon[95839]: from='client.14178 -' entity='client.admin' cmd=[{"prefix": "orch host add", "hostname": "smithi070", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T20:37:15.621 INFO:teuthology.orchestra.run.smithi026.stdout:Added host 'smithi070' with addr '172.21.15.70' 2023-09-23T20:37:15.751 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:37:15 smithi026 ceph-mon[95839]: Deploying cephadm binary to smithi070 2023-09-23T20:37:16.207 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph orch host ls --format=json 2023-09-23T20:37:16.820 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:37:16.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:37:16 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:37:16.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:37:16 smithi026 ceph-mon[95839]: Added host smithi070 2023-09-23T20:37:16.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:37:16 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:37:16.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:37:16 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:37:16.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:37:16 smithi026 ceph-mon[95839]: pgmap v3: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T20:37:18.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:37:17 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:37:18.254 INFO:teuthology.orchestra.run.smithi026.stdout: 2023-09-23T20:37:18.254 INFO:teuthology.orchestra.run.smithi026.stdout:[{"addr": "172.21.15.26", "hostname": "smithi026", "labels": [], "status": ""}, {"addr": "172.21.15.70", "hostname": "smithi070", "labels": [], "status": ""}] 2023-09-23T20:37:18.688 INFO:tasks.cephadm:Setting crush tunables to default 2023-09-23T20:37:18.688 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph osd crush tunables default 2023-09-23T20:37:19.294 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:37:19.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:37:19 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:37:19.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:37:19 smithi026 ceph-mon[95839]: from='client.14180 -' entity='client.admin' cmd=[{"prefix": "orch host ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:37:19.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:37:19 smithi026 ceph-mon[95839]: pgmap v4: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T20:37:21.564 INFO:teuthology.orchestra.run.smithi026.stderr:adjusted tunables profile to default 2023-09-23T20:37:21.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:37:21 smithi026 ceph-mon[95839]: pgmap v5: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T20:37:21.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:37:21 smithi026 ceph-mon[95839]: from='client.? 172.21.15.26:0/1212432566' entity='client.admin' cmd=[{"prefix": "osd crush tunables", "profile": "default"}]: dispatch 2023-09-23T20:37:21.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:37:21 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:37:21.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:37:21 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:37:21.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:37:21 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:37:21.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:37:21 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:37:21.883 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:37:21 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi070", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:37:21.883 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:37:21 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:37:21.883 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:37:21 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:37:21.959 INFO:tasks.cephadm:Adding mon.a on smithi026 2023-09-23T20:37:21.959 INFO:tasks.cephadm:Adding mon.c on smithi026 2023-09-23T20:37:21.959 INFO:tasks.cephadm:Adding mon.b on smithi070 2023-09-23T20:37:21.960 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph orch apply mon '3;smithi026:172.21.15.26=a;smithi026:[v2:172.21.15.26:3301,v1:172.21.15.26:6790]=c;smithi070:172.21.15.70=b' 2023-09-23T20:37:22.568 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-23T20:37:22.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:37:22 smithi026 ceph-mon[95839]: Updating smithi070:/etc/ceph/ceph.conf 2023-09-23T20:37:22.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:37:22 smithi026 ceph-mon[95839]: Updating smithi070:/var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-23T20:37:22.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:37:22 smithi026 ceph-mon[95839]: Updating smithi070:/etc/ceph/ceph.client.admin.keyring 2023-09-23T20:37:22.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:37:22 smithi026 ceph-mon[95839]: from='client.? 172.21.15.26:0/1212432566' entity='client.admin' cmd='[{"prefix": "osd crush tunables", "profile": "default"}]': finished 2023-09-23T20:37:22.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:37:22 smithi026 ceph-mon[95839]: osdmap e4: 0 total, 0 up, 0 in 2023-09-23T20:37:22.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:37:22 smithi026 ceph-mon[95839]: Updating smithi070:/var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/config/ceph.client.admin.keyring 2023-09-23T20:37:22.883 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:37:22 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:37:22.883 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:37:22 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:37:22.883 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:37:22 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:37:23.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:37:23 smithi026 ceph-mon[95839]: pgmap v7: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T20:37:23.983 INFO:teuthology.orchestra.run.smithi070.stdout:Scheduled mon update... 2023-09-23T20:37:24.422 DEBUG:teuthology.orchestra.run.smithi026:mon.c> sudo journalctl -f -n 0 -u ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7@mon.c.service 2023-09-23T20:37:24.425 DEBUG:teuthology.orchestra.run.smithi070:mon.b> sudo journalctl -f -n 0 -u ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7@mon.b.service 2023-09-23T20:37:24.427 INFO:tasks.cephadm:Waiting for 3 mons in monmap... 2023-09-23T20:37:24.428 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph mon dump -f json 2023-09-23T20:37:24.456 INFO:journalctl@ceph.mon.b.smithi070.stdout:-- Logs begin at Sat 2023-09-23 20:21:16 UTC. -- 2023-09-23T20:37:24.462 INFO:journalctl@ceph.mon.c.smithi026.stdout:-- Logs begin at Sat 2023-09-23 20:20:42 UTC. -- 2023-09-23T20:37:25.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:37:24 smithi026 ceph-mon[95839]: from='client.14184 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "placement": "3;smithi026:172.21.15.26=a;smithi026:[v2:172.21.15.26:3301,v1:172.21.15.26:6790]=c;smithi070:172.21.15.70=b", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T20:37:25.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:37:24 smithi026 ceph-mon[95839]: Saving service mon spec with placement smithi026:172.21.15.26=a;smithi026:[v2:172.21.15.26:3301,v1:172.21.15.26:6790]=c;smithi070:172.21.15.70=b;count:3 2023-09-23T20:37:25.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:37:24 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:37:25.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:37:24 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:37:25.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:37:24 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:37:25.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:37:24 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:37:25.383 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:37:24 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:37:25.383 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:37:24 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-23T20:37:25.383 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:37:24 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:37:25.383 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:37:24 smithi026 ceph-mon[95839]: Deploying daemon mon.b on smithi070 2023-09-23T20:37:25.383 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:37:24 smithi026 ceph-mon[95839]: pgmap v8: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T20:37:25.457 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-23T20:37:27.266 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:26 smithi070 systemd[1]: Starting Ceph mon.b for b3f8e4ec-5a50-11ee-8db4-212e2dc638e7... 2023-09-23T20:37:27.316 INFO:teuthology.orchestra.run.smithi070.stdout: 2023-09-23T20:37:27.316 INFO:teuthology.orchestra.run.smithi070.stdout:{"epoch":1,"fsid":"b3f8e4ec-5a50-11ee-8db4-212e2dc638e7","modified":"2023-09-23T20:35:56.712408Z","created":"2023-09-23T20:35:56.712408Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","removed_ranks: ":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"a","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:3300","nonce":0},{"type":"v1","addr":"172.21.15.26:6789","nonce":0}]},"addr":"172.21.15.26:6789/0","public_addr":"172.21.15.26:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2023-09-23T20:37:27.318 INFO:teuthology.orchestra.run.smithi070.stderr:dumped monmap epoch 1 2023-09-23T20:37:27.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:27 smithi070 podman[100691]: 2023-09-23T20:37:27.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:37:27 smithi026 ceph-mon[95839]: pgmap v9: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T20:37:27.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:37:27 smithi026 ceph-mon[95839]: from='client.? 172.21.15.70:0/1019581227' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-09-23T20:37:28.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:27 smithi070 ceph-mon[100755]: set uid:gid to 167:167 (ceph:ceph) 2023-09-23T20:37:28.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:27 smithi070 ceph-mon[100755]: ceph version 17.2.6-1123-g61431f6b (61431f6ba2752649b506867ec856b4a6236f90dd) quincy (stable), process ceph-mon, pid 7 2023-09-23T20:37:28.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:27 smithi070 ceph-mon[100755]: pidfile_write: ignore empty --pid-file 2023-09-23T20:37:28.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:27 smithi070 ceph-mon[100755]: load: jerasure load: lrc 2023-09-23T20:37:28.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:27 smithi070 ceph-mon[100755]: rocksdb: RocksDB version: 6.15.5 2023-09-23T20:37:28.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:27 smithi070 ceph-mon[100755]: rocksdb: Git sha rocksdb_build_git_sha:@0@ 2023-09-23T20:37:28.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:27 smithi070 ceph-mon[100755]: rocksdb: Compile date Sep 22 2023 2023-09-23T20:37:28.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:27 smithi070 ceph-mon[100755]: rocksdb: DB SUMMARY 2023-09-23T20:37:28.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:27 smithi070 ceph-mon[100755]: rocksdb: DB Session ID: QYGHN1WK7G7M7IQKU7KN 2023-09-23T20:37:28.121 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:27 smithi070 ceph-mon[100755]: rocksdb: CURRENT file: CURRENT 2023-09-23T20:37:28.121 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:27 smithi070 ceph-mon[100755]: rocksdb: IDENTITY file: IDENTITY 2023-09-23T20:37:28.121 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:27 smithi070 ceph-mon[100755]: rocksdb: MANIFEST file: MANIFEST-000003 size: 57 Bytes 2023-09-23T20:37:28.121 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:27 smithi070 ceph-mon[100755]: rocksdb: SST files in /var/lib/ceph/mon/ceph-b/store.db dir, Total Num: 0, files: 2023-09-23T20:37:28.121 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:27 smithi070 ceph-mon[100755]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-b/store.db: 000004.log size: 511 ; 2023-09-23T20:37:28.121 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:27 smithi070 ceph-mon[100755]: rocksdb: Options.error_if_exists: 0 2023-09-23T20:37:28.121 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:27 smithi070 ceph-mon[100755]: rocksdb: Options.create_if_missing: 0 2023-09-23T20:37:28.121 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:27 smithi070 ceph-mon[100755]: rocksdb: Options.paranoid_checks: 1 2023-09-23T20:37:28.121 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:27 smithi070 ceph-mon[100755]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2023-09-23T20:37:28.121 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:27 smithi070 ceph-mon[100755]: rocksdb: Options.env: 0x564c97586a80 2023-09-23T20:37:28.121 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:27 smithi070 ceph-mon[100755]: rocksdb: Options.fs: Posix File System 2023-09-23T20:37:28.121 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:27 smithi070 ceph-mon[100755]: rocksdb: Options.info_log: 0x564c997a3d80 2023-09-23T20:37:28.122 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:27 smithi070 ceph-mon[100755]: rocksdb: Options.max_file_opening_threads: 16 2023-09-23T20:37:28.122 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:27 smithi070 ceph-mon[100755]: rocksdb: Options.statistics: (nil) 2023-09-23T20:37:28.122 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:27 smithi070 ceph-mon[100755]: rocksdb: Options.use_fsync: 0 2023-09-23T20:37:28.122 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:27 smithi070 ceph-mon[100755]: rocksdb: Options.max_log_file_size: 0 2023-09-23T20:37:28.122 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:27 smithi070 ceph-mon[100755]: rocksdb: Options.max_manifest_file_size: 1073741824 2023-09-23T20:37:28.122 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:27 smithi070 ceph-mon[100755]: rocksdb: Options.log_file_time_to_roll: 0 2023-09-23T20:37:28.122 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:27 smithi070 ceph-mon[100755]: rocksdb: Options.keep_log_file_num: 1000 2023-09-23T20:37:28.122 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:27 smithi070 ceph-mon[100755]: rocksdb: Options.recycle_log_file_num: 0 2023-09-23T20:37:28.122 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:27 smithi070 ceph-mon[100755]: rocksdb: Options.allow_fallocate: 1 2023-09-23T20:37:28.122 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:27 smithi070 ceph-mon[100755]: rocksdb: Options.allow_mmap_reads: 0 2023-09-23T20:37:28.122 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:27 smithi070 ceph-mon[100755]: rocksdb: Options.allow_mmap_writes: 0 2023-09-23T20:37:28.122 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:27 smithi070 ceph-mon[100755]: rocksdb: Options.use_direct_reads: 0 2023-09-23T20:37:28.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:27 smithi070 ceph-mon[100755]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2023-09-23T20:37:28.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:27 smithi070 ceph-mon[100755]: rocksdb: Options.create_missing_column_families: 0 2023-09-23T20:37:28.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:27 smithi070 ceph-mon[100755]: rocksdb: Options.db_log_dir: 2023-09-23T20:37:28.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:27 smithi070 ceph-mon[100755]: rocksdb: Options.wal_dir: /var/lib/ceph/mon/ceph-b/store.db 2023-09-23T20:37:28.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:27 smithi070 ceph-mon[100755]: rocksdb: Options.table_cache_numshardbits: 6 2023-09-23T20:37:28.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:27 smithi070 ceph-mon[100755]: rocksdb: Options.WAL_ttl_seconds: 0 2023-09-23T20:37:28.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:27 smithi070 ceph-mon[100755]: rocksdb: Options.WAL_size_limit_MB: 0 2023-09-23T20:37:28.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:27 smithi070 ceph-mon[100755]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2023-09-23T20:37:28.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:27 smithi070 ceph-mon[100755]: rocksdb: Options.manifest_preallocation_size: 4194304 2023-09-23T20:37:28.125 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:27 smithi070 ceph-mon[100755]: rocksdb: Options.is_fd_close_on_exec: 1 2023-09-23T20:37:28.125 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:27 smithi070 ceph-mon[100755]: rocksdb: Options.advise_random_on_open: 1 2023-09-23T20:37:28.125 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:27 smithi070 ceph-mon[100755]: rocksdb: Options.db_write_buffer_size: 0 2023-09-23T20:37:28.125 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:27 smithi070 ceph-mon[100755]: rocksdb: Options.write_buffer_manager: 0x564c9a496960 2023-09-23T20:37:28.125 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:27 smithi070 ceph-mon[100755]: rocksdb: Options.access_hint_on_compaction_start: 1 2023-09-23T20:37:28.125 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:27 smithi070 ceph-mon[100755]: rocksdb: Options.new_table_reader_for_compaction_inputs: 0 2023-09-23T20:37:28.125 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:27 smithi070 ceph-mon[100755]: rocksdb: Options.random_access_max_buffer_size: 1048576 2023-09-23T20:37:28.125 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:27 smithi070 ceph-mon[100755]: rocksdb: Options.use_adaptive_mutex: 0 2023-09-23T20:37:28.125 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:27 smithi070 ceph-mon[100755]: rocksdb: Options.rate_limiter: (nil) 2023-09-23T20:37:28.125 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:27 smithi070 ceph-mon[100755]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2023-09-23T20:37:28.125 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:27 smithi070 ceph-mon[100755]: rocksdb: Options.wal_recovery_mode: 2 2023-09-23T20:37:28.125 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:27 smithi070 ceph-mon[100755]: rocksdb: Options.enable_thread_tracking: 0 2023-09-23T20:37:28.126 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:27 smithi070 ceph-mon[100755]: rocksdb: Options.enable_pipelined_write: 0 2023-09-23T20:37:28.126 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:27 smithi070 ceph-mon[100755]: rocksdb: Options.unordered_write: 0 2023-09-23T20:37:28.126 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:27 smithi070 ceph-mon[100755]: rocksdb: Options.allow_concurrent_memtable_write: 1 2023-09-23T20:37:28.126 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:27 smithi070 ceph-mon[100755]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2023-09-23T20:37:28.126 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:27 smithi070 ceph-mon[100755]: rocksdb: Options.write_thread_max_yield_usec: 100 2023-09-23T20:37:28.126 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:27 smithi070 ceph-mon[100755]: rocksdb: Options.write_thread_slow_yield_usec: 3 2023-09-23T20:37:28.126 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:27 smithi070 ceph-mon[100755]: rocksdb: Options.row_cache: None 2023-09-23T20:37:28.126 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:27 smithi070 ceph-mon[100755]: rocksdb: Options.wal_filter: None 2023-09-23T20:37:28.126 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:27 smithi070 ceph-mon[100755]: rocksdb: Options.avoid_flush_during_recovery: 0 2023-09-23T20:37:28.126 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:27 smithi070 ceph-mon[100755]: rocksdb: Options.allow_ingest_behind: 0 2023-09-23T20:37:28.126 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:27 smithi070 ceph-mon[100755]: rocksdb: Options.preserve_deletes: 0 2023-09-23T20:37:28.126 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:27 smithi070 ceph-mon[100755]: rocksdb: Options.two_write_queues: 0 2023-09-23T20:37:28.126 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:27 smithi070 ceph-mon[100755]: rocksdb: Options.manual_wal_flush: 0 2023-09-23T20:37:28.127 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:27 smithi070 ceph-mon[100755]: rocksdb: Options.atomic_flush: 0 2023-09-23T20:37:28.127 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:27 smithi070 ceph-mon[100755]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2023-09-23T20:37:28.127 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:27 smithi070 ceph-mon[100755]: rocksdb: Options.persist_stats_to_disk: 0 2023-09-23T20:37:28.127 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:27 smithi070 ceph-mon[100755]: rocksdb: Options.write_dbid_to_manifest: 0 2023-09-23T20:37:28.127 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:27 smithi070 ceph-mon[100755]: rocksdb: Options.log_readahead_size: 0 2023-09-23T20:37:28.127 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:27 smithi070 ceph-mon[100755]: rocksdb: Options.file_checksum_gen_factory: Unknown 2023-09-23T20:37:28.127 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:27 smithi070 ceph-mon[100755]: rocksdb: Options.best_efforts_recovery: 0 2023-09-23T20:37:28.127 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:27 smithi070 ceph-mon[100755]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2023-09-23T20:37:28.127 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:27 smithi070 ceph-mon[100755]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2023-09-23T20:37:28.127 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:27 smithi070 ceph-mon[100755]: rocksdb: Options.allow_data_in_errors: 0 2023-09-23T20:37:28.127 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:27 smithi070 ceph-mon[100755]: rocksdb: Options.db_host_id: __hostname__ 2023-09-23T20:37:28.127 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:27 smithi070 ceph-mon[100755]: rocksdb: Options.max_background_jobs: 2 2023-09-23T20:37:28.127 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:27 smithi070 ceph-mon[100755]: rocksdb: Options.max_background_compactions: -1 2023-09-23T20:37:28.128 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:27 smithi070 ceph-mon[100755]: rocksdb: Options.max_subcompactions: 1 2023-09-23T20:37:28.128 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:27 smithi070 ceph-mon[100755]: rocksdb: Options.avoid_flush_during_shutdown: 0 2023-09-23T20:37:28.128 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:27 smithi070 ceph-mon[100755]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2023-09-23T20:37:28.128 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:27 smithi070 ceph-mon[100755]: rocksdb: Options.delayed_write_rate : 16777216 2023-09-23T20:37:28.128 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:27 smithi070 ceph-mon[100755]: rocksdb: Options.max_total_wal_size: 0 2023-09-23T20:37:28.128 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:27 smithi070 ceph-mon[100755]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2023-09-23T20:37:28.128 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:27 smithi070 ceph-mon[100755]: rocksdb: Options.stats_dump_period_sec: 600 2023-09-23T20:37:28.128 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:27 smithi070 ceph-mon[100755]: rocksdb: Options.stats_persist_period_sec: 600 2023-09-23T20:37:28.128 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:27 smithi070 ceph-mon[100755]: rocksdb: Options.stats_history_buffer_size: 1048576 2023-09-23T20:37:28.128 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:27 smithi070 ceph-mon[100755]: rocksdb: Options.max_open_files: -1 2023-09-23T20:37:28.128 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:27 smithi070 ceph-mon[100755]: rocksdb: Options.bytes_per_sync: 0 2023-09-23T20:37:28.128 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:27 smithi070 ceph-mon[100755]: rocksdb: Options.wal_bytes_per_sync: 0 2023-09-23T20:37:28.129 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:27 smithi070 ceph-mon[100755]: rocksdb: Options.strict_bytes_per_sync: 0 2023-09-23T20:37:28.129 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:27 smithi070 ceph-mon[100755]: rocksdb: Options.compaction_readahead_size: 0 2023-09-23T20:37:28.129 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:27 smithi070 ceph-mon[100755]: rocksdb: Options.max_background_flushes: -1 2023-09-23T20:37:28.129 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:27 smithi070 ceph-mon[100755]: rocksdb: Compression algorithms supported: 2023-09-23T20:37:28.129 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:27 smithi070 ceph-mon[100755]: rocksdb: kZSTDNotFinalCompression supported: 0 2023-09-23T20:37:28.129 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:27 smithi070 ceph-mon[100755]: rocksdb: kZSTD supported: 0 2023-09-23T20:37:28.129 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:27 smithi070 ceph-mon[100755]: rocksdb: kXpressCompression supported: 0 2023-09-23T20:37:28.129 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:27 smithi070 ceph-mon[100755]: rocksdb: kLZ4HCCompression supported: 1 2023-09-23T20:37:28.129 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:27 smithi070 ceph-mon[100755]: rocksdb: kZlibCompression supported: 1 2023-09-23T20:37:28.129 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:27 smithi070 ceph-mon[100755]: rocksdb: kSnappyCompression supported: 1 2023-09-23T20:37:28.129 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:27 smithi070 ceph-mon[100755]: rocksdb: kLZ4Compression supported: 1 2023-09-23T20:37:28.129 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:27 smithi070 ceph-mon[100755]: rocksdb: kBZip2Compression supported: 0 2023-09-23T20:37:28.129 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:27 smithi070 ceph-mon[100755]: rocksdb: Fast CRC32 supported: Supported on x86 2023-09-23T20:37:28.129 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:27 smithi070 ceph-mon[100755]: rocksdb: [db/version_set.cc:4724] Recovering from manifest file: /var/lib/ceph/mon/ceph-b/store.db/MANIFEST-000003 2023-09-23T20:37:28.130 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:27 smithi070 ceph-mon[100755]: rocksdb: [db/column_family.cc:595] --------------- Options for column family [default]: 2023-09-23T20:37:28.130 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:27 smithi070 ceph-mon[100755]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2023-09-23T20:37:28.130 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:27 smithi070 ceph-mon[100755]: rocksdb: Options.merge_operator: 2023-09-23T20:37:28.130 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:27 smithi070 ceph-mon[100755]: rocksdb: Options.compaction_filter: None 2023-09-23T20:37:28.130 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:27 smithi070 ceph-mon[100755]: rocksdb: Options.compaction_filter_factory: None 2023-09-23T20:37:28.130 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:27 smithi070 ceph-mon[100755]: rocksdb: Options.sst_partitioner_factory: None 2023-09-23T20:37:28.130 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:27 smithi070 ceph-mon[100755]: rocksdb: Options.memtable_factory: SkipListFactory 2023-09-23T20:37:28.130 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:27 smithi070 ceph-mon[100755]: rocksdb: Options.table_factory: BlockBasedTable 2023-09-23T20:37:28.130 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:27 smithi070 ceph-mon[100755]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x564c99776fa0) 2023-09-23T20:37:28.130 INFO:journalctl@ceph.mon.b.smithi070.stdout: cache_index_and_filter_blocks: 1 2023-09-23T20:37:28.130 INFO:journalctl@ceph.mon.b.smithi070.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2023-09-23T20:37:28.130 INFO:journalctl@ceph.mon.b.smithi070.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2023-09-23T20:37:28.131 INFO:journalctl@ceph.mon.b.smithi070.stdout: pin_top_level_index_and_filter: 1 2023-09-23T20:37:28.131 INFO:journalctl@ceph.mon.b.smithi070.stdout: index_type: 0 2023-09-23T20:37:28.131 INFO:journalctl@ceph.mon.b.smithi070.stdout: data_block_index_type: 0 2023-09-23T20:37:28.131 INFO:journalctl@ceph.mon.b.smithi070.stdout: index_shortening: 1 2023-09-23T20:37:28.131 INFO:journalctl@ceph.mon.b.smithi070.stdout: data_block_hash_table_util_ratio: 0.750000 2023-09-23T20:37:28.131 INFO:journalctl@ceph.mon.b.smithi070.stdout: hash_index_allow_collision: 1 2023-09-23T20:37:28.131 INFO:journalctl@ceph.mon.b.smithi070.stdout: checksum: 1 2023-09-23T20:37:28.131 INFO:journalctl@ceph.mon.b.smithi070.stdout: no_block_cache: 0 2023-09-23T20:37:28.131 INFO:journalctl@ceph.mon.b.smithi070.stdout: block_cache: 0x564c997a9090 2023-09-23T20:37:28.131 INFO:journalctl@ceph.mon.b.smithi070.stdout: block_cache_name: BinnedLRUCache 2023-09-23T20:37:28.131 INFO:journalctl@ceph.mon.b.smithi070.stdout: block_cache_options: 2023-09-23T20:37:28.131 INFO:journalctl@ceph.mon.b.smithi070.stdout: capacity : 536870912 2023-09-23T20:37:28.132 INFO:journalctl@ceph.mon.b.smithi070.stdout: num_shard_bits : 4 2023-09-23T20:37:28.132 INFO:journalctl@ceph.mon.b.smithi070.stdout: strict_capacity_limit : 0 2023-09-23T20:37:28.132 INFO:journalctl@ceph.mon.b.smithi070.stdout: high_pri_pool_ratio: 0.000 2023-09-23T20:37:28.132 INFO:journalctl@ceph.mon.b.smithi070.stdout: block_cache_compressed: (nil) 2023-09-23T20:37:28.132 INFO:journalctl@ceph.mon.b.smithi070.stdout: persistent_cache: (nil) 2023-09-23T20:37:28.132 INFO:journalctl@ceph.mon.b.smithi070.stdout: block_size: 4096 2023-09-23T20:37:28.132 INFO:journalctl@ceph.mon.b.smithi070.stdout: block_size_deviation: 10 2023-09-23T20:37:28.132 INFO:journalctl@ceph.mon.b.smithi070.stdout: block_restart_interval: 16 2023-09-23T20:37:28.132 INFO:journalctl@ceph.mon.b.smithi070.stdout: index_block_restart_interval: 1 2023-09-23T20:37:28.132 INFO:journalctl@ceph.mon.b.smithi070.stdout: metadata_block_size: 4096 2023-09-23T20:37:28.132 INFO:journalctl@ceph.mon.b.smithi070.stdout: partition_filters: 0 2023-09-23T20:37:28.132 INFO:journalctl@ceph.mon.b.smithi070.stdout: use_delta_encoding: 1 2023-09-23T20:37:28.132 INFO:journalctl@ceph.mon.b.smithi070.stdout: filter_policy: rocksdb.BuiltinBloomFilter 2023-09-23T20:37:28.133 INFO:journalctl@ceph.mon.b.smithi070.stdout: whole_key_filtering: 1 2023-09-23T20:37:28.133 INFO:journalctl@ceph.mon.b.smithi070.stdout: verify_compression: 0 2023-09-23T20:37:28.133 INFO:journalctl@ceph.mon.b.smithi070.stdout: read_amp_bytes_per_bit: 0 2023-09-23T20:37:28.133 INFO:journalctl@ceph.mon.b.smithi070.stdout: format_version: 4 2023-09-23T20:37:28.133 INFO:journalctl@ceph.mon.b.smithi070.stdout: enable_index_compression: 1 2023-09-23T20:37:28.133 INFO:journalctl@ceph.mon.b.smithi070.stdout: block_align: 0 2023-09-23T20:37:28.133 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:27 smithi070 ceph-mon[100755]: rocksdb: Options.write_buffer_size: 33554432 2023-09-23T20:37:28.133 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:27 smithi070 ceph-mon[100755]: rocksdb: Options.max_write_buffer_number: 2 2023-09-23T20:37:28.133 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:27 smithi070 ceph-mon[100755]: rocksdb: Options.compression: NoCompression 2023-09-23T20:37:28.133 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:27 smithi070 ceph-mon[100755]: rocksdb: Options.bottommost_compression: Disabled 2023-09-23T20:37:28.133 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:27 smithi070 ceph-mon[100755]: rocksdb: Options.prefix_extractor: nullptr 2023-09-23T20:37:28.133 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:27 smithi070 ceph-mon[100755]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2023-09-23T20:37:28.133 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:27 smithi070 ceph-mon[100755]: rocksdb: Options.num_levels: 7 2023-09-23T20:37:28.134 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:27 smithi070 bash[100691]: 2c8a2924d3d7c05a42d5d43cab674cda783d7a232a49fabfd6f04a2470cff550 2023-09-23T20:37:28.134 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:27 smithi070 ceph-mon[100755]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2023-09-23T20:37:28.134 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:27 smithi070 ceph-mon[100755]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2023-09-23T20:37:28.134 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:27 smithi070 ceph-mon[100755]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2023-09-23T20:37:28.134 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:27 smithi070 ceph-mon[100755]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2023-09-23T20:37:28.134 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:27 smithi070 ceph-mon[100755]: rocksdb: Options.bottommost_compression_opts.level: 32767 2023-09-23T20:37:28.134 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:27 smithi070 ceph-mon[100755]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2023-09-23T20:37:28.134 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:27 smithi070 ceph-mon[100755]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2023-09-23T20:37:28.134 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:27 smithi070 ceph-mon[100755]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2023-09-23T20:37:28.134 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:27 smithi070 ceph-mon[100755]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2023-09-23T20:37:28.134 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:27 smithi070 ceph-mon[100755]: rocksdb: Options.bottommost_compression_opts.enabled: false 2023-09-23T20:37:28.134 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:27 smithi070 ceph-mon[100755]: rocksdb: Options.compression_opts.window_bits: -14 2023-09-23T20:37:28.135 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:27 smithi070 ceph-mon[100755]: rocksdb: Options.compression_opts.level: 32767 2023-09-23T20:37:28.135 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:27 smithi070 ceph-mon[100755]: rocksdb: Options.compression_opts.strategy: 0 2023-09-23T20:37:28.135 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:27 smithi070 ceph-mon[100755]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2023-09-23T20:37:28.135 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:27 smithi070 ceph-mon[100755]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2023-09-23T20:37:28.135 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:27 smithi070 ceph-mon[100755]: rocksdb: Options.compression_opts.parallel_threads: 1 2023-09-23T20:37:28.135 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:27 smithi070 ceph-mon[100755]: rocksdb: Options.compression_opts.enabled: false 2023-09-23T20:37:28.135 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:27 smithi070 ceph-mon[100755]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2023-09-23T20:37:28.135 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:27 smithi070 ceph-mon[100755]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2023-09-23T20:37:28.135 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:27 smithi070 ceph-mon[100755]: rocksdb: Options.level0_stop_writes_trigger: 36 2023-09-23T20:37:28.135 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:27 smithi070 ceph-mon[100755]: rocksdb: Options.target_file_size_base: 67108864 2023-09-23T20:37:28.135 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:27 smithi070 ceph-mon[100755]: rocksdb: Options.target_file_size_multiplier: 1 2023-09-23T20:37:28.135 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:27 smithi070 ceph-mon[100755]: rocksdb: Options.max_bytes_for_level_base: 268435456 2023-09-23T20:37:28.135 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:27 smithi070 ceph-mon[100755]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2023-09-23T20:37:28.136 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:27 smithi070 ceph-mon[100755]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2023-09-23T20:37:28.136 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:27 smithi070 ceph-mon[100755]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2023-09-23T20:37:28.136 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:27 smithi070 ceph-mon[100755]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2023-09-23T20:37:28.136 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:27 smithi070 ceph-mon[100755]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2023-09-23T20:37:28.136 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:27 smithi070 ceph-mon[100755]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2023-09-23T20:37:28.136 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:27 smithi070 ceph-mon[100755]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2023-09-23T20:37:28.136 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:27 smithi070 ceph-mon[100755]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2023-09-23T20:37:28.136 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:27 smithi070 ceph-mon[100755]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2023-09-23T20:37:28.136 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:27 smithi070 ceph-mon[100755]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2023-09-23T20:37:28.136 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:27 smithi070 ceph-mon[100755]: rocksdb: Options.max_compaction_bytes: 1677721600 2023-09-23T20:37:28.136 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:27 smithi070 ceph-mon[100755]: rocksdb: Options.arena_block_size: 4194304 2023-09-23T20:37:28.136 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:27 smithi070 ceph-mon[100755]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2023-09-23T20:37:28.136 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:27 smithi070 ceph-mon[100755]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2023-09-23T20:37:28.137 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:27 smithi070 ceph-mon[100755]: rocksdb: Options.rate_limit_delay_max_milliseconds: 100 2023-09-23T20:37:28.137 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:27 smithi070 ceph-mon[100755]: rocksdb: Options.disable_auto_compactions: 0 2023-09-23T20:37:28.137 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:27 smithi070 ceph-mon[100755]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2023-09-23T20:37:28.137 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:27 smithi070 ceph-mon[100755]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2023-09-23T20:37:28.137 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:27 smithi070 ceph-mon[100755]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2023-09-23T20:37:28.137 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:27 smithi070 ceph-mon[100755]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2023-09-23T20:37:28.137 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:27 smithi070 ceph-mon[100755]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2023-09-23T20:37:28.137 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:27 smithi070 ceph-mon[100755]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2023-09-23T20:37:28.137 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:27 smithi070 ceph-mon[100755]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2023-09-23T20:37:28.137 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:27 smithi070 ceph-mon[100755]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2023-09-23T20:37:28.137 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:27 smithi070 ceph-mon[100755]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2023-09-23T20:37:28.137 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:27 smithi070 ceph-mon[100755]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2023-09-23T20:37:28.138 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:27 smithi070 ceph-mon[100755]: rocksdb: Options.table_properties_collectors: 2023-09-23T20:37:28.138 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:27 smithi070 ceph-mon[100755]: rocksdb: Options.inplace_update_support: 0 2023-09-23T20:37:28.138 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:27 smithi070 ceph-mon[100755]: rocksdb: Options.inplace_update_num_locks: 10000 2023-09-23T20:37:28.138 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:27 smithi070 ceph-mon[100755]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2023-09-23T20:37:28.138 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:27 smithi070 ceph-mon[100755]: rocksdb: Options.memtable_whole_key_filtering: 0 2023-09-23T20:37:28.138 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:27 smithi070 ceph-mon[100755]: rocksdb: Options.memtable_huge_page_size: 0 2023-09-23T20:37:28.138 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:27 smithi070 ceph-mon[100755]: rocksdb: Options.bloom_locality: 0 2023-09-23T20:37:28.138 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:27 smithi070 ceph-mon[100755]: rocksdb: Options.max_successive_merges: 0 2023-09-23T20:37:28.138 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:27 smithi070 ceph-mon[100755]: rocksdb: Options.optimize_filters_for_hits: 0 2023-09-23T20:37:28.138 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:27 smithi070 ceph-mon[100755]: rocksdb: Options.paranoid_file_checks: 0 2023-09-23T20:37:28.138 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:27 smithi070 ceph-mon[100755]: rocksdb: Options.force_consistency_checks: 1 2023-09-23T20:37:28.138 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:27 smithi070 ceph-mon[100755]: rocksdb: Options.report_bg_io_stats: 0 2023-09-23T20:37:28.139 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:27 smithi070 ceph-mon[100755]: rocksdb: Options.ttl: 2592000 2023-09-23T20:37:28.139 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:27 smithi070 ceph-mon[100755]: rocksdb: Options.periodic_compaction_seconds: 0 2023-09-23T20:37:28.139 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:27 smithi070 ceph-mon[100755]: rocksdb: Options.enable_blob_files: false 2023-09-23T20:37:28.139 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:27 smithi070 ceph-mon[100755]: rocksdb: Options.min_blob_size: 0 2023-09-23T20:37:28.139 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:27 smithi070 ceph-mon[100755]: rocksdb: Options.blob_file_size: 268435456 2023-09-23T20:37:28.139 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:27 smithi070 ceph-mon[100755]: rocksdb: Options.blob_compression_type: NoCompression 2023-09-23T20:37:28.139 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:27 smithi070 ceph-mon[100755]: rocksdb: Options.enable_blob_garbage_collection: false 2023-09-23T20:37:28.139 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:27 smithi070 ceph-mon[100755]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2023-09-23T20:37:28.139 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:27 smithi070 ceph-mon[100755]: rocksdb: [db/version_set.cc:4764] Recovered from manifest file:/var/lib/ceph/mon/ceph-b/store.db/MANIFEST-000003 succeeded,manifest_file_number is 3, next_file_number is 5, last_sequence is 0, log_number is 0,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2023-09-23T20:37:28.139 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:27 smithi070 ceph-mon[100755]: rocksdb: [db/version_set.cc:4779] Column family [default] (ID 0), log number is 0 2023-09-23T20:37:28.140 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:27 smithi070 ceph-mon[100755]: rocksdb: [db/version_set.cc:4082] Creating manifest 7 2023-09-23T20:37:28.140 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:27 smithi070 ceph-mon[100755]: rocksdb: EVENT_LOG_v1 {"time_micros": 1695501447651565, "job": 1, "event": "recovery_started", "wal_files": [4]} 2023-09-23T20:37:28.140 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:27 smithi070 ceph-mon[100755]: rocksdb: [db/db_impl/db_impl_open.cc:845] Recovering log #4 mode 2 2023-09-23T20:37:28.140 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:27 smithi070 ceph-mon[100755]: rocksdb: [table/block_based/filter_policy.cc:991] Using legacy Bloom filter with high (20) bits/key. Dramatic filter space and/or accuracy improvement is available with format_version>=5. 2023-09-23T20:37:28.140 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:27 smithi070 ceph-mon[100755]: rocksdb: EVENT_LOG_v1 {"time_micros": 1695501447651829, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 8, "file_size": 1545, "file_checksum": "", "file_checksum_func_name": "Unknown", "table_properties": {"data_size": 523, "index_size": 31, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 69, "raw_key_size": 115, "raw_average_key_size": 23, "raw_value_size": 401, "raw_average_value_size": 80, "num_data_blocks": 1, "num_entries": 5, "num_deletions": 0, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "rocksdb.BuiltinBloomFilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; ", "creation_time": 1695501447, "oldest_key_time": 0, "file_creation_time": 0, "db_id": "e8f18e04-a51b-4277-8901-73c605241e6c", "db_session_id": "QYGHN1WK7G7M7IQKU7KN"}} 2023-09-23T20:37:28.140 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:27 smithi070 ceph-mon[100755]: rocksdb: [db/version_set.cc:4082] Creating manifest 9 2023-09-23T20:37:28.140 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:27 smithi070 ceph-mon[100755]: rocksdb: EVENT_LOG_v1 {"time_micros": 1695501447652142, "job": 1, "event": "recovery_finished"} 2023-09-23T20:37:28.140 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:27 smithi070 ceph-mon[100755]: rocksdb: [file/delete_scheduler.cc:69] Deleted file /var/lib/ceph/mon/ceph-b/store.db/000004.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2023-09-23T20:37:28.140 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:27 smithi070 ceph-mon[100755]: rocksdb: [db/db_impl/db_impl_open.cc:1700] SstFileManager instance 0x564c9973d180 2023-09-23T20:37:28.140 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:27 smithi070 ceph-mon[100755]: rocksdb: DB pointer 0x564c99812000 2023-09-23T20:37:28.140 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:27 smithi070 ceph-mon[100755]: rocksdb: [db/db_impl/db_impl.cc:901] ------- DUMPING STATS ------- 2023-09-23T20:37:28.140 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:27 smithi070 ceph-mon[100755]: rocksdb: [db/db_impl/db_impl.cc:903] 2023-09-23T20:37:28.140 INFO:journalctl@ceph.mon.b.smithi070.stdout: ** DB Stats ** 2023-09-23T20:37:28.141 INFO:journalctl@ceph.mon.b.smithi070.stdout: Uptime(secs): 0.0 total, 0.0 interval 2023-09-23T20:37:28.141 INFO:journalctl@ceph.mon.b.smithi070.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2023-09-23T20:37:28.141 INFO:journalctl@ceph.mon.b.smithi070.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2023-09-23T20:37:28.141 INFO:journalctl@ceph.mon.b.smithi070.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2023-09-23T20:37:28.141 INFO:journalctl@ceph.mon.b.smithi070.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2023-09-23T20:37:28.141 INFO:journalctl@ceph.mon.b.smithi070.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 MB, 0.00 MB/s 2023-09-23T20:37:28.141 INFO:journalctl@ceph.mon.b.smithi070.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2023-09-23T20:37:28.141 INFO:journalctl@ceph.mon.b.smithi070.stdout: 2023-09-23T20:37:28.141 INFO:journalctl@ceph.mon.b.smithi070.stdout: ** Compaction Stats [default] ** 2023-09-23T20:37:28.141 INFO:journalctl@ceph.mon.b.smithi070.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-09-23T20:37:28.141 INFO:journalctl@ceph.mon.b.smithi070.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-09-23T20:37:28.141 INFO:journalctl@ceph.mon.b.smithi070.stdout: L0 1/0 1.51 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 6.0 0.00 0.00 1 0.000 0 0 2023-09-23T20:37:28.141 INFO:journalctl@ceph.mon.b.smithi070.stdout: Sum 1/0 1.51 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 6.0 0.00 0.00 1 0.000 0 0 2023-09-23T20:37:28.142 INFO:journalctl@ceph.mon.b.smithi070.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 6.0 0.00 0.00 1 0.000 0 0 2023-09-23T20:37:28.142 INFO:journalctl@ceph.mon.b.smithi070.stdout: 2023-09-23T20:37:28.142 INFO:journalctl@ceph.mon.b.smithi070.stdout: ** Compaction Stats [default] ** 2023-09-23T20:37:28.142 INFO:journalctl@ceph.mon.b.smithi070.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-09-23T20:37:28.142 INFO:journalctl@ceph.mon.b.smithi070.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-09-23T20:37:28.142 INFO:journalctl@ceph.mon.b.smithi070.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 6.0 0.00 0.00 1 0.000 0 0 2023-09-23T20:37:28.142 INFO:journalctl@ceph.mon.b.smithi070.stdout: Uptime(secs): 0.0 total, 0.0 interval 2023-09-23T20:37:28.142 INFO:journalctl@ceph.mon.b.smithi070.stdout: Flush(GB): cumulative 0.000, interval 0.000 2023-09-23T20:37:28.142 INFO:journalctl@ceph.mon.b.smithi070.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2023-09-23T20:37:28.142 INFO:journalctl@ceph.mon.b.smithi070.stdout: AddFile(Total Files): cumulative 0, interval 0 2023-09-23T20:37:28.142 INFO:journalctl@ceph.mon.b.smithi070.stdout: AddFile(L0 Files): cumulative 0, interval 0 2023-09-23T20:37:28.142 INFO:journalctl@ceph.mon.b.smithi070.stdout: AddFile(Keys): cumulative 0, interval 0 2023-09-23T20:37:28.143 INFO:journalctl@ceph.mon.b.smithi070.stdout: Cumulative compaction: 0.00 GB write, 0.59 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-09-23T20:37:28.143 INFO:journalctl@ceph.mon.b.smithi070.stdout: Interval compaction: 0.00 GB write, 0.59 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-09-23T20:37:28.143 INFO:journalctl@ceph.mon.b.smithi070.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2023-09-23T20:37:28.143 INFO:journalctl@ceph.mon.b.smithi070.stdout: 2023-09-23T20:37:28.143 INFO:journalctl@ceph.mon.b.smithi070.stdout: ** File Read Latency Histogram By Level [default] ** 2023-09-23T20:37:28.143 INFO:journalctl@ceph.mon.b.smithi070.stdout: 2023-09-23T20:37:28.143 INFO:journalctl@ceph.mon.b.smithi070.stdout: ** Compaction Stats [default] ** 2023-09-23T20:37:28.143 INFO:journalctl@ceph.mon.b.smithi070.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-09-23T20:37:28.143 INFO:journalctl@ceph.mon.b.smithi070.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-09-23T20:37:28.143 INFO:journalctl@ceph.mon.b.smithi070.stdout: L0 1/0 1.51 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 6.0 0.00 0.00 1 0.000 0 0 2023-09-23T20:37:28.143 INFO:journalctl@ceph.mon.b.smithi070.stdout: Sum 1/0 1.51 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 6.0 0.00 0.00 1 0.000 0 0 2023-09-23T20:37:28.144 INFO:journalctl@ceph.mon.b.smithi070.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2023-09-23T20:37:28.144 INFO:journalctl@ceph.mon.b.smithi070.stdout: 2023-09-23T20:37:28.144 INFO:journalctl@ceph.mon.b.smithi070.stdout: ** Compaction Stats [default] ** 2023-09-23T20:37:28.144 INFO:journalctl@ceph.mon.b.smithi070.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-09-23T20:37:28.144 INFO:journalctl@ceph.mon.b.smithi070.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-09-23T20:37:28.144 INFO:journalctl@ceph.mon.b.smithi070.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 6.0 0.00 0.00 1 0.000 0 0 2023-09-23T20:37:28.144 INFO:journalctl@ceph.mon.b.smithi070.stdout: Uptime(secs): 0.0 total, 0.0 interval 2023-09-23T20:37:28.144 INFO:journalctl@ceph.mon.b.smithi070.stdout: Flush(GB): cumulative 0.000, interval 0.000 2023-09-23T20:37:28.144 INFO:journalctl@ceph.mon.b.smithi070.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2023-09-23T20:37:28.144 INFO:journalctl@ceph.mon.b.smithi070.stdout: AddFile(Total Files): cumulative 0, interval 0 2023-09-23T20:37:28.144 INFO:journalctl@ceph.mon.b.smithi070.stdout: AddFile(L0 Files): cumulative 0, interval 0 2023-09-23T20:37:28.144 INFO:journalctl@ceph.mon.b.smithi070.stdout: AddFile(Keys): cumulative 0, interval 0 2023-09-23T20:37:28.144 INFO:journalctl@ceph.mon.b.smithi070.stdout: Cumulative compaction: 0.00 GB write, 0.58 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-09-23T20:37:28.145 INFO:journalctl@ceph.mon.b.smithi070.stdout: Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-09-23T20:37:28.145 INFO:journalctl@ceph.mon.b.smithi070.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2023-09-23T20:37:28.145 INFO:journalctl@ceph.mon.b.smithi070.stdout: 2023-09-23T20:37:28.145 INFO:journalctl@ceph.mon.b.smithi070.stdout: ** File Read Latency Histogram By Level [default] ** 2023-09-23T20:37:28.145 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:27 smithi070 ceph-mon[100755]: mon.b does not exist in monmap, will attempt to join an existing cluster 2023-09-23T20:37:28.145 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:27 smithi070 ceph-mon[100755]: using public_addr v2:172.21.15.70:0/0 -> [v2:172.21.15.70:3300/0,v1:172.21.15.70:6789/0] 2023-09-23T20:37:28.145 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:27 smithi070 ceph-mon[100755]: starting mon.b rank -1 at public addrs [v2:172.21.15.70:3300/0,v1:172.21.15.70:6789/0] at bind addrs [v2:172.21.15.70:3300/0,v1:172.21.15.70:6789/0] mon_data /var/lib/ceph/mon/ceph-b fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 2023-09-23T20:37:28.145 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:27 smithi070 ceph-mon[100755]: mon.b@-1(???) e0 preinit fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 2023-09-23T20:37:28.145 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:27 smithi070 ceph-mon[100755]: mon.b@-1(synchronizing).mds e1 new map 2023-09-23T20:37:28.145 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:27 smithi070 ceph-mon[100755]: mon.b@-1(synchronizing).mds e1 print_map 2023-09-23T20:37:28.145 INFO:journalctl@ceph.mon.b.smithi070.stdout: e1 2023-09-23T20:37:28.145 INFO:journalctl@ceph.mon.b.smithi070.stdout: enable_multiple, ever_enabled_multiple: 1,1 2023-09-23T20:37:28.146 INFO:journalctl@ceph.mon.b.smithi070.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2023-09-23T20:37:28.146 INFO:journalctl@ceph.mon.b.smithi070.stdout: legacy client fscid: -1 2023-09-23T20:37:28.146 INFO:journalctl@ceph.mon.b.smithi070.stdout: 2023-09-23T20:37:28.146 INFO:journalctl@ceph.mon.b.smithi070.stdout: No filesystems configured 2023-09-23T20:37:28.146 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:27 smithi070 ceph-mon[100755]: mon.b@-1(synchronizing).osd e0 _set_cache_ratios kv ratio 0.25 inc ratio 0.375 full ratio 0.375 2023-09-23T20:37:28.146 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:27 smithi070 ceph-mon[100755]: mon.b@-1(synchronizing).osd e0 register_cache_with_pcm pcm target: 2147483648 pcm max: 1020054732 pcm min: 134217728 inc_osd_cache size: 1 2023-09-23T20:37:28.146 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:27 smithi070 ceph-mon[100755]: mon.b@-1(synchronizing).osd e1 e1: 0 total, 0 up, 0 in 2023-09-23T20:37:28.146 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:27 smithi070 ceph-mon[100755]: mon.b@-1(synchronizing).osd e2 e2: 0 total, 0 up, 0 in 2023-09-23T20:37:28.146 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:27 smithi070 ceph-mon[100755]: mon.b@-1(synchronizing).osd e3 e3: 0 total, 0 up, 0 in 2023-09-23T20:37:28.146 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:27 smithi070 ceph-mon[100755]: mon.b@-1(synchronizing).osd e4 e4: 0 total, 0 up, 0 in 2023-09-23T20:37:28.146 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:27 smithi070 ceph-mon[100755]: mon.b@-1(synchronizing).osd e4 crush map has features 3314932999778484224, adjusting msgr requires 2023-09-23T20:37:28.146 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:27 smithi070 ceph-mon[100755]: mon.b@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2023-09-23T20:37:28.147 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:27 smithi070 ceph-mon[100755]: mon.b@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2023-09-23T20:37:28.147 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:27 smithi070 ceph-mon[100755]: mon.b@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2023-09-23T20:37:28.147 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:27 smithi070 ceph-mon[100755]: pgmap v5: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T20:37:28.147 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:27 smithi070 ceph-mon[100755]: from='client.? 172.21.15.26:0/1212432566' entity='client.admin' cmd=[{"prefix": "osd crush tunables", "profile": "default"}]: dispatch 2023-09-23T20:37:28.147 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:27 smithi070 ceph-mon[100755]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:37:28.147 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:27 smithi070 ceph-mon[100755]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:37:28.147 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:27 smithi070 ceph-mon[100755]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:37:28.147 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:27 smithi070 ceph-mon[100755]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:37:28.147 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:27 smithi070 ceph-mon[100755]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi070", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:37:28.147 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:27 smithi070 ceph-mon[100755]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:37:28.147 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:27 smithi070 ceph-mon[100755]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:37:28.148 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:27 smithi070 ceph-mon[100755]: Updating smithi070:/etc/ceph/ceph.conf 2023-09-23T20:37:28.148 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:27 smithi070 ceph-mon[100755]: Updating smithi070:/var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-23T20:37:28.148 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:27 smithi070 ceph-mon[100755]: Updating smithi070:/etc/ceph/ceph.client.admin.keyring 2023-09-23T20:37:28.148 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:27 smithi070 ceph-mon[100755]: from='client.? 172.21.15.26:0/1212432566' entity='client.admin' cmd='[{"prefix": "osd crush tunables", "profile": "default"}]': finished 2023-09-23T20:37:28.148 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:27 smithi070 ceph-mon[100755]: osdmap e4: 0 total, 0 up, 0 in 2023-09-23T20:37:28.148 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:27 smithi070 ceph-mon[100755]: Updating smithi070:/var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/config/ceph.client.admin.keyring 2023-09-23T20:37:28.148 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:27 smithi070 ceph-mon[100755]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:37:28.148 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:27 smithi070 ceph-mon[100755]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:37:28.148 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:27 smithi070 ceph-mon[100755]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:37:28.148 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:27 smithi070 ceph-mon[100755]: pgmap v7: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T20:37:28.148 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:27 smithi070 ceph-mon[100755]: from='client.14184 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "placement": "3;smithi026:172.21.15.26=a;smithi026:[v2:172.21.15.26:3301,v1:172.21.15.26:6790]=c;smithi070:172.21.15.70=b", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T20:37:28.148 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:27 smithi070 ceph-mon[100755]: Saving service mon spec with placement smithi026:172.21.15.26=a;smithi026:[v2:172.21.15.26:3301,v1:172.21.15.26:6790]=c;smithi070:172.21.15.70=b;count:3 2023-09-23T20:37:28.148 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:27 smithi070 ceph-mon[100755]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:37:28.149 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:27 smithi070 ceph-mon[100755]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:37:28.149 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:27 smithi070 ceph-mon[100755]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:37:28.149 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:27 smithi070 ceph-mon[100755]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:37:28.149 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:27 smithi070 ceph-mon[100755]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:37:28.149 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:27 smithi070 ceph-mon[100755]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-23T20:37:28.149 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:27 smithi070 ceph-mon[100755]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:37:28.149 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:27 smithi070 ceph-mon[100755]: Deploying daemon mon.b on smithi070 2023-09-23T20:37:28.149 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:27 smithi070 ceph-mon[100755]: pgmap v8: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T20:37:28.149 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:27 smithi070 ceph-mon[100755]: pgmap v9: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T20:37:28.149 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:27 smithi070 ceph-mon[100755]: from='client.? 172.21.15.70:0/1019581227' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-09-23T20:37:28.149 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:27 smithi070 ceph-mon[100755]: mon.b@-1(synchronizing).paxosservice(auth 1..3) refresh upgraded, format 0 -> 3 2023-09-23T20:37:28.149 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:27 smithi070 ceph-mon[100755]: mon.b@-1(synchronizing) e1 handle_conf_change mon_allow_pool_delete 2023-09-23T20:37:28.149 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:27 smithi070 systemd[1]: Started Ceph mon.b for b3f8e4ec-5a50-11ee-8db4-212e2dc638e7. 2023-09-23T20:37:28.735 INFO:tasks.cephadm:Waiting for 3 mons in monmap... 2023-09-23T20:37:28.735 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph mon dump -f json 2023-09-23T20:37:29.337 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:37:33.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:32 smithi070 ceph-mon[100755]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-23T20:37:33.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:32 smithi070 ceph-mon[100755]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T20:37:33.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:32 smithi070 ceph-mon[100755]: mon.a calling monitor election 2023-09-23T20:37:33.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:32 smithi070 ceph-mon[100755]: pgmap v10: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T20:37:33.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:32 smithi070 ceph-mon[100755]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T20:37:33.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:32 smithi070 ceph-mon[100755]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T20:37:33.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:32 smithi070 ceph-mon[100755]: mon.b calling monitor election 2023-09-23T20:37:33.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:32 smithi070 ceph-mon[100755]: pgmap v11: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T20:37:33.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:32 smithi070 ceph-mon[100755]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T20:37:33.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:32 smithi070 ceph-mon[100755]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T20:37:33.121 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:32 smithi070 ceph-mon[100755]: pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T20:37:33.121 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:32 smithi070 ceph-mon[100755]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T20:37:33.121 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:32 smithi070 ceph-mon[100755]: mon.a is new leader, mons a,b in quorum (ranks 0,1) 2023-09-23T20:37:33.121 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:32 smithi070 ceph-mon[100755]: monmap e2: 2 mons at {a=[v2:172.21.15.26:3300/0,v1:172.21.15.26:6789/0],b=[v2:172.21.15.70:3300/0,v1:172.21.15.70:6789/0]} removed_ranks: {} 2023-09-23T20:37:33.121 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:32 smithi070 ceph-mon[100755]: fsmap 2023-09-23T20:37:33.121 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:32 smithi070 ceph-mon[100755]: osdmap e4: 0 total, 0 up, 0 in 2023-09-23T20:37:33.121 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:32 smithi070 ceph-mon[100755]: mgrmap e13: y(active, since 36s) 2023-09-23T20:37:33.121 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:32 smithi070 ceph-mon[100755]: overall HEALTH_OK 2023-09-23T20:37:33.121 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:32 smithi070 ceph-mon[100755]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:37:33.121 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:32 smithi070 ceph-mon[100755]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:37:33.121 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:32 smithi070 ceph-mon[100755]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:37:33.121 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:32 smithi070 ceph-mon[100755]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-23T20:37:33.122 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:32 smithi070 ceph-mon[100755]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:37:33.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:37:32 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-23T20:37:33.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:37:32 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T20:37:33.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:37:32 smithi026 ceph-mon[95839]: mon.a calling monitor election 2023-09-23T20:37:33.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:37:32 smithi026 ceph-mon[95839]: pgmap v10: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T20:37:33.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:37:32 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T20:37:33.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:37:32 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T20:37:33.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:37:32 smithi026 ceph-mon[95839]: mon.b calling monitor election 2023-09-23T20:37:33.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:37:32 smithi026 ceph-mon[95839]: pgmap v11: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T20:37:33.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:37:32 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T20:37:33.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:37:32 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T20:37:33.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:37:32 smithi026 ceph-mon[95839]: pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T20:37:33.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:37:32 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T20:37:33.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:37:32 smithi026 ceph-mon[95839]: mon.a is new leader, mons a,b in quorum (ranks 0,1) 2023-09-23T20:37:33.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:37:32 smithi026 ceph-mon[95839]: monmap e2: 2 mons at {a=[v2:172.21.15.26:3300/0,v1:172.21.15.26:6789/0],b=[v2:172.21.15.70:3300/0,v1:172.21.15.70:6789/0]} removed_ranks: {} 2023-09-23T20:37:33.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:37:32 smithi026 ceph-mon[95839]: fsmap 2023-09-23T20:37:33.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:37:32 smithi026 ceph-mon[95839]: osdmap e4: 0 total, 0 up, 0 in 2023-09-23T20:37:33.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:37:32 smithi026 ceph-mon[95839]: mgrmap e13: y(active, since 36s) 2023-09-23T20:37:33.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:37:32 smithi026 ceph-mon[95839]: overall HEALTH_OK 2023-09-23T20:37:33.134 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:37:32 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:37:33.134 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:37:32 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:37:33.134 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:37:32 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:37:33.134 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:37:32 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-23T20:37:33.134 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:37:32 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:37:33.801 INFO:teuthology.orchestra.run.smithi070.stdout: 2023-09-23T20:37:33.801 INFO:teuthology.orchestra.run.smithi070.stdout:{"epoch":2,"fsid":"b3f8e4ec-5a50-11ee-8db4-212e2dc638e7","modified":"2023-09-23T20:37:27.671888Z","created":"2023-09-23T20:35:56.712408Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","removed_ranks: ":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"a","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:3300","nonce":0},{"type":"v1","addr":"172.21.15.26:6789","nonce":0}]},"addr":"172.21.15.26:6789/0","public_addr":"172.21.15.26:6789/0","priority":0,"weight":0,"crush_location":"{}"},{"rank":1,"name":"b","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:3300","nonce":0},{"type":"v1","addr":"172.21.15.70:6789","nonce":0}]},"addr":"172.21.15.70:6789/0","public_addr":"172.21.15.70:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0,1]} 2023-09-23T20:37:33.802 INFO:teuthology.orchestra.run.smithi070.stderr:dumped monmap epoch 2 2023-09-23T20:37:34.631 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:37:34 smithi026 ceph-mon[95839]: Deploying daemon mon.c on smithi026 2023-09-23T20:37:34.631 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:37:34 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T20:37:34.631 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:37:34 smithi026 ceph-mon[95839]: from='client.? 172.21.15.70:0/1175766312' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-09-23T20:37:34.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:34 smithi070 ceph-mon[100755]: Deploying daemon mon.c on smithi026 2023-09-23T20:37:34.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:34 smithi070 ceph-mon[100755]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T20:37:34.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:34 smithi070 ceph-mon[100755]: from='client.? 172.21.15.70:0/1175766312' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-09-23T20:37:35.199 INFO:tasks.cephadm:Waiting for 3 mons in monmap... 2023-09-23T20:37:35.200 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph mon dump -f json 2023-09-23T20:37:35.383 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:35 smithi026 systemd[1]: Starting Ceph mon.c for b3f8e4ec-5a50-11ee-8db4-212e2dc638e7... 2023-09-23T20:37:35.799 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:37:35.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:35 smithi070 ceph-mon[100755]: pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T20:37:35.881 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:37:35 smithi026 ceph-mon[95839]: pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T20:37:35.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:35 smithi026 podman[109873]: 2023-09-23T20:37:35.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:35 smithi026 ceph-mon[109912]: set uid:gid to 167:167 (ceph:ceph) 2023-09-23T20:37:35.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:35 smithi026 ceph-mon[109912]: ceph version 17.2.6-1123-g61431f6b (61431f6ba2752649b506867ec856b4a6236f90dd) quincy (stable), process ceph-mon, pid 7 2023-09-23T20:37:35.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:35 smithi026 ceph-mon[109912]: pidfile_write: ignore empty --pid-file 2023-09-23T20:37:35.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:35 smithi026 bash[109873]: 069bb324bbad559e13eaa741a1d25503db8a007784b730822f9220eb12b4c973 2023-09-23T20:37:35.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:35 smithi026 ceph-mon[109912]: load: jerasure load: lrc 2023-09-23T20:37:35.883 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:35 smithi026 ceph-mon[109912]: rocksdb: RocksDB version: 6.15.5 2023-09-23T20:37:35.883 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:35 smithi026 ceph-mon[109912]: rocksdb: Git sha rocksdb_build_git_sha:@0@ 2023-09-23T20:37:35.883 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:35 smithi026 ceph-mon[109912]: rocksdb: Compile date Sep 22 2023 2023-09-23T20:37:35.883 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:35 smithi026 ceph-mon[109912]: rocksdb: DB SUMMARY 2023-09-23T20:37:35.883 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:35 smithi026 ceph-mon[109912]: rocksdb: DB Session ID: 21M2ZAGE7BTOG6HGLU1U 2023-09-23T20:37:35.883 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:35 smithi026 ceph-mon[109912]: rocksdb: CURRENT file: CURRENT 2023-09-23T20:37:35.883 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:35 smithi026 ceph-mon[109912]: rocksdb: IDENTITY file: IDENTITY 2023-09-23T20:37:35.883 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:35 smithi026 ceph-mon[109912]: rocksdb: MANIFEST file: MANIFEST-000003 size: 57 Bytes 2023-09-23T20:37:35.883 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:35 smithi026 ceph-mon[109912]: rocksdb: SST files in /var/lib/ceph/mon/ceph-c/store.db dir, Total Num: 0, files: 2023-09-23T20:37:35.883 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:35 smithi026 ceph-mon[109912]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-c/store.db: 000004.log size: 636 ; 2023-09-23T20:37:35.883 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:35 smithi026 ceph-mon[109912]: rocksdb: Options.error_if_exists: 0 2023-09-23T20:37:35.883 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:35 smithi026 ceph-mon[109912]: rocksdb: Options.create_if_missing: 0 2023-09-23T20:37:35.884 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:35 smithi026 ceph-mon[109912]: rocksdb: Options.paranoid_checks: 1 2023-09-23T20:37:35.884 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:35 smithi026 ceph-mon[109912]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2023-09-23T20:37:35.884 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:35 smithi026 ceph-mon[109912]: rocksdb: Options.env: 0x5596c6ff9a80 2023-09-23T20:37:35.884 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:35 smithi026 ceph-mon[109912]: rocksdb: Options.fs: Posix File System 2023-09-23T20:37:35.884 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:35 smithi026 ceph-mon[109912]: rocksdb: Options.info_log: 0x5596c84f3d80 2023-09-23T20:37:35.884 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:35 smithi026 ceph-mon[109912]: rocksdb: Options.max_file_opening_threads: 16 2023-09-23T20:37:35.884 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:35 smithi026 ceph-mon[109912]: rocksdb: Options.statistics: (nil) 2023-09-23T20:37:35.884 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:35 smithi026 ceph-mon[109912]: rocksdb: Options.use_fsync: 0 2023-09-23T20:37:35.884 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:35 smithi026 ceph-mon[109912]: rocksdb: Options.max_log_file_size: 0 2023-09-23T20:37:35.884 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:35 smithi026 ceph-mon[109912]: rocksdb: Options.max_manifest_file_size: 1073741824 2023-09-23T20:37:35.884 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:35 smithi026 ceph-mon[109912]: rocksdb: Options.log_file_time_to_roll: 0 2023-09-23T20:37:35.884 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:35 smithi026 ceph-mon[109912]: rocksdb: Options.keep_log_file_num: 1000 2023-09-23T20:37:35.884 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:35 smithi026 ceph-mon[109912]: rocksdb: Options.recycle_log_file_num: 0 2023-09-23T20:37:35.885 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:35 smithi026 ceph-mon[109912]: rocksdb: Options.allow_fallocate: 1 2023-09-23T20:37:35.885 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:35 smithi026 ceph-mon[109912]: rocksdb: Options.allow_mmap_reads: 0 2023-09-23T20:37:35.885 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:35 smithi026 ceph-mon[109912]: rocksdb: Options.allow_mmap_writes: 0 2023-09-23T20:37:35.885 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:35 smithi026 ceph-mon[109912]: rocksdb: Options.use_direct_reads: 0 2023-09-23T20:37:35.885 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:35 smithi026 ceph-mon[109912]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2023-09-23T20:37:35.885 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:35 smithi026 ceph-mon[109912]: rocksdb: Options.create_missing_column_families: 0 2023-09-23T20:37:35.885 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:35 smithi026 ceph-mon[109912]: rocksdb: Options.db_log_dir: 2023-09-23T20:37:35.885 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:35 smithi026 ceph-mon[109912]: rocksdb: Options.wal_dir: /var/lib/ceph/mon/ceph-c/store.db 2023-09-23T20:37:35.885 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:35 smithi026 ceph-mon[109912]: rocksdb: Options.table_cache_numshardbits: 6 2023-09-23T20:37:35.887 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:35 smithi026 ceph-mon[109912]: rocksdb: Options.WAL_ttl_seconds: 0 2023-09-23T20:37:35.887 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:35 smithi026 ceph-mon[109912]: rocksdb: Options.WAL_size_limit_MB: 0 2023-09-23T20:37:35.887 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:35 smithi026 ceph-mon[109912]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2023-09-23T20:37:35.887 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:35 smithi026 ceph-mon[109912]: rocksdb: Options.manifest_preallocation_size: 4194304 2023-09-23T20:37:35.887 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:35 smithi026 ceph-mon[109912]: rocksdb: Options.is_fd_close_on_exec: 1 2023-09-23T20:37:35.887 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:35 smithi026 ceph-mon[109912]: rocksdb: Options.advise_random_on_open: 1 2023-09-23T20:37:35.888 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:35 smithi026 ceph-mon[109912]: rocksdb: Options.db_write_buffer_size: 0 2023-09-23T20:37:35.888 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:35 smithi026 ceph-mon[109912]: rocksdb: Options.write_buffer_manager: 0x5596c91e6930 2023-09-23T20:37:35.888 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:35 smithi026 ceph-mon[109912]: rocksdb: Options.access_hint_on_compaction_start: 1 2023-09-23T20:37:35.888 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:35 smithi026 ceph-mon[109912]: rocksdb: Options.new_table_reader_for_compaction_inputs: 0 2023-09-23T20:37:35.888 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:35 smithi026 ceph-mon[109912]: rocksdb: Options.random_access_max_buffer_size: 1048576 2023-09-23T20:37:35.888 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:35 smithi026 ceph-mon[109912]: rocksdb: Options.use_adaptive_mutex: 0 2023-09-23T20:37:35.888 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:35 smithi026 ceph-mon[109912]: rocksdb: Options.rate_limiter: (nil) 2023-09-23T20:37:35.888 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:35 smithi026 ceph-mon[109912]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2023-09-23T20:37:35.888 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:35 smithi026 ceph-mon[109912]: rocksdb: Options.wal_recovery_mode: 2 2023-09-23T20:37:35.888 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:35 smithi026 ceph-mon[109912]: rocksdb: Options.enable_thread_tracking: 0 2023-09-23T20:37:35.888 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:35 smithi026 ceph-mon[109912]: rocksdb: Options.enable_pipelined_write: 0 2023-09-23T20:37:35.888 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:35 smithi026 ceph-mon[109912]: rocksdb: Options.unordered_write: 0 2023-09-23T20:37:35.889 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:35 smithi026 ceph-mon[109912]: rocksdb: Options.allow_concurrent_memtable_write: 1 2023-09-23T20:37:35.889 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:35 smithi026 ceph-mon[109912]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2023-09-23T20:37:35.889 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:35 smithi026 ceph-mon[109912]: rocksdb: Options.write_thread_max_yield_usec: 100 2023-09-23T20:37:35.889 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:35 smithi026 ceph-mon[109912]: rocksdb: Options.write_thread_slow_yield_usec: 3 2023-09-23T20:37:35.889 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:35 smithi026 ceph-mon[109912]: rocksdb: Options.row_cache: None 2023-09-23T20:37:35.889 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:35 smithi026 ceph-mon[109912]: rocksdb: Options.wal_filter: None 2023-09-23T20:37:35.889 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:35 smithi026 ceph-mon[109912]: rocksdb: Options.avoid_flush_during_recovery: 0 2023-09-23T20:37:35.889 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:35 smithi026 ceph-mon[109912]: rocksdb: Options.allow_ingest_behind: 0 2023-09-23T20:37:35.889 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:35 smithi026 ceph-mon[109912]: rocksdb: Options.preserve_deletes: 0 2023-09-23T20:37:35.889 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:35 smithi026 ceph-mon[109912]: rocksdb: Options.two_write_queues: 0 2023-09-23T20:37:35.889 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:35 smithi026 ceph-mon[109912]: rocksdb: Options.manual_wal_flush: 0 2023-09-23T20:37:35.889 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:35 smithi026 ceph-mon[109912]: rocksdb: Options.atomic_flush: 0 2023-09-23T20:37:35.889 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:35 smithi026 ceph-mon[109912]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2023-09-23T20:37:35.890 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:35 smithi026 ceph-mon[109912]: rocksdb: Options.persist_stats_to_disk: 0 2023-09-23T20:37:35.890 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:35 smithi026 ceph-mon[109912]: rocksdb: Options.write_dbid_to_manifest: 0 2023-09-23T20:37:35.890 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:35 smithi026 ceph-mon[109912]: rocksdb: Options.log_readahead_size: 0 2023-09-23T20:37:35.890 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:35 smithi026 ceph-mon[109912]: rocksdb: Options.file_checksum_gen_factory: Unknown 2023-09-23T20:37:35.890 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:35 smithi026 ceph-mon[109912]: rocksdb: Options.best_efforts_recovery: 0 2023-09-23T20:37:35.890 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:35 smithi026 ceph-mon[109912]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2023-09-23T20:37:35.890 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:35 smithi026 ceph-mon[109912]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2023-09-23T20:37:35.890 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:35 smithi026 ceph-mon[109912]: rocksdb: Options.allow_data_in_errors: 0 2023-09-23T20:37:35.890 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:35 smithi026 ceph-mon[109912]: rocksdb: Options.db_host_id: __hostname__ 2023-09-23T20:37:35.890 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:35 smithi026 ceph-mon[109912]: rocksdb: Options.max_background_jobs: 2 2023-09-23T20:37:35.890 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:35 smithi026 ceph-mon[109912]: rocksdb: Options.max_background_compactions: -1 2023-09-23T20:37:35.890 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:35 smithi026 ceph-mon[109912]: rocksdb: Options.max_subcompactions: 1 2023-09-23T20:37:35.891 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:35 smithi026 ceph-mon[109912]: rocksdb: Options.avoid_flush_during_shutdown: 0 2023-09-23T20:37:35.891 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:35 smithi026 ceph-mon[109912]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2023-09-23T20:37:35.891 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:35 smithi026 ceph-mon[109912]: rocksdb: Options.delayed_write_rate : 16777216 2023-09-23T20:37:35.891 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:35 smithi026 ceph-mon[109912]: rocksdb: Options.max_total_wal_size: 0 2023-09-23T20:37:35.891 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:35 smithi026 ceph-mon[109912]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2023-09-23T20:37:35.891 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:35 smithi026 ceph-mon[109912]: rocksdb: Options.stats_dump_period_sec: 600 2023-09-23T20:37:35.891 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:35 smithi026 ceph-mon[109912]: rocksdb: Options.stats_persist_period_sec: 600 2023-09-23T20:37:35.891 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:35 smithi026 ceph-mon[109912]: rocksdb: Options.stats_history_buffer_size: 1048576 2023-09-23T20:37:35.891 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:35 smithi026 ceph-mon[109912]: rocksdb: Options.max_open_files: -1 2023-09-23T20:37:35.891 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:35 smithi026 ceph-mon[109912]: rocksdb: Options.bytes_per_sync: 0 2023-09-23T20:37:35.891 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:35 smithi026 ceph-mon[109912]: rocksdb: Options.wal_bytes_per_sync: 0 2023-09-23T20:37:35.891 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:35 smithi026 ceph-mon[109912]: rocksdb: Options.strict_bytes_per_sync: 0 2023-09-23T20:37:35.891 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:35 smithi026 ceph-mon[109912]: rocksdb: Options.compaction_readahead_size: 0 2023-09-23T20:37:35.892 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:35 smithi026 ceph-mon[109912]: rocksdb: Options.max_background_flushes: -1 2023-09-23T20:37:35.892 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:35 smithi026 ceph-mon[109912]: rocksdb: Compression algorithms supported: 2023-09-23T20:37:35.892 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:35 smithi026 ceph-mon[109912]: rocksdb: kZSTDNotFinalCompression supported: 0 2023-09-23T20:37:35.892 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:35 smithi026 ceph-mon[109912]: rocksdb: kZSTD supported: 0 2023-09-23T20:37:35.892 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:35 smithi026 ceph-mon[109912]: rocksdb: kXpressCompression supported: 0 2023-09-23T20:37:35.892 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:35 smithi026 ceph-mon[109912]: rocksdb: kLZ4HCCompression supported: 1 2023-09-23T20:37:35.892 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:35 smithi026 ceph-mon[109912]: rocksdb: kZlibCompression supported: 1 2023-09-23T20:37:35.892 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:35 smithi026 ceph-mon[109912]: rocksdb: kSnappyCompression supported: 1 2023-09-23T20:37:35.892 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:35 smithi026 ceph-mon[109912]: rocksdb: kLZ4Compression supported: 1 2023-09-23T20:37:35.892 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:35 smithi026 ceph-mon[109912]: rocksdb: kBZip2Compression supported: 0 2023-09-23T20:37:35.892 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:35 smithi026 ceph-mon[109912]: rocksdb: Fast CRC32 supported: Supported on x86 2023-09-23T20:37:35.892 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:35 smithi026 ceph-mon[109912]: rocksdb: [db/version_set.cc:4724] Recovering from manifest file: /var/lib/ceph/mon/ceph-c/store.db/MANIFEST-000003 2023-09-23T20:37:35.893 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:35 smithi026 ceph-mon[109912]: rocksdb: [db/column_family.cc:595] --------------- Options for column family [default]: 2023-09-23T20:37:35.893 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:35 smithi026 ceph-mon[109912]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2023-09-23T20:37:35.893 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:35 smithi026 ceph-mon[109912]: rocksdb: Options.merge_operator: 2023-09-23T20:37:35.893 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:35 smithi026 ceph-mon[109912]: rocksdb: Options.compaction_filter: None 2023-09-23T20:37:35.893 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:35 smithi026 ceph-mon[109912]: rocksdb: Options.compaction_filter_factory: None 2023-09-23T20:37:35.893 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:35 smithi026 ceph-mon[109912]: rocksdb: Options.sst_partitioner_factory: None 2023-09-23T20:37:35.893 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:35 smithi026 ceph-mon[109912]: rocksdb: Options.memtable_factory: SkipListFactory 2023-09-23T20:37:35.893 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:35 smithi026 ceph-mon[109912]: rocksdb: Options.table_factory: BlockBasedTable 2023-09-23T20:37:35.893 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:35 smithi026 ceph-mon[109912]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x5596c84c6fa0) 2023-09-23T20:37:35.893 INFO:journalctl@ceph.mon.c.smithi026.stdout: cache_index_and_filter_blocks: 1 2023-09-23T20:37:35.893 INFO:journalctl@ceph.mon.c.smithi026.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2023-09-23T20:37:35.894 INFO:journalctl@ceph.mon.c.smithi026.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2023-09-23T20:37:35.894 INFO:journalctl@ceph.mon.c.smithi026.stdout: pin_top_level_index_and_filter: 1 2023-09-23T20:37:35.894 INFO:journalctl@ceph.mon.c.smithi026.stdout: index_type: 0 2023-09-23T20:37:35.894 INFO:journalctl@ceph.mon.c.smithi026.stdout: data_block_index_type: 0 2023-09-23T20:37:35.894 INFO:journalctl@ceph.mon.c.smithi026.stdout: index_shortening: 1 2023-09-23T20:37:35.894 INFO:journalctl@ceph.mon.c.smithi026.stdout: data_block_hash_table_util_ratio: 0.750000 2023-09-23T20:37:35.894 INFO:journalctl@ceph.mon.c.smithi026.stdout: hash_index_allow_collision: 1 2023-09-23T20:37:35.894 INFO:journalctl@ceph.mon.c.smithi026.stdout: checksum: 1 2023-09-23T20:37:35.894 INFO:journalctl@ceph.mon.c.smithi026.stdout: no_block_cache: 0 2023-09-23T20:37:35.894 INFO:journalctl@ceph.mon.c.smithi026.stdout: block_cache: 0x5596c84f9090 2023-09-23T20:37:35.894 INFO:journalctl@ceph.mon.c.smithi026.stdout: block_cache_name: BinnedLRUCache 2023-09-23T20:37:35.894 INFO:journalctl@ceph.mon.c.smithi026.stdout: block_cache_options: 2023-09-23T20:37:35.895 INFO:journalctl@ceph.mon.c.smithi026.stdout: capacity : 536870912 2023-09-23T20:37:35.895 INFO:journalctl@ceph.mon.c.smithi026.stdout: num_shard_bits : 4 2023-09-23T20:37:35.895 INFO:journalctl@ceph.mon.c.smithi026.stdout: strict_capacity_limit : 0 2023-09-23T20:37:35.895 INFO:journalctl@ceph.mon.c.smithi026.stdout: high_pri_pool_ratio: 0.000 2023-09-23T20:37:35.895 INFO:journalctl@ceph.mon.c.smithi026.stdout: block_cache_compressed: (nil) 2023-09-23T20:37:35.895 INFO:journalctl@ceph.mon.c.smithi026.stdout: persistent_cache: (nil) 2023-09-23T20:37:35.895 INFO:journalctl@ceph.mon.c.smithi026.stdout: block_size: 4096 2023-09-23T20:37:35.895 INFO:journalctl@ceph.mon.c.smithi026.stdout: block_size_deviation: 10 2023-09-23T20:37:35.895 INFO:journalctl@ceph.mon.c.smithi026.stdout: block_restart_interval: 16 2023-09-23T20:37:35.895 INFO:journalctl@ceph.mon.c.smithi026.stdout: index_block_restart_interval: 1 2023-09-23T20:37:35.896 INFO:journalctl@ceph.mon.c.smithi026.stdout: metadata_block_size: 4096 2023-09-23T20:37:35.896 INFO:journalctl@ceph.mon.c.smithi026.stdout: partition_filters: 0 2023-09-23T20:37:35.896 INFO:journalctl@ceph.mon.c.smithi026.stdout: use_delta_encoding: 1 2023-09-23T20:37:35.896 INFO:journalctl@ceph.mon.c.smithi026.stdout: filter_policy: rocksdb.BuiltinBloomFilter 2023-09-23T20:37:35.896 INFO:journalctl@ceph.mon.c.smithi026.stdout: whole_key_filtering: 1 2023-09-23T20:37:35.896 INFO:journalctl@ceph.mon.c.smithi026.stdout: verify_compression: 0 2023-09-23T20:37:35.896 INFO:journalctl@ceph.mon.c.smithi026.stdout: read_amp_bytes_per_bit: 0 2023-09-23T20:37:35.896 INFO:journalctl@ceph.mon.c.smithi026.stdout: format_version: 4 2023-09-23T20:37:35.896 INFO:journalctl@ceph.mon.c.smithi026.stdout: enable_index_compression: 1 2023-09-23T20:37:35.896 INFO:journalctl@ceph.mon.c.smithi026.stdout: block_align: 0 2023-09-23T20:37:35.897 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:35 smithi026 ceph-mon[109912]: rocksdb: Options.write_buffer_size: 33554432 2023-09-23T20:37:35.897 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:35 smithi026 ceph-mon[109912]: rocksdb: Options.max_write_buffer_number: 2 2023-09-23T20:37:35.897 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:35 smithi026 ceph-mon[109912]: rocksdb: Options.compression: NoCompression 2023-09-23T20:37:35.897 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:35 smithi026 ceph-mon[109912]: rocksdb: Options.bottommost_compression: Disabled 2023-09-23T20:37:35.897 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:35 smithi026 ceph-mon[109912]: rocksdb: Options.prefix_extractor: nullptr 2023-09-23T20:37:35.897 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:35 smithi026 ceph-mon[109912]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2023-09-23T20:37:35.897 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:35 smithi026 ceph-mon[109912]: rocksdb: Options.num_levels: 7 2023-09-23T20:37:35.897 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:35 smithi026 ceph-mon[109912]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2023-09-23T20:37:35.897 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:35 smithi026 ceph-mon[109912]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2023-09-23T20:37:35.897 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:35 smithi026 ceph-mon[109912]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2023-09-23T20:37:35.897 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:35 smithi026 ceph-mon[109912]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2023-09-23T20:37:35.898 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:35 smithi026 ceph-mon[109912]: rocksdb: Options.bottommost_compression_opts.level: 32767 2023-09-23T20:37:35.898 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:35 smithi026 ceph-mon[109912]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2023-09-23T20:37:35.898 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:35 smithi026 ceph-mon[109912]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2023-09-23T20:37:35.898 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:35 smithi026 ceph-mon[109912]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2023-09-23T20:37:35.898 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:35 smithi026 ceph-mon[109912]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2023-09-23T20:37:35.898 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:35 smithi026 ceph-mon[109912]: rocksdb: Options.bottommost_compression_opts.enabled: false 2023-09-23T20:37:35.898 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:35 smithi026 ceph-mon[109912]: rocksdb: Options.compression_opts.window_bits: -14 2023-09-23T20:37:35.898 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:35 smithi026 ceph-mon[109912]: rocksdb: Options.compression_opts.level: 32767 2023-09-23T20:37:35.898 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:35 smithi026 ceph-mon[109912]: rocksdb: Options.compression_opts.strategy: 0 2023-09-23T20:37:35.898 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:35 smithi026 ceph-mon[109912]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2023-09-23T20:37:35.898 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:35 smithi026 ceph-mon[109912]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2023-09-23T20:37:35.898 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:35 smithi026 ceph-mon[109912]: rocksdb: Options.compression_opts.parallel_threads: 1 2023-09-23T20:37:35.899 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:35 smithi026 ceph-mon[109912]: rocksdb: Options.compression_opts.enabled: false 2023-09-23T20:37:35.899 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:35 smithi026 ceph-mon[109912]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2023-09-23T20:37:35.899 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:35 smithi026 ceph-mon[109912]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2023-09-23T20:37:35.899 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:35 smithi026 ceph-mon[109912]: rocksdb: Options.level0_stop_writes_trigger: 36 2023-09-23T20:37:35.899 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:35 smithi026 ceph-mon[109912]: rocksdb: Options.target_file_size_base: 67108864 2023-09-23T20:37:35.899 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:35 smithi026 ceph-mon[109912]: rocksdb: Options.target_file_size_multiplier: 1 2023-09-23T20:37:35.899 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:35 smithi026 ceph-mon[109912]: rocksdb: Options.max_bytes_for_level_base: 268435456 2023-09-23T20:37:35.899 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:35 smithi026 ceph-mon[109912]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2023-09-23T20:37:35.899 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:35 smithi026 ceph-mon[109912]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2023-09-23T20:37:35.899 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:35 smithi026 ceph-mon[109912]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2023-09-23T20:37:35.899 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:35 smithi026 ceph-mon[109912]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2023-09-23T20:37:35.900 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:35 smithi026 ceph-mon[109912]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2023-09-23T20:37:35.900 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:35 smithi026 ceph-mon[109912]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2023-09-23T20:37:35.900 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:35 smithi026 ceph-mon[109912]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2023-09-23T20:37:35.900 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:35 smithi026 ceph-mon[109912]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2023-09-23T20:37:35.900 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:35 smithi026 ceph-mon[109912]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2023-09-23T20:37:35.900 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:35 smithi026 ceph-mon[109912]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2023-09-23T20:37:35.900 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:35 smithi026 ceph-mon[109912]: rocksdb: Options.max_compaction_bytes: 1677721600 2023-09-23T20:37:35.900 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:35 smithi026 ceph-mon[109912]: rocksdb: Options.arena_block_size: 4194304 2023-09-23T20:37:35.900 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:35 smithi026 ceph-mon[109912]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2023-09-23T20:37:35.900 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:35 smithi026 ceph-mon[109912]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2023-09-23T20:37:35.900 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:35 smithi026 ceph-mon[109912]: rocksdb: Options.rate_limit_delay_max_milliseconds: 100 2023-09-23T20:37:35.900 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:35 smithi026 ceph-mon[109912]: rocksdb: Options.disable_auto_compactions: 0 2023-09-23T20:37:35.900 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:35 smithi026 ceph-mon[109912]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2023-09-23T20:37:35.901 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:35 smithi026 ceph-mon[109912]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2023-09-23T20:37:35.901 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:35 smithi026 ceph-mon[109912]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2023-09-23T20:37:35.901 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:35 smithi026 ceph-mon[109912]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2023-09-23T20:37:35.901 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:35 smithi026 ceph-mon[109912]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2023-09-23T20:37:35.901 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:35 smithi026 ceph-mon[109912]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2023-09-23T20:37:35.901 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:35 smithi026 ceph-mon[109912]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2023-09-23T20:37:35.901 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:35 smithi026 ceph-mon[109912]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2023-09-23T20:37:35.901 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:35 smithi026 ceph-mon[109912]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2023-09-23T20:37:35.901 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:35 smithi026 ceph-mon[109912]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2023-09-23T20:37:35.901 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:35 smithi026 ceph-mon[109912]: rocksdb: Options.table_properties_collectors: 2023-09-23T20:37:35.901 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:35 smithi026 ceph-mon[109912]: rocksdb: Options.inplace_update_support: 0 2023-09-23T20:37:35.901 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:35 smithi026 ceph-mon[109912]: rocksdb: Options.inplace_update_num_locks: 10000 2023-09-23T20:37:35.902 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:35 smithi026 ceph-mon[109912]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2023-09-23T20:37:35.902 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:35 smithi026 ceph-mon[109912]: rocksdb: Options.memtable_whole_key_filtering: 0 2023-09-23T20:37:35.902 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:35 smithi026 ceph-mon[109912]: rocksdb: Options.memtable_huge_page_size: 0 2023-09-23T20:37:35.902 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:35 smithi026 ceph-mon[109912]: rocksdb: Options.bloom_locality: 0 2023-09-23T20:37:35.902 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:35 smithi026 ceph-mon[109912]: rocksdb: Options.max_successive_merges: 0 2023-09-23T20:37:35.902 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:35 smithi026 ceph-mon[109912]: rocksdb: Options.optimize_filters_for_hits: 0 2023-09-23T20:37:35.902 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:35 smithi026 ceph-mon[109912]: rocksdb: Options.paranoid_file_checks: 0 2023-09-23T20:37:35.902 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:35 smithi026 ceph-mon[109912]: rocksdb: Options.force_consistency_checks: 1 2023-09-23T20:37:35.902 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:35 smithi026 ceph-mon[109912]: rocksdb: Options.report_bg_io_stats: 0 2023-09-23T20:37:35.902 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:35 smithi026 ceph-mon[109912]: rocksdb: Options.ttl: 2592000 2023-09-23T20:37:35.902 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:35 smithi026 ceph-mon[109912]: rocksdb: Options.periodic_compaction_seconds: 0 2023-09-23T20:37:35.902 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:35 smithi026 ceph-mon[109912]: rocksdb: Options.enable_blob_files: false 2023-09-23T20:37:35.903 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:35 smithi026 ceph-mon[109912]: rocksdb: Options.min_blob_size: 0 2023-09-23T20:37:35.903 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:35 smithi026 ceph-mon[109912]: rocksdb: Options.blob_file_size: 268435456 2023-09-23T20:37:35.903 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:35 smithi026 ceph-mon[109912]: rocksdb: Options.blob_compression_type: NoCompression 2023-09-23T20:37:35.903 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:35 smithi026 ceph-mon[109912]: rocksdb: Options.enable_blob_garbage_collection: false 2023-09-23T20:37:35.903 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:35 smithi026 ceph-mon[109912]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2023-09-23T20:37:35.903 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:35 smithi026 ceph-mon[109912]: rocksdb: [db/version_set.cc:4764] Recovered from manifest file:/var/lib/ceph/mon/ceph-c/store.db/MANIFEST-000003 succeeded,manifest_file_number is 3, next_file_number is 5, last_sequence is 0, log_number is 0,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2023-09-23T20:37:35.903 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:35 smithi026 ceph-mon[109912]: rocksdb: [db/version_set.cc:4779] Column family [default] (ID 0), log number is 0 2023-09-23T20:37:35.903 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:35 smithi026 ceph-mon[109912]: rocksdb: [db/version_set.cc:4082] Creating manifest 7 2023-09-23T20:37:35.903 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:35 smithi026 ceph-mon[109912]: rocksdb: EVENT_LOG_v1 {"time_micros": 1695501455643301, "job": 1, "event": "recovery_started", "wal_files": [4]} 2023-09-23T20:37:35.903 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:35 smithi026 ceph-mon[109912]: rocksdb: [db/db_impl/db_impl_open.cc:845] Recovering log #4 mode 2 2023-09-23T20:37:35.904 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:35 smithi026 ceph-mon[109912]: rocksdb: [table/block_based/filter_policy.cc:991] Using legacy Bloom filter with high (20) bits/key. Dramatic filter space and/or accuracy improvement is available with format_version>=5. 2023-09-23T20:37:35.904 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:35 smithi026 ceph-mon[109912]: rocksdb: EVENT_LOG_v1 {"time_micros": 1695501455643587, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 8, "file_size": 1670, "file_checksum": "", "file_checksum_func_name": "Unknown", "table_properties": {"data_size": 648, "index_size": 31, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 69, "raw_key_size": 115, "raw_average_key_size": 23, "raw_value_size": 526, "raw_average_value_size": 105, "num_data_blocks": 1, "num_entries": 5, "num_deletions": 0, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "rocksdb.BuiltinBloomFilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; ", "creation_time": 1695501455, "oldest_key_time": 0, "file_creation_time": 0, "db_id": "72687895-73c9-4fa1-ac87-a7ad648c7688", "db_session_id": "21M2ZAGE7BTOG6HGLU1U"}} 2023-09-23T20:37:35.904 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:35 smithi026 ceph-mon[109912]: rocksdb: [db/version_set.cc:4082] Creating manifest 9 2023-09-23T20:37:35.904 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:35 smithi026 ceph-mon[109912]: rocksdb: EVENT_LOG_v1 {"time_micros": 1695501455643928, "job": 1, "event": "recovery_finished"} 2023-09-23T20:37:35.904 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:35 smithi026 ceph-mon[109912]: rocksdb: [file/delete_scheduler.cc:69] Deleted file /var/lib/ceph/mon/ceph-c/store.db/000004.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2023-09-23T20:37:35.904 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:35 smithi026 ceph-mon[109912]: rocksdb: [db/db_impl/db_impl_open.cc:1700] SstFileManager instance 0x5596c848d180 2023-09-23T20:37:35.904 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:35 smithi026 ceph-mon[109912]: rocksdb: DB pointer 0x5596c8562000 2023-09-23T20:37:35.904 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:35 smithi026 ceph-mon[109912]: rocksdb: [db/db_impl/db_impl.cc:901] ------- DUMPING STATS ------- 2023-09-23T20:37:35.904 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:35 smithi026 ceph-mon[109912]: rocksdb: [db/db_impl/db_impl.cc:903] 2023-09-23T20:37:35.904 INFO:journalctl@ceph.mon.c.smithi026.stdout: ** DB Stats ** 2023-09-23T20:37:35.904 INFO:journalctl@ceph.mon.c.smithi026.stdout: Uptime(secs): 0.0 total, 0.0 interval 2023-09-23T20:37:35.904 INFO:journalctl@ceph.mon.c.smithi026.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2023-09-23T20:37:35.905 INFO:journalctl@ceph.mon.c.smithi026.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2023-09-23T20:37:35.905 INFO:journalctl@ceph.mon.c.smithi026.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2023-09-23T20:37:35.905 INFO:journalctl@ceph.mon.c.smithi026.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2023-09-23T20:37:35.905 INFO:journalctl@ceph.mon.c.smithi026.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 MB, 0.00 MB/s 2023-09-23T20:37:35.905 INFO:journalctl@ceph.mon.c.smithi026.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2023-09-23T20:37:35.905 INFO:journalctl@ceph.mon.c.smithi026.stdout: 2023-09-23T20:37:35.905 INFO:journalctl@ceph.mon.c.smithi026.stdout: ** Compaction Stats [default] ** 2023-09-23T20:37:35.905 INFO:journalctl@ceph.mon.c.smithi026.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-09-23T20:37:35.905 INFO:journalctl@ceph.mon.c.smithi026.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-09-23T20:37:35.905 INFO:journalctl@ceph.mon.c.smithi026.stdout: L0 1/0 1.63 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 5.9 0.00 0.00 1 0.000 0 0 2023-09-23T20:37:35.905 INFO:journalctl@ceph.mon.c.smithi026.stdout: Sum 1/0 1.63 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 5.9 0.00 0.00 1 0.000 0 0 2023-09-23T20:37:35.905 INFO:journalctl@ceph.mon.c.smithi026.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 5.9 0.00 0.00 1 0.000 0 0 2023-09-23T20:37:35.905 INFO:journalctl@ceph.mon.c.smithi026.stdout: 2023-09-23T20:37:35.906 INFO:journalctl@ceph.mon.c.smithi026.stdout: ** Compaction Stats [default] ** 2023-09-23T20:37:35.906 INFO:journalctl@ceph.mon.c.smithi026.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-09-23T20:37:35.906 INFO:journalctl@ceph.mon.c.smithi026.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-09-23T20:37:35.906 INFO:journalctl@ceph.mon.c.smithi026.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 5.9 0.00 0.00 1 0.000 0 0 2023-09-23T20:37:35.906 INFO:journalctl@ceph.mon.c.smithi026.stdout: Uptime(secs): 0.0 total, 0.0 interval 2023-09-23T20:37:35.906 INFO:journalctl@ceph.mon.c.smithi026.stdout: Flush(GB): cumulative 0.000, interval 0.000 2023-09-23T20:37:35.906 INFO:journalctl@ceph.mon.c.smithi026.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2023-09-23T20:37:35.906 INFO:journalctl@ceph.mon.c.smithi026.stdout: AddFile(Total Files): cumulative 0, interval 0 2023-09-23T20:37:35.906 INFO:journalctl@ceph.mon.c.smithi026.stdout: AddFile(L0 Files): cumulative 0, interval 0 2023-09-23T20:37:35.906 INFO:journalctl@ceph.mon.c.smithi026.stdout: AddFile(Keys): cumulative 0, interval 0 2023-09-23T20:37:35.906 INFO:journalctl@ceph.mon.c.smithi026.stdout: Cumulative compaction: 0.00 GB write, 0.61 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-09-23T20:37:35.906 INFO:journalctl@ceph.mon.c.smithi026.stdout: Interval compaction: 0.00 GB write, 0.61 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-09-23T20:37:35.907 INFO:journalctl@ceph.mon.c.smithi026.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2023-09-23T20:37:35.907 INFO:journalctl@ceph.mon.c.smithi026.stdout: 2023-09-23T20:37:35.907 INFO:journalctl@ceph.mon.c.smithi026.stdout: ** File Read Latency Histogram By Level [default] ** 2023-09-23T20:37:35.907 INFO:journalctl@ceph.mon.c.smithi026.stdout: 2023-09-23T20:37:35.907 INFO:journalctl@ceph.mon.c.smithi026.stdout: ** Compaction Stats [default] ** 2023-09-23T20:37:35.907 INFO:journalctl@ceph.mon.c.smithi026.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-09-23T20:37:35.907 INFO:journalctl@ceph.mon.c.smithi026.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-09-23T20:37:35.907 INFO:journalctl@ceph.mon.c.smithi026.stdout: L0 1/0 1.63 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 5.9 0.00 0.00 1 0.000 0 0 2023-09-23T20:37:35.908 INFO:journalctl@ceph.mon.c.smithi026.stdout: Sum 1/0 1.63 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 5.9 0.00 0.00 1 0.000 0 0 2023-09-23T20:37:35.908 INFO:journalctl@ceph.mon.c.smithi026.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2023-09-23T20:37:35.908 INFO:journalctl@ceph.mon.c.smithi026.stdout: 2023-09-23T20:37:35.908 INFO:journalctl@ceph.mon.c.smithi026.stdout: ** Compaction Stats [default] ** 2023-09-23T20:37:35.908 INFO:journalctl@ceph.mon.c.smithi026.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-09-23T20:37:35.908 INFO:journalctl@ceph.mon.c.smithi026.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-09-23T20:37:35.908 INFO:journalctl@ceph.mon.c.smithi026.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 5.9 0.00 0.00 1 0.000 0 0 2023-09-23T20:37:35.908 INFO:journalctl@ceph.mon.c.smithi026.stdout: Uptime(secs): 0.0 total, 0.0 interval 2023-09-23T20:37:35.908 INFO:journalctl@ceph.mon.c.smithi026.stdout: Flush(GB): cumulative 0.000, interval 0.000 2023-09-23T20:37:35.908 INFO:journalctl@ceph.mon.c.smithi026.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2023-09-23T20:37:35.908 INFO:journalctl@ceph.mon.c.smithi026.stdout: AddFile(Total Files): cumulative 0, interval 0 2023-09-23T20:37:35.908 INFO:journalctl@ceph.mon.c.smithi026.stdout: AddFile(L0 Files): cumulative 0, interval 0 2023-09-23T20:37:35.908 INFO:journalctl@ceph.mon.c.smithi026.stdout: AddFile(Keys): cumulative 0, interval 0 2023-09-23T20:37:35.909 INFO:journalctl@ceph.mon.c.smithi026.stdout: Cumulative compaction: 0.00 GB write, 0.60 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-09-23T20:37:35.909 INFO:journalctl@ceph.mon.c.smithi026.stdout: Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-09-23T20:37:35.909 INFO:journalctl@ceph.mon.c.smithi026.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2023-09-23T20:37:35.909 INFO:journalctl@ceph.mon.c.smithi026.stdout: 2023-09-23T20:37:35.909 INFO:journalctl@ceph.mon.c.smithi026.stdout: ** File Read Latency Histogram By Level [default] ** 2023-09-23T20:37:35.909 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:35 smithi026 ceph-mon[109912]: mon.c does not exist in monmap, will attempt to join an existing cluster 2023-09-23T20:37:35.909 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:35 smithi026 ceph-mon[109912]: using public_addrv [v2:172.21.15.26:3301/0,v1:172.21.15.26:6790/0] 2023-09-23T20:37:35.909 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:35 smithi026 ceph-mon[109912]: starting mon.c rank -1 at public addrs [v2:172.21.15.26:3301/0,v1:172.21.15.26:6790/0] at bind addrs [v2:172.21.15.26:3301/0,v1:172.21.15.26:6790/0] mon_data /var/lib/ceph/mon/ceph-c fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 2023-09-23T20:37:35.909 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:35 smithi026 ceph-mon[109912]: mon.c@-1(???) e0 preinit fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 2023-09-23T20:37:35.909 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:35 smithi026 systemd[1]: Started Ceph mon.c for b3f8e4ec-5a50-11ee-8db4-212e2dc638e7. 2023-09-23T20:37:35.909 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:35 smithi026 ceph-mon[109912]: mon.c@-1(synchronizing).mds e1 new map 2023-09-23T20:37:35.909 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:35 smithi026 ceph-mon[109912]: mon.c@-1(synchronizing).mds e1 print_map 2023-09-23T20:37:35.909 INFO:journalctl@ceph.mon.c.smithi026.stdout: e1 2023-09-23T20:37:35.910 INFO:journalctl@ceph.mon.c.smithi026.stdout: enable_multiple, ever_enabled_multiple: 1,1 2023-09-23T20:37:35.910 INFO:journalctl@ceph.mon.c.smithi026.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2023-09-23T20:37:35.910 INFO:journalctl@ceph.mon.c.smithi026.stdout: legacy client fscid: -1 2023-09-23T20:37:35.910 INFO:journalctl@ceph.mon.c.smithi026.stdout: 2023-09-23T20:37:35.910 INFO:journalctl@ceph.mon.c.smithi026.stdout: No filesystems configured 2023-09-23T20:37:35.910 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:35 smithi026 ceph-mon[109912]: mon.c@-1(synchronizing).osd e0 _set_cache_ratios kv ratio 0.25 inc ratio 0.375 full ratio 0.375 2023-09-23T20:37:35.910 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:35 smithi026 ceph-mon[109912]: mon.c@-1(synchronizing).osd e0 register_cache_with_pcm pcm target: 2147483648 pcm max: 1020054732 pcm min: 134217728 inc_osd_cache size: 1 2023-09-23T20:37:35.910 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:35 smithi026 ceph-mon[109912]: mon.c@-1(synchronizing).osd e1 e1: 0 total, 0 up, 0 in 2023-09-23T20:37:35.910 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:35 smithi026 ceph-mon[109912]: mon.c@-1(synchronizing).osd e2 e2: 0 total, 0 up, 0 in 2023-09-23T20:37:35.910 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:35 smithi026 ceph-mon[109912]: mon.c@-1(synchronizing).osd e3 e3: 0 total, 0 up, 0 in 2023-09-23T20:37:35.910 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:35 smithi026 ceph-mon[109912]: mon.c@-1(synchronizing).osd e4 e4: 0 total, 0 up, 0 in 2023-09-23T20:37:35.910 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:35 smithi026 ceph-mon[109912]: mon.c@-1(synchronizing).osd e4 crush map has features 3314932999778484224, adjusting msgr requires 2023-09-23T20:37:35.910 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:35 smithi026 ceph-mon[109912]: mon.c@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2023-09-23T20:37:35.911 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:35 smithi026 ceph-mon[109912]: mon.c@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2023-09-23T20:37:35.911 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:35 smithi026 ceph-mon[109912]: mon.c@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2023-09-23T20:37:35.911 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:35 smithi026 ceph-mon[109912]: pgmap v5: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T20:37:35.911 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:35 smithi026 ceph-mon[109912]: from='client.? 172.21.15.26:0/1212432566' entity='client.admin' cmd=[{"prefix": "osd crush tunables", "profile": "default"}]: dispatch 2023-09-23T20:37:35.911 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:35 smithi026 ceph-mon[109912]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:37:35.911 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:35 smithi026 ceph-mon[109912]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:37:35.911 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:35 smithi026 ceph-mon[109912]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:37:35.911 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:35 smithi026 ceph-mon[109912]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:37:35.911 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:35 smithi026 ceph-mon[109912]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi070", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:37:35.911 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:35 smithi026 ceph-mon[109912]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:37:35.911 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:35 smithi026 ceph-mon[109912]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:37:35.911 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:35 smithi026 ceph-mon[109912]: Updating smithi070:/etc/ceph/ceph.conf 2023-09-23T20:37:35.911 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:35 smithi026 ceph-mon[109912]: Updating smithi070:/var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-23T20:37:35.912 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:35 smithi026 ceph-mon[109912]: Updating smithi070:/etc/ceph/ceph.client.admin.keyring 2023-09-23T20:37:35.912 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:35 smithi026 ceph-mon[109912]: from='client.? 172.21.15.26:0/1212432566' entity='client.admin' cmd='[{"prefix": "osd crush tunables", "profile": "default"}]': finished 2023-09-23T20:37:35.912 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:35 smithi026 ceph-mon[109912]: osdmap e4: 0 total, 0 up, 0 in 2023-09-23T20:37:35.912 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:35 smithi026 ceph-mon[109912]: Updating smithi070:/var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/config/ceph.client.admin.keyring 2023-09-23T20:37:35.912 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:35 smithi026 ceph-mon[109912]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:37:35.912 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:35 smithi026 ceph-mon[109912]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:37:35.912 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:35 smithi026 ceph-mon[109912]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:37:35.912 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:35 smithi026 ceph-mon[109912]: pgmap v7: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T20:37:35.912 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:35 smithi026 ceph-mon[109912]: from='client.14184 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "placement": "3;smithi026:172.21.15.26=a;smithi026:[v2:172.21.15.26:3301,v1:172.21.15.26:6790]=c;smithi070:172.21.15.70=b", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T20:37:35.912 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:35 smithi026 ceph-mon[109912]: Saving service mon spec with placement smithi026:172.21.15.26=a;smithi026:[v2:172.21.15.26:3301,v1:172.21.15.26:6790]=c;smithi070:172.21.15.70=b;count:3 2023-09-23T20:37:35.912 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:35 smithi026 ceph-mon[109912]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:37:35.912 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:35 smithi026 ceph-mon[109912]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:37:35.913 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:35 smithi026 ceph-mon[109912]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:37:35.913 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:35 smithi026 ceph-mon[109912]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:37:35.913 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:35 smithi026 ceph-mon[109912]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:37:35.913 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:35 smithi026 ceph-mon[109912]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-23T20:37:35.913 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:35 smithi026 ceph-mon[109912]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:37:35.913 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:35 smithi026 ceph-mon[109912]: Deploying daemon mon.b on smithi070 2023-09-23T20:37:35.913 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:35 smithi026 ceph-mon[109912]: pgmap v8: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T20:37:35.913 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:35 smithi026 ceph-mon[109912]: pgmap v9: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T20:37:35.913 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:35 smithi026 ceph-mon[109912]: from='client.? 172.21.15.70:0/1019581227' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-09-23T20:37:35.913 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:35 smithi026 ceph-mon[109912]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-23T20:37:35.913 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:35 smithi026 ceph-mon[109912]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T20:37:35.913 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:35 smithi026 ceph-mon[109912]: mon.a calling monitor election 2023-09-23T20:37:35.914 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:35 smithi026 ceph-mon[109912]: pgmap v10: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T20:37:35.914 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:35 smithi026 ceph-mon[109912]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T20:37:35.914 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:35 smithi026 ceph-mon[109912]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T20:37:35.914 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:35 smithi026 ceph-mon[109912]: mon.b calling monitor election 2023-09-23T20:37:35.914 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:35 smithi026 ceph-mon[109912]: pgmap v11: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T20:37:35.914 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:35 smithi026 ceph-mon[109912]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T20:37:35.914 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:35 smithi026 ceph-mon[109912]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T20:37:35.914 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:35 smithi026 ceph-mon[109912]: pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T20:37:35.914 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:35 smithi026 ceph-mon[109912]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T20:37:35.914 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:35 smithi026 ceph-mon[109912]: mon.a is new leader, mons a,b in quorum (ranks 0,1) 2023-09-23T20:37:35.915 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:35 smithi026 ceph-mon[109912]: monmap e2: 2 mons at {a=[v2:172.21.15.26:3300/0,v1:172.21.15.26:6789/0],b=[v2:172.21.15.70:3300/0,v1:172.21.15.70:6789/0]} removed_ranks: {} 2023-09-23T20:37:35.915 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:35 smithi026 ceph-mon[109912]: fsmap 2023-09-23T20:37:35.915 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:35 smithi026 ceph-mon[109912]: osdmap e4: 0 total, 0 up, 0 in 2023-09-23T20:37:35.915 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:35 smithi026 ceph-mon[109912]: mgrmap e13: y(active, since 36s) 2023-09-23T20:37:35.915 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:35 smithi026 ceph-mon[109912]: overall HEALTH_OK 2023-09-23T20:37:35.915 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:35 smithi026 ceph-mon[109912]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:37:35.915 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:35 smithi026 ceph-mon[109912]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:37:35.915 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:35 smithi026 ceph-mon[109912]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:37:35.915 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:35 smithi026 ceph-mon[109912]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-23T20:37:35.915 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:35 smithi026 ceph-mon[109912]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:37:35.915 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:35 smithi026 ceph-mon[109912]: Deploying daemon mon.c on smithi026 2023-09-23T20:37:35.915 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:35 smithi026 ceph-mon[109912]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T20:37:35.915 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:35 smithi026 ceph-mon[109912]: from='client.? 172.21.15.70:0/1175766312' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-09-23T20:37:35.916 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:35 smithi026 ceph-mon[109912]: pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T20:37:35.916 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:35 smithi026 ceph-mon[109912]: mon.c@-1(synchronizing).paxosservice(auth 1..3) refresh upgraded, format 0 -> 3 2023-09-23T20:37:35.916 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:35 smithi026 ceph-mon[109912]: mon.c@-1(synchronizing) e2 handle_conf_change mon_allow_pool_delete 2023-09-23T20:37:41.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:40 smithi070 ceph-mon[100755]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-23T20:37:41.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:40 smithi070 ceph-mon[100755]: mon.b calling monitor election 2023-09-23T20:37:41.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:40 smithi070 ceph-mon[100755]: mon.a calling monitor election 2023-09-23T20:37:41.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:40 smithi070 ceph-mon[100755]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T20:37:41.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:40 smithi070 ceph-mon[100755]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T20:37:41.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:40 smithi070 ceph-mon[100755]: pgmap v14: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T20:37:41.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:40 smithi070 ceph-mon[100755]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T20:37:41.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:40 smithi070 ceph-mon[100755]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T20:37:41.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:40 smithi070 ceph-mon[100755]: pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T20:37:41.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:40 smithi070 ceph-mon[100755]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T20:37:41.121 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:40 smithi070 ceph-mon[100755]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T20:37:41.121 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:40 smithi070 ceph-mon[100755]: pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T20:37:41.121 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:40 smithi070 ceph-mon[100755]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T20:37:41.121 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:40 smithi070 ceph-mon[100755]: mon.a is new leader, mons a,b in quorum (ranks 0,1) 2023-09-23T20:37:41.121 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:40 smithi070 ceph-mon[100755]: monmap e3: 3 mons at {a=[v2:172.21.15.26:3300/0,v1:172.21.15.26:6789/0],b=[v2:172.21.15.70:3300/0,v1:172.21.15.70:6789/0],c=[v2:172.21.15.26:3301/0,v1:172.21.15.26:6790/0]} removed_ranks: {} 2023-09-23T20:37:41.121 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:40 smithi070 ceph-mon[100755]: fsmap 2023-09-23T20:37:41.121 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:40 smithi070 ceph-mon[100755]: osdmap e4: 0 total, 0 up, 0 in 2023-09-23T20:37:41.121 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:40 smithi070 ceph-mon[100755]: mgrmap e13: y(active, since 44s) 2023-09-23T20:37:41.121 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:40 smithi070 ceph-mon[100755]: Health check failed: 1/3 mons down, quorum a,b (MON_DOWN) 2023-09-23T20:37:41.121 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:40 smithi070 ceph-mon[100755]: Health detail: HEALTH_WARN 1/3 mons down, quorum a,b 2023-09-23T20:37:41.121 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:40 smithi070 ceph-mon[100755]: [WRN] MON_DOWN: 1/3 mons down, quorum a,b 2023-09-23T20:37:41.122 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:40 smithi070 ceph-mon[100755]: mon.c (rank 2) addr [v2:172.21.15.26:3301/0,v1:172.21.15.26:6790/0] is down (out of quorum) 2023-09-23T20:37:41.122 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:40 smithi070 ceph-mon[100755]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:37:41.122 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:40 smithi070 ceph-mon[100755]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:37:41.122 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:40 smithi070 ceph-mon[100755]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:37:41.122 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:40 smithi070 ceph-mon[100755]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:37:41.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:37:40 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-23T20:37:41.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:37:40 smithi026 ceph-mon[95839]: mon.b calling monitor election 2023-09-23T20:37:41.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:37:40 smithi026 ceph-mon[95839]: mon.a calling monitor election 2023-09-23T20:37:41.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:37:40 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T20:37:41.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:37:40 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T20:37:41.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:37:40 smithi026 ceph-mon[95839]: pgmap v14: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T20:37:41.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:37:40 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T20:37:41.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:37:40 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T20:37:41.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:37:40 smithi026 ceph-mon[95839]: pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T20:37:41.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:37:40 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T20:37:41.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:37:40 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T20:37:41.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:37:40 smithi026 ceph-mon[95839]: pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T20:37:41.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:37:40 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T20:37:41.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:37:40 smithi026 ceph-mon[95839]: mon.a is new leader, mons a,b in quorum (ranks 0,1) 2023-09-23T20:37:41.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:37:40 smithi026 ceph-mon[95839]: monmap e3: 3 mons at {a=[v2:172.21.15.26:3300/0,v1:172.21.15.26:6789/0],b=[v2:172.21.15.70:3300/0,v1:172.21.15.70:6789/0],c=[v2:172.21.15.26:3301/0,v1:172.21.15.26:6790/0]} removed_ranks: {} 2023-09-23T20:37:41.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:37:40 smithi026 ceph-mon[95839]: fsmap 2023-09-23T20:37:41.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:37:40 smithi026 ceph-mon[95839]: osdmap e4: 0 total, 0 up, 0 in 2023-09-23T20:37:41.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:37:40 smithi026 ceph-mon[95839]: mgrmap e13: y(active, since 44s) 2023-09-23T20:37:41.134 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:37:40 smithi026 ceph-mon[95839]: Health check failed: 1/3 mons down, quorum a,b (MON_DOWN) 2023-09-23T20:37:41.134 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:37:40 smithi026 ceph-mon[95839]: Health detail: HEALTH_WARN 1/3 mons down, quorum a,b 2023-09-23T20:37:41.134 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:37:40 smithi026 ceph-mon[95839]: [WRN] MON_DOWN: 1/3 mons down, quorum a,b 2023-09-23T20:37:41.134 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:37:40 smithi026 ceph-mon[95839]: mon.c (rank 2) addr [v2:172.21.15.26:3301/0,v1:172.21.15.26:6790/0] is down (out of quorum) 2023-09-23T20:37:41.134 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:37:40 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:37:41.134 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:37:40 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:37:41.134 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:37:40 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:37:41.134 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:37:40 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:37:41.677 INFO:teuthology.orchestra.run.smithi070.stdout: 2023-09-23T20:37:41.677 INFO:teuthology.orchestra.run.smithi070.stdout:{"epoch":3,"fsid":"b3f8e4ec-5a50-11ee-8db4-212e2dc638e7","modified":"2023-09-23T20:37:35.664536Z","created":"2023-09-23T20:35:56.712408Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","removed_ranks: ":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"a","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:3300","nonce":0},{"type":"v1","addr":"172.21.15.26:6789","nonce":0}]},"addr":"172.21.15.26:6789/0","public_addr":"172.21.15.26:6789/0","priority":0,"weight":0,"crush_location":"{}"},{"rank":1,"name":"b","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:3300","nonce":0},{"type":"v1","addr":"172.21.15.70:6789","nonce":0}]},"addr":"172.21.15.70:6789/0","public_addr":"172.21.15.70:6789/0","priority":0,"weight":0,"crush_location":"{}"},{"rank":2,"name":"c","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:3301","nonce":0},{"type":"v1","addr":"172.21.15.26:6790","nonce":0}]},"addr":"172.21.15.26:6790/0","public_addr":"172.21.15.26:6790/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0,1]} 2023-09-23T20:37:41.679 INFO:teuthology.orchestra.run.smithi070.stderr:dumped monmap epoch 3 2023-09-23T20:37:42.059 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:41 smithi070 ceph-mon[100755]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:37:42.059 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:41 smithi070 ceph-mon[100755]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:37:42.059 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:41 smithi070 ceph-mon[100755]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T20:37:42.059 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:41 smithi070 ceph-mon[100755]: from='client.? 172.21.15.70:0/1055237949' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-09-23T20:37:42.060 INFO:tasks.cephadm:Generating final ceph.conf file... 2023-09-23T20:37:42.061 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph config generate-minimal-conf 2023-09-23T20:37:42.090 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:37:41 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:37:42.090 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:37:41 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:37:42.091 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:37:41 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T20:37:42.091 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:37:41 smithi026 ceph-mon[95839]: from='client.? 172.21.15.70:0/1055237949' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-09-23T20:37:42.668 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:37:42.936 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:42 smithi026 ceph-mon[109912]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-23T20:37:42.937 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:42 smithi026 ceph-mon[109912]: mon.b calling monitor election 2023-09-23T20:37:42.937 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:42 smithi026 ceph-mon[109912]: mon.a calling monitor election 2023-09-23T20:37:42.937 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:42 smithi026 ceph-mon[109912]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T20:37:42.937 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:42 smithi026 ceph-mon[109912]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T20:37:42.937 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:42 smithi026 ceph-mon[109912]: pgmap v14: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T20:37:42.937 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:42 smithi026 ceph-mon[109912]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T20:37:42.937 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:42 smithi026 ceph-mon[109912]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T20:37:42.937 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:42 smithi026 ceph-mon[109912]: pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T20:37:42.937 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:42 smithi026 ceph-mon[109912]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T20:37:42.937 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:42 smithi026 ceph-mon[109912]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T20:37:42.938 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:42 smithi026 ceph-mon[109912]: pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T20:37:42.938 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:42 smithi026 ceph-mon[109912]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T20:37:42.938 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:42 smithi026 ceph-mon[109912]: mon.a is new leader, mons a,b in quorum (ranks 0,1) 2023-09-23T20:37:42.938 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:42 smithi026 ceph-mon[109912]: monmap e3: 3 mons at {a=[v2:172.21.15.26:3300/0,v1:172.21.15.26:6789/0],b=[v2:172.21.15.70:3300/0,v1:172.21.15.70:6789/0],c=[v2:172.21.15.26:3301/0,v1:172.21.15.26:6790/0]} removed_ranks: {} 2023-09-23T20:37:42.938 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:42 smithi026 ceph-mon[109912]: fsmap 2023-09-23T20:37:42.938 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:42 smithi026 ceph-mon[109912]: osdmap e4: 0 total, 0 up, 0 in 2023-09-23T20:37:42.938 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:42 smithi026 ceph-mon[109912]: mgrmap e13: y(active, since 44s) 2023-09-23T20:37:42.938 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:42 smithi026 ceph-mon[109912]: Health check failed: 1/3 mons down, quorum a,b (MON_DOWN) 2023-09-23T20:37:42.938 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:42 smithi026 ceph-mon[109912]: Health detail: HEALTH_WARN 1/3 mons down, quorum a,b 2023-09-23T20:37:42.938 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:42 smithi026 ceph-mon[109912]: [WRN] MON_DOWN: 1/3 mons down, quorum a,b 2023-09-23T20:37:42.938 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:42 smithi026 ceph-mon[109912]: mon.c (rank 2) addr [v2:172.21.15.26:3301/0,v1:172.21.15.26:6790/0] is down (out of quorum) 2023-09-23T20:37:42.939 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:42 smithi026 ceph-mon[109912]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:37:42.939 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:42 smithi026 ceph-mon[109912]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:37:42.939 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:42 smithi026 ceph-mon[109912]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:37:42.939 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:42 smithi026 ceph-mon[109912]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:37:42.939 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:42 smithi026 ceph-mon[109912]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:37:42.939 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:42 smithi026 ceph-mon[109912]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:37:42.939 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:42 smithi026 ceph-mon[109912]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T20:37:42.939 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:42 smithi026 ceph-mon[109912]: from='client.? 172.21.15.70:0/1055237949' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-09-23T20:37:43.871 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:37:43 smithi026 ceph-mon[95839]: mon.c calling monitor election 2023-09-23T20:37:43.871 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:37:43 smithi026 ceph-mon[95839]: mon.c calling monitor election 2023-09-23T20:37:43.871 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:37:43 smithi026 ceph-mon[95839]: mon.b calling monitor election 2023-09-23T20:37:43.871 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:37:43 smithi026 ceph-mon[95839]: mon.a calling monitor election 2023-09-23T20:37:43.872 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:37:43 smithi026 ceph-mon[95839]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2023-09-23T20:37:43.872 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:37:43 smithi026 ceph-mon[95839]: monmap e3: 3 mons at {a=[v2:172.21.15.26:3300/0,v1:172.21.15.26:6789/0],b=[v2:172.21.15.70:3300/0,v1:172.21.15.70:6789/0],c=[v2:172.21.15.26:3301/0,v1:172.21.15.26:6790/0]} removed_ranks: {} 2023-09-23T20:37:43.872 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:37:43 smithi026 ceph-mon[95839]: fsmap 2023-09-23T20:37:43.872 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:37:43 smithi026 ceph-mon[95839]: osdmap e4: 0 total, 0 up, 0 in 2023-09-23T20:37:43.872 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:37:43 smithi026 ceph-mon[95839]: mgrmap e13: y(active, since 46s) 2023-09-23T20:37:43.872 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:37:43 smithi026 ceph-mon[95839]: Health check cleared: MON_DOWN (was: 1/3 mons down, quorum a,b) 2023-09-23T20:37:43.872 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:37:43 smithi026 ceph-mon[95839]: Cluster is now healthy 2023-09-23T20:37:43.872 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:37:43 smithi026 ceph-mon[95839]: overall HEALTH_OK 2023-09-23T20:37:43.872 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:37:43 smithi026 ceph-mon[95839]: Updating smithi026:/var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-23T20:37:43.872 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:37:43 smithi026 ceph-mon[95839]: Updating smithi070:/var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-23T20:37:43.872 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:37:43 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:37:43.872 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:37:43 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:37:43.872 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:37:43 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:37:43.873 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:37:43 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:37:43.873 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:37:43 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:37:43.873 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:37:43 smithi026 ceph-mon[95839]: Reconfiguring mon.a (unknown last config time)... 2023-09-23T20:37:43.873 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:37:43 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-23T20:37:43.873 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:37:43 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-09-23T20:37:43.873 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:37:43 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:37:43.873 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:37:43 smithi026 ceph-mon[95839]: Reconfiguring daemon mon.a on smithi026 2023-09-23T20:37:43.873 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:43 smithi026 ceph-mon[109912]: mon.c calling monitor election 2023-09-23T20:37:43.873 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:43 smithi026 ceph-mon[109912]: mon.c calling monitor election 2023-09-23T20:37:43.873 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:43 smithi026 ceph-mon[109912]: mon.b calling monitor election 2023-09-23T20:37:43.873 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:43 smithi026 ceph-mon[109912]: mon.a calling monitor election 2023-09-23T20:37:43.874 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:43 smithi026 ceph-mon[109912]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2023-09-23T20:37:43.874 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:43 smithi026 ceph-mon[109912]: monmap e3: 3 mons at {a=[v2:172.21.15.26:3300/0,v1:172.21.15.26:6789/0],b=[v2:172.21.15.70:3300/0,v1:172.21.15.70:6789/0],c=[v2:172.21.15.26:3301/0,v1:172.21.15.26:6790/0]} removed_ranks: {} 2023-09-23T20:37:43.874 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:43 smithi026 ceph-mon[109912]: fsmap 2023-09-23T20:37:43.874 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:43 smithi026 ceph-mon[109912]: osdmap e4: 0 total, 0 up, 0 in 2023-09-23T20:37:43.874 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:43 smithi026 ceph-mon[109912]: mgrmap e13: y(active, since 46s) 2023-09-23T20:37:43.874 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:43 smithi026 ceph-mon[109912]: Health check cleared: MON_DOWN (was: 1/3 mons down, quorum a,b) 2023-09-23T20:37:43.874 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:43 smithi026 ceph-mon[109912]: Cluster is now healthy 2023-09-23T20:37:43.874 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:43 smithi026 ceph-mon[109912]: overall HEALTH_OK 2023-09-23T20:37:43.874 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:43 smithi026 ceph-mon[109912]: Updating smithi026:/var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-23T20:37:43.874 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:43 smithi026 ceph-mon[109912]: Updating smithi070:/var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-23T20:37:43.874 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:43 smithi026 ceph-mon[109912]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:37:43.874 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:43 smithi026 ceph-mon[109912]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:37:43.875 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:43 smithi026 ceph-mon[109912]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:37:43.875 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:43 smithi026 ceph-mon[109912]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:37:43.875 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:43 smithi026 ceph-mon[109912]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:37:43.875 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:43 smithi026 ceph-mon[109912]: Reconfiguring mon.a (unknown last config time)... 2023-09-23T20:37:43.875 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:43 smithi026 ceph-mon[109912]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-23T20:37:43.875 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:43 smithi026 ceph-mon[109912]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-09-23T20:37:43.875 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:43 smithi026 ceph-mon[109912]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:37:43.875 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:43 smithi026 ceph-mon[109912]: Reconfiguring daemon mon.a on smithi026 2023-09-23T20:37:44.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:43 smithi070 ceph-mon[100755]: mon.c calling monitor election 2023-09-23T20:37:44.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:43 smithi070 ceph-mon[100755]: mon.c calling monitor election 2023-09-23T20:37:44.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:43 smithi070 ceph-mon[100755]: mon.b calling monitor election 2023-09-23T20:37:44.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:43 smithi070 ceph-mon[100755]: mon.a calling monitor election 2023-09-23T20:37:44.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:43 smithi070 ceph-mon[100755]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2023-09-23T20:37:44.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:43 smithi070 ceph-mon[100755]: monmap e3: 3 mons at {a=[v2:172.21.15.26:3300/0,v1:172.21.15.26:6789/0],b=[v2:172.21.15.70:3300/0,v1:172.21.15.70:6789/0],c=[v2:172.21.15.26:3301/0,v1:172.21.15.26:6790/0]} removed_ranks: {} 2023-09-23T20:37:44.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:43 smithi070 ceph-mon[100755]: fsmap 2023-09-23T20:37:44.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:43 smithi070 ceph-mon[100755]: osdmap e4: 0 total, 0 up, 0 in 2023-09-23T20:37:44.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:43 smithi070 ceph-mon[100755]: mgrmap e13: y(active, since 46s) 2023-09-23T20:37:44.121 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:43 smithi070 ceph-mon[100755]: Health check cleared: MON_DOWN (was: 1/3 mons down, quorum a,b) 2023-09-23T20:37:44.121 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:43 smithi070 ceph-mon[100755]: Cluster is now healthy 2023-09-23T20:37:44.121 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:43 smithi070 ceph-mon[100755]: overall HEALTH_OK 2023-09-23T20:37:44.121 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:43 smithi070 ceph-mon[100755]: Updating smithi026:/var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-23T20:37:44.121 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:43 smithi070 ceph-mon[100755]: Updating smithi070:/var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-23T20:37:44.121 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:43 smithi070 ceph-mon[100755]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:37:44.121 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:43 smithi070 ceph-mon[100755]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:37:44.121 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:43 smithi070 ceph-mon[100755]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:37:44.121 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:43 smithi070 ceph-mon[100755]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:37:44.121 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:43 smithi070 ceph-mon[100755]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:37:44.121 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:43 smithi070 ceph-mon[100755]: Reconfiguring mon.a (unknown last config time)... 2023-09-23T20:37:44.121 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:43 smithi070 ceph-mon[100755]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-23T20:37:44.122 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:43 smithi070 ceph-mon[100755]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-09-23T20:37:44.122 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:43 smithi070 ceph-mon[100755]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:37:44.122 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:43 smithi070 ceph-mon[100755]: Reconfiguring daemon mon.a on smithi026 2023-09-23T20:37:44.438 INFO:teuthology.orchestra.run.smithi026.stdout:# minimal ceph.conf for b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 2023-09-23T20:37:44.438 INFO:teuthology.orchestra.run.smithi026.stdout:[global] 2023-09-23T20:37:44.439 INFO:teuthology.orchestra.run.smithi026.stdout: fsid = b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 2023-09-23T20:37:44.439 INFO:teuthology.orchestra.run.smithi026.stdout: mon_host = [v2:172.21.15.26:3300/0,v1:172.21.15.26:6789/0] [v2:172.21.15.70:3300/0,v1:172.21.15.70:6789/0] [v2:172.21.15.26:3301/0,v1:172.21.15.26:6790/0] 2023-09-23T20:37:44.721 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:37:44 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T20:37:44.721 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:37:44 smithi026 ceph-mon[95839]: from='client.? 172.21.15.26:0/1564418547' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:37:44.721 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:37:44 smithi026 ceph-mon[95839]: pgmap v18: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T20:37:44.722 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:44 smithi026 ceph-mon[109912]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T20:37:44.722 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:44 smithi026 ceph-mon[109912]: from='client.? 172.21.15.26:0/1564418547' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:37:44.722 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:44 smithi026 ceph-mon[109912]: pgmap v18: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T20:37:45.012 INFO:tasks.cephadm:Distributing (final) config and client.admin keyring... 2023-09-23T20:37:45.013 DEBUG:teuthology.orchestra.run.smithi026:> set -ex 2023-09-23T20:37:45.013 DEBUG:teuthology.orchestra.run.smithi026:> sudo dd of=/etc/ceph/ceph.conf 2023-09-23T20:37:45.083 DEBUG:teuthology.orchestra.run.smithi026:> set -ex 2023-09-23T20:37:45.083 DEBUG:teuthology.orchestra.run.smithi026:> sudo dd of=/etc/ceph/ceph.client.admin.keyring 2023-09-23T20:37:45.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:44 smithi070 ceph-mon[100755]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T20:37:45.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:44 smithi070 ceph-mon[100755]: from='client.? 172.21.15.26:0/1564418547' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:37:45.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:44 smithi070 ceph-mon[100755]: pgmap v18: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T20:37:45.153 DEBUG:teuthology.orchestra.run.smithi070:> set -ex 2023-09-23T20:37:45.153 DEBUG:teuthology.orchestra.run.smithi070:> sudo dd of=/etc/ceph/ceph.conf 2023-09-23T20:37:45.189 DEBUG:teuthology.orchestra.run.smithi070:> set -ex 2023-09-23T20:37:45.189 DEBUG:teuthology.orchestra.run.smithi070:> sudo dd of=/etc/ceph/ceph.client.admin.keyring 2023-09-23T20:37:45.259 INFO:tasks.cephadm:Adding mgr.y on smithi026 2023-09-23T20:37:45.259 INFO:tasks.cephadm:Adding mgr.x on smithi070 2023-09-23T20:37:45.260 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph orch apply mgr '2;smithi026=y;smithi070=x' 2023-09-23T20:37:45.889 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:37:46.099 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:45 smithi070 ceph-mon[100755]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:37:46.099 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:45 smithi070 ceph-mon[100755]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:37:46.099 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:45 smithi070 ceph-mon[100755]: Reconfiguring mon.c (monmap changed)... 2023-09-23T20:37:46.099 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:45 smithi070 ceph-mon[100755]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-23T20:37:46.099 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:45 smithi070 ceph-mon[100755]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-09-23T20:37:46.099 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:45 smithi070 ceph-mon[100755]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:37:46.099 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:45 smithi070 ceph-mon[100755]: Reconfiguring daemon mon.c on smithi026 2023-09-23T20:37:46.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:37:45 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:37:46.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:37:45 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:37:46.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:37:45 smithi026 ceph-mon[95839]: Reconfiguring mon.c (monmap changed)... 2023-09-23T20:37:46.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:37:45 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-23T20:37:46.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:37:45 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-09-23T20:37:46.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:37:45 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:37:46.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:37:45 smithi026 ceph-mon[95839]: Reconfiguring daemon mon.c on smithi026 2023-09-23T20:37:46.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:45 smithi026 ceph-mon[109912]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:37:46.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:45 smithi026 ceph-mon[109912]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:37:46.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:45 smithi026 ceph-mon[109912]: Reconfiguring mon.c (monmap changed)... 2023-09-23T20:37:46.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:45 smithi026 ceph-mon[109912]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-23T20:37:46.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:45 smithi026 ceph-mon[109912]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-09-23T20:37:46.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:45 smithi026 ceph-mon[109912]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:37:46.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:45 smithi026 ceph-mon[109912]: Reconfiguring daemon mon.c on smithi026 2023-09-23T20:37:47.261 INFO:teuthology.orchestra.run.smithi070.stdout:Scheduled mgr update... 2023-09-23T20:37:47.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:47 smithi070 ceph-mon[100755]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:37:47.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:47 smithi070 ceph-mon[100755]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:37:47.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:47 smithi070 ceph-mon[100755]: Reconfiguring mon.b (monmap changed)... 2023-09-23T20:37:47.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:47 smithi070 ceph-mon[100755]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-23T20:37:47.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:47 smithi070 ceph-mon[100755]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-09-23T20:37:47.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:47 smithi070 ceph-mon[100755]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:37:47.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:47 smithi070 ceph-mon[100755]: Reconfiguring daemon mon.b on smithi070 2023-09-23T20:37:47.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:47 smithi070 ceph-mon[100755]: pgmap v19: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T20:37:47.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:37:47 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:37:47.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:37:47 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:37:47.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:37:47 smithi026 ceph-mon[95839]: Reconfiguring mon.b (monmap changed)... 2023-09-23T20:37:47.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:37:47 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-23T20:37:47.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:37:47 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-09-23T20:37:47.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:37:47 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:37:47.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:37:47 smithi026 ceph-mon[95839]: Reconfiguring daemon mon.b on smithi070 2023-09-23T20:37:47.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:37:47 smithi026 ceph-mon[95839]: pgmap v19: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T20:37:47.383 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:47 smithi026 ceph-mon[109912]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:37:47.383 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:47 smithi026 ceph-mon[109912]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:37:47.383 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:47 smithi026 ceph-mon[109912]: Reconfiguring mon.b (monmap changed)... 2023-09-23T20:37:47.383 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:47 smithi026 ceph-mon[109912]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-23T20:37:47.383 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:47 smithi026 ceph-mon[109912]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-09-23T20:37:47.383 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:47 smithi026 ceph-mon[109912]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:37:47.383 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:47 smithi026 ceph-mon[109912]: Reconfiguring daemon mon.b on smithi070 2023-09-23T20:37:47.383 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:47 smithi026 ceph-mon[109912]: pgmap v19: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T20:37:47.927 DEBUG:teuthology.orchestra.run.smithi070:mgr.x> sudo journalctl -f -n 0 -u ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7@mgr.x.service 2023-09-23T20:37:47.929 INFO:tasks.cephadm:Deploying OSDs... 2023-09-23T20:37:47.930 DEBUG:teuthology.orchestra.run.smithi026:> set -ex 2023-09-23T20:37:47.930 DEBUG:teuthology.orchestra.run.smithi026:> dd if=/scratch_devs of=/dev/stdout 2023-09-23T20:37:47.951 DEBUG:teuthology.misc:devs=['/dev/vg_nvme/lv_1', '/dev/vg_nvme/lv_2', '/dev/vg_nvme/lv_3', '/dev/vg_nvme/lv_4'] 2023-09-23T20:37:47.951 DEBUG:teuthology.orchestra.run.smithi026:> stat /dev/vg_nvme/lv_1 2023-09-23T20:37:47.958 INFO:journalctl@ceph.mgr.x.smithi070.stdout:-- Logs begin at Sat 2023-09-23 20:21:16 UTC. -- 2023-09-23T20:37:48.012 INFO:teuthology.orchestra.run.smithi026.stdout: File: /dev/vg_nvme/lv_1 -> ../dm-0 2023-09-23T20:37:48.012 INFO:teuthology.orchestra.run.smithi026.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2023-09-23T20:37:48.012 INFO:teuthology.orchestra.run.smithi026.stdout:Device: 6h/6d Inode: 390253 Links: 1 2023-09-23T20:37:48.013 INFO:teuthology.orchestra.run.smithi026.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2023-09-23T20:37:48.013 INFO:teuthology.orchestra.run.smithi026.stdout:Context: system_u:object_r:device_t:s0 2023-09-23T20:37:48.013 INFO:teuthology.orchestra.run.smithi026.stdout:Access: 2023-09-23 20:37:35.116455199 +0000 2023-09-23T20:37:48.013 INFO:teuthology.orchestra.run.smithi026.stdout:Modify: 2023-09-23 20:30:11.141392960 +0000 2023-09-23T20:37:48.013 INFO:teuthology.orchestra.run.smithi026.stdout:Change: 2023-09-23 20:30:11.141392960 +0000 2023-09-23T20:37:48.013 INFO:teuthology.orchestra.run.smithi026.stdout: Birth: - 2023-09-23T20:37:48.013 DEBUG:teuthology.orchestra.run.smithi026:> sudo dd if=/dev/vg_nvme/lv_1 of=/dev/null count=1 2023-09-23T20:37:48.083 INFO:teuthology.orchestra.run.smithi026.stderr:1+0 records in 2023-09-23T20:37:48.083 INFO:teuthology.orchestra.run.smithi026.stderr:1+0 records out 2023-09-23T20:37:48.084 INFO:teuthology.orchestra.run.smithi026.stderr:512 bytes copied, 0.000126871 s, 4.0 MB/s 2023-09-23T20:37:48.085 DEBUG:teuthology.orchestra.run.smithi026:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_1 2023-09-23T20:37:48.146 DEBUG:teuthology.orchestra.run.smithi026:> stat /dev/vg_nvme/lv_2 2023-09-23T20:37:48.206 INFO:teuthology.orchestra.run.smithi026.stdout: File: /dev/vg_nvme/lv_2 -> ../dm-1 2023-09-23T20:37:48.206 INFO:teuthology.orchestra.run.smithi026.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2023-09-23T20:37:48.206 INFO:teuthology.orchestra.run.smithi026.stdout:Device: 6h/6d Inode: 388638 Links: 1 2023-09-23T20:37:48.206 INFO:teuthology.orchestra.run.smithi026.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2023-09-23T20:37:48.206 INFO:teuthology.orchestra.run.smithi026.stdout:Context: system_u:object_r:device_t:s0 2023-09-23T20:37:48.206 INFO:teuthology.orchestra.run.smithi026.stdout:Access: 2023-09-23 20:37:35.117455184 +0000 2023-09-23T20:37:48.206 INFO:teuthology.orchestra.run.smithi026.stdout:Modify: 2023-09-23 20:30:11.643385725 +0000 2023-09-23T20:37:48.207 INFO:teuthology.orchestra.run.smithi026.stdout:Change: 2023-09-23 20:30:11.643385725 +0000 2023-09-23T20:37:48.207 INFO:teuthology.orchestra.run.smithi026.stdout: Birth: - 2023-09-23T20:37:48.207 DEBUG:teuthology.orchestra.run.smithi026:> sudo dd if=/dev/vg_nvme/lv_2 of=/dev/null count=1 2023-09-23T20:37:48.276 INFO:teuthology.orchestra.run.smithi026.stderr:1+0 records in 2023-09-23T20:37:48.276 INFO:teuthology.orchestra.run.smithi026.stderr:1+0 records out 2023-09-23T20:37:48.277 INFO:teuthology.orchestra.run.smithi026.stderr:512 bytes copied, 0.00011528 s, 4.4 MB/s 2023-09-23T20:37:48.278 DEBUG:teuthology.orchestra.run.smithi026:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_2 2023-09-23T20:37:48.337 DEBUG:teuthology.orchestra.run.smithi026:> stat /dev/vg_nvme/lv_3 2023-09-23T20:37:48.381 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:37:48 smithi026 ceph-mon[95839]: from='client.24100 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "placement": "2;smithi026=y;smithi070=x", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T20:37:48.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:37:48 smithi026 ceph-mon[95839]: Saving service mgr spec with placement smithi026=y;smithi070=x;count:2 2023-09-23T20:37:48.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:37:48 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:37:48.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:37:48 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:37:48.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:37:48 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:37:48.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:37:48 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:37:48.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:37:48 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:37:48.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:37:48 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:37:48.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:37:48 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:37:48.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:37:48 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-09-23T20:37:48.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:37:48 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2023-09-23T20:37:48.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:37:48 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-09-23T20:37:48.383 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:37:48 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:37:48.383 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:48 smithi026 ceph-mon[109912]: from='client.24100 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "placement": "2;smithi026=y;smithi070=x", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T20:37:48.383 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:48 smithi026 ceph-mon[109912]: Saving service mgr spec with placement smithi026=y;smithi070=x;count:2 2023-09-23T20:37:48.383 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:48 smithi026 ceph-mon[109912]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:37:48.383 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:48 smithi026 ceph-mon[109912]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:37:48.383 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:48 smithi026 ceph-mon[109912]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:37:48.383 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:48 smithi026 ceph-mon[109912]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:37:48.383 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:48 smithi026 ceph-mon[109912]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:37:48.383 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:48 smithi026 ceph-mon[109912]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:37:48.384 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:48 smithi026 ceph-mon[109912]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:37:48.384 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:48 smithi026 ceph-mon[109912]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-09-23T20:37:48.384 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:48 smithi026 ceph-mon[109912]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2023-09-23T20:37:48.384 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:48 smithi026 ceph-mon[109912]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-09-23T20:37:48.384 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:48 smithi026 ceph-mon[109912]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:37:48.396 INFO:teuthology.orchestra.run.smithi026.stdout: File: /dev/vg_nvme/lv_3 -> ../dm-2 2023-09-23T20:37:48.396 INFO:teuthology.orchestra.run.smithi026.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2023-09-23T20:37:48.397 INFO:teuthology.orchestra.run.smithi026.stdout:Device: 6h/6d Inode: 390338 Links: 1 2023-09-23T20:37:48.397 INFO:teuthology.orchestra.run.smithi026.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2023-09-23T20:37:48.397 INFO:teuthology.orchestra.run.smithi026.stdout:Context: system_u:object_r:device_t:s0 2023-09-23T20:37:48.397 INFO:teuthology.orchestra.run.smithi026.stdout:Access: 2023-09-23 20:37:35.117455184 +0000 2023-09-23T20:37:48.397 INFO:teuthology.orchestra.run.smithi026.stdout:Modify: 2023-09-23 20:30:12.158378303 +0000 2023-09-23T20:37:48.397 INFO:teuthology.orchestra.run.smithi026.stdout:Change: 2023-09-23 20:30:12.158378303 +0000 2023-09-23T20:37:48.397 INFO:teuthology.orchestra.run.smithi026.stdout: Birth: - 2023-09-23T20:37:48.397 DEBUG:teuthology.orchestra.run.smithi026:> sudo dd if=/dev/vg_nvme/lv_3 of=/dev/null count=1 2023-09-23T20:37:48.465 INFO:teuthology.orchestra.run.smithi026.stderr:1+0 records in 2023-09-23T20:37:48.465 INFO:teuthology.orchestra.run.smithi026.stderr:1+0 records out 2023-09-23T20:37:48.465 INFO:teuthology.orchestra.run.smithi026.stderr:512 bytes copied, 0.000118721 s, 4.3 MB/s 2023-09-23T20:37:48.466 DEBUG:teuthology.orchestra.run.smithi026:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_3 2023-09-23T20:37:48.525 DEBUG:teuthology.orchestra.run.smithi026:> stat /dev/vg_nvme/lv_4 2023-09-23T20:37:48.583 INFO:teuthology.orchestra.run.smithi026.stdout: File: /dev/vg_nvme/lv_4 -> ../dm-3 2023-09-23T20:37:48.583 INFO:teuthology.orchestra.run.smithi026.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2023-09-23T20:37:48.583 INFO:teuthology.orchestra.run.smithi026.stdout:Device: 6h/6d Inode: 386966 Links: 1 2023-09-23T20:37:48.583 INFO:teuthology.orchestra.run.smithi026.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2023-09-23T20:37:48.584 INFO:teuthology.orchestra.run.smithi026.stdout:Context: system_u:object_r:device_t:s0 2023-09-23T20:37:48.584 INFO:teuthology.orchestra.run.smithi026.stdout:Access: 2023-09-23 20:37:35.117455184 +0000 2023-09-23T20:37:48.584 INFO:teuthology.orchestra.run.smithi026.stdout:Modify: 2023-09-23 20:30:12.655371140 +0000 2023-09-23T20:37:48.584 INFO:teuthology.orchestra.run.smithi026.stdout:Change: 2023-09-23 20:30:12.655371140 +0000 2023-09-23T20:37:48.584 INFO:teuthology.orchestra.run.smithi026.stdout: Birth: - 2023-09-23T20:37:48.584 DEBUG:teuthology.orchestra.run.smithi026:> sudo dd if=/dev/vg_nvme/lv_4 of=/dev/null count=1 2023-09-23T20:37:48.585 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:48 smithi070 ceph-mon[100755]: from='client.24100 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "placement": "2;smithi026=y;smithi070=x", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T20:37:48.585 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:48 smithi070 ceph-mon[100755]: Saving service mgr spec with placement smithi026=y;smithi070=x;count:2 2023-09-23T20:37:48.585 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:48 smithi070 ceph-mon[100755]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:37:48.585 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:48 smithi070 ceph-mon[100755]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:37:48.585 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:48 smithi070 ceph-mon[100755]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:37:48.585 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:48 smithi070 ceph-mon[100755]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:37:48.585 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:48 smithi070 ceph-mon[100755]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:37:48.585 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:48 smithi070 ceph-mon[100755]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:37:48.585 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:48 smithi070 ceph-mon[100755]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:37:48.585 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:48 smithi070 ceph-mon[100755]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-09-23T20:37:48.586 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:48 smithi070 ceph-mon[100755]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2023-09-23T20:37:48.586 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:48 smithi070 ceph-mon[100755]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-09-23T20:37:48.586 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:48 smithi070 ceph-mon[100755]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:37:48.651 INFO:teuthology.orchestra.run.smithi026.stderr:1+0 records in 2023-09-23T20:37:48.651 INFO:teuthology.orchestra.run.smithi026.stderr:1+0 records out 2023-09-23T20:37:48.651 INFO:teuthology.orchestra.run.smithi026.stderr:512 bytes copied, 0.000112856 s, 4.5 MB/s 2023-09-23T20:37:48.652 DEBUG:teuthology.orchestra.run.smithi026:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_4 2023-09-23T20:37:48.712 DEBUG:teuthology.orchestra.run.smithi070:> set -ex 2023-09-23T20:37:48.712 DEBUG:teuthology.orchestra.run.smithi070:> dd if=/scratch_devs of=/dev/stdout 2023-09-23T20:37:48.729 DEBUG:teuthology.misc:devs=['/dev/vg_nvme/lv_1', '/dev/vg_nvme/lv_2', '/dev/vg_nvme/lv_3', '/dev/vg_nvme/lv_4'] 2023-09-23T20:37:48.729 DEBUG:teuthology.orchestra.run.smithi070:> stat /dev/vg_nvme/lv_1 2023-09-23T20:37:48.786 INFO:teuthology.orchestra.run.smithi070.stdout: File: /dev/vg_nvme/lv_1 -> ../dm-0 2023-09-23T20:37:48.786 INFO:teuthology.orchestra.run.smithi070.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2023-09-23T20:37:48.786 INFO:teuthology.orchestra.run.smithi070.stdout:Device: 6h/6d Inode: 387424 Links: 1 2023-09-23T20:37:48.787 INFO:teuthology.orchestra.run.smithi070.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2023-09-23T20:37:48.787 INFO:teuthology.orchestra.run.smithi070.stdout:Context: system_u:object_r:device_t:s0 2023-09-23T20:37:48.787 INFO:teuthology.orchestra.run.smithi070.stdout:Access: 2023-09-23 20:37:26.924125572 +0000 2023-09-23T20:37:48.787 INFO:teuthology.orchestra.run.smithi070.stdout:Modify: 2023-09-23 20:29:57.401649752 +0000 2023-09-23T20:37:48.787 INFO:teuthology.orchestra.run.smithi070.stdout:Change: 2023-09-23 20:29:57.401649752 +0000 2023-09-23T20:37:48.787 INFO:teuthology.orchestra.run.smithi070.stdout: Birth: - 2023-09-23T20:37:48.787 DEBUG:teuthology.orchestra.run.smithi070:> sudo dd if=/dev/vg_nvme/lv_1 of=/dev/null count=1 2023-09-23T20:37:48.852 INFO:teuthology.orchestra.run.smithi070.stderr:1+0 records in 2023-09-23T20:37:48.853 INFO:teuthology.orchestra.run.smithi070.stderr:1+0 records out 2023-09-23T20:37:48.853 INFO:teuthology.orchestra.run.smithi070.stderr:512 bytes copied, 0.000135973 s, 3.8 MB/s 2023-09-23T20:37:48.854 DEBUG:teuthology.orchestra.run.smithi070:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_1 2023-09-23T20:37:48.911 DEBUG:teuthology.orchestra.run.smithi070:> stat /dev/vg_nvme/lv_2 2023-09-23T20:37:48.980 INFO:teuthology.orchestra.run.smithi070.stdout: File: /dev/vg_nvme/lv_2 -> ../dm-1 2023-09-23T20:37:48.980 INFO:teuthology.orchestra.run.smithi070.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2023-09-23T20:37:48.980 INFO:teuthology.orchestra.run.smithi070.stdout:Device: 6h/6d Inode: 389725 Links: 1 2023-09-23T20:37:48.980 INFO:teuthology.orchestra.run.smithi070.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2023-09-23T20:37:48.980 INFO:teuthology.orchestra.run.smithi070.stdout:Context: system_u:object_r:device_t:s0 2023-09-23T20:37:48.980 INFO:teuthology.orchestra.run.smithi070.stdout:Access: 2023-09-23 20:37:26.924125572 +0000 2023-09-23T20:37:48.980 INFO:teuthology.orchestra.run.smithi070.stdout:Modify: 2023-09-23 20:29:57.878635869 +0000 2023-09-23T20:37:48.980 INFO:teuthology.orchestra.run.smithi070.stdout:Change: 2023-09-23 20:29:57.878635869 +0000 2023-09-23T20:37:48.981 INFO:teuthology.orchestra.run.smithi070.stdout: Birth: - 2023-09-23T20:37:48.981 DEBUG:teuthology.orchestra.run.smithi070:> sudo dd if=/dev/vg_nvme/lv_2 of=/dev/null count=1 2023-09-23T20:37:49.048 INFO:teuthology.orchestra.run.smithi070.stderr:1+0 records in 2023-09-23T20:37:49.048 INFO:teuthology.orchestra.run.smithi070.stderr:1+0 records out 2023-09-23T20:37:49.048 INFO:teuthology.orchestra.run.smithi070.stderr:512 bytes copied, 8.8469e-05 s, 5.8 MB/s 2023-09-23T20:37:49.050 DEBUG:teuthology.orchestra.run.smithi070:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_2 2023-09-23T20:37:49.114 DEBUG:teuthology.orchestra.run.smithi070:> stat /dev/vg_nvme/lv_3 2023-09-23T20:37:49.172 INFO:teuthology.orchestra.run.smithi070.stdout: File: /dev/vg_nvme/lv_3 -> ../dm-2 2023-09-23T20:37:49.173 INFO:teuthology.orchestra.run.smithi070.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2023-09-23T20:37:49.173 INFO:teuthology.orchestra.run.smithi070.stdout:Device: 6h/6d Inode: 383679 Links: 1 2023-09-23T20:37:49.173 INFO:teuthology.orchestra.run.smithi070.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2023-09-23T20:37:49.173 INFO:teuthology.orchestra.run.smithi070.stdout:Context: system_u:object_r:device_t:s0 2023-09-23T20:37:49.173 INFO:teuthology.orchestra.run.smithi070.stdout:Access: 2023-09-23 20:37:49.125478578 +0000 2023-09-23T20:37:49.173 INFO:teuthology.orchestra.run.smithi070.stdout:Modify: 2023-09-23 20:29:58.370621550 +0000 2023-09-23T20:37:49.173 INFO:teuthology.orchestra.run.smithi070.stdout:Change: 2023-09-23 20:29:58.370621550 +0000 2023-09-23T20:37:49.173 INFO:teuthology.orchestra.run.smithi070.stdout: Birth: - 2023-09-23T20:37:49.174 DEBUG:teuthology.orchestra.run.smithi070:> sudo dd if=/dev/vg_nvme/lv_3 of=/dev/null count=1 2023-09-23T20:37:49.243 INFO:teuthology.orchestra.run.smithi070.stderr:1+0 records in 2023-09-23T20:37:49.244 INFO:teuthology.orchestra.run.smithi070.stderr:1+0 records out 2023-09-23T20:37:49.244 INFO:teuthology.orchestra.run.smithi070.stderr:512 bytes copied, 0.000102893 s, 5.0 MB/s 2023-09-23T20:37:49.245 DEBUG:teuthology.orchestra.run.smithi070:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_3 2023-09-23T20:37:49.304 DEBUG:teuthology.orchestra.run.smithi070:> stat /dev/vg_nvme/lv_4 2023-09-23T20:37:49.362 INFO:teuthology.orchestra.run.smithi070.stdout: File: /dev/vg_nvme/lv_4 -> ../dm-3 2023-09-23T20:37:49.362 INFO:teuthology.orchestra.run.smithi070.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2023-09-23T20:37:49.362 INFO:teuthology.orchestra.run.smithi070.stdout:Device: 6h/6d Inode: 391350 Links: 1 2023-09-23T20:37:49.362 INFO:teuthology.orchestra.run.smithi070.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2023-09-23T20:37:49.362 INFO:teuthology.orchestra.run.smithi070.stdout:Context: system_u:object_r:device_t:s0 2023-09-23T20:37:49.363 INFO:teuthology.orchestra.run.smithi070.stdout:Access: 2023-09-23 20:37:49.125478578 +0000 2023-09-23T20:37:49.363 INFO:teuthology.orchestra.run.smithi070.stdout:Modify: 2023-09-23 20:29:58.871606968 +0000 2023-09-23T20:37:49.363 INFO:teuthology.orchestra.run.smithi070.stdout:Change: 2023-09-23 20:29:58.871606968 +0000 2023-09-23T20:37:49.363 INFO:teuthology.orchestra.run.smithi070.stdout: Birth: - 2023-09-23T20:37:49.363 DEBUG:teuthology.orchestra.run.smithi070:> sudo dd if=/dev/vg_nvme/lv_4 of=/dev/null count=1 2023-09-23T20:37:49.426 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 20:37:49 smithi070 systemd[1]: Starting Ceph mgr.x for b3f8e4ec-5a50-11ee-8db4-212e2dc638e7... 2023-09-23T20:37:49.428 INFO:teuthology.orchestra.run.smithi070.stderr:1+0 records in 2023-09-23T20:37:49.429 INFO:teuthology.orchestra.run.smithi070.stderr:1+0 records out 2023-09-23T20:37:49.429 INFO:teuthology.orchestra.run.smithi070.stderr:512 bytes copied, 8.8945e-05 s, 5.8 MB/s 2023-09-23T20:37:49.430 DEBUG:teuthology.orchestra.run.smithi070:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_4 2023-09-23T20:37:49.488 INFO:tasks.cephadm:Deploying osd.0 on smithi026 with /dev/vg_nvme/lv_4... 2023-09-23T20:37:49.488 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph-volume lvm zap /dev/vg_nvme/lv_4 2023-09-23T20:37:49.714 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 20:37:49 smithi070 podman[102783]: 2023-09-23T20:37:49.714 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:49 smithi070 ceph-mon[100755]: Deploying daemon mgr.x on smithi070 2023-09-23T20:37:49.714 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:49 smithi070 ceph-mon[100755]: pgmap v20: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T20:37:49.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:37:49 smithi026 ceph-mon[95839]: Deploying daemon mgr.x on smithi070 2023-09-23T20:37:49.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:37:49 smithi026 ceph-mon[95839]: pgmap v20: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T20:37:49.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:49 smithi026 ceph-mon[109912]: Deploying daemon mgr.x on smithi070 2023-09-23T20:37:49.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:49 smithi026 ceph-mon[109912]: pgmap v20: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T20:37:49.999 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 20:37:49 smithi070 ceph-mgr[102885]: -- 172.21.15.70:0/2706346017 <== mon.1 v2:172.21.15.70:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x55997c4689c0 con 0x55997c43ec00 2023-09-23T20:37:49.999 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 20:37:49 smithi070 bash[102783]: e8decbdb0e5d12a94a03b17bc49beb6ed003bd7ea17295d64c01ed0fc4709de6 2023-09-23T20:37:49.999 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 20:37:49 smithi070 systemd[1]: Started Ceph mgr.x for b3f8e4ec-5a50-11ee-8db4-212e2dc638e7. 2023-09-23T20:37:50.000 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 20:37:49 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-x[102861]: 2023-09-23T20:37:49.831+0000 7f26a64c61c0 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2023-09-23T20:37:50.000 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 20:37:49 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-x[102861]: 2023-09-23T20:37:49.908+0000 7f26a64c61c0 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-09-23T20:37:50.000 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 20:37:49 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-x[102861]: 2023-09-23T20:37:49.998+0000 7f26a64c61c0 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-09-23T20:37:50.108 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:37:50.270 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 20:37:50 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-x[102861]: 2023-09-23T20:37:50.095+0000 7f26a64c61c0 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-09-23T20:37:50.586 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 20:37:50 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-x[102861]: 2023-09-23T20:37:50.268+0000 7f26a64c61c0 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-09-23T20:37:50.869 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 20:37:50 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-x[102861]: 2023-09-23T20:37:50.585+0000 7f26a64c61c0 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-09-23T20:37:50.869 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 20:37:50 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-x[102861]: 2023-09-23T20:37:50.677+0000 7f26a64c61c0 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-09-23T20:37:50.870 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 20:37:50 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-x[102861]: 2023-09-23T20:37:50.808+0000 7f26a64c61c0 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-09-23T20:37:50.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:50 smithi070 ceph-mon[100755]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:37:50.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:50 smithi070 ceph-mon[100755]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:37:50.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:50 smithi070 ceph-mon[100755]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:37:50.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:50 smithi070 ceph-mon[100755]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:37:50.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:50 smithi070 ceph-mon[100755]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:37:50.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:50 smithi070 ceph-mon[100755]: pgmap v21: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T20:37:50.872 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:37:50 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:37:50.872 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:37:50 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:37:50.872 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:37:50 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:37:50.872 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:37:50 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:37:50.872 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:37:50 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:37:50.872 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:37:50 smithi026 ceph-mon[95839]: pgmap v21: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T20:37:50.873 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:50 smithi026 ceph-mon[109912]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:37:50.873 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:50 smithi026 ceph-mon[109912]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:37:50.873 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:50 smithi026 ceph-mon[109912]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:37:50.873 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:50 smithi026 ceph-mon[109912]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:37:50.873 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:50 smithi026 ceph-mon[109912]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:37:50.873 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:50 smithi026 ceph-mon[109912]: pgmap v21: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T20:37:51.341 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 20:37:50 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-x[102861]: 2023-09-23T20:37:50.903+0000 7f26a64c61c0 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-09-23T20:37:51.607 INFO:teuthology.orchestra.run.smithi026.stderr:--> Zapping: /dev/vg_nvme/lv_4 2023-09-23T20:37:51.607 INFO:teuthology.orchestra.run.smithi026.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_4 bs=1M count=10 conv=fsync 2023-09-23T20:37:51.607 INFO:teuthology.orchestra.run.smithi026.stderr: stderr: 10+0 records in 2023-09-23T20:37:51.607 INFO:teuthology.orchestra.run.smithi026.stderr:10+0 records out 2023-09-23T20:37:51.607 INFO:teuthology.orchestra.run.smithi026.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.00821282 s, 1.3 GB/s 2023-09-23T20:37:51.607 INFO:teuthology.orchestra.run.smithi026.stderr:--> Zapping successful for: 2023-09-23T20:37:51.619 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 20:37:51 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-x[102861]: 2023-09-23T20:37:51.340+0000 7f26a64c61c0 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-09-23T20:37:52.153 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph orch daemon add osd smithi026:vg_nvme/lv_4 2023-09-23T20:37:52.369 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 20:37:51 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-x[102861]: 2023-09-23T20:37:51.921+0000 7f26a64c61c0 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-09-23T20:37:52.369 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 20:37:52 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-x[102861]: 2023-09-23T20:37:52.086+0000 7f26a64c61c0 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-09-23T20:37:52.720 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:37:52 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:37:52.720 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:37:52 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:37:52.720 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:37:52 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:37:52.720 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:37:52 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:37:52.720 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:37:52 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:37:52.721 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:37:52 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:37:52.721 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:37:52 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-09-23T20:37:52.721 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:37:52 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-09-23T20:37:52.721 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:37:52 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:37:52.721 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:52 smithi026 ceph-mon[109912]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:37:52.721 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:52 smithi026 ceph-mon[109912]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:37:52.721 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:52 smithi026 ceph-mon[109912]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:37:52.721 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:52 smithi026 ceph-mon[109912]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:37:52.721 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:52 smithi026 ceph-mon[109912]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:37:52.721 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:52 smithi026 ceph-mon[109912]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:37:52.722 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:52 smithi026 ceph-mon[109912]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-09-23T20:37:52.722 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:52 smithi026 ceph-mon[109912]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-09-23T20:37:52.722 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:52 smithi026 ceph-mon[109912]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:37:52.836 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:52 smithi070 ceph-mon[100755]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:37:52.836 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:52 smithi070 ceph-mon[100755]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:37:52.836 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:52 smithi070 ceph-mon[100755]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:37:52.836 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:52 smithi070 ceph-mon[100755]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:37:52.836 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:52 smithi070 ceph-mon[100755]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:37:52.836 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:52 smithi070 ceph-mon[100755]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:37:52.836 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:52 smithi070 ceph-mon[100755]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-09-23T20:37:52.836 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:52 smithi070 ceph-mon[100755]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-09-23T20:37:52.836 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:52 smithi070 ceph-mon[100755]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:37:53.032 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:37:53.089 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 20:37:52 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-x[102861]: 2023-09-23T20:37:52.835+0000 7f26a64c61c0 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-09-23T20:37:53.090 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 20:37:52 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-x[102861]: 2023-09-23T20:37:52.923+0000 7f26a64c61c0 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2023-09-23T20:37:53.360 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 20:37:53 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-x[102861]: 2023-09-23T20:37:53.088+0000 7f26a64c61c0 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-09-23T20:37:53.619 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 20:37:53 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-x[102861]: 2023-09-23T20:37:53.359+0000 7f26a64c61c0 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2023-09-23T20:37:53.619 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 20:37:53 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-x[102861]: 2023-09-23T20:37:53.444+0000 7f26a64c61c0 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-09-23T20:37:53.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:53 smithi070 ceph-mon[100755]: Reconfiguring mgr.y (unknown last config time)... 2023-09-23T20:37:53.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:53 smithi070 ceph-mon[100755]: Reconfiguring daemon mgr.y on smithi026 2023-09-23T20:37:53.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:53 smithi070 ceph-mon[100755]: pgmap v22: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T20:37:53.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:53 smithi070 ceph-mon[100755]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:37:53.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:53 smithi070 ceph-mon[100755]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:37:53.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:53 smithi070 ceph-mon[100755]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:37:53.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:53 smithi070 ceph-mon[100755]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:37:53.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:53 smithi070 ceph-mon[100755]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:37:53.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:53 smithi070 ceph-mon[100755]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:37:53.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:53 smithi026 ceph-mon[109912]: Reconfiguring mgr.y (unknown last config time)... 2023-09-23T20:37:53.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:53 smithi026 ceph-mon[109912]: Reconfiguring daemon mgr.y on smithi026 2023-09-23T20:37:53.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:53 smithi026 ceph-mon[109912]: pgmap v22: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T20:37:53.883 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:53 smithi026 ceph-mon[109912]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:37:53.883 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:53 smithi026 ceph-mon[109912]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:37:53.883 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:53 smithi026 ceph-mon[109912]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:37:53.883 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:53 smithi026 ceph-mon[109912]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:37:53.883 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:53 smithi026 ceph-mon[109912]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:37:53.883 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:53 smithi026 ceph-mon[109912]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:37:53.883 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:37:53 smithi026 ceph-mon[95839]: Reconfiguring mgr.y (unknown last config time)... 2023-09-23T20:37:53.883 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:37:53 smithi026 ceph-mon[95839]: Reconfiguring daemon mgr.y on smithi026 2023-09-23T20:37:53.883 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:37:53 smithi026 ceph-mon[95839]: pgmap v22: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T20:37:53.883 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:37:53 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:37:53.883 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:37:53 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:37:53.884 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:37:53 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:37:53.884 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:37:53 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:37:53.884 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:37:53 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:37:53.884 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:37:53 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:37:53.977 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 20:37:53 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-x[102861]: 2023-09-23T20:37:53.725+0000 7f26a64c61c0 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-09-23T20:37:54.369 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 20:37:53 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-x[102861]: 2023-09-23T20:37:53.976+0000 7f26a64c61c0 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-09-23T20:37:54.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:54 smithi070 ceph-mon[100755]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-09-23T20:37:54.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:54 smithi070 ceph-mon[100755]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-09-23T20:37:54.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:54 smithi070 ceph-mon[100755]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:37:54.870 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 20:37:54 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-x[102861]: 2023-09-23T20:37:54.661+0000 7f26a64c61c0 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-09-23T20:37:54.870 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 20:37:54 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-x[102861]: 2023-09-23T20:37:54.790+0000 7f26a64c61c0 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-09-23T20:37:54.881 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:54 smithi026 ceph-mon[109912]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-09-23T20:37:54.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:54 smithi026 ceph-mon[109912]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-09-23T20:37:54.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:54 smithi026 ceph-mon[109912]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:37:54.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:37:54 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-09-23T20:37:54.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:37:54 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-09-23T20:37:54.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:37:54 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:37:55.294 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 20:37:54 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-x[102861]: 2023-09-23T20:37:54.974+0000 7f26a64c61c0 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-09-23T20:37:55.294 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 20:37:55 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-x[102861]: 2023-09-23T20:37:55.085+0000 7f26a64c61c0 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-09-23T20:37:55.571 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 20:37:55 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-x[102861]: 2023-09-23T20:37:55.293+0000 7f26a64c61c0 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-09-23T20:37:55.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:55 smithi070 ceph-mon[100755]: from='client.24116 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi026:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T20:37:55.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:55 smithi070 ceph-mon[100755]: pgmap v23: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T20:37:55.881 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:37:55 smithi026 ceph-mon[95839]: from='client.24116 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi026:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T20:37:55.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:37:55 smithi026 ceph-mon[95839]: pgmap v23: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T20:37:55.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:55 smithi026 ceph-mon[109912]: from='client.24116 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi026:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T20:37:55.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:55 smithi026 ceph-mon[109912]: pgmap v23: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T20:37:56.572 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 20:37:56 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-x[102861]: 2023-09-23T20:37:56.126+0000 7f26a64c61c0 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-09-23T20:37:56.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:56 smithi070 ceph-mon[100755]: Standby manager daemon x started 2023-09-23T20:37:56.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:56 smithi070 ceph-mon[100755]: from='mgr.? 172.21.15.70:0/3415522603' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-09-23T20:37:56.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:56 smithi070 ceph-mon[100755]: from='mgr.? 172.21.15.70:0/3415522603' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-09-23T20:37:56.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:56 smithi070 ceph-mon[100755]: from='mgr.? 172.21.15.70:0/3415522603' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-09-23T20:37:56.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:56 smithi070 ceph-mon[100755]: from='mgr.? 172.21.15.70:0/3415522603' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-09-23T20:37:56.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:37:56 smithi026 ceph-mon[95839]: Standby manager daemon x started 2023-09-23T20:37:56.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:37:56 smithi026 ceph-mon[95839]: from='mgr.? 172.21.15.70:0/3415522603' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-09-23T20:37:56.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:37:56 smithi026 ceph-mon[95839]: from='mgr.? 172.21.15.70:0/3415522603' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-09-23T20:37:56.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:37:56 smithi026 ceph-mon[95839]: from='mgr.? 172.21.15.70:0/3415522603' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-09-23T20:37:56.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:37:56 smithi026 ceph-mon[95839]: from='mgr.? 172.21.15.70:0/3415522603' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-09-23T20:37:56.883 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:56 smithi026 ceph-mon[109912]: Standby manager daemon x started 2023-09-23T20:37:56.883 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:56 smithi026 ceph-mon[109912]: from='mgr.? 172.21.15.70:0/3415522603' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-09-23T20:37:56.883 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:56 smithi026 ceph-mon[109912]: from='mgr.? 172.21.15.70:0/3415522603' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-09-23T20:37:56.883 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:56 smithi026 ceph-mon[109912]: from='mgr.? 172.21.15.70:0/3415522603' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-09-23T20:37:56.883 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:56 smithi026 ceph-mon[109912]: from='mgr.? 172.21.15.70:0/3415522603' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-09-23T20:37:57.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:57 smithi070 ceph-mon[100755]: pgmap v24: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T20:37:57.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:57 smithi070 ceph-mon[100755]: mgrmap e14: y(active, since 60s), standbys: x 2023-09-23T20:37:57.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:57 smithi070 ceph-mon[100755]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-09-23T20:37:57.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:57 smithi070 ceph-mon[100755]: from='client.? 172.21.15.26:0/4064730960' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "f837feb7-060e-4965-9b48-bed9f3672d05"}]: dispatch 2023-09-23T20:37:57.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:57 smithi070 ceph-mon[100755]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "f837feb7-060e-4965-9b48-bed9f3672d05"}]: dispatch 2023-09-23T20:37:57.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:57 smithi070 ceph-mon[100755]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "f837feb7-060e-4965-9b48-bed9f3672d05"}]': finished 2023-09-23T20:37:57.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:57 smithi070 ceph-mon[100755]: osdmap e5: 1 total, 0 up, 1 in 2023-09-23T20:37:57.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:57 smithi070 ceph-mon[100755]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-23T20:37:57.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:57 smithi070 ceph-mon[100755]: from='client.? 172.21.15.26:0/2978213770' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-09-23T20:37:57.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:37:57 smithi026 ceph-mon[95839]: pgmap v24: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T20:37:57.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:37:57 smithi026 ceph-mon[95839]: mgrmap e14: y(active, since 60s), standbys: x 2023-09-23T20:37:57.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:37:57 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-09-23T20:37:57.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:37:57 smithi026 ceph-mon[95839]: from='client.? 172.21.15.26:0/4064730960' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "f837feb7-060e-4965-9b48-bed9f3672d05"}]: dispatch 2023-09-23T20:37:57.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:37:57 smithi026 ceph-mon[95839]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "f837feb7-060e-4965-9b48-bed9f3672d05"}]: dispatch 2023-09-23T20:37:57.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:37:57 smithi026 ceph-mon[95839]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "f837feb7-060e-4965-9b48-bed9f3672d05"}]': finished 2023-09-23T20:37:57.883 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:37:57 smithi026 ceph-mon[95839]: osdmap e5: 1 total, 0 up, 1 in 2023-09-23T20:37:57.883 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:37:57 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-23T20:37:57.883 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:37:57 smithi026 ceph-mon[95839]: from='client.? 172.21.15.26:0/2978213770' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-09-23T20:37:57.883 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:57 smithi026 ceph-mon[109912]: pgmap v24: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T20:37:57.883 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:57 smithi026 ceph-mon[109912]: mgrmap e14: y(active, since 60s), standbys: x 2023-09-23T20:37:57.883 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:57 smithi026 ceph-mon[109912]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-09-23T20:37:57.883 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:57 smithi026 ceph-mon[109912]: from='client.? 172.21.15.26:0/4064730960' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "f837feb7-060e-4965-9b48-bed9f3672d05"}]: dispatch 2023-09-23T20:37:57.883 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:57 smithi026 ceph-mon[109912]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "f837feb7-060e-4965-9b48-bed9f3672d05"}]: dispatch 2023-09-23T20:37:57.883 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:57 smithi026 ceph-mon[109912]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "f837feb7-060e-4965-9b48-bed9f3672d05"}]': finished 2023-09-23T20:37:57.883 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:57 smithi026 ceph-mon[109912]: osdmap e5: 1 total, 0 up, 1 in 2023-09-23T20:37:57.883 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:57 smithi026 ceph-mon[109912]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-23T20:37:57.884 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:57 smithi026 ceph-mon[109912]: from='client.? 172.21.15.26:0/2978213770' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-09-23T20:37:59.766 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:37:59 smithi026 ceph-mon[95839]: pgmap v26: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T20:37:59.766 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:37:59 smithi026 ceph-mon[109912]: pgmap v26: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T20:37:59.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:37:59 smithi070 ceph-mon[100755]: pgmap v26: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T20:38:01.833 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:38:01 smithi026 ceph-mon[95839]: pgmap v27: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T20:38:01.834 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:38:01 smithi026 ceph-mon[109912]: pgmap v27: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T20:38:01.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:38:01 smithi070 ceph-mon[100755]: pgmap v27: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T20:38:02.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:38:02 smithi070 ceph-mon[100755]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2023-09-23T20:38:02.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:38:02 smithi070 ceph-mon[100755]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:38:02.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:38:02 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2023-09-23T20:38:02.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:38:02 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:38:02.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:38:02 smithi026 ceph-mon[109912]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2023-09-23T20:38:02.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:38:02 smithi026 ceph-mon[109912]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:38:03.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:38:03 smithi070 ceph-mon[100755]: pgmap v28: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T20:38:03.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:38:03 smithi070 ceph-mon[100755]: Deploying daemon osd.0 on smithi026 2023-09-23T20:38:03.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:38:03 smithi026 ceph-mon[95839]: pgmap v28: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T20:38:03.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:38:03 smithi026 ceph-mon[95839]: Deploying daemon osd.0 on smithi026 2023-09-23T20:38:03.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:38:03 smithi026 ceph-mon[109912]: pgmap v28: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T20:38:03.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:38:03 smithi026 ceph-mon[109912]: Deploying daemon osd.0 on smithi026 2023-09-23T20:38:05.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:38:05 smithi070 ceph-mon[100755]: pgmap v29: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T20:38:05.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:38:05 smithi026 ceph-mon[95839]: pgmap v29: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T20:38:05.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:38:05 smithi026 ceph-mon[109912]: pgmap v29: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T20:38:07.631 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:38:07 smithi026 ceph-mon[95839]: pgmap v30: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T20:38:07.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:38:07 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:38:07.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:38:07 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:38:07.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:38:07 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:38:07.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:38:07 smithi026 ceph-mon[109912]: pgmap v30: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T20:38:07.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:38:07 smithi026 ceph-mon[109912]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:38:07.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:38:07 smithi026 ceph-mon[109912]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:38:07.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:38:07 smithi026 ceph-mon[109912]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:38:07.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:38:07 smithi070 ceph-mon[100755]: pgmap v30: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T20:38:07.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:38:07 smithi070 ceph-mon[100755]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:38:07.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:38:07 smithi070 ceph-mon[100755]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:38:07.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:38:07 smithi070 ceph-mon[100755]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:38:09.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:38:09 smithi026 ceph-mon[109912]: pgmap v31: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T20:38:09.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:38:09 smithi026 ceph-mon[109912]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:38:09.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:38:09 smithi026 ceph-mon[109912]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:38:09.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:38:09 smithi026 ceph-mon[95839]: pgmap v31: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T20:38:09.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:38:09 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:38:09.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:38:09 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:38:09.745 INFO:teuthology.orchestra.run.smithi026.stdout:Created osd(s) 0 on host 'smithi026' 2023-09-23T20:38:09.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:38:09 smithi070 ceph-mon[100755]: pgmap v31: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T20:38:09.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:38:09 smithi070 ceph-mon[100755]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:38:09.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:38:09 smithi070 ceph-mon[100755]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:38:10.125 DEBUG:teuthology.orchestra.run.smithi026:osd.0> sudo journalctl -f -n 0 -u ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7@osd.0.service 2023-09-23T20:38:10.127 INFO:tasks.cephadm:Deploying osd.1 on smithi026 with /dev/vg_nvme/lv_3... 2023-09-23T20:38:10.127 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph-volume lvm zap /dev/vg_nvme/lv_3 2023-09-23T20:38:10.171 INFO:journalctl@ceph.osd.0.smithi026.stdout:-- Logs begin at Sat 2023-09-23 20:20:42 UTC. -- 2023-09-23T20:38:11.021 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:38:10 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:38:11.021 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:38:10 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:38:11.021 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:38:10 smithi026 ceph-mon[95839]: from='osd.0 [v2:172.21.15.26:6802/2638118366,v1:172.21.15.26:6803/2638118366]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2023-09-23T20:38:11.021 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:38:10 smithi026 ceph-mon[95839]: pgmap v32: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T20:38:11.022 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:38:10 smithi026 ceph-mon[109912]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:38:11.022 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:38:10 smithi026 ceph-mon[109912]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:38:11.022 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:38:10 smithi026 ceph-mon[109912]: from='osd.0 [v2:172.21.15.26:6802/2638118366,v1:172.21.15.26:6803/2638118366]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2023-09-23T20:38:11.022 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:38:10 smithi026 ceph-mon[109912]: pgmap v32: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T20:38:11.052 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:38:11.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:38:10 smithi070 ceph-mon[100755]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:38:11.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:38:10 smithi070 ceph-mon[100755]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:38:11.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:38:10 smithi070 ceph-mon[100755]: from='osd.0 [v2:172.21.15.26:6802/2638118366,v1:172.21.15.26:6803/2638118366]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2023-09-23T20:38:11.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:38:10 smithi070 ceph-mon[100755]: pgmap v32: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T20:38:12.034 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:38:11 smithi026 ceph-mon[95839]: from='osd.0 [v2:172.21.15.26:6802/2638118366,v1:172.21.15.26:6803/2638118366]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2023-09-23T20:38:12.035 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:38:11 smithi026 ceph-mon[95839]: osdmap e6: 1 total, 0 up, 1 in 2023-09-23T20:38:12.035 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:38:11 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-23T20:38:12.035 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:38:11 smithi026 ceph-mon[95839]: from='osd.0 [v2:172.21.15.26:6802/2638118366,v1:172.21.15.26:6803/2638118366]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi026", "root=default"]}]: dispatch 2023-09-23T20:38:12.035 INFO:journalctl@ceph.osd.0.smithi026.stdout:Sep 23 20:38:11 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd-0[114366]: 2023-09-23T20:38:11.706+0000 7f1867935700 -1 osd.0 0 waiting for initial osdmap 2023-09-23T20:38:12.035 INFO:journalctl@ceph.osd.0.smithi026.stdout:Sep 23 20:38:11 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd-0[114366]: 2023-09-23T20:38:11.709+0000 7f1861f2f700 -1 osd.0 7 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-09-23T20:38:12.035 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:38:11 smithi026 ceph-mon[109912]: from='osd.0 [v2:172.21.15.26:6802/2638118366,v1:172.21.15.26:6803/2638118366]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2023-09-23T20:38:12.036 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:38:11 smithi026 ceph-mon[109912]: osdmap e6: 1 total, 0 up, 1 in 2023-09-23T20:38:12.036 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:38:11 smithi026 ceph-mon[109912]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-23T20:38:12.036 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:38:11 smithi026 ceph-mon[109912]: from='osd.0 [v2:172.21.15.26:6802/2638118366,v1:172.21.15.26:6803/2638118366]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi026", "root=default"]}]: dispatch 2023-09-23T20:38:12.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:38:11 smithi070 ceph-mon[100755]: from='osd.0 [v2:172.21.15.26:6802/2638118366,v1:172.21.15.26:6803/2638118366]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2023-09-23T20:38:12.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:38:11 smithi070 ceph-mon[100755]: osdmap e6: 1 total, 0 up, 1 in 2023-09-23T20:38:12.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:38:11 smithi070 ceph-mon[100755]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-23T20:38:12.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:38:11 smithi070 ceph-mon[100755]: from='osd.0 [v2:172.21.15.26:6802/2638118366,v1:172.21.15.26:6803/2638118366]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi026", "root=default"]}]: dispatch 2023-09-23T20:38:12.789 INFO:teuthology.orchestra.run.smithi026.stderr:--> Zapping: /dev/vg_nvme/lv_3 2023-09-23T20:38:12.789 INFO:teuthology.orchestra.run.smithi026.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_3 bs=1M count=10 conv=fsync 2023-09-23T20:38:12.790 INFO:teuthology.orchestra.run.smithi026.stderr: stderr: 10+0 records in 2023-09-23T20:38:12.790 INFO:teuthology.orchestra.run.smithi026.stderr:10+0 records out 2023-09-23T20:38:12.790 INFO:teuthology.orchestra.run.smithi026.stderr:10485760 bytes (10 MB, 10 MiB) copied, 0.00894612 s, 1.2 GB/s 2023-09-23T20:38:12.790 INFO:teuthology.orchestra.run.smithi026.stderr:--> Zapping successful for: 2023-09-23T20:38:12.993 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:38:12 smithi026 ceph-mon[95839]: from='osd.0 [v2:172.21.15.26:6802/2638118366,v1:172.21.15.26:6803/2638118366]' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi026", "root=default"]}]': finished 2023-09-23T20:38:12.993 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:38:12 smithi026 ceph-mon[95839]: osdmap e7: 1 total, 0 up, 1 in 2023-09-23T20:38:12.993 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:38:12 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-23T20:38:12.993 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:38:12 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-23T20:38:12.993 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:38:12 smithi026 ceph-mon[95839]: pgmap v35: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T20:38:12.994 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:38:12 smithi026 ceph-mon[109912]: from='osd.0 [v2:172.21.15.26:6802/2638118366,v1:172.21.15.26:6803/2638118366]' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi026", "root=default"]}]': finished 2023-09-23T20:38:12.994 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:38:12 smithi026 ceph-mon[109912]: osdmap e7: 1 total, 0 up, 1 in 2023-09-23T20:38:12.994 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:38:12 smithi026 ceph-mon[109912]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-23T20:38:12.994 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:38:12 smithi026 ceph-mon[109912]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-23T20:38:12.994 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:38:12 smithi026 ceph-mon[109912]: pgmap v35: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T20:38:13.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:38:12 smithi070 ceph-mon[100755]: from='osd.0 [v2:172.21.15.26:6802/2638118366,v1:172.21.15.26:6803/2638118366]' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi026", "root=default"]}]': finished 2023-09-23T20:38:13.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:38:12 smithi070 ceph-mon[100755]: osdmap e7: 1 total, 0 up, 1 in 2023-09-23T20:38:13.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:38:12 smithi070 ceph-mon[100755]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-23T20:38:13.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:38:12 smithi070 ceph-mon[100755]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-23T20:38:13.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:38:12 smithi070 ceph-mon[100755]: pgmap v35: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T20:38:13.466 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph orch daemon add osd smithi026:vg_nvme/lv_3 2023-09-23T20:38:14.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:38:13 smithi070 ceph-mon[100755]: purged_snaps scrub starts 2023-09-23T20:38:14.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:38:13 smithi070 ceph-mon[100755]: purged_snaps scrub ok 2023-09-23T20:38:14.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:38:13 smithi070 ceph-mon[100755]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-23T20:38:14.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:38:13 smithi070 ceph-mon[100755]: osd.0 [v2:172.21.15.26:6802/2638118366,v1:172.21.15.26:6803/2638118366] boot 2023-09-23T20:38:14.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:38:13 smithi070 ceph-mon[100755]: osdmap e8: 1 total, 1 up, 1 in 2023-09-23T20:38:14.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:38:13 smithi070 ceph-mon[100755]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-23T20:38:14.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:38:13 smithi070 ceph-mon[100755]: Detected new or changed devices on smithi026 2023-09-23T20:38:14.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:38:13 smithi070 ceph-mon[100755]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:38:14.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:38:13 smithi070 ceph-mon[100755]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:38:14.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:38:13 smithi070 ceph-mon[100755]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:38:14.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:38:13 smithi070 ceph-mon[100755]: Adjusting osd_memory_target on smithi026 to 16149M 2023-09-23T20:38:14.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:38:13 smithi070 ceph-mon[100755]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:38:14.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:38:13 smithi070 ceph-mon[100755]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:38:14.121 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:38:13 smithi070 ceph-mon[100755]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:38:14.121 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:38:13 smithi070 ceph-mon[100755]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:38:14.125 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:38:14.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:38:13 smithi026 ceph-mon[95839]: purged_snaps scrub starts 2023-09-23T20:38:14.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:38:13 smithi026 ceph-mon[95839]: purged_snaps scrub ok 2023-09-23T20:38:14.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:38:13 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-23T20:38:14.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:38:13 smithi026 ceph-mon[95839]: osd.0 [v2:172.21.15.26:6802/2638118366,v1:172.21.15.26:6803/2638118366] boot 2023-09-23T20:38:14.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:38:13 smithi026 ceph-mon[95839]: osdmap e8: 1 total, 1 up, 1 in 2023-09-23T20:38:14.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:38:13 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-23T20:38:14.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:38:13 smithi026 ceph-mon[95839]: Detected new or changed devices on smithi026 2023-09-23T20:38:14.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:38:13 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:38:14.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:38:13 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:38:14.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:38:13 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:38:14.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:38:13 smithi026 ceph-mon[95839]: Adjusting osd_memory_target on smithi026 to 16149M 2023-09-23T20:38:14.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:38:13 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:38:14.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:38:13 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:38:14.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:38:13 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:38:14.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:38:13 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:38:14.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:38:13 smithi026 ceph-mon[109912]: purged_snaps scrub starts 2023-09-23T20:38:14.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:38:13 smithi026 ceph-mon[109912]: purged_snaps scrub ok 2023-09-23T20:38:14.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:38:13 smithi026 ceph-mon[109912]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-23T20:38:14.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:38:13 smithi026 ceph-mon[109912]: osd.0 [v2:172.21.15.26:6802/2638118366,v1:172.21.15.26:6803/2638118366] boot 2023-09-23T20:38:14.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:38:13 smithi026 ceph-mon[109912]: osdmap e8: 1 total, 1 up, 1 in 2023-09-23T20:38:14.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:38:13 smithi026 ceph-mon[109912]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-23T20:38:14.134 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:38:13 smithi026 ceph-mon[109912]: Detected new or changed devices on smithi026 2023-09-23T20:38:14.134 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:38:13 smithi026 ceph-mon[109912]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:38:14.134 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:38:13 smithi026 ceph-mon[109912]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:38:14.134 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:38:13 smithi026 ceph-mon[109912]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:38:14.134 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:38:13 smithi026 ceph-mon[109912]: Adjusting osd_memory_target on smithi026 to 16149M 2023-09-23T20:38:14.134 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:38:13 smithi026 ceph-mon[109912]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:38:14.134 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:38:13 smithi026 ceph-mon[109912]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:38:14.134 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:38:13 smithi026 ceph-mon[109912]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:38:14.134 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:38:13 smithi026 ceph-mon[109912]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:38:14.946 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:38:14 smithi026 ceph-mon[95839]: osdmap e9: 1 total, 1 up, 1 in 2023-09-23T20:38:14.946 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:38:14 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:38:14.946 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:38:14 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:38:14.946 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:38:14 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:38:14.946 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:38:14 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:38:14.946 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:38:14 smithi026 ceph-mon[95839]: pgmap v38: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2023-09-23T20:38:14.947 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:38:14 smithi026 ceph-mon[109912]: osdmap e9: 1 total, 1 up, 1 in 2023-09-23T20:38:14.947 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:38:14 smithi026 ceph-mon[109912]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:38:14.947 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:38:14 smithi026 ceph-mon[109912]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:38:14.947 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:38:14 smithi026 ceph-mon[109912]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:38:14.947 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:38:14 smithi026 ceph-mon[109912]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:38:14.947 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:38:14 smithi026 ceph-mon[109912]: pgmap v38: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2023-09-23T20:38:15.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:38:14 smithi070 ceph-mon[100755]: osdmap e9: 1 total, 1 up, 1 in 2023-09-23T20:38:15.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:38:14 smithi070 ceph-mon[100755]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:38:15.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:38:14 smithi070 ceph-mon[100755]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:38:15.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:38:14 smithi070 ceph-mon[100755]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:38:15.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:38:14 smithi070 ceph-mon[100755]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:38:15.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:38:14 smithi070 ceph-mon[100755]: pgmap v38: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2023-09-23T20:38:16.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:38:15 smithi070 ceph-mon[100755]: from='client.14244 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi026:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T20:38:16.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:38:15 smithi070 ceph-mon[100755]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-09-23T20:38:16.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:38:15 smithi070 ceph-mon[100755]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-09-23T20:38:16.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:38:15 smithi070 ceph-mon[100755]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:38:16.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:38:15 smithi026 ceph-mon[109912]: from='client.14244 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi026:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T20:38:16.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:38:15 smithi026 ceph-mon[109912]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-09-23T20:38:16.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:38:15 smithi026 ceph-mon[109912]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-09-23T20:38:16.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:38:15 smithi026 ceph-mon[109912]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:38:16.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:38:15 smithi026 ceph-mon[95839]: from='client.14244 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi026:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T20:38:16.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:38:15 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-09-23T20:38:16.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:38:15 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-09-23T20:38:16.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:38:15 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:38:17.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:38:16 smithi070 ceph-mon[100755]: pgmap v39: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2023-09-23T20:38:17.123 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:38:16 smithi026 ceph-mon[95839]: pgmap v39: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2023-09-23T20:38:17.123 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:38:16 smithi026 ceph-mon[109912]: pgmap v39: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2023-09-23T20:38:18.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:38:17 smithi070 ceph-mon[100755]: from='client.? 172.21.15.26:0/1194461001' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "917f65aa-5963-4576-988a-396698b7c6a8"}]: dispatch 2023-09-23T20:38:18.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:38:17 smithi070 ceph-mon[100755]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "917f65aa-5963-4576-988a-396698b7c6a8"}]: dispatch 2023-09-23T20:38:18.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:38:17 smithi070 ceph-mon[100755]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "917f65aa-5963-4576-988a-396698b7c6a8"}]': finished 2023-09-23T20:38:18.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:38:17 smithi070 ceph-mon[100755]: osdmap e10: 2 total, 1 up, 2 in 2023-09-23T20:38:18.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:38:17 smithi070 ceph-mon[100755]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-23T20:38:18.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:38:17 smithi026 ceph-mon[95839]: from='client.? 172.21.15.26:0/1194461001' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "917f65aa-5963-4576-988a-396698b7c6a8"}]: dispatch 2023-09-23T20:38:18.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:38:17 smithi026 ceph-mon[95839]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "917f65aa-5963-4576-988a-396698b7c6a8"}]: dispatch 2023-09-23T20:38:18.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:38:17 smithi026 ceph-mon[95839]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "917f65aa-5963-4576-988a-396698b7c6a8"}]': finished 2023-09-23T20:38:18.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:38:17 smithi026 ceph-mon[95839]: osdmap e10: 2 total, 1 up, 2 in 2023-09-23T20:38:18.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:38:17 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-23T20:38:18.383 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:38:17 smithi026 ceph-mon[109912]: from='client.? 172.21.15.26:0/1194461001' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "917f65aa-5963-4576-988a-396698b7c6a8"}]: dispatch 2023-09-23T20:38:18.383 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:38:17 smithi026 ceph-mon[109912]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "917f65aa-5963-4576-988a-396698b7c6a8"}]: dispatch 2023-09-23T20:38:18.383 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:38:17 smithi026 ceph-mon[109912]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "917f65aa-5963-4576-988a-396698b7c6a8"}]': finished 2023-09-23T20:38:18.383 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:38:17 smithi026 ceph-mon[109912]: osdmap e10: 2 total, 1 up, 2 in 2023-09-23T20:38:18.383 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:38:17 smithi026 ceph-mon[109912]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-23T20:38:19.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:38:18 smithi070 ceph-mon[100755]: from='client.? 172.21.15.26:0/712702076' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-09-23T20:38:19.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:38:18 smithi070 ceph-mon[100755]: pgmap v41: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2023-09-23T20:38:19.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:38:18 smithi026 ceph-mon[95839]: from='client.? 172.21.15.26:0/712702076' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-09-23T20:38:19.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:38:18 smithi026 ceph-mon[95839]: pgmap v41: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2023-09-23T20:38:19.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:38:18 smithi026 ceph-mon[109912]: from='client.? 172.21.15.26:0/712702076' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-09-23T20:38:19.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:38:18 smithi026 ceph-mon[109912]: pgmap v41: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2023-09-23T20:38:21.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:38:21 smithi070 ceph-mon[100755]: pgmap v42: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2023-09-23T20:38:21.881 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:38:21 smithi026 ceph-mon[95839]: pgmap v42: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2023-09-23T20:38:21.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:38:21 smithi026 ceph-mon[109912]: pgmap v42: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2023-09-23T20:38:23.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:38:23 smithi070 ceph-mon[100755]: pgmap v43: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2023-09-23T20:38:23.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:38:23 smithi026 ceph-mon[95839]: pgmap v43: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2023-09-23T20:38:23.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:38:23 smithi026 ceph-mon[109912]: pgmap v43: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2023-09-23T20:38:24.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:38:24 smithi070 ceph-mon[100755]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2023-09-23T20:38:24.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:38:24 smithi070 ceph-mon[100755]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:38:24.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:38:24 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2023-09-23T20:38:24.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:38:24 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:38:24.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:38:24 smithi026 ceph-mon[109912]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2023-09-23T20:38:24.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:38:24 smithi026 ceph-mon[109912]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:38:25.794 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:38:25 smithi026 ceph-mon[95839]: Deploying daemon osd.1 on smithi026 2023-09-23T20:38:25.794 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:38:25 smithi026 ceph-mon[95839]: pgmap v44: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2023-09-23T20:38:25.795 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:38:25 smithi026 ceph-mon[109912]: Deploying daemon osd.1 on smithi026 2023-09-23T20:38:25.795 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:38:25 smithi026 ceph-mon[109912]: pgmap v44: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2023-09-23T20:38:25.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:38:25 smithi070 ceph-mon[100755]: Deploying daemon osd.1 on smithi026 2023-09-23T20:38:25.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:38:25 smithi070 ceph-mon[100755]: pgmap v44: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2023-09-23T20:38:27.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:38:27 smithi070 ceph-mon[100755]: pgmap v45: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2023-09-23T20:38:27.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:38:27 smithi026 ceph-mon[95839]: pgmap v45: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2023-09-23T20:38:27.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:38:27 smithi026 ceph-mon[109912]: pgmap v45: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2023-09-23T20:38:29.381 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:38:29 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:38:29.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:38:29 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:38:29.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:38:29 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:38:29.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:38:29 smithi026 ceph-mon[95839]: pgmap v46: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2023-09-23T20:38:29.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:38:29 smithi026 ceph-mon[109912]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:38:29.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:38:29 smithi026 ceph-mon[109912]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:38:29.383 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:38:29 smithi026 ceph-mon[109912]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:38:29.383 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:38:29 smithi026 ceph-mon[109912]: pgmap v46: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2023-09-23T20:38:29.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:38:29 smithi070 ceph-mon[100755]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:38:29.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:38:29 smithi070 ceph-mon[100755]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:38:29.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:38:29 smithi070 ceph-mon[100755]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:38:29.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:38:29 smithi070 ceph-mon[100755]: pgmap v46: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2023-09-23T20:38:31.078 INFO:teuthology.orchestra.run.smithi026.stdout:Created osd(s) 1 on host 'smithi026' 2023-09-23T20:38:31.516 DEBUG:teuthology.orchestra.run.smithi026:osd.1> sudo journalctl -f -n 0 -u ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7@osd.1.service 2023-09-23T20:38:31.518 INFO:tasks.cephadm:Deploying osd.2 on smithi026 with /dev/vg_nvme/lv_2... 2023-09-23T20:38:31.518 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph-volume lvm zap /dev/vg_nvme/lv_2 2023-09-23T20:38:31.550 INFO:journalctl@ceph.osd.1.smithi026.stdout:-- Logs begin at Sat 2023-09-23 20:20:42 UTC. -- 2023-09-23T20:38:31.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:38:31 smithi070 ceph-mon[100755]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:38:31.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:38:31 smithi070 ceph-mon[100755]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:38:31.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:38:31 smithi070 ceph-mon[100755]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:38:31.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:38:31 smithi070 ceph-mon[100755]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:38:31.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:38:31 smithi070 ceph-mon[100755]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:38:31.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:38:31 smithi070 ceph-mon[100755]: pgmap v47: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2023-09-23T20:38:31.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:38:31 smithi070 ceph-mon[100755]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:38:31.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:38:31 smithi070 ceph-mon[100755]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:38:31.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:38:31 smithi070 ceph-mon[100755]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:38:31.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:38:31 smithi070 ceph-mon[100755]: from='osd.1 [v2:172.21.15.26:6810/3250108152,v1:172.21.15.26:6811/3250108152]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2023-09-23T20:38:31.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:38:31 smithi070 ceph-mon[100755]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2023-09-23T20:38:31.881 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:38:31 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:38:31.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:38:31 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:38:31.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:38:31 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:38:31.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:38:31 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:38:31.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:38:31 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:38:31.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:38:31 smithi026 ceph-mon[95839]: pgmap v47: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2023-09-23T20:38:31.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:38:31 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:38:31.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:38:31 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:38:31.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:38:31 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:38:31.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:38:31 smithi026 ceph-mon[95839]: from='osd.1 [v2:172.21.15.26:6810/3250108152,v1:172.21.15.26:6811/3250108152]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2023-09-23T20:38:31.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:38:31 smithi026 ceph-mon[95839]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2023-09-23T20:38:31.883 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:38:31 smithi026 ceph-mon[109912]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:38:31.883 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:38:31 smithi026 ceph-mon[109912]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:38:31.883 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:38:31 smithi026 ceph-mon[109912]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:38:31.883 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:38:31 smithi026 ceph-mon[109912]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:38:31.883 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:38:31 smithi026 ceph-mon[109912]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:38:31.883 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:38:31 smithi026 ceph-mon[109912]: pgmap v47: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2023-09-23T20:38:31.883 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:38:31 smithi026 ceph-mon[109912]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:38:31.883 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:38:31 smithi026 ceph-mon[109912]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:38:31.883 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:38:31 smithi026 ceph-mon[109912]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:38:31.883 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:38:31 smithi026 ceph-mon[109912]: from='osd.1 [v2:172.21.15.26:6810/3250108152,v1:172.21.15.26:6811/3250108152]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2023-09-23T20:38:31.883 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:38:31 smithi026 ceph-mon[109912]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2023-09-23T20:38:32.627 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:38:33.256 INFO:journalctl@ceph.osd.1.smithi026.stdout:Sep 23 20:38:33 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd-1[121458]: 2023-09-23T20:38:33.080+0000 7f64735c5700 -1 osd.1 0 waiting for initial osdmap 2023-09-23T20:38:33.256 INFO:journalctl@ceph.osd.1.smithi026.stdout:Sep 23 20:38:33 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd-1[121458]: 2023-09-23T20:38:33.084+0000 7f646dbbf700 -1 osd.1 12 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-09-23T20:38:33.256 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:38:33 smithi026 ceph-mon[95839]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2023-09-23T20:38:33.256 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:38:33 smithi026 ceph-mon[95839]: osdmap e11: 2 total, 1 up, 2 in 2023-09-23T20:38:33.256 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:38:33 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-23T20:38:33.256 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:38:33 smithi026 ceph-mon[95839]: from='osd.1 [v2:172.21.15.26:6810/3250108152,v1:172.21.15.26:6811/3250108152]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi026", "root=default"]}]: dispatch 2023-09-23T20:38:33.257 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:38:33 smithi026 ceph-mon[95839]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi026", "root=default"]}]: dispatch 2023-09-23T20:38:33.257 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:38:33 smithi026 ceph-mon[95839]: pgmap v49: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2023-09-23T20:38:33.257 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:38:33 smithi026 ceph-mon[109912]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2023-09-23T20:38:33.257 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:38:33 smithi026 ceph-mon[109912]: osdmap e11: 2 total, 1 up, 2 in 2023-09-23T20:38:33.257 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:38:33 smithi026 ceph-mon[109912]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-23T20:38:33.257 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:38:33 smithi026 ceph-mon[109912]: from='osd.1 [v2:172.21.15.26:6810/3250108152,v1:172.21.15.26:6811/3250108152]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi026", "root=default"]}]: dispatch 2023-09-23T20:38:33.257 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:38:33 smithi026 ceph-mon[109912]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi026", "root=default"]}]: dispatch 2023-09-23T20:38:33.257 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:38:33 smithi026 ceph-mon[109912]: pgmap v49: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2023-09-23T20:38:33.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:38:33 smithi070 ceph-mon[100755]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2023-09-23T20:38:33.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:38:33 smithi070 ceph-mon[100755]: osdmap e11: 2 total, 1 up, 2 in 2023-09-23T20:38:33.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:38:33 smithi070 ceph-mon[100755]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-23T20:38:33.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:38:33 smithi070 ceph-mon[100755]: from='osd.1 [v2:172.21.15.26:6810/3250108152,v1:172.21.15.26:6811/3250108152]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi026", "root=default"]}]: dispatch 2023-09-23T20:38:33.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:38:33 smithi070 ceph-mon[100755]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi026", "root=default"]}]: dispatch 2023-09-23T20:38:33.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:38:33 smithi070 ceph-mon[100755]: pgmap v49: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2023-09-23T20:38:34.131 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:38:34 smithi026 ceph-mon[109912]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi026", "root=default"]}]': finished 2023-09-23T20:38:34.131 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:38:34 smithi026 ceph-mon[109912]: osdmap e12: 2 total, 1 up, 2 in 2023-09-23T20:38:34.131 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:38:34 smithi026 ceph-mon[109912]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-23T20:38:34.131 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:38:34 smithi026 ceph-mon[109912]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-23T20:38:34.131 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:38:34 smithi026 ceph-mon[95839]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi026", "root=default"]}]': finished 2023-09-23T20:38:34.131 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:38:34 smithi026 ceph-mon[95839]: osdmap e12: 2 total, 1 up, 2 in 2023-09-23T20:38:34.131 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:38:34 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-23T20:38:34.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:38:34 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-23T20:38:34.331 INFO:teuthology.orchestra.run.smithi026.stderr:--> Zapping: /dev/vg_nvme/lv_2 2023-09-23T20:38:34.332 INFO:teuthology.orchestra.run.smithi026.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_2 bs=1M count=10 conv=fsync 2023-09-23T20:38:34.332 INFO:teuthology.orchestra.run.smithi026.stderr: stderr: 10+0 records in 2023-09-23T20:38:34.332 INFO:teuthology.orchestra.run.smithi026.stderr:10+0 records out 2023-09-23T20:38:34.332 INFO:teuthology.orchestra.run.smithi026.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.008341 s, 1.3 GB/s 2023-09-23T20:38:34.332 INFO:teuthology.orchestra.run.smithi026.stderr:--> Zapping successful for: 2023-09-23T20:38:34.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:38:34 smithi070 ceph-mon[100755]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi026", "root=default"]}]': finished 2023-09-23T20:38:34.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:38:34 smithi070 ceph-mon[100755]: osdmap e12: 2 total, 1 up, 2 in 2023-09-23T20:38:34.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:38:34 smithi070 ceph-mon[100755]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-23T20:38:34.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:38:34 smithi070 ceph-mon[100755]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-23T20:38:34.858 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph orch daemon add osd smithi026:vg_nvme/lv_2 2023-09-23T20:38:35.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:38:35 smithi026 ceph-mon[109912]: purged_snaps scrub starts 2023-09-23T20:38:35.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:38:35 smithi026 ceph-mon[109912]: purged_snaps scrub ok 2023-09-23T20:38:35.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:38:35 smithi026 ceph-mon[109912]: osd.1 [v2:172.21.15.26:6810/3250108152,v1:172.21.15.26:6811/3250108152] boot 2023-09-23T20:38:35.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:38:35 smithi026 ceph-mon[109912]: osdmap e13: 2 total, 2 up, 2 in 2023-09-23T20:38:35.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:38:35 smithi026 ceph-mon[109912]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-23T20:38:35.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:38:35 smithi026 ceph-mon[109912]: Detected new or changed devices on smithi026 2023-09-23T20:38:35.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:38:35 smithi026 ceph-mon[109912]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:38:35.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:38:35 smithi026 ceph-mon[109912]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:38:35.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:38:35 smithi026 ceph-mon[109912]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:38:35.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:38:35 smithi026 ceph-mon[109912]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:38:35.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:38:35 smithi026 ceph-mon[109912]: Adjusting osd_memory_target on smithi026 to 8074M 2023-09-23T20:38:35.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:38:35 smithi026 ceph-mon[109912]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:38:35.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:38:35 smithi026 ceph-mon[109912]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:38:35.134 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:38:35 smithi026 ceph-mon[109912]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:38:35.134 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:38:35 smithi026 ceph-mon[109912]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:38:35.134 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:38:35 smithi026 ceph-mon[109912]: pgmap v52: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2023-09-23T20:38:35.134 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:38:35 smithi026 ceph-mon[95839]: purged_snaps scrub starts 2023-09-23T20:38:35.134 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:38:35 smithi026 ceph-mon[95839]: purged_snaps scrub ok 2023-09-23T20:38:35.134 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:38:35 smithi026 ceph-mon[95839]: osd.1 [v2:172.21.15.26:6810/3250108152,v1:172.21.15.26:6811/3250108152] boot 2023-09-23T20:38:35.134 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:38:35 smithi026 ceph-mon[95839]: osdmap e13: 2 total, 2 up, 2 in 2023-09-23T20:38:35.134 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:38:35 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-23T20:38:35.134 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:38:35 smithi026 ceph-mon[95839]: Detected new or changed devices on smithi026 2023-09-23T20:38:35.134 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:38:35 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:38:35.134 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:38:35 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:38:35.135 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:38:35 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:38:35.135 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:38:35 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:38:35.135 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:38:35 smithi026 ceph-mon[95839]: Adjusting osd_memory_target on smithi026 to 8074M 2023-09-23T20:38:35.135 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:38:35 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:38:35.135 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:38:35 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:38:35.135 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:38:35 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:38:35.135 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:38:35 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:38:35.135 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:38:35 smithi026 ceph-mon[95839]: pgmap v52: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2023-09-23T20:38:35.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:38:35 smithi070 ceph-mon[100755]: purged_snaps scrub starts 2023-09-23T20:38:35.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:38:35 smithi070 ceph-mon[100755]: purged_snaps scrub ok 2023-09-23T20:38:35.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:38:35 smithi070 ceph-mon[100755]: osd.1 [v2:172.21.15.26:6810/3250108152,v1:172.21.15.26:6811/3250108152] boot 2023-09-23T20:38:35.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:38:35 smithi070 ceph-mon[100755]: osdmap e13: 2 total, 2 up, 2 in 2023-09-23T20:38:35.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:38:35 smithi070 ceph-mon[100755]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-23T20:38:35.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:38:35 smithi070 ceph-mon[100755]: Detected new or changed devices on smithi026 2023-09-23T20:38:35.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:38:35 smithi070 ceph-mon[100755]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:38:35.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:38:35 smithi070 ceph-mon[100755]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:38:35.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:38:35 smithi070 ceph-mon[100755]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:38:35.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:38:35 smithi070 ceph-mon[100755]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:38:35.371 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:38:35 smithi070 ceph-mon[100755]: Adjusting osd_memory_target on smithi026 to 8074M 2023-09-23T20:38:35.371 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:38:35 smithi070 ceph-mon[100755]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:38:35.371 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:38:35 smithi070 ceph-mon[100755]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:38:35.371 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:38:35 smithi070 ceph-mon[100755]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:38:35.371 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:38:35 smithi070 ceph-mon[100755]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:38:35.371 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:38:35 smithi070 ceph-mon[100755]: pgmap v52: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2023-09-23T20:38:35.467 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:38:36.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:38:36 smithi070 ceph-mon[100755]: osdmap e14: 2 total, 2 up, 2 in 2023-09-23T20:38:36.381 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:38:36 smithi026 ceph-mon[95839]: osdmap e14: 2 total, 2 up, 2 in 2023-09-23T20:38:36.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:38:36 smithi026 ceph-mon[109912]: osdmap e14: 2 total, 2 up, 2 in 2023-09-23T20:38:37.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:38:37 smithi070 ceph-mon[100755]: pgmap v54: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2023-09-23T20:38:37.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:38:37 smithi070 ceph-mon[100755]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-09-23T20:38:37.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:38:37 smithi070 ceph-mon[100755]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-09-23T20:38:37.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:38:37 smithi070 ceph-mon[100755]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:38:37.381 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:38:37 smithi026 ceph-mon[95839]: pgmap v54: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2023-09-23T20:38:37.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:38:37 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-09-23T20:38:37.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:38:37 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-09-23T20:38:37.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:38:37 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:38:37.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:38:37 smithi026 ceph-mon[109912]: pgmap v54: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2023-09-23T20:38:37.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:38:37 smithi026 ceph-mon[109912]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-09-23T20:38:37.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:38:37 smithi026 ceph-mon[109912]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-09-23T20:38:37.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:38:37 smithi026 ceph-mon[109912]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:38:38.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:38:38 smithi070 ceph-mon[100755]: from='client.24170 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi026:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T20:38:38.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:38:38 smithi026 ceph-mon[95839]: from='client.24170 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi026:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T20:38:38.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:38:38 smithi026 ceph-mon[109912]: from='client.24170 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi026:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T20:38:39.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:38:39 smithi070 ceph-mon[100755]: pgmap v55: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2023-09-23T20:38:39.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:38:39 smithi070 ceph-mon[100755]: from='client.? 172.21.15.26:0/866691354' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "2e115328-76c6-478e-915c-5786903be49f"}]: dispatch 2023-09-23T20:38:39.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:38:39 smithi070 ceph-mon[100755]: from='client.? 172.21.15.26:0/866691354' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "2e115328-76c6-478e-915c-5786903be49f"}]': finished 2023-09-23T20:38:39.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:38:39 smithi070 ceph-mon[100755]: osdmap e15: 3 total, 2 up, 3 in 2023-09-23T20:38:39.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:38:39 smithi070 ceph-mon[100755]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-23T20:38:39.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:38:39 smithi026 ceph-mon[95839]: pgmap v55: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2023-09-23T20:38:39.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:38:39 smithi026 ceph-mon[95839]: from='client.? 172.21.15.26:0/866691354' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "2e115328-76c6-478e-915c-5786903be49f"}]: dispatch 2023-09-23T20:38:39.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:38:39 smithi026 ceph-mon[95839]: from='client.? 172.21.15.26:0/866691354' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "2e115328-76c6-478e-915c-5786903be49f"}]': finished 2023-09-23T20:38:39.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:38:39 smithi026 ceph-mon[95839]: osdmap e15: 3 total, 2 up, 3 in 2023-09-23T20:38:39.883 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:38:39 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-23T20:38:39.883 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:38:39 smithi026 ceph-mon[109912]: pgmap v55: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2023-09-23T20:38:39.883 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:38:39 smithi026 ceph-mon[109912]: from='client.? 172.21.15.26:0/866691354' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "2e115328-76c6-478e-915c-5786903be49f"}]: dispatch 2023-09-23T20:38:39.883 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:38:39 smithi026 ceph-mon[109912]: from='client.? 172.21.15.26:0/866691354' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "2e115328-76c6-478e-915c-5786903be49f"}]': finished 2023-09-23T20:38:39.883 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:38:39 smithi026 ceph-mon[109912]: osdmap e15: 3 total, 2 up, 3 in 2023-09-23T20:38:39.883 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:38:39 smithi026 ceph-mon[109912]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-23T20:38:40.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:38:40 smithi070 ceph-mon[100755]: from='client.? 172.21.15.26:0/2132177759' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-09-23T20:38:40.881 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:38:40 smithi026 ceph-mon[95839]: from='client.? 172.21.15.26:0/2132177759' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-09-23T20:38:40.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:38:40 smithi026 ceph-mon[109912]: from='client.? 172.21.15.26:0/2132177759' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-09-23T20:38:41.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:38:41 smithi070 ceph-mon[100755]: pgmap v57: 0 pgs: ; 0 B data, 9.9 MiB used, 179 GiB / 179 GiB avail 2023-09-23T20:38:41.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:38:41 smithi026 ceph-mon[95839]: pgmap v57: 0 pgs: ; 0 B data, 9.9 MiB used, 179 GiB / 179 GiB avail 2023-09-23T20:38:41.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:38:41 smithi026 ceph-mon[109912]: pgmap v57: 0 pgs: ; 0 B data, 9.9 MiB used, 179 GiB / 179 GiB avail 2023-09-23T20:38:43.829 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:38:43 smithi026 ceph-mon[95839]: pgmap v58: 0 pgs: ; 0 B data, 9.9 MiB used, 179 GiB / 179 GiB avail 2023-09-23T20:38:43.829 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:38:43 smithi026 ceph-mon[109912]: pgmap v58: 0 pgs: ; 0 B data, 9.9 MiB used, 179 GiB / 179 GiB avail 2023-09-23T20:38:43.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:38:43 smithi070 ceph-mon[100755]: pgmap v58: 0 pgs: ; 0 B data, 9.9 MiB used, 179 GiB / 179 GiB avail 2023-09-23T20:38:45.839 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:38:45 smithi026 ceph-mon[95839]: pgmap v59: 0 pgs: ; 0 B data, 10 MiB used, 179 GiB / 179 GiB avail 2023-09-23T20:38:45.839 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:38:45 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2023-09-23T20:38:45.839 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:38:45 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:38:45.840 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:38:45 smithi026 ceph-mon[109912]: pgmap v59: 0 pgs: ; 0 B data, 10 MiB used, 179 GiB / 179 GiB avail 2023-09-23T20:38:45.840 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:38:45 smithi026 ceph-mon[109912]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2023-09-23T20:38:45.840 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:38:45 smithi026 ceph-mon[109912]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:38:45.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:38:45 smithi070 ceph-mon[100755]: pgmap v59: 0 pgs: ; 0 B data, 10 MiB used, 179 GiB / 179 GiB avail 2023-09-23T20:38:45.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:38:45 smithi070 ceph-mon[100755]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2023-09-23T20:38:45.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:38:45 smithi070 ceph-mon[100755]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:38:46.669 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:38:46 smithi026 ceph-mon[95839]: Deploying daemon osd.2 on smithi026 2023-09-23T20:38:46.670 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:38:46 smithi026 ceph-mon[109912]: Deploying daemon osd.2 on smithi026 2023-09-23T20:38:46.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:38:46 smithi070 ceph-mon[100755]: Deploying daemon osd.2 on smithi026 2023-09-23T20:38:47.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:38:47 smithi026 ceph-mon[95839]: pgmap v60: 0 pgs: ; 0 B data, 10 MiB used, 179 GiB / 179 GiB avail 2023-09-23T20:38:47.633 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:38:47 smithi026 ceph-mon[109912]: pgmap v60: 0 pgs: ; 0 B data, 10 MiB used, 179 GiB / 179 GiB avail 2023-09-23T20:38:47.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:38:47 smithi070 ceph-mon[100755]: pgmap v60: 0 pgs: ; 0 B data, 10 MiB used, 179 GiB / 179 GiB avail 2023-09-23T20:38:49.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:38:49 smithi070 ceph-mon[100755]: pgmap v61: 0 pgs: ; 0 B data, 10 MiB used, 179 GiB / 179 GiB avail 2023-09-23T20:38:49.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:38:49 smithi070 ceph-mon[100755]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:38:49.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:38:49 smithi070 ceph-mon[100755]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:38:49.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:38:49 smithi070 ceph-mon[100755]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:38:49.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:38:49 smithi026 ceph-mon[95839]: pgmap v61: 0 pgs: ; 0 B data, 10 MiB used, 179 GiB / 179 GiB avail 2023-09-23T20:38:49.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:38:49 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:38:49.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:38:49 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:38:49.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:38:49 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:38:49.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:38:49 smithi026 ceph-mon[109912]: pgmap v61: 0 pgs: ; 0 B data, 10 MiB used, 179 GiB / 179 GiB avail 2023-09-23T20:38:49.883 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:38:49 smithi026 ceph-mon[109912]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:38:49.883 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:38:49 smithi026 ceph-mon[109912]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:38:49.883 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:38:49 smithi026 ceph-mon[109912]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:38:51.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:38:51 smithi070 ceph-mon[100755]: pgmap v62: 0 pgs: ; 0 B data, 10 MiB used, 179 GiB / 179 GiB avail 2023-09-23T20:38:51.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:38:51 smithi026 ceph-mon[95839]: pgmap v62: 0 pgs: ; 0 B data, 10 MiB used, 179 GiB / 179 GiB avail 2023-09-23T20:38:51.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:38:51 smithi026 ceph-mon[109912]: pgmap v62: 0 pgs: ; 0 B data, 10 MiB used, 179 GiB / 179 GiB avail 2023-09-23T20:38:52.238 INFO:teuthology.orchestra.run.smithi026.stdout:Created osd(s) 2 on host 'smithi026' 2023-09-23T20:38:52.762 DEBUG:teuthology.orchestra.run.smithi026:osd.2> sudo journalctl -f -n 0 -u ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7@osd.2.service 2023-09-23T20:38:52.765 INFO:tasks.cephadm:Deploying osd.3 on smithi026 with /dev/vg_nvme/lv_1... 2023-09-23T20:38:52.765 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph-volume lvm zap /dev/vg_nvme/lv_1 2023-09-23T20:38:52.795 INFO:journalctl@ceph.osd.2.smithi026.stdout:-- Logs begin at Sat 2023-09-23 20:20:42 UTC. -- 2023-09-23T20:38:53.238 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:38:52 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:38:53.238 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:38:52 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:38:53.238 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:38:52 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:38:53.238 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:38:52 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:38:53.239 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:38:52 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:38:53.239 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:38:52 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:38:53.239 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:38:52 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:38:53.239 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:38:52 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:38:53.239 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:38:52 smithi026 ceph-mon[95839]: pgmap v63: 0 pgs: ; 0 B data, 10 MiB used, 179 GiB / 179 GiB avail 2023-09-23T20:38:53.239 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:38:52 smithi026 ceph-mon[95839]: from='osd.2 [v2:172.21.15.26:6818/1325135181,v1:172.21.15.26:6819/1325135181]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2023-09-23T20:38:53.239 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:38:52 smithi026 ceph-mon[109912]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:38:53.239 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:38:52 smithi026 ceph-mon[109912]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:38:53.239 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:38:52 smithi026 ceph-mon[109912]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:38:53.239 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:38:52 smithi026 ceph-mon[109912]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:38:53.240 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:38:52 smithi026 ceph-mon[109912]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:38:53.240 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:38:52 smithi026 ceph-mon[109912]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:38:53.240 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:38:52 smithi026 ceph-mon[109912]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:38:53.240 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:38:52 smithi026 ceph-mon[109912]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:38:53.240 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:38:52 smithi026 ceph-mon[109912]: pgmap v63: 0 pgs: ; 0 B data, 10 MiB used, 179 GiB / 179 GiB avail 2023-09-23T20:38:53.240 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:38:52 smithi026 ceph-mon[109912]: from='osd.2 [v2:172.21.15.26:6818/1325135181,v1:172.21.15.26:6819/1325135181]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2023-09-23T20:38:53.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:38:52 smithi070 ceph-mon[100755]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:38:53.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:38:52 smithi070 ceph-mon[100755]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:38:53.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:38:52 smithi070 ceph-mon[100755]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:38:53.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:38:52 smithi070 ceph-mon[100755]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:38:53.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:38:52 smithi070 ceph-mon[100755]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:38:53.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:38:52 smithi070 ceph-mon[100755]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:38:53.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:38:52 smithi070 ceph-mon[100755]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:38:53.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:38:52 smithi070 ceph-mon[100755]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:38:53.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:38:52 smithi070 ceph-mon[100755]: pgmap v63: 0 pgs: ; 0 B data, 10 MiB used, 179 GiB / 179 GiB avail 2023-09-23T20:38:53.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:38:52 smithi070 ceph-mon[100755]: from='osd.2 [v2:172.21.15.26:6818/1325135181,v1:172.21.15.26:6819/1325135181]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2023-09-23T20:38:53.965 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:38:54.535 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:38:54 smithi026 ceph-mon[95839]: from='osd.2 [v2:172.21.15.26:6818/1325135181,v1:172.21.15.26:6819/1325135181]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2023-09-23T20:38:54.536 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:38:54 smithi026 ceph-mon[95839]: osdmap e16: 3 total, 2 up, 3 in 2023-09-23T20:38:54.536 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:38:54 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-23T20:38:54.536 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:38:54 smithi026 ceph-mon[95839]: from='osd.2 [v2:172.21.15.26:6818/1325135181,v1:172.21.15.26:6819/1325135181]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi026", "root=default"]}]: dispatch 2023-09-23T20:38:54.536 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:38:54 smithi026 ceph-mon[109912]: from='osd.2 [v2:172.21.15.26:6818/1325135181,v1:172.21.15.26:6819/1325135181]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2023-09-23T20:38:54.536 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:38:54 smithi026 ceph-mon[109912]: osdmap e16: 3 total, 2 up, 3 in 2023-09-23T20:38:54.536 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:38:54 smithi026 ceph-mon[109912]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-23T20:38:54.536 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:38:54 smithi026 ceph-mon[109912]: from='osd.2 [v2:172.21.15.26:6818/1325135181,v1:172.21.15.26:6819/1325135181]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi026", "root=default"]}]: dispatch 2023-09-23T20:38:54.537 INFO:journalctl@ceph.osd.2.smithi026.stdout:Sep 23 20:38:54 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd-2[128665]: 2023-09-23T20:38:54.242+0000 7fee8b98d700 -1 osd.2 0 waiting for initial osdmap 2023-09-23T20:38:54.537 INFO:journalctl@ceph.osd.2.smithi026.stdout:Sep 23 20:38:54 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd-2[128665]: 2023-09-23T20:38:54.246+0000 7fee86788700 -1 osd.2 17 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-09-23T20:38:54.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:38:54 smithi070 ceph-mon[100755]: from='osd.2 [v2:172.21.15.26:6818/1325135181,v1:172.21.15.26:6819/1325135181]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2023-09-23T20:38:54.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:38:54 smithi070 ceph-mon[100755]: osdmap e16: 3 total, 2 up, 3 in 2023-09-23T20:38:54.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:38:54 smithi070 ceph-mon[100755]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-23T20:38:54.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:38:54 smithi070 ceph-mon[100755]: from='osd.2 [v2:172.21.15.26:6818/1325135181,v1:172.21.15.26:6819/1325135181]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi026", "root=default"]}]: dispatch 2023-09-23T20:38:55.537 INFO:teuthology.orchestra.run.smithi026.stderr:--> Zapping: /dev/vg_nvme/lv_1 2023-09-23T20:38:55.537 INFO:teuthology.orchestra.run.smithi026.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_1 bs=1M count=10 conv=fsync 2023-09-23T20:38:55.537 INFO:teuthology.orchestra.run.smithi026.stderr: stderr: 10+0 records in 2023-09-23T20:38:55.538 INFO:teuthology.orchestra.run.smithi026.stderr:10+0 records out 2023-09-23T20:38:55.538 INFO:teuthology.orchestra.run.smithi026.stderr:10485760 bytes (10 MB, 10 MiB) copied, 0.00883787 s, 1.2 GB/s 2023-09-23T20:38:55.538 INFO:teuthology.orchestra.run.smithi026.stderr:--> Zapping successful for: 2023-09-23T20:38:55.578 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:38:55 smithi026 ceph-mon[95839]: from='osd.2 [v2:172.21.15.26:6818/1325135181,v1:172.21.15.26:6819/1325135181]' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi026", "root=default"]}]': finished 2023-09-23T20:38:55.578 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:38:55 smithi026 ceph-mon[95839]: osdmap e17: 3 total, 2 up, 3 in 2023-09-23T20:38:55.578 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:38:55 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-23T20:38:55.578 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:38:55 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-23T20:38:55.578 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:38:55 smithi026 ceph-mon[95839]: pgmap v66: 0 pgs: ; 0 B data, 10 MiB used, 179 GiB / 179 GiB avail 2023-09-23T20:38:55.579 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:38:55 smithi026 ceph-mon[109912]: from='osd.2 [v2:172.21.15.26:6818/1325135181,v1:172.21.15.26:6819/1325135181]' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi026", "root=default"]}]': finished 2023-09-23T20:38:55.579 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:38:55 smithi026 ceph-mon[109912]: osdmap e17: 3 total, 2 up, 3 in 2023-09-23T20:38:55.579 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:38:55 smithi026 ceph-mon[109912]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-23T20:38:55.579 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:38:55 smithi026 ceph-mon[109912]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-23T20:38:55.579 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:38:55 smithi026 ceph-mon[109912]: pgmap v66: 0 pgs: ; 0 B data, 10 MiB used, 179 GiB / 179 GiB avail 2023-09-23T20:38:55.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:38:55 smithi070 ceph-mon[100755]: from='osd.2 [v2:172.21.15.26:6818/1325135181,v1:172.21.15.26:6819/1325135181]' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi026", "root=default"]}]': finished 2023-09-23T20:38:55.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:38:55 smithi070 ceph-mon[100755]: osdmap e17: 3 total, 2 up, 3 in 2023-09-23T20:38:55.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:38:55 smithi070 ceph-mon[100755]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-23T20:38:55.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:38:55 smithi070 ceph-mon[100755]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-23T20:38:55.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:38:55 smithi070 ceph-mon[100755]: pgmap v66: 0 pgs: ; 0 B data, 10 MiB used, 179 GiB / 179 GiB avail 2023-09-23T20:38:56.030 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph orch daemon add osd smithi026:vg_nvme/lv_1 2023-09-23T20:38:56.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:38:56 smithi070 ceph-mon[100755]: purged_snaps scrub starts 2023-09-23T20:38:56.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:38:56 smithi070 ceph-mon[100755]: purged_snaps scrub ok 2023-09-23T20:38:56.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:38:56 smithi070 ceph-mon[100755]: osd.2 [v2:172.21.15.26:6818/1325135181,v1:172.21.15.26:6819/1325135181] boot 2023-09-23T20:38:56.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:38:56 smithi070 ceph-mon[100755]: osdmap e18: 3 total, 3 up, 3 in 2023-09-23T20:38:56.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:38:56 smithi070 ceph-mon[100755]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-23T20:38:56.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:38:56 smithi026 ceph-mon[95839]: purged_snaps scrub starts 2023-09-23T20:38:56.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:38:56 smithi026 ceph-mon[95839]: purged_snaps scrub ok 2023-09-23T20:38:56.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:38:56 smithi026 ceph-mon[95839]: osd.2 [v2:172.21.15.26:6818/1325135181,v1:172.21.15.26:6819/1325135181] boot 2023-09-23T20:38:56.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:38:56 smithi026 ceph-mon[95839]: osdmap e18: 3 total, 3 up, 3 in 2023-09-23T20:38:56.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:38:56 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-23T20:38:56.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:38:56 smithi026 ceph-mon[109912]: purged_snaps scrub starts 2023-09-23T20:38:56.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:38:56 smithi026 ceph-mon[109912]: purged_snaps scrub ok 2023-09-23T20:38:56.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:38:56 smithi026 ceph-mon[109912]: osd.2 [v2:172.21.15.26:6818/1325135181,v1:172.21.15.26:6819/1325135181] boot 2023-09-23T20:38:56.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:38:56 smithi026 ceph-mon[109912]: osdmap e18: 3 total, 3 up, 3 in 2023-09-23T20:38:56.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:38:56 smithi026 ceph-mon[109912]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-23T20:38:56.645 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:38:57.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:38:57 smithi070 ceph-mon[100755]: osdmap e19: 3 total, 3 up, 3 in 2023-09-23T20:38:57.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:38:57 smithi070 ceph-mon[100755]: Detected new or changed devices on smithi026 2023-09-23T20:38:57.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:38:57 smithi070 ceph-mon[100755]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:38:57.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:38:57 smithi070 ceph-mon[100755]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:38:57.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:38:57 smithi070 ceph-mon[100755]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:38:57.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:38:57 smithi070 ceph-mon[100755]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:38:57.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:38:57 smithi070 ceph-mon[100755]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:38:57.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:38:57 smithi070 ceph-mon[100755]: Adjusting osd_memory_target on smithi026 to 5383M 2023-09-23T20:38:57.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:38:57 smithi070 ceph-mon[100755]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:38:57.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:38:57 smithi070 ceph-mon[100755]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:38:57.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:38:57 smithi070 ceph-mon[100755]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:38:57.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:38:57 smithi070 ceph-mon[100755]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:38:57.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:38:57 smithi070 ceph-mon[100755]: pgmap v69: 0 pgs: ; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2023-09-23T20:38:57.621 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:38:57 smithi070 ceph-mon[100755]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2023-09-23T20:38:57.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:38:57 smithi026 ceph-mon[109912]: osdmap e19: 3 total, 3 up, 3 in 2023-09-23T20:38:57.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:38:57 smithi026 ceph-mon[109912]: Detected new or changed devices on smithi026 2023-09-23T20:38:57.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:38:57 smithi026 ceph-mon[109912]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:38:57.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:38:57 smithi026 ceph-mon[109912]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:38:57.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:38:57 smithi026 ceph-mon[109912]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:38:57.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:38:57 smithi026 ceph-mon[109912]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:38:57.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:38:57 smithi026 ceph-mon[109912]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:38:57.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:38:57 smithi026 ceph-mon[109912]: Adjusting osd_memory_target on smithi026 to 5383M 2023-09-23T20:38:57.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:38:57 smithi026 ceph-mon[109912]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:38:57.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:38:57 smithi026 ceph-mon[109912]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:38:57.633 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:38:57 smithi026 ceph-mon[109912]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:38:57.633 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:38:57 smithi026 ceph-mon[109912]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:38:57.633 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:38:57 smithi026 ceph-mon[109912]: pgmap v69: 0 pgs: ; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2023-09-23T20:38:57.633 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:38:57 smithi026 ceph-mon[109912]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2023-09-23T20:38:57.633 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:38:57 smithi026 ceph-mon[95839]: osdmap e19: 3 total, 3 up, 3 in 2023-09-23T20:38:57.633 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:38:57 smithi026 ceph-mon[95839]: Detected new or changed devices on smithi026 2023-09-23T20:38:57.633 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:38:57 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:38:57.633 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:38:57 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:38:57.633 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:38:57 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:38:57.633 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:38:57 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:38:57.634 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:38:57 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:38:57.634 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:38:57 smithi026 ceph-mon[95839]: Adjusting osd_memory_target on smithi026 to 5383M 2023-09-23T20:38:57.634 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:38:57 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:38:57.634 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:38:57 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:38:57.634 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:38:57 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:38:57.634 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:38:57 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:38:57.634 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:38:57 smithi026 ceph-mon[95839]: pgmap v69: 0 pgs: ; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2023-09-23T20:38:57.634 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:38:57 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2023-09-23T20:38:58.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:38:58 smithi070 ceph-mon[100755]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd='[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]': finished 2023-09-23T20:38:58.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:38:58 smithi070 ceph-mon[100755]: osdmap e20: 3 total, 3 up, 3 in 2023-09-23T20:38:58.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:38:58 smithi070 ceph-mon[100755]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T20:38:58.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:38:58 smithi070 ceph-mon[100755]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-09-23T20:38:58.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:38:58 smithi070 ceph-mon[100755]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-09-23T20:38:58.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:38:58 smithi070 ceph-mon[100755]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:38:58.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:38:58 smithi026 ceph-mon[109912]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd='[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]': finished 2023-09-23T20:38:58.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:38:58 smithi026 ceph-mon[109912]: osdmap e20: 3 total, 3 up, 3 in 2023-09-23T20:38:58.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:38:58 smithi026 ceph-mon[109912]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T20:38:58.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:38:58 smithi026 ceph-mon[109912]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-09-23T20:38:58.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:38:58 smithi026 ceph-mon[109912]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-09-23T20:38:58.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:38:58 smithi026 ceph-mon[109912]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:38:58.633 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:38:58 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd='[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]': finished 2023-09-23T20:38:58.633 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:38:58 smithi026 ceph-mon[95839]: osdmap e20: 3 total, 3 up, 3 in 2023-09-23T20:38:58.633 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:38:58 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T20:38:58.633 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:38:58 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-09-23T20:38:58.633 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:38:58 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-09-23T20:38:58.633 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:38:58 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:38:59.132 INFO:journalctl@ceph.osd.0.smithi026.stdout:Sep 23 20:38:58 smithi026 sudo[133804]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2023-09-23T20:38:59.132 INFO:journalctl@ceph.osd.0.smithi026.stdout:Sep 23 20:38:58 smithi026 sudo[133804]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2023-09-23T20:38:59.132 INFO:journalctl@ceph.osd.0.smithi026.stdout:Sep 23 20:38:58 smithi026 sudo[133804]: pam_unix(sudo:session): session opened for user root by (uid=0) 2023-09-23T20:38:59.132 INFO:journalctl@ceph.osd.0.smithi026.stdout:Sep 23 20:38:58 smithi026 sudo[133804]: pam_unix(sudo:session): session closed for user root 2023-09-23T20:38:59.434 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:38:59 smithi026 ceph-mon[95839]: from='client.14298 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi026:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T20:38:59.434 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:38:59 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2023-09-23T20:38:59.434 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:38:59 smithi026 ceph-mon[95839]: osdmap e21: 3 total, 3 up, 3 in 2023-09-23T20:38:59.434 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:38:59 smithi026 ceph-mon[95839]: pgmap v72: 1 pgs: 1 unknown; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2023-09-23T20:38:59.434 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:38:59 smithi026 ceph-mon[109912]: from='client.14298 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi026:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T20:38:59.435 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:38:59 smithi026 ceph-mon[109912]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2023-09-23T20:38:59.435 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:38:59 smithi026 ceph-mon[109912]: osdmap e21: 3 total, 3 up, 3 in 2023-09-23T20:38:59.435 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:38:59 smithi026 ceph-mon[109912]: pgmap v72: 1 pgs: 1 unknown; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2023-09-23T20:38:59.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:38:59 smithi070 ceph-mon[100755]: from='client.14298 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi026:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T20:38:59.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:38:59 smithi070 ceph-mon[100755]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2023-09-23T20:38:59.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:38:59 smithi070 ceph-mon[100755]: osdmap e21: 3 total, 3 up, 3 in 2023-09-23T20:38:59.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:38:59 smithi070 ceph-mon[100755]: pgmap v72: 1 pgs: 1 unknown; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2023-09-23T20:38:59.692 INFO:journalctl@ceph.osd.0.smithi026.stdout:Sep 23 20:38:59 smithi026 sudo[133834]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2023-09-23T20:38:59.692 INFO:journalctl@ceph.osd.0.smithi026.stdout:Sep 23 20:38:59 smithi026 sudo[133834]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2023-09-23T20:38:59.692 INFO:journalctl@ceph.osd.0.smithi026.stdout:Sep 23 20:38:59 smithi026 sudo[133834]: pam_unix(sudo:session): session opened for user root by (uid=0) 2023-09-23T20:38:59.692 INFO:journalctl@ceph.osd.0.smithi026.stdout:Sep 23 20:38:59 smithi026 sudo[133834]: pam_unix(sudo:session): session closed for user root 2023-09-23T20:39:00.132 INFO:journalctl@ceph.osd.1.smithi026.stdout:Sep 23 20:38:59 smithi026 sudo[133887]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2023-09-23T20:39:00.134 INFO:journalctl@ceph.osd.1.smithi026.stdout:Sep 23 20:38:59 smithi026 sudo[133887]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2023-09-23T20:39:00.134 INFO:journalctl@ceph.osd.1.smithi026.stdout:Sep 23 20:38:59 smithi026 sudo[133887]: pam_unix(sudo:session): session opened for user root by (uid=0) 2023-09-23T20:39:00.134 INFO:journalctl@ceph.osd.1.smithi026.stdout:Sep 23 20:38:59 smithi026 sudo[133887]: pam_unix(sudo:session): session closed for user root 2023-09-23T20:39:00.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:39:00 smithi070 ceph-mon[100755]: osdmap e22: 3 total, 3 up, 3 in 2023-09-23T20:39:00.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:39:00 smithi070 ceph-mon[100755]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:39:00.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:39:00 smithi070 ceph-mon[100755]: mgrmap e15: y(active, since 2m), standbys: x 2023-09-23T20:39:00.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:39:00 smithi026 ceph-mon[95839]: osdmap e22: 3 total, 3 up, 3 in 2023-09-23T20:39:00.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:39:00 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:39:00.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:39:00 smithi026 ceph-mon[95839]: mgrmap e15: y(active, since 2m), standbys: x 2023-09-23T20:39:00.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:39:00 smithi026 ceph-mon[109912]: osdmap e22: 3 total, 3 up, 3 in 2023-09-23T20:39:00.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:39:00 smithi026 ceph-mon[109912]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:39:00.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:39:00 smithi026 ceph-mon[109912]: mgrmap e15: y(active, since 2m), standbys: x 2023-09-23T20:39:00.633 INFO:journalctl@ceph.osd.1.smithi026.stdout:Sep 23 20:39:00 smithi026 sudo[133925]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2023-09-23T20:39:00.633 INFO:journalctl@ceph.osd.1.smithi026.stdout:Sep 23 20:39:00 smithi026 sudo[133925]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2023-09-23T20:39:00.633 INFO:journalctl@ceph.osd.1.smithi026.stdout:Sep 23 20:39:00 smithi026 sudo[133925]: pam_unix(sudo:session): session opened for user root by (uid=0) 2023-09-23T20:39:00.633 INFO:journalctl@ceph.osd.1.smithi026.stdout:Sep 23 20:39:00 smithi026 sudo[133925]: pam_unix(sudo:session): session closed for user root 2023-09-23T20:39:01.132 INFO:journalctl@ceph.osd.2.smithi026.stdout:Sep 23 20:39:00 smithi026 sudo[133943]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2023-09-23T20:39:01.132 INFO:journalctl@ceph.osd.2.smithi026.stdout:Sep 23 20:39:00 smithi026 sudo[133943]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2023-09-23T20:39:01.132 INFO:journalctl@ceph.osd.2.smithi026.stdout:Sep 23 20:39:00 smithi026 sudo[133943]: pam_unix(sudo:session): session opened for user root by (uid=0) 2023-09-23T20:39:01.132 INFO:journalctl@ceph.osd.2.smithi026.stdout:Sep 23 20:39:00 smithi026 sudo[133943]: pam_unix(sudo:session): session closed for user root 2023-09-23T20:39:01.631 INFO:journalctl@ceph.osd.2.smithi026.stdout:Sep 23 20:39:01 smithi026 sudo[134016]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2023-09-23T20:39:01.632 INFO:journalctl@ceph.osd.2.smithi026.stdout:Sep 23 20:39:01 smithi026 sudo[134016]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2023-09-23T20:39:01.632 INFO:journalctl@ceph.osd.2.smithi026.stdout:Sep 23 20:39:01 smithi026 sudo[134016]: pam_unix(sudo:session): session opened for user root by (uid=0) 2023-09-23T20:39:01.632 INFO:journalctl@ceph.osd.2.smithi026.stdout:Sep 23 20:39:01 smithi026 sudo[134016]: pam_unix(sudo:session): session closed for user root 2023-09-23T20:39:02.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:39:01 smithi026 sudo[134021]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2023-09-23T20:39:02.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:39:01 smithi026 sudo[134021]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2023-09-23T20:39:02.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:39:01 smithi026 sudo[134021]: pam_unix(sudo:session): session opened for user root by (uid=0) 2023-09-23T20:39:02.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:39:01 smithi026 sudo[134021]: pam_unix(sudo:session): session closed for user root 2023-09-23T20:39:02.415 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:39:02 smithi070 ceph-mon[100755]: from='client.? 172.21.15.26:0/6547130' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "c3dd2c3f-5cf5-4fb0-964c-1b8926f75c8a"}]: dispatch 2023-09-23T20:39:02.415 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:39:02 smithi070 ceph-mon[100755]: from='client.? 172.21.15.26:0/6547130' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "c3dd2c3f-5cf5-4fb0-964c-1b8926f75c8a"}]': finished 2023-09-23T20:39:02.415 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:39:02 smithi070 ceph-mon[100755]: osdmap e23: 4 total, 3 up, 4 in 2023-09-23T20:39:02.416 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:39:02 smithi070 ceph-mon[100755]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-23T20:39:02.416 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:39:02 smithi070 ceph-mon[100755]: pgmap v75: 1 pgs: 1 active+clean; 577 KiB data, 17 MiB used, 268 GiB / 268 GiB avail 2023-09-23T20:39:02.416 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:39:02 smithi070 ceph-mon[100755]: from='client.? 172.21.15.26:0/3813076504' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-09-23T20:39:02.416 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:39:02 smithi070 ceph-mon[100755]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2023-09-23T20:39:02.416 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:39:02 smithi070 sudo[103570]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2023-09-23T20:39:02.416 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:39:02 smithi070 sudo[103570]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2023-09-23T20:39:02.416 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:39:02 smithi070 sudo[103570]: pam_unix(sudo:session): session opened for user root by (uid=0) 2023-09-23T20:39:02.416 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:39:02 smithi070 sudo[103570]: pam_unix(sudo:session): session closed for user root 2023-09-23T20:39:02.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:39:02 smithi026 ceph-mon[109912]: from='client.? 172.21.15.26:0/6547130' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "c3dd2c3f-5cf5-4fb0-964c-1b8926f75c8a"}]: dispatch 2023-09-23T20:39:02.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:39:02 smithi026 ceph-mon[109912]: from='client.? 172.21.15.26:0/6547130' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "c3dd2c3f-5cf5-4fb0-964c-1b8926f75c8a"}]': finished 2023-09-23T20:39:02.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:39:02 smithi026 ceph-mon[109912]: osdmap e23: 4 total, 3 up, 4 in 2023-09-23T20:39:02.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:39:02 smithi026 ceph-mon[109912]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-23T20:39:02.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:39:02 smithi026 ceph-mon[109912]: pgmap v75: 1 pgs: 1 active+clean; 577 KiB data, 17 MiB used, 268 GiB / 268 GiB avail 2023-09-23T20:39:02.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:39:02 smithi026 ceph-mon[109912]: from='client.? 172.21.15.26:0/3813076504' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-09-23T20:39:02.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:39:02 smithi026 ceph-mon[109912]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2023-09-23T20:39:02.633 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:39:02 smithi026 sudo[134395]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2023-09-23T20:39:02.633 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:39:02 smithi026 sudo[134395]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2023-09-23T20:39:02.633 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:39:02 smithi026 sudo[134395]: pam_unix(sudo:session): session opened for user root by (uid=0) 2023-09-23T20:39:02.633 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:39:02 smithi026 sudo[134395]: pam_unix(sudo:session): session closed for user root 2023-09-23T20:39:02.633 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:39:02 smithi026 ceph-mon[95839]: from='client.? 172.21.15.26:0/6547130' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "c3dd2c3f-5cf5-4fb0-964c-1b8926f75c8a"}]: dispatch 2023-09-23T20:39:02.633 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:39:02 smithi026 ceph-mon[95839]: from='client.? 172.21.15.26:0/6547130' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "c3dd2c3f-5cf5-4fb0-964c-1b8926f75c8a"}]': finished 2023-09-23T20:39:02.633 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:39:02 smithi026 ceph-mon[95839]: osdmap e23: 4 total, 3 up, 4 in 2023-09-23T20:39:02.633 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:39:02 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-23T20:39:02.633 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:39:02 smithi026 ceph-mon[95839]: pgmap v75: 1 pgs: 1 active+clean; 577 KiB data, 17 MiB used, 268 GiB / 268 GiB avail 2023-09-23T20:39:02.633 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:39:02 smithi026 ceph-mon[95839]: from='client.? 172.21.15.26:0/3813076504' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-09-23T20:39:02.634 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:39:02 smithi026 ceph-mon[95839]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2023-09-23T20:39:02.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:39:02 smithi070 sudo[103573]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2023-09-23T20:39:02.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:39:02 smithi070 sudo[103573]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2023-09-23T20:39:02.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:39:02 smithi070 sudo[103573]: pam_unix(sudo:session): session opened for user root by (uid=0) 2023-09-23T20:39:02.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:39:02 smithi070 sudo[103573]: pam_unix(sudo:session): session closed for user root 2023-09-23T20:39:03.380 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:39:03 smithi026 sudo[134781]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2023-09-23T20:39:03.380 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:39:03 smithi026 sudo[134781]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2023-09-23T20:39:03.380 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:39:03 smithi026 sudo[134781]: pam_unix(sudo:session): session opened for user root by (uid=0) 2023-09-23T20:39:03.380 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:39:03 smithi026 sudo[134781]: pam_unix(sudo:session): session closed for user root 2023-09-23T20:39:03.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:39:03 smithi026 ceph-mon[95839]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2023-09-23T20:39:03.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:39:03 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-23T20:39:03.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:39:03 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T20:39:03.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:39:03 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T20:39:03.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:39:03 smithi026 ceph-mon[95839]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2023-09-23T20:39:03.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:39:03 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-23T20:39:03.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:39:03 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T20:39:03.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:39:03 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T20:39:03.633 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:39:03 smithi026 ceph-mon[95839]: pgmap v76: 1 pgs: 1 active+clean; 577 KiB data, 17 MiB used, 268 GiB / 268 GiB avail 2023-09-23T20:39:03.633 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:39:03 smithi026 ceph-mon[95839]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2023-09-23T20:39:03.633 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:39:03 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:39:03.633 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:39:03 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-23T20:39:03.633 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:39:03 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T20:39:03.633 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:39:03 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T20:39:03.633 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:39:03 smithi026 sudo[134924]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2023-09-23T20:39:03.633 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:39:03 smithi026 sudo[134924]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2023-09-23T20:39:03.633 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:39:03 smithi026 sudo[134924]: pam_unix(sudo:session): session opened for user root by (uid=0) 2023-09-23T20:39:03.633 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:39:03 smithi026 sudo[134924]: pam_unix(sudo:session): session closed for user root 2023-09-23T20:39:03.634 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:39:03 smithi026 ceph-mon[109912]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2023-09-23T20:39:03.634 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:39:03 smithi026 ceph-mon[109912]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-23T20:39:03.634 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:39:03 smithi026 ceph-mon[109912]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T20:39:03.634 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:39:03 smithi026 ceph-mon[109912]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T20:39:03.634 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:39:03 smithi026 ceph-mon[109912]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2023-09-23T20:39:03.634 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:39:03 smithi026 ceph-mon[109912]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-23T20:39:03.634 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:39:03 smithi026 ceph-mon[109912]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T20:39:03.634 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:39:03 smithi026 ceph-mon[109912]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T20:39:03.634 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:39:03 smithi026 ceph-mon[109912]: pgmap v76: 1 pgs: 1 active+clean; 577 KiB data, 17 MiB used, 268 GiB / 268 GiB avail 2023-09-23T20:39:03.634 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:39:03 smithi026 ceph-mon[109912]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2023-09-23T20:39:03.634 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:39:03 smithi026 ceph-mon[109912]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:39:03.634 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:39:03 smithi026 ceph-mon[109912]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-23T20:39:03.635 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:39:03 smithi026 ceph-mon[109912]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T20:39:03.635 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:39:03 smithi026 ceph-mon[109912]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T20:39:03.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:39:03 smithi070 ceph-mon[100755]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2023-09-23T20:39:03.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:39:03 smithi070 ceph-mon[100755]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-23T20:39:03.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:39:03 smithi070 ceph-mon[100755]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T20:39:03.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:39:03 smithi070 ceph-mon[100755]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T20:39:03.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:39:03 smithi070 ceph-mon[100755]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2023-09-23T20:39:03.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:39:03 smithi070 ceph-mon[100755]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-23T20:39:03.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:39:03 smithi070 ceph-mon[100755]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T20:39:03.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:39:03 smithi070 ceph-mon[100755]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T20:39:03.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:39:03 smithi070 ceph-mon[100755]: pgmap v76: 1 pgs: 1 active+clean; 577 KiB data, 17 MiB used, 268 GiB / 268 GiB avail 2023-09-23T20:39:03.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:39:03 smithi070 ceph-mon[100755]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2023-09-23T20:39:03.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:39:03 smithi070 ceph-mon[100755]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:39:03.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:39:03 smithi070 ceph-mon[100755]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-23T20:39:03.871 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:39:03 smithi070 ceph-mon[100755]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T20:39:03.871 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:39:03 smithi070 ceph-mon[100755]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T20:39:04.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:39:04 smithi070 ceph-mon[100755]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2023-09-23T20:39:04.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:39:04 smithi070 ceph-mon[100755]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2023-09-23T20:39:04.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:39:04 smithi026 ceph-mon[95839]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2023-09-23T20:39:04.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:39:04 smithi026 ceph-mon[95839]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2023-09-23T20:39:04.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:39:04 smithi026 ceph-mon[109912]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2023-09-23T20:39:04.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:39:04 smithi026 ceph-mon[109912]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2023-09-23T20:39:05.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:39:05 smithi070 ceph-mon[100755]: pgmap v77: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail 2023-09-23T20:39:05.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:39:05 smithi026 ceph-mon[95839]: pgmap v77: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail 2023-09-23T20:39:05.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:39:05 smithi026 ceph-mon[109912]: pgmap v77: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail 2023-09-23T20:39:06.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:39:06 smithi070 ceph-mon[100755]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2023-09-23T20:39:06.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:39:06 smithi070 ceph-mon[100755]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:39:06.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:39:06 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2023-09-23T20:39:06.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:39:06 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:39:06.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:39:06 smithi026 ceph-mon[109912]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2023-09-23T20:39:06.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:39:06 smithi026 ceph-mon[109912]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:39:07.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:39:07 smithi070 ceph-mon[100755]: Deploying daemon osd.3 on smithi026 2023-09-23T20:39:07.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:39:07 smithi070 ceph-mon[100755]: pgmap v78: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail; 3.0 KiB/s rd, 130 KiB/s wr, 9 op/s 2023-09-23T20:39:07.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:39:07 smithi026 ceph-mon[95839]: Deploying daemon osd.3 on smithi026 2023-09-23T20:39:07.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:39:07 smithi026 ceph-mon[95839]: pgmap v78: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail; 3.0 KiB/s rd, 130 KiB/s wr, 9 op/s 2023-09-23T20:39:07.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:39:07 smithi026 ceph-mon[109912]: Deploying daemon osd.3 on smithi026 2023-09-23T20:39:07.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:39:07 smithi026 ceph-mon[109912]: pgmap v78: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail; 3.0 KiB/s rd, 130 KiB/s wr, 9 op/s 2023-09-23T20:39:09.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:39:09 smithi026 ceph-mon[95839]: pgmap v79: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail; 2.2 KiB/s rd, 97 KiB/s wr, 6 op/s 2023-09-23T20:39:09.633 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:39:09 smithi026 ceph-mon[109912]: pgmap v79: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail; 2.2 KiB/s rd, 97 KiB/s wr, 6 op/s 2023-09-23T20:39:09.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:39:09 smithi070 ceph-mon[100755]: pgmap v79: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail; 2.2 KiB/s rd, 97 KiB/s wr, 6 op/s 2023-09-23T20:39:11.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:39:11 smithi070 ceph-mon[100755]: pgmap v80: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail; 1.8 KiB/s rd, 78 KiB/s wr, 5 op/s 2023-09-23T20:39:11.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:39:11 smithi070 ceph-mon[100755]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:39:11.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:39:11 smithi070 ceph-mon[100755]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:39:11.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:39:11 smithi070 ceph-mon[100755]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:39:11.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:39:11 smithi026 ceph-mon[109912]: pgmap v80: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail; 1.8 KiB/s rd, 78 KiB/s wr, 5 op/s 2023-09-23T20:39:11.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:39:11 smithi026 ceph-mon[109912]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:39:11.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:39:11 smithi026 ceph-mon[109912]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:39:11.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:39:11 smithi026 ceph-mon[109912]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:39:11.883 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:39:11 smithi026 ceph-mon[95839]: pgmap v80: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail; 1.8 KiB/s rd, 78 KiB/s wr, 5 op/s 2023-09-23T20:39:11.883 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:39:11 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:39:11.883 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:39:11 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:39:11.883 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:39:11 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:39:13.429 INFO:teuthology.orchestra.run.smithi026.stdout:Created osd(s) 3 on host 'smithi026' 2023-09-23T20:39:13.807 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:39:13 smithi026 ceph-mon[95839]: pgmap v81: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail; 1.5 KiB/s rd, 65 KiB/s wr, 4 op/s 2023-09-23T20:39:13.807 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:39:13 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:39:13.807 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:39:13 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:39:13.807 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:39:13 smithi026 ceph-mon[109912]: pgmap v81: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail; 1.5 KiB/s rd, 65 KiB/s wr, 4 op/s 2023-09-23T20:39:13.808 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:39:13 smithi026 ceph-mon[109912]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:39:13.808 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:39:13 smithi026 ceph-mon[109912]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:39:13.836 DEBUG:teuthology.orchestra.run.smithi026:osd.3> sudo journalctl -f -n 0 -u ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7@osd.3.service 2023-09-23T20:39:13.838 INFO:tasks.cephadm:Deploying osd.4 on smithi070 with /dev/vg_nvme/lv_4... 2023-09-23T20:39:13.838 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph-volume lvm zap /dev/vg_nvme/lv_4 2023-09-23T20:39:13.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:39:13 smithi070 ceph-mon[100755]: pgmap v81: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail; 1.5 KiB/s rd, 65 KiB/s wr, 4 op/s 2023-09-23T20:39:13.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:39:13 smithi070 ceph-mon[100755]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:39:13.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:39:13 smithi070 ceph-mon[100755]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:39:13.873 INFO:journalctl@ceph.osd.3.smithi026.stdout:-- Logs begin at Sat 2023-09-23 20:20:42 UTC. -- 2023-09-23T20:39:14.449 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:39:14.850 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:39:14 smithi070 ceph-mon[100755]: from='osd.3 [v2:172.21.15.26:6826/3610585082,v1:172.21.15.26:6827/3610585082]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2023-09-23T20:39:14.850 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:39:14 smithi070 ceph-mon[100755]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:39:14.850 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:39:14 smithi070 ceph-mon[100755]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:39:14.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:39:14 smithi026 ceph-mon[95839]: from='osd.3 [v2:172.21.15.26:6826/3610585082,v1:172.21.15.26:6827/3610585082]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2023-09-23T20:39:14.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:39:14 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:39:14.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:39:14 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:39:14.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:39:14 smithi026 ceph-mon[109912]: from='osd.3 [v2:172.21.15.26:6826/3610585082,v1:172.21.15.26:6827/3610585082]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2023-09-23T20:39:14.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:39:14 smithi026 ceph-mon[109912]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:39:14.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:39:14 smithi026 ceph-mon[109912]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:39:15.758 INFO:journalctl@ceph.osd.3.smithi026.stdout:Sep 23 20:39:15 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd-3[135828]: 2023-09-23T20:39:15.590+0000 7f3066709700 -1 osd.3 0 waiting for initial osdmap 2023-09-23T20:39:15.758 INFO:journalctl@ceph.osd.3.smithi026.stdout:Sep 23 20:39:15 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd-3[135828]: 2023-09-23T20:39:15.595+0000 7f3062506700 -1 osd.3 25 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-09-23T20:39:15.758 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:39:15 smithi026 ceph-mon[109912]: pgmap v82: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail; 1.5 KiB/s rd, 65 KiB/s wr, 4 op/s 2023-09-23T20:39:15.758 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:39:15 smithi026 ceph-mon[109912]: from='osd.3 [v2:172.21.15.26:6826/3610585082,v1:172.21.15.26:6827/3610585082]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2023-09-23T20:39:15.758 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:39:15 smithi026 ceph-mon[109912]: osdmap e24: 4 total, 3 up, 4 in 2023-09-23T20:39:15.758 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:39:15 smithi026 ceph-mon[109912]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-23T20:39:15.759 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:39:15 smithi026 ceph-mon[109912]: from='osd.3 [v2:172.21.15.26:6826/3610585082,v1:172.21.15.26:6827/3610585082]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi026", "root=default"]}]: dispatch 2023-09-23T20:39:15.759 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:39:15 smithi026 ceph-mon[95839]: pgmap v82: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail; 1.5 KiB/s rd, 65 KiB/s wr, 4 op/s 2023-09-23T20:39:15.759 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:39:15 smithi026 ceph-mon[95839]: from='osd.3 [v2:172.21.15.26:6826/3610585082,v1:172.21.15.26:6827/3610585082]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2023-09-23T20:39:15.759 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:39:15 smithi026 ceph-mon[95839]: osdmap e24: 4 total, 3 up, 4 in 2023-09-23T20:39:15.759 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:39:15 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-23T20:39:15.759 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:39:15 smithi026 ceph-mon[95839]: from='osd.3 [v2:172.21.15.26:6826/3610585082,v1:172.21.15.26:6827/3610585082]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi026", "root=default"]}]: dispatch 2023-09-23T20:39:15.859 INFO:teuthology.orchestra.run.smithi070.stderr:--> Zapping: /dev/vg_nvme/lv_4 2023-09-23T20:39:15.859 INFO:teuthology.orchestra.run.smithi070.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_4 bs=1M count=10 conv=fsync 2023-09-23T20:39:15.859 INFO:teuthology.orchestra.run.smithi070.stderr: stderr: 10+0 records in 2023-09-23T20:39:15.859 INFO:teuthology.orchestra.run.smithi070.stderr:10+0 records out 2023-09-23T20:39:15.859 INFO:teuthology.orchestra.run.smithi070.stderr:10485760 bytes (10 MB, 10 MiB) copied, 0.012359 s, 848 MB/s 2023-09-23T20:39:15.859 INFO:teuthology.orchestra.run.smithi070.stderr:--> Zapping successful for: 2023-09-23T20:39:15.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:39:15 smithi070 ceph-mon[100755]: pgmap v82: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail; 1.5 KiB/s rd, 65 KiB/s wr, 4 op/s 2023-09-23T20:39:15.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:39:15 smithi070 ceph-mon[100755]: from='osd.3 [v2:172.21.15.26:6826/3610585082,v1:172.21.15.26:6827/3610585082]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2023-09-23T20:39:15.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:39:15 smithi070 ceph-mon[100755]: osdmap e24: 4 total, 3 up, 4 in 2023-09-23T20:39:15.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:39:15 smithi070 ceph-mon[100755]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-23T20:39:15.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:39:15 smithi070 ceph-mon[100755]: from='osd.3 [v2:172.21.15.26:6826/3610585082,v1:172.21.15.26:6827/3610585082]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi026", "root=default"]}]: dispatch 2023-09-23T20:39:16.188 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph orch daemon add osd smithi070:vg_nvme/lv_4 2023-09-23T20:39:16.815 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:39:16.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:39:16 smithi070 ceph-mon[100755]: from='osd.3 [v2:172.21.15.26:6826/3610585082,v1:172.21.15.26:6827/3610585082]' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi026", "root=default"]}]': finished 2023-09-23T20:39:16.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:39:16 smithi070 ceph-mon[100755]: osdmap e25: 4 total, 3 up, 4 in 2023-09-23T20:39:16.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:39:16 smithi070 ceph-mon[100755]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-23T20:39:16.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:39:16 smithi070 ceph-mon[100755]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-23T20:39:16.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:39:16 smithi026 ceph-mon[95839]: from='osd.3 [v2:172.21.15.26:6826/3610585082,v1:172.21.15.26:6827/3610585082]' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi026", "root=default"]}]': finished 2023-09-23T20:39:16.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:39:16 smithi026 ceph-mon[95839]: osdmap e25: 4 total, 3 up, 4 in 2023-09-23T20:39:16.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:39:16 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-23T20:39:16.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:39:16 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-23T20:39:16.883 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:39:16 smithi026 ceph-mon[109912]: from='osd.3 [v2:172.21.15.26:6826/3610585082,v1:172.21.15.26:6827/3610585082]' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi026", "root=default"]}]': finished 2023-09-23T20:39:16.883 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:39:16 smithi026 ceph-mon[109912]: osdmap e25: 4 total, 3 up, 4 in 2023-09-23T20:39:16.883 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:39:16 smithi026 ceph-mon[109912]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-23T20:39:16.883 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:39:16 smithi026 ceph-mon[109912]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-23T20:39:17.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:39:17 smithi070 ceph-mon[100755]: purged_snaps scrub starts 2023-09-23T20:39:17.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:39:17 smithi070 ceph-mon[100755]: purged_snaps scrub ok 2023-09-23T20:39:17.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:39:17 smithi070 ceph-mon[100755]: pgmap v85: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail 2023-09-23T20:39:17.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:39:17 smithi070 ceph-mon[100755]: osd.3 [v2:172.21.15.26:6826/3610585082,v1:172.21.15.26:6827/3610585082] boot 2023-09-23T20:39:17.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:39:17 smithi070 ceph-mon[100755]: osdmap e26: 4 total, 4 up, 4 in 2023-09-23T20:39:17.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:39:17 smithi070 ceph-mon[100755]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-23T20:39:17.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:39:17 smithi070 ceph-mon[100755]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:39:17.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:39:17 smithi070 ceph-mon[100755]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:39:17.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:39:17 smithi070 ceph-mon[100755]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:39:17.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:39:17 smithi070 ceph-mon[100755]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:39:17.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:39:17 smithi070 ceph-mon[100755]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:39:17.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:39:17 smithi070 ceph-mon[100755]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:39:17.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:39:17 smithi070 ceph-mon[100755]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:39:17.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:39:17 smithi070 ceph-mon[100755]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:39:17.621 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:39:17 smithi070 ceph-mon[100755]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:39:17.621 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:39:17 smithi070 ceph-mon[100755]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:39:17.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:39:17 smithi026 ceph-mon[109912]: purged_snaps scrub starts 2023-09-23T20:39:17.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:39:17 smithi026 ceph-mon[109912]: purged_snaps scrub ok 2023-09-23T20:39:17.883 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:39:17 smithi026 ceph-mon[109912]: pgmap v85: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail 2023-09-23T20:39:17.883 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:39:17 smithi026 ceph-mon[109912]: osd.3 [v2:172.21.15.26:6826/3610585082,v1:172.21.15.26:6827/3610585082] boot 2023-09-23T20:39:17.883 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:39:17 smithi026 ceph-mon[109912]: osdmap e26: 4 total, 4 up, 4 in 2023-09-23T20:39:17.883 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:39:17 smithi026 ceph-mon[109912]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-23T20:39:17.883 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:39:17 smithi026 ceph-mon[109912]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:39:17.883 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:39:17 smithi026 ceph-mon[109912]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:39:17.883 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:39:17 smithi026 ceph-mon[109912]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:39:17.883 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:39:17 smithi026 ceph-mon[109912]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:39:17.883 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:39:17 smithi026 ceph-mon[109912]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:39:17.883 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:39:17 smithi026 ceph-mon[109912]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:39:17.883 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:39:17 smithi026 ceph-mon[109912]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:39:17.883 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:39:17 smithi026 ceph-mon[109912]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:39:17.883 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:39:17 smithi026 ceph-mon[109912]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:39:17.884 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:39:17 smithi026 ceph-mon[109912]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:39:17.884 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:39:17 smithi026 ceph-mon[95839]: purged_snaps scrub starts 2023-09-23T20:39:17.884 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:39:17 smithi026 ceph-mon[95839]: purged_snaps scrub ok 2023-09-23T20:39:17.884 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:39:17 smithi026 ceph-mon[95839]: pgmap v85: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail 2023-09-23T20:39:17.884 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:39:17 smithi026 ceph-mon[95839]: osd.3 [v2:172.21.15.26:6826/3610585082,v1:172.21.15.26:6827/3610585082] boot 2023-09-23T20:39:17.884 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:39:17 smithi026 ceph-mon[95839]: osdmap e26: 4 total, 4 up, 4 in 2023-09-23T20:39:17.884 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:39:17 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-23T20:39:17.884 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:39:17 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:39:17.884 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:39:17 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:39:17.884 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:39:17 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:39:17.884 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:39:17 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:39:17.885 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:39:17 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:39:17.885 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:39:17 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:39:17.885 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:39:17 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:39:17.885 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:39:17 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:39:17.885 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:39:17 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:39:17.885 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:39:17 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:39:18.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:39:18 smithi070 ceph-mon[100755]: Detected new or changed devices on smithi026 2023-09-23T20:39:18.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:39:18 smithi070 ceph-mon[100755]: Adjusting osd_memory_target on smithi026 to 4037M 2023-09-23T20:39:18.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:39:18 smithi070 ceph-mon[100755]: osdmap e27: 4 total, 4 up, 4 in 2023-09-23T20:39:18.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:39:18 smithi070 ceph-mon[100755]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:39:18.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:39:18 smithi070 ceph-mon[100755]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:39:18.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:39:18 smithi070 ceph-mon[100755]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:39:18.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:39:18 smithi070 ceph-mon[100755]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:39:18.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:39:18 smithi070 ceph-mon[100755]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-09-23T20:39:18.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:39:18 smithi070 ceph-mon[100755]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-09-23T20:39:18.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:39:18 smithi070 ceph-mon[100755]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:39:18.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:39:18 smithi026 ceph-mon[95839]: Detected new or changed devices on smithi026 2023-09-23T20:39:18.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:39:18 smithi026 ceph-mon[95839]: Adjusting osd_memory_target on smithi026 to 4037M 2023-09-23T20:39:18.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:39:18 smithi026 ceph-mon[95839]: osdmap e27: 4 total, 4 up, 4 in 2023-09-23T20:39:18.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:39:18 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:39:18.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:39:18 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:39:18.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:39:18 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:39:18.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:39:18 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:39:18.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:39:18 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-09-23T20:39:18.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:39:18 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-09-23T20:39:18.883 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:39:18 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:39:18.883 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:39:18 smithi026 ceph-mon[109912]: Detected new or changed devices on smithi026 2023-09-23T20:39:18.883 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:39:18 smithi026 ceph-mon[109912]: Adjusting osd_memory_target on smithi026 to 4037M 2023-09-23T20:39:18.883 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:39:18 smithi026 ceph-mon[109912]: osdmap e27: 4 total, 4 up, 4 in 2023-09-23T20:39:18.883 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:39:18 smithi026 ceph-mon[109912]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:39:18.883 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:39:18 smithi026 ceph-mon[109912]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:39:18.883 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:39:18 smithi026 ceph-mon[109912]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:39:18.883 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:39:18 smithi026 ceph-mon[109912]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:39:18.883 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:39:18 smithi026 ceph-mon[109912]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-09-23T20:39:18.883 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:39:18 smithi026 ceph-mon[109912]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-09-23T20:39:18.884 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:39:18 smithi026 ceph-mon[109912]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:39:19.819 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:39:19 smithi070 ceph-mon[100755]: from='client.24205 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi070:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T20:39:19.819 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:39:19 smithi070 ceph-mon[100755]: pgmap v88: 1 pgs: 1 active+clean; 577 KiB data, 24 MiB used, 358 GiB / 358 GiB avail 2023-09-23T20:39:19.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:39:19 smithi026 ceph-mon[95839]: from='client.24205 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi070:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T20:39:19.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:39:19 smithi026 ceph-mon[95839]: pgmap v88: 1 pgs: 1 active+clean; 577 KiB data, 24 MiB used, 358 GiB / 358 GiB avail 2023-09-23T20:39:19.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:39:19 smithi026 ceph-mon[109912]: from='client.24205 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi070:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T20:39:19.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:39:19 smithi026 ceph-mon[109912]: pgmap v88: 1 pgs: 1 active+clean; 577 KiB data, 24 MiB used, 358 GiB / 358 GiB avail 2023-09-23T20:39:20.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:39:20 smithi070 ceph-mon[100755]: from='client.? 172.21.15.70:0/2832753284' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "a5f31996-1b38-491b-956b-5aebc72d0170"}]: dispatch 2023-09-23T20:39:20.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:39:20 smithi070 ceph-mon[100755]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "a5f31996-1b38-491b-956b-5aebc72d0170"}]: dispatch 2023-09-23T20:39:20.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:39:20 smithi070 ceph-mon[100755]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "a5f31996-1b38-491b-956b-5aebc72d0170"}]': finished 2023-09-23T20:39:20.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:39:20 smithi070 ceph-mon[100755]: osdmap e28: 5 total, 4 up, 5 in 2023-09-23T20:39:20.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:39:20 smithi070 ceph-mon[100755]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-09-23T20:39:20.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:39:20 smithi026 ceph-mon[95839]: from='client.? 172.21.15.70:0/2832753284' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "a5f31996-1b38-491b-956b-5aebc72d0170"}]: dispatch 2023-09-23T20:39:20.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:39:20 smithi026 ceph-mon[95839]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "a5f31996-1b38-491b-956b-5aebc72d0170"}]: dispatch 2023-09-23T20:39:20.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:39:20 smithi026 ceph-mon[95839]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "a5f31996-1b38-491b-956b-5aebc72d0170"}]': finished 2023-09-23T20:39:20.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:39:20 smithi026 ceph-mon[95839]: osdmap e28: 5 total, 4 up, 5 in 2023-09-23T20:39:20.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:39:20 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-09-23T20:39:20.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:39:20 smithi026 ceph-mon[109912]: from='client.? 172.21.15.70:0/2832753284' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "a5f31996-1b38-491b-956b-5aebc72d0170"}]: dispatch 2023-09-23T20:39:20.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:39:20 smithi026 ceph-mon[109912]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "a5f31996-1b38-491b-956b-5aebc72d0170"}]: dispatch 2023-09-23T20:39:20.883 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:39:20 smithi026 ceph-mon[109912]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "a5f31996-1b38-491b-956b-5aebc72d0170"}]': finished 2023-09-23T20:39:20.883 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:39:20 smithi026 ceph-mon[109912]: osdmap e28: 5 total, 4 up, 5 in 2023-09-23T20:39:20.883 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:39:20 smithi026 ceph-mon[109912]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-09-23T20:39:21.871 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:39:21 smithi070 ceph-mon[100755]: pgmap v89: 1 pgs: 1 active+clean; 577 KiB data, 24 MiB used, 358 GiB / 358 GiB avail 2023-09-23T20:39:21.871 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:39:21 smithi070 ceph-mon[100755]: from='client.? 172.21.15.70:0/1061742495' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-09-23T20:39:21.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:39:21 smithi026 ceph-mon[95839]: pgmap v89: 1 pgs: 1 active+clean; 577 KiB data, 24 MiB used, 358 GiB / 358 GiB avail 2023-09-23T20:39:21.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:39:21 smithi026 ceph-mon[95839]: from='client.? 172.21.15.70:0/1061742495' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-09-23T20:39:21.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:39:21 smithi026 ceph-mon[109912]: pgmap v89: 1 pgs: 1 active+clean; 577 KiB data, 24 MiB used, 358 GiB / 358 GiB avail 2023-09-23T20:39:21.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:39:21 smithi026 ceph-mon[109912]: from='client.? 172.21.15.70:0/1061742495' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-09-23T20:39:23.712 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:39:23 smithi070 ceph-mon[100755]: pgmap v91: 1 pgs: 1 active+clean; 577 KiB data, 24 MiB used, 358 GiB / 358 GiB avail 2023-09-23T20:39:23.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:39:23 smithi026 ceph-mon[95839]: pgmap v91: 1 pgs: 1 active+clean; 577 KiB data, 24 MiB used, 358 GiB / 358 GiB avail 2023-09-23T20:39:23.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:39:23 smithi026 ceph-mon[109912]: pgmap v91: 1 pgs: 1 active+clean; 577 KiB data, 24 MiB used, 358 GiB / 358 GiB avail 2023-09-23T20:39:25.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:39:25 smithi070 ceph-mon[100755]: pgmap v92: 1 pgs: 1 active+clean; 577 KiB data, 24 MiB used, 358 GiB / 358 GiB avail 2023-09-23T20:39:25.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:39:25 smithi026 ceph-mon[95839]: pgmap v92: 1 pgs: 1 active+clean; 577 KiB data, 24 MiB used, 358 GiB / 358 GiB avail 2023-09-23T20:39:25.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:39:25 smithi026 ceph-mon[109912]: pgmap v92: 1 pgs: 1 active+clean; 577 KiB data, 24 MiB used, 358 GiB / 358 GiB avail 2023-09-23T20:39:26.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:39:26 smithi070 ceph-mon[100755]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2023-09-23T20:39:26.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:39:26 smithi070 ceph-mon[100755]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:39:26.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:39:26 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2023-09-23T20:39:26.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:39:26 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:39:26.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:39:26 smithi026 ceph-mon[109912]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2023-09-23T20:39:26.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:39:26 smithi026 ceph-mon[109912]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:39:27.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:39:27 smithi070 ceph-mon[100755]: Deploying daemon osd.4 on smithi070 2023-09-23T20:39:27.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:39:27 smithi070 ceph-mon[100755]: pgmap v93: 1 pgs: 1 active+clean; 577 KiB data, 24 MiB used, 358 GiB / 358 GiB avail 2023-09-23T20:39:27.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:39:27 smithi026 ceph-mon[95839]: Deploying daemon osd.4 on smithi070 2023-09-23T20:39:27.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:39:27 smithi026 ceph-mon[95839]: pgmap v93: 1 pgs: 1 active+clean; 577 KiB data, 24 MiB used, 358 GiB / 358 GiB avail 2023-09-23T20:39:27.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:39:27 smithi026 ceph-mon[109912]: Deploying daemon osd.4 on smithi070 2023-09-23T20:39:27.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:39:27 smithi026 ceph-mon[109912]: pgmap v93: 1 pgs: 1 active+clean; 577 KiB data, 24 MiB used, 358 GiB / 358 GiB avail 2023-09-23T20:39:29.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:39:29 smithi070 ceph-mon[100755]: pgmap v94: 1 pgs: 1 active+clean; 577 KiB data, 24 MiB used, 358 GiB / 358 GiB avail 2023-09-23T20:39:29.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:39:29 smithi026 ceph-mon[95839]: pgmap v94: 1 pgs: 1 active+clean; 577 KiB data, 24 MiB used, 358 GiB / 358 GiB avail 2023-09-23T20:39:29.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:39:29 smithi026 ceph-mon[109912]: pgmap v94: 1 pgs: 1 active+clean; 577 KiB data, 24 MiB used, 358 GiB / 358 GiB avail 2023-09-23T20:39:31.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:39:31 smithi070 ceph-mon[100755]: pgmap v95: 1 pgs: 1 active+clean; 577 KiB data, 24 MiB used, 358 GiB / 358 GiB avail 2023-09-23T20:39:31.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:39:31 smithi070 ceph-mon[100755]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:39:31.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:39:31 smithi070 ceph-mon[100755]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:39:31.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:39:31 smithi070 ceph-mon[100755]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:39:31.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:39:31 smithi026 ceph-mon[109912]: pgmap v95: 1 pgs: 1 active+clean; 577 KiB data, 24 MiB used, 358 GiB / 358 GiB avail 2023-09-23T20:39:31.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:39:31 smithi026 ceph-mon[109912]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:39:31.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:39:31 smithi026 ceph-mon[109912]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:39:31.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:39:31 smithi026 ceph-mon[109912]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:39:31.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:39:31 smithi026 ceph-mon[95839]: pgmap v95: 1 pgs: 1 active+clean; 577 KiB data, 24 MiB used, 358 GiB / 358 GiB avail 2023-09-23T20:39:31.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:39:31 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:39:31.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:39:31 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:39:31.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:39:31 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:39:33.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:39:33 smithi070 ceph-mon[100755]: pgmap v96: 1 pgs: 1 active+clean; 577 KiB data, 24 MiB used, 358 GiB / 358 GiB avail 2023-09-23T20:39:33.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:39:33 smithi070 ceph-mon[100755]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:39:33.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:39:33 smithi070 ceph-mon[100755]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:39:33.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:39:33 smithi070 ceph-mon[100755]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:39:33.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:39:33 smithi070 ceph-mon[100755]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:39:33.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:39:33 smithi070 ceph-mon[100755]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:39:33.821 INFO:teuthology.orchestra.run.smithi070.stdout:Created osd(s) 4 on host 'smithi070' 2023-09-23T20:39:33.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:39:33 smithi026 ceph-mon[95839]: pgmap v96: 1 pgs: 1 active+clean; 577 KiB data, 24 MiB used, 358 GiB / 358 GiB avail 2023-09-23T20:39:33.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:39:33 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:39:33.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:39:33 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:39:33.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:39:33 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:39:33.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:39:33 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:39:33.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:39:33 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:39:33.883 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:39:33 smithi026 ceph-mon[109912]: pgmap v96: 1 pgs: 1 active+clean; 577 KiB data, 24 MiB used, 358 GiB / 358 GiB avail 2023-09-23T20:39:33.883 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:39:33 smithi026 ceph-mon[109912]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:39:33.883 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:39:33 smithi026 ceph-mon[109912]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:39:33.883 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:39:33 smithi026 ceph-mon[109912]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:39:33.883 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:39:33 smithi026 ceph-mon[109912]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:39:33.883 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:39:33 smithi026 ceph-mon[109912]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:39:34.203 DEBUG:teuthology.orchestra.run.smithi070:osd.4> sudo journalctl -f -n 0 -u ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7@osd.4.service 2023-09-23T20:39:34.205 INFO:tasks.cephadm:Deploying osd.5 on smithi070 with /dev/vg_nvme/lv_3... 2023-09-23T20:39:34.205 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph-volume lvm zap /dev/vg_nvme/lv_3 2023-09-23T20:39:34.234 INFO:journalctl@ceph.osd.4.smithi070.stdout:-- Logs begin at Sat 2023-09-23 20:21:16 UTC. -- 2023-09-23T20:39:35.013 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:39:35.117 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:39:34 smithi070 ceph-mon[100755]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:39:35.118 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:39:34 smithi070 ceph-mon[100755]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:39:35.118 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:39:34 smithi070 ceph-mon[100755]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:39:35.118 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:39:34 smithi070 ceph-mon[100755]: from='osd.4 [v2:172.21.15.70:6800/1528249138,v1:172.21.15.70:6801/1528249138]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2023-09-23T20:39:35.118 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:39:34 smithi070 ceph-mon[100755]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2023-09-23T20:39:35.118 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:39:34 smithi070 ceph-mon[100755]: pgmap v97: 1 pgs: 1 active+clean; 577 KiB data, 24 MiB used, 358 GiB / 358 GiB avail 2023-09-23T20:39:35.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:39:34 smithi026 ceph-mon[109912]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:39:35.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:39:34 smithi026 ceph-mon[109912]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:39:35.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:39:34 smithi026 ceph-mon[109912]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:39:35.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:39:34 smithi026 ceph-mon[109912]: from='osd.4 [v2:172.21.15.70:6800/1528249138,v1:172.21.15.70:6801/1528249138]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2023-09-23T20:39:35.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:39:34 smithi026 ceph-mon[109912]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2023-09-23T20:39:35.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:39:34 smithi026 ceph-mon[109912]: pgmap v97: 1 pgs: 1 active+clean; 577 KiB data, 24 MiB used, 358 GiB / 358 GiB avail 2023-09-23T20:39:35.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:39:34 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:39:35.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:39:34 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:39:35.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:39:34 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:39:35.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:39:34 smithi026 ceph-mon[95839]: from='osd.4 [v2:172.21.15.70:6800/1528249138,v1:172.21.15.70:6801/1528249138]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2023-09-23T20:39:35.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:39:34 smithi026 ceph-mon[95839]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2023-09-23T20:39:35.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:39:34 smithi026 ceph-mon[95839]: pgmap v97: 1 pgs: 1 active+clean; 577 KiB data, 24 MiB used, 358 GiB / 358 GiB avail 2023-09-23T20:39:36.118 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:39:35 smithi070 ceph-mon[100755]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2023-09-23T20:39:36.118 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:39:35 smithi070 ceph-mon[100755]: osdmap e29: 5 total, 4 up, 5 in 2023-09-23T20:39:36.118 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:39:35 smithi070 ceph-mon[100755]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-09-23T20:39:36.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:39:35 smithi070 ceph-mon[100755]: from='osd.4 [v2:172.21.15.70:6800/1528249138,v1:172.21.15.70:6801/1528249138]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi070", "root=default"]}]: dispatch 2023-09-23T20:39:36.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:39:35 smithi070 ceph-mon[100755]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi070", "root=default"]}]: dispatch 2023-09-23T20:39:36.119 INFO:journalctl@ceph.osd.4.smithi070.stdout:Sep 23 20:39:35 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd-4[106327]: 2023-09-23T20:39:35.782+0000 7f9b3ad63700 -1 osd.4 0 waiting for initial osdmap 2023-09-23T20:39:36.119 INFO:journalctl@ceph.osd.4.smithi070.stdout:Sep 23 20:39:35 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd-4[106327]: 2023-09-23T20:39:35.786+0000 7f9b37b62700 -1 osd.4 30 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-09-23T20:39:36.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:39:35 smithi026 ceph-mon[95839]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2023-09-23T20:39:36.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:39:35 smithi026 ceph-mon[95839]: osdmap e29: 5 total, 4 up, 5 in 2023-09-23T20:39:36.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:39:35 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-09-23T20:39:36.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:39:35 smithi026 ceph-mon[95839]: from='osd.4 [v2:172.21.15.70:6800/1528249138,v1:172.21.15.70:6801/1528249138]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi070", "root=default"]}]: dispatch 2023-09-23T20:39:36.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:39:35 smithi026 ceph-mon[95839]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi070", "root=default"]}]: dispatch 2023-09-23T20:39:36.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:39:35 smithi026 ceph-mon[109912]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2023-09-23T20:39:36.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:39:35 smithi026 ceph-mon[109912]: osdmap e29: 5 total, 4 up, 5 in 2023-09-23T20:39:36.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:39:35 smithi026 ceph-mon[109912]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-09-23T20:39:36.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:39:35 smithi026 ceph-mon[109912]: from='osd.4 [v2:172.21.15.70:6800/1528249138,v1:172.21.15.70:6801/1528249138]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi070", "root=default"]}]: dispatch 2023-09-23T20:39:36.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:39:35 smithi026 ceph-mon[109912]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi070", "root=default"]}]: dispatch 2023-09-23T20:39:37.006 INFO:teuthology.orchestra.run.smithi070.stderr:--> Zapping: /dev/vg_nvme/lv_3 2023-09-23T20:39:37.006 INFO:teuthology.orchestra.run.smithi070.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_3 bs=1M count=10 conv=fsync 2023-09-23T20:39:37.006 INFO:teuthology.orchestra.run.smithi070.stderr: stderr: 10+0 records in 2023-09-23T20:39:37.006 INFO:teuthology.orchestra.run.smithi070.stderr:10+0 records out 2023-09-23T20:39:37.006 INFO:teuthology.orchestra.run.smithi070.stderr:10485760 bytes (10 MB, 10 MiB) copied, 0.00840058 s, 1.2 GB/s 2023-09-23T20:39:37.007 INFO:teuthology.orchestra.run.smithi070.stderr:--> Zapping successful for: 2023-09-23T20:39:37.046 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:39:36 smithi070 ceph-mon[100755]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi070", "root=default"]}]': finished 2023-09-23T20:39:37.046 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:39:36 smithi070 ceph-mon[100755]: osdmap e30: 5 total, 4 up, 5 in 2023-09-23T20:39:37.046 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:39:36 smithi070 ceph-mon[100755]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-09-23T20:39:37.047 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:39:36 smithi070 ceph-mon[100755]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-09-23T20:39:37.047 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:39:36 smithi070 ceph-mon[100755]: pgmap v100: 1 pgs: 1 active+clean; 577 KiB data, 24 MiB used, 358 GiB / 358 GiB avail 2023-09-23T20:39:37.047 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:39:36 smithi070 ceph-mon[100755]: osd.4 [v2:172.21.15.70:6800/1528249138,v1:172.21.15.70:6801/1528249138] boot 2023-09-23T20:39:37.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:39:36 smithi026 ceph-mon[95839]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi070", "root=default"]}]': finished 2023-09-23T20:39:37.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:39:36 smithi026 ceph-mon[95839]: osdmap e30: 5 total, 4 up, 5 in 2023-09-23T20:39:37.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:39:36 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-09-23T20:39:37.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:39:36 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-09-23T20:39:37.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:39:36 smithi026 ceph-mon[95839]: pgmap v100: 1 pgs: 1 active+clean; 577 KiB data, 24 MiB used, 358 GiB / 358 GiB avail 2023-09-23T20:39:37.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:39:36 smithi026 ceph-mon[95839]: osd.4 [v2:172.21.15.70:6800/1528249138,v1:172.21.15.70:6801/1528249138] boot 2023-09-23T20:39:37.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:39:36 smithi026 ceph-mon[109912]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi070", "root=default"]}]': finished 2023-09-23T20:39:37.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:39:36 smithi026 ceph-mon[109912]: osdmap e30: 5 total, 4 up, 5 in 2023-09-23T20:39:37.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:39:36 smithi026 ceph-mon[109912]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-09-23T20:39:37.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:39:36 smithi026 ceph-mon[109912]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-09-23T20:39:37.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:39:36 smithi026 ceph-mon[109912]: pgmap v100: 1 pgs: 1 active+clean; 577 KiB data, 24 MiB used, 358 GiB / 358 GiB avail 2023-09-23T20:39:37.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:39:36 smithi026 ceph-mon[109912]: osd.4 [v2:172.21.15.70:6800/1528249138,v1:172.21.15.70:6801/1528249138] boot 2023-09-23T20:39:37.419 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph orch daemon add osd smithi070:vg_nvme/lv_3 2023-09-23T20:39:38.009 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:39:38.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:39:37 smithi070 ceph-mon[100755]: purged_snaps scrub starts 2023-09-23T20:39:38.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:39:37 smithi070 ceph-mon[100755]: purged_snaps scrub ok 2023-09-23T20:39:38.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:39:37 smithi070 ceph-mon[100755]: osdmap e31: 5 total, 5 up, 5 in 2023-09-23T20:39:38.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:39:37 smithi070 ceph-mon[100755]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-09-23T20:39:38.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:39:37 smithi070 ceph-mon[100755]: Detected new or changed devices on smithi070 2023-09-23T20:39:38.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:39:37 smithi070 ceph-mon[100755]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:39:38.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:39:37 smithi070 ceph-mon[100755]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:39:38.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:39:37 smithi070 ceph-mon[100755]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:39:38.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:39:37 smithi070 ceph-mon[100755]: Adjusting osd_memory_target on smithi070 to 17168M 2023-09-23T20:39:38.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:39:37 smithi070 ceph-mon[100755]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:39:38.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:39:37 smithi070 ceph-mon[100755]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:39:38.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:39:37 smithi070 ceph-mon[100755]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:39:38.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:39:37 smithi070 ceph-mon[100755]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:39:38.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:39:37 smithi070 ceph-mon[100755]: osdmap e32: 5 total, 5 up, 5 in 2023-09-23T20:39:38.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:39:37 smithi026 ceph-mon[95839]: purged_snaps scrub starts 2023-09-23T20:39:38.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:39:37 smithi026 ceph-mon[95839]: purged_snaps scrub ok 2023-09-23T20:39:38.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:39:37 smithi026 ceph-mon[95839]: osdmap e31: 5 total, 5 up, 5 in 2023-09-23T20:39:38.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:39:37 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-09-23T20:39:38.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:39:37 smithi026 ceph-mon[95839]: Detected new or changed devices on smithi070 2023-09-23T20:39:38.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:39:37 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:39:38.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:39:37 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:39:38.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:39:37 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:39:38.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:39:37 smithi026 ceph-mon[95839]: Adjusting osd_memory_target on smithi070 to 17168M 2023-09-23T20:39:38.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:39:37 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:39:38.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:39:37 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:39:38.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:39:37 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:39:38.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:39:37 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:39:38.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:39:37 smithi026 ceph-mon[95839]: osdmap e32: 5 total, 5 up, 5 in 2023-09-23T20:39:38.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:39:37 smithi026 ceph-mon[109912]: purged_snaps scrub starts 2023-09-23T20:39:38.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:39:37 smithi026 ceph-mon[109912]: purged_snaps scrub ok 2023-09-23T20:39:38.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:39:37 smithi026 ceph-mon[109912]: osdmap e31: 5 total, 5 up, 5 in 2023-09-23T20:39:38.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:39:37 smithi026 ceph-mon[109912]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-09-23T20:39:38.134 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:39:37 smithi026 ceph-mon[109912]: Detected new or changed devices on smithi070 2023-09-23T20:39:38.134 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:39:37 smithi026 ceph-mon[109912]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:39:38.134 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:39:37 smithi026 ceph-mon[109912]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:39:38.134 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:39:37 smithi026 ceph-mon[109912]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:39:38.134 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:39:37 smithi026 ceph-mon[109912]: Adjusting osd_memory_target on smithi070 to 17168M 2023-09-23T20:39:38.134 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:39:37 smithi026 ceph-mon[109912]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:39:38.134 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:39:37 smithi026 ceph-mon[109912]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:39:38.134 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:39:37 smithi026 ceph-mon[109912]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:39:38.134 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:39:37 smithi026 ceph-mon[109912]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:39:38.134 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:39:37 smithi026 ceph-mon[109912]: osdmap e32: 5 total, 5 up, 5 in 2023-09-23T20:39:39.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:39:39 smithi070 ceph-mon[100755]: pgmap v103: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2023-09-23T20:39:39.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:39:39 smithi070 ceph-mon[100755]: osdmap e33: 5 total, 5 up, 5 in 2023-09-23T20:39:39.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:39:39 smithi070 ceph-mon[100755]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-09-23T20:39:39.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:39:39 smithi070 ceph-mon[100755]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-09-23T20:39:39.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:39:39 smithi070 ceph-mon[100755]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:39:39.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:39:39 smithi026 ceph-mon[109912]: pgmap v103: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2023-09-23T20:39:39.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:39:39 smithi026 ceph-mon[109912]: osdmap e33: 5 total, 5 up, 5 in 2023-09-23T20:39:39.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:39:39 smithi026 ceph-mon[109912]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-09-23T20:39:39.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:39:39 smithi026 ceph-mon[109912]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-09-23T20:39:39.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:39:39 smithi026 ceph-mon[109912]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:39:39.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:39:39 smithi026 ceph-mon[95839]: pgmap v103: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2023-09-23T20:39:39.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:39:39 smithi026 ceph-mon[95839]: osdmap e33: 5 total, 5 up, 5 in 2023-09-23T20:39:39.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:39:39 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-09-23T20:39:39.883 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:39:39 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-09-23T20:39:39.883 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:39:39 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:39:40.826 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:39:40 smithi070 ceph-mon[100755]: from='client.24232 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi070:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T20:39:40.826 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:39:40 smithi070 ceph-mon[100755]: osdmap e34: 5 total, 5 up, 5 in 2023-09-23T20:39:40.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:39:40 smithi026 ceph-mon[95839]: from='client.24232 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi070:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T20:39:40.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:39:40 smithi026 ceph-mon[95839]: osdmap e34: 5 total, 5 up, 5 in 2023-09-23T20:39:40.883 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:39:40 smithi026 ceph-mon[109912]: from='client.24232 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi070:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T20:39:40.883 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:39:40 smithi026 ceph-mon[109912]: osdmap e34: 5 total, 5 up, 5 in 2023-09-23T20:39:41.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:39:41 smithi070 ceph-mon[100755]: pgmap v106: 1 pgs: 1 peering; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 144 KiB/s, 0 objects/s recovering 2023-09-23T20:39:41.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:39:41 smithi070 ceph-mon[100755]: osdmap e35: 5 total, 5 up, 5 in 2023-09-23T20:39:41.881 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:39:41 smithi026 ceph-mon[95839]: pgmap v106: 1 pgs: 1 peering; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 144 KiB/s, 0 objects/s recovering 2023-09-23T20:39:41.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:39:41 smithi026 ceph-mon[95839]: osdmap e35: 5 total, 5 up, 5 in 2023-09-23T20:39:41.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:39:41 smithi026 ceph-mon[109912]: pgmap v106: 1 pgs: 1 peering; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 144 KiB/s, 0 objects/s recovering 2023-09-23T20:39:41.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:39:41 smithi026 ceph-mon[109912]: osdmap e35: 5 total, 5 up, 5 in 2023-09-23T20:39:42.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:39:42 smithi070 ceph-mon[100755]: from='client.? 172.21.15.70:0/772325681' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "21dad198-b9db-42c6-8823-112df1ecd874"}]: dispatch 2023-09-23T20:39:42.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:39:42 smithi070 ceph-mon[100755]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "21dad198-b9db-42c6-8823-112df1ecd874"}]: dispatch 2023-09-23T20:39:42.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:39:42 smithi070 ceph-mon[100755]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "21dad198-b9db-42c6-8823-112df1ecd874"}]': finished 2023-09-23T20:39:42.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:39:42 smithi070 ceph-mon[100755]: osdmap e36: 6 total, 5 up, 6 in 2023-09-23T20:39:42.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:39:42 smithi070 ceph-mon[100755]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-09-23T20:39:42.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:39:42 smithi070 ceph-mon[100755]: from='client.? 172.21.15.70:0/2603589783' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-09-23T20:39:42.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:39:42 smithi026 ceph-mon[95839]: from='client.? 172.21.15.70:0/772325681' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "21dad198-b9db-42c6-8823-112df1ecd874"}]: dispatch 2023-09-23T20:39:42.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:39:42 smithi026 ceph-mon[95839]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "21dad198-b9db-42c6-8823-112df1ecd874"}]: dispatch 2023-09-23T20:39:42.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:39:42 smithi026 ceph-mon[95839]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "21dad198-b9db-42c6-8823-112df1ecd874"}]': finished 2023-09-23T20:39:42.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:39:42 smithi026 ceph-mon[95839]: osdmap e36: 6 total, 5 up, 6 in 2023-09-23T20:39:42.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:39:42 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-09-23T20:39:42.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:39:42 smithi026 ceph-mon[95839]: from='client.? 172.21.15.70:0/2603589783' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-09-23T20:39:42.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:39:42 smithi026 ceph-mon[109912]: from='client.? 172.21.15.70:0/772325681' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "21dad198-b9db-42c6-8823-112df1ecd874"}]: dispatch 2023-09-23T20:39:42.883 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:39:42 smithi026 ceph-mon[109912]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "21dad198-b9db-42c6-8823-112df1ecd874"}]: dispatch 2023-09-23T20:39:42.883 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:39:42 smithi026 ceph-mon[109912]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "21dad198-b9db-42c6-8823-112df1ecd874"}]': finished 2023-09-23T20:39:42.883 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:39:42 smithi026 ceph-mon[109912]: osdmap e36: 6 total, 5 up, 6 in 2023-09-23T20:39:42.883 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:39:42 smithi026 ceph-mon[109912]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-09-23T20:39:42.883 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:39:42 smithi026 ceph-mon[109912]: from='client.? 172.21.15.70:0/2603589783' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-09-23T20:39:43.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:39:43 smithi070 ceph-mon[100755]: pgmap v109: 1 pgs: 1 peering; 577 KiB data, 30 MiB used, 447 GiB / 447 GiB avail; 144 KiB/s, 0 objects/s recovering 2023-09-23T20:39:43.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:39:43 smithi026 ceph-mon[95839]: pgmap v109: 1 pgs: 1 peering; 577 KiB data, 30 MiB used, 447 GiB / 447 GiB avail; 144 KiB/s, 0 objects/s recovering 2023-09-23T20:39:43.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:39:43 smithi026 ceph-mon[109912]: pgmap v109: 1 pgs: 1 peering; 577 KiB data, 30 MiB used, 447 GiB / 447 GiB avail; 144 KiB/s, 0 objects/s recovering 2023-09-23T20:39:45.854 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:39:45 smithi070 ceph-mon[100755]: pgmap v110: 1 pgs: 1 peering; 577 KiB data, 30 MiB used, 447 GiB / 447 GiB avail; 100 KiB/s, 0 objects/s recovering 2023-09-23T20:39:45.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:39:45 smithi026 ceph-mon[95839]: pgmap v110: 1 pgs: 1 peering; 577 KiB data, 30 MiB used, 447 GiB / 447 GiB avail; 100 KiB/s, 0 objects/s recovering 2023-09-23T20:39:45.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:39:45 smithi026 ceph-mon[109912]: pgmap v110: 1 pgs: 1 peering; 577 KiB data, 30 MiB used, 447 GiB / 447 GiB avail; 100 KiB/s, 0 objects/s recovering 2023-09-23T20:39:47.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:39:47 smithi070 ceph-mon[100755]: pgmap v111: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 85 KiB/s, 0 objects/s recovering 2023-09-23T20:39:47.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:39:47 smithi070 ceph-mon[100755]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2023-09-23T20:39:47.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:39:47 smithi070 ceph-mon[100755]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:39:47.881 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:39:47 smithi026 ceph-mon[95839]: pgmap v111: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 85 KiB/s, 0 objects/s recovering 2023-09-23T20:39:47.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:39:47 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2023-09-23T20:39:47.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:39:47 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:39:47.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:39:47 smithi026 ceph-mon[109912]: pgmap v111: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 85 KiB/s, 0 objects/s recovering 2023-09-23T20:39:47.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:39:47 smithi026 ceph-mon[109912]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2023-09-23T20:39:47.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:39:47 smithi026 ceph-mon[109912]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:39:48.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:39:48 smithi070 ceph-mon[100755]: Deploying daemon osd.5 on smithi070 2023-09-23T20:39:48.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:39:48 smithi026 ceph-mon[95839]: Deploying daemon osd.5 on smithi070 2023-09-23T20:39:48.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:39:48 smithi026 ceph-mon[109912]: Deploying daemon osd.5 on smithi070 2023-09-23T20:39:49.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:39:49 smithi070 ceph-mon[100755]: pgmap v112: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2023-09-23T20:39:49.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:39:49 smithi026 ceph-mon[95839]: pgmap v112: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2023-09-23T20:39:49.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:39:49 smithi026 ceph-mon[109912]: pgmap v112: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2023-09-23T20:39:51.787 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:39:51 smithi070 ceph-mon[100755]: pgmap v113: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2023-09-23T20:39:51.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:39:51 smithi026 ceph-mon[95839]: pgmap v113: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2023-09-23T20:39:51.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:39:51 smithi026 ceph-mon[109912]: pgmap v113: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2023-09-23T20:39:53.064 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:39:52 smithi070 ceph-mon[100755]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:39:53.064 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:39:52 smithi070 ceph-mon[100755]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:39:53.064 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:39:52 smithi070 ceph-mon[100755]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:39:53.064 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:39:52 smithi070 ceph-mon[100755]: pgmap v114: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2023-09-23T20:39:53.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:39:52 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:39:53.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:39:52 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:39:53.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:39:52 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:39:53.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:39:52 smithi026 ceph-mon[95839]: pgmap v114: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2023-09-23T20:39:53.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:39:52 smithi026 ceph-mon[109912]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:39:53.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:39:52 smithi026 ceph-mon[109912]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:39:53.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:39:52 smithi026 ceph-mon[109912]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:39:53.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:39:52 smithi026 ceph-mon[109912]: pgmap v114: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2023-09-23T20:39:54.685 INFO:teuthology.orchestra.run.smithi070.stdout:Created osd(s) 5 on host 'smithi070' 2023-09-23T20:39:55.094 DEBUG:teuthology.orchestra.run.smithi070:osd.5> sudo journalctl -f -n 0 -u ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7@osd.5.service 2023-09-23T20:39:55.096 INFO:tasks.cephadm:Deploying osd.6 on smithi070 with /dev/vg_nvme/lv_2... 2023-09-23T20:39:55.096 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph-volume lvm zap /dev/vg_nvme/lv_2 2023-09-23T20:39:55.127 INFO:journalctl@ceph.osd.5.smithi070.stdout:-- Logs begin at Sat 2023-09-23 20:21:16 UTC. -- 2023-09-23T20:39:55.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:39:55 smithi070 ceph-mon[100755]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:39:55.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:39:55 smithi070 ceph-mon[100755]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:39:55.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:39:55 smithi070 ceph-mon[100755]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:39:55.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:39:55 smithi070 ceph-mon[100755]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:39:55.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:39:55 smithi070 ceph-mon[100755]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:39:55.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:39:55 smithi070 ceph-mon[100755]: pgmap v115: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2023-09-23T20:39:55.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:39:55 smithi070 ceph-mon[100755]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:39:55.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:39:55 smithi070 ceph-mon[100755]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:39:55.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:39:55 smithi070 ceph-mon[100755]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:39:55.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:39:55 smithi070 ceph-mon[100755]: from='osd.5 [v2:172.21.15.70:6808/3802232906,v1:172.21.15.70:6809/3802232906]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2023-09-23T20:39:55.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:39:55 smithi070 ceph-mon[100755]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2023-09-23T20:39:55.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:39:55 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:39:55.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:39:55 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:39:55.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:39:55 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:39:55.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:39:55 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:39:55.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:39:55 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:39:55.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:39:55 smithi026 ceph-mon[95839]: pgmap v115: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2023-09-23T20:39:55.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:39:55 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:39:55.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:39:55 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:39:55.633 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:39:55 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:39:55.633 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:39:55 smithi026 ceph-mon[95839]: from='osd.5 [v2:172.21.15.70:6808/3802232906,v1:172.21.15.70:6809/3802232906]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2023-09-23T20:39:55.633 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:39:55 smithi026 ceph-mon[95839]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2023-09-23T20:39:55.633 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:39:55 smithi026 ceph-mon[109912]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:39:55.633 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:39:55 smithi026 ceph-mon[109912]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:39:55.633 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:39:55 smithi026 ceph-mon[109912]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:39:55.633 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:39:55 smithi026 ceph-mon[109912]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:39:55.633 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:39:55 smithi026 ceph-mon[109912]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:39:55.633 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:39:55 smithi026 ceph-mon[109912]: pgmap v115: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2023-09-23T20:39:55.634 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:39:55 smithi026 ceph-mon[109912]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:39:55.634 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:39:55 smithi026 ceph-mon[109912]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:39:55.634 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:39:55 smithi026 ceph-mon[109912]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:39:55.634 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:39:55 smithi026 ceph-mon[109912]: from='osd.5 [v2:172.21.15.70:6808/3802232906,v1:172.21.15.70:6809/3802232906]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2023-09-23T20:39:55.634 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:39:55 smithi026 ceph-mon[109912]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2023-09-23T20:39:56.133 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:39:56.960 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:39:56 smithi070 ceph-mon[100755]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2023-09-23T20:39:56.960 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:39:56 smithi070 ceph-mon[100755]: osdmap e37: 6 total, 5 up, 6 in 2023-09-23T20:39:56.960 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:39:56 smithi070 ceph-mon[100755]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-09-23T20:39:56.960 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:39:56 smithi070 ceph-mon[100755]: from='osd.5 [v2:172.21.15.70:6808/3802232906,v1:172.21.15.70:6809/3802232906]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi070", "root=default"]}]: dispatch 2023-09-23T20:39:56.961 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:39:56 smithi070 ceph-mon[100755]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi070", "root=default"]}]: dispatch 2023-09-23T20:39:56.961 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:39:56 smithi070 ceph-mon[100755]: pgmap v117: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2023-09-23T20:39:56.961 INFO:journalctl@ceph.osd.5.smithi070.stdout:Sep 23 20:39:56 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd-5[113356]: 2023-09-23T20:39:56.665+0000 7ff1a68ca700 -1 osd.5 0 waiting for initial osdmap 2023-09-23T20:39:56.961 INFO:journalctl@ceph.osd.5.smithi070.stdout:Sep 23 20:39:56 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd-5[113356]: 2023-09-23T20:39:56.670+0000 7ff1a16c5700 -1 osd.5 38 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-09-23T20:39:57.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:39:56 smithi026 ceph-mon[95839]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2023-09-23T20:39:57.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:39:56 smithi026 ceph-mon[95839]: osdmap e37: 6 total, 5 up, 6 in 2023-09-23T20:39:57.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:39:56 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-09-23T20:39:57.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:39:56 smithi026 ceph-mon[95839]: from='osd.5 [v2:172.21.15.70:6808/3802232906,v1:172.21.15.70:6809/3802232906]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi070", "root=default"]}]: dispatch 2023-09-23T20:39:57.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:39:56 smithi026 ceph-mon[95839]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi070", "root=default"]}]: dispatch 2023-09-23T20:39:57.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:39:56 smithi026 ceph-mon[95839]: pgmap v117: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2023-09-23T20:39:57.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:39:56 smithi026 ceph-mon[109912]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2023-09-23T20:39:57.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:39:56 smithi026 ceph-mon[109912]: osdmap e37: 6 total, 5 up, 6 in 2023-09-23T20:39:57.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:39:56 smithi026 ceph-mon[109912]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-09-23T20:39:57.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:39:56 smithi026 ceph-mon[109912]: from='osd.5 [v2:172.21.15.70:6808/3802232906,v1:172.21.15.70:6809/3802232906]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi070", "root=default"]}]: dispatch 2023-09-23T20:39:57.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:39:56 smithi026 ceph-mon[109912]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi070", "root=default"]}]: dispatch 2023-09-23T20:39:57.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:39:56 smithi026 ceph-mon[109912]: pgmap v117: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2023-09-23T20:39:57.759 INFO:teuthology.orchestra.run.smithi070.stderr:--> Zapping: /dev/vg_nvme/lv_2 2023-09-23T20:39:57.760 INFO:teuthology.orchestra.run.smithi070.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_2 bs=1M count=10 conv=fsync 2023-09-23T20:39:57.760 INFO:teuthology.orchestra.run.smithi070.stderr: stderr: 10+0 records in 2023-09-23T20:39:57.760 INFO:teuthology.orchestra.run.smithi070.stderr:10+0 records out 2023-09-23T20:39:57.760 INFO:teuthology.orchestra.run.smithi070.stderr:10485760 bytes (10 MB, 10 MiB) copied, 0.00808115 s, 1.3 GB/s 2023-09-23T20:39:57.760 INFO:teuthology.orchestra.run.smithi070.stderr: stderr: 2023-09-23T20:39:57.760 INFO:teuthology.orchestra.run.smithi070.stderr:--> Zapping successful for: 2023-09-23T20:39:57.793 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:39:57 smithi070 ceph-mon[100755]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi070", "root=default"]}]': finished 2023-09-23T20:39:57.793 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:39:57 smithi070 ceph-mon[100755]: osdmap e38: 6 total, 5 up, 6 in 2023-09-23T20:39:57.793 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:39:57 smithi070 ceph-mon[100755]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-09-23T20:39:57.794 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:39:57 smithi070 ceph-mon[100755]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-09-23T20:39:58.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:39:57 smithi026 ceph-mon[95839]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi070", "root=default"]}]': finished 2023-09-23T20:39:58.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:39:57 smithi026 ceph-mon[95839]: osdmap e38: 6 total, 5 up, 6 in 2023-09-23T20:39:58.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:39:57 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-09-23T20:39:58.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:39:57 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-09-23T20:39:58.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:39:57 smithi026 ceph-mon[109912]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi070", "root=default"]}]': finished 2023-09-23T20:39:58.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:39:57 smithi026 ceph-mon[109912]: osdmap e38: 6 total, 5 up, 6 in 2023-09-23T20:39:58.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:39:57 smithi026 ceph-mon[109912]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-09-23T20:39:58.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:39:57 smithi026 ceph-mon[109912]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-09-23T20:39:58.185 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph orch daemon add osd smithi070:vg_nvme/lv_2 2023-09-23T20:39:58.778 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:39:58.952 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:39:58 smithi070 ceph-mon[100755]: purged_snaps scrub starts 2023-09-23T20:39:58.952 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:39:58 smithi070 ceph-mon[100755]: purged_snaps scrub ok 2023-09-23T20:39:58.952 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:39:58 smithi070 ceph-mon[100755]: osd.5 [v2:172.21.15.70:6808/3802232906,v1:172.21.15.70:6809/3802232906] boot 2023-09-23T20:39:58.952 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:39:58 smithi070 ceph-mon[100755]: osdmap e39: 6 total, 6 up, 6 in 2023-09-23T20:39:58.952 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:39:58 smithi070 ceph-mon[100755]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-09-23T20:39:58.952 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:39:58 smithi070 ceph-mon[100755]: Detected new or changed devices on smithi070 2023-09-23T20:39:58.953 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:39:58 smithi070 ceph-mon[100755]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:39:58.953 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:39:58 smithi070 ceph-mon[100755]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:39:58.953 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:39:58 smithi070 ceph-mon[100755]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:39:58.953 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:39:58 smithi070 ceph-mon[100755]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:39:58.953 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:39:58 smithi070 ceph-mon[100755]: Adjusting osd_memory_target on smithi070 to 8584M 2023-09-23T20:39:58.953 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:39:58 smithi070 ceph-mon[100755]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:39:58.953 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:39:58 smithi070 ceph-mon[100755]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:39:58.953 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:39:58 smithi070 ceph-mon[100755]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:39:58.953 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:39:58 smithi070 ceph-mon[100755]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:39:58.953 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:39:58 smithi070 ceph-mon[100755]: osdmap e40: 6 total, 6 up, 6 in 2023-09-23T20:39:58.953 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:39:58 smithi070 ceph-mon[100755]: pgmap v121: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2023-09-23T20:39:59.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:39:58 smithi026 ceph-mon[95839]: purged_snaps scrub starts 2023-09-23T20:39:59.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:39:58 smithi026 ceph-mon[95839]: purged_snaps scrub ok 2023-09-23T20:39:59.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:39:58 smithi026 ceph-mon[95839]: osd.5 [v2:172.21.15.70:6808/3802232906,v1:172.21.15.70:6809/3802232906] boot 2023-09-23T20:39:59.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:39:58 smithi026 ceph-mon[95839]: osdmap e39: 6 total, 6 up, 6 in 2023-09-23T20:39:59.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:39:58 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-09-23T20:39:59.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:39:58 smithi026 ceph-mon[95839]: Detected new or changed devices on smithi070 2023-09-23T20:39:59.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:39:58 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:39:59.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:39:58 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:39:59.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:39:58 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:39:59.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:39:58 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:39:59.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:39:58 smithi026 ceph-mon[95839]: Adjusting osd_memory_target on smithi070 to 8584M 2023-09-23T20:39:59.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:39:58 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:39:59.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:39:58 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:39:59.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:39:58 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:39:59.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:39:58 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:39:59.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:39:58 smithi026 ceph-mon[95839]: osdmap e40: 6 total, 6 up, 6 in 2023-09-23T20:39:59.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:39:58 smithi026 ceph-mon[95839]: pgmap v121: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2023-09-23T20:39:59.134 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:39:58 smithi026 ceph-mon[109912]: purged_snaps scrub starts 2023-09-23T20:39:59.134 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:39:58 smithi026 ceph-mon[109912]: purged_snaps scrub ok 2023-09-23T20:39:59.134 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:39:58 smithi026 ceph-mon[109912]: osd.5 [v2:172.21.15.70:6808/3802232906,v1:172.21.15.70:6809/3802232906] boot 2023-09-23T20:39:59.134 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:39:58 smithi026 ceph-mon[109912]: osdmap e39: 6 total, 6 up, 6 in 2023-09-23T20:39:59.134 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:39:58 smithi026 ceph-mon[109912]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-09-23T20:39:59.134 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:39:58 smithi026 ceph-mon[109912]: Detected new or changed devices on smithi070 2023-09-23T20:39:59.134 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:39:58 smithi026 ceph-mon[109912]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:39:59.134 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:39:58 smithi026 ceph-mon[109912]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:39:59.134 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:39:58 smithi026 ceph-mon[109912]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:39:59.135 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:39:58 smithi026 ceph-mon[109912]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:39:59.135 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:39:58 smithi026 ceph-mon[109912]: Adjusting osd_memory_target on smithi070 to 8584M 2023-09-23T20:39:59.135 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:39:58 smithi026 ceph-mon[109912]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:39:59.135 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:39:58 smithi026 ceph-mon[109912]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:39:59.135 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:39:58 smithi026 ceph-mon[109912]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:39:59.135 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:39:58 smithi026 ceph-mon[109912]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:39:59.135 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:39:58 smithi026 ceph-mon[109912]: osdmap e40: 6 total, 6 up, 6 in 2023-09-23T20:39:59.135 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:39:58 smithi026 ceph-mon[109912]: pgmap v121: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2023-09-23T20:40:00.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:40:00 smithi070 ceph-mon[100755]: osdmap e41: 6 total, 6 up, 6 in 2023-09-23T20:40:00.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:40:00 smithi070 ceph-mon[100755]: overall HEALTH_OK 2023-09-23T20:40:00.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:40:00 smithi070 ceph-mon[100755]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-09-23T20:40:00.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:40:00 smithi070 ceph-mon[100755]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-09-23T20:40:00.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:40:00 smithi070 ceph-mon[100755]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:40:00.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:40:00 smithi026 ceph-mon[95839]: osdmap e41: 6 total, 6 up, 6 in 2023-09-23T20:40:00.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:40:00 smithi026 ceph-mon[95839]: overall HEALTH_OK 2023-09-23T20:40:00.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:40:00 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-09-23T20:40:00.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:40:00 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-09-23T20:40:00.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:40:00 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:40:00.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:40:00 smithi026 ceph-mon[109912]: osdmap e41: 6 total, 6 up, 6 in 2023-09-23T20:40:00.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:40:00 smithi026 ceph-mon[109912]: overall HEALTH_OK 2023-09-23T20:40:00.883 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:40:00 smithi026 ceph-mon[109912]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-09-23T20:40:00.883 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:40:00 smithi026 ceph-mon[109912]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-09-23T20:40:00.883 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:40:00 smithi026 ceph-mon[109912]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:40:01.809 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:40:01 smithi070 ceph-mon[100755]: from='client.24259 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi070:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T20:40:01.809 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:40:01 smithi070 ceph-mon[100755]: pgmap v124: 1 pgs: 1 remapped+peering; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2023-09-23T20:40:01.810 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:40:01 smithi070 ceph-mon[100755]: osdmap e42: 6 total, 6 up, 6 in 2023-09-23T20:40:01.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:40:01 smithi026 ceph-mon[95839]: from='client.24259 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi070:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T20:40:01.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:40:01 smithi026 ceph-mon[95839]: pgmap v124: 1 pgs: 1 remapped+peering; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2023-09-23T20:40:01.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:40:01 smithi026 ceph-mon[95839]: osdmap e42: 6 total, 6 up, 6 in 2023-09-23T20:40:01.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:40:01 smithi026 ceph-mon[109912]: from='client.24259 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi070:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T20:40:01.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:40:01 smithi026 ceph-mon[109912]: pgmap v124: 1 pgs: 1 remapped+peering; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2023-09-23T20:40:01.883 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:40:01 smithi026 ceph-mon[109912]: osdmap e42: 6 total, 6 up, 6 in 2023-09-23T20:40:02.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:40:02 smithi070 ceph-mon[100755]: osdmap e43: 6 total, 6 up, 6 in 2023-09-23T20:40:02.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:40:02 smithi070 ceph-mon[100755]: from='client.? 172.21.15.70:0/454950753' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "6f3be784-7149-4f13-81b5-98924e97c056"}]: dispatch 2023-09-23T20:40:02.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:40:02 smithi070 ceph-mon[100755]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "6f3be784-7149-4f13-81b5-98924e97c056"}]: dispatch 2023-09-23T20:40:02.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:40:02 smithi070 ceph-mon[100755]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "6f3be784-7149-4f13-81b5-98924e97c056"}]': finished 2023-09-23T20:40:02.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:40:02 smithi070 ceph-mon[100755]: osdmap e44: 7 total, 6 up, 7 in 2023-09-23T20:40:02.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:40:02 smithi070 ceph-mon[100755]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-09-23T20:40:02.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:40:02 smithi026 ceph-mon[109912]: osdmap e43: 6 total, 6 up, 6 in 2023-09-23T20:40:02.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:40:02 smithi026 ceph-mon[109912]: from='client.? 172.21.15.70:0/454950753' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "6f3be784-7149-4f13-81b5-98924e97c056"}]: dispatch 2023-09-23T20:40:02.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:40:02 smithi026 ceph-mon[109912]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "6f3be784-7149-4f13-81b5-98924e97c056"}]: dispatch 2023-09-23T20:40:02.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:40:02 smithi026 ceph-mon[109912]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "6f3be784-7149-4f13-81b5-98924e97c056"}]': finished 2023-09-23T20:40:02.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:40:02 smithi026 ceph-mon[109912]: osdmap e44: 7 total, 6 up, 7 in 2023-09-23T20:40:02.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:40:02 smithi026 ceph-mon[109912]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-09-23T20:40:02.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:40:02 smithi026 ceph-mon[95839]: osdmap e43: 6 total, 6 up, 6 in 2023-09-23T20:40:02.883 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:40:02 smithi026 ceph-mon[95839]: from='client.? 172.21.15.70:0/454950753' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "6f3be784-7149-4f13-81b5-98924e97c056"}]: dispatch 2023-09-23T20:40:02.883 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:40:02 smithi026 ceph-mon[95839]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "6f3be784-7149-4f13-81b5-98924e97c056"}]: dispatch 2023-09-23T20:40:02.883 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:40:02 smithi026 ceph-mon[95839]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "6f3be784-7149-4f13-81b5-98924e97c056"}]': finished 2023-09-23T20:40:02.883 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:40:02 smithi026 ceph-mon[95839]: osdmap e44: 7 total, 6 up, 7 in 2023-09-23T20:40:02.883 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:40:02 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-09-23T20:40:03.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:40:03 smithi070 ceph-mon[100755]: pgmap v127: 1 pgs: 1 remapped+peering; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail 2023-09-23T20:40:03.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:40:03 smithi070 ceph-mon[100755]: from='client.? 172.21.15.70:0/1709916710' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-09-23T20:40:03.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:40:03 smithi026 ceph-mon[95839]: pgmap v127: 1 pgs: 1 remapped+peering; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail 2023-09-23T20:40:03.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:40:03 smithi026 ceph-mon[95839]: from='client.? 172.21.15.70:0/1709916710' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-09-23T20:40:03.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:40:03 smithi026 ceph-mon[109912]: pgmap v127: 1 pgs: 1 remapped+peering; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail 2023-09-23T20:40:03.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:40:03 smithi026 ceph-mon[109912]: from='client.? 172.21.15.70:0/1709916710' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-09-23T20:40:05.807 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:40:05 smithi070 ceph-mon[100755]: pgmap v128: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 115 KiB/s, 0 objects/s recovering 2023-09-23T20:40:05.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:40:05 smithi026 ceph-mon[109912]: pgmap v128: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 115 KiB/s, 0 objects/s recovering 2023-09-23T20:40:05.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:40:05 smithi026 ceph-mon[95839]: pgmap v128: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 115 KiB/s, 0 objects/s recovering 2023-09-23T20:40:07.865 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:40:07 smithi070 ceph-mon[100755]: pgmap v129: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 96 KiB/s, 0 objects/s recovering 2023-09-23T20:40:07.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:40:07 smithi026 ceph-mon[95839]: pgmap v129: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 96 KiB/s, 0 objects/s recovering 2023-09-23T20:40:07.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:40:07 smithi026 ceph-mon[109912]: pgmap v129: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 96 KiB/s, 0 objects/s recovering 2023-09-23T20:40:08.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:40:08 smithi070 ceph-mon[100755]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2023-09-23T20:40:08.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:40:08 smithi070 ceph-mon[100755]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:40:08.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:40:08 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2023-09-23T20:40:08.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:40:08 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:40:08.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:40:08 smithi026 ceph-mon[109912]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2023-09-23T20:40:08.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:40:08 smithi026 ceph-mon[109912]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:40:09.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:40:09 smithi070 ceph-mon[100755]: Deploying daemon osd.6 on smithi070 2023-09-23T20:40:09.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:40:09 smithi070 ceph-mon[100755]: pgmap v130: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 72 KiB/s, 0 objects/s recovering 2023-09-23T20:40:09.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:40:09 smithi026 ceph-mon[95839]: Deploying daemon osd.6 on smithi070 2023-09-23T20:40:09.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:40:09 smithi026 ceph-mon[95839]: pgmap v130: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 72 KiB/s, 0 objects/s recovering 2023-09-23T20:40:09.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:40:09 smithi026 ceph-mon[109912]: Deploying daemon osd.6 on smithi070 2023-09-23T20:40:09.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:40:09 smithi026 ceph-mon[109912]: pgmap v130: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 72 KiB/s, 0 objects/s recovering 2023-09-23T20:40:11.827 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:40:11 smithi070 ceph-mon[100755]: pgmap v131: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 64 KiB/s, 0 objects/s recovering 2023-09-23T20:40:11.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:40:11 smithi026 ceph-mon[95839]: pgmap v131: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 64 KiB/s, 0 objects/s recovering 2023-09-23T20:40:11.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:40:11 smithi026 ceph-mon[109912]: pgmap v131: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 64 KiB/s, 0 objects/s recovering 2023-09-23T20:40:13.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:40:13 smithi070 ceph-mon[100755]: pgmap v132: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 58 KiB/s, 0 objects/s recovering 2023-09-23T20:40:13.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:40:13 smithi070 ceph-mon[100755]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:40:13.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:40:13 smithi070 ceph-mon[100755]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:40:13.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:40:13 smithi070 ceph-mon[100755]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:40:13.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:40:13 smithi026 ceph-mon[95839]: pgmap v132: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 58 KiB/s, 0 objects/s recovering 2023-09-23T20:40:13.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:40:13 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:40:13.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:40:13 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:40:13.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:40:13 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:40:13.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:40:13 smithi026 ceph-mon[109912]: pgmap v132: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 58 KiB/s, 0 objects/s recovering 2023-09-23T20:40:13.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:40:13 smithi026 ceph-mon[109912]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:40:13.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:40:13 smithi026 ceph-mon[109912]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:40:13.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:40:13 smithi026 ceph-mon[109912]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:40:15.709 INFO:teuthology.orchestra.run.smithi070.stdout:Created osd(s) 6 on host 'smithi070' 2023-09-23T20:40:15.843 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:40:15 smithi070 ceph-mon[100755]: pgmap v133: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 48 KiB/s, 0 objects/s recovering 2023-09-23T20:40:15.843 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:40:15 smithi070 ceph-mon[100755]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:40:15.843 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:40:15 smithi070 ceph-mon[100755]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:40:15.843 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:40:15 smithi070 ceph-mon[100755]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:40:15.843 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:40:15 smithi070 ceph-mon[100755]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:40:15.843 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:40:15 smithi070 ceph-mon[100755]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:40:15.881 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:40:15 smithi026 ceph-mon[109912]: pgmap v133: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 48 KiB/s, 0 objects/s recovering 2023-09-23T20:40:15.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:40:15 smithi026 ceph-mon[109912]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:40:15.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:40:15 smithi026 ceph-mon[109912]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:40:15.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:40:15 smithi026 ceph-mon[109912]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:40:15.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:40:15 smithi026 ceph-mon[109912]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:40:15.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:40:15 smithi026 ceph-mon[109912]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:40:15.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:40:15 smithi026 ceph-mon[95839]: pgmap v133: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 48 KiB/s, 0 objects/s recovering 2023-09-23T20:40:15.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:40:15 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:40:15.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:40:15 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:40:15.883 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:40:15 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:40:15.883 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:40:15 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:40:15.883 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:40:15 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:40:16.066 DEBUG:teuthology.orchestra.run.smithi070:osd.6> sudo journalctl -f -n 0 -u ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7@osd.6.service 2023-09-23T20:40:16.068 INFO:tasks.cephadm:Deploying osd.7 on smithi070 with /dev/vg_nvme/lv_1... 2023-09-23T20:40:16.068 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph-volume lvm zap /dev/vg_nvme/lv_1 2023-09-23T20:40:16.097 INFO:journalctl@ceph.osd.6.smithi070.stdout:-- Logs begin at Sat 2023-09-23 20:21:16 UTC. -- 2023-09-23T20:40:16.814 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:40:16.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:40:16 smithi070 ceph-mon[100755]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:40:16.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:40:16 smithi070 ceph-mon[100755]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:40:16.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:40:16 smithi070 ceph-mon[100755]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:40:16.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:40:16 smithi070 ceph-mon[100755]: from='osd.6 [v2:172.21.15.70:6816/2944175278,v1:172.21.15.70:6817/2944175278]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2023-09-23T20:40:16.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:40:16 smithi070 ceph-mon[100755]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2023-09-23T20:40:16.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:40:16 smithi070 ceph-mon[100755]: pgmap v134: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2023-09-23T20:40:17.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:40:16 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:40:17.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:40:16 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:40:17.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:40:16 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:40:17.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:40:16 smithi026 ceph-mon[95839]: from='osd.6 [v2:172.21.15.70:6816/2944175278,v1:172.21.15.70:6817/2944175278]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2023-09-23T20:40:17.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:40:16 smithi026 ceph-mon[95839]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2023-09-23T20:40:17.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:40:16 smithi026 ceph-mon[95839]: pgmap v134: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2023-09-23T20:40:17.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:40:16 smithi026 ceph-mon[109912]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:40:17.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:40:16 smithi026 ceph-mon[109912]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:40:17.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:40:16 smithi026 ceph-mon[109912]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:40:17.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:40:16 smithi026 ceph-mon[109912]: from='osd.6 [v2:172.21.15.70:6816/2944175278,v1:172.21.15.70:6817/2944175278]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2023-09-23T20:40:17.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:40:16 smithi026 ceph-mon[109912]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2023-09-23T20:40:17.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:40:16 smithi026 ceph-mon[109912]: pgmap v134: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2023-09-23T20:40:17.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:40:17 smithi070 ceph-mon[100755]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2023-09-23T20:40:17.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:40:17 smithi070 ceph-mon[100755]: osdmap e45: 7 total, 6 up, 7 in 2023-09-23T20:40:17.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:40:17 smithi070 ceph-mon[100755]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-09-23T20:40:17.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:40:17 smithi070 ceph-mon[100755]: from='osd.6 [v2:172.21.15.70:6816/2944175278,v1:172.21.15.70:6817/2944175278]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi070", "root=default"]}]: dispatch 2023-09-23T20:40:17.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:40:17 smithi070 ceph-mon[100755]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi070", "root=default"]}]: dispatch 2023-09-23T20:40:17.870 INFO:journalctl@ceph.osd.6.smithi070.stdout:Sep 23 20:40:17 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd-6[121137]: 2023-09-23T20:40:17.690+0000 7ffa9d95a700 -1 osd.6 0 waiting for initial osdmap 2023-09-23T20:40:17.870 INFO:journalctl@ceph.osd.6.smithi070.stdout:Sep 23 20:40:17 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd-6[121137]: 2023-09-23T20:40:17.698+0000 7ffa99757700 -1 osd.6 46 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-09-23T20:40:18.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:40:17 smithi026 ceph-mon[95839]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2023-09-23T20:40:18.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:40:17 smithi026 ceph-mon[95839]: osdmap e45: 7 total, 6 up, 7 in 2023-09-23T20:40:18.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:40:17 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-09-23T20:40:18.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:40:17 smithi026 ceph-mon[95839]: from='osd.6 [v2:172.21.15.70:6816/2944175278,v1:172.21.15.70:6817/2944175278]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi070", "root=default"]}]: dispatch 2023-09-23T20:40:18.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:40:17 smithi026 ceph-mon[95839]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi070", "root=default"]}]: dispatch 2023-09-23T20:40:18.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:40:17 smithi026 ceph-mon[109912]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2023-09-23T20:40:18.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:40:17 smithi026 ceph-mon[109912]: osdmap e45: 7 total, 6 up, 7 in 2023-09-23T20:40:18.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:40:17 smithi026 ceph-mon[109912]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-09-23T20:40:18.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:40:17 smithi026 ceph-mon[109912]: from='osd.6 [v2:172.21.15.70:6816/2944175278,v1:172.21.15.70:6817/2944175278]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi070", "root=default"]}]: dispatch 2023-09-23T20:40:18.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:40:17 smithi026 ceph-mon[109912]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi070", "root=default"]}]: dispatch 2023-09-23T20:40:18.835 INFO:teuthology.orchestra.run.smithi070.stderr:--> Zapping: /dev/vg_nvme/lv_1 2023-09-23T20:40:18.835 INFO:teuthology.orchestra.run.smithi070.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_1 bs=1M count=10 conv=fsync 2023-09-23T20:40:18.835 INFO:teuthology.orchestra.run.smithi070.stderr: stderr: 10+0 records in 2023-09-23T20:40:18.836 INFO:teuthology.orchestra.run.smithi070.stderr:10+0 records out 2023-09-23T20:40:18.836 INFO:teuthology.orchestra.run.smithi070.stderr:10485760 bytes (10 MB, 10 MiB) copied, 0.00812724 s, 1.3 GB/s 2023-09-23T20:40:18.836 INFO:teuthology.orchestra.run.smithi070.stderr: stderr: 2023-09-23T20:40:18.836 INFO:teuthology.orchestra.run.smithi070.stderr:--> Zapping successful for: 2023-09-23T20:40:19.034 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:40:18 smithi070 ceph-mon[100755]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi070", "root=default"]}]': finished 2023-09-23T20:40:19.034 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:40:18 smithi070 ceph-mon[100755]: osdmap e46: 7 total, 6 up, 7 in 2023-09-23T20:40:19.034 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:40:18 smithi070 ceph-mon[100755]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-09-23T20:40:19.034 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:40:18 smithi070 ceph-mon[100755]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-09-23T20:40:19.035 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:40:18 smithi070 ceph-mon[100755]: osd.6 [v2:172.21.15.70:6816/2944175278,v1:172.21.15.70:6817/2944175278] boot 2023-09-23T20:40:19.035 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:40:18 smithi070 ceph-mon[100755]: osdmap e47: 7 total, 7 up, 7 in 2023-09-23T20:40:19.035 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:40:18 smithi070 ceph-mon[100755]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-09-23T20:40:19.035 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:40:18 smithi070 ceph-mon[100755]: pgmap v138: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2023-09-23T20:40:19.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:40:18 smithi026 ceph-mon[95839]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi070", "root=default"]}]': finished 2023-09-23T20:40:19.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:40:18 smithi026 ceph-mon[95839]: osdmap e46: 7 total, 6 up, 7 in 2023-09-23T20:40:19.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:40:18 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-09-23T20:40:19.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:40:18 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-09-23T20:40:19.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:40:18 smithi026 ceph-mon[95839]: osd.6 [v2:172.21.15.70:6816/2944175278,v1:172.21.15.70:6817/2944175278] boot 2023-09-23T20:40:19.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:40:18 smithi026 ceph-mon[95839]: osdmap e47: 7 total, 7 up, 7 in 2023-09-23T20:40:19.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:40:18 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-09-23T20:40:19.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:40:18 smithi026 ceph-mon[95839]: pgmap v138: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2023-09-23T20:40:19.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:40:18 smithi026 ceph-mon[109912]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi070", "root=default"]}]': finished 2023-09-23T20:40:19.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:40:18 smithi026 ceph-mon[109912]: osdmap e46: 7 total, 6 up, 7 in 2023-09-23T20:40:19.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:40:18 smithi026 ceph-mon[109912]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-09-23T20:40:19.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:40:18 smithi026 ceph-mon[109912]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-09-23T20:40:19.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:40:18 smithi026 ceph-mon[109912]: osd.6 [v2:172.21.15.70:6816/2944175278,v1:172.21.15.70:6817/2944175278] boot 2023-09-23T20:40:19.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:40:18 smithi026 ceph-mon[109912]: osdmap e47: 7 total, 7 up, 7 in 2023-09-23T20:40:19.134 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:40:18 smithi026 ceph-mon[109912]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-09-23T20:40:19.134 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:40:18 smithi026 ceph-mon[109912]: pgmap v138: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2023-09-23T20:40:19.289 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph orch daemon add osd smithi070:vg_nvme/lv_1 2023-09-23T20:40:19.884 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:40:20.081 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:40:19 smithi070 ceph-mon[100755]: purged_snaps scrub starts 2023-09-23T20:40:20.081 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:40:19 smithi070 ceph-mon[100755]: purged_snaps scrub ok 2023-09-23T20:40:20.081 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:40:19 smithi070 ceph-mon[100755]: Detected new or changed devices on smithi070 2023-09-23T20:40:20.081 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:40:19 smithi070 ceph-mon[100755]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:40:20.081 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:40:19 smithi070 ceph-mon[100755]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:40:20.081 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:40:19 smithi070 ceph-mon[100755]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:40:20.081 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:40:19 smithi070 ceph-mon[100755]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:40:20.082 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:40:19 smithi070 ceph-mon[100755]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:40:20.082 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:40:19 smithi070 ceph-mon[100755]: Adjusting osd_memory_target on smithi070 to 5722M 2023-09-23T20:40:20.082 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:40:19 smithi070 ceph-mon[100755]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:40:20.082 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:40:19 smithi070 ceph-mon[100755]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:40:20.082 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:40:19 smithi070 ceph-mon[100755]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:40:20.082 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:40:19 smithi070 ceph-mon[100755]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:40:20.082 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:40:19 smithi070 ceph-mon[100755]: osdmap e48: 7 total, 7 up, 7 in 2023-09-23T20:40:20.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:40:19 smithi026 ceph-mon[109912]: purged_snaps scrub starts 2023-09-23T20:40:20.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:40:19 smithi026 ceph-mon[109912]: purged_snaps scrub ok 2023-09-23T20:40:20.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:40:19 smithi026 ceph-mon[109912]: Detected new or changed devices on smithi070 2023-09-23T20:40:20.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:40:19 smithi026 ceph-mon[109912]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:40:20.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:40:19 smithi026 ceph-mon[109912]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:40:20.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:40:19 smithi026 ceph-mon[109912]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:40:20.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:40:19 smithi026 ceph-mon[109912]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:40:20.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:40:19 smithi026 ceph-mon[109912]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:40:20.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:40:19 smithi026 ceph-mon[109912]: Adjusting osd_memory_target on smithi070 to 5722M 2023-09-23T20:40:20.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:40:19 smithi026 ceph-mon[109912]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:40:20.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:40:19 smithi026 ceph-mon[109912]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:40:20.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:40:19 smithi026 ceph-mon[109912]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:40:20.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:40:19 smithi026 ceph-mon[109912]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:40:20.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:40:19 smithi026 ceph-mon[109912]: osdmap e48: 7 total, 7 up, 7 in 2023-09-23T20:40:20.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:40:19 smithi026 ceph-mon[95839]: purged_snaps scrub starts 2023-09-23T20:40:20.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:40:19 smithi026 ceph-mon[95839]: purged_snaps scrub ok 2023-09-23T20:40:20.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:40:19 smithi026 ceph-mon[95839]: Detected new or changed devices on smithi070 2023-09-23T20:40:20.134 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:40:19 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:40:20.134 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:40:19 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:40:20.134 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:40:19 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:40:20.134 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:40:19 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:40:20.134 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:40:19 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:40:20.134 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:40:19 smithi026 ceph-mon[95839]: Adjusting osd_memory_target on smithi070 to 5722M 2023-09-23T20:40:20.134 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:40:19 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:40:20.134 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:40:19 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:40:20.134 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:40:19 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:40:20.134 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:40:19 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:40:20.134 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:40:19 smithi026 ceph-mon[95839]: osdmap e48: 7 total, 7 up, 7 in 2023-09-23T20:40:21.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:40:21 smithi070 ceph-mon[100755]: pgmap v141: 1 pgs: 1 remapped+peering; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2023-09-23T20:40:21.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:40:21 smithi070 ceph-mon[100755]: osdmap e49: 7 total, 7 up, 7 in 2023-09-23T20:40:21.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:40:21 smithi070 ceph-mon[100755]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-09-23T20:40:21.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:40:21 smithi070 ceph-mon[100755]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-09-23T20:40:21.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:40:21 smithi070 ceph-mon[100755]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:40:21.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:40:21 smithi026 ceph-mon[95839]: pgmap v141: 1 pgs: 1 remapped+peering; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2023-09-23T20:40:21.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:40:21 smithi026 ceph-mon[95839]: osdmap e49: 7 total, 7 up, 7 in 2023-09-23T20:40:21.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:40:21 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-09-23T20:40:21.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:40:21 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-09-23T20:40:21.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:40:21 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:40:21.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:40:21 smithi026 ceph-mon[109912]: pgmap v141: 1 pgs: 1 remapped+peering; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2023-09-23T20:40:21.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:40:21 smithi026 ceph-mon[109912]: osdmap e49: 7 total, 7 up, 7 in 2023-09-23T20:40:21.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:40:21 smithi026 ceph-mon[109912]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-09-23T20:40:21.883 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:40:21 smithi026 ceph-mon[109912]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-09-23T20:40:21.883 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:40:21 smithi026 ceph-mon[109912]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:40:22.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:40:22 smithi070 ceph-mon[100755]: from='client.24286 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi070:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T20:40:22.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:40:22 smithi070 ceph-mon[100755]: osdmap e50: 7 total, 7 up, 7 in 2023-09-23T20:40:22.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:40:22 smithi026 ceph-mon[95839]: from='client.24286 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi070:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T20:40:22.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:40:22 smithi026 ceph-mon[95839]: osdmap e50: 7 total, 7 up, 7 in 2023-09-23T20:40:22.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:40:22 smithi026 ceph-mon[109912]: from='client.24286 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi070:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T20:40:22.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:40:22 smithi026 ceph-mon[109912]: osdmap e50: 7 total, 7 up, 7 in 2023-09-23T20:40:23.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:40:23 smithi070 ceph-mon[100755]: pgmap v143: 1 pgs: 1 remapped+peering; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2023-09-23T20:40:23.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:40:23 smithi070 ceph-mon[100755]: osdmap e51: 7 total, 7 up, 7 in 2023-09-23T20:40:23.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:40:23 smithi026 ceph-mon[95839]: pgmap v143: 1 pgs: 1 remapped+peering; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2023-09-23T20:40:23.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:40:23 smithi026 ceph-mon[95839]: osdmap e51: 7 total, 7 up, 7 in 2023-09-23T20:40:23.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:40:23 smithi026 ceph-mon[109912]: pgmap v143: 1 pgs: 1 remapped+peering; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2023-09-23T20:40:23.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:40:23 smithi026 ceph-mon[109912]: osdmap e51: 7 total, 7 up, 7 in 2023-09-23T20:40:24.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:40:24 smithi070 ceph-mon[100755]: from='client.? 172.21.15.70:0/522408141' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "499f5f85-f4b2-40a7-ac0f-79cd22738d30"}]: dispatch 2023-09-23T20:40:24.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:40:24 smithi070 ceph-mon[100755]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "499f5f85-f4b2-40a7-ac0f-79cd22738d30"}]: dispatch 2023-09-23T20:40:24.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:40:24 smithi070 ceph-mon[100755]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "499f5f85-f4b2-40a7-ac0f-79cd22738d30"}]': finished 2023-09-23T20:40:24.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:40:24 smithi070 ceph-mon[100755]: osdmap e52: 8 total, 7 up, 8 in 2023-09-23T20:40:24.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:40:24 smithi070 ceph-mon[100755]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-09-23T20:40:24.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:40:24 smithi070 ceph-mon[100755]: from='client.? 172.21.15.70:0/3920379822' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-09-23T20:40:24.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:40:24 smithi026 ceph-mon[95839]: from='client.? 172.21.15.70:0/522408141' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "499f5f85-f4b2-40a7-ac0f-79cd22738d30"}]: dispatch 2023-09-23T20:40:24.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:40:24 smithi026 ceph-mon[95839]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "499f5f85-f4b2-40a7-ac0f-79cd22738d30"}]: dispatch 2023-09-23T20:40:24.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:40:24 smithi026 ceph-mon[95839]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "499f5f85-f4b2-40a7-ac0f-79cd22738d30"}]': finished 2023-09-23T20:40:24.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:40:24 smithi026 ceph-mon[95839]: osdmap e52: 8 total, 7 up, 8 in 2023-09-23T20:40:24.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:40:24 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-09-23T20:40:24.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:40:24 smithi026 ceph-mon[95839]: from='client.? 172.21.15.70:0/3920379822' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-09-23T20:40:24.883 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:40:24 smithi026 ceph-mon[109912]: from='client.? 172.21.15.70:0/522408141' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "499f5f85-f4b2-40a7-ac0f-79cd22738d30"}]: dispatch 2023-09-23T20:40:24.883 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:40:24 smithi026 ceph-mon[109912]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "499f5f85-f4b2-40a7-ac0f-79cd22738d30"}]: dispatch 2023-09-23T20:40:24.883 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:40:24 smithi026 ceph-mon[109912]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "499f5f85-f4b2-40a7-ac0f-79cd22738d30"}]': finished 2023-09-23T20:40:24.883 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:40:24 smithi026 ceph-mon[109912]: osdmap e52: 8 total, 7 up, 8 in 2023-09-23T20:40:24.883 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:40:24 smithi026 ceph-mon[109912]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-09-23T20:40:24.883 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:40:24 smithi026 ceph-mon[109912]: from='client.? 172.21.15.70:0/3920379822' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-09-23T20:40:25.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:40:25 smithi070 ceph-mon[100755]: pgmap v146: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 144 KiB/s, 0 objects/s recovering 2023-09-23T20:40:25.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:40:25 smithi026 ceph-mon[95839]: pgmap v146: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 144 KiB/s, 0 objects/s recovering 2023-09-23T20:40:25.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:40:25 smithi026 ceph-mon[109912]: pgmap v146: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 144 KiB/s, 0 objects/s recovering 2023-09-23T20:40:27.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:40:27 smithi070 ceph-mon[100755]: pgmap v147: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail; 96 KiB/s, 0 objects/s recovering 2023-09-23T20:40:27.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:40:27 smithi026 ceph-mon[95839]: pgmap v147: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail; 96 KiB/s, 0 objects/s recovering 2023-09-23T20:40:27.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:40:27 smithi026 ceph-mon[109912]: pgmap v147: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail; 96 KiB/s, 0 objects/s recovering 2023-09-23T20:40:29.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:40:29 smithi070 ceph-mon[100755]: pgmap v148: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail; 82 KiB/s, 0 objects/s recovering 2023-09-23T20:40:29.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:40:29 smithi070 ceph-mon[100755]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2023-09-23T20:40:29.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:40:29 smithi070 ceph-mon[100755]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:40:29.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:40:29 smithi026 ceph-mon[95839]: pgmap v148: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail; 82 KiB/s, 0 objects/s recovering 2023-09-23T20:40:29.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:40:29 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2023-09-23T20:40:29.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:40:29 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:40:29.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:40:29 smithi026 ceph-mon[109912]: pgmap v148: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail; 82 KiB/s, 0 objects/s recovering 2023-09-23T20:40:29.883 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:40:29 smithi026 ceph-mon[109912]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2023-09-23T20:40:29.883 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:40:29 smithi026 ceph-mon[109912]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:40:30.829 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:40:30 smithi070 ceph-mon[100755]: Deploying daemon osd.7 on smithi070 2023-09-23T20:40:30.881 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:40:30 smithi026 ceph-mon[95839]: Deploying daemon osd.7 on smithi070 2023-09-23T20:40:30.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:40:30 smithi026 ceph-mon[109912]: Deploying daemon osd.7 on smithi070 2023-09-23T20:40:31.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:40:31 smithi070 ceph-mon[100755]: pgmap v149: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail; 72 KiB/s, 0 objects/s recovering 2023-09-23T20:40:31.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:40:31 smithi026 ceph-mon[95839]: pgmap v149: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail; 72 KiB/s, 0 objects/s recovering 2023-09-23T20:40:31.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:40:31 smithi026 ceph-mon[109912]: pgmap v149: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail; 72 KiB/s, 0 objects/s recovering 2023-09-23T20:40:33.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:40:33 smithi070 ceph-mon[100755]: pgmap v150: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail; 58 KiB/s, 0 objects/s recovering 2023-09-23T20:40:33.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:40:33 smithi070 ceph-mon[100755]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:40:33.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:40:33 smithi070 ceph-mon[100755]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:40:33.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:40:33 smithi070 ceph-mon[100755]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:40:33.881 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:40:33 smithi026 ceph-mon[109912]: pgmap v150: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail; 58 KiB/s, 0 objects/s recovering 2023-09-23T20:40:33.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:40:33 smithi026 ceph-mon[109912]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:40:33.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:40:33 smithi026 ceph-mon[109912]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:40:33.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:40:33 smithi026 ceph-mon[109912]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:40:33.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:40:33 smithi026 ceph-mon[95839]: pgmap v150: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail; 58 KiB/s, 0 objects/s recovering 2023-09-23T20:40:33.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:40:33 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:40:33.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:40:33 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:40:33.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:40:33 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:40:35.854 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:40:35 smithi070 ceph-mon[100755]: pgmap v151: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail; 53 KiB/s, 0 objects/s recovering 2023-09-23T20:40:35.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:40:35 smithi026 ceph-mon[95839]: pgmap v151: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail; 53 KiB/s, 0 objects/s recovering 2023-09-23T20:40:35.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:40:35 smithi026 ceph-mon[109912]: pgmap v151: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail; 53 KiB/s, 0 objects/s recovering 2023-09-23T20:40:36.105 INFO:teuthology.orchestra.run.smithi070.stdout:Created osd(s) 7 on host 'smithi070' 2023-09-23T20:40:36.556 DEBUG:teuthology.orchestra.run.smithi070:osd.7> sudo journalctl -f -n 0 -u ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7@osd.7.service 2023-09-23T20:40:36.558 INFO:tasks.cephadm:Waiting for 8 OSDs to come up... 2023-09-23T20:40:36.558 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph osd stat -f json 2023-09-23T20:40:36.588 INFO:journalctl@ceph.osd.7.smithi070.stdout:-- Logs begin at Sat 2023-09-23 20:21:16 UTC. -- 2023-09-23T20:40:37.079 INFO:journalctl@ceph.osd.7.smithi070.stdout:Sep 23 20:40:36 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd-7[128949]: 2023-09-23T20:40:36.783+0000 7f121f7ca540 -1 osd.7 0 log_to_monitors true 2023-09-23T20:40:37.177 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:40:37.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:40:37 smithi070 ceph-mon[100755]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:40:37.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:40:37 smithi070 ceph-mon[100755]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:40:37.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:40:37 smithi070 ceph-mon[100755]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:40:37.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:40:37 smithi070 ceph-mon[100755]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:40:37.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:40:37 smithi070 ceph-mon[100755]: pgmap v152: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail 2023-09-23T20:40:37.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:40:37 smithi070 ceph-mon[100755]: from='osd.7 [v2:172.21.15.70:6824/2824974165,v1:172.21.15.70:6825/2824974165]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2023-09-23T20:40:37.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:40:37 smithi026 ceph-mon[109912]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:40:37.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:40:37 smithi026 ceph-mon[109912]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:40:37.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:40:37 smithi026 ceph-mon[109912]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:40:37.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:40:37 smithi026 ceph-mon[109912]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:40:37.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:40:37 smithi026 ceph-mon[109912]: pgmap v152: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail 2023-09-23T20:40:37.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:40:37 smithi026 ceph-mon[109912]: from='osd.7 [v2:172.21.15.70:6824/2824974165,v1:172.21.15.70:6825/2824974165]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2023-09-23T20:40:37.383 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:40:37 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:40:37.383 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:40:37 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:40:37.383 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:40:37 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:40:37.383 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:40:37 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:40:37.383 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:40:37 smithi026 ceph-mon[95839]: pgmap v152: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail 2023-09-23T20:40:37.383 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:40:37 smithi026 ceph-mon[95839]: from='osd.7 [v2:172.21.15.70:6824/2824974165,v1:172.21.15.70:6825/2824974165]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2023-09-23T20:40:38.435 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:40:38 smithi026 ceph-mon[95839]: from='osd.7 [v2:172.21.15.70:6824/2824974165,v1:172.21.15.70:6825/2824974165]' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2023-09-23T20:40:38.435 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:40:38 smithi026 ceph-mon[95839]: osdmap e53: 8 total, 7 up, 8 in 2023-09-23T20:40:38.435 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:40:38 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-09-23T20:40:38.436 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:40:38 smithi026 ceph-mon[95839]: from='osd.7 [v2:172.21.15.70:6824/2824974165,v1:172.21.15.70:6825/2824974165]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi070", "root=default"]}]: dispatch 2023-09-23T20:40:38.436 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:40:38 smithi026 ceph-mon[109912]: from='osd.7 [v2:172.21.15.70:6824/2824974165,v1:172.21.15.70:6825/2824974165]' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2023-09-23T20:40:38.436 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:40:38 smithi026 ceph-mon[109912]: osdmap e53: 8 total, 7 up, 8 in 2023-09-23T20:40:38.436 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:40:38 smithi026 ceph-mon[109912]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-09-23T20:40:38.436 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:40:38 smithi026 ceph-mon[109912]: from='osd.7 [v2:172.21.15.70:6824/2824974165,v1:172.21.15.70:6825/2824974165]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi070", "root=default"]}]: dispatch 2023-09-23T20:40:38.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:40:38 smithi070 ceph-mon[100755]: from='osd.7 [v2:172.21.15.70:6824/2824974165,v1:172.21.15.70:6825/2824974165]' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2023-09-23T20:40:38.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:40:38 smithi070 ceph-mon[100755]: osdmap e53: 8 total, 7 up, 8 in 2023-09-23T20:40:38.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:40:38 smithi070 ceph-mon[100755]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-09-23T20:40:38.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:40:38 smithi070 ceph-mon[100755]: from='osd.7 [v2:172.21.15.70:6824/2824974165,v1:172.21.15.70:6825/2824974165]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi070", "root=default"]}]: dispatch 2023-09-23T20:40:38.621 INFO:journalctl@ceph.osd.7.smithi070.stdout:Sep 23 20:40:38 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd-7[128949]: 2023-09-23T20:40:38.168+0000 7f1215fb0700 -1 osd.7 0 waiting for initial osdmap 2023-09-23T20:40:38.621 INFO:journalctl@ceph.osd.7.smithi070.stdout:Sep 23 20:40:38 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd-7[128949]: 2023-09-23T20:40:38.175+0000 7f120eda7700 -1 osd.7 54 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-09-23T20:40:38.816 INFO:teuthology.orchestra.run.smithi026.stdout: 2023-09-23T20:40:39.232 INFO:teuthology.orchestra.run.smithi026.stdout:{"epoch":55,"num_osds":8,"num_up_osds":8,"osd_up_since":1695501638,"num_in_osds":8,"osd_in_since":1695501623,"num_remapped_pgs":1} 2023-09-23T20:40:39.232 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph osd dump --format=json 2023-09-23T20:40:39.511 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:40:39 smithi070 ceph-mon[100755]: from='osd.7 [v2:172.21.15.70:6824/2824974165,v1:172.21.15.70:6825/2824974165]' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi070", "root=default"]}]': finished 2023-09-23T20:40:39.511 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:40:39 smithi070 ceph-mon[100755]: osdmap e54: 8 total, 7 up, 8 in 2023-09-23T20:40:39.511 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:40:39 smithi070 ceph-mon[100755]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-09-23T20:40:39.511 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:40:39 smithi070 ceph-mon[100755]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-09-23T20:40:39.512 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:40:39 smithi070 ceph-mon[100755]: osd.7 [v2:172.21.15.70:6824/2824974165,v1:172.21.15.70:6825/2824974165] boot 2023-09-23T20:40:39.512 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:40:39 smithi070 ceph-mon[100755]: osdmap e55: 8 total, 8 up, 8 in 2023-09-23T20:40:39.512 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:40:39 smithi070 ceph-mon[100755]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-09-23T20:40:39.512 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:40:39 smithi070 ceph-mon[100755]: pgmap v156: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail 2023-09-23T20:40:39.512 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:40:39 smithi070 ceph-mon[100755]: from='client.? 172.21.15.26:0/14086976' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2023-09-23T20:40:39.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:40:39 smithi026 ceph-mon[95839]: from='osd.7 [v2:172.21.15.70:6824/2824974165,v1:172.21.15.70:6825/2824974165]' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi070", "root=default"]}]': finished 2023-09-23T20:40:39.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:40:39 smithi026 ceph-mon[95839]: osdmap e54: 8 total, 7 up, 8 in 2023-09-23T20:40:39.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:40:39 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-09-23T20:40:39.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:40:39 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-09-23T20:40:39.633 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:40:39 smithi026 ceph-mon[95839]: osd.7 [v2:172.21.15.70:6824/2824974165,v1:172.21.15.70:6825/2824974165] boot 2023-09-23T20:40:39.633 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:40:39 smithi026 ceph-mon[95839]: osdmap e55: 8 total, 8 up, 8 in 2023-09-23T20:40:39.633 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:40:39 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-09-23T20:40:39.633 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:40:39 smithi026 ceph-mon[95839]: pgmap v156: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail 2023-09-23T20:40:39.633 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:40:39 smithi026 ceph-mon[95839]: from='client.? 172.21.15.26:0/14086976' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2023-09-23T20:40:39.633 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:40:39 smithi026 ceph-mon[109912]: from='osd.7 [v2:172.21.15.70:6824/2824974165,v1:172.21.15.70:6825/2824974165]' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi070", "root=default"]}]': finished 2023-09-23T20:40:39.633 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:40:39 smithi026 ceph-mon[109912]: osdmap e54: 8 total, 7 up, 8 in 2023-09-23T20:40:39.633 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:40:39 smithi026 ceph-mon[109912]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-09-23T20:40:39.633 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:40:39 smithi026 ceph-mon[109912]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-09-23T20:40:39.633 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:40:39 smithi026 ceph-mon[109912]: osd.7 [v2:172.21.15.70:6824/2824974165,v1:172.21.15.70:6825/2824974165] boot 2023-09-23T20:40:39.633 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:40:39 smithi026 ceph-mon[109912]: osdmap e55: 8 total, 8 up, 8 in 2023-09-23T20:40:39.634 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:40:39 smithi026 ceph-mon[109912]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-09-23T20:40:39.634 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:40:39 smithi026 ceph-mon[109912]: pgmap v156: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail 2023-09-23T20:40:39.634 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:40:39 smithi026 ceph-mon[109912]: from='client.? 172.21.15.26:0/14086976' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2023-09-23T20:40:39.839 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:40:40.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:40:40 smithi026 ceph-mon[95839]: purged_snaps scrub starts 2023-09-23T20:40:40.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:40:40 smithi026 ceph-mon[95839]: purged_snaps scrub ok 2023-09-23T20:40:40.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:40:40 smithi026 ceph-mon[95839]: osdmap e56: 8 total, 8 up, 8 in 2023-09-23T20:40:40.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:40:40 smithi026 ceph-mon[95839]: Detected new or changed devices on smithi070 2023-09-23T20:40:40.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:40:40 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:40:40.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:40:40 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:40:40.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:40:40 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:40:40.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:40:40 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:40:40.633 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:40:40 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:40:40.633 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:40:40 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:40:40.633 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:40:40 smithi026 ceph-mon[95839]: Adjusting osd_memory_target on smithi070 to 4292M 2023-09-23T20:40:40.633 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:40:40 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:40:40.633 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:40:40 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:40:40.633 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:40:40 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:40:40.633 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:40:40 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:40:40.633 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:40:40 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:40:40.633 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:40:40 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:40:40.633 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:40:40 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:40:40.633 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:40:40 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:40:40.634 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:40:40 smithi026 ceph-mon[109912]: purged_snaps scrub starts 2023-09-23T20:40:40.634 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:40:40 smithi026 ceph-mon[109912]: purged_snaps scrub ok 2023-09-23T20:40:40.634 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:40:40 smithi026 ceph-mon[109912]: osdmap e56: 8 total, 8 up, 8 in 2023-09-23T20:40:40.634 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:40:40 smithi026 ceph-mon[109912]: Detected new or changed devices on smithi070 2023-09-23T20:40:40.634 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:40:40 smithi026 ceph-mon[109912]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:40:40.634 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:40:40 smithi026 ceph-mon[109912]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:40:40.634 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:40:40 smithi026 ceph-mon[109912]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:40:40.634 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:40:40 smithi026 ceph-mon[109912]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:40:40.634 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:40:40 smithi026 ceph-mon[109912]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:40:40.634 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:40:40 smithi026 ceph-mon[109912]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:40:40.634 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:40:40 smithi026 ceph-mon[109912]: Adjusting osd_memory_target on smithi070 to 4292M 2023-09-23T20:40:40.634 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:40:40 smithi026 ceph-mon[109912]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:40:40.635 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:40:40 smithi026 ceph-mon[109912]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:40:40.635 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:40:40 smithi026 ceph-mon[109912]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:40:40.635 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:40:40 smithi026 ceph-mon[109912]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:40:40.635 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:40:40 smithi026 ceph-mon[109912]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:40:40.635 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:40:40 smithi026 ceph-mon[109912]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:40:40.635 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:40:40 smithi026 ceph-mon[109912]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:40:40.635 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:40:40 smithi026 ceph-mon[109912]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:40:40.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:40:40 smithi070 ceph-mon[100755]: purged_snaps scrub starts 2023-09-23T20:40:40.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:40:40 smithi070 ceph-mon[100755]: purged_snaps scrub ok 2023-09-23T20:40:40.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:40:40 smithi070 ceph-mon[100755]: osdmap e56: 8 total, 8 up, 8 in 2023-09-23T20:40:40.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:40:40 smithi070 ceph-mon[100755]: Detected new or changed devices on smithi070 2023-09-23T20:40:40.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:40:40 smithi070 ceph-mon[100755]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:40:40.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:40:40 smithi070 ceph-mon[100755]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:40:40.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:40:40 smithi070 ceph-mon[100755]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:40:40.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:40:40 smithi070 ceph-mon[100755]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:40:40.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:40:40 smithi070 ceph-mon[100755]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:40:40.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:40:40 smithi070 ceph-mon[100755]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:40:40.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:40:40 smithi070 ceph-mon[100755]: Adjusting osd_memory_target on smithi070 to 4292M 2023-09-23T20:40:40.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:40:40 smithi070 ceph-mon[100755]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:40:40.871 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:40:40 smithi070 ceph-mon[100755]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:40:40.871 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:40:40 smithi070 ceph-mon[100755]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:40:40.871 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:40:40 smithi070 ceph-mon[100755]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:40:40.871 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:40:40 smithi070 ceph-mon[100755]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:40:40.871 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:40:40 smithi070 ceph-mon[100755]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:40:40.871 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:40:40 smithi070 ceph-mon[100755]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:40:40.871 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:40:40 smithi070 ceph-mon[100755]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:40:41.212 INFO:teuthology.orchestra.run.smithi026.stdout: 2023-09-23T20:40:41.212 INFO:teuthology.orchestra.run.smithi026.stdout:{"epoch":57,"fsid":"b3f8e4ec-5a50-11ee-8db4-212e2dc638e7","created":"2023-09-23T20:35:58.919705+0000","modified":"2023-09-23T20:40:40.540477+0000","last_up_change":"2023-09-23T20:40:38.528428+0000","last_in_change":"2023-09-23T20:40:23.597125+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2023-09-23T20:38:57.022844+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}}],"osds":[{"osd":0,"uuid":"f837feb7-060e-4965-9b48-bed9f3672d05","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":56,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6802","nonce":2638118366},{"type":"v1","addr":"172.21.15.26:6803","nonce":2638118366}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6804","nonce":2638118366},{"type":"v1","addr":"172.21.15.26:6805","nonce":2638118366}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6808","nonce":2638118366},{"type":"v1","addr":"172.21.15.26:6809","nonce":2638118366}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6806","nonce":2638118366},{"type":"v1","addr":"172.21.15.26:6807","nonce":2638118366}]},"public_addr":"172.21.15.26:6803/2638118366","cluster_addr":"172.21.15.26:6805/2638118366","heartbeat_back_addr":"172.21.15.26:6809/2638118366","heartbeat_front_addr":"172.21.15.26:6807/2638118366","state":["exists","up"]},{"osd":1,"uuid":"917f65aa-5963-4576-988a-396698b7c6a8","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":34,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6810","nonce":3250108152},{"type":"v1","addr":"172.21.15.26:6811","nonce":3250108152}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6812","nonce":3250108152},{"type":"v1","addr":"172.21.15.26:6813","nonce":3250108152}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6816","nonce":3250108152},{"type":"v1","addr":"172.21.15.26:6817","nonce":3250108152}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6814","nonce":3250108152},{"type":"v1","addr":"172.21.15.26:6815","nonce":3250108152}]},"public_addr":"172.21.15.26:6811/3250108152","cluster_addr":"172.21.15.26:6813/3250108152","heartbeat_back_addr":"172.21.15.26:6817/3250108152","heartbeat_front_addr":"172.21.15.26:6815/3250108152","state":["exists","up"]},{"osd":2,"uuid":"2e115328-76c6-478e-915c-5786903be49f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6818","nonce":1325135181},{"type":"v1","addr":"172.21.15.26:6819","nonce":1325135181}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6820","nonce":1325135181},{"type":"v1","addr":"172.21.15.26:6821","nonce":1325135181}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6824","nonce":1325135181},{"type":"v1","addr":"172.21.15.26:6825","nonce":1325135181}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6822","nonce":1325135181},{"type":"v1","addr":"172.21.15.26:6823","nonce":1325135181}]},"public_addr":"172.21.15.26:6819/1325135181","cluster_addr":"172.21.15.26:6821/1325135181","heartbeat_back_addr":"172.21.15.26:6825/1325135181","heartbeat_front_addr":"172.21.15.26:6823/1325135181","state":["exists","up"]},{"osd":3,"uuid":"c3dd2c3f-5cf5-4fb0-964c-1b8926f75c8a","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":26,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6826","nonce":3610585082},{"type":"v1","addr":"172.21.15.26:6827","nonce":3610585082}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6828","nonce":3610585082},{"type":"v1","addr":"172.21.15.26:6829","nonce":3610585082}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6832","nonce":3610585082},{"type":"v1","addr":"172.21.15.26:6833","nonce":3610585082}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6830","nonce":3610585082},{"type":"v1","addr":"172.21.15.26:6831","nonce":3610585082}]},"public_addr":"172.21.15.26:6827/3610585082","cluster_addr":"172.21.15.26:6829/3610585082","heartbeat_back_addr":"172.21.15.26:6833/3610585082","heartbeat_front_addr":"172.21.15.26:6831/3610585082","state":["exists","up"]},{"osd":4,"uuid":"a5f31996-1b38-491b-956b-5aebc72d0170","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":31,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6800","nonce":1528249138},{"type":"v1","addr":"172.21.15.70:6801","nonce":1528249138}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6802","nonce":1528249138},{"type":"v1","addr":"172.21.15.70:6803","nonce":1528249138}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6806","nonce":1528249138},{"type":"v1","addr":"172.21.15.70:6807","nonce":1528249138}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6804","nonce":1528249138},{"type":"v1","addr":"172.21.15.70:6805","nonce":1528249138}]},"public_addr":"172.21.15.70:6801/1528249138","cluster_addr":"172.21.15.70:6803/1528249138","heartbeat_back_addr":"172.21.15.70:6807/1528249138","heartbeat_front_addr":"172.21.15.70:6805/1528249138","state":["exists","up"]},{"osd":5,"uuid":"21dad198-b9db-42c6-8823-112df1ecd874","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":39,"up_thru":42,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6808","nonce":3802232906},{"type":"v1","addr":"172.21.15.70:6809","nonce":3802232906}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6810","nonce":3802232906},{"type":"v1","addr":"172.21.15.70:6811","nonce":3802232906}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6814","nonce":3802232906},{"type":"v1","addr":"172.21.15.70:6815","nonce":3802232906}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6812","nonce":3802232906},{"type":"v1","addr":"172.21.15.70:6813","nonce":3802232906}]},"public_addr":"172.21.15.70:6809/3802232906","cluster_addr":"172.21.15.70:6811/3802232906","heartbeat_back_addr":"172.21.15.70:6815/3802232906","heartbeat_front_addr":"172.21.15.70:6813/3802232906","state":["exists","up"]},{"osd":6,"uuid":"6f3be784-7149-4f13-81b5-98924e97c056","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":47,"up_thru":50,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6816","nonce":2944175278},{"type":"v1","addr":"172.21.15.70:6817","nonce":2944175278}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6818","nonce":2944175278},{"type":"v1","addr":"172.21.15.70:6819","nonce":2944175278}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6822","nonce":2944175278},{"type":"v1","addr":"172.21.15.70:6823","nonce":2944175278}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6820","nonce":2944175278},{"type":"v1","addr":"172.21.15.70:6821","nonce":2944175278}]},"public_addr":"172.21.15.70:6817/2944175278","cluster_addr":"172.21.15.70:6819/2944175278","heartbeat_back_addr":"172.21.15.70:6823/2944175278","heartbeat_front_addr":"172.21.15.70:6821/2944175278","state":["exists","up"]},{"osd":7,"uuid":"499f5f85-f4b2-40a7-ac0f-79cd22738d30","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":55,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6824","nonce":2824974165},{"type":"v1","addr":"172.21.15.70:6825","nonce":2824974165}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6826","nonce":2824974165},{"type":"v1","addr":"172.21.15.70:6827","nonce":2824974165}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6830","nonce":2824974165},{"type":"v1","addr":"172.21.15.70:6831","nonce":2824974165}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6828","nonce":2824974165},{"type":"v1","addr":"172.21.15.70:6829","nonce":2824974165}]},"public_addr":"172.21.15.70:6825/2824974165","cluster_addr":"172.21.15.70:6827/2824974165","heartbeat_back_addr":"172.21.15.70:6831/2824974165","heartbeat_front_addr":"172.21.15.70:6829/2824974165","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T20:38:10.875468+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T20:38:32.449963+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T20:38:53.633316+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T20:39:14.828940+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T20:39:34.945663+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T20:39:55.954288+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T20:40:16.847102+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T20:40:37.820404+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[{"pgid":"1.0","osds":[0,6]}],"primary_temp":[],"blocklist":{"172.21.15.26:0/2838335646":"2023-09-24T20:36:56.498559+0000","172.21.15.26:6801/952511372":"2023-09-24T20:36:56.498559+0000","172.21.15.26:6800/952511372":"2023-09-24T20:36:56.498559+0000","172.21.15.26:0/1625326392":"2023-09-24T20:36:56.498559+0000","172.21.15.26:0/3175144810":"2023-09-24T20:36:31.159649+0000","172.21.15.26:0/1815190465":"2023-09-24T20:36:31.159649+0000","172.21.15.26:0/289994511":"2023-09-24T20:36:56.498559+0000","172.21.15.26:0/686734212":"2023-09-24T20:36:31.159649+0000","172.21.15.26:6800/4099624907":"2023-09-24T20:36:31.159649+0000","172.21.15.26:6801/4099624907":"2023-09-24T20:36:31.159649+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2023-09-23T20:40:41.585 INFO:tasks.cephadm.ceph_manager.ceph:[{'pool': 1, 'pool_name': '.mgr', 'create_time': '2023-09-23T20:38:57.022844+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 3, 'min_size': 2, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 1, 'pg_placement_num': 1, 'pg_placement_num_target': 1, 'pg_num_target': 1, 'pg_num_pending': 1, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '22', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_max': 32, 'pg_num_min': 1}, 'application_metadata': {'mgr': {}}}] 2023-09-23T20:40:41.586 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph osd pool get .mgr pg_num 2023-09-23T20:40:41.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:40:41 smithi070 ceph-mon[100755]: osdmap e57: 8 total, 8 up, 8 in 2023-09-23T20:40:41.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:40:41 smithi070 ceph-mon[100755]: pgmap v159: 1 pgs: 1 remapped+peering; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:40:41.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:40:41 smithi070 ceph-mon[100755]: from='client.? 172.21.15.26:0/1155842406' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-09-23T20:40:41.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:40:41 smithi026 ceph-mon[109912]: osdmap e57: 8 total, 8 up, 8 in 2023-09-23T20:40:41.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:40:41 smithi026 ceph-mon[109912]: pgmap v159: 1 pgs: 1 remapped+peering; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:40:41.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:40:41 smithi026 ceph-mon[109912]: from='client.? 172.21.15.26:0/1155842406' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-09-23T20:40:41.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:40:41 smithi026 ceph-mon[95839]: osdmap e57: 8 total, 8 up, 8 in 2023-09-23T20:40:41.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:40:41 smithi026 ceph-mon[95839]: pgmap v159: 1 pgs: 1 remapped+peering; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:40:41.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:40:41 smithi026 ceph-mon[95839]: from='client.? 172.21.15.26:0/1155842406' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-09-23T20:40:42.188 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:40:42.693 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:40:42 smithi026 ceph-mon[95839]: osdmap e58: 8 total, 8 up, 8 in 2023-09-23T20:40:42.694 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:40:42 smithi026 ceph-mon[109912]: osdmap e58: 8 total, 8 up, 8 in 2023-09-23T20:40:42.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:40:42 smithi070 ceph-mon[100755]: osdmap e58: 8 total, 8 up, 8 in 2023-09-23T20:40:43.712 INFO:teuthology.orchestra.run.smithi026.stdout:pg_num: 1 2023-09-23T20:40:43.860 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:40:43 smithi026 ceph-mon[95839]: pgmap v161: 1 pgs: 1 remapped+peering; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:40:43.861 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:40:43 smithi026 ceph-mon[95839]: osdmap e59: 8 total, 8 up, 8 in 2023-09-23T20:40:43.861 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:40:43 smithi026 ceph-mon[109912]: pgmap v161: 1 pgs: 1 remapped+peering; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:40:43.861 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:40:43 smithi026 ceph-mon[109912]: osdmap e59: 8 total, 8 up, 8 in 2023-09-23T20:40:43.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:40:43 smithi070 ceph-mon[100755]: pgmap v161: 1 pgs: 1 remapped+peering; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:40:43.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:40:43 smithi070 ceph-mon[100755]: osdmap e59: 8 total, 8 up, 8 in 2023-09-23T20:40:44.142 INFO:tasks.cephadm:Adding prometheus.a on smithi070 2023-09-23T20:40:44.142 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph orch apply prometheus '1;smithi070=a' 2023-09-23T20:40:44.748 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:40:44.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:40:44 smithi070 ceph-mon[100755]: from='client.? 172.21.15.26:0/1635391482' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2023-09-23T20:40:44.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:40:44 smithi026 ceph-mon[95839]: from='client.? 172.21.15.26:0/1635391482' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2023-09-23T20:40:44.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:40:44 smithi026 ceph-mon[109912]: from='client.? 172.21.15.26:0/1635391482' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2023-09-23T20:40:45.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:40:45 smithi070 ceph-mon[100755]: pgmap v163: 1 pgs: 1 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 115 KiB/s, 0 objects/s recovering 2023-09-23T20:40:45.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:40:45 smithi026 ceph-mon[95839]: pgmap v163: 1 pgs: 1 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 115 KiB/s, 0 objects/s recovering 2023-09-23T20:40:45.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:40:45 smithi026 ceph-mon[109912]: pgmap v163: 1 pgs: 1 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 115 KiB/s, 0 objects/s recovering 2023-09-23T20:40:46.044 INFO:teuthology.orchestra.run.smithi070.stdout:Scheduled prometheus update... 2023-09-23T20:40:46.400 DEBUG:teuthology.orchestra.run.smithi070:prometheus.a> sudo journalctl -f -n 0 -u ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7@prometheus.a.service 2023-09-23T20:40:46.403 INFO:tasks.cephadm:Adding node-exporter.a on smithi026 2023-09-23T20:40:46.403 INFO:tasks.cephadm:Adding node-exporter.b on smithi070 2023-09-23T20:40:46.403 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph orch apply node-exporter '2;smithi026=a;smithi070=b' 2023-09-23T20:40:46.431 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:-- Logs begin at Sat 2023-09-23 20:21:16 UTC. -- 2023-09-23T20:40:47.027 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:40:47.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:40:47 smithi070 ceph-mon[100755]: from='client.24328 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "prometheus", "placement": "1;smithi070=a", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T20:40:47.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:40:47 smithi070 ceph-mon[100755]: Saving service prometheus spec with placement smithi070=a;count:1 2023-09-23T20:40:47.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:40:47 smithi070 ceph-mon[100755]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:40:47.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:40:47 smithi070 ceph-mon[100755]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:40:47.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:40:47 smithi070 ceph-mon[100755]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:40:47.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:40:47 smithi070 ceph-mon[100755]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:40:47.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:40:47 smithi070 ceph-mon[100755]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:40:47.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:40:47 smithi070 ceph-mon[100755]: Deploying daemon prometheus.a on smithi070 2023-09-23T20:40:47.371 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:40:47 smithi070 ceph-mon[100755]: pgmap v164: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 96 KiB/s, 0 objects/s recovering 2023-09-23T20:40:47.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:40:47 smithi026 ceph-mon[109912]: from='client.24328 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "prometheus", "placement": "1;smithi070=a", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T20:40:47.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:40:47 smithi026 ceph-mon[109912]: Saving service prometheus spec with placement smithi070=a;count:1 2023-09-23T20:40:47.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:40:47 smithi026 ceph-mon[109912]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:40:47.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:40:47 smithi026 ceph-mon[109912]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:40:47.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:40:47 smithi026 ceph-mon[109912]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:40:47.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:40:47 smithi026 ceph-mon[109912]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:40:47.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:40:47 smithi026 ceph-mon[109912]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:40:47.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:40:47 smithi026 ceph-mon[109912]: Deploying daemon prometheus.a on smithi070 2023-09-23T20:40:47.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:40:47 smithi026 ceph-mon[109912]: pgmap v164: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 96 KiB/s, 0 objects/s recovering 2023-09-23T20:40:47.383 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:40:47 smithi026 ceph-mon[95839]: from='client.24328 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "prometheus", "placement": "1;smithi070=a", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T20:40:47.383 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:40:47 smithi026 ceph-mon[95839]: Saving service prometheus spec with placement smithi070=a;count:1 2023-09-23T20:40:47.383 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:40:47 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:40:47.383 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:40:47 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:40:47.383 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:40:47 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:40:47.383 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:40:47 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:40:47.383 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:40:47 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:40:47.383 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:40:47 smithi026 ceph-mon[95839]: Deploying daemon prometheus.a on smithi070 2023-09-23T20:40:47.383 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:40:47 smithi026 ceph-mon[95839]: pgmap v164: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 96 KiB/s, 0 objects/s recovering 2023-09-23T20:40:49.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:40:49 smithi070 ceph-mon[100755]: pgmap v165: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 72 KiB/s, 0 objects/s recovering 2023-09-23T20:40:49.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:40:49 smithi026 ceph-mon[95839]: pgmap v165: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 72 KiB/s, 0 objects/s recovering 2023-09-23T20:40:49.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:40:49 smithi026 ceph-mon[109912]: pgmap v165: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 72 KiB/s, 0 objects/s recovering 2023-09-23T20:40:50.789 INFO:teuthology.orchestra.run.smithi070.stdout:Scheduled node-exporter update... 2023-09-23T20:40:51.503 DEBUG:teuthology.orchestra.run.smithi026:node-exporter.a> sudo journalctl -f -n 0 -u ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7@node-exporter.a.service 2023-09-23T20:40:51.505 DEBUG:teuthology.orchestra.run.smithi070:node-exporter.b> sudo journalctl -f -n 0 -u ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7@node-exporter.b.service 2023-09-23T20:40:51.508 INFO:tasks.cephadm:Adding alertmanager.a on smithi026 2023-09-23T20:40:51.508 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph orch apply alertmanager '1;smithi026=a' 2023-09-23T20:40:51.537 INFO:journalctl@ceph.node-exporter.b.smithi070.stdout:-- Logs begin at Sat 2023-09-23 20:21:16 UTC. -- 2023-09-23T20:40:51.544 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:-- Logs begin at Sat 2023-09-23 20:20:42 UTC. -- 2023-09-23T20:40:51.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:40:51 smithi070 ceph-mon[100755]: pgmap v166: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 64 KiB/s, 0 objects/s recovering 2023-09-23T20:40:51.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:40:51 smithi070 ceph-mon[100755]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:40:51.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:40:51 smithi026 ceph-mon[95839]: pgmap v166: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 64 KiB/s, 0 objects/s recovering 2023-09-23T20:40:51.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:40:51 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:40:51.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:40:51 smithi026 ceph-mon[109912]: pgmap v166: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 64 KiB/s, 0 objects/s recovering 2023-09-23T20:40:51.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:40:51 smithi026 ceph-mon[109912]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:40:52.455 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:40:52.861 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:40:52 smithi070 ceph-mon[100755]: from='client.24334 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "node-exporter", "placement": "2;smithi026=a;smithi070=b", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T20:40:52.861 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:40:52 smithi070 ceph-mon[100755]: Saving service node-exporter spec with placement smithi026=a;smithi070=b;count:2 2023-09-23T20:40:52.881 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:40:52 smithi026 ceph-mon[95839]: from='client.24334 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "node-exporter", "placement": "2;smithi026=a;smithi070=b", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T20:40:52.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:40:52 smithi026 ceph-mon[95839]: Saving service node-exporter spec with placement smithi026=a;smithi070=b;count:2 2023-09-23T20:40:52.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:40:52 smithi026 ceph-mon[109912]: from='client.24334 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "node-exporter", "placement": "2;smithi026=a;smithi070=b", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T20:40:52.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:40:52 smithi026 ceph-mon[109912]: Saving service node-exporter spec with placement smithi026=a;smithi070=b;count:2 2023-09-23T20:40:53.813 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:40:53 smithi070 ceph-mon[100755]: pgmap v167: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 58 KiB/s, 0 objects/s recovering 2023-09-23T20:40:53.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:40:53 smithi026 ceph-mon[95839]: pgmap v167: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 58 KiB/s, 0 objects/s recovering 2023-09-23T20:40:53.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:40:53 smithi026 ceph-mon[109912]: pgmap v167: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 58 KiB/s, 0 objects/s recovering 2023-09-23T20:40:54.346 INFO:teuthology.orchestra.run.smithi070.stdout:Scheduled alertmanager update... 2023-09-23T20:40:54.619 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 20:40:54 smithi070 podman[134776]: 2023-09-23T20:40:54.797 DEBUG:teuthology.orchestra.run.smithi026:alertmanager.a> sudo journalctl -f -n 0 -u ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7@alertmanager.a.service 2023-09-23T20:40:54.800 INFO:tasks.cephadm:Adding grafana.a on smithi070 2023-09-23T20:40:54.800 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph orch apply grafana '1;smithi070=a' 2023-09-23T20:40:54.835 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:-- Logs begin at Sat 2023-09-23 20:20:42 UTC. -- 2023-09-23T20:40:55.120 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 20:40:54 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-prometheus-a[134794]: ts=2023-09-23T20:40:54.688Z caller=main.go:564 level=info msg="Starting Prometheus Server" mode=server version="(version=2.43.0, branch=HEAD, revision=edfc3bcd025dd6fe296c167a14a216cab1e552ee)" 2023-09-23T20:40:55.120 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 20:40:54 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-prometheus-a[134794]: ts=2023-09-23T20:40:54.688Z caller=main.go:569 level=info build_context="(go=go1.19.7, platform=linux/amd64, user=root@8a0ee342e522, date=20230321-12:56:07, tags=netgo,builtinassets)" 2023-09-23T20:40:55.120 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 20:40:54 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-prometheus-a[134794]: ts=2023-09-23T20:40:54.688Z caller=main.go:570 level=info host_details="(Linux 4.18.0-372.9.1.el8.x86_64 #1 SMP Fri Apr 15 22:12:19 EDT 2022 x86_64 smithi070 (none))" 2023-09-23T20:40:55.120 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 20:40:54 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-prometheus-a[134794]: ts=2023-09-23T20:40:54.688Z caller=main.go:571 level=info fd_limits="(soft=1048576, hard=1048576)" 2023-09-23T20:40:55.120 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 20:40:54 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-prometheus-a[134794]: ts=2023-09-23T20:40:54.688Z caller=main.go:572 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2023-09-23T20:40:55.120 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 20:40:54 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-prometheus-a[134794]: ts=2023-09-23T20:40:54.689Z caller=web.go:561 level=info component=web msg="Start listening for connections" address=:9095 2023-09-23T20:40:55.120 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 20:40:54 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-prometheus-a[134794]: ts=2023-09-23T20:40:54.690Z caller=main.go:1005 level=info msg="Starting TSDB ..." 2023-09-23T20:40:55.121 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 20:40:54 smithi070 bash[134776]: 3ac457fce8eb6e9f1b45357ac531e2adb5b6e574dbb4badeb13d5f101e2cac03 2023-09-23T20:40:55.121 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 20:40:54 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-prometheus-a[134794]: ts=2023-09-23T20:40:54.691Z caller=tls_config.go:232 level=info component=web msg="Listening on" address=[::]:9095 2023-09-23T20:40:55.121 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 20:40:54 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-prometheus-a[134794]: ts=2023-09-23T20:40:54.691Z caller=tls_config.go:235 level=info component=web msg="TLS is disabled." http2=false address=[::]:9095 2023-09-23T20:40:55.121 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 20:40:54 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-prometheus-a[134794]: ts=2023-09-23T20:40:54.693Z caller=head.go:587 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2023-09-23T20:40:55.121 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 20:40:54 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-prometheus-a[134794]: ts=2023-09-23T20:40:54.693Z caller=head.go:658 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=1.752µs 2023-09-23T20:40:55.121 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 20:40:54 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-prometheus-a[134794]: ts=2023-09-23T20:40:54.693Z caller=head.go:664 level=info component=tsdb msg="Replaying WAL, this may take a while" 2023-09-23T20:40:55.121 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 20:40:54 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-prometheus-a[134794]: ts=2023-09-23T20:40:54.694Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=0 2023-09-23T20:40:55.121 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 20:40:54 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-prometheus-a[134794]: ts=2023-09-23T20:40:54.694Z caller=head.go:772 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=19.139µs wal_replay_duration=236.263µs wbl_replay_duration=143ns total_replay_duration=272.856µs 2023-09-23T20:40:55.121 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 20:40:54 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-prometheus-a[134794]: ts=2023-09-23T20:40:54.694Z caller=main.go:1026 level=info fs_type=XFS_SUPER_MAGIC 2023-09-23T20:40:55.121 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 20:40:54 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-prometheus-a[134794]: ts=2023-09-23T20:40:54.694Z caller=main.go:1029 level=info msg="TSDB started" 2023-09-23T20:40:55.121 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 20:40:54 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-prometheus-a[134794]: ts=2023-09-23T20:40:54.694Z caller=main.go:1209 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2023-09-23T20:40:55.121 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 20:40:54 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-prometheus-a[134794]: ts=2023-09-23T20:40:54.709Z caller=main.go:1246 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=14.335777ms db_storage=682ns remote_storage=1.024µs web_handler=283ns query_engine=548ns scrape=441.121µs scrape_sd=18.154µs notify=686ns notify_sd=1.305µs rules=13.724822ms tracing=5.338µs 2023-09-23T20:40:55.122 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 20:40:54 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-prometheus-a[134794]: ts=2023-09-23T20:40:54.709Z caller=main.go:990 level=info msg="Server is ready to receive web requests." 2023-09-23T20:40:55.122 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 20:40:54 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-prometheus-a[134794]: ts=2023-09-23T20:40:54.709Z caller=manager.go:974 level=info component="rule manager" msg="Starting rule manager..." 2023-09-23T20:40:55.122 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 20:40:54 smithi070 systemd[1]: Started Ceph prometheus.a for b3f8e4ec-5a50-11ee-8db4-212e2dc638e7. 2023-09-23T20:40:55.394 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:40:55.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:40:55 smithi070 ceph-mon[100755]: from='client.24340 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "alertmanager", "placement": "1;smithi026=a", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T20:40:55.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:40:55 smithi070 ceph-mon[100755]: Saving service alertmanager spec with placement smithi026=a;count:1 2023-09-23T20:40:55.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:40:55 smithi070 ceph-mon[100755]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:40:55.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:40:55 smithi070 ceph-mon[100755]: pgmap v168: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 48 KiB/s, 0 objects/s recovering 2023-09-23T20:40:55.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:40:55 smithi070 ceph-mon[100755]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:40:55.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:40:55 smithi070 ceph-mon[100755]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:40:55.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:40:55 smithi070 ceph-mon[100755]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:40:55.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:40:55 smithi070 ceph-mon[100755]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2023-09-23T20:40:55.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:40:55 smithi026 ceph-mon[95839]: from='client.24340 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "alertmanager", "placement": "1;smithi026=a", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T20:40:55.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:40:55 smithi026 ceph-mon[95839]: Saving service alertmanager spec with placement smithi026=a;count:1 2023-09-23T20:40:55.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:40:55 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:40:55.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:40:55 smithi026 ceph-mon[95839]: pgmap v168: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 48 KiB/s, 0 objects/s recovering 2023-09-23T20:40:55.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:40:55 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:40:55.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:40:55 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:40:55.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:40:55 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:40:55.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:40:55 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2023-09-23T20:40:55.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:40:55 smithi026 ceph-mon[109912]: from='client.24340 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "alertmanager", "placement": "1;smithi026=a", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T20:40:55.633 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:40:55 smithi026 ceph-mon[109912]: Saving service alertmanager spec with placement smithi026=a;count:1 2023-09-23T20:40:55.633 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:40:55 smithi026 ceph-mon[109912]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:40:55.633 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:40:55 smithi026 ceph-mon[109912]: pgmap v168: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 48 KiB/s, 0 objects/s recovering 2023-09-23T20:40:55.633 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:40:55 smithi026 ceph-mon[109912]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:40:55.633 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:40:55 smithi026 ceph-mon[109912]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:40:55.633 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:40:55 smithi026 ceph-mon[109912]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' 2023-09-23T20:40:55.633 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:40:55 smithi026 ceph-mon[109912]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2023-09-23T20:40:56.118 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:40:55 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[96086]: ignoring --setuser ceph since I am not root 2023-09-23T20:40:56.118 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:40:55 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[96086]: ignoring --setgroup ceph since I am not root 2023-09-23T20:40:56.118 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:40:55 smithi026 ceph-mgr[96110]: -- 172.21.15.26:0/2051438880 <== mon.0 v2:172.21.15.26:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x55dc07c4a9c0 con 0x55dc07c20800 2023-09-23T20:40:56.118 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:40:56 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[96086]: 2023-09-23T20:40:56.022+0000 7ff4949a61c0 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2023-09-23T20:40:56.118 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:40:56 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[96086]: 2023-09-23T20:40:56.116+0000 7ff4949a61c0 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-09-23T20:40:56.171 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 20:40:55 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-x[102861]: ignoring --setuser ceph since I am not root 2023-09-23T20:40:56.172 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 20:40:55 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-x[102861]: ignoring --setgroup ceph since I am not root 2023-09-23T20:40:56.172 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 20:40:56 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-x[102861]: 2023-09-23T20:40:56.004+0000 7f45ecf6d1c0 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2023-09-23T20:40:56.172 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 20:40:56 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-x[102861]: 2023-09-23T20:40:56.081+0000 7f45ecf6d1c0 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-09-23T20:40:56.382 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:40:56 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[96086]: 2023-09-23T20:40:56.223+0000 7ff4949a61c0 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-09-23T20:40:56.382 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:40:56 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[96086]: 2023-09-23T20:40:56.332+0000 7ff4949a61c0 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-09-23T20:40:56.449 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 20:40:56 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-x[102861]: 2023-09-23T20:40:56.171+0000 7f45ecf6d1c0 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-09-23T20:40:56.449 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 20:40:56 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-x[102861]: 2023-09-23T20:40:56.270+0000 7f45ecf6d1c0 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-09-23T20:40:56.764 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 20:40:56 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-x[102861]: 2023-09-23T20:40:56.448+0000 7f45ecf6d1c0 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-09-23T20:40:56.831 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:40:56 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[96086]: 2023-09-23T20:40:56.536+0000 7ff4949a61c0 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-09-23T20:40:57.068 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:40:56 smithi070 ceph-mon[100755]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd='[{"prefix": "mgr module enable", "module": "prometheus"}]': finished 2023-09-23T20:40:57.069 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:40:56 smithi070 ceph-mon[100755]: mgrmap e16: y(active, since 3m), standbys: x 2023-09-23T20:40:57.069 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 20:40:56 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-x[102861]: 2023-09-23T20:40:56.764+0000 7f45ecf6d1c0 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-09-23T20:40:57.069 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 20:40:56 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-x[102861]: 2023-09-23T20:40:56.852+0000 7f45ecf6d1c0 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-09-23T20:40:57.069 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 20:40:56 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-x[102861]: 2023-09-23T20:40:56.977+0000 7f45ecf6d1c0 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-09-23T20:40:57.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:40:56 smithi026 ceph-mon[109912]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd='[{"prefix": "mgr module enable", "module": "prometheus"}]': finished 2023-09-23T20:40:57.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:40:56 smithi026 ceph-mon[109912]: mgrmap e16: y(active, since 3m), standbys: x 2023-09-23T20:40:57.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:40:56 smithi026 ceph-mon[95839]: from='mgr.14152 172.21.15.26:0/2149668047' entity='mgr.y' cmd='[{"prefix": "mgr module enable", "module": "prometheus"}]': finished 2023-09-23T20:40:57.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:40:56 smithi026 ceph-mon[95839]: mgrmap e16: y(active, since 3m), standbys: x 2023-09-23T20:40:57.132 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:40:56 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[96086]: 2023-09-23T20:40:56.934+0000 7ff4949a61c0 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-09-23T20:40:57.132 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:40:57 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[96086]: 2023-09-23T20:40:57.040+0000 7ff4949a61c0 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-09-23T20:40:57.369 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 20:40:57 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-x[102861]: 2023-09-23T20:40:57.068+0000 7f45ecf6d1c0 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-09-23T20:40:57.631 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:40:57 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[96086]: 2023-09-23T20:40:57.189+0000 7ff4949a61c0 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-09-23T20:40:57.632 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:40:57 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[96086]: 2023-09-23T20:40:57.303+0000 7ff4949a61c0 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-09-23T20:40:57.870 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 20:40:57 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-x[102861]: 2023-09-23T20:40:57.480+0000 7f45ecf6d1c0 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-09-23T20:40:58.131 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:40:57 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[96086]: 2023-09-23T20:40:57.813+0000 7ff4949a61c0 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-09-23T20:40:58.369 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 20:40:58 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-x[102861]: 2023-09-23T20:40:58.033+0000 7f45ecf6d1c0 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-09-23T20:40:58.369 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 20:40:58 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-x[102861]: 2023-09-23T20:40:58.200+0000 7f45ecf6d1c0 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-09-23T20:40:58.882 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:40:58 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[96086]: 2023-09-23T20:40:58.497+0000 7ff4949a61c0 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-09-23T20:40:58.882 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:40:58 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[96086]: 2023-09-23T20:40:58.689+0000 7ff4949a61c0 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-09-23T20:40:59.208 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 20:40:58 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-x[102861]: 2023-09-23T20:40:58.951+0000 7f45ecf6d1c0 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-09-23T20:40:59.208 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 20:40:59 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-x[102861]: 2023-09-23T20:40:59.041+0000 7f45ecf6d1c0 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2023-09-23T20:40:59.480 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 20:40:59 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-x[102861]: 2023-09-23T20:40:59.207+0000 7f45ecf6d1c0 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-09-23T20:40:59.848 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 20:40:59 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-x[102861]: 2023-09-23T20:40:59.479+0000 7f45ecf6d1c0 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2023-09-23T20:40:59.848 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 20:40:59 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-x[102861]: 2023-09-23T20:40:59.564+0000 7f45ecf6d1c0 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-09-23T20:40:59.982 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:40:59 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[96086]: 2023-09-23T20:40:59.651+0000 7ff4949a61c0 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-09-23T20:40:59.982 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:40:59 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[96086]: 2023-09-23T20:40:59.761+0000 7ff4949a61c0 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2023-09-23T20:41:00.101 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 20:40:59 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-x[102861]: 2023-09-23T20:40:59.847+0000 7f45ecf6d1c0 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-09-23T20:41:00.328 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:40:59 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[96086]: 2023-09-23T20:40:59.980+0000 7ff4949a61c0 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-09-23T20:41:00.369 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 20:41:00 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-x[102861]: 2023-09-23T20:41:00.100+0000 7f45ecf6d1c0 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-09-23T20:41:00.631 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:41:00 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[96086]: 2023-09-23T20:41:00.326+0000 7ff4949a61c0 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2023-09-23T20:41:00.632 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:41:00 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[96086]: 2023-09-23T20:41:00.432+0000 7ff4949a61c0 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-09-23T20:41:01.090 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:41:00 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[96086]: 2023-09-23T20:41:00.763+0000 7ff4949a61c0 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-09-23T20:41:01.110 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 20:41:00 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-x[102861]: 2023-09-23T20:41:00.795+0000 7f45ecf6d1c0 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-09-23T20:41:01.110 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 20:41:00 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-x[102861]: 2023-09-23T20:41:00.924+0000 7f45ecf6d1c0 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-09-23T20:41:01.369 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 20:41:01 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-x[102861]: 2023-09-23T20:41:01.109+0000 7f45ecf6d1c0 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-09-23T20:41:01.369 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 20:41:01 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-x[102861]: 2023-09-23T20:41:01.221+0000 7f45ecf6d1c0 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-09-23T20:41:01.869 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 20:41:01 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-x[102861]: 2023-09-23T20:41:01.431+0000 7f45ecf6d1c0 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-09-23T20:41:02.325 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:41:01 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[96086]: 2023-09-23T20:41:01.089+0000 7ff4949a61c0 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-09-23T20:41:02.325 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:41:01 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[96086]: 2023-09-23T20:41:01.923+0000 7ff4949a61c0 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-09-23T20:41:02.325 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:41:02 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[96086]: 2023-09-23T20:41:02.083+0000 7ff4949a61c0 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-09-23T20:41:02.325 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:41:02 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[96086]: 2023-09-23T20:41:02.325+0000 7ff4949a61c0 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-09-23T20:41:02.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:41:02 smithi070 ceph-mon[100755]: Standby manager daemon x restarted 2023-09-23T20:41:02.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:41:02 smithi070 ceph-mon[100755]: Standby manager daemon x started 2023-09-23T20:41:02.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:41:02 smithi070 ceph-mon[100755]: from='mgr.? 172.21.15.70:0/329275811' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-09-23T20:41:02.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:41:02 smithi070 ceph-mon[100755]: from='mgr.? 172.21.15.70:0/329275811' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-09-23T20:41:02.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:41:02 smithi070 ceph-mon[100755]: from='mgr.? 172.21.15.70:0/329275811' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-09-23T20:41:02.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:41:02 smithi070 ceph-mon[100755]: from='mgr.? 172.21.15.70:0/329275811' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-09-23T20:41:02.620 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 20:41:02 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-x[102861]: 2023-09-23T20:41:02.270+0000 7f45ecf6d1c0 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-09-23T20:41:02.620 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 20:41:02 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-x[102861]: [23/Sep/2023:20:41:02] ENGINE Bus STARTING 2023-09-23T20:41:02.620 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 20:41:02 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-x[102861]: CherryPy Checker: 2023-09-23T20:41:02.620 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 20:41:02 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-x[102861]: The Application mounted at '' has an empty config. 2023-09-23T20:41:02.620 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 20:41:02 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-x[102861]: 2023-09-23T20:41:02.620 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 20:41:02 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-x[102861]: [23/Sep/2023:20:41:02] ENGINE Serving on http://:::9283 2023-09-23T20:41:02.620 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 20:41:02 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-x[102861]: [23/Sep/2023:20:41:02] ENGINE Bus STARTED 2023-09-23T20:41:02.631 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:41:02 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[96086]: 2023-09-23T20:41:02.465+0000 7ff4949a61c0 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-09-23T20:41:02.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:41:02 smithi026 ceph-mon[109912]: Standby manager daemon x restarted 2023-09-23T20:41:02.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:41:02 smithi026 ceph-mon[109912]: Standby manager daemon x started 2023-09-23T20:41:02.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:41:02 smithi026 ceph-mon[109912]: from='mgr.? 172.21.15.70:0/329275811' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-09-23T20:41:02.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:41:02 smithi026 ceph-mon[109912]: from='mgr.? 172.21.15.70:0/329275811' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-09-23T20:41:02.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:41:02 smithi026 ceph-mon[109912]: from='mgr.? 172.21.15.70:0/329275811' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-09-23T20:41:02.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:41:02 smithi026 ceph-mon[109912]: from='mgr.? 172.21.15.70:0/329275811' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-09-23T20:41:02.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:41:02 smithi026 ceph-mon[95839]: Standby manager daemon x restarted 2023-09-23T20:41:02.633 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:41:02 smithi026 ceph-mon[95839]: Standby manager daemon x started 2023-09-23T20:41:02.633 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:41:02 smithi026 ceph-mon[95839]: from='mgr.? 172.21.15.70:0/329275811' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-09-23T20:41:02.633 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:41:02 smithi026 ceph-mon[95839]: from='mgr.? 172.21.15.70:0/329275811' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-09-23T20:41:02.633 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:41:02 smithi026 ceph-mon[95839]: from='mgr.? 172.21.15.70:0/329275811' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-09-23T20:41:02.633 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:41:02 smithi026 ceph-mon[95839]: from='mgr.? 172.21.15.70:0/329275811' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-09-23T20:41:03.132 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:41:02 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[96086]: 2023-09-23T20:41:02.734+0000 7ff4949a61c0 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-09-23T20:41:03.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:41:03 smithi070 ceph-mon[100755]: mgrmap e17: y(active, since 4m), standbys: x 2023-09-23T20:41:03.631 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:41:03 smithi026 ceph-mon[109912]: mgrmap e17: y(active, since 4m), standbys: x 2023-09-23T20:41:03.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:41:03 smithi026 ceph-mon[95839]: mgrmap e17: y(active, since 4m), standbys: x 2023-09-23T20:41:04.132 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:41:03 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[96086]: 2023-09-23T20:41:03.776+0000 7ff4949a61c0 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-09-23T20:41:04.132 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:41:03 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[96086]: [23/Sep/2023:20:41:03] ENGINE Bus STARTING 2023-09-23T20:41:04.132 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:41:04 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[96086]: CherryPy Checker: 2023-09-23T20:41:04.133 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:41:04 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[96086]: The Application mounted at '' has an empty config. 2023-09-23T20:41:04.133 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:41:04 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[96086]: 2023-09-23T20:41:04.133 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:41:04 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[96086]: [23/Sep/2023:20:41:04] ENGINE Serving on http://:::9283 2023-09-23T20:41:04.133 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:41:04 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[96086]: [23/Sep/2023:20:41:04] ENGINE Bus STARTED 2023-09-23T20:41:04.813 INFO:teuthology.orchestra.run.smithi070.stdout:Scheduled grafana update... 2023-09-23T20:41:04.840 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:41:04 smithi070 ceph-mon[100755]: Active manager daemon y restarted 2023-09-23T20:41:04.840 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:41:04 smithi070 ceph-mon[100755]: Activating manager daemon y 2023-09-23T20:41:04.841 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:41:04 smithi070 ceph-mon[100755]: osdmap e60: 8 total, 8 up, 8 in 2023-09-23T20:41:04.841 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:41:04 smithi070 ceph-mon[100755]: mgrmap e18: y(active, starting, since 0.00738302s), standbys: x 2023-09-23T20:41:04.841 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:41:04 smithi070 ceph-mon[100755]: from='mgr.14451 172.21.15.26:0/1329296977' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-23T20:41:04.841 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:41:04 smithi070 ceph-mon[100755]: from='mgr.14451 172.21.15.26:0/1329296977' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T20:41:04.841 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:41:04 smithi070 ceph-mon[100755]: from='mgr.14451 172.21.15.26:0/1329296977' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T20:41:04.841 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:41:04 smithi070 ceph-mon[100755]: from='mgr.14451 172.21.15.26:0/1329296977' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-09-23T20:41:04.841 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:41:04 smithi070 ceph-mon[100755]: from='mgr.14451 172.21.15.26:0/1329296977' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-09-23T20:41:04.841 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:41:04 smithi070 ceph-mon[100755]: from='mgr.14451 172.21.15.26:0/1329296977' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-23T20:41:04.841 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:41:04 smithi070 ceph-mon[100755]: from='mgr.14451 172.21.15.26:0/1329296977' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-23T20:41:04.841 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:41:04 smithi070 ceph-mon[100755]: from='mgr.14451 172.21.15.26:0/1329296977' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-23T20:41:04.841 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:41:04 smithi070 ceph-mon[100755]: from='mgr.14451 172.21.15.26:0/1329296977' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-23T20:41:04.841 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:41:04 smithi070 ceph-mon[100755]: from='mgr.14451 172.21.15.26:0/1329296977' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-09-23T20:41:04.842 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:41:04 smithi070 ceph-mon[100755]: from='mgr.14451 172.21.15.26:0/1329296977' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-09-23T20:41:04.842 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:41:04 smithi070 ceph-mon[100755]: from='mgr.14451 172.21.15.26:0/1329296977' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-09-23T20:41:04.842 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:41:04 smithi070 ceph-mon[100755]: from='mgr.14451 172.21.15.26:0/1329296977' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-09-23T20:41:04.842 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:41:04 smithi070 ceph-mon[100755]: from='mgr.14451 172.21.15.26:0/1329296977' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-09-23T20:41:04.842 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:41:04 smithi070 ceph-mon[100755]: from='mgr.14451 172.21.15.26:0/1329296977' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-09-23T20:41:04.842 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:41:04 smithi070 ceph-mon[100755]: from='mgr.14451 172.21.15.26:0/1329296977' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-09-23T20:41:04.842 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:41:04 smithi070 ceph-mon[100755]: Manager daemon y is now available 2023-09-23T20:41:04.842 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:41:04 smithi070 ceph-mon[100755]: from='mgr.14451 172.21.15.26:0/1329296977' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:41:04.842 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:41:04 smithi070 ceph-mon[100755]: from='mgr.14451 172.21.15.26:0/1329296977' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:41:04.842 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:41:04 smithi070 ceph-mon[100755]: from='mgr.14451 ' entity='mgr.y' 2023-09-23T20:41:04.842 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:41:04 smithi070 ceph-mon[100755]: from='mgr.14451 172.21.15.26:0/1329296977' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:41:04.843 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:41:04 smithi070 ceph-mon[100755]: from='mgr.14451 172.21.15.26:0/1329296977' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-09-23T20:41:04.843 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:41:04 smithi070 ceph-mon[100755]: from='mgr.14451 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-09-23T20:41:04.843 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:41:04 smithi070 ceph-mon[100755]: from='mgr.14451 172.21.15.26:0/1329296977' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-09-23T20:41:04.843 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:41:04 smithi070 ceph-mon[100755]: from='mgr.14451 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-09-23T20:41:04.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:41:04 smithi026 ceph-mon[109912]: Active manager daemon y restarted 2023-09-23T20:41:04.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:41:04 smithi026 ceph-mon[109912]: Activating manager daemon y 2023-09-23T20:41:04.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:41:04 smithi026 ceph-mon[109912]: osdmap e60: 8 total, 8 up, 8 in 2023-09-23T20:41:04.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:41:04 smithi026 ceph-mon[109912]: mgrmap e18: y(active, starting, since 0.00738302s), standbys: x 2023-09-23T20:41:04.883 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:41:04 smithi026 ceph-mon[109912]: from='mgr.14451 172.21.15.26:0/1329296977' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-23T20:41:04.883 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:41:04 smithi026 ceph-mon[109912]: from='mgr.14451 172.21.15.26:0/1329296977' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T20:41:04.883 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:41:04 smithi026 ceph-mon[109912]: from='mgr.14451 172.21.15.26:0/1329296977' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T20:41:04.883 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:41:04 smithi026 ceph-mon[109912]: from='mgr.14451 172.21.15.26:0/1329296977' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-09-23T20:41:04.883 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:41:04 smithi026 ceph-mon[109912]: from='mgr.14451 172.21.15.26:0/1329296977' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-09-23T20:41:04.883 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:41:04 smithi026 ceph-mon[109912]: from='mgr.14451 172.21.15.26:0/1329296977' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-23T20:41:04.883 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:41:04 smithi026 ceph-mon[109912]: from='mgr.14451 172.21.15.26:0/1329296977' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-23T20:41:04.883 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:41:04 smithi026 ceph-mon[109912]: from='mgr.14451 172.21.15.26:0/1329296977' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-23T20:41:04.883 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:41:04 smithi026 ceph-mon[109912]: from='mgr.14451 172.21.15.26:0/1329296977' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-23T20:41:04.883 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:41:04 smithi026 ceph-mon[109912]: from='mgr.14451 172.21.15.26:0/1329296977' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-09-23T20:41:04.883 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:41:04 smithi026 ceph-mon[109912]: from='mgr.14451 172.21.15.26:0/1329296977' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-09-23T20:41:04.883 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:41:04 smithi026 ceph-mon[109912]: from='mgr.14451 172.21.15.26:0/1329296977' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-09-23T20:41:04.884 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:41:04 smithi026 ceph-mon[109912]: from='mgr.14451 172.21.15.26:0/1329296977' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-09-23T20:41:04.884 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:41:04 smithi026 ceph-mon[109912]: from='mgr.14451 172.21.15.26:0/1329296977' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-09-23T20:41:04.884 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:41:04 smithi026 ceph-mon[109912]: from='mgr.14451 172.21.15.26:0/1329296977' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-09-23T20:41:04.884 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:41:04 smithi026 ceph-mon[109912]: from='mgr.14451 172.21.15.26:0/1329296977' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-09-23T20:41:04.884 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:41:04 smithi026 ceph-mon[109912]: Manager daemon y is now available 2023-09-23T20:41:04.884 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:41:04 smithi026 ceph-mon[109912]: from='mgr.14451 172.21.15.26:0/1329296977' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:41:04.884 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:41:04 smithi026 ceph-mon[109912]: from='mgr.14451 172.21.15.26:0/1329296977' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:41:04.884 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:41:04 smithi026 ceph-mon[109912]: from='mgr.14451 ' entity='mgr.y' 2023-09-23T20:41:04.884 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:41:04 smithi026 ceph-mon[109912]: from='mgr.14451 172.21.15.26:0/1329296977' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:41:04.884 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:41:04 smithi026 ceph-mon[109912]: from='mgr.14451 172.21.15.26:0/1329296977' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-09-23T20:41:04.884 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:41:04 smithi026 ceph-mon[109912]: from='mgr.14451 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-09-23T20:41:04.884 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:41:04 smithi026 ceph-mon[109912]: from='mgr.14451 172.21.15.26:0/1329296977' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-09-23T20:41:04.884 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:41:04 smithi026 ceph-mon[109912]: from='mgr.14451 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-09-23T20:41:04.885 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:41:04 smithi026 ceph-mon[95839]: Active manager daemon y restarted 2023-09-23T20:41:04.885 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:41:04 smithi026 ceph-mon[95839]: Activating manager daemon y 2023-09-23T20:41:04.885 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:41:04 smithi026 ceph-mon[95839]: osdmap e60: 8 total, 8 up, 8 in 2023-09-23T20:41:04.885 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:41:04 smithi026 ceph-mon[95839]: mgrmap e18: y(active, starting, since 0.00738302s), standbys: x 2023-09-23T20:41:04.885 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:41:04 smithi026 ceph-mon[95839]: from='mgr.14451 172.21.15.26:0/1329296977' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-23T20:41:04.885 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:41:04 smithi026 ceph-mon[95839]: from='mgr.14451 172.21.15.26:0/1329296977' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T20:41:04.885 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:41:04 smithi026 ceph-mon[95839]: from='mgr.14451 172.21.15.26:0/1329296977' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T20:41:04.885 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:41:04 smithi026 ceph-mon[95839]: from='mgr.14451 172.21.15.26:0/1329296977' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-09-23T20:41:04.885 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:41:04 smithi026 ceph-mon[95839]: from='mgr.14451 172.21.15.26:0/1329296977' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-09-23T20:41:04.885 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:41:04 smithi026 ceph-mon[95839]: from='mgr.14451 172.21.15.26:0/1329296977' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-23T20:41:04.886 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:41:04 smithi026 ceph-mon[95839]: from='mgr.14451 172.21.15.26:0/1329296977' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-23T20:41:04.886 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:41:04 smithi026 ceph-mon[95839]: from='mgr.14451 172.21.15.26:0/1329296977' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-23T20:41:04.886 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:41:04 smithi026 ceph-mon[95839]: from='mgr.14451 172.21.15.26:0/1329296977' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-23T20:41:04.886 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:41:04 smithi026 ceph-mon[95839]: from='mgr.14451 172.21.15.26:0/1329296977' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-09-23T20:41:04.886 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:41:04 smithi026 ceph-mon[95839]: from='mgr.14451 172.21.15.26:0/1329296977' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-09-23T20:41:04.886 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:41:04 smithi026 ceph-mon[95839]: from='mgr.14451 172.21.15.26:0/1329296977' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-09-23T20:41:04.886 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:41:04 smithi026 ceph-mon[95839]: from='mgr.14451 172.21.15.26:0/1329296977' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-09-23T20:41:04.886 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:41:04 smithi026 ceph-mon[95839]: from='mgr.14451 172.21.15.26:0/1329296977' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-09-23T20:41:04.886 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:41:04 smithi026 ceph-mon[95839]: from='mgr.14451 172.21.15.26:0/1329296977' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-09-23T20:41:04.886 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:41:04 smithi026 ceph-mon[95839]: from='mgr.14451 172.21.15.26:0/1329296977' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-09-23T20:41:04.886 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:41:04 smithi026 ceph-mon[95839]: Manager daemon y is now available 2023-09-23T20:41:04.886 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:41:04 smithi026 ceph-mon[95839]: from='mgr.14451 172.21.15.26:0/1329296977' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:41:04.887 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:41:04 smithi026 ceph-mon[95839]: from='mgr.14451 172.21.15.26:0/1329296977' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:41:04.887 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:41:04 smithi026 ceph-mon[95839]: from='mgr.14451 ' entity='mgr.y' 2023-09-23T20:41:04.887 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:41:04 smithi026 ceph-mon[95839]: from='mgr.14451 172.21.15.26:0/1329296977' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:41:04.887 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:41:04 smithi026 ceph-mon[95839]: from='mgr.14451 172.21.15.26:0/1329296977' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-09-23T20:41:04.887 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:41:04 smithi026 ceph-mon[95839]: from='mgr.14451 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-09-23T20:41:04.887 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:41:04 smithi026 ceph-mon[95839]: from='mgr.14451 172.21.15.26:0/1329296977' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-09-23T20:41:04.887 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:41:04 smithi026 ceph-mon[95839]: from='mgr.14451 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-09-23T20:41:05.358 DEBUG:teuthology.orchestra.run.smithi070:grafana.a> sudo journalctl -f -n 0 -u ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7@grafana.a.service 2023-09-23T20:41:05.360 INFO:tasks.cephadm:Setting up client nodes... 2023-09-23T20:41:05.360 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph auth get-or-create client.0 mon 'allow *' osd 'allow *' mds 'allow *' mgr 'allow *' 2023-09-23T20:41:05.407 INFO:journalctl@ceph.grafana.a.smithi070.stdout:-- Logs begin at Sat 2023-09-23 20:21:16 UTC. -- 2023-09-23T20:41:05.968 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:41:06.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:41:05 smithi070 ceph-mon[100755]: mgrmap e19: y(active, since 1.012s), standbys: x 2023-09-23T20:41:06.121 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:41:05 smithi070 ceph-mon[100755]: Saving service grafana spec with placement smithi070=a;count:1 2023-09-23T20:41:06.121 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:41:05 smithi070 ceph-mon[100755]: pgmap v3: 1 pgs: 1 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:41:06.121 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:41:05 smithi070 ceph-mon[100755]: from='mgr.14451 ' entity='mgr.y' 2023-09-23T20:41:06.121 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:41:05 smithi070 ceph-mon[100755]: from='mgr.14451 ' entity='mgr.y' 2023-09-23T20:41:06.131 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:41:05 smithi026 ceph-mon[95839]: mgrmap e19: y(active, since 1.012s), standbys: x 2023-09-23T20:41:06.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:41:05 smithi026 ceph-mon[95839]: Saving service grafana spec with placement smithi070=a;count:1 2023-09-23T20:41:06.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:41:05 smithi026 ceph-mon[95839]: pgmap v3: 1 pgs: 1 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:41:06.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:41:05 smithi026 ceph-mon[95839]: from='mgr.14451 ' entity='mgr.y' 2023-09-23T20:41:06.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:41:05 smithi026 ceph-mon[95839]: from='mgr.14451 ' entity='mgr.y' 2023-09-23T20:41:06.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:41:05 smithi026 ceph-mon[109912]: mgrmap e19: y(active, since 1.012s), standbys: x 2023-09-23T20:41:06.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:41:05 smithi026 ceph-mon[109912]: Saving service grafana spec with placement smithi070=a;count:1 2023-09-23T20:41:06.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:41:05 smithi026 ceph-mon[109912]: pgmap v3: 1 pgs: 1 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:41:06.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:41:05 smithi026 ceph-mon[109912]: from='mgr.14451 ' entity='mgr.y' 2023-09-23T20:41:06.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:41:05 smithi026 ceph-mon[109912]: from='mgr.14451 ' entity='mgr.y' 2023-09-23T20:41:07.032 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:41:06 smithi026 ceph-mon[95839]: [23/Sep/2023:20:41:05] ENGINE Bus STARTING 2023-09-23T20:41:07.032 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:41:06 smithi026 ceph-mon[95839]: [23/Sep/2023:20:41:05] ENGINE Serving on https://172.21.15.26:7150 2023-09-23T20:41:07.032 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:41:06 smithi026 ceph-mon[95839]: [23/Sep/2023:20:41:05] ENGINE Bus STARTED 2023-09-23T20:41:07.032 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:41:06 smithi026 ceph-mon[95839]: pgmap v4: 1 pgs: 1 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:41:07.033 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:41:06 smithi026 ceph-mon[109912]: [23/Sep/2023:20:41:05] ENGINE Bus STARTING 2023-09-23T20:41:07.033 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:41:06 smithi026 ceph-mon[109912]: [23/Sep/2023:20:41:05] ENGINE Serving on https://172.21.15.26:7150 2023-09-23T20:41:07.033 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:41:06 smithi026 ceph-mon[109912]: [23/Sep/2023:20:41:05] ENGINE Bus STARTED 2023-09-23T20:41:07.033 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:41:06 smithi026 ceph-mon[109912]: pgmap v4: 1 pgs: 1 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:41:07.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:41:06 smithi070 ceph-mon[100755]: [23/Sep/2023:20:41:05] ENGINE Bus STARTING 2023-09-23T20:41:07.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:41:06 smithi070 ceph-mon[100755]: [23/Sep/2023:20:41:05] ENGINE Serving on https://172.21.15.26:7150 2023-09-23T20:41:07.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:41:06 smithi070 ceph-mon[100755]: [23/Sep/2023:20:41:05] ENGINE Bus STARTED 2023-09-23T20:41:07.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:41:06 smithi070 ceph-mon[100755]: pgmap v4: 1 pgs: 1 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:41:07.492 INFO:teuthology.orchestra.run.smithi026.stdout:[client.0] 2023-09-23T20:41:07.492 INFO:teuthology.orchestra.run.smithi026.stdout: key = AQBjTQ9l2P4WHRAAclIqU13nwnZCqoJyxmzUWA== 2023-09-23T20:41:08.018 DEBUG:teuthology.orchestra.run.smithi026:> set -ex 2023-09-23T20:41:08.018 DEBUG:teuthology.orchestra.run.smithi026:> sudo dd of=/etc/ceph/ceph.client.0.keyring 2023-09-23T20:41:08.018 DEBUG:teuthology.orchestra.run.smithi026:> sudo chmod 0644 /etc/ceph/ceph.client.0.keyring 2023-09-23T20:41:08.056 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:41:07 smithi026 ceph-mon[95839]: mgrmap e20: y(active, since 3s), standbys: x 2023-09-23T20:41:08.057 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:41:07 smithi026 ceph-mon[95839]: from='mgr.14451 ' entity='mgr.y' 2023-09-23T20:41:08.057 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:41:07 smithi026 ceph-mon[95839]: from='mgr.14451 ' entity='mgr.y' 2023-09-23T20:41:08.057 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:41:07 smithi026 ceph-mon[95839]: from='mgr.14451 ' entity='mgr.y' 2023-09-23T20:41:08.057 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:41:07 smithi026 ceph-mon[95839]: from='mgr.14451 ' entity='mgr.y' 2023-09-23T20:41:08.057 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:41:07 smithi026 ceph-mon[95839]: from='client.? 172.21.15.26:0/676480685' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2023-09-23T20:41:08.057 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:41:07 smithi026 ceph-mon[95839]: from='client.? 172.21.15.26:0/676480685' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2023-09-23T20:41:08.057 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:41:07 smithi026 ceph-mon[109912]: mgrmap e20: y(active, since 3s), standbys: x 2023-09-23T20:41:08.057 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:41:07 smithi026 ceph-mon[109912]: from='mgr.14451 ' entity='mgr.y' 2023-09-23T20:41:08.057 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:41:07 smithi026 ceph-mon[109912]: from='mgr.14451 ' entity='mgr.y' 2023-09-23T20:41:08.058 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:41:07 smithi026 ceph-mon[109912]: from='mgr.14451 ' entity='mgr.y' 2023-09-23T20:41:08.058 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:41:07 smithi026 ceph-mon[109912]: from='mgr.14451 ' entity='mgr.y' 2023-09-23T20:41:08.058 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:41:07 smithi026 ceph-mon[109912]: from='client.? 172.21.15.26:0/676480685' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2023-09-23T20:41:08.058 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:41:07 smithi026 ceph-mon[109912]: from='client.? 172.21.15.26:0/676480685' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2023-09-23T20:41:08.061 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph auth get-or-create client.1 mon 'allow *' osd 'allow *' mds 'allow *' mgr 'allow *' 2023-09-23T20:41:08.085 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:41:07 smithi070 ceph-mon[100755]: mgrmap e20: y(active, since 3s), standbys: x 2023-09-23T20:41:08.085 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:41:07 smithi070 ceph-mon[100755]: from='mgr.14451 ' entity='mgr.y' 2023-09-23T20:41:08.085 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:41:07 smithi070 ceph-mon[100755]: from='mgr.14451 ' entity='mgr.y' 2023-09-23T20:41:08.085 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:41:07 smithi070 ceph-mon[100755]: from='mgr.14451 ' entity='mgr.y' 2023-09-23T20:41:08.085 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:41:07 smithi070 ceph-mon[100755]: from='mgr.14451 ' entity='mgr.y' 2023-09-23T20:41:08.085 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:41:07 smithi070 ceph-mon[100755]: from='client.? 172.21.15.26:0/676480685' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2023-09-23T20:41:08.085 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:41:07 smithi070 ceph-mon[100755]: from='client.? 172.21.15.26:0/676480685' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2023-09-23T20:41:08.662 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:41:09.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:41:08 smithi070 ceph-mon[100755]: pgmap v5: 1 pgs: 1 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:41:09.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:41:08 smithi070 ceph-mon[100755]: from='mgr.14451 ' entity='mgr.y' 2023-09-23T20:41:09.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:41:08 smithi070 ceph-mon[100755]: from='mgr.14451 ' entity='mgr.y' 2023-09-23T20:41:09.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:41:08 smithi070 ceph-mon[100755]: from='mgr.14451 ' entity='mgr.y' 2023-09-23T20:41:09.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:41:08 smithi070 ceph-mon[100755]: from='mgr.14451 ' entity='mgr.y' 2023-09-23T20:41:09.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:41:08 smithi070 ceph-mon[100755]: from='mgr.14451 172.21.15.26:0/1329296977' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:41:09.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:41:08 smithi070 ceph-mon[100755]: from='mgr.14451 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:41:09.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:41:08 smithi070 ceph-mon[100755]: from='mgr.14451 172.21.15.26:0/1329296977' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:41:09.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:41:08 smithi070 ceph-mon[100755]: from='mgr.14451 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:41:09.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:41:08 smithi070 ceph-mon[100755]: from='mgr.14451 172.21.15.26:0/1329296977' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:41:09.121 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:41:08 smithi070 ceph-mon[100755]: from='mgr.14451 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:41:09.121 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:41:08 smithi070 ceph-mon[100755]: from='mgr.14451 172.21.15.26:0/1329296977' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:41:09.121 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:41:08 smithi070 ceph-mon[100755]: from='mgr.14451 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:41:09.121 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:41:08 smithi070 ceph-mon[100755]: from='mgr.14451 ' entity='mgr.y' 2023-09-23T20:41:09.121 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:41:08 smithi070 ceph-mon[100755]: from='mgr.14451 172.21.15.26:0/1329296977' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:41:09.121 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:41:08 smithi070 ceph-mon[100755]: from='mgr.14451 172.21.15.26:0/1329296977' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:41:09.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:41:08 smithi026 ceph-mon[109912]: pgmap v5: 1 pgs: 1 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:41:09.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:41:08 smithi026 ceph-mon[109912]: from='mgr.14451 ' entity='mgr.y' 2023-09-23T20:41:09.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:41:08 smithi026 ceph-mon[109912]: from='mgr.14451 ' entity='mgr.y' 2023-09-23T20:41:09.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:41:08 smithi026 ceph-mon[109912]: from='mgr.14451 ' entity='mgr.y' 2023-09-23T20:41:09.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:41:08 smithi026 ceph-mon[109912]: from='mgr.14451 ' entity='mgr.y' 2023-09-23T20:41:09.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:41:08 smithi026 ceph-mon[109912]: from='mgr.14451 172.21.15.26:0/1329296977' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:41:09.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:41:08 smithi026 ceph-mon[109912]: from='mgr.14451 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:41:09.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:41:08 smithi026 ceph-mon[109912]: from='mgr.14451 172.21.15.26:0/1329296977' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:41:09.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:41:08 smithi026 ceph-mon[109912]: from='mgr.14451 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:41:09.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:41:08 smithi026 ceph-mon[109912]: from='mgr.14451 172.21.15.26:0/1329296977' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:41:09.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:41:08 smithi026 ceph-mon[109912]: from='mgr.14451 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:41:09.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:41:08 smithi026 ceph-mon[109912]: from='mgr.14451 172.21.15.26:0/1329296977' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:41:09.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:41:08 smithi026 ceph-mon[109912]: from='mgr.14451 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:41:09.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:41:08 smithi026 ceph-mon[109912]: from='mgr.14451 ' entity='mgr.y' 2023-09-23T20:41:09.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:41:08 smithi026 ceph-mon[109912]: from='mgr.14451 172.21.15.26:0/1329296977' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:41:09.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:41:08 smithi026 ceph-mon[109912]: from='mgr.14451 172.21.15.26:0/1329296977' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:41:09.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:41:08 smithi026 ceph-mon[95839]: pgmap v5: 1 pgs: 1 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:41:09.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:41:08 smithi026 ceph-mon[95839]: from='mgr.14451 ' entity='mgr.y' 2023-09-23T20:41:09.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:41:08 smithi026 ceph-mon[95839]: from='mgr.14451 ' entity='mgr.y' 2023-09-23T20:41:09.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:41:08 smithi026 ceph-mon[95839]: from='mgr.14451 ' entity='mgr.y' 2023-09-23T20:41:09.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:41:08 smithi026 ceph-mon[95839]: from='mgr.14451 ' entity='mgr.y' 2023-09-23T20:41:09.134 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:41:08 smithi026 ceph-mon[95839]: from='mgr.14451 172.21.15.26:0/1329296977' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:41:09.134 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:41:08 smithi026 ceph-mon[95839]: from='mgr.14451 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:41:09.134 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:41:08 smithi026 ceph-mon[95839]: from='mgr.14451 172.21.15.26:0/1329296977' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:41:09.134 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:41:08 smithi026 ceph-mon[95839]: from='mgr.14451 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:41:09.134 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:41:08 smithi026 ceph-mon[95839]: from='mgr.14451 172.21.15.26:0/1329296977' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:41:09.134 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:41:08 smithi026 ceph-mon[95839]: from='mgr.14451 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:41:09.134 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:41:08 smithi026 ceph-mon[95839]: from='mgr.14451 172.21.15.26:0/1329296977' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:41:09.134 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:41:08 smithi026 ceph-mon[95839]: from='mgr.14451 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:41:09.134 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:41:08 smithi026 ceph-mon[95839]: from='mgr.14451 ' entity='mgr.y' 2023-09-23T20:41:09.134 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:41:08 smithi026 ceph-mon[95839]: from='mgr.14451 172.21.15.26:0/1329296977' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:41:09.134 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:41:08 smithi026 ceph-mon[95839]: from='mgr.14451 172.21.15.26:0/1329296977' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:41:10.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:41:09 smithi070 ceph-mon[100755]: Adjusting osd_memory_target on smithi070 to 4036M 2023-09-23T20:41:10.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:41:09 smithi070 ceph-mon[100755]: Updating smithi026:/etc/ceph/ceph.conf 2023-09-23T20:41:10.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:41:09 smithi070 ceph-mon[100755]: Updating smithi070:/etc/ceph/ceph.conf 2023-09-23T20:41:10.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:41:09 smithi070 ceph-mon[100755]: Updating smithi070:/var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-23T20:41:10.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:41:09 smithi070 ceph-mon[100755]: Updating smithi026:/var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-23T20:41:10.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:41:09 smithi070 ceph-mon[100755]: Updating smithi070:/etc/ceph/ceph.client.admin.keyring 2023-09-23T20:41:10.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:41:09 smithi070 ceph-mon[100755]: Updating smithi026:/etc/ceph/ceph.client.admin.keyring 2023-09-23T20:41:10.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:41:09 smithi070 ceph-mon[100755]: from='mgr.14451 ' entity='mgr.y' 2023-09-23T20:41:10.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:41:09 smithi070 ceph-mon[100755]: from='mgr.14451 ' entity='mgr.y' 2023-09-23T20:41:10.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:41:09 smithi070 ceph-mon[100755]: from='mgr.14451 ' entity='mgr.y' 2023-09-23T20:41:10.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:41:09 smithi070 ceph-mon[100755]: from='mgr.14451 ' entity='mgr.y' 2023-09-23T20:41:10.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:41:09 smithi070 ceph-mon[100755]: from='mgr.14451 ' entity='mgr.y' 2023-09-23T20:41:10.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:41:09 smithi026 ceph-mon[95839]: Adjusting osd_memory_target on smithi070 to 4036M 2023-09-23T20:41:10.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:41:09 smithi026 ceph-mon[95839]: Updating smithi026:/etc/ceph/ceph.conf 2023-09-23T20:41:10.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:41:09 smithi026 ceph-mon[95839]: Updating smithi070:/etc/ceph/ceph.conf 2023-09-23T20:41:10.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:41:09 smithi026 ceph-mon[95839]: Updating smithi070:/var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-23T20:41:10.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:41:09 smithi026 ceph-mon[95839]: Updating smithi026:/var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-23T20:41:10.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:41:09 smithi026 ceph-mon[95839]: Updating smithi070:/etc/ceph/ceph.client.admin.keyring 2023-09-23T20:41:10.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:41:09 smithi026 ceph-mon[95839]: Updating smithi026:/etc/ceph/ceph.client.admin.keyring 2023-09-23T20:41:10.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:41:09 smithi026 ceph-mon[95839]: from='mgr.14451 ' entity='mgr.y' 2023-09-23T20:41:10.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:41:09 smithi026 ceph-mon[95839]: from='mgr.14451 ' entity='mgr.y' 2023-09-23T20:41:10.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:41:09 smithi026 ceph-mon[95839]: from='mgr.14451 ' entity='mgr.y' 2023-09-23T20:41:10.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:41:09 smithi026 ceph-mon[95839]: from='mgr.14451 ' entity='mgr.y' 2023-09-23T20:41:10.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:41:09 smithi026 ceph-mon[95839]: from='mgr.14451 ' entity='mgr.y' 2023-09-23T20:41:10.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:41:09 smithi026 ceph-mon[109912]: Adjusting osd_memory_target on smithi070 to 4036M 2023-09-23T20:41:10.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:41:09 smithi026 ceph-mon[109912]: Updating smithi026:/etc/ceph/ceph.conf 2023-09-23T20:41:10.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:41:09 smithi026 ceph-mon[109912]: Updating smithi070:/etc/ceph/ceph.conf 2023-09-23T20:41:10.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:41:09 smithi026 ceph-mon[109912]: Updating smithi070:/var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-23T20:41:10.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:41:09 smithi026 ceph-mon[109912]: Updating smithi026:/var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-23T20:41:10.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:41:09 smithi026 ceph-mon[109912]: Updating smithi070:/etc/ceph/ceph.client.admin.keyring 2023-09-23T20:41:10.134 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:41:09 smithi026 ceph-mon[109912]: Updating smithi026:/etc/ceph/ceph.client.admin.keyring 2023-09-23T20:41:10.134 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:41:09 smithi026 ceph-mon[109912]: from='mgr.14451 ' entity='mgr.y' 2023-09-23T20:41:10.134 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:41:09 smithi026 ceph-mon[109912]: from='mgr.14451 ' entity='mgr.y' 2023-09-23T20:41:10.134 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:41:09 smithi026 ceph-mon[109912]: from='mgr.14451 ' entity='mgr.y' 2023-09-23T20:41:10.134 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:41:09 smithi026 ceph-mon[109912]: from='mgr.14451 ' entity='mgr.y' 2023-09-23T20:41:10.134 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:41:09 smithi026 ceph-mon[109912]: from='mgr.14451 ' entity='mgr.y' 2023-09-23T20:41:10.285 INFO:teuthology.orchestra.run.smithi070.stdout:[client.1] 2023-09-23T20:41:10.285 INFO:teuthology.orchestra.run.smithi070.stdout: key = AQBmTQ9l/QzOEBAAdnmQ2PiAdJTOqCHg+Nd9cw== 2023-09-23T20:41:10.706 DEBUG:teuthology.orchestra.run.smithi070:> set -ex 2023-09-23T20:41:10.707 DEBUG:teuthology.orchestra.run.smithi070:> sudo dd of=/etc/ceph/ceph.client.1.keyring 2023-09-23T20:41:10.707 DEBUG:teuthology.orchestra.run.smithi070:> sudo chmod 0644 /etc/ceph/ceph.client.1.keyring 2023-09-23T20:41:10.789 INFO:tasks.cephadm:Waiting for OSDs to come up 2023-09-23T20:41:10.789 DEBUG:teuthology.orchestra.run.smithi026:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph --cluster ceph osd dump --format=json 2023-09-23T20:41:11.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:41:10 smithi070 ceph-mon[100755]: Updating smithi070:/var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/config/ceph.client.admin.keyring 2023-09-23T20:41:11.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:41:10 smithi070 ceph-mon[100755]: Updating smithi026:/var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/config/ceph.client.admin.keyring 2023-09-23T20:41:11.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:41:10 smithi070 ceph-mon[100755]: Deploying daemon alertmanager.a on smithi026 2023-09-23T20:41:11.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:41:10 smithi070 ceph-mon[100755]: pgmap v6: 1 pgs: 1 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:41:11.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:41:10 smithi070 ceph-mon[100755]: from='client.? 172.21.15.70:0/380994486' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2023-09-23T20:41:11.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:41:10 smithi070 ceph-mon[100755]: from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2023-09-23T20:41:11.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:41:10 smithi070 ceph-mon[100755]: from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2023-09-23T20:41:11.131 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:41:10 smithi026 ceph-mon[109912]: Updating smithi070:/var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/config/ceph.client.admin.keyring 2023-09-23T20:41:11.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:41:10 smithi026 ceph-mon[109912]: Updating smithi026:/var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/config/ceph.client.admin.keyring 2023-09-23T20:41:11.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:41:10 smithi026 ceph-mon[109912]: Deploying daemon alertmanager.a on smithi026 2023-09-23T20:41:11.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:41:10 smithi026 ceph-mon[109912]: pgmap v6: 1 pgs: 1 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:41:11.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:41:10 smithi026 ceph-mon[109912]: from='client.? 172.21.15.70:0/380994486' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2023-09-23T20:41:11.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:41:10 smithi026 ceph-mon[109912]: from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2023-09-23T20:41:11.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:41:10 smithi026 ceph-mon[109912]: from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2023-09-23T20:41:11.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:41:10 smithi026 ceph-mon[95839]: Updating smithi070:/var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/config/ceph.client.admin.keyring 2023-09-23T20:41:11.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:41:10 smithi026 ceph-mon[95839]: Updating smithi026:/var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/config/ceph.client.admin.keyring 2023-09-23T20:41:11.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:41:10 smithi026 ceph-mon[95839]: Deploying daemon alertmanager.a on smithi026 2023-09-23T20:41:11.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:41:10 smithi026 ceph-mon[95839]: pgmap v6: 1 pgs: 1 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:41:11.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:41:10 smithi026 ceph-mon[95839]: from='client.? 172.21.15.70:0/380994486' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2023-09-23T20:41:11.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:41:10 smithi026 ceph-mon[95839]: from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2023-09-23T20:41:11.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:41:10 smithi026 ceph-mon[95839]: from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2023-09-23T20:41:11.188 INFO:teuthology.misc.health.smithi026.stdout: 2023-09-23T20:41:11.188 INFO:teuthology.misc.health.smithi026.stdout:{"epoch":60,"fsid":"b3f8e4ec-5a50-11ee-8db4-212e2dc638e7","created":"2023-09-23T20:35:58.919705+0000","modified":"2023-09-23T20:41:03.790716+0000","last_up_change":"2023-09-23T20:40:38.528428+0000","last_in_change":"2023-09-23T20:40:23.597125+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2023-09-23T20:38:57.022844+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}}],"osds":[{"osd":0,"uuid":"f837feb7-060e-4965-9b48-bed9f3672d05","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6802","nonce":2638118366},{"type":"v1","addr":"172.21.15.26:6803","nonce":2638118366}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6804","nonce":2638118366},{"type":"v1","addr":"172.21.15.26:6805","nonce":2638118366}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6808","nonce":2638118366},{"type":"v1","addr":"172.21.15.26:6809","nonce":2638118366}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6806","nonce":2638118366},{"type":"v1","addr":"172.21.15.26:6807","nonce":2638118366}]},"public_addr":"172.21.15.26:6803/2638118366","cluster_addr":"172.21.15.26:6805/2638118366","heartbeat_back_addr":"172.21.15.26:6809/2638118366","heartbeat_front_addr":"172.21.15.26:6807/2638118366","state":["exists","up"]},{"osd":1,"uuid":"917f65aa-5963-4576-988a-396698b7c6a8","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":34,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6810","nonce":3250108152},{"type":"v1","addr":"172.21.15.26:6811","nonce":3250108152}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6812","nonce":3250108152},{"type":"v1","addr":"172.21.15.26:6813","nonce":3250108152}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6816","nonce":3250108152},{"type":"v1","addr":"172.21.15.26:6817","nonce":3250108152}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6814","nonce":3250108152},{"type":"v1","addr":"172.21.15.26:6815","nonce":3250108152}]},"public_addr":"172.21.15.26:6811/3250108152","cluster_addr":"172.21.15.26:6813/3250108152","heartbeat_back_addr":"172.21.15.26:6817/3250108152","heartbeat_front_addr":"172.21.15.26:6815/3250108152","state":["exists","up"]},{"osd":2,"uuid":"2e115328-76c6-478e-915c-5786903be49f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6818","nonce":1325135181},{"type":"v1","addr":"172.21.15.26:6819","nonce":1325135181}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6820","nonce":1325135181},{"type":"v1","addr":"172.21.15.26:6821","nonce":1325135181}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6824","nonce":1325135181},{"type":"v1","addr":"172.21.15.26:6825","nonce":1325135181}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6822","nonce":1325135181},{"type":"v1","addr":"172.21.15.26:6823","nonce":1325135181}]},"public_addr":"172.21.15.26:6819/1325135181","cluster_addr":"172.21.15.26:6821/1325135181","heartbeat_back_addr":"172.21.15.26:6825/1325135181","heartbeat_front_addr":"172.21.15.26:6823/1325135181","state":["exists","up"]},{"osd":3,"uuid":"c3dd2c3f-5cf5-4fb0-964c-1b8926f75c8a","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":26,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6826","nonce":3610585082},{"type":"v1","addr":"172.21.15.26:6827","nonce":3610585082}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6828","nonce":3610585082},{"type":"v1","addr":"172.21.15.26:6829","nonce":3610585082}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6832","nonce":3610585082},{"type":"v1","addr":"172.21.15.26:6833","nonce":3610585082}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6830","nonce":3610585082},{"type":"v1","addr":"172.21.15.26:6831","nonce":3610585082}]},"public_addr":"172.21.15.26:6827/3610585082","cluster_addr":"172.21.15.26:6829/3610585082","heartbeat_back_addr":"172.21.15.26:6833/3610585082","heartbeat_front_addr":"172.21.15.26:6831/3610585082","state":["exists","up"]},{"osd":4,"uuid":"a5f31996-1b38-491b-956b-5aebc72d0170","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":31,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6800","nonce":1528249138},{"type":"v1","addr":"172.21.15.70:6801","nonce":1528249138}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6802","nonce":1528249138},{"type":"v1","addr":"172.21.15.70:6803","nonce":1528249138}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6806","nonce":1528249138},{"type":"v1","addr":"172.21.15.70:6807","nonce":1528249138}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6804","nonce":1528249138},{"type":"v1","addr":"172.21.15.70:6805","nonce":1528249138}]},"public_addr":"172.21.15.70:6801/1528249138","cluster_addr":"172.21.15.70:6803/1528249138","heartbeat_back_addr":"172.21.15.70:6807/1528249138","heartbeat_front_addr":"172.21.15.70:6805/1528249138","state":["exists","up"]},{"osd":5,"uuid":"21dad198-b9db-42c6-8823-112df1ecd874","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":39,"up_thru":42,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6808","nonce":3802232906},{"type":"v1","addr":"172.21.15.70:6809","nonce":3802232906}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6810","nonce":3802232906},{"type":"v1","addr":"172.21.15.70:6811","nonce":3802232906}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6814","nonce":3802232906},{"type":"v1","addr":"172.21.15.70:6815","nonce":3802232906}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6812","nonce":3802232906},{"type":"v1","addr":"172.21.15.70:6813","nonce":3802232906}]},"public_addr":"172.21.15.70:6809/3802232906","cluster_addr":"172.21.15.70:6811/3802232906","heartbeat_back_addr":"172.21.15.70:6815/3802232906","heartbeat_front_addr":"172.21.15.70:6813/3802232906","state":["exists","up"]},{"osd":6,"uuid":"6f3be784-7149-4f13-81b5-98924e97c056","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":47,"up_thru":50,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6816","nonce":2944175278},{"type":"v1","addr":"172.21.15.70:6817","nonce":2944175278}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6818","nonce":2944175278},{"type":"v1","addr":"172.21.15.70:6819","nonce":2944175278}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6822","nonce":2944175278},{"type":"v1","addr":"172.21.15.70:6823","nonce":2944175278}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6820","nonce":2944175278},{"type":"v1","addr":"172.21.15.70:6821","nonce":2944175278}]},"public_addr":"172.21.15.70:6817/2944175278","cluster_addr":"172.21.15.70:6819/2944175278","heartbeat_back_addr":"172.21.15.70:6823/2944175278","heartbeat_front_addr":"172.21.15.70:6821/2944175278","state":["exists","up"]},{"osd":7,"uuid":"499f5f85-f4b2-40a7-ac0f-79cd22738d30","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":55,"up_thru":58,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6824","nonce":2824974165},{"type":"v1","addr":"172.21.15.70:6825","nonce":2824974165}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6826","nonce":2824974165},{"type":"v1","addr":"172.21.15.70:6827","nonce":2824974165}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6830","nonce":2824974165},{"type":"v1","addr":"172.21.15.70:6831","nonce":2824974165}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6828","nonce":2824974165},{"type":"v1","addr":"172.21.15.70:6829","nonce":2824974165}]},"public_addr":"172.21.15.70:6825/2824974165","cluster_addr":"172.21.15.70:6827/2824974165","heartbeat_back_addr":"172.21.15.70:6831/2824974165","heartbeat_front_addr":"172.21.15.70:6829/2824974165","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T20:38:10.875468+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T20:38:32.449963+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T20:38:53.633316+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T20:39:14.828940+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T20:39:34.945663+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T20:39:55.954288+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T20:40:16.847102+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T20:40:37.820404+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.26:6801/91363814":"2023-09-24T20:41:03.790648+0000","172.21.15.26:6800/91363814":"2023-09-24T20:41:03.790648+0000","172.21.15.26:0/3461760126":"2023-09-24T20:41:03.790648+0000","172.21.15.26:0/394447930":"2023-09-24T20:41:03.790648+0000","172.21.15.26:0/1083572999":"2023-09-24T20:41:03.790648+0000","172.21.15.26:0/2838335646":"2023-09-24T20:36:56.498559+0000","172.21.15.26:6801/952511372":"2023-09-24T20:36:56.498559+0000","172.21.15.26:6800/952511372":"2023-09-24T20:36:56.498559+0000","172.21.15.26:0/1625326392":"2023-09-24T20:36:56.498559+0000","172.21.15.26:0/3175144810":"2023-09-24T20:36:31.159649+0000","172.21.15.26:0/1815190465":"2023-09-24T20:36:31.159649+0000","172.21.15.26:0/188049283":"2023-09-24T20:41:03.790648+0000","172.21.15.26:0/289994511":"2023-09-24T20:36:56.498559+0000","172.21.15.26:0/686734212":"2023-09-24T20:36:31.159649+0000","172.21.15.26:6800/4099624907":"2023-09-24T20:36:31.159649+0000","172.21.15.26:6801/4099624907":"2023-09-24T20:36:31.159649+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2023-09-23T20:41:11.202 DEBUG:teuthology.misc:8 of 8 OSDs are up 2023-09-23T20:41:11.202 INFO:tasks.cephadm:Creating RBD pool 2023-09-23T20:41:11.202 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- sudo ceph --cluster ceph osd pool create rbd 8 2023-09-23T20:41:11.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:41:11 smithi070 ceph-mon[100755]: from='client.? 172.21.15.26:0/564392668' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-09-23T20:41:11.870 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 20:41:11 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-x[102861]: ::ffff:172.21.15.70 - - [23/Sep/2023:20:41:11] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2023-09-23T20:41:12.131 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:41:11 smithi026 ceph-mon[95839]: from='client.? 172.21.15.26:0/564392668' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-09-23T20:41:12.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:41:11 smithi026 ceph-mon[109912]: from='client.? 172.21.15.26:0/564392668' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-09-23T20:41:12.223 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:41:13.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:41:12 smithi070 ceph-mon[100755]: pgmap v7: 1 pgs: 1 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:41:13.125 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:41:12 smithi026 ceph-mon[95839]: pgmap v7: 1 pgs: 1 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:41:13.125 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:41:12 smithi026 ceph-mon[109912]: pgmap v7: 1 pgs: 1 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:41:14.658 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:41:14 smithi026 ceph-mon[95839]: pgmap v8: 1 pgs: 1 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:41:14.658 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:41:14 smithi026 ceph-mon[109912]: pgmap v8: 1 pgs: 1 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:41:14.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:41:14 smithi070 ceph-mon[100755]: pgmap v8: 1 pgs: 1 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:41:15.278 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Sep 23 20:41:14 smithi026 systemd[1]: Starting Ceph alertmanager.a for b3f8e4ec-5a50-11ee-8db4-212e2dc638e7... 2023-09-23T20:41:15.535 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Sep 23 20:41:15 smithi026 podman[143851]: 2023-09-23T20:41:15.559 INFO:teuthology.orchestra.run.smithi026.stderr:pool 'rbd' created 2023-09-23T20:41:15.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:41:15 smithi070 ceph-mon[100755]: from='client.? 172.21.15.26:0/1074882875' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]: dispatch 2023-09-23T20:41:15.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:41:15 smithi026 ceph-mon[95839]: from='client.? 172.21.15.26:0/1074882875' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]: dispatch 2023-09-23T20:41:15.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:41:15 smithi026 ceph-mon[109912]: from='client.? 172.21.15.26:0/1074882875' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]: dispatch 2023-09-23T20:41:15.882 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Sep 23 20:41:15 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-alertmanager-a[143867]: ts=2023-09-23T20:41:15.534Z caller=main.go:240 level=info msg="Starting Alertmanager" version="(version=0.25.0, branch=HEAD, revision=258fab7cdd551f2cf251ed0348f0ad7289aee789)" 2023-09-23T20:41:15.882 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Sep 23 20:41:15 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-alertmanager-a[143867]: ts=2023-09-23T20:41:15.534Z caller=main.go:241 level=info build_context="(go=go1.19.4, user=root@abe866dd5717, date=20221222-14:51:36)" 2023-09-23T20:41:15.882 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Sep 23 20:41:15 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-alertmanager-a[143867]: ts=2023-09-23T20:41:15.536Z caller=cluster.go:185 level=info component=cluster msg="setting advertise address explicitly" addr=172.21.15.26 port=9094 2023-09-23T20:41:15.882 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Sep 23 20:41:15 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-alertmanager-a[143867]: ts=2023-09-23T20:41:15.537Z caller=cluster.go:681 level=info component=cluster msg="Waiting for gossip to settle..." interval=2s 2023-09-23T20:41:15.883 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Sep 23 20:41:15 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-alertmanager-a[143867]: ts=2023-09-23T20:41:15.568Z caller=coordinator.go:113 level=info component=configuration msg="Loading configuration file" file=/etc/alertmanager/alertmanager.yml 2023-09-23T20:41:15.883 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Sep 23 20:41:15 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-alertmanager-a[143867]: ts=2023-09-23T20:41:15.569Z caller=coordinator.go:126 level=info component=configuration msg="Completed loading of configuration file" file=/etc/alertmanager/alertmanager.yml 2023-09-23T20:41:15.883 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Sep 23 20:41:15 smithi026 bash[143851]: b0f6e313eea3d5ddb06210a843226c9fb49c6634bfc0944dfdfae8e51e2d03a3 2023-09-23T20:41:15.883 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Sep 23 20:41:15 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-alertmanager-a[143867]: ts=2023-09-23T20:41:15.571Z caller=tls_config.go:232 level=info msg="Listening on" address=[::]:9093 2023-09-23T20:41:15.883 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Sep 23 20:41:15 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-alertmanager-a[143867]: ts=2023-09-23T20:41:15.571Z caller=tls_config.go:235 level=info msg="TLS is disabled." http2=false address=[::]:9093 2023-09-23T20:41:15.883 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Sep 23 20:41:15 smithi026 systemd[1]: Started Ceph alertmanager.a for b3f8e4ec-5a50-11ee-8db4-212e2dc638e7. 2023-09-23T20:41:15.998 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- sudo ceph --cluster ceph osd pool application enable rbd rbd --yes-i-really-mean-it 2023-09-23T20:41:16.640 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:41:16.825 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:41:16 smithi026 ceph-mon[95839]: from='client.? 172.21.15.26:0/1074882875' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]': finished 2023-09-23T20:41:16.826 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:41:16 smithi026 ceph-mon[95839]: osdmap e61: 8 total, 8 up, 8 in 2023-09-23T20:41:16.826 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:41:16 smithi026 ceph-mon[95839]: from='mgr.14451 ' entity='mgr.y' 2023-09-23T20:41:16.826 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:41:16 smithi026 ceph-mon[95839]: from='mgr.14451 ' entity='mgr.y' 2023-09-23T20:41:16.826 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:41:16 smithi026 ceph-mon[95839]: from='mgr.14451 ' entity='mgr.y' 2023-09-23T20:41:16.826 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:41:16 smithi026 ceph-mon[95839]: from='mgr.14451 ' entity='mgr.y' 2023-09-23T20:41:16.826 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:41:16 smithi026 ceph-mon[95839]: Deploying daemon node-exporter.a on smithi026 2023-09-23T20:41:16.826 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:41:16 smithi026 ceph-mon[95839]: pgmap v10: 9 pgs: 8 unknown, 1 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:41:16.827 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:41:16 smithi026 ceph-mon[109912]: from='client.? 172.21.15.26:0/1074882875' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]': finished 2023-09-23T20:41:16.827 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:41:16 smithi026 ceph-mon[109912]: osdmap e61: 8 total, 8 up, 8 in 2023-09-23T20:41:16.827 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:41:16 smithi026 ceph-mon[109912]: from='mgr.14451 ' entity='mgr.y' 2023-09-23T20:41:16.827 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:41:16 smithi026 ceph-mon[109912]: from='mgr.14451 ' entity='mgr.y' 2023-09-23T20:41:16.827 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:41:16 smithi026 ceph-mon[109912]: from='mgr.14451 ' entity='mgr.y' 2023-09-23T20:41:16.827 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:41:16 smithi026 ceph-mon[109912]: from='mgr.14451 ' entity='mgr.y' 2023-09-23T20:41:16.827 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:41:16 smithi026 ceph-mon[109912]: Deploying daemon node-exporter.a on smithi026 2023-09-23T20:41:16.827 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:41:16 smithi026 ceph-mon[109912]: pgmap v10: 9 pgs: 8 unknown, 1 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:41:16.827 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 23 20:41:16 smithi026 systemd[1]: Starting Ceph node-exporter.a for b3f8e4ec-5a50-11ee-8db4-212e2dc638e7... 2023-09-23T20:41:16.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:41:16 smithi070 ceph-mon[100755]: from='client.? 172.21.15.26:0/1074882875' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]': finished 2023-09-23T20:41:16.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:41:16 smithi070 ceph-mon[100755]: osdmap e61: 8 total, 8 up, 8 in 2023-09-23T20:41:16.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:41:16 smithi070 ceph-mon[100755]: from='mgr.14451 ' entity='mgr.y' 2023-09-23T20:41:16.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:41:16 smithi070 ceph-mon[100755]: from='mgr.14451 ' entity='mgr.y' 2023-09-23T20:41:16.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:41:16 smithi070 ceph-mon[100755]: from='mgr.14451 ' entity='mgr.y' 2023-09-23T20:41:16.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:41:16 smithi070 ceph-mon[100755]: from='mgr.14451 ' entity='mgr.y' 2023-09-23T20:41:16.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:41:16 smithi070 ceph-mon[100755]: Deploying daemon node-exporter.a on smithi026 2023-09-23T20:41:16.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:41:16 smithi070 ceph-mon[100755]: pgmap v10: 9 pgs: 8 unknown, 1 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:41:17.132 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 23 20:41:16 smithi026 bash[144213]: Trying to pull quay.io/prometheus/node-exporter:v1.5.0... 2023-09-23T20:41:17.538 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 23 20:41:17 smithi026 bash[144213]: Getting image source signatures 2023-09-23T20:41:17.538 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 23 20:41:17 smithi026 bash[144213]: Copying blob sha256:c0e87333d3805a7b7e1c25783dd25f27da0fe70a8e262eee7ed190c8dc843fa7 2023-09-23T20:41:17.538 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 23 20:41:17 smithi026 bash[144213]: Copying blob sha256:c0e87333d3805a7b7e1c25783dd25f27da0fe70a8e262eee7ed190c8dc843fa7 2023-09-23T20:41:17.538 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 23 20:41:17 smithi026 bash[144213]: Copying blob sha256:22b70bddd3acadc892fca4c2af4260629bfda5dfd11ebc106a93ce24e752b5ed 2023-09-23T20:41:17.538 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 23 20:41:17 smithi026 bash[144213]: Copying blob sha256:22b70bddd3acadc892fca4c2af4260629bfda5dfd11ebc106a93ce24e752b5ed 2023-09-23T20:41:17.538 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 23 20:41:17 smithi026 bash[144213]: Copying blob sha256:5c12815fee558b157dee7f7509dedbaba0a8379098858a65ec869e1f1526ea0c 2023-09-23T20:41:17.538 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 23 20:41:17 smithi026 bash[144213]: Copying blob sha256:5c12815fee558b157dee7f7509dedbaba0a8379098858a65ec869e1f1526ea0c 2023-09-23T20:41:17.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:41:17 smithi070 ceph-mon[100755]: osdmap e62: 8 total, 8 up, 8 in 2023-09-23T20:41:17.881 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:41:17 smithi026 ceph-mon[95839]: osdmap e62: 8 total, 8 up, 8 in 2023-09-23T20:41:17.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:41:17 smithi026 ceph-mon[109912]: osdmap e62: 8 total, 8 up, 8 in 2023-09-23T20:41:17.882 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Sep 23 20:41:17 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-alertmanager-a[143867]: ts=2023-09-23T20:41:17.537Z caller=cluster.go:706 level=info component=cluster msg="gossip not settled" polls=0 before=0 now=1 elapsed=2.000039998s 2023-09-23T20:41:18.567 INFO:teuthology.orchestra.run.smithi026.stderr:enabled application 'rbd' on pool 'rbd' 2023-09-23T20:41:18.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:41:18 smithi070 ceph-mon[100755]: osdmap e63: 8 total, 8 up, 8 in 2023-09-23T20:41:18.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:41:18 smithi070 ceph-mon[100755]: pgmap v13: 9 pgs: 8 unknown, 1 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:41:18.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:41:18 smithi070 ceph-mon[100755]: from='client.? 172.21.15.26:0/382618194' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T20:41:18.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:41:18 smithi070 ceph-mon[100755]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T20:41:18.882 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 23 20:41:18 smithi026 bash[144213]: Copying config sha256:0da6a335fe1356545476b749c68f022c897de3a2139e8f0054f6937349ee2b83 2023-09-23T20:41:18.882 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 23 20:41:18 smithi026 bash[144213]: Writing manifest to image destination 2023-09-23T20:41:18.882 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 23 20:41:18 smithi026 bash[144213]: Storing signatures 2023-09-23T20:41:18.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:41:18 smithi026 ceph-mon[95839]: osdmap e63: 8 total, 8 up, 8 in 2023-09-23T20:41:18.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:41:18 smithi026 ceph-mon[95839]: pgmap v13: 9 pgs: 8 unknown, 1 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:41:18.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:41:18 smithi026 ceph-mon[95839]: from='client.? 172.21.15.26:0/382618194' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T20:41:18.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:41:18 smithi026 ceph-mon[95839]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T20:41:18.883 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:41:18 smithi026 ceph-mon[109912]: osdmap e63: 8 total, 8 up, 8 in 2023-09-23T20:41:18.883 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:41:18 smithi026 ceph-mon[109912]: pgmap v13: 9 pgs: 8 unknown, 1 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:41:18.883 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:41:18 smithi026 ceph-mon[109912]: from='client.? 172.21.15.26:0/382618194' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T20:41:18.883 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:41:18 smithi026 ceph-mon[109912]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T20:41:19.232 INFO:tasks.ceph:Waiting until ceph daemons up and pgs clean... 2023-09-23T20:41:19.232 INFO:tasks.cephadm.ceph_manager.ceph:waiting for mgr available 2023-09-23T20:41:19.232 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph mgr dump --format=json 2023-09-23T20:41:19.381 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 23 20:41:19 smithi026 podman[144213]: 2023-09-23T20:41:19.854 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:41:19.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:41:19 smithi070 ceph-mon[100755]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]': finished 2023-09-23T20:41:19.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:41:19 smithi070 ceph-mon[100755]: osdmap e64: 8 total, 8 up, 8 in 2023-09-23T20:41:19.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:41:19 smithi070 ceph-mon[100755]: from='mgr.14451 ' entity='mgr.y' 2023-09-23T20:41:19.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:41:19 smithi070 ceph-mon[100755]: from='mgr.14451 172.21.15.26:0/1329296977' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:41:19.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:41:19 smithi026 ceph-mon[95839]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]': finished 2023-09-23T20:41:19.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:41:19 smithi026 ceph-mon[95839]: osdmap e64: 8 total, 8 up, 8 in 2023-09-23T20:41:19.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:41:19 smithi026 ceph-mon[95839]: from='mgr.14451 ' entity='mgr.y' 2023-09-23T20:41:19.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:41:19 smithi026 ceph-mon[95839]: from='mgr.14451 172.21.15.26:0/1329296977' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:41:19.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:41:19 smithi026 ceph-mon[109912]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]': finished 2023-09-23T20:41:19.883 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:41:19 smithi026 ceph-mon[109912]: osdmap e64: 8 total, 8 up, 8 in 2023-09-23T20:41:19.883 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:41:19 smithi026 ceph-mon[109912]: from='mgr.14451 ' entity='mgr.y' 2023-09-23T20:41:19.883 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:41:19 smithi026 ceph-mon[109912]: from='mgr.14451 172.21.15.26:0/1329296977' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:41:19.883 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 23 20:41:19 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-node-exporter-a[144430]: ts=2023-09-23T20:41:19.547Z caller=node_exporter.go:180 level=info msg="Starting node_exporter" version="(version=1.5.0, branch=HEAD, revision=1b48970ffcf5630534fb00bb0687d73c66d1c959)" 2023-09-23T20:41:19.884 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 23 20:41:19 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-node-exporter-a[144430]: ts=2023-09-23T20:41:19.547Z caller=node_exporter.go:181 level=info msg="Build context" build_context="(go=go1.19.3, user=root@6e7732a7b81b, date=20221129-18:59:09)" 2023-09-23T20:41:19.884 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 23 20:41:19 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-node-exporter-a[144430]: ts=2023-09-23T20:41:19.547Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+|var/lib/containers/storage/.+)($|/) 2023-09-23T20:41:19.884 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 23 20:41:19 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-node-exporter-a[144430]: ts=2023-09-23T20:41:19.547Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2023-09-23T20:41:19.884 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 23 20:41:19 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-node-exporter-a[144430]: ts=2023-09-23T20:41:19.547Z caller=diskstats_common.go:111 level=info collector=diskstats msg="Parsed flag --collector.diskstats.device-exclude" flag=^(ram|loop|fd|(h|s|v|xv)d[a-z]|nvme\d+n\d+p)\d+$ 2023-09-23T20:41:19.884 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 23 20:41:19 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-node-exporter-a[144430]: ts=2023-09-23T20:41:19.547Z caller=diskstats_linux.go:264 level=error collector=diskstats msg="Failed to open directory, disabling udev device properties" path=/run/udev/data 2023-09-23T20:41:19.884 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 23 20:41:19 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-node-exporter-a[144430]: ts=2023-09-23T20:41:19.548Z caller=node_exporter.go:110 level=info msg="Enabled collectors" 2023-09-23T20:41:19.884 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 23 20:41:19 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-node-exporter-a[144430]: ts=2023-09-23T20:41:19.548Z caller=node_exporter.go:117 level=info collector=arp 2023-09-23T20:41:19.885 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 23 20:41:19 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-node-exporter-a[144430]: ts=2023-09-23T20:41:19.548Z caller=node_exporter.go:117 level=info collector=bcache 2023-09-23T20:41:19.885 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 23 20:41:19 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-node-exporter-a[144430]: ts=2023-09-23T20:41:19.548Z caller=node_exporter.go:117 level=info collector=bonding 2023-09-23T20:41:19.885 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 23 20:41:19 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-node-exporter-a[144430]: ts=2023-09-23T20:41:19.548Z caller=node_exporter.go:117 level=info collector=btrfs 2023-09-23T20:41:19.885 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 23 20:41:19 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-node-exporter-a[144430]: ts=2023-09-23T20:41:19.548Z caller=node_exporter.go:117 level=info collector=conntrack 2023-09-23T20:41:19.885 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 23 20:41:19 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-node-exporter-a[144430]: ts=2023-09-23T20:41:19.548Z caller=node_exporter.go:117 level=info collector=cpu 2023-09-23T20:41:19.885 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 23 20:41:19 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-node-exporter-a[144430]: ts=2023-09-23T20:41:19.548Z caller=node_exporter.go:117 level=info collector=cpufreq 2023-09-23T20:41:19.885 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 23 20:41:19 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-node-exporter-a[144430]: ts=2023-09-23T20:41:19.548Z caller=node_exporter.go:117 level=info collector=diskstats 2023-09-23T20:41:19.885 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 23 20:41:19 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-node-exporter-a[144430]: ts=2023-09-23T20:41:19.548Z caller=node_exporter.go:117 level=info collector=dmi 2023-09-23T20:41:19.886 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 23 20:41:19 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-node-exporter-a[144430]: ts=2023-09-23T20:41:19.548Z caller=node_exporter.go:117 level=info collector=edac 2023-09-23T20:41:19.886 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 23 20:41:19 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-node-exporter-a[144430]: ts=2023-09-23T20:41:19.548Z caller=node_exporter.go:117 level=info collector=entropy 2023-09-23T20:41:19.886 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 23 20:41:19 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-node-exporter-a[144430]: ts=2023-09-23T20:41:19.548Z caller=node_exporter.go:117 level=info collector=fibrechannel 2023-09-23T20:41:19.886 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 23 20:41:19 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-node-exporter-a[144430]: ts=2023-09-23T20:41:19.548Z caller=node_exporter.go:117 level=info collector=filefd 2023-09-23T20:41:19.886 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 23 20:41:19 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-node-exporter-a[144430]: ts=2023-09-23T20:41:19.548Z caller=node_exporter.go:117 level=info collector=filesystem 2023-09-23T20:41:19.886 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 23 20:41:19 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-node-exporter-a[144430]: ts=2023-09-23T20:41:19.548Z caller=node_exporter.go:117 level=info collector=hwmon 2023-09-23T20:41:19.887 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 23 20:41:19 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-node-exporter-a[144430]: ts=2023-09-23T20:41:19.548Z caller=node_exporter.go:117 level=info collector=infiniband 2023-09-23T20:41:19.887 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 23 20:41:19 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-node-exporter-a[144430]: ts=2023-09-23T20:41:19.548Z caller=node_exporter.go:117 level=info collector=ipvs 2023-09-23T20:41:19.887 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 23 20:41:19 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-node-exporter-a[144430]: ts=2023-09-23T20:41:19.548Z caller=node_exporter.go:117 level=info collector=loadavg 2023-09-23T20:41:19.887 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 23 20:41:19 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-node-exporter-a[144430]: ts=2023-09-23T20:41:19.548Z caller=node_exporter.go:117 level=info collector=mdadm 2023-09-23T20:41:19.887 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 23 20:41:19 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-node-exporter-a[144430]: ts=2023-09-23T20:41:19.548Z caller=node_exporter.go:117 level=info collector=meminfo 2023-09-23T20:41:19.887 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 23 20:41:19 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-node-exporter-a[144430]: ts=2023-09-23T20:41:19.548Z caller=node_exporter.go:117 level=info collector=netclass 2023-09-23T20:41:19.887 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 23 20:41:19 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-node-exporter-a[144430]: ts=2023-09-23T20:41:19.548Z caller=node_exporter.go:117 level=info collector=netdev 2023-09-23T20:41:19.887 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 23 20:41:19 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-node-exporter-a[144430]: ts=2023-09-23T20:41:19.548Z caller=node_exporter.go:117 level=info collector=netstat 2023-09-23T20:41:19.887 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 23 20:41:19 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-node-exporter-a[144430]: ts=2023-09-23T20:41:19.548Z caller=node_exporter.go:117 level=info collector=nfs 2023-09-23T20:41:19.887 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 23 20:41:19 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-node-exporter-a[144430]: ts=2023-09-23T20:41:19.548Z caller=node_exporter.go:117 level=info collector=nfsd 2023-09-23T20:41:19.888 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 23 20:41:19 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-node-exporter-a[144430]: ts=2023-09-23T20:41:19.548Z caller=node_exporter.go:117 level=info collector=nvme 2023-09-23T20:41:19.888 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 23 20:41:19 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-node-exporter-a[144430]: ts=2023-09-23T20:41:19.548Z caller=node_exporter.go:117 level=info collector=os 2023-09-23T20:41:19.888 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 23 20:41:19 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-node-exporter-a[144430]: ts=2023-09-23T20:41:19.548Z caller=node_exporter.go:117 level=info collector=powersupplyclass 2023-09-23T20:41:19.888 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 23 20:41:19 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-node-exporter-a[144430]: ts=2023-09-23T20:41:19.548Z caller=node_exporter.go:117 level=info collector=pressure 2023-09-23T20:41:19.888 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 23 20:41:19 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-node-exporter-a[144430]: ts=2023-09-23T20:41:19.548Z caller=node_exporter.go:117 level=info collector=rapl 2023-09-23T20:41:19.888 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 23 20:41:19 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-node-exporter-a[144430]: ts=2023-09-23T20:41:19.548Z caller=node_exporter.go:117 level=info collector=schedstat 2023-09-23T20:41:19.888 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 23 20:41:19 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-node-exporter-a[144430]: ts=2023-09-23T20:41:19.548Z caller=node_exporter.go:117 level=info collector=selinux 2023-09-23T20:41:19.888 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 23 20:41:19 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-node-exporter-a[144430]: ts=2023-09-23T20:41:19.548Z caller=node_exporter.go:117 level=info collector=sockstat 2023-09-23T20:41:19.888 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 23 20:41:19 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-node-exporter-a[144430]: ts=2023-09-23T20:41:19.548Z caller=node_exporter.go:117 level=info collector=softnet 2023-09-23T20:41:19.889 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 23 20:41:19 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-node-exporter-a[144430]: ts=2023-09-23T20:41:19.548Z caller=node_exporter.go:117 level=info collector=stat 2023-09-23T20:41:19.889 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 23 20:41:19 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-node-exporter-a[144430]: ts=2023-09-23T20:41:19.548Z caller=node_exporter.go:117 level=info collector=tapestats 2023-09-23T20:41:19.889 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 23 20:41:19 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-node-exporter-a[144430]: ts=2023-09-23T20:41:19.548Z caller=node_exporter.go:117 level=info collector=textfile 2023-09-23T20:41:19.889 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 23 20:41:19 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-node-exporter-a[144430]: ts=2023-09-23T20:41:19.548Z caller=node_exporter.go:117 level=info collector=thermal_zone 2023-09-23T20:41:19.889 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 23 20:41:19 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-node-exporter-a[144430]: ts=2023-09-23T20:41:19.548Z caller=node_exporter.go:117 level=info collector=time 2023-09-23T20:41:19.889 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 23 20:41:19 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-node-exporter-a[144430]: ts=2023-09-23T20:41:19.548Z caller=node_exporter.go:117 level=info collector=udp_queues 2023-09-23T20:41:19.889 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 23 20:41:19 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-node-exporter-a[144430]: ts=2023-09-23T20:41:19.548Z caller=node_exporter.go:117 level=info collector=uname 2023-09-23T20:41:19.889 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 23 20:41:19 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-node-exporter-a[144430]: ts=2023-09-23T20:41:19.548Z caller=node_exporter.go:117 level=info collector=vmstat 2023-09-23T20:41:19.889 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 23 20:41:19 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-node-exporter-a[144430]: ts=2023-09-23T20:41:19.548Z caller=node_exporter.go:117 level=info collector=xfs 2023-09-23T20:41:19.889 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 23 20:41:19 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-node-exporter-a[144430]: ts=2023-09-23T20:41:19.548Z caller=node_exporter.go:117 level=info collector=zfs 2023-09-23T20:41:19.889 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 23 20:41:19 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-node-exporter-a[144430]: ts=2023-09-23T20:41:19.549Z caller=tls_config.go:232 level=info msg="Listening on" address=[::]:9100 2023-09-23T20:41:19.889 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 23 20:41:19 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-node-exporter-a[144430]: ts=2023-09-23T20:41:19.549Z caller=tls_config.go:235 level=info msg="TLS is disabled." http2=false address=[::]:9100 2023-09-23T20:41:19.889 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 23 20:41:19 smithi026 bash[144213]: 3e12faf27022a29014dfc40438b4f004f2c3d434afc6df4eeecf062919fc9120 2023-09-23T20:41:19.890 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 23 20:41:19 smithi026 systemd[1]: Started Ceph node-exporter.a for b3f8e4ec-5a50-11ee-8db4-212e2dc638e7. 2023-09-23T20:41:20.606 INFO:journalctl@ceph.node-exporter.b.smithi070.stdout:Sep 23 20:41:20 smithi070 systemd[1]: Starting Ceph node-exporter.b for b3f8e4ec-5a50-11ee-8db4-212e2dc638e7... 2023-09-23T20:41:20.650 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:41:20 smithi026 ceph-mon[95839]: from='mgr.14451 ' entity='mgr.y' 2023-09-23T20:41:20.651 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:41:20 smithi026 ceph-mon[109912]: from='mgr.14451 ' entity='mgr.y' 2023-09-23T20:41:20.651 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:41:20 smithi026 ceph-mon[109912]: from='mgr.14451 ' entity='mgr.y' 2023-09-23T20:41:20.651 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:41:20 smithi026 ceph-mon[109912]: from='mgr.14451 ' entity='mgr.y' 2023-09-23T20:41:20.651 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:41:20 smithi026 ceph-mon[109912]: Deploying daemon node-exporter.b on smithi070 2023-09-23T20:41:20.651 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:41:20 smithi026 ceph-mon[109912]: pgmap v15: 9 pgs: 8 unknown, 1 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:41:20.869 INFO:journalctl@ceph.node-exporter.b.smithi070.stdout:Sep 23 20:41:20 smithi070 bash[137018]: Trying to pull quay.io/prometheus/node-exporter:v1.5.0... 2023-09-23T20:41:20.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:41:20 smithi070 ceph-mon[100755]: from='mgr.14451 ' entity='mgr.y' 2023-09-23T20:41:20.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:41:20 smithi070 ceph-mon[100755]: from='mgr.14451 ' entity='mgr.y' 2023-09-23T20:41:20.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:41:20 smithi070 ceph-mon[100755]: from='mgr.14451 ' entity='mgr.y' 2023-09-23T20:41:20.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:41:20 smithi070 ceph-mon[100755]: Deploying daemon node-exporter.b on smithi070 2023-09-23T20:41:20.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:41:20 smithi070 ceph-mon[100755]: pgmap v15: 9 pgs: 8 unknown, 1 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:41:20.903 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:41:20 smithi026 ceph-mon[95839]: from='mgr.14451 ' entity='mgr.y' 2023-09-23T20:41:20.903 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:41:20 smithi026 ceph-mon[95839]: from='mgr.14451 ' entity='mgr.y' 2023-09-23T20:41:20.903 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:41:20 smithi026 ceph-mon[95839]: Deploying daemon node-exporter.b on smithi070 2023-09-23T20:41:20.903 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:41:20 smithi026 ceph-mon[95839]: pgmap v15: 9 pgs: 8 unknown, 1 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:41:21.313 INFO:teuthology.orchestra.run.smithi026.stdout: 2023-09-23T20:41:21.369 INFO:journalctl@ceph.node-exporter.b.smithi070.stdout:Sep 23 20:41:21 smithi070 bash[137018]: Getting image source signatures 2023-09-23T20:41:21.369 INFO:journalctl@ceph.node-exporter.b.smithi070.stdout:Sep 23 20:41:21 smithi070 bash[137018]: Copying blob sha256:c0e87333d3805a7b7e1c25783dd25f27da0fe70a8e262eee7ed190c8dc843fa7 2023-09-23T20:41:21.369 INFO:journalctl@ceph.node-exporter.b.smithi070.stdout:Sep 23 20:41:21 smithi070 bash[137018]: Copying blob sha256:c0e87333d3805a7b7e1c25783dd25f27da0fe70a8e262eee7ed190c8dc843fa7 2023-09-23T20:41:21.369 INFO:journalctl@ceph.node-exporter.b.smithi070.stdout:Sep 23 20:41:21 smithi070 bash[137018]: Copying blob sha256:22b70bddd3acadc892fca4c2af4260629bfda5dfd11ebc106a93ce24e752b5ed 2023-09-23T20:41:21.369 INFO:journalctl@ceph.node-exporter.b.smithi070.stdout:Sep 23 20:41:21 smithi070 bash[137018]: Copying blob sha256:5c12815fee558b157dee7f7509dedbaba0a8379098858a65ec869e1f1526ea0c 2023-09-23T20:41:21.369 INFO:journalctl@ceph.node-exporter.b.smithi070.stdout:Sep 23 20:41:21 smithi070 bash[137018]: Copying blob sha256:22b70bddd3acadc892fca4c2af4260629bfda5dfd11ebc106a93ce24e752b5ed 2023-09-23T20:41:21.370 INFO:journalctl@ceph.node-exporter.b.smithi070.stdout:Sep 23 20:41:21 smithi070 bash[137018]: Copying blob sha256:5c12815fee558b157dee7f7509dedbaba0a8379098858a65ec869e1f1526ea0c 2023-09-23T20:41:21.705 INFO:teuthology.orchestra.run.smithi026.stdout:{"epoch":20,"active_gid":14451,"active_name":"y","active_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6800","nonce":2440434873},{"type":"v1","addr":"172.21.15.26:6801","nonce":2440434873}]},"active_addr":"172.21.15.26:6801/2440434873","active_change":"2023-09-23T20:41:03.790839+0000","active_mgr_features":4540138320759226367,"available":true,"standbys":[{"gid":24346,"name":"x","mgr_features":4540138320759226367,"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2400","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"7","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.25.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:9.4.7","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.1.5","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.5.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.43.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"secs","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"noautoscale":{"name":"noautoscale","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"global autoscale flag","long_desc":"Option to turn on/off the autoscaler for all pools","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"drive_group_interval":{"name":"drive_group_interval","type":"float","level":"advanced","flags":0,"default_value":"300.0","min":"","max":"","enum_allowed":[],"desc":"interval in seconds between re-application of applied drive_groups","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}]}],"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2400","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"7","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.25.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:9.4.7","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.1.5","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.5.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.43.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"secs","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"noautoscale":{"name":"noautoscale","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"global autoscale flag","long_desc":"Option to turn on/off the autoscaler for all pools","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"drive_group_interval":{"name":"drive_group_interval","type":"float","level":"advanced","flags":0,"default_value":"300.0","min":"","max":"","enum_allowed":[],"desc":"interval in seconds between re-application of applied drive_groups","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}],"services":{"dashboard":"https://172.21.15.26:8443/","prometheus":"http://172.21.15.26:9283/"},"always_on_modules":{"octopus":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"pacific":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"quincy":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"]},"last_failure_osd_epoch":60,"active_clients":[{"name":"devicehealth","addrvec":[{"type":"v2","addr":"172.21.15.26:0","nonce":237900280}]},{"name":"libcephsqlite","addrvec":[{"type":"v2","addr":"172.21.15.26:0","nonce":1376232363}]},{"name":"rbd_support","addrvec":[{"type":"v2","addr":"172.21.15.26:0","nonce":4215395943}]},{"name":"volumes","addrvec":[{"type":"v2","addr":"172.21.15.26:0","nonce":635571892}]}]} 2023-09-23T20:41:21.709 INFO:tasks.cephadm.ceph_manager.ceph:mgr available! 2023-09-23T20:41:21.709 INFO:tasks.cephadm.ceph_manager.ceph:waiting for all up 2023-09-23T20:41:21.709 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph osd dump --format=json 2023-09-23T20:41:21.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:41:21 smithi070 ceph-mon[100755]: from='client.? 172.21.15.26:0/3773353367' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2023-09-23T20:41:21.870 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 20:41:21 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-x[102861]: ::ffff:172.21.15.70 - - [23/Sep/2023:20:41:21] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2023-09-23T20:41:22.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:41:21 smithi026 ceph-mon[95839]: from='client.? 172.21.15.26:0/3773353367' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2023-09-23T20:41:22.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:41:21 smithi026 ceph-mon[109912]: from='client.? 172.21.15.26:0/3773353367' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2023-09-23T20:41:22.315 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:41:22.845 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:41:22 smithi026 ceph-mon[95839]: pgmap v16: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:41:22.846 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:41:22 smithi026 ceph-mon[109912]: pgmap v16: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:41:23.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:41:22 smithi070 ceph-mon[100755]: pgmap v16: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:41:23.783 INFO:teuthology.orchestra.run.smithi026.stdout: 2023-09-23T20:41:23.783 INFO:teuthology.orchestra.run.smithi026.stdout:{"epoch":64,"fsid":"b3f8e4ec-5a50-11ee-8db4-212e2dc638e7","created":"2023-09-23T20:35:58.919705+0000","modified":"2023-09-23T20:41:18.557547+0000","last_up_change":"2023-09-23T20:40:38.528428+0000","last_in_change":"2023-09-23T20:40:23.597125+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":2,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2023-09-23T20:38:57.022844+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}},{"pool":2,"pool_name":"rbd","create_time":"2023-09-23T20:41:15.182110+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"64","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}}],"osds":[{"osd":0,"uuid":"f837feb7-060e-4965-9b48-bed9f3672d05","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6802","nonce":2638118366},{"type":"v1","addr":"172.21.15.26:6803","nonce":2638118366}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6804","nonce":2638118366},{"type":"v1","addr":"172.21.15.26:6805","nonce":2638118366}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6808","nonce":2638118366},{"type":"v1","addr":"172.21.15.26:6809","nonce":2638118366}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6806","nonce":2638118366},{"type":"v1","addr":"172.21.15.26:6807","nonce":2638118366}]},"public_addr":"172.21.15.26:6803/2638118366","cluster_addr":"172.21.15.26:6805/2638118366","heartbeat_back_addr":"172.21.15.26:6809/2638118366","heartbeat_front_addr":"172.21.15.26:6807/2638118366","state":["exists","up"]},{"osd":1,"uuid":"917f65aa-5963-4576-988a-396698b7c6a8","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":61,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6810","nonce":3250108152},{"type":"v1","addr":"172.21.15.26:6811","nonce":3250108152}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6812","nonce":3250108152},{"type":"v1","addr":"172.21.15.26:6813","nonce":3250108152}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6816","nonce":3250108152},{"type":"v1","addr":"172.21.15.26:6817","nonce":3250108152}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6814","nonce":3250108152},{"type":"v1","addr":"172.21.15.26:6815","nonce":3250108152}]},"public_addr":"172.21.15.26:6811/3250108152","cluster_addr":"172.21.15.26:6813/3250108152","heartbeat_back_addr":"172.21.15.26:6817/3250108152","heartbeat_front_addr":"172.21.15.26:6815/3250108152","state":["exists","up"]},{"osd":2,"uuid":"2e115328-76c6-478e-915c-5786903be49f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":61,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6818","nonce":1325135181},{"type":"v1","addr":"172.21.15.26:6819","nonce":1325135181}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6820","nonce":1325135181},{"type":"v1","addr":"172.21.15.26:6821","nonce":1325135181}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6824","nonce":1325135181},{"type":"v1","addr":"172.21.15.26:6825","nonce":1325135181}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6822","nonce":1325135181},{"type":"v1","addr":"172.21.15.26:6823","nonce":1325135181}]},"public_addr":"172.21.15.26:6819/1325135181","cluster_addr":"172.21.15.26:6821/1325135181","heartbeat_back_addr":"172.21.15.26:6825/1325135181","heartbeat_front_addr":"172.21.15.26:6823/1325135181","state":["exists","up"]},{"osd":3,"uuid":"c3dd2c3f-5cf5-4fb0-964c-1b8926f75c8a","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":26,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6826","nonce":3610585082},{"type":"v1","addr":"172.21.15.26:6827","nonce":3610585082}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6828","nonce":3610585082},{"type":"v1","addr":"172.21.15.26:6829","nonce":3610585082}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6832","nonce":3610585082},{"type":"v1","addr":"172.21.15.26:6833","nonce":3610585082}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6830","nonce":3610585082},{"type":"v1","addr":"172.21.15.26:6831","nonce":3610585082}]},"public_addr":"172.21.15.26:6827/3610585082","cluster_addr":"172.21.15.26:6829/3610585082","heartbeat_back_addr":"172.21.15.26:6833/3610585082","heartbeat_front_addr":"172.21.15.26:6831/3610585082","state":["exists","up"]},{"osd":4,"uuid":"a5f31996-1b38-491b-956b-5aebc72d0170","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":31,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6800","nonce":1528249138},{"type":"v1","addr":"172.21.15.70:6801","nonce":1528249138}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6802","nonce":1528249138},{"type":"v1","addr":"172.21.15.70:6803","nonce":1528249138}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6806","nonce":1528249138},{"type":"v1","addr":"172.21.15.70:6807","nonce":1528249138}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6804","nonce":1528249138},{"type":"v1","addr":"172.21.15.70:6805","nonce":1528249138}]},"public_addr":"172.21.15.70:6801/1528249138","cluster_addr":"172.21.15.70:6803/1528249138","heartbeat_back_addr":"172.21.15.70:6807/1528249138","heartbeat_front_addr":"172.21.15.70:6805/1528249138","state":["exists","up"]},{"osd":5,"uuid":"21dad198-b9db-42c6-8823-112df1ecd874","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":39,"up_thru":61,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6808","nonce":3802232906},{"type":"v1","addr":"172.21.15.70:6809","nonce":3802232906}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6810","nonce":3802232906},{"type":"v1","addr":"172.21.15.70:6811","nonce":3802232906}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6814","nonce":3802232906},{"type":"v1","addr":"172.21.15.70:6815","nonce":3802232906}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6812","nonce":3802232906},{"type":"v1","addr":"172.21.15.70:6813","nonce":3802232906}]},"public_addr":"172.21.15.70:6809/3802232906","cluster_addr":"172.21.15.70:6811/3802232906","heartbeat_back_addr":"172.21.15.70:6815/3802232906","heartbeat_front_addr":"172.21.15.70:6813/3802232906","state":["exists","up"]},{"osd":6,"uuid":"6f3be784-7149-4f13-81b5-98924e97c056","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":47,"up_thru":61,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6816","nonce":2944175278},{"type":"v1","addr":"172.21.15.70:6817","nonce":2944175278}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6818","nonce":2944175278},{"type":"v1","addr":"172.21.15.70:6819","nonce":2944175278}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6822","nonce":2944175278},{"type":"v1","addr":"172.21.15.70:6823","nonce":2944175278}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6820","nonce":2944175278},{"type":"v1","addr":"172.21.15.70:6821","nonce":2944175278}]},"public_addr":"172.21.15.70:6817/2944175278","cluster_addr":"172.21.15.70:6819/2944175278","heartbeat_back_addr":"172.21.15.70:6823/2944175278","heartbeat_front_addr":"172.21.15.70:6821/2944175278","state":["exists","up"]},{"osd":7,"uuid":"499f5f85-f4b2-40a7-ac0f-79cd22738d30","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":55,"up_thru":61,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6824","nonce":2824974165},{"type":"v1","addr":"172.21.15.70:6825","nonce":2824974165}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6826","nonce":2824974165},{"type":"v1","addr":"172.21.15.70:6827","nonce":2824974165}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6830","nonce":2824974165},{"type":"v1","addr":"172.21.15.70:6831","nonce":2824974165}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6828","nonce":2824974165},{"type":"v1","addr":"172.21.15.70:6829","nonce":2824974165}]},"public_addr":"172.21.15.70:6825/2824974165","cluster_addr":"172.21.15.70:6827/2824974165","heartbeat_back_addr":"172.21.15.70:6831/2824974165","heartbeat_front_addr":"172.21.15.70:6829/2824974165","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T20:38:10.875468+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T20:38:32.449963+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T20:38:53.633316+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T20:39:14.828940+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T20:39:34.945663+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T20:39:55.954288+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T20:40:16.847102+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T20:40:37.820404+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.26:6801/91363814":"2023-09-24T20:41:03.790648+0000","172.21.15.26:6800/91363814":"2023-09-24T20:41:03.790648+0000","172.21.15.26:0/3461760126":"2023-09-24T20:41:03.790648+0000","172.21.15.26:0/394447930":"2023-09-24T20:41:03.790648+0000","172.21.15.26:0/1083572999":"2023-09-24T20:41:03.790648+0000","172.21.15.26:0/2838335646":"2023-09-24T20:36:56.498559+0000","172.21.15.26:6801/952511372":"2023-09-24T20:36:56.498559+0000","172.21.15.26:6800/952511372":"2023-09-24T20:36:56.498559+0000","172.21.15.26:0/1625326392":"2023-09-24T20:36:56.498559+0000","172.21.15.26:0/3175144810":"2023-09-24T20:36:31.159649+0000","172.21.15.26:0/1815190465":"2023-09-24T20:36:31.159649+0000","172.21.15.26:0/188049283":"2023-09-24T20:41:03.790648+0000","172.21.15.26:0/289994511":"2023-09-24T20:36:56.498559+0000","172.21.15.26:0/686734212":"2023-09-24T20:36:31.159649+0000","172.21.15.26:6800/4099624907":"2023-09-24T20:36:31.159649+0000","172.21.15.26:6801/4099624907":"2023-09-24T20:36:31.159649+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2023-09-23T20:41:24.119 INFO:journalctl@ceph.node-exporter.b.smithi070.stdout:Sep 23 20:41:23 smithi070 bash[137018]: Copying config sha256:0da6a335fe1356545476b749c68f022c897de3a2139e8f0054f6937349ee2b83 2023-09-23T20:41:24.120 INFO:journalctl@ceph.node-exporter.b.smithi070.stdout:Sep 23 20:41:23 smithi070 bash[137018]: Writing manifest to image destination 2023-09-23T20:41:24.120 INFO:journalctl@ceph.node-exporter.b.smithi070.stdout:Sep 23 20:41:23 smithi070 bash[137018]: Storing signatures 2023-09-23T20:41:24.176 INFO:tasks.cephadm.ceph_manager.ceph:all up! 2023-09-23T20:41:24.177 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph osd dump --format=json 2023-09-23T20:41:24.451 INFO:journalctl@ceph.node-exporter.b.smithi070.stdout:Sep 23 20:41:24 smithi070 podman[137018]: 2023-09-23T20:41:24.451 INFO:journalctl@ceph.node-exporter.b.smithi070.stdout:Sep 23 20:41:24 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-node-exporter-b[137069]: ts=2023-09-23T20:41:24.420Z caller=node_exporter.go:180 level=info msg="Starting node_exporter" version="(version=1.5.0, branch=HEAD, revision=1b48970ffcf5630534fb00bb0687d73c66d1c959)" 2023-09-23T20:41:24.451 INFO:journalctl@ceph.node-exporter.b.smithi070.stdout:Sep 23 20:41:24 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-node-exporter-b[137069]: ts=2023-09-23T20:41:24.420Z caller=node_exporter.go:181 level=info msg="Build context" build_context="(go=go1.19.3, user=root@6e7732a7b81b, date=20221129-18:59:09)" 2023-09-23T20:41:24.452 INFO:journalctl@ceph.node-exporter.b.smithi070.stdout:Sep 23 20:41:24 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-node-exporter-b[137069]: ts=2023-09-23T20:41:24.421Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+|var/lib/containers/storage/.+)($|/) 2023-09-23T20:41:24.452 INFO:journalctl@ceph.node-exporter.b.smithi070.stdout:Sep 23 20:41:24 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-node-exporter-b[137069]: ts=2023-09-23T20:41:24.421Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2023-09-23T20:41:24.452 INFO:journalctl@ceph.node-exporter.b.smithi070.stdout:Sep 23 20:41:24 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-node-exporter-b[137069]: ts=2023-09-23T20:41:24.421Z caller=diskstats_common.go:111 level=info collector=diskstats msg="Parsed flag --collector.diskstats.device-exclude" flag=^(ram|loop|fd|(h|s|v|xv)d[a-z]|nvme\d+n\d+p)\d+$ 2023-09-23T20:41:24.452 INFO:journalctl@ceph.node-exporter.b.smithi070.stdout:Sep 23 20:41:24 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-node-exporter-b[137069]: ts=2023-09-23T20:41:24.421Z caller=diskstats_linux.go:264 level=error collector=diskstats msg="Failed to open directory, disabling udev device properties" path=/run/udev/data 2023-09-23T20:41:24.452 INFO:journalctl@ceph.node-exporter.b.smithi070.stdout:Sep 23 20:41:24 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-node-exporter-b[137069]: ts=2023-09-23T20:41:24.421Z caller=node_exporter.go:110 level=info msg="Enabled collectors" 2023-09-23T20:41:24.452 INFO:journalctl@ceph.node-exporter.b.smithi070.stdout:Sep 23 20:41:24 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-node-exporter-b[137069]: ts=2023-09-23T20:41:24.421Z caller=node_exporter.go:117 level=info collector=arp 2023-09-23T20:41:24.452 INFO:journalctl@ceph.node-exporter.b.smithi070.stdout:Sep 23 20:41:24 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-node-exporter-b[137069]: ts=2023-09-23T20:41:24.421Z caller=node_exporter.go:117 level=info collector=bcache 2023-09-23T20:41:24.452 INFO:journalctl@ceph.node-exporter.b.smithi070.stdout:Sep 23 20:41:24 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-node-exporter-b[137069]: ts=2023-09-23T20:41:24.421Z caller=node_exporter.go:117 level=info collector=bonding 2023-09-23T20:41:24.452 INFO:journalctl@ceph.node-exporter.b.smithi070.stdout:Sep 23 20:41:24 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-node-exporter-b[137069]: ts=2023-09-23T20:41:24.421Z caller=node_exporter.go:117 level=info collector=btrfs 2023-09-23T20:41:24.452 INFO:journalctl@ceph.node-exporter.b.smithi070.stdout:Sep 23 20:41:24 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-node-exporter-b[137069]: ts=2023-09-23T20:41:24.421Z caller=node_exporter.go:117 level=info collector=conntrack 2023-09-23T20:41:24.452 INFO:journalctl@ceph.node-exporter.b.smithi070.stdout:Sep 23 20:41:24 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-node-exporter-b[137069]: ts=2023-09-23T20:41:24.421Z caller=node_exporter.go:117 level=info collector=cpu 2023-09-23T20:41:24.453 INFO:journalctl@ceph.node-exporter.b.smithi070.stdout:Sep 23 20:41:24 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-node-exporter-b[137069]: ts=2023-09-23T20:41:24.421Z caller=node_exporter.go:117 level=info collector=cpufreq 2023-09-23T20:41:24.453 INFO:journalctl@ceph.node-exporter.b.smithi070.stdout:Sep 23 20:41:24 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-node-exporter-b[137069]: ts=2023-09-23T20:41:24.421Z caller=node_exporter.go:117 level=info collector=diskstats 2023-09-23T20:41:24.453 INFO:journalctl@ceph.node-exporter.b.smithi070.stdout:Sep 23 20:41:24 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-node-exporter-b[137069]: ts=2023-09-23T20:41:24.421Z caller=node_exporter.go:117 level=info collector=dmi 2023-09-23T20:41:24.453 INFO:journalctl@ceph.node-exporter.b.smithi070.stdout:Sep 23 20:41:24 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-node-exporter-b[137069]: ts=2023-09-23T20:41:24.421Z caller=node_exporter.go:117 level=info collector=edac 2023-09-23T20:41:24.453 INFO:journalctl@ceph.node-exporter.b.smithi070.stdout:Sep 23 20:41:24 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-node-exporter-b[137069]: ts=2023-09-23T20:41:24.421Z caller=node_exporter.go:117 level=info collector=entropy 2023-09-23T20:41:24.453 INFO:journalctl@ceph.node-exporter.b.smithi070.stdout:Sep 23 20:41:24 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-node-exporter-b[137069]: ts=2023-09-23T20:41:24.421Z caller=node_exporter.go:117 level=info collector=fibrechannel 2023-09-23T20:41:24.453 INFO:journalctl@ceph.node-exporter.b.smithi070.stdout:Sep 23 20:41:24 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-node-exporter-b[137069]: ts=2023-09-23T20:41:24.421Z caller=node_exporter.go:117 level=info collector=filefd 2023-09-23T20:41:24.453 INFO:journalctl@ceph.node-exporter.b.smithi070.stdout:Sep 23 20:41:24 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-node-exporter-b[137069]: ts=2023-09-23T20:41:24.421Z caller=node_exporter.go:117 level=info collector=filesystem 2023-09-23T20:41:24.453 INFO:journalctl@ceph.node-exporter.b.smithi070.stdout:Sep 23 20:41:24 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-node-exporter-b[137069]: ts=2023-09-23T20:41:24.421Z caller=node_exporter.go:117 level=info collector=hwmon 2023-09-23T20:41:24.453 INFO:journalctl@ceph.node-exporter.b.smithi070.stdout:Sep 23 20:41:24 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-node-exporter-b[137069]: ts=2023-09-23T20:41:24.421Z caller=node_exporter.go:117 level=info collector=infiniband 2023-09-23T20:41:24.453 INFO:journalctl@ceph.node-exporter.b.smithi070.stdout:Sep 23 20:41:24 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-node-exporter-b[137069]: ts=2023-09-23T20:41:24.421Z caller=node_exporter.go:117 level=info collector=ipvs 2023-09-23T20:41:24.453 INFO:journalctl@ceph.node-exporter.b.smithi070.stdout:Sep 23 20:41:24 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-node-exporter-b[137069]: ts=2023-09-23T20:41:24.421Z caller=node_exporter.go:117 level=info collector=loadavg 2023-09-23T20:41:24.454 INFO:journalctl@ceph.node-exporter.b.smithi070.stdout:Sep 23 20:41:24 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-node-exporter-b[137069]: ts=2023-09-23T20:41:24.421Z caller=node_exporter.go:117 level=info collector=mdadm 2023-09-23T20:41:24.454 INFO:journalctl@ceph.node-exporter.b.smithi070.stdout:Sep 23 20:41:24 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-node-exporter-b[137069]: ts=2023-09-23T20:41:24.421Z caller=node_exporter.go:117 level=info collector=meminfo 2023-09-23T20:41:24.454 INFO:journalctl@ceph.node-exporter.b.smithi070.stdout:Sep 23 20:41:24 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-node-exporter-b[137069]: ts=2023-09-23T20:41:24.421Z caller=node_exporter.go:117 level=info collector=netclass 2023-09-23T20:41:24.454 INFO:journalctl@ceph.node-exporter.b.smithi070.stdout:Sep 23 20:41:24 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-node-exporter-b[137069]: ts=2023-09-23T20:41:24.421Z caller=node_exporter.go:117 level=info collector=netdev 2023-09-23T20:41:24.454 INFO:journalctl@ceph.node-exporter.b.smithi070.stdout:Sep 23 20:41:24 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-node-exporter-b[137069]: ts=2023-09-23T20:41:24.421Z caller=node_exporter.go:117 level=info collector=netstat 2023-09-23T20:41:24.454 INFO:journalctl@ceph.node-exporter.b.smithi070.stdout:Sep 23 20:41:24 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-node-exporter-b[137069]: ts=2023-09-23T20:41:24.421Z caller=node_exporter.go:117 level=info collector=nfs 2023-09-23T20:41:24.454 INFO:journalctl@ceph.node-exporter.b.smithi070.stdout:Sep 23 20:41:24 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-node-exporter-b[137069]: ts=2023-09-23T20:41:24.421Z caller=node_exporter.go:117 level=info collector=nfsd 2023-09-23T20:41:24.454 INFO:journalctl@ceph.node-exporter.b.smithi070.stdout:Sep 23 20:41:24 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-node-exporter-b[137069]: ts=2023-09-23T20:41:24.421Z caller=node_exporter.go:117 level=info collector=nvme 2023-09-23T20:41:24.454 INFO:journalctl@ceph.node-exporter.b.smithi070.stdout:Sep 23 20:41:24 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-node-exporter-b[137069]: ts=2023-09-23T20:41:24.421Z caller=node_exporter.go:117 level=info collector=os 2023-09-23T20:41:24.454 INFO:journalctl@ceph.node-exporter.b.smithi070.stdout:Sep 23 20:41:24 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-node-exporter-b[137069]: ts=2023-09-23T20:41:24.421Z caller=node_exporter.go:117 level=info collector=powersupplyclass 2023-09-23T20:41:24.454 INFO:journalctl@ceph.node-exporter.b.smithi070.stdout:Sep 23 20:41:24 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-node-exporter-b[137069]: ts=2023-09-23T20:41:24.421Z caller=node_exporter.go:117 level=info collector=pressure 2023-09-23T20:41:24.454 INFO:journalctl@ceph.node-exporter.b.smithi070.stdout:Sep 23 20:41:24 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-node-exporter-b[137069]: ts=2023-09-23T20:41:24.421Z caller=node_exporter.go:117 level=info collector=rapl 2023-09-23T20:41:24.455 INFO:journalctl@ceph.node-exporter.b.smithi070.stdout:Sep 23 20:41:24 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-node-exporter-b[137069]: ts=2023-09-23T20:41:24.421Z caller=node_exporter.go:117 level=info collector=schedstat 2023-09-23T20:41:24.455 INFO:journalctl@ceph.node-exporter.b.smithi070.stdout:Sep 23 20:41:24 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-node-exporter-b[137069]: ts=2023-09-23T20:41:24.421Z caller=node_exporter.go:117 level=info collector=selinux 2023-09-23T20:41:24.455 INFO:journalctl@ceph.node-exporter.b.smithi070.stdout:Sep 23 20:41:24 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-node-exporter-b[137069]: ts=2023-09-23T20:41:24.421Z caller=node_exporter.go:117 level=info collector=sockstat 2023-09-23T20:41:24.455 INFO:journalctl@ceph.node-exporter.b.smithi070.stdout:Sep 23 20:41:24 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-node-exporter-b[137069]: ts=2023-09-23T20:41:24.421Z caller=node_exporter.go:117 level=info collector=softnet 2023-09-23T20:41:24.455 INFO:journalctl@ceph.node-exporter.b.smithi070.stdout:Sep 23 20:41:24 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-node-exporter-b[137069]: ts=2023-09-23T20:41:24.421Z caller=node_exporter.go:117 level=info collector=stat 2023-09-23T20:41:24.455 INFO:journalctl@ceph.node-exporter.b.smithi070.stdout:Sep 23 20:41:24 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-node-exporter-b[137069]: ts=2023-09-23T20:41:24.421Z caller=node_exporter.go:117 level=info collector=tapestats 2023-09-23T20:41:24.455 INFO:journalctl@ceph.node-exporter.b.smithi070.stdout:Sep 23 20:41:24 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-node-exporter-b[137069]: ts=2023-09-23T20:41:24.421Z caller=node_exporter.go:117 level=info collector=textfile 2023-09-23T20:41:24.455 INFO:journalctl@ceph.node-exporter.b.smithi070.stdout:Sep 23 20:41:24 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-node-exporter-b[137069]: ts=2023-09-23T20:41:24.421Z caller=node_exporter.go:117 level=info collector=thermal_zone 2023-09-23T20:41:24.455 INFO:journalctl@ceph.node-exporter.b.smithi070.stdout:Sep 23 20:41:24 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-node-exporter-b[137069]: ts=2023-09-23T20:41:24.421Z caller=node_exporter.go:117 level=info collector=time 2023-09-23T20:41:24.455 INFO:journalctl@ceph.node-exporter.b.smithi070.stdout:Sep 23 20:41:24 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-node-exporter-b[137069]: ts=2023-09-23T20:41:24.421Z caller=node_exporter.go:117 level=info collector=udp_queues 2023-09-23T20:41:24.455 INFO:journalctl@ceph.node-exporter.b.smithi070.stdout:Sep 23 20:41:24 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-node-exporter-b[137069]: ts=2023-09-23T20:41:24.421Z caller=node_exporter.go:117 level=info collector=uname 2023-09-23T20:41:24.455 INFO:journalctl@ceph.node-exporter.b.smithi070.stdout:Sep 23 20:41:24 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-node-exporter-b[137069]: ts=2023-09-23T20:41:24.421Z caller=node_exporter.go:117 level=info collector=vmstat 2023-09-23T20:41:24.456 INFO:journalctl@ceph.node-exporter.b.smithi070.stdout:Sep 23 20:41:24 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-node-exporter-b[137069]: ts=2023-09-23T20:41:24.421Z caller=node_exporter.go:117 level=info collector=xfs 2023-09-23T20:41:24.456 INFO:journalctl@ceph.node-exporter.b.smithi070.stdout:Sep 23 20:41:24 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-node-exporter-b[137069]: ts=2023-09-23T20:41:24.421Z caller=node_exporter.go:117 level=info collector=zfs 2023-09-23T20:41:24.456 INFO:journalctl@ceph.node-exporter.b.smithi070.stdout:Sep 23 20:41:24 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-node-exporter-b[137069]: ts=2023-09-23T20:41:24.422Z caller=tls_config.go:232 level=info msg="Listening on" address=[::]:9100 2023-09-23T20:41:24.456 INFO:journalctl@ceph.node-exporter.b.smithi070.stdout:Sep 23 20:41:24 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-node-exporter-b[137069]: ts=2023-09-23T20:41:24.422Z caller=tls_config.go:235 level=info msg="TLS is disabled." http2=false address=[::]:9100 2023-09-23T20:41:24.456 INFO:journalctl@ceph.node-exporter.b.smithi070.stdout:Sep 23 20:41:24 smithi070 bash[137018]: 8305df31701d900e829f9e1abb06df9fae1d24846fc6c239ef27fc5f7680cf66 2023-09-23T20:41:24.790 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:41:24.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:41:24 smithi070 ceph-mon[100755]: from='client.? 172.21.15.26:0/1412414632' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-09-23T20:41:24.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:41:24 smithi070 ceph-mon[100755]: pgmap v17: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:41:24.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:41:24 smithi070 ceph-mon[100755]: from='mgr.14451 ' entity='mgr.y' 2023-09-23T20:41:24.870 INFO:journalctl@ceph.node-exporter.b.smithi070.stdout:Sep 23 20:41:24 smithi070 systemd[1]: Started Ceph node-exporter.b for b3f8e4ec-5a50-11ee-8db4-212e2dc638e7. 2023-09-23T20:41:24.881 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:41:24 smithi026 ceph-mon[95839]: from='client.? 172.21.15.26:0/1412414632' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-09-23T20:41:24.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:41:24 smithi026 ceph-mon[95839]: pgmap v17: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:41:24.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:41:24 smithi026 ceph-mon[95839]: from='mgr.14451 ' entity='mgr.y' 2023-09-23T20:41:24.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:41:24 smithi026 ceph-mon[109912]: from='client.? 172.21.15.26:0/1412414632' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-09-23T20:41:24.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:41:24 smithi026 ceph-mon[109912]: pgmap v17: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:41:24.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:41:24 smithi026 ceph-mon[109912]: from='mgr.14451 ' entity='mgr.y' 2023-09-23T20:41:24.883 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:41:24 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[96086]: [23/Sep/2023:20:41:24] ENGINE Bus STOPPING 2023-09-23T20:41:24.883 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:41:24 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[96086]: [23/Sep/2023:20:41:24] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2023-09-23T20:41:24.883 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:41:24 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[96086]: [23/Sep/2023:20:41:24] ENGINE Bus STOPPED 2023-09-23T20:41:24.883 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:41:24 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[96086]: [23/Sep/2023:20:41:24] ENGINE Bus STARTING 2023-09-23T20:41:24.883 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:41:24 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[96086]: [23/Sep/2023:20:41:24] ENGINE Serving on http://:::9283 2023-09-23T20:41:24.883 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:41:24 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[96086]: [23/Sep/2023:20:41:24] ENGINE Bus STARTED 2023-09-23T20:41:25.632 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Sep 23 20:41:25 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-alertmanager-a[143867]: ts=2023-09-23T20:41:25.539Z caller=cluster.go:698 level=info component=cluster msg="gossip settled; proceeding" elapsed=10.002194327s 2023-09-23T20:41:25.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:41:25 smithi026 ceph-mon[95839]: from='mgr.14451 ' entity='mgr.y' 2023-09-23T20:41:25.633 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:41:25 smithi026 ceph-mon[95839]: from='mgr.14451 ' entity='mgr.y' 2023-09-23T20:41:25.633 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:41:25 smithi026 ceph-mon[95839]: from='mgr.14451 ' entity='mgr.y' 2023-09-23T20:41:25.633 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:41:25 smithi026 ceph-mon[95839]: from='mgr.14451 ' entity='mgr.y' 2023-09-23T20:41:25.633 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:41:25 smithi026 ceph-mon[95839]: from='mgr.14451 ' entity='mgr.y' 2023-09-23T20:41:25.633 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:41:25 smithi026 ceph-mon[95839]: from='mgr.14451 ' entity='mgr.y' 2023-09-23T20:41:25.633 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:41:25 smithi026 ceph-mon[95839]: from='mgr.14451 172.21.15.26:0/1329296977' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2023-09-23T20:41:25.633 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:41:25 smithi026 ceph-mon[95839]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2023-09-23T20:41:25.633 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:41:25 smithi026 ceph-mon[95839]: from='mgr.14451 ' entity='mgr.y' 2023-09-23T20:41:25.633 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:41:25 smithi026 ceph-mon[95839]: Deploying daemon grafana.a on smithi070 2023-09-23T20:41:25.634 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:41:25 smithi026 ceph-mon[109912]: from='mgr.14451 ' entity='mgr.y' 2023-09-23T20:41:25.634 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:41:25 smithi026 ceph-mon[109912]: from='mgr.14451 ' entity='mgr.y' 2023-09-23T20:41:25.634 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:41:25 smithi026 ceph-mon[109912]: from='mgr.14451 ' entity='mgr.y' 2023-09-23T20:41:25.634 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:41:25 smithi026 ceph-mon[109912]: from='mgr.14451 ' entity='mgr.y' 2023-09-23T20:41:25.634 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:41:25 smithi026 ceph-mon[109912]: from='mgr.14451 ' entity='mgr.y' 2023-09-23T20:41:25.634 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:41:25 smithi026 ceph-mon[109912]: from='mgr.14451 ' entity='mgr.y' 2023-09-23T20:41:25.634 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:41:25 smithi026 ceph-mon[109912]: from='mgr.14451 172.21.15.26:0/1329296977' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2023-09-23T20:41:25.634 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:41:25 smithi026 ceph-mon[109912]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2023-09-23T20:41:25.634 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:41:25 smithi026 ceph-mon[109912]: from='mgr.14451 ' entity='mgr.y' 2023-09-23T20:41:25.634 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:41:25 smithi026 ceph-mon[109912]: Deploying daemon grafana.a on smithi070 2023-09-23T20:41:25.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:41:25 smithi070 ceph-mon[100755]: from='mgr.14451 ' entity='mgr.y' 2023-09-23T20:41:25.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:41:25 smithi070 ceph-mon[100755]: from='mgr.14451 ' entity='mgr.y' 2023-09-23T20:41:25.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:41:25 smithi070 ceph-mon[100755]: from='mgr.14451 ' entity='mgr.y' 2023-09-23T20:41:25.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:41:25 smithi070 ceph-mon[100755]: from='mgr.14451 ' entity='mgr.y' 2023-09-23T20:41:25.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:41:25 smithi070 ceph-mon[100755]: from='mgr.14451 ' entity='mgr.y' 2023-09-23T20:41:25.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:41:25 smithi070 ceph-mon[100755]: from='mgr.14451 ' entity='mgr.y' 2023-09-23T20:41:25.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:41:25 smithi070 ceph-mon[100755]: from='mgr.14451 172.21.15.26:0/1329296977' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2023-09-23T20:41:25.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:41:25 smithi070 ceph-mon[100755]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2023-09-23T20:41:25.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:41:25 smithi070 ceph-mon[100755]: from='mgr.14451 ' entity='mgr.y' 2023-09-23T20:41:25.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:41:25 smithi070 ceph-mon[100755]: Deploying daemon grafana.a on smithi070 2023-09-23T20:41:26.197 INFO:teuthology.orchestra.run.smithi026.stdout: 2023-09-23T20:41:26.197 INFO:teuthology.orchestra.run.smithi026.stdout:{"epoch":64,"fsid":"b3f8e4ec-5a50-11ee-8db4-212e2dc638e7","created":"2023-09-23T20:35:58.919705+0000","modified":"2023-09-23T20:41:18.557547+0000","last_up_change":"2023-09-23T20:40:38.528428+0000","last_in_change":"2023-09-23T20:40:23.597125+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":2,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2023-09-23T20:38:57.022844+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}},{"pool":2,"pool_name":"rbd","create_time":"2023-09-23T20:41:15.182110+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"64","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}}],"osds":[{"osd":0,"uuid":"f837feb7-060e-4965-9b48-bed9f3672d05","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6802","nonce":2638118366},{"type":"v1","addr":"172.21.15.26:6803","nonce":2638118366}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6804","nonce":2638118366},{"type":"v1","addr":"172.21.15.26:6805","nonce":2638118366}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6808","nonce":2638118366},{"type":"v1","addr":"172.21.15.26:6809","nonce":2638118366}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6806","nonce":2638118366},{"type":"v1","addr":"172.21.15.26:6807","nonce":2638118366}]},"public_addr":"172.21.15.26:6803/2638118366","cluster_addr":"172.21.15.26:6805/2638118366","heartbeat_back_addr":"172.21.15.26:6809/2638118366","heartbeat_front_addr":"172.21.15.26:6807/2638118366","state":["exists","up"]},{"osd":1,"uuid":"917f65aa-5963-4576-988a-396698b7c6a8","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":61,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6810","nonce":3250108152},{"type":"v1","addr":"172.21.15.26:6811","nonce":3250108152}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6812","nonce":3250108152},{"type":"v1","addr":"172.21.15.26:6813","nonce":3250108152}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6816","nonce":3250108152},{"type":"v1","addr":"172.21.15.26:6817","nonce":3250108152}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6814","nonce":3250108152},{"type":"v1","addr":"172.21.15.26:6815","nonce":3250108152}]},"public_addr":"172.21.15.26:6811/3250108152","cluster_addr":"172.21.15.26:6813/3250108152","heartbeat_back_addr":"172.21.15.26:6817/3250108152","heartbeat_front_addr":"172.21.15.26:6815/3250108152","state":["exists","up"]},{"osd":2,"uuid":"2e115328-76c6-478e-915c-5786903be49f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":61,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6818","nonce":1325135181},{"type":"v1","addr":"172.21.15.26:6819","nonce":1325135181}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6820","nonce":1325135181},{"type":"v1","addr":"172.21.15.26:6821","nonce":1325135181}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6824","nonce":1325135181},{"type":"v1","addr":"172.21.15.26:6825","nonce":1325135181}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6822","nonce":1325135181},{"type":"v1","addr":"172.21.15.26:6823","nonce":1325135181}]},"public_addr":"172.21.15.26:6819/1325135181","cluster_addr":"172.21.15.26:6821/1325135181","heartbeat_back_addr":"172.21.15.26:6825/1325135181","heartbeat_front_addr":"172.21.15.26:6823/1325135181","state":["exists","up"]},{"osd":3,"uuid":"c3dd2c3f-5cf5-4fb0-964c-1b8926f75c8a","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":26,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6826","nonce":3610585082},{"type":"v1","addr":"172.21.15.26:6827","nonce":3610585082}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6828","nonce":3610585082},{"type":"v1","addr":"172.21.15.26:6829","nonce":3610585082}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6832","nonce":3610585082},{"type":"v1","addr":"172.21.15.26:6833","nonce":3610585082}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6830","nonce":3610585082},{"type":"v1","addr":"172.21.15.26:6831","nonce":3610585082}]},"public_addr":"172.21.15.26:6827/3610585082","cluster_addr":"172.21.15.26:6829/3610585082","heartbeat_back_addr":"172.21.15.26:6833/3610585082","heartbeat_front_addr":"172.21.15.26:6831/3610585082","state":["exists","up"]},{"osd":4,"uuid":"a5f31996-1b38-491b-956b-5aebc72d0170","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":31,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6800","nonce":1528249138},{"type":"v1","addr":"172.21.15.70:6801","nonce":1528249138}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6802","nonce":1528249138},{"type":"v1","addr":"172.21.15.70:6803","nonce":1528249138}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6806","nonce":1528249138},{"type":"v1","addr":"172.21.15.70:6807","nonce":1528249138}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6804","nonce":1528249138},{"type":"v1","addr":"172.21.15.70:6805","nonce":1528249138}]},"public_addr":"172.21.15.70:6801/1528249138","cluster_addr":"172.21.15.70:6803/1528249138","heartbeat_back_addr":"172.21.15.70:6807/1528249138","heartbeat_front_addr":"172.21.15.70:6805/1528249138","state":["exists","up"]},{"osd":5,"uuid":"21dad198-b9db-42c6-8823-112df1ecd874","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":39,"up_thru":61,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6808","nonce":3802232906},{"type":"v1","addr":"172.21.15.70:6809","nonce":3802232906}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6810","nonce":3802232906},{"type":"v1","addr":"172.21.15.70:6811","nonce":3802232906}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6814","nonce":3802232906},{"type":"v1","addr":"172.21.15.70:6815","nonce":3802232906}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6812","nonce":3802232906},{"type":"v1","addr":"172.21.15.70:6813","nonce":3802232906}]},"public_addr":"172.21.15.70:6809/3802232906","cluster_addr":"172.21.15.70:6811/3802232906","heartbeat_back_addr":"172.21.15.70:6815/3802232906","heartbeat_front_addr":"172.21.15.70:6813/3802232906","state":["exists","up"]},{"osd":6,"uuid":"6f3be784-7149-4f13-81b5-98924e97c056","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":47,"up_thru":61,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6816","nonce":2944175278},{"type":"v1","addr":"172.21.15.70:6817","nonce":2944175278}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6818","nonce":2944175278},{"type":"v1","addr":"172.21.15.70:6819","nonce":2944175278}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6822","nonce":2944175278},{"type":"v1","addr":"172.21.15.70:6823","nonce":2944175278}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6820","nonce":2944175278},{"type":"v1","addr":"172.21.15.70:6821","nonce":2944175278}]},"public_addr":"172.21.15.70:6817/2944175278","cluster_addr":"172.21.15.70:6819/2944175278","heartbeat_back_addr":"172.21.15.70:6823/2944175278","heartbeat_front_addr":"172.21.15.70:6821/2944175278","state":["exists","up"]},{"osd":7,"uuid":"499f5f85-f4b2-40a7-ac0f-79cd22738d30","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":55,"up_thru":61,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6824","nonce":2824974165},{"type":"v1","addr":"172.21.15.70:6825","nonce":2824974165}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6826","nonce":2824974165},{"type":"v1","addr":"172.21.15.70:6827","nonce":2824974165}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6830","nonce":2824974165},{"type":"v1","addr":"172.21.15.70:6831","nonce":2824974165}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6828","nonce":2824974165},{"type":"v1","addr":"172.21.15.70:6829","nonce":2824974165}]},"public_addr":"172.21.15.70:6825/2824974165","cluster_addr":"172.21.15.70:6827/2824974165","heartbeat_back_addr":"172.21.15.70:6831/2824974165","heartbeat_front_addr":"172.21.15.70:6829/2824974165","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T20:38:10.875468+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T20:38:32.449963+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T20:38:53.633316+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T20:39:14.828940+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T20:39:34.945663+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T20:39:55.954288+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T20:40:16.847102+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T20:40:37.820404+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.26:6801/91363814":"2023-09-24T20:41:03.790648+0000","172.21.15.26:6800/91363814":"2023-09-24T20:41:03.790648+0000","172.21.15.26:0/3461760126":"2023-09-24T20:41:03.790648+0000","172.21.15.26:0/394447930":"2023-09-24T20:41:03.790648+0000","172.21.15.26:0/1083572999":"2023-09-24T20:41:03.790648+0000","172.21.15.26:0/2838335646":"2023-09-24T20:36:56.498559+0000","172.21.15.26:6801/952511372":"2023-09-24T20:36:56.498559+0000","172.21.15.26:6800/952511372":"2023-09-24T20:36:56.498559+0000","172.21.15.26:0/1625326392":"2023-09-24T20:36:56.498559+0000","172.21.15.26:0/3175144810":"2023-09-24T20:36:31.159649+0000","172.21.15.26:0/1815190465":"2023-09-24T20:36:31.159649+0000","172.21.15.26:0/188049283":"2023-09-24T20:41:03.790648+0000","172.21.15.26:0/289994511":"2023-09-24T20:36:56.498559+0000","172.21.15.26:0/686734212":"2023-09-24T20:36:31.159649+0000","172.21.15.26:6800/4099624907":"2023-09-24T20:36:31.159649+0000","172.21.15.26:6801/4099624907":"2023-09-24T20:36:31.159649+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2023-09-23T20:41:26.645 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph tell osd.0 flush_pg_stats 2023-09-23T20:41:26.646 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph tell osd.1 flush_pg_stats 2023-09-23T20:41:26.646 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph tell osd.2 flush_pg_stats 2023-09-23T20:41:26.647 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph tell osd.3 flush_pg_stats 2023-09-23T20:41:26.647 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph tell osd.4 flush_pg_stats 2023-09-23T20:41:26.648 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph tell osd.5 flush_pg_stats 2023-09-23T20:41:26.648 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph tell osd.6 flush_pg_stats 2023-09-23T20:41:26.648 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph tell osd.7 flush_pg_stats 2023-09-23T20:41:26.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:41:26 smithi070 ceph-mon[100755]: pgmap v18: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:41:26.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:41:26 smithi070 ceph-mon[100755]: from='client.? 172.21.15.26:0/4189655037' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-09-23T20:41:26.894 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:41:26 smithi026 ceph-mon[109912]: pgmap v18: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:41:26.894 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:41:26 smithi026 ceph-mon[109912]: from='client.? 172.21.15.26:0/4189655037' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-09-23T20:41:26.905 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:41:26 smithi026 ceph-mon[95839]: pgmap v18: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:41:26.905 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:41:26 smithi026 ceph-mon[95839]: from='client.? 172.21.15.26:0/4189655037' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-09-23T20:41:27.365 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:41:27.405 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:41:27.418 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:41:27.505 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:41:27.523 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:41:27.609 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:41:27.919 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:41:27.919 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:41:29.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:41:28 smithi070 ceph-mon[100755]: pgmap v19: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:41:29.128 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:41:28 smithi026 ceph-mon[95839]: pgmap v19: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:41:29.128 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:41:28 smithi026 ceph-mon[109912]: pgmap v19: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:41:30.009 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:41:29 smithi026 ceph-mon[95839]: from='mgr.14451 ' entity='mgr.y' 2023-09-23T20:41:30.009 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:41:29 smithi026 ceph-mon[109912]: from='mgr.14451 ' entity='mgr.y' 2023-09-23T20:41:30.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:41:29 smithi070 ceph-mon[100755]: from='mgr.14451 ' entity='mgr.y' 2023-09-23T20:41:31.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:41:30 smithi026 ceph-mon[95839]: pgmap v20: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:41:31.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:41:30 smithi026 ceph-mon[109912]: pgmap v20: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:41:31.371 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:41:30 smithi070 ceph-mon[100755]: pgmap v20: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:41:31.869 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 20:41:31 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-x[102861]: ::ffff:172.21.15.70 - - [23/Sep/2023:20:41:31] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2023-09-23T20:41:32.336 INFO:teuthology.orchestra.run.smithi026.stdout:55834574885 2023-09-23T20:41:32.336 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.1 2023-09-23T20:41:32.948 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:41:32 smithi026 ceph-mon[95839]: pgmap v21: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:41:32.948 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:41:32 smithi026 ceph-mon[109912]: pgmap v21: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:41:33.366 INFO:teuthology.orchestra.run.smithi026.stdout:111669149724 2023-09-23T20:41:33.366 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.3 2023-09-23T20:41:33.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:41:32 smithi070 ceph-mon[100755]: pgmap v21: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:41:34.279 INFO:teuthology.orchestra.run.smithi026.stdout:133143986200 2023-09-23T20:41:34.279 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.4 2023-09-23T20:41:34.382 INFO:teuthology.orchestra.run.smithi026.stdout:201863462928 2023-09-23T20:41:34.382 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.6 2023-09-23T20:41:34.838 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:41:34 smithi070 ceph-mon[100755]: pgmap v22: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:41:34.839 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:41:34 smithi070 ceph-mon[100755]: from='mgr.14451 172.21.15.26:0/1329296977' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:41:34.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:41:34 smithi026 ceph-mon[95839]: pgmap v22: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:41:34.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:41:34 smithi026 ceph-mon[95839]: from='mgr.14451 172.21.15.26:0/1329296977' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:41:34.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:41:34 smithi026 ceph-mon[109912]: pgmap v22: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:41:34.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:41:34 smithi026 ceph-mon[109912]: from='mgr.14451 172.21.15.26:0/1329296977' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:41:35.441 INFO:teuthology.orchestra.run.smithi026.stdout:236223201291 2023-09-23T20:41:35.441 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.7 2023-09-23T20:41:35.483 INFO:teuthology.orchestra.run.smithi026.stdout:77309411360 2023-09-23T20:41:35.483 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.2 2023-09-23T20:41:35.672 INFO:teuthology.orchestra.run.smithi026.stdout:34359738410 2023-09-23T20:41:35.673 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.0 2023-09-23T20:41:35.907 INFO:teuthology.orchestra.run.smithi026.stdout:167503724565 2023-09-23T20:41:35.908 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.5 2023-09-23T20:41:36.028 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:41:36.080 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:41:36.144 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:41:36.145 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:41:36.524 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:41:36.525 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:41:36.776 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:36 smithi070 systemd[1]: Starting Ceph grafana.a for b3f8e4ec-5a50-11ee-8db4-212e2dc638e7... 2023-09-23T20:41:37.051 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:41:36 smithi026 ceph-mon[109912]: pgmap v23: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:41:37.052 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:41:36 smithi026 ceph-mon[95839]: pgmap v23: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:41:37.062 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:41:36 smithi070 ceph-mon[100755]: pgmap v23: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:41:37.063 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:36 smithi070 podman[137462]: 2023-09-23T20:41:37.063 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 bash[137462]: 12943b9df698106f0f3327ca95dcaac1cbca33fb788a557d9e609cd4d840b155 2023-09-23T20:41:37.315 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 systemd[1]: Started Ceph grafana.a for b3f8e4ec-5a50-11ee-8db4-212e2dc638e7. 2023-09-23T20:41:37.315 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=settings t=2023-09-23T20:41:37.159318243Z level=info msg="Starting Grafana" version=9.4.7 commit=4add91f03d branch=HEAD compiled=2023-03-16T23:56:52Z 2023-09-23T20:41:37.315 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=settings t=2023-09-23T20:41:37.15949785Z level=warn msg="\"sentry\" frontend logging provider is deprecated and will be removed in the next major version. Use \"grafana\" provider instead." 2023-09-23T20:41:37.315 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=settings t=2023-09-23T20:41:37.159516099Z level=info msg="Config loaded from" file=/usr/share/grafana/conf/defaults.ini 2023-09-23T20:41:37.315 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=settings t=2023-09-23T20:41:37.15954013Z level=info msg="Config loaded from" file=/etc/grafana/grafana.ini 2023-09-23T20:41:37.316 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=settings t=2023-09-23T20:41:37.159546398Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_DATA=/var/lib/grafana" 2023-09-23T20:41:37.316 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=settings t=2023-09-23T20:41:37.159553533Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_LOGS=/var/log/grafana" 2023-09-23T20:41:37.316 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=settings t=2023-09-23T20:41:37.159559039Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_PLUGINS=/var/lib/grafana/plugins" 2023-09-23T20:41:37.316 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=settings t=2023-09-23T20:41:37.159565151Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_PROVISIONING=/etc/grafana/provisioning" 2023-09-23T20:41:37.316 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=settings t=2023-09-23T20:41:37.159570838Z level=info msg="Path Home" path=/usr/share/grafana 2023-09-23T20:41:37.316 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=settings t=2023-09-23T20:41:37.159576692Z level=info msg="Path Data" path=/var/lib/grafana 2023-09-23T20:41:37.316 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=settings t=2023-09-23T20:41:37.159582056Z level=info msg="Path Logs" path=/var/log/grafana 2023-09-23T20:41:37.316 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=settings t=2023-09-23T20:41:37.159587319Z level=info msg="Path Plugins" path=/var/lib/grafana/plugins 2023-09-23T20:41:37.316 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=settings t=2023-09-23T20:41:37.159592844Z level=info msg="Path Provisioning" path=/etc/grafana/provisioning 2023-09-23T20:41:37.316 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=settings t=2023-09-23T20:41:37.159598364Z level=info msg="App mode production" 2023-09-23T20:41:37.317 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=sqlstore t=2023-09-23T20:41:37.159647731Z level=info msg="Connecting to DB" dbtype=sqlite3 2023-09-23T20:41:37.317 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=sqlstore t=2023-09-23T20:41:37.15966381Z level=warn msg="SQLite database file has broader permissions than it should" path=/var/lib/grafana/grafana.db mode=-rw-r--r-- expected=-rw-r----- 2023-09-23T20:41:37.317 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.160526026Z level=info msg="Starting DB migrations" 2023-09-23T20:41:37.317 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.160687949Z level=info msg="Executing migration" id="create migration_log table" 2023-09-23T20:41:37.317 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.16130115Z level=info msg="Executing migration" id="create user table" 2023-09-23T20:41:37.317 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.161697237Z level=info msg="Executing migration" id="add unique index user.login" 2023-09-23T20:41:37.317 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.162098702Z level=info msg="Executing migration" id="add unique index user.email" 2023-09-23T20:41:37.317 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.162489904Z level=info msg="Executing migration" id="drop index UQE_user_login - v1" 2023-09-23T20:41:37.317 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.162916473Z level=info msg="Executing migration" id="drop index UQE_user_email - v1" 2023-09-23T20:41:37.317 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.163252526Z level=info msg="Executing migration" id="Rename table user to user_v1 - v1" 2023-09-23T20:41:37.317 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.163930711Z level=info msg="Executing migration" id="create user table v2" 2023-09-23T20:41:37.318 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.164261791Z level=info msg="Executing migration" id="create index UQE_user_login - v2" 2023-09-23T20:41:37.318 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.164682216Z level=info msg="Executing migration" id="create index UQE_user_email - v2" 2023-09-23T20:41:37.318 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.16508075Z level=info msg="Executing migration" id="copy data_source v1 to v2" 2023-09-23T20:41:37.318 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.165398811Z level=info msg="Executing migration" id="Drop old table user_v1" 2023-09-23T20:41:37.318 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.165717453Z level=info msg="Executing migration" id="Add column help_flags1 to user table" 2023-09-23T20:41:37.318 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.166114122Z level=info msg="Executing migration" id="Update user table charset" 2023-09-23T20:41:37.318 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.166403761Z level=info msg="Executing migration" id="Add last_seen_at column to user" 2023-09-23T20:41:37.318 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.166782724Z level=info msg="Executing migration" id="Add missing user data" 2023-09-23T20:41:37.318 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.167081169Z level=info msg="Executing migration" id="Add is_disabled column to user" 2023-09-23T20:41:37.318 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.167478095Z level=info msg="Executing migration" id="Add index user.login/user.email" 2023-09-23T20:41:37.318 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.167861447Z level=info msg="Executing migration" id="Add is_service_account column to user" 2023-09-23T20:41:37.318 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.168217722Z level=info msg="Executing migration" id="Update is_service_account column to nullable" 2023-09-23T20:41:37.319 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.169345779Z level=info msg="Executing migration" id="create temp user table v1-7" 2023-09-23T20:41:37.319 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.169827269Z level=info msg="Executing migration" id="create index IDX_temp_user_email - v1-7" 2023-09-23T20:41:37.319 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.170340884Z level=info msg="Executing migration" id="create index IDX_temp_user_org_id - v1-7" 2023-09-23T20:41:37.319 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.17084012Z level=info msg="Executing migration" id="create index IDX_temp_user_code - v1-7" 2023-09-23T20:41:37.319 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.171311758Z level=info msg="Executing migration" id="create index IDX_temp_user_status - v1-7" 2023-09-23T20:41:37.319 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.171789102Z level=info msg="Executing migration" id="Update temp_user table charset" 2023-09-23T20:41:37.319 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.172089968Z level=info msg="Executing migration" id="drop index IDX_temp_user_email - v1" 2023-09-23T20:41:37.319 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.172491394Z level=info msg="Executing migration" id="drop index IDX_temp_user_org_id - v1" 2023-09-23T20:41:37.319 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.172830508Z level=info msg="Executing migration" id="drop index IDX_temp_user_code - v1" 2023-09-23T20:41:37.319 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.173187966Z level=info msg="Executing migration" id="drop index IDX_temp_user_status - v1" 2023-09-23T20:41:37.319 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.17361721Z level=info msg="Executing migration" id="Rename table temp_user to temp_user_tmp_qwerty - v1" 2023-09-23T20:41:37.320 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.174515119Z level=info msg="Executing migration" id="create temp_user v2" 2023-09-23T20:41:37.320 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.175012095Z level=info msg="Executing migration" id="create index IDX_temp_user_email - v2" 2023-09-23T20:41:37.320 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.175487389Z level=info msg="Executing migration" id="create index IDX_temp_user_org_id - v2" 2023-09-23T20:41:37.320 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.175843551Z level=info msg="Executing migration" id="create index IDX_temp_user_code - v2" 2023-09-23T20:41:37.320 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.176151Z level=info msg="Executing migration" id="create index IDX_temp_user_status - v2" 2023-09-23T20:41:37.320 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.176525399Z level=info msg="Executing migration" id="copy temp_user v1 to v2" 2023-09-23T20:41:37.320 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.176809832Z level=info msg="Executing migration" id="drop temp_user_tmp_qwerty" 2023-09-23T20:41:37.320 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.177070857Z level=info msg="Executing migration" id="Set created for temp users that will otherwise prematurely expire" 2023-09-23T20:41:37.320 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.177367387Z level=info msg="Executing migration" id="create star table" 2023-09-23T20:41:37.320 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.177680126Z level=info msg="Executing migration" id="add unique index star.user_id_dashboard_id" 2023-09-23T20:41:37.320 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.178111567Z level=info msg="Executing migration" id="create org table v1" 2023-09-23T20:41:37.321 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.178505776Z level=info msg="Executing migration" id="create index UQE_org_name - v1" 2023-09-23T20:41:37.321 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.178915392Z level=info msg="Executing migration" id="create org_user table v1" 2023-09-23T20:41:37.321 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.179229781Z level=info msg="Executing migration" id="create index IDX_org_user_org_id - v1" 2023-09-23T20:41:37.321 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.179588235Z level=info msg="Executing migration" id="create index UQE_org_user_org_id_user_id - v1" 2023-09-23T20:41:37.321 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.179931691Z level=info msg="Executing migration" id="create index IDX_org_user_user_id - v1" 2023-09-23T20:41:37.321 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.180271002Z level=info msg="Executing migration" id="Update org table charset" 2023-09-23T20:41:37.321 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.18051389Z level=info msg="Executing migration" id="Update org_user table charset" 2023-09-23T20:41:37.321 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.180725966Z level=info msg="Executing migration" id="Migrate all Read Only Viewers to Viewers" 2023-09-23T20:41:37.321 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.180956356Z level=info msg="Executing migration" id="create dashboard table" 2023-09-23T20:41:37.321 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.181290486Z level=info msg="Executing migration" id="add index dashboard.account_id" 2023-09-23T20:41:37.321 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.181636167Z level=info msg="Executing migration" id="add unique index dashboard_account_id_slug" 2023-09-23T20:41:37.321 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.181981099Z level=info msg="Executing migration" id="create dashboard_tag table" 2023-09-23T20:41:37.322 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.182361538Z level=info msg="Executing migration" id="add unique index dashboard_tag.dasboard_id_term" 2023-09-23T20:41:37.322 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.182728586Z level=info msg="Executing migration" id="drop index UQE_dashboard_tag_dashboard_id_term - v1" 2023-09-23T20:41:37.322 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.183035196Z level=info msg="Executing migration" id="Rename table dashboard to dashboard_v1 - v1" 2023-09-23T20:41:37.322 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.183835992Z level=info msg="Executing migration" id="create dashboard v2" 2023-09-23T20:41:37.322 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.184171078Z level=info msg="Executing migration" id="create index IDX_dashboard_org_id - v2" 2023-09-23T20:41:37.322 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.184531956Z level=info msg="Executing migration" id="create index UQE_dashboard_org_id_slug - v2" 2023-09-23T20:41:37.322 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.184907766Z level=info msg="Executing migration" id="copy dashboard v1 to v2" 2023-09-23T20:41:37.322 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.185156966Z level=info msg="Executing migration" id="drop table dashboard_v1" 2023-09-23T20:41:37.322 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.185506126Z level=info msg="Executing migration" id="alter dashboard.data to mediumtext v1" 2023-09-23T20:41:37.322 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.185780092Z level=info msg="Executing migration" id="Add column updated_by in dashboard - v2" 2023-09-23T20:41:37.322 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.18617901Z level=info msg="Executing migration" id="Add column created_by in dashboard - v2" 2023-09-23T20:41:37.323 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.186660337Z level=info msg="Executing migration" id="Add column gnetId in dashboard" 2023-09-23T20:41:37.323 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.187058777Z level=info msg="Executing migration" id="Add index for gnetId in dashboard" 2023-09-23T20:41:37.323 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.187438217Z level=info msg="Executing migration" id="Add column plugin_id in dashboard" 2023-09-23T20:41:37.323 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.187880833Z level=info msg="Executing migration" id="Add index for plugin_id in dashboard" 2023-09-23T20:41:37.323 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.188214192Z level=info msg="Executing migration" id="Add index for dashboard_id in dashboard_tag" 2023-09-23T20:41:37.323 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.188577501Z level=info msg="Executing migration" id="Update dashboard table charset" 2023-09-23T20:41:37.323 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.188790918Z level=info msg="Executing migration" id="Update dashboard_tag table charset" 2023-09-23T20:41:37.323 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.189001083Z level=info msg="Executing migration" id="Add column folder_id in dashboard" 2023-09-23T20:41:37.323 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.189462149Z level=info msg="Executing migration" id="Add column isFolder in dashboard" 2023-09-23T20:41:37.323 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.189939865Z level=info msg="Executing migration" id="Add column has_acl in dashboard" 2023-09-23T20:41:37.324 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.190385534Z level=info msg="Executing migration" id="Add column uid in dashboard" 2023-09-23T20:41:37.324 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.190869587Z level=info msg="Executing migration" id="Update uid column values in dashboard" 2023-09-23T20:41:37.324 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.191115479Z level=info msg="Executing migration" id="Add unique index dashboard_org_id_uid" 2023-09-23T20:41:37.324 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.191485664Z level=info msg="Executing migration" id="Remove unique index org_id_slug" 2023-09-23T20:41:37.324 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.191826387Z level=info msg="Executing migration" id="Update dashboard title length" 2023-09-23T20:41:37.324 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.192038801Z level=info msg="Executing migration" id="Add unique index for dashboard_org_id_title_folder_id" 2023-09-23T20:41:37.324 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.192412144Z level=info msg="Executing migration" id="create dashboard_provisioning" 2023-09-23T20:41:37.324 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.192740645Z level=info msg="Executing migration" id="Rename table dashboard_provisioning to dashboard_provisioning_tmp_qwerty - v1" 2023-09-23T20:41:37.324 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.193596708Z level=info msg="Executing migration" id="create dashboard_provisioning v2" 2023-09-23T20:41:37.324 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.19393674Z level=info msg="Executing migration" id="create index IDX_dashboard_provisioning_dashboard_id - v2" 2023-09-23T20:41:37.324 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.19429349Z level=info msg="Executing migration" id="create index IDX_dashboard_provisioning_dashboard_id_name - v2" 2023-09-23T20:41:37.324 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.19466196Z level=info msg="Executing migration" id="copy dashboard_provisioning v1 to v2" 2023-09-23T20:41:37.325 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.194910437Z level=info msg="Executing migration" id="drop dashboard_provisioning_tmp_qwerty" 2023-09-23T20:41:37.325 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.195190005Z level=info msg="Executing migration" id="Add check_sum column" 2023-09-23T20:41:37.325 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.195720018Z level=info msg="Executing migration" id="Add index for dashboard_title" 2023-09-23T20:41:37.325 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.196046515Z level=info msg="Executing migration" id="delete tags for deleted dashboards" 2023-09-23T20:41:37.325 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.196278829Z level=info msg="Executing migration" id="delete stars for deleted dashboards" 2023-09-23T20:41:37.325 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.196542707Z level=info msg="Executing migration" id="Add index for dashboard_is_folder" 2023-09-23T20:41:37.325 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.196890946Z level=info msg="Executing migration" id="Add isPublic for dashboard" 2023-09-23T20:41:37.325 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.197374476Z level=info msg="Executing migration" id="create data_source table" 2023-09-23T20:41:37.325 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.197742466Z level=info msg="Executing migration" id="add index data_source.account_id" 2023-09-23T20:41:37.325 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.198099777Z level=info msg="Executing migration" id="add unique index data_source.account_id_name" 2023-09-23T20:41:37.325 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.198472516Z level=info msg="Executing migration" id="drop index IDX_data_source_account_id - v1" 2023-09-23T20:41:37.326 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.198811127Z level=info msg="Executing migration" id="drop index UQE_data_source_account_id_name - v1" 2023-09-23T20:41:37.326 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.199121873Z level=info msg="Executing migration" id="Rename table data_source to data_source_v1 - v1" 2023-09-23T20:41:37.326 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.200078376Z level=info msg="Executing migration" id="create data_source table v2" 2023-09-23T20:41:37.326 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.200462036Z level=info msg="Executing migration" id="create index IDX_data_source_org_id - v2" 2023-09-23T20:41:37.326 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.200819438Z level=info msg="Executing migration" id="create index UQE_data_source_org_id_name - v2" 2023-09-23T20:41:37.326 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.201191136Z level=info msg="Executing migration" id="Drop old table data_source_v1 #2" 2023-09-23T20:41:37.326 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.201507629Z level=info msg="Executing migration" id="Add column with_credentials" 2023-09-23T20:41:37.326 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.201949095Z level=info msg="Executing migration" id="Add secure json data column" 2023-09-23T20:41:37.326 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.202422201Z level=info msg="Executing migration" id="Update data_source table charset" 2023-09-23T20:41:37.326 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.202671264Z level=info msg="Executing migration" id="Update initial version to 1" 2023-09-23T20:41:37.326 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.202902276Z level=info msg="Executing migration" id="Add read_only data column" 2023-09-23T20:41:37.327 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.203376607Z level=info msg="Executing migration" id="Migrate logging ds to loki ds" 2023-09-23T20:41:37.327 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.203645097Z level=info msg="Executing migration" id="Update json_data with nulls" 2023-09-23T20:41:37.327 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.203873943Z level=info msg="Executing migration" id="Add uid column" 2023-09-23T20:41:37.327 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.204370402Z level=info msg="Executing migration" id="Update uid value" 2023-09-23T20:41:37.327 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.20463468Z level=info msg="Executing migration" id="Add unique index datasource_org_id_uid" 2023-09-23T20:41:37.327 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.204972488Z level=info msg="Executing migration" id="add unique index datasource_org_id_is_default" 2023-09-23T20:41:37.327 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.205333602Z level=info msg="Executing migration" id="create api_key table" 2023-09-23T20:41:37.327 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.205691015Z level=info msg="Executing migration" id="add index api_key.account_id" 2023-09-23T20:41:37.327 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.206043165Z level=info msg="Executing migration" id="add index api_key.key" 2023-09-23T20:41:37.327 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.206403559Z level=info msg="Executing migration" id="add index api_key.account_id_name" 2023-09-23T20:41:37.327 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.20681907Z level=info msg="Executing migration" id="drop index IDX_api_key_account_id - v1" 2023-09-23T20:41:37.327 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.20713063Z level=info msg="Executing migration" id="drop index UQE_api_key_key - v1" 2023-09-23T20:41:37.328 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.207500052Z level=info msg="Executing migration" id="drop index UQE_api_key_account_id_name - v1" 2023-09-23T20:41:37.328 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.207847627Z level=info msg="Executing migration" id="Rename table api_key to api_key_v1 - v1" 2023-09-23T20:41:37.328 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.208930492Z level=info msg="Executing migration" id="create api_key table v2" 2023-09-23T20:41:37.328 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.209317052Z level=info msg="Executing migration" id="create index IDX_api_key_org_id - v2" 2023-09-23T20:41:37.328 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.209699456Z level=info msg="Executing migration" id="create index UQE_api_key_key - v2" 2023-09-23T20:41:37.328 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.210057251Z level=info msg="Executing migration" id="create index UQE_api_key_org_id_name - v2" 2023-09-23T20:41:37.328 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.210432679Z level=info msg="Executing migration" id="copy api_key v1 to v2" 2023-09-23T20:41:37.328 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.210712517Z level=info msg="Executing migration" id="Drop old table api_key_v1" 2023-09-23T20:41:37.328 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.210988086Z level=info msg="Executing migration" id="Update api_key table charset" 2023-09-23T20:41:37.328 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.211198733Z level=info msg="Executing migration" id="Add expires to api_key table" 2023-09-23T20:41:37.328 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.211699046Z level=info msg="Executing migration" id="Add service account foreign key" 2023-09-23T20:41:37.328 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.212149322Z level=info msg="Executing migration" id="set service account foreign key to nil if 0" 2023-09-23T20:41:37.329 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.212449962Z level=info msg="Executing migration" id="Add last_used_at to api_key table" 2023-09-23T20:41:37.329 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.212969117Z level=info msg="Executing migration" id="Add is_revoked column to api_key table" 2023-09-23T20:41:37.329 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.213465229Z level=info msg="Executing migration" id="create dashboard_snapshot table v4" 2023-09-23T20:41:37.329 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.213799517Z level=info msg="Executing migration" id="drop table dashboard_snapshot_v4 #1" 2023-09-23T20:41:37.329 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.214091884Z level=info msg="Executing migration" id="create dashboard_snapshot table v5 #2" 2023-09-23T20:41:37.329 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.214441762Z level=info msg="Executing migration" id="create index UQE_dashboard_snapshot_key - v5" 2023-09-23T20:41:37.329 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.214805832Z level=info msg="Executing migration" id="create index UQE_dashboard_snapshot_delete_key - v5" 2023-09-23T20:41:37.329 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.215154292Z level=info msg="Executing migration" id="create index IDX_dashboard_snapshot_user_id - v5" 2023-09-23T20:41:37.329 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.215522617Z level=info msg="Executing migration" id="alter dashboard_snapshot to mediumtext v2" 2023-09-23T20:41:37.329 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.215769315Z level=info msg="Executing migration" id="Update dashboard_snapshot table charset" 2023-09-23T20:41:37.329 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.21597913Z level=info msg="Executing migration" id="Add column external_delete_url to dashboard_snapshots table" 2023-09-23T20:41:37.330 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.216498288Z level=info msg="Executing migration" id="Add encrypted dashboard json column" 2023-09-23T20:41:37.330 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.217035043Z level=info msg="Executing migration" id="Change dashboard_encrypted column to MEDIUMBLOB" 2023-09-23T20:41:37.330 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.217285612Z level=info msg="Executing migration" id="create quota table v1" 2023-09-23T20:41:37.330 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.217630518Z level=info msg="Executing migration" id="create index UQE_quota_org_id_user_id_target - v1" 2023-09-23T20:41:37.330 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.218004239Z level=info msg="Executing migration" id="Update quota table charset" 2023-09-23T20:41:37.330 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.218226757Z level=info msg="Executing migration" id="create plugin_setting table" 2023-09-23T20:41:37.330 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.21859357Z level=info msg="Executing migration" id="create index UQE_plugin_setting_org_id_plugin_id - v1" 2023-09-23T20:41:37.330 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.218959861Z level=info msg="Executing migration" id="Add column plugin_version to plugin_settings" 2023-09-23T20:41:37.330 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.219482585Z level=info msg="Executing migration" id="Update plugin_setting table charset" 2023-09-23T20:41:37.330 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.219730869Z level=info msg="Executing migration" id="create session table" 2023-09-23T20:41:37.330 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.22010175Z level=info msg="Executing migration" id="Drop old table playlist table" 2023-09-23T20:41:37.331 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.220383689Z level=info msg="Executing migration" id="Drop old table playlist_item table" 2023-09-23T20:41:37.331 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.220636178Z level=info msg="Executing migration" id="create playlist table v2" 2023-09-23T20:41:37.331 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.220969085Z level=info msg="Executing migration" id="create playlist item table v2" 2023-09-23T20:41:37.331 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.221329659Z level=info msg="Executing migration" id="Update playlist table charset" 2023-09-23T20:41:37.331 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.221571385Z level=info msg="Executing migration" id="Update playlist_item table charset" 2023-09-23T20:41:37.331 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.221779631Z level=info msg="Executing migration" id="drop preferences table v2" 2023-09-23T20:41:37.331 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.221992192Z level=info msg="Executing migration" id="drop preferences table v3" 2023-09-23T20:41:37.331 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.222207658Z level=info msg="Executing migration" id="create preferences table v3" 2023-09-23T20:41:37.331 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.222555881Z level=info msg="Executing migration" id="Update preferences table charset" 2023-09-23T20:41:37.331 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.222771655Z level=info msg="Executing migration" id="Add column team_id in preferences" 2023-09-23T20:41:37.331 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.223313808Z level=info msg="Executing migration" id="Update team_id column values in preferences" 2023-09-23T20:41:37.332 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.223579791Z level=info msg="Executing migration" id="Add column week_start in preferences" 2023-09-23T20:41:37.332 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.224102981Z level=info msg="Executing migration" id="Add column preferences.json_data" 2023-09-23T20:41:37.332 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.224708519Z level=info msg="Executing migration" id="alter preferences.json_data to mediumtext v1" 2023-09-23T20:41:37.332 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.224960163Z level=info msg="Executing migration" id="Add preferences index org_id" 2023-09-23T20:41:37.332 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.225339919Z level=info msg="Executing migration" id="Add preferences index user_id" 2023-09-23T20:41:37.332 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.225767177Z level=info msg="Executing migration" id="create alert table v1" 2023-09-23T20:41:37.332 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.226152933Z level=info msg="Executing migration" id="add index alert org_id & id " 2023-09-23T20:41:37.332 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.226583951Z level=info msg="Executing migration" id="add index alert state" 2023-09-23T20:41:37.332 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.226947765Z level=info msg="Executing migration" id="add index alert dashboard_id" 2023-09-23T20:41:37.332 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.227310538Z level=info msg="Executing migration" id="Create alert_rule_tag table v1" 2023-09-23T20:41:37.332 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.227649492Z level=info msg="Executing migration" id="Add unique index alert_rule_tag.alert_id_tag_id" 2023-09-23T20:41:37.333 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.228008616Z level=info msg="Executing migration" id="drop index UQE_alert_rule_tag_alert_id_tag_id - v1" 2023-09-23T20:41:37.333 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.228381025Z level=info msg="Executing migration" id="Rename table alert_rule_tag to alert_rule_tag_v1 - v1" 2023-09-23T20:41:37.333 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.229756936Z level=info msg="Executing migration" id="Create alert_rule_tag table v2" 2023-09-23T20:41:37.333 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.230089456Z level=info msg="Executing migration" id="create index UQE_alert_rule_tag_alert_id_tag_id - Add unique index alert_rule_tag.alert_id_tag_id V2" 2023-09-23T20:41:37.333 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.230472444Z level=info msg="Executing migration" id="copy alert_rule_tag v1 to v2" 2023-09-23T20:41:37.333 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.230774978Z level=info msg="Executing migration" id="drop table alert_rule_tag_v1" 2023-09-23T20:41:37.333 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.231050646Z level=info msg="Executing migration" id="create alert_notification table v1" 2023-09-23T20:41:37.333 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.231392058Z level=info msg="Executing migration" id="Add column is_default" 2023-09-23T20:41:37.333 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.231953313Z level=info msg="Executing migration" id="Add column frequency" 2023-09-23T20:41:37.333 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.23255797Z level=info msg="Executing migration" id="Add column send_reminder" 2023-09-23T20:41:37.333 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.233149402Z level=info msg="Executing migration" id="Add column disable_resolve_message" 2023-09-23T20:41:37.334 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.233765554Z level=info msg="Executing migration" id="add index alert_notification org_id & name" 2023-09-23T20:41:37.334 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.234142151Z level=info msg="Executing migration" id="Update alert table charset" 2023-09-23T20:41:37.334 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.234391113Z level=info msg="Executing migration" id="Update alert_notification table charset" 2023-09-23T20:41:37.334 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.234634007Z level=info msg="Executing migration" id="create notification_journal table v1" 2023-09-23T20:41:37.334 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.234969661Z level=info msg="Executing migration" id="add index notification_journal org_id & alert_id & notifier_id" 2023-09-23T20:41:37.334 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.23534796Z level=info msg="Executing migration" id="drop alert_notification_journal" 2023-09-23T20:41:37.334 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.235680897Z level=info msg="Executing migration" id="create alert_notification_state table v1" 2023-09-23T20:41:37.334 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.23598891Z level=info msg="Executing migration" id="add index alert_notification_state org_id & alert_id & notifier_id" 2023-09-23T20:41:37.335 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.236385642Z level=info msg="Executing migration" id="Add for to alert table" 2023-09-23T20:41:37.335 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.23699886Z level=info msg="Executing migration" id="Add column uid in alert_notification" 2023-09-23T20:41:37.335 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.237592954Z level=info msg="Executing migration" id="Update uid column values in alert_notification" 2023-09-23T20:41:37.335 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.237861804Z level=info msg="Executing migration" id="Add unique index alert_notification_org_id_uid" 2023-09-23T20:41:37.335 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.238248023Z level=info msg="Executing migration" id="Remove unique index org_id_name" 2023-09-23T20:41:37.335 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.238605005Z level=info msg="Executing migration" id="Add column secure_settings in alert_notification" 2023-09-23T20:41:37.335 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.239147291Z level=info msg="Executing migration" id="alter alert.settings to mediumtext" 2023-09-23T20:41:37.335 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.239409544Z level=info msg="Executing migration" id="Add non-unique index alert_notification_state_alert_id" 2023-09-23T20:41:37.335 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.239774614Z level=info msg="Executing migration" id="Add non-unique index alert_rule_tag_alert_id" 2023-09-23T20:41:37.336 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.240133411Z level=info msg="Executing migration" id="Drop old annotation table v4" 2023-09-23T20:41:37.336 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.240444152Z level=info msg="Executing migration" id="create annotation table v5" 2023-09-23T20:41:37.336 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.240881815Z level=info msg="Executing migration" id="add index annotation 0 v3" 2023-09-23T20:41:37.336 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.241281241Z level=info msg="Executing migration" id="add index annotation 1 v3" 2023-09-23T20:41:37.336 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.241682499Z level=info msg="Executing migration" id="add index annotation 2 v3" 2023-09-23T20:41:37.336 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.242077167Z level=info msg="Executing migration" id="add index annotation 3 v3" 2023-09-23T20:41:37.336 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.242533003Z level=info msg="Executing migration" id="add index annotation 4 v3" 2023-09-23T20:41:37.336 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.242963584Z level=info msg="Executing migration" id="Update annotation table charset" 2023-09-23T20:41:37.336 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.243206486Z level=info msg="Executing migration" id="Add column region_id to annotation table" 2023-09-23T20:41:37.336 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.243956447Z level=info msg="Executing migration" id="Drop category_id index" 2023-09-23T20:41:37.336 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.244293727Z level=info msg="Executing migration" id="Add column tags to annotation table" 2023-09-23T20:41:37.336 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.245039995Z level=info msg="Executing migration" id="Create annotation_tag table v2" 2023-09-23T20:41:37.337 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.245467292Z level=info msg="Executing migration" id="Add unique index annotation_tag.annotation_id_tag_id" 2023-09-23T20:41:37.337 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.245884975Z level=info msg="Executing migration" id="drop index UQE_annotation_tag_annotation_id_tag_id - v2" 2023-09-23T20:41:37.337 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.246226034Z level=info msg="Executing migration" id="Rename table annotation_tag to annotation_tag_v2 - v2" 2023-09-23T20:41:37.337 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.248065563Z level=info msg="Executing migration" id="Create annotation_tag table v3" 2023-09-23T20:41:37.337 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.248444092Z level=info msg="Executing migration" id="create index UQE_annotation_tag_annotation_id_tag_id - Add unique index annotation_tag.annotation_id_tag_id V3" 2023-09-23T20:41:37.337 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.248915106Z level=info msg="Executing migration" id="copy annotation_tag v2 to v3" 2023-09-23T20:41:37.337 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.249162711Z level=info msg="Executing migration" id="drop table annotation_tag_v2" 2023-09-23T20:41:37.337 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.249520332Z level=info msg="Executing migration" id="Update alert annotations and set TEXT to empty" 2023-09-23T20:41:37.337 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.249867563Z level=info msg="Executing migration" id="Add created time to annotation table" 2023-09-23T20:41:37.337 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.250524404Z level=info msg="Executing migration" id="Add updated time to annotation table" 2023-09-23T20:41:37.337 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.25117942Z level=info msg="Executing migration" id="Add index for created in annotation table" 2023-09-23T20:41:37.337 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.251653902Z level=info msg="Executing migration" id="Add index for updated in annotation table" 2023-09-23T20:41:37.337 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.252046423Z level=info msg="Executing migration" id="Convert existing annotations from seconds to milliseconds" 2023-09-23T20:41:37.338 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.252307069Z level=info msg="Executing migration" id="Add epoch_end column" 2023-09-23T20:41:37.338 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.253059663Z level=info msg="Executing migration" id="Add index for epoch_end" 2023-09-23T20:41:37.338 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.253493662Z level=info msg="Executing migration" id="Make epoch_end the same as epoch" 2023-09-23T20:41:37.338 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.253805841Z level=info msg="Executing migration" id="Move region to single row" 2023-09-23T20:41:37.338 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.254145981Z level=info msg="Executing migration" id="Remove index org_id_epoch from annotation table" 2023-09-23T20:41:37.338 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.254534237Z level=info msg="Executing migration" id="Remove index org_id_dashboard_id_panel_id_epoch from annotation table" 2023-09-23T20:41:37.338 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.254900637Z level=info msg="Executing migration" id="Add index for org_id_dashboard_id_epoch_end_epoch on annotation table" 2023-09-23T20:41:37.338 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.255252976Z level=info msg="Executing migration" id="Add index for org_id_epoch_end_epoch on annotation table" 2023-09-23T20:41:37.338 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.255620795Z level=info msg="Executing migration" id="Remove index org_id_epoch_epoch_end from annotation table" 2023-09-23T20:41:37.338 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.255937757Z level=info msg="Executing migration" id="Add index for alert_id on annotation table" 2023-09-23T20:41:37.339 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.256290243Z level=info msg="Executing migration" id="Increase tags column to length 4096" 2023-09-23T20:41:37.339 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.256584753Z level=info msg="Executing migration" id="create test_data table" 2023-09-23T20:41:37.339 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.256939685Z level=info msg="Executing migration" id="create dashboard_version table v1" 2023-09-23T20:41:37.339 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.257291662Z level=info msg="Executing migration" id="add index dashboard_version.dashboard_id" 2023-09-23T20:41:37.339 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.257671362Z level=info msg="Executing migration" id="add unique index dashboard_version.dashboard_id and dashboard_version.version" 2023-09-23T20:41:37.339 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.258034682Z level=info msg="Executing migration" id="Set dashboard version to 1 where 0" 2023-09-23T20:41:37.339 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.258294645Z level=info msg="Executing migration" id="save existing dashboard data in dashboard_version table v1" 2023-09-23T20:41:37.339 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.258593199Z level=info msg="Executing migration" id="alter dashboard_version.data to mediumtext v1" 2023-09-23T20:41:37.339 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.258821361Z level=info msg="Executing migration" id="create team table" 2023-09-23T20:41:37.340 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.259242326Z level=info msg="Executing migration" id="add index team.org_id" 2023-09-23T20:41:37.340 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.259656246Z level=info msg="Executing migration" id="add unique index team_org_id_name" 2023-09-23T20:41:37.340 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.260036722Z level=info msg="Executing migration" id="create team member table" 2023-09-23T20:41:37.341 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.26050059Z level=info msg="Executing migration" id="add index team_member.org_id" 2023-09-23T20:41:37.341 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.260965198Z level=info msg="Executing migration" id="add unique index team_member_org_id_team_id_user_id" 2023-09-23T20:41:37.341 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.261392618Z level=info msg="Executing migration" id="add index team_member.team_id" 2023-09-23T20:41:37.341 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.261827555Z level=info msg="Executing migration" id="Add column email to team table" 2023-09-23T20:41:37.341 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.262594176Z level=info msg="Executing migration" id="Add column external to team_member table" 2023-09-23T20:41:37.341 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.263358514Z level=info msg="Executing migration" id="Add column permission to team_member table" 2023-09-23T20:41:37.341 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.264179126Z level=info msg="Executing migration" id="create dashboard acl table" 2023-09-23T20:41:37.341 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.264598626Z level=info msg="Executing migration" id="add index dashboard_acl_dashboard_id" 2023-09-23T20:41:37.341 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.265076727Z level=info msg="Executing migration" id="add unique index dashboard_acl_dashboard_id_user_id" 2023-09-23T20:41:37.341 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.265483879Z level=info msg="Executing migration" id="add unique index dashboard_acl_dashboard_id_team_id" 2023-09-23T20:41:37.341 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.265884481Z level=info msg="Executing migration" id="add index dashboard_acl_user_id" 2023-09-23T20:41:37.342 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.266334812Z level=info msg="Executing migration" id="add index dashboard_acl_team_id" 2023-09-23T20:41:37.342 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.266773023Z level=info msg="Executing migration" id="add index dashboard_acl_org_id_role" 2023-09-23T20:41:37.342 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.267173627Z level=info msg="Executing migration" id="add index dashboard_permission" 2023-09-23T20:41:37.342 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.267622994Z level=info msg="Executing migration" id="save default acl rules in dashboard_acl table" 2023-09-23T20:41:37.342 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.267994Z level=info msg="Executing migration" id="delete acl rules for deleted dashboards and folders" 2023-09-23T20:41:37.342 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.268305904Z level=info msg="Executing migration" id="create tag table" 2023-09-23T20:41:37.342 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.268697395Z level=info msg="Executing migration" id="add index tag.key_value" 2023-09-23T20:41:37.342 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.269161109Z level=info msg="Executing migration" id="create login attempt table" 2023-09-23T20:41:37.342 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.269546759Z level=info msg="Executing migration" id="add index login_attempt.username" 2023-09-23T20:41:37.342 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.269969509Z level=info msg="Executing migration" id="drop index IDX_login_attempt_username - v1" 2023-09-23T20:41:37.342 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.270430951Z level=info msg="Executing migration" id="Rename table login_attempt to login_attempt_tmp_qwerty - v1" 2023-09-23T20:41:37.343 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.272493145Z level=info msg="Executing migration" id="create login_attempt v2" 2023-09-23T20:41:37.343 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.272845726Z level=info msg="Executing migration" id="create index IDX_login_attempt_username - v2" 2023-09-23T20:41:37.343 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.273218527Z level=info msg="Executing migration" id="copy login_attempt v1 to v2" 2023-09-23T20:41:37.343 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.273504983Z level=info msg="Executing migration" id="drop login_attempt_tmp_qwerty" 2023-09-23T20:41:37.343 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.273792497Z level=info msg="Executing migration" id="create user auth table" 2023-09-23T20:41:37.343 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.274096778Z level=info msg="Executing migration" id="create index IDX_user_auth_auth_module_auth_id - v1" 2023-09-23T20:41:37.343 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.274478936Z level=info msg="Executing migration" id="alter user_auth.auth_id to length 190" 2023-09-23T20:41:37.343 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.274779504Z level=info msg="Executing migration" id="Add OAuth access token to user_auth" 2023-09-23T20:41:37.343 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.275501549Z level=info msg="Executing migration" id="Add OAuth refresh token to user_auth" 2023-09-23T20:41:37.343 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.276210591Z level=info msg="Executing migration" id="Add OAuth token type to user_auth" 2023-09-23T20:41:37.343 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.276934511Z level=info msg="Executing migration" id="Add OAuth expiry to user_auth" 2023-09-23T20:41:37.344 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.277688987Z level=info msg="Executing migration" id="Add index to user_id column in user_auth" 2023-09-23T20:41:37.344 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.27809155Z level=info msg="Executing migration" id="Add OAuth ID token to user_auth" 2023-09-23T20:41:37.344 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.278815815Z level=info msg="Executing migration" id="create server_lock table" 2023-09-23T20:41:37.344 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.2791582Z level=info msg="Executing migration" id="add index server_lock.operation_uid" 2023-09-23T20:41:37.344 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.279582595Z level=info msg="Executing migration" id="create user auth token table" 2023-09-23T20:41:37.344 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.279964722Z level=info msg="Executing migration" id="add unique index user_auth_token.auth_token" 2023-09-23T20:41:37.344 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.280342384Z level=info msg="Executing migration" id="add unique index user_auth_token.prev_auth_token" 2023-09-23T20:41:37.344 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.280716598Z level=info msg="Executing migration" id="add index user_auth_token.user_id" 2023-09-23T20:41:37.344 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.281087074Z level=info msg="Executing migration" id="Add revoked_at to the user auth token" 2023-09-23T20:41:37.344 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.28183896Z level=info msg="Executing migration" id="create cache_data table" 2023-09-23T20:41:37.344 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.282196153Z level=info msg="Executing migration" id="add unique index cache_data.cache_key" 2023-09-23T20:41:37.345 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.282590231Z level=info msg="Executing migration" id="create short_url table v1" 2023-09-23T20:41:37.345 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.282963582Z level=info msg="Executing migration" id="add index short_url.org_id-uid" 2023-09-23T20:41:37.345 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.283340029Z level=info msg="Executing migration" id="alter table short_url alter column created_by type to bigint" 2023-09-23T20:41:37.345 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.283593721Z level=info msg="Executing migration" id="delete alert_definition table" 2023-09-23T20:41:37.345 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.28381842Z level=info msg="Executing migration" id="recreate alert_definition table" 2023-09-23T20:41:37.345 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.284221337Z level=info msg="Executing migration" id="add index in alert_definition on org_id and title columns" 2023-09-23T20:41:37.345 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.284636303Z level=info msg="Executing migration" id="add index in alert_definition on org_id and uid columns" 2023-09-23T20:41:37.345 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.285003203Z level=info msg="Executing migration" id="alter alert_definition table data column to mediumtext in mysql" 2023-09-23T20:41:37.345 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.28522091Z level=info msg="Executing migration" id="drop index in alert_definition on org_id and title columns" 2023-09-23T20:41:37.345 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.285589636Z level=info msg="Executing migration" id="drop index in alert_definition on org_id and uid columns" 2023-09-23T20:41:37.345 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.285930905Z level=info msg="Executing migration" id="add unique index in alert_definition on org_id and title columns" 2023-09-23T20:41:37.346 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.286291178Z level=info msg="Executing migration" id="add unique index in alert_definition on org_id and uid columns" 2023-09-23T20:41:37.346 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.286695285Z level=info msg="Executing migration" id="Add column paused in alert_definition" 2023-09-23T20:41:37.346 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.287428608Z level=info msg="Executing migration" id="drop alert_definition table" 2023-09-23T20:41:37.346 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.287818713Z level=info msg="Executing migration" id="delete alert_definition_version table" 2023-09-23T20:41:37.346 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.288054451Z level=info msg="Executing migration" id="recreate alert_definition_version table" 2023-09-23T20:41:37.346 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.288417434Z level=info msg="Executing migration" id="add index in alert_definition_version table on alert_definition_id and version columns" 2023-09-23T20:41:37.346 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.288818366Z level=info msg="Executing migration" id="add index in alert_definition_version table on alert_definition_uid and version columns" 2023-09-23T20:41:37.346 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.289161874Z level=info msg="Executing migration" id="alter alert_definition_version table data column to mediumtext in mysql" 2023-09-23T20:41:37.346 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.289419757Z level=info msg="Executing migration" id="drop alert_definition_version table" 2023-09-23T20:41:37.346 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.289791418Z level=info msg="Executing migration" id="create alert_instance table" 2023-09-23T20:41:37.346 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.290169496Z level=info msg="Executing migration" id="add index in alert_instance table on def_org_id, def_uid and current_state columns" 2023-09-23T20:41:37.347 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.290605348Z level=info msg="Executing migration" id="add index in alert_instance table on def_org_id, current_state columns" 2023-09-23T20:41:37.347 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.290977743Z level=info msg="Executing migration" id="add column current_state_end to alert_instance" 2023-09-23T20:41:37.347 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.291760067Z level=info msg="Executing migration" id="remove index def_org_id, def_uid, current_state on alert_instance" 2023-09-23T20:41:37.347 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.292124412Z level=info msg="Executing migration" id="remove index def_org_id, current_state on alert_instance" 2023-09-23T20:41:37.347 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.292495025Z level=info msg="Executing migration" id="rename def_org_id to rule_org_id in alert_instance" 2023-09-23T20:41:37.347 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.29583305Z level=info msg="Executing migration" id="rename def_uid to rule_uid in alert_instance" 2023-09-23T20:41:37.347 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.299166733Z level=info msg="Executing migration" id="add index rule_org_id, rule_uid, current_state on alert_instance" 2023-09-23T20:41:37.347 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.299589822Z level=info msg="Executing migration" id="add index rule_org_id, current_state on alert_instance" 2023-09-23T20:41:37.347 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.299963702Z level=info msg="Executing migration" id="add current_reason column related to current_state" 2023-09-23T20:41:37.347 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.300756534Z level=info msg="Executing migration" id="create alert_rule table" 2023-09-23T20:41:37.347 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.301126774Z level=info msg="Executing migration" id="add index in alert_rule on org_id and title columns" 2023-09-23T20:41:37.348 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.301511137Z level=info msg="Executing migration" id="add index in alert_rule on org_id and uid columns" 2023-09-23T20:41:37.348 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.301891223Z level=info msg="Executing migration" id="add index in alert_rule on org_id, namespace_uid, group_uid columns" 2023-09-23T20:41:37.348 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.302262858Z level=info msg="Executing migration" id="alter alert_rule table data column to mediumtext in mysql" 2023-09-23T20:41:37.348 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.302514772Z level=info msg="Executing migration" id="add column for to alert_rule" 2023-09-23T20:41:37.348 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.30326901Z level=info msg="Executing migration" id="add column annotations to alert_rule" 2023-09-23T20:41:37.348 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.304053409Z level=info msg="Executing migration" id="add column labels to alert_rule" 2023-09-23T20:41:37.348 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.304851296Z level=info msg="Executing migration" id="remove unique index from alert_rule on org_id, title columns" 2023-09-23T20:41:37.348 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.305218104Z level=info msg="Executing migration" id="add index in alert_rule on org_id, namespase_uid and title columns" 2023-09-23T20:41:37.348 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.305612467Z level=info msg="Executing migration" id="add dashboard_uid column to alert_rule" 2023-09-23T20:41:37.348 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.306368761Z level=info msg="Executing migration" id="add panel_id column to alert_rule" 2023-09-23T20:41:37.348 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.307158884Z level=info msg="Executing migration" id="add index in alert_rule on org_id, dashboard_uid and panel_id columns" 2023-09-23T20:41:37.349 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.307603186Z level=info msg="Executing migration" id="add rule_group_idx column to alert_rule" 2023-09-23T20:41:37.349 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.308385771Z level=info msg="Executing migration" id="add is_paused column to alert_rule table" 2023-09-23T20:41:37.349 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.309178816Z level=info msg="Executing migration" id="fix is_paused column for alert_rule table" 2023-09-23T20:41:37.349 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.309474634Z level=info msg="Executing migration" id="create alert_rule_version table" 2023-09-23T20:41:37.349 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.309912556Z level=info msg="Executing migration" id="add index in alert_rule_version table on rule_org_id, rule_uid and version columns" 2023-09-23T20:41:37.349 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.31033594Z level=info msg="Executing migration" id="add index in alert_rule_version table on rule_org_id, rule_namespace_uid and rule_group columns" 2023-09-23T20:41:37.349 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.310750523Z level=info msg="Executing migration" id="alter alert_rule_version table data column to mediumtext in mysql" 2023-09-23T20:41:37.349 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.310985592Z level=info msg="Executing migration" id="add column for to alert_rule_version" 2023-09-23T20:41:37.349 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.311841357Z level=info msg="Executing migration" id="add column annotations to alert_rule_version" 2023-09-23T20:41:37.349 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.312665765Z level=info msg="Executing migration" id="add column labels to alert_rule_version" 2023-09-23T20:41:37.349 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.313464728Z level=info msg="Executing migration" id="add rule_group_idx column to alert_rule_version" 2023-09-23T20:41:37.349 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.314652562Z level=info msg="Executing migration" id="add is_paused column to alert_rule_versions table" 2023-09-23T20:41:37.350 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.315874104Z level=info msg="Executing migration" id="fix is_paused column for alert_rule_version table" 2023-09-23T20:41:37.350 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.31622171Z level=info msg="Executing migration" id=create_alert_configuration_table 2023-09-23T20:41:37.350 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.316718965Z level=info msg="Executing migration" id="Add column default in alert_configuration" 2023-09-23T20:41:37.620 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.317940047Z level=info msg="Executing migration" id="alert alert_configuration alertmanager_configuration column from TEXT to MEDIUMTEXT if mysql" 2023-09-23T20:41:37.620 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.318328377Z level=info msg="Executing migration" id="add column org_id in alert_configuration" 2023-09-23T20:41:37.620 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.319630549Z level=info msg="Executing migration" id="add index in alert_configuration table on org_id column" 2023-09-23T20:41:37.621 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.320108694Z level=info msg="Executing migration" id="add configuration_hash column to alert_configuration" 2023-09-23T20:41:37.621 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.320961604Z level=info msg="Executing migration" id=create_ngalert_configuration_table 2023-09-23T20:41:37.621 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.321385634Z level=info msg="Executing migration" id="add index in ngalert_configuration on org_id column" 2023-09-23T20:41:37.621 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.32181453Z level=info msg="Executing migration" id="add column send_alerts_to in ngalert_configuration" 2023-09-23T20:41:37.621 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.322696718Z level=info msg="Executing migration" id="create provenance_type table" 2023-09-23T20:41:37.621 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.323082178Z level=info msg="Executing migration" id="add index to uniquify (record_key, record_type, org_id) columns" 2023-09-23T20:41:37.621 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.323537905Z level=info msg="Executing migration" id="create alert_image table" 2023-09-23T20:41:37.621 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.323924679Z level=info msg="Executing migration" id="add unique index on token to alert_image table" 2023-09-23T20:41:37.621 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.324393796Z level=info msg="Executing migration" id="support longer URLs in alert_image table" 2023-09-23T20:41:37.621 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.324685523Z level=info msg="Executing migration" id=create_alert_configuration_history_table 2023-09-23T20:41:37.621 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.325101931Z level=info msg="Executing migration" id="drop non-unique orgID index on alert_configuration" 2023-09-23T20:41:37.622 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.325523458Z level=info msg="Executing migration" id="drop unique orgID index on alert_configuration if exists" 2023-09-23T20:41:37.622 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.325628291Z level=warn msg="Skipping migration: Already executed, but not recorded in migration log" id="drop unique orgID index on alert_configuration if exists" 2023-09-23T20:41:37.622 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.325886547Z level=info msg="Executing migration" id="extract alertmanager configuration history to separate table" 2023-09-23T20:41:37.622 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.32619171Z level=info msg="Executing migration" id="add unique index on orgID to alert_configuration" 2023-09-23T20:41:37.622 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.326612393Z level=info msg="Executing migration" id="clear migration entry \"remove unified alerting data\"" 2023-09-23T20:41:37.622 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.326753935Z level=info msg="Executing migration" id="move dashboard alerts to unified alerting" 2023-09-23T20:41:37.622 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.326896721Z level=info msg="alerts found to migrate" alerts=0 2023-09-23T20:41:37.622 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.327252935Z level=info msg="Executing migration" id="create library_element table v1" 2023-09-23T20:41:37.622 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.327714926Z level=info msg="Executing migration" id="add index library_element org_id-folder_id-name-kind" 2023-09-23T20:41:37.622 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.328171712Z level=info msg="Executing migration" id="create library_element_connection table v1" 2023-09-23T20:41:37.622 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.328600398Z level=info msg="Executing migration" id="add index library_element_connection element_id-kind-connection_id" 2023-09-23T20:41:37.622 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.328991118Z level=info msg="Executing migration" id="add unique index library_element org_id_uid" 2023-09-23T20:41:37.623 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.329415926Z level=info msg="Executing migration" id="increase max description length to 2048" 2023-09-23T20:41:37.623 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.32970675Z level=info msg="Executing migration" id="clone move dashboard alerts to unified alerting" 2023-09-23T20:41:37.623 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.330070725Z level=info msg="Executing migration" id="create data_keys table" 2023-09-23T20:41:37.623 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.330516506Z level=info msg="Executing migration" id="create secrets table" 2023-09-23T20:41:37.623 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.330939247Z level=info msg="Executing migration" id="rename data_keys name column to id" 2023-09-23T20:41:37.623 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.335642839Z level=info msg="Executing migration" id="add name column into data_keys" 2023-09-23T20:41:37.623 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.336550586Z level=info msg="Executing migration" id="copy data_keys id column values into name" 2023-09-23T20:41:37.623 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.336882038Z level=info msg="Executing migration" id="rename data_keys name column to label" 2023-09-23T20:41:37.623 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.341109918Z level=info msg="Executing migration" id="rename data_keys id column back to name" 2023-09-23T20:41:37.623 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.345656839Z level=info msg="Executing migration" id="create kv_store table v1" 2023-09-23T20:41:37.623 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.346110315Z level=info msg="Executing migration" id="add index kv_store.org_id-namespace-key" 2023-09-23T20:41:37.623 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.346672342Z level=info msg="Executing migration" id="update dashboard_uid and panel_id from existing annotations" 2023-09-23T20:41:37.624 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.34704769Z level=info msg="Executing migration" id="create permission table" 2023-09-23T20:41:37.624 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.347509897Z level=info msg="Executing migration" id="add unique index permission.role_id" 2023-09-23T20:41:37.624 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.347959709Z level=info msg="Executing migration" id="add unique index role_id_action_scope" 2023-09-23T20:41:37.624 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.348433115Z level=info msg="Executing migration" id="create role table" 2023-09-23T20:41:37.624 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.348798673Z level=info msg="Executing migration" id="add column display_name" 2023-09-23T20:41:37.624 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.349702764Z level=info msg="Executing migration" id="add column group_name" 2023-09-23T20:41:37.624 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.3506138Z level=info msg="Executing migration" id="add index role.org_id" 2023-09-23T20:41:37.624 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.35103104Z level=info msg="Executing migration" id="add unique index role_org_id_name" 2023-09-23T20:41:37.624 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.351472976Z level=info msg="Executing migration" id="add index role_org_id_uid" 2023-09-23T20:41:37.624 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.351888136Z level=info msg="Executing migration" id="create team role table" 2023-09-23T20:41:37.624 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.352252592Z level=info msg="Executing migration" id="add index team_role.org_id" 2023-09-23T20:41:37.624 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.352671444Z level=info msg="Executing migration" id="add unique index team_role_org_id_team_id_role_id" 2023-09-23T20:41:37.625 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.353114534Z level=info msg="Executing migration" id="add index team_role.team_id" 2023-09-23T20:41:37.625 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.353524846Z level=info msg="Executing migration" id="create user role table" 2023-09-23T20:41:37.625 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.353884628Z level=info msg="Executing migration" id="add index user_role.org_id" 2023-09-23T20:41:37.625 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.354294905Z level=info msg="Executing migration" id="add unique index user_role_org_id_user_id_role_id" 2023-09-23T20:41:37.625 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.354707165Z level=info msg="Executing migration" id="add index user_role.user_id" 2023-09-23T20:41:37.625 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.355107108Z level=info msg="Executing migration" id="create builtin role table" 2023-09-23T20:41:37.625 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.355515816Z level=info msg="Executing migration" id="add index builtin_role.role_id" 2023-09-23T20:41:37.625 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.355940061Z level=info msg="Executing migration" id="add index builtin_role.name" 2023-09-23T20:41:37.625 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.35635905Z level=info msg="Executing migration" id="Add column org_id to builtin_role table" 2023-09-23T20:41:37.625 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.357294344Z level=info msg="Executing migration" id="add index builtin_role.org_id" 2023-09-23T20:41:37.625 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.35776143Z level=info msg="Executing migration" id="add unique index builtin_role_org_id_role_id_role" 2023-09-23T20:41:37.626 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.358211959Z level=info msg="Executing migration" id="Remove unique index role_org_id_uid" 2023-09-23T20:41:37.626 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.358697756Z level=info msg="Executing migration" id="add unique index role.uid" 2023-09-23T20:41:37.626 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.359089866Z level=info msg="Executing migration" id="create seed assignment table" 2023-09-23T20:41:37.626 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.359473745Z level=info msg="Executing migration" id="add unique index builtin_role_role_name" 2023-09-23T20:41:37.626 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.359902392Z level=info msg="Executing migration" id="add column hidden to role table" 2023-09-23T20:41:37.626 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.36085267Z level=info msg="Executing migration" id="create query_history table v1" 2023-09-23T20:41:37.626 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.361219007Z level=info msg="Executing migration" id="add index query_history.org_id-created_by-datasource_uid" 2023-09-23T20:41:37.626 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.361663802Z level=info msg="Executing migration" id="alter table query_history alter column created_by type to bigint" 2023-09-23T20:41:37.626 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.361900711Z level=info msg="Executing migration" id="teams permissions migration" 2023-09-23T20:41:37.626 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.362291864Z level=info msg="Executing migration" id="dashboard permissions" 2023-09-23T20:41:37.626 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.362774531Z level=info msg="Executing migration" id="dashboard permissions uid scopes" 2023-09-23T20:41:37.627 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.363086092Z level=info msg="Executing migration" id="drop managed folder create actions" 2023-09-23T20:41:37.627 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.363370254Z level=info msg="Executing migration" id="alerting notification permissions" 2023-09-23T20:41:37.627 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.363698814Z level=info msg="Executing migration" id="create query_history_star table v1" 2023-09-23T20:41:37.627 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.364059874Z level=info msg="Executing migration" id="add index query_history.user_id-query_uid" 2023-09-23T20:41:37.628 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.364485493Z level=info msg="Executing migration" id="add column org_id in query_history_star" 2023-09-23T20:41:37.628 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.365417585Z level=info msg="Executing migration" id="alter table query_history_star_mig column user_id type to bigint" 2023-09-23T20:41:37.628 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.365693669Z level=info msg="Executing migration" id="create correlation table v1" 2023-09-23T20:41:37.628 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.366090496Z level=info msg="Executing migration" id="add index correlations.uid" 2023-09-23T20:41:37.628 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.366524673Z level=info msg="Executing migration" id="add index correlations.source_uid" 2023-09-23T20:41:37.628 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.366931336Z level=info msg="Executing migration" id="add correlation config column" 2023-09-23T20:41:37.628 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.367896092Z level=info msg="Executing migration" id="create entity_events table" 2023-09-23T20:41:37.628 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.368302016Z level=info msg="Executing migration" id="create dashboard public config v1" 2023-09-23T20:41:37.628 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.368868912Z level=info msg="Executing migration" id="drop index UQE_dashboard_public_config_uid - v1" 2023-09-23T20:41:37.628 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.368961693Z level=warn msg="Skipping migration: Already executed, but not recorded in migration log" id="drop index UQE_dashboard_public_config_uid - v1" 2023-09-23T20:41:37.628 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.369228618Z level=info msg="Executing migration" id="drop index IDX_dashboard_public_config_org_id_dashboard_uid - v1" 2023-09-23T20:41:37.628 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.369323375Z level=warn msg="Skipping migration: Already executed, but not recorded in migration log" id="drop index IDX_dashboard_public_config_org_id_dashboard_uid - v1" 2023-09-23T20:41:37.629 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.369566526Z level=info msg="Executing migration" id="Drop old dashboard public config table" 2023-09-23T20:41:37.629 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.369918047Z level=info msg="Executing migration" id="recreate dashboard public config v1" 2023-09-23T20:41:37.629 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.370308449Z level=info msg="Executing migration" id="create index UQE_dashboard_public_config_uid - v1" 2023-09-23T20:41:37.629 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.370761122Z level=info msg="Executing migration" id="create index IDX_dashboard_public_config_org_id_dashboard_uid - v1" 2023-09-23T20:41:37.629 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.371223172Z level=info msg="Executing migration" id="drop index UQE_dashboard_public_config_uid - v2" 2023-09-23T20:41:37.629 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.37165631Z level=info msg="Executing migration" id="drop index IDX_dashboard_public_config_org_id_dashboard_uid - v2" 2023-09-23T20:41:37.629 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.372038863Z level=info msg="Executing migration" id="Drop public config table" 2023-09-23T20:41:37.629 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.372529792Z level=info msg="Executing migration" id="Recreate dashboard public config v2" 2023-09-23T20:41:37.629 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.372940615Z level=info msg="Executing migration" id="create index UQE_dashboard_public_config_uid - v2" 2023-09-23T20:41:37.629 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.373399801Z level=info msg="Executing migration" id="create index IDX_dashboard_public_config_org_id_dashboard_uid - v2" 2023-09-23T20:41:37.629 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.37386562Z level=info msg="Executing migration" id="create index UQE_dashboard_public_config_access_token - v2" 2023-09-23T20:41:37.629 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.374298583Z level=info msg="Executing migration" id="Rename table dashboard_public_config to dashboard_public - v2" 2023-09-23T20:41:37.630 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.377350308Z level=info msg="Executing migration" id="add annotations_enabled column" 2023-09-23T20:41:37.630 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.378345953Z level=info msg="Executing migration" id="add time_selection_enabled column" 2023-09-23T20:41:37.630 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.379319857Z level=info msg="Executing migration" id="delete orphaned public dashboards" 2023-09-23T20:41:37.630 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.379636089Z level=info msg="Executing migration" id="add share column" 2023-09-23T20:41:37.630 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.380634506Z level=info msg="Executing migration" id="create default alerting folders" 2023-09-23T20:41:37.630 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.381071245Z level=info msg="Executing migration" id="create file table" 2023-09-23T20:41:37.630 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.381510562Z level=info msg="Executing migration" id="file table idx: path natural pk" 2023-09-23T20:41:37.630 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.3819405Z level=info msg="Executing migration" id="file table idx: parent_folder_path_hash fast folder retrieval" 2023-09-23T20:41:37.630 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.382424628Z level=info msg="Executing migration" id="create file_meta table" 2023-09-23T20:41:37.630 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.382838347Z level=info msg="Executing migration" id="file table idx: path key" 2023-09-23T20:41:37.630 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.383266485Z level=info msg="Executing migration" id="set path collation in file table" 2023-09-23T20:41:37.630 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.38354919Z level=info msg="Executing migration" id="managed permissions migration" 2023-09-23T20:41:37.631 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.383882645Z level=info msg="Executing migration" id="managed folder permissions alert actions migration" 2023-09-23T20:41:37.631 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.384165882Z level=info msg="Executing migration" id="RBAC action name migrator" 2023-09-23T20:41:37.631 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.384603234Z level=info msg="Executing migration" id="Add UID column to playlist" 2023-09-23T20:41:37.631 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.38561661Z level=info msg="Executing migration" id="Update uid column values in playlist" 2023-09-23T20:41:37.631 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.385910859Z level=info msg="Executing migration" id="Add index for uid in playlist" 2023-09-23T20:41:37.631 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.386342166Z level=info msg="Executing migration" id="update group index for alert rules" 2023-09-23T20:41:37.631 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.386723288Z level=info msg="Executing migration" id="managed folder permissions alert actions repeated migration" 2023-09-23T20:41:37.631 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.386997366Z level=info msg="Executing migration" id="admin only folder/dashboard permission" 2023-09-23T20:41:37.631 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.387382538Z level=info msg="Executing migration" id="add action column to seed_assignment" 2023-09-23T20:41:37.631 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.38837871Z level=info msg="Executing migration" id="add scope column to seed_assignment" 2023-09-23T20:41:37.631 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.389404345Z level=info msg="Executing migration" id="remove unique index builtin_role_role_name before nullable update" 2023-09-23T20:41:37.631 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.389823113Z level=info msg="Executing migration" id="update seed_assignment role_name column to nullable" 2023-09-23T20:41:37.632 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.399624854Z level=info msg="Executing migration" id="add unique index builtin_role_name back" 2023-09-23T20:41:37.632 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.400086208Z level=info msg="Executing migration" id="add unique index builtin_role_action_scope" 2023-09-23T20:41:37.632 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.400564912Z level=info msg="Executing migration" id="add primary key to seed_assigment" 2023-09-23T20:41:37.632 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.404019523Z level=info msg="Executing migration" id="managed folder permissions alert actions repeated fixed migration" 2023-09-23T20:41:37.632 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.404350192Z level=info msg="Executing migration" id="migrate external alertmanagers to datsourcse" 2023-09-23T20:41:37.632 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.404682275Z level=info msg="Executing migration" id="create folder table" 2023-09-23T20:41:37.632 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.405036699Z level=info msg="Executing migration" id="Add index for parent_uid" 2023-09-23T20:41:37.632 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.40547784Z level=info msg="Executing migration" id="Add unique index for folder.uid and folder.org_id" 2023-09-23T20:41:37.632 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.40590562Z level=info msg="Executing migration" id="Update folder title length" 2023-09-23T20:41:37.632 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.406154672Z level=info msg="Executing migration" id="Add unique index for folder.title and folder.parent_uid" 2023-09-23T20:41:37.632 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=migrator t=2023-09-23T20:41:37.40660294Z level=info msg="migrations completed" performed=485 skipped=0 duration=245.935785ms 2023-09-23T20:41:37.632 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=sqlstore t=2023-09-23T20:41:37.407041488Z level=info msg="Created default organization" 2023-09-23T20:41:37.633 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=plugin.loader t=2023-09-23T20:41:37.462823441Z level=info msg="Plugin registered" pluginID=input 2023-09-23T20:41:37.633 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=plugin.loader t=2023-09-23T20:41:37.491238525Z level=info msg="Plugin registered" pluginID=grafana-piechart-panel 2023-09-23T20:41:37.633 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=plugin.loader t=2023-09-23T20:41:37.491292943Z level=info msg="Plugin registered" pluginID=vonage-status-panel 2023-09-23T20:41:37.633 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=secrets t=2023-09-23T20:41:37.491365358Z level=info msg="Envelope encryption state" enabled=true currentprovider=secretKey.v1 2023-09-23T20:41:37.633 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=query_data t=2023-09-23T20:41:37.494518953Z level=info msg="Query Service initialization" 2023-09-23T20:41:37.633 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=live.push_http t=2023-09-23T20:41:37.497689049Z level=info msg="Live Push Gateway initialization" 2023-09-23T20:41:38.413 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:41:38 smithi070 ceph-mon[100755]: from='mgr.14451 ' entity='mgr.y' 2023-09-23T20:41:38.413 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:41:38 smithi070 ceph-mon[100755]: from='mgr.14451 ' entity='mgr.y' 2023-09-23T20:41:38.413 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:41:38 smithi070 ceph-mon[100755]: from='mgr.14451 ' entity='mgr.y' 2023-09-23T20:41:38.413 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:41:38 smithi070 ceph-mon[100755]: from='mgr.14451 ' entity='mgr.y' 2023-09-23T20:41:38.413 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:41:38 smithi070 ceph-mon[100755]: from='mgr.14451 172.21.15.26:0/1329296977' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:41:38.414 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:41:38 smithi070 ceph-mon[100755]: pgmap v24: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:41:38.414 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:38 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=infra.usagestats.collector t=2023-09-23T20:41:38.236248677Z level=info msg="registering usage stat providers" usageStatsProvidersLen=2 2023-09-23T20:41:38.414 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:38 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=provisioning.datasources t=2023-09-23T20:41:38.236541292Z level=warn msg="[Deprecated] the datasource provisioning config is outdated. please upgrade" filename=/etc/grafana/provisioning/datasources/ceph-dashboard.yml 2023-09-23T20:41:38.414 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:38 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=provisioning.datasources t=2023-09-23T20:41:38.270274837Z level=info msg="inserting datasource from configuration " name=Dashboard1 uid=P43CA22E17D0F9596 2023-09-23T20:41:38.415 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:38 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=provisioning.datasources t=2023-09-23T20:41:38.303450699Z level=info msg="inserting datasource from configuration " name=Loki uid=P8E80F9AEF21F6940 2023-09-23T20:41:38.415 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:38 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=provisioning.alerting t=2023-09-23T20:41:38.325160322Z level=info msg="starting to provision alerting" 2023-09-23T20:41:38.415 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:38 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=provisioning.alerting t=2023-09-23T20:41:38.325180109Z level=info msg="finished to provision alerting" 2023-09-23T20:41:38.415 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:38 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=grafanaStorageLogger t=2023-09-23T20:41:38.325377796Z level=info msg="storage starting" 2023-09-23T20:41:38.415 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:38 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=ngalert.state.manager t=2023-09-23T20:41:38.325420791Z level=info msg="Warming state cache for startup" 2023-09-23T20:41:38.416 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:38 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=http.server t=2023-09-23T20:41:38.327323748Z level=info msg="HTTP Server Listen" address=[::]:3000 protocol=https subUrl= socket= 2023-09-23T20:41:38.416 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:38 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=ngalert.state.manager t=2023-09-23T20:41:38.335673037Z level=info msg="State cache has been initialized" states=0 duration=10.250802ms 2023-09-23T20:41:38.416 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:38 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=ngalert.multiorg.alertmanager t=2023-09-23T20:41:38.335722938Z level=info msg="starting MultiOrg Alertmanager" 2023-09-23T20:41:38.416 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:38 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=ticker t=2023-09-23T20:41:38.335747125Z level=info msg=starting first_tick=2023-09-23T20:41:40Z 2023-09-23T20:41:38.416 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:38 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=sqlstore.transactions t=2023-09-23T20:41:38.335763983Z level=info msg="Database locked, sleeping then retrying" error="database is locked" retry=0 code="database is locked" 2023-09-23T20:41:38.416 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:38 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=sqlstore.transactions t=2023-09-23T20:41:38.372159394Z level=info msg="Database locked, sleeping then retrying" error="database is locked" retry=0 code="database is locked" 2023-09-23T20:41:38.417 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:38 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=sqlstore.transactions t=2023-09-23T20:41:38.412756493Z level=info msg="Database locked, sleeping then retrying" error="database is locked" retry=0 code="database is locked" 2023-09-23T20:41:38.593 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:41:38 smithi026 ceph-mon[109912]: from='mgr.14451 ' entity='mgr.y' 2023-09-23T20:41:38.593 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:41:38 smithi026 ceph-mon[109912]: from='mgr.14451 ' entity='mgr.y' 2023-09-23T20:41:38.593 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:41:38 smithi026 ceph-mon[109912]: from='mgr.14451 ' entity='mgr.y' 2023-09-23T20:41:38.593 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:41:38 smithi026 ceph-mon[109912]: from='mgr.14451 ' entity='mgr.y' 2023-09-23T20:41:38.593 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:41:38 smithi026 ceph-mon[109912]: from='mgr.14451 172.21.15.26:0/1329296977' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:41:38.595 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:41:38 smithi026 ceph-mon[109912]: pgmap v24: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:41:38.595 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:41:38 smithi026 ceph-mon[95839]: from='mgr.14451 ' entity='mgr.y' 2023-09-23T20:41:38.595 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:41:38 smithi026 ceph-mon[95839]: from='mgr.14451 ' entity='mgr.y' 2023-09-23T20:41:38.595 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:41:38 smithi026 ceph-mon[95839]: from='mgr.14451 ' entity='mgr.y' 2023-09-23T20:41:38.596 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:41:38 smithi026 ceph-mon[95839]: from='mgr.14451 ' entity='mgr.y' 2023-09-23T20:41:38.596 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:41:38 smithi026 ceph-mon[95839]: from='mgr.14451 172.21.15.26:0/1329296977' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:41:38.596 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:41:38 smithi026 ceph-mon[95839]: pgmap v24: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:41:38.869 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:41:38 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[137477]: logger=sqlstore.transactions t=2023-09-23T20:41:38.423334716Z level=info msg="Database locked, sleeping then retrying" error="database is locked" retry=1 code="database is locked" 2023-09-23T20:41:39.191 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:41:39.645 INFO:teuthology.orchestra.run.smithi026.stdout:55834574886 2023-09-23T20:41:39.765 INFO:teuthology.orchestra.run.smithi026.stdout:236223201292 2023-09-23T20:41:40.188 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:41:39 smithi026 ceph-mon[109912]: from='mgr.14451 ' entity='mgr.y' 2023-09-23T20:41:40.188 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:41:39 smithi026 ceph-mon[109912]: from='client.? 172.21.15.26:0/1941607208' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2023-09-23T20:41:40.188 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:41:39 smithi026 ceph-mon[109912]: from='client.? 172.21.15.26:0/2912954720' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2023-09-23T20:41:40.188 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:41:39 smithi026 ceph-mon[95839]: from='mgr.14451 ' entity='mgr.y' 2023-09-23T20:41:40.188 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:41:39 smithi026 ceph-mon[95839]: from='client.? 172.21.15.26:0/1941607208' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2023-09-23T20:41:40.188 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:41:39 smithi026 ceph-mon[95839]: from='client.? 172.21.15.26:0/2912954720' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2023-09-23T20:41:40.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:41:39 smithi070 ceph-mon[100755]: from='mgr.14451 ' entity='mgr.y' 2023-09-23T20:41:40.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:41:39 smithi070 ceph-mon[100755]: from='client.? 172.21.15.26:0/1941607208' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2023-09-23T20:41:40.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:41:39 smithi070 ceph-mon[100755]: from='client.? 172.21.15.26:0/2912954720' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2023-09-23T20:41:40.625 INFO:tasks.cephadm.ceph_manager.ceph:need seq 55834574885 got 55834574886 for osd.1 2023-09-23T20:41:40.625 DEBUG:teuthology.parallel:result is None 2023-09-23T20:41:40.714 INFO:tasks.cephadm.ceph_manager.ceph:need seq 236223201291 got 236223201292 for osd.7 2023-09-23T20:41:40.714 DEBUG:teuthology.parallel:result is None 2023-09-23T20:41:41.163 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:41:40 smithi026 ceph-mon[95839]: pgmap v25: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:41:41.163 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:41:40 smithi026 ceph-mon[109912]: pgmap v25: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:41:41.258 INFO:teuthology.orchestra.run.smithi026.stdout:111669149725 2023-09-23T20:41:41.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:41:40 smithi070 ceph-mon[100755]: pgmap v25: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:41:41.409 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:41:41.607 INFO:teuthology.orchestra.run.smithi026.stdout:77309411362 2023-09-23T20:41:41.825 INFO:tasks.cephadm.ceph_manager.ceph:need seq 111669149724 got 111669149725 for osd.3 2023-09-23T20:41:41.825 DEBUG:teuthology.parallel:result is None 2023-09-23T20:41:41.869 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 20:41:41 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-x[102861]: ::ffff:172.21.15.70 - - [23/Sep/2023:20:41:41] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2023-09-23T20:41:42.192 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:41:41 smithi026 ceph-mon[109912]: from='client.? 172.21.15.26:0/1465470800' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2023-09-23T20:41:42.192 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:41:41 smithi026 ceph-mon[109912]: from='client.? 172.21.15.26:0/2827554985' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2023-09-23T20:41:42.192 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:41:41 smithi026 ceph-mon[109912]: from='mgr.14451 ' entity='mgr.y' 2023-09-23T20:41:42.192 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:41:41 smithi026 ceph-mon[109912]: from='mgr.14451 ' entity='mgr.y' 2023-09-23T20:41:42.192 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:41:41 smithi026 ceph-mon[95839]: from='client.? 172.21.15.26:0/1465470800' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2023-09-23T20:41:42.193 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:41:41 smithi026 ceph-mon[95839]: from='client.? 172.21.15.26:0/2827554985' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2023-09-23T20:41:42.193 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:41:41 smithi026 ceph-mon[95839]: from='mgr.14451 ' entity='mgr.y' 2023-09-23T20:41:42.193 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:41:41 smithi026 ceph-mon[95839]: from='mgr.14451 ' entity='mgr.y' 2023-09-23T20:41:42.349 INFO:teuthology.orchestra.run.smithi026.stdout:201863462930 2023-09-23T20:41:42.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:41:41 smithi070 ceph-mon[100755]: from='client.? 172.21.15.26:0/1465470800' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2023-09-23T20:41:42.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:41:41 smithi070 ceph-mon[100755]: from='client.? 172.21.15.26:0/2827554985' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2023-09-23T20:41:42.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:41:41 smithi070 ceph-mon[100755]: from='mgr.14451 ' entity='mgr.y' 2023-09-23T20:41:42.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:41:41 smithi070 ceph-mon[100755]: from='mgr.14451 ' entity='mgr.y' 2023-09-23T20:41:42.999 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:41:42 smithi026 ceph-mon[109912]: pgmap v26: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:41:42.999 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:41:42 smithi026 ceph-mon[109912]: from='client.? 172.21.15.26:0/541591020' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2023-09-23T20:41:43.000 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:41:42 smithi026 ceph-mon[95839]: pgmap v26: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:41:43.000 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:41:42 smithi026 ceph-mon[95839]: from='client.? 172.21.15.26:0/541591020' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2023-09-23T20:41:43.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:41:42 smithi070 ceph-mon[100755]: pgmap v26: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:41:43.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:41:42 smithi070 ceph-mon[100755]: from='client.? 172.21.15.26:0/541591020' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2023-09-23T20:41:43.877 INFO:teuthology.orchestra.run.smithi026.stdout:133143986202 2023-09-23T20:41:44.436 INFO:tasks.cephadm.ceph_manager.ceph:need seq 201863462928 got 201863462930 for osd.6 2023-09-23T20:41:44.436 DEBUG:teuthology.parallel:result is None 2023-09-23T20:41:44.623 INFO:tasks.cephadm.ceph_manager.ceph:need seq 77309411360 got 77309411362 for osd.2 2023-09-23T20:41:44.623 DEBUG:teuthology.parallel:result is None 2023-09-23T20:41:44.670 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:41:44 smithi026 ceph-mon[109912]: pgmap v27: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:41:44.670 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:41:44 smithi026 ceph-mon[109912]: from='client.? 172.21.15.26:0/3720120613' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2023-09-23T20:41:44.671 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:41:44 smithi026 ceph-mon[95839]: pgmap v27: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:41:44.671 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:41:44 smithi026 ceph-mon[95839]: from='client.? 172.21.15.26:0/3720120613' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2023-09-23T20:41:44.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:41:44 smithi070 ceph-mon[100755]: pgmap v27: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:41:44.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:41:44 smithi070 ceph-mon[100755]: from='client.? 172.21.15.26:0/3720120613' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2023-09-23T20:41:44.928 INFO:teuthology.orchestra.run.smithi026.stdout:34359738412 2023-09-23T20:41:44.930 INFO:tasks.cephadm.ceph_manager.ceph:need seq 133143986200 got 133143986202 for osd.4 2023-09-23T20:41:44.931 DEBUG:teuthology.parallel:result is None 2023-09-23T20:41:45.469 INFO:tasks.cephadm.ceph_manager.ceph:need seq 34359738410 got 34359738412 for osd.0 2023-09-23T20:41:45.470 DEBUG:teuthology.parallel:result is None 2023-09-23T20:41:45.806 INFO:teuthology.orchestra.run.smithi026.stdout:167503724567 2023-09-23T20:41:45.836 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:41:45 smithi026 ceph-mon[109912]: from='client.? 172.21.15.26:0/3083178120' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2023-09-23T20:41:45.836 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:41:45 smithi026 ceph-mon[95839]: from='client.? 172.21.15.26:0/3083178120' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2023-09-23T20:41:45.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:41:45 smithi070 ceph-mon[100755]: from='client.? 172.21.15.26:0/3083178120' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2023-09-23T20:41:46.401 INFO:tasks.cephadm.ceph_manager.ceph:need seq 167503724565 got 167503724567 for osd.5 2023-09-23T20:41:46.402 DEBUG:teuthology.parallel:result is None 2023-09-23T20:41:46.402 INFO:tasks.cephadm.ceph_manager.ceph:waiting for clean 2023-09-23T20:41:46.402 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T20:41:46.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:41:46 smithi070 ceph-mon[100755]: from='client.? 172.21.15.26:0/344652723' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2023-09-23T20:41:46.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:41:46 smithi070 ceph-mon[100755]: pgmap v28: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:41:46.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:41:46 smithi026 ceph-mon[95839]: from='client.? 172.21.15.26:0/344652723' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2023-09-23T20:41:46.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:41:46 smithi026 ceph-mon[95839]: pgmap v28: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:41:46.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:41:46 smithi026 ceph-mon[109912]: from='client.? 172.21.15.26:0/344652723' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2023-09-23T20:41:46.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:41:46 smithi026 ceph-mon[109912]: pgmap v28: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:41:47.165 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:41:48.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:41:48 smithi070 ceph-mon[100755]: from='mgr.14451 ' entity='mgr.y' 2023-09-23T20:41:48.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:41:48 smithi070 ceph-mon[100755]: from='mgr.14451 ' entity='mgr.y' 2023-09-23T20:41:48.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:41:48 smithi070 ceph-mon[100755]: from='mgr.14451 172.21.15.26:0/1329296977' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:41:48.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:41:48 smithi070 ceph-mon[100755]: from='mgr.14451 172.21.15.26:0/1329296977' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:41:48.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:41:48 smithi070 ceph-mon[100755]: from='mgr.14451 ' entity='mgr.y' 2023-09-23T20:41:48.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:41:48 smithi070 ceph-mon[100755]: Reconfiguring alertmanager.a (dependencies changed)... 2023-09-23T20:41:48.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:41:48 smithi070 ceph-mon[100755]: Reconfiguring daemon alertmanager.a on smithi026 2023-09-23T20:41:48.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:41:48 smithi070 ceph-mon[100755]: pgmap v29: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:41:48.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:41:48 smithi026 ceph-mon[95839]: from='mgr.14451 ' entity='mgr.y' 2023-09-23T20:41:48.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:41:48 smithi026 ceph-mon[95839]: from='mgr.14451 ' entity='mgr.y' 2023-09-23T20:41:48.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:41:48 smithi026 ceph-mon[95839]: from='mgr.14451 172.21.15.26:0/1329296977' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:41:48.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:41:48 smithi026 ceph-mon[95839]: from='mgr.14451 172.21.15.26:0/1329296977' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:41:48.633 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:41:48 smithi026 ceph-mon[95839]: from='mgr.14451 ' entity='mgr.y' 2023-09-23T20:41:48.633 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:41:48 smithi026 ceph-mon[95839]: Reconfiguring alertmanager.a (dependencies changed)... 2023-09-23T20:41:48.633 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:41:48 smithi026 ceph-mon[95839]: Reconfiguring daemon alertmanager.a on smithi026 2023-09-23T20:41:48.633 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:41:48 smithi026 ceph-mon[95839]: pgmap v29: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:41:48.633 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:41:48 smithi026 ceph-mon[109912]: from='mgr.14451 ' entity='mgr.y' 2023-09-23T20:41:48.633 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:41:48 smithi026 ceph-mon[109912]: from='mgr.14451 ' entity='mgr.y' 2023-09-23T20:41:48.633 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:41:48 smithi026 ceph-mon[109912]: from='mgr.14451 172.21.15.26:0/1329296977' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:41:48.633 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:41:48 smithi026 ceph-mon[109912]: from='mgr.14451 172.21.15.26:0/1329296977' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:41:48.633 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:41:48 smithi026 ceph-mon[109912]: from='mgr.14451 ' entity='mgr.y' 2023-09-23T20:41:48.634 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:41:48 smithi026 ceph-mon[109912]: Reconfiguring alertmanager.a (dependencies changed)... 2023-09-23T20:41:48.634 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:41:48 smithi026 ceph-mon[109912]: Reconfiguring daemon alertmanager.a on smithi026 2023-09-23T20:41:48.634 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:41:48 smithi026 ceph-mon[109912]: pgmap v29: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:41:48.701 INFO:teuthology.orchestra.run.smithi026.stdout: 2023-09-23T20:41:48.701 INFO:teuthology.orchestra.run.smithi026.stderr:dumped all 2023-09-23T20:41:49.257 INFO:teuthology.orchestra.run.smithi026.stdout:{"pg_ready":true,"pg_map":{"version":29,"stamp":"2023-09-23T20:41:47.810412+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":128,"ondisk_log_size":128,"up":27,"acting":27,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":27,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":50364,"kb_used_data":6716,"kb_used_omap":0,"kb_used_meta":43584,"kb_avail":749910852,"statfs":{"total":767960285184,"available":767908712448,"internally_reserved":0,"allocated":6877184,"data_stored":3898264,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":44630016},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002125"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":11,"reported_epoch":63,"state":"active+clean","last_fresh":"2023-09-23T20:41:18.429548+0000","last_change":"2023-09-23T20:41:16.555449+0000","last_active":"2023-09-23T20:41:18.429548+0000","last_peered":"2023-09-23T20:41:18.429548+0000","last_clean":"2023-09-23T20:41:18.429548+0000","last_became_active":"2023-09-23T20:41:16.555054+0000","last_became_peered":"2023-09-23T20:41:16.555054+0000","last_unstale":"2023-09-23T20:41:18.429548+0000","last_undegraded":"2023-09-23T20:41:18.429548+0000","last_fullsized":"2023-09-23T20:41:18.429548+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:19:29.226436+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":11,"reported_epoch":63,"state":"active+clean","last_fresh":"2023-09-23T20:41:17.555337+0000","last_change":"2023-09-23T20:41:16.554495+0000","last_active":"2023-09-23T20:41:17.555337+0000","last_peered":"2023-09-23T20:41:17.555337+0000","last_clean":"2023-09-23T20:41:17.555337+0000","last_became_active":"2023-09-23T20:41:16.554217+0000","last_became_peered":"2023-09-23T20:41:16.554217+0000","last_unstale":"2023-09-23T20:41:17.555337+0000","last_undegraded":"2023-09-23T20:41:17.555337+0000","last_fullsized":"2023-09-23T20:41:17.555337+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:35:48.172198+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":11,"reported_epoch":63,"state":"active+clean","last_fresh":"2023-09-23T20:41:18.429615+0000","last_change":"2023-09-23T20:41:16.684629+0000","last_active":"2023-09-23T20:41:18.429615+0000","last_peered":"2023-09-23T20:41:18.429615+0000","last_clean":"2023-09-23T20:41:18.429615+0000","last_became_active":"2023-09-23T20:41:16.684297+0000","last_became_peered":"2023-09-23T20:41:16.684297+0000","last_unstale":"2023-09-23T20:41:18.429615+0000","last_undegraded":"2023-09-23T20:41:18.429615+0000","last_fullsized":"2023-09-23T20:41:18.429615+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:28:01.891279+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":11,"reported_epoch":63,"state":"active+clean","last_fresh":"2023-09-23T20:41:18.429545+0000","last_change":"2023-09-23T20:41:16.684597+0000","last_active":"2023-09-23T20:41:18.429545+0000","last_peered":"2023-09-23T20:41:18.429545+0000","last_clean":"2023-09-23T20:41:18.429545+0000","last_became_active":"2023-09-23T20:41:16.684172+0000","last_became_peered":"2023-09-23T20:41:16.684172+0000","last_unstale":"2023-09-23T20:41:18.429545+0000","last_undegraded":"2023-09-23T20:41:18.429545+0000","last_fullsized":"2023-09-23T20:41:18.429545+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:35:12.680090+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.0","version":"60'128","reported_seq":271,"reported_epoch":63,"state":"active+clean","last_fresh":"2023-09-23T20:41:18.429553+0000","last_change":"2023-09-23T20:40:42.550542+0000","last_active":"2023-09-23T20:41:18.429553+0000","last_peered":"2023-09-23T20:41:18.429553+0000","last_clean":"2023-09-23T20:41:18.429553+0000","last_became_active":"2023-09-23T20:40:42.550069+0000","last_became_peered":"2023-09-23T20:40:42.550069+0000","last_unstale":"2023-09-23T20:41:18.429553+0000","last_undegraded":"2023-09-23T20:41:18.429553+0000","last_fullsized":"2023-09-23T20:41:18.429553+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:38:57.324224+0000","objects_scrubbed":0,"log_size":128,"ondisk_log_size":128,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:53:50.698837+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":11,"reported_epoch":63,"state":"active+clean","last_fresh":"2023-09-23T20:41:18.429758+0000","last_change":"2023-09-23T20:41:16.555751+0000","last_active":"2023-09-23T20:41:18.429758+0000","last_peered":"2023-09-23T20:41:18.429758+0000","last_clean":"2023-09-23T20:41:18.429758+0000","last_became_active":"2023-09-23T20:41:16.555417+0000","last_became_peered":"2023-09-23T20:41:16.555417+0000","last_unstale":"2023-09-23T20:41:18.429758+0000","last_undegraded":"2023-09-23T20:41:18.429758+0000","last_fullsized":"2023-09-23T20:41:18.429758+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:54:54.137765+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":11,"reported_epoch":63,"state":"active+clean","last_fresh":"2023-09-23T20:41:17.556602+0000","last_change":"2023-09-23T20:41:16.686010+0000","last_active":"2023-09-23T20:41:17.556602+0000","last_peered":"2023-09-23T20:41:17.556602+0000","last_clean":"2023-09-23T20:41:17.556602+0000","last_became_active":"2023-09-23T20:41:16.685615+0000","last_became_peered":"2023-09-23T20:41:16.685615+0000","last_unstale":"2023-09-23T20:41:17.556602+0000","last_undegraded":"2023-09-23T20:41:17.556602+0000","last_fullsized":"2023-09-23T20:41:17.556602+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:46:04.405920+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":11,"reported_epoch":63,"state":"active+clean","last_fresh":"2023-09-23T20:41:18.429694+0000","last_change":"2023-09-23T20:41:16.556552+0000","last_active":"2023-09-23T20:41:18.429694+0000","last_peered":"2023-09-23T20:41:18.429694+0000","last_clean":"2023-09-23T20:41:18.429694+0000","last_became_active":"2023-09-23T20:41:16.556226+0000","last_became_peered":"2023-09-23T20:41:16.556226+0000","last_unstale":"2023-09-23T20:41:18.429694+0000","last_undegraded":"2023-09-23T20:41:18.429694+0000","last_fullsized":"2023-09-23T20:41:18.429694+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:21:42.514384+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":11,"reported_epoch":63,"state":"active+clean","last_fresh":"2023-09-23T20:41:17.555230+0000","last_change":"2023-09-23T20:41:16.684748+0000","last_active":"2023-09-23T20:41:17.555230+0000","last_peered":"2023-09-23T20:41:17.555230+0000","last_clean":"2023-09-23T20:41:17.555230+0000","last_became_active":"2023-09-23T20:41:16.684361+0000","last_became_peered":"2023-09-23T20:41:16.684361+0000","last_unstale":"2023-09-23T20:41:17.555230+0000","last_undegraded":"2023-09-23T20:41:17.555230+0000","last_fullsized":"2023-09-23T20:41:17.555230+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:39:15.420185+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":128,"ondisk_log_size":128,"up":3,"acting":3,"num_store_stats":3}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201294,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6328,"kb_used_data":1200,"kb_used_omap":0,"kb_used_meta":5120,"kb_avail":93738824,"statfs":{"total":95995035648,"available":95988555776,"internally_reserved":0,"allocated":1228800,"data_stored":856087,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5242880},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:41:40 2023","interfaces":[{"interface":"back","average":{"1min":0.54100000000000004,"5min":0.54100000000000004,"15min":0.54100000000000004},"min":{"1min":0.28999999999999998,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":0.85499999999999998,"5min":0.85499999999999998,"15min":0.85499999999999998},"last":0.47299999999999998},{"interface":"front","average":{"1min":0.54300000000000004,"5min":0.54300000000000004,"15min":0.54300000000000004},"min":{"1min":0.39000000000000001,"5min":0.39000000000000001,"15min":0.39000000000000001},"max":{"1min":0.79400000000000004,"5min":0.79400000000000004,"15min":0.79400000000000004},"last":0.64300000000000002}]},{"osd":1,"last update":"Sat Sep 23 20:41:40 2023","interfaces":[{"interface":"back","average":{"1min":0.54400000000000004,"5min":0.54400000000000004,"15min":0.54400000000000004},"min":{"1min":0.27500000000000002,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":0.81499999999999995,"5min":0.81499999999999995,"15min":0.81499999999999995},"last":0.57499999999999996},{"interface":"front","average":{"1min":0.55000000000000004,"5min":0.55000000000000004,"15min":0.55000000000000004},"min":{"1min":0.35999999999999999,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":0.78300000000000003,"5min":0.78300000000000003,"15min":0.78300000000000003},"last":0.44600000000000001}]},{"osd":2,"last update":"Sat Sep 23 20:41:40 2023","interfaces":[{"interface":"back","average":{"1min":0.52500000000000002,"5min":0.52500000000000002,"15min":0.52500000000000002},"min":{"1min":0.32700000000000001,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":0.82699999999999996,"5min":0.82699999999999996,"15min":0.82699999999999996},"last":0.60699999999999998},{"interface":"front","average":{"1min":0.57599999999999996,"5min":0.57599999999999996,"15min":0.57599999999999996},"min":{"1min":0.309,"5min":0.309,"15min":0.309},"max":{"1min":0.93100000000000005,"5min":0.93100000000000005,"15min":0.93100000000000005},"last":0.51100000000000001}]},{"osd":3,"last update":"Sat Sep 23 20:41:40 2023","interfaces":[{"interface":"back","average":{"1min":0.55200000000000005,"5min":0.55200000000000005,"15min":0.55200000000000005},"min":{"1min":0.37,"5min":0.37,"15min":0.37},"max":{"1min":0.73399999999999999,"5min":0.73399999999999999,"15min":0.73399999999999999},"last":0.56200000000000006},{"interface":"front","average":{"1min":0.58799999999999997,"5min":0.58799999999999997,"15min":0.58799999999999997},"min":{"1min":0.35199999999999998,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":0.88800000000000001,"5min":0.88800000000000001,"15min":0.88800000000000001},"last":0.67900000000000005}]},{"osd":4,"last update":"Sat Sep 23 20:41:40 2023","interfaces":[{"interface":"back","average":{"1min":0.47099999999999997,"5min":0.47099999999999997,"15min":0.47099999999999997},"min":{"1min":0.312,"5min":0.312,"15min":0.312},"max":{"1min":0.59499999999999997,"5min":0.59499999999999997,"15min":0.59499999999999997},"last":0.54000000000000004},{"interface":"front","average":{"1min":0.51300000000000001,"5min":0.51300000000000001,"15min":0.51300000000000001},"min":{"1min":0.36699999999999999,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":0.79500000000000004,"5min":0.79500000000000004,"15min":0.79500000000000004},"last":0.66500000000000004}]},{"osd":5,"last update":"Sat Sep 23 20:41:40 2023","interfaces":[{"interface":"back","average":{"1min":0.53000000000000003,"5min":0.53000000000000003,"15min":0.53000000000000003},"min":{"1min":0.33600000000000002,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":0.77800000000000002,"5min":0.77800000000000002,"15min":0.77800000000000002},"last":0.629},{"interface":"front","average":{"1min":0.55000000000000004,"5min":0.55000000000000004,"15min":0.55000000000000004},"min":{"1min":0.35999999999999999,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":0.78500000000000003,"5min":0.78500000000000003,"15min":0.78500000000000003},"last":0.69399999999999995}]},{"osd":6,"last update":"Sat Sep 23 20:41:40 2023","interfaces":[{"interface":"back","average":{"1min":0.56200000000000006,"5min":0.56200000000000006,"15min":0.56200000000000006},"min":{"1min":0.38,"5min":0.38,"15min":0.38},"max":{"1min":0.874,"5min":0.874,"15min":0.874},"last":0.59199999999999997},{"interface":"front","average":{"1min":0.58899999999999997,"5min":0.58899999999999997,"15min":0.58899999999999997},"min":{"1min":0.316,"5min":0.316,"15min":0.316},"max":{"1min":0.85099999999999998,"5min":0.85099999999999998,"15min":0.85099999999999998},"last":0.70699999999999996}]}]},{"osd":6,"up_from":47,"seq":201863462931,"num_pgs":4,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6388,"kb_used_data":1196,"kb_used_omap":0,"kb_used_meta":5184,"kb_avail":93738764,"statfs":{"total":95995035648,"available":95988494336,"internally_reserved":0,"allocated":1224704,"data_stored":855822,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5308416},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:41:20 2023","interfaces":[{"interface":"back","average":{"1min":0.51200000000000001,"5min":0.51200000000000001,"15min":0.51200000000000001},"min":{"1min":0.34100000000000003,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":0.77300000000000002,"5min":0.77300000000000002,"15min":0.77300000000000002},"last":0.46300000000000002},{"interface":"front","average":{"1min":0.53100000000000003,"5min":0.53100000000000003,"15min":0.53100000000000003},"min":{"1min":0.35199999999999998,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":0.82899999999999996,"5min":0.82899999999999996,"15min":0.82899999999999996},"last":0.47999999999999998}]},{"osd":1,"last update":"Sat Sep 23 20:41:20 2023","interfaces":[{"interface":"back","average":{"1min":0.55200000000000005,"5min":0.55200000000000005,"15min":0.55200000000000005},"min":{"1min":0.36199999999999999,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":0.81200000000000006,"5min":0.81200000000000006,"15min":0.81200000000000006},"last":0.60899999999999999},{"interface":"front","average":{"1min":0.52500000000000002,"5min":0.52500000000000002,"15min":0.52500000000000002},"min":{"1min":0.36799999999999999,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":0.75800000000000001,"5min":0.75800000000000001,"15min":0.75800000000000001},"last":0.41299999999999998}]},{"osd":2,"last update":"Sat Sep 23 20:41:20 2023","interfaces":[{"interface":"back","average":{"1min":0.50900000000000001,"5min":0.50900000000000001,"15min":0.50900000000000001},"min":{"1min":0.29799999999999999,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":0.80000000000000004,"5min":0.80000000000000004,"15min":0.80000000000000004},"last":0.51300000000000001},{"interface":"front","average":{"1min":0.54700000000000004,"5min":0.54700000000000004,"15min":0.54700000000000004},"min":{"1min":0.40000000000000002,"5min":0.40000000000000002,"15min":0.40000000000000002},"max":{"1min":0.79100000000000004,"5min":0.79100000000000004,"15min":0.79100000000000004},"last":0.434}]},{"osd":3,"last update":"Sat Sep 23 20:41:20 2023","interfaces":[{"interface":"back","average":{"1min":0.53800000000000003,"5min":0.53800000000000003,"15min":0.53800000000000003},"min":{"1min":0.38,"5min":0.38,"15min":0.38},"max":{"1min":0.751,"5min":0.751,"15min":0.751},"last":0.53400000000000003},{"interface":"front","average":{"1min":0.55800000000000005,"5min":0.55800000000000005,"15min":0.55800000000000005},"min":{"1min":0.379,"5min":0.379,"15min":0.379},"max":{"1min":0.83999999999999997,"5min":0.83999999999999997,"15min":0.83999999999999997},"last":0.54800000000000004}]},{"osd":4,"last update":"Sat Sep 23 20:41:20 2023","interfaces":[{"interface":"back","average":{"1min":0.42899999999999999,"5min":0.42899999999999999,"15min":0.42899999999999999},"min":{"1min":0.27800000000000002,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":0.56499999999999995,"5min":0.56499999999999995,"15min":0.56499999999999995},"last":0.56000000000000005},{"interface":"front","average":{"1min":0.47899999999999998,"5min":0.47899999999999998,"15min":0.47899999999999998},"min":{"1min":0.33700000000000002,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":0.64600000000000002,"5min":0.64600000000000002,"15min":0.64600000000000002},"last":0.58099999999999996}]},{"osd":5,"last update":"Sat Sep 23 20:41:20 2023","interfaces":[{"interface":"back","average":{"1min":0.46000000000000002,"5min":0.46000000000000002,"15min":0.46000000000000002},"min":{"1min":0.33500000000000002,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":0.65000000000000002,"5min":0.65000000000000002,"15min":0.65000000000000002},"last":0.64400000000000002},{"interface":"front","average":{"1min":0.49399999999999999,"5min":0.49399999999999999,"15min":0.49399999999999999},"min":{"1min":0.311,"5min":0.311,"15min":0.311},"max":{"1min":0.72199999999999998,"5min":0.72199999999999998,"15min":0.72199999999999998},"last":0.65800000000000003}]},{"osd":7,"last update":"Sat Sep 23 20:41:38 2023","interfaces":[{"interface":"back","average":{"1min":0.50600000000000001,"5min":0.50600000000000001,"15min":0.50600000000000001},"min":{"1min":0.32500000000000001,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":0.70199999999999996,"5min":0.70199999999999996,"15min":0.70199999999999996},"last":0.68899999999999995},{"interface":"front","average":{"1min":0.55900000000000005,"5min":0.55900000000000005,"15min":0.55900000000000005},"min":{"1min":0.41599999999999998,"5min":0.41599999999999998,"15min":0.41599999999999998},"max":{"1min":0.77400000000000002,"5min":0.77400000000000002,"15min":0.77400000000000002},"last":0.70199999999999996}]}]},{"osd":1,"up_from":13,"seq":55834574888,"num_pgs":4,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6520,"kb_used_data":624,"kb_used_omap":0,"kb_used_meta":5888,"kb_avail":93738632,"statfs":{"total":95995035648,"available":95988359168,"internally_reserved":0,"allocated":638976,"data_stored":266042,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":6029312},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:41:38 2023","interfaces":[{"interface":"back","average":{"1min":0.43099999999999999,"5min":0.40999999999999998,"15min":0.40400000000000003},"min":{"1min":0.27300000000000002,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":0.94499999999999995,"5min":0.96499999999999997,"15min":0.96499999999999997},"last":0.32000000000000001},{"interface":"front","average":{"1min":0.437,"5min":0.442,"15min":0.439},"min":{"1min":0.23999999999999999,"5min":0.218,"15min":0.218},"max":{"1min":1.2549999999999999,"5min":1.2549999999999999,"15min":1.2549999999999999},"last":0.34300000000000003}]},{"osd":2,"last update":"Sat Sep 23 20:41:00 2023","interfaces":[{"interface":"back","average":{"1min":0.504,"5min":0.47099999999999997,"15min":0.46500000000000002},"min":{"1min":0.30199999999999999,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":0.72199999999999998,"5min":0.76500000000000001,"15min":0.76500000000000001},"last":0.27300000000000002},{"interface":"front","average":{"1min":0.44900000000000001,"5min":0.42199999999999999,"15min":0.41799999999999998},"min":{"1min":0.29899999999999999,"5min":0.254,"15min":0.254},"max":{"1min":0.58999999999999997,"5min":0.748,"15min":0.748},"last":0.436}]},{"osd":3,"last update":"Sat Sep 23 20:41:23 2023","interfaces":[{"interface":"back","average":{"1min":0.40999999999999998,"5min":0.45600000000000002,"15min":0.46400000000000002},"min":{"1min":0.27900000000000003,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":0.61599999999999999,"5min":0.68600000000000005,"15min":0.68600000000000005},"last":0.41799999999999998},{"interface":"front","average":{"1min":0.46100000000000002,"5min":0.49399999999999999,"15min":0.5},"min":{"1min":0.28100000000000003,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":0.72999999999999998,"5min":0.72999999999999998,"15min":0.72999999999999998},"last":0.38800000000000001}]},{"osd":4,"last update":"Sat Sep 23 20:41:38 2023","interfaces":[{"interface":"back","average":{"1min":0.63300000000000001,"5min":0.58899999999999997,"15min":0.58199999999999996},"min":{"1min":0.38300000000000001,"5min":0.38300000000000001,"15min":0.38300000000000001},"max":{"1min":0.90600000000000003,"5min":0.98699999999999999,"15min":0.98699999999999999},"last":0.56299999999999994},{"interface":"front","average":{"1min":0.70399999999999996,"5min":0.63600000000000001,"15min":0.624},"min":{"1min":0.45000000000000001,"5min":0.40000000000000002,"15min":0.40000000000000002},"max":{"1min":1.5760000000000001,"5min":1.5760000000000001,"15min":1.5760000000000001},"last":0.54200000000000004}]},{"osd":5,"last update":"Sat Sep 23 20:41:00 2023","interfaces":[{"interface":"back","average":{"1min":0.64200000000000002,"5min":0.64200000000000002,"15min":0.64200000000000002},"min":{"1min":0.46600000000000003,"5min":0.46600000000000003,"15min":0.46600000000000003},"max":{"1min":0.871,"5min":0.871,"15min":0.871},"last":0.57899999999999996},{"interface":"front","average":{"1min":0.63500000000000001,"5min":0.63500000000000001,"15min":0.63500000000000001},"min":{"1min":0.43099999999999999,"5min":0.43099999999999999,"15min":0.43099999999999999},"max":{"1min":0.86899999999999999,"5min":0.86899999999999999,"15min":0.86899999999999999},"last":0.53100000000000003}]},{"osd":6,"last update":"Sat Sep 23 20:41:18 2023","interfaces":[{"interface":"back","average":{"1min":0.61399999999999999,"5min":0.61399999999999999,"15min":0.61399999999999999},"min":{"1min":0.436,"5min":0.436,"15min":0.436},"max":{"1min":0.92000000000000004,"5min":0.92000000000000004,"15min":0.92000000000000004},"last":0.46999999999999997},{"interface":"front","average":{"1min":0.65700000000000003,"5min":0.65700000000000003,"15min":0.65700000000000003},"min":{"1min":0.41399999999999998,"5min":0.41399999999999998,"15min":0.41399999999999998},"max":{"1min":0.96299999999999997,"5min":0.96299999999999997,"15min":0.96299999999999997},"last":0.45100000000000001}]},{"osd":7,"last update":"Sat Sep 23 20:41:40 2023","interfaces":[{"interface":"back","average":{"1min":0.69899999999999995,"5min":0.69899999999999995,"15min":0.69899999999999995},"min":{"1min":0.439,"5min":0.439,"15min":0.439},"max":{"1min":1.45,"5min":1.45,"15min":1.45},"last":0.47999999999999998},{"interface":"front","average":{"1min":0.65800000000000003,"5min":0.65800000000000003,"15min":0.65800000000000003},"min":{"1min":0.442,"5min":0.442,"15min":0.442},"max":{"1min":0.86199999999999999,"5min":0.86199999999999999,"15min":0.86199999999999999},"last":0.51500000000000001}]}]},{"osd":0,"up_from":8,"seq":34359738413,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":7160,"kb_used_data":1200,"kb_used_omap":0,"kb_used_meta":5952,"kb_avail":93737992,"statfs":{"total":95995035648,"available":95987703808,"internally_reserved":0,"allocated":1228800,"data_stored":856145,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":6094848},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 20:41:40 2023","interfaces":[{"interface":"back","average":{"1min":0.34499999999999997,"5min":0.35899999999999999,"15min":0.34399999999999997},"min":{"1min":0.26300000000000001,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":0.51000000000000001,"5min":0.65900000000000003,"15min":0.65900000000000003},"last":0.441},{"interface":"front","average":{"1min":0.39500000000000002,"5min":0.36399999999999999,"15min":0.34699999999999998},"min":{"1min":0.26300000000000001,"5min":0.22,"15min":0.22},"max":{"1min":0.55700000000000005,"5min":0.623,"15min":0.623},"last":0.49099999999999999}]},{"osd":2,"last update":"Sat Sep 23 20:41:00 2023","interfaces":[{"interface":"back","average":{"1min":0.44900000000000001,"5min":0.42499999999999999,"15min":0.42099999999999999},"min":{"1min":0.313,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":0.63900000000000001,"5min":0.66400000000000003,"15min":0.66400000000000003},"last":0.40200000000000002},{"interface":"front","average":{"1min":0.45900000000000002,"5min":0.42399999999999999,"15min":0.41799999999999998},"min":{"1min":0.316,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":0.62,"5min":0.81000000000000005,"15min":0.81000000000000005},"last":0.50700000000000001}]},{"osd":3,"last update":"Sat Sep 23 20:41:23 2023","interfaces":[{"interface":"back","average":{"1min":0.45600000000000002,"5min":0.48399999999999999,"15min":0.48899999999999999},"min":{"1min":0.29099999999999998,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":0.61899999999999999,"5min":0.61899999999999999,"15min":0.61899999999999999},"last":0.47599999999999998},{"interface":"front","average":{"1min":0.42499999999999999,"5min":0.46100000000000002,"15min":0.46700000000000003},"min":{"1min":0.28399999999999997,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":0.59099999999999997,"5min":0.59199999999999997,"15min":0.59199999999999997},"last":0.54200000000000004}]},{"osd":4,"last update":"Sat Sep 23 20:41:40 2023","interfaces":[{"interface":"back","average":{"1min":0.60899999999999999,"5min":0.61099999999999999,"15min":0.61099999999999999},"min":{"1min":0.43099999999999999,"5min":0.42299999999999999,"15min":0.42299999999999999},"max":{"1min":1.734,"5min":1.734,"15min":1.734},"last":0.91600000000000004},{"interface":"front","average":{"1min":0.61899999999999999,"5min":0.64300000000000002,"15min":0.64700000000000002},"min":{"1min":0.42099999999999999,"5min":0.42099999999999999,"15min":0.42099999999999999},"max":{"1min":1.7170000000000001,"5min":1.7170000000000001,"15min":1.7170000000000001},"last":0.89300000000000002}]},{"osd":5,"last update":"Sat Sep 23 20:41:00 2023","interfaces":[{"interface":"back","average":{"1min":0.60299999999999998,"5min":0.60299999999999998,"15min":0.60299999999999998},"min":{"1min":0.46899999999999997,"5min":0.46899999999999997,"15min":0.46899999999999997},"max":{"1min":1.0740000000000001,"5min":1.0740000000000001,"15min":1.0740000000000001},"last":0.80800000000000005},{"interface":"front","average":{"1min":0.64300000000000002,"5min":0.64300000000000002,"15min":0.64300000000000002},"min":{"1min":0.44,"5min":0.44,"15min":0.44},"max":{"1min":1.0660000000000001,"5min":1.0660000000000001,"15min":1.0660000000000001},"last":0.88100000000000001}]},{"osd":6,"last update":"Sat Sep 23 20:41:20 2023","interfaces":[{"interface":"back","average":{"1min":0.58299999999999996,"5min":0.58299999999999996,"15min":0.58299999999999996},"min":{"1min":0.432,"5min":0.432,"15min":0.432},"max":{"1min":0.76600000000000001,"5min":0.76600000000000001,"15min":0.76600000000000001},"last":0.95899999999999996},{"interface":"front","average":{"1min":0.59499999999999997,"5min":0.59499999999999997,"15min":0.59499999999999997},"min":{"1min":0.47799999999999998,"5min":0.47799999999999998,"15min":0.47799999999999998},"max":{"1min":0.74399999999999999,"5min":0.74399999999999999,"15min":0.74399999999999999},"last":0.93600000000000005}]},{"osd":7,"last update":"Sat Sep 23 20:41:40 2023","interfaces":[{"interface":"back","average":{"1min":0.626,"5min":0.626,"15min":0.626},"min":{"1min":0.38600000000000001,"5min":0.38600000000000001,"15min":0.38600000000000001},"max":{"1min":1.9890000000000001,"5min":1.9890000000000001,"15min":1.9890000000000001},"last":0.86799999999999999},{"interface":"front","average":{"1min":0.67900000000000005,"5min":0.67900000000000005,"15min":0.67900000000000005},"min":{"1min":0.41499999999999998,"5min":0.41499999999999998,"15min":0.41499999999999998},"max":{"1min":2.0169999999999999,"5min":2.0169999999999999,"15min":2.0169999999999999},"last":0.83599999999999997}]}]},{"osd":2,"up_from":18,"seq":77309411363,"num_pgs":3,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6456,"kb_used_data":624,"kb_used_omap":0,"kb_used_meta":5824,"kb_avail":93738696,"statfs":{"total":95995035648,"available":95988424704,"internally_reserved":0,"allocated":638976,"data_stored":266042,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5963776},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:40:59 2023","interfaces":[{"interface":"back","average":{"1min":0.41299999999999998,"5min":0.39700000000000002,"15min":0.39500000000000002},"min":{"1min":0.252,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":0.68999999999999995,"5min":0.68999999999999995,"15min":0.68999999999999995},"last":0.27900000000000003},{"interface":"front","average":{"1min":0.47099999999999997,"5min":0.42799999999999999,"15min":0.42099999999999999},"min":{"1min":0.29799999999999999,"5min":0.253,"15min":0.253},"max":{"1min":0.64900000000000002,"5min":0.64900000000000002,"15min":0.64900000000000002},"last":0.33100000000000002}]},{"osd":1,"last update":"Sat Sep 23 20:40:59 2023","interfaces":[{"interface":"back","average":{"1min":0.45900000000000002,"5min":0.437,"15min":0.433},"min":{"1min":0.23799999999999999,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":0.67900000000000005,"5min":0.67900000000000005,"15min":0.67900000000000005},"last":0.40500000000000003},{"interface":"front","average":{"1min":0.48299999999999998,"5min":0.46100000000000002,"15min":0.45700000000000002},"min":{"1min":0.32000000000000001,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":0.628,"5min":0.628,"15min":0.628},"last":0.314}]},{"osd":3,"last update":"Sat Sep 23 20:41:21 2023","interfaces":[{"interface":"back","average":{"1min":0.47299999999999998,"5min":0.501,"15min":0.50600000000000001},"min":{"1min":0.312,"5min":0.312,"15min":0.312},"max":{"1min":1.3480000000000001,"5min":1.3480000000000001,"15min":1.3480000000000001},"last":0.29399999999999998},{"interface":"front","average":{"1min":0.44800000000000001,"5min":0.52500000000000002,"15min":0.53800000000000003},"min":{"1min":0.31,"5min":0.31,"15min":0.31},"max":{"1min":0.72499999999999998,"5min":0.72499999999999998,"15min":0.72499999999999998},"last":0.36599999999999999}]},{"osd":4,"last update":"Sat Sep 23 20:41:40 2023","interfaces":[{"interface":"back","average":{"1min":0.65100000000000002,"5min":0.66300000000000003,"15min":0.66500000000000004},"min":{"1min":0.39500000000000002,"5min":0.39500000000000002,"15min":0.39500000000000002},"max":{"1min":2.327,"5min":2.327,"15min":2.327},"last":0.441},{"interface":"front","average":{"1min":0.67400000000000004,"5min":0.68400000000000005,"15min":0.68600000000000005},"min":{"1min":0.38800000000000001,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":2.9809999999999999,"5min":2.9809999999999999,"15min":2.9809999999999999},"last":0.45400000000000001}]},{"osd":5,"last update":"Sat Sep 23 20:40:59 2023","interfaces":[{"interface":"back","average":{"1min":0.61199999999999999,"5min":0.61199999999999999,"15min":0.61199999999999999},"min":{"1min":0.45200000000000001,"5min":0.45200000000000001,"15min":0.45200000000000001},"max":{"1min":0.79000000000000004,"5min":0.79000000000000004,"15min":0.79000000000000004},"last":0.432},{"interface":"front","average":{"1min":0.61199999999999999,"5min":0.61199999999999999,"15min":0.61199999999999999},"min":{"1min":0.40699999999999997,"5min":0.40699999999999997,"15min":0.40699999999999997},"max":{"1min":0.80100000000000005,"5min":0.80100000000000005,"15min":0.80100000000000005},"last":0.38800000000000001}]},{"osd":6,"last update":"Sat Sep 23 20:41:21 2023","interfaces":[{"interface":"back","average":{"1min":0.60099999999999998,"5min":0.60099999999999998,"15min":0.60099999999999998},"min":{"1min":0.42399999999999999,"5min":0.42399999999999999,"15min":0.42399999999999999},"max":{"1min":0.87,"5min":0.87,"15min":0.87},"last":0.41599999999999998},{"interface":"front","average":{"1min":0.65600000000000003,"5min":0.65600000000000003,"15min":0.65600000000000003},"min":{"1min":0.443,"5min":0.443,"15min":0.443},"max":{"1min":0.93500000000000005,"5min":0.93500000000000005,"15min":0.93500000000000005},"last":0.49399999999999999}]},{"osd":7,"last update":"Sat Sep 23 20:41:40 2023","interfaces":[{"interface":"back","average":{"1min":0.69499999999999995,"5min":0.69499999999999995,"15min":0.69499999999999995},"min":{"1min":0.41999999999999998,"5min":0.41999999999999998,"15min":0.41999999999999998},"max":{"1min":2.3700000000000001,"5min":2.3700000000000001,"15min":2.3700000000000001},"last":0.52100000000000002},{"interface":"front","average":{"1min":0.70399999999999996,"5min":0.70399999999999996,"15min":0.70399999999999996},"min":{"1min":0.46400000000000002,"5min":0.46400000000000002,"15min":0.46400000000000002},"max":{"1min":2.27,"5min":2.27,"15min":2.27},"last":0.46400000000000002}]}]},{"osd":3,"up_from":26,"seq":111669149727,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":5816,"kb_used_data":624,"kb_used_omap":0,"kb_used_meta":5184,"kb_avail":93739336,"statfs":{"total":95995035648,"available":95989080064,"internally_reserved":0,"allocated":638976,"data_stored":266042,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5308416},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:41:20 2023","interfaces":[{"interface":"back","average":{"1min":0.39100000000000001,"5min":0.41899999999999998,"15min":0.42399999999999999},"min":{"1min":0.24199999999999999,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":0.79100000000000004,"5min":0.79100000000000004,"15min":0.79100000000000004},"last":0.625},{"interface":"front","average":{"1min":0.40200000000000002,"5min":0.49099999999999999,"15min":0.50600000000000001},"min":{"1min":0.22800000000000001,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":0.70799999999999996,"5min":0.70799999999999996,"15min":0.70799999999999996},"last":0.748}]},{"osd":1,"last update":"Sat Sep 23 20:41:20 2023","interfaces":[{"interface":"back","average":{"1min":0.43099999999999999,"5min":0.48299999999999998,"15min":0.49199999999999999},"min":{"1min":0.27400000000000002,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":0.621,"5min":0.65500000000000003,"15min":0.65500000000000003},"last":0.68700000000000006},{"interface":"front","average":{"1min":0.45900000000000002,"5min":0.51200000000000001,"15min":0.52100000000000002},"min":{"1min":0.316,"5min":0.316,"15min":0.316},"max":{"1min":0.72899999999999998,"5min":0.72899999999999998,"15min":0.72899999999999998},"last":0.72099999999999997}]},{"osd":2,"last update":"Sat Sep 23 20:41:20 2023","interfaces":[{"interface":"back","average":{"1min":0.47699999999999998,"5min":0.53000000000000003,"15min":0.53900000000000003},"min":{"1min":0.33700000000000002,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":0.79800000000000004,"5min":0.79800000000000004,"15min":0.79800000000000004},"last":0.79700000000000004},{"interface":"front","average":{"1min":0.45200000000000001,"5min":0.51100000000000001,"15min":0.52100000000000002},"min":{"1min":0.28000000000000003,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":0.65800000000000003,"5min":0.65800000000000003,"15min":0.65800000000000003},"last":0.66700000000000004}]},{"osd":4,"last update":"Sat Sep 23 20:41:41 2023","interfaces":[{"interface":"back","average":{"1min":0.59499999999999997,"5min":0.65200000000000002,"15min":0.66200000000000003},"min":{"1min":0.36699999999999999,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":0.88,"5min":0.88100000000000001,"15min":0.88100000000000001},"last":0.81299999999999994},{"interface":"front","average":{"1min":0.60699999999999998,"5min":0.64200000000000002,"15min":0.64800000000000002},"min":{"1min":0.40699999999999997,"5min":0.40699999999999997,"15min":0.40699999999999997},"max":{"1min":0.90900000000000003,"5min":0.90900000000000003,"15min":0.90900000000000003},"last":0.76800000000000002}]},{"osd":5,"last update":"Sat Sep 23 20:41:00 2023","interfaces":[{"interface":"back","average":{"1min":0.65500000000000003,"5min":0.65500000000000003,"15min":0.65500000000000003},"min":{"1min":0.49099999999999999,"5min":0.49099999999999999,"15min":0.49099999999999999},"max":{"1min":0.97499999999999998,"5min":0.97499999999999998,"15min":0.97499999999999998},"last":0.73899999999999999},{"interface":"front","average":{"1min":0.68100000000000005,"5min":0.68100000000000005,"15min":0.68100000000000005},"min":{"1min":0.501,"5min":0.501,"15min":0.501},"max":{"1min":0.91500000000000004,"5min":0.91500000000000004,"15min":0.91500000000000004},"last":0.83499999999999996}]},{"osd":6,"last update":"Sat Sep 23 20:41:20 2023","interfaces":[{"interface":"back","average":{"1min":0.66600000000000004,"5min":0.66600000000000004,"15min":0.66600000000000004},"min":{"1min":0.39600000000000002,"5min":0.39600000000000002,"15min":0.39600000000000002},"max":{"1min":0.97299999999999998,"5min":0.97299999999999998,"15min":0.97299999999999998},"last":0.82499999999999996},{"interface":"front","average":{"1min":0.63600000000000001,"5min":0.63600000000000001,"15min":0.63600000000000001},"min":{"1min":0.38400000000000001,"5min":0.38400000000000001,"15min":0.38400000000000001},"max":{"1min":0.92700000000000005,"5min":0.92700000000000005,"15min":0.92700000000000005},"last":0.78100000000000003}]},{"osd":7,"last update":"Sat Sep 23 20:41:41 2023","interfaces":[{"interface":"back","average":{"1min":0.66700000000000004,"5min":0.66700000000000004,"15min":0.66700000000000004},"min":{"1min":0.47599999999999998,"5min":0.47599999999999998,"15min":0.47599999999999998},"max":{"1min":0.93700000000000006,"5min":0.93700000000000006,"15min":0.93700000000000006},"last":0.79000000000000004},{"interface":"front","average":{"1min":0.65300000000000002,"5min":0.65300000000000002,"15min":0.65300000000000002},"min":{"1min":0.40300000000000002,"5min":0.40300000000000002,"15min":0.40300000000000002},"max":{"1min":1.0049999999999999,"5min":1.0049999999999999,"15min":1.0049999999999999},"last":0.84499999999999997}]}]},{"osd":4,"up_from":31,"seq":133143986203,"num_pgs":2,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":5880,"kb_used_data":624,"kb_used_omap":0,"kb_used_meta":5248,"kb_avail":93739272,"statfs":{"total":95995035648,"available":95989014528,"internally_reserved":0,"allocated":638976,"data_stored":266042,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5373952},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:41:39 2023","interfaces":[{"interface":"back","average":{"1min":0.46500000000000002,"5min":0.52100000000000002,"15min":0.53000000000000003},"min":{"1min":0.315,"5min":0.31,"15min":0.31},"max":{"1min":0.72599999999999998,"5min":0.996,"15min":0.996},"last":0.67900000000000005},{"interface":"front","average":{"1min":0.57099999999999995,"5min":0.54600000000000004,"15min":0.54200000000000004},"min":{"1min":0.42399999999999999,"5min":0.41199999999999998,"15min":0.41199999999999998},"max":{"1min":0.77100000000000002,"5min":0.871,"15min":0.871},"last":0.52600000000000002}]},{"osd":1,"last update":"Sat Sep 23 20:41:39 2023","interfaces":[{"interface":"back","average":{"1min":0.503,"5min":0.53900000000000003,"15min":0.54500000000000004},"min":{"1min":0.33200000000000002,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":0.72199999999999998,"5min":0.88900000000000001,"15min":0.88900000000000001},"last":0.55800000000000005},{"interface":"front","average":{"1min":0.53500000000000003,"5min":0.52000000000000002,"15min":0.51800000000000002},"min":{"1min":0.32700000000000001,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":0.79600000000000004,"5min":1.0289999999999999,"15min":1.0289999999999999},"last":0.69299999999999995}]},{"osd":2,"last update":"Sat Sep 23 20:41:39 2023","interfaces":[{"interface":"back","average":{"1min":0.53600000000000003,"5min":0.55400000000000005,"15min":0.55700000000000005},"min":{"1min":0.33700000000000002,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":0.79700000000000004,"5min":1.0049999999999999,"15min":1.0049999999999999},"last":0.58499999999999996},{"interface":"front","average":{"1min":0.53100000000000003,"5min":0.53400000000000003,"15min":0.53400000000000003},"min":{"1min":0.40200000000000002,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":0.81100000000000005,"5min":0.85399999999999998,"15min":0.85399999999999998},"last":0.57499999999999996}]},{"osd":3,"last update":"Sat Sep 23 20:41:39 2023","interfaces":[{"interface":"back","average":{"1min":0.53900000000000003,"5min":0.56399999999999995,"15min":0.56799999999999995},"min":{"1min":0.35799999999999998,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":0.82999999999999996,"5min":0.90100000000000002,"15min":0.90100000000000002},"last":0.61199999999999999},{"interface":"front","average":{"1min":0.55000000000000004,"5min":0.55400000000000005,"15min":0.55500000000000005},"min":{"1min":0.38700000000000001,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":0.76900000000000002,"5min":0.91500000000000004,"15min":0.91500000000000004},"last":0.64100000000000001}]},{"osd":5,"last update":"Sat Sep 23 20:41:00 2023","interfaces":[{"interface":"back","average":{"1min":0.38300000000000001,"5min":0.38300000000000001,"15min":0.38300000000000001},"min":{"1min":0.26000000000000001,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":0.68100000000000005,"5min":0.68100000000000005,"15min":0.68100000000000005},"last":0.628},{"interface":"front","average":{"1min":0.39000000000000001,"5min":0.39000000000000001,"15min":0.39000000000000001},"min":{"1min":0.28799999999999998,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":0.46899999999999997,"5min":0.46899999999999997,"15min":0.46899999999999997},"last":0.47999999999999998}]},{"osd":6,"last update":"Sat Sep 23 20:41:18 2023","interfaces":[{"interface":"back","average":{"1min":0.52800000000000002,"5min":0.52800000000000002,"15min":0.52800000000000002},"min":{"1min":0.34899999999999998,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":1.1319999999999999,"5min":1.1319999999999999,"15min":1.1319999999999999},"last":0.496},{"interface":"front","average":{"1min":0.57499999999999996,"5min":0.57499999999999996,"15min":0.57499999999999996},"min":{"1min":0.33900000000000002,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":1.095,"5min":1.095,"15min":1.095},"last":0.65900000000000003}]},{"osd":7,"last update":"Sat Sep 23 20:41:39 2023","interfaces":[{"interface":"back","average":{"1min":0.55500000000000005,"5min":0.55500000000000005,"15min":0.55500000000000005},"min":{"1min":0.32700000000000001,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":0.77700000000000002,"5min":0.77700000000000002,"15min":0.77700000000000002},"last":0.72199999999999998},{"interface":"front","average":{"1min":0.57099999999999995,"5min":0.57099999999999995,"15min":0.57099999999999995},"min":{"1min":0.32700000000000001,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":0.86299999999999999,"5min":0.86299999999999999,"15min":0.86299999999999999},"last":0.54000000000000004}]}]},{"osd":5,"up_from":39,"seq":167503724568,"num_pgs":2,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":5816,"kb_used_data":624,"kb_used_omap":0,"kb_used_meta":5184,"kb_avail":93739336,"statfs":{"total":95995035648,"available":95989080064,"internally_reserved":0,"allocated":638976,"data_stored":266042,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5308416},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:40:59 2023","interfaces":[{"interface":"back","average":{"1min":0.54000000000000004,"5min":0.54000000000000004,"15min":0.54000000000000004},"min":{"1min":0.35399999999999998,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":0.82299999999999995,"5min":0.82299999999999995,"15min":0.82299999999999995},"last":0.38300000000000001},{"interface":"front","average":{"1min":0.56399999999999995,"5min":0.56399999999999995,"15min":0.56399999999999995},"min":{"1min":0.36399999999999999,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":0.79600000000000004,"5min":0.79600000000000004,"15min":0.79600000000000004},"last":0.44600000000000001}]},{"osd":1,"last update":"Sat Sep 23 20:40:59 2023","interfaces":[{"interface":"back","average":{"1min":0.56699999999999995,"5min":0.56699999999999995,"15min":0.56699999999999995},"min":{"1min":0.372,"5min":0.372,"15min":0.372},"max":{"1min":0.81499999999999995,"5min":0.81499999999999995,"15min":0.81499999999999995},"last":0.45500000000000002},{"interface":"front","average":{"1min":0.53100000000000003,"5min":0.53100000000000003,"15min":0.53100000000000003},"min":{"1min":0.35799999999999998,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":0.76900000000000002,"5min":0.76900000000000002,"15min":0.76900000000000002},"last":0.50900000000000001}]},{"osd":2,"last update":"Sat Sep 23 20:40:59 2023","interfaces":[{"interface":"back","average":{"1min":0.59399999999999997,"5min":0.59399999999999997,"15min":0.59399999999999997},"min":{"1min":0.41499999999999998,"5min":0.41499999999999998,"15min":0.41499999999999998},"max":{"1min":0.79100000000000004,"5min":0.79100000000000004,"15min":0.79100000000000004},"last":0.41699999999999998},{"interface":"front","average":{"1min":0.55900000000000005,"5min":0.55900000000000005,"15min":0.55900000000000005},"min":{"1min":0.38700000000000001,"5min":0.38700000000000001,"15min":0.38700000000000001},"max":{"1min":0.77700000000000002,"5min":0.77700000000000002,"15min":0.77700000000000002},"last":0.42599999999999999}]},{"osd":3,"last update":"Sat Sep 23 20:40:59 2023","interfaces":[{"interface":"back","average":{"1min":0.57799999999999996,"5min":0.57799999999999996,"15min":0.57799999999999996},"min":{"1min":0.373,"5min":0.373,"15min":0.373},"max":{"1min":0.86499999999999999,"5min":0.86499999999999999,"15min":0.86499999999999999},"last":0.436},{"interface":"front","average":{"1min":0.58399999999999996,"5min":0.58399999999999996,"15min":0.58399999999999996},"min":{"1min":0.42499999999999999,"5min":0.42499999999999999,"15min":0.42499999999999999},"max":{"1min":0.85099999999999998,"5min":0.85099999999999998,"15min":0.85099999999999998},"last":0.39400000000000002}]},{"osd":4,"last update":"Sat Sep 23 20:40:59 2023","interfaces":[{"interface":"back","average":{"1min":0.41199999999999998,"5min":0.41199999999999998,"15min":0.41199999999999998},"min":{"1min":0.33300000000000002,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":0.64700000000000002,"5min":0.64700000000000002,"15min":0.64700000000000002},"last":0.34899999999999998},{"interface":"front","average":{"1min":0.46700000000000003,"5min":0.46700000000000003,"15min":0.46700000000000003},"min":{"1min":0.30299999999999999,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":0.82799999999999996,"5min":0.82799999999999996,"15min":0.82799999999999996},"last":0.46300000000000002}]},{"osd":6,"last update":"Sat Sep 23 20:41:22 2023","interfaces":[{"interface":"back","average":{"1min":0.47199999999999998,"5min":0.47199999999999998,"15min":0.47199999999999998},"min":{"1min":0.28599999999999998,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":0.72499999999999998,"5min":0.72499999999999998,"15min":0.72499999999999998},"last":0.313},{"interface":"front","average":{"1min":0.52800000000000002,"5min":0.52800000000000002,"15min":0.52800000000000002},"min":{"1min":0.33300000000000002,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":0.82299999999999995,"5min":0.82299999999999995,"15min":0.82299999999999995},"last":0.40100000000000002}]},{"osd":7,"last update":"Sat Sep 23 20:41:41 2023","interfaces":[{"interface":"back","average":{"1min":0.55200000000000005,"5min":0.55200000000000005,"15min":0.55200000000000005},"min":{"1min":0.32200000000000001,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":0.79800000000000004,"5min":0.79800000000000004,"15min":0.79800000000000004},"last":0.376},{"interface":"front","average":{"1min":0.56399999999999995,"5min":0.56399999999999995,"15min":0.56399999999999995},"min":{"1min":0.375,"5min":0.375,"15min":0.375},"max":{"1min":0.84999999999999998,"5min":0.84999999999999998,"15min":0.84999999999999998},"last":0.47399999999999998}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T20:41:49.260 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T20:41:49.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:41:49 smithi070 ceph-mon[100755]: from='client.14628 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:41:49.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:41:49 smithi070 ceph-mon[100755]: from='mgr.14451 172.21.15.26:0/1329296977' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:41:49.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:41:49 smithi026 ceph-mon[95839]: from='client.14628 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:41:49.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:41:49 smithi026 ceph-mon[95839]: from='mgr.14451 172.21.15.26:0/1329296977' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:41:49.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:41:49 smithi026 ceph-mon[109912]: from='client.14628 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:41:49.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:41:49 smithi026 ceph-mon[109912]: from='mgr.14451 172.21.15.26:0/1329296977' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:41:50.304 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:41:50.536 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Sep 23 20:41:50 smithi026 systemd[1]: Stopping Ceph alertmanager.a for b3f8e4ec-5a50-11ee-8db4-212e2dc638e7... 2023-09-23T20:41:50.821 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:41:50 smithi026 ceph-mon[109912]: pgmap v30: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:41:50.821 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:41:50 smithi026 ceph-mon[95839]: pgmap v30: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:41:50.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:41:50 smithi070 ceph-mon[100755]: pgmap v30: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:41:51.132 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Sep 23 20:41:51 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-alertmanager-a[143867]: ts=2023-09-23T20:41:51.066Z caller=main.go:583 level=info msg="Received SIGTERM, exiting gracefully..." 2023-09-23T20:41:51.540 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Sep 23 20:41:51 smithi026 bash[149679]: ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-alertmanager-a 2023-09-23T20:41:51.869 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 20:41:51 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-x[102861]: ::ffff:172.21.15.70 - - [23/Sep/2023:20:41:51] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2023-09-23T20:41:51.882 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Sep 23 20:41:51 smithi026 systemd[1]: ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7@alertmanager.a.service: Succeeded. 2023-09-23T20:41:51.882 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Sep 23 20:41:51 smithi026 systemd[1]: Stopped Ceph alertmanager.a for b3f8e4ec-5a50-11ee-8db4-212e2dc638e7. 2023-09-23T20:41:51.882 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Sep 23 20:41:51 smithi026 systemd[1]: Starting Ceph alertmanager.a for b3f8e4ec-5a50-11ee-8db4-212e2dc638e7... 2023-09-23T20:41:51.942 INFO:teuthology.orchestra.run.smithi026.stdout: 2023-09-23T20:41:51.943 INFO:teuthology.orchestra.run.smithi026.stderr:dumped all 2023-09-23T20:41:52.289 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Sep 23 20:41:52 smithi026 podman[149886]: 2023-09-23T20:41:52.544 INFO:teuthology.orchestra.run.smithi026.stdout:{"pg_ready":true,"pg_map":{"version":31,"stamp":"2023-09-23T20:41:51.811275+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":128,"ondisk_log_size":128,"up":27,"acting":27,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":27,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":50364,"kb_used_data":6716,"kb_used_omap":0,"kb_used_meta":43584,"kb_avail":749910852,"statfs":{"total":767960285184,"available":767908712448,"internally_reserved":0,"allocated":6877184,"data_stored":3898264,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":44630016},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002274"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":11,"reported_epoch":63,"state":"active+clean","last_fresh":"2023-09-23T20:41:18.429548+0000","last_change":"2023-09-23T20:41:16.555449+0000","last_active":"2023-09-23T20:41:18.429548+0000","last_peered":"2023-09-23T20:41:18.429548+0000","last_clean":"2023-09-23T20:41:18.429548+0000","last_became_active":"2023-09-23T20:41:16.555054+0000","last_became_peered":"2023-09-23T20:41:16.555054+0000","last_unstale":"2023-09-23T20:41:18.429548+0000","last_undegraded":"2023-09-23T20:41:18.429548+0000","last_fullsized":"2023-09-23T20:41:18.429548+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:19:29.226436+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":11,"reported_epoch":63,"state":"active+clean","last_fresh":"2023-09-23T20:41:17.555337+0000","last_change":"2023-09-23T20:41:16.554495+0000","last_active":"2023-09-23T20:41:17.555337+0000","last_peered":"2023-09-23T20:41:17.555337+0000","last_clean":"2023-09-23T20:41:17.555337+0000","last_became_active":"2023-09-23T20:41:16.554217+0000","last_became_peered":"2023-09-23T20:41:16.554217+0000","last_unstale":"2023-09-23T20:41:17.555337+0000","last_undegraded":"2023-09-23T20:41:17.555337+0000","last_fullsized":"2023-09-23T20:41:17.555337+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:35:48.172198+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":11,"reported_epoch":63,"state":"active+clean","last_fresh":"2023-09-23T20:41:18.429615+0000","last_change":"2023-09-23T20:41:16.684629+0000","last_active":"2023-09-23T20:41:18.429615+0000","last_peered":"2023-09-23T20:41:18.429615+0000","last_clean":"2023-09-23T20:41:18.429615+0000","last_became_active":"2023-09-23T20:41:16.684297+0000","last_became_peered":"2023-09-23T20:41:16.684297+0000","last_unstale":"2023-09-23T20:41:18.429615+0000","last_undegraded":"2023-09-23T20:41:18.429615+0000","last_fullsized":"2023-09-23T20:41:18.429615+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:28:01.891279+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":11,"reported_epoch":63,"state":"active+clean","last_fresh":"2023-09-23T20:41:18.429545+0000","last_change":"2023-09-23T20:41:16.684597+0000","last_active":"2023-09-23T20:41:18.429545+0000","last_peered":"2023-09-23T20:41:18.429545+0000","last_clean":"2023-09-23T20:41:18.429545+0000","last_became_active":"2023-09-23T20:41:16.684172+0000","last_became_peered":"2023-09-23T20:41:16.684172+0000","last_unstale":"2023-09-23T20:41:18.429545+0000","last_undegraded":"2023-09-23T20:41:18.429545+0000","last_fullsized":"2023-09-23T20:41:18.429545+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:35:12.680090+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.0","version":"60'128","reported_seq":271,"reported_epoch":63,"state":"active+clean","last_fresh":"2023-09-23T20:41:18.429553+0000","last_change":"2023-09-23T20:40:42.550542+0000","last_active":"2023-09-23T20:41:18.429553+0000","last_peered":"2023-09-23T20:41:18.429553+0000","last_clean":"2023-09-23T20:41:18.429553+0000","last_became_active":"2023-09-23T20:40:42.550069+0000","last_became_peered":"2023-09-23T20:40:42.550069+0000","last_unstale":"2023-09-23T20:41:18.429553+0000","last_undegraded":"2023-09-23T20:41:18.429553+0000","last_fullsized":"2023-09-23T20:41:18.429553+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:38:57.324224+0000","objects_scrubbed":0,"log_size":128,"ondisk_log_size":128,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:53:50.698837+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":11,"reported_epoch":63,"state":"active+clean","last_fresh":"2023-09-23T20:41:18.429758+0000","last_change":"2023-09-23T20:41:16.555751+0000","last_active":"2023-09-23T20:41:18.429758+0000","last_peered":"2023-09-23T20:41:18.429758+0000","last_clean":"2023-09-23T20:41:18.429758+0000","last_became_active":"2023-09-23T20:41:16.555417+0000","last_became_peered":"2023-09-23T20:41:16.555417+0000","last_unstale":"2023-09-23T20:41:18.429758+0000","last_undegraded":"2023-09-23T20:41:18.429758+0000","last_fullsized":"2023-09-23T20:41:18.429758+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:54:54.137765+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":11,"reported_epoch":63,"state":"active+clean","last_fresh":"2023-09-23T20:41:17.556602+0000","last_change":"2023-09-23T20:41:16.686010+0000","last_active":"2023-09-23T20:41:17.556602+0000","last_peered":"2023-09-23T20:41:17.556602+0000","last_clean":"2023-09-23T20:41:17.556602+0000","last_became_active":"2023-09-23T20:41:16.685615+0000","last_became_peered":"2023-09-23T20:41:16.685615+0000","last_unstale":"2023-09-23T20:41:17.556602+0000","last_undegraded":"2023-09-23T20:41:17.556602+0000","last_fullsized":"2023-09-23T20:41:17.556602+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:46:04.405920+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":11,"reported_epoch":63,"state":"active+clean","last_fresh":"2023-09-23T20:41:18.429694+0000","last_change":"2023-09-23T20:41:16.556552+0000","last_active":"2023-09-23T20:41:18.429694+0000","last_peered":"2023-09-23T20:41:18.429694+0000","last_clean":"2023-09-23T20:41:18.429694+0000","last_became_active":"2023-09-23T20:41:16.556226+0000","last_became_peered":"2023-09-23T20:41:16.556226+0000","last_unstale":"2023-09-23T20:41:18.429694+0000","last_undegraded":"2023-09-23T20:41:18.429694+0000","last_fullsized":"2023-09-23T20:41:18.429694+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:21:42.514384+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":11,"reported_epoch":63,"state":"active+clean","last_fresh":"2023-09-23T20:41:17.555230+0000","last_change":"2023-09-23T20:41:16.684748+0000","last_active":"2023-09-23T20:41:17.555230+0000","last_peered":"2023-09-23T20:41:17.555230+0000","last_clean":"2023-09-23T20:41:17.555230+0000","last_became_active":"2023-09-23T20:41:16.684361+0000","last_became_peered":"2023-09-23T20:41:16.684361+0000","last_unstale":"2023-09-23T20:41:17.555230+0000","last_undegraded":"2023-09-23T20:41:17.555230+0000","last_fullsized":"2023-09-23T20:41:17.555230+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:39:15.420185+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":128,"ondisk_log_size":128,"up":3,"acting":3,"num_store_stats":3}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201295,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6328,"kb_used_data":1200,"kb_used_omap":0,"kb_used_meta":5120,"kb_avail":93738824,"statfs":{"total":95995035648,"available":95988555776,"internally_reserved":0,"allocated":1228800,"data_stored":856087,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5242880},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:41:40 2023","interfaces":[{"interface":"back","average":{"1min":0.54100000000000004,"5min":0.54100000000000004,"15min":0.54100000000000004},"min":{"1min":0.28999999999999998,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":0.85499999999999998,"5min":0.85499999999999998,"15min":0.85499999999999998},"last":0.495},{"interface":"front","average":{"1min":0.54300000000000004,"5min":0.54300000000000004,"15min":0.54300000000000004},"min":{"1min":0.39000000000000001,"5min":0.39000000000000001,"15min":0.39000000000000001},"max":{"1min":0.79400000000000004,"5min":0.79400000000000004,"15min":0.79400000000000004},"last":0.48299999999999998}]},{"osd":1,"last update":"Sat Sep 23 20:41:40 2023","interfaces":[{"interface":"back","average":{"1min":0.54400000000000004,"5min":0.54400000000000004,"15min":0.54400000000000004},"min":{"1min":0.27500000000000002,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":0.81499999999999995,"5min":0.81499999999999995,"15min":0.81499999999999995},"last":0.45700000000000002},{"interface":"front","average":{"1min":0.55000000000000004,"5min":0.55000000000000004,"15min":0.55000000000000004},"min":{"1min":0.35999999999999999,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":0.78300000000000003,"5min":0.78300000000000003,"15min":0.78300000000000003},"last":0.39200000000000002}]},{"osd":2,"last update":"Sat Sep 23 20:41:40 2023","interfaces":[{"interface":"back","average":{"1min":0.52500000000000002,"5min":0.52500000000000002,"15min":0.52500000000000002},"min":{"1min":0.32700000000000001,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":0.82699999999999996,"5min":0.82699999999999996,"15min":0.82699999999999996},"last":0.371},{"interface":"front","average":{"1min":0.57599999999999996,"5min":0.57599999999999996,"15min":0.57599999999999996},"min":{"1min":0.309,"5min":0.309,"15min":0.309},"max":{"1min":0.93100000000000005,"5min":0.93100000000000005,"15min":0.93100000000000005},"last":0.54400000000000004}]},{"osd":3,"last update":"Sat Sep 23 20:41:40 2023","interfaces":[{"interface":"back","average":{"1min":0.55200000000000005,"5min":0.55200000000000005,"15min":0.55200000000000005},"min":{"1min":0.37,"5min":0.37,"15min":0.37},"max":{"1min":0.73399999999999999,"5min":0.73399999999999999,"15min":0.73399999999999999},"last":0.57099999999999995},{"interface":"front","average":{"1min":0.58799999999999997,"5min":0.58799999999999997,"15min":0.58799999999999997},"min":{"1min":0.35199999999999998,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":0.88800000000000001,"5min":0.88800000000000001,"15min":0.88800000000000001},"last":0.58399999999999996}]},{"osd":4,"last update":"Sat Sep 23 20:41:40 2023","interfaces":[{"interface":"back","average":{"1min":0.47099999999999997,"5min":0.47099999999999997,"15min":0.47099999999999997},"min":{"1min":0.312,"5min":0.312,"15min":0.312},"max":{"1min":0.59499999999999997,"5min":0.59499999999999997,"15min":0.59499999999999997},"last":0.439},{"interface":"front","average":{"1min":0.51300000000000001,"5min":0.51300000000000001,"15min":0.51300000000000001},"min":{"1min":0.36699999999999999,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":0.79500000000000004,"5min":0.79500000000000004,"15min":0.79500000000000004},"last":0.59999999999999998}]},{"osd":5,"last update":"Sat Sep 23 20:41:40 2023","interfaces":[{"interface":"back","average":{"1min":0.53000000000000003,"5min":0.53000000000000003,"15min":0.53000000000000003},"min":{"1min":0.33600000000000002,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":0.77800000000000002,"5min":0.77800000000000002,"15min":0.77800000000000002},"last":0.61899999999999999},{"interface":"front","average":{"1min":0.55000000000000004,"5min":0.55000000000000004,"15min":0.55000000000000004},"min":{"1min":0.35999999999999999,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":0.78500000000000003,"5min":0.78500000000000003,"15min":0.78500000000000003},"last":0.55800000000000005}]},{"osd":6,"last update":"Sat Sep 23 20:41:40 2023","interfaces":[{"interface":"back","average":{"1min":0.56200000000000006,"5min":0.56200000000000006,"15min":0.56200000000000006},"min":{"1min":0.38,"5min":0.38,"15min":0.38},"max":{"1min":0.874,"5min":0.874,"15min":0.874},"last":0.53300000000000003},{"interface":"front","average":{"1min":0.58899999999999997,"5min":0.58899999999999997,"15min":0.58899999999999997},"min":{"1min":0.316,"5min":0.316,"15min":0.316},"max":{"1min":0.85099999999999998,"5min":0.85099999999999998,"15min":0.85099999999999998},"last":0.63100000000000001}]}]},{"osd":6,"up_from":47,"seq":201863462932,"num_pgs":4,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6388,"kb_used_data":1196,"kb_used_omap":0,"kb_used_meta":5184,"kb_avail":93738764,"statfs":{"total":95995035648,"available":95988494336,"internally_reserved":0,"allocated":1224704,"data_stored":855822,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5308416},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:41:20 2023","interfaces":[{"interface":"back","average":{"1min":0.51200000000000001,"5min":0.51200000000000001,"15min":0.51200000000000001},"min":{"1min":0.34100000000000003,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":0.77300000000000002,"5min":0.77300000000000002,"15min":0.77300000000000002},"last":0.58099999999999996},{"interface":"front","average":{"1min":0.53100000000000003,"5min":0.53100000000000003,"15min":0.53100000000000003},"min":{"1min":0.35199999999999998,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":0.82899999999999996,"5min":0.82899999999999996,"15min":0.82899999999999996},"last":0.38500000000000001}]},{"osd":1,"last update":"Sat Sep 23 20:41:20 2023","interfaces":[{"interface":"back","average":{"1min":0.55200000000000005,"5min":0.55200000000000005,"15min":0.55200000000000005},"min":{"1min":0.36199999999999999,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":0.81200000000000006,"5min":0.81200000000000006,"15min":0.81200000000000006},"last":0.45400000000000001},{"interface":"front","average":{"1min":0.52500000000000002,"5min":0.52500000000000002,"15min":0.52500000000000002},"min":{"1min":0.36799999999999999,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":0.75800000000000001,"5min":0.75800000000000001,"15min":0.75800000000000001},"last":0.55400000000000005}]},{"osd":2,"last update":"Sat Sep 23 20:41:20 2023","interfaces":[{"interface":"back","average":{"1min":0.50900000000000001,"5min":0.50900000000000001,"15min":0.50900000000000001},"min":{"1min":0.29799999999999999,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":0.80000000000000004,"5min":0.80000000000000004,"15min":0.80000000000000004},"last":0.49299999999999999},{"interface":"front","average":{"1min":0.54700000000000004,"5min":0.54700000000000004,"15min":0.54700000000000004},"min":{"1min":0.40000000000000002,"5min":0.40000000000000002,"15min":0.40000000000000002},"max":{"1min":0.79100000000000004,"5min":0.79100000000000004,"15min":0.79100000000000004},"last":0.47299999999999998}]},{"osd":3,"last update":"Sat Sep 23 20:41:20 2023","interfaces":[{"interface":"back","average":{"1min":0.53800000000000003,"5min":0.53800000000000003,"15min":0.53800000000000003},"min":{"1min":0.38,"5min":0.38,"15min":0.38},"max":{"1min":0.751,"5min":0.751,"15min":0.751},"last":0.53400000000000003},{"interface":"front","average":{"1min":0.55800000000000005,"5min":0.55800000000000005,"15min":0.55800000000000005},"min":{"1min":0.379,"5min":0.379,"15min":0.379},"max":{"1min":0.83999999999999997,"5min":0.83999999999999997,"15min":0.83999999999999997},"last":0.60099999999999998}]},{"osd":4,"last update":"Sat Sep 23 20:41:20 2023","interfaces":[{"interface":"back","average":{"1min":0.42899999999999999,"5min":0.42899999999999999,"15min":0.42899999999999999},"min":{"1min":0.27800000000000002,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":0.56499999999999995,"5min":0.56499999999999995,"15min":0.56499999999999995},"last":0.51700000000000002},{"interface":"front","average":{"1min":0.47899999999999998,"5min":0.47899999999999998,"15min":0.47899999999999998},"min":{"1min":0.33700000000000002,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":0.64600000000000002,"5min":0.64600000000000002,"15min":0.64600000000000002},"last":0.63400000000000001}]},{"osd":5,"last update":"Sat Sep 23 20:41:20 2023","interfaces":[{"interface":"back","average":{"1min":0.46000000000000002,"5min":0.46000000000000002,"15min":0.46000000000000002},"min":{"1min":0.33500000000000002,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":0.65000000000000002,"5min":0.65000000000000002,"15min":0.65000000000000002},"last":0.56999999999999995},{"interface":"front","average":{"1min":0.49399999999999999,"5min":0.49399999999999999,"15min":0.49399999999999999},"min":{"1min":0.311,"5min":0.311,"15min":0.311},"max":{"1min":0.72199999999999998,"5min":0.72199999999999998,"15min":0.72199999999999998},"last":0.67200000000000004}]},{"osd":7,"last update":"Sat Sep 23 20:41:38 2023","interfaces":[{"interface":"back","average":{"1min":0.50600000000000001,"5min":0.50600000000000001,"15min":0.50600000000000001},"min":{"1min":0.32500000000000001,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":0.70199999999999996,"5min":0.70199999999999996,"15min":0.70199999999999996},"last":0.66000000000000003},{"interface":"front","average":{"1min":0.55900000000000005,"5min":0.55900000000000005,"15min":0.55900000000000005},"min":{"1min":0.41599999999999998,"5min":0.41599999999999998,"15min":0.41599999999999998},"max":{"1min":0.77400000000000002,"5min":0.77400000000000002,"15min":0.77400000000000002},"last":0.69599999999999995}]}]},{"osd":1,"up_from":13,"seq":55834574889,"num_pgs":4,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6520,"kb_used_data":624,"kb_used_omap":0,"kb_used_meta":5888,"kb_avail":93738632,"statfs":{"total":95995035648,"available":95988359168,"internally_reserved":0,"allocated":638976,"data_stored":266042,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":6029312},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:41:38 2023","interfaces":[{"interface":"back","average":{"1min":0.43099999999999999,"5min":0.40999999999999998,"15min":0.40400000000000003},"min":{"1min":0.27300000000000002,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":0.94499999999999995,"5min":0.96499999999999997,"15min":0.96499999999999997},"last":0.35999999999999999},{"interface":"front","average":{"1min":0.437,"5min":0.442,"15min":0.439},"min":{"1min":0.23999999999999999,"5min":0.218,"15min":0.218},"max":{"1min":1.2549999999999999,"5min":1.2549999999999999,"15min":1.2549999999999999},"last":0.26400000000000001}]},{"osd":2,"last update":"Sat Sep 23 20:41:00 2023","interfaces":[{"interface":"back","average":{"1min":0.504,"5min":0.47099999999999997,"15min":0.46500000000000002},"min":{"1min":0.30199999999999999,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":0.72199999999999998,"5min":0.76500000000000001,"15min":0.76500000000000001},"last":0.33800000000000002},{"interface":"front","average":{"1min":0.44900000000000001,"5min":0.42199999999999999,"15min":0.41799999999999998},"min":{"1min":0.29899999999999999,"5min":0.254,"15min":0.254},"max":{"1min":0.58999999999999997,"5min":0.748,"15min":0.748},"last":0.29699999999999999}]},{"osd":3,"last update":"Sat Sep 23 20:41:23 2023","interfaces":[{"interface":"back","average":{"1min":0.40999999999999998,"5min":0.45600000000000002,"15min":0.46400000000000002},"min":{"1min":0.27900000000000003,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":0.61599999999999999,"5min":0.68600000000000005,"15min":0.68600000000000005},"last":0.38700000000000001},{"interface":"front","average":{"1min":0.46100000000000002,"5min":0.49399999999999999,"15min":0.5},"min":{"1min":0.28100000000000003,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":0.72999999999999998,"5min":0.72999999999999998,"15min":0.72999999999999998},"last":0.312}]},{"osd":4,"last update":"Sat Sep 23 20:41:38 2023","interfaces":[{"interface":"back","average":{"1min":0.63300000000000001,"5min":0.58899999999999997,"15min":0.58199999999999996},"min":{"1min":0.38300000000000001,"5min":0.38300000000000001,"15min":0.38300000000000001},"max":{"1min":0.90600000000000003,"5min":0.98699999999999999,"15min":0.98699999999999999},"last":0.66900000000000004},{"interface":"front","average":{"1min":0.70399999999999996,"5min":0.63600000000000001,"15min":0.624},"min":{"1min":0.45000000000000001,"5min":0.40000000000000002,"15min":0.40000000000000002},"max":{"1min":1.5760000000000001,"5min":1.5760000000000001,"15min":1.5760000000000001},"last":0.71099999999999997}]},{"osd":5,"last update":"Sat Sep 23 20:41:00 2023","interfaces":[{"interface":"back","average":{"1min":0.64200000000000002,"5min":0.64200000000000002,"15min":0.64200000000000002},"min":{"1min":0.46600000000000003,"5min":0.46600000000000003,"15min":0.46600000000000003},"max":{"1min":0.871,"5min":0.871,"15min":0.871},"last":0.67800000000000005},{"interface":"front","average":{"1min":0.63500000000000001,"5min":0.63500000000000001,"15min":0.63500000000000001},"min":{"1min":0.43099999999999999,"5min":0.43099999999999999,"15min":0.43099999999999999},"max":{"1min":0.86899999999999999,"5min":0.86899999999999999,"15min":0.86899999999999999},"last":0.52400000000000002}]},{"osd":6,"last update":"Sat Sep 23 20:41:18 2023","interfaces":[{"interface":"back","average":{"1min":0.61399999999999999,"5min":0.61399999999999999,"15min":0.61399999999999999},"min":{"1min":0.436,"5min":0.436,"15min":0.436},"max":{"1min":0.92000000000000004,"5min":0.92000000000000004,"15min":0.92000000000000004},"last":0.69099999999999995},{"interface":"front","average":{"1min":0.65700000000000003,"5min":0.65700000000000003,"15min":0.65700000000000003},"min":{"1min":0.41399999999999998,"5min":0.41399999999999998,"15min":0.41399999999999998},"max":{"1min":0.96299999999999997,"5min":0.96299999999999997,"15min":0.96299999999999997},"last":0.53700000000000003}]},{"osd":7,"last update":"Sat Sep 23 20:41:40 2023","interfaces":[{"interface":"back","average":{"1min":0.69899999999999995,"5min":0.69899999999999995,"15min":0.69899999999999995},"min":{"1min":0.439,"5min":0.439,"15min":0.439},"max":{"1min":1.45,"5min":1.45,"15min":1.45},"last":0.51300000000000001},{"interface":"front","average":{"1min":0.65800000000000003,"5min":0.65800000000000003,"15min":0.65800000000000003},"min":{"1min":0.442,"5min":0.442,"15min":0.442},"max":{"1min":0.86199999999999999,"5min":0.86199999999999999,"15min":0.86199999999999999},"last":0.72199999999999998}]}]},{"osd":0,"up_from":8,"seq":34359738414,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":7160,"kb_used_data":1200,"kb_used_omap":0,"kb_used_meta":5952,"kb_avail":93737992,"statfs":{"total":95995035648,"available":95987703808,"internally_reserved":0,"allocated":1228800,"data_stored":856145,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":6094848},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 20:41:40 2023","interfaces":[{"interface":"back","average":{"1min":0.34499999999999997,"5min":0.35899999999999999,"15min":0.34399999999999997},"min":{"1min":0.26300000000000001,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":0.51000000000000001,"5min":0.65900000000000003,"15min":0.65900000000000003},"last":0.39600000000000002},{"interface":"front","average":{"1min":0.39500000000000002,"5min":0.36399999999999999,"15min":0.34699999999999998},"min":{"1min":0.26300000000000001,"5min":0.22,"15min":0.22},"max":{"1min":0.55700000000000005,"5min":0.623,"15min":0.623},"last":0.41899999999999998}]},{"osd":2,"last update":"Sat Sep 23 20:41:00 2023","interfaces":[{"interface":"back","average":{"1min":0.44900000000000001,"5min":0.42499999999999999,"15min":0.42099999999999999},"min":{"1min":0.313,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":0.63900000000000001,"5min":0.66400000000000003,"15min":0.66400000000000003},"last":0.45300000000000001},{"interface":"front","average":{"1min":0.45900000000000002,"5min":0.42399999999999999,"15min":0.41799999999999998},"min":{"1min":0.316,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":0.62,"5min":0.81000000000000005,"15min":0.81000000000000005},"last":0.26400000000000001}]},{"osd":3,"last update":"Sat Sep 23 20:41:23 2023","interfaces":[{"interface":"back","average":{"1min":0.45600000000000002,"5min":0.48399999999999999,"15min":0.48899999999999999},"min":{"1min":0.29099999999999998,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":0.61899999999999999,"5min":0.61899999999999999,"15min":0.61899999999999999},"last":0.34000000000000002},{"interface":"front","average":{"1min":0.42499999999999999,"5min":0.46100000000000002,"15min":0.46700000000000003},"min":{"1min":0.28399999999999997,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":0.59099999999999997,"5min":0.59199999999999997,"15min":0.59199999999999997},"last":0.34899999999999998}]},{"osd":4,"last update":"Sat Sep 23 20:41:40 2023","interfaces":[{"interface":"back","average":{"1min":0.60899999999999999,"5min":0.61099999999999999,"15min":0.61099999999999999},"min":{"1min":0.43099999999999999,"5min":0.42299999999999999,"15min":0.42299999999999999},"max":{"1min":1.734,"5min":1.734,"15min":1.734},"last":0.59799999999999998},{"interface":"front","average":{"1min":0.61899999999999999,"5min":0.64300000000000002,"15min":0.64700000000000002},"min":{"1min":0.42099999999999999,"5min":0.42099999999999999,"15min":0.42099999999999999},"max":{"1min":1.7170000000000001,"5min":1.7170000000000001,"15min":1.7170000000000001},"last":0.73699999999999999}]},{"osd":5,"last update":"Sat Sep 23 20:41:00 2023","interfaces":[{"interface":"back","average":{"1min":0.60299999999999998,"5min":0.60299999999999998,"15min":0.60299999999999998},"min":{"1min":0.46899999999999997,"5min":0.46899999999999997,"15min":0.46899999999999997},"max":{"1min":1.0740000000000001,"5min":1.0740000000000001,"15min":1.0740000000000001},"last":0.78000000000000003},{"interface":"front","average":{"1min":0.64300000000000002,"5min":0.64300000000000002,"15min":0.64300000000000002},"min":{"1min":0.44,"5min":0.44,"15min":0.44},"max":{"1min":1.0660000000000001,"5min":1.0660000000000001,"15min":1.0660000000000001},"last":0.68700000000000006}]},{"osd":6,"last update":"Sat Sep 23 20:41:20 2023","interfaces":[{"interface":"back","average":{"1min":0.58299999999999996,"5min":0.58299999999999996,"15min":0.58299999999999996},"min":{"1min":0.432,"5min":0.432,"15min":0.432},"max":{"1min":0.76600000000000001,"5min":0.76600000000000001,"15min":0.76600000000000001},"last":0.80300000000000005},{"interface":"front","average":{"1min":0.59499999999999997,"5min":0.59499999999999997,"15min":0.59499999999999997},"min":{"1min":0.47799999999999998,"5min":0.47799999999999998,"15min":0.47799999999999998},"max":{"1min":0.74399999999999999,"5min":0.74399999999999999,"15min":0.74399999999999999},"last":0.58199999999999996}]},{"osd":7,"last update":"Sat Sep 23 20:41:40 2023","interfaces":[{"interface":"back","average":{"1min":0.626,"5min":0.626,"15min":0.626},"min":{"1min":0.38600000000000001,"5min":0.38600000000000001,"15min":0.38600000000000001},"max":{"1min":1.9890000000000001,"5min":1.9890000000000001,"15min":1.9890000000000001},"last":0.64000000000000001},{"interface":"front","average":{"1min":0.67900000000000005,"5min":0.67900000000000005,"15min":0.67900000000000005},"min":{"1min":0.41499999999999998,"5min":0.41499999999999998,"15min":0.41499999999999998},"max":{"1min":2.0169999999999999,"5min":2.0169999999999999,"15min":2.0169999999999999},"last":0.75900000000000001}]}]},{"osd":2,"up_from":18,"seq":77309411364,"num_pgs":3,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6456,"kb_used_data":624,"kb_used_omap":0,"kb_used_meta":5824,"kb_avail":93738696,"statfs":{"total":95995035648,"available":95988424704,"internally_reserved":0,"allocated":638976,"data_stored":266042,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5963776},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:40:59 2023","interfaces":[{"interface":"back","average":{"1min":0.41299999999999998,"5min":0.39700000000000002,"15min":0.39500000000000002},"min":{"1min":0.252,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":0.68999999999999995,"5min":0.68999999999999995,"15min":0.68999999999999995},"last":0.28000000000000003},{"interface":"front","average":{"1min":0.47099999999999997,"5min":0.42799999999999999,"15min":0.42099999999999999},"min":{"1min":0.29799999999999999,"5min":0.253,"15min":0.253},"max":{"1min":0.64900000000000002,"5min":0.64900000000000002,"15min":0.64900000000000002},"last":0.377}]},{"osd":1,"last update":"Sat Sep 23 20:40:59 2023","interfaces":[{"interface":"back","average":{"1min":0.45900000000000002,"5min":0.437,"15min":0.433},"min":{"1min":0.23799999999999999,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":0.67900000000000005,"5min":0.67900000000000005,"15min":0.67900000000000005},"last":0.30199999999999999},{"interface":"front","average":{"1min":0.48299999999999998,"5min":0.46100000000000002,"15min":0.45700000000000002},"min":{"1min":0.32000000000000001,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":0.628,"5min":0.628,"15min":0.628},"last":0.33300000000000002}]},{"osd":3,"last update":"Sat Sep 23 20:41:21 2023","interfaces":[{"interface":"back","average":{"1min":0.47299999999999998,"5min":0.501,"15min":0.50600000000000001},"min":{"1min":0.312,"5min":0.312,"15min":0.312},"max":{"1min":1.3480000000000001,"5min":1.3480000000000001,"15min":1.3480000000000001},"last":0.41599999999999998},{"interface":"front","average":{"1min":0.44800000000000001,"5min":0.52500000000000002,"15min":0.53800000000000003},"min":{"1min":0.31,"5min":0.31,"15min":0.31},"max":{"1min":0.72499999999999998,"5min":0.72499999999999998,"15min":0.72499999999999998},"last":0.35099999999999998}]},{"osd":4,"last update":"Sat Sep 23 20:41:40 2023","interfaces":[{"interface":"back","average":{"1min":0.65100000000000002,"5min":0.66300000000000003,"15min":0.66500000000000004},"min":{"1min":0.39500000000000002,"5min":0.39500000000000002,"15min":0.39500000000000002},"max":{"1min":2.327,"5min":2.327,"15min":2.327},"last":0.50800000000000001},{"interface":"front","average":{"1min":0.67400000000000004,"5min":0.68400000000000005,"15min":0.68600000000000005},"min":{"1min":0.38800000000000001,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":2.9809999999999999,"5min":2.9809999999999999,"15min":2.9809999999999999},"last":0.501}]},{"osd":5,"last update":"Sat Sep 23 20:40:59 2023","interfaces":[{"interface":"back","average":{"1min":0.61199999999999999,"5min":0.61199999999999999,"15min":0.61199999999999999},"min":{"1min":0.45200000000000001,"5min":0.45200000000000001,"15min":0.45200000000000001},"max":{"1min":0.79000000000000004,"5min":0.79000000000000004,"15min":0.79000000000000004},"last":0.435},{"interface":"front","average":{"1min":0.61199999999999999,"5min":0.61199999999999999,"15min":0.61199999999999999},"min":{"1min":0.40699999999999997,"5min":0.40699999999999997,"15min":0.40699999999999997},"max":{"1min":0.80100000000000005,"5min":0.80100000000000005,"15min":0.80100000000000005},"last":0.63600000000000001}]},{"osd":6,"last update":"Sat Sep 23 20:41:21 2023","interfaces":[{"interface":"back","average":{"1min":0.60099999999999998,"5min":0.60099999999999998,"15min":0.60099999999999998},"min":{"1min":0.42399999999999999,"5min":0.42399999999999999,"15min":0.42399999999999999},"max":{"1min":0.87,"5min":0.87,"15min":0.87},"last":0.60599999999999998},{"interface":"front","average":{"1min":0.65600000000000003,"5min":0.65600000000000003,"15min":0.65600000000000003},"min":{"1min":0.443,"5min":0.443,"15min":0.443},"max":{"1min":0.93500000000000005,"5min":0.93500000000000005,"15min":0.93500000000000005},"last":0.44500000000000001}]},{"osd":7,"last update":"Sat Sep 23 20:41:40 2023","interfaces":[{"interface":"back","average":{"1min":0.69499999999999995,"5min":0.69499999999999995,"15min":0.69499999999999995},"min":{"1min":0.41999999999999998,"5min":0.41999999999999998,"15min":0.41999999999999998},"max":{"1min":2.3700000000000001,"5min":2.3700000000000001,"15min":2.3700000000000001},"last":0.48599999999999999},{"interface":"front","average":{"1min":0.70399999999999996,"5min":0.70399999999999996,"15min":0.70399999999999996},"min":{"1min":0.46400000000000002,"5min":0.46400000000000002,"15min":0.46400000000000002},"max":{"1min":2.27,"5min":2.27,"15min":2.27},"last":0.66800000000000004}]}]},{"osd":3,"up_from":26,"seq":111669149728,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":5816,"kb_used_data":624,"kb_used_omap":0,"kb_used_meta":5184,"kb_avail":93739336,"statfs":{"total":95995035648,"available":95989080064,"internally_reserved":0,"allocated":638976,"data_stored":266042,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5308416},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:41:20 2023","interfaces":[{"interface":"back","average":{"1min":0.39100000000000001,"5min":0.41899999999999998,"15min":0.42399999999999999},"min":{"1min":0.24199999999999999,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":0.79100000000000004,"5min":0.79100000000000004,"15min":0.79100000000000004},"last":0.29999999999999999},{"interface":"front","average":{"1min":0.40200000000000002,"5min":0.49099999999999999,"15min":0.50600000000000001},"min":{"1min":0.22800000000000001,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":0.70799999999999996,"5min":0.70799999999999996,"15min":0.70799999999999996},"last":0.27600000000000002}]},{"osd":1,"last update":"Sat Sep 23 20:41:20 2023","interfaces":[{"interface":"back","average":{"1min":0.43099999999999999,"5min":0.48299999999999998,"15min":0.49199999999999999},"min":{"1min":0.27400000000000002,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":0.621,"5min":0.65500000000000003,"15min":0.65500000000000003},"last":0.34399999999999997},{"interface":"front","average":{"1min":0.45900000000000002,"5min":0.51200000000000001,"15min":0.52100000000000002},"min":{"1min":0.316,"5min":0.316,"15min":0.316},"max":{"1min":0.72899999999999998,"5min":0.72899999999999998,"15min":0.72899999999999998},"last":0.26200000000000001}]},{"osd":2,"last update":"Sat Sep 23 20:41:20 2023","interfaces":[{"interface":"back","average":{"1min":0.47699999999999998,"5min":0.53000000000000003,"15min":0.53900000000000003},"min":{"1min":0.33700000000000002,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":0.79800000000000004,"5min":0.79800000000000004,"15min":0.79800000000000004},"last":0.32700000000000001},{"interface":"front","average":{"1min":0.45200000000000001,"5min":0.51100000000000001,"15min":0.52100000000000002},"min":{"1min":0.28000000000000003,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":0.65800000000000003,"5min":0.65800000000000003,"15min":0.65800000000000003},"last":0.316}]},{"osd":4,"last update":"Sat Sep 23 20:41:41 2023","interfaces":[{"interface":"back","average":{"1min":0.59499999999999997,"5min":0.65200000000000002,"15min":0.66200000000000003},"min":{"1min":0.36699999999999999,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":0.88,"5min":0.88100000000000001,"15min":0.88100000000000001},"last":0.53700000000000003},{"interface":"front","average":{"1min":0.60699999999999998,"5min":0.64200000000000002,"15min":0.64800000000000002},"min":{"1min":0.40699999999999997,"5min":0.40699999999999997,"15min":0.40699999999999997},"max":{"1min":0.90900000000000003,"5min":0.90900000000000003,"15min":0.90900000000000003},"last":0.59199999999999997}]},{"osd":5,"last update":"Sat Sep 23 20:41:00 2023","interfaces":[{"interface":"back","average":{"1min":0.65500000000000003,"5min":0.65500000000000003,"15min":0.65500000000000003},"min":{"1min":0.49099999999999999,"5min":0.49099999999999999,"15min":0.49099999999999999},"max":{"1min":0.97499999999999998,"5min":0.97499999999999998,"15min":0.97499999999999998},"last":0.52400000000000002},{"interface":"front","average":{"1min":0.68100000000000005,"5min":0.68100000000000005,"15min":0.68100000000000005},"min":{"1min":0.501,"5min":0.501,"15min":0.501},"max":{"1min":0.91500000000000004,"5min":0.91500000000000004,"15min":0.91500000000000004},"last":0.57199999999999995}]},{"osd":6,"last update":"Sat Sep 23 20:41:20 2023","interfaces":[{"interface":"back","average":{"1min":0.66600000000000004,"5min":0.66600000000000004,"15min":0.66600000000000004},"min":{"1min":0.39600000000000002,"5min":0.39600000000000002,"15min":0.39600000000000002},"max":{"1min":0.97299999999999998,"5min":0.97299999999999998,"15min":0.97299999999999998},"last":0.622},{"interface":"front","average":{"1min":0.63600000000000001,"5min":0.63600000000000001,"15min":0.63600000000000001},"min":{"1min":0.38400000000000001,"5min":0.38400000000000001,"15min":0.38400000000000001},"max":{"1min":0.92700000000000005,"5min":0.92700000000000005,"15min":0.92700000000000005},"last":0.56000000000000005}]},{"osd":7,"last update":"Sat Sep 23 20:41:41 2023","interfaces":[{"interface":"back","average":{"1min":0.66700000000000004,"5min":0.66700000000000004,"15min":0.66700000000000004},"min":{"1min":0.47599999999999998,"5min":0.47599999999999998,"15min":0.47599999999999998},"max":{"1min":0.93700000000000006,"5min":0.93700000000000006,"15min":0.93700000000000006},"last":0.60499999999999998},{"interface":"front","average":{"1min":0.65300000000000002,"5min":0.65300000000000002,"15min":0.65300000000000002},"min":{"1min":0.40300000000000002,"5min":0.40300000000000002,"15min":0.40300000000000002},"max":{"1min":1.0049999999999999,"5min":1.0049999999999999,"15min":1.0049999999999999},"last":0.54700000000000004}]}]},{"osd":4,"up_from":31,"seq":133143986204,"num_pgs":2,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":5880,"kb_used_data":624,"kb_used_omap":0,"kb_used_meta":5248,"kb_avail":93739272,"statfs":{"total":95995035648,"available":95989014528,"internally_reserved":0,"allocated":638976,"data_stored":266042,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5373952},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:41:39 2023","interfaces":[{"interface":"back","average":{"1min":0.46500000000000002,"5min":0.52100000000000002,"15min":0.53000000000000003},"min":{"1min":0.315,"5min":0.31,"15min":0.31},"max":{"1min":0.72599999999999998,"5min":0.996,"15min":0.996},"last":0.39100000000000001},{"interface":"front","average":{"1min":0.57099999999999995,"5min":0.54600000000000004,"15min":0.54200000000000004},"min":{"1min":0.42399999999999999,"5min":0.41199999999999998,"15min":0.41199999999999998},"max":{"1min":0.77100000000000002,"5min":0.871,"15min":0.871},"last":0.497}]},{"osd":1,"last update":"Sat Sep 23 20:41:39 2023","interfaces":[{"interface":"back","average":{"1min":0.503,"5min":0.53900000000000003,"15min":0.54500000000000004},"min":{"1min":0.33200000000000002,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":0.72199999999999998,"5min":0.88900000000000001,"15min":0.88900000000000001},"last":0.47199999999999998},{"interface":"front","average":{"1min":0.53500000000000003,"5min":0.52000000000000002,"15min":0.51800000000000002},"min":{"1min":0.32700000000000001,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":0.79600000000000004,"5min":1.0289999999999999,"15min":1.0289999999999999},"last":0.58299999999999996}]},{"osd":2,"last update":"Sat Sep 23 20:41:39 2023","interfaces":[{"interface":"back","average":{"1min":0.53600000000000003,"5min":0.55400000000000005,"15min":0.55700000000000005},"min":{"1min":0.33700000000000002,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":0.79700000000000004,"5min":1.0049999999999999,"15min":1.0049999999999999},"last":0.42699999999999999},{"interface":"front","average":{"1min":0.53100000000000003,"5min":0.53400000000000003,"15min":0.53400000000000003},"min":{"1min":0.40200000000000002,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":0.81100000000000005,"5min":0.85399999999999998,"15min":0.85399999999999998},"last":0.52000000000000002}]},{"osd":3,"last update":"Sat Sep 23 20:41:39 2023","interfaces":[{"interface":"back","average":{"1min":0.53900000000000003,"5min":0.56399999999999995,"15min":0.56799999999999995},"min":{"1min":0.35799999999999998,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":0.82999999999999996,"5min":0.90100000000000002,"15min":0.90100000000000002},"last":0.56499999999999995},{"interface":"front","average":{"1min":0.55000000000000004,"5min":0.55400000000000005,"15min":0.55500000000000005},"min":{"1min":0.38700000000000001,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":0.76900000000000002,"5min":0.91500000000000004,"15min":0.91500000000000004},"last":0.53900000000000003}]},{"osd":5,"last update":"Sat Sep 23 20:41:00 2023","interfaces":[{"interface":"back","average":{"1min":0.38300000000000001,"5min":0.38300000000000001,"15min":0.38300000000000001},"min":{"1min":0.26000000000000001,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":0.68100000000000005,"5min":0.68100000000000005,"15min":0.68100000000000005},"last":0.35099999999999998},{"interface":"front","average":{"1min":0.39000000000000001,"5min":0.39000000000000001,"15min":0.39000000000000001},"min":{"1min":0.28799999999999998,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":0.46899999999999997,"5min":0.46899999999999997,"15min":0.46899999999999997},"last":0.443}]},{"osd":6,"last update":"Sat Sep 23 20:41:18 2023","interfaces":[{"interface":"back","average":{"1min":0.52800000000000002,"5min":0.52800000000000002,"15min":0.52800000000000002},"min":{"1min":0.34899999999999998,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":1.1319999999999999,"5min":1.1319999999999999,"15min":1.1319999999999999},"last":0.61199999999999999},{"interface":"front","average":{"1min":0.57499999999999996,"5min":0.57499999999999996,"15min":0.57499999999999996},"min":{"1min":0.33900000000000002,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":1.095,"5min":1.095,"15min":1.095},"last":0.64000000000000001}]},{"osd":7,"last update":"Sat Sep 23 20:41:39 2023","interfaces":[{"interface":"back","average":{"1min":0.55500000000000005,"5min":0.55500000000000005,"15min":0.55500000000000005},"min":{"1min":0.32700000000000001,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":0.77700000000000002,"5min":0.77700000000000002,"15min":0.77700000000000002},"last":0.67400000000000004},{"interface":"front","average":{"1min":0.57099999999999995,"5min":0.57099999999999995,"15min":0.57099999999999995},"min":{"1min":0.32700000000000001,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":0.86299999999999999,"5min":0.86299999999999999,"15min":0.86299999999999999},"last":0.627}]}]},{"osd":5,"up_from":39,"seq":167503724569,"num_pgs":2,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":5816,"kb_used_data":624,"kb_used_omap":0,"kb_used_meta":5184,"kb_avail":93739336,"statfs":{"total":95995035648,"available":95989080064,"internally_reserved":0,"allocated":638976,"data_stored":266042,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5308416},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:40:59 2023","interfaces":[{"interface":"back","average":{"1min":0.54000000000000004,"5min":0.54000000000000004,"15min":0.54000000000000004},"min":{"1min":0.35399999999999998,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":0.82299999999999995,"5min":0.82299999999999995,"15min":0.82299999999999995},"last":0.625},{"interface":"front","average":{"1min":0.56399999999999995,"5min":0.56399999999999995,"15min":0.56399999999999995},"min":{"1min":0.36399999999999999,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":0.79600000000000004,"5min":0.79600000000000004,"15min":0.79600000000000004},"last":0.41299999999999998}]},{"osd":1,"last update":"Sat Sep 23 20:40:59 2023","interfaces":[{"interface":"back","average":{"1min":0.56699999999999995,"5min":0.56699999999999995,"15min":0.56699999999999995},"min":{"1min":0.372,"5min":0.372,"15min":0.372},"max":{"1min":0.81499999999999995,"5min":0.81499999999999995,"15min":0.81499999999999995},"last":0.49099999999999999},{"interface":"front","average":{"1min":0.53100000000000003,"5min":0.53100000000000003,"15min":0.53100000000000003},"min":{"1min":0.35799999999999998,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":0.76900000000000002,"5min":0.76900000000000002,"15min":0.76900000000000002},"last":0.57999999999999996}]},{"osd":2,"last update":"Sat Sep 23 20:40:59 2023","interfaces":[{"interface":"back","average":{"1min":0.59399999999999997,"5min":0.59399999999999997,"15min":0.59399999999999997},"min":{"1min":0.41499999999999998,"5min":0.41499999999999998,"15min":0.41499999999999998},"max":{"1min":0.79100000000000004,"5min":0.79100000000000004,"15min":0.79100000000000004},"last":0.55400000000000005},{"interface":"front","average":{"1min":0.55900000000000005,"5min":0.55900000000000005,"15min":0.55900000000000005},"min":{"1min":0.38700000000000001,"5min":0.38700000000000001,"15min":0.38700000000000001},"max":{"1min":0.77700000000000002,"5min":0.77700000000000002,"15min":0.77700000000000002},"last":0.53900000000000003}]},{"osd":3,"last update":"Sat Sep 23 20:40:59 2023","interfaces":[{"interface":"back","average":{"1min":0.57799999999999996,"5min":0.57799999999999996,"15min":0.57799999999999996},"min":{"1min":0.373,"5min":0.373,"15min":0.373},"max":{"1min":0.86499999999999999,"5min":0.86499999999999999,"15min":0.86499999999999999},"last":0.69399999999999995},{"interface":"front","average":{"1min":0.58399999999999996,"5min":0.58399999999999996,"15min":0.58399999999999996},"min":{"1min":0.42499999999999999,"5min":0.42499999999999999,"15min":0.42499999999999999},"max":{"1min":0.85099999999999998,"5min":0.85099999999999998,"15min":0.85099999999999998},"last":0.60699999999999998}]},{"osd":4,"last update":"Sat Sep 23 20:40:59 2023","interfaces":[{"interface":"back","average":{"1min":0.41199999999999998,"5min":0.41199999999999998,"15min":0.41199999999999998},"min":{"1min":0.33300000000000002,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":0.64700000000000002,"5min":0.64700000000000002,"15min":0.64700000000000002},"last":0.67200000000000004},{"interface":"front","average":{"1min":0.46700000000000003,"5min":0.46700000000000003,"15min":0.46700000000000003},"min":{"1min":0.30299999999999999,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":0.82799999999999996,"5min":0.82799999999999996,"15min":0.82799999999999996},"last":0.66100000000000003}]},{"osd":6,"last update":"Sat Sep 23 20:41:22 2023","interfaces":[{"interface":"back","average":{"1min":0.47199999999999998,"5min":0.47199999999999998,"15min":0.47199999999999998},"min":{"1min":0.28599999999999998,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":0.72499999999999998,"5min":0.72499999999999998,"15min":0.72499999999999998},"last":0.70699999999999996},{"interface":"front","average":{"1min":0.52800000000000002,"5min":0.52800000000000002,"15min":0.52800000000000002},"min":{"1min":0.33300000000000002,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":0.82299999999999995,"5min":0.82299999999999995,"15min":0.82299999999999995},"last":0.75600000000000001}]},{"osd":7,"last update":"Sat Sep 23 20:41:41 2023","interfaces":[{"interface":"back","average":{"1min":0.55200000000000005,"5min":0.55200000000000005,"15min":0.55200000000000005},"min":{"1min":0.32200000000000001,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":0.79800000000000004,"5min":0.79800000000000004,"15min":0.79800000000000004},"last":0.73899999999999999},{"interface":"front","average":{"1min":0.56399999999999995,"5min":0.56399999999999995,"15min":0.56399999999999995},"min":{"1min":0.375,"5min":0.375,"15min":0.375},"max":{"1min":0.84999999999999998,"5min":0.84999999999999998,"15min":0.84999999999999998},"last":0.78900000000000003}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T20:41:52.547 INFO:tasks.cephadm.ceph_manager.ceph:clean! 2023-09-23T20:41:52.547 INFO:tasks.ceph:Waiting until ceph cluster ceph is healthy... 2023-09-23T20:41:52.547 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy 2023-09-23T20:41:52.547 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph health --format=json 2023-09-23T20:41:52.866 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Sep 23 20:41:52 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-alertmanager-a[149923]: ts=2023-09-23T20:41:52.487Z caller=main.go:240 level=info msg="Starting Alertmanager" version="(version=0.25.0, branch=HEAD, revision=258fab7cdd551f2cf251ed0348f0ad7289aee789)" 2023-09-23T20:41:52.866 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Sep 23 20:41:52 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-alertmanager-a[149923]: ts=2023-09-23T20:41:52.487Z caller=main.go:241 level=info build_context="(go=go1.19.4, user=root@abe866dd5717, date=20221222-14:51:36)" 2023-09-23T20:41:52.866 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Sep 23 20:41:52 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-alertmanager-a[149923]: ts=2023-09-23T20:41:52.489Z caller=cluster.go:185 level=info component=cluster msg="setting advertise address explicitly" addr=172.21.15.26 port=9094 2023-09-23T20:41:52.867 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Sep 23 20:41:52 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-alertmanager-a[149923]: ts=2023-09-23T20:41:52.491Z caller=cluster.go:681 level=info component=cluster msg="Waiting for gossip to settle..." interval=2s 2023-09-23T20:41:52.867 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Sep 23 20:41:52 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-alertmanager-a[149923]: ts=2023-09-23T20:41:52.520Z caller=coordinator.go:113 level=info component=configuration msg="Loading configuration file" file=/etc/alertmanager/alertmanager.yml 2023-09-23T20:41:52.867 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Sep 23 20:41:52 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-alertmanager-a[149923]: ts=2023-09-23T20:41:52.520Z caller=coordinator.go:126 level=info component=configuration msg="Completed loading of configuration file" file=/etc/alertmanager/alertmanager.yml 2023-09-23T20:41:52.867 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Sep 23 20:41:52 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-alertmanager-a[149923]: ts=2023-09-23T20:41:52.524Z caller=tls_config.go:232 level=info msg="Listening on" address=[::]:9093 2023-09-23T20:41:52.867 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Sep 23 20:41:52 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-alertmanager-a[149923]: ts=2023-09-23T20:41:52.524Z caller=tls_config.go:235 level=info msg="TLS is disabled." http2=false address=[::]:9093 2023-09-23T20:41:52.867 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Sep 23 20:41:52 smithi026 bash[149886]: 4ed9bf8e51d9b4bc61fdfeaea493c0d11bcbcc6b3a4f8f14aa3f775ac9f0523f 2023-09-23T20:41:52.867 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Sep 23 20:41:52 smithi026 systemd[1]: Started Ceph alertmanager.a for b3f8e4ec-5a50-11ee-8db4-212e2dc638e7. 2023-09-23T20:41:53.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:41:52 smithi070 ceph-mon[100755]: pgmap v31: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:41:53.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:41:52 smithi070 ceph-mon[100755]: from='mgr.14451 ' entity='mgr.y' 2023-09-23T20:41:53.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:41:52 smithi070 ceph-mon[100755]: from='mgr.14451 ' entity='mgr.y' 2023-09-23T20:41:53.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:41:52 smithi026 ceph-mon[95839]: pgmap v31: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:41:53.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:41:52 smithi026 ceph-mon[95839]: from='mgr.14451 ' entity='mgr.y' 2023-09-23T20:41:53.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:41:52 smithi026 ceph-mon[95839]: from='mgr.14451 ' entity='mgr.y' 2023-09-23T20:41:53.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:41:52 smithi026 ceph-mon[109912]: pgmap v31: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:41:53.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:41:52 smithi026 ceph-mon[109912]: from='mgr.14451 ' entity='mgr.y' 2023-09-23T20:41:53.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:41:52 smithi026 ceph-mon[109912]: from='mgr.14451 ' entity='mgr.y' 2023-09-23T20:41:53.161 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:41:54.130 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:41:53 smithi070 ceph-mon[100755]: from='client.24524 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:41:54.130 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:41:53 smithi070 ceph-mon[100755]: Reconfiguring prometheus.a (dependencies changed)... 2023-09-23T20:41:54.131 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:41:53 smithi070 ceph-mon[100755]: Reconfiguring daemon prometheus.a on smithi070 2023-09-23T20:41:54.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:41:53 smithi026 ceph-mon[95839]: from='client.24524 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:41:54.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:41:53 smithi026 ceph-mon[95839]: Reconfiguring prometheus.a (dependencies changed)... 2023-09-23T20:41:54.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:41:53 smithi026 ceph-mon[95839]: Reconfiguring daemon prometheus.a on smithi070 2023-09-23T20:41:54.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:41:53 smithi026 ceph-mon[109912]: from='client.24524 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:41:54.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:41:53 smithi026 ceph-mon[109912]: Reconfiguring prometheus.a (dependencies changed)... 2023-09-23T20:41:54.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:41:53 smithi026 ceph-mon[109912]: Reconfiguring daemon prometheus.a on smithi070 2023-09-23T20:41:54.611 INFO:teuthology.orchestra.run.smithi026.stdout: 2023-09-23T20:41:54.612 INFO:teuthology.orchestra.run.smithi026.stdout:{"status":"HEALTH_OK","checks":{},"mutes":[]} 2023-09-23T20:41:54.823 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Sep 23 20:41:54 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-alertmanager-a[149923]: ts=2023-09-23T20:41:54.492Z caller=cluster.go:706 level=info component=cluster msg="gossip not settled" polls=0 before=0 now=1 elapsed=2.001007991s 2023-09-23T20:41:55.042 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:41:54 smithi070 ceph-mon[100755]: pgmap v32: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:41:55.042 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:41:54 smithi070 ceph-mon[100755]: from='client.? 172.21.15.26:0/2762558990' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2023-09-23T20:41:55.042 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 20:41:54 smithi070 systemd[1]: Stopping Ceph prometheus.a for b3f8e4ec-5a50-11ee-8db4-212e2dc638e7... 2023-09-23T20:41:55.120 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:41:54 smithi026 ceph-mon[95839]: pgmap v32: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:41:55.120 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:41:54 smithi026 ceph-mon[95839]: from='client.? 172.21.15.26:0/2762558990' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2023-09-23T20:41:55.121 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:41:54 smithi026 ceph-mon[109912]: pgmap v32: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:41:55.121 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:41:54 smithi026 ceph-mon[109912]: from='client.? 172.21.15.26:0/2762558990' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2023-09-23T20:41:55.122 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy done 2023-09-23T20:41:55.122 INFO:tasks.cephadm:Setup complete, yielding 2023-09-23T20:41:55.122 INFO:teuthology.run_tasks:Running task cephadm.shell... 2023-09-23T20:41:55.132 INFO:tasks.cephadm:Running commands on role mon.a host ubuntu@smithi026.front.sepia.ceph.com 2023-09-23T20:41:55.133 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- bash -c 'ceph fs volume create foo' 2023-09-23T20:41:55.369 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 20:41:55 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-prometheus-a[134794]: ts=2023-09-23T20:41:55.041Z caller=main.go:840 level=warn msg="Received SIGTERM, exiting gracefully..." 2023-09-23T20:41:55.369 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 20:41:55 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-prometheus-a[134794]: ts=2023-09-23T20:41:55.042Z caller=main.go:864 level=info msg="Stopping scrape discovery manager..." 2023-09-23T20:41:55.370 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 20:41:55 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-prometheus-a[134794]: ts=2023-09-23T20:41:55.042Z caller=main.go:878 level=info msg="Stopping notify discovery manager..." 2023-09-23T20:41:55.370 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 20:41:55 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-prometheus-a[134794]: ts=2023-09-23T20:41:55.042Z caller=manager.go:988 level=info component="rule manager" msg="Stopping rule manager..." 2023-09-23T20:41:55.370 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 20:41:55 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-prometheus-a[134794]: ts=2023-09-23T20:41:55.042Z caller=main.go:860 level=info msg="Scrape discovery manager stopped" 2023-09-23T20:41:55.370 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 20:41:55 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-prometheus-a[134794]: ts=2023-09-23T20:41:55.042Z caller=manager.go:998 level=info component="rule manager" msg="Rule manager stopped" 2023-09-23T20:41:55.370 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 20:41:55 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-prometheus-a[134794]: ts=2023-09-23T20:41:55.042Z caller=main.go:874 level=info msg="Notify discovery manager stopped" 2023-09-23T20:41:55.370 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 20:41:55 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-prometheus-a[134794]: ts=2023-09-23T20:41:55.042Z caller=main.go:915 level=info msg="Stopping scrape manager..." 2023-09-23T20:41:55.370 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 20:41:55 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-prometheus-a[134794]: ts=2023-09-23T20:41:55.042Z caller=main.go:907 level=info msg="Scrape manager stopped" 2023-09-23T20:41:55.370 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 20:41:55 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-prometheus-a[134794]: ts=2023-09-23T20:41:55.042Z caller=notifier.go:606 level=info component=notifier msg="Stopping notification manager..." 2023-09-23T20:41:55.370 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 20:41:55 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-prometheus-a[134794]: ts=2023-09-23T20:41:55.042Z caller=main.go:1135 level=info msg="Notifier manager stopped" 2023-09-23T20:41:55.370 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 20:41:55 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-prometheus-a[134794]: ts=2023-09-23T20:41:55.042Z caller=main.go:1147 level=info msg="See you next time!" 2023-09-23T20:41:55.739 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:41:55.869 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 20:41:55 smithi070 bash[138372]: ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-prometheus-a 2023-09-23T20:41:55.869 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 20:41:55 smithi070 systemd[1]: ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7@prometheus.a.service: Succeeded. 2023-09-23T20:41:55.870 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 20:41:55 smithi070 systemd[1]: Stopped Ceph prometheus.a for b3f8e4ec-5a50-11ee-8db4-212e2dc638e7. 2023-09-23T20:41:55.870 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 20:41:55 smithi070 systemd[1]: Starting Ceph prometheus.a for b3f8e4ec-5a50-11ee-8db4-212e2dc638e7... 2023-09-23T20:41:56.218 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 20:41:55 smithi070 podman[138496]: 2023-09-23T20:41:56.553 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:41:56 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[96086]: [23/Sep/2023:20:41:56] ENGINE Bus STOPPING 2023-09-23T20:41:56.553 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:41:56 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[96086]: [23/Sep/2023:20:41:56] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2023-09-23T20:41:56.553 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:41:56 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[96086]: [23/Sep/2023:20:41:56] ENGINE Bus STOPPED 2023-09-23T20:41:56.553 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:41:56 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[96086]: [23/Sep/2023:20:41:56] ENGINE Bus STARTING 2023-09-23T20:41:56.553 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:41:56 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[96086]: [23/Sep/2023:20:41:56] ENGINE Serving on http://:::9283 2023-09-23T20:41:56.553 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:41:56 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[96086]: [23/Sep/2023:20:41:56] ENGINE Bus STARTED 2023-09-23T20:41:56.553 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:41:56 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[96086]: [23/Sep/2023:20:41:56] ENGINE Bus STOPPING 2023-09-23T20:41:56.554 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:41:56 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[96086]: [23/Sep/2023:20:41:56] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2023-09-23T20:41:56.554 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:41:56 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[96086]: [23/Sep/2023:20:41:56] ENGINE Bus STOPPED 2023-09-23T20:41:56.554 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:41:56 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[96086]: [23/Sep/2023:20:41:56] ENGINE Bus STARTING 2023-09-23T20:41:56.554 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:41:56 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[96086]: [23/Sep/2023:20:41:56] ENGINE Serving on http://:::9283 2023-09-23T20:41:56.554 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:41:56 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[96086]: [23/Sep/2023:20:41:56] ENGINE Bus STARTED 2023-09-23T20:41:56.620 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 20:41:56 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-prometheus-a[138511]: ts=2023-09-23T20:41:56.218Z caller=main.go:564 level=info msg="Starting Prometheus Server" mode=server version="(version=2.43.0, branch=HEAD, revision=edfc3bcd025dd6fe296c167a14a216cab1e552ee)" 2023-09-23T20:41:56.620 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 20:41:56 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-prometheus-a[138511]: ts=2023-09-23T20:41:56.218Z caller=main.go:569 level=info build_context="(go=go1.19.7, platform=linux/amd64, user=root@8a0ee342e522, date=20230321-12:56:07, tags=netgo,builtinassets)" 2023-09-23T20:41:56.620 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 20:41:56 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-prometheus-a[138511]: ts=2023-09-23T20:41:56.218Z caller=main.go:570 level=info host_details="(Linux 4.18.0-372.9.1.el8.x86_64 #1 SMP Fri Apr 15 22:12:19 EDT 2022 x86_64 smithi070 (none))" 2023-09-23T20:41:56.620 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 20:41:56 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-prometheus-a[138511]: ts=2023-09-23T20:41:56.218Z caller=main.go:571 level=info fd_limits="(soft=1048576, hard=1048576)" 2023-09-23T20:41:56.620 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 20:41:56 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-prometheus-a[138511]: ts=2023-09-23T20:41:56.218Z caller=main.go:572 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2023-09-23T20:41:56.620 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 20:41:56 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-prometheus-a[138511]: ts=2023-09-23T20:41:56.219Z caller=web.go:561 level=info component=web msg="Start listening for connections" address=:9095 2023-09-23T20:41:56.621 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 20:41:56 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-prometheus-a[138511]: ts=2023-09-23T20:41:56.220Z caller=main.go:1005 level=info msg="Starting TSDB ..." 2023-09-23T20:41:56.621 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 20:41:56 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-prometheus-a[138511]: ts=2023-09-23T20:41:56.220Z caller=tls_config.go:232 level=info component=web msg="Listening on" address=[::]:9095 2023-09-23T20:41:56.621 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 20:41:56 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-prometheus-a[138511]: ts=2023-09-23T20:41:56.220Z caller=tls_config.go:235 level=info component=web msg="TLS is disabled." http2=false address=[::]:9095 2023-09-23T20:41:56.621 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 20:41:56 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-prometheus-a[138511]: ts=2023-09-23T20:41:56.223Z caller=head.go:587 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2023-09-23T20:41:56.621 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 20:41:56 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-prometheus-a[138511]: ts=2023-09-23T20:41:56.223Z caller=head.go:658 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=2.109µs 2023-09-23T20:41:56.621 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 20:41:56 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-prometheus-a[138511]: ts=2023-09-23T20:41:56.223Z caller=head.go:664 level=info component=tsdb msg="Replaying WAL, this may take a while" 2023-09-23T20:41:56.621 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 20:41:56 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-prometheus-a[138511]: ts=2023-09-23T20:41:56.223Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=1 2023-09-23T20:41:56.621 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 20:41:56 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-prometheus-a[138511]: ts=2023-09-23T20:41:56.224Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=1 maxSegment=1 2023-09-23T20:41:56.621 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 20:41:56 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-prometheus-a[138511]: ts=2023-09-23T20:41:56.224Z caller=head.go:772 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=29.195µs wal_replay_duration=485.072µs wbl_replay_duration=84ns total_replay_duration=539.058µs 2023-09-23T20:41:56.621 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 20:41:56 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-prometheus-a[138511]: ts=2023-09-23T20:41:56.224Z caller=main.go:1026 level=info fs_type=XFS_SUPER_MAGIC 2023-09-23T20:41:56.621 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 20:41:56 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-prometheus-a[138511]: ts=2023-09-23T20:41:56.224Z caller=main.go:1029 level=info msg="TSDB started" 2023-09-23T20:41:56.622 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 20:41:56 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-prometheus-a[138511]: ts=2023-09-23T20:41:56.224Z caller=main.go:1209 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2023-09-23T20:41:56.622 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 20:41:56 smithi070 bash[138496]: 20647f147400fdd294dad4342f29f32b4e6b5c6e1f13e4ada6e301d25b159b81 2023-09-23T20:41:56.622 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 20:41:56 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-prometheus-a[138511]: ts=2023-09-23T20:41:56.238Z caller=main.go:1246 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=14.103792ms db_storage=798ns remote_storage=1.065µs web_handler=253ns query_engine=483ns scrape=188.593µs scrape_sd=23.156µs notify=18.314µs notify_sd=5.28µs rules=13.673791ms tracing=7.169µs 2023-09-23T20:41:56.622 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 20:41:56 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-prometheus-a[138511]: ts=2023-09-23T20:41:56.238Z caller=main.go:990 level=info msg="Server is ready to receive web requests." 2023-09-23T20:41:56.622 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 20:41:56 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-prometheus-a[138511]: ts=2023-09-23T20:41:56.238Z caller=manager.go:974 level=info component="rule manager" msg="Starting rule manager..." 2023-09-23T20:41:56.622 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 20:41:56 smithi070 systemd[1]: Started Ceph prometheus.a for b3f8e4ec-5a50-11ee-8db4-212e2dc638e7. 2023-09-23T20:41:56.878 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:41:56 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[96086]: [23/Sep/2023:20:41:56] ENGINE Bus STOPPING 2023-09-23T20:41:56.878 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:41:56 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[96086]: [23/Sep/2023:20:41:56] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2023-09-23T20:41:56.878 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:41:56 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[96086]: [23/Sep/2023:20:41:56] ENGINE Bus STOPPED 2023-09-23T20:41:56.878 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:41:56 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[96086]: [23/Sep/2023:20:41:56] ENGINE Bus STARTING 2023-09-23T20:41:56.878 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:41:56 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[96086]: [23/Sep/2023:20:41:56] ENGINE Serving on http://:::9283 2023-09-23T20:41:56.878 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:41:56 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[96086]: [23/Sep/2023:20:41:56] ENGINE Bus STARTED 2023-09-23T20:41:56.879 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:41:56 smithi026 ceph-mon[95839]: pgmap v33: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:41:56.879 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:41:56 smithi026 ceph-mon[95839]: from='mgr.14451 ' entity='mgr.y' 2023-09-23T20:41:56.879 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:41:56 smithi026 ceph-mon[95839]: from='mgr.14451 ' entity='mgr.y' 2023-09-23T20:41:56.879 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:41:56 smithi026 ceph-mon[95839]: from='mgr.14451 172.21.15.26:0/1329296977' entity='mgr.y' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2023-09-23T20:41:56.879 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:41:56 smithi026 ceph-mon[95839]: from='mgr.14451 172.21.15.26:0/1329296977' entity='mgr.y' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi026:9093"}]: dispatch 2023-09-23T20:41:56.879 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:41:56 smithi026 ceph-mon[95839]: from='mgr.14451 ' entity='mgr.y' 2023-09-23T20:41:56.879 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:41:56 smithi026 ceph-mon[95839]: from='mgr.14451 172.21.15.26:0/1329296977' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-09-23T20:41:56.879 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:41:56 smithi026 ceph-mon[95839]: from='mgr.14451 172.21.15.26:0/1329296977' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi070.front.sepia.ceph.com:9095"}]: dispatch 2023-09-23T20:41:56.879 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:41:56 smithi026 ceph-mon[95839]: from='mgr.14451 ' entity='mgr.y' 2023-09-23T20:41:56.879 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:41:56 smithi026 ceph-mon[95839]: from='mgr.14451 172.21.15.26:0/1329296977' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2023-09-23T20:41:56.879 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:41:56 smithi026 ceph-mon[95839]: from='mgr.14451 172.21.15.26:0/1329296977' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi070.front.sepia.ceph.com:3000"}]: dispatch 2023-09-23T20:41:56.880 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:41:56 smithi026 ceph-mon[95839]: from='mgr.14451 ' entity='mgr.y' 2023-09-23T20:41:56.880 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:41:56 smithi026 ceph-mon[95839]: from='mgr.14451 172.21.15.26:0/1329296977' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:41:56.880 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:41:56 smithi026 ceph-mon[109912]: pgmap v33: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:41:56.880 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:41:56 smithi026 ceph-mon[109912]: from='mgr.14451 ' entity='mgr.y' 2023-09-23T20:41:56.880 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:41:56 smithi026 ceph-mon[109912]: from='mgr.14451 ' entity='mgr.y' 2023-09-23T20:41:56.880 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:41:56 smithi026 ceph-mon[109912]: from='mgr.14451 172.21.15.26:0/1329296977' entity='mgr.y' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2023-09-23T20:41:56.880 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:41:56 smithi026 ceph-mon[109912]: from='mgr.14451 172.21.15.26:0/1329296977' entity='mgr.y' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi026:9093"}]: dispatch 2023-09-23T20:41:56.880 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:41:56 smithi026 ceph-mon[109912]: from='mgr.14451 ' entity='mgr.y' 2023-09-23T20:41:56.880 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:41:56 smithi026 ceph-mon[109912]: from='mgr.14451 172.21.15.26:0/1329296977' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-09-23T20:41:56.880 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:41:56 smithi026 ceph-mon[109912]: from='mgr.14451 172.21.15.26:0/1329296977' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi070.front.sepia.ceph.com:9095"}]: dispatch 2023-09-23T20:41:56.881 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:41:56 smithi026 ceph-mon[109912]: from='mgr.14451 ' entity='mgr.y' 2023-09-23T20:41:56.881 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:41:56 smithi026 ceph-mon[109912]: from='mgr.14451 172.21.15.26:0/1329296977' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2023-09-23T20:41:56.881 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:41:56 smithi026 ceph-mon[109912]: from='mgr.14451 172.21.15.26:0/1329296977' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi070.front.sepia.ceph.com:3000"}]: dispatch 2023-09-23T20:41:56.881 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:41:56 smithi026 ceph-mon[109912]: from='mgr.14451 ' entity='mgr.y' 2023-09-23T20:41:56.881 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:41:56 smithi026 ceph-mon[109912]: from='mgr.14451 172.21.15.26:0/1329296977' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:41:57.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:41:56 smithi070 ceph-mon[100755]: pgmap v33: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:41:57.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:41:56 smithi070 ceph-mon[100755]: from='mgr.14451 ' entity='mgr.y' 2023-09-23T20:41:57.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:41:56 smithi070 ceph-mon[100755]: from='mgr.14451 ' entity='mgr.y' 2023-09-23T20:41:57.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:41:56 smithi070 ceph-mon[100755]: from='mgr.14451 172.21.15.26:0/1329296977' entity='mgr.y' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2023-09-23T20:41:57.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:41:56 smithi070 ceph-mon[100755]: from='mgr.14451 172.21.15.26:0/1329296977' entity='mgr.y' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi026:9093"}]: dispatch 2023-09-23T20:41:57.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:41:56 smithi070 ceph-mon[100755]: from='mgr.14451 ' entity='mgr.y' 2023-09-23T20:41:57.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:41:56 smithi070 ceph-mon[100755]: from='mgr.14451 172.21.15.26:0/1329296977' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-09-23T20:41:57.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:41:56 smithi070 ceph-mon[100755]: from='mgr.14451 172.21.15.26:0/1329296977' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi070.front.sepia.ceph.com:9095"}]: dispatch 2023-09-23T20:41:57.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:41:56 smithi070 ceph-mon[100755]: from='mgr.14451 ' entity='mgr.y' 2023-09-23T20:41:57.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:41:56 smithi070 ceph-mon[100755]: from='mgr.14451 172.21.15.26:0/1329296977' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2023-09-23T20:41:57.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:41:56 smithi070 ceph-mon[100755]: from='mgr.14451 172.21.15.26:0/1329296977' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi070.front.sepia.ceph.com:3000"}]: dispatch 2023-09-23T20:41:57.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:41:56 smithi070 ceph-mon[100755]: from='mgr.14451 ' entity='mgr.y' 2023-09-23T20:41:57.371 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:41:56 smithi070 ceph-mon[100755]: from='mgr.14451 172.21.15.26:0/1329296977' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:41:58.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:41:57 smithi026 ceph-mon[109912]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2023-09-23T20:41:58.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:41:57 smithi026 ceph-mon[109912]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi026:9093"}]: dispatch 2023-09-23T20:41:58.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:41:57 smithi026 ceph-mon[109912]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-09-23T20:41:58.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:41:57 smithi026 ceph-mon[109912]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi070.front.sepia.ceph.com:9095"}]: dispatch 2023-09-23T20:41:58.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:41:57 smithi026 ceph-mon[109912]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2023-09-23T20:41:58.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:41:57 smithi026 ceph-mon[109912]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi070.front.sepia.ceph.com:3000"}]: dispatch 2023-09-23T20:41:58.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:41:57 smithi026 ceph-mon[109912]: from='mgr.14451 172.21.15.26:0/1329296977' entity='mgr.y' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]: dispatch 2023-09-23T20:41:58.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:41:57 smithi026 ceph-mon[109912]: from='mgr.14451 ' entity='mgr.y' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]: dispatch 2023-09-23T20:41:58.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:41:57 smithi026 ceph-mon[95839]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2023-09-23T20:41:58.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:41:57 smithi026 ceph-mon[95839]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi026:9093"}]: dispatch 2023-09-23T20:41:58.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:41:57 smithi026 ceph-mon[95839]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-09-23T20:41:58.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:41:57 smithi026 ceph-mon[95839]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi070.front.sepia.ceph.com:9095"}]: dispatch 2023-09-23T20:41:58.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:41:57 smithi026 ceph-mon[95839]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2023-09-23T20:41:58.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:41:57 smithi026 ceph-mon[95839]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi070.front.sepia.ceph.com:3000"}]: dispatch 2023-09-23T20:41:58.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:41:57 smithi026 ceph-mon[95839]: from='mgr.14451 172.21.15.26:0/1329296977' entity='mgr.y' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]: dispatch 2023-09-23T20:41:58.134 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:41:57 smithi026 ceph-mon[95839]: from='mgr.14451 ' entity='mgr.y' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]: dispatch 2023-09-23T20:41:58.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:41:57 smithi070 ceph-mon[100755]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2023-09-23T20:41:58.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:41:57 smithi070 ceph-mon[100755]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi026:9093"}]: dispatch 2023-09-23T20:41:58.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:41:57 smithi070 ceph-mon[100755]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-09-23T20:41:58.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:41:57 smithi070 ceph-mon[100755]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi070.front.sepia.ceph.com:9095"}]: dispatch 2023-09-23T20:41:58.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:41:57 smithi070 ceph-mon[100755]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2023-09-23T20:41:58.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:41:57 smithi070 ceph-mon[100755]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi070.front.sepia.ceph.com:3000"}]: dispatch 2023-09-23T20:41:58.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:41:57 smithi070 ceph-mon[100755]: from='mgr.14451 172.21.15.26:0/1329296977' entity='mgr.y' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]: dispatch 2023-09-23T20:41:58.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:41:57 smithi070 ceph-mon[100755]: from='mgr.14451 ' entity='mgr.y' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]: dispatch 2023-09-23T20:41:59.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:41:58 smithi026 ceph-mon[109912]: from='client.24536 -' entity='client.admin' cmd=[{"prefix": "fs volume create", "name": "foo", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T20:41:59.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:41:58 smithi026 ceph-mon[109912]: pgmap v34: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:41:59.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:41:58 smithi026 ceph-mon[109912]: from='mgr.14451 ' entity='mgr.y' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]': finished 2023-09-23T20:41:59.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:41:58 smithi026 ceph-mon[109912]: osdmap e65: 8 total, 8 up, 8 in 2023-09-23T20:41:59.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:41:58 smithi026 ceph-mon[109912]: from='mgr.14451 172.21.15.26:0/1329296977' entity='mgr.y' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foo.data"}]: dispatch 2023-09-23T20:41:59.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:41:58 smithi026 ceph-mon[109912]: from='mgr.14451 ' entity='mgr.y' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foo.data"}]: dispatch 2023-09-23T20:41:59.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:41:58 smithi026 ceph-mon[95839]: from='client.24536 -' entity='client.admin' cmd=[{"prefix": "fs volume create", "name": "foo", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T20:41:59.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:41:58 smithi026 ceph-mon[95839]: pgmap v34: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:41:59.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:41:58 smithi026 ceph-mon[95839]: from='mgr.14451 ' entity='mgr.y' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]': finished 2023-09-23T20:41:59.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:41:58 smithi026 ceph-mon[95839]: osdmap e65: 8 total, 8 up, 8 in 2023-09-23T20:41:59.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:41:58 smithi026 ceph-mon[95839]: from='mgr.14451 172.21.15.26:0/1329296977' entity='mgr.y' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foo.data"}]: dispatch 2023-09-23T20:41:59.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:41:58 smithi026 ceph-mon[95839]: from='mgr.14451 ' entity='mgr.y' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foo.data"}]: dispatch 2023-09-23T20:41:59.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:41:58 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mon-a[95815]: 2023-09-23T20:41:58.903+0000 7fd050ce8700 -1 log_channel(cluster) log [ERR] : Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2023-09-23T20:41:59.299 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- bash -c 'ceph config set mon mon_warn_on_insecure_global_id_reclaim false --force' 2023-09-23T20:41:59.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:41:58 smithi070 ceph-mon[100755]: from='client.24536 -' entity='client.admin' cmd=[{"prefix": "fs volume create", "name": "foo", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T20:41:59.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:41:58 smithi070 ceph-mon[100755]: pgmap v34: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:41:59.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:41:58 smithi070 ceph-mon[100755]: from='mgr.14451 ' entity='mgr.y' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]': finished 2023-09-23T20:41:59.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:41:58 smithi070 ceph-mon[100755]: osdmap e65: 8 total, 8 up, 8 in 2023-09-23T20:41:59.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:41:58 smithi070 ceph-mon[100755]: from='mgr.14451 172.21.15.26:0/1329296977' entity='mgr.y' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foo.data"}]: dispatch 2023-09-23T20:41:59.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:41:58 smithi070 ceph-mon[100755]: from='mgr.14451 ' entity='mgr.y' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foo.data"}]: dispatch 2023-09-23T20:41:59.920 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:42:00.215 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:41:59 smithi026 ceph-mon[95839]: from='mgr.14451 ' entity='mgr.y' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foo.data"}]': finished 2023-09-23T20:42:00.215 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:41:59 smithi026 ceph-mon[95839]: osdmap e66: 8 total, 8 up, 8 in 2023-09-23T20:42:00.215 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:41:59 smithi026 ceph-mon[95839]: from='mgr.14451 172.21.15.26:0/1329296977' entity='mgr.y' cmd=[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]: dispatch 2023-09-23T20:42:00.215 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:41:59 smithi026 ceph-mon[95839]: from='mgr.14451 ' entity='mgr.y' cmd=[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]: dispatch 2023-09-23T20:42:00.216 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:41:59 smithi026 ceph-mon[95839]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2023-09-23T20:42:00.216 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:41:59 smithi026 ceph-mon[95839]: Health check failed: 1 filesystem is online with fewer MDS than max_mds (MDS_UP_LESS_THAN_MAX) 2023-09-23T20:42:00.216 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:41:59 smithi026 ceph-mon[95839]: osdmap e67: 8 total, 8 up, 8 in 2023-09-23T20:42:00.216 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:41:59 smithi026 ceph-mon[95839]: from='mgr.14451 ' entity='mgr.y' cmd='[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]': finished 2023-09-23T20:42:00.216 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:41:59 smithi026 ceph-mon[95839]: fsmap foo:0 2023-09-23T20:42:00.216 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:41:59 smithi026 ceph-mon[95839]: from='mgr.14451 ' entity='mgr.y' 2023-09-23T20:42:00.216 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:41:59 smithi026 ceph-mon[109912]: from='mgr.14451 ' entity='mgr.y' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foo.data"}]': finished 2023-09-23T20:42:00.217 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:41:59 smithi026 ceph-mon[109912]: osdmap e66: 8 total, 8 up, 8 in 2023-09-23T20:42:00.217 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:41:59 smithi026 ceph-mon[109912]: from='mgr.14451 172.21.15.26:0/1329296977' entity='mgr.y' cmd=[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]: dispatch 2023-09-23T20:42:00.217 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:41:59 smithi026 ceph-mon[109912]: from='mgr.14451 ' entity='mgr.y' cmd=[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]: dispatch 2023-09-23T20:42:00.217 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:41:59 smithi026 ceph-mon[109912]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2023-09-23T20:42:00.217 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:41:59 smithi026 ceph-mon[109912]: Health check failed: 1 filesystem is online with fewer MDS than max_mds (MDS_UP_LESS_THAN_MAX) 2023-09-23T20:42:00.217 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:41:59 smithi026 ceph-mon[109912]: osdmap e67: 8 total, 8 up, 8 in 2023-09-23T20:42:00.217 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:41:59 smithi026 ceph-mon[109912]: from='mgr.14451 ' entity='mgr.y' cmd='[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]': finished 2023-09-23T20:42:00.217 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:41:59 smithi026 ceph-mon[109912]: fsmap foo:0 2023-09-23T20:42:00.217 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:41:59 smithi026 ceph-mon[109912]: from='mgr.14451 ' entity='mgr.y' 2023-09-23T20:42:00.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:41:59 smithi070 ceph-mon[100755]: from='mgr.14451 ' entity='mgr.y' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foo.data"}]': finished 2023-09-23T20:42:00.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:41:59 smithi070 ceph-mon[100755]: osdmap e66: 8 total, 8 up, 8 in 2023-09-23T20:42:00.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:41:59 smithi070 ceph-mon[100755]: from='mgr.14451 172.21.15.26:0/1329296977' entity='mgr.y' cmd=[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]: dispatch 2023-09-23T20:42:00.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:41:59 smithi070 ceph-mon[100755]: from='mgr.14451 ' entity='mgr.y' cmd=[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]: dispatch 2023-09-23T20:42:00.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:41:59 smithi070 ceph-mon[100755]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2023-09-23T20:42:00.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:41:59 smithi070 ceph-mon[100755]: Health check failed: 1 filesystem is online with fewer MDS than max_mds (MDS_UP_LESS_THAN_MAX) 2023-09-23T20:42:00.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:41:59 smithi070 ceph-mon[100755]: osdmap e67: 8 total, 8 up, 8 in 2023-09-23T20:42:00.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:41:59 smithi070 ceph-mon[100755]: from='mgr.14451 ' entity='mgr.y' cmd='[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]': finished 2023-09-23T20:42:00.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:41:59 smithi070 ceph-mon[100755]: fsmap foo:0 2023-09-23T20:42:00.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:41:59 smithi070 ceph-mon[100755]: from='mgr.14451 ' entity='mgr.y' 2023-09-23T20:42:01.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:42:00 smithi070 ceph-mon[100755]: Saving service mds.foo spec with placement count:2 2023-09-23T20:42:01.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:42:00 smithi070 ceph-mon[100755]: pgmap v38: 73 pgs: 73 active+clean; 577 KiB data, 50 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:42:01.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:42:00 smithi070 ceph-mon[100755]: osdmap e68: 8 total, 8 up, 8 in 2023-09-23T20:42:01.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:42:00 smithi070 ceph-mon[100755]: from='mgr.14451 ' entity='mgr.y' 2023-09-23T20:42:01.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:42:00 smithi070 ceph-mon[100755]: from='mgr.14451 ' entity='mgr.y' 2023-09-23T20:42:01.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:42:00 smithi070 ceph-mon[100755]: from='mgr.14451 ' entity='mgr.y' 2023-09-23T20:42:01.371 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:42:00 smithi070 ceph-mon[100755]: from='mgr.14451 ' entity='mgr.y' 2023-09-23T20:42:01.371 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:42:00 smithi070 ceph-mon[100755]: from='mgr.14451 172.21.15.26:0/1329296977' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:42:01.371 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:42:00 smithi070 ceph-mon[100755]: from='mgr.14451 172.21.15.26:0/1329296977' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:42:01.371 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:42:00 smithi070 ceph-mon[100755]: from='mgr.14451 ' entity='mgr.y' 2023-09-23T20:42:01.371 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:42:00 smithi070 ceph-mon[100755]: from='mgr.14451 172.21.15.26:0/1329296977' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi026.znfpnl", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-09-23T20:42:01.371 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:42:00 smithi070 ceph-mon[100755]: from='mgr.14451 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi026.znfpnl", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-09-23T20:42:01.371 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:42:00 smithi070 ceph-mon[100755]: from='mgr.14451 ' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi026.znfpnl", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2023-09-23T20:42:01.372 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:42:00 smithi070 ceph-mon[100755]: from='mgr.14451 172.21.15.26:0/1329296977' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:42:01.381 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:42:00 smithi026 ceph-mon[109912]: Saving service mds.foo spec with placement count:2 2023-09-23T20:42:01.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:42:00 smithi026 ceph-mon[109912]: pgmap v38: 73 pgs: 73 active+clean; 577 KiB data, 50 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:42:01.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:42:00 smithi026 ceph-mon[109912]: osdmap e68: 8 total, 8 up, 8 in 2023-09-23T20:42:01.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:42:00 smithi026 ceph-mon[109912]: from='mgr.14451 ' entity='mgr.y' 2023-09-23T20:42:01.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:42:00 smithi026 ceph-mon[109912]: from='mgr.14451 ' entity='mgr.y' 2023-09-23T20:42:01.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:42:00 smithi026 ceph-mon[109912]: from='mgr.14451 ' entity='mgr.y' 2023-09-23T20:42:01.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:42:00 smithi026 ceph-mon[109912]: from='mgr.14451 ' entity='mgr.y' 2023-09-23T20:42:01.383 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:42:00 smithi026 ceph-mon[109912]: from='mgr.14451 172.21.15.26:0/1329296977' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:42:01.383 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:42:00 smithi026 ceph-mon[109912]: from='mgr.14451 172.21.15.26:0/1329296977' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:42:01.383 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:42:00 smithi026 ceph-mon[109912]: from='mgr.14451 ' entity='mgr.y' 2023-09-23T20:42:01.383 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:42:00 smithi026 ceph-mon[109912]: from='mgr.14451 172.21.15.26:0/1329296977' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi026.znfpnl", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-09-23T20:42:01.383 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:42:00 smithi026 ceph-mon[109912]: from='mgr.14451 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi026.znfpnl", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-09-23T20:42:01.383 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:42:00 smithi026 ceph-mon[109912]: from='mgr.14451 ' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi026.znfpnl", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2023-09-23T20:42:01.383 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:42:00 smithi026 ceph-mon[109912]: from='mgr.14451 172.21.15.26:0/1329296977' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:42:01.384 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:42:00 smithi026 ceph-mon[95839]: Saving service mds.foo spec with placement count:2 2023-09-23T20:42:01.384 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:42:00 smithi026 ceph-mon[95839]: pgmap v38: 73 pgs: 73 active+clean; 577 KiB data, 50 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:42:01.384 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:42:00 smithi026 ceph-mon[95839]: osdmap e68: 8 total, 8 up, 8 in 2023-09-23T20:42:01.384 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:42:00 smithi026 ceph-mon[95839]: from='mgr.14451 ' entity='mgr.y' 2023-09-23T20:42:01.384 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:42:00 smithi026 ceph-mon[95839]: from='mgr.14451 ' entity='mgr.y' 2023-09-23T20:42:01.384 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:42:00 smithi026 ceph-mon[95839]: from='mgr.14451 ' entity='mgr.y' 2023-09-23T20:42:01.385 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:42:00 smithi026 ceph-mon[95839]: from='mgr.14451 ' entity='mgr.y' 2023-09-23T20:42:01.385 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:42:00 smithi026 ceph-mon[95839]: from='mgr.14451 172.21.15.26:0/1329296977' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:42:01.386 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:42:00 smithi026 ceph-mon[95839]: from='mgr.14451 172.21.15.26:0/1329296977' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:42:01.386 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:42:00 smithi026 ceph-mon[95839]: from='mgr.14451 ' entity='mgr.y' 2023-09-23T20:42:01.386 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:42:00 smithi026 ceph-mon[95839]: from='mgr.14451 172.21.15.26:0/1329296977' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi026.znfpnl", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-09-23T20:42:01.386 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:42:00 smithi026 ceph-mon[95839]: from='mgr.14451 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi026.znfpnl", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-09-23T20:42:01.386 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:42:00 smithi026 ceph-mon[95839]: from='mgr.14451 ' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi026.znfpnl", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2023-09-23T20:42:01.386 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:42:00 smithi026 ceph-mon[95839]: from='mgr.14451 172.21.15.26:0/1329296977' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:42:01.869 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 20:42:01 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-x[102861]: ::ffff:172.21.15.70 - - [23/Sep/2023:20:42:01] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2023-09-23T20:42:02.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:42:01 smithi026 ceph-mon[95839]: Deploying daemon mds.foo.smithi026.znfpnl on smithi026 2023-09-23T20:42:02.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:42:01 smithi026 ceph-mon[109912]: Deploying daemon mds.foo.smithi026.znfpnl on smithi026 2023-09-23T20:42:02.289 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- bash -c 'ceph config set mon mon_warn_on_insecure_global_id_reclaim_allowed false --force' 2023-09-23T20:42:02.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:42:01 smithi070 ceph-mon[100755]: Deploying daemon mds.foo.smithi026.znfpnl on smithi026 2023-09-23T20:42:02.780 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Sep 23 20:42:02 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-alertmanager-a[149923]: ts=2023-09-23T20:42:02.494Z caller=cluster.go:698 level=info component=cluster msg="gossip settled; proceeding" elapsed=10.003621242s 2023-09-23T20:42:03.062 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:42:03.295 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:42:02 smithi026 ceph-mon[109912]: pgmap v40: 73 pgs: 73 active+clean; 577 KiB data, 50 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:42:03.296 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:42:02 smithi026 ceph-mon[95839]: pgmap v40: 73 pgs: 73 active+clean; 577 KiB data, 50 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:42:03.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:42:02 smithi070 ceph-mon[100755]: pgmap v40: 73 pgs: 73 active+clean; 577 KiB data, 50 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:42:04.324 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:42:04 smithi070 ceph-mon[100755]: from='mgr.14451 ' entity='mgr.y' 2023-09-23T20:42:04.324 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:42:04 smithi070 ceph-mon[100755]: from='mgr.14451 ' entity='mgr.y' 2023-09-23T20:42:04.325 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:42:04 smithi070 ceph-mon[100755]: from='mgr.14451 ' entity='mgr.y' 2023-09-23T20:42:04.325 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:42:04 smithi070 ceph-mon[100755]: from='mgr.14451 172.21.15.26:0/1329296977' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi070.imqhqy", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-09-23T20:42:04.325 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:42:04 smithi070 ceph-mon[100755]: from='mgr.14451 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi070.imqhqy", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-09-23T20:42:04.325 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:42:04 smithi070 ceph-mon[100755]: from='mgr.14451 ' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi070.imqhqy", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2023-09-23T20:42:04.325 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:42:04 smithi070 ceph-mon[100755]: from='mgr.14451 172.21.15.26:0/1329296977' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:42:04.325 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:42:04 smithi070 ceph-mon[100755]: Deploying daemon mds.foo.smithi070.imqhqy on smithi070 2023-09-23T20:42:04.325 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:42:04 smithi070 ceph-mon[100755]: daemon mds.foo.smithi026.znfpnl assigned to filesystem foo as rank 0 (now has 1 ranks) 2023-09-23T20:42:04.325 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:42:04 smithi070 ceph-mon[100755]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2023-09-23T20:42:04.325 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:42:04 smithi070 ceph-mon[100755]: Health check cleared: MDS_UP_LESS_THAN_MAX (was: 1 filesystem is online with fewer MDS than max_mds) 2023-09-23T20:42:04.325 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:42:04 smithi070 ceph-mon[100755]: Cluster is now healthy 2023-09-23T20:42:04.325 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:42:04 smithi070 ceph-mon[100755]: mds.? [v2:172.21.15.26:6834/417830853,v1:172.21.15.26:6835/417830853] up:boot 2023-09-23T20:42:04.326 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:42:04 smithi070 ceph-mon[100755]: fsmap foo:1 {0=foo.smithi026.znfpnl=up:creating} 2023-09-23T20:42:04.326 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:42:04 smithi070 ceph-mon[100755]: from='mgr.14451 172.21.15.26:0/1329296977' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi026.znfpnl"}]: dispatch 2023-09-23T20:42:04.326 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:42:04 smithi070 ceph-mon[100755]: daemon mds.foo.smithi026.znfpnl is now active in filesystem foo as rank 0 2023-09-23T20:42:04.326 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:42:04 smithi070 ceph-mon[100755]: pgmap v41: 73 pgs: 73 active+clean; 577 KiB data, 50 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:42:04.326 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:42:04 smithi070 ceph-mon[100755]: from='mgr.14451 172.21.15.26:0/1329296977' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.b", "id": [1, 2]}]: dispatch 2023-09-23T20:42:04.326 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:42:04 smithi070 ceph-mon[100755]: from='mgr.14451 172.21.15.26:0/1329296977' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.18", "id": [1, 2]}]: dispatch 2023-09-23T20:42:04.326 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:42:04 smithi070 ceph-mon[100755]: from='mgr.14451 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.b", "id": [1, 2]}]: dispatch 2023-09-23T20:42:04.326 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:42:04 smithi070 ceph-mon[100755]: from='mgr.14451 172.21.15.26:0/1329296977' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1c", "id": [1, 5]}]: dispatch 2023-09-23T20:42:04.326 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:42:04 smithi070 ceph-mon[100755]: from='mgr.14451 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.18", "id": [1, 2]}]: dispatch 2023-09-23T20:42:04.326 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:42:04 smithi070 ceph-mon[100755]: from='mgr.14451 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1c", "id": [1, 5]}]: dispatch 2023-09-23T20:42:04.326 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:42:04 smithi070 ceph-mon[100755]: from='mgr.14451 172.21.15.26:0/1329296977' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:42:04.383 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:42:04 smithi026 ceph-mon[95839]: from='mgr.14451 ' entity='mgr.y' 2023-09-23T20:42:04.383 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:42:04 smithi026 ceph-mon[95839]: from='mgr.14451 ' entity='mgr.y' 2023-09-23T20:42:04.383 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:42:04 smithi026 ceph-mon[95839]: from='mgr.14451 ' entity='mgr.y' 2023-09-23T20:42:04.384 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:42:04 smithi026 ceph-mon[95839]: from='mgr.14451 172.21.15.26:0/1329296977' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi070.imqhqy", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-09-23T20:42:04.384 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:42:04 smithi026 ceph-mon[95839]: from='mgr.14451 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi070.imqhqy", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-09-23T20:42:04.384 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:42:04 smithi026 ceph-mon[95839]: from='mgr.14451 ' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi070.imqhqy", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2023-09-23T20:42:04.384 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:42:04 smithi026 ceph-mon[95839]: from='mgr.14451 172.21.15.26:0/1329296977' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:42:04.384 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:42:04 smithi026 ceph-mon[95839]: Deploying daemon mds.foo.smithi070.imqhqy on smithi070 2023-09-23T20:42:04.384 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:42:04 smithi026 ceph-mon[95839]: daemon mds.foo.smithi026.znfpnl assigned to filesystem foo as rank 0 (now has 1 ranks) 2023-09-23T20:42:04.385 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:42:04 smithi026 ceph-mon[95839]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2023-09-23T20:42:04.385 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:42:04 smithi026 ceph-mon[95839]: Health check cleared: MDS_UP_LESS_THAN_MAX (was: 1 filesystem is online with fewer MDS than max_mds) 2023-09-23T20:42:04.385 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:42:04 smithi026 ceph-mon[95839]: Cluster is now healthy 2023-09-23T20:42:04.385 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:42:04 smithi026 ceph-mon[95839]: mds.? [v2:172.21.15.26:6834/417830853,v1:172.21.15.26:6835/417830853] up:boot 2023-09-23T20:42:04.385 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:42:04 smithi026 ceph-mon[95839]: fsmap foo:1 {0=foo.smithi026.znfpnl=up:creating} 2023-09-23T20:42:04.385 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:42:04 smithi026 ceph-mon[95839]: from='mgr.14451 172.21.15.26:0/1329296977' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi026.znfpnl"}]: dispatch 2023-09-23T20:42:04.385 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:42:04 smithi026 ceph-mon[95839]: daemon mds.foo.smithi026.znfpnl is now active in filesystem foo as rank 0 2023-09-23T20:42:04.386 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:42:04 smithi026 ceph-mon[95839]: pgmap v41: 73 pgs: 73 active+clean; 577 KiB data, 50 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:42:04.386 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:42:04 smithi026 ceph-mon[95839]: from='mgr.14451 172.21.15.26:0/1329296977' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.b", "id": [1, 2]}]: dispatch 2023-09-23T20:42:04.386 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:42:04 smithi026 ceph-mon[95839]: from='mgr.14451 172.21.15.26:0/1329296977' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.18", "id": [1, 2]}]: dispatch 2023-09-23T20:42:04.386 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:42:04 smithi026 ceph-mon[95839]: from='mgr.14451 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.b", "id": [1, 2]}]: dispatch 2023-09-23T20:42:04.386 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:42:04 smithi026 ceph-mon[95839]: from='mgr.14451 172.21.15.26:0/1329296977' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1c", "id": [1, 5]}]: dispatch 2023-09-23T20:42:04.386 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:42:04 smithi026 ceph-mon[95839]: from='mgr.14451 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.18", "id": [1, 2]}]: dispatch 2023-09-23T20:42:04.386 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:42:04 smithi026 ceph-mon[95839]: from='mgr.14451 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1c", "id": [1, 5]}]: dispatch 2023-09-23T20:42:04.387 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:42:04 smithi026 ceph-mon[95839]: from='mgr.14451 172.21.15.26:0/1329296977' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:42:04.387 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:42:04 smithi026 ceph-mon[109912]: from='mgr.14451 ' entity='mgr.y' 2023-09-23T20:42:04.387 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:42:04 smithi026 ceph-mon[109912]: from='mgr.14451 ' entity='mgr.y' 2023-09-23T20:42:04.387 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:42:04 smithi026 ceph-mon[109912]: from='mgr.14451 ' entity='mgr.y' 2023-09-23T20:42:04.387 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:42:04 smithi026 ceph-mon[109912]: from='mgr.14451 172.21.15.26:0/1329296977' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi070.imqhqy", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-09-23T20:42:04.388 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:42:04 smithi026 ceph-mon[109912]: from='mgr.14451 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi070.imqhqy", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-09-23T20:42:04.388 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:42:04 smithi026 ceph-mon[109912]: from='mgr.14451 ' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi070.imqhqy", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2023-09-23T20:42:04.388 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:42:04 smithi026 ceph-mon[109912]: from='mgr.14451 172.21.15.26:0/1329296977' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:42:04.388 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:42:04 smithi026 ceph-mon[109912]: Deploying daemon mds.foo.smithi070.imqhqy on smithi070 2023-09-23T20:42:04.388 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:42:04 smithi026 ceph-mon[109912]: daemon mds.foo.smithi026.znfpnl assigned to filesystem foo as rank 0 (now has 1 ranks) 2023-09-23T20:42:04.389 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:42:04 smithi026 ceph-mon[109912]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2023-09-23T20:42:04.389 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:42:04 smithi026 ceph-mon[109912]: Health check cleared: MDS_UP_LESS_THAN_MAX (was: 1 filesystem is online with fewer MDS than max_mds) 2023-09-23T20:42:04.389 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:42:04 smithi026 ceph-mon[109912]: Cluster is now healthy 2023-09-23T20:42:04.389 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:42:04 smithi026 ceph-mon[109912]: mds.? [v2:172.21.15.26:6834/417830853,v1:172.21.15.26:6835/417830853] up:boot 2023-09-23T20:42:04.389 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:42:04 smithi026 ceph-mon[109912]: fsmap foo:1 {0=foo.smithi026.znfpnl=up:creating} 2023-09-23T20:42:04.389 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:42:04 smithi026 ceph-mon[109912]: from='mgr.14451 172.21.15.26:0/1329296977' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi026.znfpnl"}]: dispatch 2023-09-23T20:42:04.390 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:42:04 smithi026 ceph-mon[109912]: daemon mds.foo.smithi026.znfpnl is now active in filesystem foo as rank 0 2023-09-23T20:42:04.390 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:42:04 smithi026 ceph-mon[109912]: pgmap v41: 73 pgs: 73 active+clean; 577 KiB data, 50 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:42:04.390 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:42:04 smithi026 ceph-mon[109912]: from='mgr.14451 172.21.15.26:0/1329296977' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.b", "id": [1, 2]}]: dispatch 2023-09-23T20:42:04.390 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:42:04 smithi026 ceph-mon[109912]: from='mgr.14451 172.21.15.26:0/1329296977' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.18", "id": [1, 2]}]: dispatch 2023-09-23T20:42:04.390 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:42:04 smithi026 ceph-mon[109912]: from='mgr.14451 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.b", "id": [1, 2]}]: dispatch 2023-09-23T20:42:04.390 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:42:04 smithi026 ceph-mon[109912]: from='mgr.14451 172.21.15.26:0/1329296977' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1c", "id": [1, 5]}]: dispatch 2023-09-23T20:42:04.390 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:42:04 smithi026 ceph-mon[109912]: from='mgr.14451 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.18", "id": [1, 2]}]: dispatch 2023-09-23T20:42:04.391 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:42:04 smithi026 ceph-mon[109912]: from='mgr.14451 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1c", "id": [1, 5]}]: dispatch 2023-09-23T20:42:04.391 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:42:04 smithi026 ceph-mon[109912]: from='mgr.14451 172.21.15.26:0/1329296977' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:42:04.929 INFO:teuthology.run_tasks:Running task ceph.healthy... 2023-09-23T20:42:04.940 INFO:tasks.ceph:Waiting until ceph daemons up and pgs clean... 2023-09-23T20:42:04.940 INFO:tasks.cephadm.ceph_manager.ceph:waiting for mgr available 2023-09-23T20:42:04.941 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph mgr dump --format=json 2023-09-23T20:42:05.551 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:42:05.810 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:42:05 smithi026 ceph-mon[109912]: from='mgr.14451 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.b", "id": [1, 2]}]': finished 2023-09-23T20:42:05.811 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:42:05 smithi026 ceph-mon[109912]: from='mgr.14451 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.18", "id": [1, 2]}]': finished 2023-09-23T20:42:05.811 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:42:05 smithi026 ceph-mon[109912]: from='mgr.14451 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1c", "id": [1, 5]}]': finished 2023-09-23T20:42:05.811 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:42:05 smithi026 ceph-mon[109912]: osdmap e69: 8 total, 8 up, 8 in 2023-09-23T20:42:05.811 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:42:05 smithi026 ceph-mon[109912]: mds.? [v2:172.21.15.26:6834/417830853,v1:172.21.15.26:6835/417830853] up:active 2023-09-23T20:42:05.811 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:42:05 smithi026 ceph-mon[109912]: fsmap foo:1 {0=foo.smithi026.znfpnl=up:active} 2023-09-23T20:42:05.811 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:42:05 smithi026 ceph-mon[109912]: from='mgr.14451 ' entity='mgr.y' 2023-09-23T20:42:05.811 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:42:05 smithi026 ceph-mon[109912]: from='mgr.14451 ' entity='mgr.y' 2023-09-23T20:42:05.811 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:42:05 smithi026 ceph-mon[109912]: from='mgr.14451 ' entity='mgr.y' 2023-09-23T20:42:05.811 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:42:05 smithi026 ceph-mon[109912]: from='mgr.14451 ' entity='mgr.y' 2023-09-23T20:42:05.811 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:42:05 smithi026 ceph-mon[109912]: from='mgr.14451 ' entity='mgr.y' 2023-09-23T20:42:05.811 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:42:05 smithi026 ceph-mon[109912]: from='mgr.14451 172.21.15.26:0/1329296977' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:42:05.812 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:42:05 smithi026 ceph-mon[95839]: from='mgr.14451 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.b", "id": [1, 2]}]': finished 2023-09-23T20:42:05.812 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:42:05 smithi026 ceph-mon[95839]: from='mgr.14451 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.18", "id": [1, 2]}]': finished 2023-09-23T20:42:05.812 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:42:05 smithi026 ceph-mon[95839]: from='mgr.14451 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1c", "id": [1, 5]}]': finished 2023-09-23T20:42:05.812 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:42:05 smithi026 ceph-mon[95839]: osdmap e69: 8 total, 8 up, 8 in 2023-09-23T20:42:05.812 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:42:05 smithi026 ceph-mon[95839]: mds.? [v2:172.21.15.26:6834/417830853,v1:172.21.15.26:6835/417830853] up:active 2023-09-23T20:42:05.812 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:42:05 smithi026 ceph-mon[95839]: fsmap foo:1 {0=foo.smithi026.znfpnl=up:active} 2023-09-23T20:42:05.812 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:42:05 smithi026 ceph-mon[95839]: from='mgr.14451 ' entity='mgr.y' 2023-09-23T20:42:05.812 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:42:05 smithi026 ceph-mon[95839]: from='mgr.14451 ' entity='mgr.y' 2023-09-23T20:42:05.812 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:42:05 smithi026 ceph-mon[95839]: from='mgr.14451 ' entity='mgr.y' 2023-09-23T20:42:05.812 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:42:05 smithi026 ceph-mon[95839]: from='mgr.14451 ' entity='mgr.y' 2023-09-23T20:42:05.813 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:42:05 smithi026 ceph-mon[95839]: from='mgr.14451 ' entity='mgr.y' 2023-09-23T20:42:05.813 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:42:05 smithi026 ceph-mon[95839]: from='mgr.14451 172.21.15.26:0/1329296977' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:42:05.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:42:05 smithi070 ceph-mon[100755]: from='mgr.14451 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.b", "id": [1, 2]}]': finished 2023-09-23T20:42:05.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:42:05 smithi070 ceph-mon[100755]: from='mgr.14451 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.18", "id": [1, 2]}]': finished 2023-09-23T20:42:05.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:42:05 smithi070 ceph-mon[100755]: from='mgr.14451 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1c", "id": [1, 5]}]': finished 2023-09-23T20:42:05.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:42:05 smithi070 ceph-mon[100755]: osdmap e69: 8 total, 8 up, 8 in 2023-09-23T20:42:05.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:42:05 smithi070 ceph-mon[100755]: mds.? [v2:172.21.15.26:6834/417830853,v1:172.21.15.26:6835/417830853] up:active 2023-09-23T20:42:05.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:42:05 smithi070 ceph-mon[100755]: fsmap foo:1 {0=foo.smithi026.znfpnl=up:active} 2023-09-23T20:42:05.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:42:05 smithi070 ceph-mon[100755]: from='mgr.14451 ' entity='mgr.y' 2023-09-23T20:42:05.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:42:05 smithi070 ceph-mon[100755]: from='mgr.14451 ' entity='mgr.y' 2023-09-23T20:42:05.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:42:05 smithi070 ceph-mon[100755]: from='mgr.14451 ' entity='mgr.y' 2023-09-23T20:42:05.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:42:05 smithi070 ceph-mon[100755]: from='mgr.14451 ' entity='mgr.y' 2023-09-23T20:42:05.871 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:42:05 smithi070 ceph-mon[100755]: from='mgr.14451 ' entity='mgr.y' 2023-09-23T20:42:05.871 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:42:05 smithi070 ceph-mon[100755]: from='mgr.14451 172.21.15.26:0/1329296977' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:42:06.835 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:42:06 smithi026 ceph-mon[109912]: osdmap e70: 8 total, 8 up, 8 in 2023-09-23T20:42:06.835 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:42:06 smithi026 ceph-mon[109912]: mds.? [v2:172.21.15.70:6832/241309560,v1:172.21.15.70:6833/241309560] up:boot 2023-09-23T20:42:06.835 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:42:06 smithi026 ceph-mon[109912]: from='mgr.14451 172.21.15.26:0/1329296977' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi070.imqhqy"}]: dispatch 2023-09-23T20:42:06.835 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:42:06 smithi026 ceph-mon[109912]: fsmap foo:1 {0=foo.smithi026.znfpnl=up:active} 1 up:standby 2023-09-23T20:42:06.836 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:42:06 smithi026 ceph-mon[109912]: fsmap foo:1 {0=foo.smithi026.znfpnl=up:active} 1 up:standby 2023-09-23T20:42:06.836 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:42:06 smithi026 ceph-mon[109912]: pgmap v44: 73 pgs: 3 peering, 70 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 2.2 KiB/s wr, 7 op/s 2023-09-23T20:42:06.836 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:42:06 smithi026 ceph-mon[95839]: osdmap e70: 8 total, 8 up, 8 in 2023-09-23T20:42:06.836 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:42:06 smithi026 ceph-mon[95839]: mds.? [v2:172.21.15.70:6832/241309560,v1:172.21.15.70:6833/241309560] up:boot 2023-09-23T20:42:06.836 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:42:06 smithi026 ceph-mon[95839]: from='mgr.14451 172.21.15.26:0/1329296977' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi070.imqhqy"}]: dispatch 2023-09-23T20:42:06.836 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:42:06 smithi026 ceph-mon[95839]: fsmap foo:1 {0=foo.smithi026.znfpnl=up:active} 1 up:standby 2023-09-23T20:42:06.836 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:42:06 smithi026 ceph-mon[95839]: fsmap foo:1 {0=foo.smithi026.znfpnl=up:active} 1 up:standby 2023-09-23T20:42:06.836 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:42:06 smithi026 ceph-mon[95839]: pgmap v44: 73 pgs: 3 peering, 70 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 2.2 KiB/s wr, 7 op/s 2023-09-23T20:42:06.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:42:06 smithi070 ceph-mon[100755]: osdmap e70: 8 total, 8 up, 8 in 2023-09-23T20:42:06.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:42:06 smithi070 ceph-mon[100755]: mds.? [v2:172.21.15.70:6832/241309560,v1:172.21.15.70:6833/241309560] up:boot 2023-09-23T20:42:06.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:42:06 smithi070 ceph-mon[100755]: from='mgr.14451 172.21.15.26:0/1329296977' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi070.imqhqy"}]: dispatch 2023-09-23T20:42:06.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:42:06 smithi070 ceph-mon[100755]: fsmap foo:1 {0=foo.smithi026.znfpnl=up:active} 1 up:standby 2023-09-23T20:42:06.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:42:06 smithi070 ceph-mon[100755]: fsmap foo:1 {0=foo.smithi026.znfpnl=up:active} 1 up:standby 2023-09-23T20:42:06.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:42:06 smithi070 ceph-mon[100755]: pgmap v44: 73 pgs: 3 peering, 70 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 2.2 KiB/s wr, 7 op/s 2023-09-23T20:42:07.262 INFO:teuthology.orchestra.run.smithi026.stdout: 2023-09-23T20:42:07.572 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:42:07 smithi026 ceph-mon[109912]: from='client.? 172.21.15.26:0/3505850393' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2023-09-23T20:42:07.629 INFO:teuthology.orchestra.run.smithi026.stdout:{"epoch":20,"active_gid":14451,"active_name":"y","active_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6800","nonce":2440434873},{"type":"v1","addr":"172.21.15.26:6801","nonce":2440434873}]},"active_addr":"172.21.15.26:6801/2440434873","active_change":"2023-09-23T20:41:03.790839+0000","active_mgr_features":4540138320759226367,"available":true,"standbys":[{"gid":24346,"name":"x","mgr_features":4540138320759226367,"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2400","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"7","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.25.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:9.4.7","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.1.5","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.5.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.43.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"secs","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"noautoscale":{"name":"noautoscale","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"global autoscale flag","long_desc":"Option to turn on/off the autoscaler for all pools","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"drive_group_interval":{"name":"drive_group_interval","type":"float","level":"advanced","flags":0,"default_value":"300.0","min":"","max":"","enum_allowed":[],"desc":"interval in seconds between re-application of applied drive_groups","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}]}],"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2400","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"7","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.25.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:9.4.7","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.1.5","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.5.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.43.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"secs","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"noautoscale":{"name":"noautoscale","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"global autoscale flag","long_desc":"Option to turn on/off the autoscaler for all pools","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"drive_group_interval":{"name":"drive_group_interval","type":"float","level":"advanced","flags":0,"default_value":"300.0","min":"","max":"","enum_allowed":[],"desc":"interval in seconds between re-application of applied drive_groups","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}],"services":{"dashboard":"https://172.21.15.26:8443/","prometheus":"http://172.21.15.26:9283/"},"always_on_modules":{"octopus":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"pacific":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"quincy":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"]},"last_failure_osd_epoch":60,"active_clients":[{"name":"devicehealth","addrvec":[{"type":"v2","addr":"172.21.15.26:0","nonce":237900280}]},{"name":"libcephsqlite","addrvec":[{"type":"v2","addr":"172.21.15.26:0","nonce":1376232363}]},{"name":"rbd_support","addrvec":[{"type":"v2","addr":"172.21.15.26:0","nonce":4215395943}]},{"name":"volumes","addrvec":[{"type":"v2","addr":"172.21.15.26:0","nonce":635571892}]}]} 2023-09-23T20:42:07.634 INFO:tasks.cephadm.ceph_manager.ceph:mgr available! 2023-09-23T20:42:07.634 INFO:tasks.cephadm.ceph_manager.ceph:waiting for all up 2023-09-23T20:42:07.634 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph osd dump --format=json 2023-09-23T20:42:07.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:42:07 smithi070 ceph-mon[100755]: from='client.? 172.21.15.26:0/3505850393' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2023-09-23T20:42:07.881 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:42:07 smithi026 ceph-mon[95839]: from='client.? 172.21.15.26:0/3505850393' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2023-09-23T20:42:08.252 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:42:08.754 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:42:08 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[96086]: ::ffff:172.21.15.70 - - [23/Sep/2023:20:42:08] "GET /metrics HTTP/1.1" 200 237999 "" "Prometheus/2.43.0" 2023-09-23T20:42:08.755 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:42:08 smithi026 ceph-mon[95839]: mds.? [v2:172.21.15.26:6834/417830853,v1:172.21.15.26:6835/417830853] up:active 2023-09-23T20:42:08.755 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:42:08 smithi026 ceph-mon[95839]: fsmap foo:1 {0=foo.smithi026.znfpnl=up:active} 1 up:standby 2023-09-23T20:42:08.755 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:42:08 smithi026 ceph-mon[95839]: pgmap v45: 73 pgs: 3 peering, 70 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 1.6 KiB/s wr, 5 op/s 2023-09-23T20:42:08.755 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:42:08 smithi026 ceph-mon[109912]: mds.? [v2:172.21.15.26:6834/417830853,v1:172.21.15.26:6835/417830853] up:active 2023-09-23T20:42:08.755 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:42:08 smithi026 ceph-mon[109912]: fsmap foo:1 {0=foo.smithi026.znfpnl=up:active} 1 up:standby 2023-09-23T20:42:08.756 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:42:08 smithi026 ceph-mon[109912]: pgmap v45: 73 pgs: 3 peering, 70 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 1.6 KiB/s wr, 5 op/s 2023-09-23T20:42:08.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:42:08 smithi070 ceph-mon[100755]: mds.? [v2:172.21.15.26:6834/417830853,v1:172.21.15.26:6835/417830853] up:active 2023-09-23T20:42:08.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:42:08 smithi070 ceph-mon[100755]: fsmap foo:1 {0=foo.smithi026.znfpnl=up:active} 1 up:standby 2023-09-23T20:42:08.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:42:08 smithi070 ceph-mon[100755]: pgmap v45: 73 pgs: 3 peering, 70 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 1.6 KiB/s wr, 5 op/s 2023-09-23T20:42:09.894 INFO:teuthology.orchestra.run.smithi026.stdout: 2023-09-23T20:42:09.895 INFO:teuthology.orchestra.run.smithi026.stdout:{"epoch":70,"fsid":"b3f8e4ec-5a50-11ee-8db4-212e2dc638e7","created":"2023-09-23T20:35:58.919705+0000","modified":"2023-09-23T20:42:05.556840+0000","last_up_change":"2023-09-23T20:40:38.528428+0000","last_in_change":"2023-09-23T20:40:23.597125+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":4,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2023-09-23T20:38:57.022844+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}},{"pool":2,"pool_name":"rbd","create_time":"2023-09-23T20:41:15.182110+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"64","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}},{"pool":3,"pool_name":"cephfs.foo.meta","create_time":"2023-09-23T20:41:57.679446+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"68","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"foo"}}},{"pool":4,"pool_name":"cephfs.foo.data","create_time":"2023-09-23T20:41:58.331482+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"68","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"cephfs":{"data":"foo"}}}],"osds":[{"osd":0,"uuid":"f837feb7-060e-4965-9b48-bed9f3672d05","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":69,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6802","nonce":2638118366},{"type":"v1","addr":"172.21.15.26:6803","nonce":2638118366}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6804","nonce":2638118366},{"type":"v1","addr":"172.21.15.26:6805","nonce":2638118366}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6808","nonce":2638118366},{"type":"v1","addr":"172.21.15.26:6809","nonce":2638118366}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6806","nonce":2638118366},{"type":"v1","addr":"172.21.15.26:6807","nonce":2638118366}]},"public_addr":"172.21.15.26:6803/2638118366","cluster_addr":"172.21.15.26:6805/2638118366","heartbeat_back_addr":"172.21.15.26:6809/2638118366","heartbeat_front_addr":"172.21.15.26:6807/2638118366","state":["exists","up"]},{"osd":1,"uuid":"917f65aa-5963-4576-988a-396698b7c6a8","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":66,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6810","nonce":3250108152},{"type":"v1","addr":"172.21.15.26:6811","nonce":3250108152}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6812","nonce":3250108152},{"type":"v1","addr":"172.21.15.26:6813","nonce":3250108152}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6816","nonce":3250108152},{"type":"v1","addr":"172.21.15.26:6817","nonce":3250108152}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6814","nonce":3250108152},{"type":"v1","addr":"172.21.15.26:6815","nonce":3250108152}]},"public_addr":"172.21.15.26:6811/3250108152","cluster_addr":"172.21.15.26:6813/3250108152","heartbeat_back_addr":"172.21.15.26:6817/3250108152","heartbeat_front_addr":"172.21.15.26:6815/3250108152","state":["exists","up"]},{"osd":2,"uuid":"2e115328-76c6-478e-915c-5786903be49f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":69,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6818","nonce":1325135181},{"type":"v1","addr":"172.21.15.26:6819","nonce":1325135181}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6820","nonce":1325135181},{"type":"v1","addr":"172.21.15.26:6821","nonce":1325135181}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6824","nonce":1325135181},{"type":"v1","addr":"172.21.15.26:6825","nonce":1325135181}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6822","nonce":1325135181},{"type":"v1","addr":"172.21.15.26:6823","nonce":1325135181}]},"public_addr":"172.21.15.26:6819/1325135181","cluster_addr":"172.21.15.26:6821/1325135181","heartbeat_back_addr":"172.21.15.26:6825/1325135181","heartbeat_front_addr":"172.21.15.26:6823/1325135181","state":["exists","up"]},{"osd":3,"uuid":"c3dd2c3f-5cf5-4fb0-964c-1b8926f75c8a","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":26,"up_thru":66,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6826","nonce":3610585082},{"type":"v1","addr":"172.21.15.26:6827","nonce":3610585082}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6828","nonce":3610585082},{"type":"v1","addr":"172.21.15.26:6829","nonce":3610585082}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6832","nonce":3610585082},{"type":"v1","addr":"172.21.15.26:6833","nonce":3610585082}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6830","nonce":3610585082},{"type":"v1","addr":"172.21.15.26:6831","nonce":3610585082}]},"public_addr":"172.21.15.26:6827/3610585082","cluster_addr":"172.21.15.26:6829/3610585082","heartbeat_back_addr":"172.21.15.26:6833/3610585082","heartbeat_front_addr":"172.21.15.26:6831/3610585082","state":["exists","up"]},{"osd":4,"uuid":"a5f31996-1b38-491b-956b-5aebc72d0170","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":31,"up_thru":69,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6800","nonce":1528249138},{"type":"v1","addr":"172.21.15.70:6801","nonce":1528249138}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6802","nonce":1528249138},{"type":"v1","addr":"172.21.15.70:6803","nonce":1528249138}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6806","nonce":1528249138},{"type":"v1","addr":"172.21.15.70:6807","nonce":1528249138}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6804","nonce":1528249138},{"type":"v1","addr":"172.21.15.70:6805","nonce":1528249138}]},"public_addr":"172.21.15.70:6801/1528249138","cluster_addr":"172.21.15.70:6803/1528249138","heartbeat_back_addr":"172.21.15.70:6807/1528249138","heartbeat_front_addr":"172.21.15.70:6805/1528249138","state":["exists","up"]},{"osd":5,"uuid":"21dad198-b9db-42c6-8823-112df1ecd874","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":39,"up_thru":66,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6808","nonce":3802232906},{"type":"v1","addr":"172.21.15.70:6809","nonce":3802232906}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6810","nonce":3802232906},{"type":"v1","addr":"172.21.15.70:6811","nonce":3802232906}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6814","nonce":3802232906},{"type":"v1","addr":"172.21.15.70:6815","nonce":3802232906}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6812","nonce":3802232906},{"type":"v1","addr":"172.21.15.70:6813","nonce":3802232906}]},"public_addr":"172.21.15.70:6809/3802232906","cluster_addr":"172.21.15.70:6811/3802232906","heartbeat_back_addr":"172.21.15.70:6815/3802232906","heartbeat_front_addr":"172.21.15.70:6813/3802232906","state":["exists","up"]},{"osd":6,"uuid":"6f3be784-7149-4f13-81b5-98924e97c056","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":47,"up_thru":66,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6816","nonce":2944175278},{"type":"v1","addr":"172.21.15.70:6817","nonce":2944175278}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6818","nonce":2944175278},{"type":"v1","addr":"172.21.15.70:6819","nonce":2944175278}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6822","nonce":2944175278},{"type":"v1","addr":"172.21.15.70:6823","nonce":2944175278}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6820","nonce":2944175278},{"type":"v1","addr":"172.21.15.70:6821","nonce":2944175278}]},"public_addr":"172.21.15.70:6817/2944175278","cluster_addr":"172.21.15.70:6819/2944175278","heartbeat_back_addr":"172.21.15.70:6823/2944175278","heartbeat_front_addr":"172.21.15.70:6821/2944175278","state":["exists","up"]},{"osd":7,"uuid":"499f5f85-f4b2-40a7-ac0f-79cd22738d30","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":55,"up_thru":65,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6824","nonce":2824974165},{"type":"v1","addr":"172.21.15.70:6825","nonce":2824974165}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6826","nonce":2824974165},{"type":"v1","addr":"172.21.15.70:6827","nonce":2824974165}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6830","nonce":2824974165},{"type":"v1","addr":"172.21.15.70:6831","nonce":2824974165}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6828","nonce":2824974165},{"type":"v1","addr":"172.21.15.70:6829","nonce":2824974165}]},"public_addr":"172.21.15.70:6825/2824974165","cluster_addr":"172.21.15.70:6827/2824974165","heartbeat_back_addr":"172.21.15.70:6831/2824974165","heartbeat_front_addr":"172.21.15.70:6829/2824974165","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T20:38:10.875468+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T20:38:32.449963+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T20:38:53.633316+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T20:39:14.828940+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T20:39:34.945663+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T20:39:55.954288+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T20:40:16.847102+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T20:40:37.820404+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"4.b","mappings":[{"from":1,"to":2}]},{"pgid":"4.18","mappings":[{"from":1,"to":2}]},{"pgid":"4.1c","mappings":[{"from":1,"to":5}]}],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.26:6801/91363814":"2023-09-24T20:41:03.790648+0000","172.21.15.26:6800/91363814":"2023-09-24T20:41:03.790648+0000","172.21.15.26:0/3461760126":"2023-09-24T20:41:03.790648+0000","172.21.15.26:0/394447930":"2023-09-24T20:41:03.790648+0000","172.21.15.26:0/1083572999":"2023-09-24T20:41:03.790648+0000","172.21.15.26:0/2838335646":"2023-09-24T20:36:56.498559+0000","172.21.15.26:6801/952511372":"2023-09-24T20:36:56.498559+0000","172.21.15.26:6800/952511372":"2023-09-24T20:36:56.498559+0000","172.21.15.26:0/1625326392":"2023-09-24T20:36:56.498559+0000","172.21.15.26:0/3175144810":"2023-09-24T20:36:31.159649+0000","172.21.15.26:0/1815190465":"2023-09-24T20:36:31.159649+0000","172.21.15.26:0/188049283":"2023-09-24T20:41:03.790648+0000","172.21.15.26:0/289994511":"2023-09-24T20:36:56.498559+0000","172.21.15.26:0/686734212":"2023-09-24T20:36:31.159649+0000","172.21.15.26:6800/4099624907":"2023-09-24T20:36:31.159649+0000","172.21.15.26:6801/4099624907":"2023-09-24T20:36:31.159649+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2023-09-23T20:42:10.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:42:09 smithi070 ceph-mon[100755]: from='mgr.14451 ' entity='mgr.y' 2023-09-23T20:42:10.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:42:09 smithi026 ceph-mon[109912]: from='mgr.14451 ' entity='mgr.y' 2023-09-23T20:42:10.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:42:09 smithi026 ceph-mon[95839]: from='mgr.14451 ' entity='mgr.y' 2023-09-23T20:42:10.722 INFO:tasks.cephadm.ceph_manager.ceph:all up! 2023-09-23T20:42:10.722 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph osd dump --format=json 2023-09-23T20:42:11.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:42:10 smithi026 ceph-mon[109912]: pgmap v46: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 1.6 KiB/s wr, 5 op/s 2023-09-23T20:42:11.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:42:10 smithi026 ceph-mon[109912]: from='client.? 172.21.15.26:0/3325201919' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-09-23T20:42:11.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:42:10 smithi026 ceph-mon[109912]: mds.? [v2:172.21.15.70:6832/241309560,v1:172.21.15.70:6833/241309560] up:standby 2023-09-23T20:42:11.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:42:10 smithi026 ceph-mon[109912]: fsmap foo:1 {0=foo.smithi026.znfpnl=up:active} 1 up:standby 2023-09-23T20:42:11.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:42:10 smithi026 ceph-mon[109912]: from='mgr.14451 ' entity='mgr.y' 2023-09-23T20:42:11.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:42:10 smithi026 ceph-mon[109912]: from='mgr.14451 ' entity='mgr.y' 2023-09-23T20:42:11.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:42:10 smithi026 ceph-mon[95839]: pgmap v46: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 1.6 KiB/s wr, 5 op/s 2023-09-23T20:42:11.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:42:10 smithi026 ceph-mon[95839]: from='client.? 172.21.15.26:0/3325201919' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-09-23T20:42:11.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:42:10 smithi026 ceph-mon[95839]: mds.? [v2:172.21.15.70:6832/241309560,v1:172.21.15.70:6833/241309560] up:standby 2023-09-23T20:42:11.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:42:10 smithi026 ceph-mon[95839]: fsmap foo:1 {0=foo.smithi026.znfpnl=up:active} 1 up:standby 2023-09-23T20:42:11.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:42:10 smithi026 ceph-mon[95839]: from='mgr.14451 ' entity='mgr.y' 2023-09-23T20:42:11.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:42:10 smithi026 ceph-mon[95839]: from='mgr.14451 ' entity='mgr.y' 2023-09-23T20:42:11.350 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:42:11.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:42:10 smithi070 ceph-mon[100755]: pgmap v46: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 1.6 KiB/s wr, 5 op/s 2023-09-23T20:42:11.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:42:10 smithi070 ceph-mon[100755]: from='client.? 172.21.15.26:0/3325201919' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-09-23T20:42:11.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:42:10 smithi070 ceph-mon[100755]: mds.? [v2:172.21.15.70:6832/241309560,v1:172.21.15.70:6833/241309560] up:standby 2023-09-23T20:42:11.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:42:10 smithi070 ceph-mon[100755]: fsmap foo:1 {0=foo.smithi026.znfpnl=up:active} 1 up:standby 2023-09-23T20:42:11.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:42:10 smithi070 ceph-mon[100755]: from='mgr.14451 ' entity='mgr.y' 2023-09-23T20:42:11.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:42:10 smithi070 ceph-mon[100755]: from='mgr.14451 ' entity='mgr.y' 2023-09-23T20:42:11.869 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 20:42:11 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-x[102861]: ::ffff:172.21.15.70 - - [23/Sep/2023:20:42:11] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2023-09-23T20:42:12.754 INFO:teuthology.orchestra.run.smithi026.stdout: 2023-09-23T20:42:12.755 INFO:teuthology.orchestra.run.smithi026.stdout:{"epoch":70,"fsid":"b3f8e4ec-5a50-11ee-8db4-212e2dc638e7","created":"2023-09-23T20:35:58.919705+0000","modified":"2023-09-23T20:42:05.556840+0000","last_up_change":"2023-09-23T20:40:38.528428+0000","last_in_change":"2023-09-23T20:40:23.597125+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":4,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2023-09-23T20:38:57.022844+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}},{"pool":2,"pool_name":"rbd","create_time":"2023-09-23T20:41:15.182110+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"64","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}},{"pool":3,"pool_name":"cephfs.foo.meta","create_time":"2023-09-23T20:41:57.679446+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"68","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"foo"}}},{"pool":4,"pool_name":"cephfs.foo.data","create_time":"2023-09-23T20:41:58.331482+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"68","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"cephfs":{"data":"foo"}}}],"osds":[{"osd":0,"uuid":"f837feb7-060e-4965-9b48-bed9f3672d05","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":69,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6802","nonce":2638118366},{"type":"v1","addr":"172.21.15.26:6803","nonce":2638118366}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6804","nonce":2638118366},{"type":"v1","addr":"172.21.15.26:6805","nonce":2638118366}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6808","nonce":2638118366},{"type":"v1","addr":"172.21.15.26:6809","nonce":2638118366}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6806","nonce":2638118366},{"type":"v1","addr":"172.21.15.26:6807","nonce":2638118366}]},"public_addr":"172.21.15.26:6803/2638118366","cluster_addr":"172.21.15.26:6805/2638118366","heartbeat_back_addr":"172.21.15.26:6809/2638118366","heartbeat_front_addr":"172.21.15.26:6807/2638118366","state":["exists","up"]},{"osd":1,"uuid":"917f65aa-5963-4576-988a-396698b7c6a8","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":66,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6810","nonce":3250108152},{"type":"v1","addr":"172.21.15.26:6811","nonce":3250108152}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6812","nonce":3250108152},{"type":"v1","addr":"172.21.15.26:6813","nonce":3250108152}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6816","nonce":3250108152},{"type":"v1","addr":"172.21.15.26:6817","nonce":3250108152}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6814","nonce":3250108152},{"type":"v1","addr":"172.21.15.26:6815","nonce":3250108152}]},"public_addr":"172.21.15.26:6811/3250108152","cluster_addr":"172.21.15.26:6813/3250108152","heartbeat_back_addr":"172.21.15.26:6817/3250108152","heartbeat_front_addr":"172.21.15.26:6815/3250108152","state":["exists","up"]},{"osd":2,"uuid":"2e115328-76c6-478e-915c-5786903be49f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":69,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6818","nonce":1325135181},{"type":"v1","addr":"172.21.15.26:6819","nonce":1325135181}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6820","nonce":1325135181},{"type":"v1","addr":"172.21.15.26:6821","nonce":1325135181}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6824","nonce":1325135181},{"type":"v1","addr":"172.21.15.26:6825","nonce":1325135181}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6822","nonce":1325135181},{"type":"v1","addr":"172.21.15.26:6823","nonce":1325135181}]},"public_addr":"172.21.15.26:6819/1325135181","cluster_addr":"172.21.15.26:6821/1325135181","heartbeat_back_addr":"172.21.15.26:6825/1325135181","heartbeat_front_addr":"172.21.15.26:6823/1325135181","state":["exists","up"]},{"osd":3,"uuid":"c3dd2c3f-5cf5-4fb0-964c-1b8926f75c8a","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":26,"up_thru":66,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6826","nonce":3610585082},{"type":"v1","addr":"172.21.15.26:6827","nonce":3610585082}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6828","nonce":3610585082},{"type":"v1","addr":"172.21.15.26:6829","nonce":3610585082}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6832","nonce":3610585082},{"type":"v1","addr":"172.21.15.26:6833","nonce":3610585082}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6830","nonce":3610585082},{"type":"v1","addr":"172.21.15.26:6831","nonce":3610585082}]},"public_addr":"172.21.15.26:6827/3610585082","cluster_addr":"172.21.15.26:6829/3610585082","heartbeat_back_addr":"172.21.15.26:6833/3610585082","heartbeat_front_addr":"172.21.15.26:6831/3610585082","state":["exists","up"]},{"osd":4,"uuid":"a5f31996-1b38-491b-956b-5aebc72d0170","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":31,"up_thru":69,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6800","nonce":1528249138},{"type":"v1","addr":"172.21.15.70:6801","nonce":1528249138}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6802","nonce":1528249138},{"type":"v1","addr":"172.21.15.70:6803","nonce":1528249138}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6806","nonce":1528249138},{"type":"v1","addr":"172.21.15.70:6807","nonce":1528249138}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6804","nonce":1528249138},{"type":"v1","addr":"172.21.15.70:6805","nonce":1528249138}]},"public_addr":"172.21.15.70:6801/1528249138","cluster_addr":"172.21.15.70:6803/1528249138","heartbeat_back_addr":"172.21.15.70:6807/1528249138","heartbeat_front_addr":"172.21.15.70:6805/1528249138","state":["exists","up"]},{"osd":5,"uuid":"21dad198-b9db-42c6-8823-112df1ecd874","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":39,"up_thru":66,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6808","nonce":3802232906},{"type":"v1","addr":"172.21.15.70:6809","nonce":3802232906}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6810","nonce":3802232906},{"type":"v1","addr":"172.21.15.70:6811","nonce":3802232906}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6814","nonce":3802232906},{"type":"v1","addr":"172.21.15.70:6815","nonce":3802232906}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6812","nonce":3802232906},{"type":"v1","addr":"172.21.15.70:6813","nonce":3802232906}]},"public_addr":"172.21.15.70:6809/3802232906","cluster_addr":"172.21.15.70:6811/3802232906","heartbeat_back_addr":"172.21.15.70:6815/3802232906","heartbeat_front_addr":"172.21.15.70:6813/3802232906","state":["exists","up"]},{"osd":6,"uuid":"6f3be784-7149-4f13-81b5-98924e97c056","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":47,"up_thru":66,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6816","nonce":2944175278},{"type":"v1","addr":"172.21.15.70:6817","nonce":2944175278}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6818","nonce":2944175278},{"type":"v1","addr":"172.21.15.70:6819","nonce":2944175278}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6822","nonce":2944175278},{"type":"v1","addr":"172.21.15.70:6823","nonce":2944175278}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6820","nonce":2944175278},{"type":"v1","addr":"172.21.15.70:6821","nonce":2944175278}]},"public_addr":"172.21.15.70:6817/2944175278","cluster_addr":"172.21.15.70:6819/2944175278","heartbeat_back_addr":"172.21.15.70:6823/2944175278","heartbeat_front_addr":"172.21.15.70:6821/2944175278","state":["exists","up"]},{"osd":7,"uuid":"499f5f85-f4b2-40a7-ac0f-79cd22738d30","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":55,"up_thru":65,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6824","nonce":2824974165},{"type":"v1","addr":"172.21.15.70:6825","nonce":2824974165}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6826","nonce":2824974165},{"type":"v1","addr":"172.21.15.70:6827","nonce":2824974165}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6830","nonce":2824974165},{"type":"v1","addr":"172.21.15.70:6831","nonce":2824974165}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6828","nonce":2824974165},{"type":"v1","addr":"172.21.15.70:6829","nonce":2824974165}]},"public_addr":"172.21.15.70:6825/2824974165","cluster_addr":"172.21.15.70:6827/2824974165","heartbeat_back_addr":"172.21.15.70:6831/2824974165","heartbeat_front_addr":"172.21.15.70:6829/2824974165","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T20:38:10.875468+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T20:38:32.449963+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T20:38:53.633316+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T20:39:14.828940+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T20:39:34.945663+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T20:39:55.954288+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T20:40:16.847102+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T20:40:37.820404+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"4.b","mappings":[{"from":1,"to":2}]},{"pgid":"4.18","mappings":[{"from":1,"to":2}]},{"pgid":"4.1c","mappings":[{"from":1,"to":5}]}],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.26:6801/91363814":"2023-09-24T20:41:03.790648+0000","172.21.15.26:6800/91363814":"2023-09-24T20:41:03.790648+0000","172.21.15.26:0/3461760126":"2023-09-24T20:41:03.790648+0000","172.21.15.26:0/394447930":"2023-09-24T20:41:03.790648+0000","172.21.15.26:0/1083572999":"2023-09-24T20:41:03.790648+0000","172.21.15.26:0/2838335646":"2023-09-24T20:36:56.498559+0000","172.21.15.26:6801/952511372":"2023-09-24T20:36:56.498559+0000","172.21.15.26:6800/952511372":"2023-09-24T20:36:56.498559+0000","172.21.15.26:0/1625326392":"2023-09-24T20:36:56.498559+0000","172.21.15.26:0/3175144810":"2023-09-24T20:36:31.159649+0000","172.21.15.26:0/1815190465":"2023-09-24T20:36:31.159649+0000","172.21.15.26:0/188049283":"2023-09-24T20:41:03.790648+0000","172.21.15.26:0/289994511":"2023-09-24T20:36:56.498559+0000","172.21.15.26:0/686734212":"2023-09-24T20:36:31.159649+0000","172.21.15.26:6800/4099624907":"2023-09-24T20:36:31.159649+0000","172.21.15.26:6801/4099624907":"2023-09-24T20:36:31.159649+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2023-09-23T20:42:13.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:42:12 smithi026 ceph-mon[95839]: pgmap v47: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 1.6 KiB/s wr, 5 op/s 2023-09-23T20:42:13.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:42:12 smithi026 ceph-mon[95839]: from='client.? 172.21.15.26:0/1231149976' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-09-23T20:42:13.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:42:12 smithi026 ceph-mon[109912]: pgmap v47: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 1.6 KiB/s wr, 5 op/s 2023-09-23T20:42:13.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:42:12 smithi026 ceph-mon[109912]: from='client.? 172.21.15.26:0/1231149976' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-09-23T20:42:13.203 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph tell osd.0 flush_pg_stats 2023-09-23T20:42:13.203 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph tell osd.1 flush_pg_stats 2023-09-23T20:42:13.204 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph tell osd.2 flush_pg_stats 2023-09-23T20:42:13.204 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph tell osd.3 flush_pg_stats 2023-09-23T20:42:13.205 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph tell osd.4 flush_pg_stats 2023-09-23T20:42:13.205 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph tell osd.5 flush_pg_stats 2023-09-23T20:42:13.205 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph tell osd.6 flush_pg_stats 2023-09-23T20:42:13.206 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph tell osd.7 flush_pg_stats 2023-09-23T20:42:13.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:42:12 smithi070 ceph-mon[100755]: pgmap v47: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 1.6 KiB/s wr, 5 op/s 2023-09-23T20:42:13.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:42:12 smithi070 ceph-mon[100755]: from='client.? 172.21.15.26:0/1231149976' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-09-23T20:42:14.134 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:42:14.250 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:42:14.288 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:42:14.345 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:42:14 smithi026 ceph-mon[95839]: from='mgr.14451 ' entity='mgr.y' 2023-09-23T20:42:14.345 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:42:14 smithi026 ceph-mon[95839]: from='mgr.14451 ' entity='mgr.y' 2023-09-23T20:42:14.345 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:42:14 smithi026 ceph-mon[95839]: from='mgr.14451 172.21.15.26:0/1329296977' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:42:14.345 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:42:14 smithi026 ceph-mon[95839]: from='mgr.14451 172.21.15.26:0/1329296977' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:42:14.345 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:42:14 smithi026 ceph-mon[95839]: from='mgr.14451 ' entity='mgr.y' 2023-09-23T20:42:14.345 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:42:14 smithi026 ceph-mon[95839]: from='mgr.14451 172.21.15.26:0/1329296977' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:42:14.345 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:42:14 smithi026 ceph-mon[95839]: pgmap v48: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 1.4 KiB/s wr, 4 op/s 2023-09-23T20:42:14.346 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:42:14 smithi026 ceph-mon[109912]: from='mgr.14451 ' entity='mgr.y' 2023-09-23T20:42:14.346 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:42:14 smithi026 ceph-mon[109912]: from='mgr.14451 ' entity='mgr.y' 2023-09-23T20:42:14.346 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:42:14 smithi026 ceph-mon[109912]: from='mgr.14451 172.21.15.26:0/1329296977' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:42:14.346 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:42:14 smithi026 ceph-mon[109912]: from='mgr.14451 172.21.15.26:0/1329296977' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:42:14.346 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:42:14 smithi026 ceph-mon[109912]: from='mgr.14451 ' entity='mgr.y' 2023-09-23T20:42:14.346 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:42:14 smithi026 ceph-mon[109912]: from='mgr.14451 172.21.15.26:0/1329296977' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:42:14.346 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:42:14 smithi026 ceph-mon[109912]: pgmap v48: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 1.4 KiB/s wr, 4 op/s 2023-09-23T20:42:14.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:42:14 smithi070 ceph-mon[100755]: from='mgr.14451 ' entity='mgr.y' 2023-09-23T20:42:14.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:42:14 smithi070 ceph-mon[100755]: from='mgr.14451 ' entity='mgr.y' 2023-09-23T20:42:14.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:42:14 smithi070 ceph-mon[100755]: from='mgr.14451 172.21.15.26:0/1329296977' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:42:14.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:42:14 smithi070 ceph-mon[100755]: from='mgr.14451 172.21.15.26:0/1329296977' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:42:14.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:42:14 smithi070 ceph-mon[100755]: from='mgr.14451 ' entity='mgr.y' 2023-09-23T20:42:14.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:42:14 smithi070 ceph-mon[100755]: from='mgr.14451 172.21.15.26:0/1329296977' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:42:14.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:42:14 smithi070 ceph-mon[100755]: pgmap v48: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 1.4 KiB/s wr, 4 op/s 2023-09-23T20:42:14.460 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:42:14.543 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:42:14.554 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:42:15.164 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:42:17.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:42:16 smithi026 ceph-mon[95839]: pgmap v49: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2023-09-23T20:42:17.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:42:16 smithi026 ceph-mon[109912]: pgmap v49: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2023-09-23T20:42:17.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:42:16 smithi070 ceph-mon[100755]: pgmap v49: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2023-09-23T20:42:17.502 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:42:18.213 INFO:teuthology.orchestra.run.smithi026.stdout:77309411370 2023-09-23T20:42:18.214 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.2 2023-09-23T20:42:18.872 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:42:18 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[96086]: ::ffff:172.21.15.70 - - [23/Sep/2023:20:42:18] "GET /metrics HTTP/1.1" 200 237999 "" "Prometheus/2.43.0" 2023-09-23T20:42:19.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:42:18 smithi026 ceph-mon[95839]: pgmap v50: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2023-09-23T20:42:19.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:42:18 smithi026 ceph-mon[95839]: from='mgr.14451 ' entity='mgr.y' 2023-09-23T20:42:19.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:42:18 smithi026 ceph-mon[95839]: from='mgr.14451 ' entity='mgr.y' 2023-09-23T20:42:19.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:42:18 smithi026 ceph-mon[109912]: pgmap v50: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2023-09-23T20:42:19.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:42:18 smithi026 ceph-mon[109912]: from='mgr.14451 ' entity='mgr.y' 2023-09-23T20:42:19.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:42:18 smithi026 ceph-mon[109912]: from='mgr.14451 ' entity='mgr.y' 2023-09-23T20:42:19.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:42:18 smithi070 ceph-mon[100755]: pgmap v50: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2023-09-23T20:42:19.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:42:18 smithi070 ceph-mon[100755]: from='mgr.14451 ' entity='mgr.y' 2023-09-23T20:42:19.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:42:18 smithi070 ceph-mon[100755]: from='mgr.14451 ' entity='mgr.y' 2023-09-23T20:42:19.906 INFO:teuthology.orchestra.run.smithi026.stdout:167503724575 2023-09-23T20:42:19.907 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.5 2023-09-23T20:42:20.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:42:19 smithi026 ceph-mon[109912]: from='mgr.14451 172.21.15.26:0/1329296977' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:42:20.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:42:19 smithi026 ceph-mon[95839]: from='mgr.14451 172.21.15.26:0/1329296977' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:42:20.272 INFO:teuthology.orchestra.run.smithi026.stdout:111669149734 2023-09-23T20:42:20.273 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.3 2023-09-23T20:42:20.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:42:19 smithi070 ceph-mon[100755]: from='mgr.14451 172.21.15.26:0/1329296977' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:42:21.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:42:20 smithi026 ceph-mon[95839]: pgmap v51: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2023-09-23T20:42:21.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:42:20 smithi026 ceph-mon[109912]: pgmap v51: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2023-09-23T20:42:21.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:42:20 smithi070 ceph-mon[100755]: pgmap v51: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2023-09-23T20:42:21.475 INFO:teuthology.orchestra.run.smithi026.stdout:201863462938 2023-09-23T20:42:21.476 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.6 2023-09-23T20:42:21.682 INFO:teuthology.orchestra.run.smithi026.stdout:55834574896 2023-09-23T20:42:21.683 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.1 2023-09-23T20:42:21.869 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 20:42:21 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-x[102861]: ::ffff:172.21.15.70 - - [23/Sep/2023:20:42:21] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2023-09-23T20:42:22.116 INFO:teuthology.orchestra.run.smithi026.stdout:133143986211 2023-09-23T20:42:22.116 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.4 2023-09-23T20:42:22.395 INFO:teuthology.orchestra.run.smithi026.stdout:34359738421 2023-09-23T20:42:22.395 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.0 2023-09-23T20:42:22.725 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:42:22.767 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:42:22.767 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:42:22.772 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:42:22.931 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:42:23.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:42:22 smithi026 ceph-mon[95839]: pgmap v52: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:42:23.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:42:22 smithi026 ceph-mon[109912]: pgmap v52: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:42:23.278 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:42:23.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:42:22 smithi070 ceph-mon[100755]: pgmap v52: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:42:23.458 INFO:teuthology.orchestra.run.smithi026.stdout:236223201302 2023-09-23T20:42:23.458 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.7 2023-09-23T20:42:24.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:42:24 smithi070 ceph-mon[100755]: pgmap v53: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:42:24.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:42:24 smithi026 ceph-mon[95839]: pgmap v53: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:42:24.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:42:24 smithi026 ceph-mon[109912]: pgmap v53: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:42:25.306 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:42:26.637 INFO:teuthology.orchestra.run.smithi026.stdout:77309411372 2023-09-23T20:42:26.965 INFO:teuthology.orchestra.run.smithi026.stdout:111669149736 2023-09-23T20:42:27.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:42:26 smithi026 ceph-mon[95839]: pgmap v54: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:42:27.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:42:26 smithi026 ceph-mon[95839]: from='client.? 172.21.15.26:0/4158603979' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2023-09-23T20:42:27.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:42:26 smithi026 ceph-mon[109912]: pgmap v54: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:42:27.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:42:26 smithi026 ceph-mon[109912]: from='client.? 172.21.15.26:0/4158603979' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2023-09-23T20:42:27.320 INFO:teuthology.orchestra.run.smithi026.stdout:201863462940 2023-09-23T20:42:27.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:42:26 smithi070 ceph-mon[100755]: pgmap v54: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:42:27.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:42:26 smithi070 ceph-mon[100755]: from='client.? 172.21.15.26:0/4158603979' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2023-09-23T20:42:27.574 INFO:teuthology.orchestra.run.smithi026.stdout:167503724577 2023-09-23T20:42:27.877 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:42:27 smithi026 ceph-mon[95839]: from='client.? 172.21.15.26:0/1117207288' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2023-09-23T20:42:27.877 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:42:27 smithi026 ceph-mon[95839]: from='client.? 172.21.15.26:0/1069419680' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2023-09-23T20:42:27.877 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:42:27 smithi026 ceph-mon[95839]: from='client.? 172.21.15.26:0/993204083' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2023-09-23T20:42:28.131 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:42:27 smithi026 ceph-mon[109912]: from='client.? 172.21.15.26:0/1117207288' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2023-09-23T20:42:28.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:42:27 smithi026 ceph-mon[109912]: from='client.? 172.21.15.26:0/1069419680' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2023-09-23T20:42:28.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:42:27 smithi026 ceph-mon[109912]: from='client.? 172.21.15.26:0/993204083' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2023-09-23T20:42:28.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:42:27 smithi070 ceph-mon[100755]: from='client.? 172.21.15.26:0/1117207288' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2023-09-23T20:42:28.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:42:27 smithi070 ceph-mon[100755]: from='client.? 172.21.15.26:0/1069419680' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2023-09-23T20:42:28.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:42:27 smithi070 ceph-mon[100755]: from='client.? 172.21.15.26:0/993204083' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2023-09-23T20:42:28.449 INFO:tasks.cephadm.ceph_manager.ceph:need seq 111669149734 got 111669149736 for osd.3 2023-09-23T20:42:28.450 DEBUG:teuthology.parallel:result is None 2023-09-23T20:42:28.813 INFO:teuthology.orchestra.run.smithi026.stdout:55834574897 2023-09-23T20:42:28.838 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:42:28 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[96086]: ::ffff:172.21.15.70 - - [23/Sep/2023:20:42:28] "GET /metrics HTTP/1.1" 200 238624 "" "Prometheus/2.43.0" 2023-09-23T20:42:29.059 INFO:tasks.cephadm.ceph_manager.ceph:need seq 201863462938 got 201863462940 for osd.6 2023-09-23T20:42:29.059 DEBUG:teuthology.parallel:result is None 2023-09-23T20:42:29.078 INFO:tasks.cephadm.ceph_manager.ceph:need seq 77309411370 got 77309411372 for osd.2 2023-09-23T20:42:29.079 DEBUG:teuthology.parallel:result is None 2023-09-23T20:42:29.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:42:28 smithi026 ceph-mon[95839]: pgmap v55: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:42:29.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:42:28 smithi026 ceph-mon[95839]: from='client.? 172.21.15.26:0/3118201623' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2023-09-23T20:42:29.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:42:28 smithi026 ceph-mon[109912]: pgmap v55: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:42:29.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:42:28 smithi026 ceph-mon[109912]: from='client.? 172.21.15.26:0/3118201623' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2023-09-23T20:42:29.280 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:42:29.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:42:28 smithi070 ceph-mon[100755]: pgmap v55: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:42:29.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:42:28 smithi070 ceph-mon[100755]: from='client.? 172.21.15.26:0/3118201623' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2023-09-23T20:42:29.719 INFO:tasks.cephadm.ceph_manager.ceph:need seq 167503724575 got 167503724577 for osd.5 2023-09-23T20:42:29.719 DEBUG:teuthology.parallel:result is None 2023-09-23T20:42:29.938 INFO:teuthology.orchestra.run.smithi026.stdout:133143986212 2023-09-23T20:42:30.140 INFO:tasks.cephadm.ceph_manager.ceph:need seq 55834574896 got 55834574897 for osd.1 2023-09-23T20:42:30.140 DEBUG:teuthology.parallel:result is None 2023-09-23T20:42:30.764 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:42:30 smithi026 ceph-mon[109912]: from='mgr.14451 ' entity='mgr.y' 2023-09-23T20:42:30.765 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:42:30 smithi026 ceph-mon[109912]: from='mgr.14451 ' entity='mgr.y' 2023-09-23T20:42:30.765 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:42:30 smithi026 ceph-mon[109912]: from='mgr.14451 172.21.15.26:0/1329296977' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:42:30.765 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:42:30 smithi026 ceph-mon[109912]: from='mgr.14451 172.21.15.26:0/1329296977' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:42:30.765 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:42:30 smithi026 ceph-mon[109912]: from='mgr.14451 ' entity='mgr.y' 2023-09-23T20:42:30.765 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:42:30 smithi026 ceph-mon[109912]: pgmap v56: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:42:30.765 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:42:30 smithi026 ceph-mon[109912]: from='client.? 172.21.15.26:0/3399687313' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2023-09-23T20:42:30.765 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:42:30 smithi026 ceph-mon[95839]: from='mgr.14451 ' entity='mgr.y' 2023-09-23T20:42:30.765 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:42:30 smithi026 ceph-mon[95839]: from='mgr.14451 ' entity='mgr.y' 2023-09-23T20:42:30.766 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:42:30 smithi026 ceph-mon[95839]: from='mgr.14451 172.21.15.26:0/1329296977' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:42:30.766 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:42:30 smithi026 ceph-mon[95839]: from='mgr.14451 172.21.15.26:0/1329296977' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:42:30.766 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:42:30 smithi026 ceph-mon[95839]: from='mgr.14451 ' entity='mgr.y' 2023-09-23T20:42:30.766 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:42:30 smithi026 ceph-mon[95839]: pgmap v56: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:42:30.766 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:42:30 smithi026 ceph-mon[95839]: from='client.? 172.21.15.26:0/3399687313' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2023-09-23T20:42:30.780 INFO:tasks.cephadm.ceph_manager.ceph:need seq 133143986211 got 133143986212 for osd.4 2023-09-23T20:42:30.780 DEBUG:teuthology.parallel:result is None 2023-09-23T20:42:30.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:42:30 smithi070 ceph-mon[100755]: from='mgr.14451 ' entity='mgr.y' 2023-09-23T20:42:30.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:42:30 smithi070 ceph-mon[100755]: from='mgr.14451 ' entity='mgr.y' 2023-09-23T20:42:30.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:42:30 smithi070 ceph-mon[100755]: from='mgr.14451 172.21.15.26:0/1329296977' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:42:30.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:42:30 smithi070 ceph-mon[100755]: from='mgr.14451 172.21.15.26:0/1329296977' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:42:30.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:42:30 smithi070 ceph-mon[100755]: from='mgr.14451 ' entity='mgr.y' 2023-09-23T20:42:30.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:42:30 smithi070 ceph-mon[100755]: pgmap v56: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:42:30.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:42:30 smithi070 ceph-mon[100755]: from='client.? 172.21.15.26:0/3399687313' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2023-09-23T20:42:31.166 INFO:teuthology.orchestra.run.smithi026.stdout:34359738423 2023-09-23T20:42:31.455 INFO:teuthology.orchestra.run.smithi026.stdout:236223201304 2023-09-23T20:42:31.610 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:42:31 smithi026 ceph-mon[95839]: from='client.? 172.21.15.26:0/2102047826' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2023-09-23T20:42:31.610 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:42:31 smithi026 ceph-mon[109912]: from='client.? 172.21.15.26:0/2102047826' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2023-09-23T20:42:31.629 INFO:tasks.cephadm.ceph_manager.ceph:need seq 34359738421 got 34359738423 for osd.0 2023-09-23T20:42:31.629 DEBUG:teuthology.parallel:result is None 2023-09-23T20:42:31.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:42:31 smithi070 ceph-mon[100755]: from='client.? 172.21.15.26:0/2102047826' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2023-09-23T20:42:31.870 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 20:42:31 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-x[102861]: ::ffff:172.21.15.70 - - [23/Sep/2023:20:42:31] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2023-09-23T20:42:31.902 INFO:tasks.cephadm.ceph_manager.ceph:need seq 236223201302 got 236223201304 for osd.7 2023-09-23T20:42:31.903 DEBUG:teuthology.parallel:result is None 2023-09-23T20:42:31.903 INFO:tasks.cephadm.ceph_manager.ceph:waiting for clean 2023-09-23T20:42:31.903 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T20:42:32.509 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:42:32.724 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:42:32 smithi026 ceph-mon[95839]: from='client.? 172.21.15.26:0/1850710684' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2023-09-23T20:42:32.724 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:42:32 smithi026 ceph-mon[95839]: pgmap v57: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:42:32.724 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:42:32 smithi026 ceph-mon[109912]: from='client.? 172.21.15.26:0/1850710684' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2023-09-23T20:42:32.724 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:42:32 smithi026 ceph-mon[109912]: pgmap v57: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:42:32.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:42:32 smithi070 ceph-mon[100755]: from='client.? 172.21.15.26:0/1850710684' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2023-09-23T20:42:32.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:42:32 smithi070 ceph-mon[100755]: pgmap v57: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:42:33.946 INFO:teuthology.orchestra.run.smithi026.stdout: 2023-09-23T20:42:33.947 INFO:teuthology.orchestra.run.smithi026.stderr:dumped all 2023-09-23T20:42:34.373 INFO:teuthology.orchestra.run.smithi026.stdout:{"pg_ready":true,"pg_map":{"version":58,"stamp":"2023-09-23T20:42:33.820544+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":592674,"num_objects":24,"num_object_clones":0,"num_object_copies":72,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":24,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":224,"num_write_kb":2429,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":167,"ondisk_log_size":167,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":52700,"kb_used_data":7388,"kb_used_omap":0,"kb_used_meta":45248,"kb_avail":749908516,"statfs":{"total":767960285184,"available":767906320384,"internally_reserved":0,"allocated":7565312,"data_stored":4253366,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":46333952},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002532"},"pg_stats":[{"pgid":"4.18","version":"0'0","reported_seq":22,"reported_epoch":70,"state":"active+clean","last_fresh":"2023-09-23T20:42:05.790328+0000","last_change":"2023-09-23T20:42:05.790328+0000","last_active":"2023-09-23T20:42:05.790328+0000","last_peered":"2023-09-23T20:42:05.790328+0000","last_clean":"2023-09-23T20:42:05.790328+0000","last_became_active":"2023-09-23T20:42:05.789915+0000","last_became_peered":"2023-09-23T20:42:05.789915+0000","last_unstale":"2023-09-23T20:42:05.790328+0000","last_undegraded":"2023-09-23T20:42:05.790328+0000","last_fullsized":"2023-09-23T20:42:05.790328+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:28:47.862378+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":17,"reported_epoch":69,"state":"active+clean","last_fresh":"2023-09-23T20:42:04.563924+0000","last_change":"2023-09-23T20:41:58.901188+0000","last_active":"2023-09-23T20:42:04.563924+0000","last_peered":"2023-09-23T20:42:04.563924+0000","last_clean":"2023-09-23T20:42:04.563924+0000","last_became_active":"2023-09-23T20:41:58.900733+0000","last_became_peered":"2023-09-23T20:41:58.900733+0000","last_unstale":"2023-09-23T20:42:04.563924+0000","last_undegraded":"2023-09-23T20:42:04.563924+0000","last_fullsized":"2023-09-23T20:42:04.563924+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":4,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:14:09.190468+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":12,"reported_epoch":69,"state":"active+clean","last_fresh":"2023-09-23T20:42:04.564949+0000","last_change":"2023-09-23T20:41:58.926106+0000","last_active":"2023-09-23T20:42:04.564949+0000","last_peered":"2023-09-23T20:42:04.564949+0000","last_clean":"2023-09-23T20:42:04.564949+0000","last_became_active":"2023-09-23T20:41:58.925811+0000","last_became_peered":"2023-09-23T20:41:58.925811+0000","last_unstale":"2023-09-23T20:42:04.564949+0000","last_undegraded":"2023-09-23T20:42:04.564949+0000","last_fullsized":"2023-09-23T20:42:04.564949+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:54:31.291402+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":14,"reported_epoch":69,"state":"active+clean","last_fresh":"2023-09-23T20:42:04.565694+0000","last_change":"2023-09-23T20:41:58.899392+0000","last_active":"2023-09-23T20:42:04.565694+0000","last_peered":"2023-09-23T20:42:04.565694+0000","last_clean":"2023-09-23T20:42:04.565694+0000","last_became_active":"2023-09-23T20:41:58.899090+0000","last_became_peered":"2023-09-23T20:41:58.899090+0000","last_unstale":"2023-09-23T20:42:04.565694+0000","last_undegraded":"2023-09-23T20:42:04.565694+0000","last_fullsized":"2023-09-23T20:42:04.565694+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:12:24.515412+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":12,"reported_epoch":69,"state":"active+clean","last_fresh":"2023-09-23T20:42:04.559676+0000","last_change":"2023-09-23T20:41:58.924613+0000","last_active":"2023-09-23T20:42:04.559676+0000","last_peered":"2023-09-23T20:42:04.559676+0000","last_clean":"2023-09-23T20:42:04.559676+0000","last_became_active":"2023-09-23T20:41:58.924221+0000","last_became_peered":"2023-09-23T20:41:58.924221+0000","last_unstale":"2023-09-23T20:42:04.559676+0000","last_undegraded":"2023-09-23T20:42:04.559676+0000","last_fullsized":"2023-09-23T20:42:04.559676+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:28:20.833451+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":15,"reported_epoch":69,"state":"active+clean","last_fresh":"2023-09-23T20:42:04.564540+0000","last_change":"2023-09-23T20:41:58.906744+0000","last_active":"2023-09-23T20:42:04.564540+0000","last_peered":"2023-09-23T20:42:04.564540+0000","last_clean":"2023-09-23T20:42:04.564540+0000","last_became_active":"2023-09-23T20:41:58.906409+0000","last_became_peered":"2023-09-23T20:41:58.906409+0000","last_unstale":"2023-09-23T20:42:04.564540+0000","last_undegraded":"2023-09-23T20:42:04.564540+0000","last_fullsized":"2023-09-23T20:42:04.564540+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:15:42.250523+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":12,"reported_epoch":69,"state":"active+clean","last_fresh":"2023-09-23T20:42:04.560547+0000","last_change":"2023-09-23T20:41:58.927547+0000","last_active":"2023-09-23T20:42:04.560547+0000","last_peered":"2023-09-23T20:42:04.560547+0000","last_clean":"2023-09-23T20:42:04.560547+0000","last_became_active":"2023-09-23T20:41:58.927269+0000","last_became_peered":"2023-09-23T20:41:58.927269+0000","last_unstale":"2023-09-23T20:42:04.560547+0000","last_undegraded":"2023-09-23T20:42:04.560547+0000","last_fullsized":"2023-09-23T20:42:04.560547+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:54:12.824329+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":15,"reported_epoch":69,"state":"active+clean","last_fresh":"2023-09-23T20:42:04.564449+0000","last_change":"2023-09-23T20:41:58.906725+0000","last_active":"2023-09-23T20:42:04.564449+0000","last_peered":"2023-09-23T20:42:04.564449+0000","last_clean":"2023-09-23T20:42:04.564449+0000","last_became_active":"2023-09-23T20:41:58.906455+0000","last_became_peered":"2023-09-23T20:41:58.906455+0000","last_unstale":"2023-09-23T20:42:04.564449+0000","last_undegraded":"2023-09-23T20:42:04.564449+0000","last_fullsized":"2023-09-23T20:42:04.564449+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:32:38.304955+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":22,"reported_epoch":70,"state":"active+clean","last_fresh":"2023-09-23T20:42:05.790247+0000","last_change":"2023-09-23T20:42:05.790247+0000","last_active":"2023-09-23T20:42:05.790247+0000","last_peered":"2023-09-23T20:42:05.790247+0000","last_clean":"2023-09-23T20:42:05.790247+0000","last_became_active":"2023-09-23T20:42:05.789961+0000","last_became_peered":"2023-09-23T20:42:05.789961+0000","last_unstale":"2023-09-23T20:42:05.790247+0000","last_undegraded":"2023-09-23T20:42:05.790247+0000","last_fullsized":"2023-09-23T20:42:05.790247+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:11:50.767403+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":14,"reported_epoch":69,"state":"active+clean","last_fresh":"2023-09-23T20:42:04.564221+0000","last_change":"2023-09-23T20:41:58.906156+0000","last_active":"2023-09-23T20:42:04.564221+0000","last_peered":"2023-09-23T20:42:04.564221+0000","last_clean":"2023-09-23T20:42:04.564221+0000","last_became_active":"2023-09-23T20:41:58.905838+0000","last_became_peered":"2023-09-23T20:41:58.905838+0000","last_unstale":"2023-09-23T20:42:04.564221+0000","last_undegraded":"2023-09-23T20:42:04.564221+0000","last_fullsized":"2023-09-23T20:42:04.564221+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:42:08.292319+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":12,"reported_epoch":69,"state":"active+clean","last_fresh":"2023-09-23T20:42:04.559478+0000","last_change":"2023-09-23T20:41:58.918255+0000","last_active":"2023-09-23T20:42:04.559478+0000","last_peered":"2023-09-23T20:42:04.559478+0000","last_clean":"2023-09-23T20:42:04.559478+0000","last_became_active":"2023-09-23T20:41:58.917956+0000","last_became_peered":"2023-09-23T20:41:58.917956+0000","last_unstale":"2023-09-23T20:42:04.559478+0000","last_undegraded":"2023-09-23T20:42:04.559478+0000","last_fullsized":"2023-09-23T20:42:04.559478+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:42:21.698474+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.6","version":"70'1","reported_seq":14,"reported_epoch":70,"state":"active+clean","last_fresh":"2023-09-23T20:42:05.569773+0000","last_change":"2023-09-23T20:41:58.906360+0000","last_active":"2023-09-23T20:42:05.569773+0000","last_peered":"2023-09-23T20:42:05.569773+0000","last_clean":"2023-09-23T20:42:05.569773+0000","last_became_active":"2023-09-23T20:41:58.905683+0000","last_became_peered":"2023-09-23T20:41:58.905683+0000","last_unstale":"2023-09-23T20:42:05.569773+0000","last_undegraded":"2023-09-23T20:42:05.569773+0000","last_fullsized":"2023-09-23T20:42:05.569773+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:19:20.821454+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":17,"reported_epoch":69,"state":"active+clean","last_fresh":"2023-09-23T20:42:04.561014+0000","last_change":"2023-09-23T20:41:16.555449+0000","last_active":"2023-09-23T20:42:04.561014+0000","last_peered":"2023-09-23T20:42:04.561014+0000","last_clean":"2023-09-23T20:42:04.561014+0000","last_became_active":"2023-09-23T20:41:16.555054+0000","last_became_peered":"2023-09-23T20:41:16.555054+0000","last_unstale":"2023-09-23T20:42:04.561014+0000","last_undegraded":"2023-09-23T20:42:04.561014+0000","last_fullsized":"2023-09-23T20:42:04.561014+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:19:29.226436+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":12,"reported_epoch":69,"state":"active+clean","last_fresh":"2023-09-23T20:42:04.564782+0000","last_change":"2023-09-23T20:41:59.061304+0000","last_active":"2023-09-23T20:42:04.564782+0000","last_peered":"2023-09-23T20:42:04.564782+0000","last_clean":"2023-09-23T20:42:04.564782+0000","last_became_active":"2023-09-23T20:41:59.060972+0000","last_became_peered":"2023-09-23T20:41:59.060972+0000","last_unstale":"2023-09-23T20:42:04.564782+0000","last_undegraded":"2023-09-23T20:42:04.564782+0000","last_fullsized":"2023-09-23T20:42:04.564782+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:56:08.771697+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":17,"reported_epoch":69,"state":"active+clean","last_fresh":"2023-09-23T20:42:04.561660+0000","last_change":"2023-09-23T20:41:16.554495+0000","last_active":"2023-09-23T20:42:04.561660+0000","last_peered":"2023-09-23T20:42:04.561660+0000","last_clean":"2023-09-23T20:42:04.561660+0000","last_became_active":"2023-09-23T20:41:16.554217+0000","last_became_peered":"2023-09-23T20:41:16.554217+0000","last_unstale":"2023-09-23T20:42:04.561660+0000","last_undegraded":"2023-09-23T20:42:04.561660+0000","last_fullsized":"2023-09-23T20:42:04.561660+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:35:48.172198+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":15,"reported_epoch":69,"state":"active+clean","last_fresh":"2023-09-23T20:42:04.565228+0000","last_change":"2023-09-23T20:41:58.903926+0000","last_active":"2023-09-23T20:42:04.565228+0000","last_peered":"2023-09-23T20:42:04.565228+0000","last_clean":"2023-09-23T20:42:04.565228+0000","last_became_active":"2023-09-23T20:41:58.903634+0000","last_became_peered":"2023-09-23T20:41:58.903634+0000","last_unstale":"2023-09-23T20:42:04.565228+0000","last_undegraded":"2023-09-23T20:42:04.565228+0000","last_fullsized":"2023-09-23T20:42:04.565228+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:18:08.164482+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":12,"reported_epoch":69,"state":"active+clean","last_fresh":"2023-09-23T20:42:04.564207+0000","last_change":"2023-09-23T20:41:59.060510+0000","last_active":"2023-09-23T20:42:04.564207+0000","last_peered":"2023-09-23T20:42:04.564207+0000","last_clean":"2023-09-23T20:42:04.564207+0000","last_became_active":"2023-09-23T20:41:59.059962+0000","last_became_peered":"2023-09-23T20:41:59.059962+0000","last_unstale":"2023-09-23T20:42:04.564207+0000","last_undegraded":"2023-09-23T20:42:04.564207+0000","last_fullsized":"2023-09-23T20:42:04.564207+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:53:51.728457+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2023-09-23T20:42:04.562197+0000","last_change":"2023-09-23T20:41:58.898729+0000","last_active":"2023-09-23T20:42:04.562197+0000","last_peered":"2023-09-23T20:42:04.562197+0000","last_clean":"2023-09-23T20:42:04.562197+0000","last_became_active":"2023-09-23T20:41:58.898458+0000","last_became_peered":"2023-09-23T20:41:58.898458+0000","last_unstale":"2023-09-23T20:42:04.562197+0000","last_undegraded":"2023-09-23T20:42:04.562197+0000","last_fullsized":"2023-09-23T20:42:04.562197+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:11:25.895709+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":17,"reported_epoch":69,"state":"active+clean","last_fresh":"2023-09-23T20:42:04.558432+0000","last_change":"2023-09-23T20:41:16.684629+0000","last_active":"2023-09-23T20:42:04.558432+0000","last_peered":"2023-09-23T20:42:04.558432+0000","last_clean":"2023-09-23T20:42:04.558432+0000","last_became_active":"2023-09-23T20:41:16.684297+0000","last_became_peered":"2023-09-23T20:41:16.684297+0000","last_unstale":"2023-09-23T20:42:04.558432+0000","last_undegraded":"2023-09-23T20:42:04.558432+0000","last_fullsized":"2023-09-23T20:42:04.558432+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:28:01.891279+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":12,"reported_epoch":69,"state":"active+clean","last_fresh":"2023-09-23T20:42:04.563820+0000","last_change":"2023-09-23T20:41:58.925171+0000","last_active":"2023-09-23T20:42:04.563820+0000","last_peered":"2023-09-23T20:42:04.563820+0000","last_clean":"2023-09-23T20:42:04.563820+0000","last_became_active":"2023-09-23T20:41:58.924883+0000","last_became_peered":"2023-09-23T20:41:58.924883+0000","last_unstale":"2023-09-23T20:42:04.563820+0000","last_undegraded":"2023-09-23T20:42:04.563820+0000","last_fullsized":"2023-09-23T20:42:04.563820+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:28:03.874579+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2023-09-23T20:42:04.563109+0000","last_change":"2023-09-23T20:41:58.906198+0000","last_active":"2023-09-23T20:42:04.563109+0000","last_peered":"2023-09-23T20:42:04.563109+0000","last_clean":"2023-09-23T20:42:04.563109+0000","last_became_active":"2023-09-23T20:41:58.905808+0000","last_became_peered":"2023-09-23T20:41:58.905808+0000","last_unstale":"2023-09-23T20:42:04.563109+0000","last_undegraded":"2023-09-23T20:42:04.563109+0000","last_fullsized":"2023-09-23T20:42:04.563109+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:45:45.033621+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":17,"reported_epoch":69,"state":"active+clean","last_fresh":"2023-09-23T20:42:04.558188+0000","last_change":"2023-09-23T20:41:16.684597+0000","last_active":"2023-09-23T20:42:04.558188+0000","last_peered":"2023-09-23T20:42:04.558188+0000","last_clean":"2023-09-23T20:42:04.558188+0000","last_became_active":"2023-09-23T20:41:16.684172+0000","last_became_peered":"2023-09-23T20:41:16.684172+0000","last_unstale":"2023-09-23T20:42:04.558188+0000","last_undegraded":"2023-09-23T20:42:04.558188+0000","last_fullsized":"2023-09-23T20:42:04.558188+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:35:12.680090+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":12,"reported_epoch":69,"state":"active+clean","last_fresh":"2023-09-23T20:42:04.561128+0000","last_change":"2023-09-23T20:41:59.062981+0000","last_active":"2023-09-23T20:42:04.561128+0000","last_peered":"2023-09-23T20:42:04.561128+0000","last_clean":"2023-09-23T20:42:04.561128+0000","last_became_active":"2023-09-23T20:41:59.062717+0000","last_became_peered":"2023-09-23T20:41:59.062717+0000","last_unstale":"2023-09-23T20:42:04.561128+0000","last_undegraded":"2023-09-23T20:42:04.561128+0000","last_fullsized":"2023-09-23T20:42:04.561128+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:20:58.386461+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2023-09-23T20:42:04.565010+0000","last_change":"2023-09-23T20:41:58.902762+0000","last_active":"2023-09-23T20:42:04.565010+0000","last_peered":"2023-09-23T20:42:04.565010+0000","last_clean":"2023-09-23T20:42:04.565010+0000","last_became_active":"2023-09-23T20:41:58.902448+0000","last_became_peered":"2023-09-23T20:41:58.902448+0000","last_unstale":"2023-09-23T20:42:04.565010+0000","last_undegraded":"2023-09-23T20:42:04.565010+0000","last_fullsized":"2023-09-23T20:42:04.565010+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:06:06.890898+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.0","version":"60'128","reported_seq":277,"reported_epoch":69,"state":"active+clean","last_fresh":"2023-09-23T20:42:04.558150+0000","last_change":"2023-09-23T20:40:42.550542+0000","last_active":"2023-09-23T20:42:04.558150+0000","last_peered":"2023-09-23T20:42:04.558150+0000","last_clean":"2023-09-23T20:42:04.558150+0000","last_became_active":"2023-09-23T20:40:42.550069+0000","last_became_peered":"2023-09-23T20:40:42.550069+0000","last_unstale":"2023-09-23T20:42:04.558150+0000","last_undegraded":"2023-09-23T20:42:04.558150+0000","last_fullsized":"2023-09-23T20:42:04.558150+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:38:57.324224+0000","objects_scrubbed":0,"log_size":128,"ondisk_log_size":128,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:53:50.698837+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":17,"reported_epoch":69,"state":"active+clean","last_fresh":"2023-09-23T20:42:04.564278+0000","last_change":"2023-09-23T20:41:16.555751+0000","last_active":"2023-09-23T20:42:04.564278+0000","last_peered":"2023-09-23T20:42:04.564278+0000","last_clean":"2023-09-23T20:42:04.564278+0000","last_became_active":"2023-09-23T20:41:16.555417+0000","last_became_peered":"2023-09-23T20:41:16.555417+0000","last_unstale":"2023-09-23T20:42:04.564278+0000","last_undegraded":"2023-09-23T20:42:04.564278+0000","last_fullsized":"2023-09-23T20:42:04.564278+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:54:54.137765+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":12,"reported_epoch":69,"state":"active+clean","last_fresh":"2023-09-23T20:42:04.561141+0000","last_change":"2023-09-23T20:41:58.923989+0000","last_active":"2023-09-23T20:42:04.561141+0000","last_peered":"2023-09-23T20:42:04.561141+0000","last_clean":"2023-09-23T20:42:04.561141+0000","last_became_active":"2023-09-23T20:41:58.923702+0000","last_became_peered":"2023-09-23T20:41:58.923702+0000","last_unstale":"2023-09-23T20:42:04.561141+0000","last_undegraded":"2023-09-23T20:42:04.561141+0000","last_fullsized":"2023-09-23T20:42:04.561141+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:44:37.813515+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2023-09-23T20:42:04.561524+0000","last_change":"2023-09-23T20:41:58.897342+0000","last_active":"2023-09-23T20:42:04.561524+0000","last_peered":"2023-09-23T20:42:04.561524+0000","last_clean":"2023-09-23T20:42:04.561524+0000","last_became_active":"2023-09-23T20:41:58.897041+0000","last_became_peered":"2023-09-23T20:41:58.897041+0000","last_unstale":"2023-09-23T20:42:04.561524+0000","last_undegraded":"2023-09-23T20:42:04.561524+0000","last_fullsized":"2023-09-23T20:42:04.561524+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:57:54.858274+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":17,"reported_epoch":69,"state":"active+clean","last_fresh":"2023-09-23T20:42:04.561101+0000","last_change":"2023-09-23T20:41:16.686010+0000","last_active":"2023-09-23T20:42:04.561101+0000","last_peered":"2023-09-23T20:42:04.561101+0000","last_clean":"2023-09-23T20:42:04.561101+0000","last_became_active":"2023-09-23T20:41:16.685615+0000","last_became_peered":"2023-09-23T20:41:16.685615+0000","last_unstale":"2023-09-23T20:42:04.561101+0000","last_undegraded":"2023-09-23T20:42:04.561101+0000","last_fullsized":"2023-09-23T20:42:04.561101+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:46:04.405920+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":12,"reported_epoch":69,"state":"active+clean","last_fresh":"2023-09-23T20:42:04.562189+0000","last_change":"2023-09-23T20:41:58.919583+0000","last_active":"2023-09-23T20:42:04.562189+0000","last_peered":"2023-09-23T20:42:04.562189+0000","last_clean":"2023-09-23T20:42:04.562189+0000","last_became_active":"2023-09-23T20:41:58.919281+0000","last_became_peered":"2023-09-23T20:41:58.919281+0000","last_unstale":"2023-09-23T20:42:04.562189+0000","last_undegraded":"2023-09-23T20:42:04.562189+0000","last_fullsized":"2023-09-23T20:42:04.562189+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:37:23.934877+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2023-09-23T20:42:04.560410+0000","last_change":"2023-09-23T20:41:58.905896+0000","last_active":"2023-09-23T20:42:04.560410+0000","last_peered":"2023-09-23T20:42:04.560410+0000","last_clean":"2023-09-23T20:42:04.560410+0000","last_became_active":"2023-09-23T20:41:58.905654+0000","last_became_peered":"2023-09-23T20:41:58.905654+0000","last_unstale":"2023-09-23T20:42:04.560410+0000","last_undegraded":"2023-09-23T20:42:04.560410+0000","last_fullsized":"2023-09-23T20:42:04.560410+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:41:11.238836+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":17,"reported_epoch":69,"state":"active+clean","last_fresh":"2023-09-23T20:42:04.564283+0000","last_change":"2023-09-23T20:41:16.556552+0000","last_active":"2023-09-23T20:42:04.564283+0000","last_peered":"2023-09-23T20:42:04.564283+0000","last_clean":"2023-09-23T20:42:04.564283+0000","last_became_active":"2023-09-23T20:41:16.556226+0000","last_became_peered":"2023-09-23T20:41:16.556226+0000","last_unstale":"2023-09-23T20:42:04.564283+0000","last_undegraded":"2023-09-23T20:42:04.564283+0000","last_fullsized":"2023-09-23T20:42:04.564283+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:21:42.514384+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":12,"reported_epoch":69,"state":"active+clean","last_fresh":"2023-09-23T20:42:04.561546+0000","last_change":"2023-09-23T20:41:58.917668+0000","last_active":"2023-09-23T20:42:04.561546+0000","last_peered":"2023-09-23T20:42:04.561546+0000","last_clean":"2023-09-23T20:42:04.561546+0000","last_became_active":"2023-09-23T20:41:58.917308+0000","last_became_peered":"2023-09-23T20:41:58.917308+0000","last_unstale":"2023-09-23T20:42:04.561546+0000","last_undegraded":"2023-09-23T20:42:04.561546+0000","last_fullsized":"2023-09-23T20:42:04.561546+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:40:38.700565+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2023-09-23T20:42:04.564516+0000","last_change":"2023-09-23T20:41:58.903585+0000","last_active":"2023-09-23T20:42:04.564516+0000","last_peered":"2023-09-23T20:42:04.564516+0000","last_clean":"2023-09-23T20:42:04.564516+0000","last_became_active":"2023-09-23T20:41:58.902845+0000","last_became_peered":"2023-09-23T20:41:58.902845+0000","last_unstale":"2023-09-23T20:42:04.564516+0000","last_undegraded":"2023-09-23T20:42:04.564516+0000","last_fullsized":"2023-09-23T20:42:04.564516+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:25:22.419537+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":17,"reported_epoch":69,"state":"active+clean","last_fresh":"2023-09-23T20:42:04.562135+0000","last_change":"2023-09-23T20:41:16.684748+0000","last_active":"2023-09-23T20:42:04.562135+0000","last_peered":"2023-09-23T20:42:04.562135+0000","last_clean":"2023-09-23T20:42:04.562135+0000","last_became_active":"2023-09-23T20:41:16.684361+0000","last_became_peered":"2023-09-23T20:41:16.684361+0000","last_unstale":"2023-09-23T20:42:04.562135+0000","last_undegraded":"2023-09-23T20:42:04.562135+0000","last_fullsized":"2023-09-23T20:42:04.562135+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:39:15.420185+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":12,"reported_epoch":69,"state":"active+clean","last_fresh":"2023-09-23T20:42:04.561218+0000","last_change":"2023-09-23T20:41:58.920027+0000","last_active":"2023-09-23T20:42:04.561218+0000","last_peered":"2023-09-23T20:42:04.561218+0000","last_clean":"2023-09-23T20:42:04.561218+0000","last_became_active":"2023-09-23T20:41:58.919731+0000","last_became_peered":"2023-09-23T20:41:58.919731+0000","last_unstale":"2023-09-23T20:42:04.561218+0000","last_undegraded":"2023-09-23T20:42:04.561218+0000","last_fullsized":"2023-09-23T20:42:04.561218+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:30:06.925226+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2023-09-23T20:42:04.564785+0000","last_change":"2023-09-23T20:41:58.898493+0000","last_active":"2023-09-23T20:42:04.564785+0000","last_peered":"2023-09-23T20:42:04.564785+0000","last_clean":"2023-09-23T20:42:04.564785+0000","last_became_active":"2023-09-23T20:41:58.898205+0000","last_became_peered":"2023-09-23T20:41:58.898205+0000","last_unstale":"2023-09-23T20:42:04.564785+0000","last_undegraded":"2023-09-23T20:42:04.564785+0000","last_fullsized":"2023-09-23T20:42:04.564785+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:16:08.144482+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":12,"reported_epoch":69,"state":"active+clean","last_fresh":"2023-09-23T20:42:04.559603+0000","last_change":"2023-09-23T20:41:58.921014+0000","last_active":"2023-09-23T20:42:04.559603+0000","last_peered":"2023-09-23T20:42:04.559603+0000","last_clean":"2023-09-23T20:42:04.559603+0000","last_became_active":"2023-09-23T20:41:58.920805+0000","last_became_peered":"2023-09-23T20:41:58.920805+0000","last_unstale":"2023-09-23T20:42:04.559603+0000","last_undegraded":"2023-09-23T20:42:04.559603+0000","last_fullsized":"2023-09-23T20:42:04.559603+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:03:21.219355+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2023-09-23T20:42:04.559522+0000","last_change":"2023-09-23T20:41:58.902484+0000","last_active":"2023-09-23T20:42:04.559522+0000","last_peered":"2023-09-23T20:42:04.559522+0000","last_clean":"2023-09-23T20:42:04.559522+0000","last_became_active":"2023-09-23T20:41:58.902177+0000","last_became_peered":"2023-09-23T20:41:58.902177+0000","last_unstale":"2023-09-23T20:42:04.559522+0000","last_undegraded":"2023-09-23T20:42:04.559522+0000","last_fullsized":"2023-09-23T20:42:04.559522+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:00:00.624317+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":12,"reported_epoch":69,"state":"active+clean","last_fresh":"2023-09-23T20:42:04.559129+0000","last_change":"2023-09-23T20:41:58.920596+0000","last_active":"2023-09-23T20:42:04.559129+0000","last_peered":"2023-09-23T20:42:04.559129+0000","last_clean":"2023-09-23T20:42:04.559129+0000","last_became_active":"2023-09-23T20:41:58.920360+0000","last_became_peered":"2023-09-23T20:41:58.920360+0000","last_unstale":"2023-09-23T20:42:04.559129+0000","last_undegraded":"2023-09-23T20:42:04.559129+0000","last_fullsized":"2023-09-23T20:42:04.559129+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:36:30.646529+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2023-09-23T20:42:04.560949+0000","last_change":"2023-09-23T20:41:58.906865+0000","last_active":"2023-09-23T20:42:04.560949+0000","last_peered":"2023-09-23T20:42:04.560949+0000","last_clean":"2023-09-23T20:42:04.560949+0000","last_became_active":"2023-09-23T20:41:58.906505+0000","last_became_peered":"2023-09-23T20:41:58.906505+0000","last_unstale":"2023-09-23T20:42:04.560949+0000","last_undegraded":"2023-09-23T20:42:04.560949+0000","last_fullsized":"2023-09-23T20:42:04.560949+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:52:53.776236+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":12,"reported_epoch":69,"state":"active+clean","last_fresh":"2023-09-23T20:42:04.559994+0000","last_change":"2023-09-23T20:41:58.920103+0000","last_active":"2023-09-23T20:42:04.559994+0000","last_peered":"2023-09-23T20:42:04.559994+0000","last_clean":"2023-09-23T20:42:04.559994+0000","last_became_active":"2023-09-23T20:41:58.919764+0000","last_became_peered":"2023-09-23T20:41:58.919764+0000","last_unstale":"2023-09-23T20:42:04.559994+0000","last_undegraded":"2023-09-23T20:42:04.559994+0000","last_fullsized":"2023-09-23T20:42:04.559994+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:44:40.228355+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":15,"reported_epoch":69,"state":"active+clean","last_fresh":"2023-09-23T20:42:04.565221+0000","last_change":"2023-09-23T20:41:58.908699+0000","last_active":"2023-09-23T20:42:04.565221+0000","last_peered":"2023-09-23T20:42:04.565221+0000","last_clean":"2023-09-23T20:42:04.565221+0000","last_became_active":"2023-09-23T20:41:58.908315+0000","last_became_peered":"2023-09-23T20:41:58.908315+0000","last_unstale":"2023-09-23T20:42:04.565221+0000","last_undegraded":"2023-09-23T20:42:04.565221+0000","last_fullsized":"2023-09-23T20:42:04.565221+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:25:39.622409+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":22,"reported_epoch":70,"state":"active+clean","last_fresh":"2023-09-23T20:42:05.571127+0000","last_change":"2023-09-23T20:42:05.571127+0000","last_active":"2023-09-23T20:42:05.571127+0000","last_peered":"2023-09-23T20:42:05.571127+0000","last_clean":"2023-09-23T20:42:05.571127+0000","last_became_active":"2023-09-23T20:42:05.570777+0000","last_became_peered":"2023-09-23T20:42:05.570777+0000","last_unstale":"2023-09-23T20:42:05.571127+0000","last_undegraded":"2023-09-23T20:42:05.571127+0000","last_fullsized":"2023-09-23T20:42:05.571127+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:18:07.563540+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2023-09-23T20:42:04.564383+0000","last_change":"2023-09-23T20:41:58.903766+0000","last_active":"2023-09-23T20:42:04.564383+0000","last_peered":"2023-09-23T20:42:04.564383+0000","last_clean":"2023-09-23T20:42:04.564383+0000","last_became_active":"2023-09-23T20:41:58.902298+0000","last_became_peered":"2023-09-23T20:41:58.902298+0000","last_unstale":"2023-09-23T20:42:04.564383+0000","last_undegraded":"2023-09-23T20:42:04.564383+0000","last_fullsized":"2023-09-23T20:42:04.564383+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:56:10.559104+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":12,"reported_epoch":69,"state":"active+clean","last_fresh":"2023-09-23T20:42:04.560902+0000","last_change":"2023-09-23T20:41:59.060269+0000","last_active":"2023-09-23T20:42:04.560902+0000","last_peered":"2023-09-23T20:42:04.560902+0000","last_clean":"2023-09-23T20:42:04.560902+0000","last_became_active":"2023-09-23T20:41:59.059913+0000","last_became_peered":"2023-09-23T20:41:59.059913+0000","last_unstale":"2023-09-23T20:42:04.560902+0000","last_undegraded":"2023-09-23T20:42:04.560902+0000","last_fullsized":"2023-09-23T20:42:04.560902+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:46:45.365100+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":14,"reported_epoch":69,"state":"active+clean","last_fresh":"2023-09-23T20:42:04.558502+0000","last_change":"2023-09-23T20:41:58.898346+0000","last_active":"2023-09-23T20:42:04.558502+0000","last_peered":"2023-09-23T20:42:04.558502+0000","last_clean":"2023-09-23T20:42:04.558502+0000","last_became_active":"2023-09-23T20:41:58.898065+0000","last_became_peered":"2023-09-23T20:41:58.898065+0000","last_unstale":"2023-09-23T20:42:04.558502+0000","last_undegraded":"2023-09-23T20:42:04.558502+0000","last_fullsized":"2023-09-23T20:42:04.558502+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:51:57.491171+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":12,"reported_epoch":69,"state":"active+clean","last_fresh":"2023-09-23T20:42:04.559272+0000","last_change":"2023-09-23T20:41:58.919426+0000","last_active":"2023-09-23T20:42:04.559272+0000","last_peered":"2023-09-23T20:42:04.559272+0000","last_clean":"2023-09-23T20:42:04.559272+0000","last_became_active":"2023-09-23T20:41:58.919238+0000","last_became_peered":"2023-09-23T20:41:58.919238+0000","last_unstale":"2023-09-23T20:42:04.559272+0000","last_undegraded":"2023-09-23T20:42:04.559272+0000","last_fullsized":"2023-09-23T20:42:04.559272+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:33:14.594665+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":15,"reported_epoch":69,"state":"active+clean","last_fresh":"2023-09-23T20:42:04.558899+0000","last_change":"2023-09-23T20:41:58.894683+0000","last_active":"2023-09-23T20:42:04.558899+0000","last_peered":"2023-09-23T20:42:04.558899+0000","last_clean":"2023-09-23T20:42:04.558899+0000","last_became_active":"2023-09-23T20:41:58.894407+0000","last_became_peered":"2023-09-23T20:41:58.894407+0000","last_unstale":"2023-09-23T20:42:04.558899+0000","last_undegraded":"2023-09-23T20:42:04.558899+0000","last_fullsized":"2023-09-23T20:42:04.558899+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:41:38.552538+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":12,"reported_epoch":69,"state":"active+clean","last_fresh":"2023-09-23T20:42:04.564201+0000","last_change":"2023-09-23T20:41:59.061405+0000","last_active":"2023-09-23T20:42:04.564201+0000","last_peered":"2023-09-23T20:42:04.564201+0000","last_clean":"2023-09-23T20:42:04.564201+0000","last_became_active":"2023-09-23T20:41:59.061032+0000","last_became_peered":"2023-09-23T20:41:59.061032+0000","last_unstale":"2023-09-23T20:42:04.564201+0000","last_undegraded":"2023-09-23T20:42:04.564201+0000","last_fullsized":"2023-09-23T20:42:04.564201+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:43:09.716350+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":14,"reported_epoch":69,"state":"active+clean","last_fresh":"2023-09-23T20:42:04.558705+0000","last_change":"2023-09-23T20:41:58.895045+0000","last_active":"2023-09-23T20:42:04.558705+0000","last_peered":"2023-09-23T20:42:04.558705+0000","last_clean":"2023-09-23T20:42:04.558705+0000","last_became_active":"2023-09-23T20:41:58.894739+0000","last_became_peered":"2023-09-23T20:41:58.894739+0000","last_unstale":"2023-09-23T20:42:04.558705+0000","last_undegraded":"2023-09-23T20:42:04.558705+0000","last_fullsized":"2023-09-23T20:42:04.558705+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:51:39.089390+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":12,"reported_epoch":69,"state":"active+clean","last_fresh":"2023-09-23T20:42:04.565149+0000","last_change":"2023-09-23T20:41:59.060886+0000","last_active":"2023-09-23T20:42:04.565149+0000","last_peered":"2023-09-23T20:42:04.565149+0000","last_clean":"2023-09-23T20:42:04.565149+0000","last_became_active":"2023-09-23T20:41:59.059915+0000","last_became_peered":"2023-09-23T20:41:59.059915+0000","last_unstale":"2023-09-23T20:42:04.565149+0000","last_undegraded":"2023-09-23T20:42:04.565149+0000","last_fullsized":"2023-09-23T20:42:04.565149+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:03:29.688908+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":21,"reported_epoch":69,"state":"active+clean","last_fresh":"2023-09-23T20:42:04.561396+0000","last_change":"2023-09-23T20:41:58.900378+0000","last_active":"2023-09-23T20:42:04.561396+0000","last_peered":"2023-09-23T20:42:04.561396+0000","last_clean":"2023-09-23T20:42:04.561396+0000","last_became_active":"2023-09-23T20:41:58.900098+0000","last_became_peered":"2023-09-23T20:41:58.900098+0000","last_unstale":"2023-09-23T20:42:04.561396+0000","last_undegraded":"2023-09-23T20:42:04.561396+0000","last_fullsized":"2023-09-23T20:42:04.561396+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":8,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:32:49.020622+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":12,"reported_epoch":69,"state":"active+clean","last_fresh":"2023-09-23T20:42:04.563560+0000","last_change":"2023-09-23T20:41:59.060309+0000","last_active":"2023-09-23T20:42:04.563560+0000","last_peered":"2023-09-23T20:42:04.563560+0000","last_clean":"2023-09-23T20:42:04.563560+0000","last_became_active":"2023-09-23T20:41:59.059896+0000","last_became_peered":"2023-09-23T20:41:59.059896+0000","last_unstale":"2023-09-23T20:42:04.563560+0000","last_undegraded":"2023-09-23T20:42:04.563560+0000","last_fullsized":"2023-09-23T20:42:04.563560+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:12:55.804003+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2023-09-23T20:42:04.558325+0000","last_change":"2023-09-23T20:41:58.898224+0000","last_active":"2023-09-23T20:42:04.558325+0000","last_peered":"2023-09-23T20:42:04.558325+0000","last_clean":"2023-09-23T20:42:04.558325+0000","last_became_active":"2023-09-23T20:41:58.898036+0000","last_became_peered":"2023-09-23T20:41:58.898036+0000","last_unstale":"2023-09-23T20:42:04.558325+0000","last_undegraded":"2023-09-23T20:42:04.558325+0000","last_fullsized":"2023-09-23T20:42:04.558325+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:53:15.361560+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":12,"reported_epoch":69,"state":"active+clean","last_fresh":"2023-09-23T20:42:04.564426+0000","last_change":"2023-09-23T20:41:59.061779+0000","last_active":"2023-09-23T20:42:04.564426+0000","last_peered":"2023-09-23T20:42:04.564426+0000","last_clean":"2023-09-23T20:42:04.564426+0000","last_became_active":"2023-09-23T20:41:59.061503+0000","last_became_peered":"2023-09-23T20:41:59.061503+0000","last_unstale":"2023-09-23T20:42:04.564426+0000","last_undegraded":"2023-09-23T20:42:04.564426+0000","last_fullsized":"2023-09-23T20:42:04.564426+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:03:29.221377+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2023-09-23T20:42:04.563387+0000","last_change":"2023-09-23T20:41:58.899881+0000","last_active":"2023-09-23T20:42:04.563387+0000","last_peered":"2023-09-23T20:42:04.563387+0000","last_clean":"2023-09-23T20:42:04.563387+0000","last_became_active":"2023-09-23T20:41:58.899189+0000","last_became_peered":"2023-09-23T20:41:58.899189+0000","last_unstale":"2023-09-23T20:42:04.563387+0000","last_undegraded":"2023-09-23T20:42:04.563387+0000","last_fullsized":"2023-09-23T20:42:04.563387+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:01:03.685609+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":12,"reported_epoch":69,"state":"active+clean","last_fresh":"2023-09-23T20:42:04.565332+0000","last_change":"2023-09-23T20:41:59.060226+0000","last_active":"2023-09-23T20:42:04.565332+0000","last_peered":"2023-09-23T20:42:04.565332+0000","last_clean":"2023-09-23T20:42:04.565332+0000","last_became_active":"2023-09-23T20:41:59.059889+0000","last_became_peered":"2023-09-23T20:41:59.059889+0000","last_unstale":"2023-09-23T20:42:04.565332+0000","last_undegraded":"2023-09-23T20:42:04.565332+0000","last_fullsized":"2023-09-23T20:42:04.565332+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:36:53.062096+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":15,"reported_epoch":69,"state":"active+clean","last_fresh":"2023-09-23T20:42:04.559423+0000","last_change":"2023-09-23T20:41:58.897223+0000","last_active":"2023-09-23T20:42:04.559423+0000","last_peered":"2023-09-23T20:42:04.559423+0000","last_clean":"2023-09-23T20:42:04.559423+0000","last_became_active":"2023-09-23T20:41:58.896599+0000","last_became_peered":"2023-09-23T20:41:58.896599+0000","last_unstale":"2023-09-23T20:42:04.559423+0000","last_undegraded":"2023-09-23T20:42:04.559423+0000","last_fullsized":"2023-09-23T20:42:04.559423+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:44:37.392444+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":12,"reported_epoch":69,"state":"active+clean","last_fresh":"2023-09-23T20:42:04.560546+0000","last_change":"2023-09-23T20:41:58.921718+0000","last_active":"2023-09-23T20:42:04.560546+0000","last_peered":"2023-09-23T20:42:04.560546+0000","last_clean":"2023-09-23T20:42:04.560546+0000","last_became_active":"2023-09-23T20:41:58.921443+0000","last_became_peered":"2023-09-23T20:41:58.921443+0000","last_unstale":"2023-09-23T20:42:04.560546+0000","last_undegraded":"2023-09-23T20:42:04.560546+0000","last_fullsized":"2023-09-23T20:42:04.560546+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:10:39.135164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"68'6","reported_seq":19,"reported_epoch":69,"state":"active+clean","last_fresh":"2023-09-23T20:42:04.559924+0000","last_change":"2023-09-23T20:41:58.905866+0000","last_active":"2023-09-23T20:42:04.559924+0000","last_peered":"2023-09-23T20:42:04.559924+0000","last_clean":"2023-09-23T20:42:04.559924+0000","last_became_active":"2023-09-23T20:41:58.905556+0000","last_became_peered":"2023-09-23T20:41:58.905556+0000","last_unstale":"2023-09-23T20:42:04.559924+0000","last_undegraded":"2023-09-23T20:42:04.559924+0000","last_fullsized":"2023-09-23T20:42:04.559924+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":6,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:03:14.027348+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":1536,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6,"num_write_kb":3,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":12,"reported_epoch":69,"state":"active+clean","last_fresh":"2023-09-23T20:42:04.561452+0000","last_change":"2023-09-23T20:41:58.921966+0000","last_active":"2023-09-23T20:42:04.561452+0000","last_peered":"2023-09-23T20:42:04.561452+0000","last_clean":"2023-09-23T20:42:04.561452+0000","last_became_active":"2023-09-23T20:41:58.921649+0000","last_became_peered":"2023-09-23T20:41:58.921649+0000","last_unstale":"2023-09-23T20:42:04.561452+0000","last_undegraded":"2023-09-23T20:42:04.561452+0000","last_fullsized":"2023-09-23T20:42:04.561452+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:05:43.041172+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":16,"reported_epoch":69,"state":"active+clean","last_fresh":"2023-09-23T20:42:04.558364+0000","last_change":"2023-09-23T20:41:58.895940+0000","last_active":"2023-09-23T20:42:04.558364+0000","last_peered":"2023-09-23T20:42:04.558364+0000","last_clean":"2023-09-23T20:42:04.558364+0000","last_became_active":"2023-09-23T20:41:58.895688+0000","last_became_peered":"2023-09-23T20:41:58.895688+0000","last_unstale":"2023-09-23T20:42:04.558364+0000","last_undegraded":"2023-09-23T20:42:04.558364+0000","last_fullsized":"2023-09-23T20:42:04.558364+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":3,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:04:40.907902+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":12,"reported_epoch":69,"state":"active+clean","last_fresh":"2023-09-23T20:42:04.564996+0000","last_change":"2023-09-23T20:41:59.060781+0000","last_active":"2023-09-23T20:42:04.564996+0000","last_peered":"2023-09-23T20:42:04.564996+0000","last_clean":"2023-09-23T20:42:04.564996+0000","last_became_active":"2023-09-23T20:41:59.059897+0000","last_became_peered":"2023-09-23T20:41:59.059897+0000","last_unstale":"2023-09-23T20:42:04.564996+0000","last_undegraded":"2023-09-23T20:42:04.564996+0000","last_fullsized":"2023-09-23T20:42:04.564996+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:55:42.494186+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2023-09-23T20:42:04.564476+0000","last_change":"2023-09-23T20:41:58.901830+0000","last_active":"2023-09-23T20:42:04.564476+0000","last_peered":"2023-09-23T20:42:04.564476+0000","last_clean":"2023-09-23T20:42:04.564476+0000","last_became_active":"2023-09-23T20:41:58.901536+0000","last_became_peered":"2023-09-23T20:41:58.901536+0000","last_unstale":"2023-09-23T20:42:04.564476+0000","last_undegraded":"2023-09-23T20:42:04.564476+0000","last_fullsized":"2023-09-23T20:42:04.564476+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:07:53.066435+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":12,"reported_epoch":69,"state":"active+clean","last_fresh":"2023-09-23T20:42:04.564869+0000","last_change":"2023-09-23T20:41:58.923598+0000","last_active":"2023-09-23T20:42:04.564869+0000","last_peered":"2023-09-23T20:42:04.564869+0000","last_clean":"2023-09-23T20:42:04.564869+0000","last_became_active":"2023-09-23T20:41:58.923297+0000","last_became_peered":"2023-09-23T20:41:58.923297+0000","last_unstale":"2023-09-23T20:42:04.564869+0000","last_undegraded":"2023-09-23T20:42:04.564869+0000","last_fullsized":"2023-09-23T20:42:04.564869+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:57:10.155613+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2023-09-23T20:42:04.564008+0000","last_change":"2023-09-23T20:41:58.902008+0000","last_active":"2023-09-23T20:42:04.564008+0000","last_peered":"2023-09-23T20:42:04.564008+0000","last_clean":"2023-09-23T20:42:04.564008+0000","last_became_active":"2023-09-23T20:41:58.901717+0000","last_became_peered":"2023-09-23T20:41:58.901717+0000","last_unstale":"2023-09-23T20:42:04.564008+0000","last_undegraded":"2023-09-23T20:42:04.564008+0000","last_fullsized":"2023-09-23T20:42:04.564008+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:33:32.458651+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":12,"reported_epoch":69,"state":"active+clean","last_fresh":"2023-09-23T20:42:04.560991+0000","last_change":"2023-09-23T20:41:58.920656+0000","last_active":"2023-09-23T20:42:04.560991+0000","last_peered":"2023-09-23T20:42:04.560991+0000","last_clean":"2023-09-23T20:42:04.560991+0000","last_became_active":"2023-09-23T20:41:58.920420+0000","last_became_peered":"2023-09-23T20:41:58.920420+0000","last_unstale":"2023-09-23T20:42:04.560991+0000","last_undegraded":"2023-09-23T20:42:04.560991+0000","last_fullsized":"2023-09-23T20:42:04.560991+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:59:16.325436+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2023-09-23T20:42:04.564880+0000","last_change":"2023-09-23T20:41:58.901955+0000","last_active":"2023-09-23T20:42:04.564880+0000","last_peered":"2023-09-23T20:42:04.564880+0000","last_clean":"2023-09-23T20:42:04.564880+0000","last_became_active":"2023-09-23T20:41:58.901659+0000","last_became_peered":"2023-09-23T20:41:58.901659+0000","last_unstale":"2023-09-23T20:42:04.564880+0000","last_undegraded":"2023-09-23T20:42:04.564880+0000","last_fullsized":"2023-09-23T20:42:04.564880+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:16:06.225477+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":12,"reported_epoch":69,"state":"active+clean","last_fresh":"2023-09-23T20:42:04.563635+0000","last_change":"2023-09-23T20:41:59.061254+0000","last_active":"2023-09-23T20:42:04.563635+0000","last_peered":"2023-09-23T20:42:04.563635+0000","last_clean":"2023-09-23T20:42:04.563635+0000","last_became_active":"2023-09-23T20:41:59.060968+0000","last_became_peered":"2023-09-23T20:41:59.060968+0000","last_unstale":"2023-09-23T20:42:04.563635+0000","last_undegraded":"2023-09-23T20:42:04.563635+0000","last_fullsized":"2023-09-23T20:42:04.563635+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:39:11.919325+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2023-09-23T20:42:04.561166+0000","last_change":"2023-09-23T20:41:58.907143+0000","last_active":"2023-09-23T20:42:04.561166+0000","last_peered":"2023-09-23T20:42:04.561166+0000","last_clean":"2023-09-23T20:42:04.561166+0000","last_became_active":"2023-09-23T20:41:58.906851+0000","last_became_peered":"2023-09-23T20:41:58.906851+0000","last_unstale":"2023-09-23T20:42:04.561166+0000","last_undegraded":"2023-09-23T20:42:04.561166+0000","last_fullsized":"2023-09-23T20:42:04.561166+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:34:34.423958+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":12,"reported_epoch":69,"state":"active+clean","last_fresh":"2023-09-23T20:42:04.565240+0000","last_change":"2023-09-23T20:41:58.923643+0000","last_active":"2023-09-23T20:42:04.565240+0000","last_peered":"2023-09-23T20:42:04.565240+0000","last_clean":"2023-09-23T20:42:04.565240+0000","last_became_active":"2023-09-23T20:41:58.923348+0000","last_became_peered":"2023-09-23T20:41:58.923348+0000","last_unstale":"2023-09-23T20:42:04.565240+0000","last_undegraded":"2023-09-23T20:42:04.565240+0000","last_fullsized":"2023-09-23T20:42:04.565240+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:01:02.077218+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"68'1","reported_seq":14,"reported_epoch":69,"state":"active+clean","last_fresh":"2023-09-23T20:42:04.559605+0000","last_change":"2023-09-23T20:41:58.902573+0000","last_active":"2023-09-23T20:42:04.559605+0000","last_peered":"2023-09-23T20:42:04.559605+0000","last_clean":"2023-09-23T20:42:04.559605+0000","last_became_active":"2023-09-23T20:41:58.902320+0000","last_became_peered":"2023-09-23T20:41:58.902320+0000","last_unstale":"2023-09-23T20:42:04.559605+0000","last_undegraded":"2023-09-23T20:42:04.559605+0000","last_fullsized":"2023-09-23T20:42:04.559605+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:39:32.963286+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":2306,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":45,"num_write_kb":13,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":6918,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":39,"ondisk_log_size":39,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":128,"ondisk_log_size":128,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201304,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6620,"kb_used_data":1300,"kb_used_omap":0,"kb_used_meta":5312,"kb_avail":93738532,"statfs":{"total":95995035648,"available":95988256768,"internally_reserved":0,"allocated":1331200,"data_stored":901362,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5439488},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:41:40 2023","interfaces":[{"interface":"back","average":{"1min":0.54100000000000004,"5min":0.54100000000000004,"15min":0.54100000000000004},"min":{"1min":0.28999999999999998,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":0.85499999999999998,"5min":0.85499999999999998,"15min":0.85499999999999998},"last":0.51500000000000001},{"interface":"front","average":{"1min":0.54300000000000004,"5min":0.54300000000000004,"15min":0.54300000000000004},"min":{"1min":0.39000000000000001,"5min":0.39000000000000001,"15min":0.39000000000000001},"max":{"1min":0.79400000000000004,"5min":0.79400000000000004,"15min":0.79400000000000004},"last":0.505}]},{"osd":1,"last update":"Sat Sep 23 20:41:40 2023","interfaces":[{"interface":"back","average":{"1min":0.54400000000000004,"5min":0.54400000000000004,"15min":0.54400000000000004},"min":{"1min":0.27500000000000002,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":0.81499999999999995,"5min":0.81499999999999995,"15min":0.81499999999999995},"last":0.48599999999999999},{"interface":"front","average":{"1min":0.55000000000000004,"5min":0.55000000000000004,"15min":0.55000000000000004},"min":{"1min":0.35999999999999999,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":0.78300000000000003,"5min":0.78300000000000003,"15min":0.78300000000000003},"last":0.47299999999999998}]},{"osd":2,"last update":"Sat Sep 23 20:41:40 2023","interfaces":[{"interface":"back","average":{"1min":0.52500000000000002,"5min":0.52500000000000002,"15min":0.52500000000000002},"min":{"1min":0.32700000000000001,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":0.82699999999999996,"5min":0.82699999999999996,"15min":0.82699999999999996},"last":0.45700000000000002},{"interface":"front","average":{"1min":0.57599999999999996,"5min":0.57599999999999996,"15min":0.57599999999999996},"min":{"1min":0.309,"5min":0.309,"15min":0.309},"max":{"1min":0.93100000000000005,"5min":0.93100000000000005,"15min":0.93100000000000005},"last":0.58399999999999996}]},{"osd":3,"last update":"Sat Sep 23 20:41:40 2023","interfaces":[{"interface":"back","average":{"1min":0.55200000000000005,"5min":0.55200000000000005,"15min":0.55200000000000005},"min":{"1min":0.37,"5min":0.37,"15min":0.37},"max":{"1min":0.73399999999999999,"5min":0.73399999999999999,"15min":0.73399999999999999},"last":0.64300000000000002},{"interface":"front","average":{"1min":0.58799999999999997,"5min":0.58799999999999997,"15min":0.58799999999999997},"min":{"1min":0.35199999999999998,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":0.88800000000000001,"5min":0.88800000000000001,"15min":0.88800000000000001},"last":0.435}]},{"osd":4,"last update":"Sat Sep 23 20:41:40 2023","interfaces":[{"interface":"back","average":{"1min":0.47099999999999997,"5min":0.47099999999999997,"15min":0.47099999999999997},"min":{"1min":0.312,"5min":0.312,"15min":0.312},"max":{"1min":0.59499999999999997,"5min":0.59499999999999997,"15min":0.59499999999999997},"last":0.40899999999999997},{"interface":"front","average":{"1min":0.51300000000000001,"5min":0.51300000000000001,"15min":0.51300000000000001},"min":{"1min":0.36699999999999999,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":0.79500000000000004,"5min":0.79500000000000004,"15min":0.79500000000000004},"last":0.55600000000000005}]},{"osd":5,"last update":"Sat Sep 23 20:41:40 2023","interfaces":[{"interface":"back","average":{"1min":0.53000000000000003,"5min":0.53000000000000003,"15min":0.53000000000000003},"min":{"1min":0.33600000000000002,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":0.77800000000000002,"5min":0.77800000000000002,"15min":0.77800000000000002},"last":0.53200000000000003},{"interface":"front","average":{"1min":0.55000000000000004,"5min":0.55000000000000004,"15min":0.55000000000000004},"min":{"1min":0.35999999999999999,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":0.78500000000000003,"5min":0.78500000000000003,"15min":0.78500000000000003},"last":0.59599999999999997}]},{"osd":6,"last update":"Sat Sep 23 20:41:40 2023","interfaces":[{"interface":"back","average":{"1min":0.56200000000000006,"5min":0.56200000000000006,"15min":0.56200000000000006},"min":{"1min":0.38,"5min":0.38,"15min":0.38},"max":{"1min":0.874,"5min":0.874,"15min":0.874},"last":0.54600000000000004},{"interface":"front","average":{"1min":0.58899999999999997,"5min":0.58899999999999997,"15min":0.58899999999999997},"min":{"1min":0.316,"5min":0.316,"15min":0.316},"max":{"1min":0.85099999999999998,"5min":0.85099999999999998,"15min":0.85099999999999998},"last":0.61299999999999999}]}]},{"osd":6,"up_from":47,"seq":201863462941,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6668,"kb_used_data":1284,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":93738484,"statfs":{"total":95995035648,"available":95988207616,"internally_reserved":0,"allocated":1314816,"data_stored":900951,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:42:23 2023","interfaces":[{"interface":"back","average":{"1min":0.48099999999999998,"5min":0.505,"15min":0.50900000000000001},"min":{"1min":0.28399999999999997,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":0.80200000000000005,"5min":0.80200000000000005,"15min":0.80200000000000005},"last":0.57099999999999995},{"interface":"front","average":{"1min":0.51500000000000001,"5min":0.52700000000000002,"15min":0.52900000000000003},"min":{"1min":0.35599999999999998,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":1.206,"5min":1.206,"15min":1.206},"last":0.41699999999999998}]},{"osd":1,"last update":"Sat Sep 23 20:42:23 2023","interfaces":[{"interface":"back","average":{"1min":0.52500000000000002,"5min":0.54600000000000004,"15min":0.55000000000000004},"min":{"1min":0.32300000000000001,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":1.1759999999999999,"5min":1.1759999999999999,"15min":1.1759999999999999},"last":0.53000000000000003},{"interface":"front","average":{"1min":0.50600000000000001,"5min":0.52100000000000002,"15min":0.52300000000000002},"min":{"1min":0.308,"5min":0.308,"15min":0.308},"max":{"1min":0.90400000000000003,"5min":0.90400000000000003,"15min":0.90400000000000003},"last":0.38400000000000001}]},{"osd":2,"last update":"Sat Sep 23 20:42:23 2023","interfaces":[{"interface":"back","average":{"1min":0.51200000000000001,"5min":0.50900000000000001,"15min":0.50900000000000001},"min":{"1min":0.39400000000000002,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":0.84399999999999997,"5min":0.84399999999999997,"15min":0.84399999999999997},"last":0.44400000000000001},{"interface":"front","average":{"1min":0.54900000000000004,"5min":0.54700000000000004,"15min":0.54700000000000004},"min":{"1min":0.36599999999999999,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":1.1499999999999999,"5min":1.1499999999999999,"15min":1.1499999999999999},"last":0.70499999999999996}]},{"osd":3,"last update":"Sat Sep 23 20:42:23 2023","interfaces":[{"interface":"back","average":{"1min":0.57399999999999995,"5min":0.54500000000000004,"15min":0.54000000000000004},"min":{"1min":0.38600000000000001,"5min":0.38,"15min":0.38},"max":{"1min":0.96099999999999997,"5min":0.96099999999999997,"15min":0.96099999999999997},"last":0.48999999999999999},{"interface":"front","average":{"1min":0.53600000000000003,"5min":0.55300000000000005,"15min":0.55600000000000005},"min":{"1min":0.36399999999999999,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":0.874,"5min":0.874,"15min":0.874},"last":0.501}]},{"osd":4,"last update":"Sat Sep 23 20:42:23 2023","interfaces":[{"interface":"back","average":{"1min":0.47599999999999998,"5min":0.438,"15min":0.432},"min":{"1min":0.28799999999999998,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":1.1779999999999999,"5min":1.1779999999999999,"15min":1.1779999999999999},"last":0.45700000000000002},{"interface":"front","average":{"1min":0.46100000000000002,"5min":0.47499999999999998,"15min":0.47699999999999998},"min":{"1min":0.313,"5min":0.313,"15min":0.313},"max":{"1min":0.72899999999999998,"5min":0.72899999999999998,"15min":0.72899999999999998},"last":0.64500000000000002}]},{"osd":5,"last update":"Sat Sep 23 20:42:23 2023","interfaces":[{"interface":"back","average":{"1min":0.48199999999999998,"5min":0.46400000000000002,"15min":0.46100000000000002},"min":{"1min":0.32900000000000001,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":1.081,"5min":1.081,"15min":1.081},"last":0.78500000000000003},{"interface":"front","average":{"1min":0.51100000000000001,"5min":0.497,"15min":0.495},"min":{"1min":0.31,"5min":0.31,"15min":0.31},"max":{"1min":0.89000000000000001,"5min":0.89000000000000001,"15min":0.89000000000000001},"last":0.61299999999999999}]},{"osd":7,"last update":"Sat Sep 23 20:41:38 2023","interfaces":[{"interface":"back","average":{"1min":0.50600000000000001,"5min":0.50600000000000001,"15min":0.50600000000000001},"min":{"1min":0.32500000000000001,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":0.70199999999999996,"5min":0.70199999999999996,"15min":0.70199999999999996},"last":0.752},{"interface":"front","average":{"1min":0.55900000000000005,"5min":0.55900000000000005,"15min":0.55900000000000005},"min":{"1min":0.41599999999999998,"5min":0.41599999999999998,"15min":0.41599999999999998},"max":{"1min":0.77400000000000002,"5min":0.77400000000000002,"15min":0.77400000000000002},"last":0.67900000000000005}]}]},{"osd":1,"up_from":13,"seq":55834574898,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6848,"kb_used_data":696,"kb_used_omap":0,"kb_used_meta":6144,"kb_avail":93738304,"statfs":{"total":95995035648,"available":95988023296,"internally_reserved":0,"allocated":712704,"data_stored":309565,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":6291456},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:41:38 2023","interfaces":[{"interface":"back","average":{"1min":0.43099999999999999,"5min":0.40999999999999998,"15min":0.40400000000000003},"min":{"1min":0.27300000000000002,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":0.94499999999999995,"5min":0.96499999999999997,"15min":0.96499999999999997},"last":0.317},{"interface":"front","average":{"1min":0.437,"5min":0.442,"15min":0.439},"min":{"1min":0.23999999999999999,"5min":0.218,"15min":0.218},"max":{"1min":1.2549999999999999,"5min":1.2549999999999999,"15min":1.2549999999999999},"last":0.34000000000000002}]},{"osd":2,"last update":"Sat Sep 23 20:42:01 2023","interfaces":[{"interface":"back","average":{"1min":0.45200000000000001,"5min":0.46899999999999997,"15min":0.46500000000000002},"min":{"1min":0.27300000000000002,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":1.544,"5min":1.544,"15min":1.544},"last":0.378},{"interface":"front","average":{"1min":0.39100000000000001,"5min":0.41699999999999998,"15min":0.41599999999999998},"min":{"1min":0.26200000000000001,"5min":0.254,"15min":0.254},"max":{"1min":0.66000000000000003,"5min":0.748,"15min":0.748},"last":0.26800000000000002}]},{"osd":3,"last update":"Sat Sep 23 20:42:24 2023","interfaces":[{"interface":"back","average":{"1min":0.498,"5min":0.46200000000000002,"15min":0.46600000000000003},"min":{"1min":0.30099999999999999,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":1.8320000000000001,"5min":1.8320000000000001,"15min":1.8320000000000001},"last":0.29399999999999998},{"interface":"front","average":{"1min":0.47499999999999998,"5min":0.48899999999999999,"15min":0.498},"min":{"1min":0.312,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":1.2929999999999999,"5min":1.2929999999999999,"15min":1.2929999999999999},"last":0.57399999999999995}]},{"osd":4,"last update":"Sat Sep 23 20:41:38 2023","interfaces":[{"interface":"back","average":{"1min":0.63300000000000001,"5min":0.58899999999999997,"15min":0.58199999999999996},"min":{"1min":0.38300000000000001,"5min":0.38300000000000001,"15min":0.38300000000000001},"max":{"1min":0.90600000000000003,"5min":0.98699999999999999,"15min":0.98699999999999999},"last":0.54000000000000004},{"interface":"front","average":{"1min":0.70399999999999996,"5min":0.63600000000000001,"15min":0.624},"min":{"1min":0.45000000000000001,"5min":0.40000000000000002,"15min":0.40000000000000002},"max":{"1min":1.5760000000000001,"5min":1.5760000000000001,"15min":1.5760000000000001},"last":0.55600000000000005}]},{"osd":5,"last update":"Sat Sep 23 20:42:01 2023","interfaces":[{"interface":"back","average":{"1min":0.63800000000000001,"5min":0.64100000000000001,"15min":0.64100000000000001},"min":{"1min":0.379,"5min":0.379,"15min":0.379},"max":{"1min":1.516,"5min":1.516,"15min":1.516},"last":0.432},{"interface":"front","average":{"1min":0.61299999999999999,"5min":0.63,"15min":0.63300000000000001},"min":{"1min":0.371,"5min":0.371,"15min":0.371},"max":{"1min":0.91700000000000004,"5min":0.91700000000000004,"15min":0.91700000000000004},"last":0.45900000000000002}]},{"osd":6,"last update":"Sat Sep 23 20:42:20 2023","interfaces":[{"interface":"back","average":{"1min":0.623,"5min":0.61499999999999999,"15min":0.61399999999999999},"min":{"1min":0.40400000000000003,"5min":0.40400000000000003,"15min":0.40400000000000003},"max":{"1min":0.89200000000000002,"5min":0.92000000000000004,"15min":0.92000000000000004},"last":0.497},{"interface":"front","average":{"1min":0.65400000000000003,"5min":0.65600000000000003,"15min":0.65600000000000003},"min":{"1min":0.45100000000000001,"5min":0.41399999999999998,"15min":0.41399999999999998},"max":{"1min":1.4850000000000001,"5min":1.4850000000000001,"15min":1.4850000000000001},"last":0.40999999999999998}]},{"osd":7,"last update":"Sat Sep 23 20:41:40 2023","interfaces":[{"interface":"back","average":{"1min":0.69899999999999995,"5min":0.69899999999999995,"15min":0.69899999999999995},"min":{"1min":0.439,"5min":0.439,"15min":0.439},"max":{"1min":1.45,"5min":1.45,"15min":1.45},"last":0.52600000000000002},{"interface":"front","average":{"1min":0.65800000000000003,"5min":0.65800000000000003,"15min":0.65800000000000003},"min":{"1min":0.442,"5min":0.442,"15min":0.442},"max":{"1min":0.86199999999999999,"5min":0.86199999999999999,"15min":0.86199999999999999},"last":0.51400000000000001}]}]},{"osd":0,"up_from":8,"seq":34359738423,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":7432,"kb_used_data":1280,"kb_used_omap":0,"kb_used_meta":6144,"kb_avail":93737720,"statfs":{"total":95995035648,"available":95987425280,"internally_reserved":0,"allocated":1310720,"data_stored":900244,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":6291456},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 20:41:40 2023","interfaces":[{"interface":"back","average":{"1min":0.34499999999999997,"5min":0.35899999999999999,"15min":0.34399999999999997},"min":{"1min":0.26300000000000001,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":0.51000000000000001,"5min":0.65900000000000003,"15min":0.65900000000000003},"last":0.36899999999999999},{"interface":"front","average":{"1min":0.39500000000000002,"5min":0.36399999999999999,"15min":0.34699999999999998},"min":{"1min":0.26300000000000001,"5min":0.22,"15min":0.22},"max":{"1min":0.55700000000000005,"5min":0.623,"15min":0.623},"last":0.29199999999999998}]},{"osd":2,"last update":"Sat Sep 23 20:42:02 2023","interfaces":[{"interface":"back","average":{"1min":0.36199999999999999,"5min":0.41399999999999998,"15min":0.41799999999999998},"min":{"1min":0.23599999999999999,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":0.45300000000000001,"5min":0.66400000000000003,"15min":0.66400000000000003},"last":0.33100000000000002},{"interface":"front","average":{"1min":0.39100000000000001,"5min":0.41899999999999998,"15min":0.41699999999999998},"min":{"1min":0.26100000000000001,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":0.54100000000000004,"5min":0.81000000000000005,"15min":0.81000000000000005},"last":0.46999999999999997}]},{"osd":3,"last update":"Sat Sep 23 20:42:26 2023","interfaces":[{"interface":"back","average":{"1min":0.496,"5min":0.48499999999999999,"15min":0.48899999999999999},"min":{"1min":0.29899999999999999,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":1.6899999999999999,"5min":1.6899999999999999,"15min":1.6899999999999999},"last":0.41699999999999998},{"interface":"front","average":{"1min":0.433,"5min":0.45300000000000001,"15min":0.46400000000000002},"min":{"1min":0.28899999999999998,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":0.95299999999999996,"5min":0.95299999999999996,"15min":0.95299999999999996},"last":0.35699999999999998}]},{"osd":4,"last update":"Sat Sep 23 20:41:40 2023","interfaces":[{"interface":"back","average":{"1min":0.60899999999999999,"5min":0.61099999999999999,"15min":0.61099999999999999},"min":{"1min":0.43099999999999999,"5min":0.42299999999999999,"15min":0.42299999999999999},"max":{"1min":1.734,"5min":1.734,"15min":1.734},"last":0.60699999999999998},{"interface":"front","average":{"1min":0.61899999999999999,"5min":0.64300000000000002,"15min":0.64700000000000002},"min":{"1min":0.42099999999999999,"5min":0.42099999999999999,"15min":0.42099999999999999},"max":{"1min":1.7170000000000001,"5min":1.7170000000000001,"15min":1.7170000000000001},"last":0.752}]},{"osd":5,"last update":"Sat Sep 23 20:42:02 2023","interfaces":[{"interface":"back","average":{"1min":0.60999999999999999,"5min":0.60399999999999998,"15min":0.60299999999999998},"min":{"1min":0.39800000000000002,"5min":0.39800000000000002,"15min":0.39800000000000002},"max":{"1min":1.6890000000000001,"5min":1.6890000000000001,"15min":1.6890000000000001},"last":0.68000000000000005},{"interface":"front","average":{"1min":0.622,"5min":0.63800000000000001,"15min":0.64100000000000001},"min":{"1min":0.38300000000000001,"5min":0.38300000000000001,"15min":0.38300000000000001},"max":{"1min":1.9390000000000001,"5min":1.9390000000000001,"15min":1.9390000000000001},"last":0.73599999999999999}]},{"osd":6,"last update":"Sat Sep 23 20:42:20 2023","interfaces":[{"interface":"back","average":{"1min":0.66300000000000003,"5min":0.59899999999999998,"15min":0.58799999999999997},"min":{"1min":0.39100000000000001,"5min":0.39100000000000001,"15min":0.39100000000000001},"max":{"1min":1.752,"5min":1.752,"15min":1.752},"last":0.70599999999999996},{"interface":"front","average":{"1min":0.71599999999999997,"5min":0.61899999999999999,"15min":0.60299999999999998},"min":{"1min":0.39400000000000002,"5min":0.39400000000000002,"15min":0.39400000000000002},"max":{"1min":2.0009999999999999,"5min":2.0009999999999999,"15min":2.0009999999999999},"last":0.78000000000000003}]},{"osd":7,"last update":"Sat Sep 23 20:41:40 2023","interfaces":[{"interface":"back","average":{"1min":0.626,"5min":0.626,"15min":0.626},"min":{"1min":0.38600000000000001,"5min":0.38600000000000001,"15min":0.38600000000000001},"max":{"1min":1.9890000000000001,"5min":1.9890000000000001,"15min":1.9890000000000001},"last":0.58399999999999996},{"interface":"front","average":{"1min":0.67900000000000005,"5min":0.67900000000000005,"15min":0.67900000000000005},"min":{"1min":0.41499999999999998,"5min":0.41499999999999998,"15min":0.41499999999999998},"max":{"1min":2.0169999999999999,"5min":2.0169999999999999,"15min":2.0169999999999999},"last":0.72399999999999998}]}]},{"osd":2,"up_from":18,"seq":77309411373,"num_pgs":19,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6664,"kb_used_data":704,"kb_used_omap":0,"kb_used_meta":5952,"kb_avail":93738488,"statfs":{"total":95995035648,"available":95988211712,"internally_reserved":0,"allocated":720896,"data_stored":310153,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":6094848},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:42:02 2023","interfaces":[{"interface":"back","average":{"1min":0.46300000000000002,"5min":0.41099999999999998,"15min":0.39900000000000002},"min":{"1min":0.25900000000000001,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":2.4350000000000001,"5min":2.4350000000000001,"15min":2.4350000000000001},"last":0.34699999999999998},{"interface":"front","average":{"1min":1.026,"5min":0.55000000000000004,"15min":0.46200000000000002},"min":{"1min":0.255,"5min":0.253,"15min":0.253},"max":{"1min":13.180999999999999,"5min":13.180999999999999,"15min":13.180999999999999},"last":0.40699999999999997}]},{"osd":1,"last update":"Sat Sep 23 20:42:02 2023","interfaces":[{"interface":"back","average":{"1min":0.53800000000000003,"5min":0.45800000000000002,"15min":0.44},"min":{"1min":0.26700000000000002,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":2.9049999999999998,"5min":2.9049999999999998,"15min":2.9049999999999998},"last":0.441},{"interface":"front","average":{"1min":0.51100000000000001,"5min":0.47199999999999998,"15min":0.46100000000000002},"min":{"1min":0.29899999999999999,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":2.4609999999999999,"5min":2.4609999999999999,"15min":2.4609999999999999},"last":0.38900000000000001}]},{"osd":3,"last update":"Sat Sep 23 20:42:22 2023","interfaces":[{"interface":"back","average":{"1min":1.05,"5min":0.60999999999999999,"15min":0.54200000000000004},"min":{"1min":0.27600000000000002,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":13.239000000000001,"5min":13.239000000000001,"15min":13.239000000000001},"last":0.52700000000000002},{"interface":"front","average":{"1min":0.52200000000000002,"5min":0.52100000000000002,"15min":0.53700000000000003},"min":{"1min":0.29299999999999998,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":3.0110000000000001,"5min":3.0110000000000001,"15min":3.0110000000000001},"last":0.498}]},{"osd":4,"last update":"Sat Sep 23 20:41:40 2023","interfaces":[{"interface":"back","average":{"1min":0.65100000000000002,"5min":0.66300000000000003,"15min":0.66500000000000004},"min":{"1min":0.39500000000000002,"5min":0.39500000000000002,"15min":0.39500000000000002},"max":{"1min":2.327,"5min":2.327,"15min":2.327},"last":0.48199999999999998},{"interface":"front","average":{"1min":0.67400000000000004,"5min":0.68400000000000005,"15min":0.68600000000000005},"min":{"1min":0.38800000000000001,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":2.9809999999999999,"5min":2.9809999999999999,"15min":2.9809999999999999},"last":0.57199999999999995}]},{"osd":5,"last update":"Sat Sep 23 20:42:02 2023","interfaces":[{"interface":"back","average":{"1min":0.68700000000000006,"5min":0.627,"15min":0.61699999999999999},"min":{"1min":0.432,"5min":0.432,"15min":0.432},"max":{"1min":2.948,"5min":2.948,"15min":2.948},"last":0.51900000000000002},{"interface":"front","average":{"1min":0.67200000000000004,"5min":0.624,"15min":0.61599999999999999},"min":{"1min":0.379,"5min":0.379,"15min":0.379},"max":{"1min":2.2280000000000002,"5min":2.2280000000000002,"15min":2.2280000000000002},"last":0.63600000000000001}]},{"osd":6,"last update":"Sat Sep 23 20:42:22 2023","interfaces":[{"interface":"back","average":{"1min":0.67300000000000004,"5min":0.61499999999999999,"15min":0.60499999999999998},"min":{"1min":0.41599999999999998,"5min":0.41599999999999998,"15min":0.41599999999999998},"max":{"1min":2.2989999999999999,"5min":2.2989999999999999,"15min":2.2989999999999999},"last":0.58999999999999997},{"interface":"front","average":{"1min":0.69999999999999996,"5min":0.66400000000000003,"15min":0.65800000000000003},"min":{"1min":0.44500000000000001,"5min":0.443,"15min":0.443},"max":{"1min":2.407,"5min":2.407,"15min":2.407},"last":0.58099999999999996}]},{"osd":7,"last update":"Sat Sep 23 20:41:40 2023","interfaces":[{"interface":"back","average":{"1min":0.69499999999999995,"5min":0.69499999999999995,"15min":0.69499999999999995},"min":{"1min":0.41999999999999998,"5min":0.41999999999999998,"15min":0.41999999999999998},"max":{"1min":2.3700000000000001,"5min":2.3700000000000001,"15min":2.3700000000000001},"last":0.50700000000000001},{"interface":"front","average":{"1min":0.70399999999999996,"5min":0.70399999999999996,"15min":0.70399999999999996},"min":{"1min":0.46400000000000002,"5min":0.46400000000000002,"15min":0.46400000000000002},"max":{"1min":2.27,"5min":2.27,"15min":2.27},"last":0.61699999999999999}]}]},{"osd":3,"up_from":26,"seq":111669149737,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6148,"kb_used_data":700,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":93739004,"statfs":{"total":95995035648,"available":95988740096,"internally_reserved":0,"allocated":716800,"data_stored":309655,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:42:23 2023","interfaces":[{"interface":"back","average":{"1min":0.36699999999999999,"5min":0.40699999999999997,"15min":0.41999999999999998},"min":{"1min":0.23699999999999999,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":0.625,"5min":0.79100000000000004,"15min":0.79100000000000004},"last":0.28999999999999998},{"interface":"front","average":{"1min":0.46200000000000002,"5min":0.48099999999999998,"15min":0.503},"min":{"1min":0.26000000000000001,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":1.7070000000000001,"5min":1.7070000000000001,"15min":1.7070000000000001},"last":0.33800000000000002}]},{"osd":1,"last update":"Sat Sep 23 20:42:23 2023","interfaces":[{"interface":"back","average":{"1min":0.442,"5min":0.47199999999999998,"15min":0.48799999999999999},"min":{"1min":0.27400000000000002,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":1.724,"5min":1.724,"15min":1.724},"last":0.27200000000000002},{"interface":"front","average":{"1min":0.41399999999999998,"5min":0.48999999999999999,"15min":0.51400000000000001},"min":{"1min":0.26200000000000001,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":0.72099999999999997,"5min":0.72899999999999998,"15min":0.72899999999999998},"last":0.40300000000000002}]},{"osd":2,"last update":"Sat Sep 23 20:42:23 2023","interfaces":[{"interface":"back","average":{"1min":0.39900000000000002,"5min":0.501,"15min":0.52900000000000003},"min":{"1min":0.307,"5min":0.307,"15min":0.307},"max":{"1min":0.79700000000000004,"5min":0.79800000000000004,"15min":0.79800000000000004},"last":0.35599999999999998},{"interface":"front","average":{"1min":0.41399999999999998,"5min":0.48799999999999999,"15min":0.51300000000000001},"min":{"1min":0.30599999999999999,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":0.77400000000000002,"5min":0.77400000000000002,"15min":0.77400000000000002},"last":0.33000000000000002}]},{"osd":4,"last update":"Sat Sep 23 20:41:41 2023","interfaces":[{"interface":"back","average":{"1min":0.59499999999999997,"5min":0.65200000000000002,"15min":0.66200000000000003},"min":{"1min":0.36699999999999999,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":0.88,"5min":0.88100000000000001,"15min":0.88100000000000001},"last":0.68400000000000005},{"interface":"front","average":{"1min":0.60699999999999998,"5min":0.64200000000000002,"15min":0.64800000000000002},"min":{"1min":0.40699999999999997,"5min":0.40699999999999997,"15min":0.40699999999999997},"max":{"1min":0.90900000000000003,"5min":0.90900000000000003,"15min":0.90900000000000003},"last":0.56299999999999994}]},{"osd":5,"last update":"Sat Sep 23 20:42:03 2023","interfaces":[{"interface":"back","average":{"1min":0.53700000000000003,"5min":0.63100000000000001,"15min":0.64700000000000002},"min":{"1min":0.28299999999999997,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":0.84099999999999997,"5min":0.97499999999999998,"15min":0.97499999999999998},"last":0.67000000000000004},{"interface":"front","average":{"1min":0.56699999999999995,"5min":0.65800000000000003,"15min":0.67300000000000004},"min":{"1min":0.38600000000000001,"5min":0.38600000000000001,"15min":0.38600000000000001},"max":{"1min":0.83499999999999996,"5min":0.91500000000000004,"15min":0.91500000000000004},"last":0.72199999999999998}]},{"osd":6,"last update":"Sat Sep 23 20:42:23 2023","interfaces":[{"interface":"back","average":{"1min":0.58599999999999997,"5min":0.65000000000000002,"15min":0.66000000000000003},"min":{"1min":0.47099999999999997,"5min":0.39600000000000002,"15min":0.39600000000000002},"max":{"1min":0.82499999999999996,"5min":0.97299999999999998,"15min":0.97299999999999998},"last":0.70299999999999996},{"interface":"front","average":{"1min":0.58299999999999996,"5min":0.625,"15min":0.63200000000000001},"min":{"1min":0.41399999999999998,"5min":0.38400000000000001,"15min":0.38400000000000001},"max":{"1min":0.85099999999999998,"5min":0.92700000000000005,"15min":0.92700000000000005},"last":0.63700000000000001}]},{"osd":7,"last update":"Sat Sep 23 20:41:41 2023","interfaces":[{"interface":"back","average":{"1min":0.66700000000000004,"5min":0.66700000000000004,"15min":0.66700000000000004},"min":{"1min":0.47599999999999998,"5min":0.47599999999999998,"15min":0.47599999999999998},"max":{"1min":0.93700000000000006,"5min":0.93700000000000006,"15min":0.93700000000000006},"last":0.623},{"interface":"front","average":{"1min":0.65300000000000002,"5min":0.65300000000000002,"15min":0.65300000000000002},"min":{"1min":0.40300000000000002,"5min":0.40300000000000002,"15min":0.40300000000000002},"max":{"1min":1.0049999999999999,"5min":1.0049999999999999,"15min":1.0049999999999999},"last":0.73299999999999998}]}]},{"osd":4,"up_from":31,"seq":133143986213,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6232,"kb_used_data":720,"kb_used_omap":0,"kb_used_meta":5504,"kb_avail":93738920,"statfs":{"total":95995035648,"available":95988654080,"internally_reserved":0,"allocated":737280,"data_stored":311247,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5636096},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:41:39 2023","interfaces":[{"interface":"back","average":{"1min":0.46500000000000002,"5min":0.52100000000000002,"15min":0.53000000000000003},"min":{"1min":0.315,"5min":0.31,"15min":0.31},"max":{"1min":0.72599999999999998,"5min":0.996,"15min":0.996},"last":0.41599999999999998},{"interface":"front","average":{"1min":0.57099999999999995,"5min":0.54600000000000004,"15min":0.54200000000000004},"min":{"1min":0.42399999999999999,"5min":0.41199999999999998,"15min":0.41199999999999998},"max":{"1min":0.77100000000000002,"5min":0.871,"15min":0.871},"last":0.58299999999999996}]},{"osd":1,"last update":"Sat Sep 23 20:41:39 2023","interfaces":[{"interface":"back","average":{"1min":0.503,"5min":0.53900000000000003,"15min":0.54500000000000004},"min":{"1min":0.33200000000000002,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":0.72199999999999998,"5min":0.88900000000000001,"15min":0.88900000000000001},"last":0.46600000000000003},{"interface":"front","average":{"1min":0.53500000000000003,"5min":0.52000000000000002,"15min":0.51800000000000002},"min":{"1min":0.32700000000000001,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":0.79600000000000004,"5min":1.0289999999999999,"15min":1.0289999999999999},"last":0.36299999999999999}]},{"osd":2,"last update":"Sat Sep 23 20:41:39 2023","interfaces":[{"interface":"back","average":{"1min":0.53600000000000003,"5min":0.55400000000000005,"15min":0.55700000000000005},"min":{"1min":0.33700000000000002,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":0.79700000000000004,"5min":1.0049999999999999,"15min":1.0049999999999999},"last":0.49299999999999999},{"interface":"front","average":{"1min":0.53100000000000003,"5min":0.53400000000000003,"15min":0.53400000000000003},"min":{"1min":0.40200000000000002,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":0.81100000000000005,"5min":0.85399999999999998,"15min":0.85399999999999998},"last":0.51300000000000001}]},{"osd":3,"last update":"Sat Sep 23 20:41:39 2023","interfaces":[{"interface":"back","average":{"1min":0.53900000000000003,"5min":0.56399999999999995,"15min":0.56799999999999995},"min":{"1min":0.35799999999999998,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":0.82999999999999996,"5min":0.90100000000000002,"15min":0.90100000000000002},"last":0.59799999999999998},{"interface":"front","average":{"1min":0.55000000000000004,"5min":0.55400000000000005,"15min":0.55500000000000005},"min":{"1min":0.38700000000000001,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":0.76900000000000002,"5min":0.91500000000000004,"15min":0.91500000000000004},"last":0.56200000000000006}]},{"osd":5,"last update":"Sat Sep 23 20:42:03 2023","interfaces":[{"interface":"back","average":{"1min":0.52900000000000003,"5min":0.41199999999999998,"15min":0.39200000000000002},"min":{"1min":0.35099999999999998,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":0.83599999999999997,"5min":0.83599999999999997,"15min":0.83599999999999997},"last":0.61599999999999999},{"interface":"front","average":{"1min":0.52600000000000002,"5min":0.41699999999999998,"15min":0.39900000000000002},"min":{"1min":0.35799999999999998,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":0.90000000000000002,"5min":0.90000000000000002,"15min":0.90000000000000002},"last":0.63800000000000001}]},{"osd":6,"last update":"Sat Sep 23 20:42:22 2023","interfaces":[{"interface":"back","average":{"1min":0.47999999999999998,"5min":0.51800000000000002,"15min":0.52400000000000002},"min":{"1min":0.33600000000000002,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":0.80700000000000005,"5min":1.1319999999999999,"15min":1.1319999999999999},"last":0.629},{"interface":"front","average":{"1min":0.56599999999999995,"5min":0.57299999999999995,"15min":0.57399999999999995},"min":{"1min":0.35399999999999998,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":0.85399999999999998,"5min":1.095,"15min":1.095},"last":0.67100000000000004}]},{"osd":7,"last update":"Sat Sep 23 20:41:39 2023","interfaces":[{"interface":"back","average":{"1min":0.55500000000000005,"5min":0.55500000000000005,"15min":0.55500000000000005},"min":{"1min":0.32700000000000001,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":0.77700000000000002,"5min":0.77700000000000002,"15min":0.77700000000000002},"last":0.65100000000000002},{"interface":"front","average":{"1min":0.57099999999999995,"5min":0.57099999999999995,"15min":0.57099999999999995},"min":{"1min":0.32700000000000001,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":0.86299999999999999,"5min":0.86299999999999999,"15min":0.86299999999999999},"last":0.68400000000000005}]}]},{"osd":5,"up_from":39,"seq":167503724578,"num_pgs":21,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6088,"kb_used_data":704,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":93739064,"statfs":{"total":95995035648,"available":95988801536,"internally_reserved":0,"allocated":720896,"data_stored":310189,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:42:00 2023","interfaces":[{"interface":"back","average":{"1min":0.46800000000000003,"5min":0.52500000000000002,"15min":0.53500000000000003},"min":{"1min":0.29199999999999998,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":0.63100000000000001,"5min":0.82299999999999995,"15min":0.82299999999999995},"last":0.46899999999999997},{"interface":"front","average":{"1min":0.46700000000000003,"5min":0.54400000000000004,"15min":0.55700000000000005},"min":{"1min":0.30499999999999999,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":0.78300000000000003,"5min":0.79600000000000004,"15min":0.79600000000000004},"last":0.48599999999999999}]},{"osd":1,"last update":"Sat Sep 23 20:42:00 2023","interfaces":[{"interface":"back","average":{"1min":0.49199999999999999,"5min":0.55200000000000005,"15min":0.56200000000000006},"min":{"1min":0.32400000000000001,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":0.65700000000000003,"5min":0.81499999999999995,"15min":0.81499999999999995},"last":0.40799999999999997},{"interface":"front","average":{"1min":0.48199999999999998,"5min":0.52100000000000002,"15min":0.52700000000000002},"min":{"1min":0.315,"5min":0.315,"15min":0.315},"max":{"1min":0.73099999999999998,"5min":0.76900000000000002,"15min":0.76900000000000002},"last":0.53500000000000003}]},{"osd":2,"last update":"Sat Sep 23 20:42:00 2023","interfaces":[{"interface":"back","average":{"1min":0.51400000000000001,"5min":0.57799999999999996,"15min":0.58799999999999997},"min":{"1min":0.32900000000000001,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":0.67000000000000004,"5min":0.79100000000000004,"15min":0.79100000000000004},"last":0.42699999999999999},{"interface":"front","average":{"1min":0.51200000000000001,"5min":0.54900000000000004,"15min":0.55500000000000005},"min":{"1min":0.375,"5min":0.375,"15min":0.375},"max":{"1min":0.66200000000000003,"5min":0.77700000000000002,"15min":0.77700000000000002},"last":0.45600000000000002}]},{"osd":3,"last update":"Sat Sep 23 20:42:00 2023","interfaces":[{"interface":"back","average":{"1min":0.51600000000000001,"5min":0.56499999999999995,"15min":0.57299999999999995},"min":{"1min":0.35999999999999999,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":0.69399999999999995,"5min":0.86499999999999999,"15min":0.86499999999999999},"last":0.36599999999999999},{"interface":"front","average":{"1min":0.54800000000000004,"5min":0.57599999999999996,"15min":0.58099999999999996},"min":{"1min":0.39300000000000002,"5min":0.39300000000000002,"15min":0.39300000000000002},"max":{"1min":0.70299999999999996,"5min":0.85099999999999998,"15min":0.85099999999999998},"last":0.60299999999999998}]},{"osd":4,"last update":"Sat Sep 23 20:42:00 2023","interfaces":[{"interface":"back","average":{"1min":0.46899999999999997,"5min":0.42299999999999999,"15min":0.41499999999999998},"min":{"1min":0.34899999999999998,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":0.67200000000000004,"5min":0.67200000000000004,"15min":0.67200000000000004},"last":0.437},{"interface":"front","average":{"1min":0.51400000000000001,"5min":0.47599999999999998,"15min":0.46999999999999997},"min":{"1min":0.318,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":0.76800000000000002,"5min":0.82799999999999996,"15min":0.82799999999999996},"last":0.499}]},{"osd":6,"last update":"Sat Sep 23 20:42:26 2023","interfaces":[{"interface":"back","average":{"1min":0.504,"5min":0.47799999999999998,"15min":0.47399999999999998},"min":{"1min":0.313,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":0.746,"5min":0.746,"15min":0.746},"last":0.56200000000000006},{"interface":"front","average":{"1min":0.52400000000000002,"5min":0.52700000000000002,"15min":0.52700000000000002},"min":{"1min":0.38,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":0.75600000000000001,"5min":0.82299999999999995,"15min":0.82299999999999995},"last":0.58799999999999997}]},{"osd":7,"last update":"Sat Sep 23 20:41:41 2023","interfaces":[{"interface":"back","average":{"1min":0.55200000000000005,"5min":0.55200000000000005,"15min":0.55200000000000005},"min":{"1min":0.32200000000000001,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":0.79800000000000004,"5min":0.79800000000000004,"15min":0.79800000000000004},"last":0.51500000000000001},{"interface":"front","average":{"1min":0.56399999999999995,"5min":0.56399999999999995,"15min":0.56399999999999995},"min":{"1min":0.375,"5min":0.375,"15min":0.375},"max":{"1min":0.84999999999999998,"5min":0.84999999999999998,"15min":0.84999999999999998},"last":0.67700000000000005}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":1682,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":1606,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":1752,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T20:42:34.378 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T20:42:34.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:42:34 smithi026 ceph-mon[95839]: pgmap v58: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:42:34.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:42:34 smithi026 ceph-mon[95839]: from='mgr.14451 172.21.15.26:0/1329296977' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:42:34.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:42:34 smithi026 ceph-mon[109912]: pgmap v58: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:42:34.633 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:42:34 smithi026 ceph-mon[109912]: from='mgr.14451 172.21.15.26:0/1329296977' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:42:34.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:42:34 smithi070 ceph-mon[100755]: pgmap v58: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:42:34.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:42:34 smithi070 ceph-mon[100755]: from='mgr.14451 172.21.15.26:0/1329296977' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:42:34.986 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:42:35.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:42:35 smithi070 ceph-mon[100755]: from='client.14787 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:42:35.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:42:35 smithi026 ceph-mon[95839]: from='client.14787 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:42:35.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:42:35 smithi026 ceph-mon[109912]: from='client.14787 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:42:36.454 INFO:teuthology.orchestra.run.smithi026.stdout: 2023-09-23T20:42:36.455 INFO:teuthology.orchestra.run.smithi026.stderr:dumped all 2023-09-23T20:42:36.832 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:42:36 smithi026 ceph-mon[95839]: pgmap v59: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:42:36.832 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:42:36 smithi026 ceph-mon[109912]: pgmap v59: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:42:36.848 INFO:teuthology.orchestra.run.smithi026.stdout:{"pg_ready":true,"pg_map":{"version":59,"stamp":"2023-09-23T20:42:35.820875+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":592674,"num_objects":24,"num_object_clones":0,"num_object_copies":72,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":24,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":224,"num_write_kb":2429,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":167,"ondisk_log_size":167,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":52700,"kb_used_data":7388,"kb_used_omap":0,"kb_used_meta":45248,"kb_avail":749908516,"statfs":{"total":767960285184,"available":767906320384,"internally_reserved":0,"allocated":7565312,"data_stored":4253366,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":46333952},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002464"},"pg_stats":[{"pgid":"4.18","version":"0'0","reported_seq":22,"reported_epoch":70,"state":"active+clean","last_fresh":"2023-09-23T20:42:05.790328+0000","last_change":"2023-09-23T20:42:05.790328+0000","last_active":"2023-09-23T20:42:05.790328+0000","last_peered":"2023-09-23T20:42:05.790328+0000","last_clean":"2023-09-23T20:42:05.790328+0000","last_became_active":"2023-09-23T20:42:05.789915+0000","last_became_peered":"2023-09-23T20:42:05.789915+0000","last_unstale":"2023-09-23T20:42:05.790328+0000","last_undegraded":"2023-09-23T20:42:05.790328+0000","last_fullsized":"2023-09-23T20:42:05.790328+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:28:47.862378+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":17,"reported_epoch":69,"state":"active+clean","last_fresh":"2023-09-23T20:42:04.563924+0000","last_change":"2023-09-23T20:41:58.901188+0000","last_active":"2023-09-23T20:42:04.563924+0000","last_peered":"2023-09-23T20:42:04.563924+0000","last_clean":"2023-09-23T20:42:04.563924+0000","last_became_active":"2023-09-23T20:41:58.900733+0000","last_became_peered":"2023-09-23T20:41:58.900733+0000","last_unstale":"2023-09-23T20:42:04.563924+0000","last_undegraded":"2023-09-23T20:42:04.563924+0000","last_fullsized":"2023-09-23T20:42:04.563924+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":4,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:14:09.190468+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":12,"reported_epoch":69,"state":"active+clean","last_fresh":"2023-09-23T20:42:04.564949+0000","last_change":"2023-09-23T20:41:58.926106+0000","last_active":"2023-09-23T20:42:04.564949+0000","last_peered":"2023-09-23T20:42:04.564949+0000","last_clean":"2023-09-23T20:42:04.564949+0000","last_became_active":"2023-09-23T20:41:58.925811+0000","last_became_peered":"2023-09-23T20:41:58.925811+0000","last_unstale":"2023-09-23T20:42:04.564949+0000","last_undegraded":"2023-09-23T20:42:04.564949+0000","last_fullsized":"2023-09-23T20:42:04.564949+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:54:31.291402+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":14,"reported_epoch":69,"state":"active+clean","last_fresh":"2023-09-23T20:42:04.565694+0000","last_change":"2023-09-23T20:41:58.899392+0000","last_active":"2023-09-23T20:42:04.565694+0000","last_peered":"2023-09-23T20:42:04.565694+0000","last_clean":"2023-09-23T20:42:04.565694+0000","last_became_active":"2023-09-23T20:41:58.899090+0000","last_became_peered":"2023-09-23T20:41:58.899090+0000","last_unstale":"2023-09-23T20:42:04.565694+0000","last_undegraded":"2023-09-23T20:42:04.565694+0000","last_fullsized":"2023-09-23T20:42:04.565694+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:12:24.515412+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":12,"reported_epoch":69,"state":"active+clean","last_fresh":"2023-09-23T20:42:04.559676+0000","last_change":"2023-09-23T20:41:58.924613+0000","last_active":"2023-09-23T20:42:04.559676+0000","last_peered":"2023-09-23T20:42:04.559676+0000","last_clean":"2023-09-23T20:42:04.559676+0000","last_became_active":"2023-09-23T20:41:58.924221+0000","last_became_peered":"2023-09-23T20:41:58.924221+0000","last_unstale":"2023-09-23T20:42:04.559676+0000","last_undegraded":"2023-09-23T20:42:04.559676+0000","last_fullsized":"2023-09-23T20:42:04.559676+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:28:20.833451+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":15,"reported_epoch":69,"state":"active+clean","last_fresh":"2023-09-23T20:42:04.564540+0000","last_change":"2023-09-23T20:41:58.906744+0000","last_active":"2023-09-23T20:42:04.564540+0000","last_peered":"2023-09-23T20:42:04.564540+0000","last_clean":"2023-09-23T20:42:04.564540+0000","last_became_active":"2023-09-23T20:41:58.906409+0000","last_became_peered":"2023-09-23T20:41:58.906409+0000","last_unstale":"2023-09-23T20:42:04.564540+0000","last_undegraded":"2023-09-23T20:42:04.564540+0000","last_fullsized":"2023-09-23T20:42:04.564540+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:15:42.250523+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":12,"reported_epoch":69,"state":"active+clean","last_fresh":"2023-09-23T20:42:04.560547+0000","last_change":"2023-09-23T20:41:58.927547+0000","last_active":"2023-09-23T20:42:04.560547+0000","last_peered":"2023-09-23T20:42:04.560547+0000","last_clean":"2023-09-23T20:42:04.560547+0000","last_became_active":"2023-09-23T20:41:58.927269+0000","last_became_peered":"2023-09-23T20:41:58.927269+0000","last_unstale":"2023-09-23T20:42:04.560547+0000","last_undegraded":"2023-09-23T20:42:04.560547+0000","last_fullsized":"2023-09-23T20:42:04.560547+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:54:12.824329+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":15,"reported_epoch":69,"state":"active+clean","last_fresh":"2023-09-23T20:42:04.564449+0000","last_change":"2023-09-23T20:41:58.906725+0000","last_active":"2023-09-23T20:42:04.564449+0000","last_peered":"2023-09-23T20:42:04.564449+0000","last_clean":"2023-09-23T20:42:04.564449+0000","last_became_active":"2023-09-23T20:41:58.906455+0000","last_became_peered":"2023-09-23T20:41:58.906455+0000","last_unstale":"2023-09-23T20:42:04.564449+0000","last_undegraded":"2023-09-23T20:42:04.564449+0000","last_fullsized":"2023-09-23T20:42:04.564449+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:32:38.304955+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":22,"reported_epoch":70,"state":"active+clean","last_fresh":"2023-09-23T20:42:05.790247+0000","last_change":"2023-09-23T20:42:05.790247+0000","last_active":"2023-09-23T20:42:05.790247+0000","last_peered":"2023-09-23T20:42:05.790247+0000","last_clean":"2023-09-23T20:42:05.790247+0000","last_became_active":"2023-09-23T20:42:05.789961+0000","last_became_peered":"2023-09-23T20:42:05.789961+0000","last_unstale":"2023-09-23T20:42:05.790247+0000","last_undegraded":"2023-09-23T20:42:05.790247+0000","last_fullsized":"2023-09-23T20:42:05.790247+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:11:50.767403+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":14,"reported_epoch":69,"state":"active+clean","last_fresh":"2023-09-23T20:42:04.564221+0000","last_change":"2023-09-23T20:41:58.906156+0000","last_active":"2023-09-23T20:42:04.564221+0000","last_peered":"2023-09-23T20:42:04.564221+0000","last_clean":"2023-09-23T20:42:04.564221+0000","last_became_active":"2023-09-23T20:41:58.905838+0000","last_became_peered":"2023-09-23T20:41:58.905838+0000","last_unstale":"2023-09-23T20:42:04.564221+0000","last_undegraded":"2023-09-23T20:42:04.564221+0000","last_fullsized":"2023-09-23T20:42:04.564221+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:42:08.292319+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":12,"reported_epoch":69,"state":"active+clean","last_fresh":"2023-09-23T20:42:04.559478+0000","last_change":"2023-09-23T20:41:58.918255+0000","last_active":"2023-09-23T20:42:04.559478+0000","last_peered":"2023-09-23T20:42:04.559478+0000","last_clean":"2023-09-23T20:42:04.559478+0000","last_became_active":"2023-09-23T20:41:58.917956+0000","last_became_peered":"2023-09-23T20:41:58.917956+0000","last_unstale":"2023-09-23T20:42:04.559478+0000","last_undegraded":"2023-09-23T20:42:04.559478+0000","last_fullsized":"2023-09-23T20:42:04.559478+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:42:21.698474+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.6","version":"70'1","reported_seq":14,"reported_epoch":70,"state":"active+clean","last_fresh":"2023-09-23T20:42:05.569773+0000","last_change":"2023-09-23T20:41:58.906360+0000","last_active":"2023-09-23T20:42:05.569773+0000","last_peered":"2023-09-23T20:42:05.569773+0000","last_clean":"2023-09-23T20:42:05.569773+0000","last_became_active":"2023-09-23T20:41:58.905683+0000","last_became_peered":"2023-09-23T20:41:58.905683+0000","last_unstale":"2023-09-23T20:42:05.569773+0000","last_undegraded":"2023-09-23T20:42:05.569773+0000","last_fullsized":"2023-09-23T20:42:05.569773+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:19:20.821454+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":17,"reported_epoch":69,"state":"active+clean","last_fresh":"2023-09-23T20:42:04.561014+0000","last_change":"2023-09-23T20:41:16.555449+0000","last_active":"2023-09-23T20:42:04.561014+0000","last_peered":"2023-09-23T20:42:04.561014+0000","last_clean":"2023-09-23T20:42:04.561014+0000","last_became_active":"2023-09-23T20:41:16.555054+0000","last_became_peered":"2023-09-23T20:41:16.555054+0000","last_unstale":"2023-09-23T20:42:04.561014+0000","last_undegraded":"2023-09-23T20:42:04.561014+0000","last_fullsized":"2023-09-23T20:42:04.561014+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:19:29.226436+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":12,"reported_epoch":69,"state":"active+clean","last_fresh":"2023-09-23T20:42:04.564782+0000","last_change":"2023-09-23T20:41:59.061304+0000","last_active":"2023-09-23T20:42:04.564782+0000","last_peered":"2023-09-23T20:42:04.564782+0000","last_clean":"2023-09-23T20:42:04.564782+0000","last_became_active":"2023-09-23T20:41:59.060972+0000","last_became_peered":"2023-09-23T20:41:59.060972+0000","last_unstale":"2023-09-23T20:42:04.564782+0000","last_undegraded":"2023-09-23T20:42:04.564782+0000","last_fullsized":"2023-09-23T20:42:04.564782+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:56:08.771697+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":17,"reported_epoch":69,"state":"active+clean","last_fresh":"2023-09-23T20:42:04.561660+0000","last_change":"2023-09-23T20:41:16.554495+0000","last_active":"2023-09-23T20:42:04.561660+0000","last_peered":"2023-09-23T20:42:04.561660+0000","last_clean":"2023-09-23T20:42:04.561660+0000","last_became_active":"2023-09-23T20:41:16.554217+0000","last_became_peered":"2023-09-23T20:41:16.554217+0000","last_unstale":"2023-09-23T20:42:04.561660+0000","last_undegraded":"2023-09-23T20:42:04.561660+0000","last_fullsized":"2023-09-23T20:42:04.561660+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:35:48.172198+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":15,"reported_epoch":69,"state":"active+clean","last_fresh":"2023-09-23T20:42:04.565228+0000","last_change":"2023-09-23T20:41:58.903926+0000","last_active":"2023-09-23T20:42:04.565228+0000","last_peered":"2023-09-23T20:42:04.565228+0000","last_clean":"2023-09-23T20:42:04.565228+0000","last_became_active":"2023-09-23T20:41:58.903634+0000","last_became_peered":"2023-09-23T20:41:58.903634+0000","last_unstale":"2023-09-23T20:42:04.565228+0000","last_undegraded":"2023-09-23T20:42:04.565228+0000","last_fullsized":"2023-09-23T20:42:04.565228+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:18:08.164482+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":12,"reported_epoch":69,"state":"active+clean","last_fresh":"2023-09-23T20:42:04.564207+0000","last_change":"2023-09-23T20:41:59.060510+0000","last_active":"2023-09-23T20:42:04.564207+0000","last_peered":"2023-09-23T20:42:04.564207+0000","last_clean":"2023-09-23T20:42:04.564207+0000","last_became_active":"2023-09-23T20:41:59.059962+0000","last_became_peered":"2023-09-23T20:41:59.059962+0000","last_unstale":"2023-09-23T20:42:04.564207+0000","last_undegraded":"2023-09-23T20:42:04.564207+0000","last_fullsized":"2023-09-23T20:42:04.564207+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:53:51.728457+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2023-09-23T20:42:04.562197+0000","last_change":"2023-09-23T20:41:58.898729+0000","last_active":"2023-09-23T20:42:04.562197+0000","last_peered":"2023-09-23T20:42:04.562197+0000","last_clean":"2023-09-23T20:42:04.562197+0000","last_became_active":"2023-09-23T20:41:58.898458+0000","last_became_peered":"2023-09-23T20:41:58.898458+0000","last_unstale":"2023-09-23T20:42:04.562197+0000","last_undegraded":"2023-09-23T20:42:04.562197+0000","last_fullsized":"2023-09-23T20:42:04.562197+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:11:25.895709+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":17,"reported_epoch":69,"state":"active+clean","last_fresh":"2023-09-23T20:42:04.558432+0000","last_change":"2023-09-23T20:41:16.684629+0000","last_active":"2023-09-23T20:42:04.558432+0000","last_peered":"2023-09-23T20:42:04.558432+0000","last_clean":"2023-09-23T20:42:04.558432+0000","last_became_active":"2023-09-23T20:41:16.684297+0000","last_became_peered":"2023-09-23T20:41:16.684297+0000","last_unstale":"2023-09-23T20:42:04.558432+0000","last_undegraded":"2023-09-23T20:42:04.558432+0000","last_fullsized":"2023-09-23T20:42:04.558432+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:28:01.891279+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":12,"reported_epoch":69,"state":"active+clean","last_fresh":"2023-09-23T20:42:04.563820+0000","last_change":"2023-09-23T20:41:58.925171+0000","last_active":"2023-09-23T20:42:04.563820+0000","last_peered":"2023-09-23T20:42:04.563820+0000","last_clean":"2023-09-23T20:42:04.563820+0000","last_became_active":"2023-09-23T20:41:58.924883+0000","last_became_peered":"2023-09-23T20:41:58.924883+0000","last_unstale":"2023-09-23T20:42:04.563820+0000","last_undegraded":"2023-09-23T20:42:04.563820+0000","last_fullsized":"2023-09-23T20:42:04.563820+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:28:03.874579+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2023-09-23T20:42:04.563109+0000","last_change":"2023-09-23T20:41:58.906198+0000","last_active":"2023-09-23T20:42:04.563109+0000","last_peered":"2023-09-23T20:42:04.563109+0000","last_clean":"2023-09-23T20:42:04.563109+0000","last_became_active":"2023-09-23T20:41:58.905808+0000","last_became_peered":"2023-09-23T20:41:58.905808+0000","last_unstale":"2023-09-23T20:42:04.563109+0000","last_undegraded":"2023-09-23T20:42:04.563109+0000","last_fullsized":"2023-09-23T20:42:04.563109+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:45:45.033621+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":17,"reported_epoch":69,"state":"active+clean","last_fresh":"2023-09-23T20:42:04.558188+0000","last_change":"2023-09-23T20:41:16.684597+0000","last_active":"2023-09-23T20:42:04.558188+0000","last_peered":"2023-09-23T20:42:04.558188+0000","last_clean":"2023-09-23T20:42:04.558188+0000","last_became_active":"2023-09-23T20:41:16.684172+0000","last_became_peered":"2023-09-23T20:41:16.684172+0000","last_unstale":"2023-09-23T20:42:04.558188+0000","last_undegraded":"2023-09-23T20:42:04.558188+0000","last_fullsized":"2023-09-23T20:42:04.558188+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:35:12.680090+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":12,"reported_epoch":69,"state":"active+clean","last_fresh":"2023-09-23T20:42:04.561128+0000","last_change":"2023-09-23T20:41:59.062981+0000","last_active":"2023-09-23T20:42:04.561128+0000","last_peered":"2023-09-23T20:42:04.561128+0000","last_clean":"2023-09-23T20:42:04.561128+0000","last_became_active":"2023-09-23T20:41:59.062717+0000","last_became_peered":"2023-09-23T20:41:59.062717+0000","last_unstale":"2023-09-23T20:42:04.561128+0000","last_undegraded":"2023-09-23T20:42:04.561128+0000","last_fullsized":"2023-09-23T20:42:04.561128+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:20:58.386461+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2023-09-23T20:42:04.565010+0000","last_change":"2023-09-23T20:41:58.902762+0000","last_active":"2023-09-23T20:42:04.565010+0000","last_peered":"2023-09-23T20:42:04.565010+0000","last_clean":"2023-09-23T20:42:04.565010+0000","last_became_active":"2023-09-23T20:41:58.902448+0000","last_became_peered":"2023-09-23T20:41:58.902448+0000","last_unstale":"2023-09-23T20:42:04.565010+0000","last_undegraded":"2023-09-23T20:42:04.565010+0000","last_fullsized":"2023-09-23T20:42:04.565010+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:06:06.890898+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.0","version":"60'128","reported_seq":277,"reported_epoch":69,"state":"active+clean","last_fresh":"2023-09-23T20:42:04.558150+0000","last_change":"2023-09-23T20:40:42.550542+0000","last_active":"2023-09-23T20:42:04.558150+0000","last_peered":"2023-09-23T20:42:04.558150+0000","last_clean":"2023-09-23T20:42:04.558150+0000","last_became_active":"2023-09-23T20:40:42.550069+0000","last_became_peered":"2023-09-23T20:40:42.550069+0000","last_unstale":"2023-09-23T20:42:04.558150+0000","last_undegraded":"2023-09-23T20:42:04.558150+0000","last_fullsized":"2023-09-23T20:42:04.558150+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:38:57.324224+0000","objects_scrubbed":0,"log_size":128,"ondisk_log_size":128,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:53:50.698837+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":17,"reported_epoch":69,"state":"active+clean","last_fresh":"2023-09-23T20:42:04.564278+0000","last_change":"2023-09-23T20:41:16.555751+0000","last_active":"2023-09-23T20:42:04.564278+0000","last_peered":"2023-09-23T20:42:04.564278+0000","last_clean":"2023-09-23T20:42:04.564278+0000","last_became_active":"2023-09-23T20:41:16.555417+0000","last_became_peered":"2023-09-23T20:41:16.555417+0000","last_unstale":"2023-09-23T20:42:04.564278+0000","last_undegraded":"2023-09-23T20:42:04.564278+0000","last_fullsized":"2023-09-23T20:42:04.564278+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:54:54.137765+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":12,"reported_epoch":69,"state":"active+clean","last_fresh":"2023-09-23T20:42:04.561141+0000","last_change":"2023-09-23T20:41:58.923989+0000","last_active":"2023-09-23T20:42:04.561141+0000","last_peered":"2023-09-23T20:42:04.561141+0000","last_clean":"2023-09-23T20:42:04.561141+0000","last_became_active":"2023-09-23T20:41:58.923702+0000","last_became_peered":"2023-09-23T20:41:58.923702+0000","last_unstale":"2023-09-23T20:42:04.561141+0000","last_undegraded":"2023-09-23T20:42:04.561141+0000","last_fullsized":"2023-09-23T20:42:04.561141+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:44:37.813515+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2023-09-23T20:42:04.561524+0000","last_change":"2023-09-23T20:41:58.897342+0000","last_active":"2023-09-23T20:42:04.561524+0000","last_peered":"2023-09-23T20:42:04.561524+0000","last_clean":"2023-09-23T20:42:04.561524+0000","last_became_active":"2023-09-23T20:41:58.897041+0000","last_became_peered":"2023-09-23T20:41:58.897041+0000","last_unstale":"2023-09-23T20:42:04.561524+0000","last_undegraded":"2023-09-23T20:42:04.561524+0000","last_fullsized":"2023-09-23T20:42:04.561524+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:57:54.858274+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":17,"reported_epoch":69,"state":"active+clean","last_fresh":"2023-09-23T20:42:04.561101+0000","last_change":"2023-09-23T20:41:16.686010+0000","last_active":"2023-09-23T20:42:04.561101+0000","last_peered":"2023-09-23T20:42:04.561101+0000","last_clean":"2023-09-23T20:42:04.561101+0000","last_became_active":"2023-09-23T20:41:16.685615+0000","last_became_peered":"2023-09-23T20:41:16.685615+0000","last_unstale":"2023-09-23T20:42:04.561101+0000","last_undegraded":"2023-09-23T20:42:04.561101+0000","last_fullsized":"2023-09-23T20:42:04.561101+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:46:04.405920+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":12,"reported_epoch":69,"state":"active+clean","last_fresh":"2023-09-23T20:42:04.562189+0000","last_change":"2023-09-23T20:41:58.919583+0000","last_active":"2023-09-23T20:42:04.562189+0000","last_peered":"2023-09-23T20:42:04.562189+0000","last_clean":"2023-09-23T20:42:04.562189+0000","last_became_active":"2023-09-23T20:41:58.919281+0000","last_became_peered":"2023-09-23T20:41:58.919281+0000","last_unstale":"2023-09-23T20:42:04.562189+0000","last_undegraded":"2023-09-23T20:42:04.562189+0000","last_fullsized":"2023-09-23T20:42:04.562189+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:37:23.934877+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2023-09-23T20:42:04.560410+0000","last_change":"2023-09-23T20:41:58.905896+0000","last_active":"2023-09-23T20:42:04.560410+0000","last_peered":"2023-09-23T20:42:04.560410+0000","last_clean":"2023-09-23T20:42:04.560410+0000","last_became_active":"2023-09-23T20:41:58.905654+0000","last_became_peered":"2023-09-23T20:41:58.905654+0000","last_unstale":"2023-09-23T20:42:04.560410+0000","last_undegraded":"2023-09-23T20:42:04.560410+0000","last_fullsized":"2023-09-23T20:42:04.560410+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:41:11.238836+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":17,"reported_epoch":69,"state":"active+clean","last_fresh":"2023-09-23T20:42:04.564283+0000","last_change":"2023-09-23T20:41:16.556552+0000","last_active":"2023-09-23T20:42:04.564283+0000","last_peered":"2023-09-23T20:42:04.564283+0000","last_clean":"2023-09-23T20:42:04.564283+0000","last_became_active":"2023-09-23T20:41:16.556226+0000","last_became_peered":"2023-09-23T20:41:16.556226+0000","last_unstale":"2023-09-23T20:42:04.564283+0000","last_undegraded":"2023-09-23T20:42:04.564283+0000","last_fullsized":"2023-09-23T20:42:04.564283+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:21:42.514384+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":12,"reported_epoch":69,"state":"active+clean","last_fresh":"2023-09-23T20:42:04.561546+0000","last_change":"2023-09-23T20:41:58.917668+0000","last_active":"2023-09-23T20:42:04.561546+0000","last_peered":"2023-09-23T20:42:04.561546+0000","last_clean":"2023-09-23T20:42:04.561546+0000","last_became_active":"2023-09-23T20:41:58.917308+0000","last_became_peered":"2023-09-23T20:41:58.917308+0000","last_unstale":"2023-09-23T20:42:04.561546+0000","last_undegraded":"2023-09-23T20:42:04.561546+0000","last_fullsized":"2023-09-23T20:42:04.561546+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:40:38.700565+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2023-09-23T20:42:04.564516+0000","last_change":"2023-09-23T20:41:58.903585+0000","last_active":"2023-09-23T20:42:04.564516+0000","last_peered":"2023-09-23T20:42:04.564516+0000","last_clean":"2023-09-23T20:42:04.564516+0000","last_became_active":"2023-09-23T20:41:58.902845+0000","last_became_peered":"2023-09-23T20:41:58.902845+0000","last_unstale":"2023-09-23T20:42:04.564516+0000","last_undegraded":"2023-09-23T20:42:04.564516+0000","last_fullsized":"2023-09-23T20:42:04.564516+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:25:22.419537+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":17,"reported_epoch":69,"state":"active+clean","last_fresh":"2023-09-23T20:42:04.562135+0000","last_change":"2023-09-23T20:41:16.684748+0000","last_active":"2023-09-23T20:42:04.562135+0000","last_peered":"2023-09-23T20:42:04.562135+0000","last_clean":"2023-09-23T20:42:04.562135+0000","last_became_active":"2023-09-23T20:41:16.684361+0000","last_became_peered":"2023-09-23T20:41:16.684361+0000","last_unstale":"2023-09-23T20:42:04.562135+0000","last_undegraded":"2023-09-23T20:42:04.562135+0000","last_fullsized":"2023-09-23T20:42:04.562135+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:39:15.420185+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":12,"reported_epoch":69,"state":"active+clean","last_fresh":"2023-09-23T20:42:04.561218+0000","last_change":"2023-09-23T20:41:58.920027+0000","last_active":"2023-09-23T20:42:04.561218+0000","last_peered":"2023-09-23T20:42:04.561218+0000","last_clean":"2023-09-23T20:42:04.561218+0000","last_became_active":"2023-09-23T20:41:58.919731+0000","last_became_peered":"2023-09-23T20:41:58.919731+0000","last_unstale":"2023-09-23T20:42:04.561218+0000","last_undegraded":"2023-09-23T20:42:04.561218+0000","last_fullsized":"2023-09-23T20:42:04.561218+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:30:06.925226+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2023-09-23T20:42:04.564785+0000","last_change":"2023-09-23T20:41:58.898493+0000","last_active":"2023-09-23T20:42:04.564785+0000","last_peered":"2023-09-23T20:42:04.564785+0000","last_clean":"2023-09-23T20:42:04.564785+0000","last_became_active":"2023-09-23T20:41:58.898205+0000","last_became_peered":"2023-09-23T20:41:58.898205+0000","last_unstale":"2023-09-23T20:42:04.564785+0000","last_undegraded":"2023-09-23T20:42:04.564785+0000","last_fullsized":"2023-09-23T20:42:04.564785+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:16:08.144482+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":12,"reported_epoch":69,"state":"active+clean","last_fresh":"2023-09-23T20:42:04.559603+0000","last_change":"2023-09-23T20:41:58.921014+0000","last_active":"2023-09-23T20:42:04.559603+0000","last_peered":"2023-09-23T20:42:04.559603+0000","last_clean":"2023-09-23T20:42:04.559603+0000","last_became_active":"2023-09-23T20:41:58.920805+0000","last_became_peered":"2023-09-23T20:41:58.920805+0000","last_unstale":"2023-09-23T20:42:04.559603+0000","last_undegraded":"2023-09-23T20:42:04.559603+0000","last_fullsized":"2023-09-23T20:42:04.559603+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:03:21.219355+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2023-09-23T20:42:04.559522+0000","last_change":"2023-09-23T20:41:58.902484+0000","last_active":"2023-09-23T20:42:04.559522+0000","last_peered":"2023-09-23T20:42:04.559522+0000","last_clean":"2023-09-23T20:42:04.559522+0000","last_became_active":"2023-09-23T20:41:58.902177+0000","last_became_peered":"2023-09-23T20:41:58.902177+0000","last_unstale":"2023-09-23T20:42:04.559522+0000","last_undegraded":"2023-09-23T20:42:04.559522+0000","last_fullsized":"2023-09-23T20:42:04.559522+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:00:00.624317+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":12,"reported_epoch":69,"state":"active+clean","last_fresh":"2023-09-23T20:42:04.559129+0000","last_change":"2023-09-23T20:41:58.920596+0000","last_active":"2023-09-23T20:42:04.559129+0000","last_peered":"2023-09-23T20:42:04.559129+0000","last_clean":"2023-09-23T20:42:04.559129+0000","last_became_active":"2023-09-23T20:41:58.920360+0000","last_became_peered":"2023-09-23T20:41:58.920360+0000","last_unstale":"2023-09-23T20:42:04.559129+0000","last_undegraded":"2023-09-23T20:42:04.559129+0000","last_fullsized":"2023-09-23T20:42:04.559129+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:36:30.646529+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2023-09-23T20:42:04.560949+0000","last_change":"2023-09-23T20:41:58.906865+0000","last_active":"2023-09-23T20:42:04.560949+0000","last_peered":"2023-09-23T20:42:04.560949+0000","last_clean":"2023-09-23T20:42:04.560949+0000","last_became_active":"2023-09-23T20:41:58.906505+0000","last_became_peered":"2023-09-23T20:41:58.906505+0000","last_unstale":"2023-09-23T20:42:04.560949+0000","last_undegraded":"2023-09-23T20:42:04.560949+0000","last_fullsized":"2023-09-23T20:42:04.560949+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:52:53.776236+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":12,"reported_epoch":69,"state":"active+clean","last_fresh":"2023-09-23T20:42:04.559994+0000","last_change":"2023-09-23T20:41:58.920103+0000","last_active":"2023-09-23T20:42:04.559994+0000","last_peered":"2023-09-23T20:42:04.559994+0000","last_clean":"2023-09-23T20:42:04.559994+0000","last_became_active":"2023-09-23T20:41:58.919764+0000","last_became_peered":"2023-09-23T20:41:58.919764+0000","last_unstale":"2023-09-23T20:42:04.559994+0000","last_undegraded":"2023-09-23T20:42:04.559994+0000","last_fullsized":"2023-09-23T20:42:04.559994+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:44:40.228355+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":15,"reported_epoch":69,"state":"active+clean","last_fresh":"2023-09-23T20:42:04.565221+0000","last_change":"2023-09-23T20:41:58.908699+0000","last_active":"2023-09-23T20:42:04.565221+0000","last_peered":"2023-09-23T20:42:04.565221+0000","last_clean":"2023-09-23T20:42:04.565221+0000","last_became_active":"2023-09-23T20:41:58.908315+0000","last_became_peered":"2023-09-23T20:41:58.908315+0000","last_unstale":"2023-09-23T20:42:04.565221+0000","last_undegraded":"2023-09-23T20:42:04.565221+0000","last_fullsized":"2023-09-23T20:42:04.565221+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:25:39.622409+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":22,"reported_epoch":70,"state":"active+clean","last_fresh":"2023-09-23T20:42:05.571127+0000","last_change":"2023-09-23T20:42:05.571127+0000","last_active":"2023-09-23T20:42:05.571127+0000","last_peered":"2023-09-23T20:42:05.571127+0000","last_clean":"2023-09-23T20:42:05.571127+0000","last_became_active":"2023-09-23T20:42:05.570777+0000","last_became_peered":"2023-09-23T20:42:05.570777+0000","last_unstale":"2023-09-23T20:42:05.571127+0000","last_undegraded":"2023-09-23T20:42:05.571127+0000","last_fullsized":"2023-09-23T20:42:05.571127+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:18:07.563540+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2023-09-23T20:42:04.564383+0000","last_change":"2023-09-23T20:41:58.903766+0000","last_active":"2023-09-23T20:42:04.564383+0000","last_peered":"2023-09-23T20:42:04.564383+0000","last_clean":"2023-09-23T20:42:04.564383+0000","last_became_active":"2023-09-23T20:41:58.902298+0000","last_became_peered":"2023-09-23T20:41:58.902298+0000","last_unstale":"2023-09-23T20:42:04.564383+0000","last_undegraded":"2023-09-23T20:42:04.564383+0000","last_fullsized":"2023-09-23T20:42:04.564383+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:56:10.559104+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":12,"reported_epoch":69,"state":"active+clean","last_fresh":"2023-09-23T20:42:04.560902+0000","last_change":"2023-09-23T20:41:59.060269+0000","last_active":"2023-09-23T20:42:04.560902+0000","last_peered":"2023-09-23T20:42:04.560902+0000","last_clean":"2023-09-23T20:42:04.560902+0000","last_became_active":"2023-09-23T20:41:59.059913+0000","last_became_peered":"2023-09-23T20:41:59.059913+0000","last_unstale":"2023-09-23T20:42:04.560902+0000","last_undegraded":"2023-09-23T20:42:04.560902+0000","last_fullsized":"2023-09-23T20:42:04.560902+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:46:45.365100+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":14,"reported_epoch":69,"state":"active+clean","last_fresh":"2023-09-23T20:42:04.558502+0000","last_change":"2023-09-23T20:41:58.898346+0000","last_active":"2023-09-23T20:42:04.558502+0000","last_peered":"2023-09-23T20:42:04.558502+0000","last_clean":"2023-09-23T20:42:04.558502+0000","last_became_active":"2023-09-23T20:41:58.898065+0000","last_became_peered":"2023-09-23T20:41:58.898065+0000","last_unstale":"2023-09-23T20:42:04.558502+0000","last_undegraded":"2023-09-23T20:42:04.558502+0000","last_fullsized":"2023-09-23T20:42:04.558502+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:51:57.491171+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":12,"reported_epoch":69,"state":"active+clean","last_fresh":"2023-09-23T20:42:04.559272+0000","last_change":"2023-09-23T20:41:58.919426+0000","last_active":"2023-09-23T20:42:04.559272+0000","last_peered":"2023-09-23T20:42:04.559272+0000","last_clean":"2023-09-23T20:42:04.559272+0000","last_became_active":"2023-09-23T20:41:58.919238+0000","last_became_peered":"2023-09-23T20:41:58.919238+0000","last_unstale":"2023-09-23T20:42:04.559272+0000","last_undegraded":"2023-09-23T20:42:04.559272+0000","last_fullsized":"2023-09-23T20:42:04.559272+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:33:14.594665+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":15,"reported_epoch":69,"state":"active+clean","last_fresh":"2023-09-23T20:42:04.558899+0000","last_change":"2023-09-23T20:41:58.894683+0000","last_active":"2023-09-23T20:42:04.558899+0000","last_peered":"2023-09-23T20:42:04.558899+0000","last_clean":"2023-09-23T20:42:04.558899+0000","last_became_active":"2023-09-23T20:41:58.894407+0000","last_became_peered":"2023-09-23T20:41:58.894407+0000","last_unstale":"2023-09-23T20:42:04.558899+0000","last_undegraded":"2023-09-23T20:42:04.558899+0000","last_fullsized":"2023-09-23T20:42:04.558899+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:41:38.552538+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":12,"reported_epoch":69,"state":"active+clean","last_fresh":"2023-09-23T20:42:04.564201+0000","last_change":"2023-09-23T20:41:59.061405+0000","last_active":"2023-09-23T20:42:04.564201+0000","last_peered":"2023-09-23T20:42:04.564201+0000","last_clean":"2023-09-23T20:42:04.564201+0000","last_became_active":"2023-09-23T20:41:59.061032+0000","last_became_peered":"2023-09-23T20:41:59.061032+0000","last_unstale":"2023-09-23T20:42:04.564201+0000","last_undegraded":"2023-09-23T20:42:04.564201+0000","last_fullsized":"2023-09-23T20:42:04.564201+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:43:09.716350+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":14,"reported_epoch":69,"state":"active+clean","last_fresh":"2023-09-23T20:42:04.558705+0000","last_change":"2023-09-23T20:41:58.895045+0000","last_active":"2023-09-23T20:42:04.558705+0000","last_peered":"2023-09-23T20:42:04.558705+0000","last_clean":"2023-09-23T20:42:04.558705+0000","last_became_active":"2023-09-23T20:41:58.894739+0000","last_became_peered":"2023-09-23T20:41:58.894739+0000","last_unstale":"2023-09-23T20:42:04.558705+0000","last_undegraded":"2023-09-23T20:42:04.558705+0000","last_fullsized":"2023-09-23T20:42:04.558705+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:51:39.089390+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":12,"reported_epoch":69,"state":"active+clean","last_fresh":"2023-09-23T20:42:04.565149+0000","last_change":"2023-09-23T20:41:59.060886+0000","last_active":"2023-09-23T20:42:04.565149+0000","last_peered":"2023-09-23T20:42:04.565149+0000","last_clean":"2023-09-23T20:42:04.565149+0000","last_became_active":"2023-09-23T20:41:59.059915+0000","last_became_peered":"2023-09-23T20:41:59.059915+0000","last_unstale":"2023-09-23T20:42:04.565149+0000","last_undegraded":"2023-09-23T20:42:04.565149+0000","last_fullsized":"2023-09-23T20:42:04.565149+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:03:29.688908+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":21,"reported_epoch":69,"state":"active+clean","last_fresh":"2023-09-23T20:42:04.561396+0000","last_change":"2023-09-23T20:41:58.900378+0000","last_active":"2023-09-23T20:42:04.561396+0000","last_peered":"2023-09-23T20:42:04.561396+0000","last_clean":"2023-09-23T20:42:04.561396+0000","last_became_active":"2023-09-23T20:41:58.900098+0000","last_became_peered":"2023-09-23T20:41:58.900098+0000","last_unstale":"2023-09-23T20:42:04.561396+0000","last_undegraded":"2023-09-23T20:42:04.561396+0000","last_fullsized":"2023-09-23T20:42:04.561396+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":8,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:32:49.020622+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":12,"reported_epoch":69,"state":"active+clean","last_fresh":"2023-09-23T20:42:04.563560+0000","last_change":"2023-09-23T20:41:59.060309+0000","last_active":"2023-09-23T20:42:04.563560+0000","last_peered":"2023-09-23T20:42:04.563560+0000","last_clean":"2023-09-23T20:42:04.563560+0000","last_became_active":"2023-09-23T20:41:59.059896+0000","last_became_peered":"2023-09-23T20:41:59.059896+0000","last_unstale":"2023-09-23T20:42:04.563560+0000","last_undegraded":"2023-09-23T20:42:04.563560+0000","last_fullsized":"2023-09-23T20:42:04.563560+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:12:55.804003+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2023-09-23T20:42:04.558325+0000","last_change":"2023-09-23T20:41:58.898224+0000","last_active":"2023-09-23T20:42:04.558325+0000","last_peered":"2023-09-23T20:42:04.558325+0000","last_clean":"2023-09-23T20:42:04.558325+0000","last_became_active":"2023-09-23T20:41:58.898036+0000","last_became_peered":"2023-09-23T20:41:58.898036+0000","last_unstale":"2023-09-23T20:42:04.558325+0000","last_undegraded":"2023-09-23T20:42:04.558325+0000","last_fullsized":"2023-09-23T20:42:04.558325+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:53:15.361560+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":12,"reported_epoch":69,"state":"active+clean","last_fresh":"2023-09-23T20:42:04.564426+0000","last_change":"2023-09-23T20:41:59.061779+0000","last_active":"2023-09-23T20:42:04.564426+0000","last_peered":"2023-09-23T20:42:04.564426+0000","last_clean":"2023-09-23T20:42:04.564426+0000","last_became_active":"2023-09-23T20:41:59.061503+0000","last_became_peered":"2023-09-23T20:41:59.061503+0000","last_unstale":"2023-09-23T20:42:04.564426+0000","last_undegraded":"2023-09-23T20:42:04.564426+0000","last_fullsized":"2023-09-23T20:42:04.564426+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:03:29.221377+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2023-09-23T20:42:04.563387+0000","last_change":"2023-09-23T20:41:58.899881+0000","last_active":"2023-09-23T20:42:04.563387+0000","last_peered":"2023-09-23T20:42:04.563387+0000","last_clean":"2023-09-23T20:42:04.563387+0000","last_became_active":"2023-09-23T20:41:58.899189+0000","last_became_peered":"2023-09-23T20:41:58.899189+0000","last_unstale":"2023-09-23T20:42:04.563387+0000","last_undegraded":"2023-09-23T20:42:04.563387+0000","last_fullsized":"2023-09-23T20:42:04.563387+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:01:03.685609+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":12,"reported_epoch":69,"state":"active+clean","last_fresh":"2023-09-23T20:42:04.565332+0000","last_change":"2023-09-23T20:41:59.060226+0000","last_active":"2023-09-23T20:42:04.565332+0000","last_peered":"2023-09-23T20:42:04.565332+0000","last_clean":"2023-09-23T20:42:04.565332+0000","last_became_active":"2023-09-23T20:41:59.059889+0000","last_became_peered":"2023-09-23T20:41:59.059889+0000","last_unstale":"2023-09-23T20:42:04.565332+0000","last_undegraded":"2023-09-23T20:42:04.565332+0000","last_fullsized":"2023-09-23T20:42:04.565332+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:36:53.062096+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":15,"reported_epoch":69,"state":"active+clean","last_fresh":"2023-09-23T20:42:04.559423+0000","last_change":"2023-09-23T20:41:58.897223+0000","last_active":"2023-09-23T20:42:04.559423+0000","last_peered":"2023-09-23T20:42:04.559423+0000","last_clean":"2023-09-23T20:42:04.559423+0000","last_became_active":"2023-09-23T20:41:58.896599+0000","last_became_peered":"2023-09-23T20:41:58.896599+0000","last_unstale":"2023-09-23T20:42:04.559423+0000","last_undegraded":"2023-09-23T20:42:04.559423+0000","last_fullsized":"2023-09-23T20:42:04.559423+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:44:37.392444+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":12,"reported_epoch":69,"state":"active+clean","last_fresh":"2023-09-23T20:42:04.560546+0000","last_change":"2023-09-23T20:41:58.921718+0000","last_active":"2023-09-23T20:42:04.560546+0000","last_peered":"2023-09-23T20:42:04.560546+0000","last_clean":"2023-09-23T20:42:04.560546+0000","last_became_active":"2023-09-23T20:41:58.921443+0000","last_became_peered":"2023-09-23T20:41:58.921443+0000","last_unstale":"2023-09-23T20:42:04.560546+0000","last_undegraded":"2023-09-23T20:42:04.560546+0000","last_fullsized":"2023-09-23T20:42:04.560546+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:10:39.135164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"68'6","reported_seq":19,"reported_epoch":69,"state":"active+clean","last_fresh":"2023-09-23T20:42:04.559924+0000","last_change":"2023-09-23T20:41:58.905866+0000","last_active":"2023-09-23T20:42:04.559924+0000","last_peered":"2023-09-23T20:42:04.559924+0000","last_clean":"2023-09-23T20:42:04.559924+0000","last_became_active":"2023-09-23T20:41:58.905556+0000","last_became_peered":"2023-09-23T20:41:58.905556+0000","last_unstale":"2023-09-23T20:42:04.559924+0000","last_undegraded":"2023-09-23T20:42:04.559924+0000","last_fullsized":"2023-09-23T20:42:04.559924+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":6,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:03:14.027348+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":1536,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6,"num_write_kb":3,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":12,"reported_epoch":69,"state":"active+clean","last_fresh":"2023-09-23T20:42:04.561452+0000","last_change":"2023-09-23T20:41:58.921966+0000","last_active":"2023-09-23T20:42:04.561452+0000","last_peered":"2023-09-23T20:42:04.561452+0000","last_clean":"2023-09-23T20:42:04.561452+0000","last_became_active":"2023-09-23T20:41:58.921649+0000","last_became_peered":"2023-09-23T20:41:58.921649+0000","last_unstale":"2023-09-23T20:42:04.561452+0000","last_undegraded":"2023-09-23T20:42:04.561452+0000","last_fullsized":"2023-09-23T20:42:04.561452+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:05:43.041172+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":16,"reported_epoch":69,"state":"active+clean","last_fresh":"2023-09-23T20:42:04.558364+0000","last_change":"2023-09-23T20:41:58.895940+0000","last_active":"2023-09-23T20:42:04.558364+0000","last_peered":"2023-09-23T20:42:04.558364+0000","last_clean":"2023-09-23T20:42:04.558364+0000","last_became_active":"2023-09-23T20:41:58.895688+0000","last_became_peered":"2023-09-23T20:41:58.895688+0000","last_unstale":"2023-09-23T20:42:04.558364+0000","last_undegraded":"2023-09-23T20:42:04.558364+0000","last_fullsized":"2023-09-23T20:42:04.558364+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":3,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:04:40.907902+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":12,"reported_epoch":69,"state":"active+clean","last_fresh":"2023-09-23T20:42:04.564996+0000","last_change":"2023-09-23T20:41:59.060781+0000","last_active":"2023-09-23T20:42:04.564996+0000","last_peered":"2023-09-23T20:42:04.564996+0000","last_clean":"2023-09-23T20:42:04.564996+0000","last_became_active":"2023-09-23T20:41:59.059897+0000","last_became_peered":"2023-09-23T20:41:59.059897+0000","last_unstale":"2023-09-23T20:42:04.564996+0000","last_undegraded":"2023-09-23T20:42:04.564996+0000","last_fullsized":"2023-09-23T20:42:04.564996+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:55:42.494186+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2023-09-23T20:42:04.564476+0000","last_change":"2023-09-23T20:41:58.901830+0000","last_active":"2023-09-23T20:42:04.564476+0000","last_peered":"2023-09-23T20:42:04.564476+0000","last_clean":"2023-09-23T20:42:04.564476+0000","last_became_active":"2023-09-23T20:41:58.901536+0000","last_became_peered":"2023-09-23T20:41:58.901536+0000","last_unstale":"2023-09-23T20:42:04.564476+0000","last_undegraded":"2023-09-23T20:42:04.564476+0000","last_fullsized":"2023-09-23T20:42:04.564476+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:07:53.066435+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":12,"reported_epoch":69,"state":"active+clean","last_fresh":"2023-09-23T20:42:04.564869+0000","last_change":"2023-09-23T20:41:58.923598+0000","last_active":"2023-09-23T20:42:04.564869+0000","last_peered":"2023-09-23T20:42:04.564869+0000","last_clean":"2023-09-23T20:42:04.564869+0000","last_became_active":"2023-09-23T20:41:58.923297+0000","last_became_peered":"2023-09-23T20:41:58.923297+0000","last_unstale":"2023-09-23T20:42:04.564869+0000","last_undegraded":"2023-09-23T20:42:04.564869+0000","last_fullsized":"2023-09-23T20:42:04.564869+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:57:10.155613+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2023-09-23T20:42:04.564008+0000","last_change":"2023-09-23T20:41:58.902008+0000","last_active":"2023-09-23T20:42:04.564008+0000","last_peered":"2023-09-23T20:42:04.564008+0000","last_clean":"2023-09-23T20:42:04.564008+0000","last_became_active":"2023-09-23T20:41:58.901717+0000","last_became_peered":"2023-09-23T20:41:58.901717+0000","last_unstale":"2023-09-23T20:42:04.564008+0000","last_undegraded":"2023-09-23T20:42:04.564008+0000","last_fullsized":"2023-09-23T20:42:04.564008+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:33:32.458651+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":12,"reported_epoch":69,"state":"active+clean","last_fresh":"2023-09-23T20:42:04.560991+0000","last_change":"2023-09-23T20:41:58.920656+0000","last_active":"2023-09-23T20:42:04.560991+0000","last_peered":"2023-09-23T20:42:04.560991+0000","last_clean":"2023-09-23T20:42:04.560991+0000","last_became_active":"2023-09-23T20:41:58.920420+0000","last_became_peered":"2023-09-23T20:41:58.920420+0000","last_unstale":"2023-09-23T20:42:04.560991+0000","last_undegraded":"2023-09-23T20:42:04.560991+0000","last_fullsized":"2023-09-23T20:42:04.560991+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:59:16.325436+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2023-09-23T20:42:04.564880+0000","last_change":"2023-09-23T20:41:58.901955+0000","last_active":"2023-09-23T20:42:04.564880+0000","last_peered":"2023-09-23T20:42:04.564880+0000","last_clean":"2023-09-23T20:42:04.564880+0000","last_became_active":"2023-09-23T20:41:58.901659+0000","last_became_peered":"2023-09-23T20:41:58.901659+0000","last_unstale":"2023-09-23T20:42:04.564880+0000","last_undegraded":"2023-09-23T20:42:04.564880+0000","last_fullsized":"2023-09-23T20:42:04.564880+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:16:06.225477+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":12,"reported_epoch":69,"state":"active+clean","last_fresh":"2023-09-23T20:42:04.563635+0000","last_change":"2023-09-23T20:41:59.061254+0000","last_active":"2023-09-23T20:42:04.563635+0000","last_peered":"2023-09-23T20:42:04.563635+0000","last_clean":"2023-09-23T20:42:04.563635+0000","last_became_active":"2023-09-23T20:41:59.060968+0000","last_became_peered":"2023-09-23T20:41:59.060968+0000","last_unstale":"2023-09-23T20:42:04.563635+0000","last_undegraded":"2023-09-23T20:42:04.563635+0000","last_fullsized":"2023-09-23T20:42:04.563635+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:39:11.919325+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2023-09-23T20:42:04.561166+0000","last_change":"2023-09-23T20:41:58.907143+0000","last_active":"2023-09-23T20:42:04.561166+0000","last_peered":"2023-09-23T20:42:04.561166+0000","last_clean":"2023-09-23T20:42:04.561166+0000","last_became_active":"2023-09-23T20:41:58.906851+0000","last_became_peered":"2023-09-23T20:41:58.906851+0000","last_unstale":"2023-09-23T20:42:04.561166+0000","last_undegraded":"2023-09-23T20:42:04.561166+0000","last_fullsized":"2023-09-23T20:42:04.561166+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:34:34.423958+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":12,"reported_epoch":69,"state":"active+clean","last_fresh":"2023-09-23T20:42:04.565240+0000","last_change":"2023-09-23T20:41:58.923643+0000","last_active":"2023-09-23T20:42:04.565240+0000","last_peered":"2023-09-23T20:42:04.565240+0000","last_clean":"2023-09-23T20:42:04.565240+0000","last_became_active":"2023-09-23T20:41:58.923348+0000","last_became_peered":"2023-09-23T20:41:58.923348+0000","last_unstale":"2023-09-23T20:42:04.565240+0000","last_undegraded":"2023-09-23T20:42:04.565240+0000","last_fullsized":"2023-09-23T20:42:04.565240+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:01:02.077218+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"68'1","reported_seq":14,"reported_epoch":69,"state":"active+clean","last_fresh":"2023-09-23T20:42:04.559605+0000","last_change":"2023-09-23T20:41:58.902573+0000","last_active":"2023-09-23T20:42:04.559605+0000","last_peered":"2023-09-23T20:42:04.559605+0000","last_clean":"2023-09-23T20:42:04.559605+0000","last_became_active":"2023-09-23T20:41:58.902320+0000","last_became_peered":"2023-09-23T20:41:58.902320+0000","last_unstale":"2023-09-23T20:42:04.559605+0000","last_undegraded":"2023-09-23T20:42:04.559605+0000","last_fullsized":"2023-09-23T20:42:04.559605+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:39:32.963286+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":2306,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":45,"num_write_kb":13,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":6918,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":39,"ondisk_log_size":39,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":128,"ondisk_log_size":128,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201305,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6620,"kb_used_data":1300,"kb_used_omap":0,"kb_used_meta":5312,"kb_avail":93738532,"statfs":{"total":95995035648,"available":95988256768,"internally_reserved":0,"allocated":1331200,"data_stored":901362,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5439488},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:41:40 2023","interfaces":[{"interface":"back","average":{"1min":0.54100000000000004,"5min":0.54100000000000004,"15min":0.54100000000000004},"min":{"1min":0.28999999999999998,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":0.85499999999999998,"5min":0.85499999999999998,"15min":0.85499999999999998},"last":0.58899999999999997},{"interface":"front","average":{"1min":0.54300000000000004,"5min":0.54300000000000004,"15min":0.54300000000000004},"min":{"1min":0.39000000000000001,"5min":0.39000000000000001,"15min":0.39000000000000001},"max":{"1min":0.79400000000000004,"5min":0.79400000000000004,"15min":0.79400000000000004},"last":0.57199999999999995}]},{"osd":1,"last update":"Sat Sep 23 20:41:40 2023","interfaces":[{"interface":"back","average":{"1min":0.54400000000000004,"5min":0.54400000000000004,"15min":0.54400000000000004},"min":{"1min":0.27500000000000002,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":0.81499999999999995,"5min":0.81499999999999995,"15min":0.81499999999999995},"last":0.53900000000000003},{"interface":"front","average":{"1min":0.55000000000000004,"5min":0.55000000000000004,"15min":0.55000000000000004},"min":{"1min":0.35999999999999999,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":0.78300000000000003,"5min":0.78300000000000003,"15min":0.78300000000000003},"last":0.39200000000000002}]},{"osd":2,"last update":"Sat Sep 23 20:41:40 2023","interfaces":[{"interface":"back","average":{"1min":0.52500000000000002,"5min":0.52500000000000002,"15min":0.52500000000000002},"min":{"1min":0.32700000000000001,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":0.82699999999999996,"5min":0.82699999999999996,"15min":0.82699999999999996},"last":0.65000000000000002},{"interface":"front","average":{"1min":0.57599999999999996,"5min":0.57599999999999996,"15min":0.57599999999999996},"min":{"1min":0.309,"5min":0.309,"15min":0.309},"max":{"1min":0.93100000000000005,"5min":0.93100000000000005,"15min":0.93100000000000005},"last":0.45200000000000001}]},{"osd":3,"last update":"Sat Sep 23 20:41:40 2023","interfaces":[{"interface":"back","average":{"1min":0.55200000000000005,"5min":0.55200000000000005,"15min":0.55200000000000005},"min":{"1min":0.37,"5min":0.37,"15min":0.37},"max":{"1min":0.73399999999999999,"5min":0.73399999999999999,"15min":0.73399999999999999},"last":0.68500000000000005},{"interface":"front","average":{"1min":0.58799999999999997,"5min":0.58799999999999997,"15min":0.58799999999999997},"min":{"1min":0.35199999999999998,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":0.88800000000000001,"5min":0.88800000000000001,"15min":0.88800000000000001},"last":0.66400000000000003}]},{"osd":4,"last update":"Sat Sep 23 20:41:40 2023","interfaces":[{"interface":"back","average":{"1min":0.47099999999999997,"5min":0.47099999999999997,"15min":0.47099999999999997},"min":{"1min":0.312,"5min":0.312,"15min":0.312},"max":{"1min":0.59499999999999997,"5min":0.59499999999999997,"15min":0.59499999999999997},"last":0.49099999999999999},{"interface":"front","average":{"1min":0.51300000000000001,"5min":0.51300000000000001,"15min":0.51300000000000001},"min":{"1min":0.36699999999999999,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":0.79500000000000004,"5min":0.79500000000000004,"15min":0.79500000000000004},"last":0.70999999999999996}]},{"osd":5,"last update":"Sat Sep 23 20:41:40 2023","interfaces":[{"interface":"back","average":{"1min":0.53000000000000003,"5min":0.53000000000000003,"15min":0.53000000000000003},"min":{"1min":0.33600000000000002,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":0.77800000000000002,"5min":0.77800000000000002,"15min":0.77800000000000002},"last":0.622},{"interface":"front","average":{"1min":0.55000000000000004,"5min":0.55000000000000004,"15min":0.55000000000000004},"min":{"1min":0.35999999999999999,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":0.78500000000000003,"5min":0.78500000000000003,"15min":0.78500000000000003},"last":0.61199999999999999}]},{"osd":6,"last update":"Sat Sep 23 20:41:40 2023","interfaces":[{"interface":"back","average":{"1min":0.56200000000000006,"5min":0.56200000000000006,"15min":0.56200000000000006},"min":{"1min":0.38,"5min":0.38,"15min":0.38},"max":{"1min":0.874,"5min":0.874,"15min":0.874},"last":0.64000000000000001},{"interface":"front","average":{"1min":0.58899999999999997,"5min":0.58899999999999997,"15min":0.58899999999999997},"min":{"1min":0.316,"5min":0.316,"15min":0.316},"max":{"1min":0.85099999999999998,"5min":0.85099999999999998,"15min":0.85099999999999998},"last":0.73099999999999998}]}]},{"osd":6,"up_from":47,"seq":201863462942,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6668,"kb_used_data":1284,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":93738484,"statfs":{"total":95995035648,"available":95988207616,"internally_reserved":0,"allocated":1314816,"data_stored":900951,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:42:23 2023","interfaces":[{"interface":"back","average":{"1min":0.48099999999999998,"5min":0.505,"15min":0.50900000000000001},"min":{"1min":0.28399999999999997,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":0.80200000000000005,"5min":0.80200000000000005,"15min":0.80200000000000005},"last":0.50800000000000001},{"interface":"front","average":{"1min":0.51500000000000001,"5min":0.52700000000000002,"15min":0.52900000000000003},"min":{"1min":0.35599999999999998,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":1.206,"5min":1.206,"15min":1.206},"last":0.51700000000000002}]},{"osd":1,"last update":"Sat Sep 23 20:42:23 2023","interfaces":[{"interface":"back","average":{"1min":0.52500000000000002,"5min":0.54600000000000004,"15min":0.55000000000000004},"min":{"1min":0.32300000000000001,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":1.1759999999999999,"5min":1.1759999999999999,"15min":1.1759999999999999},"last":0.47299999999999998},{"interface":"front","average":{"1min":0.50600000000000001,"5min":0.52100000000000002,"15min":0.52300000000000002},"min":{"1min":0.308,"5min":0.308,"15min":0.308},"max":{"1min":0.90400000000000003,"5min":0.90400000000000003,"15min":0.90400000000000003},"last":0.40999999999999998}]},{"osd":2,"last update":"Sat Sep 23 20:42:23 2023","interfaces":[{"interface":"back","average":{"1min":0.51200000000000001,"5min":0.50900000000000001,"15min":0.50900000000000001},"min":{"1min":0.39400000000000002,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":0.84399999999999997,"5min":0.84399999999999997,"15min":0.84399999999999997},"last":0.45900000000000002},{"interface":"front","average":{"1min":0.54900000000000004,"5min":0.54700000000000004,"15min":0.54700000000000004},"min":{"1min":0.36599999999999999,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":1.1499999999999999,"5min":1.1499999999999999,"15min":1.1499999999999999},"last":0.54200000000000004}]},{"osd":3,"last update":"Sat Sep 23 20:42:23 2023","interfaces":[{"interface":"back","average":{"1min":0.57399999999999995,"5min":0.54500000000000004,"15min":0.54000000000000004},"min":{"1min":0.38600000000000001,"5min":0.38,"15min":0.38},"max":{"1min":0.96099999999999997,"5min":0.96099999999999997,"15min":0.96099999999999997},"last":0.55200000000000005},{"interface":"front","average":{"1min":0.53600000000000003,"5min":0.55300000000000005,"15min":0.55600000000000005},"min":{"1min":0.36399999999999999,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":0.874,"5min":0.874,"15min":0.874},"last":0.497}]},{"osd":4,"last update":"Sat Sep 23 20:42:23 2023","interfaces":[{"interface":"back","average":{"1min":0.47599999999999998,"5min":0.438,"15min":0.432},"min":{"1min":0.28799999999999998,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":1.1779999999999999,"5min":1.1779999999999999,"15min":1.1779999999999999},"last":0.442},{"interface":"front","average":{"1min":0.46100000000000002,"5min":0.47499999999999998,"15min":0.47699999999999998},"min":{"1min":0.313,"5min":0.313,"15min":0.313},"max":{"1min":0.72899999999999998,"5min":0.72899999999999998,"15min":0.72899999999999998},"last":0.434}]},{"osd":5,"last update":"Sat Sep 23 20:42:23 2023","interfaces":[{"interface":"back","average":{"1min":0.48199999999999998,"5min":0.46400000000000002,"15min":0.46100000000000002},"min":{"1min":0.32900000000000001,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":1.081,"5min":1.081,"15min":1.081},"last":0.56299999999999994},{"interface":"front","average":{"1min":0.51100000000000001,"5min":0.497,"15min":0.495},"min":{"1min":0.31,"5min":0.31,"15min":0.31},"max":{"1min":0.89000000000000001,"5min":0.89000000000000001,"15min":0.89000000000000001},"last":0.38600000000000001}]},{"osd":7,"last update":"Sat Sep 23 20:41:38 2023","interfaces":[{"interface":"back","average":{"1min":0.50600000000000001,"5min":0.50600000000000001,"15min":0.50600000000000001},"min":{"1min":0.32500000000000001,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":0.70199999999999996,"5min":0.70199999999999996,"15min":0.70199999999999996},"last":0.48899999999999999},{"interface":"front","average":{"1min":0.55900000000000005,"5min":0.55900000000000005,"15min":0.55900000000000005},"min":{"1min":0.41599999999999998,"5min":0.41599999999999998,"15min":0.41599999999999998},"max":{"1min":0.77400000000000002,"5min":0.77400000000000002,"15min":0.77400000000000002},"last":0.53100000000000003}]}]},{"osd":1,"up_from":13,"seq":55834574899,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6848,"kb_used_data":696,"kb_used_omap":0,"kb_used_meta":6144,"kb_avail":93738304,"statfs":{"total":95995035648,"available":95988023296,"internally_reserved":0,"allocated":712704,"data_stored":309565,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":6291456},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:41:38 2023","interfaces":[{"interface":"back","average":{"1min":0.43099999999999999,"5min":0.40999999999999998,"15min":0.40400000000000003},"min":{"1min":0.27300000000000002,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":0.94499999999999995,"5min":0.96499999999999997,"15min":0.96499999999999997},"last":0.42599999999999999},{"interface":"front","average":{"1min":0.437,"5min":0.442,"15min":0.439},"min":{"1min":0.23999999999999999,"5min":0.218,"15min":0.218},"max":{"1min":1.2549999999999999,"5min":1.2549999999999999,"15min":1.2549999999999999},"last":0.36099999999999999}]},{"osd":2,"last update":"Sat Sep 23 20:42:01 2023","interfaces":[{"interface":"back","average":{"1min":0.45200000000000001,"5min":0.46899999999999997,"15min":0.46500000000000002},"min":{"1min":0.27300000000000002,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":1.544,"5min":1.544,"15min":1.544},"last":0.40999999999999998},{"interface":"front","average":{"1min":0.39100000000000001,"5min":0.41699999999999998,"15min":0.41599999999999998},"min":{"1min":0.26200000000000001,"5min":0.254,"15min":0.254},"max":{"1min":0.66000000000000003,"5min":0.748,"15min":0.748},"last":0.39000000000000001}]},{"osd":3,"last update":"Sat Sep 23 20:42:24 2023","interfaces":[{"interface":"back","average":{"1min":0.498,"5min":0.46200000000000002,"15min":0.46600000000000003},"min":{"1min":0.30099999999999999,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":1.8320000000000001,"5min":1.8320000000000001,"15min":1.8320000000000001},"last":0.376},{"interface":"front","average":{"1min":0.47499999999999998,"5min":0.48899999999999999,"15min":0.498},"min":{"1min":0.312,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":1.2929999999999999,"5min":1.2929999999999999,"15min":1.2929999999999999},"last":0.46700000000000003}]},{"osd":4,"last update":"Sat Sep 23 20:41:38 2023","interfaces":[{"interface":"back","average":{"1min":0.63300000000000001,"5min":0.58899999999999997,"15min":0.58199999999999996},"min":{"1min":0.38300000000000001,"5min":0.38300000000000001,"15min":0.38300000000000001},"max":{"1min":0.90600000000000003,"5min":0.98699999999999999,"15min":0.98699999999999999},"last":0.56899999999999995},{"interface":"front","average":{"1min":0.70399999999999996,"5min":0.63600000000000001,"15min":0.624},"min":{"1min":0.45000000000000001,"5min":0.40000000000000002,"15min":0.40000000000000002},"max":{"1min":1.5760000000000001,"5min":1.5760000000000001,"15min":1.5760000000000001},"last":0.57599999999999996}]},{"osd":5,"last update":"Sat Sep 23 20:42:01 2023","interfaces":[{"interface":"back","average":{"1min":0.63800000000000001,"5min":0.64100000000000001,"15min":0.64100000000000001},"min":{"1min":0.379,"5min":0.379,"15min":0.379},"max":{"1min":1.516,"5min":1.516,"15min":1.516},"last":0.55700000000000005},{"interface":"front","average":{"1min":0.61299999999999999,"5min":0.63,"15min":0.63300000000000001},"min":{"1min":0.371,"5min":0.371,"15min":0.371},"max":{"1min":0.91700000000000004,"5min":0.91700000000000004,"15min":0.91700000000000004},"last":0.63500000000000001}]},{"osd":6,"last update":"Sat Sep 23 20:42:20 2023","interfaces":[{"interface":"back","average":{"1min":0.623,"5min":0.61499999999999999,"15min":0.61399999999999999},"min":{"1min":0.40400000000000003,"5min":0.40400000000000003,"15min":0.40400000000000003},"max":{"1min":0.89200000000000002,"5min":0.92000000000000004,"15min":0.92000000000000004},"last":0.68899999999999995},{"interface":"front","average":{"1min":0.65400000000000003,"5min":0.65600000000000003,"15min":0.65600000000000003},"min":{"1min":0.45100000000000001,"5min":0.41399999999999998,"15min":0.41399999999999998},"max":{"1min":1.4850000000000001,"5min":1.4850000000000001,"15min":1.4850000000000001},"last":0.66500000000000004}]},{"osd":7,"last update":"Sat Sep 23 20:41:40 2023","interfaces":[{"interface":"back","average":{"1min":0.69899999999999995,"5min":0.69899999999999995,"15min":0.69899999999999995},"min":{"1min":0.439,"5min":0.439,"15min":0.439},"max":{"1min":1.45,"5min":1.45,"15min":1.45},"last":0.72899999999999998},{"interface":"front","average":{"1min":0.65800000000000003,"5min":0.65800000000000003,"15min":0.65800000000000003},"min":{"1min":0.442,"5min":0.442,"15min":0.442},"max":{"1min":0.86199999999999999,"5min":0.86199999999999999,"15min":0.86199999999999999},"last":0.71999999999999997}]}]},{"osd":0,"up_from":8,"seq":34359738424,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":7432,"kb_used_data":1280,"kb_used_omap":0,"kb_used_meta":6144,"kb_avail":93737720,"statfs":{"total":95995035648,"available":95987425280,"internally_reserved":0,"allocated":1310720,"data_stored":900244,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":6291456},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 20:41:40 2023","interfaces":[{"interface":"back","average":{"1min":0.34499999999999997,"5min":0.35899999999999999,"15min":0.34399999999999997},"min":{"1min":0.26300000000000001,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":0.51000000000000001,"5min":0.65900000000000003,"15min":0.65900000000000003},"last":0.28100000000000003},{"interface":"front","average":{"1min":0.39500000000000002,"5min":0.36399999999999999,"15min":0.34699999999999998},"min":{"1min":0.26300000000000001,"5min":0.22,"15min":0.22},"max":{"1min":0.55700000000000005,"5min":0.623,"15min":0.623},"last":0.23300000000000001}]},{"osd":2,"last update":"Sat Sep 23 20:42:02 2023","interfaces":[{"interface":"back","average":{"1min":0.36199999999999999,"5min":0.41399999999999998,"15min":0.41799999999999998},"min":{"1min":0.23599999999999999,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":0.45300000000000001,"5min":0.66400000000000003,"15min":0.66400000000000003},"last":0.30099999999999999},{"interface":"front","average":{"1min":0.39100000000000001,"5min":0.41899999999999998,"15min":0.41699999999999998},"min":{"1min":0.26100000000000001,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":0.54100000000000004,"5min":0.81000000000000005,"15min":0.81000000000000005},"last":0.39700000000000002}]},{"osd":3,"last update":"Sat Sep 23 20:42:26 2023","interfaces":[{"interface":"back","average":{"1min":0.496,"5min":0.48499999999999999,"15min":0.48899999999999999},"min":{"1min":0.29899999999999999,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":1.6899999999999999,"5min":1.6899999999999999,"15min":1.6899999999999999},"last":0.373},{"interface":"front","average":{"1min":0.433,"5min":0.45300000000000001,"15min":0.46400000000000002},"min":{"1min":0.28899999999999998,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":0.95299999999999996,"5min":0.95299999999999996,"15min":0.95299999999999996},"last":0.36299999999999999}]},{"osd":4,"last update":"Sat Sep 23 20:41:40 2023","interfaces":[{"interface":"back","average":{"1min":0.60899999999999999,"5min":0.61099999999999999,"15min":0.61099999999999999},"min":{"1min":0.43099999999999999,"5min":0.42299999999999999,"15min":0.42299999999999999},"max":{"1min":1.734,"5min":1.734,"15min":1.734},"last":0.55700000000000005},{"interface":"front","average":{"1min":0.61899999999999999,"5min":0.64300000000000002,"15min":0.64700000000000002},"min":{"1min":0.42099999999999999,"5min":0.42099999999999999,"15min":0.42099999999999999},"max":{"1min":1.7170000000000001,"5min":1.7170000000000001,"15min":1.7170000000000001},"last":0.53600000000000003}]},{"osd":5,"last update":"Sat Sep 23 20:42:02 2023","interfaces":[{"interface":"back","average":{"1min":0.60999999999999999,"5min":0.60399999999999998,"15min":0.60299999999999998},"min":{"1min":0.39800000000000002,"5min":0.39800000000000002,"15min":0.39800000000000002},"max":{"1min":1.6890000000000001,"5min":1.6890000000000001,"15min":1.6890000000000001},"last":0.61399999999999999},{"interface":"front","average":{"1min":0.622,"5min":0.63800000000000001,"15min":0.64100000000000001},"min":{"1min":0.38300000000000001,"5min":0.38300000000000001,"15min":0.38300000000000001},"max":{"1min":1.9390000000000001,"5min":1.9390000000000001,"15min":1.9390000000000001},"last":0.45700000000000002}]},{"osd":6,"last update":"Sat Sep 23 20:42:20 2023","interfaces":[{"interface":"back","average":{"1min":0.66300000000000003,"5min":0.59899999999999998,"15min":0.58799999999999997},"min":{"1min":0.39100000000000001,"5min":0.39100000000000001,"15min":0.39100000000000001},"max":{"1min":1.752,"5min":1.752,"15min":1.752},"last":0.57399999999999995},{"interface":"front","average":{"1min":0.71599999999999997,"5min":0.61899999999999999,"15min":0.60299999999999998},"min":{"1min":0.39400000000000002,"5min":0.39400000000000002,"15min":0.39400000000000002},"max":{"1min":2.0009999999999999,"5min":2.0009999999999999,"15min":2.0009999999999999},"last":0.48299999999999998}]},{"osd":7,"last update":"Sat Sep 23 20:41:40 2023","interfaces":[{"interface":"back","average":{"1min":0.626,"5min":0.626,"15min":0.626},"min":{"1min":0.38600000000000001,"5min":0.38600000000000001,"15min":0.38600000000000001},"max":{"1min":1.9890000000000001,"5min":1.9890000000000001,"15min":1.9890000000000001},"last":0.60199999999999998},{"interface":"front","average":{"1min":0.67900000000000005,"5min":0.67900000000000005,"15min":0.67900000000000005},"min":{"1min":0.41499999999999998,"5min":0.41499999999999998,"15min":0.41499999999999998},"max":{"1min":2.0169999999999999,"5min":2.0169999999999999,"15min":2.0169999999999999},"last":0.47299999999999998}]}]},{"osd":2,"up_from":18,"seq":77309411374,"num_pgs":19,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6664,"kb_used_data":704,"kb_used_omap":0,"kb_used_meta":5952,"kb_avail":93738488,"statfs":{"total":95995035648,"available":95988211712,"internally_reserved":0,"allocated":720896,"data_stored":310153,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":6094848},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:42:02 2023","interfaces":[{"interface":"back","average":{"1min":0.46300000000000002,"5min":0.41099999999999998,"15min":0.39900000000000002},"min":{"1min":0.25900000000000001,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":2.4350000000000001,"5min":2.4350000000000001,"15min":2.4350000000000001},"last":0.52900000000000003},{"interface":"front","average":{"1min":1.026,"5min":0.55000000000000004,"15min":0.46200000000000002},"min":{"1min":0.255,"5min":0.253,"15min":0.253},"max":{"1min":13.180999999999999,"5min":13.180999999999999,"15min":13.180999999999999},"last":0.28299999999999997}]},{"osd":1,"last update":"Sat Sep 23 20:42:02 2023","interfaces":[{"interface":"back","average":{"1min":0.53800000000000003,"5min":0.45800000000000002,"15min":0.44},"min":{"1min":0.26700000000000002,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":2.9049999999999998,"5min":2.9049999999999998,"15min":2.9049999999999998},"last":0.42099999999999999},{"interface":"front","average":{"1min":0.51100000000000001,"5min":0.47199999999999998,"15min":0.46100000000000002},"min":{"1min":0.29899999999999999,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":2.4609999999999999,"5min":2.4609999999999999,"15min":2.4609999999999999},"last":0.48799999999999999}]},{"osd":3,"last update":"Sat Sep 23 20:42:22 2023","interfaces":[{"interface":"back","average":{"1min":1.05,"5min":0.60999999999999999,"15min":0.54200000000000004},"min":{"1min":0.27600000000000002,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":13.239000000000001,"5min":13.239000000000001,"15min":13.239000000000001},"last":0.46600000000000003},{"interface":"front","average":{"1min":0.52200000000000002,"5min":0.52100000000000002,"15min":0.53700000000000003},"min":{"1min":0.29299999999999998,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":3.0110000000000001,"5min":3.0110000000000001,"15min":3.0110000000000001},"last":0.33300000000000002}]},{"osd":4,"last update":"Sat Sep 23 20:41:40 2023","interfaces":[{"interface":"back","average":{"1min":0.65100000000000002,"5min":0.66300000000000003,"15min":0.66500000000000004},"min":{"1min":0.39500000000000002,"5min":0.39500000000000002,"15min":0.39500000000000002},"max":{"1min":2.327,"5min":2.327,"15min":2.327},"last":0.69499999999999995},{"interface":"front","average":{"1min":0.67400000000000004,"5min":0.68400000000000005,"15min":0.68600000000000005},"min":{"1min":0.38800000000000001,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":2.9809999999999999,"5min":2.9809999999999999,"15min":2.9809999999999999},"last":0.57699999999999996}]},{"osd":5,"last update":"Sat Sep 23 20:42:02 2023","interfaces":[{"interface":"back","average":{"1min":0.68700000000000006,"5min":0.627,"15min":0.61699999999999999},"min":{"1min":0.432,"5min":0.432,"15min":0.432},"max":{"1min":2.948,"5min":2.948,"15min":2.948},"last":1.0369999999999999},{"interface":"front","average":{"1min":0.67200000000000004,"5min":0.624,"15min":0.61599999999999999},"min":{"1min":0.379,"5min":0.379,"15min":0.379},"max":{"1min":2.2280000000000002,"5min":2.2280000000000002,"15min":2.2280000000000002},"last":1.075}]},{"osd":6,"last update":"Sat Sep 23 20:42:22 2023","interfaces":[{"interface":"back","average":{"1min":0.67300000000000004,"5min":0.61499999999999999,"15min":0.60499999999999998},"min":{"1min":0.41599999999999998,"5min":0.41599999999999998,"15min":0.41599999999999998},"max":{"1min":2.2989999999999999,"5min":2.2989999999999999,"15min":2.2989999999999999},"last":0.621},{"interface":"front","average":{"1min":0.69999999999999996,"5min":0.66400000000000003,"15min":0.65800000000000003},"min":{"1min":0.44500000000000001,"5min":0.443,"15min":0.443},"max":{"1min":2.407,"5min":2.407,"15min":2.407},"last":0.77500000000000002}]},{"osd":7,"last update":"Sat Sep 23 20:41:40 2023","interfaces":[{"interface":"back","average":{"1min":0.69499999999999995,"5min":0.69499999999999995,"15min":0.69499999999999995},"min":{"1min":0.41999999999999998,"5min":0.41999999999999998,"15min":0.41999999999999998},"max":{"1min":2.3700000000000001,"5min":2.3700000000000001,"15min":2.3700000000000001},"last":0.73699999999999999},{"interface":"front","average":{"1min":0.70399999999999996,"5min":0.70399999999999996,"15min":0.70399999999999996},"min":{"1min":0.46400000000000002,"5min":0.46400000000000002,"15min":0.46400000000000002},"max":{"1min":2.27,"5min":2.27,"15min":2.27},"last":0.66600000000000004}]}]},{"osd":3,"up_from":26,"seq":111669149738,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6148,"kb_used_data":700,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":93739004,"statfs":{"total":95995035648,"available":95988740096,"internally_reserved":0,"allocated":716800,"data_stored":309655,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:42:23 2023","interfaces":[{"interface":"back","average":{"1min":0.36699999999999999,"5min":0.40699999999999997,"15min":0.41999999999999998},"min":{"1min":0.23699999999999999,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":0.625,"5min":0.79100000000000004,"15min":0.79100000000000004},"last":0.311},{"interface":"front","average":{"1min":0.46200000000000002,"5min":0.48099999999999998,"15min":0.503},"min":{"1min":0.26000000000000001,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":1.7070000000000001,"5min":1.7070000000000001,"15min":1.7070000000000001},"last":0.40600000000000003}]},{"osd":1,"last update":"Sat Sep 23 20:42:23 2023","interfaces":[{"interface":"back","average":{"1min":0.442,"5min":0.47199999999999998,"15min":0.48799999999999999},"min":{"1min":0.27400000000000002,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":1.724,"5min":1.724,"15min":1.724},"last":0.34100000000000003},{"interface":"front","average":{"1min":0.41399999999999998,"5min":0.48999999999999999,"15min":0.51400000000000001},"min":{"1min":0.26200000000000001,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":0.72099999999999997,"5min":0.72899999999999998,"15min":0.72899999999999998},"last":0.36199999999999999}]},{"osd":2,"last update":"Sat Sep 23 20:42:23 2023","interfaces":[{"interface":"back","average":{"1min":0.39900000000000002,"5min":0.501,"15min":0.52900000000000003},"min":{"1min":0.307,"5min":0.307,"15min":0.307},"max":{"1min":0.79700000000000004,"5min":0.79800000000000004,"15min":0.79800000000000004},"last":0.442},{"interface":"front","average":{"1min":0.41399999999999998,"5min":0.48799999999999999,"15min":0.51300000000000001},"min":{"1min":0.30599999999999999,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":0.77400000000000002,"5min":0.77400000000000002,"15min":0.77400000000000002},"last":0.42199999999999999}]},{"osd":4,"last update":"Sat Sep 23 20:41:41 2023","interfaces":[{"interface":"back","average":{"1min":0.59499999999999997,"5min":0.65200000000000002,"15min":0.66200000000000003},"min":{"1min":0.36699999999999999,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":0.88,"5min":0.88100000000000001,"15min":0.88100000000000001},"last":0.48199999999999998},{"interface":"front","average":{"1min":0.60699999999999998,"5min":0.64200000000000002,"15min":0.64800000000000002},"min":{"1min":0.40699999999999997,"5min":0.40699999999999997,"15min":0.40699999999999997},"max":{"1min":0.90900000000000003,"5min":0.90900000000000003,"15min":0.90900000000000003},"last":0.58799999999999997}]},{"osd":5,"last update":"Sat Sep 23 20:42:03 2023","interfaces":[{"interface":"back","average":{"1min":0.53700000000000003,"5min":0.63100000000000001,"15min":0.64700000000000002},"min":{"1min":0.28299999999999997,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":0.84099999999999997,"5min":0.97499999999999998,"15min":0.97499999999999998},"last":0.53100000000000003},{"interface":"front","average":{"1min":0.56699999999999995,"5min":0.65800000000000003,"15min":0.67300000000000004},"min":{"1min":0.38600000000000001,"5min":0.38600000000000001,"15min":0.38600000000000001},"max":{"1min":0.83499999999999996,"5min":0.91500000000000004,"15min":0.91500000000000004},"last":0.60699999999999998}]},{"osd":6,"last update":"Sat Sep 23 20:42:23 2023","interfaces":[{"interface":"back","average":{"1min":0.58599999999999997,"5min":0.65000000000000002,"15min":0.66000000000000003},"min":{"1min":0.47099999999999997,"5min":0.39600000000000002,"15min":0.39600000000000002},"max":{"1min":0.82499999999999996,"5min":0.97299999999999998,"15min":0.97299999999999998},"last":0.61899999999999999},{"interface":"front","average":{"1min":0.58299999999999996,"5min":0.625,"15min":0.63200000000000001},"min":{"1min":0.41399999999999998,"5min":0.38400000000000001,"15min":0.38400000000000001},"max":{"1min":0.85099999999999998,"5min":0.92700000000000005,"15min":0.92700000000000005},"last":0.55800000000000005}]},{"osd":7,"last update":"Sat Sep 23 20:41:41 2023","interfaces":[{"interface":"back","average":{"1min":0.66700000000000004,"5min":0.66700000000000004,"15min":0.66700000000000004},"min":{"1min":0.47599999999999998,"5min":0.47599999999999998,"15min":0.47599999999999998},"max":{"1min":0.93700000000000006,"5min":0.93700000000000006,"15min":0.93700000000000006},"last":0.56599999999999995},{"interface":"front","average":{"1min":0.65300000000000002,"5min":0.65300000000000002,"15min":0.65300000000000002},"min":{"1min":0.40300000000000002,"5min":0.40300000000000002,"15min":0.40300000000000002},"max":{"1min":1.0049999999999999,"5min":1.0049999999999999,"15min":1.0049999999999999},"last":0.67000000000000004}]}]},{"osd":4,"up_from":31,"seq":133143986214,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6232,"kb_used_data":720,"kb_used_omap":0,"kb_used_meta":5504,"kb_avail":93738920,"statfs":{"total":95995035648,"available":95988654080,"internally_reserved":0,"allocated":737280,"data_stored":311247,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5636096},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:41:39 2023","interfaces":[{"interface":"back","average":{"1min":0.46500000000000002,"5min":0.52100000000000002,"15min":0.53000000000000003},"min":{"1min":0.315,"5min":0.31,"15min":0.31},"max":{"1min":0.72599999999999998,"5min":0.996,"15min":0.996},"last":0.628},{"interface":"front","average":{"1min":0.57099999999999995,"5min":0.54600000000000004,"15min":0.54200000000000004},"min":{"1min":0.42399999999999999,"5min":0.41199999999999998,"15min":0.41199999999999998},"max":{"1min":0.77100000000000002,"5min":0.871,"15min":0.871},"last":0.56399999999999995}]},{"osd":1,"last update":"Sat Sep 23 20:41:39 2023","interfaces":[{"interface":"back","average":{"1min":0.503,"5min":0.53900000000000003,"15min":0.54500000000000004},"min":{"1min":0.33200000000000002,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":0.72199999999999998,"5min":0.88900000000000001,"15min":0.88900000000000001},"last":0.502},{"interface":"front","average":{"1min":0.53500000000000003,"5min":0.52000000000000002,"15min":0.51800000000000002},"min":{"1min":0.32700000000000001,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":0.79600000000000004,"5min":1.0289999999999999,"15min":1.0289999999999999},"last":0.61099999999999999}]},{"osd":2,"last update":"Sat Sep 23 20:41:39 2023","interfaces":[{"interface":"back","average":{"1min":0.53600000000000003,"5min":0.55400000000000005,"15min":0.55700000000000005},"min":{"1min":0.33700000000000002,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":0.79700000000000004,"5min":1.0049999999999999,"15min":1.0049999999999999},"last":0.51700000000000002},{"interface":"front","average":{"1min":0.53100000000000003,"5min":0.53400000000000003,"15min":0.53400000000000003},"min":{"1min":0.40200000000000002,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":0.81100000000000005,"5min":0.85399999999999998,"15min":0.85399999999999998},"last":0.59599999999999997}]},{"osd":3,"last update":"Sat Sep 23 20:41:39 2023","interfaces":[{"interface":"back","average":{"1min":0.53900000000000003,"5min":0.56399999999999995,"15min":0.56799999999999995},"min":{"1min":0.35799999999999998,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":0.82999999999999996,"5min":0.90100000000000002,"15min":0.90100000000000002},"last":0.64800000000000002},{"interface":"front","average":{"1min":0.55000000000000004,"5min":0.55400000000000005,"15min":0.55500000000000005},"min":{"1min":0.38700000000000001,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":0.76900000000000002,"5min":0.91500000000000004,"15min":0.91500000000000004},"last":0.54900000000000004}]},{"osd":5,"last update":"Sat Sep 23 20:42:03 2023","interfaces":[{"interface":"back","average":{"1min":0.52900000000000003,"5min":0.41199999999999998,"15min":0.39200000000000002},"min":{"1min":0.35099999999999998,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":0.83599999999999997,"5min":0.83599999999999997,"15min":0.83599999999999997},"last":0.39200000000000002},{"interface":"front","average":{"1min":0.52600000000000002,"5min":0.41699999999999998,"15min":0.39900000000000002},"min":{"1min":0.35799999999999998,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":0.90000000000000002,"5min":0.90000000000000002,"15min":0.90000000000000002},"last":0.45800000000000002}]},{"osd":6,"last update":"Sat Sep 23 20:42:22 2023","interfaces":[{"interface":"back","average":{"1min":0.47999999999999998,"5min":0.51800000000000002,"15min":0.52400000000000002},"min":{"1min":0.33600000000000002,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":0.80700000000000005,"5min":1.1319999999999999,"15min":1.1319999999999999},"last":0.58099999999999996},{"interface":"front","average":{"1min":0.56599999999999995,"5min":0.57299999999999995,"15min":0.57399999999999995},"min":{"1min":0.35399999999999998,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":0.85399999999999998,"5min":1.095,"15min":1.095},"last":0.48199999999999998}]},{"osd":7,"last update":"Sat Sep 23 20:41:39 2023","interfaces":[{"interface":"back","average":{"1min":0.55500000000000005,"5min":0.55500000000000005,"15min":0.55500000000000005},"min":{"1min":0.32700000000000001,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":0.77700000000000002,"5min":0.77700000000000002,"15min":0.77700000000000002},"last":0.44600000000000001},{"interface":"front","average":{"1min":0.57099999999999995,"5min":0.57099999999999995,"15min":0.57099999999999995},"min":{"1min":0.32700000000000001,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":0.86299999999999999,"5min":0.86299999999999999,"15min":0.86299999999999999},"last":0.52700000000000002}]}]},{"osd":5,"up_from":39,"seq":167503724579,"num_pgs":21,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6088,"kb_used_data":704,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":93739064,"statfs":{"total":95995035648,"available":95988801536,"internally_reserved":0,"allocated":720896,"data_stored":310189,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:42:00 2023","interfaces":[{"interface":"back","average":{"1min":0.46800000000000003,"5min":0.52500000000000002,"15min":0.53500000000000003},"min":{"1min":0.29199999999999998,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":0.63100000000000001,"5min":0.82299999999999995,"15min":0.82299999999999995},"last":0.58399999999999996},{"interface":"front","average":{"1min":0.46700000000000003,"5min":0.54400000000000004,"15min":0.55700000000000005},"min":{"1min":0.30499999999999999,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":0.78300000000000003,"5min":0.79600000000000004,"15min":0.79600000000000004},"last":0.60999999999999999}]},{"osd":1,"last update":"Sat Sep 23 20:42:00 2023","interfaces":[{"interface":"back","average":{"1min":0.49199999999999999,"5min":0.55200000000000005,"15min":0.56200000000000006},"min":{"1min":0.32400000000000001,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":0.65700000000000003,"5min":0.81499999999999995,"15min":0.81499999999999995},"last":0.59499999999999997},{"interface":"front","average":{"1min":0.48199999999999998,"5min":0.52100000000000002,"15min":0.52700000000000002},"min":{"1min":0.315,"5min":0.315,"15min":0.315},"max":{"1min":0.73099999999999998,"5min":0.76900000000000002,"15min":0.76900000000000002},"last":0.45400000000000001}]},{"osd":2,"last update":"Sat Sep 23 20:42:00 2023","interfaces":[{"interface":"back","average":{"1min":0.51400000000000001,"5min":0.57799999999999996,"15min":0.58799999999999997},"min":{"1min":0.32900000000000001,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":0.67000000000000004,"5min":0.79100000000000004,"15min":0.79100000000000004},"last":0.47599999999999998},{"interface":"front","average":{"1min":0.51200000000000001,"5min":0.54900000000000004,"15min":0.55500000000000005},"min":{"1min":0.375,"5min":0.375,"15min":0.375},"max":{"1min":0.66200000000000003,"5min":0.77700000000000002,"15min":0.77700000000000002},"last":0.42599999999999999}]},{"osd":3,"last update":"Sat Sep 23 20:42:00 2023","interfaces":[{"interface":"back","average":{"1min":0.51600000000000001,"5min":0.56499999999999995,"15min":0.57299999999999995},"min":{"1min":0.35999999999999999,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":0.69399999999999995,"5min":0.86499999999999999,"15min":0.86499999999999999},"last":0.54900000000000004},{"interface":"front","average":{"1min":0.54800000000000004,"5min":0.57599999999999996,"15min":0.58099999999999996},"min":{"1min":0.39300000000000002,"5min":0.39300000000000002,"15min":0.39300000000000002},"max":{"1min":0.70299999999999996,"5min":0.85099999999999998,"15min":0.85099999999999998},"last":0.625}]},{"osd":4,"last update":"Sat Sep 23 20:42:00 2023","interfaces":[{"interface":"back","average":{"1min":0.46899999999999997,"5min":0.42299999999999999,"15min":0.41499999999999998},"min":{"1min":0.34899999999999998,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":0.67200000000000004,"5min":0.67200000000000004,"15min":0.67200000000000004},"last":0.50900000000000001},{"interface":"front","average":{"1min":0.51400000000000001,"5min":0.47599999999999998,"15min":0.46999999999999997},"min":{"1min":0.318,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":0.76800000000000002,"5min":0.82799999999999996,"15min":0.82799999999999996},"last":0.63700000000000001}]},{"osd":6,"last update":"Sat Sep 23 20:42:26 2023","interfaces":[{"interface":"back","average":{"1min":0.504,"5min":0.47799999999999998,"15min":0.47399999999999998},"min":{"1min":0.313,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":0.746,"5min":0.746,"15min":0.746},"last":0.52700000000000002},{"interface":"front","average":{"1min":0.52400000000000002,"5min":0.52700000000000002,"15min":0.52700000000000002},"min":{"1min":0.38,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":0.75600000000000001,"5min":0.82299999999999995,"15min":0.82299999999999995},"last":0.56399999999999995}]},{"osd":7,"last update":"Sat Sep 23 20:41:41 2023","interfaces":[{"interface":"back","average":{"1min":0.55200000000000005,"5min":0.55200000000000005,"15min":0.55200000000000005},"min":{"1min":0.32200000000000001,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":0.79800000000000004,"5min":0.79800000000000004,"15min":0.79800000000000004},"last":0.66400000000000003},{"interface":"front","average":{"1min":0.56399999999999995,"5min":0.56399999999999995,"15min":0.56399999999999995},"min":{"1min":0.375,"5min":0.375,"15min":0.375},"max":{"1min":0.84999999999999998,"5min":0.84999999999999998,"15min":0.84999999999999998},"last":0.65000000000000002}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":1682,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":1606,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":1752,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T20:42:36.855 INFO:tasks.cephadm.ceph_manager.ceph:clean! 2023-09-23T20:42:36.855 INFO:tasks.ceph:Waiting until ceph cluster ceph is healthy... 2023-09-23T20:42:36.855 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy 2023-09-23T20:42:36.856 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph health --format=json 2023-09-23T20:42:36.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:42:36 smithi070 ceph-mon[100755]: pgmap v59: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:42:37.460 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:42:37.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:42:37 smithi070 ceph-mon[100755]: from='client.24686 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:42:37.919 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:42:37 smithi026 ceph-mon[95839]: from='client.24686 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:42:37.920 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:42:37 smithi026 ceph-mon[109912]: from='client.24686 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:42:38.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:42:38 smithi070 ceph-mon[100755]: pgmap v60: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:42:38.881 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:42:38 smithi026 ceph-mon[95839]: pgmap v60: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:42:38.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:42:38 smithi026 ceph-mon[109912]: pgmap v60: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:42:38.882 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:42:38 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[96086]: ::ffff:172.21.15.70 - - [23/Sep/2023:20:42:38] "GET /metrics HTTP/1.1" 200 238609 "" "Prometheus/2.43.0" 2023-09-23T20:42:38.917 INFO:teuthology.orchestra.run.smithi026.stdout: 2023-09-23T20:42:38.917 INFO:teuthology.orchestra.run.smithi026.stdout:{"status":"HEALTH_OK","checks":{},"mutes":[]} 2023-09-23T20:42:39.323 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy done 2023-09-23T20:42:39.323 INFO:teuthology.run_tasks:Running task print... 2023-09-23T20:42:39.337 INFO:teuthology.task.print:**** upgrading first half of cluster, with stress **** 2023-09-23T20:42:39.337 INFO:teuthology.run_tasks:Running task parallel... 2023-09-23T20:42:39.359 INFO:teuthology.task.parallel:starting parallel... 2023-09-23T20:42:39.359 INFO:teuthology.task.parallel:In parallel, running task sequential... 2023-09-23T20:42:39.360 INFO:teuthology.task.sequential:In sequential, running task workunit... 2023-09-23T20:42:39.363 INFO:tasks.workunit:Pulling workunits from ref quincy 2023-09-23T20:42:39.363 INFO:tasks.workunit:Making a separate scratch dir for every client... 2023-09-23T20:42:39.363 DEBUG:teuthology.orchestra.run.smithi026:> stat -- /home/ubuntu/cephtest/mnt.0 2023-09-23T20:42:39.364 INFO:teuthology.task.parallel:In parallel, running task sequential... 2023-09-23T20:42:39.364 INFO:teuthology.task.sequential:In sequential, running task cephadm.shell... 2023-09-23T20:42:39.364 INFO:tasks.cephadm:Running commands on role mon.a host ubuntu@smithi026.front.sepia.ceph.com 2023-09-23T20:42:39.364 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -e sha1=bb0daeff97f020f686596fca6eba20c310e26fb7 -- bash -c 'ceph config set mgr mgr/cephadm/daemon_cache_timeout 60' 2023-09-23T20:42:39.383 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:42:39.383 INFO:teuthology.orchestra.run.smithi026.stderr:stat: cannot statx '/home/ubuntu/cephtest/mnt.0': No such file or directory 2023-09-23T20:42:39.383 DEBUG:teuthology.orchestra.run.smithi026:> mkdir -- /home/ubuntu/cephtest/mnt.0 2023-09-23T20:42:39.440 INFO:tasks.workunit:Created dir /home/ubuntu/cephtest/mnt.0 2023-09-23T20:42:39.440 DEBUG:teuthology.orchestra.run.smithi026:> cd -- /home/ubuntu/cephtest/mnt.0 && mkdir -- client.0 2023-09-23T20:42:39.497 INFO:tasks.workunit:timeout=3h 2023-09-23T20:42:39.497 INFO:tasks.workunit:cleanup=True 2023-09-23T20:42:39.498 DEBUG:teuthology.orchestra.run.smithi026:> rm -rf /home/ubuntu/cephtest/clone.client.0 && git clone --depth 1 --branch quincy https://github.com/ceph/ceph.git /home/ubuntu/cephtest/clone.client.0 && cd /home/ubuntu/cephtest/clone.client.0 2023-09-23T20:42:39.554 INFO:tasks.workunit.client.0.smithi026.stderr:Cloning into '/home/ubuntu/cephtest/clone.client.0'... 2023-09-23T20:42:39.586 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:42:39 smithi026 ceph-mon[95839]: from='client.? 172.21.15.26:0/2244557918' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2023-09-23T20:42:39.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:42:39 smithi070 ceph-mon[100755]: from='client.? 172.21.15.26:0/2244557918' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2023-09-23T20:42:39.881 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:42:39 smithi026 ceph-mon[109912]: from='client.? 172.21.15.26:0/2244557918' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2023-09-23T20:42:39.975 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:42:40.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:42:40 smithi070 ceph-mon[100755]: pgmap v61: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:42:40.881 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:42:40 smithi026 ceph-mon[95839]: pgmap v61: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:42:40.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:42:40 smithi026 ceph-mon[109912]: pgmap v61: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:42:41.707 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:42:41 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[96086]: [23/Sep/2023:20:42:41] ENGINE Bus STOPPING 2023-09-23T20:42:41.708 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:42:41 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[96086]: [23/Sep/2023:20:42:41] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2023-09-23T20:42:41.708 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:42:41 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[96086]: [23/Sep/2023:20:42:41] ENGINE Bus STOPPED 2023-09-23T20:42:41.708 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:42:41 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[96086]: [23/Sep/2023:20:42:41] ENGINE Bus STARTING 2023-09-23T20:42:41.708 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:42:41 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[96086]: [23/Sep/2023:20:42:41] ENGINE Serving on http://:::9283 2023-09-23T20:42:41.708 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:42:41 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[96086]: [23/Sep/2023:20:42:41] ENGINE Bus STARTED 2023-09-23T20:42:41.869 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 20:42:41 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-x[102861]: ::ffff:172.21.15.70 - - [23/Sep/2023:20:42:41] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2023-09-23T20:42:42.030 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -e sha1=bb0daeff97f020f686596fca6eba20c310e26fb7 -- bash -c 'ceph config set global log_to_journald false --force' 2023-09-23T20:42:42.682 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:42:42.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:42:42 smithi070 ceph-mon[100755]: from='client.? ' entity='client.admin' 2023-09-23T20:42:42.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:42:42 smithi070 ceph-mon[100755]: from='mgr.14451 172.21.15.26:0/1329296977' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:42:42.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:42:42 smithi070 ceph-mon[100755]: from='mgr.14451 172.21.15.26:0/1329296977' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:42:42.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:42:42 smithi070 ceph-mon[100755]: from='mgr.14451 172.21.15.26:0/1329296977' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:42:42.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:42:42 smithi070 ceph-mon[100755]: from='mgr.14451 ' entity='mgr.y' 2023-09-23T20:42:42.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:42:42 smithi070 ceph-mon[100755]: pgmap v62: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:42:42.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:42:42 smithi026 ceph-mon[95839]: from='client.? ' entity='client.admin' 2023-09-23T20:42:42.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:42:42 smithi026 ceph-mon[95839]: from='mgr.14451 172.21.15.26:0/1329296977' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:42:42.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:42:42 smithi026 ceph-mon[95839]: from='mgr.14451 172.21.15.26:0/1329296977' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:42:42.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:42:42 smithi026 ceph-mon[95839]: from='mgr.14451 172.21.15.26:0/1329296977' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:42:42.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:42:42 smithi026 ceph-mon[95839]: from='mgr.14451 ' entity='mgr.y' 2023-09-23T20:42:42.883 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:42:42 smithi026 ceph-mon[95839]: pgmap v62: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:42:42.883 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:42:42 smithi026 ceph-mon[109912]: from='client.? ' entity='client.admin' 2023-09-23T20:42:42.883 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:42:42 smithi026 ceph-mon[109912]: from='mgr.14451 172.21.15.26:0/1329296977' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:42:42.883 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:42:42 smithi026 ceph-mon[109912]: from='mgr.14451 172.21.15.26:0/1329296977' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:42:42.884 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:42:42 smithi026 ceph-mon[109912]: from='mgr.14451 172.21.15.26:0/1329296977' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:42:42.884 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:42:42 smithi026 ceph-mon[109912]: from='mgr.14451 ' entity='mgr.y' 2023-09-23T20:42:42.884 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:42:42 smithi026 ceph-mon[109912]: pgmap v62: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:42:44.077 DEBUG:teuthology.orchestra.run.smithi026:> cd -- /home/ubuntu/cephtest/clone.client.0/qa/workunits && if test -e Makefile ; then make ; fi && find -executable -type f -printf '%P\0' >/home/ubuntu/cephtest/workunits.list.client.0 2023-09-23T20:42:44.114 INFO:tasks.workunit.client.0.smithi026.stdout:for d in direct_io fs ; do ( cd $d ; make all ) ; done 2023-09-23T20:42:44.116 INFO:tasks.workunit.client.0.smithi026.stdout:make[1]: Entering directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/direct_io' 2023-09-23T20:42:44.116 INFO:tasks.workunit.client.0.smithi026.stdout:cc -Wall -Wextra -D_GNU_SOURCE direct_io_test.c -o direct_io_test 2023-09-23T20:42:44.572 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:42:44 smithi026 ceph-mon[95839]: pgmap v63: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:42:44.573 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:42:44 smithi026 ceph-mon[109912]: pgmap v63: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:42:44.794 INFO:tasks.workunit.client.0.smithi026.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_sync_io.c -o test_sync_io 2023-09-23T20:42:44.845 INFO:tasks.workunit.client.0.smithi026.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_short_dio_read.c -o test_short_dio_read 2023-09-23T20:42:44.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:42:44 smithi070 ceph-mon[100755]: pgmap v63: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:42:44.886 INFO:tasks.workunit.client.0.smithi026.stdout:make[1]: Leaving directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/direct_io' 2023-09-23T20:42:44.888 INFO:tasks.workunit.client.0.smithi026.stdout:make[1]: Entering directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/fs' 2023-09-23T20:42:44.888 INFO:tasks.workunit.client.0.smithi026.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_o_trunc.c -o test_o_trunc 2023-09-23T20:42:44.927 INFO:tasks.workunit.client.0.smithi026.stdout:make[1]: Leaving directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/fs' 2023-09-23T20:42:44.931 DEBUG:teuthology.orchestra.run.smithi026:> set -ex 2023-09-23T20:42:44.931 DEBUG:teuthology.orchestra.run.smithi026:> dd if=/home/ubuntu/cephtest/workunits.list.client.0 of=/dev/stdout 2023-09-23T20:42:44.989 INFO:tasks.workunit:Running workunits matching cls/test_cls_rbd.sh on client.0... 2023-09-23T20:42:44.990 INFO:tasks.workunit:Running workunit cls/test_cls_rbd.sh... 2023-09-23T20:42:44.990 DEBUG:teuthology.orchestra.run.smithi026:workunit test cls/test_cls_rbd.sh> mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=quincy TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/cls/test_cls_rbd.sh 2023-09-23T20:42:45.059 INFO:tasks.workunit.client.0.smithi026.stdout:Running main() from gmock_main.cc 2023-09-23T20:42:45.059 INFO:tasks.workunit.client.0.smithi026.stdout:[==========] Running 70 tests from 1 test suite. 2023-09-23T20:42:45.059 INFO:tasks.workunit.client.0.smithi026.stdout:[----------] Global test environment set-up. 2023-09-23T20:42:45.059 INFO:tasks.workunit.client.0.smithi026.stdout:[----------] 70 tests from TestClsRbd 2023-09-23T20:42:45.255 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -e sha1=bb0daeff97f020f686596fca6eba20c310e26fb7 -- bash -c 'ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1' 2023-09-23T20:42:45.901 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:42:46.585 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] TestClsRbd.get_all_features 2023-09-23T20:42:46.600 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] TestClsRbd.get_all_features (15 ms) 2023-09-23T20:42:46.600 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] TestClsRbd.copyup 2023-09-23T20:42:46.784 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] TestClsRbd.copyup (184 ms) 2023-09-23T20:42:46.784 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] TestClsRbd.sparse_copyup 2023-09-23T20:42:46.795 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] TestClsRbd.sparse_copyup (12 ms) 2023-09-23T20:42:46.796 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] TestClsRbd.get_and_set_id 2023-09-23T20:42:46.803 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] TestClsRbd.get_and_set_id (7 ms) 2023-09-23T20:42:46.803 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] TestClsRbd.add_remove_child 2023-09-23T20:42:46.813 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] TestClsRbd.add_remove_child (10 ms) 2023-09-23T20:42:46.813 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] TestClsRbd.directory_methods 2023-09-23T20:42:46.839 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] TestClsRbd.directory_methods (26 ms) 2023-09-23T20:42:46.839 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] TestClsRbd.create 2023-09-23T20:42:46.853 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] TestClsRbd.create (14 ms) 2023-09-23T20:42:46.853 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] TestClsRbd.get_features 2023-09-23T20:42:46.854 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] TestClsRbd.get_features (1 ms) 2023-09-23T20:42:46.854 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] TestClsRbd.get_object_prefix 2023-09-23T20:42:46.855 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] TestClsRbd.get_object_prefix (2 ms) 2023-09-23T20:42:46.856 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] TestClsRbd.get_create_timestamp 2023-09-23T20:42:46.857 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] TestClsRbd.get_create_timestamp (2 ms) 2023-09-23T20:42:46.857 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] TestClsRbd.get_access_timestamp 2023-09-23T20:42:46.858 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] TestClsRbd.get_access_timestamp (1 ms) 2023-09-23T20:42:46.858 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] TestClsRbd.get_modify_timestamp 2023-09-23T20:42:46.860 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] TestClsRbd.get_modify_timestamp (1 ms) 2023-09-23T20:42:46.860 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] TestClsRbd.get_data_pool 2023-09-23T20:42:46.863 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] TestClsRbd.get_data_pool (4 ms) 2023-09-23T20:42:46.863 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] TestClsRbd.get_size 2023-09-23T20:42:46.867 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] TestClsRbd.get_size (3 ms) 2023-09-23T20:42:46.867 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] TestClsRbd.set_size 2023-09-23T20:42:46.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:42:46 smithi070 ceph-mon[100755]: osdmap e71: 8 total, 8 up, 8 in 2023-09-23T20:42:46.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:42:46 smithi070 ceph-mon[100755]: from='client.? 172.21.15.26:0/326794743' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-158336-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2023-09-23T20:42:46.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:42:46 smithi070 ceph-mon[100755]: pgmap v65: 105 pgs: 32 unknown, 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:42:46.871 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] TestClsRbd.set_size (5 ms) 2023-09-23T20:42:46.871 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] TestClsRbd.protection_status 2023-09-23T20:42:46.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:42:46 smithi026 ceph-mon[95839]: osdmap e71: 8 total, 8 up, 8 in 2023-09-23T20:42:46.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:42:46 smithi026 ceph-mon[95839]: from='client.? 172.21.15.26:0/326794743' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-158336-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2023-09-23T20:42:46.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:42:46 smithi026 ceph-mon[95839]: pgmap v65: 105 pgs: 32 unknown, 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:42:46.883 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:42:46 smithi026 ceph-mon[109912]: osdmap e71: 8 total, 8 up, 8 in 2023-09-23T20:42:46.883 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:42:46 smithi026 ceph-mon[109912]: from='client.? 172.21.15.26:0/326794743' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-158336-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2023-09-23T20:42:46.883 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:42:46 smithi026 ceph-mon[109912]: pgmap v65: 105 pgs: 32 unknown, 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:42:46.888 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] TestClsRbd.protection_status (17 ms) 2023-09-23T20:42:46.888 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] TestClsRbd.snapshot_limits 2023-09-23T20:42:46.895 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] TestClsRbd.snapshot_limits (7 ms) 2023-09-23T20:42:46.895 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] TestClsRbd.parents_v1 2023-09-23T20:42:46.939 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] TestClsRbd.parents_v1 (44 ms) 2023-09-23T20:42:46.939 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] TestClsRbd.parents_v2 2023-09-23T20:42:46.969 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] TestClsRbd.parents_v2 (30 ms) 2023-09-23T20:42:46.969 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] TestClsRbd.snapshots 2023-09-23T20:42:46.990 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] TestClsRbd.snapshots (20 ms) 2023-09-23T20:42:46.990 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] TestClsRbd.snapid_race 2023-09-23T20:42:46.994 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] TestClsRbd.snapid_race (5 ms) 2023-09-23T20:42:46.994 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] TestClsRbd.stripingv2 2023-09-23T20:42:47.003 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] TestClsRbd.stripingv2 (8 ms) 2023-09-23T20:42:47.003 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] TestClsRbd.object_map_save 2023-09-23T20:42:47.004 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] TestClsRbd.object_map_save (2 ms) 2023-09-23T20:42:47.005 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] TestClsRbd.object_map_resize 2023-09-23T20:42:47.009 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] TestClsRbd.object_map_resize (4 ms) 2023-09-23T20:42:47.009 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] TestClsRbd.object_map_update 2023-09-23T20:42:47.013 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] TestClsRbd.object_map_update (4 ms) 2023-09-23T20:42:47.013 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] TestClsRbd.object_map_load_enoent 2023-09-23T20:42:47.013 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] TestClsRbd.object_map_load_enoent (1 ms) 2023-09-23T20:42:47.014 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] TestClsRbd.object_map_snap_add 2023-09-23T20:42:47.017 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] TestClsRbd.object_map_snap_add (3 ms) 2023-09-23T20:42:47.017 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] TestClsRbd.object_map_snap_remove 2023-09-23T20:42:47.021 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] TestClsRbd.object_map_snap_remove (4 ms) 2023-09-23T20:42:47.021 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] TestClsRbd.flags 2023-09-23T20:42:47.026 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] TestClsRbd.flags (5 ms) 2023-09-23T20:42:47.026 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] TestClsRbd.metadata 2023-09-23T20:42:47.050 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] TestClsRbd.metadata (24 ms) 2023-09-23T20:42:47.050 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] TestClsRbd.set_features 2023-09-23T20:42:47.056 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] TestClsRbd.set_features (6 ms) 2023-09-23T20:42:47.056 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] TestClsRbd.mirror 2023-09-23T20:42:47.087 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] TestClsRbd.mirror (31 ms) 2023-09-23T20:42:47.087 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] TestClsRbd.mirror_image 2023-09-23T20:42:47.099 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] TestClsRbd.mirror_image (13 ms) 2023-09-23T20:42:47.100 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] TestClsRbd.mirror_image_status 2023-09-23T20:42:47.429 INFO:teuthology.orchestra.run.smithi026.stdout:Initiating upgrade to quay.ceph.io/ceph-ci/ceph:bb0daeff97f020f686596fca6eba20c310e26fb7 2023-09-23T20:42:47.827 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:42:47 smithi026 ceph-mon[109912]: from='client.? 172.21.15.26:0/326794743' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-158336-1","app": "rados","yes_i_really_mean_it": true}]': finished 2023-09-23T20:42:47.827 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:42:47 smithi026 ceph-mon[109912]: osdmap e72: 8 total, 8 up, 8 in 2023-09-23T20:42:47.827 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:42:47 smithi026 ceph-mon[109912]: from='mgr.14451 ' entity='mgr.y' 2023-09-23T20:42:47.827 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:42:47 smithi026 ceph-mon[109912]: from='mgr.14451 172.21.15.26:0/1329296977' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:42:47.827 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:42:47 smithi026 ceph-mon[109912]: from='mgr.14451 172.21.15.26:0/1329296977' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:42:47.828 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:42:47 smithi026 ceph-mon[109912]: from='mgr.14451 172.21.15.26:0/1329296977' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:42:47.828 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:42:47 smithi026 ceph-mon[109912]: from='mgr.14451 ' entity='mgr.y' 2023-09-23T20:42:47.828 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:42:47 smithi026 ceph-mon[95839]: from='client.? 172.21.15.26:0/326794743' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-158336-1","app": "rados","yes_i_really_mean_it": true}]': finished 2023-09-23T20:42:47.828 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:42:47 smithi026 ceph-mon[95839]: osdmap e72: 8 total, 8 up, 8 in 2023-09-23T20:42:47.828 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:42:47 smithi026 ceph-mon[95839]: from='mgr.14451 ' entity='mgr.y' 2023-09-23T20:42:47.828 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:42:47 smithi026 ceph-mon[95839]: from='mgr.14451 172.21.15.26:0/1329296977' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:42:47.829 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:42:47 smithi026 ceph-mon[95839]: from='mgr.14451 172.21.15.26:0/1329296977' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:42:47.829 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:42:47 smithi026 ceph-mon[95839]: from='mgr.14451 172.21.15.26:0/1329296977' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:42:47.829 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:42:47 smithi026 ceph-mon[95839]: from='mgr.14451 ' entity='mgr.y' 2023-09-23T20:42:47.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:42:47 smithi070 ceph-mon[100755]: from='client.? 172.21.15.26:0/326794743' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-158336-1","app": "rados","yes_i_really_mean_it": true}]': finished 2023-09-23T20:42:47.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:42:47 smithi070 ceph-mon[100755]: osdmap e72: 8 total, 8 up, 8 in 2023-09-23T20:42:47.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:42:47 smithi070 ceph-mon[100755]: from='mgr.14451 ' entity='mgr.y' 2023-09-23T20:42:47.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:42:47 smithi070 ceph-mon[100755]: from='mgr.14451 172.21.15.26:0/1329296977' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:42:47.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:42:47 smithi070 ceph-mon[100755]: from='mgr.14451 172.21.15.26:0/1329296977' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:42:47.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:42:47 smithi070 ceph-mon[100755]: from='mgr.14451 172.21.15.26:0/1329296977' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:42:47.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:42:47 smithi070 ceph-mon[100755]: from='mgr.14451 ' entity='mgr.y' 2023-09-23T20:42:47.875 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -e sha1=bb0daeff97f020f686596fca6eba20c310e26fb7 -- bash -c 'ceph orch ps' 2023-09-23T20:42:48.531 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:42:48.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:42:48 smithi070 ceph-mon[100755]: from='client.24667 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:bb0daeff97f020f686596fca6eba20c310e26fb7", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T20:42:48.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:42:48 smithi070 ceph-mon[100755]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:bb0daeff97f020f686596fca6eba20c310e26fb7 2023-09-23T20:42:48.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:42:48 smithi070 ceph-mon[100755]: osdmap e73: 8 total, 8 up, 8 in 2023-09-23T20:42:48.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:42:48 smithi070 ceph-mon[100755]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:bb0daeff97f020f686596fca6eba20c310e26fb7 2023-09-23T20:42:48.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:42:48 smithi070 ceph-mon[100755]: pgmap v68: 105 pgs: 32 unknown, 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:42:48.881 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:42:48 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[96086]: ::ffff:172.21.15.70 - - [23/Sep/2023:20:42:48] "GET /metrics HTTP/1.1" 200 238609 "" "Prometheus/2.43.0" 2023-09-23T20:42:48.883 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:42:48 smithi026 ceph-mon[109912]: from='client.24667 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:bb0daeff97f020f686596fca6eba20c310e26fb7", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T20:42:48.883 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:42:48 smithi026 ceph-mon[109912]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:bb0daeff97f020f686596fca6eba20c310e26fb7 2023-09-23T20:42:48.884 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:42:48 smithi026 ceph-mon[109912]: osdmap e73: 8 total, 8 up, 8 in 2023-09-23T20:42:48.884 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:42:48 smithi026 ceph-mon[109912]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:bb0daeff97f020f686596fca6eba20c310e26fb7 2023-09-23T20:42:48.884 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:42:48 smithi026 ceph-mon[109912]: pgmap v68: 105 pgs: 32 unknown, 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:42:48.884 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:42:48 smithi026 ceph-mon[95839]: from='client.24667 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:bb0daeff97f020f686596fca6eba20c310e26fb7", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T20:42:48.884 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:42:48 smithi026 ceph-mon[95839]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:bb0daeff97f020f686596fca6eba20c310e26fb7 2023-09-23T20:42:48.884 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:42:48 smithi026 ceph-mon[95839]: osdmap e73: 8 total, 8 up, 8 in 2023-09-23T20:42:48.884 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:42:48 smithi026 ceph-mon[95839]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:bb0daeff97f020f686596fca6eba20c310e26fb7 2023-09-23T20:42:48.884 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:42:48 smithi026 ceph-mon[95839]: pgmap v68: 105 pgs: 32 unknown, 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:42:49.432 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] TestClsRbd.mirror_image_status (2332 ms) 2023-09-23T20:42:49.433 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] TestClsRbd.mirror_image_map 2023-09-23T20:42:49.515 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] TestClsRbd.mirror_image_map (84 ms) 2023-09-23T20:42:49.516 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] TestClsRbd.mirror_instances 2023-09-23T20:42:49.524 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] TestClsRbd.mirror_instances (9 ms) 2023-09-23T20:42:49.524 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] TestClsRbd.mirror_snapshot 2023-09-23T20:42:49.535 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] TestClsRbd.mirror_snapshot (11 ms) 2023-09-23T20:42:49.535 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] TestClsRbd.group_dir_list 2023-09-23T20:42:49.537 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] TestClsRbd.group_dir_list (2 ms) 2023-09-23T20:42:49.538 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] TestClsRbd.group_dir_add 2023-09-23T20:42:49.539 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] TestClsRbd.group_dir_add (2 ms) 2023-09-23T20:42:49.539 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] TestClsRbd.dir_add_already_existing 2023-09-23T20:42:49.542 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] TestClsRbd.dir_add_already_existing (3 ms) 2023-09-23T20:42:49.542 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] TestClsRbd.group_dir_rename 2023-09-23T20:42:49.547 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] TestClsRbd.group_dir_rename (5 ms) 2023-09-23T20:42:49.548 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] TestClsRbd.group_dir_remove 2023-09-23T20:42:49.551 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] TestClsRbd.group_dir_remove (3 ms) 2023-09-23T20:42:49.551 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] TestClsRbd.group_dir_remove_missing 2023-09-23T20:42:49.554 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] TestClsRbd.group_dir_remove_missing (4 ms) 2023-09-23T20:42:49.554 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] TestClsRbd.group_image_add 2023-09-23T20:42:49.558 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] TestClsRbd.group_image_add (2 ms) 2023-09-23T20:42:49.558 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] TestClsRbd.group_image_remove 2023-09-23T20:42:49.559 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] TestClsRbd.group_image_remove (3 ms) 2023-09-23T20:42:49.559 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] TestClsRbd.group_image_list 2023-09-23T20:42:49.563 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] TestClsRbd.group_image_list (3 ms) 2023-09-23T20:42:49.563 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] TestClsRbd.group_image_clean 2023-09-23T20:42:49.568 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] TestClsRbd.group_image_clean (5 ms) 2023-09-23T20:42:49.568 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] TestClsRbd.image_group_add 2023-09-23T20:42:49.570 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] TestClsRbd.image_group_add (3 ms) 2023-09-23T20:42:49.570 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] TestClsRbd.image_group_remove 2023-09-23T20:42:49.573 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] TestClsRbd.image_group_remove (3 ms) 2023-09-23T20:42:49.573 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] TestClsRbd.image_group_get 2023-09-23T20:42:49.576 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] TestClsRbd.image_group_get (2 ms) 2023-09-23T20:42:49.576 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] TestClsRbd.group_snap_set_empty_name 2023-09-23T20:42:49.578 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] TestClsRbd.group_snap_set_empty_name (2 ms) 2023-09-23T20:42:49.578 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] TestClsRbd.group_snap_set_empty_id 2023-09-23T20:42:49.579 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] TestClsRbd.group_snap_set_empty_id (2 ms) 2023-09-23T20:42:49.579 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] TestClsRbd.group_snap_set_duplicate_id 2023-09-23T20:42:49.582 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] TestClsRbd.group_snap_set_duplicate_id (3 ms) 2023-09-23T20:42:49.582 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] TestClsRbd.group_snap_set_duplicate_name 2023-09-23T20:42:49.585 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] TestClsRbd.group_snap_set_duplicate_name (2 ms) 2023-09-23T20:42:49.585 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] TestClsRbd.group_snap_set 2023-09-23T20:42:49.589 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] TestClsRbd.group_snap_set (3 ms) 2023-09-23T20:42:49.589 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] TestClsRbd.group_snap_list 2023-09-23T20:42:49.592 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] TestClsRbd.group_snap_list (5 ms) 2023-09-23T20:42:49.592 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] TestClsRbd.group_snap_list_max_return 2023-09-23T20:42:49.614 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] TestClsRbd.group_snap_list_max_return (21 ms) 2023-09-23T20:42:49.614 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] TestClsRbd.group_snap_list_max_read 2023-09-23T20:42:49.804 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] TestClsRbd.group_snap_list_max_read (190 ms) 2023-09-23T20:42:49.804 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] TestClsRbd.group_snap_remove 2023-09-23T20:42:49.807 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] TestClsRbd.group_snap_remove (4 ms) 2023-09-23T20:42:49.807 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] TestClsRbd.group_snap_get_by_id 2023-09-23T20:42:49.810 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] TestClsRbd.group_snap_get_by_id (2 ms) 2023-09-23T20:42:49.810 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] TestClsRbd.trash_methods 2023-09-23T20:42:49.819 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] TestClsRbd.trash_methods (9 ms) 2023-09-23T20:42:49.819 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] TestClsRbd.op_features 2023-09-23T20:42:49.826 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] TestClsRbd.op_features (8 ms) 2023-09-23T20:42:49.826 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] TestClsRbd.clone_parent 2023-09-23T20:42:49.846 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] TestClsRbd.clone_parent (19 ms) 2023-09-23T20:42:49.846 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] TestClsRbd.clone_parent_ns 2023-09-23T20:42:49.854 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] TestClsRbd.clone_parent_ns (9 ms) 2023-09-23T20:42:49.854 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] TestClsRbd.clone_child 2023-09-23T20:42:49.868 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] TestClsRbd.clone_child (13 ms) 2023-09-23T20:42:49.868 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] TestClsRbd.namespace_methods 2023-09-23T20:42:49.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:42:49 smithi070 ceph-mon[100755]: from='mgr.14451 172.21.15.26:0/1329296977' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:42:49.875 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] TestClsRbd.namespace_methods (7 ms) 2023-09-23T20:42:49.875 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] TestClsRbd.migration 2023-09-23T20:42:49.881 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:42:49 smithi026 ceph-mon[109912]: from='mgr.14451 172.21.15.26:0/1329296977' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:42:49.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:42:49 smithi026 ceph-mon[95839]: from='mgr.14451 172.21.15.26:0/1329296977' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:42:49.884 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] TestClsRbd.migration (9 ms) 2023-09-23T20:42:49.884 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] TestClsRbd.migration_v1 2023-09-23T20:42:49.895 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] TestClsRbd.migration_v1 (12 ms) 2023-09-23T20:42:49.895 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] TestClsRbd.assert_snapc_seq 2023-09-23T20:42:50.079 INFO:teuthology.orchestra.run.smithi026.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2023-09-23T20:42:50.079 INFO:teuthology.orchestra.run.smithi026.stdout:alertmanager.a smithi026 *:9093,9094 running (58s) 20s ago 94s 17.6M - 0.25.0 c8568f914cd2 4ed9bf8e51d9 2023-09-23T20:42:50.079 INFO:teuthology.orchestra.run.smithi026.stdout:grafana.a smithi070 *:3000 running (73s) 31s ago 73s 87.1M - 9.4.7 2c41d148cca3 12943b9df698 2023-09-23T20:42:50.079 INFO:teuthology.orchestra.run.smithi026.stdout:mds.foo.smithi026.znfpnl smithi026 running (47s) 20s ago 47s 16.2M - 17.2.6-1123-g61431f6b 0db14aa01094 b6861ac0ff55 2023-09-23T20:42:50.080 INFO:teuthology.orchestra.run.smithi026.stdout:mds.foo.smithi070.imqhqy smithi070 running (45s) 31s ago 44s 15.6M - 17.2.6-1123-g61431f6b 0db14aa01094 ca26bf551eb9 2023-09-23T20:42:50.080 INFO:teuthology.orchestra.run.smithi026.stdout:mgr.x smithi070 *:8443 running (5m) 31s ago 5m 439M - 17.2.6-1123-g61431f6b 0db14aa01094 e8decbdb0e5d 2023-09-23T20:42:50.080 INFO:teuthology.orchestra.run.smithi026.stdout:mgr.y smithi026 *:9283 running (6m) 20s ago 6m 493M - 17.2.6-1123-g61431f6b 0db14aa01094 e022dce09ba0 2023-09-23T20:42:50.080 INFO:teuthology.orchestra.run.smithi026.stdout:mon.a smithi026 running (6m) 20s ago 6m 76.1M 2048M 17.2.6-1123-g61431f6b 0db14aa01094 24cf5254b0b8 2023-09-23T20:42:50.080 INFO:teuthology.orchestra.run.smithi026.stdout:mon.b smithi070 running (5m) 31s ago 5m 56.3M 2048M 17.2.6-1123-g61431f6b 0db14aa01094 2c8a2924d3d7 2023-09-23T20:42:50.080 INFO:teuthology.orchestra.run.smithi026.stdout:mon.c smithi026 running (5m) 20s ago 5m 57.0M 2048M 17.2.6-1123-g61431f6b 0db14aa01094 069bb324bbad 2023-09-23T20:42:50.080 INFO:teuthology.orchestra.run.smithi026.stdout:node-exporter.a smithi026 *:9100 running (90s) 20s ago 90s 8648k - 1.5.0 0da6a335fe13 3e12faf27022 2023-09-23T20:42:50.080 INFO:teuthology.orchestra.run.smithi026.stdout:node-exporter.b smithi070 *:9100 running (85s) 31s ago 85s 16.8M - 1.5.0 0da6a335fe13 8305df31701d 2023-09-23T20:42:50.080 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0 smithi026 running (4m) 20s ago 4m 58.6M 4037M 17.2.6-1123-g61431f6b 0db14aa01094 e40b0407b8cf 2023-09-23T20:42:50.080 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1 smithi026 running (4m) 20s ago 4m 60.4M 4037M 17.2.6-1123-g61431f6b 0db14aa01094 a73d47580165 2023-09-23T20:42:50.080 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2 smithi026 running (4m) 20s ago 4m 49.7M 4037M 17.2.6-1123-g61431f6b 0db14aa01094 fc4783fb1a89 2023-09-23T20:42:50.080 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3 smithi026 running (3m) 20s ago 3m 49.7M 4037M 17.2.6-1123-g61431f6b 0db14aa01094 5f0339ddc2c4 2023-09-23T20:42:50.080 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4 smithi070 running (3m) 31s ago 3m 53.7M 4036M 17.2.6-1123-g61431f6b 0db14aa01094 308a2af3101e 2023-09-23T20:42:50.081 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5 smithi070 running (2m) 31s ago 2m 48.6M 4036M 17.2.6-1123-g61431f6b 0db14aa01094 16cd72517077 2023-09-23T20:42:50.081 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6 smithi070 running (2m) 31s ago 2m 48.3M 4036M 17.2.6-1123-g61431f6b 0db14aa01094 5d24c21e8be7 2023-09-23T20:42:50.081 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7 smithi070 running (2m) 31s ago 2m 52.9M 4036M 17.2.6-1123-g61431f6b 0db14aa01094 8c1b7f314c41 2023-09-23T20:42:50.081 INFO:teuthology.orchestra.run.smithi026.stdout:prometheus.a smithi070 *:9095 running (54s) 31s ago 115s 38.0M - 2.43.0 a07b618ecd1d 20647f147400 2023-09-23T20:42:50.431 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -e sha1=bb0daeff97f020f686596fca6eba20c310e26fb7 -- bash -c 'echo wait for minority of mons to upgrade' 2023-09-23T20:42:50.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:42:50 smithi070 ceph-mon[100755]: pgmap v69: 105 pgs: 105 active+clean; 579 KiB data, 84 MiB used, 715 GiB / 715 GiB avail; 3.1 MiB/s rd, 1.4 MiB/s wr, 2.40k op/s 2023-09-23T20:42:50.881 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:42:50 smithi026 ceph-mon[95839]: pgmap v69: 105 pgs: 105 active+clean; 579 KiB data, 84 MiB used, 715 GiB / 715 GiB avail; 3.1 MiB/s rd, 1.4 MiB/s wr, 2.40k op/s 2023-09-23T20:42:50.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:42:50 smithi026 ceph-mon[109912]: pgmap v69: 105 pgs: 105 active+clean; 579 KiB data, 84 MiB used, 715 GiB / 715 GiB avail; 3.1 MiB/s rd, 1.4 MiB/s wr, 2.40k op/s 2023-09-23T20:42:51.077 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:42:51.606 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] TestClsRbd.assert_snapc_seq (1710 ms) 2023-09-23T20:42:51.606 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] TestClsRbd.sparsify 2023-09-23T20:42:51.633 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] TestClsRbd.sparsify (27 ms) 2023-09-23T20:42:51.846 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:42:51 smithi026 ceph-mon[95839]: from='client.14829 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T20:42:51.846 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:42:51 smithi026 ceph-mon[95839]: osdmap e74: 8 total, 8 up, 8 in 2023-09-23T20:42:51.847 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:42:51 smithi026 ceph-mon[109912]: from='client.14829 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T20:42:51.847 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:42:51 smithi026 ceph-mon[109912]: osdmap e74: 8 total, 8 up, 8 in 2023-09-23T20:42:51.869 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 20:42:51 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-x[102861]: ::ffff:172.21.15.70 - - [23/Sep/2023:20:42:51] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2023-09-23T20:42:51.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:42:51 smithi070 ceph-mon[100755]: from='client.14829 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T20:42:51.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:42:51 smithi070 ceph-mon[100755]: osdmap e74: 8 total, 8 up, 8 in 2023-09-23T20:42:52.202 INFO:teuthology.orchestra.run.smithi026.stdout:wait for minority of mons to upgrade 2023-09-23T20:42:52.617 INFO:tasks.workunit.client.0.smithi026.stdout:[----------] 70 tests from TestClsRbd (5048 ms total) 2023-09-23T20:42:52.617 INFO:tasks.workunit.client.0.smithi026.stdout: 2023-09-23T20:42:52.617 INFO:tasks.workunit.client.0.smithi026.stdout:[----------] Global test environment tear-down 2023-09-23T20:42:52.618 INFO:tasks.workunit.client.0.smithi026.stdout:[==========] 70 tests from 1 test suite ran. (7558 ms total) 2023-09-23T20:42:52.618 INFO:tasks.workunit.client.0.smithi026.stdout:[ PASSED ] 70 tests. 2023-09-23T20:42:52.619 INFO:teuthology.orchestra.run:Running command with timeout 3600 2023-09-23T20:42:52.619 DEBUG:teuthology.orchestra.run.smithi026:> sudo rm -rf -- /home/ubuntu/cephtest/mnt.0/client.0/tmp 2023-09-23T20:42:52.654 INFO:tasks.workunit:Stopping ['cls/test_cls_rbd.sh'] on client.0... 2023-09-23T20:42:52.654 DEBUG:teuthology.orchestra.run.smithi026:> sudo rm -rf -- /home/ubuntu/cephtest/workunits.list.client.0 /home/ubuntu/cephtest/clone.client.0 2023-09-23T20:42:52.722 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:42:52 smithi026 ceph-mon[109912]: osdmap e75: 8 total, 8 up, 8 in 2023-09-23T20:42:52.722 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:42:52 smithi026 ceph-mon[109912]: pgmap v72: 105 pgs: 105 active+clean; 579 KiB data, 84 MiB used, 715 GiB / 715 GiB avail; 3.5 MiB/s rd, 1.6 MiB/s wr, 2.74k op/s 2023-09-23T20:42:52.722 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:42:52 smithi026 ceph-mon[95839]: osdmap e75: 8 total, 8 up, 8 in 2023-09-23T20:42:52.722 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:42:52 smithi026 ceph-mon[95839]: pgmap v72: 105 pgs: 105 active+clean; 579 KiB data, 84 MiB used, 715 GiB / 715 GiB avail; 3.5 MiB/s rd, 1.6 MiB/s wr, 2.74k op/s 2023-09-23T20:42:52.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:42:52 smithi070 ceph-mon[100755]: osdmap e75: 8 total, 8 up, 8 in 2023-09-23T20:42:52.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:42:52 smithi070 ceph-mon[100755]: pgmap v72: 105 pgs: 105 active+clean; 579 KiB data, 84 MiB used, 715 GiB / 715 GiB avail; 3.5 MiB/s rd, 1.6 MiB/s wr, 2.74k op/s 2023-09-23T20:42:52.913 DEBUG:teuthology.parallel:result is None 2023-09-23T20:42:52.913 DEBUG:teuthology.orchestra.run.smithi026:> sudo rm -rf -- /home/ubuntu/cephtest/mnt.0/client.0 2023-09-23T20:42:52.982 INFO:tasks.workunit:Deleted dir /home/ubuntu/cephtest/mnt.0/client.0 2023-09-23T20:42:52.982 DEBUG:teuthology.orchestra.run.smithi026:> rmdir -- /home/ubuntu/cephtest/mnt.0 2023-09-23T20:42:53.053 INFO:tasks.workunit:Deleted artificial mount point /home/ubuntu/cephtest/mnt.0/client.0 2023-09-23T20:42:53.053 INFO:teuthology.task.sequential:In sequential, running task print... 2023-09-23T20:42:53.053 INFO:teuthology.task.print:**** done cls/test_cls_rbd.sh 5-workload 2023-09-23T20:42:53.053 DEBUG:teuthology.parallel:result is None 2023-09-23T20:42:53.136 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -e sha1=bb0daeff97f020f686596fca6eba20c310e26fb7 -- bash -c 'while ! ceph mon versions | grep $sha1 ; do sleep 2 ; done' 2023-09-23T20:42:53.758 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:42:53.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:42:53 smithi070 ceph-mon[100755]: osdmap e76: 8 total, 8 up, 8 in 2023-09-23T20:42:53.881 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:42:53 smithi026 ceph-mon[95839]: osdmap e76: 8 total, 8 up, 8 in 2023-09-23T20:42:53.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:42:53 smithi026 ceph-mon[109912]: osdmap e76: 8 total, 8 up, 8 in 2023-09-23T20:42:54.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:42:54 smithi070 ceph-mon[100755]: pgmap v74: 73 pgs: 73 active+clean; 579 KiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:42:54.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:42:54 smithi026 ceph-mon[95839]: pgmap v74: 73 pgs: 73 active+clean; 579 KiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:42:54.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:42:54 smithi026 ceph-mon[109912]: pgmap v74: 73 pgs: 73 active+clean; 579 KiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:42:55.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:42:55 smithi070 ceph-mon[100755]: from='client.? 172.21.15.26:0/771899051' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:42:55.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:42:55 smithi026 ceph-mon[95839]: from='client.? 172.21.15.26:0/771899051' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:42:55.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:42:55 smithi026 ceph-mon[109912]: from='client.? 172.21.15.26:0/771899051' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:42:56.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:42:56 smithi070 ceph-mon[100755]: pgmap v75: 73 pgs: 73 active+clean; 579 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:42:56.881 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:42:56 smithi026 ceph-mon[95839]: pgmap v75: 73 pgs: 73 active+clean; 579 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:42:56.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:42:56 smithi026 ceph-mon[109912]: pgmap v75: 73 pgs: 73 active+clean; 579 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:42:58.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:42:57 smithi070 ceph-mon[100755]: from='client.? 172.21.15.26:0/3861107868' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:42:58.131 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:42:57 smithi026 ceph-mon[95839]: from='client.? 172.21.15.26:0/3861107868' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:42:58.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:42:57 smithi026 ceph-mon[109912]: from='client.? 172.21.15.26:0/3861107868' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:42:58.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:42:58 smithi026 ceph-mon[95839]: pgmap v76: 73 pgs: 73 active+clean; 579 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:42:58.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:42:58 smithi026 ceph-mon[109912]: pgmap v76: 73 pgs: 73 active+clean; 579 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:42:58.882 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:42:58 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[96086]: ::ffff:172.21.15.70 - - [23/Sep/2023:20:42:58] "GET /metrics HTTP/1.1" 200 240893 "" "Prometheus/2.43.0" 2023-09-23T20:42:59.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:42:58 smithi070 ceph-mon[100755]: pgmap v76: 73 pgs: 73 active+clean; 579 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:43:01.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:43:00 smithi026 ceph-mon[95839]: pgmap v77: 73 pgs: 73 active+clean; 579 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:43:01.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:43:00 smithi026 ceph-mon[95839]: from='client.? 172.21.15.26:0/316085735' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:43:01.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:43:00 smithi026 ceph-mon[109912]: pgmap v77: 73 pgs: 73 active+clean; 579 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:43:01.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:43:00 smithi026 ceph-mon[109912]: from='client.? 172.21.15.26:0/316085735' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:43:01.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:43:00 smithi070 ceph-mon[100755]: pgmap v77: 73 pgs: 73 active+clean; 579 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:43:01.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:43:00 smithi070 ceph-mon[100755]: from='client.? 172.21.15.26:0/316085735' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:43:01.869 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 20:43:01 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-x[102861]: ::ffff:172.21.15.70 - - [23/Sep/2023:20:43:01] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2023-09-23T20:43:03.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:43:02 smithi026 ceph-mon[95839]: pgmap v78: 73 pgs: 73 active+clean; 579 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:43:03.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:43:02 smithi026 ceph-mon[95839]: from='client.? 172.21.15.26:0/1831677758' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:43:03.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:43:02 smithi026 ceph-mon[109912]: pgmap v78: 73 pgs: 73 active+clean; 579 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:43:03.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:43:02 smithi026 ceph-mon[109912]: from='client.? 172.21.15.26:0/1831677758' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:43:03.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:43:02 smithi070 ceph-mon[100755]: pgmap v78: 73 pgs: 73 active+clean; 579 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:43:03.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:43:02 smithi070 ceph-mon[100755]: from='client.? 172.21.15.26:0/1831677758' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:43:04.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:43:04 smithi070 ceph-mon[100755]: pgmap v79: 73 pgs: 73 active+clean; 579 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:43:04.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:43:04 smithi070 ceph-mon[100755]: from='mgr.14451 172.21.15.26:0/1329296977' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:43:04.881 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:43:04 smithi026 ceph-mon[109912]: pgmap v79: 73 pgs: 73 active+clean; 579 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:43:04.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:43:04 smithi026 ceph-mon[109912]: from='mgr.14451 172.21.15.26:0/1329296977' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:43:04.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:43:04 smithi026 ceph-mon[95839]: pgmap v79: 73 pgs: 73 active+clean; 579 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:43:04.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:43:04 smithi026 ceph-mon[95839]: from='mgr.14451 172.21.15.26:0/1329296977' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:43:05.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:43:05 smithi070 ceph-mon[100755]: from='client.? 172.21.15.26:0/3534125369' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:43:05.881 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:43:05 smithi026 ceph-mon[109912]: from='client.? 172.21.15.26:0/3534125369' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:43:05.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:43:05 smithi026 ceph-mon[95839]: from='client.? 172.21.15.26:0/3534125369' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:43:06.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:43:06 smithi070 ceph-mon[100755]: pgmap v80: 73 pgs: 73 active+clean; 579 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:43:06.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:43:06 smithi026 ceph-mon[95839]: pgmap v80: 73 pgs: 73 active+clean; 579 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:43:06.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:43:06 smithi026 ceph-mon[109912]: pgmap v80: 73 pgs: 73 active+clean; 579 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:43:07.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:43:07 smithi070 ceph-mon[100755]: from='client.? 172.21.15.26:0/2605773755' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:43:07.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:43:07 smithi026 ceph-mon[95839]: from='client.? 172.21.15.26:0/2605773755' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:43:07.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:43:07 smithi026 ceph-mon[109912]: from='client.? 172.21.15.26:0/2605773755' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:43:08.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:43:08 smithi070 ceph-mon[100755]: pgmap v81: 73 pgs: 73 active+clean; 579 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:43:08.883 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:43:08 smithi026 ceph-mon[95839]: pgmap v81: 73 pgs: 73 active+clean; 579 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:43:08.883 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:43:08 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[96086]: ::ffff:172.21.15.70 - - [23/Sep/2023:20:43:08] "GET /metrics HTTP/1.1" 200 239524 "" "Prometheus/2.43.0" 2023-09-23T20:43:08.884 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:43:08 smithi026 ceph-mon[109912]: pgmap v81: 73 pgs: 73 active+clean; 579 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:43:10.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:43:09 smithi070 ceph-mon[100755]: pgmap v82: 73 pgs: 73 active+clean; 579 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:43:10.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:43:09 smithi070 ceph-mon[100755]: from='client.? 172.21.15.26:0/769553571' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:43:10.381 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:43:09 smithi026 ceph-mon[109912]: pgmap v82: 73 pgs: 73 active+clean; 579 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:43:10.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:43:09 smithi026 ceph-mon[109912]: from='client.? 172.21.15.26:0/769553571' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:43:10.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:43:09 smithi026 ceph-mon[95839]: pgmap v82: 73 pgs: 73 active+clean; 579 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:43:10.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:43:09 smithi026 ceph-mon[95839]: from='client.? 172.21.15.26:0/769553571' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:43:11.869 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 20:43:11 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-x[102861]: ::ffff:172.21.15.70 - - [23/Sep/2023:20:43:11] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2023-09-23T20:43:13.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:43:12 smithi026 ceph-mon[109912]: pgmap v83: 73 pgs: 73 active+clean; 579 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:43:13.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:43:12 smithi026 ceph-mon[109912]: from='client.? 172.21.15.26:0/891608836' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:43:13.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:43:12 smithi026 ceph-mon[109912]: from='mgr.14451 ' entity='mgr.y' 2023-09-23T20:43:13.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:43:12 smithi026 ceph-mon[109912]: from='mgr.14451 172.21.15.26:0/1329296977' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:43:13.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:43:12 smithi026 ceph-mon[109912]: from='mgr.14451 ' entity='mgr.y' 2023-09-23T20:43:13.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:43:12 smithi026 ceph-mon[95839]: pgmap v83: 73 pgs: 73 active+clean; 579 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:43:13.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:43:12 smithi026 ceph-mon[95839]: from='client.? 172.21.15.26:0/891608836' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:43:13.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:43:12 smithi026 ceph-mon[95839]: from='mgr.14451 ' entity='mgr.y' 2023-09-23T20:43:13.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:43:12 smithi026 ceph-mon[95839]: from='mgr.14451 172.21.15.26:0/1329296977' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:43:13.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:43:12 smithi026 ceph-mon[95839]: from='mgr.14451 ' entity='mgr.y' 2023-09-23T20:43:13.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:43:12 smithi070 ceph-mon[100755]: pgmap v83: 73 pgs: 73 active+clean; 579 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:43:13.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:43:12 smithi070 ceph-mon[100755]: from='client.? 172.21.15.26:0/891608836' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:43:13.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:43:12 smithi070 ceph-mon[100755]: from='mgr.14451 ' entity='mgr.y' 2023-09-23T20:43:13.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:43:12 smithi070 ceph-mon[100755]: from='mgr.14451 172.21.15.26:0/1329296977' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:43:13.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:43:12 smithi070 ceph-mon[100755]: from='mgr.14451 ' entity='mgr.y' 2023-09-23T20:43:14.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:43:13 smithi070 ceph-mon[100755]: Upgrade: Target is version 18.2.0-460-gbb0daeff (unknown) 2023-09-23T20:43:14.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:43:13 smithi070 ceph-mon[100755]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d, digests ['quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d'] 2023-09-23T20:43:14.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:43:13 smithi070 ceph-mon[100755]: Upgrade: Need to upgrade myself (mgr.y) 2023-09-23T20:43:14.381 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:43:13 smithi026 ceph-mon[109912]: Upgrade: Target is version 18.2.0-460-gbb0daeff (unknown) 2023-09-23T20:43:14.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:43:13 smithi026 ceph-mon[109912]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d, digests ['quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d'] 2023-09-23T20:43:14.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:43:13 smithi026 ceph-mon[109912]: Upgrade: Need to upgrade myself (mgr.y) 2023-09-23T20:43:14.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:43:13 smithi026 ceph-mon[95839]: Upgrade: Target is version 18.2.0-460-gbb0daeff (unknown) 2023-09-23T20:43:14.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:43:13 smithi026 ceph-mon[95839]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d, digests ['quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d'] 2023-09-23T20:43:14.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:43:13 smithi026 ceph-mon[95839]: Upgrade: Need to upgrade myself (mgr.y) 2023-09-23T20:43:15.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:43:14 smithi070 ceph-mon[100755]: Upgrade: Pulling quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d on smithi070 2023-09-23T20:43:15.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:43:14 smithi070 ceph-mon[100755]: pgmap v84: 73 pgs: 73 active+clean; 579 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:43:15.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:43:14 smithi070 ceph-mon[100755]: from='client.? 172.21.15.26:0/158819678' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:43:15.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:43:14 smithi026 ceph-mon[95839]: Upgrade: Pulling quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d on smithi070 2023-09-23T20:43:15.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:43:14 smithi026 ceph-mon[95839]: pgmap v84: 73 pgs: 73 active+clean; 579 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:43:15.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:43:14 smithi026 ceph-mon[95839]: from='client.? 172.21.15.26:0/158819678' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:43:15.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:43:14 smithi026 ceph-mon[109912]: Upgrade: Pulling quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d on smithi070 2023-09-23T20:43:15.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:43:14 smithi026 ceph-mon[109912]: pgmap v84: 73 pgs: 73 active+clean; 579 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:43:15.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:43:14 smithi026 ceph-mon[109912]: from='client.? 172.21.15.26:0/158819678' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:43:17.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:43:16 smithi070 ceph-mon[100755]: pgmap v85: 73 pgs: 73 active+clean; 579 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:43:17.381 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:43:16 smithi026 ceph-mon[95839]: pgmap v85: 73 pgs: 73 active+clean; 579 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:43:17.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:43:16 smithi026 ceph-mon[109912]: pgmap v85: 73 pgs: 73 active+clean; 579 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:43:18.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:43:17 smithi070 ceph-mon[100755]: from='client.? 172.21.15.26:0/636823597' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:43:18.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:43:17 smithi026 ceph-mon[95839]: from='client.? 172.21.15.26:0/636823597' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:43:18.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:43:17 smithi026 ceph-mon[109912]: from='client.? 172.21.15.26:0/636823597' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:43:18.882 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:43:18 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[96086]: ::ffff:172.21.15.70 - - [23/Sep/2023:20:43:18] "GET /metrics HTTP/1.1" 200 239524 "" "Prometheus/2.43.0" 2023-09-23T20:43:19.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:43:18 smithi070 ceph-mon[100755]: pgmap v86: 73 pgs: 73 active+clean; 579 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:43:19.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:43:18 smithi070 ceph-mon[100755]: from='mgr.14451 172.21.15.26:0/1329296977' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:43:19.381 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:43:18 smithi026 ceph-mon[95839]: pgmap v86: 73 pgs: 73 active+clean; 579 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:43:19.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:43:18 smithi026 ceph-mon[95839]: from='mgr.14451 172.21.15.26:0/1329296977' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:43:19.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:43:18 smithi026 ceph-mon[109912]: pgmap v86: 73 pgs: 73 active+clean; 579 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:43:19.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:43:18 smithi026 ceph-mon[109912]: from='mgr.14451 172.21.15.26:0/1329296977' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:43:20.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:43:19 smithi070 ceph-mon[100755]: from='client.? 172.21.15.26:0/1350452867' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:43:20.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:43:19 smithi026 ceph-mon[95839]: from='client.? 172.21.15.26:0/1350452867' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:43:20.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:43:19 smithi026 ceph-mon[109912]: from='client.? 172.21.15.26:0/1350452867' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:43:21.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:43:20 smithi070 ceph-mon[100755]: pgmap v87: 73 pgs: 73 active+clean; 579 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:43:21.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:43:20 smithi026 ceph-mon[95839]: pgmap v87: 73 pgs: 73 active+clean; 579 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:43:21.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:43:20 smithi026 ceph-mon[109912]: pgmap v87: 73 pgs: 73 active+clean; 579 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:43:21.869 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 20:43:21 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-x[102861]: ::ffff:172.21.15.70 - - [23/Sep/2023:20:43:21] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2023-09-23T20:43:22.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:43:21 smithi070 ceph-mon[100755]: from='client.? 172.21.15.26:0/1271338435' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:43:22.381 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:43:21 smithi026 ceph-mon[95839]: from='client.? 172.21.15.26:0/1271338435' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:43:22.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:43:21 smithi026 ceph-mon[109912]: from='client.? 172.21.15.26:0/1271338435' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:43:23.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:43:22 smithi070 ceph-mon[100755]: pgmap v88: 73 pgs: 73 active+clean; 579 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:43:23.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:43:22 smithi026 ceph-mon[95839]: pgmap v88: 73 pgs: 73 active+clean; 579 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:43:23.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:43:22 smithi026 ceph-mon[109912]: pgmap v88: 73 pgs: 73 active+clean; 579 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:43:24.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:43:24 smithi070 ceph-mon[100755]: pgmap v89: 73 pgs: 73 active+clean; 579 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:43:24.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:43:24 smithi070 ceph-mon[100755]: from='client.? 172.21.15.26:0/347320870' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:43:24.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:43:24 smithi026 ceph-mon[95839]: pgmap v89: 73 pgs: 73 active+clean; 579 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:43:24.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:43:24 smithi026 ceph-mon[95839]: from='client.? 172.21.15.26:0/347320870' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:43:24.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:43:24 smithi026 ceph-mon[109912]: pgmap v89: 73 pgs: 73 active+clean; 579 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:43:24.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:43:24 smithi026 ceph-mon[109912]: from='client.? 172.21.15.26:0/347320870' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:43:27.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:43:26 smithi070 ceph-mon[100755]: pgmap v90: 73 pgs: 73 active+clean; 579 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:43:27.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:43:26 smithi070 ceph-mon[100755]: from='client.? 172.21.15.26:0/1258754283' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:43:27.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:43:26 smithi026 ceph-mon[95839]: pgmap v90: 73 pgs: 73 active+clean; 579 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:43:27.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:43:26 smithi026 ceph-mon[95839]: from='client.? 172.21.15.26:0/1258754283' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:43:27.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:43:26 smithi026 ceph-mon[109912]: pgmap v90: 73 pgs: 73 active+clean; 579 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:43:27.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:43:26 smithi026 ceph-mon[109912]: from='client.? 172.21.15.26:0/1258754283' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:43:28.882 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:43:28 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[96086]: ::ffff:172.21.15.70 - - [23/Sep/2023:20:43:28] "GET /metrics HTTP/1.1" 200 239514 "" "Prometheus/2.43.0" 2023-09-23T20:43:29.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:43:28 smithi070 ceph-mon[100755]: pgmap v91: 73 pgs: 73 active+clean; 579 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:43:29.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:43:28 smithi026 ceph-mon[95839]: pgmap v91: 73 pgs: 73 active+clean; 579 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:43:29.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:43:28 smithi026 ceph-mon[109912]: pgmap v91: 73 pgs: 73 active+clean; 579 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:43:30.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:43:29 smithi070 ceph-mon[100755]: from='client.? 172.21.15.26:0/2024519236' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:43:30.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:43:29 smithi026 ceph-mon[95839]: from='client.? 172.21.15.26:0/2024519236' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:43:30.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:43:29 smithi026 ceph-mon[109912]: from='client.? 172.21.15.26:0/2024519236' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:43:31.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:43:30 smithi070 ceph-mon[100755]: pgmap v92: 73 pgs: 73 active+clean; 579 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:43:31.381 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:43:30 smithi026 ceph-mon[95839]: pgmap v92: 73 pgs: 73 active+clean; 579 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:43:31.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:43:30 smithi026 ceph-mon[109912]: pgmap v92: 73 pgs: 73 active+clean; 579 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:43:31.871 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 20:43:31 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-x[102861]: ::ffff:172.21.15.70 - - [23/Sep/2023:20:43:31] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2023-09-23T20:43:32.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:43:31 smithi070 ceph-mon[100755]: from='client.? 172.21.15.26:0/4040337130' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:43:32.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:43:31 smithi026 ceph-mon[95839]: from='client.? 172.21.15.26:0/4040337130' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:43:32.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:43:31 smithi026 ceph-mon[109912]: from='client.? 172.21.15.26:0/4040337130' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:43:33.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:43:32 smithi070 ceph-mon[100755]: pgmap v93: 73 pgs: 73 active+clean; 579 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:43:33.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:43:32 smithi026 ceph-mon[95839]: pgmap v93: 73 pgs: 73 active+clean; 579 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:43:33.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:43:32 smithi026 ceph-mon[109912]: pgmap v93: 73 pgs: 73 active+clean; 579 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:43:34.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:43:34 smithi070 ceph-mon[100755]: pgmap v94: 73 pgs: 73 active+clean; 579 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:43:34.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:43:34 smithi070 ceph-mon[100755]: from='mgr.14451 172.21.15.26:0/1329296977' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:43:34.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:43:34 smithi070 ceph-mon[100755]: from='client.? 172.21.15.26:0/3715297213' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:43:34.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:43:34 smithi026 ceph-mon[95839]: pgmap v94: 73 pgs: 73 active+clean; 579 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:43:34.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:43:34 smithi026 ceph-mon[95839]: from='mgr.14451 172.21.15.26:0/1329296977' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:43:34.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:43:34 smithi026 ceph-mon[95839]: from='client.? 172.21.15.26:0/3715297213' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:43:34.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:43:34 smithi026 ceph-mon[109912]: pgmap v94: 73 pgs: 73 active+clean; 579 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:43:34.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:43:34 smithi026 ceph-mon[109912]: from='mgr.14451 172.21.15.26:0/1329296977' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:43:34.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:43:34 smithi026 ceph-mon[109912]: from='client.? 172.21.15.26:0/3715297213' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:43:37.262 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:43:36 smithi070 ceph-mon[100755]: pgmap v95: 73 pgs: 73 active+clean; 579 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:43:37.263 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:43:36 smithi070 ceph-mon[100755]: from='client.? 172.21.15.26:0/476010301' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:43:37.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:43:36 smithi026 ceph-mon[95839]: pgmap v95: 73 pgs: 73 active+clean; 579 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:43:37.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:43:36 smithi026 ceph-mon[95839]: from='client.? 172.21.15.26:0/476010301' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:43:37.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:43:36 smithi026 ceph-mon[109912]: pgmap v95: 73 pgs: 73 active+clean; 579 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:43:37.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:43:36 smithi026 ceph-mon[109912]: from='client.? 172.21.15.26:0/476010301' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:43:38.882 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:43:38 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[96086]: ::ffff:172.21.15.70 - - [23/Sep/2023:20:43:38] "GET /metrics HTTP/1.1" 200 239485 "" "Prometheus/2.43.0" 2023-09-23T20:43:39.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:43:38 smithi070 ceph-mon[100755]: pgmap v96: 73 pgs: 73 active+clean; 579 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:43:39.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:43:38 smithi070 ceph-mon[100755]: from='mgr.14451 ' entity='mgr.y' 2023-09-23T20:43:39.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:43:38 smithi070 ceph-mon[100755]: from='mgr.14451 172.21.15.26:0/1329296977' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-09-23T20:43:39.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:43:38 smithi070 ceph-mon[100755]: from='mgr.14451 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-09-23T20:43:39.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:43:38 smithi070 ceph-mon[100755]: from='mgr.14451 172.21.15.26:0/1329296977' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-09-23T20:43:39.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:43:38 smithi070 ceph-mon[100755]: from='mgr.14451 172.21.15.26:0/1329296977' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:43:39.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:43:38 smithi070 ceph-mon[100755]: from='client.? 172.21.15.26:0/3171514874' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:43:39.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:43:38 smithi026 ceph-mon[95839]: pgmap v96: 73 pgs: 73 active+clean; 579 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:43:39.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:43:38 smithi026 ceph-mon[95839]: from='mgr.14451 ' entity='mgr.y' 2023-09-23T20:43:39.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:43:38 smithi026 ceph-mon[95839]: from='mgr.14451 172.21.15.26:0/1329296977' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-09-23T20:43:39.383 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:43:38 smithi026 ceph-mon[95839]: from='mgr.14451 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-09-23T20:43:39.383 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:43:38 smithi026 ceph-mon[95839]: from='mgr.14451 172.21.15.26:0/1329296977' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-09-23T20:43:39.383 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:43:38 smithi026 ceph-mon[95839]: from='mgr.14451 172.21.15.26:0/1329296977' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:43:39.383 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:43:38 smithi026 ceph-mon[95839]: from='client.? 172.21.15.26:0/3171514874' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:43:39.383 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:43:38 smithi026 ceph-mon[109912]: pgmap v96: 73 pgs: 73 active+clean; 579 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:43:39.383 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:43:38 smithi026 ceph-mon[109912]: from='mgr.14451 ' entity='mgr.y' 2023-09-23T20:43:39.383 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:43:38 smithi026 ceph-mon[109912]: from='mgr.14451 172.21.15.26:0/1329296977' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-09-23T20:43:39.383 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:43:38 smithi026 ceph-mon[109912]: from='mgr.14451 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-09-23T20:43:39.383 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:43:38 smithi026 ceph-mon[109912]: from='mgr.14451 172.21.15.26:0/1329296977' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-09-23T20:43:39.384 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:43:38 smithi026 ceph-mon[109912]: from='mgr.14451 172.21.15.26:0/1329296977' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:43:39.384 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:43:38 smithi026 ceph-mon[109912]: from='client.? 172.21.15.26:0/3171514874' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:43:39.555 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 20:43:39 smithi070 systemd[1]: Stopping Ceph mgr.x for b3f8e4ec-5a50-11ee-8db4-212e2dc638e7... 2023-09-23T20:43:39.827 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 20:43:39 smithi070 bash[141306]: ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-x 2023-09-23T20:43:39.827 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 20:43:39 smithi070 systemd[1]: ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7@mgr.x.service: Main process exited, code=exited, status=143/n/a 2023-09-23T20:43:40.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:43:39 smithi070 ceph-mon[100755]: Upgrade: Updating mgr.x 2023-09-23T20:43:40.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:43:39 smithi070 ceph-mon[100755]: Deploying daemon mgr.x on smithi070 2023-09-23T20:43:40.370 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 20:43:40 smithi070 systemd[1]: ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7@mgr.x.service: Failed with result 'exit-code'. 2023-09-23T20:43:40.370 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 20:43:40 smithi070 systemd[1]: Stopped Ceph mgr.x for b3f8e4ec-5a50-11ee-8db4-212e2dc638e7. 2023-09-23T20:43:40.370 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 20:43:40 smithi070 systemd[1]: Starting Ceph mgr.x for b3f8e4ec-5a50-11ee-8db4-212e2dc638e7... 2023-09-23T20:43:40.381 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:43:39 smithi026 ceph-mon[95839]: Upgrade: Updating mgr.x 2023-09-23T20:43:40.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:43:39 smithi026 ceph-mon[95839]: Deploying daemon mgr.x on smithi070 2023-09-23T20:43:40.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:43:39 smithi026 ceph-mon[109912]: Upgrade: Updating mgr.x 2023-09-23T20:43:40.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:43:39 smithi026 ceph-mon[109912]: Deploying daemon mgr.x on smithi070 2023-09-23T20:43:40.702 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 20:43:40 smithi070 podman[141454]: 2023-09-23T20:43:41.002 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:43:40 smithi070 ceph-mon[100755]: pgmap v97: 73 pgs: 73 active+clean; 579 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:43:41.002 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:43:40 smithi070 ceph-mon[100755]: from='mgr.14451 ' entity='mgr.y' 2023-09-23T20:43:41.002 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:43:40 smithi070 ceph-mon[100755]: from='mgr.14451 ' entity='mgr.y' 2023-09-23T20:43:41.002 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:43:40 smithi070 ceph-mon[100755]: from='mgr.14451 172.21.15.26:0/1329296977' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:43:41.002 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 20:43:40 smithi070 ceph-mgr[141494]: -- 172.21.15.70:0/3887294547 <== mon.1 v2:172.21.15.70:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x558eba5571e0 con 0x558eba52c800 2023-09-23T20:43:41.003 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 20:43:40 smithi070 bash[141454]: 11a9d7e48ea730967c33e78b70d6cfad44b0211d0ca5d53f1123435ca1e06854 2023-09-23T20:43:41.003 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 20:43:40 smithi070 systemd[1]: Started Ceph mgr.x for b3f8e4ec-5a50-11ee-8db4-212e2dc638e7. 2023-09-23T20:43:41.003 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 20:43:40 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-x[141470]: 2023-09-23T20:43:40.825+0000 7fed337d9200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2023-09-23T20:43:41.003 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 20:43:40 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-x[141470]: 2023-09-23T20:43:40.909+0000 7fed337d9200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-09-23T20:43:41.003 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 20:43:41 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-x[141470]: 2023-09-23T20:43:41.002+0000 7fed337d9200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-09-23T20:43:41.281 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 20:43:41 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-x[141470]: 2023-09-23T20:43:41.104+0000 7fed337d9200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-09-23T20:43:41.281 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 20:43:41 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-x[141470]: 2023-09-23T20:43:41.280+0000 7fed337d9200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-09-23T20:43:41.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:43:40 smithi026 ceph-mon[95839]: pgmap v97: 73 pgs: 73 active+clean; 579 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:43:41.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:43:40 smithi026 ceph-mon[95839]: from='mgr.14451 ' entity='mgr.y' 2023-09-23T20:43:41.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:43:40 smithi026 ceph-mon[95839]: from='mgr.14451 ' entity='mgr.y' 2023-09-23T20:43:41.383 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:43:40 smithi026 ceph-mon[95839]: from='mgr.14451 172.21.15.26:0/1329296977' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:43:41.383 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:43:40 smithi026 ceph-mon[109912]: pgmap v97: 73 pgs: 73 active+clean; 579 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:43:41.383 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:43:40 smithi026 ceph-mon[109912]: from='mgr.14451 ' entity='mgr.y' 2023-09-23T20:43:41.383 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:43:40 smithi026 ceph-mon[109912]: from='mgr.14451 ' entity='mgr.y' 2023-09-23T20:43:41.383 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:43:40 smithi026 ceph-mon[109912]: from='mgr.14451 172.21.15.26:0/1329296977' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:43:42.012 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:43:41 smithi070 ceph-mon[100755]: from='client.? 172.21.15.26:0/487037829' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:43:42.012 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 20:43:41 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-x[141470]: 2023-09-23T20:43:41.674+0000 7fed337d9200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-09-23T20:43:42.012 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 20:43:41 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-x[141470]: 2023-09-23T20:43:41.810+0000 7fed337d9200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-09-23T20:43:42.012 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 20:43:41 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-x[141470]: 2023-09-23T20:43:41.903+0000 7fed337d9200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-09-23T20:43:42.369 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 20:43:42 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-x[141470]: 2023-09-23T20:43:42.012+0000 7fed337d9200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-09-23T20:43:42.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:43:41 smithi026 ceph-mon[95839]: from='client.? 172.21.15.26:0/487037829' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:43:42.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:43:41 smithi026 ceph-mon[109912]: from='client.? 172.21.15.26:0/487037829' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:43:42.869 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 20:43:42 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-x[141470]: 2023-09-23T20:43:42.453+0000 7fed337d9200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-09-23T20:43:43.159 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:43:42 smithi070 ceph-mon[100755]: pgmap v98: 73 pgs: 73 active+clean; 579 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:43:43.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:43:42 smithi026 ceph-mon[95839]: pgmap v98: 73 pgs: 73 active+clean; 579 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:43:43.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:43:42 smithi026 ceph-mon[109912]: pgmap v98: 73 pgs: 73 active+clean; 579 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:43:43.619 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 20:43:43 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-x[141470]: 2023-09-23T20:43:43.159+0000 7fed337d9200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-09-23T20:43:43.619 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 20:43:43 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-x[141470]: 2023-09-23T20:43:43.243+0000 7fed337d9200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-09-23T20:43:44.192 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:43:43 smithi070 ceph-mon[100755]: from='client.? 172.21.15.26:0/1249326497' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:43:44.192 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 20:43:44 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-x[141470]: 2023-09-23T20:43:44.097+0000 7fed337d9200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-09-23T20:43:44.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:43:43 smithi026 ceph-mon[95839]: from='client.? 172.21.15.26:0/1249326497' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:43:44.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:43:43 smithi026 ceph-mon[109912]: from='client.? 172.21.15.26:0/1249326497' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:43:44.619 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 20:43:44 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-x[141470]: 2023-09-23T20:43:44.191+0000 7fed337d9200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2023-09-23T20:43:44.619 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 20:43:44 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-x[141470]: 2023-09-23T20:43:44.371+0000 7fed337d9200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-09-23T20:43:45.081 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:43:44 smithi070 ceph-mon[100755]: pgmap v99: 73 pgs: 73 active+clean; 579 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:43:45.081 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 20:43:44 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-x[141470]: 2023-09-23T20:43:44.670+0000 7fed337d9200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2023-09-23T20:43:45.081 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 20:43:44 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-x[141470]: 2023-09-23T20:43:44.915+0000 7fed337d9200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-09-23T20:43:45.357 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 20:43:45 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-x[141470]: 2023-09-23T20:43:45.081+0000 7fed337d9200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-09-23T20:43:45.381 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:43:44 smithi026 ceph-mon[109912]: pgmap v99: 73 pgs: 73 active+clean; 579 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:43:45.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:43:44 smithi026 ceph-mon[95839]: pgmap v99: 73 pgs: 73 active+clean; 579 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:43:45.619 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 20:43:45 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-x[141470]: 2023-09-23T20:43:45.356+0000 7fed337d9200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-09-23T20:43:46.369 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 20:43:46 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-x[141470]: 2023-09-23T20:43:46.111+0000 7fed337d9200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-09-23T20:43:46.369 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 20:43:46 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-x[141470]: 2023-09-23T20:43:46.250+0000 7fed337d9200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-09-23T20:43:46.815 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 20:43:46 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-x[141470]: 2023-09-23T20:43:46.453+0000 7fed337d9200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-09-23T20:43:46.815 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 20:43:46 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-x[141470]: 2023-09-23T20:43:46.574+0000 7fed337d9200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-09-23T20:43:46.815 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 20:43:46 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-x[141470]: 2023-09-23T20:43:46.814+0000 7fed337d9200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-09-23T20:43:47.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:43:46 smithi070 ceph-mon[100755]: pgmap v100: 73 pgs: 73 active+clean; 579 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:43:47.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:43:46 smithi070 ceph-mon[100755]: from='mgr.14451 ' entity='mgr.y' 2023-09-23T20:43:47.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:43:46 smithi070 ceph-mon[100755]: from='mgr.14451 ' entity='mgr.y' 2023-09-23T20:43:47.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:43:46 smithi070 ceph-mon[100755]: from='client.? 172.21.15.26:0/2977792977' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:43:47.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:43:46 smithi026 ceph-mon[95839]: pgmap v100: 73 pgs: 73 active+clean; 579 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:43:47.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:43:46 smithi026 ceph-mon[95839]: from='mgr.14451 ' entity='mgr.y' 2023-09-23T20:43:47.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:43:46 smithi026 ceph-mon[95839]: from='mgr.14451 ' entity='mgr.y' 2023-09-23T20:43:47.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:43:46 smithi026 ceph-mon[95839]: from='client.? 172.21.15.26:0/2977792977' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:43:47.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:43:46 smithi026 ceph-mon[109912]: pgmap v100: 73 pgs: 73 active+clean; 579 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:43:47.383 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:43:46 smithi026 ceph-mon[109912]: from='mgr.14451 ' entity='mgr.y' 2023-09-23T20:43:47.383 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:43:46 smithi026 ceph-mon[109912]: from='mgr.14451 ' entity='mgr.y' 2023-09-23T20:43:47.383 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:43:46 smithi026 ceph-mon[109912]: from='client.? 172.21.15.26:0/2977792977' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:43:48.032 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 20:43:47 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-x[141470]: 2023-09-23T20:43:47.735+0000 7fed337d9200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-09-23T20:43:48.033 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 20:43:47 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-x[141470]: [23/Sep/2023:20:43:47] ENGINE Bus STARTING 2023-09-23T20:43:48.033 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 20:43:47 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-x[141470]: CherryPy Checker: 2023-09-23T20:43:48.033 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 20:43:47 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-x[141470]: The Application mounted at '' has an empty config. 2023-09-23T20:43:48.033 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 20:43:47 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-x[141470]: 2023-09-23T20:43:48.033 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 20:43:47 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-x[141470]: [23/Sep/2023:20:43:47] ENGINE Serving on http://:::9283 2023-09-23T20:43:48.033 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 20:43:47 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-x[141470]: [23/Sep/2023:20:43:47] ENGINE Bus STARTED 2023-09-23T20:43:48.033 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:43:47 smithi070 ceph-mon[100755]: from='mgr.14451 ' entity='mgr.y' 2023-09-23T20:43:48.033 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:43:47 smithi070 ceph-mon[100755]: from='mgr.14451 ' entity='mgr.y' 2023-09-23T20:43:48.033 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:43:47 smithi070 ceph-mon[100755]: Standby manager daemon x restarted 2023-09-23T20:43:48.034 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:43:47 smithi070 ceph-mon[100755]: Standby manager daemon x started 2023-09-23T20:43:48.034 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:43:47 smithi070 ceph-mon[100755]: from='mgr.? 172.21.15.70:0/208042876' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-09-23T20:43:48.034 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:43:47 smithi070 ceph-mon[100755]: from='mgr.? 172.21.15.70:0/208042876' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-09-23T20:43:48.034 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:43:47 smithi070 ceph-mon[100755]: from='mgr.? 172.21.15.70:0/208042876' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-09-23T20:43:48.034 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:43:47 smithi070 ceph-mon[100755]: from='mgr.? 172.21.15.70:0/208042876' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-09-23T20:43:48.034 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:43:47 smithi070 ceph-mon[100755]: pgmap v101: 73 pgs: 73 active+clean; 579 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:43:48.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:43:47 smithi026 ceph-mon[95839]: from='mgr.14451 ' entity='mgr.y' 2023-09-23T20:43:48.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:43:47 smithi026 ceph-mon[95839]: from='mgr.14451 ' entity='mgr.y' 2023-09-23T20:43:48.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:43:47 smithi026 ceph-mon[95839]: Standby manager daemon x restarted 2023-09-23T20:43:48.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:43:47 smithi026 ceph-mon[95839]: Standby manager daemon x started 2023-09-23T20:43:48.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:43:47 smithi026 ceph-mon[95839]: from='mgr.? 172.21.15.70:0/208042876' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-09-23T20:43:48.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:43:47 smithi026 ceph-mon[95839]: from='mgr.? 172.21.15.70:0/208042876' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-09-23T20:43:48.383 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:43:47 smithi026 ceph-mon[95839]: from='mgr.? 172.21.15.70:0/208042876' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-09-23T20:43:48.383 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:43:47 smithi026 ceph-mon[95839]: from='mgr.? 172.21.15.70:0/208042876' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-09-23T20:43:48.383 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:43:47 smithi026 ceph-mon[95839]: pgmap v101: 73 pgs: 73 active+clean; 579 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:43:48.383 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:43:47 smithi026 ceph-mon[109912]: from='mgr.14451 ' entity='mgr.y' 2023-09-23T20:43:48.383 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:43:47 smithi026 ceph-mon[109912]: from='mgr.14451 ' entity='mgr.y' 2023-09-23T20:43:48.383 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:43:47 smithi026 ceph-mon[109912]: Standby manager daemon x restarted 2023-09-23T20:43:48.383 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:43:47 smithi026 ceph-mon[109912]: Standby manager daemon x started 2023-09-23T20:43:48.383 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:43:47 smithi026 ceph-mon[109912]: from='mgr.? 172.21.15.70:0/208042876' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-09-23T20:43:48.383 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:43:47 smithi026 ceph-mon[109912]: from='mgr.? 172.21.15.70:0/208042876' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-09-23T20:43:48.383 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:43:47 smithi026 ceph-mon[109912]: from='mgr.? 172.21.15.70:0/208042876' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-09-23T20:43:48.384 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:43:47 smithi026 ceph-mon[109912]: from='mgr.? 172.21.15.70:0/208042876' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-09-23T20:43:48.384 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:43:47 smithi026 ceph-mon[109912]: pgmap v101: 73 pgs: 73 active+clean; 579 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:43:48.882 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:43:48 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[96086]: ::ffff:172.21.15.70 - - [23/Sep/2023:20:43:48] "GET /metrics HTTP/1.1" 200 239485 "" "Prometheus/2.43.0" 2023-09-23T20:43:49.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:43:48 smithi070 ceph-mon[100755]: mgrmap e21: y(active, since 2m), standbys: x 2023-09-23T20:43:49.371 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:43:48 smithi070 ceph-mon[100755]: from='client.? 172.21.15.26:0/2336273976' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:43:49.371 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:43:48 smithi070 ceph-mon[100755]: from='mgr.14451 172.21.15.26:0/1329296977' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:43:49.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:43:48 smithi026 ceph-mon[109912]: mgrmap e21: y(active, since 2m), standbys: x 2023-09-23T20:43:49.383 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:43:48 smithi026 ceph-mon[109912]: from='client.? 172.21.15.26:0/2336273976' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:43:49.383 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:43:48 smithi026 ceph-mon[109912]: from='mgr.14451 172.21.15.26:0/1329296977' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:43:49.384 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:43:48 smithi026 ceph-mon[95839]: mgrmap e21: y(active, since 2m), standbys: x 2023-09-23T20:43:49.384 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:43:48 smithi026 ceph-mon[95839]: from='client.? 172.21.15.26:0/2336273976' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:43:49.384 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:43:48 smithi026 ceph-mon[95839]: from='mgr.14451 172.21.15.26:0/1329296977' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:43:50.361 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:43:50 smithi070 ceph-mon[100755]: from='mgr.14451 ' entity='mgr.y' 2023-09-23T20:43:50.361 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:43:50 smithi070 ceph-mon[100755]: from='mgr.14451 ' entity='mgr.y' 2023-09-23T20:43:50.361 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:43:50 smithi070 ceph-mon[100755]: pgmap v102: 73 pgs: 73 active+clean; 579 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:43:50.383 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:43:50 smithi026 ceph-mon[109912]: from='mgr.14451 ' entity='mgr.y' 2023-09-23T20:43:50.383 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:43:50 smithi026 ceph-mon[109912]: from='mgr.14451 ' entity='mgr.y' 2023-09-23T20:43:50.383 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:43:50 smithi026 ceph-mon[109912]: pgmap v102: 73 pgs: 73 active+clean; 579 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:43:50.383 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:43:50 smithi026 ceph-mon[95839]: from='mgr.14451 ' entity='mgr.y' 2023-09-23T20:43:50.383 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:43:50 smithi026 ceph-mon[95839]: from='mgr.14451 ' entity='mgr.y' 2023-09-23T20:43:50.383 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:43:50 smithi026 ceph-mon[95839]: pgmap v102: 73 pgs: 73 active+clean; 579 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:43:50.935 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:43:50 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[96086]: ignoring --setuser ceph since I am not root 2023-09-23T20:43:50.935 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:43:50 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[96086]: ignoring --setgroup ceph since I am not root 2023-09-23T20:43:50.935 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:43:50 smithi026 ceph-mgr[96110]: -- 172.21.15.26:0/1112682778 <== mon.2 v2:172.21.15.26:3301/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x558510a309c0 con 0x558510a06c00 2023-09-23T20:43:50.935 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:43:50 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[96086]: 2023-09-23T20:43:50.836+0000 7f83f06b21c0 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2023-09-23T20:43:50.935 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:43:50 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[96086]: 2023-09-23T20:43:50.934+0000 7f83f06b21c0 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-09-23T20:43:51.119 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 20:43:50 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-x[141470]: [23/Sep/2023:20:43:50] ENGINE Bus STOPPING 2023-09-23T20:43:51.120 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 20:43:50 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-x[141470]: [23/Sep/2023:20:43:50] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2023-09-23T20:43:51.120 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 20:43:50 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-x[141470]: [23/Sep/2023:20:43:50] ENGINE Bus STOPPED 2023-09-23T20:43:51.120 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 20:43:50 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-x[141470]: [23/Sep/2023:20:43:50] ENGINE Bus STARTING 2023-09-23T20:43:51.120 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 20:43:50 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-x[141470]: 2023-09-23T20:43:50.798+0000 7fec90c4b700 -1 client.0 error registering admin socket command: (17) File exists 2023-09-23T20:43:51.120 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 20:43:50 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-x[141470]: 2023-09-23T20:43:50.798+0000 7fec90c4b700 -1 client.0 error registering admin socket command: (17) File exists 2023-09-23T20:43:51.120 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 20:43:50 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-x[141470]: 2023-09-23T20:43:50.798+0000 7fec90c4b700 -1 client.0 error registering admin socket command: (17) File exists 2023-09-23T20:43:51.120 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 20:43:50 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-x[141470]: 2023-09-23T20:43:50.798+0000 7fec90c4b700 -1 client.0 error registering admin socket command: (17) File exists 2023-09-23T20:43:51.120 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 20:43:50 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-x[141470]: 2023-09-23T20:43:50.798+0000 7fec90c4b700 -1 client.0 error registering admin socket command: (17) File exists 2023-09-23T20:43:51.120 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 20:43:50 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-x[141470]: 2023-09-23T20:43:50.798+0000 7fec8dc45700 -1 client.0 error registering admin socket command: (17) File exists 2023-09-23T20:43:51.121 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 20:43:50 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-x[141470]: 2023-09-23T20:43:50.798+0000 7fec8dc45700 -1 client.0 error registering admin socket command: (17) File exists 2023-09-23T20:43:51.121 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 20:43:50 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-x[141470]: 2023-09-23T20:43:50.799+0000 7fec8dc45700 -1 client.0 error registering admin socket command: (17) File exists 2023-09-23T20:43:51.121 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 20:43:50 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-x[141470]: 2023-09-23T20:43:50.799+0000 7fec8dc45700 -1 client.0 error registering admin socket command: (17) File exists 2023-09-23T20:43:51.121 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 20:43:50 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-x[141470]: 2023-09-23T20:43:50.799+0000 7fec8dc45700 -1 client.0 error registering admin socket command: (17) File exists 2023-09-23T20:43:51.121 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 20:43:50 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-x[141470]: [23/Sep/2023:20:43:50] ENGINE Serving on http://:::9283 2023-09-23T20:43:51.121 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 20:43:50 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-x[141470]: [23/Sep/2023:20:43:50] ENGINE Bus STARTED 2023-09-23T20:43:51.366 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:43:51 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[96086]: 2023-09-23T20:43:51.042+0000 7f83f06b21c0 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-09-23T20:43:51.367 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:43:51 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[96086]: 2023-09-23T20:43:51.157+0000 7f83f06b21c0 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-09-23T20:43:51.632 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:43:51 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[96086]: 2023-09-23T20:43:51.366+0000 7f83f06b21c0 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-09-23T20:43:51.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:43:51 smithi026 ceph-mon[109912]: from='mgr.14451 ' entity='mgr.y' 2023-09-23T20:43:51.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:43:51 smithi026 ceph-mon[109912]: from='mgr.14451 ' entity='mgr.y' 2023-09-23T20:43:51.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:43:51 smithi026 ceph-mon[109912]: from='mgr.14451 172.21.15.26:0/1329296977' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:43:51.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:43:51 smithi026 ceph-mon[109912]: from='mgr.14451 172.21.15.26:0/1329296977' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:43:51.633 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:43:51 smithi026 ceph-mon[109912]: from='mgr.14451 ' entity='mgr.y' 2023-09-23T20:43:51.633 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:43:51 smithi026 ceph-mon[109912]: from='mgr.14451 172.21.15.26:0/1329296977' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:43:51.633 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:43:51 smithi026 ceph-mon[109912]: from='mgr.14451 172.21.15.26:0/1329296977' entity='mgr.y' cmd=[{"prefix": "mgr fail", "who": "y"}]: dispatch 2023-09-23T20:43:51.633 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:43:51 smithi026 ceph-mon[109912]: from='mgr.14451 ' entity='mgr.y' cmd=[{"prefix": "mgr fail", "who": "y"}]: dispatch 2023-09-23T20:43:51.633 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:43:51 smithi026 ceph-mon[109912]: osdmap e77: 8 total, 8 up, 8 in 2023-09-23T20:43:51.633 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:43:51 smithi026 ceph-mon[109912]: from='mgr.14451 ' entity='mgr.y' cmd='[{"prefix": "mgr fail", "who": "y"}]': finished 2023-09-23T20:43:51.633 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:43:51 smithi026 ceph-mon[109912]: mgrmap e22: x(active, starting, since 0.00548728s) 2023-09-23T20:43:51.633 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:43:51 smithi026 ceph-mon[109912]: from='mgr.24781 172.21.15.70:0/208042876' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-23T20:43:51.633 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:43:51 smithi026 ceph-mon[109912]: from='mgr.24781 172.21.15.70:0/208042876' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T20:43:51.633 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:43:51 smithi026 ceph-mon[109912]: from='mgr.24781 172.21.15.70:0/208042876' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T20:43:51.633 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:43:51 smithi026 ceph-mon[109912]: from='mgr.24781 172.21.15.70:0/208042876' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi026.znfpnl"}]: dispatch 2023-09-23T20:43:51.633 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:43:51 smithi026 ceph-mon[109912]: from='mgr.24781 172.21.15.70:0/208042876' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi070.imqhqy"}]: dispatch 2023-09-23T20:43:51.634 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:43:51 smithi026 ceph-mon[109912]: from='mgr.24781 172.21.15.70:0/208042876' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-09-23T20:43:51.634 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:43:51 smithi026 ceph-mon[109912]: from='mgr.24781 172.21.15.70:0/208042876' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-23T20:43:51.634 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:43:51 smithi026 ceph-mon[109912]: from='mgr.24781 172.21.15.70:0/208042876' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-23T20:43:51.634 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:43:51 smithi026 ceph-mon[109912]: from='mgr.24781 172.21.15.70:0/208042876' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-23T20:43:51.634 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:43:51 smithi026 ceph-mon[109912]: from='mgr.24781 172.21.15.70:0/208042876' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-23T20:43:51.634 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:43:51 smithi026 ceph-mon[109912]: from='mgr.24781 172.21.15.70:0/208042876' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-09-23T20:43:51.634 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:43:51 smithi026 ceph-mon[109912]: from='mgr.24781 172.21.15.70:0/208042876' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-09-23T20:43:51.634 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:43:51 smithi026 ceph-mon[109912]: from='mgr.24781 172.21.15.70:0/208042876' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-09-23T20:43:51.634 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:43:51 smithi026 ceph-mon[109912]: from='mgr.24781 172.21.15.70:0/208042876' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-09-23T20:43:51.634 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:43:51 smithi026 ceph-mon[109912]: from='mgr.24781 172.21.15.70:0/208042876' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-09-23T20:43:51.634 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:43:51 smithi026 ceph-mon[109912]: from='mgr.24781 172.21.15.70:0/208042876' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-09-23T20:43:51.634 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:43:51 smithi026 ceph-mon[109912]: from='mgr.24781 172.21.15.70:0/208042876' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-09-23T20:43:51.634 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:43:51 smithi026 ceph-mon[109912]: Manager daemon x is now available 2023-09-23T20:43:51.635 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:43:51 smithi026 ceph-mon[95839]: from='mgr.14451 ' entity='mgr.y' 2023-09-23T20:43:51.635 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:43:51 smithi026 ceph-mon[95839]: from='mgr.14451 ' entity='mgr.y' 2023-09-23T20:43:51.635 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:43:51 smithi026 ceph-mon[95839]: from='mgr.14451 172.21.15.26:0/1329296977' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:43:51.635 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:43:51 smithi026 ceph-mon[95839]: from='mgr.14451 172.21.15.26:0/1329296977' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:43:51.635 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:43:51 smithi026 ceph-mon[95839]: from='mgr.14451 ' entity='mgr.y' 2023-09-23T20:43:51.635 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:43:51 smithi026 ceph-mon[95839]: from='mgr.14451 172.21.15.26:0/1329296977' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:43:51.635 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:43:51 smithi026 ceph-mon[95839]: from='mgr.14451 172.21.15.26:0/1329296977' entity='mgr.y' cmd=[{"prefix": "mgr fail", "who": "y"}]: dispatch 2023-09-23T20:43:51.635 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:43:51 smithi026 ceph-mon[95839]: from='mgr.14451 ' entity='mgr.y' cmd=[{"prefix": "mgr fail", "who": "y"}]: dispatch 2023-09-23T20:43:51.635 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:43:51 smithi026 ceph-mon[95839]: osdmap e77: 8 total, 8 up, 8 in 2023-09-23T20:43:51.635 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:43:51 smithi026 ceph-mon[95839]: from='mgr.14451 ' entity='mgr.y' cmd='[{"prefix": "mgr fail", "who": "y"}]': finished 2023-09-23T20:43:51.636 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:43:51 smithi026 ceph-mon[95839]: mgrmap e22: x(active, starting, since 0.00548728s) 2023-09-23T20:43:51.636 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:43:51 smithi026 ceph-mon[95839]: from='mgr.24781 172.21.15.70:0/208042876' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-23T20:43:51.636 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:43:51 smithi026 ceph-mon[95839]: from='mgr.24781 172.21.15.70:0/208042876' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T20:43:51.636 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:43:51 smithi026 ceph-mon[95839]: from='mgr.24781 172.21.15.70:0/208042876' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T20:43:51.636 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:43:51 smithi026 ceph-mon[95839]: from='mgr.24781 172.21.15.70:0/208042876' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi026.znfpnl"}]: dispatch 2023-09-23T20:43:51.636 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:43:51 smithi026 ceph-mon[95839]: from='mgr.24781 172.21.15.70:0/208042876' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi070.imqhqy"}]: dispatch 2023-09-23T20:43:51.636 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:43:51 smithi026 ceph-mon[95839]: from='mgr.24781 172.21.15.70:0/208042876' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-09-23T20:43:51.636 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:43:51 smithi026 ceph-mon[95839]: from='mgr.24781 172.21.15.70:0/208042876' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-23T20:43:51.636 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:43:51 smithi026 ceph-mon[95839]: from='mgr.24781 172.21.15.70:0/208042876' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-23T20:43:51.636 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:43:51 smithi026 ceph-mon[95839]: from='mgr.24781 172.21.15.70:0/208042876' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-23T20:43:51.636 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:43:51 smithi026 ceph-mon[95839]: from='mgr.24781 172.21.15.70:0/208042876' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-23T20:43:51.636 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:43:51 smithi026 ceph-mon[95839]: from='mgr.24781 172.21.15.70:0/208042876' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-09-23T20:43:51.637 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:43:51 smithi026 ceph-mon[95839]: from='mgr.24781 172.21.15.70:0/208042876' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-09-23T20:43:51.637 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:43:51 smithi026 ceph-mon[95839]: from='mgr.24781 172.21.15.70:0/208042876' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-09-23T20:43:51.637 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:43:51 smithi026 ceph-mon[95839]: from='mgr.24781 172.21.15.70:0/208042876' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-09-23T20:43:51.637 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:43:51 smithi026 ceph-mon[95839]: from='mgr.24781 172.21.15.70:0/208042876' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-09-23T20:43:51.637 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:43:51 smithi026 ceph-mon[95839]: from='mgr.24781 172.21.15.70:0/208042876' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-09-23T20:43:51.637 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:43:51 smithi026 ceph-mon[95839]: from='mgr.24781 172.21.15.70:0/208042876' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-09-23T20:43:51.637 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:43:51 smithi026 ceph-mon[95839]: Manager daemon x is now available 2023-09-23T20:43:51.637 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:43:51 smithi026 ceph-mon[95839]: from='mgr.24781 ' entity='mgr.x' 2023-09-23T20:43:51.637 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:43:51 smithi026 ceph-mon[109912]: from='mgr.24781 ' entity='mgr.x' 2023-09-23T20:43:51.637 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:43:51 smithi026 ceph-mon[109912]: from='mgr.24781 172.21.15.70:0/208042876' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:43:51.638 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:43:51 smithi026 ceph-mon[109912]: from='mgr.24781 172.21.15.70:0/208042876' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:43:51.638 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:43:51 smithi026 ceph-mon[109912]: from='mgr.24781 172.21.15.70:0/208042876' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:43:51.638 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:43:51 smithi026 ceph-mon[109912]: from='mgr.24781 172.21.15.70:0/208042876' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2023-09-23T20:43:51.638 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:43:51 smithi026 ceph-mon[109912]: from='mgr.24781 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2023-09-23T20:43:51.638 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:43:51 smithi026 ceph-mon[109912]: from='mgr.24781 172.21.15.70:0/208042876' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2023-09-23T20:43:51.638 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:43:51 smithi026 ceph-mon[109912]: from='mgr.24781 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2023-09-23T20:43:51.638 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:43:51 smithi026 ceph-mon[109912]: from='client.? 172.21.15.26:0/2112531417' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:43:51.638 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:43:51 smithi026 ceph-mon[109912]: from='mgr.24781 ' entity='mgr.x' 2023-09-23T20:43:51.638 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:43:51 smithi026 ceph-mon[109912]: from='mgr.24781 ' entity='mgr.x' 2023-09-23T20:43:51.638 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:43:51 smithi026 ceph-mon[95839]: from='mgr.24781 172.21.15.70:0/208042876' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:43:51.639 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:43:51 smithi026 ceph-mon[95839]: from='mgr.24781 172.21.15.70:0/208042876' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:43:51.639 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:43:51 smithi026 ceph-mon[95839]: from='mgr.24781 172.21.15.70:0/208042876' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:43:51.639 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:43:51 smithi026 ceph-mon[95839]: from='mgr.24781 172.21.15.70:0/208042876' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2023-09-23T20:43:51.639 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:43:51 smithi026 ceph-mon[95839]: from='mgr.24781 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2023-09-23T20:43:51.639 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:43:51 smithi026 ceph-mon[95839]: from='mgr.24781 172.21.15.70:0/208042876' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2023-09-23T20:43:51.639 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:43:51 smithi026 ceph-mon[95839]: from='mgr.24781 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2023-09-23T20:43:51.639 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:43:51 smithi026 ceph-mon[95839]: from='client.? 172.21.15.26:0/2112531417' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:43:51.639 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:43:51 smithi026 ceph-mon[95839]: from='mgr.24781 ' entity='mgr.x' 2023-09-23T20:43:51.639 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:43:51 smithi026 ceph-mon[95839]: from='mgr.24781 ' entity='mgr.x' 2023-09-23T20:43:51.869 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 20:43:51 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-x[141470]: ::ffff:172.21.15.70 - - [23/Sep/2023:20:43:51] "GET /metrics HTTP/1.1" 200 30867 "" "Prometheus/2.43.0" 2023-09-23T20:43:51.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:43:51 smithi070 ceph-mon[100755]: from='mgr.14451 ' entity='mgr.y' 2023-09-23T20:43:51.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:43:51 smithi070 ceph-mon[100755]: from='mgr.14451 ' entity='mgr.y' 2023-09-23T20:43:51.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:43:51 smithi070 ceph-mon[100755]: from='mgr.14451 172.21.15.26:0/1329296977' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:43:51.871 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:43:51 smithi070 ceph-mon[100755]: from='mgr.14451 172.21.15.26:0/1329296977' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:43:51.871 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:43:51 smithi070 ceph-mon[100755]: from='mgr.14451 ' entity='mgr.y' 2023-09-23T20:43:51.871 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:43:51 smithi070 ceph-mon[100755]: from='mgr.14451 172.21.15.26:0/1329296977' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:43:51.871 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:43:51 smithi070 ceph-mon[100755]: from='mgr.14451 172.21.15.26:0/1329296977' entity='mgr.y' cmd=[{"prefix": "mgr fail", "who": "y"}]: dispatch 2023-09-23T20:43:51.871 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:43:51 smithi070 ceph-mon[100755]: from='mgr.14451 ' entity='mgr.y' cmd=[{"prefix": "mgr fail", "who": "y"}]: dispatch 2023-09-23T20:43:51.871 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:43:51 smithi070 ceph-mon[100755]: osdmap e77: 8 total, 8 up, 8 in 2023-09-23T20:43:51.871 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:43:51 smithi070 ceph-mon[100755]: from='mgr.14451 ' entity='mgr.y' cmd='[{"prefix": "mgr fail", "who": "y"}]': finished 2023-09-23T20:43:51.871 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:43:51 smithi070 ceph-mon[100755]: mgrmap e22: x(active, starting, since 0.00548728s) 2023-09-23T20:43:51.871 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:43:51 smithi070 ceph-mon[100755]: from='mgr.24781 172.21.15.70:0/208042876' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-23T20:43:51.871 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:43:51 smithi070 ceph-mon[100755]: from='mgr.24781 172.21.15.70:0/208042876' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T20:43:51.871 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:43:51 smithi070 ceph-mon[100755]: from='mgr.24781 172.21.15.70:0/208042876' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T20:43:51.871 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:43:51 smithi070 ceph-mon[100755]: from='mgr.24781 172.21.15.70:0/208042876' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi026.znfpnl"}]: dispatch 2023-09-23T20:43:51.872 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:43:51 smithi070 ceph-mon[100755]: from='mgr.24781 172.21.15.70:0/208042876' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi070.imqhqy"}]: dispatch 2023-09-23T20:43:51.872 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:43:51 smithi070 ceph-mon[100755]: from='mgr.24781 172.21.15.70:0/208042876' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-09-23T20:43:51.872 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:43:51 smithi070 ceph-mon[100755]: from='mgr.24781 172.21.15.70:0/208042876' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-23T20:43:51.872 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:43:51 smithi070 ceph-mon[100755]: from='mgr.24781 172.21.15.70:0/208042876' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-23T20:43:51.872 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:43:51 smithi070 ceph-mon[100755]: from='mgr.24781 172.21.15.70:0/208042876' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-23T20:43:51.872 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:43:51 smithi070 ceph-mon[100755]: from='mgr.24781 172.21.15.70:0/208042876' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-23T20:43:51.872 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:43:51 smithi070 ceph-mon[100755]: from='mgr.24781 172.21.15.70:0/208042876' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-09-23T20:43:51.872 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:43:51 smithi070 ceph-mon[100755]: from='mgr.24781 172.21.15.70:0/208042876' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-09-23T20:43:51.872 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:43:51 smithi070 ceph-mon[100755]: from='mgr.24781 172.21.15.70:0/208042876' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-09-23T20:43:51.872 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:43:51 smithi070 ceph-mon[100755]: from='mgr.24781 172.21.15.70:0/208042876' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-09-23T20:43:51.872 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:43:51 smithi070 ceph-mon[100755]: from='mgr.24781 172.21.15.70:0/208042876' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-09-23T20:43:51.872 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:43:51 smithi070 ceph-mon[100755]: from='mgr.24781 172.21.15.70:0/208042876' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-09-23T20:43:51.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:43:51 smithi070 ceph-mon[100755]: from='mgr.24781 172.21.15.70:0/208042876' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-09-23T20:43:51.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:43:51 smithi070 ceph-mon[100755]: Manager daemon x is now available 2023-09-23T20:43:51.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:43:51 smithi070 ceph-mon[100755]: from='mgr.24781 ' entity='mgr.x' 2023-09-23T20:43:51.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:43:51 smithi070 ceph-mon[100755]: from='mgr.24781 172.21.15.70:0/208042876' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:43:51.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:43:51 smithi070 ceph-mon[100755]: from='mgr.24781 172.21.15.70:0/208042876' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:43:51.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:43:51 smithi070 ceph-mon[100755]: from='mgr.24781 172.21.15.70:0/208042876' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:43:51.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:43:51 smithi070 ceph-mon[100755]: from='mgr.24781 172.21.15.70:0/208042876' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2023-09-23T20:43:51.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:43:51 smithi070 ceph-mon[100755]: from='mgr.24781 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2023-09-23T20:43:51.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:43:51 smithi070 ceph-mon[100755]: from='mgr.24781 172.21.15.70:0/208042876' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2023-09-23T20:43:51.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:43:51 smithi070 ceph-mon[100755]: from='mgr.24781 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2023-09-23T20:43:51.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:43:51 smithi070 ceph-mon[100755]: from='client.? 172.21.15.26:0/2112531417' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:43:51.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:43:51 smithi070 ceph-mon[100755]: from='mgr.24781 ' entity='mgr.x' 2023-09-23T20:43:51.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:43:51 smithi070 ceph-mon[100755]: from='mgr.24781 ' entity='mgr.x' 2023-09-23T20:43:52.023 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:43:51 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[96086]: 2023-09-23T20:43:51.766+0000 7f83f06b21c0 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-09-23T20:43:52.024 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:43:51 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[96086]: 2023-09-23T20:43:51.872+0000 7f83f06b21c0 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-09-23T20:43:52.024 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:43:52 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[96086]: 2023-09-23T20:43:52.022+0000 7f83f06b21c0 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-09-23T20:43:52.382 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:43:52 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[96086]: 2023-09-23T20:43:52.135+0000 7f83f06b21c0 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-09-23T20:43:53.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:43:52 smithi070 ceph-mon[100755]: Deploying cephadm binary to smithi070 2023-09-23T20:43:53.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:43:52 smithi070 ceph-mon[100755]: Deploying cephadm binary to smithi026 2023-09-23T20:43:53.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:43:52 smithi070 ceph-mon[100755]: mgrmap e23: x(active, since 1.00738s) 2023-09-23T20:43:53.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:43:52 smithi070 ceph-mon[100755]: pgmap v3: 73 pgs: 73 active+clean; 580 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:43:53.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:43:52 smithi026 ceph-mon[95839]: Deploying cephadm binary to smithi070 2023-09-23T20:43:53.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:43:52 smithi026 ceph-mon[95839]: Deploying cephadm binary to smithi026 2023-09-23T20:43:53.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:43:52 smithi026 ceph-mon[95839]: mgrmap e23: x(active, since 1.00738s) 2023-09-23T20:43:53.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:43:52 smithi026 ceph-mon[95839]: pgmap v3: 73 pgs: 73 active+clean; 580 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:43:53.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:43:52 smithi026 ceph-mon[109912]: Deploying cephadm binary to smithi070 2023-09-23T20:43:53.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:43:52 smithi026 ceph-mon[109912]: Deploying cephadm binary to smithi026 2023-09-23T20:43:53.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:43:52 smithi026 ceph-mon[109912]: mgrmap e23: x(active, since 1.00738s) 2023-09-23T20:43:53.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:43:52 smithi026 ceph-mon[109912]: pgmap v3: 73 pgs: 73 active+clean; 580 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:43:53.133 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:43:52 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[96086]: 2023-09-23T20:43:52.650+0000 7f83f06b21c0 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-09-23T20:43:53.651 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:43:53 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[96086]: 2023-09-23T20:43:53.380+0000 7f83f06b21c0 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-09-23T20:43:53.651 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:43:53 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[96086]: 2023-09-23T20:43:53.581+0000 7f83f06b21c0 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-09-23T20:43:54.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:43:53 smithi070 ceph-mon[100755]: [23/Sep/2023:20:43:52] ENGINE Bus STARTING 2023-09-23T20:43:54.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:43:53 smithi070 ceph-mon[100755]: [23/Sep/2023:20:43:52] ENGINE Serving on https://172.21.15.70:7150 2023-09-23T20:43:54.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:43:53 smithi070 ceph-mon[100755]: [23/Sep/2023:20:43:52] ENGINE Serving on http://172.21.15.70:8765 2023-09-23T20:43:54.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:43:53 smithi070 ceph-mon[100755]: [23/Sep/2023:20:43:52] ENGINE Bus STARTED 2023-09-23T20:43:54.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:43:53 smithi070 ceph-mon[100755]: pgmap v4: 73 pgs: 73 active+clean; 580 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:43:54.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:43:53 smithi070 ceph-mon[100755]: mgrmap e24: x(active, since 2s) 2023-09-23T20:43:54.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:43:53 smithi070 ceph-mon[100755]: from='client.? 172.21.15.26:0/3634010949' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:43:54.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:43:53 smithi026 ceph-mon[95839]: [23/Sep/2023:20:43:52] ENGINE Bus STARTING 2023-09-23T20:43:54.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:43:53 smithi026 ceph-mon[95839]: [23/Sep/2023:20:43:52] ENGINE Serving on https://172.21.15.70:7150 2023-09-23T20:43:54.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:43:53 smithi026 ceph-mon[95839]: [23/Sep/2023:20:43:52] ENGINE Serving on http://172.21.15.70:8765 2023-09-23T20:43:54.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:43:53 smithi026 ceph-mon[95839]: [23/Sep/2023:20:43:52] ENGINE Bus STARTED 2023-09-23T20:43:54.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:43:53 smithi026 ceph-mon[95839]: pgmap v4: 73 pgs: 73 active+clean; 580 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:43:54.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:43:53 smithi026 ceph-mon[95839]: mgrmap e24: x(active, since 2s) 2023-09-23T20:43:54.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:43:53 smithi026 ceph-mon[95839]: from='client.? 172.21.15.26:0/3634010949' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:43:54.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:43:53 smithi026 ceph-mon[109912]: [23/Sep/2023:20:43:52] ENGINE Bus STARTING 2023-09-23T20:43:54.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:43:53 smithi026 ceph-mon[109912]: [23/Sep/2023:20:43:52] ENGINE Serving on https://172.21.15.70:7150 2023-09-23T20:43:54.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:43:53 smithi026 ceph-mon[109912]: [23/Sep/2023:20:43:52] ENGINE Serving on http://172.21.15.70:8765 2023-09-23T20:43:54.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:43:53 smithi026 ceph-mon[109912]: [23/Sep/2023:20:43:52] ENGINE Bus STARTED 2023-09-23T20:43:54.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:43:53 smithi026 ceph-mon[109912]: pgmap v4: 73 pgs: 73 active+clean; 580 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:43:54.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:43:53 smithi026 ceph-mon[109912]: mgrmap e24: x(active, since 2s) 2023-09-23T20:43:54.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:43:53 smithi026 ceph-mon[109912]: from='client.? 172.21.15.26:0/3634010949' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:43:54.882 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:43:54 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[96086]: 2023-09-23T20:43:54.582+0000 7f83f06b21c0 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-09-23T20:43:54.882 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:43:54 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[96086]: 2023-09-23T20:43:54.699+0000 7f83f06b21c0 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2023-09-23T20:43:55.291 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:43:54 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[96086]: 2023-09-23T20:43:54.932+0000 7f83f06b21c0 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-09-23T20:43:55.632 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:43:55 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[96086]: 2023-09-23T20:43:55.289+0000 7f83f06b21c0 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2023-09-23T20:43:55.632 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:43:55 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[96086]: 2023-09-23T20:43:55.411+0000 7f83f06b21c0 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-09-23T20:43:56.105 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:43:55 smithi026 ceph-mon[95839]: pgmap v5: 73 pgs: 73 active+clean; 580 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:43:56.105 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:43:55 smithi026 ceph-mon[109912]: pgmap v5: 73 pgs: 73 active+clean; 580 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:43:56.106 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:43:55 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[96086]: 2023-09-23T20:43:55.761+0000 7f83f06b21c0 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-09-23T20:43:56.106 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:43:56 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[96086]: 2023-09-23T20:43:56.103+0000 7f83f06b21c0 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-09-23T20:43:56.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:43:55 smithi070 ceph-mon[100755]: pgmap v5: 73 pgs: 73 active+clean; 580 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:43:56.975 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:43:56 smithi026 ceph-mon[109912]: from='client.? 172.21.15.26:0/82994638' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:43:56.976 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:43:56 smithi026 ceph-mon[109912]: from='mgr.24781 ' entity='mgr.x' 2023-09-23T20:43:56.976 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:43:56 smithi026 ceph-mon[109912]: from='mgr.24781 ' entity='mgr.x' 2023-09-23T20:43:56.976 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:43:56 smithi026 ceph-mon[95839]: from='client.? 172.21.15.26:0/82994638' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:43:56.976 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:43:56 smithi026 ceph-mon[95839]: from='mgr.24781 ' entity='mgr.x' 2023-09-23T20:43:56.977 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:43:56 smithi026 ceph-mon[95839]: from='mgr.24781 ' entity='mgr.x' 2023-09-23T20:43:57.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:43:56 smithi070 ceph-mon[100755]: from='client.? 172.21.15.26:0/82994638' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:43:57.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:43:56 smithi070 ceph-mon[100755]: from='mgr.24781 ' entity='mgr.x' 2023-09-23T20:43:57.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:43:56 smithi070 ceph-mon[100755]: from='mgr.24781 ' entity='mgr.x' 2023-09-23T20:43:57.382 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:43:56 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[96086]: 2023-09-23T20:43:56.973+0000 7f83f06b21c0 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-09-23T20:43:57.382 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:43:57 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[96086]: 2023-09-23T20:43:57.137+0000 7f83f06b21c0 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-09-23T20:43:57.707 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:43:57 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[96086]: 2023-09-23T20:43:57.390+0000 7f83f06b21c0 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-09-23T20:43:57.708 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:43:57 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[96086]: 2023-09-23T20:43:57.531+0000 7f83f06b21c0 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-09-23T20:43:58.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:43:57 smithi070 ceph-mon[100755]: pgmap v6: 73 pgs: 73 active+clean; 580 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:43:58.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:43:57 smithi070 ceph-mon[100755]: from='mgr.24781 ' entity='mgr.x' 2023-09-23T20:43:58.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:43:57 smithi070 ceph-mon[100755]: from='mgr.24781 ' entity='mgr.x' 2023-09-23T20:43:58.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:43:57 smithi070 ceph-mon[100755]: from='mgr.24781 172.21.15.70:0/208042876' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:43:58.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:43:57 smithi070 ceph-mon[100755]: from='mgr.24781 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:43:58.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:43:57 smithi070 ceph-mon[100755]: from='mgr.24781 172.21.15.70:0/208042876' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:43:58.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:43:57 smithi070 ceph-mon[100755]: from='mgr.24781 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:43:58.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:43:57 smithi070 ceph-mon[100755]: from='mgr.24781 172.21.15.70:0/208042876' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:43:58.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:43:57 smithi070 ceph-mon[100755]: from='mgr.24781 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:43:58.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:43:57 smithi070 ceph-mon[100755]: from='mgr.24781 172.21.15.70:0/208042876' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:43:58.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:43:57 smithi070 ceph-mon[100755]: from='mgr.24781 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:43:58.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:43:57 smithi070 ceph-mon[100755]: from='mgr.24781 ' entity='mgr.x' 2023-09-23T20:43:58.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:43:57 smithi070 ceph-mon[100755]: from='mgr.24781 ' entity='mgr.x' 2023-09-23T20:43:58.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:43:57 smithi070 ceph-mon[100755]: from='mgr.24781 ' entity='mgr.x' 2023-09-23T20:43:58.131 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:43:57 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[96086]: 2023-09-23T20:43:57.807+0000 7f83f06b21c0 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-09-23T20:43:58.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:43:57 smithi026 ceph-mon[95839]: pgmap v6: 73 pgs: 73 active+clean; 580 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:43:58.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:43:57 smithi026 ceph-mon[95839]: from='mgr.24781 ' entity='mgr.x' 2023-09-23T20:43:58.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:43:57 smithi026 ceph-mon[95839]: from='mgr.24781 ' entity='mgr.x' 2023-09-23T20:43:58.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:43:57 smithi026 ceph-mon[95839]: from='mgr.24781 172.21.15.70:0/208042876' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:43:58.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:43:57 smithi026 ceph-mon[95839]: from='mgr.24781 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:43:58.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:43:57 smithi026 ceph-mon[95839]: from='mgr.24781 172.21.15.70:0/208042876' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:43:58.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:43:57 smithi026 ceph-mon[95839]: from='mgr.24781 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:43:58.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:43:57 smithi026 ceph-mon[95839]: from='mgr.24781 172.21.15.70:0/208042876' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:43:58.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:43:57 smithi026 ceph-mon[95839]: from='mgr.24781 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:43:58.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:43:57 smithi026 ceph-mon[95839]: from='mgr.24781 172.21.15.70:0/208042876' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:43:58.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:43:57 smithi026 ceph-mon[95839]: from='mgr.24781 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:43:58.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:43:57 smithi026 ceph-mon[95839]: from='mgr.24781 ' entity='mgr.x' 2023-09-23T20:43:58.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:43:57 smithi026 ceph-mon[95839]: from='mgr.24781 ' entity='mgr.x' 2023-09-23T20:43:58.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:43:57 smithi026 ceph-mon[95839]: from='mgr.24781 ' entity='mgr.x' 2023-09-23T20:43:58.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:43:57 smithi026 ceph-mon[109912]: pgmap v6: 73 pgs: 73 active+clean; 580 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:43:58.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:43:57 smithi026 ceph-mon[109912]: from='mgr.24781 ' entity='mgr.x' 2023-09-23T20:43:58.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:43:57 smithi026 ceph-mon[109912]: from='mgr.24781 ' entity='mgr.x' 2023-09-23T20:43:58.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:43:57 smithi026 ceph-mon[109912]: from='mgr.24781 172.21.15.70:0/208042876' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:43:58.134 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:43:57 smithi026 ceph-mon[109912]: from='mgr.24781 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:43:58.134 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:43:57 smithi026 ceph-mon[109912]: from='mgr.24781 172.21.15.70:0/208042876' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:43:58.134 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:43:57 smithi026 ceph-mon[109912]: from='mgr.24781 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:43:58.134 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:43:57 smithi026 ceph-mon[109912]: from='mgr.24781 172.21.15.70:0/208042876' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:43:58.134 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:43:57 smithi026 ceph-mon[109912]: from='mgr.24781 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:43:58.134 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:43:57 smithi026 ceph-mon[109912]: from='mgr.24781 172.21.15.70:0/208042876' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:43:58.134 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:43:57 smithi026 ceph-mon[109912]: from='mgr.24781 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:43:58.134 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:43:57 smithi026 ceph-mon[109912]: from='mgr.24781 ' entity='mgr.x' 2023-09-23T20:43:58.134 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:43:57 smithi026 ceph-mon[109912]: from='mgr.24781 ' entity='mgr.x' 2023-09-23T20:43:58.134 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:43:57 smithi026 ceph-mon[109912]: from='mgr.24781 ' entity='mgr.x' 2023-09-23T20:43:58.995 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:43:58 smithi026 ceph-mon[95839]: Adjusting osd_memory_target on smithi026 to 2501M 2023-09-23T20:43:58.995 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:43:58 smithi026 ceph-mon[95839]: from='client.? 172.21.15.26:0/3230988950' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:43:58.995 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:43:58 smithi026 ceph-mon[95839]: from='mgr.24781 ' entity='mgr.x' 2023-09-23T20:43:58.996 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:43:58 smithi026 ceph-mon[95839]: from='mgr.24781 ' entity='mgr.x' 2023-09-23T20:43:58.996 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:43:58 smithi026 ceph-mon[95839]: pgmap v7: 73 pgs: 73 active+clean; 580 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:43:58.996 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:43:58 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[96086]: 2023-09-23T20:43:58.873+0000 7f83f06b21c0 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-09-23T20:43:58.996 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:43:58 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[96086]: [23/Sep/2023:20:43:58] ENGINE Bus STARTING 2023-09-23T20:43:58.996 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:43:58 smithi026 ceph-mon[109912]: Adjusting osd_memory_target on smithi026 to 2501M 2023-09-23T20:43:58.996 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:43:58 smithi026 ceph-mon[109912]: from='client.? 172.21.15.26:0/3230988950' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:43:58.997 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:43:58 smithi026 ceph-mon[109912]: from='mgr.24781 ' entity='mgr.x' 2023-09-23T20:43:58.997 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:43:58 smithi026 ceph-mon[109912]: from='mgr.24781 ' entity='mgr.x' 2023-09-23T20:43:58.997 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:43:58 smithi026 ceph-mon[109912]: pgmap v7: 73 pgs: 73 active+clean; 580 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:43:59.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:43:58 smithi070 ceph-mon[100755]: Adjusting osd_memory_target on smithi026 to 2501M 2023-09-23T20:43:59.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:43:58 smithi070 ceph-mon[100755]: from='client.? 172.21.15.26:0/3230988950' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:43:59.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:43:58 smithi070 ceph-mon[100755]: from='mgr.24781 ' entity='mgr.x' 2023-09-23T20:43:59.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:43:58 smithi070 ceph-mon[100755]: from='mgr.24781 ' entity='mgr.x' 2023-09-23T20:43:59.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:43:58 smithi070 ceph-mon[100755]: pgmap v7: 73 pgs: 73 active+clean; 580 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:43:59.382 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:43:58 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[96086]: CherryPy Checker: 2023-09-23T20:43:59.382 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:43:58 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[96086]: The Application mounted at '' has an empty config. 2023-09-23T20:43:59.382 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:43:58 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[96086]: 2023-09-23T20:43:59.382 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:43:58 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[96086]: [23/Sep/2023:20:43:58] ENGINE Serving on http://:::9283 2023-09-23T20:43:59.382 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:43:58 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[96086]: [23/Sep/2023:20:43:58] ENGINE Bus STARTED 2023-09-23T20:43:59.841 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:43:59 smithi070 ceph-mon[100755]: Standby manager daemon y started 2023-09-23T20:43:59.841 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:43:59 smithi070 ceph-mon[100755]: from='mgr.? 172.21.15.26:0/844133129' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2023-09-23T20:43:59.841 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:43:59 smithi070 ceph-mon[100755]: from='mgr.? 172.21.15.26:0/844133129' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-09-23T20:43:59.841 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:43:59 smithi070 ceph-mon[100755]: from='mgr.? 172.21.15.26:0/844133129' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2023-09-23T20:43:59.842 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:43:59 smithi070 ceph-mon[100755]: from='mgr.? 172.21.15.26:0/844133129' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-09-23T20:44:00.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:43:59 smithi026 ceph-mon[95839]: Standby manager daemon y started 2023-09-23T20:44:00.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:43:59 smithi026 ceph-mon[95839]: from='mgr.? 172.21.15.26:0/844133129' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2023-09-23T20:44:00.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:43:59 smithi026 ceph-mon[95839]: from='mgr.? 172.21.15.26:0/844133129' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-09-23T20:44:00.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:43:59 smithi026 ceph-mon[95839]: from='mgr.? 172.21.15.26:0/844133129' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2023-09-23T20:44:00.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:43:59 smithi026 ceph-mon[95839]: from='mgr.? 172.21.15.26:0/844133129' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-09-23T20:44:00.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:43:59 smithi026 ceph-mon[109912]: Standby manager daemon y started 2023-09-23T20:44:00.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:43:59 smithi026 ceph-mon[109912]: from='mgr.? 172.21.15.26:0/844133129' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2023-09-23T20:44:00.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:43:59 smithi026 ceph-mon[109912]: from='mgr.? 172.21.15.26:0/844133129' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-09-23T20:44:00.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:43:59 smithi026 ceph-mon[109912]: from='mgr.? 172.21.15.26:0/844133129' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2023-09-23T20:44:00.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:43:59 smithi026 ceph-mon[109912]: from='mgr.? 172.21.15.26:0/844133129' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-09-23T20:44:01.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:44:00 smithi070 ceph-mon[100755]: from='mgr.24781 172.21.15.70:0/208042876' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-09-23T20:44:01.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:44:00 smithi070 ceph-mon[100755]: mgrmap e25: x(active, since 9s), standbys: y 2023-09-23T20:44:01.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:44:00 smithi070 ceph-mon[100755]: from='client.? 172.21.15.26:0/3370085006' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:44:01.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:44:00 smithi070 ceph-mon[100755]: pgmap v8: 73 pgs: 73 active+clean; 580 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:44:01.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:44:00 smithi026 ceph-mon[109912]: from='mgr.24781 172.21.15.70:0/208042876' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-09-23T20:44:01.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:44:00 smithi026 ceph-mon[109912]: mgrmap e25: x(active, since 9s), standbys: y 2023-09-23T20:44:01.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:44:00 smithi026 ceph-mon[109912]: from='client.? 172.21.15.26:0/3370085006' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:44:01.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:44:00 smithi026 ceph-mon[109912]: pgmap v8: 73 pgs: 73 active+clean; 580 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:44:01.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:44:00 smithi026 ceph-mon[95839]: from='mgr.24781 172.21.15.70:0/208042876' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-09-23T20:44:01.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:44:00 smithi026 ceph-mon[95839]: mgrmap e25: x(active, since 9s), standbys: y 2023-09-23T20:44:01.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:44:00 smithi026 ceph-mon[95839]: from='client.? 172.21.15.26:0/3370085006' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:44:01.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:44:00 smithi026 ceph-mon[95839]: pgmap v8: 73 pgs: 73 active+clean; 580 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:44:01.869 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 20:44:01 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-x[141470]: ::ffff:172.21.15.70 - - [23/Sep/2023:20:44:01] "GET /metrics HTTP/1.1" 200 30867 "" "Prometheus/2.43.0" 2023-09-23T20:44:04.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:44:03 smithi070 ceph-mon[100755]: pgmap v9: 73 pgs: 73 active+clean; 580 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:44:04.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:44:03 smithi070 ceph-mon[100755]: from='client.? 172.21.15.26:0/1385663600' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:44:04.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:44:03 smithi070 ceph-mon[100755]: from='mgr.24781 ' entity='mgr.x' 2023-09-23T20:44:04.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:44:03 smithi070 ceph-mon[100755]: from='mgr.24781 ' entity='mgr.x' 2023-09-23T20:44:04.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:44:03 smithi070 ceph-mon[100755]: from='mgr.24781 172.21.15.70:0/208042876' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:44:04.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:44:03 smithi070 ceph-mon[100755]: from='mgr.24781 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:44:04.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:44:03 smithi070 ceph-mon[100755]: from='mgr.24781 172.21.15.70:0/208042876' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:44:04.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:44:03 smithi070 ceph-mon[100755]: from='mgr.24781 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:44:04.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:44:03 smithi070 ceph-mon[100755]: from='mgr.24781 172.21.15.70:0/208042876' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:44:04.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:44:03 smithi070 ceph-mon[100755]: from='mgr.24781 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:44:04.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:44:03 smithi070 ceph-mon[100755]: from='mgr.24781 172.21.15.70:0/208042876' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:44:04.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:44:03 smithi070 ceph-mon[100755]: from='mgr.24781 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:44:04.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:44:03 smithi070 ceph-mon[100755]: from='mgr.24781 ' entity='mgr.x' 2023-09-23T20:44:04.121 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:44:03 smithi070 ceph-mon[100755]: from='mgr.24781 172.21.15.70:0/208042876' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:44:04.121 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:44:03 smithi070 ceph-mon[100755]: from='mgr.24781 172.21.15.70:0/208042876' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:44:04.131 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:44:03 smithi026 ceph-mon[95839]: pgmap v9: 73 pgs: 73 active+clean; 580 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:44:04.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:44:03 smithi026 ceph-mon[95839]: from='client.? 172.21.15.26:0/1385663600' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:44:04.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:44:03 smithi026 ceph-mon[95839]: from='mgr.24781 ' entity='mgr.x' 2023-09-23T20:44:04.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:44:03 smithi026 ceph-mon[95839]: from='mgr.24781 ' entity='mgr.x' 2023-09-23T20:44:04.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:44:03 smithi026 ceph-mon[95839]: from='mgr.24781 172.21.15.70:0/208042876' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:44:04.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:44:03 smithi026 ceph-mon[95839]: from='mgr.24781 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:44:04.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:44:03 smithi026 ceph-mon[95839]: from='mgr.24781 172.21.15.70:0/208042876' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:44:04.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:44:03 smithi026 ceph-mon[95839]: from='mgr.24781 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:44:04.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:44:03 smithi026 ceph-mon[95839]: from='mgr.24781 172.21.15.70:0/208042876' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:44:04.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:44:03 smithi026 ceph-mon[95839]: from='mgr.24781 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:44:04.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:44:03 smithi026 ceph-mon[95839]: from='mgr.24781 172.21.15.70:0/208042876' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:44:04.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:44:03 smithi026 ceph-mon[95839]: from='mgr.24781 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:44:04.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:44:03 smithi026 ceph-mon[95839]: from='mgr.24781 ' entity='mgr.x' 2023-09-23T20:44:04.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:44:03 smithi026 ceph-mon[95839]: from='mgr.24781 172.21.15.70:0/208042876' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:44:04.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:44:03 smithi026 ceph-mon[95839]: from='mgr.24781 172.21.15.70:0/208042876' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:44:04.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:44:03 smithi026 ceph-mon[109912]: pgmap v9: 73 pgs: 73 active+clean; 580 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:44:04.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:44:03 smithi026 ceph-mon[109912]: from='client.? 172.21.15.26:0/1385663600' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:44:04.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:44:03 smithi026 ceph-mon[109912]: from='mgr.24781 ' entity='mgr.x' 2023-09-23T20:44:04.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:44:03 smithi026 ceph-mon[109912]: from='mgr.24781 ' entity='mgr.x' 2023-09-23T20:44:04.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:44:03 smithi026 ceph-mon[109912]: from='mgr.24781 172.21.15.70:0/208042876' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:44:04.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:44:03 smithi026 ceph-mon[109912]: from='mgr.24781 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:44:04.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:44:03 smithi026 ceph-mon[109912]: from='mgr.24781 172.21.15.70:0/208042876' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:44:04.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:44:03 smithi026 ceph-mon[109912]: from='mgr.24781 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:44:04.134 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:44:03 smithi026 ceph-mon[109912]: from='mgr.24781 172.21.15.70:0/208042876' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:44:04.134 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:44:03 smithi026 ceph-mon[109912]: from='mgr.24781 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:44:04.134 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:44:03 smithi026 ceph-mon[109912]: from='mgr.24781 172.21.15.70:0/208042876' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:44:04.134 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:44:03 smithi026 ceph-mon[109912]: from='mgr.24781 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:44:04.134 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:44:03 smithi026 ceph-mon[109912]: from='mgr.24781 ' entity='mgr.x' 2023-09-23T20:44:04.134 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:44:03 smithi026 ceph-mon[109912]: from='mgr.24781 172.21.15.70:0/208042876' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:44:04.134 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:44:03 smithi026 ceph-mon[109912]: from='mgr.24781 172.21.15.70:0/208042876' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:44:05.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:44:04 smithi070 ceph-mon[100755]: Adjusting osd_memory_target on smithi070 to 2500M 2023-09-23T20:44:05.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:44:04 smithi070 ceph-mon[100755]: Updating smithi026:/etc/ceph/ceph.conf 2023-09-23T20:44:05.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:44:04 smithi070 ceph-mon[100755]: Updating smithi070:/etc/ceph/ceph.conf 2023-09-23T20:44:05.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:44:04 smithi070 ceph-mon[100755]: Updating smithi070:/var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-23T20:44:05.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:44:04 smithi070 ceph-mon[100755]: Updating smithi026:/var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-23T20:44:05.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:44:04 smithi070 ceph-mon[100755]: from='mgr.24781 ' entity='mgr.x' 2023-09-23T20:44:05.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:44:04 smithi070 ceph-mon[100755]: from='mgr.24781 ' entity='mgr.x' 2023-09-23T20:44:05.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:44:04 smithi070 ceph-mon[100755]: from='mgr.24781 ' entity='mgr.x' 2023-09-23T20:44:05.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:44:04 smithi070 ceph-mon[100755]: from='mgr.24781 ' entity='mgr.x' 2023-09-23T20:44:05.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:44:04 smithi070 ceph-mon[100755]: from='mgr.24781 ' entity='mgr.x' 2023-09-23T20:44:05.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:44:04 smithi026 ceph-mon[95839]: Adjusting osd_memory_target on smithi070 to 2500M 2023-09-23T20:44:05.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:44:04 smithi026 ceph-mon[95839]: Updating smithi026:/etc/ceph/ceph.conf 2023-09-23T20:44:05.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:44:04 smithi026 ceph-mon[95839]: Updating smithi070:/etc/ceph/ceph.conf 2023-09-23T20:44:05.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:44:04 smithi026 ceph-mon[95839]: Updating smithi070:/var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-23T20:44:05.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:44:04 smithi026 ceph-mon[95839]: Updating smithi026:/var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-23T20:44:05.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:44:04 smithi026 ceph-mon[95839]: from='mgr.24781 ' entity='mgr.x' 2023-09-23T20:44:05.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:44:04 smithi026 ceph-mon[95839]: from='mgr.24781 ' entity='mgr.x' 2023-09-23T20:44:05.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:44:04 smithi026 ceph-mon[95839]: from='mgr.24781 ' entity='mgr.x' 2023-09-23T20:44:05.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:44:04 smithi026 ceph-mon[95839]: from='mgr.24781 ' entity='mgr.x' 2023-09-23T20:44:05.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:44:04 smithi026 ceph-mon[95839]: from='mgr.24781 ' entity='mgr.x' 2023-09-23T20:44:05.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:44:04 smithi026 ceph-mon[109912]: Adjusting osd_memory_target on smithi070 to 2500M 2023-09-23T20:44:05.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:44:04 smithi026 ceph-mon[109912]: Updating smithi026:/etc/ceph/ceph.conf 2023-09-23T20:44:05.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:44:04 smithi026 ceph-mon[109912]: Updating smithi070:/etc/ceph/ceph.conf 2023-09-23T20:44:05.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:44:04 smithi026 ceph-mon[109912]: Updating smithi070:/var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-23T20:44:05.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:44:04 smithi026 ceph-mon[109912]: Updating smithi026:/var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-23T20:44:05.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:44:04 smithi026 ceph-mon[109912]: from='mgr.24781 ' entity='mgr.x' 2023-09-23T20:44:05.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:44:04 smithi026 ceph-mon[109912]: from='mgr.24781 ' entity='mgr.x' 2023-09-23T20:44:05.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:44:04 smithi026 ceph-mon[109912]: from='mgr.24781 ' entity='mgr.x' 2023-09-23T20:44:05.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:44:04 smithi026 ceph-mon[109912]: from='mgr.24781 ' entity='mgr.x' 2023-09-23T20:44:05.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:44:04 smithi026 ceph-mon[109912]: from='mgr.24781 ' entity='mgr.x' 2023-09-23T20:44:06.004 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:44:05 smithi026 ceph-mon[95839]: Updating smithi026:/etc/ceph/ceph.client.admin.keyring 2023-09-23T20:44:06.004 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:44:05 smithi026 ceph-mon[95839]: Updating smithi070:/etc/ceph/ceph.client.admin.keyring 2023-09-23T20:44:06.004 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:44:05 smithi026 ceph-mon[95839]: Updating smithi026:/var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/config/ceph.client.admin.keyring 2023-09-23T20:44:06.004 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:44:05 smithi026 ceph-mon[95839]: Updating smithi070:/var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/config/ceph.client.admin.keyring 2023-09-23T20:44:06.005 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:44:05 smithi026 ceph-mon[95839]: pgmap v10: 73 pgs: 73 active+clean; 580 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:44:06.005 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:44:05 smithi026 ceph-mon[95839]: from='mgr.24781 ' entity='mgr.x' 2023-09-23T20:44:06.005 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:44:05 smithi026 ceph-mon[95839]: from='mgr.24781 ' entity='mgr.x' 2023-09-23T20:44:06.005 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:44:05 smithi026 ceph-mon[95839]: from='mgr.24781 ' entity='mgr.x' 2023-09-23T20:44:06.005 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:44:05 smithi026 ceph-mon[95839]: from='mgr.24781 ' entity='mgr.x' 2023-09-23T20:44:06.005 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:44:05 smithi026 ceph-mon[95839]: from='client.? 172.21.15.26:0/2286305968' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:44:06.006 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:44:05 smithi026 ceph-mon[109912]: Updating smithi026:/etc/ceph/ceph.client.admin.keyring 2023-09-23T20:44:06.006 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:44:05 smithi026 ceph-mon[109912]: Updating smithi070:/etc/ceph/ceph.client.admin.keyring 2023-09-23T20:44:06.006 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:44:05 smithi026 ceph-mon[109912]: Updating smithi026:/var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/config/ceph.client.admin.keyring 2023-09-23T20:44:06.006 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:44:05 smithi026 ceph-mon[109912]: Updating smithi070:/var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/config/ceph.client.admin.keyring 2023-09-23T20:44:06.007 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:44:05 smithi026 ceph-mon[109912]: pgmap v10: 73 pgs: 73 active+clean; 580 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:44:06.007 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:44:05 smithi026 ceph-mon[109912]: from='mgr.24781 ' entity='mgr.x' 2023-09-23T20:44:06.007 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:44:05 smithi026 ceph-mon[109912]: from='mgr.24781 ' entity='mgr.x' 2023-09-23T20:44:06.007 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:44:05 smithi026 ceph-mon[109912]: from='mgr.24781 ' entity='mgr.x' 2023-09-23T20:44:06.007 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:44:05 smithi026 ceph-mon[109912]: from='mgr.24781 ' entity='mgr.x' 2023-09-23T20:44:06.007 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:44:05 smithi026 ceph-mon[109912]: from='client.? 172.21.15.26:0/2286305968' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:44:06.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:44:05 smithi070 ceph-mon[100755]: Updating smithi026:/etc/ceph/ceph.client.admin.keyring 2023-09-23T20:44:06.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:44:05 smithi070 ceph-mon[100755]: Updating smithi070:/etc/ceph/ceph.client.admin.keyring 2023-09-23T20:44:06.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:44:05 smithi070 ceph-mon[100755]: Updating smithi026:/var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/config/ceph.client.admin.keyring 2023-09-23T20:44:06.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:44:05 smithi070 ceph-mon[100755]: Updating smithi070:/var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/config/ceph.client.admin.keyring 2023-09-23T20:44:06.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:44:05 smithi070 ceph-mon[100755]: pgmap v10: 73 pgs: 73 active+clean; 580 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:44:06.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:44:05 smithi070 ceph-mon[100755]: from='mgr.24781 ' entity='mgr.x' 2023-09-23T20:44:06.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:44:05 smithi070 ceph-mon[100755]: from='mgr.24781 ' entity='mgr.x' 2023-09-23T20:44:06.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:44:05 smithi070 ceph-mon[100755]: from='mgr.24781 ' entity='mgr.x' 2023-09-23T20:44:06.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:44:05 smithi070 ceph-mon[100755]: from='mgr.24781 ' entity='mgr.x' 2023-09-23T20:44:06.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:44:05 smithi070 ceph-mon[100755]: from='client.? 172.21.15.26:0/2286305968' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:44:06.961 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:44:06 smithi026 ceph-mon[95839]: Reconfiguring alertmanager.a (dependencies changed)... 2023-09-23T20:44:06.962 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:44:06 smithi026 ceph-mon[95839]: Reconfiguring daemon alertmanager.a on smithi026 2023-09-23T20:44:06.962 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:44:06 smithi026 ceph-mon[95839]: from='mgr.24781 172.21.15.70:0/208042876' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:44:06.962 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:44:06 smithi026 ceph-mon[95839]: pgmap v11: 73 pgs: 73 active+clean; 580 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:44:06.962 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:44:06 smithi026 ceph-mon[109912]: Reconfiguring alertmanager.a (dependencies changed)... 2023-09-23T20:44:06.962 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:44:06 smithi026 ceph-mon[109912]: Reconfiguring daemon alertmanager.a on smithi026 2023-09-23T20:44:06.962 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:44:06 smithi026 ceph-mon[109912]: from='mgr.24781 172.21.15.70:0/208042876' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:44:06.962 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:44:06 smithi026 ceph-mon[109912]: pgmap v11: 73 pgs: 73 active+clean; 580 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:44:07.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:44:06 smithi070 ceph-mon[100755]: Reconfiguring alertmanager.a (dependencies changed)... 2023-09-23T20:44:07.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:44:06 smithi070 ceph-mon[100755]: Reconfiguring daemon alertmanager.a on smithi026 2023-09-23T20:44:07.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:44:06 smithi070 ceph-mon[100755]: from='mgr.24781 172.21.15.70:0/208042876' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:44:07.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:44:06 smithi070 ceph-mon[100755]: pgmap v11: 73 pgs: 73 active+clean; 580 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:44:07.266 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Sep 23 20:44:06 smithi026 systemd[1]: Stopping Ceph alertmanager.a for b3f8e4ec-5a50-11ee-8db4-212e2dc638e7... 2023-09-23T20:44:07.626 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Sep 23 20:44:07 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-alertmanager-a[149923]: ts=2023-09-23T20:44:07.266Z caller=main.go:583 level=info msg="Received SIGTERM, exiting gracefully..." 2023-09-23T20:44:07.882 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Sep 23 20:44:07 smithi026 bash[162876]: ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-alertmanager-a 2023-09-23T20:44:07.882 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Sep 23 20:44:07 smithi026 systemd[1]: ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7@alertmanager.a.service: Succeeded. 2023-09-23T20:44:07.882 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Sep 23 20:44:07 smithi026 systemd[1]: Stopped Ceph alertmanager.a for b3f8e4ec-5a50-11ee-8db4-212e2dc638e7. 2023-09-23T20:44:07.882 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Sep 23 20:44:07 smithi026 systemd[1]: Starting Ceph alertmanager.a for b3f8e4ec-5a50-11ee-8db4-212e2dc638e7... 2023-09-23T20:44:08.159 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:44:07 smithi026 ceph-mon[95839]: from='client.? 172.21.15.26:0/304887566' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:44:08.159 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:44:07 smithi026 ceph-mon[109912]: from='client.? 172.21.15.26:0/304887566' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:44:08.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:44:07 smithi070 ceph-mon[100755]: from='client.? 172.21.15.26:0/304887566' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:44:08.409 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Sep 23 20:44:08 smithi026 podman[163020]: 2023-09-23T20:44:08.409 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Sep 23 20:44:08 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-alertmanager-a[163035]: ts=2023-09-23T20:44:08.394Z caller=main.go:240 level=info msg="Starting Alertmanager" version="(version=0.25.0, branch=HEAD, revision=258fab7cdd551f2cf251ed0348f0ad7289aee789)" 2023-09-23T20:44:08.409 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Sep 23 20:44:08 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-alertmanager-a[163035]: ts=2023-09-23T20:44:08.394Z caller=main.go:241 level=info build_context="(go=go1.19.4, user=root@abe866dd5717, date=20221222-14:51:36)" 2023-09-23T20:44:08.409 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Sep 23 20:44:08 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-alertmanager-a[163035]: ts=2023-09-23T20:44:08.396Z caller=cluster.go:185 level=info component=cluster msg="setting advertise address explicitly" addr=172.21.15.26 port=9094 2023-09-23T20:44:08.409 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Sep 23 20:44:08 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-alertmanager-a[163035]: ts=2023-09-23T20:44:08.397Z caller=cluster.go:681 level=info component=cluster msg="Waiting for gossip to settle..." interval=2s 2023-09-23T20:44:08.863 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:44:08 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[96086]: ::ffff:172.21.15.70 - - [23/Sep/2023:20:44:08] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2023-09-23T20:44:08.864 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Sep 23 20:44:08 smithi026 bash[163020]: d4b48de0f10c484c45571d53aaa5073ccf5e4b435cc592e10bf12f45db83b663 2023-09-23T20:44:08.864 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Sep 23 20:44:08 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-alertmanager-a[163035]: ts=2023-09-23T20:44:08.425Z caller=coordinator.go:113 level=info component=configuration msg="Loading configuration file" file=/etc/alertmanager/alertmanager.yml 2023-09-23T20:44:08.864 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Sep 23 20:44:08 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-alertmanager-a[163035]: ts=2023-09-23T20:44:08.425Z caller=coordinator.go:126 level=info component=configuration msg="Completed loading of configuration file" file=/etc/alertmanager/alertmanager.yml 2023-09-23T20:44:08.864 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Sep 23 20:44:08 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-alertmanager-a[163035]: ts=2023-09-23T20:44:08.428Z caller=tls_config.go:232 level=info msg="Listening on" address=[::]:9093 2023-09-23T20:44:08.864 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Sep 23 20:44:08 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-alertmanager-a[163035]: ts=2023-09-23T20:44:08.428Z caller=tls_config.go:235 level=info msg="TLS is disabled." http2=false address=[::]:9093 2023-09-23T20:44:08.864 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Sep 23 20:44:08 smithi026 systemd[1]: Started Ceph alertmanager.a for b3f8e4ec-5a50-11ee-8db4-212e2dc638e7. 2023-09-23T20:44:09.132 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 23 20:44:08 smithi026 systemd[1]: Stopping Ceph node-exporter.a for b3f8e4ec-5a50-11ee-8db4-212e2dc638e7... 2023-09-23T20:44:09.514 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:44:09 smithi026 ceph-mon[95839]: from='mgr.24781 ' entity='mgr.x' 2023-09-23T20:44:09.514 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:44:09 smithi026 ceph-mon[95839]: from='mgr.24781 ' entity='mgr.x' 2023-09-23T20:44:09.514 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:44:09 smithi026 ceph-mon[95839]: Reconfiguring node-exporter.a (dependencies changed)... 2023-09-23T20:44:09.514 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:44:09 smithi026 ceph-mon[95839]: Reconfiguring daemon node-exporter.a on smithi026 2023-09-23T20:44:09.514 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:44:09 smithi026 ceph-mon[95839]: pgmap v12: 73 pgs: 73 active+clean; 580 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:44:09.515 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 23 20:44:09 smithi026 bash[163193]: ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-node-exporter-a 2023-09-23T20:44:09.515 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:44:09 smithi026 ceph-mon[109912]: from='mgr.24781 ' entity='mgr.x' 2023-09-23T20:44:09.515 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:44:09 smithi026 ceph-mon[109912]: from='mgr.24781 ' entity='mgr.x' 2023-09-23T20:44:09.515 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:44:09 smithi026 ceph-mon[109912]: Reconfiguring node-exporter.a (dependencies changed)... 2023-09-23T20:44:09.515 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:44:09 smithi026 ceph-mon[109912]: Reconfiguring daemon node-exporter.a on smithi026 2023-09-23T20:44:09.515 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:44:09 smithi026 ceph-mon[109912]: pgmap v12: 73 pgs: 73 active+clean; 580 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:44:09.618 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 23 20:44:09 smithi026 systemd[1]: ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7@node-exporter.a.service: Main process exited, code=exited, status=143/n/a 2023-09-23T20:44:09.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:44:09 smithi070 ceph-mon[100755]: from='mgr.24781 ' entity='mgr.x' 2023-09-23T20:44:09.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:44:09 smithi070 ceph-mon[100755]: from='mgr.24781 ' entity='mgr.x' 2023-09-23T20:44:09.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:44:09 smithi070 ceph-mon[100755]: Reconfiguring node-exporter.a (dependencies changed)... 2023-09-23T20:44:09.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:44:09 smithi070 ceph-mon[100755]: Reconfiguring daemon node-exporter.a on smithi026 2023-09-23T20:44:09.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:44:09 smithi070 ceph-mon[100755]: pgmap v12: 73 pgs: 73 active+clean; 580 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:44:09.881 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 23 20:44:09 smithi026 systemd[1]: ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7@node-exporter.a.service: Failed with result 'exit-code'. 2023-09-23T20:44:09.882 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 23 20:44:09 smithi026 systemd[1]: Stopped Ceph node-exporter.a for b3f8e4ec-5a50-11ee-8db4-212e2dc638e7. 2023-09-23T20:44:09.882 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 23 20:44:09 smithi026 systemd[1]: Starting Ceph node-exporter.a for b3f8e4ec-5a50-11ee-8db4-212e2dc638e7... 2023-09-23T20:44:10.172 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 23 20:44:09 smithi026 podman[163317]: 2023-09-23T20:44:10.632 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Sep 23 20:44:10 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-alertmanager-a[163035]: ts=2023-09-23T20:44:10.397Z caller=cluster.go:706 level=info component=cluster msg="gossip not settled" polls=0 before=0 now=1 elapsed=2.000134229s 2023-09-23T20:44:10.632 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 23 20:44:10 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-node-exporter-a[163355]: ts=2023-09-23T20:44:10.172Z caller=node_exporter.go:180 level=info msg="Starting node_exporter" version="(version=1.5.0, branch=HEAD, revision=1b48970ffcf5630534fb00bb0687d73c66d1c959)" 2023-09-23T20:44:10.632 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 23 20:44:10 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-node-exporter-a[163355]: ts=2023-09-23T20:44:10.172Z caller=node_exporter.go:181 level=info msg="Build context" build_context="(go=go1.19.3, user=root@6e7732a7b81b, date=20221129-18:59:09)" 2023-09-23T20:44:10.632 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 23 20:44:10 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-node-exporter-a[163355]: ts=2023-09-23T20:44:10.172Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+|var/lib/containers/storage/.+)($|/) 2023-09-23T20:44:10.633 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 23 20:44:10 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-node-exporter-a[163355]: ts=2023-09-23T20:44:10.172Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2023-09-23T20:44:10.633 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 23 20:44:10 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-node-exporter-a[163355]: ts=2023-09-23T20:44:10.173Z caller=diskstats_common.go:111 level=info collector=diskstats msg="Parsed flag --collector.diskstats.device-exclude" flag=^(ram|loop|fd|(h|s|v|xv)d[a-z]|nvme\d+n\d+p)\d+$ 2023-09-23T20:44:10.633 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 23 20:44:10 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-node-exporter-a[163355]: ts=2023-09-23T20:44:10.173Z caller=diskstats_linux.go:264 level=error collector=diskstats msg="Failed to open directory, disabling udev device properties" path=/run/udev/data 2023-09-23T20:44:10.633 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 23 20:44:10 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-node-exporter-a[163355]: ts=2023-09-23T20:44:10.174Z caller=node_exporter.go:110 level=info msg="Enabled collectors" 2023-09-23T20:44:10.633 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 23 20:44:10 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-node-exporter-a[163355]: ts=2023-09-23T20:44:10.174Z caller=node_exporter.go:117 level=info collector=arp 2023-09-23T20:44:10.633 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 23 20:44:10 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-node-exporter-a[163355]: ts=2023-09-23T20:44:10.174Z caller=node_exporter.go:117 level=info collector=bcache 2023-09-23T20:44:10.633 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 23 20:44:10 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-node-exporter-a[163355]: ts=2023-09-23T20:44:10.174Z caller=node_exporter.go:117 level=info collector=bonding 2023-09-23T20:44:10.633 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 23 20:44:10 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-node-exporter-a[163355]: ts=2023-09-23T20:44:10.174Z caller=node_exporter.go:117 level=info collector=btrfs 2023-09-23T20:44:10.633 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 23 20:44:10 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-node-exporter-a[163355]: ts=2023-09-23T20:44:10.174Z caller=node_exporter.go:117 level=info collector=conntrack 2023-09-23T20:44:10.633 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 23 20:44:10 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-node-exporter-a[163355]: ts=2023-09-23T20:44:10.174Z caller=node_exporter.go:117 level=info collector=cpu 2023-09-23T20:44:10.633 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 23 20:44:10 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-node-exporter-a[163355]: ts=2023-09-23T20:44:10.174Z caller=node_exporter.go:117 level=info collector=cpufreq 2023-09-23T20:44:10.634 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 23 20:44:10 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-node-exporter-a[163355]: ts=2023-09-23T20:44:10.174Z caller=node_exporter.go:117 level=info collector=diskstats 2023-09-23T20:44:10.634 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 23 20:44:10 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-node-exporter-a[163355]: ts=2023-09-23T20:44:10.174Z caller=node_exporter.go:117 level=info collector=dmi 2023-09-23T20:44:10.634 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 23 20:44:10 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-node-exporter-a[163355]: ts=2023-09-23T20:44:10.174Z caller=node_exporter.go:117 level=info collector=edac 2023-09-23T20:44:10.634 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 23 20:44:10 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-node-exporter-a[163355]: ts=2023-09-23T20:44:10.174Z caller=node_exporter.go:117 level=info collector=entropy 2023-09-23T20:44:10.634 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 23 20:44:10 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-node-exporter-a[163355]: ts=2023-09-23T20:44:10.174Z caller=node_exporter.go:117 level=info collector=fibrechannel 2023-09-23T20:44:10.634 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 23 20:44:10 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-node-exporter-a[163355]: ts=2023-09-23T20:44:10.174Z caller=node_exporter.go:117 level=info collector=filefd 2023-09-23T20:44:10.634 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 23 20:44:10 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-node-exporter-a[163355]: ts=2023-09-23T20:44:10.174Z caller=node_exporter.go:117 level=info collector=filesystem 2023-09-23T20:44:10.634 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 23 20:44:10 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-node-exporter-a[163355]: ts=2023-09-23T20:44:10.174Z caller=node_exporter.go:117 level=info collector=hwmon 2023-09-23T20:44:10.634 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 23 20:44:10 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-node-exporter-a[163355]: ts=2023-09-23T20:44:10.174Z caller=node_exporter.go:117 level=info collector=infiniband 2023-09-23T20:44:10.634 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 23 20:44:10 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-node-exporter-a[163355]: ts=2023-09-23T20:44:10.174Z caller=node_exporter.go:117 level=info collector=ipvs 2023-09-23T20:44:10.634 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 23 20:44:10 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-node-exporter-a[163355]: ts=2023-09-23T20:44:10.174Z caller=node_exporter.go:117 level=info collector=loadavg 2023-09-23T20:44:10.634 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 23 20:44:10 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-node-exporter-a[163355]: ts=2023-09-23T20:44:10.174Z caller=node_exporter.go:117 level=info collector=mdadm 2023-09-23T20:44:10.635 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 23 20:44:10 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-node-exporter-a[163355]: ts=2023-09-23T20:44:10.174Z caller=node_exporter.go:117 level=info collector=meminfo 2023-09-23T20:44:10.635 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 23 20:44:10 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-node-exporter-a[163355]: ts=2023-09-23T20:44:10.174Z caller=node_exporter.go:117 level=info collector=netclass 2023-09-23T20:44:10.635 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 23 20:44:10 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-node-exporter-a[163355]: ts=2023-09-23T20:44:10.174Z caller=node_exporter.go:117 level=info collector=netdev 2023-09-23T20:44:10.635 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 23 20:44:10 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-node-exporter-a[163355]: ts=2023-09-23T20:44:10.174Z caller=node_exporter.go:117 level=info collector=netstat 2023-09-23T20:44:10.635 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 23 20:44:10 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-node-exporter-a[163355]: ts=2023-09-23T20:44:10.174Z caller=node_exporter.go:117 level=info collector=nfs 2023-09-23T20:44:10.635 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 23 20:44:10 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-node-exporter-a[163355]: ts=2023-09-23T20:44:10.174Z caller=node_exporter.go:117 level=info collector=nfsd 2023-09-23T20:44:10.635 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 23 20:44:10 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-node-exporter-a[163355]: ts=2023-09-23T20:44:10.174Z caller=node_exporter.go:117 level=info collector=nvme 2023-09-23T20:44:10.635 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 23 20:44:10 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-node-exporter-a[163355]: ts=2023-09-23T20:44:10.174Z caller=node_exporter.go:117 level=info collector=os 2023-09-23T20:44:10.635 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 23 20:44:10 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-node-exporter-a[163355]: ts=2023-09-23T20:44:10.174Z caller=node_exporter.go:117 level=info collector=powersupplyclass 2023-09-23T20:44:10.635 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 23 20:44:10 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-node-exporter-a[163355]: ts=2023-09-23T20:44:10.174Z caller=node_exporter.go:117 level=info collector=pressure 2023-09-23T20:44:10.635 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 23 20:44:10 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-node-exporter-a[163355]: ts=2023-09-23T20:44:10.174Z caller=node_exporter.go:117 level=info collector=rapl 2023-09-23T20:44:10.635 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 23 20:44:10 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-node-exporter-a[163355]: ts=2023-09-23T20:44:10.174Z caller=node_exporter.go:117 level=info collector=schedstat 2023-09-23T20:44:10.635 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 23 20:44:10 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-node-exporter-a[163355]: ts=2023-09-23T20:44:10.174Z caller=node_exporter.go:117 level=info collector=selinux 2023-09-23T20:44:10.636 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 23 20:44:10 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-node-exporter-a[163355]: ts=2023-09-23T20:44:10.174Z caller=node_exporter.go:117 level=info collector=sockstat 2023-09-23T20:44:10.636 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 23 20:44:10 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-node-exporter-a[163355]: ts=2023-09-23T20:44:10.174Z caller=node_exporter.go:117 level=info collector=softnet 2023-09-23T20:44:10.636 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 23 20:44:10 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-node-exporter-a[163355]: ts=2023-09-23T20:44:10.174Z caller=node_exporter.go:117 level=info collector=stat 2023-09-23T20:44:10.636 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 23 20:44:10 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-node-exporter-a[163355]: ts=2023-09-23T20:44:10.174Z caller=node_exporter.go:117 level=info collector=tapestats 2023-09-23T20:44:10.636 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 23 20:44:10 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-node-exporter-a[163355]: ts=2023-09-23T20:44:10.174Z caller=node_exporter.go:117 level=info collector=textfile 2023-09-23T20:44:10.636 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 23 20:44:10 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-node-exporter-a[163355]: ts=2023-09-23T20:44:10.174Z caller=node_exporter.go:117 level=info collector=thermal_zone 2023-09-23T20:44:10.636 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 23 20:44:10 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-node-exporter-a[163355]: ts=2023-09-23T20:44:10.174Z caller=node_exporter.go:117 level=info collector=time 2023-09-23T20:44:10.636 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 23 20:44:10 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-node-exporter-a[163355]: ts=2023-09-23T20:44:10.174Z caller=node_exporter.go:117 level=info collector=udp_queues 2023-09-23T20:44:10.636 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 23 20:44:10 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-node-exporter-a[163355]: ts=2023-09-23T20:44:10.174Z caller=node_exporter.go:117 level=info collector=uname 2023-09-23T20:44:10.636 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 23 20:44:10 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-node-exporter-a[163355]: ts=2023-09-23T20:44:10.174Z caller=node_exporter.go:117 level=info collector=vmstat 2023-09-23T20:44:10.636 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 23 20:44:10 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-node-exporter-a[163355]: ts=2023-09-23T20:44:10.174Z caller=node_exporter.go:117 level=info collector=xfs 2023-09-23T20:44:10.636 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 23 20:44:10 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-node-exporter-a[163355]: ts=2023-09-23T20:44:10.174Z caller=node_exporter.go:117 level=info collector=zfs 2023-09-23T20:44:10.637 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 23 20:44:10 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-node-exporter-a[163355]: ts=2023-09-23T20:44:10.174Z caller=tls_config.go:232 level=info msg="Listening on" address=[::]:9100 2023-09-23T20:44:10.637 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 23 20:44:10 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-node-exporter-a[163355]: ts=2023-09-23T20:44:10.174Z caller=tls_config.go:235 level=info msg="TLS is disabled." http2=false address=[::]:9100 2023-09-23T20:44:10.637 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 23 20:44:10 smithi026 bash[163317]: 95269f65f9af99750e6333509f491c1cef8b1a2b7f49c07be12793f754e51c44 2023-09-23T20:44:10.637 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 23 20:44:10 smithi026 systemd[1]: Started Ceph node-exporter.a for b3f8e4ec-5a50-11ee-8db4-212e2dc638e7. 2023-09-23T20:44:11.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:44:11 smithi070 ceph-mon[100755]: from='mgr.24781 ' entity='mgr.x' 2023-09-23T20:44:11.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:44:11 smithi070 ceph-mon[100755]: from='mgr.24781 ' entity='mgr.x' 2023-09-23T20:44:11.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:44:11 smithi070 ceph-mon[100755]: Reconfiguring prometheus.a (dependencies changed)... 2023-09-23T20:44:11.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:44:11 smithi070 ceph-mon[100755]: from='client.? 172.21.15.26:0/3467603169' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:44:11.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:44:11 smithi070 ceph-mon[100755]: Reconfiguring daemon prometheus.a on smithi070 2023-09-23T20:44:11.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:44:11 smithi070 ceph-mon[100755]: pgmap v13: 73 pgs: 73 active+clean; 580 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:44:11.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:44:11 smithi026 ceph-mon[95839]: from='mgr.24781 ' entity='mgr.x' 2023-09-23T20:44:11.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:44:11 smithi026 ceph-mon[95839]: from='mgr.24781 ' entity='mgr.x' 2023-09-23T20:44:11.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:44:11 smithi026 ceph-mon[95839]: Reconfiguring prometheus.a (dependencies changed)... 2023-09-23T20:44:11.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:44:11 smithi026 ceph-mon[95839]: from='client.? 172.21.15.26:0/3467603169' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:44:11.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:44:11 smithi026 ceph-mon[95839]: Reconfiguring daemon prometheus.a on smithi070 2023-09-23T20:44:11.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:44:11 smithi026 ceph-mon[95839]: pgmap v13: 73 pgs: 73 active+clean; 580 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:44:11.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:44:11 smithi026 ceph-mon[109912]: from='mgr.24781 ' entity='mgr.x' 2023-09-23T20:44:11.633 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:44:11 smithi026 ceph-mon[109912]: from='mgr.24781 ' entity='mgr.x' 2023-09-23T20:44:11.633 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:44:11 smithi026 ceph-mon[109912]: Reconfiguring prometheus.a (dependencies changed)... 2023-09-23T20:44:11.633 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:44:11 smithi026 ceph-mon[109912]: from='client.? 172.21.15.26:0/3467603169' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:44:11.633 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:44:11 smithi026 ceph-mon[109912]: Reconfiguring daemon prometheus.a on smithi070 2023-09-23T20:44:11.633 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:44:11 smithi026 ceph-mon[109912]: pgmap v13: 73 pgs: 73 active+clean; 580 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:44:11.869 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 20:44:11 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-x[141470]: ::ffff:172.21.15.70 - - [23/Sep/2023:20:44:11] "GET /metrics HTTP/1.1" 200 33704 "" "Prometheus/2.43.0" 2023-09-23T20:44:12.830 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:44:12 smithi070 ceph-mon[100755]: from='client.? 172.21.15.26:0/2157105056' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:44:12.831 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 20:44:12 smithi070 systemd[1]: Stopping Ceph prometheus.a for b3f8e4ec-5a50-11ee-8db4-212e2dc638e7... 2023-09-23T20:44:13.120 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 20:44:12 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-prometheus-a[138511]: ts=2023-09-23T20:44:12.830Z caller=main.go:840 level=warn msg="Received SIGTERM, exiting gracefully..." 2023-09-23T20:44:13.120 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 20:44:12 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-prometheus-a[138511]: ts=2023-09-23T20:44:12.830Z caller=main.go:864 level=info msg="Stopping scrape discovery manager..." 2023-09-23T20:44:13.120 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 20:44:12 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-prometheus-a[138511]: ts=2023-09-23T20:44:12.830Z caller=main.go:878 level=info msg="Stopping notify discovery manager..." 2023-09-23T20:44:13.120 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 20:44:12 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-prometheus-a[138511]: ts=2023-09-23T20:44:12.830Z caller=manager.go:988 level=info component="rule manager" msg="Stopping rule manager..." 2023-09-23T20:44:13.120 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 20:44:12 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-prometheus-a[138511]: ts=2023-09-23T20:44:12.830Z caller=main.go:860 level=info msg="Scrape discovery manager stopped" 2023-09-23T20:44:13.120 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 20:44:12 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-prometheus-a[138511]: ts=2023-09-23T20:44:12.830Z caller=manager.go:998 level=info component="rule manager" msg="Rule manager stopped" 2023-09-23T20:44:13.120 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 20:44:12 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-prometheus-a[138511]: ts=2023-09-23T20:44:12.830Z caller=main.go:874 level=info msg="Notify discovery manager stopped" 2023-09-23T20:44:13.120 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 20:44:12 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-prometheus-a[138511]: ts=2023-09-23T20:44:12.830Z caller=main.go:915 level=info msg="Stopping scrape manager..." 2023-09-23T20:44:13.120 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 20:44:12 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-prometheus-a[138511]: ts=2023-09-23T20:44:12.831Z caller=main.go:907 level=info msg="Scrape manager stopped" 2023-09-23T20:44:13.120 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 20:44:12 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-prometheus-a[138511]: ts=2023-09-23T20:44:12.831Z caller=notifier.go:606 level=info component=notifier msg="Stopping notification manager..." 2023-09-23T20:44:13.120 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 20:44:12 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-prometheus-a[138511]: ts=2023-09-23T20:44:12.831Z caller=main.go:1135 level=info msg="Notifier manager stopped" 2023-09-23T20:44:13.121 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 20:44:12 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-prometheus-a[138511]: ts=2023-09-23T20:44:12.831Z caller=main.go:1147 level=info msg="See you next time!" 2023-09-23T20:44:13.121 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 20:44:13 smithi070 bash[152281]: ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-prometheus-a 2023-09-23T20:44:13.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:44:12 smithi026 ceph-mon[95839]: from='client.? 172.21.15.26:0/2157105056' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:44:13.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:44:12 smithi026 ceph-mon[109912]: from='client.? 172.21.15.26:0/2157105056' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:44:13.535 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 20:44:13 smithi070 systemd[1]: ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7@prometheus.a.service: Succeeded. 2023-09-23T20:44:13.535 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 20:44:13 smithi070 systemd[1]: Stopped Ceph prometheus.a for b3f8e4ec-5a50-11ee-8db4-212e2dc638e7. 2023-09-23T20:44:13.535 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 20:44:13 smithi070 systemd[1]: Starting Ceph prometheus.a for b3f8e4ec-5a50-11ee-8db4-212e2dc638e7... 2023-09-23T20:44:13.793 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:44:13 smithi070 ceph-mon[100755]: pgmap v14: 73 pgs: 73 active+clean; 580 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:44:13.793 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 20:44:13 smithi070 podman[152399]: 2023-09-23T20:44:13.793 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 20:44:13 smithi070 bash[152399]: de9b1d1498a0179623cac4aead91670e7f17c965b76d89dd3d5aef3f457ae092 2023-09-23T20:44:13.793 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 20:44:13 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-prometheus-a[152415]: ts=2023-09-23T20:44:13.771Z caller=main.go:564 level=info msg="Starting Prometheus Server" mode=server version="(version=2.43.0, branch=HEAD, revision=edfc3bcd025dd6fe296c167a14a216cab1e552ee)" 2023-09-23T20:44:13.793 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 20:44:13 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-prometheus-a[152415]: ts=2023-09-23T20:44:13.771Z caller=main.go:569 level=info build_context="(go=go1.19.7, platform=linux/amd64, user=root@8a0ee342e522, date=20230321-12:56:07, tags=netgo,builtinassets)" 2023-09-23T20:44:13.794 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 20:44:13 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-prometheus-a[152415]: ts=2023-09-23T20:44:13.771Z caller=main.go:570 level=info host_details="(Linux 4.18.0-372.9.1.el8.x86_64 #1 SMP Fri Apr 15 22:12:19 EDT 2022 x86_64 smithi070 (none))" 2023-09-23T20:44:13.794 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 20:44:13 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-prometheus-a[152415]: ts=2023-09-23T20:44:13.771Z caller=main.go:571 level=info fd_limits="(soft=1048576, hard=1048576)" 2023-09-23T20:44:13.794 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 20:44:13 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-prometheus-a[152415]: ts=2023-09-23T20:44:13.771Z caller=main.go:572 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2023-09-23T20:44:13.794 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 20:44:13 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-prometheus-a[152415]: ts=2023-09-23T20:44:13.773Z caller=web.go:561 level=info component=web msg="Start listening for connections" address=:9095 2023-09-23T20:44:13.794 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 20:44:13 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-prometheus-a[152415]: ts=2023-09-23T20:44:13.773Z caller=main.go:1005 level=info msg="Starting TSDB ..." 2023-09-23T20:44:13.794 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 20:44:13 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-prometheus-a[152415]: ts=2023-09-23T20:44:13.775Z caller=tls_config.go:232 level=info component=web msg="Listening on" address=[::]:9095 2023-09-23T20:44:13.794 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 20:44:13 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-prometheus-a[152415]: ts=2023-09-23T20:44:13.775Z caller=tls_config.go:235 level=info component=web msg="TLS is disabled." http2=false address=[::]:9095 2023-09-23T20:44:13.794 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 20:44:13 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-prometheus-a[152415]: ts=2023-09-23T20:44:13.776Z caller=head.go:587 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2023-09-23T20:44:13.794 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 20:44:13 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-prometheus-a[152415]: ts=2023-09-23T20:44:13.776Z caller=head.go:658 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=1.64µs 2023-09-23T20:44:13.794 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 20:44:13 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-prometheus-a[152415]: ts=2023-09-23T20:44:13.776Z caller=head.go:664 level=info component=tsdb msg="Replaying WAL, this may take a while" 2023-09-23T20:44:13.795 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 20:44:13 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-prometheus-a[152415]: ts=2023-09-23T20:44:13.777Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=2 2023-09-23T20:44:13.795 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 20:44:13 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-prometheus-a[152415]: ts=2023-09-23T20:44:13.793Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=1 maxSegment=2 2023-09-23T20:44:14.119 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 20:44:13 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-prometheus-a[152415]: ts=2023-09-23T20:44:13.793Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=2 maxSegment=2 2023-09-23T20:44:14.119 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 20:44:13 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-prometheus-a[152415]: ts=2023-09-23T20:44:13.793Z caller=head.go:772 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=19.525µs wal_replay_duration=16.929867ms wbl_replay_duration=111ns total_replay_duration=16.967858ms 2023-09-23T20:44:14.120 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 20:44:13 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-prometheus-a[152415]: ts=2023-09-23T20:44:13.796Z caller=main.go:1026 level=info fs_type=XFS_SUPER_MAGIC 2023-09-23T20:44:14.120 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 20:44:13 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-prometheus-a[152415]: ts=2023-09-23T20:44:13.796Z caller=main.go:1029 level=info msg="TSDB started" 2023-09-23T20:44:14.120 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 20:44:13 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-prometheus-a[152415]: ts=2023-09-23T20:44:13.796Z caller=main.go:1209 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2023-09-23T20:44:14.120 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 20:44:13 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-prometheus-a[152415]: ts=2023-09-23T20:44:13.810Z caller=main.go:1246 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=13.956529ms db_storage=833ns remote_storage=1.15µs web_handler=385ns query_engine=591ns scrape=235.501µs scrape_sd=89.122µs notify=10.931µs notify_sd=14.938µs rules=13.398956ms tracing=5.041µs 2023-09-23T20:44:14.120 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 20:44:13 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-prometheus-a[152415]: ts=2023-09-23T20:44:13.810Z caller=main.go:990 level=info msg="Server is ready to receive web requests." 2023-09-23T20:44:14.120 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 20:44:13 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-prometheus-a[152415]: ts=2023-09-23T20:44:13.810Z caller=manager.go:974 level=info component="rule manager" msg="Starting rule manager..." 2023-09-23T20:44:14.120 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 20:44:13 smithi070 systemd[1]: Started Ceph prometheus.a for b3f8e4ec-5a50-11ee-8db4-212e2dc638e7. 2023-09-23T20:44:14.131 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:44:13 smithi026 ceph-mon[95839]: pgmap v14: 73 pgs: 73 active+clean; 580 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:44:14.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:44:13 smithi026 ceph-mon[109912]: pgmap v14: 73 pgs: 73 active+clean; 580 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:44:14.616 INFO:journalctl@ceph.node-exporter.b.smithi070.stdout:Sep 23 20:44:14 smithi070 systemd[1]: Stopping Ceph node-exporter.b for b3f8e4ec-5a50-11ee-8db4-212e2dc638e7... 2023-09-23T20:44:14.869 INFO:journalctl@ceph.node-exporter.b.smithi070.stdout:Sep 23 20:44:14 smithi070 bash[152574]: ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-node-exporter-b 2023-09-23T20:44:14.869 INFO:journalctl@ceph.node-exporter.b.smithi070.stdout:Sep 23 20:44:14 smithi070 systemd[1]: ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7@node-exporter.b.service: Main process exited, code=exited, status=143/n/a 2023-09-23T20:44:14.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:44:14 smithi070 ceph-mon[100755]: from='mgr.24781 ' entity='mgr.x' 2023-09-23T20:44:14.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:44:14 smithi070 ceph-mon[100755]: from='mgr.24781 ' entity='mgr.x' 2023-09-23T20:44:14.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:44:14 smithi070 ceph-mon[100755]: Reconfiguring node-exporter.b (dependencies changed)... 2023-09-23T20:44:14.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:44:14 smithi070 ceph-mon[100755]: Reconfiguring daemon node-exporter.b on smithi070 2023-09-23T20:44:14.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:44:14 smithi070 ceph-mon[100755]: pgmap v15: 73 pgs: 73 active+clean; 580 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:44:15.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:44:14 smithi026 ceph-mon[109912]: from='mgr.24781 ' entity='mgr.x' 2023-09-23T20:44:15.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:44:14 smithi026 ceph-mon[109912]: from='mgr.24781 ' entity='mgr.x' 2023-09-23T20:44:15.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:44:14 smithi026 ceph-mon[109912]: Reconfiguring node-exporter.b (dependencies changed)... 2023-09-23T20:44:15.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:44:14 smithi026 ceph-mon[109912]: Reconfiguring daemon node-exporter.b on smithi070 2023-09-23T20:44:15.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:44:14 smithi026 ceph-mon[109912]: pgmap v15: 73 pgs: 73 active+clean; 580 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:44:15.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:44:14 smithi026 ceph-mon[95839]: from='mgr.24781 ' entity='mgr.x' 2023-09-23T20:44:15.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:44:14 smithi026 ceph-mon[95839]: from='mgr.24781 ' entity='mgr.x' 2023-09-23T20:44:15.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:44:14 smithi026 ceph-mon[95839]: Reconfiguring node-exporter.b (dependencies changed)... 2023-09-23T20:44:15.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:44:14 smithi026 ceph-mon[95839]: Reconfiguring daemon node-exporter.b on smithi070 2023-09-23T20:44:15.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:44:14 smithi026 ceph-mon[95839]: pgmap v15: 73 pgs: 73 active+clean; 580 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:44:15.248 INFO:journalctl@ceph.node-exporter.b.smithi070.stdout:Sep 23 20:44:14 smithi070 systemd[1]: ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7@node-exporter.b.service: Failed with result 'exit-code'. 2023-09-23T20:44:15.248 INFO:journalctl@ceph.node-exporter.b.smithi070.stdout:Sep 23 20:44:14 smithi070 systemd[1]: Stopped Ceph node-exporter.b for b3f8e4ec-5a50-11ee-8db4-212e2dc638e7. 2023-09-23T20:44:15.248 INFO:journalctl@ceph.node-exporter.b.smithi070.stdout:Sep 23 20:44:14 smithi070 systemd[1]: Starting Ceph node-exporter.b for b3f8e4ec-5a50-11ee-8db4-212e2dc638e7... 2023-09-23T20:44:15.512 INFO:journalctl@ceph.node-exporter.b.smithi070.stdout:Sep 23 20:44:15 smithi070 podman[152697]: 2023-09-23T20:44:15.512 INFO:journalctl@ceph.node-exporter.b.smithi070.stdout:Sep 23 20:44:15 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-node-exporter-b[152712]: ts=2023-09-23T20:44:15.492Z caller=node_exporter.go:180 level=info msg="Starting node_exporter" version="(version=1.5.0, branch=HEAD, revision=1b48970ffcf5630534fb00bb0687d73c66d1c959)" 2023-09-23T20:44:15.512 INFO:journalctl@ceph.node-exporter.b.smithi070.stdout:Sep 23 20:44:15 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-node-exporter-b[152712]: ts=2023-09-23T20:44:15.492Z caller=node_exporter.go:181 level=info msg="Build context" build_context="(go=go1.19.3, user=root@6e7732a7b81b, date=20221129-18:59:09)" 2023-09-23T20:44:15.512 INFO:journalctl@ceph.node-exporter.b.smithi070.stdout:Sep 23 20:44:15 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-node-exporter-b[152712]: ts=2023-09-23T20:44:15.492Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+|var/lib/containers/storage/.+)($|/) 2023-09-23T20:44:15.512 INFO:journalctl@ceph.node-exporter.b.smithi070.stdout:Sep 23 20:44:15 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-node-exporter-b[152712]: ts=2023-09-23T20:44:15.492Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2023-09-23T20:44:15.513 INFO:journalctl@ceph.node-exporter.b.smithi070.stdout:Sep 23 20:44:15 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-node-exporter-b[152712]: ts=2023-09-23T20:44:15.493Z caller=diskstats_common.go:111 level=info collector=diskstats msg="Parsed flag --collector.diskstats.device-exclude" flag=^(ram|loop|fd|(h|s|v|xv)d[a-z]|nvme\d+n\d+p)\d+$ 2023-09-23T20:44:15.513 INFO:journalctl@ceph.node-exporter.b.smithi070.stdout:Sep 23 20:44:15 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-node-exporter-b[152712]: ts=2023-09-23T20:44:15.493Z caller=diskstats_linux.go:264 level=error collector=diskstats msg="Failed to open directory, disabling udev device properties" path=/run/udev/data 2023-09-23T20:44:15.513 INFO:journalctl@ceph.node-exporter.b.smithi070.stdout:Sep 23 20:44:15 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-node-exporter-b[152712]: ts=2023-09-23T20:44:15.493Z caller=node_exporter.go:110 level=info msg="Enabled collectors" 2023-09-23T20:44:15.513 INFO:journalctl@ceph.node-exporter.b.smithi070.stdout:Sep 23 20:44:15 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-node-exporter-b[152712]: ts=2023-09-23T20:44:15.493Z caller=node_exporter.go:117 level=info collector=arp 2023-09-23T20:44:15.513 INFO:journalctl@ceph.node-exporter.b.smithi070.stdout:Sep 23 20:44:15 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-node-exporter-b[152712]: ts=2023-09-23T20:44:15.493Z caller=node_exporter.go:117 level=info collector=bcache 2023-09-23T20:44:15.513 INFO:journalctl@ceph.node-exporter.b.smithi070.stdout:Sep 23 20:44:15 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-node-exporter-b[152712]: ts=2023-09-23T20:44:15.493Z caller=node_exporter.go:117 level=info collector=bonding 2023-09-23T20:44:15.513 INFO:journalctl@ceph.node-exporter.b.smithi070.stdout:Sep 23 20:44:15 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-node-exporter-b[152712]: ts=2023-09-23T20:44:15.493Z caller=node_exporter.go:117 level=info collector=btrfs 2023-09-23T20:44:15.513 INFO:journalctl@ceph.node-exporter.b.smithi070.stdout:Sep 23 20:44:15 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-node-exporter-b[152712]: ts=2023-09-23T20:44:15.493Z caller=node_exporter.go:117 level=info collector=conntrack 2023-09-23T20:44:15.513 INFO:journalctl@ceph.node-exporter.b.smithi070.stdout:Sep 23 20:44:15 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-node-exporter-b[152712]: ts=2023-09-23T20:44:15.493Z caller=node_exporter.go:117 level=info collector=cpu 2023-09-23T20:44:15.513 INFO:journalctl@ceph.node-exporter.b.smithi070.stdout:Sep 23 20:44:15 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-node-exporter-b[152712]: ts=2023-09-23T20:44:15.493Z caller=node_exporter.go:117 level=info collector=cpufreq 2023-09-23T20:44:15.513 INFO:journalctl@ceph.node-exporter.b.smithi070.stdout:Sep 23 20:44:15 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-node-exporter-b[152712]: ts=2023-09-23T20:44:15.493Z caller=node_exporter.go:117 level=info collector=diskstats 2023-09-23T20:44:15.514 INFO:journalctl@ceph.node-exporter.b.smithi070.stdout:Sep 23 20:44:15 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-node-exporter-b[152712]: ts=2023-09-23T20:44:15.493Z caller=node_exporter.go:117 level=info collector=dmi 2023-09-23T20:44:15.514 INFO:journalctl@ceph.node-exporter.b.smithi070.stdout:Sep 23 20:44:15 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-node-exporter-b[152712]: ts=2023-09-23T20:44:15.493Z caller=node_exporter.go:117 level=info collector=edac 2023-09-23T20:44:15.514 INFO:journalctl@ceph.node-exporter.b.smithi070.stdout:Sep 23 20:44:15 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-node-exporter-b[152712]: ts=2023-09-23T20:44:15.493Z caller=node_exporter.go:117 level=info collector=entropy 2023-09-23T20:44:15.514 INFO:journalctl@ceph.node-exporter.b.smithi070.stdout:Sep 23 20:44:15 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-node-exporter-b[152712]: ts=2023-09-23T20:44:15.493Z caller=node_exporter.go:117 level=info collector=fibrechannel 2023-09-23T20:44:15.514 INFO:journalctl@ceph.node-exporter.b.smithi070.stdout:Sep 23 20:44:15 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-node-exporter-b[152712]: ts=2023-09-23T20:44:15.493Z caller=node_exporter.go:117 level=info collector=filefd 2023-09-23T20:44:15.514 INFO:journalctl@ceph.node-exporter.b.smithi070.stdout:Sep 23 20:44:15 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-node-exporter-b[152712]: ts=2023-09-23T20:44:15.493Z caller=node_exporter.go:117 level=info collector=filesystem 2023-09-23T20:44:15.514 INFO:journalctl@ceph.node-exporter.b.smithi070.stdout:Sep 23 20:44:15 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-node-exporter-b[152712]: ts=2023-09-23T20:44:15.493Z caller=node_exporter.go:117 level=info collector=hwmon 2023-09-23T20:44:15.514 INFO:journalctl@ceph.node-exporter.b.smithi070.stdout:Sep 23 20:44:15 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-node-exporter-b[152712]: ts=2023-09-23T20:44:15.493Z caller=node_exporter.go:117 level=info collector=infiniband 2023-09-23T20:44:15.514 INFO:journalctl@ceph.node-exporter.b.smithi070.stdout:Sep 23 20:44:15 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-node-exporter-b[152712]: ts=2023-09-23T20:44:15.493Z caller=node_exporter.go:117 level=info collector=ipvs 2023-09-23T20:44:15.514 INFO:journalctl@ceph.node-exporter.b.smithi070.stdout:Sep 23 20:44:15 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-node-exporter-b[152712]: ts=2023-09-23T20:44:15.493Z caller=node_exporter.go:117 level=info collector=loadavg 2023-09-23T20:44:15.514 INFO:journalctl@ceph.node-exporter.b.smithi070.stdout:Sep 23 20:44:15 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-node-exporter-b[152712]: ts=2023-09-23T20:44:15.493Z caller=node_exporter.go:117 level=info collector=mdadm 2023-09-23T20:44:15.515 INFO:journalctl@ceph.node-exporter.b.smithi070.stdout:Sep 23 20:44:15 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-node-exporter-b[152712]: ts=2023-09-23T20:44:15.493Z caller=node_exporter.go:117 level=info collector=meminfo 2023-09-23T20:44:15.515 INFO:journalctl@ceph.node-exporter.b.smithi070.stdout:Sep 23 20:44:15 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-node-exporter-b[152712]: ts=2023-09-23T20:44:15.493Z caller=node_exporter.go:117 level=info collector=netclass 2023-09-23T20:44:15.515 INFO:journalctl@ceph.node-exporter.b.smithi070.stdout:Sep 23 20:44:15 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-node-exporter-b[152712]: ts=2023-09-23T20:44:15.493Z caller=node_exporter.go:117 level=info collector=netdev 2023-09-23T20:44:15.515 INFO:journalctl@ceph.node-exporter.b.smithi070.stdout:Sep 23 20:44:15 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-node-exporter-b[152712]: ts=2023-09-23T20:44:15.493Z caller=node_exporter.go:117 level=info collector=netstat 2023-09-23T20:44:15.515 INFO:journalctl@ceph.node-exporter.b.smithi070.stdout:Sep 23 20:44:15 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-node-exporter-b[152712]: ts=2023-09-23T20:44:15.493Z caller=node_exporter.go:117 level=info collector=nfs 2023-09-23T20:44:15.515 INFO:journalctl@ceph.node-exporter.b.smithi070.stdout:Sep 23 20:44:15 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-node-exporter-b[152712]: ts=2023-09-23T20:44:15.493Z caller=node_exporter.go:117 level=info collector=nfsd 2023-09-23T20:44:15.515 INFO:journalctl@ceph.node-exporter.b.smithi070.stdout:Sep 23 20:44:15 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-node-exporter-b[152712]: ts=2023-09-23T20:44:15.493Z caller=node_exporter.go:117 level=info collector=nvme 2023-09-23T20:44:15.515 INFO:journalctl@ceph.node-exporter.b.smithi070.stdout:Sep 23 20:44:15 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-node-exporter-b[152712]: ts=2023-09-23T20:44:15.493Z caller=node_exporter.go:117 level=info collector=os 2023-09-23T20:44:15.515 INFO:journalctl@ceph.node-exporter.b.smithi070.stdout:Sep 23 20:44:15 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-node-exporter-b[152712]: ts=2023-09-23T20:44:15.493Z caller=node_exporter.go:117 level=info collector=powersupplyclass 2023-09-23T20:44:15.515 INFO:journalctl@ceph.node-exporter.b.smithi070.stdout:Sep 23 20:44:15 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-node-exporter-b[152712]: ts=2023-09-23T20:44:15.493Z caller=node_exporter.go:117 level=info collector=pressure 2023-09-23T20:44:15.515 INFO:journalctl@ceph.node-exporter.b.smithi070.stdout:Sep 23 20:44:15 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-node-exporter-b[152712]: ts=2023-09-23T20:44:15.493Z caller=node_exporter.go:117 level=info collector=rapl 2023-09-23T20:44:15.515 INFO:journalctl@ceph.node-exporter.b.smithi070.stdout:Sep 23 20:44:15 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-node-exporter-b[152712]: ts=2023-09-23T20:44:15.493Z caller=node_exporter.go:117 level=info collector=schedstat 2023-09-23T20:44:15.516 INFO:journalctl@ceph.node-exporter.b.smithi070.stdout:Sep 23 20:44:15 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-node-exporter-b[152712]: ts=2023-09-23T20:44:15.493Z caller=node_exporter.go:117 level=info collector=selinux 2023-09-23T20:44:15.516 INFO:journalctl@ceph.node-exporter.b.smithi070.stdout:Sep 23 20:44:15 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-node-exporter-b[152712]: ts=2023-09-23T20:44:15.493Z caller=node_exporter.go:117 level=info collector=sockstat 2023-09-23T20:44:15.516 INFO:journalctl@ceph.node-exporter.b.smithi070.stdout:Sep 23 20:44:15 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-node-exporter-b[152712]: ts=2023-09-23T20:44:15.493Z caller=node_exporter.go:117 level=info collector=softnet 2023-09-23T20:44:15.516 INFO:journalctl@ceph.node-exporter.b.smithi070.stdout:Sep 23 20:44:15 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-node-exporter-b[152712]: ts=2023-09-23T20:44:15.493Z caller=node_exporter.go:117 level=info collector=stat 2023-09-23T20:44:15.516 INFO:journalctl@ceph.node-exporter.b.smithi070.stdout:Sep 23 20:44:15 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-node-exporter-b[152712]: ts=2023-09-23T20:44:15.493Z caller=node_exporter.go:117 level=info collector=tapestats 2023-09-23T20:44:15.516 INFO:journalctl@ceph.node-exporter.b.smithi070.stdout:Sep 23 20:44:15 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-node-exporter-b[152712]: ts=2023-09-23T20:44:15.493Z caller=node_exporter.go:117 level=info collector=textfile 2023-09-23T20:44:15.516 INFO:journalctl@ceph.node-exporter.b.smithi070.stdout:Sep 23 20:44:15 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-node-exporter-b[152712]: ts=2023-09-23T20:44:15.493Z caller=node_exporter.go:117 level=info collector=thermal_zone 2023-09-23T20:44:15.516 INFO:journalctl@ceph.node-exporter.b.smithi070.stdout:Sep 23 20:44:15 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-node-exporter-b[152712]: ts=2023-09-23T20:44:15.493Z caller=node_exporter.go:117 level=info collector=time 2023-09-23T20:44:15.516 INFO:journalctl@ceph.node-exporter.b.smithi070.stdout:Sep 23 20:44:15 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-node-exporter-b[152712]: ts=2023-09-23T20:44:15.493Z caller=node_exporter.go:117 level=info collector=udp_queues 2023-09-23T20:44:15.516 INFO:journalctl@ceph.node-exporter.b.smithi070.stdout:Sep 23 20:44:15 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-node-exporter-b[152712]: ts=2023-09-23T20:44:15.493Z caller=node_exporter.go:117 level=info collector=uname 2023-09-23T20:44:15.517 INFO:journalctl@ceph.node-exporter.b.smithi070.stdout:Sep 23 20:44:15 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-node-exporter-b[152712]: ts=2023-09-23T20:44:15.493Z caller=node_exporter.go:117 level=info collector=vmstat 2023-09-23T20:44:15.517 INFO:journalctl@ceph.node-exporter.b.smithi070.stdout:Sep 23 20:44:15 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-node-exporter-b[152712]: ts=2023-09-23T20:44:15.493Z caller=node_exporter.go:117 level=info collector=xfs 2023-09-23T20:44:15.517 INFO:journalctl@ceph.node-exporter.b.smithi070.stdout:Sep 23 20:44:15 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-node-exporter-b[152712]: ts=2023-09-23T20:44:15.493Z caller=node_exporter.go:117 level=info collector=zfs 2023-09-23T20:44:15.517 INFO:journalctl@ceph.node-exporter.b.smithi070.stdout:Sep 23 20:44:15 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-node-exporter-b[152712]: ts=2023-09-23T20:44:15.494Z caller=tls_config.go:232 level=info msg="Listening on" address=[::]:9100 2023-09-23T20:44:15.517 INFO:journalctl@ceph.node-exporter.b.smithi070.stdout:Sep 23 20:44:15 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-node-exporter-b[152712]: ts=2023-09-23T20:44:15.494Z caller=tls_config.go:235 level=info msg="TLS is disabled." http2=false address=[::]:9100 2023-09-23T20:44:15.517 INFO:journalctl@ceph.node-exporter.b.smithi070.stdout:Sep 23 20:44:15 smithi070 bash[152697]: 5b56a4a415d09438a5928c763d5609f9d4174662d3fdef144e4d2acffab0d93b 2023-09-23T20:44:15.869 INFO:journalctl@ceph.node-exporter.b.smithi070.stdout:Sep 23 20:44:15 smithi070 systemd[1]: Started Ceph node-exporter.b for b3f8e4ec-5a50-11ee-8db4-212e2dc638e7. 2023-09-23T20:44:16.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:44:15 smithi070 ceph-mon[100755]: from='client.? 172.21.15.26:0/3919918459' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:44:16.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:44:15 smithi070 ceph-mon[100755]: from='mgr.24781 ' entity='mgr.x' 2023-09-23T20:44:16.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:44:15 smithi070 ceph-mon[100755]: from='mgr.24781 ' entity='mgr.x' 2023-09-23T20:44:16.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:44:15 smithi070 ceph-mon[100755]: Reconfiguring grafana.a (dependencies changed)... 2023-09-23T20:44:16.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:44:15 smithi070 ceph-mon[100755]: Reconfiguring daemon grafana.a on smithi070 2023-09-23T20:44:16.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:44:15 smithi026 ceph-mon[109912]: from='client.? 172.21.15.26:0/3919918459' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:44:16.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:44:15 smithi026 ceph-mon[109912]: from='mgr.24781 ' entity='mgr.x' 2023-09-23T20:44:16.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:44:15 smithi026 ceph-mon[109912]: from='mgr.24781 ' entity='mgr.x' 2023-09-23T20:44:16.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:44:15 smithi026 ceph-mon[109912]: Reconfiguring grafana.a (dependencies changed)... 2023-09-23T20:44:16.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:44:15 smithi026 ceph-mon[109912]: Reconfiguring daemon grafana.a on smithi070 2023-09-23T20:44:16.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:44:15 smithi026 ceph-mon[95839]: from='client.? 172.21.15.26:0/3919918459' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:44:16.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:44:15 smithi026 ceph-mon[95839]: from='mgr.24781 ' entity='mgr.x' 2023-09-23T20:44:16.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:44:15 smithi026 ceph-mon[95839]: from='mgr.24781 ' entity='mgr.x' 2023-09-23T20:44:16.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:44:15 smithi026 ceph-mon[95839]: Reconfiguring grafana.a (dependencies changed)... 2023-09-23T20:44:16.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:44:15 smithi026 ceph-mon[95839]: Reconfiguring daemon grafana.a on smithi070 2023-09-23T20:44:17.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:44:16 smithi026 ceph-mon[109912]: pgmap v16: 73 pgs: 73 active+clean; 580 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:44:17.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:44:16 smithi026 ceph-mon[95839]: pgmap v16: 73 pgs: 73 active+clean; 580 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:44:17.153 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:44:16 smithi070 ceph-mon[100755]: pgmap v16: 73 pgs: 73 active+clean; 580 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:44:17.869 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:44:17 smithi070 systemd[1]: Stopping Ceph grafana.a for b3f8e4ec-5a50-11ee-8db4-212e2dc638e7... 2023-09-23T20:44:18.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:44:17 smithi026 ceph-mon[95839]: from='client.? 172.21.15.26:0/1803956201' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:44:18.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:44:17 smithi026 ceph-mon[109912]: from='client.? 172.21.15.26:0/1803956201' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:44:18.178 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:44:17 smithi070 ceph-mon[100755]: from='client.? 172.21.15.26:0/1803956201' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:44:18.455 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:44:18 smithi070 bash[152992]: ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a 2023-09-23T20:44:18.455 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:44:18 smithi070 systemd[1]: ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7@grafana.a.service: Main process exited, code=exited, status=143/n/a 2023-09-23T20:44:18.793 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:44:18 smithi070 systemd[1]: ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7@grafana.a.service: Failed with result 'exit-code'. 2023-09-23T20:44:18.794 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:44:18 smithi070 systemd[1]: Stopped Ceph grafana.a for b3f8e4ec-5a50-11ee-8db4-212e2dc638e7. 2023-09-23T20:44:18.794 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:44:18 smithi070 systemd[1]: Starting Ceph grafana.a for b3f8e4ec-5a50-11ee-8db4-212e2dc638e7... 2023-09-23T20:44:18.881 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Sep 23 20:44:18 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-alertmanager-a[163035]: ts=2023-09-23T20:44:18.400Z caller=cluster.go:698 level=info component=cluster msg="gossip settled; proceeding" elapsed=10.003014202s 2023-09-23T20:44:19.078 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:44:18 smithi070 podman[153114]: 2023-09-23T20:44:19.369 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 20:44:19 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-x[141470]: [23/Sep/2023:20:44:19] ENGINE Bus STOPPING 2023-09-23T20:44:19.369 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 20:44:19 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-x[141470]: [23/Sep/2023:20:44:19] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2023-09-23T20:44:19.370 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 20:44:19 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-x[141470]: [23/Sep/2023:20:44:19] ENGINE Bus STOPPED 2023-09-23T20:44:19.370 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 20:44:19 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-x[141470]: [23/Sep/2023:20:44:19] ENGINE Bus STARTING 2023-09-23T20:44:19.370 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 20:44:19 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-x[141470]: [23/Sep/2023:20:44:19] ENGINE Serving on http://:::9283 2023-09-23T20:44:19.370 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 20:44:19 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-x[141470]: [23/Sep/2023:20:44:19] ENGINE Bus STARTED 2023-09-23T20:44:19.370 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:44:19 smithi070 bash[153114]: ed7e189354a57e87564eb766157fbb26818bf16eb37fed3142ddf2bf2548e74f 2023-09-23T20:44:19.370 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:44:19 smithi070 systemd[1]: Started Ceph grafana.a for b3f8e4ec-5a50-11ee-8db4-212e2dc638e7. 2023-09-23T20:44:19.370 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:44:19 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[153130]: logger=settings t=2023-09-23T20:44:19.208091285Z level=info msg="Starting Grafana" version=9.4.7 commit=4add91f03d branch=HEAD compiled=2023-03-16T23:56:52Z 2023-09-23T20:44:19.371 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:44:19 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[153130]: logger=settings t=2023-09-23T20:44:19.208289127Z level=warn msg="\"sentry\" frontend logging provider is deprecated and will be removed in the next major version. Use \"grafana\" provider instead." 2023-09-23T20:44:19.371 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:44:19 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[153130]: logger=settings t=2023-09-23T20:44:19.208311904Z level=info msg="Config loaded from" file=/usr/share/grafana/conf/defaults.ini 2023-09-23T20:44:19.371 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:44:19 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[153130]: logger=settings t=2023-09-23T20:44:19.208319515Z level=info msg="Config loaded from" file=/etc/grafana/grafana.ini 2023-09-23T20:44:19.371 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:44:19 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[153130]: logger=settings t=2023-09-23T20:44:19.208325813Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_DATA=/var/lib/grafana" 2023-09-23T20:44:19.371 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:44:19 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[153130]: logger=settings t=2023-09-23T20:44:19.208332767Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_LOGS=/var/log/grafana" 2023-09-23T20:44:19.371 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:44:19 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[153130]: logger=settings t=2023-09-23T20:44:19.208338719Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_PLUGINS=/var/lib/grafana/plugins" 2023-09-23T20:44:19.371 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:44:19 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[153130]: logger=settings t=2023-09-23T20:44:19.208345416Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_PROVISIONING=/etc/grafana/provisioning" 2023-09-23T20:44:19.371 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:44:19 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[153130]: logger=settings t=2023-09-23T20:44:19.208351469Z level=info msg="Path Home" path=/usr/share/grafana 2023-09-23T20:44:19.371 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:44:19 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[153130]: logger=settings t=2023-09-23T20:44:19.208357374Z level=info msg="Path Data" path=/var/lib/grafana 2023-09-23T20:44:19.371 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:44:19 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[153130]: logger=settings t=2023-09-23T20:44:19.208363118Z level=info msg="Path Logs" path=/var/log/grafana 2023-09-23T20:44:19.371 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:44:19 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[153130]: logger=settings t=2023-09-23T20:44:19.208368733Z level=info msg="Path Plugins" path=/var/lib/grafana/plugins 2023-09-23T20:44:19.371 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:44:19 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[153130]: logger=settings t=2023-09-23T20:44:19.208374644Z level=info msg="Path Provisioning" path=/etc/grafana/provisioning 2023-09-23T20:44:19.372 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:44:19 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[153130]: logger=settings t=2023-09-23T20:44:19.208380605Z level=info msg="App mode production" 2023-09-23T20:44:19.372 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:44:19 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[153130]: logger=sqlstore t=2023-09-23T20:44:19.208431585Z level=info msg="Connecting to DB" dbtype=sqlite3 2023-09-23T20:44:19.372 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:44:19 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[153130]: logger=sqlstore t=2023-09-23T20:44:19.20845124Z level=warn msg="SQLite database file has broader permissions than it should" path=/var/lib/grafana/grafana.db mode=-rw-r--r-- expected=-rw-r----- 2023-09-23T20:44:19.372 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:44:19 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[153130]: logger=migrator t=2023-09-23T20:44:19.225032286Z level=info msg="Starting DB migrations" 2023-09-23T20:44:19.372 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:44:19 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[153130]: logger=migrator t=2023-09-23T20:44:19.230975588Z level=info msg="migrations completed" performed=0 skipped=481 duration=373.85µs 2023-09-23T20:44:19.372 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:44:19 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[153130]: logger=sqlstore t=2023-09-23T20:44:19.231412805Z level=info msg="Created default organization" 2023-09-23T20:44:19.372 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:44:19 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[153130]: logger=plugin.loader t=2023-09-23T20:44:19.287575109Z level=info msg="Plugin registered" pluginID=input 2023-09-23T20:44:19.372 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:44:19 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[153130]: logger=plugin.loader t=2023-09-23T20:44:19.314655913Z level=info msg="Plugin registered" pluginID=vonage-status-panel 2023-09-23T20:44:19.372 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:44:19 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[153130]: logger=plugin.loader t=2023-09-23T20:44:19.314676273Z level=info msg="Plugin registered" pluginID=grafana-piechart-panel 2023-09-23T20:44:19.372 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:44:19 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[153130]: logger=secrets t=2023-09-23T20:44:19.314709124Z level=info msg="Envelope encryption state" enabled=true currentprovider=secretKey.v1 2023-09-23T20:44:19.372 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:44:19 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[153130]: logger=query_data t=2023-09-23T20:44:19.316421457Z level=info msg="Query Service initialization" 2023-09-23T20:44:19.372 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:44:19 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[153130]: logger=live.push_http t=2023-09-23T20:44:19.318960545Z level=info msg="Live Push Gateway initialization" 2023-09-23T20:44:19.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:44:19 smithi070 ceph-mon[100755]: pgmap v17: 73 pgs: 73 active+clean; 580 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:44:19.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:44:19 smithi070 ceph-mon[100755]: from='mgr.24781 ' entity='mgr.x' 2023-09-23T20:44:19.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:44:19 smithi070 ceph-mon[100755]: from='mgr.24781 ' entity='mgr.x' 2023-09-23T20:44:19.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:44:19 smithi070 ceph-mon[100755]: from='mgr.24781 172.21.15.70:0/208042876' entity='mgr.x' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2023-09-23T20:44:19.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:44:19 smithi070 ceph-mon[100755]: from='mgr.24781 172.21.15.70:0/208042876' entity='mgr.x' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi026.front.sepia.ceph.com:9093"}]: dispatch 2023-09-23T20:44:19.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:44:19 smithi070 ceph-mon[100755]: from='mgr.24781 ' entity='mgr.x' 2023-09-23T20:44:19.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:44:19 smithi070 ceph-mon[100755]: from='mgr.24781 172.21.15.70:0/208042876' entity='mgr.x' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-09-23T20:44:19.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:44:19 smithi070 ceph-mon[100755]: from='mgr.24781 172.21.15.70:0/208042876' entity='mgr.x' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2023-09-23T20:44:19.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:44:19 smithi070 ceph-mon[100755]: from='mgr.24781 172.21.15.70:0/208042876' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:44:19.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:44:19 smithi026 ceph-mon[95839]: pgmap v17: 73 pgs: 73 active+clean; 580 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:44:19.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:44:19 smithi026 ceph-mon[95839]: from='mgr.24781 ' entity='mgr.x' 2023-09-23T20:44:19.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:44:19 smithi026 ceph-mon[95839]: from='mgr.24781 ' entity='mgr.x' 2023-09-23T20:44:19.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:44:19 smithi026 ceph-mon[95839]: from='mgr.24781 172.21.15.70:0/208042876' entity='mgr.x' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2023-09-23T20:44:19.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:44:19 smithi026 ceph-mon[95839]: from='mgr.24781 172.21.15.70:0/208042876' entity='mgr.x' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi026.front.sepia.ceph.com:9093"}]: dispatch 2023-09-23T20:44:19.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:44:19 smithi026 ceph-mon[95839]: from='mgr.24781 ' entity='mgr.x' 2023-09-23T20:44:19.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:44:19 smithi026 ceph-mon[95839]: from='mgr.24781 172.21.15.70:0/208042876' entity='mgr.x' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-09-23T20:44:19.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:44:19 smithi026 ceph-mon[95839]: from='mgr.24781 172.21.15.70:0/208042876' entity='mgr.x' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2023-09-23T20:44:19.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:44:19 smithi026 ceph-mon[95839]: from='mgr.24781 172.21.15.70:0/208042876' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:44:19.883 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:44:19 smithi026 ceph-mon[109912]: pgmap v17: 73 pgs: 73 active+clean; 580 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:44:19.883 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:44:19 smithi026 ceph-mon[109912]: from='mgr.24781 ' entity='mgr.x' 2023-09-23T20:44:19.883 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:44:19 smithi026 ceph-mon[109912]: from='mgr.24781 ' entity='mgr.x' 2023-09-23T20:44:19.883 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:44:19 smithi026 ceph-mon[109912]: from='mgr.24781 172.21.15.70:0/208042876' entity='mgr.x' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2023-09-23T20:44:19.883 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:44:19 smithi026 ceph-mon[109912]: from='mgr.24781 172.21.15.70:0/208042876' entity='mgr.x' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi026.front.sepia.ceph.com:9093"}]: dispatch 2023-09-23T20:44:19.883 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:44:19 smithi026 ceph-mon[109912]: from='mgr.24781 ' entity='mgr.x' 2023-09-23T20:44:19.883 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:44:19 smithi026 ceph-mon[109912]: from='mgr.24781 172.21.15.70:0/208042876' entity='mgr.x' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-09-23T20:44:19.883 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:44:19 smithi026 ceph-mon[109912]: from='mgr.24781 172.21.15.70:0/208042876' entity='mgr.x' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2023-09-23T20:44:19.883 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:44:19 smithi026 ceph-mon[109912]: from='mgr.24781 172.21.15.70:0/208042876' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:44:20.370 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:44:20 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[153130]: logger=infra.usagestats.collector t=2023-09-23T20:44:20.022840706Z level=info msg="registering usage stat providers" usageStatsProvidersLen=2 2023-09-23T20:44:20.370 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:44:20 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[153130]: logger=provisioning.datasources t=2023-09-23T20:44:20.109041824Z level=info msg="deleted correlations based on configuration" ds_name=Dashboard1 2023-09-23T20:44:20.370 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:44:20 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[153130]: logger=provisioning.datasources t=2023-09-23T20:44:20.109103039Z level=info msg="deleted datasource based on configuration" name=Dashboard1 2023-09-23T20:44:20.370 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:44:20 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[153130]: logger=provisioning.datasources t=2023-09-23T20:44:20.1205551Z level=info msg="inserting datasource from configuration " name=Dashboard1 uid=P43CA22E17D0F9596 2023-09-23T20:44:20.370 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:44:20 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[153130]: logger=provisioning.alerting t=2023-09-23T20:44:20.199299245Z level=info msg="starting to provision alerting" 2023-09-23T20:44:20.371 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:44:20 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[153130]: logger=provisioning.alerting t=2023-09-23T20:44:20.199319897Z level=info msg="finished to provision alerting" 2023-09-23T20:44:20.371 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:44:20 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[153130]: logger=grafanaStorageLogger t=2023-09-23T20:44:20.199489176Z level=info msg="storage starting" 2023-09-23T20:44:20.371 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:44:20 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[153130]: logger=ngalert.state.manager t=2023-09-23T20:44:20.199707029Z level=info msg="Warming state cache for startup" 2023-09-23T20:44:20.371 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:44:20 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[153130]: logger=http.server t=2023-09-23T20:44:20.201083563Z level=info msg="HTTP Server Listen" address=[::]:3000 protocol=https subUrl= socket= 2023-09-23T20:44:20.371 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:44:20 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[153130]: logger=ngalert.state.manager t=2023-09-23T20:44:20.210009945Z level=info msg="State cache has been initialized" states=0 duration=10.301868ms 2023-09-23T20:44:20.371 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:44:20 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[153130]: logger=ticker t=2023-09-23T20:44:20.210063108Z level=info msg=starting first_tick=2023-09-23T20:44:30Z 2023-09-23T20:44:20.371 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:44:20 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[153130]: logger=ngalert.multiorg.alertmanager t=2023-09-23T20:44:20.210067058Z level=info msg="starting MultiOrg Alertmanager" 2023-09-23T20:44:20.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:44:20 smithi070 ceph-mon[100755]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2023-09-23T20:44:20.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:44:20 smithi070 ceph-mon[100755]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi026.front.sepia.ceph.com:9093"}]: dispatch 2023-09-23T20:44:20.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:44:20 smithi070 ceph-mon[100755]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-09-23T20:44:20.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:44:20 smithi070 ceph-mon[100755]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2023-09-23T20:44:20.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:44:20 smithi070 ceph-mon[100755]: Upgrade: Need to upgrade myself (mgr.x) 2023-09-23T20:44:20.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:44:20 smithi070 ceph-mon[100755]: Upgrade: Need to upgrade myself (mgr.x) 2023-09-23T20:44:20.871 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:44:20 smithi070 ceph-mon[100755]: from='client.? 172.21.15.26:0/1188476382' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:44:20.871 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:44:20 smithi070 ceph-mon[100755]: from='mgr.24781 ' entity='mgr.x' 2023-09-23T20:44:20.871 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:44:20 smithi070 ceph-mon[100755]: from='mgr.24781 172.21.15.70:0/208042876' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-09-23T20:44:20.871 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:44:20 smithi070 ceph-mon[100755]: from='mgr.24781 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-09-23T20:44:20.871 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:44:20 smithi070 ceph-mon[100755]: from='mgr.24781 172.21.15.70:0/208042876' entity='mgr.x' cmd=[{"prefix": "mgr services"}]: dispatch 2023-09-23T20:44:20.881 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:44:20 smithi026 ceph-mon[95839]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2023-09-23T20:44:20.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:44:20 smithi026 ceph-mon[95839]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi026.front.sepia.ceph.com:9093"}]: dispatch 2023-09-23T20:44:20.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:44:20 smithi026 ceph-mon[95839]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-09-23T20:44:20.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:44:20 smithi026 ceph-mon[95839]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2023-09-23T20:44:20.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:44:20 smithi026 ceph-mon[95839]: Upgrade: Need to upgrade myself (mgr.x) 2023-09-23T20:44:20.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:44:20 smithi026 ceph-mon[95839]: Upgrade: Need to upgrade myself (mgr.x) 2023-09-23T20:44:20.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:44:20 smithi026 ceph-mon[95839]: from='client.? 172.21.15.26:0/1188476382' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:44:20.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:44:20 smithi026 ceph-mon[95839]: from='mgr.24781 ' entity='mgr.x' 2023-09-23T20:44:20.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:44:20 smithi026 ceph-mon[95839]: from='mgr.24781 172.21.15.70:0/208042876' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-09-23T20:44:20.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:44:20 smithi026 ceph-mon[95839]: from='mgr.24781 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-09-23T20:44:20.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:44:20 smithi026 ceph-mon[95839]: from='mgr.24781 172.21.15.70:0/208042876' entity='mgr.x' cmd=[{"prefix": "mgr services"}]: dispatch 2023-09-23T20:44:20.883 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:44:20 smithi026 ceph-mon[109912]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2023-09-23T20:44:20.883 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:44:20 smithi026 ceph-mon[109912]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi026.front.sepia.ceph.com:9093"}]: dispatch 2023-09-23T20:44:20.883 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:44:20 smithi026 ceph-mon[109912]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-09-23T20:44:20.883 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:44:20 smithi026 ceph-mon[109912]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2023-09-23T20:44:20.883 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:44:20 smithi026 ceph-mon[109912]: Upgrade: Need to upgrade myself (mgr.x) 2023-09-23T20:44:20.883 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:44:20 smithi026 ceph-mon[109912]: Upgrade: Need to upgrade myself (mgr.x) 2023-09-23T20:44:20.883 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:44:20 smithi026 ceph-mon[109912]: from='client.? 172.21.15.26:0/1188476382' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:44:20.883 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:44:20 smithi026 ceph-mon[109912]: from='mgr.24781 ' entity='mgr.x' 2023-09-23T20:44:20.883 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:44:20 smithi026 ceph-mon[109912]: from='mgr.24781 172.21.15.70:0/208042876' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-09-23T20:44:20.883 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:44:20 smithi026 ceph-mon[109912]: from='mgr.24781 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-09-23T20:44:20.884 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:44:20 smithi026 ceph-mon[109912]: from='mgr.24781 172.21.15.70:0/208042876' entity='mgr.x' cmd=[{"prefix": "mgr services"}]: dispatch 2023-09-23T20:44:21.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:44:21 smithi070 ceph-mon[100755]: Upgrade: Updating mgr.y 2023-09-23T20:44:21.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:44:21 smithi070 ceph-mon[100755]: from='mgr.24781 172.21.15.70:0/208042876' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:44:21.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:44:21 smithi070 ceph-mon[100755]: Deploying daemon mgr.y on smithi026 2023-09-23T20:44:21.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:44:21 smithi070 ceph-mon[100755]: pgmap v18: 73 pgs: 73 active+clean; 580 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:44:21.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:44:21 smithi070 ceph-mon[100755]: from='mgr.24781 172.21.15.70:0/208042876' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:44:21.874 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:44:21 smithi026 ceph-mon[95839]: Upgrade: Updating mgr.y 2023-09-23T20:44:21.874 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:44:21 smithi026 ceph-mon[95839]: from='mgr.24781 172.21.15.70:0/208042876' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:44:21.874 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:44:21 smithi026 ceph-mon[95839]: Deploying daemon mgr.y on smithi026 2023-09-23T20:44:21.874 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:44:21 smithi026 ceph-mon[95839]: pgmap v18: 73 pgs: 73 active+clean; 580 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:44:21.875 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:44:21 smithi026 ceph-mon[95839]: from='mgr.24781 172.21.15.70:0/208042876' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:44:21.875 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:44:21 smithi026 ceph-mon[109912]: Upgrade: Updating mgr.y 2023-09-23T20:44:21.875 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:44:21 smithi026 ceph-mon[109912]: from='mgr.24781 172.21.15.70:0/208042876' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:44:21.875 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:44:21 smithi026 ceph-mon[109912]: Deploying daemon mgr.y on smithi026 2023-09-23T20:44:21.875 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:44:21 smithi026 ceph-mon[109912]: pgmap v18: 73 pgs: 73 active+clean; 580 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:44:21.875 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:44:21 smithi026 ceph-mon[109912]: from='mgr.24781 172.21.15.70:0/208042876' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:44:22.594 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:44:22 smithi026 systemd[1]: Stopping Ceph mgr.y for b3f8e4ec-5a50-11ee-8db4-212e2dc638e7... 2023-09-23T20:44:22.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:44:22 smithi070 ceph-mon[100755]: from='client.? 172.21.15.26:0/2895282891' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:44:22.872 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:44:22 smithi026 ceph-mon[109912]: from='client.? 172.21.15.26:0/2895282891' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:44:22.872 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:44:22 smithi026 ceph-mon[95839]: from='client.? 172.21.15.26:0/2895282891' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:44:22.872 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:44:22 smithi026 bash[163866]: ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y 2023-09-23T20:44:23.198 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:44:22 smithi026 systemd[1]: ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7@mgr.y.service: Main process exited, code=exited, status=143/n/a 2023-09-23T20:44:23.198 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:44:23 smithi026 systemd[1]: ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7@mgr.y.service: Failed with result 'exit-code'. 2023-09-23T20:44:23.198 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:44:23 smithi026 systemd[1]: Stopped Ceph mgr.y for b3f8e4ec-5a50-11ee-8db4-212e2dc638e7. 2023-09-23T20:44:23.603 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 20:44:23 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-x[141470]: ::ffff:172.21.15.70 - - [23/Sep/2023:20:44:23] "GET /metrics HTTP/1.1" 200 33701 "" "Prometheus/2.43.0" 2023-09-23T20:44:23.765 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:44:23 smithi026 ceph-mon[109912]: pgmap v19: 73 pgs: 73 active+clean; 580 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:44:23.765 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:44:23 smithi026 systemd[1]: Starting Ceph mgr.y for b3f8e4ec-5a50-11ee-8db4-212e2dc638e7... 2023-09-23T20:44:23.765 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:44:23 smithi026 podman[164013]: 2023-09-23T20:44:23.766 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:44:23 smithi026 ceph-mon[95839]: pgmap v19: 73 pgs: 73 active+clean; 580 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:44:23.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:44:23 smithi070 ceph-mon[100755]: pgmap v19: 73 pgs: 73 active+clean; 580 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:44:24.017 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:44:23 smithi026 ceph-mgr[164053]: -- 172.21.15.26:0/968115535 <== mon.2 v2:172.21.15.26:3301/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x55d5e2cfb1e0 con 0x55d5e2cd0c00 2023-09-23T20:44:24.017 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:44:23 smithi026 bash[164013]: 5cc7ece3b7e85f51669a19e7d2080126c848b942dd94935312ce498cf59d3050 2023-09-23T20:44:24.017 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:44:23 smithi026 systemd[1]: Started Ceph mgr.y for b3f8e4ec-5a50-11ee-8db4-212e2dc638e7. 2023-09-23T20:44:24.017 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:44:23 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[164029]: 2023-09-23T20:44:23.916+0000 7ffba324b200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2023-09-23T20:44:24.017 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:44:24 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[164029]: 2023-09-23T20:44:24.015+0000 7ffba324b200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-09-23T20:44:24.382 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:44:24 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[164029]: 2023-09-23T20:44:24.128+0000 7ffba324b200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-09-23T20:44:24.382 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:44:24 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[164029]: 2023-09-23T20:44:24.243+0000 7ffba324b200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-09-23T20:44:24.858 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:44:24 smithi026 ceph-mon[109912]: from='mgr.24781 ' entity='mgr.x' 2023-09-23T20:44:24.858 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:44:24 smithi026 ceph-mon[109912]: from='mgr.24781 ' entity='mgr.x' 2023-09-23T20:44:24.858 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:44:24 smithi026 ceph-mon[109912]: from='mgr.24781 172.21.15.70:0/208042876' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:44:24.859 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:44:24 smithi026 ceph-mon[109912]: pgmap v20: 73 pgs: 73 active+clean; 580 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:44:24.859 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:44:24 smithi026 ceph-mon[109912]: from='client.? 172.21.15.26:0/757524949' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:44:24.859 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:44:24 smithi026 ceph-mon[95839]: from='mgr.24781 ' entity='mgr.x' 2023-09-23T20:44:24.859 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:44:24 smithi026 ceph-mon[95839]: from='mgr.24781 ' entity='mgr.x' 2023-09-23T20:44:24.859 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:44:24 smithi026 ceph-mon[95839]: from='mgr.24781 172.21.15.70:0/208042876' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:44:24.859 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:44:24 smithi026 ceph-mon[95839]: pgmap v20: 73 pgs: 73 active+clean; 580 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:44:24.859 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:44:24 smithi026 ceph-mon[95839]: from='client.? 172.21.15.26:0/757524949' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:44:24.860 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:44:24 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[164029]: 2023-09-23T20:44:24.475+0000 7ffba324b200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-09-23T20:44:25.122 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:44:24 smithi070 ceph-mon[100755]: from='mgr.24781 ' entity='mgr.x' 2023-09-23T20:44:25.122 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:44:24 smithi070 ceph-mon[100755]: from='mgr.24781 ' entity='mgr.x' 2023-09-23T20:44:25.122 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:44:24 smithi070 ceph-mon[100755]: from='mgr.24781 172.21.15.70:0/208042876' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:44:25.122 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:44:24 smithi070 ceph-mon[100755]: pgmap v20: 73 pgs: 73 active+clean; 580 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:44:25.122 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:44:24 smithi070 ceph-mon[100755]: from='client.? 172.21.15.26:0/757524949' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:44:25.132 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:44:24 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[164029]: 2023-09-23T20:44:24.987+0000 7ffba324b200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-09-23T20:44:25.632 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:44:25 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[164029]: 2023-09-23T20:44:25.148+0000 7ffba324b200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-09-23T20:44:25.632 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:44:25 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[164029]: 2023-09-23T20:44:25.256+0000 7ffba324b200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-09-23T20:44:25.632 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:44:25 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[164029]: 2023-09-23T20:44:25.384+0000 7ffba324b200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-09-23T20:44:26.382 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:44:25 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[164029]: 2023-09-23T20:44:25.931+0000 7ffba324b200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-09-23T20:44:27.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:44:26 smithi070 ceph-mon[100755]: pgmap v21: 73 pgs: 73 active+clean; 580 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:44:27.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:44:26 smithi026 ceph-mon[95839]: pgmap v21: 73 pgs: 73 active+clean; 580 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:44:27.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:44:26 smithi026 ceph-mon[109912]: pgmap v21: 73 pgs: 73 active+clean; 580 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:44:27.132 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:44:26 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[164029]: 2023-09-23T20:44:26.759+0000 7ffba324b200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-09-23T20:44:27.133 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:44:26 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[164029]: 2023-09-23T20:44:26.873+0000 7ffba324b200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-09-23T20:44:28.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:44:27 smithi070 ceph-mon[100755]: from='client.? 172.21.15.26:0/2735044718' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:44:28.131 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:44:27 smithi026 ceph-mon[95839]: from='client.? 172.21.15.26:0/2735044718' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:44:28.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:44:27 smithi026 ceph-mon[109912]: from='client.? 172.21.15.26:0/2735044718' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:44:28.132 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:44:27 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[164029]: 2023-09-23T20:44:27.894+0000 7ffba324b200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-09-23T20:44:28.132 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:44:28 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[164029]: 2023-09-23T20:44:28.012+0000 7ffba324b200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2023-09-23T20:44:28.618 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:44:28 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[164029]: 2023-09-23T20:44:28.242+0000 7ffba324b200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-09-23T20:44:28.881 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:44:28 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[164029]: 2023-09-23T20:44:28.617+0000 7ffba324b200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2023-09-23T20:44:29.119 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 20:44:28 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-prometheus-a[152415]: ts=2023-09-23T20:44:28.659Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1123-g61431f6b (61431f6ba2752649b506867ec856b4a6236f90dd) quincy (stable)\", cluster_addr=\"172.21.15.26\", device_class=\"ssd\", hostname=\"smithi026\", instance=\"smithi070:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.26\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1123-g61431f6b (61431f6ba2752649b506867ec856b4a6236f90dd) quincy (stable)\", cluster_addr=\"172.21.15.26\", device_class=\"ssd\", hostname=\"smithi026\", instance=\"smithi070.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.26\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-09-23T20:44:29.382 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:44:28 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[164029]: 2023-09-23T20:44:28.883+0000 7ffba324b200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-09-23T20:44:29.382 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:44:29 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[164029]: 2023-09-23T20:44:29.074+0000 7ffba324b200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-09-23T20:44:29.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:44:29 smithi070 ceph-mon[100755]: from='mgr.24781 ' entity='mgr.x' 2023-09-23T20:44:29.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:44:29 smithi070 ceph-mon[100755]: from='mgr.24781 ' entity='mgr.x' 2023-09-23T20:44:29.871 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:44:29 smithi070 ceph-mon[100755]: pgmap v22: 73 pgs: 73 active+clean; 580 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:44:29.871 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:44:29 smithi070 ceph-mon[100755]: from='mgr.24781 ' entity='mgr.x' 2023-09-23T20:44:29.871 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:44:29 smithi070 ceph-mon[100755]: from='mgr.24781 ' entity='mgr.x' 2023-09-23T20:44:29.871 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:44:29 smithi070 ceph-mon[100755]: from='mgr.24781 ' entity='mgr.x' 2023-09-23T20:44:29.871 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:44:29 smithi070 ceph-mon[100755]: from='mgr.24781 ' entity='mgr.x' 2023-09-23T20:44:29.881 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:44:29 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[164029]: 2023-09-23T20:44:29.434+0000 7ffba324b200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-09-23T20:44:29.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:44:29 smithi026 ceph-mon[95839]: from='mgr.24781 ' entity='mgr.x' 2023-09-23T20:44:29.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:44:29 smithi026 ceph-mon[95839]: from='mgr.24781 ' entity='mgr.x' 2023-09-23T20:44:29.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:44:29 smithi026 ceph-mon[95839]: pgmap v22: 73 pgs: 73 active+clean; 580 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:44:29.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:44:29 smithi026 ceph-mon[95839]: from='mgr.24781 ' entity='mgr.x' 2023-09-23T20:44:29.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:44:29 smithi026 ceph-mon[95839]: from='mgr.24781 ' entity='mgr.x' 2023-09-23T20:44:29.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:44:29 smithi026 ceph-mon[95839]: from='mgr.24781 ' entity='mgr.x' 2023-09-23T20:44:29.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:44:29 smithi026 ceph-mon[95839]: from='mgr.24781 ' entity='mgr.x' 2023-09-23T20:44:29.883 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:44:29 smithi026 ceph-mon[109912]: from='mgr.24781 ' entity='mgr.x' 2023-09-23T20:44:29.883 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:44:29 smithi026 ceph-mon[109912]: from='mgr.24781 ' entity='mgr.x' 2023-09-23T20:44:29.883 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:44:29 smithi026 ceph-mon[109912]: pgmap v22: 73 pgs: 73 active+clean; 580 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:44:29.883 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:44:29 smithi026 ceph-mon[109912]: from='mgr.24781 ' entity='mgr.x' 2023-09-23T20:44:29.883 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:44:29 smithi026 ceph-mon[109912]: from='mgr.24781 ' entity='mgr.x' 2023-09-23T20:44:29.883 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:44:29 smithi026 ceph-mon[109912]: from='mgr.24781 ' entity='mgr.x' 2023-09-23T20:44:29.883 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:44:29 smithi026 ceph-mon[109912]: from='mgr.24781 ' entity='mgr.x' 2023-09-23T20:44:30.501 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:44:30 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[164029]: 2023-09-23T20:44:30.322+0000 7ffba324b200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-09-23T20:44:30.501 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:44:30 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[164029]: 2023-09-23T20:44:30.500+0000 7ffba324b200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-09-23T20:44:30.770 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:44:30 smithi026 ceph-mon[109912]: from='client.? 172.21.15.26:0/1945171349' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:44:30.770 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:44:30 smithi026 ceph-mon[95839]: from='client.? 172.21.15.26:0/1945171349' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:44:30.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:44:30 smithi070 ceph-mon[100755]: from='client.? 172.21.15.26:0/1945171349' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:44:31.030 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:44:30 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[164029]: 2023-09-23T20:44:30.769+0000 7ffba324b200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-09-23T20:44:31.030 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:44:30 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[164029]: 2023-09-23T20:44:30.915+0000 7ffba324b200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-09-23T20:44:31.382 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:44:31 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[164029]: 2023-09-23T20:44:31.209+0000 7ffba324b200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-09-23T20:44:31.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:44:31 smithi070 ceph-mon[100755]: pgmap v23: 73 pgs: 73 active+clean; 580 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:44:31.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:44:31 smithi026 ceph-mon[95839]: pgmap v23: 73 pgs: 73 active+clean; 580 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:44:31.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:44:31 smithi026 ceph-mon[109912]: pgmap v23: 73 pgs: 73 active+clean; 580 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:44:32.478 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:44:32 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[164029]: 2023-09-23T20:44:32.357+0000 7ffba324b200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-09-23T20:44:32.479 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:44:32 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[164029]: [23/Sep/2023:20:44:32] ENGINE Bus STARTING 2023-09-23T20:44:32.479 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:44:32 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[164029]: CherryPy Checker: 2023-09-23T20:44:32.849 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:44:32 smithi026 ceph-mon[95839]: from='client.? 172.21.15.26:0/1484219247' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:44:32.850 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:44:32 smithi026 ceph-mon[95839]: Standby manager daemon y restarted 2023-09-23T20:44:32.850 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:44:32 smithi026 ceph-mon[95839]: Standby manager daemon y started 2023-09-23T20:44:32.850 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:44:32 smithi026 ceph-mon[95839]: from='mgr.? 172.21.15.26:0/503544817' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2023-09-23T20:44:32.850 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:44:32 smithi026 ceph-mon[95839]: from='mgr.? 172.21.15.26:0/503544817' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-09-23T20:44:32.850 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:44:32 smithi026 ceph-mon[95839]: from='mgr.? 172.21.15.26:0/503544817' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2023-09-23T20:44:32.850 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:44:32 smithi026 ceph-mon[95839]: from='mgr.? 172.21.15.26:0/503544817' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-09-23T20:44:32.850 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:44:32 smithi026 ceph-mon[109912]: from='client.? 172.21.15.26:0/1484219247' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:44:32.851 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:44:32 smithi026 ceph-mon[109912]: Standby manager daemon y restarted 2023-09-23T20:44:32.851 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:44:32 smithi026 ceph-mon[109912]: Standby manager daemon y started 2023-09-23T20:44:32.851 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:44:32 smithi026 ceph-mon[109912]: from='mgr.? 172.21.15.26:0/503544817' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2023-09-23T20:44:32.851 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:44:32 smithi026 ceph-mon[109912]: from='mgr.? 172.21.15.26:0/503544817' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-09-23T20:44:32.851 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:44:32 smithi026 ceph-mon[109912]: from='mgr.? 172.21.15.26:0/503544817' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2023-09-23T20:44:32.851 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:44:32 smithi026 ceph-mon[109912]: from='mgr.? 172.21.15.26:0/503544817' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-09-23T20:44:32.851 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:44:32 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[164029]: The Application mounted at '' has an empty config. 2023-09-23T20:44:32.851 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:44:32 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[164029]: 2023-09-23T20:44:32.851 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:44:32 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[164029]: [23/Sep/2023:20:44:32] ENGINE Serving on http://:::9283 2023-09-23T20:44:32.851 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:44:32 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[164029]: [23/Sep/2023:20:44:32] ENGINE Bus STARTED 2023-09-23T20:44:32.852 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:44:32 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[164029]: [23/Sep/2023:20:44:32] ENGINE Bus STOPPING 2023-09-23T20:44:32.868 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 20:44:32 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-x[141470]: 2023-09-23T20:44:32.839+0000 7fed26e87700 -1 mgr handle_mgr_map I was active but no longer am 2023-09-23T20:44:32.868 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:44:32 smithi070 ceph-mon[100755]: from='client.? 172.21.15.26:0/1484219247' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:44:32.868 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:44:32 smithi070 ceph-mon[100755]: Standby manager daemon y restarted 2023-09-23T20:44:32.868 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:44:32 smithi070 ceph-mon[100755]: Standby manager daemon y started 2023-09-23T20:44:32.868 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:44:32 smithi070 ceph-mon[100755]: from='mgr.? 172.21.15.26:0/503544817' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2023-09-23T20:44:32.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:44:32 smithi070 ceph-mon[100755]: from='mgr.? 172.21.15.26:0/503544817' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-09-23T20:44:32.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:44:32 smithi070 ceph-mon[100755]: from='mgr.? 172.21.15.26:0/503544817' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2023-09-23T20:44:32.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:44:32 smithi070 ceph-mon[100755]: from='mgr.? 172.21.15.26:0/503544817' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-09-23T20:44:33.119 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 20:44:32 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-x[141470]: ignoring --setuser ceph since I am not root 2023-09-23T20:44:33.119 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 20:44:32 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-x[141470]: ignoring --setgroup ceph since I am not root 2023-09-23T20:44:33.120 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 20:44:32 smithi070 ceph-mgr[141494]: -- 172.21.15.70:0/633189811 <== mon.1 v2:172.21.15.70:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x559b7c5891e0 con 0x559b7c55f000 2023-09-23T20:44:33.120 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 20:44:33 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-x[141470]: 2023-09-23T20:44:33.012+0000 7f2fd6ca0200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2023-09-23T20:44:33.120 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 20:44:33 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-x[141470]: 2023-09-23T20:44:33.095+0000 7f2fd6ca0200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-09-23T20:44:33.132 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:44:32 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[164029]: [23/Sep/2023:20:44:32] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2023-09-23T20:44:33.132 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:44:32 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[164029]: [23/Sep/2023:20:44:32] ENGINE Bus STOPPED 2023-09-23T20:44:33.132 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:44:32 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[164029]: [23/Sep/2023:20:44:32] ENGINE Bus STARTING 2023-09-23T20:44:33.132 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:44:33 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[164029]: 2023-09-23T20:44:33.002+0000 7ffafeee4700 -1 client.0 error registering admin socket command: (17) File exists 2023-09-23T20:44:33.132 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:44:33 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[164029]: 2023-09-23T20:44:33.002+0000 7ffafeee4700 -1 client.0 error registering admin socket command: (17) File exists 2023-09-23T20:44:33.133 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:44:33 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[164029]: 2023-09-23T20:44:33.002+0000 7ffafeee4700 -1 client.0 error registering admin socket command: (17) File exists 2023-09-23T20:44:33.133 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:44:33 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[164029]: 2023-09-23T20:44:33.002+0000 7ffafeee4700 -1 client.0 error registering admin socket command: (17) File exists 2023-09-23T20:44:33.133 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:44:33 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[164029]: 2023-09-23T20:44:33.002+0000 7ffafeee4700 -1 client.0 error registering admin socket command: (17) File exists 2023-09-23T20:44:33.133 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:44:33 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[164029]: 2023-09-23T20:44:33.002+0000 7ffafcee0700 -1 client.0 error registering admin socket command: (17) File exists 2023-09-23T20:44:33.133 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:44:33 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[164029]: 2023-09-23T20:44:33.002+0000 7ffafcee0700 -1 client.0 error registering admin socket command: (17) File exists 2023-09-23T20:44:33.133 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:44:33 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[164029]: 2023-09-23T20:44:33.002+0000 7ffafcee0700 -1 client.0 error registering admin socket command: (17) File exists 2023-09-23T20:44:33.133 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:44:33 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[164029]: 2023-09-23T20:44:33.002+0000 7ffafcee0700 -1 client.0 error registering admin socket command: (17) File exists 2023-09-23T20:44:33.133 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:44:33 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[164029]: 2023-09-23T20:44:33.002+0000 7ffafcee0700 -1 client.0 error registering admin socket command: (17) File exists 2023-09-23T20:44:33.133 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:44:33 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[164029]: [23/Sep/2023:20:44:33] ENGINE Serving on http://:::9283 2023-09-23T20:44:33.133 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:44:33 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[164029]: [23/Sep/2023:20:44:33] ENGINE Bus STARTED 2023-09-23T20:44:33.462 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 20:44:33 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-x[141470]: 2023-09-23T20:44:33.183+0000 7f2fd6ca0200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-09-23T20:44:33.462 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 20:44:33 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-x[141470]: 2023-09-23T20:44:33.278+0000 7f2fd6ca0200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-09-23T20:44:33.847 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 20:44:33 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-x[141470]: 2023-09-23T20:44:33.461+0000 7f2fd6ca0200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-09-23T20:44:33.847 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:44:33 smithi070 ceph-mon[100755]: mgrmap e26: x(active, since 41s), standbys: y 2023-09-23T20:44:33.847 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:44:33 smithi070 ceph-mon[100755]: pgmap v24: 73 pgs: 73 active+clean; 580 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:44:33.847 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:44:33 smithi070 ceph-mon[100755]: from='mgr.24781 ' entity='mgr.x' 2023-09-23T20:44:33.848 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:44:33 smithi070 ceph-mon[100755]: from='mgr.24781 ' entity='mgr.x' 2023-09-23T20:44:33.848 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:44:33 smithi070 ceph-mon[100755]: from='mgr.24781 172.21.15.70:0/208042876' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:44:33.848 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:44:33 smithi070 ceph-mon[100755]: from='mgr.24781 172.21.15.70:0/208042876' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:44:33.848 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:44:33 smithi070 ceph-mon[100755]: from='mgr.24781 ' entity='mgr.x' 2023-09-23T20:44:33.848 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:44:33 smithi070 ceph-mon[100755]: from='mgr.24781 172.21.15.70:0/208042876' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:44:33.848 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:44:33 smithi070 ceph-mon[100755]: from='mgr.24781 172.21.15.70:0/208042876' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2023-09-23T20:44:33.848 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:44:33 smithi070 ceph-mon[100755]: from='mgr.24781 ' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2023-09-23T20:44:33.848 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:44:33 smithi070 ceph-mon[100755]: osdmap e78: 8 total, 8 up, 8 in 2023-09-23T20:44:33.849 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:44:33 smithi070 ceph-mon[100755]: from='mgr.24781 ' entity='mgr.x' cmd='[{"prefix": "mgr fail", "who": "x"}]': finished 2023-09-23T20:44:33.849 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:44:33 smithi070 ceph-mon[100755]: mgrmap e27: y(active, starting, since 0.00556218s) 2023-09-23T20:44:33.849 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:44:33 smithi070 ceph-mon[100755]: from='mgr.24989 172.21.15.26:0/503544817' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-23T20:44:33.849 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:44:33 smithi070 ceph-mon[100755]: from='mgr.24989 172.21.15.26:0/503544817' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T20:44:33.849 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:44:33 smithi070 ceph-mon[100755]: from='mgr.24989 172.21.15.26:0/503544817' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T20:44:33.849 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:44:33 smithi070 ceph-mon[100755]: from='mgr.24989 172.21.15.26:0/503544817' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi026.znfpnl"}]: dispatch 2023-09-23T20:44:33.849 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:44:33 smithi070 ceph-mon[100755]: from='mgr.24989 172.21.15.26:0/503544817' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi070.imqhqy"}]: dispatch 2023-09-23T20:44:33.850 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:44:33 smithi070 ceph-mon[100755]: from='mgr.24989 172.21.15.26:0/503544817' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-09-23T20:44:33.850 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:44:33 smithi070 ceph-mon[100755]: from='mgr.24989 172.21.15.26:0/503544817' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-23T20:44:33.850 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:44:33 smithi070 ceph-mon[100755]: from='mgr.24989 172.21.15.26:0/503544817' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-23T20:44:33.850 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:44:33 smithi070 ceph-mon[100755]: from='mgr.24989 172.21.15.26:0/503544817' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-23T20:44:33.850 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:44:33 smithi070 ceph-mon[100755]: from='mgr.24989 172.21.15.26:0/503544817' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-23T20:44:33.850 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:44:33 smithi070 ceph-mon[100755]: from='mgr.24989 172.21.15.26:0/503544817' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-09-23T20:44:33.850 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:44:33 smithi070 ceph-mon[100755]: from='mgr.24989 172.21.15.26:0/503544817' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-09-23T20:44:33.850 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:44:33 smithi070 ceph-mon[100755]: from='mgr.24989 172.21.15.26:0/503544817' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-09-23T20:44:33.851 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:44:33 smithi070 ceph-mon[100755]: from='mgr.24989 172.21.15.26:0/503544817' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-09-23T20:44:33.851 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:44:33 smithi070 ceph-mon[100755]: from='mgr.24989 172.21.15.26:0/503544817' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-09-23T20:44:33.851 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:44:33 smithi070 ceph-mon[100755]: from='mgr.24989 172.21.15.26:0/503544817' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-09-23T20:44:33.851 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:44:33 smithi070 ceph-mon[100755]: from='mgr.24989 172.21.15.26:0/503544817' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-09-23T20:44:33.851 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:44:33 smithi070 ceph-mon[100755]: Manager daemon y is now available 2023-09-23T20:44:33.851 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:44:33 smithi070 ceph-mon[100755]: from='mgr.24989 172.21.15.26:0/503544817' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:44:33.851 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:44:33 smithi070 ceph-mon[100755]: from='mgr.24989 172.21.15.26:0/503544817' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:44:33.851 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:44:33 smithi070 ceph-mon[100755]: from='mgr.24989 172.21.15.26:0/503544817' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:44:33.852 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:44:33 smithi070 ceph-mon[100755]: from='mgr.24989 172.21.15.26:0/503544817' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-09-23T20:44:33.852 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:44:33 smithi070 ceph-mon[100755]: from='mgr.24989 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-09-23T20:44:33.852 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:44:33 smithi070 ceph-mon[100755]: from='mgr.24989 172.21.15.26:0/503544817' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-09-23T20:44:33.852 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:44:33 smithi070 ceph-mon[100755]: from='mgr.24989 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-09-23T20:44:33.882 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Sep 23 20:44:33 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-alertmanager-a[163035]: ts=2023-09-23T20:44:33.508Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi070.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.70:8443: connect: connection refused" 2023-09-23T20:44:33.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:44:33 smithi026 ceph-mon[109912]: mgrmap e26: x(active, since 41s), standbys: y 2023-09-23T20:44:33.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:44:33 smithi026 ceph-mon[109912]: pgmap v24: 73 pgs: 73 active+clean; 580 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:44:33.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:44:33 smithi026 ceph-mon[109912]: from='mgr.24781 ' entity='mgr.x' 2023-09-23T20:44:33.883 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:44:33 smithi026 ceph-mon[109912]: from='mgr.24781 ' entity='mgr.x' 2023-09-23T20:44:33.883 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:44:33 smithi026 ceph-mon[109912]: from='mgr.24781 172.21.15.70:0/208042876' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:44:33.883 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:44:33 smithi026 ceph-mon[109912]: from='mgr.24781 172.21.15.70:0/208042876' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:44:33.883 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:44:33 smithi026 ceph-mon[109912]: from='mgr.24781 ' entity='mgr.x' 2023-09-23T20:44:33.883 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:44:33 smithi026 ceph-mon[109912]: from='mgr.24781 172.21.15.70:0/208042876' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:44:33.883 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:44:33 smithi026 ceph-mon[109912]: from='mgr.24781 172.21.15.70:0/208042876' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2023-09-23T20:44:33.883 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:44:33 smithi026 ceph-mon[109912]: from='mgr.24781 ' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2023-09-23T20:44:33.883 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:44:33 smithi026 ceph-mon[109912]: osdmap e78: 8 total, 8 up, 8 in 2023-09-23T20:44:33.884 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:44:33 smithi026 ceph-mon[109912]: from='mgr.24781 ' entity='mgr.x' cmd='[{"prefix": "mgr fail", "who": "x"}]': finished 2023-09-23T20:44:33.884 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:44:33 smithi026 ceph-mon[109912]: mgrmap e27: y(active, starting, since 0.00556218s) 2023-09-23T20:44:33.884 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:44:33 smithi026 ceph-mon[109912]: from='mgr.24989 172.21.15.26:0/503544817' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-23T20:44:33.884 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:44:33 smithi026 ceph-mon[109912]: from='mgr.24989 172.21.15.26:0/503544817' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T20:44:33.884 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:44:33 smithi026 ceph-mon[109912]: from='mgr.24989 172.21.15.26:0/503544817' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T20:44:33.884 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:44:33 smithi026 ceph-mon[109912]: from='mgr.24989 172.21.15.26:0/503544817' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi026.znfpnl"}]: dispatch 2023-09-23T20:44:33.884 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:44:33 smithi026 ceph-mon[109912]: from='mgr.24989 172.21.15.26:0/503544817' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi070.imqhqy"}]: dispatch 2023-09-23T20:44:33.884 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:44:33 smithi026 ceph-mon[109912]: from='mgr.24989 172.21.15.26:0/503544817' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-09-23T20:44:33.884 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:44:33 smithi026 ceph-mon[109912]: from='mgr.24989 172.21.15.26:0/503544817' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-23T20:44:33.884 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:44:33 smithi026 ceph-mon[109912]: from='mgr.24989 172.21.15.26:0/503544817' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-23T20:44:33.885 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:44:33 smithi026 ceph-mon[109912]: from='mgr.24989 172.21.15.26:0/503544817' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-23T20:44:33.885 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:44:33 smithi026 ceph-mon[109912]: from='mgr.24989 172.21.15.26:0/503544817' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-23T20:44:33.885 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:44:33 smithi026 ceph-mon[109912]: from='mgr.24989 172.21.15.26:0/503544817' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-09-23T20:44:33.885 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:44:33 smithi026 ceph-mon[109912]: from='mgr.24989 172.21.15.26:0/503544817' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-09-23T20:44:33.885 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:44:33 smithi026 ceph-mon[109912]: from='mgr.24989 172.21.15.26:0/503544817' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-09-23T20:44:33.885 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:44:33 smithi026 ceph-mon[109912]: from='mgr.24989 172.21.15.26:0/503544817' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-09-23T20:44:33.885 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:44:33 smithi026 ceph-mon[109912]: from='mgr.24989 172.21.15.26:0/503544817' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-09-23T20:44:33.885 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:44:33 smithi026 ceph-mon[109912]: from='mgr.24989 172.21.15.26:0/503544817' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-09-23T20:44:33.886 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:44:33 smithi026 ceph-mon[95839]: mgrmap e26: x(active, since 41s), standbys: y 2023-09-23T20:44:33.886 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:44:33 smithi026 ceph-mon[95839]: pgmap v24: 73 pgs: 73 active+clean; 580 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:44:33.886 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:44:33 smithi026 ceph-mon[95839]: from='mgr.24781 ' entity='mgr.x' 2023-09-23T20:44:33.886 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:44:33 smithi026 ceph-mon[95839]: from='mgr.24781 ' entity='mgr.x' 2023-09-23T20:44:33.886 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:44:33 smithi026 ceph-mon[95839]: from='mgr.24781 172.21.15.70:0/208042876' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:44:33.887 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:44:33 smithi026 ceph-mon[95839]: from='mgr.24781 172.21.15.70:0/208042876' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:44:33.887 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:44:33 smithi026 ceph-mon[95839]: from='mgr.24781 ' entity='mgr.x' 2023-09-23T20:44:33.887 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:44:33 smithi026 ceph-mon[95839]: from='mgr.24781 172.21.15.70:0/208042876' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:44:33.887 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:44:33 smithi026 ceph-mon[95839]: from='mgr.24781 172.21.15.70:0/208042876' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2023-09-23T20:44:33.887 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:44:33 smithi026 ceph-mon[95839]: from='mgr.24781 ' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2023-09-23T20:44:33.887 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:44:33 smithi026 ceph-mon[95839]: osdmap e78: 8 total, 8 up, 8 in 2023-09-23T20:44:33.887 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:44:33 smithi026 ceph-mon[95839]: from='mgr.24781 ' entity='mgr.x' cmd='[{"prefix": "mgr fail", "who": "x"}]': finished 2023-09-23T20:44:33.887 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:44:33 smithi026 ceph-mon[95839]: mgrmap e27: y(active, starting, since 0.00556218s) 2023-09-23T20:44:33.887 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:44:33 smithi026 ceph-mon[95839]: from='mgr.24989 172.21.15.26:0/503544817' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-23T20:44:33.887 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:44:33 smithi026 ceph-mon[95839]: from='mgr.24989 172.21.15.26:0/503544817' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T20:44:33.888 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:44:33 smithi026 ceph-mon[95839]: from='mgr.24989 172.21.15.26:0/503544817' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T20:44:33.888 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:44:33 smithi026 ceph-mon[95839]: from='mgr.24989 172.21.15.26:0/503544817' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi026.znfpnl"}]: dispatch 2023-09-23T20:44:33.888 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:44:33 smithi026 ceph-mon[95839]: from='mgr.24989 172.21.15.26:0/503544817' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi070.imqhqy"}]: dispatch 2023-09-23T20:44:33.888 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:44:33 smithi026 ceph-mon[95839]: from='mgr.24989 172.21.15.26:0/503544817' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-09-23T20:44:33.888 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:44:33 smithi026 ceph-mon[95839]: from='mgr.24989 172.21.15.26:0/503544817' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-23T20:44:33.888 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:44:33 smithi026 ceph-mon[95839]: from='mgr.24989 172.21.15.26:0/503544817' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-23T20:44:33.888 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:44:33 smithi026 ceph-mon[95839]: from='mgr.24989 172.21.15.26:0/503544817' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-23T20:44:33.888 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:44:33 smithi026 ceph-mon[95839]: from='mgr.24989 172.21.15.26:0/503544817' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-23T20:44:33.888 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:44:33 smithi026 ceph-mon[95839]: from='mgr.24989 172.21.15.26:0/503544817' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-09-23T20:44:33.888 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:44:33 smithi026 ceph-mon[95839]: from='mgr.24989 172.21.15.26:0/503544817' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-09-23T20:44:33.888 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:44:33 smithi026 ceph-mon[95839]: from='mgr.24989 172.21.15.26:0/503544817' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-09-23T20:44:33.889 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:44:33 smithi026 ceph-mon[95839]: from='mgr.24989 172.21.15.26:0/503544817' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-09-23T20:44:33.889 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:44:33 smithi026 ceph-mon[95839]: from='mgr.24989 172.21.15.26:0/503544817' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-09-23T20:44:33.889 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:44:33 smithi026 ceph-mon[95839]: from='mgr.24989 172.21.15.26:0/503544817' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-09-23T20:44:33.889 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:44:33 smithi026 ceph-mon[95839]: from='mgr.24989 172.21.15.26:0/503544817' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-09-23T20:44:33.889 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:44:33 smithi026 ceph-mon[95839]: Manager daemon y is now available 2023-09-23T20:44:33.889 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:44:33 smithi026 ceph-mon[95839]: from='mgr.24989 172.21.15.26:0/503544817' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:44:33.889 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:44:33 smithi026 ceph-mon[95839]: from='mgr.24989 172.21.15.26:0/503544817' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:44:33.889 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:44:33 smithi026 ceph-mon[95839]: from='mgr.24989 172.21.15.26:0/503544817' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:44:33.889 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:44:33 smithi026 ceph-mon[95839]: from='mgr.24989 172.21.15.26:0/503544817' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-09-23T20:44:33.889 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:44:33 smithi026 ceph-mon[95839]: from='mgr.24989 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-09-23T20:44:33.889 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:44:33 smithi026 ceph-mon[95839]: from='mgr.24989 172.21.15.26:0/503544817' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-09-23T20:44:33.889 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:44:33 smithi026 ceph-mon[95839]: from='mgr.24989 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-09-23T20:44:33.890 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:44:33 smithi026 ceph-mon[109912]: from='mgr.24989 172.21.15.26:0/503544817' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-09-23T20:44:33.890 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:44:33 smithi026 ceph-mon[109912]: Manager daemon y is now available 2023-09-23T20:44:33.890 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:44:33 smithi026 ceph-mon[109912]: from='mgr.24989 172.21.15.26:0/503544817' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:44:33.890 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:44:33 smithi026 ceph-mon[109912]: from='mgr.24989 172.21.15.26:0/503544817' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:44:33.890 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:44:33 smithi026 ceph-mon[109912]: from='mgr.24989 172.21.15.26:0/503544817' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:44:33.890 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:44:33 smithi026 ceph-mon[109912]: from='mgr.24989 172.21.15.26:0/503544817' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-09-23T20:44:33.890 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:44:33 smithi026 ceph-mon[109912]: from='mgr.24989 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-09-23T20:44:33.890 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:44:33 smithi026 ceph-mon[109912]: from='mgr.24989 172.21.15.26:0/503544817' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-09-23T20:44:33.890 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:44:33 smithi026 ceph-mon[109912]: from='mgr.24989 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-09-23T20:44:34.119 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 20:44:33 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-x[141470]: 2023-09-23T20:44:33.845+0000 7f2fd6ca0200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-09-23T20:44:34.119 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 20:44:33 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-x[141470]: 2023-09-23T20:44:33.984+0000 7f2fd6ca0200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-09-23T20:44:34.119 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 20:44:34 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-x[141470]: 2023-09-23T20:44:34.077+0000 7f2fd6ca0200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-09-23T20:44:34.619 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 20:44:34 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-x[141470]: 2023-09-23T20:44:34.175+0000 7f2fd6ca0200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-09-23T20:44:35.119 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 20:44:34 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-x[141470]: 2023-09-23T20:44:34.621+0000 7f2fd6ca0200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-09-23T20:44:35.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:44:34 smithi070 ceph-mon[100755]: mgrmap e28: y(active, since 1.00824s) 2023-09-23T20:44:35.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:44:34 smithi070 ceph-mon[100755]: [23/Sep/2023:20:44:34] ENGINE Bus STARTING 2023-09-23T20:44:35.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:44:34 smithi070 ceph-mon[100755]: [23/Sep/2023:20:44:34] ENGINE Serving on https://172.21.15.26:7150 2023-09-23T20:44:35.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:44:34 smithi070 ceph-mon[100755]: [23/Sep/2023:20:44:34] ENGINE Serving on http://172.21.15.26:8765 2023-09-23T20:44:35.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:44:34 smithi070 ceph-mon[100755]: [23/Sep/2023:20:44:34] ENGINE Bus STARTED 2023-09-23T20:44:35.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:44:34 smithi070 ceph-mon[100755]: from='client.? 172.21.15.26:0/861638206' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:44:35.131 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:44:34 smithi026 ceph-mon[109912]: mgrmap e28: y(active, since 1.00824s) 2023-09-23T20:44:35.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:44:34 smithi026 ceph-mon[109912]: [23/Sep/2023:20:44:34] ENGINE Bus STARTING 2023-09-23T20:44:35.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:44:34 smithi026 ceph-mon[109912]: [23/Sep/2023:20:44:34] ENGINE Serving on https://172.21.15.26:7150 2023-09-23T20:44:35.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:44:34 smithi026 ceph-mon[109912]: [23/Sep/2023:20:44:34] ENGINE Serving on http://172.21.15.26:8765 2023-09-23T20:44:35.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:44:34 smithi026 ceph-mon[109912]: [23/Sep/2023:20:44:34] ENGINE Bus STARTED 2023-09-23T20:44:35.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:44:34 smithi026 ceph-mon[109912]: from='client.? 172.21.15.26:0/861638206' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:44:35.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:44:34 smithi026 ceph-mon[95839]: mgrmap e28: y(active, since 1.00824s) 2023-09-23T20:44:35.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:44:34 smithi026 ceph-mon[95839]: [23/Sep/2023:20:44:34] ENGINE Bus STARTING 2023-09-23T20:44:35.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:44:34 smithi026 ceph-mon[95839]: [23/Sep/2023:20:44:34] ENGINE Serving on https://172.21.15.26:7150 2023-09-23T20:44:35.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:44:34 smithi026 ceph-mon[95839]: [23/Sep/2023:20:44:34] ENGINE Serving on http://172.21.15.26:8765 2023-09-23T20:44:35.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:44:34 smithi026 ceph-mon[95839]: [23/Sep/2023:20:44:34] ENGINE Bus STARTED 2023-09-23T20:44:35.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:44:34 smithi026 ceph-mon[95839]: from='client.? 172.21.15.26:0/861638206' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:44:35.620 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 20:44:35 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-x[141470]: 2023-09-23T20:44:35.310+0000 7f2fd6ca0200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-09-23T20:44:35.621 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 20:44:35 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-x[141470]: 2023-09-23T20:44:35.395+0000 7f2fd6ca0200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-09-23T20:44:36.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:44:35 smithi070 ceph-mon[100755]: pgmap v4: 73 pgs: 73 active+clean; 581 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:44:36.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:44:35 smithi070 ceph-mon[100755]: mgrmap e29: y(active, since 2s) 2023-09-23T20:44:36.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:44:35 smithi026 ceph-mon[95839]: pgmap v4: 73 pgs: 73 active+clean; 581 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:44:36.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:44:35 smithi026 ceph-mon[95839]: mgrmap e29: y(active, since 2s) 2023-09-23T20:44:36.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:44:35 smithi026 ceph-mon[109912]: pgmap v4: 73 pgs: 73 active+clean; 581 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:44:36.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:44:35 smithi026 ceph-mon[109912]: mgrmap e29: y(active, since 2s) 2023-09-23T20:44:36.513 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 20:44:36 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-x[141470]: 2023-09-23T20:44:36.221+0000 7f2fd6ca0200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-09-23T20:44:36.513 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 20:44:36 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-x[141470]: 2023-09-23T20:44:36.313+0000 7f2fd6ca0200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2023-09-23T20:44:36.819 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 20:44:36 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-x[141470]: 2023-09-23T20:44:36.512+0000 7f2fd6ca0200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-09-23T20:44:37.119 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 20:44:36 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-x[141470]: 2023-09-23T20:44:36.818+0000 7f2fd6ca0200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2023-09-23T20:44:37.119 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 20:44:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-x[141470]: 2023-09-23T20:44:37.053+0000 7f2fd6ca0200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-09-23T20:44:37.493 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 20:44:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-x[141470]: 2023-09-23T20:44:37.210+0000 7f2fd6ca0200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-09-23T20:44:37.870 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 20:44:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-x[141470]: 2023-09-23T20:44:37.492+0000 7f2fd6ca0200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-09-23T20:44:38.258 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:44:37 smithi070 ceph-mon[100755]: pgmap v5: 73 pgs: 73 active+clean; 581 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:44:38.258 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:44:37 smithi070 ceph-mon[100755]: from='client.? 172.21.15.26:0/319357143' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:44:38.258 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:44:37 smithi070 ceph-mon[100755]: from='mgr.24989 ' entity='mgr.y' 2023-09-23T20:44:38.258 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:44:37 smithi070 ceph-mon[100755]: from='mgr.24989 ' entity='mgr.y' 2023-09-23T20:44:38.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:44:37 smithi026 ceph-mon[109912]: pgmap v5: 73 pgs: 73 active+clean; 581 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:44:38.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:44:37 smithi026 ceph-mon[109912]: from='client.? 172.21.15.26:0/319357143' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:44:38.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:44:37 smithi026 ceph-mon[109912]: from='mgr.24989 ' entity='mgr.y' 2023-09-23T20:44:38.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:44:37 smithi026 ceph-mon[109912]: from='mgr.24989 ' entity='mgr.y' 2023-09-23T20:44:38.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:44:37 smithi026 ceph-mon[95839]: pgmap v5: 73 pgs: 73 active+clean; 581 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:44:38.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:44:37 smithi026 ceph-mon[95839]: from='client.? 172.21.15.26:0/319357143' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:44:38.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:44:37 smithi026 ceph-mon[95839]: from='mgr.24989 ' entity='mgr.y' 2023-09-23T20:44:38.383 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:44:37 smithi026 ceph-mon[95839]: from='mgr.24989 ' entity='mgr.y' 2023-09-23T20:44:38.605 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 20:44:38 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-x[141470]: 2023-09-23T20:44:38.257+0000 7f2fd6ca0200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-09-23T20:44:38.605 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 20:44:38 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-x[141470]: 2023-09-23T20:44:38.398+0000 7f2fd6ca0200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-09-23T20:44:38.869 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 20:44:38 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-x[141470]: 2023-09-23T20:44:38.604+0000 7f2fd6ca0200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-09-23T20:44:38.869 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 20:44:38 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-x[141470]: 2023-09-23T20:44:38.729+0000 7f2fd6ca0200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-09-23T20:44:39.369 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 20:44:38 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-x[141470]: 2023-09-23T20:44:38.964+0000 7f2fd6ca0200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-09-23T20:44:39.633 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:44:39 smithi026 ceph-mon[95839]: from='mgr.24989 ' entity='mgr.y' 2023-09-23T20:44:39.633 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:44:39 smithi026 ceph-mon[95839]: from='mgr.24989 ' entity='mgr.y' 2023-09-23T20:44:39.634 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:44:39 smithi026 ceph-mon[95839]: from='mgr.24989 ' entity='mgr.y' 2023-09-23T20:44:39.634 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:44:39 smithi026 ceph-mon[95839]: from='mgr.24989 ' entity='mgr.y' 2023-09-23T20:44:39.634 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:44:39 smithi026 ceph-mon[95839]: from='mgr.24989 ' entity='mgr.y' 2023-09-23T20:44:39.634 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:44:39 smithi026 ceph-mon[95839]: from='client.? 172.21.15.26:0/4294473115' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:44:39.634 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:44:39 smithi026 ceph-mon[95839]: from='mgr.24989 ' entity='mgr.y' 2023-09-23T20:44:39.634 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:44:39 smithi026 ceph-mon[109912]: from='mgr.24989 ' entity='mgr.y' 2023-09-23T20:44:39.634 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:44:39 smithi026 ceph-mon[109912]: from='mgr.24989 ' entity='mgr.y' 2023-09-23T20:44:39.635 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:44:39 smithi026 ceph-mon[109912]: from='mgr.24989 ' entity='mgr.y' 2023-09-23T20:44:39.635 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:44:39 smithi026 ceph-mon[109912]: from='mgr.24989 ' entity='mgr.y' 2023-09-23T20:44:39.635 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:44:39 smithi026 ceph-mon[109912]: from='mgr.24989 ' entity='mgr.y' 2023-09-23T20:44:39.635 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:44:39 smithi026 ceph-mon[109912]: from='client.? 172.21.15.26:0/4294473115' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:44:39.635 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:44:39 smithi026 ceph-mon[109912]: from='mgr.24989 ' entity='mgr.y' 2023-09-23T20:44:39.850 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:44:39 smithi070 ceph-mon[100755]: from='mgr.24989 ' entity='mgr.y' 2023-09-23T20:44:39.850 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:44:39 smithi070 ceph-mon[100755]: from='mgr.24989 ' entity='mgr.y' 2023-09-23T20:44:39.850 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:44:39 smithi070 ceph-mon[100755]: from='mgr.24989 ' entity='mgr.y' 2023-09-23T20:44:39.850 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:44:39 smithi070 ceph-mon[100755]: from='mgr.24989 ' entity='mgr.y' 2023-09-23T20:44:39.850 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:44:39 smithi070 ceph-mon[100755]: from='mgr.24989 ' entity='mgr.y' 2023-09-23T20:44:39.850 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:44:39 smithi070 ceph-mon[100755]: from='client.? 172.21.15.26:0/4294473115' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:44:39.850 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:44:39 smithi070 ceph-mon[100755]: from='mgr.24989 ' entity='mgr.y' 2023-09-23T20:44:40.119 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 20:44:39 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-x[141470]: 2023-09-23T20:44:39.848+0000 7f2fd6ca0200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-09-23T20:44:40.120 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 20:44:39 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-x[141470]: [23/Sep/2023:20:44:39] ENGINE Bus STARTING 2023-09-23T20:44:40.120 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 20:44:39 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-x[141470]: CherryPy Checker: 2023-09-23T20:44:40.120 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 20:44:39 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-x[141470]: The Application mounted at '' has an empty config. 2023-09-23T20:44:40.120 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 20:44:39 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-x[141470]: 2023-09-23T20:44:40.120 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 20:44:39 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-x[141470]: [23/Sep/2023:20:44:39] ENGINE Serving on http://:::9283 2023-09-23T20:44:40.121 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 20:44:39 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-x[141470]: [23/Sep/2023:20:44:39] ENGINE Bus STARTED 2023-09-23T20:44:40.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:44:40 smithi070 ceph-mon[100755]: pgmap v6: 73 pgs: 73 active+clean; 581 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:44:40.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:44:40 smithi070 ceph-mon[100755]: Standby manager daemon x started 2023-09-23T20:44:40.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:44:40 smithi070 ceph-mon[100755]: from='mgr.? 172.21.15.70:0/753378707' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-09-23T20:44:40.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:44:40 smithi070 ceph-mon[100755]: from='mgr.? 172.21.15.70:0/753378707' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-09-23T20:44:40.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:44:40 smithi070 ceph-mon[100755]: from='mgr.? 172.21.15.70:0/753378707' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-09-23T20:44:40.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:44:40 smithi070 ceph-mon[100755]: from='mgr.? 172.21.15.70:0/753378707' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-09-23T20:44:40.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:44:40 smithi026 ceph-mon[109912]: pgmap v6: 73 pgs: 73 active+clean; 581 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:44:40.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:44:40 smithi026 ceph-mon[109912]: Standby manager daemon x started 2023-09-23T20:44:40.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:44:40 smithi026 ceph-mon[109912]: from='mgr.? 172.21.15.70:0/753378707' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-09-23T20:44:40.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:44:40 smithi026 ceph-mon[109912]: from='mgr.? 172.21.15.70:0/753378707' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-09-23T20:44:40.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:44:40 smithi026 ceph-mon[109912]: from='mgr.? 172.21.15.70:0/753378707' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-09-23T20:44:40.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:44:40 smithi026 ceph-mon[109912]: from='mgr.? 172.21.15.70:0/753378707' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-09-23T20:44:40.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:44:40 smithi026 ceph-mon[95839]: pgmap v6: 73 pgs: 73 active+clean; 581 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:44:40.883 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:44:40 smithi026 ceph-mon[95839]: Standby manager daemon x started 2023-09-23T20:44:40.883 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:44:40 smithi026 ceph-mon[95839]: from='mgr.? 172.21.15.70:0/753378707' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-09-23T20:44:40.883 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:44:40 smithi026 ceph-mon[95839]: from='mgr.? 172.21.15.70:0/753378707' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-09-23T20:44:40.883 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:44:40 smithi026 ceph-mon[95839]: from='mgr.? 172.21.15.70:0/753378707' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-09-23T20:44:40.883 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:44:40 smithi026 ceph-mon[95839]: from='mgr.? 172.21.15.70:0/753378707' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-09-23T20:44:41.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:44:41 smithi070 ceph-mon[100755]: mgrmap e30: y(active, since 7s), standbys: x 2023-09-23T20:44:41.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:44:41 smithi070 ceph-mon[100755]: from='mgr.24989 172.21.15.26:0/503544817' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-09-23T20:44:41.881 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:44:41 smithi026 ceph-mon[109912]: mgrmap e30: y(active, since 7s), standbys: x 2023-09-23T20:44:41.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:44:41 smithi026 ceph-mon[109912]: from='mgr.24989 172.21.15.26:0/503544817' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-09-23T20:44:41.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:44:41 smithi026 ceph-mon[95839]: mgrmap e30: y(active, since 7s), standbys: x 2023-09-23T20:44:41.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:44:41 smithi026 ceph-mon[95839]: from='mgr.24989 172.21.15.26:0/503544817' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-09-23T20:44:42.741 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Sep 23 20:44:42 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-alertmanager-a[163035]: ts=2023-09-23T20:44:42.474Z caller=notify.go:743 level=info component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify success" attempts=7 2023-09-23T20:44:42.742 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:44:42 smithi026 ceph-mon[95839]: pgmap v7: 73 pgs: 73 active+clean; 581 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:44:42.743 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:44:42 smithi026 ceph-mon[95839]: from='mgr.24989 ' entity='mgr.y' 2023-09-23T20:44:42.743 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:44:42 smithi026 ceph-mon[95839]: from='mgr.24989 ' entity='mgr.y' 2023-09-23T20:44:42.743 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:44:42 smithi026 ceph-mon[95839]: from='mgr.24989 172.21.15.26:0/503544817' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:44:42.743 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:44:42 smithi026 ceph-mon[95839]: from='mgr.24989 172.21.15.26:0/503544817' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:44:42.743 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:44:42 smithi026 ceph-mon[95839]: Updating smithi026:/etc/ceph/ceph.conf 2023-09-23T20:44:42.743 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:44:42 smithi026 ceph-mon[95839]: Updating smithi070:/etc/ceph/ceph.conf 2023-09-23T20:44:42.743 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:44:42 smithi026 ceph-mon[95839]: from='client.? 172.21.15.26:0/1927071392' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:44:43.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:44:42 smithi070 ceph-mon[100755]: pgmap v7: 73 pgs: 73 active+clean; 581 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:44:43.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:44:42 smithi070 ceph-mon[100755]: from='mgr.24989 ' entity='mgr.y' 2023-09-23T20:44:43.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:44:42 smithi070 ceph-mon[100755]: from='mgr.24989 ' entity='mgr.y' 2023-09-23T20:44:43.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:44:42 smithi070 ceph-mon[100755]: from='mgr.24989 172.21.15.26:0/503544817' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:44:43.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:44:42 smithi070 ceph-mon[100755]: from='mgr.24989 172.21.15.26:0/503544817' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:44:43.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:44:42 smithi070 ceph-mon[100755]: Updating smithi026:/etc/ceph/ceph.conf 2023-09-23T20:44:43.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:44:42 smithi070 ceph-mon[100755]: Updating smithi070:/etc/ceph/ceph.conf 2023-09-23T20:44:43.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:44:42 smithi070 ceph-mon[100755]: from='client.? 172.21.15.26:0/1927071392' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:44:43.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:44:42 smithi026 ceph-mon[109912]: pgmap v7: 73 pgs: 73 active+clean; 581 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:44:43.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:44:42 smithi026 ceph-mon[109912]: from='mgr.24989 ' entity='mgr.y' 2023-09-23T20:44:43.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:44:42 smithi026 ceph-mon[109912]: from='mgr.24989 ' entity='mgr.y' 2023-09-23T20:44:43.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:44:42 smithi026 ceph-mon[109912]: from='mgr.24989 172.21.15.26:0/503544817' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:44:43.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:44:42 smithi026 ceph-mon[109912]: from='mgr.24989 172.21.15.26:0/503544817' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:44:43.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:44:42 smithi026 ceph-mon[109912]: Updating smithi026:/etc/ceph/ceph.conf 2023-09-23T20:44:43.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:44:42 smithi026 ceph-mon[109912]: Updating smithi070:/etc/ceph/ceph.conf 2023-09-23T20:44:43.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:44:42 smithi026 ceph-mon[109912]: from='client.? 172.21.15.26:0/1927071392' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:44:43.619 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 20:44:43 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-x[141470]: ::ffff:172.21.15.70 - - [23/Sep/2023:20:44:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2023-09-23T20:44:44.099 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:44:43 smithi070 ceph-mon[100755]: Updating smithi070:/var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-23T20:44:44.100 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:44:43 smithi070 ceph-mon[100755]: Updating smithi026:/var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-23T20:44:44.100 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:44:43 smithi070 ceph-mon[100755]: Updating smithi070:/etc/ceph/ceph.client.admin.keyring 2023-09-23T20:44:44.100 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:44:43 smithi070 ceph-mon[100755]: Updating smithi026:/etc/ceph/ceph.client.admin.keyring 2023-09-23T20:44:44.100 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:44:43 smithi070 ceph-mon[100755]: Updating smithi026:/var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/config/ceph.client.admin.keyring 2023-09-23T20:44:44.100 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:44:43 smithi070 ceph-mon[100755]: Updating smithi070:/var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/config/ceph.client.admin.keyring 2023-09-23T20:44:44.100 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:44:43 smithi070 ceph-mon[100755]: from='mgr.24989 ' entity='mgr.y' 2023-09-23T20:44:44.100 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:44:43 smithi070 ceph-mon[100755]: from='mgr.24989 ' entity='mgr.y' 2023-09-23T20:44:44.100 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:44:43 smithi070 ceph-mon[100755]: from='mgr.24989 ' entity='mgr.y' 2023-09-23T20:44:44.101 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:44:43 smithi070 ceph-mon[100755]: from='mgr.24989 ' entity='mgr.y' 2023-09-23T20:44:44.101 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:44:43 smithi070 ceph-mon[100755]: from='mgr.24989 ' entity='mgr.y' 2023-09-23T20:44:44.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:44:43 smithi026 ceph-mon[109912]: Updating smithi070:/var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-23T20:44:44.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:44:43 smithi026 ceph-mon[109912]: Updating smithi026:/var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-23T20:44:44.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:44:43 smithi026 ceph-mon[109912]: Updating smithi070:/etc/ceph/ceph.client.admin.keyring 2023-09-23T20:44:44.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:44:43 smithi026 ceph-mon[109912]: Updating smithi026:/etc/ceph/ceph.client.admin.keyring 2023-09-23T20:44:44.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:44:43 smithi026 ceph-mon[109912]: Updating smithi026:/var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/config/ceph.client.admin.keyring 2023-09-23T20:44:44.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:44:43 smithi026 ceph-mon[109912]: Updating smithi070:/var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/config/ceph.client.admin.keyring 2023-09-23T20:44:44.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:44:43 smithi026 ceph-mon[109912]: from='mgr.24989 ' entity='mgr.y' 2023-09-23T20:44:44.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:44:43 smithi026 ceph-mon[109912]: from='mgr.24989 ' entity='mgr.y' 2023-09-23T20:44:44.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:44:43 smithi026 ceph-mon[109912]: from='mgr.24989 ' entity='mgr.y' 2023-09-23T20:44:44.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:44:43 smithi026 ceph-mon[109912]: from='mgr.24989 ' entity='mgr.y' 2023-09-23T20:44:44.134 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:44:43 smithi026 ceph-mon[109912]: from='mgr.24989 ' entity='mgr.y' 2023-09-23T20:44:44.134 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:44:43 smithi026 ceph-mon[95839]: Updating smithi070:/var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-23T20:44:44.134 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:44:43 smithi026 ceph-mon[95839]: Updating smithi026:/var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-23T20:44:44.134 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:44:43 smithi026 ceph-mon[95839]: Updating smithi070:/etc/ceph/ceph.client.admin.keyring 2023-09-23T20:44:44.134 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:44:43 smithi026 ceph-mon[95839]: Updating smithi026:/etc/ceph/ceph.client.admin.keyring 2023-09-23T20:44:44.134 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:44:43 smithi026 ceph-mon[95839]: Updating smithi026:/var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/config/ceph.client.admin.keyring 2023-09-23T20:44:44.134 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:44:43 smithi026 ceph-mon[95839]: Updating smithi070:/var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/config/ceph.client.admin.keyring 2023-09-23T20:44:44.134 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:44:43 smithi026 ceph-mon[95839]: from='mgr.24989 ' entity='mgr.y' 2023-09-23T20:44:44.135 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:44:43 smithi026 ceph-mon[95839]: from='mgr.24989 ' entity='mgr.y' 2023-09-23T20:44:44.135 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:44:43 smithi026 ceph-mon[95839]: from='mgr.24989 ' entity='mgr.y' 2023-09-23T20:44:44.135 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:44:43 smithi026 ceph-mon[95839]: from='mgr.24989 ' entity='mgr.y' 2023-09-23T20:44:44.135 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:44:43 smithi026 ceph-mon[95839]: from='mgr.24989 ' entity='mgr.y' 2023-09-23T20:44:44.967 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:44:44 smithi070 ceph-mon[100755]: pgmap v8: 73 pgs: 73 active+clean; 581 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:44:44.967 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:44:44 smithi070 ceph-mon[100755]: Reconfiguring prometheus.a (dependencies changed)... 2023-09-23T20:44:44.967 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:44:44 smithi070 ceph-mon[100755]: Reconfiguring daemon prometheus.a on smithi070 2023-09-23T20:44:44.967 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:44:44 smithi070 ceph-mon[100755]: from='client.? 172.21.15.26:0/1900613323' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:44:45.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:44:44 smithi026 ceph-mon[95839]: pgmap v8: 73 pgs: 73 active+clean; 581 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:44:45.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:44:44 smithi026 ceph-mon[95839]: Reconfiguring prometheus.a (dependencies changed)... 2023-09-23T20:44:45.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:44:44 smithi026 ceph-mon[95839]: Reconfiguring daemon prometheus.a on smithi070 2023-09-23T20:44:45.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:44:44 smithi026 ceph-mon[95839]: from='client.? 172.21.15.26:0/1900613323' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:44:45.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:44:44 smithi026 ceph-mon[109912]: pgmap v8: 73 pgs: 73 active+clean; 581 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:44:45.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:44:44 smithi026 ceph-mon[109912]: Reconfiguring prometheus.a (dependencies changed)... 2023-09-23T20:44:45.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:44:44 smithi026 ceph-mon[109912]: Reconfiguring daemon prometheus.a on smithi070 2023-09-23T20:44:45.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:44:44 smithi026 ceph-mon[109912]: from='client.? 172.21.15.26:0/1900613323' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:44:45.619 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 20:44:45 smithi070 systemd[1]: Stopping Ceph prometheus.a for b3f8e4ec-5a50-11ee-8db4-212e2dc638e7... 2023-09-23T20:44:45.904 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 20:44:45 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-prometheus-a[152415]: ts=2023-09-23T20:44:45.653Z caller=main.go:840 level=warn msg="Received SIGTERM, exiting gracefully..." 2023-09-23T20:44:45.904 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 20:44:45 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-prometheus-a[152415]: ts=2023-09-23T20:44:45.653Z caller=main.go:864 level=info msg="Stopping scrape discovery manager..." 2023-09-23T20:44:45.904 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 20:44:45 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-prometheus-a[152415]: ts=2023-09-23T20:44:45.653Z caller=main.go:878 level=info msg="Stopping notify discovery manager..." 2023-09-23T20:44:45.905 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 20:44:45 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-prometheus-a[152415]: ts=2023-09-23T20:44:45.653Z caller=manager.go:988 level=info component="rule manager" msg="Stopping rule manager..." 2023-09-23T20:44:45.905 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 20:44:45 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-prometheus-a[152415]: ts=2023-09-23T20:44:45.653Z caller=main.go:860 level=info msg="Scrape discovery manager stopped" 2023-09-23T20:44:45.905 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 20:44:45 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-prometheus-a[152415]: ts=2023-09-23T20:44:45.653Z caller=main.go:874 level=info msg="Notify discovery manager stopped" 2023-09-23T20:44:45.905 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 20:44:45 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-prometheus-a[152415]: ts=2023-09-23T20:44:45.653Z caller=manager.go:998 level=info component="rule manager" msg="Rule manager stopped" 2023-09-23T20:44:45.905 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 20:44:45 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-prometheus-a[152415]: ts=2023-09-23T20:44:45.653Z caller=main.go:915 level=info msg="Stopping scrape manager..." 2023-09-23T20:44:45.905 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 20:44:45 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-prometheus-a[152415]: ts=2023-09-23T20:44:45.653Z caller=main.go:907 level=info msg="Scrape manager stopped" 2023-09-23T20:44:45.905 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 20:44:45 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-prometheus-a[152415]: ts=2023-09-23T20:44:45.653Z caller=notifier.go:606 level=info component=notifier msg="Stopping notification manager..." 2023-09-23T20:44:45.905 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 20:44:45 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-prometheus-a[152415]: ts=2023-09-23T20:44:45.653Z caller=main.go:1135 level=info msg="Notifier manager stopped" 2023-09-23T20:44:45.906 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 20:44:45 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-prometheus-a[152415]: ts=2023-09-23T20:44:45.653Z caller=main.go:1147 level=info msg="See you next time!" 2023-09-23T20:44:45.906 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 20:44:45 smithi070 bash[155838]: ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-prometheus-a 2023-09-23T20:44:46.368 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 20:44:46 smithi070 systemd[1]: ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7@prometheus.a.service: Succeeded. 2023-09-23T20:44:46.369 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 20:44:46 smithi070 systemd[1]: Stopped Ceph prometheus.a for b3f8e4ec-5a50-11ee-8db4-212e2dc638e7. 2023-09-23T20:44:46.369 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 20:44:46 smithi070 systemd[1]: Starting Ceph prometheus.a for b3f8e4ec-5a50-11ee-8db4-212e2dc638e7... 2023-09-23T20:44:46.622 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 20:44:46 smithi070 podman[155959]: 2023-09-23T20:44:47.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:44:46 smithi070 ceph-mon[100755]: pgmap v9: 73 pgs: 73 active+clean; 581 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:44:47.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:44:46 smithi070 ceph-mon[100755]: from='client.? 172.21.15.26:0/2125900448' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:44:47.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:44:46 smithi070 ceph-mon[100755]: from='mgr.24989 ' entity='mgr.y' 2023-09-23T20:44:47.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:44:46 smithi070 ceph-mon[100755]: from='mgr.24989 ' entity='mgr.y' 2023-09-23T20:44:47.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:44:46 smithi070 ceph-mon[100755]: from='mgr.24989 172.21.15.26:0/503544817' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-09-23T20:44:47.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:44:46 smithi070 ceph-mon[100755]: from='mgr.24989 172.21.15.26:0/503544817' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:44:47.120 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 20:44:46 smithi070 bash[155959]: dda227f9352403e4744b3468bfbf4c9108cab28e4f16e9edff87ce9d8e8eb74b 2023-09-23T20:44:47.120 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 20:44:46 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-prometheus-a[155975]: ts=2023-09-23T20:44:46.630Z caller=main.go:564 level=info msg="Starting Prometheus Server" mode=server version="(version=2.43.0, branch=HEAD, revision=edfc3bcd025dd6fe296c167a14a216cab1e552ee)" 2023-09-23T20:44:47.121 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 20:44:46 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-prometheus-a[155975]: ts=2023-09-23T20:44:46.630Z caller=main.go:569 level=info build_context="(go=go1.19.7, platform=linux/amd64, user=root@8a0ee342e522, date=20230321-12:56:07, tags=netgo,builtinassets)" 2023-09-23T20:44:47.121 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 20:44:46 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-prometheus-a[155975]: ts=2023-09-23T20:44:46.630Z caller=main.go:570 level=info host_details="(Linux 4.18.0-372.9.1.el8.x86_64 #1 SMP Fri Apr 15 22:12:19 EDT 2022 x86_64 smithi070 (none))" 2023-09-23T20:44:47.121 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 20:44:46 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-prometheus-a[155975]: ts=2023-09-23T20:44:46.630Z caller=main.go:571 level=info fd_limits="(soft=1048576, hard=1048576)" 2023-09-23T20:44:47.121 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 20:44:46 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-prometheus-a[155975]: ts=2023-09-23T20:44:46.630Z caller=main.go:572 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2023-09-23T20:44:47.121 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 20:44:46 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-prometheus-a[155975]: ts=2023-09-23T20:44:46.632Z caller=web.go:561 level=info component=web msg="Start listening for connections" address=:9095 2023-09-23T20:44:47.121 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 20:44:46 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-prometheus-a[155975]: ts=2023-09-23T20:44:46.632Z caller=main.go:1005 level=info msg="Starting TSDB ..." 2023-09-23T20:44:47.121 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 20:44:46 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-prometheus-a[155975]: ts=2023-09-23T20:44:46.633Z caller=tls_config.go:232 level=info component=web msg="Listening on" address=[::]:9095 2023-09-23T20:44:47.121 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 20:44:46 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-prometheus-a[155975]: ts=2023-09-23T20:44:46.633Z caller=tls_config.go:235 level=info component=web msg="TLS is disabled." http2=false address=[::]:9095 2023-09-23T20:44:47.121 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 20:44:46 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-prometheus-a[155975]: ts=2023-09-23T20:44:46.634Z caller=head.go:587 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2023-09-23T20:44:47.121 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 20:44:46 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-prometheus-a[155975]: ts=2023-09-23T20:44:46.634Z caller=head.go:658 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=1.598µs 2023-09-23T20:44:47.121 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 20:44:46 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-prometheus-a[155975]: ts=2023-09-23T20:44:46.634Z caller=head.go:664 level=info component=tsdb msg="Replaying WAL, this may take a while" 2023-09-23T20:44:47.122 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 20:44:46 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-prometheus-a[155975]: ts=2023-09-23T20:44:46.635Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=3 2023-09-23T20:44:47.122 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 20:44:46 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-prometheus-a[155975]: ts=2023-09-23T20:44:46.649Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=1 maxSegment=3 2023-09-23T20:44:47.122 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 20:44:46 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-prometheus-a[155975]: ts=2023-09-23T20:44:46.650Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=2 maxSegment=3 2023-09-23T20:44:47.122 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 20:44:46 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-prometheus-a[155975]: ts=2023-09-23T20:44:46.651Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=3 maxSegment=3 2023-09-23T20:44:47.122 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 20:44:46 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-prometheus-a[155975]: ts=2023-09-23T20:44:46.651Z caller=head.go:772 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=31.285µs wal_replay_duration=16.325825ms wbl_replay_duration=109ns total_replay_duration=16.378342ms 2023-09-23T20:44:47.122 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 20:44:46 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-prometheus-a[155975]: ts=2023-09-23T20:44:46.654Z caller=main.go:1026 level=info fs_type=XFS_SUPER_MAGIC 2023-09-23T20:44:47.122 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 20:44:46 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-prometheus-a[155975]: ts=2023-09-23T20:44:46.654Z caller=main.go:1029 level=info msg="TSDB started" 2023-09-23T20:44:47.122 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 20:44:46 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-prometheus-a[155975]: ts=2023-09-23T20:44:46.654Z caller=main.go:1209 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2023-09-23T20:44:47.122 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 20:44:46 smithi070 systemd[1]: Started Ceph prometheus.a for b3f8e4ec-5a50-11ee-8db4-212e2dc638e7. 2023-09-23T20:44:47.122 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 20:44:46 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-prometheus-a[155975]: ts=2023-09-23T20:44:46.669Z caller=main.go:1246 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=15.787624ms db_storage=826ns remote_storage=1.113µs web_handler=441ns query_engine=668ns scrape=261.226µs scrape_sd=84.834µs notify=32.903µs notify_sd=17.044µs rules=15.164452ms tracing=7.187µs 2023-09-23T20:44:47.123 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 20:44:46 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-prometheus-a[155975]: ts=2023-09-23T20:44:46.669Z caller=main.go:990 level=info msg="Server is ready to receive web requests." 2023-09-23T20:44:47.123 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 20:44:46 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-prometheus-a[155975]: ts=2023-09-23T20:44:46.669Z caller=manager.go:974 level=info component="rule manager" msg="Starting rule manager..." 2023-09-23T20:44:47.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:44:46 smithi026 ceph-mon[109912]: pgmap v9: 73 pgs: 73 active+clean; 581 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:44:47.134 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:44:46 smithi026 ceph-mon[109912]: from='client.? 172.21.15.26:0/2125900448' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:44:47.134 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:44:46 smithi026 ceph-mon[109912]: from='mgr.24989 ' entity='mgr.y' 2023-09-23T20:44:47.134 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:44:46 smithi026 ceph-mon[109912]: from='mgr.24989 ' entity='mgr.y' 2023-09-23T20:44:47.134 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:44:46 smithi026 ceph-mon[109912]: from='mgr.24989 172.21.15.26:0/503544817' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-09-23T20:44:47.134 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:44:46 smithi026 ceph-mon[109912]: from='mgr.24989 172.21.15.26:0/503544817' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:44:47.134 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:44:46 smithi026 ceph-mon[95839]: pgmap v9: 73 pgs: 73 active+clean; 581 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:44:47.134 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:44:46 smithi026 ceph-mon[95839]: from='client.? 172.21.15.26:0/2125900448' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:44:47.135 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:44:46 smithi026 ceph-mon[95839]: from='mgr.24989 ' entity='mgr.y' 2023-09-23T20:44:47.135 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:44:46 smithi026 ceph-mon[95839]: from='mgr.24989 ' entity='mgr.y' 2023-09-23T20:44:47.135 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:44:46 smithi026 ceph-mon[95839]: from='mgr.24989 172.21.15.26:0/503544817' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-09-23T20:44:47.135 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:44:46 smithi026 ceph-mon[95839]: from='mgr.24989 172.21.15.26:0/503544817' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:44:47.995 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:44:47 smithi070 ceph-mon[100755]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-09-23T20:44:48.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:44:47 smithi026 ceph-mon[95839]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-09-23T20:44:48.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:44:47 smithi026 ceph-mon[109912]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-09-23T20:44:48.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:44:48 smithi070 ceph-mon[100755]: pgmap v10: 73 pgs: 73 active+clean; 581 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:44:48.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:44:48 smithi070 ceph-mon[100755]: from='mgr.24989 172.21.15.26:0/503544817' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:44:48.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:44:48 smithi070 ceph-mon[100755]: from='mgr.24989 172.21.15.26:0/503544817' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-09-23T20:44:48.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:44:48 smithi070 ceph-mon[100755]: from='mgr.24989 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-09-23T20:44:48.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:44:48 smithi070 ceph-mon[100755]: from='mgr.24989 172.21.15.26:0/503544817' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-09-23T20:44:48.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:44:48 smithi070 ceph-mon[100755]: from='mgr.24989 172.21.15.26:0/503544817' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:44:49.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:44:48 smithi026 ceph-mon[95839]: pgmap v10: 73 pgs: 73 active+clean; 581 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:44:49.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:44:48 smithi026 ceph-mon[95839]: from='mgr.24989 172.21.15.26:0/503544817' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:44:49.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:44:48 smithi026 ceph-mon[95839]: from='mgr.24989 172.21.15.26:0/503544817' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-09-23T20:44:49.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:44:48 smithi026 ceph-mon[95839]: from='mgr.24989 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-09-23T20:44:49.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:44:48 smithi026 ceph-mon[95839]: from='mgr.24989 172.21.15.26:0/503544817' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-09-23T20:44:49.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:44:48 smithi026 ceph-mon[95839]: from='mgr.24989 172.21.15.26:0/503544817' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:44:49.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:44:48 smithi026 ceph-mon[109912]: pgmap v10: 73 pgs: 73 active+clean; 581 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:44:49.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:44:48 smithi026 ceph-mon[109912]: from='mgr.24989 172.21.15.26:0/503544817' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:44:49.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:44:48 smithi026 ceph-mon[109912]: from='mgr.24989 172.21.15.26:0/503544817' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-09-23T20:44:49.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:44:48 smithi026 ceph-mon[109912]: from='mgr.24989 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-09-23T20:44:49.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:44:48 smithi026 ceph-mon[109912]: from='mgr.24989 172.21.15.26:0/503544817' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-09-23T20:44:49.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:44:48 smithi026 ceph-mon[109912]: from='mgr.24989 172.21.15.26:0/503544817' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:44:49.620 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 20:44:49 smithi070 systemd[1]: Stopping Ceph mgr.x for b3f8e4ec-5a50-11ee-8db4-212e2dc638e7... 2023-09-23T20:44:49.964 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 20:44:49 smithi070 bash[156370]: ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-x 2023-09-23T20:44:49.964 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:44:49 smithi070 ceph-mon[100755]: Upgrade: Updating mgr.x 2023-09-23T20:44:49.964 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:44:49 smithi070 ceph-mon[100755]: Deploying daemon mgr.x on smithi070 2023-09-23T20:44:49.965 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:44:49 smithi070 ceph-mon[100755]: from='client.? 172.21.15.26:0/3982039333' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:44:50.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:44:49 smithi026 ceph-mon[109912]: Upgrade: Updating mgr.x 2023-09-23T20:44:50.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:44:49 smithi026 ceph-mon[109912]: Deploying daemon mgr.x on smithi070 2023-09-23T20:44:50.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:44:49 smithi026 ceph-mon[109912]: from='client.? 172.21.15.26:0/3982039333' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:44:50.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:44:49 smithi026 ceph-mon[95839]: Upgrade: Updating mgr.x 2023-09-23T20:44:50.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:44:49 smithi026 ceph-mon[95839]: Deploying daemon mgr.x on smithi070 2023-09-23T20:44:50.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:44:49 smithi026 ceph-mon[95839]: from='client.? 172.21.15.26:0/3982039333' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:44:50.264 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 20:44:50 smithi070 systemd[1]: ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7@mgr.x.service: Main process exited, code=exited, status=143/n/a 2023-09-23T20:44:50.265 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 20:44:50 smithi070 systemd[1]: ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7@mgr.x.service: Failed with result 'exit-code'. 2023-09-23T20:44:50.265 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 20:44:50 smithi070 systemd[1]: Stopped Ceph mgr.x for b3f8e4ec-5a50-11ee-8db4-212e2dc638e7. 2023-09-23T20:44:50.556 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 20:44:50 smithi070 systemd[1]: Starting Ceph mgr.x for b3f8e4ec-5a50-11ee-8db4-212e2dc638e7... 2023-09-23T20:44:50.829 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:44:50 smithi070 ceph-mon[100755]: pgmap v11: 73 pgs: 73 active+clean; 581 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:44:50.829 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 20:44:50 smithi070 podman[156518]: 2023-09-23T20:44:51.120 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 20:44:50 smithi070 bash[156518]: 35059b639121397917545fb2cb771c8e1bac01e98bd913b967a8918e4f446a44 2023-09-23T20:44:51.120 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 20:44:50 smithi070 ceph-mgr[156558]: -- 172.21.15.70:0/1867089431 <== mon.1 v2:172.21.15.70:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x557c3f0471e0 con 0x557c3f01cc00 2023-09-23T20:44:51.120 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 20:44:50 smithi070 systemd[1]: Started Ceph mgr.x for b3f8e4ec-5a50-11ee-8db4-212e2dc638e7. 2023-09-23T20:44:51.121 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 20:44:50 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-x[156534]: 2023-09-23T20:44:50.952+0000 7f35bb8ba200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2023-09-23T20:44:51.121 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 20:44:51 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-x[156534]: 2023-09-23T20:44:51.036+0000 7f35bb8ba200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-09-23T20:44:51.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:44:50 smithi026 ceph-mon[109912]: pgmap v11: 73 pgs: 73 active+clean; 581 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:44:51.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:44:50 smithi026 ceph-mon[95839]: pgmap v11: 73 pgs: 73 active+clean; 581 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:44:51.413 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 20:44:51 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-x[156534]: 2023-09-23T20:44:51.130+0000 7f35bb8ba200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-09-23T20:44:51.413 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 20:44:51 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-x[156534]: 2023-09-23T20:44:51.239+0000 7f35bb8ba200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-09-23T20:44:51.817 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 20:44:51 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-x[156534]: 2023-09-23T20:44:51.413+0000 7f35bb8ba200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-09-23T20:44:52.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:44:51 smithi070 ceph-mon[100755]: pgmap v12: 73 pgs: 73 active+clean; 581 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:44:52.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:44:51 smithi070 ceph-mon[100755]: from='mgr.24989 ' entity='mgr.y' 2023-09-23T20:44:52.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:44:51 smithi070 ceph-mon[100755]: from='mgr.24989 ' entity='mgr.y' 2023-09-23T20:44:52.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:44:51 smithi070 ceph-mon[100755]: from='mgr.24989 172.21.15.26:0/503544817' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:44:52.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:44:51 smithi070 ceph-mon[100755]: from='client.? 172.21.15.26:0/1944351267' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:44:52.120 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 20:44:51 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-x[156534]: 2023-09-23T20:44:51.816+0000 7f35bb8ba200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-09-23T20:44:52.120 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 20:44:51 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-x[156534]: 2023-09-23T20:44:51.957+0000 7f35bb8ba200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-09-23T20:44:52.120 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 20:44:52 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-x[156534]: 2023-09-23T20:44:52.047+0000 7f35bb8ba200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-09-23T20:44:52.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:44:51 smithi026 ceph-mon[95839]: pgmap v12: 73 pgs: 73 active+clean; 581 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:44:52.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:44:51 smithi026 ceph-mon[95839]: from='mgr.24989 ' entity='mgr.y' 2023-09-23T20:44:52.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:44:51 smithi026 ceph-mon[95839]: from='mgr.24989 ' entity='mgr.y' 2023-09-23T20:44:52.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:44:51 smithi026 ceph-mon[95839]: from='mgr.24989 172.21.15.26:0/503544817' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:44:52.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:44:51 smithi026 ceph-mon[95839]: from='client.? 172.21.15.26:0/1944351267' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:44:52.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:44:51 smithi026 ceph-mon[109912]: pgmap v12: 73 pgs: 73 active+clean; 581 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:44:52.383 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:44:51 smithi026 ceph-mon[109912]: from='mgr.24989 ' entity='mgr.y' 2023-09-23T20:44:52.383 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:44:51 smithi026 ceph-mon[109912]: from='mgr.24989 ' entity='mgr.y' 2023-09-23T20:44:52.383 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:44:51 smithi026 ceph-mon[109912]: from='mgr.24989 172.21.15.26:0/503544817' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:44:52.383 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:44:51 smithi026 ceph-mon[109912]: from='client.? 172.21.15.26:0/1944351267' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:44:52.612 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 20:44:52 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-x[156534]: 2023-09-23T20:44:52.149+0000 7f35bb8ba200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-09-23T20:44:52.869 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 20:44:52 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-x[156534]: 2023-09-23T20:44:52.611+0000 7f35bb8ba200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-09-23T20:44:53.619 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 20:44:53 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-x[156534]: 2023-09-23T20:44:53.323+0000 7f35bb8ba200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-09-23T20:44:53.619 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 20:44:53 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-x[156534]: 2023-09-23T20:44:53.406+0000 7f35bb8ba200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-09-23T20:44:54.516 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 20:44:54 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-x[156534]: 2023-09-23T20:44:54.241+0000 7f35bb8ba200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-09-23T20:44:54.516 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 20:44:54 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-x[156534]: 2023-09-23T20:44:54.334+0000 7f35bb8ba200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2023-09-23T20:44:54.829 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 20:44:54 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-x[156534]: 2023-09-23T20:44:54.516+0000 7f35bb8ba200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-09-23T20:44:54.829 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:44:54 smithi070 ceph-mon[100755]: pgmap v13: 73 pgs: 73 active+clean; 581 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:44:54.829 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:44:54 smithi070 ceph-mon[100755]: from='client.? 172.21.15.26:0/5957962' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:44:54.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:44:54 smithi026 ceph-mon[95839]: pgmap v13: 73 pgs: 73 active+clean; 581 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:44:54.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:44:54 smithi026 ceph-mon[95839]: from='client.? 172.21.15.26:0/5957962' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:44:54.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:44:54 smithi026 ceph-mon[109912]: pgmap v13: 73 pgs: 73 active+clean; 581 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:44:54.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:44:54 smithi026 ceph-mon[109912]: from='client.? 172.21.15.26:0/5957962' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:44:55.119 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 20:44:54 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-x[156534]: 2023-09-23T20:44:54.828+0000 7f35bb8ba200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2023-09-23T20:44:55.119 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 20:44:55 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-x[156534]: 2023-09-23T20:44:55.065+0000 7f35bb8ba200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-09-23T20:44:55.498 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 20:44:55 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-x[156534]: 2023-09-23T20:44:55.221+0000 7f35bb8ba200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-09-23T20:44:55.869 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 20:44:55 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-x[156534]: 2023-09-23T20:44:55.498+0000 7f35bb8ba200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-09-23T20:44:56.596 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 20:44:56 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-x[156534]: 2023-09-23T20:44:56.248+0000 7f35bb8ba200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-09-23T20:44:56.596 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 20:44:56 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-x[156534]: 2023-09-23T20:44:56.393+0000 7f35bb8ba200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-09-23T20:44:56.869 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 20:44:56 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-x[156534]: 2023-09-23T20:44:56.596+0000 7f35bb8ba200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-09-23T20:44:56.869 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 20:44:56 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-x[156534]: 2023-09-23T20:44:56.718+0000 7f35bb8ba200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-09-23T20:44:56.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:44:56 smithi070 ceph-mon[100755]: pgmap v14: 73 pgs: 73 active+clean; 581 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:44:56.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:44:56 smithi070 ceph-mon[100755]: from='mgr.24989 ' entity='mgr.y' 2023-09-23T20:44:56.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:44:56 smithi070 ceph-mon[100755]: from='mgr.24989 ' entity='mgr.y' 2023-09-23T20:44:56.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:44:56 smithi070 ceph-mon[100755]: from='client.? 172.21.15.26:0/155245574' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:44:56.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:44:56 smithi026 ceph-mon[109912]: pgmap v14: 73 pgs: 73 active+clean; 581 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:44:56.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:44:56 smithi026 ceph-mon[109912]: from='mgr.24989 ' entity='mgr.y' 2023-09-23T20:44:56.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:44:56 smithi026 ceph-mon[109912]: from='mgr.24989 ' entity='mgr.y' 2023-09-23T20:44:56.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:44:56 smithi026 ceph-mon[109912]: from='client.? 172.21.15.26:0/155245574' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:44:56.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:44:56 smithi026 ceph-mon[95839]: pgmap v14: 73 pgs: 73 active+clean; 581 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:44:56.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:44:56 smithi026 ceph-mon[95839]: from='mgr.24989 ' entity='mgr.y' 2023-09-23T20:44:56.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:44:56 smithi026 ceph-mon[95839]: from='mgr.24989 ' entity='mgr.y' 2023-09-23T20:44:56.883 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:44:56 smithi026 ceph-mon[95839]: from='client.? 172.21.15.26:0/155245574' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:44:57.369 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 20:44:56 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-x[156534]: 2023-09-23T20:44:56.956+0000 7f35bb8ba200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-09-23T20:44:58.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:44:58 smithi070 ceph-mon[100755]: pgmap v15: 73 pgs: 73 active+clean; 581 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:44:58.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:44:58 smithi070 ceph-mon[100755]: from='mgr.24989 ' entity='mgr.y' 2023-09-23T20:44:58.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:44:58 smithi070 ceph-mon[100755]: from='mgr.24989 ' entity='mgr.y' 2023-09-23T20:44:58.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:44:58 smithi070 ceph-mon[100755]: Standby manager daemon x restarted 2023-09-23T20:44:58.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:44:58 smithi070 ceph-mon[100755]: Standby manager daemon x started 2023-09-23T20:44:58.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:44:58 smithi070 ceph-mon[100755]: from='mgr.? 172.21.15.70:0/3389554759' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-09-23T20:44:58.121 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:44:58 smithi070 ceph-mon[100755]: from='mgr.? 172.21.15.70:0/3389554759' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-09-23T20:44:58.121 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:44:58 smithi070 ceph-mon[100755]: from='mgr.? 172.21.15.70:0/3389554759' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-09-23T20:44:58.121 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:44:58 smithi070 ceph-mon[100755]: from='mgr.? 172.21.15.70:0/3389554759' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-09-23T20:44:58.121 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 20:44:57 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-x[156534]: 2023-09-23T20:44:57.868+0000 7f35bb8ba200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-09-23T20:44:58.121 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 20:44:57 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-x[156534]: [23/Sep/2023:20:44:57] ENGINE Bus STARTING 2023-09-23T20:44:58.121 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 20:44:57 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-x[156534]: CherryPy Checker: 2023-09-23T20:44:58.121 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 20:44:57 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-x[156534]: The Application mounted at '' has an empty config. 2023-09-23T20:44:58.121 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 20:44:57 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-x[156534]: 2023-09-23T20:44:58.121 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 20:44:57 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-x[156534]: [23/Sep/2023:20:44:57] ENGINE Serving on http://:::9283 2023-09-23T20:44:58.121 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 20:44:57 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-x[156534]: [23/Sep/2023:20:44:57] ENGINE Bus STARTED 2023-09-23T20:44:58.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:44:58 smithi026 ceph-mon[109912]: pgmap v15: 73 pgs: 73 active+clean; 581 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:44:58.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:44:58 smithi026 ceph-mon[109912]: from='mgr.24989 ' entity='mgr.y' 2023-09-23T20:44:58.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:44:58 smithi026 ceph-mon[109912]: from='mgr.24989 ' entity='mgr.y' 2023-09-23T20:44:58.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:44:58 smithi026 ceph-mon[109912]: Standby manager daemon x restarted 2023-09-23T20:44:58.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:44:58 smithi026 ceph-mon[109912]: Standby manager daemon x started 2023-09-23T20:44:58.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:44:58 smithi026 ceph-mon[109912]: from='mgr.? 172.21.15.70:0/3389554759' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-09-23T20:44:58.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:44:58 smithi026 ceph-mon[109912]: from='mgr.? 172.21.15.70:0/3389554759' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-09-23T20:44:58.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:44:58 smithi026 ceph-mon[109912]: from='mgr.? 172.21.15.70:0/3389554759' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-09-23T20:44:58.383 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:44:58 smithi026 ceph-mon[109912]: from='mgr.? 172.21.15.70:0/3389554759' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-09-23T20:44:58.383 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:44:58 smithi026 ceph-mon[95839]: pgmap v15: 73 pgs: 73 active+clean; 581 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:44:58.383 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:44:58 smithi026 ceph-mon[95839]: from='mgr.24989 ' entity='mgr.y' 2023-09-23T20:44:58.383 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:44:58 smithi026 ceph-mon[95839]: from='mgr.24989 ' entity='mgr.y' 2023-09-23T20:44:58.383 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:44:58 smithi026 ceph-mon[95839]: Standby manager daemon x restarted 2023-09-23T20:44:58.383 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:44:58 smithi026 ceph-mon[95839]: Standby manager daemon x started 2023-09-23T20:44:58.383 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:44:58 smithi026 ceph-mon[95839]: from='mgr.? 172.21.15.70:0/3389554759' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-09-23T20:44:58.383 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:44:58 smithi026 ceph-mon[95839]: from='mgr.? 172.21.15.70:0/3389554759' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-09-23T20:44:58.383 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:44:58 smithi026 ceph-mon[95839]: from='mgr.? 172.21.15.70:0/3389554759' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-09-23T20:44:58.383 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:44:58 smithi026 ceph-mon[95839]: from='mgr.? 172.21.15.70:0/3389554759' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-09-23T20:44:59.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:44:59 smithi070 ceph-mon[100755]: mgrmap e31: y(active, since 25s), standbys: x 2023-09-23T20:44:59.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:44:59 smithi070 ceph-mon[100755]: from='client.? 172.21.15.26:0/21219551' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:44:59.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:44:59 smithi026 ceph-mon[95839]: mgrmap e31: y(active, since 25s), standbys: x 2023-09-23T20:44:59.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:44:59 smithi026 ceph-mon[95839]: from='client.? 172.21.15.26:0/21219551' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:44:59.882 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:44:59 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[164029]: ::ffff:172.21.15.70 - - [23/Sep/2023:20:44:59] "GET /metrics HTTP/1.1" 200 33701 "" "Prometheus/2.43.0" 2023-09-23T20:44:59.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:44:59 smithi026 ceph-mon[109912]: mgrmap e31: y(active, since 25s), standbys: x 2023-09-23T20:44:59.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:44:59 smithi026 ceph-mon[109912]: from='client.? 172.21.15.26:0/21219551' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:45:00.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:45:00 smithi070 ceph-mon[100755]: pgmap v16: 73 pgs: 73 active+clean; 581 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:45:00.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:45:00 smithi070 ceph-mon[100755]: from='mgr.24989 ' entity='mgr.y' 2023-09-23T20:45:00.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:45:00 smithi070 ceph-mon[100755]: from='mgr.24989 ' entity='mgr.y' 2023-09-23T20:45:00.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:45:00 smithi070 ceph-mon[100755]: from='mgr.24989 172.21.15.26:0/503544817' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:45:00.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:45:00 smithi070 ceph-mon[100755]: from='mgr.24989 172.21.15.26:0/503544817' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:45:00.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:45:00 smithi070 ceph-mon[100755]: from='mgr.24989 ' entity='mgr.y' 2023-09-23T20:45:00.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:45:00 smithi070 ceph-mon[100755]: from='mgr.24989 172.21.15.26:0/503544817' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:45:00.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:45:00 smithi070 ceph-mon[100755]: from='mgr.24989 172.21.15.26:0/503544817' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T20:45:00.871 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:45:00 smithi070 ceph-mon[100755]: from='mgr.24989 ' entity='mgr.y' 2023-09-23T20:45:00.871 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:45:00 smithi070 ceph-mon[100755]: from='mgr.24989 172.21.15.26:0/503544817' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2023-09-23T20:45:00.871 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:45:00 smithi070 ceph-mon[100755]: from='mgr.24989 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2023-09-23T20:45:00.871 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:45:00 smithi070 ceph-mon[100755]: from='mgr.24989 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]': finished 2023-09-23T20:45:00.871 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:45:00 smithi070 ceph-mon[100755]: from='mgr.24989 172.21.15.26:0/503544817' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2023-09-23T20:45:00.871 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:45:00 smithi070 ceph-mon[100755]: from='mgr.24989 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2023-09-23T20:45:00.871 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:45:00 smithi070 ceph-mon[100755]: from='mgr.24989 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]': finished 2023-09-23T20:45:00.871 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:45:00 smithi070 ceph-mon[100755]: from='mgr.24989 172.21.15.26:0/503544817' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2023-09-23T20:45:00.871 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:45:00 smithi070 ceph-mon[100755]: from='mgr.24989 172.21.15.26:0/503544817' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["a"]}]: dispatch 2023-09-23T20:45:00.875 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:00 smithi026 ceph-mon[95839]: pgmap v16: 73 pgs: 73 active+clean; 581 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:45:00.876 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:00 smithi026 ceph-mon[95839]: from='mgr.24989 ' entity='mgr.y' 2023-09-23T20:45:00.876 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:00 smithi026 ceph-mon[95839]: from='mgr.24989 ' entity='mgr.y' 2023-09-23T20:45:00.876 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:00 smithi026 ceph-mon[95839]: from='mgr.24989 172.21.15.26:0/503544817' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:45:00.876 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:00 smithi026 ceph-mon[95839]: from='mgr.24989 172.21.15.26:0/503544817' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:45:00.876 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:00 smithi026 ceph-mon[95839]: from='mgr.24989 ' entity='mgr.y' 2023-09-23T20:45:00.876 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:00 smithi026 ceph-mon[95839]: from='mgr.24989 172.21.15.26:0/503544817' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:45:00.876 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:00 smithi026 ceph-mon[95839]: from='mgr.24989 172.21.15.26:0/503544817' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T20:45:00.876 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:00 smithi026 ceph-mon[95839]: from='mgr.24989 ' entity='mgr.y' 2023-09-23T20:45:00.876 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:00 smithi026 ceph-mon[95839]: from='mgr.24989 172.21.15.26:0/503544817' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2023-09-23T20:45:00.877 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:00 smithi026 ceph-mon[95839]: from='mgr.24989 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2023-09-23T20:45:00.877 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:00 smithi026 ceph-mon[95839]: from='mgr.24989 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]': finished 2023-09-23T20:45:00.877 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:00 smithi026 ceph-mon[95839]: from='mgr.24989 172.21.15.26:0/503544817' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2023-09-23T20:45:00.877 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:00 smithi026 ceph-mon[95839]: from='mgr.24989 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2023-09-23T20:45:00.877 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:00 smithi026 ceph-mon[95839]: from='mgr.24989 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]': finished 2023-09-23T20:45:00.877 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:00 smithi026 ceph-mon[95839]: from='mgr.24989 172.21.15.26:0/503544817' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2023-09-23T20:45:00.877 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:00 smithi026 ceph-mon[95839]: from='mgr.24989 172.21.15.26:0/503544817' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["a"]}]: dispatch 2023-09-23T20:45:00.877 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:45:00 smithi026 ceph-mon[109912]: pgmap v16: 73 pgs: 73 active+clean; 581 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:45:00.877 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:45:00 smithi026 ceph-mon[109912]: from='mgr.24989 ' entity='mgr.y' 2023-09-23T20:45:00.877 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:45:00 smithi026 ceph-mon[109912]: from='mgr.24989 ' entity='mgr.y' 2023-09-23T20:45:00.877 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:45:00 smithi026 ceph-mon[109912]: from='mgr.24989 172.21.15.26:0/503544817' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:45:00.878 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:45:00 smithi026 ceph-mon[109912]: from='mgr.24989 172.21.15.26:0/503544817' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:45:00.878 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:45:00 smithi026 ceph-mon[109912]: from='mgr.24989 ' entity='mgr.y' 2023-09-23T20:45:00.878 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:45:00 smithi026 ceph-mon[109912]: from='mgr.24989 172.21.15.26:0/503544817' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:45:00.878 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:45:00 smithi026 ceph-mon[109912]: from='mgr.24989 172.21.15.26:0/503544817' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T20:45:00.878 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:45:00 smithi026 ceph-mon[109912]: from='mgr.24989 ' entity='mgr.y' 2023-09-23T20:45:00.878 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:45:00 smithi026 ceph-mon[109912]: from='mgr.24989 172.21.15.26:0/503544817' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2023-09-23T20:45:00.878 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:45:00 smithi026 ceph-mon[109912]: from='mgr.24989 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2023-09-23T20:45:00.878 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:45:00 smithi026 ceph-mon[109912]: from='mgr.24989 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]': finished 2023-09-23T20:45:00.878 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:45:00 smithi026 ceph-mon[109912]: from='mgr.24989 172.21.15.26:0/503544817' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2023-09-23T20:45:00.878 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:45:00 smithi026 ceph-mon[109912]: from='mgr.24989 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2023-09-23T20:45:00.878 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:45:00 smithi026 ceph-mon[109912]: from='mgr.24989 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]': finished 2023-09-23T20:45:00.879 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:45:00 smithi026 ceph-mon[109912]: from='mgr.24989 172.21.15.26:0/503544817' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2023-09-23T20:45:00.879 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:45:00 smithi026 ceph-mon[109912]: from='mgr.24989 172.21.15.26:0/503544817' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["a"]}]: dispatch 2023-09-23T20:45:01.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:45:01 smithi070 ceph-mon[100755]: Upgrade: Setting container_image for all mgr 2023-09-23T20:45:01.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:45:01 smithi070 ceph-mon[100755]: Upgrade: It appears safe to stop mon.a 2023-09-23T20:45:01.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:45:01 smithi070 ceph-mon[100755]: from='client.? 172.21.15.26:0/1949444443' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:45:01.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:45:01 smithi026 ceph-mon[109912]: Upgrade: Setting container_image for all mgr 2023-09-23T20:45:01.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:45:01 smithi026 ceph-mon[109912]: Upgrade: It appears safe to stop mon.a 2023-09-23T20:45:01.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:45:01 smithi026 ceph-mon[109912]: from='client.? 172.21.15.26:0/1949444443' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:45:01.883 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:01 smithi026 ceph-mon[95839]: Upgrade: Setting container_image for all mgr 2023-09-23T20:45:01.883 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:01 smithi026 ceph-mon[95839]: Upgrade: It appears safe to stop mon.a 2023-09-23T20:45:01.883 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:01 smithi026 ceph-mon[95839]: from='client.? 172.21.15.26:0/1949444443' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:45:02.795 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:45:02 smithi026 ceph-mon[109912]: pgmap v17: 73 pgs: 73 active+clean; 581 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:45:02.796 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:45:02 smithi026 ceph-mon[109912]: Upgrade: Updating mon.a 2023-09-23T20:45:02.796 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:45:02 smithi026 ceph-mon[109912]: from='mgr.24989 ' entity='mgr.y' 2023-09-23T20:45:02.796 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:45:02 smithi026 ceph-mon[109912]: from='mgr.24989 172.21.15.26:0/503544817' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-23T20:45:02.796 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:45:02 smithi026 ceph-mon[109912]: from='mgr.24989 172.21.15.26:0/503544817' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-09-23T20:45:02.796 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:45:02 smithi026 ceph-mon[109912]: from='mgr.24989 172.21.15.26:0/503544817' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:45:02.796 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:45:02 smithi026 ceph-mon[109912]: Deploying daemon mon.a on smithi026 2023-09-23T20:45:02.796 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:02 smithi026 ceph-mon[95839]: pgmap v17: 73 pgs: 73 active+clean; 581 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:45:02.796 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:02 smithi026 ceph-mon[95839]: Upgrade: Updating mon.a 2023-09-23T20:45:02.797 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:02 smithi026 ceph-mon[95839]: from='mgr.24989 ' entity='mgr.y' 2023-09-23T20:45:02.797 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:02 smithi026 ceph-mon[95839]: from='mgr.24989 172.21.15.26:0/503544817' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-23T20:45:02.797 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:02 smithi026 ceph-mon[95839]: from='mgr.24989 172.21.15.26:0/503544817' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-09-23T20:45:02.797 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:02 smithi026 ceph-mon[95839]: from='mgr.24989 172.21.15.26:0/503544817' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:45:02.797 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:02 smithi026 ceph-mon[95839]: Deploying daemon mon.a on smithi026 2023-09-23T20:45:03.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:45:02 smithi070 ceph-mon[100755]: pgmap v17: 73 pgs: 73 active+clean; 581 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:45:03.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:45:02 smithi070 ceph-mon[100755]: Upgrade: Updating mon.a 2023-09-23T20:45:03.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:45:02 smithi070 ceph-mon[100755]: from='mgr.24989 ' entity='mgr.y' 2023-09-23T20:45:03.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:45:02 smithi070 ceph-mon[100755]: from='mgr.24989 172.21.15.26:0/503544817' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-23T20:45:03.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:45:02 smithi070 ceph-mon[100755]: from='mgr.24989 172.21.15.26:0/503544817' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-09-23T20:45:03.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:45:02 smithi070 ceph-mon[100755]: from='mgr.24989 172.21.15.26:0/503544817' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:45:03.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:45:02 smithi070 ceph-mon[100755]: Deploying daemon mon.a on smithi026 2023-09-23T20:45:03.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:02 smithi026 systemd[1]: Stopping Ceph mon.a for b3f8e4ec-5a50-11ee-8db4-212e2dc638e7... 2023-09-23T20:45:03.567 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:03 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mon-a[95815]: 2023-09-23T20:45:03.260+0000 7fd0564f3700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-mon -n mon.a -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2023-09-23T20:45:03.567 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:03 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mon-a[95815]: 2023-09-23T20:45:03.260+0000 7fd0564f3700 -1 mon.a@0(leader) e3 *** Got Signal Terminated *** 2023-09-23T20:45:03.881 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:03 smithi026 bash[174931]: ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mon-a 2023-09-23T20:45:04.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:03 smithi026 systemd[1]: ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7@mon.a.service: Succeeded. 2023-09-23T20:45:04.383 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:03 smithi026 systemd[1]: Stopped Ceph mon.a for b3f8e4ec-5a50-11ee-8db4-212e2dc638e7. 2023-09-23T20:45:04.383 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:04 smithi026 systemd[1]: Starting Ceph mon.a for b3f8e4ec-5a50-11ee-8db4-212e2dc638e7... 2023-09-23T20:45:04.669 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:04 smithi026 podman[175111]: 2023-09-23T20:45:05.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:04 smithi026 ceph-mon[175151]: set uid:gid to 167:167 (ceph:ceph) 2023-09-23T20:45:05.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:04 smithi026 ceph-mon[175151]: ceph version 18.2.0-460-gbb0daeff (bb0daeff97f020f686596fca6eba20c310e26fb7) reef (stable), process ceph-mon, pid 7 2023-09-23T20:45:05.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:04 smithi026 ceph-mon[175151]: pidfile_write: ignore empty --pid-file 2023-09-23T20:45:05.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:04 smithi026 ceph-mon[175151]: load: jerasure load: lrc 2023-09-23T20:45:05.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:04 smithi026 ceph-mon[175151]: rocksdb: RocksDB version: 7.9.2 2023-09-23T20:45:05.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:04 smithi026 ceph-mon[175151]: rocksdb: Git sha 0 2023-09-23T20:45:05.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:04 smithi026 ceph-mon[175151]: rocksdb: Compile date 2023-09-22 15:19:43 2023-09-23T20:45:05.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:04 smithi026 ceph-mon[175151]: rocksdb: DB SUMMARY 2023-09-23T20:45:05.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:04 smithi026 ceph-mon[175151]: rocksdb: DB Session ID: TW2LGVHIET6MDVK314PV 2023-09-23T20:45:05.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:04 smithi026 ceph-mon[175151]: rocksdb: CURRENT file: CURRENT 2023-09-23T20:45:05.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:04 smithi026 ceph-mon[175151]: rocksdb: IDENTITY file: IDENTITY 2023-09-23T20:45:05.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:04 smithi026 ceph-mon[175151]: rocksdb: MANIFEST file: MANIFEST-000015 size: 765 Bytes 2023-09-23T20:45:05.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:04 smithi026 ceph-mon[175151]: rocksdb: SST files in /var/lib/ceph/mon/ceph-a/store.db dir, Total Num: 1, files: 000027.sst 2023-09-23T20:45:05.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:04 smithi026 ceph-mon[175151]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-a/store.db: 000025.log size: 2548176 ; 2023-09-23T20:45:05.134 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:04 smithi026 ceph-mon[175151]: rocksdb: Options.error_if_exists: 0 2023-09-23T20:45:05.134 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:04 smithi026 ceph-mon[175151]: rocksdb: Options.create_if_missing: 0 2023-09-23T20:45:05.134 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:04 smithi026 ceph-mon[175151]: rocksdb: Options.paranoid_checks: 1 2023-09-23T20:45:05.134 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:04 smithi026 ceph-mon[175151]: rocksdb: Options.flush_verify_memtable_count: 1 2023-09-23T20:45:05.134 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:04 smithi026 ceph-mon[175151]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2023-09-23T20:45:05.134 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:04 smithi026 ceph-mon[175151]: rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2023-09-23T20:45:05.134 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:04 smithi026 ceph-mon[175151]: rocksdb: Options.env: 0x559e110a0720 2023-09-23T20:45:05.134 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:04 smithi026 ceph-mon[175151]: rocksdb: Options.fs: PosixFileSystem 2023-09-23T20:45:05.134 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:04 smithi026 ceph-mon[175151]: rocksdb: Options.info_log: 0x559e13c03420 2023-09-23T20:45:05.134 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:04 smithi026 ceph-mon[175151]: rocksdb: Options.max_file_opening_threads: 16 2023-09-23T20:45:05.134 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:04 smithi026 ceph-mon[175151]: rocksdb: Options.statistics: (nil) 2023-09-23T20:45:05.135 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:04 smithi026 ceph-mon[175151]: rocksdb: Options.use_fsync: 0 2023-09-23T20:45:05.135 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:04 smithi026 ceph-mon[175151]: rocksdb: Options.max_log_file_size: 0 2023-09-23T20:45:05.135 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:04 smithi026 ceph-mon[175151]: rocksdb: Options.max_manifest_file_size: 1073741824 2023-09-23T20:45:05.135 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:04 smithi026 ceph-mon[175151]: rocksdb: Options.log_file_time_to_roll: 0 2023-09-23T20:45:05.135 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:04 smithi026 ceph-mon[175151]: rocksdb: Options.keep_log_file_num: 1000 2023-09-23T20:45:05.135 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:04 smithi026 ceph-mon[175151]: rocksdb: Options.recycle_log_file_num: 0 2023-09-23T20:45:05.135 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:04 smithi026 ceph-mon[175151]: rocksdb: Options.allow_fallocate: 1 2023-09-23T20:45:05.135 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:04 smithi026 ceph-mon[175151]: rocksdb: Options.allow_mmap_reads: 0 2023-09-23T20:45:05.135 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:04 smithi026 ceph-mon[175151]: rocksdb: Options.allow_mmap_writes: 0 2023-09-23T20:45:05.135 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:04 smithi026 ceph-mon[175151]: rocksdb: Options.use_direct_reads: 0 2023-09-23T20:45:05.135 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:04 smithi026 ceph-mon[175151]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2023-09-23T20:45:05.136 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:04 smithi026 ceph-mon[175151]: rocksdb: Options.create_missing_column_families: 0 2023-09-23T20:45:05.136 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:04 smithi026 ceph-mon[175151]: rocksdb: Options.db_log_dir: 2023-09-23T20:45:05.136 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:04 smithi026 ceph-mon[175151]: rocksdb: Options.wal_dir: 2023-09-23T20:45:05.136 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:04 smithi026 ceph-mon[175151]: rocksdb: Options.table_cache_numshardbits: 6 2023-09-23T20:45:05.136 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:04 smithi026 ceph-mon[175151]: rocksdb: Options.WAL_ttl_seconds: 0 2023-09-23T20:45:05.136 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:04 smithi026 ceph-mon[175151]: rocksdb: Options.WAL_size_limit_MB: 0 2023-09-23T20:45:05.136 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:04 smithi026 ceph-mon[175151]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2023-09-23T20:45:05.136 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:04 smithi026 ceph-mon[175151]: rocksdb: Options.manifest_preallocation_size: 4194304 2023-09-23T20:45:05.136 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:04 smithi026 ceph-mon[175151]: rocksdb: Options.is_fd_close_on_exec: 1 2023-09-23T20:45:05.136 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:04 smithi026 ceph-mon[175151]: rocksdb: Options.advise_random_on_open: 1 2023-09-23T20:45:05.136 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:04 smithi026 ceph-mon[175151]: rocksdb: Options.db_write_buffer_size: 0 2023-09-23T20:45:05.136 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:04 smithi026 ceph-mon[175151]: rocksdb: Options.write_buffer_manager: 0x559e12e945a0 2023-09-23T20:45:05.136 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:04 smithi026 ceph-mon[175151]: rocksdb: Options.access_hint_on_compaction_start: 1 2023-09-23T20:45:05.137 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:04 smithi026 ceph-mon[175151]: rocksdb: Options.random_access_max_buffer_size: 1048576 2023-09-23T20:45:05.137 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:04 smithi026 ceph-mon[175151]: rocksdb: Options.use_adaptive_mutex: 0 2023-09-23T20:45:05.137 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:04 smithi026 ceph-mon[175151]: rocksdb: Options.rate_limiter: (nil) 2023-09-23T20:45:05.137 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:04 smithi026 ceph-mon[175151]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2023-09-23T20:45:05.137 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:04 smithi026 ceph-mon[175151]: rocksdb: Options.wal_recovery_mode: 2 2023-09-23T20:45:05.137 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:04 smithi026 ceph-mon[175151]: rocksdb: Options.enable_thread_tracking: 0 2023-09-23T20:45:05.137 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:04 smithi026 ceph-mon[175151]: rocksdb: Options.enable_pipelined_write: 0 2023-09-23T20:45:05.137 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:04 smithi026 ceph-mon[175151]: rocksdb: Options.unordered_write: 0 2023-09-23T20:45:05.137 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:04 smithi026 ceph-mon[175151]: rocksdb: Options.allow_concurrent_memtable_write: 1 2023-09-23T20:45:05.137 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:04 smithi026 ceph-mon[175151]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2023-09-23T20:45:05.137 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:04 smithi026 ceph-mon[175151]: rocksdb: Options.write_thread_max_yield_usec: 100 2023-09-23T20:45:05.137 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:04 smithi026 ceph-mon[175151]: rocksdb: Options.write_thread_slow_yield_usec: 3 2023-09-23T20:45:05.138 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:04 smithi026 ceph-mon[175151]: rocksdb: Options.row_cache: None 2023-09-23T20:45:05.138 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:04 smithi026 ceph-mon[175151]: rocksdb: Options.wal_filter: None 2023-09-23T20:45:05.138 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:04 smithi026 ceph-mon[175151]: rocksdb: Options.avoid_flush_during_recovery: 0 2023-09-23T20:45:05.138 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:04 smithi026 ceph-mon[175151]: rocksdb: Options.allow_ingest_behind: 0 2023-09-23T20:45:05.138 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:04 smithi026 ceph-mon[175151]: rocksdb: Options.two_write_queues: 0 2023-09-23T20:45:05.138 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:04 smithi026 ceph-mon[175151]: rocksdb: Options.manual_wal_flush: 0 2023-09-23T20:45:05.138 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:04 smithi026 ceph-mon[175151]: rocksdb: Options.wal_compression: 0 2023-09-23T20:45:05.138 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:04 smithi026 ceph-mon[175151]: rocksdb: Options.atomic_flush: 0 2023-09-23T20:45:05.138 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:04 smithi026 ceph-mon[175151]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2023-09-23T20:45:05.138 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:04 smithi026 ceph-mon[175151]: rocksdb: Options.persist_stats_to_disk: 0 2023-09-23T20:45:05.138 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:04 smithi026 ceph-mon[175151]: rocksdb: Options.write_dbid_to_manifest: 0 2023-09-23T20:45:05.138 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:04 smithi026 ceph-mon[175151]: rocksdb: Options.log_readahead_size: 0 2023-09-23T20:45:05.139 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:04 smithi026 ceph-mon[175151]: rocksdb: Options.file_checksum_gen_factory: Unknown 2023-09-23T20:45:05.139 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:04 smithi026 ceph-mon[175151]: rocksdb: Options.best_efforts_recovery: 0 2023-09-23T20:45:05.139 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:04 smithi026 ceph-mon[175151]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2023-09-23T20:45:05.139 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:04 smithi026 ceph-mon[175151]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2023-09-23T20:45:05.139 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:04 smithi026 ceph-mon[175151]: rocksdb: Options.allow_data_in_errors: 0 2023-09-23T20:45:05.139 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:04 smithi026 ceph-mon[175151]: rocksdb: Options.db_host_id: __hostname__ 2023-09-23T20:45:05.140 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:04 smithi026 ceph-mon[175151]: rocksdb: Options.enforce_single_del_contracts: true 2023-09-23T20:45:05.140 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:04 smithi026 ceph-mon[175151]: rocksdb: Options.max_background_jobs: 2 2023-09-23T20:45:05.140 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:04 smithi026 ceph-mon[175151]: rocksdb: Options.max_background_compactions: -1 2023-09-23T20:45:05.140 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:04 smithi026 ceph-mon[175151]: rocksdb: Options.max_subcompactions: 1 2023-09-23T20:45:05.140 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:04 smithi026 ceph-mon[175151]: rocksdb: Options.avoid_flush_during_shutdown: 0 2023-09-23T20:45:05.140 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:04 smithi026 ceph-mon[175151]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2023-09-23T20:45:05.141 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:04 smithi026 ceph-mon[175151]: rocksdb: Options.delayed_write_rate : 16777216 2023-09-23T20:45:05.141 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:04 smithi026 ceph-mon[175151]: rocksdb: Options.max_total_wal_size: 0 2023-09-23T20:45:05.141 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:04 smithi026 ceph-mon[175151]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2023-09-23T20:45:05.141 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:04 smithi026 ceph-mon[175151]: rocksdb: Options.stats_dump_period_sec: 600 2023-09-23T20:45:05.141 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:04 smithi026 ceph-mon[175151]: rocksdb: Options.stats_persist_period_sec: 600 2023-09-23T20:45:05.141 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:04 smithi026 ceph-mon[175151]: rocksdb: Options.stats_history_buffer_size: 1048576 2023-09-23T20:45:05.141 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:04 smithi026 ceph-mon[175151]: rocksdb: Options.max_open_files: -1 2023-09-23T20:45:05.141 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:04 smithi026 ceph-mon[175151]: rocksdb: Options.bytes_per_sync: 0 2023-09-23T20:45:05.141 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:04 smithi026 ceph-mon[175151]: rocksdb: Options.wal_bytes_per_sync: 0 2023-09-23T20:45:05.141 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:04 smithi026 ceph-mon[175151]: rocksdb: Options.strict_bytes_per_sync: 0 2023-09-23T20:45:05.141 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:04 smithi026 ceph-mon[175151]: rocksdb: Options.compaction_readahead_size: 0 2023-09-23T20:45:05.141 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:04 smithi026 ceph-mon[175151]: rocksdb: Options.max_background_flushes: -1 2023-09-23T20:45:05.142 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:04 smithi026 ceph-mon[175151]: rocksdb: Compression algorithms supported: 2023-09-23T20:45:05.142 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:04 smithi026 ceph-mon[175151]: rocksdb: kZSTDNotFinalCompression supported: 0 2023-09-23T20:45:05.142 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:04 smithi026 ceph-mon[175151]: rocksdb: kZSTD supported: 0 2023-09-23T20:45:05.142 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:04 smithi026 ceph-mon[175151]: rocksdb: kXpressCompression supported: 0 2023-09-23T20:45:05.142 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:04 smithi026 ceph-mon[175151]: rocksdb: kLZ4HCCompression supported: 1 2023-09-23T20:45:05.142 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:04 smithi026 ceph-mon[175151]: rocksdb: kZlibCompression supported: 1 2023-09-23T20:45:05.142 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:04 smithi026 ceph-mon[175151]: rocksdb: kSnappyCompression supported: 1 2023-09-23T20:45:05.142 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:04 smithi026 ceph-mon[175151]: rocksdb: kLZ4Compression supported: 1 2023-09-23T20:45:05.142 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:04 smithi026 ceph-mon[175151]: rocksdb: kBZip2Compression supported: 0 2023-09-23T20:45:05.142 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:04 smithi026 ceph-mon[175151]: rocksdb: Fast CRC32 supported: Supported on x86 2023-09-23T20:45:05.142 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:04 smithi026 ceph-mon[175151]: rocksdb: DMutex implementation: pthread_mutex_t 2023-09-23T20:45:05.142 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:04 smithi026 ceph-mon[175151]: rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000015 2023-09-23T20:45:05.143 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:04 smithi026 ceph-mon[175151]: rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2023-09-23T20:45:05.143 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:04 smithi026 ceph-mon[175151]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2023-09-23T20:45:05.143 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:04 smithi026 ceph-mon[175151]: rocksdb: Options.merge_operator: 2023-09-23T20:45:05.143 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:04 smithi026 ceph-mon[175151]: rocksdb: Options.compaction_filter: None 2023-09-23T20:45:05.143 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:04 smithi026 ceph-mon[175151]: rocksdb: Options.compaction_filter_factory: None 2023-09-23T20:45:05.143 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:04 smithi026 ceph-mon[175151]: rocksdb: Options.sst_partitioner_factory: None 2023-09-23T20:45:05.143 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:04 smithi026 ceph-mon[175151]: rocksdb: Options.memtable_factory: SkipListFactory 2023-09-23T20:45:05.143 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:04 smithi026 ceph-mon[175151]: rocksdb: Options.table_factory: BlockBasedTable 2023-09-23T20:45:05.143 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:04 smithi026 ceph-mon[175151]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x559e13c03520) 2023-09-23T20:45:05.143 INFO:journalctl@ceph.mon.a.smithi026.stdout: cache_index_and_filter_blocks: 1 2023-09-23T20:45:05.143 INFO:journalctl@ceph.mon.a.smithi026.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2023-09-23T20:45:05.143 INFO:journalctl@ceph.mon.a.smithi026.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2023-09-23T20:45:05.143 INFO:journalctl@ceph.mon.a.smithi026.stdout: pin_top_level_index_and_filter: 1 2023-09-23T20:45:05.144 INFO:journalctl@ceph.mon.a.smithi026.stdout: index_type: 0 2023-09-23T20:45:05.144 INFO:journalctl@ceph.mon.a.smithi026.stdout: data_block_index_type: 0 2023-09-23T20:45:05.144 INFO:journalctl@ceph.mon.a.smithi026.stdout: index_shortening: 1 2023-09-23T20:45:05.144 INFO:journalctl@ceph.mon.a.smithi026.stdout: data_block_hash_table_util_ratio: 0.750000 2023-09-23T20:45:05.144 INFO:journalctl@ceph.mon.a.smithi026.stdout: checksum: 4 2023-09-23T20:45:05.144 INFO:journalctl@ceph.mon.a.smithi026.stdout: no_block_cache: 0 2023-09-23T20:45:05.144 INFO:journalctl@ceph.mon.a.smithi026.stdout: block_cache: 0x559e12f15350 2023-09-23T20:45:05.144 INFO:journalctl@ceph.mon.a.smithi026.stdout: block_cache_name: BinnedLRUCache 2023-09-23T20:45:05.144 INFO:journalctl@ceph.mon.a.smithi026.stdout: block_cache_options: 2023-09-23T20:45:05.144 INFO:journalctl@ceph.mon.a.smithi026.stdout: capacity : 536870912 2023-09-23T20:45:05.144 INFO:journalctl@ceph.mon.a.smithi026.stdout: num_shard_bits : 4 2023-09-23T20:45:05.144 INFO:journalctl@ceph.mon.a.smithi026.stdout: strict_capacity_limit : 0 2023-09-23T20:45:05.145 INFO:journalctl@ceph.mon.a.smithi026.stdout: high_pri_pool_ratio: 0.000 2023-09-23T20:45:05.145 INFO:journalctl@ceph.mon.a.smithi026.stdout: block_cache_compressed: (nil) 2023-09-23T20:45:05.145 INFO:journalctl@ceph.mon.a.smithi026.stdout: persistent_cache: (nil) 2023-09-23T20:45:05.145 INFO:journalctl@ceph.mon.a.smithi026.stdout: block_size: 4096 2023-09-23T20:45:05.145 INFO:journalctl@ceph.mon.a.smithi026.stdout: block_size_deviation: 10 2023-09-23T20:45:05.145 INFO:journalctl@ceph.mon.a.smithi026.stdout: block_restart_interval: 16 2023-09-23T20:45:05.145 INFO:journalctl@ceph.mon.a.smithi026.stdout: index_block_restart_interval: 1 2023-09-23T20:45:05.145 INFO:journalctl@ceph.mon.a.smithi026.stdout: metadata_block_size: 4096 2023-09-23T20:45:05.145 INFO:journalctl@ceph.mon.a.smithi026.stdout: partition_filters: 0 2023-09-23T20:45:05.145 INFO:journalctl@ceph.mon.a.smithi026.stdout: use_delta_encoding: 1 2023-09-23T20:45:05.145 INFO:journalctl@ceph.mon.a.smithi026.stdout: filter_policy: bloomfilter 2023-09-23T20:45:05.145 INFO:journalctl@ceph.mon.a.smithi026.stdout: whole_key_filtering: 1 2023-09-23T20:45:05.146 INFO:journalctl@ceph.mon.a.smithi026.stdout: verify_compression: 0 2023-09-23T20:45:05.146 INFO:journalctl@ceph.mon.a.smithi026.stdout: read_amp_bytes_per_bit: 0 2023-09-23T20:45:05.146 INFO:journalctl@ceph.mon.a.smithi026.stdout: format_version: 5 2023-09-23T20:45:05.146 INFO:journalctl@ceph.mon.a.smithi026.stdout: enable_index_compression: 1 2023-09-23T20:45:05.146 INFO:journalctl@ceph.mon.a.smithi026.stdout: block_align: 0 2023-09-23T20:45:05.146 INFO:journalctl@ceph.mon.a.smithi026.stdout: max_auto_readahead_size: 262144 2023-09-23T20:45:05.146 INFO:journalctl@ceph.mon.a.smithi026.stdout: prepopulate_block_cache: 0 2023-09-23T20:45:05.146 INFO:journalctl@ceph.mon.a.smithi026.stdout: initial_auto_readahead_size: 8192 2023-09-23T20:45:05.146 INFO:journalctl@ceph.mon.a.smithi026.stdout: num_file_reads_for_auto_readahead: 2 2023-09-23T20:45:05.146 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:04 smithi026 ceph-mon[175151]: rocksdb: Options.write_buffer_size: 33554432 2023-09-23T20:45:05.146 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:04 smithi026 ceph-mon[175151]: rocksdb: Options.max_write_buffer_number: 2 2023-09-23T20:45:05.146 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:04 smithi026 ceph-mon[175151]: rocksdb: Options.compression: NoCompression 2023-09-23T20:45:05.146 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:04 smithi026 ceph-mon[175151]: rocksdb: Options.bottommost_compression: Disabled 2023-09-23T20:45:05.147 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:04 smithi026 ceph-mon[175151]: rocksdb: Options.prefix_extractor: nullptr 2023-09-23T20:45:05.147 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:04 smithi026 ceph-mon[175151]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2023-09-23T20:45:05.147 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:04 smithi026 ceph-mon[175151]: rocksdb: Options.num_levels: 7 2023-09-23T20:45:05.147 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:04 smithi026 ceph-mon[175151]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2023-09-23T20:45:05.147 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:04 smithi026 ceph-mon[175151]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2023-09-23T20:45:05.147 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:04 smithi026 ceph-mon[175151]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2023-09-23T20:45:05.147 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:04 smithi026 ceph-mon[175151]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2023-09-23T20:45:05.147 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:04 smithi026 ceph-mon[175151]: rocksdb: Options.bottommost_compression_opts.level: 32767 2023-09-23T20:45:05.147 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:04 smithi026 ceph-mon[175151]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2023-09-23T20:45:05.147 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:04 smithi026 ceph-mon[175151]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2023-09-23T20:45:05.147 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:04 smithi026 ceph-mon[175151]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2023-09-23T20:45:05.147 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:04 smithi026 ceph-mon[175151]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2023-09-23T20:45:05.148 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:04 smithi026 ceph-mon[175151]: rocksdb: Options.bottommost_compression_opts.enabled: false 2023-09-23T20:45:05.148 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:04 smithi026 ceph-mon[175151]: rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2023-09-23T20:45:05.148 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:04 smithi026 ceph-mon[175151]: rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2023-09-23T20:45:05.148 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:04 smithi026 ceph-mon[175151]: rocksdb: Options.compression_opts.window_bits: -14 2023-09-23T20:45:05.148 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:04 smithi026 ceph-mon[175151]: rocksdb: Options.compression_opts.level: 32767 2023-09-23T20:45:05.148 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:04 smithi026 ceph-mon[175151]: rocksdb: Options.compression_opts.strategy: 0 2023-09-23T20:45:05.148 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:04 smithi026 ceph-mon[175151]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2023-09-23T20:45:05.148 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:04 smithi026 ceph-mon[175151]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2023-09-23T20:45:05.148 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:04 smithi026 ceph-mon[175151]: rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2023-09-23T20:45:05.148 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:04 smithi026 ceph-mon[175151]: rocksdb: Options.compression_opts.parallel_threads: 1 2023-09-23T20:45:05.148 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:04 smithi026 ceph-mon[175151]: rocksdb: Options.compression_opts.enabled: false 2023-09-23T20:45:05.148 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:04 smithi026 ceph-mon[175151]: rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2023-09-23T20:45:05.149 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:04 smithi026 ceph-mon[175151]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2023-09-23T20:45:05.149 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:04 smithi026 ceph-mon[175151]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2023-09-23T20:45:05.149 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:04 smithi026 ceph-mon[175151]: rocksdb: Options.level0_stop_writes_trigger: 36 2023-09-23T20:45:05.149 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:04 smithi026 ceph-mon[175151]: rocksdb: Options.target_file_size_base: 67108864 2023-09-23T20:45:05.149 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:04 smithi026 ceph-mon[175151]: rocksdb: Options.target_file_size_multiplier: 1 2023-09-23T20:45:05.149 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:04 smithi026 ceph-mon[175151]: rocksdb: Options.max_bytes_for_level_base: 268435456 2023-09-23T20:45:05.149 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:04 smithi026 ceph-mon[175151]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2023-09-23T20:45:05.149 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:04 smithi026 ceph-mon[175151]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2023-09-23T20:45:05.149 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:04 smithi026 ceph-mon[175151]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2023-09-23T20:45:05.149 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:04 smithi026 ceph-mon[175151]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2023-09-23T20:45:05.149 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:04 smithi026 ceph-mon[175151]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2023-09-23T20:45:05.149 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:04 smithi026 ceph-mon[175151]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2023-09-23T20:45:05.150 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:04 smithi026 ceph-mon[175151]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2023-09-23T20:45:05.150 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:04 smithi026 ceph-mon[175151]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2023-09-23T20:45:05.150 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:04 smithi026 ceph-mon[175151]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2023-09-23T20:45:05.150 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:04 smithi026 ceph-mon[175151]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2023-09-23T20:45:05.150 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:04 smithi026 ceph-mon[175151]: rocksdb: Options.max_compaction_bytes: 1677721600 2023-09-23T20:45:05.150 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:04 smithi026 ceph-mon[175151]: rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2023-09-23T20:45:05.150 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:04 smithi026 ceph-mon[175151]: rocksdb: Options.arena_block_size: 1048576 2023-09-23T20:45:05.150 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:04 smithi026 ceph-mon[175151]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2023-09-23T20:45:05.150 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:04 smithi026 ceph-mon[175151]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2023-09-23T20:45:05.150 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:04 smithi026 ceph-mon[175151]: rocksdb: Options.disable_auto_compactions: 0 2023-09-23T20:45:05.150 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:04 smithi026 ceph-mon[175151]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2023-09-23T20:45:05.150 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:04 smithi026 ceph-mon[175151]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2023-09-23T20:45:05.151 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:04 smithi026 ceph-mon[175151]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2023-09-23T20:45:05.151 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:04 smithi026 ceph-mon[175151]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2023-09-23T20:45:05.151 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:04 smithi026 ceph-mon[175151]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2023-09-23T20:45:05.151 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:04 smithi026 ceph-mon[175151]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2023-09-23T20:45:05.151 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:04 smithi026 ceph-mon[175151]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2023-09-23T20:45:05.151 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:04 smithi026 ceph-mon[175151]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2023-09-23T20:45:05.151 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:04 smithi026 ceph-mon[175151]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2023-09-23T20:45:05.151 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:04 smithi026 ceph-mon[175151]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2023-09-23T20:45:05.151 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:04 smithi026 ceph-mon[175151]: rocksdb: Options.table_properties_collectors: 2023-09-23T20:45:05.151 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:04 smithi026 ceph-mon[175151]: rocksdb: Options.inplace_update_support: 0 2023-09-23T20:45:05.151 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:04 smithi026 ceph-mon[175151]: rocksdb: Options.inplace_update_num_locks: 10000 2023-09-23T20:45:05.151 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:04 smithi026 ceph-mon[175151]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2023-09-23T20:45:05.152 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:04 smithi026 ceph-mon[175151]: rocksdb: Options.memtable_whole_key_filtering: 0 2023-09-23T20:45:05.152 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:04 smithi026 ceph-mon[175151]: rocksdb: Options.memtable_huge_page_size: 0 2023-09-23T20:45:05.152 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:04 smithi026 ceph-mon[175151]: rocksdb: Options.bloom_locality: 0 2023-09-23T20:45:05.152 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:04 smithi026 ceph-mon[175151]: rocksdb: Options.max_successive_merges: 0 2023-09-23T20:45:05.152 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:04 smithi026 ceph-mon[175151]: rocksdb: Options.optimize_filters_for_hits: 0 2023-09-23T20:45:05.152 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:04 smithi026 ceph-mon[175151]: rocksdb: Options.paranoid_file_checks: 0 2023-09-23T20:45:05.152 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:04 smithi026 ceph-mon[175151]: rocksdb: Options.force_consistency_checks: 1 2023-09-23T20:45:05.152 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:04 smithi026 ceph-mon[175151]: rocksdb: Options.report_bg_io_stats: 0 2023-09-23T20:45:05.152 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:04 smithi026 ceph-mon[175151]: rocksdb: Options.ttl: 2592000 2023-09-23T20:45:05.152 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:04 smithi026 ceph-mon[175151]: rocksdb: Options.periodic_compaction_seconds: 0 2023-09-23T20:45:05.152 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:04 smithi026 ceph-mon[175151]: rocksdb: Options.preclude_last_level_data_seconds: 0 2023-09-23T20:45:05.152 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:04 smithi026 ceph-mon[175151]: rocksdb: Options.preserve_internal_time_seconds: 0 2023-09-23T20:45:05.153 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:04 smithi026 ceph-mon[175151]: rocksdb: Options.enable_blob_files: false 2023-09-23T20:45:05.153 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:04 smithi026 ceph-mon[175151]: rocksdb: Options.min_blob_size: 0 2023-09-23T20:45:05.153 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:04 smithi026 ceph-mon[175151]: rocksdb: Options.blob_file_size: 268435456 2023-09-23T20:45:05.153 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:04 smithi026 ceph-mon[175151]: rocksdb: Options.blob_compression_type: NoCompression 2023-09-23T20:45:05.153 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:04 smithi026 ceph-mon[175151]: rocksdb: Options.enable_blob_garbage_collection: false 2023-09-23T20:45:05.153 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:04 smithi026 ceph-mon[175151]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2023-09-23T20:45:05.153 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:04 smithi026 ceph-mon[175151]: rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2023-09-23T20:45:05.153 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:04 smithi026 ceph-mon[175151]: rocksdb: Options.blob_compaction_readahead_size: 0 2023-09-23T20:45:05.153 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:04 smithi026 ceph-mon[175151]: rocksdb: Options.blob_file_starting_level: 0 2023-09-23T20:45:05.153 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:04 smithi026 ceph-mon[175151]: rocksdb: Options.experimental_mempurge_threshold: 0.000000 2023-09-23T20:45:05.153 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:04 smithi026 ceph-mon[175151]: rocksdb: [table/block_based/block_based_table_reader.cc:721] At least one SST file opened without unique ID to verify: 27.sst 2023-09-23T20:45:05.153 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:04 smithi026 ceph-mon[175151]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2023-09-23T20:45:05.154 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:04 smithi026 ceph-mon[175151]: rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000015 succeeded,manifest_file_number is 15, next_file_number is 29, last_sequence is 10178, log_number is 25,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2023-09-23T20:45:05.154 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:04 smithi026 ceph-mon[175151]: rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 25 2023-09-23T20:45:05.154 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:04 smithi026 ceph-mon[175151]: rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: 9fa03219-303e-487f-a76c-0f910ed72ae6 2023-09-23T20:45:05.154 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:04 smithi026 ceph-mon[175151]: rocksdb: EVENT_LOG_v1 {"time_micros": 1695501904675750, "job": 1, "event": "recovery_started", "wal_files": [25]} 2023-09-23T20:45:05.154 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:04 smithi026 ceph-mon[175151]: rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #25 mode 2 2023-09-23T20:45:05.154 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:04 smithi026 ceph-mon[175151]: rocksdb: EVENT_LOG_v1 {"time_micros": 1695501904683372, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 30, "file_size": 2314853, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 10179, "largest_seqno": 11394, "table_properties": {"data_size": 2308795, "index_size": 3631, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 1413, "raw_key_size": 12762, "raw_average_key_size": 23, "raw_value_size": 2296965, "raw_average_value_size": 4214, "num_data_blocks": 168, "num_entries": 545, "num_filter_entries": 545, "num_deletions": 2, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1695501904, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "9fa03219-303e-487f-a76c-0f910ed72ae6", "db_session_id": "TW2LGVHIET6MDVK314PV", "orig_file_number": 30, "seqno_to_time_mapping": "N/A"}} 2023-09-23T20:45:05.154 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:04 smithi026 ceph-mon[175151]: rocksdb: EVENT_LOG_v1 {"time_micros": 1695501904683463, "job": 1, "event": "recovery_finished"} 2023-09-23T20:45:05.154 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:04 smithi026 ceph-mon[175151]: rocksdb: [db/version_set.cc:5047] Creating manifest 32 2023-09-23T20:45:05.154 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:04 smithi026 ceph-mon[175151]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2023-09-23T20:45:05.154 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:04 smithi026 bash[175111]: f0ba51b53f505569a6aa13c3f5778032a80772deee9b11987b6759fd55b59166 2023-09-23T20:45:05.154 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:04 smithi026 ceph-mon[175151]: rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-a/store.db/000025.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2023-09-23T20:45:05.155 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:04 smithi026 ceph-mon[175151]: rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x559e12fb2000 2023-09-23T20:45:05.155 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:04 smithi026 ceph-mon[175151]: rocksdb: DB pointer 0x559e12f9e000 2023-09-23T20:45:05.155 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:04 smithi026 ceph-mon[175151]: rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2023-09-23T20:45:05.155 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:04 smithi026 ceph-mon[175151]: rocksdb: [db/db_impl/db_impl.cc:1111] 2023-09-23T20:45:05.155 INFO:journalctl@ceph.mon.a.smithi026.stdout: ** DB Stats ** 2023-09-23T20:45:05.155 INFO:journalctl@ceph.mon.a.smithi026.stdout: Uptime(secs): 0.0 total, 0.0 interval 2023-09-23T20:45:05.155 INFO:journalctl@ceph.mon.a.smithi026.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2023-09-23T20:45:05.155 INFO:journalctl@ceph.mon.a.smithi026.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2023-09-23T20:45:05.155 INFO:journalctl@ceph.mon.a.smithi026.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2023-09-23T20:45:05.155 INFO:journalctl@ceph.mon.a.smithi026.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2023-09-23T20:45:05.155 INFO:journalctl@ceph.mon.a.smithi026.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2023-09-23T20:45:05.155 INFO:journalctl@ceph.mon.a.smithi026.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2023-09-23T20:45:05.156 INFO:journalctl@ceph.mon.a.smithi026.stdout: 2023-09-23T20:45:05.156 INFO:journalctl@ceph.mon.a.smithi026.stdout: ** Compaction Stats [default] ** 2023-09-23T20:45:05.156 INFO:journalctl@ceph.mon.a.smithi026.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2023-09-23T20:45:05.156 INFO:journalctl@ceph.mon.a.smithi026.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2023-09-23T20:45:05.156 INFO:journalctl@ceph.mon.a.smithi026.stdout: L0 1/0 2.21 MB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 581.0 0.00 0.00 1 0.004 0 0 0.0 0.0 2023-09-23T20:45:05.156 INFO:journalctl@ceph.mon.a.smithi026.stdout: L6 1/0 6.48 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 0.0 0.0 2023-09-23T20:45:05.156 INFO:journalctl@ceph.mon.a.smithi026.stdout: Sum 2/0 8.69 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 581.0 0.00 0.00 1 0.004 0 0 0.0 0.0 2023-09-23T20:45:05.156 INFO:journalctl@ceph.mon.a.smithi026.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 581.0 0.00 0.00 1 0.004 0 0 0.0 0.0 2023-09-23T20:45:05.156 INFO:journalctl@ceph.mon.a.smithi026.stdout: 2023-09-23T20:45:05.156 INFO:journalctl@ceph.mon.a.smithi026.stdout: ** Compaction Stats [default] ** 2023-09-23T20:45:05.156 INFO:journalctl@ceph.mon.a.smithi026.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2023-09-23T20:45:05.156 INFO:journalctl@ceph.mon.a.smithi026.stdout: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-09-23T20:45:05.156 INFO:journalctl@ceph.mon.a.smithi026.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 581.0 0.00 0.00 1 0.004 0 0 0.0 0.0 2023-09-23T20:45:05.157 INFO:journalctl@ceph.mon.a.smithi026.stdout: 2023-09-23T20:45:05.157 INFO:journalctl@ceph.mon.a.smithi026.stdout: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2023-09-23T20:45:05.157 INFO:journalctl@ceph.mon.a.smithi026.stdout: 2023-09-23T20:45:05.157 INFO:journalctl@ceph.mon.a.smithi026.stdout: Uptime(secs): 0.0 total, 0.0 interval 2023-09-23T20:45:05.157 INFO:journalctl@ceph.mon.a.smithi026.stdout: Flush(GB): cumulative 0.002, interval 0.002 2023-09-23T20:45:05.157 INFO:journalctl@ceph.mon.a.smithi026.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2023-09-23T20:45:05.157 INFO:journalctl@ceph.mon.a.smithi026.stdout: AddFile(Total Files): cumulative 0, interval 0 2023-09-23T20:45:05.157 INFO:journalctl@ceph.mon.a.smithi026.stdout: AddFile(L0 Files): cumulative 0, interval 0 2023-09-23T20:45:05.157 INFO:journalctl@ceph.mon.a.smithi026.stdout: AddFile(Keys): cumulative 0, interval 0 2023-09-23T20:45:05.157 INFO:journalctl@ceph.mon.a.smithi026.stdout: Cumulative compaction: 0.00 GB write, 220.23 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-09-23T20:45:05.157 INFO:journalctl@ceph.mon.a.smithi026.stdout: Interval compaction: 0.00 GB write, 220.23 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-09-23T20:45:05.157 INFO:journalctl@ceph.mon.a.smithi026.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2023-09-23T20:45:05.157 INFO:journalctl@ceph.mon.a.smithi026.stdout: Block cache BinnedLRUCache@0x559e12f15350#7 capacity: 512.00 MB usage: 5.61 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 1.1e-05 secs_since: 0 2023-09-23T20:45:05.158 INFO:journalctl@ceph.mon.a.smithi026.stdout: Block cache entry stats(count,size,portion): FilterBlock(1,1.53 KB,0.000292063%) IndexBlock(1,4.08 KB,0.000777841%) Misc(1,0.00 KB,0%) 2023-09-23T20:45:05.158 INFO:journalctl@ceph.mon.a.smithi026.stdout: 2023-09-23T20:45:05.158 INFO:journalctl@ceph.mon.a.smithi026.stdout: ** File Read Latency Histogram By Level [default] ** 2023-09-23T20:45:05.158 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:04 smithi026 ceph-mon[175151]: starting mon.a rank 0 at public addrs [v2:172.21.15.26:3300/0,v1:172.21.15.26:6789/0] at bind addrs [v2:172.21.15.26:3300/0,v1:172.21.15.26:6789/0] mon_data /var/lib/ceph/mon/ceph-a fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 2023-09-23T20:45:05.158 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:04 smithi026 ceph-mon[175151]: mon.a@-1(???) e3 preinit fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 2023-09-23T20:45:05.158 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:04 smithi026 ceph-mon[175151]: mon.a@-1(???).mds e8 new map 2023-09-23T20:45:05.158 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:04 smithi026 ceph-mon[175151]: mon.a@-1(???).mds e8 print_map 2023-09-23T20:45:05.158 INFO:journalctl@ceph.mon.a.smithi026.stdout: e8 2023-09-23T20:45:05.158 INFO:journalctl@ceph.mon.a.smithi026.stdout: enable_multiple, ever_enabled_multiple: 1,1 2023-09-23T20:45:05.158 INFO:journalctl@ceph.mon.a.smithi026.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2023-09-23T20:45:05.158 INFO:journalctl@ceph.mon.a.smithi026.stdout: legacy client fscid: 1 2023-09-23T20:45:05.158 INFO:journalctl@ceph.mon.a.smithi026.stdout: 2023-09-23T20:45:05.159 INFO:journalctl@ceph.mon.a.smithi026.stdout: Filesystem 'foo' (1) 2023-09-23T20:45:05.159 INFO:journalctl@ceph.mon.a.smithi026.stdout: fs_name foo 2023-09-23T20:45:05.159 INFO:journalctl@ceph.mon.a.smithi026.stdout: epoch 7 2023-09-23T20:45:05.159 INFO:journalctl@ceph.mon.a.smithi026.stdout: flags 12 joinable allow_snaps allow_multimds_snaps 2023-09-23T20:45:05.159 INFO:journalctl@ceph.mon.a.smithi026.stdout: created 2023-09-23T20:41:58.904985+0000 2023-09-23T20:45:05.159 INFO:journalctl@ceph.mon.a.smithi026.stdout: modified 2023-09-23T20:42:07.570998+0000 2023-09-23T20:45:05.159 INFO:journalctl@ceph.mon.a.smithi026.stdout: tableserver 0 2023-09-23T20:45:05.159 INFO:journalctl@ceph.mon.a.smithi026.stdout: root 0 2023-09-23T20:45:05.159 INFO:journalctl@ceph.mon.a.smithi026.stdout: session_timeout 60 2023-09-23T20:45:05.159 INFO:journalctl@ceph.mon.a.smithi026.stdout: session_autoclose 300 2023-09-23T20:45:05.159 INFO:journalctl@ceph.mon.a.smithi026.stdout: max_file_size 1099511627776 2023-09-23T20:45:05.159 INFO:journalctl@ceph.mon.a.smithi026.stdout: required_client_features {} 2023-09-23T20:45:05.159 INFO:journalctl@ceph.mon.a.smithi026.stdout: last_failure 0 2023-09-23T20:45:05.160 INFO:journalctl@ceph.mon.a.smithi026.stdout: last_failure_osd_epoch 0 2023-09-23T20:45:05.160 INFO:journalctl@ceph.mon.a.smithi026.stdout: compat compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,7=mds uses inline data,8=no anchor table,9=file layout v2,10=snaprealm v2} 2023-09-23T20:45:05.160 INFO:journalctl@ceph.mon.a.smithi026.stdout: max_mds 1 2023-09-23T20:45:05.160 INFO:journalctl@ceph.mon.a.smithi026.stdout: in 0 2023-09-23T20:45:05.160 INFO:journalctl@ceph.mon.a.smithi026.stdout: up {0=14658} 2023-09-23T20:45:05.160 INFO:journalctl@ceph.mon.a.smithi026.stdout: failed 2023-09-23T20:45:05.160 INFO:journalctl@ceph.mon.a.smithi026.stdout: damaged 2023-09-23T20:45:05.160 INFO:journalctl@ceph.mon.a.smithi026.stdout: stopped 2023-09-23T20:45:05.160 INFO:journalctl@ceph.mon.a.smithi026.stdout: data_pools [4] 2023-09-23T20:45:05.160 INFO:journalctl@ceph.mon.a.smithi026.stdout: metadata_pool 3 2023-09-23T20:45:05.160 INFO:journalctl@ceph.mon.a.smithi026.stdout: inline_data disabled 2023-09-23T20:45:05.160 INFO:journalctl@ceph.mon.a.smithi026.stdout: balancer 2023-09-23T20:45:05.160 INFO:journalctl@ceph.mon.a.smithi026.stdout: bal_rank_mask -1 2023-09-23T20:45:05.161 INFO:journalctl@ceph.mon.a.smithi026.stdout: standby_count_wanted 1 2023-09-23T20:45:05.161 INFO:journalctl@ceph.mon.a.smithi026.stdout: [mds.foo.smithi026.znfpnl{0:14658} state up:active seq 3 join_fscid=1 addr [v2:172.21.15.26:6834/417830853,v1:172.21.15.26:6835/417830853] compat {c=[1],r=[1],i=[7ff]}] 2023-09-23T20:45:05.161 INFO:journalctl@ceph.mon.a.smithi026.stdout: 2023-09-23T20:45:05.161 INFO:journalctl@ceph.mon.a.smithi026.stdout: 2023-09-23T20:45:05.161 INFO:journalctl@ceph.mon.a.smithi026.stdout: Standby daemons: 2023-09-23T20:45:05.161 INFO:journalctl@ceph.mon.a.smithi026.stdout: 2023-09-23T20:45:05.161 INFO:journalctl@ceph.mon.a.smithi026.stdout: [mds.foo.smithi070.imqhqy{-1:24529} state up:standby seq 2 join_fscid=1 addr [v2:172.21.15.70:6832/241309560,v1:172.21.15.70:6833/241309560] compat {c=[1],r=[1],i=[7ff]}] 2023-09-23T20:45:05.161 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:04 smithi026 ceph-mon[175151]: mon.a@-1(???).osd e78 crush map has features 3314933000854323200, adjusting msgr requires 2023-09-23T20:45:05.161 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:04 smithi026 ceph-mon[175151]: mon.a@-1(???).osd e78 crush map has features 432629239337189376, adjusting msgr requires 2023-09-23T20:45:05.161 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:04 smithi026 ceph-mon[175151]: mon.a@-1(???).osd e78 crush map has features 432629239337189376, adjusting msgr requires 2023-09-23T20:45:05.161 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:04 smithi026 ceph-mon[175151]: mon.a@-1(???).osd e78 crush map has features 432629239337189376, adjusting msgr requires 2023-09-23T20:45:05.161 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:04 smithi026 ceph-mon[175151]: mon.a@-1(???).paxosservice(auth 1..18) refresh upgraded, format 0 -> 3 2023-09-23T20:45:05.161 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:04 smithi026 systemd[1]: Started Ceph mon.a for b3f8e4ec-5a50-11ee-8db4-212e2dc638e7. 2023-09-23T20:45:08.028 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:45:07 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[164029]: ignoring --setuser ceph since I am not root 2023-09-23T20:45:08.028 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:45:07 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[164029]: ignoring --setgroup ceph since I am not root 2023-09-23T20:45:08.028 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:45:07 smithi026 ceph-mgr[164053]: -- 172.21.15.26:0/2355070152 <== mon.0 v2:172.21.15.26:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x55bbe9b231e0 con 0x55bbe9af8800 2023-09-23T20:45:08.028 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:45:07 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[164029]: 2023-09-23T20:45:07.929+0000 7fd5e5868200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2023-09-23T20:45:08.028 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:45:08 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[164029]: 2023-09-23T20:45:08.026+0000 7fd5e5868200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-09-23T20:45:08.028 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:45:07 smithi026 ceph-mon[109912]: pgmap v18: 73 pgs: 73 active+clean; 581 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:45:08.029 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:45:07 smithi026 ceph-mon[109912]: from='client.? 172.21.15.26:0/1628749677' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:45:08.029 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:45:07 smithi026 ceph-mon[109912]: pgmap v19: 73 pgs: 73 active+clean; 581 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:45:08.029 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:45:07 smithi026 ceph-mon[109912]: from='client.? 172.21.15.26:0/3442381556' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:45:08.029 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:45:07 smithi026 ceph-mon[109912]: mon.a calling monitor election 2023-09-23T20:45:08.029 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:45:07 smithi026 ceph-mon[109912]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2023-09-23T20:45:08.029 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:45:07 smithi026 ceph-mon[109912]: monmap e3: 3 mons at {a=[v2:172.21.15.26:3300/0,v1:172.21.15.26:6789/0],b=[v2:172.21.15.70:3300/0,v1:172.21.15.70:6789/0],c=[v2:172.21.15.26:3301/0,v1:172.21.15.26:6790/0]} removed_ranks: {} 2023-09-23T20:45:08.029 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:45:07 smithi026 ceph-mon[109912]: fsmap foo:1 {0=foo.smithi026.znfpnl=up:active} 1 up:standby 2023-09-23T20:45:08.029 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:45:07 smithi026 ceph-mon[109912]: osdmap e78: 8 total, 8 up, 8 in 2023-09-23T20:45:08.029 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:45:07 smithi026 ceph-mon[109912]: mgrmap e31: y(active, since 34s), standbys: x 2023-09-23T20:45:08.029 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:45:07 smithi026 ceph-mon[109912]: mgrmap e32: y(active, since 34s), standbys: x 2023-09-23T20:45:08.029 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:45:07 smithi026 ceph-mon[109912]: overall HEALTH_OK 2023-09-23T20:45:08.030 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:45:07 smithi026 ceph-mon[109912]: from='mgr.24989 ' entity='' 2023-09-23T20:45:08.030 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:07 smithi026 ceph-mon[175151]: pgmap v18: 73 pgs: 73 active+clean; 581 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:45:08.030 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:07 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/1628749677' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:45:08.030 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:07 smithi026 ceph-mon[175151]: pgmap v19: 73 pgs: 73 active+clean; 581 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:45:08.030 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:07 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/3442381556' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:45:08.030 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:07 smithi026 ceph-mon[175151]: mon.a calling monitor election 2023-09-23T20:45:08.030 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:07 smithi026 ceph-mon[175151]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2023-09-23T20:45:08.030 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:07 smithi026 ceph-mon[175151]: monmap e3: 3 mons at {a=[v2:172.21.15.26:3300/0,v1:172.21.15.26:6789/0],b=[v2:172.21.15.70:3300/0,v1:172.21.15.70:6789/0],c=[v2:172.21.15.26:3301/0,v1:172.21.15.26:6790/0]} removed_ranks: {} 2023-09-23T20:45:08.030 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:07 smithi026 ceph-mon[175151]: fsmap foo:1 {0=foo.smithi026.znfpnl=up:active} 1 up:standby 2023-09-23T20:45:08.031 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:07 smithi026 ceph-mon[175151]: osdmap e78: 8 total, 8 up, 8 in 2023-09-23T20:45:08.031 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:07 smithi026 ceph-mon[175151]: mgrmap e31: y(active, since 34s), standbys: x 2023-09-23T20:45:08.031 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:07 smithi026 ceph-mon[175151]: mgrmap e32: y(active, since 34s), standbys: x 2023-09-23T20:45:08.031 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:07 smithi026 ceph-mon[175151]: overall HEALTH_OK 2023-09-23T20:45:08.031 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:07 smithi026 ceph-mon[175151]: from='mgr.24989 ' entity='' 2023-09-23T20:45:08.064 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 20:45:07 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-x[156534]: ignoring --setuser ceph since I am not root 2023-09-23T20:45:08.064 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 20:45:07 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-x[156534]: ignoring --setgroup ceph since I am not root 2023-09-23T20:45:08.064 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 20:45:07 smithi070 ceph-mgr[156558]: -- 172.21.15.70:0/724322811 <== mon.1 v2:172.21.15.70:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x55dcc80731e0 con 0x55dcc8048800 2023-09-23T20:45:08.064 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 20:45:07 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-x[156534]: 2023-09-23T20:45:07.894+0000 7fde95c11200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2023-09-23T20:45:08.064 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 20:45:07 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-x[156534]: 2023-09-23T20:45:07.973+0000 7fde95c11200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-09-23T20:45:08.064 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 20:45:08 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-x[156534]: 2023-09-23T20:45:08.062+0000 7fde95c11200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-09-23T20:45:08.065 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:45:07 smithi070 ceph-mon[100755]: pgmap v18: 73 pgs: 73 active+clean; 581 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:45:08.065 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:45:07 smithi070 ceph-mon[100755]: from='client.? 172.21.15.26:0/1628749677' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:45:08.065 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:45:07 smithi070 ceph-mon[100755]: pgmap v19: 73 pgs: 73 active+clean; 581 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:45:08.065 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:45:07 smithi070 ceph-mon[100755]: from='client.? 172.21.15.26:0/3442381556' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:45:08.065 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:45:07 smithi070 ceph-mon[100755]: mon.a calling monitor election 2023-09-23T20:45:08.065 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:45:07 smithi070 ceph-mon[100755]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2023-09-23T20:45:08.065 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:45:07 smithi070 ceph-mon[100755]: monmap e3: 3 mons at {a=[v2:172.21.15.26:3300/0,v1:172.21.15.26:6789/0],b=[v2:172.21.15.70:3300/0,v1:172.21.15.70:6789/0],c=[v2:172.21.15.26:3301/0,v1:172.21.15.26:6790/0]} removed_ranks: {} 2023-09-23T20:45:08.065 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:45:07 smithi070 ceph-mon[100755]: fsmap foo:1 {0=foo.smithi026.znfpnl=up:active} 1 up:standby 2023-09-23T20:45:08.065 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:45:07 smithi070 ceph-mon[100755]: osdmap e78: 8 total, 8 up, 8 in 2023-09-23T20:45:08.066 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:45:07 smithi070 ceph-mon[100755]: mgrmap e31: y(active, since 34s), standbys: x 2023-09-23T20:45:08.066 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:45:07 smithi070 ceph-mon[100755]: mgrmap e32: y(active, since 34s), standbys: x 2023-09-23T20:45:08.066 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:45:07 smithi070 ceph-mon[100755]: overall HEALTH_OK 2023-09-23T20:45:08.066 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:45:07 smithi070 ceph-mon[100755]: from='mgr.24989 ' entity='' 2023-09-23T20:45:08.327 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 20:45:08 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-x[156534]: 2023-09-23T20:45:08.157+0000 7fde95c11200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-09-23T20:45:08.382 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:45:08 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[164029]: 2023-09-23T20:45:08.140+0000 7fd5e5868200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-09-23T20:45:08.382 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:45:08 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[164029]: 2023-09-23T20:45:08.253+0000 7fd5e5868200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-09-23T20:45:08.386 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 18.2.0-460-gbb0daeff (bb0daeff97f020f686596fca6eba20c310e26fb7) reef (stable)": 1 2023-09-23T20:45:08.619 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 20:45:08 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-x[156534]: 2023-09-23T20:45:08.326+0000 7fde95c11200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-09-23T20:45:08.761 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:45:08 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[164029]: 2023-09-23T20:45:08.469+0000 7fd5e5868200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-09-23T20:45:08.806 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -e sha1=bb0daeff97f020f686596fca6eba20c310e26fb7 -- bash -c 'ceph orch ps' 2023-09-23T20:45:09.004 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:45:08 smithi070 ceph-mon[100755]: from='client.? 172.21.15.26:0/2389149347' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:45:09.004 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 20:45:08 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-x[156534]: 2023-09-23T20:45:08.687+0000 7fde95c11200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-09-23T20:45:09.004 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 20:45:08 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-x[156534]: 2023-09-23T20:45:08.820+0000 7fde95c11200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-09-23T20:45:09.005 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 20:45:08 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-x[156534]: 2023-09-23T20:45:08.907+0000 7fde95c11200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-09-23T20:45:09.005 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 20:45:08 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-prometheus-a[155975]: ts=2023-09-23T20:45:08.658Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1123-g61431f6b (61431f6ba2752649b506867ec856b4a6236f90dd) quincy (stable)\", cluster_addr=\"172.21.15.26\", device_class=\"ssd\", hostname=\"smithi026\", instance=\"smithi070.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.26\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1123-g61431f6b (61431f6ba2752649b506867ec856b4a6236f90dd) quincy (stable)\", cluster_addr=\"172.21.15.26\", device_class=\"ssd\", hostname=\"smithi026\", instance=\"smithi026:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.26\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-09-23T20:45:09.118 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:45:08 smithi026 ceph-mon[109912]: from='client.? 172.21.15.26:0/2389149347' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:45:09.118 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:08 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/2389149347' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:45:09.118 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:45:08 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[164029]: 2023-09-23T20:45:08.953+0000 7fd5e5868200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-09-23T20:45:09.369 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 20:45:09 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-x[156534]: 2023-09-23T20:45:09.003+0000 7fde95c11200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-09-23T20:45:09.381 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:45:09 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[164029]: 2023-09-23T20:45:09.116+0000 7fd5e5868200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-09-23T20:45:09.382 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:45:09 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[164029]: 2023-09-23T20:45:09.231+0000 7fd5e5868200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-09-23T20:45:09.382 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:45:09 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[164029]: 2023-09-23T20:45:09.351+0000 7fd5e5868200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-09-23T20:45:09.440 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:45:09.869 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 20:45:09 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-x[156534]: 2023-09-23T20:45:09.422+0000 7fde95c11200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-09-23T20:45:10.001 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:45:09 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[164029]: 2023-09-23T20:45:09.883+0000 7fd5e5868200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-09-23T20:45:10.369 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 20:45:10 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-x[156534]: 2023-09-23T20:45:10.074+0000 7fde95c11200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-09-23T20:45:10.370 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 20:45:10 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-x[156534]: 2023-09-23T20:45:10.158+0000 7fde95c11200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-09-23T20:45:10.882 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:45:10 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[164029]: 2023-09-23T20:45:10.696+0000 7fd5e5868200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-09-23T20:45:10.882 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:45:10 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[164029]: 2023-09-23T20:45:10.803+0000 7fd5e5868200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-09-23T20:45:11.200 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 20:45:10 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-x[156534]: 2023-09-23T20:45:10.931+0000 7fde95c11200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-09-23T20:45:11.200 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 20:45:11 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-x[156534]: 2023-09-23T20:45:11.024+0000 7fde95c11200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2023-09-23T20:45:11.485 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 20:45:11 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-x[156534]: 2023-09-23T20:45:11.199+0000 7fde95c11200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-09-23T20:45:11.860 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 20:45:11 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-x[156534]: 2023-09-23T20:45:11.484+0000 7fde95c11200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2023-09-23T20:45:11.860 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 20:45:11 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-x[156534]: 2023-09-23T20:45:11.710+0000 7fde95c11200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-09-23T20:45:12.119 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 20:45:11 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-x[156534]: 2023-09-23T20:45:11.859+0000 7fde95c11200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-09-23T20:45:12.130 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:45:11 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[164029]: 2023-09-23T20:45:11.790+0000 7fd5e5868200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-09-23T20:45:12.130 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:45:11 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[164029]: 2023-09-23T20:45:11.903+0000 7fd5e5868200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2023-09-23T20:45:12.130 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:45:12 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[164029]: 2023-09-23T20:45:12.128+0000 7fd5e5868200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-09-23T20:45:12.619 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 20:45:12 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-x[156534]: 2023-09-23T20:45:12.127+0000 7fde95c11200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-09-23T20:45:12.745 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:45:12 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[164029]: 2023-09-23T20:45:12.486+0000 7fd5e5868200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2023-09-23T20:45:13.119 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 20:45:12 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-x[156534]: 2023-09-23T20:45:12.831+0000 7fde95c11200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-09-23T20:45:13.119 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 20:45:12 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-x[156534]: 2023-09-23T20:45:12.962+0000 7fde95c11200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-09-23T20:45:13.131 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:45:12 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[164029]: 2023-09-23T20:45:12.744+0000 7fd5e5868200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-09-23T20:45:13.132 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:45:12 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[164029]: 2023-09-23T20:45:12.930+0000 7fd5e5868200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-09-23T20:45:13.499 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 20:45:13 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-x[156534]: 2023-09-23T20:45:13.160+0000 7fde95c11200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-09-23T20:45:13.499 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 20:45:13 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-x[156534]: 2023-09-23T20:45:13.276+0000 7fde95c11200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-09-23T20:45:13.632 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:45:13 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[164029]: 2023-09-23T20:45:13.269+0000 7fd5e5868200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-09-23T20:45:13.869 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 20:45:13 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-x[156534]: 2023-09-23T20:45:13.498+0000 7fde95c11200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-09-23T20:45:14.381 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:45:14 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[164029]: 2023-09-23T20:45:14.110+0000 7fd5e5868200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-09-23T20:45:14.382 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:45:14 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[164029]: 2023-09-23T20:45:14.272+0000 7fd5e5868200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-09-23T20:45:14.621 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:45:14 smithi070 ceph-mon[100755]: Standby manager daemon x restarted 2023-09-23T20:45:14.622 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:45:14 smithi070 ceph-mon[100755]: Standby manager daemon x started 2023-09-23T20:45:14.622 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:45:14 smithi070 ceph-mon[100755]: from='mgr.? 172.21.15.70:0/335003092' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-09-23T20:45:14.622 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:45:14 smithi070 ceph-mon[100755]: from='mgr.? 172.21.15.70:0/335003092' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-09-23T20:45:14.622 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:45:14 smithi070 ceph-mon[100755]: from='mgr.? 172.21.15.70:0/335003092' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-09-23T20:45:14.622 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:45:14 smithi070 ceph-mon[100755]: from='mgr.? 172.21.15.70:0/335003092' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-09-23T20:45:14.622 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 20:45:14 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-x[156534]: 2023-09-23T20:45:14.362+0000 7fde95c11200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-09-23T20:45:14.622 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 20:45:14 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-x[156534]: [23/Sep/2023:20:45:14] ENGINE Bus STARTING 2023-09-23T20:45:14.622 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 20:45:14 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-x[156534]: CherryPy Checker: 2023-09-23T20:45:14.623 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 20:45:14 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-x[156534]: The Application mounted at '' has an empty config. 2023-09-23T20:45:14.623 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 20:45:14 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-x[156534]: 2023-09-23T20:45:14.623 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 20:45:14 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-x[156534]: [23/Sep/2023:20:45:14] ENGINE Serving on http://:::9283 2023-09-23T20:45:14.623 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 20:45:14 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-x[156534]: [23/Sep/2023:20:45:14] ENGINE Bus STARTED 2023-09-23T20:45:14.882 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:45:14 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[164029]: 2023-09-23T20:45:14.529+0000 7fd5e5868200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-09-23T20:45:14.882 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:45:14 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[164029]: 2023-09-23T20:45:14.670+0000 7fd5e5868200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-09-23T20:45:14.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:14 smithi026 ceph-mon[175151]: Standby manager daemon x restarted 2023-09-23T20:45:14.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:14 smithi026 ceph-mon[175151]: Standby manager daemon x started 2023-09-23T20:45:14.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:14 smithi026 ceph-mon[175151]: from='mgr.? 172.21.15.70:0/335003092' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-09-23T20:45:14.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:14 smithi026 ceph-mon[175151]: from='mgr.? 172.21.15.70:0/335003092' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-09-23T20:45:14.883 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:14 smithi026 ceph-mon[175151]: from='mgr.? 172.21.15.70:0/335003092' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-09-23T20:45:14.883 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:14 smithi026 ceph-mon[175151]: from='mgr.? 172.21.15.70:0/335003092' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-09-23T20:45:14.883 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:45:14 smithi026 ceph-mon[109912]: Standby manager daemon x restarted 2023-09-23T20:45:14.883 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:45:14 smithi026 ceph-mon[109912]: Standby manager daemon x started 2023-09-23T20:45:14.883 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:45:14 smithi026 ceph-mon[109912]: from='mgr.? 172.21.15.70:0/335003092' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-09-23T20:45:14.883 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:45:14 smithi026 ceph-mon[109912]: from='mgr.? 172.21.15.70:0/335003092' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-09-23T20:45:14.883 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:45:14 smithi026 ceph-mon[109912]: from='mgr.? 172.21.15.70:0/335003092' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-09-23T20:45:14.883 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:45:14 smithi026 ceph-mon[109912]: from='mgr.? 172.21.15.70:0/335003092' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-09-23T20:45:15.382 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:45:14 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[164029]: 2023-09-23T20:45:14.956+0000 7fd5e5868200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-09-23T20:45:15.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:45:15 smithi070 ceph-mon[100755]: mgrmap e33: y(active, since 41s), standbys: x 2023-09-23T20:45:15.881 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:15 smithi026 ceph-mon[175151]: mgrmap e33: y(active, since 41s), standbys: x 2023-09-23T20:45:15.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:45:15 smithi026 ceph-mon[109912]: mgrmap e33: y(active, since 41s), standbys: x 2023-09-23T20:45:16.275 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:45:16 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[164029]: 2023-09-23T20:45:16.022+0000 7fd5e5868200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-09-23T20:45:16.275 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:45:16 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[164029]: [23/Sep/2023:20:45:16] ENGINE Bus STARTING 2023-09-23T20:45:16.275 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:45:16 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[164029]: 2023-09-23T20:45:16.198+0000 7fd53fbd4700 -1 client.0 error registering admin socket command: (17) File exists 2023-09-23T20:45:16.275 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:45:16 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[164029]: 2023-09-23T20:45:16.198+0000 7fd53fbd4700 -1 client.0 error registering admin socket command: (17) File exists 2023-09-23T20:45:16.275 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:45:16 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[164029]: 2023-09-23T20:45:16.198+0000 7fd53fbd4700 -1 client.0 error registering admin socket command: (17) File exists 2023-09-23T20:45:16.275 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:45:16 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[164029]: 2023-09-23T20:45:16.198+0000 7fd53fbd4700 -1 client.0 error registering admin socket command: (17) File exists 2023-09-23T20:45:16.275 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:45:16 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[164029]: 2023-09-23T20:45:16.198+0000 7fd53fbd4700 -1 client.0 error registering admin socket command: (17) File exists 2023-09-23T20:45:16.275 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:45:16 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[164029]: 2023-09-23T20:45:16.198+0000 7fd545be0700 -1 client.0 error registering admin socket command: (17) File exists 2023-09-23T20:45:16.276 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:45:16 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[164029]: 2023-09-23T20:45:16.198+0000 7fd545be0700 -1 client.0 error registering admin socket command: (17) File exists 2023-09-23T20:45:16.276 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:45:16 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[164029]: 2023-09-23T20:45:16.198+0000 7fd545be0700 -1 client.0 error registering admin socket command: (17) File exists 2023-09-23T20:45:16.276 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:45:16 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[164029]: 2023-09-23T20:45:16.198+0000 7fd545be0700 -1 client.0 error registering admin socket command: (17) File exists 2023-09-23T20:45:16.276 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:45:16 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[164029]: 2023-09-23T20:45:16.198+0000 7fd545be0700 -1 client.0 error registering admin socket command: (17) File exists 2023-09-23T20:45:16.276 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:45:16 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[164029]: CherryPy Checker: 2023-09-23T20:45:16.276 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:45:16 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[164029]: The Application mounted at '' has an empty config. 2023-09-23T20:45:16.276 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:45:16 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[164029]: 2023-09-23T20:45:16.276 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:45:16 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[164029]: [23/Sep/2023:20:45:16] ENGINE Serving on http://:::9283 2023-09-23T20:45:16.276 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:45:16 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[164029]: [23/Sep/2023:20:45:16] ENGINE Bus STARTED 2023-09-23T20:45:16.276 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:45:16 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mon-c[109888]: 2023-09-23T20:45:16.039+0000 7f31549a1700 -1 mon.c@2(peon).osd e79 update_from_paxos full map CRC mismatch, resetting to canonical 2023-09-23T20:45:16.294 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:45:16 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mon-b[100708]: 2023-09-23T20:45:16.039+0000 7f4bbfa06700 -1 mon.b@1(peon).osd e79 update_from_paxos full map CRC mismatch, resetting to canonical 2023-09-23T20:45:16.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:45:16 smithi070 ceph-mon[100755]: Active manager daemon y restarted 2023-09-23T20:45:16.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:45:16 smithi070 ceph-mon[100755]: Activating manager daemon y 2023-09-23T20:45:16.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:45:16 smithi070 ceph-mon[100755]: osdmap e79: 8 total, 8 up, 8 in 2023-09-23T20:45:16.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:45:16 smithi070 ceph-mon[100755]: mgrmap e34: y(active, starting, since 0.00507152s), standbys: x 2023-09-23T20:45:16.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:45:16 smithi070 ceph-mon[100755]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-23T20:45:16.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:45:16 smithi070 ceph-mon[100755]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T20:45:16.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:45:16 smithi070 ceph-mon[100755]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T20:45:16.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:45:16 smithi070 ceph-mon[100755]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi026.znfpnl"}]: dispatch 2023-09-23T20:45:16.621 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:45:16 smithi070 ceph-mon[100755]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi070.imqhqy"}]: dispatch 2023-09-23T20:45:16.621 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:45:16 smithi070 ceph-mon[100755]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-09-23T20:45:16.621 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:45:16 smithi070 ceph-mon[100755]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-09-23T20:45:16.621 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:45:16 smithi070 ceph-mon[100755]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-23T20:45:16.621 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:45:16 smithi070 ceph-mon[100755]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-23T20:45:16.621 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:45:16 smithi070 ceph-mon[100755]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-23T20:45:16.621 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:45:16 smithi070 ceph-mon[100755]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-23T20:45:16.621 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:45:16 smithi070 ceph-mon[100755]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-09-23T20:45:16.621 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:45:16 smithi070 ceph-mon[100755]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-09-23T20:45:16.622 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:45:16 smithi070 ceph-mon[100755]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-09-23T20:45:16.622 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:45:16 smithi070 ceph-mon[100755]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-09-23T20:45:16.622 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:45:16 smithi070 ceph-mon[100755]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-09-23T20:45:16.622 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:45:16 smithi070 ceph-mon[100755]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-09-23T20:45:16.622 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:45:16 smithi070 ceph-mon[100755]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-09-23T20:45:16.622 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:45:16 smithi070 ceph-mon[100755]: Manager daemon y is now available 2023-09-23T20:45:16.622 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:45:16 smithi070 ceph-mon[100755]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:45:16.622 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:45:16 smithi070 ceph-mon[100755]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:45:16.622 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:45:16 smithi070 ceph-mon[100755]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:45:16.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:45:16 smithi070 ceph-mon[100755]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-09-23T20:45:16.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:45:16 smithi070 ceph-mon[100755]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-09-23T20:45:16.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:45:16 smithi026 ceph-mon[109912]: Active manager daemon y restarted 2023-09-23T20:45:16.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:45:16 smithi026 ceph-mon[109912]: Activating manager daemon y 2023-09-23T20:45:16.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:45:16 smithi026 ceph-mon[109912]: osdmap e79: 8 total, 8 up, 8 in 2023-09-23T20:45:16.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:45:16 smithi026 ceph-mon[109912]: mgrmap e34: y(active, starting, since 0.00507152s), standbys: x 2023-09-23T20:45:16.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:45:16 smithi026 ceph-mon[109912]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-23T20:45:16.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:45:16 smithi026 ceph-mon[109912]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T20:45:16.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:45:16 smithi026 ceph-mon[109912]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T20:45:16.633 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:45:16 smithi026 ceph-mon[109912]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi026.znfpnl"}]: dispatch 2023-09-23T20:45:16.633 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:45:16 smithi026 ceph-mon[109912]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi070.imqhqy"}]: dispatch 2023-09-23T20:45:16.633 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:45:16 smithi026 ceph-mon[109912]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-09-23T20:45:16.633 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:45:16 smithi026 ceph-mon[109912]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-09-23T20:45:16.633 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:45:16 smithi026 ceph-mon[109912]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-23T20:45:16.633 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:45:16 smithi026 ceph-mon[109912]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-23T20:45:16.633 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:45:16 smithi026 ceph-mon[109912]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-23T20:45:16.633 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:45:16 smithi026 ceph-mon[109912]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-23T20:45:16.633 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:45:16 smithi026 ceph-mon[109912]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-09-23T20:45:16.633 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:45:16 smithi026 ceph-mon[109912]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-09-23T20:45:16.633 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:45:16 smithi026 ceph-mon[109912]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-09-23T20:45:16.633 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:45:16 smithi026 ceph-mon[109912]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-09-23T20:45:16.633 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:45:16 smithi026 ceph-mon[109912]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-09-23T20:45:16.634 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:45:16 smithi026 ceph-mon[109912]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-09-23T20:45:16.634 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:45:16 smithi026 ceph-mon[109912]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-09-23T20:45:16.634 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:45:16 smithi026 ceph-mon[109912]: Manager daemon y is now available 2023-09-23T20:45:16.634 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:45:16 smithi026 ceph-mon[109912]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:45:16.634 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:45:16 smithi026 ceph-mon[109912]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:45:16.634 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:45:16 smithi026 ceph-mon[109912]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:45:16.634 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:45:16 smithi026 ceph-mon[109912]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-09-23T20:45:16.634 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:45:16 smithi026 ceph-mon[109912]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-09-23T20:45:16.634 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:16 smithi026 ceph-mon[175151]: Active manager daemon y restarted 2023-09-23T20:45:16.634 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:16 smithi026 ceph-mon[175151]: Activating manager daemon y 2023-09-23T20:45:16.634 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:16 smithi026 ceph-mon[175151]: osdmap e79: 8 total, 8 up, 8 in 2023-09-23T20:45:16.635 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:16 smithi026 ceph-mon[175151]: mgrmap e34: y(active, starting, since 0.00507152s), standbys: x 2023-09-23T20:45:16.635 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:16 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-23T20:45:16.635 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:16 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T20:45:16.635 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:16 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T20:45:16.635 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:16 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi026.znfpnl"}]: dispatch 2023-09-23T20:45:16.635 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:16 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi070.imqhqy"}]: dispatch 2023-09-23T20:45:16.635 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:16 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-09-23T20:45:16.635 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:16 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-09-23T20:45:16.635 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:16 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-23T20:45:16.635 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:16 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-23T20:45:16.635 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:16 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-23T20:45:16.635 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:16 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-23T20:45:16.635 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:16 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-09-23T20:45:16.636 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:16 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-09-23T20:45:16.636 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:16 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-09-23T20:45:16.636 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:16 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-09-23T20:45:16.636 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:16 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-09-23T20:45:16.636 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:16 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-09-23T20:45:16.636 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:16 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-09-23T20:45:16.636 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:16 smithi026 ceph-mon[175151]: Manager daemon y is now available 2023-09-23T20:45:16.636 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:16 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:45:16.636 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:16 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:45:16.636 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:16 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:45:16.636 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:16 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-09-23T20:45:16.636 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:16 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-09-23T20:45:17.067 INFO:teuthology.orchestra.run.smithi026.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2023-09-23T20:45:17.067 INFO:teuthology.orchestra.run.smithi026.stdout:alertmanager.a smithi026 *:9093,9094 running (69s) - 4m 18.0M - 0.25.0 c8568f914cd2 d4b48de0f10c 2023-09-23T20:45:17.067 INFO:teuthology.orchestra.run.smithi026.stdout:grafana.a smithi070 *:3000 running (58s) - 3m 81.0M - 9.4.7 2c41d148cca3 ed7e189354a5 2023-09-23T20:45:17.067 INFO:teuthology.orchestra.run.smithi026.stdout:mds.foo.smithi026.znfpnl smithi026 running (3m) - 3m 17.9M - 17.2.6-1123-g61431f6b 0db14aa01094 b6861ac0ff55 2023-09-23T20:45:17.067 INFO:teuthology.orchestra.run.smithi026.stdout:mds.foo.smithi070.imqhqy smithi070 running (3m) - 3m 17.0M - 17.2.6-1123-g61431f6b 0db14aa01094 ca26bf551eb9 2023-09-23T20:45:17.068 INFO:teuthology.orchestra.run.smithi026.stdout:mgr.x smithi070 *:8443,9283,8765 running (26s) - 7m 58.7M - 18.2.0-460-gbb0daeff e32c2a64c126 35059b639121 2023-09-23T20:45:17.068 INFO:teuthology.orchestra.run.smithi026.stdout:mgr.y smithi026 *:8443,9283,8765 running (53s) - 9m 477M - 18.2.0-460-gbb0daeff e32c2a64c126 5cc7ece3b7e8 2023-09-23T20:45:17.068 INFO:teuthology.orchestra.run.smithi026.stdout:mon.a smithi026 running (9m) - 9m 94.7M 2048M 17.2.6-1123-g61431f6b 0db14aa01094 24cf5254b0b8 2023-09-23T20:45:17.068 INFO:teuthology.orchestra.run.smithi026.stdout:mon.b smithi070 running (7m) - 7m 71.3M 2048M 17.2.6-1123-g61431f6b 0db14aa01094 2c8a2924d3d7 2023-09-23T20:45:17.068 INFO:teuthology.orchestra.run.smithi026.stdout:mon.c smithi026 running (7m) - 7m 69.3M 2048M 17.2.6-1123-g61431f6b 0db14aa01094 069bb324bbad 2023-09-23T20:45:17.068 INFO:teuthology.orchestra.run.smithi026.stdout:node-exporter.a smithi026 *:9100 running (67s) - 3m 9198k - 1.5.0 0da6a335fe13 95269f65f9af 2023-09-23T20:45:17.068 INFO:teuthology.orchestra.run.smithi026.stdout:node-exporter.b smithi070 *:9100 running (61s) - 3m 14.8M - 1.5.0 0da6a335fe13 5b56a4a415d0 2023-09-23T20:45:17.068 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0 smithi026 running (7m) - 7m 73.8M 2501M 17.2.6-1123-g61431f6b 0db14aa01094 e40b0407b8cf 2023-09-23T20:45:17.068 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1 smithi026 running (6m) - 6m 74.2M 2501M 17.2.6-1123-g61431f6b 0db14aa01094 a73d47580165 2023-09-23T20:45:17.068 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2 smithi026 running (6m) - 6m 145M 2501M 17.2.6-1123-g61431f6b 0db14aa01094 fc4783fb1a89 2023-09-23T20:45:17.069 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3 smithi026 running (6m) - 6m 63.9M 2501M 17.2.6-1123-g61431f6b 0db14aa01094 5f0339ddc2c4 2023-09-23T20:45:17.069 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4 smithi070 running (5m) - 5m 83.0M 2500M 17.2.6-1123-g61431f6b 0db14aa01094 308a2af3101e 2023-09-23T20:45:17.069 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5 smithi070 running (5m) - 5m 97.1M 2500M 17.2.6-1123-g61431f6b 0db14aa01094 16cd72517077 2023-09-23T20:45:17.069 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6 smithi070 running (5m) - 5m 61.3M 2500M 17.2.6-1123-g61431f6b 0db14aa01094 5d24c21e8be7 2023-09-23T20:45:17.069 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7 smithi070 running (4m) - 4m 101M 2500M 17.2.6-1123-g61431f6b 0db14aa01094 8c1b7f314c41 2023-09-23T20:45:17.069 INFO:teuthology.orchestra.run.smithi026.stdout:prometheus.a smithi070 *:9095 running (30s) - 4m 41.0M - 2.43.0 a07b618ecd1d dda227f93524 2023-09-23T20:45:17.381 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:45:17 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[164029]: 2023-09-23T20:45:17.048+0000 7fd59efc8700 -1 mgr.server handle_report got status from non-daemon mon.a 2023-09-23T20:45:17.515 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -e sha1=bb0daeff97f020f686596fca6eba20c310e26fb7 -- bash -c 'ceph orch upgrade pause' 2023-09-23T20:45:17.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:45:17 smithi070 ceph-mon[100755]: failed to encode map e79 with expected crc 2023-09-23T20:45:17.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:45:17 smithi070 ceph-mon[100755]: failed to encode map e79 with expected crc 2023-09-23T20:45:17.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:45:17 smithi070 ceph-mon[100755]: failed to encode map e79 with expected crc 2023-09-23T20:45:17.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:45:17 smithi070 ceph-mon[100755]: failed to encode map e79 with expected crc 2023-09-23T20:45:17.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:45:17 smithi070 ceph-mon[100755]: failed to encode map e79 with expected crc 2023-09-23T20:45:17.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:45:17 smithi070 ceph-mon[100755]: [23/Sep/2023:20:45:16] ENGINE Bus STARTING 2023-09-23T20:45:17.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:45:17 smithi070 ceph-mon[100755]: [23/Sep/2023:20:45:16] ENGINE Serving on http://172.21.15.26:8765 2023-09-23T20:45:17.871 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:45:17 smithi070 ceph-mon[100755]: [23/Sep/2023:20:45:16] ENGINE Serving on https://172.21.15.26:7150 2023-09-23T20:45:17.871 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:45:17 smithi070 ceph-mon[100755]: [23/Sep/2023:20:45:16] ENGINE Bus STARTED 2023-09-23T20:45:17.871 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:45:17 smithi070 ceph-mon[100755]: mgrmap e35: y(active, since 1.00917s), standbys: x 2023-09-23T20:45:17.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:45:17 smithi026 ceph-mon[109912]: failed to encode map e79 with expected crc 2023-09-23T20:45:17.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:45:17 smithi026 ceph-mon[109912]: failed to encode map e79 with expected crc 2023-09-23T20:45:17.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:45:17 smithi026 ceph-mon[109912]: failed to encode map e79 with expected crc 2023-09-23T20:45:17.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:45:17 smithi026 ceph-mon[109912]: failed to encode map e79 with expected crc 2023-09-23T20:45:17.883 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:45:17 smithi026 ceph-mon[109912]: failed to encode map e79 with expected crc 2023-09-23T20:45:17.883 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:45:17 smithi026 ceph-mon[109912]: [23/Sep/2023:20:45:16] ENGINE Bus STARTING 2023-09-23T20:45:17.883 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:45:17 smithi026 ceph-mon[109912]: [23/Sep/2023:20:45:16] ENGINE Serving on http://172.21.15.26:8765 2023-09-23T20:45:17.883 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:45:17 smithi026 ceph-mon[109912]: [23/Sep/2023:20:45:16] ENGINE Serving on https://172.21.15.26:7150 2023-09-23T20:45:17.883 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:45:17 smithi026 ceph-mon[109912]: [23/Sep/2023:20:45:16] ENGINE Bus STARTED 2023-09-23T20:45:17.883 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:45:17 smithi026 ceph-mon[109912]: mgrmap e35: y(active, since 1.00917s), standbys: x 2023-09-23T20:45:17.884 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:17 smithi026 ceph-mon[175151]: failed to encode map e79 with expected crc 2023-09-23T20:45:17.884 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:17 smithi026 ceph-mon[175151]: failed to encode map e79 with expected crc 2023-09-23T20:45:17.884 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:17 smithi026 ceph-mon[175151]: failed to encode map e79 with expected crc 2023-09-23T20:45:17.884 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:17 smithi026 ceph-mon[175151]: failed to encode map e79 with expected crc 2023-09-23T20:45:17.884 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:17 smithi026 ceph-mon[175151]: failed to encode map e79 with expected crc 2023-09-23T20:45:17.884 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:17 smithi026 ceph-mon[175151]: [23/Sep/2023:20:45:16] ENGINE Bus STARTING 2023-09-23T20:45:17.884 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:17 smithi026 ceph-mon[175151]: [23/Sep/2023:20:45:16] ENGINE Serving on http://172.21.15.26:8765 2023-09-23T20:45:17.884 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:17 smithi026 ceph-mon[175151]: [23/Sep/2023:20:45:16] ENGINE Serving on https://172.21.15.26:7150 2023-09-23T20:45:17.885 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:17 smithi026 ceph-mon[175151]: [23/Sep/2023:20:45:16] ENGINE Bus STARTED 2023-09-23T20:45:17.885 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:17 smithi026 ceph-mon[175151]: mgrmap e35: y(active, since 1.00917s), standbys: x 2023-09-23T20:45:18.275 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:45:18.689 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:18 smithi026 ceph-mon[175151]: from='client.25108 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T20:45:18.689 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:18 smithi026 ceph-mon[175151]: pgmap v3: 73 pgs: 73 active+clean; 582 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:45:18.689 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:45:18 smithi026 ceph-mon[109912]: from='client.25108 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T20:45:18.689 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:45:18 smithi026 ceph-mon[109912]: pgmap v3: 73 pgs: 73 active+clean; 582 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:45:18.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:45:18 smithi070 ceph-mon[100755]: from='client.25108 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T20:45:18.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:45:18 smithi070 ceph-mon[100755]: pgmap v3: 73 pgs: 73 active+clean; 582 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:45:19.698 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:19 smithi026 ceph-mon[175151]: pgmap v4: 73 pgs: 73 active+clean; 582 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:45:19.698 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:19 smithi026 ceph-mon[175151]: mgrmap e36: y(active, since 3s), standbys: x 2023-09-23T20:45:19.699 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:45:19 smithi026 ceph-mon[109912]: pgmap v4: 73 pgs: 73 active+clean; 582 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:45:19.699 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:45:19 smithi026 ceph-mon[109912]: mgrmap e36: y(active, since 3s), standbys: x 2023-09-23T20:45:19.699 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:45:19 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[164029]: ::ffff:172.21.15.70 - - [23/Sep/2023:20:45:19] "GET /metrics HTTP/1.1" 200 31071 "" "Prometheus/2.43.0" 2023-09-23T20:45:19.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:45:19 smithi070 ceph-mon[100755]: pgmap v4: 73 pgs: 73 active+clean; 582 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:45:19.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:45:19 smithi070 ceph-mon[100755]: mgrmap e36: y(active, since 3s), standbys: x 2023-09-23T20:45:20.162 INFO:teuthology.orchestra.run.smithi026.stdout:Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d 2023-09-23T20:45:20.737 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -e sha1=bb0daeff97f020f686596fca6eba20c310e26fb7 -- bash -c 'sleep 60' 2023-09-23T20:45:21.447 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:45:21.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:45:21 smithi070 ceph-mon[100755]: pgmap v5: 73 pgs: 73 active+clean; 582 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:45:21.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:45:21 smithi070 ceph-mon[100755]: from='client.25193 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T20:45:21.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:45:21 smithi070 ceph-mon[100755]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d 2023-09-23T20:45:21.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:45:21 smithi070 ceph-mon[100755]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:45:21.631 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:45:21 smithi026 ceph-mon[109912]: pgmap v5: 73 pgs: 73 active+clean; 582 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:45:21.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:45:21 smithi026 ceph-mon[109912]: from='client.25193 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T20:45:21.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:45:21 smithi026 ceph-mon[109912]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d 2023-09-23T20:45:21.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:45:21 smithi026 ceph-mon[109912]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:45:21.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:21 smithi026 ceph-mon[175151]: pgmap v5: 73 pgs: 73 active+clean; 582 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:45:21.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:21 smithi026 ceph-mon[175151]: from='client.25193 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T20:45:21.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:21 smithi026 ceph-mon[175151]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d 2023-09-23T20:45:21.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:21 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:45:22.682 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:45:22 smithi026 ceph-mon[109912]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:45:22.683 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:45:22 smithi026 ceph-mon[109912]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:45:22.683 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:45:22 smithi026 ceph-mon[109912]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:45:22.683 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:45:22 smithi026 ceph-mon[109912]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:45:22.683 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:45:22 smithi026 ceph-mon[109912]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:45:22.683 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:45:22 smithi026 ceph-mon[109912]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:45:22.683 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:22 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:45:22.683 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:22 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:45:22.683 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:22 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:45:22.684 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:22 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:45:22.684 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:22 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:45:22.684 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:22 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:45:22.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:45:22 smithi070 ceph-mon[100755]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:45:22.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:45:22 smithi070 ceph-mon[100755]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:45:22.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:45:22 smithi070 ceph-mon[100755]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:45:22.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:45:22 smithi070 ceph-mon[100755]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:45:22.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:45:22 smithi070 ceph-mon[100755]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:45:22.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:45:22 smithi070 ceph-mon[100755]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:45:23.860 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:45:23 smithi070 ceph-mon[100755]: pgmap v6: 73 pgs: 73 active+clean; 582 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:45:23.860 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:45:23 smithi070 ceph-mon[100755]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:45:23.860 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:45:23 smithi070 ceph-mon[100755]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:45:23.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:23 smithi026 ceph-mon[175151]: pgmap v6: 73 pgs: 73 active+clean; 582 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:45:23.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:23 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:45:23.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:23 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:45:23.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:45:23 smithi026 ceph-mon[109912]: pgmap v6: 73 pgs: 73 active+clean; 582 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:45:23.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:45:23 smithi026 ceph-mon[109912]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:45:23.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:45:23 smithi026 ceph-mon[109912]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:45:26.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:45:25 smithi070 ceph-mon[100755]: pgmap v7: 73 pgs: 73 active+clean; 582 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:45:26.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:45:25 smithi070 ceph-mon[100755]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:45:26.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:45:25 smithi070 ceph-mon[100755]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:45:26.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:45:25 smithi070 ceph-mon[100755]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:45:26.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:45:25 smithi070 ceph-mon[100755]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:45:26.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:45:25 smithi026 ceph-mon[109912]: pgmap v7: 73 pgs: 73 active+clean; 582 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:45:26.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:45:25 smithi026 ceph-mon[109912]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:45:26.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:45:25 smithi026 ceph-mon[109912]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:45:26.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:45:25 smithi026 ceph-mon[109912]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:45:26.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:45:25 smithi026 ceph-mon[109912]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:45:26.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:25 smithi026 ceph-mon[175151]: pgmap v7: 73 pgs: 73 active+clean; 582 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:45:26.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:25 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:45:26.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:25 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:45:26.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:25 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:45:26.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:25 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:45:27.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:45:26 smithi070 ceph-mon[100755]: Updating smithi026:/etc/ceph/ceph.conf 2023-09-23T20:45:27.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:45:26 smithi070 ceph-mon[100755]: Updating smithi070:/etc/ceph/ceph.conf 2023-09-23T20:45:27.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:45:26 smithi070 ceph-mon[100755]: Updating smithi070:/var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-23T20:45:27.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:45:26 smithi070 ceph-mon[100755]: Updating smithi026:/var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-23T20:45:27.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:45:26 smithi070 ceph-mon[100755]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:45:27.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:45:26 smithi070 ceph-mon[100755]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:45:27.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:45:26 smithi070 ceph-mon[100755]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:45:27.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:45:26 smithi070 ceph-mon[100755]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:45:27.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:45:26 smithi070 ceph-mon[100755]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:45:27.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:26 smithi026 ceph-mon[175151]: Updating smithi026:/etc/ceph/ceph.conf 2023-09-23T20:45:27.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:26 smithi026 ceph-mon[175151]: Updating smithi070:/etc/ceph/ceph.conf 2023-09-23T20:45:27.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:26 smithi026 ceph-mon[175151]: Updating smithi070:/var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-23T20:45:27.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:26 smithi026 ceph-mon[175151]: Updating smithi026:/var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-23T20:45:27.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:26 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:45:27.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:26 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:45:27.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:26 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:45:27.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:26 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:45:27.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:26 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:45:27.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:45:26 smithi026 ceph-mon[109912]: Updating smithi026:/etc/ceph/ceph.conf 2023-09-23T20:45:27.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:45:26 smithi026 ceph-mon[109912]: Updating smithi070:/etc/ceph/ceph.conf 2023-09-23T20:45:27.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:45:26 smithi026 ceph-mon[109912]: Updating smithi070:/var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-23T20:45:27.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:45:26 smithi026 ceph-mon[109912]: Updating smithi026:/var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-23T20:45:27.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:45:26 smithi026 ceph-mon[109912]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:45:27.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:45:26 smithi026 ceph-mon[109912]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:45:27.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:45:26 smithi026 ceph-mon[109912]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:45:27.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:45:26 smithi026 ceph-mon[109912]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:45:27.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:45:26 smithi026 ceph-mon[109912]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:45:28.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:45:27 smithi070 ceph-mon[100755]: Updating smithi070:/etc/ceph/ceph.client.admin.keyring 2023-09-23T20:45:28.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:45:27 smithi070 ceph-mon[100755]: Updating smithi026:/etc/ceph/ceph.client.admin.keyring 2023-09-23T20:45:28.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:45:27 smithi070 ceph-mon[100755]: pgmap v8: 73 pgs: 73 active+clean; 582 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:45:28.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:45:27 smithi070 ceph-mon[100755]: Updating smithi070:/var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/config/ceph.client.admin.keyring 2023-09-23T20:45:28.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:45:27 smithi070 ceph-mon[100755]: Updating smithi026:/var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/config/ceph.client.admin.keyring 2023-09-23T20:45:28.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:45:27 smithi070 ceph-mon[100755]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:45:28.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:45:27 smithi070 ceph-mon[100755]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:45:28.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:45:27 smithi070 ceph-mon[100755]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:45:28.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:45:27 smithi070 ceph-mon[100755]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:45:28.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:45:27 smithi026 ceph-mon[109912]: Updating smithi070:/etc/ceph/ceph.client.admin.keyring 2023-09-23T20:45:28.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:45:27 smithi026 ceph-mon[109912]: Updating smithi026:/etc/ceph/ceph.client.admin.keyring 2023-09-23T20:45:28.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:45:27 smithi026 ceph-mon[109912]: pgmap v8: 73 pgs: 73 active+clean; 582 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:45:28.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:45:27 smithi026 ceph-mon[109912]: Updating smithi070:/var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/config/ceph.client.admin.keyring 2023-09-23T20:45:28.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:45:27 smithi026 ceph-mon[109912]: Updating smithi026:/var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/config/ceph.client.admin.keyring 2023-09-23T20:45:28.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:45:27 smithi026 ceph-mon[109912]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:45:28.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:45:27 smithi026 ceph-mon[109912]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:45:28.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:45:27 smithi026 ceph-mon[109912]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:45:28.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:45:27 smithi026 ceph-mon[109912]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:45:28.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:27 smithi026 ceph-mon[175151]: Updating smithi070:/etc/ceph/ceph.client.admin.keyring 2023-09-23T20:45:28.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:27 smithi026 ceph-mon[175151]: Updating smithi026:/etc/ceph/ceph.client.admin.keyring 2023-09-23T20:45:28.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:27 smithi026 ceph-mon[175151]: pgmap v8: 73 pgs: 73 active+clean; 582 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:45:28.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:27 smithi026 ceph-mon[175151]: Updating smithi070:/var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/config/ceph.client.admin.keyring 2023-09-23T20:45:28.134 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:27 smithi026 ceph-mon[175151]: Updating smithi026:/var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/config/ceph.client.admin.keyring 2023-09-23T20:45:28.134 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:27 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:45:28.134 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:27 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:45:28.134 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:27 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:45:28.134 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:27 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:45:29.119 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 20:45:28 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-prometheus-a[155975]: ts=2023-09-23T20:45:28.660Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1123-g61431f6b (61431f6ba2752649b506867ec856b4a6236f90dd) quincy (stable)\", cluster_addr=\"172.21.15.26\", device_class=\"ssd\", hostname=\"smithi026\", instance=\"smithi070.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.26\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1123-g61431f6b (61431f6ba2752649b506867ec856b4a6236f90dd) quincy (stable)\", cluster_addr=\"172.21.15.26\", device_class=\"ssd\", hostname=\"smithi026\", instance=\"smithi026:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.26\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-09-23T20:45:29.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:45:29 smithi026 ceph-mon[109912]: pgmap v9: 73 pgs: 73 active+clean; 582 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:45:29.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:29 smithi026 ceph-mon[175151]: pgmap v9: 73 pgs: 73 active+clean; 582 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:45:29.883 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:45:29 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[164029]: ::ffff:172.21.15.70 - - [23/Sep/2023:20:45:29] "GET /metrics HTTP/1.1" 200 31071 "" "Prometheus/2.43.0" 2023-09-23T20:45:30.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:45:29 smithi070 ceph-mon[100755]: pgmap v9: 73 pgs: 73 active+clean; 582 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:45:32.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:45:31 smithi070 ceph-mon[100755]: pgmap v10: 73 pgs: 73 active+clean; 582 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:45:32.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:45:31 smithi070 ceph-mon[100755]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:45:32.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:45:31 smithi026 ceph-mon[109912]: pgmap v10: 73 pgs: 73 active+clean; 582 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:45:32.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:45:31 smithi026 ceph-mon[109912]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:45:32.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:31 smithi026 ceph-mon[175151]: pgmap v10: 73 pgs: 73 active+clean; 582 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:45:32.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:31 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:45:34.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:45:33 smithi070 ceph-mon[100755]: pgmap v11: 73 pgs: 73 active+clean; 582 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:45:34.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:33 smithi026 ceph-mon[175151]: pgmap v11: 73 pgs: 73 active+clean; 582 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:45:34.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:45:33 smithi026 ceph-mon[109912]: pgmap v11: 73 pgs: 73 active+clean; 582 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:45:36.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:45:35 smithi070 ceph-mon[100755]: pgmap v12: 73 pgs: 73 active+clean; 582 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:45:36.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:35 smithi026 ceph-mon[175151]: pgmap v12: 73 pgs: 73 active+clean; 582 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:45:36.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:45:35 smithi026 ceph-mon[109912]: pgmap v12: 73 pgs: 73 active+clean; 582 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:45:38.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:45:37 smithi070 ceph-mon[100755]: pgmap v13: 73 pgs: 73 active+clean; 582 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:45:38.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:37 smithi026 ceph-mon[175151]: pgmap v13: 73 pgs: 73 active+clean; 582 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:45:38.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:45:37 smithi026 ceph-mon[109912]: pgmap v13: 73 pgs: 73 active+clean; 582 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:45:39.119 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 20:45:38 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-prometheus-a[155975]: ts=2023-09-23T20:45:38.660Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1123-g61431f6b (61431f6ba2752649b506867ec856b4a6236f90dd) quincy (stable)\", cluster_addr=\"172.21.15.26\", device_class=\"ssd\", hostname=\"smithi026\", instance=\"smithi070.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.26\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1123-g61431f6b (61431f6ba2752649b506867ec856b4a6236f90dd) quincy (stable)\", cluster_addr=\"172.21.15.26\", device_class=\"ssd\", hostname=\"smithi026\", instance=\"smithi026:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.26\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-09-23T20:45:39.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:39 smithi026 ceph-mon[175151]: pgmap v14: 73 pgs: 73 active+clean; 582 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:45:39.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:45:39 smithi026 ceph-mon[109912]: pgmap v14: 73 pgs: 73 active+clean; 582 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:45:39.882 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:45:39 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[164029]: ::ffff:172.21.15.70 - - [23/Sep/2023:20:45:39] "GET /metrics HTTP/1.1" 200 33700 "" "Prometheus/2.43.0" 2023-09-23T20:45:40.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:45:39 smithi070 ceph-mon[100755]: pgmap v14: 73 pgs: 73 active+clean; 582 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:45:42.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:45:41 smithi070 ceph-mon[100755]: pgmap v15: 73 pgs: 73 active+clean; 582 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:45:42.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:41 smithi026 ceph-mon[175151]: pgmap v15: 73 pgs: 73 active+clean; 582 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:45:42.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:45:41 smithi026 ceph-mon[109912]: pgmap v15: 73 pgs: 73 active+clean; 582 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:45:44.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:45:43 smithi070 ceph-mon[100755]: pgmap v16: 73 pgs: 73 active+clean; 582 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:45:44.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:43 smithi026 ceph-mon[175151]: pgmap v16: 73 pgs: 73 active+clean; 582 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:45:44.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:45:43 smithi026 ceph-mon[109912]: pgmap v16: 73 pgs: 73 active+clean; 582 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:45:46.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:45:45 smithi070 ceph-mon[100755]: pgmap v17: 73 pgs: 73 active+clean; 582 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:45:46.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:45:45 smithi026 ceph-mon[109912]: pgmap v17: 73 pgs: 73 active+clean; 582 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:45:46.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:45 smithi026 ceph-mon[175151]: pgmap v17: 73 pgs: 73 active+clean; 582 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:45:47.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:45:46 smithi070 ceph-mon[100755]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:45:47.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:46 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:45:47.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:45:46 smithi026 ceph-mon[109912]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:45:48.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:45:47 smithi070 ceph-mon[100755]: pgmap v18: 73 pgs: 73 active+clean; 582 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:45:48.131 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:47 smithi026 ceph-mon[175151]: pgmap v18: 73 pgs: 73 active+clean; 582 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:45:48.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:45:47 smithi026 ceph-mon[109912]: pgmap v18: 73 pgs: 73 active+clean; 582 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:45:49.119 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 20:45:48 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-prometheus-a[155975]: ts=2023-09-23T20:45:48.660Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1123-g61431f6b (61431f6ba2752649b506867ec856b4a6236f90dd) quincy (stable)\", cluster_addr=\"172.21.15.26\", device_class=\"ssd\", hostname=\"smithi026\", instance=\"smithi070.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.26\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1123-g61431f6b (61431f6ba2752649b506867ec856b4a6236f90dd) quincy (stable)\", cluster_addr=\"172.21.15.26\", device_class=\"ssd\", hostname=\"smithi026\", instance=\"smithi026:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.26\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-09-23T20:45:49.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:49 smithi026 ceph-mon[175151]: pgmap v19: 73 pgs: 73 active+clean; 582 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:45:49.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:45:49 smithi026 ceph-mon[109912]: pgmap v19: 73 pgs: 73 active+clean; 582 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:45:49.882 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:45:49 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[164029]: ::ffff:172.21.15.70 - - [23/Sep/2023:20:45:49] "GET /metrics HTTP/1.1" 200 33697 "" "Prometheus/2.43.0" 2023-09-23T20:45:50.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:45:49 smithi070 ceph-mon[100755]: pgmap v19: 73 pgs: 73 active+clean; 582 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:45:52.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:45:51 smithi070 ceph-mon[100755]: pgmap v20: 73 pgs: 73 active+clean; 582 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:45:52.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:51 smithi026 ceph-mon[175151]: pgmap v20: 73 pgs: 73 active+clean; 582 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:45:52.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:45:51 smithi026 ceph-mon[109912]: pgmap v20: 73 pgs: 73 active+clean; 582 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:45:54.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:45:53 smithi070 ceph-mon[100755]: pgmap v21: 73 pgs: 73 active+clean; 582 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:45:54.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:53 smithi026 ceph-mon[175151]: pgmap v21: 73 pgs: 73 active+clean; 582 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:45:54.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:45:53 smithi026 ceph-mon[109912]: pgmap v21: 73 pgs: 73 active+clean; 582 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:45:56.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:45:55 smithi070 ceph-mon[100755]: pgmap v22: 73 pgs: 73 active+clean; 582 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:45:56.131 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:55 smithi026 ceph-mon[175151]: pgmap v22: 73 pgs: 73 active+clean; 582 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:45:56.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:45:55 smithi026 ceph-mon[109912]: pgmap v22: 73 pgs: 73 active+clean; 582 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:45:58.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:45:57 smithi070 ceph-mon[100755]: pgmap v23: 73 pgs: 73 active+clean; 582 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:45:58.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:57 smithi026 ceph-mon[175151]: pgmap v23: 73 pgs: 73 active+clean; 582 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:45:58.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:45:57 smithi026 ceph-mon[109912]: pgmap v23: 73 pgs: 73 active+clean; 582 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:45:59.119 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 20:45:58 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-prometheus-a[155975]: ts=2023-09-23T20:45:58.660Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1123-g61431f6b (61431f6ba2752649b506867ec856b4a6236f90dd) quincy (stable)\", cluster_addr=\"172.21.15.26\", device_class=\"ssd\", hostname=\"smithi026\", instance=\"smithi070.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.26\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1123-g61431f6b (61431f6ba2752649b506867ec856b4a6236f90dd) quincy (stable)\", cluster_addr=\"172.21.15.26\", device_class=\"ssd\", hostname=\"smithi026\", instance=\"smithi026:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.26\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-09-23T20:45:59.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:45:59 smithi026 ceph-mon[175151]: pgmap v24: 73 pgs: 73 active+clean; 582 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:45:59.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:45:59 smithi026 ceph-mon[109912]: pgmap v24: 73 pgs: 73 active+clean; 582 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:45:59.882 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:45:59 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[164029]: ::ffff:172.21.15.70 - - [23/Sep/2023:20:45:59] "GET /metrics HTTP/1.1" 200 33697 "" "Prometheus/2.43.0" 2023-09-23T20:46:00.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:45:59 smithi070 ceph-mon[100755]: pgmap v24: 73 pgs: 73 active+clean; 582 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:46:02.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:46:01 smithi070 ceph-mon[100755]: pgmap v25: 73 pgs: 73 active+clean; 582 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:46:02.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:46:01 smithi070 ceph-mon[100755]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:46:02.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:46:01 smithi026 ceph-mon[175151]: pgmap v25: 73 pgs: 73 active+clean; 582 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:46:02.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:46:01 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:46:02.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:46:01 smithi026 ceph-mon[109912]: pgmap v25: 73 pgs: 73 active+clean; 582 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:46:02.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:46:01 smithi026 ceph-mon[109912]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:46:04.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:46:03 smithi070 ceph-mon[100755]: pgmap v26: 73 pgs: 73 active+clean; 582 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:46:04.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:46:03 smithi026 ceph-mon[175151]: pgmap v26: 73 pgs: 73 active+clean; 582 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:46:04.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:46:03 smithi026 ceph-mon[109912]: pgmap v26: 73 pgs: 73 active+clean; 582 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:46:06.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:46:05 smithi070 ceph-mon[100755]: pgmap v27: 73 pgs: 73 active+clean; 582 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:46:06.131 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:46:05 smithi026 ceph-mon[175151]: pgmap v27: 73 pgs: 73 active+clean; 582 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:46:06.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:46:05 smithi026 ceph-mon[109912]: pgmap v27: 73 pgs: 73 active+clean; 582 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:46:08.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:46:07 smithi070 ceph-mon[100755]: pgmap v28: 73 pgs: 73 active+clean; 582 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:46:08.131 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:46:07 smithi026 ceph-mon[175151]: pgmap v28: 73 pgs: 73 active+clean; 582 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:46:08.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:46:07 smithi026 ceph-mon[109912]: pgmap v28: 73 pgs: 73 active+clean; 582 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:46:09.119 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 20:46:08 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-prometheus-a[155975]: ts=2023-09-23T20:46:08.660Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1123-g61431f6b (61431f6ba2752649b506867ec856b4a6236f90dd) quincy (stable)\", cluster_addr=\"172.21.15.26\", device_class=\"ssd\", hostname=\"smithi026\", instance=\"smithi070.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.26\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1123-g61431f6b (61431f6ba2752649b506867ec856b4a6236f90dd) quincy (stable)\", cluster_addr=\"172.21.15.26\", device_class=\"ssd\", hostname=\"smithi026\", instance=\"smithi026:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.26\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-09-23T20:46:09.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:46:09 smithi026 ceph-mon[175151]: pgmap v29: 73 pgs: 73 active+clean; 582 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:46:09.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:46:09 smithi026 ceph-mon[109912]: pgmap v29: 73 pgs: 73 active+clean; 582 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:46:09.882 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:46:09 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[164029]: ::ffff:172.21.15.70 - - [23/Sep/2023:20:46:09] "GET /metrics HTTP/1.1" 200 33693 "" "Prometheus/2.43.0" 2023-09-23T20:46:10.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:46:09 smithi070 ceph-mon[100755]: pgmap v29: 73 pgs: 73 active+clean; 582 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:46:12.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:46:11 smithi070 ceph-mon[100755]: pgmap v30: 73 pgs: 73 active+clean; 582 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:46:12.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:46:11 smithi026 ceph-mon[175151]: pgmap v30: 73 pgs: 73 active+clean; 582 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:46:12.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:46:11 smithi026 ceph-mon[109912]: pgmap v30: 73 pgs: 73 active+clean; 582 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:46:14.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:46:13 smithi070 ceph-mon[100755]: pgmap v31: 73 pgs: 73 active+clean; 582 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:46:14.131 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:46:13 smithi026 ceph-mon[175151]: pgmap v31: 73 pgs: 73 active+clean; 582 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:46:14.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:46:13 smithi026 ceph-mon[109912]: pgmap v31: 73 pgs: 73 active+clean; 582 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:46:16.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:46:15 smithi070 ceph-mon[100755]: pgmap v32: 73 pgs: 73 active+clean; 582 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:46:16.131 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:46:15 smithi026 ceph-mon[109912]: pgmap v32: 73 pgs: 73 active+clean; 582 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:46:16.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:46:15 smithi026 ceph-mon[175151]: pgmap v32: 73 pgs: 73 active+clean; 582 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:46:17.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:46:16 smithi070 ceph-mon[100755]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:46:17.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:46:16 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:46:17.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:46:16 smithi026 ceph-mon[109912]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:46:18.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:46:17 smithi070 ceph-mon[100755]: pgmap v33: 73 pgs: 73 active+clean; 582 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:46:18.131 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:46:17 smithi026 ceph-mon[175151]: pgmap v33: 73 pgs: 73 active+clean; 582 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:46:18.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:46:17 smithi026 ceph-mon[109912]: pgmap v33: 73 pgs: 73 active+clean; 582 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:46:19.122 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 20:46:18 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-prometheus-a[155975]: ts=2023-09-23T20:46:18.661Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1123-g61431f6b (61431f6ba2752649b506867ec856b4a6236f90dd) quincy (stable)\", cluster_addr=\"172.21.15.26\", device_class=\"ssd\", hostname=\"smithi026\", instance=\"smithi070.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.26\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1123-g61431f6b (61431f6ba2752649b506867ec856b4a6236f90dd) quincy (stable)\", cluster_addr=\"172.21.15.26\", device_class=\"ssd\", hostname=\"smithi026\", instance=\"smithi026:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.26\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-09-23T20:46:19.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:46:19 smithi026 ceph-mon[109912]: pgmap v34: 73 pgs: 73 active+clean; 582 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:46:19.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:46:19 smithi026 ceph-mon[175151]: pgmap v34: 73 pgs: 73 active+clean; 582 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:46:19.883 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:46:19 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[164029]: ::ffff:172.21.15.70 - - [23/Sep/2023:20:46:19] "GET /metrics HTTP/1.1" 200 33693 "" "Prometheus/2.43.0" 2023-09-23T20:46:20.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:46:19 smithi070 ceph-mon[100755]: pgmap v34: 73 pgs: 73 active+clean; 582 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:46:22.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:46:21 smithi070 ceph-mon[100755]: pgmap v35: 73 pgs: 73 active+clean; 582 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:46:22.131 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:46:21 smithi026 ceph-mon[175151]: pgmap v35: 73 pgs: 73 active+clean; 582 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:46:22.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:46:21 smithi026 ceph-mon[109912]: pgmap v35: 73 pgs: 73 active+clean; 582 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:46:23.100 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -e sha1=bb0daeff97f020f686596fca6eba20c310e26fb7 -- bash -c 'ceph orch upgrade resume' 2023-09-23T20:46:23.711 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:46:24.070 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:46:23 smithi026 ceph-mon[175151]: pgmap v36: 73 pgs: 73 active+clean; 582 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:46:24.070 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:46:23 smithi026 ceph-mon[109912]: pgmap v36: 73 pgs: 73 active+clean; 582 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:46:24.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:46:23 smithi070 ceph-mon[100755]: pgmap v36: 73 pgs: 73 active+clean; 582 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:46:25.501 INFO:teuthology.orchestra.run.smithi026.stdout:Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d 2023-09-23T20:46:25.975 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -e sha1=bb0daeff97f020f686596fca6eba20c310e26fb7 -- bash -c 'echo wait for majority of mons to upgrade' 2023-09-23T20:46:26.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:46:25 smithi070 ceph-mon[100755]: pgmap v37: 73 pgs: 73 active+clean; 582 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:46:26.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:46:25 smithi070 ceph-mon[100755]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:46:26.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:46:25 smithi070 ceph-mon[100755]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:46:26.131 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:46:25 smithi026 ceph-mon[109912]: pgmap v37: 73 pgs: 73 active+clean; 582 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:46:26.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:46:25 smithi026 ceph-mon[109912]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:46:26.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:46:25 smithi026 ceph-mon[109912]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:46:26.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:46:25 smithi026 ceph-mon[175151]: pgmap v37: 73 pgs: 73 active+clean; 582 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:46:26.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:46:25 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:46:26.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:46:25 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:46:26.583 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:46:27.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:46:26 smithi070 ceph-mon[100755]: from='client.25199 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T20:46:27.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:46:26 smithi070 ceph-mon[100755]: Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d 2023-09-23T20:46:27.120 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:46:26 smithi026 ceph-mon[175151]: from='client.25199 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T20:46:27.120 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:46:26 smithi026 ceph-mon[175151]: Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d 2023-09-23T20:46:27.120 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:46:26 smithi026 ceph-mon[109912]: from='client.25199 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T20:46:27.121 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:46:26 smithi026 ceph-mon[109912]: Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d 2023-09-23T20:46:28.053 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:46:27 smithi026 ceph-mon[109912]: pgmap v38: 73 pgs: 73 active+clean; 582 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:46:28.053 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:46:27 smithi026 ceph-mon[175151]: pgmap v38: 73 pgs: 73 active+clean; 582 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:46:28.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:46:27 smithi070 ceph-mon[100755]: pgmap v38: 73 pgs: 73 active+clean; 582 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:46:28.186 INFO:teuthology.orchestra.run.smithi026.stdout:wait for majority of mons to upgrade 2023-09-23T20:46:28.598 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -e sha1=bb0daeff97f020f686596fca6eba20c310e26fb7 -- bash -c 'while ! ceph mon versions | grep $sha1 | egrep '"'"': [23]'"'"' ; do sleep 2 ; done' 2023-09-23T20:46:29.119 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 20:46:28 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-prometheus-a[155975]: ts=2023-09-23T20:46:28.659Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1123-g61431f6b (61431f6ba2752649b506867ec856b4a6236f90dd) quincy (stable)\", cluster_addr=\"172.21.15.26\", device_class=\"ssd\", hostname=\"smithi026\", instance=\"smithi070.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.26\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1123-g61431f6b (61431f6ba2752649b506867ec856b4a6236f90dd) quincy (stable)\", cluster_addr=\"172.21.15.26\", device_class=\"ssd\", hostname=\"smithi026\", instance=\"smithi026:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.26\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-09-23T20:46:29.260 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:46:29.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:46:29 smithi026 ceph-mon[175151]: pgmap v39: 73 pgs: 73 active+clean; 582 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:46:29.882 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:46:29 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[164029]: ::ffff:172.21.15.70 - - [23/Sep/2023:20:46:29] "GET /metrics HTTP/1.1" 200 33693 "" "Prometheus/2.43.0" 2023-09-23T20:46:29.883 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:46:29 smithi026 ceph-mon[109912]: pgmap v39: 73 pgs: 73 active+clean; 582 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:46:30.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:46:29 smithi070 ceph-mon[100755]: pgmap v39: 73 pgs: 73 active+clean; 582 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:46:31.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:46:31 smithi070 ceph-mon[100755]: pgmap v40: 73 pgs: 73 active+clean; 582 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:46:31.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:46:31 smithi070 ceph-mon[100755]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:46:31.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:46:31 smithi070 ceph-mon[100755]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:46:31.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:46:31 smithi070 ceph-mon[100755]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:46:31.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:46:31 smithi070 ceph-mon[100755]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:46:31.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:46:31 smithi070 ceph-mon[100755]: from='client.? 172.21.15.26:0/4215918595' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:46:31.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:46:31 smithi070 ceph-mon[100755]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:46:31.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:46:31 smithi070 ceph-mon[100755]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:46:31.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:46:31 smithi070 ceph-mon[100755]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:46:31.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:46:31 smithi070 ceph-mon[100755]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:46:31.881 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:46:31 smithi026 ceph-mon[109912]: pgmap v40: 73 pgs: 73 active+clean; 582 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:46:31.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:46:31 smithi026 ceph-mon[109912]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:46:31.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:46:31 smithi026 ceph-mon[109912]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:46:31.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:46:31 smithi026 ceph-mon[109912]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:46:31.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:46:31 smithi026 ceph-mon[109912]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:46:31.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:46:31 smithi026 ceph-mon[109912]: from='client.? 172.21.15.26:0/4215918595' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:46:31.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:46:31 smithi026 ceph-mon[109912]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:46:31.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:46:31 smithi026 ceph-mon[109912]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:46:31.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:46:31 smithi026 ceph-mon[109912]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:46:31.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:46:31 smithi026 ceph-mon[109912]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:46:31.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:46:31 smithi026 ceph-mon[175151]: pgmap v40: 73 pgs: 73 active+clean; 582 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:46:31.883 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:46:31 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:46:31.883 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:46:31 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:46:31.883 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:46:31 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:46:31.883 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:46:31 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:46:31.883 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:46:31 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/4215918595' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:46:31.883 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:46:31 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:46:31.883 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:46:31 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:46:31.883 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:46:31 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:46:31.883 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:46:31 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:46:32.758 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:46:32 smithi026 ceph-mon[109912]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:46:32.758 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:46:32 smithi026 ceph-mon[109912]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T20:46:32.758 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:46:32 smithi026 ceph-mon[109912]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2023-09-23T20:46:32.758 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:46:32 smithi026 ceph-mon[109912]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["c"]}]: dispatch 2023-09-23T20:46:32.758 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:46:32 smithi026 ceph-mon[109912]: Upgrade: It appears safe to stop mon.c 2023-09-23T20:46:32.758 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:46:32 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:46:32.759 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:46:32 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T20:46:32.759 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:46:32 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2023-09-23T20:46:32.759 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:46:32 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["c"]}]: dispatch 2023-09-23T20:46:32.759 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:46:32 smithi026 ceph-mon[175151]: Upgrade: It appears safe to stop mon.c 2023-09-23T20:46:32.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:46:32 smithi070 ceph-mon[100755]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:46:32.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:46:32 smithi070 ceph-mon[100755]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T20:46:32.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:46:32 smithi070 ceph-mon[100755]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2023-09-23T20:46:32.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:46:32 smithi070 ceph-mon[100755]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["c"]}]: dispatch 2023-09-23T20:46:32.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:46:32 smithi070 ceph-mon[100755]: Upgrade: It appears safe to stop mon.c 2023-09-23T20:46:33.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:46:33 smithi026 ceph-mon[175151]: pgmap v41: 73 pgs: 73 active+clean; 582 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:46:33.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:46:33 smithi026 ceph-mon[175151]: Upgrade: Updating mon.c 2023-09-23T20:46:33.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:46:33 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:46:33.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:46:33 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-23T20:46:33.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:46:33 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-09-23T20:46:33.883 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:46:33 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:46:33.883 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:46:33 smithi026 ceph-mon[175151]: Deploying daemon mon.c on smithi026 2023-09-23T20:46:33.883 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:46:33 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/3010325894' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:46:33.883 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:46:33 smithi026 ceph-mon[109912]: pgmap v41: 73 pgs: 73 active+clean; 582 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:46:33.883 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:46:33 smithi026 ceph-mon[109912]: Upgrade: Updating mon.c 2023-09-23T20:46:33.883 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:46:33 smithi026 ceph-mon[109912]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:46:33.883 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:46:33 smithi026 ceph-mon[109912]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-23T20:46:33.883 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:46:33 smithi026 ceph-mon[109912]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-09-23T20:46:33.883 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:46:33 smithi026 ceph-mon[109912]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:46:33.883 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:46:33 smithi026 ceph-mon[109912]: Deploying daemon mon.c on smithi026 2023-09-23T20:46:33.884 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:46:33 smithi026 ceph-mon[109912]: from='client.? 172.21.15.26:0/3010325894' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:46:34.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:46:33 smithi070 ceph-mon[100755]: pgmap v41: 73 pgs: 73 active+clean; 582 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:46:34.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:46:33 smithi070 ceph-mon[100755]: Upgrade: Updating mon.c 2023-09-23T20:46:34.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:46:33 smithi070 ceph-mon[100755]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:46:34.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:46:33 smithi070 ceph-mon[100755]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-23T20:46:34.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:46:33 smithi070 ceph-mon[100755]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-09-23T20:46:34.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:46:33 smithi070 ceph-mon[100755]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:46:34.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:46:33 smithi070 ceph-mon[100755]: Deploying daemon mon.c on smithi026 2023-09-23T20:46:34.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:46:33 smithi070 ceph-mon[100755]: from='client.? 172.21.15.26:0/3010325894' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:46:34.364 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:46:34 smithi026 systemd[1]: Stopping Ceph mon.c for b3f8e4ec-5a50-11ee-8db4-212e2dc638e7... 2023-09-23T20:46:34.619 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:46:34 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mon-c[109888]: 2023-09-23T20:46:34.361+0000 7f315a1ac700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-mon -n mon.c -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2023-09-23T20:46:34.619 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:46:34 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mon-c[109888]: 2023-09-23T20:46:34.361+0000 7f315a1ac700 -1 mon.c@2(peon) e3 *** Got Signal Terminated *** 2023-09-23T20:46:34.619 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:46:34 smithi026 bash[179469]: ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mon-c 2023-09-23T20:46:35.368 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:46:34 smithi026 systemd[1]: ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7@mon.c.service: Succeeded. 2023-09-23T20:46:35.368 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:46:34 smithi026 systemd[1]: Stopped Ceph mon.c for b3f8e4ec-5a50-11ee-8db4-212e2dc638e7. 2023-09-23T20:46:35.368 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:46:35 smithi026 systemd[1]: Starting Ceph mon.c for b3f8e4ec-5a50-11ee-8db4-212e2dc638e7... 2023-09-23T20:46:35.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:46:35 smithi026 podman[179624]: 2023-09-23T20:46:35.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:46:35 smithi026 bash[179624]: ad100e9213d343eb7caed617f94b273ebce228251d7e210d5157f4ae1fabb62b 2023-09-23T20:46:35.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:46:35 smithi026 ceph-mon[179688]: set uid:gid to 167:167 (ceph:ceph) 2023-09-23T20:46:35.633 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:46:35 smithi026 ceph-mon[179688]: ceph version 18.2.0-460-gbb0daeff (bb0daeff97f020f686596fca6eba20c310e26fb7) reef (stable), process ceph-mon, pid 8 2023-09-23T20:46:35.633 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:46:35 smithi026 ceph-mon[179688]: pidfile_write: ignore empty --pid-file 2023-09-23T20:46:35.633 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:46:35 smithi026 ceph-mon[179688]: load: jerasure load: lrc 2023-09-23T20:46:35.633 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:46:35 smithi026 ceph-mon[179688]: rocksdb: RocksDB version: 7.9.2 2023-09-23T20:46:35.633 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:46:35 smithi026 ceph-mon[179688]: rocksdb: Git sha 0 2023-09-23T20:46:35.633 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:46:35 smithi026 ceph-mon[179688]: rocksdb: Compile date 2023-09-22 15:19:43 2023-09-23T20:46:35.633 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:46:35 smithi026 ceph-mon[179688]: rocksdb: DB SUMMARY 2023-09-23T20:46:35.633 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:46:35 smithi026 ceph-mon[179688]: rocksdb: DB Session ID: 3GHFTF2FEJKD53JUBMNE 2023-09-23T20:46:35.633 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:46:35 smithi026 ceph-mon[179688]: rocksdb: CURRENT file: CURRENT 2023-09-23T20:46:35.633 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:46:35 smithi026 ceph-mon[179688]: rocksdb: IDENTITY file: IDENTITY 2023-09-23T20:46:35.633 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:46:35 smithi026 ceph-mon[179688]: rocksdb: MANIFEST file: MANIFEST-000009 size: 1069 Bytes 2023-09-23T20:46:35.633 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:46:35 smithi026 ceph-mon[179688]: rocksdb: SST files in /var/lib/ceph/mon/ceph-c/store.db dir, Total Num: 1, files: 000027.sst 2023-09-23T20:46:35.634 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:46:35 smithi026 ceph-mon[179688]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-c/store.db: 000025.log size: 381255 ; 2023-09-23T20:46:35.634 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:46:35 smithi026 ceph-mon[179688]: rocksdb: Options.error_if_exists: 0 2023-09-23T20:46:35.634 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:46:35 smithi026 ceph-mon[179688]: rocksdb: Options.create_if_missing: 0 2023-09-23T20:46:35.634 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:46:35 smithi026 ceph-mon[179688]: rocksdb: Options.paranoid_checks: 1 2023-09-23T20:46:35.634 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:46:35 smithi026 ceph-mon[179688]: rocksdb: Options.flush_verify_memtable_count: 1 2023-09-23T20:46:35.634 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:46:35 smithi026 ceph-mon[179688]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2023-09-23T20:46:35.634 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:46:35 smithi026 ceph-mon[179688]: rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2023-09-23T20:46:35.634 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:46:35 smithi026 ceph-mon[179688]: rocksdb: Options.env: 0x556a22284720 2023-09-23T20:46:35.634 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:46:35 smithi026 ceph-mon[179688]: rocksdb: Options.fs: PosixFileSystem 2023-09-23T20:46:35.634 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:46:35 smithi026 ceph-mon[179688]: rocksdb: Options.info_log: 0x556a24a45420 2023-09-23T20:46:35.634 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:46:35 smithi026 ceph-mon[179688]: rocksdb: Options.max_file_opening_threads: 16 2023-09-23T20:46:35.634 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:46:35 smithi026 ceph-mon[179688]: rocksdb: Options.statistics: (nil) 2023-09-23T20:46:35.635 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:46:35 smithi026 ceph-mon[179688]: rocksdb: Options.use_fsync: 0 2023-09-23T20:46:35.635 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:46:35 smithi026 ceph-mon[179688]: rocksdb: Options.max_log_file_size: 0 2023-09-23T20:46:35.635 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:46:35 smithi026 ceph-mon[179688]: rocksdb: Options.max_manifest_file_size: 1073741824 2023-09-23T20:46:35.635 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:46:35 smithi026 ceph-mon[179688]: rocksdb: Options.log_file_time_to_roll: 0 2023-09-23T20:46:35.635 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:46:35 smithi026 ceph-mon[179688]: rocksdb: Options.keep_log_file_num: 1000 2023-09-23T20:46:35.635 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:46:35 smithi026 ceph-mon[179688]: rocksdb: Options.recycle_log_file_num: 0 2023-09-23T20:46:35.635 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:46:35 smithi026 ceph-mon[179688]: rocksdb: Options.allow_fallocate: 1 2023-09-23T20:46:35.635 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:46:35 smithi026 ceph-mon[179688]: rocksdb: Options.allow_mmap_reads: 0 2023-09-23T20:46:35.635 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:46:35 smithi026 ceph-mon[179688]: rocksdb: Options.allow_mmap_writes: 0 2023-09-23T20:46:35.635 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:46:35 smithi026 ceph-mon[179688]: rocksdb: Options.use_direct_reads: 0 2023-09-23T20:46:35.635 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:46:35 smithi026 ceph-mon[179688]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2023-09-23T20:46:35.635 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:46:35 smithi026 ceph-mon[179688]: rocksdb: Options.create_missing_column_families: 0 2023-09-23T20:46:35.635 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:46:35 smithi026 ceph-mon[179688]: rocksdb: Options.db_log_dir: 2023-09-23T20:46:35.637 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:46:35 smithi026 ceph-mon[179688]: rocksdb: Options.wal_dir: 2023-09-23T20:46:35.637 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:46:35 smithi026 ceph-mon[179688]: rocksdb: Options.table_cache_numshardbits: 6 2023-09-23T20:46:35.637 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:46:35 smithi026 ceph-mon[179688]: rocksdb: Options.WAL_ttl_seconds: 0 2023-09-23T20:46:35.637 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:46:35 smithi026 ceph-mon[179688]: rocksdb: Options.WAL_size_limit_MB: 0 2023-09-23T20:46:35.637 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:46:35 smithi026 ceph-mon[179688]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2023-09-23T20:46:35.637 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:46:35 smithi026 ceph-mon[179688]: rocksdb: Options.manifest_preallocation_size: 4194304 2023-09-23T20:46:35.637 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:46:35 smithi026 ceph-mon[179688]: rocksdb: Options.is_fd_close_on_exec: 1 2023-09-23T20:46:35.637 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:46:35 smithi026 ceph-mon[179688]: rocksdb: Options.advise_random_on_open: 1 2023-09-23T20:46:35.637 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:46:35 smithi026 ceph-mon[179688]: rocksdb: Options.db_write_buffer_size: 0 2023-09-23T20:46:35.638 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:46:35 smithi026 ceph-mon[179688]: rocksdb: Options.write_buffer_manager: 0x556a23cd65a0 2023-09-23T20:46:35.638 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:46:35 smithi026 ceph-mon[179688]: rocksdb: Options.access_hint_on_compaction_start: 1 2023-09-23T20:46:35.638 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:46:35 smithi026 ceph-mon[179688]: rocksdb: Options.random_access_max_buffer_size: 1048576 2023-09-23T20:46:35.638 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:46:35 smithi026 ceph-mon[179688]: rocksdb: Options.use_adaptive_mutex: 0 2023-09-23T20:46:35.638 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:46:35 smithi026 ceph-mon[179688]: rocksdb: Options.rate_limiter: (nil) 2023-09-23T20:46:35.638 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:46:35 smithi026 ceph-mon[179688]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2023-09-23T20:46:35.638 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:46:35 smithi026 ceph-mon[179688]: rocksdb: Options.wal_recovery_mode: 2 2023-09-23T20:46:35.638 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:46:35 smithi026 ceph-mon[179688]: rocksdb: Options.enable_thread_tracking: 0 2023-09-23T20:46:35.638 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:46:35 smithi026 ceph-mon[179688]: rocksdb: Options.enable_pipelined_write: 0 2023-09-23T20:46:35.638 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:46:35 smithi026 ceph-mon[179688]: rocksdb: Options.unordered_write: 0 2023-09-23T20:46:35.638 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:46:35 smithi026 ceph-mon[179688]: rocksdb: Options.allow_concurrent_memtable_write: 1 2023-09-23T20:46:35.638 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:46:35 smithi026 ceph-mon[179688]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2023-09-23T20:46:35.639 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:46:35 smithi026 ceph-mon[179688]: rocksdb: Options.write_thread_max_yield_usec: 100 2023-09-23T20:46:35.639 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:46:35 smithi026 ceph-mon[179688]: rocksdb: Options.write_thread_slow_yield_usec: 3 2023-09-23T20:46:35.639 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:46:35 smithi026 ceph-mon[179688]: rocksdb: Options.row_cache: None 2023-09-23T20:46:35.639 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:46:35 smithi026 ceph-mon[179688]: rocksdb: Options.wal_filter: None 2023-09-23T20:46:35.639 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:46:35 smithi026 ceph-mon[179688]: rocksdb: Options.avoid_flush_during_recovery: 0 2023-09-23T20:46:35.639 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:46:35 smithi026 ceph-mon[179688]: rocksdb: Options.allow_ingest_behind: 0 2023-09-23T20:46:35.639 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:46:35 smithi026 ceph-mon[179688]: rocksdb: Options.two_write_queues: 0 2023-09-23T20:46:35.639 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:46:35 smithi026 ceph-mon[179688]: rocksdb: Options.manual_wal_flush: 0 2023-09-23T20:46:35.639 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:46:35 smithi026 ceph-mon[179688]: rocksdb: Options.wal_compression: 0 2023-09-23T20:46:35.639 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:46:35 smithi026 ceph-mon[179688]: rocksdb: Options.atomic_flush: 0 2023-09-23T20:46:35.639 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:46:35 smithi026 ceph-mon[179688]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2023-09-23T20:46:35.639 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:46:35 smithi026 ceph-mon[179688]: rocksdb: Options.persist_stats_to_disk: 0 2023-09-23T20:46:35.639 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:46:35 smithi026 ceph-mon[179688]: rocksdb: Options.write_dbid_to_manifest: 0 2023-09-23T20:46:35.640 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:46:35 smithi026 ceph-mon[179688]: rocksdb: Options.log_readahead_size: 0 2023-09-23T20:46:35.640 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:46:35 smithi026 ceph-mon[179688]: rocksdb: Options.file_checksum_gen_factory: Unknown 2023-09-23T20:46:35.640 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:46:35 smithi026 ceph-mon[179688]: rocksdb: Options.best_efforts_recovery: 0 2023-09-23T20:46:35.640 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:46:35 smithi026 ceph-mon[179688]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2023-09-23T20:46:35.640 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:46:35 smithi026 ceph-mon[179688]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2023-09-23T20:46:35.640 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:46:35 smithi026 ceph-mon[179688]: rocksdb: Options.allow_data_in_errors: 0 2023-09-23T20:46:35.640 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:46:35 smithi026 ceph-mon[179688]: rocksdb: Options.db_host_id: __hostname__ 2023-09-23T20:46:35.640 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:46:35 smithi026 ceph-mon[179688]: rocksdb: Options.enforce_single_del_contracts: true 2023-09-23T20:46:35.640 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:46:35 smithi026 ceph-mon[179688]: rocksdb: Options.max_background_jobs: 2 2023-09-23T20:46:35.640 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:46:35 smithi026 ceph-mon[179688]: rocksdb: Options.max_background_compactions: -1 2023-09-23T20:46:35.640 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:46:35 smithi026 ceph-mon[179688]: rocksdb: Options.max_subcompactions: 1 2023-09-23T20:46:35.640 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:46:35 smithi026 ceph-mon[179688]: rocksdb: Options.avoid_flush_during_shutdown: 0 2023-09-23T20:46:35.640 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:46:35 smithi026 ceph-mon[179688]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2023-09-23T20:46:35.641 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:46:35 smithi026 ceph-mon[179688]: rocksdb: Options.delayed_write_rate : 16777216 2023-09-23T20:46:35.641 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:46:35 smithi026 ceph-mon[179688]: rocksdb: Options.max_total_wal_size: 0 2023-09-23T20:46:35.641 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:46:35 smithi026 ceph-mon[179688]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2023-09-23T20:46:35.641 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:46:35 smithi026 ceph-mon[179688]: rocksdb: Options.stats_dump_period_sec: 600 2023-09-23T20:46:35.641 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:46:35 smithi026 ceph-mon[179688]: rocksdb: Options.stats_persist_period_sec: 600 2023-09-23T20:46:35.641 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:46:35 smithi026 ceph-mon[179688]: rocksdb: Options.stats_history_buffer_size: 1048576 2023-09-23T20:46:35.641 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:46:35 smithi026 ceph-mon[179688]: rocksdb: Options.max_open_files: -1 2023-09-23T20:46:35.641 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:46:35 smithi026 ceph-mon[179688]: rocksdb: Options.bytes_per_sync: 0 2023-09-23T20:46:35.641 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:46:35 smithi026 ceph-mon[179688]: rocksdb: Options.wal_bytes_per_sync: 0 2023-09-23T20:46:35.641 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:46:35 smithi026 ceph-mon[179688]: rocksdb: Options.strict_bytes_per_sync: 0 2023-09-23T20:46:35.641 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:46:35 smithi026 ceph-mon[179688]: rocksdb: Options.compaction_readahead_size: 0 2023-09-23T20:46:35.641 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:46:35 smithi026 ceph-mon[179688]: rocksdb: Options.max_background_flushes: -1 2023-09-23T20:46:35.642 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:46:35 smithi026 ceph-mon[179688]: rocksdb: Compression algorithms supported: 2023-09-23T20:46:35.642 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:46:35 smithi026 ceph-mon[179688]: rocksdb: kZSTDNotFinalCompression supported: 0 2023-09-23T20:46:35.642 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:46:35 smithi026 ceph-mon[179688]: rocksdb: kZSTD supported: 0 2023-09-23T20:46:35.642 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:46:35 smithi026 ceph-mon[179688]: rocksdb: kXpressCompression supported: 0 2023-09-23T20:46:35.642 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:46:35 smithi026 ceph-mon[179688]: rocksdb: kLZ4HCCompression supported: 1 2023-09-23T20:46:35.642 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:46:35 smithi026 ceph-mon[179688]: rocksdb: kZlibCompression supported: 1 2023-09-23T20:46:35.642 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:46:35 smithi026 ceph-mon[179688]: rocksdb: kSnappyCompression supported: 1 2023-09-23T20:46:35.642 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:46:35 smithi026 ceph-mon[179688]: rocksdb: kLZ4Compression supported: 1 2023-09-23T20:46:35.642 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:46:35 smithi026 ceph-mon[179688]: rocksdb: kBZip2Compression supported: 0 2023-09-23T20:46:35.642 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:46:35 smithi026 ceph-mon[179688]: rocksdb: Fast CRC32 supported: Supported on x86 2023-09-23T20:46:35.642 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:46:35 smithi026 ceph-mon[179688]: rocksdb: DMutex implementation: pthread_mutex_t 2023-09-23T20:46:35.642 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:46:35 smithi026 ceph-mon[179688]: rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-c/store.db/MANIFEST-000009 2023-09-23T20:46:35.642 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:46:35 smithi026 ceph-mon[179688]: rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2023-09-23T20:46:35.643 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:46:35 smithi026 ceph-mon[179688]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2023-09-23T20:46:35.643 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:46:35 smithi026 ceph-mon[179688]: rocksdb: Options.merge_operator: 2023-09-23T20:46:35.643 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:46:35 smithi026 ceph-mon[179688]: rocksdb: Options.compaction_filter: None 2023-09-23T20:46:35.643 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:46:35 smithi026 ceph-mon[179688]: rocksdb: Options.compaction_filter_factory: None 2023-09-23T20:46:35.643 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:46:35 smithi026 ceph-mon[179688]: rocksdb: Options.sst_partitioner_factory: None 2023-09-23T20:46:35.643 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:46:35 smithi026 ceph-mon[179688]: rocksdb: Options.memtable_factory: SkipListFactory 2023-09-23T20:46:35.643 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:46:35 smithi026 ceph-mon[179688]: rocksdb: Options.table_factory: BlockBasedTable 2023-09-23T20:46:35.643 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:46:35 smithi026 ceph-mon[179688]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x556a24a45520) 2023-09-23T20:46:35.643 INFO:journalctl@ceph.mon.c.smithi026.stdout: cache_index_and_filter_blocks: 1 2023-09-23T20:46:35.643 INFO:journalctl@ceph.mon.c.smithi026.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2023-09-23T20:46:35.643 INFO:journalctl@ceph.mon.c.smithi026.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2023-09-23T20:46:35.643 INFO:journalctl@ceph.mon.c.smithi026.stdout: pin_top_level_index_and_filter: 1 2023-09-23T20:46:35.644 INFO:journalctl@ceph.mon.c.smithi026.stdout: index_type: 0 2023-09-23T20:46:35.644 INFO:journalctl@ceph.mon.c.smithi026.stdout: data_block_index_type: 0 2023-09-23T20:46:35.644 INFO:journalctl@ceph.mon.c.smithi026.stdout: index_shortening: 1 2023-09-23T20:46:35.644 INFO:journalctl@ceph.mon.c.smithi026.stdout: data_block_hash_table_util_ratio: 0.750000 2023-09-23T20:46:35.644 INFO:journalctl@ceph.mon.c.smithi026.stdout: checksum: 4 2023-09-23T20:46:35.644 INFO:journalctl@ceph.mon.c.smithi026.stdout: no_block_cache: 0 2023-09-23T20:46:35.644 INFO:journalctl@ceph.mon.c.smithi026.stdout: block_cache: 0x556a23d57350 2023-09-23T20:46:35.644 INFO:journalctl@ceph.mon.c.smithi026.stdout: block_cache_name: BinnedLRUCache 2023-09-23T20:46:35.644 INFO:journalctl@ceph.mon.c.smithi026.stdout: block_cache_options: 2023-09-23T20:46:35.644 INFO:journalctl@ceph.mon.c.smithi026.stdout: capacity : 536870912 2023-09-23T20:46:35.644 INFO:journalctl@ceph.mon.c.smithi026.stdout: num_shard_bits : 4 2023-09-23T20:46:35.644 INFO:journalctl@ceph.mon.c.smithi026.stdout: strict_capacity_limit : 0 2023-09-23T20:46:35.644 INFO:journalctl@ceph.mon.c.smithi026.stdout: high_pri_pool_ratio: 0.000 2023-09-23T20:46:35.645 INFO:journalctl@ceph.mon.c.smithi026.stdout: block_cache_compressed: (nil) 2023-09-23T20:46:35.645 INFO:journalctl@ceph.mon.c.smithi026.stdout: persistent_cache: (nil) 2023-09-23T20:46:35.645 INFO:journalctl@ceph.mon.c.smithi026.stdout: block_size: 4096 2023-09-23T20:46:35.645 INFO:journalctl@ceph.mon.c.smithi026.stdout: block_size_deviation: 10 2023-09-23T20:46:35.645 INFO:journalctl@ceph.mon.c.smithi026.stdout: block_restart_interval: 16 2023-09-23T20:46:35.645 INFO:journalctl@ceph.mon.c.smithi026.stdout: index_block_restart_interval: 1 2023-09-23T20:46:35.645 INFO:journalctl@ceph.mon.c.smithi026.stdout: metadata_block_size: 4096 2023-09-23T20:46:35.645 INFO:journalctl@ceph.mon.c.smithi026.stdout: partition_filters: 0 2023-09-23T20:46:35.645 INFO:journalctl@ceph.mon.c.smithi026.stdout: use_delta_encoding: 1 2023-09-23T20:46:35.645 INFO:journalctl@ceph.mon.c.smithi026.stdout: filter_policy: bloomfilter 2023-09-23T20:46:35.645 INFO:journalctl@ceph.mon.c.smithi026.stdout: whole_key_filtering: 1 2023-09-23T20:46:35.645 INFO:journalctl@ceph.mon.c.smithi026.stdout: verify_compression: 0 2023-09-23T20:46:35.645 INFO:journalctl@ceph.mon.c.smithi026.stdout: read_amp_bytes_per_bit: 0 2023-09-23T20:46:35.646 INFO:journalctl@ceph.mon.c.smithi026.stdout: format_version: 5 2023-09-23T20:46:35.646 INFO:journalctl@ceph.mon.c.smithi026.stdout: enable_index_compression: 1 2023-09-23T20:46:35.646 INFO:journalctl@ceph.mon.c.smithi026.stdout: block_align: 0 2023-09-23T20:46:35.646 INFO:journalctl@ceph.mon.c.smithi026.stdout: max_auto_readahead_size: 262144 2023-09-23T20:46:35.646 INFO:journalctl@ceph.mon.c.smithi026.stdout: prepopulate_block_cache: 0 2023-09-23T20:46:35.646 INFO:journalctl@ceph.mon.c.smithi026.stdout: initial_auto_readahead_size: 8192 2023-09-23T20:46:35.646 INFO:journalctl@ceph.mon.c.smithi026.stdout: num_file_reads_for_auto_readahead: 2 2023-09-23T20:46:35.646 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:46:35 smithi026 ceph-mon[179688]: rocksdb: Options.write_buffer_size: 33554432 2023-09-23T20:46:35.646 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:46:35 smithi026 ceph-mon[179688]: rocksdb: Options.max_write_buffer_number: 2 2023-09-23T20:46:35.646 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:46:35 smithi026 ceph-mon[179688]: rocksdb: Options.compression: NoCompression 2023-09-23T20:46:35.646 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:46:35 smithi026 ceph-mon[179688]: rocksdb: Options.bottommost_compression: Disabled 2023-09-23T20:46:35.646 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:46:35 smithi026 ceph-mon[179688]: rocksdb: Options.prefix_extractor: nullptr 2023-09-23T20:46:35.646 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:46:35 smithi026 ceph-mon[179688]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2023-09-23T20:46:35.647 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:46:35 smithi026 ceph-mon[179688]: rocksdb: Options.num_levels: 7 2023-09-23T20:46:35.647 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:46:35 smithi026 ceph-mon[179688]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2023-09-23T20:46:35.647 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:46:35 smithi026 ceph-mon[179688]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2023-09-23T20:46:35.647 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:46:35 smithi026 ceph-mon[179688]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2023-09-23T20:46:35.647 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:46:35 smithi026 ceph-mon[179688]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2023-09-23T20:46:35.647 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:46:35 smithi026 ceph-mon[179688]: rocksdb: Options.bottommost_compression_opts.level: 32767 2023-09-23T20:46:35.647 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:46:35 smithi026 ceph-mon[179688]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2023-09-23T20:46:35.647 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:46:35 smithi026 ceph-mon[179688]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2023-09-23T20:46:35.647 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:46:35 smithi026 ceph-mon[179688]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2023-09-23T20:46:35.647 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:46:35 smithi026 ceph-mon[179688]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2023-09-23T20:46:35.647 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:46:35 smithi026 ceph-mon[179688]: rocksdb: Options.bottommost_compression_opts.enabled: false 2023-09-23T20:46:35.647 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:46:35 smithi026 ceph-mon[179688]: rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2023-09-23T20:46:35.647 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:46:35 smithi026 ceph-mon[179688]: rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2023-09-23T20:46:35.648 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:46:35 smithi026 ceph-mon[179688]: rocksdb: Options.compression_opts.window_bits: -14 2023-09-23T20:46:35.648 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:46:35 smithi026 ceph-mon[179688]: rocksdb: Options.compression_opts.level: 32767 2023-09-23T20:46:35.648 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:46:35 smithi026 ceph-mon[179688]: rocksdb: Options.compression_opts.strategy: 0 2023-09-23T20:46:35.648 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:46:35 smithi026 ceph-mon[179688]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2023-09-23T20:46:35.648 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:46:35 smithi026 ceph-mon[179688]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2023-09-23T20:46:35.648 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:46:35 smithi026 ceph-mon[179688]: rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2023-09-23T20:46:35.648 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:46:35 smithi026 ceph-mon[179688]: rocksdb: Options.compression_opts.parallel_threads: 1 2023-09-23T20:46:35.648 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:46:35 smithi026 ceph-mon[179688]: rocksdb: Options.compression_opts.enabled: false 2023-09-23T20:46:35.648 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:46:35 smithi026 ceph-mon[179688]: rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2023-09-23T20:46:35.648 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:46:35 smithi026 ceph-mon[179688]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2023-09-23T20:46:35.648 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:46:35 smithi026 ceph-mon[179688]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2023-09-23T20:46:35.648 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:46:35 smithi026 ceph-mon[179688]: rocksdb: Options.level0_stop_writes_trigger: 36 2023-09-23T20:46:35.648 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:46:35 smithi026 ceph-mon[179688]: rocksdb: Options.target_file_size_base: 67108864 2023-09-23T20:46:35.649 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:46:35 smithi026 ceph-mon[179688]: rocksdb: Options.target_file_size_multiplier: 1 2023-09-23T20:46:35.649 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:46:35 smithi026 ceph-mon[179688]: rocksdb: Options.max_bytes_for_level_base: 268435456 2023-09-23T20:46:35.649 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:46:35 smithi026 ceph-mon[179688]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2023-09-23T20:46:35.649 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:46:35 smithi026 ceph-mon[179688]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2023-09-23T20:46:35.649 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:46:35 smithi026 ceph-mon[179688]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2023-09-23T20:46:35.649 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:46:35 smithi026 ceph-mon[179688]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2023-09-23T20:46:35.649 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:46:35 smithi026 ceph-mon[179688]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2023-09-23T20:46:35.649 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:46:35 smithi026 ceph-mon[179688]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2023-09-23T20:46:35.649 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:46:35 smithi026 ceph-mon[179688]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2023-09-23T20:46:35.649 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:46:35 smithi026 ceph-mon[179688]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2023-09-23T20:46:35.649 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:46:35 smithi026 ceph-mon[179688]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2023-09-23T20:46:35.649 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:46:35 smithi026 ceph-mon[179688]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2023-09-23T20:46:35.649 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:46:35 smithi026 ceph-mon[179688]: rocksdb: Options.max_compaction_bytes: 1677721600 2023-09-23T20:46:35.650 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:46:35 smithi026 ceph-mon[179688]: rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2023-09-23T20:46:35.650 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:46:35 smithi026 ceph-mon[179688]: rocksdb: Options.arena_block_size: 1048576 2023-09-23T20:46:35.650 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:46:35 smithi026 ceph-mon[179688]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2023-09-23T20:46:35.650 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:46:35 smithi026 ceph-mon[179688]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2023-09-23T20:46:35.650 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:46:35 smithi026 ceph-mon[179688]: rocksdb: Options.disable_auto_compactions: 0 2023-09-23T20:46:35.650 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:46:35 smithi026 ceph-mon[179688]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2023-09-23T20:46:35.650 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:46:35 smithi026 ceph-mon[179688]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2023-09-23T20:46:35.650 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:46:35 smithi026 ceph-mon[179688]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2023-09-23T20:46:35.650 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:46:35 smithi026 ceph-mon[179688]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2023-09-23T20:46:35.650 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:46:35 smithi026 ceph-mon[179688]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2023-09-23T20:46:35.650 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:46:35 smithi026 ceph-mon[179688]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2023-09-23T20:46:35.650 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:46:35 smithi026 ceph-mon[179688]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2023-09-23T20:46:35.651 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:46:35 smithi026 ceph-mon[179688]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2023-09-23T20:46:35.651 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:46:35 smithi026 ceph-mon[179688]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2023-09-23T20:46:35.651 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:46:35 smithi026 ceph-mon[179688]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2023-09-23T20:46:35.651 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:46:35 smithi026 ceph-mon[179688]: rocksdb: Options.table_properties_collectors: 2023-09-23T20:46:35.651 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:46:35 smithi026 ceph-mon[179688]: rocksdb: Options.inplace_update_support: 0 2023-09-23T20:46:35.651 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:46:35 smithi026 ceph-mon[179688]: rocksdb: Options.inplace_update_num_locks: 10000 2023-09-23T20:46:35.651 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:46:35 smithi026 ceph-mon[179688]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2023-09-23T20:46:35.651 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:46:35 smithi026 ceph-mon[179688]: rocksdb: Options.memtable_whole_key_filtering: 0 2023-09-23T20:46:35.651 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:46:35 smithi026 ceph-mon[179688]: rocksdb: Options.memtable_huge_page_size: 0 2023-09-23T20:46:35.651 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:46:35 smithi026 ceph-mon[179688]: rocksdb: Options.bloom_locality: 0 2023-09-23T20:46:35.651 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:46:35 smithi026 ceph-mon[179688]: rocksdb: Options.max_successive_merges: 0 2023-09-23T20:46:35.651 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:46:35 smithi026 ceph-mon[179688]: rocksdb: Options.optimize_filters_for_hits: 0 2023-09-23T20:46:35.651 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:46:35 smithi026 ceph-mon[179688]: rocksdb: Options.paranoid_file_checks: 0 2023-09-23T20:46:35.652 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:46:35 smithi026 ceph-mon[179688]: rocksdb: Options.force_consistency_checks: 1 2023-09-23T20:46:35.652 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:46:35 smithi026 ceph-mon[179688]: rocksdb: Options.report_bg_io_stats: 0 2023-09-23T20:46:35.652 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:46:35 smithi026 ceph-mon[179688]: rocksdb: Options.ttl: 2592000 2023-09-23T20:46:35.652 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:46:35 smithi026 ceph-mon[179688]: rocksdb: Options.periodic_compaction_seconds: 0 2023-09-23T20:46:35.652 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:46:35 smithi026 ceph-mon[179688]: rocksdb: Options.preclude_last_level_data_seconds: 0 2023-09-23T20:46:35.652 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:46:35 smithi026 ceph-mon[179688]: rocksdb: Options.preserve_internal_time_seconds: 0 2023-09-23T20:46:35.652 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:46:35 smithi026 ceph-mon[179688]: rocksdb: Options.enable_blob_files: false 2023-09-23T20:46:35.652 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:46:35 smithi026 ceph-mon[179688]: rocksdb: Options.min_blob_size: 0 2023-09-23T20:46:35.652 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:46:35 smithi026 ceph-mon[179688]: rocksdb: Options.blob_file_size: 268435456 2023-09-23T20:46:35.652 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:46:35 smithi026 ceph-mon[179688]: rocksdb: Options.blob_compression_type: NoCompression 2023-09-23T20:46:35.652 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:46:35 smithi026 ceph-mon[179688]: rocksdb: Options.enable_blob_garbage_collection: false 2023-09-23T20:46:35.652 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:46:35 smithi026 ceph-mon[179688]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2023-09-23T20:46:35.652 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:46:35 smithi026 ceph-mon[179688]: rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2023-09-23T20:46:35.653 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:46:35 smithi026 ceph-mon[179688]: rocksdb: Options.blob_compaction_readahead_size: 0 2023-09-23T20:46:35.653 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:46:35 smithi026 ceph-mon[179688]: rocksdb: Options.blob_file_starting_level: 0 2023-09-23T20:46:35.653 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:46:35 smithi026 ceph-mon[179688]: rocksdb: Options.experimental_mempurge_threshold: 0.000000 2023-09-23T20:46:35.653 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:46:35 smithi026 ceph-mon[179688]: rocksdb: [table/block_based/block_based_table_reader.cc:721] At least one SST file opened without unique ID to verify: 27.sst 2023-09-23T20:46:35.653 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:46:35 smithi026 ceph-mon[179688]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2023-09-23T20:46:35.653 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:46:35 smithi026 ceph-mon[179688]: rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-c/store.db/MANIFEST-000009 succeeded,manifest_file_number is 9, next_file_number is 29, last_sequence is 13389, log_number is 25,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2023-09-23T20:46:35.653 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:46:35 smithi026 ceph-mon[179688]: rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 25 2023-09-23T20:46:35.653 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:46:35 smithi026 ceph-mon[179688]: rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: 72687895-73c9-4fa1-ac87-a7ad648c7688 2023-09-23T20:46:35.653 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:46:35 smithi026 ceph-mon[179688]: rocksdb: EVENT_LOG_v1 {"time_micros": 1695501995577148, "job": 1, "event": "recovery_started", "wal_files": [25]} 2023-09-23T20:46:35.653 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:46:35 smithi026 ceph-mon[179688]: rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #25 mode 2 2023-09-23T20:46:35.653 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:46:35 smithi026 ceph-mon[179688]: rocksdb: EVENT_LOG_v1 {"time_micros": 1695501995579768, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 30, "file_size": 250691, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 13384, "largest_seqno": 13689, "table_properties": {"data_size": 248559, "index_size": 800, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 325, "raw_key_size": 2933, "raw_average_key_size": 23, "raw_value_size": 245785, "raw_average_value_size": 1998, "num_data_blocks": 34, "num_entries": 123, "num_filter_entries": 123, "num_deletions": 0, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1695501995, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "72687895-73c9-4fa1-ac87-a7ad648c7688", "db_session_id": "3GHFTF2FEJKD53JUBMNE", "orig_file_number": 30, "seqno_to_time_mapping": "N/A"}} 2023-09-23T20:46:35.653 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:46:35 smithi026 ceph-mon[179688]: rocksdb: EVENT_LOG_v1 {"time_micros": 1695501995579877, "job": 1, "event": "recovery_finished"} 2023-09-23T20:46:35.654 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:46:35 smithi026 ceph-mon[179688]: rocksdb: [db/version_set.cc:5047] Creating manifest 32 2023-09-23T20:46:35.654 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:46:35 smithi026 ceph-mon[179688]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2023-09-23T20:46:35.654 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:46:35 smithi026 ceph-mon[179688]: rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-c/store.db/000025.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2023-09-23T20:46:35.654 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:46:35 smithi026 ceph-mon[179688]: rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x556a23df4000 2023-09-23T20:46:35.654 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:46:35 smithi026 ceph-mon[179688]: rocksdb: DB pointer 0x556a23de0000 2023-09-23T20:46:35.654 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:46:35 smithi026 ceph-mon[179688]: rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2023-09-23T20:46:35.654 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:46:35 smithi026 ceph-mon[179688]: rocksdb: [db/db_impl/db_impl.cc:1111] 2023-09-23T20:46:35.654 INFO:journalctl@ceph.mon.c.smithi026.stdout: ** DB Stats ** 2023-09-23T20:46:35.654 INFO:journalctl@ceph.mon.c.smithi026.stdout: Uptime(secs): 0.0 total, 0.0 interval 2023-09-23T20:46:35.654 INFO:journalctl@ceph.mon.c.smithi026.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2023-09-23T20:46:35.654 INFO:journalctl@ceph.mon.c.smithi026.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2023-09-23T20:46:35.654 INFO:journalctl@ceph.mon.c.smithi026.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2023-09-23T20:46:35.655 INFO:journalctl@ceph.mon.c.smithi026.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2023-09-23T20:46:35.655 INFO:journalctl@ceph.mon.c.smithi026.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2023-09-23T20:46:35.655 INFO:journalctl@ceph.mon.c.smithi026.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2023-09-23T20:46:35.655 INFO:journalctl@ceph.mon.c.smithi026.stdout: 2023-09-23T20:46:35.655 INFO:journalctl@ceph.mon.c.smithi026.stdout: ** Compaction Stats [default] ** 2023-09-23T20:46:35.655 INFO:journalctl@ceph.mon.c.smithi026.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2023-09-23T20:46:35.655 INFO:journalctl@ceph.mon.c.smithi026.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2023-09-23T20:46:35.655 INFO:journalctl@ceph.mon.c.smithi026.stdout: L0 1/0 244.82 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 175.7 0.00 0.00 1 0.001 0 0 0.0 0.0 2023-09-23T20:46:35.655 INFO:journalctl@ceph.mon.c.smithi026.stdout: L6 1/0 7.89 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 0.0 0.0 2023-09-23T20:46:35.655 INFO:journalctl@ceph.mon.c.smithi026.stdout: Sum 2/0 8.13 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 175.7 0.00 0.00 1 0.001 0 0 0.0 0.0 2023-09-23T20:46:35.655 INFO:journalctl@ceph.mon.c.smithi026.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 175.7 0.00 0.00 1 0.001 0 0 0.0 0.0 2023-09-23T20:46:35.655 INFO:journalctl@ceph.mon.c.smithi026.stdout: 2023-09-23T20:46:35.655 INFO:journalctl@ceph.mon.c.smithi026.stdout: ** Compaction Stats [default] ** 2023-09-23T20:46:35.656 INFO:journalctl@ceph.mon.c.smithi026.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2023-09-23T20:46:35.656 INFO:journalctl@ceph.mon.c.smithi026.stdout: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-09-23T20:46:35.656 INFO:journalctl@ceph.mon.c.smithi026.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 175.7 0.00 0.00 1 0.001 0 0 0.0 0.0 2023-09-23T20:46:35.656 INFO:journalctl@ceph.mon.c.smithi026.stdout: 2023-09-23T20:46:35.656 INFO:journalctl@ceph.mon.c.smithi026.stdout: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2023-09-23T20:46:35.656 INFO:journalctl@ceph.mon.c.smithi026.stdout: 2023-09-23T20:46:35.656 INFO:journalctl@ceph.mon.c.smithi026.stdout: Uptime(secs): 0.0 total, 0.0 interval 2023-09-23T20:46:35.656 INFO:journalctl@ceph.mon.c.smithi026.stdout: Flush(GB): cumulative 0.000, interval 0.000 2023-09-23T20:46:35.656 INFO:journalctl@ceph.mon.c.smithi026.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2023-09-23T20:46:35.656 INFO:journalctl@ceph.mon.c.smithi026.stdout: AddFile(Total Files): cumulative 0, interval 0 2023-09-23T20:46:35.656 INFO:journalctl@ceph.mon.c.smithi026.stdout: AddFile(L0 Files): cumulative 0, interval 0 2023-09-23T20:46:35.656 INFO:journalctl@ceph.mon.c.smithi026.stdout: AddFile(Keys): cumulative 0, interval 0 2023-09-23T20:46:35.656 INFO:journalctl@ceph.mon.c.smithi026.stdout: Cumulative compaction: 0.00 GB write, 40.44 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-09-23T20:46:35.657 INFO:journalctl@ceph.mon.c.smithi026.stdout: Interval compaction: 0.00 GB write, 40.44 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-09-23T20:46:35.657 INFO:journalctl@ceph.mon.c.smithi026.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2023-09-23T20:46:35.657 INFO:journalctl@ceph.mon.c.smithi026.stdout: Block cache BinnedLRUCache@0x556a23d57350#8 capacity: 512.00 MB usage: 1.33 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 1e-05 secs_since: 0 2023-09-23T20:46:35.657 INFO:journalctl@ceph.mon.c.smithi026.stdout: Block cache entry stats(count,size,portion): FilterBlock(1,0.38 KB,7.15256e-05%) IndexBlock(1,0.95 KB,0.000181794%) Misc(1,0.00 KB,0%) 2023-09-23T20:46:35.657 INFO:journalctl@ceph.mon.c.smithi026.stdout: 2023-09-23T20:46:35.657 INFO:journalctl@ceph.mon.c.smithi026.stdout: ** File Read Latency Histogram By Level [default] ** 2023-09-23T20:46:35.657 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:46:35 smithi026 ceph-mon[179688]: starting mon.c rank 2 at public addrs [v2:172.21.15.26:3301/0,v1:172.21.15.26:6790/0] at bind addrs [v2:172.21.15.26:3301/0,v1:172.21.15.26:6790/0] mon_data /var/lib/ceph/mon/ceph-c fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 2023-09-23T20:46:35.657 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:46:35 smithi026 ceph-mon[179688]: mon.c@-1(???) e3 preinit fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 2023-09-23T20:46:35.657 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:46:35 smithi026 ceph-mon[179688]: mon.c@-1(???).mds e8 new map 2023-09-23T20:46:35.657 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:46:35 smithi026 ceph-mon[179688]: mon.c@-1(???).mds e8 print_map 2023-09-23T20:46:35.657 INFO:journalctl@ceph.mon.c.smithi026.stdout: e8 2023-09-23T20:46:35.657 INFO:journalctl@ceph.mon.c.smithi026.stdout: enable_multiple, ever_enabled_multiple: 1,1 2023-09-23T20:46:35.658 INFO:journalctl@ceph.mon.c.smithi026.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2023-09-23T20:46:35.658 INFO:journalctl@ceph.mon.c.smithi026.stdout: legacy client fscid: 1 2023-09-23T20:46:35.658 INFO:journalctl@ceph.mon.c.smithi026.stdout: 2023-09-23T20:46:35.658 INFO:journalctl@ceph.mon.c.smithi026.stdout: Filesystem 'foo' (1) 2023-09-23T20:46:35.658 INFO:journalctl@ceph.mon.c.smithi026.stdout: fs_name foo 2023-09-23T20:46:35.658 INFO:journalctl@ceph.mon.c.smithi026.stdout: epoch 7 2023-09-23T20:46:35.658 INFO:journalctl@ceph.mon.c.smithi026.stdout: flags 12 joinable allow_snaps allow_multimds_snaps 2023-09-23T20:46:35.658 INFO:journalctl@ceph.mon.c.smithi026.stdout: created 2023-09-23T20:41:58.904985+0000 2023-09-23T20:46:35.658 INFO:journalctl@ceph.mon.c.smithi026.stdout: modified 2023-09-23T20:42:07.570998+0000 2023-09-23T20:46:35.658 INFO:journalctl@ceph.mon.c.smithi026.stdout: tableserver 0 2023-09-23T20:46:35.658 INFO:journalctl@ceph.mon.c.smithi026.stdout: root 0 2023-09-23T20:46:35.658 INFO:journalctl@ceph.mon.c.smithi026.stdout: session_timeout 60 2023-09-23T20:46:35.659 INFO:journalctl@ceph.mon.c.smithi026.stdout: session_autoclose 300 2023-09-23T20:46:35.659 INFO:journalctl@ceph.mon.c.smithi026.stdout: max_file_size 1099511627776 2023-09-23T20:46:35.659 INFO:journalctl@ceph.mon.c.smithi026.stdout: required_client_features {} 2023-09-23T20:46:35.659 INFO:journalctl@ceph.mon.c.smithi026.stdout: last_failure 0 2023-09-23T20:46:35.659 INFO:journalctl@ceph.mon.c.smithi026.stdout: last_failure_osd_epoch 0 2023-09-23T20:46:35.659 INFO:journalctl@ceph.mon.c.smithi026.stdout: compat compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,7=mds uses inline data,8=no anchor table,9=file layout v2,10=snaprealm v2} 2023-09-23T20:46:35.659 INFO:journalctl@ceph.mon.c.smithi026.stdout: max_mds 1 2023-09-23T20:46:35.659 INFO:journalctl@ceph.mon.c.smithi026.stdout: in 0 2023-09-23T20:46:35.659 INFO:journalctl@ceph.mon.c.smithi026.stdout: up {0=14658} 2023-09-23T20:46:35.659 INFO:journalctl@ceph.mon.c.smithi026.stdout: failed 2023-09-23T20:46:35.659 INFO:journalctl@ceph.mon.c.smithi026.stdout: damaged 2023-09-23T20:46:35.659 INFO:journalctl@ceph.mon.c.smithi026.stdout: stopped 2023-09-23T20:46:35.659 INFO:journalctl@ceph.mon.c.smithi026.stdout: data_pools [4] 2023-09-23T20:46:35.660 INFO:journalctl@ceph.mon.c.smithi026.stdout: metadata_pool 3 2023-09-23T20:46:35.660 INFO:journalctl@ceph.mon.c.smithi026.stdout: inline_data disabled 2023-09-23T20:46:35.660 INFO:journalctl@ceph.mon.c.smithi026.stdout: balancer 2023-09-23T20:46:35.660 INFO:journalctl@ceph.mon.c.smithi026.stdout: bal_rank_mask -1 2023-09-23T20:46:35.660 INFO:journalctl@ceph.mon.c.smithi026.stdout: standby_count_wanted 1 2023-09-23T20:46:35.660 INFO:journalctl@ceph.mon.c.smithi026.stdout: [mds.foo.smithi026.znfpnl{0:14658} state up:active seq 3 join_fscid=1 addr [v2:172.21.15.26:6834/417830853,v1:172.21.15.26:6835/417830853] compat {c=[1],r=[1],i=[7ff]}] 2023-09-23T20:46:35.660 INFO:journalctl@ceph.mon.c.smithi026.stdout: 2023-09-23T20:46:35.660 INFO:journalctl@ceph.mon.c.smithi026.stdout: 2023-09-23T20:46:35.660 INFO:journalctl@ceph.mon.c.smithi026.stdout: Standby daemons: 2023-09-23T20:46:35.660 INFO:journalctl@ceph.mon.c.smithi026.stdout: 2023-09-23T20:46:35.660 INFO:journalctl@ceph.mon.c.smithi026.stdout: [mds.foo.smithi070.imqhqy{-1:24529} state up:standby seq 2 join_fscid=1 addr [v2:172.21.15.70:6832/241309560,v1:172.21.15.70:6833/241309560] compat {c=[1],r=[1],i=[7ff]}] 2023-09-23T20:46:35.660 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:46:35 smithi026 ceph-mon[179688]: mon.c@-1(???).osd e79 crush map has features 3314933000854323200, adjusting msgr requires 2023-09-23T20:46:35.660 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:46:35 smithi026 ceph-mon[179688]: mon.c@-1(???).osd e79 crush map has features 432629239337189376, adjusting msgr requires 2023-09-23T20:46:35.661 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:46:35 smithi026 ceph-mon[179688]: mon.c@-1(???).osd e79 crush map has features 432629239337189376, adjusting msgr requires 2023-09-23T20:46:35.661 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:46:35 smithi026 ceph-mon[179688]: mon.c@-1(???).osd e79 crush map has features 432629239337189376, adjusting msgr requires 2023-09-23T20:46:35.661 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:46:35 smithi026 ceph-mon[179688]: mon.c@-1(???).paxosservice(auth 1..19) refresh upgraded, format 0 -> 3 2023-09-23T20:46:35.661 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:46:35 smithi026 ceph-mon[179688]: expand_channel_meta expand map: {default=false} 2023-09-23T20:46:35.661 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:46:35 smithi026 ceph-mon[179688]: expand_channel_meta from 'false' to 'false' 2023-09-23T20:46:35.661 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:46:35 smithi026 ceph-mon[179688]: expand_channel_meta expanded map: {default=false} 2023-09-23T20:46:35.661 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:46:35 smithi026 ceph-mon[179688]: expand_channel_meta expand map: {default=info} 2023-09-23T20:46:35.661 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:46:35 smithi026 ceph-mon[179688]: expand_channel_meta from 'info' to 'info' 2023-09-23T20:46:35.661 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:46:35 smithi026 ceph-mon[179688]: expand_channel_meta expanded map: {default=info} 2023-09-23T20:46:35.661 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:46:35 smithi026 ceph-mon[179688]: expand_channel_meta expand map: {default=daemon} 2023-09-23T20:46:35.661 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:46:35 smithi026 ceph-mon[179688]: expand_channel_meta from 'daemon' to 'daemon' 2023-09-23T20:46:35.661 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:46:35 smithi026 ceph-mon[179688]: expand_channel_meta expanded map: {default=daemon} 2023-09-23T20:46:35.662 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:46:35 smithi026 ceph-mon[179688]: expand_channel_meta expand map: {default=debug} 2023-09-23T20:46:35.662 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:46:35 smithi026 ceph-mon[179688]: expand_channel_meta from 'debug' to 'debug' 2023-09-23T20:46:35.662 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:46:35 smithi026 ceph-mon[179688]: expand_channel_meta expanded map: {default=debug} 2023-09-23T20:46:36.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:46:35 smithi026 systemd[1]: Started Ceph mon.c for b3f8e4ec-5a50-11ee-8db4-212e2dc638e7. 2023-09-23T20:46:39.119 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 20:46:38 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-prometheus-a[155975]: ts=2023-09-23T20:46:38.660Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1123-g61431f6b (61431f6ba2752649b506867ec856b4a6236f90dd) quincy (stable)\", cluster_addr=\"172.21.15.26\", device_class=\"ssd\", hostname=\"smithi026\", instance=\"smithi070.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.26\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1123-g61431f6b (61431f6ba2752649b506867ec856b4a6236f90dd) quincy (stable)\", cluster_addr=\"172.21.15.26\", device_class=\"ssd\", hostname=\"smithi026\", instance=\"smithi026:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.26\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-09-23T20:46:39.882 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:46:39 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[164029]: ::ffff:172.21.15.70 - - [23/Sep/2023:20:46:39] "GET /metrics HTTP/1.1" 200 33696 "" "Prometheus/2.43.0" 2023-09-23T20:46:42.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:46:41 smithi070 ceph-mon[100755]: mon.c calling monitor election 2023-09-23T20:46:42.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:46:41 smithi070 ceph-mon[100755]: mon.b calling monitor election 2023-09-23T20:46:42.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:46:41 smithi070 ceph-mon[100755]: mon.a calling monitor election 2023-09-23T20:46:42.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:46:41 smithi070 ceph-mon[100755]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2023-09-23T20:46:42.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:46:41 smithi070 ceph-mon[100755]: monmap e3: 3 mons at {a=[v2:172.21.15.26:3300/0,v1:172.21.15.26:6789/0],b=[v2:172.21.15.70:3300/0,v1:172.21.15.70:6789/0],c=[v2:172.21.15.26:3301/0,v1:172.21.15.26:6790/0]} removed_ranks: {} 2023-09-23T20:46:42.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:46:41 smithi070 ceph-mon[100755]: fsmap foo:1 {0=foo.smithi026.znfpnl=up:active} 1 up:standby 2023-09-23T20:46:42.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:46:41 smithi070 ceph-mon[100755]: osdmap e79: 8 total, 8 up, 8 in 2023-09-23T20:46:42.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:46:41 smithi070 ceph-mon[100755]: mgrmap e36: y(active, since 85s), standbys: x 2023-09-23T20:46:42.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:46:41 smithi070 ceph-mon[100755]: Health check cleared: MON_DOWN (was: 1/3 mons down, quorum a,b) 2023-09-23T20:46:42.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:46:41 smithi070 ceph-mon[100755]: Cluster is now healthy 2023-09-23T20:46:42.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:46:41 smithi070 ceph-mon[100755]: overall HEALTH_OK 2023-09-23T20:46:42.131 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:46:41 smithi026 ceph-mon[175151]: mon.c calling monitor election 2023-09-23T20:46:42.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:46:41 smithi026 ceph-mon[175151]: mon.b calling monitor election 2023-09-23T20:46:42.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:46:41 smithi026 ceph-mon[175151]: mon.a calling monitor election 2023-09-23T20:46:42.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:46:41 smithi026 ceph-mon[175151]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2023-09-23T20:46:42.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:46:41 smithi026 ceph-mon[175151]: monmap e3: 3 mons at {a=[v2:172.21.15.26:3300/0,v1:172.21.15.26:6789/0],b=[v2:172.21.15.70:3300/0,v1:172.21.15.70:6789/0],c=[v2:172.21.15.26:3301/0,v1:172.21.15.26:6790/0]} removed_ranks: {} 2023-09-23T20:46:42.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:46:41 smithi026 ceph-mon[175151]: fsmap foo:1 {0=foo.smithi026.znfpnl=up:active} 1 up:standby 2023-09-23T20:46:42.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:46:41 smithi026 ceph-mon[175151]: osdmap e79: 8 total, 8 up, 8 in 2023-09-23T20:46:42.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:46:41 smithi026 ceph-mon[175151]: mgrmap e36: y(active, since 85s), standbys: x 2023-09-23T20:46:42.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:46:41 smithi026 ceph-mon[175151]: Health check cleared: MON_DOWN (was: 1/3 mons down, quorum a,b) 2023-09-23T20:46:42.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:46:41 smithi026 ceph-mon[175151]: Cluster is now healthy 2023-09-23T20:46:42.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:46:41 smithi026 ceph-mon[175151]: overall HEALTH_OK 2023-09-23T20:46:42.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:46:41 smithi026 ceph-mon[179688]: mon.c calling monitor election 2023-09-23T20:46:42.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:46:41 smithi026 ceph-mon[179688]: mon.b calling monitor election 2023-09-23T20:46:42.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:46:41 smithi026 ceph-mon[179688]: mon.a calling monitor election 2023-09-23T20:46:42.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:46:41 smithi026 ceph-mon[179688]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2023-09-23T20:46:42.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:46:41 smithi026 ceph-mon[179688]: monmap e3: 3 mons at {a=[v2:172.21.15.26:3300/0,v1:172.21.15.26:6789/0],b=[v2:172.21.15.70:3300/0,v1:172.21.15.70:6789/0],c=[v2:172.21.15.26:3301/0,v1:172.21.15.26:6790/0]} removed_ranks: {} 2023-09-23T20:46:42.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:46:41 smithi026 ceph-mon[179688]: fsmap foo:1 {0=foo.smithi026.znfpnl=up:active} 1 up:standby 2023-09-23T20:46:42.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:46:41 smithi026 ceph-mon[179688]: osdmap e79: 8 total, 8 up, 8 in 2023-09-23T20:46:42.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:46:41 smithi026 ceph-mon[179688]: mgrmap e36: y(active, since 85s), standbys: x 2023-09-23T20:46:42.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:46:41 smithi026 ceph-mon[179688]: Health check cleared: MON_DOWN (was: 1/3 mons down, quorum a,b) 2023-09-23T20:46:42.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:46:41 smithi026 ceph-mon[179688]: Cluster is now healthy 2023-09-23T20:46:42.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:46:41 smithi026 ceph-mon[179688]: overall HEALTH_OK 2023-09-23T20:46:43.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:46:42 smithi070 ceph-mon[100755]: pgmap v46: 73 pgs: 73 active+clean; 582 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:46:43.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:46:42 smithi026 ceph-mon[175151]: pgmap v46: 73 pgs: 73 active+clean; 582 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:46:43.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:46:42 smithi026 ceph-mon[179688]: pgmap v46: 73 pgs: 73 active+clean; 582 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:46:43.684 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 18.2.0-460-gbb0daeff (bb0daeff97f020f686596fca6eba20c310e26fb7) reef (stable)": 2 2023-09-23T20:46:44.050 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:46:43 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/381828703' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:46:44.050 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:46:43 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/381828703' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:46:44.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:46:43 smithi070 ceph-mon[100755]: from='client.? 172.21.15.26:0/381828703' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:46:44.144 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -e sha1=bb0daeff97f020f686596fca6eba20c310e26fb7 -- bash -c 'ceph orch ps' 2023-09-23T20:46:44.868 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:46:45.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:46:45 smithi026 ceph-mon[175151]: pgmap v47: 73 pgs: 73 active+clean; 582 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:46:45.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:46:45 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:46:45.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:46:45 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:46:45.882 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:46:45 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[164029]: 2023-09-23T20:46:45.593+0000 7fd59efc8700 -1 mgr.server handle_report got status from non-daemon mon.c 2023-09-23T20:46:45.883 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:46:45 smithi026 ceph-mon[179688]: pgmap v47: 73 pgs: 73 active+clean; 582 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:46:45.883 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:46:45 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:46:45.883 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:46:45 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:46:46.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:46:45 smithi070 ceph-mon[100755]: pgmap v47: 73 pgs: 73 active+clean; 582 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:46:46.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:46:45 smithi070 ceph-mon[100755]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:46:46.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:46:45 smithi070 ceph-mon[100755]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:46:46.579 INFO:teuthology.orchestra.run.smithi026.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2023-09-23T20:46:46.579 INFO:teuthology.orchestra.run.smithi026.stdout:alertmanager.a smithi026 *:9093,9094 running (2m) - 5m 28.6M - 0.25.0 c8568f914cd2 d4b48de0f10c 2023-09-23T20:46:46.579 INFO:teuthology.orchestra.run.smithi026.stdout:grafana.a smithi070 *:3000 running (2m) - 5m 84.0M - 9.4.7 2c41d148cca3 ed7e189354a5 2023-09-23T20:46:46.579 INFO:teuthology.orchestra.run.smithi026.stdout:mds.foo.smithi026.znfpnl smithi026 running (4m) - 4m 18.6M - 17.2.6-1123-g61431f6b 0db14aa01094 b6861ac0ff55 2023-09-23T20:46:46.579 INFO:teuthology.orchestra.run.smithi026.stdout:mds.foo.smithi070.imqhqy smithi070 running (4m) - 4m 17.4M - 17.2.6-1123-g61431f6b 0db14aa01094 ca26bf551eb9 2023-09-23T20:46:46.579 INFO:teuthology.orchestra.run.smithi026.stdout:mgr.x smithi070 *:8443,9283,8765 running (116s) - 8m 447M - 18.2.0-460-gbb0daeff e32c2a64c126 35059b639121 2023-09-23T20:46:46.580 INFO:teuthology.orchestra.run.smithi026.stdout:mgr.y smithi026 *:8443,9283,8765 running (2m) - 10m 502M - 18.2.0-460-gbb0daeff e32c2a64c126 5cc7ece3b7e8 2023-09-23T20:46:46.580 INFO:teuthology.orchestra.run.smithi026.stdout:mon.a smithi026 running (102s) - 10m 55.6M 2048M 18.2.0-460-gbb0daeff e32c2a64c126 f0ba51b53f50 2023-09-23T20:46:46.580 INFO:teuthology.orchestra.run.smithi026.stdout:mon.b smithi070 running (9m) - 9m 76.3M 2048M 17.2.6-1123-g61431f6b 0db14aa01094 2c8a2924d3d7 2023-09-23T20:46:46.580 INFO:teuthology.orchestra.run.smithi026.stdout:mon.c smithi026 running (11s) - 9m 18.2M 2048M 18.2.0-460-gbb0daeff e32c2a64c126 ad100e9213d3 2023-09-23T20:46:46.580 INFO:teuthology.orchestra.run.smithi026.stdout:node-exporter.a smithi026 *:9100 running (2m) - 5m 16.9M - 1.5.0 0da6a335fe13 95269f65f9af 2023-09-23T20:46:46.580 INFO:teuthology.orchestra.run.smithi026.stdout:node-exporter.b smithi070 *:9100 running (2m) - 5m 17.8M - 1.5.0 0da6a335fe13 5b56a4a415d0 2023-09-23T20:46:46.580 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0 smithi026 running (8m) - 8m 79.8M 2501M 17.2.6-1123-g61431f6b 0db14aa01094 e40b0407b8cf 2023-09-23T20:46:46.580 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1 smithi026 running (8m) - 8m 80.4M 2501M 17.2.6-1123-g61431f6b 0db14aa01094 a73d47580165 2023-09-23T20:46:46.580 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2 smithi026 running (7m) - 7m 149M 2501M 17.2.6-1123-g61431f6b 0db14aa01094 fc4783fb1a89 2023-09-23T20:46:46.580 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3 smithi026 running (7m) - 7m 70.5M 2501M 17.2.6-1123-g61431f6b 0db14aa01094 5f0339ddc2c4 2023-09-23T20:46:46.580 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4 smithi070 running (7m) - 7m 87.9M 2500M 17.2.6-1123-g61431f6b 0db14aa01094 308a2af3101e 2023-09-23T20:46:46.580 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5 smithi070 running (6m) - 6m 100M 2500M 17.2.6-1123-g61431f6b 0db14aa01094 16cd72517077 2023-09-23T20:46:46.581 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6 smithi070 running (6m) - 6m 65.5M 2500M 17.2.6-1123-g61431f6b 0db14aa01094 5d24c21e8be7 2023-09-23T20:46:46.581 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7 smithi070 running (6m) - 6m 105M 2500M 17.2.6-1123-g61431f6b 0db14aa01094 8c1b7f314c41 2023-09-23T20:46:46.581 INFO:teuthology.orchestra.run.smithi026.stdout:prometheus.a smithi070 *:9095 running (2m) - 5m 52.6M - 2.43.0 a07b618ecd1d dda227f93524 2023-09-23T20:46:46.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:46:46 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:46:46.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:46:46 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:46:46.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:46:46 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:46:46.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:46:46 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:46:46.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:46:46 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:46:46.883 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:46:46 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:46:46.980 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -e sha1=bb0daeff97f020f686596fca6eba20c310e26fb7 -- bash -c 'ceph orch upgrade pause' 2023-09-23T20:46:47.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:46:46 smithi070 ceph-mon[100755]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:46:47.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:46:46 smithi070 ceph-mon[100755]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:46:47.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:46:46 smithi070 ceph-mon[100755]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:46:47.694 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:46:47.777 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:46:47 smithi026 ceph-mon[179688]: pgmap v48: 73 pgs: 73 active+clean; 582 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:46:47.777 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:46:47 smithi026 ceph-mon[179688]: from='client.44102 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T20:46:47.777 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:46:47 smithi026 ceph-mon[179688]: mgrmap e37: y(active, since 90s), standbys: x 2023-09-23T20:46:47.777 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:46:47 smithi026 ceph-mon[175151]: pgmap v48: 73 pgs: 73 active+clean; 582 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:46:47.778 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:46:47 smithi026 ceph-mon[175151]: from='client.44102 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T20:46:47.778 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:46:47 smithi026 ceph-mon[175151]: mgrmap e37: y(active, since 90s), standbys: x 2023-09-23T20:46:48.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:46:47 smithi070 ceph-mon[100755]: pgmap v48: 73 pgs: 73 active+clean; 582 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:46:48.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:46:47 smithi070 ceph-mon[100755]: from='client.44102 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T20:46:48.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:46:47 smithi070 ceph-mon[100755]: mgrmap e37: y(active, since 90s), standbys: x 2023-09-23T20:46:49.119 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 20:46:48 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-prometheus-a[155975]: ts=2023-09-23T20:46:48.660Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1123-g61431f6b (61431f6ba2752649b506867ec856b4a6236f90dd) quincy (stable)\", cluster_addr=\"172.21.15.26\", device_class=\"ssd\", hostname=\"smithi026\", instance=\"smithi070.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.26\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1123-g61431f6b (61431f6ba2752649b506867ec856b4a6236f90dd) quincy (stable)\", cluster_addr=\"172.21.15.26\", device_class=\"ssd\", hostname=\"smithi026\", instance=\"smithi026:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.26\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-09-23T20:46:49.616 INFO:teuthology.orchestra.run.smithi026.stdout:Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d 2023-09-23T20:46:49.785 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:46:49 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[164029]: ::ffff:172.21.15.70 - - [23/Sep/2023:20:46:49] "GET /metrics HTTP/1.1" 200 33693 "" "Prometheus/2.43.0" 2023-09-23T20:46:49.785 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:46:49 smithi026 ceph-mon[175151]: pgmap v49: 73 pgs: 73 active+clean; 582 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2023-09-23T20:46:49.785 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:46:49 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:46:49.785 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:46:49 smithi026 ceph-mon[179688]: pgmap v49: 73 pgs: 73 active+clean; 582 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2023-09-23T20:46:49.786 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:46:49 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:46:50.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:46:49 smithi070 ceph-mon[100755]: pgmap v49: 73 pgs: 73 active+clean; 582 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2023-09-23T20:46:50.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:46:49 smithi070 ceph-mon[100755]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:46:50.121 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -e sha1=bb0daeff97f020f686596fca6eba20c310e26fb7 -- bash -c 'sleep 60' 2023-09-23T20:46:50.732 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:46:50.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:46:50 smithi026 ceph-mon[179688]: from='client.44108 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T20:46:50.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:46:50 smithi026 ceph-mon[179688]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d 2023-09-23T20:46:50.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:46:50 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:46:50.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:46:50 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:46:50.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:46:50 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:46:50.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:46:50 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:46:50.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:46:50 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:46:50.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:46:50 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:46:50.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:46:50 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:46:50.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:46:50 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:46:50.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:46:50 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:46:50.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:46:50 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:46:50.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:46:50 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:46:50.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:46:50 smithi026 ceph-mon[175151]: from='client.44108 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T20:46:50.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:46:50 smithi026 ceph-mon[175151]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d 2023-09-23T20:46:50.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:46:50 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:46:50.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:46:50 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:46:50.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:46:50 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:46:50.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:46:50 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:46:50.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:46:50 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:46:50.987 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:46:50 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:46:50.987 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:46:50 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:46:50.987 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:46:50 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:46:50.987 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:46:50 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:46:50.987 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:46:50 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:46:50.987 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:46:50 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:46:51.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:46:50 smithi070 ceph-mon[100755]: from='client.44108 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T20:46:51.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:46:50 smithi070 ceph-mon[100755]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d 2023-09-23T20:46:51.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:46:50 smithi070 ceph-mon[100755]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:46:51.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:46:50 smithi070 ceph-mon[100755]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:46:51.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:46:50 smithi070 ceph-mon[100755]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:46:51.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:46:50 smithi070 ceph-mon[100755]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:46:51.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:46:50 smithi070 ceph-mon[100755]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:46:51.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:46:50 smithi070 ceph-mon[100755]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:46:51.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:46:50 smithi070 ceph-mon[100755]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:46:51.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:46:50 smithi070 ceph-mon[100755]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:46:51.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:46:50 smithi070 ceph-mon[100755]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:46:51.121 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:46:50 smithi070 ceph-mon[100755]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:46:51.121 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:46:50 smithi070 ceph-mon[100755]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:46:51.800 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:46:51 smithi026 ceph-mon[175151]: pgmap v50: 73 pgs: 73 active+clean; 582 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2023-09-23T20:46:51.801 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:46:51 smithi026 ceph-mon[179688]: pgmap v50: 73 pgs: 73 active+clean; 582 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2023-09-23T20:46:52.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:46:51 smithi070 ceph-mon[100755]: pgmap v50: 73 pgs: 73 active+clean; 582 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2023-09-23T20:46:54.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:46:53 smithi070 ceph-mon[100755]: pgmap v51: 73 pgs: 73 active+clean; 582 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2023-09-23T20:46:54.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:46:53 smithi026 ceph-mon[175151]: pgmap v51: 73 pgs: 73 active+clean; 582 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2023-09-23T20:46:54.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:46:53 smithi026 ceph-mon[179688]: pgmap v51: 73 pgs: 73 active+clean; 582 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2023-09-23T20:46:56.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:46:55 smithi070 ceph-mon[100755]: pgmap v52: 73 pgs: 73 active+clean; 582 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2023-09-23T20:46:56.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:46:55 smithi026 ceph-mon[175151]: pgmap v52: 73 pgs: 73 active+clean; 582 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2023-09-23T20:46:56.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:46:55 smithi026 ceph-mon[179688]: pgmap v52: 73 pgs: 73 active+clean; 582 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2023-09-23T20:46:58.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:46:57 smithi070 ceph-mon[100755]: pgmap v53: 73 pgs: 73 active+clean; 582 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2023-09-23T20:46:58.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:46:57 smithi026 ceph-mon[175151]: pgmap v53: 73 pgs: 73 active+clean; 582 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2023-09-23T20:46:58.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:46:57 smithi026 ceph-mon[179688]: pgmap v53: 73 pgs: 73 active+clean; 582 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2023-09-23T20:46:59.120 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 20:46:58 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-prometheus-a[155975]: ts=2023-09-23T20:46:58.659Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1123-g61431f6b (61431f6ba2752649b506867ec856b4a6236f90dd) quincy (stable)\", cluster_addr=\"172.21.15.26\", device_class=\"ssd\", hostname=\"smithi026\", instance=\"smithi070.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.26\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1123-g61431f6b (61431f6ba2752649b506867ec856b4a6236f90dd) quincy (stable)\", cluster_addr=\"172.21.15.26\", device_class=\"ssd\", hostname=\"smithi026\", instance=\"smithi026:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.26\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-09-23T20:46:59.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:46:59 smithi026 ceph-mon[175151]: pgmap v54: 73 pgs: 73 active+clean; 582 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2023-09-23T20:46:59.882 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:46:59 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[164029]: ::ffff:172.21.15.70 - - [23/Sep/2023:20:46:59] "GET /metrics HTTP/1.1" 200 33693 "" "Prometheus/2.43.0" 2023-09-23T20:46:59.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:46:59 smithi026 ceph-mon[179688]: pgmap v54: 73 pgs: 73 active+clean; 582 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2023-09-23T20:47:00.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:46:59 smithi070 ceph-mon[100755]: pgmap v54: 73 pgs: 73 active+clean; 582 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2023-09-23T20:47:02.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:47:01 smithi070 ceph-mon[100755]: pgmap v55: 73 pgs: 73 active+clean; 582 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:47:02.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:47:01 smithi070 ceph-mon[100755]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:47:02.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:47:01 smithi026 ceph-mon[175151]: pgmap v55: 73 pgs: 73 active+clean; 582 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:47:02.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:47:01 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:47:02.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:47:01 smithi026 ceph-mon[179688]: pgmap v55: 73 pgs: 73 active+clean; 582 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:47:02.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:47:01 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:47:04.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:47:03 smithi070 ceph-mon[100755]: pgmap v56: 73 pgs: 73 active+clean; 582 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:47:04.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:47:03 smithi026 ceph-mon[175151]: pgmap v56: 73 pgs: 73 active+clean; 582 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:47:04.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:47:03 smithi026 ceph-mon[179688]: pgmap v56: 73 pgs: 73 active+clean; 582 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:47:06.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:47:05 smithi070 ceph-mon[100755]: pgmap v57: 73 pgs: 73 active+clean; 582 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:47:06.131 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:47:05 smithi026 ceph-mon[175151]: pgmap v57: 73 pgs: 73 active+clean; 582 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:47:06.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:47:05 smithi026 ceph-mon[179688]: pgmap v57: 73 pgs: 73 active+clean; 582 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:47:08.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:47:07 smithi070 ceph-mon[100755]: pgmap v58: 73 pgs: 73 active+clean; 582 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:47:08.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:47:07 smithi026 ceph-mon[175151]: pgmap v58: 73 pgs: 73 active+clean; 582 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:47:08.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:47:07 smithi026 ceph-mon[179688]: pgmap v58: 73 pgs: 73 active+clean; 582 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:47:09.119 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 20:47:08 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-prometheus-a[155975]: ts=2023-09-23T20:47:08.660Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1123-g61431f6b (61431f6ba2752649b506867ec856b4a6236f90dd) quincy (stable)\", cluster_addr=\"172.21.15.26\", device_class=\"ssd\", hostname=\"smithi026\", instance=\"smithi070.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.26\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1123-g61431f6b (61431f6ba2752649b506867ec856b4a6236f90dd) quincy (stable)\", cluster_addr=\"172.21.15.26\", device_class=\"ssd\", hostname=\"smithi026\", instance=\"smithi026:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.26\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-09-23T20:47:09.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:47:09 smithi026 ceph-mon[175151]: pgmap v59: 73 pgs: 73 active+clean; 582 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:47:09.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:47:09 smithi026 ceph-mon[179688]: pgmap v59: 73 pgs: 73 active+clean; 582 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:47:09.883 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:47:09 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[164029]: ::ffff:172.21.15.70 - - [23/Sep/2023:20:47:09] "GET /metrics HTTP/1.1" 200 33691 "" "Prometheus/2.43.0" 2023-09-23T20:47:10.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:47:09 smithi070 ceph-mon[100755]: pgmap v59: 73 pgs: 73 active+clean; 582 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:47:12.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:47:11 smithi070 ceph-mon[100755]: pgmap v60: 73 pgs: 73 active+clean; 582 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:47:12.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:47:11 smithi026 ceph-mon[175151]: pgmap v60: 73 pgs: 73 active+clean; 582 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:47:12.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:47:11 smithi026 ceph-mon[179688]: pgmap v60: 73 pgs: 73 active+clean; 582 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:47:14.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:47:13 smithi070 ceph-mon[100755]: pgmap v61: 73 pgs: 73 active+clean; 582 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:47:14.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:47:13 smithi026 ceph-mon[175151]: pgmap v61: 73 pgs: 73 active+clean; 582 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:47:14.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:47:13 smithi026 ceph-mon[179688]: pgmap v61: 73 pgs: 73 active+clean; 582 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:47:16.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:47:15 smithi070 ceph-mon[100755]: pgmap v62: 73 pgs: 73 active+clean; 582 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:47:16.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:47:15 smithi026 ceph-mon[175151]: pgmap v62: 73 pgs: 73 active+clean; 582 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:47:16.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:47:15 smithi026 ceph-mon[179688]: pgmap v62: 73 pgs: 73 active+clean; 582 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:47:17.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:47:16 smithi070 ceph-mon[100755]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:47:17.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:47:16 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:47:17.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:47:16 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:47:18.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:47:17 smithi070 ceph-mon[100755]: pgmap v63: 73 pgs: 73 active+clean; 582 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:47:18.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:47:17 smithi026 ceph-mon[175151]: pgmap v63: 73 pgs: 73 active+clean; 582 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:47:18.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:47:17 smithi026 ceph-mon[179688]: pgmap v63: 73 pgs: 73 active+clean; 582 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:47:19.119 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 20:47:18 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-prometheus-a[155975]: ts=2023-09-23T20:47:18.660Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1123-g61431f6b (61431f6ba2752649b506867ec856b4a6236f90dd) quincy (stable)\", cluster_addr=\"172.21.15.26\", device_class=\"ssd\", hostname=\"smithi026\", instance=\"smithi070.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.26\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1123-g61431f6b (61431f6ba2752649b506867ec856b4a6236f90dd) quincy (stable)\", cluster_addr=\"172.21.15.26\", device_class=\"ssd\", hostname=\"smithi026\", instance=\"smithi026:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.26\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-09-23T20:47:19.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:47:19 smithi026 ceph-mon[175151]: pgmap v64: 73 pgs: 73 active+clean; 582 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:47:19.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:47:19 smithi026 ceph-mon[179688]: pgmap v64: 73 pgs: 73 active+clean; 582 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:47:19.883 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:47:19 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[164029]: ::ffff:172.21.15.70 - - [23/Sep/2023:20:47:19] "GET /metrics HTTP/1.1" 200 33687 "" "Prometheus/2.43.0" 2023-09-23T20:47:20.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:47:19 smithi070 ceph-mon[100755]: pgmap v64: 73 pgs: 73 active+clean; 582 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:47:22.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:47:21 smithi070 ceph-mon[100755]: pgmap v65: 73 pgs: 73 active+clean; 582 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:47:22.131 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:47:21 smithi026 ceph-mon[175151]: pgmap v65: 73 pgs: 73 active+clean; 582 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:47:22.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:47:21 smithi026 ceph-mon[179688]: pgmap v65: 73 pgs: 73 active+clean; 582 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:47:24.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:47:23 smithi070 ceph-mon[100755]: pgmap v66: 73 pgs: 73 active+clean; 582 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:47:24.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:47:23 smithi026 ceph-mon[175151]: pgmap v66: 73 pgs: 73 active+clean; 582 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:47:24.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:47:23 smithi026 ceph-mon[179688]: pgmap v66: 73 pgs: 73 active+clean; 582 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:47:26.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:47:25 smithi070 ceph-mon[100755]: pgmap v67: 73 pgs: 73 active+clean; 582 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:47:26.131 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:47:25 smithi026 ceph-mon[175151]: pgmap v67: 73 pgs: 73 active+clean; 582 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:47:26.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:47:25 smithi026 ceph-mon[179688]: pgmap v67: 73 pgs: 73 active+clean; 582 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:47:28.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:47:27 smithi070 ceph-mon[100755]: pgmap v68: 73 pgs: 73 active+clean; 582 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:47:28.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:47:27 smithi026 ceph-mon[175151]: pgmap v68: 73 pgs: 73 active+clean; 582 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:47:28.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:47:27 smithi026 ceph-mon[179688]: pgmap v68: 73 pgs: 73 active+clean; 582 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:47:29.119 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 20:47:28 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-prometheus-a[155975]: ts=2023-09-23T20:47:28.659Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1123-g61431f6b (61431f6ba2752649b506867ec856b4a6236f90dd) quincy (stable)\", cluster_addr=\"172.21.15.26\", device_class=\"ssd\", hostname=\"smithi026\", instance=\"smithi070.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.26\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1123-g61431f6b (61431f6ba2752649b506867ec856b4a6236f90dd) quincy (stable)\", cluster_addr=\"172.21.15.26\", device_class=\"ssd\", hostname=\"smithi026\", instance=\"smithi026:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.26\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-09-23T20:47:29.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:47:29 smithi026 ceph-mon[175151]: pgmap v69: 73 pgs: 73 active+clean; 582 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:47:29.883 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:47:29 smithi026 ceph-mon[179688]: pgmap v69: 73 pgs: 73 active+clean; 582 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:47:29.883 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:47:29 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[164029]: ::ffff:172.21.15.70 - - [23/Sep/2023:20:47:29] "GET /metrics HTTP/1.1" 200 33687 "" "Prometheus/2.43.0" 2023-09-23T20:47:30.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:47:29 smithi070 ceph-mon[100755]: pgmap v69: 73 pgs: 73 active+clean; 582 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:47:32.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:47:31 smithi070 ceph-mon[100755]: pgmap v70: 73 pgs: 73 active+clean; 582 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:47:32.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:47:31 smithi070 ceph-mon[100755]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:47:32.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:47:31 smithi026 ceph-mon[175151]: pgmap v70: 73 pgs: 73 active+clean; 582 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:47:32.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:47:31 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:47:32.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:47:31 smithi026 ceph-mon[179688]: pgmap v70: 73 pgs: 73 active+clean; 582 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:47:32.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:47:31 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:47:34.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:47:33 smithi070 ceph-mon[100755]: pgmap v71: 73 pgs: 73 active+clean; 582 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:47:34.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:47:33 smithi026 ceph-mon[175151]: pgmap v71: 73 pgs: 73 active+clean; 582 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:47:34.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:47:33 smithi026 ceph-mon[179688]: pgmap v71: 73 pgs: 73 active+clean; 582 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:47:36.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:47:35 smithi070 ceph-mon[100755]: pgmap v72: 73 pgs: 73 active+clean; 582 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:47:36.131 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:47:35 smithi026 ceph-mon[175151]: pgmap v72: 73 pgs: 73 active+clean; 582 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:47:36.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:47:35 smithi026 ceph-mon[179688]: pgmap v72: 73 pgs: 73 active+clean; 582 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:47:38.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:47:37 smithi070 ceph-mon[100755]: pgmap v73: 73 pgs: 73 active+clean; 582 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:47:38.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:47:37 smithi026 ceph-mon[175151]: pgmap v73: 73 pgs: 73 active+clean; 582 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:47:38.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:47:37 smithi026 ceph-mon[179688]: pgmap v73: 73 pgs: 73 active+clean; 582 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:47:39.119 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 20:47:38 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-prometheus-a[155975]: ts=2023-09-23T20:47:38.660Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1123-g61431f6b (61431f6ba2752649b506867ec856b4a6236f90dd) quincy (stable)\", cluster_addr=\"172.21.15.26\", device_class=\"ssd\", hostname=\"smithi026\", instance=\"smithi070.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.26\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1123-g61431f6b (61431f6ba2752649b506867ec856b4a6236f90dd) quincy (stable)\", cluster_addr=\"172.21.15.26\", device_class=\"ssd\", hostname=\"smithi026\", instance=\"smithi026:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.26\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-09-23T20:47:39.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:47:39 smithi026 ceph-mon[175151]: pgmap v74: 73 pgs: 73 active+clean; 582 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:47:39.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:47:39 smithi026 ceph-mon[179688]: pgmap v74: 73 pgs: 73 active+clean; 582 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:47:39.882 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:47:39 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[164029]: ::ffff:172.21.15.70 - - [23/Sep/2023:20:47:39] "GET /metrics HTTP/1.1" 200 33700 "" "Prometheus/2.43.0" 2023-09-23T20:47:40.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:47:39 smithi070 ceph-mon[100755]: pgmap v74: 73 pgs: 73 active+clean; 582 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:47:42.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:47:41 smithi070 ceph-mon[100755]: pgmap v75: 73 pgs: 73 active+clean; 582 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:47:42.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:47:41 smithi026 ceph-mon[175151]: pgmap v75: 73 pgs: 73 active+clean; 582 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:47:42.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:47:41 smithi026 ceph-mon[179688]: pgmap v75: 73 pgs: 73 active+clean; 582 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:47:44.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:47:43 smithi070 ceph-mon[100755]: pgmap v76: 73 pgs: 73 active+clean; 582 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:47:44.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:47:43 smithi026 ceph-mon[175151]: pgmap v76: 73 pgs: 73 active+clean; 582 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:47:44.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:47:43 smithi026 ceph-mon[179688]: pgmap v76: 73 pgs: 73 active+clean; 582 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:47:46.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:47:45 smithi070 ceph-mon[100755]: pgmap v77: 73 pgs: 73 active+clean; 582 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:47:46.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:47:45 smithi026 ceph-mon[175151]: pgmap v77: 73 pgs: 73 active+clean; 582 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:47:46.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:47:45 smithi026 ceph-mon[179688]: pgmap v77: 73 pgs: 73 active+clean; 582 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:47:47.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:47:46 smithi070 ceph-mon[100755]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:47:47.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:47:46 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:47:47.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:47:46 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:47:48.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:47:47 smithi070 ceph-mon[100755]: pgmap v78: 73 pgs: 73 active+clean; 582 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:47:48.131 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:47:47 smithi026 ceph-mon[175151]: pgmap v78: 73 pgs: 73 active+clean; 582 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:47:48.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:47:47 smithi026 ceph-mon[179688]: pgmap v78: 73 pgs: 73 active+clean; 582 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:47:49.119 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 20:47:48 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-prometheus-a[155975]: ts=2023-09-23T20:47:48.660Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1123-g61431f6b (61431f6ba2752649b506867ec856b4a6236f90dd) quincy (stable)\", cluster_addr=\"172.21.15.26\", device_class=\"ssd\", hostname=\"smithi026\", instance=\"smithi070.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.26\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1123-g61431f6b (61431f6ba2752649b506867ec856b4a6236f90dd) quincy (stable)\", cluster_addr=\"172.21.15.26\", device_class=\"ssd\", hostname=\"smithi026\", instance=\"smithi026:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.26\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-09-23T20:47:49.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:47:49 smithi026 ceph-mon[175151]: pgmap v79: 73 pgs: 73 active+clean; 582 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:47:49.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:47:49 smithi026 ceph-mon[179688]: pgmap v79: 73 pgs: 73 active+clean; 582 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:47:49.882 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:47:49 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[164029]: ::ffff:172.21.15.70 - - [23/Sep/2023:20:47:49] "GET /metrics HTTP/1.1" 200 33701 "" "Prometheus/2.43.0" 2023-09-23T20:47:50.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:47:49 smithi070 ceph-mon[100755]: pgmap v79: 73 pgs: 73 active+clean; 582 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:47:51.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:47:50 smithi070 ceph-mon[100755]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:47:51.427 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:47:50 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:47:51.428 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:47:50 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:47:52.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:47:51 smithi070 ceph-mon[100755]: pgmap v80: 73 pgs: 73 active+clean; 582 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:47:52.148 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:47:51 smithi070 ceph-mon[100755]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:47:52.148 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:47:51 smithi070 ceph-mon[100755]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:47:52.149 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:47:51 smithi026 ceph-mon[175151]: pgmap v80: 73 pgs: 73 active+clean; 582 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:47:52.149 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:47:51 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:47:52.149 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:47:51 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:47:52.149 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:47:51 smithi026 ceph-mon[179688]: pgmap v80: 73 pgs: 73 active+clean; 582 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:47:52.150 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:47:51 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:47:52.150 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:47:51 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:47:52.444 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -e sha1=bb0daeff97f020f686596fca6eba20c310e26fb7 -- bash -c 'ceph orch upgrade resume' 2023-09-23T20:47:53.325 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:47:53.814 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:47:53 smithi026 ceph-mon[179688]: pgmap v81: 73 pgs: 73 active+clean; 582 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:47:54.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:47:53 smithi070 ceph-mon[100755]: pgmap v81: 73 pgs: 73 active+clean; 582 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:47:54.131 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:47:53 smithi026 ceph-mon[175151]: pgmap v81: 73 pgs: 73 active+clean; 582 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:47:54.840 INFO:teuthology.orchestra.run.smithi026.stdout:Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d 2023-09-23T20:47:55.484 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -e sha1=bb0daeff97f020f686596fca6eba20c310e26fb7 -- bash -c 'echo wait for all mons to upgrade' 2023-09-23T20:47:56.094 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:47:56.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:47:55 smithi070 ceph-mon[100755]: pgmap v82: 73 pgs: 73 active+clean; 582 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:47:56.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:47:55 smithi070 ceph-mon[100755]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:47:56.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:47:55 smithi070 ceph-mon[100755]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:47:56.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:47:55 smithi070 ceph-mon[100755]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:47:56.131 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:47:55 smithi026 ceph-mon[179688]: pgmap v82: 73 pgs: 73 active+clean; 582 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:47:56.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:47:55 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:47:56.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:47:55 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:47:56.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:47:55 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:47:56.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:47:55 smithi026 ceph-mon[175151]: pgmap v82: 73 pgs: 73 active+clean; 582 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:47:56.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:47:55 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:47:56.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:47:55 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:47:56.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:47:55 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:47:56.911 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:47:56 smithi026 ceph-mon[179688]: from='client.34209 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T20:47:56.911 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:47:56 smithi026 ceph-mon[179688]: Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d 2023-09-23T20:47:56.911 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:47:56 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:47:56.911 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:47:56 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:47:56.911 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:47:56 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:47:56.911 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:47:56 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:47:56.911 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:47:56 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:47:56.911 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:47:56 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:47:56.912 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:47:56 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T20:47:56.912 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:47:56 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2023-09-23T20:47:56.912 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:47:56 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["b"]}]: dispatch 2023-09-23T20:47:56.912 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:47:56 smithi026 ceph-mon[175151]: from='client.34209 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T20:47:56.912 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:47:56 smithi026 ceph-mon[175151]: Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d 2023-09-23T20:47:56.912 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:47:56 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:47:56.912 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:47:56 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:47:56.912 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:47:56 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:47:56.912 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:47:56 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:47:56.913 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:47:56 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:47:56.913 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:47:56 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:47:56.913 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:47:56 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T20:47:56.913 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:47:56 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2023-09-23T20:47:56.913 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:47:56 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["b"]}]: dispatch 2023-09-23T20:47:57.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:47:56 smithi070 ceph-mon[100755]: from='client.34209 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T20:47:57.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:47:56 smithi070 ceph-mon[100755]: Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d 2023-09-23T20:47:57.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:47:56 smithi070 ceph-mon[100755]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:47:57.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:47:56 smithi070 ceph-mon[100755]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:47:57.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:47:56 smithi070 ceph-mon[100755]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:47:57.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:47:56 smithi070 ceph-mon[100755]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:47:57.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:47:56 smithi070 ceph-mon[100755]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:47:57.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:47:56 smithi070 ceph-mon[100755]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:47:57.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:47:56 smithi070 ceph-mon[100755]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T20:47:57.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:47:56 smithi070 ceph-mon[100755]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2023-09-23T20:47:57.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:47:56 smithi070 ceph-mon[100755]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["b"]}]: dispatch 2023-09-23T20:47:57.259 INFO:teuthology.orchestra.run.smithi026.stdout:wait for all mons to upgrade 2023-09-23T20:47:57.658 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -e sha1=bb0daeff97f020f686596fca6eba20c310e26fb7 -- bash -c 'while ! ceph mon versions | grep $sha1 | grep '"'"': 3'"'"' ; do sleep 2 ; done' 2023-09-23T20:47:57.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:47:57 smithi070 ceph-mon[100755]: pgmap v83: 73 pgs: 73 active+clean; 582 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:47:57.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:47:57 smithi070 ceph-mon[100755]: Upgrade: It appears safe to stop mon.b 2023-09-23T20:47:58.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:47:57 smithi026 ceph-mon[179688]: pgmap v83: 73 pgs: 73 active+clean; 582 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:47:58.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:47:57 smithi026 ceph-mon[179688]: Upgrade: It appears safe to stop mon.b 2023-09-23T20:47:58.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:47:57 smithi026 ceph-mon[175151]: pgmap v83: 73 pgs: 73 active+clean; 582 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:47:58.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:47:57 smithi026 ceph-mon[175151]: Upgrade: It appears safe to stop mon.b 2023-09-23T20:47:58.265 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:47:58.996 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 20:47:58 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-prometheus-a[155975]: ts=2023-09-23T20:47:58.660Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1123-g61431f6b (61431f6ba2752649b506867ec856b4a6236f90dd) quincy (stable)\", cluster_addr=\"172.21.15.26\", device_class=\"ssd\", hostname=\"smithi026\", instance=\"smithi070.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.26\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1123-g61431f6b (61431f6ba2752649b506867ec856b4a6236f90dd) quincy (stable)\", cluster_addr=\"172.21.15.26\", device_class=\"ssd\", hostname=\"smithi026\", instance=\"smithi026:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.26\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-09-23T20:47:59.256 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:47:59 smithi070 ceph-mon[100755]: pgmap v84: 73 pgs: 73 active+clean; 582 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:47:59.257 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:47:59 smithi070 ceph-mon[100755]: Upgrade: Updating mon.b 2023-09-23T20:47:59.257 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:47:59 smithi070 ceph-mon[100755]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:47:59.257 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:47:59 smithi070 ceph-mon[100755]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-23T20:47:59.257 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:47:59 smithi070 ceph-mon[100755]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-09-23T20:47:59.257 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:47:59 smithi070 ceph-mon[100755]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:47:59.257 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:47:59 smithi070 ceph-mon[100755]: Deploying daemon mon.b on smithi070 2023-09-23T20:47:59.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:47:59 smithi026 ceph-mon[175151]: pgmap v84: 73 pgs: 73 active+clean; 582 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:47:59.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:47:59 smithi026 ceph-mon[175151]: Upgrade: Updating mon.b 2023-09-23T20:47:59.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:47:59 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:47:59.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:47:59 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-23T20:47:59.383 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:47:59 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-09-23T20:47:59.383 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:47:59 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:47:59.383 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:47:59 smithi026 ceph-mon[175151]: Deploying daemon mon.b on smithi070 2023-09-23T20:47:59.383 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:47:59 smithi026 ceph-mon[179688]: pgmap v84: 73 pgs: 73 active+clean; 582 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:47:59.383 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:47:59 smithi026 ceph-mon[179688]: Upgrade: Updating mon.b 2023-09-23T20:47:59.383 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:47:59 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:47:59.383 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:47:59 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-23T20:47:59.383 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:47:59 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-09-23T20:47:59.383 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:47:59 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:47:59.384 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:47:59 smithi026 ceph-mon[179688]: Deploying daemon mon.b on smithi070 2023-09-23T20:47:59.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:47:59 smithi070 systemd[1]: Stopping Ceph mon.b for b3f8e4ec-5a50-11ee-8db4-212e2dc638e7... 2023-09-23T20:47:59.882 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:47:59 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[164029]: ::ffff:172.21.15.70 - - [23/Sep/2023:20:47:59] "GET /metrics HTTP/1.1" 200 33701 "" "Prometheus/2.43.0" 2023-09-23T20:48:00.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:47:59 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mon-b[100708]: 2023-09-23T20:47:59.800+0000 7f4bc5211700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-mon -n mon.b -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2023-09-23T20:48:00.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:47:59 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mon-b[100708]: 2023-09-23T20:47:59.800+0000 7f4bc5211700 -1 mon.b@1(peon) e3 *** Got Signal Terminated *** 2023-09-23T20:48:00.553 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:00 smithi070 bash[168096]: ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mon-b 2023-09-23T20:48:00.553 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:00 smithi070 systemd[1]: ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7@mon.b.service: Succeeded. 2023-09-23T20:48:00.553 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:00 smithi070 systemd[1]: Stopped Ceph mon.b for b3f8e4ec-5a50-11ee-8db4-212e2dc638e7. 2023-09-23T20:48:00.847 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:00 smithi070 systemd[1]: Starting Ceph mon.b for b3f8e4ec-5a50-11ee-8db4-212e2dc638e7... 2023-09-23T20:48:01.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:00 smithi070 podman[168242]: 2023-09-23T20:48:01.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:01 smithi070 ceph-mon[168282]: set uid:gid to 167:167 (ceph:ceph) 2023-09-23T20:48:01.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:01 smithi070 ceph-mon[168282]: ceph version 18.2.0-460-gbb0daeff (bb0daeff97f020f686596fca6eba20c310e26fb7) reef (stable), process ceph-mon, pid 7 2023-09-23T20:48:01.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:01 smithi070 bash[168242]: 957198c68713170a46d197efc0119f5cab4b9f2289f3c1a79abf7e69120d75cb 2023-09-23T20:48:01.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:01 smithi070 ceph-mon[168282]: pidfile_write: ignore empty --pid-file 2023-09-23T20:48:01.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:01 smithi070 ceph-mon[168282]: load: jerasure load: lrc 2023-09-23T20:48:01.121 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:01 smithi070 ceph-mon[168282]: rocksdb: RocksDB version: 7.9.2 2023-09-23T20:48:01.121 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:01 smithi070 ceph-mon[168282]: rocksdb: Git sha 0 2023-09-23T20:48:01.121 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:01 smithi070 ceph-mon[168282]: rocksdb: Compile date 2023-09-22 15:19:43 2023-09-23T20:48:01.121 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:01 smithi070 ceph-mon[168282]: rocksdb: DB SUMMARY 2023-09-23T20:48:01.121 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:01 smithi070 ceph-mon[168282]: rocksdb: DB Session ID: JYQY3284W0GMNMTWXP2P 2023-09-23T20:48:01.121 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:01 smithi070 ceph-mon[168282]: rocksdb: CURRENT file: CURRENT 2023-09-23T20:48:01.121 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:01 smithi070 ceph-mon[168282]: rocksdb: IDENTITY file: IDENTITY 2023-09-23T20:48:01.121 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:01 smithi070 ceph-mon[168282]: rocksdb: MANIFEST file: MANIFEST-000009 size: 1069 Bytes 2023-09-23T20:48:01.121 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:01 smithi070 ceph-mon[168282]: rocksdb: SST files in /var/lib/ceph/mon/ceph-b/store.db dir, Total Num: 1, files: 000027.sst 2023-09-23T20:48:01.121 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:01 smithi070 ceph-mon[168282]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-b/store.db: 000025.log size: 2118909 ; 2023-09-23T20:48:01.121 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:01 smithi070 ceph-mon[168282]: rocksdb: Options.error_if_exists: 0 2023-09-23T20:48:01.121 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:01 smithi070 ceph-mon[168282]: rocksdb: Options.create_if_missing: 0 2023-09-23T20:48:01.122 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:01 smithi070 ceph-mon[168282]: rocksdb: Options.paranoid_checks: 1 2023-09-23T20:48:01.122 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:01 smithi070 ceph-mon[168282]: rocksdb: Options.flush_verify_memtable_count: 1 2023-09-23T20:48:01.122 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:01 smithi070 ceph-mon[168282]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2023-09-23T20:48:01.122 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:01 smithi070 ceph-mon[168282]: rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2023-09-23T20:48:01.122 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:01 smithi070 ceph-mon[168282]: rocksdb: Options.env: 0x55e8e7972720 2023-09-23T20:48:01.122 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:01 smithi070 ceph-mon[168282]: rocksdb: Options.fs: PosixFileSystem 2023-09-23T20:48:01.122 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:01 smithi070 ceph-mon[168282]: rocksdb: Options.info_log: 0x55e8e92e9420 2023-09-23T20:48:01.122 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:01 smithi070 ceph-mon[168282]: rocksdb: Options.max_file_opening_threads: 16 2023-09-23T20:48:01.122 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:01 smithi070 ceph-mon[168282]: rocksdb: Options.statistics: (nil) 2023-09-23T20:48:01.122 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:01 smithi070 ceph-mon[168282]: rocksdb: Options.use_fsync: 0 2023-09-23T20:48:01.122 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:01 smithi070 ceph-mon[168282]: rocksdb: Options.max_log_file_size: 0 2023-09-23T20:48:01.122 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:01 smithi070 ceph-mon[168282]: rocksdb: Options.max_manifest_file_size: 1073741824 2023-09-23T20:48:01.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:01 smithi070 ceph-mon[168282]: rocksdb: Options.log_file_time_to_roll: 0 2023-09-23T20:48:01.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:01 smithi070 ceph-mon[168282]: rocksdb: Options.keep_log_file_num: 1000 2023-09-23T20:48:01.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:01 smithi070 ceph-mon[168282]: rocksdb: Options.recycle_log_file_num: 0 2023-09-23T20:48:01.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:01 smithi070 ceph-mon[168282]: rocksdb: Options.allow_fallocate: 1 2023-09-23T20:48:01.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:01 smithi070 ceph-mon[168282]: rocksdb: Options.allow_mmap_reads: 0 2023-09-23T20:48:01.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:01 smithi070 ceph-mon[168282]: rocksdb: Options.allow_mmap_writes: 0 2023-09-23T20:48:01.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:01 smithi070 ceph-mon[168282]: rocksdb: Options.use_direct_reads: 0 2023-09-23T20:48:01.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:01 smithi070 ceph-mon[168282]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2023-09-23T20:48:01.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:01 smithi070 ceph-mon[168282]: rocksdb: Options.create_missing_column_families: 0 2023-09-23T20:48:01.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:01 smithi070 ceph-mon[168282]: rocksdb: Options.db_log_dir: 2023-09-23T20:48:01.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:01 smithi070 ceph-mon[168282]: rocksdb: Options.wal_dir: 2023-09-23T20:48:01.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:01 smithi070 ceph-mon[168282]: rocksdb: Options.table_cache_numshardbits: 6 2023-09-23T20:48:01.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:01 smithi070 ceph-mon[168282]: rocksdb: Options.WAL_ttl_seconds: 0 2023-09-23T20:48:01.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:01 smithi070 ceph-mon[168282]: rocksdb: Options.WAL_size_limit_MB: 0 2023-09-23T20:48:01.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:01 smithi070 ceph-mon[168282]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2023-09-23T20:48:01.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:01 smithi070 ceph-mon[168282]: rocksdb: Options.manifest_preallocation_size: 4194304 2023-09-23T20:48:01.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:01 smithi070 ceph-mon[168282]: rocksdb: Options.is_fd_close_on_exec: 1 2023-09-23T20:48:01.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:01 smithi070 ceph-mon[168282]: rocksdb: Options.advise_random_on_open: 1 2023-09-23T20:48:01.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:01 smithi070 ceph-mon[168282]: rocksdb: Options.db_write_buffer_size: 0 2023-09-23T20:48:01.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:01 smithi070 ceph-mon[168282]: rocksdb: Options.write_buffer_manager: 0x55e8e857a5a0 2023-09-23T20:48:01.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:01 smithi070 ceph-mon[168282]: rocksdb: Options.access_hint_on_compaction_start: 1 2023-09-23T20:48:01.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:01 smithi070 ceph-mon[168282]: rocksdb: Options.random_access_max_buffer_size: 1048576 2023-09-23T20:48:01.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:01 smithi070 ceph-mon[168282]: rocksdb: Options.use_adaptive_mutex: 0 2023-09-23T20:48:01.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:01 smithi070 ceph-mon[168282]: rocksdb: Options.rate_limiter: (nil) 2023-09-23T20:48:01.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:01 smithi070 ceph-mon[168282]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2023-09-23T20:48:01.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:01 smithi070 ceph-mon[168282]: rocksdb: Options.wal_recovery_mode: 2 2023-09-23T20:48:01.125 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:01 smithi070 ceph-mon[168282]: rocksdb: Options.enable_thread_tracking: 0 2023-09-23T20:48:01.125 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:01 smithi070 ceph-mon[168282]: rocksdb: Options.enable_pipelined_write: 0 2023-09-23T20:48:01.125 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:01 smithi070 ceph-mon[168282]: rocksdb: Options.unordered_write: 0 2023-09-23T20:48:01.125 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:01 smithi070 ceph-mon[168282]: rocksdb: Options.allow_concurrent_memtable_write: 1 2023-09-23T20:48:01.125 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:01 smithi070 ceph-mon[168282]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2023-09-23T20:48:01.125 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:01 smithi070 ceph-mon[168282]: rocksdb: Options.write_thread_max_yield_usec: 100 2023-09-23T20:48:01.125 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:01 smithi070 ceph-mon[168282]: rocksdb: Options.write_thread_slow_yield_usec: 3 2023-09-23T20:48:01.125 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:01 smithi070 ceph-mon[168282]: rocksdb: Options.row_cache: None 2023-09-23T20:48:01.125 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:01 smithi070 ceph-mon[168282]: rocksdb: Options.wal_filter: None 2023-09-23T20:48:01.125 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:01 smithi070 ceph-mon[168282]: rocksdb: Options.avoid_flush_during_recovery: 0 2023-09-23T20:48:01.125 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:01 smithi070 ceph-mon[168282]: rocksdb: Options.allow_ingest_behind: 0 2023-09-23T20:48:01.125 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:01 smithi070 ceph-mon[168282]: rocksdb: Options.two_write_queues: 0 2023-09-23T20:48:01.126 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:01 smithi070 ceph-mon[168282]: rocksdb: Options.manual_wal_flush: 0 2023-09-23T20:48:01.126 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:01 smithi070 ceph-mon[168282]: rocksdb: Options.wal_compression: 0 2023-09-23T20:48:01.126 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:01 smithi070 ceph-mon[168282]: rocksdb: Options.atomic_flush: 0 2023-09-23T20:48:01.126 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:01 smithi070 ceph-mon[168282]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2023-09-23T20:48:01.127 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:01 smithi070 ceph-mon[168282]: rocksdb: Options.persist_stats_to_disk: 0 2023-09-23T20:48:01.127 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:01 smithi070 ceph-mon[168282]: rocksdb: Options.write_dbid_to_manifest: 0 2023-09-23T20:48:01.127 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:01 smithi070 ceph-mon[168282]: rocksdb: Options.log_readahead_size: 0 2023-09-23T20:48:01.127 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:01 smithi070 ceph-mon[168282]: rocksdb: Options.file_checksum_gen_factory: Unknown 2023-09-23T20:48:01.127 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:01 smithi070 ceph-mon[168282]: rocksdb: Options.best_efforts_recovery: 0 2023-09-23T20:48:01.127 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:01 smithi070 ceph-mon[168282]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2023-09-23T20:48:01.127 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:01 smithi070 ceph-mon[168282]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2023-09-23T20:48:01.127 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:01 smithi070 ceph-mon[168282]: rocksdb: Options.allow_data_in_errors: 0 2023-09-23T20:48:01.127 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:01 smithi070 ceph-mon[168282]: rocksdb: Options.db_host_id: __hostname__ 2023-09-23T20:48:01.127 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:01 smithi070 ceph-mon[168282]: rocksdb: Options.enforce_single_del_contracts: true 2023-09-23T20:48:01.127 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:01 smithi070 ceph-mon[168282]: rocksdb: Options.max_background_jobs: 2 2023-09-23T20:48:01.127 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:01 smithi070 ceph-mon[168282]: rocksdb: Options.max_background_compactions: -1 2023-09-23T20:48:01.128 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:01 smithi070 ceph-mon[168282]: rocksdb: Options.max_subcompactions: 1 2023-09-23T20:48:01.128 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:01 smithi070 ceph-mon[168282]: rocksdb: Options.avoid_flush_during_shutdown: 0 2023-09-23T20:48:01.128 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:01 smithi070 ceph-mon[168282]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2023-09-23T20:48:01.128 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:01 smithi070 ceph-mon[168282]: rocksdb: Options.delayed_write_rate : 16777216 2023-09-23T20:48:01.128 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:01 smithi070 ceph-mon[168282]: rocksdb: Options.max_total_wal_size: 0 2023-09-23T20:48:01.128 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:01 smithi070 ceph-mon[168282]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2023-09-23T20:48:01.128 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:01 smithi070 ceph-mon[168282]: rocksdb: Options.stats_dump_period_sec: 600 2023-09-23T20:48:01.128 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:01 smithi070 ceph-mon[168282]: rocksdb: Options.stats_persist_period_sec: 600 2023-09-23T20:48:01.128 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:01 smithi070 ceph-mon[168282]: rocksdb: Options.stats_history_buffer_size: 1048576 2023-09-23T20:48:01.128 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:01 smithi070 ceph-mon[168282]: rocksdb: Options.max_open_files: -1 2023-09-23T20:48:01.128 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:01 smithi070 ceph-mon[168282]: rocksdb: Options.bytes_per_sync: 0 2023-09-23T20:48:01.128 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:01 smithi070 ceph-mon[168282]: rocksdb: Options.wal_bytes_per_sync: 0 2023-09-23T20:48:01.128 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:01 smithi070 ceph-mon[168282]: rocksdb: Options.strict_bytes_per_sync: 0 2023-09-23T20:48:01.129 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:01 smithi070 ceph-mon[168282]: rocksdb: Options.compaction_readahead_size: 0 2023-09-23T20:48:01.129 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:01 smithi070 ceph-mon[168282]: rocksdb: Options.max_background_flushes: -1 2023-09-23T20:48:01.129 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:01 smithi070 ceph-mon[168282]: rocksdb: Compression algorithms supported: 2023-09-23T20:48:01.129 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:01 smithi070 ceph-mon[168282]: rocksdb: kZSTDNotFinalCompression supported: 0 2023-09-23T20:48:01.129 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:01 smithi070 ceph-mon[168282]: rocksdb: kZSTD supported: 0 2023-09-23T20:48:01.129 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:01 smithi070 ceph-mon[168282]: rocksdb: kXpressCompression supported: 0 2023-09-23T20:48:01.129 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:01 smithi070 ceph-mon[168282]: rocksdb: kLZ4HCCompression supported: 1 2023-09-23T20:48:01.129 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:01 smithi070 ceph-mon[168282]: rocksdb: kZlibCompression supported: 1 2023-09-23T20:48:01.129 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:01 smithi070 ceph-mon[168282]: rocksdb: kSnappyCompression supported: 1 2023-09-23T20:48:01.129 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:01 smithi070 ceph-mon[168282]: rocksdb: kLZ4Compression supported: 1 2023-09-23T20:48:01.129 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:01 smithi070 ceph-mon[168282]: rocksdb: kBZip2Compression supported: 0 2023-09-23T20:48:01.129 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:01 smithi070 ceph-mon[168282]: rocksdb: Fast CRC32 supported: Supported on x86 2023-09-23T20:48:01.129 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:01 smithi070 ceph-mon[168282]: rocksdb: DMutex implementation: pthread_mutex_t 2023-09-23T20:48:01.130 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:01 smithi070 ceph-mon[168282]: rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-b/store.db/MANIFEST-000009 2023-09-23T20:48:01.130 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:01 smithi070 ceph-mon[168282]: rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2023-09-23T20:48:01.130 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:01 smithi070 ceph-mon[168282]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2023-09-23T20:48:01.130 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:01 smithi070 ceph-mon[168282]: rocksdb: Options.merge_operator: 2023-09-23T20:48:01.130 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:01 smithi070 ceph-mon[168282]: rocksdb: Options.compaction_filter: None 2023-09-23T20:48:01.130 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:01 smithi070 ceph-mon[168282]: rocksdb: Options.compaction_filter_factory: None 2023-09-23T20:48:01.130 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:01 smithi070 ceph-mon[168282]: rocksdb: Options.sst_partitioner_factory: None 2023-09-23T20:48:01.130 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:01 smithi070 ceph-mon[168282]: rocksdb: Options.memtable_factory: SkipListFactory 2023-09-23T20:48:01.130 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:01 smithi070 ceph-mon[168282]: rocksdb: Options.table_factory: BlockBasedTable 2023-09-23T20:48:01.130 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:01 smithi070 ceph-mon[168282]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x55e8e92e9520) 2023-09-23T20:48:01.130 INFO:journalctl@ceph.mon.b.smithi070.stdout: cache_index_and_filter_blocks: 1 2023-09-23T20:48:01.130 INFO:journalctl@ceph.mon.b.smithi070.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2023-09-23T20:48:01.131 INFO:journalctl@ceph.mon.b.smithi070.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2023-09-23T20:48:01.131 INFO:journalctl@ceph.mon.b.smithi070.stdout: pin_top_level_index_and_filter: 1 2023-09-23T20:48:01.131 INFO:journalctl@ceph.mon.b.smithi070.stdout: index_type: 0 2023-09-23T20:48:01.131 INFO:journalctl@ceph.mon.b.smithi070.stdout: data_block_index_type: 0 2023-09-23T20:48:01.131 INFO:journalctl@ceph.mon.b.smithi070.stdout: index_shortening: 1 2023-09-23T20:48:01.131 INFO:journalctl@ceph.mon.b.smithi070.stdout: data_block_hash_table_util_ratio: 0.750000 2023-09-23T20:48:01.131 INFO:journalctl@ceph.mon.b.smithi070.stdout: checksum: 4 2023-09-23T20:48:01.131 INFO:journalctl@ceph.mon.b.smithi070.stdout: no_block_cache: 0 2023-09-23T20:48:01.131 INFO:journalctl@ceph.mon.b.smithi070.stdout: block_cache: 0x55e8e85fb350 2023-09-23T20:48:01.131 INFO:journalctl@ceph.mon.b.smithi070.stdout: block_cache_name: BinnedLRUCache 2023-09-23T20:48:01.131 INFO:journalctl@ceph.mon.b.smithi070.stdout: block_cache_options: 2023-09-23T20:48:01.131 INFO:journalctl@ceph.mon.b.smithi070.stdout: capacity : 536870912 2023-09-23T20:48:01.131 INFO:journalctl@ceph.mon.b.smithi070.stdout: num_shard_bits : 4 2023-09-23T20:48:01.132 INFO:journalctl@ceph.mon.b.smithi070.stdout: strict_capacity_limit : 0 2023-09-23T20:48:01.132 INFO:journalctl@ceph.mon.b.smithi070.stdout: high_pri_pool_ratio: 0.000 2023-09-23T20:48:01.132 INFO:journalctl@ceph.mon.b.smithi070.stdout: block_cache_compressed: (nil) 2023-09-23T20:48:01.132 INFO:journalctl@ceph.mon.b.smithi070.stdout: persistent_cache: (nil) 2023-09-23T20:48:01.132 INFO:journalctl@ceph.mon.b.smithi070.stdout: block_size: 4096 2023-09-23T20:48:01.132 INFO:journalctl@ceph.mon.b.smithi070.stdout: block_size_deviation: 10 2023-09-23T20:48:01.132 INFO:journalctl@ceph.mon.b.smithi070.stdout: block_restart_interval: 16 2023-09-23T20:48:01.132 INFO:journalctl@ceph.mon.b.smithi070.stdout: index_block_restart_interval: 1 2023-09-23T20:48:01.132 INFO:journalctl@ceph.mon.b.smithi070.stdout: metadata_block_size: 4096 2023-09-23T20:48:01.132 INFO:journalctl@ceph.mon.b.smithi070.stdout: partition_filters: 0 2023-09-23T20:48:01.132 INFO:journalctl@ceph.mon.b.smithi070.stdout: use_delta_encoding: 1 2023-09-23T20:48:01.132 INFO:journalctl@ceph.mon.b.smithi070.stdout: filter_policy: bloomfilter 2023-09-23T20:48:01.133 INFO:journalctl@ceph.mon.b.smithi070.stdout: whole_key_filtering: 1 2023-09-23T20:48:01.133 INFO:journalctl@ceph.mon.b.smithi070.stdout: verify_compression: 0 2023-09-23T20:48:01.133 INFO:journalctl@ceph.mon.b.smithi070.stdout: read_amp_bytes_per_bit: 0 2023-09-23T20:48:01.133 INFO:journalctl@ceph.mon.b.smithi070.stdout: format_version: 5 2023-09-23T20:48:01.133 INFO:journalctl@ceph.mon.b.smithi070.stdout: enable_index_compression: 1 2023-09-23T20:48:01.133 INFO:journalctl@ceph.mon.b.smithi070.stdout: block_align: 0 2023-09-23T20:48:01.133 INFO:journalctl@ceph.mon.b.smithi070.stdout: max_auto_readahead_size: 262144 2023-09-23T20:48:01.133 INFO:journalctl@ceph.mon.b.smithi070.stdout: prepopulate_block_cache: 0 2023-09-23T20:48:01.133 INFO:journalctl@ceph.mon.b.smithi070.stdout: initial_auto_readahead_size: 8192 2023-09-23T20:48:01.133 INFO:journalctl@ceph.mon.b.smithi070.stdout: num_file_reads_for_auto_readahead: 2 2023-09-23T20:48:01.133 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:01 smithi070 ceph-mon[168282]: rocksdb: Options.write_buffer_size: 33554432 2023-09-23T20:48:01.133 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:01 smithi070 ceph-mon[168282]: rocksdb: Options.max_write_buffer_number: 2 2023-09-23T20:48:01.133 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:01 smithi070 ceph-mon[168282]: rocksdb: Options.compression: NoCompression 2023-09-23T20:48:01.134 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:01 smithi070 ceph-mon[168282]: rocksdb: Options.bottommost_compression: Disabled 2023-09-23T20:48:01.134 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:01 smithi070 ceph-mon[168282]: rocksdb: Options.prefix_extractor: nullptr 2023-09-23T20:48:01.134 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:01 smithi070 ceph-mon[168282]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2023-09-23T20:48:01.134 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:01 smithi070 ceph-mon[168282]: rocksdb: Options.num_levels: 7 2023-09-23T20:48:01.134 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:01 smithi070 ceph-mon[168282]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2023-09-23T20:48:01.134 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:01 smithi070 ceph-mon[168282]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2023-09-23T20:48:01.134 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:01 smithi070 ceph-mon[168282]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2023-09-23T20:48:01.134 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:01 smithi070 ceph-mon[168282]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2023-09-23T20:48:01.134 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:01 smithi070 ceph-mon[168282]: rocksdb: Options.bottommost_compression_opts.level: 32767 2023-09-23T20:48:01.134 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:01 smithi070 ceph-mon[168282]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2023-09-23T20:48:01.134 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:01 smithi070 ceph-mon[168282]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2023-09-23T20:48:01.135 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:01 smithi070 ceph-mon[168282]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2023-09-23T20:48:01.135 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:01 smithi070 ceph-mon[168282]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2023-09-23T20:48:01.135 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:01 smithi070 ceph-mon[168282]: rocksdb: Options.bottommost_compression_opts.enabled: false 2023-09-23T20:48:01.135 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:01 smithi070 ceph-mon[168282]: rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2023-09-23T20:48:01.135 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:01 smithi070 ceph-mon[168282]: rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2023-09-23T20:48:01.135 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:01 smithi070 ceph-mon[168282]: rocksdb: Options.compression_opts.window_bits: -14 2023-09-23T20:48:01.135 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:01 smithi070 ceph-mon[168282]: rocksdb: Options.compression_opts.level: 32767 2023-09-23T20:48:01.135 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:01 smithi070 ceph-mon[168282]: rocksdb: Options.compression_opts.strategy: 0 2023-09-23T20:48:01.135 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:01 smithi070 ceph-mon[168282]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2023-09-23T20:48:01.135 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:01 smithi070 ceph-mon[168282]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2023-09-23T20:48:01.135 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:01 smithi070 ceph-mon[168282]: rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2023-09-23T20:48:01.135 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:01 smithi070 ceph-mon[168282]: rocksdb: Options.compression_opts.parallel_threads: 1 2023-09-23T20:48:01.136 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:01 smithi070 ceph-mon[168282]: rocksdb: Options.compression_opts.enabled: false 2023-09-23T20:48:01.136 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:01 smithi070 ceph-mon[168282]: rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2023-09-23T20:48:01.136 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:01 smithi070 ceph-mon[168282]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2023-09-23T20:48:01.136 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:01 smithi070 ceph-mon[168282]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2023-09-23T20:48:01.136 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:01 smithi070 ceph-mon[168282]: rocksdb: Options.level0_stop_writes_trigger: 36 2023-09-23T20:48:01.136 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:01 smithi070 ceph-mon[168282]: rocksdb: Options.target_file_size_base: 67108864 2023-09-23T20:48:01.136 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:01 smithi070 ceph-mon[168282]: rocksdb: Options.target_file_size_multiplier: 1 2023-09-23T20:48:01.136 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:01 smithi070 ceph-mon[168282]: rocksdb: Options.max_bytes_for_level_base: 268435456 2023-09-23T20:48:01.136 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:01 smithi070 ceph-mon[168282]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2023-09-23T20:48:01.136 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:01 smithi070 ceph-mon[168282]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2023-09-23T20:48:01.136 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:01 smithi070 ceph-mon[168282]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2023-09-23T20:48:01.136 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:01 smithi070 ceph-mon[168282]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2023-09-23T20:48:01.136 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:01 smithi070 ceph-mon[168282]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2023-09-23T20:48:01.137 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:01 smithi070 ceph-mon[168282]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2023-09-23T20:48:01.137 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:01 smithi070 ceph-mon[168282]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2023-09-23T20:48:01.137 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:01 smithi070 ceph-mon[168282]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2023-09-23T20:48:01.137 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:01 smithi070 ceph-mon[168282]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2023-09-23T20:48:01.137 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:01 smithi070 ceph-mon[168282]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2023-09-23T20:48:01.137 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:01 smithi070 ceph-mon[168282]: rocksdb: Options.max_compaction_bytes: 1677721600 2023-09-23T20:48:01.137 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:01 smithi070 ceph-mon[168282]: rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2023-09-23T20:48:01.137 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:01 smithi070 ceph-mon[168282]: rocksdb: Options.arena_block_size: 1048576 2023-09-23T20:48:01.137 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:01 smithi070 ceph-mon[168282]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2023-09-23T20:48:01.137 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:01 smithi070 ceph-mon[168282]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2023-09-23T20:48:01.137 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:01 smithi070 ceph-mon[168282]: rocksdb: Options.disable_auto_compactions: 0 2023-09-23T20:48:01.138 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:01 smithi070 ceph-mon[168282]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2023-09-23T20:48:01.138 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:01 smithi070 ceph-mon[168282]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2023-09-23T20:48:01.138 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:01 smithi070 ceph-mon[168282]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2023-09-23T20:48:01.139 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:01 smithi070 ceph-mon[168282]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2023-09-23T20:48:01.139 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:01 smithi070 ceph-mon[168282]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2023-09-23T20:48:01.139 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:01 smithi070 ceph-mon[168282]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2023-09-23T20:48:01.139 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:01 smithi070 ceph-mon[168282]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2023-09-23T20:48:01.139 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:01 smithi070 ceph-mon[168282]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2023-09-23T20:48:01.139 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:01 smithi070 ceph-mon[168282]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2023-09-23T20:48:01.139 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:01 smithi070 ceph-mon[168282]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2023-09-23T20:48:01.139 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:01 smithi070 ceph-mon[168282]: rocksdb: Options.table_properties_collectors: 2023-09-23T20:48:01.139 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:01 smithi070 ceph-mon[168282]: rocksdb: Options.inplace_update_support: 0 2023-09-23T20:48:01.139 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:01 smithi070 ceph-mon[168282]: rocksdb: Options.inplace_update_num_locks: 10000 2023-09-23T20:48:01.139 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:01 smithi070 ceph-mon[168282]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2023-09-23T20:48:01.139 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:01 smithi070 ceph-mon[168282]: rocksdb: Options.memtable_whole_key_filtering: 0 2023-09-23T20:48:01.140 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:01 smithi070 ceph-mon[168282]: rocksdb: Options.memtable_huge_page_size: 0 2023-09-23T20:48:01.140 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:01 smithi070 ceph-mon[168282]: rocksdb: Options.bloom_locality: 0 2023-09-23T20:48:01.140 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:01 smithi070 ceph-mon[168282]: rocksdb: Options.max_successive_merges: 0 2023-09-23T20:48:01.140 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:01 smithi070 ceph-mon[168282]: rocksdb: Options.optimize_filters_for_hits: 0 2023-09-23T20:48:01.140 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:01 smithi070 ceph-mon[168282]: rocksdb: Options.paranoid_file_checks: 0 2023-09-23T20:48:01.140 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:01 smithi070 ceph-mon[168282]: rocksdb: Options.force_consistency_checks: 1 2023-09-23T20:48:01.140 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:01 smithi070 ceph-mon[168282]: rocksdb: Options.report_bg_io_stats: 0 2023-09-23T20:48:01.140 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:01 smithi070 ceph-mon[168282]: rocksdb: Options.ttl: 2592000 2023-09-23T20:48:01.140 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:01 smithi070 ceph-mon[168282]: rocksdb: Options.periodic_compaction_seconds: 0 2023-09-23T20:48:01.140 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:01 smithi070 ceph-mon[168282]: rocksdb: Options.preclude_last_level_data_seconds: 0 2023-09-23T20:48:01.140 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:01 smithi070 ceph-mon[168282]: rocksdb: Options.preserve_internal_time_seconds: 0 2023-09-23T20:48:01.140 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:01 smithi070 ceph-mon[168282]: rocksdb: Options.enable_blob_files: false 2023-09-23T20:48:01.140 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:01 smithi070 ceph-mon[168282]: rocksdb: Options.min_blob_size: 0 2023-09-23T20:48:01.141 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:01 smithi070 ceph-mon[168282]: rocksdb: Options.blob_file_size: 268435456 2023-09-23T20:48:01.141 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:01 smithi070 ceph-mon[168282]: rocksdb: Options.blob_compression_type: NoCompression 2023-09-23T20:48:01.141 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:01 smithi070 ceph-mon[168282]: rocksdb: Options.enable_blob_garbage_collection: false 2023-09-23T20:48:01.141 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:01 smithi070 ceph-mon[168282]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2023-09-23T20:48:01.141 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:01 smithi070 ceph-mon[168282]: rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2023-09-23T20:48:01.141 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:01 smithi070 ceph-mon[168282]: rocksdb: Options.blob_compaction_readahead_size: 0 2023-09-23T20:48:01.141 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:01 smithi070 ceph-mon[168282]: rocksdb: Options.blob_file_starting_level: 0 2023-09-23T20:48:01.141 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:01 smithi070 ceph-mon[168282]: rocksdb: Options.experimental_mempurge_threshold: 0.000000 2023-09-23T20:48:01.141 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:01 smithi070 ceph-mon[168282]: rocksdb: [table/block_based/block_based_table_reader.cc:721] At least one SST file opened without unique ID to verify: 27.sst 2023-09-23T20:48:01.141 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:01 smithi070 ceph-mon[168282]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2023-09-23T20:48:01.141 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:01 smithi070 ceph-mon[168282]: rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-b/store.db/MANIFEST-000009 succeeded,manifest_file_number is 9, next_file_number is 29, last_sequence is 13495, log_number is 25,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2023-09-23T20:48:01.141 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:01 smithi070 ceph-mon[168282]: rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 25 2023-09-23T20:48:01.142 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:01 smithi070 ceph-mon[168282]: rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: e8f18e04-a51b-4277-8901-73c605241e6c 2023-09-23T20:48:01.142 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:01 smithi070 ceph-mon[168282]: rocksdb: EVENT_LOG_v1 {"time_micros": 1695502081058403, "job": 1, "event": "recovery_started", "wal_files": [25]} 2023-09-23T20:48:01.142 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:01 smithi070 ceph-mon[168282]: rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #25 mode 2 2023-09-23T20:48:01.142 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:01 smithi070 ceph-mon[168282]: rocksdb: EVENT_LOG_v1 {"time_micros": 1695502081064272, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 30, "file_size": 1289215, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 13490, "largest_seqno": 14966, "table_properties": {"data_size": 1283588, "index_size": 3396, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 1221, "raw_key_size": 10350, "raw_average_key_size": 21, "raw_value_size": 1273163, "raw_average_value_size": 2669, "num_data_blocks": 156, "num_entries": 477, "num_filter_entries": 477, "num_deletions": 0, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1695502081, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "e8f18e04-a51b-4277-8901-73c605241e6c", "db_session_id": "JYQY3284W0GMNMTWXP2P", "orig_file_number": 30, "seqno_to_time_mapping": "N/A"}} 2023-09-23T20:48:01.142 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:01 smithi070 ceph-mon[168282]: rocksdb: EVENT_LOG_v1 {"time_micros": 1695502081064351, "job": 1, "event": "recovery_finished"} 2023-09-23T20:48:01.142 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:01 smithi070 ceph-mon[168282]: rocksdb: [db/version_set.cc:5047] Creating manifest 32 2023-09-23T20:48:01.142 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:01 smithi070 ceph-mon[168282]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2023-09-23T20:48:01.142 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:01 smithi070 ceph-mon[168282]: rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-b/store.db/000025.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2023-09-23T20:48:01.142 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:01 smithi070 ceph-mon[168282]: rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x55e8e8698000 2023-09-23T20:48:01.142 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:01 smithi070 ceph-mon[168282]: rocksdb: DB pointer 0x55e8e8684000 2023-09-23T20:48:01.142 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:01 smithi070 ceph-mon[168282]: rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2023-09-23T20:48:01.142 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:01 smithi070 ceph-mon[168282]: rocksdb: [db/db_impl/db_impl.cc:1111] 2023-09-23T20:48:01.142 INFO:journalctl@ceph.mon.b.smithi070.stdout: ** DB Stats ** 2023-09-23T20:48:01.143 INFO:journalctl@ceph.mon.b.smithi070.stdout: Uptime(secs): 0.0 total, 0.0 interval 2023-09-23T20:48:01.143 INFO:journalctl@ceph.mon.b.smithi070.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2023-09-23T20:48:01.143 INFO:journalctl@ceph.mon.b.smithi070.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2023-09-23T20:48:01.143 INFO:journalctl@ceph.mon.b.smithi070.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2023-09-23T20:48:01.143 INFO:journalctl@ceph.mon.b.smithi070.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2023-09-23T20:48:01.143 INFO:journalctl@ceph.mon.b.smithi070.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2023-09-23T20:48:01.143 INFO:journalctl@ceph.mon.b.smithi070.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2023-09-23T20:48:01.143 INFO:journalctl@ceph.mon.b.smithi070.stdout: 2023-09-23T20:48:01.143 INFO:journalctl@ceph.mon.b.smithi070.stdout: ** Compaction Stats [default] ** 2023-09-23T20:48:01.143 INFO:journalctl@ceph.mon.b.smithi070.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2023-09-23T20:48:01.143 INFO:journalctl@ceph.mon.b.smithi070.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2023-09-23T20:48:01.143 INFO:journalctl@ceph.mon.b.smithi070.stdout: L0 1/0 1.23 MB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 487.1 0.00 0.00 1 0.003 0 0 0.0 0.0 2023-09-23T20:48:01.144 INFO:journalctl@ceph.mon.b.smithi070.stdout: L6 1/0 7.89 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 0.0 0.0 2023-09-23T20:48:01.144 INFO:journalctl@ceph.mon.b.smithi070.stdout: Sum 2/0 9.12 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 487.1 0.00 0.00 1 0.003 0 0 0.0 0.0 2023-09-23T20:48:01.144 INFO:journalctl@ceph.mon.b.smithi070.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 487.1 0.00 0.00 1 0.003 0 0 0.0 0.0 2023-09-23T20:48:01.144 INFO:journalctl@ceph.mon.b.smithi070.stdout: 2023-09-23T20:48:01.144 INFO:journalctl@ceph.mon.b.smithi070.stdout: ** Compaction Stats [default] ** 2023-09-23T20:48:01.144 INFO:journalctl@ceph.mon.b.smithi070.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2023-09-23T20:48:01.144 INFO:journalctl@ceph.mon.b.smithi070.stdout: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-09-23T20:48:01.144 INFO:journalctl@ceph.mon.b.smithi070.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 487.1 0.00 0.00 1 0.003 0 0 0.0 0.0 2023-09-23T20:48:01.144 INFO:journalctl@ceph.mon.b.smithi070.stdout: 2023-09-23T20:48:01.144 INFO:journalctl@ceph.mon.b.smithi070.stdout: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2023-09-23T20:48:01.144 INFO:journalctl@ceph.mon.b.smithi070.stdout: 2023-09-23T20:48:01.144 INFO:journalctl@ceph.mon.b.smithi070.stdout: Uptime(secs): 0.0 total, 0.0 interval 2023-09-23T20:48:01.145 INFO:journalctl@ceph.mon.b.smithi070.stdout: Flush(GB): cumulative 0.001, interval 0.001 2023-09-23T20:48:01.145 INFO:journalctl@ceph.mon.b.smithi070.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2023-09-23T20:48:01.145 INFO:journalctl@ceph.mon.b.smithi070.stdout: AddFile(Total Files): cumulative 0, interval 0 2023-09-23T20:48:01.145 INFO:journalctl@ceph.mon.b.smithi070.stdout: AddFile(L0 Files): cumulative 0, interval 0 2023-09-23T20:48:01.145 INFO:journalctl@ceph.mon.b.smithi070.stdout: AddFile(Keys): cumulative 0, interval 0 2023-09-23T20:48:01.145 INFO:journalctl@ceph.mon.b.smithi070.stdout: Cumulative compaction: 0.00 GB write, 151.10 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-09-23T20:48:01.145 INFO:journalctl@ceph.mon.b.smithi070.stdout: Interval compaction: 0.00 GB write, 151.10 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-09-23T20:48:01.145 INFO:journalctl@ceph.mon.b.smithi070.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2023-09-23T20:48:01.145 INFO:journalctl@ceph.mon.b.smithi070.stdout: Block cache BinnedLRUCache@0x55e8e85fb350#7 capacity: 512.00 MB usage: 5.36 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 1.1e-05 secs_since: 0 2023-09-23T20:48:01.145 INFO:journalctl@ceph.mon.b.smithi070.stdout: Block cache entry stats(count,size,portion): FilterBlock(1,1.28 KB,0.000244379%) IndexBlock(1,4.08 KB,0.000777841%) Misc(1,0.00 KB,0%) 2023-09-23T20:48:01.145 INFO:journalctl@ceph.mon.b.smithi070.stdout: 2023-09-23T20:48:01.145 INFO:journalctl@ceph.mon.b.smithi070.stdout: ** File Read Latency Histogram By Level [default] ** 2023-09-23T20:48:01.145 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:01 smithi070 ceph-mon[168282]: starting mon.b rank 1 at public addrs [v2:172.21.15.70:3300/0,v1:172.21.15.70:6789/0] at bind addrs [v2:172.21.15.70:3300/0,v1:172.21.15.70:6789/0] mon_data /var/lib/ceph/mon/ceph-b fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 2023-09-23T20:48:01.146 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:01 smithi070 ceph-mon[168282]: mon.b@-1(???) e3 preinit fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 2023-09-23T20:48:01.146 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:01 smithi070 ceph-mon[168282]: mon.b@-1(???).mds e8 new map 2023-09-23T20:48:01.146 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:01 smithi070 ceph-mon[168282]: mon.b@-1(???).mds e8 print_map 2023-09-23T20:48:01.146 INFO:journalctl@ceph.mon.b.smithi070.stdout: e8 2023-09-23T20:48:01.146 INFO:journalctl@ceph.mon.b.smithi070.stdout: enable_multiple, ever_enabled_multiple: 1,1 2023-09-23T20:48:01.146 INFO:journalctl@ceph.mon.b.smithi070.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2023-09-23T20:48:01.146 INFO:journalctl@ceph.mon.b.smithi070.stdout: legacy client fscid: 1 2023-09-23T20:48:01.146 INFO:journalctl@ceph.mon.b.smithi070.stdout: 2023-09-23T20:48:01.146 INFO:journalctl@ceph.mon.b.smithi070.stdout: Filesystem 'foo' (1) 2023-09-23T20:48:01.146 INFO:journalctl@ceph.mon.b.smithi070.stdout: fs_name foo 2023-09-23T20:48:01.146 INFO:journalctl@ceph.mon.b.smithi070.stdout: epoch 7 2023-09-23T20:48:01.146 INFO:journalctl@ceph.mon.b.smithi070.stdout: flags 12 joinable allow_snaps allow_multimds_snaps 2023-09-23T20:48:01.147 INFO:journalctl@ceph.mon.b.smithi070.stdout: created 2023-09-23T20:41:58.904985+0000 2023-09-23T20:48:01.147 INFO:journalctl@ceph.mon.b.smithi070.stdout: modified 2023-09-23T20:42:07.570998+0000 2023-09-23T20:48:01.147 INFO:journalctl@ceph.mon.b.smithi070.stdout: tableserver 0 2023-09-23T20:48:01.147 INFO:journalctl@ceph.mon.b.smithi070.stdout: root 0 2023-09-23T20:48:01.147 INFO:journalctl@ceph.mon.b.smithi070.stdout: session_timeout 60 2023-09-23T20:48:01.147 INFO:journalctl@ceph.mon.b.smithi070.stdout: session_autoclose 300 2023-09-23T20:48:01.147 INFO:journalctl@ceph.mon.b.smithi070.stdout: max_file_size 1099511627776 2023-09-23T20:48:01.147 INFO:journalctl@ceph.mon.b.smithi070.stdout: required_client_features {} 2023-09-23T20:48:01.147 INFO:journalctl@ceph.mon.b.smithi070.stdout: last_failure 0 2023-09-23T20:48:01.147 INFO:journalctl@ceph.mon.b.smithi070.stdout: last_failure_osd_epoch 0 2023-09-23T20:48:01.147 INFO:journalctl@ceph.mon.b.smithi070.stdout: compat compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,7=mds uses inline data,8=no anchor table,9=file layout v2,10=snaprealm v2} 2023-09-23T20:48:01.147 INFO:journalctl@ceph.mon.b.smithi070.stdout: max_mds 1 2023-09-23T20:48:01.147 INFO:journalctl@ceph.mon.b.smithi070.stdout: in 0 2023-09-23T20:48:01.148 INFO:journalctl@ceph.mon.b.smithi070.stdout: up {0=14658} 2023-09-23T20:48:01.148 INFO:journalctl@ceph.mon.b.smithi070.stdout: failed 2023-09-23T20:48:01.148 INFO:journalctl@ceph.mon.b.smithi070.stdout: damaged 2023-09-23T20:48:01.148 INFO:journalctl@ceph.mon.b.smithi070.stdout: stopped 2023-09-23T20:48:01.148 INFO:journalctl@ceph.mon.b.smithi070.stdout: data_pools [4] 2023-09-23T20:48:01.148 INFO:journalctl@ceph.mon.b.smithi070.stdout: metadata_pool 3 2023-09-23T20:48:01.148 INFO:journalctl@ceph.mon.b.smithi070.stdout: inline_data disabled 2023-09-23T20:48:01.148 INFO:journalctl@ceph.mon.b.smithi070.stdout: balancer 2023-09-23T20:48:01.148 INFO:journalctl@ceph.mon.b.smithi070.stdout: bal_rank_mask -1 2023-09-23T20:48:01.148 INFO:journalctl@ceph.mon.b.smithi070.stdout: standby_count_wanted 1 2023-09-23T20:48:01.148 INFO:journalctl@ceph.mon.b.smithi070.stdout: [mds.foo.smithi026.znfpnl{0:14658} state up:active seq 3 join_fscid=1 addr [v2:172.21.15.26:6834/417830853,v1:172.21.15.26:6835/417830853] compat {c=[1],r=[1],i=[7ff]}] 2023-09-23T20:48:01.148 INFO:journalctl@ceph.mon.b.smithi070.stdout: 2023-09-23T20:48:01.148 INFO:journalctl@ceph.mon.b.smithi070.stdout: 2023-09-23T20:48:01.149 INFO:journalctl@ceph.mon.b.smithi070.stdout: Standby daemons: 2023-09-23T20:48:01.149 INFO:journalctl@ceph.mon.b.smithi070.stdout: 2023-09-23T20:48:01.149 INFO:journalctl@ceph.mon.b.smithi070.stdout: [mds.foo.smithi070.imqhqy{-1:24529} state up:standby seq 2 join_fscid=1 addr [v2:172.21.15.70:6832/241309560,v1:172.21.15.70:6833/241309560] compat {c=[1],r=[1],i=[7ff]}] 2023-09-23T20:48:01.149 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:01 smithi070 ceph-mon[168282]: mon.b@-1(???).osd e79 crush map has features 3314933000854323200, adjusting msgr requires 2023-09-23T20:48:01.149 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:01 smithi070 ceph-mon[168282]: mon.b@-1(???).osd e79 crush map has features 432629239337189376, adjusting msgr requires 2023-09-23T20:48:01.149 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:01 smithi070 ceph-mon[168282]: mon.b@-1(???).osd e79 crush map has features 432629239337189376, adjusting msgr requires 2023-09-23T20:48:01.149 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:01 smithi070 ceph-mon[168282]: mon.b@-1(???).osd e79 crush map has features 432629239337189376, adjusting msgr requires 2023-09-23T20:48:01.149 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:01 smithi070 ceph-mon[168282]: mon.b@-1(???).paxosservice(auth 1..21) refresh upgraded, format 0 -> 3 2023-09-23T20:48:01.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:01 smithi070 systemd[1]: Started Ceph mon.b for b3f8e4ec-5a50-11ee-8db4-212e2dc638e7. 2023-09-23T20:48:02.293 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 18.2.0-460-gbb0daeff (bb0daeff97f020f686596fca6eba20c310e26fb7) reef (stable)": 3 2023-09-23T20:48:02.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:02 smithi070 ceph-mon[168282]: overall HEALTH_OK 2023-09-23T20:48:02.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:02 smithi070 ceph-mon[168282]: mon.c calling monitor election 2023-09-23T20:48:02.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:02 smithi070 ceph-mon[168282]: mon.b calling monitor election 2023-09-23T20:48:02.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:02 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-23T20:48:02.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:02 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T20:48:02.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:02 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T20:48:02.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:02 smithi070 ceph-mon[168282]: mon.a calling monitor election 2023-09-23T20:48:02.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:02 smithi070 ceph-mon[168282]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2023-09-23T20:48:02.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:02 smithi070 ceph-mon[168282]: monmap e4: 3 mons at {a=[v2:172.21.15.26:3300/0,v1:172.21.15.26:6789/0],b=[v2:172.21.15.70:3300/0,v1:172.21.15.70:6789/0],c=[v2:172.21.15.26:3301/0,v1:172.21.15.26:6790/0]} removed_ranks: {} 2023-09-23T20:48:02.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:02 smithi070 ceph-mon[168282]: fsmap foo:1 {0=foo.smithi026.znfpnl=up:active} 1 up:standby 2023-09-23T20:48:02.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:02 smithi070 ceph-mon[168282]: osdmap e79: 8 total, 8 up, 8 in 2023-09-23T20:48:02.621 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:02 smithi070 ceph-mon[168282]: mgrmap e37: y(active, since 2m), standbys: x 2023-09-23T20:48:02.621 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:02 smithi070 ceph-mon[168282]: overall HEALTH_OK 2023-09-23T20:48:02.621 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:02 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:48:02.621 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:02 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:48:02.621 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:02 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:48:02.621 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:02 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/637740067' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:48:02.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:48:02 smithi026 ceph-mon[179688]: overall HEALTH_OK 2023-09-23T20:48:02.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:48:02 smithi026 ceph-mon[179688]: mon.c calling monitor election 2023-09-23T20:48:02.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:48:02 smithi026 ceph-mon[179688]: mon.b calling monitor election 2023-09-23T20:48:02.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:48:02 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-23T20:48:02.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:48:02 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T20:48:02.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:48:02 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T20:48:02.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:48:02 smithi026 ceph-mon[179688]: mon.a calling monitor election 2023-09-23T20:48:02.633 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:48:02 smithi026 ceph-mon[179688]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2023-09-23T20:48:02.633 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:48:02 smithi026 ceph-mon[179688]: monmap e4: 3 mons at {a=[v2:172.21.15.26:3300/0,v1:172.21.15.26:6789/0],b=[v2:172.21.15.70:3300/0,v1:172.21.15.70:6789/0],c=[v2:172.21.15.26:3301/0,v1:172.21.15.26:6790/0]} removed_ranks: {} 2023-09-23T20:48:02.633 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:48:02 smithi026 ceph-mon[179688]: fsmap foo:1 {0=foo.smithi026.znfpnl=up:active} 1 up:standby 2023-09-23T20:48:02.633 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:48:02 smithi026 ceph-mon[179688]: osdmap e79: 8 total, 8 up, 8 in 2023-09-23T20:48:02.633 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:48:02 smithi026 ceph-mon[179688]: mgrmap e37: y(active, since 2m), standbys: x 2023-09-23T20:48:02.633 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:48:02 smithi026 ceph-mon[179688]: overall HEALTH_OK 2023-09-23T20:48:02.633 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:48:02 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:48:02.633 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:48:02 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:48:02.633 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:48:02 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:48:02.633 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:48:02 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/637740067' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:48:02.633 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:48:02 smithi026 ceph-mon[175151]: overall HEALTH_OK 2023-09-23T20:48:02.634 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:48:02 smithi026 ceph-mon[175151]: mon.c calling monitor election 2023-09-23T20:48:02.634 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:48:02 smithi026 ceph-mon[175151]: mon.b calling monitor election 2023-09-23T20:48:02.634 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:48:02 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-23T20:48:02.634 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:48:02 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T20:48:02.634 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:48:02 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T20:48:02.634 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:48:02 smithi026 ceph-mon[175151]: mon.a calling monitor election 2023-09-23T20:48:02.634 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:48:02 smithi026 ceph-mon[175151]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2023-09-23T20:48:02.634 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:48:02 smithi026 ceph-mon[175151]: monmap e4: 3 mons at {a=[v2:172.21.15.26:3300/0,v1:172.21.15.26:6789/0],b=[v2:172.21.15.70:3300/0,v1:172.21.15.70:6789/0],c=[v2:172.21.15.26:3301/0,v1:172.21.15.26:6790/0]} removed_ranks: {} 2023-09-23T20:48:02.634 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:48:02 smithi026 ceph-mon[175151]: fsmap foo:1 {0=foo.smithi026.znfpnl=up:active} 1 up:standby 2023-09-23T20:48:02.634 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:48:02 smithi026 ceph-mon[175151]: osdmap e79: 8 total, 8 up, 8 in 2023-09-23T20:48:02.634 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:48:02 smithi026 ceph-mon[175151]: mgrmap e37: y(active, since 2m), standbys: x 2023-09-23T20:48:02.634 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:48:02 smithi026 ceph-mon[175151]: overall HEALTH_OK 2023-09-23T20:48:02.635 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:48:02 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:48:02.635 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:48:02 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:48:02.635 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:48:02 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:48:02.635 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:48:02 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/637740067' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:48:02.733 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -e sha1=bb0daeff97f020f686596fca6eba20c310e26fb7 -- bash -c 'ceph orch ps' 2023-09-23T20:48:03.339 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:48:03.742 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:48:03 smithi026 ceph-mon[179688]: pgmap v86: 73 pgs: 73 active+clean; 582 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:48:03.742 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:48:03 smithi026 ceph-mon[175151]: pgmap v86: 73 pgs: 73 active+clean; 582 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:48:03.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:03 smithi070 ceph-mon[168282]: pgmap v86: 73 pgs: 73 active+clean; 582 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:48:04.869 INFO:teuthology.orchestra.run.smithi026.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2023-09-23T20:48:04.869 INFO:teuthology.orchestra.run.smithi026.stdout:alertmanager.a smithi026 *:9093,9094 running (3m) - 6m 28.6M - 0.25.0 c8568f914cd2 d4b48de0f10c 2023-09-23T20:48:04.869 INFO:teuthology.orchestra.run.smithi026.stdout:grafana.a smithi070 *:3000 running (3m) - 6m 91.7M - 9.4.7 2c41d148cca3 ed7e189354a5 2023-09-23T20:48:04.869 INFO:teuthology.orchestra.run.smithi026.stdout:mds.foo.smithi026.znfpnl smithi026 running (6m) - 6m 18.9M - 17.2.6-1123-g61431f6b 0db14aa01094 b6861ac0ff55 2023-09-23T20:48:04.869 INFO:teuthology.orchestra.run.smithi026.stdout:mds.foo.smithi070.imqhqy smithi070 running (6m) - 5m 17.7M - 17.2.6-1123-g61431f6b 0db14aa01094 ca26bf551eb9 2023-09-23T20:48:04.869 INFO:teuthology.orchestra.run.smithi026.stdout:mgr.x smithi070 *:8443,9283,8765 running (3m) - 10m 448M - 18.2.0-460-gbb0daeff e32c2a64c126 35059b639121 2023-09-23T20:48:04.869 INFO:teuthology.orchestra.run.smithi026.stdout:mgr.y smithi026 *:8443,9283,8765 running (3m) - 11m 505M - 18.2.0-460-gbb0daeff e32c2a64c126 5cc7ece3b7e8 2023-09-23T20:48:04.869 INFO:teuthology.orchestra.run.smithi026.stdout:mon.a smithi026 running (3m) - 12m 61.0M 2048M 18.2.0-460-gbb0daeff e32c2a64c126 f0ba51b53f50 2023-09-23T20:48:04.869 INFO:teuthology.orchestra.run.smithi026.stdout:mon.b smithi070 starting - - - 2048M 2023-09-23T20:48:04.870 INFO:teuthology.orchestra.run.smithi026.stdout:mon.c smithi026 running (89s) - 10m 35.4M 2048M 18.2.0-460-gbb0daeff e32c2a64c126 ad100e9213d3 2023-09-23T20:48:04.870 INFO:teuthology.orchestra.run.smithi026.stdout:node-exporter.a smithi026 *:9100 running (3m) - 6m 17.0M - 1.5.0 0da6a335fe13 95269f65f9af 2023-09-23T20:48:04.870 INFO:teuthology.orchestra.run.smithi026.stdout:node-exporter.b smithi070 *:9100 running (3m) - 6m 17.9M - 1.5.0 0da6a335fe13 5b56a4a415d0 2023-09-23T20:48:04.870 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0 smithi026 running (9m) - 9m 82.7M 2501M 17.2.6-1123-g61431f6b 0db14aa01094 e40b0407b8cf 2023-09-23T20:48:04.870 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1 smithi026 running (9m) - 9m 83.4M 2501M 17.2.6-1123-g61431f6b 0db14aa01094 a73d47580165 2023-09-23T20:48:04.870 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2 smithi026 running (9m) - 9m 151M 2501M 17.2.6-1123-g61431f6b 0db14aa01094 fc4783fb1a89 2023-09-23T20:48:04.870 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3 smithi026 running (8m) - 8m 73.6M 2501M 17.2.6-1123-g61431f6b 0db14aa01094 5f0339ddc2c4 2023-09-23T20:48:04.870 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4 smithi070 running (8m) - 8m 92.0M 2500M 17.2.6-1123-g61431f6b 0db14aa01094 308a2af3101e 2023-09-23T20:48:04.870 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5 smithi070 running (8m) - 8m 103M 2500M 17.2.6-1123-g61431f6b 0db14aa01094 16cd72517077 2023-09-23T20:48:04.870 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6 smithi070 running (7m) - 7m 68.8M 2500M 17.2.6-1123-g61431f6b 0db14aa01094 5d24c21e8be7 2023-09-23T20:48:04.871 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7 smithi070 running (7m) - 7m 109M 2500M 17.2.6-1123-g61431f6b 0db14aa01094 8c1b7f314c41 2023-09-23T20:48:04.871 INFO:teuthology.orchestra.run.smithi026.stdout:prometheus.a smithi070 *:9095 running (3m) - 7m 57.2M - 2.43.0 a07b618ecd1d dda227f93524 2023-09-23T20:48:05.398 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -e sha1=bb0daeff97f020f686596fca6eba20c310e26fb7 -- bash -c 'ceph orch upgrade pause' 2023-09-23T20:48:06.004 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:48:06.074 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:48:05 smithi026 ceph-mon[175151]: pgmap v87: 73 pgs: 73 active+clean; 582 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:48:06.075 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:48:05 smithi026 ceph-mon[179688]: pgmap v87: 73 pgs: 73 active+clean; 582 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:48:06.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:05 smithi070 ceph-mon[168282]: pgmap v87: 73 pgs: 73 active+clean; 582 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:48:06.382 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:48:06 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[164029]: 2023-09-23T20:48:06.073+0000 7fd59efc8700 -1 mgr.server handle_report got status from non-daemon mon.b 2023-09-23T20:48:07.038 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:48:06 smithi026 ceph-mon[175151]: from='client.44132 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T20:48:07.038 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:48:06 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:48:07.038 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:48:06 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:48:07.038 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:48:06 smithi026 ceph-mon[179688]: from='client.44132 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T20:48:07.038 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:48:06 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:48:07.039 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:48:06 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:48:07.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:06 smithi070 ceph-mon[168282]: from='client.44132 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T20:48:07.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:06 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:48:07.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:06 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:48:07.470 INFO:teuthology.orchestra.run.smithi026.stdout:Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d 2023-09-23T20:48:07.863 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:48:07 smithi026 ceph-mon[179688]: pgmap v88: 73 pgs: 73 active+clean; 582 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:48:07.863 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:48:07 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:48:07.863 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:48:07 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:48:07.863 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:48:07 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:48:07.863 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:48:07 smithi026 ceph-mon[175151]: pgmap v88: 73 pgs: 73 active+clean; 582 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:48:07.863 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:48:07 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:48:07.863 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:48:07 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:48:07.863 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:48:07 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:48:07.953 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -e sha1=bb0daeff97f020f686596fca6eba20c310e26fb7 -- bash -c 'sleep 60' 2023-09-23T20:48:08.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:07 smithi070 ceph-mon[168282]: pgmap v88: 73 pgs: 73 active+clean; 582 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:48:08.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:07 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:48:08.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:07 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:48:08.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:07 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:48:08.559 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:48:08.738 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 20:48:08 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-prometheus-a[155975]: ts=2023-09-23T20:48:08.659Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1123-g61431f6b (61431f6ba2752649b506867ec856b4a6236f90dd) quincy (stable)\", cluster_addr=\"172.21.15.26\", device_class=\"ssd\", hostname=\"smithi026\", instance=\"smithi070.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.26\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1123-g61431f6b (61431f6ba2752649b506867ec856b4a6236f90dd) quincy (stable)\", cluster_addr=\"172.21.15.26\", device_class=\"ssd\", hostname=\"smithi026\", instance=\"smithi026:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.26\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-09-23T20:48:09.001 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:08 smithi070 ceph-mon[168282]: from='client.44138 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T20:48:09.001 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:08 smithi070 ceph-mon[168282]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d 2023-09-23T20:48:09.086 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:48:08 smithi026 ceph-mon[175151]: from='client.44138 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T20:48:09.086 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:48:08 smithi026 ceph-mon[175151]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d 2023-09-23T20:48:09.086 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:48:08 smithi026 ceph-mon[179688]: from='client.44138 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T20:48:09.086 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:48:08 smithi026 ceph-mon[179688]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d 2023-09-23T20:48:09.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:48:09 smithi026 ceph-mon[175151]: pgmap v89: 73 pgs: 73 active+clean; 582 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:48:09.883 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:48:09 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[164029]: ::ffff:172.21.15.70 - - [23/Sep/2023:20:48:09] "GET /metrics HTTP/1.1" 200 33626 "" "Prometheus/2.43.0" 2023-09-23T20:48:09.883 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:48:09 smithi026 ceph-mon[179688]: pgmap v89: 73 pgs: 73 active+clean; 582 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:48:10.052 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:09 smithi070 ceph-mon[168282]: pgmap v89: 73 pgs: 73 active+clean; 582 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:48:11.598 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:48:11 smithi026 ceph-mon[175151]: pgmap v90: 73 pgs: 73 active+clean; 582 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:48:11.598 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:48:11 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:48:11.599 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:48:11 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:48:11.599 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:48:11 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:48:11.599 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:48:11 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:48:11.599 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:48:11 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:48:11.599 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:48:11 smithi026 ceph-mon[175151]: Reconfiguring mon.a (monmap changed)... 2023-09-23T20:48:11.599 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:48:11 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-23T20:48:11.599 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:48:11 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-09-23T20:48:11.599 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:48:11 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:48:11.599 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:48:11 smithi026 ceph-mon[175151]: Reconfiguring daemon mon.a on smithi026 2023-09-23T20:48:11.599 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:48:11 smithi026 ceph-mon[179688]: pgmap v90: 73 pgs: 73 active+clean; 582 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:48:11.600 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:48:11 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:48:11.600 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:48:11 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:48:11.600 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:48:11 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:48:11.600 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:48:11 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:48:11.600 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:48:11 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:48:11.600 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:48:11 smithi026 ceph-mon[179688]: Reconfiguring mon.a (monmap changed)... 2023-09-23T20:48:11.600 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:48:11 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-23T20:48:11.600 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:48:11 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-09-23T20:48:11.600 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:48:11 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:48:11.600 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:48:11 smithi026 ceph-mon[179688]: Reconfiguring daemon mon.a on smithi026 2023-09-23T20:48:11.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:11 smithi070 ceph-mon[168282]: pgmap v90: 73 pgs: 73 active+clean; 582 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:48:11.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:11 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:48:11.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:11 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:48:11.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:11 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:48:11.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:11 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:48:11.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:11 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:48:11.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:11 smithi070 ceph-mon[168282]: Reconfiguring mon.a (monmap changed)... 2023-09-23T20:48:11.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:11 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-23T20:48:11.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:11 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-09-23T20:48:11.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:11 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:48:11.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:11 smithi070 ceph-mon[168282]: Reconfiguring daemon mon.a on smithi026 2023-09-23T20:48:12.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:12 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:48:12.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:12 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:48:12.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:12 smithi070 ceph-mon[168282]: Reconfiguring mgr.y (monmap changed)... 2023-09-23T20:48:12.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:12 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-09-23T20:48:12.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:12 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-09-23T20:48:12.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:12 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:48:12.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:12 smithi070 ceph-mon[168282]: Reconfiguring daemon mgr.y on smithi026 2023-09-23T20:48:12.881 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:48:12 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:48:12.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:48:12 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:48:12.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:48:12 smithi026 ceph-mon[179688]: Reconfiguring mgr.y (monmap changed)... 2023-09-23T20:48:12.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:48:12 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-09-23T20:48:12.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:48:12 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-09-23T20:48:12.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:48:12 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:48:12.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:48:12 smithi026 ceph-mon[179688]: Reconfiguring daemon mgr.y on smithi026 2023-09-23T20:48:12.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:48:12 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:48:12.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:48:12 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:48:12.883 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:48:12 smithi026 ceph-mon[175151]: Reconfiguring mgr.y (monmap changed)... 2023-09-23T20:48:12.883 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:48:12 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-09-23T20:48:12.883 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:48:12 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-09-23T20:48:12.883 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:48:12 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:48:12.883 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:48:12 smithi026 ceph-mon[175151]: Reconfiguring daemon mgr.y on smithi026 2023-09-23T20:48:14.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:13 smithi070 ceph-mon[168282]: pgmap v91: 73 pgs: 73 active+clean; 582 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:48:14.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:13 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:48:14.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:13 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:48:14.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:13 smithi070 ceph-mon[168282]: Reconfiguring mon.c (monmap changed)... 2023-09-23T20:48:14.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:13 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-23T20:48:14.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:13 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-09-23T20:48:14.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:13 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:48:14.121 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:13 smithi070 ceph-mon[168282]: Reconfiguring daemon mon.c on smithi026 2023-09-23T20:48:14.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:48:13 smithi026 ceph-mon[179688]: pgmap v91: 73 pgs: 73 active+clean; 582 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:48:14.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:48:13 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:48:14.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:48:13 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:48:14.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:48:13 smithi026 ceph-mon[179688]: Reconfiguring mon.c (monmap changed)... 2023-09-23T20:48:14.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:48:13 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-23T20:48:14.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:48:13 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-09-23T20:48:14.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:48:13 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:48:14.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:48:13 smithi026 ceph-mon[179688]: Reconfiguring daemon mon.c on smithi026 2023-09-23T20:48:14.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:48:13 smithi026 ceph-mon[175151]: pgmap v91: 73 pgs: 73 active+clean; 582 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:48:14.134 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:48:13 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:48:14.134 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:48:13 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:48:14.134 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:48:13 smithi026 ceph-mon[175151]: Reconfiguring mon.c (monmap changed)... 2023-09-23T20:48:14.134 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:48:13 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-23T20:48:14.134 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:48:13 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-09-23T20:48:14.134 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:48:13 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:48:14.134 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:48:13 smithi026 ceph-mon[175151]: Reconfiguring daemon mon.c on smithi026 2023-09-23T20:48:15.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:48:14 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:48:15.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:48:14 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:48:15.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:48:14 smithi026 ceph-mon[179688]: Reconfiguring osd.0 (monmap changed)... 2023-09-23T20:48:15.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:48:14 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2023-09-23T20:48:15.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:48:14 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:48:15.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:48:14 smithi026 ceph-mon[179688]: Reconfiguring daemon osd.0 on smithi026 2023-09-23T20:48:15.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:48:14 smithi026 ceph-mon[179688]: pgmap v92: 73 pgs: 73 active+clean; 582 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:48:15.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:48:14 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:48:15.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:48:14 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:48:15.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:48:14 smithi026 ceph-mon[175151]: Reconfiguring osd.0 (monmap changed)... 2023-09-23T20:48:15.134 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:48:14 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2023-09-23T20:48:15.134 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:48:14 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:48:15.134 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:48:14 smithi026 ceph-mon[175151]: Reconfiguring daemon osd.0 on smithi026 2023-09-23T20:48:15.134 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:48:14 smithi026 ceph-mon[175151]: pgmap v92: 73 pgs: 73 active+clean; 582 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:48:15.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:14 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:48:15.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:14 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:48:15.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:14 smithi070 ceph-mon[168282]: Reconfiguring osd.0 (monmap changed)... 2023-09-23T20:48:15.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:14 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2023-09-23T20:48:15.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:14 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:48:15.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:14 smithi070 ceph-mon[168282]: Reconfiguring daemon osd.0 on smithi026 2023-09-23T20:48:15.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:14 smithi070 ceph-mon[168282]: pgmap v92: 73 pgs: 73 active+clean; 582 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:48:16.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:16 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:48:16.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:16 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:48:16.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:16 smithi070 ceph-mon[168282]: Reconfiguring osd.1 (monmap changed)... 2023-09-23T20:48:16.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:16 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2023-09-23T20:48:16.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:16 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:48:16.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:16 smithi070 ceph-mon[168282]: Reconfiguring daemon osd.1 on smithi026 2023-09-23T20:48:16.381 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:48:16 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:48:16.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:48:16 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:48:16.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:48:16 smithi026 ceph-mon[175151]: Reconfiguring osd.1 (monmap changed)... 2023-09-23T20:48:16.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:48:16 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2023-09-23T20:48:16.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:48:16 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:48:16.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:48:16 smithi026 ceph-mon[175151]: Reconfiguring daemon osd.1 on smithi026 2023-09-23T20:48:16.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:48:16 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:48:16.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:48:16 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:48:16.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:48:16 smithi026 ceph-mon[179688]: Reconfiguring osd.1 (monmap changed)... 2023-09-23T20:48:16.383 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:48:16 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2023-09-23T20:48:16.383 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:48:16 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:48:16.383 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:48:16 smithi026 ceph-mon[179688]: Reconfiguring daemon osd.1 on smithi026 2023-09-23T20:48:17.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:17 smithi070 ceph-mon[168282]: pgmap v93: 73 pgs: 73 active+clean; 582 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:48:17.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:17 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:48:17.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:17 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:48:17.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:17 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:48:17.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:17 smithi070 ceph-mon[168282]: Reconfiguring osd.2 (monmap changed)... 2023-09-23T20:48:17.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:17 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2023-09-23T20:48:17.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:17 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:48:17.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:17 smithi070 ceph-mon[168282]: Reconfiguring daemon osd.2 on smithi026 2023-09-23T20:48:17.376 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:48:17 smithi026 ceph-mon[179688]: pgmap v93: 73 pgs: 73 active+clean; 582 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:48:17.376 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:48:17 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:48:17.376 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:48:17 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:48:17.376 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:48:17 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:48:17.376 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:48:17 smithi026 ceph-mon[179688]: Reconfiguring osd.2 (monmap changed)... 2023-09-23T20:48:17.376 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:48:17 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2023-09-23T20:48:17.376 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:48:17 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:48:17.376 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:48:17 smithi026 ceph-mon[179688]: Reconfiguring daemon osd.2 on smithi026 2023-09-23T20:48:17.377 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:48:17 smithi026 ceph-mon[175151]: pgmap v93: 73 pgs: 73 active+clean; 582 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:48:17.377 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:48:17 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:48:17.377 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:48:17 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:48:17.377 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:48:17 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:48:17.377 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:48:17 smithi026 ceph-mon[175151]: Reconfiguring osd.2 (monmap changed)... 2023-09-23T20:48:17.377 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:48:17 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2023-09-23T20:48:17.377 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:48:17 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:48:17.377 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:48:17 smithi026 ceph-mon[175151]: Reconfiguring daemon osd.2 on smithi026 2023-09-23T20:48:18.547 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:48:18 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:48:18.547 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:48:18 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:48:18.547 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:48:18 smithi026 ceph-mon[179688]: Reconfiguring osd.3 (monmap changed)... 2023-09-23T20:48:18.547 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:48:18 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2023-09-23T20:48:18.547 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:48:18 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:48:18.547 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:48:18 smithi026 ceph-mon[179688]: Reconfiguring daemon osd.3 on smithi026 2023-09-23T20:48:18.547 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:48:18 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:48:18.547 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:48:18 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:48:18.548 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:48:18 smithi026 ceph-mon[175151]: Reconfiguring osd.3 (monmap changed)... 2023-09-23T20:48:18.548 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:48:18 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2023-09-23T20:48:18.548 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:48:18 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:48:18.548 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:48:18 smithi026 ceph-mon[175151]: Reconfiguring daemon osd.3 on smithi026 2023-09-23T20:48:18.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:18 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:48:18.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:18 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:48:18.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:18 smithi070 ceph-mon[168282]: Reconfiguring osd.3 (monmap changed)... 2023-09-23T20:48:18.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:18 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2023-09-23T20:48:18.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:18 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:48:18.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:18 smithi070 ceph-mon[168282]: Reconfiguring daemon osd.3 on smithi026 2023-09-23T20:48:18.870 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 20:48:18 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-prometheus-a[155975]: ts=2023-09-23T20:48:18.660Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1123-g61431f6b (61431f6ba2752649b506867ec856b4a6236f90dd) quincy (stable)\", cluster_addr=\"172.21.15.26\", device_class=\"ssd\", hostname=\"smithi026\", instance=\"smithi070.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.26\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1123-g61431f6b (61431f6ba2752649b506867ec856b4a6236f90dd) quincy (stable)\", cluster_addr=\"172.21.15.26\", device_class=\"ssd\", hostname=\"smithi026\", instance=\"smithi026:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.26\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-09-23T20:48:19.882 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:48:19 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[164029]: ::ffff:172.21.15.70 - - [23/Sep/2023:20:48:19] "GET /metrics HTTP/1.1" 200 33696 "" "Prometheus/2.43.0" 2023-09-23T20:48:19.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:48:19 smithi026 ceph-mon[179688]: pgmap v94: 73 pgs: 73 active+clean; 582 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:48:19.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:48:19 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:48:19.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:48:19 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:48:19.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:48:19 smithi026 ceph-mon[179688]: Reconfiguring mds.foo.smithi026.znfpnl (monmap changed)... 2023-09-23T20:48:19.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:48:19 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi026.znfpnl", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-09-23T20:48:19.883 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:48:19 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:48:19.883 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:48:19 smithi026 ceph-mon[179688]: Reconfiguring daemon mds.foo.smithi026.znfpnl on smithi026 2023-09-23T20:48:19.883 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:48:19 smithi026 ceph-mon[175151]: pgmap v94: 73 pgs: 73 active+clean; 582 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:48:19.883 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:48:19 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:48:19.883 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:48:19 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:48:19.883 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:48:19 smithi026 ceph-mon[175151]: Reconfiguring mds.foo.smithi026.znfpnl (monmap changed)... 2023-09-23T20:48:19.883 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:48:19 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi026.znfpnl", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-09-23T20:48:19.883 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:48:19 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:48:19.884 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:48:19 smithi026 ceph-mon[175151]: Reconfiguring daemon mds.foo.smithi026.znfpnl on smithi026 2023-09-23T20:48:20.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:19 smithi070 ceph-mon[168282]: pgmap v94: 73 pgs: 73 active+clean; 582 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:48:20.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:19 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:48:20.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:19 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:48:20.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:19 smithi070 ceph-mon[168282]: Reconfiguring mds.foo.smithi026.znfpnl (monmap changed)... 2023-09-23T20:48:20.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:19 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi026.znfpnl", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-09-23T20:48:20.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:19 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:48:20.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:19 smithi070 ceph-mon[168282]: Reconfiguring daemon mds.foo.smithi026.znfpnl on smithi026 2023-09-23T20:48:21.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:20 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:48:21.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:20 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:48:21.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:20 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-23T20:48:21.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:20 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-09-23T20:48:21.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:20 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:48:21.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:48:20 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:48:21.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:48:20 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:48:21.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:48:20 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-23T20:48:21.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:48:20 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-09-23T20:48:21.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:48:20 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:48:21.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:48:20 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:48:21.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:48:20 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:48:21.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:48:20 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-23T20:48:21.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:48:20 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-09-23T20:48:21.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:48:20 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:48:21.841 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:21 smithi070 ceph-mon[168282]: Reconfiguring mon.b (monmap changed)... 2023-09-23T20:48:21.841 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:21 smithi070 ceph-mon[168282]: Reconfiguring daemon mon.b on smithi070 2023-09-23T20:48:21.841 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:21 smithi070 ceph-mon[168282]: pgmap v95: 73 pgs: 73 active+clean; 582 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:48:21.841 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:21 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:48:21.841 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:21 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:48:21.841 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:21 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-09-23T20:48:21.841 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:21 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-09-23T20:48:21.841 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:21 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:48:22.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:48:21 smithi026 ceph-mon[175151]: Reconfiguring mon.b (monmap changed)... 2023-09-23T20:48:22.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:48:21 smithi026 ceph-mon[175151]: Reconfiguring daemon mon.b on smithi070 2023-09-23T20:48:22.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:48:21 smithi026 ceph-mon[175151]: pgmap v95: 73 pgs: 73 active+clean; 582 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:48:22.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:48:21 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:48:22.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:48:21 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:48:22.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:48:21 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-09-23T20:48:22.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:48:21 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-09-23T20:48:22.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:48:21 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:48:22.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:48:21 smithi026 ceph-mon[179688]: Reconfiguring mon.b (monmap changed)... 2023-09-23T20:48:22.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:48:21 smithi026 ceph-mon[179688]: Reconfiguring daemon mon.b on smithi070 2023-09-23T20:48:22.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:48:21 smithi026 ceph-mon[179688]: pgmap v95: 73 pgs: 73 active+clean; 582 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:48:22.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:48:21 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:48:22.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:48:21 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:48:22.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:48:21 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-09-23T20:48:22.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:48:21 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-09-23T20:48:22.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:48:21 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:48:23.032 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:22 smithi070 ceph-mon[168282]: Reconfiguring mgr.x (monmap changed)... 2023-09-23T20:48:23.032 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:22 smithi070 ceph-mon[168282]: Reconfiguring daemon mgr.x on smithi070 2023-09-23T20:48:23.032 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:22 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:48:23.033 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:22 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:48:23.033 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:22 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2023-09-23T20:48:23.033 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:22 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:48:23.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:48:22 smithi026 ceph-mon[179688]: Reconfiguring mgr.x (monmap changed)... 2023-09-23T20:48:23.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:48:22 smithi026 ceph-mon[179688]: Reconfiguring daemon mgr.x on smithi070 2023-09-23T20:48:23.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:48:22 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:48:23.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:48:22 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:48:23.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:48:22 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2023-09-23T20:48:23.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:48:22 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:48:23.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:48:22 smithi026 ceph-mon[175151]: Reconfiguring mgr.x (monmap changed)... 2023-09-23T20:48:23.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:48:22 smithi026 ceph-mon[175151]: Reconfiguring daemon mgr.x on smithi070 2023-09-23T20:48:23.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:48:22 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:48:23.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:48:22 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:48:23.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:48:22 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2023-09-23T20:48:23.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:48:22 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:48:24.061 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:23 smithi070 ceph-mon[168282]: pgmap v96: 73 pgs: 73 active+clean; 582 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:48:24.061 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:23 smithi070 ceph-mon[168282]: Reconfiguring osd.4 (monmap changed)... 2023-09-23T20:48:24.061 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:23 smithi070 ceph-mon[168282]: Reconfiguring daemon osd.4 on smithi070 2023-09-23T20:48:24.061 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:23 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:48:24.061 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:23 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:48:24.061 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:23 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2023-09-23T20:48:24.061 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:23 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:48:24.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:48:23 smithi026 ceph-mon[175151]: pgmap v96: 73 pgs: 73 active+clean; 582 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:48:24.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:48:23 smithi026 ceph-mon[175151]: Reconfiguring osd.4 (monmap changed)... 2023-09-23T20:48:24.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:48:23 smithi026 ceph-mon[175151]: Reconfiguring daemon osd.4 on smithi070 2023-09-23T20:48:24.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:48:23 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:48:24.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:48:23 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:48:24.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:48:23 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2023-09-23T20:48:24.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:48:23 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:48:24.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:48:23 smithi026 ceph-mon[179688]: pgmap v96: 73 pgs: 73 active+clean; 582 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:48:24.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:48:23 smithi026 ceph-mon[179688]: Reconfiguring osd.4 (monmap changed)... 2023-09-23T20:48:24.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:48:23 smithi026 ceph-mon[179688]: Reconfiguring daemon osd.4 on smithi070 2023-09-23T20:48:24.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:48:23 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:48:24.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:48:23 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:48:24.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:48:23 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2023-09-23T20:48:24.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:48:23 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:48:25.103 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:24 smithi070 ceph-mon[168282]: Reconfiguring osd.5 (monmap changed)... 2023-09-23T20:48:25.103 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:24 smithi070 ceph-mon[168282]: Reconfiguring daemon osd.5 on smithi070 2023-09-23T20:48:25.103 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:24 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:48:25.103 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:24 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:48:25.104 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:24 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2023-09-23T20:48:25.104 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:24 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:48:25.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:48:24 smithi026 ceph-mon[175151]: Reconfiguring osd.5 (monmap changed)... 2023-09-23T20:48:25.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:48:24 smithi026 ceph-mon[175151]: Reconfiguring daemon osd.5 on smithi070 2023-09-23T20:48:25.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:48:24 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:48:25.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:48:24 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:48:25.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:48:24 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2023-09-23T20:48:25.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:48:24 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:48:25.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:48:24 smithi026 ceph-mon[179688]: Reconfiguring osd.5 (monmap changed)... 2023-09-23T20:48:25.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:48:24 smithi026 ceph-mon[179688]: Reconfiguring daemon osd.5 on smithi070 2023-09-23T20:48:25.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:48:24 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:48:25.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:48:24 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:48:25.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:48:24 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2023-09-23T20:48:25.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:48:24 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:48:25.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:25 smithi070 ceph-mon[168282]: pgmap v97: 73 pgs: 73 active+clean; 582 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:48:25.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:25 smithi070 ceph-mon[168282]: Reconfiguring osd.6 (monmap changed)... 2023-09-23T20:48:25.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:25 smithi070 ceph-mon[168282]: Reconfiguring daemon osd.6 on smithi070 2023-09-23T20:48:26.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:48:25 smithi026 ceph-mon[175151]: pgmap v97: 73 pgs: 73 active+clean; 582 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:48:26.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:48:25 smithi026 ceph-mon[175151]: Reconfiguring osd.6 (monmap changed)... 2023-09-23T20:48:26.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:48:25 smithi026 ceph-mon[175151]: Reconfiguring daemon osd.6 on smithi070 2023-09-23T20:48:26.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:48:25 smithi026 ceph-mon[179688]: pgmap v97: 73 pgs: 73 active+clean; 582 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:48:26.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:48:25 smithi026 ceph-mon[179688]: Reconfiguring osd.6 (monmap changed)... 2023-09-23T20:48:26.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:48:25 smithi026 ceph-mon[179688]: Reconfiguring daemon osd.6 on smithi070 2023-09-23T20:48:27.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:26 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:48:27.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:26 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:48:27.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:26 smithi070 ceph-mon[168282]: Reconfiguring osd.7 (monmap changed)... 2023-09-23T20:48:27.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:26 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2023-09-23T20:48:27.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:26 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:48:27.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:26 smithi070 ceph-mon[168282]: Reconfiguring daemon osd.7 on smithi070 2023-09-23T20:48:27.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:26 smithi070 ceph-mon[168282]: pgmap v98: 73 pgs: 73 active+clean; 582 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:48:27.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:48:26 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:48:27.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:48:26 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:48:27.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:48:26 smithi026 ceph-mon[179688]: Reconfiguring osd.7 (monmap changed)... 2023-09-23T20:48:27.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:48:26 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2023-09-23T20:48:27.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:48:26 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:48:27.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:48:26 smithi026 ceph-mon[179688]: Reconfiguring daemon osd.7 on smithi070 2023-09-23T20:48:27.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:48:26 smithi026 ceph-mon[179688]: pgmap v98: 73 pgs: 73 active+clean; 582 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:48:27.383 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:48:26 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:48:27.383 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:48:26 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:48:27.383 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:48:26 smithi026 ceph-mon[175151]: Reconfiguring osd.7 (monmap changed)... 2023-09-23T20:48:27.383 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:48:26 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2023-09-23T20:48:27.383 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:48:26 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:48:27.383 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:48:26 smithi026 ceph-mon[175151]: Reconfiguring daemon osd.7 on smithi070 2023-09-23T20:48:27.383 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:48:26 smithi026 ceph-mon[175151]: pgmap v98: 73 pgs: 73 active+clean; 582 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:48:28.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:28 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:48:28.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:28 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:48:28.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:28 smithi070 ceph-mon[168282]: Reconfiguring mds.foo.smithi070.imqhqy (monmap changed)... 2023-09-23T20:48:28.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:28 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi070.imqhqy", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-09-23T20:48:28.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:28 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:48:28.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:28 smithi070 ceph-mon[168282]: Reconfiguring daemon mds.foo.smithi070.imqhqy on smithi070 2023-09-23T20:48:28.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:48:28 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:48:28.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:48:28 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:48:28.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:48:28 smithi026 ceph-mon[175151]: Reconfiguring mds.foo.smithi070.imqhqy (monmap changed)... 2023-09-23T20:48:28.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:48:28 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi070.imqhqy", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-09-23T20:48:28.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:48:28 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:48:28.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:48:28 smithi026 ceph-mon[175151]: Reconfiguring daemon mds.foo.smithi070.imqhqy on smithi070 2023-09-23T20:48:28.633 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:48:28 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:48:28.633 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:48:28 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:48:28.633 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:48:28 smithi026 ceph-mon[179688]: Reconfiguring mds.foo.smithi070.imqhqy (monmap changed)... 2023-09-23T20:48:28.633 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:48:28 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi070.imqhqy", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-09-23T20:48:28.633 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:48:28 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:48:28.633 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:48:28 smithi026 ceph-mon[179688]: Reconfiguring daemon mds.foo.smithi070.imqhqy on smithi070 2023-09-23T20:48:29.119 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 20:48:28 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-prometheus-a[155975]: ts=2023-09-23T20:48:28.660Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1123-g61431f6b (61431f6ba2752649b506867ec856b4a6236f90dd) quincy (stable)\", cluster_addr=\"172.21.15.26\", device_class=\"ssd\", hostname=\"smithi026\", instance=\"smithi070.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.26\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1123-g61431f6b (61431f6ba2752649b506867ec856b4a6236f90dd) quincy (stable)\", cluster_addr=\"172.21.15.26\", device_class=\"ssd\", hostname=\"smithi026\", instance=\"smithi026:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.26\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-09-23T20:48:29.632 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:48:29 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[164029]: ::ffff:172.21.15.70 - - [23/Sep/2023:20:48:29] "GET /metrics HTTP/1.1" 200 33696 "" "Prometheus/2.43.0" 2023-09-23T20:48:29.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:48:29 smithi026 ceph-mon[175151]: pgmap v99: 73 pgs: 73 active+clean; 582 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:48:29.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:48:29 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:48:29.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:48:29 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:48:29.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:48:29 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:48:29.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:48:29 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:48:29.633 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:48:29 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:48:29.633 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:48:29 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:48:29.633 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:48:29 smithi026 ceph-mon[179688]: pgmap v99: 73 pgs: 73 active+clean; 582 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:48:29.633 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:48:29 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:48:29.633 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:48:29 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:48:29.633 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:48:29 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:48:29.633 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:48:29 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:48:29.633 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:48:29 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:48:29.633 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:48:29 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:48:29.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:29 smithi070 ceph-mon[168282]: pgmap v99: 73 pgs: 73 active+clean; 582 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:48:29.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:29 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:48:29.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:29 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:48:29.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:29 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:48:29.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:29 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:48:29.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:29 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:48:29.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:29 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:48:32.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:31 smithi070 ceph-mon[168282]: pgmap v100: 73 pgs: 73 active+clean; 582 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:48:32.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:31 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:48:32.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:48:31 smithi026 ceph-mon[179688]: pgmap v100: 73 pgs: 73 active+clean; 582 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:48:32.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:48:31 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:48:32.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:48:31 smithi026 ceph-mon[175151]: pgmap v100: 73 pgs: 73 active+clean; 582 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:48:32.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:48:31 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:48:34.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:33 smithi070 ceph-mon[168282]: pgmap v101: 73 pgs: 73 active+clean; 582 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:48:34.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:48:33 smithi026 ceph-mon[175151]: pgmap v101: 73 pgs: 73 active+clean; 582 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:48:34.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:48:33 smithi026 ceph-mon[179688]: pgmap v101: 73 pgs: 73 active+clean; 582 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:48:36.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:35 smithi070 ceph-mon[168282]: pgmap v102: 73 pgs: 73 active+clean; 582 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:48:36.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:48:35 smithi026 ceph-mon[175151]: pgmap v102: 73 pgs: 73 active+clean; 582 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:48:36.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:48:35 smithi026 ceph-mon[179688]: pgmap v102: 73 pgs: 73 active+clean; 582 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:48:38.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:37 smithi070 ceph-mon[168282]: pgmap v103: 73 pgs: 73 active+clean; 582 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:48:38.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:48:37 smithi026 ceph-mon[175151]: pgmap v103: 73 pgs: 73 active+clean; 582 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:48:38.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:48:37 smithi026 ceph-mon[179688]: pgmap v103: 73 pgs: 73 active+clean; 582 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:48:39.120 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 20:48:38 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-prometheus-a[155975]: ts=2023-09-23T20:48:38.661Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1123-g61431f6b (61431f6ba2752649b506867ec856b4a6236f90dd) quincy (stable)\", cluster_addr=\"172.21.15.26\", device_class=\"ssd\", hostname=\"smithi026\", instance=\"smithi070.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.26\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1123-g61431f6b (61431f6ba2752649b506867ec856b4a6236f90dd) quincy (stable)\", cluster_addr=\"172.21.15.26\", device_class=\"ssd\", hostname=\"smithi026\", instance=\"smithi026:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.26\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-09-23T20:48:39.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:48:39 smithi026 ceph-mon[175151]: pgmap v104: 73 pgs: 73 active+clean; 582 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:48:39.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:48:39 smithi026 ceph-mon[179688]: pgmap v104: 73 pgs: 73 active+clean; 582 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:48:39.882 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:48:39 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[164029]: ::ffff:172.21.15.70 - - [23/Sep/2023:20:48:39] "GET /metrics HTTP/1.1" 200 33694 "" "Prometheus/2.43.0" 2023-09-23T20:48:40.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:39 smithi070 ceph-mon[168282]: pgmap v104: 73 pgs: 73 active+clean; 582 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:48:42.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:41 smithi070 ceph-mon[168282]: pgmap v105: 73 pgs: 73 active+clean; 582 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:48:42.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:48:41 smithi026 ceph-mon[175151]: pgmap v105: 73 pgs: 73 active+clean; 582 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:48:42.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:48:41 smithi026 ceph-mon[179688]: pgmap v105: 73 pgs: 73 active+clean; 582 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:48:44.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:43 smithi070 ceph-mon[168282]: pgmap v106: 73 pgs: 73 active+clean; 582 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:48:44.131 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:48:43 smithi026 ceph-mon[175151]: pgmap v106: 73 pgs: 73 active+clean; 582 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:48:44.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:48:43 smithi026 ceph-mon[179688]: pgmap v106: 73 pgs: 73 active+clean; 582 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:48:46.121 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:45 smithi070 ceph-mon[168282]: pgmap v107: 73 pgs: 73 active+clean; 582 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:48:46.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:48:45 smithi026 ceph-mon[175151]: pgmap v107: 73 pgs: 73 active+clean; 582 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:48:46.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:48:45 smithi026 ceph-mon[179688]: pgmap v107: 73 pgs: 73 active+clean; 582 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:48:47.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:46 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:48:47.131 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:48:46 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:48:47.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:48:46 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:48:48.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:47 smithi070 ceph-mon[168282]: pgmap v108: 73 pgs: 73 active+clean; 582 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:48:48.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:48:47 smithi026 ceph-mon[175151]: pgmap v108: 73 pgs: 73 active+clean; 582 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:48:48.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:48:47 smithi026 ceph-mon[179688]: pgmap v108: 73 pgs: 73 active+clean; 582 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:48:49.119 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 20:48:48 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-prometheus-a[155975]: ts=2023-09-23T20:48:48.659Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1123-g61431f6b (61431f6ba2752649b506867ec856b4a6236f90dd) quincy (stable)\", cluster_addr=\"172.21.15.26\", device_class=\"ssd\", hostname=\"smithi026\", instance=\"smithi070.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.26\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1123-g61431f6b (61431f6ba2752649b506867ec856b4a6236f90dd) quincy (stable)\", cluster_addr=\"172.21.15.26\", device_class=\"ssd\", hostname=\"smithi026\", instance=\"smithi026:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.26\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-09-23T20:48:49.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:48:49 smithi026 ceph-mon[175151]: pgmap v109: 73 pgs: 73 active+clean; 582 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:48:49.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:48:49 smithi026 ceph-mon[179688]: pgmap v109: 73 pgs: 73 active+clean; 582 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:48:49.883 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:48:49 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[164029]: ::ffff:172.21.15.70 - - [23/Sep/2023:20:48:49] "GET /metrics HTTP/1.1" 200 33697 "" "Prometheus/2.43.0" 2023-09-23T20:48:50.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:49 smithi070 ceph-mon[168282]: pgmap v109: 73 pgs: 73 active+clean; 582 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:48:52.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:51 smithi070 ceph-mon[168282]: pgmap v110: 73 pgs: 73 active+clean; 582 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:48:52.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:48:51 smithi026 ceph-mon[175151]: pgmap v110: 73 pgs: 73 active+clean; 582 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:48:52.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:48:51 smithi026 ceph-mon[179688]: pgmap v110: 73 pgs: 73 active+clean; 582 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:48:54.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:53 smithi070 ceph-mon[168282]: pgmap v111: 73 pgs: 73 active+clean; 582 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:48:54.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:48:53 smithi026 ceph-mon[175151]: pgmap v111: 73 pgs: 73 active+clean; 582 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:48:54.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:48:53 smithi026 ceph-mon[179688]: pgmap v111: 73 pgs: 73 active+clean; 582 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:48:56.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:55 smithi070 ceph-mon[168282]: pgmap v112: 73 pgs: 73 active+clean; 582 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:48:56.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:48:55 smithi026 ceph-mon[175151]: pgmap v112: 73 pgs: 73 active+clean; 582 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:48:56.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:48:55 smithi026 ceph-mon[179688]: pgmap v112: 73 pgs: 73 active+clean; 582 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:48:58.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:57 smithi070 ceph-mon[168282]: pgmap v113: 73 pgs: 73 active+clean; 582 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:48:58.131 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:48:57 smithi026 ceph-mon[175151]: pgmap v113: 73 pgs: 73 active+clean; 582 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:48:58.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:48:57 smithi026 ceph-mon[179688]: pgmap v113: 73 pgs: 73 active+clean; 582 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:48:59.119 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 20:48:58 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-prometheus-a[155975]: ts=2023-09-23T20:48:58.660Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1123-g61431f6b (61431f6ba2752649b506867ec856b4a6236f90dd) quincy (stable)\", cluster_addr=\"172.21.15.26\", device_class=\"ssd\", hostname=\"smithi026\", instance=\"smithi070.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.26\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1123-g61431f6b (61431f6ba2752649b506867ec856b4a6236f90dd) quincy (stable)\", cluster_addr=\"172.21.15.26\", device_class=\"ssd\", hostname=\"smithi026\", instance=\"smithi026:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.26\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-09-23T20:48:59.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:48:59 smithi026 ceph-mon[175151]: pgmap v114: 73 pgs: 73 active+clean; 582 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:48:59.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:48:59 smithi026 ceph-mon[179688]: pgmap v114: 73 pgs: 73 active+clean; 582 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:48:59.882 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:48:59 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[164029]: ::ffff:172.21.15.70 - - [23/Sep/2023:20:48:59] "GET /metrics HTTP/1.1" 200 33697 "" "Prometheus/2.43.0" 2023-09-23T20:49:00.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:48:59 smithi070 ceph-mon[168282]: pgmap v114: 73 pgs: 73 active+clean; 582 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:49:02.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:49:01 smithi070 ceph-mon[168282]: pgmap v115: 73 pgs: 73 active+clean; 582 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:49:02.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:49:01 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:49:02.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:49:01 smithi026 ceph-mon[175151]: pgmap v115: 73 pgs: 73 active+clean; 582 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:49:02.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:49:01 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:49:02.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:49:01 smithi026 ceph-mon[179688]: pgmap v115: 73 pgs: 73 active+clean; 582 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:49:02.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:49:01 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:49:04.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:49:03 smithi070 ceph-mon[168282]: pgmap v116: 73 pgs: 73 active+clean; 582 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:49:04.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:49:03 smithi026 ceph-mon[175151]: pgmap v116: 73 pgs: 73 active+clean; 582 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:49:04.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:49:03 smithi026 ceph-mon[179688]: pgmap v116: 73 pgs: 73 active+clean; 582 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:49:06.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:49:05 smithi070 ceph-mon[168282]: pgmap v117: 73 pgs: 73 active+clean; 582 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:49:06.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:49:05 smithi026 ceph-mon[179688]: pgmap v117: 73 pgs: 73 active+clean; 582 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:49:06.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:49:05 smithi026 ceph-mon[175151]: pgmap v117: 73 pgs: 73 active+clean; 582 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:49:08.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:49:07 smithi070 ceph-mon[168282]: pgmap v118: 73 pgs: 73 active+clean; 582 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:49:08.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:49:07 smithi026 ceph-mon[175151]: pgmap v118: 73 pgs: 73 active+clean; 582 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:49:08.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:49:07 smithi026 ceph-mon[179688]: pgmap v118: 73 pgs: 73 active+clean; 582 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:49:09.119 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 20:49:08 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-prometheus-a[155975]: ts=2023-09-23T20:49:08.660Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1123-g61431f6b (61431f6ba2752649b506867ec856b4a6236f90dd) quincy (stable)\", cluster_addr=\"172.21.15.26\", device_class=\"ssd\", hostname=\"smithi026\", instance=\"smithi070.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.26\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1123-g61431f6b (61431f6ba2752649b506867ec856b4a6236f90dd) quincy (stable)\", cluster_addr=\"172.21.15.26\", device_class=\"ssd\", hostname=\"smithi026\", instance=\"smithi026:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.26\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-09-23T20:49:09.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:49:09 smithi026 ceph-mon[175151]: pgmap v119: 73 pgs: 73 active+clean; 582 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:49:09.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:49:09 smithi026 ceph-mon[179688]: pgmap v119: 73 pgs: 73 active+clean; 582 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:49:09.882 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:49:09 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[164029]: ::ffff:172.21.15.70 - - [23/Sep/2023:20:49:09] "GET /metrics HTTP/1.1" 200 33692 "" "Prometheus/2.43.0" 2023-09-23T20:49:10.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:49:09 smithi070 ceph-mon[168282]: pgmap v119: 73 pgs: 73 active+clean; 582 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:49:10.261 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -e sha1=bb0daeff97f020f686596fca6eba20c310e26fb7 -- bash -c 'ceph orch upgrade resume' 2023-09-23T20:49:10.882 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:49:11.796 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:49:11 smithi026 ceph-mon[179688]: pgmap v120: 73 pgs: 73 active+clean; 582 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:49:11.796 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:49:11 smithi026 ceph-mon[175151]: pgmap v120: 73 pgs: 73 active+clean; 582 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:49:12.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:49:11 smithi070 ceph-mon[168282]: pgmap v120: 73 pgs: 73 active+clean; 582 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:49:12.313 INFO:teuthology.orchestra.run.smithi026.stdout:Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d 2023-09-23T20:49:12.728 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -e sha1=bb0daeff97f020f686596fca6eba20c310e26fb7 -- bash -c 'echo wait for half of osds to upgrade' 2023-09-23T20:49:13.341 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:49:13.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:49:13 smithi070 ceph-mon[168282]: pgmap v121: 73 pgs: 73 active+clean; 582 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:49:13.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:49:13 smithi070 ceph-mon[168282]: from='client.44144 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T20:49:13.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:49:13 smithi070 ceph-mon[168282]: Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d 2023-09-23T20:49:13.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:49:13 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:49:13.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:49:13 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:49:13.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:49:13 smithi026 ceph-mon[175151]: pgmap v121: 73 pgs: 73 active+clean; 582 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:49:13.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:49:13 smithi026 ceph-mon[175151]: from='client.44144 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T20:49:13.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:49:13 smithi026 ceph-mon[175151]: Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d 2023-09-23T20:49:13.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:49:13 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:49:13.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:49:13 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:49:13.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:49:13 smithi026 ceph-mon[179688]: pgmap v121: 73 pgs: 73 active+clean; 582 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:49:13.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:49:13 smithi026 ceph-mon[179688]: from='client.44144 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T20:49:13.633 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:49:13 smithi026 ceph-mon[179688]: Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d 2023-09-23T20:49:13.633 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:49:13 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:49:13.633 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:49:13 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:49:14.877 INFO:teuthology.orchestra.run.smithi026.stdout:wait for half of osds to upgrade 2023-09-23T20:49:15.493 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -e sha1=bb0daeff97f020f686596fca6eba20c310e26fb7 -- bash -c 'while ! ceph osd versions | grep $sha1 | egrep '"'"': [45678]'"'"'; do sleep 2 ; done' 2023-09-23T20:49:15.747 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:49:15 smithi026 ceph-mon[179688]: pgmap v122: 73 pgs: 73 active+clean; 582 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:49:15.748 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:49:15 smithi026 ceph-mon[175151]: pgmap v122: 73 pgs: 73 active+clean; 582 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:49:16.110 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:49:16.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:49:15 smithi070 ceph-mon[168282]: pgmap v122: 73 pgs: 73 active+clean; 582 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:49:17.006 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:49:16 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:49:17.007 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:49:16 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:49:17.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:49:16 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:49:18.121 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:49:17 smithi070 ceph-mon[168282]: pgmap v123: 73 pgs: 73 active+clean; 582 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:49:18.122 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:49:17 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:49:18.122 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:49:17 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:49:18.122 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:49:17 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/1298307185' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:49:18.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:49:17 smithi026 ceph-mon[179688]: pgmap v123: 73 pgs: 73 active+clean; 582 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:49:18.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:49:17 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:49:18.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:49:17 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:49:18.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:49:17 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/1298307185' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:49:18.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:49:17 smithi026 ceph-mon[175151]: pgmap v123: 73 pgs: 73 active+clean; 582 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:49:18.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:49:17 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:49:18.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:49:17 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:49:18.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:49:17 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/1298307185' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:49:19.292 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:49:18 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:49:19.292 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:49:18 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:49:19.292 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:49:18 smithi026 ceph-mon[175151]: pgmap v124: 73 pgs: 73 active+clean; 582 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:49:19.292 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:49:18 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:49:19.292 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:49:18 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:49:19.292 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:49:18 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:49:19.293 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:49:18 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:49:19.293 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:49:18 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T20:49:19.293 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:49:18 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T20:49:19.293 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:49:18 smithi026 ceph-mon[175151]: Upgrade: Setting container_image for all mon 2023-09-23T20:49:19.293 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:49:18 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:49:19.293 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:49:18 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2023-09-23T20:49:19.293 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:49:18 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]': finished 2023-09-23T20:49:19.293 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:49:18 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2023-09-23T20:49:19.293 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:49:18 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]': finished 2023-09-23T20:49:19.293 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:49:18 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2023-09-23T20:49:19.293 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:49:18 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]': finished 2023-09-23T20:49:19.294 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:49:18 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T20:49:19.294 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:49:18 smithi026 ceph-mon[175151]: Upgrade: Setting container_image for all crash 2023-09-23T20:49:19.294 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:49:18 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:49:19.294 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:49:18 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2023-09-23T20:49:19.294 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:49:18 smithi026 ceph-mon[175151]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2023-09-23T20:49:19.294 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:49:18 smithi026 ceph-mon[175151]: Upgrade: osd.0 is safe to restart 2023-09-23T20:49:19.294 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:49:18 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:49:19.294 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:49:18 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:49:19.295 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:49:18 smithi026 ceph-mon[179688]: pgmap v124: 73 pgs: 73 active+clean; 582 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:49:19.295 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:49:18 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:49:19.295 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:49:18 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:49:19.295 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:49:18 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:49:19.295 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:49:18 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:49:19.295 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:49:18 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T20:49:19.295 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:49:18 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T20:49:19.295 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:49:18 smithi026 ceph-mon[179688]: Upgrade: Setting container_image for all mon 2023-09-23T20:49:19.295 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:49:18 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:49:19.295 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:49:18 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2023-09-23T20:49:19.296 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:49:18 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]': finished 2023-09-23T20:49:19.296 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:49:18 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2023-09-23T20:49:19.296 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:49:18 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]': finished 2023-09-23T20:49:19.296 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:49:18 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2023-09-23T20:49:19.296 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:49:18 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]': finished 2023-09-23T20:49:19.296 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:49:18 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T20:49:19.296 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:49:18 smithi026 ceph-mon[179688]: Upgrade: Setting container_image for all crash 2023-09-23T20:49:19.296 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:49:18 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:49:19.296 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:49:18 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2023-09-23T20:49:19.296 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:49:18 smithi026 ceph-mon[179688]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2023-09-23T20:49:19.296 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:49:18 smithi026 ceph-mon[179688]: Upgrade: osd.0 is safe to restart 2023-09-23T20:49:19.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:49:18 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:49:19.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:49:18 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:49:19.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:49:18 smithi070 ceph-mon[168282]: pgmap v124: 73 pgs: 73 active+clean; 582 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:49:19.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:49:18 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:49:19.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:49:18 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:49:19.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:49:18 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:49:19.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:49:18 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:49:19.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:49:18 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T20:49:19.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:49:18 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T20:49:19.371 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:49:18 smithi070 ceph-mon[168282]: Upgrade: Setting container_image for all mon 2023-09-23T20:49:19.371 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:49:18 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:49:19.371 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:49:18 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2023-09-23T20:49:19.371 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:49:18 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]': finished 2023-09-23T20:49:19.371 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:49:18 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2023-09-23T20:49:19.371 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:49:18 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]': finished 2023-09-23T20:49:19.371 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:49:18 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2023-09-23T20:49:19.371 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:49:18 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]': finished 2023-09-23T20:49:19.371 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:49:18 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T20:49:19.371 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:49:18 smithi070 ceph-mon[168282]: Upgrade: Setting container_image for all crash 2023-09-23T20:49:19.371 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:49:18 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:49:19.372 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:49:18 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2023-09-23T20:49:19.372 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:49:18 smithi070 ceph-mon[168282]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2023-09-23T20:49:19.372 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:49:18 smithi070 ceph-mon[168282]: Upgrade: osd.0 is safe to restart 2023-09-23T20:49:19.881 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:49:19 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[164029]: ::ffff:172.21.15.70 - - [23/Sep/2023:20:49:19] "GET /metrics HTTP/1.1" 200 33697 "" "Prometheus/2.43.0" 2023-09-23T20:49:21.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:49:20 smithi026 ceph-mon[175151]: Upgrade: Updating osd.0 2023-09-23T20:49:21.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:49:20 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:49:21.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:49:20 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2023-09-23T20:49:21.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:49:20 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:49:21.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:49:20 smithi026 ceph-mon[175151]: Deploying daemon osd.0 on smithi026 2023-09-23T20:49:21.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:49:20 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/2173944379' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:49:21.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:49:20 smithi026 ceph-mon[175151]: pgmap v125: 73 pgs: 73 active+clean; 582 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:49:21.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:49:20 smithi026 ceph-mon[179688]: Upgrade: Updating osd.0 2023-09-23T20:49:21.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:49:20 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:49:21.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:49:20 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2023-09-23T20:49:21.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:49:20 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:49:21.134 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:49:20 smithi026 ceph-mon[179688]: Deploying daemon osd.0 on smithi026 2023-09-23T20:49:21.134 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:49:20 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/2173944379' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:49:21.134 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:49:20 smithi026 ceph-mon[179688]: pgmap v125: 73 pgs: 73 active+clean; 582 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:49:21.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:49:20 smithi070 ceph-mon[168282]: Upgrade: Updating osd.0 2023-09-23T20:49:21.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:49:20 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:49:21.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:49:20 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2023-09-23T20:49:21.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:49:20 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:49:21.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:49:20 smithi070 ceph-mon[168282]: Deploying daemon osd.0 on smithi026 2023-09-23T20:49:21.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:49:20 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/2173944379' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:49:21.371 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:49:20 smithi070 ceph-mon[168282]: pgmap v125: 73 pgs: 73 active+clean; 582 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:49:22.882 INFO:journalctl@ceph.osd.0.smithi026.stdout:Sep 23 20:49:22 smithi026 systemd[1]: Stopping Ceph osd.0 for b3f8e4ec-5a50-11ee-8db4-212e2dc638e7... 2023-09-23T20:49:23.181 INFO:journalctl@ceph.osd.0.smithi026.stdout:Sep 23 20:49:22 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd-0[114366]: 2023-09-23T20:49:22.910+0000 7f1868937700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-osd -n osd.0 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2023-09-23T20:49:23.182 INFO:journalctl@ceph.osd.0.smithi026.stdout:Sep 23 20:49:22 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd-0[114366]: 2023-09-23T20:49:22.910+0000 7f1868937700 -1 osd.0 79 *** Got signal Terminated *** 2023-09-23T20:49:23.182 INFO:journalctl@ceph.osd.0.smithi026.stdout:Sep 23 20:49:22 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd-0[114366]: 2023-09-23T20:49:22.910+0000 7f1868937700 -1 osd.0 79 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-09-23T20:49:23.182 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:49:23 smithi026 ceph-mon[175151]: pgmap v126: 73 pgs: 73 active+clean; 582 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:49:23.182 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:49:23 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/1104032870' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:49:23.182 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:49:23 smithi026 ceph-mon[175151]: osd.0 marked itself down and dead 2023-09-23T20:49:23.182 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:49:23 smithi026 ceph-mon[179688]: pgmap v126: 73 pgs: 73 active+clean; 582 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:49:23.183 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:49:23 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/1104032870' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:49:23.183 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:49:23 smithi026 ceph-mon[179688]: osd.0 marked itself down and dead 2023-09-23T20:49:23.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:49:23 smithi070 ceph-mon[168282]: pgmap v126: 73 pgs: 73 active+clean; 582 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:49:23.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:49:23 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/1104032870' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:49:23.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:49:23 smithi070 ceph-mon[168282]: osd.0 marked itself down and dead 2023-09-23T20:49:24.184 INFO:journalctl@ceph.osd.0.smithi026.stdout:Sep 23 20:49:23 smithi026 bash[189829]: ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd-0 2023-09-23T20:49:24.185 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:49:24 smithi026 ceph-mon[175151]: Health check failed: 1 osds down (OSD_DOWN) 2023-09-23T20:49:24.185 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:49:24 smithi026 ceph-mon[175151]: failed to encode map e80 with expected crc 2023-09-23T20:49:24.185 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:49:24 smithi026 ceph-mon[175151]: failed to encode map e80 with expected crc 2023-09-23T20:49:24.185 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:49:24 smithi026 ceph-mon[175151]: osdmap e80: 8 total, 7 up, 8 in 2023-09-23T20:49:24.186 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:49:24 smithi026 ceph-mon[179688]: Health check failed: 1 osds down (OSD_DOWN) 2023-09-23T20:49:24.186 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:49:24 smithi026 ceph-mon[179688]: failed to encode map e80 with expected crc 2023-09-23T20:49:24.186 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:49:24 smithi026 ceph-mon[179688]: failed to encode map e80 with expected crc 2023-09-23T20:49:24.186 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:49:24 smithi026 ceph-mon[179688]: osdmap e80: 8 total, 7 up, 8 in 2023-09-23T20:49:24.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:49:24 smithi070 ceph-mon[168282]: Health check failed: 1 osds down (OSD_DOWN) 2023-09-23T20:49:24.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:49:24 smithi070 ceph-mon[168282]: failed to encode map e80 with expected crc 2023-09-23T20:49:24.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:49:24 smithi070 ceph-mon[168282]: failed to encode map e80 with expected crc 2023-09-23T20:49:24.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:49:24 smithi070 ceph-mon[168282]: osdmap e80: 8 total, 7 up, 8 in 2023-09-23T20:49:25.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:49:25 smithi026 ceph-mon[175151]: pgmap v128: 73 pgs: 11 stale+active+clean, 62 active+clean; 582 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:49:25.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:49:25 smithi026 ceph-mon[175151]: failed to encode map e81 with expected crc 2023-09-23T20:49:25.383 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:49:25 smithi026 ceph-mon[175151]: failed to encode map e81 with expected crc 2023-09-23T20:49:25.383 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:49:25 smithi026 ceph-mon[175151]: osdmap e81: 8 total, 7 up, 8 in 2023-09-23T20:49:25.383 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:49:25 smithi026 ceph-mon[175151]: failed to encode map e81 with expected crc 2023-09-23T20:49:25.383 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:49:25 smithi026 ceph-mon[175151]: failed to encode map e81 with expected crc 2023-09-23T20:49:25.383 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:49:25 smithi026 ceph-mon[175151]: failed to encode map e81 with expected crc 2023-09-23T20:49:25.383 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:49:25 smithi026 ceph-mon[175151]: failed to encode map e81 with expected crc 2023-09-23T20:49:25.383 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:49:25 smithi026 ceph-mon[175151]: failed to encode map e81 with expected crc 2023-09-23T20:49:25.383 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:49:25 smithi026 ceph-mon[175151]: failed to encode map e81 with expected crc 2023-09-23T20:49:25.383 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:49:25 smithi026 ceph-mon[175151]: failed to encode map e81 with expected crc 2023-09-23T20:49:25.383 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:49:25 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/690425686' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:49:25.384 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:49:25 smithi026 ceph-mon[179688]: pgmap v128: 73 pgs: 11 stale+active+clean, 62 active+clean; 582 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:49:25.384 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:49:25 smithi026 ceph-mon[179688]: failed to encode map e81 with expected crc 2023-09-23T20:49:25.384 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:49:25 smithi026 ceph-mon[179688]: failed to encode map e81 with expected crc 2023-09-23T20:49:25.384 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:49:25 smithi026 ceph-mon[179688]: osdmap e81: 8 total, 7 up, 8 in 2023-09-23T20:49:25.384 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:49:25 smithi026 ceph-mon[179688]: failed to encode map e81 with expected crc 2023-09-23T20:49:25.384 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:49:25 smithi026 ceph-mon[179688]: failed to encode map e81 with expected crc 2023-09-23T20:49:25.384 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:49:25 smithi026 ceph-mon[179688]: failed to encode map e81 with expected crc 2023-09-23T20:49:25.384 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:49:25 smithi026 ceph-mon[179688]: failed to encode map e81 with expected crc 2023-09-23T20:49:25.384 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:49:25 smithi026 ceph-mon[179688]: failed to encode map e81 with expected crc 2023-09-23T20:49:25.384 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:49:25 smithi026 ceph-mon[179688]: failed to encode map e81 with expected crc 2023-09-23T20:49:25.384 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:49:25 smithi026 ceph-mon[179688]: failed to encode map e81 with expected crc 2023-09-23T20:49:25.385 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:49:25 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/690425686' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:49:25.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:49:25 smithi070 ceph-mon[168282]: pgmap v128: 73 pgs: 11 stale+active+clean, 62 active+clean; 582 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:49:25.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:49:25 smithi070 ceph-mon[168282]: failed to encode map e81 with expected crc 2023-09-23T20:49:25.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:49:25 smithi070 ceph-mon[168282]: failed to encode map e81 with expected crc 2023-09-23T20:49:25.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:49:25 smithi070 ceph-mon[168282]: osdmap e81: 8 total, 7 up, 8 in 2023-09-23T20:49:25.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:49:25 smithi070 ceph-mon[168282]: failed to encode map e81 with expected crc 2023-09-23T20:49:25.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:49:25 smithi070 ceph-mon[168282]: failed to encode map e81 with expected crc 2023-09-23T20:49:25.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:49:25 smithi070 ceph-mon[168282]: failed to encode map e81 with expected crc 2023-09-23T20:49:25.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:49:25 smithi070 ceph-mon[168282]: failed to encode map e81 with expected crc 2023-09-23T20:49:25.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:49:25 smithi070 ceph-mon[168282]: failed to encode map e81 with expected crc 2023-09-23T20:49:25.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:49:25 smithi070 ceph-mon[168282]: failed to encode map e81 with expected crc 2023-09-23T20:49:25.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:49:25 smithi070 ceph-mon[168282]: failed to encode map e81 with expected crc 2023-09-23T20:49:25.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:49:25 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/690425686' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:49:25.882 INFO:journalctl@ceph.osd.0.smithi026.stdout:Sep 23 20:49:25 smithi026 systemd[1]: ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7@osd.0.service: Succeeded. 2023-09-23T20:49:25.882 INFO:journalctl@ceph.osd.0.smithi026.stdout:Sep 23 20:49:25 smithi026 systemd[1]: Stopped Ceph osd.0 for b3f8e4ec-5a50-11ee-8db4-212e2dc638e7. 2023-09-23T20:49:25.883 INFO:journalctl@ceph.osd.0.smithi026.stdout:Sep 23 20:49:25 smithi026 systemd[1]: Starting Ceph osd.0 for b3f8e4ec-5a50-11ee-8db4-212e2dc638e7... 2023-09-23T20:49:26.882 INFO:journalctl@ceph.osd.0.smithi026.stdout:Sep 23 20:49:26 smithi026 bash[190136]: --> Failed to activate via raw: did not find any matching OSD to activate 2023-09-23T20:49:26.882 INFO:journalctl@ceph.osd.0.smithi026.stdout:Sep 23 20:49:26 smithi026 bash[190136]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2023-09-23T20:49:26.882 INFO:journalctl@ceph.osd.0.smithi026.stdout:Sep 23 20:49:26 smithi026 bash[190136]: Running command: /usr/bin/ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev /dev/vg_nvme/lv_4 --path /var/lib/ceph/osd/ceph-0 --no-mon-config 2023-09-23T20:49:26.882 INFO:journalctl@ceph.osd.0.smithi026.stdout:Sep 23 20:49:26 smithi026 bash[190136]: Running command: /usr/bin/ln -snf /dev/vg_nvme/lv_4 /var/lib/ceph/osd/ceph-0/block 2023-09-23T20:49:26.882 INFO:journalctl@ceph.osd.0.smithi026.stdout:Sep 23 20:49:26 smithi026 bash[190136]: Running command: /usr/bin/chown -h ceph:ceph /var/lib/ceph/osd/ceph-0/block 2023-09-23T20:49:26.882 INFO:journalctl@ceph.osd.0.smithi026.stdout:Sep 23 20:49:26 smithi026 bash[190136]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2023-09-23T20:49:26.883 INFO:journalctl@ceph.osd.0.smithi026.stdout:Sep 23 20:49:26 smithi026 bash[190136]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2023-09-23T20:49:26.883 INFO:journalctl@ceph.osd.0.smithi026.stdout:Sep 23 20:49:26 smithi026 bash[190136]: --> ceph-volume lvm activate successful for osd ID: 0 2023-09-23T20:49:27.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:49:27 smithi070 ceph-mon[168282]: pgmap v130: 73 pgs: 11 stale+active+clean, 62 active+clean; 582 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:49:27.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:49:27 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/1647862811' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:49:27.631 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:49:27 smithi026 ceph-mon[179688]: pgmap v130: 73 pgs: 11 stale+active+clean, 62 active+clean; 582 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:49:27.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:49:27 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/1647862811' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:49:27.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:49:27 smithi026 ceph-mon[175151]: pgmap v130: 73 pgs: 11 stale+active+clean, 62 active+clean; 582 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:49:27.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:49:27 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/1647862811' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:49:27.632 INFO:journalctl@ceph.osd.0.smithi026.stdout:Sep 23 20:49:27 smithi026 podman[190423]: 2023-09-23T20:49:27.632 INFO:journalctl@ceph.osd.0.smithi026.stdout:Sep 23 20:49:27 smithi026 ceph-osd[190463]: -- 172.21.15.26:0/3608190900 <== mon.2 v2:172.21.15.26:3301/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x5605e49124e0 con 0x5605e48e2c00 2023-09-23T20:49:27.633 INFO:journalctl@ceph.osd.0.smithi026.stdout:Sep 23 20:49:27 smithi026 bash[190423]: c815c65a946ddcbc71ec237505f88d56b76581cf3b2b66337880d2dbae7a1f7f 2023-09-23T20:49:27.633 INFO:journalctl@ceph.osd.0.smithi026.stdout:Sep 23 20:49:27 smithi026 systemd[1]: Started Ceph osd.0 for b3f8e4ec-5a50-11ee-8db4-212e2dc638e7. 2023-09-23T20:49:28.382 INFO:journalctl@ceph.osd.0.smithi026.stdout:Sep 23 20:49:27 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd-0[190439]: 2023-09-23T20:49:27.910+0000 7f27a8033640 -1 Falling back to public interface 2023-09-23T20:49:28.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:49:28 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:49:28.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:49:28 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:49:28.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:49:28 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:49:28.881 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:49:28 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:49:28.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:49:28 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:49:28.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:49:28 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:49:28.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:49:28 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:49:28.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:49:28 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:49:28.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:49:28 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:49:29.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:49:29 smithi070 ceph-mon[168282]: pgmap v131: 73 pgs: 7 active+undersized+degraded, 19 active+undersized, 47 active+clean; 582 KiB data, 88 MiB used, 715 GiB / 715 GiB avail; 12/72 objects degraded (16.667%) 2023-09-23T20:49:29.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:49:29 smithi070 ceph-mon[168282]: Health check failed: Degraded data redundancy: 12/72 objects degraded (16.667%), 7 pgs degraded (PG_DEGRADED) 2023-09-23T20:49:29.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:49:29 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/3111354927' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:49:29.881 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:49:29 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[164029]: ::ffff:172.21.15.70 - - [23/Sep/2023:20:49:29] "GET /metrics HTTP/1.1" 200 33697 "" "Prometheus/2.43.0" 2023-09-23T20:49:29.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:49:29 smithi026 ceph-mon[179688]: pgmap v131: 73 pgs: 7 active+undersized+degraded, 19 active+undersized, 47 active+clean; 582 KiB data, 88 MiB used, 715 GiB / 715 GiB avail; 12/72 objects degraded (16.667%) 2023-09-23T20:49:29.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:49:29 smithi026 ceph-mon[179688]: Health check failed: Degraded data redundancy: 12/72 objects degraded (16.667%), 7 pgs degraded (PG_DEGRADED) 2023-09-23T20:49:29.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:49:29 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/3111354927' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:49:29.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:49:29 smithi026 ceph-mon[175151]: pgmap v131: 73 pgs: 7 active+undersized+degraded, 19 active+undersized, 47 active+clean; 582 KiB data, 88 MiB used, 715 GiB / 715 GiB avail; 12/72 objects degraded (16.667%) 2023-09-23T20:49:29.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:49:29 smithi026 ceph-mon[175151]: Health check failed: Degraded data redundancy: 12/72 objects degraded (16.667%), 7 pgs degraded (PG_DEGRADED) 2023-09-23T20:49:29.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:49:29 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/3111354927' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:49:31.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:49:30 smithi070 ceph-mon[168282]: from='osd.0 [v2:172.21.15.26:6802/3105265146,v1:172.21.15.26:6803/3105265146]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2023-09-23T20:49:31.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:49:30 smithi070 ceph-mon[168282]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2023-09-23T20:49:31.131 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:49:30 smithi026 ceph-mon[179688]: from='osd.0 [v2:172.21.15.26:6802/3105265146,v1:172.21.15.26:6803/3105265146]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2023-09-23T20:49:31.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:49:30 smithi026 ceph-mon[179688]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2023-09-23T20:49:31.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:49:30 smithi026 ceph-mon[175151]: from='osd.0 [v2:172.21.15.26:6802/3105265146,v1:172.21.15.26:6803/3105265146]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2023-09-23T20:49:31.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:49:30 smithi026 ceph-mon[175151]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2023-09-23T20:49:31.132 INFO:journalctl@ceph.osd.0.smithi026.stdout:Sep 23 20:49:30 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd-0[190439]: 2023-09-23T20:49:30.645+0000 7f27a8033640 -1 osd.0 79 log_to_monitors true 2023-09-23T20:49:31.132 INFO:journalctl@ceph.osd.0.smithi026.stdout:Sep 23 20:49:30 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd-0[190439]: 2023-09-23T20:49:30.779+0000 7f2795498700 -1 osd.0 79 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-09-23T20:49:32.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:49:31 smithi070 ceph-mon[168282]: pgmap v132: 73 pgs: 7 active+undersized+degraded, 19 active+undersized, 47 active+clean; 582 KiB data, 88 MiB used, 715 GiB / 715 GiB avail; 12/72 objects degraded (16.667%) 2023-09-23T20:49:32.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:49:31 smithi070 ceph-mon[168282]: failed to encode map e82 with expected crc 2023-09-23T20:49:32.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:49:31 smithi070 ceph-mon[168282]: failed to encode map e82 with expected crc 2023-09-23T20:49:32.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:49:31 smithi070 ceph-mon[168282]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2023-09-23T20:49:32.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:49:31 smithi070 ceph-mon[168282]: osdmap e82: 8 total, 7 up, 8 in 2023-09-23T20:49:32.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:49:31 smithi070 ceph-mon[168282]: from='osd.0 [v2:172.21.15.26:6802/3105265146,v1:172.21.15.26:6803/3105265146]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi026", "root=default"]}]: dispatch 2023-09-23T20:49:32.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:49:31 smithi070 ceph-mon[168282]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi026", "root=default"]}]: dispatch 2023-09-23T20:49:32.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:49:31 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:49:32.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:49:31 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:49:32.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:49:31 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/2412929740' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:49:32.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:49:31 smithi026 ceph-mon[175151]: pgmap v132: 73 pgs: 7 active+undersized+degraded, 19 active+undersized, 47 active+clean; 582 KiB data, 88 MiB used, 715 GiB / 715 GiB avail; 12/72 objects degraded (16.667%) 2023-09-23T20:49:32.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:49:31 smithi026 ceph-mon[175151]: failed to encode map e82 with expected crc 2023-09-23T20:49:32.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:49:31 smithi026 ceph-mon[175151]: failed to encode map e82 with expected crc 2023-09-23T20:49:32.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:49:31 smithi026 ceph-mon[175151]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2023-09-23T20:49:32.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:49:31 smithi026 ceph-mon[175151]: osdmap e82: 8 total, 7 up, 8 in 2023-09-23T20:49:32.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:49:31 smithi026 ceph-mon[175151]: from='osd.0 [v2:172.21.15.26:6802/3105265146,v1:172.21.15.26:6803/3105265146]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi026", "root=default"]}]: dispatch 2023-09-23T20:49:32.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:49:31 smithi026 ceph-mon[175151]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi026", "root=default"]}]: dispatch 2023-09-23T20:49:32.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:49:31 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:49:32.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:49:31 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:49:32.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:49:31 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/2412929740' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:49:32.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:49:31 smithi026 ceph-mon[179688]: pgmap v132: 73 pgs: 7 active+undersized+degraded, 19 active+undersized, 47 active+clean; 582 KiB data, 88 MiB used, 715 GiB / 715 GiB avail; 12/72 objects degraded (16.667%) 2023-09-23T20:49:32.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:49:31 smithi026 ceph-mon[179688]: failed to encode map e82 with expected crc 2023-09-23T20:49:32.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:49:31 smithi026 ceph-mon[179688]: failed to encode map e82 with expected crc 2023-09-23T20:49:32.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:49:31 smithi026 ceph-mon[179688]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2023-09-23T20:49:32.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:49:31 smithi026 ceph-mon[179688]: osdmap e82: 8 total, 7 up, 8 in 2023-09-23T20:49:32.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:49:31 smithi026 ceph-mon[179688]: from='osd.0 [v2:172.21.15.26:6802/3105265146,v1:172.21.15.26:6803/3105265146]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi026", "root=default"]}]: dispatch 2023-09-23T20:49:32.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:49:31 smithi026 ceph-mon[179688]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi026", "root=default"]}]: dispatch 2023-09-23T20:49:32.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:49:31 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:49:32.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:49:31 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:49:32.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:49:31 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/2412929740' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:49:33.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:49:32 smithi070 ceph-mon[168282]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-09-23T20:49:33.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:49:32 smithi070 ceph-mon[168282]: failed to encode map e83 with expected crc 2023-09-23T20:49:33.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:49:32 smithi070 ceph-mon[168282]: osd.0 [v2:172.21.15.26:6802/3105265146,v1:172.21.15.26:6803/3105265146] boot 2023-09-23T20:49:33.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:49:32 smithi070 ceph-mon[168282]: osdmap e83: 8 total, 8 up, 8 in 2023-09-23T20:49:33.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:49:32 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-23T20:49:33.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:49:32 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:49:33.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:49:32 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:49:33.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:49:32 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:49:33.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:49:32 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:49:33.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:49:32 smithi070 ceph-mon[168282]: osdmap e84: 8 total, 8 up, 8 in 2023-09-23T20:49:33.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:49:32 smithi026 ceph-mon[179688]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-09-23T20:49:33.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:49:32 smithi026 ceph-mon[179688]: failed to encode map e83 with expected crc 2023-09-23T20:49:33.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:49:32 smithi026 ceph-mon[179688]: osd.0 [v2:172.21.15.26:6802/3105265146,v1:172.21.15.26:6803/3105265146] boot 2023-09-23T20:49:33.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:49:32 smithi026 ceph-mon[179688]: osdmap e83: 8 total, 8 up, 8 in 2023-09-23T20:49:33.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:49:32 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-23T20:49:33.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:49:32 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:49:33.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:49:32 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:49:33.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:49:32 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:49:33.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:49:32 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:49:33.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:49:32 smithi026 ceph-mon[179688]: osdmap e84: 8 total, 8 up, 8 in 2023-09-23T20:49:33.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:49:32 smithi026 ceph-mon[175151]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-09-23T20:49:33.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:49:32 smithi026 ceph-mon[175151]: failed to encode map e83 with expected crc 2023-09-23T20:49:33.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:49:32 smithi026 ceph-mon[175151]: osd.0 [v2:172.21.15.26:6802/3105265146,v1:172.21.15.26:6803/3105265146] boot 2023-09-23T20:49:33.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:49:32 smithi026 ceph-mon[175151]: osdmap e83: 8 total, 8 up, 8 in 2023-09-23T20:49:33.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:49:32 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-23T20:49:33.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:49:32 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:49:33.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:49:32 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:49:33.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:49:32 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:49:33.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:49:32 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:49:33.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:49:32 smithi026 ceph-mon[175151]: osdmap e84: 8 total, 8 up, 8 in 2023-09-23T20:49:33.785 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:49:33 smithi026 ceph-mon[179688]: pgmap v135: 73 pgs: 15 peering, 2 active+undersized+degraded, 9 active+undersized, 47 active+clean; 582 KiB data, 108 MiB used, 715 GiB / 715 GiB avail; 3/72 objects degraded (4.167%) 2023-09-23T20:49:33.785 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:49:33 smithi026 ceph-mon[179688]: failed to encode map e84 with expected crc 2023-09-23T20:49:33.785 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:49:33 smithi026 ceph-mon[179688]: failed to encode map e84 with expected crc 2023-09-23T20:49:33.785 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:49:33 smithi026 ceph-mon[179688]: failed to encode map e84 with expected crc 2023-09-23T20:49:33.785 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:49:33 smithi026 ceph-mon[179688]: failed to encode map e84 with expected crc 2023-09-23T20:49:33.785 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:49:33 smithi026 ceph-mon[179688]: failed to encode map e84 with expected crc 2023-09-23T20:49:33.786 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:49:33 smithi026 ceph-mon[179688]: failed to encode map e84 with expected crc 2023-09-23T20:49:34.121 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:49:33 smithi070 ceph-mon[168282]: pgmap v135: 73 pgs: 15 peering, 2 active+undersized+degraded, 9 active+undersized, 47 active+clean; 582 KiB data, 108 MiB used, 715 GiB / 715 GiB avail; 3/72 objects degraded (4.167%) 2023-09-23T20:49:34.121 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:49:33 smithi070 ceph-mon[168282]: failed to encode map e84 with expected crc 2023-09-23T20:49:34.121 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:49:33 smithi070 ceph-mon[168282]: failed to encode map e84 with expected crc 2023-09-23T20:49:34.121 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:49:33 smithi070 ceph-mon[168282]: failed to encode map e84 with expected crc 2023-09-23T20:49:34.121 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:49:33 smithi070 ceph-mon[168282]: failed to encode map e84 with expected crc 2023-09-23T20:49:34.121 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:49:33 smithi070 ceph-mon[168282]: failed to encode map e84 with expected crc 2023-09-23T20:49:34.121 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:49:33 smithi070 ceph-mon[168282]: failed to encode map e84 with expected crc 2023-09-23T20:49:34.122 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:49:33 smithi070 ceph-mon[168282]: failed to encode map e84 with expected crc 2023-09-23T20:49:34.122 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:49:33 smithi070 ceph-mon[168282]: failed to encode map e84 with expected crc 2023-09-23T20:49:34.131 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:49:33 smithi026 ceph-mon[179688]: failed to encode map e84 with expected crc 2023-09-23T20:49:34.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:49:33 smithi026 ceph-mon[179688]: failed to encode map e84 with expected crc 2023-09-23T20:49:34.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:49:33 smithi026 ceph-mon[175151]: pgmap v135: 73 pgs: 15 peering, 2 active+undersized+degraded, 9 active+undersized, 47 active+clean; 582 KiB data, 108 MiB used, 715 GiB / 715 GiB avail; 3/72 objects degraded (4.167%) 2023-09-23T20:49:34.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:49:33 smithi026 ceph-mon[175151]: failed to encode map e84 with expected crc 2023-09-23T20:49:34.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:49:33 smithi026 ceph-mon[175151]: failed to encode map e84 with expected crc 2023-09-23T20:49:34.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:49:33 smithi026 ceph-mon[175151]: failed to encode map e84 with expected crc 2023-09-23T20:49:34.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:49:33 smithi026 ceph-mon[175151]: failed to encode map e84 with expected crc 2023-09-23T20:49:34.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:49:33 smithi026 ceph-mon[175151]: failed to encode map e84 with expected crc 2023-09-23T20:49:34.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:49:33 smithi026 ceph-mon[175151]: failed to encode map e84 with expected crc 2023-09-23T20:49:34.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:49:33 smithi026 ceph-mon[175151]: failed to encode map e84 with expected crc 2023-09-23T20:49:34.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:49:33 smithi026 ceph-mon[175151]: failed to encode map e84 with expected crc 2023-09-23T20:49:35.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:49:34 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/2585862057' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:49:35.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:49:34 smithi070 ceph-mon[168282]: Health check update: Degraded data redundancy: 3/72 objects degraded (4.167%), 2 pgs degraded (PG_DEGRADED) 2023-09-23T20:49:35.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:49:34 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/2585862057' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:49:35.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:49:34 smithi026 ceph-mon[175151]: Health check update: Degraded data redundancy: 3/72 objects degraded (4.167%), 2 pgs degraded (PG_DEGRADED) 2023-09-23T20:49:35.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:49:34 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/2585862057' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:49:35.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:49:34 smithi026 ceph-mon[179688]: Health check update: Degraded data redundancy: 3/72 objects degraded (4.167%), 2 pgs degraded (PG_DEGRADED) 2023-09-23T20:49:35.882 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:49:35 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[164029]: 2023-09-23T20:49:35.855+0000 7fd59efc8700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (6 PGs are or would become offline) 2023-09-23T20:49:35.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:49:35 smithi026 ceph-mon[179688]: pgmap v137: 73 pgs: 15 peering, 2 active+undersized+degraded, 9 active+undersized, 47 active+clean; 582 KiB data, 108 MiB used, 715 GiB / 715 GiB avail; 3/72 objects degraded (4.167%) 2023-09-23T20:49:35.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:49:35 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:49:35.883 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:49:35 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:49:35.883 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:49:35 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:49:35.883 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:49:35 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:49:35.883 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:49:35 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:49:35.883 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:49:35 smithi026 ceph-mon[175151]: pgmap v137: 73 pgs: 15 peering, 2 active+undersized+degraded, 9 active+undersized, 47 active+clean; 582 KiB data, 108 MiB used, 715 GiB / 715 GiB avail; 3/72 objects degraded (4.167%) 2023-09-23T20:49:35.883 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:49:35 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:49:35.883 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:49:35 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:49:35.883 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:49:35 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:49:35.883 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:49:35 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:49:35.883 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:49:35 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:49:36.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:49:35 smithi070 ceph-mon[168282]: pgmap v137: 73 pgs: 15 peering, 2 active+undersized+degraded, 9 active+undersized, 47 active+clean; 582 KiB data, 108 MiB used, 715 GiB / 715 GiB avail; 3/72 objects degraded (4.167%) 2023-09-23T20:49:36.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:49:35 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:49:36.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:49:35 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:49:36.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:49:35 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:49:36.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:49:35 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:49:36.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:49:35 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:49:37.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:49:36 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:49:37.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:49:36 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T20:49:37.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:49:36 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T20:49:37.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:49:36 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T20:49:37.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:49:36 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-09-23T20:49:37.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:49:36 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/2610366302' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:49:37.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:49:36 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:49:37.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:49:36 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T20:49:37.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:49:36 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T20:49:37.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:49:36 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T20:49:37.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:49:36 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-09-23T20:49:37.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:49:36 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/2610366302' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:49:37.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:49:36 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:49:37.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:49:36 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T20:49:37.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:49:36 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T20:49:37.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:49:36 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T20:49:37.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:49:36 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-09-23T20:49:37.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:49:36 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/2610366302' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:49:38.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:49:37 smithi070 ceph-mon[168282]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-09-23T20:49:38.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:49:37 smithi070 ceph-mon[168282]: Upgrade: unsafe to stop osd(s) at this time (6 PGs are or would become offline) 2023-09-23T20:49:38.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:49:37 smithi070 ceph-mon[168282]: pgmap v138: 73 pgs: 15 peering, 58 active+clean; 582 KiB data, 108 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:49:38.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:49:37 smithi070 ceph-mon[168282]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 3/72 objects degraded (4.167%), 2 pgs degraded) 2023-09-23T20:49:38.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:49:37 smithi070 ceph-mon[168282]: Cluster is now healthy 2023-09-23T20:49:38.131 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:49:37 smithi026 ceph-mon[175151]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-09-23T20:49:38.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:49:37 smithi026 ceph-mon[175151]: Upgrade: unsafe to stop osd(s) at this time (6 PGs are or would become offline) 2023-09-23T20:49:38.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:49:37 smithi026 ceph-mon[175151]: pgmap v138: 73 pgs: 15 peering, 58 active+clean; 582 KiB data, 108 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:49:38.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:49:37 smithi026 ceph-mon[175151]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 3/72 objects degraded (4.167%), 2 pgs degraded) 2023-09-23T20:49:38.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:49:37 smithi026 ceph-mon[175151]: Cluster is now healthy 2023-09-23T20:49:38.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:49:37 smithi026 ceph-mon[179688]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-09-23T20:49:38.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:49:37 smithi026 ceph-mon[179688]: Upgrade: unsafe to stop osd(s) at this time (6 PGs are or would become offline) 2023-09-23T20:49:38.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:49:37 smithi026 ceph-mon[179688]: pgmap v138: 73 pgs: 15 peering, 58 active+clean; 582 KiB data, 108 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:49:38.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:49:37 smithi026 ceph-mon[179688]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 3/72 objects degraded (4.167%), 2 pgs degraded) 2023-09-23T20:49:38.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:49:37 smithi026 ceph-mon[179688]: Cluster is now healthy 2023-09-23T20:49:39.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:49:38 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/1196836481' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:49:39.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:49:38 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/1196836481' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:49:39.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:49:38 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/1196836481' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:49:39.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:49:39 smithi026 ceph-mon[175151]: pgmap v139: 73 pgs: 73 active+clean; 582 KiB data, 109 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:49:39.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:49:39 smithi026 ceph-mon[179688]: pgmap v139: 73 pgs: 73 active+clean; 582 KiB data, 109 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:49:39.883 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:49:39 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[164029]: ::ffff:172.21.15.70 - - [23/Sep/2023:20:49:39] "GET /metrics HTTP/1.1" 200 33679 "" "Prometheus/2.43.0" 2023-09-23T20:49:40.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:49:39 smithi070 ceph-mon[168282]: pgmap v139: 73 pgs: 73 active+clean; 582 KiB data, 109 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:49:42.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:49:41 smithi070 ceph-mon[168282]: pgmap v140: 73 pgs: 73 active+clean; 582 KiB data, 109 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:49:42.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:49:41 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/408620882' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:49:42.131 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:49:41 smithi026 ceph-mon[179688]: pgmap v140: 73 pgs: 73 active+clean; 582 KiB data, 109 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:49:42.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:49:41 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/408620882' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:49:42.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:49:41 smithi026 ceph-mon[175151]: pgmap v140: 73 pgs: 73 active+clean; 582 KiB data, 109 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:49:42.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:49:41 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/408620882' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:49:44.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:49:43 smithi070 ceph-mon[168282]: pgmap v141: 73 pgs: 73 active+clean; 582 KiB data, 109 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:49:44.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:49:43 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/570000162' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:49:44.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:49:43 smithi026 ceph-mon[175151]: pgmap v141: 73 pgs: 73 active+clean; 582 KiB data, 109 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:49:44.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:49:43 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/570000162' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:49:44.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:49:43 smithi026 ceph-mon[179688]: pgmap v141: 73 pgs: 73 active+clean; 582 KiB data, 109 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:49:44.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:49:43 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/570000162' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:49:46.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:49:45 smithi070 ceph-mon[168282]: pgmap v142: 73 pgs: 73 active+clean; 582 KiB data, 109 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:49:46.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:49:45 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/2991247965' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:49:46.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:49:45 smithi026 ceph-mon[175151]: pgmap v142: 73 pgs: 73 active+clean; 582 KiB data, 109 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:49:46.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:49:45 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/2991247965' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:49:46.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:49:45 smithi026 ceph-mon[179688]: pgmap v142: 73 pgs: 73 active+clean; 582 KiB data, 109 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:49:46.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:49:45 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/2991247965' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:49:47.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:49:47 smithi070 ceph-mon[168282]: pgmap v143: 73 pgs: 73 active+clean; 582 KiB data, 109 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:49:47.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:49:47 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:49:47.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:49:47 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:49:47.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:49:47 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:49:47.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:49:47 smithi026 ceph-mon[179688]: pgmap v143: 73 pgs: 73 active+clean; 582 KiB data, 109 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:49:47.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:49:47 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:49:47.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:49:47 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:49:47.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:49:47 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:49:47.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:49:47 smithi026 ceph-mon[175151]: pgmap v143: 73 pgs: 73 active+clean; 582 KiB data, 109 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:49:47.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:49:47 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:49:47.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:49:47 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:49:47.633 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:49:47 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:49:48.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:49:48 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/746221318' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:49:48.631 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:49:48 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/746221318' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:49:48.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:49:48 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/746221318' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:49:49.611 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:49:49 smithi026 ceph-mon[175151]: pgmap v144: 73 pgs: 73 active+clean; 582 KiB data, 109 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:49:49.611 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:49:49 smithi026 ceph-mon[179688]: pgmap v144: 73 pgs: 73 active+clean; 582 KiB data, 109 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:49:49.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:49:49 smithi070 ceph-mon[168282]: pgmap v144: 73 pgs: 73 active+clean; 582 KiB data, 109 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:49:49.882 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:49:49 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[164029]: ::ffff:172.21.15.70 - - [23/Sep/2023:20:49:49] "GET /metrics HTTP/1.1" 200 33826 "" "Prometheus/2.43.0" 2023-09-23T20:49:51.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:49:50 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/151932107' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:49:51.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:49:50 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/151932107' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:49:51.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:49:50 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/151932107' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:49:51.850 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:49:51 smithi026 ceph-mon[175151]: pgmap v145: 73 pgs: 73 active+clean; 582 KiB data, 109 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:49:51.851 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:49:51 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-09-23T20:49:51.851 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:49:51 smithi026 ceph-mon[179688]: pgmap v145: 73 pgs: 73 active+clean; 582 KiB data, 109 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:49:51.851 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:49:51 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-09-23T20:49:52.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:49:51 smithi070 ceph-mon[168282]: pgmap v145: 73 pgs: 73 active+clean; 582 KiB data, 109 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:49:52.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:49:51 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-09-23T20:49:53.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:49:52 smithi070 ceph-mon[168282]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-09-23T20:49:53.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:49:52 smithi070 ceph-mon[168282]: Upgrade: osd.1 is safe to restart 2023-09-23T20:49:53.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:49:52 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:49:53.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:49:52 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2023-09-23T20:49:53.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:49:52 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:49:53.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:49:52 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/2252454344' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:49:53.120 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:49:52 smithi026 ceph-mon[179688]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-09-23T20:49:53.120 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:49:52 smithi026 ceph-mon[179688]: Upgrade: osd.1 is safe to restart 2023-09-23T20:49:53.120 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:49:52 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:49:53.120 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:49:52 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2023-09-23T20:49:53.121 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:49:52 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:49:53.121 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:49:52 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/2252454344' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:49:53.121 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:49:52 smithi026 ceph-mon[175151]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-09-23T20:49:53.121 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:49:52 smithi026 ceph-mon[175151]: Upgrade: osd.1 is safe to restart 2023-09-23T20:49:53.121 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:49:52 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:49:53.121 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:49:52 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2023-09-23T20:49:53.121 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:49:52 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:49:53.121 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:49:52 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/2252454344' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:49:54.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:49:53 smithi070 ceph-mon[168282]: pgmap v146: 73 pgs: 73 active+clean; 582 KiB data, 109 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:49:54.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:49:53 smithi070 ceph-mon[168282]: Upgrade: Updating osd.1 2023-09-23T20:49:54.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:49:53 smithi070 ceph-mon[168282]: Deploying daemon osd.1 on smithi026 2023-09-23T20:49:54.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:49:53 smithi026 ceph-mon[179688]: pgmap v146: 73 pgs: 73 active+clean; 582 KiB data, 109 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:49:54.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:49:53 smithi026 ceph-mon[179688]: Upgrade: Updating osd.1 2023-09-23T20:49:54.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:49:53 smithi026 ceph-mon[179688]: Deploying daemon osd.1 on smithi026 2023-09-23T20:49:54.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:49:53 smithi026 ceph-mon[175151]: pgmap v146: 73 pgs: 73 active+clean; 582 KiB data, 109 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:49:54.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:49:53 smithi026 ceph-mon[175151]: Upgrade: Updating osd.1 2023-09-23T20:49:54.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:49:53 smithi026 ceph-mon[175151]: Deploying daemon osd.1 on smithi026 2023-09-23T20:49:55.382 INFO:journalctl@ceph.osd.1.smithi026.stdout:Sep 23 20:49:55 smithi026 systemd[1]: Stopping Ceph osd.1 for b3f8e4ec-5a50-11ee-8db4-212e2dc638e7... 2023-09-23T20:49:55.766 INFO:journalctl@ceph.osd.1.smithi026.stdout:Sep 23 20:49:55 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd-1[121458]: 2023-09-23T20:49:55.393+0000 7f64745c7700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-osd -n osd.1 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2023-09-23T20:49:55.766 INFO:journalctl@ceph.osd.1.smithi026.stdout:Sep 23 20:49:55 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd-1[121458]: 2023-09-23T20:49:55.393+0000 7f64745c7700 -1 osd.1 84 *** Got signal Terminated *** 2023-09-23T20:49:55.766 INFO:journalctl@ceph.osd.1.smithi026.stdout:Sep 23 20:49:55 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd-1[121458]: 2023-09-23T20:49:55.393+0000 7f64745c7700 -1 osd.1 84 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-09-23T20:49:56.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:49:55 smithi070 ceph-mon[168282]: pgmap v147: 73 pgs: 73 active+clean; 582 KiB data, 109 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:49:56.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:49:55 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/3916560799' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:49:56.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:49:55 smithi070 ceph-mon[168282]: osd.1 marked itself down and dead 2023-09-23T20:49:56.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:49:55 smithi026 ceph-mon[179688]: pgmap v147: 73 pgs: 73 active+clean; 582 KiB data, 109 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:49:56.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:49:55 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/3916560799' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:49:56.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:49:55 smithi026 ceph-mon[179688]: osd.1 marked itself down and dead 2023-09-23T20:49:56.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:49:55 smithi026 ceph-mon[175151]: pgmap v147: 73 pgs: 73 active+clean; 582 KiB data, 109 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:49:56.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:49:55 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/3916560799' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:49:56.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:49:55 smithi026 ceph-mon[175151]: osd.1 marked itself down and dead 2023-09-23T20:49:56.557 INFO:journalctl@ceph.osd.1.smithi026.stdout:Sep 23 20:49:56 smithi026 bash[195891]: ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd-1 2023-09-23T20:49:57.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:49:56 smithi070 ceph-mon[168282]: Health check failed: 1 osds down (OSD_DOWN) 2023-09-23T20:49:57.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:49:56 smithi070 ceph-mon[168282]: failed to encode map e85 with expected crc 2023-09-23T20:49:57.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:49:56 smithi070 ceph-mon[168282]: failed to encode map e85 with expected crc 2023-09-23T20:49:57.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:49:56 smithi070 ceph-mon[168282]: osdmap e85: 8 total, 7 up, 8 in 2023-09-23T20:49:57.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:49:56 smithi070 ceph-mon[168282]: failed to encode map e85 with expected crc 2023-09-23T20:49:57.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:49:56 smithi070 ceph-mon[168282]: failed to encode map e85 with expected crc 2023-09-23T20:49:57.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:49:56 smithi070 ceph-mon[168282]: failed to encode map e85 with expected crc 2023-09-23T20:49:57.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:49:56 smithi070 ceph-mon[168282]: failed to encode map e85 with expected crc 2023-09-23T20:49:57.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:49:56 smithi026 ceph-mon[175151]: Health check failed: 1 osds down (OSD_DOWN) 2023-09-23T20:49:57.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:49:56 smithi026 ceph-mon[175151]: failed to encode map e85 with expected crc 2023-09-23T20:49:57.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:49:56 smithi026 ceph-mon[175151]: failed to encode map e85 with expected crc 2023-09-23T20:49:57.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:49:56 smithi026 ceph-mon[175151]: osdmap e85: 8 total, 7 up, 8 in 2023-09-23T20:49:57.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:49:56 smithi026 ceph-mon[175151]: failed to encode map e85 with expected crc 2023-09-23T20:49:57.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:49:56 smithi026 ceph-mon[175151]: failed to encode map e85 with expected crc 2023-09-23T20:49:57.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:49:56 smithi026 ceph-mon[175151]: failed to encode map e85 with expected crc 2023-09-23T20:49:57.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:49:56 smithi026 ceph-mon[175151]: failed to encode map e85 with expected crc 2023-09-23T20:49:57.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:49:56 smithi026 ceph-mon[179688]: Health check failed: 1 osds down (OSD_DOWN) 2023-09-23T20:49:57.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:49:56 smithi026 ceph-mon[179688]: failed to encode map e85 with expected crc 2023-09-23T20:49:57.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:49:56 smithi026 ceph-mon[179688]: failed to encode map e85 with expected crc 2023-09-23T20:49:57.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:49:56 smithi026 ceph-mon[179688]: osdmap e85: 8 total, 7 up, 8 in 2023-09-23T20:49:57.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:49:56 smithi026 ceph-mon[179688]: failed to encode map e85 with expected crc 2023-09-23T20:49:57.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:49:56 smithi026 ceph-mon[179688]: failed to encode map e85 with expected crc 2023-09-23T20:49:57.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:49:56 smithi026 ceph-mon[179688]: failed to encode map e85 with expected crc 2023-09-23T20:49:57.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:49:56 smithi026 ceph-mon[179688]: failed to encode map e85 with expected crc 2023-09-23T20:49:57.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:49:57 smithi026 ceph-mon[175151]: pgmap v149: 73 pgs: 10 stale+active+clean, 63 active+clean; 582 KiB data, 109 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:49:57.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:49:57 smithi026 ceph-mon[175151]: failed to encode map e86 with expected crc 2023-09-23T20:49:57.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:49:57 smithi026 ceph-mon[175151]: failed to encode map e86 with expected crc 2023-09-23T20:49:57.883 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:49:57 smithi026 ceph-mon[175151]: osdmap e86: 8 total, 7 up, 8 in 2023-09-23T20:49:57.883 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:49:57 smithi026 ceph-mon[175151]: failed to encode map e86 with expected crc 2023-09-23T20:49:57.883 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:49:57 smithi026 ceph-mon[175151]: failed to encode map e86 with expected crc 2023-09-23T20:49:57.883 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:49:57 smithi026 ceph-mon[175151]: failed to encode map e86 with expected crc 2023-09-23T20:49:57.883 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:49:57 smithi026 ceph-mon[175151]: failed to encode map e86 with expected crc 2023-09-23T20:49:57.883 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:49:57 smithi026 ceph-mon[175151]: failed to encode map e86 with expected crc 2023-09-23T20:49:57.883 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:49:57 smithi026 ceph-mon[175151]: failed to encode map e86 with expected crc 2023-09-23T20:49:57.883 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:49:57 smithi026 ceph-mon[175151]: failed to encode map e86 with expected crc 2023-09-23T20:49:57.883 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:49:57 smithi026 ceph-mon[175151]: failed to encode map e86 with expected crc 2023-09-23T20:49:57.883 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:49:57 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/3553112559' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:49:57.884 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:49:57 smithi026 ceph-mon[179688]: pgmap v149: 73 pgs: 10 stale+active+clean, 63 active+clean; 582 KiB data, 109 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:49:57.884 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:49:57 smithi026 ceph-mon[179688]: failed to encode map e86 with expected crc 2023-09-23T20:49:57.884 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:49:57 smithi026 ceph-mon[179688]: failed to encode map e86 with expected crc 2023-09-23T20:49:57.884 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:49:57 smithi026 ceph-mon[179688]: osdmap e86: 8 total, 7 up, 8 in 2023-09-23T20:49:57.884 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:49:57 smithi026 ceph-mon[179688]: failed to encode map e86 with expected crc 2023-09-23T20:49:57.884 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:49:57 smithi026 ceph-mon[179688]: failed to encode map e86 with expected crc 2023-09-23T20:49:57.884 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:49:57 smithi026 ceph-mon[179688]: failed to encode map e86 with expected crc 2023-09-23T20:49:57.884 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:49:57 smithi026 ceph-mon[179688]: failed to encode map e86 with expected crc 2023-09-23T20:49:57.884 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:49:57 smithi026 ceph-mon[179688]: failed to encode map e86 with expected crc 2023-09-23T20:49:57.885 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:49:57 smithi026 ceph-mon[179688]: failed to encode map e86 with expected crc 2023-09-23T20:49:57.885 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:49:57 smithi026 ceph-mon[179688]: failed to encode map e86 with expected crc 2023-09-23T20:49:57.885 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:49:57 smithi026 ceph-mon[179688]: failed to encode map e86 with expected crc 2023-09-23T20:49:57.885 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:49:57 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/3553112559' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:49:58.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:49:57 smithi070 ceph-mon[168282]: pgmap v149: 73 pgs: 10 stale+active+clean, 63 active+clean; 582 KiB data, 109 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:49:58.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:49:57 smithi070 ceph-mon[168282]: failed to encode map e86 with expected crc 2023-09-23T20:49:58.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:49:57 smithi070 ceph-mon[168282]: failed to encode map e86 with expected crc 2023-09-23T20:49:58.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:49:57 smithi070 ceph-mon[168282]: osdmap e86: 8 total, 7 up, 8 in 2023-09-23T20:49:58.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:49:57 smithi070 ceph-mon[168282]: failed to encode map e86 with expected crc 2023-09-23T20:49:58.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:49:57 smithi070 ceph-mon[168282]: failed to encode map e86 with expected crc 2023-09-23T20:49:58.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:49:57 smithi070 ceph-mon[168282]: failed to encode map e86 with expected crc 2023-09-23T20:49:58.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:49:57 smithi070 ceph-mon[168282]: failed to encode map e86 with expected crc 2023-09-23T20:49:58.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:49:57 smithi070 ceph-mon[168282]: failed to encode map e86 with expected crc 2023-09-23T20:49:58.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:49:57 smithi070 ceph-mon[168282]: failed to encode map e86 with expected crc 2023-09-23T20:49:58.121 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:49:57 smithi070 ceph-mon[168282]: failed to encode map e86 with expected crc 2023-09-23T20:49:58.121 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:49:57 smithi070 ceph-mon[168282]: failed to encode map e86 with expected crc 2023-09-23T20:49:58.121 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:49:57 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/3553112559' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:49:58.382 INFO:journalctl@ceph.osd.1.smithi026.stdout:Sep 23 20:49:58 smithi026 systemd[1]: ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7@osd.1.service: Succeeded. 2023-09-23T20:49:58.382 INFO:journalctl@ceph.osd.1.smithi026.stdout:Sep 23 20:49:58 smithi026 systemd[1]: Stopped Ceph osd.1 for b3f8e4ec-5a50-11ee-8db4-212e2dc638e7. 2023-09-23T20:49:58.382 INFO:journalctl@ceph.osd.1.smithi026.stdout:Sep 23 20:49:58 smithi026 systemd[1]: Starting Ceph osd.1 for b3f8e4ec-5a50-11ee-8db4-212e2dc638e7... 2023-09-23T20:49:58.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:49:58 smithi026 ceph-mon[175151]: Health check failed: Degraded data redundancy: 2/72 objects degraded (2.778%), 2 pgs degraded (PG_DEGRADED) 2023-09-23T20:49:58.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:49:58 smithi026 ceph-mon[179688]: Health check failed: Degraded data redundancy: 2/72 objects degraded (2.778%), 2 pgs degraded (PG_DEGRADED) 2023-09-23T20:49:59.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:49:58 smithi070 ceph-mon[168282]: Health check failed: Degraded data redundancy: 2/72 objects degraded (2.778%), 2 pgs degraded (PG_DEGRADED) 2023-09-23T20:49:59.382 INFO:journalctl@ceph.osd.1.smithi026.stdout:Sep 23 20:49:59 smithi026 bash[196184]: --> Failed to activate via raw: did not find any matching OSD to activate 2023-09-23T20:49:59.382 INFO:journalctl@ceph.osd.1.smithi026.stdout:Sep 23 20:49:59 smithi026 bash[196184]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2023-09-23T20:49:59.382 INFO:journalctl@ceph.osd.1.smithi026.stdout:Sep 23 20:49:59 smithi026 bash[196184]: Running command: /usr/bin/ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev /dev/vg_nvme/lv_3 --path /var/lib/ceph/osd/ceph-1 --no-mon-config 2023-09-23T20:49:59.382 INFO:journalctl@ceph.osd.1.smithi026.stdout:Sep 23 20:49:59 smithi026 bash[196184]: Running command: /usr/bin/ln -snf /dev/vg_nvme/lv_3 /var/lib/ceph/osd/ceph-1/block 2023-09-23T20:49:59.382 INFO:journalctl@ceph.osd.1.smithi026.stdout:Sep 23 20:49:59 smithi026 bash[196184]: Running command: /usr/bin/chown -h ceph:ceph /var/lib/ceph/osd/ceph-1/block 2023-09-23T20:49:59.382 INFO:journalctl@ceph.osd.1.smithi026.stdout:Sep 23 20:49:59 smithi026 bash[196184]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2 2023-09-23T20:49:59.382 INFO:journalctl@ceph.osd.1.smithi026.stdout:Sep 23 20:49:59 smithi026 bash[196184]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2023-09-23T20:49:59.382 INFO:journalctl@ceph.osd.1.smithi026.stdout:Sep 23 20:49:59 smithi026 bash[196184]: --> ceph-volume lvm activate successful for osd ID: 1 2023-09-23T20:49:59.881 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:49:59 smithi026 ceph-mon[179688]: pgmap v151: 73 pgs: 2 active+undersized+degraded, 1 stale+active+clean, 27 active+undersized, 43 active+clean; 582 KiB data, 109 MiB used, 715 GiB / 715 GiB avail; 2/72 objects degraded (2.778%) 2023-09-23T20:49:59.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:49:59 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/3589792285' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:49:59.882 INFO:journalctl@ceph.osd.1.smithi026.stdout:Sep 23 20:49:59 smithi026 podman[196474]: 2023-09-23T20:49:59.883 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:49:59 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[164029]: ::ffff:172.21.15.70 - - [23/Sep/2023:20:49:59] "GET /metrics HTTP/1.1" 200 33826 "" "Prometheus/2.43.0" 2023-09-23T20:49:59.883 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:49:59 smithi026 ceph-mon[175151]: pgmap v151: 73 pgs: 2 active+undersized+degraded, 1 stale+active+clean, 27 active+undersized, 43 active+clean; 582 KiB data, 109 MiB used, 715 GiB / 715 GiB avail; 2/72 objects degraded (2.778%) 2023-09-23T20:49:59.883 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:49:59 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/3589792285' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:50:00.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:49:59 smithi070 ceph-mon[168282]: pgmap v151: 73 pgs: 2 active+undersized+degraded, 1 stale+active+clean, 27 active+undersized, 43 active+clean; 582 KiB data, 109 MiB used, 715 GiB / 715 GiB avail; 2/72 objects degraded (2.778%) 2023-09-23T20:50:00.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:49:59 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/3589792285' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:50:00.382 INFO:journalctl@ceph.osd.1.smithi026.stdout:Sep 23 20:49:59 smithi026 ceph-osd[196517]: -- 172.21.15.26:0/2657273680 <== mon.2 v2:172.21.15.26:3301/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x55f1bdf724e0 con 0x55f1bdf42c00 2023-09-23T20:50:00.382 INFO:journalctl@ceph.osd.1.smithi026.stdout:Sep 23 20:50:00 smithi026 bash[196474]: 0e3111ea70d36607984f03648a813a4f35a1d29cc3c8c44d13ca1cdecf958d34 2023-09-23T20:50:00.382 INFO:journalctl@ceph.osd.1.smithi026.stdout:Sep 23 20:50:00 smithi026 systemd[1]: Started Ceph osd.1 for b3f8e4ec-5a50-11ee-8db4-212e2dc638e7. 2023-09-23T20:50:00.834 INFO:journalctl@ceph.osd.1.smithi026.stdout:Sep 23 20:50:00 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd-1[196493]: 2023-09-23T20:50:00.492+0000 7f8e8aad3640 -1 Falling back to public interface 2023-09-23T20:50:01.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:50:00 smithi070 ceph-mon[168282]: Health detail: HEALTH_WARN 1 osds down; Degraded data redundancy: 2/72 objects degraded (2.778%), 2 pgs degraded 2023-09-23T20:50:01.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:50:00 smithi070 ceph-mon[168282]: [WRN] OSD_DOWN: 1 osds down 2023-09-23T20:50:01.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:50:00 smithi070 ceph-mon[168282]: osd.1 (root=default,host=smithi026) is down 2023-09-23T20:50:01.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:50:00 smithi070 ceph-mon[168282]: [WRN] PG_DEGRADED: Degraded data redundancy: 2/72 objects degraded (2.778%), 2 pgs degraded 2023-09-23T20:50:01.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:50:00 smithi070 ceph-mon[168282]: pg 3.e is active+undersized+degraded, acting [7,4] 2023-09-23T20:50:01.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:50:00 smithi070 ceph-mon[168282]: pg 3.1c is active+undersized+degraded, acting [5,4] 2023-09-23T20:50:01.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:50:00 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:50:01.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:50:00 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:50:01.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:50:00 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:50:01.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:50:00 smithi026 ceph-mon[175151]: Health detail: HEALTH_WARN 1 osds down; Degraded data redundancy: 2/72 objects degraded (2.778%), 2 pgs degraded 2023-09-23T20:50:01.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:50:00 smithi026 ceph-mon[175151]: [WRN] OSD_DOWN: 1 osds down 2023-09-23T20:50:01.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:50:00 smithi026 ceph-mon[175151]: osd.1 (root=default,host=smithi026) is down 2023-09-23T20:50:01.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:50:00 smithi026 ceph-mon[175151]: [WRN] PG_DEGRADED: Degraded data redundancy: 2/72 objects degraded (2.778%), 2 pgs degraded 2023-09-23T20:50:01.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:50:00 smithi026 ceph-mon[175151]: pg 3.e is active+undersized+degraded, acting [7,4] 2023-09-23T20:50:01.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:50:00 smithi026 ceph-mon[175151]: pg 3.1c is active+undersized+degraded, acting [5,4] 2023-09-23T20:50:01.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:50:00 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:50:01.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:50:00 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:50:01.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:50:00 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:50:01.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:50:00 smithi026 ceph-mon[179688]: Health detail: HEALTH_WARN 1 osds down; Degraded data redundancy: 2/72 objects degraded (2.778%), 2 pgs degraded 2023-09-23T20:50:01.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:50:00 smithi026 ceph-mon[179688]: [WRN] OSD_DOWN: 1 osds down 2023-09-23T20:50:01.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:50:00 smithi026 ceph-mon[179688]: osd.1 (root=default,host=smithi026) is down 2023-09-23T20:50:01.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:50:00 smithi026 ceph-mon[179688]: [WRN] PG_DEGRADED: Degraded data redundancy: 2/72 objects degraded (2.778%), 2 pgs degraded 2023-09-23T20:50:01.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:50:00 smithi026 ceph-mon[179688]: pg 3.e is active+undersized+degraded, acting [7,4] 2023-09-23T20:50:01.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:50:00 smithi026 ceph-mon[179688]: pg 3.1c is active+undersized+degraded, acting [5,4] 2023-09-23T20:50:01.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:50:00 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:50:01.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:50:00 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:50:01.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:50:00 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:50:02.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:50:01 smithi070 ceph-mon[168282]: pgmap v152: 73 pgs: 2 active+undersized+degraded, 1 stale+active+clean, 27 active+undersized, 43 active+clean; 582 KiB data, 109 MiB used, 715 GiB / 715 GiB avail; 2/72 objects degraded (2.778%) 2023-09-23T20:50:02.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:50:01 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:50:02.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:50:01 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:50:02.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:50:01 smithi026 ceph-mon[179688]: pgmap v152: 73 pgs: 2 active+undersized+degraded, 1 stale+active+clean, 27 active+undersized, 43 active+clean; 582 KiB data, 109 MiB used, 715 GiB / 715 GiB avail; 2/72 objects degraded (2.778%) 2023-09-23T20:50:02.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:50:01 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:50:02.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:50:01 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:50:02.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:50:01 smithi026 ceph-mon[175151]: pgmap v152: 73 pgs: 2 active+undersized+degraded, 1 stale+active+clean, 27 active+undersized, 43 active+clean; 582 KiB data, 109 MiB used, 715 GiB / 715 GiB avail; 2/72 objects degraded (2.778%) 2023-09-23T20:50:02.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:50:01 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:50:02.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:50:01 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:50:03.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:50:02 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/2332943398' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:50:03.131 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:50:02 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/2332943398' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:50:03.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:50:02 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/2332943398' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:50:03.631 INFO:journalctl@ceph.osd.1.smithi026.stdout:Sep 23 20:50:03 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd-1[196493]: 2023-09-23T20:50:03.254+0000 7f8e8aad3640 -1 osd.1 84 log_to_monitors true 2023-09-23T20:50:04.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:50:03 smithi070 ceph-mon[168282]: pgmap v153: 73 pgs: 3 active+undersized+degraded, 29 active+undersized, 41 active+clean; 582 KiB data, 109 MiB used, 715 GiB / 715 GiB avail; 3/72 objects degraded (4.167%) 2023-09-23T20:50:04.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:50:03 smithi070 ceph-mon[168282]: from='osd.1 [v2:172.21.15.26:6810/3442737755,v1:172.21.15.26:6811/3442737755]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2023-09-23T20:50:04.131 INFO:journalctl@ceph.osd.1.smithi026.stdout:Sep 23 20:50:03 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd-1[196493]: 2023-09-23T20:50:03.860+0000 7f8e79f3c700 -1 osd.1 84 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-09-23T20:50:04.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:50:03 smithi026 ceph-mon[175151]: pgmap v153: 73 pgs: 3 active+undersized+degraded, 29 active+undersized, 41 active+clean; 582 KiB data, 109 MiB used, 715 GiB / 715 GiB avail; 3/72 objects degraded (4.167%) 2023-09-23T20:50:04.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:50:03 smithi026 ceph-mon[175151]: from='osd.1 [v2:172.21.15.26:6810/3442737755,v1:172.21.15.26:6811/3442737755]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2023-09-23T20:50:04.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:50:03 smithi026 ceph-mon[179688]: pgmap v153: 73 pgs: 3 active+undersized+degraded, 29 active+undersized, 41 active+clean; 582 KiB data, 109 MiB used, 715 GiB / 715 GiB avail; 3/72 objects degraded (4.167%) 2023-09-23T20:50:04.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:50:03 smithi026 ceph-mon[179688]: from='osd.1 [v2:172.21.15.26:6810/3442737755,v1:172.21.15.26:6811/3442737755]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2023-09-23T20:50:05.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:50:04 smithi070 ceph-mon[168282]: failed to encode map e87 with expected crc 2023-09-23T20:50:05.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:50:04 smithi070 ceph-mon[168282]: failed to encode map e87 with expected crc 2023-09-23T20:50:05.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:50:04 smithi070 ceph-mon[168282]: from='osd.1 [v2:172.21.15.26:6810/3442737755,v1:172.21.15.26:6811/3442737755]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2023-09-23T20:50:05.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:50:04 smithi070 ceph-mon[168282]: osdmap e87: 8 total, 7 up, 8 in 2023-09-23T20:50:05.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:50:04 smithi070 ceph-mon[168282]: from='osd.1 [v2:172.21.15.26:6810/3442737755,v1:172.21.15.26:6811/3442737755]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi026", "root=default"]}]: dispatch 2023-09-23T20:50:05.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:50:04 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/854607474' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:50:05.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:50:04 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:50:05.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:50:04 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:50:05.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:50:04 smithi070 ceph-mon[168282]: Health check update: Degraded data redundancy: 3/72 objects degraded (4.167%), 3 pgs degraded (PG_DEGRADED) 2023-09-23T20:50:05.131 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:50:04 smithi026 ceph-mon[179688]: failed to encode map e87 with expected crc 2023-09-23T20:50:05.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:50:04 smithi026 ceph-mon[179688]: failed to encode map e87 with expected crc 2023-09-23T20:50:05.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:50:04 smithi026 ceph-mon[179688]: from='osd.1 [v2:172.21.15.26:6810/3442737755,v1:172.21.15.26:6811/3442737755]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2023-09-23T20:50:05.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:50:04 smithi026 ceph-mon[179688]: osdmap e87: 8 total, 7 up, 8 in 2023-09-23T20:50:05.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:50:04 smithi026 ceph-mon[179688]: from='osd.1 [v2:172.21.15.26:6810/3442737755,v1:172.21.15.26:6811/3442737755]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi026", "root=default"]}]: dispatch 2023-09-23T20:50:05.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:50:04 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/854607474' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:50:05.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:50:04 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:50:05.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:50:04 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:50:05.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:50:04 smithi026 ceph-mon[179688]: Health check update: Degraded data redundancy: 3/72 objects degraded (4.167%), 3 pgs degraded (PG_DEGRADED) 2023-09-23T20:50:05.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:50:04 smithi026 ceph-mon[175151]: failed to encode map e87 with expected crc 2023-09-23T20:50:05.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:50:04 smithi026 ceph-mon[175151]: failed to encode map e87 with expected crc 2023-09-23T20:50:05.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:50:04 smithi026 ceph-mon[175151]: from='osd.1 [v2:172.21.15.26:6810/3442737755,v1:172.21.15.26:6811/3442737755]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2023-09-23T20:50:05.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:50:04 smithi026 ceph-mon[175151]: osdmap e87: 8 total, 7 up, 8 in 2023-09-23T20:50:05.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:50:04 smithi026 ceph-mon[175151]: from='osd.1 [v2:172.21.15.26:6810/3442737755,v1:172.21.15.26:6811/3442737755]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi026", "root=default"]}]: dispatch 2023-09-23T20:50:05.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:50:04 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/854607474' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:50:05.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:50:04 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:50:05.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:50:04 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:50:05.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:50:04 smithi026 ceph-mon[175151]: Health check update: Degraded data redundancy: 3/72 objects degraded (4.167%), 3 pgs degraded (PG_DEGRADED) 2023-09-23T20:50:06.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:50:05 smithi070 ceph-mon[168282]: pgmap v155: 73 pgs: 3 active+undersized+degraded, 29 active+undersized, 41 active+clean; 582 KiB data, 129 MiB used, 715 GiB / 715 GiB avail; 3/72 objects degraded (4.167%) 2023-09-23T20:50:06.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:50:05 smithi070 ceph-mon[168282]: failed to encode map e87 with expected crc 2023-09-23T20:50:06.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:50:05 smithi070 ceph-mon[168282]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-09-23T20:50:06.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:50:05 smithi070 ceph-mon[168282]: failed to encode map e88 with expected crc 2023-09-23T20:50:06.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:50:05 smithi070 ceph-mon[168282]: osd.1 [v2:172.21.15.26:6810/3442737755,v1:172.21.15.26:6811/3442737755] boot 2023-09-23T20:50:06.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:50:05 smithi070 ceph-mon[168282]: osdmap e88: 8 total, 8 up, 8 in 2023-09-23T20:50:06.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:50:05 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-23T20:50:06.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:50:05 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:50:06.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:50:05 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:50:06.131 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:50:05 smithi026 ceph-mon[179688]: pgmap v155: 73 pgs: 3 active+undersized+degraded, 29 active+undersized, 41 active+clean; 582 KiB data, 129 MiB used, 715 GiB / 715 GiB avail; 3/72 objects degraded (4.167%) 2023-09-23T20:50:06.131 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:50:05 smithi026 ceph-mon[179688]: failed to encode map e87 with expected crc 2023-09-23T20:50:06.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:50:05 smithi026 ceph-mon[179688]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-09-23T20:50:06.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:50:05 smithi026 ceph-mon[179688]: failed to encode map e88 with expected crc 2023-09-23T20:50:06.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:50:05 smithi026 ceph-mon[179688]: osd.1 [v2:172.21.15.26:6810/3442737755,v1:172.21.15.26:6811/3442737755] boot 2023-09-23T20:50:06.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:50:05 smithi026 ceph-mon[179688]: osdmap e88: 8 total, 8 up, 8 in 2023-09-23T20:50:06.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:50:05 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-23T20:50:06.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:50:05 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:50:06.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:50:05 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:50:06.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:50:05 smithi026 ceph-mon[175151]: pgmap v155: 73 pgs: 3 active+undersized+degraded, 29 active+undersized, 41 active+clean; 582 KiB data, 129 MiB used, 715 GiB / 715 GiB avail; 3/72 objects degraded (4.167%) 2023-09-23T20:50:06.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:50:05 smithi026 ceph-mon[175151]: failed to encode map e87 with expected crc 2023-09-23T20:50:06.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:50:05 smithi026 ceph-mon[175151]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-09-23T20:50:06.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:50:05 smithi026 ceph-mon[175151]: failed to encode map e88 with expected crc 2023-09-23T20:50:06.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:50:05 smithi026 ceph-mon[175151]: osd.1 [v2:172.21.15.26:6810/3442737755,v1:172.21.15.26:6811/3442737755] boot 2023-09-23T20:50:06.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:50:05 smithi026 ceph-mon[175151]: osdmap e88: 8 total, 8 up, 8 in 2023-09-23T20:50:06.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:50:05 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-23T20:50:06.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:50:05 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:50:06.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:50:05 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:50:07.085 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:50:06 smithi026 ceph-mon[175151]: failed to encode map e89 with expected crc 2023-09-23T20:50:07.085 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:50:06 smithi026 ceph-mon[175151]: failed to encode map e89 with expected crc 2023-09-23T20:50:07.085 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:50:06 smithi026 ceph-mon[175151]: osdmap e89: 8 total, 8 up, 8 in 2023-09-23T20:50:07.085 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:50:06 smithi026 ceph-mon[175151]: failed to encode map e89 with expected crc 2023-09-23T20:50:07.085 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:50:06 smithi026 ceph-mon[175151]: failed to encode map e89 with expected crc 2023-09-23T20:50:07.085 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:50:06 smithi026 ceph-mon[175151]: failed to encode map e89 with expected crc 2023-09-23T20:50:07.085 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:50:06 smithi026 ceph-mon[175151]: failed to encode map e89 with expected crc 2023-09-23T20:50:07.085 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:50:06 smithi026 ceph-mon[175151]: failed to encode map e89 with expected crc 2023-09-23T20:50:07.085 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:50:06 smithi026 ceph-mon[175151]: pgmap v158: 73 pgs: 2 peering, 2 active+undersized+degraded, 28 active+undersized, 41 active+clean; 582 KiB data, 129 MiB used, 715 GiB / 715 GiB avail; 2/72 objects degraded (2.778%) 2023-09-23T20:50:07.086 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:50:06 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/49621794' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:50:07.086 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:50:06 smithi026 ceph-mon[179688]: failed to encode map e89 with expected crc 2023-09-23T20:50:07.086 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:50:06 smithi026 ceph-mon[179688]: failed to encode map e89 with expected crc 2023-09-23T20:50:07.086 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:50:06 smithi026 ceph-mon[179688]: osdmap e89: 8 total, 8 up, 8 in 2023-09-23T20:50:07.086 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:50:06 smithi026 ceph-mon[179688]: failed to encode map e89 with expected crc 2023-09-23T20:50:07.086 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:50:06 smithi026 ceph-mon[179688]: failed to encode map e89 with expected crc 2023-09-23T20:50:07.086 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:50:06 smithi026 ceph-mon[179688]: failed to encode map e89 with expected crc 2023-09-23T20:50:07.086 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:50:06 smithi026 ceph-mon[179688]: failed to encode map e89 with expected crc 2023-09-23T20:50:07.086 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:50:06 smithi026 ceph-mon[179688]: failed to encode map e89 with expected crc 2023-09-23T20:50:07.086 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:50:06 smithi026 ceph-mon[179688]: pgmap v158: 73 pgs: 2 peering, 2 active+undersized+degraded, 28 active+undersized, 41 active+clean; 582 KiB data, 129 MiB used, 715 GiB / 715 GiB avail; 2/72 objects degraded (2.778%) 2023-09-23T20:50:07.087 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:50:06 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/49621794' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:50:07.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:50:06 smithi070 ceph-mon[168282]: failed to encode map e89 with expected crc 2023-09-23T20:50:07.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:50:06 smithi070 ceph-mon[168282]: failed to encode map e89 with expected crc 2023-09-23T20:50:07.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:50:06 smithi070 ceph-mon[168282]: osdmap e89: 8 total, 8 up, 8 in 2023-09-23T20:50:07.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:50:06 smithi070 ceph-mon[168282]: failed to encode map e89 with expected crc 2023-09-23T20:50:07.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:50:06 smithi070 ceph-mon[168282]: failed to encode map e89 with expected crc 2023-09-23T20:50:07.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:50:06 smithi070 ceph-mon[168282]: failed to encode map e89 with expected crc 2023-09-23T20:50:07.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:50:06 smithi070 ceph-mon[168282]: failed to encode map e89 with expected crc 2023-09-23T20:50:07.121 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:50:06 smithi070 ceph-mon[168282]: failed to encode map e89 with expected crc 2023-09-23T20:50:07.121 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:50:06 smithi070 ceph-mon[168282]: pgmap v158: 73 pgs: 2 peering, 2 active+undersized+degraded, 28 active+undersized, 41 active+clean; 582 KiB data, 129 MiB used, 715 GiB / 715 GiB avail; 2/72 objects degraded (2.778%) 2023-09-23T20:50:07.121 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:50:06 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/49621794' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:50:08.531 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:50:08 smithi026 ceph-mon[175151]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 2/72 objects degraded (2.778%), 2 pgs degraded) 2023-09-23T20:50:08.531 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:50:08 smithi026 ceph-mon[175151]: Cluster is now healthy 2023-09-23T20:50:08.532 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:50:08 smithi026 ceph-mon[179688]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 2/72 objects degraded (2.778%), 2 pgs degraded) 2023-09-23T20:50:08.532 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:50:08 smithi026 ceph-mon[179688]: Cluster is now healthy 2023-09-23T20:50:08.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:50:08 smithi070 ceph-mon[168282]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 2/72 objects degraded (2.778%), 2 pgs degraded) 2023-09-23T20:50:08.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:50:08 smithi070 ceph-mon[168282]: Cluster is now healthy 2023-09-23T20:50:08.881 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:50:08 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[164029]: 2023-09-23T20:50:08.529+0000 7fd59efc8700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (5 PGs are or would become offline) 2023-09-23T20:50:09.632 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:50:09 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[164029]: ::ffff:172.21.15.70 - - [23/Sep/2023:20:50:09] "GET /metrics HTTP/1.1" 200 33826 "" "Prometheus/2.43.0" 2023-09-23T20:50:09.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:50:09 smithi026 ceph-mon[175151]: pgmap v159: 73 pgs: 2 peering, 10 active+undersized, 61 active+clean; 582 KiB data, 130 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:50:09.633 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:50:09 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:50:09.633 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:50:09 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:50:09.633 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:50:09 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:50:09.633 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:50:09 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:50:09.633 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:50:09 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:50:09.633 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:50:09 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:50:09.634 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:50:09 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T20:50:09.634 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:50:09 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T20:50:09.634 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:50:09 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T20:50:09.634 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:50:09 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-09-23T20:50:09.634 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:50:09 smithi026 ceph-mon[175151]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-09-23T20:50:09.634 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:50:09 smithi026 ceph-mon[175151]: Upgrade: unsafe to stop osd(s) at this time (5 PGs are or would become offline) 2023-09-23T20:50:09.634 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:50:09 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/2282516390' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:50:09.635 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:50:09 smithi026 ceph-mon[179688]: pgmap v159: 73 pgs: 2 peering, 10 active+undersized, 61 active+clean; 582 KiB data, 130 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:50:09.635 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:50:09 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:50:09.635 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:50:09 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:50:09.635 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:50:09 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:50:09.635 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:50:09 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:50:09.635 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:50:09 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:50:09.636 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:50:09 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:50:09.636 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:50:09 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T20:50:09.636 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:50:09 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T20:50:09.636 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:50:09 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T20:50:09.636 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:50:09 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-09-23T20:50:09.636 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:50:09 smithi026 ceph-mon[179688]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-09-23T20:50:09.636 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:50:09 smithi026 ceph-mon[179688]: Upgrade: unsafe to stop osd(s) at this time (5 PGs are or would become offline) 2023-09-23T20:50:09.636 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:50:09 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/2282516390' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:50:09.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:50:09 smithi070 ceph-mon[168282]: pgmap v159: 73 pgs: 2 peering, 10 active+undersized, 61 active+clean; 582 KiB data, 130 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:50:09.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:50:09 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:50:09.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:50:09 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:50:09.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:50:09 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:50:09.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:50:09 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:50:09.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:50:09 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:50:09.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:50:09 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:50:09.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:50:09 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T20:50:09.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:50:09 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T20:50:09.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:50:09 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T20:50:09.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:50:09 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-09-23T20:50:09.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:50:09 smithi070 ceph-mon[168282]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-09-23T20:50:09.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:50:09 smithi070 ceph-mon[168282]: Upgrade: unsafe to stop osd(s) at this time (5 PGs are or would become offline) 2023-09-23T20:50:09.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:50:09 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/2282516390' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:50:12.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:50:11 smithi070 ceph-mon[168282]: pgmap v160: 73 pgs: 2 peering, 71 active+clean; 582 KiB data, 130 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:50:12.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:50:11 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:50:12.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:50:11 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/1105329485' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:50:12.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:50:11 smithi026 ceph-mon[175151]: pgmap v160: 73 pgs: 2 peering, 71 active+clean; 582 KiB data, 130 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:50:12.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:50:11 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:50:12.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:50:11 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/1105329485' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:50:12.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:50:11 smithi026 ceph-mon[179688]: pgmap v160: 73 pgs: 2 peering, 71 active+clean; 582 KiB data, 130 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:50:12.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:50:11 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:50:12.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:50:11 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/1105329485' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:50:14.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:50:13 smithi070 ceph-mon[168282]: pgmap v161: 73 pgs: 73 active+clean; 582 KiB data, 130 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:50:14.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:50:13 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/3003449621' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:50:14.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:50:13 smithi026 ceph-mon[175151]: pgmap v161: 73 pgs: 73 active+clean; 582 KiB data, 130 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:50:14.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:50:13 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/3003449621' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:50:14.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:50:13 smithi026 ceph-mon[179688]: pgmap v161: 73 pgs: 73 active+clean; 582 KiB data, 130 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:50:14.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:50:13 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/3003449621' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:50:16.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:50:15 smithi070 ceph-mon[168282]: pgmap v162: 73 pgs: 73 active+clean; 582 KiB data, 130 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:50:16.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:50:15 smithi026 ceph-mon[175151]: pgmap v162: 73 pgs: 73 active+clean; 582 KiB data, 130 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:50:16.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:50:15 smithi026 ceph-mon[179688]: pgmap v162: 73 pgs: 73 active+clean; 582 KiB data, 130 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:50:17.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:50:16 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/3262004063' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:50:17.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:50:16 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:50:17.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:50:16 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:50:17.131 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:50:16 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/3262004063' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:50:17.131 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:50:16 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:50:17.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:50:16 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:50:17.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:50:16 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/3262004063' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:50:17.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:50:16 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:50:17.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:50:16 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:50:18.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:50:17 smithi070 ceph-mon[168282]: pgmap v163: 73 pgs: 73 active+clean; 582 KiB data, 130 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:50:18.131 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:50:17 smithi026 ceph-mon[179688]: pgmap v163: 73 pgs: 73 active+clean; 582 KiB data, 130 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:50:18.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:50:17 smithi026 ceph-mon[175151]: pgmap v163: 73 pgs: 73 active+clean; 582 KiB data, 130 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:50:19.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:50:18 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/2879860181' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:50:19.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:50:18 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/2879860181' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:50:19.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:50:18 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/2879860181' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:50:19.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:50:19 smithi026 ceph-mon[175151]: pgmap v164: 73 pgs: 73 active+clean; 582 KiB data, 130 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:50:19.882 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:50:19 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[164029]: ::ffff:172.21.15.70 - - [23/Sep/2023:20:50:19] "GET /metrics HTTP/1.1" 200 33824 "" "Prometheus/2.43.0" 2023-09-23T20:50:19.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:50:19 smithi026 ceph-mon[179688]: pgmap v164: 73 pgs: 73 active+clean; 582 KiB data, 130 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:50:20.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:50:19 smithi070 ceph-mon[168282]: pgmap v164: 73 pgs: 73 active+clean; 582 KiB data, 130 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:50:21.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:50:20 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/2570021881' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:50:21.131 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:50:20 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/2570021881' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:50:21.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:50:20 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/2570021881' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:50:22.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:50:21 smithi070 ceph-mon[168282]: pgmap v165: 73 pgs: 73 active+clean; 582 KiB data, 130 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:50:22.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:50:21 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:50:22.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:50:21 smithi026 ceph-mon[175151]: pgmap v165: 73 pgs: 73 active+clean; 582 KiB data, 130 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:50:22.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:50:21 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:50:22.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:50:21 smithi026 ceph-mon[179688]: pgmap v165: 73 pgs: 73 active+clean; 582 KiB data, 130 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:50:22.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:50:21 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:50:24.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:50:23 smithi070 ceph-mon[168282]: pgmap v166: 73 pgs: 73 active+clean; 582 KiB data, 130 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:50:24.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:50:23 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/2025038442' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:50:24.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:50:23 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-09-23T20:50:24.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:50:23 smithi026 ceph-mon[175151]: pgmap v166: 73 pgs: 73 active+clean; 582 KiB data, 130 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:50:24.134 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:50:23 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/2025038442' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:50:24.134 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:50:23 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-09-23T20:50:24.134 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:50:23 smithi026 ceph-mon[179688]: pgmap v166: 73 pgs: 73 active+clean; 582 KiB data, 130 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:50:24.134 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:50:23 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/2025038442' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:50:24.134 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:50:23 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-09-23T20:50:25.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:50:24 smithi070 ceph-mon[168282]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-09-23T20:50:25.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:50:24 smithi070 ceph-mon[168282]: Upgrade: osd.2 is safe to restart 2023-09-23T20:50:25.131 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:50:24 smithi026 ceph-mon[179688]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-09-23T20:50:25.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:50:24 smithi026 ceph-mon[179688]: Upgrade: osd.2 is safe to restart 2023-09-23T20:50:25.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:50:24 smithi026 ceph-mon[175151]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-09-23T20:50:25.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:50:24 smithi026 ceph-mon[175151]: Upgrade: osd.2 is safe to restart 2023-09-23T20:50:26.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:50:25 smithi070 ceph-mon[168282]: pgmap v167: 73 pgs: 73 active+clean; 582 KiB data, 130 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:50:26.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:50:25 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:50:26.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:50:25 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2023-09-23T20:50:26.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:50:25 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:50:26.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:50:25 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/4166559259' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:50:26.131 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:50:25 smithi026 ceph-mon[179688]: pgmap v167: 73 pgs: 73 active+clean; 582 KiB data, 130 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:50:26.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:50:25 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:50:26.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:50:25 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2023-09-23T20:50:26.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:50:25 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:50:26.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:50:25 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/4166559259' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:50:26.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:50:25 smithi026 ceph-mon[175151]: pgmap v167: 73 pgs: 73 active+clean; 582 KiB data, 130 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:50:26.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:50:25 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:50:26.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:50:25 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2023-09-23T20:50:26.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:50:25 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:50:26.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:50:25 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/4166559259' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:50:27.080 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:50:26 smithi026 ceph-mon[175151]: Upgrade: Updating osd.2 2023-09-23T20:50:27.081 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:50:26 smithi026 ceph-mon[175151]: Deploying daemon osd.2 on smithi026 2023-09-23T20:50:27.081 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:50:26 smithi026 ceph-mon[179688]: Upgrade: Updating osd.2 2023-09-23T20:50:27.081 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:50:26 smithi026 ceph-mon[179688]: Deploying daemon osd.2 on smithi026 2023-09-23T20:50:27.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:50:26 smithi070 ceph-mon[168282]: Upgrade: Updating osd.2 2023-09-23T20:50:27.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:50:26 smithi070 ceph-mon[168282]: Deploying daemon osd.2 on smithi026 2023-09-23T20:50:27.748 INFO:journalctl@ceph.osd.2.smithi026.stdout:Sep 23 20:50:27 smithi026 systemd[1]: Stopping Ceph osd.2 for b3f8e4ec-5a50-11ee-8db4-212e2dc638e7... 2023-09-23T20:50:28.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:50:27 smithi070 ceph-mon[168282]: pgmap v168: 73 pgs: 73 active+clean; 582 KiB data, 130 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:50:28.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:50:27 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/1748576622' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:50:28.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:50:27 smithi070 ceph-mon[168282]: osd.2 marked itself down and dead 2023-09-23T20:50:28.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:50:27 smithi026 ceph-mon[175151]: pgmap v168: 73 pgs: 73 active+clean; 582 KiB data, 130 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:50:28.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:50:27 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/1748576622' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:50:28.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:50:27 smithi026 ceph-mon[175151]: osd.2 marked itself down and dead 2023-09-23T20:50:28.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:50:27 smithi026 ceph-mon[179688]: pgmap v168: 73 pgs: 73 active+clean; 582 KiB data, 130 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:50:28.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:50:27 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/1748576622' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:50:28.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:50:27 smithi026 ceph-mon[179688]: osd.2 marked itself down and dead 2023-09-23T20:50:28.132 INFO:journalctl@ceph.osd.2.smithi026.stdout:Sep 23 20:50:27 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd-2[128665]: 2023-09-23T20:50:27.747+0000 7fee8c98f700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-osd -n osd.2 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2023-09-23T20:50:28.133 INFO:journalctl@ceph.osd.2.smithi026.stdout:Sep 23 20:50:27 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd-2[128665]: 2023-09-23T20:50:27.747+0000 7fee8c98f700 -1 osd.2 89 *** Got signal Terminated *** 2023-09-23T20:50:28.133 INFO:journalctl@ceph.osd.2.smithi026.stdout:Sep 23 20:50:27 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd-2[128665]: 2023-09-23T20:50:27.747+0000 7fee8c98f700 -1 osd.2 89 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-09-23T20:50:29.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:50:28 smithi070 ceph-mon[168282]: Health check failed: 1 osds down (OSD_DOWN) 2023-09-23T20:50:29.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:50:28 smithi070 ceph-mon[168282]: failed to encode map e90 with expected crc 2023-09-23T20:50:29.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:50:28 smithi070 ceph-mon[168282]: osdmap e90: 8 total, 7 up, 8 in 2023-09-23T20:50:29.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:50:28 smithi070 ceph-mon[168282]: failed to encode map e90 with expected crc 2023-09-23T20:50:29.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:50:28 smithi070 ceph-mon[168282]: failed to encode map e90 with expected crc 2023-09-23T20:50:29.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:50:28 smithi070 ceph-mon[168282]: failed to encode map e90 with expected crc 2023-09-23T20:50:29.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:50:28 smithi070 ceph-mon[168282]: failed to encode map e90 with expected crc 2023-09-23T20:50:29.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:50:28 smithi070 ceph-mon[168282]: failed to encode map e90 with expected crc 2023-09-23T20:50:29.121 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:50:28 smithi070 ceph-mon[168282]: failed to encode map e90 with expected crc 2023-09-23T20:50:29.121 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:50:28 smithi070 ceph-mon[168282]: osdmap e91: 8 total, 7 up, 8 in 2023-09-23T20:50:29.132 INFO:journalctl@ceph.osd.2.smithi026.stdout:Sep 23 20:50:28 smithi026 bash[201951]: ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd-2 2023-09-23T20:50:29.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:50:28 smithi026 ceph-mon[175151]: Health check failed: 1 osds down (OSD_DOWN) 2023-09-23T20:50:29.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:50:28 smithi026 ceph-mon[175151]: failed to encode map e90 with expected crc 2023-09-23T20:50:29.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:50:28 smithi026 ceph-mon[175151]: osdmap e90: 8 total, 7 up, 8 in 2023-09-23T20:50:29.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:50:28 smithi026 ceph-mon[175151]: failed to encode map e90 with expected crc 2023-09-23T20:50:29.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:50:28 smithi026 ceph-mon[175151]: failed to encode map e90 with expected crc 2023-09-23T20:50:29.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:50:28 smithi026 ceph-mon[175151]: failed to encode map e90 with expected crc 2023-09-23T20:50:29.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:50:28 smithi026 ceph-mon[175151]: failed to encode map e90 with expected crc 2023-09-23T20:50:29.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:50:28 smithi026 ceph-mon[175151]: failed to encode map e90 with expected crc 2023-09-23T20:50:29.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:50:28 smithi026 ceph-mon[175151]: failed to encode map e90 with expected crc 2023-09-23T20:50:29.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:50:28 smithi026 ceph-mon[175151]: osdmap e91: 8 total, 7 up, 8 in 2023-09-23T20:50:29.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:50:28 smithi026 ceph-mon[179688]: Health check failed: 1 osds down (OSD_DOWN) 2023-09-23T20:50:29.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:50:28 smithi026 ceph-mon[179688]: failed to encode map e90 with expected crc 2023-09-23T20:50:29.134 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:50:28 smithi026 ceph-mon[179688]: osdmap e90: 8 total, 7 up, 8 in 2023-09-23T20:50:29.134 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:50:28 smithi026 ceph-mon[179688]: failed to encode map e90 with expected crc 2023-09-23T20:50:29.134 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:50:28 smithi026 ceph-mon[179688]: failed to encode map e90 with expected crc 2023-09-23T20:50:29.134 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:50:28 smithi026 ceph-mon[179688]: failed to encode map e90 with expected crc 2023-09-23T20:50:29.134 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:50:28 smithi026 ceph-mon[179688]: failed to encode map e90 with expected crc 2023-09-23T20:50:29.134 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:50:28 smithi026 ceph-mon[179688]: failed to encode map e90 with expected crc 2023-09-23T20:50:29.134 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:50:28 smithi026 ceph-mon[179688]: failed to encode map e90 with expected crc 2023-09-23T20:50:29.134 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:50:28 smithi026 ceph-mon[179688]: osdmap e91: 8 total, 7 up, 8 in 2023-09-23T20:50:29.882 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:50:29 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[164029]: ::ffff:172.21.15.70 - - [23/Sep/2023:20:50:29] "GET /metrics HTTP/1.1" 200 33824 "" "Prometheus/2.43.0" 2023-09-23T20:50:29.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:50:29 smithi026 ceph-mon[179688]: pgmap v170: 73 pgs: 2 stale+active+clean, 71 active+clean; 582 KiB data, 130 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:50:29.891 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:50:29 smithi026 ceph-mon[179688]: failed to encode map e91 with expected crc 2023-09-23T20:50:29.891 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:50:29 smithi026 ceph-mon[179688]: failed to encode map e91 with expected crc 2023-09-23T20:50:29.891 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:50:29 smithi026 ceph-mon[179688]: failed to encode map e91 with expected crc 2023-09-23T20:50:29.891 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:50:29 smithi026 ceph-mon[179688]: failed to encode map e91 with expected crc 2023-09-23T20:50:29.891 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:50:29 smithi026 ceph-mon[179688]: failed to encode map e91 with expected crc 2023-09-23T20:50:29.891 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:50:29 smithi026 ceph-mon[179688]: failed to encode map e91 with expected crc 2023-09-23T20:50:29.891 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:50:29 smithi026 ceph-mon[179688]: failed to encode map e91 with expected crc 2023-09-23T20:50:29.892 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:50:29 smithi026 ceph-mon[179688]: failed to encode map e91 with expected crc 2023-09-23T20:50:29.892 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:50:29 smithi026 ceph-mon[175151]: pgmap v170: 73 pgs: 2 stale+active+clean, 71 active+clean; 582 KiB data, 130 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:50:29.892 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:50:29 smithi026 ceph-mon[175151]: failed to encode map e91 with expected crc 2023-09-23T20:50:29.892 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:50:29 smithi026 ceph-mon[175151]: failed to encode map e91 with expected crc 2023-09-23T20:50:29.892 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:50:29 smithi026 ceph-mon[175151]: failed to encode map e91 with expected crc 2023-09-23T20:50:29.892 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:50:29 smithi026 ceph-mon[175151]: failed to encode map e91 with expected crc 2023-09-23T20:50:29.892 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:50:29 smithi026 ceph-mon[175151]: failed to encode map e91 with expected crc 2023-09-23T20:50:29.892 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:50:29 smithi026 ceph-mon[175151]: failed to encode map e91 with expected crc 2023-09-23T20:50:29.892 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:50:29 smithi026 ceph-mon[175151]: failed to encode map e91 with expected crc 2023-09-23T20:50:29.893 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:50:29 smithi026 ceph-mon[175151]: failed to encode map e91 with expected crc 2023-09-23T20:50:30.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:50:29 smithi070 ceph-mon[168282]: pgmap v170: 73 pgs: 2 stale+active+clean, 71 active+clean; 582 KiB data, 130 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:50:30.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:50:29 smithi070 ceph-mon[168282]: failed to encode map e91 with expected crc 2023-09-23T20:50:30.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:50:29 smithi070 ceph-mon[168282]: failed to encode map e91 with expected crc 2023-09-23T20:50:30.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:50:29 smithi070 ceph-mon[168282]: failed to encode map e91 with expected crc 2023-09-23T20:50:30.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:50:29 smithi070 ceph-mon[168282]: failed to encode map e91 with expected crc 2023-09-23T20:50:30.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:50:29 smithi070 ceph-mon[168282]: failed to encode map e91 with expected crc 2023-09-23T20:50:30.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:50:29 smithi070 ceph-mon[168282]: failed to encode map e91 with expected crc 2023-09-23T20:50:30.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:50:29 smithi070 ceph-mon[168282]: failed to encode map e91 with expected crc 2023-09-23T20:50:30.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:50:29 smithi070 ceph-mon[168282]: failed to encode map e91 with expected crc 2023-09-23T20:50:30.382 INFO:journalctl@ceph.osd.2.smithi026.stdout:Sep 23 20:50:29 smithi026 systemd[1]: ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7@osd.2.service: Succeeded. 2023-09-23T20:50:30.382 INFO:journalctl@ceph.osd.2.smithi026.stdout:Sep 23 20:50:30 smithi026 systemd[1]: Stopped Ceph osd.2 for b3f8e4ec-5a50-11ee-8db4-212e2dc638e7. 2023-09-23T20:50:30.382 INFO:journalctl@ceph.osd.2.smithi026.stdout:Sep 23 20:50:30 smithi026 systemd[1]: Starting Ceph osd.2 for b3f8e4ec-5a50-11ee-8db4-212e2dc638e7... 2023-09-23T20:50:31.095 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:50:30 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/2569577860' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:50:31.096 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:50:30 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/2569577860' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:50:31.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:50:30 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/2569577860' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:50:31.382 INFO:journalctl@ceph.osd.2.smithi026.stdout:Sep 23 20:50:31 smithi026 bash[202245]: --> Failed to activate via raw: did not find any matching OSD to activate 2023-09-23T20:50:31.383 INFO:journalctl@ceph.osd.2.smithi026.stdout:Sep 23 20:50:31 smithi026 bash[202245]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2023-09-23T20:50:31.383 INFO:journalctl@ceph.osd.2.smithi026.stdout:Sep 23 20:50:31 smithi026 bash[202245]: Running command: /usr/bin/ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev /dev/vg_nvme/lv_2 --path /var/lib/ceph/osd/ceph-2 --no-mon-config 2023-09-23T20:50:31.383 INFO:journalctl@ceph.osd.2.smithi026.stdout:Sep 23 20:50:31 smithi026 bash[202245]: Running command: /usr/bin/ln -snf /dev/vg_nvme/lv_2 /var/lib/ceph/osd/ceph-2/block 2023-09-23T20:50:31.383 INFO:journalctl@ceph.osd.2.smithi026.stdout:Sep 23 20:50:31 smithi026 bash[202245]: Running command: /usr/bin/chown -h ceph:ceph /var/lib/ceph/osd/ceph-2/block 2023-09-23T20:50:31.383 INFO:journalctl@ceph.osd.2.smithi026.stdout:Sep 23 20:50:31 smithi026 bash[202245]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-1 2023-09-23T20:50:31.383 INFO:journalctl@ceph.osd.2.smithi026.stdout:Sep 23 20:50:31 smithi026 bash[202245]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2023-09-23T20:50:31.383 INFO:journalctl@ceph.osd.2.smithi026.stdout:Sep 23 20:50:31 smithi026 bash[202245]: --> ceph-volume lvm activate successful for osd ID: 2 2023-09-23T20:50:32.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:50:31 smithi070 ceph-mon[168282]: pgmap v172: 73 pgs: 2 stale+active+clean, 4 active+undersized, 67 active+clean; 582 KiB data, 130 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:50:32.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:50:31 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:50:32.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:50:31 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:50:32.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:50:31 smithi026 ceph-mon[175151]: pgmap v172: 73 pgs: 2 stale+active+clean, 4 active+undersized, 67 active+clean; 582 KiB data, 130 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:50:32.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:50:31 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:50:32.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:50:31 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:50:32.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:50:31 smithi026 ceph-mon[179688]: pgmap v172: 73 pgs: 2 stale+active+clean, 4 active+undersized, 67 active+clean; 582 KiB data, 130 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:50:32.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:50:31 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:50:32.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:50:31 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:50:32.132 INFO:journalctl@ceph.osd.2.smithi026.stdout:Sep 23 20:50:31 smithi026 podman[202494]: 2023-09-23T20:50:32.133 INFO:journalctl@ceph.osd.2.smithi026.stdout:Sep 23 20:50:31 smithi026 ceph-osd[202534]: -- 172.21.15.26:0/1901329832 <== mon.0 v2:172.21.15.26:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x559f517c44e0 con 0x559f51795400 2023-09-23T20:50:32.133 INFO:journalctl@ceph.osd.2.smithi026.stdout:Sep 23 20:50:31 smithi026 bash[202494]: 26e1c16041a9e28fa758d09278cb7571be5521477891f4a6ee2087f412407247 2023-09-23T20:50:32.133 INFO:journalctl@ceph.osd.2.smithi026.stdout:Sep 23 20:50:31 smithi026 systemd[1]: Started Ceph osd.2 for b3f8e4ec-5a50-11ee-8db4-212e2dc638e7. 2023-09-23T20:50:32.882 INFO:journalctl@ceph.osd.2.smithi026.stdout:Sep 23 20:50:32 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd-2[202510]: 2023-09-23T20:50:32.416+0000 7f1283f08640 -1 Falling back to public interface 2023-09-23T20:50:33.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:50:32 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:50:33.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:50:32 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:50:33.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:50:32 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:50:33.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:50:32 smithi070 ceph-mon[168282]: pgmap v173: 73 pgs: 2 active+undersized+degraded, 17 active+undersized, 54 active+clean; 582 KiB data, 134 MiB used, 715 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-09-23T20:50:33.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:50:32 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/1193784713' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:50:33.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:50:32 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:50:33.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:50:32 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:50:33.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:50:32 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:50:33.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:50:32 smithi026 ceph-mon[175151]: pgmap v173: 73 pgs: 2 active+undersized+degraded, 17 active+undersized, 54 active+clean; 582 KiB data, 134 MiB used, 715 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-09-23T20:50:33.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:50:32 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/1193784713' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:50:33.383 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:50:32 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:50:33.383 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:50:32 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:50:33.383 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:50:32 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:50:33.383 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:50:32 smithi026 ceph-mon[179688]: pgmap v173: 73 pgs: 2 active+undersized+degraded, 17 active+undersized, 54 active+clean; 582 KiB data, 134 MiB used, 715 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-09-23T20:50:33.383 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:50:32 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/1193784713' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:50:34.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:50:33 smithi070 ceph-mon[168282]: Health check failed: Degraded data redundancy: 4/72 objects degraded (5.556%), 2 pgs degraded (PG_DEGRADED) 2023-09-23T20:50:34.381 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:50:33 smithi026 ceph-mon[179688]: Health check failed: Degraded data redundancy: 4/72 objects degraded (5.556%), 2 pgs degraded (PG_DEGRADED) 2023-09-23T20:50:34.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:50:33 smithi026 ceph-mon[175151]: Health check failed: Degraded data redundancy: 4/72 objects degraded (5.556%), 2 pgs degraded (PG_DEGRADED) 2023-09-23T20:50:35.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:50:34 smithi070 ceph-mon[168282]: pgmap v174: 73 pgs: 2 active+undersized+degraded, 17 active+undersized, 54 active+clean; 582 KiB data, 134 MiB used, 715 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-09-23T20:50:35.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:50:34 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/1270865709' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:50:35.382 INFO:journalctl@ceph.osd.2.smithi026.stdout:Sep 23 20:50:35 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd-2[202510]: 2023-09-23T20:50:35.190+0000 7f1283f08640 -1 osd.2 89 log_to_monitors true 2023-09-23T20:50:35.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:50:34 smithi026 ceph-mon[175151]: pgmap v174: 73 pgs: 2 active+undersized+degraded, 17 active+undersized, 54 active+clean; 582 KiB data, 134 MiB used, 715 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-09-23T20:50:35.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:50:34 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/1270865709' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:50:35.383 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:50:34 smithi026 ceph-mon[179688]: pgmap v174: 73 pgs: 2 active+undersized+degraded, 17 active+undersized, 54 active+clean; 582 KiB data, 134 MiB used, 715 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-09-23T20:50:35.383 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:50:34 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/1270865709' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:50:36.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:50:35 smithi070 ceph-mon[168282]: from='osd.2 [v2:172.21.15.26:6818/38979282,v1:172.21.15.26:6819/38979282]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2023-09-23T20:50:36.381 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:50:35 smithi026 ceph-mon[179688]: from='osd.2 [v2:172.21.15.26:6818/38979282,v1:172.21.15.26:6819/38979282]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2023-09-23T20:50:36.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:50:35 smithi026 ceph-mon[175151]: from='osd.2 [v2:172.21.15.26:6818/38979282,v1:172.21.15.26:6819/38979282]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2023-09-23T20:50:36.382 INFO:journalctl@ceph.osd.2.smithi026.stdout:Sep 23 20:50:35 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd-2[202510]: 2023-09-23T20:50:35.986+0000 7f1272b70700 -1 osd.2 89 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-09-23T20:50:37.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:50:36 smithi070 ceph-mon[168282]: from='osd.2 [v2:172.21.15.26:6818/38979282,v1:172.21.15.26:6819/38979282]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2023-09-23T20:50:37.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:50:36 smithi070 ceph-mon[168282]: osdmap e92: 8 total, 7 up, 8 in 2023-09-23T20:50:37.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:50:36 smithi070 ceph-mon[168282]: from='osd.2 [v2:172.21.15.26:6818/38979282,v1:172.21.15.26:6819/38979282]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi026", "root=default"]}]: dispatch 2023-09-23T20:50:37.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:50:36 smithi070 ceph-mon[168282]: pgmap v176: 73 pgs: 2 active+undersized+degraded, 17 active+undersized, 54 active+clean; 582 KiB data, 145 MiB used, 715 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-09-23T20:50:37.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:50:36 smithi070 ceph-mon[168282]: failed to encode map e92 with expected crc 2023-09-23T20:50:37.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:50:36 smithi070 ceph-mon[168282]: failed to encode map e92 with expected crc 2023-09-23T20:50:37.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:50:36 smithi070 ceph-mon[168282]: failed to encode map e92 with expected crc 2023-09-23T20:50:37.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:50:36 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:50:37.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:50:36 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:50:37.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:50:36 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:50:37.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:50:36 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:50:37.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:50:36 smithi026 ceph-mon[175151]: from='osd.2 [v2:172.21.15.26:6818/38979282,v1:172.21.15.26:6819/38979282]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2023-09-23T20:50:37.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:50:36 smithi026 ceph-mon[175151]: osdmap e92: 8 total, 7 up, 8 in 2023-09-23T20:50:37.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:50:36 smithi026 ceph-mon[175151]: from='osd.2 [v2:172.21.15.26:6818/38979282,v1:172.21.15.26:6819/38979282]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi026", "root=default"]}]: dispatch 2023-09-23T20:50:37.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:50:36 smithi026 ceph-mon[175151]: pgmap v176: 73 pgs: 2 active+undersized+degraded, 17 active+undersized, 54 active+clean; 582 KiB data, 145 MiB used, 715 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-09-23T20:50:37.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:50:36 smithi026 ceph-mon[175151]: failed to encode map e92 with expected crc 2023-09-23T20:50:37.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:50:36 smithi026 ceph-mon[175151]: failed to encode map e92 with expected crc 2023-09-23T20:50:37.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:50:36 smithi026 ceph-mon[175151]: failed to encode map e92 with expected crc 2023-09-23T20:50:37.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:50:36 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:50:37.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:50:36 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:50:37.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:50:36 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:50:37.383 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:50:36 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:50:37.383 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:50:36 smithi026 ceph-mon[179688]: from='osd.2 [v2:172.21.15.26:6818/38979282,v1:172.21.15.26:6819/38979282]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2023-09-23T20:50:37.383 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:50:36 smithi026 ceph-mon[179688]: osdmap e92: 8 total, 7 up, 8 in 2023-09-23T20:50:37.383 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:50:36 smithi026 ceph-mon[179688]: from='osd.2 [v2:172.21.15.26:6818/38979282,v1:172.21.15.26:6819/38979282]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi026", "root=default"]}]: dispatch 2023-09-23T20:50:37.383 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:50:36 smithi026 ceph-mon[179688]: pgmap v176: 73 pgs: 2 active+undersized+degraded, 17 active+undersized, 54 active+clean; 582 KiB data, 145 MiB used, 715 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-09-23T20:50:37.383 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:50:36 smithi026 ceph-mon[179688]: failed to encode map e92 with expected crc 2023-09-23T20:50:37.383 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:50:36 smithi026 ceph-mon[179688]: failed to encode map e92 with expected crc 2023-09-23T20:50:37.383 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:50:36 smithi026 ceph-mon[179688]: failed to encode map e92 with expected crc 2023-09-23T20:50:37.383 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:50:36 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:50:37.384 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:50:36 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:50:37.384 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:50:36 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:50:37.384 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:50:36 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:50:38.297 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:50:37 smithi026 ceph-mon[175151]: failed to encode map e92 with expected crc 2023-09-23T20:50:38.297 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:50:37 smithi026 ceph-mon[175151]: failed to encode map e92 with expected crc 2023-09-23T20:50:38.297 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:50:37 smithi026 ceph-mon[175151]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-09-23T20:50:38.297 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:50:37 smithi026 ceph-mon[175151]: failed to encode map e93 with expected crc 2023-09-23T20:50:38.297 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:50:37 smithi026 ceph-mon[175151]: failed to encode map e93 with expected crc 2023-09-23T20:50:38.297 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:50:37 smithi026 ceph-mon[175151]: osd.2 [v2:172.21.15.26:6818/38979282,v1:172.21.15.26:6819/38979282] boot 2023-09-23T20:50:38.297 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:50:37 smithi026 ceph-mon[175151]: osdmap e93: 8 total, 8 up, 8 in 2023-09-23T20:50:38.297 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:50:37 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-23T20:50:38.297 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:50:37 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/1576214263' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:50:38.297 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:50:37 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:50:38.298 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:50:37 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:50:38.298 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:50:37 smithi026 ceph-mon[179688]: failed to encode map e92 with expected crc 2023-09-23T20:50:38.298 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:50:37 smithi026 ceph-mon[179688]: failed to encode map e92 with expected crc 2023-09-23T20:50:38.298 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:50:37 smithi026 ceph-mon[179688]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-09-23T20:50:38.298 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:50:37 smithi026 ceph-mon[179688]: failed to encode map e93 with expected crc 2023-09-23T20:50:38.298 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:50:37 smithi026 ceph-mon[179688]: failed to encode map e93 with expected crc 2023-09-23T20:50:38.298 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:50:37 smithi026 ceph-mon[179688]: osd.2 [v2:172.21.15.26:6818/38979282,v1:172.21.15.26:6819/38979282] boot 2023-09-23T20:50:38.298 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:50:37 smithi026 ceph-mon[179688]: osdmap e93: 8 total, 8 up, 8 in 2023-09-23T20:50:38.298 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:50:37 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-23T20:50:38.299 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:50:37 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/1576214263' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:50:38.299 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:50:37 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:50:38.299 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:50:37 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:50:38.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:50:37 smithi070 ceph-mon[168282]: failed to encode map e92 with expected crc 2023-09-23T20:50:38.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:50:37 smithi070 ceph-mon[168282]: failed to encode map e92 with expected crc 2023-09-23T20:50:38.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:50:37 smithi070 ceph-mon[168282]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-09-23T20:50:38.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:50:37 smithi070 ceph-mon[168282]: failed to encode map e93 with expected crc 2023-09-23T20:50:38.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:50:37 smithi070 ceph-mon[168282]: failed to encode map e93 with expected crc 2023-09-23T20:50:38.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:50:37 smithi070 ceph-mon[168282]: osd.2 [v2:172.21.15.26:6818/38979282,v1:172.21.15.26:6819/38979282] boot 2023-09-23T20:50:38.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:50:37 smithi070 ceph-mon[168282]: osdmap e93: 8 total, 8 up, 8 in 2023-09-23T20:50:38.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:50:37 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-23T20:50:38.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:50:37 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/1576214263' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:50:38.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:50:37 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:50:38.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:50:37 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:50:39.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:50:38 smithi070 ceph-mon[168282]: failed to encode map e94 with expected crc 2023-09-23T20:50:39.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:50:38 smithi070 ceph-mon[168282]: failed to encode map e94 with expected crc 2023-09-23T20:50:39.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:50:38 smithi070 ceph-mon[168282]: osdmap e94: 8 total, 8 up, 8 in 2023-09-23T20:50:39.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:50:38 smithi070 ceph-mon[168282]: failed to encode map e94 with expected crc 2023-09-23T20:50:39.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:50:38 smithi070 ceph-mon[168282]: failed to encode map e94 with expected crc 2023-09-23T20:50:39.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:50:38 smithi070 ceph-mon[168282]: failed to encode map e94 with expected crc 2023-09-23T20:50:39.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:50:38 smithi070 ceph-mon[168282]: failed to encode map e94 with expected crc 2023-09-23T20:50:39.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:50:38 smithi070 ceph-mon[168282]: failed to encode map e94 with expected crc 2023-09-23T20:50:39.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:50:38 smithi070 ceph-mon[168282]: failed to encode map e94 with expected crc 2023-09-23T20:50:39.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:50:38 smithi070 ceph-mon[168282]: failed to encode map e94 with expected crc 2023-09-23T20:50:39.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:50:38 smithi070 ceph-mon[168282]: pgmap v179: 73 pgs: 10 peering, 1 active+undersized+degraded, 8 active+undersized, 54 active+clean; 582 KiB data, 145 MiB used, 715 GiB / 715 GiB avail; 2/72 objects degraded (2.778%) 2023-09-23T20:50:39.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:50:38 smithi026 ceph-mon[175151]: failed to encode map e94 with expected crc 2023-09-23T20:50:39.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:50:38 smithi026 ceph-mon[175151]: failed to encode map e94 with expected crc 2023-09-23T20:50:39.383 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:50:38 smithi026 ceph-mon[175151]: osdmap e94: 8 total, 8 up, 8 in 2023-09-23T20:50:39.383 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:50:38 smithi026 ceph-mon[175151]: failed to encode map e94 with expected crc 2023-09-23T20:50:39.383 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:50:38 smithi026 ceph-mon[175151]: failed to encode map e94 with expected crc 2023-09-23T20:50:39.383 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:50:38 smithi026 ceph-mon[175151]: failed to encode map e94 with expected crc 2023-09-23T20:50:39.383 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:50:38 smithi026 ceph-mon[175151]: failed to encode map e94 with expected crc 2023-09-23T20:50:39.383 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:50:38 smithi026 ceph-mon[175151]: failed to encode map e94 with expected crc 2023-09-23T20:50:39.384 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:50:38 smithi026 ceph-mon[175151]: failed to encode map e94 with expected crc 2023-09-23T20:50:39.384 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:50:38 smithi026 ceph-mon[175151]: failed to encode map e94 with expected crc 2023-09-23T20:50:39.384 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:50:38 smithi026 ceph-mon[175151]: pgmap v179: 73 pgs: 10 peering, 1 active+undersized+degraded, 8 active+undersized, 54 active+clean; 582 KiB data, 145 MiB used, 715 GiB / 715 GiB avail; 2/72 objects degraded (2.778%) 2023-09-23T20:50:39.384 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:50:38 smithi026 ceph-mon[179688]: failed to encode map e94 with expected crc 2023-09-23T20:50:39.384 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:50:38 smithi026 ceph-mon[179688]: failed to encode map e94 with expected crc 2023-09-23T20:50:39.384 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:50:38 smithi026 ceph-mon[179688]: osdmap e94: 8 total, 8 up, 8 in 2023-09-23T20:50:39.384 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:50:38 smithi026 ceph-mon[179688]: failed to encode map e94 with expected crc 2023-09-23T20:50:39.385 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:50:38 smithi026 ceph-mon[179688]: failed to encode map e94 with expected crc 2023-09-23T20:50:39.385 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:50:38 smithi026 ceph-mon[179688]: failed to encode map e94 with expected crc 2023-09-23T20:50:39.385 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:50:38 smithi026 ceph-mon[179688]: failed to encode map e94 with expected crc 2023-09-23T20:50:39.385 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:50:38 smithi026 ceph-mon[179688]: failed to encode map e94 with expected crc 2023-09-23T20:50:39.385 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:50:38 smithi026 ceph-mon[179688]: failed to encode map e94 with expected crc 2023-09-23T20:50:39.385 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:50:38 smithi026 ceph-mon[179688]: failed to encode map e94 with expected crc 2023-09-23T20:50:39.385 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:50:38 smithi026 ceph-mon[179688]: pgmap v179: 73 pgs: 10 peering, 1 active+undersized+degraded, 8 active+undersized, 54 active+clean; 582 KiB data, 145 MiB used, 715 GiB / 715 GiB avail; 2/72 objects degraded (2.778%) 2023-09-23T20:50:39.881 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:50:39 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[164029]: ::ffff:172.21.15.70 - - [23/Sep/2023:20:50:39] "GET /metrics HTTP/1.1" 200 33822 "" "Prometheus/2.43.0" 2023-09-23T20:50:40.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:50:39 smithi070 ceph-mon[168282]: Health check update: Degraded data redundancy: 2/72 objects degraded (2.778%), 1 pg degraded (PG_DEGRADED) 2023-09-23T20:50:40.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:50:39 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/942273378' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:50:40.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:50:39 smithi026 ceph-mon[179688]: Health check update: Degraded data redundancy: 2/72 objects degraded (2.778%), 1 pg degraded (PG_DEGRADED) 2023-09-23T20:50:40.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:50:39 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/942273378' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:50:40.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:50:39 smithi026 ceph-mon[175151]: Health check update: Degraded data redundancy: 2/72 objects degraded (2.778%), 1 pg degraded (PG_DEGRADED) 2023-09-23T20:50:40.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:50:39 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/942273378' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:50:40.882 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:50:40 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[164029]: 2023-09-23T20:50:40.454+0000 7fd59efc8700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (5 PGs are or would become offline) 2023-09-23T20:50:41.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:50:41 smithi070 ceph-mon[168282]: pgmap v180: 73 pgs: 10 peering, 1 active+undersized+degraded, 4 active+undersized, 58 active+clean; 582 KiB data, 145 MiB used, 715 GiB / 715 GiB avail; 2/72 objects degraded (2.778%) 2023-09-23T20:50:41.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:50:41 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:50:41.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:50:41 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:50:41.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:50:41 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:50:41.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:50:41 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:50:41.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:50:41 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:50:41.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:50:41 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:50:41.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:50:41 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T20:50:41.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:50:41 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T20:50:41.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:50:41 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T20:50:41.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:50:41 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-09-23T20:50:41.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:50:41 smithi070 ceph-mon[168282]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-09-23T20:50:41.621 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:50:41 smithi070 ceph-mon[168282]: Upgrade: unsafe to stop osd(s) at this time (5 PGs are or would become offline) 2023-09-23T20:50:41.621 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:50:41 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:50:41.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:50:41 smithi026 ceph-mon[179688]: pgmap v180: 73 pgs: 10 peering, 1 active+undersized+degraded, 4 active+undersized, 58 active+clean; 582 KiB data, 145 MiB used, 715 GiB / 715 GiB avail; 2/72 objects degraded (2.778%) 2023-09-23T20:50:41.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:50:41 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:50:41.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:50:41 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:50:41.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:50:41 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:50:41.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:50:41 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:50:41.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:50:41 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:50:41.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:50:41 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:50:41.633 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:50:41 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T20:50:41.633 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:50:41 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T20:50:41.633 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:50:41 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T20:50:41.633 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:50:41 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-09-23T20:50:41.633 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:50:41 smithi026 ceph-mon[179688]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-09-23T20:50:41.633 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:50:41 smithi026 ceph-mon[179688]: Upgrade: unsafe to stop osd(s) at this time (5 PGs are or would become offline) 2023-09-23T20:50:41.633 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:50:41 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:50:41.633 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:50:41 smithi026 ceph-mon[175151]: pgmap v180: 73 pgs: 10 peering, 1 active+undersized+degraded, 4 active+undersized, 58 active+clean; 582 KiB data, 145 MiB used, 715 GiB / 715 GiB avail; 2/72 objects degraded (2.778%) 2023-09-23T20:50:41.633 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:50:41 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:50:41.633 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:50:41 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:50:41.634 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:50:41 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:50:41.634 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:50:41 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:50:41.634 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:50:41 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:50:41.634 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:50:41 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:50:41.634 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:50:41 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T20:50:41.634 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:50:41 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T20:50:41.634 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:50:41 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T20:50:41.634 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:50:41 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-09-23T20:50:41.634 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:50:41 smithi026 ceph-mon[175151]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-09-23T20:50:41.634 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:50:41 smithi026 ceph-mon[175151]: Upgrade: unsafe to stop osd(s) at this time (5 PGs are or would become offline) 2023-09-23T20:50:41.634 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:50:41 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:50:42.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:50:42 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/3544925385' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:50:42.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:50:42 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/3544925385' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:50:42.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:50:42 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/3544925385' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:50:43.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:50:43 smithi070 ceph-mon[168282]: pgmap v181: 73 pgs: 73 active+clean; 582 KiB data, 149 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:50:43.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:50:43 smithi070 ceph-mon[168282]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 2/72 objects degraded (2.778%), 1 pg degraded) 2023-09-23T20:50:43.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:50:43 smithi070 ceph-mon[168282]: Cluster is now healthy 2023-09-23T20:50:43.631 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:50:43 smithi026 ceph-mon[175151]: pgmap v181: 73 pgs: 73 active+clean; 582 KiB data, 149 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:50:43.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:50:43 smithi026 ceph-mon[175151]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 2/72 objects degraded (2.778%), 1 pg degraded) 2023-09-23T20:50:43.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:50:43 smithi026 ceph-mon[175151]: Cluster is now healthy 2023-09-23T20:50:43.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:50:43 smithi026 ceph-mon[179688]: pgmap v181: 73 pgs: 73 active+clean; 582 KiB data, 149 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:50:43.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:50:43 smithi026 ceph-mon[179688]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 2/72 objects degraded (2.778%), 1 pg degraded) 2023-09-23T20:50:43.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:50:43 smithi026 ceph-mon[179688]: Cluster is now healthy 2023-09-23T20:50:44.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:50:44 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/11002756' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:50:44.631 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:50:44 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/11002756' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:50:44.631 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:50:44 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/11002756' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:50:46.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:50:45 smithi070 ceph-mon[168282]: pgmap v182: 73 pgs: 73 active+clean; 582 KiB data, 149 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:50:46.131 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:50:45 smithi026 ceph-mon[179688]: pgmap v182: 73 pgs: 73 active+clean; 582 KiB data, 149 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:50:46.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:50:45 smithi026 ceph-mon[175151]: pgmap v182: 73 pgs: 73 active+clean; 582 KiB data, 149 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:50:47.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:50:47 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:50:47.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:50:47 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:50:47.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:50:47 smithi070 ceph-mon[168282]: pgmap v183: 73 pgs: 73 active+clean; 582 KiB data, 149 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:50:47.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:50:47 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/157306506' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:50:47.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:50:47 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:50:47.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:50:47 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:50:47.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:50:47 smithi026 ceph-mon[179688]: pgmap v183: 73 pgs: 73 active+clean; 582 KiB data, 149 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:50:47.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:50:47 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/157306506' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:50:47.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:50:47 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:50:47.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:50:47 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:50:47.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:50:47 smithi026 ceph-mon[175151]: pgmap v183: 73 pgs: 73 active+clean; 582 KiB data, 149 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:50:47.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:50:47 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/157306506' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:50:49.612 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:50:49 smithi026 ceph-mon[175151]: pgmap v184: 73 pgs: 73 active+clean; 582 KiB data, 149 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:50:49.612 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:50:49 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/19617021' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:50:49.613 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:50:49 smithi026 ceph-mon[179688]: pgmap v184: 73 pgs: 73 active+clean; 582 KiB data, 149 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:50:49.613 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:50:49 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/19617021' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:50:49.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:50:49 smithi070 ceph-mon[168282]: pgmap v184: 73 pgs: 73 active+clean; 582 KiB data, 149 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:50:49.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:50:49 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/19617021' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:50:49.882 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:50:49 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[164029]: ::ffff:172.21.15.70 - - [23/Sep/2023:20:50:49] "GET /metrics HTTP/1.1" 200 33816 "" "Prometheus/2.43.0" 2023-09-23T20:50:52.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:50:51 smithi070 ceph-mon[168282]: pgmap v185: 73 pgs: 73 active+clean; 582 KiB data, 149 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:50:52.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:50:51 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/379126747' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:50:52.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:50:51 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:50:52.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:50:51 smithi026 ceph-mon[179688]: pgmap v185: 73 pgs: 73 active+clean; 582 KiB data, 149 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:50:52.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:50:51 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/379126747' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:50:52.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:50:51 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:50:52.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:50:51 smithi026 ceph-mon[175151]: pgmap v185: 73 pgs: 73 active+clean; 582 KiB data, 149 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:50:52.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:50:51 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/379126747' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:50:52.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:50:51 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:50:54.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:50:53 smithi070 ceph-mon[168282]: pgmap v186: 73 pgs: 73 active+clean; 582 KiB data, 149 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:50:54.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:50:53 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/1082716783' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:50:54.131 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:50:53 smithi026 ceph-mon[179688]: pgmap v186: 73 pgs: 73 active+clean; 582 KiB data, 149 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:50:54.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:50:53 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/1082716783' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:50:54.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:50:53 smithi026 ceph-mon[175151]: pgmap v186: 73 pgs: 73 active+clean; 582 KiB data, 149 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:50:54.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:50:53 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/1082716783' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:50:56.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:50:55 smithi070 ceph-mon[168282]: pgmap v187: 73 pgs: 73 active+clean; 582 KiB data, 149 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:50:56.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:50:55 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-09-23T20:50:56.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:50:55 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/952405528' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:50:56.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:50:55 smithi026 ceph-mon[179688]: pgmap v187: 73 pgs: 73 active+clean; 582 KiB data, 149 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:50:56.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:50:55 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-09-23T20:50:56.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:50:55 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/952405528' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:50:56.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:50:55 smithi026 ceph-mon[175151]: pgmap v187: 73 pgs: 73 active+clean; 582 KiB data, 149 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:50:56.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:50:55 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-09-23T20:50:56.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:50:55 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/952405528' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:50:57.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:50:56 smithi070 ceph-mon[168282]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-09-23T20:50:57.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:50:56 smithi070 ceph-mon[168282]: Upgrade: osd.3 is safe to restart 2023-09-23T20:50:57.131 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:50:56 smithi026 ceph-mon[179688]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-09-23T20:50:57.131 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:50:56 smithi026 ceph-mon[179688]: Upgrade: osd.3 is safe to restart 2023-09-23T20:50:57.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:50:56 smithi026 ceph-mon[175151]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-09-23T20:50:57.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:50:56 smithi026 ceph-mon[175151]: Upgrade: osd.3 is safe to restart 2023-09-23T20:50:58.071 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:50:57 smithi026 ceph-mon[179688]: pgmap v188: 73 pgs: 73 active+clean; 582 KiB data, 149 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:50:58.071 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:50:57 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:50:58.071 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:50:57 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2023-09-23T20:50:58.071 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:50:57 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:50:58.072 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:50:57 smithi026 ceph-mon[175151]: pgmap v188: 73 pgs: 73 active+clean; 582 KiB data, 149 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:50:58.072 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:50:57 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:50:58.072 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:50:57 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2023-09-23T20:50:58.072 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:50:57 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:50:58.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:50:57 smithi070 ceph-mon[168282]: pgmap v188: 73 pgs: 73 active+clean; 582 KiB data, 149 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:50:58.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:50:57 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:50:58.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:50:57 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2023-09-23T20:50:58.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:50:57 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:50:58.785 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:50:58 smithi026 ceph-mon[175151]: Upgrade: Updating osd.3 2023-09-23T20:50:59.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:50:58 smithi070 ceph-mon[168282]: Upgrade: Updating osd.3 2023-09-23T20:50:59.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:50:58 smithi070 ceph-mon[168282]: Deploying daemon osd.3 on smithi026 2023-09-23T20:50:59.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:50:58 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/1901544730' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:50:59.131 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:50:58 smithi026 ceph-mon[175151]: Deploying daemon osd.3 on smithi026 2023-09-23T20:50:59.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:50:58 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/1901544730' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:50:59.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:50:58 smithi026 ceph-mon[179688]: Upgrade: Updating osd.3 2023-09-23T20:50:59.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:50:58 smithi026 ceph-mon[179688]: Deploying daemon osd.3 on smithi026 2023-09-23T20:50:59.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:50:58 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/1901544730' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:50:59.786 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:50:59 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[164029]: ::ffff:172.21.15.70 - - [23/Sep/2023:20:50:59] "GET /metrics HTTP/1.1" 200 33816 "" "Prometheus/2.43.0" 2023-09-23T20:50:59.786 INFO:journalctl@ceph.osd.3.smithi026.stdout:Sep 23 20:50:59 smithi026 systemd[1]: Stopping Ceph osd.3 for b3f8e4ec-5a50-11ee-8db4-212e2dc638e7... 2023-09-23T20:51:00.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:50:59 smithi070 ceph-mon[168282]: pgmap v189: 73 pgs: 73 active+clean; 582 KiB data, 149 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:51:00.131 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:50:59 smithi026 ceph-mon[175151]: pgmap v189: 73 pgs: 73 active+clean; 582 KiB data, 149 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:51:00.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:50:59 smithi026 ceph-mon[179688]: pgmap v189: 73 pgs: 73 active+clean; 582 KiB data, 149 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:51:00.132 INFO:journalctl@ceph.osd.3.smithi026.stdout:Sep 23 20:50:59 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd-3[135828]: 2023-09-23T20:50:59.935+0000 7f306770b700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-osd -n osd.3 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2023-09-23T20:51:00.132 INFO:journalctl@ceph.osd.3.smithi026.stdout:Sep 23 20:50:59 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd-3[135828]: 2023-09-23T20:50:59.935+0000 7f306770b700 -1 osd.3 94 *** Got signal Terminated *** 2023-09-23T20:51:00.132 INFO:journalctl@ceph.osd.3.smithi026.stdout:Sep 23 20:50:59 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd-3[135828]: 2023-09-23T20:50:59.935+0000 7f306770b700 -1 osd.3 94 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-09-23T20:51:01.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:51:00 smithi070 ceph-mon[168282]: osd.3 marked itself down and dead 2023-09-23T20:51:01.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:51:00 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/1890241182' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:51:01.131 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:51:00 smithi026 ceph-mon[175151]: osd.3 marked itself down and dead 2023-09-23T20:51:01.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:51:00 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/1890241182' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:51:01.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:51:00 smithi026 ceph-mon[179688]: osd.3 marked itself down and dead 2023-09-23T20:51:01.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:51:00 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/1890241182' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:51:01.882 INFO:journalctl@ceph.osd.3.smithi026.stdout:Sep 23 20:51:01 smithi026 bash[207954]: ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd-3 2023-09-23T20:51:01.883 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:51:01 smithi026 ceph-mon[175151]: pgmap v190: 73 pgs: 73 active+clean; 582 KiB data, 149 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:51:01.883 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:51:01 smithi026 ceph-mon[175151]: Health check failed: 1 osds down (OSD_DOWN) 2023-09-23T20:51:01.883 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:51:01 smithi026 ceph-mon[175151]: failed to encode map e95 with expected crc 2023-09-23T20:51:01.883 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:51:01 smithi026 ceph-mon[175151]: failed to encode map e95 with expected crc 2023-09-23T20:51:01.883 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:51:01 smithi026 ceph-mon[175151]: osdmap e95: 8 total, 7 up, 8 in 2023-09-23T20:51:01.883 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:51:01 smithi026 ceph-mon[175151]: failed to encode map e95 with expected crc 2023-09-23T20:51:01.883 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:51:01 smithi026 ceph-mon[175151]: failed to encode map e95 with expected crc 2023-09-23T20:51:01.883 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:51:01 smithi026 ceph-mon[175151]: failed to encode map e95 with expected crc 2023-09-23T20:51:01.883 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:51:01 smithi026 ceph-mon[175151]: failed to encode map e95 with expected crc 2023-09-23T20:51:01.883 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:51:01 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:51:01.883 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:51:01 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:51:01.884 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:51:01 smithi026 ceph-mon[179688]: pgmap v190: 73 pgs: 73 active+clean; 582 KiB data, 149 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:51:01.884 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:51:01 smithi026 ceph-mon[179688]: Health check failed: 1 osds down (OSD_DOWN) 2023-09-23T20:51:01.884 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:51:01 smithi026 ceph-mon[179688]: failed to encode map e95 with expected crc 2023-09-23T20:51:01.884 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:51:01 smithi026 ceph-mon[179688]: failed to encode map e95 with expected crc 2023-09-23T20:51:01.884 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:51:01 smithi026 ceph-mon[179688]: osdmap e95: 8 total, 7 up, 8 in 2023-09-23T20:51:01.884 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:51:01 smithi026 ceph-mon[179688]: failed to encode map e95 with expected crc 2023-09-23T20:51:01.884 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:51:01 smithi026 ceph-mon[179688]: failed to encode map e95 with expected crc 2023-09-23T20:51:01.884 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:51:01 smithi026 ceph-mon[179688]: failed to encode map e95 with expected crc 2023-09-23T20:51:01.884 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:51:01 smithi026 ceph-mon[179688]: failed to encode map e95 with expected crc 2023-09-23T20:51:01.884 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:51:01 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:51:01.884 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:51:01 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:51:02.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:51:01 smithi070 ceph-mon[168282]: pgmap v190: 73 pgs: 73 active+clean; 582 KiB data, 149 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:51:02.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:51:01 smithi070 ceph-mon[168282]: Health check failed: 1 osds down (OSD_DOWN) 2023-09-23T20:51:02.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:51:01 smithi070 ceph-mon[168282]: failed to encode map e95 with expected crc 2023-09-23T20:51:02.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:51:01 smithi070 ceph-mon[168282]: failed to encode map e95 with expected crc 2023-09-23T20:51:02.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:51:01 smithi070 ceph-mon[168282]: osdmap e95: 8 total, 7 up, 8 in 2023-09-23T20:51:02.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:51:01 smithi070 ceph-mon[168282]: failed to encode map e95 with expected crc 2023-09-23T20:51:02.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:51:01 smithi070 ceph-mon[168282]: failed to encode map e95 with expected crc 2023-09-23T20:51:02.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:51:01 smithi070 ceph-mon[168282]: failed to encode map e95 with expected crc 2023-09-23T20:51:02.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:51:01 smithi070 ceph-mon[168282]: failed to encode map e95 with expected crc 2023-09-23T20:51:02.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:51:01 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:51:02.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:51:01 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:51:03.006 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:51:02 smithi026 ceph-mon[175151]: failed to encode map e96 with expected crc 2023-09-23T20:51:03.006 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:51:02 smithi026 ceph-mon[175151]: osdmap e96: 8 total, 7 up, 8 in 2023-09-23T20:51:03.006 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:51:02 smithi026 ceph-mon[175151]: failed to encode map e96 with expected crc 2023-09-23T20:51:03.006 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:51:02 smithi026 ceph-mon[175151]: failed to encode map e96 with expected crc 2023-09-23T20:51:03.006 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:51:02 smithi026 ceph-mon[175151]: failed to encode map e96 with expected crc 2023-09-23T20:51:03.007 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:51:02 smithi026 ceph-mon[175151]: failed to encode map e96 with expected crc 2023-09-23T20:51:03.007 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:51:02 smithi026 ceph-mon[175151]: failed to encode map e96 with expected crc 2023-09-23T20:51:03.007 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:51:02 smithi026 ceph-mon[175151]: failed to encode map e96 with expected crc 2023-09-23T20:51:03.007 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:51:02 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/1087095698' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:51:03.007 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:51:02 smithi026 ceph-mon[179688]: failed to encode map e96 with expected crc 2023-09-23T20:51:03.007 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:51:02 smithi026 ceph-mon[179688]: osdmap e96: 8 total, 7 up, 8 in 2023-09-23T20:51:03.007 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:51:02 smithi026 ceph-mon[179688]: failed to encode map e96 with expected crc 2023-09-23T20:51:03.007 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:51:02 smithi026 ceph-mon[179688]: failed to encode map e96 with expected crc 2023-09-23T20:51:03.007 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:51:02 smithi026 ceph-mon[179688]: failed to encode map e96 with expected crc 2023-09-23T20:51:03.007 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:51:02 smithi026 ceph-mon[179688]: failed to encode map e96 with expected crc 2023-09-23T20:51:03.007 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:51:02 smithi026 ceph-mon[179688]: failed to encode map e96 with expected crc 2023-09-23T20:51:03.008 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:51:02 smithi026 ceph-mon[179688]: failed to encode map e96 with expected crc 2023-09-23T20:51:03.008 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:51:02 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/1087095698' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:51:03.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:51:02 smithi070 ceph-mon[168282]: failed to encode map e96 with expected crc 2023-09-23T20:51:03.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:51:02 smithi070 ceph-mon[168282]: osdmap e96: 8 total, 7 up, 8 in 2023-09-23T20:51:03.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:51:02 smithi070 ceph-mon[168282]: failed to encode map e96 with expected crc 2023-09-23T20:51:03.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:51:02 smithi070 ceph-mon[168282]: failed to encode map e96 with expected crc 2023-09-23T20:51:03.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:51:02 smithi070 ceph-mon[168282]: failed to encode map e96 with expected crc 2023-09-23T20:51:03.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:51:02 smithi070 ceph-mon[168282]: failed to encode map e96 with expected crc 2023-09-23T20:51:03.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:51:02 smithi070 ceph-mon[168282]: failed to encode map e96 with expected crc 2023-09-23T20:51:03.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:51:02 smithi070 ceph-mon[168282]: failed to encode map e96 with expected crc 2023-09-23T20:51:03.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:51:02 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/1087095698' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:51:03.382 INFO:journalctl@ceph.osd.3.smithi026.stdout:Sep 23 20:51:03 smithi026 systemd[1]: ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7@osd.3.service: Succeeded. 2023-09-23T20:51:03.382 INFO:journalctl@ceph.osd.3.smithi026.stdout:Sep 23 20:51:03 smithi026 systemd[1]: Stopped Ceph osd.3 for b3f8e4ec-5a50-11ee-8db4-212e2dc638e7. 2023-09-23T20:51:03.382 INFO:journalctl@ceph.osd.3.smithi026.stdout:Sep 23 20:51:03 smithi026 systemd[1]: Starting Ceph osd.3 for b3f8e4ec-5a50-11ee-8db4-212e2dc638e7... 2023-09-23T20:51:03.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:51:03 smithi026 ceph-mon[175151]: pgmap v193: 73 pgs: 2 active+undersized+degraded, 3 peering, 6 stale+active+clean, 12 active+undersized, 50 active+clean; 582 KiB data, 149 MiB used, 715 GiB / 715 GiB avail; 3/72 objects degraded (4.167%) 2023-09-23T20:51:03.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:51:03 smithi026 ceph-mon[175151]: Health check failed: Degraded data redundancy: 3/72 objects degraded (4.167%), 2 pgs degraded (PG_DEGRADED) 2023-09-23T20:51:03.883 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:51:03 smithi026 ceph-mon[179688]: pgmap v193: 73 pgs: 2 active+undersized+degraded, 3 peering, 6 stale+active+clean, 12 active+undersized, 50 active+clean; 582 KiB data, 149 MiB used, 715 GiB / 715 GiB avail; 3/72 objects degraded (4.167%) 2023-09-23T20:51:03.883 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:51:03 smithi026 ceph-mon[179688]: Health check failed: Degraded data redundancy: 3/72 objects degraded (4.167%), 2 pgs degraded (PG_DEGRADED) 2023-09-23T20:51:04.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:51:03 smithi070 ceph-mon[168282]: pgmap v193: 73 pgs: 2 active+undersized+degraded, 3 peering, 6 stale+active+clean, 12 active+undersized, 50 active+clean; 582 KiB data, 149 MiB used, 715 GiB / 715 GiB avail; 3/72 objects degraded (4.167%) 2023-09-23T20:51:04.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:51:03 smithi070 ceph-mon[168282]: Health check failed: Degraded data redundancy: 3/72 objects degraded (4.167%), 2 pgs degraded (PG_DEGRADED) 2023-09-23T20:51:04.382 INFO:journalctl@ceph.osd.3.smithi026.stdout:Sep 23 20:51:04 smithi026 bash[208283]: --> Failed to activate via raw: did not find any matching OSD to activate 2023-09-23T20:51:04.382 INFO:journalctl@ceph.osd.3.smithi026.stdout:Sep 23 20:51:04 smithi026 bash[208283]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2023-09-23T20:51:04.382 INFO:journalctl@ceph.osd.3.smithi026.stdout:Sep 23 20:51:04 smithi026 bash[208283]: Running command: /usr/bin/ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev /dev/vg_nvme/lv_1 --path /var/lib/ceph/osd/ceph-3 --no-mon-config 2023-09-23T20:51:04.382 INFO:journalctl@ceph.osd.3.smithi026.stdout:Sep 23 20:51:04 smithi026 bash[208283]: Running command: /usr/bin/ln -snf /dev/vg_nvme/lv_1 /var/lib/ceph/osd/ceph-3/block 2023-09-23T20:51:04.382 INFO:journalctl@ceph.osd.3.smithi026.stdout:Sep 23 20:51:04 smithi026 bash[208283]: Running command: /usr/bin/chown -h ceph:ceph /var/lib/ceph/osd/ceph-3/block 2023-09-23T20:51:04.382 INFO:journalctl@ceph.osd.3.smithi026.stdout:Sep 23 20:51:04 smithi026 bash[208283]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-0 2023-09-23T20:51:04.382 INFO:journalctl@ceph.osd.3.smithi026.stdout:Sep 23 20:51:04 smithi026 bash[208283]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2023-09-23T20:51:04.382 INFO:journalctl@ceph.osd.3.smithi026.stdout:Sep 23 20:51:04 smithi026 bash[208283]: --> ceph-volume lvm activate successful for osd ID: 3 2023-09-23T20:51:05.131 INFO:journalctl@ceph.osd.3.smithi026.stdout:Sep 23 20:51:04 smithi026 podman[208556]: 2023-09-23T20:51:05.132 INFO:journalctl@ceph.osd.3.smithi026.stdout:Sep 23 20:51:04 smithi026 ceph-osd[208616]: -- 172.21.15.26:0/503207595 <== mon.2 v2:172.21.15.26:3301/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x5595f48384e0 con 0x5595f4809000 2023-09-23T20:51:05.132 INFO:journalctl@ceph.osd.3.smithi026.stdout:Sep 23 20:51:05 smithi026 bash[208556]: 745262093b5fcf222dfd403817072b87f700ba12be7e7e886f6e4a84c58b5d3b 2023-09-23T20:51:05.132 INFO:journalctl@ceph.osd.3.smithi026.stdout:Sep 23 20:51:05 smithi026 systemd[1]: Started Ceph osd.3 for b3f8e4ec-5a50-11ee-8db4-212e2dc638e7. 2023-09-23T20:51:05.804 INFO:journalctl@ceph.osd.3.smithi026.stdout:Sep 23 20:51:05 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd-3[208592]: 2023-09-23T20:51:05.526+0000 7fc52e381640 -1 Falling back to public interface 2023-09-23T20:51:06.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:51:05 smithi070 ceph-mon[168282]: pgmap v194: 73 pgs: 2 active+undersized+degraded, 3 peering, 2 stale+active+clean, 19 active+undersized, 47 active+clean; 582 KiB data, 149 MiB used, 715 GiB / 715 GiB avail; 3/72 objects degraded (4.167%) 2023-09-23T20:51:06.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:51:05 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/763251469' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:51:06.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:51:05 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:51:06.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:51:05 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:51:06.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:51:05 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:51:06.131 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:51:05 smithi026 ceph-mon[179688]: pgmap v194: 73 pgs: 2 active+undersized+degraded, 3 peering, 2 stale+active+clean, 19 active+undersized, 47 active+clean; 582 KiB data, 149 MiB used, 715 GiB / 715 GiB avail; 3/72 objects degraded (4.167%) 2023-09-23T20:51:06.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:51:05 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/763251469' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:51:06.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:51:05 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:51:06.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:51:05 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:51:06.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:51:05 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:51:06.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:51:05 smithi026 ceph-mon[175151]: pgmap v194: 73 pgs: 2 active+undersized+degraded, 3 peering, 2 stale+active+clean, 19 active+undersized, 47 active+clean; 582 KiB data, 149 MiB used, 715 GiB / 715 GiB avail; 3/72 objects degraded (4.167%) 2023-09-23T20:51:06.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:51:05 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/763251469' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:51:06.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:51:05 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:51:06.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:51:05 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:51:06.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:51:05 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:51:08.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:51:07 smithi070 ceph-mon[168282]: pgmap v195: 73 pgs: 3 active+undersized+degraded, 28 active+undersized, 42 active+clean; 582 KiB data, 149 MiB used, 715 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-09-23T20:51:08.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:51:07 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/1315779365' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:51:08.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:51:07 smithi026 ceph-mon[175151]: pgmap v195: 73 pgs: 3 active+undersized+degraded, 28 active+undersized, 42 active+clean; 582 KiB data, 149 MiB used, 715 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-09-23T20:51:08.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:51:07 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/1315779365' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:51:08.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:51:07 smithi026 ceph-mon[179688]: pgmap v195: 73 pgs: 3 active+undersized+degraded, 28 active+undersized, 42 active+clean; 582 KiB data, 149 MiB used, 715 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-09-23T20:51:08.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:51:07 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/1315779365' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:51:08.632 INFO:journalctl@ceph.osd.3.smithi026.stdout:Sep 23 20:51:08 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd-3[208592]: 2023-09-23T20:51:08.278+0000 7fc52e381640 -1 osd.3 94 log_to_monitors true 2023-09-23T20:51:09.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:51:08 smithi070 ceph-mon[168282]: from='osd.3 [v2:172.21.15.26:6826/2947396145,v1:172.21.15.26:6827/2947396145]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2023-09-23T20:51:09.131 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:51:08 smithi026 ceph-mon[175151]: from='osd.3 [v2:172.21.15.26:6826/2947396145,v1:172.21.15.26:6827/2947396145]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2023-09-23T20:51:09.132 INFO:journalctl@ceph.osd.3.smithi026.stdout:Sep 23 20:51:08 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd-3[208592]: 2023-09-23T20:51:08.838+0000 7fc51d7ea700 -1 osd.3 94 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-09-23T20:51:09.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:51:08 smithi026 ceph-mon[179688]: from='osd.3 [v2:172.21.15.26:6826/2947396145,v1:172.21.15.26:6827/2947396145]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2023-09-23T20:51:09.881 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:51:09 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[164029]: ::ffff:172.21.15.70 - - [23/Sep/2023:20:51:09] "GET /metrics HTTP/1.1" 200 33820 "" "Prometheus/2.43.0" 2023-09-23T20:51:09.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:51:09 smithi026 ceph-mon[179688]: pgmap v196: 73 pgs: 3 active+undersized+degraded, 28 active+undersized, 42 active+clean; 582 KiB data, 149 MiB used, 715 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-09-23T20:51:09.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:51:09 smithi026 ceph-mon[179688]: failed to encode map e97 with expected crc 2023-09-23T20:51:09.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:51:09 smithi026 ceph-mon[179688]: from='osd.3 [v2:172.21.15.26:6826/2947396145,v1:172.21.15.26:6827/2947396145]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2023-09-23T20:51:09.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:51:09 smithi026 ceph-mon[179688]: osdmap e97: 8 total, 7 up, 8 in 2023-09-23T20:51:09.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:51:09 smithi026 ceph-mon[179688]: from='osd.3 [v2:172.21.15.26:6826/2947396145,v1:172.21.15.26:6827/2947396145]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi026", "root=default"]}]: dispatch 2023-09-23T20:51:09.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:51:09 smithi026 ceph-mon[179688]: failed to encode map e97 with expected crc 2023-09-23T20:51:09.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:51:09 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:51:09.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:51:09 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:51:09.883 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:51:09 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/799534421' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:51:09.883 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:51:09 smithi026 ceph-mon[179688]: Health check update: Degraded data redundancy: 4/72 objects degraded (5.556%), 3 pgs degraded (PG_DEGRADED) 2023-09-23T20:51:09.883 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:51:09 smithi026 ceph-mon[175151]: pgmap v196: 73 pgs: 3 active+undersized+degraded, 28 active+undersized, 42 active+clean; 582 KiB data, 149 MiB used, 715 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-09-23T20:51:09.883 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:51:09 smithi026 ceph-mon[175151]: failed to encode map e97 with expected crc 2023-09-23T20:51:09.883 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:51:09 smithi026 ceph-mon[175151]: from='osd.3 [v2:172.21.15.26:6826/2947396145,v1:172.21.15.26:6827/2947396145]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2023-09-23T20:51:09.883 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:51:09 smithi026 ceph-mon[175151]: osdmap e97: 8 total, 7 up, 8 in 2023-09-23T20:51:09.883 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:51:09 smithi026 ceph-mon[175151]: from='osd.3 [v2:172.21.15.26:6826/2947396145,v1:172.21.15.26:6827/2947396145]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi026", "root=default"]}]: dispatch 2023-09-23T20:51:09.883 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:51:09 smithi026 ceph-mon[175151]: failed to encode map e97 with expected crc 2023-09-23T20:51:09.883 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:51:09 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:51:09.883 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:51:09 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:51:09.884 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:51:09 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/799534421' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:51:09.884 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:51:09 smithi026 ceph-mon[175151]: Health check update: Degraded data redundancy: 4/72 objects degraded (5.556%), 3 pgs degraded (PG_DEGRADED) 2023-09-23T20:51:10.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:51:09 smithi070 ceph-mon[168282]: pgmap v196: 73 pgs: 3 active+undersized+degraded, 28 active+undersized, 42 active+clean; 582 KiB data, 149 MiB used, 715 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-09-23T20:51:10.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:51:09 smithi070 ceph-mon[168282]: failed to encode map e97 with expected crc 2023-09-23T20:51:10.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:51:09 smithi070 ceph-mon[168282]: from='osd.3 [v2:172.21.15.26:6826/2947396145,v1:172.21.15.26:6827/2947396145]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2023-09-23T20:51:10.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:51:09 smithi070 ceph-mon[168282]: osdmap e97: 8 total, 7 up, 8 in 2023-09-23T20:51:10.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:51:09 smithi070 ceph-mon[168282]: from='osd.3 [v2:172.21.15.26:6826/2947396145,v1:172.21.15.26:6827/2947396145]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi026", "root=default"]}]: dispatch 2023-09-23T20:51:10.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:51:09 smithi070 ceph-mon[168282]: failed to encode map e97 with expected crc 2023-09-23T20:51:10.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:51:09 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:51:10.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:51:09 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:51:10.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:51:09 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/799534421' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:51:10.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:51:09 smithi070 ceph-mon[168282]: Health check update: Degraded data redundancy: 4/72 objects degraded (5.556%), 3 pgs degraded (PG_DEGRADED) 2023-09-23T20:51:11.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:51:10 smithi070 ceph-mon[168282]: failed to encode map e97 with expected crc 2023-09-23T20:51:11.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:51:10 smithi070 ceph-mon[168282]: failed to encode map e97 with expected crc 2023-09-23T20:51:11.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:51:10 smithi070 ceph-mon[168282]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-09-23T20:51:11.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:51:10 smithi070 ceph-mon[168282]: failed to encode map e98 with expected crc 2023-09-23T20:51:11.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:51:10 smithi070 ceph-mon[168282]: failed to encode map e98 with expected crc 2023-09-23T20:51:11.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:51:10 smithi070 ceph-mon[168282]: osd.3 [v2:172.21.15.26:6826/2947396145,v1:172.21.15.26:6827/2947396145] boot 2023-09-23T20:51:11.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:51:10 smithi070 ceph-mon[168282]: osdmap e98: 8 total, 8 up, 8 in 2023-09-23T20:51:11.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:51:10 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-23T20:51:11.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:51:10 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:51:11.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:51:10 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:51:11.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:51:10 smithi026 ceph-mon[175151]: failed to encode map e97 with expected crc 2023-09-23T20:51:11.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:51:10 smithi026 ceph-mon[175151]: failed to encode map e97 with expected crc 2023-09-23T20:51:11.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:51:10 smithi026 ceph-mon[175151]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-09-23T20:51:11.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:51:10 smithi026 ceph-mon[175151]: failed to encode map e98 with expected crc 2023-09-23T20:51:11.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:51:10 smithi026 ceph-mon[175151]: failed to encode map e98 with expected crc 2023-09-23T20:51:11.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:51:10 smithi026 ceph-mon[175151]: osd.3 [v2:172.21.15.26:6826/2947396145,v1:172.21.15.26:6827/2947396145] boot 2023-09-23T20:51:11.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:51:10 smithi026 ceph-mon[175151]: osdmap e98: 8 total, 8 up, 8 in 2023-09-23T20:51:11.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:51:10 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-23T20:51:11.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:51:10 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:51:11.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:51:10 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:51:11.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:51:10 smithi026 ceph-mon[179688]: failed to encode map e97 with expected crc 2023-09-23T20:51:11.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:51:10 smithi026 ceph-mon[179688]: failed to encode map e97 with expected crc 2023-09-23T20:51:11.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:51:10 smithi026 ceph-mon[179688]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-09-23T20:51:11.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:51:10 smithi026 ceph-mon[179688]: failed to encode map e98 with expected crc 2023-09-23T20:51:11.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:51:10 smithi026 ceph-mon[179688]: failed to encode map e98 with expected crc 2023-09-23T20:51:11.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:51:10 smithi026 ceph-mon[179688]: osd.3 [v2:172.21.15.26:6826/2947396145,v1:172.21.15.26:6827/2947396145] boot 2023-09-23T20:51:11.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:51:10 smithi026 ceph-mon[179688]: osdmap e98: 8 total, 8 up, 8 in 2023-09-23T20:51:11.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:51:10 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-23T20:51:11.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:51:10 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:51:11.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:51:10 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:51:12.018 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 18.2.0-460-gbb0daeff (bb0daeff97f020f686596fca6eba20c310e26fb7) reef (stable)": 4 2023-09-23T20:51:12.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:51:11 smithi070 ceph-mon[168282]: pgmap v199: 73 pgs: 3 active+undersized+degraded, 28 active+undersized, 42 active+clean; 582 KiB data, 170 MiB used, 715 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-09-23T20:51:12.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:51:11 smithi070 ceph-mon[168282]: failed to encode map e99 with expected crc 2023-09-23T20:51:12.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:51:11 smithi070 ceph-mon[168282]: failed to encode map e99 with expected crc 2023-09-23T20:51:12.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:51:11 smithi070 ceph-mon[168282]: osdmap e99: 8 total, 8 up, 8 in 2023-09-23T20:51:12.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:51:11 smithi070 ceph-mon[168282]: failed to encode map e99 with expected crc 2023-09-23T20:51:12.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:51:11 smithi070 ceph-mon[168282]: failed to encode map e99 with expected crc 2023-09-23T20:51:12.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:51:11 smithi070 ceph-mon[168282]: failed to encode map e99 with expected crc 2023-09-23T20:51:12.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:51:11 smithi070 ceph-mon[168282]: failed to encode map e99 with expected crc 2023-09-23T20:51:12.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:51:11 smithi070 ceph-mon[168282]: failed to encode map e99 with expected crc 2023-09-23T20:51:12.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:51:11 smithi070 ceph-mon[168282]: failed to encode map e99 with expected crc 2023-09-23T20:51:12.131 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:51:11 smithi026 ceph-mon[179688]: pgmap v199: 73 pgs: 3 active+undersized+degraded, 28 active+undersized, 42 active+clean; 582 KiB data, 170 MiB used, 715 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-09-23T20:51:12.131 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:51:11 smithi026 ceph-mon[179688]: failed to encode map e99 with expected crc 2023-09-23T20:51:12.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:51:11 smithi026 ceph-mon[179688]: failed to encode map e99 with expected crc 2023-09-23T20:51:12.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:51:11 smithi026 ceph-mon[179688]: osdmap e99: 8 total, 8 up, 8 in 2023-09-23T20:51:12.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:51:11 smithi026 ceph-mon[179688]: failed to encode map e99 with expected crc 2023-09-23T20:51:12.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:51:11 smithi026 ceph-mon[179688]: failed to encode map e99 with expected crc 2023-09-23T20:51:12.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:51:11 smithi026 ceph-mon[179688]: failed to encode map e99 with expected crc 2023-09-23T20:51:12.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:51:11 smithi026 ceph-mon[179688]: failed to encode map e99 with expected crc 2023-09-23T20:51:12.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:51:11 smithi026 ceph-mon[179688]: failed to encode map e99 with expected crc 2023-09-23T20:51:12.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:51:11 smithi026 ceph-mon[179688]: failed to encode map e99 with expected crc 2023-09-23T20:51:12.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:51:11 smithi026 ceph-mon[175151]: pgmap v199: 73 pgs: 3 active+undersized+degraded, 28 active+undersized, 42 active+clean; 582 KiB data, 170 MiB used, 715 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-09-23T20:51:12.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:51:11 smithi026 ceph-mon[175151]: failed to encode map e99 with expected crc 2023-09-23T20:51:12.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:51:11 smithi026 ceph-mon[175151]: failed to encode map e99 with expected crc 2023-09-23T20:51:12.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:51:11 smithi026 ceph-mon[175151]: osdmap e99: 8 total, 8 up, 8 in 2023-09-23T20:51:12.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:51:11 smithi026 ceph-mon[175151]: failed to encode map e99 with expected crc 2023-09-23T20:51:12.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:51:11 smithi026 ceph-mon[175151]: failed to encode map e99 with expected crc 2023-09-23T20:51:12.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:51:11 smithi026 ceph-mon[175151]: failed to encode map e99 with expected crc 2023-09-23T20:51:12.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:51:11 smithi026 ceph-mon[175151]: failed to encode map e99 with expected crc 2023-09-23T20:51:12.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:51:11 smithi026 ceph-mon[175151]: failed to encode map e99 with expected crc 2023-09-23T20:51:12.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:51:11 smithi026 ceph-mon[175151]: failed to encode map e99 with expected crc 2023-09-23T20:51:12.515 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -e sha1=bb0daeff97f020f686596fca6eba20c310e26fb7 -- bash -c 'ceph orch upgrade pause' 2023-09-23T20:51:12.827 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:51:12 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/2895464428' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:51:13.066 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:51:13.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:51:12 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/2895464428' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:51:13.131 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:51:12 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/2895464428' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:51:13.828 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:51:13 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[164029]: 2023-09-23T20:51:13.678+0000 7fd59efc8700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (5 PGs are or would become offline) 2023-09-23T20:51:14.092 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:51:13 smithi026 ceph-mon[179688]: pgmap v201: 73 pgs: 5 peering, 2 active+undersized+degraded, 14 active+undersized, 52 active+clean; 582 KiB data, 171 MiB used, 715 GiB / 715 GiB avail; 2/72 objects degraded (2.778%) 2023-09-23T20:51:14.093 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:51:13 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:51:14.093 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:51:13 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:51:14.093 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:51:13 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:51:14.093 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:51:13 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:51:14.093 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:51:13 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:51:14.093 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:51:13 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:51:14.093 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:51:13 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T20:51:14.093 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:51:13 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T20:51:14.093 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:51:13 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T20:51:14.093 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:51:13 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2023-09-23T20:51:14.094 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:51:13 smithi026 ceph-mon[175151]: pgmap v201: 73 pgs: 5 peering, 2 active+undersized+degraded, 14 active+undersized, 52 active+clean; 582 KiB data, 171 MiB used, 715 GiB / 715 GiB avail; 2/72 objects degraded (2.778%) 2023-09-23T20:51:14.094 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:51:13 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:51:14.094 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:51:13 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:51:14.094 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:51:13 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:51:14.094 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:51:13 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:51:14.094 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:51:13 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:51:14.094 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:51:13 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:51:14.094 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:51:13 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T20:51:14.094 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:51:13 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T20:51:14.094 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:51:13 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T20:51:14.095 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:51:13 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2023-09-23T20:51:14.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:51:13 smithi070 ceph-mon[168282]: pgmap v201: 73 pgs: 5 peering, 2 active+undersized+degraded, 14 active+undersized, 52 active+clean; 582 KiB data, 171 MiB used, 715 GiB / 715 GiB avail; 2/72 objects degraded (2.778%) 2023-09-23T20:51:14.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:51:13 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:51:14.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:51:13 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:51:14.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:51:13 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:51:14.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:51:13 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:51:14.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:51:13 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:51:14.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:51:13 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:51:14.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:51:13 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T20:51:14.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:51:13 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T20:51:14.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:51:13 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T20:51:14.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:51:13 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2023-09-23T20:51:14.830 INFO:teuthology.orchestra.run.smithi026.stdout:Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d 2023-09-23T20:51:15.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:51:14 smithi070 ceph-mon[168282]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2023-09-23T20:51:15.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:51:14 smithi070 ceph-mon[168282]: Upgrade: unsafe to stop osd(s) at this time (5 PGs are or would become offline) 2023-09-23T20:51:15.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:51:14 smithi070 ceph-mon[168282]: Health check update: Degraded data redundancy: 2/72 objects degraded (2.778%), 2 pgs degraded (PG_DEGRADED) 2023-09-23T20:51:15.131 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:51:14 smithi026 ceph-mon[175151]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2023-09-23T20:51:15.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:51:14 smithi026 ceph-mon[175151]: Upgrade: unsafe to stop osd(s) at this time (5 PGs are or would become offline) 2023-09-23T20:51:15.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:51:14 smithi026 ceph-mon[175151]: Health check update: Degraded data redundancy: 2/72 objects degraded (2.778%), 2 pgs degraded (PG_DEGRADED) 2023-09-23T20:51:15.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:51:14 smithi026 ceph-mon[179688]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2023-09-23T20:51:15.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:51:14 smithi026 ceph-mon[179688]: Upgrade: unsafe to stop osd(s) at this time (5 PGs are or would become offline) 2023-09-23T20:51:15.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:51:14 smithi026 ceph-mon[179688]: Health check update: Degraded data redundancy: 2/72 objects degraded (2.778%), 2 pgs degraded (PG_DEGRADED) 2023-09-23T20:51:15.261 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -e sha1=bb0daeff97f020f686596fca6eba20c310e26fb7 -- bash -c 'ceph orch ps' 2023-09-23T20:51:15.871 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:51:16.108 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:51:15 smithi026 ceph-mon[175151]: pgmap v202: 73 pgs: 5 peering, 68 active+clean; 582 KiB data, 171 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:51:16.108 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:51:15 smithi026 ceph-mon[175151]: from='client.34566 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T20:51:16.108 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:51:15 smithi026 ceph-mon[175151]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d 2023-09-23T20:51:16.108 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:51:15 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:51:16.108 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:51:15 smithi026 ceph-mon[175151]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 2/72 objects degraded (2.778%), 2 pgs degraded) 2023-09-23T20:51:16.108 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:51:15 smithi026 ceph-mon[175151]: Cluster is now healthy 2023-09-23T20:51:16.108 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:51:15 smithi026 ceph-mon[179688]: pgmap v202: 73 pgs: 5 peering, 68 active+clean; 582 KiB data, 171 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:51:16.108 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:51:15 smithi026 ceph-mon[179688]: from='client.34566 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T20:51:16.109 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:51:15 smithi026 ceph-mon[179688]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d 2023-09-23T20:51:16.109 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:51:15 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:51:16.109 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:51:15 smithi026 ceph-mon[179688]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 2/72 objects degraded (2.778%), 2 pgs degraded) 2023-09-23T20:51:16.109 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:51:15 smithi026 ceph-mon[179688]: Cluster is now healthy 2023-09-23T20:51:16.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:51:15 smithi070 ceph-mon[168282]: pgmap v202: 73 pgs: 5 peering, 68 active+clean; 582 KiB data, 171 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:51:16.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:51:15 smithi070 ceph-mon[168282]: from='client.34566 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T20:51:16.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:51:15 smithi070 ceph-mon[168282]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d 2023-09-23T20:51:16.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:51:15 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:51:16.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:51:15 smithi070 ceph-mon[168282]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 2/72 objects degraded (2.778%), 2 pgs degraded) 2023-09-23T20:51:16.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:51:15 smithi070 ceph-mon[168282]: Cluster is now healthy 2023-09-23T20:51:17.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:51:17 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:51:17.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:51:17 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:51:17.383 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:51:17 smithi026 ceph-mon[175151]: pgmap v203: 73 pgs: 73 active+clean; 582 KiB data, 171 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:51:17.383 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:51:17 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:51:17.383 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:51:17 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:51:17.383 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:51:17 smithi026 ceph-mon[179688]: pgmap v203: 73 pgs: 73 active+clean; 582 KiB data, 171 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:51:17.457 INFO:teuthology.orchestra.run.smithi026.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2023-09-23T20:51:17.457 INFO:teuthology.orchestra.run.smithi026.stdout:alertmanager.a smithi026 *:9093,9094 running (7m) - 10m 28.6M - 0.25.0 c8568f914cd2 d4b48de0f10c 2023-09-23T20:51:17.457 INFO:teuthology.orchestra.run.smithi026.stdout:grafana.a smithi070 *:3000 running (6m) - 9m 93.3M - 9.4.7 2c41d148cca3 ed7e189354a5 2023-09-23T20:51:17.458 INFO:teuthology.orchestra.run.smithi026.stdout:mds.foo.smithi026.znfpnl smithi026 running (9m) - 9m 20.2M - 17.2.6-1123-g61431f6b 0db14aa01094 b6861ac0ff55 2023-09-23T20:51:17.458 INFO:teuthology.orchestra.run.smithi026.stdout:mds.foo.smithi070.imqhqy smithi070 running (9m) - 9m 18.3M - 17.2.6-1123-g61431f6b 0db14aa01094 ca26bf551eb9 2023-09-23T20:51:17.458 INFO:teuthology.orchestra.run.smithi026.stdout:mgr.x smithi070 *:8443,9283,8765 running (6m) - 13m 449M - 18.2.0-460-gbb0daeff e32c2a64c126 35059b639121 2023-09-23T20:51:17.458 INFO:teuthology.orchestra.run.smithi026.stdout:mgr.y smithi026 *:8443,9283,8765 running (6m) - 15m 515M - 18.2.0-460-gbb0daeff e32c2a64c126 5cc7ece3b7e8 2023-09-23T20:51:17.458 INFO:teuthology.orchestra.run.smithi026.stdout:mon.a smithi026 running (6m) - 15m 83.6M 2048M 18.2.0-460-gbb0daeff e32c2a64c126 f0ba51b53f50 2023-09-23T20:51:17.458 INFO:teuthology.orchestra.run.smithi026.stdout:mon.b smithi070 running (3m) - 13m 48.6M 2048M 18.2.0-460-gbb0daeff e32c2a64c126 957198c68713 2023-09-23T20:51:17.458 INFO:teuthology.orchestra.run.smithi026.stdout:mon.c smithi026 running (4m) - 13m 57.4M 2048M 18.2.0-460-gbb0daeff e32c2a64c126 ad100e9213d3 2023-09-23T20:51:17.458 INFO:teuthology.orchestra.run.smithi026.stdout:node-exporter.a smithi026 *:9100 running (7m) - 9m 16.8M - 1.5.0 0da6a335fe13 95269f65f9af 2023-09-23T20:51:17.458 INFO:teuthology.orchestra.run.smithi026.stdout:node-exporter.b smithi070 *:9100 running (7m) - 9m 18.9M - 1.5.0 0da6a335fe13 5b56a4a415d0 2023-09-23T20:51:17.458 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0 smithi026 running (110s) - 13m 56.5M 2501M 18.2.0-460-gbb0daeff e32c2a64c126 c815c65a946d 2023-09-23T20:51:17.458 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1 smithi026 running (77s) - 12m 54.9M 2501M 18.2.0-460-gbb0daeff e32c2a64c126 0e3111ea70d3 2023-09-23T20:51:17.458 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2 smithi026 running (45s) - 12m 51.6M 2501M 18.2.0-460-gbb0daeff e32c2a64c126 26e1c16041a9 2023-09-23T20:51:17.459 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3 smithi026 running (12s) - 12m 12.5M 2501M 18.2.0-460-gbb0daeff e32c2a64c126 745262093b5f 2023-09-23T20:51:17.459 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4 smithi070 running (11m) - 11m 108M 2500M 17.2.6-1123-g61431f6b 0db14aa01094 308a2af3101e 2023-09-23T20:51:17.459 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5 smithi070 running (11m) - 11m 115M 2500M 17.2.6-1123-g61431f6b 0db14aa01094 16cd72517077 2023-09-23T20:51:17.459 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6 smithi070 running (11m) - 11m 80.8M 2500M 17.2.6-1123-g61431f6b 0db14aa01094 5d24c21e8be7 2023-09-23T20:51:17.459 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7 smithi070 running (10m) - 10m 122M 2500M 17.2.6-1123-g61431f6b 0db14aa01094 8c1b7f314c41 2023-09-23T20:51:17.459 INFO:teuthology.orchestra.run.smithi026.stdout:prometheus.a smithi070 *:9095 running (6m) - 10m 61.7M - 2.43.0 a07b618ecd1d dda227f93524 2023-09-23T20:51:17.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:51:17 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:51:17.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:51:17 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:51:17.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:51:17 smithi070 ceph-mon[168282]: pgmap v203: 73 pgs: 73 active+clean; 582 KiB data, 171 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:51:17.832 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -e sha1=bb0daeff97f020f686596fca6eba20c310e26fb7 -- bash -c 'ceph orch ps' 2023-09-23T20:51:18.439 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:51:18.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:51:18 smithi070 ceph-mon[168282]: from='client.44477 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T20:51:18.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:51:18 smithi026 ceph-mon[175151]: from='client.44477 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T20:51:18.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:51:18 smithi026 ceph-mon[179688]: from='client.44477 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T20:51:19.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:51:19 smithi026 ceph-mon[175151]: pgmap v204: 73 pgs: 73 active+clean; 582 KiB data, 171 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:51:19.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:51:19 smithi026 ceph-mon[179688]: pgmap v204: 73 pgs: 73 active+clean; 582 KiB data, 171 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:51:19.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:51:19 smithi070 ceph-mon[168282]: pgmap v204: 73 pgs: 73 active+clean; 582 KiB data, 171 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:51:19.882 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:51:19 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[164029]: ::ffff:172.21.15.70 - - [23/Sep/2023:20:51:19] "GET /metrics HTTP/1.1" 200 33813 "" "Prometheus/2.43.0" 2023-09-23T20:51:19.887 INFO:teuthology.orchestra.run.smithi026.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2023-09-23T20:51:19.887 INFO:teuthology.orchestra.run.smithi026.stdout:alertmanager.a smithi026 *:9093,9094 running (7m) - 10m 28.6M - 0.25.0 c8568f914cd2 d4b48de0f10c 2023-09-23T20:51:19.887 INFO:teuthology.orchestra.run.smithi026.stdout:grafana.a smithi070 *:3000 running (7m) - 9m 93.3M - 9.4.7 2c41d148cca3 ed7e189354a5 2023-09-23T20:51:19.887 INFO:teuthology.orchestra.run.smithi026.stdout:mds.foo.smithi026.znfpnl smithi026 running (9m) - 9m 20.2M - 17.2.6-1123-g61431f6b 0db14aa01094 b6861ac0ff55 2023-09-23T20:51:19.887 INFO:teuthology.orchestra.run.smithi026.stdout:mds.foo.smithi070.imqhqy smithi070 running (9m) - 9m 18.3M - 17.2.6-1123-g61431f6b 0db14aa01094 ca26bf551eb9 2023-09-23T20:51:19.887 INFO:teuthology.orchestra.run.smithi026.stdout:mgr.x smithi070 *:8443,9283,8765 running (6m) - 13m 449M - 18.2.0-460-gbb0daeff e32c2a64c126 35059b639121 2023-09-23T20:51:19.887 INFO:teuthology.orchestra.run.smithi026.stdout:mgr.y smithi026 *:8443,9283,8765 running (6m) - 15m 515M - 18.2.0-460-gbb0daeff e32c2a64c126 5cc7ece3b7e8 2023-09-23T20:51:19.887 INFO:teuthology.orchestra.run.smithi026.stdout:mon.a smithi026 running (6m) - 15m 83.6M 2048M 18.2.0-460-gbb0daeff e32c2a64c126 f0ba51b53f50 2023-09-23T20:51:19.887 INFO:teuthology.orchestra.run.smithi026.stdout:mon.b smithi070 running (3m) - 13m 48.6M 2048M 18.2.0-460-gbb0daeff e32c2a64c126 957198c68713 2023-09-23T20:51:19.888 INFO:teuthology.orchestra.run.smithi026.stdout:mon.c smithi026 running (4m) - 13m 57.4M 2048M 18.2.0-460-gbb0daeff e32c2a64c126 ad100e9213d3 2023-09-23T20:51:19.888 INFO:teuthology.orchestra.run.smithi026.stdout:node-exporter.a smithi026 *:9100 running (7m) - 10m 16.8M - 1.5.0 0da6a335fe13 95269f65f9af 2023-09-23T20:51:19.888 INFO:teuthology.orchestra.run.smithi026.stdout:node-exporter.b smithi070 *:9100 running (7m) - 9m 18.9M - 1.5.0 0da6a335fe13 5b56a4a415d0 2023-09-23T20:51:19.888 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0 smithi026 running (112s) - 13m 56.5M 2501M 18.2.0-460-gbb0daeff e32c2a64c126 c815c65a946d 2023-09-23T20:51:19.888 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1 smithi026 running (80s) - 12m 54.9M 2501M 18.2.0-460-gbb0daeff e32c2a64c126 0e3111ea70d3 2023-09-23T20:51:19.888 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2 smithi026 running (48s) - 12m 51.6M 2501M 18.2.0-460-gbb0daeff e32c2a64c126 26e1c16041a9 2023-09-23T20:51:19.888 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3 smithi026 running (15s) - 12m 12.5M 2501M 18.2.0-460-gbb0daeff e32c2a64c126 745262093b5f 2023-09-23T20:51:19.888 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4 smithi070 running (11m) - 11m 108M 2500M 17.2.6-1123-g61431f6b 0db14aa01094 308a2af3101e 2023-09-23T20:51:19.888 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5 smithi070 running (11m) - 11m 115M 2500M 17.2.6-1123-g61431f6b 0db14aa01094 16cd72517077 2023-09-23T20:51:19.888 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6 smithi070 running (11m) - 11m 80.8M 2500M 17.2.6-1123-g61431f6b 0db14aa01094 5d24c21e8be7 2023-09-23T20:51:19.888 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7 smithi070 running (10m) - 10m 122M 2500M 17.2.6-1123-g61431f6b 0db14aa01094 8c1b7f314c41 2023-09-23T20:51:19.888 INFO:teuthology.orchestra.run.smithi026.stdout:prometheus.a smithi070 *:9095 running (6m) - 10m 61.7M - 2.43.0 a07b618ecd1d dda227f93524 2023-09-23T20:51:20.288 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -e sha1=bb0daeff97f020f686596fca6eba20c310e26fb7 -- bash -c 'ceph versions' 2023-09-23T20:51:20.899 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:51:22.101 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:51:21 smithi026 ceph-mon[179688]: from='client.34578 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T20:51:22.101 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:51:21 smithi026 ceph-mon[179688]: pgmap v205: 73 pgs: 73 active+clean; 582 KiB data, 171 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:51:22.101 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:51:21 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:51:22.101 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:51:21 smithi026 ceph-mon[175151]: from='client.34578 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T20:51:22.102 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:51:21 smithi026 ceph-mon[175151]: pgmap v205: 73 pgs: 73 active+clean; 582 KiB data, 171 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:51:22.102 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:51:21 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:51:22.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:51:21 smithi070 ceph-mon[168282]: from='client.34578 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T20:51:22.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:51:21 smithi070 ceph-mon[168282]: pgmap v205: 73 pgs: 73 active+clean; 582 KiB data, 171 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:51:22.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:51:21 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:51:22.558 INFO:teuthology.orchestra.run.smithi026.stdout:{ 2023-09-23T20:51:22.558 INFO:teuthology.orchestra.run.smithi026.stdout: "mon": { 2023-09-23T20:51:22.558 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 18.2.0-460-gbb0daeff (bb0daeff97f020f686596fca6eba20c310e26fb7) reef (stable)": 3 2023-09-23T20:51:22.558 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:51:22.558 INFO:teuthology.orchestra.run.smithi026.stdout: "mgr": { 2023-09-23T20:51:22.558 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 18.2.0-460-gbb0daeff (bb0daeff97f020f686596fca6eba20c310e26fb7) reef (stable)": 2 2023-09-23T20:51:22.558 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:51:22.558 INFO:teuthology.orchestra.run.smithi026.stdout: "osd": { 2023-09-23T20:51:22.559 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 17.2.6-1123-g61431f6b (61431f6ba2752649b506867ec856b4a6236f90dd) quincy (stable)": 4, 2023-09-23T20:51:22.559 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 18.2.0-460-gbb0daeff (bb0daeff97f020f686596fca6eba20c310e26fb7) reef (stable)": 4 2023-09-23T20:51:22.559 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:51:22.559 INFO:teuthology.orchestra.run.smithi026.stdout: "mds": { 2023-09-23T20:51:22.559 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 17.2.6-1123-g61431f6b (61431f6ba2752649b506867ec856b4a6236f90dd) quincy (stable)": 2 2023-09-23T20:51:22.559 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:51:22.559 INFO:teuthology.orchestra.run.smithi026.stdout: "overall": { 2023-09-23T20:51:22.559 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 17.2.6-1123-g61431f6b (61431f6ba2752649b506867ec856b4a6236f90dd) quincy (stable)": 6, 2023-09-23T20:51:22.559 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 18.2.0-460-gbb0daeff (bb0daeff97f020f686596fca6eba20c310e26fb7) reef (stable)": 9 2023-09-23T20:51:22.559 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:51:22.559 INFO:teuthology.orchestra.run.smithi026.stdout:} 2023-09-23T20:51:22.881 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:51:22 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/3619505490' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T20:51:22.881 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:51:22 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/3619505490' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T20:51:22.944 DEBUG:teuthology.parallel:result is None 2023-09-23T20:51:22.945 INFO:teuthology.run_tasks:Running task print... 2023-09-23T20:51:22.954 INFO:teuthology.task.print:**** done upgrading first half of cluster **** 2023-09-23T20:51:22.955 INFO:teuthology.run_tasks:Running task ceph.healthy... 2023-09-23T20:51:22.999 INFO:tasks.ceph:Waiting until ceph daemons up and pgs clean... 2023-09-23T20:51:22.999 INFO:tasks.cephadm.ceph_manager.ceph:waiting for mgr available 2023-09-23T20:51:23.000 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph mgr dump --format=json 2023-09-23T20:51:23.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:51:22 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/3619505490' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T20:51:23.604 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:51:24.051 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:51:23 smithi026 ceph-mon[175151]: pgmap v206: 73 pgs: 73 active+clean; 582 KiB data, 171 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:51:24.051 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:51:23 smithi026 ceph-mon[179688]: pgmap v206: 73 pgs: 73 active+clean; 582 KiB data, 171 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:51:24.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:51:23 smithi070 ceph-mon[168282]: pgmap v206: 73 pgs: 73 active+clean; 582 KiB data, 171 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:51:25.064 INFO:teuthology.orchestra.run.smithi026.stdout: 2023-09-23T20:51:25.451 INFO:teuthology.orchestra.run.smithi026.stdout:{"epoch":37,"active_gid":25142,"active_name":"y","active_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6800","nonce":946685950},{"type":"v1","addr":"172.21.15.26:6801","nonce":946685950}]},"active_addr":"172.21.15.26:6801/946685950","active_change":"2023-09-23T20:45:16.037648+0000","active_mgr_features":4540138322906710015,"available":true,"standbys":[{"gid":25090,"name":"x","mgr_features":4540138322906710015,"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2359","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.25.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph:v18","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_elasticsearch":{"name":"container_image_elasticsearch","type":"str","level":"advanced","flags":0,"default_value":"quay.io/omrizeneva/elasticsearch:6.8.23","min":"","max":"","enum_allowed":[],"desc":"elasticsearch container image","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:9.4.7","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_agent":{"name":"container_image_jaeger_agent","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-agent:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger agent container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_collector":{"name":"container_image_jaeger_collector","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-collector:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger collector container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_query":{"name":"container_image_jaeger_query","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-query:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger query container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.2.4","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.5.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_nvmeof":{"name":"container_image_nvmeof","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/nvmeof:0.0.2","min":"","max":"","enum_allowed":[],"desc":"Nvme-of container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.43.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"secs","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"secure_monitoring_stack":{"name":"secure_monitoring_stack","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable TLS security for all the monitoring stack daemons","long_desc":"","tags":[],"see_also":[]},"service_discovery_port":{"name":"service_discovery_port","type":"int","level":"advanced","flags":0,"default_value":"8765","min":"","max":"","enum_allowed":[],"desc":"cephadm service discovery port","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"fail_fs":{"name":"fail_fs","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Fail filesystem for rapid multi-rank mds upgrade","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"noautoscale":{"name":"noautoscale","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"global autoscale flag","long_desc":"Option to turn on/off the autoscaler for all pools","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"exclude_perf_counters":{"name":"exclude_perf_counters","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Do not include perf-counters in the metrics output","long_desc":"Gathering perf-counters from a single Prometheus exporter can degrade ceph-mgr performance, especially in large clusters. Instead, Ceph-exporter daemons are now used by default for perf-counter gathering. This should only be disabled when no ceph-exporters are deployed.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"drive_group_interval":{"name":"drive_group_interval","type":"float","level":"advanced","flags":0,"default_value":"300.0","min":"","max":"","enum_allowed":[],"desc":"interval in seconds between re-application of applied drive_groups","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard_description":{"name":"leaderboard_description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}]}],"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2359","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.25.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph:v18","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_elasticsearch":{"name":"container_image_elasticsearch","type":"str","level":"advanced","flags":0,"default_value":"quay.io/omrizeneva/elasticsearch:6.8.23","min":"","max":"","enum_allowed":[],"desc":"elasticsearch container image","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:9.4.7","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_agent":{"name":"container_image_jaeger_agent","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-agent:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger agent container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_collector":{"name":"container_image_jaeger_collector","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-collector:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger collector container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_query":{"name":"container_image_jaeger_query","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-query:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger query container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.2.4","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.5.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_nvmeof":{"name":"container_image_nvmeof","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/nvmeof:0.0.2","min":"","max":"","enum_allowed":[],"desc":"Nvme-of container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.43.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"secs","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"secure_monitoring_stack":{"name":"secure_monitoring_stack","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable TLS security for all the monitoring stack daemons","long_desc":"","tags":[],"see_also":[]},"service_discovery_port":{"name":"service_discovery_port","type":"int","level":"advanced","flags":0,"default_value":"8765","min":"","max":"","enum_allowed":[],"desc":"cephadm service discovery port","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"fail_fs":{"name":"fail_fs","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Fail filesystem for rapid multi-rank mds upgrade","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"noautoscale":{"name":"noautoscale","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"global autoscale flag","long_desc":"Option to turn on/off the autoscaler for all pools","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"exclude_perf_counters":{"name":"exclude_perf_counters","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Do not include perf-counters in the metrics output","long_desc":"Gathering perf-counters from a single Prometheus exporter can degrade ceph-mgr performance, especially in large clusters. Instead, Ceph-exporter daemons are now used by default for perf-counter gathering. This should only be disabled when no ceph-exporters are deployed.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"drive_group_interval":{"name":"drive_group_interval","type":"float","level":"advanced","flags":0,"default_value":"300.0","min":"","max":"","enum_allowed":[],"desc":"interval in seconds between re-application of applied drive_groups","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard_description":{"name":"leaderboard_description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}],"services":{"dashboard":"https://172.21.15.26:8443/","prometheus":"http://172.21.15.26:9283/"},"always_on_modules":{"octopus":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"pacific":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"quincy":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"reef":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"]},"last_failure_osd_epoch":79,"active_clients":[{"name":"devicehealth","addrvec":[{"type":"v2","addr":"172.21.15.26:0","nonce":2522590450}]},{"name":"libcephsqlite","addrvec":[{"type":"v2","addr":"172.21.15.26:0","nonce":750765754}]},{"name":"rbd_support","addrvec":[{"type":"v2","addr":"172.21.15.26:0","nonce":4267148466}]},{"name":"volumes","addrvec":[{"type":"v2","addr":"172.21.15.26:0","nonce":1124861819}]}]} 2023-09-23T20:51:25.456 INFO:tasks.cephadm.ceph_manager.ceph:mgr available! 2023-09-23T20:51:25.456 INFO:tasks.cephadm.ceph_manager.ceph:waiting for all up 2023-09-23T20:51:25.456 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph osd dump --format=json 2023-09-23T20:51:26.063 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:51:26.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:51:25 smithi070 ceph-mon[168282]: pgmap v207: 73 pgs: 73 active+clean; 582 KiB data, 171 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:51:26.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:51:25 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/3476008350' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2023-09-23T20:51:26.131 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:51:25 smithi026 ceph-mon[175151]: pgmap v207: 73 pgs: 73 active+clean; 582 KiB data, 171 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:51:26.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:51:25 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/3476008350' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2023-09-23T20:51:26.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:51:25 smithi026 ceph-mon[179688]: pgmap v207: 73 pgs: 73 active+clean; 582 KiB data, 171 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:51:26.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:51:25 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/3476008350' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2023-09-23T20:51:27.503 INFO:teuthology.orchestra.run.smithi026.stdout: 2023-09-23T20:51:27.503 INFO:teuthology.orchestra.run.smithi026.stdout:{"epoch":99,"fsid":"b3f8e4ec-5a50-11ee-8db4-212e2dc638e7","created":"2023-09-23T20:35:58.919705+0000","modified":"2023-09-23T20:51:10.816433+0000","last_up_change":"2023-09-23T20:51:09.811245+0000","last_in_change":"2023-09-23T20:40:23.597125+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":22,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":5,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"quincy","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2023-09-23T20:38:57.022844+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_acting":7.8899998664855957,"score_stable":7.8899998664855957,"optimal_score":0.37999999523162842,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2023-09-23T20:41:15.182110+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"64","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_acting":2,"score_stable":2,"optimal_score":1,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs.foo.meta","create_time":"2023-09-23T20:41:57.679446+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"68","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"foo"}},"read_balance":{"score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":4,"pool_name":"cephfs.foo.data","create_time":"2023-09-23T20:41:58.331482+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"68","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"cephfs":{"data":"foo"}},"read_balance":{"score_acting":2.25,"score_stable":2.25,"optimal_score":1,"raw_score_acting":2.25,"raw_score_stable":2.25,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"f837feb7-060e-4965-9b48-bed9f3672d05","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":79,"up_from":83,"up_thru":98,"down_at":80,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6802","nonce":3105265146},{"type":"v1","addr":"172.21.15.26:6803","nonce":3105265146}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6804","nonce":3105265146},{"type":"v1","addr":"172.21.15.26:6805","nonce":3105265146}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6808","nonce":3105265146},{"type":"v1","addr":"172.21.15.26:6809","nonce":3105265146}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6806","nonce":3105265146},{"type":"v1","addr":"172.21.15.26:6807","nonce":3105265146}]},"public_addr":"172.21.15.26:6803/3105265146","cluster_addr":"172.21.15.26:6805/3105265146","heartbeat_back_addr":"172.21.15.26:6809/3105265146","heartbeat_front_addr":"172.21.15.26:6807/3105265146","state":["exists","up"]},{"osd":1,"uuid":"917f65aa-5963-4576-988a-396698b7c6a8","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":13,"last_clean_end":84,"up_from":88,"up_thru":98,"down_at":85,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6810","nonce":3442737755},{"type":"v1","addr":"172.21.15.26:6811","nonce":3442737755}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6812","nonce":3442737755},{"type":"v1","addr":"172.21.15.26:6813","nonce":3442737755}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6816","nonce":3442737755},{"type":"v1","addr":"172.21.15.26:6817","nonce":3442737755}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6814","nonce":3442737755},{"type":"v1","addr":"172.21.15.26:6815","nonce":3442737755}]},"public_addr":"172.21.15.26:6811/3442737755","cluster_addr":"172.21.15.26:6813/3442737755","heartbeat_back_addr":"172.21.15.26:6817/3442737755","heartbeat_front_addr":"172.21.15.26:6815/3442737755","state":["exists","up"]},{"osd":2,"uuid":"2e115328-76c6-478e-915c-5786903be49f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":18,"last_clean_end":89,"up_from":93,"up_thru":98,"down_at":90,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6818","nonce":38979282},{"type":"v1","addr":"172.21.15.26:6819","nonce":38979282}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6820","nonce":38979282},{"type":"v1","addr":"172.21.15.26:6821","nonce":38979282}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6824","nonce":38979282},{"type":"v1","addr":"172.21.15.26:6825","nonce":38979282}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6822","nonce":38979282},{"type":"v1","addr":"172.21.15.26:6823","nonce":38979282}]},"public_addr":"172.21.15.26:6819/38979282","cluster_addr":"172.21.15.26:6821/38979282","heartbeat_back_addr":"172.21.15.26:6825/38979282","heartbeat_front_addr":"172.21.15.26:6823/38979282","state":["exists","up"]},{"osd":3,"uuid":"c3dd2c3f-5cf5-4fb0-964c-1b8926f75c8a","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":26,"last_clean_end":94,"up_from":98,"up_thru":98,"down_at":95,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6826","nonce":2947396145},{"type":"v1","addr":"172.21.15.26:6827","nonce":2947396145}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6828","nonce":2947396145},{"type":"v1","addr":"172.21.15.26:6829","nonce":2947396145}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6832","nonce":2947396145},{"type":"v1","addr":"172.21.15.26:6833","nonce":2947396145}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6830","nonce":2947396145},{"type":"v1","addr":"172.21.15.26:6831","nonce":2947396145}]},"public_addr":"172.21.15.26:6827/2947396145","cluster_addr":"172.21.15.26:6829/2947396145","heartbeat_back_addr":"172.21.15.26:6833/2947396145","heartbeat_front_addr":"172.21.15.26:6831/2947396145","state":["exists","up"]},{"osd":4,"uuid":"a5f31996-1b38-491b-956b-5aebc72d0170","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":31,"up_thru":98,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6800","nonce":1528249138},{"type":"v1","addr":"172.21.15.70:6801","nonce":1528249138}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6802","nonce":1528249138},{"type":"v1","addr":"172.21.15.70:6803","nonce":1528249138}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6806","nonce":1528249138},{"type":"v1","addr":"172.21.15.70:6807","nonce":1528249138}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6804","nonce":1528249138},{"type":"v1","addr":"172.21.15.70:6805","nonce":1528249138}]},"public_addr":"172.21.15.70:6801/1528249138","cluster_addr":"172.21.15.70:6803/1528249138","heartbeat_back_addr":"172.21.15.70:6807/1528249138","heartbeat_front_addr":"172.21.15.70:6805/1528249138","state":["exists","up"]},{"osd":5,"uuid":"21dad198-b9db-42c6-8823-112df1ecd874","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":39,"up_thru":98,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6808","nonce":3802232906},{"type":"v1","addr":"172.21.15.70:6809","nonce":3802232906}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6810","nonce":3802232906},{"type":"v1","addr":"172.21.15.70:6811","nonce":3802232906}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6814","nonce":3802232906},{"type":"v1","addr":"172.21.15.70:6815","nonce":3802232906}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6812","nonce":3802232906},{"type":"v1","addr":"172.21.15.70:6813","nonce":3802232906}]},"public_addr":"172.21.15.70:6809/3802232906","cluster_addr":"172.21.15.70:6811/3802232906","heartbeat_back_addr":"172.21.15.70:6815/3802232906","heartbeat_front_addr":"172.21.15.70:6813/3802232906","state":["exists","up"]},{"osd":6,"uuid":"6f3be784-7149-4f13-81b5-98924e97c056","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":47,"up_thru":95,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6816","nonce":2944175278},{"type":"v1","addr":"172.21.15.70:6817","nonce":2944175278}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6818","nonce":2944175278},{"type":"v1","addr":"172.21.15.70:6819","nonce":2944175278}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6822","nonce":2944175278},{"type":"v1","addr":"172.21.15.70:6823","nonce":2944175278}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6820","nonce":2944175278},{"type":"v1","addr":"172.21.15.70:6821","nonce":2944175278}]},"public_addr":"172.21.15.70:6817/2944175278","cluster_addr":"172.21.15.70:6819/2944175278","heartbeat_back_addr":"172.21.15.70:6823/2944175278","heartbeat_front_addr":"172.21.15.70:6821/2944175278","state":["exists","up"]},{"osd":7,"uuid":"499f5f85-f4b2-40a7-ac0f-79cd22738d30","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":55,"up_thru":98,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6824","nonce":2824974165},{"type":"v1","addr":"172.21.15.70:6825","nonce":2824974165}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6826","nonce":2824974165},{"type":"v1","addr":"172.21.15.70:6827","nonce":2824974165}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6830","nonce":2824974165},{"type":"v1","addr":"172.21.15.70:6831","nonce":2824974165}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6828","nonce":2824974165},{"type":"v1","addr":"172.21.15.70:6829","nonce":2824974165}]},"public_addr":"172.21.15.70:6825/2824974165","cluster_addr":"172.21.15.70:6827/2824974165","heartbeat_back_addr":"172.21.15.70:6831/2824974165","heartbeat_front_addr":"172.21.15.70:6829/2824974165","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T20:38:10.875468+0000","dead_epoch":79},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T20:38:32.449963+0000","dead_epoch":84},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T20:38:53.633316+0000","dead_epoch":89},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T20:39:14.828940+0000","dead_epoch":94},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T20:39:34.945663+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T20:39:55.954288+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T20:40:16.847102+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T20:40:37.820404+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"4.b","mappings":[{"from":1,"to":2}]},{"pgid":"4.18","mappings":[{"from":1,"to":2}]},{"pgid":"4.1c","mappings":[{"from":1,"to":5}]}],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.26:6801/91363814":"2023-09-24T20:41:03.790648+0000","172.21.15.26:0/3175144810":"2023-09-24T20:36:31.159649+0000","172.21.15.26:0/1625326392":"2023-09-24T20:36:56.498559+0000","172.21.15.70:0/3393166416":"2023-09-24T20:44:32.835628+0000","172.21.15.26:0/1083572999":"2023-09-24T20:41:03.790648+0000","172.21.15.26:6801/2440434873":"2023-09-24T20:43:50.641728+0000","172.21.15.70:6834/1275773750":"2023-09-24T20:44:32.835628+0000","172.21.15.26:0/289994511":"2023-09-24T20:36:56.498559+0000","172.21.15.70:0/2989870914":"2023-09-24T20:44:32.835628+0000","172.21.15.26:6800/2440434873":"2023-09-24T20:43:50.641728+0000","172.21.15.70:0/302414747":"2023-09-24T20:44:32.835628+0000","172.21.15.70:0/413499922":"2023-09-24T20:44:32.835628+0000","172.21.15.26:0/4215395943":"2023-09-24T20:43:50.641728+0000","172.21.15.26:0/233963796":"2023-09-24T20:45:16.037413+0000","172.21.15.26:0/934875067":"2023-09-24T20:45:16.037413+0000","172.21.15.70:0/1035724255":"2023-09-24T20:44:32.835628+0000","172.21.15.26:0/1376232363":"2023-09-24T20:43:50.641728+0000","172.21.15.26:0/394447930":"2023-09-24T20:41:03.790648+0000","172.21.15.70:6835/1275773750":"2023-09-24T20:44:32.835628+0000","172.21.15.26:0/190503462":"2023-09-24T20:45:16.037413+0000","172.21.15.70:0/2198692933":"2023-09-24T20:44:32.835628+0000","172.21.15.26:6800/952511372":"2023-09-24T20:36:56.498559+0000","172.21.15.26:0/2633606443":"2023-09-24T20:45:16.037413+0000","172.21.15.26:0/237900280":"2023-09-24T20:43:50.641728+0000","172.21.15.26:0/3461760126":"2023-09-24T20:41:03.790648+0000","172.21.15.26:0/635571892":"2023-09-24T20:43:50.641728+0000","172.21.15.26:0/188049283":"2023-09-24T20:41:03.790648+0000","172.21.15.26:0/1815190465":"2023-09-24T20:36:31.159649+0000","172.21.15.26:0/686734212":"2023-09-24T20:36:31.159649+0000","172.21.15.26:6801/952511372":"2023-09-24T20:36:56.498559+0000","172.21.15.26:0/2838335646":"2023-09-24T20:36:56.498559+0000","172.21.15.26:6801/1446178988":"2023-09-24T20:45:16.037413+0000","172.21.15.26:6800/1446178988":"2023-09-24T20:45:16.037413+0000","172.21.15.26:6800/4099624907":"2023-09-24T20:36:31.159649+0000","172.21.15.26:6801/4099624907":"2023-09-24T20:36:31.159649+0000","172.21.15.26:0/3408257869":"2023-09-24T20:45:16.037413+0000","172.21.15.26:6800/91363814":"2023-09-24T20:41:03.790648+0000","172.21.15.26:0/2817523682":"2023-09-24T20:45:16.037413+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[{"pool":5,"snaps":[{"begin":2,"length":1}]}],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2023-09-23T20:51:27.916 INFO:tasks.cephadm.ceph_manager.ceph:all up! 2023-09-23T20:51:27.917 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph osd dump --format=json 2023-09-23T20:51:28.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:51:27 smithi070 ceph-mon[168282]: pgmap v208: 73 pgs: 73 active+clean; 582 KiB data, 171 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:51:28.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:51:27 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/2543448832' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-09-23T20:51:28.131 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:51:27 smithi026 ceph-mon[175151]: pgmap v208: 73 pgs: 73 active+clean; 582 KiB data, 171 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:51:28.131 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:51:27 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/2543448832' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-09-23T20:51:28.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:51:27 smithi026 ceph-mon[179688]: pgmap v208: 73 pgs: 73 active+clean; 582 KiB data, 171 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:51:28.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:51:27 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/2543448832' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-09-23T20:51:28.523 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:51:29.107 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:51:28 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:51:29.107 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:51:28 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:51:29.107 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:51:28 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:51:29.107 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:51:28 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:51:29.107 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:51:28 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:51:29.108 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:51:28 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:51:29.108 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:51:28 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:51:29.108 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:51:28 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:51:29.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:51:28 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:51:29.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:51:28 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:51:29.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:51:28 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:51:29.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:51:28 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:51:29.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:51:29 smithi026 ceph-mon[175151]: pgmap v209: 73 pgs: 73 active+clean; 582 KiB data, 171 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:51:29.883 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:51:29 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:51:29.883 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:51:29 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:51:29.883 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:51:29 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:51:29.883 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:51:29 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:51:29.883 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:51:29 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[164029]: ::ffff:172.21.15.70 - - [23/Sep/2023:20:51:29] "GET /metrics HTTP/1.1" 200 33813 "" "Prometheus/2.43.0" 2023-09-23T20:51:29.883 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:51:29 smithi026 ceph-mon[179688]: pgmap v209: 73 pgs: 73 active+clean; 582 KiB data, 171 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:51:29.884 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:51:29 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:51:29.884 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:51:29 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:51:29.884 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:51:29 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:51:29.884 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:51:29 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:51:30.016 INFO:teuthology.orchestra.run.smithi026.stdout: 2023-09-23T20:51:30.016 INFO:teuthology.orchestra.run.smithi026.stdout:{"epoch":99,"fsid":"b3f8e4ec-5a50-11ee-8db4-212e2dc638e7","created":"2023-09-23T20:35:58.919705+0000","modified":"2023-09-23T20:51:10.816433+0000","last_up_change":"2023-09-23T20:51:09.811245+0000","last_in_change":"2023-09-23T20:40:23.597125+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":22,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":5,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"quincy","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2023-09-23T20:38:57.022844+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_acting":7.8899998664855957,"score_stable":7.8899998664855957,"optimal_score":0.37999999523162842,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2023-09-23T20:41:15.182110+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"64","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_acting":2,"score_stable":2,"optimal_score":1,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs.foo.meta","create_time":"2023-09-23T20:41:57.679446+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"68","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"foo"}},"read_balance":{"score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":4,"pool_name":"cephfs.foo.data","create_time":"2023-09-23T20:41:58.331482+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"68","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"cephfs":{"data":"foo"}},"read_balance":{"score_acting":2.25,"score_stable":2.25,"optimal_score":1,"raw_score_acting":2.25,"raw_score_stable":2.25,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"f837feb7-060e-4965-9b48-bed9f3672d05","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":79,"up_from":83,"up_thru":98,"down_at":80,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6802","nonce":3105265146},{"type":"v1","addr":"172.21.15.26:6803","nonce":3105265146}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6804","nonce":3105265146},{"type":"v1","addr":"172.21.15.26:6805","nonce":3105265146}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6808","nonce":3105265146},{"type":"v1","addr":"172.21.15.26:6809","nonce":3105265146}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6806","nonce":3105265146},{"type":"v1","addr":"172.21.15.26:6807","nonce":3105265146}]},"public_addr":"172.21.15.26:6803/3105265146","cluster_addr":"172.21.15.26:6805/3105265146","heartbeat_back_addr":"172.21.15.26:6809/3105265146","heartbeat_front_addr":"172.21.15.26:6807/3105265146","state":["exists","up"]},{"osd":1,"uuid":"917f65aa-5963-4576-988a-396698b7c6a8","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":13,"last_clean_end":84,"up_from":88,"up_thru":98,"down_at":85,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6810","nonce":3442737755},{"type":"v1","addr":"172.21.15.26:6811","nonce":3442737755}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6812","nonce":3442737755},{"type":"v1","addr":"172.21.15.26:6813","nonce":3442737755}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6816","nonce":3442737755},{"type":"v1","addr":"172.21.15.26:6817","nonce":3442737755}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6814","nonce":3442737755},{"type":"v1","addr":"172.21.15.26:6815","nonce":3442737755}]},"public_addr":"172.21.15.26:6811/3442737755","cluster_addr":"172.21.15.26:6813/3442737755","heartbeat_back_addr":"172.21.15.26:6817/3442737755","heartbeat_front_addr":"172.21.15.26:6815/3442737755","state":["exists","up"]},{"osd":2,"uuid":"2e115328-76c6-478e-915c-5786903be49f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":18,"last_clean_end":89,"up_from":93,"up_thru":98,"down_at":90,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6818","nonce":38979282},{"type":"v1","addr":"172.21.15.26:6819","nonce":38979282}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6820","nonce":38979282},{"type":"v1","addr":"172.21.15.26:6821","nonce":38979282}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6824","nonce":38979282},{"type":"v1","addr":"172.21.15.26:6825","nonce":38979282}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6822","nonce":38979282},{"type":"v1","addr":"172.21.15.26:6823","nonce":38979282}]},"public_addr":"172.21.15.26:6819/38979282","cluster_addr":"172.21.15.26:6821/38979282","heartbeat_back_addr":"172.21.15.26:6825/38979282","heartbeat_front_addr":"172.21.15.26:6823/38979282","state":["exists","up"]},{"osd":3,"uuid":"c3dd2c3f-5cf5-4fb0-964c-1b8926f75c8a","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":26,"last_clean_end":94,"up_from":98,"up_thru":98,"down_at":95,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6826","nonce":2947396145},{"type":"v1","addr":"172.21.15.26:6827","nonce":2947396145}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6828","nonce":2947396145},{"type":"v1","addr":"172.21.15.26:6829","nonce":2947396145}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6832","nonce":2947396145},{"type":"v1","addr":"172.21.15.26:6833","nonce":2947396145}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6830","nonce":2947396145},{"type":"v1","addr":"172.21.15.26:6831","nonce":2947396145}]},"public_addr":"172.21.15.26:6827/2947396145","cluster_addr":"172.21.15.26:6829/2947396145","heartbeat_back_addr":"172.21.15.26:6833/2947396145","heartbeat_front_addr":"172.21.15.26:6831/2947396145","state":["exists","up"]},{"osd":4,"uuid":"a5f31996-1b38-491b-956b-5aebc72d0170","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":31,"up_thru":98,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6800","nonce":1528249138},{"type":"v1","addr":"172.21.15.70:6801","nonce":1528249138}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6802","nonce":1528249138},{"type":"v1","addr":"172.21.15.70:6803","nonce":1528249138}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6806","nonce":1528249138},{"type":"v1","addr":"172.21.15.70:6807","nonce":1528249138}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6804","nonce":1528249138},{"type":"v1","addr":"172.21.15.70:6805","nonce":1528249138}]},"public_addr":"172.21.15.70:6801/1528249138","cluster_addr":"172.21.15.70:6803/1528249138","heartbeat_back_addr":"172.21.15.70:6807/1528249138","heartbeat_front_addr":"172.21.15.70:6805/1528249138","state":["exists","up"]},{"osd":5,"uuid":"21dad198-b9db-42c6-8823-112df1ecd874","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":39,"up_thru":98,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6808","nonce":3802232906},{"type":"v1","addr":"172.21.15.70:6809","nonce":3802232906}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6810","nonce":3802232906},{"type":"v1","addr":"172.21.15.70:6811","nonce":3802232906}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6814","nonce":3802232906},{"type":"v1","addr":"172.21.15.70:6815","nonce":3802232906}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6812","nonce":3802232906},{"type":"v1","addr":"172.21.15.70:6813","nonce":3802232906}]},"public_addr":"172.21.15.70:6809/3802232906","cluster_addr":"172.21.15.70:6811/3802232906","heartbeat_back_addr":"172.21.15.70:6815/3802232906","heartbeat_front_addr":"172.21.15.70:6813/3802232906","state":["exists","up"]},{"osd":6,"uuid":"6f3be784-7149-4f13-81b5-98924e97c056","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":47,"up_thru":95,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6816","nonce":2944175278},{"type":"v1","addr":"172.21.15.70:6817","nonce":2944175278}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6818","nonce":2944175278},{"type":"v1","addr":"172.21.15.70:6819","nonce":2944175278}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6822","nonce":2944175278},{"type":"v1","addr":"172.21.15.70:6823","nonce":2944175278}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6820","nonce":2944175278},{"type":"v1","addr":"172.21.15.70:6821","nonce":2944175278}]},"public_addr":"172.21.15.70:6817/2944175278","cluster_addr":"172.21.15.70:6819/2944175278","heartbeat_back_addr":"172.21.15.70:6823/2944175278","heartbeat_front_addr":"172.21.15.70:6821/2944175278","state":["exists","up"]},{"osd":7,"uuid":"499f5f85-f4b2-40a7-ac0f-79cd22738d30","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":55,"up_thru":98,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6824","nonce":2824974165},{"type":"v1","addr":"172.21.15.70:6825","nonce":2824974165}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6826","nonce":2824974165},{"type":"v1","addr":"172.21.15.70:6827","nonce":2824974165}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6830","nonce":2824974165},{"type":"v1","addr":"172.21.15.70:6831","nonce":2824974165}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6828","nonce":2824974165},{"type":"v1","addr":"172.21.15.70:6829","nonce":2824974165}]},"public_addr":"172.21.15.70:6825/2824974165","cluster_addr":"172.21.15.70:6827/2824974165","heartbeat_back_addr":"172.21.15.70:6831/2824974165","heartbeat_front_addr":"172.21.15.70:6829/2824974165","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T20:38:10.875468+0000","dead_epoch":79},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T20:38:32.449963+0000","dead_epoch":84},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T20:38:53.633316+0000","dead_epoch":89},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T20:39:14.828940+0000","dead_epoch":94},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T20:39:34.945663+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T20:39:55.954288+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T20:40:16.847102+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T20:40:37.820404+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"4.b","mappings":[{"from":1,"to":2}]},{"pgid":"4.18","mappings":[{"from":1,"to":2}]},{"pgid":"4.1c","mappings":[{"from":1,"to":5}]}],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.26:6801/91363814":"2023-09-24T20:41:03.790648+0000","172.21.15.26:0/3175144810":"2023-09-24T20:36:31.159649+0000","172.21.15.26:0/1625326392":"2023-09-24T20:36:56.498559+0000","172.21.15.70:0/3393166416":"2023-09-24T20:44:32.835628+0000","172.21.15.26:0/1083572999":"2023-09-24T20:41:03.790648+0000","172.21.15.26:6801/2440434873":"2023-09-24T20:43:50.641728+0000","172.21.15.70:6834/1275773750":"2023-09-24T20:44:32.835628+0000","172.21.15.26:0/289994511":"2023-09-24T20:36:56.498559+0000","172.21.15.70:0/2989870914":"2023-09-24T20:44:32.835628+0000","172.21.15.26:6800/2440434873":"2023-09-24T20:43:50.641728+0000","172.21.15.70:0/302414747":"2023-09-24T20:44:32.835628+0000","172.21.15.70:0/413499922":"2023-09-24T20:44:32.835628+0000","172.21.15.26:0/4215395943":"2023-09-24T20:43:50.641728+0000","172.21.15.26:0/233963796":"2023-09-24T20:45:16.037413+0000","172.21.15.26:0/934875067":"2023-09-24T20:45:16.037413+0000","172.21.15.70:0/1035724255":"2023-09-24T20:44:32.835628+0000","172.21.15.26:0/1376232363":"2023-09-24T20:43:50.641728+0000","172.21.15.26:0/394447930":"2023-09-24T20:41:03.790648+0000","172.21.15.70:6835/1275773750":"2023-09-24T20:44:32.835628+0000","172.21.15.26:0/190503462":"2023-09-24T20:45:16.037413+0000","172.21.15.70:0/2198692933":"2023-09-24T20:44:32.835628+0000","172.21.15.26:6800/952511372":"2023-09-24T20:36:56.498559+0000","172.21.15.26:0/2633606443":"2023-09-24T20:45:16.037413+0000","172.21.15.26:0/237900280":"2023-09-24T20:43:50.641728+0000","172.21.15.26:0/3461760126":"2023-09-24T20:41:03.790648+0000","172.21.15.26:0/635571892":"2023-09-24T20:43:50.641728+0000","172.21.15.26:0/188049283":"2023-09-24T20:41:03.790648+0000","172.21.15.26:0/1815190465":"2023-09-24T20:36:31.159649+0000","172.21.15.26:0/686734212":"2023-09-24T20:36:31.159649+0000","172.21.15.26:6801/952511372":"2023-09-24T20:36:56.498559+0000","172.21.15.26:0/2838335646":"2023-09-24T20:36:56.498559+0000","172.21.15.26:6801/1446178988":"2023-09-24T20:45:16.037413+0000","172.21.15.26:6800/1446178988":"2023-09-24T20:45:16.037413+0000","172.21.15.26:6800/4099624907":"2023-09-24T20:36:31.159649+0000","172.21.15.26:6801/4099624907":"2023-09-24T20:36:31.159649+0000","172.21.15.26:0/3408257869":"2023-09-24T20:45:16.037413+0000","172.21.15.26:6800/91363814":"2023-09-24T20:41:03.790648+0000","172.21.15.26:0/2817523682":"2023-09-24T20:45:16.037413+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[{"pool":5,"snaps":[{"begin":2,"length":1}]}],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2023-09-23T20:51:30.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:51:29 smithi070 ceph-mon[168282]: pgmap v209: 73 pgs: 73 active+clean; 582 KiB data, 171 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:51:30.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:51:29 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:51:30.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:51:29 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:51:30.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:51:29 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:51:30.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:51:29 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:51:30.573 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph tell osd.0 flush_pg_stats 2023-09-23T20:51:30.574 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph tell osd.1 flush_pg_stats 2023-09-23T20:51:30.574 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph tell osd.2 flush_pg_stats 2023-09-23T20:51:30.575 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph tell osd.3 flush_pg_stats 2023-09-23T20:51:30.575 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph tell osd.4 flush_pg_stats 2023-09-23T20:51:30.575 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph tell osd.5 flush_pg_stats 2023-09-23T20:51:30.576 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph tell osd.6 flush_pg_stats 2023-09-23T20:51:30.576 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph tell osd.7 flush_pg_stats 2023-09-23T20:51:30.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:51:30 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/2581375590' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-09-23T20:51:30.892 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:51:30 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/2581375590' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-09-23T20:51:31.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:51:30 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/2581375590' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-09-23T20:51:31.696 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:51:31.714 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:51:31.856 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:51:31.857 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:51:31.984 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:51:32.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:51:31 smithi070 ceph-mon[168282]: pgmap v210: 73 pgs: 73 active+clean; 582 KiB data, 171 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:51:32.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:51:31 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:51:32.122 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:51:31 smithi026 ceph-mon[179688]: pgmap v210: 73 pgs: 73 active+clean; 582 KiB data, 171 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:51:32.123 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:51:31 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:51:32.123 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:51:31 smithi026 ceph-mon[175151]: pgmap v210: 73 pgs: 73 active+clean; 582 KiB data, 171 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:51:32.123 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:51:31 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:51:32.236 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:51:32.675 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:51:32.967 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:51:34.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:51:33 smithi070 ceph-mon[168282]: pgmap v211: 73 pgs: 73 active+clean; 582 KiB data, 171 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:51:34.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:51:33 smithi026 ceph-mon[175151]: pgmap v211: 73 pgs: 73 active+clean; 582 KiB data, 171 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:51:34.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:51:33 smithi026 ceph-mon[179688]: pgmap v211: 73 pgs: 73 active+clean; 582 KiB data, 171 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:51:35.926 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:51:35 smithi026 ceph-mon[179688]: pgmap v212: 73 pgs: 73 active+clean; 582 KiB data, 171 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:51:35.926 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:51:35 smithi026 ceph-mon[175151]: pgmap v212: 73 pgs: 73 active+clean; 582 KiB data, 171 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:51:36.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:51:35 smithi070 ceph-mon[168282]: pgmap v212: 73 pgs: 73 active+clean; 582 KiB data, 171 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:51:37.475 INFO:teuthology.orchestra.run.smithi026.stdout:201863463050 2023-09-23T20:51:37.475 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.6 2023-09-23T20:51:37.507 INFO:teuthology.orchestra.run.smithi026.stdout:133143986322 2023-09-23T20:51:37.507 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.4 2023-09-23T20:51:38.097 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:51:37 smithi026 ceph-mon[175151]: pgmap v213: 73 pgs: 73 active+clean; 582 KiB data, 171 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:51:38.098 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:51:37 smithi026 ceph-mon[179688]: pgmap v213: 73 pgs: 73 active+clean; 582 KiB data, 171 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:51:38.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:51:37 smithi070 ceph-mon[168282]: pgmap v213: 73 pgs: 73 active+clean; 582 KiB data, 171 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:51:38.366 INFO:teuthology.orchestra.run.smithi026.stdout:399431958541 2023-09-23T20:51:38.367 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.2 2023-09-23T20:51:38.716 INFO:teuthology.orchestra.run.smithi026.stdout:236223201414 2023-09-23T20:51:38.717 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.7 2023-09-23T20:51:38.975 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:51:38 smithi026 ceph-mon[179688]: pgmap v214: 73 pgs: 73 active+clean; 582 KiB data, 171 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:51:38.975 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:51:38 smithi026 ceph-mon[175151]: pgmap v214: 73 pgs: 73 active+clean; 582 KiB data, 171 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:51:38.976 INFO:teuthology.orchestra.run.smithi026.stdout:420906795015 2023-09-23T20:51:38.976 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.3 2023-09-23T20:51:39.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:51:38 smithi070 ceph-mon[168282]: pgmap v214: 73 pgs: 73 active+clean; 582 KiB data, 171 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:51:39.536 INFO:teuthology.orchestra.run.smithi026.stdout:356482285595 2023-09-23T20:51:39.536 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.0 2023-09-23T20:51:39.610 INFO:teuthology.orchestra.run.smithi026.stdout:377957122068 2023-09-23T20:51:39.610 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.1 2023-09-23T20:51:39.882 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:51:39 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[164029]: ::ffff:172.21.15.70 - - [23/Sep/2023:20:51:39] "GET /metrics HTTP/1.1" 200 33815 "" "Prometheus/2.43.0" 2023-09-23T20:51:40.275 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:51:40.326 INFO:teuthology.orchestra.run.smithi026.stdout:167503724688 2023-09-23T20:51:40.327 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.5 2023-09-23T20:51:40.361 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:51:40.367 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:51:40.500 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:51:40.619 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:51:40.620 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:51:40.662 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:51:42.021 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:51:41 smithi026 ceph-mon[179688]: pgmap v215: 73 pgs: 73 active+clean; 582 KiB data, 171 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:51:42.021 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:51:41 smithi026 ceph-mon[175151]: pgmap v215: 73 pgs: 73 active+clean; 582 KiB data, 171 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:51:42.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:51:41 smithi070 ceph-mon[168282]: pgmap v215: 73 pgs: 73 active+clean; 582 KiB data, 171 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:51:43.241 INFO:teuthology.orchestra.run.smithi026.stdout:399431958543 2023-09-23T20:51:44.056 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:51:43 smithi026 ceph-mon[179688]: pgmap v216: 73 pgs: 73 active+clean; 582 KiB data, 171 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:51:44.056 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:51:43 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/3845115494' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2023-09-23T20:51:44.056 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:51:43 smithi026 ceph-mon[175151]: pgmap v216: 73 pgs: 73 active+clean; 582 KiB data, 171 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:51:44.056 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:51:43 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/3845115494' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2023-09-23T20:51:44.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:51:43 smithi070 ceph-mon[168282]: pgmap v216: 73 pgs: 73 active+clean; 582 KiB data, 171 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:51:44.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:51:43 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/3845115494' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2023-09-23T20:51:44.232 INFO:tasks.cephadm.ceph_manager.ceph:need seq 399431958541 got 399431958543 for osd.2 2023-09-23T20:51:44.233 DEBUG:teuthology.parallel:result is None 2023-09-23T20:51:44.263 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:51:44.763 INFO:teuthology.orchestra.run.smithi026.stdout:201863463052 2023-09-23T20:51:44.881 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:51:44 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/1047563953' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2023-09-23T20:51:44.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:51:44 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/1047563953' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2023-09-23T20:51:45.037 INFO:teuthology.orchestra.run.smithi026.stdout:236223201415 2023-09-23T20:51:45.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:51:44 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/1047563953' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2023-09-23T20:51:45.649 INFO:teuthology.orchestra.run.smithi026.stdout:377957122070 2023-09-23T20:51:45.877 INFO:teuthology.orchestra.run.smithi026.stdout:356482285596 2023-09-23T20:51:46.021 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:51:45 smithi026 ceph-mon[179688]: pgmap v217: 73 pgs: 73 active+clean; 582 KiB data, 171 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:51:46.021 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:51:45 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/1230973007' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2023-09-23T20:51:46.021 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:51:45 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/904342092' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2023-09-23T20:51:46.021 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:51:45 smithi026 ceph-mon[175151]: pgmap v217: 73 pgs: 73 active+clean; 582 KiB data, 171 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:51:46.021 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:51:45 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/1230973007' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2023-09-23T20:51:46.021 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:51:45 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/904342092' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2023-09-23T20:51:46.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:51:45 smithi070 ceph-mon[168282]: pgmap v217: 73 pgs: 73 active+clean; 582 KiB data, 171 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:51:46.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:51:45 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/1230973007' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2023-09-23T20:51:46.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:51:45 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/904342092' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2023-09-23T20:51:46.197 INFO:tasks.cephadm.ceph_manager.ceph:need seq 236223201414 got 236223201415 for osd.7 2023-09-23T20:51:46.197 DEBUG:teuthology.parallel:result is None 2023-09-23T20:51:46.514 INFO:tasks.cephadm.ceph_manager.ceph:need seq 356482285595 got 356482285596 for osd.0 2023-09-23T20:51:46.514 DEBUG:teuthology.parallel:result is None 2023-09-23T20:51:46.727 INFO:tasks.cephadm.ceph_manager.ceph:need seq 201863463050 got 201863463052 for osd.6 2023-09-23T20:51:46.727 DEBUG:teuthology.parallel:result is None 2023-09-23T20:51:46.799 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:51:46 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/1744847680' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2023-09-23T20:51:46.799 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:51:46 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:51:46.799 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:51:46 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/1744847680' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2023-09-23T20:51:46.799 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:51:46 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:51:47.010 INFO:teuthology.orchestra.run.smithi026.stdout:133143986324 2023-09-23T20:51:47.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:51:46 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/1744847680' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2023-09-23T20:51:47.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:51:46 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:51:47.208 INFO:tasks.cephadm.ceph_manager.ceph:need seq 377957122068 got 377957122070 for osd.1 2023-09-23T20:51:47.208 DEBUG:teuthology.parallel:result is None 2023-09-23T20:51:47.516 INFO:tasks.cephadm.ceph_manager.ceph:need seq 133143986322 got 133143986324 for osd.4 2023-09-23T20:51:47.516 DEBUG:teuthology.parallel:result is None 2023-09-23T20:51:47.944 INFO:teuthology.orchestra.run.smithi026.stdout:420906795017 2023-09-23T20:51:48.089 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:51:47 smithi026 ceph-mon[175151]: pgmap v218: 73 pgs: 73 active+clean; 582 KiB data, 171 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:51:48.089 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:51:47 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/2561073143' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2023-09-23T20:51:48.090 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:51:47 smithi026 ceph-mon[179688]: pgmap v218: 73 pgs: 73 active+clean; 582 KiB data, 171 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:51:48.090 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:51:47 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/2561073143' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2023-09-23T20:51:48.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:51:47 smithi070 ceph-mon[168282]: pgmap v218: 73 pgs: 73 active+clean; 582 KiB data, 171 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:51:48.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:51:47 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/2561073143' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2023-09-23T20:51:48.171 INFO:teuthology.orchestra.run.smithi026.stdout:167503724689 2023-09-23T20:51:48.700 INFO:tasks.cephadm.ceph_manager.ceph:need seq 420906795015 got 420906795017 for osd.3 2023-09-23T20:51:48.701 DEBUG:teuthology.parallel:result is None 2023-09-23T20:51:48.721 INFO:tasks.cephadm.ceph_manager.ceph:need seq 167503724688 got 167503724689 for osd.5 2023-09-23T20:51:48.721 DEBUG:teuthology.parallel:result is None 2023-09-23T20:51:48.721 INFO:tasks.cephadm.ceph_manager.ceph:waiting for clean 2023-09-23T20:51:48.721 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T20:51:49.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:51:48 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/1303834880' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2023-09-23T20:51:49.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:51:48 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/598379727' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2023-09-23T20:51:49.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:51:48 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/1303834880' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2023-09-23T20:51:49.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:51:48 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/598379727' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2023-09-23T20:51:49.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:51:48 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/1303834880' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2023-09-23T20:51:49.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:51:48 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/598379727' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2023-09-23T20:51:49.372 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:51:49.806 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:51:49 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[164029]: ::ffff:172.21.15.70 - - [23/Sep/2023:20:51:49] "GET /metrics HTTP/1.1" 200 33816 "" "Prometheus/2.43.0" 2023-09-23T20:51:50.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:51:49 smithi070 ceph-mon[168282]: pgmap v219: 73 pgs: 73 active+clean; 582 KiB data, 171 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:51:50.131 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:51:49 smithi026 ceph-mon[179688]: pgmap v219: 73 pgs: 73 active+clean; 582 KiB data, 171 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:51:50.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:51:49 smithi026 ceph-mon[175151]: pgmap v219: 73 pgs: 73 active+clean; 582 KiB data, 171 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:51:50.764 INFO:teuthology.orchestra.run.smithi026.stdout: 2023-09-23T20:51:50.766 INFO:teuthology.orchestra.run.smithi026.stderr:dumped all 2023-09-23T20:51:51.121 INFO:teuthology.orchestra.run.smithi026.stdout:{"pg_ready":true,"pg_map":{"version":220,"stamp":"2023-09-23T20:51:50.157638+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":595872,"num_objects":24,"num_object_clones":0,"num_object_copies":72,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":24,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":300,"num_write_kb":2445,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":216,"ondisk_log_size":216,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":175016,"kb_used_data":9704,"kb_used_omap":126,"kb_used_meta":165121,"kb_avail":749786200,"statfs":{"total":767960285184,"available":767781068800,"internally_reserved":0,"allocated":9936896,"data_stored":6089406,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":129413,"internal_metadata":169084539},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.003126"},"pg_stats":[{"pgid":"4.18","version":"0'0","reported_seq":68,"reported_epoch":98,"state":"active+clean","last_fresh":"2023-09-23T20:51:09.825543+0000","last_change":"2023-09-23T20:50:37.984570+0000","last_active":"2023-09-23T20:51:09.825543+0000","last_peered":"2023-09-23T20:51:09.825543+0000","last_clean":"2023-09-23T20:51:09.825543+0000","last_became_active":"2023-09-23T20:50:37.984262+0000","last_became_peered":"2023-09-23T20:50:37.984262+0000","last_unstale":"2023-09-23T20:51:09.825543+0000","last_undegraded":"2023-09-23T20:51:09.825543+0000","last_fullsized":"2023-09-23T20:51:09.825543+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:47:39.024252+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":56,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-09-23T20:51:10.829873+0000","last_change":"2023-09-23T20:50:37.982897+0000","last_active":"2023-09-23T20:51:10.829873+0000","last_peered":"2023-09-23T20:51:10.829873+0000","last_clean":"2023-09-23T20:51:10.829873+0000","last_became_active":"2023-09-23T20:50:37.982566+0000","last_became_peered":"2023-09-23T20:50:37.982566+0000","last_unstale":"2023-09-23T20:51:10.829873+0000","last_undegraded":"2023-09-23T20:51:10.829873+0000","last_fullsized":"2023-09-23T20:51:10.829873+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:00:00.068989+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":82,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-09-23T20:51:10.838334+0000","last_change":"2023-09-23T20:51:10.838334+0000","last_active":"2023-09-23T20:51:10.838334+0000","last_peered":"2023-09-23T20:51:10.838334+0000","last_clean":"2023-09-23T20:51:10.838334+0000","last_became_active":"2023-09-23T20:51:10.838026+0000","last_became_peered":"2023-09-23T20:51:10.838026+0000","last_unstale":"2023-09-23T20:51:10.838334+0000","last_undegraded":"2023-09-23T20:51:10.838334+0000","last_fullsized":"2023-09-23T20:51:10.838334+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:04:37.009457+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":67,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-09-23T20:51:10.833849+0000","last_change":"2023-09-23T20:51:10.833849+0000","last_active":"2023-09-23T20:51:10.833849+0000","last_peered":"2023-09-23T20:51:10.833849+0000","last_clean":"2023-09-23T20:51:10.833849+0000","last_became_active":"2023-09-23T20:51:10.833524+0000","last_became_peered":"2023-09-23T20:51:10.833524+0000","last_unstale":"2023-09-23T20:51:10.833849+0000","last_undegraded":"2023-09-23T20:51:10.833849+0000","last_fullsized":"2023-09-23T20:51:10.833849+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:23:12.501924+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":75,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-09-23T20:51:10.835094+0000","last_change":"2023-09-23T20:51:10.835094+0000","last_active":"2023-09-23T20:51:10.835094+0000","last_peered":"2023-09-23T20:51:10.835094+0000","last_clean":"2023-09-23T20:51:10.835094+0000","last_became_active":"2023-09-23T20:51:10.834732+0000","last_became_peered":"2023-09-23T20:51:10.834732+0000","last_unstale":"2023-09-23T20:51:10.835094+0000","last_undegraded":"2023-09-23T20:51:10.835094+0000","last_fullsized":"2023-09-23T20:51:10.835094+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:21:08.414737+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":44,"reported_epoch":98,"state":"active+clean","last_fresh":"2023-09-23T20:51:09.819115+0000","last_change":"2023-09-23T20:41:58.906744+0000","last_active":"2023-09-23T20:51:09.819115+0000","last_peered":"2023-09-23T20:51:09.819115+0000","last_clean":"2023-09-23T20:51:09.819115+0000","last_became_active":"2023-09-23T20:41:58.906409+0000","last_became_peered":"2023-09-23T20:41:58.906409+0000","last_unstale":"2023-09-23T20:51:09.819115+0000","last_undegraded":"2023-09-23T20:51:09.819115+0000","last_fullsized":"2023-09-23T20:51:09.819115+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:15:42.250523+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":75,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-09-23T20:51:10.833684+0000","last_change":"2023-09-23T20:51:10.833684+0000","last_active":"2023-09-23T20:51:10.833684+0000","last_peered":"2023-09-23T20:51:10.833684+0000","last_clean":"2023-09-23T20:51:10.833684+0000","last_became_active":"2023-09-23T20:51:10.833360+0000","last_became_peered":"2023-09-23T20:51:10.833360+0000","last_unstale":"2023-09-23T20:51:10.833684+0000","last_undegraded":"2023-09-23T20:51:10.833684+0000","last_fullsized":"2023-09-23T20:51:10.833684+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:11:12.175158+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":61,"reported_epoch":98,"state":"active+clean","last_fresh":"2023-09-23T20:51:09.818991+0000","last_change":"2023-09-23T20:50:05.866461+0000","last_active":"2023-09-23T20:51:09.818991+0000","last_peered":"2023-09-23T20:51:09.818991+0000","last_clean":"2023-09-23T20:51:09.818991+0000","last_became_active":"2023-09-23T20:50:05.866053+0000","last_became_peered":"2023-09-23T20:50:05.866053+0000","last_unstale":"2023-09-23T20:51:09.818991+0000","last_undegraded":"2023-09-23T20:51:09.818991+0000","last_fullsized":"2023-09-23T20:51:09.818991+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:29:36.203264+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":55,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-09-23T20:51:10.835609+0000","last_change":"2023-09-23T20:51:10.835609+0000","last_active":"2023-09-23T20:51:10.835609+0000","last_peered":"2023-09-23T20:51:10.835609+0000","last_clean":"2023-09-23T20:51:10.835609+0000","last_became_active":"2023-09-23T20:51:10.835302+0000","last_became_peered":"2023-09-23T20:51:10.835302+0000","last_unstale":"2023-09-23T20:51:10.835609+0000","last_undegraded":"2023-09-23T20:51:10.835609+0000","last_fullsized":"2023-09-23T20:51:10.835609+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:07:20.599461+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":40,"reported_epoch":98,"state":"active+clean","last_fresh":"2023-09-23T20:51:09.823784+0000","last_change":"2023-09-23T20:49:32.775562+0000","last_active":"2023-09-23T20:51:09.823784+0000","last_peered":"2023-09-23T20:51:09.823784+0000","last_clean":"2023-09-23T20:51:09.823784+0000","last_became_active":"2023-09-23T20:49:32.775094+0000","last_became_peered":"2023-09-23T20:49:32.775094+0000","last_unstale":"2023-09-23T20:51:09.823784+0000","last_undegraded":"2023-09-23T20:51:09.823784+0000","last_fullsized":"2023-09-23T20:51:09.823784+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:17:52.944043+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":41,"reported_epoch":98,"state":"active+clean","last_fresh":"2023-09-23T20:51:09.826361+0000","last_change":"2023-09-23T20:41:58.918255+0000","last_active":"2023-09-23T20:51:09.826361+0000","last_peered":"2023-09-23T20:51:09.826361+0000","last_clean":"2023-09-23T20:51:09.826361+0000","last_became_active":"2023-09-23T20:41:58.917956+0000","last_became_peered":"2023-09-23T20:41:58.917956+0000","last_unstale":"2023-09-23T20:51:09.826361+0000","last_undegraded":"2023-09-23T20:51:09.826361+0000","last_fullsized":"2023-09-23T20:51:09.826361+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:42:21.698474+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.6","version":"70'1","reported_seq":58,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-09-23T20:51:10.833668+0000","last_change":"2023-09-23T20:50:05.871179+0000","last_active":"2023-09-23T20:51:10.833668+0000","last_peered":"2023-09-23T20:51:10.833668+0000","last_clean":"2023-09-23T20:51:10.833668+0000","last_became_active":"2023-09-23T20:50:05.870816+0000","last_became_peered":"2023-09-23T20:50:05.870816+0000","last_unstale":"2023-09-23T20:51:10.833668+0000","last_undegraded":"2023-09-23T20:51:10.833668+0000","last_fullsized":"2023-09-23T20:51:10.833668+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:25:24.455835+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":63,"reported_epoch":98,"state":"active+clean","last_fresh":"2023-09-23T20:51:09.824227+0000","last_change":"2023-09-23T20:50:37.985663+0000","last_active":"2023-09-23T20:51:09.824227+0000","last_peered":"2023-09-23T20:51:09.824227+0000","last_clean":"2023-09-23T20:51:09.824227+0000","last_became_active":"2023-09-23T20:50:37.985325+0000","last_became_peered":"2023-09-23T20:50:37.985325+0000","last_unstale":"2023-09-23T20:51:09.824227+0000","last_undegraded":"2023-09-23T20:51:09.824227+0000","last_fullsized":"2023-09-23T20:51:09.824227+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:24:30.838843+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":56,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-09-23T20:51:10.833546+0000","last_change":"2023-09-23T20:51:10.833546+0000","last_active":"2023-09-23T20:51:10.833546+0000","last_peered":"2023-09-23T20:51:10.833546+0000","last_clean":"2023-09-23T20:51:10.833546+0000","last_became_active":"2023-09-23T20:51:10.833137+0000","last_became_peered":"2023-09-23T20:51:10.833137+0000","last_unstale":"2023-09-23T20:51:10.833546+0000","last_undegraded":"2023-09-23T20:51:10.833546+0000","last_fullsized":"2023-09-23T20:51:10.833546+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:16:42.840695+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":32,"reported_epoch":98,"state":"active+clean","last_fresh":"2023-09-23T20:51:09.833862+0000","last_change":"2023-09-23T20:50:05.868392+0000","last_active":"2023-09-23T20:51:09.833862+0000","last_peered":"2023-09-23T20:51:09.833862+0000","last_clean":"2023-09-23T20:51:09.833862+0000","last_became_active":"2023-09-23T20:50:05.867910+0000","last_became_peered":"2023-09-23T20:50:05.867910+0000","last_unstale":"2023-09-23T20:51:09.833862+0000","last_undegraded":"2023-09-23T20:51:09.833862+0000","last_fullsized":"2023-09-23T20:51:09.833862+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:25:00.099993+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":59,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-09-23T20:51:10.836910+0000","last_change":"2023-09-23T20:51:10.836910+0000","last_active":"2023-09-23T20:51:10.836910+0000","last_peered":"2023-09-23T20:51:10.836910+0000","last_clean":"2023-09-23T20:51:10.836910+0000","last_became_active":"2023-09-23T20:51:10.836557+0000","last_became_peered":"2023-09-23T20:51:10.836557+0000","last_unstale":"2023-09-23T20:51:10.836910+0000","last_undegraded":"2023-09-23T20:51:10.836910+0000","last_fullsized":"2023-09-23T20:51:10.836910+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:51:54.019969+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":38,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-09-23T20:51:10.827806+0000","last_change":"2023-09-23T20:49:32.775934+0000","last_active":"2023-09-23T20:51:10.827806+0000","last_peered":"2023-09-23T20:51:10.827806+0000","last_clean":"2023-09-23T20:51:10.827806+0000","last_became_active":"2023-09-23T20:49:32.775500+0000","last_became_peered":"2023-09-23T20:49:32.775500+0000","last_unstale":"2023-09-23T20:51:10.827806+0000","last_undegraded":"2023-09-23T20:51:10.827806+0000","last_fullsized":"2023-09-23T20:51:10.827806+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:30:43.381073+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":47,"reported_epoch":98,"state":"active+clean","last_fresh":"2023-09-23T20:51:09.835354+0000","last_change":"2023-09-23T20:50:37.982593+0000","last_active":"2023-09-23T20:51:09.835354+0000","last_peered":"2023-09-23T20:51:09.835354+0000","last_clean":"2023-09-23T20:51:09.835354+0000","last_became_active":"2023-09-23T20:50:37.982303+0000","last_became_peered":"2023-09-23T20:50:37.982303+0000","last_unstale":"2023-09-23T20:51:09.835354+0000","last_undegraded":"2023-09-23T20:51:09.835354+0000","last_fullsized":"2023-09-23T20:51:09.835354+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:53:06.301515+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":63,"reported_epoch":98,"state":"active+clean","last_fresh":"2023-09-23T20:51:09.823655+0000","last_change":"2023-09-23T20:49:32.776947+0000","last_active":"2023-09-23T20:51:09.823655+0000","last_peered":"2023-09-23T20:51:09.823655+0000","last_clean":"2023-09-23T20:51:09.823655+0000","last_became_active":"2023-09-23T20:49:32.776646+0000","last_became_peered":"2023-09-23T20:49:32.776646+0000","last_unstale":"2023-09-23T20:51:09.823655+0000","last_undegraded":"2023-09-23T20:51:09.823655+0000","last_fullsized":"2023-09-23T20:51:09.823655+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:26:50.532203+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":68,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-09-23T20:51:10.833744+0000","last_change":"2023-09-23T20:50:37.982235+0000","last_active":"2023-09-23T20:51:10.833744+0000","last_peered":"2023-09-23T20:51:10.833744+0000","last_clean":"2023-09-23T20:51:10.833744+0000","last_became_active":"2023-09-23T20:50:37.981913+0000","last_became_peered":"2023-09-23T20:50:37.981913+0000","last_unstale":"2023-09-23T20:51:10.833744+0000","last_undegraded":"2023-09-23T20:51:10.833744+0000","last_fullsized":"2023-09-23T20:51:10.833744+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:59:57.122584+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":53,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-09-23T20:51:10.841702+0000","last_change":"2023-09-23T20:51:10.841350+0000","last_active":"2023-09-23T20:51:10.841702+0000","last_peered":"2023-09-23T20:51:10.841702+0000","last_clean":"2023-09-23T20:51:10.841702+0000","last_became_active":"2023-09-23T20:51:10.841029+0000","last_became_peered":"2023-09-23T20:51:10.841029+0000","last_unstale":"2023-09-23T20:51:10.841702+0000","last_undegraded":"2023-09-23T20:51:10.841702+0000","last_fullsized":"2023-09-23T20:51:10.841702+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:26:05.095206+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":80,"reported_epoch":98,"state":"active+clean","last_fresh":"2023-09-23T20:51:09.821811+0000","last_change":"2023-09-23T20:50:05.866898+0000","last_active":"2023-09-23T20:51:09.821811+0000","last_peered":"2023-09-23T20:51:09.821811+0000","last_clean":"2023-09-23T20:51:09.821811+0000","last_became_active":"2023-09-23T20:50:05.866541+0000","last_became_peered":"2023-09-23T20:50:05.866541+0000","last_unstale":"2023-09-23T20:51:09.821811+0000","last_undegraded":"2023-09-23T20:51:09.821811+0000","last_fullsized":"2023-09-23T20:51:09.821811+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:17:02.353010+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":41,"reported_epoch":98,"state":"active+clean","last_fresh":"2023-09-23T20:51:09.823968+0000","last_change":"2023-09-23T20:41:59.062981+0000","last_active":"2023-09-23T20:51:09.823968+0000","last_peered":"2023-09-23T20:51:09.823968+0000","last_clean":"2023-09-23T20:51:09.823968+0000","last_became_active":"2023-09-23T20:41:59.062717+0000","last_became_peered":"2023-09-23T20:41:59.062717+0000","last_unstale":"2023-09-23T20:51:09.823968+0000","last_undegraded":"2023-09-23T20:51:09.823968+0000","last_fullsized":"2023-09-23T20:51:09.823968+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:20:58.386461+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":22,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-09-23T20:51:10.836794+0000","last_change":"2023-09-23T20:51:10.836794+0000","last_active":"2023-09-23T20:51:10.836794+0000","last_peered":"2023-09-23T20:51:10.836794+0000","last_clean":"2023-09-23T20:51:10.836794+0000","last_became_active":"2023-09-23T20:51:10.836449+0000","last_became_peered":"2023-09-23T20:51:10.836449+0000","last_unstale":"2023-09-23T20:51:10.836794+0000","last_undegraded":"2023-09-23T20:51:10.836794+0000","last_fullsized":"2023-09-23T20:51:10.836794+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:00:35.572203+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.0","version":"79'161","reported_seq":440,"reported_epoch":98,"state":"active+clean","last_fresh":"2023-09-23T20:51:09.821731+0000","last_change":"2023-09-23T20:49:32.777246+0000","last_active":"2023-09-23T20:51:09.821731+0000","last_peered":"2023-09-23T20:51:09.821731+0000","last_clean":"2023-09-23T20:51:09.821731+0000","last_became_active":"2023-09-23T20:49:32.776822+0000","last_became_peered":"2023-09-23T20:49:32.776822+0000","last_unstale":"2023-09-23T20:51:09.821731+0000","last_undegraded":"2023-09-23T20:51:09.821731+0000","last_fullsized":"2023-09-23T20:51:09.821731+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:38:57.324224+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:05:46.881275+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":63,"reported_epoch":98,"state":"active+clean","last_fresh":"2023-09-23T20:51:09.817117+0000","last_change":"2023-09-23T20:50:37.984163+0000","last_active":"2023-09-23T20:51:09.817117+0000","last_peered":"2023-09-23T20:51:09.817117+0000","last_clean":"2023-09-23T20:51:09.817117+0000","last_became_active":"2023-09-23T20:50:37.983838+0000","last_became_peered":"2023-09-23T20:50:37.983838+0000","last_unstale":"2023-09-23T20:51:09.817117+0000","last_undegraded":"2023-09-23T20:51:09.817117+0000","last_fullsized":"2023-09-23T20:51:09.817117+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:02:41.792156+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":65,"reported_epoch":98,"state":"active+clean","last_fresh":"2023-09-23T20:51:09.835854+0000","last_change":"2023-09-23T20:50:05.867499+0000","last_active":"2023-09-23T20:51:09.835854+0000","last_peered":"2023-09-23T20:51:09.835854+0000","last_clean":"2023-09-23T20:51:09.835854+0000","last_became_active":"2023-09-23T20:50:05.866571+0000","last_became_peered":"2023-09-23T20:50:05.866571+0000","last_unstale":"2023-09-23T20:51:09.835854+0000","last_undegraded":"2023-09-23T20:51:09.835854+0000","last_fullsized":"2023-09-23T20:51:09.835854+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:18:10.735501+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":47,"reported_epoch":98,"state":"active+clean","last_fresh":"2023-09-23T20:51:09.836383+0000","last_change":"2023-09-23T20:50:37.983072+0000","last_active":"2023-09-23T20:51:09.836383+0000","last_peered":"2023-09-23T20:51:09.836383+0000","last_clean":"2023-09-23T20:51:09.836383+0000","last_became_active":"2023-09-23T20:50:37.982777+0000","last_became_peered":"2023-09-23T20:50:37.982777+0000","last_unstale":"2023-09-23T20:51:09.836383+0000","last_undegraded":"2023-09-23T20:51:09.836383+0000","last_fullsized":"2023-09-23T20:51:09.836383+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:58:11.972144+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":81,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-09-23T20:51:10.838054+0000","last_change":"2023-09-23T20:51:10.838054+0000","last_active":"2023-09-23T20:51:10.838054+0000","last_peered":"2023-09-23T20:51:10.838054+0000","last_clean":"2023-09-23T20:51:10.838054+0000","last_became_active":"2023-09-23T20:51:10.837750+0000","last_became_peered":"2023-09-23T20:51:10.837750+0000","last_unstale":"2023-09-23T20:51:10.838054+0000","last_undegraded":"2023-09-23T20:51:10.838054+0000","last_fullsized":"2023-09-23T20:51:10.838054+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:15:51.648085+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":62,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-09-23T20:51:10.829889+0000","last_change":"2023-09-23T20:51:10.829889+0000","last_active":"2023-09-23T20:51:10.829889+0000","last_peered":"2023-09-23T20:51:10.829889+0000","last_clean":"2023-09-23T20:51:10.829889+0000","last_became_active":"2023-09-23T20:51:10.829533+0000","last_became_peered":"2023-09-23T20:51:10.829533+0000","last_unstale":"2023-09-23T20:51:10.829889+0000","last_undegraded":"2023-09-23T20:51:10.829889+0000","last_fullsized":"2023-09-23T20:51:10.829889+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:43:04.603460+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":59,"reported_epoch":98,"state":"active+clean","last_fresh":"2023-09-23T20:51:09.826642+0000","last_change":"2023-09-23T20:49:32.771117+0000","last_active":"2023-09-23T20:51:09.826642+0000","last_peered":"2023-09-23T20:51:09.826642+0000","last_clean":"2023-09-23T20:51:09.826642+0000","last_became_active":"2023-09-23T20:49:32.770686+0000","last_became_peered":"2023-09-23T20:49:32.770686+0000","last_unstale":"2023-09-23T20:51:09.826642+0000","last_undegraded":"2023-09-23T20:51:09.826642+0000","last_fullsized":"2023-09-23T20:51:09.826642+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:15:46.598698+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":63,"reported_epoch":98,"state":"active+clean","last_fresh":"2023-09-23T20:51:09.817089+0000","last_change":"2023-09-23T20:50:05.864620+0000","last_active":"2023-09-23T20:51:09.817089+0000","last_peered":"2023-09-23T20:51:09.817089+0000","last_clean":"2023-09-23T20:51:09.817089+0000","last_became_active":"2023-09-23T20:50:05.864276+0000","last_became_peered":"2023-09-23T20:50:05.864276+0000","last_unstale":"2023-09-23T20:51:09.817089+0000","last_undegraded":"2023-09-23T20:51:09.817089+0000","last_fullsized":"2023-09-23T20:51:09.817089+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:20:19.862901+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":32,"reported_epoch":98,"state":"active+clean","last_fresh":"2023-09-23T20:51:09.834324+0000","last_change":"2023-09-23T20:50:05.868321+0000","last_active":"2023-09-23T20:51:09.834324+0000","last_peered":"2023-09-23T20:51:09.834324+0000","last_clean":"2023-09-23T20:51:09.834324+0000","last_became_active":"2023-09-23T20:50:05.868120+0000","last_became_peered":"2023-09-23T20:50:05.868120+0000","last_unstale":"2023-09-23T20:51:09.834324+0000","last_undegraded":"2023-09-23T20:51:09.834324+0000","last_fullsized":"2023-09-23T20:51:09.834324+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:23:14.856477+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":76,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-09-23T20:51:10.840273+0000","last_change":"2023-09-23T20:51:10.840273+0000","last_active":"2023-09-23T20:51:10.840273+0000","last_peered":"2023-09-23T20:51:10.840273+0000","last_clean":"2023-09-23T20:51:10.840273+0000","last_became_active":"2023-09-23T20:51:10.840017+0000","last_became_peered":"2023-09-23T20:51:10.840017+0000","last_unstale":"2023-09-23T20:51:10.840273+0000","last_undegraded":"2023-09-23T20:51:10.840273+0000","last_fullsized":"2023-09-23T20:51:10.840273+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:58:17.982729+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":70,"reported_epoch":98,"state":"active+clean","last_fresh":"2023-09-23T20:51:09.835307+0000","last_change":"2023-09-23T20:50:05.867130+0000","last_active":"2023-09-23T20:51:09.835307+0000","last_peered":"2023-09-23T20:51:09.835307+0000","last_clean":"2023-09-23T20:51:09.835307+0000","last_became_active":"2023-09-23T20:50:05.866758+0000","last_became_peered":"2023-09-23T20:50:05.866758+0000","last_unstale":"2023-09-23T20:51:09.835307+0000","last_undegraded":"2023-09-23T20:51:09.835307+0000","last_fullsized":"2023-09-23T20:51:09.835307+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:55:07.548209+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":47,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-09-23T20:51:10.833850+0000","last_change":"2023-09-23T20:51:10.833850+0000","last_active":"2023-09-23T20:51:10.833850+0000","last_peered":"2023-09-23T20:51:10.833850+0000","last_clean":"2023-09-23T20:51:10.833850+0000","last_became_active":"2023-09-23T20:51:10.833539+0000","last_became_peered":"2023-09-23T20:51:10.833539+0000","last_unstale":"2023-09-23T20:51:10.833850+0000","last_undegraded":"2023-09-23T20:51:10.833850+0000","last_fullsized":"2023-09-23T20:51:10.833850+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:01:36.445616+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":62,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-09-23T20:51:10.831197+0000","last_change":"2023-09-23T20:51:10.831197+0000","last_active":"2023-09-23T20:51:10.831197+0000","last_peered":"2023-09-23T20:51:10.831197+0000","last_clean":"2023-09-23T20:51:10.831197+0000","last_became_active":"2023-09-23T20:51:10.830878+0000","last_became_peered":"2023-09-23T20:51:10.830878+0000","last_unstale":"2023-09-23T20:51:10.831197+0000","last_undegraded":"2023-09-23T20:51:10.831197+0000","last_fullsized":"2023-09-23T20:51:10.831197+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:05:24.314478+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":58,"reported_epoch":98,"state":"active+clean","last_fresh":"2023-09-23T20:51:09.826968+0000","last_change":"2023-09-23T20:50:05.871406+0000","last_active":"2023-09-23T20:51:09.826968+0000","last_peered":"2023-09-23T20:51:09.826968+0000","last_clean":"2023-09-23T20:51:09.826968+0000","last_became_active":"2023-09-23T20:50:05.871127+0000","last_became_peered":"2023-09-23T20:50:05.871127+0000","last_unstale":"2023-09-23T20:51:09.826968+0000","last_undegraded":"2023-09-23T20:51:09.826968+0000","last_fullsized":"2023-09-23T20:51:09.826968+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:37:48.850229+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":59,"reported_epoch":98,"state":"active+clean","last_fresh":"2023-09-23T20:51:09.826268+0000","last_change":"2023-09-23T20:50:37.983914+0000","last_active":"2023-09-23T20:51:09.826268+0000","last_peered":"2023-09-23T20:51:09.826268+0000","last_clean":"2023-09-23T20:51:09.826268+0000","last_became_active":"2023-09-23T20:50:37.983617+0000","last_became_peered":"2023-09-23T20:50:37.983617+0000","last_unstale":"2023-09-23T20:51:09.826268+0000","last_undegraded":"2023-09-23T20:51:09.826268+0000","last_fullsized":"2023-09-23T20:51:09.826268+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:15:50.863306+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":75,"reported_epoch":98,"state":"active+clean","last_fresh":"2023-09-23T20:51:09.826833+0000","last_change":"2023-09-23T20:50:37.985593+0000","last_active":"2023-09-23T20:51:09.826833+0000","last_peered":"2023-09-23T20:51:09.826833+0000","last_clean":"2023-09-23T20:51:09.826833+0000","last_became_active":"2023-09-23T20:50:37.985324+0000","last_became_peered":"2023-09-23T20:50:37.985324+0000","last_unstale":"2023-09-23T20:51:09.826833+0000","last_undegraded":"2023-09-23T20:51:09.826833+0000","last_fullsized":"2023-09-23T20:51:09.826833+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:34:07.801771+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":59,"reported_epoch":98,"state":"active+clean","last_fresh":"2023-09-23T20:51:09.824658+0000","last_change":"2023-09-23T20:50:05.867646+0000","last_active":"2023-09-23T20:51:09.824658+0000","last_peered":"2023-09-23T20:51:09.824658+0000","last_clean":"2023-09-23T20:51:09.824658+0000","last_became_active":"2023-09-23T20:50:05.867343+0000","last_became_peered":"2023-09-23T20:50:05.867343+0000","last_unstale":"2023-09-23T20:51:09.824658+0000","last_undegraded":"2023-09-23T20:51:09.824658+0000","last_fullsized":"2023-09-23T20:51:09.824658+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:53:34.985580+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":58,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-09-23T20:51:10.833630+0000","last_change":"2023-09-23T20:51:10.833630+0000","last_active":"2023-09-23T20:51:10.833630+0000","last_peered":"2023-09-23T20:51:10.833630+0000","last_clean":"2023-09-23T20:51:10.833630+0000","last_became_active":"2023-09-23T20:51:10.833265+0000","last_became_peered":"2023-09-23T20:51:10.833265+0000","last_unstale":"2023-09-23T20:51:10.833630+0000","last_undegraded":"2023-09-23T20:51:10.833630+0000","last_fullsized":"2023-09-23T20:51:10.833630+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:12:01.557967+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":59,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-09-23T20:51:10.836752+0000","last_change":"2023-09-23T20:51:10.836752+0000","last_active":"2023-09-23T20:51:10.836752+0000","last_peered":"2023-09-23T20:51:10.836752+0000","last_clean":"2023-09-23T20:51:10.836752+0000","last_became_active":"2023-09-23T20:51:10.836213+0000","last_became_peered":"2023-09-23T20:51:10.836213+0000","last_unstale":"2023-09-23T20:51:10.836752+0000","last_undegraded":"2023-09-23T20:51:10.836752+0000","last_fullsized":"2023-09-23T20:51:10.836752+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:35:01.825635+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":66,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-09-23T20:51:10.827903+0000","last_change":"2023-09-23T20:50:37.982263+0000","last_active":"2023-09-23T20:51:10.827903+0000","last_peered":"2023-09-23T20:51:10.827903+0000","last_clean":"2023-09-23T20:51:10.827903+0000","last_became_active":"2023-09-23T20:50:37.981797+0000","last_became_peered":"2023-09-23T20:50:37.981797+0000","last_unstale":"2023-09-23T20:51:10.827903+0000","last_undegraded":"2023-09-23T20:51:10.827903+0000","last_fullsized":"2023-09-23T20:51:10.827903+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:56:45.027447+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":59,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-09-23T20:51:10.839104+0000","last_change":"2023-09-23T20:51:10.839104+0000","last_active":"2023-09-23T20:51:10.839104+0000","last_peered":"2023-09-23T20:51:10.839104+0000","last_clean":"2023-09-23T20:51:10.839104+0000","last_became_active":"2023-09-23T20:51:10.838783+0000","last_became_peered":"2023-09-23T20:51:10.838783+0000","last_unstale":"2023-09-23T20:51:10.839104+0000","last_undegraded":"2023-09-23T20:51:10.839104+0000","last_fullsized":"2023-09-23T20:51:10.839104+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:24:32.550241+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":58,"reported_epoch":98,"state":"active+clean","last_fresh":"2023-09-23T20:51:09.824495+0000","last_change":"2023-09-23T20:50:05.867973+0000","last_active":"2023-09-23T20:51:09.824495+0000","last_peered":"2023-09-23T20:51:09.824495+0000","last_clean":"2023-09-23T20:51:09.824495+0000","last_became_active":"2023-09-23T20:50:05.867622+0000","last_became_peered":"2023-09-23T20:50:05.867622+0000","last_unstale":"2023-09-23T20:51:09.824495+0000","last_undegraded":"2023-09-23T20:51:09.824495+0000","last_fullsized":"2023-09-23T20:51:09.824495+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:10:12.617597+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":43,"reported_epoch":98,"state":"active+clean","last_fresh":"2023-09-23T20:51:09.823751+0000","last_change":"2023-09-23T20:41:58.898346+0000","last_active":"2023-09-23T20:51:09.823751+0000","last_peered":"2023-09-23T20:51:09.823751+0000","last_clean":"2023-09-23T20:51:09.823751+0000","last_became_active":"2023-09-23T20:41:58.898065+0000","last_became_peered":"2023-09-23T20:41:58.898065+0000","last_unstale":"2023-09-23T20:51:09.823751+0000","last_undegraded":"2023-09-23T20:51:09.823751+0000","last_fullsized":"2023-09-23T20:51:09.823751+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:51:57.491171+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":75,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-09-23T20:51:10.838338+0000","last_change":"2023-09-23T20:51:10.838338+0000","last_active":"2023-09-23T20:51:10.838338+0000","last_peered":"2023-09-23T20:51:10.838338+0000","last_clean":"2023-09-23T20:51:10.838338+0000","last_became_active":"2023-09-23T20:51:10.838049+0000","last_became_peered":"2023-09-23T20:51:10.838049+0000","last_unstale":"2023-09-23T20:51:10.838338+0000","last_undegraded":"2023-09-23T20:51:10.838338+0000","last_fullsized":"2023-09-23T20:51:10.838338+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:40:23.602402+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":61,"reported_epoch":98,"state":"active+clean","last_fresh":"2023-09-23T20:51:09.823256+0000","last_change":"2023-09-23T20:50:05.866983+0000","last_active":"2023-09-23T20:51:09.823256+0000","last_peered":"2023-09-23T20:51:09.823256+0000","last_clean":"2023-09-23T20:51:09.823256+0000","last_became_active":"2023-09-23T20:50:05.866601+0000","last_became_peered":"2023-09-23T20:50:05.866601+0000","last_unstale":"2023-09-23T20:51:09.823256+0000","last_undegraded":"2023-09-23T20:51:09.823256+0000","last_fullsized":"2023-09-23T20:51:09.823256+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:15:00.955166+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":41,"reported_epoch":98,"state":"active+clean","last_fresh":"2023-09-23T20:51:09.816955+0000","last_change":"2023-09-23T20:41:59.061405+0000","last_active":"2023-09-23T20:51:09.816955+0000","last_peered":"2023-09-23T20:51:09.816955+0000","last_clean":"2023-09-23T20:51:09.816955+0000","last_became_active":"2023-09-23T20:41:59.061032+0000","last_became_peered":"2023-09-23T20:41:59.061032+0000","last_unstale":"2023-09-23T20:51:09.816955+0000","last_undegraded":"2023-09-23T20:51:09.816955+0000","last_fullsized":"2023-09-23T20:51:09.816955+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:43:09.716350+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":60,"reported_epoch":98,"state":"active+clean","last_fresh":"2023-09-23T20:51:09.822945+0000","last_change":"2023-09-23T20:49:32.777943+0000","last_active":"2023-09-23T20:51:09.822945+0000","last_peered":"2023-09-23T20:51:09.822945+0000","last_clean":"2023-09-23T20:51:09.822945+0000","last_became_active":"2023-09-23T20:49:32.776212+0000","last_became_peered":"2023-09-23T20:49:32.776212+0000","last_unstale":"2023-09-23T20:51:09.822945+0000","last_undegraded":"2023-09-23T20:51:09.822945+0000","last_fullsized":"2023-09-23T20:51:09.822945+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:21:05.725926+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":61,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-09-23T20:51:10.833234+0000","last_change":"2023-09-23T20:51:10.833234+0000","last_active":"2023-09-23T20:51:10.833234+0000","last_peered":"2023-09-23T20:51:10.833234+0000","last_clean":"2023-09-23T20:51:10.833234+0000","last_became_active":"2023-09-23T20:51:10.832763+0000","last_became_peered":"2023-09-23T20:51:10.832763+0000","last_unstale":"2023-09-23T20:51:10.833234+0000","last_undegraded":"2023-09-23T20:51:10.833234+0000","last_fullsized":"2023-09-23T20:51:10.833234+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:24:08.283578+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":67,"reported_epoch":98,"state":"active+clean","last_fresh":"2023-09-23T20:51:09.824402+0000","last_change":"2023-09-23T20:49:32.777437+0000","last_active":"2023-09-23T20:51:09.824402+0000","last_peered":"2023-09-23T20:51:09.824402+0000","last_clean":"2023-09-23T20:51:09.824402+0000","last_became_active":"2023-09-23T20:49:32.776999+0000","last_became_peered":"2023-09-23T20:49:32.776999+0000","last_unstale":"2023-09-23T20:51:09.824402+0000","last_undegraded":"2023-09-23T20:51:09.824402+0000","last_fullsized":"2023-09-23T20:51:09.824402+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:42:21.816073+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":52,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-09-23T20:51:10.840849+0000","last_change":"2023-09-23T20:51:10.840849+0000","last_active":"2023-09-23T20:51:10.840849+0000","last_peered":"2023-09-23T20:51:10.840849+0000","last_clean":"2023-09-23T20:51:10.840849+0000","last_became_active":"2023-09-23T20:51:10.840593+0000","last_became_peered":"2023-09-23T20:51:10.840593+0000","last_unstale":"2023-09-23T20:51:10.840849+0000","last_undegraded":"2023-09-23T20:51:10.840849+0000","last_fullsized":"2023-09-23T20:51:10.840849+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:30:52.297625+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":42,"reported_epoch":98,"state":"active+clean","last_fresh":"2023-09-23T20:51:09.821305+0000","last_change":"2023-09-23T20:41:58.898224+0000","last_active":"2023-09-23T20:51:09.821305+0000","last_peered":"2023-09-23T20:51:09.821305+0000","last_clean":"2023-09-23T20:51:09.821305+0000","last_became_active":"2023-09-23T20:41:58.898036+0000","last_became_peered":"2023-09-23T20:41:58.898036+0000","last_unstale":"2023-09-23T20:51:09.821305+0000","last_undegraded":"2023-09-23T20:51:09.821305+0000","last_fullsized":"2023-09-23T20:51:09.821305+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:53:15.361560+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":58,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-09-23T20:51:10.839174+0000","last_change":"2023-09-23T20:51:10.839174+0000","last_active":"2023-09-23T20:51:10.839174+0000","last_peered":"2023-09-23T20:51:10.839174+0000","last_clean":"2023-09-23T20:51:10.839174+0000","last_became_active":"2023-09-23T20:51:10.838826+0000","last_became_peered":"2023-09-23T20:51:10.838826+0000","last_unstale":"2023-09-23T20:51:10.839174+0000","last_undegraded":"2023-09-23T20:51:10.839174+0000","last_fullsized":"2023-09-23T20:51:10.839174+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:43:00.944119+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":52,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-09-23T20:51:10.840590+0000","last_change":"2023-09-23T20:51:10.840590+0000","last_active":"2023-09-23T20:51:10.840590+0000","last_peered":"2023-09-23T20:51:10.840590+0000","last_clean":"2023-09-23T20:51:10.840590+0000","last_became_active":"2023-09-23T20:51:10.840221+0000","last_became_peered":"2023-09-23T20:51:10.840221+0000","last_unstale":"2023-09-23T20:51:10.840590+0000","last_undegraded":"2023-09-23T20:51:10.840590+0000","last_fullsized":"2023-09-23T20:51:10.840590+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:03:39.138638+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":61,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-09-23T20:51:10.831262+0000","last_change":"2023-09-23T20:51:10.831262+0000","last_active":"2023-09-23T20:51:10.831262+0000","last_peered":"2023-09-23T20:51:10.831262+0000","last_clean":"2023-09-23T20:51:10.831262+0000","last_became_active":"2023-09-23T20:51:10.830931+0000","last_became_peered":"2023-09-23T20:51:10.830931+0000","last_unstale":"2023-09-23T20:51:10.831262+0000","last_undegraded":"2023-09-23T20:51:10.831262+0000","last_fullsized":"2023-09-23T20:51:10.831262+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:51:54.982505+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":61,"reported_epoch":98,"state":"active+clean","last_fresh":"2023-09-23T20:51:09.821700+0000","last_change":"2023-09-23T20:50:37.986729+0000","last_active":"2023-09-23T20:51:09.821700+0000","last_peered":"2023-09-23T20:51:09.821700+0000","last_clean":"2023-09-23T20:51:09.821700+0000","last_became_active":"2023-09-23T20:50:37.986433+0000","last_became_peered":"2023-09-23T20:50:37.986433+0000","last_unstale":"2023-09-23T20:51:09.821700+0000","last_undegraded":"2023-09-23T20:51:09.821700+0000","last_fullsized":"2023-09-23T20:51:09.821700+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:28:28.853091+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":58,"reported_epoch":98,"state":"active+clean","last_fresh":"2023-09-23T20:51:09.826698+0000","last_change":"2023-09-23T20:50:05.871199+0000","last_active":"2023-09-23T20:51:09.826698+0000","last_peered":"2023-09-23T20:51:09.826698+0000","last_clean":"2023-09-23T20:51:09.826698+0000","last_became_active":"2023-09-23T20:50:05.870912+0000","last_became_peered":"2023-09-23T20:50:05.870912+0000","last_unstale":"2023-09-23T20:51:09.826698+0000","last_undegraded":"2023-09-23T20:51:09.826698+0000","last_fullsized":"2023-09-23T20:51:09.826698+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:16:22.718106+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"79'22","reported_seq":64,"reported_epoch":98,"state":"active+clean","last_fresh":"2023-09-23T20:51:09.827615+0000","last_change":"2023-09-23T20:41:58.905866+0000","last_active":"2023-09-23T20:51:09.827615+0000","last_peered":"2023-09-23T20:51:09.827615+0000","last_clean":"2023-09-23T20:51:09.827615+0000","last_became_active":"2023-09-23T20:41:58.905556+0000","last_became_peered":"2023-09-23T20:41:58.905556+0000","last_unstale":"2023-09-23T20:51:09.827615+0000","last_undegraded":"2023-09-23T20:51:09.827615+0000","last_fullsized":"2023-09-23T20:51:09.827615+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:03:14.027348+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":47,"reported_epoch":98,"state":"active+clean","last_fresh":"2023-09-23T20:51:09.834463+0000","last_change":"2023-09-23T20:50:37.982103+0000","last_active":"2023-09-23T20:51:09.834463+0000","last_peered":"2023-09-23T20:51:09.834463+0000","last_clean":"2023-09-23T20:51:09.834463+0000","last_became_active":"2023-09-23T20:50:37.981791+0000","last_became_peered":"2023-09-23T20:50:37.981791+0000","last_unstale":"2023-09-23T20:51:09.834463+0000","last_undegraded":"2023-09-23T20:51:09.834463+0000","last_fullsized":"2023-09-23T20:51:09.834463+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:32:52.286186+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":62,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-09-23T20:51:10.834618+0000","last_change":"2023-09-23T20:51:10.834618+0000","last_active":"2023-09-23T20:51:10.834618+0000","last_peered":"2023-09-23T20:51:10.834618+0000","last_clean":"2023-09-23T20:51:10.834618+0000","last_became_active":"2023-09-23T20:51:10.834267+0000","last_became_peered":"2023-09-23T20:51:10.834267+0000","last_unstale":"2023-09-23T20:51:10.834618+0000","last_undegraded":"2023-09-23T20:51:10.834618+0000","last_fullsized":"2023-09-23T20:51:10.834618+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:38:59.356003+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":22,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-09-23T20:51:10.833790+0000","last_change":"2023-09-23T20:51:10.833790+0000","last_active":"2023-09-23T20:51:10.833790+0000","last_peered":"2023-09-23T20:51:10.833790+0000","last_clean":"2023-09-23T20:51:10.833790+0000","last_became_active":"2023-09-23T20:51:10.832804+0000","last_became_peered":"2023-09-23T20:51:10.832804+0000","last_unstale":"2023-09-23T20:51:10.833790+0000","last_undegraded":"2023-09-23T20:51:10.833790+0000","last_fullsized":"2023-09-23T20:51:10.833790+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:27:42.601674+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":59,"reported_epoch":98,"state":"active+clean","last_fresh":"2023-09-23T20:51:09.817393+0000","last_change":"2023-09-23T20:50:05.863737+0000","last_active":"2023-09-23T20:51:09.817393+0000","last_peered":"2023-09-23T20:51:09.817393+0000","last_clean":"2023-09-23T20:51:09.817393+0000","last_became_active":"2023-09-23T20:50:05.863466+0000","last_became_peered":"2023-09-23T20:50:05.863466+0000","last_unstale":"2023-09-23T20:51:09.817393+0000","last_undegraded":"2023-09-23T20:51:09.817393+0000","last_fullsized":"2023-09-23T20:51:09.817393+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:28:57.602682+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":61,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-09-23T20:51:10.833424+0000","last_change":"2023-09-23T20:51:10.833424+0000","last_active":"2023-09-23T20:51:10.833424+0000","last_peered":"2023-09-23T20:51:10.833424+0000","last_clean":"2023-09-23T20:51:10.833424+0000","last_became_active":"2023-09-23T20:51:10.832808+0000","last_became_peered":"2023-09-23T20:51:10.832808+0000","last_unstale":"2023-09-23T20:51:10.833424+0000","last_undegraded":"2023-09-23T20:51:10.833424+0000","last_fullsized":"2023-09-23T20:51:10.833424+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:32:42.819583+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":52,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-09-23T20:51:10.840587+0000","last_change":"2023-09-23T20:51:10.840587+0000","last_active":"2023-09-23T20:51:10.840587+0000","last_peered":"2023-09-23T20:51:10.840587+0000","last_clean":"2023-09-23T20:51:10.840587+0000","last_became_active":"2023-09-23T20:51:10.840242+0000","last_became_peered":"2023-09-23T20:51:10.840242+0000","last_unstale":"2023-09-23T20:51:10.840587+0000","last_undegraded":"2023-09-23T20:51:10.840587+0000","last_fullsized":"2023-09-23T20:51:10.840587+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:44:07.171314+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":58,"reported_epoch":98,"state":"active+clean","last_fresh":"2023-09-23T20:51:09.824327+0000","last_change":"2023-09-23T20:50:05.867622+0000","last_active":"2023-09-23T20:51:09.824327+0000","last_peered":"2023-09-23T20:51:09.824327+0000","last_clean":"2023-09-23T20:51:09.824327+0000","last_became_active":"2023-09-23T20:50:05.867311+0000","last_became_peered":"2023-09-23T20:50:05.867311+0000","last_unstale":"2023-09-23T20:51:09.824327+0000","last_undegraded":"2023-09-23T20:51:09.824327+0000","last_fullsized":"2023-09-23T20:51:09.824327+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:31:53.285316+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":79,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-09-23T20:51:10.833613+0000","last_change":"2023-09-23T20:51:10.833613+0000","last_active":"2023-09-23T20:51:10.833613+0000","last_peered":"2023-09-23T20:51:10.833613+0000","last_clean":"2023-09-23T20:51:10.833613+0000","last_became_active":"2023-09-23T20:51:10.833208+0000","last_became_peered":"2023-09-23T20:51:10.833208+0000","last_unstale":"2023-09-23T20:51:10.833613+0000","last_undegraded":"2023-09-23T20:51:10.833613+0000","last_fullsized":"2023-09-23T20:51:10.833613+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:51:28.246014+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":53,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-09-23T20:51:10.841782+0000","last_change":"2023-09-23T20:51:10.841420+0000","last_active":"2023-09-23T20:51:10.841782+0000","last_peered":"2023-09-23T20:51:10.841782+0000","last_clean":"2023-09-23T20:51:10.841782+0000","last_became_active":"2023-09-23T20:51:10.841112+0000","last_became_peered":"2023-09-23T20:51:10.841112+0000","last_unstale":"2023-09-23T20:51:10.841782+0000","last_undegraded":"2023-09-23T20:51:10.841782+0000","last_fullsized":"2023-09-23T20:51:10.841782+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:44:02.236650+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":47,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-09-23T20:51:10.833035+0000","last_change":"2023-09-23T20:51:10.833035+0000","last_active":"2023-09-23T20:51:10.833035+0000","last_peered":"2023-09-23T20:51:10.833035+0000","last_clean":"2023-09-23T20:51:10.833035+0000","last_became_active":"2023-09-23T20:51:10.832741+0000","last_became_peered":"2023-09-23T20:51:10.832741+0000","last_unstale":"2023-09-23T20:51:10.833035+0000","last_undegraded":"2023-09-23T20:51:10.833035+0000","last_fullsized":"2023-09-23T20:51:10.833035+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:53:16.471852+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":61,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-09-23T20:51:10.835170+0000","last_change":"2023-09-23T20:51:10.835170+0000","last_active":"2023-09-23T20:51:10.835170+0000","last_peered":"2023-09-23T20:51:10.835170+0000","last_clean":"2023-09-23T20:51:10.835170+0000","last_became_active":"2023-09-23T20:51:10.834808+0000","last_became_peered":"2023-09-23T20:51:10.834808+0000","last_unstale":"2023-09-23T20:51:10.835170+0000","last_undegraded":"2023-09-23T20:51:10.835170+0000","last_fullsized":"2023-09-23T20:51:10.835170+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:51:25.153742+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"68'1","reported_seq":77,"reported_epoch":98,"state":"active+clean","last_fresh":"2023-09-23T20:51:09.826718+0000","last_change":"2023-09-23T20:50:37.985580+0000","last_active":"2023-09-23T20:51:09.826718+0000","last_peered":"2023-09-23T20:51:09.826718+0000","last_clean":"2023-09-23T20:51:09.826718+0000","last_became_active":"2023-09-23T20:50:37.985229+0000","last_became_peered":"2023-09-23T20:50:37.985229+0000","last_unstale":"2023-09-23T20:51:09.826718+0000","last_undegraded":"2023-09-23T20:51:09.826718+0000","last_fullsized":"2023-09-23T20:51:09.826718+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:52:50.591507+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":161,"ondisk_log_size":161,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201416,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":22696,"kb_used_data":1568,"kb_used_omap":0,"kb_used_meta":21120,"kb_avail":93722456,"statfs":{"total":95995035648,"available":95971794944,"internally_reserved":0,"allocated":1605632,"data_stored":1128632,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":21626880},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:51:35 2023","interfaces":[{"interface":"back","average":{"1min":0.622,"5min":0.51700000000000002,"15min":0.499},"min":{"1min":0.45900000000000002,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":0.80400000000000005,"5min":0.80400000000000005,"15min":0.80400000000000005},"last":0.69399999999999995},{"interface":"front","average":{"1min":0.55200000000000005,"5min":0.56000000000000005,"15min":0.56200000000000006},"min":{"1min":0.33700000000000002,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":0.79000000000000004,"5min":0.79000000000000004,"15min":0.79000000000000004},"last":0.60499999999999998}]},{"osd":1,"last update":"Sat Sep 23 20:51:05 2023","interfaces":[{"interface":"back","average":{"1min":0.54900000000000004,"5min":0.54900000000000004,"15min":0.54900000000000004},"min":{"1min":0.40999999999999998,"5min":0.40999999999999998,"15min":0.40999999999999998},"max":{"1min":0.73099999999999998,"5min":0.73099999999999998,"15min":0.73099999999999998},"last":0.51400000000000001},{"interface":"front","average":{"1min":0.54400000000000004,"5min":0.54400000000000004,"15min":0.54400000000000004},"min":{"1min":0.32100000000000001,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":0.71599999999999997,"5min":0.71599999999999997,"15min":0.71599999999999997},"last":0.61699999999999999}]},{"osd":2,"last update":"Sat Sep 23 20:51:41 2023","interfaces":[{"interface":"back","average":{"1min":0.55100000000000005,"5min":0.55100000000000005,"15min":0.55100000000000005},"min":{"1min":0.33300000000000002,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":0.83099999999999996,"5min":0.83099999999999996,"15min":0.83099999999999996},"last":0.59099999999999997},{"interface":"front","average":{"1min":0.58399999999999996,"5min":0.58399999999999996,"15min":0.58399999999999996},"min":{"1min":0.375,"5min":0.375,"15min":0.375},"max":{"1min":0.81299999999999994,"5min":0.81299999999999994,"15min":0.81299999999999994},"last":0.755}]},{"osd":3,"last update":"Sat Sep 23 20:50:58 2023","interfaces":[{"interface":"back","average":{"1min":0.58199999999999996,"5min":0.58599999999999997,"15min":0.56499999999999995},"min":{"1min":0.40000000000000002,"5min":0.309,"15min":0.28000000000000003},"max":{"1min":0.755,"5min":0.83299999999999996,"15min":1.159},"last":0.66100000000000003},{"interface":"front","average":{"1min":0.60699999999999998,"5min":0.61099999999999999,"15min":0.58799999999999997},"min":{"1min":0.435,"5min":0.376,"15min":0.32900000000000001},"max":{"1min":0.77800000000000002,"5min":0.84099999999999997,"15min":1.5840000000000001},"last":0.67900000000000005}]},{"osd":4,"last update":"Sat Sep 23 20:50:58 2023","interfaces":[{"interface":"back","average":{"1min":0.55000000000000004,"5min":0.51100000000000001,"15min":0.48099999999999998},"min":{"1min":0.34999999999999998,"5min":0.312,"15min":0.254},"max":{"1min":0.77000000000000002,"5min":0.82099999999999995,"15min":0.82099999999999995},"last":0.55800000000000005},{"interface":"front","average":{"1min":0.56999999999999995,"5min":0.54100000000000004,"15min":0.52000000000000002},"min":{"1min":0.39200000000000002,"5min":0.35999999999999999,"15min":0.29599999999999999},"max":{"1min":0.80000000000000004,"5min":0.873,"15min":1.1100000000000001},"last":0.64000000000000001}]},{"osd":5,"last update":"Sat Sep 23 20:50:58 2023","interfaces":[{"interface":"back","average":{"1min":0.59199999999999997,"5min":0.55400000000000005,"15min":0.53100000000000003},"min":{"1min":0.42399999999999999,"5min":0.318,"15min":0.30199999999999999},"max":{"1min":0.78500000000000003,"5min":0.89900000000000002,"15min":1.4610000000000001},"last":0.70999999999999996},{"interface":"front","average":{"1min":0.63400000000000001,"5min":0.58199999999999996,"15min":0.55100000000000005},"min":{"1min":0.39500000000000002,"5min":0.34300000000000003,"15min":0.30599999999999999},"max":{"1min":0.84999999999999998,"5min":0.85899999999999999,"15min":0.85899999999999999},"last":0.65300000000000002}]},{"osd":6,"last update":"Sat Sep 23 20:50:58 2023","interfaces":[{"interface":"back","average":{"1min":0.623,"5min":0.60599999999999998,"15min":0.56899999999999995},"min":{"1min":0.433,"5min":0.41599999999999998,"15min":0.32600000000000001},"max":{"1min":0.88200000000000001,"5min":0.88300000000000001,"15min":0.88900000000000001},"last":0.72499999999999998},{"interface":"front","average":{"1min":0.64800000000000002,"5min":0.60599999999999998,"15min":0.58499999999999996},"min":{"1min":0.47599999999999998,"5min":0.36299999999999999,"15min":0.316},"max":{"1min":0.83999999999999997,"5min":0.90700000000000003,"15min":1.0760000000000001},"last":0.73199999999999998}]}]},{"osd":6,"up_from":47,"seq":201863463053,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":7832,"kb_used_data":1552,"kb_used_omap":0,"kb_used_meta":6272,"kb_avail":93737320,"statfs":{"total":95995035648,"available":95987015680,"internally_reserved":0,"allocated":1589248,"data_stored":1128221,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":6422528},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:51:35 2023","interfaces":[{"interface":"back","average":{"1min":0.50900000000000001,"5min":0.56499999999999995,"15min":0.57399999999999995},"min":{"1min":0.33400000000000002,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":0.75700000000000001,"5min":0.79200000000000004,"15min":0.79200000000000004},"last":0.45800000000000002},{"interface":"front","average":{"1min":0.54600000000000004,"5min":0.58099999999999996,"15min":0.58699999999999997},"min":{"1min":0.34200000000000003,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":0.68799999999999994,"5min":0.89100000000000001,"15min":0.89100000000000001},"last":0.39500000000000002}]},{"osd":1,"last update":"Sat Sep 23 20:51:06 2023","interfaces":[{"interface":"back","average":{"1min":0.53400000000000003,"5min":0.53400000000000003,"15min":0.53400000000000003},"min":{"1min":0.371,"5min":0.371,"15min":0.371},"max":{"1min":0.72499999999999998,"5min":0.72499999999999998,"15min":0.72499999999999998},"last":0.41599999999999998},{"interface":"front","average":{"1min":0.56999999999999995,"5min":0.56999999999999995,"15min":0.56999999999999995},"min":{"1min":0.38300000000000001,"5min":0.38300000000000001,"15min":0.38300000000000001},"max":{"1min":0.79700000000000004,"5min":0.79700000000000004,"15min":0.79700000000000004},"last":0.51000000000000001}]},{"osd":2,"last update":"Sat Sep 23 20:51:41 2023","interfaces":[{"interface":"back","average":{"1min":0.54800000000000004,"5min":0.54800000000000004,"15min":0.54800000000000004},"min":{"1min":0.40899999999999997,"5min":0.40899999999999997,"15min":0.40899999999999997},"max":{"1min":0.71999999999999997,"5min":0.71999999999999997,"15min":0.71999999999999997},"last":0.56599999999999995},{"interface":"front","average":{"1min":0.58599999999999997,"5min":0.58599999999999997,"15min":0.58599999999999997},"min":{"1min":0.35799999999999998,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":0.73399999999999999,"5min":0.73399999999999999,"15min":0.73399999999999999},"last":0.65900000000000003}]},{"osd":3,"last update":"Sat Sep 23 20:50:38 2023","interfaces":[{"interface":"back","average":{"1min":0.61599999999999999,"5min":0.60199999999999998,"15min":0.56200000000000006},"min":{"1min":0.439,"5min":0.34699999999999998,"15min":0.33200000000000002},"max":{"1min":0.82699999999999996,"5min":0.92100000000000004,"15min":0.96099999999999997},"last":0.59199999999999997},{"interface":"front","average":{"1min":0.60699999999999998,"5min":0.59599999999999997,"15min":0.56100000000000005},"min":{"1min":0.42899999999999999,"5min":0.39900000000000002,"15min":0.34100000000000003},"max":{"1min":0.81999999999999995,"5min":0.83999999999999997,"15min":0.874},"last":0.60699999999999998}]},{"osd":4,"last update":"Sat Sep 23 20:51:41 2023","interfaces":[{"interface":"back","average":{"1min":0.503,"5min":0.47599999999999998,"15min":0.45700000000000002},"min":{"1min":0.374,"5min":0.28000000000000003,"15min":0.27800000000000002},"max":{"1min":0.68400000000000005,"5min":0.81000000000000005,"15min":1.1779999999999999},"last":0.63300000000000001},{"interface":"front","average":{"1min":0.54500000000000004,"5min":0.52300000000000002,"15min":0.49399999999999999},"min":{"1min":0.376,"5min":0.30599999999999999,"15min":0.29199999999999998},"max":{"1min":0.77000000000000002,"5min":0.81599999999999995,"15min":0.81599999999999995},"last":0.64500000000000002}]},{"osd":5,"last update":"Sat Sep 23 20:51:41 2023","interfaces":[{"interface":"back","average":{"1min":0.53600000000000003,"5min":0.51600000000000001,"15min":0.49199999999999999},"min":{"1min":0.39600000000000002,"5min":0.32600000000000001,"15min":0.28399999999999997},"max":{"1min":0.69999999999999996,"5min":0.82699999999999996,"15min":1.081},"last":0.71999999999999997},{"interface":"front","average":{"1min":0.59299999999999997,"5min":0.56000000000000005,"15min":0.52300000000000002},"min":{"1min":0.374,"5min":0.33100000000000002,"15min":0.28499999999999998},"max":{"1min":0.755,"5min":0.81000000000000005,"15min":0.89000000000000001},"last":0.72999999999999998}]},{"osd":7,"last update":"Sat Sep 23 20:51:02 2023","interfaces":[{"interface":"back","average":{"1min":0.58299999999999996,"5min":0.56899999999999995,"15min":0.53500000000000003},"min":{"1min":0.38600000000000001,"5min":0.33000000000000002,"15min":0.32500000000000001},"max":{"1min":0.78900000000000003,"5min":0.82299999999999995,"15min":1.119},"last":0.68799999999999994},{"interface":"front","average":{"1min":0.63600000000000001,"5min":0.61599999999999999,"15min":0.57899999999999996},"min":{"1min":0.42199999999999999,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":0.79200000000000004,"5min":0.94099999999999995,"15min":0.94099999999999995},"last":0.70199999999999996}]}]},{"osd":1,"up_from":88,"seq":377957122071,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28480,"kb_used_data":1016,"kb_used_omap":9,"kb_used_meta":27446,"kb_avail":93716672,"statfs":{"total":95995035648,"available":95965872128,"internally_reserved":0,"allocated":1040384,"data_stored":544044,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9239,"internal_metadata":28105705},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:51:04 2023","interfaces":[{"interface":"back","average":{"1min":0.40899999999999997,"5min":0.40899999999999997,"15min":0.40899999999999997},"min":{"1min":0.27300000000000002,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":0.63200000000000001,"5min":0.63200000000000001,"15min":0.63200000000000001},"last":0.46800000000000003},{"interface":"front","average":{"1min":0.435,"5min":0.435,"15min":0.435},"min":{"1min":0.31,"5min":0.31,"15min":0.31},"max":{"1min":0.73999999999999999,"5min":0.73999999999999999,"15min":0.73999999999999999},"last":0.36599999999999999}]},{"osd":2,"last update":"Sat Sep 23 20:51:37 2023","interfaces":[{"interface":"back","average":{"1min":0.38700000000000001,"5min":0.38700000000000001,"15min":0.38700000000000001},"min":{"1min":0.26400000000000001,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":0.503,"5min":0.503,"15min":0.503},"last":0.505},{"interface":"front","average":{"1min":0.58499999999999996,"5min":0.58499999999999996,"15min":0.58499999999999996},"min":{"1min":0.29099999999999998,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":3.5219999999999998,"5min":3.5219999999999998,"15min":3.5219999999999998},"last":0.41599999999999998}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.443}]},{"osd":4,"last update":"Sat Sep 23 20:51:04 2023","interfaces":[{"interface":"back","average":{"1min":0.64700000000000002,"5min":0.64700000000000002,"15min":0.64700000000000002},"min":{"1min":0.435,"5min":0.435,"15min":0.435},"max":{"1min":0.88200000000000001,"5min":0.88200000000000001,"15min":0.88200000000000001},"last":0.81299999999999994},{"interface":"front","average":{"1min":0.67300000000000004,"5min":0.67300000000000004,"15min":0.67300000000000004},"min":{"1min":0.44400000000000001,"5min":0.44400000000000001,"15min":0.44400000000000001},"max":{"1min":0.92900000000000005,"5min":0.92900000000000005,"15min":0.92900000000000005},"last":0.78900000000000003}]},{"osd":5,"last update":"Sat Sep 23 20:51:04 2023","interfaces":[{"interface":"back","average":{"1min":0.69799999999999995,"5min":0.69799999999999995,"15min":0.69799999999999995},"min":{"1min":0.39500000000000002,"5min":0.39500000000000002,"15min":0.39500000000000002},"max":{"1min":0.86899999999999999,"5min":0.86899999999999999,"15min":0.86899999999999999},"last":0.76700000000000002},{"interface":"front","average":{"1min":0.64700000000000002,"5min":0.64700000000000002,"15min":0.64700000000000002},"min":{"1min":0.47899999999999998,"5min":0.47899999999999998,"15min":0.47899999999999998},"max":{"1min":0.79200000000000004,"5min":0.79200000000000004,"15min":0.79200000000000004},"last":0.65000000000000002}]},{"osd":6,"last update":"Sat Sep 23 20:51:04 2023","interfaces":[{"interface":"back","average":{"1min":0.68799999999999994,"5min":0.68799999999999994,"15min":0.68799999999999994},"min":{"1min":0.47999999999999998,"5min":0.47999999999999998,"15min":0.47999999999999998},"max":{"1min":0.98799999999999999,"5min":0.98799999999999999,"15min":0.98799999999999999},"last":0.86699999999999999},{"interface":"front","average":{"1min":0.65800000000000003,"5min":0.65800000000000003,"15min":0.65800000000000003},"min":{"1min":0.41799999999999998,"5min":0.41799999999999998,"15min":0.41799999999999998},"max":{"1min":0.90900000000000003,"5min":0.90900000000000003,"15min":0.90900000000000003},"last":0.72399999999999998}]},{"osd":7,"last update":"Sat Sep 23 20:51:04 2023","interfaces":[{"interface":"back","average":{"1min":0.67400000000000004,"5min":0.67400000000000004,"15min":0.67400000000000004},"min":{"1min":0.46999999999999997,"5min":0.46999999999999997,"15min":0.46999999999999997},"max":{"1min":1.004,"5min":1.004,"15min":1.004},"last":0.85199999999999998},{"interface":"front","average":{"1min":0.68600000000000005,"5min":0.68600000000000005,"15min":0.68600000000000005},"min":{"1min":0.502,"5min":0.502,"15min":0.502},"max":{"1min":1.036,"5min":1.036,"15min":1.036},"last":0.80300000000000005}]}]},{"osd":0,"up_from":83,"seq":356482285597,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":29140,"kb_used_data":1612,"kb_used_omap":28,"kb_used_meta":27491,"kb_avail":93716012,"statfs":{"total":95995035648,"available":95965196288,"internally_reserved":0,"allocated":1650688,"data_stored":1136306,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":28848,"internal_metadata":28151632},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 20:51:07 2023","interfaces":[{"interface":"back","average":{"1min":0.33300000000000002,"5min":0.33300000000000002,"15min":0.33300000000000002},"min":{"1min":0.23400000000000001,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":0.503,"5min":0.503,"15min":0.503},"last":0.51200000000000001},{"interface":"front","average":{"1min":0.38700000000000001,"5min":0.38700000000000001,"15min":0.38700000000000001},"min":{"1min":0.26100000000000001,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":0.54100000000000004,"5min":0.54100000000000004,"15min":0.54100000000000004},"last":0.58999999999999997}]},{"osd":2,"last update":"Sat Sep 23 20:51:37 2023","interfaces":[{"interface":"back","average":{"1min":0.41499999999999998,"5min":0.41499999999999998,"15min":0.41499999999999998},"min":{"1min":0.248,"5min":0.248,"15min":0.248},"max":{"1min":0.61799999999999999,"5min":0.61799999999999999,"15min":0.61799999999999999},"last":0.67700000000000005},{"interface":"front","average":{"1min":0.39000000000000001,"5min":0.39000000000000001,"15min":0.39000000000000001},"min":{"1min":0.25700000000000001,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":0.55900000000000005,"5min":0.55900000000000005,"15min":0.55900000000000005},"last":0.86099999999999999}]},{"osd":3,"last update":"Sat Sep 23 20:50:33 2023","interfaces":[{"interface":"back","average":{"1min":0.46000000000000002,"5min":0.46000000000000002,"15min":0.46000000000000002},"min":{"1min":0.28899999999999998,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":0.72699999999999998,"5min":0.72699999999999998,"15min":0.72699999999999998},"last":0.70099999999999996},{"interface":"front","average":{"1min":0.45500000000000002,"5min":0.45500000000000002,"15min":0.45500000000000002},"min":{"1min":0.33100000000000002,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":0.64400000000000002,"5min":0.64400000000000002,"15min":0.64400000000000002},"last":0.98799999999999999}]},{"osd":4,"last update":"Sat Sep 23 20:51:37 2023","interfaces":[{"interface":"back","average":{"1min":0.64800000000000002,"5min":0.64800000000000002,"15min":0.64800000000000002},"min":{"1min":0.377,"5min":0.377,"15min":0.377},"max":{"1min":1.9870000000000001,"5min":1.9870000000000001,"15min":1.9870000000000001},"last":0.96699999999999997},{"interface":"front","average":{"1min":0.68899999999999995,"5min":0.66500000000000004,"15min":0.66100000000000003},"min":{"1min":0.41399999999999998,"5min":0.41399999999999998,"15min":0.41399999999999998},"max":{"1min":2.0049999999999999,"5min":2.0049999999999999,"15min":2.0049999999999999},"last":0.753}]},{"osd":5,"last update":"Sat Sep 23 20:51:37 2023","interfaces":[{"interface":"back","average":{"1min":0.60099999999999998,"5min":0.63900000000000001,"15min":0.64500000000000002},"min":{"1min":0.42299999999999999,"5min":0.42299999999999999,"15min":0.42299999999999999},"max":{"1min":0.80400000000000005,"5min":0.83799999999999997,"15min":0.83799999999999997},"last":0.84399999999999997},{"interface":"front","average":{"1min":0.60799999999999998,"5min":0.68700000000000006,"15min":0.69999999999999996},"min":{"1min":0.42499999999999999,"5min":0.42499999999999999,"15min":0.42499999999999999},"max":{"1min":0.88400000000000001,"5min":0.90000000000000002,"15min":0.90000000000000002},"last":0.93899999999999995}]},{"osd":6,"last update":"Sat Sep 23 20:51:37 2023","interfaces":[{"interface":"back","average":{"1min":0.60799999999999998,"5min":0.67400000000000004,"15min":0.68500000000000005},"min":{"1min":0.433,"5min":0.433,"15min":0.433},"max":{"1min":0.876,"5min":0.93999999999999995,"15min":0.93999999999999995},"last":0.90400000000000003},{"interface":"front","average":{"1min":0.72099999999999997,"5min":0.68899999999999995,"15min":0.68300000000000005},"min":{"1min":0.45000000000000001,"5min":0.44600000000000001,"15min":0.44600000000000001},"max":{"1min":2.0489999999999999,"5min":2.0489999999999999,"15min":2.0489999999999999},"last":0.81999999999999995}]},{"osd":7,"last update":"Sat Sep 23 20:51:37 2023","interfaces":[{"interface":"back","average":{"1min":0.64000000000000001,"5min":0.68799999999999994,"15min":0.69599999999999995},"min":{"1min":0.42799999999999999,"5min":0.42799999999999999,"15min":0.42799999999999999},"max":{"1min":0.82399999999999995,"5min":0.92000000000000004,"15min":0.92000000000000004},"last":0.91700000000000004},{"interface":"front","average":{"1min":0.63900000000000001,"5min":0.66700000000000004,"15min":0.67100000000000004},"min":{"1min":0.442,"5min":0.442,"15min":0.442},"max":{"1min":0.85799999999999998,"5min":0.90600000000000003,"15min":0.90600000000000003},"last":0.879}]}]},{"osd":2,"up_from":93,"seq":399431958544,"num_pgs":19,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28600,"kb_used_data":1008,"kb_used_omap":81,"kb_used_meta":27502,"kb_avail":93716552,"statfs":{"total":95995035648,"available":95965749248,"internally_reserved":0,"allocated":1032192,"data_stored":541255,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":83750,"internal_metadata":28162266},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:51:38 2023","interfaces":[{"interface":"back","average":{"1min":0.35799999999999998,"5min":0.35799999999999998,"15min":0.35799999999999998},"min":{"1min":0.249,"5min":0.249,"15min":0.249},"max":{"1min":0.54500000000000004,"5min":0.54500000000000004,"15min":0.54500000000000004},"last":0.30099999999999999},{"interface":"front","average":{"1min":0.40699999999999997,"5min":0.40699999999999997,"15min":0.40699999999999997},"min":{"1min":0.309,"5min":0.309,"15min":0.309},"max":{"1min":0.64800000000000002,"5min":0.64800000000000002,"15min":0.64800000000000002},"last":0.33600000000000002}]},{"osd":1,"last update":"Sat Sep 23 20:51:38 2023","interfaces":[{"interface":"back","average":{"1min":0.41699999999999998,"5min":0.41699999999999998,"15min":0.41699999999999998},"min":{"1min":0.31,"5min":0.31,"15min":0.31},"max":{"1min":0.61699999999999999,"5min":0.61699999999999999,"15min":0.61699999999999999},"last":0.39100000000000001},{"interface":"front","average":{"1min":0.42099999999999999,"5min":0.42099999999999999,"15min":0.42099999999999999},"min":{"1min":0.33200000000000002,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":0.55600000000000005,"5min":0.55600000000000005,"15min":0.55600000000000005},"last":0.379}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.28199999999999997}]},{"osd":4,"last update":"Sat Sep 23 20:51:38 2023","interfaces":[{"interface":"back","average":{"1min":0.61899999999999999,"5min":0.61899999999999999,"15min":0.61899999999999999},"min":{"1min":0.45200000000000001,"5min":0.45200000000000001,"15min":0.45200000000000001},"max":{"1min":0.86099999999999999,"5min":0.86099999999999999,"15min":0.86099999999999999},"last":0.54900000000000004},{"interface":"front","average":{"1min":0.67100000000000004,"5min":0.67100000000000004,"15min":0.67100000000000004},"min":{"1min":0.47299999999999998,"5min":0.47299999999999998,"15min":0.47299999999999998},"max":{"1min":0.92400000000000004,"5min":0.92400000000000004,"15min":0.92400000000000004},"last":0.47899999999999998}]},{"osd":5,"last update":"Sat Sep 23 20:51:38 2023","interfaces":[{"interface":"back","average":{"1min":0.65200000000000002,"5min":0.65200000000000002,"15min":0.65200000000000002},"min":{"1min":0.46300000000000002,"5min":0.46300000000000002,"15min":0.46300000000000002},"max":{"1min":0.95999999999999996,"5min":0.95999999999999996,"15min":0.95999999999999996},"last":0.51400000000000001},{"interface":"front","average":{"1min":0.63600000000000001,"5min":0.63600000000000001,"15min":0.63600000000000001},"min":{"1min":0.42299999999999999,"5min":0.42299999999999999,"15min":0.42299999999999999},"max":{"1min":0.89600000000000002,"5min":0.89600000000000002,"15min":0.89600000000000002},"last":0.56399999999999995}]},{"osd":6,"last update":"Sat Sep 23 20:51:38 2023","interfaces":[{"interface":"back","average":{"1min":0.70599999999999996,"5min":0.70599999999999996,"15min":0.70599999999999996},"min":{"1min":0.502,"5min":0.502,"15min":0.502},"max":{"1min":0.97299999999999998,"5min":0.97299999999999998,"15min":0.97299999999999998},"last":0.504},{"interface":"front","average":{"1min":0.68600000000000005,"5min":0.68600000000000005,"15min":0.68600000000000005},"min":{"1min":0.49399999999999999,"5min":0.49399999999999999,"15min":0.49399999999999999},"max":{"1min":0.93999999999999995,"5min":0.93999999999999995,"15min":0.93999999999999995},"last":0.61699999999999999}]},{"osd":7,"last update":"Sat Sep 23 20:51:38 2023","interfaces":[{"interface":"back","average":{"1min":0.63500000000000001,"5min":0.63500000000000001,"15min":0.63500000000000001},"min":{"1min":0.498,"5min":0.498,"15min":0.498},"max":{"1min":0.90600000000000003,"5min":0.90600000000000003,"15min":0.90600000000000003},"last":0.57599999999999996},{"interface":"front","average":{"1min":0.66200000000000003,"5min":0.66200000000000003,"15min":0.66200000000000003},"min":{"1min":0.433,"5min":0.433,"15min":0.433},"max":{"1min":0.90500000000000003,"5min":0.90500000000000003,"15min":0.90500000000000003},"last":0.46800000000000003}]}]},{"osd":3,"up_from":98,"seq":420906795018,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28452,"kb_used_data":988,"kb_used_omap":7,"kb_used_meta":27448,"kb_avail":93716700,"statfs":{"total":95995035648,"available":95965900800,"internally_reserved":0,"allocated":1011712,"data_stored":538170,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7576,"internal_metadata":28107368},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.36699999999999999}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.38600000000000001}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.47299999999999998}]},{"osd":4,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.63200000000000001}]},{"osd":5,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.67300000000000004}]},{"osd":6,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.73899999999999999}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.70699999999999996}]}]},{"osd":4,"up_from":31,"seq":133143986325,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":11876,"kb_used_data":988,"kb_used_omap":0,"kb_used_meta":10880,"kb_avail":93733276,"statfs":{"total":95995035648,"available":95982874624,"internally_reserved":0,"allocated":1011712,"data_stored":538517,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":11141120},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:51:35 2023","interfaces":[{"interface":"back","average":{"1min":0.745,"5min":0.65600000000000003,"15min":0.64100000000000001},"min":{"1min":0.40000000000000002,"5min":0.38900000000000001,"15min":0.38900000000000001},"max":{"1min":4.4050000000000002,"5min":4.4050000000000002,"15min":4.4050000000000002},"last":0.52400000000000002},{"interface":"front","average":{"1min":0.748,"5min":0.88200000000000001,"15min":0.90400000000000003},"min":{"1min":0.46899999999999997,"5min":0.46899999999999997,"15min":0.46899999999999997},"max":{"1min":4.468,"5min":5.2409999999999997,"15min":5.2409999999999997},"last":0.56999999999999995}]},{"osd":1,"last update":"Sat Sep 23 20:51:06 2023","interfaces":[{"interface":"back","average":{"1min":0.55900000000000005,"5min":0.55900000000000005,"15min":0.55900000000000005},"min":{"1min":0.44700000000000001,"5min":0.44700000000000001,"15min":0.44700000000000001},"max":{"1min":0.79900000000000004,"5min":0.79900000000000004,"15min":0.79900000000000004},"last":0.45200000000000001},{"interface":"front","average":{"1min":0.59899999999999998,"5min":0.59899999999999998,"15min":0.59899999999999998},"min":{"1min":0.44500000000000001,"5min":0.44500000000000001,"15min":0.44500000000000001},"max":{"1min":0.72399999999999998,"5min":0.72399999999999998,"15min":0.72399999999999998},"last":0.70499999999999996}]},{"osd":2,"last update":"Sat Sep 23 20:51:41 2023","interfaces":[{"interface":"back","average":{"1min":0.68600000000000005,"5min":0.68600000000000005,"15min":0.68600000000000005},"min":{"1min":0.442,"5min":0.442,"15min":0.442},"max":{"1min":2.5030000000000001,"5min":2.5030000000000001,"15min":2.5030000000000001},"last":0.751},{"interface":"front","average":{"1min":0.77400000000000002,"5min":0.77400000000000002,"15min":0.77400000000000002},"min":{"1min":0.38200000000000001,"5min":0.38200000000000001,"15min":0.38200000000000001},"max":{"1min":4.3289999999999997,"5min":4.3289999999999997,"15min":4.3289999999999997},"last":0.78400000000000003}]},{"osd":3,"last update":"Sat Sep 23 20:50:04 2023","interfaces":[{"interface":"back","average":{"1min":0.63300000000000001,"5min":0.59999999999999998,"15min":0.56799999999999995},"min":{"1min":0.41799999999999998,"5min":0.35299999999999998,"15min":0.33100000000000002},"max":{"1min":2.1320000000000001,"5min":2.1320000000000001,"15min":2.1320000000000001},"last":0.59899999999999998},{"interface":"front","average":{"1min":0.53500000000000003,"5min":0.57799999999999996,"15min":0.55900000000000005},"min":{"1min":0.35699999999999998,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":0.83499999999999996,"5min":0.88,"15min":0.91500000000000004},"last":0.80900000000000005}]},{"osd":5,"last update":"Sat Sep 23 20:51:17 2023","interfaces":[{"interface":"back","average":{"1min":0.57399999999999995,"5min":0.502,"15min":0.45100000000000001},"min":{"1min":0.40000000000000002,"5min":0.29099999999999998,"15min":0.251},"max":{"1min":0.80900000000000005,"5min":0.80900000000000005,"15min":0.83599999999999997},"last":0.83799999999999997},{"interface":"front","average":{"1min":0.54400000000000004,"5min":0.51100000000000001,"15min":0.46000000000000002},"min":{"1min":0.371,"5min":0.33400000000000002,"15min":0.28799999999999998},"max":{"1min":0.79500000000000004,"5min":0.79500000000000004,"15min":0.90000000000000002},"last":0.67900000000000005}]},{"osd":6,"last update":"Sat Sep 23 20:50:49 2023","interfaces":[{"interface":"back","average":{"1min":0.52600000000000002,"5min":0.51200000000000001,"15min":0.51100000000000001},"min":{"1min":0.36399999999999999,"5min":0.30299999999999999,"15min":0.28100000000000003},"max":{"1min":0.67000000000000004,"5min":0.748,"15min":1.1319999999999999},"last":0.72199999999999998},{"interface":"front","average":{"1min":0.59499999999999997,"5min":0.56799999999999995,"15min":0.56399999999999995},"min":{"1min":0.42999999999999999,"5min":0.316,"15min":0.29399999999999998},"max":{"1min":0.79200000000000004,"5min":0.82699999999999996,"15min":1.095},"last":0.85999999999999999}]},{"osd":7,"last update":"Sat Sep 23 20:51:06 2023","interfaces":[{"interface":"back","average":{"1min":0.63500000000000001,"5min":0.59299999999999997,"15min":0.56599999999999995},"min":{"1min":0.44700000000000001,"5min":0.35099999999999998,"15min":0.32500000000000001},"max":{"1min":0.77100000000000002,"5min":0.79100000000000004,"15min":0.877},"last":0.88400000000000001},{"interface":"front","average":{"1min":0.62,"5min":0.58599999999999997,"15min":0.56499999999999995},"min":{"1min":0.42499999999999999,"5min":0.33500000000000002,"15min":0.32700000000000001},"max":{"1min":0.79600000000000004,"5min":0.79600000000000004,"15min":0.86299999999999999},"last":0.82399999999999995}]}]},{"osd":5,"up_from":39,"seq":167503724690,"num_pgs":21,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":17940,"kb_used_data":972,"kb_used_omap":0,"kb_used_meta":16960,"kb_avail":93727212,"statfs":{"total":95995035648,"available":95976665088,"internally_reserved":0,"allocated":995328,"data_stored":534261,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":17367040},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:51:34 2023","interfaces":[{"interface":"back","average":{"1min":0.51300000000000001,"5min":0.499,"15min":0.497},"min":{"1min":0.37,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":0.745,"5min":0.745,"15min":0.745},"last":0.57299999999999995},{"interface":"front","average":{"1min":0.57399999999999995,"5min":0.57499999999999996,"15min":0.57499999999999996},"min":{"1min":0.35399999999999998,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":0.88400000000000001,"5min":0.88400000000000001,"15min":0.88400000000000001},"last":0.438}]},{"osd":1,"last update":"Sat Sep 23 20:51:05 2023","interfaces":[{"interface":"back","average":{"1min":0.56599999999999995,"5min":0.56599999999999995,"15min":0.56599999999999995},"min":{"1min":0.42499999999999999,"5min":0.42499999999999999,"15min":0.42499999999999999},"max":{"1min":0.83899999999999997,"5min":0.83899999999999997,"15min":0.83899999999999997},"last":0.46100000000000002},{"interface":"front","average":{"1min":0.54900000000000004,"5min":0.54900000000000004,"15min":0.54900000000000004},"min":{"1min":0.39000000000000001,"5min":0.39000000000000001,"15min":0.39000000000000001},"max":{"1min":0.70899999999999996,"5min":0.70899999999999996,"15min":0.70899999999999996},"last":0.38800000000000001}]},{"osd":2,"last update":"Sat Sep 23 20:51:39 2023","interfaces":[{"interface":"back","average":{"1min":0.54200000000000004,"5min":0.54200000000000004,"15min":0.54200000000000004},"min":{"1min":0.436,"5min":0.436,"15min":0.436},"max":{"1min":0.68500000000000005,"5min":0.68500000000000005,"15min":0.68500000000000005},"last":0.495},{"interface":"front","average":{"1min":0.58399999999999996,"5min":0.58399999999999996,"15min":0.58399999999999996},"min":{"1min":0.40200000000000002,"5min":0.40200000000000002,"15min":0.40200000000000002},"max":{"1min":0.81699999999999995,"5min":0.81699999999999995,"15min":0.81699999999999995},"last":0.41499999999999998}]},{"osd":3,"last update":"Sat Sep 23 20:50:16 2023","interfaces":[{"interface":"back","average":{"1min":0.61099999999999999,"5min":0.59199999999999997,"15min":0.56599999999999995},"min":{"1min":0.45600000000000002,"5min":0.39000000000000001,"15min":0.29399999999999998},"max":{"1min":0.79300000000000004,"5min":0.81999999999999995,"15min":0.96099999999999997},"last":0.47999999999999998},{"interface":"front","average":{"1min":0.59099999999999997,"5min":0.61299999999999999,"15min":0.58399999999999996},"min":{"1min":0.38,"5min":0.37,"15min":0.33100000000000002},"max":{"1min":0.78500000000000003,"5min":0.90400000000000003,"15min":0.97699999999999998},"last":0.61599999999999999}]},{"osd":4,"last update":"Sat Sep 23 20:51:17 2023","interfaces":[{"interface":"back","average":{"1min":0.53000000000000003,"5min":0.50900000000000001,"15min":0.45900000000000002},"min":{"1min":0.32700000000000001,"5min":0.317,"15min":0.253},"max":{"1min":0.75900000000000001,"5min":0.80900000000000005,"15min":0.80900000000000005},"last":0.56200000000000006},{"interface":"front","average":{"1min":0.54400000000000004,"5min":0.53500000000000003,"15min":0.496},"min":{"1min":0.28499999999999998,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":0.83499999999999996,"5min":0.83499999999999996,"15min":0.89500000000000002},"last":0.65600000000000003}]},{"osd":6,"last update":"Sat Sep 23 20:51:44 2023","interfaces":[{"interface":"back","average":{"1min":0.57399999999999995,"5min":0.54800000000000004,"15min":0.51100000000000001},"min":{"1min":0.34599999999999997,"5min":0.309,"15min":0.28599999999999998},"max":{"1min":0.86799999999999999,"5min":0.86899999999999999,"15min":0.91400000000000003},"last":0.53600000000000003},{"interface":"front","average":{"1min":0.63700000000000001,"5min":0.57799999999999996,"15min":0.54200000000000004},"min":{"1min":0.36299999999999999,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":0.85399999999999998,"5min":0.85399999999999998,"15min":0.85399999999999998},"last":0.60099999999999998}]},{"osd":7,"last update":"Sat Sep 23 20:51:02 2023","interfaces":[{"interface":"back","average":{"1min":0.57299999999999995,"5min":0.58199999999999996,"15min":0.56100000000000005},"min":{"1min":0.38,"5min":0.33600000000000002,"15min":0.314},"max":{"1min":0.77900000000000003,"5min":0.873,"15min":0.874},"last":0.59299999999999997},{"interface":"front","average":{"1min":0.624,"5min":0.61699999999999999,"15min":0.57899999999999996},"min":{"1min":0.41399999999999998,"5min":0.36099999999999999,"15min":0.34100000000000003},"max":{"1min":0.80100000000000005,"5min":0.88500000000000001,"15min":0.88500000000000001},"last":0.64300000000000002}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T20:51:51.128 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T20:51:51.734 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:51:52.081 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:51:51 smithi026 ceph-mon[179688]: pgmap v220: 73 pgs: 73 active+clean; 582 KiB data, 171 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:51:52.081 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:51:51 smithi026 ceph-mon[179688]: from='client.34704 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:51:52.081 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:51:51 smithi026 ceph-mon[175151]: pgmap v220: 73 pgs: 73 active+clean; 582 KiB data, 171 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:51:52.082 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:51:51 smithi026 ceph-mon[175151]: from='client.34704 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:51:52.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:51:51 smithi070 ceph-mon[168282]: pgmap v220: 73 pgs: 73 active+clean; 582 KiB data, 171 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:51:52.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:51:51 smithi070 ceph-mon[168282]: from='client.34704 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:51:53.135 INFO:teuthology.orchestra.run.smithi026.stdout: 2023-09-23T20:51:53.136 INFO:teuthology.orchestra.run.smithi026.stderr:dumped all 2023-09-23T20:51:53.494 INFO:teuthology.orchestra.run.smithi026.stdout:{"pg_ready":true,"pg_map":{"version":221,"stamp":"2023-09-23T20:51:52.158036+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":595872,"num_objects":24,"num_object_clones":0,"num_object_copies":72,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":24,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":300,"num_write_kb":2445,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":216,"ondisk_log_size":216,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":175016,"kb_used_data":9704,"kb_used_omap":126,"kb_used_meta":165121,"kb_avail":749786200,"statfs":{"total":767960285184,"available":767781068800,"internally_reserved":0,"allocated":9936896,"data_stored":6089406,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":129413,"internal_metadata":169084539},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002783"},"pg_stats":[{"pgid":"4.18","version":"0'0","reported_seq":68,"reported_epoch":98,"state":"active+clean","last_fresh":"2023-09-23T20:51:09.825543+0000","last_change":"2023-09-23T20:50:37.984570+0000","last_active":"2023-09-23T20:51:09.825543+0000","last_peered":"2023-09-23T20:51:09.825543+0000","last_clean":"2023-09-23T20:51:09.825543+0000","last_became_active":"2023-09-23T20:50:37.984262+0000","last_became_peered":"2023-09-23T20:50:37.984262+0000","last_unstale":"2023-09-23T20:51:09.825543+0000","last_undegraded":"2023-09-23T20:51:09.825543+0000","last_fullsized":"2023-09-23T20:51:09.825543+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:47:39.024252+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":56,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-09-23T20:51:10.829873+0000","last_change":"2023-09-23T20:50:37.982897+0000","last_active":"2023-09-23T20:51:10.829873+0000","last_peered":"2023-09-23T20:51:10.829873+0000","last_clean":"2023-09-23T20:51:10.829873+0000","last_became_active":"2023-09-23T20:50:37.982566+0000","last_became_peered":"2023-09-23T20:50:37.982566+0000","last_unstale":"2023-09-23T20:51:10.829873+0000","last_undegraded":"2023-09-23T20:51:10.829873+0000","last_fullsized":"2023-09-23T20:51:10.829873+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:00:00.068989+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":82,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-09-23T20:51:10.838334+0000","last_change":"2023-09-23T20:51:10.838334+0000","last_active":"2023-09-23T20:51:10.838334+0000","last_peered":"2023-09-23T20:51:10.838334+0000","last_clean":"2023-09-23T20:51:10.838334+0000","last_became_active":"2023-09-23T20:51:10.838026+0000","last_became_peered":"2023-09-23T20:51:10.838026+0000","last_unstale":"2023-09-23T20:51:10.838334+0000","last_undegraded":"2023-09-23T20:51:10.838334+0000","last_fullsized":"2023-09-23T20:51:10.838334+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:04:37.009457+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":67,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-09-23T20:51:10.833849+0000","last_change":"2023-09-23T20:51:10.833849+0000","last_active":"2023-09-23T20:51:10.833849+0000","last_peered":"2023-09-23T20:51:10.833849+0000","last_clean":"2023-09-23T20:51:10.833849+0000","last_became_active":"2023-09-23T20:51:10.833524+0000","last_became_peered":"2023-09-23T20:51:10.833524+0000","last_unstale":"2023-09-23T20:51:10.833849+0000","last_undegraded":"2023-09-23T20:51:10.833849+0000","last_fullsized":"2023-09-23T20:51:10.833849+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:23:12.501924+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":75,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-09-23T20:51:10.835094+0000","last_change":"2023-09-23T20:51:10.835094+0000","last_active":"2023-09-23T20:51:10.835094+0000","last_peered":"2023-09-23T20:51:10.835094+0000","last_clean":"2023-09-23T20:51:10.835094+0000","last_became_active":"2023-09-23T20:51:10.834732+0000","last_became_peered":"2023-09-23T20:51:10.834732+0000","last_unstale":"2023-09-23T20:51:10.835094+0000","last_undegraded":"2023-09-23T20:51:10.835094+0000","last_fullsized":"2023-09-23T20:51:10.835094+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:21:08.414737+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":44,"reported_epoch":98,"state":"active+clean","last_fresh":"2023-09-23T20:51:09.819115+0000","last_change":"2023-09-23T20:41:58.906744+0000","last_active":"2023-09-23T20:51:09.819115+0000","last_peered":"2023-09-23T20:51:09.819115+0000","last_clean":"2023-09-23T20:51:09.819115+0000","last_became_active":"2023-09-23T20:41:58.906409+0000","last_became_peered":"2023-09-23T20:41:58.906409+0000","last_unstale":"2023-09-23T20:51:09.819115+0000","last_undegraded":"2023-09-23T20:51:09.819115+0000","last_fullsized":"2023-09-23T20:51:09.819115+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:15:42.250523+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":75,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-09-23T20:51:10.833684+0000","last_change":"2023-09-23T20:51:10.833684+0000","last_active":"2023-09-23T20:51:10.833684+0000","last_peered":"2023-09-23T20:51:10.833684+0000","last_clean":"2023-09-23T20:51:10.833684+0000","last_became_active":"2023-09-23T20:51:10.833360+0000","last_became_peered":"2023-09-23T20:51:10.833360+0000","last_unstale":"2023-09-23T20:51:10.833684+0000","last_undegraded":"2023-09-23T20:51:10.833684+0000","last_fullsized":"2023-09-23T20:51:10.833684+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:11:12.175158+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":61,"reported_epoch":98,"state":"active+clean","last_fresh":"2023-09-23T20:51:09.818991+0000","last_change":"2023-09-23T20:50:05.866461+0000","last_active":"2023-09-23T20:51:09.818991+0000","last_peered":"2023-09-23T20:51:09.818991+0000","last_clean":"2023-09-23T20:51:09.818991+0000","last_became_active":"2023-09-23T20:50:05.866053+0000","last_became_peered":"2023-09-23T20:50:05.866053+0000","last_unstale":"2023-09-23T20:51:09.818991+0000","last_undegraded":"2023-09-23T20:51:09.818991+0000","last_fullsized":"2023-09-23T20:51:09.818991+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:29:36.203264+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":55,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-09-23T20:51:10.835609+0000","last_change":"2023-09-23T20:51:10.835609+0000","last_active":"2023-09-23T20:51:10.835609+0000","last_peered":"2023-09-23T20:51:10.835609+0000","last_clean":"2023-09-23T20:51:10.835609+0000","last_became_active":"2023-09-23T20:51:10.835302+0000","last_became_peered":"2023-09-23T20:51:10.835302+0000","last_unstale":"2023-09-23T20:51:10.835609+0000","last_undegraded":"2023-09-23T20:51:10.835609+0000","last_fullsized":"2023-09-23T20:51:10.835609+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:07:20.599461+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":40,"reported_epoch":98,"state":"active+clean","last_fresh":"2023-09-23T20:51:09.823784+0000","last_change":"2023-09-23T20:49:32.775562+0000","last_active":"2023-09-23T20:51:09.823784+0000","last_peered":"2023-09-23T20:51:09.823784+0000","last_clean":"2023-09-23T20:51:09.823784+0000","last_became_active":"2023-09-23T20:49:32.775094+0000","last_became_peered":"2023-09-23T20:49:32.775094+0000","last_unstale":"2023-09-23T20:51:09.823784+0000","last_undegraded":"2023-09-23T20:51:09.823784+0000","last_fullsized":"2023-09-23T20:51:09.823784+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:17:52.944043+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":41,"reported_epoch":98,"state":"active+clean","last_fresh":"2023-09-23T20:51:09.826361+0000","last_change":"2023-09-23T20:41:58.918255+0000","last_active":"2023-09-23T20:51:09.826361+0000","last_peered":"2023-09-23T20:51:09.826361+0000","last_clean":"2023-09-23T20:51:09.826361+0000","last_became_active":"2023-09-23T20:41:58.917956+0000","last_became_peered":"2023-09-23T20:41:58.917956+0000","last_unstale":"2023-09-23T20:51:09.826361+0000","last_undegraded":"2023-09-23T20:51:09.826361+0000","last_fullsized":"2023-09-23T20:51:09.826361+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:42:21.698474+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.6","version":"70'1","reported_seq":58,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-09-23T20:51:10.833668+0000","last_change":"2023-09-23T20:50:05.871179+0000","last_active":"2023-09-23T20:51:10.833668+0000","last_peered":"2023-09-23T20:51:10.833668+0000","last_clean":"2023-09-23T20:51:10.833668+0000","last_became_active":"2023-09-23T20:50:05.870816+0000","last_became_peered":"2023-09-23T20:50:05.870816+0000","last_unstale":"2023-09-23T20:51:10.833668+0000","last_undegraded":"2023-09-23T20:51:10.833668+0000","last_fullsized":"2023-09-23T20:51:10.833668+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:25:24.455835+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":63,"reported_epoch":98,"state":"active+clean","last_fresh":"2023-09-23T20:51:09.824227+0000","last_change":"2023-09-23T20:50:37.985663+0000","last_active":"2023-09-23T20:51:09.824227+0000","last_peered":"2023-09-23T20:51:09.824227+0000","last_clean":"2023-09-23T20:51:09.824227+0000","last_became_active":"2023-09-23T20:50:37.985325+0000","last_became_peered":"2023-09-23T20:50:37.985325+0000","last_unstale":"2023-09-23T20:51:09.824227+0000","last_undegraded":"2023-09-23T20:51:09.824227+0000","last_fullsized":"2023-09-23T20:51:09.824227+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:24:30.838843+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":56,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-09-23T20:51:10.833546+0000","last_change":"2023-09-23T20:51:10.833546+0000","last_active":"2023-09-23T20:51:10.833546+0000","last_peered":"2023-09-23T20:51:10.833546+0000","last_clean":"2023-09-23T20:51:10.833546+0000","last_became_active":"2023-09-23T20:51:10.833137+0000","last_became_peered":"2023-09-23T20:51:10.833137+0000","last_unstale":"2023-09-23T20:51:10.833546+0000","last_undegraded":"2023-09-23T20:51:10.833546+0000","last_fullsized":"2023-09-23T20:51:10.833546+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:16:42.840695+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":32,"reported_epoch":98,"state":"active+clean","last_fresh":"2023-09-23T20:51:09.833862+0000","last_change":"2023-09-23T20:50:05.868392+0000","last_active":"2023-09-23T20:51:09.833862+0000","last_peered":"2023-09-23T20:51:09.833862+0000","last_clean":"2023-09-23T20:51:09.833862+0000","last_became_active":"2023-09-23T20:50:05.867910+0000","last_became_peered":"2023-09-23T20:50:05.867910+0000","last_unstale":"2023-09-23T20:51:09.833862+0000","last_undegraded":"2023-09-23T20:51:09.833862+0000","last_fullsized":"2023-09-23T20:51:09.833862+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:25:00.099993+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":59,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-09-23T20:51:10.836910+0000","last_change":"2023-09-23T20:51:10.836910+0000","last_active":"2023-09-23T20:51:10.836910+0000","last_peered":"2023-09-23T20:51:10.836910+0000","last_clean":"2023-09-23T20:51:10.836910+0000","last_became_active":"2023-09-23T20:51:10.836557+0000","last_became_peered":"2023-09-23T20:51:10.836557+0000","last_unstale":"2023-09-23T20:51:10.836910+0000","last_undegraded":"2023-09-23T20:51:10.836910+0000","last_fullsized":"2023-09-23T20:51:10.836910+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:51:54.019969+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":38,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-09-23T20:51:10.827806+0000","last_change":"2023-09-23T20:49:32.775934+0000","last_active":"2023-09-23T20:51:10.827806+0000","last_peered":"2023-09-23T20:51:10.827806+0000","last_clean":"2023-09-23T20:51:10.827806+0000","last_became_active":"2023-09-23T20:49:32.775500+0000","last_became_peered":"2023-09-23T20:49:32.775500+0000","last_unstale":"2023-09-23T20:51:10.827806+0000","last_undegraded":"2023-09-23T20:51:10.827806+0000","last_fullsized":"2023-09-23T20:51:10.827806+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:30:43.381073+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":47,"reported_epoch":98,"state":"active+clean","last_fresh":"2023-09-23T20:51:09.835354+0000","last_change":"2023-09-23T20:50:37.982593+0000","last_active":"2023-09-23T20:51:09.835354+0000","last_peered":"2023-09-23T20:51:09.835354+0000","last_clean":"2023-09-23T20:51:09.835354+0000","last_became_active":"2023-09-23T20:50:37.982303+0000","last_became_peered":"2023-09-23T20:50:37.982303+0000","last_unstale":"2023-09-23T20:51:09.835354+0000","last_undegraded":"2023-09-23T20:51:09.835354+0000","last_fullsized":"2023-09-23T20:51:09.835354+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:53:06.301515+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":63,"reported_epoch":98,"state":"active+clean","last_fresh":"2023-09-23T20:51:09.823655+0000","last_change":"2023-09-23T20:49:32.776947+0000","last_active":"2023-09-23T20:51:09.823655+0000","last_peered":"2023-09-23T20:51:09.823655+0000","last_clean":"2023-09-23T20:51:09.823655+0000","last_became_active":"2023-09-23T20:49:32.776646+0000","last_became_peered":"2023-09-23T20:49:32.776646+0000","last_unstale":"2023-09-23T20:51:09.823655+0000","last_undegraded":"2023-09-23T20:51:09.823655+0000","last_fullsized":"2023-09-23T20:51:09.823655+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:26:50.532203+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":68,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-09-23T20:51:10.833744+0000","last_change":"2023-09-23T20:50:37.982235+0000","last_active":"2023-09-23T20:51:10.833744+0000","last_peered":"2023-09-23T20:51:10.833744+0000","last_clean":"2023-09-23T20:51:10.833744+0000","last_became_active":"2023-09-23T20:50:37.981913+0000","last_became_peered":"2023-09-23T20:50:37.981913+0000","last_unstale":"2023-09-23T20:51:10.833744+0000","last_undegraded":"2023-09-23T20:51:10.833744+0000","last_fullsized":"2023-09-23T20:51:10.833744+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:59:57.122584+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":53,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-09-23T20:51:10.841702+0000","last_change":"2023-09-23T20:51:10.841350+0000","last_active":"2023-09-23T20:51:10.841702+0000","last_peered":"2023-09-23T20:51:10.841702+0000","last_clean":"2023-09-23T20:51:10.841702+0000","last_became_active":"2023-09-23T20:51:10.841029+0000","last_became_peered":"2023-09-23T20:51:10.841029+0000","last_unstale":"2023-09-23T20:51:10.841702+0000","last_undegraded":"2023-09-23T20:51:10.841702+0000","last_fullsized":"2023-09-23T20:51:10.841702+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:26:05.095206+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":80,"reported_epoch":98,"state":"active+clean","last_fresh":"2023-09-23T20:51:09.821811+0000","last_change":"2023-09-23T20:50:05.866898+0000","last_active":"2023-09-23T20:51:09.821811+0000","last_peered":"2023-09-23T20:51:09.821811+0000","last_clean":"2023-09-23T20:51:09.821811+0000","last_became_active":"2023-09-23T20:50:05.866541+0000","last_became_peered":"2023-09-23T20:50:05.866541+0000","last_unstale":"2023-09-23T20:51:09.821811+0000","last_undegraded":"2023-09-23T20:51:09.821811+0000","last_fullsized":"2023-09-23T20:51:09.821811+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:17:02.353010+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":41,"reported_epoch":98,"state":"active+clean","last_fresh":"2023-09-23T20:51:09.823968+0000","last_change":"2023-09-23T20:41:59.062981+0000","last_active":"2023-09-23T20:51:09.823968+0000","last_peered":"2023-09-23T20:51:09.823968+0000","last_clean":"2023-09-23T20:51:09.823968+0000","last_became_active":"2023-09-23T20:41:59.062717+0000","last_became_peered":"2023-09-23T20:41:59.062717+0000","last_unstale":"2023-09-23T20:51:09.823968+0000","last_undegraded":"2023-09-23T20:51:09.823968+0000","last_fullsized":"2023-09-23T20:51:09.823968+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:20:58.386461+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":22,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-09-23T20:51:10.836794+0000","last_change":"2023-09-23T20:51:10.836794+0000","last_active":"2023-09-23T20:51:10.836794+0000","last_peered":"2023-09-23T20:51:10.836794+0000","last_clean":"2023-09-23T20:51:10.836794+0000","last_became_active":"2023-09-23T20:51:10.836449+0000","last_became_peered":"2023-09-23T20:51:10.836449+0000","last_unstale":"2023-09-23T20:51:10.836794+0000","last_undegraded":"2023-09-23T20:51:10.836794+0000","last_fullsized":"2023-09-23T20:51:10.836794+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:00:35.572203+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.0","version":"79'161","reported_seq":440,"reported_epoch":98,"state":"active+clean","last_fresh":"2023-09-23T20:51:09.821731+0000","last_change":"2023-09-23T20:49:32.777246+0000","last_active":"2023-09-23T20:51:09.821731+0000","last_peered":"2023-09-23T20:51:09.821731+0000","last_clean":"2023-09-23T20:51:09.821731+0000","last_became_active":"2023-09-23T20:49:32.776822+0000","last_became_peered":"2023-09-23T20:49:32.776822+0000","last_unstale":"2023-09-23T20:51:09.821731+0000","last_undegraded":"2023-09-23T20:51:09.821731+0000","last_fullsized":"2023-09-23T20:51:09.821731+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:38:57.324224+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:05:46.881275+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":63,"reported_epoch":98,"state":"active+clean","last_fresh":"2023-09-23T20:51:09.817117+0000","last_change":"2023-09-23T20:50:37.984163+0000","last_active":"2023-09-23T20:51:09.817117+0000","last_peered":"2023-09-23T20:51:09.817117+0000","last_clean":"2023-09-23T20:51:09.817117+0000","last_became_active":"2023-09-23T20:50:37.983838+0000","last_became_peered":"2023-09-23T20:50:37.983838+0000","last_unstale":"2023-09-23T20:51:09.817117+0000","last_undegraded":"2023-09-23T20:51:09.817117+0000","last_fullsized":"2023-09-23T20:51:09.817117+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:02:41.792156+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":65,"reported_epoch":98,"state":"active+clean","last_fresh":"2023-09-23T20:51:09.835854+0000","last_change":"2023-09-23T20:50:05.867499+0000","last_active":"2023-09-23T20:51:09.835854+0000","last_peered":"2023-09-23T20:51:09.835854+0000","last_clean":"2023-09-23T20:51:09.835854+0000","last_became_active":"2023-09-23T20:50:05.866571+0000","last_became_peered":"2023-09-23T20:50:05.866571+0000","last_unstale":"2023-09-23T20:51:09.835854+0000","last_undegraded":"2023-09-23T20:51:09.835854+0000","last_fullsized":"2023-09-23T20:51:09.835854+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:18:10.735501+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":47,"reported_epoch":98,"state":"active+clean","last_fresh":"2023-09-23T20:51:09.836383+0000","last_change":"2023-09-23T20:50:37.983072+0000","last_active":"2023-09-23T20:51:09.836383+0000","last_peered":"2023-09-23T20:51:09.836383+0000","last_clean":"2023-09-23T20:51:09.836383+0000","last_became_active":"2023-09-23T20:50:37.982777+0000","last_became_peered":"2023-09-23T20:50:37.982777+0000","last_unstale":"2023-09-23T20:51:09.836383+0000","last_undegraded":"2023-09-23T20:51:09.836383+0000","last_fullsized":"2023-09-23T20:51:09.836383+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:58:11.972144+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":81,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-09-23T20:51:10.838054+0000","last_change":"2023-09-23T20:51:10.838054+0000","last_active":"2023-09-23T20:51:10.838054+0000","last_peered":"2023-09-23T20:51:10.838054+0000","last_clean":"2023-09-23T20:51:10.838054+0000","last_became_active":"2023-09-23T20:51:10.837750+0000","last_became_peered":"2023-09-23T20:51:10.837750+0000","last_unstale":"2023-09-23T20:51:10.838054+0000","last_undegraded":"2023-09-23T20:51:10.838054+0000","last_fullsized":"2023-09-23T20:51:10.838054+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:15:51.648085+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":62,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-09-23T20:51:10.829889+0000","last_change":"2023-09-23T20:51:10.829889+0000","last_active":"2023-09-23T20:51:10.829889+0000","last_peered":"2023-09-23T20:51:10.829889+0000","last_clean":"2023-09-23T20:51:10.829889+0000","last_became_active":"2023-09-23T20:51:10.829533+0000","last_became_peered":"2023-09-23T20:51:10.829533+0000","last_unstale":"2023-09-23T20:51:10.829889+0000","last_undegraded":"2023-09-23T20:51:10.829889+0000","last_fullsized":"2023-09-23T20:51:10.829889+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:43:04.603460+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":59,"reported_epoch":98,"state":"active+clean","last_fresh":"2023-09-23T20:51:09.826642+0000","last_change":"2023-09-23T20:49:32.771117+0000","last_active":"2023-09-23T20:51:09.826642+0000","last_peered":"2023-09-23T20:51:09.826642+0000","last_clean":"2023-09-23T20:51:09.826642+0000","last_became_active":"2023-09-23T20:49:32.770686+0000","last_became_peered":"2023-09-23T20:49:32.770686+0000","last_unstale":"2023-09-23T20:51:09.826642+0000","last_undegraded":"2023-09-23T20:51:09.826642+0000","last_fullsized":"2023-09-23T20:51:09.826642+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:15:46.598698+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":63,"reported_epoch":98,"state":"active+clean","last_fresh":"2023-09-23T20:51:09.817089+0000","last_change":"2023-09-23T20:50:05.864620+0000","last_active":"2023-09-23T20:51:09.817089+0000","last_peered":"2023-09-23T20:51:09.817089+0000","last_clean":"2023-09-23T20:51:09.817089+0000","last_became_active":"2023-09-23T20:50:05.864276+0000","last_became_peered":"2023-09-23T20:50:05.864276+0000","last_unstale":"2023-09-23T20:51:09.817089+0000","last_undegraded":"2023-09-23T20:51:09.817089+0000","last_fullsized":"2023-09-23T20:51:09.817089+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:20:19.862901+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":32,"reported_epoch":98,"state":"active+clean","last_fresh":"2023-09-23T20:51:09.834324+0000","last_change":"2023-09-23T20:50:05.868321+0000","last_active":"2023-09-23T20:51:09.834324+0000","last_peered":"2023-09-23T20:51:09.834324+0000","last_clean":"2023-09-23T20:51:09.834324+0000","last_became_active":"2023-09-23T20:50:05.868120+0000","last_became_peered":"2023-09-23T20:50:05.868120+0000","last_unstale":"2023-09-23T20:51:09.834324+0000","last_undegraded":"2023-09-23T20:51:09.834324+0000","last_fullsized":"2023-09-23T20:51:09.834324+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:23:14.856477+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":76,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-09-23T20:51:10.840273+0000","last_change":"2023-09-23T20:51:10.840273+0000","last_active":"2023-09-23T20:51:10.840273+0000","last_peered":"2023-09-23T20:51:10.840273+0000","last_clean":"2023-09-23T20:51:10.840273+0000","last_became_active":"2023-09-23T20:51:10.840017+0000","last_became_peered":"2023-09-23T20:51:10.840017+0000","last_unstale":"2023-09-23T20:51:10.840273+0000","last_undegraded":"2023-09-23T20:51:10.840273+0000","last_fullsized":"2023-09-23T20:51:10.840273+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:58:17.982729+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":70,"reported_epoch":98,"state":"active+clean","last_fresh":"2023-09-23T20:51:09.835307+0000","last_change":"2023-09-23T20:50:05.867130+0000","last_active":"2023-09-23T20:51:09.835307+0000","last_peered":"2023-09-23T20:51:09.835307+0000","last_clean":"2023-09-23T20:51:09.835307+0000","last_became_active":"2023-09-23T20:50:05.866758+0000","last_became_peered":"2023-09-23T20:50:05.866758+0000","last_unstale":"2023-09-23T20:51:09.835307+0000","last_undegraded":"2023-09-23T20:51:09.835307+0000","last_fullsized":"2023-09-23T20:51:09.835307+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:55:07.548209+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":47,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-09-23T20:51:10.833850+0000","last_change":"2023-09-23T20:51:10.833850+0000","last_active":"2023-09-23T20:51:10.833850+0000","last_peered":"2023-09-23T20:51:10.833850+0000","last_clean":"2023-09-23T20:51:10.833850+0000","last_became_active":"2023-09-23T20:51:10.833539+0000","last_became_peered":"2023-09-23T20:51:10.833539+0000","last_unstale":"2023-09-23T20:51:10.833850+0000","last_undegraded":"2023-09-23T20:51:10.833850+0000","last_fullsized":"2023-09-23T20:51:10.833850+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:01:36.445616+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":62,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-09-23T20:51:10.831197+0000","last_change":"2023-09-23T20:51:10.831197+0000","last_active":"2023-09-23T20:51:10.831197+0000","last_peered":"2023-09-23T20:51:10.831197+0000","last_clean":"2023-09-23T20:51:10.831197+0000","last_became_active":"2023-09-23T20:51:10.830878+0000","last_became_peered":"2023-09-23T20:51:10.830878+0000","last_unstale":"2023-09-23T20:51:10.831197+0000","last_undegraded":"2023-09-23T20:51:10.831197+0000","last_fullsized":"2023-09-23T20:51:10.831197+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:05:24.314478+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":58,"reported_epoch":98,"state":"active+clean","last_fresh":"2023-09-23T20:51:09.826968+0000","last_change":"2023-09-23T20:50:05.871406+0000","last_active":"2023-09-23T20:51:09.826968+0000","last_peered":"2023-09-23T20:51:09.826968+0000","last_clean":"2023-09-23T20:51:09.826968+0000","last_became_active":"2023-09-23T20:50:05.871127+0000","last_became_peered":"2023-09-23T20:50:05.871127+0000","last_unstale":"2023-09-23T20:51:09.826968+0000","last_undegraded":"2023-09-23T20:51:09.826968+0000","last_fullsized":"2023-09-23T20:51:09.826968+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:37:48.850229+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":59,"reported_epoch":98,"state":"active+clean","last_fresh":"2023-09-23T20:51:09.826268+0000","last_change":"2023-09-23T20:50:37.983914+0000","last_active":"2023-09-23T20:51:09.826268+0000","last_peered":"2023-09-23T20:51:09.826268+0000","last_clean":"2023-09-23T20:51:09.826268+0000","last_became_active":"2023-09-23T20:50:37.983617+0000","last_became_peered":"2023-09-23T20:50:37.983617+0000","last_unstale":"2023-09-23T20:51:09.826268+0000","last_undegraded":"2023-09-23T20:51:09.826268+0000","last_fullsized":"2023-09-23T20:51:09.826268+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:15:50.863306+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":75,"reported_epoch":98,"state":"active+clean","last_fresh":"2023-09-23T20:51:09.826833+0000","last_change":"2023-09-23T20:50:37.985593+0000","last_active":"2023-09-23T20:51:09.826833+0000","last_peered":"2023-09-23T20:51:09.826833+0000","last_clean":"2023-09-23T20:51:09.826833+0000","last_became_active":"2023-09-23T20:50:37.985324+0000","last_became_peered":"2023-09-23T20:50:37.985324+0000","last_unstale":"2023-09-23T20:51:09.826833+0000","last_undegraded":"2023-09-23T20:51:09.826833+0000","last_fullsized":"2023-09-23T20:51:09.826833+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:34:07.801771+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":59,"reported_epoch":98,"state":"active+clean","last_fresh":"2023-09-23T20:51:09.824658+0000","last_change":"2023-09-23T20:50:05.867646+0000","last_active":"2023-09-23T20:51:09.824658+0000","last_peered":"2023-09-23T20:51:09.824658+0000","last_clean":"2023-09-23T20:51:09.824658+0000","last_became_active":"2023-09-23T20:50:05.867343+0000","last_became_peered":"2023-09-23T20:50:05.867343+0000","last_unstale":"2023-09-23T20:51:09.824658+0000","last_undegraded":"2023-09-23T20:51:09.824658+0000","last_fullsized":"2023-09-23T20:51:09.824658+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:53:34.985580+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":58,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-09-23T20:51:10.833630+0000","last_change":"2023-09-23T20:51:10.833630+0000","last_active":"2023-09-23T20:51:10.833630+0000","last_peered":"2023-09-23T20:51:10.833630+0000","last_clean":"2023-09-23T20:51:10.833630+0000","last_became_active":"2023-09-23T20:51:10.833265+0000","last_became_peered":"2023-09-23T20:51:10.833265+0000","last_unstale":"2023-09-23T20:51:10.833630+0000","last_undegraded":"2023-09-23T20:51:10.833630+0000","last_fullsized":"2023-09-23T20:51:10.833630+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:12:01.557967+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":59,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-09-23T20:51:10.836752+0000","last_change":"2023-09-23T20:51:10.836752+0000","last_active":"2023-09-23T20:51:10.836752+0000","last_peered":"2023-09-23T20:51:10.836752+0000","last_clean":"2023-09-23T20:51:10.836752+0000","last_became_active":"2023-09-23T20:51:10.836213+0000","last_became_peered":"2023-09-23T20:51:10.836213+0000","last_unstale":"2023-09-23T20:51:10.836752+0000","last_undegraded":"2023-09-23T20:51:10.836752+0000","last_fullsized":"2023-09-23T20:51:10.836752+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:35:01.825635+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":66,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-09-23T20:51:10.827903+0000","last_change":"2023-09-23T20:50:37.982263+0000","last_active":"2023-09-23T20:51:10.827903+0000","last_peered":"2023-09-23T20:51:10.827903+0000","last_clean":"2023-09-23T20:51:10.827903+0000","last_became_active":"2023-09-23T20:50:37.981797+0000","last_became_peered":"2023-09-23T20:50:37.981797+0000","last_unstale":"2023-09-23T20:51:10.827903+0000","last_undegraded":"2023-09-23T20:51:10.827903+0000","last_fullsized":"2023-09-23T20:51:10.827903+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:56:45.027447+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":59,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-09-23T20:51:10.839104+0000","last_change":"2023-09-23T20:51:10.839104+0000","last_active":"2023-09-23T20:51:10.839104+0000","last_peered":"2023-09-23T20:51:10.839104+0000","last_clean":"2023-09-23T20:51:10.839104+0000","last_became_active":"2023-09-23T20:51:10.838783+0000","last_became_peered":"2023-09-23T20:51:10.838783+0000","last_unstale":"2023-09-23T20:51:10.839104+0000","last_undegraded":"2023-09-23T20:51:10.839104+0000","last_fullsized":"2023-09-23T20:51:10.839104+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:24:32.550241+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":58,"reported_epoch":98,"state":"active+clean","last_fresh":"2023-09-23T20:51:09.824495+0000","last_change":"2023-09-23T20:50:05.867973+0000","last_active":"2023-09-23T20:51:09.824495+0000","last_peered":"2023-09-23T20:51:09.824495+0000","last_clean":"2023-09-23T20:51:09.824495+0000","last_became_active":"2023-09-23T20:50:05.867622+0000","last_became_peered":"2023-09-23T20:50:05.867622+0000","last_unstale":"2023-09-23T20:51:09.824495+0000","last_undegraded":"2023-09-23T20:51:09.824495+0000","last_fullsized":"2023-09-23T20:51:09.824495+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:10:12.617597+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":43,"reported_epoch":98,"state":"active+clean","last_fresh":"2023-09-23T20:51:09.823751+0000","last_change":"2023-09-23T20:41:58.898346+0000","last_active":"2023-09-23T20:51:09.823751+0000","last_peered":"2023-09-23T20:51:09.823751+0000","last_clean":"2023-09-23T20:51:09.823751+0000","last_became_active":"2023-09-23T20:41:58.898065+0000","last_became_peered":"2023-09-23T20:41:58.898065+0000","last_unstale":"2023-09-23T20:51:09.823751+0000","last_undegraded":"2023-09-23T20:51:09.823751+0000","last_fullsized":"2023-09-23T20:51:09.823751+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:51:57.491171+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":75,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-09-23T20:51:10.838338+0000","last_change":"2023-09-23T20:51:10.838338+0000","last_active":"2023-09-23T20:51:10.838338+0000","last_peered":"2023-09-23T20:51:10.838338+0000","last_clean":"2023-09-23T20:51:10.838338+0000","last_became_active":"2023-09-23T20:51:10.838049+0000","last_became_peered":"2023-09-23T20:51:10.838049+0000","last_unstale":"2023-09-23T20:51:10.838338+0000","last_undegraded":"2023-09-23T20:51:10.838338+0000","last_fullsized":"2023-09-23T20:51:10.838338+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:40:23.602402+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":61,"reported_epoch":98,"state":"active+clean","last_fresh":"2023-09-23T20:51:09.823256+0000","last_change":"2023-09-23T20:50:05.866983+0000","last_active":"2023-09-23T20:51:09.823256+0000","last_peered":"2023-09-23T20:51:09.823256+0000","last_clean":"2023-09-23T20:51:09.823256+0000","last_became_active":"2023-09-23T20:50:05.866601+0000","last_became_peered":"2023-09-23T20:50:05.866601+0000","last_unstale":"2023-09-23T20:51:09.823256+0000","last_undegraded":"2023-09-23T20:51:09.823256+0000","last_fullsized":"2023-09-23T20:51:09.823256+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:15:00.955166+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":41,"reported_epoch":98,"state":"active+clean","last_fresh":"2023-09-23T20:51:09.816955+0000","last_change":"2023-09-23T20:41:59.061405+0000","last_active":"2023-09-23T20:51:09.816955+0000","last_peered":"2023-09-23T20:51:09.816955+0000","last_clean":"2023-09-23T20:51:09.816955+0000","last_became_active":"2023-09-23T20:41:59.061032+0000","last_became_peered":"2023-09-23T20:41:59.061032+0000","last_unstale":"2023-09-23T20:51:09.816955+0000","last_undegraded":"2023-09-23T20:51:09.816955+0000","last_fullsized":"2023-09-23T20:51:09.816955+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:43:09.716350+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":60,"reported_epoch":98,"state":"active+clean","last_fresh":"2023-09-23T20:51:09.822945+0000","last_change":"2023-09-23T20:49:32.777943+0000","last_active":"2023-09-23T20:51:09.822945+0000","last_peered":"2023-09-23T20:51:09.822945+0000","last_clean":"2023-09-23T20:51:09.822945+0000","last_became_active":"2023-09-23T20:49:32.776212+0000","last_became_peered":"2023-09-23T20:49:32.776212+0000","last_unstale":"2023-09-23T20:51:09.822945+0000","last_undegraded":"2023-09-23T20:51:09.822945+0000","last_fullsized":"2023-09-23T20:51:09.822945+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:21:05.725926+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":61,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-09-23T20:51:10.833234+0000","last_change":"2023-09-23T20:51:10.833234+0000","last_active":"2023-09-23T20:51:10.833234+0000","last_peered":"2023-09-23T20:51:10.833234+0000","last_clean":"2023-09-23T20:51:10.833234+0000","last_became_active":"2023-09-23T20:51:10.832763+0000","last_became_peered":"2023-09-23T20:51:10.832763+0000","last_unstale":"2023-09-23T20:51:10.833234+0000","last_undegraded":"2023-09-23T20:51:10.833234+0000","last_fullsized":"2023-09-23T20:51:10.833234+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:24:08.283578+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":67,"reported_epoch":98,"state":"active+clean","last_fresh":"2023-09-23T20:51:09.824402+0000","last_change":"2023-09-23T20:49:32.777437+0000","last_active":"2023-09-23T20:51:09.824402+0000","last_peered":"2023-09-23T20:51:09.824402+0000","last_clean":"2023-09-23T20:51:09.824402+0000","last_became_active":"2023-09-23T20:49:32.776999+0000","last_became_peered":"2023-09-23T20:49:32.776999+0000","last_unstale":"2023-09-23T20:51:09.824402+0000","last_undegraded":"2023-09-23T20:51:09.824402+0000","last_fullsized":"2023-09-23T20:51:09.824402+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:42:21.816073+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":52,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-09-23T20:51:10.840849+0000","last_change":"2023-09-23T20:51:10.840849+0000","last_active":"2023-09-23T20:51:10.840849+0000","last_peered":"2023-09-23T20:51:10.840849+0000","last_clean":"2023-09-23T20:51:10.840849+0000","last_became_active":"2023-09-23T20:51:10.840593+0000","last_became_peered":"2023-09-23T20:51:10.840593+0000","last_unstale":"2023-09-23T20:51:10.840849+0000","last_undegraded":"2023-09-23T20:51:10.840849+0000","last_fullsized":"2023-09-23T20:51:10.840849+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:30:52.297625+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":42,"reported_epoch":98,"state":"active+clean","last_fresh":"2023-09-23T20:51:09.821305+0000","last_change":"2023-09-23T20:41:58.898224+0000","last_active":"2023-09-23T20:51:09.821305+0000","last_peered":"2023-09-23T20:51:09.821305+0000","last_clean":"2023-09-23T20:51:09.821305+0000","last_became_active":"2023-09-23T20:41:58.898036+0000","last_became_peered":"2023-09-23T20:41:58.898036+0000","last_unstale":"2023-09-23T20:51:09.821305+0000","last_undegraded":"2023-09-23T20:51:09.821305+0000","last_fullsized":"2023-09-23T20:51:09.821305+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:53:15.361560+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":58,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-09-23T20:51:10.839174+0000","last_change":"2023-09-23T20:51:10.839174+0000","last_active":"2023-09-23T20:51:10.839174+0000","last_peered":"2023-09-23T20:51:10.839174+0000","last_clean":"2023-09-23T20:51:10.839174+0000","last_became_active":"2023-09-23T20:51:10.838826+0000","last_became_peered":"2023-09-23T20:51:10.838826+0000","last_unstale":"2023-09-23T20:51:10.839174+0000","last_undegraded":"2023-09-23T20:51:10.839174+0000","last_fullsized":"2023-09-23T20:51:10.839174+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:43:00.944119+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":53,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-09-23T20:51:10.841667+0000","last_change":"2023-09-23T20:51:10.840590+0000","last_active":"2023-09-23T20:51:10.841667+0000","last_peered":"2023-09-23T20:51:10.841667+0000","last_clean":"2023-09-23T20:51:10.841667+0000","last_became_active":"2023-09-23T20:51:10.840221+0000","last_became_peered":"2023-09-23T20:51:10.840221+0000","last_unstale":"2023-09-23T20:51:10.841667+0000","last_undegraded":"2023-09-23T20:51:10.841667+0000","last_fullsized":"2023-09-23T20:51:10.841667+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:03:39.138638+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":61,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-09-23T20:51:10.831262+0000","last_change":"2023-09-23T20:51:10.831262+0000","last_active":"2023-09-23T20:51:10.831262+0000","last_peered":"2023-09-23T20:51:10.831262+0000","last_clean":"2023-09-23T20:51:10.831262+0000","last_became_active":"2023-09-23T20:51:10.830931+0000","last_became_peered":"2023-09-23T20:51:10.830931+0000","last_unstale":"2023-09-23T20:51:10.831262+0000","last_undegraded":"2023-09-23T20:51:10.831262+0000","last_fullsized":"2023-09-23T20:51:10.831262+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:51:54.982505+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":61,"reported_epoch":98,"state":"active+clean","last_fresh":"2023-09-23T20:51:09.821700+0000","last_change":"2023-09-23T20:50:37.986729+0000","last_active":"2023-09-23T20:51:09.821700+0000","last_peered":"2023-09-23T20:51:09.821700+0000","last_clean":"2023-09-23T20:51:09.821700+0000","last_became_active":"2023-09-23T20:50:37.986433+0000","last_became_peered":"2023-09-23T20:50:37.986433+0000","last_unstale":"2023-09-23T20:51:09.821700+0000","last_undegraded":"2023-09-23T20:51:09.821700+0000","last_fullsized":"2023-09-23T20:51:09.821700+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:28:28.853091+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":58,"reported_epoch":98,"state":"active+clean","last_fresh":"2023-09-23T20:51:09.826698+0000","last_change":"2023-09-23T20:50:05.871199+0000","last_active":"2023-09-23T20:51:09.826698+0000","last_peered":"2023-09-23T20:51:09.826698+0000","last_clean":"2023-09-23T20:51:09.826698+0000","last_became_active":"2023-09-23T20:50:05.870912+0000","last_became_peered":"2023-09-23T20:50:05.870912+0000","last_unstale":"2023-09-23T20:51:09.826698+0000","last_undegraded":"2023-09-23T20:51:09.826698+0000","last_fullsized":"2023-09-23T20:51:09.826698+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:16:22.718106+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"79'22","reported_seq":64,"reported_epoch":98,"state":"active+clean","last_fresh":"2023-09-23T20:51:09.827615+0000","last_change":"2023-09-23T20:41:58.905866+0000","last_active":"2023-09-23T20:51:09.827615+0000","last_peered":"2023-09-23T20:51:09.827615+0000","last_clean":"2023-09-23T20:51:09.827615+0000","last_became_active":"2023-09-23T20:41:58.905556+0000","last_became_peered":"2023-09-23T20:41:58.905556+0000","last_unstale":"2023-09-23T20:51:09.827615+0000","last_undegraded":"2023-09-23T20:51:09.827615+0000","last_fullsized":"2023-09-23T20:51:09.827615+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:03:14.027348+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":47,"reported_epoch":98,"state":"active+clean","last_fresh":"2023-09-23T20:51:09.834463+0000","last_change":"2023-09-23T20:50:37.982103+0000","last_active":"2023-09-23T20:51:09.834463+0000","last_peered":"2023-09-23T20:51:09.834463+0000","last_clean":"2023-09-23T20:51:09.834463+0000","last_became_active":"2023-09-23T20:50:37.981791+0000","last_became_peered":"2023-09-23T20:50:37.981791+0000","last_unstale":"2023-09-23T20:51:09.834463+0000","last_undegraded":"2023-09-23T20:51:09.834463+0000","last_fullsized":"2023-09-23T20:51:09.834463+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:32:52.286186+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":62,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-09-23T20:51:10.834618+0000","last_change":"2023-09-23T20:51:10.834618+0000","last_active":"2023-09-23T20:51:10.834618+0000","last_peered":"2023-09-23T20:51:10.834618+0000","last_clean":"2023-09-23T20:51:10.834618+0000","last_became_active":"2023-09-23T20:51:10.834267+0000","last_became_peered":"2023-09-23T20:51:10.834267+0000","last_unstale":"2023-09-23T20:51:10.834618+0000","last_undegraded":"2023-09-23T20:51:10.834618+0000","last_fullsized":"2023-09-23T20:51:10.834618+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:38:59.356003+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":22,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-09-23T20:51:10.833790+0000","last_change":"2023-09-23T20:51:10.833790+0000","last_active":"2023-09-23T20:51:10.833790+0000","last_peered":"2023-09-23T20:51:10.833790+0000","last_clean":"2023-09-23T20:51:10.833790+0000","last_became_active":"2023-09-23T20:51:10.832804+0000","last_became_peered":"2023-09-23T20:51:10.832804+0000","last_unstale":"2023-09-23T20:51:10.833790+0000","last_undegraded":"2023-09-23T20:51:10.833790+0000","last_fullsized":"2023-09-23T20:51:10.833790+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:27:42.601674+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":59,"reported_epoch":98,"state":"active+clean","last_fresh":"2023-09-23T20:51:09.817393+0000","last_change":"2023-09-23T20:50:05.863737+0000","last_active":"2023-09-23T20:51:09.817393+0000","last_peered":"2023-09-23T20:51:09.817393+0000","last_clean":"2023-09-23T20:51:09.817393+0000","last_became_active":"2023-09-23T20:50:05.863466+0000","last_became_peered":"2023-09-23T20:50:05.863466+0000","last_unstale":"2023-09-23T20:51:09.817393+0000","last_undegraded":"2023-09-23T20:51:09.817393+0000","last_fullsized":"2023-09-23T20:51:09.817393+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:28:57.602682+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":61,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-09-23T20:51:10.833424+0000","last_change":"2023-09-23T20:51:10.833424+0000","last_active":"2023-09-23T20:51:10.833424+0000","last_peered":"2023-09-23T20:51:10.833424+0000","last_clean":"2023-09-23T20:51:10.833424+0000","last_became_active":"2023-09-23T20:51:10.832808+0000","last_became_peered":"2023-09-23T20:51:10.832808+0000","last_unstale":"2023-09-23T20:51:10.833424+0000","last_undegraded":"2023-09-23T20:51:10.833424+0000","last_fullsized":"2023-09-23T20:51:10.833424+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:32:42.819583+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":52,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-09-23T20:51:10.840587+0000","last_change":"2023-09-23T20:51:10.840587+0000","last_active":"2023-09-23T20:51:10.840587+0000","last_peered":"2023-09-23T20:51:10.840587+0000","last_clean":"2023-09-23T20:51:10.840587+0000","last_became_active":"2023-09-23T20:51:10.840242+0000","last_became_peered":"2023-09-23T20:51:10.840242+0000","last_unstale":"2023-09-23T20:51:10.840587+0000","last_undegraded":"2023-09-23T20:51:10.840587+0000","last_fullsized":"2023-09-23T20:51:10.840587+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:44:07.171314+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":58,"reported_epoch":98,"state":"active+clean","last_fresh":"2023-09-23T20:51:09.824327+0000","last_change":"2023-09-23T20:50:05.867622+0000","last_active":"2023-09-23T20:51:09.824327+0000","last_peered":"2023-09-23T20:51:09.824327+0000","last_clean":"2023-09-23T20:51:09.824327+0000","last_became_active":"2023-09-23T20:50:05.867311+0000","last_became_peered":"2023-09-23T20:50:05.867311+0000","last_unstale":"2023-09-23T20:51:09.824327+0000","last_undegraded":"2023-09-23T20:51:09.824327+0000","last_fullsized":"2023-09-23T20:51:09.824327+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:31:53.285316+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":79,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-09-23T20:51:10.833613+0000","last_change":"2023-09-23T20:51:10.833613+0000","last_active":"2023-09-23T20:51:10.833613+0000","last_peered":"2023-09-23T20:51:10.833613+0000","last_clean":"2023-09-23T20:51:10.833613+0000","last_became_active":"2023-09-23T20:51:10.833208+0000","last_became_peered":"2023-09-23T20:51:10.833208+0000","last_unstale":"2023-09-23T20:51:10.833613+0000","last_undegraded":"2023-09-23T20:51:10.833613+0000","last_fullsized":"2023-09-23T20:51:10.833613+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:51:28.246014+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":53,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-09-23T20:51:10.841782+0000","last_change":"2023-09-23T20:51:10.841420+0000","last_active":"2023-09-23T20:51:10.841782+0000","last_peered":"2023-09-23T20:51:10.841782+0000","last_clean":"2023-09-23T20:51:10.841782+0000","last_became_active":"2023-09-23T20:51:10.841112+0000","last_became_peered":"2023-09-23T20:51:10.841112+0000","last_unstale":"2023-09-23T20:51:10.841782+0000","last_undegraded":"2023-09-23T20:51:10.841782+0000","last_fullsized":"2023-09-23T20:51:10.841782+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:44:02.236650+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":47,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-09-23T20:51:10.833035+0000","last_change":"2023-09-23T20:51:10.833035+0000","last_active":"2023-09-23T20:51:10.833035+0000","last_peered":"2023-09-23T20:51:10.833035+0000","last_clean":"2023-09-23T20:51:10.833035+0000","last_became_active":"2023-09-23T20:51:10.832741+0000","last_became_peered":"2023-09-23T20:51:10.832741+0000","last_unstale":"2023-09-23T20:51:10.833035+0000","last_undegraded":"2023-09-23T20:51:10.833035+0000","last_fullsized":"2023-09-23T20:51:10.833035+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:53:16.471852+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":61,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-09-23T20:51:10.835170+0000","last_change":"2023-09-23T20:51:10.835170+0000","last_active":"2023-09-23T20:51:10.835170+0000","last_peered":"2023-09-23T20:51:10.835170+0000","last_clean":"2023-09-23T20:51:10.835170+0000","last_became_active":"2023-09-23T20:51:10.834808+0000","last_became_peered":"2023-09-23T20:51:10.834808+0000","last_unstale":"2023-09-23T20:51:10.835170+0000","last_undegraded":"2023-09-23T20:51:10.835170+0000","last_fullsized":"2023-09-23T20:51:10.835170+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:51:25.153742+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"68'1","reported_seq":77,"reported_epoch":98,"state":"active+clean","last_fresh":"2023-09-23T20:51:09.826718+0000","last_change":"2023-09-23T20:50:37.985580+0000","last_active":"2023-09-23T20:51:09.826718+0000","last_peered":"2023-09-23T20:51:09.826718+0000","last_clean":"2023-09-23T20:51:09.826718+0000","last_became_active":"2023-09-23T20:50:37.985229+0000","last_became_peered":"2023-09-23T20:50:37.985229+0000","last_unstale":"2023-09-23T20:51:09.826718+0000","last_undegraded":"2023-09-23T20:51:09.826718+0000","last_fullsized":"2023-09-23T20:51:09.826718+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:52:50.591507+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":161,"ondisk_log_size":161,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201417,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":22696,"kb_used_data":1568,"kb_used_omap":0,"kb_used_meta":21120,"kb_avail":93722456,"statfs":{"total":95995035648,"available":95971794944,"internally_reserved":0,"allocated":1605632,"data_stored":1128632,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":21626880},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:51:35 2023","interfaces":[{"interface":"back","average":{"1min":0.622,"5min":0.51700000000000002,"15min":0.499},"min":{"1min":0.45900000000000002,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":0.80400000000000005,"5min":0.80400000000000005,"15min":0.80400000000000005},"last":0.69399999999999995},{"interface":"front","average":{"1min":0.55200000000000005,"5min":0.56000000000000005,"15min":0.56200000000000006},"min":{"1min":0.33700000000000002,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":0.79000000000000004,"5min":0.79000000000000004,"15min":0.79000000000000004},"last":0.60499999999999998}]},{"osd":1,"last update":"Sat Sep 23 20:51:05 2023","interfaces":[{"interface":"back","average":{"1min":0.54900000000000004,"5min":0.54900000000000004,"15min":0.54900000000000004},"min":{"1min":0.40999999999999998,"5min":0.40999999999999998,"15min":0.40999999999999998},"max":{"1min":0.73099999999999998,"5min":0.73099999999999998,"15min":0.73099999999999998},"last":0.51400000000000001},{"interface":"front","average":{"1min":0.54400000000000004,"5min":0.54400000000000004,"15min":0.54400000000000004},"min":{"1min":0.32100000000000001,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":0.71599999999999997,"5min":0.71599999999999997,"15min":0.71599999999999997},"last":0.61699999999999999}]},{"osd":2,"last update":"Sat Sep 23 20:51:41 2023","interfaces":[{"interface":"back","average":{"1min":0.55100000000000005,"5min":0.55100000000000005,"15min":0.55100000000000005},"min":{"1min":0.33300000000000002,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":0.83099999999999996,"5min":0.83099999999999996,"15min":0.83099999999999996},"last":0.59099999999999997},{"interface":"front","average":{"1min":0.58399999999999996,"5min":0.58399999999999996,"15min":0.58399999999999996},"min":{"1min":0.375,"5min":0.375,"15min":0.375},"max":{"1min":0.81299999999999994,"5min":0.81299999999999994,"15min":0.81299999999999994},"last":0.755}]},{"osd":3,"last update":"Sat Sep 23 20:50:58 2023","interfaces":[{"interface":"back","average":{"1min":0.58199999999999996,"5min":0.58599999999999997,"15min":0.56499999999999995},"min":{"1min":0.40000000000000002,"5min":0.309,"15min":0.28000000000000003},"max":{"1min":0.755,"5min":0.83299999999999996,"15min":1.159},"last":0.66100000000000003},{"interface":"front","average":{"1min":0.60699999999999998,"5min":0.61099999999999999,"15min":0.58799999999999997},"min":{"1min":0.435,"5min":0.376,"15min":0.32900000000000001},"max":{"1min":0.77800000000000002,"5min":0.84099999999999997,"15min":1.5840000000000001},"last":0.67900000000000005}]},{"osd":4,"last update":"Sat Sep 23 20:50:58 2023","interfaces":[{"interface":"back","average":{"1min":0.55000000000000004,"5min":0.51100000000000001,"15min":0.48099999999999998},"min":{"1min":0.34999999999999998,"5min":0.312,"15min":0.254},"max":{"1min":0.77000000000000002,"5min":0.82099999999999995,"15min":0.82099999999999995},"last":0.55800000000000005},{"interface":"front","average":{"1min":0.56999999999999995,"5min":0.54100000000000004,"15min":0.52000000000000002},"min":{"1min":0.39200000000000002,"5min":0.35999999999999999,"15min":0.29599999999999999},"max":{"1min":0.80000000000000004,"5min":0.873,"15min":1.1100000000000001},"last":0.64000000000000001}]},{"osd":5,"last update":"Sat Sep 23 20:50:58 2023","interfaces":[{"interface":"back","average":{"1min":0.59199999999999997,"5min":0.55400000000000005,"15min":0.53100000000000003},"min":{"1min":0.42399999999999999,"5min":0.318,"15min":0.30199999999999999},"max":{"1min":0.78500000000000003,"5min":0.89900000000000002,"15min":1.4610000000000001},"last":0.70999999999999996},{"interface":"front","average":{"1min":0.63400000000000001,"5min":0.58199999999999996,"15min":0.55100000000000005},"min":{"1min":0.39500000000000002,"5min":0.34300000000000003,"15min":0.30599999999999999},"max":{"1min":0.84999999999999998,"5min":0.85899999999999999,"15min":0.85899999999999999},"last":0.65300000000000002}]},{"osd":6,"last update":"Sat Sep 23 20:50:58 2023","interfaces":[{"interface":"back","average":{"1min":0.623,"5min":0.60599999999999998,"15min":0.56899999999999995},"min":{"1min":0.433,"5min":0.41599999999999998,"15min":0.32600000000000001},"max":{"1min":0.88200000000000001,"5min":0.88300000000000001,"15min":0.88900000000000001},"last":0.72499999999999998},{"interface":"front","average":{"1min":0.64800000000000002,"5min":0.60599999999999998,"15min":0.58499999999999996},"min":{"1min":0.47599999999999998,"5min":0.36299999999999999,"15min":0.316},"max":{"1min":0.83999999999999997,"5min":0.90700000000000003,"15min":1.0760000000000001},"last":0.73199999999999998}]}]},{"osd":6,"up_from":47,"seq":201863463054,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":7832,"kb_used_data":1552,"kb_used_omap":0,"kb_used_meta":6272,"kb_avail":93737320,"statfs":{"total":95995035648,"available":95987015680,"internally_reserved":0,"allocated":1589248,"data_stored":1128221,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":6422528},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:51:35 2023","interfaces":[{"interface":"back","average":{"1min":0.50900000000000001,"5min":0.56499999999999995,"15min":0.57399999999999995},"min":{"1min":0.33400000000000002,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":0.75700000000000001,"5min":0.79200000000000004,"15min":0.79200000000000004},"last":0.49399999999999999},{"interface":"front","average":{"1min":0.54600000000000004,"5min":0.58099999999999996,"15min":0.58699999999999997},"min":{"1min":0.34200000000000003,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":0.68799999999999994,"5min":0.89100000000000001,"15min":0.89100000000000001},"last":0.51800000000000002}]},{"osd":1,"last update":"Sat Sep 23 20:51:06 2023","interfaces":[{"interface":"back","average":{"1min":0.53400000000000003,"5min":0.53400000000000003,"15min":0.53400000000000003},"min":{"1min":0.371,"5min":0.371,"15min":0.371},"max":{"1min":0.72499999999999998,"5min":0.72499999999999998,"15min":0.72499999999999998},"last":0.39900000000000002},{"interface":"front","average":{"1min":0.56999999999999995,"5min":0.56999999999999995,"15min":0.56999999999999995},"min":{"1min":0.38300000000000001,"5min":0.38300000000000001,"15min":0.38300000000000001},"max":{"1min":0.79700000000000004,"5min":0.79700000000000004,"15min":0.79700000000000004},"last":0.62}]},{"osd":2,"last update":"Sat Sep 23 20:51:41 2023","interfaces":[{"interface":"back","average":{"1min":0.54800000000000004,"5min":0.54800000000000004,"15min":0.54800000000000004},"min":{"1min":0.40899999999999997,"5min":0.40899999999999997,"15min":0.40899999999999997},"max":{"1min":0.71999999999999997,"5min":0.71999999999999997,"15min":0.71999999999999997},"last":0.54000000000000004},{"interface":"front","average":{"1min":0.58599999999999997,"5min":0.58599999999999997,"15min":0.58599999999999997},"min":{"1min":0.35799999999999998,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":0.73399999999999999,"5min":0.73399999999999999,"15min":0.73399999999999999},"last":0.56499999999999995}]},{"osd":3,"last update":"Sat Sep 23 20:50:38 2023","interfaces":[{"interface":"back","average":{"1min":0.61599999999999999,"5min":0.60199999999999998,"15min":0.56200000000000006},"min":{"1min":0.439,"5min":0.34699999999999998,"15min":0.33200000000000002},"max":{"1min":0.82699999999999996,"5min":0.92100000000000004,"15min":0.96099999999999997},"last":0.55300000000000005},{"interface":"front","average":{"1min":0.60699999999999998,"5min":0.59599999999999997,"15min":0.56100000000000005},"min":{"1min":0.42899999999999999,"5min":0.39900000000000002,"15min":0.34100000000000003},"max":{"1min":0.81999999999999995,"5min":0.83999999999999997,"15min":0.874},"last":0.42099999999999999}]},{"osd":4,"last update":"Sat Sep 23 20:51:41 2023","interfaces":[{"interface":"back","average":{"1min":0.503,"5min":0.47599999999999998,"15min":0.45700000000000002},"min":{"1min":0.374,"5min":0.28000000000000003,"15min":0.27800000000000002},"max":{"1min":0.68400000000000005,"5min":0.81000000000000005,"15min":1.1779999999999999},"last":0.44500000000000001},{"interface":"front","average":{"1min":0.54500000000000004,"5min":0.52300000000000002,"15min":0.49399999999999999},"min":{"1min":0.376,"5min":0.30599999999999999,"15min":0.29199999999999998},"max":{"1min":0.77000000000000002,"5min":0.81599999999999995,"15min":0.81599999999999995},"last":0.63400000000000001}]},{"osd":5,"last update":"Sat Sep 23 20:51:41 2023","interfaces":[{"interface":"back","average":{"1min":0.53600000000000003,"5min":0.51600000000000001,"15min":0.49199999999999999},"min":{"1min":0.39600000000000002,"5min":0.32600000000000001,"15min":0.28399999999999997},"max":{"1min":0.69999999999999996,"5min":0.82699999999999996,"15min":1.081},"last":0.60499999999999998},{"interface":"front","average":{"1min":0.59299999999999997,"5min":0.56000000000000005,"15min":0.52300000000000002},"min":{"1min":0.374,"5min":0.33100000000000002,"15min":0.28499999999999998},"max":{"1min":0.755,"5min":0.81000000000000005,"15min":0.89000000000000001},"last":0.58799999999999997}]},{"osd":7,"last update":"Sat Sep 23 20:51:02 2023","interfaces":[{"interface":"back","average":{"1min":0.58299999999999996,"5min":0.56899999999999995,"15min":0.53500000000000003},"min":{"1min":0.38600000000000001,"5min":0.33000000000000002,"15min":0.32500000000000001},"max":{"1min":0.78900000000000003,"5min":0.82299999999999995,"15min":1.119},"last":0.57899999999999996},{"interface":"front","average":{"1min":0.63600000000000001,"5min":0.61599999999999999,"15min":0.57899999999999996},"min":{"1min":0.42199999999999999,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":0.79200000000000004,"5min":0.94099999999999995,"15min":0.94099999999999995},"last":0.66500000000000004}]}]},{"osd":1,"up_from":88,"seq":377957122071,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28480,"kb_used_data":1016,"kb_used_omap":9,"kb_used_meta":27446,"kb_avail":93716672,"statfs":{"total":95995035648,"available":95965872128,"internally_reserved":0,"allocated":1040384,"data_stored":544044,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9239,"internal_metadata":28105705},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:51:04 2023","interfaces":[{"interface":"back","average":{"1min":0.40899999999999997,"5min":0.40899999999999997,"15min":0.40899999999999997},"min":{"1min":0.27300000000000002,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":0.63200000000000001,"5min":0.63200000000000001,"15min":0.63200000000000001},"last":0.46800000000000003},{"interface":"front","average":{"1min":0.435,"5min":0.435,"15min":0.435},"min":{"1min":0.31,"5min":0.31,"15min":0.31},"max":{"1min":0.73999999999999999,"5min":0.73999999999999999,"15min":0.73999999999999999},"last":0.36599999999999999}]},{"osd":2,"last update":"Sat Sep 23 20:51:37 2023","interfaces":[{"interface":"back","average":{"1min":0.38700000000000001,"5min":0.38700000000000001,"15min":0.38700000000000001},"min":{"1min":0.26400000000000001,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":0.503,"5min":0.503,"15min":0.503},"last":0.505},{"interface":"front","average":{"1min":0.58499999999999996,"5min":0.58499999999999996,"15min":0.58499999999999996},"min":{"1min":0.29099999999999998,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":3.5219999999999998,"5min":3.5219999999999998,"15min":3.5219999999999998},"last":0.41599999999999998}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.443}]},{"osd":4,"last update":"Sat Sep 23 20:51:04 2023","interfaces":[{"interface":"back","average":{"1min":0.64700000000000002,"5min":0.64700000000000002,"15min":0.64700000000000002},"min":{"1min":0.435,"5min":0.435,"15min":0.435},"max":{"1min":0.88200000000000001,"5min":0.88200000000000001,"15min":0.88200000000000001},"last":0.81299999999999994},{"interface":"front","average":{"1min":0.67300000000000004,"5min":0.67300000000000004,"15min":0.67300000000000004},"min":{"1min":0.44400000000000001,"5min":0.44400000000000001,"15min":0.44400000000000001},"max":{"1min":0.92900000000000005,"5min":0.92900000000000005,"15min":0.92900000000000005},"last":0.78900000000000003}]},{"osd":5,"last update":"Sat Sep 23 20:51:04 2023","interfaces":[{"interface":"back","average":{"1min":0.69799999999999995,"5min":0.69799999999999995,"15min":0.69799999999999995},"min":{"1min":0.39500000000000002,"5min":0.39500000000000002,"15min":0.39500000000000002},"max":{"1min":0.86899999999999999,"5min":0.86899999999999999,"15min":0.86899999999999999},"last":0.76700000000000002},{"interface":"front","average":{"1min":0.64700000000000002,"5min":0.64700000000000002,"15min":0.64700000000000002},"min":{"1min":0.47899999999999998,"5min":0.47899999999999998,"15min":0.47899999999999998},"max":{"1min":0.79200000000000004,"5min":0.79200000000000004,"15min":0.79200000000000004},"last":0.65000000000000002}]},{"osd":6,"last update":"Sat Sep 23 20:51:04 2023","interfaces":[{"interface":"back","average":{"1min":0.68799999999999994,"5min":0.68799999999999994,"15min":0.68799999999999994},"min":{"1min":0.47999999999999998,"5min":0.47999999999999998,"15min":0.47999999999999998},"max":{"1min":0.98799999999999999,"5min":0.98799999999999999,"15min":0.98799999999999999},"last":0.86699999999999999},{"interface":"front","average":{"1min":0.65800000000000003,"5min":0.65800000000000003,"15min":0.65800000000000003},"min":{"1min":0.41799999999999998,"5min":0.41799999999999998,"15min":0.41799999999999998},"max":{"1min":0.90900000000000003,"5min":0.90900000000000003,"15min":0.90900000000000003},"last":0.72399999999999998}]},{"osd":7,"last update":"Sat Sep 23 20:51:04 2023","interfaces":[{"interface":"back","average":{"1min":0.67400000000000004,"5min":0.67400000000000004,"15min":0.67400000000000004},"min":{"1min":0.46999999999999997,"5min":0.46999999999999997,"15min":0.46999999999999997},"max":{"1min":1.004,"5min":1.004,"15min":1.004},"last":0.85199999999999998},{"interface":"front","average":{"1min":0.68600000000000005,"5min":0.68600000000000005,"15min":0.68600000000000005},"min":{"1min":0.502,"5min":0.502,"15min":0.502},"max":{"1min":1.036,"5min":1.036,"15min":1.036},"last":0.80300000000000005}]}]},{"osd":0,"up_from":83,"seq":356482285598,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":29140,"kb_used_data":1612,"kb_used_omap":28,"kb_used_meta":27491,"kb_avail":93716012,"statfs":{"total":95995035648,"available":95965196288,"internally_reserved":0,"allocated":1650688,"data_stored":1136306,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":28848,"internal_metadata":28151632},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 20:51:07 2023","interfaces":[{"interface":"back","average":{"1min":0.33300000000000002,"5min":0.33300000000000002,"15min":0.33300000000000002},"min":{"1min":0.23400000000000001,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":0.503,"5min":0.503,"15min":0.503},"last":0.27500000000000002},{"interface":"front","average":{"1min":0.38700000000000001,"5min":0.38700000000000001,"15min":0.38700000000000001},"min":{"1min":0.26100000000000001,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":0.54100000000000004,"5min":0.54100000000000004,"15min":0.54100000000000004},"last":0.26000000000000001}]},{"osd":2,"last update":"Sat Sep 23 20:51:37 2023","interfaces":[{"interface":"back","average":{"1min":0.41499999999999998,"5min":0.41499999999999998,"15min":0.41499999999999998},"min":{"1min":0.248,"5min":0.248,"15min":0.248},"max":{"1min":0.61799999999999999,"5min":0.61799999999999999,"15min":0.61799999999999999},"last":0.376},{"interface":"front","average":{"1min":0.39000000000000001,"5min":0.39000000000000001,"15min":0.39000000000000001},"min":{"1min":0.25700000000000001,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":0.55900000000000005,"5min":0.55900000000000005,"15min":0.55900000000000005},"last":0.309}]},{"osd":3,"last update":"Sat Sep 23 20:50:33 2023","interfaces":[{"interface":"back","average":{"1min":0.46000000000000002,"5min":0.46000000000000002,"15min":0.46000000000000002},"min":{"1min":0.28899999999999998,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":0.72699999999999998,"5min":0.72699999999999998,"15min":0.72699999999999998},"last":0.36499999999999999},{"interface":"front","average":{"1min":0.45500000000000002,"5min":0.45500000000000002,"15min":0.45500000000000002},"min":{"1min":0.33100000000000002,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":0.64400000000000002,"5min":0.64400000000000002,"15min":0.64400000000000002},"last":0.41399999999999998}]},{"osd":4,"last update":"Sat Sep 23 20:51:37 2023","interfaces":[{"interface":"back","average":{"1min":0.64800000000000002,"5min":0.64800000000000002,"15min":0.64800000000000002},"min":{"1min":0.377,"5min":0.377,"15min":0.377},"max":{"1min":1.9870000000000001,"5min":1.9870000000000001,"15min":1.9870000000000001},"last":0.498},{"interface":"front","average":{"1min":0.68899999999999995,"5min":0.66500000000000004,"15min":0.66100000000000003},"min":{"1min":0.41399999999999998,"5min":0.41399999999999998,"15min":0.41399999999999998},"max":{"1min":2.0049999999999999,"5min":2.0049999999999999,"15min":2.0049999999999999},"last":0.54600000000000004}]},{"osd":5,"last update":"Sat Sep 23 20:51:37 2023","interfaces":[{"interface":"back","average":{"1min":0.60099999999999998,"5min":0.63900000000000001,"15min":0.64500000000000002},"min":{"1min":0.42299999999999999,"5min":0.42299999999999999,"15min":0.42299999999999999},"max":{"1min":0.80400000000000005,"5min":0.83799999999999997,"15min":0.83799999999999997},"last":0.55800000000000005},{"interface":"front","average":{"1min":0.60799999999999998,"5min":0.68700000000000006,"15min":0.69999999999999996},"min":{"1min":0.42499999999999999,"5min":0.42499999999999999,"15min":0.42499999999999999},"max":{"1min":0.88400000000000001,"5min":0.90000000000000002,"15min":0.90000000000000002},"last":0.46100000000000002}]},{"osd":6,"last update":"Sat Sep 23 20:51:37 2023","interfaces":[{"interface":"back","average":{"1min":0.60799999999999998,"5min":0.67400000000000004,"15min":0.68500000000000005},"min":{"1min":0.433,"5min":0.433,"15min":0.433},"max":{"1min":0.876,"5min":0.93999999999999995,"15min":0.93999999999999995},"last":0.51600000000000001},{"interface":"front","average":{"1min":0.72099999999999997,"5min":0.68899999999999995,"15min":0.68300000000000005},"min":{"1min":0.45000000000000001,"5min":0.44600000000000001,"15min":0.44600000000000001},"max":{"1min":2.0489999999999999,"5min":2.0489999999999999,"15min":2.0489999999999999},"last":0.50900000000000001}]},{"osd":7,"last update":"Sat Sep 23 20:51:37 2023","interfaces":[{"interface":"back","average":{"1min":0.64000000000000001,"5min":0.68799999999999994,"15min":0.69599999999999995},"min":{"1min":0.42799999999999999,"5min":0.42799999999999999,"15min":0.42799999999999999},"max":{"1min":0.82399999999999995,"5min":0.92000000000000004,"15min":0.92000000000000004},"last":0.47299999999999998},{"interface":"front","average":{"1min":0.63900000000000001,"5min":0.66700000000000004,"15min":0.67100000000000004},"min":{"1min":0.442,"5min":0.442,"15min":0.442},"max":{"1min":0.85799999999999998,"5min":0.90600000000000003,"15min":0.90600000000000003},"last":0.57099999999999995}]}]},{"osd":2,"up_from":93,"seq":399431958545,"num_pgs":19,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28600,"kb_used_data":1008,"kb_used_omap":81,"kb_used_meta":27502,"kb_avail":93716552,"statfs":{"total":95995035648,"available":95965749248,"internally_reserved":0,"allocated":1032192,"data_stored":541255,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":83750,"internal_metadata":28162266},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:51:38 2023","interfaces":[{"interface":"back","average":{"1min":0.35799999999999998,"5min":0.35799999999999998,"15min":0.35799999999999998},"min":{"1min":0.249,"5min":0.249,"15min":0.249},"max":{"1min":0.54500000000000004,"5min":0.54500000000000004,"15min":0.54500000000000004},"last":0.28199999999999997},{"interface":"front","average":{"1min":0.40699999999999997,"5min":0.40699999999999997,"15min":0.40699999999999997},"min":{"1min":0.309,"5min":0.309,"15min":0.309},"max":{"1min":0.64800000000000002,"5min":0.64800000000000002,"15min":0.64800000000000002},"last":0.309}]},{"osd":1,"last update":"Sat Sep 23 20:51:38 2023","interfaces":[{"interface":"back","average":{"1min":0.41699999999999998,"5min":0.41699999999999998,"15min":0.41699999999999998},"min":{"1min":0.31,"5min":0.31,"15min":0.31},"max":{"1min":0.61699999999999999,"5min":0.61699999999999999,"15min":0.61699999999999999},"last":0.33300000000000002},{"interface":"front","average":{"1min":0.42099999999999999,"5min":0.42099999999999999,"15min":0.42099999999999999},"min":{"1min":0.33200000000000002,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":0.55600000000000005,"5min":0.55600000000000005,"15min":0.55600000000000005},"last":0.35699999999999998}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.38300000000000001}]},{"osd":4,"last update":"Sat Sep 23 20:51:38 2023","interfaces":[{"interface":"back","average":{"1min":0.61899999999999999,"5min":0.61899999999999999,"15min":0.61899999999999999},"min":{"1min":0.45200000000000001,"5min":0.45200000000000001,"15min":0.45200000000000001},"max":{"1min":0.86099999999999999,"5min":0.86099999999999999,"15min":0.86099999999999999},"last":0.54300000000000004},{"interface":"front","average":{"1min":0.67100000000000004,"5min":0.67100000000000004,"15min":0.67100000000000004},"min":{"1min":0.47299999999999998,"5min":0.47299999999999998,"15min":0.47299999999999998},"max":{"1min":0.92400000000000004,"5min":0.92400000000000004,"15min":0.92400000000000004},"last":0.53300000000000003}]},{"osd":5,"last update":"Sat Sep 23 20:51:38 2023","interfaces":[{"interface":"back","average":{"1min":0.65200000000000002,"5min":0.65200000000000002,"15min":0.65200000000000002},"min":{"1min":0.46300000000000002,"5min":0.46300000000000002,"15min":0.46300000000000002},"max":{"1min":0.95999999999999996,"5min":0.95999999999999996,"15min":0.95999999999999996},"last":0.58899999999999997},{"interface":"front","average":{"1min":0.63600000000000001,"5min":0.63600000000000001,"15min":0.63600000000000001},"min":{"1min":0.42299999999999999,"5min":0.42299999999999999,"15min":0.42299999999999999},"max":{"1min":0.89600000000000002,"5min":0.89600000000000002,"15min":0.89600000000000002},"last":0.51000000000000001}]},{"osd":6,"last update":"Sat Sep 23 20:51:38 2023","interfaces":[{"interface":"back","average":{"1min":0.70599999999999996,"5min":0.70599999999999996,"15min":0.70599999999999996},"min":{"1min":0.502,"5min":0.502,"15min":0.502},"max":{"1min":0.97299999999999998,"5min":0.97299999999999998,"15min":0.97299999999999998},"last":0.55000000000000004},{"interface":"front","average":{"1min":0.68600000000000005,"5min":0.68600000000000005,"15min":0.68600000000000005},"min":{"1min":0.49399999999999999,"5min":0.49399999999999999,"15min":0.49399999999999999},"max":{"1min":0.93999999999999995,"5min":0.93999999999999995,"15min":0.93999999999999995},"last":0.60999999999999999}]},{"osd":7,"last update":"Sat Sep 23 20:51:38 2023","interfaces":[{"interface":"back","average":{"1min":0.63500000000000001,"5min":0.63500000000000001,"15min":0.63500000000000001},"min":{"1min":0.498,"5min":0.498,"15min":0.498},"max":{"1min":0.90600000000000003,"5min":0.90600000000000003,"15min":0.90600000000000003},"last":0.63300000000000001},{"interface":"front","average":{"1min":0.66200000000000003,"5min":0.66200000000000003,"15min":0.66200000000000003},"min":{"1min":0.433,"5min":0.433,"15min":0.433},"max":{"1min":0.90500000000000003,"5min":0.90500000000000003,"15min":0.90500000000000003},"last":0.501}]}]},{"osd":3,"up_from":98,"seq":420906795018,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28452,"kb_used_data":988,"kb_used_omap":7,"kb_used_meta":27448,"kb_avail":93716700,"statfs":{"total":95995035648,"available":95965900800,"internally_reserved":0,"allocated":1011712,"data_stored":538170,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7576,"internal_metadata":28107368},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.36699999999999999}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.38600000000000001}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.47299999999999998}]},{"osd":4,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.63200000000000001}]},{"osd":5,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.67300000000000004}]},{"osd":6,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.73899999999999999}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.70699999999999996}]}]},{"osd":4,"up_from":31,"seq":133143986326,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":11876,"kb_used_data":988,"kb_used_omap":0,"kb_used_meta":10880,"kb_avail":93733276,"statfs":{"total":95995035648,"available":95982874624,"internally_reserved":0,"allocated":1011712,"data_stored":538517,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":11141120},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:51:35 2023","interfaces":[{"interface":"back","average":{"1min":0.745,"5min":0.65600000000000003,"15min":0.64100000000000001},"min":{"1min":0.40000000000000002,"5min":0.38900000000000001,"15min":0.38900000000000001},"max":{"1min":4.4050000000000002,"5min":4.4050000000000002,"15min":4.4050000000000002},"last":0.47299999999999998},{"interface":"front","average":{"1min":0.748,"5min":0.88200000000000001,"15min":0.90400000000000003},"min":{"1min":0.46899999999999997,"5min":0.46899999999999997,"15min":0.46899999999999997},"max":{"1min":4.468,"5min":5.2409999999999997,"15min":5.2409999999999997},"last":0.53100000000000003}]},{"osd":1,"last update":"Sat Sep 23 20:51:06 2023","interfaces":[{"interface":"back","average":{"1min":0.55900000000000005,"5min":0.55900000000000005,"15min":0.55900000000000005},"min":{"1min":0.44700000000000001,"5min":0.44700000000000001,"15min":0.44700000000000001},"max":{"1min":0.79900000000000004,"5min":0.79900000000000004,"15min":0.79900000000000004},"last":0.44800000000000001},{"interface":"front","average":{"1min":0.59899999999999998,"5min":0.59899999999999998,"15min":0.59899999999999998},"min":{"1min":0.44500000000000001,"5min":0.44500000000000001,"15min":0.44500000000000001},"max":{"1min":0.72399999999999998,"5min":0.72399999999999998,"15min":0.72399999999999998},"last":0.621}]},{"osd":2,"last update":"Sat Sep 23 20:51:41 2023","interfaces":[{"interface":"back","average":{"1min":0.68600000000000005,"5min":0.68600000000000005,"15min":0.68600000000000005},"min":{"1min":0.442,"5min":0.442,"15min":0.442},"max":{"1min":2.5030000000000001,"5min":2.5030000000000001,"15min":2.5030000000000001},"last":0.51900000000000002},{"interface":"front","average":{"1min":0.77400000000000002,"5min":0.77400000000000002,"15min":0.77400000000000002},"min":{"1min":0.38200000000000001,"5min":0.38200000000000001,"15min":0.38200000000000001},"max":{"1min":4.3289999999999997,"5min":4.3289999999999997,"15min":4.3289999999999997},"last":0.64300000000000002}]},{"osd":3,"last update":"Sat Sep 23 20:50:04 2023","interfaces":[{"interface":"back","average":{"1min":0.63300000000000001,"5min":0.59999999999999998,"15min":0.56799999999999995},"min":{"1min":0.41799999999999998,"5min":0.35299999999999998,"15min":0.33100000000000002},"max":{"1min":2.1320000000000001,"5min":2.1320000000000001,"15min":2.1320000000000001},"last":0.54700000000000004},{"interface":"front","average":{"1min":0.53500000000000003,"5min":0.57799999999999996,"15min":0.55900000000000005},"min":{"1min":0.35699999999999998,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":0.83499999999999996,"5min":0.88,"15min":0.91500000000000004},"last":0.66500000000000004}]},{"osd":5,"last update":"Sat Sep 23 20:51:17 2023","interfaces":[{"interface":"back","average":{"1min":0.57399999999999995,"5min":0.502,"15min":0.45100000000000001},"min":{"1min":0.40000000000000002,"5min":0.29099999999999998,"15min":0.251},"max":{"1min":0.80900000000000005,"5min":0.80900000000000005,"15min":0.83599999999999997},"last":0.59699999999999998},{"interface":"front","average":{"1min":0.54400000000000004,"5min":0.51100000000000001,"15min":0.46000000000000002},"min":{"1min":0.371,"5min":0.33400000000000002,"15min":0.28799999999999998},"max":{"1min":0.79500000000000004,"5min":0.79500000000000004,"15min":0.90000000000000002},"last":0.49399999999999999}]},{"osd":6,"last update":"Sat Sep 23 20:50:49 2023","interfaces":[{"interface":"back","average":{"1min":0.52600000000000002,"5min":0.51200000000000001,"15min":0.51100000000000001},"min":{"1min":0.36399999999999999,"5min":0.30299999999999999,"15min":0.28100000000000003},"max":{"1min":0.67000000000000004,"5min":0.748,"15min":1.1319999999999999},"last":0.50800000000000001},{"interface":"front","average":{"1min":0.59499999999999997,"5min":0.56799999999999995,"15min":0.56399999999999995},"min":{"1min":0.42999999999999999,"5min":0.316,"15min":0.29399999999999998},"max":{"1min":0.79200000000000004,"5min":0.82699999999999996,"15min":1.095},"last":0.56200000000000006}]},{"osd":7,"last update":"Sat Sep 23 20:51:06 2023","interfaces":[{"interface":"back","average":{"1min":0.63500000000000001,"5min":0.59299999999999997,"15min":0.56599999999999995},"min":{"1min":0.44700000000000001,"5min":0.35099999999999998,"15min":0.32500000000000001},"max":{"1min":0.77100000000000002,"5min":0.79100000000000004,"15min":0.877},"last":0.68700000000000006},{"interface":"front","average":{"1min":0.62,"5min":0.58599999999999997,"15min":0.56499999999999995},"min":{"1min":0.42499999999999999,"5min":0.33500000000000002,"15min":0.32700000000000001},"max":{"1min":0.79600000000000004,"5min":0.79600000000000004,"15min":0.86299999999999999},"last":0.42599999999999999}]}]},{"osd":5,"up_from":39,"seq":167503724691,"num_pgs":21,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":17940,"kb_used_data":972,"kb_used_omap":0,"kb_used_meta":16960,"kb_avail":93727212,"statfs":{"total":95995035648,"available":95976665088,"internally_reserved":0,"allocated":995328,"data_stored":534261,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":17367040},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:51:34 2023","interfaces":[{"interface":"back","average":{"1min":0.51300000000000001,"5min":0.499,"15min":0.497},"min":{"1min":0.37,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":0.745,"5min":0.745,"15min":0.745},"last":1.04},{"interface":"front","average":{"1min":0.57399999999999995,"5min":0.57499999999999996,"15min":0.57499999999999996},"min":{"1min":0.35399999999999998,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":0.88400000000000001,"5min":0.88400000000000001,"15min":0.88400000000000001},"last":1.022}]},{"osd":1,"last update":"Sat Sep 23 20:51:05 2023","interfaces":[{"interface":"back","average":{"1min":0.56599999999999995,"5min":0.56599999999999995,"15min":0.56599999999999995},"min":{"1min":0.42499999999999999,"5min":0.42499999999999999,"15min":0.42499999999999999},"max":{"1min":0.83899999999999997,"5min":0.83899999999999997,"15min":0.83899999999999997},"last":1.0669999999999999},{"interface":"front","average":{"1min":0.54900000000000004,"5min":0.54900000000000004,"15min":0.54900000000000004},"min":{"1min":0.39000000000000001,"5min":0.39000000000000001,"15min":0.39000000000000001},"max":{"1min":0.70899999999999996,"5min":0.70899999999999996,"15min":0.70899999999999996},"last":0.95199999999999996}]},{"osd":2,"last update":"Sat Sep 23 20:51:39 2023","interfaces":[{"interface":"back","average":{"1min":0.54200000000000004,"5min":0.54200000000000004,"15min":0.54200000000000004},"min":{"1min":0.436,"5min":0.436,"15min":0.436},"max":{"1min":0.68500000000000005,"5min":0.68500000000000005,"15min":0.68500000000000005},"last":1.1379999999999999},{"interface":"front","average":{"1min":0.58399999999999996,"5min":0.58399999999999996,"15min":0.58399999999999996},"min":{"1min":0.40200000000000002,"5min":0.40200000000000002,"15min":0.40200000000000002},"max":{"1min":0.81699999999999995,"5min":0.81699999999999995,"15min":0.81699999999999995},"last":0.997}]},{"osd":3,"last update":"Sat Sep 23 20:50:16 2023","interfaces":[{"interface":"back","average":{"1min":0.61099999999999999,"5min":0.59199999999999997,"15min":0.56599999999999995},"min":{"1min":0.45600000000000002,"5min":0.39000000000000001,"15min":0.29399999999999998},"max":{"1min":0.79300000000000004,"5min":0.81999999999999995,"15min":0.96099999999999997},"last":1.087},{"interface":"front","average":{"1min":0.59099999999999997,"5min":0.61299999999999999,"15min":0.58399999999999996},"min":{"1min":0.38,"5min":0.37,"15min":0.33100000000000002},"max":{"1min":0.78500000000000003,"5min":0.90400000000000003,"15min":0.97699999999999998},"last":1.105}]},{"osd":4,"last update":"Sat Sep 23 20:51:17 2023","interfaces":[{"interface":"back","average":{"1min":0.53000000000000003,"5min":0.50900000000000001,"15min":0.45900000000000002},"min":{"1min":0.32700000000000001,"5min":0.317,"15min":0.253},"max":{"1min":0.75900000000000001,"5min":0.80900000000000005,"15min":0.80900000000000005},"last":0.43099999999999999},{"interface":"front","average":{"1min":0.54400000000000004,"5min":0.53500000000000003,"15min":0.496},"min":{"1min":0.28499999999999998,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":0.83499999999999996,"5min":0.83499999999999996,"15min":0.89500000000000002},"last":0.36699999999999999}]},{"osd":6,"last update":"Sat Sep 23 20:51:44 2023","interfaces":[{"interface":"back","average":{"1min":0.57399999999999995,"5min":0.54800000000000004,"15min":0.51100000000000001},"min":{"1min":0.34599999999999997,"5min":0.309,"15min":0.28599999999999998},"max":{"1min":0.86799999999999999,"5min":0.86899999999999999,"15min":0.91400000000000003},"last":0.39100000000000001},{"interface":"front","average":{"1min":0.63700000000000001,"5min":0.57799999999999996,"15min":0.54200000000000004},"min":{"1min":0.36299999999999999,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":0.85399999999999998,"5min":0.85399999999999998,"15min":0.85399999999999998},"last":0.52400000000000002}]},{"osd":7,"last update":"Sat Sep 23 20:51:02 2023","interfaces":[{"interface":"back","average":{"1min":0.57299999999999995,"5min":0.58199999999999996,"15min":0.56100000000000005},"min":{"1min":0.38,"5min":0.33600000000000002,"15min":0.314},"max":{"1min":0.77900000000000003,"5min":0.873,"15min":0.874},"last":0.41999999999999998},{"interface":"front","average":{"1min":0.624,"5min":0.61699999999999999,"15min":0.57899999999999996},"min":{"1min":0.41399999999999998,"5min":0.36099999999999999,"15min":0.34100000000000003},"max":{"1min":0.80100000000000005,"5min":0.88500000000000001,"15min":0.88500000000000001},"last":0.61899999999999999}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T20:51:53.499 INFO:tasks.cephadm.ceph_manager.ceph:clean! 2023-09-23T20:51:53.499 INFO:tasks.ceph:Waiting until ceph cluster ceph is healthy... 2023-09-23T20:51:53.499 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy 2023-09-23T20:51:53.499 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph health --format=json 2023-09-23T20:51:54.105 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:51:54.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:51:53 smithi070 ceph-mon[168282]: pgmap v221: 73 pgs: 73 active+clean; 582 KiB data, 171 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:51:54.131 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:51:53 smithi026 ceph-mon[179688]: pgmap v221: 73 pgs: 73 active+clean; 582 KiB data, 171 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:51:54.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:51:53 smithi026 ceph-mon[175151]: pgmap v221: 73 pgs: 73 active+clean; 582 KiB data, 171 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:51:54.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:51:54 smithi026 ceph-mon[175151]: from='client.44615 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:51:54.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:51:54 smithi026 ceph-mon[179688]: from='client.44615 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:51:55.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:51:54 smithi070 ceph-mon[168282]: from='client.44615 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:51:55.560 INFO:teuthology.orchestra.run.smithi026.stdout: 2023-09-23T20:51:55.560 INFO:teuthology.orchestra.run.smithi026.stdout:{"status":"HEALTH_OK","checks":{},"mutes":[]} 2023-09-23T20:51:55.881 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:51:55 smithi026 ceph-mon[175151]: pgmap v222: 73 pgs: 73 active+clean; 582 KiB data, 171 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:51:55.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:51:55 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/2491264075' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2023-09-23T20:51:55.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:51:55 smithi026 ceph-mon[179688]: pgmap v222: 73 pgs: 73 active+clean; 582 KiB data, 171 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:51:55.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:51:55 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/2491264075' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2023-09-23T20:51:55.957 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy done 2023-09-23T20:51:55.958 INFO:teuthology.run_tasks:Running task print... 2023-09-23T20:51:55.969 INFO:teuthology.task.print:**** applying stress + thrashing to mixed-version cluster **** 2023-09-23T20:51:55.969 INFO:teuthology.run_tasks:Running task parallel... 2023-09-23T20:51:55.991 INFO:teuthology.task.parallel:starting parallel... 2023-09-23T20:51:55.991 INFO:teuthology.task.parallel:In parallel, running task sequential... 2023-09-23T20:51:55.991 INFO:teuthology.task.sequential:In sequential, running task thrashosds... 2023-09-23T20:51:55.993 INFO:tasks.thrashosds:config is {'aggressive_pg_num_changes': False, 'chance_force_recovery': 0, 'chance_pgnum_grow': 1, 'chance_pgpnum_fix': 1, 'chance_thrash_cluster_full': 0, 'chance_thrash_pg_upmap': 0, 'chance_thrash_pg_upmap_items': 0, 'disable_objectstore_tool_tests': True, 'timeout': 1200, 'sighup_delay': 0.1, 'optrack_toggle_delay': 2.0, 'dump_ops_enable': 'true', 'noscrub_toggle_delay': 2.0, 'random_eio': 0.0} 2023-09-23T20:51:55.994 INFO:tasks.thrashosds:overrides is {'admin_socket': {'branch': 'reef'}, 'ceph': {'conf': {'global': {'mon election default strategy': 3}, 'mgr': {'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20}, 'osd': {'debug ms': 1, 'debug osd': 20, 'osd shutdown pgref assert': True}}, 'create_rbd_pool': True, 'flavor': 'default', 'log-ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)'], 'sha1': 'bb0daeff97f020f686596fca6eba20c310e26fb7'}, 'ceph-deploy': {'conf': {'client': {'log file': '/var/log/ceph/ceph-$name.$pid.log'}, 'mon': {}}}, 'install': {'ceph': {'flavor': 'default', 'sha1': 'bb0daeff97f020f686596fca6eba20c310e26fb7'}}, 'selinux': {'whitelist': ['scontext=system_u:system_r:logrotate_t:s0']}, 'workunit': {'branch': 'reef', 'sha1': 'bb0daeff97f020f686596fca6eba20c310e26fb7'}} 2023-09-23T20:51:55.994 INFO:tasks.thrashosds:config is {'aggressive_pg_num_changes': False, 'chance_force_recovery': 0, 'chance_pgnum_grow': 1, 'chance_pgpnum_fix': 1, 'chance_thrash_cluster_full': 0, 'chance_thrash_pg_upmap': 0, 'chance_thrash_pg_upmap_items': 0, 'disable_objectstore_tool_tests': True, 'timeout': 1200, 'sighup_delay': 0.1, 'optrack_toggle_delay': 2.0, 'dump_ops_enable': 'true', 'noscrub_toggle_delay': 2.0, 'random_eio': 0.0} 2023-09-23T20:51:55.994 INFO:tasks.thrashosds:Beginning thrashosds... 2023-09-23T20:51:55.994 INFO:tasks.cephadm.ceph_manager.ceph:waiting for clean 2023-09-23T20:51:55.994 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T20:51:56.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:51:55 smithi070 ceph-mon[168282]: pgmap v222: 73 pgs: 73 active+clean; 582 KiB data, 171 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:51:56.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:51:55 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/2491264075' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2023-09-23T20:51:56.600 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:51:58.053 INFO:teuthology.orchestra.run.smithi026.stdout: 2023-09-23T20:51:58.055 INFO:teuthology.orchestra.run.smithi026.stderr:dumped all 2023-09-23T20:51:58.086 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:51:57 smithi026 ceph-mon[175151]: pgmap v223: 73 pgs: 73 active+clean; 582 KiB data, 171 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:51:58.086 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:51:57 smithi026 ceph-mon[179688]: pgmap v223: 73 pgs: 73 active+clean; 582 KiB data, 171 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:51:58.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:51:57 smithi070 ceph-mon[168282]: pgmap v223: 73 pgs: 73 active+clean; 582 KiB data, 171 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:51:58.462 INFO:teuthology.orchestra.run.smithi026.stdout:{"pg_ready":true,"pg_map":{"version":223,"stamp":"2023-09-23T20:51:56.159125+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":595872,"num_objects":24,"num_object_clones":0,"num_object_copies":72,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":24,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":300,"num_write_kb":2445,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":216,"ondisk_log_size":216,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":175016,"kb_used_data":9704,"kb_used_omap":126,"kb_used_meta":165121,"kb_avail":749786200,"statfs":{"total":767960285184,"available":767781068800,"internally_reserved":0,"allocated":9936896,"data_stored":6089406,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":129413,"internal_metadata":169084539},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002763"},"pg_stats":[{"pgid":"4.18","version":"0'0","reported_seq":68,"reported_epoch":98,"state":"active+clean","last_fresh":"2023-09-23T20:51:09.825543+0000","last_change":"2023-09-23T20:50:37.984570+0000","last_active":"2023-09-23T20:51:09.825543+0000","last_peered":"2023-09-23T20:51:09.825543+0000","last_clean":"2023-09-23T20:51:09.825543+0000","last_became_active":"2023-09-23T20:50:37.984262+0000","last_became_peered":"2023-09-23T20:50:37.984262+0000","last_unstale":"2023-09-23T20:51:09.825543+0000","last_undegraded":"2023-09-23T20:51:09.825543+0000","last_fullsized":"2023-09-23T20:51:09.825543+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:47:39.024252+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":56,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-09-23T20:51:10.829873+0000","last_change":"2023-09-23T20:50:37.982897+0000","last_active":"2023-09-23T20:51:10.829873+0000","last_peered":"2023-09-23T20:51:10.829873+0000","last_clean":"2023-09-23T20:51:10.829873+0000","last_became_active":"2023-09-23T20:50:37.982566+0000","last_became_peered":"2023-09-23T20:50:37.982566+0000","last_unstale":"2023-09-23T20:51:10.829873+0000","last_undegraded":"2023-09-23T20:51:10.829873+0000","last_fullsized":"2023-09-23T20:51:10.829873+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:00:00.068989+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":82,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-09-23T20:51:10.838334+0000","last_change":"2023-09-23T20:51:10.838334+0000","last_active":"2023-09-23T20:51:10.838334+0000","last_peered":"2023-09-23T20:51:10.838334+0000","last_clean":"2023-09-23T20:51:10.838334+0000","last_became_active":"2023-09-23T20:51:10.838026+0000","last_became_peered":"2023-09-23T20:51:10.838026+0000","last_unstale":"2023-09-23T20:51:10.838334+0000","last_undegraded":"2023-09-23T20:51:10.838334+0000","last_fullsized":"2023-09-23T20:51:10.838334+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:04:37.009457+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":67,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-09-23T20:51:10.833849+0000","last_change":"2023-09-23T20:51:10.833849+0000","last_active":"2023-09-23T20:51:10.833849+0000","last_peered":"2023-09-23T20:51:10.833849+0000","last_clean":"2023-09-23T20:51:10.833849+0000","last_became_active":"2023-09-23T20:51:10.833524+0000","last_became_peered":"2023-09-23T20:51:10.833524+0000","last_unstale":"2023-09-23T20:51:10.833849+0000","last_undegraded":"2023-09-23T20:51:10.833849+0000","last_fullsized":"2023-09-23T20:51:10.833849+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:23:12.501924+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":75,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-09-23T20:51:10.835094+0000","last_change":"2023-09-23T20:51:10.835094+0000","last_active":"2023-09-23T20:51:10.835094+0000","last_peered":"2023-09-23T20:51:10.835094+0000","last_clean":"2023-09-23T20:51:10.835094+0000","last_became_active":"2023-09-23T20:51:10.834732+0000","last_became_peered":"2023-09-23T20:51:10.834732+0000","last_unstale":"2023-09-23T20:51:10.835094+0000","last_undegraded":"2023-09-23T20:51:10.835094+0000","last_fullsized":"2023-09-23T20:51:10.835094+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:21:08.414737+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":44,"reported_epoch":98,"state":"active+clean","last_fresh":"2023-09-23T20:51:09.819115+0000","last_change":"2023-09-23T20:41:58.906744+0000","last_active":"2023-09-23T20:51:09.819115+0000","last_peered":"2023-09-23T20:51:09.819115+0000","last_clean":"2023-09-23T20:51:09.819115+0000","last_became_active":"2023-09-23T20:41:58.906409+0000","last_became_peered":"2023-09-23T20:41:58.906409+0000","last_unstale":"2023-09-23T20:51:09.819115+0000","last_undegraded":"2023-09-23T20:51:09.819115+0000","last_fullsized":"2023-09-23T20:51:09.819115+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:15:42.250523+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":75,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-09-23T20:51:10.833684+0000","last_change":"2023-09-23T20:51:10.833684+0000","last_active":"2023-09-23T20:51:10.833684+0000","last_peered":"2023-09-23T20:51:10.833684+0000","last_clean":"2023-09-23T20:51:10.833684+0000","last_became_active":"2023-09-23T20:51:10.833360+0000","last_became_peered":"2023-09-23T20:51:10.833360+0000","last_unstale":"2023-09-23T20:51:10.833684+0000","last_undegraded":"2023-09-23T20:51:10.833684+0000","last_fullsized":"2023-09-23T20:51:10.833684+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:11:12.175158+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":61,"reported_epoch":98,"state":"active+clean","last_fresh":"2023-09-23T20:51:09.818991+0000","last_change":"2023-09-23T20:50:05.866461+0000","last_active":"2023-09-23T20:51:09.818991+0000","last_peered":"2023-09-23T20:51:09.818991+0000","last_clean":"2023-09-23T20:51:09.818991+0000","last_became_active":"2023-09-23T20:50:05.866053+0000","last_became_peered":"2023-09-23T20:50:05.866053+0000","last_unstale":"2023-09-23T20:51:09.818991+0000","last_undegraded":"2023-09-23T20:51:09.818991+0000","last_fullsized":"2023-09-23T20:51:09.818991+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:29:36.203264+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":55,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-09-23T20:51:10.835609+0000","last_change":"2023-09-23T20:51:10.835609+0000","last_active":"2023-09-23T20:51:10.835609+0000","last_peered":"2023-09-23T20:51:10.835609+0000","last_clean":"2023-09-23T20:51:10.835609+0000","last_became_active":"2023-09-23T20:51:10.835302+0000","last_became_peered":"2023-09-23T20:51:10.835302+0000","last_unstale":"2023-09-23T20:51:10.835609+0000","last_undegraded":"2023-09-23T20:51:10.835609+0000","last_fullsized":"2023-09-23T20:51:10.835609+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:07:20.599461+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":40,"reported_epoch":98,"state":"active+clean","last_fresh":"2023-09-23T20:51:09.823784+0000","last_change":"2023-09-23T20:49:32.775562+0000","last_active":"2023-09-23T20:51:09.823784+0000","last_peered":"2023-09-23T20:51:09.823784+0000","last_clean":"2023-09-23T20:51:09.823784+0000","last_became_active":"2023-09-23T20:49:32.775094+0000","last_became_peered":"2023-09-23T20:49:32.775094+0000","last_unstale":"2023-09-23T20:51:09.823784+0000","last_undegraded":"2023-09-23T20:51:09.823784+0000","last_fullsized":"2023-09-23T20:51:09.823784+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:17:52.944043+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":41,"reported_epoch":98,"state":"active+clean","last_fresh":"2023-09-23T20:51:09.826361+0000","last_change":"2023-09-23T20:41:58.918255+0000","last_active":"2023-09-23T20:51:09.826361+0000","last_peered":"2023-09-23T20:51:09.826361+0000","last_clean":"2023-09-23T20:51:09.826361+0000","last_became_active":"2023-09-23T20:41:58.917956+0000","last_became_peered":"2023-09-23T20:41:58.917956+0000","last_unstale":"2023-09-23T20:51:09.826361+0000","last_undegraded":"2023-09-23T20:51:09.826361+0000","last_fullsized":"2023-09-23T20:51:09.826361+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:42:21.698474+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.6","version":"70'1","reported_seq":58,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-09-23T20:51:10.833668+0000","last_change":"2023-09-23T20:50:05.871179+0000","last_active":"2023-09-23T20:51:10.833668+0000","last_peered":"2023-09-23T20:51:10.833668+0000","last_clean":"2023-09-23T20:51:10.833668+0000","last_became_active":"2023-09-23T20:50:05.870816+0000","last_became_peered":"2023-09-23T20:50:05.870816+0000","last_unstale":"2023-09-23T20:51:10.833668+0000","last_undegraded":"2023-09-23T20:51:10.833668+0000","last_fullsized":"2023-09-23T20:51:10.833668+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:25:24.455835+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":63,"reported_epoch":98,"state":"active+clean","last_fresh":"2023-09-23T20:51:09.824227+0000","last_change":"2023-09-23T20:50:37.985663+0000","last_active":"2023-09-23T20:51:09.824227+0000","last_peered":"2023-09-23T20:51:09.824227+0000","last_clean":"2023-09-23T20:51:09.824227+0000","last_became_active":"2023-09-23T20:50:37.985325+0000","last_became_peered":"2023-09-23T20:50:37.985325+0000","last_unstale":"2023-09-23T20:51:09.824227+0000","last_undegraded":"2023-09-23T20:51:09.824227+0000","last_fullsized":"2023-09-23T20:51:09.824227+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:24:30.838843+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":56,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-09-23T20:51:10.833546+0000","last_change":"2023-09-23T20:51:10.833546+0000","last_active":"2023-09-23T20:51:10.833546+0000","last_peered":"2023-09-23T20:51:10.833546+0000","last_clean":"2023-09-23T20:51:10.833546+0000","last_became_active":"2023-09-23T20:51:10.833137+0000","last_became_peered":"2023-09-23T20:51:10.833137+0000","last_unstale":"2023-09-23T20:51:10.833546+0000","last_undegraded":"2023-09-23T20:51:10.833546+0000","last_fullsized":"2023-09-23T20:51:10.833546+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:16:42.840695+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":32,"reported_epoch":98,"state":"active+clean","last_fresh":"2023-09-23T20:51:09.833862+0000","last_change":"2023-09-23T20:50:05.868392+0000","last_active":"2023-09-23T20:51:09.833862+0000","last_peered":"2023-09-23T20:51:09.833862+0000","last_clean":"2023-09-23T20:51:09.833862+0000","last_became_active":"2023-09-23T20:50:05.867910+0000","last_became_peered":"2023-09-23T20:50:05.867910+0000","last_unstale":"2023-09-23T20:51:09.833862+0000","last_undegraded":"2023-09-23T20:51:09.833862+0000","last_fullsized":"2023-09-23T20:51:09.833862+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:25:00.099993+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":59,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-09-23T20:51:10.836910+0000","last_change":"2023-09-23T20:51:10.836910+0000","last_active":"2023-09-23T20:51:10.836910+0000","last_peered":"2023-09-23T20:51:10.836910+0000","last_clean":"2023-09-23T20:51:10.836910+0000","last_became_active":"2023-09-23T20:51:10.836557+0000","last_became_peered":"2023-09-23T20:51:10.836557+0000","last_unstale":"2023-09-23T20:51:10.836910+0000","last_undegraded":"2023-09-23T20:51:10.836910+0000","last_fullsized":"2023-09-23T20:51:10.836910+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:51:54.019969+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":38,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-09-23T20:51:10.827806+0000","last_change":"2023-09-23T20:49:32.775934+0000","last_active":"2023-09-23T20:51:10.827806+0000","last_peered":"2023-09-23T20:51:10.827806+0000","last_clean":"2023-09-23T20:51:10.827806+0000","last_became_active":"2023-09-23T20:49:32.775500+0000","last_became_peered":"2023-09-23T20:49:32.775500+0000","last_unstale":"2023-09-23T20:51:10.827806+0000","last_undegraded":"2023-09-23T20:51:10.827806+0000","last_fullsized":"2023-09-23T20:51:10.827806+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:30:43.381073+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":47,"reported_epoch":98,"state":"active+clean","last_fresh":"2023-09-23T20:51:09.835354+0000","last_change":"2023-09-23T20:50:37.982593+0000","last_active":"2023-09-23T20:51:09.835354+0000","last_peered":"2023-09-23T20:51:09.835354+0000","last_clean":"2023-09-23T20:51:09.835354+0000","last_became_active":"2023-09-23T20:50:37.982303+0000","last_became_peered":"2023-09-23T20:50:37.982303+0000","last_unstale":"2023-09-23T20:51:09.835354+0000","last_undegraded":"2023-09-23T20:51:09.835354+0000","last_fullsized":"2023-09-23T20:51:09.835354+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:53:06.301515+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":63,"reported_epoch":98,"state":"active+clean","last_fresh":"2023-09-23T20:51:09.823655+0000","last_change":"2023-09-23T20:49:32.776947+0000","last_active":"2023-09-23T20:51:09.823655+0000","last_peered":"2023-09-23T20:51:09.823655+0000","last_clean":"2023-09-23T20:51:09.823655+0000","last_became_active":"2023-09-23T20:49:32.776646+0000","last_became_peered":"2023-09-23T20:49:32.776646+0000","last_unstale":"2023-09-23T20:51:09.823655+0000","last_undegraded":"2023-09-23T20:51:09.823655+0000","last_fullsized":"2023-09-23T20:51:09.823655+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:26:50.532203+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":68,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-09-23T20:51:10.833744+0000","last_change":"2023-09-23T20:50:37.982235+0000","last_active":"2023-09-23T20:51:10.833744+0000","last_peered":"2023-09-23T20:51:10.833744+0000","last_clean":"2023-09-23T20:51:10.833744+0000","last_became_active":"2023-09-23T20:50:37.981913+0000","last_became_peered":"2023-09-23T20:50:37.981913+0000","last_unstale":"2023-09-23T20:51:10.833744+0000","last_undegraded":"2023-09-23T20:51:10.833744+0000","last_fullsized":"2023-09-23T20:51:10.833744+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:59:57.122584+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":53,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-09-23T20:51:10.841702+0000","last_change":"2023-09-23T20:51:10.841350+0000","last_active":"2023-09-23T20:51:10.841702+0000","last_peered":"2023-09-23T20:51:10.841702+0000","last_clean":"2023-09-23T20:51:10.841702+0000","last_became_active":"2023-09-23T20:51:10.841029+0000","last_became_peered":"2023-09-23T20:51:10.841029+0000","last_unstale":"2023-09-23T20:51:10.841702+0000","last_undegraded":"2023-09-23T20:51:10.841702+0000","last_fullsized":"2023-09-23T20:51:10.841702+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:26:05.095206+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":80,"reported_epoch":98,"state":"active+clean","last_fresh":"2023-09-23T20:51:09.821811+0000","last_change":"2023-09-23T20:50:05.866898+0000","last_active":"2023-09-23T20:51:09.821811+0000","last_peered":"2023-09-23T20:51:09.821811+0000","last_clean":"2023-09-23T20:51:09.821811+0000","last_became_active":"2023-09-23T20:50:05.866541+0000","last_became_peered":"2023-09-23T20:50:05.866541+0000","last_unstale":"2023-09-23T20:51:09.821811+0000","last_undegraded":"2023-09-23T20:51:09.821811+0000","last_fullsized":"2023-09-23T20:51:09.821811+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:17:02.353010+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":41,"reported_epoch":98,"state":"active+clean","last_fresh":"2023-09-23T20:51:09.823968+0000","last_change":"2023-09-23T20:41:59.062981+0000","last_active":"2023-09-23T20:51:09.823968+0000","last_peered":"2023-09-23T20:51:09.823968+0000","last_clean":"2023-09-23T20:51:09.823968+0000","last_became_active":"2023-09-23T20:41:59.062717+0000","last_became_peered":"2023-09-23T20:41:59.062717+0000","last_unstale":"2023-09-23T20:51:09.823968+0000","last_undegraded":"2023-09-23T20:51:09.823968+0000","last_fullsized":"2023-09-23T20:51:09.823968+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:20:58.386461+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":22,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-09-23T20:51:10.836794+0000","last_change":"2023-09-23T20:51:10.836794+0000","last_active":"2023-09-23T20:51:10.836794+0000","last_peered":"2023-09-23T20:51:10.836794+0000","last_clean":"2023-09-23T20:51:10.836794+0000","last_became_active":"2023-09-23T20:51:10.836449+0000","last_became_peered":"2023-09-23T20:51:10.836449+0000","last_unstale":"2023-09-23T20:51:10.836794+0000","last_undegraded":"2023-09-23T20:51:10.836794+0000","last_fullsized":"2023-09-23T20:51:10.836794+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:00:35.572203+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.0","version":"79'161","reported_seq":440,"reported_epoch":98,"state":"active+clean","last_fresh":"2023-09-23T20:51:09.821731+0000","last_change":"2023-09-23T20:49:32.777246+0000","last_active":"2023-09-23T20:51:09.821731+0000","last_peered":"2023-09-23T20:51:09.821731+0000","last_clean":"2023-09-23T20:51:09.821731+0000","last_became_active":"2023-09-23T20:49:32.776822+0000","last_became_peered":"2023-09-23T20:49:32.776822+0000","last_unstale":"2023-09-23T20:51:09.821731+0000","last_undegraded":"2023-09-23T20:51:09.821731+0000","last_fullsized":"2023-09-23T20:51:09.821731+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:38:57.324224+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:05:46.881275+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":63,"reported_epoch":98,"state":"active+clean","last_fresh":"2023-09-23T20:51:09.817117+0000","last_change":"2023-09-23T20:50:37.984163+0000","last_active":"2023-09-23T20:51:09.817117+0000","last_peered":"2023-09-23T20:51:09.817117+0000","last_clean":"2023-09-23T20:51:09.817117+0000","last_became_active":"2023-09-23T20:50:37.983838+0000","last_became_peered":"2023-09-23T20:50:37.983838+0000","last_unstale":"2023-09-23T20:51:09.817117+0000","last_undegraded":"2023-09-23T20:51:09.817117+0000","last_fullsized":"2023-09-23T20:51:09.817117+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:02:41.792156+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":65,"reported_epoch":98,"state":"active+clean","last_fresh":"2023-09-23T20:51:09.835854+0000","last_change":"2023-09-23T20:50:05.867499+0000","last_active":"2023-09-23T20:51:09.835854+0000","last_peered":"2023-09-23T20:51:09.835854+0000","last_clean":"2023-09-23T20:51:09.835854+0000","last_became_active":"2023-09-23T20:50:05.866571+0000","last_became_peered":"2023-09-23T20:50:05.866571+0000","last_unstale":"2023-09-23T20:51:09.835854+0000","last_undegraded":"2023-09-23T20:51:09.835854+0000","last_fullsized":"2023-09-23T20:51:09.835854+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:18:10.735501+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":47,"reported_epoch":98,"state":"active+clean","last_fresh":"2023-09-23T20:51:09.836383+0000","last_change":"2023-09-23T20:50:37.983072+0000","last_active":"2023-09-23T20:51:09.836383+0000","last_peered":"2023-09-23T20:51:09.836383+0000","last_clean":"2023-09-23T20:51:09.836383+0000","last_became_active":"2023-09-23T20:50:37.982777+0000","last_became_peered":"2023-09-23T20:50:37.982777+0000","last_unstale":"2023-09-23T20:51:09.836383+0000","last_undegraded":"2023-09-23T20:51:09.836383+0000","last_fullsized":"2023-09-23T20:51:09.836383+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:58:11.972144+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":81,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-09-23T20:51:10.838054+0000","last_change":"2023-09-23T20:51:10.838054+0000","last_active":"2023-09-23T20:51:10.838054+0000","last_peered":"2023-09-23T20:51:10.838054+0000","last_clean":"2023-09-23T20:51:10.838054+0000","last_became_active":"2023-09-23T20:51:10.837750+0000","last_became_peered":"2023-09-23T20:51:10.837750+0000","last_unstale":"2023-09-23T20:51:10.838054+0000","last_undegraded":"2023-09-23T20:51:10.838054+0000","last_fullsized":"2023-09-23T20:51:10.838054+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:15:51.648085+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":62,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-09-23T20:51:10.829889+0000","last_change":"2023-09-23T20:51:10.829889+0000","last_active":"2023-09-23T20:51:10.829889+0000","last_peered":"2023-09-23T20:51:10.829889+0000","last_clean":"2023-09-23T20:51:10.829889+0000","last_became_active":"2023-09-23T20:51:10.829533+0000","last_became_peered":"2023-09-23T20:51:10.829533+0000","last_unstale":"2023-09-23T20:51:10.829889+0000","last_undegraded":"2023-09-23T20:51:10.829889+0000","last_fullsized":"2023-09-23T20:51:10.829889+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:43:04.603460+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":59,"reported_epoch":98,"state":"active+clean","last_fresh":"2023-09-23T20:51:09.826642+0000","last_change":"2023-09-23T20:49:32.771117+0000","last_active":"2023-09-23T20:51:09.826642+0000","last_peered":"2023-09-23T20:51:09.826642+0000","last_clean":"2023-09-23T20:51:09.826642+0000","last_became_active":"2023-09-23T20:49:32.770686+0000","last_became_peered":"2023-09-23T20:49:32.770686+0000","last_unstale":"2023-09-23T20:51:09.826642+0000","last_undegraded":"2023-09-23T20:51:09.826642+0000","last_fullsized":"2023-09-23T20:51:09.826642+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:15:46.598698+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":63,"reported_epoch":98,"state":"active+clean","last_fresh":"2023-09-23T20:51:09.817089+0000","last_change":"2023-09-23T20:50:05.864620+0000","last_active":"2023-09-23T20:51:09.817089+0000","last_peered":"2023-09-23T20:51:09.817089+0000","last_clean":"2023-09-23T20:51:09.817089+0000","last_became_active":"2023-09-23T20:50:05.864276+0000","last_became_peered":"2023-09-23T20:50:05.864276+0000","last_unstale":"2023-09-23T20:51:09.817089+0000","last_undegraded":"2023-09-23T20:51:09.817089+0000","last_fullsized":"2023-09-23T20:51:09.817089+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:20:19.862901+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":32,"reported_epoch":98,"state":"active+clean","last_fresh":"2023-09-23T20:51:09.834324+0000","last_change":"2023-09-23T20:50:05.868321+0000","last_active":"2023-09-23T20:51:09.834324+0000","last_peered":"2023-09-23T20:51:09.834324+0000","last_clean":"2023-09-23T20:51:09.834324+0000","last_became_active":"2023-09-23T20:50:05.868120+0000","last_became_peered":"2023-09-23T20:50:05.868120+0000","last_unstale":"2023-09-23T20:51:09.834324+0000","last_undegraded":"2023-09-23T20:51:09.834324+0000","last_fullsized":"2023-09-23T20:51:09.834324+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:23:14.856477+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":76,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-09-23T20:51:10.840273+0000","last_change":"2023-09-23T20:51:10.840273+0000","last_active":"2023-09-23T20:51:10.840273+0000","last_peered":"2023-09-23T20:51:10.840273+0000","last_clean":"2023-09-23T20:51:10.840273+0000","last_became_active":"2023-09-23T20:51:10.840017+0000","last_became_peered":"2023-09-23T20:51:10.840017+0000","last_unstale":"2023-09-23T20:51:10.840273+0000","last_undegraded":"2023-09-23T20:51:10.840273+0000","last_fullsized":"2023-09-23T20:51:10.840273+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:58:17.982729+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":70,"reported_epoch":98,"state":"active+clean","last_fresh":"2023-09-23T20:51:09.835307+0000","last_change":"2023-09-23T20:50:05.867130+0000","last_active":"2023-09-23T20:51:09.835307+0000","last_peered":"2023-09-23T20:51:09.835307+0000","last_clean":"2023-09-23T20:51:09.835307+0000","last_became_active":"2023-09-23T20:50:05.866758+0000","last_became_peered":"2023-09-23T20:50:05.866758+0000","last_unstale":"2023-09-23T20:51:09.835307+0000","last_undegraded":"2023-09-23T20:51:09.835307+0000","last_fullsized":"2023-09-23T20:51:09.835307+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:55:07.548209+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":47,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-09-23T20:51:10.833850+0000","last_change":"2023-09-23T20:51:10.833850+0000","last_active":"2023-09-23T20:51:10.833850+0000","last_peered":"2023-09-23T20:51:10.833850+0000","last_clean":"2023-09-23T20:51:10.833850+0000","last_became_active":"2023-09-23T20:51:10.833539+0000","last_became_peered":"2023-09-23T20:51:10.833539+0000","last_unstale":"2023-09-23T20:51:10.833850+0000","last_undegraded":"2023-09-23T20:51:10.833850+0000","last_fullsized":"2023-09-23T20:51:10.833850+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:01:36.445616+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":62,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-09-23T20:51:10.831197+0000","last_change":"2023-09-23T20:51:10.831197+0000","last_active":"2023-09-23T20:51:10.831197+0000","last_peered":"2023-09-23T20:51:10.831197+0000","last_clean":"2023-09-23T20:51:10.831197+0000","last_became_active":"2023-09-23T20:51:10.830878+0000","last_became_peered":"2023-09-23T20:51:10.830878+0000","last_unstale":"2023-09-23T20:51:10.831197+0000","last_undegraded":"2023-09-23T20:51:10.831197+0000","last_fullsized":"2023-09-23T20:51:10.831197+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:05:24.314478+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":58,"reported_epoch":98,"state":"active+clean","last_fresh":"2023-09-23T20:51:09.826968+0000","last_change":"2023-09-23T20:50:05.871406+0000","last_active":"2023-09-23T20:51:09.826968+0000","last_peered":"2023-09-23T20:51:09.826968+0000","last_clean":"2023-09-23T20:51:09.826968+0000","last_became_active":"2023-09-23T20:50:05.871127+0000","last_became_peered":"2023-09-23T20:50:05.871127+0000","last_unstale":"2023-09-23T20:51:09.826968+0000","last_undegraded":"2023-09-23T20:51:09.826968+0000","last_fullsized":"2023-09-23T20:51:09.826968+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:37:48.850229+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":59,"reported_epoch":98,"state":"active+clean","last_fresh":"2023-09-23T20:51:09.826268+0000","last_change":"2023-09-23T20:50:37.983914+0000","last_active":"2023-09-23T20:51:09.826268+0000","last_peered":"2023-09-23T20:51:09.826268+0000","last_clean":"2023-09-23T20:51:09.826268+0000","last_became_active":"2023-09-23T20:50:37.983617+0000","last_became_peered":"2023-09-23T20:50:37.983617+0000","last_unstale":"2023-09-23T20:51:09.826268+0000","last_undegraded":"2023-09-23T20:51:09.826268+0000","last_fullsized":"2023-09-23T20:51:09.826268+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:15:50.863306+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":75,"reported_epoch":98,"state":"active+clean","last_fresh":"2023-09-23T20:51:09.826833+0000","last_change":"2023-09-23T20:50:37.985593+0000","last_active":"2023-09-23T20:51:09.826833+0000","last_peered":"2023-09-23T20:51:09.826833+0000","last_clean":"2023-09-23T20:51:09.826833+0000","last_became_active":"2023-09-23T20:50:37.985324+0000","last_became_peered":"2023-09-23T20:50:37.985324+0000","last_unstale":"2023-09-23T20:51:09.826833+0000","last_undegraded":"2023-09-23T20:51:09.826833+0000","last_fullsized":"2023-09-23T20:51:09.826833+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:34:07.801771+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":59,"reported_epoch":98,"state":"active+clean","last_fresh":"2023-09-23T20:51:09.824658+0000","last_change":"2023-09-23T20:50:05.867646+0000","last_active":"2023-09-23T20:51:09.824658+0000","last_peered":"2023-09-23T20:51:09.824658+0000","last_clean":"2023-09-23T20:51:09.824658+0000","last_became_active":"2023-09-23T20:50:05.867343+0000","last_became_peered":"2023-09-23T20:50:05.867343+0000","last_unstale":"2023-09-23T20:51:09.824658+0000","last_undegraded":"2023-09-23T20:51:09.824658+0000","last_fullsized":"2023-09-23T20:51:09.824658+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:53:34.985580+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":58,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-09-23T20:51:10.833630+0000","last_change":"2023-09-23T20:51:10.833630+0000","last_active":"2023-09-23T20:51:10.833630+0000","last_peered":"2023-09-23T20:51:10.833630+0000","last_clean":"2023-09-23T20:51:10.833630+0000","last_became_active":"2023-09-23T20:51:10.833265+0000","last_became_peered":"2023-09-23T20:51:10.833265+0000","last_unstale":"2023-09-23T20:51:10.833630+0000","last_undegraded":"2023-09-23T20:51:10.833630+0000","last_fullsized":"2023-09-23T20:51:10.833630+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:12:01.557967+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":59,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-09-23T20:51:10.836752+0000","last_change":"2023-09-23T20:51:10.836752+0000","last_active":"2023-09-23T20:51:10.836752+0000","last_peered":"2023-09-23T20:51:10.836752+0000","last_clean":"2023-09-23T20:51:10.836752+0000","last_became_active":"2023-09-23T20:51:10.836213+0000","last_became_peered":"2023-09-23T20:51:10.836213+0000","last_unstale":"2023-09-23T20:51:10.836752+0000","last_undegraded":"2023-09-23T20:51:10.836752+0000","last_fullsized":"2023-09-23T20:51:10.836752+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:35:01.825635+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":66,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-09-23T20:51:10.827903+0000","last_change":"2023-09-23T20:50:37.982263+0000","last_active":"2023-09-23T20:51:10.827903+0000","last_peered":"2023-09-23T20:51:10.827903+0000","last_clean":"2023-09-23T20:51:10.827903+0000","last_became_active":"2023-09-23T20:50:37.981797+0000","last_became_peered":"2023-09-23T20:50:37.981797+0000","last_unstale":"2023-09-23T20:51:10.827903+0000","last_undegraded":"2023-09-23T20:51:10.827903+0000","last_fullsized":"2023-09-23T20:51:10.827903+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:56:45.027447+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":59,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-09-23T20:51:10.839104+0000","last_change":"2023-09-23T20:51:10.839104+0000","last_active":"2023-09-23T20:51:10.839104+0000","last_peered":"2023-09-23T20:51:10.839104+0000","last_clean":"2023-09-23T20:51:10.839104+0000","last_became_active":"2023-09-23T20:51:10.838783+0000","last_became_peered":"2023-09-23T20:51:10.838783+0000","last_unstale":"2023-09-23T20:51:10.839104+0000","last_undegraded":"2023-09-23T20:51:10.839104+0000","last_fullsized":"2023-09-23T20:51:10.839104+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:24:32.550241+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":58,"reported_epoch":98,"state":"active+clean","last_fresh":"2023-09-23T20:51:09.824495+0000","last_change":"2023-09-23T20:50:05.867973+0000","last_active":"2023-09-23T20:51:09.824495+0000","last_peered":"2023-09-23T20:51:09.824495+0000","last_clean":"2023-09-23T20:51:09.824495+0000","last_became_active":"2023-09-23T20:50:05.867622+0000","last_became_peered":"2023-09-23T20:50:05.867622+0000","last_unstale":"2023-09-23T20:51:09.824495+0000","last_undegraded":"2023-09-23T20:51:09.824495+0000","last_fullsized":"2023-09-23T20:51:09.824495+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:10:12.617597+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":43,"reported_epoch":98,"state":"active+clean","last_fresh":"2023-09-23T20:51:09.823751+0000","last_change":"2023-09-23T20:41:58.898346+0000","last_active":"2023-09-23T20:51:09.823751+0000","last_peered":"2023-09-23T20:51:09.823751+0000","last_clean":"2023-09-23T20:51:09.823751+0000","last_became_active":"2023-09-23T20:41:58.898065+0000","last_became_peered":"2023-09-23T20:41:58.898065+0000","last_unstale":"2023-09-23T20:51:09.823751+0000","last_undegraded":"2023-09-23T20:51:09.823751+0000","last_fullsized":"2023-09-23T20:51:09.823751+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:51:57.491171+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":75,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-09-23T20:51:10.838338+0000","last_change":"2023-09-23T20:51:10.838338+0000","last_active":"2023-09-23T20:51:10.838338+0000","last_peered":"2023-09-23T20:51:10.838338+0000","last_clean":"2023-09-23T20:51:10.838338+0000","last_became_active":"2023-09-23T20:51:10.838049+0000","last_became_peered":"2023-09-23T20:51:10.838049+0000","last_unstale":"2023-09-23T20:51:10.838338+0000","last_undegraded":"2023-09-23T20:51:10.838338+0000","last_fullsized":"2023-09-23T20:51:10.838338+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:40:23.602402+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":61,"reported_epoch":98,"state":"active+clean","last_fresh":"2023-09-23T20:51:09.823256+0000","last_change":"2023-09-23T20:50:05.866983+0000","last_active":"2023-09-23T20:51:09.823256+0000","last_peered":"2023-09-23T20:51:09.823256+0000","last_clean":"2023-09-23T20:51:09.823256+0000","last_became_active":"2023-09-23T20:50:05.866601+0000","last_became_peered":"2023-09-23T20:50:05.866601+0000","last_unstale":"2023-09-23T20:51:09.823256+0000","last_undegraded":"2023-09-23T20:51:09.823256+0000","last_fullsized":"2023-09-23T20:51:09.823256+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:15:00.955166+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":41,"reported_epoch":98,"state":"active+clean","last_fresh":"2023-09-23T20:51:09.816955+0000","last_change":"2023-09-23T20:41:59.061405+0000","last_active":"2023-09-23T20:51:09.816955+0000","last_peered":"2023-09-23T20:51:09.816955+0000","last_clean":"2023-09-23T20:51:09.816955+0000","last_became_active":"2023-09-23T20:41:59.061032+0000","last_became_peered":"2023-09-23T20:41:59.061032+0000","last_unstale":"2023-09-23T20:51:09.816955+0000","last_undegraded":"2023-09-23T20:51:09.816955+0000","last_fullsized":"2023-09-23T20:51:09.816955+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:43:09.716350+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":60,"reported_epoch":98,"state":"active+clean","last_fresh":"2023-09-23T20:51:09.822945+0000","last_change":"2023-09-23T20:49:32.777943+0000","last_active":"2023-09-23T20:51:09.822945+0000","last_peered":"2023-09-23T20:51:09.822945+0000","last_clean":"2023-09-23T20:51:09.822945+0000","last_became_active":"2023-09-23T20:49:32.776212+0000","last_became_peered":"2023-09-23T20:49:32.776212+0000","last_unstale":"2023-09-23T20:51:09.822945+0000","last_undegraded":"2023-09-23T20:51:09.822945+0000","last_fullsized":"2023-09-23T20:51:09.822945+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:21:05.725926+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":61,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-09-23T20:51:10.833234+0000","last_change":"2023-09-23T20:51:10.833234+0000","last_active":"2023-09-23T20:51:10.833234+0000","last_peered":"2023-09-23T20:51:10.833234+0000","last_clean":"2023-09-23T20:51:10.833234+0000","last_became_active":"2023-09-23T20:51:10.832763+0000","last_became_peered":"2023-09-23T20:51:10.832763+0000","last_unstale":"2023-09-23T20:51:10.833234+0000","last_undegraded":"2023-09-23T20:51:10.833234+0000","last_fullsized":"2023-09-23T20:51:10.833234+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:24:08.283578+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":67,"reported_epoch":98,"state":"active+clean","last_fresh":"2023-09-23T20:51:09.824402+0000","last_change":"2023-09-23T20:49:32.777437+0000","last_active":"2023-09-23T20:51:09.824402+0000","last_peered":"2023-09-23T20:51:09.824402+0000","last_clean":"2023-09-23T20:51:09.824402+0000","last_became_active":"2023-09-23T20:49:32.776999+0000","last_became_peered":"2023-09-23T20:49:32.776999+0000","last_unstale":"2023-09-23T20:51:09.824402+0000","last_undegraded":"2023-09-23T20:51:09.824402+0000","last_fullsized":"2023-09-23T20:51:09.824402+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:42:21.816073+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":53,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-09-23T20:51:10.841582+0000","last_change":"2023-09-23T20:51:10.840849+0000","last_active":"2023-09-23T20:51:10.841582+0000","last_peered":"2023-09-23T20:51:10.841582+0000","last_clean":"2023-09-23T20:51:10.841582+0000","last_became_active":"2023-09-23T20:51:10.840593+0000","last_became_peered":"2023-09-23T20:51:10.840593+0000","last_unstale":"2023-09-23T20:51:10.841582+0000","last_undegraded":"2023-09-23T20:51:10.841582+0000","last_fullsized":"2023-09-23T20:51:10.841582+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:30:52.297625+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":42,"reported_epoch":98,"state":"active+clean","last_fresh":"2023-09-23T20:51:09.821305+0000","last_change":"2023-09-23T20:41:58.898224+0000","last_active":"2023-09-23T20:51:09.821305+0000","last_peered":"2023-09-23T20:51:09.821305+0000","last_clean":"2023-09-23T20:51:09.821305+0000","last_became_active":"2023-09-23T20:41:58.898036+0000","last_became_peered":"2023-09-23T20:41:58.898036+0000","last_unstale":"2023-09-23T20:51:09.821305+0000","last_undegraded":"2023-09-23T20:51:09.821305+0000","last_fullsized":"2023-09-23T20:51:09.821305+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:53:15.361560+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":58,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-09-23T20:51:10.839174+0000","last_change":"2023-09-23T20:51:10.839174+0000","last_active":"2023-09-23T20:51:10.839174+0000","last_peered":"2023-09-23T20:51:10.839174+0000","last_clean":"2023-09-23T20:51:10.839174+0000","last_became_active":"2023-09-23T20:51:10.838826+0000","last_became_peered":"2023-09-23T20:51:10.838826+0000","last_unstale":"2023-09-23T20:51:10.839174+0000","last_undegraded":"2023-09-23T20:51:10.839174+0000","last_fullsized":"2023-09-23T20:51:10.839174+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:43:00.944119+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":53,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-09-23T20:51:10.841667+0000","last_change":"2023-09-23T20:51:10.840590+0000","last_active":"2023-09-23T20:51:10.841667+0000","last_peered":"2023-09-23T20:51:10.841667+0000","last_clean":"2023-09-23T20:51:10.841667+0000","last_became_active":"2023-09-23T20:51:10.840221+0000","last_became_peered":"2023-09-23T20:51:10.840221+0000","last_unstale":"2023-09-23T20:51:10.841667+0000","last_undegraded":"2023-09-23T20:51:10.841667+0000","last_fullsized":"2023-09-23T20:51:10.841667+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:03:39.138638+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":61,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-09-23T20:51:10.831262+0000","last_change":"2023-09-23T20:51:10.831262+0000","last_active":"2023-09-23T20:51:10.831262+0000","last_peered":"2023-09-23T20:51:10.831262+0000","last_clean":"2023-09-23T20:51:10.831262+0000","last_became_active":"2023-09-23T20:51:10.830931+0000","last_became_peered":"2023-09-23T20:51:10.830931+0000","last_unstale":"2023-09-23T20:51:10.831262+0000","last_undegraded":"2023-09-23T20:51:10.831262+0000","last_fullsized":"2023-09-23T20:51:10.831262+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:51:54.982505+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":61,"reported_epoch":98,"state":"active+clean","last_fresh":"2023-09-23T20:51:09.821700+0000","last_change":"2023-09-23T20:50:37.986729+0000","last_active":"2023-09-23T20:51:09.821700+0000","last_peered":"2023-09-23T20:51:09.821700+0000","last_clean":"2023-09-23T20:51:09.821700+0000","last_became_active":"2023-09-23T20:50:37.986433+0000","last_became_peered":"2023-09-23T20:50:37.986433+0000","last_unstale":"2023-09-23T20:51:09.821700+0000","last_undegraded":"2023-09-23T20:51:09.821700+0000","last_fullsized":"2023-09-23T20:51:09.821700+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:28:28.853091+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":58,"reported_epoch":98,"state":"active+clean","last_fresh":"2023-09-23T20:51:09.826698+0000","last_change":"2023-09-23T20:50:05.871199+0000","last_active":"2023-09-23T20:51:09.826698+0000","last_peered":"2023-09-23T20:51:09.826698+0000","last_clean":"2023-09-23T20:51:09.826698+0000","last_became_active":"2023-09-23T20:50:05.870912+0000","last_became_peered":"2023-09-23T20:50:05.870912+0000","last_unstale":"2023-09-23T20:51:09.826698+0000","last_undegraded":"2023-09-23T20:51:09.826698+0000","last_fullsized":"2023-09-23T20:51:09.826698+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:16:22.718106+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"79'22","reported_seq":64,"reported_epoch":98,"state":"active+clean","last_fresh":"2023-09-23T20:51:09.827615+0000","last_change":"2023-09-23T20:41:58.905866+0000","last_active":"2023-09-23T20:51:09.827615+0000","last_peered":"2023-09-23T20:51:09.827615+0000","last_clean":"2023-09-23T20:51:09.827615+0000","last_became_active":"2023-09-23T20:41:58.905556+0000","last_became_peered":"2023-09-23T20:41:58.905556+0000","last_unstale":"2023-09-23T20:51:09.827615+0000","last_undegraded":"2023-09-23T20:51:09.827615+0000","last_fullsized":"2023-09-23T20:51:09.827615+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:03:14.027348+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":47,"reported_epoch":98,"state":"active+clean","last_fresh":"2023-09-23T20:51:09.834463+0000","last_change":"2023-09-23T20:50:37.982103+0000","last_active":"2023-09-23T20:51:09.834463+0000","last_peered":"2023-09-23T20:51:09.834463+0000","last_clean":"2023-09-23T20:51:09.834463+0000","last_became_active":"2023-09-23T20:50:37.981791+0000","last_became_peered":"2023-09-23T20:50:37.981791+0000","last_unstale":"2023-09-23T20:51:09.834463+0000","last_undegraded":"2023-09-23T20:51:09.834463+0000","last_fullsized":"2023-09-23T20:51:09.834463+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:32:52.286186+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":62,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-09-23T20:51:10.834618+0000","last_change":"2023-09-23T20:51:10.834618+0000","last_active":"2023-09-23T20:51:10.834618+0000","last_peered":"2023-09-23T20:51:10.834618+0000","last_clean":"2023-09-23T20:51:10.834618+0000","last_became_active":"2023-09-23T20:51:10.834267+0000","last_became_peered":"2023-09-23T20:51:10.834267+0000","last_unstale":"2023-09-23T20:51:10.834618+0000","last_undegraded":"2023-09-23T20:51:10.834618+0000","last_fullsized":"2023-09-23T20:51:10.834618+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:38:59.356003+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":22,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-09-23T20:51:10.833790+0000","last_change":"2023-09-23T20:51:10.833790+0000","last_active":"2023-09-23T20:51:10.833790+0000","last_peered":"2023-09-23T20:51:10.833790+0000","last_clean":"2023-09-23T20:51:10.833790+0000","last_became_active":"2023-09-23T20:51:10.832804+0000","last_became_peered":"2023-09-23T20:51:10.832804+0000","last_unstale":"2023-09-23T20:51:10.833790+0000","last_undegraded":"2023-09-23T20:51:10.833790+0000","last_fullsized":"2023-09-23T20:51:10.833790+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:27:42.601674+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":59,"reported_epoch":98,"state":"active+clean","last_fresh":"2023-09-23T20:51:09.817393+0000","last_change":"2023-09-23T20:50:05.863737+0000","last_active":"2023-09-23T20:51:09.817393+0000","last_peered":"2023-09-23T20:51:09.817393+0000","last_clean":"2023-09-23T20:51:09.817393+0000","last_became_active":"2023-09-23T20:50:05.863466+0000","last_became_peered":"2023-09-23T20:50:05.863466+0000","last_unstale":"2023-09-23T20:51:09.817393+0000","last_undegraded":"2023-09-23T20:51:09.817393+0000","last_fullsized":"2023-09-23T20:51:09.817393+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:28:57.602682+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":61,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-09-23T20:51:10.833424+0000","last_change":"2023-09-23T20:51:10.833424+0000","last_active":"2023-09-23T20:51:10.833424+0000","last_peered":"2023-09-23T20:51:10.833424+0000","last_clean":"2023-09-23T20:51:10.833424+0000","last_became_active":"2023-09-23T20:51:10.832808+0000","last_became_peered":"2023-09-23T20:51:10.832808+0000","last_unstale":"2023-09-23T20:51:10.833424+0000","last_undegraded":"2023-09-23T20:51:10.833424+0000","last_fullsized":"2023-09-23T20:51:10.833424+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:32:42.819583+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":53,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-09-23T20:51:10.841551+0000","last_change":"2023-09-23T20:51:10.840587+0000","last_active":"2023-09-23T20:51:10.841551+0000","last_peered":"2023-09-23T20:51:10.841551+0000","last_clean":"2023-09-23T20:51:10.841551+0000","last_became_active":"2023-09-23T20:51:10.840242+0000","last_became_peered":"2023-09-23T20:51:10.840242+0000","last_unstale":"2023-09-23T20:51:10.841551+0000","last_undegraded":"2023-09-23T20:51:10.841551+0000","last_fullsized":"2023-09-23T20:51:10.841551+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:44:07.171314+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":58,"reported_epoch":98,"state":"active+clean","last_fresh":"2023-09-23T20:51:09.824327+0000","last_change":"2023-09-23T20:50:05.867622+0000","last_active":"2023-09-23T20:51:09.824327+0000","last_peered":"2023-09-23T20:51:09.824327+0000","last_clean":"2023-09-23T20:51:09.824327+0000","last_became_active":"2023-09-23T20:50:05.867311+0000","last_became_peered":"2023-09-23T20:50:05.867311+0000","last_unstale":"2023-09-23T20:51:09.824327+0000","last_undegraded":"2023-09-23T20:51:09.824327+0000","last_fullsized":"2023-09-23T20:51:09.824327+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:31:53.285316+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":79,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-09-23T20:51:10.833613+0000","last_change":"2023-09-23T20:51:10.833613+0000","last_active":"2023-09-23T20:51:10.833613+0000","last_peered":"2023-09-23T20:51:10.833613+0000","last_clean":"2023-09-23T20:51:10.833613+0000","last_became_active":"2023-09-23T20:51:10.833208+0000","last_became_peered":"2023-09-23T20:51:10.833208+0000","last_unstale":"2023-09-23T20:51:10.833613+0000","last_undegraded":"2023-09-23T20:51:10.833613+0000","last_fullsized":"2023-09-23T20:51:10.833613+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:51:28.246014+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":53,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-09-23T20:51:10.841782+0000","last_change":"2023-09-23T20:51:10.841420+0000","last_active":"2023-09-23T20:51:10.841782+0000","last_peered":"2023-09-23T20:51:10.841782+0000","last_clean":"2023-09-23T20:51:10.841782+0000","last_became_active":"2023-09-23T20:51:10.841112+0000","last_became_peered":"2023-09-23T20:51:10.841112+0000","last_unstale":"2023-09-23T20:51:10.841782+0000","last_undegraded":"2023-09-23T20:51:10.841782+0000","last_fullsized":"2023-09-23T20:51:10.841782+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:44:02.236650+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":47,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-09-23T20:51:10.833035+0000","last_change":"2023-09-23T20:51:10.833035+0000","last_active":"2023-09-23T20:51:10.833035+0000","last_peered":"2023-09-23T20:51:10.833035+0000","last_clean":"2023-09-23T20:51:10.833035+0000","last_became_active":"2023-09-23T20:51:10.832741+0000","last_became_peered":"2023-09-23T20:51:10.832741+0000","last_unstale":"2023-09-23T20:51:10.833035+0000","last_undegraded":"2023-09-23T20:51:10.833035+0000","last_fullsized":"2023-09-23T20:51:10.833035+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:53:16.471852+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":61,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-09-23T20:51:10.835170+0000","last_change":"2023-09-23T20:51:10.835170+0000","last_active":"2023-09-23T20:51:10.835170+0000","last_peered":"2023-09-23T20:51:10.835170+0000","last_clean":"2023-09-23T20:51:10.835170+0000","last_became_active":"2023-09-23T20:51:10.834808+0000","last_became_peered":"2023-09-23T20:51:10.834808+0000","last_unstale":"2023-09-23T20:51:10.835170+0000","last_undegraded":"2023-09-23T20:51:10.835170+0000","last_fullsized":"2023-09-23T20:51:10.835170+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:51:25.153742+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"68'1","reported_seq":77,"reported_epoch":98,"state":"active+clean","last_fresh":"2023-09-23T20:51:09.826718+0000","last_change":"2023-09-23T20:50:37.985580+0000","last_active":"2023-09-23T20:51:09.826718+0000","last_peered":"2023-09-23T20:51:09.826718+0000","last_clean":"2023-09-23T20:51:09.826718+0000","last_became_active":"2023-09-23T20:50:37.985229+0000","last_became_peered":"2023-09-23T20:50:37.985229+0000","last_unstale":"2023-09-23T20:51:09.826718+0000","last_undegraded":"2023-09-23T20:51:09.826718+0000","last_fullsized":"2023-09-23T20:51:09.826718+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:52:50.591507+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":161,"ondisk_log_size":161,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201417,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":22696,"kb_used_data":1568,"kb_used_omap":0,"kb_used_meta":21120,"kb_avail":93722456,"statfs":{"total":95995035648,"available":95971794944,"internally_reserved":0,"allocated":1605632,"data_stored":1128632,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":21626880},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:51:35 2023","interfaces":[{"interface":"back","average":{"1min":0.622,"5min":0.51700000000000002,"15min":0.499},"min":{"1min":0.45900000000000002,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":0.80400000000000005,"5min":0.80400000000000005,"15min":0.80400000000000005},"last":0.69399999999999995},{"interface":"front","average":{"1min":0.55200000000000005,"5min":0.56000000000000005,"15min":0.56200000000000006},"min":{"1min":0.33700000000000002,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":0.79000000000000004,"5min":0.79000000000000004,"15min":0.79000000000000004},"last":0.60499999999999998}]},{"osd":1,"last update":"Sat Sep 23 20:51:05 2023","interfaces":[{"interface":"back","average":{"1min":0.54900000000000004,"5min":0.54900000000000004,"15min":0.54900000000000004},"min":{"1min":0.40999999999999998,"5min":0.40999999999999998,"15min":0.40999999999999998},"max":{"1min":0.73099999999999998,"5min":0.73099999999999998,"15min":0.73099999999999998},"last":0.51400000000000001},{"interface":"front","average":{"1min":0.54400000000000004,"5min":0.54400000000000004,"15min":0.54400000000000004},"min":{"1min":0.32100000000000001,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":0.71599999999999997,"5min":0.71599999999999997,"15min":0.71599999999999997},"last":0.61699999999999999}]},{"osd":2,"last update":"Sat Sep 23 20:51:41 2023","interfaces":[{"interface":"back","average":{"1min":0.55100000000000005,"5min":0.55100000000000005,"15min":0.55100000000000005},"min":{"1min":0.33300000000000002,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":0.83099999999999996,"5min":0.83099999999999996,"15min":0.83099999999999996},"last":0.59099999999999997},{"interface":"front","average":{"1min":0.58399999999999996,"5min":0.58399999999999996,"15min":0.58399999999999996},"min":{"1min":0.375,"5min":0.375,"15min":0.375},"max":{"1min":0.81299999999999994,"5min":0.81299999999999994,"15min":0.81299999999999994},"last":0.755}]},{"osd":3,"last update":"Sat Sep 23 20:50:58 2023","interfaces":[{"interface":"back","average":{"1min":0.58199999999999996,"5min":0.58599999999999997,"15min":0.56499999999999995},"min":{"1min":0.40000000000000002,"5min":0.309,"15min":0.28000000000000003},"max":{"1min":0.755,"5min":0.83299999999999996,"15min":1.159},"last":0.66100000000000003},{"interface":"front","average":{"1min":0.60699999999999998,"5min":0.61099999999999999,"15min":0.58799999999999997},"min":{"1min":0.435,"5min":0.376,"15min":0.32900000000000001},"max":{"1min":0.77800000000000002,"5min":0.84099999999999997,"15min":1.5840000000000001},"last":0.67900000000000005}]},{"osd":4,"last update":"Sat Sep 23 20:50:58 2023","interfaces":[{"interface":"back","average":{"1min":0.55000000000000004,"5min":0.51100000000000001,"15min":0.48099999999999998},"min":{"1min":0.34999999999999998,"5min":0.312,"15min":0.254},"max":{"1min":0.77000000000000002,"5min":0.82099999999999995,"15min":0.82099999999999995},"last":0.55800000000000005},{"interface":"front","average":{"1min":0.56999999999999995,"5min":0.54100000000000004,"15min":0.52000000000000002},"min":{"1min":0.39200000000000002,"5min":0.35999999999999999,"15min":0.29599999999999999},"max":{"1min":0.80000000000000004,"5min":0.873,"15min":1.1100000000000001},"last":0.64000000000000001}]},{"osd":5,"last update":"Sat Sep 23 20:50:58 2023","interfaces":[{"interface":"back","average":{"1min":0.59199999999999997,"5min":0.55400000000000005,"15min":0.53100000000000003},"min":{"1min":0.42399999999999999,"5min":0.318,"15min":0.30199999999999999},"max":{"1min":0.78500000000000003,"5min":0.89900000000000002,"15min":1.4610000000000001},"last":0.70999999999999996},{"interface":"front","average":{"1min":0.63400000000000001,"5min":0.58199999999999996,"15min":0.55100000000000005},"min":{"1min":0.39500000000000002,"5min":0.34300000000000003,"15min":0.30599999999999999},"max":{"1min":0.84999999999999998,"5min":0.85899999999999999,"15min":0.85899999999999999},"last":0.65300000000000002}]},{"osd":6,"last update":"Sat Sep 23 20:50:58 2023","interfaces":[{"interface":"back","average":{"1min":0.623,"5min":0.60599999999999998,"15min":0.56899999999999995},"min":{"1min":0.433,"5min":0.41599999999999998,"15min":0.32600000000000001},"max":{"1min":0.88200000000000001,"5min":0.88300000000000001,"15min":0.88900000000000001},"last":0.72499999999999998},{"interface":"front","average":{"1min":0.64800000000000002,"5min":0.60599999999999998,"15min":0.58499999999999996},"min":{"1min":0.47599999999999998,"5min":0.36299999999999999,"15min":0.316},"max":{"1min":0.83999999999999997,"5min":0.90700000000000003,"15min":1.0760000000000001},"last":0.73199999999999998}]}]},{"osd":6,"up_from":47,"seq":201863463054,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":7832,"kb_used_data":1552,"kb_used_omap":0,"kb_used_meta":6272,"kb_avail":93737320,"statfs":{"total":95995035648,"available":95987015680,"internally_reserved":0,"allocated":1589248,"data_stored":1128221,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":6422528},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:51:35 2023","interfaces":[{"interface":"back","average":{"1min":0.50900000000000001,"5min":0.56499999999999995,"15min":0.57399999999999995},"min":{"1min":0.33400000000000002,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":0.75700000000000001,"5min":0.79200000000000004,"15min":0.79200000000000004},"last":0.49399999999999999},{"interface":"front","average":{"1min":0.54600000000000004,"5min":0.58099999999999996,"15min":0.58699999999999997},"min":{"1min":0.34200000000000003,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":0.68799999999999994,"5min":0.89100000000000001,"15min":0.89100000000000001},"last":0.51800000000000002}]},{"osd":1,"last update":"Sat Sep 23 20:51:06 2023","interfaces":[{"interface":"back","average":{"1min":0.53400000000000003,"5min":0.53400000000000003,"15min":0.53400000000000003},"min":{"1min":0.371,"5min":0.371,"15min":0.371},"max":{"1min":0.72499999999999998,"5min":0.72499999999999998,"15min":0.72499999999999998},"last":0.39900000000000002},{"interface":"front","average":{"1min":0.56999999999999995,"5min":0.56999999999999995,"15min":0.56999999999999995},"min":{"1min":0.38300000000000001,"5min":0.38300000000000001,"15min":0.38300000000000001},"max":{"1min":0.79700000000000004,"5min":0.79700000000000004,"15min":0.79700000000000004},"last":0.62}]},{"osd":2,"last update":"Sat Sep 23 20:51:41 2023","interfaces":[{"interface":"back","average":{"1min":0.54800000000000004,"5min":0.54800000000000004,"15min":0.54800000000000004},"min":{"1min":0.40899999999999997,"5min":0.40899999999999997,"15min":0.40899999999999997},"max":{"1min":0.71999999999999997,"5min":0.71999999999999997,"15min":0.71999999999999997},"last":0.54000000000000004},{"interface":"front","average":{"1min":0.58599999999999997,"5min":0.58599999999999997,"15min":0.58599999999999997},"min":{"1min":0.35799999999999998,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":0.73399999999999999,"5min":0.73399999999999999,"15min":0.73399999999999999},"last":0.56499999999999995}]},{"osd":3,"last update":"Sat Sep 23 20:50:38 2023","interfaces":[{"interface":"back","average":{"1min":0.61599999999999999,"5min":0.60199999999999998,"15min":0.56200000000000006},"min":{"1min":0.439,"5min":0.34699999999999998,"15min":0.33200000000000002},"max":{"1min":0.82699999999999996,"5min":0.92100000000000004,"15min":0.96099999999999997},"last":0.55300000000000005},{"interface":"front","average":{"1min":0.60699999999999998,"5min":0.59599999999999997,"15min":0.56100000000000005},"min":{"1min":0.42899999999999999,"5min":0.39900000000000002,"15min":0.34100000000000003},"max":{"1min":0.81999999999999995,"5min":0.83999999999999997,"15min":0.874},"last":0.42099999999999999}]},{"osd":4,"last update":"Sat Sep 23 20:51:41 2023","interfaces":[{"interface":"back","average":{"1min":0.503,"5min":0.47599999999999998,"15min":0.45700000000000002},"min":{"1min":0.374,"5min":0.28000000000000003,"15min":0.27800000000000002},"max":{"1min":0.68400000000000005,"5min":0.81000000000000005,"15min":1.1779999999999999},"last":0.44500000000000001},{"interface":"front","average":{"1min":0.54500000000000004,"5min":0.52300000000000002,"15min":0.49399999999999999},"min":{"1min":0.376,"5min":0.30599999999999999,"15min":0.29199999999999998},"max":{"1min":0.77000000000000002,"5min":0.81599999999999995,"15min":0.81599999999999995},"last":0.63400000000000001}]},{"osd":5,"last update":"Sat Sep 23 20:51:41 2023","interfaces":[{"interface":"back","average":{"1min":0.53600000000000003,"5min":0.51600000000000001,"15min":0.49199999999999999},"min":{"1min":0.39600000000000002,"5min":0.32600000000000001,"15min":0.28399999999999997},"max":{"1min":0.69999999999999996,"5min":0.82699999999999996,"15min":1.081},"last":0.60499999999999998},{"interface":"front","average":{"1min":0.59299999999999997,"5min":0.56000000000000005,"15min":0.52300000000000002},"min":{"1min":0.374,"5min":0.33100000000000002,"15min":0.28499999999999998},"max":{"1min":0.755,"5min":0.81000000000000005,"15min":0.89000000000000001},"last":0.58799999999999997}]},{"osd":7,"last update":"Sat Sep 23 20:51:02 2023","interfaces":[{"interface":"back","average":{"1min":0.58299999999999996,"5min":0.56899999999999995,"15min":0.53500000000000003},"min":{"1min":0.38600000000000001,"5min":0.33000000000000002,"15min":0.32500000000000001},"max":{"1min":0.78900000000000003,"5min":0.82299999999999995,"15min":1.119},"last":0.57899999999999996},{"interface":"front","average":{"1min":0.63600000000000001,"5min":0.61599999999999999,"15min":0.57899999999999996},"min":{"1min":0.42199999999999999,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":0.79200000000000004,"5min":0.94099999999999995,"15min":0.94099999999999995},"last":0.66500000000000004}]}]},{"osd":1,"up_from":88,"seq":377957122072,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28480,"kb_used_data":1016,"kb_used_omap":9,"kb_used_meta":27446,"kb_avail":93716672,"statfs":{"total":95995035648,"available":95965872128,"internally_reserved":0,"allocated":1040384,"data_stored":544044,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9239,"internal_metadata":28105705},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:51:04 2023","interfaces":[{"interface":"back","average":{"1min":0.40899999999999997,"5min":0.40899999999999997,"15min":0.40899999999999997},"min":{"1min":0.27300000000000002,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":0.63200000000000001,"5min":0.63200000000000001,"15min":0.63200000000000001},"last":0.307},{"interface":"front","average":{"1min":0.435,"5min":0.435,"15min":0.435},"min":{"1min":0.31,"5min":0.31,"15min":0.31},"max":{"1min":0.73999999999999999,"5min":0.73999999999999999,"15min":0.73999999999999999},"last":0.32100000000000001}]},{"osd":2,"last update":"Sat Sep 23 20:51:37 2023","interfaces":[{"interface":"back","average":{"1min":0.38700000000000001,"5min":0.38700000000000001,"15min":0.38700000000000001},"min":{"1min":0.26400000000000001,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":0.503,"5min":0.503,"15min":0.503},"last":0.26500000000000001},{"interface":"front","average":{"1min":0.58499999999999996,"5min":0.58499999999999996,"15min":0.58499999999999996},"min":{"1min":0.29099999999999998,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":3.5219999999999998,"5min":3.5219999999999998,"15min":3.5219999999999998},"last":0.36399999999999999}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.33800000000000002}]},{"osd":4,"last update":"Sat Sep 23 20:51:04 2023","interfaces":[{"interface":"back","average":{"1min":0.64700000000000002,"5min":0.64700000000000002,"15min":0.64700000000000002},"min":{"1min":0.435,"5min":0.435,"15min":0.435},"max":{"1min":0.88200000000000001,"5min":0.88200000000000001,"15min":0.88200000000000001},"last":0.56000000000000005},{"interface":"front","average":{"1min":0.67300000000000004,"5min":0.67300000000000004,"15min":0.67300000000000004},"min":{"1min":0.44400000000000001,"5min":0.44400000000000001,"15min":0.44400000000000001},"max":{"1min":0.92900000000000005,"5min":0.92900000000000005,"15min":0.92900000000000005},"last":0.442}]},{"osd":5,"last update":"Sat Sep 23 20:51:04 2023","interfaces":[{"interface":"back","average":{"1min":0.69799999999999995,"5min":0.69799999999999995,"15min":0.69799999999999995},"min":{"1min":0.39500000000000002,"5min":0.39500000000000002,"15min":0.39500000000000002},"max":{"1min":0.86899999999999999,"5min":0.86899999999999999,"15min":0.86899999999999999},"last":0.42999999999999999},{"interface":"front","average":{"1min":0.64700000000000002,"5min":0.64700000000000002,"15min":0.64700000000000002},"min":{"1min":0.47899999999999998,"5min":0.47899999999999998,"15min":0.47899999999999998},"max":{"1min":0.79200000000000004,"5min":0.79200000000000004,"15min":0.79200000000000004},"last":0.54000000000000004}]},{"osd":6,"last update":"Sat Sep 23 20:51:04 2023","interfaces":[{"interface":"back","average":{"1min":0.68799999999999994,"5min":0.68799999999999994,"15min":0.68799999999999994},"min":{"1min":0.47999999999999998,"5min":0.47999999999999998,"15min":0.47999999999999998},"max":{"1min":0.98799999999999999,"5min":0.98799999999999999,"15min":0.98799999999999999},"last":0.47199999999999998},{"interface":"front","average":{"1min":0.65800000000000003,"5min":0.65800000000000003,"15min":0.65800000000000003},"min":{"1min":0.41799999999999998,"5min":0.41799999999999998,"15min":0.41799999999999998},"max":{"1min":0.90900000000000003,"5min":0.90900000000000003,"15min":0.90900000000000003},"last":0.48399999999999999}]},{"osd":7,"last update":"Sat Sep 23 20:51:04 2023","interfaces":[{"interface":"back","average":{"1min":0.67400000000000004,"5min":0.67400000000000004,"15min":0.67400000000000004},"min":{"1min":0.46999999999999997,"5min":0.46999999999999997,"15min":0.46999999999999997},"max":{"1min":1.004,"5min":1.004,"15min":1.004},"last":0.42199999999999999},{"interface":"front","average":{"1min":0.68600000000000005,"5min":0.68600000000000005,"15min":0.68600000000000005},"min":{"1min":0.502,"5min":0.502,"15min":0.502},"max":{"1min":1.036,"5min":1.036,"15min":1.036},"last":0.51100000000000001}]}]},{"osd":0,"up_from":83,"seq":356482285599,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":29140,"kb_used_data":1612,"kb_used_omap":28,"kb_used_meta":27491,"kb_avail":93716012,"statfs":{"total":95995035648,"available":95965196288,"internally_reserved":0,"allocated":1650688,"data_stored":1136306,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":28848,"internal_metadata":28151632},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 20:51:07 2023","interfaces":[{"interface":"back","average":{"1min":0.33300000000000002,"5min":0.33300000000000002,"15min":0.33300000000000002},"min":{"1min":0.23400000000000001,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":0.503,"5min":0.503,"15min":0.503},"last":0.27200000000000002},{"interface":"front","average":{"1min":0.38700000000000001,"5min":0.38700000000000001,"15min":0.38700000000000001},"min":{"1min":0.26100000000000001,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":0.54100000000000004,"5min":0.54100000000000004,"15min":0.54100000000000004},"last":0.28999999999999998}]},{"osd":2,"last update":"Sat Sep 23 20:51:37 2023","interfaces":[{"interface":"back","average":{"1min":0.41499999999999998,"5min":0.41499999999999998,"15min":0.41499999999999998},"min":{"1min":0.248,"5min":0.248,"15min":0.248},"max":{"1min":0.61799999999999999,"5min":0.61799999999999999,"15min":0.61799999999999999},"last":0.33700000000000002},{"interface":"front","average":{"1min":0.39000000000000001,"5min":0.39000000000000001,"15min":0.39000000000000001},"min":{"1min":0.25700000000000001,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":0.55900000000000005,"5min":0.55900000000000005,"15min":0.55900000000000005},"last":0.34699999999999998}]},{"osd":3,"last update":"Sat Sep 23 20:50:33 2023","interfaces":[{"interface":"back","average":{"1min":0.46000000000000002,"5min":0.46000000000000002,"15min":0.46000000000000002},"min":{"1min":0.28899999999999998,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":0.72699999999999998,"5min":0.72699999999999998,"15min":0.72699999999999998},"last":0.308},{"interface":"front","average":{"1min":0.45500000000000002,"5min":0.45500000000000002,"15min":0.45500000000000002},"min":{"1min":0.33100000000000002,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":0.64400000000000002,"5min":0.64400000000000002,"15min":0.64400000000000002},"last":0.36099999999999999}]},{"osd":4,"last update":"Sat Sep 23 20:51:37 2023","interfaces":[{"interface":"back","average":{"1min":0.64800000000000002,"5min":0.64800000000000002,"15min":0.64800000000000002},"min":{"1min":0.377,"5min":0.377,"15min":0.377},"max":{"1min":1.9870000000000001,"5min":1.9870000000000001,"15min":1.9870000000000001},"last":0.45300000000000001},{"interface":"front","average":{"1min":0.68899999999999995,"5min":0.66500000000000004,"15min":0.66100000000000003},"min":{"1min":0.41399999999999998,"5min":0.41399999999999998,"15min":0.41399999999999998},"max":{"1min":2.0049999999999999,"5min":2.0049999999999999,"15min":2.0049999999999999},"last":0.51900000000000002}]},{"osd":5,"last update":"Sat Sep 23 20:51:37 2023","interfaces":[{"interface":"back","average":{"1min":0.60099999999999998,"5min":0.63900000000000001,"15min":0.64500000000000002},"min":{"1min":0.42299999999999999,"5min":0.42299999999999999,"15min":0.42299999999999999},"max":{"1min":0.80400000000000005,"5min":0.83799999999999997,"15min":0.83799999999999997},"last":0.437},{"interface":"front","average":{"1min":0.60799999999999998,"5min":0.68700000000000006,"15min":0.69999999999999996},"min":{"1min":0.42499999999999999,"5min":0.42499999999999999,"15min":0.42499999999999999},"max":{"1min":0.88400000000000001,"5min":0.90000000000000002,"15min":0.90000000000000002},"last":0.42699999999999999}]},{"osd":6,"last update":"Sat Sep 23 20:51:37 2023","interfaces":[{"interface":"back","average":{"1min":0.60799999999999998,"5min":0.67400000000000004,"15min":0.68500000000000005},"min":{"1min":0.433,"5min":0.433,"15min":0.433},"max":{"1min":0.876,"5min":0.93999999999999995,"15min":0.93999999999999995},"last":0.5},{"interface":"front","average":{"1min":0.72099999999999997,"5min":0.68899999999999995,"15min":0.68300000000000005},"min":{"1min":0.45000000000000001,"5min":0.44600000000000001,"15min":0.44600000000000001},"max":{"1min":2.0489999999999999,"5min":2.0489999999999999,"15min":2.0489999999999999},"last":0.41399999999999998}]},{"osd":7,"last update":"Sat Sep 23 20:51:37 2023","interfaces":[{"interface":"back","average":{"1min":0.64000000000000001,"5min":0.68799999999999994,"15min":0.69599999999999995},"min":{"1min":0.42799999999999999,"5min":0.42799999999999999,"15min":0.42799999999999999},"max":{"1min":0.82399999999999995,"5min":0.92000000000000004,"15min":0.92000000000000004},"last":0.45800000000000002},{"interface":"front","average":{"1min":0.63900000000000001,"5min":0.66700000000000004,"15min":0.67100000000000004},"min":{"1min":0.442,"5min":0.442,"15min":0.442},"max":{"1min":0.85799999999999998,"5min":0.90600000000000003,"15min":0.90600000000000003},"last":0.48699999999999999}]}]},{"osd":2,"up_from":93,"seq":399431958546,"num_pgs":19,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28600,"kb_used_data":1008,"kb_used_omap":81,"kb_used_meta":27502,"kb_avail":93716552,"statfs":{"total":95995035648,"available":95965749248,"internally_reserved":0,"allocated":1032192,"data_stored":541255,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":83750,"internal_metadata":28162266},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:51:38 2023","interfaces":[{"interface":"back","average":{"1min":0.35799999999999998,"5min":0.35799999999999998,"15min":0.35799999999999998},"min":{"1min":0.249,"5min":0.249,"15min":0.249},"max":{"1min":0.54500000000000004,"5min":0.54500000000000004,"15min":0.54500000000000004},"last":0.20499999999999999},{"interface":"front","average":{"1min":0.40699999999999997,"5min":0.40699999999999997,"15min":0.40699999999999997},"min":{"1min":0.309,"5min":0.309,"15min":0.309},"max":{"1min":0.64800000000000002,"5min":0.64800000000000002,"15min":0.64800000000000002},"last":0.27000000000000002}]},{"osd":1,"last update":"Sat Sep 23 20:51:38 2023","interfaces":[{"interface":"back","average":{"1min":0.41699999999999998,"5min":0.41699999999999998,"15min":0.41699999999999998},"min":{"1min":0.31,"5min":0.31,"15min":0.31},"max":{"1min":0.61699999999999999,"5min":0.61699999999999999,"15min":0.61699999999999999},"last":0.28999999999999998},{"interface":"front","average":{"1min":0.42099999999999999,"5min":0.42099999999999999,"15min":0.42099999999999999},"min":{"1min":0.33200000000000002,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":0.55600000000000005,"5min":0.55600000000000005,"15min":0.55600000000000005},"last":0.33400000000000002}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.313}]},{"osd":4,"last update":"Sat Sep 23 20:51:38 2023","interfaces":[{"interface":"back","average":{"1min":0.61899999999999999,"5min":0.61899999999999999,"15min":0.61899999999999999},"min":{"1min":0.45200000000000001,"5min":0.45200000000000001,"15min":0.45200000000000001},"max":{"1min":0.86099999999999999,"5min":0.86099999999999999,"15min":0.86099999999999999},"last":0.65000000000000002},{"interface":"front","average":{"1min":0.67100000000000004,"5min":0.67100000000000004,"15min":0.67100000000000004},"min":{"1min":0.47299999999999998,"5min":0.47299999999999998,"15min":0.47299999999999998},"max":{"1min":0.92400000000000004,"5min":0.92400000000000004,"15min":0.92400000000000004},"last":0.55800000000000005}]},{"osd":5,"last update":"Sat Sep 23 20:51:38 2023","interfaces":[{"interface":"back","average":{"1min":0.65200000000000002,"5min":0.65200000000000002,"15min":0.65200000000000002},"min":{"1min":0.46300000000000002,"5min":0.46300000000000002,"15min":0.46300000000000002},"max":{"1min":0.95999999999999996,"5min":0.95999999999999996,"15min":0.95999999999999996},"last":0.67800000000000005},{"interface":"front","average":{"1min":0.63600000000000001,"5min":0.63600000000000001,"15min":0.63600000000000001},"min":{"1min":0.42299999999999999,"5min":0.42299999999999999,"15min":0.42299999999999999},"max":{"1min":0.89600000000000002,"5min":0.89600000000000002,"15min":0.89600000000000002},"last":0.54200000000000004}]},{"osd":6,"last update":"Sat Sep 23 20:51:38 2023","interfaces":[{"interface":"back","average":{"1min":0.70599999999999996,"5min":0.70599999999999996,"15min":0.70599999999999996},"min":{"1min":0.502,"5min":0.502,"15min":0.502},"max":{"1min":0.97299999999999998,"5min":0.97299999999999998,"15min":0.97299999999999998},"last":0.51800000000000002},{"interface":"front","average":{"1min":0.68600000000000005,"5min":0.68600000000000005,"15min":0.68600000000000005},"min":{"1min":0.49399999999999999,"5min":0.49399999999999999,"15min":0.49399999999999999},"max":{"1min":0.93999999999999995,"5min":0.93999999999999995,"15min":0.93999999999999995},"last":0.628}]},{"osd":7,"last update":"Sat Sep 23 20:51:38 2023","interfaces":[{"interface":"back","average":{"1min":0.63500000000000001,"5min":0.63500000000000001,"15min":0.63500000000000001},"min":{"1min":0.498,"5min":0.498,"15min":0.498},"max":{"1min":0.90600000000000003,"5min":0.90600000000000003,"15min":0.90600000000000003},"last":0.56699999999999995},{"interface":"front","average":{"1min":0.66200000000000003,"5min":0.66200000000000003,"15min":0.66200000000000003},"min":{"1min":0.433,"5min":0.433,"15min":0.433},"max":{"1min":0.90500000000000003,"5min":0.90500000000000003,"15min":0.90500000000000003},"last":0.69399999999999995}]}]},{"osd":3,"up_from":98,"seq":420906795019,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28452,"kb_used_data":988,"kb_used_omap":7,"kb_used_meta":27448,"kb_avail":93716700,"statfs":{"total":95995035648,"available":95965900800,"internally_reserved":0,"allocated":1011712,"data_stored":538170,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7576,"internal_metadata":28107368},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.253}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.32500000000000001}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.40200000000000002}]},{"osd":4,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.76300000000000001}]},{"osd":5,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.623}]},{"osd":6,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.86099999999999999}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.65900000000000003}]}]},{"osd":4,"up_from":31,"seq":133143986326,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":11876,"kb_used_data":988,"kb_used_omap":0,"kb_used_meta":10880,"kb_avail":93733276,"statfs":{"total":95995035648,"available":95982874624,"internally_reserved":0,"allocated":1011712,"data_stored":538517,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":11141120},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:51:35 2023","interfaces":[{"interface":"back","average":{"1min":0.745,"5min":0.65600000000000003,"15min":0.64100000000000001},"min":{"1min":0.40000000000000002,"5min":0.38900000000000001,"15min":0.38900000000000001},"max":{"1min":4.4050000000000002,"5min":4.4050000000000002,"15min":4.4050000000000002},"last":0.47299999999999998},{"interface":"front","average":{"1min":0.748,"5min":0.88200000000000001,"15min":0.90400000000000003},"min":{"1min":0.46899999999999997,"5min":0.46899999999999997,"15min":0.46899999999999997},"max":{"1min":4.468,"5min":5.2409999999999997,"15min":5.2409999999999997},"last":0.53100000000000003}]},{"osd":1,"last update":"Sat Sep 23 20:51:06 2023","interfaces":[{"interface":"back","average":{"1min":0.55900000000000005,"5min":0.55900000000000005,"15min":0.55900000000000005},"min":{"1min":0.44700000000000001,"5min":0.44700000000000001,"15min":0.44700000000000001},"max":{"1min":0.79900000000000004,"5min":0.79900000000000004,"15min":0.79900000000000004},"last":0.44800000000000001},{"interface":"front","average":{"1min":0.59899999999999998,"5min":0.59899999999999998,"15min":0.59899999999999998},"min":{"1min":0.44500000000000001,"5min":0.44500000000000001,"15min":0.44500000000000001},"max":{"1min":0.72399999999999998,"5min":0.72399999999999998,"15min":0.72399999999999998},"last":0.621}]},{"osd":2,"last update":"Sat Sep 23 20:51:41 2023","interfaces":[{"interface":"back","average":{"1min":0.68600000000000005,"5min":0.68600000000000005,"15min":0.68600000000000005},"min":{"1min":0.442,"5min":0.442,"15min":0.442},"max":{"1min":2.5030000000000001,"5min":2.5030000000000001,"15min":2.5030000000000001},"last":0.51900000000000002},{"interface":"front","average":{"1min":0.77400000000000002,"5min":0.77400000000000002,"15min":0.77400000000000002},"min":{"1min":0.38200000000000001,"5min":0.38200000000000001,"15min":0.38200000000000001},"max":{"1min":4.3289999999999997,"5min":4.3289999999999997,"15min":4.3289999999999997},"last":0.64300000000000002}]},{"osd":3,"last update":"Sat Sep 23 20:50:04 2023","interfaces":[{"interface":"back","average":{"1min":0.63300000000000001,"5min":0.59999999999999998,"15min":0.56799999999999995},"min":{"1min":0.41799999999999998,"5min":0.35299999999999998,"15min":0.33100000000000002},"max":{"1min":2.1320000000000001,"5min":2.1320000000000001,"15min":2.1320000000000001},"last":0.54700000000000004},{"interface":"front","average":{"1min":0.53500000000000003,"5min":0.57799999999999996,"15min":0.55900000000000005},"min":{"1min":0.35699999999999998,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":0.83499999999999996,"5min":0.88,"15min":0.91500000000000004},"last":0.66500000000000004}]},{"osd":5,"last update":"Sat Sep 23 20:51:17 2023","interfaces":[{"interface":"back","average":{"1min":0.57399999999999995,"5min":0.502,"15min":0.45100000000000001},"min":{"1min":0.40000000000000002,"5min":0.29099999999999998,"15min":0.251},"max":{"1min":0.80900000000000005,"5min":0.80900000000000005,"15min":0.83599999999999997},"last":0.59699999999999998},{"interface":"front","average":{"1min":0.54400000000000004,"5min":0.51100000000000001,"15min":0.46000000000000002},"min":{"1min":0.371,"5min":0.33400000000000002,"15min":0.28799999999999998},"max":{"1min":0.79500000000000004,"5min":0.79500000000000004,"15min":0.90000000000000002},"last":0.49399999999999999}]},{"osd":6,"last update":"Sat Sep 23 20:50:49 2023","interfaces":[{"interface":"back","average":{"1min":0.52600000000000002,"5min":0.51200000000000001,"15min":0.51100000000000001},"min":{"1min":0.36399999999999999,"5min":0.30299999999999999,"15min":0.28100000000000003},"max":{"1min":0.67000000000000004,"5min":0.748,"15min":1.1319999999999999},"last":0.50800000000000001},{"interface":"front","average":{"1min":0.59499999999999997,"5min":0.56799999999999995,"15min":0.56399999999999995},"min":{"1min":0.42999999999999999,"5min":0.316,"15min":0.29399999999999998},"max":{"1min":0.79200000000000004,"5min":0.82699999999999996,"15min":1.095},"last":0.56200000000000006}]},{"osd":7,"last update":"Sat Sep 23 20:51:06 2023","interfaces":[{"interface":"back","average":{"1min":0.63500000000000001,"5min":0.59299999999999997,"15min":0.56599999999999995},"min":{"1min":0.44700000000000001,"5min":0.35099999999999998,"15min":0.32500000000000001},"max":{"1min":0.77100000000000002,"5min":0.79100000000000004,"15min":0.877},"last":0.68700000000000006},{"interface":"front","average":{"1min":0.62,"5min":0.58599999999999997,"15min":0.56499999999999995},"min":{"1min":0.42499999999999999,"5min":0.33500000000000002,"15min":0.32700000000000001},"max":{"1min":0.79600000000000004,"5min":0.79600000000000004,"15min":0.86299999999999999},"last":0.42599999999999999}]}]},{"osd":5,"up_from":39,"seq":167503724691,"num_pgs":21,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":17940,"kb_used_data":972,"kb_used_omap":0,"kb_used_meta":16960,"kb_avail":93727212,"statfs":{"total":95995035648,"available":95976665088,"internally_reserved":0,"allocated":995328,"data_stored":534261,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":17367040},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:51:34 2023","interfaces":[{"interface":"back","average":{"1min":0.51300000000000001,"5min":0.499,"15min":0.497},"min":{"1min":0.37,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":0.745,"5min":0.745,"15min":0.745},"last":1.04},{"interface":"front","average":{"1min":0.57399999999999995,"5min":0.57499999999999996,"15min":0.57499999999999996},"min":{"1min":0.35399999999999998,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":0.88400000000000001,"5min":0.88400000000000001,"15min":0.88400000000000001},"last":1.022}]},{"osd":1,"last update":"Sat Sep 23 20:51:05 2023","interfaces":[{"interface":"back","average":{"1min":0.56599999999999995,"5min":0.56599999999999995,"15min":0.56599999999999995},"min":{"1min":0.42499999999999999,"5min":0.42499999999999999,"15min":0.42499999999999999},"max":{"1min":0.83899999999999997,"5min":0.83899999999999997,"15min":0.83899999999999997},"last":1.0669999999999999},{"interface":"front","average":{"1min":0.54900000000000004,"5min":0.54900000000000004,"15min":0.54900000000000004},"min":{"1min":0.39000000000000001,"5min":0.39000000000000001,"15min":0.39000000000000001},"max":{"1min":0.70899999999999996,"5min":0.70899999999999996,"15min":0.70899999999999996},"last":0.95199999999999996}]},{"osd":2,"last update":"Sat Sep 23 20:51:39 2023","interfaces":[{"interface":"back","average":{"1min":0.54200000000000004,"5min":0.54200000000000004,"15min":0.54200000000000004},"min":{"1min":0.436,"5min":0.436,"15min":0.436},"max":{"1min":0.68500000000000005,"5min":0.68500000000000005,"15min":0.68500000000000005},"last":1.1379999999999999},{"interface":"front","average":{"1min":0.58399999999999996,"5min":0.58399999999999996,"15min":0.58399999999999996},"min":{"1min":0.40200000000000002,"5min":0.40200000000000002,"15min":0.40200000000000002},"max":{"1min":0.81699999999999995,"5min":0.81699999999999995,"15min":0.81699999999999995},"last":0.997}]},{"osd":3,"last update":"Sat Sep 23 20:50:16 2023","interfaces":[{"interface":"back","average":{"1min":0.61099999999999999,"5min":0.59199999999999997,"15min":0.56599999999999995},"min":{"1min":0.45600000000000002,"5min":0.39000000000000001,"15min":0.29399999999999998},"max":{"1min":0.79300000000000004,"5min":0.81999999999999995,"15min":0.96099999999999997},"last":1.087},{"interface":"front","average":{"1min":0.59099999999999997,"5min":0.61299999999999999,"15min":0.58399999999999996},"min":{"1min":0.38,"5min":0.37,"15min":0.33100000000000002},"max":{"1min":0.78500000000000003,"5min":0.90400000000000003,"15min":0.97699999999999998},"last":1.105}]},{"osd":4,"last update":"Sat Sep 23 20:51:17 2023","interfaces":[{"interface":"back","average":{"1min":0.53000000000000003,"5min":0.50900000000000001,"15min":0.45900000000000002},"min":{"1min":0.32700000000000001,"5min":0.317,"15min":0.253},"max":{"1min":0.75900000000000001,"5min":0.80900000000000005,"15min":0.80900000000000005},"last":0.43099999999999999},{"interface":"front","average":{"1min":0.54400000000000004,"5min":0.53500000000000003,"15min":0.496},"min":{"1min":0.28499999999999998,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":0.83499999999999996,"5min":0.83499999999999996,"15min":0.89500000000000002},"last":0.36699999999999999}]},{"osd":6,"last update":"Sat Sep 23 20:51:44 2023","interfaces":[{"interface":"back","average":{"1min":0.57399999999999995,"5min":0.54800000000000004,"15min":0.51100000000000001},"min":{"1min":0.34599999999999997,"5min":0.309,"15min":0.28599999999999998},"max":{"1min":0.86799999999999999,"5min":0.86899999999999999,"15min":0.91400000000000003},"last":0.39100000000000001},{"interface":"front","average":{"1min":0.63700000000000001,"5min":0.57799999999999996,"15min":0.54200000000000004},"min":{"1min":0.36299999999999999,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":0.85399999999999998,"5min":0.85399999999999998,"15min":0.85399999999999998},"last":0.52400000000000002}]},{"osd":7,"last update":"Sat Sep 23 20:51:02 2023","interfaces":[{"interface":"back","average":{"1min":0.57299999999999995,"5min":0.58199999999999996,"15min":0.56100000000000005},"min":{"1min":0.38,"5min":0.33600000000000002,"15min":0.314},"max":{"1min":0.77900000000000003,"5min":0.873,"15min":0.874},"last":0.41999999999999998},{"interface":"front","average":{"1min":0.624,"5min":0.61699999999999999,"15min":0.57899999999999996},"min":{"1min":0.41399999999999998,"5min":0.36099999999999999,"15min":0.34100000000000003},"max":{"1min":0.80100000000000005,"5min":0.88500000000000001,"15min":0.88500000000000001},"last":0.61899999999999999}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T20:51:58.468 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T20:51:59.113 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:51:59.882 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:51:59 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[164029]: ::ffff:172.21.15.70 - - [23/Sep/2023:20:51:59] "GET /metrics HTTP/1.1" 200 33816 "" "Prometheus/2.43.0" 2023-09-23T20:51:59.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:51:59 smithi026 ceph-mon[175151]: from='client.34722 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:51:59.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:51:59 smithi026 ceph-mon[175151]: pgmap v224: 73 pgs: 73 active+clean; 582 KiB data, 171 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:51:59.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:51:59 smithi026 ceph-mon[179688]: from='client.34722 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:51:59.883 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:51:59 smithi026 ceph-mon[179688]: pgmap v224: 73 pgs: 73 active+clean; 582 KiB data, 171 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:52:00.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:51:59 smithi070 ceph-mon[168282]: from='client.34722 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:52:00.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:51:59 smithi070 ceph-mon[168282]: pgmap v224: 73 pgs: 73 active+clean; 582 KiB data, 171 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:52:00.538 INFO:teuthology.orchestra.run.smithi026.stdout: 2023-09-23T20:52:00.539 INFO:teuthology.orchestra.run.smithi026.stderr:dumped all 2023-09-23T20:52:01.073 INFO:teuthology.orchestra.run.smithi026.stdout:{"pg_ready":true,"pg_map":{"version":225,"stamp":"2023-09-23T20:52:00.159963+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":595872,"num_objects":24,"num_object_clones":0,"num_object_copies":72,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":24,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":300,"num_write_kb":2445,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":216,"ondisk_log_size":216,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":175016,"kb_used_data":9704,"kb_used_omap":126,"kb_used_meta":165121,"kb_avail":749786200,"statfs":{"total":767960285184,"available":767781068800,"internally_reserved":0,"allocated":9936896,"data_stored":6089406,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":129413,"internal_metadata":169084539},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002765"},"pg_stats":[{"pgid":"4.18","version":"0'0","reported_seq":68,"reported_epoch":98,"state":"active+clean","last_fresh":"2023-09-23T20:51:09.825543+0000","last_change":"2023-09-23T20:50:37.984570+0000","last_active":"2023-09-23T20:51:09.825543+0000","last_peered":"2023-09-23T20:51:09.825543+0000","last_clean":"2023-09-23T20:51:09.825543+0000","last_became_active":"2023-09-23T20:50:37.984262+0000","last_became_peered":"2023-09-23T20:50:37.984262+0000","last_unstale":"2023-09-23T20:51:09.825543+0000","last_undegraded":"2023-09-23T20:51:09.825543+0000","last_fullsized":"2023-09-23T20:51:09.825543+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:47:39.024252+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":56,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-09-23T20:51:10.829873+0000","last_change":"2023-09-23T20:50:37.982897+0000","last_active":"2023-09-23T20:51:10.829873+0000","last_peered":"2023-09-23T20:51:10.829873+0000","last_clean":"2023-09-23T20:51:10.829873+0000","last_became_active":"2023-09-23T20:50:37.982566+0000","last_became_peered":"2023-09-23T20:50:37.982566+0000","last_unstale":"2023-09-23T20:51:10.829873+0000","last_undegraded":"2023-09-23T20:51:10.829873+0000","last_fullsized":"2023-09-23T20:51:10.829873+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:00:00.068989+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":82,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-09-23T20:51:10.838334+0000","last_change":"2023-09-23T20:51:10.838334+0000","last_active":"2023-09-23T20:51:10.838334+0000","last_peered":"2023-09-23T20:51:10.838334+0000","last_clean":"2023-09-23T20:51:10.838334+0000","last_became_active":"2023-09-23T20:51:10.838026+0000","last_became_peered":"2023-09-23T20:51:10.838026+0000","last_unstale":"2023-09-23T20:51:10.838334+0000","last_undegraded":"2023-09-23T20:51:10.838334+0000","last_fullsized":"2023-09-23T20:51:10.838334+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:04:37.009457+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":67,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-09-23T20:51:10.833849+0000","last_change":"2023-09-23T20:51:10.833849+0000","last_active":"2023-09-23T20:51:10.833849+0000","last_peered":"2023-09-23T20:51:10.833849+0000","last_clean":"2023-09-23T20:51:10.833849+0000","last_became_active":"2023-09-23T20:51:10.833524+0000","last_became_peered":"2023-09-23T20:51:10.833524+0000","last_unstale":"2023-09-23T20:51:10.833849+0000","last_undegraded":"2023-09-23T20:51:10.833849+0000","last_fullsized":"2023-09-23T20:51:10.833849+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:23:12.501924+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":75,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-09-23T20:51:10.835094+0000","last_change":"2023-09-23T20:51:10.835094+0000","last_active":"2023-09-23T20:51:10.835094+0000","last_peered":"2023-09-23T20:51:10.835094+0000","last_clean":"2023-09-23T20:51:10.835094+0000","last_became_active":"2023-09-23T20:51:10.834732+0000","last_became_peered":"2023-09-23T20:51:10.834732+0000","last_unstale":"2023-09-23T20:51:10.835094+0000","last_undegraded":"2023-09-23T20:51:10.835094+0000","last_fullsized":"2023-09-23T20:51:10.835094+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:21:08.414737+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":44,"reported_epoch":98,"state":"active+clean","last_fresh":"2023-09-23T20:51:09.819115+0000","last_change":"2023-09-23T20:41:58.906744+0000","last_active":"2023-09-23T20:51:09.819115+0000","last_peered":"2023-09-23T20:51:09.819115+0000","last_clean":"2023-09-23T20:51:09.819115+0000","last_became_active":"2023-09-23T20:41:58.906409+0000","last_became_peered":"2023-09-23T20:41:58.906409+0000","last_unstale":"2023-09-23T20:51:09.819115+0000","last_undegraded":"2023-09-23T20:51:09.819115+0000","last_fullsized":"2023-09-23T20:51:09.819115+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:15:42.250523+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":75,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-09-23T20:51:10.833684+0000","last_change":"2023-09-23T20:51:10.833684+0000","last_active":"2023-09-23T20:51:10.833684+0000","last_peered":"2023-09-23T20:51:10.833684+0000","last_clean":"2023-09-23T20:51:10.833684+0000","last_became_active":"2023-09-23T20:51:10.833360+0000","last_became_peered":"2023-09-23T20:51:10.833360+0000","last_unstale":"2023-09-23T20:51:10.833684+0000","last_undegraded":"2023-09-23T20:51:10.833684+0000","last_fullsized":"2023-09-23T20:51:10.833684+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:11:12.175158+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":61,"reported_epoch":98,"state":"active+clean","last_fresh":"2023-09-23T20:51:09.818991+0000","last_change":"2023-09-23T20:50:05.866461+0000","last_active":"2023-09-23T20:51:09.818991+0000","last_peered":"2023-09-23T20:51:09.818991+0000","last_clean":"2023-09-23T20:51:09.818991+0000","last_became_active":"2023-09-23T20:50:05.866053+0000","last_became_peered":"2023-09-23T20:50:05.866053+0000","last_unstale":"2023-09-23T20:51:09.818991+0000","last_undegraded":"2023-09-23T20:51:09.818991+0000","last_fullsized":"2023-09-23T20:51:09.818991+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:29:36.203264+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":55,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-09-23T20:51:10.835609+0000","last_change":"2023-09-23T20:51:10.835609+0000","last_active":"2023-09-23T20:51:10.835609+0000","last_peered":"2023-09-23T20:51:10.835609+0000","last_clean":"2023-09-23T20:51:10.835609+0000","last_became_active":"2023-09-23T20:51:10.835302+0000","last_became_peered":"2023-09-23T20:51:10.835302+0000","last_unstale":"2023-09-23T20:51:10.835609+0000","last_undegraded":"2023-09-23T20:51:10.835609+0000","last_fullsized":"2023-09-23T20:51:10.835609+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:07:20.599461+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":40,"reported_epoch":98,"state":"active+clean","last_fresh":"2023-09-23T20:51:09.823784+0000","last_change":"2023-09-23T20:49:32.775562+0000","last_active":"2023-09-23T20:51:09.823784+0000","last_peered":"2023-09-23T20:51:09.823784+0000","last_clean":"2023-09-23T20:51:09.823784+0000","last_became_active":"2023-09-23T20:49:32.775094+0000","last_became_peered":"2023-09-23T20:49:32.775094+0000","last_unstale":"2023-09-23T20:51:09.823784+0000","last_undegraded":"2023-09-23T20:51:09.823784+0000","last_fullsized":"2023-09-23T20:51:09.823784+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:17:52.944043+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":41,"reported_epoch":98,"state":"active+clean","last_fresh":"2023-09-23T20:51:09.826361+0000","last_change":"2023-09-23T20:41:58.918255+0000","last_active":"2023-09-23T20:51:09.826361+0000","last_peered":"2023-09-23T20:51:09.826361+0000","last_clean":"2023-09-23T20:51:09.826361+0000","last_became_active":"2023-09-23T20:41:58.917956+0000","last_became_peered":"2023-09-23T20:41:58.917956+0000","last_unstale":"2023-09-23T20:51:09.826361+0000","last_undegraded":"2023-09-23T20:51:09.826361+0000","last_fullsized":"2023-09-23T20:51:09.826361+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:42:21.698474+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.6","version":"70'1","reported_seq":58,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-09-23T20:51:10.833668+0000","last_change":"2023-09-23T20:50:05.871179+0000","last_active":"2023-09-23T20:51:10.833668+0000","last_peered":"2023-09-23T20:51:10.833668+0000","last_clean":"2023-09-23T20:51:10.833668+0000","last_became_active":"2023-09-23T20:50:05.870816+0000","last_became_peered":"2023-09-23T20:50:05.870816+0000","last_unstale":"2023-09-23T20:51:10.833668+0000","last_undegraded":"2023-09-23T20:51:10.833668+0000","last_fullsized":"2023-09-23T20:51:10.833668+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:25:24.455835+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":63,"reported_epoch":98,"state":"active+clean","last_fresh":"2023-09-23T20:51:09.824227+0000","last_change":"2023-09-23T20:50:37.985663+0000","last_active":"2023-09-23T20:51:09.824227+0000","last_peered":"2023-09-23T20:51:09.824227+0000","last_clean":"2023-09-23T20:51:09.824227+0000","last_became_active":"2023-09-23T20:50:37.985325+0000","last_became_peered":"2023-09-23T20:50:37.985325+0000","last_unstale":"2023-09-23T20:51:09.824227+0000","last_undegraded":"2023-09-23T20:51:09.824227+0000","last_fullsized":"2023-09-23T20:51:09.824227+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:24:30.838843+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":56,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-09-23T20:51:10.833546+0000","last_change":"2023-09-23T20:51:10.833546+0000","last_active":"2023-09-23T20:51:10.833546+0000","last_peered":"2023-09-23T20:51:10.833546+0000","last_clean":"2023-09-23T20:51:10.833546+0000","last_became_active":"2023-09-23T20:51:10.833137+0000","last_became_peered":"2023-09-23T20:51:10.833137+0000","last_unstale":"2023-09-23T20:51:10.833546+0000","last_undegraded":"2023-09-23T20:51:10.833546+0000","last_fullsized":"2023-09-23T20:51:10.833546+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:16:42.840695+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":32,"reported_epoch":98,"state":"active+clean","last_fresh":"2023-09-23T20:51:09.833862+0000","last_change":"2023-09-23T20:50:05.868392+0000","last_active":"2023-09-23T20:51:09.833862+0000","last_peered":"2023-09-23T20:51:09.833862+0000","last_clean":"2023-09-23T20:51:09.833862+0000","last_became_active":"2023-09-23T20:50:05.867910+0000","last_became_peered":"2023-09-23T20:50:05.867910+0000","last_unstale":"2023-09-23T20:51:09.833862+0000","last_undegraded":"2023-09-23T20:51:09.833862+0000","last_fullsized":"2023-09-23T20:51:09.833862+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:25:00.099993+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":59,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-09-23T20:51:10.836910+0000","last_change":"2023-09-23T20:51:10.836910+0000","last_active":"2023-09-23T20:51:10.836910+0000","last_peered":"2023-09-23T20:51:10.836910+0000","last_clean":"2023-09-23T20:51:10.836910+0000","last_became_active":"2023-09-23T20:51:10.836557+0000","last_became_peered":"2023-09-23T20:51:10.836557+0000","last_unstale":"2023-09-23T20:51:10.836910+0000","last_undegraded":"2023-09-23T20:51:10.836910+0000","last_fullsized":"2023-09-23T20:51:10.836910+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:51:54.019969+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":38,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-09-23T20:51:10.827806+0000","last_change":"2023-09-23T20:49:32.775934+0000","last_active":"2023-09-23T20:51:10.827806+0000","last_peered":"2023-09-23T20:51:10.827806+0000","last_clean":"2023-09-23T20:51:10.827806+0000","last_became_active":"2023-09-23T20:49:32.775500+0000","last_became_peered":"2023-09-23T20:49:32.775500+0000","last_unstale":"2023-09-23T20:51:10.827806+0000","last_undegraded":"2023-09-23T20:51:10.827806+0000","last_fullsized":"2023-09-23T20:51:10.827806+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:30:43.381073+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":47,"reported_epoch":98,"state":"active+clean","last_fresh":"2023-09-23T20:51:09.835354+0000","last_change":"2023-09-23T20:50:37.982593+0000","last_active":"2023-09-23T20:51:09.835354+0000","last_peered":"2023-09-23T20:51:09.835354+0000","last_clean":"2023-09-23T20:51:09.835354+0000","last_became_active":"2023-09-23T20:50:37.982303+0000","last_became_peered":"2023-09-23T20:50:37.982303+0000","last_unstale":"2023-09-23T20:51:09.835354+0000","last_undegraded":"2023-09-23T20:51:09.835354+0000","last_fullsized":"2023-09-23T20:51:09.835354+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:53:06.301515+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":63,"reported_epoch":98,"state":"active+clean","last_fresh":"2023-09-23T20:51:09.823655+0000","last_change":"2023-09-23T20:49:32.776947+0000","last_active":"2023-09-23T20:51:09.823655+0000","last_peered":"2023-09-23T20:51:09.823655+0000","last_clean":"2023-09-23T20:51:09.823655+0000","last_became_active":"2023-09-23T20:49:32.776646+0000","last_became_peered":"2023-09-23T20:49:32.776646+0000","last_unstale":"2023-09-23T20:51:09.823655+0000","last_undegraded":"2023-09-23T20:51:09.823655+0000","last_fullsized":"2023-09-23T20:51:09.823655+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:26:50.532203+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":68,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-09-23T20:51:10.833744+0000","last_change":"2023-09-23T20:50:37.982235+0000","last_active":"2023-09-23T20:51:10.833744+0000","last_peered":"2023-09-23T20:51:10.833744+0000","last_clean":"2023-09-23T20:51:10.833744+0000","last_became_active":"2023-09-23T20:50:37.981913+0000","last_became_peered":"2023-09-23T20:50:37.981913+0000","last_unstale":"2023-09-23T20:51:10.833744+0000","last_undegraded":"2023-09-23T20:51:10.833744+0000","last_fullsized":"2023-09-23T20:51:10.833744+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:59:57.122584+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":53,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-09-23T20:51:10.841702+0000","last_change":"2023-09-23T20:51:10.841350+0000","last_active":"2023-09-23T20:51:10.841702+0000","last_peered":"2023-09-23T20:51:10.841702+0000","last_clean":"2023-09-23T20:51:10.841702+0000","last_became_active":"2023-09-23T20:51:10.841029+0000","last_became_peered":"2023-09-23T20:51:10.841029+0000","last_unstale":"2023-09-23T20:51:10.841702+0000","last_undegraded":"2023-09-23T20:51:10.841702+0000","last_fullsized":"2023-09-23T20:51:10.841702+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:26:05.095206+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":80,"reported_epoch":98,"state":"active+clean","last_fresh":"2023-09-23T20:51:09.821811+0000","last_change":"2023-09-23T20:50:05.866898+0000","last_active":"2023-09-23T20:51:09.821811+0000","last_peered":"2023-09-23T20:51:09.821811+0000","last_clean":"2023-09-23T20:51:09.821811+0000","last_became_active":"2023-09-23T20:50:05.866541+0000","last_became_peered":"2023-09-23T20:50:05.866541+0000","last_unstale":"2023-09-23T20:51:09.821811+0000","last_undegraded":"2023-09-23T20:51:09.821811+0000","last_fullsized":"2023-09-23T20:51:09.821811+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:17:02.353010+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":41,"reported_epoch":98,"state":"active+clean","last_fresh":"2023-09-23T20:51:09.823968+0000","last_change":"2023-09-23T20:41:59.062981+0000","last_active":"2023-09-23T20:51:09.823968+0000","last_peered":"2023-09-23T20:51:09.823968+0000","last_clean":"2023-09-23T20:51:09.823968+0000","last_became_active":"2023-09-23T20:41:59.062717+0000","last_became_peered":"2023-09-23T20:41:59.062717+0000","last_unstale":"2023-09-23T20:51:09.823968+0000","last_undegraded":"2023-09-23T20:51:09.823968+0000","last_fullsized":"2023-09-23T20:51:09.823968+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:20:58.386461+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":22,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-09-23T20:51:10.836794+0000","last_change":"2023-09-23T20:51:10.836794+0000","last_active":"2023-09-23T20:51:10.836794+0000","last_peered":"2023-09-23T20:51:10.836794+0000","last_clean":"2023-09-23T20:51:10.836794+0000","last_became_active":"2023-09-23T20:51:10.836449+0000","last_became_peered":"2023-09-23T20:51:10.836449+0000","last_unstale":"2023-09-23T20:51:10.836794+0000","last_undegraded":"2023-09-23T20:51:10.836794+0000","last_fullsized":"2023-09-23T20:51:10.836794+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:00:35.572203+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.0","version":"79'161","reported_seq":440,"reported_epoch":98,"state":"active+clean","last_fresh":"2023-09-23T20:51:09.821731+0000","last_change":"2023-09-23T20:49:32.777246+0000","last_active":"2023-09-23T20:51:09.821731+0000","last_peered":"2023-09-23T20:51:09.821731+0000","last_clean":"2023-09-23T20:51:09.821731+0000","last_became_active":"2023-09-23T20:49:32.776822+0000","last_became_peered":"2023-09-23T20:49:32.776822+0000","last_unstale":"2023-09-23T20:51:09.821731+0000","last_undegraded":"2023-09-23T20:51:09.821731+0000","last_fullsized":"2023-09-23T20:51:09.821731+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:38:57.324224+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:05:46.881275+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":63,"reported_epoch":98,"state":"active+clean","last_fresh":"2023-09-23T20:51:09.817117+0000","last_change":"2023-09-23T20:50:37.984163+0000","last_active":"2023-09-23T20:51:09.817117+0000","last_peered":"2023-09-23T20:51:09.817117+0000","last_clean":"2023-09-23T20:51:09.817117+0000","last_became_active":"2023-09-23T20:50:37.983838+0000","last_became_peered":"2023-09-23T20:50:37.983838+0000","last_unstale":"2023-09-23T20:51:09.817117+0000","last_undegraded":"2023-09-23T20:51:09.817117+0000","last_fullsized":"2023-09-23T20:51:09.817117+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:02:41.792156+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":65,"reported_epoch":98,"state":"active+clean","last_fresh":"2023-09-23T20:51:09.835854+0000","last_change":"2023-09-23T20:50:05.867499+0000","last_active":"2023-09-23T20:51:09.835854+0000","last_peered":"2023-09-23T20:51:09.835854+0000","last_clean":"2023-09-23T20:51:09.835854+0000","last_became_active":"2023-09-23T20:50:05.866571+0000","last_became_peered":"2023-09-23T20:50:05.866571+0000","last_unstale":"2023-09-23T20:51:09.835854+0000","last_undegraded":"2023-09-23T20:51:09.835854+0000","last_fullsized":"2023-09-23T20:51:09.835854+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:18:10.735501+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":47,"reported_epoch":98,"state":"active+clean","last_fresh":"2023-09-23T20:51:09.836383+0000","last_change":"2023-09-23T20:50:37.983072+0000","last_active":"2023-09-23T20:51:09.836383+0000","last_peered":"2023-09-23T20:51:09.836383+0000","last_clean":"2023-09-23T20:51:09.836383+0000","last_became_active":"2023-09-23T20:50:37.982777+0000","last_became_peered":"2023-09-23T20:50:37.982777+0000","last_unstale":"2023-09-23T20:51:09.836383+0000","last_undegraded":"2023-09-23T20:51:09.836383+0000","last_fullsized":"2023-09-23T20:51:09.836383+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:58:11.972144+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":81,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-09-23T20:51:10.838054+0000","last_change":"2023-09-23T20:51:10.838054+0000","last_active":"2023-09-23T20:51:10.838054+0000","last_peered":"2023-09-23T20:51:10.838054+0000","last_clean":"2023-09-23T20:51:10.838054+0000","last_became_active":"2023-09-23T20:51:10.837750+0000","last_became_peered":"2023-09-23T20:51:10.837750+0000","last_unstale":"2023-09-23T20:51:10.838054+0000","last_undegraded":"2023-09-23T20:51:10.838054+0000","last_fullsized":"2023-09-23T20:51:10.838054+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:15:51.648085+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":62,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-09-23T20:51:10.829889+0000","last_change":"2023-09-23T20:51:10.829889+0000","last_active":"2023-09-23T20:51:10.829889+0000","last_peered":"2023-09-23T20:51:10.829889+0000","last_clean":"2023-09-23T20:51:10.829889+0000","last_became_active":"2023-09-23T20:51:10.829533+0000","last_became_peered":"2023-09-23T20:51:10.829533+0000","last_unstale":"2023-09-23T20:51:10.829889+0000","last_undegraded":"2023-09-23T20:51:10.829889+0000","last_fullsized":"2023-09-23T20:51:10.829889+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:43:04.603460+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":59,"reported_epoch":98,"state":"active+clean","last_fresh":"2023-09-23T20:51:09.826642+0000","last_change":"2023-09-23T20:49:32.771117+0000","last_active":"2023-09-23T20:51:09.826642+0000","last_peered":"2023-09-23T20:51:09.826642+0000","last_clean":"2023-09-23T20:51:09.826642+0000","last_became_active":"2023-09-23T20:49:32.770686+0000","last_became_peered":"2023-09-23T20:49:32.770686+0000","last_unstale":"2023-09-23T20:51:09.826642+0000","last_undegraded":"2023-09-23T20:51:09.826642+0000","last_fullsized":"2023-09-23T20:51:09.826642+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:15:46.598698+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":63,"reported_epoch":98,"state":"active+clean","last_fresh":"2023-09-23T20:51:09.817089+0000","last_change":"2023-09-23T20:50:05.864620+0000","last_active":"2023-09-23T20:51:09.817089+0000","last_peered":"2023-09-23T20:51:09.817089+0000","last_clean":"2023-09-23T20:51:09.817089+0000","last_became_active":"2023-09-23T20:50:05.864276+0000","last_became_peered":"2023-09-23T20:50:05.864276+0000","last_unstale":"2023-09-23T20:51:09.817089+0000","last_undegraded":"2023-09-23T20:51:09.817089+0000","last_fullsized":"2023-09-23T20:51:09.817089+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:20:19.862901+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":32,"reported_epoch":98,"state":"active+clean","last_fresh":"2023-09-23T20:51:09.834324+0000","last_change":"2023-09-23T20:50:05.868321+0000","last_active":"2023-09-23T20:51:09.834324+0000","last_peered":"2023-09-23T20:51:09.834324+0000","last_clean":"2023-09-23T20:51:09.834324+0000","last_became_active":"2023-09-23T20:50:05.868120+0000","last_became_peered":"2023-09-23T20:50:05.868120+0000","last_unstale":"2023-09-23T20:51:09.834324+0000","last_undegraded":"2023-09-23T20:51:09.834324+0000","last_fullsized":"2023-09-23T20:51:09.834324+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:23:14.856477+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":76,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-09-23T20:51:10.840273+0000","last_change":"2023-09-23T20:51:10.840273+0000","last_active":"2023-09-23T20:51:10.840273+0000","last_peered":"2023-09-23T20:51:10.840273+0000","last_clean":"2023-09-23T20:51:10.840273+0000","last_became_active":"2023-09-23T20:51:10.840017+0000","last_became_peered":"2023-09-23T20:51:10.840017+0000","last_unstale":"2023-09-23T20:51:10.840273+0000","last_undegraded":"2023-09-23T20:51:10.840273+0000","last_fullsized":"2023-09-23T20:51:10.840273+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:58:17.982729+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":70,"reported_epoch":98,"state":"active+clean","last_fresh":"2023-09-23T20:51:09.835307+0000","last_change":"2023-09-23T20:50:05.867130+0000","last_active":"2023-09-23T20:51:09.835307+0000","last_peered":"2023-09-23T20:51:09.835307+0000","last_clean":"2023-09-23T20:51:09.835307+0000","last_became_active":"2023-09-23T20:50:05.866758+0000","last_became_peered":"2023-09-23T20:50:05.866758+0000","last_unstale":"2023-09-23T20:51:09.835307+0000","last_undegraded":"2023-09-23T20:51:09.835307+0000","last_fullsized":"2023-09-23T20:51:09.835307+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:55:07.548209+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":47,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-09-23T20:51:10.833850+0000","last_change":"2023-09-23T20:51:10.833850+0000","last_active":"2023-09-23T20:51:10.833850+0000","last_peered":"2023-09-23T20:51:10.833850+0000","last_clean":"2023-09-23T20:51:10.833850+0000","last_became_active":"2023-09-23T20:51:10.833539+0000","last_became_peered":"2023-09-23T20:51:10.833539+0000","last_unstale":"2023-09-23T20:51:10.833850+0000","last_undegraded":"2023-09-23T20:51:10.833850+0000","last_fullsized":"2023-09-23T20:51:10.833850+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:01:36.445616+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":62,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-09-23T20:51:10.831197+0000","last_change":"2023-09-23T20:51:10.831197+0000","last_active":"2023-09-23T20:51:10.831197+0000","last_peered":"2023-09-23T20:51:10.831197+0000","last_clean":"2023-09-23T20:51:10.831197+0000","last_became_active":"2023-09-23T20:51:10.830878+0000","last_became_peered":"2023-09-23T20:51:10.830878+0000","last_unstale":"2023-09-23T20:51:10.831197+0000","last_undegraded":"2023-09-23T20:51:10.831197+0000","last_fullsized":"2023-09-23T20:51:10.831197+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:05:24.314478+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":58,"reported_epoch":98,"state":"active+clean","last_fresh":"2023-09-23T20:51:09.826968+0000","last_change":"2023-09-23T20:50:05.871406+0000","last_active":"2023-09-23T20:51:09.826968+0000","last_peered":"2023-09-23T20:51:09.826968+0000","last_clean":"2023-09-23T20:51:09.826968+0000","last_became_active":"2023-09-23T20:50:05.871127+0000","last_became_peered":"2023-09-23T20:50:05.871127+0000","last_unstale":"2023-09-23T20:51:09.826968+0000","last_undegraded":"2023-09-23T20:51:09.826968+0000","last_fullsized":"2023-09-23T20:51:09.826968+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:37:48.850229+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":59,"reported_epoch":98,"state":"active+clean","last_fresh":"2023-09-23T20:51:09.826268+0000","last_change":"2023-09-23T20:50:37.983914+0000","last_active":"2023-09-23T20:51:09.826268+0000","last_peered":"2023-09-23T20:51:09.826268+0000","last_clean":"2023-09-23T20:51:09.826268+0000","last_became_active":"2023-09-23T20:50:37.983617+0000","last_became_peered":"2023-09-23T20:50:37.983617+0000","last_unstale":"2023-09-23T20:51:09.826268+0000","last_undegraded":"2023-09-23T20:51:09.826268+0000","last_fullsized":"2023-09-23T20:51:09.826268+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:15:50.863306+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":75,"reported_epoch":98,"state":"active+clean","last_fresh":"2023-09-23T20:51:09.826833+0000","last_change":"2023-09-23T20:50:37.985593+0000","last_active":"2023-09-23T20:51:09.826833+0000","last_peered":"2023-09-23T20:51:09.826833+0000","last_clean":"2023-09-23T20:51:09.826833+0000","last_became_active":"2023-09-23T20:50:37.985324+0000","last_became_peered":"2023-09-23T20:50:37.985324+0000","last_unstale":"2023-09-23T20:51:09.826833+0000","last_undegraded":"2023-09-23T20:51:09.826833+0000","last_fullsized":"2023-09-23T20:51:09.826833+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:34:07.801771+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":59,"reported_epoch":98,"state":"active+clean","last_fresh":"2023-09-23T20:51:09.824658+0000","last_change":"2023-09-23T20:50:05.867646+0000","last_active":"2023-09-23T20:51:09.824658+0000","last_peered":"2023-09-23T20:51:09.824658+0000","last_clean":"2023-09-23T20:51:09.824658+0000","last_became_active":"2023-09-23T20:50:05.867343+0000","last_became_peered":"2023-09-23T20:50:05.867343+0000","last_unstale":"2023-09-23T20:51:09.824658+0000","last_undegraded":"2023-09-23T20:51:09.824658+0000","last_fullsized":"2023-09-23T20:51:09.824658+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:53:34.985580+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":58,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-09-23T20:51:10.833630+0000","last_change":"2023-09-23T20:51:10.833630+0000","last_active":"2023-09-23T20:51:10.833630+0000","last_peered":"2023-09-23T20:51:10.833630+0000","last_clean":"2023-09-23T20:51:10.833630+0000","last_became_active":"2023-09-23T20:51:10.833265+0000","last_became_peered":"2023-09-23T20:51:10.833265+0000","last_unstale":"2023-09-23T20:51:10.833630+0000","last_undegraded":"2023-09-23T20:51:10.833630+0000","last_fullsized":"2023-09-23T20:51:10.833630+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:12:01.557967+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":59,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-09-23T20:51:10.836752+0000","last_change":"2023-09-23T20:51:10.836752+0000","last_active":"2023-09-23T20:51:10.836752+0000","last_peered":"2023-09-23T20:51:10.836752+0000","last_clean":"2023-09-23T20:51:10.836752+0000","last_became_active":"2023-09-23T20:51:10.836213+0000","last_became_peered":"2023-09-23T20:51:10.836213+0000","last_unstale":"2023-09-23T20:51:10.836752+0000","last_undegraded":"2023-09-23T20:51:10.836752+0000","last_fullsized":"2023-09-23T20:51:10.836752+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:35:01.825635+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":66,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-09-23T20:51:10.827903+0000","last_change":"2023-09-23T20:50:37.982263+0000","last_active":"2023-09-23T20:51:10.827903+0000","last_peered":"2023-09-23T20:51:10.827903+0000","last_clean":"2023-09-23T20:51:10.827903+0000","last_became_active":"2023-09-23T20:50:37.981797+0000","last_became_peered":"2023-09-23T20:50:37.981797+0000","last_unstale":"2023-09-23T20:51:10.827903+0000","last_undegraded":"2023-09-23T20:51:10.827903+0000","last_fullsized":"2023-09-23T20:51:10.827903+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:56:45.027447+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":59,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-09-23T20:51:10.839104+0000","last_change":"2023-09-23T20:51:10.839104+0000","last_active":"2023-09-23T20:51:10.839104+0000","last_peered":"2023-09-23T20:51:10.839104+0000","last_clean":"2023-09-23T20:51:10.839104+0000","last_became_active":"2023-09-23T20:51:10.838783+0000","last_became_peered":"2023-09-23T20:51:10.838783+0000","last_unstale":"2023-09-23T20:51:10.839104+0000","last_undegraded":"2023-09-23T20:51:10.839104+0000","last_fullsized":"2023-09-23T20:51:10.839104+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:24:32.550241+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":58,"reported_epoch":98,"state":"active+clean","last_fresh":"2023-09-23T20:51:09.824495+0000","last_change":"2023-09-23T20:50:05.867973+0000","last_active":"2023-09-23T20:51:09.824495+0000","last_peered":"2023-09-23T20:51:09.824495+0000","last_clean":"2023-09-23T20:51:09.824495+0000","last_became_active":"2023-09-23T20:50:05.867622+0000","last_became_peered":"2023-09-23T20:50:05.867622+0000","last_unstale":"2023-09-23T20:51:09.824495+0000","last_undegraded":"2023-09-23T20:51:09.824495+0000","last_fullsized":"2023-09-23T20:51:09.824495+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:10:12.617597+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":43,"reported_epoch":98,"state":"active+clean","last_fresh":"2023-09-23T20:51:09.823751+0000","last_change":"2023-09-23T20:41:58.898346+0000","last_active":"2023-09-23T20:51:09.823751+0000","last_peered":"2023-09-23T20:51:09.823751+0000","last_clean":"2023-09-23T20:51:09.823751+0000","last_became_active":"2023-09-23T20:41:58.898065+0000","last_became_peered":"2023-09-23T20:41:58.898065+0000","last_unstale":"2023-09-23T20:51:09.823751+0000","last_undegraded":"2023-09-23T20:51:09.823751+0000","last_fullsized":"2023-09-23T20:51:09.823751+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:51:57.491171+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":75,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-09-23T20:51:10.838338+0000","last_change":"2023-09-23T20:51:10.838338+0000","last_active":"2023-09-23T20:51:10.838338+0000","last_peered":"2023-09-23T20:51:10.838338+0000","last_clean":"2023-09-23T20:51:10.838338+0000","last_became_active":"2023-09-23T20:51:10.838049+0000","last_became_peered":"2023-09-23T20:51:10.838049+0000","last_unstale":"2023-09-23T20:51:10.838338+0000","last_undegraded":"2023-09-23T20:51:10.838338+0000","last_fullsized":"2023-09-23T20:51:10.838338+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:40:23.602402+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":61,"reported_epoch":98,"state":"active+clean","last_fresh":"2023-09-23T20:51:09.823256+0000","last_change":"2023-09-23T20:50:05.866983+0000","last_active":"2023-09-23T20:51:09.823256+0000","last_peered":"2023-09-23T20:51:09.823256+0000","last_clean":"2023-09-23T20:51:09.823256+0000","last_became_active":"2023-09-23T20:50:05.866601+0000","last_became_peered":"2023-09-23T20:50:05.866601+0000","last_unstale":"2023-09-23T20:51:09.823256+0000","last_undegraded":"2023-09-23T20:51:09.823256+0000","last_fullsized":"2023-09-23T20:51:09.823256+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:15:00.955166+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":41,"reported_epoch":98,"state":"active+clean","last_fresh":"2023-09-23T20:51:09.816955+0000","last_change":"2023-09-23T20:41:59.061405+0000","last_active":"2023-09-23T20:51:09.816955+0000","last_peered":"2023-09-23T20:51:09.816955+0000","last_clean":"2023-09-23T20:51:09.816955+0000","last_became_active":"2023-09-23T20:41:59.061032+0000","last_became_peered":"2023-09-23T20:41:59.061032+0000","last_unstale":"2023-09-23T20:51:09.816955+0000","last_undegraded":"2023-09-23T20:51:09.816955+0000","last_fullsized":"2023-09-23T20:51:09.816955+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:43:09.716350+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":60,"reported_epoch":98,"state":"active+clean","last_fresh":"2023-09-23T20:51:09.822945+0000","last_change":"2023-09-23T20:49:32.777943+0000","last_active":"2023-09-23T20:51:09.822945+0000","last_peered":"2023-09-23T20:51:09.822945+0000","last_clean":"2023-09-23T20:51:09.822945+0000","last_became_active":"2023-09-23T20:49:32.776212+0000","last_became_peered":"2023-09-23T20:49:32.776212+0000","last_unstale":"2023-09-23T20:51:09.822945+0000","last_undegraded":"2023-09-23T20:51:09.822945+0000","last_fullsized":"2023-09-23T20:51:09.822945+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:21:05.725926+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":61,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-09-23T20:51:10.833234+0000","last_change":"2023-09-23T20:51:10.833234+0000","last_active":"2023-09-23T20:51:10.833234+0000","last_peered":"2023-09-23T20:51:10.833234+0000","last_clean":"2023-09-23T20:51:10.833234+0000","last_became_active":"2023-09-23T20:51:10.832763+0000","last_became_peered":"2023-09-23T20:51:10.832763+0000","last_unstale":"2023-09-23T20:51:10.833234+0000","last_undegraded":"2023-09-23T20:51:10.833234+0000","last_fullsized":"2023-09-23T20:51:10.833234+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:24:08.283578+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":67,"reported_epoch":98,"state":"active+clean","last_fresh":"2023-09-23T20:51:09.824402+0000","last_change":"2023-09-23T20:49:32.777437+0000","last_active":"2023-09-23T20:51:09.824402+0000","last_peered":"2023-09-23T20:51:09.824402+0000","last_clean":"2023-09-23T20:51:09.824402+0000","last_became_active":"2023-09-23T20:49:32.776999+0000","last_became_peered":"2023-09-23T20:49:32.776999+0000","last_unstale":"2023-09-23T20:51:09.824402+0000","last_undegraded":"2023-09-23T20:51:09.824402+0000","last_fullsized":"2023-09-23T20:51:09.824402+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:42:21.816073+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":53,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-09-23T20:51:10.841582+0000","last_change":"2023-09-23T20:51:10.840849+0000","last_active":"2023-09-23T20:51:10.841582+0000","last_peered":"2023-09-23T20:51:10.841582+0000","last_clean":"2023-09-23T20:51:10.841582+0000","last_became_active":"2023-09-23T20:51:10.840593+0000","last_became_peered":"2023-09-23T20:51:10.840593+0000","last_unstale":"2023-09-23T20:51:10.841582+0000","last_undegraded":"2023-09-23T20:51:10.841582+0000","last_fullsized":"2023-09-23T20:51:10.841582+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:30:52.297625+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":42,"reported_epoch":98,"state":"active+clean","last_fresh":"2023-09-23T20:51:09.821305+0000","last_change":"2023-09-23T20:41:58.898224+0000","last_active":"2023-09-23T20:51:09.821305+0000","last_peered":"2023-09-23T20:51:09.821305+0000","last_clean":"2023-09-23T20:51:09.821305+0000","last_became_active":"2023-09-23T20:41:58.898036+0000","last_became_peered":"2023-09-23T20:41:58.898036+0000","last_unstale":"2023-09-23T20:51:09.821305+0000","last_undegraded":"2023-09-23T20:51:09.821305+0000","last_fullsized":"2023-09-23T20:51:09.821305+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:53:15.361560+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":58,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-09-23T20:51:10.839174+0000","last_change":"2023-09-23T20:51:10.839174+0000","last_active":"2023-09-23T20:51:10.839174+0000","last_peered":"2023-09-23T20:51:10.839174+0000","last_clean":"2023-09-23T20:51:10.839174+0000","last_became_active":"2023-09-23T20:51:10.838826+0000","last_became_peered":"2023-09-23T20:51:10.838826+0000","last_unstale":"2023-09-23T20:51:10.839174+0000","last_undegraded":"2023-09-23T20:51:10.839174+0000","last_fullsized":"2023-09-23T20:51:10.839174+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:43:00.944119+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":53,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-09-23T20:51:10.841667+0000","last_change":"2023-09-23T20:51:10.840590+0000","last_active":"2023-09-23T20:51:10.841667+0000","last_peered":"2023-09-23T20:51:10.841667+0000","last_clean":"2023-09-23T20:51:10.841667+0000","last_became_active":"2023-09-23T20:51:10.840221+0000","last_became_peered":"2023-09-23T20:51:10.840221+0000","last_unstale":"2023-09-23T20:51:10.841667+0000","last_undegraded":"2023-09-23T20:51:10.841667+0000","last_fullsized":"2023-09-23T20:51:10.841667+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:03:39.138638+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":61,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-09-23T20:51:10.831262+0000","last_change":"2023-09-23T20:51:10.831262+0000","last_active":"2023-09-23T20:51:10.831262+0000","last_peered":"2023-09-23T20:51:10.831262+0000","last_clean":"2023-09-23T20:51:10.831262+0000","last_became_active":"2023-09-23T20:51:10.830931+0000","last_became_peered":"2023-09-23T20:51:10.830931+0000","last_unstale":"2023-09-23T20:51:10.831262+0000","last_undegraded":"2023-09-23T20:51:10.831262+0000","last_fullsized":"2023-09-23T20:51:10.831262+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:51:54.982505+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":61,"reported_epoch":98,"state":"active+clean","last_fresh":"2023-09-23T20:51:09.821700+0000","last_change":"2023-09-23T20:50:37.986729+0000","last_active":"2023-09-23T20:51:09.821700+0000","last_peered":"2023-09-23T20:51:09.821700+0000","last_clean":"2023-09-23T20:51:09.821700+0000","last_became_active":"2023-09-23T20:50:37.986433+0000","last_became_peered":"2023-09-23T20:50:37.986433+0000","last_unstale":"2023-09-23T20:51:09.821700+0000","last_undegraded":"2023-09-23T20:51:09.821700+0000","last_fullsized":"2023-09-23T20:51:09.821700+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:28:28.853091+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":58,"reported_epoch":98,"state":"active+clean","last_fresh":"2023-09-23T20:51:09.826698+0000","last_change":"2023-09-23T20:50:05.871199+0000","last_active":"2023-09-23T20:51:09.826698+0000","last_peered":"2023-09-23T20:51:09.826698+0000","last_clean":"2023-09-23T20:51:09.826698+0000","last_became_active":"2023-09-23T20:50:05.870912+0000","last_became_peered":"2023-09-23T20:50:05.870912+0000","last_unstale":"2023-09-23T20:51:09.826698+0000","last_undegraded":"2023-09-23T20:51:09.826698+0000","last_fullsized":"2023-09-23T20:51:09.826698+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:16:22.718106+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"79'22","reported_seq":64,"reported_epoch":98,"state":"active+clean","last_fresh":"2023-09-23T20:51:09.827615+0000","last_change":"2023-09-23T20:41:58.905866+0000","last_active":"2023-09-23T20:51:09.827615+0000","last_peered":"2023-09-23T20:51:09.827615+0000","last_clean":"2023-09-23T20:51:09.827615+0000","last_became_active":"2023-09-23T20:41:58.905556+0000","last_became_peered":"2023-09-23T20:41:58.905556+0000","last_unstale":"2023-09-23T20:51:09.827615+0000","last_undegraded":"2023-09-23T20:51:09.827615+0000","last_fullsized":"2023-09-23T20:51:09.827615+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:03:14.027348+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":47,"reported_epoch":98,"state":"active+clean","last_fresh":"2023-09-23T20:51:09.834463+0000","last_change":"2023-09-23T20:50:37.982103+0000","last_active":"2023-09-23T20:51:09.834463+0000","last_peered":"2023-09-23T20:51:09.834463+0000","last_clean":"2023-09-23T20:51:09.834463+0000","last_became_active":"2023-09-23T20:50:37.981791+0000","last_became_peered":"2023-09-23T20:50:37.981791+0000","last_unstale":"2023-09-23T20:51:09.834463+0000","last_undegraded":"2023-09-23T20:51:09.834463+0000","last_fullsized":"2023-09-23T20:51:09.834463+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:32:52.286186+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":62,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-09-23T20:51:10.834618+0000","last_change":"2023-09-23T20:51:10.834618+0000","last_active":"2023-09-23T20:51:10.834618+0000","last_peered":"2023-09-23T20:51:10.834618+0000","last_clean":"2023-09-23T20:51:10.834618+0000","last_became_active":"2023-09-23T20:51:10.834267+0000","last_became_peered":"2023-09-23T20:51:10.834267+0000","last_unstale":"2023-09-23T20:51:10.834618+0000","last_undegraded":"2023-09-23T20:51:10.834618+0000","last_fullsized":"2023-09-23T20:51:10.834618+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:38:59.356003+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":22,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-09-23T20:51:10.833790+0000","last_change":"2023-09-23T20:51:10.833790+0000","last_active":"2023-09-23T20:51:10.833790+0000","last_peered":"2023-09-23T20:51:10.833790+0000","last_clean":"2023-09-23T20:51:10.833790+0000","last_became_active":"2023-09-23T20:51:10.832804+0000","last_became_peered":"2023-09-23T20:51:10.832804+0000","last_unstale":"2023-09-23T20:51:10.833790+0000","last_undegraded":"2023-09-23T20:51:10.833790+0000","last_fullsized":"2023-09-23T20:51:10.833790+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:27:42.601674+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":59,"reported_epoch":98,"state":"active+clean","last_fresh":"2023-09-23T20:51:09.817393+0000","last_change":"2023-09-23T20:50:05.863737+0000","last_active":"2023-09-23T20:51:09.817393+0000","last_peered":"2023-09-23T20:51:09.817393+0000","last_clean":"2023-09-23T20:51:09.817393+0000","last_became_active":"2023-09-23T20:50:05.863466+0000","last_became_peered":"2023-09-23T20:50:05.863466+0000","last_unstale":"2023-09-23T20:51:09.817393+0000","last_undegraded":"2023-09-23T20:51:09.817393+0000","last_fullsized":"2023-09-23T20:51:09.817393+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:28:57.602682+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":61,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-09-23T20:51:10.833424+0000","last_change":"2023-09-23T20:51:10.833424+0000","last_active":"2023-09-23T20:51:10.833424+0000","last_peered":"2023-09-23T20:51:10.833424+0000","last_clean":"2023-09-23T20:51:10.833424+0000","last_became_active":"2023-09-23T20:51:10.832808+0000","last_became_peered":"2023-09-23T20:51:10.832808+0000","last_unstale":"2023-09-23T20:51:10.833424+0000","last_undegraded":"2023-09-23T20:51:10.833424+0000","last_fullsized":"2023-09-23T20:51:10.833424+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:32:42.819583+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":53,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-09-23T20:51:10.841551+0000","last_change":"2023-09-23T20:51:10.840587+0000","last_active":"2023-09-23T20:51:10.841551+0000","last_peered":"2023-09-23T20:51:10.841551+0000","last_clean":"2023-09-23T20:51:10.841551+0000","last_became_active":"2023-09-23T20:51:10.840242+0000","last_became_peered":"2023-09-23T20:51:10.840242+0000","last_unstale":"2023-09-23T20:51:10.841551+0000","last_undegraded":"2023-09-23T20:51:10.841551+0000","last_fullsized":"2023-09-23T20:51:10.841551+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:44:07.171314+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":58,"reported_epoch":98,"state":"active+clean","last_fresh":"2023-09-23T20:51:09.824327+0000","last_change":"2023-09-23T20:50:05.867622+0000","last_active":"2023-09-23T20:51:09.824327+0000","last_peered":"2023-09-23T20:51:09.824327+0000","last_clean":"2023-09-23T20:51:09.824327+0000","last_became_active":"2023-09-23T20:50:05.867311+0000","last_became_peered":"2023-09-23T20:50:05.867311+0000","last_unstale":"2023-09-23T20:51:09.824327+0000","last_undegraded":"2023-09-23T20:51:09.824327+0000","last_fullsized":"2023-09-23T20:51:09.824327+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:31:53.285316+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":79,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-09-23T20:51:10.833613+0000","last_change":"2023-09-23T20:51:10.833613+0000","last_active":"2023-09-23T20:51:10.833613+0000","last_peered":"2023-09-23T20:51:10.833613+0000","last_clean":"2023-09-23T20:51:10.833613+0000","last_became_active":"2023-09-23T20:51:10.833208+0000","last_became_peered":"2023-09-23T20:51:10.833208+0000","last_unstale":"2023-09-23T20:51:10.833613+0000","last_undegraded":"2023-09-23T20:51:10.833613+0000","last_fullsized":"2023-09-23T20:51:10.833613+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:51:28.246014+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":53,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-09-23T20:51:10.841782+0000","last_change":"2023-09-23T20:51:10.841420+0000","last_active":"2023-09-23T20:51:10.841782+0000","last_peered":"2023-09-23T20:51:10.841782+0000","last_clean":"2023-09-23T20:51:10.841782+0000","last_became_active":"2023-09-23T20:51:10.841112+0000","last_became_peered":"2023-09-23T20:51:10.841112+0000","last_unstale":"2023-09-23T20:51:10.841782+0000","last_undegraded":"2023-09-23T20:51:10.841782+0000","last_fullsized":"2023-09-23T20:51:10.841782+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:44:02.236650+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":47,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-09-23T20:51:10.833035+0000","last_change":"2023-09-23T20:51:10.833035+0000","last_active":"2023-09-23T20:51:10.833035+0000","last_peered":"2023-09-23T20:51:10.833035+0000","last_clean":"2023-09-23T20:51:10.833035+0000","last_became_active":"2023-09-23T20:51:10.832741+0000","last_became_peered":"2023-09-23T20:51:10.832741+0000","last_unstale":"2023-09-23T20:51:10.833035+0000","last_undegraded":"2023-09-23T20:51:10.833035+0000","last_fullsized":"2023-09-23T20:51:10.833035+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:53:16.471852+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":61,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-09-23T20:51:10.835170+0000","last_change":"2023-09-23T20:51:10.835170+0000","last_active":"2023-09-23T20:51:10.835170+0000","last_peered":"2023-09-23T20:51:10.835170+0000","last_clean":"2023-09-23T20:51:10.835170+0000","last_became_active":"2023-09-23T20:51:10.834808+0000","last_became_peered":"2023-09-23T20:51:10.834808+0000","last_unstale":"2023-09-23T20:51:10.835170+0000","last_undegraded":"2023-09-23T20:51:10.835170+0000","last_fullsized":"2023-09-23T20:51:10.835170+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:51:25.153742+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"68'1","reported_seq":77,"reported_epoch":98,"state":"active+clean","last_fresh":"2023-09-23T20:51:09.826718+0000","last_change":"2023-09-23T20:50:37.985580+0000","last_active":"2023-09-23T20:51:09.826718+0000","last_peered":"2023-09-23T20:51:09.826718+0000","last_clean":"2023-09-23T20:51:09.826718+0000","last_became_active":"2023-09-23T20:50:37.985229+0000","last_became_peered":"2023-09-23T20:50:37.985229+0000","last_unstale":"2023-09-23T20:51:09.826718+0000","last_undegraded":"2023-09-23T20:51:09.826718+0000","last_fullsized":"2023-09-23T20:51:09.826718+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:52:50.591507+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":161,"ondisk_log_size":161,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201418,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":22696,"kb_used_data":1568,"kb_used_omap":0,"kb_used_meta":21120,"kb_avail":93722456,"statfs":{"total":95995035648,"available":95971794944,"internally_reserved":0,"allocated":1605632,"data_stored":1128632,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":21626880},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:51:35 2023","interfaces":[{"interface":"back","average":{"1min":0.622,"5min":0.51700000000000002,"15min":0.499},"min":{"1min":0.45900000000000002,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":0.80400000000000005,"5min":0.80400000000000005,"15min":0.80400000000000005},"last":0.63800000000000001},{"interface":"front","average":{"1min":0.55200000000000005,"5min":0.56000000000000005,"15min":0.56200000000000006},"min":{"1min":0.33700000000000002,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":0.79000000000000004,"5min":0.79000000000000004,"15min":0.79000000000000004},"last":0.60699999999999998}]},{"osd":1,"last update":"Sat Sep 23 20:51:05 2023","interfaces":[{"interface":"back","average":{"1min":0.54900000000000004,"5min":0.54900000000000004,"15min":0.54900000000000004},"min":{"1min":0.40999999999999998,"5min":0.40999999999999998,"15min":0.40999999999999998},"max":{"1min":0.73099999999999998,"5min":0.73099999999999998,"15min":0.73099999999999998},"last":0.69499999999999995},{"interface":"front","average":{"1min":0.54400000000000004,"5min":0.54400000000000004,"15min":0.54400000000000004},"min":{"1min":0.32100000000000001,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":0.71599999999999997,"5min":0.71599999999999997,"15min":0.71599999999999997},"last":0.52900000000000003}]},{"osd":2,"last update":"Sat Sep 23 20:51:41 2023","interfaces":[{"interface":"back","average":{"1min":0.55100000000000005,"5min":0.55100000000000005,"15min":0.55100000000000005},"min":{"1min":0.33300000000000002,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":0.83099999999999996,"5min":0.83099999999999996,"15min":0.83099999999999996},"last":0.627},{"interface":"front","average":{"1min":0.58399999999999996,"5min":0.58399999999999996,"15min":0.58399999999999996},"min":{"1min":0.375,"5min":0.375,"15min":0.375},"max":{"1min":0.81299999999999994,"5min":0.81299999999999994,"15min":0.81299999999999994},"last":0.76000000000000001}]},{"osd":3,"last update":"Sat Sep 23 20:50:58 2023","interfaces":[{"interface":"back","average":{"1min":0.58199999999999996,"5min":0.58599999999999997,"15min":0.56499999999999995},"min":{"1min":0.40000000000000002,"5min":0.309,"15min":0.28000000000000003},"max":{"1min":0.755,"5min":0.83299999999999996,"15min":1.159},"last":0.57299999999999995},{"interface":"front","average":{"1min":0.60699999999999998,"5min":0.61099999999999999,"15min":0.58799999999999997},"min":{"1min":0.435,"5min":0.376,"15min":0.32900000000000001},"max":{"1min":0.77800000000000002,"5min":0.84099999999999997,"15min":1.5840000000000001},"last":0.65900000000000003}]},{"osd":4,"last update":"Sat Sep 23 20:50:58 2023","interfaces":[{"interface":"back","average":{"1min":0.55000000000000004,"5min":0.51100000000000001,"15min":0.48099999999999998},"min":{"1min":0.34999999999999998,"5min":0.312,"15min":0.254},"max":{"1min":0.77000000000000002,"5min":0.82099999999999995,"15min":0.82099999999999995},"last":0.58799999999999997},{"interface":"front","average":{"1min":0.56999999999999995,"5min":0.54100000000000004,"15min":0.52000000000000002},"min":{"1min":0.39200000000000002,"5min":0.35999999999999999,"15min":0.29599999999999999},"max":{"1min":0.80000000000000004,"5min":0.873,"15min":1.1100000000000001},"last":0.71399999999999997}]},{"osd":5,"last update":"Sat Sep 23 20:50:58 2023","interfaces":[{"interface":"back","average":{"1min":0.59199999999999997,"5min":0.55400000000000005,"15min":0.53100000000000003},"min":{"1min":0.42399999999999999,"5min":0.318,"15min":0.30199999999999999},"max":{"1min":0.78500000000000003,"5min":0.89900000000000002,"15min":1.4610000000000001},"last":0.78800000000000003},{"interface":"front","average":{"1min":0.63400000000000001,"5min":0.58199999999999996,"15min":0.55100000000000005},"min":{"1min":0.39500000000000002,"5min":0.34300000000000003,"15min":0.30599999999999999},"max":{"1min":0.84999999999999998,"5min":0.85899999999999999,"15min":0.85899999999999999},"last":0.73399999999999999}]},{"osd":6,"last update":"Sat Sep 23 20:50:58 2023","interfaces":[{"interface":"back","average":{"1min":0.623,"5min":0.60599999999999998,"15min":0.56899999999999995},"min":{"1min":0.433,"5min":0.41599999999999998,"15min":0.32600000000000001},"max":{"1min":0.88200000000000001,"5min":0.88300000000000001,"15min":0.88900000000000001},"last":0.68300000000000005},{"interface":"front","average":{"1min":0.64800000000000002,"5min":0.60599999999999998,"15min":0.58499999999999996},"min":{"1min":0.47599999999999998,"5min":0.36299999999999999,"15min":0.316},"max":{"1min":0.83999999999999997,"5min":0.90700000000000003,"15min":1.0760000000000001},"last":0.77100000000000002}]}]},{"osd":6,"up_from":47,"seq":201863463055,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":7832,"kb_used_data":1552,"kb_used_omap":0,"kb_used_meta":6272,"kb_avail":93737320,"statfs":{"total":95995035648,"available":95987015680,"internally_reserved":0,"allocated":1589248,"data_stored":1128221,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":6422528},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:51:35 2023","interfaces":[{"interface":"back","average":{"1min":0.50900000000000001,"5min":0.56499999999999995,"15min":0.57399999999999995},"min":{"1min":0.33400000000000002,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":0.75700000000000001,"5min":0.79200000000000004,"15min":0.79200000000000004},"last":0.81100000000000005},{"interface":"front","average":{"1min":0.54600000000000004,"5min":0.58099999999999996,"15min":0.58699999999999997},"min":{"1min":0.34200000000000003,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":0.68799999999999994,"5min":0.89100000000000001,"15min":0.89100000000000001},"last":0.58599999999999997}]},{"osd":1,"last update":"Sat Sep 23 20:51:06 2023","interfaces":[{"interface":"back","average":{"1min":0.53400000000000003,"5min":0.53400000000000003,"15min":0.53400000000000003},"min":{"1min":0.371,"5min":0.371,"15min":0.371},"max":{"1min":0.72499999999999998,"5min":0.72499999999999998,"15min":0.72499999999999998},"last":0.64800000000000002},{"interface":"front","average":{"1min":0.56999999999999995,"5min":0.56999999999999995,"15min":0.56999999999999995},"min":{"1min":0.38300000000000001,"5min":0.38300000000000001,"15min":0.38300000000000001},"max":{"1min":0.79700000000000004,"5min":0.79700000000000004,"15min":0.79700000000000004},"last":0.73299999999999998}]},{"osd":2,"last update":"Sat Sep 23 20:51:41 2023","interfaces":[{"interface":"back","average":{"1min":0.54800000000000004,"5min":0.54800000000000004,"15min":0.54800000000000004},"min":{"1min":0.40899999999999997,"5min":0.40899999999999997,"15min":0.40899999999999997},"max":{"1min":0.71999999999999997,"5min":0.71999999999999997,"15min":0.71999999999999997},"last":0.68200000000000005},{"interface":"front","average":{"1min":0.58599999999999997,"5min":0.58599999999999997,"15min":0.58599999999999997},"min":{"1min":0.35799999999999998,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":0.73399999999999999,"5min":0.73399999999999999,"15min":0.73399999999999999},"last":0.76200000000000001}]},{"osd":3,"last update":"Sat Sep 23 20:50:38 2023","interfaces":[{"interface":"back","average":{"1min":0.61599999999999999,"5min":0.60199999999999998,"15min":0.56200000000000006},"min":{"1min":0.439,"5min":0.34699999999999998,"15min":0.33200000000000002},"max":{"1min":0.82699999999999996,"5min":0.92100000000000004,"15min":0.96099999999999997},"last":0.78500000000000003},{"interface":"front","average":{"1min":0.60699999999999998,"5min":0.59599999999999997,"15min":0.56100000000000005},"min":{"1min":0.42899999999999999,"5min":0.39900000000000002,"15min":0.34100000000000003},"max":{"1min":0.81999999999999995,"5min":0.83999999999999997,"15min":0.874},"last":0.66800000000000004}]},{"osd":4,"last update":"Sat Sep 23 20:51:41 2023","interfaces":[{"interface":"back","average":{"1min":0.503,"5min":0.47599999999999998,"15min":0.45700000000000002},"min":{"1min":0.374,"5min":0.28000000000000003,"15min":0.27800000000000002},"max":{"1min":0.68400000000000005,"5min":0.81000000000000005,"15min":1.1779999999999999},"last":0.623},{"interface":"front","average":{"1min":0.54500000000000004,"5min":0.52300000000000002,"15min":0.49399999999999999},"min":{"1min":0.376,"5min":0.30599999999999999,"15min":0.29199999999999998},"max":{"1min":0.77000000000000002,"5min":0.81599999999999995,"15min":0.81599999999999995},"last":0.748}]},{"osd":5,"last update":"Sat Sep 23 20:51:41 2023","interfaces":[{"interface":"back","average":{"1min":0.53600000000000003,"5min":0.51600000000000001,"15min":0.49199999999999999},"min":{"1min":0.39600000000000002,"5min":0.32600000000000001,"15min":0.28399999999999997},"max":{"1min":0.69999999999999996,"5min":0.82699999999999996,"15min":1.081},"last":0.71199999999999997},{"interface":"front","average":{"1min":0.59299999999999997,"5min":0.56000000000000005,"15min":0.52300000000000002},"min":{"1min":0.374,"5min":0.33100000000000002,"15min":0.28499999999999998},"max":{"1min":0.755,"5min":0.81000000000000005,"15min":0.89000000000000001},"last":0.69999999999999996}]},{"osd":7,"last update":"Sat Sep 23 20:51:02 2023","interfaces":[{"interface":"back","average":{"1min":0.58299999999999996,"5min":0.56899999999999995,"15min":0.53500000000000003},"min":{"1min":0.38600000000000001,"5min":0.33000000000000002,"15min":0.32500000000000001},"max":{"1min":0.78900000000000003,"5min":0.82299999999999995,"15min":1.119},"last":0.77300000000000002},{"interface":"front","average":{"1min":0.63600000000000001,"5min":0.61599999999999999,"15min":0.57899999999999996},"min":{"1min":0.42199999999999999,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":0.79200000000000004,"5min":0.94099999999999995,"15min":0.94099999999999995},"last":0.83399999999999996}]}]},{"osd":1,"up_from":88,"seq":377957122073,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28480,"kb_used_data":1016,"kb_used_omap":9,"kb_used_meta":27446,"kb_avail":93716672,"statfs":{"total":95995035648,"available":95965872128,"internally_reserved":0,"allocated":1040384,"data_stored":544044,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9239,"internal_metadata":28105705},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:51:04 2023","interfaces":[{"interface":"back","average":{"1min":0.40899999999999997,"5min":0.40899999999999997,"15min":0.40899999999999997},"min":{"1min":0.27300000000000002,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":0.63200000000000001,"5min":0.63200000000000001,"15min":0.63200000000000001},"last":0.314},{"interface":"front","average":{"1min":0.435,"5min":0.435,"15min":0.435},"min":{"1min":0.31,"5min":0.31,"15min":0.31},"max":{"1min":0.73999999999999999,"5min":0.73999999999999999,"15min":0.73999999999999999},"last":0.376}]},{"osd":2,"last update":"Sat Sep 23 20:51:37 2023","interfaces":[{"interface":"back","average":{"1min":0.38700000000000001,"5min":0.38700000000000001,"15min":0.38700000000000001},"min":{"1min":0.26400000000000001,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":0.503,"5min":0.503,"15min":0.503},"last":0.39500000000000002},{"interface":"front","average":{"1min":0.58499999999999996,"5min":0.58499999999999996,"15min":0.58499999999999996},"min":{"1min":0.29099999999999998,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":3.5219999999999998,"5min":3.5219999999999998,"15min":3.5219999999999998},"last":0.29099999999999998}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.28000000000000003}]},{"osd":4,"last update":"Sat Sep 23 20:51:04 2023","interfaces":[{"interface":"back","average":{"1min":0.64700000000000002,"5min":0.64700000000000002,"15min":0.64700000000000002},"min":{"1min":0.435,"5min":0.435,"15min":0.435},"max":{"1min":0.88200000000000001,"5min":0.88200000000000001,"15min":0.88200000000000001},"last":0.623},{"interface":"front","average":{"1min":0.67300000000000004,"5min":0.67300000000000004,"15min":0.67300000000000004},"min":{"1min":0.44400000000000001,"5min":0.44400000000000001,"15min":0.44400000000000001},"max":{"1min":0.92900000000000005,"5min":0.92900000000000005,"15min":0.92900000000000005},"last":0.51200000000000001}]},{"osd":5,"last update":"Sat Sep 23 20:51:04 2023","interfaces":[{"interface":"back","average":{"1min":0.69799999999999995,"5min":0.69799999999999995,"15min":0.69799999999999995},"min":{"1min":0.39500000000000002,"5min":0.39500000000000002,"15min":0.39500000000000002},"max":{"1min":0.86899999999999999,"5min":0.86899999999999999,"15min":0.86899999999999999},"last":0.504},{"interface":"front","average":{"1min":0.64700000000000002,"5min":0.64700000000000002,"15min":0.64700000000000002},"min":{"1min":0.47899999999999998,"5min":0.47899999999999998,"15min":0.47899999999999998},"max":{"1min":0.79200000000000004,"5min":0.79200000000000004,"15min":0.79200000000000004},"last":0.56399999999999995}]},{"osd":6,"last update":"Sat Sep 23 20:51:04 2023","interfaces":[{"interface":"back","average":{"1min":0.68799999999999994,"5min":0.68799999999999994,"15min":0.68799999999999994},"min":{"1min":0.47999999999999998,"5min":0.47999999999999998,"15min":0.47999999999999998},"max":{"1min":0.98799999999999999,"5min":0.98799999999999999,"15min":0.98799999999999999},"last":0.61199999999999999},{"interface":"front","average":{"1min":0.65800000000000003,"5min":0.65800000000000003,"15min":0.65800000000000003},"min":{"1min":0.41799999999999998,"5min":0.41799999999999998,"15min":0.41799999999999998},"max":{"1min":0.90900000000000003,"5min":0.90900000000000003,"15min":0.90900000000000003},"last":0.52700000000000002}]},{"osd":7,"last update":"Sat Sep 23 20:51:04 2023","interfaces":[{"interface":"back","average":{"1min":0.67400000000000004,"5min":0.67400000000000004,"15min":0.67400000000000004},"min":{"1min":0.46999999999999997,"5min":0.46999999999999997,"15min":0.46999999999999997},"max":{"1min":1.004,"5min":1.004,"15min":1.004},"last":0.54100000000000004},{"interface":"front","average":{"1min":0.68600000000000005,"5min":0.68600000000000005,"15min":0.68600000000000005},"min":{"1min":0.502,"5min":0.502,"15min":0.502},"max":{"1min":1.036,"5min":1.036,"15min":1.036},"last":0.54900000000000004}]}]},{"osd":0,"up_from":83,"seq":356482285599,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":29140,"kb_used_data":1612,"kb_used_omap":28,"kb_used_meta":27491,"kb_avail":93716012,"statfs":{"total":95995035648,"available":95965196288,"internally_reserved":0,"allocated":1650688,"data_stored":1136306,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":28848,"internal_metadata":28151632},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 20:51:07 2023","interfaces":[{"interface":"back","average":{"1min":0.33300000000000002,"5min":0.33300000000000002,"15min":0.33300000000000002},"min":{"1min":0.23400000000000001,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":0.503,"5min":0.503,"15min":0.503},"last":0.27200000000000002},{"interface":"front","average":{"1min":0.38700000000000001,"5min":0.38700000000000001,"15min":0.38700000000000001},"min":{"1min":0.26100000000000001,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":0.54100000000000004,"5min":0.54100000000000004,"15min":0.54100000000000004},"last":0.28999999999999998}]},{"osd":2,"last update":"Sat Sep 23 20:51:37 2023","interfaces":[{"interface":"back","average":{"1min":0.41499999999999998,"5min":0.41499999999999998,"15min":0.41499999999999998},"min":{"1min":0.248,"5min":0.248,"15min":0.248},"max":{"1min":0.61799999999999999,"5min":0.61799999999999999,"15min":0.61799999999999999},"last":0.33700000000000002},{"interface":"front","average":{"1min":0.39000000000000001,"5min":0.39000000000000001,"15min":0.39000000000000001},"min":{"1min":0.25700000000000001,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":0.55900000000000005,"5min":0.55900000000000005,"15min":0.55900000000000005},"last":0.34699999999999998}]},{"osd":3,"last update":"Sat Sep 23 20:50:33 2023","interfaces":[{"interface":"back","average":{"1min":0.46000000000000002,"5min":0.46000000000000002,"15min":0.46000000000000002},"min":{"1min":0.28899999999999998,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":0.72699999999999998,"5min":0.72699999999999998,"15min":0.72699999999999998},"last":0.308},{"interface":"front","average":{"1min":0.45500000000000002,"5min":0.45500000000000002,"15min":0.45500000000000002},"min":{"1min":0.33100000000000002,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":0.64400000000000002,"5min":0.64400000000000002,"15min":0.64400000000000002},"last":0.36099999999999999}]},{"osd":4,"last update":"Sat Sep 23 20:51:37 2023","interfaces":[{"interface":"back","average":{"1min":0.64800000000000002,"5min":0.64800000000000002,"15min":0.64800000000000002},"min":{"1min":0.377,"5min":0.377,"15min":0.377},"max":{"1min":1.9870000000000001,"5min":1.9870000000000001,"15min":1.9870000000000001},"last":0.45300000000000001},{"interface":"front","average":{"1min":0.68899999999999995,"5min":0.66500000000000004,"15min":0.66100000000000003},"min":{"1min":0.41399999999999998,"5min":0.41399999999999998,"15min":0.41399999999999998},"max":{"1min":2.0049999999999999,"5min":2.0049999999999999,"15min":2.0049999999999999},"last":0.51900000000000002}]},{"osd":5,"last update":"Sat Sep 23 20:51:37 2023","interfaces":[{"interface":"back","average":{"1min":0.60099999999999998,"5min":0.63900000000000001,"15min":0.64500000000000002},"min":{"1min":0.42299999999999999,"5min":0.42299999999999999,"15min":0.42299999999999999},"max":{"1min":0.80400000000000005,"5min":0.83799999999999997,"15min":0.83799999999999997},"last":0.437},{"interface":"front","average":{"1min":0.60799999999999998,"5min":0.68700000000000006,"15min":0.69999999999999996},"min":{"1min":0.42499999999999999,"5min":0.42499999999999999,"15min":0.42499999999999999},"max":{"1min":0.88400000000000001,"5min":0.90000000000000002,"15min":0.90000000000000002},"last":0.42699999999999999}]},{"osd":6,"last update":"Sat Sep 23 20:51:37 2023","interfaces":[{"interface":"back","average":{"1min":0.60799999999999998,"5min":0.67400000000000004,"15min":0.68500000000000005},"min":{"1min":0.433,"5min":0.433,"15min":0.433},"max":{"1min":0.876,"5min":0.93999999999999995,"15min":0.93999999999999995},"last":0.5},{"interface":"front","average":{"1min":0.72099999999999997,"5min":0.68899999999999995,"15min":0.68300000000000005},"min":{"1min":0.45000000000000001,"5min":0.44600000000000001,"15min":0.44600000000000001},"max":{"1min":2.0489999999999999,"5min":2.0489999999999999,"15min":2.0489999999999999},"last":0.41399999999999998}]},{"osd":7,"last update":"Sat Sep 23 20:51:37 2023","interfaces":[{"interface":"back","average":{"1min":0.64000000000000001,"5min":0.68799999999999994,"15min":0.69599999999999995},"min":{"1min":0.42799999999999999,"5min":0.42799999999999999,"15min":0.42799999999999999},"max":{"1min":0.82399999999999995,"5min":0.92000000000000004,"15min":0.92000000000000004},"last":0.45800000000000002},{"interface":"front","average":{"1min":0.63900000000000001,"5min":0.66700000000000004,"15min":0.67100000000000004},"min":{"1min":0.442,"5min":0.442,"15min":0.442},"max":{"1min":0.85799999999999998,"5min":0.90600000000000003,"15min":0.90600000000000003},"last":0.48699999999999999}]}]},{"osd":2,"up_from":93,"seq":399431958546,"num_pgs":19,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28600,"kb_used_data":1008,"kb_used_omap":81,"kb_used_meta":27502,"kb_avail":93716552,"statfs":{"total":95995035648,"available":95965749248,"internally_reserved":0,"allocated":1032192,"data_stored":541255,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":83750,"internal_metadata":28162266},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:51:38 2023","interfaces":[{"interface":"back","average":{"1min":0.35799999999999998,"5min":0.35799999999999998,"15min":0.35799999999999998},"min":{"1min":0.249,"5min":0.249,"15min":0.249},"max":{"1min":0.54500000000000004,"5min":0.54500000000000004,"15min":0.54500000000000004},"last":0.20499999999999999},{"interface":"front","average":{"1min":0.40699999999999997,"5min":0.40699999999999997,"15min":0.40699999999999997},"min":{"1min":0.309,"5min":0.309,"15min":0.309},"max":{"1min":0.64800000000000002,"5min":0.64800000000000002,"15min":0.64800000000000002},"last":0.27000000000000002}]},{"osd":1,"last update":"Sat Sep 23 20:51:38 2023","interfaces":[{"interface":"back","average":{"1min":0.41699999999999998,"5min":0.41699999999999998,"15min":0.41699999999999998},"min":{"1min":0.31,"5min":0.31,"15min":0.31},"max":{"1min":0.61699999999999999,"5min":0.61699999999999999,"15min":0.61699999999999999},"last":0.28999999999999998},{"interface":"front","average":{"1min":0.42099999999999999,"5min":0.42099999999999999,"15min":0.42099999999999999},"min":{"1min":0.33200000000000002,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":0.55600000000000005,"5min":0.55600000000000005,"15min":0.55600000000000005},"last":0.33400000000000002}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.313}]},{"osd":4,"last update":"Sat Sep 23 20:51:38 2023","interfaces":[{"interface":"back","average":{"1min":0.61899999999999999,"5min":0.61899999999999999,"15min":0.61899999999999999},"min":{"1min":0.45200000000000001,"5min":0.45200000000000001,"15min":0.45200000000000001},"max":{"1min":0.86099999999999999,"5min":0.86099999999999999,"15min":0.86099999999999999},"last":0.65000000000000002},{"interface":"front","average":{"1min":0.67100000000000004,"5min":0.67100000000000004,"15min":0.67100000000000004},"min":{"1min":0.47299999999999998,"5min":0.47299999999999998,"15min":0.47299999999999998},"max":{"1min":0.92400000000000004,"5min":0.92400000000000004,"15min":0.92400000000000004},"last":0.55800000000000005}]},{"osd":5,"last update":"Sat Sep 23 20:51:38 2023","interfaces":[{"interface":"back","average":{"1min":0.65200000000000002,"5min":0.65200000000000002,"15min":0.65200000000000002},"min":{"1min":0.46300000000000002,"5min":0.46300000000000002,"15min":0.46300000000000002},"max":{"1min":0.95999999999999996,"5min":0.95999999999999996,"15min":0.95999999999999996},"last":0.67800000000000005},{"interface":"front","average":{"1min":0.63600000000000001,"5min":0.63600000000000001,"15min":0.63600000000000001},"min":{"1min":0.42299999999999999,"5min":0.42299999999999999,"15min":0.42299999999999999},"max":{"1min":0.89600000000000002,"5min":0.89600000000000002,"15min":0.89600000000000002},"last":0.54200000000000004}]},{"osd":6,"last update":"Sat Sep 23 20:51:38 2023","interfaces":[{"interface":"back","average":{"1min":0.70599999999999996,"5min":0.70599999999999996,"15min":0.70599999999999996},"min":{"1min":0.502,"5min":0.502,"15min":0.502},"max":{"1min":0.97299999999999998,"5min":0.97299999999999998,"15min":0.97299999999999998},"last":0.51800000000000002},{"interface":"front","average":{"1min":0.68600000000000005,"5min":0.68600000000000005,"15min":0.68600000000000005},"min":{"1min":0.49399999999999999,"5min":0.49399999999999999,"15min":0.49399999999999999},"max":{"1min":0.93999999999999995,"5min":0.93999999999999995,"15min":0.93999999999999995},"last":0.628}]},{"osd":7,"last update":"Sat Sep 23 20:51:38 2023","interfaces":[{"interface":"back","average":{"1min":0.63500000000000001,"5min":0.63500000000000001,"15min":0.63500000000000001},"min":{"1min":0.498,"5min":0.498,"15min":0.498},"max":{"1min":0.90600000000000003,"5min":0.90600000000000003,"15min":0.90600000000000003},"last":0.56699999999999995},{"interface":"front","average":{"1min":0.66200000000000003,"5min":0.66200000000000003,"15min":0.66200000000000003},"min":{"1min":0.433,"5min":0.433,"15min":0.433},"max":{"1min":0.90500000000000003,"5min":0.90500000000000003,"15min":0.90500000000000003},"last":0.69399999999999995}]}]},{"osd":3,"up_from":98,"seq":420906795020,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28452,"kb_used_data":988,"kb_used_omap":7,"kb_used_meta":27448,"kb_avail":93716700,"statfs":{"total":95995035648,"available":95965900800,"internally_reserved":0,"allocated":1011712,"data_stored":538170,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7576,"internal_metadata":28107368},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.27900000000000003}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.25700000000000001}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.36599999999999999}]},{"osd":4,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.45300000000000001}]},{"osd":5,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.53700000000000003}]},{"osd":6,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.437}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.52300000000000002}]}]},{"osd":4,"up_from":31,"seq":133143986327,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":11876,"kb_used_data":988,"kb_used_omap":0,"kb_used_meta":10880,"kb_avail":93733276,"statfs":{"total":95995035648,"available":95982874624,"internally_reserved":0,"allocated":1011712,"data_stored":538517,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":11141120},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:51:35 2023","interfaces":[{"interface":"back","average":{"1min":0.745,"5min":0.65600000000000003,"15min":0.64100000000000001},"min":{"1min":0.40000000000000002,"5min":0.38900000000000001,"15min":0.38900000000000001},"max":{"1min":4.4050000000000002,"5min":4.4050000000000002,"15min":4.4050000000000002},"last":0.81000000000000005},{"interface":"front","average":{"1min":0.748,"5min":0.88200000000000001,"15min":0.90400000000000003},"min":{"1min":0.46899999999999997,"5min":0.46899999999999997,"15min":0.46899999999999997},"max":{"1min":4.468,"5min":5.2409999999999997,"15min":5.2409999999999997},"last":0.66000000000000003}]},{"osd":1,"last update":"Sat Sep 23 20:51:06 2023","interfaces":[{"interface":"back","average":{"1min":0.55900000000000005,"5min":0.55900000000000005,"15min":0.55900000000000005},"min":{"1min":0.44700000000000001,"5min":0.44700000000000001,"15min":0.44700000000000001},"max":{"1min":0.79900000000000004,"5min":0.79900000000000004,"15min":0.79900000000000004},"last":0.64700000000000002},{"interface":"front","average":{"1min":0.59899999999999998,"5min":0.59899999999999998,"15min":0.59899999999999998},"min":{"1min":0.44500000000000001,"5min":0.44500000000000001,"15min":0.44500000000000001},"max":{"1min":0.72399999999999998,"5min":0.72399999999999998,"15min":0.72399999999999998},"last":0.73999999999999999}]},{"osd":2,"last update":"Sat Sep 23 20:51:41 2023","interfaces":[{"interface":"back","average":{"1min":0.68600000000000005,"5min":0.68600000000000005,"15min":0.68600000000000005},"min":{"1min":0.442,"5min":0.442,"15min":0.442},"max":{"1min":2.5030000000000001,"5min":2.5030000000000001,"15min":2.5030000000000001},"last":0.70799999999999996},{"interface":"front","average":{"1min":0.77400000000000002,"5min":0.77400000000000002,"15min":0.77400000000000002},"min":{"1min":0.38200000000000001,"5min":0.38200000000000001,"15min":0.38200000000000001},"max":{"1min":4.3289999999999997,"5min":4.3289999999999997,"15min":4.3289999999999997},"last":0.55300000000000005}]},{"osd":3,"last update":"Sat Sep 23 20:50:04 2023","interfaces":[{"interface":"back","average":{"1min":0.63300000000000001,"5min":0.59999999999999998,"15min":0.56799999999999995},"min":{"1min":0.41799999999999998,"5min":0.35299999999999998,"15min":0.33100000000000002},"max":{"1min":2.1320000000000001,"5min":2.1320000000000001,"15min":2.1320000000000001},"last":0.59299999999999997},{"interface":"front","average":{"1min":0.53500000000000003,"5min":0.57799999999999996,"15min":0.55900000000000005},"min":{"1min":0.35699999999999998,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":0.83499999999999996,"5min":0.88,"15min":0.91500000000000004},"last":0.77500000000000002}]},{"osd":5,"last update":"Sat Sep 23 20:51:17 2023","interfaces":[{"interface":"back","average":{"1min":0.57399999999999995,"5min":0.502,"15min":0.45100000000000001},"min":{"1min":0.40000000000000002,"5min":0.29099999999999998,"15min":0.251},"max":{"1min":0.80900000000000005,"5min":0.80900000000000005,"15min":0.83599999999999997},"last":0.45600000000000002},{"interface":"front","average":{"1min":0.54400000000000004,"5min":0.51100000000000001,"15min":0.46000000000000002},"min":{"1min":0.371,"5min":0.33400000000000002,"15min":0.28799999999999998},"max":{"1min":0.79500000000000004,"5min":0.79500000000000004,"15min":0.90000000000000002},"last":0.499}]},{"osd":6,"last update":"Sat Sep 23 20:51:53 2023","interfaces":[{"interface":"back","average":{"1min":0.56599999999999995,"5min":0.51900000000000002,"15min":0.51400000000000001},"min":{"1min":0.439,"5min":0.30299999999999999,"15min":0.28100000000000003},"max":{"1min":0.72199999999999998,"5min":0.748,"15min":1.1319999999999999},"last":0.52800000000000002},{"interface":"front","average":{"1min":0.66500000000000004,"5min":0.58499999999999996,"15min":0.56999999999999995},"min":{"1min":0.47299999999999998,"5min":0.316,"15min":0.29399999999999998},"max":{"1min":0.85999999999999999,"5min":0.85999999999999999,"15min":1.095},"last":0.623}]},{"osd":7,"last update":"Sat Sep 23 20:51:06 2023","interfaces":[{"interface":"back","average":{"1min":0.63500000000000001,"5min":0.59299999999999997,"15min":0.56599999999999995},"min":{"1min":0.44700000000000001,"5min":0.35099999999999998,"15min":0.32500000000000001},"max":{"1min":0.77100000000000002,"5min":0.79100000000000004,"15min":0.877},"last":0.67300000000000004},{"interface":"front","average":{"1min":0.62,"5min":0.58599999999999997,"15min":0.56499999999999995},"min":{"1min":0.42499999999999999,"5min":0.33500000000000002,"15min":0.32700000000000001},"max":{"1min":0.79600000000000004,"5min":0.79600000000000004,"15min":0.86299999999999999},"last":0.60999999999999999}]}]},{"osd":5,"up_from":39,"seq":167503724692,"num_pgs":21,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":17940,"kb_used_data":972,"kb_used_omap":0,"kb_used_meta":16960,"kb_avail":93727212,"statfs":{"total":95995035648,"available":95976665088,"internally_reserved":0,"allocated":995328,"data_stored":534261,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":17367040},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:51:34 2023","interfaces":[{"interface":"back","average":{"1min":0.51300000000000001,"5min":0.499,"15min":0.497},"min":{"1min":0.37,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":0.745,"5min":0.745,"15min":0.745},"last":0.58199999999999996},{"interface":"front","average":{"1min":0.57399999999999995,"5min":0.57499999999999996,"15min":0.57499999999999996},"min":{"1min":0.35399999999999998,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":0.88400000000000001,"5min":0.88400000000000001,"15min":0.88400000000000001},"last":0.59599999999999997}]},{"osd":1,"last update":"Sat Sep 23 20:51:05 2023","interfaces":[{"interface":"back","average":{"1min":0.56599999999999995,"5min":0.56599999999999995,"15min":0.56599999999999995},"min":{"1min":0.42499999999999999,"5min":0.42499999999999999,"15min":0.42499999999999999},"max":{"1min":0.83899999999999997,"5min":0.83899999999999997,"15min":0.83899999999999997},"last":0.56499999999999995},{"interface":"front","average":{"1min":0.54900000000000004,"5min":0.54900000000000004,"15min":0.54900000000000004},"min":{"1min":0.39000000000000001,"5min":0.39000000000000001,"15min":0.39000000000000001},"max":{"1min":0.70899999999999996,"5min":0.70899999999999996,"15min":0.70899999999999996},"last":0.64300000000000002}]},{"osd":2,"last update":"Sat Sep 23 20:51:39 2023","interfaces":[{"interface":"back","average":{"1min":0.54200000000000004,"5min":0.54200000000000004,"15min":0.54200000000000004},"min":{"1min":0.436,"5min":0.436,"15min":0.436},"max":{"1min":0.68500000000000005,"5min":0.68500000000000005,"15min":0.68500000000000005},"last":0.65900000000000003},{"interface":"front","average":{"1min":0.58399999999999996,"5min":0.58399999999999996,"15min":0.58399999999999996},"min":{"1min":0.40200000000000002,"5min":0.40200000000000002,"15min":0.40200000000000002},"max":{"1min":0.81699999999999995,"5min":0.81699999999999995,"15min":0.81699999999999995},"last":0.48399999999999999}]},{"osd":3,"last update":"Sat Sep 23 20:50:16 2023","interfaces":[{"interface":"back","average":{"1min":0.61099999999999999,"5min":0.59199999999999997,"15min":0.56599999999999995},"min":{"1min":0.45600000000000002,"5min":0.39000000000000001,"15min":0.29399999999999998},"max":{"1min":0.79300000000000004,"5min":0.81999999999999995,"15min":0.96099999999999997},"last":0.68200000000000005},{"interface":"front","average":{"1min":0.59099999999999997,"5min":0.61299999999999999,"15min":0.58399999999999996},"min":{"1min":0.38,"5min":0.37,"15min":0.33100000000000002},"max":{"1min":0.78500000000000003,"5min":0.90400000000000003,"15min":0.97699999999999998},"last":0.61799999999999999}]},{"osd":4,"last update":"Sat Sep 23 20:51:17 2023","interfaces":[{"interface":"back","average":{"1min":0.53000000000000003,"5min":0.50900000000000001,"15min":0.45900000000000002},"min":{"1min":0.32700000000000001,"5min":0.317,"15min":0.253},"max":{"1min":0.75900000000000001,"5min":0.80900000000000005,"15min":0.80900000000000005},"last":0.45100000000000001},{"interface":"front","average":{"1min":0.54400000000000004,"5min":0.53500000000000003,"15min":0.496},"min":{"1min":0.28499999999999998,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":0.83499999999999996,"5min":0.83499999999999996,"15min":0.89500000000000002},"last":0.52500000000000002}]},{"osd":6,"last update":"Sat Sep 23 20:51:44 2023","interfaces":[{"interface":"back","average":{"1min":0.57399999999999995,"5min":0.54800000000000004,"15min":0.51100000000000001},"min":{"1min":0.34599999999999997,"5min":0.309,"15min":0.28599999999999998},"max":{"1min":0.86799999999999999,"5min":0.86899999999999999,"15min":0.91400000000000003},"last":0.69799999999999995},{"interface":"front","average":{"1min":0.63700000000000001,"5min":0.57799999999999996,"15min":0.54200000000000004},"min":{"1min":0.36299999999999999,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":0.85399999999999998,"5min":0.85399999999999998,"15min":0.85399999999999998},"last":0.55000000000000004}]},{"osd":7,"last update":"Sat Sep 23 20:51:02 2023","interfaces":[{"interface":"back","average":{"1min":0.57299999999999995,"5min":0.58199999999999996,"15min":0.56100000000000005},"min":{"1min":0.38,"5min":0.33600000000000002,"15min":0.314},"max":{"1min":0.77900000000000003,"5min":0.873,"15min":0.874},"last":0.63100000000000001},{"interface":"front","average":{"1min":0.624,"5min":0.61699999999999999,"15min":0.57899999999999996},"min":{"1min":0.41399999999999998,"5min":0.36099999999999999,"15min":0.34100000000000003},"max":{"1min":0.80100000000000005,"5min":0.88500000000000001,"15min":0.88500000000000001},"last":0.71199999999999997}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T20:52:01.079 INFO:tasks.cephadm.ceph_manager.ceph:clean! 2023-09-23T20:52:01.079 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph osd dump 2023-09-23T20:52:01.686 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:52:02.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:52:01 smithi070 ceph-mon[168282]: pgmap v225: 73 pgs: 73 active+clean; 582 KiB data, 171 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:52:02.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:52:01 smithi070 ceph-mon[168282]: from='client.44633 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:52:02.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:52:01 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:52:02.131 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:52:01 smithi026 ceph-mon[175151]: pgmap v225: 73 pgs: 73 active+clean; 582 KiB data, 171 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:52:02.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:52:01 smithi026 ceph-mon[175151]: from='client.44633 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:52:02.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:52:01 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:52:02.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:52:01 smithi026 ceph-mon[179688]: pgmap v225: 73 pgs: 73 active+clean; 582 KiB data, 171 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:52:02.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:52:01 smithi026 ceph-mon[179688]: from='client.44633 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:52:02.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:52:01 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:52:03.180 INFO:teuthology.orchestra.run.smithi026.stdout:epoch 99 2023-09-23T20:52:03.180 INFO:teuthology.orchestra.run.smithi026.stdout:fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 2023-09-23T20:52:03.181 INFO:teuthology.orchestra.run.smithi026.stdout:created 2023-09-23T20:35:58.919705+0000 2023-09-23T20:52:03.181 INFO:teuthology.orchestra.run.smithi026.stdout:modified 2023-09-23T20:51:10.816433+0000 2023-09-23T20:52:03.181 INFO:teuthology.orchestra.run.smithi026.stdout:flags sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit 2023-09-23T20:52:03.181 INFO:teuthology.orchestra.run.smithi026.stdout:crush_version 22 2023-09-23T20:52:03.181 INFO:teuthology.orchestra.run.smithi026.stdout:full_ratio 0.95 2023-09-23T20:52:03.181 INFO:teuthology.orchestra.run.smithi026.stdout:backfillfull_ratio 0.9 2023-09-23T20:52:03.181 INFO:teuthology.orchestra.run.smithi026.stdout:nearfull_ratio 0.85 2023-09-23T20:52:03.181 INFO:teuthology.orchestra.run.smithi026.stdout:require_min_compat_client luminous 2023-09-23T20:52:03.181 INFO:teuthology.orchestra.run.smithi026.stdout:min_compat_client luminous 2023-09-23T20:52:03.181 INFO:teuthology.orchestra.run.smithi026.stdout:require_osd_release quincy 2023-09-23T20:52:03.181 INFO:teuthology.orchestra.run.smithi026.stdout:stretch_mode_enabled false 2023-09-23T20:52:03.181 INFO:teuthology.orchestra.run.smithi026.stdout:pool 1 '.mgr' replicated size 3 min_size 2 crush_rule 0 object_hash rjenkins pg_num 1 pgp_num 1 autoscale_mode off last_change 22 flags hashpspool stripe_width 0 pg_num_max 32 pg_num_min 1 application mgr read_balance_score 7.89 2023-09-23T20:52:03.182 INFO:teuthology.orchestra.run.smithi026.stdout:pool 2 'rbd' replicated size 3 min_size 2 crush_rule 0 object_hash rjenkins pg_num 8 pgp_num 8 autoscale_mode off last_change 64 flags hashpspool stripe_width 0 application rbd read_balance_score 2.00 2023-09-23T20:52:03.182 INFO:teuthology.orchestra.run.smithi026.stdout:pool 3 'cephfs.foo.meta' replicated size 3 min_size 2 crush_rule 0 object_hash rjenkins pg_num 32 pgp_num 32 autoscale_mode off last_change 68 flags hashpspool stripe_width 0 pg_autoscale_bias 4 pg_num_min 16 recovery_priority 5 application cephfs read_balance_score 1.50 2023-09-23T20:52:03.182 INFO:teuthology.orchestra.run.smithi026.stdout:pool 4 'cephfs.foo.data' replicated size 3 min_size 2 crush_rule 0 object_hash rjenkins pg_num 32 pgp_num 32 autoscale_mode off last_change 68 flags hashpspool stripe_width 0 application cephfs read_balance_score 2.25 2023-09-23T20:52:03.182 INFO:teuthology.orchestra.run.smithi026.stdout:max_osd 8 2023-09-23T20:52:03.182 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0 up in weight 1 up_from 83 up_thru 98 down_at 80 last_clean_interval [8,79) [v2:172.21.15.26:6802/3105265146,v1:172.21.15.26:6803/3105265146] [v2:172.21.15.26:6804/3105265146,v1:172.21.15.26:6805/3105265146] exists,up f837feb7-060e-4965-9b48-bed9f3672d05 2023-09-23T20:52:03.182 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1 up in weight 1 up_from 88 up_thru 98 down_at 85 last_clean_interval [13,84) [v2:172.21.15.26:6810/3442737755,v1:172.21.15.26:6811/3442737755] [v2:172.21.15.26:6812/3442737755,v1:172.21.15.26:6813/3442737755] exists,up 917f65aa-5963-4576-988a-396698b7c6a8 2023-09-23T20:52:03.182 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2 up in weight 1 up_from 93 up_thru 98 down_at 90 last_clean_interval [18,89) [v2:172.21.15.26:6818/38979282,v1:172.21.15.26:6819/38979282] [v2:172.21.15.26:6820/38979282,v1:172.21.15.26:6821/38979282] exists,up 2e115328-76c6-478e-915c-5786903be49f 2023-09-23T20:52:03.182 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3 up in weight 1 up_from 98 up_thru 98 down_at 95 last_clean_interval [26,94) [v2:172.21.15.26:6826/2947396145,v1:172.21.15.26:6827/2947396145] [v2:172.21.15.26:6828/2947396145,v1:172.21.15.26:6829/2947396145] exists,up c3dd2c3f-5cf5-4fb0-964c-1b8926f75c8a 2023-09-23T20:52:03.182 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4 up in weight 1 up_from 31 up_thru 98 down_at 0 last_clean_interval [0,0) [v2:172.21.15.70:6800/1528249138,v1:172.21.15.70:6801/1528249138] [v2:172.21.15.70:6802/1528249138,v1:172.21.15.70:6803/1528249138] exists,up a5f31996-1b38-491b-956b-5aebc72d0170 2023-09-23T20:52:03.182 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5 up in weight 1 up_from 39 up_thru 98 down_at 0 last_clean_interval [0,0) [v2:172.21.15.70:6808/3802232906,v1:172.21.15.70:6809/3802232906] [v2:172.21.15.70:6810/3802232906,v1:172.21.15.70:6811/3802232906] exists,up 21dad198-b9db-42c6-8823-112df1ecd874 2023-09-23T20:52:03.182 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6 up in weight 1 up_from 47 up_thru 95 down_at 0 last_clean_interval [0,0) [v2:172.21.15.70:6816/2944175278,v1:172.21.15.70:6817/2944175278] [v2:172.21.15.70:6818/2944175278,v1:172.21.15.70:6819/2944175278] exists,up 6f3be784-7149-4f13-81b5-98924e97c056 2023-09-23T20:52:03.182 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7 up in weight 1 up_from 55 up_thru 98 down_at 0 last_clean_interval [0,0) [v2:172.21.15.70:6824/2824974165,v1:172.21.15.70:6825/2824974165] [v2:172.21.15.70:6826/2824974165,v1:172.21.15.70:6827/2824974165] exists,up 499f5f85-f4b2-40a7-ac0f-79cd22738d30 2023-09-23T20:52:03.183 INFO:teuthology.orchestra.run.smithi026.stdout:pg_upmap_items 4.b [1,2] 2023-09-23T20:52:03.183 INFO:teuthology.orchestra.run.smithi026.stdout:pg_upmap_items 4.18 [1,2] 2023-09-23T20:52:03.183 INFO:teuthology.orchestra.run.smithi026.stdout:pg_upmap_items 4.1c [1,5] 2023-09-23T20:52:03.183 INFO:teuthology.orchestra.run.smithi026.stdout:blocklist 172.21.15.26:0/2817523682 expires 2023-09-24T20:45:16.037413+0000 2023-09-23T20:52:03.183 INFO:teuthology.orchestra.run.smithi026.stdout:blocklist 172.21.15.26:0/934875067 expires 2023-09-24T20:45:16.037413+0000 2023-09-23T20:52:03.183 INFO:teuthology.orchestra.run.smithi026.stdout:blocklist 172.21.15.26:0/190503462 expires 2023-09-24T20:45:16.037413+0000 2023-09-23T20:52:03.183 INFO:teuthology.orchestra.run.smithi026.stdout:blocklist 172.21.15.26:0/3175144810 expires 2023-09-24T20:36:31.159649+0000 2023-09-23T20:52:03.183 INFO:teuthology.orchestra.run.smithi026.stdout:blocklist 172.21.15.26:0/188049283 expires 2023-09-24T20:41:03.790648+0000 2023-09-23T20:52:03.183 INFO:teuthology.orchestra.run.smithi026.stdout:blocklist 172.21.15.26:0/1815190465 expires 2023-09-24T20:36:31.159649+0000 2023-09-23T20:52:03.183 INFO:teuthology.orchestra.run.smithi026.stdout:blocklist 172.21.15.26:0/289994511 expires 2023-09-24T20:36:56.498559+0000 2023-09-23T20:52:03.183 INFO:teuthology.orchestra.run.smithi026.stdout:blocklist 172.21.15.26:6801/91363814 expires 2023-09-24T20:41:03.790648+0000 2023-09-23T20:52:03.183 INFO:teuthology.orchestra.run.smithi026.stdout:blocklist 172.21.15.70:0/2989870914 expires 2023-09-24T20:44:32.835628+0000 2023-09-23T20:52:03.184 INFO:teuthology.orchestra.run.smithi026.stdout:blocklist 172.21.15.26:6800/2440434873 expires 2023-09-24T20:43:50.641728+0000 2023-09-23T20:52:03.184 INFO:teuthology.orchestra.run.smithi026.stdout:blocklist 172.21.15.26:0/1083572999 expires 2023-09-24T20:41:03.790648+0000 2023-09-23T20:52:03.184 INFO:teuthology.orchestra.run.smithi026.stdout:blocklist 172.21.15.70:0/3393166416 expires 2023-09-24T20:44:32.835628+0000 2023-09-23T20:52:03.184 INFO:teuthology.orchestra.run.smithi026.stdout:blocklist 172.21.15.26:6800/952511372 expires 2023-09-24T20:36:56.498559+0000 2023-09-23T20:52:03.184 INFO:teuthology.orchestra.run.smithi026.stdout:blocklist 172.21.15.70:0/2198692933 expires 2023-09-24T20:44:32.835628+0000 2023-09-23T20:52:03.184 INFO:teuthology.orchestra.run.smithi026.stdout:blocklist 172.21.15.26:6801/2440434873 expires 2023-09-24T20:43:50.641728+0000 2023-09-23T20:52:03.184 INFO:teuthology.orchestra.run.smithi026.stdout:blocklist 172.21.15.70:0/302414747 expires 2023-09-24T20:44:32.835628+0000 2023-09-23T20:52:03.184 INFO:teuthology.orchestra.run.smithi026.stdout:blocklist 172.21.15.70:0/413499922 expires 2023-09-24T20:44:32.835628+0000 2023-09-23T20:52:03.184 INFO:teuthology.orchestra.run.smithi026.stdout:blocklist 172.21.15.26:0/686734212 expires 2023-09-24T20:36:31.159649+0000 2023-09-23T20:52:03.184 INFO:teuthology.orchestra.run.smithi026.stdout:blocklist 172.21.15.70:6834/1275773750 expires 2023-09-24T20:44:32.835628+0000 2023-09-23T20:52:03.184 INFO:teuthology.orchestra.run.smithi026.stdout:blocklist 172.21.15.26:0/635571892 expires 2023-09-24T20:43:50.641728+0000 2023-09-23T20:52:03.184 INFO:teuthology.orchestra.run.smithi026.stdout:blocklist 172.21.15.26:0/1376232363 expires 2023-09-24T20:43:50.641728+0000 2023-09-23T20:52:03.185 INFO:teuthology.orchestra.run.smithi026.stdout:blocklist 172.21.15.26:0/394447930 expires 2023-09-24T20:41:03.790648+0000 2023-09-23T20:52:03.185 INFO:teuthology.orchestra.run.smithi026.stdout:blocklist 172.21.15.70:6835/1275773750 expires 2023-09-24T20:44:32.835628+0000 2023-09-23T20:52:03.185 INFO:teuthology.orchestra.run.smithi026.stdout:blocklist 172.21.15.26:0/1625326392 expires 2023-09-24T20:36:56.498559+0000 2023-09-23T20:52:03.185 INFO:teuthology.orchestra.run.smithi026.stdout:blocklist 172.21.15.26:6801/952511372 expires 2023-09-24T20:36:56.498559+0000 2023-09-23T20:52:03.185 INFO:teuthology.orchestra.run.smithi026.stdout:blocklist 172.21.15.26:0/2838335646 expires 2023-09-24T20:36:56.498559+0000 2023-09-23T20:52:03.185 INFO:teuthology.orchestra.run.smithi026.stdout:blocklist 172.21.15.26:6801/4099624907 expires 2023-09-24T20:36:31.159649+0000 2023-09-23T20:52:03.185 INFO:teuthology.orchestra.run.smithi026.stdout:blocklist 172.21.15.26:0/3408257869 expires 2023-09-24T20:45:16.037413+0000 2023-09-23T20:52:03.185 INFO:teuthology.orchestra.run.smithi026.stdout:blocklist 172.21.15.26:6800/91363814 expires 2023-09-24T20:41:03.790648+0000 2023-09-23T20:52:03.185 INFO:teuthology.orchestra.run.smithi026.stdout:blocklist 172.21.15.70:0/1035724255 expires 2023-09-24T20:44:32.835628+0000 2023-09-23T20:52:03.185 INFO:teuthology.orchestra.run.smithi026.stdout:blocklist 172.21.15.26:0/4215395943 expires 2023-09-24T20:43:50.641728+0000 2023-09-23T20:52:03.185 INFO:teuthology.orchestra.run.smithi026.stdout:blocklist 172.21.15.26:0/233963796 expires 2023-09-24T20:45:16.037413+0000 2023-09-23T20:52:03.185 INFO:teuthology.orchestra.run.smithi026.stdout:blocklist 172.21.15.26:0/3461760126 expires 2023-09-24T20:41:03.790648+0000 2023-09-23T20:52:03.186 INFO:teuthology.orchestra.run.smithi026.stdout:blocklist 172.21.15.26:0/237900280 expires 2023-09-24T20:43:50.641728+0000 2023-09-23T20:52:03.186 INFO:teuthology.orchestra.run.smithi026.stdout:blocklist 172.21.15.26:6801/1446178988 expires 2023-09-24T20:45:16.037413+0000 2023-09-23T20:52:03.186 INFO:teuthology.orchestra.run.smithi026.stdout:blocklist 172.21.15.26:0/2633606443 expires 2023-09-24T20:45:16.037413+0000 2023-09-23T20:52:03.186 INFO:teuthology.orchestra.run.smithi026.stdout:blocklist 172.21.15.26:6800/4099624907 expires 2023-09-24T20:36:31.159649+0000 2023-09-23T20:52:03.186 INFO:teuthology.orchestra.run.smithi026.stdout:blocklist 172.21.15.26:6800/1446178988 expires 2023-09-24T20:45:16.037413+0000 2023-09-23T20:52:03.535 INFO:tasks.cephadm.ceph_manager.ceph:['osd.0 up in weight 1 up_from 83 up_thru 98 down_at 80 last_clean_interval [8,79) [v2:172.21.15.26:6802/3105265146,v1:172.21.15.26:6803/3105265146] [v2:172.21.15.26:6804/3105265146,v1:172.21.15.26:6805/3105265146] exists,up f837feb7-060e-4965-9b48-bed9f3672d05', 'osd.1 up in weight 1 up_from 88 up_thru 98 down_at 85 last_clean_interval [13,84) [v2:172.21.15.26:6810/3442737755,v1:172.21.15.26:6811/3442737755] [v2:172.21.15.26:6812/3442737755,v1:172.21.15.26:6813/3442737755] exists,up 917f65aa-5963-4576-988a-396698b7c6a8', 'osd.2 up in weight 1 up_from 93 up_thru 98 down_at 90 last_clean_interval [18,89) [v2:172.21.15.26:6818/38979282,v1:172.21.15.26:6819/38979282] [v2:172.21.15.26:6820/38979282,v1:172.21.15.26:6821/38979282] exists,up 2e115328-76c6-478e-915c-5786903be49f', 'osd.3 up in weight 1 up_from 98 up_thru 98 down_at 95 last_clean_interval [26,94) [v2:172.21.15.26:6826/2947396145,v1:172.21.15.26:6827/2947396145] [v2:172.21.15.26:6828/2947396145,v1:172.21.15.26:6829/2947396145] exists,up c3dd2c3f-5cf5-4fb0-964c-1b8926f75c8a', 'osd.4 up in weight 1 up_from 31 up_thru 98 down_at 0 last_clean_interval [0,0) [v2:172.21.15.70:6800/1528249138,v1:172.21.15.70:6801/1528249138] [v2:172.21.15.70:6802/1528249138,v1:172.21.15.70:6803/1528249138] exists,up a5f31996-1b38-491b-956b-5aebc72d0170', 'osd.5 up in weight 1 up_from 39 up_thru 98 down_at 0 last_clean_interval [0,0) [v2:172.21.15.70:6808/3802232906,v1:172.21.15.70:6809/3802232906] [v2:172.21.15.70:6810/3802232906,v1:172.21.15.70:6811/3802232906] exists,up 21dad198-b9db-42c6-8823-112df1ecd874', 'osd.6 up in weight 1 up_from 47 up_thru 95 down_at 0 last_clean_interval [0,0) [v2:172.21.15.70:6816/2944175278,v1:172.21.15.70:6817/2944175278] [v2:172.21.15.70:6818/2944175278,v1:172.21.15.70:6819/2944175278] exists,up 6f3be784-7149-4f13-81b5-98924e97c056', 'osd.7 up in weight 1 up_from 55 up_thru 98 down_at 0 last_clean_interval [0,0) [v2:172.21.15.70:6824/2824974165,v1:172.21.15.70:6825/2824974165] [v2:172.21.15.70:6826/2824974165,v1:172.21.15.70:6827/2824974165] exists,up 499f5f85-f4b2-40a7-ac0f-79cd22738d30'] 2023-09-23T20:52:03.535 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon mon.a config show 2023-09-23T20:52:04.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:52:03 smithi070 ceph-mon[168282]: pgmap v226: 73 pgs: 73 active+clean; 582 KiB data, 171 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:52:04.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:52:03 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/3203116217' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2023-09-23T20:52:04.131 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:52:03 smithi026 ceph-mon[175151]: pgmap v226: 73 pgs: 73 active+clean; 582 KiB data, 171 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:52:04.131 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:52:03 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/3203116217' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2023-09-23T20:52:04.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:52:03 smithi026 ceph-mon[179688]: pgmap v226: 73 pgs: 73 active+clean; 582 KiB data, 171 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:52:04.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:52:03 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/3203116217' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2023-09-23T20:52:04.140 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:52:05.459 INFO:teuthology.orchestra.run.smithi026.stdout:{ 2023-09-23T20:52:05.459 INFO:teuthology.orchestra.run.smithi026.stdout: "name": "mon.a", 2023-09-23T20:52:05.459 INFO:teuthology.orchestra.run.smithi026.stdout: "cluster": "ceph", 2023-09-23T20:52:05.459 INFO:teuthology.orchestra.run.smithi026.stdout: "admin_socket": "/var/run/ceph/ceph-mon.a.asok", 2023-09-23T20:52:05.459 INFO:teuthology.orchestra.run.smithi026.stdout: "admin_socket_mode": "", 2023-09-23T20:52:05.459 INFO:teuthology.orchestra.run.smithi026.stdout: "auth_allow_insecure_global_id_reclaim": "false", 2023-09-23T20:52:05.459 INFO:teuthology.orchestra.run.smithi026.stdout: "auth_client_required": "cephx, none", 2023-09-23T20:52:05.459 INFO:teuthology.orchestra.run.smithi026.stdout: "auth_cluster_required": "cephx", 2023-09-23T20:52:05.459 INFO:teuthology.orchestra.run.smithi026.stdout: "auth_debug": "true", 2023-09-23T20:52:05.460 INFO:teuthology.orchestra.run.smithi026.stdout: "auth_expose_insecure_global_id_reclaim": "true", 2023-09-23T20:52:05.460 INFO:teuthology.orchestra.run.smithi026.stdout: "auth_mon_ticket_ttl": "660.000000", 2023-09-23T20:52:05.460 INFO:teuthology.orchestra.run.smithi026.stdout: "auth_service_required": "cephx", 2023-09-23T20:52:05.460 INFO:teuthology.orchestra.run.smithi026.stdout: "auth_service_ticket_ttl": "240.000000", 2023-09-23T20:52:05.460 INFO:teuthology.orchestra.run.smithi026.stdout: "auth_supported": "", 2023-09-23T20:52:05.460 INFO:teuthology.orchestra.run.smithi026.stdout: "bdev_aio": "true", 2023-09-23T20:52:05.460 INFO:teuthology.orchestra.run.smithi026.stdout: "bdev_aio_max_queue_depth": "1024", 2023-09-23T20:52:05.460 INFO:teuthology.orchestra.run.smithi026.stdout: "bdev_aio_poll_ms": "250", 2023-09-23T20:52:05.460 INFO:teuthology.orchestra.run.smithi026.stdout: "bdev_aio_reap_max": "16", 2023-09-23T20:52:05.460 INFO:teuthology.orchestra.run.smithi026.stdout: "bdev_async_discard": "false", 2023-09-23T20:52:05.460 INFO:teuthology.orchestra.run.smithi026.stdout: "bdev_block_size": "4096", 2023-09-23T20:52:05.461 INFO:teuthology.orchestra.run.smithi026.stdout: "bdev_debug_aio": "false", 2023-09-23T20:52:05.461 INFO:teuthology.orchestra.run.smithi026.stdout: "bdev_debug_aio_log_age": "5.000000", 2023-09-23T20:52:05.461 INFO:teuthology.orchestra.run.smithi026.stdout: "bdev_debug_aio_suicide_timeout": "60.000000", 2023-09-23T20:52:05.461 INFO:teuthology.orchestra.run.smithi026.stdout: "bdev_debug_inflight_ios": "false", 2023-09-23T20:52:05.461 INFO:teuthology.orchestra.run.smithi026.stdout: "bdev_enable_discard": "false", 2023-09-23T20:52:05.461 INFO:teuthology.orchestra.run.smithi026.stdout: "bdev_flock_retry": "3", 2023-09-23T20:52:05.461 INFO:teuthology.orchestra.run.smithi026.stdout: "bdev_flock_retry_interval": "0.100000", 2023-09-23T20:52:05.461 INFO:teuthology.orchestra.run.smithi026.stdout: "bdev_inject_crash": "0", 2023-09-23T20:52:05.461 INFO:teuthology.orchestra.run.smithi026.stdout: "bdev_inject_crash_flush_delay": "2", 2023-09-23T20:52:05.461 INFO:teuthology.orchestra.run.smithi026.stdout: "bdev_ioring": "false", 2023-09-23T20:52:05.461 INFO:teuthology.orchestra.run.smithi026.stdout: "bdev_ioring_hipri": "false", 2023-09-23T20:52:05.461 INFO:teuthology.orchestra.run.smithi026.stdout: "bdev_ioring_sqthread_poll": "false", 2023-09-23T20:52:05.462 INFO:teuthology.orchestra.run.smithi026.stdout: "bdev_nvme_unbind_from_kernel": "false", 2023-09-23T20:52:05.462 INFO:teuthology.orchestra.run.smithi026.stdout: "bdev_read_buffer_alignment": "4096", 2023-09-23T20:52:05.462 INFO:teuthology.orchestra.run.smithi026.stdout: "bdev_read_preallocated_huge_buffers": "", 2023-09-23T20:52:05.462 INFO:teuthology.orchestra.run.smithi026.stdout: "bdev_type": "", 2023-09-23T20:52:05.462 INFO:teuthology.orchestra.run.smithi026.stdout: "bluefs_alloc_size": "1048576", 2023-09-23T20:52:05.462 INFO:teuthology.orchestra.run.smithi026.stdout: "bluefs_allocator": "hybrid", 2023-09-23T20:52:05.462 INFO:teuthology.orchestra.run.smithi026.stdout: "bluefs_buffered_io": "true", 2023-09-23T20:52:05.462 INFO:teuthology.orchestra.run.smithi026.stdout: "bluefs_check_for_zeros": "false", 2023-09-23T20:52:05.462 INFO:teuthology.orchestra.run.smithi026.stdout: "bluefs_check_volume_selector_often": "false", 2023-09-23T20:52:05.462 INFO:teuthology.orchestra.run.smithi026.stdout: "bluefs_check_volume_selector_on_umount": "false", 2023-09-23T20:52:05.462 INFO:teuthology.orchestra.run.smithi026.stdout: "bluefs_compact_log_sync": "false", 2023-09-23T20:52:05.463 INFO:teuthology.orchestra.run.smithi026.stdout: "bluefs_failed_shared_alloc_cooldown": "600.000000", 2023-09-23T20:52:05.463 INFO:teuthology.orchestra.run.smithi026.stdout: "bluefs_log_compact_min_ratio": "5.000000", 2023-09-23T20:52:05.463 INFO:teuthology.orchestra.run.smithi026.stdout: "bluefs_log_compact_min_size": "16777216", 2023-09-23T20:52:05.463 INFO:teuthology.orchestra.run.smithi026.stdout: "bluefs_log_replay_check_allocations": "true", 2023-09-23T20:52:05.463 INFO:teuthology.orchestra.run.smithi026.stdout: "bluefs_max_log_runway": "4194304", 2023-09-23T20:52:05.463 INFO:teuthology.orchestra.run.smithi026.stdout: "bluefs_max_prefetch": "1048576", 2023-09-23T20:52:05.463 INFO:teuthology.orchestra.run.smithi026.stdout: "bluefs_min_flush_size": "524288", 2023-09-23T20:52:05.463 INFO:teuthology.orchestra.run.smithi026.stdout: "bluefs_min_log_runway": "1048576", 2023-09-23T20:52:05.463 INFO:teuthology.orchestra.run.smithi026.stdout: "bluefs_replay_recovery": "false", 2023-09-23T20:52:05.463 INFO:teuthology.orchestra.run.smithi026.stdout: "bluefs_replay_recovery_disable_compact": "false", 2023-09-23T20:52:05.463 INFO:teuthology.orchestra.run.smithi026.stdout: "bluefs_shared_alloc_size": "65536", 2023-09-23T20:52:05.463 INFO:teuthology.orchestra.run.smithi026.stdout: "bluefs_sync_write": "false", 2023-09-23T20:52:05.463 INFO:teuthology.orchestra.run.smithi026.stdout: "bluestore_2q_cache_kin_ratio": "0.500000", 2023-09-23T20:52:05.464 INFO:teuthology.orchestra.run.smithi026.stdout: "bluestore_2q_cache_kout_ratio": "0.500000", 2023-09-23T20:52:05.464 INFO:teuthology.orchestra.run.smithi026.stdout: "bluestore_alloc_stats_dump_interval": "86400.000000", 2023-09-23T20:52:05.464 INFO:teuthology.orchestra.run.smithi026.stdout: "bluestore_allocation_from_file": "true", 2023-09-23T20:52:05.464 INFO:teuthology.orchestra.run.smithi026.stdout: "bluestore_allocator": "hybrid", 2023-09-23T20:52:05.464 INFO:teuthology.orchestra.run.smithi026.stdout: "bluestore_avl_alloc_bf_free_pct": "4", 2023-09-23T20:52:05.464 INFO:teuthology.orchestra.run.smithi026.stdout: "bluestore_avl_alloc_bf_threshold": "131072", 2023-09-23T20:52:05.464 INFO:teuthology.orchestra.run.smithi026.stdout: "bluestore_avl_alloc_ff_max_search_bytes": "16777216", 2023-09-23T20:52:05.464 INFO:teuthology.orchestra.run.smithi026.stdout: "bluestore_avl_alloc_ff_max_search_count": "100", 2023-09-23T20:52:05.464 INFO:teuthology.orchestra.run.smithi026.stdout: "bluestore_bitmapallocator_blocks_per_zone": "1024", 2023-09-23T20:52:05.464 INFO:teuthology.orchestra.run.smithi026.stdout: "bluestore_bitmapallocator_span_size": "1024", 2023-09-23T20:52:05.464 INFO:teuthology.orchestra.run.smithi026.stdout: "bluestore_blobid_prealloc": "10240", 2023-09-23T20:52:05.464 INFO:teuthology.orchestra.run.smithi026.stdout: "bluestore_block_create": "true", 2023-09-23T20:52:05.465 INFO:teuthology.orchestra.run.smithi026.stdout: "bluestore_block_db_create": "false", 2023-09-23T20:52:05.465 INFO:teuthology.orchestra.run.smithi026.stdout: "bluestore_block_db_path": "", 2023-09-23T20:52:05.465 INFO:teuthology.orchestra.run.smithi026.stdout: "bluestore_block_db_size": "0", 2023-09-23T20:52:05.465 INFO:teuthology.orchestra.run.smithi026.stdout: "bluestore_block_path": "", 2023-09-23T20:52:05.465 INFO:teuthology.orchestra.run.smithi026.stdout: "bluestore_block_preallocate_file": "false", 2023-09-23T20:52:05.465 INFO:teuthology.orchestra.run.smithi026.stdout: "bluestore_block_size": "107374182400", 2023-09-23T20:52:05.465 INFO:teuthology.orchestra.run.smithi026.stdout: "bluestore_block_wal_create": "false", 2023-09-23T20:52:05.465 INFO:teuthology.orchestra.run.smithi026.stdout: "bluestore_block_wal_path": "", 2023-09-23T20:52:05.465 INFO:teuthology.orchestra.run.smithi026.stdout: "bluestore_block_wal_size": "100663296", 2023-09-23T20:52:05.465 INFO:teuthology.orchestra.run.smithi026.stdout: "bluestore_bluefs": "true", 2023-09-23T20:52:05.465 INFO:teuthology.orchestra.run.smithi026.stdout: "bluestore_bluefs_alloc_failure_dump_interval": "0.000000", 2023-09-23T20:52:05.465 INFO:teuthology.orchestra.run.smithi026.stdout: "bluestore_bluefs_env_mirror": "false", 2023-09-23T20:52:05.466 INFO:teuthology.orchestra.run.smithi026.stdout: "bluestore_bluefs_max_free": "10737418240", 2023-09-23T20:52:05.466 INFO:teuthology.orchestra.run.smithi026.stdout: "bluestore_cache_age_bin_interval": "1.000000", 2023-09-23T20:52:05.466 INFO:teuthology.orchestra.run.smithi026.stdout: "bluestore_cache_age_bins_data": "1 2 6 24 120 720 0 0 0 0", 2023-09-23T20:52:05.466 INFO:teuthology.orchestra.run.smithi026.stdout: "bluestore_cache_age_bins_kv": "1 2 6 24 120 720 0 0 0 0", 2023-09-23T20:52:05.466 INFO:teuthology.orchestra.run.smithi026.stdout: "bluestore_cache_age_bins_kv_onode": "0 0 0 0 0 0 0 0 0 720", 2023-09-23T20:52:05.466 INFO:teuthology.orchestra.run.smithi026.stdout: "bluestore_cache_age_bins_meta": "1 2 6 24 120 720 0 0 0 0", 2023-09-23T20:52:05.466 INFO:teuthology.orchestra.run.smithi026.stdout: "bluestore_cache_autotune": "true", 2023-09-23T20:52:05.466 INFO:teuthology.orchestra.run.smithi026.stdout: "bluestore_cache_autotune_interval": "5.000000", 2023-09-23T20:52:05.466 INFO:teuthology.orchestra.run.smithi026.stdout: "bluestore_cache_kv_onode_ratio": "0.040000", 2023-09-23T20:52:05.466 INFO:teuthology.orchestra.run.smithi026.stdout: "bluestore_cache_kv_ratio": "0.450000", 2023-09-23T20:52:05.466 INFO:teuthology.orchestra.run.smithi026.stdout: "bluestore_cache_meta_ratio": "0.450000", 2023-09-23T20:52:05.466 INFO:teuthology.orchestra.run.smithi026.stdout: "bluestore_cache_size": "0", 2023-09-23T20:52:05.467 INFO:teuthology.orchestra.run.smithi026.stdout: "bluestore_cache_size_hdd": "1073741824", 2023-09-23T20:52:05.467 INFO:teuthology.orchestra.run.smithi026.stdout: "bluestore_cache_size_ssd": "3221225472", 2023-09-23T20:52:05.467 INFO:teuthology.orchestra.run.smithi026.stdout: "bluestore_cache_trim_interval": "0.050000", 2023-09-23T20:52:05.467 INFO:teuthology.orchestra.run.smithi026.stdout: "bluestore_cache_trim_max_skip_pinned": "1000", 2023-09-23T20:52:05.467 INFO:teuthology.orchestra.run.smithi026.stdout: "bluestore_cache_type": "2q", 2023-09-23T20:52:05.467 INFO:teuthology.orchestra.run.smithi026.stdout: "bluestore_cleaner_sleep_interval": "5.000000", 2023-09-23T20:52:05.467 INFO:teuthology.orchestra.run.smithi026.stdout: "bluestore_clone_cow": "true", 2023-09-23T20:52:05.467 INFO:teuthology.orchestra.run.smithi026.stdout: "bluestore_compression_algorithm": "snappy", 2023-09-23T20:52:05.467 INFO:teuthology.orchestra.run.smithi026.stdout: "bluestore_compression_max_blob_size": "0", 2023-09-23T20:52:05.467 INFO:teuthology.orchestra.run.smithi026.stdout: "bluestore_compression_max_blob_size_hdd": "65536", 2023-09-23T20:52:05.467 INFO:teuthology.orchestra.run.smithi026.stdout: "bluestore_compression_max_blob_size_ssd": "65536", 2023-09-23T20:52:05.467 INFO:teuthology.orchestra.run.smithi026.stdout: "bluestore_compression_min_blob_size": "0", 2023-09-23T20:52:05.468 INFO:teuthology.orchestra.run.smithi026.stdout: "bluestore_compression_min_blob_size_hdd": "8192", 2023-09-23T20:52:05.468 INFO:teuthology.orchestra.run.smithi026.stdout: "bluestore_compression_min_blob_size_ssd": "65536", 2023-09-23T20:52:05.468 INFO:teuthology.orchestra.run.smithi026.stdout: "bluestore_compression_mode": "none", 2023-09-23T20:52:05.468 INFO:teuthology.orchestra.run.smithi026.stdout: "bluestore_compression_required_ratio": "0.875000", 2023-09-23T20:52:05.468 INFO:teuthology.orchestra.run.smithi026.stdout: "bluestore_csum_type": "crc32c", 2023-09-23T20:52:05.468 INFO:teuthology.orchestra.run.smithi026.stdout: "bluestore_debug_enforce_settings": "default", 2023-09-23T20:52:05.468 INFO:teuthology.orchestra.run.smithi026.stdout: "bluestore_debug_freelist": "false", 2023-09-23T20:52:05.468 INFO:teuthology.orchestra.run.smithi026.stdout: "bluestore_debug_fsck_abort": "false", 2023-09-23T20:52:05.468 INFO:teuthology.orchestra.run.smithi026.stdout: "bluestore_debug_inject_allocation_from_file_failure": "0.000000", 2023-09-23T20:52:05.468 INFO:teuthology.orchestra.run.smithi026.stdout: "bluestore_debug_inject_bug21040": "false", 2023-09-23T20:52:05.468 INFO:teuthology.orchestra.run.smithi026.stdout: "bluestore_debug_inject_csum_err_probability": "0.000000", 2023-09-23T20:52:05.468 INFO:teuthology.orchestra.run.smithi026.stdout: "bluestore_debug_inject_read_err": "false", 2023-09-23T20:52:05.468 INFO:teuthology.orchestra.run.smithi026.stdout: "bluestore_debug_legacy_omap": "false", 2023-09-23T20:52:05.469 INFO:teuthology.orchestra.run.smithi026.stdout: "bluestore_debug_no_reuse_blocks": "false", 2023-09-23T20:52:05.469 INFO:teuthology.orchestra.run.smithi026.stdout: "bluestore_debug_omit_block_device_write": "false", 2023-09-23T20:52:05.469 INFO:teuthology.orchestra.run.smithi026.stdout: "bluestore_debug_omit_kv_commit": "false", 2023-09-23T20:52:05.469 INFO:teuthology.orchestra.run.smithi026.stdout: "bluestore_debug_permit_any_bdev_label": "false", 2023-09-23T20:52:05.469 INFO:teuthology.orchestra.run.smithi026.stdout: "bluestore_debug_prefill": "0.000000", 2023-09-23T20:52:05.469 INFO:teuthology.orchestra.run.smithi026.stdout: "bluestore_debug_prefragment_max": "1048576", 2023-09-23T20:52:05.469 INFO:teuthology.orchestra.run.smithi026.stdout: "bluestore_debug_random_read_err": "0.000000", 2023-09-23T20:52:05.469 INFO:teuthology.orchestra.run.smithi026.stdout: "bluestore_debug_randomize_serial_transaction": "0", 2023-09-23T20:52:05.469 INFO:teuthology.orchestra.run.smithi026.stdout: "bluestore_debug_small_allocations": "0", 2023-09-23T20:52:05.469 INFO:teuthology.orchestra.run.smithi026.stdout: "bluestore_debug_too_many_blobs_threshold": "24576", 2023-09-23T20:52:05.469 INFO:teuthology.orchestra.run.smithi026.stdout: "bluestore_default_buffered_read": "true", 2023-09-23T20:52:05.469 INFO:teuthology.orchestra.run.smithi026.stdout: "bluestore_default_buffered_write": "false", 2023-09-23T20:52:05.470 INFO:teuthology.orchestra.run.smithi026.stdout: "bluestore_deferred_batch_ops": "0", 2023-09-23T20:52:05.470 INFO:teuthology.orchestra.run.smithi026.stdout: "bluestore_deferred_batch_ops_hdd": "64", 2023-09-23T20:52:05.470 INFO:teuthology.orchestra.run.smithi026.stdout: "bluestore_deferred_batch_ops_ssd": "16", 2023-09-23T20:52:05.470 INFO:teuthology.orchestra.run.smithi026.stdout: "bluestore_extent_map_inline_shard_prealloc_size": "256", 2023-09-23T20:52:05.470 INFO:teuthology.orchestra.run.smithi026.stdout: "bluestore_extent_map_shard_max_size": "1200", 2023-09-23T20:52:05.470 INFO:teuthology.orchestra.run.smithi026.stdout: "bluestore_extent_map_shard_min_size": "150", 2023-09-23T20:52:05.470 INFO:teuthology.orchestra.run.smithi026.stdout: "bluestore_extent_map_shard_target_size": "500", 2023-09-23T20:52:05.470 INFO:teuthology.orchestra.run.smithi026.stdout: "bluestore_extent_map_shard_target_size_slop": "0.200000", 2023-09-23T20:52:05.470 INFO:teuthology.orchestra.run.smithi026.stdout: "bluestore_fail_eio": "false", 2023-09-23T20:52:05.470 INFO:teuthology.orchestra.run.smithi026.stdout: "bluestore_freelist_blocks_per_key": "128", 2023-09-23T20:52:05.470 INFO:teuthology.orchestra.run.smithi026.stdout: "bluestore_fsck_error_on_no_per_pg_omap": "false", 2023-09-23T20:52:05.471 INFO:teuthology.orchestra.run.smithi026.stdout: "bluestore_fsck_error_on_no_per_pool_omap": "false", 2023-09-23T20:52:05.471 INFO:teuthology.orchestra.run.smithi026.stdout: "bluestore_fsck_error_on_no_per_pool_stats": "false", 2023-09-23T20:52:05.471 INFO:teuthology.orchestra.run.smithi026.stdout: "bluestore_fsck_on_mkfs": "true", 2023-09-23T20:52:05.471 INFO:teuthology.orchestra.run.smithi026.stdout: "bluestore_fsck_on_mkfs_deep": "false", 2023-09-23T20:52:05.471 INFO:teuthology.orchestra.run.smithi026.stdout: "bluestore_fsck_on_mount": "false", 2023-09-23T20:52:05.471 INFO:teuthology.orchestra.run.smithi026.stdout: "bluestore_fsck_on_mount_deep": "false", 2023-09-23T20:52:05.471 INFO:teuthology.orchestra.run.smithi026.stdout: "bluestore_fsck_on_umount": "false", 2023-09-23T20:52:05.471 INFO:teuthology.orchestra.run.smithi026.stdout: "bluestore_fsck_on_umount_deep": "false", 2023-09-23T20:52:05.471 INFO:teuthology.orchestra.run.smithi026.stdout: "bluestore_fsck_quick_fix_on_mount": "false", 2023-09-23T20:52:05.471 INFO:teuthology.orchestra.run.smithi026.stdout: "bluestore_fsck_quick_fix_threads": "2", 2023-09-23T20:52:05.471 INFO:teuthology.orchestra.run.smithi026.stdout: "bluestore_fsck_read_bytes_cap": "67108864", 2023-09-23T20:52:05.471 INFO:teuthology.orchestra.run.smithi026.stdout: "bluestore_fsck_shared_blob_tracker_size": "0.031250", 2023-09-23T20:52:05.471 INFO:teuthology.orchestra.run.smithi026.stdout: "bluestore_gc_enable_blob_threshold": "0", 2023-09-23T20:52:05.472 INFO:teuthology.orchestra.run.smithi026.stdout: "bluestore_gc_enable_total_threshold": "0", 2023-09-23T20:52:05.472 INFO:teuthology.orchestra.run.smithi026.stdout: "bluestore_hybrid_alloc_mem_cap": "67108864", 2023-09-23T20:52:05.472 INFO:teuthology.orchestra.run.smithi026.stdout: "bluestore_ignore_data_csum": "false", 2023-09-23T20:52:05.472 INFO:teuthology.orchestra.run.smithi026.stdout: "bluestore_kv_sync_util_logging_s": "10.000000", 2023-09-23T20:52:05.472 INFO:teuthology.orchestra.run.smithi026.stdout: "bluestore_kvbackend": "rocksdb", 2023-09-23T20:52:05.472 INFO:teuthology.orchestra.run.smithi026.stdout: "bluestore_log_collection_list_age": "60.000000", 2023-09-23T20:52:05.472 INFO:teuthology.orchestra.run.smithi026.stdout: "bluestore_log_omap_iterator_age": "5.000000", 2023-09-23T20:52:05.472 INFO:teuthology.orchestra.run.smithi026.stdout: "bluestore_log_op_age": "5.000000", 2023-09-23T20:52:05.472 INFO:teuthology.orchestra.run.smithi026.stdout: "bluestore_max_alloc_size": "0", 2023-09-23T20:52:05.472 INFO:teuthology.orchestra.run.smithi026.stdout: "bluestore_max_blob_size": "0", 2023-09-23T20:52:05.472 INFO:teuthology.orchestra.run.smithi026.stdout: "bluestore_max_blob_size_hdd": "65536", 2023-09-23T20:52:05.472 INFO:teuthology.orchestra.run.smithi026.stdout: "bluestore_max_blob_size_ssd": "65536", 2023-09-23T20:52:05.473 INFO:teuthology.orchestra.run.smithi026.stdout: "bluestore_max_defer_interval": "3.000000", 2023-09-23T20:52:05.473 INFO:teuthology.orchestra.run.smithi026.stdout: "bluestore_max_deferred_txc": "32", 2023-09-23T20:52:05.473 INFO:teuthology.orchestra.run.smithi026.stdout: "bluestore_min_alloc_size": "0", 2023-09-23T20:52:05.473 INFO:teuthology.orchestra.run.smithi026.stdout: "bluestore_min_alloc_size_hdd": "4096", 2023-09-23T20:52:05.473 INFO:teuthology.orchestra.run.smithi026.stdout: "bluestore_min_alloc_size_ssd": "4096", 2023-09-23T20:52:05.473 INFO:teuthology.orchestra.run.smithi026.stdout: "bluestore_nid_prealloc": "1024", 2023-09-23T20:52:05.473 INFO:teuthology.orchestra.run.smithi026.stdout: "bluestore_prefer_deferred_size": "0", 2023-09-23T20:52:05.473 INFO:teuthology.orchestra.run.smithi026.stdout: "bluestore_prefer_deferred_size_hdd": "65536", 2023-09-23T20:52:05.473 INFO:teuthology.orchestra.run.smithi026.stdout: "bluestore_prefer_deferred_size_ssd": "0", 2023-09-23T20:52:05.473 INFO:teuthology.orchestra.run.smithi026.stdout: "bluestore_qfsck_on_mount": "true", 2023-09-23T20:52:05.473 INFO:teuthology.orchestra.run.smithi026.stdout: "bluestore_retry_disk_reads": "3", 2023-09-23T20:52:05.473 INFO:teuthology.orchestra.run.smithi026.stdout: "bluestore_rocksdb_cf": "true", 2023-09-23T20:52:05.473 INFO:teuthology.orchestra.run.smithi026.stdout: "bluestore_rocksdb_cfs": "m(3) p(3,0-12) O(3,0-13)=block_cache={type=binned_lru} L=min_write_buffer_number_to_merge=32 P=min_write_buffer_number_to_merge=32", 2023-09-23T20:52:05.474 INFO:teuthology.orchestra.run.smithi026.stdout: "bluestore_rocksdb_options": "compression=kNoCompression,max_write_buffer_number=64,min_write_buffer_number_to_merge=6,compaction_style=kCompactionStyleLevel,write_buffer_size=16777216,max_background_jobs=4,level0_file_num_compaction_trigger=8,max_bytes_for_level_base=1073741824,max_bytes_for_level_multiplier=8,compaction_readahead_size=2MB,max_total_wal_size=1073741824,writable_file_max_buffer_size=0", 2023-09-23T20:52:05.474 INFO:teuthology.orchestra.run.smithi026.stdout: "bluestore_rocksdb_options_annex": "", 2023-09-23T20:52:05.474 INFO:teuthology.orchestra.run.smithi026.stdout: "bluestore_spdk_coremask": "0x1", 2023-09-23T20:52:05.474 INFO:teuthology.orchestra.run.smithi026.stdout: "bluestore_spdk_io_sleep": "5", 2023-09-23T20:52:05.474 INFO:teuthology.orchestra.run.smithi026.stdout: "bluestore_spdk_max_io_completion": "0", 2023-09-23T20:52:05.474 INFO:teuthology.orchestra.run.smithi026.stdout: "bluestore_spdk_mem": "512", 2023-09-23T20:52:05.474 INFO:teuthology.orchestra.run.smithi026.stdout: "bluestore_sync_submit_transaction": "false", 2023-09-23T20:52:05.474 INFO:teuthology.orchestra.run.smithi026.stdout: "bluestore_throttle_bytes": "67108864", 2023-09-23T20:52:05.474 INFO:teuthology.orchestra.run.smithi026.stdout: "bluestore_throttle_cost_per_io": "0", 2023-09-23T20:52:05.474 INFO:teuthology.orchestra.run.smithi026.stdout: "bluestore_throttle_cost_per_io_hdd": "670000", 2023-09-23T20:52:05.474 INFO:teuthology.orchestra.run.smithi026.stdout: "bluestore_throttle_cost_per_io_ssd": "4000", 2023-09-23T20:52:05.475 INFO:teuthology.orchestra.run.smithi026.stdout: "bluestore_throttle_deferred_bytes": "134217728", 2023-09-23T20:52:05.475 INFO:teuthology.orchestra.run.smithi026.stdout: "bluestore_throttle_trace_rate": "0.000000", 2023-09-23T20:52:05.475 INFO:teuthology.orchestra.run.smithi026.stdout: "bluestore_tracing": "false", 2023-09-23T20:52:05.475 INFO:teuthology.orchestra.run.smithi026.stdout: "bluestore_use_optimal_io_size_for_min_alloc_size": "false", 2023-09-23T20:52:05.475 INFO:teuthology.orchestra.run.smithi026.stdout: "bluestore_volume_selection_policy": "use_some_extra", 2023-09-23T20:52:05.475 INFO:teuthology.orchestra.run.smithi026.stdout: "bluestore_volume_selection_reserved": "0", 2023-09-23T20:52:05.475 INFO:teuthology.orchestra.run.smithi026.stdout: "bluestore_volume_selection_reserved_factor": "2.000000", 2023-09-23T20:52:05.475 INFO:teuthology.orchestra.run.smithi026.stdout: "bluestore_warn_on_bluefs_spillover": "true", 2023-09-23T20:52:05.475 INFO:teuthology.orchestra.run.smithi026.stdout: "bluestore_warn_on_legacy_statfs": "true", 2023-09-23T20:52:05.475 INFO:teuthology.orchestra.run.smithi026.stdout: "bluestore_warn_on_no_per_pg_omap": "false", 2023-09-23T20:52:05.475 INFO:teuthology.orchestra.run.smithi026.stdout: "bluestore_warn_on_no_per_pool_omap": "true", 2023-09-23T20:52:05.476 INFO:teuthology.orchestra.run.smithi026.stdout: "bluestore_warn_on_spurious_read_errors": "true", 2023-09-23T20:52:05.476 INFO:teuthology.orchestra.run.smithi026.stdout: "bluestore_zero_block_detection": "false", 2023-09-23T20:52:05.476 INFO:teuthology.orchestra.run.smithi026.stdout: "cephadm_path": "/usr/sbin/cephadm", 2023-09-23T20:52:05.476 INFO:teuthology.orchestra.run.smithi026.stdout: "cephfs_mirror_action_update_interval": "2", 2023-09-23T20:52:05.476 INFO:teuthology.orchestra.run.smithi026.stdout: "cephfs_mirror_directory_scan_interval": "10", 2023-09-23T20:52:05.476 INFO:teuthology.orchestra.run.smithi026.stdout: "cephfs_mirror_max_concurrent_directory_syncs": "3", 2023-09-23T20:52:05.476 INFO:teuthology.orchestra.run.smithi026.stdout: "cephfs_mirror_max_consecutive_failures_per_directory": "10", 2023-09-23T20:52:05.476 INFO:teuthology.orchestra.run.smithi026.stdout: "cephfs_mirror_max_snapshot_sync_per_cycle": "3", 2023-09-23T20:52:05.476 INFO:teuthology.orchestra.run.smithi026.stdout: "cephfs_mirror_mount_timeout": "10", 2023-09-23T20:52:05.476 INFO:teuthology.orchestra.run.smithi026.stdout: "cephfs_mirror_restart_mirror_on_blocklist_interval": "30", 2023-09-23T20:52:05.476 INFO:teuthology.orchestra.run.smithi026.stdout: "cephfs_mirror_restart_mirror_on_failure_interval": "20", 2023-09-23T20:52:05.476 INFO:teuthology.orchestra.run.smithi026.stdout: "cephfs_mirror_retry_failed_directories_interval": "60", 2023-09-23T20:52:05.476 INFO:teuthology.orchestra.run.smithi026.stdout: "cephsqlite_blocklist_dead_locker": "true", 2023-09-23T20:52:05.477 INFO:teuthology.orchestra.run.smithi026.stdout: "cephsqlite_lock_renewal_interval": "2000", 2023-09-23T20:52:05.477 INFO:teuthology.orchestra.run.smithi026.stdout: "cephsqlite_lock_renewal_timeout": "30000", 2023-09-23T20:52:05.477 INFO:teuthology.orchestra.run.smithi026.stdout: "cephx_cluster_require_signatures": "false", 2023-09-23T20:52:05.477 INFO:teuthology.orchestra.run.smithi026.stdout: "cephx_cluster_require_version": "2", 2023-09-23T20:52:05.477 INFO:teuthology.orchestra.run.smithi026.stdout: "cephx_require_signatures": "false", 2023-09-23T20:52:05.477 INFO:teuthology.orchestra.run.smithi026.stdout: "cephx_require_version": "2", 2023-09-23T20:52:05.477 INFO:teuthology.orchestra.run.smithi026.stdout: "cephx_service_require_signatures": "false", 2023-09-23T20:52:05.477 INFO:teuthology.orchestra.run.smithi026.stdout: "cephx_service_require_version": "2", 2023-09-23T20:52:05.477 INFO:teuthology.orchestra.run.smithi026.stdout: "cephx_sign_messages": "true", 2023-09-23T20:52:05.477 INFO:teuthology.orchestra.run.smithi026.stdout: "chdir": "", 2023-09-23T20:52:05.477 INFO:teuthology.orchestra.run.smithi026.stdout: "client_acl_type": "", 2023-09-23T20:52:05.477 INFO:teuthology.orchestra.run.smithi026.stdout: "client_asio_thread_count": "2", 2023-09-23T20:52:05.477 INFO:teuthology.orchestra.run.smithi026.stdout: "client_cache_mid": "0.750000", 2023-09-23T20:52:05.478 INFO:teuthology.orchestra.run.smithi026.stdout: "client_cache_size": "16384", 2023-09-23T20:52:05.478 INFO:teuthology.orchestra.run.smithi026.stdout: "client_caps_release_delay": "5", 2023-09-23T20:52:05.478 INFO:teuthology.orchestra.run.smithi026.stdout: "client_check_pool_perm": "true", 2023-09-23T20:52:05.478 INFO:teuthology.orchestra.run.smithi026.stdout: "client_collect_and_send_global_metrics": "false", 2023-09-23T20:52:05.478 INFO:teuthology.orchestra.run.smithi026.stdout: "client_debug_force_sync_read": "false", 2023-09-23T20:52:05.478 INFO:teuthology.orchestra.run.smithi026.stdout: "client_debug_getattr_caps": "false", 2023-09-23T20:52:05.478 INFO:teuthology.orchestra.run.smithi026.stdout: "client_debug_inject_tick_delay": "0", 2023-09-23T20:52:05.478 INFO:teuthology.orchestra.run.smithi026.stdout: "client_die_on_failed_dentry_invalidate": "true", 2023-09-23T20:52:05.478 INFO:teuthology.orchestra.run.smithi026.stdout: "client_die_on_failed_remount": "false", 2023-09-23T20:52:05.478 INFO:teuthology.orchestra.run.smithi026.stdout: "client_dirsize_rbytes": "true", 2023-09-23T20:52:05.478 INFO:teuthology.orchestra.run.smithi026.stdout: "client_force_lazyio": "false", 2023-09-23T20:52:05.478 INFO:teuthology.orchestra.run.smithi026.stdout: "client_fs": "", 2023-09-23T20:52:05.479 INFO:teuthology.orchestra.run.smithi026.stdout: "client_inject_fixed_oldest_tid": "false", 2023-09-23T20:52:05.479 INFO:teuthology.orchestra.run.smithi026.stdout: "client_inject_release_failure": "false", 2023-09-23T20:52:05.479 INFO:teuthology.orchestra.run.smithi026.stdout: "client_max_inline_size": "4096", 2023-09-23T20:52:05.479 INFO:teuthology.orchestra.run.smithi026.stdout: "client_max_retries_on_remount_failure": "5", 2023-09-23T20:52:05.479 INFO:teuthology.orchestra.run.smithi026.stdout: "client_mds_namespace": "", 2023-09-23T20:52:05.479 INFO:teuthology.orchestra.run.smithi026.stdout: "client_metadata": "", 2023-09-23T20:52:05.479 INFO:teuthology.orchestra.run.smithi026.stdout: "client_mount_gid": "-1", 2023-09-23T20:52:05.479 INFO:teuthology.orchestra.run.smithi026.stdout: "client_mount_timeout": "300", 2023-09-23T20:52:05.479 INFO:teuthology.orchestra.run.smithi026.stdout: "client_mount_uid": "-1", 2023-09-23T20:52:05.479 INFO:teuthology.orchestra.run.smithi026.stdout: "client_mountpoint": "/", 2023-09-23T20:52:05.479 INFO:teuthology.orchestra.run.smithi026.stdout: "client_notify_timeout": "10", 2023-09-23T20:52:05.479 INFO:teuthology.orchestra.run.smithi026.stdout: "client_oc": "true", 2023-09-23T20:52:05.480 INFO:teuthology.orchestra.run.smithi026.stdout: "client_oc_max_dirty": "104857600", 2023-09-23T20:52:05.480 INFO:teuthology.orchestra.run.smithi026.stdout: "client_oc_max_dirty_age": "5.000000", 2023-09-23T20:52:05.480 INFO:teuthology.orchestra.run.smithi026.stdout: "client_oc_max_objects": "1000", 2023-09-23T20:52:05.480 INFO:teuthology.orchestra.run.smithi026.stdout: "client_oc_size": "209715200", 2023-09-23T20:52:05.480 INFO:teuthology.orchestra.run.smithi026.stdout: "client_oc_target_dirty": "8388608", 2023-09-23T20:52:05.480 INFO:teuthology.orchestra.run.smithi026.stdout: "client_permissions": "true", 2023-09-23T20:52:05.480 INFO:teuthology.orchestra.run.smithi026.stdout: "client_quota": "true", 2023-09-23T20:52:05.480 INFO:teuthology.orchestra.run.smithi026.stdout: "client_quota_df": "true", 2023-09-23T20:52:05.480 INFO:teuthology.orchestra.run.smithi026.stdout: "client_readahead_max_bytes": "0", 2023-09-23T20:52:05.480 INFO:teuthology.orchestra.run.smithi026.stdout: "client_readahead_max_periods": "4", 2023-09-23T20:52:05.480 INFO:teuthology.orchestra.run.smithi026.stdout: "client_readahead_min": "131072", 2023-09-23T20:52:05.480 INFO:teuthology.orchestra.run.smithi026.stdout: "client_reconnect_stale": "false", 2023-09-23T20:52:05.480 INFO:teuthology.orchestra.run.smithi026.stdout: "client_shutdown_timeout": "30", 2023-09-23T20:52:05.481 INFO:teuthology.orchestra.run.smithi026.stdout: "client_snapdir": ".snap", 2023-09-23T20:52:05.481 INFO:teuthology.orchestra.run.smithi026.stdout: "client_tick_interval": "1", 2023-09-23T20:52:05.481 INFO:teuthology.orchestra.run.smithi026.stdout: "client_trace": "", 2023-09-23T20:52:05.481 INFO:teuthology.orchestra.run.smithi026.stdout: "client_try_dentry_invalidate": "false", 2023-09-23T20:52:05.481 INFO:teuthology.orchestra.run.smithi026.stdout: "client_use_faked_inos": "false", 2023-09-23T20:52:05.481 INFO:teuthology.orchestra.run.smithi026.stdout: "client_use_random_mds": "false", 2023-09-23T20:52:05.481 INFO:teuthology.orchestra.run.smithi026.stdout: "clog_to_graylog": "false", 2023-09-23T20:52:05.481 INFO:teuthology.orchestra.run.smithi026.stdout: "clog_to_graylog_host": "127.0.0.1", 2023-09-23T20:52:05.481 INFO:teuthology.orchestra.run.smithi026.stdout: "clog_to_graylog_port": "12201", 2023-09-23T20:52:05.481 INFO:teuthology.orchestra.run.smithi026.stdout: "clog_to_monitors": "default=true", 2023-09-23T20:52:05.481 INFO:teuthology.orchestra.run.smithi026.stdout: "clog_to_syslog": "false", 2023-09-23T20:52:05.481 INFO:teuthology.orchestra.run.smithi026.stdout: "clog_to_syslog_facility": "default=daemon audit=local0", 2023-09-23T20:52:05.481 INFO:teuthology.orchestra.run.smithi026.stdout: "clog_to_syslog_level": "info", 2023-09-23T20:52:05.482 INFO:teuthology.orchestra.run.smithi026.stdout: "cluster_addr": "-", 2023-09-23T20:52:05.482 INFO:teuthology.orchestra.run.smithi026.stdout: "cluster_network": "", 2023-09-23T20:52:05.482 INFO:teuthology.orchestra.run.smithi026.stdout: "cluster_network_interface": "", 2023-09-23T20:52:05.482 INFO:teuthology.orchestra.run.smithi026.stdout: "compressor_zlib_isal": "false", 2023-09-23T20:52:05.482 INFO:teuthology.orchestra.run.smithi026.stdout: "compressor_zlib_level": "5", 2023-09-23T20:52:05.482 INFO:teuthology.orchestra.run.smithi026.stdout: "compressor_zlib_winsize": "-15", 2023-09-23T20:52:05.482 INFO:teuthology.orchestra.run.smithi026.stdout: "compressor_zstd_level": "1", 2023-09-23T20:52:05.482 INFO:teuthology.orchestra.run.smithi026.stdout: "container_image": "quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d", 2023-09-23T20:52:05.482 INFO:teuthology.orchestra.run.smithi026.stdout: "crash_dir": "/var/lib/ceph/crash", 2023-09-23T20:52:05.482 INFO:teuthology.orchestra.run.smithi026.stdout: "crimson_alien_op_num_threads": "6", 2023-09-23T20:52:05.482 INFO:teuthology.orchestra.run.smithi026.stdout: "crimson_alien_thread_cpu_cores": "", 2023-09-23T20:52:05.483 INFO:teuthology.orchestra.run.smithi026.stdout: "crimson_osd_obc_lru_size": "10", 2023-09-23T20:52:05.483 INFO:teuthology.orchestra.run.smithi026.stdout: "crimson_osd_scheduler_concurrency": "0", 2023-09-23T20:52:05.483 INFO:teuthology.orchestra.run.smithi026.stdout: "crimson_seastar_smp": "1", 2023-09-23T20:52:05.483 INFO:teuthology.orchestra.run.smithi026.stdout: "crush_location": "", 2023-09-23T20:52:05.483 INFO:teuthology.orchestra.run.smithi026.stdout: "crush_location_hook": "", 2023-09-23T20:52:05.483 INFO:teuthology.orchestra.run.smithi026.stdout: "crush_location_hook_timeout": "10", 2023-09-23T20:52:05.483 INFO:teuthology.orchestra.run.smithi026.stdout: "daemonize": "false", 2023-09-23T20:52:05.483 INFO:teuthology.orchestra.run.smithi026.stdout: "daos_pool": "tank", 2023-09-23T20:52:05.483 INFO:teuthology.orchestra.run.smithi026.stdout: "dbstore_config_uri": "file:/var/lib/ceph/radosgw/dbstore-config.db", 2023-09-23T20:52:05.483 INFO:teuthology.orchestra.run.smithi026.stdout: "dbstore_db_dir": "/var/lib/ceph/radosgw", 2023-09-23T20:52:05.483 INFO:teuthology.orchestra.run.smithi026.stdout: "dbstore_db_name_prefix": "dbstore", 2023-09-23T20:52:05.483 INFO:teuthology.orchestra.run.smithi026.stdout: "debug_alienstore": "0/5", 2023-09-23T20:52:05.484 INFO:teuthology.orchestra.run.smithi026.stdout: "debug_allow_any_pool_priority": "false", 2023-09-23T20:52:05.484 INFO:teuthology.orchestra.run.smithi026.stdout: "debug_asok": "1/5", 2023-09-23T20:52:05.484 INFO:teuthology.orchestra.run.smithi026.stdout: "debug_asok_assert_abort": "false", 2023-09-23T20:52:05.484 INFO:teuthology.orchestra.run.smithi026.stdout: "debug_asserts_on_shutdown": "true", 2023-09-23T20:52:05.484 INFO:teuthology.orchestra.run.smithi026.stdout: "debug_auth": "1/5", 2023-09-23T20:52:05.484 INFO:teuthology.orchestra.run.smithi026.stdout: "debug_bdev": "1/3", 2023-09-23T20:52:05.484 INFO:teuthology.orchestra.run.smithi026.stdout: "debug_bluefs": "1/5", 2023-09-23T20:52:05.484 INFO:teuthology.orchestra.run.smithi026.stdout: "debug_bluestore": "1/5", 2023-09-23T20:52:05.484 INFO:teuthology.orchestra.run.smithi026.stdout: "debug_buffer": "0/1", 2023-09-23T20:52:05.484 INFO:teuthology.orchestra.run.smithi026.stdout: "debug_ceph_exporter": "1/5", 2023-09-23T20:52:05.484 INFO:teuthology.orchestra.run.smithi026.stdout: "debug_cephfs_mirror": "0/5", 2023-09-23T20:52:05.484 INFO:teuthology.orchestra.run.smithi026.stdout: "debug_cephsqlite": "0/5", 2023-09-23T20:52:05.485 INFO:teuthology.orchestra.run.smithi026.stdout: "debug_client": "0/5", 2023-09-23T20:52:05.485 INFO:teuthology.orchestra.run.smithi026.stdout: "debug_compressor": "1/5", 2023-09-23T20:52:05.485 INFO:teuthology.orchestra.run.smithi026.stdout: "debug_context": "0/1", 2023-09-23T20:52:05.485 INFO:teuthology.orchestra.run.smithi026.stdout: "debug_crush": "1/1", 2023-09-23T20:52:05.485 INFO:teuthology.orchestra.run.smithi026.stdout: "debug_crypto": "1/5", 2023-09-23T20:52:05.485 INFO:teuthology.orchestra.run.smithi026.stdout: "debug_cyanstore": "0/5", 2023-09-23T20:52:05.485 INFO:teuthology.orchestra.run.smithi026.stdout: "debug_deliberately_leak_memory": "false", 2023-09-23T20:52:05.485 INFO:teuthology.orchestra.run.smithi026.stdout: "debug_disable_randomized_ping": "false", 2023-09-23T20:52:05.485 INFO:teuthology.orchestra.run.smithi026.stdout: "debug_dpdk": "1/5", 2023-09-23T20:52:05.485 INFO:teuthology.orchestra.run.smithi026.stdout: "debug_eventtrace": "1/5", 2023-09-23T20:52:05.485 INFO:teuthology.orchestra.run.smithi026.stdout: "debug_filer": "0/1", 2023-09-23T20:52:05.485 INFO:teuthology.orchestra.run.smithi026.stdout: "debug_filestore": "1/3", 2023-09-23T20:52:05.485 INFO:teuthology.orchestra.run.smithi026.stdout: "debug_finisher": "1/1", 2023-09-23T20:52:05.486 INFO:teuthology.orchestra.run.smithi026.stdout: "debug_fuse": "1/5", 2023-09-23T20:52:05.486 INFO:teuthology.orchestra.run.smithi026.stdout: "debug_heartbeat_testing_span": "0", 2023-09-23T20:52:05.486 INFO:teuthology.orchestra.run.smithi026.stdout: "debug_heartbeatmap": "1/5", 2023-09-23T20:52:05.486 INFO:teuthology.orchestra.run.smithi026.stdout: "debug_immutable_obj_cache": "0/5", 2023-09-23T20:52:05.486 INFO:teuthology.orchestra.run.smithi026.stdout: "debug_javaclient": "1/5", 2023-09-23T20:52:05.486 INFO:teuthology.orchestra.run.smithi026.stdout: "debug_journal": "1/3", 2023-09-23T20:52:05.486 INFO:teuthology.orchestra.run.smithi026.stdout: "debug_journaler": "0/5", 2023-09-23T20:52:05.486 INFO:teuthology.orchestra.run.smithi026.stdout: "debug_kstore": "1/5", 2023-09-23T20:52:05.486 INFO:teuthology.orchestra.run.smithi026.stdout: "debug_leveldb": "4/5", 2023-09-23T20:52:05.486 INFO:teuthology.orchestra.run.smithi026.stdout: "debug_lockdep": "0/1", 2023-09-23T20:52:05.486 INFO:teuthology.orchestra.run.smithi026.stdout: "debug_mclock": "1/5", 2023-09-23T20:52:05.487 INFO:teuthology.orchestra.run.smithi026.stdout: "debug_mds": "1/5", 2023-09-23T20:52:05.487 INFO:teuthology.orchestra.run.smithi026.stdout: "debug_mds_balancer": "1/5", 2023-09-23T20:52:05.487 INFO:teuthology.orchestra.run.smithi026.stdout: "debug_mds_locker": "1/5", 2023-09-23T20:52:05.487 INFO:teuthology.orchestra.run.smithi026.stdout: "debug_mds_log": "1/5", 2023-09-23T20:52:05.487 INFO:teuthology.orchestra.run.smithi026.stdout: "debug_mds_log_expire": "1/5", 2023-09-23T20:52:05.487 INFO:teuthology.orchestra.run.smithi026.stdout: "debug_mds_migrator": "1/5", 2023-09-23T20:52:05.487 INFO:teuthology.orchestra.run.smithi026.stdout: "debug_memstore": "1/5", 2023-09-23T20:52:05.487 INFO:teuthology.orchestra.run.smithi026.stdout: "debug_mgr": "2/5", 2023-09-23T20:52:05.487 INFO:teuthology.orchestra.run.smithi026.stdout: "debug_mgrc": "1/5", 2023-09-23T20:52:05.487 INFO:teuthology.orchestra.run.smithi026.stdout: "debug_mon": "20/20", 2023-09-23T20:52:05.487 INFO:teuthology.orchestra.run.smithi026.stdout: "debug_monc": "0/10", 2023-09-23T20:52:05.487 INFO:teuthology.orchestra.run.smithi026.stdout: "debug_ms": "1/1", 2023-09-23T20:52:05.488 INFO:teuthology.orchestra.run.smithi026.stdout: "debug_none": "0/5", 2023-09-23T20:52:05.488 INFO:teuthology.orchestra.run.smithi026.stdout: "debug_objclass": "0/5", 2023-09-23T20:52:05.488 INFO:teuthology.orchestra.run.smithi026.stdout: "debug_objectcacher": "0/5", 2023-09-23T20:52:05.488 INFO:teuthology.orchestra.run.smithi026.stdout: "debug_objecter": "0/1", 2023-09-23T20:52:05.488 INFO:teuthology.orchestra.run.smithi026.stdout: "debug_optracker": "0/5", 2023-09-23T20:52:05.488 INFO:teuthology.orchestra.run.smithi026.stdout: "debug_osd": "1/5", 2023-09-23T20:52:05.488 INFO:teuthology.orchestra.run.smithi026.stdout: "debug_paxos": "20/20", 2023-09-23T20:52:05.488 INFO:teuthology.orchestra.run.smithi026.stdout: "debug_perfcounter": "1/5", 2023-09-23T20:52:05.488 INFO:teuthology.orchestra.run.smithi026.stdout: "debug_prioritycache": "1/5", 2023-09-23T20:52:05.488 INFO:teuthology.orchestra.run.smithi026.stdout: "debug_rados": "0/5", 2023-09-23T20:52:05.488 INFO:teuthology.orchestra.run.smithi026.stdout: "debug_rbd": "0/5", 2023-09-23T20:52:05.489 INFO:teuthology.orchestra.run.smithi026.stdout: "debug_rbd_mirror": "0/5", 2023-09-23T20:52:05.489 INFO:teuthology.orchestra.run.smithi026.stdout: "debug_rbd_pwl": "0/5", 2023-09-23T20:52:05.489 INFO:teuthology.orchestra.run.smithi026.stdout: "debug_rbd_replay": "0/5", 2023-09-23T20:52:05.489 INFO:teuthology.orchestra.run.smithi026.stdout: "debug_refs": "0/0", 2023-09-23T20:52:05.489 INFO:teuthology.orchestra.run.smithi026.stdout: "debug_reserver": "1/1", 2023-09-23T20:52:05.489 INFO:teuthology.orchestra.run.smithi026.stdout: "debug_rgw": "1/5", 2023-09-23T20:52:05.489 INFO:teuthology.orchestra.run.smithi026.stdout: "debug_rgw_access": "1/5", 2023-09-23T20:52:05.489 INFO:teuthology.orchestra.run.smithi026.stdout: "debug_rgw_datacache": "1/5", 2023-09-23T20:52:05.489 INFO:teuthology.orchestra.run.smithi026.stdout: "debug_rgw_dbstore": "1/5", 2023-09-23T20:52:05.489 INFO:teuthology.orchestra.run.smithi026.stdout: "debug_rgw_flight": "1/5", 2023-09-23T20:52:05.489 INFO:teuthology.orchestra.run.smithi026.stdout: "debug_rgw_sync": "1/5", 2023-09-23T20:52:05.490 INFO:teuthology.orchestra.run.smithi026.stdout: "debug_rocksdb": "4/5", 2023-09-23T20:52:05.490 INFO:teuthology.orchestra.run.smithi026.stdout: "debug_seastore": "0/5", 2023-09-23T20:52:05.490 INFO:teuthology.orchestra.run.smithi026.stdout: "debug_seastore_backref": "0/5", 2023-09-23T20:52:05.490 INFO:teuthology.orchestra.run.smithi026.stdout: "debug_seastore_cache": "0/5", 2023-09-23T20:52:05.490 INFO:teuthology.orchestra.run.smithi026.stdout: "debug_seastore_cleaner": "0/5", 2023-09-23T20:52:05.490 INFO:teuthology.orchestra.run.smithi026.stdout: "debug_seastore_device": "0/5", 2023-09-23T20:52:05.490 INFO:teuthology.orchestra.run.smithi026.stdout: "debug_seastore_epm": "0/5", 2023-09-23T20:52:05.490 INFO:teuthology.orchestra.run.smithi026.stdout: "debug_seastore_fixedkv_tree": "0/5", 2023-09-23T20:52:05.491 INFO:teuthology.orchestra.run.smithi026.stdout: "debug_seastore_journal": "0/5", 2023-09-23T20:52:05.491 INFO:teuthology.orchestra.run.smithi026.stdout: "debug_seastore_lba": "0/5", 2023-09-23T20:52:05.491 INFO:teuthology.orchestra.run.smithi026.stdout: "debug_seastore_odata": "0/5", 2023-09-23T20:52:05.491 INFO:teuthology.orchestra.run.smithi026.stdout: "debug_seastore_omap": "0/5", 2023-09-23T20:52:05.491 INFO:teuthology.orchestra.run.smithi026.stdout: "debug_seastore_onode": "0/5", 2023-09-23T20:52:05.491 INFO:teuthology.orchestra.run.smithi026.stdout: "debug_seastore_t": "0/5", 2023-09-23T20:52:05.491 INFO:teuthology.orchestra.run.smithi026.stdout: "debug_seastore_tm": "0/5", 2023-09-23T20:52:05.491 INFO:teuthology.orchestra.run.smithi026.stdout: "debug_striper": "0/1", 2023-09-23T20:52:05.491 INFO:teuthology.orchestra.run.smithi026.stdout: "debug_test": "0/5", 2023-09-23T20:52:05.491 INFO:teuthology.orchestra.run.smithi026.stdout: "debug_throttle": "1/1", 2023-09-23T20:52:05.491 INFO:teuthology.orchestra.run.smithi026.stdout: "debug_timer": "0/1", 2023-09-23T20:52:05.491 INFO:teuthology.orchestra.run.smithi026.stdout: "debug_tp": "0/5", 2023-09-23T20:52:05.491 INFO:teuthology.orchestra.run.smithi026.stdout: "device_failure_prediction_mode": "none", 2023-09-23T20:52:05.492 INFO:teuthology.orchestra.run.smithi026.stdout: "enable_experimental_unrecoverable_data_corrupting_features": "", 2023-09-23T20:52:05.492 INFO:teuthology.orchestra.run.smithi026.stdout: "erasure_code_dir": "/usr/lib64/ceph/erasure-code", 2023-09-23T20:52:05.492 INFO:teuthology.orchestra.run.smithi026.stdout: "err_to_graylog": "false", 2023-09-23T20:52:05.492 INFO:teuthology.orchestra.run.smithi026.stdout: "err_to_journald": "false", 2023-09-23T20:52:05.492 INFO:teuthology.orchestra.run.smithi026.stdout: "err_to_stderr": "true", 2023-09-23T20:52:05.492 INFO:teuthology.orchestra.run.smithi026.stdout: "err_to_syslog": "false", 2023-09-23T20:52:05.492 INFO:teuthology.orchestra.run.smithi026.stdout: "event_tracing": "false", 2023-09-23T20:52:05.492 INFO:teuthology.orchestra.run.smithi026.stdout: "exporter_addr": "0.0.0.0", 2023-09-23T20:52:05.492 INFO:teuthology.orchestra.run.smithi026.stdout: "exporter_http_port": "9926", 2023-09-23T20:52:05.492 INFO:teuthology.orchestra.run.smithi026.stdout: "exporter_prio_limit": "5", 2023-09-23T20:52:05.492 INFO:teuthology.orchestra.run.smithi026.stdout: "exporter_sock_dir": "/var/run/ceph/", 2023-09-23T20:52:05.492 INFO:teuthology.orchestra.run.smithi026.stdout: "exporter_sort_metrics": "true", 2023-09-23T20:52:05.492 INFO:teuthology.orchestra.run.smithi026.stdout: "exporter_stats_period": "5", 2023-09-23T20:52:05.493 INFO:teuthology.orchestra.run.smithi026.stdout: "fake_statfs_for_testing": "0", 2023-09-23T20:52:05.493 INFO:teuthology.orchestra.run.smithi026.stdout: "fatal_signal_handlers": "true", 2023-09-23T20:52:05.493 INFO:teuthology.orchestra.run.smithi026.stdout: "filer_max_purge_ops": "10", 2023-09-23T20:52:05.493 INFO:teuthology.orchestra.run.smithi026.stdout: "filer_max_truncate_ops": "128", 2023-09-23T20:52:05.493 INFO:teuthology.orchestra.run.smithi026.stdout: "filestore_apply_finisher_threads": "1", 2023-09-23T20:52:05.493 INFO:teuthology.orchestra.run.smithi026.stdout: "filestore_blackhole": "false", 2023-09-23T20:52:05.493 INFO:teuthology.orchestra.run.smithi026.stdout: "filestore_btrfs_clone_range": "true", 2023-09-23T20:52:05.493 INFO:teuthology.orchestra.run.smithi026.stdout: "filestore_btrfs_snap": "true", 2023-09-23T20:52:05.493 INFO:teuthology.orchestra.run.smithi026.stdout: "filestore_caller_concurrency": "10", 2023-09-23T20:52:05.493 INFO:teuthology.orchestra.run.smithi026.stdout: "filestore_collect_device_partition_information": "true", 2023-09-23T20:52:05.493 INFO:teuthology.orchestra.run.smithi026.stdout: "filestore_commit_timeout": "600.000000", 2023-09-23T20:52:05.493 INFO:teuthology.orchestra.run.smithi026.stdout: "filestore_debug_inject_read_err": "false", 2023-09-23T20:52:05.494 INFO:teuthology.orchestra.run.smithi026.stdout: "filestore_debug_omap_check": "false", 2023-09-23T20:52:05.494 INFO:teuthology.orchestra.run.smithi026.stdout: "filestore_debug_random_read_err": "0.000000", 2023-09-23T20:52:05.494 INFO:teuthology.orchestra.run.smithi026.stdout: "filestore_debug_verify_split": "false", 2023-09-23T20:52:05.494 INFO:teuthology.orchestra.run.smithi026.stdout: "filestore_dump_file": "", 2023-09-23T20:52:05.494 INFO:teuthology.orchestra.run.smithi026.stdout: "filestore_expected_throughput_bytes": "209715200.000000", 2023-09-23T20:52:05.494 INFO:teuthology.orchestra.run.smithi026.stdout: "filestore_expected_throughput_ops": "200.000000", 2023-09-23T20:52:05.494 INFO:teuthology.orchestra.run.smithi026.stdout: "filestore_fadvise": "true", 2023-09-23T20:52:05.494 INFO:teuthology.orchestra.run.smithi026.stdout: "filestore_fail_eio": "true", 2023-09-23T20:52:05.494 INFO:teuthology.orchestra.run.smithi026.stdout: "filestore_fd_cache_shards": "16", 2023-09-23T20:52:05.494 INFO:teuthology.orchestra.run.smithi026.stdout: "filestore_fd_cache_size": "128", 2023-09-23T20:52:05.494 INFO:teuthology.orchestra.run.smithi026.stdout: "filestore_fiemap": "false", 2023-09-23T20:52:05.495 INFO:teuthology.orchestra.run.smithi026.stdout: "filestore_fiemap_threshold": "4096", 2023-09-23T20:52:05.495 INFO:teuthology.orchestra.run.smithi026.stdout: "filestore_fsync_flushes_journal_data": "false", 2023-09-23T20:52:05.495 INFO:teuthology.orchestra.run.smithi026.stdout: "filestore_index_retry_probability": "0.000000", 2023-09-23T20:52:05.495 INFO:teuthology.orchestra.run.smithi026.stdout: "filestore_inject_stall": "0", 2023-09-23T20:52:05.495 INFO:teuthology.orchestra.run.smithi026.stdout: "filestore_journal_parallel": "false", 2023-09-23T20:52:05.495 INFO:teuthology.orchestra.run.smithi026.stdout: "filestore_journal_trailing": "false", 2023-09-23T20:52:05.495 INFO:teuthology.orchestra.run.smithi026.stdout: "filestore_journal_writeahead": "false", 2023-09-23T20:52:05.495 INFO:teuthology.orchestra.run.smithi026.stdout: "filestore_kill_at": "0", 2023-09-23T20:52:05.495 INFO:teuthology.orchestra.run.smithi026.stdout: "filestore_max_alloc_hint_size": "1048576", 2023-09-23T20:52:05.495 INFO:teuthology.orchestra.run.smithi026.stdout: "filestore_max_inline_xattr_size": "0", 2023-09-23T20:52:05.495 INFO:teuthology.orchestra.run.smithi026.stdout: "filestore_max_inline_xattr_size_btrfs": "2048", 2023-09-23T20:52:05.495 INFO:teuthology.orchestra.run.smithi026.stdout: "filestore_max_inline_xattr_size_other": "512", 2023-09-23T20:52:05.495 INFO:teuthology.orchestra.run.smithi026.stdout: "filestore_max_inline_xattr_size_xfs": "65536", 2023-09-23T20:52:05.496 INFO:teuthology.orchestra.run.smithi026.stdout: "filestore_max_inline_xattrs": "0", 2023-09-23T20:52:05.496 INFO:teuthology.orchestra.run.smithi026.stdout: "filestore_max_inline_xattrs_btrfs": "10", 2023-09-23T20:52:05.496 INFO:teuthology.orchestra.run.smithi026.stdout: "filestore_max_inline_xattrs_other": "2", 2023-09-23T20:52:05.496 INFO:teuthology.orchestra.run.smithi026.stdout: "filestore_max_inline_xattrs_xfs": "10", 2023-09-23T20:52:05.496 INFO:teuthology.orchestra.run.smithi026.stdout: "filestore_max_sync_interval": "5.000000", 2023-09-23T20:52:05.496 INFO:teuthology.orchestra.run.smithi026.stdout: "filestore_max_xattr_value_size": "0", 2023-09-23T20:52:05.496 INFO:teuthology.orchestra.run.smithi026.stdout: "filestore_max_xattr_value_size_btrfs": "65536", 2023-09-23T20:52:05.496 INFO:teuthology.orchestra.run.smithi026.stdout: "filestore_max_xattr_value_size_other": "1024", 2023-09-23T20:52:05.496 INFO:teuthology.orchestra.run.smithi026.stdout: "filestore_max_xattr_value_size_xfs": "65536", 2023-09-23T20:52:05.496 INFO:teuthology.orchestra.run.smithi026.stdout: "filestore_merge_threshold": "-10", 2023-09-23T20:52:05.496 INFO:teuthology.orchestra.run.smithi026.stdout: "filestore_min_sync_interval": "0.010000", 2023-09-23T20:52:05.496 INFO:teuthology.orchestra.run.smithi026.stdout: "filestore_odsync_write": "false", 2023-09-23T20:52:05.497 INFO:teuthology.orchestra.run.smithi026.stdout: "filestore_omap_backend": "rocksdb", 2023-09-23T20:52:05.497 INFO:teuthology.orchestra.run.smithi026.stdout: "filestore_omap_backend_path": "", 2023-09-23T20:52:05.497 INFO:teuthology.orchestra.run.smithi026.stdout: "filestore_omap_header_cache_size": "1024", 2023-09-23T20:52:05.497 INFO:teuthology.orchestra.run.smithi026.stdout: "filestore_ondisk_finisher_threads": "1", 2023-09-23T20:52:05.497 INFO:teuthology.orchestra.run.smithi026.stdout: "filestore_op_thread_suicide_timeout": "180", 2023-09-23T20:52:05.497 INFO:teuthology.orchestra.run.smithi026.stdout: "filestore_op_thread_timeout": "60", 2023-09-23T20:52:05.497 INFO:teuthology.orchestra.run.smithi026.stdout: "filestore_op_threads": "2", 2023-09-23T20:52:05.497 INFO:teuthology.orchestra.run.smithi026.stdout: "filestore_punch_hole": "false", 2023-09-23T20:52:05.497 INFO:teuthology.orchestra.run.smithi026.stdout: "filestore_queue_high_delay_multiple": "0.000000", 2023-09-23T20:52:05.497 INFO:teuthology.orchestra.run.smithi026.stdout: "filestore_queue_high_delay_multiple_bytes": "0.000000", 2023-09-23T20:52:05.497 INFO:teuthology.orchestra.run.smithi026.stdout: "filestore_queue_high_delay_multiple_ops": "0.000000", 2023-09-23T20:52:05.497 INFO:teuthology.orchestra.run.smithi026.stdout: "filestore_queue_high_threshhold": "0.900000", 2023-09-23T20:52:05.498 INFO:teuthology.orchestra.run.smithi026.stdout: "filestore_queue_low_threshhold": "0.300000", 2023-09-23T20:52:05.498 INFO:teuthology.orchestra.run.smithi026.stdout: "filestore_queue_max_bytes": "104857600", 2023-09-23T20:52:05.498 INFO:teuthology.orchestra.run.smithi026.stdout: "filestore_queue_max_delay_multiple": "0.000000", 2023-09-23T20:52:05.498 INFO:teuthology.orchestra.run.smithi026.stdout: "filestore_queue_max_delay_multiple_bytes": "0.000000", 2023-09-23T20:52:05.498 INFO:teuthology.orchestra.run.smithi026.stdout: "filestore_queue_max_delay_multiple_ops": "0.000000", 2023-09-23T20:52:05.498 INFO:teuthology.orchestra.run.smithi026.stdout: "filestore_queue_max_ops": "50", 2023-09-23T20:52:05.498 INFO:teuthology.orchestra.run.smithi026.stdout: "filestore_rocksdb_options": "max_background_jobs=10,compaction_readahead_size=2097152,compression=kNoCompression", 2023-09-23T20:52:05.498 INFO:teuthology.orchestra.run.smithi026.stdout: "filestore_seek_data_hole": "false", 2023-09-23T20:52:05.498 INFO:teuthology.orchestra.run.smithi026.stdout: "filestore_sloppy_crc": "false", 2023-09-23T20:52:05.498 INFO:teuthology.orchestra.run.smithi026.stdout: "filestore_sloppy_crc_block_size": "65536", 2023-09-23T20:52:05.499 INFO:teuthology.orchestra.run.smithi026.stdout: "filestore_splice": "false", 2023-09-23T20:52:05.499 INFO:teuthology.orchestra.run.smithi026.stdout: "filestore_split_multiple": "2", 2023-09-23T20:52:05.499 INFO:teuthology.orchestra.run.smithi026.stdout: "filestore_split_rand_factor": "20", 2023-09-23T20:52:05.499 INFO:teuthology.orchestra.run.smithi026.stdout: "filestore_update_to": "1000", 2023-09-23T20:52:05.499 INFO:teuthology.orchestra.run.smithi026.stdout: "filestore_wbthrottle_btrfs_bytes_hard_limit": "419430400", 2023-09-23T20:52:05.499 INFO:teuthology.orchestra.run.smithi026.stdout: "filestore_wbthrottle_btrfs_bytes_start_flusher": "41943040", 2023-09-23T20:52:05.499 INFO:teuthology.orchestra.run.smithi026.stdout: "filestore_wbthrottle_btrfs_inodes_hard_limit": "5000", 2023-09-23T20:52:05.499 INFO:teuthology.orchestra.run.smithi026.stdout: "filestore_wbthrottle_btrfs_inodes_start_flusher": "500", 2023-09-23T20:52:05.499 INFO:teuthology.orchestra.run.smithi026.stdout: "filestore_wbthrottle_btrfs_ios_hard_limit": "5000", 2023-09-23T20:52:05.499 INFO:teuthology.orchestra.run.smithi026.stdout: "filestore_wbthrottle_btrfs_ios_start_flusher": "500", 2023-09-23T20:52:05.499 INFO:teuthology.orchestra.run.smithi026.stdout: "filestore_wbthrottle_enable": "true", 2023-09-23T20:52:05.499 INFO:teuthology.orchestra.run.smithi026.stdout: "filestore_wbthrottle_xfs_bytes_hard_limit": "419430400", 2023-09-23T20:52:05.500 INFO:teuthology.orchestra.run.smithi026.stdout: "filestore_wbthrottle_xfs_bytes_start_flusher": "41943040", 2023-09-23T20:52:05.500 INFO:teuthology.orchestra.run.smithi026.stdout: "filestore_wbthrottle_xfs_inodes_hard_limit": "5000", 2023-09-23T20:52:05.500 INFO:teuthology.orchestra.run.smithi026.stdout: "filestore_wbthrottle_xfs_inodes_start_flusher": "500", 2023-09-23T20:52:05.500 INFO:teuthology.orchestra.run.smithi026.stdout: "filestore_wbthrottle_xfs_ios_hard_limit": "5000", 2023-09-23T20:52:05.500 INFO:teuthology.orchestra.run.smithi026.stdout: "filestore_wbthrottle_xfs_ios_start_flusher": "500", 2023-09-23T20:52:05.500 INFO:teuthology.orchestra.run.smithi026.stdout: "filestore_xfs_extsize": "false", 2023-09-23T20:52:05.500 INFO:teuthology.orchestra.run.smithi026.stdout: "filestore_zfs_snap": "false", 2023-09-23T20:52:05.500 INFO:teuthology.orchestra.run.smithi026.stdout: "fio_dir": "/tmp/fio", 2023-09-23T20:52:05.500 INFO:teuthology.orchestra.run.smithi026.stdout: "fsid": "b3f8e4ec-5a50-11ee-8db4-212e2dc638e7", 2023-09-23T20:52:05.500 INFO:teuthology.orchestra.run.smithi026.stdout: "fuse_allow_other": "true", 2023-09-23T20:52:05.500 INFO:teuthology.orchestra.run.smithi026.stdout: "fuse_atomic_o_trunc": "true", 2023-09-23T20:52:05.500 INFO:teuthology.orchestra.run.smithi026.stdout: "fuse_big_writes": "true", 2023-09-23T20:52:05.501 INFO:teuthology.orchestra.run.smithi026.stdout: "fuse_debug": "false", 2023-09-23T20:52:05.501 INFO:teuthology.orchestra.run.smithi026.stdout: "fuse_default_permissions": "false", 2023-09-23T20:52:05.501 INFO:teuthology.orchestra.run.smithi026.stdout: "fuse_disable_pagecache": "false", 2023-09-23T20:52:05.501 INFO:teuthology.orchestra.run.smithi026.stdout: "fuse_max_write": "0", 2023-09-23T20:52:05.501 INFO:teuthology.orchestra.run.smithi026.stdout: "fuse_multithreaded": "true", 2023-09-23T20:52:05.501 INFO:teuthology.orchestra.run.smithi026.stdout: "fuse_require_active_mds": "true", 2023-09-23T20:52:05.501 INFO:teuthology.orchestra.run.smithi026.stdout: "fuse_set_user_groups": "true", 2023-09-23T20:52:05.501 INFO:teuthology.orchestra.run.smithi026.stdout: "fuse_splice_move": "true", 2023-09-23T20:52:05.501 INFO:teuthology.orchestra.run.smithi026.stdout: "fuse_splice_read": "true", 2023-09-23T20:52:05.501 INFO:teuthology.orchestra.run.smithi026.stdout: "fuse_splice_write": "true", 2023-09-23T20:52:05.501 INFO:teuthology.orchestra.run.smithi026.stdout: "fuse_syncfs_on_mksnap": "true", 2023-09-23T20:52:05.501 INFO:teuthology.orchestra.run.smithi026.stdout: "fuse_use_invalidate_cb": "true", 2023-09-23T20:52:05.502 INFO:teuthology.orchestra.run.smithi026.stdout: "gss_ktab_client_file": "/var/lib/ceph/mon.a/gss_client_mon.a.ktab", 2023-09-23T20:52:05.502 INFO:teuthology.orchestra.run.smithi026.stdout: "gss_target_name": "ceph", 2023-09-23T20:52:05.502 INFO:teuthology.orchestra.run.smithi026.stdout: "heartbeat_file": "", 2023-09-23T20:52:05.502 INFO:teuthology.orchestra.run.smithi026.stdout: "heartbeat_inject_failure": "0", 2023-09-23T20:52:05.502 INFO:teuthology.orchestra.run.smithi026.stdout: "heartbeat_interval": "5", 2023-09-23T20:52:05.502 INFO:teuthology.orchestra.run.smithi026.stdout: "host": "smithi026", 2023-09-23T20:52:05.502 INFO:teuthology.orchestra.run.smithi026.stdout: "immutable_object_cache_client_dedicated_thread_num": "2", 2023-09-23T20:52:05.502 INFO:teuthology.orchestra.run.smithi026.stdout: "immutable_object_cache_max_inflight_ops": "128", 2023-09-23T20:52:05.502 INFO:teuthology.orchestra.run.smithi026.stdout: "immutable_object_cache_max_size": "1073741824", 2023-09-23T20:52:05.502 INFO:teuthology.orchestra.run.smithi026.stdout: "immutable_object_cache_path": "/tmp/ceph_immutable_object_cache", 2023-09-23T20:52:05.502 INFO:teuthology.orchestra.run.smithi026.stdout: "immutable_object_cache_qos_bps_burst": "0", 2023-09-23T20:52:05.503 INFO:teuthology.orchestra.run.smithi026.stdout: "immutable_object_cache_qos_bps_burst_seconds": "1", 2023-09-23T20:52:05.503 INFO:teuthology.orchestra.run.smithi026.stdout: "immutable_object_cache_qos_bps_limit": "0", 2023-09-23T20:52:05.503 INFO:teuthology.orchestra.run.smithi026.stdout: "immutable_object_cache_qos_iops_burst": "0", 2023-09-23T20:52:05.503 INFO:teuthology.orchestra.run.smithi026.stdout: "immutable_object_cache_qos_iops_burst_seconds": "1", 2023-09-23T20:52:05.503 INFO:teuthology.orchestra.run.smithi026.stdout: "immutable_object_cache_qos_iops_limit": "0", 2023-09-23T20:52:05.503 INFO:teuthology.orchestra.run.smithi026.stdout: "immutable_object_cache_qos_schedule_tick_min": "50", 2023-09-23T20:52:05.503 INFO:teuthology.orchestra.run.smithi026.stdout: "immutable_object_cache_sock": "/var/run/ceph/immutable_object_cache_sock", 2023-09-23T20:52:05.503 INFO:teuthology.orchestra.run.smithi026.stdout: "immutable_object_cache_watermark": "0.900000", 2023-09-23T20:52:05.503 INFO:teuthology.orchestra.run.smithi026.stdout: "inject_early_sigterm": "false", 2023-09-23T20:52:05.503 INFO:teuthology.orchestra.run.smithi026.stdout: "jaeger_agent_port": "6799", 2023-09-23T20:52:05.503 INFO:teuthology.orchestra.run.smithi026.stdout: "jaeger_tracing_enable": "false", 2023-09-23T20:52:05.503 INFO:teuthology.orchestra.run.smithi026.stdout: "journal_aio": "true", 2023-09-23T20:52:05.504 INFO:teuthology.orchestra.run.smithi026.stdout: "journal_align_min_size": "65536", 2023-09-23T20:52:05.504 INFO:teuthology.orchestra.run.smithi026.stdout: "journal_block_align": "true", 2023-09-23T20:52:05.504 INFO:teuthology.orchestra.run.smithi026.stdout: "journal_block_size": "4096", 2023-09-23T20:52:05.504 INFO:teuthology.orchestra.run.smithi026.stdout: "journal_dio": "true", 2023-09-23T20:52:05.504 INFO:teuthology.orchestra.run.smithi026.stdout: "journal_discard": "false", 2023-09-23T20:52:05.504 INFO:teuthology.orchestra.run.smithi026.stdout: "journal_force_aio": "false", 2023-09-23T20:52:05.504 INFO:teuthology.orchestra.run.smithi026.stdout: "journal_ignore_corruption": "false", 2023-09-23T20:52:05.504 INFO:teuthology.orchestra.run.smithi026.stdout: "journal_max_write_bytes": "10485760", 2023-09-23T20:52:05.504 INFO:teuthology.orchestra.run.smithi026.stdout: "journal_max_write_entries": "100", 2023-09-23T20:52:05.504 INFO:teuthology.orchestra.run.smithi026.stdout: "journal_replay_from": "0", 2023-09-23T20:52:05.504 INFO:teuthology.orchestra.run.smithi026.stdout: "journal_throttle_high_multiple": "0.000000", 2023-09-23T20:52:05.504 INFO:teuthology.orchestra.run.smithi026.stdout: "journal_throttle_high_threshhold": "0.900000", 2023-09-23T20:52:05.505 INFO:teuthology.orchestra.run.smithi026.stdout: "journal_throttle_low_threshhold": "0.600000", 2023-09-23T20:52:05.505 INFO:teuthology.orchestra.run.smithi026.stdout: "journal_throttle_max_multiple": "0.000000", 2023-09-23T20:52:05.505 INFO:teuthology.orchestra.run.smithi026.stdout: "journal_write_header_frequency": "0", 2023-09-23T20:52:05.505 INFO:teuthology.orchestra.run.smithi026.stdout: "journal_zero_on_create": "false", 2023-09-23T20:52:05.505 INFO:teuthology.orchestra.run.smithi026.stdout: "journaler_prefetch_periods": "10", 2023-09-23T20:52:05.505 INFO:teuthology.orchestra.run.smithi026.stdout: "journaler_prezero_periods": "5", 2023-09-23T20:52:05.505 INFO:teuthology.orchestra.run.smithi026.stdout: "journaler_write_head_interval": "15", 2023-09-23T20:52:05.505 INFO:teuthology.orchestra.run.smithi026.stdout: "key": "", 2023-09-23T20:52:05.505 INFO:teuthology.orchestra.run.smithi026.stdout: "keyfile": "", 2023-09-23T20:52:05.505 INFO:teuthology.orchestra.run.smithi026.stdout: "keyring": "/var/lib/ceph/mon/ceph-a/keyring", 2023-09-23T20:52:05.505 INFO:teuthology.orchestra.run.smithi026.stdout: "kstore_backend": "rocksdb", 2023-09-23T20:52:05.505 INFO:teuthology.orchestra.run.smithi026.stdout: "kstore_default_stripe_size": "65536", 2023-09-23T20:52:05.506 INFO:teuthology.orchestra.run.smithi026.stdout: "kstore_fsck_on_mount": "false", 2023-09-23T20:52:05.506 INFO:teuthology.orchestra.run.smithi026.stdout: "kstore_fsck_on_mount_deep": "true", 2023-09-23T20:52:05.506 INFO:teuthology.orchestra.run.smithi026.stdout: "kstore_max_bytes": "67108864", 2023-09-23T20:52:05.506 INFO:teuthology.orchestra.run.smithi026.stdout: "kstore_max_ops": "512", 2023-09-23T20:52:05.506 INFO:teuthology.orchestra.run.smithi026.stdout: "kstore_nid_prealloc": "1024", 2023-09-23T20:52:05.506 INFO:teuthology.orchestra.run.smithi026.stdout: "kstore_onode_map_size": "1024", 2023-09-23T20:52:05.506 INFO:teuthology.orchestra.run.smithi026.stdout: "kstore_rocksdb_options": "compression=kNoCompression", 2023-09-23T20:52:05.506 INFO:teuthology.orchestra.run.smithi026.stdout: "kstore_sync_submit_transaction": "false", 2023-09-23T20:52:05.506 INFO:teuthology.orchestra.run.smithi026.stdout: "kstore_sync_transaction": "false", 2023-09-23T20:52:05.506 INFO:teuthology.orchestra.run.smithi026.stdout: "leveldb_block_size": "65536", 2023-09-23T20:52:05.506 INFO:teuthology.orchestra.run.smithi026.stdout: "leveldb_bloom_size": "0", 2023-09-23T20:52:05.506 INFO:teuthology.orchestra.run.smithi026.stdout: "leveldb_cache_size": "536870912", 2023-09-23T20:52:05.507 INFO:teuthology.orchestra.run.smithi026.stdout: "leveldb_compact_on_mount": "false", 2023-09-23T20:52:05.507 INFO:teuthology.orchestra.run.smithi026.stdout: "leveldb_compression": "false", 2023-09-23T20:52:05.507 INFO:teuthology.orchestra.run.smithi026.stdout: "leveldb_log": "", 2023-09-23T20:52:05.507 INFO:teuthology.orchestra.run.smithi026.stdout: "leveldb_log_to_ceph_log": "true", 2023-09-23T20:52:05.507 INFO:teuthology.orchestra.run.smithi026.stdout: "leveldb_max_open_files": "0", 2023-09-23T20:52:05.507 INFO:teuthology.orchestra.run.smithi026.stdout: "leveldb_paranoid": "false", 2023-09-23T20:52:05.507 INFO:teuthology.orchestra.run.smithi026.stdout: "leveldb_write_buffer_size": "33554432", 2023-09-23T20:52:05.507 INFO:teuthology.orchestra.run.smithi026.stdout: "librados_thread_count": "2", 2023-09-23T20:52:05.507 INFO:teuthology.orchestra.run.smithi026.stdout: "lockdep": "false", 2023-09-23T20:52:05.507 INFO:teuthology.orchestra.run.smithi026.stdout: "lockdep_force_backtrace": "false", 2023-09-23T20:52:05.507 INFO:teuthology.orchestra.run.smithi026.stdout: "log_coarse_timestamps": "true", 2023-09-23T20:52:05.507 INFO:teuthology.orchestra.run.smithi026.stdout: "log_file": "/var/log/ceph/ceph-mon.a.log", 2023-09-23T20:52:05.508 INFO:teuthology.orchestra.run.smithi026.stdout: "log_flush_on_exit": "false", 2023-09-23T20:52:05.508 INFO:teuthology.orchestra.run.smithi026.stdout: "log_graylog_host": "127.0.0.1", 2023-09-23T20:52:05.508 INFO:teuthology.orchestra.run.smithi026.stdout: "log_graylog_port": "12201", 2023-09-23T20:52:05.508 INFO:teuthology.orchestra.run.smithi026.stdout: "log_max_new": "1000", 2023-09-23T20:52:05.508 INFO:teuthology.orchestra.run.smithi026.stdout: "log_max_recent": "10000", 2023-09-23T20:52:05.508 INFO:teuthology.orchestra.run.smithi026.stdout: "log_stderr_prefix": "", 2023-09-23T20:52:05.508 INFO:teuthology.orchestra.run.smithi026.stdout: "log_stop_at_utilization": "0.970000", 2023-09-23T20:52:05.508 INFO:teuthology.orchestra.run.smithi026.stdout: "log_to_file": "true", 2023-09-23T20:52:05.508 INFO:teuthology.orchestra.run.smithi026.stdout: "log_to_graylog": "false", 2023-09-23T20:52:05.508 INFO:teuthology.orchestra.run.smithi026.stdout: "log_to_journald": "false", 2023-09-23T20:52:05.508 INFO:teuthology.orchestra.run.smithi026.stdout: "log_to_stderr": "false", 2023-09-23T20:52:05.508 INFO:teuthology.orchestra.run.smithi026.stdout: "log_to_syslog": "false", 2023-09-23T20:52:05.509 INFO:teuthology.orchestra.run.smithi026.stdout: "max_rotating_auth_attempts": "10", 2023-09-23T20:52:05.509 INFO:teuthology.orchestra.run.smithi026.stdout: "mds_abort_on_newly_corrupt_dentry": "true", 2023-09-23T20:52:05.509 INFO:teuthology.orchestra.run.smithi026.stdout: "mds_action_on_write_error": "1", 2023-09-23T20:52:05.509 INFO:teuthology.orchestra.run.smithi026.stdout: "mds_alternate_name_max": "8192", 2023-09-23T20:52:05.509 INFO:teuthology.orchestra.run.smithi026.stdout: "mds_asio_thread_count": "2", 2023-09-23T20:52:05.509 INFO:teuthology.orchestra.run.smithi026.stdout: "mds_bal_export_pin": "true", 2023-09-23T20:52:05.509 INFO:teuthology.orchestra.run.smithi026.stdout: "mds_bal_fragment_dirs": "true", 2023-09-23T20:52:05.509 INFO:teuthology.orchestra.run.smithi026.stdout: "mds_bal_fragment_fast_factor": "1.500000", 2023-09-23T20:52:05.509 INFO:teuthology.orchestra.run.smithi026.stdout: "mds_bal_fragment_interval": "5", 2023-09-23T20:52:05.509 INFO:teuthology.orchestra.run.smithi026.stdout: "mds_bal_fragment_size_max": "100000", 2023-09-23T20:52:05.509 INFO:teuthology.orchestra.run.smithi026.stdout: "mds_bal_idle_threshold": "0.000000", 2023-09-23T20:52:05.509 INFO:teuthology.orchestra.run.smithi026.stdout: "mds_bal_interval": "10", 2023-09-23T20:52:05.510 INFO:teuthology.orchestra.run.smithi026.stdout: "mds_bal_max": "-1", 2023-09-23T20:52:05.510 INFO:teuthology.orchestra.run.smithi026.stdout: "mds_bal_max_until": "-1", 2023-09-23T20:52:05.510 INFO:teuthology.orchestra.run.smithi026.stdout: "mds_bal_merge_size": "50", 2023-09-23T20:52:05.510 INFO:teuthology.orchestra.run.smithi026.stdout: "mds_bal_midchunk": "0.300000", 2023-09-23T20:52:05.510 INFO:teuthology.orchestra.run.smithi026.stdout: "mds_bal_min_rebalance": "0.100000", 2023-09-23T20:52:05.510 INFO:teuthology.orchestra.run.smithi026.stdout: "mds_bal_min_start": "0.200000", 2023-09-23T20:52:05.510 INFO:teuthology.orchestra.run.smithi026.stdout: "mds_bal_minchunk": "0.001000", 2023-09-23T20:52:05.510 INFO:teuthology.orchestra.run.smithi026.stdout: "mds_bal_mode": "0", 2023-09-23T20:52:05.510 INFO:teuthology.orchestra.run.smithi026.stdout: "mds_bal_need_max": "1.200000", 2023-09-23T20:52:05.510 INFO:teuthology.orchestra.run.smithi026.stdout: "mds_bal_need_min": "0.800000", 2023-09-23T20:52:05.510 INFO:teuthology.orchestra.run.smithi026.stdout: "mds_bal_replicate_threshold": "8000.000000", 2023-09-23T20:52:05.511 INFO:teuthology.orchestra.run.smithi026.stdout: "mds_bal_sample_interval": "3.000000", 2023-09-23T20:52:05.511 INFO:teuthology.orchestra.run.smithi026.stdout: "mds_bal_split_bits": "3", 2023-09-23T20:52:05.511 INFO:teuthology.orchestra.run.smithi026.stdout: "mds_bal_split_rd": "25000.000000", 2023-09-23T20:52:05.511 INFO:teuthology.orchestra.run.smithi026.stdout: "mds_bal_split_size": "10000", 2023-09-23T20:52:05.511 INFO:teuthology.orchestra.run.smithi026.stdout: "mds_bal_split_wr": "10000.000000", 2023-09-23T20:52:05.511 INFO:teuthology.orchestra.run.smithi026.stdout: "mds_bal_target_decay": "10.000000", 2023-09-23T20:52:05.511 INFO:teuthology.orchestra.run.smithi026.stdout: "mds_bal_unreplicate_threshold": "0.000000", 2023-09-23T20:52:05.511 INFO:teuthology.orchestra.run.smithi026.stdout: "mds_beacon_grace": "15.000000", 2023-09-23T20:52:05.511 INFO:teuthology.orchestra.run.smithi026.stdout: "mds_beacon_interval": "4.000000", 2023-09-23T20:52:05.511 INFO:teuthology.orchestra.run.smithi026.stdout: "mds_beacon_mon_down_grace": "60", 2023-09-23T20:52:05.511 INFO:teuthology.orchestra.run.smithi026.stdout: "mds_cache_memory_limit": "4294967296", 2023-09-23T20:52:05.511 INFO:teuthology.orchestra.run.smithi026.stdout: "mds_cache_mid": "0.700000", 2023-09-23T20:52:05.512 INFO:teuthology.orchestra.run.smithi026.stdout: "mds_cache_release_free_interval": "10", 2023-09-23T20:52:05.512 INFO:teuthology.orchestra.run.smithi026.stdout: "mds_cache_reservation": "0.050000", 2023-09-23T20:52:05.512 INFO:teuthology.orchestra.run.smithi026.stdout: "mds_cache_trim_decay_rate": "1.000000", 2023-09-23T20:52:05.512 INFO:teuthology.orchestra.run.smithi026.stdout: "mds_cache_trim_interval": "1", 2023-09-23T20:52:05.512 INFO:teuthology.orchestra.run.smithi026.stdout: "mds_cache_trim_threshold": "262144", 2023-09-23T20:52:05.512 INFO:teuthology.orchestra.run.smithi026.stdout: "mds_cap_acquisition_throttle_retry_request_timeout": "0.500000", 2023-09-23T20:52:05.512 INFO:teuthology.orchestra.run.smithi026.stdout: "mds_cap_revoke_eviction_timeout": "0.000000", 2023-09-23T20:52:05.512 INFO:teuthology.orchestra.run.smithi026.stdout: "mds_client_delegate_inos_pct": "50", 2023-09-23T20:52:05.512 INFO:teuthology.orchestra.run.smithi026.stdout: "mds_client_prealloc_inos": "1000", 2023-09-23T20:52:05.512 INFO:teuthology.orchestra.run.smithi026.stdout: "mds_client_writeable_range_max_inc_objs": "1024", 2023-09-23T20:52:05.512 INFO:teuthology.orchestra.run.smithi026.stdout: "mds_connect_bootstrapping": "false", 2023-09-23T20:52:05.512 INFO:teuthology.orchestra.run.smithi026.stdout: "mds_damage_table_max_entries": "10000", 2023-09-23T20:52:05.512 INFO:teuthology.orchestra.run.smithi026.stdout: "mds_data": "/var/lib/ceph/mds/ceph-a", 2023-09-23T20:52:05.513 INFO:teuthology.orchestra.run.smithi026.stdout: "mds_debug_auth_pins": "false", 2023-09-23T20:52:05.513 INFO:teuthology.orchestra.run.smithi026.stdout: "mds_debug_frag": "false", 2023-09-23T20:52:05.513 INFO:teuthology.orchestra.run.smithi026.stdout: "mds_debug_scatterstat": "false", 2023-09-23T20:52:05.513 INFO:teuthology.orchestra.run.smithi026.stdout: "mds_debug_subtrees": "false", 2023-09-23T20:52:05.513 INFO:teuthology.orchestra.run.smithi026.stdout: "mds_decay_halflife": "5.000000", 2023-09-23T20:52:05.513 INFO:teuthology.orchestra.run.smithi026.stdout: "mds_default_dir_hash": "2", 2023-09-23T20:52:05.513 INFO:teuthology.orchestra.run.smithi026.stdout: "mds_defer_session_stale": "true", 2023-09-23T20:52:05.513 INFO:teuthology.orchestra.run.smithi026.stdout: "mds_deny_all_reconnect": "false", 2023-09-23T20:52:05.513 INFO:teuthology.orchestra.run.smithi026.stdout: "mds_dir_keys_per_op": "16384", 2023-09-23T20:52:05.513 INFO:teuthology.orchestra.run.smithi026.stdout: "mds_dir_max_commit_size": "10", 2023-09-23T20:52:05.513 INFO:teuthology.orchestra.run.smithi026.stdout: "mds_dir_max_entries": "0", 2023-09-23T20:52:05.513 INFO:teuthology.orchestra.run.smithi026.stdout: "mds_dir_prefetch": "true", 2023-09-23T20:52:05.514 INFO:teuthology.orchestra.run.smithi026.stdout: "mds_dirstat_min_interval": "1.000000", 2023-09-23T20:52:05.514 INFO:teuthology.orchestra.run.smithi026.stdout: "mds_dump_cache_after_rejoin": "false", 2023-09-23T20:52:05.514 INFO:teuthology.orchestra.run.smithi026.stdout: "mds_dump_cache_on_map": "false", 2023-09-23T20:52:05.514 INFO:teuthology.orchestra.run.smithi026.stdout: "mds_dump_cache_threshold_file": "0", 2023-09-23T20:52:05.514 INFO:teuthology.orchestra.run.smithi026.stdout: "mds_dump_cache_threshold_formatter": "1073741824", 2023-09-23T20:52:05.514 INFO:teuthology.orchestra.run.smithi026.stdout: "mds_early_reply": "true", 2023-09-23T20:52:05.514 INFO:teuthology.orchestra.run.smithi026.stdout: "mds_enable_op_tracker": "true", 2023-09-23T20:52:05.514 INFO:teuthology.orchestra.run.smithi026.stdout: "mds_enforce_unique_name": "true", 2023-09-23T20:52:05.514 INFO:teuthology.orchestra.run.smithi026.stdout: "mds_export_ephemeral_distributed": "true", 2023-09-23T20:52:05.514 INFO:teuthology.orchestra.run.smithi026.stdout: "mds_export_ephemeral_distributed_factor": "2.000000", 2023-09-23T20:52:05.514 INFO:teuthology.orchestra.run.smithi026.stdout: "mds_export_ephemeral_random": "true", 2023-09-23T20:52:05.515 INFO:teuthology.orchestra.run.smithi026.stdout: "mds_export_ephemeral_random_max": "0.010000", 2023-09-23T20:52:05.515 INFO:teuthology.orchestra.run.smithi026.stdout: "mds_extraordinary_events_dump_interval": "0", 2023-09-23T20:52:05.515 INFO:teuthology.orchestra.run.smithi026.stdout: "mds_forward_all_requests_to_auth": "false", 2023-09-23T20:52:05.515 INFO:teuthology.orchestra.run.smithi026.stdout: "mds_freeze_tree_timeout": "30.000000", 2023-09-23T20:52:05.515 INFO:teuthology.orchestra.run.smithi026.stdout: "mds_fscrypt_last_block_max_size": "4096", 2023-09-23T20:52:05.515 INFO:teuthology.orchestra.run.smithi026.stdout: "mds_go_bad_corrupt_dentry": "true", 2023-09-23T20:52:05.515 INFO:teuthology.orchestra.run.smithi026.stdout: "mds_hack_allow_loading_invalid_metadata": "false", 2023-09-23T20:52:05.515 INFO:teuthology.orchestra.run.smithi026.stdout: "mds_health_cache_threshold": "1.500000", 2023-09-23T20:52:05.515 INFO:teuthology.orchestra.run.smithi026.stdout: "mds_health_summarize_threshold": "10", 2023-09-23T20:52:05.515 INFO:teuthology.orchestra.run.smithi026.stdout: "mds_heartbeat_grace": "15.000000", 2023-09-23T20:52:05.515 INFO:teuthology.orchestra.run.smithi026.stdout: "mds_heartbeat_reset_grace": "1000", 2023-09-23T20:52:05.515 INFO:teuthology.orchestra.run.smithi026.stdout: "mds_inject_health_dummy": "false", 2023-09-23T20:52:05.516 INFO:teuthology.orchestra.run.smithi026.stdout: "mds_inject_journal_corrupt_dentry_first": "0.000000", 2023-09-23T20:52:05.516 INFO:teuthology.orchestra.run.smithi026.stdout: "mds_inject_migrator_session_race": "false", 2023-09-23T20:52:05.516 INFO:teuthology.orchestra.run.smithi026.stdout: "mds_inject_rename_corrupt_dentry_first": "0.000000", 2023-09-23T20:52:05.516 INFO:teuthology.orchestra.run.smithi026.stdout: "mds_inject_skip_replaying_inotable": "false", 2023-09-23T20:52:05.516 INFO:teuthology.orchestra.run.smithi026.stdout: "mds_inject_traceless_reply_probability": "0.000000", 2023-09-23T20:52:05.516 INFO:teuthology.orchestra.run.smithi026.stdout: "mds_join_fs": "", 2023-09-23T20:52:05.516 INFO:teuthology.orchestra.run.smithi026.stdout: "mds_journal_format": "1", 2023-09-23T20:52:05.516 INFO:teuthology.orchestra.run.smithi026.stdout: "mds_kill_create_at": "0", 2023-09-23T20:52:05.516 INFO:teuthology.orchestra.run.smithi026.stdout: "mds_kill_export_at": "0", 2023-09-23T20:52:05.516 INFO:teuthology.orchestra.run.smithi026.stdout: "mds_kill_import_at": "0", 2023-09-23T20:52:05.516 INFO:teuthology.orchestra.run.smithi026.stdout: "mds_kill_journal_at": "0", 2023-09-23T20:52:05.516 INFO:teuthology.orchestra.run.smithi026.stdout: "mds_kill_journal_expire_at": "0", 2023-09-23T20:52:05.517 INFO:teuthology.orchestra.run.smithi026.stdout: "mds_kill_journal_replay_at": "0", 2023-09-23T20:52:05.517 INFO:teuthology.orchestra.run.smithi026.stdout: "mds_kill_link_at": "0", 2023-09-23T20:52:05.517 INFO:teuthology.orchestra.run.smithi026.stdout: "mds_kill_mdstable_at": "0", 2023-09-23T20:52:05.517 INFO:teuthology.orchestra.run.smithi026.stdout: "mds_kill_openc_at": "0", 2023-09-23T20:52:05.517 INFO:teuthology.orchestra.run.smithi026.stdout: "mds_kill_rename_at": "0", 2023-09-23T20:52:05.517 INFO:teuthology.orchestra.run.smithi026.stdout: "mds_kill_skip_replaying_inotable": "false", 2023-09-23T20:52:05.517 INFO:teuthology.orchestra.run.smithi026.stdout: "mds_log_events_per_segment": "1024", 2023-09-23T20:52:05.517 INFO:teuthology.orchestra.run.smithi026.stdout: "mds_log_max_events": "-1", 2023-09-23T20:52:05.517 INFO:teuthology.orchestra.run.smithi026.stdout: "mds_log_max_segments": "128", 2023-09-23T20:52:05.517 INFO:teuthology.orchestra.run.smithi026.stdout: "mds_log_pause": "false", 2023-09-23T20:52:05.517 INFO:teuthology.orchestra.run.smithi026.stdout: "mds_log_segment_size": "0", 2023-09-23T20:52:05.517 INFO:teuthology.orchestra.run.smithi026.stdout: "mds_log_skip_corrupt_events": "false", 2023-09-23T20:52:05.518 INFO:teuthology.orchestra.run.smithi026.stdout: "mds_log_warn_factor": "2.000000", 2023-09-23T20:52:05.518 INFO:teuthology.orchestra.run.smithi026.stdout: "mds_max_caps_per_client": "1048576", 2023-09-23T20:52:05.518 INFO:teuthology.orchestra.run.smithi026.stdout: "mds_max_completed_flushes": "100000", 2023-09-23T20:52:05.518 INFO:teuthology.orchestra.run.smithi026.stdout: "mds_max_completed_requests": "100000", 2023-09-23T20:52:05.518 INFO:teuthology.orchestra.run.smithi026.stdout: "mds_max_export_size": "20971520", 2023-09-23T20:52:05.518 INFO:teuthology.orchestra.run.smithi026.stdout: "mds_max_file_recover": "32", 2023-09-23T20:52:05.518 INFO:teuthology.orchestra.run.smithi026.stdout: "mds_max_purge_files": "64", 2023-09-23T20:52:05.518 INFO:teuthology.orchestra.run.smithi026.stdout: "mds_max_purge_ops": "8192", 2023-09-23T20:52:05.518 INFO:teuthology.orchestra.run.smithi026.stdout: "mds_max_purge_ops_per_pg": "0.500000", 2023-09-23T20:52:05.518 INFO:teuthology.orchestra.run.smithi026.stdout: "mds_max_scrub_ops_in_progress": "5", 2023-09-23T20:52:05.518 INFO:teuthology.orchestra.run.smithi026.stdout: "mds_max_snaps_per_dir": "100", 2023-09-23T20:52:05.519 INFO:teuthology.orchestra.run.smithi026.stdout: "mds_max_xattr_pairs_size": "65536", 2023-09-23T20:52:05.519 INFO:teuthology.orchestra.run.smithi026.stdout: "mds_metrics_update_interval": "2", 2023-09-23T20:52:05.519 INFO:teuthology.orchestra.run.smithi026.stdout: "mds_min_caps_per_client": "100", 2023-09-23T20:52:05.519 INFO:teuthology.orchestra.run.smithi026.stdout: "mds_min_caps_working_set": "10000", 2023-09-23T20:52:05.519 INFO:teuthology.orchestra.run.smithi026.stdout: "mds_mon_shutdown_timeout": "5.000000", 2023-09-23T20:52:05.519 INFO:teuthology.orchestra.run.smithi026.stdout: "mds_numa_node": "-1", 2023-09-23T20:52:05.519 INFO:teuthology.orchestra.run.smithi026.stdout: "mds_oft_prefetch_dirfrags": "false", 2023-09-23T20:52:05.519 INFO:teuthology.orchestra.run.smithi026.stdout: "mds_op_complaint_time": "30.000000", 2023-09-23T20:52:05.519 INFO:teuthology.orchestra.run.smithi026.stdout: "mds_op_history_duration": "600", 2023-09-23T20:52:05.519 INFO:teuthology.orchestra.run.smithi026.stdout: "mds_op_history_size": "20", 2023-09-23T20:52:05.519 INFO:teuthology.orchestra.run.smithi026.stdout: "mds_op_log_threshold": "5", 2023-09-23T20:52:05.519 INFO:teuthology.orchestra.run.smithi026.stdout: "mds_ping_grace": "15", 2023-09-23T20:52:05.519 INFO:teuthology.orchestra.run.smithi026.stdout: "mds_ping_interval": "5", 2023-09-23T20:52:05.520 INFO:teuthology.orchestra.run.smithi026.stdout: "mds_purge_queue_busy_flush_period": "1.000000", 2023-09-23T20:52:05.520 INFO:teuthology.orchestra.run.smithi026.stdout: "mds_recall_global_max_decay_threshold": "131072", 2023-09-23T20:52:05.520 INFO:teuthology.orchestra.run.smithi026.stdout: "mds_recall_max_caps": "30000", 2023-09-23T20:52:05.520 INFO:teuthology.orchestra.run.smithi026.stdout: "mds_recall_max_decay_rate": "1.500000", 2023-09-23T20:52:05.520 INFO:teuthology.orchestra.run.smithi026.stdout: "mds_recall_max_decay_threshold": "131072", 2023-09-23T20:52:05.520 INFO:teuthology.orchestra.run.smithi026.stdout: "mds_recall_warning_decay_rate": "60.000000", 2023-09-23T20:52:05.520 INFO:teuthology.orchestra.run.smithi026.stdout: "mds_recall_warning_threshold": "262144", 2023-09-23T20:52:05.520 INFO:teuthology.orchestra.run.smithi026.stdout: "mds_reconnect_timeout": "45.000000", 2023-09-23T20:52:05.520 INFO:teuthology.orchestra.run.smithi026.stdout: "mds_replay_interval": "1.000000", 2023-09-23T20:52:05.520 INFO:teuthology.orchestra.run.smithi026.stdout: "mds_replay_unsafe_with_closed_session": "false", 2023-09-23T20:52:05.520 INFO:teuthology.orchestra.run.smithi026.stdout: "mds_request_load_average_decay_rate": "60.000000", 2023-09-23T20:52:05.520 INFO:teuthology.orchestra.run.smithi026.stdout: "mds_root_ino_gid": "0", 2023-09-23T20:52:05.521 INFO:teuthology.orchestra.run.smithi026.stdout: "mds_root_ino_uid": "0", 2023-09-23T20:52:05.521 INFO:teuthology.orchestra.run.smithi026.stdout: "mds_scatter_nudge_interval": "5.000000", 2023-09-23T20:52:05.521 INFO:teuthology.orchestra.run.smithi026.stdout: "mds_session_blocklist_on_evict": "true", 2023-09-23T20:52:05.521 INFO:teuthology.orchestra.run.smithi026.stdout: "mds_session_blocklist_on_timeout": "true", 2023-09-23T20:52:05.521 INFO:teuthology.orchestra.run.smithi026.stdout: "mds_session_cache_liveness_decay_rate": "300.000000", 2023-09-23T20:52:05.521 INFO:teuthology.orchestra.run.smithi026.stdout: "mds_session_cache_liveness_magnitude": "10", 2023-09-23T20:52:05.521 INFO:teuthology.orchestra.run.smithi026.stdout: "mds_session_cap_acquisition_decay_rate": "30.000000", 2023-09-23T20:52:05.521 INFO:teuthology.orchestra.run.smithi026.stdout: "mds_session_cap_acquisition_throttle": "100000", 2023-09-23T20:52:05.521 INFO:teuthology.orchestra.run.smithi026.stdout: "mds_session_max_caps_throttle_ratio": "1.100000", 2023-09-23T20:52:05.521 INFO:teuthology.orchestra.run.smithi026.stdout: "mds_sessionmap_keys_per_op": "1024", 2023-09-23T20:52:05.521 INFO:teuthology.orchestra.run.smithi026.stdout: "mds_shutdown_check": "0", 2023-09-23T20:52:05.521 INFO:teuthology.orchestra.run.smithi026.stdout: "mds_skip_ino": "0", 2023-09-23T20:52:05.521 INFO:teuthology.orchestra.run.smithi026.stdout: "mds_sleep_rank_change": "0.000000", 2023-09-23T20:52:05.522 INFO:teuthology.orchestra.run.smithi026.stdout: "mds_snap_max_uid": "4294967294", 2023-09-23T20:52:05.522 INFO:teuthology.orchestra.run.smithi026.stdout: "mds_snap_min_uid": "0", 2023-09-23T20:52:05.522 INFO:teuthology.orchestra.run.smithi026.stdout: "mds_snap_rstat": "false", 2023-09-23T20:52:05.522 INFO:teuthology.orchestra.run.smithi026.stdout: "mds_standby_replay_damaged": "false", 2023-09-23T20:52:05.522 INFO:teuthology.orchestra.run.smithi026.stdout: "mds_symlink_recovery": "true", 2023-09-23T20:52:05.522 INFO:teuthology.orchestra.run.smithi026.stdout: "mds_task_status_update_interval": "2.000000", 2023-09-23T20:52:05.522 INFO:teuthology.orchestra.run.smithi026.stdout: "mds_thrash_exports": "0", 2023-09-23T20:52:05.522 INFO:teuthology.orchestra.run.smithi026.stdout: "mds_thrash_fragments": "0", 2023-09-23T20:52:05.522 INFO:teuthology.orchestra.run.smithi026.stdout: "mds_tick_interval": "5.000000", 2023-09-23T20:52:05.523 INFO:teuthology.orchestra.run.smithi026.stdout: "mds_valgrind_exit": "false", 2023-09-23T20:52:05.523 INFO:teuthology.orchestra.run.smithi026.stdout: "mds_verify_backtrace": "1", 2023-09-23T20:52:05.523 INFO:teuthology.orchestra.run.smithi026.stdout: "mds_verify_scatter": "false", 2023-09-23T20:52:05.523 INFO:teuthology.orchestra.run.smithi026.stdout: "mds_wipe_ino_prealloc": "false", 2023-09-23T20:52:05.523 INFO:teuthology.orchestra.run.smithi026.stdout: "mds_wipe_sessions": "false", 2023-09-23T20:52:05.523 INFO:teuthology.orchestra.run.smithi026.stdout: "mempool_debug": "false", 2023-09-23T20:52:05.523 INFO:teuthology.orchestra.run.smithi026.stdout: "memstore_debug_omit_block_device_write": "false", 2023-09-23T20:52:05.523 INFO:teuthology.orchestra.run.smithi026.stdout: "memstore_device_bytes": "1073741824", 2023-09-23T20:52:05.523 INFO:teuthology.orchestra.run.smithi026.stdout: "memstore_page_set": "false", 2023-09-23T20:52:05.523 INFO:teuthology.orchestra.run.smithi026.stdout: "memstore_page_size": "65536", 2023-09-23T20:52:05.523 INFO:teuthology.orchestra.run.smithi026.stdout: "mgr_client_bytes": "134217728", 2023-09-23T20:52:05.523 INFO:teuthology.orchestra.run.smithi026.stdout: "mgr_client_messages": "512", 2023-09-23T20:52:05.524 INFO:teuthology.orchestra.run.smithi026.stdout: "mgr_client_service_daemon_unregister_timeout": "1.000000", 2023-09-23T20:52:05.524 INFO:teuthology.orchestra.run.smithi026.stdout: "mgr_connect_retry_interval": "1.000000", 2023-09-23T20:52:05.524 INFO:teuthology.orchestra.run.smithi026.stdout: "mgr_data": "/var/lib/ceph/mgr/ceph-a", 2023-09-23T20:52:05.524 INFO:teuthology.orchestra.run.smithi026.stdout: "mgr_debug_aggressive_pg_num_changes": "false", 2023-09-23T20:52:05.524 INFO:teuthology.orchestra.run.smithi026.stdout: "mgr_disabled_modules": "", 2023-09-23T20:52:05.524 INFO:teuthology.orchestra.run.smithi026.stdout: "mgr_initial_modules": "restful iostat nfs", 2023-09-23T20:52:05.524 INFO:teuthology.orchestra.run.smithi026.stdout: "mgr_max_pg_num_change": "128", 2023-09-23T20:52:05.524 INFO:teuthology.orchestra.run.smithi026.stdout: "mgr_mds_bytes": "134217728", 2023-09-23T20:52:05.524 INFO:teuthology.orchestra.run.smithi026.stdout: "mgr_mds_messages": "128", 2023-09-23T20:52:05.524 INFO:teuthology.orchestra.run.smithi026.stdout: "mgr_module_path": "/usr/share/ceph/mgr", 2023-09-23T20:52:05.524 INFO:teuthology.orchestra.run.smithi026.stdout: "mgr_mon_bytes": "134217728", 2023-09-23T20:52:05.524 INFO:teuthology.orchestra.run.smithi026.stdout: "mgr_mon_messages": "128", 2023-09-23T20:52:05.525 INFO:teuthology.orchestra.run.smithi026.stdout: "mgr_osd_bytes": "536870912", 2023-09-23T20:52:05.525 INFO:teuthology.orchestra.run.smithi026.stdout: "mgr_osd_messages": "8192", 2023-09-23T20:52:05.525 INFO:teuthology.orchestra.run.smithi026.stdout: "mgr_pool": "true", 2023-09-23T20:52:05.525 INFO:teuthology.orchestra.run.smithi026.stdout: "mgr_service_beacon_grace": "60.000000", 2023-09-23T20:52:05.525 INFO:teuthology.orchestra.run.smithi026.stdout: "mgr_standby_modules": "true", 2023-09-23T20:52:05.525 INFO:teuthology.orchestra.run.smithi026.stdout: "mgr_stats_period": "5", 2023-09-23T20:52:05.525 INFO:teuthology.orchestra.run.smithi026.stdout: "mgr_stats_threshold": "5", 2023-09-23T20:52:05.525 INFO:teuthology.orchestra.run.smithi026.stdout: "mgr_tick_period": "2", 2023-09-23T20:52:05.525 INFO:teuthology.orchestra.run.smithi026.stdout: "mgr_ttl_cache_expire_seconds": "0", 2023-09-23T20:52:05.525 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_accept_timeout_factor": "2.000000", 2023-09-23T20:52:05.525 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_allow_pool_delete": "true", 2023-09-23T20:52:05.525 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_allow_pool_size_one": "false", 2023-09-23T20:52:05.526 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_auth_validate_all_caps": "true", 2023-09-23T20:52:05.526 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_cache_target_full_warn_ratio": "0.660000", 2023-09-23T20:52:05.526 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_clean_pg_upmaps_per_chunk": "256", 2023-09-23T20:52:05.526 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_client_bytes": "104857600", 2023-09-23T20:52:05.526 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_client_directed_command_retry": "2", 2023-09-23T20:52:05.526 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_client_hunt_interval": "3.000000", 2023-09-23T20:52:05.526 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_client_hunt_interval_backoff": "1.500000", 2023-09-23T20:52:05.526 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_client_hunt_interval_max_multiple": "10.000000", 2023-09-23T20:52:05.526 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_client_hunt_interval_min_multiple": "1.000000", 2023-09-23T20:52:05.526 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_client_hunt_parallel": "3", 2023-09-23T20:52:05.526 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_client_log_interval": "1.000000", 2023-09-23T20:52:05.526 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_client_max_log_entries_per_message": "1000", 2023-09-23T20:52:05.527 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_client_ping_interval": "10.000000", 2023-09-23T20:52:05.527 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_client_ping_timeout": "30.000000", 2023-09-23T20:52:05.527 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_clock_drift_allowed": "1.000000", 2023-09-23T20:52:05.527 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_clock_drift_warn_backoff": "5.000000", 2023-09-23T20:52:05.527 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_cluster_log_file": "default=/var/log/ceph/ceph.$channel.log cluster=/var/log/ceph/ceph.log", 2023-09-23T20:52:05.527 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_cluster_log_file_level": "debug", 2023-09-23T20:52:05.527 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_cluster_log_to_file": "false", 2023-09-23T20:52:05.527 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_cluster_log_to_graylog": "false", 2023-09-23T20:52:05.527 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_cluster_log_to_graylog_host": "127.0.0.1", 2023-09-23T20:52:05.527 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_cluster_log_to_graylog_port": "12201", 2023-09-23T20:52:05.527 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_cluster_log_to_journald": "true", 2023-09-23T20:52:05.527 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_cluster_log_to_stderr": "false", 2023-09-23T20:52:05.527 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_cluster_log_to_syslog": "default=false", 2023-09-23T20:52:05.528 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_cluster_log_to_syslog_facility": "daemon", 2023-09-23T20:52:05.528 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_cluster_log_to_syslog_level": "info", 2023-09-23T20:52:05.528 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_compact_on_bootstrap": "false", 2023-09-23T20:52:05.528 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_compact_on_start": "false", 2023-09-23T20:52:05.528 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_compact_on_trim": "true", 2023-09-23T20:52:05.528 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_con_tracker_persist_interval": "10", 2023-09-23T20:52:05.528 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_con_tracker_score_halflife": "43200", 2023-09-23T20:52:05.528 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_config_key_max_entry_size": "65536", 2023-09-23T20:52:05.528 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_cpu_threads": "4", 2023-09-23T20:52:05.528 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_crush_min_required_version": "hammer", 2023-09-23T20:52:05.528 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_daemon_bytes": "419430400", 2023-09-23T20:52:05.528 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_data": "/var/lib/ceph/mon/ceph-a", 2023-09-23T20:52:05.529 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_data_avail_crit": "5", 2023-09-23T20:52:05.529 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_data_avail_warn": "5", 2023-09-23T20:52:05.529 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_data_size_warn": "16106127360", 2023-09-23T20:52:05.529 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_debug_block_osdmap_trim": "false", 2023-09-23T20:52:05.529 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_debug_deprecated_as_obsolete": "false", 2023-09-23T20:52:05.529 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_debug_dump_json": "false", 2023-09-23T20:52:05.529 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_debug_dump_location": "/var/log/ceph/ceph-mon.a.tdump", 2023-09-23T20:52:05.529 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_debug_dump_transactions": "false", 2023-09-23T20:52:05.529 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_debug_extra_checks": "false", 2023-09-23T20:52:05.529 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_debug_no_initial_persistent_features": "false", 2023-09-23T20:52:05.529 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_debug_no_require_bluestore_for_ec_overwrites": "false", 2023-09-23T20:52:05.529 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_debug_no_require_quincy": "false", 2023-09-23T20:52:05.529 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_debug_no_require_reef": "false", 2023-09-23T20:52:05.530 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_debug_unsafe_allow_tier_with_nonempty_snaps": "false", 2023-09-23T20:52:05.530 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_delta_reset_interval": "10.000000", 2023-09-23T20:52:05.530 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_dns_srv_name": "ceph-mon", 2023-09-23T20:52:05.530 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_down_mkfs_grace": "60", 2023-09-23T20:52:05.530 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_election_default_strategy": "3", 2023-09-23T20:52:05.530 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_election_timeout": "5.000000", 2023-09-23T20:52:05.530 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_elector_ignore_propose_margin": "0.000500", 2023-09-23T20:52:05.530 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_elector_ping_divisor": "2", 2023-09-23T20:52:05.530 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_elector_ping_timeout": "2.000000", 2023-09-23T20:52:05.530 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_enable_op_tracker": "true", 2023-09-23T20:52:05.530 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_fake_pool_delete": "false", 2023-09-23T20:52:05.531 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_force_quorum_join": "false", 2023-09-23T20:52:05.531 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_globalid_prealloc": "10000", 2023-09-23T20:52:05.531 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_health_detail_to_clog": "true", 2023-09-23T20:52:05.531 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_health_log_update_period": "5", 2023-09-23T20:52:05.531 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_health_max_detail": "50", 2023-09-23T20:52:05.531 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_health_to_clog": "true", 2023-09-23T20:52:05.531 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_health_to_clog_interval": "600", 2023-09-23T20:52:05.531 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_health_to_clog_tick_interval": "60.000000", 2023-09-23T20:52:05.531 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_host": "[v2:172.21.15.26:3300/0,v1:172.21.15.26:6789/0] [v2:172.21.15.70:3300/0,v1:172.21.15.70:6789/0] [v2:172.21.15.26:3301/0,v1:172.21.15.26:6790/0]", 2023-09-23T20:52:05.531 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_host_override": "", 2023-09-23T20:52:05.531 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_initial_members": "", 2023-09-23T20:52:05.531 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_inject_pg_merge_bounce_probability": "0.000000", 2023-09-23T20:52:05.531 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_inject_sync_get_chunk_delay": "0.000000", 2023-09-23T20:52:05.532 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_inject_transaction_delay_max": "10.000000", 2023-09-23T20:52:05.532 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_inject_transaction_delay_probability": "0.000000", 2023-09-23T20:52:05.532 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_keyvaluedb": "rocksdb", 2023-09-23T20:52:05.532 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_lease": "5.000000", 2023-09-23T20:52:05.532 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_lease_ack_timeout_factor": "2.000000", 2023-09-23T20:52:05.532 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_lease_renew_interval_factor": "0.600000", 2023-09-23T20:52:05.532 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_log_full_interval": "50", 2023-09-23T20:52:05.532 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_log_max": "10000", 2023-09-23T20:52:05.532 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_log_max_summary": "50", 2023-09-23T20:52:05.532 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_max_log_entries_per_event": "4096", 2023-09-23T20:52:05.532 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_max_log_epochs": "500", 2023-09-23T20:52:05.532 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_max_mdsmap_epochs": "500", 2023-09-23T20:52:05.533 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_max_mgrmap_epochs": "500", 2023-09-23T20:52:05.533 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_max_osd": "10000", 2023-09-23T20:52:05.533 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_max_pg_per_osd": "10000", 2023-09-23T20:52:05.533 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_max_pool_pg_num": "65536", 2023-09-23T20:52:05.533 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_max_snap_prune_per_epoch": "100", 2023-09-23T20:52:05.533 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_mds_blocklist_interval": "86400.000000", 2023-09-23T20:52:05.533 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_mds_force_trim_to": "0", 2023-09-23T20:52:05.533 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_mds_skip_sanity": "true", 2023-09-23T20:52:05.533 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_memory_autotune": "true", 2023-09-23T20:52:05.533 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_memory_target": "2147483648", 2023-09-23T20:52:05.533 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_mgr_beacon_grace": "30", 2023-09-23T20:52:05.533 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_mgr_blocklist_interval": "86400.000000", 2023-09-23T20:52:05.534 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_mgr_digest_period": "5", 2023-09-23T20:52:05.534 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_mgr_inactive_grace": "60", 2023-09-23T20:52:05.534 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_mgr_mkfs_grace": "240", 2023-09-23T20:52:05.534 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_mgr_proxy_client_bytes_ratio": "0.300000", 2023-09-23T20:52:05.534 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_min_osdmap_epochs": "500", 2023-09-23T20:52:05.534 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_op_complaint_time": "30", 2023-09-23T20:52:05.534 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_op_history_duration": "600", 2023-09-23T20:52:05.534 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_op_history_size": "20", 2023-09-23T20:52:05.534 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_op_history_slow_op_size": "20", 2023-09-23T20:52:05.534 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_op_history_slow_op_threshold": "10", 2023-09-23T20:52:05.534 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_op_log_threshold": "5", 2023-09-23T20:52:05.535 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_osd_adjust_down_out_interval": "true", 2023-09-23T20:52:05.535 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_osd_adjust_heartbeat_grace": "true", 2023-09-23T20:52:05.535 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_osd_auto_mark_auto_out_in": "true", 2023-09-23T20:52:05.535 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_osd_auto_mark_in": "false", 2023-09-23T20:52:05.535 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_osd_auto_mark_new_in": "true", 2023-09-23T20:52:05.535 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_osd_backfillfull_ratio": "0.900000", 2023-09-23T20:52:05.535 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_osd_blocklist_default_expire": "3600.000000", 2023-09-23T20:52:05.535 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_osd_cache_size": "500", 2023-09-23T20:52:05.535 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_osd_cache_size_min": "134217728", 2023-09-23T20:52:05.535 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_osd_crush_smoke_test": "true", 2023-09-23T20:52:05.535 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_osd_destroyed_out_interval": "600", 2023-09-23T20:52:05.535 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_osd_down_out_interval": "600", 2023-09-23T20:52:05.535 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_osd_down_out_subtree_limit": "rack", 2023-09-23T20:52:05.536 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_osd_err_op_age_ratio": "128.000000", 2023-09-23T20:52:05.536 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_osd_force_trim_to": "0", 2023-09-23T20:52:05.536 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_osd_full_ratio": "0.950000", 2023-09-23T20:52:05.536 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_osd_initial_require_min_compat_client": "luminous", 2023-09-23T20:52:05.536 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_osd_laggy_halflife": "3600", 2023-09-23T20:52:05.536 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_osd_laggy_max_interval": "300", 2023-09-23T20:52:05.536 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_osd_laggy_weight": "0.300000", 2023-09-23T20:52:05.536 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_osd_mapping_pgs_per_chunk": "4096", 2023-09-23T20:52:05.536 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_osd_max_creating_pgs": "1024", 2023-09-23T20:52:05.536 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_osd_max_initial_pgs": "1024", 2023-09-23T20:52:05.536 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_osd_min_down_reporters": "2", 2023-09-23T20:52:05.536 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_osd_min_in_ratio": "0.750000", 2023-09-23T20:52:05.536 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_osd_min_up_ratio": "0.300000", 2023-09-23T20:52:05.537 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_osd_nearfull_ratio": "0.850000", 2023-09-23T20:52:05.537 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_osd_prime_pg_temp": "true", 2023-09-23T20:52:05.537 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_osd_prime_pg_temp_max_estimate": "0.250000", 2023-09-23T20:52:05.537 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_osd_prime_pg_temp_max_time": "0.500000", 2023-09-23T20:52:05.537 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_osd_report_timeout": "900", 2023-09-23T20:52:05.537 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_osd_reporter_subtree_level": "osd", 2023-09-23T20:52:05.537 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_osd_snap_trim_queue_warn_on": "32768", 2023-09-23T20:52:05.537 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_osd_warn_num_repaired": "10", 2023-09-23T20:52:05.537 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_osd_warn_op_age": "32.000000", 2023-09-23T20:52:05.537 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_osdmap_full_prune_enabled": "true", 2023-09-23T20:52:05.537 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_osdmap_full_prune_interval": "10", 2023-09-23T20:52:05.537 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_osdmap_full_prune_min": "10000", 2023-09-23T20:52:05.537 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_osdmap_full_prune_txsize": "100", 2023-09-23T20:52:05.538 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_pg_check_down_all_threshold": "0.500000", 2023-09-23T20:52:05.538 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_pg_stuck_threshold": "60", 2023-09-23T20:52:05.538 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_pg_warn_max_object_skew": "0.000000", 2023-09-23T20:52:05.538 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_pg_warn_min_objects": "10000", 2023-09-23T20:52:05.538 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_pg_warn_min_per_osd": "0", 2023-09-23T20:52:05.538 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_pg_warn_min_pool_objects": "1000", 2023-09-23T20:52:05.538 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_pool_quota_crit_threshold": "0", 2023-09-23T20:52:05.538 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_pool_quota_warn_threshold": "0", 2023-09-23T20:52:05.538 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_probe_timeout": "2.000000", 2023-09-23T20:52:05.538 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_reweight_max_change": "0.050000", 2023-09-23T20:52:05.538 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_reweight_max_osds": "4", 2023-09-23T20:52:05.538 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_reweight_min_bytes_per_osd": "10", 2023-09-23T20:52:05.539 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_reweight_min_pgs_per_osd": "4", 2023-09-23T20:52:05.539 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_rocksdb_options": "write_buffer_size=33554432,compression=kNoCompression,level_compaction_dynamic_level_bytes=true", 2023-09-23T20:52:05.539 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_scrub_inject_crc_mismatch": "0.000000", 2023-09-23T20:52:05.539 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_scrub_inject_missing_keys": "0.000000", 2023-09-23T20:52:05.539 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_scrub_interval": "86400", 2023-09-23T20:52:05.539 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_scrub_max_keys": "100", 2023-09-23T20:52:05.539 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_scrub_timeout": "300", 2023-09-23T20:52:05.539 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_session_timeout": "300", 2023-09-23T20:52:05.539 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_smart_report_timeout": "5", 2023-09-23T20:52:05.539 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_stat_smooth_intervals": "6", 2023-09-23T20:52:05.539 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_stretch_cluster_recovery_ratio": "0.600000", 2023-09-23T20:52:05.539 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_stretch_pool_min_size": "2", 2023-09-23T20:52:05.539 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_stretch_pool_size": "4", 2023-09-23T20:52:05.540 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_stretch_recovery_min_wait": "15.000000", 2023-09-23T20:52:05.540 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_subscribe_interval": "86400.000000", 2023-09-23T20:52:05.540 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_sync_debug": "false", 2023-09-23T20:52:05.540 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_sync_max_payload_keys": "2000", 2023-09-23T20:52:05.540 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_sync_max_payload_size": "1048576", 2023-09-23T20:52:05.540 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_sync_provider_kill_at": "0", 2023-09-23T20:52:05.540 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_sync_requester_kill_at": "0", 2023-09-23T20:52:05.540 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_sync_timeout": "60.000000", 2023-09-23T20:52:05.540 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_target_pg_per_osd": "100", 2023-09-23T20:52:05.540 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_tick_interval": "5", 2023-09-23T20:52:05.540 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_timecheck_interval": "300.000000", 2023-09-23T20:52:05.540 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_timecheck_skew_interval": "30.000000", 2023-09-23T20:52:05.540 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_use_min_delay_socket": "false", 2023-09-23T20:52:05.541 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_warn_older_version_delay": "604800", 2023-09-23T20:52:05.541 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_warn_on_cache_pools_without_hit_sets": "true", 2023-09-23T20:52:05.541 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_warn_on_crush_straw_calc_version_zero": "false", 2023-09-23T20:52:05.541 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_warn_on_degraded_stretch_mode": "true", 2023-09-23T20:52:05.541 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_warn_on_filestore_osds": "true", 2023-09-23T20:52:05.541 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_warn_on_insecure_global_id_reclaim": "false", 2023-09-23T20:52:05.541 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_warn_on_insecure_global_id_reclaim_allowed": "false", 2023-09-23T20:52:05.541 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_warn_on_legacy_crush_tunables": "false", 2023-09-23T20:52:05.541 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_warn_on_misplaced": "false", 2023-09-23T20:52:05.541 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_warn_on_msgr2_not_enabled": "true", 2023-09-23T20:52:05.541 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_warn_on_older_version": "true", 2023-09-23T20:52:05.541 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_warn_on_osd_down_out_interval_zero": "false", 2023-09-23T20:52:05.542 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_warn_on_pool_no_app": "true", 2023-09-23T20:52:05.542 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_warn_on_pool_no_redundancy": "true", 2023-09-23T20:52:05.542 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_warn_on_pool_pg_num_not_power_of_two": "false", 2023-09-23T20:52:05.542 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_warn_on_slow_ping_ratio": "0.050000", 2023-09-23T20:52:05.542 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_warn_on_slow_ping_time": "0.000000", 2023-09-23T20:52:05.542 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_warn_on_too_few_osds": "false", 2023-09-23T20:52:05.542 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_warn_pg_not_deep_scrubbed_ratio": "0.750000", 2023-09-23T20:52:05.542 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_warn_pg_not_scrubbed_ratio": "0.500000", 2023-09-23T20:52:05.542 INFO:teuthology.orchestra.run.smithi026.stdout: "monmap": "", 2023-09-23T20:52:05.542 INFO:teuthology.orchestra.run.smithi026.stdout: "motr_admin_endpoint": "192.168.180.182@tcp:12345:4:1", 2023-09-23T20:52:05.542 INFO:teuthology.orchestra.run.smithi026.stdout: "motr_admin_fid": "0x7200000000000001:0x0", 2023-09-23T20:52:05.542 INFO:teuthology.orchestra.run.smithi026.stdout: "motr_ha_endpoint": "192.168.180.182@tcp:12345:1:1", 2023-09-23T20:52:05.543 INFO:teuthology.orchestra.run.smithi026.stdout: "motr_my_endpoint": "192.168.180.182@tcp:12345:4:1", 2023-09-23T20:52:05.543 INFO:teuthology.orchestra.run.smithi026.stdout: "motr_my_fid": "0x7200000000000001:0x0", 2023-09-23T20:52:05.543 INFO:teuthology.orchestra.run.smithi026.stdout: "motr_profile_fid": "0x7000000000000001:0x0", 2023-09-23T20:52:05.543 INFO:teuthology.orchestra.run.smithi026.stdout: "motr_tracing_enabled": "false", 2023-09-23T20:52:05.543 INFO:teuthology.orchestra.run.smithi026.stdout: "ms_async_op_threads": "3", 2023-09-23T20:52:05.543 INFO:teuthology.orchestra.run.smithi026.stdout: "ms_async_rdma_buffer_size": "131072", 2023-09-23T20:52:05.543 INFO:teuthology.orchestra.run.smithi026.stdout: "ms_async_rdma_cm": "false", 2023-09-23T20:52:05.543 INFO:teuthology.orchestra.run.smithi026.stdout: "ms_async_rdma_device_name": "", 2023-09-23T20:52:05.543 INFO:teuthology.orchestra.run.smithi026.stdout: "ms_async_rdma_dscp": "96", 2023-09-23T20:52:05.543 INFO:teuthology.orchestra.run.smithi026.stdout: "ms_async_rdma_enable_hugepage": "false", 2023-09-23T20:52:05.543 INFO:teuthology.orchestra.run.smithi026.stdout: "ms_async_rdma_gid_idx": "0", 2023-09-23T20:52:05.543 INFO:teuthology.orchestra.run.smithi026.stdout: "ms_async_rdma_local_gid": "", 2023-09-23T20:52:05.544 INFO:teuthology.orchestra.run.smithi026.stdout: "ms_async_rdma_polling_us": "1000", 2023-09-23T20:52:05.544 INFO:teuthology.orchestra.run.smithi026.stdout: "ms_async_rdma_port_num": "1", 2023-09-23T20:52:05.544 INFO:teuthology.orchestra.run.smithi026.stdout: "ms_async_rdma_receive_buffers": "32768", 2023-09-23T20:52:05.544 INFO:teuthology.orchestra.run.smithi026.stdout: "ms_async_rdma_receive_queue_len": "4096", 2023-09-23T20:52:05.544 INFO:teuthology.orchestra.run.smithi026.stdout: "ms_async_rdma_roce_ver": "1", 2023-09-23T20:52:05.544 INFO:teuthology.orchestra.run.smithi026.stdout: "ms_async_rdma_send_buffers": "1024", 2023-09-23T20:52:05.544 INFO:teuthology.orchestra.run.smithi026.stdout: "ms_async_rdma_sl": "3", 2023-09-23T20:52:05.544 INFO:teuthology.orchestra.run.smithi026.stdout: "ms_async_rdma_support_srq": "true", 2023-09-23T20:52:05.544 INFO:teuthology.orchestra.run.smithi026.stdout: "ms_async_rdma_type": "ib", 2023-09-23T20:52:05.544 INFO:teuthology.orchestra.run.smithi026.stdout: "ms_async_reap_threshold": "5", 2023-09-23T20:52:05.544 INFO:teuthology.orchestra.run.smithi026.stdout: "ms_bind_before_connect": "false", 2023-09-23T20:52:05.544 INFO:teuthology.orchestra.run.smithi026.stdout: "ms_bind_ipv4": "true", 2023-09-23T20:52:05.544 INFO:teuthology.orchestra.run.smithi026.stdout: "ms_bind_ipv6": "false", 2023-09-23T20:52:05.545 INFO:teuthology.orchestra.run.smithi026.stdout: "ms_bind_msgr1": "true", 2023-09-23T20:52:05.545 INFO:teuthology.orchestra.run.smithi026.stdout: "ms_bind_msgr2": "true", 2023-09-23T20:52:05.545 INFO:teuthology.orchestra.run.smithi026.stdout: "ms_bind_port_max": "7568", 2023-09-23T20:52:05.545 INFO:teuthology.orchestra.run.smithi026.stdout: "ms_bind_port_min": "6800", 2023-09-23T20:52:05.545 INFO:teuthology.orchestra.run.smithi026.stdout: "ms_bind_prefer_ipv4": "false", 2023-09-23T20:52:05.545 INFO:teuthology.orchestra.run.smithi026.stdout: "ms_bind_retry_count": "3", 2023-09-23T20:52:05.545 INFO:teuthology.orchestra.run.smithi026.stdout: "ms_bind_retry_delay": "5", 2023-09-23T20:52:05.545 INFO:teuthology.orchestra.run.smithi026.stdout: "ms_blackhole_client": "false", 2023-09-23T20:52:05.545 INFO:teuthology.orchestra.run.smithi026.stdout: "ms_blackhole_mds": "false", 2023-09-23T20:52:05.545 INFO:teuthology.orchestra.run.smithi026.stdout: "ms_blackhole_mgr": "false", 2023-09-23T20:52:05.545 INFO:teuthology.orchestra.run.smithi026.stdout: "ms_blackhole_mon": "false", 2023-09-23T20:52:05.545 INFO:teuthology.orchestra.run.smithi026.stdout: "ms_blackhole_osd": "false", 2023-09-23T20:52:05.545 INFO:teuthology.orchestra.run.smithi026.stdout: "ms_client_mode": "crc secure", 2023-09-23T20:52:05.546 INFO:teuthology.orchestra.run.smithi026.stdout: "ms_cluster_mode": "crc secure", 2023-09-23T20:52:05.546 INFO:teuthology.orchestra.run.smithi026.stdout: "ms_cluster_type": "", 2023-09-23T20:52:05.546 INFO:teuthology.orchestra.run.smithi026.stdout: "ms_compress_secure": "false", 2023-09-23T20:52:05.546 INFO:teuthology.orchestra.run.smithi026.stdout: "ms_connection_idle_timeout": "900", 2023-09-23T20:52:05.546 INFO:teuthology.orchestra.run.smithi026.stdout: "ms_connection_ready_timeout": "10", 2023-09-23T20:52:05.546 INFO:teuthology.orchestra.run.smithi026.stdout: "ms_crc_data": "true", 2023-09-23T20:52:05.546 INFO:teuthology.orchestra.run.smithi026.stdout: "ms_crc_header": "true", 2023-09-23T20:52:05.546 INFO:teuthology.orchestra.run.smithi026.stdout: "ms_die_on_bad_msg": "false", 2023-09-23T20:52:05.546 INFO:teuthology.orchestra.run.smithi026.stdout: "ms_die_on_bug": "true", 2023-09-23T20:52:05.546 INFO:teuthology.orchestra.run.smithi026.stdout: "ms_die_on_old_message": "true", 2023-09-23T20:52:05.546 INFO:teuthology.orchestra.run.smithi026.stdout: "ms_die_on_skipped_message": "false", 2023-09-23T20:52:05.546 INFO:teuthology.orchestra.run.smithi026.stdout: "ms_die_on_unhandled_msg": "false", 2023-09-23T20:52:05.547 INFO:teuthology.orchestra.run.smithi026.stdout: "ms_dispatch_throttle_bytes": "104857600", 2023-09-23T20:52:05.547 INFO:teuthology.orchestra.run.smithi026.stdout: "ms_dpdk_coremask": "0xF", 2023-09-23T20:52:05.547 INFO:teuthology.orchestra.run.smithi026.stdout: "ms_dpdk_debug_allow_loopback": "false", 2023-09-23T20:52:05.547 INFO:teuthology.orchestra.run.smithi026.stdout: "ms_dpdk_devs_allowlist": "", 2023-09-23T20:52:05.547 INFO:teuthology.orchestra.run.smithi026.stdout: "ms_dpdk_enable_tso": "true", 2023-09-23T20:52:05.547 INFO:teuthology.orchestra.run.smithi026.stdout: "ms_dpdk_gateway_ipv4_addr": "", 2023-09-23T20:52:05.547 INFO:teuthology.orchestra.run.smithi026.stdout: "ms_dpdk_host_ipv4_addr": "", 2023-09-23T20:52:05.547 INFO:teuthology.orchestra.run.smithi026.stdout: "ms_dpdk_hugepages": "", 2023-09-23T20:52:05.547 INFO:teuthology.orchestra.run.smithi026.stdout: "ms_dpdk_hw_flow_control": "true", 2023-09-23T20:52:05.547 INFO:teuthology.orchestra.run.smithi026.stdout: "ms_dpdk_hw_queue_weight": "1.000000", 2023-09-23T20:52:05.547 INFO:teuthology.orchestra.run.smithi026.stdout: "ms_dpdk_lro": "true", 2023-09-23T20:52:05.547 INFO:teuthology.orchestra.run.smithi026.stdout: "ms_dpdk_memory_channel": "4", 2023-09-23T20:52:05.547 INFO:teuthology.orchestra.run.smithi026.stdout: "ms_dpdk_netmask_ipv4_addr": "", 2023-09-23T20:52:05.547 INFO:teuthology.orchestra.run.smithi026.stdout: "ms_dpdk_pmd": "", 2023-09-23T20:52:05.548 INFO:teuthology.orchestra.run.smithi026.stdout: "ms_dpdk_port_id": "0", 2023-09-23T20:52:05.548 INFO:teuthology.orchestra.run.smithi026.stdout: "ms_dpdk_rx_buffer_count_per_core": "8192", 2023-09-23T20:52:05.548 INFO:teuthology.orchestra.run.smithi026.stdout: "ms_dump_corrupt_message_level": "1", 2023-09-23T20:52:05.548 INFO:teuthology.orchestra.run.smithi026.stdout: "ms_dump_on_send": "false", 2023-09-23T20:52:05.548 INFO:teuthology.orchestra.run.smithi026.stdout: "ms_initial_backoff": "0.200000", 2023-09-23T20:52:05.548 INFO:teuthology.orchestra.run.smithi026.stdout: "ms_inject_delay_max": "1.000000", 2023-09-23T20:52:05.548 INFO:teuthology.orchestra.run.smithi026.stdout: "ms_inject_delay_probability": "0.000000", 2023-09-23T20:52:05.548 INFO:teuthology.orchestra.run.smithi026.stdout: "ms_inject_delay_type": "", 2023-09-23T20:52:05.548 INFO:teuthology.orchestra.run.smithi026.stdout: "ms_inject_internal_delays": "0.000000", 2023-09-23T20:52:05.548 INFO:teuthology.orchestra.run.smithi026.stdout: "ms_inject_network_congestion": "0", 2023-09-23T20:52:05.548 INFO:teuthology.orchestra.run.smithi026.stdout: "ms_inject_socket_failures": "0", 2023-09-23T20:52:05.548 INFO:teuthology.orchestra.run.smithi026.stdout: "ms_learn_addr_from_peer": "true", 2023-09-23T20:52:05.549 INFO:teuthology.orchestra.run.smithi026.stdout: "ms_max_accept_failures": "4", 2023-09-23T20:52:05.549 INFO:teuthology.orchestra.run.smithi026.stdout: "ms_max_backoff": "15.000000", 2023-09-23T20:52:05.549 INFO:teuthology.orchestra.run.smithi026.stdout: "ms_mon_client_mode": "secure crc", 2023-09-23T20:52:05.549 INFO:teuthology.orchestra.run.smithi026.stdout: "ms_mon_cluster_mode": "secure crc", 2023-09-23T20:52:05.549 INFO:teuthology.orchestra.run.smithi026.stdout: "ms_mon_service_mode": "secure crc", 2023-09-23T20:52:05.549 INFO:teuthology.orchestra.run.smithi026.stdout: "ms_osd_compress_min_size": "1024", 2023-09-23T20:52:05.549 INFO:teuthology.orchestra.run.smithi026.stdout: "ms_osd_compress_mode": "none", 2023-09-23T20:52:05.549 INFO:teuthology.orchestra.run.smithi026.stdout: "ms_osd_compression_algorithm": "snappy", 2023-09-23T20:52:05.549 INFO:teuthology.orchestra.run.smithi026.stdout: "ms_pq_max_tokens_per_priority": "16777216", 2023-09-23T20:52:05.549 INFO:teuthology.orchestra.run.smithi026.stdout: "ms_pq_min_cost": "65536", 2023-09-23T20:52:05.549 INFO:teuthology.orchestra.run.smithi026.stdout: "ms_public_type": "", 2023-09-23T20:52:05.549 INFO:teuthology.orchestra.run.smithi026.stdout: "ms_service_mode": "crc secure", 2023-09-23T20:52:05.550 INFO:teuthology.orchestra.run.smithi026.stdout: "ms_tcp_listen_backlog": "512", 2023-09-23T20:52:05.550 INFO:teuthology.orchestra.run.smithi026.stdout: "ms_tcp_nodelay": "true", 2023-09-23T20:52:05.550 INFO:teuthology.orchestra.run.smithi026.stdout: "ms_tcp_prefetch_max_size": "4096", 2023-09-23T20:52:05.550 INFO:teuthology.orchestra.run.smithi026.stdout: "ms_tcp_rcvbuf": "0", 2023-09-23T20:52:05.550 INFO:teuthology.orchestra.run.smithi026.stdout: "ms_type": "async+posix", 2023-09-23T20:52:05.550 INFO:teuthology.orchestra.run.smithi026.stdout: "no_config_file": "false", 2023-09-23T20:52:05.550 INFO:teuthology.orchestra.run.smithi026.stdout: "objecter_completion_locks_per_session": "32", 2023-09-23T20:52:05.550 INFO:teuthology.orchestra.run.smithi026.stdout: "objecter_debug_inject_relock_delay": "false", 2023-09-23T20:52:05.550 INFO:teuthology.orchestra.run.smithi026.stdout: "objecter_inflight_op_bytes": "104857600", 2023-09-23T20:52:05.550 INFO:teuthology.orchestra.run.smithi026.stdout: "objecter_inflight_ops": "1024", 2023-09-23T20:52:05.550 INFO:teuthology.orchestra.run.smithi026.stdout: "objecter_inject_no_watch_ping": "false", 2023-09-23T20:52:05.550 INFO:teuthology.orchestra.run.smithi026.stdout: "objecter_retry_writes_after_first_reply": "false", 2023-09-23T20:52:05.551 INFO:teuthology.orchestra.run.smithi026.stdout: "objecter_tick_interval": "5.000000", 2023-09-23T20:52:05.551 INFO:teuthology.orchestra.run.smithi026.stdout: "objecter_timeout": "10.000000", 2023-09-23T20:52:05.551 INFO:teuthology.orchestra.run.smithi026.stdout: "objectstore_blackhole": "false", 2023-09-23T20:52:05.551 INFO:teuthology.orchestra.run.smithi026.stdout: "openssl_engine_opts": "", 2023-09-23T20:52:05.551 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_agent_delay_time": "5.000000", 2023-09-23T20:52:05.551 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_agent_hist_halflife": "1000", 2023-09-23T20:52:05.551 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_agent_max_low_ops": "2", 2023-09-23T20:52:05.551 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_agent_max_ops": "4", 2023-09-23T20:52:05.551 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_agent_min_evict_effort": "0.100000", 2023-09-23T20:52:05.551 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_agent_quantize_effort": "0.100000", 2023-09-23T20:52:05.551 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_agent_slop": "0.020000", 2023-09-23T20:52:05.551 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_aggregated_slow_ops_logging": "true", 2023-09-23T20:52:05.551 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_allow_recovery_below_min_size": "true", 2023-09-23T20:52:05.552 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_asio_thread_count": "2", 2023-09-23T20:52:05.552 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_async_recovery_min_cost": "100", 2023-09-23T20:52:05.552 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_auto_mark_unfound_lost": "false", 2023-09-23T20:52:05.552 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_backfill_retry_interval": "30.000000", 2023-09-23T20:52:05.552 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_backfill_scan_max": "512", 2023-09-23T20:52:05.552 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_backfill_scan_min": "64", 2023-09-23T20:52:05.552 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_backoff_on_degraded": "false", 2023-09-23T20:52:05.552 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_backoff_on_peering": "false", 2023-09-23T20:52:05.552 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_backoff_on_unfound": "true", 2023-09-23T20:52:05.552 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_beacon_report_interval": "300", 2023-09-23T20:52:05.552 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_bench_duration": "30", 2023-09-23T20:52:05.552 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_bench_large_size_max_throughput": "104857600", 2023-09-23T20:52:05.553 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_bench_max_block_size": "67108864", 2023-09-23T20:52:05.553 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_bench_small_size_max_iops": "100", 2023-09-23T20:52:05.553 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_blkin_trace_all": "false", 2023-09-23T20:52:05.553 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_blocked_scrub_grace_period": "120", 2023-09-23T20:52:05.553 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_calc_pg_upmaps_aggressively": "true", 2023-09-23T20:52:05.553 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_calc_pg_upmaps_aggressively_fast": "true", 2023-09-23T20:52:05.553 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_calc_pg_upmaps_local_fallback_retries": "100", 2023-09-23T20:52:05.553 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_check_for_log_corruption": "false", 2023-09-23T20:52:05.553 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_check_max_object_name_len_on_startup": "true", 2023-09-23T20:52:05.553 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_class_default_list": "cephfs hello journal lock log numops otp rbd refcount rgw rgw_gc timeindex user version cas cmpomap queue 2pc_queue fifo", 2023-09-23T20:52:05.553 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_class_dir": "/usr/lib64/rados-classes", 2023-09-23T20:52:05.553 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_class_load_list": "cephfs hello journal lock log numops otp rbd refcount rgw rgw_gc timeindex user version cas cmpomap queue 2pc_queue fifo", 2023-09-23T20:52:05.554 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_class_update_on_start": "true", 2023-09-23T20:52:05.554 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_client_message_cap": "256", 2023-09-23T20:52:05.554 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_client_message_size_cap": "524288000", 2023-09-23T20:52:05.554 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_client_op_priority": "63", 2023-09-23T20:52:05.554 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_client_watch_timeout": "30", 2023-09-23T20:52:05.554 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_command_max_records": "256", 2023-09-23T20:52:05.554 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_compact_on_start": "false", 2023-09-23T20:52:05.554 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_copyfrom_max_chunk": "8388608", 2023-09-23T20:52:05.554 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_crush_chooseleaf_type": "1", 2023-09-23T20:52:05.554 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_crush_initial_weight": "-1.000000", 2023-09-23T20:52:05.554 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_crush_update_on_start": "true", 2023-09-23T20:52:05.554 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_crush_update_weight_set": "true", 2023-09-23T20:52:05.555 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_data": "/var/lib/ceph/osd/ceph-a", 2023-09-23T20:52:05.555 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_debug_crash_on_ignored_backoff": "false", 2023-09-23T20:52:05.555 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_debug_deep_scrub_sleep": "0.000000", 2023-09-23T20:52:05.555 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_debug_drop_ping_duration": "0", 2023-09-23T20:52:05.555 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_debug_drop_ping_probability": "0.000000", 2023-09-23T20:52:05.555 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_debug_feed_pullee": "-1", 2023-09-23T20:52:05.555 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_debug_inject_copyfrom_error": "false", 2023-09-23T20:52:05.555 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_debug_inject_dispatch_delay_duration": "0.100000", 2023-09-23T20:52:05.555 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_debug_inject_dispatch_delay_probability": "0.000000", 2023-09-23T20:52:05.555 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_debug_misdirected_ops": "false", 2023-09-23T20:52:05.555 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_debug_no_acting_change": "false", 2023-09-23T20:52:05.555 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_debug_no_purge_strays": "false", 2023-09-23T20:52:05.556 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_debug_op_order": "false", 2023-09-23T20:52:05.556 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_debug_pg_log_writeout": "false", 2023-09-23T20:52:05.556 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_debug_pretend_recovery_active": "false", 2023-09-23T20:52:05.556 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_debug_random_push_read_error": "0.000000", 2023-09-23T20:52:05.556 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_debug_reject_backfill_probability": "0.000000", 2023-09-23T20:52:05.556 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_debug_shutdown": "false", 2023-09-23T20:52:05.556 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_debug_skip_full_check_in_backfill_reservation": "false", 2023-09-23T20:52:05.556 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_debug_skip_full_check_in_recovery": "false", 2023-09-23T20:52:05.556 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_debug_verify_cached_snaps": "false", 2023-09-23T20:52:05.556 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_debug_verify_missing_on_start": "false", 2023-09-23T20:52:05.556 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_debug_verify_snaps": "false", 2023-09-23T20:52:05.556 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_debug_verify_stray_on_activate": "false", 2023-09-23T20:52:05.557 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_deep_scrub_interval": "604800.000000", 2023-09-23T20:52:05.557 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_deep_scrub_keys": "1024", 2023-09-23T20:52:05.557 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_deep_scrub_large_omap_object_key_threshold": "200000", 2023-09-23T20:52:05.557 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_deep_scrub_large_omap_object_value_sum_threshold": "1073741824", 2023-09-23T20:52:05.557 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_deep_scrub_randomize_ratio": "0.150000", 2023-09-23T20:52:05.557 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_deep_scrub_stride": "524288", 2023-09-23T20:52:05.557 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_deep_scrub_update_digest_min_age": "7200", 2023-09-23T20:52:05.557 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_default_data_pool_replay_window": "45", 2023-09-23T20:52:05.557 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_default_notify_timeout": "30", 2023-09-23T20:52:05.557 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_delete_sleep": "0.000000", 2023-09-23T20:52:05.557 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_delete_sleep_hdd": "5.000000", 2023-09-23T20:52:05.557 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_delete_sleep_hybrid": "1.000000", 2023-09-23T20:52:05.558 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_delete_sleep_ssd": "1.000000", 2023-09-23T20:52:05.558 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_discard_disconnected_ops": "true", 2023-09-23T20:52:05.558 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_enable_op_tracker": "true", 2023-09-23T20:52:05.558 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_erasure_code_plugins": "jerasure lrc", 2023-09-23T20:52:05.558 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_extblkdev_plugins": "vdo", 2023-09-23T20:52:05.558 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_failsafe_full_ratio": "0.970000", 2023-09-23T20:52:05.558 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_fast_fail_on_connection_refused": "true", 2023-09-23T20:52:05.558 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_fast_info": "true", 2023-09-23T20:52:05.558 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_fast_shutdown": "true", 2023-09-23T20:52:05.558 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_fast_shutdown_notify_mon": "true", 2023-09-23T20:52:05.558 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_fast_shutdown_timeout": "15", 2023-09-23T20:52:05.558 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_find_best_info_ignore_history_les": "false", 2023-09-23T20:52:05.559 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_force_auth_primary_missing_objects": "100", 2023-09-23T20:52:05.559 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_force_recovery_pg_log_entries_factor": "1.300000", 2023-09-23T20:52:05.559 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_function_tracing": "false", 2023-09-23T20:52:05.559 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_heartbeat_grace": "20", 2023-09-23T20:52:05.559 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_heartbeat_interval": "6", 2023-09-23T20:52:05.559 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_heartbeat_min_healthy_ratio": "0.330000", 2023-09-23T20:52:05.559 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_heartbeat_min_peers": "10", 2023-09-23T20:52:05.559 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_heartbeat_min_size": "2000", 2023-09-23T20:52:05.559 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_heartbeat_stale": "600", 2023-09-23T20:52:05.559 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_heartbeat_use_min_delay_socket": "false", 2023-09-23T20:52:05.559 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_hit_set_max_size": "100000", 2023-09-23T20:52:05.559 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_hit_set_min_size": "1000", 2023-09-23T20:52:05.560 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_hit_set_namespace": ".ceph-internal", 2023-09-23T20:52:05.560 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_ignore_stale_divergent_priors": "false", 2023-09-23T20:52:05.560 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_inject_bad_map_crc_probability": "0.000000", 2023-09-23T20:52:05.560 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_inject_failure_on_pg_removal": "false", 2023-09-23T20:52:05.560 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_journal": "/var/lib/ceph/osd/ceph-a/journal", 2023-09-23T20:52:05.560 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_journal_flush_on_shutdown": "true", 2023-09-23T20:52:05.560 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_journal_size": "5120", 2023-09-23T20:52:05.560 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_kill_backfill_at": "0", 2023-09-23T20:52:05.560 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_loop_before_reset_tphandle": "64", 2023-09-23T20:52:05.560 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_map_cache_size": "50", 2023-09-23T20:52:05.560 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_map_dedup": "true", 2023-09-23T20:52:05.560 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_map_message_max": "40", 2023-09-23T20:52:05.561 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_map_message_max_bytes": "10485760", 2023-09-23T20:52:05.561 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_map_share_max_epochs": "40", 2023-09-23T20:52:05.561 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_max_attr_name_len": "100", 2023-09-23T20:52:05.561 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_max_attr_size": "0", 2023-09-23T20:52:05.561 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_max_backfills": "1", 2023-09-23T20:52:05.561 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_max_markdown_count": "5", 2023-09-23T20:52:05.561 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_max_markdown_period": "600", 2023-09-23T20:52:05.561 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_max_object_name_len": "2048", 2023-09-23T20:52:05.561 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_max_object_namespace_len": "256", 2023-09-23T20:52:05.561 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_max_object_size": "134217728", 2023-09-23T20:52:05.561 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_max_omap_bytes_per_request": "1073741824", 2023-09-23T20:52:05.561 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_max_omap_entries_per_request": "1024", 2023-09-23T20:52:05.562 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_max_pg_blocked_by": "16", 2023-09-23T20:52:05.562 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_max_pg_log_entries": "10000", 2023-09-23T20:52:05.562 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_max_pg_per_osd_hard_ratio": "3.000000", 2023-09-23T20:52:05.562 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_max_pgls": "1024", 2023-09-23T20:52:05.562 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_max_push_cost": "8388608", 2023-09-23T20:52:05.562 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_max_push_objects": "10", 2023-09-23T20:52:05.562 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_max_scrubs": "1", 2023-09-23T20:52:05.562 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_max_snap_prune_intervals_per_epoch": "512", 2023-09-23T20:52:05.562 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_max_trimming_pgs": "2", 2023-09-23T20:52:05.562 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_max_write_op_reply_len": "64", 2023-09-23T20:52:05.562 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_max_write_size": "90", 2023-09-23T20:52:05.562 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_mclock_force_run_benchmark_on_init": "false", 2023-09-23T20:52:05.563 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_mclock_iops_capacity_threshold_hdd": "500.000000", 2023-09-23T20:52:05.563 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_mclock_iops_capacity_threshold_ssd": "80000.000000", 2023-09-23T20:52:05.563 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_mclock_max_capacity_iops_hdd": "315.000000", 2023-09-23T20:52:05.563 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_mclock_max_capacity_iops_ssd": "21500.000000", 2023-09-23T20:52:05.563 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_mclock_max_sequential_bandwidth_hdd": "157286400", 2023-09-23T20:52:05.563 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_mclock_max_sequential_bandwidth_ssd": "1258291200", 2023-09-23T20:52:05.563 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_mclock_override_recovery_settings": "false", 2023-09-23T20:52:05.563 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_mclock_profile": "balanced", 2023-09-23T20:52:05.563 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_mclock_scheduler_anticipation_timeout": "0.000000", 2023-09-23T20:52:05.563 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_mclock_scheduler_background_best_effort_lim": "0.000000", 2023-09-23T20:52:05.563 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_mclock_scheduler_background_best_effort_res": "0.000000", 2023-09-23T20:52:05.563 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_mclock_scheduler_background_best_effort_wgt": "1", 2023-09-23T20:52:05.563 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_mclock_scheduler_background_recovery_lim": "0.000000", 2023-09-23T20:52:05.564 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_mclock_scheduler_background_recovery_res": "0.000000", 2023-09-23T20:52:05.564 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_mclock_scheduler_background_recovery_wgt": "1", 2023-09-23T20:52:05.564 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_mclock_scheduler_client_lim": "0.000000", 2023-09-23T20:52:05.564 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_mclock_scheduler_client_res": "0.000000", 2023-09-23T20:52:05.564 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_mclock_scheduler_client_wgt": "1", 2023-09-23T20:52:05.564 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_mclock_skip_benchmark": "false", 2023-09-23T20:52:05.564 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_memory_base": "805306368", 2023-09-23T20:52:05.564 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_memory_cache_min": "134217728", 2023-09-23T20:52:05.564 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_memory_cache_resize_interval": "1.000000", 2023-09-23T20:52:05.564 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_memory_expected_fragmentation": "0.150000", 2023-09-23T20:52:05.564 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_memory_target": "4294967296", 2023-09-23T20:52:05.564 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_memory_target_autotune": "false", 2023-09-23T20:52:05.564 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_memory_target_cgroup_limit_ratio": "0.800000", 2023-09-23T20:52:05.565 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_min_pg_log_entries": "250", 2023-09-23T20:52:05.565 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_min_recovery_priority": "0", 2023-09-23T20:52:05.565 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_mon_heartbeat_interval": "30", 2023-09-23T20:52:05.565 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_mon_heartbeat_stat_stale": "3600", 2023-09-23T20:52:05.565 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_mon_report_interval": "5", 2023-09-23T20:52:05.565 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_mon_report_max_in_flight": "2", 2023-09-23T20:52:05.565 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_mon_shutdown_timeout": "5.000000", 2023-09-23T20:52:05.565 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_num_cache_shards": "32", 2023-09-23T20:52:05.565 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_num_op_tracker_shard": "32", 2023-09-23T20:52:05.565 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_numa_auto_affinity": "true", 2023-09-23T20:52:05.565 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_numa_node": "-1", 2023-09-23T20:52:05.565 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_numa_prefer_iface": "true", 2023-09-23T20:52:05.565 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_object_clean_region_max_num_intervals": "10", 2023-09-23T20:52:05.566 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_objecter_finishers": "1", 2023-09-23T20:52:05.566 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_objectstore": "bluestore", 2023-09-23T20:52:05.566 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_objectstore_fuse": "false", 2023-09-23T20:52:05.566 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_objectstore_tracing": "false", 2023-09-23T20:52:05.566 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_op_complaint_time": "30.000000", 2023-09-23T20:52:05.566 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_op_history_duration": "600", 2023-09-23T20:52:05.566 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_op_history_size": "20", 2023-09-23T20:52:05.566 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_op_history_slow_op_size": "20", 2023-09-23T20:52:05.566 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_op_history_slow_op_threshold": "10.000000", 2023-09-23T20:52:05.566 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_op_log_threshold": "5", 2023-09-23T20:52:05.566 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_op_num_shards": "0", 2023-09-23T20:52:05.566 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_op_num_shards_hdd": "5", 2023-09-23T20:52:05.567 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_op_num_shards_ssd": "8", 2023-09-23T20:52:05.567 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_op_num_threads_per_shard": "0", 2023-09-23T20:52:05.567 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_op_num_threads_per_shard_hdd": "1", 2023-09-23T20:52:05.567 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_op_num_threads_per_shard_ssd": "2", 2023-09-23T20:52:05.567 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_op_pq_max_tokens_per_priority": "4194304", 2023-09-23T20:52:05.567 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_op_pq_min_cost": "65536", 2023-09-23T20:52:05.567 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_op_queue": "mclock_scheduler", 2023-09-23T20:52:05.567 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_op_queue_cut_off": "high", 2023-09-23T20:52:05.567 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_op_thread_suicide_timeout": "150", 2023-09-23T20:52:05.567 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_op_thread_timeout": "15", 2023-09-23T20:52:05.567 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_open_classes_on_start": "true", 2023-09-23T20:52:05.567 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_os_flags": "0", 2023-09-23T20:52:05.567 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_peering_op_priority": "255", 2023-09-23T20:52:05.568 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_pg_delete_cost": "1048576", 2023-09-23T20:52:05.568 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_pg_delete_priority": "5", 2023-09-23T20:52:05.568 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_pg_epoch_max_lag_factor": "2.000000", 2023-09-23T20:52:05.568 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_pg_epoch_persisted_max_stale": "40", 2023-09-23T20:52:05.568 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_pg_log_dups_tracked": "3000", 2023-09-23T20:52:05.568 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_pg_log_trim_max": "10000", 2023-09-23T20:52:05.568 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_pg_log_trim_min": "100", 2023-09-23T20:52:05.568 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_pg_max_concurrent_snap_trims": "2", 2023-09-23T20:52:05.568 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_pg_object_context_cache_count": "64", 2023-09-23T20:52:05.568 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_pg_stat_report_interval_max": "500", 2023-09-23T20:52:05.568 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_pool_default_cache_max_evict_check_size": "10", 2023-09-23T20:52:05.568 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_pool_default_cache_min_evict_age": "0", 2023-09-23T20:52:05.569 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_pool_default_cache_min_flush_age": "0", 2023-09-23T20:52:05.569 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_pool_default_cache_target_dirty_high_ratio": "0.600000", 2023-09-23T20:52:05.569 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_pool_default_cache_target_dirty_ratio": "0.400000", 2023-09-23T20:52:05.569 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_pool_default_cache_target_full_ratio": "0.800000", 2023-09-23T20:52:05.569 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_pool_default_crimson": "false", 2023-09-23T20:52:05.569 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_pool_default_crush_rule": "-1", 2023-09-23T20:52:05.569 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_pool_default_ec_fast_read": "false", 2023-09-23T20:52:05.569 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_pool_default_erasure_code_profile": "plugin=jerasure technique=reed_sol_van k=2 m=1 crush-failure-domain=osd", 2023-09-23T20:52:05.569 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_pool_default_flag_bulk": "false", 2023-09-23T20:52:05.569 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_pool_default_flag_hashpspool": "true", 2023-09-23T20:52:05.569 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_pool_default_flag_nodelete": "false", 2023-09-23T20:52:05.569 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_pool_default_flag_nopgchange": "false", 2023-09-23T20:52:05.569 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_pool_default_flag_nosizechange": "false", 2023-09-23T20:52:05.570 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_pool_default_flags": "0", 2023-09-23T20:52:05.570 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_pool_default_hit_set_bloom_fpp": "0.050000", 2023-09-23T20:52:05.570 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_pool_default_min_size": "0", 2023-09-23T20:52:05.570 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_pool_default_pg_autoscale_mode": "off", 2023-09-23T20:52:05.570 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_pool_default_pg_num": "32", 2023-09-23T20:52:05.570 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_pool_default_pgp_num": "0", 2023-09-23T20:52:05.570 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_pool_default_read_lease_ratio": "0.800000", 2023-09-23T20:52:05.570 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_pool_default_size": "3", 2023-09-23T20:52:05.570 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_pool_default_type": "replicated", 2023-09-23T20:52:05.570 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_pool_erasure_code_stripe_unit": "4096", 2023-09-23T20:52:05.570 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_pool_use_gmt_hitset": "true", 2023-09-23T20:52:05.570 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_push_per_object_cost": "1000", 2023-09-23T20:52:05.571 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_read_ec_check_for_errors": "false", 2023-09-23T20:52:05.571 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_recover_clone_overlap": "true", 2023-09-23T20:52:05.571 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_recover_clone_overlap_limit": "10", 2023-09-23T20:52:05.571 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_recovery_cost": "20971520", 2023-09-23T20:52:05.571 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_recovery_delay_start": "0.000000", 2023-09-23T20:52:05.571 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_recovery_max_active": "0", 2023-09-23T20:52:05.571 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_recovery_max_active_hdd": "3", 2023-09-23T20:52:05.571 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_recovery_max_active_ssd": "10", 2023-09-23T20:52:05.571 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_recovery_max_chunk": "8388608", 2023-09-23T20:52:05.571 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_recovery_max_omap_entries_per_chunk": "8096", 2023-09-23T20:52:05.571 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_recovery_max_single_start": "1", 2023-09-23T20:52:05.571 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_recovery_op_priority": "3", 2023-09-23T20:52:05.572 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_recovery_op_warn_multiple": "16", 2023-09-23T20:52:05.572 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_recovery_priority": "5", 2023-09-23T20:52:05.572 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_recovery_retry_interval": "30.000000", 2023-09-23T20:52:05.572 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_recovery_sleep": "0.000000", 2023-09-23T20:52:05.572 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_recovery_sleep_hdd": "0.100000", 2023-09-23T20:52:05.572 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_recovery_sleep_hybrid": "0.025000", 2023-09-23T20:52:05.572 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_recovery_sleep_ssd": "0.000000", 2023-09-23T20:52:05.572 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_repair_during_recovery": "false", 2023-09-23T20:52:05.572 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_requested_scrub_priority": "120", 2023-09-23T20:52:05.573 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_rocksdb_iterator_bounds_enabled": "true", 2023-09-23T20:52:05.573 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_rollback_to_cluster_snap": "", 2023-09-23T20:52:05.573 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_scrub_auto_repair": "false", 2023-09-23T20:52:05.573 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_scrub_auto_repair_num_errors": "5", 2023-09-23T20:52:05.573 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_scrub_backoff_ratio": "0.660000", 2023-09-23T20:52:05.573 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_scrub_begin_hour": "0", 2023-09-23T20:52:05.573 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_scrub_begin_week_day": "0", 2023-09-23T20:52:05.573 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_scrub_chunk_max": "25", 2023-09-23T20:52:05.573 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_scrub_chunk_min": "5", 2023-09-23T20:52:05.573 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_scrub_cost": "52428800", 2023-09-23T20:52:05.573 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_scrub_during_recovery": "false", 2023-09-23T20:52:05.573 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_scrub_end_hour": "0", 2023-09-23T20:52:05.574 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_scrub_end_week_day": "0", 2023-09-23T20:52:05.574 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_scrub_event_cost": "4096", 2023-09-23T20:52:05.574 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_scrub_extended_sleep": "0.000000", 2023-09-23T20:52:05.574 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_scrub_interval_randomize_ratio": "0.500000", 2023-09-23T20:52:05.574 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_scrub_invalid_stats": "true", 2023-09-23T20:52:05.574 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_scrub_load_threshold": "0.500000", 2023-09-23T20:52:05.574 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_scrub_max_interval": "604800.000000", 2023-09-23T20:52:05.574 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_scrub_max_preemptions": "5", 2023-09-23T20:52:05.574 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_scrub_min_interval": "86400.000000", 2023-09-23T20:52:05.574 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_scrub_priority": "5", 2023-09-23T20:52:05.574 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_scrub_reservation_timeout": "5000", 2023-09-23T20:52:05.574 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_scrub_sleep": "0.000000", 2023-09-23T20:52:05.575 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_scrub_slow_reservation_response": "2200", 2023-09-23T20:52:05.575 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_shallow_scrub_chunk_max": "100", 2023-09-23T20:52:05.575 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_shallow_scrub_chunk_min": "50", 2023-09-23T20:52:05.575 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_shutdown_pgref_assert": "false", 2023-09-23T20:52:05.575 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_skip_data_digest": "false", 2023-09-23T20:52:05.575 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_smart_report_timeout": "5", 2023-09-23T20:52:05.575 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_snap_trim_cost": "1048576", 2023-09-23T20:52:05.575 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_snap_trim_priority": "5", 2023-09-23T20:52:05.575 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_snap_trim_sleep": "0.000000", 2023-09-23T20:52:05.575 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_snap_trim_sleep_hdd": "5.000000", 2023-09-23T20:52:05.575 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_snap_trim_sleep_hybrid": "2.000000", 2023-09-23T20:52:05.575 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_snap_trim_sleep_ssd": "0.000000", 2023-09-23T20:52:05.576 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_stats_update_period_not_scrubbing": "120", 2023-09-23T20:52:05.576 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_stats_update_period_scrubbing": "15", 2023-09-23T20:52:05.576 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_target_pg_log_entries_per_osd": "300000", 2023-09-23T20:52:05.576 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_target_transaction_size": "30", 2023-09-23T20:52:05.576 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_tier_default_cache_hit_set_count": "4", 2023-09-23T20:52:05.576 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_tier_default_cache_hit_set_grade_decay_rate": "20", 2023-09-23T20:52:05.576 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_tier_default_cache_hit_set_period": "1200", 2023-09-23T20:52:05.576 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_tier_default_cache_hit_set_search_last_n": "1", 2023-09-23T20:52:05.576 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_tier_default_cache_hit_set_type": "bloom", 2023-09-23T20:52:05.576 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_tier_default_cache_min_read_recency_for_promote": "1", 2023-09-23T20:52:05.576 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_tier_default_cache_min_write_recency_for_promote": "1", 2023-09-23T20:52:05.576 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_tier_default_cache_mode": "writeback", 2023-09-23T20:52:05.577 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_tier_promote_max_bytes_sec": "5242880", 2023-09-23T20:52:05.577 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_tier_promote_max_objects_sec": "25", 2023-09-23T20:52:05.577 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_tracing": "false", 2023-09-23T20:52:05.577 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_use_stale_snap": "false", 2023-09-23T20:52:05.577 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_uuid": "00000000-0000-0000-0000-000000000000", 2023-09-23T20:52:05.577 INFO:teuthology.orchestra.run.smithi026.stdout: "osdc_blkin_trace_all": "false", 2023-09-23T20:52:05.577 INFO:teuthology.orchestra.run.smithi026.stdout: "paxos_kill_at": "0", 2023-09-23T20:52:05.577 INFO:teuthology.orchestra.run.smithi026.stdout: "paxos_max_join_drift": "10", 2023-09-23T20:52:05.577 INFO:teuthology.orchestra.run.smithi026.stdout: "paxos_min": "500", 2023-09-23T20:52:05.577 INFO:teuthology.orchestra.run.smithi026.stdout: "paxos_min_wait": "0.050000", 2023-09-23T20:52:05.577 INFO:teuthology.orchestra.run.smithi026.stdout: "paxos_propose_interval": "1.000000", 2023-09-23T20:52:05.577 INFO:teuthology.orchestra.run.smithi026.stdout: "paxos_service_trim_max": "500", 2023-09-23T20:52:05.578 INFO:teuthology.orchestra.run.smithi026.stdout: "paxos_service_trim_max_multiplier": "20", 2023-09-23T20:52:05.578 INFO:teuthology.orchestra.run.smithi026.stdout: "paxos_service_trim_min": "250", 2023-09-23T20:52:05.578 INFO:teuthology.orchestra.run.smithi026.stdout: "paxos_stash_full_interval": "25", 2023-09-23T20:52:05.578 INFO:teuthology.orchestra.run.smithi026.stdout: "paxos_trim_max": "500", 2023-09-23T20:52:05.578 INFO:teuthology.orchestra.run.smithi026.stdout: "paxos_trim_min": "250", 2023-09-23T20:52:05.578 INFO:teuthology.orchestra.run.smithi026.stdout: "perf": "true", 2023-09-23T20:52:05.578 INFO:teuthology.orchestra.run.smithi026.stdout: "pid_file": "", 2023-09-23T20:52:05.578 INFO:teuthology.orchestra.run.smithi026.stdout: "plugin_crypto_accelerator": "crypto_isal", 2023-09-23T20:52:05.578 INFO:teuthology.orchestra.run.smithi026.stdout: "plugin_dir": "/usr/lib64/ceph", 2023-09-23T20:52:05.578 INFO:teuthology.orchestra.run.smithi026.stdout: "public_addr": "-", 2023-09-23T20:52:05.578 INFO:teuthology.orchestra.run.smithi026.stdout: "public_addrv": "", 2023-09-23T20:52:05.578 INFO:teuthology.orchestra.run.smithi026.stdout: "public_bind_addr": "-", 2023-09-23T20:52:05.579 INFO:teuthology.orchestra.run.smithi026.stdout: "public_network": "172.21.0.0/20", 2023-09-23T20:52:05.579 INFO:teuthology.orchestra.run.smithi026.stdout: "public_network_interface": "", 2023-09-23T20:52:05.579 INFO:teuthology.orchestra.run.smithi026.stdout: "qat_compressor_enabled": "false", 2023-09-23T20:52:05.579 INFO:teuthology.orchestra.run.smithi026.stdout: "qat_compressor_session_max_number": "256", 2023-09-23T20:52:05.579 INFO:teuthology.orchestra.run.smithi026.stdout: "rados_mon_op_timeout": "0", 2023-09-23T20:52:05.579 INFO:teuthology.orchestra.run.smithi026.stdout: "rados_osd_op_timeout": "0", 2023-09-23T20:52:05.579 INFO:teuthology.orchestra.run.smithi026.stdout: "rados_tracing": "false", 2023-09-23T20:52:05.579 INFO:teuthology.orchestra.run.smithi026.stdout: "rbd_atime_update_interval": "60", 2023-09-23T20:52:05.579 INFO:teuthology.orchestra.run.smithi026.stdout: "rbd_auto_exclusive_lock_until_manual_request": "true", 2023-09-23T20:52:05.579 INFO:teuthology.orchestra.run.smithi026.stdout: "rbd_balance_parent_reads": "false", 2023-09-23T20:52:05.579 INFO:teuthology.orchestra.run.smithi026.stdout: "rbd_balance_snap_reads": "false", 2023-09-23T20:52:05.579 INFO:teuthology.orchestra.run.smithi026.stdout: "rbd_blkin_trace_all": "false", 2023-09-23T20:52:05.580 INFO:teuthology.orchestra.run.smithi026.stdout: "rbd_blocklist_expire_seconds": "0", 2023-09-23T20:52:05.580 INFO:teuthology.orchestra.run.smithi026.stdout: "rbd_blocklist_on_break_lock": "true", 2023-09-23T20:52:05.580 INFO:teuthology.orchestra.run.smithi026.stdout: "rbd_cache": "true", 2023-09-23T20:52:05.580 INFO:teuthology.orchestra.run.smithi026.stdout: "rbd_cache_block_writes_upfront": "false", 2023-09-23T20:52:05.580 INFO:teuthology.orchestra.run.smithi026.stdout: "rbd_cache_max_dirty": "25165824", 2023-09-23T20:52:05.580 INFO:teuthology.orchestra.run.smithi026.stdout: "rbd_cache_max_dirty_age": "1.000000", 2023-09-23T20:52:05.580 INFO:teuthology.orchestra.run.smithi026.stdout: "rbd_cache_max_dirty_object": "0", 2023-09-23T20:52:05.580 INFO:teuthology.orchestra.run.smithi026.stdout: "rbd_cache_policy": "writearound", 2023-09-23T20:52:05.580 INFO:teuthology.orchestra.run.smithi026.stdout: "rbd_cache_size": "33554432", 2023-09-23T20:52:05.580 INFO:teuthology.orchestra.run.smithi026.stdout: "rbd_cache_target_dirty": "16777216", 2023-09-23T20:52:05.580 INFO:teuthology.orchestra.run.smithi026.stdout: "rbd_cache_writethrough_until_flush": "true", 2023-09-23T20:52:05.580 INFO:teuthology.orchestra.run.smithi026.stdout: "rbd_clone_copy_on_read": "false", 2023-09-23T20:52:05.581 INFO:teuthology.orchestra.run.smithi026.stdout: "rbd_compression_hint": "none", 2023-09-23T20:52:05.581 INFO:teuthology.orchestra.run.smithi026.stdout: "rbd_concurrent_management_ops": "10", 2023-09-23T20:52:05.581 INFO:teuthology.orchestra.run.smithi026.stdout: "rbd_config_pool_override_update_timestamp": "0", 2023-09-23T20:52:05.581 INFO:teuthology.orchestra.run.smithi026.stdout: "rbd_default_clone_format": "auto", 2023-09-23T20:52:05.581 INFO:teuthology.orchestra.run.smithi026.stdout: "rbd_default_data_pool": "", 2023-09-23T20:52:05.581 INFO:teuthology.orchestra.run.smithi026.stdout: "rbd_default_features": "61", 2023-09-23T20:52:05.581 INFO:teuthology.orchestra.run.smithi026.stdout: "rbd_default_format": "2", 2023-09-23T20:52:05.581 INFO:teuthology.orchestra.run.smithi026.stdout: "rbd_default_map_options": "", 2023-09-23T20:52:05.581 INFO:teuthology.orchestra.run.smithi026.stdout: "rbd_default_order": "22", 2023-09-23T20:52:05.581 INFO:teuthology.orchestra.run.smithi026.stdout: "rbd_default_pool": "rbd", 2023-09-23T20:52:05.581 INFO:teuthology.orchestra.run.smithi026.stdout: "rbd_default_snapshot_quiesce_mode": "required", 2023-09-23T20:52:05.581 INFO:teuthology.orchestra.run.smithi026.stdout: "rbd_default_stripe_count": "0", 2023-09-23T20:52:05.582 INFO:teuthology.orchestra.run.smithi026.stdout: "rbd_default_stripe_unit": "0", 2023-09-23T20:52:05.582 INFO:teuthology.orchestra.run.smithi026.stdout: "rbd_disable_zero_copy_writes": "true", 2023-09-23T20:52:05.582 INFO:teuthology.orchestra.run.smithi026.stdout: "rbd_discard_granularity_bytes": "65536", 2023-09-23T20:52:05.582 INFO:teuthology.orchestra.run.smithi026.stdout: "rbd_discard_on_zeroed_write_same": "true", 2023-09-23T20:52:05.582 INFO:teuthology.orchestra.run.smithi026.stdout: "rbd_enable_alloc_hint": "true", 2023-09-23T20:52:05.582 INFO:teuthology.orchestra.run.smithi026.stdout: "rbd_invalidate_object_map_on_timeout": "true", 2023-09-23T20:52:05.582 INFO:teuthology.orchestra.run.smithi026.stdout: "rbd_io_scheduler": "simple", 2023-09-23T20:52:05.582 INFO:teuthology.orchestra.run.smithi026.stdout: "rbd_io_scheduler_simple_max_delay": "0", 2023-09-23T20:52:05.582 INFO:teuthology.orchestra.run.smithi026.stdout: "rbd_journal_commit_age": "5.000000", 2023-09-23T20:52:05.582 INFO:teuthology.orchestra.run.smithi026.stdout: "rbd_journal_max_concurrent_object_sets": "0", 2023-09-23T20:52:05.582 INFO:teuthology.orchestra.run.smithi026.stdout: "rbd_journal_max_payload_bytes": "16384", 2023-09-23T20:52:05.582 INFO:teuthology.orchestra.run.smithi026.stdout: "rbd_journal_object_flush_age": "0.000000", 2023-09-23T20:52:05.583 INFO:teuthology.orchestra.run.smithi026.stdout: "rbd_journal_object_flush_bytes": "1048576", 2023-09-23T20:52:05.583 INFO:teuthology.orchestra.run.smithi026.stdout: "rbd_journal_object_flush_interval": "0", 2023-09-23T20:52:05.583 INFO:teuthology.orchestra.run.smithi026.stdout: "rbd_journal_object_max_in_flight_appends": "0", 2023-09-23T20:52:05.583 INFO:teuthology.orchestra.run.smithi026.stdout: "rbd_journal_object_writethrough_until_flush": "true", 2023-09-23T20:52:05.583 INFO:teuthology.orchestra.run.smithi026.stdout: "rbd_journal_order": "24", 2023-09-23T20:52:05.583 INFO:teuthology.orchestra.run.smithi026.stdout: "rbd_journal_pool": "", 2023-09-23T20:52:05.583 INFO:teuthology.orchestra.run.smithi026.stdout: "rbd_journal_splay_width": "4", 2023-09-23T20:52:05.583 INFO:teuthology.orchestra.run.smithi026.stdout: "rbd_localize_parent_reads": "false", 2023-09-23T20:52:05.583 INFO:teuthology.orchestra.run.smithi026.stdout: "rbd_localize_snap_reads": "false", 2023-09-23T20:52:05.583 INFO:teuthology.orchestra.run.smithi026.stdout: "rbd_mirror_concurrent_image_deletions": "1", 2023-09-23T20:52:05.583 INFO:teuthology.orchestra.run.smithi026.stdout: "rbd_mirror_concurrent_image_syncs": "5", 2023-09-23T20:52:05.583 INFO:teuthology.orchestra.run.smithi026.stdout: "rbd_mirror_delete_retry_interval": "30.000000", 2023-09-23T20:52:05.584 INFO:teuthology.orchestra.run.smithi026.stdout: "rbd_mirror_image_perf_stats_prio": "5", 2023-09-23T20:52:05.584 INFO:teuthology.orchestra.run.smithi026.stdout: "rbd_mirror_image_policy_migration_throttle": "300", 2023-09-23T20:52:05.584 INFO:teuthology.orchestra.run.smithi026.stdout: "rbd_mirror_image_policy_rebalance_timeout": "0.000000", 2023-09-23T20:52:05.584 INFO:teuthology.orchestra.run.smithi026.stdout: "rbd_mirror_image_policy_type": "simple", 2023-09-23T20:52:05.584 INFO:teuthology.orchestra.run.smithi026.stdout: "rbd_mirror_image_policy_update_throttle_interval": "1.000000", 2023-09-23T20:52:05.584 INFO:teuthology.orchestra.run.smithi026.stdout: "rbd_mirror_image_state_check_interval": "30", 2023-09-23T20:52:05.584 INFO:teuthology.orchestra.run.smithi026.stdout: "rbd_mirror_journal_commit_age": "5.000000", 2023-09-23T20:52:05.584 INFO:teuthology.orchestra.run.smithi026.stdout: "rbd_mirror_journal_poll_age": "5.000000", 2023-09-23T20:52:05.584 INFO:teuthology.orchestra.run.smithi026.stdout: "rbd_mirror_leader_heartbeat_interval": "5", 2023-09-23T20:52:05.584 INFO:teuthology.orchestra.run.smithi026.stdout: "rbd_mirror_leader_max_acquire_attempts_before_break": "3", 2023-09-23T20:52:05.584 INFO:teuthology.orchestra.run.smithi026.stdout: "rbd_mirror_leader_max_missed_heartbeats": "2", 2023-09-23T20:52:05.584 INFO:teuthology.orchestra.run.smithi026.stdout: "rbd_mirror_memory_autotune": "true", 2023-09-23T20:52:05.585 INFO:teuthology.orchestra.run.smithi026.stdout: "rbd_mirror_memory_base": "805306368", 2023-09-23T20:52:05.585 INFO:teuthology.orchestra.run.smithi026.stdout: "rbd_mirror_memory_cache_autotune_interval": "30.000000", 2023-09-23T20:52:05.585 INFO:teuthology.orchestra.run.smithi026.stdout: "rbd_mirror_memory_cache_min": "134217728", 2023-09-23T20:52:05.585 INFO:teuthology.orchestra.run.smithi026.stdout: "rbd_mirror_memory_cache_resize_interval": "5.000000", 2023-09-23T20:52:05.585 INFO:teuthology.orchestra.run.smithi026.stdout: "rbd_mirror_memory_expected_fragmentation": "0.150000", 2023-09-23T20:52:05.585 INFO:teuthology.orchestra.run.smithi026.stdout: "rbd_mirror_memory_target": "4294967296", 2023-09-23T20:52:05.585 INFO:teuthology.orchestra.run.smithi026.stdout: "rbd_mirror_perf_stats_prio": "5", 2023-09-23T20:52:05.585 INFO:teuthology.orchestra.run.smithi026.stdout: "rbd_mirror_pool_replayers_refresh_interval": "30", 2023-09-23T20:52:05.585 INFO:teuthology.orchestra.run.smithi026.stdout: "rbd_mirror_sync_point_update_age": "30.000000", 2023-09-23T20:52:05.585 INFO:teuthology.orchestra.run.smithi026.stdout: "rbd_mirroring_delete_delay": "0", 2023-09-23T20:52:05.585 INFO:teuthology.orchestra.run.smithi026.stdout: "rbd_mirroring_max_mirroring_snapshots": "5", 2023-09-23T20:52:05.585 INFO:teuthology.orchestra.run.smithi026.stdout: "rbd_mirroring_replay_delay": "0", 2023-09-23T20:52:05.585 INFO:teuthology.orchestra.run.smithi026.stdout: "rbd_mirroring_resync_after_disconnect": "false", 2023-09-23T20:52:05.586 INFO:teuthology.orchestra.run.smithi026.stdout: "rbd_move_parent_to_trash_on_remove": "false", 2023-09-23T20:52:05.586 INFO:teuthology.orchestra.run.smithi026.stdout: "rbd_move_to_trash_on_remove": "false", 2023-09-23T20:52:05.586 INFO:teuthology.orchestra.run.smithi026.stdout: "rbd_move_to_trash_on_remove_expire_seconds": "0", 2023-09-23T20:52:05.586 INFO:teuthology.orchestra.run.smithi026.stdout: "rbd_mtime_update_interval": "60", 2023-09-23T20:52:05.586 INFO:teuthology.orchestra.run.smithi026.stdout: "rbd_non_blocking_aio": "true", 2023-09-23T20:52:05.586 INFO:teuthology.orchestra.run.smithi026.stdout: "rbd_op_thread_timeout": "60", 2023-09-23T20:52:05.586 INFO:teuthology.orchestra.run.smithi026.stdout: "rbd_op_threads": "1", 2023-09-23T20:52:05.586 INFO:teuthology.orchestra.run.smithi026.stdout: "rbd_parent_cache_enabled": "false", 2023-09-23T20:52:05.586 INFO:teuthology.orchestra.run.smithi026.stdout: "rbd_persistent_cache_mode": "disabled", 2023-09-23T20:52:05.586 INFO:teuthology.orchestra.run.smithi026.stdout: "rbd_persistent_cache_path": "/tmp", 2023-09-23T20:52:05.586 INFO:teuthology.orchestra.run.smithi026.stdout: "rbd_persistent_cache_size": "1073741824", 2023-09-23T20:52:05.586 INFO:teuthology.orchestra.run.smithi026.stdout: "rbd_plugins": "", 2023-09-23T20:52:05.587 INFO:teuthology.orchestra.run.smithi026.stdout: "rbd_qos_bps_burst": "0", 2023-09-23T20:52:05.587 INFO:teuthology.orchestra.run.smithi026.stdout: "rbd_qos_bps_burst_seconds": "1", 2023-09-23T20:52:05.587 INFO:teuthology.orchestra.run.smithi026.stdout: "rbd_qos_bps_limit": "0", 2023-09-23T20:52:05.587 INFO:teuthology.orchestra.run.smithi026.stdout: "rbd_qos_exclude_ops": "0", 2023-09-23T20:52:05.587 INFO:teuthology.orchestra.run.smithi026.stdout: "rbd_qos_iops_burst": "0", 2023-09-23T20:52:05.587 INFO:teuthology.orchestra.run.smithi026.stdout: "rbd_qos_iops_burst_seconds": "1", 2023-09-23T20:52:05.587 INFO:teuthology.orchestra.run.smithi026.stdout: "rbd_qos_iops_limit": "0", 2023-09-23T20:52:05.587 INFO:teuthology.orchestra.run.smithi026.stdout: "rbd_qos_read_bps_burst": "0", 2023-09-23T20:52:05.587 INFO:teuthology.orchestra.run.smithi026.stdout: "rbd_qos_read_bps_burst_seconds": "1", 2023-09-23T20:52:05.587 INFO:teuthology.orchestra.run.smithi026.stdout: "rbd_qos_read_bps_limit": "0", 2023-09-23T20:52:05.587 INFO:teuthology.orchestra.run.smithi026.stdout: "rbd_qos_read_iops_burst": "0", 2023-09-23T20:52:05.587 INFO:teuthology.orchestra.run.smithi026.stdout: "rbd_qos_read_iops_burst_seconds": "1", 2023-09-23T20:52:05.588 INFO:teuthology.orchestra.run.smithi026.stdout: "rbd_qos_read_iops_limit": "0", 2023-09-23T20:52:05.588 INFO:teuthology.orchestra.run.smithi026.stdout: "rbd_qos_schedule_tick_min": "50", 2023-09-23T20:52:05.588 INFO:teuthology.orchestra.run.smithi026.stdout: "rbd_qos_write_bps_burst": "0", 2023-09-23T20:52:05.588 INFO:teuthology.orchestra.run.smithi026.stdout: "rbd_qos_write_bps_burst_seconds": "1", 2023-09-23T20:52:05.588 INFO:teuthology.orchestra.run.smithi026.stdout: "rbd_qos_write_bps_limit": "0", 2023-09-23T20:52:05.588 INFO:teuthology.orchestra.run.smithi026.stdout: "rbd_qos_write_iops_burst": "0", 2023-09-23T20:52:05.588 INFO:teuthology.orchestra.run.smithi026.stdout: "rbd_qos_write_iops_burst_seconds": "1", 2023-09-23T20:52:05.588 INFO:teuthology.orchestra.run.smithi026.stdout: "rbd_qos_write_iops_limit": "0", 2023-09-23T20:52:05.588 INFO:teuthology.orchestra.run.smithi026.stdout: "rbd_quiesce_notification_attempts": "10", 2023-09-23T20:52:05.588 INFO:teuthology.orchestra.run.smithi026.stdout: "rbd_read_from_replica_policy": "default", 2023-09-23T20:52:05.588 INFO:teuthology.orchestra.run.smithi026.stdout: "rbd_readahead_disable_after_bytes": "52428800", 2023-09-23T20:52:05.588 INFO:teuthology.orchestra.run.smithi026.stdout: "rbd_readahead_max_bytes": "524288", 2023-09-23T20:52:05.589 INFO:teuthology.orchestra.run.smithi026.stdout: "rbd_readahead_trigger_requests": "10", 2023-09-23T20:52:05.589 INFO:teuthology.orchestra.run.smithi026.stdout: "rbd_request_timed_out_seconds": "30", 2023-09-23T20:52:05.589 INFO:teuthology.orchestra.run.smithi026.stdout: "rbd_skip_partial_discard": "true", 2023-09-23T20:52:05.589 INFO:teuthology.orchestra.run.smithi026.stdout: "rbd_sparse_read_threshold_bytes": "65536", 2023-09-23T20:52:05.589 INFO:teuthology.orchestra.run.smithi026.stdout: "rbd_tracing": "false", 2023-09-23T20:52:05.589 INFO:teuthology.orchestra.run.smithi026.stdout: "rbd_validate_names": "true", 2023-09-23T20:52:05.589 INFO:teuthology.orchestra.run.smithi026.stdout: "rbd_validate_pool": "true", 2023-09-23T20:52:05.589 INFO:teuthology.orchestra.run.smithi026.stdout: "restapi_base_url": "", 2023-09-23T20:52:05.589 INFO:teuthology.orchestra.run.smithi026.stdout: "restapi_log_level": "", 2023-09-23T20:52:05.589 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_acl_grants_max_num": "100", 2023-09-23T20:52:05.589 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_admin_entry": "admin", 2023-09-23T20:52:05.589 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_allow_notification_secrets_in_cleartext": "false", 2023-09-23T20:52:05.590 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_backend_store": "rados", 2023-09-23T20:52:05.590 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_barbican_url": "", 2023-09-23T20:52:05.590 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_beast_enable_async": "true", 2023-09-23T20:52:05.590 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_bucket_default_quota_max_objects": "-1", 2023-09-23T20:52:05.590 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_bucket_default_quota_max_size": "-1", 2023-09-23T20:52:05.590 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_bucket_index_max_aio": "128", 2023-09-23T20:52:05.590 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_bucket_index_transaction_instrumentation": "false", 2023-09-23T20:52:05.590 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_bucket_quota_cache_size": "10000", 2023-09-23T20:52:05.590 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_bucket_quota_ttl": "600", 2023-09-23T20:52:05.590 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_bucket_sync_spawn_window": "20", 2023-09-23T20:52:05.590 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_cache_enabled": "true", 2023-09-23T20:52:05.591 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_cache_expiry_interval": "900", 2023-09-23T20:52:05.591 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_cache_lru_size": "10000", 2023-09-23T20:52:05.591 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_config_store": "rados", 2023-09-23T20:52:05.591 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_content_length_compat": "false", 2023-09-23T20:52:05.591 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_copy_obj_progress": "true", 2023-09-23T20:52:05.591 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_copy_obj_progress_every_bytes": "1048576", 2023-09-23T20:52:05.591 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_cors_rules_max_num": "100", 2023-09-23T20:52:05.591 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_cross_domain_policy": "", 2023-09-23T20:52:05.591 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_crypt_default_encryption_key": "", 2023-09-23T20:52:05.591 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_crypt_kmip_addr": "", 2023-09-23T20:52:05.591 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_crypt_kmip_ca_path": "", 2023-09-23T20:52:05.591 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_crypt_kmip_client_cert": "", 2023-09-23T20:52:05.592 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_crypt_kmip_client_key": "", 2023-09-23T20:52:05.592 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_crypt_kmip_kms_key_template": "", 2023-09-23T20:52:05.592 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_crypt_kmip_password": "", 2023-09-23T20:52:05.592 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_crypt_kmip_s3_key_template": "$keyid", 2023-09-23T20:52:05.592 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_crypt_kmip_username": "", 2023-09-23T20:52:05.592 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_crypt_require_ssl": "true", 2023-09-23T20:52:05.592 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_crypt_s3_kms_backend": "barbican", 2023-09-23T20:52:05.592 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_crypt_s3_kms_encryption_keys": "", 2023-09-23T20:52:05.592 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_crypt_sse_s3_backend": "vault", 2023-09-23T20:52:05.592 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_crypt_sse_s3_key_template": "%bucket_id", 2023-09-23T20:52:05.592 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_crypt_sse_s3_vault_addr": "", 2023-09-23T20:52:05.592 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_crypt_sse_s3_vault_auth": "token", 2023-09-23T20:52:05.593 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_crypt_sse_s3_vault_namespace": "", 2023-09-23T20:52:05.593 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_crypt_sse_s3_vault_prefix": "", 2023-09-23T20:52:05.593 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_crypt_sse_s3_vault_secret_engine": "transit", 2023-09-23T20:52:05.593 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_crypt_sse_s3_vault_ssl_cacert": "", 2023-09-23T20:52:05.593 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_crypt_sse_s3_vault_ssl_clientcert": "", 2023-09-23T20:52:05.593 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_crypt_sse_s3_vault_ssl_clientkey": "", 2023-09-23T20:52:05.593 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_crypt_sse_s3_vault_token_file": "", 2023-09-23T20:52:05.593 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_crypt_sse_s3_vault_verify_ssl": "true", 2023-09-23T20:52:05.593 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_crypt_suppress_logs": "true", 2023-09-23T20:52:05.593 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_crypt_vault_addr": "", 2023-09-23T20:52:05.593 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_crypt_vault_auth": "token", 2023-09-23T20:52:05.593 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_crypt_vault_namespace": "", 2023-09-23T20:52:05.593 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_crypt_vault_prefix": "", 2023-09-23T20:52:05.594 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_crypt_vault_secret_engine": "transit", 2023-09-23T20:52:05.594 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_crypt_vault_ssl_cacert": "", 2023-09-23T20:52:05.594 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_crypt_vault_ssl_clientcert": "", 2023-09-23T20:52:05.594 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_crypt_vault_ssl_clientkey": "", 2023-09-23T20:52:05.594 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_crypt_vault_token_file": "", 2023-09-23T20:52:05.594 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_crypt_vault_verify_ssl": "true", 2023-09-23T20:52:05.594 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_curl_buffersize": "524288", 2023-09-23T20:52:05.594 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_curl_low_speed_limit": "1024", 2023-09-23T20:52:05.594 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_curl_low_speed_time": "300", 2023-09-23T20:52:05.594 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_curl_tcp_keepalive": "0", 2023-09-23T20:52:05.594 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_curl_wait_timeout_ms": "1000", 2023-09-23T20:52:05.594 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_d3n_l1_datacache_persistent_path": "/tmp/rgw_datacache/", 2023-09-23T20:52:05.595 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_d3n_l1_datacache_size": "1073741824", 2023-09-23T20:52:05.595 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_d3n_l1_evict_cache_on_start": "true", 2023-09-23T20:52:05.595 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_d3n_l1_eviction_policy": "lru", 2023-09-23T20:52:05.595 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_d3n_l1_fadvise": "4", 2023-09-23T20:52:05.595 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_d3n_l1_local_datacache_enabled": "false", 2023-09-23T20:52:05.595 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_d3n_libaio_aio_num": "64", 2023-09-23T20:52:05.595 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_d3n_libaio_aio_threads": "20", 2023-09-23T20:52:05.595 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_data": "/var/lib/ceph/radosgw/ceph-a", 2023-09-23T20:52:05.595 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_data_log_changes_size": "1000", 2023-09-23T20:52:05.595 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_data_log_num_shards": "128", 2023-09-23T20:52:05.597 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_data_log_obj_prefix": "data_log", 2023-09-23T20:52:05.597 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_data_log_window": "30", 2023-09-23T20:52:05.597 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_data_notify_interval_msec": "0", 2023-09-23T20:52:05.597 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_data_sync_poll_interval": "20", 2023-09-23T20:52:05.597 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_data_sync_spawn_window": "20", 2023-09-23T20:52:05.597 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_debug_inject_olh_cancel_modification_err": "false", 2023-09-23T20:52:05.597 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_debug_inject_set_olh_err": "0", 2023-09-23T20:52:05.597 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_default_data_log_backing": "fifo", 2023-09-23T20:52:05.597 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_default_realm_info_oid": "default.realm", 2023-09-23T20:52:05.597 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_default_region_info_oid": "default.region", 2023-09-23T20:52:05.597 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_default_zone_info_oid": "default.zone", 2023-09-23T20:52:05.598 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_default_zonegroup_info_oid": "default.zonegroup", 2023-09-23T20:52:05.598 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_defer_to_bucket_acls": "", 2023-09-23T20:52:05.598 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_delete_multi_obj_max_num": "1000", 2023-09-23T20:52:05.598 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_dmclock_admin_lim": "0.000000", 2023-09-23T20:52:05.598 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_dmclock_admin_res": "100.000000", 2023-09-23T20:52:05.598 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_dmclock_admin_wgt": "100.000000", 2023-09-23T20:52:05.598 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_dmclock_auth_lim": "0.000000", 2023-09-23T20:52:05.598 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_dmclock_auth_res": "200.000000", 2023-09-23T20:52:05.598 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_dmclock_auth_wgt": "100.000000", 2023-09-23T20:52:05.598 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_dmclock_data_lim": "0.000000", 2023-09-23T20:52:05.598 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_dmclock_data_res": "500.000000", 2023-09-23T20:52:05.598 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_dmclock_data_wgt": "500.000000", 2023-09-23T20:52:05.599 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_dmclock_metadata_lim": "0.000000", 2023-09-23T20:52:05.599 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_dmclock_metadata_res": "500.000000", 2023-09-23T20:52:05.599 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_dmclock_metadata_wgt": "500.000000", 2023-09-23T20:52:05.599 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_dns_name": "", 2023-09-23T20:52:05.599 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_dns_s3website_name": "", 2023-09-23T20:52:05.599 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_dynamic_resharding": "true", 2023-09-23T20:52:05.599 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_enable_apis": "s3, s3website, swift, swift_auth, admin, sts, iam, notifications", 2023-09-23T20:52:05.599 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_enable_gc_threads": "true", 2023-09-23T20:52:05.599 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_enable_lc_threads": "true", 2023-09-23T20:52:05.599 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_enable_ops_log": "false", 2023-09-23T20:52:05.599 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_enable_quota_threads": "true", 2023-09-23T20:52:05.599 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_enable_static_website": "false", 2023-09-23T20:52:05.600 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_enable_usage_log": "false", 2023-09-23T20:52:05.600 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_enforce_swift_acls": "true", 2023-09-23T20:52:05.600 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_exit_timeout_secs": "120", 2023-09-23T20:52:05.600 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_expose_bucket": "false", 2023-09-23T20:52:05.600 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_extended_http_attrs": "", 2023-09-23T20:52:05.600 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_filter": "none", 2023-09-23T20:52:05.600 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_frontend_defaults": "beast ssl_certificate=config://rgw/cert/$realm/$zone.crt ssl_private_key=config://rgw/cert/$realm/$zone.key", 2023-09-23T20:52:05.600 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_frontends": "beast port=7480", 2023-09-23T20:52:05.600 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_gc_max_concurrent_io": "10", 2023-09-23T20:52:05.600 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_gc_max_deferred": "50", 2023-09-23T20:52:05.600 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_gc_max_deferred_entries_size": "3072", 2023-09-23T20:52:05.600 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_gc_max_objs": "32", 2023-09-23T20:52:05.601 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_gc_max_queue_size": "134213632", 2023-09-23T20:52:05.601 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_gc_max_trim_chunk": "16", 2023-09-23T20:52:05.601 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_gc_obj_min_wait": "7200", 2023-09-23T20:52:05.601 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_gc_processor_max_time": "3600", 2023-09-23T20:52:05.601 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_gc_processor_period": "3600", 2023-09-23T20:52:05.601 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_get_obj_max_req_size": "4194304", 2023-09-23T20:52:05.601 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_get_obj_window_size": "16777216", 2023-09-23T20:52:05.601 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_healthcheck_disabling_path": "", 2023-09-23T20:52:05.601 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_ignore_get_invalid_range": "false", 2023-09-23T20:52:05.601 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_init_timeout": "300", 2023-09-23T20:52:05.601 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_inject_notify_timeout_probability": "0.000000", 2023-09-23T20:52:05.601 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_json_config": "/var/lib/ceph/radosgw/config.json", 2023-09-23T20:52:05.602 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_keystone_accepted_admin_roles": "", 2023-09-23T20:52:05.602 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_keystone_accepted_roles": "Member, admin", 2023-09-23T20:52:05.602 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_keystone_admin_domain": "", 2023-09-23T20:52:05.602 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_keystone_admin_password": "", 2023-09-23T20:52:05.602 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_keystone_admin_password_path": "", 2023-09-23T20:52:05.602 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_keystone_admin_project": "", 2023-09-23T20:52:05.602 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_keystone_admin_tenant": "", 2023-09-23T20:52:05.602 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_keystone_admin_token": "", 2023-09-23T20:52:05.602 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_keystone_admin_token_path": "", 2023-09-23T20:52:05.602 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_keystone_admin_user": "", 2023-09-23T20:52:05.602 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_keystone_api_version": "2", 2023-09-23T20:52:05.602 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_keystone_barbican_domain": "", 2023-09-23T20:52:05.603 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_keystone_barbican_password": "", 2023-09-23T20:52:05.603 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_keystone_barbican_project": "", 2023-09-23T20:52:05.603 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_keystone_barbican_tenant": "", 2023-09-23T20:52:05.603 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_keystone_barbican_user": "", 2023-09-23T20:52:05.603 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_keystone_expired_token_cache_expiration": "3600", 2023-09-23T20:52:05.603 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_keystone_implicit_tenants": "false", 2023-09-23T20:52:05.603 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_keystone_service_token_accepted_roles": "admin", 2023-09-23T20:52:05.603 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_keystone_service_token_enabled": "false", 2023-09-23T20:52:05.603 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_keystone_token_cache_size": "10000", 2023-09-23T20:52:05.603 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_keystone_url": "", 2023-09-23T20:52:05.603 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_keystone_verify_ssl": "true", 2023-09-23T20:52:05.603 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_lc_debug_interval": "-1", 2023-09-23T20:52:05.604 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_lc_lock_max_time": "90", 2023-09-23T20:52:05.604 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_lc_max_objs": "32", 2023-09-23T20:52:05.604 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_lc_max_rules": "1000", 2023-09-23T20:52:05.604 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_lc_max_worker": "3", 2023-09-23T20:52:05.604 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_lc_max_wp_worker": "3", 2023-09-23T20:52:05.604 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_lc_thread_delay": "0", 2023-09-23T20:52:05.604 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_ldap_binddn": "uid=admin,cn=users,dc=example,dc=com", 2023-09-23T20:52:05.604 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_ldap_dnattr": "uid", 2023-09-23T20:52:05.604 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_ldap_searchdn": "cn=users,cn=accounts,dc=example,dc=com", 2023-09-23T20:52:05.604 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_ldap_searchfilter": "", 2023-09-23T20:52:05.604 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_ldap_secret": "/etc/openldap/secret", 2023-09-23T20:52:05.604 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_ldap_uri": "ldaps://", 2023-09-23T20:52:05.604 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_lifecycle_work_time": "00:00-06:00", 2023-09-23T20:52:05.605 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_list_bucket_min_readahead": "1000", 2023-09-23T20:52:05.605 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_list_buckets_max_chunk": "1000", 2023-09-23T20:52:05.605 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_log_http_headers": "", 2023-09-23T20:52:05.605 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_log_nonexistent_bucket": "false", 2023-09-23T20:52:05.605 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_log_object_name": "%Y-%m-%d-%H-%i-%n", 2023-09-23T20:52:05.605 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_log_object_name_utc": "false", 2023-09-23T20:52:05.605 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_luarocks_location": "/tmp/luarocks", 2023-09-23T20:52:05.605 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_max_attr_name_len": "0", 2023-09-23T20:52:05.605 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_max_attr_size": "0", 2023-09-23T20:52:05.605 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_max_attrs_num_in_req": "0", 2023-09-23T20:52:05.605 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_max_chunk_size": "4194304", 2023-09-23T20:52:05.605 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_max_concurrent_requests": "1024", 2023-09-23T20:52:05.606 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_max_copy_obj_concurrent_io": "10", 2023-09-23T20:52:05.606 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_max_dynamic_shards": "1999", 2023-09-23T20:52:05.606 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_max_listing_results": "1000", 2023-09-23T20:52:05.606 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_max_notify_retries": "10", 2023-09-23T20:52:05.606 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_max_objs_per_shard": "100000", 2023-09-23T20:52:05.606 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_max_put_param_size": "1048576", 2023-09-23T20:52:05.606 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_max_put_size": "5368709120", 2023-09-23T20:52:05.606 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_max_slo_entries": "1000", 2023-09-23T20:52:05.606 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_md_log_max_shards": "64", 2023-09-23T20:52:05.606 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_md_notify_interval_msec": "200", 2023-09-23T20:52:05.606 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_meta_sync_poll_interval": "20", 2023-09-23T20:52:05.606 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_meta_sync_spawn_window": "20", 2023-09-23T20:52:05.607 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_mime_types_file": "/etc/mime.types", 2023-09-23T20:52:05.607 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_mp_lock_max_time": "600", 2023-09-23T20:52:05.607 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_multi_obj_del_max_aio": "16", 2023-09-23T20:52:05.607 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_multipart_min_part_size": "5242880", 2023-09-23T20:52:05.607 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_multipart_part_upload_limit": "10000", 2023-09-23T20:52:05.607 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_nfs_fhcache_partitions": "3", 2023-09-23T20:52:05.607 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_nfs_fhcache_size": "2017", 2023-09-23T20:52:05.607 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_nfs_frontends": "rgw-nfs", 2023-09-23T20:52:05.607 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_nfs_lru_lane_hiwat": "911", 2023-09-23T20:52:05.607 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_nfs_lru_lanes": "5", 2023-09-23T20:52:05.607 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_nfs_max_gc": "300", 2023-09-23T20:52:05.607 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_nfs_namespace_expire_secs": "300", 2023-09-23T20:52:05.607 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_nfs_run_gc_threads": "false", 2023-09-23T20:52:05.608 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_nfs_run_lc_threads": "false", 2023-09-23T20:52:05.608 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_nfs_run_quota_threads": "false", 2023-09-23T20:52:05.608 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_nfs_run_sync_thread": "false", 2023-09-23T20:52:05.608 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_nfs_s3_fast_attrs": "false", 2023-09-23T20:52:05.608 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_nfs_write_completion_interval_s": "10", 2023-09-23T20:52:05.608 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_num_async_rados_threads": "32", 2023-09-23T20:52:05.608 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_num_control_oids": "8", 2023-09-23T20:52:05.608 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_numa_node": "-1", 2023-09-23T20:52:05.608 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_obj_stripe_size": "4194304", 2023-09-23T20:52:05.608 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_obj_tombstone_cache_size": "1000", 2023-09-23T20:52:05.608 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_objexp_chunk_size": "100", 2023-09-23T20:52:05.608 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_objexp_gc_interval": "600", 2023-09-23T20:52:05.608 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_objexp_hints_num_shards": "127", 2023-09-23T20:52:05.609 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_olh_pending_timeout_sec": "3600", 2023-09-23T20:52:05.609 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_op_thread_suicide_timeout": "0", 2023-09-23T20:52:05.609 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_op_thread_timeout": "600", 2023-09-23T20:52:05.609 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_op_tracing": "false", 2023-09-23T20:52:05.609 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_opa_token": "", 2023-09-23T20:52:05.609 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_opa_url": "", 2023-09-23T20:52:05.609 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_opa_verify_ssl": "true", 2023-09-23T20:52:05.609 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_ops_log_data_backlog": "5242880", 2023-09-23T20:52:05.609 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_ops_log_file_path": "/var/log/ceph/ops-log-ceph-mon.a.log", 2023-09-23T20:52:05.609 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_ops_log_rados": "false", 2023-09-23T20:52:05.609 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_ops_log_socket_path": "", 2023-09-23T20:52:05.609 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_override_bucket_index_max_shards": "0", 2023-09-23T20:52:05.609 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_pending_bucket_index_op_expiration": "120", 2023-09-23T20:52:05.610 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_period_latest_epoch_info_oid": ".latest_epoch", 2023-09-23T20:52:05.610 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_period_push_interval": "2.000000", 2023-09-23T20:52:05.610 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_period_push_interval_max": "30.000000", 2023-09-23T20:52:05.610 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_period_root_pool": ".rgw.root", 2023-09-23T20:52:05.610 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_policy_reject_invalid_principals": "true", 2023-09-23T20:52:05.610 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_print_continue": "true", 2023-09-23T20:52:05.610 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_print_prohibited_content_length": "false", 2023-09-23T20:52:05.610 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_put_obj_max_window_size": "67108864", 2023-09-23T20:52:05.610 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_put_obj_min_window_size": "16777216", 2023-09-23T20:52:05.610 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_rados_pool_autoscale_bias": "4.000000", 2023-09-23T20:52:05.610 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_rados_pool_recovery_priority": "5", 2023-09-23T20:52:05.610 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_rados_tracing": "false", 2023-09-23T20:52:05.610 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_realm": "", 2023-09-23T20:52:05.611 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_realm_id": "", 2023-09-23T20:52:05.611 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_realm_root_pool": ".rgw.root", 2023-09-23T20:52:05.611 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_region": "", 2023-09-23T20:52:05.611 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_region_root_pool": ".rgw.root", 2023-09-23T20:52:05.611 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_relaxed_region_enforcement": "false", 2023-09-23T20:52:05.611 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_relaxed_s3_bucket_names": "false", 2023-09-23T20:52:05.611 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_remote_addr_param": "REMOTE_ADDR", 2023-09-23T20:52:05.611 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_request_uri": "", 2023-09-23T20:52:05.611 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_reshard_batch_size": "64", 2023-09-23T20:52:05.611 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_reshard_bucket_lock_duration": "360", 2023-09-23T20:52:05.611 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_reshard_max_aio": "128", 2023-09-23T20:52:05.611 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_reshard_num_logs": "16", 2023-09-23T20:52:05.611 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_reshard_thread_interval": "600", 2023-09-23T20:52:05.612 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_resolve_cname": "false", 2023-09-23T20:52:05.612 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_rest_getusage_op_compat": "false", 2023-09-23T20:52:05.612 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_run_sync_thread": "true", 2023-09-23T20:52:05.612 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_s3_auth_order": "sts, external, local", 2023-09-23T20:52:05.612 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_s3_auth_use_keystone": "false", 2023-09-23T20:52:05.612 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_s3_auth_use_ldap": "false", 2023-09-23T20:52:05.612 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_s3_auth_use_rados": "true", 2023-09-23T20:52:05.612 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_s3_auth_use_sts": "false", 2023-09-23T20:52:05.612 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_s3_client_max_sig_ver": "-1", 2023-09-23T20:52:05.612 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_s3_success_create_obj_status": "0", 2023-09-23T20:52:05.612 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_safe_max_objects_per_shard": "102400", 2023-09-23T20:52:05.612 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_scheduler_type": "throttler", 2023-09-23T20:52:05.612 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_script_uri": "", 2023-09-23T20:52:05.613 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_service_provider_name": "", 2023-09-23T20:52:05.613 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_shard_warning_threshold": "90.000000", 2023-09-23T20:52:05.613 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_sts_client_id": "", 2023-09-23T20:52:05.613 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_sts_client_secret": "", 2023-09-23T20:52:05.613 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_sts_entry": "sts", 2023-09-23T20:52:05.613 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_sts_key": "sts", 2023-09-23T20:52:05.613 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_sts_max_session_duration": "43200", 2023-09-23T20:52:05.613 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_sts_min_session_duration": "900", 2023-09-23T20:52:05.613 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_sts_token_introspection_url": "", 2023-09-23T20:52:05.613 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_swift_account_in_url": "false", 2023-09-23T20:52:05.613 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_swift_auth_entry": "auth", 2023-09-23T20:52:05.613 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_swift_auth_url": "", 2023-09-23T20:52:05.613 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_swift_custom_header": "", 2023-09-23T20:52:05.614 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_swift_enforce_content_length": "false", 2023-09-23T20:52:05.614 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_swift_need_stats": "true", 2023-09-23T20:52:05.614 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_swift_tenant_name": "", 2023-09-23T20:52:05.614 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_swift_token_expiration": "86400", 2023-09-23T20:52:05.614 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_swift_url": "", 2023-09-23T20:52:05.614 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_swift_url_prefix": "swift", 2023-09-23T20:52:05.614 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_swift_versioning_enabled": "false", 2023-09-23T20:52:05.614 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_sync_data_full_inject_err_probability": "0.000000", 2023-09-23T20:52:05.614 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_sync_data_inject_err_probability": "0.000000", 2023-09-23T20:52:05.614 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_sync_lease_period": "120", 2023-09-23T20:52:05.614 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_sync_log_trim_concurrent_buckets": "4", 2023-09-23T20:52:05.614 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_sync_log_trim_interval": "1200", 2023-09-23T20:52:05.615 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_sync_log_trim_max_buckets": "16", 2023-09-23T20:52:05.615 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_sync_log_trim_min_cold_buckets": "4", 2023-09-23T20:52:05.615 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_sync_meta_inject_err_probability": "0.000000", 2023-09-23T20:52:05.615 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_sync_obj_etag_verify": "false", 2023-09-23T20:52:05.615 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_sync_trace_history_size": "4096", 2023-09-23T20:52:05.615 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_sync_trace_per_node_log_size": "32", 2023-09-23T20:52:05.615 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_sync_trace_servicemap_update_interval": "10", 2023-09-23T20:52:05.615 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_thread_pool_size": "512", 2023-09-23T20:52:05.615 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_torrent_comment": "", 2023-09-23T20:52:05.615 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_torrent_createby": "", 2023-09-23T20:52:05.615 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_torrent_encoding": "", 2023-09-23T20:52:05.615 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_torrent_flag": "false", 2023-09-23T20:52:05.616 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_torrent_origin": "", 2023-09-23T20:52:05.616 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_torrent_sha_unit": "524288", 2023-09-23T20:52:05.616 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_torrent_tracker": "", 2023-09-23T20:52:05.616 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_trust_forwarded_https": "false", 2023-09-23T20:52:05.616 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_usage_log_flush_threshold": "1024", 2023-09-23T20:52:05.616 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_usage_log_tick_interval": "30", 2023-09-23T20:52:05.616 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_usage_max_shards": "32", 2023-09-23T20:52:05.616 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_usage_max_user_shards": "1", 2023-09-23T20:52:05.616 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_use_opa_authz": "false", 2023-09-23T20:52:05.616 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_user_default_quota_max_objects": "-1", 2023-09-23T20:52:05.616 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_user_default_quota_max_size": "-1", 2023-09-23T20:52:05.616 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_user_max_buckets": "1000", 2023-09-23T20:52:05.616 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_user_policies_max_num": "100", 2023-09-23T20:52:05.617 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_user_quota_bucket_sync_interval": "180", 2023-09-23T20:52:05.617 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_user_quota_sync_idle_users": "false", 2023-09-23T20:52:05.617 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_user_quota_sync_interval": "86400", 2023-09-23T20:52:05.617 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_user_quota_sync_wait_time": "86400", 2023-09-23T20:52:05.617 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_user_unique_email": "true", 2023-09-23T20:52:05.617 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_verify_ssl": "true", 2023-09-23T20:52:05.617 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_website_routing_rules_max_num": "50", 2023-09-23T20:52:05.617 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_zone": "", 2023-09-23T20:52:05.617 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_zone_id": "", 2023-09-23T20:52:05.617 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_zone_root_pool": ".rgw.root", 2023-09-23T20:52:05.617 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_zonegroup": "", 2023-09-23T20:52:05.617 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_zonegroup_id": "", 2023-09-23T20:52:05.617 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_zonegroup_root_pool": ".rgw.root", 2023-09-23T20:52:05.618 INFO:teuthology.orchestra.run.smithi026.stdout: "rgwlc_auto_session_clear": "true", 2023-09-23T20:52:05.618 INFO:teuthology.orchestra.run.smithi026.stdout: "rgwlc_skip_bucket_step": "false", 2023-09-23T20:52:05.618 INFO:teuthology.orchestra.run.smithi026.stdout: "rocksdb_block_size": "4096", 2023-09-23T20:52:05.618 INFO:teuthology.orchestra.run.smithi026.stdout: "rocksdb_bloom_bits_per_key": "20", 2023-09-23T20:52:05.618 INFO:teuthology.orchestra.run.smithi026.stdout: "rocksdb_cache_index_and_filter_blocks": "true", 2023-09-23T20:52:05.618 INFO:teuthology.orchestra.run.smithi026.stdout: "rocksdb_cache_index_and_filter_blocks_with_high_priority": "false", 2023-09-23T20:52:05.618 INFO:teuthology.orchestra.run.smithi026.stdout: "rocksdb_cache_row_ratio": "0.000000", 2023-09-23T20:52:05.618 INFO:teuthology.orchestra.run.smithi026.stdout: "rocksdb_cache_shard_bits": "4", 2023-09-23T20:52:05.618 INFO:teuthology.orchestra.run.smithi026.stdout: "rocksdb_cache_size": "536870912", 2023-09-23T20:52:05.618 INFO:teuthology.orchestra.run.smithi026.stdout: "rocksdb_cache_type": "binned_lru", 2023-09-23T20:52:05.618 INFO:teuthology.orchestra.run.smithi026.stdout: "rocksdb_cf_compact_on_deletion": "true", 2023-09-23T20:52:05.618 INFO:teuthology.orchestra.run.smithi026.stdout: "rocksdb_cf_compact_on_deletion_sliding_window": "32768", 2023-09-23T20:52:05.619 INFO:teuthology.orchestra.run.smithi026.stdout: "rocksdb_cf_compact_on_deletion_trigger": "16384", 2023-09-23T20:52:05.619 INFO:teuthology.orchestra.run.smithi026.stdout: "rocksdb_collect_compaction_stats": "false", 2023-09-23T20:52:05.619 INFO:teuthology.orchestra.run.smithi026.stdout: "rocksdb_collect_extended_stats": "false", 2023-09-23T20:52:05.619 INFO:teuthology.orchestra.run.smithi026.stdout: "rocksdb_collect_memory_stats": "false", 2023-09-23T20:52:05.619 INFO:teuthology.orchestra.run.smithi026.stdout: "rocksdb_delete_range_threshold": "1048576", 2023-09-23T20:52:05.619 INFO:teuthology.orchestra.run.smithi026.stdout: "rocksdb_index_type": "binary_search", 2023-09-23T20:52:05.619 INFO:teuthology.orchestra.run.smithi026.stdout: "rocksdb_log_to_ceph_log": "true", 2023-09-23T20:52:05.619 INFO:teuthology.orchestra.run.smithi026.stdout: "rocksdb_metadata_block_size": "4096", 2023-09-23T20:52:05.619 INFO:teuthology.orchestra.run.smithi026.stdout: "rocksdb_partition_filters": "false", 2023-09-23T20:52:05.619 INFO:teuthology.orchestra.run.smithi026.stdout: "rocksdb_perf": "false", 2023-09-23T20:52:05.619 INFO:teuthology.orchestra.run.smithi026.stdout: "rocksdb_pin_l0_filter_and_index_blocks_in_cache": "false", 2023-09-23T20:52:05.619 INFO:teuthology.orchestra.run.smithi026.stdout: "rotating_keys_bootstrap_timeout": "30", 2023-09-23T20:52:05.619 INFO:teuthology.orchestra.run.smithi026.stdout: "rotating_keys_renewal_timeout": "10", 2023-09-23T20:52:05.620 INFO:teuthology.orchestra.run.smithi026.stdout: "run_dir": "/var/run/ceph", 2023-09-23T20:52:05.620 INFO:teuthology.orchestra.run.smithi026.stdout: "seastore_block_create": "true", 2023-09-23T20:52:05.620 INFO:teuthology.orchestra.run.smithi026.stdout: "seastore_cache_lru_size": "67108864", 2023-09-23T20:52:05.620 INFO:teuthology.orchestra.run.smithi026.stdout: "seastore_cbjournal_size": "5368709120", 2023-09-23T20:52:05.620 INFO:teuthology.orchestra.run.smithi026.stdout: "seastore_default_max_object_size": "16777216", 2023-09-23T20:52:05.620 INFO:teuthology.orchestra.run.smithi026.stdout: "seastore_default_object_metadata_reservation": "16777216", 2023-09-23T20:52:05.620 INFO:teuthology.orchestra.run.smithi026.stdout: "seastore_device_size": "53687091200", 2023-09-23T20:52:05.620 INFO:teuthology.orchestra.run.smithi026.stdout: "seastore_journal_batch_capacity": "16", 2023-09-23T20:52:05.620 INFO:teuthology.orchestra.run.smithi026.stdout: "seastore_journal_batch_flush_size": "16777216", 2023-09-23T20:52:05.620 INFO:teuthology.orchestra.run.smithi026.stdout: "seastore_journal_batch_preferred_fullness": "0.950000", 2023-09-23T20:52:05.620 INFO:teuthology.orchestra.run.smithi026.stdout: "seastore_journal_iodepth_limit": "5", 2023-09-23T20:52:05.620 INFO:teuthology.orchestra.run.smithi026.stdout: "seastore_main_device_type": "SSD", 2023-09-23T20:52:05.621 INFO:teuthology.orchestra.run.smithi026.stdout: "seastore_max_concurrent_transactions": "8", 2023-09-23T20:52:05.621 INFO:teuthology.orchestra.run.smithi026.stdout: "seastore_multiple_tiers_default_evict_ratio": "0.600000", 2023-09-23T20:52:05.621 INFO:teuthology.orchestra.run.smithi026.stdout: "seastore_multiple_tiers_fast_evict_ratio": "0.700000", 2023-09-23T20:52:05.621 INFO:teuthology.orchestra.run.smithi026.stdout: "seastore_multiple_tiers_stop_evict_ratio": "0.500000", 2023-09-23T20:52:05.621 INFO:teuthology.orchestra.run.smithi026.stdout: "seastore_obj_data_write_amplification": "1.250000", 2023-09-23T20:52:05.621 INFO:teuthology.orchestra.run.smithi026.stdout: "seastore_segment_size": "67108864", 2023-09-23T20:52:05.621 INFO:teuthology.orchestra.run.smithi026.stdout: "set_keepcaps": "false", 2023-09-23T20:52:05.621 INFO:teuthology.orchestra.run.smithi026.stdout: "setgroup": "ceph", 2023-09-23T20:52:05.621 INFO:teuthology.orchestra.run.smithi026.stdout: "setuser": "ceph", 2023-09-23T20:52:05.621 INFO:teuthology.orchestra.run.smithi026.stdout: "setuser_match_path": "", 2023-09-23T20:52:05.621 INFO:teuthology.orchestra.run.smithi026.stdout: "target_max_misplaced_ratio": "0.050000", 2023-09-23T20:52:05.621 INFO:teuthology.orchestra.run.smithi026.stdout: "thp": "false", 2023-09-23T20:52:05.622 INFO:teuthology.orchestra.run.smithi026.stdout: "threadpool_default_timeout": "60", 2023-09-23T20:52:05.622 INFO:teuthology.orchestra.run.smithi026.stdout: "threadpool_empty_queue_max_wait": "2", 2023-09-23T20:52:05.622 INFO:teuthology.orchestra.run.smithi026.stdout: "throttler_perf_counter": "true" 2023-09-23T20:52:05.622 INFO:teuthology.orchestra.run.smithi026.stdout:} 2023-09-23T20:52:05.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:52:05 smithi026 ceph-mon[175151]: pgmap v227: 73 pgs: 73 active+clean; 582 KiB data, 171 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:52:05.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:52:05 smithi026 ceph-mon[179688]: pgmap v227: 73 pgs: 73 active+clean; 582 KiB data, 171 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:52:05.904 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph -- tell 'mon.*' injectargs --mon_osd_down_out_interval=0 2023-09-23T20:52:06.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:52:05 smithi070 ceph-mon[168282]: pgmap v227: 73 pgs: 73 active+clean; 582 KiB data, 171 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:52:06.509 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:52:07.734 INFO:teuthology.orchestra.run.smithi026.stdout:mon.a: {} 2023-09-23T20:52:07.745 INFO:teuthology.orchestra.run.smithi026.stdout:mon.b: {} 2023-09-23T20:52:07.755 INFO:teuthology.orchestra.run.smithi026.stdout:mon.c: {} 2023-09-23T20:52:07.756 INFO:teuthology.orchestra.run.smithi026.stderr:mon.a: mon_osd_down_out_interval = '0' (not observed, change may require restart) 2023-09-23T20:52:07.756 INFO:teuthology.orchestra.run.smithi026.stderr:mon.b: mon_osd_down_out_interval = '0' (not observed, change may require restart) 2023-09-23T20:52:07.756 INFO:teuthology.orchestra.run.smithi026.stderr:mon.c: mon_osd_down_out_interval = '0' (not observed, change may require restart) 2023-09-23T20:52:08.106 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:52:07 smithi026 ceph-mon[175151]: pgmap v228: 73 pgs: 73 active+clean; 582 KiB data, 171 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:52:08.106 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:52:07 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/2837458697' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-09-23T20:52:08.106 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:52:07 smithi026 ceph-mon[179688]: pgmap v228: 73 pgs: 73 active+clean; 582 KiB data, 171 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:52:08.106 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:52:07 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/2837458697' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-09-23T20:52:08.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:52:07 smithi070 ceph-mon[168282]: pgmap v228: 73 pgs: 73 active+clean; 582 KiB data, 171 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:52:08.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:52:07 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/2837458697' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-09-23T20:52:08.169 INFO:tasks.thrashosds.thrasher:Unable to test ceph-objectstore-tool, not available on all OSD nodes 2023-09-23T20:52:08.169 INFO:teuthology.task.sequential:In sequential, running task full_sequential... 2023-09-23T20:52:08.170 INFO:teuthology.task.full_sequential:In full_sequential, running task radosbench... 2023-09-23T20:52:08.171 INFO:tasks.radosbench:Beginning radosbench... 2023-09-23T20:52:08.172 INFO:tasks.cephadm.ceph_manager.ceph:creating pool_name unique_pool_0 2023-09-23T20:52:08.172 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph osd pool create unique_pool_0 16 2023-09-23T20:52:08.173 INFO:tasks.thrashosds.thrasher:starting do_thrash 2023-09-23T20:52:08.173 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6, 7] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-09-23T20:52:08.173 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T20:52:08.173 INFO:tasks.thrashosds.thrasher:Removing osd 5, in_osds are: [0, 1, 2, 3, 4, 5, 6, 7] 2023-09-23T20:52:08.173 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph osd out 5 2023-09-23T20:52:08.173 INFO:tasks.thrashosds.thrasher:starting do_sighup with a delay of 0.1 2023-09-23T20:52:08.174 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:52:08.174 INFO:tasks.thrashosds.thrasher:starting do_optrack_toggle with a delay of 2.0 2023-09-23T20:52:08.174 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T20:52:08.174 INFO:tasks.thrashosds.thrasher:starting do_dump_ops 2023-09-23T20:52:08.175 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_ops_in_flight 2023-09-23T20:52:08.175 INFO:tasks.thrashosds.thrasher:starting do_noscrub_toggle with a delay of 2.0 2023-09-23T20:52:08.175 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T20:52:08.230 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:52:08.289 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:52:08.294 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:52:08.294 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:52:08.395 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:52:08.459 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:52:08.499 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:52:08.501 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:52:08.502 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:52:08.603 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:52:08.632 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:52:08.665 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:52:08.667 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:52:08.668 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:52:08.769 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:52:08.794 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:52:08.825 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:52:08.827 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:52:08.828 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:52:08.850 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:52:08.879 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:52:08.898 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:52:08.929 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:52:08.957 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:52:08.979 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:52:08.979 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:52:09.002 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:52:09.008 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:52:09.008 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:52:09.109 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:52:09.135 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:52:09.167 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:52:09.170 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:52:09.170 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:52:09.271 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:52:09.299 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:52:09.535 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:52:09.539 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:52:09.539 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:52:09.640 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:52:09.667 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:52:09.699 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:52:09.702 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:52:09.702 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:52:09.803 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:52:09.829 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:52:09.861 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:52:09.863 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:52:09.864 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:52:09.881 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:52:09 smithi026 ceph-mon[175151]: pgmap v229: 73 pgs: 73 active+clean; 582 KiB data, 171 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:52:09.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:52:09 smithi026 ceph-mon[179688]: pgmap v229: 73 pgs: 73 active+clean; 582 KiB data, 171 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:52:09.882 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:52:09 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[164029]: ::ffff:172.21.15.70 - - [23/Sep/2023:20:52:09] "GET /metrics HTTP/1.1" 200 33812 "" "Prometheus/2.43.0" 2023-09-23T20:52:09.964 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:52:10.000 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:52:10.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:52:09 smithi070 ceph-mon[168282]: pgmap v229: 73 pgs: 73 active+clean; 582 KiB data, 171 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:52:11.122 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:52:11.125 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:52:11.126 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:52:11.226 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:52:11.258 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:52:11.294 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:52:11.296 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:52:11.297 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:52:11.398 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:52:11.426 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:52:11.459 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:52:11.462 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:52:11.462 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:52:11.563 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:52:11.590 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:52:11.622 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:52:11.624 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:52:11.625 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:52:11.725 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:52:11.753 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:52:11 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mon-a[175127]: 2023-09-23T20:52:11.651+0000 7f1e2d487700 -1 mon.a@0(leader).osd e99 definitely_dead 0 2023-09-23T20:52:11.756 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:52:11.850 INFO:teuthology.orchestra.run.smithi026.stderr:marked out osd.5. 2023-09-23T20:52:11.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:52:11 smithi070 ceph-mon[168282]: pgmap v230: 73 pgs: 73 active+clean; 582 KiB data, 171 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:52:11.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:52:11 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/3589423328' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["5"]}]: dispatch 2023-09-23T20:52:11.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:52:11 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["5"]}]: dispatch 2023-09-23T20:52:11.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:52:11 smithi070 ceph-mon[168282]: Client client.admin marked osd.5 out, while it was still marked up 2023-09-23T20:52:12.016 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:52:11 smithi026 ceph-mon[179688]: pgmap v230: 73 pgs: 73 active+clean; 582 KiB data, 171 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:52:12.016 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:52:11 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/3589423328' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["5"]}]: dispatch 2023-09-23T20:52:12.016 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:52:11 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["5"]}]: dispatch 2023-09-23T20:52:12.017 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:52:11 smithi026 ceph-mon[179688]: Client client.admin marked osd.5 out, while it was still marked up 2023-09-23T20:52:12.017 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:52:11 smithi026 ceph-mon[175151]: pgmap v230: 73 pgs: 73 active+clean; 582 KiB data, 171 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:52:12.017 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:52:11 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/3589423328' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["5"]}]: dispatch 2023-09-23T20:52:12.017 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:52:11 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["5"]}]: dispatch 2023-09-23T20:52:12.017 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:52:11 smithi026 ceph-mon[175151]: Client client.admin marked osd.5 out, while it was still marked up 2023-09-23T20:52:12.046 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:52:12.050 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:52:12.050 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:52:12.151 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:52:12.178 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:52:12.210 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:52:12.213 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:52:12.213 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:52:12.226 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0: {} 2023-09-23T20:52:12.244 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1: {} 2023-09-23T20:52:12.262 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2: {} 2023-09-23T20:52:12.281 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3: {} 2023-09-23T20:52:12.300 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4: {} 2023-09-23T20:52:12.314 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:52:12.318 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T20:52:12.338 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6: {} 2023-09-23T20:52:12.339 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:52:12.357 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T20:52:12.358 INFO:teuthology.orchestra.run.smithi026.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T20:52:12.358 INFO:teuthology.orchestra.run.smithi026.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T20:52:12.358 INFO:teuthology.orchestra.run.smithi026.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T20:52:12.358 INFO:teuthology.orchestra.run.smithi026.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T20:52:12.359 INFO:teuthology.orchestra.run.smithi026.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T20:52:12.359 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T20:52:12.359 INFO:teuthology.orchestra.run.smithi026.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T20:52:12.359 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T20:52:12.371 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:52:12.373 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:52:12.374 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:52:12.474 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:52:12.507 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:52:12.549 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:52:12.552 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:52:12.553 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:52:12.654 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:52:12.683 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:52:12.864 INFO:teuthology.orchestra.run.smithi026.stderr:noscrub is set 2023-09-23T20:52:12.933 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:52:12 smithi026 ceph-mon[179688]: failed to encode map e100 with expected crc 2023-09-23T20:52:12.933 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:52:12 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["5"]}]': finished 2023-09-23T20:52:12.933 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:52:12 smithi026 ceph-mon[179688]: osdmap e100: 8 total, 8 up, 7 in 2023-09-23T20:52:12.934 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:52:12 smithi026 ceph-mon[179688]: failed to encode map e100 with expected crc 2023-09-23T20:52:12.934 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:52:12 smithi026 ceph-mon[179688]: failed to encode map e100 with expected crc 2023-09-23T20:52:12.934 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:52:12 smithi026 ceph-mon[179688]: failed to encode map e100 with expected crc 2023-09-23T20:52:12.934 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:52:12 smithi026 ceph-mon[179688]: failed to encode map e100 with expected crc 2023-09-23T20:52:12.934 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:52:12 smithi026 ceph-mon[179688]: failed to encode map e100 with expected crc 2023-09-23T20:52:12.934 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:52:12 smithi026 ceph-mon[179688]: failed to encode map e100 with expected crc 2023-09-23T20:52:12.934 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:52:12 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/294358195' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:52:12.934 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:52:12 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/2404512947' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:52:12.934 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:52:12 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:52:12.935 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:52:12 smithi026 ceph-mon[175151]: failed to encode map e100 with expected crc 2023-09-23T20:52:12.935 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:52:12 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["5"]}]': finished 2023-09-23T20:52:12.935 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:52:12 smithi026 ceph-mon[175151]: osdmap e100: 8 total, 8 up, 7 in 2023-09-23T20:52:12.935 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:52:12 smithi026 ceph-mon[175151]: failed to encode map e100 with expected crc 2023-09-23T20:52:12.935 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:52:12 smithi026 ceph-mon[175151]: failed to encode map e100 with expected crc 2023-09-23T20:52:12.935 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:52:12 smithi026 ceph-mon[175151]: failed to encode map e100 with expected crc 2023-09-23T20:52:12.935 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:52:12 smithi026 ceph-mon[175151]: failed to encode map e100 with expected crc 2023-09-23T20:52:12.935 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:52:12 smithi026 ceph-mon[175151]: failed to encode map e100 with expected crc 2023-09-23T20:52:12.935 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:52:12 smithi026 ceph-mon[175151]: failed to encode map e100 with expected crc 2023-09-23T20:52:12.935 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:52:12 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/294358195' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:52:12.935 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:52:12 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/2404512947' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:52:12.936 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:52:12 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:52:13.118 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:52:13.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:52:12 smithi070 ceph-mon[168282]: failed to encode map e100 with expected crc 2023-09-23T20:52:13.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:52:12 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["5"]}]': finished 2023-09-23T20:52:13.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:52:12 smithi070 ceph-mon[168282]: osdmap e100: 8 total, 8 up, 7 in 2023-09-23T20:52:13.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:52:12 smithi070 ceph-mon[168282]: failed to encode map e100 with expected crc 2023-09-23T20:52:13.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:52:12 smithi070 ceph-mon[168282]: failed to encode map e100 with expected crc 2023-09-23T20:52:13.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:52:12 smithi070 ceph-mon[168282]: failed to encode map e100 with expected crc 2023-09-23T20:52:13.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:52:12 smithi070 ceph-mon[168282]: failed to encode map e100 with expected crc 2023-09-23T20:52:13.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:52:12 smithi070 ceph-mon[168282]: failed to encode map e100 with expected crc 2023-09-23T20:52:13.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:52:12 smithi070 ceph-mon[168282]: failed to encode map e100 with expected crc 2023-09-23T20:52:13.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:52:12 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/294358195' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:52:13.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:52:12 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/2404512947' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:52:13.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:52:12 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:52:13.124 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:52:13.124 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:52:13.225 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:52:13.254 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:52:13.288 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:52:13.290 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:52:13.291 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:52:13.293 INFO:teuthology.orchestra.run.smithi026.stdout:ERROR: (22) Invalid argument 2023-09-23T20:52:13.294 INFO:teuthology.orchestra.run.smithi026.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:52:13.391 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:52:13.425 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:52:13.816 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:52:13.820 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:52:13.821 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:52:13.840 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_blocked_ops 2023-09-23T20:52:13.856 INFO:teuthology.orchestra.run.smithi026.stderr:pool 'unique_pool_0' created 2023-09-23T20:52:13.921 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:52:13.954 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:52:13.991 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:52:13.994 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:52:13.994 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:52:14.095 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:52:14.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:52:13 smithi070 ceph-mon[168282]: pgmap v232: 73 pgs: 9 peering, 64 active+clean; 582 KiB data, 153 MiB used, 626 GiB / 626 GiB avail 2023-09-23T20:52:14.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:52:13 smithi070 ceph-mon[168282]: Health check failed: Reduced data availability: 3 pgs inactive, 9 pgs peering (PG_AVAILABILITY) 2023-09-23T20:52:14.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:52:13 smithi070 ceph-mon[168282]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:52:14.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:52:13 smithi070 ceph-mon[168282]: failed to encode map e101 with expected crc 2023-09-23T20:52:14.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:52:13 smithi070 ceph-mon[168282]: failed to encode map e101 with expected crc 2023-09-23T20:52:14.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:52:13 smithi070 ceph-mon[168282]: failed to encode map e101 with expected crc 2023-09-23T20:52:14.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:52:13 smithi070 ceph-mon[168282]: failed to encode map e101 with expected crc 2023-09-23T20:52:14.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:52:13 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T20:52:14.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:52:13 smithi070 ceph-mon[168282]: osdmap e101: 8 total, 8 up, 7 in 2023-09-23T20:52:14.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:52:13 smithi070 ceph-mon[168282]: failed to encode map e101 with expected crc 2023-09-23T20:52:14.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:52:13 smithi070 ceph-mon[168282]: failed to encode map e101 with expected crc 2023-09-23T20:52:14.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:52:13 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/2219962345' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16}]: dispatch 2023-09-23T20:52:14.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:52:13 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16}]: dispatch 2023-09-23T20:52:14.124 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:52:14.131 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:52:13 smithi026 ceph-mon[175151]: pgmap v232: 73 pgs: 9 peering, 64 active+clean; 582 KiB data, 153 MiB used, 626 GiB / 626 GiB avail 2023-09-23T20:52:14.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:52:13 smithi026 ceph-mon[175151]: Health check failed: Reduced data availability: 3 pgs inactive, 9 pgs peering (PG_AVAILABILITY) 2023-09-23T20:52:14.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:52:13 smithi026 ceph-mon[175151]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:52:14.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:52:13 smithi026 ceph-mon[175151]: failed to encode map e101 with expected crc 2023-09-23T20:52:14.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:52:13 smithi026 ceph-mon[175151]: failed to encode map e101 with expected crc 2023-09-23T20:52:14.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:52:13 smithi026 ceph-mon[175151]: failed to encode map e101 with expected crc 2023-09-23T20:52:14.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:52:13 smithi026 ceph-mon[175151]: failed to encode map e101 with expected crc 2023-09-23T20:52:14.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:52:13 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T20:52:14.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:52:13 smithi026 ceph-mon[175151]: osdmap e101: 8 total, 8 up, 7 in 2023-09-23T20:52:14.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:52:13 smithi026 ceph-mon[175151]: failed to encode map e101 with expected crc 2023-09-23T20:52:14.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:52:13 smithi026 ceph-mon[175151]: failed to encode map e101 with expected crc 2023-09-23T20:52:14.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:52:13 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/2219962345' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16}]: dispatch 2023-09-23T20:52:14.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:52:13 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16}]: dispatch 2023-09-23T20:52:14.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:52:13 smithi026 ceph-mon[179688]: pgmap v232: 73 pgs: 9 peering, 64 active+clean; 582 KiB data, 153 MiB used, 626 GiB / 626 GiB avail 2023-09-23T20:52:14.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:52:13 smithi026 ceph-mon[179688]: Health check failed: Reduced data availability: 3 pgs inactive, 9 pgs peering (PG_AVAILABILITY) 2023-09-23T20:52:14.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:52:13 smithi026 ceph-mon[179688]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:52:14.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:52:13 smithi026 ceph-mon[179688]: failed to encode map e101 with expected crc 2023-09-23T20:52:14.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:52:13 smithi026 ceph-mon[179688]: failed to encode map e101 with expected crc 2023-09-23T20:52:14.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:52:13 smithi026 ceph-mon[179688]: failed to encode map e101 with expected crc 2023-09-23T20:52:14.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:52:13 smithi026 ceph-mon[179688]: failed to encode map e101 with expected crc 2023-09-23T20:52:14.134 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:52:13 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T20:52:14.134 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:52:13 smithi026 ceph-mon[179688]: osdmap e101: 8 total, 8 up, 7 in 2023-09-23T20:52:14.134 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:52:13 smithi026 ceph-mon[179688]: failed to encode map e101 with expected crc 2023-09-23T20:52:14.134 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:52:13 smithi026 ceph-mon[179688]: failed to encode map e101 with expected crc 2023-09-23T20:52:14.134 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:52:13 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/2219962345' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16}]: dispatch 2023-09-23T20:52:14.134 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:52:13 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16}]: dispatch 2023-09-23T20:52:14.158 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:52:14.160 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:52:14.161 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:52:14.262 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:52:14.287 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph osd pool application enable unique_pool_0 rados --yes-i-really-mean-it || true 2023-09-23T20:52:14.289 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:52:14.321 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:52:14.324 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:52:14.324 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:52:14.425 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:52:14.455 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:52:14.488 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:52:14.491 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:52:14.491 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:52:14.544 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:52:14.592 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:52:14.619 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:52:14.652 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:52:14.655 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:52:14.655 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:52:14.756 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:52:14.786 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:52:14.819 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:52:14.822 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:52:14.823 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:52:14.924 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:52:14.951 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:52:14.983 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:52:14.985 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:52:14.985 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:52:15.039 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T20:52:15.064 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:52:14 smithi026 ceph-mon[175151]: failed to encode map e102 with expected crc 2023-09-23T20:52:15.064 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:52:14 smithi026 ceph-mon[175151]: failed to encode map e102 with expected crc 2023-09-23T20:52:15.064 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:52:14 smithi026 ceph-mon[175151]: failed to encode map e102 with expected crc 2023-09-23T20:52:15.064 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:52:14 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16}]': finished 2023-09-23T20:52:15.064 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:52:14 smithi026 ceph-mon[175151]: osdmap e102: 8 total, 8 up, 7 in 2023-09-23T20:52:15.065 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:52:14 smithi026 ceph-mon[179688]: failed to encode map e102 with expected crc 2023-09-23T20:52:15.065 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:52:14 smithi026 ceph-mon[179688]: failed to encode map e102 with expected crc 2023-09-23T20:52:15.065 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:52:14 smithi026 ceph-mon[179688]: failed to encode map e102 with expected crc 2023-09-23T20:52:15.065 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:52:14 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16}]': finished 2023-09-23T20:52:15.065 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:52:14 smithi026 ceph-mon[179688]: osdmap e102: 8 total, 8 up, 7 in 2023-09-23T20:52:15.075 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:52:15.086 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:52:15.110 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:52:14 smithi070 ceph-mon[168282]: failed to encode map e102 with expected crc 2023-09-23T20:52:15.110 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:52:14 smithi070 ceph-mon[168282]: failed to encode map e102 with expected crc 2023-09-23T20:52:15.110 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:52:14 smithi070 ceph-mon[168282]: failed to encode map e102 with expected crc 2023-09-23T20:52:15.110 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:52:14 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16}]': finished 2023-09-23T20:52:15.110 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:52:14 smithi070 ceph-mon[168282]: osdmap e102: 8 total, 8 up, 7 in 2023-09-23T20:52:15.112 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:52:15.144 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:52:15.146 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:52:15.146 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:52:15.247 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:52:15.276 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:52:15.610 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:52:15.613 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:52:15.613 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:52:15.714 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:52:15.742 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:52:15.776 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:52:15.778 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:52:15.779 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:52:15.801 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T20:52:15.879 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:52:15.906 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:52:15.938 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:52:15.940 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:52:15.941 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:52:16.042 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:52:16.065 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:52:15 smithi070 ceph-mon[168282]: pgmap v235: 89 pgs: 3 creating+peering, 13 unknown, 9 peering, 64 active+clean; 582 KiB data, 153 MiB used, 626 GiB / 626 GiB avail 2023-09-23T20:52:16.065 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:52:15 smithi070 ceph-mon[168282]: failed to encode map e103 with expected crc 2023-09-23T20:52:16.066 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:52:15 smithi070 ceph-mon[168282]: failed to encode map e103 with expected crc 2023-09-23T20:52:16.066 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:52:15 smithi070 ceph-mon[168282]: osdmap e103: 8 total, 8 up, 7 in 2023-09-23T20:52:16.066 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:52:15 smithi070 ceph-mon[168282]: failed to encode map e103 with expected crc 2023-09-23T20:52:16.066 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:52:15 smithi070 ceph-mon[168282]: failed to encode map e103 with expected crc 2023-09-23T20:52:16.067 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:52:16.098 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:52:16.101 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:52:16.101 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:52:16.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:52:15 smithi026 ceph-mon[175151]: pgmap v235: 89 pgs: 3 creating+peering, 13 unknown, 9 peering, 64 active+clean; 582 KiB data, 153 MiB used, 626 GiB / 626 GiB avail 2023-09-23T20:52:16.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:52:15 smithi026 ceph-mon[175151]: failed to encode map e103 with expected crc 2023-09-23T20:52:16.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:52:15 smithi026 ceph-mon[175151]: failed to encode map e103 with expected crc 2023-09-23T20:52:16.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:52:15 smithi026 ceph-mon[175151]: osdmap e103: 8 total, 8 up, 7 in 2023-09-23T20:52:16.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:52:15 smithi026 ceph-mon[175151]: failed to encode map e103 with expected crc 2023-09-23T20:52:16.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:52:15 smithi026 ceph-mon[175151]: failed to encode map e103 with expected crc 2023-09-23T20:52:16.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:52:15 smithi026 ceph-mon[179688]: pgmap v235: 89 pgs: 3 creating+peering, 13 unknown, 9 peering, 64 active+clean; 582 KiB data, 153 MiB used, 626 GiB / 626 GiB avail 2023-09-23T20:52:16.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:52:15 smithi026 ceph-mon[179688]: failed to encode map e103 with expected crc 2023-09-23T20:52:16.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:52:15 smithi026 ceph-mon[179688]: failed to encode map e103 with expected crc 2023-09-23T20:52:16.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:52:15 smithi026 ceph-mon[179688]: osdmap e103: 8 total, 8 up, 7 in 2023-09-23T20:52:16.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:52:15 smithi026 ceph-mon[179688]: failed to encode map e103 with expected crc 2023-09-23T20:52:16.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:52:15 smithi026 ceph-mon[179688]: failed to encode map e103 with expected crc 2023-09-23T20:52:16.202 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:52:16.227 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:52:16.259 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:52:16.261 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:52:16.261 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:52:16.362 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:52:16.393 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:52:16.434 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:52:16.434 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:52:16.437 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:52:16.438 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:52:16.456 INFO:teuthology.orchestra.run.smithi026.stdout:ERROR: (22) Invalid argument 2023-09-23T20:52:16.456 INFO:teuthology.orchestra.run.smithi026.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:52:16.539 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:52:16.565 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:52:16.596 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:52:16.598 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:52:16.599 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:52:16.700 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:52:16.732 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:52:16.769 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:52:16.772 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:52:16.772 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:52:16.873 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:52:16.899 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:52:16.930 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:52:16.932 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:52:16.933 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:52:17.034 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:52:17.059 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:52:16 smithi026 ceph-mon[179688]: failed to encode map e104 with expected crc 2023-09-23T20:52:17.059 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:52:16 smithi026 ceph-mon[179688]: failed to encode map e104 with expected crc 2023-09-23T20:52:17.060 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:52:16 smithi026 ceph-mon[179688]: osdmap e104: 8 total, 8 up, 7 in 2023-09-23T20:52:17.060 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:52:16 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:52:17.060 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:52:16 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:52:17.060 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:52:16 smithi026 ceph-mon[175151]: failed to encode map e104 with expected crc 2023-09-23T20:52:17.060 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:52:16 smithi026 ceph-mon[175151]: failed to encode map e104 with expected crc 2023-09-23T20:52:17.060 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:52:16 smithi026 ceph-mon[175151]: osdmap e104: 8 total, 8 up, 7 in 2023-09-23T20:52:17.060 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:52:16 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:52:17.060 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:52:16 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:52:17.062 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:52:17.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:52:16 smithi070 ceph-mon[168282]: failed to encode map e104 with expected crc 2023-09-23T20:52:17.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:52:16 smithi070 ceph-mon[168282]: failed to encode map e104 with expected crc 2023-09-23T20:52:17.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:52:16 smithi070 ceph-mon[168282]: osdmap e104: 8 total, 8 up, 7 in 2023-09-23T20:52:17.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:52:16 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:52:17.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:52:16 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:52:17.220 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:52:17.223 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:52:17.223 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:52:17.265 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:52:17.317 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_historic_ops 2023-09-23T20:52:17.324 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:52:17.350 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:52:17.381 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:52:17.384 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:52:17.384 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:52:17.485 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:52:17.513 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:52:17.740 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 6, 7] out_osds: [5] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-09-23T20:52:17.740 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph osd reweight 0 1 2023-09-23T20:52:17.754 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:52:17.763 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:52:17.764 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:52:17.864 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:52:17.870 INFO:teuthology.orchestra.run.smithi026.stderr:enabled application 'rados' on pool 'unique_pool_0' 2023-09-23T20:52:17.891 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:52:17.921 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:52:17 smithi026 ceph-mon[175151]: pgmap v238: 89 pgs: 3 creating+peering, 10 unknown, 9 peering, 67 active+clean; 582 KiB data, 154 MiB used, 626 GiB / 626 GiB avail; 138 B/s, 0 objects/s recovering 2023-09-23T20:52:17.921 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:52:17 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/797476275' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T20:52:17.922 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:52:17 smithi026 ceph-mon[179688]: pgmap v238: 89 pgs: 3 creating+peering, 10 unknown, 9 peering, 67 active+clean; 582 KiB data, 154 MiB used, 626 GiB / 626 GiB avail; 138 B/s, 0 objects/s recovering 2023-09-23T20:52:17.922 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:52:17 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/797476275' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T20:52:17.923 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:52:17.926 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:52:17.926 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:52:18.027 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:52:18.055 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:52:18.091 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:52:18.093 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:52:18.094 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:52:18.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:52:17 smithi070 ceph-mon[168282]: pgmap v238: 89 pgs: 3 creating+peering, 10 unknown, 9 peering, 67 active+clean; 582 KiB data, 154 MiB used, 626 GiB / 626 GiB avail; 138 B/s, 0 objects/s recovering 2023-09-23T20:52:18.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:52:17 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/797476275' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T20:52:18.195 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:52:18.227 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:52:18.622 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0: {} 2023-09-23T20:52:18.622 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1: {} 2023-09-23T20:52:18.622 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2: {} 2023-09-23T20:52:18.622 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3: {} 2023-09-23T20:52:18.623 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4: {} 2023-09-23T20:52:18.623 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T20:52:18.623 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6: {} 2023-09-23T20:52:18.623 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T20:52:18.623 INFO:teuthology.orchestra.run.smithi026.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T20:52:18.623 INFO:teuthology.orchestra.run.smithi026.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T20:52:18.623 INFO:teuthology.orchestra.run.smithi026.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T20:52:18.623 INFO:teuthology.orchestra.run.smithi026.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T20:52:18.623 INFO:teuthology.orchestra.run.smithi026.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T20:52:18.623 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T20:52:18.624 INFO:teuthology.orchestra.run.smithi026.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T20:52:18.624 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T20:52:18.717 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:52:18.720 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:52:18.720 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:52:18.821 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:52:18.852 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:52:18.852 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:52:18.888 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:52:18.890 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:52:18.891 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:52:18.992 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:52:19.021 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:52:19.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:52:18 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/797476275' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-09-23T20:52:19.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:52:18 smithi070 ceph-mon[168282]: osdmap e105: 8 total, 8 up, 7 in 2023-09-23T20:52:19.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:52:18 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/370405328' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:52:19.131 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:52:18 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/797476275' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-09-23T20:52:19.131 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:52:18 smithi026 ceph-mon[175151]: osdmap e105: 8 total, 8 up, 7 in 2023-09-23T20:52:19.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:52:18 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/370405328' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:52:19.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:52:18 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/797476275' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-09-23T20:52:19.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:52:18 smithi026 ceph-mon[179688]: osdmap e105: 8 total, 8 up, 7 in 2023-09-23T20:52:19.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:52:18 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/370405328' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:52:19.207 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:52:19.209 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:52:19.210 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:52:19.310 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:52:19.341 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:52:19.373 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:52:19.480 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:52:19.484 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:52:19.485 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:52:19.585 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:52:19.617 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:52:19.704 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:52:19.706 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:52:19.709 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:52:19.775 DEBUG:teuthology.orchestra.run.smithi026:> /bin/sh -c 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados --no-log-to-stderr --name client.0 -b 65536 --object-size 65536 -p unique_pool_0 bench 90 write' 2023-09-23T20:52:19.778 INFO:tasks.radosbench:joining radosbench (timing out after 3000s) 2023-09-23T20:52:19.778 INFO:teuthology.orchestra.run:waiting for 3000 2023-09-23T20:52:19.810 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:52:19.827 INFO:tasks.radosbench.radosbench.0.smithi026.stdout:hints = 1 2023-09-23T20:52:19.827 INFO:tasks.radosbench.radosbench.0.smithi026.stdout:Maintaining 16 concurrent writes of 65536 bytes to objects of size 65536 for up to 90 seconds or 0 objects 2023-09-23T20:52:19.827 INFO:tasks.radosbench.radosbench.0.smithi026.stdout:Object prefix: benchmark_data_smithi026_223868 2023-09-23T20:52:19.827 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T20:52:19.827 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 0 0 0 0 0 0 - 0 2023-09-23T20:52:19.849 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:52:19.884 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:52:19 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[164029]: ::ffff:172.21.15.70 - - [23/Sep/2023:20:52:19] "GET /metrics HTTP/1.1" 200 36245 "" "Prometheus/2.43.0" 2023-09-23T20:52:19.903 INFO:teuthology.orchestra.run.smithi026.stderr:nodeep-scrub is set 2023-09-23T20:52:19.905 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:52:19.909 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:52:19.909 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:52:20.010 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:52:20.043 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:52:20.085 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:52:20.087 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:52:20.088 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:52:20.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:52:19 smithi070 ceph-mon[168282]: failed to encode map e105 with expected crc 2023-09-23T20:52:20.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:52:19 smithi070 ceph-mon[168282]: failed to encode map e105 with expected crc 2023-09-23T20:52:20.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:52:19 smithi070 ceph-mon[168282]: pgmap v240: 89 pgs: 3 creating+peering, 4 unknown, 82 active+clean; 582 KiB data, 154 MiB used, 626 GiB / 626 GiB avail; 144 B/s, 2 objects/s recovering 2023-09-23T20:52:20.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:52:19 smithi070 ceph-mon[168282]: failed to encode map e105 with expected crc 2023-09-23T20:52:20.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:52:19 smithi070 ceph-mon[168282]: failed to encode map e105 with expected crc 2023-09-23T20:52:20.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:52:19 smithi070 ceph-mon[168282]: failed to encode map e105 with expected crc 2023-09-23T20:52:20.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:52:19 smithi070 ceph-mon[168282]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 3 pgs inactive, 9 pgs peering) 2023-09-23T20:52:20.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:52:19 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/3174395849' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:52:20.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:52:19 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:52:20.189 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:52:20.207 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:52:19 smithi026 ceph-mon[175151]: failed to encode map e105 with expected crc 2023-09-23T20:52:20.207 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:52:19 smithi026 ceph-mon[175151]: failed to encode map e105 with expected crc 2023-09-23T20:52:20.207 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:52:19 smithi026 ceph-mon[175151]: pgmap v240: 89 pgs: 3 creating+peering, 4 unknown, 82 active+clean; 582 KiB data, 154 MiB used, 626 GiB / 626 GiB avail; 144 B/s, 2 objects/s recovering 2023-09-23T20:52:20.207 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:52:19 smithi026 ceph-mon[175151]: failed to encode map e105 with expected crc 2023-09-23T20:52:20.207 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:52:19 smithi026 ceph-mon[175151]: failed to encode map e105 with expected crc 2023-09-23T20:52:20.207 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:52:19 smithi026 ceph-mon[175151]: failed to encode map e105 with expected crc 2023-09-23T20:52:20.207 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:52:19 smithi026 ceph-mon[175151]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 3 pgs inactive, 9 pgs peering) 2023-09-23T20:52:20.207 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:52:19 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/3174395849' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:52:20.208 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:52:19 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:52:20.208 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:52:19 smithi026 ceph-mon[179688]: failed to encode map e105 with expected crc 2023-09-23T20:52:20.208 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:52:19 smithi026 ceph-mon[179688]: failed to encode map e105 with expected crc 2023-09-23T20:52:20.208 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:52:19 smithi026 ceph-mon[179688]: pgmap v240: 89 pgs: 3 creating+peering, 4 unknown, 82 active+clean; 582 KiB data, 154 MiB used, 626 GiB / 626 GiB avail; 144 B/s, 2 objects/s recovering 2023-09-23T20:52:20.208 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:52:19 smithi026 ceph-mon[179688]: failed to encode map e105 with expected crc 2023-09-23T20:52:20.208 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:52:19 smithi026 ceph-mon[179688]: failed to encode map e105 with expected crc 2023-09-23T20:52:20.208 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:52:19 smithi026 ceph-mon[179688]: failed to encode map e105 with expected crc 2023-09-23T20:52:20.208 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:52:19 smithi026 ceph-mon[179688]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 3 pgs inactive, 9 pgs peering) 2023-09-23T20:52:20.208 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:52:19 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/3174395849' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:52:20.209 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:52:19 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:52:20.222 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:52:20.270 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:52:20.274 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:52:20.274 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:52:20.375 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:52:20.432 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:52:20.747 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:52:20.755 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:52:20.755 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:52:20.827 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 1 16 3160 3144 196.467 196.5 0.0086019 0.00507904 2023-09-23T20:52:20.856 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:52:20.923 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:52:21.097 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:52:20 smithi026 ceph-mon[175151]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:52:21.097 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:52:20 smithi026 ceph-mon[175151]: failed to encode map e106 with expected crc 2023-09-23T20:52:21.097 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:52:20 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T20:52:21.097 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:52:20 smithi026 ceph-mon[175151]: osdmap e106: 8 total, 8 up, 7 in 2023-09-23T20:52:21.098 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:52:20 smithi026 ceph-mon[175151]: failed to encode map e106 with expected crc 2023-09-23T20:52:21.098 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:52:20 smithi026 ceph-mon[175151]: pgmap v242: 89 pgs: 89 active+clean; 582 KiB data, 154 MiB used, 626 GiB / 626 GiB avail; 117 B/s, 1 objects/s recovering 2023-09-23T20:52:21.098 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:52:20 smithi026 ceph-mon[179688]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:52:21.098 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:52:20 smithi026 ceph-mon[179688]: failed to encode map e106 with expected crc 2023-09-23T20:52:21.098 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:52:20 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T20:52:21.098 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:52:20 smithi026 ceph-mon[179688]: osdmap e106: 8 total, 8 up, 7 in 2023-09-23T20:52:21.098 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:52:20 smithi026 ceph-mon[179688]: failed to encode map e106 with expected crc 2023-09-23T20:52:21.098 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:52:20 smithi026 ceph-mon[179688]: pgmap v242: 89 pgs: 89 active+clean; 582 KiB data, 154 MiB used, 626 GiB / 626 GiB avail; 117 B/s, 1 objects/s recovering 2023-09-23T20:52:21.140 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:52:21.144 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:52:21.144 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:52:21.245 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:52:21.253 INFO:teuthology.orchestra.run.smithi026.stdout:{ 2023-09-23T20:52:21.253 INFO:teuthology.orchestra.run.smithi026.stdout: "size": 20, 2023-09-23T20:52:21.253 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 600, 2023-09-23T20:52:21.253 INFO:teuthology.orchestra.run.smithi026.stdout: "ops": [ 2023-09-23T20:52:21.254 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:21.254 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34797.0:202 6.9 6:99598774:::benchmark_data_smithi026_223868_object201:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e105)", 2023-09-23T20:52:21.254 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T20:52:19.882995+0000", 2023-09-23T20:52:21.254 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 1.3644860489999999, 2023-09-23T20:52:21.254 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0089774580000000007, 2023-09-23T20:52:21.254 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T20:52:21.254 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:52:21.254 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T20:52:21.254 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34797", 2023-09-23T20:52:21.254 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T20:52:21.255 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 202 2023-09-23T20:52:21.255 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:21.255 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T20:52:21.255 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:21.255 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T20:52:21.255 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:19.882995+0000", 2023-09-23T20:52:21.256 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:52:21.256 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:21.256 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:21.256 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T20:52:21.256 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:19.882995+0000", 2023-09-23T20:52:21.256 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:52:21.256 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:21.257 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:21.257 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T20:52:21.257 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:19.882994+0000", 2023-09-23T20:52:21.257 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T20:52:21.257 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:21.257 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:21.258 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T20:52:21.258 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:19.883018+0000", 2023-09-23T20:52:21.258 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.3932000000000001e-05 2023-09-23T20:52:21.258 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:21.258 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:21.259 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T20:52:21.259 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:19.883034+0000", 2023-09-23T20:52:21.259 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.6396999999999999e-05 2023-09-23T20:52:21.259 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:21.259 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:21.259 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T20:52:21.259 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:19.883041+0000", 2023-09-23T20:52:21.260 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.6100000000000002e-06 2023-09-23T20:52:21.260 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:21.260 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:21.260 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:52:21.260 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:19.884153+0000", 2023-09-23T20:52:21.260 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0011124679999999999 2023-09-23T20:52:21.260 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:21.261 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:21.261 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T20:52:21.261 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:19.884281+0000", 2023-09-23T20:52:21.261 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000127136 2023-09-23T20:52:21.261 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:21.261 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:21.261 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 2,7", 2023-09-23T20:52:21.262 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:19.884419+0000", 2023-09-23T20:52:21.262 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00013820099999999999 2023-09-23T20:52:21.262 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:21.262 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:21.262 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:52:21.262 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:19.891824+0000", 2023-09-23T20:52:21.263 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0074053490000000003 2023-09-23T20:52:21.263 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:21.263 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:21.263 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T20:52:21.263 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:19.891833+0000", 2023-09-23T20:52:21.263 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.1519999999999998e-06 2023-09-23T20:52:21.264 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:21.264 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:21.264 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:52:21.264 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:19.891869+0000", 2023-09-23T20:52:21.264 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.5689999999999999e-05 2023-09-23T20:52:21.264 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:21.264 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:21.265 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T20:52:21.265 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:19.891930+0000", 2023-09-23T20:52:21.265 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.0813999999999997e-05 2023-09-23T20:52:21.265 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:21.265 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:21.265 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T20:52:21.265 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:19.891972+0000", 2023-09-23T20:52:21.265 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.231e-05 2023-09-23T20:52:21.265 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:52:21.265 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T20:52:21.265 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:52:21.266 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:21.266 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:21.266 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34797.0:578 6.9 6:9bdcf619:::benchmark_data_smithi026_223868_object577:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e106)", 2023-09-23T20:52:21.266 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T20:52:19.993752+0000", 2023-09-23T20:52:21.266 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 1.2537286009999999, 2023-09-23T20:52:21.266 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.010781198000000001, 2023-09-23T20:52:21.266 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T20:52:21.266 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:52:21.266 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T20:52:21.266 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34797", 2023-09-23T20:52:21.266 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T20:52:21.266 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 578 2023-09-23T20:52:21.267 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:21.267 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T20:52:21.267 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:21.267 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T20:52:21.267 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:19.993752+0000", 2023-09-23T20:52:21.267 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:52:21.267 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:21.267 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:21.267 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T20:52:21.267 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:19.993752+0000", 2023-09-23T20:52:21.267 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:52:21.267 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:21.268 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:21.268 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T20:52:21.268 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:19.993751+0000", 2023-09-23T20:52:21.268 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T20:52:21.268 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:21.268 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:21.268 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T20:52:21.268 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:19.993780+0000", 2023-09-23T20:52:21.268 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.9082999999999999e-05 2023-09-23T20:52:21.268 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:21.268 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:21.268 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T20:52:21.269 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:19.993794+0000", 2023-09-23T20:52:21.269 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.4093e-05 2023-09-23T20:52:21.269 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:21.269 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:21.269 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T20:52:21.269 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:19.993802+0000", 2023-09-23T20:52:21.269 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.5110000000000003e-06 2023-09-23T20:52:21.269 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:21.269 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:21.269 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:52:21.269 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:19.994258+0000", 2023-09-23T20:52:21.269 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00045591000000000002 2023-09-23T20:52:21.269 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:21.270 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:21.270 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T20:52:21.270 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:19.994428+0000", 2023-09-23T20:52:21.270 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00016992299999999999 2023-09-23T20:52:21.270 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:21.270 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:21.270 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 2,7", 2023-09-23T20:52:21.270 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:19.994644+0000", 2023-09-23T20:52:21.270 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00021634700000000001 2023-09-23T20:52:21.270 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:21.270 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:21.270 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T20:52:21.271 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:19.995355+0000", 2023-09-23T20:52:21.271 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00071084100000000003 2023-09-23T20:52:21.271 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:21.271 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:21.271 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:52:21.271 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:19.996191+0000", 2023-09-23T20:52:21.271 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000835398 2023-09-23T20:52:21.271 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:21.271 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:21.271 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:52:21.271 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:20.004425+0000", 2023-09-23T20:52:21.271 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0082342990000000005 2023-09-23T20:52:21.272 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:21.272 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:21.272 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T20:52:21.272 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:20.004493+0000", 2023-09-23T20:52:21.272 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.7687000000000004e-05 2023-09-23T20:52:21.272 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:21.272 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:21.272 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T20:52:21.272 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:20.004533+0000", 2023-09-23T20:52:21.272 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.0604e-05 2023-09-23T20:52:21.272 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:52:21.273 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T20:52:21.273 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:52:21.273 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:21.273 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:21.273 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34797.0:581 6.0 6:07804cd1:::benchmark_data_smithi026_223868_object580:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e106)", 2023-09-23T20:52:21.273 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T20:52:19.994981+0000", 2023-09-23T20:52:21.273 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 1.252499236, 2023-09-23T20:52:21.273 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0096176720000000007, 2023-09-23T20:52:21.273 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T20:52:21.273 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:52:21.273 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T20:52:21.273 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34797", 2023-09-23T20:52:21.274 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T20:52:21.274 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 581 2023-09-23T20:52:21.274 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:21.274 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T20:52:21.274 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:21.274 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T20:52:21.274 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:19.994981+0000", 2023-09-23T20:52:21.274 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:52:21.274 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:21.274 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:21.274 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T20:52:21.274 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:19.994981+0000", 2023-09-23T20:52:21.275 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:52:21.275 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:21.275 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:21.275 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T20:52:21.275 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:19.994980+0000", 2023-09-23T20:52:21.275 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999981 2023-09-23T20:52:21.275 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:21.275 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:21.275 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T20:52:21.275 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:19.995014+0000", 2023-09-23T20:52:21.275 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.4548000000000002e-05 2023-09-23T20:52:21.275 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:21.276 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:21.276 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T20:52:21.276 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:19.995028+0000", 2023-09-23T20:52:21.276 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.3901e-05 2023-09-23T20:52:21.276 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:21.276 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:21.276 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T20:52:21.276 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:19.995043+0000", 2023-09-23T20:52:21.276 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.5113000000000001e-05 2023-09-23T20:52:21.276 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:21.276 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:21.276 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:52:21.276 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:19.995097+0000", 2023-09-23T20:52:21.277 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.4184999999999997e-05 2023-09-23T20:52:21.277 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:21.277 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:21.277 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T20:52:21.277 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:19.995220+0000", 2023-09-23T20:52:21.277 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000122457 2023-09-23T20:52:21.277 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:21.277 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:21.277 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 2,3", 2023-09-23T20:52:21.277 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:19.995367+0000", 2023-09-23T20:52:21.277 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00014747 2023-09-23T20:52:21.277 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:21.277 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:21.278 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T20:52:21.278 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:19.996193+0000", 2023-09-23T20:52:21.278 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00082558800000000004 2023-09-23T20:52:21.278 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:21.278 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:21.278 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:52:21.278 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:19.997366+0000", 2023-09-23T20:52:21.278 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.001173101 2023-09-23T20:52:21.278 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:21.278 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:21.279 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:52:21.279 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:20.004516+0000", 2023-09-23T20:52:21.279 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.007149669 2023-09-23T20:52:21.279 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:21.279 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:21.279 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T20:52:21.279 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:20.004561+0000", 2023-09-23T20:52:21.279 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.5297999999999999e-05 2023-09-23T20:52:21.279 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:21.279 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:21.279 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T20:52:21.279 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:20.004599+0000", 2023-09-23T20:52:21.279 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.8260000000000003e-05 2023-09-23T20:52:21.280 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:52:21.280 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T20:52:21.280 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:52:21.280 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:21.280 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:21.280 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34797.0:1360 6.9 6:95d21b39:::benchmark_data_smithi026_223868_object1359:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e106)", 2023-09-23T20:52:21.280 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T20:52:20.262620+0000", 2023-09-23T20:52:21.280 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 0.98486107599999995, 2023-09-23T20:52:21.280 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0092281859999999993, 2023-09-23T20:52:21.280 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T20:52:21.280 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:52:21.280 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T20:52:21.281 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34797", 2023-09-23T20:52:21.281 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T20:52:21.281 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 1360 2023-09-23T20:52:21.281 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:21.281 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T20:52:21.281 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:21.281 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T20:52:21.281 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:20.262620+0000", 2023-09-23T20:52:21.281 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:52:21.281 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:21.281 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:21.281 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T20:52:21.282 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:20.262620+0000", 2023-09-23T20:52:21.282 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:52:21.282 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:21.282 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:21.282 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T20:52:21.282 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:20.262618+0000", 2023-09-23T20:52:21.282 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T20:52:21.282 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:21.282 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:21.282 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T20:52:21.282 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:20.262641+0000", 2023-09-23T20:52:21.283 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.2223000000000001e-05 2023-09-23T20:52:21.283 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:21.283 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:21.283 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T20:52:21.283 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:20.262658+0000", 2023-09-23T20:52:21.283 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.7499e-05 2023-09-23T20:52:21.283 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:21.283 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:21.283 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T20:52:21.283 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:20.262667+0000", 2023-09-23T20:52:21.283 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.653e-06 2023-09-23T20:52:21.283 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:21.284 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:21.284 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:52:21.284 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:20.262711+0000", 2023-09-23T20:52:21.284 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.4048000000000002e-05 2023-09-23T20:52:21.284 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:21.284 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:21.284 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T20:52:21.284 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:20.262843+0000", 2023-09-23T20:52:21.285 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000132318 2023-09-23T20:52:21.285 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:21.285 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:21.285 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 2,7", 2023-09-23T20:52:21.285 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:20.262984+0000", 2023-09-23T20:52:21.285 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00014130100000000001 2023-09-23T20:52:21.285 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:21.285 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:21.285 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T20:52:21.285 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:20.263480+0000", 2023-09-23T20:52:21.285 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00049560100000000003 2023-09-23T20:52:21.285 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:21.286 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:21.286 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:52:21.286 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:20.264104+0000", 2023-09-23T20:52:21.286 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000623987 2023-09-23T20:52:21.286 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:21.286 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:21.286 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:52:21.286 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:20.271756+0000", 2023-09-23T20:52:21.286 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0076523269999999996 2023-09-23T20:52:21.286 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:21.286 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:21.287 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T20:52:21.287 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:20.271808+0000", 2023-09-23T20:52:21.287 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.1539000000000002e-05 2023-09-23T20:52:21.287 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:21.287 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:21.287 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T20:52:21.287 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:20.271848+0000", 2023-09-23T20:52:21.287 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.9969000000000003e-05 2023-09-23T20:52:21.287 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:52:21.287 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T20:52:21.287 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:52:21.287 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:21.288 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:21.288 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_repop(client.34797.0:2004 6.d e106/102 6:b4ab8e89:::benchmark_data_smithi026_223868_object2003:head v 106'122, mlcod=106'120)", 2023-09-23T20:52:21.288 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T20:52:20.469708+0000", 2023-09-23T20:52:21.288 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 0.77777301399999998, 2023-09-23T20:52:21.288 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0085295419999999993, 2023-09-23T20:52:21.288 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T20:52:21.288 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:52:21.288 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T20:52:21.288 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:21.288 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T20:52:21.288 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:20.469708+0000", 2023-09-23T20:52:21.288 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:52:21.289 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:21.289 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:21.289 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T20:52:21.289 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:20.469708+0000", 2023-09-23T20:52:21.289 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:52:21.289 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:21.289 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:21.289 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T20:52:21.289 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:20.469707+0000", 2023-09-23T20:52:21.289 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T20:52:21.289 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:21.289 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:21.289 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T20:52:21.290 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:20.469741+0000", 2023-09-23T20:52:21.290 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.3884999999999998e-05 2023-09-23T20:52:21.290 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:21.290 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:21.290 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T20:52:21.290 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:20.469757+0000", 2023-09-23T20:52:21.290 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.5829999999999999e-05 2023-09-23T20:52:21.290 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:21.290 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:21.291 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T20:52:21.291 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:20.469765+0000", 2023-09-23T20:52:21.291 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.9719999999999992e-06 2023-09-23T20:52:21.291 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:21.291 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:21.291 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:52:21.291 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:20.469819+0000", 2023-09-23T20:52:21.291 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.4656e-05 2023-09-23T20:52:21.291 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:21.291 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:21.291 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T20:52:21.291 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:20.469843+0000", 2023-09-23T20:52:21.292 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.3640000000000001e-05 2023-09-23T20:52:21.292 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:21.292 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:21.292 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T20:52:21.292 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:20.478187+0000", 2023-09-23T20:52:21.292 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0083442780000000001 2023-09-23T20:52:21.292 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:21.292 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:21.292 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T20:52:21.292 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:20.478237+0000", 2023-09-23T20:52:21.292 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.9951000000000003e-05 2023-09-23T20:52:21.292 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:52:21.293 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T20:52:21.293 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:52:21.293 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:21.293 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:21.293 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_repop(client.34797.0:2010 6.d e106/102 6:b43ec735:::benchmark_data_smithi026_223868_object2009:head v 106'123, mlcod=106'120)", 2023-09-23T20:52:21.293 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T20:52:20.469780+0000", 2023-09-23T20:52:21.293 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 0.77770086000000005, 2023-09-23T20:52:21.293 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0084834339999999998, 2023-09-23T20:52:21.293 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T20:52:21.293 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:52:21.293 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T20:52:21.293 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:21.294 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T20:52:21.294 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:20.469780+0000", 2023-09-23T20:52:21.294 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:52:21.294 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:21.294 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:21.294 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T20:52:21.294 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:20.469780+0000", 2023-09-23T20:52:21.294 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:52:21.294 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:21.294 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:21.295 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T20:52:21.295 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:20.469779+0000", 2023-09-23T20:52:21.295 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T20:52:21.295 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:21.295 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:21.295 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T20:52:21.295 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:20.469811+0000", 2023-09-23T20:52:21.295 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.1588999999999999e-05 2023-09-23T20:52:21.295 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:21.295 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:21.296 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T20:52:21.296 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:20.469819+0000", 2023-09-23T20:52:21.296 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.2810000000000008e-06 2023-09-23T20:52:21.296 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:21.296 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:21.296 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T20:52:21.296 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:20.469825+0000", 2023-09-23T20:52:21.296 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.8379999999999998e-06 2023-09-23T20:52:21.296 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:21.296 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:21.297 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:52:21.297 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:20.470116+0000", 2023-09-23T20:52:21.297 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00029095199999999997 2023-09-23T20:52:21.297 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:21.297 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:21.297 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T20:52:21.297 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:20.470147+0000", 2023-09-23T20:52:21.297 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.0651000000000002e-05 2023-09-23T20:52:21.297 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:21.297 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:21.297 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T20:52:21.298 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:20.478240+0000", 2023-09-23T20:52:21.298 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0080932720000000003 2023-09-23T20:52:21.298 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:21.298 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:21.298 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T20:52:21.298 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:20.478263+0000", 2023-09-23T20:52:21.298 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.3217000000000001e-05 2023-09-23T20:52:21.298 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:52:21.298 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T20:52:21.298 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:52:21.298 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:21.299 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:21.299 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34797.0:2026 6.0 6:0610a7f9:::benchmark_data_smithi026_223868_object2025:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e106)", 2023-09-23T20:52:21.299 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T20:52:20.471448+0000", 2023-09-23T20:52:21.299 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 0.77603223899999996, 2023-09-23T20:52:21.299 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0088490389999999995, 2023-09-23T20:52:21.299 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T20:52:21.299 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:52:21.299 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T20:52:21.299 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34797", 2023-09-23T20:52:21.299 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T20:52:21.299 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 2026 2023-09-23T20:52:21.299 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:21.300 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T20:52:21.300 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:21.300 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T20:52:21.300 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:20.471448+0000", 2023-09-23T20:52:21.300 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:52:21.300 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:21.300 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:21.300 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T20:52:21.300 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:20.471448+0000", 2023-09-23T20:52:21.300 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:52:21.300 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:21.301 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:21.301 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T20:52:21.301 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:20.471440+0000", 2023-09-23T20:52:21.301 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999914 2023-09-23T20:52:21.301 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:21.301 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:21.301 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T20:52:21.301 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:20.471471+0000", 2023-09-23T20:52:21.301 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.1217000000000003e-05 2023-09-23T20:52:21.301 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:21.301 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:21.301 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T20:52:21.301 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:20.471520+0000", 2023-09-23T20:52:21.302 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.8730999999999998e-05 2023-09-23T20:52:21.302 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:21.302 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:21.302 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T20:52:21.302 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:20.471530+0000", 2023-09-23T20:52:21.302 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.8490000000000001e-06 2023-09-23T20:52:21.302 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:21.302 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:21.302 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:52:21.302 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:20.471581+0000", 2023-09-23T20:52:21.303 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.1532000000000001e-05 2023-09-23T20:52:21.303 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:21.303 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:21.303 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T20:52:21.303 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:20.471714+0000", 2023-09-23T20:52:21.303 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000133163 2023-09-23T20:52:21.303 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:21.303 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:21.303 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 2,3", 2023-09-23T20:52:21.303 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:20.471860+0000", 2023-09-23T20:52:21.303 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00014605799999999999 2023-09-23T20:52:21.303 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:21.304 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:21.304 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:52:21.304 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:20.477323+0000", 2023-09-23T20:52:21.304 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0054627920000000002 2023-09-23T20:52:21.304 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:21.304 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:21.304 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T20:52:21.304 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:20.478334+0000", 2023-09-23T20:52:21.304 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0010110060000000001 2023-09-23T20:52:21.304 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:21.304 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:21.304 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:52:21.305 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:20.480205+0000", 2023-09-23T20:52:21.305 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.001870441 2023-09-23T20:52:21.305 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:21.305 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:21.305 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T20:52:21.305 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:20.480259+0000", 2023-09-23T20:52:21.305 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.4360999999999997e-05 2023-09-23T20:52:21.305 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:21.305 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:21.305 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T20:52:21.305 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:20.480297+0000", 2023-09-23T20:52:21.305 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.8587999999999997e-05 2023-09-23T20:52:21.306 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:52:21.306 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T20:52:21.306 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:52:21.306 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:21.306 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:21.306 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34797.0:2220 6.9 6:91e35b9f:::benchmark_data_smithi026_223868_object2219:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e106)", 2023-09-23T20:52:21.306 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T20:52:20.531849+0000", 2023-09-23T20:52:21.306 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 0.71563136199999999, 2023-09-23T20:52:21.306 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0090040399999999996, 2023-09-23T20:52:21.306 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T20:52:21.306 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:52:21.307 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T20:52:21.307 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34797", 2023-09-23T20:52:21.307 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T20:52:21.307 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 2220 2023-09-23T20:52:21.307 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:21.307 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T20:52:21.307 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:21.307 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T20:52:21.307 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:20.531849+0000", 2023-09-23T20:52:21.307 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:52:21.308 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:21.308 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:21.308 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T20:52:21.308 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:20.531849+0000", 2023-09-23T20:52:21.308 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:52:21.308 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:21.308 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:21.308 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T20:52:21.308 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:20.531848+0000", 2023-09-23T20:52:21.308 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T20:52:21.308 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:21.308 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:21.308 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T20:52:21.309 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:20.531877+0000", 2023-09-23T20:52:21.309 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.915e-05 2023-09-23T20:52:21.309 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:21.309 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:21.309 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T20:52:21.309 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:20.531889+0000", 2023-09-23T20:52:21.309 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1715999999999999e-05 2023-09-23T20:52:21.309 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:21.309 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:21.309 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T20:52:21.309 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:20.531896+0000", 2023-09-23T20:52:21.310 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.4120000000000004e-06 2023-09-23T20:52:21.310 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:21.310 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:21.310 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:52:21.310 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:20.531954+0000", 2023-09-23T20:52:21.310 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.7577000000000001e-05 2023-09-23T20:52:21.310 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:21.310 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:21.310 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T20:52:21.310 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:20.532106+0000", 2023-09-23T20:52:21.310 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00015179899999999999 2023-09-23T20:52:21.311 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:21.311 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:21.311 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 2,7", 2023-09-23T20:52:21.311 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:20.532257+0000", 2023-09-23T20:52:21.311 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000151361 2023-09-23T20:52:21.311 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:21.311 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:21.311 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T20:52:21.311 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:20.533948+0000", 2023-09-23T20:52:21.311 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.001691304 2023-09-23T20:52:21.311 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:21.311 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:21.312 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:52:21.312 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:20.535145+0000", 2023-09-23T20:52:21.312 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.001196513 2023-09-23T20:52:21.312 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:21.312 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:21.312 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:52:21.312 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:20.540751+0000", 2023-09-23T20:52:21.312 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0056060950000000002 2023-09-23T20:52:21.312 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:21.312 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:21.312 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T20:52:21.312 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:20.540811+0000", 2023-09-23T20:52:21.313 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.0278000000000001e-05 2023-09-23T20:52:21.313 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:21.313 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:21.313 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T20:52:21.313 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:20.540853+0000", 2023-09-23T20:52:21.313 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.2144999999999997e-05 2023-09-23T20:52:21.313 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:52:21.313 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T20:52:21.313 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:52:21.313 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:21.313 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:21.313 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34797.0:2431 6.0 6:0943ddb6:::benchmark_data_smithi026_223868_object2430:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e106)", 2023-09-23T20:52:21.314 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T20:52:20.598271+0000", 2023-09-23T20:52:21.314 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 0.64920957700000004, 2023-09-23T20:52:21.314 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0087725730000000005, 2023-09-23T20:52:21.314 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T20:52:21.314 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:52:21.314 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T20:52:21.314 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34797", 2023-09-23T20:52:21.314 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T20:52:21.314 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 2431 2023-09-23T20:52:21.314 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:21.314 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T20:52:21.315 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:21.315 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T20:52:21.315 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:20.598271+0000", 2023-09-23T20:52:21.315 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:52:21.315 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:21.315 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:21.315 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T20:52:21.315 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:20.598271+0000", 2023-09-23T20:52:21.315 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:52:21.315 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:21.315 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:21.315 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T20:52:21.316 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:20.598269+0000", 2023-09-23T20:52:21.316 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T20:52:21.316 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:21.316 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:21.316 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T20:52:21.316 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:20.598292+0000", 2023-09-23T20:52:21.316 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.2603999999999999e-05 2023-09-23T20:52:21.316 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:21.316 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:21.316 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T20:52:21.316 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:20.598305+0000", 2023-09-23T20:52:21.316 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.3236e-05 2023-09-23T20:52:21.317 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:21.317 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:21.317 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T20:52:21.317 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:20.598313+0000", 2023-09-23T20:52:21.317 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.123e-06 2023-09-23T20:52:21.317 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:21.317 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:21.317 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:52:21.317 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:20.598636+0000", 2023-09-23T20:52:21.317 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00032270299999999998 2023-09-23T20:52:21.317 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:21.317 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:21.318 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T20:52:21.318 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:20.598790+0000", 2023-09-23T20:52:21.318 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000153671 2023-09-23T20:52:21.318 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:21.318 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:21.318 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 2,3", 2023-09-23T20:52:21.318 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:20.598979+0000", 2023-09-23T20:52:21.318 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00018889300000000001 2023-09-23T20:52:21.318 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:21.318 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:21.319 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T20:52:21.319 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:20.600211+0000", 2023-09-23T20:52:21.319 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0012322209999999999 2023-09-23T20:52:21.319 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:21.319 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:21.319 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:52:21.319 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:20.604373+0000", 2023-09-23T20:52:21.319 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0041619810000000004 2023-09-23T20:52:21.319 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:21.319 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:21.319 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:52:21.320 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:20.606942+0000", 2023-09-23T20:52:21.320 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0025691049999999999 2023-09-23T20:52:21.320 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:21.320 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:21.320 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T20:52:21.320 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:20.607003+0000", 2023-09-23T20:52:21.320 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.0739999999999998e-05 2023-09-23T20:52:21.320 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:21.320 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:21.320 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T20:52:21.320 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:20.607044+0000", 2023-09-23T20:52:21.320 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.0893999999999998e-05 2023-09-23T20:52:21.320 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:52:21.321 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T20:52:21.321 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:52:21.321 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:21.321 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:21.321 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34797.0:2556 6.0 6:0527cef3:::benchmark_data_smithi026_223868_object2555:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e106)", 2023-09-23T20:52:21.321 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T20:52:20.640874+0000", 2023-09-23T20:52:21.321 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 0.60660661800000004, 2023-09-23T20:52:21.321 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0087417090000000003, 2023-09-23T20:52:21.321 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T20:52:21.321 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:52:21.321 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T20:52:21.321 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34797", 2023-09-23T20:52:21.322 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T20:52:21.322 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 2556 2023-09-23T20:52:21.322 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:21.322 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T20:52:21.322 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:21.322 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T20:52:21.322 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:20.640874+0000", 2023-09-23T20:52:21.322 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:52:21.322 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:21.322 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:21.322 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T20:52:21.323 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:20.640874+0000", 2023-09-23T20:52:21.323 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:52:21.323 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:21.323 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:21.323 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T20:52:21.323 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:20.640872+0000", 2023-09-23T20:52:21.323 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T20:52:21.323 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:21.323 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:21.323 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T20:52:21.324 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:20.640898+0000", 2023-09-23T20:52:21.324 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.5372999999999999e-05 2023-09-23T20:52:21.324 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:21.324 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:21.324 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T20:52:21.324 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:20.640915+0000", 2023-09-23T20:52:21.325 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.7272999999999999e-05 2023-09-23T20:52:21.325 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:21.325 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:21.325 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T20:52:21.325 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:20.640924+0000", 2023-09-23T20:52:21.325 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.5210000000000002e-06 2023-09-23T20:52:21.326 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:21.326 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:21.326 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:52:21.326 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:20.640999+0000", 2023-09-23T20:52:21.326 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.5519999999999995e-05 2023-09-23T20:52:21.326 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:21.327 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:21.327 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T20:52:21.327 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:20.641128+0000", 2023-09-23T20:52:21.327 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00012877599999999999 2023-09-23T20:52:21.327 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:21.327 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:21.327 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 2,3", 2023-09-23T20:52:21.328 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:20.641291+0000", 2023-09-23T20:52:21.328 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000163449 2023-09-23T20:52:21.328 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:21.328 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:21.328 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T20:52:21.328 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:20.641932+0000", 2023-09-23T20:52:21.329 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000640141 2023-09-23T20:52:21.329 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:21.329 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:21.329 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:52:21.329 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:20.645417+0000", 2023-09-23T20:52:21.329 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.003485733 2023-09-23T20:52:21.329 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:21.330 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:21.330 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:52:21.330 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:20.649536+0000", 2023-09-23T20:52:21.330 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0041190580000000001 2023-09-23T20:52:21.330 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:21.331 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:21.331 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T20:52:21.331 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:20.649579+0000", 2023-09-23T20:52:21.331 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.2505e-05 2023-09-23T20:52:21.331 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:21.331 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:21.332 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T20:52:21.332 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:20.649616+0000", 2023-09-23T20:52:21.332 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.6933e-05 2023-09-23T20:52:21.332 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:52:21.332 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T20:52:21.332 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:52:21.333 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:21.333 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:21.333 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34797.0:3144 6.0 6:0281910c:::benchmark_data_smithi026_223868_object3143:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e106)", 2023-09-23T20:52:21.333 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T20:52:20.821665+0000", 2023-09-23T20:52:21.333 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 0.42581582499999998, 2023-09-23T20:52:21.333 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.014996548, 2023-09-23T20:52:21.334 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T20:52:21.334 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:52:21.334 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T20:52:21.334 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34797", 2023-09-23T20:52:21.334 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T20:52:21.335 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 3144 2023-09-23T20:52:21.335 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:21.335 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T20:52:21.335 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:21.335 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T20:52:21.335 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:20.821665+0000", 2023-09-23T20:52:21.336 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:52:21.336 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:21.336 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:21.336 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T20:52:21.336 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:20.821665+0000", 2023-09-23T20:52:21.336 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:52:21.337 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:21.337 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:21.337 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T20:52:21.337 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:20.821663+0000", 2023-09-23T20:52:21.337 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T20:52:21.337 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:21.338 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:21.338 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T20:52:21.338 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:20.821687+0000", 2023-09-23T20:52:21.338 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.4065000000000001e-05 2023-09-23T20:52:21.338 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:21.338 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:21.339 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T20:52:21.339 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:20.821700+0000", 2023-09-23T20:52:21.339 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.2979999999999999e-05 2023-09-23T20:52:21.339 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:21.339 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:21.339 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T20:52:21.340 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:20.821708+0000", 2023-09-23T20:52:21.340 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.1759999999999993e-06 2023-09-23T20:52:21.340 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:21.340 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:21.340 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:52:21.340 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:20.821770+0000", 2023-09-23T20:52:21.341 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.1760000000000005e-05 2023-09-23T20:52:21.341 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:21.341 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:21.341 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T20:52:21.341 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:20.821944+0000", 2023-09-23T20:52:21.341 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00017386699999999999 2023-09-23T20:52:21.342 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:21.342 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:21.342 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 2,3", 2023-09-23T20:52:21.342 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:20.822087+0000", 2023-09-23T20:52:21.342 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0001429 2023-09-23T20:52:21.342 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:21.343 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:21.343 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T20:52:21.343 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:20.822933+0000", 2023-09-23T20:52:21.343 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00084598900000000001 2023-09-23T20:52:21.343 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:21.344 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:21.344 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:52:21.344 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:20.824130+0000", 2023-09-23T20:52:21.344 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0011969050000000001 2023-09-23T20:52:21.344 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:21.345 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:21.345 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:52:21.345 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:20.836564+0000", 2023-09-23T20:52:21.345 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.012434338 2023-09-23T20:52:21.345 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:21.345 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:21.346 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T20:52:21.346 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:20.836621+0000", 2023-09-23T20:52:21.346 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.7265000000000001e-05 2023-09-23T20:52:21.346 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:21.346 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:21.346 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T20:52:21.347 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:20.836661+0000", 2023-09-23T20:52:21.347 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.9907999999999998e-05 2023-09-23T20:52:21.347 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:52:21.347 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T20:52:21.347 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:52:21.347 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:21.348 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:21.348 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34797.0:3150 6.9 6:9e2f5622:::benchmark_data_smithi026_223868_object3149:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e106)", 2023-09-23T20:52:21.348 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T20:52:20.827708+0000", 2023-09-23T20:52:21.348 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 0.41977292399999999, 2023-09-23T20:52:21.348 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.011478538, 2023-09-23T20:52:21.348 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T20:52:21.349 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:52:21.349 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T20:52:21.349 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34797", 2023-09-23T20:52:21.349 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T20:52:21.349 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 3150 2023-09-23T20:52:21.350 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:21.350 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T20:52:21.350 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:21.350 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T20:52:21.350 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:20.827708+0000", 2023-09-23T20:52:21.350 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:52:21.351 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:21.351 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:21.351 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T20:52:21.351 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:20.827708+0000", 2023-09-23T20:52:21.351 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:52:21.351 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:21.352 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:21.352 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T20:52:21.352 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:20.827705+0000", 2023-09-23T20:52:21.352 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999976 2023-09-23T20:52:21.352 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:21.352 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:21.353 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T20:52:21.353 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:20.827740+0000", 2023-09-23T20:52:21.353 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.5145999999999999e-05 2023-09-23T20:52:21.353 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:21.353 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:21.353 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T20:52:21.354 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:20.827764+0000", 2023-09-23T20:52:21.354 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.4032999999999998e-05 2023-09-23T20:52:21.354 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:21.354 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:21.354 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T20:52:21.354 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:20.827775+0000", 2023-09-23T20:52:21.355 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0813e-05 2023-09-23T20:52:21.355 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:21.355 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:21.355 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:52:21.355 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:20.827858+0000", 2023-09-23T20:52:21.356 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.3024999999999999e-05 2023-09-23T20:52:21.356 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:21.356 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:21.356 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T20:52:21.356 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:20.828021+0000", 2023-09-23T20:52:21.356 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00016250499999999999 2023-09-23T20:52:21.357 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:21.357 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:21.357 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 2,7", 2023-09-23T20:52:21.357 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:20.828175+0000", 2023-09-23T20:52:21.357 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00015392199999999999 2023-09-23T20:52:21.357 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:21.358 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:21.358 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:52:21.358 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:20.836117+0000", 2023-09-23T20:52:21.358 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0079420519999999998 2023-09-23T20:52:21.358 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:21.359 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:21.359 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T20:52:21.359 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:20.836458+0000", 2023-09-23T20:52:21.359 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000341217 2023-09-23T20:52:21.359 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:21.359 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:21.360 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:52:21.360 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:20.839050+0000", 2023-09-23T20:52:21.360 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00259158 2023-09-23T20:52:21.360 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:21.360 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:21.360 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T20:52:21.360 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:20.839114+0000", 2023-09-23T20:52:21.360 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.4641999999999995e-05 2023-09-23T20:52:21.361 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:21.361 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:21.361 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T20:52:21.361 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:20.839186+0000", 2023-09-23T20:52:21.361 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.2035999999999999e-05 2023-09-23T20:52:21.361 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:52:21.361 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T20:52:21.362 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:52:21.362 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:21.362 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:21.362 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34797.0:3157 6.9 6:99cb9021:::benchmark_data_smithi026_223868_object3156:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e106)", 2023-09-23T20:52:21.362 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T20:52:20.827798+0000", 2023-09-23T20:52:21.362 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 0.41968217600000002, 2023-09-23T20:52:21.362 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.011696359999999999, 2023-09-23T20:52:21.363 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T20:52:21.363 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:52:21.363 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T20:52:21.363 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34797", 2023-09-23T20:52:21.363 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T20:52:21.363 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 3157 2023-09-23T20:52:21.363 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:21.364 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T20:52:21.364 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:21.364 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T20:52:21.364 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:20.827798+0000", 2023-09-23T20:52:21.364 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:52:21.364 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:21.364 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:21.365 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T20:52:21.365 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:20.827798+0000", 2023-09-23T20:52:21.365 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:52:21.365 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:21.365 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:21.365 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T20:52:21.365 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:20.827798+0000", 2023-09-23T20:52:21.366 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.999999 2023-09-23T20:52:21.366 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:21.366 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:21.366 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T20:52:21.366 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:20.827832+0000", 2023-09-23T20:52:21.366 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.4408e-05 2023-09-23T20:52:21.366 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:21.366 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:21.366 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T20:52:21.366 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:20.827843+0000", 2023-09-23T20:52:21.367 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1241e-05 2023-09-23T20:52:21.367 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:21.367 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:21.367 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T20:52:21.367 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:20.827851+0000", 2023-09-23T20:52:21.367 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.357e-06 2023-09-23T20:52:21.367 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:21.367 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:21.367 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:52:21.367 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:20.828608+0000", 2023-09-23T20:52:21.367 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00075711099999999998 2023-09-23T20:52:21.367 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:21.367 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:21.368 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T20:52:21.368 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:20.828720+0000", 2023-09-23T20:52:21.368 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00011183000000000001 2023-09-23T20:52:21.368 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:21.368 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:21.368 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 2,7", 2023-09-23T20:52:21.368 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:20.828846+0000", 2023-09-23T20:52:21.368 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000126525 2023-09-23T20:52:21.368 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:21.368 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:21.368 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:52:21.368 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:20.836150+0000", 2023-09-23T20:52:21.368 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0073038629999999998 2023-09-23T20:52:21.369 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:21.369 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:21.369 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T20:52:21.369 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:20.836467+0000", 2023-09-23T20:52:21.369 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00031645600000000001 2023-09-23T20:52:21.369 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:21.369 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:21.369 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:52:21.369 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:20.839391+0000", 2023-09-23T20:52:21.369 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.002924116 2023-09-23T20:52:21.369 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:21.369 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:21.369 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T20:52:21.370 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:20.839440+0000", 2023-09-23T20:52:21.370 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.9361999999999999e-05 2023-09-23T20:52:21.370 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:21.370 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:21.370 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T20:52:21.370 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:20.839495+0000", 2023-09-23T20:52:21.370 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.4843000000000003e-05 2023-09-23T20:52:21.370 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:52:21.370 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T20:52:21.370 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:52:21.370 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:21.371 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:21.371 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34797.0:3228 6.0 6:045c6caa:::benchmark_data_smithi026_223868_object3227:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e106)", 2023-09-23T20:52:21.371 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T20:52:20.857573+0000", 2023-09-23T20:52:21.371 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 0.38990799599999998, 2023-09-23T20:52:21.371 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0090637440000000003, 2023-09-23T20:52:21.371 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T20:52:21.371 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:52:21.371 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T20:52:21.371 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34797", 2023-09-23T20:52:21.371 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T20:52:21.371 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 3228 2023-09-23T20:52:21.371 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:21.371 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T20:52:21.372 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:21.372 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T20:52:21.372 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:20.857573+0000", 2023-09-23T20:52:21.372 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:52:21.372 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:21.372 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:21.372 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T20:52:21.372 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:20.857573+0000", 2023-09-23T20:52:21.372 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:52:21.372 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:21.372 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:21.373 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T20:52:21.373 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:20.857571+0000", 2023-09-23T20:52:21.373 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T20:52:21.373 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:21.373 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:21.373 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T20:52:21.373 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:20.857596+0000", 2023-09-23T20:52:21.374 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.5154e-05 2023-09-23T20:52:21.374 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:21.374 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:21.374 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T20:52:21.374 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:20.857607+0000", 2023-09-23T20:52:21.374 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0496000000000001e-05 2023-09-23T20:52:21.374 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:21.375 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:21.375 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T20:52:21.375 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:20.857614+0000", 2023-09-23T20:52:21.375 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.5619999999999998e-06 2023-09-23T20:52:21.375 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:21.375 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:21.375 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:52:21.375 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:20.857775+0000", 2023-09-23T20:52:21.375 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00016072600000000001 2023-09-23T20:52:21.375 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:21.375 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:21.375 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T20:52:21.376 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:20.857922+0000", 2023-09-23T20:52:21.376 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00014641 2023-09-23T20:52:21.376 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:21.376 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:21.376 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 2,3", 2023-09-23T20:52:21.376 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:20.858090+0000", 2023-09-23T20:52:21.376 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000168553 2023-09-23T20:52:21.376 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:21.376 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:21.376 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T20:52:21.376 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:20.858724+0000", 2023-09-23T20:52:21.377 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00063435100000000001 2023-09-23T20:52:21.377 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:21.377 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:21.377 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:52:21.377 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:20.862706+0000", 2023-09-23T20:52:21.377 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0039820039999999999 2023-09-23T20:52:21.377 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:21.377 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:21.377 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:52:21.377 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:20.866542+0000", 2023-09-23T20:52:21.377 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0038351150000000001 2023-09-23T20:52:21.377 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:21.377 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:21.378 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T20:52:21.378 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:20.866595+0000", 2023-09-23T20:52:21.378 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.3684999999999998e-05 2023-09-23T20:52:21.378 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:21.378 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:21.378 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T20:52:21.378 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:20.866636+0000", 2023-09-23T20:52:21.378 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.1143000000000002e-05 2023-09-23T20:52:21.378 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:52:21.378 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T20:52:21.378 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:52:21.379 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:21.379 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:21.379 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34797.0:3303 6.0 6:06b818a2:::benchmark_data_smithi026_223868_object3302:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e106)", 2023-09-23T20:52:21.379 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T20:52:20.889523+0000", 2023-09-23T20:52:21.379 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 0.35795743299999999, 2023-09-23T20:52:21.379 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.012446586000000001, 2023-09-23T20:52:21.379 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T20:52:21.379 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:52:21.379 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T20:52:21.379 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34797", 2023-09-23T20:52:21.379 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T20:52:21.379 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 3303 2023-09-23T20:52:21.380 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:21.380 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T20:52:21.380 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:21.380 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T20:52:21.380 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:20.889523+0000", 2023-09-23T20:52:21.380 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:52:21.380 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:21.380 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:21.380 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T20:52:21.380 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:20.889523+0000", 2023-09-23T20:52:21.380 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:52:21.381 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:21.381 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:21.381 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T20:52:21.381 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:20.889521+0000", 2023-09-23T20:52:21.381 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999981 2023-09-23T20:52:21.381 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:21.381 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:21.381 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T20:52:21.381 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:20.889552+0000", 2023-09-23T20:52:21.381 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.1201000000000002e-05 2023-09-23T20:52:21.381 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:21.381 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:21.382 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T20:52:21.382 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:20.889575+0000", 2023-09-23T20:52:21.382 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.2314999999999999e-05 2023-09-23T20:52:21.382 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:21.382 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:21.382 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T20:52:21.382 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:20.889584+0000", 2023-09-23T20:52:21.382 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.0580000000000004e-06 2023-09-23T20:52:21.382 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:21.382 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:21.382 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:52:21.383 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:20.889716+0000", 2023-09-23T20:52:21.383 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000132137 2023-09-23T20:52:21.383 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:21.383 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:21.383 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T20:52:21.383 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:20.889839+0000", 2023-09-23T20:52:21.383 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00012363599999999999 2023-09-23T20:52:21.383 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:21.383 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:21.383 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 2,3", 2023-09-23T20:52:21.383 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:20.889973+0000", 2023-09-23T20:52:21.383 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00013339299999999999 2023-09-23T20:52:21.384 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:21.384 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:21.384 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T20:52:21.384 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:20.890675+0000", 2023-09-23T20:52:21.384 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000702018 2023-09-23T20:52:21.384 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:21.384 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:21.384 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:52:21.384 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:20.898052+0000", 2023-09-23T20:52:21.384 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0073775339999999998 2023-09-23T20:52:21.384 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:21.384 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:21.385 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:52:21.385 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:20.901830+0000", 2023-09-23T20:52:21.385 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.003777163 2023-09-23T20:52:21.385 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:21.385 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:21.385 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T20:52:21.385 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:20.901930+0000", 2023-09-23T20:52:21.385 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000100243 2023-09-23T20:52:21.385 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:21.385 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:21.385 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T20:52:21.385 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:20.901970+0000", 2023-09-23T20:52:21.386 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.9999000000000001e-05 2023-09-23T20:52:21.386 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:52:21.386 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T20:52:21.386 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:52:21.386 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:21.386 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:21.386 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34797.0:3326 6.0 6:05e1a295:::benchmark_data_smithi026_223868_object3325:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e106)", 2023-09-23T20:52:21.386 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T20:52:20.897197+0000", 2023-09-23T20:52:21.386 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 0.35028415400000001, 2023-09-23T20:52:21.386 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0097713509999999993, 2023-09-23T20:52:21.386 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T20:52:21.387 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:52:21.387 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T20:52:21.387 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34797", 2023-09-23T20:52:21.387 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T20:52:21.387 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 3326 2023-09-23T20:52:21.387 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:21.387 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T20:52:21.387 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:21.387 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T20:52:21.387 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:20.897197+0000", 2023-09-23T20:52:21.387 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:52:21.387 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:21.388 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:21.388 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T20:52:21.388 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:20.897197+0000", 2023-09-23T20:52:21.388 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:52:21.388 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:21.388 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:21.388 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T20:52:21.388 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:20.897195+0000", 2023-09-23T20:52:21.388 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T20:52:21.388 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:21.388 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:21.388 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T20:52:21.388 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:20.897221+0000", 2023-09-23T20:52:21.389 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.6276000000000001e-05 2023-09-23T20:52:21.389 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:21.389 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:21.389 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T20:52:21.389 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:20.897243+0000", 2023-09-23T20:52:21.389 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.2035000000000002e-05 2023-09-23T20:52:21.389 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:21.389 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:21.389 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T20:52:21.389 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:20.897253+0000", 2023-09-23T20:52:21.389 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.3489999999999998e-06 2023-09-23T20:52:21.389 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:21.390 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:21.390 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:52:21.390 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:20.897329+0000", 2023-09-23T20:52:21.390 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.6604e-05 2023-09-23T20:52:21.390 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:21.390 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:21.390 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T20:52:21.390 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:20.897496+0000", 2023-09-23T20:52:21.390 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00016664100000000001 2023-09-23T20:52:21.390 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:21.390 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:21.391 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 2,3", 2023-09-23T20:52:21.391 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:20.897633+0000", 2023-09-23T20:52:21.391 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00013693000000000001 2023-09-23T20:52:21.391 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:21.391 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:21.391 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T20:52:21.391 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:20.898880+0000", 2023-09-23T20:52:21.391 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.001247642 2023-09-23T20:52:21.391 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:21.392 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:21.392 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:52:21.392 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:20.899079+0000", 2023-09-23T20:52:21.392 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000198085 2023-09-23T20:52:21.392 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:21.392 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:21.392 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:52:21.393 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:20.906842+0000", 2023-09-23T20:52:21.393 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0077631009999999997 2023-09-23T20:52:21.393 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:21.393 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:21.393 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T20:52:21.393 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:20.906930+0000", 2023-09-23T20:52:21.394 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.8174e-05 2023-09-23T20:52:21.394 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:21.394 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:21.394 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T20:52:21.394 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:20.906968+0000", 2023-09-23T20:52:21.395 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.8062999999999998e-05 2023-09-23T20:52:21.395 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:52:21.395 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T20:52:21.395 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:52:21.395 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:21.395 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:21.395 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34797.0:3423 6.0 6:00454a12:::benchmark_data_smithi026_223868_object3422:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e106)", 2023-09-23T20:52:21.396 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T20:52:20.941069+0000", 2023-09-23T20:52:21.396 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 0.30641184199999999, 2023-09-23T20:52:21.396 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.010196007, 2023-09-23T20:52:21.396 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T20:52:21.396 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:52:21.396 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T20:52:21.397 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34797", 2023-09-23T20:52:21.397 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T20:52:21.397 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 3423 2023-09-23T20:52:21.397 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:21.397 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T20:52:21.397 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:21.397 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T20:52:21.398 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:20.941069+0000", 2023-09-23T20:52:21.398 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:52:21.398 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:21.398 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:21.398 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T20:52:21.398 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:20.941069+0000", 2023-09-23T20:52:21.399 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:52:21.399 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:21.399 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:21.399 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T20:52:21.399 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:20.941067+0000", 2023-09-23T20:52:21.399 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T20:52:21.401 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:21.401 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:21.401 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T20:52:21.401 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:20.941100+0000", 2023-09-23T20:52:21.401 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.2647000000000001e-05 2023-09-23T20:52:21.401 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:21.401 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:21.402 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T20:52:21.402 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:20.941110+0000", 2023-09-23T20:52:21.402 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.6539999999999994e-06 2023-09-23T20:52:21.402 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:21.402 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:21.402 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T20:52:21.402 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:20.941115+0000", 2023-09-23T20:52:21.402 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.3240000000000002e-06 2023-09-23T20:52:21.402 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:21.402 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:21.403 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:52:21.403 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:20.941164+0000", 2023-09-23T20:52:21.403 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.9416000000000003e-05 2023-09-23T20:52:21.403 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:21.403 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:21.403 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T20:52:21.403 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:20.941277+0000", 2023-09-23T20:52:21.403 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000113013 2023-09-23T20:52:21.403 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:21.403 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:21.403 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 2,3", 2023-09-23T20:52:21.403 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:20.941443+0000", 2023-09-23T20:52:21.404 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000165971 2023-09-23T20:52:21.404 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:21.404 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:21.404 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T20:52:21.404 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:20.942947+0000", 2023-09-23T20:52:21.404 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.001503749 2023-09-23T20:52:21.404 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:21.404 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:21.404 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:52:21.404 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:20.945346+0000", 2023-09-23T20:52:21.404 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00239843 2023-09-23T20:52:21.405 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:21.405 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:21.405 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:52:21.405 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:20.951184+0000", 2023-09-23T20:52:21.405 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0058385169999999997 2023-09-23T20:52:21.405 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:21.405 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:21.405 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T20:52:21.405 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:20.951221+0000", 2023-09-23T20:52:21.405 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.7147000000000002e-05 2023-09-23T20:52:21.405 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:21.405 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:21.406 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T20:52:21.406 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:20.951265+0000", 2023-09-23T20:52:21.406 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.3590000000000001e-05 2023-09-23T20:52:21.406 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:52:21.406 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T20:52:21.406 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:52:21.406 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:21.406 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:21.406 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34797.0:3427 6.0 6:0e850f6a:::benchmark_data_smithi026_223868_object3426:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e106)", 2023-09-23T20:52:21.406 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T20:52:20.941126+0000", 2023-09-23T20:52:21.406 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 0.30635444299999998, 2023-09-23T20:52:21.407 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.010241952, 2023-09-23T20:52:21.407 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T20:52:21.407 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:52:21.407 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T20:52:21.407 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34797", 2023-09-23T20:52:21.407 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T20:52:21.407 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 3427 2023-09-23T20:52:21.407 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:21.407 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T20:52:21.407 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:21.407 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T20:52:21.407 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:20.941126+0000", 2023-09-23T20:52:21.408 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:52:21.408 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:21.408 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:21.408 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T20:52:21.408 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:20.941126+0000", 2023-09-23T20:52:21.408 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:52:21.408 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:21.408 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:21.408 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T20:52:21.408 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:20.941126+0000", 2023-09-23T20:52:21.408 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T20:52:21.408 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:21.408 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:21.409 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T20:52:21.409 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:20.941148+0000", 2023-09-23T20:52:21.409 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.2636000000000001e-05 2023-09-23T20:52:21.409 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:21.409 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:21.409 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T20:52:21.409 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:20.941157+0000", 2023-09-23T20:52:21.409 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.4139999999999994e-06 2023-09-23T20:52:21.409 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:21.409 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:21.409 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T20:52:21.409 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:20.941163+0000", 2023-09-23T20:52:21.410 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.353e-06 2023-09-23T20:52:21.410 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:21.410 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:21.410 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:52:21.410 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:20.941910+0000", 2023-09-23T20:52:21.410 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00074695299999999996 2023-09-23T20:52:21.410 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:21.410 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:21.410 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T20:52:21.410 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:20.942024+0000", 2023-09-23T20:52:21.410 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00011394499999999999 2023-09-23T20:52:21.411 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:21.411 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:21.411 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 2,3", 2023-09-23T20:52:21.411 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:20.942179+0000", 2023-09-23T20:52:21.411 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000154597 2023-09-23T20:52:21.411 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:21.411 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:21.411 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T20:52:21.411 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:20.942953+0000", 2023-09-23T20:52:21.411 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00077438500000000005 2023-09-23T20:52:21.411 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:21.412 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:21.412 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:52:21.412 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:20.946719+0000", 2023-09-23T20:52:21.412 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0037657580000000001 2023-09-23T20:52:21.412 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:21.412 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:21.412 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:52:21.412 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:20.951303+0000", 2023-09-23T20:52:21.412 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.004584543 2023-09-23T20:52:21.412 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:21.412 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:21.413 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T20:52:21.413 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:20.951344+0000", 2023-09-23T20:52:21.413 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.1115000000000002e-05 2023-09-23T20:52:21.413 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:21.413 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:21.413 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T20:52:21.413 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:20.951368+0000", 2023-09-23T20:52:21.413 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.3774999999999999e-05 2023-09-23T20:52:21.413 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:52:21.413 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T20:52:21.413 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:52:21.413 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:21.414 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:21.414 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34797.0:3846 6.0 6:0caf5243:::benchmark_data_smithi026_223868_object3845:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e106)", 2023-09-23T20:52:21.414 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T20:52:21.090155+0000", 2023-09-23T20:52:21.414 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 0.15732577, 2023-09-23T20:52:21.414 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.010451031, 2023-09-23T20:52:21.414 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T20:52:21.414 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:52:21.414 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T20:52:21.414 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34797", 2023-09-23T20:52:21.414 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T20:52:21.415 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 3846 2023-09-23T20:52:21.415 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:21.415 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T20:52:21.415 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:21.415 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T20:52:21.415 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:21.090155+0000", 2023-09-23T20:52:21.415 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:52:21.415 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:21.415 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:21.415 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T20:52:21.415 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:21.090155+0000", 2023-09-23T20:52:21.416 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:52:21.416 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:21.416 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:21.416 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T20:52:21.416 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:21.090153+0000", 2023-09-23T20:52:21.416 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T20:52:21.416 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:21.416 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:21.416 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T20:52:21.416 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:21.090177+0000", 2023-09-23T20:52:21.416 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.3917000000000002e-05 2023-09-23T20:52:21.416 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:21.416 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:21.417 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T20:52:21.417 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:21.090204+0000", 2023-09-23T20:52:21.417 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.6486999999999999e-05 2023-09-23T20:52:21.417 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:21.417 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:21.417 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T20:52:21.417 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:21.090214+0000", 2023-09-23T20:52:21.417 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.8600000000000005e-06 2023-09-23T20:52:21.417 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:21.417 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:21.417 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:52:21.417 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:21.091684+0000", 2023-09-23T20:52:21.418 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0014701759999999999 2023-09-23T20:52:21.418 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:21.418 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:21.418 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T20:52:21.418 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:21.091804+0000", 2023-09-23T20:52:21.418 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00012079299999999999 2023-09-23T20:52:21.418 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:21.418 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:21.418 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 2,3", 2023-09-23T20:52:21.418 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:21.091936+0000", 2023-09-23T20:52:21.418 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00013112200000000001 2023-09-23T20:52:21.419 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:21.419 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:21.419 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:52:21.419 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:21.094366+0000", 2023-09-23T20:52:21.419 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0024304130000000002 2023-09-23T20:52:21.419 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:21.419 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:21.419 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T20:52:21.419 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:21.096235+0000", 2023-09-23T20:52:21.419 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.001868895 2023-09-23T20:52:21.419 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:21.419 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:21.420 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:52:21.420 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:21.100507+0000", 2023-09-23T20:52:21.420 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0042723780000000003 2023-09-23T20:52:21.420 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:21.420 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:21.420 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T20:52:21.420 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:21.100561+0000", 2023-09-23T20:52:21.420 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.3418000000000003e-05 2023-09-23T20:52:21.420 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:21.421 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:21.421 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T20:52:21.421 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:21.100606+0000", 2023-09-23T20:52:21.421 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.5221e-05 2023-09-23T20:52:21.421 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:52:21.421 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T20:52:21.421 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:52:21.421 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:21.421 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:21.421 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34797.0:3894 6.0 6:0316d832:::benchmark_data_smithi026_223868_object3893:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e106)", 2023-09-23T20:52:21.421 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T20:52:21.112797+0000", 2023-09-23T20:52:21.421 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 0.13468317399999999, 2023-09-23T20:52:21.422 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0089112080000000003, 2023-09-23T20:52:21.422 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T20:52:21.422 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:52:21.422 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T20:52:21.422 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34797", 2023-09-23T20:52:21.422 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T20:52:21.422 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 3894 2023-09-23T20:52:21.422 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:21.422 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T20:52:21.423 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:21.423 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T20:52:21.423 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:21.112797+0000", 2023-09-23T20:52:21.423 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:52:21.423 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:21.423 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:21.423 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T20:52:21.423 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:21.112797+0000", 2023-09-23T20:52:21.423 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:52:21.423 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:21.423 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:21.423 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T20:52:21.424 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:21.112796+0000", 2023-09-23T20:52:21.424 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T20:52:21.424 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:21.424 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:21.424 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T20:52:21.424 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:21.112834+0000", 2023-09-23T20:52:21.424 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.8250000000000001e-05 2023-09-23T20:52:21.424 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:21.424 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:21.424 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T20:52:21.424 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:21.112857+0000", 2023-09-23T20:52:21.424 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.2548000000000002e-05 2023-09-23T20:52:21.424 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:21.425 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:21.425 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T20:52:21.425 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:21.112870+0000", 2023-09-23T20:52:21.425 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.3521e-05 2023-09-23T20:52:21.425 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:21.425 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:21.425 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:52:21.425 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:21.113396+0000", 2023-09-23T20:52:21.425 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00052532299999999996 2023-09-23T20:52:21.425 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:21.425 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:21.425 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T20:52:21.426 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:21.113525+0000", 2023-09-23T20:52:21.426 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00012930099999999999 2023-09-23T20:52:21.426 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:21.426 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:21.426 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 2,3", 2023-09-23T20:52:21.426 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:21.113667+0000", 2023-09-23T20:52:21.426 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00014191599999999999 2023-09-23T20:52:21.426 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:21.426 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:21.426 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T20:52:21.426 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:21.114728+0000", 2023-09-23T20:52:21.426 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0010614279999999999 2023-09-23T20:52:21.427 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:21.427 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:21.427 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:52:21.427 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:21.118166+0000", 2023-09-23T20:52:21.427 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0034377850000000001 2023-09-23T20:52:21.427 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:21.427 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:21.427 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:52:21.427 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:21.121640+0000", 2023-09-23T20:52:21.427 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0034741279999999999 2023-09-23T20:52:21.427 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:21.427 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:21.428 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T20:52:21.428 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:21.121672+0000", 2023-09-23T20:52:21.428 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.1687999999999999e-05 2023-09-23T20:52:21.428 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:21.428 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:21.428 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T20:52:21.428 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:21.121709+0000", 2023-09-23T20:52:21.428 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.6934000000000002e-05 2023-09-23T20:52:21.428 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:52:21.428 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T20:52:21.428 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:52:21.428 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:52:21.428 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T20:52:21.429 INFO:teuthology.orchestra.run.smithi026.stdout:} 2023-09-23T20:52:21.429 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T20:52:21.431 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:52:20 smithi070 ceph-mon[168282]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:52:21.431 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:52:20 smithi070 ceph-mon[168282]: failed to encode map e106 with expected crc 2023-09-23T20:52:21.431 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:52:20 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T20:52:21.431 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:52:20 smithi070 ceph-mon[168282]: osdmap e106: 8 total, 8 up, 7 in 2023-09-23T20:52:21.431 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:52:20 smithi070 ceph-mon[168282]: failed to encode map e106 with expected crc 2023-09-23T20:52:21.431 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:52:20 smithi070 ceph-mon[168282]: pgmap v242: 89 pgs: 89 active+clean; 582 KiB data, 154 MiB used, 626 GiB / 626 GiB avail; 117 B/s, 1 objects/s recovering 2023-09-23T20:52:21.432 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:52:21.432 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:52:21.432 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:52:21.432 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:52:21.534 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:52:21.673 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_ops_in_flight 2023-09-23T20:52:21.761 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:52:21.827 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 2 16 5613 5597 174.883 153.312 0.00429431 0.00564119 2023-09-23T20:52:21.828 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:52:21.832 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:52:21.832 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:52:21.933 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:52:21.945 INFO:teuthology.orchestra.run.smithi026.stderr:reweighted osd.0 to 1 (10000) 2023-09-23T20:52:21.977 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:52:21 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/2001835469' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-09-23T20:52:21.977 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:52:21 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-09-23T20:52:21.978 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:52:21 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/2001835469' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-09-23T20:52:21.978 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:52:21 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-09-23T20:52:22.081 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:52:22.343 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:52:22.349 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:52:22.350 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:52:22.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:52:21 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/2001835469' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-09-23T20:52:22.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:52:21 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-09-23T20:52:22.450 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:52:22.506 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:52:22.554 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph osd reweight 1 1 2023-09-23T20:52:22.603 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:52:22.607 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:52:22.608 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:52:22.633 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:52:22.709 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:52:22.709 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:52:22.738 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:52:22.778 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:52:22.782 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:52:22.782 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:52:22.829 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 3 16 7918 7902 164.529 144.062 0.00422363 0.00606633 2023-09-23T20:52:22.843 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T20:52:22.883 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:52:22.965 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:52:23.055 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:52:23.065 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:52:23.065 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:52:23.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:52:22 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-09-23T20:52:23.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:52:22 smithi070 ceph-mon[168282]: osdmap e107: 8 total, 8 up, 7 in 2023-09-23T20:52:23.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:52:22 smithi070 ceph-mon[168282]: failed to encode map e107 with expected crc 2023-09-23T20:52:23.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:52:22 smithi070 ceph-mon[168282]: failed to encode map e107 with expected crc 2023-09-23T20:52:23.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:52:22 smithi070 ceph-mon[168282]: failed to encode map e107 with expected crc 2023-09-23T20:52:23.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:52:22 smithi070 ceph-mon[168282]: pgmap v244: 89 pgs: 89 active+clean; 180 MiB data, 639 MiB used, 625 GiB / 626 GiB avail; 30 MiB/s wr, 478 op/s; 11 B/s, 1 objects/s recovering 2023-09-23T20:52:23.166 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:52:23.182 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:52:22 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-09-23T20:52:23.182 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:52:22 smithi026 ceph-mon[175151]: osdmap e107: 8 total, 8 up, 7 in 2023-09-23T20:52:23.182 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:52:22 smithi026 ceph-mon[175151]: failed to encode map e107 with expected crc 2023-09-23T20:52:23.182 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:52:22 smithi026 ceph-mon[175151]: failed to encode map e107 with expected crc 2023-09-23T20:52:23.182 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:52:22 smithi026 ceph-mon[175151]: failed to encode map e107 with expected crc 2023-09-23T20:52:23.182 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:52:22 smithi026 ceph-mon[175151]: pgmap v244: 89 pgs: 89 active+clean; 180 MiB data, 639 MiB used, 625 GiB / 626 GiB avail; 30 MiB/s wr, 478 op/s; 11 B/s, 1 objects/s recovering 2023-09-23T20:52:23.183 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:52:22 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-09-23T20:52:23.184 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:52:22 smithi026 ceph-mon[179688]: osdmap e107: 8 total, 8 up, 7 in 2023-09-23T20:52:23.184 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:52:22 smithi026 ceph-mon[179688]: failed to encode map e107 with expected crc 2023-09-23T20:52:23.184 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:52:22 smithi026 ceph-mon[179688]: failed to encode map e107 with expected crc 2023-09-23T20:52:23.184 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:52:22 smithi026 ceph-mon[179688]: failed to encode map e107 with expected crc 2023-09-23T20:52:23.184 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:52:22 smithi026 ceph-mon[179688]: pgmap v244: 89 pgs: 89 active+clean; 180 MiB data, 639 MiB used, 625 GiB / 626 GiB avail; 30 MiB/s wr, 478 op/s; 11 B/s, 1 objects/s recovering 2023-09-23T20:52:23.248 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:52:23.362 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:52:23.367 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:52:23.367 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:52:23.468 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:52:23.504 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:52:23.550 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:52:23.553 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:52:23.553 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:52:23.654 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:52:23.746 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:52:23.829 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 4 16 10114 10098 157.709 137.25 0.00932137 0.00633229 2023-09-23T20:52:24.146 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:52:24.157 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:52:24.158 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:52:24.196 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:52:24.244 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:52:23 smithi026 ceph-mon[179688]: failed to encode map e107 with expected crc 2023-09-23T20:52:24.244 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:52:23 smithi026 ceph-mon[175151]: failed to encode map e107 with expected crc 2023-09-23T20:52:24.259 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:52:24.290 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:52:23 smithi070 ceph-mon[168282]: failed to encode map e107 with expected crc 2023-09-23T20:52:24.292 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:52:24.339 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:52:24.342 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:52:24.342 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:52:24.443 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:52:24.510 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:52:24.527 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:52:24.626 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0: {} 2023-09-23T20:52:24.669 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1: {} 2023-09-23T20:52:24.718 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2: {} 2023-09-23T20:52:24.769 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3: {} 2023-09-23T20:52:24.813 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4: {} 2023-09-23T20:52:24.829 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 5 16 12349 12333 154.096 139.688 0.0100357 0.00648329 2023-09-23T20:52:24.850 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:52:24.857 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T20:52:24.858 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:52:24.858 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:52:24.905 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6: {} 2023-09-23T20:52:24.948 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T20:52:24.952 INFO:teuthology.orchestra.run.smithi026.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T20:52:24.952 INFO:teuthology.orchestra.run.smithi026.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T20:52:24.952 INFO:teuthology.orchestra.run.smithi026.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T20:52:24.952 INFO:teuthology.orchestra.run.smithi026.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T20:52:24.953 INFO:teuthology.orchestra.run.smithi026.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T20:52:24.953 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T20:52:24.953 INFO:teuthology.orchestra.run.smithi026.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T20:52:24.953 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T20:52:24.959 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:52:24.982 INFO:teuthology.orchestra.run.smithi026.stdout:ERROR: (22) Invalid argument 2023-09-23T20:52:24.982 INFO:teuthology.orchestra.run.smithi026.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:52:24.988 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:52:25.027 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:52:25.030 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:52:25.030 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:52:25.055 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:52:24 smithi026 ceph-mon[179688]: pgmap v245: 89 pgs: 89 active+clean; 459 MiB data, 1.2 GiB used, 625 GiB / 626 GiB avail; 73 MiB/s wr, 1.16k op/s 2023-09-23T20:52:25.055 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:52:24 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/2796617235' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:52:25.055 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:52:24 smithi026 ceph-mon[175151]: pgmap v245: 89 pgs: 89 active+clean; 459 MiB data, 1.2 GiB used, 625 GiB / 626 GiB avail; 73 MiB/s wr, 1.16k op/s 2023-09-23T20:52:25.055 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:52:24 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/2796617235' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:52:25.131 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:52:25.196 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:52:25.266 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:52:25.272 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:52:25.273 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:52:25.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:52:24 smithi070 ceph-mon[168282]: pgmap v245: 89 pgs: 89 active+clean; 459 MiB data, 1.2 GiB used, 625 GiB / 626 GiB avail; 73 MiB/s wr, 1.16k op/s 2023-09-23T20:52:25.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:52:24 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/2796617235' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:52:25.374 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:52:25.442 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:52:25.608 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:52:25.613 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:52:25.614 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:52:25.715 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:52:25.772 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:52:25.830 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 6 16 14614 14598 151.996 141.562 0.00723372 0.00657168 2023-09-23T20:52:26.000 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_blocked_ops 2023-09-23T20:52:26.369 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:52:26.374 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:52:26.374 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:52:26.476 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:52:26.540 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:52:26.619 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:52:26.633 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:52:26.633 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:52:26.734 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:52:26.767 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:52:26.807 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:52:26.810 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:52:26.811 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:52:26.834 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 7 16 16820 16804 149.876 137.875 0.00647939 0.0066649 2023-09-23T20:52:26.912 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:52:26.947 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:52:26.990 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:52:26.993 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:52:26.994 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:52:27.094 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:52:27.124 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:52:27.163 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:52:27.166 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:52:27.166 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:52:27.267 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:52:27.298 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:52:27.339 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:52:27.342 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:52:27.343 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:52:27.365 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:52:27.386 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:52:27 smithi026 ceph-mon[175151]: pgmap v246: 89 pgs: 89 active+clean; 595 MiB data, 1.8 GiB used, 624 GiB / 626 GiB avail; 74 MiB/s wr, 1.19k op/s 2023-09-23T20:52:27.386 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:52:27 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:52:27.386 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:52:27 smithi026 ceph-mon[179688]: pgmap v246: 89 pgs: 89 active+clean; 595 MiB data, 1.8 GiB used, 624 GiB / 626 GiB avail; 74 MiB/s wr, 1.19k op/s 2023-09-23T20:52:27.386 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:52:27 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:52:27.443 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:52:27.470 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:52:27 smithi070 ceph-mon[168282]: pgmap v246: 89 pgs: 89 active+clean; 595 MiB data, 1.8 GiB used, 624 GiB / 626 GiB avail; 74 MiB/s wr, 1.19k op/s 2023-09-23T20:52:27.470 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:52:27 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:52:27.474 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:52:27.509 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:52:27.512 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:52:27.512 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:52:27.613 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:52:27.648 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:52:27.693 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:52:27.696 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:52:27.696 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:52:27.797 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:52:27.828 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:52:27.836 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 8 16 18848 18832 146.946 126.75 0.0106104 0.00679987 2023-09-23T20:52:27.877 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:52:27.880 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:52:27.880 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:52:27.981 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:52:28.046 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:52:28.126 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:52:28.134 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:52:28.134 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:52:28.235 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:52:28.250 INFO:teuthology.orchestra.run.smithi026.stderr:noscrub is unset 2023-09-23T20:52:28.264 INFO:teuthology.orchestra.run.smithi026.stderr:reweighted osd.1 to 1 (10000) 2023-09-23T20:52:28.267 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:52:28.293 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:52:28 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/1963240950' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-09-23T20:52:28.293 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:52:28 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/3437673942' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:52:28.293 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:52:28 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:52:28.294 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:52:28 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/1963240950' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-09-23T20:52:28.294 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:52:28 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/3437673942' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:52:28.294 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:52:28 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:52:28.308 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:52:28.312 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:52:28.313 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:52:28.414 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:52:28.451 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:52:28.465 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T20:52:28.489 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:52:28.492 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:52:28.493 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:52:28.493 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:52:28 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/1963240950' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-09-23T20:52:28.493 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:52:28 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/3437673942' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:52:28.493 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:52:28 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:52:28.594 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:52:28.625 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:52:28.668 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:52:28.671 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:52:28.672 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:52:28.772 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:52:28.837 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 9 16 20883 20867 144.747 127.188 0.00950227 0.00689891 2023-09-23T20:52:28.848 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:52:29.278 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:52:29.285 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:52:29.286 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:52:29.386 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:52:29.387 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:52:29 smithi026 ceph-mon[179688]: pgmap v247: 89 pgs: 89 active+clean; 860 MiB data, 2.6 GiB used, 623 GiB / 626 GiB avail; 104 MiB/s wr, 1.66k op/s 2023-09-23T20:52:29.387 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:52:29 smithi026 ceph-mon[179688]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:52:29.387 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:52:29 smithi026 ceph-mon[179688]: failed to encode map e108 with expected crc 2023-09-23T20:52:29.387 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:52:29 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/1963240950' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-09-23T20:52:29.388 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:52:29 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T20:52:29.388 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:52:29 smithi026 ceph-mon[179688]: osdmap e108: 8 total, 8 up, 7 in 2023-09-23T20:52:29.388 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:52:29 smithi026 ceph-mon[179688]: failed to encode map e108 with expected crc 2023-09-23T20:52:29.388 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:52:29 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:52:29.390 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph osd reweight 2 1 2023-09-23T20:52:29.390 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:52:29 smithi026 ceph-mon[175151]: pgmap v247: 89 pgs: 89 active+clean; 860 MiB data, 2.6 GiB used, 623 GiB / 626 GiB avail; 104 MiB/s wr, 1.66k op/s 2023-09-23T20:52:29.390 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:52:29 smithi026 ceph-mon[175151]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:52:29.390 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:52:29 smithi026 ceph-mon[175151]: failed to encode map e108 with expected crc 2023-09-23T20:52:29.390 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:52:29 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/1963240950' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-09-23T20:52:29.390 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:52:29 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T20:52:29.391 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:52:29 smithi026 ceph-mon[175151]: osdmap e108: 8 total, 8 up, 7 in 2023-09-23T20:52:29.391 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:52:29 smithi026 ceph-mon[175151]: failed to encode map e108 with expected crc 2023-09-23T20:52:29.391 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:52:29 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:52:29.477 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:52:29.552 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:52:29.556 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:52:29.557 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:52:29.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:52:29 smithi070 ceph-mon[168282]: pgmap v247: 89 pgs: 89 active+clean; 860 MiB data, 2.6 GiB used, 623 GiB / 626 GiB avail; 104 MiB/s wr, 1.66k op/s 2023-09-23T20:52:29.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:52:29 smithi070 ceph-mon[168282]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:52:29.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:52:29 smithi070 ceph-mon[168282]: failed to encode map e108 with expected crc 2023-09-23T20:52:29.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:52:29 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/1963240950' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-09-23T20:52:29.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:52:29 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T20:52:29.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:52:29 smithi070 ceph-mon[168282]: osdmap e108: 8 total, 8 up, 7 in 2023-09-23T20:52:29.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:52:29 smithi070 ceph-mon[168282]: failed to encode map e108 with expected crc 2023-09-23T20:52:29.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:52:29 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:52:29.638 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:52:29.658 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:52:29.669 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:52:29 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[164029]: ::ffff:172.21.15.70 - - [23/Sep/2023:20:52:29] "GET /metrics HTTP/1.1" 200 36245 "" "Prometheus/2.43.0" 2023-09-23T20:52:29.775 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:52:29.840 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 10 16 22541 22525 140.605 103.625 0.0113579 0.00710511 2023-09-23T20:52:29.916 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:52:29.921 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:52:29.921 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:52:30.022 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:52:30.034 INFO:teuthology.orchestra.run.smithi026.stdout:ERROR: (22) Invalid argument 2023-09-23T20:52:30.034 INFO:teuthology.orchestra.run.smithi026.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:52:30.109 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:52:30.334 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:52:30.337 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:52:30.338 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:52:30.439 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:52:30.484 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:52:30.529 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:52:30.534 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:52:30.535 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:52:30.636 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:52:30.678 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:52:30.699 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_historic_ops 2023-09-23T20:52:30.718 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:52:30.721 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:52:30.721 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:52:30.822 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:52:30.842 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 11 16 24365 24349 138.162 114 0.0095697 0.00723085 2023-09-23T20:52:30.877 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:52:30.886 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:52:30 smithi026 ceph-mon[175151]: failed to encode map e108 with expected crc 2023-09-23T20:52:30.886 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:52:30 smithi026 ceph-mon[179688]: failed to encode map e108 with expected crc 2023-09-23T20:52:30.953 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:52:31.085 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:52:31.097 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:52:31.098 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:52:31.116 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T20:52:31.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:52:30 smithi070 ceph-mon[168282]: failed to encode map e108 with expected crc 2023-09-23T20:52:31.198 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:52:31.304 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:52:31.424 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:52:31.428 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:52:31.429 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:52:31.529 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:52:31.663 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:52:31.822 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:52:31 smithi026 ceph-mon[175151]: pgmap v249: 89 pgs: 89 active+clean; 1.1 GiB data, 3.2 GiB used, 623 GiB / 626 GiB avail; 134 MiB/s wr, 2.15k op/s 2023-09-23T20:52:31.822 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:52:31 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:52:31.822 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:52:31 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:52:31.824 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:52:31 smithi026 ceph-mon[179688]: pgmap v249: 89 pgs: 89 active+clean; 1.1 GiB data, 3.2 GiB used, 623 GiB / 626 GiB avail; 134 MiB/s wr, 2.15k op/s 2023-09-23T20:52:31.824 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:52:31 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:52:31.824 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:52:31 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:52:31.842 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 12 16 25818 25802 134.22 90.8125 0.0065688 0.0074451 2023-09-23T20:52:32.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:52:31 smithi070 ceph-mon[168282]: pgmap v249: 89 pgs: 89 active+clean; 1.1 GiB data, 3.2 GiB used, 623 GiB / 626 GiB avail; 134 MiB/s wr, 2.15k op/s 2023-09-23T20:52:32.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:52:31 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:52:32.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:52:31 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:52:32.141 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:52:32.146 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:52:32.146 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:52:32.247 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:52:32.351 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:52:32.388 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0: {} 2023-09-23T20:52:32.425 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1: {} 2023-09-23T20:52:32.464 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2: {} 2023-09-23T20:52:32.510 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3: {} 2023-09-23T20:52:32.519 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:52:32.522 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:52:32.523 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:52:32.526 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:52:32.549 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:52:32.550 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4: {} 2023-09-23T20:52:32.585 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T20:52:32.623 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:52:32.626 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6: {} 2023-09-23T20:52:32.659 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:52:32.673 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T20:52:32.683 INFO:teuthology.orchestra.run.smithi026.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T20:52:32.683 INFO:teuthology.orchestra.run.smithi026.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T20:52:32.683 INFO:teuthology.orchestra.run.smithi026.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T20:52:32.683 INFO:teuthology.orchestra.run.smithi026.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T20:52:32.683 INFO:teuthology.orchestra.run.smithi026.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T20:52:32.683 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T20:52:32.683 INFO:teuthology.orchestra.run.smithi026.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T20:52:32.683 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T20:52:32.702 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:52:32.705 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:52:32.705 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:52:32.806 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:52:32.835 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:52:32 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/1532650771' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:52:32.849 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 13 16 27372 27356 131.347 97.125 0.0204994 0.00760391 2023-09-23T20:52:32.907 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:52:33.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:52:32 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/1532650771' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:52:33.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:52:32 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/1532650771' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:52:33.844 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 14 16 29185 29169 130.059 113.312 0.00984592 0.00768114 2023-09-23T20:52:33.860 INFO:teuthology.orchestra.run.smithi026.stderr:reweighted osd.2 to 1 (10000) 2023-09-23T20:52:33.983 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:52:33 smithi026 ceph-mon[179688]: pgmap v250: 89 pgs: 89 active+clean; 1.4 GiB data, 4.4 GiB used, 621 GiB / 626 GiB avail; 130 MiB/s wr, 2.08k op/s 2023-09-23T20:52:33.983 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:52:33 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/3819949141' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T20:52:33.983 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:52:33 smithi026 ceph-mon[175151]: pgmap v250: 89 pgs: 89 active+clean; 1.4 GiB data, 4.4 GiB used, 621 GiB / 626 GiB avail; 130 MiB/s wr, 2.08k op/s 2023-09-23T20:52:33.983 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:52:33 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/3819949141' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T20:52:34.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:52:33 smithi070 ceph-mon[168282]: pgmap v250: 89 pgs: 89 active+clean; 1.4 GiB data, 4.4 GiB used, 621 GiB / 626 GiB avail; 130 MiB/s wr, 2.08k op/s 2023-09-23T20:52:34.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:52:33 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/3819949141' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T20:52:34.143 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:52:34.153 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:52:34.154 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:52:34.254 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:52:34.317 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:52:34.412 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:52:34.424 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:52:34.425 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:52:34.525 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:52:34.605 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:52:34.844 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 15 16 30886 30870 128.477 106.312 0.00903095 0.0077775 2023-09-23T20:52:35.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:52:34 smithi070 ceph-mon[168282]: failed to encode map e109 with expected crc 2023-09-23T20:52:35.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:52:34 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/3819949141' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-09-23T20:52:35.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:52:34 smithi070 ceph-mon[168282]: osdmap e109: 8 total, 8 up, 7 in 2023-09-23T20:52:35.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:52:34 smithi070 ceph-mon[168282]: failed to encode map e109 with expected crc 2023-09-23T20:52:35.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:52:34 smithi070 ceph-mon[168282]: failed to encode map e109 with expected crc 2023-09-23T20:52:35.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:52:34 smithi070 ceph-mon[168282]: failed to encode map e109 with expected crc 2023-09-23T20:52:35.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:52:34 smithi070 ceph-mon[168282]: failed to encode map e109 with expected crc 2023-09-23T20:52:35.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:52:34 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:52:35.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:52:34 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:52:35.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:52:34 smithi026 ceph-mon[179688]: failed to encode map e109 with expected crc 2023-09-23T20:52:35.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:52:34 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/3819949141' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-09-23T20:52:35.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:52:34 smithi026 ceph-mon[179688]: osdmap e109: 8 total, 8 up, 7 in 2023-09-23T20:52:35.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:52:34 smithi026 ceph-mon[179688]: failed to encode map e109 with expected crc 2023-09-23T20:52:35.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:52:34 smithi026 ceph-mon[179688]: failed to encode map e109 with expected crc 2023-09-23T20:52:35.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:52:34 smithi026 ceph-mon[179688]: failed to encode map e109 with expected crc 2023-09-23T20:52:35.134 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:52:34 smithi026 ceph-mon[179688]: failed to encode map e109 with expected crc 2023-09-23T20:52:35.134 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:52:34 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:52:35.134 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:52:34 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:52:35.135 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:52:34 smithi026 ceph-mon[175151]: failed to encode map e109 with expected crc 2023-09-23T20:52:35.135 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:52:34 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/3819949141' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-09-23T20:52:35.135 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:52:34 smithi026 ceph-mon[175151]: osdmap e109: 8 total, 8 up, 7 in 2023-09-23T20:52:35.135 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:52:34 smithi026 ceph-mon[175151]: failed to encode map e109 with expected crc 2023-09-23T20:52:35.135 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:52:34 smithi026 ceph-mon[175151]: failed to encode map e109 with expected crc 2023-09-23T20:52:35.135 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:52:34 smithi026 ceph-mon[175151]: failed to encode map e109 with expected crc 2023-09-23T20:52:35.135 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:52:34 smithi026 ceph-mon[175151]: failed to encode map e109 with expected crc 2023-09-23T20:52:35.135 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:52:34 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:52:35.135 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:52:34 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:52:35.269 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph osd reweight 3 1 2023-09-23T20:52:35.366 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:52:35.369 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:52:35.369 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:52:35.470 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:52:35.586 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:52:35.727 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:52:35.739 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:52:35.739 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:52:35.840 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:52:35.844 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 16 16 32366 32350 126.23 92.5 0.00486379 0.00791658 2023-09-23T20:52:35.875 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:52:35.916 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:52:35.919 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:52:35.920 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:52:36.020 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:52:36.052 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:52:36.094 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:52:36.098 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:52:36.099 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:52:36.115 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:52:35 smithi026 ceph-mon[175151]: pgmap v252: 89 pgs: 89 active+clean; 1.7 GiB data, 4.9 GiB used, 621 GiB / 626 GiB avail; 138 MiB/s wr, 2.21k op/s 2023-09-23T20:52:36.116 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:52:35 smithi026 ceph-mon[179688]: pgmap v252: 89 pgs: 89 active+clean; 1.7 GiB data, 4.9 GiB used, 621 GiB / 626 GiB avail; 138 MiB/s wr, 2.21k op/s 2023-09-23T20:52:36.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:52:35 smithi070 ceph-mon[168282]: pgmap v252: 89 pgs: 89 active+clean; 1.7 GiB data, 4.9 GiB used, 621 GiB / 626 GiB avail; 138 MiB/s wr, 2.21k op/s 2023-09-23T20:52:36.199 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:52:36.231 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:52:36.278 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:52:36.281 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:52:36.282 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:52:36.380 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T20:52:36.382 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:52:36.416 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:52:36.455 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:52:36.457 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:52:36.457 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:52:36.558 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:52:36.637 INFO:teuthology.orchestra.run.smithi026.stdout:{ 2023-09-23T20:52:36.637 INFO:teuthology.orchestra.run.smithi026.stdout: "size": 20, 2023-09-23T20:52:36.637 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 600, 2023-09-23T20:52:36.637 INFO:teuthology.orchestra.run.smithi026.stdout: "ops": [ 2023-09-23T20:52:36.637 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:36.637 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34797.0:8396 6.d 6:b8497979:::benchmark_data_smithi026_223868_object8395:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e107)", 2023-09-23T20:52:36.638 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T20:52:23.081486+0000", 2023-09-23T20:52:36.638 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 13.548262121, 2023-09-23T20:52:36.638 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.026714778000000002, 2023-09-23T20:52:36.638 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T20:52:36.638 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:52:36.638 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T20:52:36.638 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34797", 2023-09-23T20:52:36.638 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T20:52:36.638 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 8396 2023-09-23T20:52:36.638 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:36.638 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T20:52:36.639 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:36.639 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T20:52:36.639 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:23.081486+0000", 2023-09-23T20:52:36.639 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:52:36.639 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:36.639 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:36.639 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T20:52:36.639 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:23.081486+0000", 2023-09-23T20:52:36.639 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:52:36.639 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:36.640 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:36.640 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T20:52:36.640 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:23.081484+0000", 2023-09-23T20:52:36.640 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T20:52:36.640 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:36.640 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:36.640 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T20:52:36.640 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:23.081507+0000", 2023-09-23T20:52:36.640 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.2374999999999998e-05 2023-09-23T20:52:36.640 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:36.640 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:36.640 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T20:52:36.641 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:23.081523+0000", 2023-09-23T20:52:36.641 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.6084999999999999e-05 2023-09-23T20:52:36.641 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:36.641 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:36.641 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T20:52:36.641 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:23.081531+0000", 2023-09-23T20:52:36.641 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.5350000000000004e-06 2023-09-23T20:52:36.641 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:36.641 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:36.641 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:52:36.641 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:23.081581+0000", 2023-09-23T20:52:36.641 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.9910000000000002e-05 2023-09-23T20:52:36.642 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:36.642 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:36.642 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T20:52:36.642 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:23.081740+0000", 2023-09-23T20:52:36.642 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00015884300000000001 2023-09-23T20:52:36.642 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:36.642 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:36.642 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 0,2", 2023-09-23T20:52:36.642 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:23.081884+0000", 2023-09-23T20:52:36.642 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000143473 2023-09-23T20:52:36.642 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:36.642 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:36.643 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T20:52:36.643 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:23.082668+0000", 2023-09-23T20:52:36.643 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00078388199999999996 2023-09-23T20:52:36.643 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:36.643 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:36.643 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:52:36.643 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:23.084990+0000", 2023-09-23T20:52:36.643 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0023225659999999999 2023-09-23T20:52:36.643 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:36.643 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:36.643 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:52:36.643 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:23.108130+0000", 2023-09-23T20:52:36.643 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.023139794000000002 2023-09-23T20:52:36.644 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:36.644 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:36.644 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T20:52:36.644 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:23.108165+0000", 2023-09-23T20:52:36.644 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.5135000000000002e-05 2023-09-23T20:52:36.644 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:36.644 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:36.644 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T20:52:36.644 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:23.108201+0000", 2023-09-23T20:52:36.644 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.5679999999999997e-05 2023-09-23T20:52:36.644 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:52:36.644 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T20:52:36.645 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:52:36.645 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:36.645 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:36.645 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_repop(client.34797.0:27342 6.e e108/102 6:7ebfe229:::benchmark_data_smithi026_223868_object27341:head v 108'1760, mlcod=108'1756)", 2023-09-23T20:52:36.645 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T20:52:32.816428+0000", 2023-09-23T20:52:36.645 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 3.8133200390000002, 2023-09-23T20:52:36.645 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.020671920999999999, 2023-09-23T20:52:36.645 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T20:52:36.645 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:52:36.645 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T20:52:36.645 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:36.645 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T20:52:36.646 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:32.816428+0000", 2023-09-23T20:52:36.646 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:52:36.646 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:36.646 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:36.646 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T20:52:36.646 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:32.816428+0000", 2023-09-23T20:52:36.646 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:52:36.646 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:36.646 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:36.646 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T20:52:36.646 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:32.816427+0000", 2023-09-23T20:52:36.646 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T20:52:36.647 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:36.647 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:36.647 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T20:52:36.647 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:32.816467+0000", 2023-09-23T20:52:36.647 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.9894000000000001e-05 2023-09-23T20:52:36.647 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:36.647 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:36.647 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T20:52:36.647 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:32.816487+0000", 2023-09-23T20:52:36.647 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.9259e-05 2023-09-23T20:52:36.647 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:36.647 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:36.648 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T20:52:36.648 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:32.816495+0000", 2023-09-23T20:52:36.648 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.4849999999999992e-06 2023-09-23T20:52:36.648 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:36.648 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:36.648 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:52:36.648 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:32.817599+0000", 2023-09-23T20:52:36.648 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.001104484 2023-09-23T20:52:36.648 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:36.648 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:36.648 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T20:52:36.648 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:32.817616+0000", 2023-09-23T20:52:36.648 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.6901999999999998e-05 2023-09-23T20:52:36.649 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:36.649 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:36.649 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T20:52:36.649 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:32.837052+0000", 2023-09-23T20:52:36.649 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.019435759 2023-09-23T20:52:36.649 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:36.649 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:36.649 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T20:52:36.649 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:32.837100+0000", 2023-09-23T20:52:36.649 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.7787999999999997e-05 2023-09-23T20:52:36.649 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:52:36.649 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T20:52:36.650 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:52:36.650 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:36.650 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:36.650 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34797.0:27357 6.4 6:290f41d7:::benchmark_data_smithi026_223868_object27356:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e108)", 2023-09-23T20:52:36.650 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T20:52:32.826184+0000", 2023-09-23T20:52:36.650 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 3.803564546, 2023-09-23T20:52:36.650 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.020895540000000001, 2023-09-23T20:52:36.650 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T20:52:36.650 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:52:36.650 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T20:52:36.650 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34797", 2023-09-23T20:52:36.651 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T20:52:36.651 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 27357 2023-09-23T20:52:36.651 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:36.651 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T20:52:36.651 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:36.651 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T20:52:36.651 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:32.826184+0000", 2023-09-23T20:52:36.651 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:52:36.651 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:36.651 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:36.651 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T20:52:36.651 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:32.826184+0000", 2023-09-23T20:52:36.652 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:52:36.652 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:36.652 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:36.652 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T20:52:36.652 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:32.826183+0000", 2023-09-23T20:52:36.652 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T20:52:36.652 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:36.652 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:36.652 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T20:52:36.652 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:32.826205+0000", 2023-09-23T20:52:36.652 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.2354e-05 2023-09-23T20:52:36.652 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:36.652 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:36.653 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T20:52:36.653 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:32.826213+0000", 2023-09-23T20:52:36.653 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.1169999999999998e-06 2023-09-23T20:52:36.653 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:36.653 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:36.653 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T20:52:36.653 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:32.826218+0000", 2023-09-23T20:52:36.653 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.485e-06 2023-09-23T20:52:36.653 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:36.653 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:36.653 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:52:36.653 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:32.826268+0000", 2023-09-23T20:52:36.654 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.0260999999999999e-05 2023-09-23T20:52:36.654 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:36.654 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:36.654 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T20:52:36.654 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:32.826412+0000", 2023-09-23T20:52:36.654 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00014357899999999999 2023-09-23T20:52:36.654 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:36.654 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:36.654 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 3,6", 2023-09-23T20:52:36.654 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:32.826572+0000", 2023-09-23T20:52:36.655 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00016051300000000001 2023-09-23T20:52:36.655 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:36.655 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:36.655 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:52:36.655 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:32.830935+0000", 2023-09-23T20:52:36.655 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0043624680000000004 2023-09-23T20:52:36.655 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:36.655 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:36.655 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T20:52:36.655 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:32.835714+0000", 2023-09-23T20:52:36.655 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0047791960000000003 2023-09-23T20:52:36.655 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:36.656 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:36.656 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:52:36.656 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:32.846988+0000", 2023-09-23T20:52:36.656 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.011274304000000001 2023-09-23T20:52:36.656 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:36.656 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:36.656 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T20:52:36.656 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:32.847043+0000", 2023-09-23T20:52:36.656 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.4481000000000002e-05 2023-09-23T20:52:36.656 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:36.656 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:36.656 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T20:52:36.657 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:32.847079+0000", 2023-09-23T20:52:36.657 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.6266999999999997e-05 2023-09-23T20:52:36.657 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:52:36.657 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T20:52:36.657 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:52:36.657 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:36.657 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:36.657 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34797.0:27358 6.4 6:20c8109c:::benchmark_data_smithi026_223868_object27357:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e108)", 2023-09-23T20:52:36.657 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T20:52:32.826231+0000", 2023-09-23T20:52:36.657 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 3.8035172930000001, 2023-09-23T20:52:36.657 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.02094594, 2023-09-23T20:52:36.657 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T20:52:36.657 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:52:36.657 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T20:52:36.658 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34797", 2023-09-23T20:52:36.658 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T20:52:36.658 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 27358 2023-09-23T20:52:36.658 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:36.658 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T20:52:36.658 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:36.658 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T20:52:36.658 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:32.826231+0000", 2023-09-23T20:52:36.658 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:52:36.658 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:36.658 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:36.658 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T20:52:36.659 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:32.826231+0000", 2023-09-23T20:52:36.659 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:52:36.659 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:36.659 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:36.659 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T20:52:36.659 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:32.826230+0000", 2023-09-23T20:52:36.659 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T20:52:36.659 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:36.659 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:36.659 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T20:52:36.659 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:32.826252+0000", 2023-09-23T20:52:36.660 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.1798999999999999e-05 2023-09-23T20:52:36.660 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:36.660 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:36.660 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T20:52:36.660 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:32.826259+0000", 2023-09-23T20:52:36.660 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.3400000000000003e-06 2023-09-23T20:52:36.660 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:36.660 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:36.660 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T20:52:36.660 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:32.826264+0000", 2023-09-23T20:52:36.660 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.3120000000000001e-06 2023-09-23T20:52:36.660 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:36.660 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:36.661 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:52:36.661 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:32.829427+0000", 2023-09-23T20:52:36.661 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0031627640000000002 2023-09-23T20:52:36.661 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:36.661 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:36.661 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T20:52:36.661 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:32.829562+0000", 2023-09-23T20:52:36.661 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00013556899999999999 2023-09-23T20:52:36.661 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:36.661 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:36.661 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 3,6", 2023-09-23T20:52:36.661 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:32.829694+0000", 2023-09-23T20:52:36.662 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000131719 2023-09-23T20:52:36.662 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:36.662 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:36.662 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:52:36.662 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:32.835465+0000", 2023-09-23T20:52:36.662 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.005771254 2023-09-23T20:52:36.662 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:36.662 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:36.662 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T20:52:36.663 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:32.835719+0000", 2023-09-23T20:52:36.663 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00025427199999999999 2023-09-23T20:52:36.663 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:36.663 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:36.663 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:52:36.663 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:32.847103+0000", 2023-09-23T20:52:36.663 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.011384093 2023-09-23T20:52:36.663 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:36.663 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:36.663 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T20:52:36.663 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:32.847143+0000", 2023-09-23T20:52:36.663 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.9907000000000003e-05 2023-09-23T20:52:36.663 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:36.664 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:36.664 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T20:52:36.664 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:32.847177+0000", 2023-09-23T20:52:36.664 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.3312000000000002e-05 2023-09-23T20:52:36.664 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:52:36.664 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T20:52:36.664 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:52:36.664 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:36.664 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:36.664 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34797.0:30046 6.d 6:b54b75cc:::benchmark_data_smithi026_223868_object30045:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e109)", 2023-09-23T20:52:36.664 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T20:52:34.337140+0000", 2023-09-23T20:52:36.664 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 2.292608263, 2023-09-23T20:52:36.664 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.022554487000000002, 2023-09-23T20:52:36.665 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T20:52:36.665 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:52:36.665 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T20:52:36.665 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34797", 2023-09-23T20:52:36.665 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T20:52:36.665 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 30046 2023-09-23T20:52:36.665 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:36.665 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T20:52:36.665 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:36.665 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T20:52:36.665 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:34.337140+0000", 2023-09-23T20:52:36.665 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:52:36.666 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:36.666 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:36.666 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T20:52:36.666 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:34.337140+0000", 2023-09-23T20:52:36.666 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:52:36.666 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:36.666 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:36.666 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T20:52:36.666 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:34.337139+0000", 2023-09-23T20:52:36.666 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.999999 2023-09-23T20:52:36.666 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:36.666 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:36.666 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T20:52:36.667 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:34.337163+0000", 2023-09-23T20:52:36.667 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.4498000000000002e-05 2023-09-23T20:52:36.667 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:36.667 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:36.667 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T20:52:36.667 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:34.337184+0000", 2023-09-23T20:52:36.667 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.1209e-05 2023-09-23T20:52:36.667 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:36.667 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:36.667 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T20:52:36.667 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:34.337195+0000", 2023-09-23T20:52:36.667 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0121e-05 2023-09-23T20:52:36.667 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:36.668 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:36.668 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:52:36.668 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:34.337235+0000", 2023-09-23T20:52:36.668 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.0478000000000001e-05 2023-09-23T20:52:36.668 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:36.668 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:36.668 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T20:52:36.668 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:34.337429+0000", 2023-09-23T20:52:36.668 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00019422499999999999 2023-09-23T20:52:36.668 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:36.668 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:36.668 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 0,2", 2023-09-23T20:52:36.669 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:34.337616+0000", 2023-09-23T20:52:36.669 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00018708599999999999 2023-09-23T20:52:36.669 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:36.669 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:36.669 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T20:52:36.669 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:34.341275+0000", 2023-09-23T20:52:36.669 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0036590120000000001 2023-09-23T20:52:36.669 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:36.669 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:36.669 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:52:36.669 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:34.345322+0000", 2023-09-23T20:52:36.669 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0040469690000000001 2023-09-23T20:52:36.670 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:36.670 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:36.670 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:52:36.670 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:34.359597+0000", 2023-09-23T20:52:36.670 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.014274505999999999 2023-09-23T20:52:36.670 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:36.670 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:36.670 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T20:52:36.670 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:34.359649+0000", 2023-09-23T20:52:36.670 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.1765999999999999e-05 2023-09-23T20:52:36.670 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:36.670 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:36.671 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T20:52:36.671 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:34.359694+0000", 2023-09-23T20:52:36.671 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.5633000000000002e-05 2023-09-23T20:52:36.671 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:52:36.671 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T20:52:36.671 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:52:36.671 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:36.671 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:36.671 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34797.0:31150 6.4 6:25f46e44:::benchmark_data_smithi026_223868_object31149:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e109)", 2023-09-23T20:52:36.671 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T20:52:35.007983+0000", 2023-09-23T20:52:36.671 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 1.6217646960000001, 2023-09-23T20:52:36.671 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.021465523, 2023-09-23T20:52:36.672 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T20:52:36.672 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:52:36.672 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T20:52:36.672 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34797", 2023-09-23T20:52:36.672 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T20:52:36.672 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 31150 2023-09-23T20:52:36.672 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:36.672 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T20:52:36.672 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:36.672 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T20:52:36.672 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:35.007983+0000", 2023-09-23T20:52:36.672 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:52:36.673 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:36.673 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:36.673 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T20:52:36.673 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:35.007983+0000", 2023-09-23T20:52:36.673 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:52:36.673 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:36.673 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:36.673 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T20:52:36.673 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:35.007983+0000", 2023-09-23T20:52:36.673 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T20:52:36.673 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:36.673 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:36.674 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T20:52:36.674 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:35.008005+0000", 2023-09-23T20:52:36.674 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.2297e-05 2023-09-23T20:52:36.674 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:36.674 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:36.674 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T20:52:36.674 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:35.008014+0000", 2023-09-23T20:52:36.674 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.4919999999999993e-06 2023-09-23T20:52:36.674 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:36.674 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:36.674 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T20:52:36.674 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:35.008020+0000", 2023-09-23T20:52:36.675 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.4559999999999997e-06 2023-09-23T20:52:36.675 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:36.675 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:36.675 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:52:36.675 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:35.008082+0000", 2023-09-23T20:52:36.675 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.2001000000000005e-05 2023-09-23T20:52:36.675 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:36.675 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:36.675 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T20:52:36.675 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:35.008212+0000", 2023-09-23T20:52:36.675 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000129567 2023-09-23T20:52:36.675 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:36.676 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:36.676 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 3,6", 2023-09-23T20:52:36.676 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:35.008350+0000", 2023-09-23T20:52:36.676 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000138202 2023-09-23T20:52:36.676 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:36.676 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:36.676 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:52:36.676 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:35.018329+0000", 2023-09-23T20:52:36.676 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0099786679999999996 2023-09-23T20:52:36.676 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:36.676 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:36.677 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T20:52:36.677 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:35.018570+0000", 2023-09-23T20:52:36.677 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000241356 2023-09-23T20:52:36.677 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:36.677 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:36.677 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:52:36.677 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:35.029364+0000", 2023-09-23T20:52:36.677 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.010794160000000001 2023-09-23T20:52:36.677 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:36.677 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:36.677 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T20:52:36.678 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:35.029413+0000", 2023-09-23T20:52:36.678 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.8767999999999998e-05 2023-09-23T20:52:36.678 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:36.678 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:36.678 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T20:52:36.678 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:35.029449+0000", 2023-09-23T20:52:36.678 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.6096999999999997e-05 2023-09-23T20:52:36.678 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:52:36.678 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T20:52:36.678 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:52:36.678 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:36.678 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:36.679 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34797.0:31709 6.1 6:837ec07f:::benchmark_data_smithi026_223868_object31708:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e109)", 2023-09-23T20:52:36.679 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T20:52:35.350472+0000", 2023-09-23T20:52:36.679 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 1.2792764810000001, 2023-09-23T20:52:36.679 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.022012647999999999, 2023-09-23T20:52:36.679 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T20:52:36.679 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:52:36.679 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T20:52:36.679 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34797", 2023-09-23T20:52:36.679 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T20:52:36.679 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 31709 2023-09-23T20:52:36.679 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:36.679 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T20:52:36.680 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:36.680 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T20:52:36.680 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:35.350472+0000", 2023-09-23T20:52:36.680 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:52:36.680 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:36.680 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:36.680 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T20:52:36.680 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:35.350472+0000", 2023-09-23T20:52:36.680 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:52:36.680 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:36.680 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:36.681 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T20:52:36.681 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:35.350470+0000", 2023-09-23T20:52:36.681 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.999999 2023-09-23T20:52:36.681 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:36.681 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:36.681 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T20:52:36.681 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:35.350518+0000", 2023-09-23T20:52:36.681 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.7827999999999997e-05 2023-09-23T20:52:36.681 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:36.681 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:36.681 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T20:52:36.681 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:35.350532+0000", 2023-09-23T20:52:36.682 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.3661e-05 2023-09-23T20:52:36.682 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:36.682 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:36.682 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T20:52:36.682 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:35.350538+0000", 2023-09-23T20:52:36.682 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.0809999999999999e-06 2023-09-23T20:52:36.682 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:36.682 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:36.682 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:52:36.682 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:35.350588+0000", 2023-09-23T20:52:36.682 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.0380999999999998e-05 2023-09-23T20:52:36.682 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:36.683 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:36.683 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T20:52:36.683 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:35.350771+0000", 2023-09-23T20:52:36.683 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000182898 2023-09-23T20:52:36.683 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:36.683 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:36.683 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 2,6", 2023-09-23T20:52:36.683 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:35.351004+0000", 2023-09-23T20:52:36.683 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000233031 2023-09-23T20:52:36.683 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:36.683 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:36.683 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:52:36.684 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:35.354778+0000", 2023-09-23T20:52:36.684 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0037741739999999999 2023-09-23T20:52:36.684 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:36.684 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:36.684 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T20:52:36.684 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:35.356132+0000", 2023-09-23T20:52:36.684 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0013539820000000001 2023-09-23T20:52:36.684 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:36.684 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:36.684 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:52:36.684 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:35.372389+0000", 2023-09-23T20:52:36.685 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.016256319000000002 2023-09-23T20:52:36.685 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:36.685 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:36.685 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T20:52:36.685 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:35.372429+0000", 2023-09-23T20:52:36.685 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.9928000000000001e-05 2023-09-23T20:52:36.685 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:36.685 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:36.685 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T20:52:36.685 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:35.372484+0000", 2023-09-23T20:52:36.685 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.5525000000000001e-05 2023-09-23T20:52:36.686 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:52:36.686 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T20:52:36.686 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:52:36.686 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:36.686 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:36.686 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34797.0:31710 6.1 6:86310ade:::benchmark_data_smithi026_223868_object31709:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e109)", 2023-09-23T20:52:36.686 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T20:52:35.350551+0000", 2023-09-23T20:52:36.686 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 1.279197449, 2023-09-23T20:52:36.686 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.022771013999999999, 2023-09-23T20:52:36.686 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T20:52:36.686 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:52:36.687 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T20:52:36.687 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34797", 2023-09-23T20:52:36.687 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T20:52:36.687 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 31710 2023-09-23T20:52:36.687 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:36.687 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T20:52:36.687 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:36.687 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T20:52:36.687 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:35.350551+0000", 2023-09-23T20:52:36.687 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:52:36.687 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:36.687 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:36.688 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T20:52:36.688 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:35.350551+0000", 2023-09-23T20:52:36.688 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:52:36.688 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:36.688 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:36.688 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T20:52:36.688 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:35.350550+0000", 2023-09-23T20:52:36.688 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T20:52:36.688 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:36.688 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:36.688 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T20:52:36.688 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:35.350579+0000", 2023-09-23T20:52:36.689 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.8636000000000002e-05 2023-09-23T20:52:36.689 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:36.689 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:36.689 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T20:52:36.689 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:35.350587+0000", 2023-09-23T20:52:36.689 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.0290000000000005e-06 2023-09-23T20:52:36.689 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:36.689 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:36.689 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T20:52:36.689 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:35.350592+0000", 2023-09-23T20:52:36.689 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.5960000000000003e-06 2023-09-23T20:52:36.689 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:36.690 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:36.690 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:52:36.690 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:35.353378+0000", 2023-09-23T20:52:36.690 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0027859590000000002 2023-09-23T20:52:36.690 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:36.690 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:36.690 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T20:52:36.690 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:35.353540+0000", 2023-09-23T20:52:36.690 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000161328 2023-09-23T20:52:36.690 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:36.690 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:36.690 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 2,6", 2023-09-23T20:52:36.691 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:35.353692+0000", 2023-09-23T20:52:36.691 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00015201 2023-09-23T20:52:36.691 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:36.691 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:36.691 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T20:52:36.691 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:35.357515+0000", 2023-09-23T20:52:36.691 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0038238230000000001 2023-09-23T20:52:36.691 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:36.691 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:36.691 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:52:36.691 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:35.358307+0000", 2023-09-23T20:52:36.691 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00079140999999999999 2023-09-23T20:52:36.692 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:36.692 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:36.692 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:52:36.692 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:35.373235+0000", 2023-09-23T20:52:36.692 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.014928469 2023-09-23T20:52:36.692 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:36.692 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:36.692 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T20:52:36.692 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:35.373285+0000", 2023-09-23T20:52:36.692 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.9619e-05 2023-09-23T20:52:36.692 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:36.693 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:36.693 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T20:52:36.693 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:35.373322+0000", 2023-09-23T20:52:36.693 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.6656000000000003e-05 2023-09-23T20:52:36.693 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:52:36.693 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T20:52:36.693 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:52:36.693 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:36.693 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:36.693 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34797.0:31713 6.d 6:b8f98635:::benchmark_data_smithi026_223868_object31712:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e109)", 2023-09-23T20:52:36.693 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T20:52:35.350604+0000", 2023-09-23T20:52:36.693 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 1.2791441889999999, 2023-09-23T20:52:36.694 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.023410212, 2023-09-23T20:52:36.694 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T20:52:36.694 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:52:36.694 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T20:52:36.694 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34797", 2023-09-23T20:52:36.694 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T20:52:36.694 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 31713 2023-09-23T20:52:36.694 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:36.694 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T20:52:36.694 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:36.694 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T20:52:36.694 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:35.350604+0000", 2023-09-23T20:52:36.695 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:52:36.695 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:36.695 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:36.695 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T20:52:36.695 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:35.350604+0000", 2023-09-23T20:52:36.695 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:52:36.695 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:36.695 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:36.695 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T20:52:36.695 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:35.350603+0000", 2023-09-23T20:52:36.695 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T20:52:36.695 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:36.696 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:36.696 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T20:52:36.696 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:35.350625+0000", 2023-09-23T20:52:36.696 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.1319e-05 2023-09-23T20:52:36.696 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:36.696 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:36.696 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T20:52:36.696 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:35.350631+0000", 2023-09-23T20:52:36.696 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.477e-06 2023-09-23T20:52:36.696 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:36.696 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:36.696 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T20:52:36.696 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:35.350637+0000", 2023-09-23T20:52:36.697 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.1240000000000002e-06 2023-09-23T20:52:36.697 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:36.697 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:36.697 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:52:36.697 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:35.350715+0000", 2023-09-23T20:52:36.697 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.7278000000000001e-05 2023-09-23T20:52:36.697 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:36.697 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:36.697 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T20:52:36.697 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:35.350872+0000", 2023-09-23T20:52:36.697 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00015791000000000001 2023-09-23T20:52:36.698 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:36.698 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:36.698 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 0,2", 2023-09-23T20:52:36.698 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:35.351024+0000", 2023-09-23T20:52:36.698 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00015124 2023-09-23T20:52:36.698 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:36.698 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:36.698 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T20:52:36.698 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:35.369110+0000", 2023-09-23T20:52:36.698 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.018086006000000002 2023-09-23T20:52:36.698 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:36.698 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:36.699 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:52:36.699 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:35.369144+0000", 2023-09-23T20:52:36.699 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.4288000000000001e-05 2023-09-23T20:52:36.699 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:36.699 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:36.699 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:52:36.699 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:35.373933+0000", 2023-09-23T20:52:36.699 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0047886760000000004 2023-09-23T20:52:36.699 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:36.699 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:36.699 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T20:52:36.699 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:35.373979+0000", 2023-09-23T20:52:36.699 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.6721999999999998e-05 2023-09-23T20:52:36.700 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:36.700 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:36.700 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T20:52:36.700 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:35.374014+0000", 2023-09-23T20:52:36.700 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.4668000000000001e-05 2023-09-23T20:52:36.700 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:52:36.700 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T20:52:36.700 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:52:36.700 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:36.700 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:36.700 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34797.0:31790 6.4 6:2245ab05:::benchmark_data_smithi026_223868_object31789:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e109)", 2023-09-23T20:52:36.700 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T20:52:35.420953+0000", 2023-09-23T20:52:36.700 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 1.208795327, 2023-09-23T20:52:36.701 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.025777683999999999, 2023-09-23T20:52:36.701 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T20:52:36.701 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:52:36.701 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T20:52:36.701 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34797", 2023-09-23T20:52:36.701 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T20:52:36.701 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 31790 2023-09-23T20:52:36.701 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:36.701 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T20:52:36.701 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:36.701 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T20:52:36.702 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:35.420953+0000", 2023-09-23T20:52:36.702 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:52:36.702 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:36.702 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:36.702 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T20:52:36.702 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:35.420953+0000", 2023-09-23T20:52:36.702 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:52:36.702 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:36.702 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:36.702 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T20:52:36.702 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:35.420952+0000", 2023-09-23T20:52:36.702 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.999999 2023-09-23T20:52:36.703 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:36.703 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:36.703 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T20:52:36.703 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:35.420983+0000", 2023-09-23T20:52:36.703 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.1158999999999998e-05 2023-09-23T20:52:36.703 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:36.703 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:36.703 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T20:52:36.703 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:35.420991+0000", 2023-09-23T20:52:36.703 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.2390000000000002e-06 2023-09-23T20:52:36.703 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:36.703 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:36.704 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T20:52:36.704 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:35.420998+0000", 2023-09-23T20:52:36.704 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.28e-06 2023-09-23T20:52:36.704 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:36.704 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:36.704 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:52:36.704 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:35.421111+0000", 2023-09-23T20:52:36.704 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000113559 2023-09-23T20:52:36.704 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:36.704 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:36.704 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T20:52:36.704 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:35.421230+0000", 2023-09-23T20:52:36.704 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000118936 2023-09-23T20:52:36.705 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:36.705 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:36.705 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 3,6", 2023-09-23T20:52:36.705 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:35.421367+0000", 2023-09-23T20:52:36.705 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00013645000000000001 2023-09-23T20:52:36.705 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:36.705 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:36.705 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T20:52:36.705 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:35.423070+0000", 2023-09-23T20:52:36.705 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.001702968 2023-09-23T20:52:36.705 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:36.705 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:36.706 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:52:36.706 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:35.424856+0000", 2023-09-23T20:52:36.706 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.001786787 2023-09-23T20:52:36.706 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:36.706 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:36.706 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:52:36.706 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:35.446627+0000", 2023-09-23T20:52:36.706 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.021770233 2023-09-23T20:52:36.706 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:36.706 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:36.706 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T20:52:36.707 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:35.446684+0000", 2023-09-23T20:52:36.707 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.7318000000000002e-05 2023-09-23T20:52:36.707 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:36.707 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:36.707 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T20:52:36.707 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:35.446730+0000", 2023-09-23T20:52:36.707 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.6485e-05 2023-09-23T20:52:36.707 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:52:36.707 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T20:52:36.707 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:52:36.707 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:36.708 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:36.708 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34797.0:31801 6.4 6:26214b1a:::benchmark_data_smithi026_223868_object31800:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e109)", 2023-09-23T20:52:36.708 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T20:52:35.424882+0000", 2023-09-23T20:52:36.708 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 1.204866182, 2023-09-23T20:52:36.708 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.021966164999999999, 2023-09-23T20:52:36.708 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T20:52:36.708 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:52:36.708 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T20:52:36.708 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34797", 2023-09-23T20:52:36.708 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T20:52:36.708 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 31801 2023-09-23T20:52:36.709 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:36.709 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T20:52:36.709 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:36.709 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T20:52:36.709 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:35.424882+0000", 2023-09-23T20:52:36.709 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:52:36.709 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:36.709 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:36.709 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T20:52:36.709 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:35.424882+0000", 2023-09-23T20:52:36.709 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:52:36.709 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:36.709 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:36.710 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T20:52:36.710 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:35.424880+0000", 2023-09-23T20:52:36.710 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T20:52:36.710 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:36.710 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:36.710 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T20:52:36.710 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:35.424905+0000", 2023-09-23T20:52:36.710 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.5120999999999999e-05 2023-09-23T20:52:36.710 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:36.710 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:36.710 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T20:52:36.710 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:35.424919+0000", 2023-09-23T20:52:36.711 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.3539e-05 2023-09-23T20:52:36.711 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:36.711 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:36.711 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T20:52:36.711 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:35.424928+0000", 2023-09-23T20:52:36.711 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.5380000000000002e-06 2023-09-23T20:52:36.711 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:36.711 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:36.711 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:52:36.711 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:35.424971+0000", 2023-09-23T20:52:36.711 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.3126e-05 2023-09-23T20:52:36.712 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:36.712 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:36.712 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T20:52:36.712 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:35.425118+0000", 2023-09-23T20:52:36.712 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00014720799999999999 2023-09-23T20:52:36.712 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:36.712 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:36.712 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 3,6", 2023-09-23T20:52:36.712 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:35.425250+0000", 2023-09-23T20:52:36.712 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00013187799999999999 2023-09-23T20:52:36.712 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:36.712 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:36.713 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:52:36.713 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:35.426889+0000", 2023-09-23T20:52:36.713 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.001638837 2023-09-23T20:52:36.713 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:36.713 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:36.713 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T20:52:36.713 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:35.426897+0000", 2023-09-23T20:52:36.713 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.3210000000000005e-06 2023-09-23T20:52:36.713 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:36.713 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:36.713 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:52:36.713 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:35.446777+0000", 2023-09-23T20:52:36.714 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.019880289999999998 2023-09-23T20:52:36.714 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:36.714 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:36.714 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T20:52:36.714 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:35.446816+0000", 2023-09-23T20:52:36.714 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.8330000000000001e-05 2023-09-23T20:52:36.714 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:36.714 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:36.714 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T20:52:36.714 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:35.446848+0000", 2023-09-23T20:52:36.714 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.2489e-05 2023-09-23T20:52:36.714 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:52:36.714 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T20:52:36.715 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:52:36.715 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:36.715 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:36.715 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34797.0:31845 6.d 6:b45ad078:::benchmark_data_smithi026_223868_object31844:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e109)", 2023-09-23T20:52:36.715 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T20:52:35.467792+0000", 2023-09-23T20:52:36.715 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 1.161956231, 2023-09-23T20:52:36.715 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.023241379, 2023-09-23T20:52:36.715 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T20:52:36.715 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:52:36.715 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T20:52:36.715 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34797", 2023-09-23T20:52:36.716 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T20:52:36.716 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 31845 2023-09-23T20:52:36.716 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:36.716 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T20:52:36.716 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:36.716 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T20:52:36.716 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:35.467792+0000", 2023-09-23T20:52:36.716 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:52:36.716 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:36.716 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:36.717 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T20:52:36.717 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:35.467792+0000", 2023-09-23T20:52:36.717 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:52:36.717 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:36.717 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:36.717 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T20:52:36.717 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:35.467790+0000", 2023-09-23T20:52:36.717 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999981 2023-09-23T20:52:36.717 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:36.717 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:36.717 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T20:52:36.717 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:35.467841+0000", 2023-09-23T20:52:36.718 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.1573000000000002e-05 2023-09-23T20:52:36.718 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:36.718 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:36.718 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T20:52:36.718 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:35.467878+0000", 2023-09-23T20:52:36.718 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.6075999999999998e-05 2023-09-23T20:52:36.718 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:36.718 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:36.718 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T20:52:36.718 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:35.467894+0000", 2023-09-23T20:52:36.718 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.6809000000000001e-05 2023-09-23T20:52:36.718 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:36.719 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:36.719 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:52:36.719 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:35.468956+0000", 2023-09-23T20:52:36.719 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0010616 2023-09-23T20:52:36.719 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:36.719 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:36.719 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T20:52:36.719 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:35.469079+0000", 2023-09-23T20:52:36.719 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000123096 2023-09-23T20:52:36.719 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:36.719 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:36.719 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 0,2", 2023-09-23T20:52:36.720 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:35.469209+0000", 2023-09-23T20:52:36.720 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000129806 2023-09-23T20:52:36.720 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:36.720 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:36.720 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T20:52:36.720 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:35.475597+0000", 2023-09-23T20:52:36.720 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0063882189999999997 2023-09-23T20:52:36.720 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:36.720 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:36.720 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:52:36.720 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:35.478593+0000", 2023-09-23T20:52:36.720 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0029959959999999999 2023-09-23T20:52:36.720 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:36.721 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:36.721 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:52:36.721 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:35.490947+0000", 2023-09-23T20:52:36.721 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.012354248 2023-09-23T20:52:36.721 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:36.721 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:36.721 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T20:52:36.721 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:35.490997+0000", 2023-09-23T20:52:36.721 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.9846999999999999e-05 2023-09-23T20:52:36.721 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:36.721 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:36.721 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T20:52:36.721 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:35.491033+0000", 2023-09-23T20:52:36.722 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.6050999999999998e-05 2023-09-23T20:52:36.722 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:52:36.722 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T20:52:36.722 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:52:36.722 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:36.722 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:36.722 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34797.0:32050 6.4 6:238d9ca0:::benchmark_data_smithi026_223868_object32049:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e109)", 2023-09-23T20:52:36.722 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T20:52:35.631413+0000", 2023-09-23T20:52:36.722 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 0.99833470300000005, 2023-09-23T20:52:36.722 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.020128390999999999, 2023-09-23T20:52:36.722 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T20:52:36.722 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:52:36.723 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T20:52:36.723 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34797", 2023-09-23T20:52:36.723 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T20:52:36.723 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 32050 2023-09-23T20:52:36.723 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:36.723 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T20:52:36.723 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:36.723 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T20:52:36.723 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:35.631413+0000", 2023-09-23T20:52:36.723 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:52:36.723 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:36.723 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:36.724 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T20:52:36.724 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:35.631413+0000", 2023-09-23T20:52:36.724 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:52:36.724 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:36.724 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:36.724 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T20:52:36.724 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:35.631413+0000", 2023-09-23T20:52:36.724 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T20:52:36.724 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:36.724 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:36.724 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T20:52:36.724 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:35.631431+0000", 2023-09-23T20:52:36.725 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.8547999999999999e-05 2023-09-23T20:52:36.725 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:36.725 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:36.725 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T20:52:36.725 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:35.631438+0000", 2023-09-23T20:52:36.725 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.1410000000000002e-06 2023-09-23T20:52:36.725 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:36.725 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:36.725 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T20:52:36.725 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:35.631443+0000", 2023-09-23T20:52:36.725 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.5979999999999996e-06 2023-09-23T20:52:36.726 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:36.726 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:36.726 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:52:36.726 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:35.631500+0000", 2023-09-23T20:52:36.726 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.7071999999999998e-05 2023-09-23T20:52:36.726 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:36.726 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:36.726 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T20:52:36.726 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:35.631681+0000", 2023-09-23T20:52:36.726 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00018085099999999999 2023-09-23T20:52:36.726 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:36.726 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:36.726 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 3,6", 2023-09-23T20:52:36.727 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:35.631831+0000", 2023-09-23T20:52:36.727 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00014990399999999999 2023-09-23T20:52:36.727 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:36.727 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:36.727 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T20:52:36.727 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:35.633995+0000", 2023-09-23T20:52:36.727 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0021640169999999999 2023-09-23T20:52:36.727 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:36.727 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:36.727 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:52:36.727 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:35.635335+0000", 2023-09-23T20:52:36.728 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0013404599999999999 2023-09-23T20:52:36.728 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:36.728 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:36.728 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:52:36.728 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:35.651443+0000", 2023-09-23T20:52:36.728 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.016107094999999998 2023-09-23T20:52:36.728 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:36.728 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:36.728 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T20:52:36.728 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:35.651504+0000", 2023-09-23T20:52:36.728 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.1582999999999996e-05 2023-09-23T20:52:36.728 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:36.728 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:36.729 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T20:52:36.729 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:35.651542+0000", 2023-09-23T20:52:36.729 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.7599999999999999e-05 2023-09-23T20:52:36.729 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:52:36.729 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T20:52:36.729 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:52:36.729 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:36.729 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:36.729 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_repop(client.34797.0:33272 6.7 e109/102 6:e312ad03:::benchmark_data_smithi026_223868_object33271:head v 109'2054, mlcod=109'2051)", 2023-09-23T20:52:36.729 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T20:52:36.459174+0000", 2023-09-23T20:52:36.729 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 0.170574271, 2023-09-23T20:52:36.729 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.028574291000000002, 2023-09-23T20:52:36.729 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T20:52:36.729 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:52:36.730 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T20:52:36.730 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:36.730 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T20:52:36.730 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:36.459174+0000", 2023-09-23T20:52:36.730 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:52:36.730 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:36.730 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:36.730 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T20:52:36.730 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:36.459174+0000", 2023-09-23T20:52:36.730 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:52:36.730 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:36.730 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:36.731 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T20:52:36.731 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:36.459173+0000", 2023-09-23T20:52:36.731 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.999999 2023-09-23T20:52:36.731 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:36.731 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:36.731 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T20:52:36.731 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:36.459195+0000", 2023-09-23T20:52:36.731 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.1968e-05 2023-09-23T20:52:36.731 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:36.731 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:36.731 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T20:52:36.731 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:36.459211+0000", 2023-09-23T20:52:36.731 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.6426000000000002e-05 2023-09-23T20:52:36.732 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:36.732 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:36.732 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T20:52:36.732 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:36.459220+0000", 2023-09-23T20:52:36.732 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.2490000000000007e-06 2023-09-23T20:52:36.732 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:36.732 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:36.732 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:52:36.732 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:36.465176+0000", 2023-09-23T20:52:36.732 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0059554600000000001 2023-09-23T20:52:36.732 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:36.732 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:36.732 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T20:52:36.733 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:36.465200+0000", 2023-09-23T20:52:36.733 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.4094e-05 2023-09-23T20:52:36.733 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:36.733 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:36.733 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T20:52:36.733 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:36.487707+0000", 2023-09-23T20:52:36.733 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.022507416999999998 2023-09-23T20:52:36.733 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:36.733 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:36.733 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T20:52:36.733 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:36.487748+0000", 2023-09-23T20:52:36.733 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.0844999999999999e-05 2023-09-23T20:52:36.733 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:52:36.734 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T20:52:36.734 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:52:36.734 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:36.734 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:36.734 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_repop(client.34797.0:33275 6.7 e109/102 6:ebf740bc:::benchmark_data_smithi026_223868_object33274:head v 109'2055, mlcod=109'2051)", 2023-09-23T20:52:36.734 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T20:52:36.459240+0000", 2023-09-23T20:52:36.734 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 0.17050801500000001, 2023-09-23T20:52:36.734 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.028535973999999999, 2023-09-23T20:52:36.734 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T20:52:36.734 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:52:36.734 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T20:52:36.735 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:36.735 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T20:52:36.735 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:36.459240+0000", 2023-09-23T20:52:36.735 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:52:36.735 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:36.735 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:36.735 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T20:52:36.735 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:36.459240+0000", 2023-09-23T20:52:36.735 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:52:36.735 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:36.735 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:36.735 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T20:52:36.736 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:36.459240+0000", 2023-09-23T20:52:36.736 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T20:52:36.736 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:36.736 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:36.736 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T20:52:36.736 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:36.459260+0000", 2023-09-23T20:52:36.736 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.0727e-05 2023-09-23T20:52:36.736 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:36.736 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:36.736 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T20:52:36.736 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:36.459268+0000", 2023-09-23T20:52:36.736 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.6909999999999997e-06 2023-09-23T20:52:36.737 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:36.737 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:36.737 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T20:52:36.737 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:36.459274+0000", 2023-09-23T20:52:36.737 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.7189999999999998e-06 2023-09-23T20:52:36.737 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:36.737 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:36.737 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:52:36.737 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:36.466401+0000", 2023-09-23T20:52:36.737 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0071270830000000002 2023-09-23T20:52:36.737 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:36.737 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:36.737 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T20:52:36.737 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:36.466422+0000", 2023-09-23T20:52:36.738 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.0985e-05 2023-09-23T20:52:36.738 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:36.738 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:36.738 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T20:52:36.738 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:36.487754+0000", 2023-09-23T20:52:36.738 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.021332482 2023-09-23T20:52:36.738 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:36.738 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:36.738 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T20:52:36.738 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:36.487776+0000", 2023-09-23T20:52:36.738 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.1603e-05 2023-09-23T20:52:36.738 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:52:36.739 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T20:52:36.739 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:52:36.739 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:36.739 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:36.739 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_repop(client.34797.0:33290 6.e e109/102 6:7aab0fdd:::benchmark_data_smithi026_223868_object33289:head v 109'2106, mlcod=109'2104)", 2023-09-23T20:52:36.739 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T20:52:36.465560+0000", 2023-09-23T20:52:36.739 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 0.16418842, 2023-09-23T20:52:36.739 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.021341742, 2023-09-23T20:52:36.739 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T20:52:36.739 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:52:36.739 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T20:52:36.740 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:36.740 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T20:52:36.740 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:36.465560+0000", 2023-09-23T20:52:36.740 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:52:36.740 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:36.740 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:36.740 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T20:52:36.740 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:36.465560+0000", 2023-09-23T20:52:36.740 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:52:36.740 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:36.740 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:36.740 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T20:52:36.740 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:36.465558+0000", 2023-09-23T20:52:36.741 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.999999 2023-09-23T20:52:36.741 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:36.741 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:36.741 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T20:52:36.741 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:36.465600+0000", 2023-09-23T20:52:36.741 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.1547e-05 2023-09-23T20:52:36.741 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:36.741 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:36.741 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T20:52:36.741 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:36.465616+0000", 2023-09-23T20:52:36.741 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.5617999999999999e-05 2023-09-23T20:52:36.741 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:36.741 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:36.742 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T20:52:36.742 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:36.465624+0000", 2023-09-23T20:52:36.742 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.8109999999999992e-06 2023-09-23T20:52:36.742 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:36.742 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:36.742 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:52:36.742 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:36.465705+0000", 2023-09-23T20:52:36.742 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.0431000000000003e-05 2023-09-23T20:52:36.742 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:36.742 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:36.742 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T20:52:36.742 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:36.465726+0000", 2023-09-23T20:52:36.743 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.1265999999999999e-05 2023-09-23T20:52:36.743 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:36.743 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:36.743 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T20:52:36.743 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:36.486867+0000", 2023-09-23T20:52:36.743 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.021141020999999999 2023-09-23T20:52:36.743 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:36.743 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:36.743 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T20:52:36.743 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:36.486901+0000", 2023-09-23T20:52:36.743 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.4214999999999997e-05 2023-09-23T20:52:36.743 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:52:36.743 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T20:52:36.744 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:52:36.744 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:36.744 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:36.744 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34797.0:33289 6.d 6:b8a35bb7:::benchmark_data_smithi026_223868_object33288:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e109)", 2023-09-23T20:52:36.744 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T20:52:36.465802+0000", 2023-09-23T20:52:36.744 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 0.16394573500000001, 2023-09-23T20:52:36.744 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.021052698000000002, 2023-09-23T20:52:36.744 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T20:52:36.744 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:52:36.744 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T20:52:36.744 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34797", 2023-09-23T20:52:36.744 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T20:52:36.744 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 33289 2023-09-23T20:52:36.745 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:36.745 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T20:52:36.745 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:36.745 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T20:52:36.745 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:36.465802+0000", 2023-09-23T20:52:36.745 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:52:36.745 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:36.745 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:36.745 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T20:52:36.745 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:36.465802+0000", 2023-09-23T20:52:36.745 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:52:36.745 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:36.745 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:36.746 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T20:52:36.746 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:36.465801+0000", 2023-09-23T20:52:36.746 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999981 2023-09-23T20:52:36.746 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:36.746 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:36.746 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T20:52:36.746 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:36.465831+0000", 2023-09-23T20:52:36.746 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.0488999999999999e-05 2023-09-23T20:52:36.746 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:36.746 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:36.746 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T20:52:36.747 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:36.465843+0000", 2023-09-23T20:52:36.747 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.2352e-05 2023-09-23T20:52:36.747 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:36.747 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:36.747 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T20:52:36.747 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:36.465852+0000", 2023-09-23T20:52:36.747 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.5089999999999993e-06 2023-09-23T20:52:36.747 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:36.747 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:36.747 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:52:36.747 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:36.465903+0000", 2023-09-23T20:52:36.747 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.1187999999999998e-05 2023-09-23T20:52:36.747 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:36.747 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:36.748 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T20:52:36.748 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:36.466028+0000", 2023-09-23T20:52:36.748 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000124855 2023-09-23T20:52:36.748 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:36.748 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:36.748 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 0,2", 2023-09-23T20:52:36.748 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:36.466152+0000", 2023-09-23T20:52:36.748 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00012359299999999999 2023-09-23T20:52:36.748 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:36.748 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:36.748 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:52:36.748 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:36.470495+0000", 2023-09-23T20:52:36.748 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0043432430000000001 2023-09-23T20:52:36.748 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:36.749 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:36.749 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:52:36.749 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:36.475052+0000", 2023-09-23T20:52:36.749 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0045574719999999999 2023-09-23T20:52:36.749 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:36.749 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:36.749 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T20:52:36.749 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:36.486760+0000", 2023-09-23T20:52:36.749 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.011708163000000001 2023-09-23T20:52:36.749 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:36.749 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:36.749 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T20:52:36.749 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:36.486814+0000", 2023-09-23T20:52:36.749 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.3913999999999999e-05 2023-09-23T20:52:36.750 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:36.750 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:36.750 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T20:52:36.750 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:36.486855+0000", 2023-09-23T20:52:36.750 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.0635e-05 2023-09-23T20:52:36.750 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:52:36.750 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T20:52:36.750 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:52:36.750 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:36.750 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:36.750 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_repop(client.34797.0:33284 6.b e109/102 6:d60135a0:::benchmark_data_smithi026_223868_object33283:head v 109'2083, mlcod=109'2078)", 2023-09-23T20:52:36.750 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T20:52:36.466716+0000", 2023-09-23T20:52:36.750 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 0.16303214499999999, 2023-09-23T20:52:36.751 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.021532979000000001, 2023-09-23T20:52:36.751 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T20:52:36.751 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:52:36.751 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T20:52:36.751 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:36.751 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T20:52:36.751 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:36.466716+0000", 2023-09-23T20:52:36.751 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:52:36.751 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:36.751 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:36.751 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T20:52:36.752 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:36.466716+0000", 2023-09-23T20:52:36.752 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:52:36.752 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:36.752 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:36.752 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T20:52:36.752 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:36.466715+0000", 2023-09-23T20:52:36.752 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.999999 2023-09-23T20:52:36.752 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:36.752 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:36.752 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T20:52:36.752 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:36.466742+0000", 2023-09-23T20:52:36.752 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.7271e-05 2023-09-23T20:52:36.752 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:36.752 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:36.753 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T20:52:36.753 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:36.466753+0000", 2023-09-23T20:52:36.753 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0529e-05 2023-09-23T20:52:36.753 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:36.753 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:36.753 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T20:52:36.753 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:36.466759+0000", 2023-09-23T20:52:36.753 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.4470000000000003e-06 2023-09-23T20:52:36.753 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:36.753 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:36.753 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:52:36.753 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:36.469266+0000", 2023-09-23T20:52:36.754 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.002506983 2023-09-23T20:52:36.754 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:36.754 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:36.754 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T20:52:36.754 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:36.469289+0000", 2023-09-23T20:52:36.754 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.2768999999999998e-05 2023-09-23T20:52:36.754 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:36.754 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:36.754 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T20:52:36.754 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:36.488222+0000", 2023-09-23T20:52:36.754 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.018933274 2023-09-23T20:52:36.754 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:36.755 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:36.755 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T20:52:36.755 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:36.488249+0000", 2023-09-23T20:52:36.755 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.6475999999999999e-05 2023-09-23T20:52:36.755 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:52:36.755 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T20:52:36.755 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:52:36.755 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:36.755 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:36.755 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34797.0:33382 6.1 6:81858130:::benchmark_data_smithi026_223868_object33381:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e109)", 2023-09-23T20:52:36.755 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T20:52:36.558547+0000", 2023-09-23T20:52:36.755 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 0.071200942000000003, 2023-09-23T20:52:36.756 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.020767243000000001, 2023-09-23T20:52:36.756 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T20:52:36.756 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:52:36.756 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T20:52:36.756 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34797", 2023-09-23T20:52:36.756 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T20:52:36.756 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 33382 2023-09-23T20:52:36.756 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:36.756 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T20:52:36.756 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:36.756 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T20:52:36.756 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:36.558547+0000", 2023-09-23T20:52:36.757 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:52:36.757 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:36.757 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:36.757 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T20:52:36.757 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:36.558547+0000", 2023-09-23T20:52:36.757 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:52:36.757 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:36.757 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:36.757 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T20:52:36.757 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:36.558546+0000", 2023-09-23T20:52:36.757 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.999999 2023-09-23T20:52:36.757 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:36.757 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:36.758 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T20:52:36.758 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:36.558571+0000", 2023-09-23T20:52:36.758 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.4406999999999999e-05 2023-09-23T20:52:36.758 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:36.758 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:36.758 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T20:52:36.758 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:36.558580+0000", 2023-09-23T20:52:36.758 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.0340000000000002e-06 2023-09-23T20:52:36.758 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:36.758 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:36.758 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T20:52:36.759 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:36.558587+0000", 2023-09-23T20:52:36.759 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.0450000000000003e-06 2023-09-23T20:52:36.759 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:36.759 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:36.759 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:52:36.759 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:36.558641+0000", 2023-09-23T20:52:36.759 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.4441000000000003e-05 2023-09-23T20:52:36.759 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:36.759 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:36.759 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T20:52:36.759 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:36.558771+0000", 2023-09-23T20:52:36.759 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000129954 2023-09-23T20:52:36.760 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:36.760 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:36.760 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 2,6", 2023-09-23T20:52:36.760 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:36.558963+0000", 2023-09-23T20:52:36.760 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00019168500000000001 2023-09-23T20:52:36.760 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:36.760 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:36.760 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T20:52:36.760 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:36.565887+0000", 2023-09-23T20:52:36.760 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0069241449999999996 2023-09-23T20:52:36.760 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:36.760 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:36.760 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:52:36.761 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:36.568224+0000", 2023-09-23T20:52:36.761 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0023365270000000001 2023-09-23T20:52:36.761 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:36.761 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:36.761 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:52:36.761 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:36.579216+0000", 2023-09-23T20:52:36.761 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.010992682 2023-09-23T20:52:36.761 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:36.761 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:36.761 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T20:52:36.761 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:36.579271+0000", 2023-09-23T20:52:36.762 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.4840999999999999e-05 2023-09-23T20:52:36.762 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:36.762 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:36.762 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T20:52:36.762 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:36.579314+0000", 2023-09-23T20:52:36.762 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.3210999999999997e-05 2023-09-23T20:52:36.762 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:52:36.762 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T20:52:36.762 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:52:36.762 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:36.762 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:36.762 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34797.0:33386 6.1 6:84ce398e:::benchmark_data_smithi026_223868_object33385:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e109)", 2023-09-23T20:52:36.763 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T20:52:36.558606+0000", 2023-09-23T20:52:36.763 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 0.071141909000000003, 2023-09-23T20:52:36.763 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.021013198, 2023-09-23T20:52:36.763 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T20:52:36.763 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:52:36.763 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T20:52:36.763 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34797", 2023-09-23T20:52:36.763 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T20:52:36.763 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 33386 2023-09-23T20:52:36.763 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:36.763 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T20:52:36.763 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:36.764 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T20:52:36.764 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:36.558606+0000", 2023-09-23T20:52:36.764 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:52:36.764 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:36.764 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:36.764 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T20:52:36.764 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:36.558606+0000", 2023-09-23T20:52:36.764 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:52:36.764 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:36.764 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:36.764 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T20:52:36.764 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:36.558606+0000", 2023-09-23T20:52:36.765 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T20:52:36.765 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:36.765 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:36.765 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T20:52:36.765 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:36.558628+0000", 2023-09-23T20:52:36.765 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.209e-05 2023-09-23T20:52:36.765 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:36.765 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:36.765 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T20:52:36.765 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:36.558635+0000", 2023-09-23T20:52:36.765 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.4150000000000002e-06 2023-09-23T20:52:36.765 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:36.766 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:36.766 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T20:52:36.766 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:36.558642+0000", 2023-09-23T20:52:36.766 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.7569999999999998e-06 2023-09-23T20:52:36.766 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:36.766 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:36.766 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:52:36.766 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:36.563332+0000", 2023-09-23T20:52:36.766 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0046905970000000003 2023-09-23T20:52:36.766 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:36.766 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:36.766 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T20:52:36.766 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:36.563487+0000", 2023-09-23T20:52:36.766 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00015436599999999999 2023-09-23T20:52:36.767 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:36.767 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:36.767 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 2,6", 2023-09-23T20:52:36.767 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:36.563630+0000", 2023-09-23T20:52:36.767 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00014358900000000001 2023-09-23T20:52:36.767 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:36.767 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:36.767 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T20:52:36.767 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:36.565893+0000", 2023-09-23T20:52:36.767 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0022621590000000001 2023-09-23T20:52:36.769 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:36.769 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:36.769 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:52:36.769 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:36.568253+0000", 2023-09-23T20:52:36.769 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0023604479999999998 2023-09-23T20:52:36.769 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:36.769 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:36.769 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:52:36.769 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:36.579534+0000", 2023-09-23T20:52:36.769 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.011280497 2023-09-23T20:52:36.770 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:36.770 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:36.770 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T20:52:36.770 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:36.579583+0000", 2023-09-23T20:52:36.770 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.9524000000000002e-05 2023-09-23T20:52:36.770 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:36.770 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:36.770 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T20:52:36.770 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:36.579619+0000", 2023-09-23T20:52:36.770 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.6319999999999998e-05 2023-09-23T20:52:36.770 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:52:36.770 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T20:52:36.771 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:52:36.771 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:52:36.771 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T20:52:36.771 INFO:teuthology.orchestra.run.smithi026.stdout:} 2023-09-23T20:52:36.771 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:52:36.772 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:52:36.845 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 17 16 33713 33697 123.752 84.1875 0.0112477 0.00807536 2023-09-23T20:52:36.903 INFO:teuthology.orchestra.run.smithi026.stderr:nodeep-scrub is unset 2023-09-23T20:52:37.086 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:52:37.096 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:52:37.097 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:52:37.097 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:52:36 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/1682769002' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:52:37.097 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:52:36 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:52:37.097 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:52:36 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/1682769002' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:52:37.097 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:52:36 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:52:37.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:52:36 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/1682769002' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:52:37.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:52:36 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:52:37.198 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:52:37.230 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:52:37.270 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:52:37.273 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:52:37.274 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:52:37.290 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_ops_in_flight 2023-09-23T20:52:37.375 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:52:37.405 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:52:37.447 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:52:37.450 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:52:37.450 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:52:37.551 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:52:37.581 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:52:37.623 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:52:37.626 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:52:37.626 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:52:37.727 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:52:37.823 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:52:37.845 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 18 16 35201 35185 122.045 93 0.00451291 0.00818584 2023-09-23T20:52:37.867 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:52:37 smithi070 ceph-mon[168282]: pgmap v253: 89 pgs: 89 active+clean; 1.8 GiB data, 5.4 GiB used, 620 GiB / 626 GiB avail; 117 MiB/s wr, 1.87k op/s 2023-09-23T20:52:37.867 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:52:37 smithi070 ceph-mon[168282]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T20:52:37.867 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:52:37 smithi070 ceph-mon[168282]: Cluster is now healthy 2023-09-23T20:52:37.867 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:52:37 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T20:52:37.868 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:52:37 smithi070 ceph-mon[168282]: osdmap e110: 8 total, 8 up, 7 in 2023-09-23T20:52:37.868 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:52:37 smithi070 ceph-mon[168282]: failed to encode map e110 with expected crc 2023-09-23T20:52:37.868 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:52:37 smithi070 ceph-mon[168282]: failed to encode map e110 with expected crc 2023-09-23T20:52:37.868 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:52:37 smithi070 ceph-mon[168282]: failed to encode map e110 with expected crc 2023-09-23T20:52:37.868 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:52:37 smithi070 ceph-mon[168282]: failed to encode map e110 with expected crc 2023-09-23T20:52:37.890 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:52:37.896 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:52:37.897 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:52:37.997 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:52:38.030 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:52:38.069 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:52:38.074 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:52:38.074 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:52:38.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:52:37 smithi026 ceph-mon[179688]: pgmap v253: 89 pgs: 89 active+clean; 1.8 GiB data, 5.4 GiB used, 620 GiB / 626 GiB avail; 117 MiB/s wr, 1.87k op/s 2023-09-23T20:52:38.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:52:37 smithi026 ceph-mon[179688]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T20:52:38.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:52:37 smithi026 ceph-mon[179688]: Cluster is now healthy 2023-09-23T20:52:38.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:52:37 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T20:52:38.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:52:37 smithi026 ceph-mon[179688]: osdmap e110: 8 total, 8 up, 7 in 2023-09-23T20:52:38.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:52:37 smithi026 ceph-mon[179688]: failed to encode map e110 with expected crc 2023-09-23T20:52:38.134 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:52:37 smithi026 ceph-mon[179688]: failed to encode map e110 with expected crc 2023-09-23T20:52:38.134 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:52:37 smithi026 ceph-mon[179688]: failed to encode map e110 with expected crc 2023-09-23T20:52:38.134 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:52:37 smithi026 ceph-mon[179688]: failed to encode map e110 with expected crc 2023-09-23T20:52:38.134 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:52:37 smithi026 ceph-mon[175151]: pgmap v253: 89 pgs: 89 active+clean; 1.8 GiB data, 5.4 GiB used, 620 GiB / 626 GiB avail; 117 MiB/s wr, 1.87k op/s 2023-09-23T20:52:38.134 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:52:37 smithi026 ceph-mon[175151]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T20:52:38.134 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:52:37 smithi026 ceph-mon[175151]: Cluster is now healthy 2023-09-23T20:52:38.134 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:52:37 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T20:52:38.134 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:52:37 smithi026 ceph-mon[175151]: osdmap e110: 8 total, 8 up, 7 in 2023-09-23T20:52:38.135 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:52:37 smithi026 ceph-mon[175151]: failed to encode map e110 with expected crc 2023-09-23T20:52:38.135 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:52:37 smithi026 ceph-mon[175151]: failed to encode map e110 with expected crc 2023-09-23T20:52:38.135 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:52:37 smithi026 ceph-mon[175151]: failed to encode map e110 with expected crc 2023-09-23T20:52:38.135 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:52:37 smithi026 ceph-mon[175151]: failed to encode map e110 with expected crc 2023-09-23T20:52:38.175 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:52:38.237 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:52:38.290 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:52:38.302 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:52:38.311 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:52:38.311 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:52:38.412 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:52:38.482 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:52:38.555 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:52:38.740 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:52:38.748 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:52:38.749 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:52:38.845 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 19 16 36578 36562 120.152 86.0625 0.00667861 0.00831722 2023-09-23T20:52:38.850 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:52:38.946 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:52:39.187 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:52:38 smithi026 ceph-mon[179688]: failed to encode map e110 with expected crc 2023-09-23T20:52:39.190 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:52:38 smithi026 ceph-mon[175151]: failed to encode map e110 with expected crc 2023-09-23T20:52:39.227 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:52:39.233 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:52:39.234 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:52:39.335 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:52:39.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:52:38 smithi070 ceph-mon[168282]: failed to encode map e110 with expected crc 2023-09-23T20:52:39.402 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:52:39.481 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:52:39.489 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:52:39.490 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:52:39.591 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:52:39.695 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:52:39.788 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:52:39 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[164029]: ::ffff:172.21.15.70 - - [23/Sep/2023:20:52:39] "GET /metrics HTTP/1.1" 200 36305 "" "Prometheus/2.43.0" 2023-09-23T20:52:39.798 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T20:52:39.846 INFO:tasks.radosbench.radosbench.0.smithi026.stdout:2023-09-23T20:52:39.846420+0000 min lat: 0.00125417 max lat: 0.0886562 avg lat: 0.00845181 2023-09-23T20:52:39.846 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T20:52:39.846 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 20 16 37892 37876 118.252 82.125 0.0155632 0.00845181 2023-09-23T20:52:40.020 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:52:40.025 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:52:40.025 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:52:40.120 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:52:39 smithi026 ceph-mon[179688]: pgmap v255: 89 pgs: 89 active+clean; 1.9 GiB data, 5.9 GiB used, 620 GiB / 626 GiB avail; 103 MiB/s wr, 1.66k op/s 2023-09-23T20:52:40.121 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:52:39 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:52:40.121 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:52:39 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:52:40.121 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:52:39 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/2687494204' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-09-23T20:52:40.121 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:52:39 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-09-23T20:52:40.121 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:52:39 smithi026 ceph-mon[175151]: pgmap v255: 89 pgs: 89 active+clean; 1.9 GiB data, 5.9 GiB used, 620 GiB / 626 GiB avail; 103 MiB/s wr, 1.66k op/s 2023-09-23T20:52:40.121 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:52:39 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:52:40.121 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:52:39 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:52:40.122 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:52:39 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/2687494204' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-09-23T20:52:40.122 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:52:39 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-09-23T20:52:40.126 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:52:40.154 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:52:39 smithi070 ceph-mon[168282]: pgmap v255: 89 pgs: 89 active+clean; 1.9 GiB data, 5.9 GiB used, 620 GiB / 626 GiB avail; 103 MiB/s wr, 1.66k op/s 2023-09-23T20:52:40.154 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:52:39 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:52:40.154 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:52:39 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:52:40.154 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:52:39 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/2687494204' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-09-23T20:52:40.154 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:52:39 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-09-23T20:52:40.156 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:52:40.194 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:52:40.197 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:52:40.197 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:52:40.298 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:52:40.381 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:52:40.453 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:52:40.459 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:52:40.460 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:52:40.561 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:52:40.649 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0: {} 2023-09-23T20:52:40.674 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:52:40.699 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1: {} 2023-09-23T20:52:40.748 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2: {} 2023-09-23T20:52:40.754 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:52:40.766 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:52:40.766 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:52:40.804 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3: {} 2023-09-23T20:52:40.847 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 21 16 39029 39013 115.996 71.0625 0.0170772 0.00861229 2023-09-23T20:52:40.856 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4: {} 2023-09-23T20:52:40.867 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:52:40.891 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:52:40.895 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:52:40.906 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T20:52:40.907 INFO:teuthology.orchestra.run.smithi026.stderr:reweighted osd.3 to 1 (10000) 2023-09-23T20:52:40.918 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:52:40 smithi026 ceph-mon[175151]: pgmap v256: 89 pgs: 89 active+clean; 2.1 GiB data, 6.4 GiB used, 619 GiB / 626 GiB avail; 88 MiB/s wr, 1.40k op/s 2023-09-23T20:52:40.918 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:52:40 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/811455485' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:52:40.918 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:52:40 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-09-23T20:52:40.918 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:52:40 smithi026 ceph-mon[175151]: osdmap e111: 8 total, 8 up, 7 in 2023-09-23T20:52:40.919 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:52:40 smithi026 ceph-mon[179688]: pgmap v256: 89 pgs: 89 active+clean; 2.1 GiB data, 6.4 GiB used, 619 GiB / 626 GiB avail; 88 MiB/s wr, 1.40k op/s 2023-09-23T20:52:40.919 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:52:40 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/811455485' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:52:40.919 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:52:40 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-09-23T20:52:40.919 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:52:40 smithi026 ceph-mon[179688]: osdmap e111: 8 total, 8 up, 7 in 2023-09-23T20:52:40.932 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:52:40.934 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:52:40.935 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:52:40.954 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6: {} 2023-09-23T20:52:40.964 INFO:teuthology.orchestra.run.smithi026.stdout:ERROR: (22) Invalid argument 2023-09-23T20:52:40.964 INFO:teuthology.orchestra.run.smithi026.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:52:41.005 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T20:52:41.018 INFO:teuthology.orchestra.run.smithi026.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T20:52:41.019 INFO:teuthology.orchestra.run.smithi026.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T20:52:41.019 INFO:teuthology.orchestra.run.smithi026.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T20:52:41.019 INFO:teuthology.orchestra.run.smithi026.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T20:52:41.019 INFO:teuthology.orchestra.run.smithi026.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T20:52:41.019 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T20:52:41.019 INFO:teuthology.orchestra.run.smithi026.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T20:52:41.019 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T20:52:41.035 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:52:41.115 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:52:41.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:52:40 smithi070 ceph-mon[168282]: pgmap v256: 89 pgs: 89 active+clean; 2.1 GiB data, 6.4 GiB used, 619 GiB / 626 GiB avail; 88 MiB/s wr, 1.40k op/s 2023-09-23T20:52:41.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:52:40 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/811455485' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:52:41.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:52:40 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-09-23T20:52:41.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:52:40 smithi070 ceph-mon[168282]: osdmap e111: 8 total, 8 up, 7 in 2023-09-23T20:52:41.487 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:52:41.495 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:52:41.495 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:52:41.596 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:52:41.673 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:52:41.848 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 22 15 40433 40418 114.715 87.8125 0.0129543 0.00871045 2023-09-23T20:52:42.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:52:41 smithi026 ceph-mon[179688]: failed to encode map e111 with expected crc 2023-09-23T20:52:42.134 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:52:41 smithi026 ceph-mon[179688]: failed to encode map e111 with expected crc 2023-09-23T20:52:42.134 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:52:41 smithi026 ceph-mon[179688]: failed to encode map e111 with expected crc 2023-09-23T20:52:42.134 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:52:41 smithi026 ceph-mon[179688]: failed to encode map e111 with expected crc 2023-09-23T20:52:42.134 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:52:41 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:52:42.134 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:52:41 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:52:42.134 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:52:41 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:52:42.134 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:52:41 smithi026 ceph-mon[175151]: failed to encode map e111 with expected crc 2023-09-23T20:52:42.135 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:52:41 smithi026 ceph-mon[175151]: failed to encode map e111 with expected crc 2023-09-23T20:52:42.135 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:52:41 smithi026 ceph-mon[175151]: failed to encode map e111 with expected crc 2023-09-23T20:52:42.135 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:52:41 smithi026 ceph-mon[175151]: failed to encode map e111 with expected crc 2023-09-23T20:52:42.135 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:52:41 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:52:42.135 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:52:41 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:52:42.135 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:52:41 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:52:42.339 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_blocked_ops 2023-09-23T20:52:42.359 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:52:42.361 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:52:42.362 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:52:42.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:52:41 smithi070 ceph-mon[168282]: failed to encode map e111 with expected crc 2023-09-23T20:52:42.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:52:41 smithi070 ceph-mon[168282]: failed to encode map e111 with expected crc 2023-09-23T20:52:42.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:52:41 smithi070 ceph-mon[168282]: failed to encode map e111 with expected crc 2023-09-23T20:52:42.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:52:41 smithi070 ceph-mon[168282]: failed to encode map e111 with expected crc 2023-09-23T20:52:42.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:52:41 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:52:42.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:52:41 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:52:42.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:52:41 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:52:42.463 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:52:42.530 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph osd reweight 4 1 2023-09-23T20:52:42.566 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:52:42.680 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:52:42.686 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:52:42.686 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:52:42.787 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:52:42.818 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:52:42.848 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 23 16 41821 41805 113.497 86.6875 0.026918 0.00880308 2023-09-23T20:52:42.856 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:52:42.861 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:52:42.861 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:52:42.963 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:52:42.993 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:52:43.031 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:52:43.033 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:52:43.034 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:52:43.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:52:42 smithi070 ceph-mon[168282]: pgmap v258: 89 pgs: 89 active+clean; 2.4 GiB data, 7.3 GiB used, 619 GiB / 626 GiB avail; 89 MiB/s wr, 1.43k op/s 2023-09-23T20:52:43.135 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:52:43.138 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:52:42 smithi026 ceph-mon[179688]: pgmap v258: 89 pgs: 89 active+clean; 2.4 GiB data, 7.3 GiB used, 619 GiB / 626 GiB avail; 89 MiB/s wr, 1.43k op/s 2023-09-23T20:52:43.138 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:52:42 smithi026 ceph-mon[175151]: pgmap v258: 89 pgs: 89 active+clean; 2.4 GiB data, 7.3 GiB used, 619 GiB / 626 GiB avail; 89 MiB/s wr, 1.43k op/s 2023-09-23T20:52:43.167 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:52:43.206 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:52:43.209 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:52:43.209 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:52:43.274 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:52:43.310 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:52:43.339 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:52:43.378 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:52:43.382 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:52:43.382 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:52:43.483 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:52:43.515 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:52:43.559 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:52:43.562 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:52:43.562 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:52:43.570 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:52:43.663 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:52:43.696 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:52:43.744 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:52:43.747 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:52:43.747 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:52:43.841 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T20:52:43.848 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:52:43.848 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 24 15 43134 43119 112.191 82.125 0.00438436 0.00890808 2023-09-23T20:52:43.944 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:52:44.008 INFO:teuthology.orchestra.run.smithi026.stderr:noscrub is set 2023-09-23T20:52:44.034 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:52:44.039 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:52:44.039 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:52:44.141 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:52:44.172 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:52:44.215 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:52:44.218 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:52:44.218 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:52:44.319 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:52:44.355 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:52:43 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/2506939894' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:52:44.355 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:52:43 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:52:44.355 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:52:44.381 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:52:43 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/2506939894' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:52:44.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:52:43 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:52:44.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:52:43 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/2506939894' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:52:44.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:52:43 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:52:44.402 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:52:44.405 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:52:44.405 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:52:44.506 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:52:44.544 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:52:44.587 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:52:44.590 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:52:44.590 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:52:44.691 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:52:44.718 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:52:44.750 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:52:44.752 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:52:44.753 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:52:44.851 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 25 16 44458 44442 111.001 82.6875 0.00662509 0.00900269 2023-09-23T20:52:44.853 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:52:44.944 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:52:45.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:52:44 smithi070 ceph-mon[168282]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:52:45.371 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:52:44 smithi070 ceph-mon[168282]: failed to encode map e112 with expected crc 2023-09-23T20:52:45.371 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:52:44 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T20:52:45.371 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:52:44 smithi070 ceph-mon[168282]: osdmap e112: 8 total, 8 up, 7 in 2023-09-23T20:52:45.371 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:52:44 smithi070 ceph-mon[168282]: failed to encode map e112 with expected crc 2023-09-23T20:52:45.371 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:52:44 smithi070 ceph-mon[168282]: pgmap v260: 89 pgs: 89 active+clean; 2.5 GiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 110 MiB/s wr, 1.75k op/s 2023-09-23T20:52:45.383 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:52:44 smithi026 ceph-mon[179688]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:52:45.383 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:52:44 smithi026 ceph-mon[179688]: failed to encode map e112 with expected crc 2023-09-23T20:52:45.383 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:52:44 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T20:52:45.383 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:52:44 smithi026 ceph-mon[179688]: osdmap e112: 8 total, 8 up, 7 in 2023-09-23T20:52:45.383 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:52:44 smithi026 ceph-mon[179688]: failed to encode map e112 with expected crc 2023-09-23T20:52:45.383 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:52:44 smithi026 ceph-mon[179688]: pgmap v260: 89 pgs: 89 active+clean; 2.5 GiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 110 MiB/s wr, 1.75k op/s 2023-09-23T20:52:45.383 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:52:44 smithi026 ceph-mon[175151]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:52:45.384 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:52:44 smithi026 ceph-mon[175151]: failed to encode map e112 with expected crc 2023-09-23T20:52:45.384 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:52:44 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T20:52:45.384 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:52:44 smithi026 ceph-mon[175151]: osdmap e112: 8 total, 8 up, 7 in 2023-09-23T20:52:45.384 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:52:44 smithi026 ceph-mon[175151]: failed to encode map e112 with expected crc 2023-09-23T20:52:45.384 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:52:44 smithi026 ceph-mon[175151]: pgmap v260: 89 pgs: 89 active+clean; 2.5 GiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 110 MiB/s wr, 1.75k op/s 2023-09-23T20:52:45.851 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 26 16 46029 46013 110.505 98.1875 0.00950491 0.00904474 2023-09-23T20:52:46.200 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:52:46 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:52:46.572 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:52:46 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:52:46.615 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:52:46.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:52:46 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:52:46.625 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:52:46.625 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:52:46.726 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:52:46.758 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:52:46.799 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:52:46.803 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:52:46.804 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:52:46.851 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 27 16 47474 47458 109.757 90.3125 0.00251525 0.00910467 2023-09-23T20:52:46.905 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:52:46.939 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:52:46.981 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:52:46.984 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:52:46.984 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:52:47.085 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:52:47.179 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:52:47.265 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:52:47.269 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:52:47.270 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:52:47.354 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:52:47.371 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:52:47.375 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:52:47 smithi026 ceph-mon[179688]: pgmap v261: 89 pgs: 89 active+clean; 2.6 GiB data, 8.0 GiB used, 618 GiB / 626 GiB avail; 87 MiB/s wr, 1.40k op/s 2023-09-23T20:52:47.375 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:52:47 smithi026 ceph-mon[175151]: pgmap v261: 89 pgs: 89 active+clean; 2.6 GiB data, 8.0 GiB used, 618 GiB / 626 GiB avail; 87 MiB/s wr, 1.40k op/s 2023-09-23T20:52:47.402 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:52:47.414 INFO:teuthology.orchestra.run.smithi026.stdout:ERROR: (22) Invalid argument 2023-09-23T20:52:47.414 INFO:teuthology.orchestra.run.smithi026.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:52:47.441 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:52:47.444 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:52:47.444 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:52:47.545 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:52:47.621 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:52:47 smithi070 ceph-mon[168282]: pgmap v261: 89 pgs: 89 active+clean; 2.6 GiB data, 8.0 GiB used, 618 GiB / 626 GiB avail; 87 MiB/s wr, 1.40k op/s 2023-09-23T20:52:47.621 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:52:47.852 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 28 15 48651 48636 108.468 73.625 0.0129009 0.00921348 2023-09-23T20:52:48.044 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:52:48.050 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:52:48.050 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:52:48.121 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_historic_ops 2023-09-23T20:52:48.151 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:52:48.244 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:52:48.307 INFO:teuthology.orchestra.run.smithi026.stderr:reweighted osd.4 to 1 (10000) 2023-09-23T20:52:48.345 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:52:48.351 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:52:48.351 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:52:48.452 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:52:48.517 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:52:48 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/1816447226' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-09-23T20:52:48.517 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:52:48 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-09-23T20:52:48.517 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:52:48 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/1816447226' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-09-23T20:52:48.517 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:52:48 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-09-23T20:52:48.526 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:52:48.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:52:48 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/1816447226' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-09-23T20:52:48.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:52:48 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-09-23T20:52:48.685 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T20:52:48.851 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 29 16 49852 49836 107.314 75 0.0056237 0.00931146 2023-09-23T20:52:48.870 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:52:48.878 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:52:48.878 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:52:48.879 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph osd reweight 6 1 2023-09-23T20:52:48.979 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:52:49.078 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:52:49.328 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:52:49.368 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:52:49.379 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:52:49.379 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:52:49.480 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:52:49.553 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:52:49 smithi026 ceph-mon[175151]: pgmap v262: 89 pgs: 89 active+clean; 2.8 GiB data, 8.5 GiB used, 617 GiB / 626 GiB avail; 82 MiB/s wr, 1.31k op/s 2023-09-23T20:52:49.553 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:52:49 smithi026 ceph-mon[175151]: failed to encode map e113 with expected crc 2023-09-23T20:52:49.553 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:52:49 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-09-23T20:52:49.553 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:52:49 smithi026 ceph-mon[175151]: osdmap e113: 8 total, 8 up, 7 in 2023-09-23T20:52:49.554 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:52:49 smithi026 ceph-mon[175151]: failed to encode map e113 with expected crc 2023-09-23T20:52:49.557 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:52:49 smithi026 ceph-mon[179688]: pgmap v262: 89 pgs: 89 active+clean; 2.8 GiB data, 8.5 GiB used, 617 GiB / 626 GiB avail; 82 MiB/s wr, 1.31k op/s 2023-09-23T20:52:49.558 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:52:49 smithi026 ceph-mon[179688]: failed to encode map e113 with expected crc 2023-09-23T20:52:49.558 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:52:49 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-09-23T20:52:49.558 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:52:49 smithi026 ceph-mon[179688]: osdmap e113: 8 total, 8 up, 7 in 2023-09-23T20:52:49.558 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:52:49 smithi026 ceph-mon[179688]: failed to encode map e113 with expected crc 2023-09-23T20:52:49.567 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:52:49.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:52:49 smithi070 ceph-mon[168282]: pgmap v262: 89 pgs: 89 active+clean; 2.8 GiB data, 8.5 GiB used, 617 GiB / 626 GiB avail; 82 MiB/s wr, 1.31k op/s 2023-09-23T20:52:49.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:52:49 smithi070 ceph-mon[168282]: failed to encode map e113 with expected crc 2023-09-23T20:52:49.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:52:49 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-09-23T20:52:49.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:52:49 smithi070 ceph-mon[168282]: osdmap e113: 8 total, 8 up, 7 in 2023-09-23T20:52:49.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:52:49 smithi070 ceph-mon[168282]: failed to encode map e113 with expected crc 2023-09-23T20:52:49.695 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:52:49.699 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:52:49.700 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:52:49.768 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0: {} 2023-09-23T20:52:49.801 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:52:49.809 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1: {} 2023-09-23T20:52:49.836 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:52:49 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[164029]: ::ffff:172.21.15.70 - - [23/Sep/2023:20:52:49] "GET /metrics HTTP/1.1" 200 36308 "" "Prometheus/2.43.0" 2023-09-23T20:52:49.843 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2: {} 2023-09-23T20:52:49.857 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 30 16 50873 50857 105.847 63.8125 0.0126641 0.00944184 2023-09-23T20:52:49.877 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:52:49.879 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3: {} 2023-09-23T20:52:49.920 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4: {} 2023-09-23T20:52:49.956 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T20:52:49.991 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6: {} 2023-09-23T20:52:50.026 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T20:52:50.057 INFO:teuthology.orchestra.run.smithi026.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T20:52:50.058 INFO:teuthology.orchestra.run.smithi026.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T20:52:50.058 INFO:teuthology.orchestra.run.smithi026.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T20:52:50.058 INFO:teuthology.orchestra.run.smithi026.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T20:52:50.058 INFO:teuthology.orchestra.run.smithi026.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T20:52:50.058 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T20:52:50.058 INFO:teuthology.orchestra.run.smithi026.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T20:52:50.058 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T20:52:50.106 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:52:50.110 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:52:50.111 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:52:50.184 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:52:50.212 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:52:50.301 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:52:50.383 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:52:50 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/4166580754' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:52:50.383 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:52:50 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/4166580754' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:52:50.598 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:52:50.607 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:52:50.607 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:52:50.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:52:50 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/4166580754' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:52:50.708 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:52:50.742 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:52:50.768 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:52:50.782 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:52:50.785 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:52:50.786 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:52:50.857 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 31 16 52090 52074 104.887 76.0625 0.00879382 0.00952756 2023-09-23T20:52:50.886 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:52:50.917 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:52:50.960 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:52:50.964 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:52:50.964 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:52:51.065 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:52:51.099 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:52:51.138 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:52:51.142 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:52:51.142 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:52:51.243 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:52:51.324 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:52:51.412 INFO:teuthology.orchestra.run.smithi026.stdout:{ 2023-09-23T20:52:51.412 INFO:teuthology.orchestra.run.smithi026.stdout: "size": 20, 2023-09-23T20:52:51.412 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 600, 2023-09-23T20:52:51.413 INFO:teuthology.orchestra.run.smithi026.stdout: "ops": [ 2023-09-23T20:52:51.413 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:51.413 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34797.0:5581 6.f 6:f9876e9a:::benchmark_data_smithi026_223868_object5580:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e106)", 2023-09-23T20:52:51.413 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T20:52:21.793639+0000", 2023-09-23T20:52:51.413 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 29.615399249999999, 2023-09-23T20:52:51.413 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.080631881000000002, 2023-09-23T20:52:51.413 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T20:52:51.413 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:52:51.413 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T20:52:51.413 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34797", 2023-09-23T20:52:51.413 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T20:52:51.413 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 5581 2023-09-23T20:52:51.413 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:51.414 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T20:52:51.414 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:51.414 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T20:52:51.414 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:21.793639+0000", 2023-09-23T20:52:51.414 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:52:51.414 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:51.414 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:51.414 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T20:52:51.414 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:21.793639+0000", 2023-09-23T20:52:51.414 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:52:51.414 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:51.414 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:51.415 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T20:52:51.415 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:21.793637+0000", 2023-09-23T20:52:51.415 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T20:52:51.415 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:51.415 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:51.415 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T20:52:51.415 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:21.793661+0000", 2023-09-23T20:52:51.415 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.4215000000000001e-05 2023-09-23T20:52:51.415 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:51.415 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:51.415 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T20:52:51.415 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:21.793673+0000", 2023-09-23T20:52:51.415 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.2011000000000001e-05 2023-09-23T20:52:51.415 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:51.416 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:51.416 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T20:52:51.416 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:21.793681+0000", 2023-09-23T20:52:51.416 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.2069999999999995e-06 2023-09-23T20:52:51.416 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:51.416 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:51.416 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:52:51.416 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:21.793739+0000", 2023-09-23T20:52:51.416 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.7176000000000003e-05 2023-09-23T20:52:51.416 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:51.416 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:51.416 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T20:52:51.416 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:21.793869+0000", 2023-09-23T20:52:51.417 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00012998400000000001 2023-09-23T20:52:51.417 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:51.417 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:51.417 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 3,4", 2023-09-23T20:52:51.417 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:21.794008+0000", 2023-09-23T20:52:51.417 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00013893900000000001 2023-09-23T20:52:51.417 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:51.417 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:51.417 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T20:52:51.417 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:21.794668+0000", 2023-09-23T20:52:51.417 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00066021500000000002 2023-09-23T20:52:51.417 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:51.417 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:51.418 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:52:51.418 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:21.797987+0000", 2023-09-23T20:52:51.418 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.003319383 2023-09-23T20:52:51.418 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:51.418 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:51.418 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:52:51.418 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:21.874167+0000", 2023-09-23T20:52:51.418 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.076179809000000001 2023-09-23T20:52:51.418 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:51.418 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:51.418 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T20:52:51.418 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:21.874227+0000", 2023-09-23T20:52:51.419 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.9590999999999999e-05 2023-09-23T20:52:51.419 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:51.419 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:51.419 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T20:52:51.419 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:21.874270+0000", 2023-09-23T20:52:51.419 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.3908e-05 2023-09-23T20:52:51.419 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:52:51.419 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T20:52:51.419 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:52:51.419 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:51.419 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:51.419 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34797.0:5587 6.f 6:fbd70a37:::benchmark_data_smithi026_223868_object5586:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e106)", 2023-09-23T20:52:51.419 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T20:52:21.804013+0000", 2023-09-23T20:52:51.420 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 29.605025216000001, 2023-09-23T20:52:51.420 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.070379024999999998, 2023-09-23T20:52:51.420 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T20:52:51.420 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:52:51.420 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T20:52:51.420 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34797", 2023-09-23T20:52:51.420 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T20:52:51.420 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 5587 2023-09-23T20:52:51.420 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:51.420 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T20:52:51.420 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:51.420 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T20:52:51.420 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:21.804013+0000", 2023-09-23T20:52:51.420 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:52:51.421 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:51.421 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:51.421 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T20:52:51.421 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:21.804013+0000", 2023-09-23T20:52:51.421 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:52:51.421 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:51.421 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:51.421 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T20:52:51.421 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:21.804011+0000", 2023-09-23T20:52:51.421 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999981 2023-09-23T20:52:51.421 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:51.421 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:51.421 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T20:52:51.422 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:21.804038+0000", 2023-09-23T20:52:51.422 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.7861e-05 2023-09-23T20:52:51.422 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:51.422 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:51.422 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T20:52:51.422 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:21.804057+0000", 2023-09-23T20:52:51.422 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.8921000000000001e-05 2023-09-23T20:52:51.422 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:51.422 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:51.422 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T20:52:51.422 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:21.804067+0000", 2023-09-23T20:52:51.422 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.8700000000000004e-06 2023-09-23T20:52:51.423 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:51.423 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:51.423 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:52:51.423 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:21.804132+0000", 2023-09-23T20:52:51.423 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.4844999999999993e-05 2023-09-23T20:52:51.423 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:51.423 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:51.423 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T20:52:51.423 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:21.804314+0000", 2023-09-23T20:52:51.423 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00018207999999999999 2023-09-23T20:52:51.423 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:51.423 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:51.423 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 3,4", 2023-09-23T20:52:51.424 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:21.804491+0000", 2023-09-23T20:52:51.424 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000177226 2023-09-23T20:52:51.424 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:51.424 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:51.424 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T20:52:51.424 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:21.805944+0000", 2023-09-23T20:52:51.424 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00145265 2023-09-23T20:52:51.424 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:51.424 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:51.424 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:52:51.424 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:21.806669+0000", 2023-09-23T20:52:51.424 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00072490399999999996 2023-09-23T20:52:51.424 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:51.425 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:51.425 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:52:51.425 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:21.874310+0000", 2023-09-23T20:52:51.425 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.067640699999999998 2023-09-23T20:52:51.425 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:51.425 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:51.425 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T20:52:51.425 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:21.874356+0000", 2023-09-23T20:52:51.425 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.6125999999999998e-05 2023-09-23T20:52:51.425 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:51.425 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:51.425 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T20:52:51.425 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:21.874392+0000", 2023-09-23T20:52:51.426 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.5876e-05 2023-09-23T20:52:51.426 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:52:51.426 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T20:52:51.426 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:52:51.426 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:51.426 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:51.426 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34797.0:5590 6.f 6:fca0bdcf:::benchmark_data_smithi026_223868_object5589:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e106)", 2023-09-23T20:52:51.426 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T20:52:21.804085+0000", 2023-09-23T20:52:51.426 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 29.604952957999998, 2023-09-23T20:52:51.426 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.070434017000000002, 2023-09-23T20:52:51.426 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T20:52:51.426 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:52:51.427 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T20:52:51.427 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34797", 2023-09-23T20:52:51.427 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T20:52:51.427 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 5590 2023-09-23T20:52:51.427 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:51.427 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T20:52:51.427 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:51.427 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T20:52:51.427 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:21.804085+0000", 2023-09-23T20:52:51.427 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:52:51.427 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:51.427 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:51.428 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T20:52:51.428 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:21.804085+0000", 2023-09-23T20:52:51.428 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:52:51.428 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:51.428 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:51.428 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T20:52:51.428 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:21.804084+0000", 2023-09-23T20:52:51.428 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T20:52:51.428 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:51.428 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:51.428 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T20:52:51.428 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:21.804107+0000", 2023-09-23T20:52:51.428 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.2421000000000001e-05 2023-09-23T20:52:51.429 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:51.429 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:51.429 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T20:52:51.429 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:21.804114+0000", 2023-09-23T20:52:51.429 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.6979999999999998e-06 2023-09-23T20:52:51.429 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:51.429 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:51.429 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T20:52:51.429 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:21.804121+0000", 2023-09-23T20:52:51.429 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.1469999999999998e-06 2023-09-23T20:52:51.429 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:51.430 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:51.430 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:52:51.430 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:21.804992+0000", 2023-09-23T20:52:51.430 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00087129000000000004 2023-09-23T20:52:51.430 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:51.430 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:51.430 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T20:52:51.430 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:21.805116+0000", 2023-09-23T20:52:51.430 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00012401600000000001 2023-09-23T20:52:51.430 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:51.430 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:51.430 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 3,4", 2023-09-23T20:52:51.431 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:21.805271+0000", 2023-09-23T20:52:51.431 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000154687 2023-09-23T20:52:51.431 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:51.431 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:51.431 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T20:52:51.431 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:21.806576+0000", 2023-09-23T20:52:51.431 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.001305795 2023-09-23T20:52:51.431 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:51.431 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:51.431 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:52:51.431 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:21.806934+0000", 2023-09-23T20:52:51.431 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00035793400000000002 2023-09-23T20:52:51.431 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:51.431 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:51.432 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:52:51.432 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:21.874426+0000", 2023-09-23T20:52:51.432 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.067491222000000003 2023-09-23T20:52:51.432 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:51.432 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:51.432 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T20:52:51.432 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:21.874467+0000", 2023-09-23T20:52:51.432 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.1768999999999999e-05 2023-09-23T20:52:51.432 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:51.432 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:51.432 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T20:52:51.432 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:21.874519+0000", 2023-09-23T20:52:51.432 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.1548000000000002e-05 2023-09-23T20:52:51.432 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:52:51.433 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T20:52:51.433 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:52:51.433 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:51.433 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:51.433 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34797.0:5606 6.f 6:f7f59783:::benchmark_data_smithi026_223868_object5605:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e106)", 2023-09-23T20:52:51.433 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T20:52:21.815135+0000", 2023-09-23T20:52:51.433 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 29.593903321999999, 2023-09-23T20:52:51.433 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.059534789999999997, 2023-09-23T20:52:51.433 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T20:52:51.433 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:52:51.433 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T20:52:51.433 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34797", 2023-09-23T20:52:51.434 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T20:52:51.434 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 5606 2023-09-23T20:52:51.434 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:51.434 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T20:52:51.434 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:51.434 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T20:52:51.434 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:21.815135+0000", 2023-09-23T20:52:51.434 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:52:51.434 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:51.434 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:51.434 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T20:52:51.434 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:21.815135+0000", 2023-09-23T20:52:51.435 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:52:51.435 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:51.435 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:51.435 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T20:52:51.435 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:21.815132+0000", 2023-09-23T20:52:51.435 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999981 2023-09-23T20:52:51.435 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:51.435 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:51.435 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T20:52:51.435 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:21.815160+0000", 2023-09-23T20:52:51.435 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.7328999999999999e-05 2023-09-23T20:52:51.435 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:51.435 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:51.435 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T20:52:51.436 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:21.815180+0000", 2023-09-23T20:52:51.436 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.035e-05 2023-09-23T20:52:51.436 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:51.436 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:51.436 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T20:52:51.436 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:21.815188+0000", 2023-09-23T20:52:51.436 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.3799999999999994e-06 2023-09-23T20:52:51.436 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:51.436 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:51.436 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:52:51.436 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:21.815243+0000", 2023-09-23T20:52:51.436 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.4536000000000001e-05 2023-09-23T20:52:51.436 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:51.437 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:51.437 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T20:52:51.437 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:21.815404+0000", 2023-09-23T20:52:51.437 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00016117700000000001 2023-09-23T20:52:51.437 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:51.437 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:51.437 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 3,4", 2023-09-23T20:52:51.437 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:21.815574+0000", 2023-09-23T20:52:51.437 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00017022000000000001 2023-09-23T20:52:51.437 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:51.437 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:51.437 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T20:52:51.437 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:21.816340+0000", 2023-09-23T20:52:51.438 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00076515700000000003 2023-09-23T20:52:51.438 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:51.438 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:51.438 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:52:51.438 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:21.819792+0000", 2023-09-23T20:52:51.438 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.003451937 2023-09-23T20:52:51.438 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:51.438 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:51.438 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:52:51.438 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:21.874587+0000", 2023-09-23T20:52:51.438 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.054795439000000001 2023-09-23T20:52:51.438 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:51.438 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:51.439 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T20:52:51.439 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:21.874637+0000", 2023-09-23T20:52:51.439 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.9790000000000003e-05 2023-09-23T20:52:51.439 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:51.439 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:51.439 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T20:52:51.439 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:21.874669+0000", 2023-09-23T20:52:51.439 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.2561000000000002e-05 2023-09-23T20:52:51.439 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:52:51.439 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T20:52:51.439 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:52:51.439 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:51.439 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:51.440 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34797.0:5616 6.f 6:f1b3da43:::benchmark_data_smithi026_223868_object5615:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e106)", 2023-09-23T20:52:51.440 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T20:52:21.831946+0000", 2023-09-23T20:52:51.440 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 29.577091886000002, 2023-09-23T20:52:51.440 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.042830383, 2023-09-23T20:52:51.440 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T20:52:51.440 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:52:51.440 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T20:52:51.440 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34797", 2023-09-23T20:52:51.440 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T20:52:51.440 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 5616 2023-09-23T20:52:51.440 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:51.440 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T20:52:51.440 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:51.441 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T20:52:51.441 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:21.831946+0000", 2023-09-23T20:52:51.441 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:52:51.441 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:51.441 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:51.441 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T20:52:51.441 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:21.831946+0000", 2023-09-23T20:52:51.441 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:52:51.441 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:51.441 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:51.441 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T20:52:51.441 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:21.831944+0000", 2023-09-23T20:52:51.442 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999981 2023-09-23T20:52:51.442 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:51.442 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:51.442 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T20:52:51.442 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:21.831971+0000", 2023-09-23T20:52:51.442 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.7078000000000001e-05 2023-09-23T20:52:51.442 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:51.442 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:51.442 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T20:52:51.442 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:21.831990+0000", 2023-09-23T20:52:51.442 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.8638e-05 2023-09-23T20:52:51.442 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:51.443 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:51.443 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T20:52:51.443 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:21.832000+0000", 2023-09-23T20:52:51.443 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0499000000000001e-05 2023-09-23T20:52:51.443 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:51.443 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:51.443 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:52:51.443 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:21.832064+0000", 2023-09-23T20:52:51.443 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.3462999999999999e-05 2023-09-23T20:52:51.443 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:51.443 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:51.443 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T20:52:51.443 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:21.832281+0000", 2023-09-23T20:52:51.444 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000217414 2023-09-23T20:52:51.444 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:51.444 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:51.444 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 3,4", 2023-09-23T20:52:51.444 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:21.832464+0000", 2023-09-23T20:52:51.444 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0001828 2023-09-23T20:52:51.444 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:51.444 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:51.444 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T20:52:51.444 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:21.833115+0000", 2023-09-23T20:52:51.444 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00065129800000000002 2023-09-23T20:52:51.444 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:51.445 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:51.445 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:52:51.445 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:21.833441+0000", 2023-09-23T20:52:51.445 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00032532699999999998 2023-09-23T20:52:51.445 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:51.445 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:51.445 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:52:51.445 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:21.874709+0000", 2023-09-23T20:52:51.445 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.041267882999999998 2023-09-23T20:52:51.445 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:51.445 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:51.445 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T20:52:51.446 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:21.874746+0000", 2023-09-23T20:52:51.446 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.7894e-05 2023-09-23T20:52:51.446 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:51.446 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:51.446 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T20:52:51.446 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:21.874776+0000", 2023-09-23T20:52:51.446 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.9865999999999998e-05 2023-09-23T20:52:51.446 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:52:51.446 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T20:52:51.446 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:52:51.446 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:51.446 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:51.447 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34797.0:8132 6.f 6:f24b17c8:::benchmark_data_smithi026_223868_object8131:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e107)", 2023-09-23T20:52:51.447 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T20:52:22.932591+0000", 2023-09-23T20:52:51.447 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 28.476446510999999, 2023-09-23T20:52:51.447 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.024257468000000001, 2023-09-23T20:52:51.447 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T20:52:51.447 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:52:51.447 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T20:52:51.447 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34797", 2023-09-23T20:52:51.447 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T20:52:51.447 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 8132 2023-09-23T20:52:51.447 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:51.447 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T20:52:51.447 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:51.448 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T20:52:51.448 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:22.932591+0000", 2023-09-23T20:52:51.448 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:52:51.448 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:51.448 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:51.448 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T20:52:51.448 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:22.932591+0000", 2023-09-23T20:52:51.448 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:52:51.448 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:51.448 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:51.448 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T20:52:51.448 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:22.932590+0000", 2023-09-23T20:52:51.448 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999981 2023-09-23T20:52:51.449 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:51.449 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:51.449 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T20:52:51.449 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:22.932620+0000", 2023-09-23T20:52:51.449 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.0148e-05 2023-09-23T20:52:51.449 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:51.449 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:51.449 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T20:52:51.449 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:22.932634+0000", 2023-09-23T20:52:51.449 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.4052e-05 2023-09-23T20:52:51.449 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:51.450 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:51.450 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T20:52:51.450 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:22.932643+0000", 2023-09-23T20:52:51.450 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.6339999999999996e-06 2023-09-23T20:52:51.450 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:51.450 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:51.450 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:52:51.450 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:22.935390+0000", 2023-09-23T20:52:51.450 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0027461510000000001 2023-09-23T20:52:51.450 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:51.450 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:51.451 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T20:52:51.451 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:22.935540+0000", 2023-09-23T20:52:51.451 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00015056499999999999 2023-09-23T20:52:51.451 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:51.451 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:51.451 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 3,4", 2023-09-23T20:52:51.451 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:22.935673+0000", 2023-09-23T20:52:51.451 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000133127 2023-09-23T20:52:51.451 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:51.451 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:51.451 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:52:51.451 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:22.936752+0000", 2023-09-23T20:52:51.451 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.001079001 2023-09-23T20:52:51.452 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:51.452 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:51.452 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T20:52:51.452 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:22.938721+0000", 2023-09-23T20:52:51.452 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0019691800000000001 2023-09-23T20:52:51.452 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:51.452 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:51.452 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:52:51.452 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:22.956756+0000", 2023-09-23T20:52:51.452 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.018035000999999998 2023-09-23T20:52:51.452 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:51.452 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:51.452 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T20:52:51.453 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:22.956808+0000", 2023-09-23T20:52:51.453 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.1264000000000002e-05 2023-09-23T20:52:51.453 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:51.453 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:51.453 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T20:52:51.453 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:22.956849+0000", 2023-09-23T20:52:51.453 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.1153999999999999e-05 2023-09-23T20:52:51.453 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:52:51.453 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T20:52:51.453 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:52:51.453 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:51.453 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:51.453 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34797.0:27000 6.f 6:ff7560a1:::benchmark_data_smithi026_223868_object26999:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e108)", 2023-09-23T20:52:51.454 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T20:52:32.577606+0000", 2023-09-23T20:52:51.454 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 18.831432336999999, 2023-09-23T20:52:51.454 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.024720974, 2023-09-23T20:52:51.454 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T20:52:51.454 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:52:51.454 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T20:52:51.454 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34797", 2023-09-23T20:52:51.454 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T20:52:51.454 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 27000 2023-09-23T20:52:51.454 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:51.455 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T20:52:51.455 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:51.455 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T20:52:51.455 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:32.577606+0000", 2023-09-23T20:52:51.455 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:52:51.455 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:51.455 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:51.455 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T20:52:51.455 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:32.577606+0000", 2023-09-23T20:52:51.455 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:52:51.455 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:51.455 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:51.455 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T20:52:51.456 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:32.577605+0000", 2023-09-23T20:52:51.456 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T20:52:51.456 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:51.456 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:51.456 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T20:52:51.456 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:32.577627+0000", 2023-09-23T20:52:51.456 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.2565000000000002e-05 2023-09-23T20:52:51.456 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:51.456 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:51.456 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T20:52:51.456 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:32.577635+0000", 2023-09-23T20:52:51.456 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.9359999999999999e-06 2023-09-23T20:52:51.456 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:51.456 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:51.457 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T20:52:51.457 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:32.577642+0000", 2023-09-23T20:52:51.457 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.0979999999999999e-06 2023-09-23T20:52:51.457 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:51.457 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:51.457 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:52:51.457 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:32.578895+0000", 2023-09-23T20:52:51.457 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0012531560000000001 2023-09-23T20:52:51.457 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:51.457 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:51.457 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T20:52:51.457 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:32.578983+0000", 2023-09-23T20:52:51.457 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.8630999999999999e-05 2023-09-23T20:52:51.458 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:51.458 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:51.458 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 3,4", 2023-09-23T20:52:51.458 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:32.579112+0000", 2023-09-23T20:52:51.458 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000128405 2023-09-23T20:52:51.458 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:51.458 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:51.458 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T20:52:51.458 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:32.584668+0000", 2023-09-23T20:52:51.458 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0055560849999999997 2023-09-23T20:52:51.458 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:51.458 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:51.459 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:52:51.459 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:32.590358+0000", 2023-09-23T20:52:51.459 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.005689922 2023-09-23T20:52:51.459 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:51.459 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:51.459 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:52:51.459 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:32.602229+0000", 2023-09-23T20:52:51.459 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.011871404 2023-09-23T20:52:51.459 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:51.459 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:51.459 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T20:52:51.459 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:32.602283+0000", 2023-09-23T20:52:51.459 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.4080999999999999e-05 2023-09-23T20:52:51.459 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:51.460 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:51.460 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T20:52:51.460 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:32.602326+0000", 2023-09-23T20:52:51.460 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.3273999999999999e-05 2023-09-23T20:52:51.460 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:52:51.460 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T20:52:51.460 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:52:51.460 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:51.460 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:51.460 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34797.0:31166 6.f 6:fffd5bad:::benchmark_data_smithi026_223868_object31165:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e109)", 2023-09-23T20:52:51.460 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T20:52:35.031008+0000", 2023-09-23T20:52:51.460 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 16.378029938000001, 2023-09-23T20:52:51.460 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.025566292000000001, 2023-09-23T20:52:51.461 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T20:52:51.461 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:52:51.461 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T20:52:51.461 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34797", 2023-09-23T20:52:51.461 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T20:52:51.461 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 31166 2023-09-23T20:52:51.461 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:51.461 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T20:52:51.461 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:51.461 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T20:52:51.461 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:35.031008+0000", 2023-09-23T20:52:51.461 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:52:51.462 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:51.462 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:51.462 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T20:52:51.462 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:35.031008+0000", 2023-09-23T20:52:51.462 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:52:51.462 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:51.462 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:51.462 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T20:52:51.462 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:35.031006+0000", 2023-09-23T20:52:51.462 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999981 2023-09-23T20:52:51.462 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:51.462 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:51.462 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T20:52:51.463 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:35.031035+0000", 2023-09-23T20:52:51.463 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.8937000000000001e-05 2023-09-23T20:52:51.463 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:51.463 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:51.463 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T20:52:51.463 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:35.031060+0000", 2023-09-23T20:52:51.463 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.478e-05 2023-09-23T20:52:51.463 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:51.463 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:51.463 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T20:52:51.463 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:35.031072+0000", 2023-09-23T20:52:51.463 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.2126e-05 2023-09-23T20:52:51.463 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:51.464 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:51.464 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:52:51.464 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:35.031220+0000", 2023-09-23T20:52:51.464 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00014874500000000001 2023-09-23T20:52:51.464 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:51.464 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:51.464 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T20:52:51.464 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:35.031362+0000", 2023-09-23T20:52:51.464 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00014158199999999999 2023-09-23T20:52:51.464 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:51.464 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:51.464 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 3,4", 2023-09-23T20:52:51.464 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:35.031511+0000", 2023-09-23T20:52:51.465 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00014892599999999999 2023-09-23T20:52:51.465 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:51.465 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:51.465 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T20:52:51.465 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:35.032846+0000", 2023-09-23T20:52:51.465 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00133467 2023-09-23T20:52:51.465 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:51.465 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:51.465 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:52:51.465 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:35.040554+0000", 2023-09-23T20:52:51.465 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0077087479999999996 2023-09-23T20:52:51.465 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:51.465 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:51.465 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:52:51.466 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:35.056469+0000", 2023-09-23T20:52:51.466 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.015914421000000002 2023-09-23T20:52:51.466 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:51.466 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:51.466 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T20:52:51.466 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:35.056537+0000", 2023-09-23T20:52:51.466 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.8172999999999999e-05 2023-09-23T20:52:51.466 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:51.466 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:51.466 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T20:52:51.466 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:35.056574+0000", 2023-09-23T20:52:51.466 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.7267000000000001e-05 2023-09-23T20:52:51.466 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:52:51.467 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T20:52:51.467 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:52:51.467 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:51.467 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:51.467 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34797.0:31176 6.f 6:ff7512c3:::benchmark_data_smithi026_223868_object31175:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e109)", 2023-09-23T20:52:51.467 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T20:52:35.031092+0000", 2023-09-23T20:52:51.467 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 16.377945833999998, 2023-09-23T20:52:51.467 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.025602918999999998, 2023-09-23T20:52:51.467 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T20:52:51.467 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:52:51.467 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T20:52:51.467 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34797", 2023-09-23T20:52:51.467 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T20:52:51.468 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 31176 2023-09-23T20:52:51.468 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:51.468 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T20:52:51.468 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:51.468 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T20:52:51.468 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:35.031092+0000", 2023-09-23T20:52:51.468 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:52:51.468 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:51.468 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:51.468 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T20:52:51.468 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:35.031092+0000", 2023-09-23T20:52:51.468 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:52:51.468 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:51.469 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:51.469 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T20:52:51.469 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:35.031091+0000", 2023-09-23T20:52:51.469 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T20:52:51.469 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:51.469 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:51.469 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T20:52:51.469 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:35.031116+0000", 2023-09-23T20:52:51.469 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.4734000000000001e-05 2023-09-23T20:52:51.469 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:51.469 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:51.469 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T20:52:51.469 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:35.031125+0000", 2023-09-23T20:52:51.469 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.8259999999999999e-06 2023-09-23T20:52:51.470 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:51.470 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:51.470 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T20:52:51.470 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:35.031132+0000", 2023-09-23T20:52:51.470 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.64e-06 2023-09-23T20:52:51.470 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:51.470 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:51.470 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:52:51.470 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:35.034592+0000", 2023-09-23T20:52:51.470 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0034602830000000002 2023-09-23T20:52:51.470 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:51.470 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:51.470 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T20:52:51.471 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:35.034703+0000", 2023-09-23T20:52:51.471 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00011123100000000001 2023-09-23T20:52:51.471 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:51.471 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:51.471 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 3,4", 2023-09-23T20:52:51.471 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:35.034813+0000", 2023-09-23T20:52:51.471 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000109888 2023-09-23T20:52:51.471 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:51.471 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:51.471 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T20:52:51.471 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:35.036052+0000", 2023-09-23T20:52:51.471 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.001239392 2023-09-23T20:52:51.471 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:51.472 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:51.472 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:52:51.472 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:35.042863+0000", 2023-09-23T20:52:51.472 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.006810623 2023-09-23T20:52:51.472 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:51.472 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:51.472 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:52:51.472 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:35.056612+0000", 2023-09-23T20:52:51.472 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.013748862000000001 2023-09-23T20:52:51.472 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:51.472 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:51.472 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T20:52:51.472 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:35.056651+0000", 2023-09-23T20:52:51.473 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.9388000000000003e-05 2023-09-23T20:52:51.473 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:51.473 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:51.473 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T20:52:51.473 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:35.056695+0000", 2023-09-23T20:52:51.473 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.3652000000000002e-05 2023-09-23T20:52:51.473 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:52:51.473 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T20:52:51.473 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:52:51.473 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:51.473 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:51.473 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34797.0:31924 6.f 6:f9e56b9f:::benchmark_data_smithi026_223868_object31923:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e109)", 2023-09-23T20:52:51.473 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T20:52:35.540533+0000", 2023-09-23T20:52:51.474 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 15.868504592000001, 2023-09-23T20:52:51.474 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.028613774000000002, 2023-09-23T20:52:51.474 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T20:52:51.474 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:52:51.474 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T20:52:51.474 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34797", 2023-09-23T20:52:51.474 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T20:52:51.474 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 31924 2023-09-23T20:52:51.474 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:51.474 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T20:52:51.474 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:51.474 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T20:52:51.474 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:35.540533+0000", 2023-09-23T20:52:51.475 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:52:51.475 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:51.475 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:51.475 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T20:52:51.475 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:35.540533+0000", 2023-09-23T20:52:51.475 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:52:51.475 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:51.475 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:51.475 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T20:52:51.475 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:35.540532+0000", 2023-09-23T20:52:51.476 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.999999 2023-09-23T20:52:51.476 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:51.476 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:51.476 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T20:52:51.476 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:35.540571+0000", 2023-09-23T20:52:51.476 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.8464000000000003e-05 2023-09-23T20:52:51.476 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:51.476 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:51.476 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T20:52:51.476 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:35.540588+0000", 2023-09-23T20:52:51.476 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.7558e-05 2023-09-23T20:52:51.476 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:51.476 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:51.477 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T20:52:51.477 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:35.540597+0000", 2023-09-23T20:52:51.477 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.0350000000000007e-06 2023-09-23T20:52:51.477 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:51.477 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:51.477 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:52:51.477 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:35.541823+0000", 2023-09-23T20:52:51.477 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.001225818 2023-09-23T20:52:51.477 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:51.477 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:51.477 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T20:52:51.477 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:35.541945+0000", 2023-09-23T20:52:51.477 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00012213299999999999 2023-09-23T20:52:51.478 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:51.478 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:51.478 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 3,4", 2023-09-23T20:52:51.478 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:35.542075+0000", 2023-09-23T20:52:51.478 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00013027399999999999 2023-09-23T20:52:51.478 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:51.478 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:51.478 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:52:51.478 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:35.545340+0000", 2023-09-23T20:52:51.478 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.003264996 2023-09-23T20:52:51.478 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:51.478 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:51.478 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T20:52:51.479 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:35.546274+0000", 2023-09-23T20:52:51.479 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00093376199999999998 2023-09-23T20:52:51.479 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:51.479 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:51.479 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:52:51.479 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:35.569048+0000", 2023-09-23T20:52:51.479 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.022773890000000001 2023-09-23T20:52:51.479 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:51.479 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:51.479 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T20:52:51.479 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:35.569107+0000", 2023-09-23T20:52:51.479 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.9036999999999997e-05 2023-09-23T20:52:51.479 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:51.479 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:51.480 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T20:52:51.480 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:35.569147+0000", 2023-09-23T20:52:51.480 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.9987999999999997e-05 2023-09-23T20:52:51.480 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:52:51.480 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T20:52:51.480 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:52:51.480 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:51.480 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:51.480 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34797.0:34418 6.f 6:f30e8cc0:::benchmark_data_smithi026_223868_object34417:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e110)", 2023-09-23T20:52:51.480 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T20:52:37.286819+0000", 2023-09-23T20:52:51.480 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 14.122218522000001, 2023-09-23T20:52:51.480 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.024538283000000001, 2023-09-23T20:52:51.481 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T20:52:51.481 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:52:51.481 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T20:52:51.481 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34797", 2023-09-23T20:52:51.481 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T20:52:51.481 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 34418 2023-09-23T20:52:51.481 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:51.481 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T20:52:51.481 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:51.481 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T20:52:51.481 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:37.286819+0000", 2023-09-23T20:52:51.481 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:52:51.481 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:51.482 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:51.482 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T20:52:51.482 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:37.286819+0000", 2023-09-23T20:52:51.482 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:52:51.482 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:51.482 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:51.482 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T20:52:51.482 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:37.286818+0000", 2023-09-23T20:52:51.482 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T20:52:51.482 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:51.482 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:51.482 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T20:52:51.483 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:37.286846+0000", 2023-09-23T20:52:51.483 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.7900000000000001e-05 2023-09-23T20:52:51.483 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:51.483 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:51.483 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T20:52:51.483 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:37.286864+0000", 2023-09-23T20:52:51.483 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.8196999999999999e-05 2023-09-23T20:52:51.483 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:51.483 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:51.483 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T20:52:51.483 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:37.286872+0000", 2023-09-23T20:52:51.483 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.259e-06 2023-09-23T20:52:51.484 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:51.484 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:51.484 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:52:51.484 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:37.292119+0000", 2023-09-23T20:52:51.484 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0052465810000000002 2023-09-23T20:52:51.484 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:51.484 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:51.484 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T20:52:51.484 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:37.292272+0000", 2023-09-23T20:52:51.484 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000153807 2023-09-23T20:52:51.484 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:51.485 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:51.485 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 3,4", 2023-09-23T20:52:51.485 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:37.292435+0000", 2023-09-23T20:52:51.485 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000162993 2023-09-23T20:52:51.485 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:51.485 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:51.485 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T20:52:51.485 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:37.298129+0000", 2023-09-23T20:52:51.485 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0056932720000000001 2023-09-23T20:52:51.485 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:51.485 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:51.485 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:52:51.485 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:37.298351+0000", 2023-09-23T20:52:51.486 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00022195400000000001 2023-09-23T20:52:51.486 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:51.486 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:51.486 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:52:51.486 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:37.311250+0000", 2023-09-23T20:52:51.486 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.012899753999999999 2023-09-23T20:52:51.486 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:51.486 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:51.486 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T20:52:51.486 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:37.311306+0000", 2023-09-23T20:52:51.486 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.5291999999999998e-05 2023-09-23T20:52:51.486 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:51.487 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:51.487 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T20:52:51.487 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:37.311358+0000", 2023-09-23T20:52:51.487 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.1861999999999999e-05 2023-09-23T20:52:51.487 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:52:51.487 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T20:52:51.487 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:52:51.487 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:51.487 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:51.487 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34797.0:34641 6.f 6:f04f2cca:::benchmark_data_smithi026_223868_object34640:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e110)", 2023-09-23T20:52:51.487 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T20:52:37.449772+0000", 2023-09-23T20:52:51.487 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 13.959265406, 2023-09-23T20:52:51.487 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.026184296999999999, 2023-09-23T20:52:51.487 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T20:52:51.488 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:52:51.488 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T20:52:51.488 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34797", 2023-09-23T20:52:51.488 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T20:52:51.488 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 34641 2023-09-23T20:52:51.488 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:51.488 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T20:52:51.488 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:51.488 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T20:52:51.488 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:37.449772+0000", 2023-09-23T20:52:51.488 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:52:51.488 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:51.488 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:51.489 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T20:52:51.489 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:37.449772+0000", 2023-09-23T20:52:51.489 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:52:51.489 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:51.489 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:51.489 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T20:52:51.489 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:37.449772+0000", 2023-09-23T20:52:51.489 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T20:52:51.489 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:51.489 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:51.489 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T20:52:51.489 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:37.449792+0000", 2023-09-23T20:52:51.489 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.9534e-05 2023-09-23T20:52:51.490 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:51.490 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:51.490 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T20:52:51.490 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:37.449798+0000", 2023-09-23T20:52:51.490 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.9630000000000001e-06 2023-09-23T20:52:51.490 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:51.490 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:51.491 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T20:52:51.491 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:37.449804+0000", 2023-09-23T20:52:51.491 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.3519999999999997e-06 2023-09-23T20:52:51.491 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:51.491 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:51.491 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:52:51.491 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:37.454848+0000", 2023-09-23T20:52:51.491 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0050438770000000004 2023-09-23T20:52:51.491 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:51.491 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:51.491 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T20:52:51.491 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:37.454975+0000", 2023-09-23T20:52:51.492 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000127794 2023-09-23T20:52:51.492 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:51.492 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:51.492 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 3,4", 2023-09-23T20:52:51.492 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:37.455066+0000", 2023-09-23T20:52:51.492 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.0224999999999998e-05 2023-09-23T20:52:51.492 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:51.492 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:51.492 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T20:52:51.492 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:37.456420+0000", 2023-09-23T20:52:51.492 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.001354339 2023-09-23T20:52:51.492 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:51.492 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:51.493 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:52:51.493 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:37.460563+0000", 2023-09-23T20:52:51.493 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0041432489999999999 2023-09-23T20:52:51.493 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:51.493 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:51.493 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:52:51.493 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:37.475872+0000", 2023-09-23T20:52:51.493 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.015309 2023-09-23T20:52:51.493 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:51.493 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:51.493 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T20:52:51.493 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:37.475917+0000", 2023-09-23T20:52:51.493 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.4425000000000002e-05 2023-09-23T20:52:51.494 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:51.494 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:51.494 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T20:52:51.494 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:37.475957+0000", 2023-09-23T20:52:51.494 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.0005000000000001e-05 2023-09-23T20:52:51.494 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:52:51.494 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T20:52:51.494 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:52:51.494 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:51.494 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:51.494 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34797.0:36050 6.f 6:f9721691:::benchmark_data_smithi026_223868_object36049:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e110)", 2023-09-23T20:52:51.494 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T20:52:38.443133+0000", 2023-09-23T20:52:51.495 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 12.965904385, 2023-09-23T20:52:51.495 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.027886886, 2023-09-23T20:52:51.495 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T20:52:51.495 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:52:51.495 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T20:52:51.495 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34797", 2023-09-23T20:52:51.495 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T20:52:51.495 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 36050 2023-09-23T20:52:51.496 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:51.496 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T20:52:51.496 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:51.496 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T20:52:51.496 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:38.443133+0000", 2023-09-23T20:52:51.496 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:52:51.496 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:51.496 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:51.496 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T20:52:51.496 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:38.443133+0000", 2023-09-23T20:52:51.496 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:52:51.496 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:51.496 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:51.497 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T20:52:51.497 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:38.443132+0000", 2023-09-23T20:52:51.497 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999981 2023-09-23T20:52:51.497 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:51.497 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:51.497 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T20:52:51.497 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:38.443154+0000", 2023-09-23T20:52:51.497 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.2413999999999999e-05 2023-09-23T20:52:51.497 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:51.497 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:51.497 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T20:52:51.497 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:38.443166+0000", 2023-09-23T20:52:51.497 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.2364999999999999e-05 2023-09-23T20:52:51.498 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:51.498 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:51.498 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T20:52:51.498 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:38.443174+0000", 2023-09-23T20:52:51.498 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.6580000000000002e-06 2023-09-23T20:52:51.498 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:51.498 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:51.498 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:52:51.498 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:38.443223+0000", 2023-09-23T20:52:51.498 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.9021999999999999e-05 2023-09-23T20:52:51.498 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:51.498 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:51.498 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T20:52:51.499 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:38.443353+0000", 2023-09-23T20:52:51.499 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00013024799999999999 2023-09-23T20:52:51.499 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:51.499 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:51.499 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 3,4", 2023-09-23T20:52:51.499 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:38.443506+0000", 2023-09-23T20:52:51.499 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000152158 2023-09-23T20:52:51.499 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:51.499 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:51.499 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T20:52:51.499 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:38.448120+0000", 2023-09-23T20:52:51.499 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0046139830000000003 2023-09-23T20:52:51.500 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:51.500 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:51.500 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:52:51.500 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:38.448244+0000", 2023-09-23T20:52:51.500 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00012423299999999999 2023-09-23T20:52:51.500 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:51.500 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:51.500 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:52:51.500 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:38.470919+0000", 2023-09-23T20:52:51.501 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.022675298999999999 2023-09-23T20:52:51.501 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:51.501 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:51.501 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T20:52:51.501 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:38.470975+0000", 2023-09-23T20:52:51.501 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.5713e-05 2023-09-23T20:52:51.501 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:51.501 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:51.501 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T20:52:51.501 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:38.471020+0000", 2023-09-23T20:52:51.501 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.5578999999999998e-05 2023-09-23T20:52:51.502 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:52:51.502 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T20:52:51.502 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:52:51.502 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:51.502 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:51.503 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34797.0:36069 6.f 6:fa4fc37c:::benchmark_data_smithi026_223868_object36068:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e110)", 2023-09-23T20:52:51.503 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T20:52:38.461168+0000", 2023-09-23T20:52:51.503 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 12.947870019, 2023-09-23T20:52:51.503 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.024636491999999999, 2023-09-23T20:52:51.503 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T20:52:51.503 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:52:51.504 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T20:52:51.504 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34797", 2023-09-23T20:52:51.504 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T20:52:51.504 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 36069 2023-09-23T20:52:51.504 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:51.504 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T20:52:51.505 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:51.505 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T20:52:51.505 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:38.461168+0000", 2023-09-23T20:52:51.505 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:52:51.505 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:51.505 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:51.505 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T20:52:51.505 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:38.461168+0000", 2023-09-23T20:52:51.506 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:52:51.506 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:51.506 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:51.506 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T20:52:51.506 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:38.461166+0000", 2023-09-23T20:52:51.506 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999981 2023-09-23T20:52:51.506 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:51.506 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:51.507 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T20:52:51.507 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:38.461192+0000", 2023-09-23T20:52:51.507 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.5822000000000001e-05 2023-09-23T20:52:51.507 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:51.507 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:51.507 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T20:52:51.507 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:38.461211+0000", 2023-09-23T20:52:51.508 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.9627e-05 2023-09-23T20:52:51.508 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:51.508 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:51.508 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T20:52:51.508 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:38.461221+0000", 2023-09-23T20:52:51.508 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.8800000000000003e-06 2023-09-23T20:52:51.509 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:51.509 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:51.509 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:52:51.509 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:38.461274+0000", 2023-09-23T20:52:51.509 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.2513999999999999e-05 2023-09-23T20:52:51.509 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:51.509 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:51.509 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T20:52:51.509 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:38.461405+0000", 2023-09-23T20:52:51.509 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000131216 2023-09-23T20:52:51.509 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:51.509 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:51.509 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 3,4", 2023-09-23T20:52:51.510 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:38.461581+0000", 2023-09-23T20:52:51.510 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00017586899999999999 2023-09-23T20:52:51.510 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:51.510 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:51.510 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T20:52:51.510 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:38.467145+0000", 2023-09-23T20:52:51.510 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0055647769999999999 2023-09-23T20:52:51.510 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:51.510 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:51.510 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:52:51.510 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:38.471062+0000", 2023-09-23T20:52:51.511 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.003916648 2023-09-23T20:52:51.511 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:51.511 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:51.511 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:52:51.511 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:38.485712+0000", 2023-09-23T20:52:51.511 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.014649798 2023-09-23T20:52:51.511 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:51.511 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:51.511 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T20:52:51.511 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:38.485764+0000", 2023-09-23T20:52:51.511 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.1938000000000003e-05 2023-09-23T20:52:51.512 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:51.512 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:51.512 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T20:52:51.512 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:38.485804+0000", 2023-09-23T20:52:51.512 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.0513999999999999e-05 2023-09-23T20:52:51.512 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:52:51.512 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T20:52:51.512 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:52:51.512 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:51.512 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:51.512 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34797.0:36290 6.f 6:fbb3bd56:::benchmark_data_smithi026_223868_object36289:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e110)", 2023-09-23T20:52:51.512 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T20:52:38.639634+0000", 2023-09-23T20:52:51.513 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 12.769403560000001, 2023-09-23T20:52:51.513 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.02667744, 2023-09-23T20:52:51.513 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T20:52:51.513 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:52:51.513 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T20:52:51.513 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34797", 2023-09-23T20:52:51.513 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T20:52:51.513 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 36290 2023-09-23T20:52:51.513 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:51.513 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T20:52:51.513 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:51.513 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T20:52:51.514 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:38.639634+0000", 2023-09-23T20:52:51.514 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:52:51.514 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:51.514 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:51.514 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T20:52:51.514 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:38.639634+0000", 2023-09-23T20:52:51.514 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:52:51.514 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:51.514 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:51.514 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T20:52:51.514 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:38.639633+0000", 2023-09-23T20:52:51.515 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T20:52:51.515 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:51.515 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:51.515 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T20:52:51.515 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:38.639657+0000", 2023-09-23T20:52:51.515 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.3841000000000001e-05 2023-09-23T20:52:51.515 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:51.515 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:51.515 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T20:52:51.515 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:38.639666+0000", 2023-09-23T20:52:51.515 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.8510000000000005e-06 2023-09-23T20:52:51.515 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:51.515 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:51.516 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T20:52:51.516 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:38.639672+0000", 2023-09-23T20:52:51.516 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.6880000000000002e-06 2023-09-23T20:52:51.516 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:51.516 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:51.516 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:52:51.516 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:38.639730+0000", 2023-09-23T20:52:51.516 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.7664999999999998e-05 2023-09-23T20:52:51.516 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:51.516 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:51.516 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T20:52:51.516 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:38.639855+0000", 2023-09-23T20:52:51.516 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000125356 2023-09-23T20:52:51.517 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:51.517 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:51.517 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 3,4", 2023-09-23T20:52:51.517 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:38.639988+0000", 2023-09-23T20:52:51.517 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00013281799999999999 2023-09-23T20:52:51.517 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:51.517 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:51.517 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T20:52:51.517 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:38.641353+0000", 2023-09-23T20:52:51.517 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.001364433 2023-09-23T20:52:51.517 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:51.517 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:51.517 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:52:51.518 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:38.644444+0000", 2023-09-23T20:52:51.518 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0030912259999999999 2023-09-23T20:52:51.518 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:51.518 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:51.518 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:52:51.518 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:38.666217+0000", 2023-09-23T20:52:51.518 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.021773153999999999 2023-09-23T20:52:51.518 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:51.518 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:51.518 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T20:52:51.518 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:38.666271+0000", 2023-09-23T20:52:51.519 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.4261000000000001e-05 2023-09-23T20:52:51.519 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:51.519 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:51.519 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T20:52:51.519 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:38.666312+0000", 2023-09-23T20:52:51.519 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.0531000000000002e-05 2023-09-23T20:52:51.519 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:52:51.519 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T20:52:51.519 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:52:51.519 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:51.519 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:51.519 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34797.0:38346 6.f 6:f8ba2755:::benchmark_data_smithi026_223868_object38345:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e110)", 2023-09-23T20:52:51.519 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T20:52:40.218268+0000", 2023-09-23T20:52:51.520 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 11.190769692, 2023-09-23T20:52:51.520 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.02327396, 2023-09-23T20:52:51.520 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T20:52:51.520 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:52:51.520 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T20:52:51.520 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34797", 2023-09-23T20:52:51.520 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T20:52:51.520 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 38346 2023-09-23T20:52:51.520 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:51.520 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T20:52:51.520 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:51.520 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T20:52:51.521 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:40.218268+0000", 2023-09-23T20:52:51.521 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:52:51.521 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:51.521 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:51.521 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T20:52:51.521 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:40.218268+0000", 2023-09-23T20:52:51.521 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:52:51.521 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:51.521 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:51.521 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T20:52:51.521 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:40.218259+0000", 2023-09-23T20:52:51.521 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999914 2023-09-23T20:52:51.521 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:51.522 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:51.522 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T20:52:51.522 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:40.218304+0000", 2023-09-23T20:52:51.522 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.4623999999999998e-05 2023-09-23T20:52:51.522 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:51.522 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:51.522 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T20:52:51.522 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:40.218321+0000", 2023-09-23T20:52:51.522 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.7333000000000002e-05 2023-09-23T20:52:51.522 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:51.522 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:51.523 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T20:52:51.523 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:40.218329+0000", 2023-09-23T20:52:51.523 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.8450000000000003e-06 2023-09-23T20:52:51.523 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:51.523 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:51.523 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:52:51.523 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:40.219491+0000", 2023-09-23T20:52:51.523 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.001162141 2023-09-23T20:52:51.523 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:51.523 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:51.523 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T20:52:51.523 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:40.219638+0000", 2023-09-23T20:52:51.523 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000146265 2023-09-23T20:52:51.524 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:51.524 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:51.524 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 3,4", 2023-09-23T20:52:51.524 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:40.219787+0000", 2023-09-23T20:52:51.524 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00014903900000000001 2023-09-23T20:52:51.524 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:51.524 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:51.524 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:52:51.524 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:40.223577+0000", 2023-09-23T20:52:51.524 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.003790491 2023-09-23T20:52:51.524 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:51.524 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:51.525 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T20:52:51.525 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:40.228710+0000", 2023-09-23T20:52:51.525 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0051329410000000002 2023-09-23T20:52:51.525 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:51.525 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:51.525 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:52:51.525 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:40.241462+0000", 2023-09-23T20:52:51.526 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.012751525999999999 2023-09-23T20:52:51.526 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:51.526 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:51.526 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T20:52:51.526 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:40.241512+0000", 2023-09-23T20:52:51.526 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.0597999999999999e-05 2023-09-23T20:52:51.527 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:51.527 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:51.527 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T20:52:51.527 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:40.241542+0000", 2023-09-23T20:52:51.527 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.9882999999999998e-05 2023-09-23T20:52:51.527 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:52:51.527 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T20:52:51.527 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:52:51.527 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:51.528 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:51.528 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34797.0:38347 6.f 6:ffe8ac96:::benchmark_data_smithi026_223868_object38346:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e110)", 2023-09-23T20:52:51.528 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T20:52:40.218352+0000", 2023-09-23T20:52:51.528 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 11.190686075, 2023-09-23T20:52:51.528 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.023309561999999999, 2023-09-23T20:52:51.528 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T20:52:51.528 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:52:51.528 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T20:52:51.528 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34797", 2023-09-23T20:52:51.528 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T20:52:51.528 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 38347 2023-09-23T20:52:51.528 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:51.528 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T20:52:51.529 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:51.529 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T20:52:51.529 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:40.218352+0000", 2023-09-23T20:52:51.529 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:52:51.529 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:51.529 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:51.529 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T20:52:51.529 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:40.218352+0000", 2023-09-23T20:52:51.529 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:52:51.529 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:51.529 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:51.529 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T20:52:51.529 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:40.218351+0000", 2023-09-23T20:52:51.530 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T20:52:51.530 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:51.530 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:51.530 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T20:52:51.530 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:40.218372+0000", 2023-09-23T20:52:51.530 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.05e-05 2023-09-23T20:52:51.530 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:51.530 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:51.530 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T20:52:51.530 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:40.218379+0000", 2023-09-23T20:52:51.531 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.3549999999999999e-06 2023-09-23T20:52:51.531 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:51.531 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:51.531 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T20:52:51.531 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:40.218388+0000", 2023-09-23T20:52:51.531 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.2660000000000007e-06 2023-09-23T20:52:51.531 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:51.531 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:51.531 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:52:51.531 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:40.221008+0000", 2023-09-23T20:52:51.531 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0026195820000000001 2023-09-23T20:52:51.531 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:51.531 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:51.532 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T20:52:51.532 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:40.221638+0000", 2023-09-23T20:52:51.532 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00063040800000000005 2023-09-23T20:52:51.532 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:51.532 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:51.532 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 3,4", 2023-09-23T20:52:51.532 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:40.221768+0000", 2023-09-23T20:52:51.532 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00012917000000000001 2023-09-23T20:52:51.533 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:51.533 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:51.533 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:52:51.533 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:40.225646+0000", 2023-09-23T20:52:51.533 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0038786519999999998 2023-09-23T20:52:51.533 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:51.533 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:51.533 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T20:52:51.533 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:40.228716+0000", 2023-09-23T20:52:51.533 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0030701880000000002 2023-09-23T20:52:51.533 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:51.534 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:51.534 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:52:51.534 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:40.241591+0000", 2023-09-23T20:52:51.534 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.012874864999999999 2023-09-23T20:52:51.534 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:51.534 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:51.534 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T20:52:51.534 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:40.241629+0000", 2023-09-23T20:52:51.534 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.7515000000000003e-05 2023-09-23T20:52:51.534 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:51.534 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:51.534 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T20:52:51.535 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:40.241661+0000", 2023-09-23T20:52:51.535 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.2567000000000002e-05 2023-09-23T20:52:51.535 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:52:51.535 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T20:52:51.535 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:52:51.535 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:51.535 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:51.535 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34797.0:38460 6.f 6:f23ed23b:::benchmark_data_smithi026_223868_object38459:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e110)", 2023-09-23T20:52:51.535 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T20:52:40.334078+0000", 2023-09-23T20:52:51.535 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 11.074960209, 2023-09-23T20:52:51.535 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.027224721, 2023-09-23T20:52:51.535 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T20:52:51.536 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:52:51.536 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T20:52:51.536 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34797", 2023-09-23T20:52:51.536 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T20:52:51.536 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 38460 2023-09-23T20:52:51.536 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:51.536 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T20:52:51.536 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:51.536 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T20:52:51.536 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:40.334078+0000", 2023-09-23T20:52:51.536 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:52:51.536 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:51.537 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:51.537 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T20:52:51.537 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:40.334078+0000", 2023-09-23T20:52:51.537 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:52:51.537 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:51.537 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:51.537 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T20:52:51.537 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:40.334076+0000", 2023-09-23T20:52:51.537 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T20:52:51.537 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:51.537 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:51.537 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T20:52:51.538 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:40.334101+0000", 2023-09-23T20:52:51.538 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.4576999999999999e-05 2023-09-23T20:52:51.538 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:51.538 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:51.538 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T20:52:51.538 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:40.334115+0000", 2023-09-23T20:52:51.538 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.4361e-05 2023-09-23T20:52:51.538 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:51.538 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:51.538 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T20:52:51.538 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:40.334125+0000", 2023-09-23T20:52:51.539 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.8209999999999997e-06 2023-09-23T20:52:51.539 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:51.539 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:51.539 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:52:51.539 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:40.343341+0000", 2023-09-23T20:52:51.539 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0092166750000000006 2023-09-23T20:52:51.539 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:51.539 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:51.539 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T20:52:51.539 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:40.343460+0000", 2023-09-23T20:52:51.539 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000118108 2023-09-23T20:52:51.539 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:51.539 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:51.540 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 3,4", 2023-09-23T20:52:51.540 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:40.343608+0000", 2023-09-23T20:52:51.540 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000148782 2023-09-23T20:52:51.540 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:51.540 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:51.540 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T20:52:51.540 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:40.349293+0000", 2023-09-23T20:52:51.540 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0056847030000000002 2023-09-23T20:52:51.541 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:51.541 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:51.541 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:52:51.541 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:40.356926+0000", 2023-09-23T20:52:51.541 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0076325539999999997 2023-09-23T20:52:51.541 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:51.541 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:51.541 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:52:51.541 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:40.361243+0000", 2023-09-23T20:52:51.541 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0043175100000000001 2023-09-23T20:52:51.541 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:51.541 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:51.541 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T20:52:51.542 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:40.361277+0000", 2023-09-23T20:52:51.542 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.3515e-05 2023-09-23T20:52:51.542 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:51.542 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:51.542 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T20:52:51.542 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:40.361302+0000", 2023-09-23T20:52:51.542 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.5726000000000001e-05 2023-09-23T20:52:51.542 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:52:51.542 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T20:52:51.542 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:52:51.542 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:51.543 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:51.543 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34797.0:52021 6.f 6:f0be42e6:::benchmark_data_smithi026_223868_object52020:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e113)", 2023-09-23T20:52:51.543 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T20:52:50.795307+0000", 2023-09-23T20:52:51.543 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 0.61373072500000003, 2023-09-23T20:52:51.543 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.023217094000000001, 2023-09-23T20:52:51.543 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T20:52:51.543 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:52:51.543 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T20:52:51.543 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34797", 2023-09-23T20:52:51.543 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T20:52:51.543 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 52021 2023-09-23T20:52:51.544 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:51.544 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T20:52:51.544 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:51.544 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T20:52:51.544 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:50.795307+0000", 2023-09-23T20:52:51.544 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:52:51.545 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:51.545 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:51.545 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T20:52:51.545 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:50.795307+0000", 2023-09-23T20:52:51.545 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:52:51.545 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:51.545 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:51.545 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T20:52:51.546 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:50.795306+0000", 2023-09-23T20:52:51.546 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T20:52:51.546 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:51.546 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:51.546 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T20:52:51.546 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:50.795334+0000", 2023-09-23T20:52:51.546 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.8065e-05 2023-09-23T20:52:51.546 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:51.546 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:51.546 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T20:52:51.546 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:50.795346+0000", 2023-09-23T20:52:51.547 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.2364000000000001e-05 2023-09-23T20:52:51.547 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:51.547 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:51.547 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T20:52:51.547 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:50.795354+0000", 2023-09-23T20:52:51.547 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.9189999999999999e-06 2023-09-23T20:52:51.547 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:51.547 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:51.547 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:52:51.547 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:50.795522+0000", 2023-09-23T20:52:51.547 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000167646 2023-09-23T20:52:51.547 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:51.547 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:51.548 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T20:52:51.548 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:50.795671+0000", 2023-09-23T20:52:51.548 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00014937800000000001 2023-09-23T20:52:51.548 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:51.548 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:51.548 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 3,4", 2023-09-23T20:52:51.548 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:50.795819+0000", 2023-09-23T20:52:51.548 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00014762299999999999 2023-09-23T20:52:51.548 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:51.548 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:51.548 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:52:51.548 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:50.809163+0000", 2023-09-23T20:52:51.548 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.013344705 2023-09-23T20:52:51.549 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:51.549 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:51.549 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:52:51.549 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:50.809195+0000", 2023-09-23T20:52:51.549 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.1699999999999998e-05 2023-09-23T20:52:51.549 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:51.549 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:51.549 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T20:52:51.549 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:50.818427+0000", 2023-09-23T20:52:51.549 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0092323649999999993 2023-09-23T20:52:51.549 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:51.549 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:51.549 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T20:52:51.550 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:50.818483+0000", 2023-09-23T20:52:51.550 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.5828000000000001e-05 2023-09-23T20:52:51.550 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:51.550 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:51.550 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T20:52:51.550 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:50.818524+0000", 2023-09-23T20:52:51.550 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.0907e-05 2023-09-23T20:52:51.550 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:52:51.550 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T20:52:51.550 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:52:51.550 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:51.551 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:51.551 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34797.0:52022 6.f 6:f34aad84:::benchmark_data_smithi026_223868_object52021:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e113)", 2023-09-23T20:52:51.551 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T20:52:50.795388+0000", 2023-09-23T20:52:51.551 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 0.61364964700000002, 2023-09-23T20:52:51.551 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.032518263999999998, 2023-09-23T20:52:51.551 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T20:52:51.551 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:52:51.552 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T20:52:51.552 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34797", 2023-09-23T20:52:51.552 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T20:52:51.552 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 52022 2023-09-23T20:52:51.552 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:51.552 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T20:52:51.552 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:51.552 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T20:52:51.552 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:50.795388+0000", 2023-09-23T20:52:51.552 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:52:51.552 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:51.552 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:51.553 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T20:52:51.553 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:50.795388+0000", 2023-09-23T20:52:51.553 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:52:51.553 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:51.553 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:51.553 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T20:52:51.553 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:50.795388+0000", 2023-09-23T20:52:51.553 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T20:52:51.553 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:51.553 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:51.553 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T20:52:51.553 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:50.795411+0000", 2023-09-23T20:52:51.553 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.3785999999999999e-05 2023-09-23T20:52:51.554 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:51.554 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:51.554 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T20:52:51.554 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:50.795419+0000", 2023-09-23T20:52:51.554 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.4309999999999997e-06 2023-09-23T20:52:51.554 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:51.554 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:51.554 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T20:52:51.554 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:50.795424+0000", 2023-09-23T20:52:51.554 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.463e-06 2023-09-23T20:52:51.554 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:51.555 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:51.555 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:52:51.555 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:50.809220+0000", 2023-09-23T20:52:51.555 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.01379555 2023-09-23T20:52:51.555 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:51.555 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:51.555 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T20:52:51.555 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:50.809352+0000", 2023-09-23T20:52:51.556 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00013243 2023-09-23T20:52:51.556 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:51.556 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:51.556 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 3,4", 2023-09-23T20:52:51.556 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:50.809521+0000", 2023-09-23T20:52:51.556 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00016826800000000001 2023-09-23T20:52:51.556 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:51.556 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:51.556 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:52:51.556 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:50.818567+0000", 2023-09-23T20:52:51.556 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0090459170000000005 2023-09-23T20:52:51.557 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:51.557 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:51.557 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T20:52:51.557 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:50.819749+0000", 2023-09-23T20:52:51.557 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0011828850000000001 2023-09-23T20:52:51.557 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:51.557 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:51.557 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:52:51.557 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:50.827817+0000", 2023-09-23T20:52:51.557 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0080671690000000008 2023-09-23T20:52:51.557 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:51.557 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:51.557 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T20:52:51.557 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:50.827867+0000", 2023-09-23T20:52:51.558 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.0710000000000001e-05 2023-09-23T20:52:51.558 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:51.558 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:51.558 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T20:52:51.558 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:50.827906+0000", 2023-09-23T20:52:51.558 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.9190000000000003e-05 2023-09-23T20:52:51.558 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:52:51.559 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T20:52:51.559 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:52:51.559 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:52:51.559 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T20:52:51.559 INFO:teuthology.orchestra.run.smithi026.stdout:} 2023-09-23T20:52:51.857 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 32 16 53312 53296 103.997 76.375 0.0132806 0.00961057 2023-09-23T20:52:51.882 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:52:51.889 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:52:51.890 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:52:51.990 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:52:52.049 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:52:51 smithi026 ceph-mon[175151]: pgmap v264: 89 pgs: 89 active+clean; 3.0 GiB data, 8.9 GiB used, 617 GiB / 626 GiB avail; 76 MiB/s wr, 1.22k op/s 2023-09-23T20:52:52.049 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:52:51 smithi026 ceph-mon[179688]: pgmap v264: 89 pgs: 89 active+clean; 3.0 GiB data, 8.9 GiB used, 617 GiB / 626 GiB avail; 76 MiB/s wr, 1.22k op/s 2023-09-23T20:52:52.054 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:52:52.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:52:51 smithi070 ceph-mon[168282]: pgmap v264: 89 pgs: 89 active+clean; 3.0 GiB data, 8.9 GiB used, 617 GiB / 626 GiB avail; 76 MiB/s wr, 1.22k op/s 2023-09-23T20:52:52.345 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:52:52.348 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:52:52.348 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:52:52.429 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_ops_in_flight 2023-09-23T20:52:52.450 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:52:52.482 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:52:52.521 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:52:52.525 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:52:52.525 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:52:52.626 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:52:52.659 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:52:52.700 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:52:52.702 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:52:52.703 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:52:52.803 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:52:52.808 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T20:52:52.833 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:52:52.859 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 33 16 54527 54511 103.138 75.9375 0.0231139 0.00968986 2023-09-23T20:52:52.873 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:52:52.877 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:52:52.878 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:52:52.978 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:52:53.005 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:52:53.046 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:52:53.049 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:52:53.050 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:52:53.150 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:52:53.178 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:52:53.219 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:52:53.222 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:52:53.222 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:52:53.323 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:52:53.353 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:52:53.390 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:52:53.393 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:52:53.394 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:52:53.476 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:52:53.494 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:52:53.527 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:52:53.567 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:52:53.571 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:52:53.572 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:52:53.672 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:52:53.706 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:52:53.748 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:52:53.753 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:52:53.754 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:52:53.854 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:52:53.866 INFO:teuthology.orchestra.run.smithi026.stderr:nodeep-scrub is set 2023-09-23T20:52:53.866 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 34 16 55522 55506 101.916 62.1875 0.0132798 0.0098056 2023-09-23T20:52:53.868 INFO:teuthology.orchestra.run.smithi026.stderr:reweighted osd.6 to 1 (10000) 2023-09-23T20:52:53.945 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:52:53.990 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:52:53 smithi026 ceph-mon[175151]: pgmap v265: 89 pgs: 89 active+clean; 3.2 GiB data, 9.7 GiB used, 616 GiB / 626 GiB avail; 77 MiB/s wr, 1.24k op/s 2023-09-23T20:52:53.990 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:52:53 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/2858018544' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-09-23T20:52:53.991 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:52:53 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-09-23T20:52:53.991 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:52:53 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/2320825012' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:52:53.991 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:52:53 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:52:53.991 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:52:53 smithi026 ceph-mon[179688]: pgmap v265: 89 pgs: 89 active+clean; 3.2 GiB data, 9.7 GiB used, 616 GiB / 626 GiB avail; 77 MiB/s wr, 1.24k op/s 2023-09-23T20:52:53.991 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:52:53 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/2858018544' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-09-23T20:52:53.991 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:52:53 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-09-23T20:52:53.991 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:52:53 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/2320825012' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:52:53.992 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:52:53 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:52:54.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:52:53 smithi070 ceph-mon[168282]: pgmap v265: 89 pgs: 89 active+clean; 3.2 GiB data, 9.7 GiB used, 616 GiB / 626 GiB avail; 77 MiB/s wr, 1.24k op/s 2023-09-23T20:52:54.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:52:53 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/2858018544' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-09-23T20:52:54.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:52:53 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-09-23T20:52:54.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:52:53 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/2320825012' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:52:54.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:52:53 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:52:54.341 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:52:54.351 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:52:54.352 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:52:54.382 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:52:54.453 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph osd reweight 7 1 2023-09-23T20:52:54.453 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:52:54.487 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:52:54.529 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:52:54.532 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:52:54.532 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:52:54.633 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:52:54.667 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:52:54.707 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:52:54.711 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:52:54.712 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:52:54.812 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:52:54.842 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:52:54.871 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:52:54 smithi026 ceph-mon[175151]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:52:54.871 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:52:54 smithi026 ceph-mon[175151]: failed to encode map e114 with expected crc 2023-09-23T20:52:54.872 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 35 16 56658 56642 101.032 71 0.025484 0.00989077 2023-09-23T20:52:54.882 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:52:54.887 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:52:54.887 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:52:54.987 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:52:55.018 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:52:55.058 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:52:55.062 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:52:55.063 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:52:55.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:52:54 smithi070 ceph-mon[168282]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:52:55.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:52:54 smithi070 ceph-mon[168282]: failed to encode map e114 with expected crc 2023-09-23T20:52:55.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:52:54 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-09-23T20:52:55.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:52:54 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T20:52:55.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:52:54 smithi070 ceph-mon[168282]: osdmap e114: 8 total, 8 up, 7 in 2023-09-23T20:52:55.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:52:54 smithi070 ceph-mon[168282]: failed to encode map e114 with expected crc 2023-09-23T20:52:55.123 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:52:54 smithi026 ceph-mon[179688]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:52:55.123 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:52:54 smithi026 ceph-mon[179688]: failed to encode map e114 with expected crc 2023-09-23T20:52:55.123 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:52:54 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-09-23T20:52:55.123 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:52:54 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T20:52:55.123 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:52:54 smithi026 ceph-mon[179688]: osdmap e114: 8 total, 8 up, 7 in 2023-09-23T20:52:55.124 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:52:54 smithi026 ceph-mon[179688]: failed to encode map e114 with expected crc 2023-09-23T20:52:55.124 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:52:54 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-09-23T20:52:55.124 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:52:54 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T20:52:55.124 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:52:54 smithi026 ceph-mon[175151]: osdmap e114: 8 total, 8 up, 7 in 2023-09-23T20:52:55.124 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:52:54 smithi026 ceph-mon[175151]: failed to encode map e114 with expected crc 2023-09-23T20:52:55.163 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:52:55.216 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:52:55.339 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:52:55.345 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:52:55.345 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:52:55.446 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:52:55.480 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:52:55.522 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:52:55.527 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:52:55.527 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:52:55.628 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:52:55.701 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:52:55.709 INFO:teuthology.orchestra.run.smithi026.stdout:{ 2023-09-23T20:52:55.709 INFO:teuthology.orchestra.run.smithi026.stdout: "ops": [ 2023-09-23T20:52:55.709 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:55.709 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_repop(client.34797.0:57613 6.8 e114/102)", 2023-09-23T20:52:55.709 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T20:52:55.708954+0000", 2023-09-23T20:52:55.709 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 5.4607000000000001e-05, 2023-09-23T20:52:55.710 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.6275999999999998e-05, 2023-09-23T20:52:55.710 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T20:52:55.710 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "queued for pg", 2023-09-23T20:52:55.710 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T20:52:55.710 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:55.710 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T20:52:55.710 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:55.708954+0000", 2023-09-23T20:52:55.710 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:52:55.710 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:55.710 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:55.710 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T20:52:55.711 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:55.708954+0000", 2023-09-23T20:52:55.711 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:52:55.711 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:55.711 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:55.711 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T20:52:55.711 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:55.708954+0000", 2023-09-23T20:52:55.711 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T20:52:55.711 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:55.711 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:55.711 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T20:52:55.711 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:55.708992+0000", 2023-09-23T20:52:55.712 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.8801999999999999e-05 2023-09-23T20:52:55.712 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:55.712 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:55.712 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T20:52:55.712 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:55.709008+0000", 2023-09-23T20:52:55.712 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.5342999999999999e-05 2023-09-23T20:52:55.712 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:52:55.712 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:52:55.712 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T20:52:55.712 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:55.709015+0000", 2023-09-23T20:52:55.712 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.9460000000000001e-06 2023-09-23T20:52:55.712 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:52:55.713 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T20:52:55.713 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:52:55.713 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:52:55.713 INFO:teuthology.orchestra.run.smithi026.stdout: ], 2023-09-23T20:52:55.713 INFO:teuthology.orchestra.run.smithi026.stdout: "num_ops": 1 2023-09-23T20:52:55.713 INFO:teuthology.orchestra.run.smithi026.stdout:} 2023-09-23T20:52:55.770 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:52:55.781 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:52:55.782 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:52:55.867 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 36 16 57778 57762 100.171 70 0.0104838 0.00997737 2023-09-23T20:52:55.882 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:52:55.914 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:52:55.958 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:52:55.962 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:52:55.962 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:52:56.061 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:52:56.063 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:52:56.094 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:52:56.112 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:52:55 smithi026 ceph-mon[179688]: pgmap v267: 89 pgs: 89 active+clean; 3.3 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 89 MiB/s wr, 1.42k op/s 2023-09-23T20:52:56.115 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:52:55 smithi026 ceph-mon[175151]: pgmap v267: 89 pgs: 89 active+clean; 3.3 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 89 MiB/s wr, 1.42k op/s 2023-09-23T20:52:56.134 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:52:56.136 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:52:55 smithi070 ceph-mon[168282]: pgmap v267: 89 pgs: 89 active+clean; 3.3 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 89 MiB/s wr, 1.42k op/s 2023-09-23T20:52:56.137 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:52:56.137 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:52:56.138 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_blocked_ops 2023-09-23T20:52:56.237 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:52:56.336 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:52:56.483 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:52:56.485 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:52:56.486 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:52:56.586 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:52:56.618 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:52:56.656 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:52:56.658 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:52:56.658 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:52:56.759 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:52:56.786 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:52:56.827 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:52:56.831 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:52:56.831 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:52:56.858 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0: {} 2023-09-23T20:52:56.867 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 37 16 58869 58853 99.3071 68.1875 0.0140807 0.0100648 2023-09-23T20:52:56.906 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1: {} 2023-09-23T20:52:56.931 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:52:56.945 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2: {} 2023-09-23T20:52:56.992 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3: {} 2023-09-23T20:52:57.024 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:52:57.047 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4: {} 2023-09-23T20:52:57.083 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T20:52:57.122 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6: {} 2023-09-23T20:52:57.131 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:52:56 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/2061228707' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:52:57.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:52:56 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/2061228707' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:52:57.154 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T20:52:57.161 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T20:52:57.168 INFO:teuthology.orchestra.run.smithi026.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T20:52:57.168 INFO:teuthology.orchestra.run.smithi026.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T20:52:57.168 INFO:teuthology.orchestra.run.smithi026.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T20:52:57.168 INFO:teuthology.orchestra.run.smithi026.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T20:52:57.168 INFO:teuthology.orchestra.run.smithi026.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T20:52:57.168 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T20:52:57.168 INFO:teuthology.orchestra.run.smithi026.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T20:52:57.168 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T20:52:57.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:52:56 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/2061228707' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:52:57.868 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 38 16 60022 60006 98.5887 72.0625 0.00701694 0.0101377 2023-09-23T20:52:58.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:52:57 smithi070 ceph-mon[168282]: pgmap v268: 89 pgs: 89 active+clean; 3.4 GiB data, 10 GiB used, 615 GiB / 626 GiB avail; 77 MiB/s wr, 1.23k op/s 2023-09-23T20:52:58.381 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:52:57 smithi026 ceph-mon[179688]: pgmap v268: 89 pgs: 89 active+clean; 3.4 GiB data, 10 GiB used, 615 GiB / 626 GiB avail; 77 MiB/s wr, 1.23k op/s 2023-09-23T20:52:58.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:52:57 smithi026 ceph-mon[175151]: pgmap v268: 89 pgs: 89 active+clean; 3.4 GiB data, 10 GiB used, 615 GiB / 626 GiB avail; 77 MiB/s wr, 1.23k op/s 2023-09-23T20:52:58.869 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 39 16 61347 61331 98.1818 82.8125 0.0178841 0.01018 2023-09-23T20:52:59.085 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:52:59.089 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:52:59.089 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:52:59.190 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:52:59.262 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:52:59.318 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:52:59.614 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:52:59 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[164029]: ::ffff:172.21.15.70 - - [23/Sep/2023:20:52:59] "GET /metrics HTTP/1.1" 200 36308 "" "Prometheus/2.43.0" 2023-09-23T20:52:59.871 INFO:tasks.radosbench.radosbench.0.smithi026.stdout:2023-09-23T20:52:59.872087+0000 min lat: 0.00125417 max lat: 0.0886562 avg lat: 0.0102326 2023-09-23T20:52:59.871 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T20:52:59.872 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 40 16 62602 62586 97.6824 78.4375 0.00840848 0.0102326 2023-09-23T20:53:00.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:52:59 smithi070 ceph-mon[168282]: pgmap v269: 89 pgs: 89 active+clean; 3.5 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 75 MiB/s wr, 1.20k op/s 2023-09-23T20:53:00.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:52:59 smithi026 ceph-mon[175151]: pgmap v269: 89 pgs: 89 active+clean; 3.5 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 75 MiB/s wr, 1.20k op/s 2023-09-23T20:53:00.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:52:59 smithi026 ceph-mon[179688]: pgmap v269: 89 pgs: 89 active+clean; 3.5 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 75 MiB/s wr, 1.20k op/s 2023-09-23T20:53:00.871 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 41 16 63880 63864 97.2483 79.875 0.0123986 0.0102772 2023-09-23T20:53:01.135 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:53:01.139 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:53:01.139 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:53:01.240 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:53:01.312 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:53:00 smithi026 ceph-mon[179688]: pgmap v270: 89 pgs: 89 active+clean; 3.7 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 72 MiB/s wr, 1.15k op/s 2023-09-23T20:53:01.312 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:53:00 smithi026 ceph-mon[175151]: pgmap v270: 89 pgs: 89 active+clean; 3.7 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 72 MiB/s wr, 1.15k op/s 2023-09-23T20:53:01.314 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:53:01.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:53:00 smithi070 ceph-mon[168282]: pgmap v270: 89 pgs: 89 active+clean; 3.7 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 72 MiB/s wr, 1.15k op/s 2023-09-23T20:53:01.873 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 42 16 65030 65014 96.642 71.875 0.0146188 0.0103428 2023-09-23T20:53:02.009 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:53:02.016 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:53:02.017 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:53:02.117 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:53:02.152 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:53:02.193 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:53:02.197 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:53:02.197 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:53:02.271 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:53:01 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:53:02.278 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:53:01 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:53:02.298 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:53:02.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:53:01 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:53:02.381 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:53:02.441 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:53:02.455 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:53:02.456 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:53:02.557 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:53:02.565 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:53:02.586 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:53:02.626 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:53:02.630 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:53:02.631 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:53:02.731 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:53:02.765 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:53:02.804 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:53:02.809 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:53:02.810 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:53:02.872 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 43 16 66072 66056 95.9096 65.125 0.0150581 0.0104214 2023-09-23T20:53:02.911 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:53:02.984 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:53:03.238 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:53:03.246 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:53:02 smithi026 ceph-mon[179688]: pgmap v271: 89 pgs: 89 active+clean; 3.9 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 73 MiB/s wr, 1.16k op/s 2023-09-23T20:53:03.247 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:53:02 smithi026 ceph-mon[175151]: pgmap v271: 89 pgs: 89 active+clean; 3.9 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 73 MiB/s wr, 1.16k op/s 2023-09-23T20:53:03.247 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:53:03.247 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:53:03.348 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:53:03.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:53:02 smithi070 ceph-mon[168282]: pgmap v271: 89 pgs: 89 active+clean; 3.9 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 73 MiB/s wr, 1.16k op/s 2023-09-23T20:53:03.380 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:53:03.420 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:53:03.424 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:53:03.424 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:53:03.525 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:53:03.613 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:53:03.873 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 44 16 67131 67115 95.2347 66.1875 0.00661302 0.0104959 2023-09-23T20:53:04.001 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T20:53:04.010 INFO:teuthology.orchestra.run.smithi026.stderr:reweighted osd.7 to 1 (10000) 2023-09-23T20:53:04.065 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:53:04.078 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:53:04.079 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:53:04.141 INFO:teuthology.orchestra.run.smithi026.stdout:ERROR: (22) Invalid argument 2023-09-23T20:53:04.141 INFO:teuthology.orchestra.run.smithi026.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:53:04.179 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:53:04.287 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:53:03 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/3599795027' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-09-23T20:53:04.288 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:53:03 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/3599795027' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-09-23T20:53:04.312 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:53:04.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:53:03 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/3599795027' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-09-23T20:53:04.511 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:53:04.520 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:53:04.520 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:53:04.621 INFO:tasks.cephadm.ceph_manager.ceph:waiting for recovery to complete 2023-09-23T20:53:04.621 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T20:53:04.622 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:53:04.707 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:53:04.870 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:53:04.874 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 45 16 68070 68054 94.4204 58.6875 0.0154432 0.0105851 2023-09-23T20:53:04.887 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:53:04.888 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:53:04.955 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_historic_ops 2023-09-23T20:53:04.988 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:53:05.111 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:53:05.212 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:53:05.232 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:53:05.237 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:53:05.237 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:53:05 smithi026 ceph-mon[179688]: failed to encode map e115 with expected crc 2023-09-23T20:53:05.237 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:53:05 smithi026 ceph-mon[179688]: failed to encode map e115 with expected crc 2023-09-23T20:53:05.238 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:53:05 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/3599795027' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-09-23T20:53:05.238 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:53:05 smithi026 ceph-mon[179688]: osdmap e115: 8 total, 8 up, 7 in 2023-09-23T20:53:05.238 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:53:05 smithi026 ceph-mon[179688]: pgmap v273: 89 pgs: 89 active+clean; 4.0 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 73 MiB/s wr, 1.17k op/s 2023-09-23T20:53:05.238 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:53:05.241 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:53:04 smithi026 ceph-mon[175151]: failed to encode map e115 with expected crc 2023-09-23T20:53:05.241 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:53:04 smithi026 ceph-mon[175151]: failed to encode map e115 with expected crc 2023-09-23T20:53:05.241 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:53:04 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/3599795027' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-09-23T20:53:05.241 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:53:04 smithi026 ceph-mon[175151]: osdmap e115: 8 total, 8 up, 7 in 2023-09-23T20:53:05.241 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:53:04 smithi026 ceph-mon[175151]: pgmap v273: 89 pgs: 89 active+clean; 4.0 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 73 MiB/s wr, 1.17k op/s 2023-09-23T20:53:05.340 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:53:05.366 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:53:04 smithi070 ceph-mon[168282]: failed to encode map e115 with expected crc 2023-09-23T20:53:05.366 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:53:04 smithi070 ceph-mon[168282]: failed to encode map e115 with expected crc 2023-09-23T20:53:05.366 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:53:04 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/3599795027' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-09-23T20:53:05.366 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:53:04 smithi070 ceph-mon[168282]: osdmap e115: 8 total, 8 up, 7 in 2023-09-23T20:53:05.366 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:53:04 smithi070 ceph-mon[168282]: pgmap v273: 89 pgs: 89 active+clean; 4.0 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 73 MiB/s wr, 1.17k op/s 2023-09-23T20:53:05.371 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:53:05.407 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:53:05.411 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:53:05.411 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:53:05.512 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:53:05.622 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:53:05.685 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:53:05.689 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:53:05.690 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:53:05.791 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:53:05.822 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:53:05.875 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 46 16 68879 68863 93.4656 50.5625 0.023995 0.0106924 2023-09-23T20:53:05.883 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:53:06.038 INFO:teuthology.orchestra.run.smithi026.stderr:noscrub is unset 2023-09-23T20:53:06.136 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:53:06 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/3831138340' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:53:06.136 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:53:06 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/3831138340' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:53:06.203 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:53:06.206 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:53:06.207 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:53:06.308 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:53:06.334 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:53:06.335 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:53:06 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/3831138340' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:53:06.340 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:53:06.380 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:53:06.383 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:53:06.384 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:53:06.484 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:53:06.576 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:53:06.875 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 47 16 69902 69886 92.8378 63.9375 0.0118087 0.010765 2023-09-23T20:53:07.132 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:53:07.138 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:53:07.139 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:53:07.239 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:53:07.264 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:53:07 smithi070 ceph-mon[168282]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:53:07.265 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:53:07 smithi070 ceph-mon[168282]: failed to encode map e116 with expected crc 2023-09-23T20:53:07.265 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:53:07 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/3831138340' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T20:53:07.265 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:53:07 smithi070 ceph-mon[168282]: osdmap e116: 8 total, 8 up, 7 in 2023-09-23T20:53:07.265 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:53:07 smithi070 ceph-mon[168282]: failed to encode map e116 with expected crc 2023-09-23T20:53:07.265 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:53:07 smithi070 ceph-mon[168282]: pgmap v275: 89 pgs: 89 active+clean; 4.1 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 74 MiB/s wr, 1.18k op/s 2023-09-23T20:53:07.267 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:53:07.313 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:53:07.317 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:53:07.317 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:53:07.386 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:53:07 smithi026 ceph-mon[175151]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:53:07.386 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:53:07 smithi026 ceph-mon[175151]: failed to encode map e116 with expected crc 2023-09-23T20:53:07.386 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:53:07 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/3831138340' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T20:53:07.386 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:53:07 smithi026 ceph-mon[175151]: osdmap e116: 8 total, 8 up, 7 in 2023-09-23T20:53:07.386 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:53:07 smithi026 ceph-mon[175151]: failed to encode map e116 with expected crc 2023-09-23T20:53:07.386 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:53:07 smithi026 ceph-mon[175151]: pgmap v275: 89 pgs: 89 active+clean; 4.1 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 74 MiB/s wr, 1.18k op/s 2023-09-23T20:53:07.386 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:53:07 smithi026 ceph-mon[179688]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:53:07.387 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:53:07 smithi026 ceph-mon[179688]: failed to encode map e116 with expected crc 2023-09-23T20:53:07.387 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:53:07 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/3831138340' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T20:53:07.387 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:53:07 smithi026 ceph-mon[179688]: osdmap e116: 8 total, 8 up, 7 in 2023-09-23T20:53:07.387 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:53:07 smithi026 ceph-mon[179688]: failed to encode map e116 with expected crc 2023-09-23T20:53:07.387 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:53:07 smithi026 ceph-mon[179688]: pgmap v275: 89 pgs: 89 active+clean; 4.1 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 74 MiB/s wr, 1.18k op/s 2023-09-23T20:53:07.418 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:53:07.493 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:53:07.876 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 48 16 70962 70946 92.2843 66.25 0.00897306 0.010831 2023-09-23T20:53:08.221 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:53:08.228 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:53:08.228 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:53:08.329 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:53:08.418 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:53:08.877 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 49 16 71895 71879 91.5885 58.3125 0.00690473 0.0109119 2023-09-23T20:53:08.924 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T20:53:09.268 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:53:09 smithi026 ceph-mon[179688]: pgmap v276: 89 pgs: 89 active+clean; 4.2 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 68 MiB/s wr, 1.09k op/s 2023-09-23T20:53:09.268 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:53:09 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/4181652111' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:53:09.268 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:53:09 smithi026 ceph-mon[175151]: pgmap v276: 89 pgs: 89 active+clean; 4.2 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 68 MiB/s wr, 1.09k op/s 2023-09-23T20:53:09.268 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:53:09 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/4181652111' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:53:09.600 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0: {} 2023-09-23T20:53:09.601 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1: {} 2023-09-23T20:53:09.601 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2: {} 2023-09-23T20:53:09.601 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3: {} 2023-09-23T20:53:09.601 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4: {} 2023-09-23T20:53:09.601 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T20:53:09.601 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6: {} 2023-09-23T20:53:09.601 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T20:53:09.601 INFO:teuthology.orchestra.run.smithi026.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T20:53:09.601 INFO:teuthology.orchestra.run.smithi026.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T20:53:09.602 INFO:teuthology.orchestra.run.smithi026.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T20:53:09.602 INFO:teuthology.orchestra.run.smithi026.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T20:53:09.602 INFO:teuthology.orchestra.run.smithi026.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T20:53:09.602 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T20:53:09.602 INFO:teuthology.orchestra.run.smithi026.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T20:53:09.602 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T20:53:09.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:53:09 smithi070 ceph-mon[168282]: pgmap v276: 89 pgs: 89 active+clean; 4.2 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 68 MiB/s wr, 1.09k op/s 2023-09-23T20:53:09.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:53:09 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/4181652111' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:53:09.877 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 50 16 72843 72827 90.9422 59.25 0.0113321 0.0109913 2023-09-23T20:53:09.882 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:53:09 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[164029]: ::ffff:172.21.15.70 - - [23/Sep/2023:20:53:09] "GET /metrics HTTP/1.1" 200 36310 "" "Prometheus/2.43.0" 2023-09-23T20:53:10.884 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 51 16 73883 73867 90.4301 65 0.0179513 0.0110535 2023-09-23T20:53:11.264 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:53:11.274 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:53:11.275 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:53:11.375 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:53:11.471 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:53:11.564 INFO:teuthology.orchestra.run.smithi026.stdout: 2023-09-23T20:53:11.572 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:53:11.575 INFO:teuthology.orchestra.run.smithi026.stderr:dumped all 2023-09-23T20:53:11.577 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:53:11.577 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:53:11.678 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:53:11.776 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:53:11.776 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:53:11.880 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 52 16 74804 74788 89.7974 57.5625 0.0133632 0.0111314 2023-09-23T20:53:11.884 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:53:11 smithi026 ceph-mon[179688]: pgmap v277: 89 pgs: 89 active+clean; 4.3 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 58 MiB/s wr, 924 op/s 2023-09-23T20:53:11.884 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:53:11 smithi026 ceph-mon[175151]: pgmap v277: 89 pgs: 89 active+clean; 4.3 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 58 MiB/s wr, 924 op/s 2023-09-23T20:53:12.035 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:53:12.041 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:53:12.041 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:53:12.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:53:11 smithi070 ceph-mon[168282]: pgmap v277: 89 pgs: 89 active+clean; 4.3 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 58 MiB/s wr, 924 op/s 2023-09-23T20:53:12.141 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:53:12.207 INFO:teuthology.orchestra.run.smithi026.stdout:{"pg_ready":true,"pg_map":{"version":277,"stamp":"2023-09-23T20:53:10.199023+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4635695008,"num_objects":70750,"num_object_clones":0,"num_object_copies":212250,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":70750,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":71026,"num_write_kb":4528909,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":15,"num_bytes_recovered":1771728,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":70942,"ondisk_log_size":70942,"up":267,"acting":267,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":267,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":13763408,"kb_used_data":13422296,"kb_used_omap":126,"kb_used_meta":340929,"kb_avail":642452656,"statfs":{"total":671965249536,"available":657871519744,"internally_reserved":0,"allocated":13744431104,"data_stored":13740888665,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":129413,"internal_metadata":349111931},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":8,"apply_latency_ms":8,"commit_latency_ns":8000000,"apply_latency_ns":8000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":484966400,"num_objects":7400,"num_object_clones":0,"num_object_copies":22200,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":7400,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7400,"num_write_kb":473600,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"8.003794"},"pg_stats":[{"pgid":"3.1a","version":"68'1","reported_seq":94,"reported_epoch":115,"state":"active+clean","last_fresh":"2023-09-23T20:53:03.998381+0000","last_change":"2023-09-23T20:50:37.985580+0000","last_active":"2023-09-23T20:53:03.998381+0000","last_peered":"2023-09-23T20:53:03.998381+0000","last_clean":"2023-09-23T20:53:03.998381+0000","last_became_active":"2023-09-23T20:50:37.985229+0000","last_became_peered":"2023-09-23T20:50:37.985229+0000","last_unstale":"2023-09-23T20:53:03.998381+0000","last_undegraded":"2023-09-23T20:53:03.998381+0000","last_fullsized":"2023-09-23T20:53:03.998381+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:52:50.591507+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":78,"reported_epoch":115,"state":"active+clean","last_fresh":"2023-09-23T20:53:04.003302+0000","last_change":"2023-09-23T20:51:10.835170+0000","last_active":"2023-09-23T20:53:04.003302+0000","last_peered":"2023-09-23T20:53:04.003302+0000","last_clean":"2023-09-23T20:53:04.003302+0000","last_became_active":"2023-09-23T20:51:10.834808+0000","last_became_peered":"2023-09-23T20:51:10.834808+0000","last_unstale":"2023-09-23T20:53:04.003302+0000","last_undegraded":"2023-09-23T20:53:04.003302+0000","last_fullsized":"2023-09-23T20:53:04.003302+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:51:25.153742+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":65,"reported_epoch":115,"state":"active+clean","last_fresh":"2023-09-23T20:53:04.027896+0000","last_change":"2023-09-23T20:51:10.833035+0000","last_active":"2023-09-23T20:53:04.027896+0000","last_peered":"2023-09-23T20:53:04.027896+0000","last_clean":"2023-09-23T20:53:04.027896+0000","last_became_active":"2023-09-23T20:51:10.832741+0000","last_became_peered":"2023-09-23T20:51:10.832741+0000","last_unstale":"2023-09-23T20:53:04.027896+0000","last_undegraded":"2023-09-23T20:53:04.027896+0000","last_fullsized":"2023-09-23T20:53:04.027896+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:53:16.471852+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":69,"reported_epoch":114,"state":"active+clean","last_fresh":"2023-09-23T20:52:53.874458+0000","last_change":"2023-09-23T20:51:10.841420+0000","last_active":"2023-09-23T20:52:53.874458+0000","last_peered":"2023-09-23T20:52:53.874458+0000","last_clean":"2023-09-23T20:52:53.874458+0000","last_became_active":"2023-09-23T20:51:10.841112+0000","last_became_peered":"2023-09-23T20:51:10.841112+0000","last_unstale":"2023-09-23T20:52:53.874458+0000","last_undegraded":"2023-09-23T20:52:53.874458+0000","last_fullsized":"2023-09-23T20:52:53.874458+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:44:02.236650+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":96,"reported_epoch":115,"state":"active+clean","last_fresh":"2023-09-23T20:53:04.002365+0000","last_change":"2023-09-23T20:51:10.833613+0000","last_active":"2023-09-23T20:53:04.002365+0000","last_peered":"2023-09-23T20:53:04.002365+0000","last_clean":"2023-09-23T20:53:04.002365+0000","last_became_active":"2023-09-23T20:51:10.833208+0000","last_became_peered":"2023-09-23T20:51:10.833208+0000","last_unstale":"2023-09-23T20:53:04.002365+0000","last_undegraded":"2023-09-23T20:53:04.002365+0000","last_fullsized":"2023-09-23T20:53:04.002365+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:51:28.246014+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":85,"reported_epoch":115,"state":"active+clean","last_fresh":"2023-09-23T20:53:04.000227+0000","last_change":"2023-09-23T20:52:12.862262+0000","last_active":"2023-09-23T20:53:04.000227+0000","last_peered":"2023-09-23T20:53:04.000227+0000","last_clean":"2023-09-23T20:53:04.000227+0000","last_became_active":"2023-09-23T20:52:12.861972+0000","last_became_peered":"2023-09-23T20:52:12.861972+0000","last_unstale":"2023-09-23T20:53:04.000227+0000","last_undegraded":"2023-09-23T20:53:04.000227+0000","last_fullsized":"2023-09-23T20:53:04.000227+0000","mapping_epoch":100,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":101,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:10:42.887918+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":78,"reported_epoch":114,"state":"active+clean","last_fresh":"2023-09-23T20:52:53.875003+0000","last_change":"2023-09-23T20:52:12.862076+0000","last_active":"2023-09-23T20:52:53.875003+0000","last_peered":"2023-09-23T20:52:53.875003+0000","last_clean":"2023-09-23T20:52:53.875003+0000","last_became_active":"2023-09-23T20:52:12.861753+0000","last_became_peered":"2023-09-23T20:52:12.861753+0000","last_unstale":"2023-09-23T20:52:53.875003+0000","last_undegraded":"2023-09-23T20:52:53.875003+0000","last_fullsized":"2023-09-23T20:52:53.875003+0000","mapping_epoch":100,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":101,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:15:39.708595+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,6],"acting":[0,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":78,"reported_epoch":115,"state":"active+clean","last_fresh":"2023-09-23T20:53:04.002452+0000","last_change":"2023-09-23T20:51:10.833424+0000","last_active":"2023-09-23T20:53:04.002452+0000","last_peered":"2023-09-23T20:53:04.002452+0000","last_clean":"2023-09-23T20:53:04.002452+0000","last_became_active":"2023-09-23T20:51:10.832808+0000","last_became_peered":"2023-09-23T20:51:10.832808+0000","last_unstale":"2023-09-23T20:53:04.002452+0000","last_undegraded":"2023-09-23T20:53:04.002452+0000","last_fullsized":"2023-09-23T20:53:04.002452+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:32:42.819583+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":68,"reported_epoch":115,"state":"active+clean","last_fresh":"2023-09-23T20:53:03.998597+0000","last_change":"2023-09-23T20:52:12.860277+0000","last_active":"2023-09-23T20:53:03.998597+0000","last_peered":"2023-09-23T20:53:03.998597+0000","last_clean":"2023-09-23T20:53:03.998597+0000","last_became_active":"2023-09-23T20:52:12.859866+0000","last_became_peered":"2023-09-23T20:52:12.859866+0000","last_unstale":"2023-09-23T20:53:03.998597+0000","last_undegraded":"2023-09-23T20:53:03.998597+0000","last_fullsized":"2023-09-23T20:53:03.998597+0000","mapping_epoch":100,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":101,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:07:45.523326+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,3],"acting":[7,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":39,"reported_epoch":115,"state":"active+clean","last_fresh":"2023-09-23T20:53:04.002239+0000","last_change":"2023-09-23T20:51:10.833790+0000","last_active":"2023-09-23T20:53:04.002239+0000","last_peered":"2023-09-23T20:53:04.002239+0000","last_clean":"2023-09-23T20:53:04.002239+0000","last_became_active":"2023-09-23T20:51:10.832804+0000","last_became_peered":"2023-09-23T20:51:10.832804+0000","last_unstale":"2023-09-23T20:53:04.002239+0000","last_undegraded":"2023-09-23T20:53:04.002239+0000","last_fullsized":"2023-09-23T20:53:04.002239+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:27:42.601674+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":79,"reported_epoch":115,"state":"active+clean","last_fresh":"2023-09-23T20:53:04.000433+0000","last_change":"2023-09-23T20:51:10.834618+0000","last_active":"2023-09-23T20:53:04.000433+0000","last_peered":"2023-09-23T20:53:04.000433+0000","last_clean":"2023-09-23T20:53:04.000433+0000","last_became_active":"2023-09-23T20:51:10.834267+0000","last_became_peered":"2023-09-23T20:51:10.834267+0000","last_unstale":"2023-09-23T20:53:04.000433+0000","last_undegraded":"2023-09-23T20:53:04.000433+0000","last_fullsized":"2023-09-23T20:53:04.000433+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:38:59.356003+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":65,"reported_epoch":115,"state":"active+clean","last_fresh":"2023-09-23T20:53:04.028516+0000","last_change":"2023-09-23T20:50:37.982103+0000","last_active":"2023-09-23T20:53:04.028516+0000","last_peered":"2023-09-23T20:53:04.028516+0000","last_clean":"2023-09-23T20:53:04.028516+0000","last_became_active":"2023-09-23T20:50:37.981791+0000","last_became_peered":"2023-09-23T20:50:37.981791+0000","last_unstale":"2023-09-23T20:53:04.028516+0000","last_undegraded":"2023-09-23T20:53:04.028516+0000","last_fullsized":"2023-09-23T20:53:04.028516+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:32:52.286186+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"79'22","reported_seq":81,"reported_epoch":115,"state":"active+clean","last_fresh":"2023-09-23T20:53:03.999190+0000","last_change":"2023-09-23T20:41:58.905866+0000","last_active":"2023-09-23T20:53:03.999190+0000","last_peered":"2023-09-23T20:53:03.999190+0000","last_clean":"2023-09-23T20:53:03.999190+0000","last_became_active":"2023-09-23T20:41:58.905556+0000","last_became_peered":"2023-09-23T20:41:58.905556+0000","last_unstale":"2023-09-23T20:53:03.999190+0000","last_undegraded":"2023-09-23T20:53:03.999190+0000","last_fullsized":"2023-09-23T20:53:03.999190+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:03:14.027348+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":75,"reported_epoch":115,"state":"active+clean","last_fresh":"2023-09-23T20:53:03.998923+0000","last_change":"2023-09-23T20:50:05.871199+0000","last_active":"2023-09-23T20:53:03.998923+0000","last_peered":"2023-09-23T20:53:03.998923+0000","last_clean":"2023-09-23T20:53:03.998923+0000","last_became_active":"2023-09-23T20:50:05.870912+0000","last_became_peered":"2023-09-23T20:50:05.870912+0000","last_unstale":"2023-09-23T20:53:03.998923+0000","last_undegraded":"2023-09-23T20:53:03.998923+0000","last_fullsized":"2023-09-23T20:53:03.998923+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:16:22.718106+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":78,"reported_epoch":115,"state":"active+clean","last_fresh":"2023-09-23T20:53:03.999111+0000","last_change":"2023-09-23T20:50:37.986729+0000","last_active":"2023-09-23T20:53:03.999111+0000","last_peered":"2023-09-23T20:53:03.999111+0000","last_clean":"2023-09-23T20:53:03.999111+0000","last_became_active":"2023-09-23T20:50:37.986433+0000","last_became_peered":"2023-09-23T20:50:37.986433+0000","last_unstale":"2023-09-23T20:53:03.999111+0000","last_undegraded":"2023-09-23T20:53:03.999111+0000","last_fullsized":"2023-09-23T20:53:03.999111+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:28:28.853091+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":78,"reported_epoch":115,"state":"active+clean","last_fresh":"2023-09-23T20:53:04.004807+0000","last_change":"2023-09-23T20:51:10.831262+0000","last_active":"2023-09-23T20:53:04.004807+0000","last_peered":"2023-09-23T20:53:04.004807+0000","last_clean":"2023-09-23T20:53:04.004807+0000","last_became_active":"2023-09-23T20:51:10.830931+0000","last_became_peered":"2023-09-23T20:51:10.830931+0000","last_unstale":"2023-09-23T20:53:04.004807+0000","last_undegraded":"2023-09-23T20:53:04.004807+0000","last_fullsized":"2023-09-23T20:53:04.004807+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:51:54.982505+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":69,"reported_epoch":114,"state":"active+clean","last_fresh":"2023-09-23T20:52:53.875891+0000","last_change":"2023-09-23T20:51:10.840590+0000","last_active":"2023-09-23T20:52:53.875891+0000","last_peered":"2023-09-23T20:52:53.875891+0000","last_clean":"2023-09-23T20:52:53.875891+0000","last_became_active":"2023-09-23T20:51:10.840221+0000","last_became_peered":"2023-09-23T20:51:10.840221+0000","last_unstale":"2023-09-23T20:52:53.875891+0000","last_undegraded":"2023-09-23T20:52:53.875891+0000","last_fullsized":"2023-09-23T20:52:53.875891+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:03:39.138638+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":77,"reported_epoch":115,"state":"active+clean","last_fresh":"2023-09-23T20:53:04.000281+0000","last_change":"2023-09-23T20:52:12.860214+0000","last_active":"2023-09-23T20:53:04.000281+0000","last_peered":"2023-09-23T20:53:04.000281+0000","last_clean":"2023-09-23T20:53:04.000281+0000","last_became_active":"2023-09-23T20:52:12.859837+0000","last_became_peered":"2023-09-23T20:52:12.859837+0000","last_unstale":"2023-09-23T20:53:04.000281+0000","last_undegraded":"2023-09-23T20:53:04.000281+0000","last_fullsized":"2023-09-23T20:53:04.000281+0000","mapping_epoch":100,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":101,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:01:38.149335+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,6],"acting":[7,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":59,"reported_epoch":115,"state":"active+clean","last_fresh":"2023-09-23T20:53:03.997917+0000","last_change":"2023-09-23T20:41:58.898224+0000","last_active":"2023-09-23T20:53:03.997917+0000","last_peered":"2023-09-23T20:53:03.997917+0000","last_clean":"2023-09-23T20:53:03.997917+0000","last_became_active":"2023-09-23T20:41:58.898036+0000","last_became_peered":"2023-09-23T20:41:58.898036+0000","last_unstale":"2023-09-23T20:53:03.997917+0000","last_undegraded":"2023-09-23T20:53:03.997917+0000","last_fullsized":"2023-09-23T20:53:03.997917+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:53:15.361560+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":69,"reported_epoch":114,"state":"active+clean","last_fresh":"2023-09-23T20:52:53.874390+0000","last_change":"2023-09-23T20:51:10.840849+0000","last_active":"2023-09-23T20:52:53.874390+0000","last_peered":"2023-09-23T20:52:53.874390+0000","last_clean":"2023-09-23T20:52:53.874390+0000","last_became_active":"2023-09-23T20:51:10.840593+0000","last_became_peered":"2023-09-23T20:51:10.840593+0000","last_unstale":"2023-09-23T20:52:53.874390+0000","last_undegraded":"2023-09-23T20:52:53.874390+0000","last_fullsized":"2023-09-23T20:52:53.874390+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:30:52.297625+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":104,"reported_epoch":115,"state":"active+clean","last_fresh":"2023-09-23T20:53:03.999734+0000","last_change":"2023-09-23T20:52:12.870672+0000","last_active":"2023-09-23T20:53:03.999734+0000","last_peered":"2023-09-23T20:53:03.999734+0000","last_clean":"2023-09-23T20:53:03.999734+0000","last_became_active":"2023-09-23T20:52:12.862084+0000","last_became_peered":"2023-09-23T20:52:12.862084+0000","last_unstale":"2023-09-23T20:53:03.999734+0000","last_undegraded":"2023-09-23T20:53:03.999734+0000","last_fullsized":"2023-09-23T20:53:03.999734+0000","mapping_epoch":100,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":101,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:32:58.322893+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":78,"reported_epoch":115,"state":"active+clean","last_fresh":"2023-09-23T20:53:04.003236+0000","last_change":"2023-09-23T20:51:10.833234+0000","last_active":"2023-09-23T20:53:04.003236+0000","last_peered":"2023-09-23T20:53:04.003236+0000","last_clean":"2023-09-23T20:53:04.003236+0000","last_became_active":"2023-09-23T20:51:10.832763+0000","last_became_peered":"2023-09-23T20:51:10.832763+0000","last_unstale":"2023-09-23T20:53:04.003236+0000","last_undegraded":"2023-09-23T20:53:04.003236+0000","last_fullsized":"2023-09-23T20:53:04.003236+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:24:08.283578+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.a","version":"116'4448","reported_seq":4470,"reported_epoch":116,"state":"active+clean","last_fresh":"2023-09-23T20:53:07.093723+0000","last_change":"2023-09-23T20:52:14.865442+0000","last_active":"2023-09-23T20:53:07.093723+0000","last_peered":"2023-09-23T20:53:07.093723+0000","last_clean":"2023-09-23T20:53:07.093723+0000","last_became_active":"2023-09-23T20:52:14.865018+0000","last_became_peered":"2023-09-23T20:52:14.865018+0000","last_unstale":"2023-09-23T20:53:07.093723+0000","last_undegraded":"2023-09-23T20:53:07.093723+0000","last_fullsized":"2023-09-23T20:53:07.093723+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":102,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:52:13.843048+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:52:13.843048+0000","last_clean_scrub_stamp":"2023-09-23T20:52:13.843048+0000","objects_scrubbed":0,"log_size":4448,"log_dups_size":0,"ondisk_log_size":4448,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:32:46.511747+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":291504128,"num_objects":4448,"num_object_clones":0,"num_object_copies":13344,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4448,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4448,"num_write_kb":284672,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,2],"acting":[6,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":77,"reported_epoch":115,"state":"active+clean","last_fresh":"2023-09-23T20:53:03.998285+0000","last_change":"2023-09-23T20:49:32.777943+0000","last_active":"2023-09-23T20:53:03.998285+0000","last_peered":"2023-09-23T20:53:03.998285+0000","last_clean":"2023-09-23T20:53:03.998285+0000","last_became_active":"2023-09-23T20:49:32.776212+0000","last_became_peered":"2023-09-23T20:49:32.776212+0000","last_unstale":"2023-09-23T20:53:03.998285+0000","last_undegraded":"2023-09-23T20:53:03.998285+0000","last_fullsized":"2023-09-23T20:53:03.998285+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:21:05.725926+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":28,"reported_epoch":115,"state":"active+clean","last_fresh":"2023-09-23T20:53:03.997784+0000","last_change":"2023-09-23T20:52:12.861197+0000","last_active":"2023-09-23T20:53:03.997784+0000","last_peered":"2023-09-23T20:53:03.997784+0000","last_clean":"2023-09-23T20:53:03.997784+0000","last_became_active":"2023-09-23T20:52:12.860833+0000","last_became_peered":"2023-09-23T20:52:12.860833+0000","last_unstale":"2023-09-23T20:53:03.997784+0000","last_undegraded":"2023-09-23T20:53:03.997784+0000","last_fullsized":"2023-09-23T20:53:03.997784+0000","mapping_epoch":100,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":101,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:48:25.460380+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,4],"acting":[7,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"6.b","version":"116'4363","reported_seq":4386,"reported_epoch":116,"state":"active+clean","last_fresh":"2023-09-23T20:53:08.851258+0000","last_change":"2023-09-23T20:52:14.868057+0000","last_active":"2023-09-23T20:53:08.851258+0000","last_peered":"2023-09-23T20:53:08.851258+0000","last_clean":"2023-09-23T20:53:08.851258+0000","last_became_active":"2023-09-23T20:52:14.867734+0000","last_became_peered":"2023-09-23T20:52:14.867734+0000","last_unstale":"2023-09-23T20:53:08.851258+0000","last_undegraded":"2023-09-23T20:53:08.851258+0000","last_fullsized":"2023-09-23T20:53:08.851258+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":102,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:52:13.843048+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:52:13.843048+0000","last_clean_scrub_stamp":"2023-09-23T20:52:13.843048+0000","objects_scrubbed":0,"log_size":4363,"log_dups_size":0,"ondisk_log_size":4363,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:40:31.506977+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":285933568,"num_objects":4363,"num_object_clones":0,"num_object_copies":13089,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4363,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4363,"num_write_kb":279232,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":78,"reported_epoch":115,"state":"active+clean","last_fresh":"2023-09-23T20:53:03.998470+0000","last_change":"2023-09-23T20:50:05.866983+0000","last_active":"2023-09-23T20:53:03.998470+0000","last_peered":"2023-09-23T20:53:03.998470+0000","last_clean":"2023-09-23T20:53:03.998470+0000","last_became_active":"2023-09-23T20:50:05.866601+0000","last_became_peered":"2023-09-23T20:50:05.866601+0000","last_unstale":"2023-09-23T20:53:03.998470+0000","last_undegraded":"2023-09-23T20:53:03.998470+0000","last_fullsized":"2023-09-23T20:53:03.998470+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:15:00.955166+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":92,"reported_epoch":115,"state":"active+clean","last_fresh":"2023-09-23T20:53:03.998392+0000","last_change":"2023-09-23T20:51:10.838338+0000","last_active":"2023-09-23T20:53:03.998392+0000","last_peered":"2023-09-23T20:53:03.998392+0000","last_clean":"2023-09-23T20:53:03.998392+0000","last_became_active":"2023-09-23T20:51:10.838049+0000","last_became_peered":"2023-09-23T20:51:10.838049+0000","last_unstale":"2023-09-23T20:53:03.998392+0000","last_undegraded":"2023-09-23T20:53:03.998392+0000","last_fullsized":"2023-09-23T20:53:03.998392+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:40:23.602402+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.8","version":"116'4413","reported_seq":4435,"reported_epoch":116,"state":"active+clean","last_fresh":"2023-09-23T20:53:07.086683+0000","last_change":"2023-09-23T20:52:14.869562+0000","last_active":"2023-09-23T20:53:07.086683+0000","last_peered":"2023-09-23T20:53:07.086683+0000","last_clean":"2023-09-23T20:53:07.086683+0000","last_became_active":"2023-09-23T20:52:14.869204+0000","last_became_peered":"2023-09-23T20:52:14.869204+0000","last_unstale":"2023-09-23T20:53:07.086683+0000","last_undegraded":"2023-09-23T20:53:07.086683+0000","last_fullsized":"2023-09-23T20:53:07.086683+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":102,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:52:13.843048+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:52:13.843048+0000","last_clean_scrub_stamp":"2023-09-23T20:52:13.843048+0000","objects_scrubbed":0,"log_size":4413,"log_dups_size":0,"ondisk_log_size":4413,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:32:14.355022+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":289210368,"num_objects":4413,"num_object_clones":0,"num_object_copies":13239,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4413,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4413,"num_write_kb":282432,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,3],"acting":[7,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":74,"reported_epoch":115,"state":"active+clean","last_fresh":"2023-09-23T20:53:04.000577+0000","last_change":"2023-09-23T20:52:12.866062+0000","last_active":"2023-09-23T20:53:04.000577+0000","last_peered":"2023-09-23T20:53:04.000577+0000","last_clean":"2023-09-23T20:53:04.000577+0000","last_became_active":"2023-09-23T20:52:12.861064+0000","last_became_peered":"2023-09-23T20:52:12.861064+0000","last_unstale":"2023-09-23T20:53:04.000577+0000","last_undegraded":"2023-09-23T20:53:04.000577+0000","last_fullsized":"2023-09-23T20:53:04.000577+0000","mapping_epoch":100,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":101,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:42:14.367193+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,3],"acting":[7,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":75,"reported_epoch":115,"state":"active+clean","last_fresh":"2023-09-23T20:53:04.000251+0000","last_change":"2023-09-23T20:50:05.867973+0000","last_active":"2023-09-23T20:53:04.000251+0000","last_peered":"2023-09-23T20:53:04.000251+0000","last_clean":"2023-09-23T20:53:04.000251+0000","last_became_active":"2023-09-23T20:50:05.867622+0000","last_became_peered":"2023-09-23T20:50:05.867622+0000","last_unstale":"2023-09-23T20:53:04.000251+0000","last_undegraded":"2023-09-23T20:53:04.000251+0000","last_fullsized":"2023-09-23T20:53:04.000251+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:10:12.617597+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.9","version":"115'4327","reported_seq":4349,"reported_epoch":115,"state":"active+clean","last_fresh":"2023-09-23T20:53:05.738925+0000","last_change":"2023-09-23T20:52:14.864297+0000","last_active":"2023-09-23T20:53:05.738925+0000","last_peered":"2023-09-23T20:53:05.738925+0000","last_clean":"2023-09-23T20:53:05.738925+0000","last_became_active":"2023-09-23T20:52:14.863986+0000","last_became_peered":"2023-09-23T20:52:14.863986+0000","last_unstale":"2023-09-23T20:53:05.738925+0000","last_undegraded":"2023-09-23T20:53:05.738925+0000","last_fullsized":"2023-09-23T20:53:05.738925+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":102,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:52:13.843048+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:52:13.843048+0000","last_clean_scrub_stamp":"2023-09-23T20:52:13.843048+0000","objects_scrubbed":0,"log_size":4327,"log_dups_size":0,"ondisk_log_size":4327,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:37:59.995892+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":283574272,"num_objects":4327,"num_object_clones":0,"num_object_copies":12981,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4327,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4327,"num_write_kb":276928,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,2],"acting":[0,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":77,"reported_epoch":115,"state":"active+clean","last_fresh":"2023-09-23T20:53:04.004997+0000","last_change":"2023-09-23T20:52:12.858198+0000","last_active":"2023-09-23T20:53:04.004997+0000","last_peered":"2023-09-23T20:53:04.004997+0000","last_clean":"2023-09-23T20:53:04.004997+0000","last_became_active":"2023-09-23T20:52:12.857937+0000","last_became_peered":"2023-09-23T20:52:12.857937+0000","last_unstale":"2023-09-23T20:53:04.004997+0000","last_undegraded":"2023-09-23T20:53:04.004997+0000","last_fullsized":"2023-09-23T20:53:04.004997+0000","mapping_epoch":100,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":101,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:18:42.976887+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":82,"reported_epoch":114,"state":"active+clean","last_fresh":"2023-09-23T20:52:53.874221+0000","last_change":"2023-09-23T20:50:37.982263+0000","last_active":"2023-09-23T20:52:53.874221+0000","last_peered":"2023-09-23T20:52:53.874221+0000","last_clean":"2023-09-23T20:52:53.874221+0000","last_became_active":"2023-09-23T20:50:37.981797+0000","last_became_peered":"2023-09-23T20:50:37.981797+0000","last_unstale":"2023-09-23T20:52:53.874221+0000","last_undegraded":"2023-09-23T20:52:53.874221+0000","last_fullsized":"2023-09-23T20:52:53.874221+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:56:45.027447+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.e","version":"116'4420","reported_seq":4442,"reported_epoch":116,"state":"active+clean","last_fresh":"2023-09-23T20:53:07.071181+0000","last_change":"2023-09-23T20:52:14.862981+0000","last_active":"2023-09-23T20:53:07.071181+0000","last_peered":"2023-09-23T20:53:07.071181+0000","last_clean":"2023-09-23T20:53:07.071181+0000","last_became_active":"2023-09-23T20:52:14.862783+0000","last_became_peered":"2023-09-23T20:52:14.862783+0000","last_unstale":"2023-09-23T20:53:07.071181+0000","last_undegraded":"2023-09-23T20:53:07.071181+0000","last_fullsized":"2023-09-23T20:53:07.071181+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":102,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:52:13.843048+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:52:13.843048+0000","last_clean_scrub_stamp":"2023-09-23T20:52:13.843048+0000","objects_scrubbed":0,"log_size":4420,"log_dups_size":0,"ondisk_log_size":4420,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:39:24.557635+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":289669120,"num_objects":4420,"num_object_clones":0,"num_object_copies":13260,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4420,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4420,"num_write_kb":282880,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":76,"reported_epoch":115,"state":"active+clean","last_fresh":"2023-09-23T20:53:04.002495+0000","last_change":"2023-09-23T20:51:10.836752+0000","last_active":"2023-09-23T20:53:04.002495+0000","last_peered":"2023-09-23T20:53:04.002495+0000","last_clean":"2023-09-23T20:53:04.002495+0000","last_became_active":"2023-09-23T20:51:10.836213+0000","last_became_peered":"2023-09-23T20:51:10.836213+0000","last_unstale":"2023-09-23T20:53:04.002495+0000","last_undegraded":"2023-09-23T20:53:04.002495+0000","last_fullsized":"2023-09-23T20:53:04.002495+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:35:01.825635+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":75,"reported_epoch":115,"state":"active+clean","last_fresh":"2023-09-23T20:53:03.999259+0000","last_change":"2023-09-23T20:51:10.833630+0000","last_active":"2023-09-23T20:53:03.999259+0000","last_peered":"2023-09-23T20:53:03.999259+0000","last_clean":"2023-09-23T20:53:03.999259+0000","last_became_active":"2023-09-23T20:51:10.833265+0000","last_became_peered":"2023-09-23T20:51:10.833265+0000","last_unstale":"2023-09-23T20:53:03.999259+0000","last_undegraded":"2023-09-23T20:53:03.999259+0000","last_fullsized":"2023-09-23T20:53:03.999259+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:12:01.557967+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.f","version":"115'4265","reported_seq":4287,"reported_epoch":115,"state":"active+clean","last_fresh":"2023-09-23T20:53:05.996055+0000","last_change":"2023-09-23T20:52:14.863531+0000","last_active":"2023-09-23T20:53:05.996055+0000","last_peered":"2023-09-23T20:53:05.996055+0000","last_clean":"2023-09-23T20:53:05.996055+0000","last_became_active":"2023-09-23T20:52:14.862972+0000","last_became_peered":"2023-09-23T20:52:14.862972+0000","last_unstale":"2023-09-23T20:53:05.996055+0000","last_undegraded":"2023-09-23T20:53:05.996055+0000","last_fullsized":"2023-09-23T20:53:05.996055+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":102,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:52:13.843048+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:52:13.843048+0000","last_clean_scrub_stamp":"2023-09-23T20:52:13.843048+0000","objects_scrubbed":0,"log_size":4265,"log_dups_size":0,"ondisk_log_size":4265,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:23:46.230633+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":279511040,"num_objects":4265,"num_object_clones":0,"num_object_copies":12795,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4265,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4265,"num_write_kb":272960,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":76,"reported_epoch":115,"state":"active+clean","last_fresh":"2023-09-23T20:53:04.000328+0000","last_change":"2023-09-23T20:50:05.867646+0000","last_active":"2023-09-23T20:53:04.000328+0000","last_peered":"2023-09-23T20:53:04.000328+0000","last_clean":"2023-09-23T20:53:04.000328+0000","last_became_active":"2023-09-23T20:50:05.867343+0000","last_became_peered":"2023-09-23T20:50:05.867343+0000","last_unstale":"2023-09-23T20:53:04.000328+0000","last_undegraded":"2023-09-23T20:53:04.000328+0000","last_fullsized":"2023-09-23T20:53:04.000328+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:53:34.985580+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":92,"reported_epoch":115,"state":"active+clean","last_fresh":"2023-09-23T20:53:03.998686+0000","last_change":"2023-09-23T20:50:37.985593+0000","last_active":"2023-09-23T20:53:03.998686+0000","last_peered":"2023-09-23T20:53:03.998686+0000","last_clean":"2023-09-23T20:53:03.998686+0000","last_became_active":"2023-09-23T20:50:37.985324+0000","last_became_peered":"2023-09-23T20:50:37.985324+0000","last_unstale":"2023-09-23T20:53:03.998686+0000","last_undegraded":"2023-09-23T20:53:03.998686+0000","last_fullsized":"2023-09-23T20:53:03.998686+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:34:07.801771+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.c","version":"116'4447","reported_seq":4470,"reported_epoch":116,"state":"active+clean","last_fresh":"2023-09-23T20:53:08.829939+0000","last_change":"2023-09-23T20:52:14.865953+0000","last_active":"2023-09-23T20:53:08.829939+0000","last_peered":"2023-09-23T20:53:08.829939+0000","last_clean":"2023-09-23T20:53:08.829939+0000","last_became_active":"2023-09-23T20:52:14.865335+0000","last_became_peered":"2023-09-23T20:52:14.865335+0000","last_unstale":"2023-09-23T20:53:08.829939+0000","last_undegraded":"2023-09-23T20:53:08.829939+0000","last_fullsized":"2023-09-23T20:53:08.829939+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":102,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:52:13.843048+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:52:13.843048+0000","last_clean_scrub_stamp":"2023-09-23T20:52:13.843048+0000","objects_scrubbed":0,"log_size":4447,"log_dups_size":0,"ondisk_log_size":4447,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:20:00.174949+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":291438592,"num_objects":4447,"num_object_clones":0,"num_object_copies":13341,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4447,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4447,"num_write_kb":284608,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":76,"reported_epoch":115,"state":"active+clean","last_fresh":"2023-09-23T20:53:03.998124+0000","last_change":"2023-09-23T20:50:37.983914+0000","last_active":"2023-09-23T20:53:03.998124+0000","last_peered":"2023-09-23T20:53:03.998124+0000","last_clean":"2023-09-23T20:53:03.998124+0000","last_became_active":"2023-09-23T20:50:37.983617+0000","last_became_peered":"2023-09-23T20:50:37.983617+0000","last_unstale":"2023-09-23T20:53:03.998124+0000","last_undegraded":"2023-09-23T20:53:03.998124+0000","last_fullsized":"2023-09-23T20:53:03.998124+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:15:50.863306+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":75,"reported_epoch":115,"state":"active+clean","last_fresh":"2023-09-23T20:53:03.998341+0000","last_change":"2023-09-23T20:50:05.871406+0000","last_active":"2023-09-23T20:53:03.998341+0000","last_peered":"2023-09-23T20:53:03.998341+0000","last_clean":"2023-09-23T20:53:03.998341+0000","last_became_active":"2023-09-23T20:50:05.871127+0000","last_became_peered":"2023-09-23T20:50:05.871127+0000","last_unstale":"2023-09-23T20:53:03.998341+0000","last_undegraded":"2023-09-23T20:53:03.998341+0000","last_fullsized":"2023-09-23T20:53:03.998341+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:37:48.850229+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.d","version":"116'4543","reported_seq":4566,"reported_epoch":116,"state":"active+clean","last_fresh":"2023-09-23T20:53:08.877889+0000","last_change":"2023-09-23T20:52:14.867585+0000","last_active":"2023-09-23T20:53:08.877889+0000","last_peered":"2023-09-23T20:53:08.877889+0000","last_clean":"2023-09-23T20:53:08.877889+0000","last_became_active":"2023-09-23T20:52:14.867310+0000","last_became_peered":"2023-09-23T20:52:14.867310+0000","last_unstale":"2023-09-23T20:53:08.877889+0000","last_undegraded":"2023-09-23T20:53:08.877889+0000","last_fullsized":"2023-09-23T20:53:08.877889+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":102,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:52:13.843048+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:52:13.843048+0000","last_clean_scrub_stamp":"2023-09-23T20:52:13.843048+0000","objects_scrubbed":0,"log_size":4543,"log_dups_size":0,"ondisk_log_size":4543,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:19:26.814547+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":297730048,"num_objects":4543,"num_object_clones":0,"num_object_copies":13629,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4543,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4543,"num_write_kb":290752,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,2],"acting":[1,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":79,"reported_epoch":115,"state":"active+clean","last_fresh":"2023-09-23T20:53:04.002171+0000","last_change":"2023-09-23T20:51:10.831197+0000","last_active":"2023-09-23T20:53:04.002171+0000","last_peered":"2023-09-23T20:53:04.002171+0000","last_clean":"2023-09-23T20:53:04.002171+0000","last_became_active":"2023-09-23T20:51:10.830878+0000","last_became_peered":"2023-09-23T20:51:10.830878+0000","last_unstale":"2023-09-23T20:53:04.002171+0000","last_undegraded":"2023-09-23T20:53:04.002171+0000","last_fullsized":"2023-09-23T20:53:04.002171+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:05:24.314478+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":65,"reported_epoch":115,"state":"active+clean","last_fresh":"2023-09-23T20:53:04.029093+0000","last_change":"2023-09-23T20:51:10.833850+0000","last_active":"2023-09-23T20:53:04.029093+0000","last_peered":"2023-09-23T20:53:04.029093+0000","last_clean":"2023-09-23T20:53:04.029093+0000","last_became_active":"2023-09-23T20:51:10.833539+0000","last_became_peered":"2023-09-23T20:51:10.833539+0000","last_unstale":"2023-09-23T20:53:04.029093+0000","last_undegraded":"2023-09-23T20:53:04.029093+0000","last_fullsized":"2023-09-23T20:53:04.029093+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:01:36.445616+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":88,"reported_epoch":115,"state":"active+clean","last_fresh":"2023-09-23T20:53:04.028422+0000","last_change":"2023-09-23T20:50:05.867130+0000","last_active":"2023-09-23T20:53:04.028422+0000","last_peered":"2023-09-23T20:53:04.028422+0000","last_clean":"2023-09-23T20:53:04.028422+0000","last_became_active":"2023-09-23T20:50:05.866758+0000","last_became_peered":"2023-09-23T20:50:05.866758+0000","last_unstale":"2023-09-23T20:53:04.028422+0000","last_undegraded":"2023-09-23T20:53:04.028422+0000","last_fullsized":"2023-09-23T20:53:04.028422+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:55:07.548209+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":94,"reported_epoch":115,"state":"active+clean","last_fresh":"2023-09-23T20:53:04.003162+0000","last_change":"2023-09-23T20:52:12.857372+0000","last_active":"2023-09-23T20:53:04.003162+0000","last_peered":"2023-09-23T20:53:04.003162+0000","last_clean":"2023-09-23T20:53:04.003162+0000","last_became_active":"2023-09-23T20:52:12.857135+0000","last_became_peered":"2023-09-23T20:52:12.857135+0000","last_unstale":"2023-09-23T20:53:04.003162+0000","last_undegraded":"2023-09-23T20:53:04.003162+0000","last_fullsized":"2023-09-23T20:53:04.003162+0000","mapping_epoch":100,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":101,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:50:17.844004+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":59,"reported_epoch":115,"state":"active+clean","last_fresh":"2023-09-23T20:53:04.028434+0000","last_change":"2023-09-23T20:52:12.854192+0000","last_active":"2023-09-23T20:53:04.028434+0000","last_peered":"2023-09-23T20:53:04.028434+0000","last_clean":"2023-09-23T20:53:04.028434+0000","last_became_active":"2023-09-23T20:52:12.853812+0000","last_became_peered":"2023-09-23T20:52:12.853812+0000","last_unstale":"2023-09-23T20:53:04.028434+0000","last_undegraded":"2023-09-23T20:53:04.028434+0000","last_fullsized":"2023-09-23T20:53:04.028434+0000","mapping_epoch":100,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":101,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:36:13.376752+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.0","version":"115'4347","reported_seq":4369,"reported_epoch":115,"state":"active+clean","last_fresh":"2023-09-23T20:53:05.779300+0000","last_change":"2023-09-23T20:52:14.866690+0000","last_active":"2023-09-23T20:53:05.779300+0000","last_peered":"2023-09-23T20:53:05.779300+0000","last_clean":"2023-09-23T20:53:05.779300+0000","last_became_active":"2023-09-23T20:52:14.866408+0000","last_became_peered":"2023-09-23T20:52:14.866408+0000","last_unstale":"2023-09-23T20:53:05.779300+0000","last_undegraded":"2023-09-23T20:53:05.779300+0000","last_fullsized":"2023-09-23T20:53:05.779300+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":102,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:52:13.843048+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:52:13.843048+0000","last_clean_scrub_stamp":"2023-09-23T20:52:13.843048+0000","objects_scrubbed":0,"log_size":4347,"log_dups_size":0,"ondisk_log_size":4347,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:29:07.905971+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":284884992,"num_objects":4347,"num_object_clones":0,"num_object_copies":13041,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4347,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4347,"num_write_kb":278208,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.6","version":"116'4417","reported_seq":4440,"reported_epoch":116,"state":"active+clean","last_fresh":"2023-09-23T20:53:08.794236+0000","last_change":"2023-09-23T20:52:14.865161+0000","last_active":"2023-09-23T20:53:08.794236+0000","last_peered":"2023-09-23T20:53:08.794236+0000","last_clean":"2023-09-23T20:53:08.794236+0000","last_became_active":"2023-09-23T20:52:14.864806+0000","last_became_peered":"2023-09-23T20:52:14.864806+0000","last_unstale":"2023-09-23T20:53:08.794236+0000","last_undegraded":"2023-09-23T20:53:08.794236+0000","last_fullsized":"2023-09-23T20:53:08.794236+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":102,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:52:13.843048+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:52:13.843048+0000","last_clean_scrub_stamp":"2023-09-23T20:52:13.843048+0000","objects_scrubbed":0,"log_size":4417,"log_dups_size":0,"ondisk_log_size":4417,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:05:57.391225+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":289472512,"num_objects":4417,"num_object_clones":0,"num_object_copies":13251,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4417,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4417,"num_write_kb":282688,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":71,"reported_epoch":115,"state":"active+clean","last_fresh":"2023-09-23T20:53:04.028290+0000","last_change":"2023-09-23T20:52:12.860753+0000","last_active":"2023-09-23T20:53:04.028290+0000","last_peered":"2023-09-23T20:53:04.028290+0000","last_clean":"2023-09-23T20:53:04.028290+0000","last_became_active":"2023-09-23T20:52:12.860394+0000","last_became_peered":"2023-09-23T20:52:12.860394+0000","last_unstale":"2023-09-23T20:53:04.028290+0000","last_undegraded":"2023-09-23T20:53:04.028290+0000","last_fullsized":"2023-09-23T20:53:04.028290+0000","mapping_epoch":100,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":101,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:44:12.690612+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":76,"reported_epoch":115,"state":"active+clean","last_fresh":"2023-09-23T20:53:03.998802+0000","last_change":"2023-09-23T20:49:32.771117+0000","last_active":"2023-09-23T20:53:03.998802+0000","last_peered":"2023-09-23T20:53:03.998802+0000","last_clean":"2023-09-23T20:53:03.998802+0000","last_became_active":"2023-09-23T20:49:32.770686+0000","last_became_peered":"2023-09-23T20:49:32.770686+0000","last_unstale":"2023-09-23T20:53:03.998802+0000","last_undegraded":"2023-09-23T20:53:03.998802+0000","last_fullsized":"2023-09-23T20:53:03.998802+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:15:46.598698+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":80,"reported_epoch":115,"state":"active+clean","last_fresh":"2023-09-23T20:53:04.028200+0000","last_change":"2023-09-23T20:51:10.829889+0000","last_active":"2023-09-23T20:53:04.028200+0000","last_peered":"2023-09-23T20:53:04.028200+0000","last_clean":"2023-09-23T20:53:04.028200+0000","last_became_active":"2023-09-23T20:51:10.829533+0000","last_became_peered":"2023-09-23T20:51:10.829533+0000","last_unstale":"2023-09-23T20:53:04.028200+0000","last_undegraded":"2023-09-23T20:53:04.028200+0000","last_fullsized":"2023-09-23T20:53:04.028200+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:43:04.603460+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.5","version":"116'4348","reported_seq":4370,"reported_epoch":116,"state":"active+clean","last_fresh":"2023-09-23T20:53:07.032313+0000","last_change":"2023-09-23T20:52:14.869287+0000","last_active":"2023-09-23T20:53:07.032313+0000","last_peered":"2023-09-23T20:53:07.032313+0000","last_clean":"2023-09-23T20:53:07.032313+0000","last_became_active":"2023-09-23T20:52:14.868955+0000","last_became_peered":"2023-09-23T20:52:14.868955+0000","last_unstale":"2023-09-23T20:53:07.032313+0000","last_undegraded":"2023-09-23T20:53:07.032313+0000","last_fullsized":"2023-09-23T20:53:07.032313+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":102,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:52:13.843048+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:52:13.843048+0000","last_clean_scrub_stamp":"2023-09-23T20:52:13.843048+0000","objects_scrubbed":0,"log_size":4348,"log_dups_size":0,"ondisk_log_size":4348,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:05:41.908990+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":284950528,"num_objects":4348,"num_object_clones":0,"num_object_copies":13044,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4348,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4348,"num_write_kb":278272,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,3],"acting":[7,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":98,"reported_epoch":114,"state":"active+clean","last_fresh":"2023-09-23T20:52:53.880559+0000","last_change":"2023-09-23T20:51:10.838054+0000","last_active":"2023-09-23T20:52:53.880559+0000","last_peered":"2023-09-23T20:52:53.880559+0000","last_clean":"2023-09-23T20:52:53.880559+0000","last_became_active":"2023-09-23T20:51:10.837750+0000","last_became_peered":"2023-09-23T20:51:10.837750+0000","last_unstale":"2023-09-23T20:52:53.880559+0000","last_undegraded":"2023-09-23T20:52:53.880559+0000","last_fullsized":"2023-09-23T20:52:53.880559+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:15:51.648085+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":65,"reported_epoch":115,"state":"active+clean","last_fresh":"2023-09-23T20:53:04.028010+0000","last_change":"2023-09-23T20:50:37.983072+0000","last_active":"2023-09-23T20:53:04.028010+0000","last_peered":"2023-09-23T20:53:04.028010+0000","last_clean":"2023-09-23T20:53:04.028010+0000","last_became_active":"2023-09-23T20:50:37.982777+0000","last_became_peered":"2023-09-23T20:50:37.982777+0000","last_unstale":"2023-09-23T20:53:04.028010+0000","last_undegraded":"2023-09-23T20:53:04.028010+0000","last_fullsized":"2023-09-23T20:53:04.028010+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:58:11.972144+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":92,"reported_epoch":115,"state":"active+clean","last_fresh":"2023-09-23T20:53:04.029166+0000","last_change":"2023-09-23T20:52:12.864016+0000","last_active":"2023-09-23T20:53:04.029166+0000","last_peered":"2023-09-23T20:53:04.029166+0000","last_clean":"2023-09-23T20:53:04.029166+0000","last_became_active":"2023-09-23T20:52:12.863714+0000","last_became_peered":"2023-09-23T20:52:12.863714+0000","last_unstale":"2023-09-23T20:53:04.029166+0000","last_undegraded":"2023-09-23T20:53:04.029166+0000","last_fullsized":"2023-09-23T20:53:04.029166+0000","mapping_epoch":100,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":101,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:20:44.035420+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.7","version":"116'4538","reported_seq":4559,"reported_epoch":116,"state":"active+clean","last_fresh":"2023-09-23T20:53:08.827227+0000","last_change":"2023-09-23T20:52:14.867968+0000","last_active":"2023-09-23T20:53:08.827227+0000","last_peered":"2023-09-23T20:53:08.827227+0000","last_clean":"2023-09-23T20:53:08.827227+0000","last_became_active":"2023-09-23T20:52:14.867682+0000","last_became_peered":"2023-09-23T20:52:14.867682+0000","last_unstale":"2023-09-23T20:53:08.827227+0000","last_undegraded":"2023-09-23T20:53:08.827227+0000","last_fullsized":"2023-09-23T20:53:08.827227+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":102,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:52:13.843048+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:52:13.843048+0000","last_clean_scrub_stamp":"2023-09-23T20:52:13.843048+0000","objects_scrubbed":0,"log_size":4538,"log_dups_size":0,"ondisk_log_size":4538,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:24:49.928264+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":297402368,"num_objects":4538,"num_object_clones":0,"num_object_copies":13614,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4538,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4538,"num_write_kb":290432,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":76,"reported_epoch":115,"state":"active+clean","last_fresh":"2023-09-23T20:53:04.005565+0000","last_change":"2023-09-23T20:52:12.853881+0000","last_active":"2023-09-23T20:53:04.005565+0000","last_peered":"2023-09-23T20:53:04.005565+0000","last_clean":"2023-09-23T20:53:04.005565+0000","last_became_active":"2023-09-23T20:52:12.853545+0000","last_became_peered":"2023-09-23T20:52:12.853545+0000","last_unstale":"2023-09-23T20:53:04.005565+0000","last_undegraded":"2023-09-23T20:53:04.005565+0000","last_fullsized":"2023-09-23T20:53:04.005565+0000","mapping_epoch":100,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":101,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:18:05.760976+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,3],"acting":[2,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"79'161","reported_seq":457,"reported_epoch":115,"state":"active+clean","last_fresh":"2023-09-23T20:53:03.997842+0000","last_change":"2023-09-23T20:49:32.777246+0000","last_active":"2023-09-23T20:53:03.997842+0000","last_peered":"2023-09-23T20:53:03.997842+0000","last_clean":"2023-09-23T20:53:03.997842+0000","last_became_active":"2023-09-23T20:49:32.776822+0000","last_became_peered":"2023-09-23T20:49:32.776822+0000","last_unstale":"2023-09-23T20:53:03.997842+0000","last_undegraded":"2023-09-23T20:53:03.997842+0000","last_fullsized":"2023-09-23T20:53:03.997842+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:38:57.324224+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:05:46.881275+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":48,"reported_epoch":115,"state":"active+clean","last_fresh":"2023-09-23T20:53:04.002674+0000","last_change":"2023-09-23T20:52:12.860483+0000","last_active":"2023-09-23T20:53:04.002674+0000","last_peered":"2023-09-23T20:53:04.002674+0000","last_clean":"2023-09-23T20:53:04.002674+0000","last_became_active":"2023-09-23T20:52:12.860093+0000","last_became_peered":"2023-09-23T20:52:12.860093+0000","last_unstale":"2023-09-23T20:53:04.002674+0000","last_undegraded":"2023-09-23T20:53:04.002674+0000","last_fullsized":"2023-09-23T20:53:04.002674+0000","mapping_epoch":100,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":101,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:01:28.713922+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,4],"acting":[3,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":58,"reported_epoch":115,"state":"active+clean","last_fresh":"2023-09-23T20:53:04.001327+0000","last_change":"2023-09-23T20:41:59.062981+0000","last_active":"2023-09-23T20:53:04.001327+0000","last_peered":"2023-09-23T20:53:04.001327+0000","last_clean":"2023-09-23T20:53:04.001327+0000","last_became_active":"2023-09-23T20:41:59.062717+0000","last_became_peered":"2023-09-23T20:41:59.062717+0000","last_unstale":"2023-09-23T20:53:04.001327+0000","last_undegraded":"2023-09-23T20:53:04.001327+0000","last_fullsized":"2023-09-23T20:53:04.001327+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:20:58.386461+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":97,"reported_epoch":115,"state":"active+clean","last_fresh":"2023-09-23T20:53:03.997858+0000","last_change":"2023-09-23T20:50:05.866898+0000","last_active":"2023-09-23T20:53:03.997858+0000","last_peered":"2023-09-23T20:53:03.997858+0000","last_clean":"2023-09-23T20:53:03.997858+0000","last_became_active":"2023-09-23T20:50:05.866541+0000","last_became_peered":"2023-09-23T20:50:05.866541+0000","last_unstale":"2023-09-23T20:53:03.997858+0000","last_undegraded":"2023-09-23T20:53:03.997858+0000","last_fullsized":"2023-09-23T20:53:03.997858+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:17:02.353010+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":69,"reported_epoch":114,"state":"active+clean","last_fresh":"2023-09-23T20:52:53.875631+0000","last_change":"2023-09-23T20:51:10.841350+0000","last_active":"2023-09-23T20:52:53.875631+0000","last_peered":"2023-09-23T20:52:53.875631+0000","last_clean":"2023-09-23T20:52:53.875631+0000","last_became_active":"2023-09-23T20:51:10.841029+0000","last_became_peered":"2023-09-23T20:51:10.841029+0000","last_unstale":"2023-09-23T20:52:53.875631+0000","last_undegraded":"2023-09-23T20:52:53.875631+0000","last_fullsized":"2023-09-23T20:52:53.875631+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:26:05.095206+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":84,"reported_epoch":114,"state":"active+clean","last_fresh":"2023-09-23T20:52:53.874628+0000","last_change":"2023-09-23T20:50:37.982235+0000","last_active":"2023-09-23T20:52:53.874628+0000","last_peered":"2023-09-23T20:52:53.874628+0000","last_clean":"2023-09-23T20:52:53.874628+0000","last_became_active":"2023-09-23T20:50:37.981913+0000","last_became_peered":"2023-09-23T20:50:37.981913+0000","last_unstale":"2023-09-23T20:52:53.874628+0000","last_undegraded":"2023-09-23T20:52:53.874628+0000","last_fullsized":"2023-09-23T20:52:53.874628+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:59:57.122584+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.4","version":"116'4522","reported_seq":4545,"reported_epoch":116,"state":"active+clean","last_fresh":"2023-09-23T20:53:08.861966+0000","last_change":"2023-09-23T20:52:14.869618+0000","last_active":"2023-09-23T20:53:08.861966+0000","last_peered":"2023-09-23T20:53:08.861966+0000","last_clean":"2023-09-23T20:53:08.861966+0000","last_became_active":"2023-09-23T20:52:14.869288+0000","last_became_peered":"2023-09-23T20:52:14.869288+0000","last_unstale":"2023-09-23T20:53:08.861966+0000","last_undegraded":"2023-09-23T20:53:08.861966+0000","last_fullsized":"2023-09-23T20:53:08.861966+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":102,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:52:13.843048+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:52:13.843048+0000","last_clean_scrub_stamp":"2023-09-23T20:52:13.843048+0000","objects_scrubbed":0,"log_size":4522,"log_dups_size":0,"ondisk_log_size":4522,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:48:22.341973+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":296353792,"num_objects":4522,"num_object_clones":0,"num_object_copies":13566,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4522,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4522,"num_write_kb":289408,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":80,"reported_epoch":115,"state":"active+clean","last_fresh":"2023-09-23T20:53:04.000501+0000","last_change":"2023-09-23T20:49:32.776947+0000","last_active":"2023-09-23T20:53:04.000501+0000","last_peered":"2023-09-23T20:53:04.000501+0000","last_clean":"2023-09-23T20:53:04.000501+0000","last_became_active":"2023-09-23T20:49:32.776646+0000","last_became_peered":"2023-09-23T20:49:32.776646+0000","last_unstale":"2023-09-23T20:53:04.000501+0000","last_undegraded":"2023-09-23T20:53:04.000501+0000","last_fullsized":"2023-09-23T20:53:04.000501+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:26:50.532203+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":74,"reported_epoch":115,"state":"active+clean","last_fresh":"2023-09-23T20:53:04.028344+0000","last_change":"2023-09-23T20:52:12.853904+0000","last_active":"2023-09-23T20:53:04.028344+0000","last_peered":"2023-09-23T20:53:04.028344+0000","last_clean":"2023-09-23T20:53:04.028344+0000","last_became_active":"2023-09-23T20:52:12.853589+0000","last_became_peered":"2023-09-23T20:52:12.853589+0000","last_unstale":"2023-09-23T20:53:04.028344+0000","last_undegraded":"2023-09-23T20:53:04.028344+0000","last_fullsized":"2023-09-23T20:53:04.028344+0000","mapping_epoch":100,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":101,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:36:58.772320+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":63,"reported_epoch":114,"state":"active+clean","last_fresh":"2023-09-23T20:52:53.874084+0000","last_change":"2023-09-23T20:52:12.861675+0000","last_active":"2023-09-23T20:52:53.874084+0000","last_peered":"2023-09-23T20:52:53.874084+0000","last_clean":"2023-09-23T20:52:53.874084+0000","last_became_active":"2023-09-23T20:52:12.861342+0000","last_became_peered":"2023-09-23T20:52:12.861342+0000","last_unstale":"2023-09-23T20:52:53.874084+0000","last_undegraded":"2023-09-23T20:52:53.874084+0000","last_fullsized":"2023-09-23T20:52:53.874084+0000","mapping_epoch":100,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":101,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:37:54.207520+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.1","version":"116'4576","reported_seq":4599,"reported_epoch":116,"state":"active+clean","last_fresh":"2023-09-23T20:53:08.829102+0000","last_change":"2023-09-23T20:52:14.866659+0000","last_active":"2023-09-23T20:53:08.829102+0000","last_peered":"2023-09-23T20:53:08.829102+0000","last_clean":"2023-09-23T20:53:08.829102+0000","last_became_active":"2023-09-23T20:52:14.866348+0000","last_became_peered":"2023-09-23T20:52:14.866348+0000","last_unstale":"2023-09-23T20:53:08.829102+0000","last_undegraded":"2023-09-23T20:53:08.829102+0000","last_fullsized":"2023-09-23T20:53:08.829102+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":102,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:52:13.843048+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:52:13.843048+0000","last_clean_scrub_stamp":"2023-09-23T20:52:13.843048+0000","objects_scrubbed":0,"log_size":4576,"log_dups_size":0,"ondisk_log_size":4576,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:51:07.595313+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":299892736,"num_objects":4576,"num_object_clones":0,"num_object_copies":13728,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4576,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4576,"num_write_kb":292864,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":76,"reported_epoch":115,"state":"active+clean","last_fresh":"2023-09-23T20:53:04.003326+0000","last_change":"2023-09-23T20:51:10.836910+0000","last_active":"2023-09-23T20:53:04.003326+0000","last_peered":"2023-09-23T20:53:04.003326+0000","last_clean":"2023-09-23T20:53:04.003326+0000","last_became_active":"2023-09-23T20:51:10.836557+0000","last_became_peered":"2023-09-23T20:51:10.836557+0000","last_unstale":"2023-09-23T20:53:04.003326+0000","last_undegraded":"2023-09-23T20:53:04.003326+0000","last_fullsized":"2023-09-23T20:53:04.003326+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:51:54.019969+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":50,"reported_epoch":115,"state":"active+clean","last_fresh":"2023-09-23T20:53:04.028730+0000","last_change":"2023-09-23T20:50:05.868392+0000","last_active":"2023-09-23T20:53:04.028730+0000","last_peered":"2023-09-23T20:53:04.028730+0000","last_clean":"2023-09-23T20:53:04.028730+0000","last_became_active":"2023-09-23T20:50:05.867910+0000","last_became_peered":"2023-09-23T20:50:05.867910+0000","last_unstale":"2023-09-23T20:53:04.028730+0000","last_undegraded":"2023-09-23T20:53:04.028730+0000","last_fullsized":"2023-09-23T20:53:04.028730+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:25:00.099993+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":74,"reported_epoch":116,"state":"active+clean","last_fresh":"2023-09-23T20:53:06.035434+0000","last_change":"2023-09-23T20:51:10.833546+0000","last_active":"2023-09-23T20:53:06.035434+0000","last_peered":"2023-09-23T20:53:06.035434+0000","last_clean":"2023-09-23T20:53:06.035434+0000","last_became_active":"2023-09-23T20:51:10.833137+0000","last_became_peered":"2023-09-23T20:51:10.833137+0000","last_unstale":"2023-09-23T20:53:06.035434+0000","last_undegraded":"2023-09-23T20:53:06.035434+0000","last_fullsized":"2023-09-23T20:53:06.035434+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:16:42.840695+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.2","version":"116'4299","reported_seq":4320,"reported_epoch":116,"state":"active+clean","last_fresh":"2023-09-23T20:53:07.073873+0000","last_change":"2023-09-23T20:52:14.866208+0000","last_active":"2023-09-23T20:53:07.073873+0000","last_peered":"2023-09-23T20:53:07.073873+0000","last_clean":"2023-09-23T20:53:07.073873+0000","last_became_active":"2023-09-23T20:52:14.865956+0000","last_became_peered":"2023-09-23T20:52:14.865956+0000","last_unstale":"2023-09-23T20:53:07.073873+0000","last_undegraded":"2023-09-23T20:53:07.073873+0000","last_fullsized":"2023-09-23T20:53:07.073873+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":102,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:52:13.843048+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:52:13.843048+0000","last_clean_scrub_stamp":"2023-09-23T20:52:13.843048+0000","objects_scrubbed":0,"log_size":4299,"log_dups_size":0,"ondisk_log_size":4299,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:38:02.533042+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":281739264,"num_objects":4299,"num_object_clones":0,"num_object_copies":12897,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4299,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4299,"num_write_kb":275136,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":80,"reported_epoch":115,"state":"active+clean","last_fresh":"2023-09-23T20:53:04.000100+0000","last_change":"2023-09-23T20:50:37.985663+0000","last_active":"2023-09-23T20:53:04.000100+0000","last_peered":"2023-09-23T20:53:04.000100+0000","last_clean":"2023-09-23T20:53:04.000100+0000","last_became_active":"2023-09-23T20:50:37.985325+0000","last_became_peered":"2023-09-23T20:50:37.985325+0000","last_unstale":"2023-09-23T20:53:04.000100+0000","last_undegraded":"2023-09-23T20:53:04.000100+0000","last_fullsized":"2023-09-23T20:53:04.000100+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:24:30.838843+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"70'1","reported_seq":74,"reported_epoch":114,"state":"active+clean","last_fresh":"2023-09-23T20:52:53.874558+0000","last_change":"2023-09-23T20:50:05.871179+0000","last_active":"2023-09-23T20:52:53.874558+0000","last_peered":"2023-09-23T20:52:53.874558+0000","last_clean":"2023-09-23T20:52:53.874558+0000","last_became_active":"2023-09-23T20:50:05.870816+0000","last_became_peered":"2023-09-23T20:50:05.870816+0000","last_unstale":"2023-09-23T20:52:53.874558+0000","last_undegraded":"2023-09-23T20:52:53.874558+0000","last_fullsized":"2023-09-23T20:52:53.874558+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:25:24.455835+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":68,"reported_epoch":115,"state":"active+clean","last_fresh":"2023-09-23T20:53:03.998207+0000","last_change":"2023-09-23T20:52:12.860323+0000","last_active":"2023-09-23T20:53:03.998207+0000","last_peered":"2023-09-23T20:53:03.998207+0000","last_clean":"2023-09-23T20:53:03.998207+0000","last_became_active":"2023-09-23T20:52:12.859984+0000","last_became_peered":"2023-09-23T20:52:12.859984+0000","last_unstale":"2023-09-23T20:53:03.998207+0000","last_undegraded":"2023-09-23T20:53:03.998207+0000","last_fullsized":"2023-09-23T20:53:03.998207+0000","mapping_epoch":100,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":101,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:53:33.240133+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.3","version":"116'4453","reported_seq":4475,"reported_epoch":116,"state":"active+clean","last_fresh":"2023-09-23T20:53:07.086874+0000","last_change":"2023-09-23T20:52:14.865640+0000","last_active":"2023-09-23T20:53:07.086874+0000","last_peered":"2023-09-23T20:53:07.086874+0000","last_clean":"2023-09-23T20:53:07.086874+0000","last_became_active":"2023-09-23T20:52:14.865328+0000","last_became_peered":"2023-09-23T20:52:14.865328+0000","last_unstale":"2023-09-23T20:53:07.086874+0000","last_undegraded":"2023-09-23T20:53:07.086874+0000","last_fullsized":"2023-09-23T20:53:07.086874+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":102,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:52:13.843048+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:52:13.843048+0000","last_clean_scrub_stamp":"2023-09-23T20:52:13.843048+0000","objects_scrubbed":0,"log_size":4453,"log_dups_size":0,"ondisk_log_size":4453,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:04:22.193022+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":291831808,"num_objects":4453,"num_object_clones":0,"num_object_copies":13359,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4453,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4453,"num_write_kb":284992,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,2],"acting":[7,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":57,"reported_epoch":114,"state":"active+clean","last_fresh":"2023-09-23T20:52:53.874304+0000","last_change":"2023-09-23T20:49:32.775562+0000","last_active":"2023-09-23T20:52:53.874304+0000","last_peered":"2023-09-23T20:52:53.874304+0000","last_clean":"2023-09-23T20:52:53.874304+0000","last_became_active":"2023-09-23T20:49:32.775094+0000","last_became_peered":"2023-09-23T20:49:32.775094+0000","last_unstale":"2023-09-23T20:52:53.874304+0000","last_undegraded":"2023-09-23T20:52:53.874304+0000","last_fullsized":"2023-09-23T20:52:53.874304+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:17:52.944043+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":82,"reported_epoch":116,"state":"active+clean","last_fresh":"2023-09-23T20:53:06.023379+0000","last_change":"2023-09-23T20:52:12.855430+0000","last_active":"2023-09-23T20:53:06.023379+0000","last_peered":"2023-09-23T20:53:06.023379+0000","last_clean":"2023-09-23T20:53:06.023379+0000","last_became_active":"2023-09-23T20:52:12.855115+0000","last_became_peered":"2023-09-23T20:52:12.855115+0000","last_unstale":"2023-09-23T20:53:06.023379+0000","last_undegraded":"2023-09-23T20:53:06.023379+0000","last_fullsized":"2023-09-23T20:53:06.023379+0000","mapping_epoch":100,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":101,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:33:23.802214+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":42,"reported_epoch":115,"state":"active+clean","last_fresh":"2023-09-23T20:53:03.999117+0000","last_change":"2023-09-23T20:52:12.863936+0000","last_active":"2023-09-23T20:53:03.999117+0000","last_peered":"2023-09-23T20:53:03.999117+0000","last_clean":"2023-09-23T20:53:03.999117+0000","last_became_active":"2023-09-23T20:52:12.855724+0000","last_became_peered":"2023-09-23T20:52:12.855724+0000","last_unstale":"2023-09-23T20:53:03.999117+0000","last_undegraded":"2023-09-23T20:53:03.999117+0000","last_fullsized":"2023-09-23T20:53:03.999117+0000","mapping_epoch":100,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":101,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:12:48.714857+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":92,"reported_epoch":115,"state":"active+clean","last_fresh":"2023-09-23T20:53:03.998985+0000","last_change":"2023-09-23T20:51:10.833684+0000","last_active":"2023-09-23T20:53:03.998985+0000","last_peered":"2023-09-23T20:53:03.998985+0000","last_clean":"2023-09-23T20:53:03.998985+0000","last_became_active":"2023-09-23T20:51:10.833360+0000","last_became_peered":"2023-09-23T20:51:10.833360+0000","last_unstale":"2023-09-23T20:53:03.998985+0000","last_undegraded":"2023-09-23T20:53:03.998985+0000","last_fullsized":"2023-09-23T20:53:03.998985+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:11:12.175158+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":42,"reported_epoch":115,"state":"active+clean","last_fresh":"2023-09-23T20:53:03.998758+0000","last_change":"2023-09-23T20:52:12.869818+0000","last_active":"2023-09-23T20:53:03.998758+0000","last_peered":"2023-09-23T20:53:03.998758+0000","last_clean":"2023-09-23T20:53:03.998758+0000","last_became_active":"2023-09-23T20:52:12.859839+0000","last_became_peered":"2023-09-23T20:52:12.859839+0000","last_unstale":"2023-09-23T20:53:03.998758+0000","last_undegraded":"2023-09-23T20:53:03.998758+0000","last_fullsized":"2023-09-23T20:53:03.998758+0000","mapping_epoch":100,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":101,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:56:46.497439+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":92,"reported_epoch":115,"state":"active+clean","last_fresh":"2023-09-23T20:53:03.998501+0000","last_change":"2023-09-23T20:51:10.835094+0000","last_active":"2023-09-23T20:53:03.998501+0000","last_peered":"2023-09-23T20:53:03.998501+0000","last_clean":"2023-09-23T20:53:03.998501+0000","last_became_active":"2023-09-23T20:51:10.834732+0000","last_became_peered":"2023-09-23T20:51:10.834732+0000","last_unstale":"2023-09-23T20:53:03.998501+0000","last_undegraded":"2023-09-23T20:53:03.998501+0000","last_fullsized":"2023-09-23T20:53:03.998501+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:21:08.414737+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":85,"reported_epoch":116,"state":"active+clean","last_fresh":"2023-09-23T20:53:06.033782+0000","last_change":"2023-09-23T20:51:10.833849+0000","last_active":"2023-09-23T20:53:06.033782+0000","last_peered":"2023-09-23T20:53:06.033782+0000","last_clean":"2023-09-23T20:53:06.033782+0000","last_became_active":"2023-09-23T20:51:10.833524+0000","last_became_peered":"2023-09-23T20:51:10.833524+0000","last_unstale":"2023-09-23T20:53:06.033782+0000","last_undegraded":"2023-09-23T20:53:06.033782+0000","last_fullsized":"2023-09-23T20:53:06.033782+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:23:12.501924+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":99,"reported_epoch":115,"state":"active+clean","last_fresh":"2023-09-23T20:53:04.002345+0000","last_change":"2023-09-23T20:51:10.838334+0000","last_active":"2023-09-23T20:53:04.002345+0000","last_peered":"2023-09-23T20:53:04.002345+0000","last_clean":"2023-09-23T20:53:04.002345+0000","last_became_active":"2023-09-23T20:51:10.838026+0000","last_became_peered":"2023-09-23T20:51:10.838026+0000","last_unstale":"2023-09-23T20:53:04.002345+0000","last_undegraded":"2023-09-23T20:53:04.002345+0000","last_fullsized":"2023-09-23T20:53:04.002345+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:04:37.009457+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":87,"reported_epoch":114,"state":"active+clean","last_fresh":"2023-09-23T20:52:53.874871+0000","last_change":"2023-09-23T20:52:12.868379+0000","last_active":"2023-09-23T20:52:53.874871+0000","last_peered":"2023-09-23T20:52:53.874871+0000","last_clean":"2023-09-23T20:52:53.874871+0000","last_became_active":"2023-09-23T20:52:12.861899+0000","last_became_peered":"2023-09-23T20:52:12.861899+0000","last_unstale":"2023-09-23T20:52:53.874871+0000","last_undegraded":"2023-09-23T20:52:53.874871+0000","last_fullsized":"2023-09-23T20:52:53.874871+0000","mapping_epoch":100,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":101,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:17:39.308666+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":554,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,6],"acting":[0,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":85,"reported_epoch":115,"state":"active+clean","last_fresh":"2023-09-23T20:53:03.998032+0000","last_change":"2023-09-23T20:50:37.984570+0000","last_active":"2023-09-23T20:53:03.998032+0000","last_peered":"2023-09-23T20:53:03.998032+0000","last_clean":"2023-09-23T20:53:03.998032+0000","last_became_active":"2023-09-23T20:50:37.984262+0000","last_became_peered":"2023-09-23T20:50:37.984262+0000","last_unstale":"2023-09-23T20:53:03.998032+0000","last_undegraded":"2023-09-23T20:53:03.998032+0000","last_fullsized":"2023-09-23T20:53:03.998032+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:47:39.024252+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":6,"num_pg":16,"stat_sum":{"num_bytes":4635099136,"num_objects":70726,"num_object_clones":0,"num_object_copies":212178,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":70726,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":70726,"num_write_kb":4526464,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":13840809984,"data_stored":13840809984,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":70726,"ondisk_log_size":70726,"up":48,"acting":48,"num_store_stats":7},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":9,"num_bytes_recovered":624,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":106496,"data_stored":17136,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":161,"ondisk_log_size":161,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201432,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1997616,"kb_used_data":1950376,"kb_used_omap":0,"kb_used_meta":47232,"kb_avail":91747536,"statfs":{"total":95995035648,"available":93949476864,"internally_reserved":0,"allocated":1997185024,"data_stored":1996705184,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":48365568},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:52:36 2023","interfaces":[{"interface":"back","average":{"1min":0.92000000000000004,"5min":0.60299999999999998,"15min":0.52800000000000002},"min":{"1min":0.25800000000000001,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":3.5209999999999999,"5min":3.5209999999999999,"15min":3.5209999999999999},"last":5.8760000000000003},{"interface":"front","average":{"1min":0.71199999999999997,"5min":0.58999999999999997,"15min":0.57199999999999995},"min":{"1min":0.28899999999999998,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":3.1779999999999999,"5min":3.1779999999999999,"15min":3.1779999999999999},"last":5.8700000000000001}]},{"osd":1,"last update":"Sat Sep 23 20:52:05 2023","interfaces":[{"interface":"back","average":{"1min":0.54300000000000004,"5min":0.54700000000000004,"15min":0.54800000000000004},"min":{"1min":0.28399999999999997,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":0.69499999999999995,"5min":0.73099999999999998,"15min":0.73099999999999998},"last":1.4099999999999999},{"interface":"front","average":{"1min":0.57699999999999996,"5min":0.55000000000000004,"15min":0.54600000000000004},"min":{"1min":0.42499999999999999,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":0.72799999999999998,"5min":0.72799999999999998,"15min":0.72799999999999998},"last":1.804}]},{"osd":2,"last update":"Sat Sep 23 20:52:45 2023","interfaces":[{"interface":"back","average":{"1min":0.89400000000000002,"5min":0.61899999999999999,"15min":0.57299999999999995},"min":{"1min":0.24399999999999999,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":3.5459999999999998,"5min":3.5459999999999998,"15min":3.5459999999999998},"last":5.7249999999999996},{"interface":"front","average":{"1min":0.93200000000000005,"5min":0.65300000000000002,"15min":0.60699999999999998},"min":{"1min":0.34399999999999997,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":3.7050000000000001,"5min":3.7050000000000001,"15min":3.7050000000000001},"last":2.5289999999999999}]},{"osd":3,"last update":"Sat Sep 23 20:52:13 2023","interfaces":[{"interface":"back","average":{"1min":0.56000000000000005,"5min":0.56000000000000005,"15min":0.56000000000000005},"min":{"1min":0.38700000000000001,"5min":0.38700000000000001,"15min":0.38700000000000001},"max":{"1min":0.77200000000000002,"5min":0.77200000000000002,"15min":0.77200000000000002},"last":2.2519999999999998},{"interface":"front","average":{"1min":0.57399999999999995,"5min":0.57399999999999995,"15min":0.57399999999999995},"min":{"1min":0.41199999999999998,"5min":0.41199999999999998,"15min":0.41199999999999998},"max":{"1min":0.73099999999999998,"5min":0.73099999999999998,"15min":0.73099999999999998},"last":1.573}]},{"osd":4,"last update":"Sat Sep 23 20:53:04 2023","interfaces":[{"interface":"back","average":{"1min":0.50800000000000001,"5min":0.52900000000000003,"15min":0.49099999999999999},"min":{"1min":0.23699999999999999,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":1.3879999999999999,"5min":1.3879999999999999,"15min":1.3879999999999999},"last":1.3879999999999999},{"interface":"front","average":{"1min":0.69199999999999995,"5min":0.57699999999999996,"15min":0.53600000000000003},"min":{"1min":0.32100000000000001,"5min":0.32100000000000001,"15min":0.29599999999999999},"max":{"1min":3.2349999999999999,"5min":3.2349999999999999,"15min":3.2349999999999999},"last":1.121}]},{"osd":5,"last update":"Sat Sep 23 20:53:04 2023","interfaces":[{"interface":"back","average":{"1min":0.52700000000000002,"5min":0.56399999999999995,"15min":0.53800000000000003},"min":{"1min":0.29999999999999999,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":1.0620000000000001,"5min":1.0620000000000001,"15min":1.4610000000000001},"last":0.40600000000000003},{"interface":"front","average":{"1min":0.59099999999999997,"5min":0.60399999999999998,"15min":0.56100000000000005},"min":{"1min":0.29199999999999998,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":1.319,"5min":1.319,"15min":1.319},"last":1.319}]},{"osd":6,"last update":"Sat Sep 23 20:53:04 2023","interfaces":[{"interface":"back","average":{"1min":0.58799999999999997,"5min":0.62,"15min":0.57899999999999996},"min":{"1min":0.27400000000000002,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":1.363,"5min":1.363,"15min":1.363},"last":1.363},{"interface":"front","average":{"1min":0.61299999999999999,"5min":0.61599999999999999,"15min":0.59199999999999997},"min":{"1min":0.376,"5min":0.36299999999999999,"15min":0.316},"max":{"1min":1.4419999999999999,"5min":1.4419999999999999,"15min":1.4419999999999999},"last":1.151}]}]},{"osd":6,"up_from":47,"seq":201863463069,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1719140,"kb_used_data":1682588,"kb_used_omap":0,"kb_used_meta":36544,"kb_avail":92026012,"statfs":{"total":95995035648,"available":94234636288,"internally_reserved":0,"allocated":1722970112,"data_stored":1722502703,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":37421056},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:52:39 2023","interfaces":[{"interface":"back","average":{"1min":0.57799999999999996,"5min":0.56399999999999995,"15min":0.57399999999999995},"min":{"1min":0.29099999999999998,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":1.4810000000000001,"5min":1.4810000000000001,"15min":1.4810000000000001},"last":2.6739999999999999},{"interface":"front","average":{"1min":0.77300000000000002,"5min":0.61699999999999999,"15min":0.59899999999999998},"min":{"1min":0.29099999999999998,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":3.3069999999999999,"5min":3.3069999999999999,"15min":3.3069999999999999},"last":1.8640000000000001}]},{"osd":1,"last update":"Sat Sep 23 20:52:10 2023","interfaces":[{"interface":"back","average":{"1min":0.55000000000000004,"5min":0.53700000000000003,"15min":0.53500000000000003},"min":{"1min":0.39000000000000001,"5min":0.371,"15min":0.371},"max":{"1min":0.80600000000000005,"5min":0.80600000000000005,"15min":0.80600000000000005},"last":7.194},{"interface":"front","average":{"1min":0.56000000000000005,"5min":0.56799999999999995,"15min":0.56899999999999995},"min":{"1min":0.35399999999999998,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":0.91700000000000004,"5min":0.91700000000000004,"15min":0.91700000000000004},"last":1.0800000000000001}]},{"osd":2,"last update":"Sat Sep 23 20:52:44 2023","interfaces":[{"interface":"back","average":{"1min":0.94999999999999996,"5min":0.628,"15min":0.57399999999999995},"min":{"1min":0.44,"5min":0.40899999999999997,"15min":0.40899999999999997},"max":{"1min":3.1179999999999999,"5min":3.1179999999999999,"15min":3.1179999999999999},"last":2.1059999999999999},{"interface":"front","average":{"1min":1.026,"5min":0.67400000000000004,"15min":0.61499999999999999},"min":{"1min":0.39700000000000002,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":3.0960000000000001,"5min":3.0960000000000001,"15min":3.0960000000000001},"last":2.1360000000000001}]},{"osd":3,"last update":"Sat Sep 23 20:52:10 2023","interfaces":[{"interface":"back","average":{"1min":0.59199999999999997,"5min":0.59199999999999997,"15min":0.59199999999999997},"min":{"1min":0.38700000000000001,"5min":0.38700000000000001,"15min":0.38700000000000001},"max":{"1min":0.85799999999999998,"5min":0.85799999999999998,"15min":0.85799999999999998},"last":1.1839999999999999},{"interface":"front","average":{"1min":0.58099999999999996,"5min":0.58099999999999996,"15min":0.58099999999999996},"min":{"1min":0.40000000000000002,"5min":0.40000000000000002,"15min":0.40000000000000002},"max":{"1min":0.83399999999999996,"5min":0.83399999999999996,"15min":0.83399999999999996},"last":1.3149999999999999}]},{"osd":4,"last update":"Sat Sep 23 20:52:44 2023","interfaces":[{"interface":"back","average":{"1min":0.46500000000000002,"5min":0.47699999999999998,"15min":0.46000000000000002},"min":{"1min":0.248,"5min":0.248,"15min":0.248},"max":{"1min":0.71599999999999997,"5min":0.81000000000000005,"15min":1.1779999999999999},"last":0.28999999999999998},{"interface":"front","average":{"1min":0.46100000000000002,"5min":0.51600000000000001,"15min":0.49299999999999999},"min":{"1min":0.28899999999999998,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":0.748,"5min":0.81599999999999995,"15min":0.81599999999999995},"last":0.31}]},{"osd":5,"last update":"Sat Sep 23 20:52:44 2023","interfaces":[{"interface":"back","average":{"1min":0.53900000000000003,"5min":0.52200000000000002,"15min":0.498},"min":{"1min":0.32000000000000001,"5min":0.32000000000000001,"15min":0.28399999999999997},"max":{"1min":0.872,"5min":0.872,"15min":1.081},"last":0.34899999999999998},{"interface":"front","average":{"1min":0.52300000000000002,"5min":0.55600000000000005,"15min":0.52500000000000002},"min":{"1min":0.38500000000000001,"5min":0.33100000000000002,"15min":0.28499999999999998},"max":{"1min":0.72999999999999998,"5min":0.80800000000000005,"15min":0.89000000000000001},"last":0.40500000000000003}]},{"osd":7,"last update":"Sat Sep 23 20:53:04 2023","interfaces":[{"interface":"back","average":{"1min":0.48999999999999999,"5min":0.55600000000000005,"15min":0.54000000000000004},"min":{"1min":0.34799999999999998,"5min":0.33000000000000002,"15min":0.32500000000000001},"max":{"1min":0.94599999999999995,"5min":0.94599999999999995,"15min":1.119},"last":0.39500000000000002},{"interface":"front","average":{"1min":0.497,"5min":0.60099999999999998,"15min":0.58199999999999996},"min":{"1min":0.39000000000000001,"5min":0.38900000000000001,"15min":0.36099999999999999},"max":{"1min":0.84599999999999997,"5min":0.86899999999999999,"15min":0.94099999999999995},"last":0.437}]}]},{"osd":1,"up_from":88,"seq":377957122087,"num_pgs":42,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1760364,"kb_used_data":1714852,"kb_used_omap":9,"kb_used_meta":45494,"kb_avail":91984788,"statfs":{"total":95995035648,"available":94192422912,"internally_reserved":0,"allocated":1756008448,"data_stored":1755476114,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9239,"internal_metadata":46586857},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:53:07 2023","interfaces":[{"interface":"back","average":{"1min":0.98299999999999998,"5min":0.50900000000000001,"15min":0.442},"min":{"1min":0.216,"5min":0.216,"15min":0.216},"max":{"1min":5.0860000000000003,"5min":5.0860000000000003,"15min":5.0860000000000003},"last":0.496},{"interface":"front","average":{"1min":1.071,"5min":0.54900000000000004,"15min":0.47299999999999998},"min":{"1min":0.31900000000000001,"5min":0.31,"15min":0.31},"max":{"1min":4.9059999999999997,"5min":4.9059999999999997,"15min":4.9059999999999997},"last":0.53700000000000003}]},{"osd":2,"last update":"Sat Sep 23 20:52:39 2023","interfaces":[{"interface":"back","average":{"1min":1.2370000000000001,"5min":0.55700000000000005,"15min":0.443},"min":{"1min":0.26500000000000001,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":6.6379999999999999,"5min":6.6379999999999999,"15min":6.6379999999999999},"last":2.677},{"interface":"front","average":{"1min":0.78400000000000003,"5min":0.624,"15min":0.59799999999999998},"min":{"1min":0.26100000000000001,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":4.9370000000000003,"5min":4.9370000000000003,"15min":4.9370000000000003},"last":0.57399999999999995}]},{"osd":3,"last update":"Sat Sep 23 20:52:10 2023","interfaces":[{"interface":"back","average":{"1min":0.35899999999999999,"5min":0.35899999999999999,"15min":0.35899999999999999},"min":{"1min":0.27100000000000002,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":0.46999999999999997,"5min":0.46999999999999997,"15min":0.46999999999999997},"last":3.0270000000000001},{"interface":"front","average":{"1min":0.40999999999999998,"5min":0.40999999999999998,"15min":0.40999999999999998},"min":{"1min":0.34899999999999998,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":0.54300000000000004,"5min":0.54300000000000004,"15min":0.54300000000000004},"last":3.0600000000000001}]},{"osd":4,"last update":"Sat Sep 23 20:53:07 2023","interfaces":[{"interface":"back","average":{"1min":1.448,"5min":0.79300000000000004,"15min":0.69499999999999995},"min":{"1min":0.35199999999999998,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":5.7279999999999998,"5min":5.7279999999999998,"15min":5.7279999999999998},"last":0.51900000000000002},{"interface":"front","average":{"1min":0.86799999999999999,"5min":0.68999999999999995,"15min":0.67800000000000005},"min":{"1min":0.30099999999999999,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":4.4290000000000003,"5min":4.4290000000000003,"15min":4.4290000000000003},"last":0.68200000000000005}]},{"osd":5,"last update":"Sat Sep 23 20:53:07 2023","interfaces":[{"interface":"back","average":{"1min":0.91500000000000004,"5min":0.71099999999999997,"15min":0.70199999999999996},"min":{"1min":0.371,"5min":0.371,"15min":0.371},"max":{"1min":3.234,"5min":3.234,"15min":3.234},"last":0.64200000000000002},{"interface":"front","average":{"1min":1.4670000000000001,"5min":0.79600000000000004,"15min":0.69599999999999995},"min":{"1min":0.377,"5min":0.377,"15min":0.377},"max":{"1min":5.766,"5min":5.766,"15min":5.766},"last":0.56599999999999995}]},{"osd":6,"last update":"Sat Sep 23 20:53:07 2023","interfaces":[{"interface":"back","average":{"1min":0.91200000000000003,"5min":0.71399999999999997,"15min":0.69599999999999995},"min":{"1min":0.36899999999999999,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":3.278,"5min":3.278,"15min":3.278},"last":0.61099999999999999},{"interface":"front","average":{"1min":1.476,"5min":0.80600000000000005,"15min":0.70699999999999996},"min":{"1min":0.41199999999999998,"5min":0.41199999999999998,"15min":0.41199999999999998},"max":{"1min":5.7489999999999997,"5min":5.7489999999999997,"15min":5.7489999999999997},"last":0.59099999999999997}]},{"osd":7,"last update":"Sat Sep 23 20:53:07 2023","interfaces":[{"interface":"back","average":{"1min":1.1140000000000001,"5min":0.74199999999999999,"15min":0.69599999999999995},"min":{"1min":0.36599999999999999,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":4.46,"5min":4.46,"15min":4.46},"last":0.72899999999999998},{"interface":"front","average":{"1min":1.0189999999999999,"5min":0.73499999999999999,"15min":0.70199999999999996},"min":{"1min":0.38300000000000001,"5min":0.38300000000000001,"15min":0.38300000000000001},"max":{"1min":4.8700000000000001,"5min":4.8700000000000001,"15min":4.8700000000000001},"last":0.80200000000000005}]}]},{"osd":0,"up_from":83,"seq":356482285613,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1141944,"kb_used_data":1106224,"kb_used_omap":28,"kb_used_meta":35683,"kb_avail":92603208,"statfs":{"total":95995035648,"available":94825684992,"internally_reserved":0,"allocated":1132773376,"data_stored":1132223131,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":28848,"internal_metadata":36540240},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 20:52:08 2023","interfaces":[{"interface":"back","average":{"1min":0.36099999999999999,"5min":0.33800000000000002,"15min":0.33400000000000002},"min":{"1min":0.23899999999999999,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":0.60999999999999999,"5min":0.60999999999999999,"15min":0.60999999999999999},"last":0.70299999999999996},{"interface":"front","average":{"1min":0.378,"5min":0.38500000000000001,"15min":0.38600000000000001},"min":{"1min":0.25800000000000001,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":0.58999999999999997,"5min":0.58999999999999997,"15min":0.58999999999999997},"last":0.68899999999999995}]},{"osd":2,"last update":"Sat Sep 23 20:52:40 2023","interfaces":[{"interface":"back","average":{"1min":1.212,"5min":0.57399999999999995,"15min":0.46800000000000003},"min":{"1min":0.27500000000000002,"5min":0.248,"15min":0.248},"max":{"1min":8.5649999999999995,"5min":8.5649999999999995,"15min":8.5649999999999995},"last":5.9589999999999996},{"interface":"front","average":{"1min":0.90100000000000002,"5min":0.49199999999999999,"15min":0.42399999999999999},"min":{"1min":0.309,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":4.7050000000000001,"5min":4.7050000000000001,"15min":4.7050000000000001},"last":0.21099999999999999}]},{"osd":3,"last update":"Sat Sep 23 20:52:11 2023","interfaces":[{"interface":"back","average":{"1min":0.40100000000000002,"5min":0.40100000000000002,"15min":0.40100000000000002},"min":{"1min":0.29299999999999998,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":0.70099999999999996,"5min":0.70099999999999996,"15min":0.70099999999999996},"last":0.745},{"interface":"front","average":{"1min":0.47799999999999998,"5min":0.47799999999999998,"15min":0.47799999999999998},"min":{"1min":0.36099999999999999,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":0.98799999999999999,"5min":0.98799999999999999,"15min":0.98799999999999999},"last":0.72199999999999998}]},{"osd":4,"last update":"Sat Sep 23 20:52:40 2023","interfaces":[{"interface":"back","average":{"1min":0.87,"5min":0.69199999999999995,"15min":0.66200000000000003},"min":{"1min":0.33700000000000002,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":4.5250000000000004,"5min":4.5250000000000004,"15min":4.5250000000000004},"last":0.32000000000000001},{"interface":"front","average":{"1min":0.79200000000000004,"5min":0.69099999999999995,"15min":0.66900000000000004},"min":{"1min":0.40799999999999997,"5min":0.40799999999999997,"15min":0.40799999999999997},"max":{"1min":4.5890000000000004,"5min":4.5890000000000004,"15min":4.5890000000000004},"last":0.77600000000000002}]},{"osd":5,"last update":"Sat Sep 23 20:52:40 2023","interfaces":[{"interface":"back","average":{"1min":1.1879999999999999,"5min":0.747,"15min":0.68100000000000005},"min":{"1min":0.41499999999999998,"5min":0.41499999999999998,"15min":0.41499999999999998},"max":{"1min":8.4559999999999995,"5min":8.4559999999999995,"15min":8.4559999999999995},"last":0.65700000000000003},{"interface":"front","average":{"1min":0.86599999999999999,"5min":0.71899999999999997,"15min":0.71099999999999997},"min":{"1min":0.375,"5min":0.375,"15min":0.375},"max":{"1min":4.4740000000000002,"5min":4.4740000000000002,"15min":4.4740000000000002},"last":0.35899999999999999}]},{"osd":6,"last update":"Sat Sep 23 20:52:40 2023","interfaces":[{"interface":"back","average":{"1min":0.92300000000000004,"5min":0.71999999999999997,"15min":0.69999999999999996},"min":{"1min":0.45700000000000002,"5min":0.433,"15min":0.433},"max":{"1min":4.4400000000000004,"5min":4.4400000000000004,"15min":4.4400000000000004},"last":0.39900000000000002},{"interface":"front","average":{"1min":0.86799999999999999,"5min":0.72599999999999998,"15min":0.69599999999999995},"min":{"1min":0.41399999999999998,"5min":0.41399999999999998,"15min":0.41399999999999998},"max":{"1min":4.6200000000000001,"5min":4.6200000000000001,"15min":4.6200000000000001},"last":0.80500000000000005}]},{"osd":7,"last update":"Sat Sep 23 20:52:40 2023","interfaces":[{"interface":"back","average":{"1min":1.1870000000000001,"5min":0.78500000000000003,"15min":0.72799999999999998},"min":{"1min":0.45800000000000002,"5min":0.42799999999999999,"15min":0.42799999999999999},"max":{"1min":8.3979999999999997,"5min":8.3979999999999997,"15min":8.3979999999999997},"last":0.60899999999999999},{"interface":"front","average":{"1min":0.89100000000000001,"5min":0.70999999999999996,"15min":0.68600000000000005},"min":{"1min":0.41899999999999998,"5min":0.41899999999999998,"15min":0.41899999999999998},"max":{"1min":4.5599999999999996,"5min":4.5599999999999996,"15min":4.5599999999999996},"last":0.83399999999999996}]}]},{"osd":2,"up_from":93,"seq":399431958560,"num_pgs":29,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2830168,"kb_used_data":2750096,"kb_used_omap":81,"kb_used_meta":79982,"kb_avail":90914984,"statfs":{"total":95995035648,"available":93096943616,"internally_reserved":0,"allocated":2816098304,"data_stored":2815575290,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":83750,"internal_metadata":81901786},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:52:39 2023","interfaces":[{"interface":"back","average":{"1min":0.52800000000000002,"5min":0.39200000000000002,"15min":0.36899999999999999},"min":{"1min":0.20499999999999999,"5min":0.20499999999999999,"15min":0.20499999999999999},"max":{"1min":2.6760000000000002,"5min":2.6760000000000002,"15min":2.6760000000000002},"last":0.40899999999999997},{"interface":"front","average":{"1min":0.52800000000000002,"5min":0.43099999999999999,"15min":0.41499999999999998},"min":{"1min":0.26800000000000002,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":1.9830000000000001,"5min":1.9830000000000001,"15min":1.9830000000000001},"last":0.46100000000000002}]},{"osd":1,"last update":"Sat Sep 23 20:52:39 2023","interfaces":[{"interface":"back","average":{"1min":0.65600000000000003,"5min":0.46400000000000002,"15min":0.432},"min":{"1min":0.24199999999999999,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":4.367,"5min":4.367,"15min":4.367},"last":16.081},{"interface":"front","average":{"1min":0.70299999999999996,"5min":0.47699999999999998,"15min":0.439},"min":{"1min":0.25700000000000001,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":5.2469999999999999,"5min":5.2469999999999999,"15min":5.2469999999999999},"last":16.041}]},{"osd":3,"last update":"Sat Sep 23 20:52:12 2023","interfaces":[{"interface":"back","average":{"1min":0.41399999999999998,"5min":0.41399999999999998,"15min":0.41399999999999998},"min":{"1min":0.255,"5min":0.255,"15min":0.255},"max":{"1min":1.571,"5min":1.571,"15min":1.571},"last":0.38800000000000001},{"interface":"front","average":{"1min":0.38,"5min":0.38,"15min":0.38},"min":{"1min":0.221,"5min":0.221,"15min":0.221},"max":{"1min":1.02,"5min":1.02,"15min":1.02},"last":0.42899999999999999}]},{"osd":4,"last update":"Sat Sep 23 20:52:39 2023","interfaces":[{"interface":"back","average":{"1min":0.69899999999999995,"5min":0.63500000000000001,"15min":0.624},"min":{"1min":0.29299999999999998,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":1.9359999999999999,"5min":1.9359999999999999,"15min":1.9359999999999999},"last":0.53500000000000003},{"interface":"front","average":{"1min":0.81200000000000006,"5min":0.69899999999999995,"15min":0.68000000000000005},"min":{"1min":0.40400000000000003,"5min":0.40400000000000003,"15min":0.40400000000000003},"max":{"1min":4.274,"5min":4.274,"15min":4.274},"last":0.55000000000000004}]},{"osd":5,"last update":"Sat Sep 23 20:52:39 2023","interfaces":[{"interface":"back","average":{"1min":0.71299999999999997,"5min":0.66400000000000003,"15min":0.65600000000000003},"min":{"1min":0.34200000000000003,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":2.0139999999999998,"5min":2.0139999999999998,"15min":2.0139999999999998},"last":0.60599999999999998},{"interface":"front","average":{"1min":0.84399999999999997,"5min":0.67700000000000005,"15min":0.64900000000000002},"min":{"1min":0.36299999999999999,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":4.3070000000000004,"5min":4.3070000000000004,"15min":4.3070000000000004},"last":0.64900000000000002}]},{"osd":6,"last update":"Sat Sep 23 20:52:39 2023","interfaces":[{"interface":"back","average":{"1min":0.82699999999999996,"5min":0.72999999999999998,"15min":0.71399999999999997},"min":{"1min":0.38800000000000001,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":4.3380000000000001,"5min":4.3380000000000001,"15min":4.3380000000000001},"last":0.68600000000000005},{"interface":"front","average":{"1min":0.77900000000000003,"5min":0.70399999999999996,"15min":0.69199999999999995},"min":{"1min":0.41899999999999998,"5min":0.41899999999999998,"15min":0.41899999999999998},"max":{"1min":2.7410000000000001,"5min":2.7410000000000001,"15min":2.7410000000000001},"last":0.61599999999999999}]},{"osd":7,"last update":"Sat Sep 23 20:52:39 2023","interfaces":[{"interface":"back","average":{"1min":0.746,"5min":0.65700000000000003,"15min":0.64200000000000002},"min":{"1min":0.42299999999999999,"5min":0.42299999999999999,"15min":0.42299999999999999},"max":{"1min":2.7719999999999998,"5min":2.7719999999999998,"15min":2.7719999999999998},"last":0.66500000000000004},{"interface":"front","average":{"1min":0.75,"5min":0.67900000000000005,"15min":0.66700000000000004},"min":{"1min":0.36599999999999999,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":3.0270000000000001,"5min":3.0270000000000001,"15min":3.0270000000000001},"last":0.58099999999999996}]}]},{"osd":3,"up_from":98,"seq":420906795034,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2890772,"kb_used_data":2827212,"kb_used_omap":7,"kb_used_meta":63544,"kb_avail":90854380,"statfs":{"total":95995035648,"available":93034885120,"internally_reserved":0,"allocated":2895065088,"data_stored":2894551526,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7576,"internal_metadata":65069672},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:52:13 2023","interfaces":[{"interface":"back","average":{"1min":0.371,"5min":0.371,"15min":0.371},"min":{"1min":0.253,"5min":0.253,"15min":0.253},"max":{"1min":0.71199999999999997,"5min":0.71199999999999997,"15min":0.71199999999999997},"last":0.39800000000000002},{"interface":"front","average":{"1min":0.35799999999999998,"5min":0.35799999999999998,"15min":0.35799999999999998},"min":{"1min":0.24399999999999999,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":0.60499999999999998,"5min":0.60499999999999998,"15min":0.60499999999999998},"last":1.335}]},{"osd":1,"last update":"Sat Sep 23 20:52:13 2023","interfaces":[{"interface":"back","average":{"1min":0.36699999999999999,"5min":0.36699999999999999,"15min":0.36699999999999999},"min":{"1min":0.253,"5min":0.253,"15min":0.253},"max":{"1min":0.65500000000000003,"5min":0.65500000000000003,"15min":0.65500000000000003},"last":1.907},{"interface":"front","average":{"1min":0.38900000000000001,"5min":0.38900000000000001,"15min":0.38900000000000001},"min":{"1min":0.28999999999999998,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":0.63100000000000001,"5min":0.63100000000000001,"15min":0.63100000000000001},"last":0.28499999999999998}]},{"osd":2,"last update":"Sat Sep 23 20:52:13 2023","interfaces":[{"interface":"back","average":{"1min":0.42499999999999999,"5min":0.42499999999999999,"15min":0.42499999999999999},"min":{"1min":0.30199999999999999,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":0.70399999999999996,"5min":0.70399999999999996,"15min":0.70399999999999996},"last":1.3660000000000001},{"interface":"front","average":{"1min":0.439,"5min":0.439,"15min":0.439},"min":{"1min":0.30499999999999999,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":0.67700000000000005,"5min":0.67700000000000005,"15min":0.67700000000000005},"last":0.47199999999999998}]},{"osd":4,"last update":"Sat Sep 23 20:52:13 2023","interfaces":[{"interface":"back","average":{"1min":0.63200000000000001,"5min":0.63200000000000001,"15min":0.63200000000000001},"min":{"1min":0.42599999999999999,"5min":0.42599999999999999,"15min":0.42599999999999999},"max":{"1min":1.0960000000000001,"5min":1.0960000000000001,"15min":1.0960000000000001},"last":1.456},{"interface":"front","average":{"1min":0.627,"5min":0.627,"15min":0.627},"min":{"1min":0.437,"5min":0.437,"15min":0.437},"max":{"1min":0.80400000000000005,"5min":0.80400000000000005,"15min":0.80400000000000005},"last":0.32700000000000001}]},{"osd":5,"last update":"Sat Sep 23 20:52:13 2023","interfaces":[{"interface":"back","average":{"1min":0.64600000000000002,"5min":0.64600000000000002,"15min":0.64600000000000002},"min":{"1min":0.42599999999999999,"5min":0.42599999999999999,"15min":0.42599999999999999},"max":{"1min":1.1240000000000001,"5min":1.1240000000000001,"15min":1.1240000000000001},"last":1.409},{"interface":"front","average":{"1min":0.63700000000000001,"5min":0.63700000000000001,"15min":0.63700000000000001},"min":{"1min":0.46899999999999997,"5min":0.46899999999999997,"15min":0.46899999999999997},"max":{"1min":0.83599999999999997,"5min":0.83599999999999997,"15min":0.83599999999999997},"last":0.441}]},{"osd":6,"last update":"Sat Sep 23 20:52:13 2023","interfaces":[{"interface":"back","average":{"1min":0.69099999999999995,"5min":0.69099999999999995,"15min":0.69099999999999995},"min":{"1min":0.41599999999999998,"5min":0.41599999999999998,"15min":0.41599999999999998},"max":{"1min":0.86099999999999999,"5min":0.86099999999999999,"15min":0.86099999999999999},"last":0.38100000000000001},{"interface":"front","average":{"1min":0.71899999999999997,"5min":0.71899999999999997,"15min":0.71899999999999997},"min":{"1min":0.499,"5min":0.499,"15min":0.499},"max":{"1min":1.149,"5min":1.149,"15min":1.149},"last":1.4930000000000001}]},{"osd":7,"last update":"Sat Sep 23 20:52:13 2023","interfaces":[{"interface":"back","average":{"1min":0.69499999999999995,"5min":0.69499999999999995,"15min":0.69499999999999995},"min":{"1min":0.51700000000000002,"5min":0.51700000000000002,"15min":0.51700000000000002},"max":{"1min":1.177,"5min":1.177,"15min":1.177},"last":1.524},{"interface":"front","average":{"1min":0.69399999999999995,"5min":0.69399999999999995,"15min":0.69399999999999995},"min":{"1min":0.46200000000000002,"5min":0.46200000000000002,"15min":0.46200000000000002},"max":{"1min":0.82999999999999996,"5min":0.82999999999999996,"15min":0.82999999999999996},"last":0.41899999999999998}]}]},{"osd":4,"up_from":31,"seq":133143986341,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1423404,"kb_used_data":1390948,"kb_used_omap":0,"kb_used_meta":32448,"kb_avail":92321748,"statfs":{"total":95995035648,"available":94537469952,"internally_reserved":0,"allocated":1424330752,"data_stored":1423854717,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":33226752},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:52:39 2023","interfaces":[{"interface":"back","average":{"1min":0.56999999999999995,"5min":0.64300000000000002,"15min":0.63700000000000001},"min":{"1min":0.249,"5min":0.249,"15min":0.249},"max":{"1min":0.92500000000000004,"5min":4.4050000000000002,"15min":4.4050000000000002},"last":0.39300000000000002},{"interface":"front","average":{"1min":0.56899999999999995,"5min":0.81299999999999994,"15min":0.88100000000000001},"min":{"1min":0.314,"5min":0.314,"15min":0.314},"max":{"1min":0.97899999999999998,"5min":5.2409999999999997,"15min":5.2409999999999997},"last":0.38}]},{"osd":1,"last update":"Sat Sep 23 20:52:09 2023","interfaces":[{"interface":"back","average":{"1min":0.82499999999999996,"5min":0.61199999999999999,"15min":0.57599999999999996},"min":{"1min":0.40300000000000002,"5min":0.40300000000000002,"15min":0.40300000000000002},"max":{"1min":5.8680000000000003,"5min":5.8680000000000003,"15min":5.8680000000000003},"last":4.742},{"interface":"front","average":{"1min":0.65500000000000003,"5min":0.60999999999999999,"15min":0.60199999999999998},"min":{"1min":0.373,"5min":0.373,"15min":0.373},"max":{"1min":2.2290000000000001,"5min":2.2290000000000001,"15min":2.2290000000000001},"last":0.52700000000000002}]},{"osd":2,"last update":"Sat Sep 23 20:52:42 2023","interfaces":[{"interface":"back","average":{"1min":0.64500000000000002,"5min":0.67700000000000005,"15min":0.68300000000000005},"min":{"1min":0.29099999999999998,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":1.4710000000000001,"5min":2.5030000000000001,"15min":2.5030000000000001},"last":3.3780000000000001},{"interface":"front","average":{"1min":0.75600000000000001,"5min":0.77000000000000002,"15min":0.77200000000000002},"min":{"1min":0.39400000000000002,"5min":0.38200000000000001,"15min":0.38200000000000001},"max":{"1min":3.1520000000000001,"5min":4.3289999999999997,"15min":4.3289999999999997},"last":0.89800000000000002}]},{"osd":3,"last update":"Sat Sep 23 20:52:09 2023","interfaces":[{"interface":"back","average":{"1min":0.65200000000000002,"5min":0.65200000000000002,"15min":0.65200000000000002},"min":{"1min":0.39600000000000002,"5min":0.39600000000000002,"15min":0.39600000000000002},"max":{"1min":2.367,"5min":2.367,"15min":2.367},"last":2.968},{"interface":"front","average":{"1min":0.76300000000000001,"5min":0.76300000000000001,"15min":0.76300000000000001},"min":{"1min":0.41899999999999998,"5min":0.41899999999999998,"15min":0.41899999999999998},"max":{"1min":2.331,"5min":2.331,"15min":2.331},"last":1.091}]},{"osd":5,"last update":"Sat Sep 23 20:52:22 2023","interfaces":[{"interface":"back","average":{"1min":0.58399999999999996,"5min":0.52200000000000002,"15min":0.46500000000000002},"min":{"1min":0.35399999999999998,"5min":0.29099999999999998,"15min":0.251},"max":{"1min":0.83799999999999997,"5min":0.83799999999999997,"15min":0.83799999999999997},"last":0.32600000000000001},{"interface":"front","average":{"1min":0.52100000000000002,"5min":0.51700000000000002,"15min":0.46899999999999997},"min":{"1min":0.311,"5min":0.311,"15min":0.28799999999999998},"max":{"1min":0.74199999999999999,"5min":0.79500000000000004,"15min":0.90000000000000002},"last":0.438}]},{"osd":6,"last update":"Sat Sep 23 20:52:55 2023","interfaces":[{"interface":"back","average":{"1min":0.48999999999999999,"5min":0.51200000000000001,"15min":0.51100000000000001},"min":{"1min":0.317,"5min":0.30299999999999999,"15min":0.28100000000000003},"max":{"1min":0.71999999999999997,"5min":0.748,"15min":1.1319999999999999},"last":0.42099999999999999},{"interface":"front","average":{"1min":0.60399999999999998,"5min":0.59099999999999997,"15min":0.57199999999999995},"min":{"1min":0.29299999999999998,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":1.925,"5min":1.925,"15min":1.925},"last":0.49399999999999999}]},{"osd":7,"last update":"Sat Sep 23 20:52:09 2023","interfaces":[{"interface":"back","average":{"1min":0.69599999999999995,"5min":0.61699999999999999,"15min":0.57499999999999996},"min":{"1min":0.51100000000000001,"5min":0.35099999999999998,"15min":0.32500000000000001},"max":{"1min":0.88400000000000001,"5min":0.88400000000000001,"15min":0.88400000000000001},"last":0.45100000000000001},{"interface":"front","average":{"1min":0.61399999999999999,"5min":0.59099999999999997,"15min":0.56799999999999995},"min":{"1min":0.42599999999999999,"5min":0.33500000000000002,"15min":0.32700000000000001},"max":{"1min":0.82399999999999995,"5min":0.82399999999999995,"15min":0.86299999999999999},"last":0.48699999999999999}]}]},{"osd":5,"up_from":0,"seq":167503724706,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5358,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1139343360,"data_stored":1139343360,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1775042560,"data_stored":1775042560,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":2841575424,"data_stored":2841575424,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":2925133824,"data_stored":2925133824,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1427767296,"data_stored":1427767296,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1731330048,"data_stored":1731330048,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":2000617472,"data_stored":2000617472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T20:53:12.213 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T20:53:12.270 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:53:12.374 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:53:12.392 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:53:12.392 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:53:12.493 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:53:12.497 INFO:teuthology.orchestra.run.smithi026.stdout:{ 2023-09-23T20:53:12.497 INFO:teuthology.orchestra.run.smithi026.stdout: "size": 20, 2023-09-23T20:53:12.497 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 600, 2023-09-23T20:53:12.497 INFO:teuthology.orchestra.run.smithi026.stdout: "ops": [ 2023-09-23T20:53:12.498 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:53:12.498 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34797.0:5580 6.7 6:e7d0e972:::benchmark_data_smithi026_223868_object5579:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e106)", 2023-09-23T20:53:12.498 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T20:52:21.792643+0000", 2023-09-23T20:53:12.498 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 50.702532601999998, 2023-09-23T20:53:12.498 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.080597706000000005, 2023-09-23T20:53:12.498 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T20:53:12.498 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:53:12.498 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T20:53:12.498 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34797", 2023-09-23T20:53:12.499 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T20:53:12.499 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 5580 2023-09-23T20:53:12.499 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:53:12.499 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T20:53:12.499 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:53:12.499 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T20:53:12.499 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:21.792643+0000", 2023-09-23T20:53:12.499 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:53:12.499 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:53:12.499 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:53:12.499 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T20:53:12.500 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:21.792643+0000", 2023-09-23T20:53:12.500 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:53:12.500 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:53:12.500 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:53:12.500 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T20:53:12.500 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:21.792642+0000", 2023-09-23T20:53:12.500 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T20:53:12.500 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:53:12.500 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:53:12.500 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T20:53:12.500 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:21.792665+0000", 2023-09-23T20:53:12.501 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.3706999999999999e-05 2023-09-23T20:53:12.501 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:53:12.501 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:53:12.501 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T20:53:12.501 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:21.792677+0000", 2023-09-23T20:53:12.501 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1398e-05 2023-09-23T20:53:12.501 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:53:12.501 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:53:12.502 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T20:53:12.502 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:21.792684+0000", 2023-09-23T20:53:12.502 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.1359999999999999e-06 2023-09-23T20:53:12.502 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:53:12.502 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:53:12.502 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:53:12.502 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:21.792745+0000", 2023-09-23T20:53:12.502 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.1409999999999996e-05 2023-09-23T20:53:12.502 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:53:12.502 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:53:12.503 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T20:53:12.503 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:21.792884+0000", 2023-09-23T20:53:12.503 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00013860599999999999 2023-09-23T20:53:12.503 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:53:12.503 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:53:12.503 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 1,4", 2023-09-23T20:53:12.503 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:21.792995+0000", 2023-09-23T20:53:12.503 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00011145399999999999 2023-09-23T20:53:12.503 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:53:12.504 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:53:12.504 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T20:53:12.504 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:21.794983+0000", 2023-09-23T20:53:12.504 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0019872370000000002 2023-09-23T20:53:12.504 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:53:12.504 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:53:12.504 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:53:12.504 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:21.797238+0000", 2023-09-23T20:53:12.504 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0022552050000000001 2023-09-23T20:53:12.504 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:53:12.505 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:53:12.505 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:53:12.505 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:21.873142+0000", 2023-09-23T20:53:12.505 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.075904504999999997 2023-09-23T20:53:12.505 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:53:12.505 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:53:12.505 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T20:53:12.505 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:21.873197+0000", 2023-09-23T20:53:12.506 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.4382000000000002e-05 2023-09-23T20:53:12.506 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:53:12.506 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:53:12.506 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T20:53:12.506 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:21.873241+0000", 2023-09-23T20:53:12.506 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.4326999999999998e-05 2023-09-23T20:53:12.506 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:53:12.506 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T20:53:12.506 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:53:12.507 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:53:12.507 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:53:12.507 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34797.0:5584 6.7 6:edbb355b:::benchmark_data_smithi026_223868_object5583:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e106)", 2023-09-23T20:53:12.507 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T20:52:21.793537+0000", 2023-09-23T20:53:12.507 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 50.701639233999998, 2023-09-23T20:53:12.507 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.079811784999999996, 2023-09-23T20:53:12.507 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T20:53:12.507 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:53:12.507 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T20:53:12.507 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34797", 2023-09-23T20:53:12.507 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T20:53:12.508 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 5584 2023-09-23T20:53:12.508 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:53:12.508 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T20:53:12.508 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:53:12.508 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T20:53:12.508 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:21.793537+0000", 2023-09-23T20:53:12.508 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:53:12.508 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:53:12.508 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:53:12.508 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T20:53:12.508 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:21.793537+0000", 2023-09-23T20:53:12.509 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:53:12.509 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:53:12.509 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:53:12.509 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T20:53:12.509 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:21.793536+0000", 2023-09-23T20:53:12.509 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.999999 2023-09-23T20:53:12.509 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:53:12.509 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:53:12.509 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T20:53:12.509 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:21.793585+0000", 2023-09-23T20:53:12.510 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.9076000000000002e-05 2023-09-23T20:53:12.510 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:53:12.510 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:53:12.510 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T20:53:12.510 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:21.793597+0000", 2023-09-23T20:53:12.510 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.2043e-05 2023-09-23T20:53:12.510 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:53:12.510 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:53:12.510 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T20:53:12.510 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:21.793604+0000", 2023-09-23T20:53:12.511 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.537e-06 2023-09-23T20:53:12.511 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:53:12.511 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:53:12.511 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:53:12.511 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:21.793691+0000", 2023-09-23T20:53:12.511 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.6510999999999999e-05 2023-09-23T20:53:12.511 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:53:12.511 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:53:12.511 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T20:53:12.511 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:21.793820+0000", 2023-09-23T20:53:12.512 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00012899499999999999 2023-09-23T20:53:12.512 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:53:12.512 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:53:12.512 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 1,4", 2023-09-23T20:53:12.512 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:21.793910+0000", 2023-09-23T20:53:12.512 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.9752000000000004e-05 2023-09-23T20:53:12.512 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:53:12.512 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:53:12.512 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T20:53:12.512 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:21.795241+0000", 2023-09-23T20:53:12.512 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0013311410000000001 2023-09-23T20:53:12.513 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:53:12.513 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:53:12.513 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:53:12.513 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:21.797443+0000", 2023-09-23T20:53:12.513 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0022027420000000002 2023-09-23T20:53:12.513 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:53:12.513 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:53:12.513 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:53:12.513 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:21.873281+0000", 2023-09-23T20:53:12.513 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.075837837000000005 2023-09-23T20:53:12.514 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:53:12.514 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:53:12.514 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T20:53:12.514 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:21.873318+0000", 2023-09-23T20:53:12.514 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.7147000000000002e-05 2023-09-23T20:53:12.514 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:53:12.514 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:53:12.514 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T20:53:12.514 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:21.873348+0000", 2023-09-23T20:53:12.514 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.0094e-05 2023-09-23T20:53:12.515 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:53:12.515 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T20:53:12.515 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:53:12.515 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:53:12.515 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:53:12.515 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34797.0:5585 6.7 6:efdab73e:::benchmark_data_smithi026_223868_object5584:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e106)", 2023-09-23T20:53:12.515 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T20:52:21.793620+0000", 2023-09-23T20:53:12.516 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 50.701555939999999, 2023-09-23T20:53:12.516 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.079867258999999996, 2023-09-23T20:53:12.516 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T20:53:12.516 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:53:12.516 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T20:53:12.516 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34797", 2023-09-23T20:53:12.517 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T20:53:12.517 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 5585 2023-09-23T20:53:12.517 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:53:12.518 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T20:53:12.518 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:53:12.518 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T20:53:12.518 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:21.793620+0000", 2023-09-23T20:53:12.518 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:53:12.518 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:53:12.518 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:53:12.518 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T20:53:12.518 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:21.793620+0000", 2023-09-23T20:53:12.518 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:53:12.519 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:53:12.519 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:53:12.519 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T20:53:12.519 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:21.793619+0000", 2023-09-23T20:53:12.519 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T20:53:12.519 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:53:12.519 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:53:12.519 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T20:53:12.519 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:21.793641+0000", 2023-09-23T20:53:12.519 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.1722999999999999e-05 2023-09-23T20:53:12.519 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:53:12.520 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:53:12.520 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T20:53:12.520 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:21.793650+0000", 2023-09-23T20:53:12.520 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.3610000000000001e-06 2023-09-23T20:53:12.520 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:53:12.520 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:53:12.520 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T20:53:12.520 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:21.793655+0000", 2023-09-23T20:53:12.520 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.4219999999999999e-06 2023-09-23T20:53:12.520 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:53:12.520 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:53:12.521 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:53:12.521 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:21.794305+0000", 2023-09-23T20:53:12.521 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00065043100000000003 2023-09-23T20:53:12.521 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:53:12.521 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:53:12.521 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T20:53:12.521 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:21.794440+0000", 2023-09-23T20:53:12.521 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000134952 2023-09-23T20:53:12.521 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:53:12.521 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:53:12.522 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 1,4", 2023-09-23T20:53:12.522 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:21.794595+0000", 2023-09-23T20:53:12.522 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00015463299999999999 2023-09-23T20:53:12.522 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:53:12.522 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:53:12.522 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T20:53:12.522 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:21.795247+0000", 2023-09-23T20:53:12.522 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00065243899999999999 2023-09-23T20:53:12.522 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:53:12.522 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:53:12.523 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:53:12.523 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:21.797689+0000", 2023-09-23T20:53:12.523 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0024413619999999999 2023-09-23T20:53:12.523 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:53:12.523 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:53:12.523 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:53:12.523 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:21.873388+0000", 2023-09-23T20:53:12.523 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.075699747999999997 2023-09-23T20:53:12.523 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:53:12.523 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:53:12.524 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T20:53:12.524 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:21.873434+0000", 2023-09-23T20:53:12.524 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.5892e-05 2023-09-23T20:53:12.524 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:53:12.524 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:53:12.524 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T20:53:12.524 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:21.873487+0000", 2023-09-23T20:53:12.524 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.2849999999999997e-05 2023-09-23T20:53:12.524 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:53:12.524 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T20:53:12.525 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:53:12.525 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:53:12.525 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:53:12.525 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34797.0:72068 6.7 6:e384b031:::benchmark_data_smithi026_223868_object72067:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e116)", 2023-09-23T20:53:12.525 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T20:53:09.059198+0000", 2023-09-23T20:53:12.525 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 3.4359783629999998, 2023-09-23T20:53:12.525 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.059698566000000002, 2023-09-23T20:53:12.525 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T20:53:12.525 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:53:12.525 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T20:53:12.526 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34797", 2023-09-23T20:53:12.526 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T20:53:12.526 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 72068 2023-09-23T20:53:12.526 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:53:12.526 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T20:53:12.526 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:53:12.526 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T20:53:12.526 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:09.059198+0000", 2023-09-23T20:53:12.526 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:53:12.526 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:53:12.527 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:53:12.527 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T20:53:12.527 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:09.059198+0000", 2023-09-23T20:53:12.527 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:53:12.527 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:53:12.527 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:53:12.527 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T20:53:12.527 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:09.059197+0000", 2023-09-23T20:53:12.527 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T20:53:12.527 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:53:12.527 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:53:12.528 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T20:53:12.528 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:09.059224+0000", 2023-09-23T20:53:12.528 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.6772e-05 2023-09-23T20:53:12.528 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:53:12.528 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:53:12.528 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T20:53:12.528 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:09.059233+0000", 2023-09-23T20:53:12.528 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.8210000000000008e-06 2023-09-23T20:53:12.528 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:53:12.528 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:53:12.528 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T20:53:12.529 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:09.059247+0000", 2023-09-23T20:53:12.529 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.4450000000000001e-05 2023-09-23T20:53:12.529 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:53:12.529 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:53:12.529 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:53:12.529 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:09.064419+0000", 2023-09-23T20:53:12.529 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0051717400000000002 2023-09-23T20:53:12.529 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:53:12.529 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:53:12.529 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T20:53:12.530 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:09.064561+0000", 2023-09-23T20:53:12.530 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00014188699999999999 2023-09-23T20:53:12.530 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:53:12.530 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:53:12.530 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 1,4", 2023-09-23T20:53:12.530 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:09.064712+0000", 2023-09-23T20:53:12.530 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000151769 2023-09-23T20:53:12.530 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:53:12.530 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:53:12.530 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:53:12.531 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:09.070703+0000", 2023-09-23T20:53:12.531 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0059908770000000004 2023-09-23T20:53:12.531 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:53:12.531 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:53:12.531 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T20:53:12.531 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:09.118748+0000", 2023-09-23T20:53:12.531 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.048044894999999997 2023-09-23T20:53:12.531 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:53:12.531 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:53:12.531 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:53:12.532 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:09.118795+0000", 2023-09-23T20:53:12.532 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.6381000000000002e-05 2023-09-23T20:53:12.532 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:53:12.532 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:53:12.532 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T20:53:12.532 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:09.118846+0000", 2023-09-23T20:53:12.532 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.1675000000000002e-05 2023-09-23T20:53:12.532 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:53:12.532 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:53:12.532 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T20:53:12.533 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:09.118896+0000", 2023-09-23T20:53:12.533 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.9826e-05 2023-09-23T20:53:12.533 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:53:12.533 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T20:53:12.533 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:53:12.533 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:53:12.533 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:53:12.533 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34797.0:72077 6.6 6:60edbd29:::benchmark_data_smithi026_223868_object72076:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e116)", 2023-09-23T20:53:12.533 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T20:53:09.070488+0000", 2023-09-23T20:53:12.533 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 3.4246876959999999, 2023-09-23T20:53:12.533 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.115914822, 2023-09-23T20:53:12.534 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T20:53:12.534 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:53:12.534 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T20:53:12.534 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34797", 2023-09-23T20:53:12.534 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T20:53:12.534 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 72077 2023-09-23T20:53:12.534 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:53:12.534 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T20:53:12.534 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:53:12.534 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T20:53:12.535 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:09.070488+0000", 2023-09-23T20:53:12.535 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:53:12.535 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:53:12.535 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:53:12.535 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T20:53:12.535 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:09.070488+0000", 2023-09-23T20:53:12.535 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:53:12.535 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:53:12.535 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:53:12.535 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T20:53:12.535 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:09.070487+0000", 2023-09-23T20:53:12.536 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T20:53:12.536 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:53:12.536 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:53:12.536 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T20:53:12.536 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:09.070514+0000", 2023-09-23T20:53:12.536 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.6794000000000001e-05 2023-09-23T20:53:12.536 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:53:12.536 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:53:12.536 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T20:53:12.536 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:09.070523+0000", 2023-09-23T20:53:12.537 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.7079999999999992e-06 2023-09-23T20:53:12.537 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:53:12.537 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:53:12.537 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T20:53:12.537 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:09.070529+0000", 2023-09-23T20:53:12.537 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.5520000000000004e-06 2023-09-23T20:53:12.537 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:53:12.537 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:53:12.537 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:53:12.537 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:09.070574+0000", 2023-09-23T20:53:12.537 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.4854000000000001e-05 2023-09-23T20:53:12.538 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:53:12.538 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:53:12.538 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T20:53:12.538 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:09.070710+0000", 2023-09-23T20:53:12.538 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000136737 2023-09-23T20:53:12.538 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:53:12.538 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:53:12.538 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 4,7", 2023-09-23T20:53:12.538 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:09.118125+0000", 2023-09-23T20:53:12.538 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.047414945999999999 2023-09-23T20:53:12.539 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:53:12.539 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:53:12.539 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:53:12.539 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:09.126216+0000", 2023-09-23T20:53:12.539 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0080901600000000008 2023-09-23T20:53:12.539 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:53:12.539 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:53:12.539 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:53:12.539 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:09.126250+0000", 2023-09-23T20:53:12.539 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.4867999999999999e-05 2023-09-23T20:53:12.540 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:53:12.540 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:53:12.540 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T20:53:12.540 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:09.186315+0000", 2023-09-23T20:53:12.540 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.060064192000000002 2023-09-23T20:53:12.540 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:53:12.540 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:53:12.540 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T20:53:12.540 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:09.186362+0000", 2023-09-23T20:53:12.540 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.7126000000000002e-05 2023-09-23T20:53:12.540 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:53:12.541 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:53:12.541 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T20:53:12.541 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:09.186403+0000", 2023-09-23T20:53:12.541 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.1349e-05 2023-09-23T20:53:12.541 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:53:12.541 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T20:53:12.541 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:53:12.541 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:53:12.541 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:53:12.542 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34797.0:72080 6.6 6:6a273beb:::benchmark_data_smithi026_223868_object72079:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e116)", 2023-09-23T20:53:12.542 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T20:53:09.070544+0000", 2023-09-23T20:53:12.542 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 3.4246314930000001, 2023-09-23T20:53:12.542 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.115949072, 2023-09-23T20:53:12.542 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T20:53:12.542 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:53:12.542 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T20:53:12.542 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34797", 2023-09-23T20:53:12.542 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T20:53:12.543 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 72080 2023-09-23T20:53:12.543 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:53:12.543 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T20:53:12.543 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:53:12.543 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T20:53:12.543 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:09.070544+0000", 2023-09-23T20:53:12.543 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:53:12.543 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:53:12.543 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:53:12.543 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T20:53:12.543 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:09.070544+0000", 2023-09-23T20:53:12.544 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:53:12.544 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:53:12.544 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:53:12.544 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T20:53:12.544 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:09.070544+0000", 2023-09-23T20:53:12.544 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T20:53:12.544 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:53:12.544 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:53:12.544 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T20:53:12.544 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:09.070565+0000", 2023-09-23T20:53:12.545 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.1067e-05 2023-09-23T20:53:12.545 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:53:12.545 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:53:12.545 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T20:53:12.545 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:09.070572+0000", 2023-09-23T20:53:12.545 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.2699999999999999e-06 2023-09-23T20:53:12.545 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:53:12.545 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:53:12.545 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T20:53:12.545 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:09.070578+0000", 2023-09-23T20:53:12.545 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.3959999999999997e-06 2023-09-23T20:53:12.546 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:53:12.546 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:53:12.546 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:53:12.546 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:09.120762+0000", 2023-09-23T20:53:12.546 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.050184281999999997 2023-09-23T20:53:12.546 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:53:12.546 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:53:12.546 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T20:53:12.546 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:09.120909+0000", 2023-09-23T20:53:12.546 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000147584 2023-09-23T20:53:12.547 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:53:12.547 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:53:12.547 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 4,7", 2023-09-23T20:53:12.547 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:09.121064+0000", 2023-09-23T20:53:12.547 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000154819 2023-09-23T20:53:12.547 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:53:12.547 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:53:12.547 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:53:12.547 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:09.126281+0000", 2023-09-23T20:53:12.547 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0052170630000000001 2023-09-23T20:53:12.547 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:53:12.548 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:53:12.548 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:53:12.548 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:09.129603+0000", 2023-09-23T20:53:12.548 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0033220210000000001 2023-09-23T20:53:12.548 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:53:12.548 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:53:12.548 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T20:53:12.548 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:09.186411+0000", 2023-09-23T20:53:12.548 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.056807189000000001 2023-09-23T20:53:12.548 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:53:12.549 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:53:12.549 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T20:53:12.549 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:09.186449+0000", 2023-09-23T20:53:12.549 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.8776000000000003e-05 2023-09-23T20:53:12.549 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:53:12.549 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:53:12.549 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T20:53:12.549 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:09.186493+0000", 2023-09-23T20:53:12.549 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.4141999999999999e-05 2023-09-23T20:53:12.549 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:53:12.549 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T20:53:12.550 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:53:12.550 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:53:12.550 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:53:12.550 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_repop(client.34797.0:72081 6.8 e116/102 6:17dd4223:::benchmark_data_smithi026_223868_object72080:head v 116'4526, mlcod=116'4524)", 2023-09-23T20:53:12.550 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T20:53:09.071108+0000", 2023-09-23T20:53:12.550 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 3.4240676199999998, 2023-09-23T20:53:12.550 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.114938104, 2023-09-23T20:53:12.550 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T20:53:12.550 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:53:12.550 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T20:53:12.550 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:53:12.551 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T20:53:12.551 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:09.071108+0000", 2023-09-23T20:53:12.551 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:53:12.551 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:53:12.551 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:53:12.551 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T20:53:12.551 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:09.071108+0000", 2023-09-23T20:53:12.551 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:53:12.551 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:53:12.551 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:53:12.552 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T20:53:12.552 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:09.071108+0000", 2023-09-23T20:53:12.552 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.999999 2023-09-23T20:53:12.552 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:53:12.552 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:53:12.552 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T20:53:12.552 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:09.071345+0000", 2023-09-23T20:53:12.552 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00023794899999999999 2023-09-23T20:53:12.552 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:53:12.552 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:53:12.552 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T20:53:12.553 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:09.118625+0000", 2023-09-23T20:53:12.553 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.047279130000000003 2023-09-23T20:53:12.553 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:53:12.553 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:53:12.553 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T20:53:12.553 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:09.118635+0000", 2023-09-23T20:53:12.553 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0416e-05 2023-09-23T20:53:12.553 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:53:12.553 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:53:12.553 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:53:12.554 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:09.120588+0000", 2023-09-23T20:53:12.554 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.001952542 2023-09-23T20:53:12.554 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:53:12.554 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:53:12.554 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T20:53:12.554 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:09.120606+0000", 2023-09-23T20:53:12.554 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.8162e-05 2023-09-23T20:53:12.554 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:53:12.554 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:53:12.554 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T20:53:12.555 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:09.186018+0000", 2023-09-23T20:53:12.555 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.065411838999999999 2023-09-23T20:53:12.555 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:53:12.555 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:53:12.555 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T20:53:12.555 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:09.186046+0000", 2023-09-23T20:53:12.555 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.8856e-05 2023-09-23T20:53:12.555 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:53:12.555 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T20:53:12.555 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:53:12.556 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:53:12.556 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:53:12.556 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_repop(client.34797.0:72078 6.0 e116/102 6:027b02cd:::benchmark_data_smithi026_223868_object72077:head v 116'4560, mlcod=116'4558)", 2023-09-23T20:53:12.556 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T20:53:09.078632+0000", 2023-09-23T20:53:12.556 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 3.4165438259999998, 2023-09-23T20:53:12.556 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.107381485, 2023-09-23T20:53:12.556 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T20:53:12.556 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:53:12.556 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T20:53:12.556 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:53:12.557 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T20:53:12.557 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:09.078632+0000", 2023-09-23T20:53:12.557 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:53:12.557 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:53:12.557 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:53:12.557 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T20:53:12.557 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:09.078632+0000", 2023-09-23T20:53:12.557 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:53:12.557 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:53:12.557 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:53:12.557 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T20:53:12.558 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:09.078631+0000", 2023-09-23T20:53:12.558 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T20:53:12.558 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:53:12.558 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:53:12.558 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T20:53:12.558 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:09.078662+0000", 2023-09-23T20:53:12.558 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.1257000000000002e-05 2023-09-23T20:53:12.558 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:53:12.558 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:53:12.559 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T20:53:12.559 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:09.117807+0000", 2023-09-23T20:53:12.559 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.039145043999999997 2023-09-23T20:53:12.559 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:53:12.559 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:53:12.559 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T20:53:12.559 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:09.117821+0000", 2023-09-23T20:53:12.559 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.4280999999999999e-05 2023-09-23T20:53:12.559 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:53:12.559 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:53:12.559 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:53:12.560 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:09.118062+0000", 2023-09-23T20:53:12.560 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00024037 2023-09-23T20:53:12.560 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:53:12.560 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:53:12.560 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T20:53:12.560 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:09.118087+0000", 2023-09-23T20:53:12.560 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.5264000000000001e-05 2023-09-23T20:53:12.560 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:53:12.560 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:53:12.560 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T20:53:12.560 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:09.185958+0000", 2023-09-23T20:53:12.561 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.067871048000000003 2023-09-23T20:53:12.561 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:53:12.561 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:53:12.561 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T20:53:12.561 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:09.186014+0000", 2023-09-23T20:53:12.561 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.5494000000000001e-05 2023-09-23T20:53:12.561 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:53:12.561 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T20:53:12.561 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:53:12.561 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:53:12.561 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:53:12.562 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_repop(client.34797.0:72083 6.0 e116/102 6:0bf8574b:::benchmark_data_smithi026_223868_object72082:head v 116'4561, mlcod=116'4558)", 2023-09-23T20:53:12.562 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T20:53:09.117845+0000", 2023-09-23T20:53:12.562 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 3.377331265, 2023-09-23T20:53:12.562 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.068226247000000004, 2023-09-23T20:53:12.562 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T20:53:12.562 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:53:12.562 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T20:53:12.562 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:53:12.562 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T20:53:12.562 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:09.117845+0000", 2023-09-23T20:53:12.563 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:53:12.563 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:53:12.563 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:53:12.563 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T20:53:12.563 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:09.117845+0000", 2023-09-23T20:53:12.563 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:53:12.563 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:53:12.563 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:53:12.563 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T20:53:12.563 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:09.117844+0000", 2023-09-23T20:53:12.564 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.999999 2023-09-23T20:53:12.564 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:53:12.564 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:53:12.564 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T20:53:12.564 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:09.117899+0000", 2023-09-23T20:53:12.564 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.4904000000000002e-05 2023-09-23T20:53:12.564 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:53:12.564 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:53:12.564 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T20:53:12.564 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:09.117910+0000", 2023-09-23T20:53:12.564 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1473e-05 2023-09-23T20:53:12.565 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:53:12.565 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:53:12.565 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T20:53:12.565 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:09.117916+0000", 2023-09-23T20:53:12.565 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.3879999999999997e-06 2023-09-23T20:53:12.565 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:53:12.565 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:53:12.565 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:53:12.565 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:09.120662+0000", 2023-09-23T20:53:12.565 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0027460760000000001 2023-09-23T20:53:12.565 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:53:12.566 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:53:12.566 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T20:53:12.566 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:09.120684+0000", 2023-09-23T20:53:12.566 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.179e-05 2023-09-23T20:53:12.566 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:53:12.566 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:53:12.566 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T20:53:12.566 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:09.186050+0000", 2023-09-23T20:53:12.566 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.065365456000000002 2023-09-23T20:53:12.566 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:53:12.567 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:53:12.567 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T20:53:12.567 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:09.186071+0000", 2023-09-23T20:53:12.567 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.1169e-05 2023-09-23T20:53:12.567 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:53:12.567 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T20:53:12.567 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:53:12.567 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:53:12.567 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:53:12.567 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_repop(client.34797.0:72087 6.4 e116/102 6:28e23fe0:::benchmark_data_smithi026_223868_object72086:head v 116'4532, mlcod=116'4530)", 2023-09-23T20:53:12.568 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T20:53:09.118038+0000", 2023-09-23T20:53:12.568 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 3.3771383770000001, 2023-09-23T20:53:12.568 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.068015682999999993, 2023-09-23T20:53:12.568 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T20:53:12.568 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:53:12.568 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T20:53:12.568 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:53:12.568 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T20:53:12.568 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:09.118038+0000", 2023-09-23T20:53:12.568 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:53:12.568 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:53:12.569 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:53:12.569 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T20:53:12.569 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:09.118038+0000", 2023-09-23T20:53:12.569 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:53:12.569 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:53:12.569 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:53:12.569 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T20:53:12.569 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:09.118037+0000", 2023-09-23T20:53:12.569 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T20:53:12.569 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:53:12.569 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:53:12.570 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T20:53:12.570 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:09.118063+0000", 2023-09-23T20:53:12.570 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.5735999999999999e-05 2023-09-23T20:53:12.570 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:53:12.570 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:53:12.570 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T20:53:12.570 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:09.118069+0000", 2023-09-23T20:53:12.570 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.4389999999999997e-06 2023-09-23T20:53:12.570 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:53:12.571 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:53:12.571 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T20:53:12.571 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:09.118074+0000", 2023-09-23T20:53:12.571 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.6949999999999999e-06 2023-09-23T20:53:12.571 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:53:12.571 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:53:12.571 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:53:12.571 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:09.118808+0000", 2023-09-23T20:53:12.571 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000733941 2023-09-23T20:53:12.571 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:53:12.571 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:53:12.572 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T20:53:12.572 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:09.118827+0000", 2023-09-23T20:53:12.572 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.8947999999999999e-05 2023-09-23T20:53:12.572 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:53:12.572 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:53:12.572 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T20:53:12.572 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:09.132656+0000", 2023-09-23T20:53:12.572 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.013828656999999999 2023-09-23T20:53:12.572 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:53:12.572 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:53:12.572 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T20:53:12.573 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:09.186053+0000", 2023-09-23T20:53:12.573 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.053397562000000003 2023-09-23T20:53:12.573 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:53:12.573 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T20:53:12.573 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:53:12.573 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:53:12.573 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:53:12.573 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34797.0:72085 6.b 6:d92d7132:::benchmark_data_smithi026_223868_object72084:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e116)", 2023-09-23T20:53:12.573 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T20:53:09.118137+0000", 2023-09-23T20:53:12.573 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 3.3770393730000001, 2023-09-23T20:53:12.574 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.068040587999999999, 2023-09-23T20:53:12.574 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T20:53:12.574 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:53:12.574 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T20:53:12.574 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34797", 2023-09-23T20:53:12.574 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T20:53:12.574 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 72085 2023-09-23T20:53:12.574 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:53:12.574 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T20:53:12.574 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:53:12.575 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T20:53:12.575 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:09.118137+0000", 2023-09-23T20:53:12.575 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:53:12.575 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:53:12.575 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:53:12.575 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T20:53:12.575 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:09.118137+0000", 2023-09-23T20:53:12.575 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:53:12.575 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:53:12.575 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:53:12.575 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T20:53:12.576 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:09.118135+0000", 2023-09-23T20:53:12.576 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T20:53:12.576 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:53:12.576 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:53:12.576 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T20:53:12.576 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:09.118161+0000", 2023-09-23T20:53:12.576 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.6271e-05 2023-09-23T20:53:12.576 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:53:12.576 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:53:12.576 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T20:53:12.577 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:09.118172+0000", 2023-09-23T20:53:12.577 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0322e-05 2023-09-23T20:53:12.577 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:53:12.577 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:53:12.577 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T20:53:12.577 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:09.118179+0000", 2023-09-23T20:53:12.577 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.8009999999999998e-06 2023-09-23T20:53:12.577 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:53:12.577 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:53:12.577 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:53:12.578 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:09.118536+0000", 2023-09-23T20:53:12.578 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00035700899999999998 2023-09-23T20:53:12.578 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:53:12.578 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:53:12.578 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T20:53:12.578 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:09.118687+0000", 2023-09-23T20:53:12.578 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00015179700000000001 2023-09-23T20:53:12.578 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:53:12.578 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:53:12.578 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 1,7", 2023-09-23T20:53:12.579 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:09.118823+0000", 2023-09-23T20:53:12.579 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00013593000000000001 2023-09-23T20:53:12.579 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:53:12.579 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:53:12.579 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:53:12.579 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:09.124261+0000", 2023-09-23T20:53:12.579 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0054375589999999998 2023-09-23T20:53:12.579 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:53:12.579 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:53:12.579 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:53:12.579 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:09.129714+0000", 2023-09-23T20:53:12.580 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0054535149999999999 2023-09-23T20:53:12.580 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:53:12.580 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:53:12.580 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T20:53:12.580 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:09.186083+0000", 2023-09-23T20:53:12.580 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.056368754 2023-09-23T20:53:12.580 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:53:12.580 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:53:12.580 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T20:53:12.580 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:09.186133+0000", 2023-09-23T20:53:12.580 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.9595999999999997e-05 2023-09-23T20:53:12.581 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:53:12.581 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:53:12.581 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T20:53:12.581 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:09.186177+0000", 2023-09-23T20:53:12.581 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.4372999999999997e-05 2023-09-23T20:53:12.581 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:53:12.581 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T20:53:12.581 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:53:12.581 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:53:12.581 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:53:12.582 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34797.0:72086 6.6 6:6ef49911:::benchmark_data_smithi026_223868_object72085:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e116)", 2023-09-23T20:53:12.582 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T20:53:09.118186+0000", 2023-09-23T20:53:12.582 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 3.3769901139999998, 2023-09-23T20:53:12.582 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.068389641000000001, 2023-09-23T20:53:12.582 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T20:53:12.582 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:53:12.582 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T20:53:12.582 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34797", 2023-09-23T20:53:12.582 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T20:53:12.582 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 72086 2023-09-23T20:53:12.583 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:53:12.583 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T20:53:12.583 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:53:12.583 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T20:53:12.583 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:09.118186+0000", 2023-09-23T20:53:12.583 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:53:12.583 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:53:12.583 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:53:12.583 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T20:53:12.583 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:09.118186+0000", 2023-09-23T20:53:12.583 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:53:12.584 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:53:12.584 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:53:12.584 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T20:53:12.584 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:09.118185+0000", 2023-09-23T20:53:12.584 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T20:53:12.584 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:53:12.584 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:53:12.584 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T20:53:12.584 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:09.118216+0000", 2023-09-23T20:53:12.584 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.0704999999999998e-05 2023-09-23T20:53:12.585 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:53:12.585 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:53:12.585 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T20:53:12.585 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:09.118223+0000", 2023-09-23T20:53:12.585 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.4510000000000003e-06 2023-09-23T20:53:12.585 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:53:12.585 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:53:12.585 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T20:53:12.585 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:09.118229+0000", 2023-09-23T20:53:12.586 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.524e-06 2023-09-23T20:53:12.586 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:53:12.586 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:53:12.586 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:53:12.586 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:09.123448+0000", 2023-09-23T20:53:12.586 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0052193589999999998 2023-09-23T20:53:12.586 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:53:12.586 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:53:12.586 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T20:53:12.586 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:09.123588+0000", 2023-09-23T20:53:12.586 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000139713 2023-09-23T20:53:12.587 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:53:12.587 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:53:12.587 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 4,7", 2023-09-23T20:53:12.587 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:09.123729+0000", 2023-09-23T20:53:12.587 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000140906 2023-09-23T20:53:12.587 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:53:12.587 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:53:12.587 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:53:12.587 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:09.126319+0000", 2023-09-23T20:53:12.587 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0025903380000000002 2023-09-23T20:53:12.588 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:53:12.588 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:53:12.588 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:53:12.588 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:09.186263+0000", 2023-09-23T20:53:12.588 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.059943862000000001 2023-09-23T20:53:12.588 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:53:12.588 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:53:12.588 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T20:53:12.588 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:09.186505+0000", 2023-09-23T20:53:12.588 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000242155 2023-09-23T20:53:12.589 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:53:12.589 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:53:12.589 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T20:53:12.589 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:09.186543+0000", 2023-09-23T20:53:12.589 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.7786e-05 2023-09-23T20:53:12.589 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:53:12.589 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:53:12.589 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T20:53:12.589 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:09.186575+0000", 2023-09-23T20:53:12.589 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.2363000000000002e-05 2023-09-23T20:53:12.589 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:53:12.590 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T20:53:12.590 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:53:12.590 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:53:12.590 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:53:12.590 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34797.0:72089 6.b 6:d8440395:::benchmark_data_smithi026_223868_object72088:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e116)", 2023-09-23T20:53:12.590 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T20:53:09.118237+0000", 2023-09-23T20:53:12.590 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 3.3769388330000001, 2023-09-23T20:53:12.590 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.068024056999999999, 2023-09-23T20:53:12.590 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T20:53:12.590 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:53:12.591 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T20:53:12.591 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34797", 2023-09-23T20:53:12.591 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T20:53:12.591 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 72089 2023-09-23T20:53:12.591 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:53:12.591 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T20:53:12.591 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:53:12.591 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T20:53:12.591 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:09.118237+0000", 2023-09-23T20:53:12.591 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:53:12.592 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:53:12.592 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:53:12.592 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T20:53:12.592 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:09.118237+0000", 2023-09-23T20:53:12.592 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:53:12.592 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:53:12.592 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:53:12.592 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T20:53:12.592 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:09.118237+0000", 2023-09-23T20:53:12.592 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T20:53:12.592 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:53:12.593 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:53:12.593 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T20:53:12.593 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:09.118261+0000", 2023-09-23T20:53:12.593 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.4409e-05 2023-09-23T20:53:12.593 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:53:12.593 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:53:12.593 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T20:53:12.593 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:09.118268+0000", 2023-09-23T20:53:12.593 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.776e-06 2023-09-23T20:53:12.593 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:53:12.594 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:53:12.594 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T20:53:12.594 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:09.118273+0000", 2023-09-23T20:53:12.594 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.3820000000000003e-06 2023-09-23T20:53:12.594 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:53:12.594 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:53:12.594 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:53:12.594 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:09.121086+0000", 2023-09-23T20:53:12.594 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0028124539999999998 2023-09-23T20:53:12.594 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:53:12.595 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:53:12.595 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T20:53:12.595 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:09.121205+0000", 2023-09-23T20:53:12.595 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000119589 2023-09-23T20:53:12.595 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:53:12.595 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:53:12.595 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 1,7", 2023-09-23T20:53:12.595 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:09.121330+0000", 2023-09-23T20:53:12.595 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00012472899999999999 2023-09-23T20:53:12.595 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:53:12.595 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:53:12.596 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:53:12.596 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:09.127035+0000", 2023-09-23T20:53:12.596 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0057047799999999996 2023-09-23T20:53:12.596 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:53:12.596 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:53:12.596 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:53:12.596 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:09.129745+0000", 2023-09-23T20:53:12.596 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0027107709999999998 2023-09-23T20:53:12.596 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:53:12.596 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:53:12.597 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T20:53:12.597 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:09.186191+0000", 2023-09-23T20:53:12.597 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.056445813999999997 2023-09-23T20:53:12.597 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:53:12.597 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:53:12.597 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T20:53:12.597 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:09.186230+0000", 2023-09-23T20:53:12.597 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.8296000000000001e-05 2023-09-23T20:53:12.597 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:53:12.597 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:53:12.597 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T20:53:12.598 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:09.186261+0000", 2023-09-23T20:53:12.598 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.1563999999999998e-05 2023-09-23T20:53:12.598 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:53:12.598 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T20:53:12.598 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:53:12.598 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:53:12.598 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:53:12.598 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34797.0:72091 6.6 6:6824f233:::benchmark_data_smithi026_223868_object72090:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e116)", 2023-09-23T20:53:12.598 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T20:53:09.118278+0000", 2023-09-23T20:53:12.598 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 3.3768981309999999, 2023-09-23T20:53:12.599 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.068381155999999998, 2023-09-23T20:53:12.599 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T20:53:12.599 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:53:12.599 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T20:53:12.599 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34797", 2023-09-23T20:53:12.599 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T20:53:12.599 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 72091 2023-09-23T20:53:12.599 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:53:12.599 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T20:53:12.599 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:53:12.599 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T20:53:12.601 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:09.118278+0000", 2023-09-23T20:53:12.601 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:53:12.601 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:53:12.601 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:53:12.601 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T20:53:12.601 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:09.118278+0000", 2023-09-23T20:53:12.601 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:53:12.602 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:53:12.602 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:53:12.602 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T20:53:12.602 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:09.118277+0000", 2023-09-23T20:53:12.602 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T20:53:12.602 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:53:12.602 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:53:12.602 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T20:53:12.602 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:09.118300+0000", 2023-09-23T20:53:12.602 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.2221e-05 2023-09-23T20:53:12.603 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:53:12.603 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:53:12.603 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T20:53:12.603 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:09.118306+0000", 2023-09-23T20:53:12.603 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.6730000000000003e-06 2023-09-23T20:53:12.603 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:53:12.603 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:53:12.603 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T20:53:12.603 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:09.118312+0000", 2023-09-23T20:53:12.603 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.2599999999999996e-06 2023-09-23T20:53:12.603 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:53:12.604 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:53:12.604 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:53:12.604 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:09.126520+0000", 2023-09-23T20:53:12.604 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0082085460000000006 2023-09-23T20:53:12.604 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:53:12.604 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:53:12.604 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T20:53:12.604 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:09.126687+0000", 2023-09-23T20:53:12.604 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00016649199999999999 2023-09-23T20:53:12.604 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:53:12.605 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:53:12.605 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 4,7", 2023-09-23T20:53:12.605 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:09.126852+0000", 2023-09-23T20:53:12.605 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00016557200000000001 2023-09-23T20:53:12.605 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:53:12.605 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:53:12.605 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:53:12.605 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:09.186049+0000", 2023-09-23T20:53:12.605 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.059197194000000002 2023-09-23T20:53:12.605 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:53:12.606 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:53:12.606 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:53:12.606 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:09.186306+0000", 2023-09-23T20:53:12.606 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00025688700000000002 2023-09-23T20:53:12.606 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:53:12.606 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:53:12.606 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T20:53:12.606 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:09.186586+0000", 2023-09-23T20:53:12.606 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00028009900000000002 2023-09-23T20:53:12.606 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:53:12.606 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:53:12.607 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T20:53:12.607 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:09.186624+0000", 2023-09-23T20:53:12.607 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.7219999999999999e-05 2023-09-23T20:53:12.607 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:53:12.607 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:53:12.607 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T20:53:12.607 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:09.186659+0000", 2023-09-23T20:53:12.607 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.5405999999999999e-05 2023-09-23T20:53:12.607 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:53:12.607 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T20:53:12.607 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:53:12.608 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:53:12.608 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:53:12.608 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_repop(client.34797.0:72093 6.f e116/102 6:ff3fbf06:::benchmark_data_smithi026_223868_object72092:head v 116'4452, mlcod=116'4450)", 2023-09-23T20:53:12.608 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T20:53:09.123564+0000", 2023-09-23T20:53:12.608 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 3.371611535, 2023-09-23T20:53:12.608 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.062462236999999997, 2023-09-23T20:53:12.608 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T20:53:12.608 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:53:12.608 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T20:53:12.608 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:53:12.609 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T20:53:12.609 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:09.123564+0000", 2023-09-23T20:53:12.609 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:53:12.609 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:53:12.609 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:53:12.609 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T20:53:12.609 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:09.123564+0000", 2023-09-23T20:53:12.609 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:53:12.609 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:53:12.609 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:53:12.609 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T20:53:12.610 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:09.123563+0000", 2023-09-23T20:53:12.610 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.999999 2023-09-23T20:53:12.610 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:53:12.610 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:53:12.610 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T20:53:12.610 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:09.123585+0000", 2023-09-23T20:53:12.610 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.1733e-05 2023-09-23T20:53:12.610 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:53:12.610 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:53:12.610 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T20:53:12.611 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:09.123596+0000", 2023-09-23T20:53:12.611 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0535e-05 2023-09-23T20:53:12.611 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:53:12.611 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:53:12.611 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T20:53:12.611 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:09.123602+0000", 2023-09-23T20:53:12.611 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.4579999999999998e-06 2023-09-23T20:53:12.611 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:53:12.611 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:53:12.611 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:53:12.612 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:09.124297+0000", 2023-09-23T20:53:12.612 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00069532299999999997 2023-09-23T20:53:12.612 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:53:12.612 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:53:12.612 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T20:53:12.612 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:09.124315+0000", 2023-09-23T20:53:12.612 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.7129000000000002e-05 2023-09-23T20:53:12.612 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:53:12.612 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:53:12.612 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T20:53:12.612 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:09.132691+0000", 2023-09-23T20:53:12.613 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0083768420000000007 2023-09-23T20:53:12.613 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:53:12.613 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:53:12.613 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T20:53:12.613 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:09.186027+0000", 2023-09-23T20:53:12.613 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.053335201999999998 2023-09-23T20:53:12.613 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:53:12.613 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T20:53:12.613 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:53:12.613 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:53:12.614 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:53:12.614 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_repop(client.34797.0:72096 6.4 e116/102 6:233218e7:::benchmark_data_smithi026_223868_object72095:head v 116'4533, mlcod=116'4530)", 2023-09-23T20:53:12.614 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T20:53:09.123701+0000", 2023-09-23T20:53:12.614 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 3.371475336, 2023-09-23T20:53:12.614 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.062383717999999998, 2023-09-23T20:53:12.614 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T20:53:12.614 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:53:12.614 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T20:53:12.614 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:53:12.614 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T20:53:12.614 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:09.123701+0000", 2023-09-23T20:53:12.615 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:53:12.615 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:53:12.615 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:53:12.615 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T20:53:12.615 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:09.123701+0000", 2023-09-23T20:53:12.615 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:53:12.615 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:53:12.615 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:53:12.615 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T20:53:12.615 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:09.123700+0000", 2023-09-23T20:53:12.616 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T20:53:12.616 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:53:12.616 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:53:12.616 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T20:53:12.616 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:09.123719+0000", 2023-09-23T20:53:12.616 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.9063e-05 2023-09-23T20:53:12.616 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:53:12.616 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:53:12.616 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T20:53:12.616 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:09.123728+0000", 2023-09-23T20:53:12.617 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.9460000000000004e-06 2023-09-23T20:53:12.617 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:53:12.617 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:53:12.617 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T20:53:12.617 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:09.123734+0000", 2023-09-23T20:53:12.617 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.4790000000000004e-06 2023-09-23T20:53:12.617 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:53:12.617 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:53:12.617 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:53:12.617 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:09.124269+0000", 2023-09-23T20:53:12.617 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00053541699999999995 2023-09-23T20:53:12.618 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:53:12.618 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:53:12.618 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T20:53:12.618 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:09.124289+0000", 2023-09-23T20:53:12.618 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.9901000000000001e-05 2023-09-23T20:53:12.618 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:53:12.618 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:53:12.618 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T20:53:12.618 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:09.186058+0000", 2023-09-23T20:53:12.619 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.061769109000000003 2023-09-23T20:53:12.619 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:53:12.619 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:53:12.619 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T20:53:12.619 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:09.186084+0000", 2023-09-23T20:53:12.619 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.6299e-05 2023-09-23T20:53:12.619 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:53:12.619 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T20:53:12.619 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:53:12.619 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:53:12.619 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:53:12.620 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34797.0:72092 6.6 6:626db0ed:::benchmark_data_smithi026_223868_object72091:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e116)", 2023-09-23T20:53:12.620 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T20:53:09.126463+0000", 2023-09-23T20:53:12.620 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 3.3687127100000001, 2023-09-23T20:53:12.620 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.060347071000000002, 2023-09-23T20:53:12.620 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T20:53:12.620 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:53:12.620 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T20:53:12.620 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34797", 2023-09-23T20:53:12.620 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T20:53:12.620 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 72092 2023-09-23T20:53:12.621 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:53:12.621 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T20:53:12.621 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:53:12.621 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T20:53:12.621 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:09.126463+0000", 2023-09-23T20:53:12.621 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:53:12.621 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:53:12.621 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:53:12.621 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T20:53:12.621 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:09.126463+0000", 2023-09-23T20:53:12.621 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:53:12.622 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:53:12.622 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:53:12.622 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T20:53:12.622 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:09.126462+0000", 2023-09-23T20:53:12.622 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T20:53:12.622 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:53:12.622 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:53:12.622 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T20:53:12.622 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:09.126500+0000", 2023-09-23T20:53:12.622 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.8241999999999997e-05 2023-09-23T20:53:12.623 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:53:12.623 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:53:12.623 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T20:53:12.623 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:09.126512+0000", 2023-09-23T20:53:12.623 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.2169e-05 2023-09-23T20:53:12.623 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:53:12.623 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:53:12.623 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T20:53:12.623 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:09.126521+0000", 2023-09-23T20:53:12.623 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.7129999999999994e-06 2023-09-23T20:53:12.624 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:53:12.624 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:53:12.624 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:53:12.624 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:09.129695+0000", 2023-09-23T20:53:12.624 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.003174484 2023-09-23T20:53:12.624 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:53:12.624 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:53:12.624 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T20:53:12.624 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:09.129851+0000", 2023-09-23T20:53:12.624 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00015627699999999999 2023-09-23T20:53:12.624 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:53:12.625 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:53:12.625 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 4,7", 2023-09-23T20:53:12.625 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:09.130011+0000", 2023-09-23T20:53:12.625 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00015930600000000001 2023-09-23T20:53:12.625 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:53:12.625 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:53:12.625 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T20:53:12.625 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:09.186666+0000", 2023-09-23T20:53:12.625 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.056655177000000001 2023-09-23T20:53:12.625 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:53:12.626 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:53:12.626 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:53:12.626 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:09.186701+0000", 2023-09-23T20:53:12.626 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.5293999999999998e-05 2023-09-23T20:53:12.626 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:53:12.626 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:53:12.626 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:53:12.626 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:09.186733+0000", 2023-09-23T20:53:12.626 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.1783999999999999e-05 2023-09-23T20:53:12.627 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:53:12.627 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:53:12.627 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T20:53:12.627 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:09.186778+0000", 2023-09-23T20:53:12.627 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.5022999999999999e-05 2023-09-23T20:53:12.627 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:53:12.627 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:53:12.627 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T20:53:12.627 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:09.186810+0000", 2023-09-23T20:53:12.627 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.2236000000000001e-05 2023-09-23T20:53:12.627 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:53:12.628 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T20:53:12.628 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:53:12.628 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:53:12.628 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:53:12.628 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34797.0:72094 6.7 6:edc6476e:::benchmark_data_smithi026_223868_object72093:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e116)", 2023-09-23T20:53:12.628 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T20:53:09.126536+0000", 2023-09-23T20:53:12.628 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 3.368640225, 2023-09-23T20:53:12.628 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.059649287000000002, 2023-09-23T20:53:12.628 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T20:53:12.628 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:53:12.629 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T20:53:12.629 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34797", 2023-09-23T20:53:12.629 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T20:53:12.629 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 72094 2023-09-23T20:53:12.629 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:53:12.629 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T20:53:12.629 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:53:12.629 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T20:53:12.629 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:09.126536+0000", 2023-09-23T20:53:12.629 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:53:12.629 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:53:12.630 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:53:12.630 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T20:53:12.630 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:09.126536+0000", 2023-09-23T20:53:12.630 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:53:12.630 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:53:12.630 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:53:12.630 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T20:53:12.630 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:09.126535+0000", 2023-09-23T20:53:12.630 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T20:53:12.630 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:53:12.631 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:53:12.631 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T20:53:12.631 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:09.126560+0000", 2023-09-23T20:53:12.631 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.4882999999999999e-05 2023-09-23T20:53:12.631 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:53:12.631 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:53:12.631 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T20:53:12.631 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:09.126569+0000", 2023-09-23T20:53:12.631 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.8750000000000006e-06 2023-09-23T20:53:12.631 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:53:12.632 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:53:12.632 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T20:53:12.632 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:09.126575+0000", 2023-09-23T20:53:12.632 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.0900000000000001e-06 2023-09-23T20:53:12.632 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:53:12.632 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:53:12.632 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:53:12.632 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:09.126612+0000", 2023-09-23T20:53:12.632 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.7076000000000002e-05 2023-09-23T20:53:12.632 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:53:12.632 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:53:12.633 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T20:53:12.633 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:09.126766+0000", 2023-09-23T20:53:12.633 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00015427799999999999 2023-09-23T20:53:12.633 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:53:12.633 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:53:12.633 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 1,4", 2023-09-23T20:53:12.633 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:09.126901+0000", 2023-09-23T20:53:12.633 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00013437099999999999 2023-09-23T20:53:12.633 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:53:12.633 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:53:12.633 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:53:12.634 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:09.129834+0000", 2023-09-23T20:53:12.634 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0029336119999999999 2023-09-23T20:53:12.634 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:53:12.634 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:53:12.634 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:53:12.634 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:09.130400+0000", 2023-09-23T20:53:12.634 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00056540300000000002 2023-09-23T20:53:12.634 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:53:12.635 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:53:12.635 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T20:53:12.635 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:09.186101+0000", 2023-09-23T20:53:12.635 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.055701514000000001 2023-09-23T20:53:12.635 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:53:12.635 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:53:12.635 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T20:53:12.635 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:09.186146+0000", 2023-09-23T20:53:12.635 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.5287999999999997e-05 2023-09-23T20:53:12.636 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:53:12.636 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:53:12.636 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T20:53:12.636 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:09.186185+0000", 2023-09-23T20:53:12.636 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.8537000000000001e-05 2023-09-23T20:53:12.636 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:53:12.636 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T20:53:12.636 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:53:12.637 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:53:12.637 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:53:12.637 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34797.0:72987 6.c 6:390f0dda:::benchmark_data_smithi026_223868_object72986:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e116)", 2023-09-23T20:53:12.637 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T20:53:10.025522+0000", 2023-09-23T20:53:12.637 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 2.4696538490000002, 2023-09-23T20:53:12.637 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.062634704999999999, 2023-09-23T20:53:12.637 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T20:53:12.637 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:53:12.637 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T20:53:12.637 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34797", 2023-09-23T20:53:12.638 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T20:53:12.638 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 72987 2023-09-23T20:53:12.638 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:53:12.638 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T20:53:12.638 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:53:12.638 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T20:53:12.638 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:10.025522+0000", 2023-09-23T20:53:12.638 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:53:12.639 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:53:12.639 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:53:12.639 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T20:53:12.639 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:10.025522+0000", 2023-09-23T20:53:12.639 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:53:12.639 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:53:12.639 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:53:12.639 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T20:53:12.639 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:10.025522+0000", 2023-09-23T20:53:12.640 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T20:53:12.640 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:53:12.640 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:53:12.640 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T20:53:12.640 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:10.025550+0000", 2023-09-23T20:53:12.640 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.8232999999999999e-05 2023-09-23T20:53:12.640 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:53:12.640 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:53:12.640 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T20:53:12.640 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:10.025558+0000", 2023-09-23T20:53:12.641 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.1540000000000002e-06 2023-09-23T20:53:12.641 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:53:12.641 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:53:12.641 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T20:53:12.641 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:10.025564+0000", 2023-09-23T20:53:12.641 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.818e-06 2023-09-23T20:53:12.641 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:53:12.641 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:53:12.641 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:53:12.641 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:10.028480+0000", 2023-09-23T20:53:12.642 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0029164590000000002 2023-09-23T20:53:12.642 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:53:12.642 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:53:12.642 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T20:53:12.642 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:10.028618+0000", 2023-09-23T20:53:12.642 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00013774800000000001 2023-09-23T20:53:12.642 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:53:12.642 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:53:12.642 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 6,7", 2023-09-23T20:53:12.642 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:10.028753+0000", 2023-09-23T20:53:12.643 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00013492300000000001 2023-09-23T20:53:12.643 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:53:12.643 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:53:12.643 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T20:53:12.643 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:10.031592+0000", 2023-09-23T20:53:12.643 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0028394710000000001 2023-09-23T20:53:12.643 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:53:12.643 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:53:12.643 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:53:12.643 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:10.087771+0000", 2023-09-23T20:53:12.644 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.056178778999999998 2023-09-23T20:53:12.644 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:53:12.644 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:53:12.644 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:53:12.644 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:10.088083+0000", 2023-09-23T20:53:12.644 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00031144199999999999 2023-09-23T20:53:12.644 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:53:12.644 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:53:12.644 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T20:53:12.644 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:10.088120+0000", 2023-09-23T20:53:12.645 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.7419999999999997e-05 2023-09-23T20:53:12.645 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:53:12.645 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:53:12.645 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T20:53:12.645 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:10.088157+0000", 2023-09-23T20:53:12.645 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.6797e-05 2023-09-23T20:53:12.645 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:53:12.645 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T20:53:12.645 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:53:12.645 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:53:12.646 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:53:12.646 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_repop(client.34797.0:73000 6.0 e116/102 6:01da73c9:::benchmark_data_smithi026_223868_object72999:head v 116'4615, mlcod=116'4612)", 2023-09-23T20:53:12.646 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T20:53:10.035617+0000", 2023-09-23T20:53:12.646 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 2.4595586460000001, 2023-09-23T20:53:12.646 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.064480148000000001, 2023-09-23T20:53:12.646 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T20:53:12.646 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:53:12.646 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T20:53:12.646 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:53:12.646 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T20:53:12.647 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:10.035617+0000", 2023-09-23T20:53:12.647 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:53:12.647 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:53:12.647 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:53:12.647 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T20:53:12.647 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:10.035617+0000", 2023-09-23T20:53:12.647 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:53:12.647 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:53:12.647 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:53:12.647 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T20:53:12.648 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:10.035617+0000", 2023-09-23T20:53:12.648 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.999999 2023-09-23T20:53:12.648 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:53:12.648 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:53:12.648 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T20:53:12.648 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:10.035640+0000", 2023-09-23T20:53:12.648 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.3210999999999999e-05 2023-09-23T20:53:12.648 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:53:12.648 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:53:12.648 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T20:53:12.648 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:10.087765+0000", 2023-09-23T20:53:12.649 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.052125328999999998 2023-09-23T20:53:12.649 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:53:12.649 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:53:12.649 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T20:53:12.649 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:10.087784+0000", 2023-09-23T20:53:12.649 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.8488e-05 2023-09-23T20:53:12.649 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:53:12.649 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:53:12.649 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:53:12.649 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:10.088143+0000", 2023-09-23T20:53:12.650 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00035935300000000003 2023-09-23T20:53:12.650 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:53:12.650 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:53:12.650 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T20:53:12.650 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:10.088169+0000", 2023-09-23T20:53:12.650 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.5956999999999999e-05 2023-09-23T20:53:12.650 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:53:12.650 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:53:12.650 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T20:53:12.650 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:10.100051+0000", 2023-09-23T20:53:12.651 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.011882075000000001 2023-09-23T20:53:12.651 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:53:12.651 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:53:12.651 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T20:53:12.651 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:10.100097+0000", 2023-09-23T20:53:12.651 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.6474999999999998e-05 2023-09-23T20:53:12.651 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:53:12.651 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T20:53:12.651 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:53:12.651 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:53:12.652 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T20:53:12.652 INFO:teuthology.orchestra.run.smithi026.stdout:} 2023-09-23T20:53:12.653 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:53:12.653 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:53:12.653 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:53:12.653 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:53:12.754 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:53:12.787 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:53:12.829 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:53:12.831 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:53:12.831 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:53:12.881 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 53 16 75716 75700 89.1785 57 0.0121342 0.0112087 2023-09-23T20:53:12.932 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:53:12.946 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_ops_in_flight 2023-09-23T20:53:12.946 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:53:12 smithi026 ceph-mon[179688]: from='client.44795 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:53:12.946 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:53:12 smithi026 ceph-mon[175151]: from='client.44795 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:53:13.012 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:53:13.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:53:12 smithi070 ceph-mon[168282]: from='client.44795 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:53:13.136 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:53:13.142 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:53:13.143 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:53:13.244 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:53:13.350 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:53:13.434 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T20:53:13.465 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:53:13.470 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:53:13.477 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:53:13.477 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:53:13.578 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:53:13.607 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:53:13.617 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:53:13.654 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:53:13.662 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:53:13.663 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:53:13.763 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:53:13.805 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:53:13.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:53:13 smithi070 ceph-mon[168282]: pgmap v278: 89 pgs: 89 active+clean; 4.5 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 58 MiB/s wr, 929 op/s 2023-09-23T20:53:13.881 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 54 16 76634 76618 88.59 57.375 0.00883161 0.0112817 2023-09-23T20:53:13.883 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:53:13 smithi026 ceph-mon[175151]: pgmap v278: 89 pgs: 89 active+clean; 4.5 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 58 MiB/s wr, 929 op/s 2023-09-23T20:53:13.883 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:53:13 smithi026 ceph-mon[179688]: pgmap v278: 89 pgs: 89 active+clean; 4.5 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 58 MiB/s wr, 929 op/s 2023-09-23T20:53:13.895 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:53:13.902 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:53:13.903 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:53:14.003 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:53:14.039 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:53:14.077 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:53:14.080 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:53:14.080 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:53:14.181 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:53:14.253 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:53:14.399 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:53:14.409 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:53:14.409 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:53:14.510 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:53:14.607 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:53:14.683 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:53:14.689 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:53:14.689 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:53:14.689 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:53:14.790 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:53:14.824 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:53:14.881 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 55 16 77531 77515 87.999 56.0625 0.0190716 0.0113587 2023-09-23T20:53:14.895 INFO:teuthology.orchestra.run.smithi026.stderr:nodeep-scrub is unset 2023-09-23T20:53:14.924 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:53:14.930 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:53:14.930 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:53:14.941 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:53:14 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/4149092558' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:53:14.941 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:53:14 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:53:14.941 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:53:14 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/4149092558' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:53:14.941 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:53:14 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:53:15.003 INFO:teuthology.orchestra.run.smithi070.stdout:{ 2023-09-23T20:53:15.003 INFO:teuthology.orchestra.run.smithi070.stdout: "ops": [ 2023-09-23T20:53:15.003 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:15.003 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.34797.0:77626 6.2 6:4f6dd74d:::benchmark_data_smithi026_223868_object77625:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e117)", 2023-09-23T20:53:15.004 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T20:53:14.987708+0000", 2023-09-23T20:53:15.004 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 0.015224361, 2023-09-23T20:53:15.004 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.015257331000000001, 2023-09-23T20:53:15.004 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T20:53:15.004 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "waiting for sub ops", 2023-09-23T20:53:15.004 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T20:53:15.004 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.34797", 2023-09-23T20:53:15.004 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T20:53:15.004 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 77626 2023-09-23T20:53:15.004 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:15.004 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T20:53:15.005 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:15.005 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T20:53:15.005 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:14.987708+0000", 2023-09-23T20:53:15.005 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T20:53:15.005 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:15.005 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:15.005 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T20:53:15.005 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:14.987708+0000", 2023-09-23T20:53:15.005 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T20:53:15.005 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:15.005 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:15.006 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T20:53:15.006 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:14.987706+0000", 2023-09-23T20:53:15.006 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999981 2023-09-23T20:53:15.006 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:15.006 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:15.006 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T20:53:15.006 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:14.987768+0000", 2023-09-23T20:53:15.006 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.1895000000000003e-05 2023-09-23T20:53:15.006 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:15.006 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:15.007 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T20:53:15.007 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:14.987784+0000", 2023-09-23T20:53:15.007 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.6351e-05 2023-09-23T20:53:15.007 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:15.007 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:15.007 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T20:53:15.007 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:14.987792+0000", 2023-09-23T20:53:15.007 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.5859999999999999e-06 2023-09-23T20:53:15.007 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:15.007 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:15.007 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T20:53:15.008 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:14.987836+0000", 2023-09-23T20:53:15.008 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.4505000000000002e-05 2023-09-23T20:53:15.008 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:15.008 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:15.008 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T20:53:15.008 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:14.987942+0000", 2023-09-23T20:53:15.008 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000105852 2023-09-23T20:53:15.008 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:15.008 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:15.008 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 2,3", 2023-09-23T20:53:15.008 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:14.988087+0000", 2023-09-23T20:53:15.008 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000145041 2023-09-23T20:53:15.009 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:15.009 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:15.009 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T20:53:15.009 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:14.993016+0000", 2023-09-23T20:53:15.009 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0049283199999999999 2023-09-23T20:53:15.009 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:15.009 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:15.009 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:53:15.009 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:14.997704+0000", 2023-09-23T20:53:15.009 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0046882929999999996 2023-09-23T20:53:15.009 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T20:53:15.009 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T20:53:15.010 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T20:53:15.010 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:15.010 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:15.010 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.34797.0:77632 6.2 6:4dde3dfc:::benchmark_data_smithi026_223868_object77631:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e117)", 2023-09-23T20:53:15.010 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T20:53:14.987877+0000", 2023-09-23T20:53:15.010 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 0.015054592, 2023-09-23T20:53:15.010 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.015183277, 2023-09-23T20:53:15.010 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T20:53:15.010 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "waiting for sub ops", 2023-09-23T20:53:15.010 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T20:53:15.010 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.34797", 2023-09-23T20:53:15.010 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T20:53:15.011 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 77632 2023-09-23T20:53:15.011 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:15.011 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T20:53:15.011 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:15.011 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T20:53:15.011 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:14.987877+0000", 2023-09-23T20:53:15.011 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T20:53:15.011 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:15.011 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:15.011 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T20:53:15.011 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:14.987877+0000", 2023-09-23T20:53:15.011 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T20:53:15.012 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:15.012 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:15.012 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T20:53:15.012 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:14.987877+0000", 2023-09-23T20:53:15.012 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999995 2023-09-23T20:53:15.012 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:15.012 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:15.012 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T20:53:15.012 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:14.987952+0000", 2023-09-23T20:53:15.012 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.5545999999999998e-05 2023-09-23T20:53:15.012 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:15.013 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:15.013 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T20:53:15.013 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:14.987960+0000", 2023-09-23T20:53:15.013 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.1149999999999994e-06 2023-09-23T20:53:15.013 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:15.013 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:15.013 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T20:53:15.013 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:14.987965+0000", 2023-09-23T20:53:15.013 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.7709999999999997e-06 2023-09-23T20:53:15.013 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:15.013 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:15.014 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T20:53:15.014 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:14.990509+0000", 2023-09-23T20:53:15.014 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0025435459999999998 2023-09-23T20:53:15.014 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:15.014 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:15.014 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T20:53:15.014 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:14.990680+0000", 2023-09-23T20:53:15.014 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000171194 2023-09-23T20:53:15.014 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:15.014 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:15.014 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 2,3", 2023-09-23T20:53:15.014 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:14.990803+0000", 2023-09-23T20:53:15.015 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00012286999999999999 2023-09-23T20:53:15.015 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:15.015 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:15.015 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T20:53:15.015 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:14.993151+0000", 2023-09-23T20:53:15.015 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0023477820000000001 2023-09-23T20:53:15.015 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:15.015 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:15.015 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:53:15.015 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:14.997726+0000", 2023-09-23T20:53:15.015 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0045750089999999997 2023-09-23T20:53:15.016 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T20:53:15.016 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T20:53:15.016 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T20:53:15.016 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:15.016 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:15.016 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.34797.0:77633 6.e 6:74c4aa50:::benchmark_data_smithi026_223868_object77632:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e117)", 2023-09-23T20:53:15.016 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T20:53:14.988099+0000", 2023-09-23T20:53:15.016 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 0.014833157, 2023-09-23T20:53:15.016 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.015048113, 2023-09-23T20:53:15.016 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T20:53:15.016 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "waiting for sub ops", 2023-09-23T20:53:15.016 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T20:53:15.017 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.34797", 2023-09-23T20:53:15.017 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T20:53:15.017 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 77633 2023-09-23T20:53:15.017 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:15.017 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T20:53:15.017 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:15.017 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T20:53:15.017 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:14.988099+0000", 2023-09-23T20:53:15.017 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T20:53:15.017 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:15.017 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:15.018 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T20:53:15.018 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:14.988099+0000", 2023-09-23T20:53:15.018 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T20:53:15.018 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:15.018 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:15.018 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T20:53:15.018 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:14.988098+0000", 2023-09-23T20:53:15.018 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.999999 2023-09-23T20:53:15.018 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:15.018 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:15.018 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T20:53:15.019 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:14.988190+0000", 2023-09-23T20:53:15.019 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.1881999999999999e-05 2023-09-23T20:53:15.019 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:15.019 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:15.019 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T20:53:15.019 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:14.988202+0000", 2023-09-23T20:53:15.019 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.2038000000000001e-05 2023-09-23T20:53:15.019 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:15.019 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:15.019 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T20:53:15.019 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:14.988208+0000", 2023-09-23T20:53:15.020 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.2559999999999999e-06 2023-09-23T20:53:15.020 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:15.020 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:15.020 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T20:53:15.020 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:14.991281+0000", 2023-09-23T20:53:15.020 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0030728650000000001 2023-09-23T20:53:15.020 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:15.020 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:15.020 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T20:53:15.020 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:14.991413+0000", 2023-09-23T20:53:15.020 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00013219699999999999 2023-09-23T20:53:15.021 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:15.021 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:15.021 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 1,2", 2023-09-23T20:53:15.021 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:14.991591+0000", 2023-09-23T20:53:15.021 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000177407 2023-09-23T20:53:15.021 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:15.021 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:15.021 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T20:53:15.021 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:14.993750+0000", 2023-09-23T20:53:15.021 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.002159691 2023-09-23T20:53:15.021 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:15.021 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:15.022 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:53:15.022 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:15.001307+0000", 2023-09-23T20:53:15.022 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0075568420000000002 2023-09-23T20:53:15.022 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T20:53:15.022 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T20:53:15.022 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T20:53:15.022 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:15.022 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:15.022 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.34797.0:77641 6.2 6:49064fb8:::benchmark_data_smithi026_223868_object77640:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e117)", 2023-09-23T20:53:15.022 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T20:53:14.998839+0000", 2023-09-23T20:53:15.022 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 0.0040933819999999996, 2023-09-23T20:53:15.022 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.004399254, 2023-09-23T20:53:15.023 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T20:53:15.023 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "waiting for sub ops", 2023-09-23T20:53:15.023 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T20:53:15.023 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.34797", 2023-09-23T20:53:15.023 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T20:53:15.023 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 77641 2023-09-23T20:53:15.023 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:15.023 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T20:53:15.023 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:15.023 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T20:53:15.023 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:14.998839+0000", 2023-09-23T20:53:15.023 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T20:53:15.023 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:15.024 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:15.024 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T20:53:15.024 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:14.998839+0000", 2023-09-23T20:53:15.024 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T20:53:15.024 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:15.024 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:15.024 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T20:53:15.024 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:14.998837+0000", 2023-09-23T20:53:15.024 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T20:53:15.024 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:15.024 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:15.024 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T20:53:15.025 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:14.998896+0000", 2023-09-23T20:53:15.025 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.8768999999999999e-05 2023-09-23T20:53:15.025 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:15.025 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:15.025 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T20:53:15.025 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:14.998910+0000", 2023-09-23T20:53:15.025 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.3854e-05 2023-09-23T20:53:15.025 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:15.025 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:15.025 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T20:53:15.025 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:14.998916+0000", 2023-09-23T20:53:15.025 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.0009999999999998e-06 2023-09-23T20:53:15.025 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:15.026 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:15.026 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T20:53:15.026 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:14.998948+0000", 2023-09-23T20:53:15.026 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.2073999999999998e-05 2023-09-23T20:53:15.026 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:15.026 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:15.026 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T20:53:15.026 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:14.999030+0000", 2023-09-23T20:53:15.026 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.1714999999999999e-05 2023-09-23T20:53:15.026 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:15.026 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:15.026 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 2,3", 2023-09-23T20:53:15.027 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:14.999111+0000", 2023-09-23T20:53:15.027 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.1594000000000005e-05 2023-09-23T20:53:15.027 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T20:53:15.027 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T20:53:15.027 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T20:53:15.027 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:15.027 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:15.027 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.34797.0:77643 6.2 6:4dedb4d4:::benchmark_data_smithi026_223868_object77642:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e117)", 2023-09-23T20:53:15.027 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T20:53:14.998928+0000", 2023-09-23T20:53:15.027 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 0.0040042949999999997, 2023-09-23T20:53:15.027 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0043839559999999996, 2023-09-23T20:53:15.027 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T20:53:15.027 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "waiting for sub ops", 2023-09-23T20:53:15.028 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T20:53:15.028 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.34797", 2023-09-23T20:53:15.028 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T20:53:15.028 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 77643 2023-09-23T20:53:15.028 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:15.028 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T20:53:15.028 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:15.028 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T20:53:15.028 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:14.998928+0000", 2023-09-23T20:53:15.028 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T20:53:15.028 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:15.028 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:15.029 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T20:53:15.029 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:14.998928+0000", 2023-09-23T20:53:15.029 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T20:53:15.029 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:15.029 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:15.029 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T20:53:15.029 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:14.998927+0000", 2023-09-23T20:53:15.029 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999995 2023-09-23T20:53:15.029 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:15.029 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:15.029 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T20:53:15.029 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:14.998959+0000", 2023-09-23T20:53:15.029 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.1714999999999997e-05 2023-09-23T20:53:15.030 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:15.030 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:15.030 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T20:53:15.030 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:14.998968+0000", 2023-09-23T20:53:15.030 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.4089999999999992e-06 2023-09-23T20:53:15.030 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:15.030 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:15.030 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T20:53:15.030 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:14.998974+0000", 2023-09-23T20:53:15.030 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.9089999999999997e-06 2023-09-23T20:53:15.030 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:15.030 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:15.031 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T20:53:15.031 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:15.001541+0000", 2023-09-23T20:53:15.031 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0025663320000000002 2023-09-23T20:53:15.031 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:15.031 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:15.031 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T20:53:15.031 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:15.001684+0000", 2023-09-23T20:53:15.031 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000142953 2023-09-23T20:53:15.031 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:15.031 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:15.031 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 2,3", 2023-09-23T20:53:15.031 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:15.001852+0000", 2023-09-23T20:53:15.032 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00016854500000000001 2023-09-23T20:53:15.032 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T20:53:15.032 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T20:53:15.032 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T20:53:15.032 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:15.032 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:15.032 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_repop_reply(client.34797.0:77626 6.2 e117/102)", 2023-09-23T20:53:15.032 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T20:53:15.001932+0000", 2023-09-23T20:53:15.032 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 0.001000272, 2023-09-23T20:53:15.032 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0014461680000000001, 2023-09-23T20:53:15.032 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T20:53:15.032 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "queued for pg", 2023-09-23T20:53:15.032 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T20:53:15.033 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:15.033 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T20:53:15.033 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:15.001932+0000", 2023-09-23T20:53:15.033 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T20:53:15.033 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:15.033 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:15.033 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T20:53:15.033 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:15.001932+0000", 2023-09-23T20:53:15.033 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T20:53:15.033 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:15.033 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:15.033 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T20:53:15.033 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:15.001931+0000", 2023-09-23T20:53:15.034 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999995 2023-09-23T20:53:15.034 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:15.034 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:15.034 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T20:53:15.034 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:15.001936+0000", 2023-09-23T20:53:15.034 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.51e-06 2023-09-23T20:53:15.034 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:15.034 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:15.034 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T20:53:15.034 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:15.001955+0000", 2023-09-23T20:53:15.034 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.9038999999999998e-05 2023-09-23T20:53:15.034 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:15.035 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:15.035 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T20:53:15.035 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:15.001968+0000", 2023-09-23T20:53:15.035 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.2855e-05 2023-09-23T20:53:15.035 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T20:53:15.035 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T20:53:15.035 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T20:53:15.035 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:15.035 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:15.035 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_repop(client.34797.0:77637 6.6 e117/102 6:692dc16c:::benchmark_data_smithi026_223868_object77636:head v 117'4808, mlcod=117'4805)", 2023-09-23T20:53:15.035 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T20:53:15.001990+0000", 2023-09-23T20:53:15.035 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 0.00094194199999999997, 2023-09-23T20:53:15.035 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0014370909999999999, 2023-09-23T20:53:15.036 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T20:53:15.036 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "started", 2023-09-23T20:53:15.036 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T20:53:15.036 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:15.036 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T20:53:15.036 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:15.001990+0000", 2023-09-23T20:53:15.036 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T20:53:15.036 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:15.036 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:15.036 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T20:53:15.036 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:15.001990+0000", 2023-09-23T20:53:15.036 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T20:53:15.036 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:15.037 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:15.037 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T20:53:15.037 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:15.001990+0000", 2023-09-23T20:53:15.037 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999995 2023-09-23T20:53:15.037 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:15.037 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:15.037 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T20:53:15.037 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:15.002023+0000", 2023-09-23T20:53:15.037 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.3330000000000001e-05 2023-09-23T20:53:15.037 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:15.037 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:15.037 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T20:53:15.038 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:15.002030+0000", 2023-09-23T20:53:15.038 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.311e-06 2023-09-23T20:53:15.038 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:15.038 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:15.038 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T20:53:15.038 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:15.002035+0000", 2023-09-23T20:53:15.038 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.4299999999999999e-06 2023-09-23T20:53:15.038 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:15.038 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:15.038 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T20:53:15.038 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:15.002067+0000", 2023-09-23T20:53:15.038 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.2662e-05 2023-09-23T20:53:15.039 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:15.039 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:15.039 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T20:53:15.039 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:15.002081+0000", 2023-09-23T20:53:15.039 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.3750999999999999e-05 2023-09-23T20:53:15.039 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T20:53:15.039 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T20:53:15.039 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T20:53:15.039 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T20:53:15.039 INFO:teuthology.orchestra.run.smithi070.stdout: ], 2023-09-23T20:53:15.039 INFO:teuthology.orchestra.run.smithi070.stdout: "num_ops": 7 2023-09-23T20:53:15.039 INFO:teuthology.orchestra.run.smithi070.stdout:} 2023-09-23T20:53:15.039 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:53:15.040 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:53:14 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/4149092558' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:53:15.040 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:53:14 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:53:15.072 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:53:15.125 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:53:15.128 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:53:15.128 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:53:15.229 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:53:15.264 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:53:15.355 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:53:15.358 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:53:15.359 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:53:15.400 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_blocked_ops 2023-09-23T20:53:15.459 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:53:15.501 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:53:15.542 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:53:15.545 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:53:15.546 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:53:15.581 INFO:teuthology.orchestra.run.smithi026.stdout: 2023-09-23T20:53:15.606 INFO:teuthology.orchestra.run.smithi026.stderr:dumped all 2023-09-23T20:53:15.646 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:53:15.686 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:53:15.735 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:53:15.740 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:53:15.740 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:53:15.841 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:53:15.881 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 56 16 78467 78451 87.4726 58.5 0.00925118 0.0114261 2023-09-23T20:53:15.895 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:53:15 smithi026 ceph-mon[179688]: pgmap v279: 89 pgs: 89 active+clean; 4.6 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 61 MiB/s wr, 972 op/s 2023-09-23T20:53:15.895 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:53:15 smithi026 ceph-mon[179688]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T20:53:15.895 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:53:15 smithi026 ceph-mon[179688]: Cluster is now healthy 2023-09-23T20:53:15.895 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:53:15 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T20:53:15.895 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:53:15 smithi026 ceph-mon[179688]: osdmap e117: 8 total, 8 up, 7 in 2023-09-23T20:53:15.896 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:53:15 smithi026 ceph-mon[179688]: failed to encode map e117 with expected crc 2023-09-23T20:53:15.896 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:53:15 smithi026 ceph-mon[179688]: failed to encode map e117 with expected crc 2023-09-23T20:53:15.896 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:53:15 smithi026 ceph-mon[179688]: failed to encode map e117 with expected crc 2023-09-23T20:53:15.896 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:53:15 smithi026 ceph-mon[179688]: failed to encode map e117 with expected crc 2023-09-23T20:53:15.896 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:53:15 smithi026 ceph-mon[175151]: pgmap v279: 89 pgs: 89 active+clean; 4.6 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 61 MiB/s wr, 972 op/s 2023-09-23T20:53:15.896 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:53:15 smithi026 ceph-mon[175151]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T20:53:15.897 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:53:15 smithi026 ceph-mon[175151]: Cluster is now healthy 2023-09-23T20:53:15.897 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:53:15 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T20:53:15.897 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:53:15 smithi026 ceph-mon[175151]: osdmap e117: 8 total, 8 up, 7 in 2023-09-23T20:53:15.897 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:53:15 smithi026 ceph-mon[175151]: failed to encode map e117 with expected crc 2023-09-23T20:53:15.897 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:53:15 smithi026 ceph-mon[175151]: failed to encode map e117 with expected crc 2023-09-23T20:53:15.897 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:53:15 smithi026 ceph-mon[175151]: failed to encode map e117 with expected crc 2023-09-23T20:53:15.897 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:53:15 smithi026 ceph-mon[175151]: failed to encode map e117 with expected crc 2023-09-23T20:53:15.939 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:53:16.088 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:53:16.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:53:15 smithi070 ceph-mon[168282]: pgmap v279: 89 pgs: 89 active+clean; 4.6 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 61 MiB/s wr, 972 op/s 2023-09-23T20:53:16.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:53:15 smithi070 ceph-mon[168282]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T20:53:16.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:53:15 smithi070 ceph-mon[168282]: Cluster is now healthy 2023-09-23T20:53:16.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:53:15 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T20:53:16.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:53:15 smithi070 ceph-mon[168282]: osdmap e117: 8 total, 8 up, 7 in 2023-09-23T20:53:16.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:53:15 smithi070 ceph-mon[168282]: failed to encode map e117 with expected crc 2023-09-23T20:53:16.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:53:15 smithi070 ceph-mon[168282]: failed to encode map e117 with expected crc 2023-09-23T20:53:16.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:53:15 smithi070 ceph-mon[168282]: failed to encode map e117 with expected crc 2023-09-23T20:53:16.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:53:15 smithi070 ceph-mon[168282]: failed to encode map e117 with expected crc 2023-09-23T20:53:16.190 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:53:16.195 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:53:16.195 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:53:16.296 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:53:16.335 INFO:teuthology.orchestra.run.smithi026.stdout:{"pg_ready":true,"pg_map":{"version":280,"stamp":"2023-09-23T20:53:14.871820+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4950988704,"num_objects":75561,"num_object_clones":0,"num_object_copies":226683,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":75561,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":75837,"num_write_kb":4836813,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":15,"num_bytes_recovered":1771728,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":75753,"ondisk_log_size":75753,"up":267,"acting":267,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":267,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":14742368,"kb_used_data":14361384,"kb_used_omap":126,"kb_used_meta":380737,"kb_avail":641473696,"statfs":{"total":671965249536,"available":656869064704,"internally_reserved":0,"allocated":14706057216,"data_stored":14702514531,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":129413,"internal_metadata":389875323},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,5,1,1,1,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":7,"apply_latency_ms":7,"commit_latency_ns":7000000,"apply_latency_ns":7000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":597032960,"num_objects":9110,"num_object_clones":0,"num_object_copies":27330,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":9110,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9110,"num_write_kb":583040,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"8.856573"},"pg_stats":[{"pgid":"3.1a","version":"68'1","reported_seq":94,"reported_epoch":115,"state":"active+clean","last_fresh":"2023-09-23T20:53:03.998381+0000","last_change":"2023-09-23T20:50:37.985580+0000","last_active":"2023-09-23T20:53:03.998381+0000","last_peered":"2023-09-23T20:53:03.998381+0000","last_clean":"2023-09-23T20:53:03.998381+0000","last_became_active":"2023-09-23T20:50:37.985229+0000","last_became_peered":"2023-09-23T20:50:37.985229+0000","last_unstale":"2023-09-23T20:53:03.998381+0000","last_undegraded":"2023-09-23T20:53:03.998381+0000","last_fullsized":"2023-09-23T20:53:03.998381+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:52:50.591507+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":78,"reported_epoch":115,"state":"active+clean","last_fresh":"2023-09-23T20:53:04.003302+0000","last_change":"2023-09-23T20:51:10.835170+0000","last_active":"2023-09-23T20:53:04.003302+0000","last_peered":"2023-09-23T20:53:04.003302+0000","last_clean":"2023-09-23T20:53:04.003302+0000","last_became_active":"2023-09-23T20:51:10.834808+0000","last_became_peered":"2023-09-23T20:51:10.834808+0000","last_unstale":"2023-09-23T20:53:04.003302+0000","last_undegraded":"2023-09-23T20:53:04.003302+0000","last_fullsized":"2023-09-23T20:53:04.003302+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:51:25.153742+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":65,"reported_epoch":115,"state":"active+clean","last_fresh":"2023-09-23T20:53:04.027896+0000","last_change":"2023-09-23T20:51:10.833035+0000","last_active":"2023-09-23T20:53:04.027896+0000","last_peered":"2023-09-23T20:53:04.027896+0000","last_clean":"2023-09-23T20:53:04.027896+0000","last_became_active":"2023-09-23T20:51:10.832741+0000","last_became_peered":"2023-09-23T20:51:10.832741+0000","last_unstale":"2023-09-23T20:53:04.027896+0000","last_undegraded":"2023-09-23T20:53:04.027896+0000","last_fullsized":"2023-09-23T20:53:04.027896+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:53:16.471852+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":70,"reported_epoch":115,"state":"active+clean","last_fresh":"2023-09-23T20:53:04.008591+0000","last_change":"2023-09-23T20:51:10.841420+0000","last_active":"2023-09-23T20:53:04.008591+0000","last_peered":"2023-09-23T20:53:04.008591+0000","last_clean":"2023-09-23T20:53:04.008591+0000","last_became_active":"2023-09-23T20:51:10.841112+0000","last_became_peered":"2023-09-23T20:51:10.841112+0000","last_unstale":"2023-09-23T20:53:04.008591+0000","last_undegraded":"2023-09-23T20:53:04.008591+0000","last_fullsized":"2023-09-23T20:53:04.008591+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:44:02.236650+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":96,"reported_epoch":115,"state":"active+clean","last_fresh":"2023-09-23T20:53:04.002365+0000","last_change":"2023-09-23T20:51:10.833613+0000","last_active":"2023-09-23T20:53:04.002365+0000","last_peered":"2023-09-23T20:53:04.002365+0000","last_clean":"2023-09-23T20:53:04.002365+0000","last_became_active":"2023-09-23T20:51:10.833208+0000","last_became_peered":"2023-09-23T20:51:10.833208+0000","last_unstale":"2023-09-23T20:53:04.002365+0000","last_undegraded":"2023-09-23T20:53:04.002365+0000","last_fullsized":"2023-09-23T20:53:04.002365+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:51:28.246014+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":85,"reported_epoch":115,"state":"active+clean","last_fresh":"2023-09-23T20:53:04.000227+0000","last_change":"2023-09-23T20:52:12.862262+0000","last_active":"2023-09-23T20:53:04.000227+0000","last_peered":"2023-09-23T20:53:04.000227+0000","last_clean":"2023-09-23T20:53:04.000227+0000","last_became_active":"2023-09-23T20:52:12.861972+0000","last_became_peered":"2023-09-23T20:52:12.861972+0000","last_unstale":"2023-09-23T20:53:04.000227+0000","last_undegraded":"2023-09-23T20:53:04.000227+0000","last_fullsized":"2023-09-23T20:53:04.000227+0000","mapping_epoch":100,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":101,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:10:42.887918+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":79,"reported_epoch":115,"state":"active+clean","last_fresh":"2023-09-23T20:53:04.008882+0000","last_change":"2023-09-23T20:52:12.862076+0000","last_active":"2023-09-23T20:53:04.008882+0000","last_peered":"2023-09-23T20:53:04.008882+0000","last_clean":"2023-09-23T20:53:04.008882+0000","last_became_active":"2023-09-23T20:52:12.861753+0000","last_became_peered":"2023-09-23T20:52:12.861753+0000","last_unstale":"2023-09-23T20:53:04.008882+0000","last_undegraded":"2023-09-23T20:53:04.008882+0000","last_fullsized":"2023-09-23T20:53:04.008882+0000","mapping_epoch":100,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":101,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:15:39.708595+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,6],"acting":[0,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":78,"reported_epoch":115,"state":"active+clean","last_fresh":"2023-09-23T20:53:04.002452+0000","last_change":"2023-09-23T20:51:10.833424+0000","last_active":"2023-09-23T20:53:04.002452+0000","last_peered":"2023-09-23T20:53:04.002452+0000","last_clean":"2023-09-23T20:53:04.002452+0000","last_became_active":"2023-09-23T20:51:10.832808+0000","last_became_peered":"2023-09-23T20:51:10.832808+0000","last_unstale":"2023-09-23T20:53:04.002452+0000","last_undegraded":"2023-09-23T20:53:04.002452+0000","last_fullsized":"2023-09-23T20:53:04.002452+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:32:42.819583+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":68,"reported_epoch":115,"state":"active+clean","last_fresh":"2023-09-23T20:53:03.998597+0000","last_change":"2023-09-23T20:52:12.860277+0000","last_active":"2023-09-23T20:53:03.998597+0000","last_peered":"2023-09-23T20:53:03.998597+0000","last_clean":"2023-09-23T20:53:03.998597+0000","last_became_active":"2023-09-23T20:52:12.859866+0000","last_became_peered":"2023-09-23T20:52:12.859866+0000","last_unstale":"2023-09-23T20:53:03.998597+0000","last_undegraded":"2023-09-23T20:53:03.998597+0000","last_fullsized":"2023-09-23T20:53:03.998597+0000","mapping_epoch":100,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":101,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:07:45.523326+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,3],"acting":[7,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":39,"reported_epoch":115,"state":"active+clean","last_fresh":"2023-09-23T20:53:04.002239+0000","last_change":"2023-09-23T20:51:10.833790+0000","last_active":"2023-09-23T20:53:04.002239+0000","last_peered":"2023-09-23T20:53:04.002239+0000","last_clean":"2023-09-23T20:53:04.002239+0000","last_became_active":"2023-09-23T20:51:10.832804+0000","last_became_peered":"2023-09-23T20:51:10.832804+0000","last_unstale":"2023-09-23T20:53:04.002239+0000","last_undegraded":"2023-09-23T20:53:04.002239+0000","last_fullsized":"2023-09-23T20:53:04.002239+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:27:42.601674+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":79,"reported_epoch":115,"state":"active+clean","last_fresh":"2023-09-23T20:53:04.000433+0000","last_change":"2023-09-23T20:51:10.834618+0000","last_active":"2023-09-23T20:53:04.000433+0000","last_peered":"2023-09-23T20:53:04.000433+0000","last_clean":"2023-09-23T20:53:04.000433+0000","last_became_active":"2023-09-23T20:51:10.834267+0000","last_became_peered":"2023-09-23T20:51:10.834267+0000","last_unstale":"2023-09-23T20:53:04.000433+0000","last_undegraded":"2023-09-23T20:53:04.000433+0000","last_fullsized":"2023-09-23T20:53:04.000433+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:38:59.356003+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":65,"reported_epoch":115,"state":"active+clean","last_fresh":"2023-09-23T20:53:04.028516+0000","last_change":"2023-09-23T20:50:37.982103+0000","last_active":"2023-09-23T20:53:04.028516+0000","last_peered":"2023-09-23T20:53:04.028516+0000","last_clean":"2023-09-23T20:53:04.028516+0000","last_became_active":"2023-09-23T20:50:37.981791+0000","last_became_peered":"2023-09-23T20:50:37.981791+0000","last_unstale":"2023-09-23T20:53:04.028516+0000","last_undegraded":"2023-09-23T20:53:04.028516+0000","last_fullsized":"2023-09-23T20:53:04.028516+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:32:52.286186+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"79'22","reported_seq":81,"reported_epoch":115,"state":"active+clean","last_fresh":"2023-09-23T20:53:03.999190+0000","last_change":"2023-09-23T20:41:58.905866+0000","last_active":"2023-09-23T20:53:03.999190+0000","last_peered":"2023-09-23T20:53:03.999190+0000","last_clean":"2023-09-23T20:53:03.999190+0000","last_became_active":"2023-09-23T20:41:58.905556+0000","last_became_peered":"2023-09-23T20:41:58.905556+0000","last_unstale":"2023-09-23T20:53:03.999190+0000","last_undegraded":"2023-09-23T20:53:03.999190+0000","last_fullsized":"2023-09-23T20:53:03.999190+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:03:14.027348+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":75,"reported_epoch":115,"state":"active+clean","last_fresh":"2023-09-23T20:53:03.998923+0000","last_change":"2023-09-23T20:50:05.871199+0000","last_active":"2023-09-23T20:53:03.998923+0000","last_peered":"2023-09-23T20:53:03.998923+0000","last_clean":"2023-09-23T20:53:03.998923+0000","last_became_active":"2023-09-23T20:50:05.870912+0000","last_became_peered":"2023-09-23T20:50:05.870912+0000","last_unstale":"2023-09-23T20:53:03.998923+0000","last_undegraded":"2023-09-23T20:53:03.998923+0000","last_fullsized":"2023-09-23T20:53:03.998923+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:16:22.718106+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":78,"reported_epoch":115,"state":"active+clean","last_fresh":"2023-09-23T20:53:03.999111+0000","last_change":"2023-09-23T20:50:37.986729+0000","last_active":"2023-09-23T20:53:03.999111+0000","last_peered":"2023-09-23T20:53:03.999111+0000","last_clean":"2023-09-23T20:53:03.999111+0000","last_became_active":"2023-09-23T20:50:37.986433+0000","last_became_peered":"2023-09-23T20:50:37.986433+0000","last_unstale":"2023-09-23T20:53:03.999111+0000","last_undegraded":"2023-09-23T20:53:03.999111+0000","last_fullsized":"2023-09-23T20:53:03.999111+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:28:28.853091+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":78,"reported_epoch":115,"state":"active+clean","last_fresh":"2023-09-23T20:53:04.004807+0000","last_change":"2023-09-23T20:51:10.831262+0000","last_active":"2023-09-23T20:53:04.004807+0000","last_peered":"2023-09-23T20:53:04.004807+0000","last_clean":"2023-09-23T20:53:04.004807+0000","last_became_active":"2023-09-23T20:51:10.830931+0000","last_became_peered":"2023-09-23T20:51:10.830931+0000","last_unstale":"2023-09-23T20:53:04.004807+0000","last_undegraded":"2023-09-23T20:53:04.004807+0000","last_fullsized":"2023-09-23T20:53:04.004807+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:51:54.982505+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":70,"reported_epoch":115,"state":"active+clean","last_fresh":"2023-09-23T20:53:04.008670+0000","last_change":"2023-09-23T20:51:10.840590+0000","last_active":"2023-09-23T20:53:04.008670+0000","last_peered":"2023-09-23T20:53:04.008670+0000","last_clean":"2023-09-23T20:53:04.008670+0000","last_became_active":"2023-09-23T20:51:10.840221+0000","last_became_peered":"2023-09-23T20:51:10.840221+0000","last_unstale":"2023-09-23T20:53:04.008670+0000","last_undegraded":"2023-09-23T20:53:04.008670+0000","last_fullsized":"2023-09-23T20:53:04.008670+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:03:39.138638+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":77,"reported_epoch":115,"state":"active+clean","last_fresh":"2023-09-23T20:53:04.000281+0000","last_change":"2023-09-23T20:52:12.860214+0000","last_active":"2023-09-23T20:53:04.000281+0000","last_peered":"2023-09-23T20:53:04.000281+0000","last_clean":"2023-09-23T20:53:04.000281+0000","last_became_active":"2023-09-23T20:52:12.859837+0000","last_became_peered":"2023-09-23T20:52:12.859837+0000","last_unstale":"2023-09-23T20:53:04.000281+0000","last_undegraded":"2023-09-23T20:53:04.000281+0000","last_fullsized":"2023-09-23T20:53:04.000281+0000","mapping_epoch":100,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":101,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:01:38.149335+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,6],"acting":[7,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":59,"reported_epoch":115,"state":"active+clean","last_fresh":"2023-09-23T20:53:03.997917+0000","last_change":"2023-09-23T20:41:58.898224+0000","last_active":"2023-09-23T20:53:03.997917+0000","last_peered":"2023-09-23T20:53:03.997917+0000","last_clean":"2023-09-23T20:53:03.997917+0000","last_became_active":"2023-09-23T20:41:58.898036+0000","last_became_peered":"2023-09-23T20:41:58.898036+0000","last_unstale":"2023-09-23T20:53:03.997917+0000","last_undegraded":"2023-09-23T20:53:03.997917+0000","last_fullsized":"2023-09-23T20:53:03.997917+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:53:15.361560+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":70,"reported_epoch":115,"state":"active+clean","last_fresh":"2023-09-23T20:53:04.008544+0000","last_change":"2023-09-23T20:51:10.840849+0000","last_active":"2023-09-23T20:53:04.008544+0000","last_peered":"2023-09-23T20:53:04.008544+0000","last_clean":"2023-09-23T20:53:04.008544+0000","last_became_active":"2023-09-23T20:51:10.840593+0000","last_became_peered":"2023-09-23T20:51:10.840593+0000","last_unstale":"2023-09-23T20:53:04.008544+0000","last_undegraded":"2023-09-23T20:53:04.008544+0000","last_fullsized":"2023-09-23T20:53:04.008544+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:30:52.297625+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":104,"reported_epoch":115,"state":"active+clean","last_fresh":"2023-09-23T20:53:03.999734+0000","last_change":"2023-09-23T20:52:12.870672+0000","last_active":"2023-09-23T20:53:03.999734+0000","last_peered":"2023-09-23T20:53:03.999734+0000","last_clean":"2023-09-23T20:53:03.999734+0000","last_became_active":"2023-09-23T20:52:12.862084+0000","last_became_peered":"2023-09-23T20:52:12.862084+0000","last_unstale":"2023-09-23T20:53:03.999734+0000","last_undegraded":"2023-09-23T20:53:03.999734+0000","last_fullsized":"2023-09-23T20:53:03.999734+0000","mapping_epoch":100,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":101,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:32:58.322893+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":78,"reported_epoch":115,"state":"active+clean","last_fresh":"2023-09-23T20:53:04.003236+0000","last_change":"2023-09-23T20:51:10.833234+0000","last_active":"2023-09-23T20:53:04.003236+0000","last_peered":"2023-09-23T20:53:04.003236+0000","last_clean":"2023-09-23T20:53:04.003236+0000","last_became_active":"2023-09-23T20:51:10.832763+0000","last_became_peered":"2023-09-23T20:51:10.832763+0000","last_unstale":"2023-09-23T20:53:04.003236+0000","last_undegraded":"2023-09-23T20:53:04.003236+0000","last_fullsized":"2023-09-23T20:53:04.003236+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:24:08.283578+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.a","version":"116'4747","reported_seq":4769,"reported_epoch":116,"state":"active+clean","last_fresh":"2023-09-23T20:53:12.060600+0000","last_change":"2023-09-23T20:52:14.865442+0000","last_active":"2023-09-23T20:53:12.060600+0000","last_peered":"2023-09-23T20:53:12.060600+0000","last_clean":"2023-09-23T20:53:12.060600+0000","last_became_active":"2023-09-23T20:52:14.865018+0000","last_became_peered":"2023-09-23T20:52:14.865018+0000","last_unstale":"2023-09-23T20:53:12.060600+0000","last_undegraded":"2023-09-23T20:53:12.060600+0000","last_fullsized":"2023-09-23T20:53:12.060600+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":102,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:52:13.843048+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:52:13.843048+0000","last_clean_scrub_stamp":"2023-09-23T20:52:13.843048+0000","objects_scrubbed":0,"log_size":4747,"log_dups_size":0,"ondisk_log_size":4747,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:32:46.511747+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":311099392,"num_objects":4747,"num_object_clones":0,"num_object_copies":14241,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4747,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4747,"num_write_kb":303808,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,2],"acting":[6,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":77,"reported_epoch":115,"state":"active+clean","last_fresh":"2023-09-23T20:53:03.998285+0000","last_change":"2023-09-23T20:49:32.777943+0000","last_active":"2023-09-23T20:53:03.998285+0000","last_peered":"2023-09-23T20:53:03.998285+0000","last_clean":"2023-09-23T20:53:03.998285+0000","last_became_active":"2023-09-23T20:49:32.776212+0000","last_became_peered":"2023-09-23T20:49:32.776212+0000","last_unstale":"2023-09-23T20:53:03.998285+0000","last_undegraded":"2023-09-23T20:53:03.998285+0000","last_fullsized":"2023-09-23T20:53:03.998285+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:21:05.725926+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":28,"reported_epoch":115,"state":"active+clean","last_fresh":"2023-09-23T20:53:03.997784+0000","last_change":"2023-09-23T20:52:12.861197+0000","last_active":"2023-09-23T20:53:03.997784+0000","last_peered":"2023-09-23T20:53:03.997784+0000","last_clean":"2023-09-23T20:53:03.997784+0000","last_became_active":"2023-09-23T20:52:12.860833+0000","last_became_peered":"2023-09-23T20:52:12.860833+0000","last_unstale":"2023-09-23T20:53:03.997784+0000","last_undegraded":"2023-09-23T20:53:03.997784+0000","last_fullsized":"2023-09-23T20:53:03.997784+0000","mapping_epoch":100,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":101,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:48:25.460380+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,4],"acting":[7,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"6.b","version":"116'4665","reported_seq":4688,"reported_epoch":116,"state":"active+clean","last_fresh":"2023-09-23T20:53:13.792420+0000","last_change":"2023-09-23T20:52:14.868057+0000","last_active":"2023-09-23T20:53:13.792420+0000","last_peered":"2023-09-23T20:53:13.792420+0000","last_clean":"2023-09-23T20:53:13.792420+0000","last_became_active":"2023-09-23T20:52:14.867734+0000","last_became_peered":"2023-09-23T20:52:14.867734+0000","last_unstale":"2023-09-23T20:53:13.792420+0000","last_undegraded":"2023-09-23T20:53:13.792420+0000","last_fullsized":"2023-09-23T20:53:13.792420+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":102,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:52:13.843048+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:52:13.843048+0000","last_clean_scrub_stamp":"2023-09-23T20:52:13.843048+0000","objects_scrubbed":0,"log_size":4665,"log_dups_size":0,"ondisk_log_size":4665,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:40:31.506977+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":305725440,"num_objects":4665,"num_object_clones":0,"num_object_copies":13995,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4665,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4665,"num_write_kb":298560,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":78,"reported_epoch":115,"state":"active+clean","last_fresh":"2023-09-23T20:53:03.998470+0000","last_change":"2023-09-23T20:50:05.866983+0000","last_active":"2023-09-23T20:53:03.998470+0000","last_peered":"2023-09-23T20:53:03.998470+0000","last_clean":"2023-09-23T20:53:03.998470+0000","last_became_active":"2023-09-23T20:50:05.866601+0000","last_became_peered":"2023-09-23T20:50:05.866601+0000","last_unstale":"2023-09-23T20:53:03.998470+0000","last_undegraded":"2023-09-23T20:53:03.998470+0000","last_fullsized":"2023-09-23T20:53:03.998470+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:15:00.955166+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":92,"reported_epoch":115,"state":"active+clean","last_fresh":"2023-09-23T20:53:03.998392+0000","last_change":"2023-09-23T20:51:10.838338+0000","last_active":"2023-09-23T20:53:03.998392+0000","last_peered":"2023-09-23T20:53:03.998392+0000","last_clean":"2023-09-23T20:53:03.998392+0000","last_became_active":"2023-09-23T20:51:10.838049+0000","last_became_peered":"2023-09-23T20:51:10.838049+0000","last_unstale":"2023-09-23T20:53:03.998392+0000","last_undegraded":"2023-09-23T20:53:03.998392+0000","last_fullsized":"2023-09-23T20:53:03.998392+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:40:23.602402+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.8","version":"116'4705","reported_seq":4727,"reported_epoch":116,"state":"active+clean","last_fresh":"2023-09-23T20:53:12.029025+0000","last_change":"2023-09-23T20:52:14.869562+0000","last_active":"2023-09-23T20:53:12.029025+0000","last_peered":"2023-09-23T20:53:12.029025+0000","last_clean":"2023-09-23T20:53:12.029025+0000","last_became_active":"2023-09-23T20:52:14.869204+0000","last_became_peered":"2023-09-23T20:52:14.869204+0000","last_unstale":"2023-09-23T20:53:12.029025+0000","last_undegraded":"2023-09-23T20:53:12.029025+0000","last_fullsized":"2023-09-23T20:53:12.029025+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":102,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:52:13.843048+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:52:13.843048+0000","last_clean_scrub_stamp":"2023-09-23T20:52:13.843048+0000","objects_scrubbed":0,"log_size":4705,"log_dups_size":0,"ondisk_log_size":4705,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:32:14.355022+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":308346880,"num_objects":4705,"num_object_clones":0,"num_object_copies":14115,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4705,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4705,"num_write_kb":301120,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,3],"acting":[7,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":74,"reported_epoch":115,"state":"active+clean","last_fresh":"2023-09-23T20:53:04.000577+0000","last_change":"2023-09-23T20:52:12.866062+0000","last_active":"2023-09-23T20:53:04.000577+0000","last_peered":"2023-09-23T20:53:04.000577+0000","last_clean":"2023-09-23T20:53:04.000577+0000","last_became_active":"2023-09-23T20:52:12.861064+0000","last_became_peered":"2023-09-23T20:52:12.861064+0000","last_unstale":"2023-09-23T20:53:04.000577+0000","last_undegraded":"2023-09-23T20:53:04.000577+0000","last_fullsized":"2023-09-23T20:53:04.000577+0000","mapping_epoch":100,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":101,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:42:14.367193+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,3],"acting":[7,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":75,"reported_epoch":115,"state":"active+clean","last_fresh":"2023-09-23T20:53:04.000251+0000","last_change":"2023-09-23T20:50:05.867973+0000","last_active":"2023-09-23T20:53:04.000251+0000","last_peered":"2023-09-23T20:53:04.000251+0000","last_clean":"2023-09-23T20:53:04.000251+0000","last_became_active":"2023-09-23T20:50:05.867622+0000","last_became_peered":"2023-09-23T20:50:05.867622+0000","last_unstale":"2023-09-23T20:53:04.000251+0000","last_undegraded":"2023-09-23T20:53:04.000251+0000","last_fullsized":"2023-09-23T20:53:04.000251+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:10:12.617597+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.9","version":"116'4624","reported_seq":4647,"reported_epoch":116,"state":"active+clean","last_fresh":"2023-09-23T20:53:10.797917+0000","last_change":"2023-09-23T20:52:14.864297+0000","last_active":"2023-09-23T20:53:10.797917+0000","last_peered":"2023-09-23T20:53:10.797917+0000","last_clean":"2023-09-23T20:53:10.797917+0000","last_became_active":"2023-09-23T20:52:14.863986+0000","last_became_peered":"2023-09-23T20:52:14.863986+0000","last_unstale":"2023-09-23T20:53:10.797917+0000","last_undegraded":"2023-09-23T20:53:10.797917+0000","last_fullsized":"2023-09-23T20:53:10.797917+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":102,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:52:13.843048+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:52:13.843048+0000","last_clean_scrub_stamp":"2023-09-23T20:52:13.843048+0000","objects_scrubbed":0,"log_size":4624,"log_dups_size":0,"ondisk_log_size":4624,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:37:59.995892+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":303038464,"num_objects":4624,"num_object_clones":0,"num_object_copies":13872,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4624,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4624,"num_write_kb":295936,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,2],"acting":[0,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":77,"reported_epoch":115,"state":"active+clean","last_fresh":"2023-09-23T20:53:04.004997+0000","last_change":"2023-09-23T20:52:12.858198+0000","last_active":"2023-09-23T20:53:04.004997+0000","last_peered":"2023-09-23T20:53:04.004997+0000","last_clean":"2023-09-23T20:53:04.004997+0000","last_became_active":"2023-09-23T20:52:12.857937+0000","last_became_peered":"2023-09-23T20:52:12.857937+0000","last_unstale":"2023-09-23T20:53:04.004997+0000","last_undegraded":"2023-09-23T20:53:04.004997+0000","last_fullsized":"2023-09-23T20:53:04.004997+0000","mapping_epoch":100,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":101,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:18:42.976887+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":83,"reported_epoch":115,"state":"active+clean","last_fresh":"2023-09-23T20:53:04.007911+0000","last_change":"2023-09-23T20:50:37.982263+0000","last_active":"2023-09-23T20:53:04.007911+0000","last_peered":"2023-09-23T20:53:04.007911+0000","last_clean":"2023-09-23T20:53:04.007911+0000","last_became_active":"2023-09-23T20:50:37.981797+0000","last_became_peered":"2023-09-23T20:50:37.981797+0000","last_unstale":"2023-09-23T20:53:04.007911+0000","last_undegraded":"2023-09-23T20:53:04.007911+0000","last_fullsized":"2023-09-23T20:53:04.007911+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:56:45.027447+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.e","version":"116'4746","reported_seq":4768,"reported_epoch":116,"state":"active+clean","last_fresh":"2023-09-23T20:53:12.058875+0000","last_change":"2023-09-23T20:52:14.862981+0000","last_active":"2023-09-23T20:53:12.058875+0000","last_peered":"2023-09-23T20:53:12.058875+0000","last_clean":"2023-09-23T20:53:12.058875+0000","last_became_active":"2023-09-23T20:52:14.862783+0000","last_became_peered":"2023-09-23T20:52:14.862783+0000","last_unstale":"2023-09-23T20:53:12.058875+0000","last_undegraded":"2023-09-23T20:53:12.058875+0000","last_fullsized":"2023-09-23T20:53:12.058875+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":102,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:52:13.843048+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:52:13.843048+0000","last_clean_scrub_stamp":"2023-09-23T20:52:13.843048+0000","objects_scrubbed":0,"log_size":4746,"log_dups_size":0,"ondisk_log_size":4746,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:39:24.557635+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":311033856,"num_objects":4746,"num_object_clones":0,"num_object_copies":14238,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4746,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4746,"num_write_kb":303744,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":76,"reported_epoch":115,"state":"active+clean","last_fresh":"2023-09-23T20:53:04.002495+0000","last_change":"2023-09-23T20:51:10.836752+0000","last_active":"2023-09-23T20:53:04.002495+0000","last_peered":"2023-09-23T20:53:04.002495+0000","last_clean":"2023-09-23T20:53:04.002495+0000","last_became_active":"2023-09-23T20:51:10.836213+0000","last_became_peered":"2023-09-23T20:51:10.836213+0000","last_unstale":"2023-09-23T20:53:04.002495+0000","last_undegraded":"2023-09-23T20:53:04.002495+0000","last_fullsized":"2023-09-23T20:53:04.002495+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:35:01.825635+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":75,"reported_epoch":115,"state":"active+clean","last_fresh":"2023-09-23T20:53:03.999259+0000","last_change":"2023-09-23T20:51:10.833630+0000","last_active":"2023-09-23T20:53:03.999259+0000","last_peered":"2023-09-23T20:53:03.999259+0000","last_clean":"2023-09-23T20:53:03.999259+0000","last_became_active":"2023-09-23T20:51:10.833265+0000","last_became_peered":"2023-09-23T20:51:10.833265+0000","last_unstale":"2023-09-23T20:53:03.999259+0000","last_undegraded":"2023-09-23T20:53:03.999259+0000","last_fullsized":"2023-09-23T20:53:03.999259+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:12:01.557967+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.f","version":"116'4581","reported_seq":4603,"reported_epoch":116,"state":"active+clean","last_fresh":"2023-09-23T20:53:11.015348+0000","last_change":"2023-09-23T20:52:14.863531+0000","last_active":"2023-09-23T20:53:11.015348+0000","last_peered":"2023-09-23T20:53:11.015348+0000","last_clean":"2023-09-23T20:53:11.015348+0000","last_became_active":"2023-09-23T20:52:14.862972+0000","last_became_peered":"2023-09-23T20:52:14.862972+0000","last_unstale":"2023-09-23T20:53:11.015348+0000","last_undegraded":"2023-09-23T20:53:11.015348+0000","last_fullsized":"2023-09-23T20:53:11.015348+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":102,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:52:13.843048+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:52:13.843048+0000","last_clean_scrub_stamp":"2023-09-23T20:52:13.843048+0000","objects_scrubbed":0,"log_size":4581,"log_dups_size":0,"ondisk_log_size":4581,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:23:46.230633+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":300220416,"num_objects":4581,"num_object_clones":0,"num_object_copies":13743,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4581,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4581,"num_write_kb":293184,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":76,"reported_epoch":115,"state":"active+clean","last_fresh":"2023-09-23T20:53:04.000328+0000","last_change":"2023-09-23T20:50:05.867646+0000","last_active":"2023-09-23T20:53:04.000328+0000","last_peered":"2023-09-23T20:53:04.000328+0000","last_clean":"2023-09-23T20:53:04.000328+0000","last_became_active":"2023-09-23T20:50:05.867343+0000","last_became_peered":"2023-09-23T20:50:05.867343+0000","last_unstale":"2023-09-23T20:53:04.000328+0000","last_undegraded":"2023-09-23T20:53:04.000328+0000","last_fullsized":"2023-09-23T20:53:04.000328+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:53:34.985580+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":92,"reported_epoch":115,"state":"active+clean","last_fresh":"2023-09-23T20:53:03.998686+0000","last_change":"2023-09-23T20:50:37.985593+0000","last_active":"2023-09-23T20:53:03.998686+0000","last_peered":"2023-09-23T20:53:03.998686+0000","last_clean":"2023-09-23T20:53:03.998686+0000","last_became_active":"2023-09-23T20:50:37.985324+0000","last_became_peered":"2023-09-23T20:50:37.985324+0000","last_unstale":"2023-09-23T20:53:03.998686+0000","last_undegraded":"2023-09-23T20:53:03.998686+0000","last_fullsized":"2023-09-23T20:53:03.998686+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:34:07.801771+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.c","version":"116'4728","reported_seq":4751,"reported_epoch":116,"state":"active+clean","last_fresh":"2023-09-23T20:53:13.847808+0000","last_change":"2023-09-23T20:52:14.865953+0000","last_active":"2023-09-23T20:53:13.847808+0000","last_peered":"2023-09-23T20:53:13.847808+0000","last_clean":"2023-09-23T20:53:13.847808+0000","last_became_active":"2023-09-23T20:52:14.865335+0000","last_became_peered":"2023-09-23T20:52:14.865335+0000","last_unstale":"2023-09-23T20:53:13.847808+0000","last_undegraded":"2023-09-23T20:53:13.847808+0000","last_fullsized":"2023-09-23T20:53:13.847808+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":102,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:52:13.843048+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:52:13.843048+0000","last_clean_scrub_stamp":"2023-09-23T20:52:13.843048+0000","objects_scrubbed":0,"log_size":4728,"log_dups_size":0,"ondisk_log_size":4728,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:20:00.174949+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":309854208,"num_objects":4728,"num_object_clones":0,"num_object_copies":14184,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4728,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4728,"num_write_kb":302592,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":76,"reported_epoch":115,"state":"active+clean","last_fresh":"2023-09-23T20:53:03.998124+0000","last_change":"2023-09-23T20:50:37.983914+0000","last_active":"2023-09-23T20:53:03.998124+0000","last_peered":"2023-09-23T20:53:03.998124+0000","last_clean":"2023-09-23T20:53:03.998124+0000","last_became_active":"2023-09-23T20:50:37.983617+0000","last_became_peered":"2023-09-23T20:50:37.983617+0000","last_unstale":"2023-09-23T20:53:03.998124+0000","last_undegraded":"2023-09-23T20:53:03.998124+0000","last_fullsized":"2023-09-23T20:53:03.998124+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:15:50.863306+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":75,"reported_epoch":115,"state":"active+clean","last_fresh":"2023-09-23T20:53:03.998341+0000","last_change":"2023-09-23T20:50:05.871406+0000","last_active":"2023-09-23T20:53:03.998341+0000","last_peered":"2023-09-23T20:53:03.998341+0000","last_clean":"2023-09-23T20:53:03.998341+0000","last_became_active":"2023-09-23T20:50:05.871127+0000","last_became_peered":"2023-09-23T20:50:05.871127+0000","last_unstale":"2023-09-23T20:53:03.998341+0000","last_undegraded":"2023-09-23T20:53:03.998341+0000","last_fullsized":"2023-09-23T20:53:03.998341+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:37:48.850229+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.d","version":"116'4830","reported_seq":4853,"reported_epoch":116,"state":"active+clean","last_fresh":"2023-09-23T20:53:13.874586+0000","last_change":"2023-09-23T20:52:14.867585+0000","last_active":"2023-09-23T20:53:13.874586+0000","last_peered":"2023-09-23T20:53:13.874586+0000","last_clean":"2023-09-23T20:53:13.874586+0000","last_became_active":"2023-09-23T20:52:14.867310+0000","last_became_peered":"2023-09-23T20:52:14.867310+0000","last_unstale":"2023-09-23T20:53:13.874586+0000","last_undegraded":"2023-09-23T20:53:13.874586+0000","last_fullsized":"2023-09-23T20:53:13.874586+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":102,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:52:13.843048+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:52:13.843048+0000","last_clean_scrub_stamp":"2023-09-23T20:52:13.843048+0000","objects_scrubbed":0,"log_size":4830,"log_dups_size":0,"ondisk_log_size":4830,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:19:26.814547+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":316538880,"num_objects":4830,"num_object_clones":0,"num_object_copies":14490,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4830,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4830,"num_write_kb":309120,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,2],"acting":[1,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":79,"reported_epoch":115,"state":"active+clean","last_fresh":"2023-09-23T20:53:04.002171+0000","last_change":"2023-09-23T20:51:10.831197+0000","last_active":"2023-09-23T20:53:04.002171+0000","last_peered":"2023-09-23T20:53:04.002171+0000","last_clean":"2023-09-23T20:53:04.002171+0000","last_became_active":"2023-09-23T20:51:10.830878+0000","last_became_peered":"2023-09-23T20:51:10.830878+0000","last_unstale":"2023-09-23T20:53:04.002171+0000","last_undegraded":"2023-09-23T20:53:04.002171+0000","last_fullsized":"2023-09-23T20:53:04.002171+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:05:24.314478+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":65,"reported_epoch":115,"state":"active+clean","last_fresh":"2023-09-23T20:53:04.029093+0000","last_change":"2023-09-23T20:51:10.833850+0000","last_active":"2023-09-23T20:53:04.029093+0000","last_peered":"2023-09-23T20:53:04.029093+0000","last_clean":"2023-09-23T20:53:04.029093+0000","last_became_active":"2023-09-23T20:51:10.833539+0000","last_became_peered":"2023-09-23T20:51:10.833539+0000","last_unstale":"2023-09-23T20:53:04.029093+0000","last_undegraded":"2023-09-23T20:53:04.029093+0000","last_fullsized":"2023-09-23T20:53:04.029093+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:01:36.445616+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":88,"reported_epoch":115,"state":"active+clean","last_fresh":"2023-09-23T20:53:04.028422+0000","last_change":"2023-09-23T20:50:05.867130+0000","last_active":"2023-09-23T20:53:04.028422+0000","last_peered":"2023-09-23T20:53:04.028422+0000","last_clean":"2023-09-23T20:53:04.028422+0000","last_became_active":"2023-09-23T20:50:05.866758+0000","last_became_peered":"2023-09-23T20:50:05.866758+0000","last_unstale":"2023-09-23T20:53:04.028422+0000","last_undegraded":"2023-09-23T20:53:04.028422+0000","last_fullsized":"2023-09-23T20:53:04.028422+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:55:07.548209+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":94,"reported_epoch":115,"state":"active+clean","last_fresh":"2023-09-23T20:53:04.003162+0000","last_change":"2023-09-23T20:52:12.857372+0000","last_active":"2023-09-23T20:53:04.003162+0000","last_peered":"2023-09-23T20:53:04.003162+0000","last_clean":"2023-09-23T20:53:04.003162+0000","last_became_active":"2023-09-23T20:52:12.857135+0000","last_became_peered":"2023-09-23T20:52:12.857135+0000","last_unstale":"2023-09-23T20:53:04.003162+0000","last_undegraded":"2023-09-23T20:53:04.003162+0000","last_fullsized":"2023-09-23T20:53:04.003162+0000","mapping_epoch":100,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":101,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:50:17.844004+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":59,"reported_epoch":115,"state":"active+clean","last_fresh":"2023-09-23T20:53:04.028434+0000","last_change":"2023-09-23T20:52:12.854192+0000","last_active":"2023-09-23T20:53:04.028434+0000","last_peered":"2023-09-23T20:53:04.028434+0000","last_clean":"2023-09-23T20:53:04.028434+0000","last_became_active":"2023-09-23T20:52:12.853812+0000","last_became_peered":"2023-09-23T20:52:12.853812+0000","last_unstale":"2023-09-23T20:53:04.028434+0000","last_undegraded":"2023-09-23T20:53:04.028434+0000","last_fullsized":"2023-09-23T20:53:04.028434+0000","mapping_epoch":100,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":101,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:36:13.376752+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.0","version":"116'4669","reported_seq":4692,"reported_epoch":116,"state":"active+clean","last_fresh":"2023-09-23T20:53:10.778982+0000","last_change":"2023-09-23T20:52:14.866690+0000","last_active":"2023-09-23T20:53:10.778982+0000","last_peered":"2023-09-23T20:53:10.778982+0000","last_clean":"2023-09-23T20:53:10.778982+0000","last_became_active":"2023-09-23T20:52:14.866408+0000","last_became_peered":"2023-09-23T20:52:14.866408+0000","last_unstale":"2023-09-23T20:53:10.778982+0000","last_undegraded":"2023-09-23T20:53:10.778982+0000","last_fullsized":"2023-09-23T20:53:10.778982+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":102,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:52:13.843048+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:52:13.843048+0000","last_clean_scrub_stamp":"2023-09-23T20:52:13.843048+0000","objects_scrubbed":0,"log_size":4669,"log_dups_size":0,"ondisk_log_size":4669,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:29:07.905971+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":305987584,"num_objects":4669,"num_object_clones":0,"num_object_copies":14007,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4669,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4669,"num_write_kb":298816,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.6","version":"116'4727","reported_seq":4750,"reported_epoch":116,"state":"active+clean","last_fresh":"2023-09-23T20:53:13.862305+0000","last_change":"2023-09-23T20:52:14.865161+0000","last_active":"2023-09-23T20:53:13.862305+0000","last_peered":"2023-09-23T20:53:13.862305+0000","last_clean":"2023-09-23T20:53:13.862305+0000","last_became_active":"2023-09-23T20:52:14.864806+0000","last_became_peered":"2023-09-23T20:52:14.864806+0000","last_unstale":"2023-09-23T20:53:13.862305+0000","last_undegraded":"2023-09-23T20:53:13.862305+0000","last_fullsized":"2023-09-23T20:53:13.862305+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":102,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:52:13.843048+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:52:13.843048+0000","last_clean_scrub_stamp":"2023-09-23T20:52:13.843048+0000","objects_scrubbed":0,"log_size":4727,"log_dups_size":0,"ondisk_log_size":4727,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:05:57.391225+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":309788672,"num_objects":4727,"num_object_clones":0,"num_object_copies":14181,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4727,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4727,"num_write_kb":302528,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":71,"reported_epoch":115,"state":"active+clean","last_fresh":"2023-09-23T20:53:04.028290+0000","last_change":"2023-09-23T20:52:12.860753+0000","last_active":"2023-09-23T20:53:04.028290+0000","last_peered":"2023-09-23T20:53:04.028290+0000","last_clean":"2023-09-23T20:53:04.028290+0000","last_became_active":"2023-09-23T20:52:12.860394+0000","last_became_peered":"2023-09-23T20:52:12.860394+0000","last_unstale":"2023-09-23T20:53:04.028290+0000","last_undegraded":"2023-09-23T20:53:04.028290+0000","last_fullsized":"2023-09-23T20:53:04.028290+0000","mapping_epoch":100,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":101,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:44:12.690612+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":76,"reported_epoch":115,"state":"active+clean","last_fresh":"2023-09-23T20:53:03.998802+0000","last_change":"2023-09-23T20:49:32.771117+0000","last_active":"2023-09-23T20:53:03.998802+0000","last_peered":"2023-09-23T20:53:03.998802+0000","last_clean":"2023-09-23T20:53:03.998802+0000","last_became_active":"2023-09-23T20:49:32.770686+0000","last_became_peered":"2023-09-23T20:49:32.770686+0000","last_unstale":"2023-09-23T20:53:03.998802+0000","last_undegraded":"2023-09-23T20:53:03.998802+0000","last_fullsized":"2023-09-23T20:53:03.998802+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:15:46.598698+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":80,"reported_epoch":115,"state":"active+clean","last_fresh":"2023-09-23T20:53:04.028200+0000","last_change":"2023-09-23T20:51:10.829889+0000","last_active":"2023-09-23T20:53:04.028200+0000","last_peered":"2023-09-23T20:53:04.028200+0000","last_clean":"2023-09-23T20:53:04.028200+0000","last_became_active":"2023-09-23T20:51:10.829533+0000","last_became_peered":"2023-09-23T20:51:10.829533+0000","last_unstale":"2023-09-23T20:53:04.028200+0000","last_undegraded":"2023-09-23T20:53:04.028200+0000","last_fullsized":"2023-09-23T20:53:04.028200+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:43:04.603460+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.5","version":"116'4641","reported_seq":4662,"reported_epoch":116,"state":"active+clean","last_fresh":"2023-09-23T20:53:12.087717+0000","last_change":"2023-09-23T20:52:14.869287+0000","last_active":"2023-09-23T20:53:12.087717+0000","last_peered":"2023-09-23T20:53:12.087717+0000","last_clean":"2023-09-23T20:53:12.087717+0000","last_became_active":"2023-09-23T20:52:14.868955+0000","last_became_peered":"2023-09-23T20:52:14.868955+0000","last_unstale":"2023-09-23T20:53:12.087717+0000","last_undegraded":"2023-09-23T20:53:12.087717+0000","last_fullsized":"2023-09-23T20:53:12.087717+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":102,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:52:13.843048+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:52:13.843048+0000","last_clean_scrub_stamp":"2023-09-23T20:52:13.843048+0000","objects_scrubbed":0,"log_size":4641,"log_dups_size":0,"ondisk_log_size":4641,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:05:41.908990+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":304152576,"num_objects":4641,"num_object_clones":0,"num_object_copies":13923,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4641,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4641,"num_write_kb":297024,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,3],"acting":[7,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":99,"reported_epoch":115,"state":"active+clean","last_fresh":"2023-09-23T20:53:04.005189+0000","last_change":"2023-09-23T20:51:10.838054+0000","last_active":"2023-09-23T20:53:04.005189+0000","last_peered":"2023-09-23T20:53:04.005189+0000","last_clean":"2023-09-23T20:53:04.005189+0000","last_became_active":"2023-09-23T20:51:10.837750+0000","last_became_peered":"2023-09-23T20:51:10.837750+0000","last_unstale":"2023-09-23T20:53:04.005189+0000","last_undegraded":"2023-09-23T20:53:04.005189+0000","last_fullsized":"2023-09-23T20:53:04.005189+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:15:51.648085+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":65,"reported_epoch":115,"state":"active+clean","last_fresh":"2023-09-23T20:53:04.028010+0000","last_change":"2023-09-23T20:50:37.983072+0000","last_active":"2023-09-23T20:53:04.028010+0000","last_peered":"2023-09-23T20:53:04.028010+0000","last_clean":"2023-09-23T20:53:04.028010+0000","last_became_active":"2023-09-23T20:50:37.982777+0000","last_became_peered":"2023-09-23T20:50:37.982777+0000","last_unstale":"2023-09-23T20:53:04.028010+0000","last_undegraded":"2023-09-23T20:53:04.028010+0000","last_fullsized":"2023-09-23T20:53:04.028010+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:58:11.972144+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":92,"reported_epoch":115,"state":"active+clean","last_fresh":"2023-09-23T20:53:04.029166+0000","last_change":"2023-09-23T20:52:12.864016+0000","last_active":"2023-09-23T20:53:04.029166+0000","last_peered":"2023-09-23T20:53:04.029166+0000","last_clean":"2023-09-23T20:53:04.029166+0000","last_became_active":"2023-09-23T20:52:12.863714+0000","last_became_peered":"2023-09-23T20:52:12.863714+0000","last_unstale":"2023-09-23T20:53:04.029166+0000","last_undegraded":"2023-09-23T20:53:04.029166+0000","last_fullsized":"2023-09-23T20:53:04.029166+0000","mapping_epoch":100,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":101,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:20:44.035420+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.7","version":"116'4852","reported_seq":4875,"reported_epoch":116,"state":"active+clean","last_fresh":"2023-09-23T20:53:13.849961+0000","last_change":"2023-09-23T20:52:14.867968+0000","last_active":"2023-09-23T20:53:13.849961+0000","last_peered":"2023-09-23T20:53:13.849961+0000","last_clean":"2023-09-23T20:53:13.849961+0000","last_became_active":"2023-09-23T20:52:14.867682+0000","last_became_peered":"2023-09-23T20:52:14.867682+0000","last_unstale":"2023-09-23T20:53:13.849961+0000","last_undegraded":"2023-09-23T20:53:13.849961+0000","last_fullsized":"2023-09-23T20:53:13.849961+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":102,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:52:13.843048+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:52:13.843048+0000","last_clean_scrub_stamp":"2023-09-23T20:52:13.843048+0000","objects_scrubbed":0,"log_size":4852,"log_dups_size":0,"ondisk_log_size":4852,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:24:49.928264+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":317980672,"num_objects":4852,"num_object_clones":0,"num_object_copies":14556,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4852,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4852,"num_write_kb":310528,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":76,"reported_epoch":115,"state":"active+clean","last_fresh":"2023-09-23T20:53:04.005565+0000","last_change":"2023-09-23T20:52:12.853881+0000","last_active":"2023-09-23T20:53:04.005565+0000","last_peered":"2023-09-23T20:53:04.005565+0000","last_clean":"2023-09-23T20:53:04.005565+0000","last_became_active":"2023-09-23T20:52:12.853545+0000","last_became_peered":"2023-09-23T20:52:12.853545+0000","last_unstale":"2023-09-23T20:53:04.005565+0000","last_undegraded":"2023-09-23T20:53:04.005565+0000","last_fullsized":"2023-09-23T20:53:04.005565+0000","mapping_epoch":100,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":101,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:18:05.760976+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,3],"acting":[2,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"79'161","reported_seq":457,"reported_epoch":115,"state":"active+clean","last_fresh":"2023-09-23T20:53:03.997842+0000","last_change":"2023-09-23T20:49:32.777246+0000","last_active":"2023-09-23T20:53:03.997842+0000","last_peered":"2023-09-23T20:53:03.997842+0000","last_clean":"2023-09-23T20:53:03.997842+0000","last_became_active":"2023-09-23T20:49:32.776822+0000","last_became_peered":"2023-09-23T20:49:32.776822+0000","last_unstale":"2023-09-23T20:53:03.997842+0000","last_undegraded":"2023-09-23T20:53:03.997842+0000","last_fullsized":"2023-09-23T20:53:03.997842+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:38:57.324224+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:05:46.881275+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":49,"reported_epoch":116,"state":"active+clean","last_fresh":"2023-09-23T20:53:06.033708+0000","last_change":"2023-09-23T20:52:12.860483+0000","last_active":"2023-09-23T20:53:06.033708+0000","last_peered":"2023-09-23T20:53:06.033708+0000","last_clean":"2023-09-23T20:53:06.033708+0000","last_became_active":"2023-09-23T20:52:12.860093+0000","last_became_peered":"2023-09-23T20:52:12.860093+0000","last_unstale":"2023-09-23T20:53:06.033708+0000","last_undegraded":"2023-09-23T20:53:06.033708+0000","last_fullsized":"2023-09-23T20:53:06.033708+0000","mapping_epoch":100,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":101,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:01:28.713922+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,4],"acting":[3,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":58,"reported_epoch":115,"state":"active+clean","last_fresh":"2023-09-23T20:53:04.001327+0000","last_change":"2023-09-23T20:41:59.062981+0000","last_active":"2023-09-23T20:53:04.001327+0000","last_peered":"2023-09-23T20:53:04.001327+0000","last_clean":"2023-09-23T20:53:04.001327+0000","last_became_active":"2023-09-23T20:41:59.062717+0000","last_became_peered":"2023-09-23T20:41:59.062717+0000","last_unstale":"2023-09-23T20:53:04.001327+0000","last_undegraded":"2023-09-23T20:53:04.001327+0000","last_fullsized":"2023-09-23T20:53:04.001327+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:20:58.386461+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":97,"reported_epoch":115,"state":"active+clean","last_fresh":"2023-09-23T20:53:03.997858+0000","last_change":"2023-09-23T20:50:05.866898+0000","last_active":"2023-09-23T20:53:03.997858+0000","last_peered":"2023-09-23T20:53:03.997858+0000","last_clean":"2023-09-23T20:53:03.997858+0000","last_became_active":"2023-09-23T20:50:05.866541+0000","last_became_peered":"2023-09-23T20:50:05.866541+0000","last_unstale":"2023-09-23T20:53:03.997858+0000","last_undegraded":"2023-09-23T20:53:03.997858+0000","last_fullsized":"2023-09-23T20:53:03.997858+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:17:02.353010+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":70,"reported_epoch":115,"state":"active+clean","last_fresh":"2023-09-23T20:53:04.007585+0000","last_change":"2023-09-23T20:51:10.841350+0000","last_active":"2023-09-23T20:53:04.007585+0000","last_peered":"2023-09-23T20:53:04.007585+0000","last_clean":"2023-09-23T20:53:04.007585+0000","last_became_active":"2023-09-23T20:51:10.841029+0000","last_became_peered":"2023-09-23T20:51:10.841029+0000","last_unstale":"2023-09-23T20:53:04.007585+0000","last_undegraded":"2023-09-23T20:53:04.007585+0000","last_fullsized":"2023-09-23T20:53:04.007585+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:26:05.095206+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":85,"reported_epoch":115,"state":"active+clean","last_fresh":"2023-09-23T20:53:04.008693+0000","last_change":"2023-09-23T20:50:37.982235+0000","last_active":"2023-09-23T20:53:04.008693+0000","last_peered":"2023-09-23T20:53:04.008693+0000","last_clean":"2023-09-23T20:53:04.008693+0000","last_became_active":"2023-09-23T20:50:37.981913+0000","last_became_peered":"2023-09-23T20:50:37.981913+0000","last_unstale":"2023-09-23T20:53:04.008693+0000","last_undegraded":"2023-09-23T20:53:04.008693+0000","last_fullsized":"2023-09-23T20:53:04.008693+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:59:57.122584+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.4","version":"116'4814","reported_seq":4837,"reported_epoch":116,"state":"active+clean","last_fresh":"2023-09-23T20:53:13.874666+0000","last_change":"2023-09-23T20:52:14.869618+0000","last_active":"2023-09-23T20:53:13.874666+0000","last_peered":"2023-09-23T20:53:13.874666+0000","last_clean":"2023-09-23T20:53:13.874666+0000","last_became_active":"2023-09-23T20:52:14.869288+0000","last_became_peered":"2023-09-23T20:52:14.869288+0000","last_unstale":"2023-09-23T20:53:13.874666+0000","last_undegraded":"2023-09-23T20:53:13.874666+0000","last_fullsized":"2023-09-23T20:53:13.874666+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":102,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:52:13.843048+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:52:13.843048+0000","last_clean_scrub_stamp":"2023-09-23T20:52:13.843048+0000","objects_scrubbed":0,"log_size":4814,"log_dups_size":0,"ondisk_log_size":4814,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:48:22.341973+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":315490304,"num_objects":4814,"num_object_clones":0,"num_object_copies":14442,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4814,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4814,"num_write_kb":308096,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":80,"reported_epoch":115,"state":"active+clean","last_fresh":"2023-09-23T20:53:04.000501+0000","last_change":"2023-09-23T20:49:32.776947+0000","last_active":"2023-09-23T20:53:04.000501+0000","last_peered":"2023-09-23T20:53:04.000501+0000","last_clean":"2023-09-23T20:53:04.000501+0000","last_became_active":"2023-09-23T20:49:32.776646+0000","last_became_peered":"2023-09-23T20:49:32.776646+0000","last_unstale":"2023-09-23T20:53:04.000501+0000","last_undegraded":"2023-09-23T20:53:04.000501+0000","last_fullsized":"2023-09-23T20:53:04.000501+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:26:50.532203+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":74,"reported_epoch":115,"state":"active+clean","last_fresh":"2023-09-23T20:53:04.028344+0000","last_change":"2023-09-23T20:52:12.853904+0000","last_active":"2023-09-23T20:53:04.028344+0000","last_peered":"2023-09-23T20:53:04.028344+0000","last_clean":"2023-09-23T20:53:04.028344+0000","last_became_active":"2023-09-23T20:52:12.853589+0000","last_became_peered":"2023-09-23T20:52:12.853589+0000","last_unstale":"2023-09-23T20:53:04.028344+0000","last_undegraded":"2023-09-23T20:53:04.028344+0000","last_fullsized":"2023-09-23T20:53:04.028344+0000","mapping_epoch":100,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":101,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:36:58.772320+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":64,"reported_epoch":115,"state":"active+clean","last_fresh":"2023-09-23T20:53:04.007766+0000","last_change":"2023-09-23T20:52:12.861675+0000","last_active":"2023-09-23T20:53:04.007766+0000","last_peered":"2023-09-23T20:53:04.007766+0000","last_clean":"2023-09-23T20:53:04.007766+0000","last_became_active":"2023-09-23T20:52:12.861342+0000","last_became_peered":"2023-09-23T20:52:12.861342+0000","last_unstale":"2023-09-23T20:53:04.007766+0000","last_undegraded":"2023-09-23T20:53:04.007766+0000","last_fullsized":"2023-09-23T20:53:04.007766+0000","mapping_epoch":100,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":101,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:37:54.207520+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.1","version":"116'4891","reported_seq":4914,"reported_epoch":116,"state":"active+clean","last_fresh":"2023-09-23T20:53:13.831360+0000","last_change":"2023-09-23T20:52:14.866659+0000","last_active":"2023-09-23T20:53:13.831360+0000","last_peered":"2023-09-23T20:53:13.831360+0000","last_clean":"2023-09-23T20:53:13.831360+0000","last_became_active":"2023-09-23T20:52:14.866348+0000","last_became_peered":"2023-09-23T20:52:14.866348+0000","last_unstale":"2023-09-23T20:53:13.831360+0000","last_undegraded":"2023-09-23T20:53:13.831360+0000","last_fullsized":"2023-09-23T20:53:13.831360+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":102,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:52:13.843048+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:52:13.843048+0000","last_clean_scrub_stamp":"2023-09-23T20:52:13.843048+0000","objects_scrubbed":0,"log_size":4891,"log_dups_size":0,"ondisk_log_size":4891,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:51:07.595313+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":320536576,"num_objects":4891,"num_object_clones":0,"num_object_copies":14673,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4891,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4891,"num_write_kb":313024,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":76,"reported_epoch":115,"state":"active+clean","last_fresh":"2023-09-23T20:53:04.003326+0000","last_change":"2023-09-23T20:51:10.836910+0000","last_active":"2023-09-23T20:53:04.003326+0000","last_peered":"2023-09-23T20:53:04.003326+0000","last_clean":"2023-09-23T20:53:04.003326+0000","last_became_active":"2023-09-23T20:51:10.836557+0000","last_became_peered":"2023-09-23T20:51:10.836557+0000","last_unstale":"2023-09-23T20:53:04.003326+0000","last_undegraded":"2023-09-23T20:53:04.003326+0000","last_fullsized":"2023-09-23T20:53:04.003326+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:51:54.019969+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":50,"reported_epoch":115,"state":"active+clean","last_fresh":"2023-09-23T20:53:04.028730+0000","last_change":"2023-09-23T20:50:05.868392+0000","last_active":"2023-09-23T20:53:04.028730+0000","last_peered":"2023-09-23T20:53:04.028730+0000","last_clean":"2023-09-23T20:53:04.028730+0000","last_became_active":"2023-09-23T20:50:05.867910+0000","last_became_peered":"2023-09-23T20:50:05.867910+0000","last_unstale":"2023-09-23T20:53:04.028730+0000","last_undegraded":"2023-09-23T20:53:04.028730+0000","last_fullsized":"2023-09-23T20:53:04.028730+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:25:00.099993+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":74,"reported_epoch":116,"state":"active+clean","last_fresh":"2023-09-23T20:53:06.035434+0000","last_change":"2023-09-23T20:51:10.833546+0000","last_active":"2023-09-23T20:53:06.035434+0000","last_peered":"2023-09-23T20:53:06.035434+0000","last_clean":"2023-09-23T20:53:06.035434+0000","last_became_active":"2023-09-23T20:51:10.833137+0000","last_became_peered":"2023-09-23T20:51:10.833137+0000","last_unstale":"2023-09-23T20:53:06.035434+0000","last_undegraded":"2023-09-23T20:53:06.035434+0000","last_fullsized":"2023-09-23T20:53:06.035434+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:16:42.840695+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.2","version":"116'4583","reported_seq":4605,"reported_epoch":116,"state":"active+clean","last_fresh":"2023-09-23T20:53:12.002118+0000","last_change":"2023-09-23T20:52:14.866208+0000","last_active":"2023-09-23T20:53:12.002118+0000","last_peered":"2023-09-23T20:53:12.002118+0000","last_clean":"2023-09-23T20:53:12.002118+0000","last_became_active":"2023-09-23T20:52:14.865956+0000","last_became_peered":"2023-09-23T20:52:14.865956+0000","last_unstale":"2023-09-23T20:53:12.002118+0000","last_undegraded":"2023-09-23T20:53:12.002118+0000","last_fullsized":"2023-09-23T20:53:12.002118+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":102,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:52:13.843048+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:52:13.843048+0000","last_clean_scrub_stamp":"2023-09-23T20:52:13.843048+0000","objects_scrubbed":0,"log_size":4583,"log_dups_size":0,"ondisk_log_size":4583,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:38:02.533042+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":300351488,"num_objects":4583,"num_object_clones":0,"num_object_copies":13749,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4583,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4583,"num_write_kb":293312,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":80,"reported_epoch":115,"state":"active+clean","last_fresh":"2023-09-23T20:53:04.000100+0000","last_change":"2023-09-23T20:50:37.985663+0000","last_active":"2023-09-23T20:53:04.000100+0000","last_peered":"2023-09-23T20:53:04.000100+0000","last_clean":"2023-09-23T20:53:04.000100+0000","last_became_active":"2023-09-23T20:50:37.985325+0000","last_became_peered":"2023-09-23T20:50:37.985325+0000","last_unstale":"2023-09-23T20:53:04.000100+0000","last_undegraded":"2023-09-23T20:53:04.000100+0000","last_fullsized":"2023-09-23T20:53:04.000100+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:24:30.838843+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"70'1","reported_seq":75,"reported_epoch":115,"state":"active+clean","last_fresh":"2023-09-23T20:53:04.008641+0000","last_change":"2023-09-23T20:50:05.871179+0000","last_active":"2023-09-23T20:53:04.008641+0000","last_peered":"2023-09-23T20:53:04.008641+0000","last_clean":"2023-09-23T20:53:04.008641+0000","last_became_active":"2023-09-23T20:50:05.870816+0000","last_became_peered":"2023-09-23T20:50:05.870816+0000","last_unstale":"2023-09-23T20:53:04.008641+0000","last_undegraded":"2023-09-23T20:53:04.008641+0000","last_fullsized":"2023-09-23T20:53:04.008641+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:25:24.455835+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":68,"reported_epoch":115,"state":"active+clean","last_fresh":"2023-09-23T20:53:03.998207+0000","last_change":"2023-09-23T20:52:12.860323+0000","last_active":"2023-09-23T20:53:03.998207+0000","last_peered":"2023-09-23T20:53:03.998207+0000","last_clean":"2023-09-23T20:53:03.998207+0000","last_became_active":"2023-09-23T20:52:12.859984+0000","last_became_peered":"2023-09-23T20:52:12.859984+0000","last_unstale":"2023-09-23T20:53:03.998207+0000","last_undegraded":"2023-09-23T20:53:03.998207+0000","last_fullsized":"2023-09-23T20:53:03.998207+0000","mapping_epoch":100,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":101,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:53:33.240133+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.3","version":"116'4734","reported_seq":4756,"reported_epoch":116,"state":"active+clean","last_fresh":"2023-09-23T20:53:12.016896+0000","last_change":"2023-09-23T20:52:14.865640+0000","last_active":"2023-09-23T20:53:12.016896+0000","last_peered":"2023-09-23T20:53:12.016896+0000","last_clean":"2023-09-23T20:53:12.016896+0000","last_became_active":"2023-09-23T20:52:14.865328+0000","last_became_peered":"2023-09-23T20:52:14.865328+0000","last_unstale":"2023-09-23T20:53:12.016896+0000","last_undegraded":"2023-09-23T20:53:12.016896+0000","last_fullsized":"2023-09-23T20:53:12.016896+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":102,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:52:13.843048+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:52:13.843048+0000","last_clean_scrub_stamp":"2023-09-23T20:52:13.843048+0000","objects_scrubbed":0,"log_size":4734,"log_dups_size":0,"ondisk_log_size":4734,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:04:22.193022+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":310247424,"num_objects":4734,"num_object_clones":0,"num_object_copies":14202,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4734,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4734,"num_write_kb":302976,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,2],"acting":[7,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":58,"reported_epoch":115,"state":"active+clean","last_fresh":"2023-09-23T20:53:04.008000+0000","last_change":"2023-09-23T20:49:32.775562+0000","last_active":"2023-09-23T20:53:04.008000+0000","last_peered":"2023-09-23T20:53:04.008000+0000","last_clean":"2023-09-23T20:53:04.008000+0000","last_became_active":"2023-09-23T20:49:32.775094+0000","last_became_peered":"2023-09-23T20:49:32.775094+0000","last_unstale":"2023-09-23T20:53:04.008000+0000","last_undegraded":"2023-09-23T20:53:04.008000+0000","last_fullsized":"2023-09-23T20:53:04.008000+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:17:52.944043+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":82,"reported_epoch":116,"state":"active+clean","last_fresh":"2023-09-23T20:53:06.023379+0000","last_change":"2023-09-23T20:52:12.855430+0000","last_active":"2023-09-23T20:53:06.023379+0000","last_peered":"2023-09-23T20:53:06.023379+0000","last_clean":"2023-09-23T20:53:06.023379+0000","last_became_active":"2023-09-23T20:52:12.855115+0000","last_became_peered":"2023-09-23T20:52:12.855115+0000","last_unstale":"2023-09-23T20:53:06.023379+0000","last_undegraded":"2023-09-23T20:53:06.023379+0000","last_fullsized":"2023-09-23T20:53:06.023379+0000","mapping_epoch":100,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":101,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:33:23.802214+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":42,"reported_epoch":115,"state":"active+clean","last_fresh":"2023-09-23T20:53:03.999117+0000","last_change":"2023-09-23T20:52:12.863936+0000","last_active":"2023-09-23T20:53:03.999117+0000","last_peered":"2023-09-23T20:53:03.999117+0000","last_clean":"2023-09-23T20:53:03.999117+0000","last_became_active":"2023-09-23T20:52:12.855724+0000","last_became_peered":"2023-09-23T20:52:12.855724+0000","last_unstale":"2023-09-23T20:53:03.999117+0000","last_undegraded":"2023-09-23T20:53:03.999117+0000","last_fullsized":"2023-09-23T20:53:03.999117+0000","mapping_epoch":100,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":101,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:12:48.714857+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":92,"reported_epoch":115,"state":"active+clean","last_fresh":"2023-09-23T20:53:03.998985+0000","last_change":"2023-09-23T20:51:10.833684+0000","last_active":"2023-09-23T20:53:03.998985+0000","last_peered":"2023-09-23T20:53:03.998985+0000","last_clean":"2023-09-23T20:53:03.998985+0000","last_became_active":"2023-09-23T20:51:10.833360+0000","last_became_peered":"2023-09-23T20:51:10.833360+0000","last_unstale":"2023-09-23T20:53:03.998985+0000","last_undegraded":"2023-09-23T20:53:03.998985+0000","last_fullsized":"2023-09-23T20:53:03.998985+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:11:12.175158+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":42,"reported_epoch":115,"state":"active+clean","last_fresh":"2023-09-23T20:53:03.998758+0000","last_change":"2023-09-23T20:52:12.869818+0000","last_active":"2023-09-23T20:53:03.998758+0000","last_peered":"2023-09-23T20:53:03.998758+0000","last_clean":"2023-09-23T20:53:03.998758+0000","last_became_active":"2023-09-23T20:52:12.859839+0000","last_became_peered":"2023-09-23T20:52:12.859839+0000","last_unstale":"2023-09-23T20:53:03.998758+0000","last_undegraded":"2023-09-23T20:53:03.998758+0000","last_fullsized":"2023-09-23T20:53:03.998758+0000","mapping_epoch":100,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":101,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:56:46.497439+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":92,"reported_epoch":115,"state":"active+clean","last_fresh":"2023-09-23T20:53:03.998501+0000","last_change":"2023-09-23T20:51:10.835094+0000","last_active":"2023-09-23T20:53:03.998501+0000","last_peered":"2023-09-23T20:53:03.998501+0000","last_clean":"2023-09-23T20:53:03.998501+0000","last_became_active":"2023-09-23T20:51:10.834732+0000","last_became_peered":"2023-09-23T20:51:10.834732+0000","last_unstale":"2023-09-23T20:53:03.998501+0000","last_undegraded":"2023-09-23T20:53:03.998501+0000","last_fullsized":"2023-09-23T20:53:03.998501+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:21:08.414737+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":85,"reported_epoch":116,"state":"active+clean","last_fresh":"2023-09-23T20:53:06.033782+0000","last_change":"2023-09-23T20:51:10.833849+0000","last_active":"2023-09-23T20:53:06.033782+0000","last_peered":"2023-09-23T20:53:06.033782+0000","last_clean":"2023-09-23T20:53:06.033782+0000","last_became_active":"2023-09-23T20:51:10.833524+0000","last_became_peered":"2023-09-23T20:51:10.833524+0000","last_unstale":"2023-09-23T20:53:06.033782+0000","last_undegraded":"2023-09-23T20:53:06.033782+0000","last_fullsized":"2023-09-23T20:53:06.033782+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:23:12.501924+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":99,"reported_epoch":115,"state":"active+clean","last_fresh":"2023-09-23T20:53:04.002345+0000","last_change":"2023-09-23T20:51:10.838334+0000","last_active":"2023-09-23T20:53:04.002345+0000","last_peered":"2023-09-23T20:53:04.002345+0000","last_clean":"2023-09-23T20:53:04.002345+0000","last_became_active":"2023-09-23T20:51:10.838026+0000","last_became_peered":"2023-09-23T20:51:10.838026+0000","last_unstale":"2023-09-23T20:53:04.002345+0000","last_undegraded":"2023-09-23T20:53:04.002345+0000","last_fullsized":"2023-09-23T20:53:04.002345+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:04:37.009457+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":88,"reported_epoch":115,"state":"active+clean","last_fresh":"2023-09-23T20:53:04.008810+0000","last_change":"2023-09-23T20:52:12.868379+0000","last_active":"2023-09-23T20:53:04.008810+0000","last_peered":"2023-09-23T20:53:04.008810+0000","last_clean":"2023-09-23T20:53:04.008810+0000","last_became_active":"2023-09-23T20:52:12.861899+0000","last_became_peered":"2023-09-23T20:52:12.861899+0000","last_unstale":"2023-09-23T20:53:04.008810+0000","last_undegraded":"2023-09-23T20:53:04.008810+0000","last_fullsized":"2023-09-23T20:53:04.008810+0000","mapping_epoch":100,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":101,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:17:39.308666+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":554,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,6],"acting":[0,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":85,"reported_epoch":115,"state":"active+clean","last_fresh":"2023-09-23T20:53:03.998032+0000","last_change":"2023-09-23T20:50:37.984570+0000","last_active":"2023-09-23T20:53:03.998032+0000","last_peered":"2023-09-23T20:53:03.998032+0000","last_clean":"2023-09-23T20:53:03.998032+0000","last_became_active":"2023-09-23T20:50:37.984262+0000","last_became_peered":"2023-09-23T20:50:37.984262+0000","last_unstale":"2023-09-23T20:53:03.998032+0000","last_undegraded":"2023-09-23T20:53:03.998032+0000","last_fullsized":"2023-09-23T20:53:03.998032+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:47:39.024252+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":6,"num_pg":16,"stat_sum":{"num_bytes":4950392832,"num_objects":75537,"num_object_clones":0,"num_object_copies":226611,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":75537,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":75537,"num_write_kb":4834368,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":14797832192,"data_stored":14797832192,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":75537,"ondisk_log_size":75537,"up":48,"acting":48,"num_store_stats":7},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":9,"num_bytes_recovered":624,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":106496,"data_stored":17136,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":161,"ondisk_log_size":161,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201433,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2125040,"kb_used_data":2083944,"kb_used_omap":0,"kb_used_meta":41088,"kb_avail":91620112,"statfs":{"total":95995035648,"available":93818994688,"internally_reserved":0,"allocated":2133958656,"data_stored":2133478816,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":42074112},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:52:36 2023","interfaces":[{"interface":"back","average":{"1min":0.92000000000000004,"5min":0.60299999999999998,"15min":0.52800000000000002},"min":{"1min":0.25800000000000001,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":3.5209999999999999,"5min":3.5209999999999999,"15min":3.5209999999999999},"last":0.33400000000000002},{"interface":"front","average":{"1min":0.71199999999999997,"5min":0.58999999999999997,"15min":0.57199999999999995},"min":{"1min":0.28899999999999998,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":3.1779999999999999,"5min":3.1779999999999999,"15min":3.1779999999999999},"last":0.29099999999999998}]},{"osd":1,"last update":"Sat Sep 23 20:53:07 2023","interfaces":[{"interface":"back","average":{"1min":2.5640000000000001,"5min":0.94999999999999996,"15min":0.68200000000000005},"min":{"1min":0.26100000000000001,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":22.591999999999999,"5min":22.591999999999999,"15min":22.591999999999999},"last":4.484},{"interface":"front","average":{"1min":1.992,"5min":0.83999999999999997,"15min":0.64200000000000002},"min":{"1min":0.25600000000000001,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":21.437000000000001,"5min":21.437000000000001,"15min":21.437000000000001},"last":3.3239999999999998}]},{"osd":2,"last update":"Sat Sep 23 20:52:45 2023","interfaces":[{"interface":"back","average":{"1min":0.89400000000000002,"5min":0.61899999999999999,"15min":0.57299999999999995},"min":{"1min":0.24399999999999999,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":3.5459999999999998,"5min":3.5459999999999998,"15min":3.5459999999999998},"last":3.1880000000000002},{"interface":"front","average":{"1min":0.93200000000000005,"5min":0.65300000000000002,"15min":0.60699999999999998},"min":{"1min":0.34399999999999997,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":3.7050000000000001,"5min":3.7050000000000001,"15min":3.7050000000000001},"last":3.3780000000000001}]},{"osd":3,"last update":"Sat Sep 23 20:52:13 2023","interfaces":[{"interface":"back","average":{"1min":0.56000000000000005,"5min":0.56000000000000005,"15min":0.56000000000000005},"min":{"1min":0.38700000000000001,"5min":0.38700000000000001,"15min":0.38700000000000001},"max":{"1min":0.77200000000000002,"5min":0.77200000000000002,"15min":0.77200000000000002},"last":4.1079999999999997},{"interface":"front","average":{"1min":0.57399999999999995,"5min":0.57399999999999995,"15min":0.57399999999999995},"min":{"1min":0.41199999999999998,"5min":0.41199999999999998,"15min":0.41199999999999998},"max":{"1min":0.73099999999999998,"5min":0.73099999999999998,"15min":0.73099999999999998},"last":1.143}]},{"osd":4,"last update":"Sat Sep 23 20:53:04 2023","interfaces":[{"interface":"back","average":{"1min":0.50800000000000001,"5min":0.52900000000000003,"15min":0.49099999999999999},"min":{"1min":0.23699999999999999,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":1.3879999999999999,"5min":1.3879999999999999,"15min":1.3879999999999999},"last":0.41999999999999998},{"interface":"front","average":{"1min":0.69199999999999995,"5min":0.57699999999999996,"15min":0.53600000000000003},"min":{"1min":0.32100000000000001,"5min":0.32100000000000001,"15min":0.29599999999999999},"max":{"1min":3.2349999999999999,"5min":3.2349999999999999,"15min":3.2349999999999999},"last":0.50900000000000001}]},{"osd":5,"last update":"Sat Sep 23 20:53:04 2023","interfaces":[{"interface":"back","average":{"1min":0.52700000000000002,"5min":0.56399999999999995,"15min":0.53800000000000003},"min":{"1min":0.29999999999999999,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":1.0620000000000001,"5min":1.0620000000000001,"15min":1.4610000000000001},"last":0.53000000000000003},{"interface":"front","average":{"1min":0.59099999999999997,"5min":0.60399999999999998,"15min":0.56100000000000005},"min":{"1min":0.29199999999999998,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":1.319,"5min":1.319,"15min":1.319},"last":0.81999999999999995}]},{"osd":6,"last update":"Sat Sep 23 20:53:04 2023","interfaces":[{"interface":"back","average":{"1min":0.58799999999999997,"5min":0.62,"15min":0.57899999999999996},"min":{"1min":0.27400000000000002,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":1.363,"5min":1.363,"15min":1.363},"last":0.56799999999999995},{"interface":"front","average":{"1min":0.61299999999999999,"5min":0.61599999999999999,"15min":0.59199999999999997},"min":{"1min":0.376,"5min":0.36299999999999999,"15min":0.316},"max":{"1min":1.4419999999999999,"5min":1.4419999999999999,"15min":1.4419999999999999},"last":0.54800000000000004}]}]},{"osd":6,"up_from":47,"seq":201863463070,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1847844,"kb_used_data":1801308,"kb_used_omap":0,"kb_used_meta":46528,"kb_avail":91897308,"statfs":{"total":95995035648,"available":94102843392,"internally_reserved":0,"allocated":1844539392,"data_stored":1844071983,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":47644672},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:52:39 2023","interfaces":[{"interface":"back","average":{"1min":0.57799999999999996,"5min":0.56399999999999995,"15min":0.57399999999999995},"min":{"1min":0.29099999999999998,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":1.4810000000000001,"5min":1.4810000000000001,"15min":1.4810000000000001},"last":0.29299999999999998},{"interface":"front","average":{"1min":0.77300000000000002,"5min":0.61699999999999999,"15min":0.59899999999999998},"min":{"1min":0.29099999999999998,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":3.3069999999999999,"5min":3.3069999999999999,"15min":3.3069999999999999},"last":1.319}]},{"osd":1,"last update":"Sat Sep 23 20:52:10 2023","interfaces":[{"interface":"back","average":{"1min":0.55000000000000004,"5min":0.53700000000000003,"15min":0.53500000000000003},"min":{"1min":0.39000000000000001,"5min":0.371,"15min":0.371},"max":{"1min":0.80600000000000005,"5min":0.80600000000000005,"15min":0.80600000000000005},"last":1.5189999999999999},{"interface":"front","average":{"1min":0.56000000000000005,"5min":0.56799999999999995,"15min":0.56899999999999995},"min":{"1min":0.35399999999999998,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":0.91700000000000004,"5min":0.91700000000000004,"15min":0.91700000000000004},"last":0.308}]},{"osd":2,"last update":"Sat Sep 23 20:52:44 2023","interfaces":[{"interface":"back","average":{"1min":0.94999999999999996,"5min":0.628,"15min":0.57399999999999995},"min":{"1min":0.44,"5min":0.40899999999999997,"15min":0.40899999999999997},"max":{"1min":3.1179999999999999,"5min":3.1179999999999999,"15min":3.1179999999999999},"last":0.61399999999999999},{"interface":"front","average":{"1min":1.026,"5min":0.67400000000000004,"15min":0.61499999999999999},"min":{"1min":0.39700000000000002,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":3.0960000000000001,"5min":3.0960000000000001,"15min":3.0960000000000001},"last":0.40500000000000003}]},{"osd":3,"last update":"Sat Sep 23 20:52:10 2023","interfaces":[{"interface":"back","average":{"1min":0.59199999999999997,"5min":0.59199999999999997,"15min":0.59199999999999997},"min":{"1min":0.38700000000000001,"5min":0.38700000000000001,"15min":0.38700000000000001},"max":{"1min":0.85799999999999998,"5min":0.85799999999999998,"15min":0.85799999999999998},"last":0.75600000000000001},{"interface":"front","average":{"1min":0.58099999999999996,"5min":0.58099999999999996,"15min":0.58099999999999996},"min":{"1min":0.40000000000000002,"5min":0.40000000000000002,"15min":0.40000000000000002},"max":{"1min":0.83399999999999996,"5min":0.83399999999999996,"15min":0.83399999999999996},"last":0.50800000000000001}]},{"osd":4,"last update":"Sat Sep 23 20:52:44 2023","interfaces":[{"interface":"back","average":{"1min":0.46500000000000002,"5min":0.47699999999999998,"15min":0.46000000000000002},"min":{"1min":0.248,"5min":0.248,"15min":0.248},"max":{"1min":0.71599999999999997,"5min":0.81000000000000005,"15min":1.1779999999999999},"last":0.55500000000000005},{"interface":"front","average":{"1min":0.46100000000000002,"5min":0.51600000000000001,"15min":0.49299999999999999},"min":{"1min":0.28899999999999998,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":0.748,"5min":0.81599999999999995,"15min":0.81599999999999995},"last":0.42299999999999999}]},{"osd":5,"last update":"Sat Sep 23 20:52:44 2023","interfaces":[{"interface":"back","average":{"1min":0.53900000000000003,"5min":0.52200000000000002,"15min":0.498},"min":{"1min":0.32000000000000001,"5min":0.32000000000000001,"15min":0.28399999999999997},"max":{"1min":0.872,"5min":0.872,"15min":1.081},"last":0.63400000000000001},{"interface":"front","average":{"1min":0.52300000000000002,"5min":0.55600000000000005,"15min":0.52500000000000002},"min":{"1min":0.38500000000000001,"5min":0.33100000000000002,"15min":0.28499999999999998},"max":{"1min":0.72999999999999998,"5min":0.80800000000000005,"15min":0.89000000000000001},"last":0.496}]},{"osd":7,"last update":"Sat Sep 23 20:53:04 2023","interfaces":[{"interface":"back","average":{"1min":0.48999999999999999,"5min":0.55600000000000005,"15min":0.54000000000000004},"min":{"1min":0.34799999999999998,"5min":0.33000000000000002,"15min":0.32500000000000001},"max":{"1min":0.94599999999999995,"5min":0.94599999999999995,"15min":1.119},"last":0.58999999999999997},{"interface":"front","average":{"1min":0.497,"5min":0.60099999999999998,"15min":0.58199999999999996},"min":{"1min":0.39000000000000001,"5min":0.38900000000000001,"15min":0.36099999999999999},"max":{"1min":0.84599999999999997,"5min":0.86899999999999999,"15min":0.94099999999999995},"last":0.46100000000000002}]}]},{"osd":1,"up_from":88,"seq":377957122088,"num_pgs":42,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1881644,"kb_used_data":1832036,"kb_used_omap":9,"kb_used_meta":49590,"kb_avail":91863508,"statfs":{"total":95995035648,"available":94068232192,"internally_reserved":0,"allocated":1876004864,"data_stored":1875472530,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9239,"internal_metadata":50781161},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:53:07 2023","interfaces":[{"interface":"back","average":{"1min":0.98299999999999998,"5min":0.50900000000000001,"15min":0.442},"min":{"1min":0.216,"5min":0.216,"15min":0.216},"max":{"1min":5.0860000000000003,"5min":5.0860000000000003,"15min":5.0860000000000003},"last":5.2069999999999999},{"interface":"front","average":{"1min":1.071,"5min":0.54900000000000004,"15min":0.47299999999999998},"min":{"1min":0.31900000000000001,"5min":0.31,"15min":0.31},"max":{"1min":4.9059999999999997,"5min":4.9059999999999997,"15min":4.9059999999999997},"last":7.2969999999999997}]},{"osd":2,"last update":"Sat Sep 23 20:52:39 2023","interfaces":[{"interface":"back","average":{"1min":1.2370000000000001,"5min":0.55700000000000005,"15min":0.443},"min":{"1min":0.26500000000000001,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":6.6379999999999999,"5min":6.6379999999999999,"15min":6.6379999999999999},"last":0.20300000000000001},{"interface":"front","average":{"1min":0.78400000000000003,"5min":0.624,"15min":0.59799999999999998},"min":{"1min":0.26100000000000001,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":4.9370000000000003,"5min":4.9370000000000003,"15min":4.9370000000000003},"last":2.0289999999999999}]},{"osd":3,"last update":"Sat Sep 23 20:53:13 2023","interfaces":[{"interface":"back","average":{"1min":2.1179999999999999,"5min":0.70999999999999996,"15min":0.47599999999999998},"min":{"1min":0.27700000000000002,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":5.6710000000000003,"5min":5.6710000000000003,"15min":5.6710000000000003},"last":1.839},{"interface":"front","average":{"1min":1.7210000000000001,"5min":0.67200000000000004,"15min":0.497},"min":{"1min":0.32500000000000001,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":4.3780000000000001,"5min":4.3780000000000001,"15min":4.3780000000000001},"last":0.49099999999999999}]},{"osd":4,"last update":"Sat Sep 23 20:53:07 2023","interfaces":[{"interface":"back","average":{"1min":1.448,"5min":0.79300000000000004,"15min":0.69499999999999995},"min":{"1min":0.35199999999999998,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":5.7279999999999998,"5min":5.7279999999999998,"15min":5.7279999999999998},"last":1.756},{"interface":"front","average":{"1min":0.86799999999999999,"5min":0.68999999999999995,"15min":0.67800000000000005},"min":{"1min":0.30099999999999999,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":4.4290000000000003,"5min":4.4290000000000003,"15min":4.4290000000000003},"last":0.41399999999999998}]},{"osd":5,"last update":"Sat Sep 23 20:53:07 2023","interfaces":[{"interface":"back","average":{"1min":0.91500000000000004,"5min":0.71099999999999997,"15min":0.70199999999999996},"min":{"1min":0.371,"5min":0.371,"15min":0.371},"max":{"1min":3.234,"5min":3.234,"15min":3.234},"last":2},{"interface":"front","average":{"1min":1.4670000000000001,"5min":0.79600000000000004,"15min":0.69599999999999995},"min":{"1min":0.377,"5min":0.377,"15min":0.377},"max":{"1min":5.766,"5min":5.766,"15min":5.766},"last":1.8160000000000001}]},{"osd":6,"last update":"Sat Sep 23 20:53:07 2023","interfaces":[{"interface":"back","average":{"1min":0.91200000000000003,"5min":0.71399999999999997,"15min":0.69599999999999995},"min":{"1min":0.36899999999999999,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":3.278,"5min":3.278,"15min":3.278},"last":2.0539999999999998},{"interface":"front","average":{"1min":1.476,"5min":0.80600000000000005,"15min":0.70699999999999996},"min":{"1min":0.41199999999999998,"5min":0.41199999999999998,"15min":0.41199999999999998},"max":{"1min":5.7489999999999997,"5min":5.7489999999999997,"15min":5.7489999999999997},"last":1.7949999999999999}]},{"osd":7,"last update":"Sat Sep 23 20:53:07 2023","interfaces":[{"interface":"back","average":{"1min":1.1140000000000001,"5min":0.74199999999999999,"15min":0.69599999999999995},"min":{"1min":0.36599999999999999,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":4.46,"5min":4.46,"15min":4.46},"last":0.45400000000000001},{"interface":"front","average":{"1min":1.0189999999999999,"5min":0.73499999999999999,"15min":0.70199999999999996},"min":{"1min":0.38300000000000001,"5min":0.38300000000000001,"15min":0.38300000000000001},"max":{"1min":4.8700000000000001,"5min":4.8700000000000001,"15min":4.8700000000000001},"last":2.0779999999999998}]}]},{"osd":0,"up_from":83,"seq":356482285614,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1239872,"kb_used_data":1182008,"kb_used_omap":28,"kb_used_meta":57827,"kb_avail":92505280,"statfs":{"total":95995035648,"available":94725406720,"internally_reserved":0,"allocated":1210376192,"data_stored":1209825824,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":28848,"internal_metadata":59215696},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,2,1,0,0,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 20:53:10 2023","interfaces":[{"interface":"back","average":{"1min":1.286,"5min":0.52900000000000003,"15min":0.39800000000000002},"min":{"1min":0.249,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":6.3710000000000004,"5min":6.3710000000000004,"15min":6.3710000000000004},"last":6.3710000000000004},{"interface":"front","average":{"1min":1.2709999999999999,"5min":0.56200000000000006,"15min":0.44500000000000001},"min":{"1min":0.255,"5min":0.255,"15min":0.255},"max":{"1min":8.4969999999999999,"5min":8.4969999999999999,"15min":8.4969999999999999},"last":0.65000000000000002}]},{"osd":2,"last update":"Sat Sep 23 20:52:40 2023","interfaces":[{"interface":"back","average":{"1min":1.212,"5min":0.57399999999999995,"15min":0.46800000000000003},"min":{"1min":0.27500000000000002,"5min":0.248,"15min":0.248},"max":{"1min":8.5649999999999995,"5min":8.5649999999999995,"15min":8.5649999999999995},"last":0.23799999999999999},{"interface":"front","average":{"1min":0.90100000000000002,"5min":0.49199999999999999,"15min":0.42399999999999999},"min":{"1min":0.309,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":4.7050000000000001,"5min":4.7050000000000001,"15min":4.7050000000000001},"last":4.7089999999999996}]},{"osd":3,"last update":"Sat Sep 23 20:52:11 2023","interfaces":[{"interface":"back","average":{"1min":0.40100000000000002,"5min":0.40100000000000002,"15min":0.40100000000000002},"min":{"1min":0.29299999999999998,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":0.70099999999999996,"5min":0.70099999999999996,"15min":0.70099999999999996},"last":1.089},{"interface":"front","average":{"1min":0.47799999999999998,"5min":0.47799999999999998,"15min":0.47799999999999998},"min":{"1min":0.36099999999999999,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":0.98799999999999999,"5min":0.98799999999999999,"15min":0.98799999999999999},"last":2.2109999999999999}]},{"osd":4,"last update":"Sat Sep 23 20:52:40 2023","interfaces":[{"interface":"back","average":{"1min":0.87,"5min":0.69199999999999995,"15min":0.66200000000000003},"min":{"1min":0.33700000000000002,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":4.5250000000000004,"5min":4.5250000000000004,"15min":4.5250000000000004},"last":4.6189999999999998},{"interface":"front","average":{"1min":0.79200000000000004,"5min":0.69099999999999995,"15min":0.66900000000000004},"min":{"1min":0.40799999999999997,"5min":0.40799999999999997,"15min":0.40799999999999997},"max":{"1min":4.5890000000000004,"5min":4.5890000000000004,"15min":4.5890000000000004},"last":0.89000000000000001}]},{"osd":5,"last update":"Sat Sep 23 20:52:40 2023","interfaces":[{"interface":"back","average":{"1min":1.1879999999999999,"5min":0.747,"15min":0.68100000000000005},"min":{"1min":0.41499999999999998,"5min":0.41499999999999998,"15min":0.41499999999999998},"max":{"1min":8.4559999999999995,"5min":8.4559999999999995,"15min":8.4559999999999995},"last":0.68300000000000005},{"interface":"front","average":{"1min":0.86599999999999999,"5min":0.71899999999999997,"15min":0.71099999999999997},"min":{"1min":0.375,"5min":0.375,"15min":0.375},"max":{"1min":4.4740000000000002,"5min":4.4740000000000002,"15min":4.4740000000000002},"last":4.6859999999999999}]},{"osd":6,"last update":"Sat Sep 23 20:52:40 2023","interfaces":[{"interface":"back","average":{"1min":0.92300000000000004,"5min":0.71999999999999997,"15min":0.69999999999999996},"min":{"1min":0.45700000000000002,"5min":0.433,"15min":0.433},"max":{"1min":4.4400000000000004,"5min":4.4400000000000004,"15min":4.4400000000000004},"last":4.6580000000000004},{"interface":"front","average":{"1min":0.86799999999999999,"5min":0.72599999999999998,"15min":0.69599999999999995},"min":{"1min":0.41399999999999998,"5min":0.41399999999999998,"15min":0.41399999999999998},"max":{"1min":4.6200000000000001,"5min":4.6200000000000001,"15min":4.6200000000000001},"last":0.96899999999999997}]},{"osd":7,"last update":"Sat Sep 23 20:52:40 2023","interfaces":[{"interface":"back","average":{"1min":1.1870000000000001,"5min":0.78500000000000003,"15min":0.72799999999999998},"min":{"1min":0.45800000000000002,"5min":0.42799999999999999,"15min":0.42799999999999999},"max":{"1min":8.3979999999999997,"5min":8.3979999999999997,"15min":8.3979999999999997},"last":0.71199999999999997},{"interface":"front","average":{"1min":0.89100000000000001,"5min":0.70999999999999996,"15min":0.68600000000000005},"min":{"1min":0.41899999999999998,"5min":0.41899999999999998,"15min":0.41899999999999998},"max":{"1min":4.5599999999999996,"5min":4.5599999999999996,"15min":4.5599999999999996},"last":0.92800000000000005}]}]},{"osd":2,"up_from":93,"seq":399431958561,"num_pgs":29,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3021216,"kb_used_data":2949336,"kb_used_omap":81,"kb_used_meta":71790,"kb_avail":90723936,"statfs":{"total":95995035648,"available":92901310464,"internally_reserved":0,"allocated":3020120064,"data_stored":3019596927,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":83750,"internal_metadata":73513178},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:52:39 2023","interfaces":[{"interface":"back","average":{"1min":0.52800000000000002,"5min":0.39200000000000002,"15min":0.36899999999999999},"min":{"1min":0.20499999999999999,"5min":0.20499999999999999,"15min":0.20499999999999999},"max":{"1min":2.6760000000000002,"5min":2.6760000000000002,"15min":2.6760000000000002},"last":2.714},{"interface":"front","average":{"1min":0.52800000000000002,"5min":0.43099999999999999,"15min":0.41499999999999998},"min":{"1min":0.26800000000000002,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":1.9830000000000001,"5min":1.9830000000000001,"15min":1.9830000000000001},"last":2.883}]},{"osd":1,"last update":"Sat Sep 23 20:52:39 2023","interfaces":[{"interface":"back","average":{"1min":0.65600000000000003,"5min":0.46400000000000002,"15min":0.432},"min":{"1min":0.24199999999999999,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":4.367,"5min":4.367,"15min":4.367},"last":2.7330000000000001},{"interface":"front","average":{"1min":0.70299999999999996,"5min":0.47699999999999998,"15min":0.439},"min":{"1min":0.25700000000000001,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":5.2469999999999999,"5min":5.2469999999999999,"15min":5.2469999999999999},"last":2.9180000000000001}]},{"osd":3,"last update":"Sat Sep 23 20:52:12 2023","interfaces":[{"interface":"back","average":{"1min":0.41399999999999998,"5min":0.41399999999999998,"15min":0.41399999999999998},"min":{"1min":0.255,"5min":0.255,"15min":0.255},"max":{"1min":1.571,"5min":1.571,"15min":1.571},"last":11.423999999999999},{"interface":"front","average":{"1min":0.38,"5min":0.38,"15min":0.38},"min":{"1min":0.221,"5min":0.221,"15min":0.221},"max":{"1min":1.02,"5min":1.02,"15min":1.02},"last":11.510999999999999}]},{"osd":4,"last update":"Sat Sep 23 20:52:39 2023","interfaces":[{"interface":"back","average":{"1min":0.69899999999999995,"5min":0.63500000000000001,"15min":0.624},"min":{"1min":0.29299999999999998,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":1.9359999999999999,"5min":1.9359999999999999,"15min":1.9359999999999999},"last":2.8250000000000002},{"interface":"front","average":{"1min":0.81200000000000006,"5min":0.69899999999999995,"15min":0.68000000000000005},"min":{"1min":0.40400000000000003,"5min":0.40400000000000003,"15min":0.40400000000000003},"max":{"1min":4.274,"5min":4.274,"15min":4.274},"last":0.42599999999999999}]},{"osd":5,"last update":"Sat Sep 23 20:52:39 2023","interfaces":[{"interface":"back","average":{"1min":0.71299999999999997,"5min":0.66400000000000003,"15min":0.65600000000000003},"min":{"1min":0.34200000000000003,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":2.0139999999999998,"5min":2.0139999999999998,"15min":2.0139999999999998},"last":2.7919999999999998},{"interface":"front","average":{"1min":0.84399999999999997,"5min":0.67700000000000005,"15min":0.64900000000000002},"min":{"1min":0.36299999999999999,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":4.3070000000000004,"5min":4.3070000000000004,"15min":4.3070000000000004},"last":0.48899999999999999}]},{"osd":6,"last update":"Sat Sep 23 20:52:39 2023","interfaces":[{"interface":"back","average":{"1min":0.82699999999999996,"5min":0.72999999999999998,"15min":0.71399999999999997},"min":{"1min":0.38800000000000001,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":4.3380000000000001,"5min":4.3380000000000001,"15min":4.3380000000000001},"last":0.35099999999999998},{"interface":"front","average":{"1min":0.77900000000000003,"5min":0.70399999999999996,"15min":0.69199999999999995},"min":{"1min":0.41899999999999998,"5min":0.41899999999999998,"15min":0.41899999999999998},"max":{"1min":2.7410000000000001,"5min":2.7410000000000001,"15min":2.7410000000000001},"last":2.6629999999999998}]},{"osd":7,"last update":"Sat Sep 23 20:52:39 2023","interfaces":[{"interface":"back","average":{"1min":0.746,"5min":0.65700000000000003,"15min":0.64200000000000002},"min":{"1min":0.42299999999999999,"5min":0.42299999999999999,"15min":0.42299999999999999},"max":{"1min":2.7719999999999998,"5min":2.7719999999999998,"15min":2.7719999999999998},"last":2.6949999999999998},{"interface":"front","average":{"1min":0.75,"5min":0.67900000000000005,"15min":0.66700000000000004},"min":{"1min":0.36599999999999999,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":3.0270000000000001,"5min":3.0270000000000001,"15min":3.0270000000000001},"last":2.8540000000000001}]}]},{"osd":3,"up_from":98,"seq":420906795035,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3093972,"kb_used_data":3022220,"kb_used_omap":7,"kb_used_meta":71736,"kb_avail":90651180,"statfs":{"total":95995035648,"available":92826808320,"internally_reserved":0,"allocated":3094753280,"data_stored":3094239718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7576,"internal_metadata":73458280},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,2,0,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:52:13 2023","interfaces":[{"interface":"back","average":{"1min":0.371,"5min":0.371,"15min":0.371},"min":{"1min":0.253,"5min":0.253,"15min":0.253},"max":{"1min":0.71199999999999997,"5min":0.71199999999999997,"15min":0.71199999999999997},"last":10.619999999999999},{"interface":"front","average":{"1min":0.35799999999999998,"5min":0.35799999999999998,"15min":0.35799999999999998},"min":{"1min":0.24399999999999999,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":0.60499999999999998,"5min":0.60499999999999998,"15min":0.60499999999999998},"last":2.016}]},{"osd":1,"last update":"Sat Sep 23 20:52:13 2023","interfaces":[{"interface":"back","average":{"1min":0.36699999999999999,"5min":0.36699999999999999,"15min":0.36699999999999999},"min":{"1min":0.253,"5min":0.253,"15min":0.253},"max":{"1min":0.65500000000000003,"5min":0.65500000000000003,"15min":0.65500000000000003},"last":5.8010000000000002},{"interface":"front","average":{"1min":0.38900000000000001,"5min":0.38900000000000001,"15min":0.38900000000000001},"min":{"1min":0.28999999999999998,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":0.63100000000000001,"5min":0.63100000000000001,"15min":0.63100000000000001},"last":1.9410000000000001}]},{"osd":2,"last update":"Sat Sep 23 20:52:13 2023","interfaces":[{"interface":"back","average":{"1min":0.42499999999999999,"5min":0.42499999999999999,"15min":0.42499999999999999},"min":{"1min":0.30199999999999999,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":0.70399999999999996,"5min":0.70399999999999996,"15min":0.70399999999999996},"last":5.7759999999999998},{"interface":"front","average":{"1min":0.439,"5min":0.439,"15min":0.439},"min":{"1min":0.30499999999999999,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":0.67700000000000005,"5min":0.67700000000000005,"15min":0.67700000000000005},"last":2.101}]},{"osd":4,"last update":"Sat Sep 23 20:52:13 2023","interfaces":[{"interface":"back","average":{"1min":0.63200000000000001,"5min":0.63200000000000001,"15min":0.63200000000000001},"min":{"1min":0.42599999999999999,"5min":0.42599999999999999,"15min":0.42599999999999999},"max":{"1min":1.0960000000000001,"5min":1.0960000000000001,"15min":1.0960000000000001},"last":5.681},{"interface":"front","average":{"1min":0.627,"5min":0.627,"15min":0.627},"min":{"1min":0.437,"5min":0.437,"15min":0.437},"max":{"1min":0.80400000000000005,"5min":0.80400000000000005,"15min":0.80400000000000005},"last":1.9710000000000001}]},{"osd":5,"last update":"Sat Sep 23 20:52:13 2023","interfaces":[{"interface":"back","average":{"1min":0.64600000000000002,"5min":0.64600000000000002,"15min":0.64600000000000002},"min":{"1min":0.42599999999999999,"5min":0.42599999999999999,"15min":0.42599999999999999},"max":{"1min":1.1240000000000001,"5min":1.1240000000000001,"15min":1.1240000000000001},"last":5.7240000000000002},{"interface":"front","average":{"1min":0.63700000000000001,"5min":0.63700000000000001,"15min":0.63700000000000001},"min":{"1min":0.46899999999999997,"5min":0.46899999999999997,"15min":0.46899999999999997},"max":{"1min":0.83599999999999997,"5min":0.83599999999999997,"15min":0.83599999999999997},"last":0.47099999999999997}]},{"osd":6,"last update":"Sat Sep 23 20:52:13 2023","interfaces":[{"interface":"back","average":{"1min":0.69099999999999995,"5min":0.69099999999999995,"15min":0.69099999999999995},"min":{"1min":0.41599999999999998,"5min":0.41599999999999998,"15min":0.41599999999999998},"max":{"1min":0.86099999999999999,"5min":0.86099999999999999,"15min":0.86099999999999999},"last":0.41099999999999998},{"interface":"front","average":{"1min":0.71899999999999997,"5min":0.71899999999999997,"15min":0.71899999999999997},"min":{"1min":0.499,"5min":0.499,"15min":0.499},"max":{"1min":1.149,"5min":1.149,"15min":1.149},"last":5.8259999999999996}]},{"osd":7,"last update":"Sat Sep 23 20:52:13 2023","interfaces":[{"interface":"back","average":{"1min":0.69499999999999995,"5min":0.69499999999999995,"15min":0.69499999999999995},"min":{"1min":0.51700000000000002,"5min":0.51700000000000002,"15min":0.51700000000000002},"max":{"1min":1.177,"5min":1.177,"15min":1.177},"last":5.7510000000000003},{"interface":"front","average":{"1min":0.69399999999999995,"5min":0.69399999999999995,"15min":0.69399999999999995},"min":{"1min":0.46200000000000002,"5min":0.46200000000000002,"15min":0.46200000000000002},"max":{"1min":0.82999999999999996,"5min":0.82999999999999996,"15min":0.82999999999999996},"last":0.44700000000000001}]}]},{"osd":4,"up_from":31,"seq":133143986342,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1532780,"kb_used_data":1490532,"kb_used_omap":0,"kb_used_meta":42176,"kb_avail":92212372,"statfs":{"total":95995035648,"available":94425468928,"internally_reserved":0,"allocated":1526304768,"data_stored":1525828733,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":43188224},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,0,0,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:52:39 2023","interfaces":[{"interface":"back","average":{"1min":0.56999999999999995,"5min":0.64300000000000002,"15min":0.63700000000000001},"min":{"1min":0.249,"5min":0.249,"15min":0.249},"max":{"1min":0.92500000000000004,"5min":4.4050000000000002,"15min":4.4050000000000002},"last":3.6840000000000002},{"interface":"front","average":{"1min":0.56899999999999995,"5min":0.81299999999999994,"15min":0.88100000000000001},"min":{"1min":0.314,"5min":0.314,"15min":0.314},"max":{"1min":0.97899999999999998,"5min":5.2409999999999997,"15min":5.2409999999999997},"last":3.6739999999999999}]},{"osd":1,"last update":"Sat Sep 23 20:53:10 2023","interfaces":[{"interface":"back","average":{"1min":1.4490000000000001,"5min":0.79000000000000004,"15min":0.63600000000000001},"min":{"1min":0.23599999999999999,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":7.9859999999999998,"5min":7.9859999999999998,"15min":7.9859999999999998},"last":0.35399999999999998},{"interface":"front","average":{"1min":0.76700000000000002,"5min":0.64300000000000002,"15min":0.61299999999999999},"min":{"1min":0.23200000000000001,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":2.6459999999999999,"5min":2.6459999999999999,"15min":2.6459999999999999},"last":0.433}]},{"osd":2,"last update":"Sat Sep 23 20:52:42 2023","interfaces":[{"interface":"back","average":{"1min":0.64500000000000002,"5min":0.67700000000000005,"15min":0.68300000000000005},"min":{"1min":0.29099999999999998,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":1.4710000000000001,"5min":2.5030000000000001,"15min":2.5030000000000001},"last":2.294},{"interface":"front","average":{"1min":0.75600000000000001,"5min":0.77000000000000002,"15min":0.77200000000000002},"min":{"1min":0.39400000000000002,"5min":0.38200000000000001,"15min":0.38200000000000001},"max":{"1min":3.1520000000000001,"5min":4.3289999999999997,"15min":4.3289999999999997},"last":2.3210000000000002}]},{"osd":3,"last update":"Sat Sep 23 20:53:10 2023","interfaces":[{"interface":"back","average":{"1min":4.3310000000000004,"5min":1.387,"15min":0.89700000000000002},"min":{"1min":0.33800000000000002,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":31.562999999999999,"5min":31.562999999999999,"15min":31.562999999999999},"last":0.50900000000000001},{"interface":"front","average":{"1min":4.2560000000000002,"5min":1.4610000000000001,"15min":0.995},"min":{"1min":0.46500000000000002,"5min":0.41899999999999998,"15min":0.41899999999999998},"max":{"1min":31.574000000000002,"5min":31.574000000000002,"15min":31.574000000000002},"last":0.55100000000000005}]},{"osd":5,"last update":"Sat Sep 23 20:52:22 2023","interfaces":[{"interface":"back","average":{"1min":0.58399999999999996,"5min":0.52200000000000002,"15min":0.46500000000000002},"min":{"1min":0.35399999999999998,"5min":0.29099999999999998,"15min":0.251},"max":{"1min":0.83799999999999997,"5min":0.83799999999999997,"15min":0.83799999999999997},"last":0.498},{"interface":"front","average":{"1min":0.52100000000000002,"5min":0.51700000000000002,"15min":0.46899999999999997},"min":{"1min":0.311,"5min":0.311,"15min":0.28799999999999998},"max":{"1min":0.74199999999999999,"5min":0.79500000000000004,"15min":0.90000000000000002},"last":0.38700000000000001}]},{"osd":6,"last update":"Sat Sep 23 20:52:55 2023","interfaces":[{"interface":"back","average":{"1min":0.48999999999999999,"5min":0.51200000000000001,"15min":0.51100000000000001},"min":{"1min":0.317,"5min":0.30299999999999999,"15min":0.28100000000000003},"max":{"1min":0.71999999999999997,"5min":0.748,"15min":1.1319999999999999},"last":0.42099999999999999},{"interface":"front","average":{"1min":0.60399999999999998,"5min":0.59099999999999997,"15min":0.57199999999999995},"min":{"1min":0.29299999999999998,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":1.925,"5min":1.925,"15min":1.925},"last":0.46899999999999997}]},{"osd":7,"last update":"Sat Sep 23 20:53:10 2023","interfaces":[{"interface":"back","average":{"1min":0.64700000000000002,"5min":0.622,"15min":0.58099999999999996},"min":{"1min":0.34399999999999997,"5min":0.34399999999999997,"15min":0.32500000000000001},"max":{"1min":2.4039999999999999,"5min":2.4039999999999999,"15min":2.4039999999999999},"last":0.52500000000000002},{"interface":"front","average":{"1min":0.746,"5min":0.623,"15min":0.57999999999999996},"min":{"1min":0.39400000000000002,"5min":0.33500000000000002,"15min":0.32700000000000001},"max":{"1min":3.9420000000000002,"5min":3.9420000000000002,"15min":3.9420000000000002},"last":0.45100000000000001}]}]},{"osd":5,"up_from":0,"seq":167503724707,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5358,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1220083712,"data_stored":1220083712,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1893924864,"data_stored":1893924864,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":3048013824,"data_stored":3048013824,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":3119251456,"data_stored":3119251456,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1528889344,"data_stored":1528889344,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1851981824,"data_stored":1851981824,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":2135687168,"data_stored":2135687168,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T20:53:16.341 INFO:tasks.cephadm.ceph_manager.ceph:recovered! 2023-09-23T20:53:16.342 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T20:53:16.342 INFO:tasks.thrashosds.thrasher:inject_pause on osd.0 2023-09-23T20:53:16.342 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-09-23T20:53:16.342 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-09-23T20:53:16.342 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 config set filestore_inject_stall 3 2023-09-23T20:53:16.394 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:53:16.480 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:53:16.489 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:53:16.489 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:53:16.590 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:53:16.640 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:53:16.795 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0: {} 2023-09-23T20:53:16.846 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1: {} 2023-09-23T20:53:16.874 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:53:16 smithi026 ceph-mon[179688]: failed to encode map e117 with expected crc 2023-09-23T20:53:16.874 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:53:16 smithi026 ceph-mon[179688]: from='client.44807 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:53:16.874 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:53:16 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.f", "id": [4, 2]}]: dispatch 2023-09-23T20:53:16.874 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:53:16 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1d", "id": [1, 2]}]: dispatch 2023-09-23T20:53:16.874 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:53:16 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:53:16.874 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:53:16 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:53:16.879 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:53:16 smithi026 ceph-mon[175151]: failed to encode map e117 with expected crc 2023-09-23T20:53:16.879 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:53:16 smithi026 ceph-mon[175151]: from='client.44807 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:53:16.879 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:53:16 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.f", "id": [4, 2]}]: dispatch 2023-09-23T20:53:16.879 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:53:16 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1d", "id": [1, 2]}]: dispatch 2023-09-23T20:53:16.879 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:53:16 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:53:16.879 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:53:16 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:53:16.879 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:53:16 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/3609294917' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:53:16.879 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2: {} 2023-09-23T20:53:16.881 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 57 16 79343 79327 86.8989 54.75 0.0334399 0.0115007 2023-09-23T20:53:16.923 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3: {} 2023-09-23T20:53:16.934 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:53:16 smithi070 ceph-mon[168282]: failed to encode map e117 with expected crc 2023-09-23T20:53:16.934 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:53:16 smithi070 ceph-mon[168282]: from='client.44807 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:53:16.935 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:53:16 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.f", "id": [4, 2]}]: dispatch 2023-09-23T20:53:16.935 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:53:16 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1d", "id": [1, 2]}]: dispatch 2023-09-23T20:53:16.935 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:53:16 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:53:16.935 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:53:16 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:53:16.935 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:53:16 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/3609294917' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:53:16.976 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:53:16.979 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4: {} 2023-09-23T20:53:16.980 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:53:16.980 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:53:17.015 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T20:53:17.054 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6: {} 2023-09-23T20:53:17.081 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:53:17.099 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T20:53:17.136 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:53:16 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/3609294917' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:53:17.144 INFO:teuthology.orchestra.run.smithi026.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T20:53:17.144 INFO:teuthology.orchestra.run.smithi026.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T20:53:17.144 INFO:teuthology.orchestra.run.smithi026.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T20:53:17.144 INFO:teuthology.orchestra.run.smithi026.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T20:53:17.144 INFO:teuthology.orchestra.run.smithi026.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T20:53:17.144 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T20:53:17.144 INFO:teuthology.orchestra.run.smithi026.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T20:53:17.144 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T20:53:17.205 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:53:17.258 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:53:17.466 INFO:teuthology.orchestra.run.smithi070.stdout:ERROR: (22) Invalid argument 2023-09-23T20:53:17.467 INFO:teuthology.orchestra.run.smithi070.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:53:17.540 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:53:17.548 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:53:17.549 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:53:17.650 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:53:17.733 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:53:17.868 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_historic_ops 2023-09-23T20:53:17.881 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 58 16 80278 80262 86.4085 58.4375 0.0210206 0.0115675 2023-09-23T20:53:17.972 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:53:17.981 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:53:17.981 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:53:18.062 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T20:53:18.082 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:53:18.116 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:53:17 smithi070 ceph-mon[168282]: pgmap v281: 89 pgs: 89 active+clean; 4.7 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 60 MiB/s wr, 963 op/s 2023-09-23T20:53:18.116 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:53:17 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.f", "id": [4, 2]}]': finished 2023-09-23T20:53:18.116 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:53:17 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1d", "id": [1, 2]}]': finished 2023-09-23T20:53:18.117 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:53:17 smithi070 ceph-mon[168282]: osdmap e118: 8 total, 8 up, 7 in 2023-09-23T20:53:18.117 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:53:17 smithi070 ceph-mon[168282]: failed to encode map e118 with expected crc 2023-09-23T20:53:18.121 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:53:18.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:53:17 smithi026 ceph-mon[175151]: pgmap v281: 89 pgs: 89 active+clean; 4.7 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 60 MiB/s wr, 963 op/s 2023-09-23T20:53:18.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:53:17 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.f", "id": [4, 2]}]': finished 2023-09-23T20:53:18.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:53:17 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1d", "id": [1, 2]}]': finished 2023-09-23T20:53:18.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:53:17 smithi026 ceph-mon[175151]: osdmap e118: 8 total, 8 up, 7 in 2023-09-23T20:53:18.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:53:17 smithi026 ceph-mon[175151]: failed to encode map e118 with expected crc 2023-09-23T20:53:18.136 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:53:17 smithi026 ceph-mon[179688]: pgmap v281: 89 pgs: 89 active+clean; 4.7 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 60 MiB/s wr, 963 op/s 2023-09-23T20:53:18.136 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:53:17 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.f", "id": [4, 2]}]': finished 2023-09-23T20:53:18.136 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:53:17 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1d", "id": [1, 2]}]': finished 2023-09-23T20:53:18.136 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:53:17 smithi026 ceph-mon[179688]: osdmap e118: 8 total, 8 up, 7 in 2023-09-23T20:53:18.136 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:53:17 smithi026 ceph-mon[179688]: failed to encode map e118 with expected crc 2023-09-23T20:53:18.168 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:53:18.171 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:53:18.172 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:53:18.273 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:53:18.396 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:53:18.535 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:53:18.545 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:53:18.545 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:53:18.557 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:53:18.646 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:53:18.679 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:53:18.820 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:53:18.825 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:53:18.825 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:53:18.888 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 59 16 81184 81168 85.8991 56.625 0.0149932 0.0116363 2023-09-23T20:53:18.926 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:53:18.980 INFO:teuthology.orchestra.run.smithi026.stdout:{ 2023-09-23T20:53:18.980 INFO:teuthology.orchestra.run.smithi026.stdout: "success": "filestore_inject_stall = '3' (not observed, change may require restart) " 2023-09-23T20:53:18.980 INFO:teuthology.orchestra.run.smithi026.stdout:} 2023-09-23T20:53:18.983 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:53:18 smithi070 ceph-mon[168282]: osdmap e119: 8 total, 8 up, 7 in 2023-09-23T20:53:18.983 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:53:18 smithi070 ceph-mon[168282]: failed to encode map e119 with expected crc 2023-09-23T20:53:18.983 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:53:18 smithi070 ceph-mon[168282]: failed to encode map e119 with expected crc 2023-09-23T20:53:18.983 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:53:18 smithi070 ceph-mon[168282]: failed to encode map e119 with expected crc 2023-09-23T20:53:19.037 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:53:19.077 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:53:19.113 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:53:19.119 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:53:19.120 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:53:19.220 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:53:19.255 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:53:19.374 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:53:18 smithi026 ceph-mon[179688]: osdmap e119: 8 total, 8 up, 7 in 2023-09-23T20:53:19.374 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:53:18 smithi026 ceph-mon[179688]: failed to encode map e119 with expected crc 2023-09-23T20:53:19.375 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:53:18 smithi026 ceph-mon[179688]: failed to encode map e119 with expected crc 2023-09-23T20:53:19.375 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:53:18 smithi026 ceph-mon[179688]: failed to encode map e119 with expected crc 2023-09-23T20:53:19.377 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:53:18 smithi026 ceph-mon[175151]: osdmap e119: 8 total, 8 up, 7 in 2023-09-23T20:53:19.377 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:53:18 smithi026 ceph-mon[175151]: failed to encode map e119 with expected crc 2023-09-23T20:53:19.377 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:53:18 smithi026 ceph-mon[175151]: failed to encode map e119 with expected crc 2023-09-23T20:53:19.377 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:53:18 smithi026 ceph-mon[175151]: failed to encode map e119 with expected crc 2023-09-23T20:53:19.620 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T20:53:19.631 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:53:19 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[164029]: ::ffff:172.21.15.70 - - [23/Sep/2023:20:53:19] "GET /metrics HTTP/1.1" 200 36309 "" "Prometheus/2.43.0" 2023-09-23T20:53:19.885 INFO:tasks.radosbench.radosbench.0.smithi026.stdout:2023-09-23T20:53:19.885565+0000 min lat: 0.00125417 max lat: 0.125744 avg lat: 0.0116988 2023-09-23T20:53:19.885 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T20:53:19.885 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 60 16 82112 82096 85.4344 58 0.0191135 0.0116988 2023-09-23T20:53:19.910 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:53:19 smithi026 ceph-mon[175151]: pgmap v284: 89 pgs: 1 peering, 88 active+clean; 4.8 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 48 MiB/s wr, 766 op/s 2023-09-23T20:53:20.206 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:53:19 smithi026 ceph-mon[179688]: pgmap v284: 89 pgs: 1 peering, 88 active+clean; 4.8 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 48 MiB/s wr, 766 op/s 2023-09-23T20:53:20.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:53:19 smithi070 ceph-mon[168282]: pgmap v284: 89 pgs: 1 peering, 88 active+clean; 4.8 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 48 MiB/s wr, 766 op/s 2023-09-23T20:53:20.838 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:53:20.885 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 61 16 83058 83042 85.0033 59.125 0.00947273 0.0117592 2023-09-23T20:53:21.145 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:53:21.148 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:53:21.148 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:53:21.249 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:53:21.287 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:53:20 smithi070 ceph-mon[168282]: pgmap v285: 89 pgs: 1 peering, 88 active+clean; 4.9 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 47 MiB/s wr, 752 op/s 2023-09-23T20:53:21.291 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:53:21.381 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:53:20 smithi026 ceph-mon[179688]: pgmap v285: 89 pgs: 1 peering, 88 active+clean; 4.9 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 47 MiB/s wr, 752 op/s 2023-09-23T20:53:21.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:53:20 smithi026 ceph-mon[175151]: pgmap v285: 89 pgs: 1 peering, 88 active+clean; 4.9 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 47 MiB/s wr, 752 op/s 2023-09-23T20:53:21.885 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 62 16 83985 83969 84.567 57.9375 0.0128886 0.0118191 2023-09-23T20:53:22.016 INFO:teuthology.orchestra.run.smithi026.stderr:noscrub is set 2023-09-23T20:53:22.051 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:53:21 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/3721600166' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:53:22.052 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:53:21 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:53:22.052 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:53:21 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/3721600166' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:53:22.052 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:53:21 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:53:22.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:53:21 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/3721600166' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:53:22.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:53:21 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:53:22.370 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:53:22.373 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:53:22.374 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:53:22.474 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:53:22.511 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:53:22.560 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:53:22.563 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:53:22.563 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:53:22.664 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:53:22.694 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:53:22.885 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 63 16 84930 84914 84.1625 59.0625 0.0155562 0.0118761 2023-09-23T20:53:23.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:53:23 smithi026 ceph-mon[179688]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:53:23.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:53:23 smithi026 ceph-mon[179688]: failed to encode map e120 with expected crc 2023-09-23T20:53:23.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:53:23 smithi026 ceph-mon[179688]: failed to encode map e120 with expected crc 2023-09-23T20:53:23.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:53:23 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T20:53:23.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:53:23 smithi026 ceph-mon[179688]: osdmap e120: 8 total, 8 up, 7 in 2023-09-23T20:53:23.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:53:23 smithi026 ceph-mon[179688]: pgmap v287: 89 pgs: 89 active+clean; 5.0 GiB data, 15 GiB used, 610 GiB / 626 GiB avail; 65 MiB/s wr, 1.04k op/s; 3 B/s, 0 objects/s recovering 2023-09-23T20:53:23.134 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:53:23 smithi026 ceph-mon[175151]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:53:23.134 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:53:23 smithi026 ceph-mon[175151]: failed to encode map e120 with expected crc 2023-09-23T20:53:23.134 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:53:23 smithi026 ceph-mon[175151]: failed to encode map e120 with expected crc 2023-09-23T20:53:23.134 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:53:23 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T20:53:23.134 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:53:23 smithi026 ceph-mon[175151]: osdmap e120: 8 total, 8 up, 7 in 2023-09-23T20:53:23.134 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:53:23 smithi026 ceph-mon[175151]: pgmap v287: 89 pgs: 89 active+clean; 5.0 GiB data, 15 GiB used, 610 GiB / 626 GiB avail; 65 MiB/s wr, 1.04k op/s; 3 B/s, 0 objects/s recovering 2023-09-23T20:53:23.187 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:53:23.190 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:53:23.190 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:53:23.291 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:53:23.321 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:53:23 smithi070 ceph-mon[168282]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:53:23.321 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:53:23 smithi070 ceph-mon[168282]: failed to encode map e120 with expected crc 2023-09-23T20:53:23.321 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:53:23 smithi070 ceph-mon[168282]: failed to encode map e120 with expected crc 2023-09-23T20:53:23.321 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:53:23 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T20:53:23.321 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:53:23 smithi070 ceph-mon[168282]: osdmap e120: 8 total, 8 up, 7 in 2023-09-23T20:53:23.321 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:53:23 smithi070 ceph-mon[168282]: pgmap v287: 89 pgs: 89 active+clean; 5.0 GiB data, 15 GiB used, 610 GiB / 626 GiB avail; 65 MiB/s wr, 1.04k op/s; 3 B/s, 0 objects/s recovering 2023-09-23T20:53:23.324 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:53:23.372 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:53:23.376 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:53:23.376 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:53:23.452 INFO:teuthology.orchestra.run.smithi070.stdout:ERROR: (22) Invalid argument 2023-09-23T20:53:23.452 INFO:teuthology.orchestra.run.smithi070.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:53:23.477 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:53:23.531 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0: {} 2023-09-23T20:53:23.565 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:53:23.581 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1: {} 2023-09-23T20:53:23.620 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2: {} 2023-09-23T20:53:23.647 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:53:23.651 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:53:23.652 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:53:23.657 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3: {} 2023-09-23T20:53:23.689 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4: {} 2023-09-23T20:53:23.725 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T20:53:23.753 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:53:23.771 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6: {} 2023-09-23T20:53:23.798 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:53:23.810 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T20:53:23.819 INFO:teuthology.orchestra.run.smithi026.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T20:53:23.819 INFO:teuthology.orchestra.run.smithi026.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T20:53:23.819 INFO:teuthology.orchestra.run.smithi026.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T20:53:23.819 INFO:teuthology.orchestra.run.smithi026.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T20:53:23.819 INFO:teuthology.orchestra.run.smithi026.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T20:53:23.820 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T20:53:23.820 INFO:teuthology.orchestra.run.smithi026.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T20:53:23.820 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T20:53:23.870 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_ops_in_flight 2023-09-23T20:53:23.885 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 64 16 85839 85823 83.7354 56.8125 0.0223767 0.0119368 2023-09-23T20:53:24.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:53:24 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/2261280909' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:53:24.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:53:24 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/2261280909' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:53:24.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:53:24 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/2261280909' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:53:24.231 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:53:24.236 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:53:24.236 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:53:24.337 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:53:24.422 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:53:24.497 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:53:24.504 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:53:24.505 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:53:24.506 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 6, 7] out_osds: [5] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-09-23T20:53:24.506 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T20:53:24.506 INFO:tasks.thrashosds.thrasher:Removing osd 4, in_osds are: [0, 1, 2, 3, 4, 6, 7] 2023-09-23T20:53:24.507 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph osd out 4 2023-09-23T20:53:24.558 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:53:24.569 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T20:53:24.605 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:53:24.640 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:53:24.691 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:53:24.694 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:53:24.694 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:53:24.795 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:53:24.828 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:53:24.868 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:53:24.872 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:53:24.872 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:53:24.886 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 65 16 86741 86725 83.3147 56.375 0.024233 0.0119965 2023-09-23T20:53:24.973 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:53:25.011 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:53:25.370 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:53:25.372 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:53:25.373 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:53:25.415 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:53:25.473 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:53:25.491 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:53:25.509 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:53:25.620 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:53:25.624 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:53:25.625 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:53:25.726 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:53:25.812 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:53:25.839 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:53:25 smithi070 ceph-mon[168282]: pgmap v288: 89 pgs: 89 active+clean; 5.2 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 70 MiB/s wr, 1.13k op/s; 2 B/s, 0 objects/s recovering 2023-09-23T20:53:25.844 INFO:teuthology.orchestra.run.smithi070.stdout:{ 2023-09-23T20:53:25.844 INFO:teuthology.orchestra.run.smithi070.stdout: "ops": [], 2023-09-23T20:53:25.845 INFO:teuthology.orchestra.run.smithi070.stdout: "num_ops": 0 2023-09-23T20:53:25.845 INFO:teuthology.orchestra.run.smithi070.stdout:} 2023-09-23T20:53:25.890 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 66 16 87658 87642 82.9179 57.3125 0.0171799 0.0120559 2023-09-23T20:53:25.993 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:53:25 smithi026 ceph-mon[175151]: pgmap v288: 89 pgs: 89 active+clean; 5.2 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 70 MiB/s wr, 1.13k op/s; 2 B/s, 0 objects/s recovering 2023-09-23T20:53:25.994 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:53:25 smithi026 ceph-mon[179688]: pgmap v288: 89 pgs: 89 active+clean; 5.2 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 70 MiB/s wr, 1.13k op/s; 2 B/s, 0 objects/s recovering 2023-09-23T20:53:26.011 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:53:26.021 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:53:26.021 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:53:26.122 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:53:26.165 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:53:26.214 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:53:26.218 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:53:26.218 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:53:26.258 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_blocked_ops 2023-09-23T20:53:26.303 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T20:53:26.320 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:53:26.373 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:53:26.421 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:53:26.424 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:53:26.425 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:53:26.525 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:53:26.677 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:53:26.835 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:53:26.845 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:53:26.845 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:53:26.890 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 67 16 88489 88473 82.453 51.9375 0.0239432 0.0121216 2023-09-23T20:53:26.947 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:53:26.966 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:53:27.049 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:53:27.230 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:53:27.237 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:53:27.237 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:53:27.338 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:53:27.375 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:53:27.416 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:53:27.424 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:53:27.424 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:53:27.525 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:53:27.590 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:53:27.632 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:53:27.716 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:53:27.732 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:53:27.733 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:53:27.833 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:53:27.885 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:53:27 smithi026 ceph-mon[175151]: pgmap v289: 89 pgs: 89 active+clean; 5.2 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 57 MiB/s wr, 905 op/s; 2 B/s, 0 objects/s recovering 2023-09-23T20:53:27.885 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:53:27 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mon-a[175127]: 2023-09-23T20:53:27.847+0000 7f1e2d487700 -1 mon.a@0(leader).osd e120 definitely_dead 0 2023-09-23T20:53:27.885 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:53:27 smithi026 ceph-mon[179688]: pgmap v289: 89 pgs: 89 active+clean; 5.2 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 57 MiB/s wr, 905 op/s; 2 B/s, 0 objects/s recovering 2023-09-23T20:53:27.890 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 68 16 89240 89224 81.9311 46.9375 0.0155333 0.0122002 2023-09-23T20:53:27.903 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:53:27.954 INFO:teuthology.orchestra.run.smithi026.stderr:marked out osd.4. 2023-09-23T20:53:27.997 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:53:28.001 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:53:28.002 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:53:28.012 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:53:27 smithi070 ceph-mon[168282]: pgmap v289: 89 pgs: 89 active+clean; 5.2 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 57 MiB/s wr, 905 op/s; 2 B/s, 0 objects/s recovering 2023-09-23T20:53:28.103 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:53:28.131 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:53:28.198 INFO:teuthology.orchestra.run.smithi070.stdout:{ 2023-09-23T20:53:28.199 INFO:teuthology.orchestra.run.smithi070.stdout: "ops": [], 2023-09-23T20:53:28.199 INFO:teuthology.orchestra.run.smithi070.stdout: "complaint_time": 30, 2023-09-23T20:53:28.199 INFO:teuthology.orchestra.run.smithi070.stdout: "num_blocked_ops": 0 2023-09-23T20:53:28.199 INFO:teuthology.orchestra.run.smithi070.stdout:} 2023-09-23T20:53:28.357 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:53:28.359 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:53:28.360 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:53:28.460 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:53:28.489 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:53:28.544 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_historic_ops 2023-09-23T20:53:28.600 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:53:28.603 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:53:28.603 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:53:28.704 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:53:28.730 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:53:28.765 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:53:28.767 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:53:28.767 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:53:28.868 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:53:28.881 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:53:28 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/2921442280' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["4"]}]: dispatch 2023-09-23T20:53:28.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:53:28 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["4"]}]: dispatch 2023-09-23T20:53:28.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:53:28 smithi026 ceph-mon[179688]: Client client.admin marked osd.4 out, while it was still marked up 2023-09-23T20:53:28.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:53:28 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["4"]}]': finished 2023-09-23T20:53:28.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:53:28 smithi026 ceph-mon[179688]: osdmap e121: 8 total, 8 up, 6 in 2023-09-23T20:53:28.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:53:28 smithi026 ceph-mon[179688]: failed to encode map e121 with expected crc 2023-09-23T20:53:28.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:53:28 smithi026 ceph-mon[179688]: failed to encode map e121 with expected crc 2023-09-23T20:53:28.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:53:28 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/249712225' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:53:28.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:53:28 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:53:28.883 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:53:28 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/2921442280' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["4"]}]: dispatch 2023-09-23T20:53:28.883 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:53:28 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["4"]}]: dispatch 2023-09-23T20:53:28.883 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:53:28 smithi026 ceph-mon[175151]: Client client.admin marked osd.4 out, while it was still marked up 2023-09-23T20:53:28.883 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:53:28 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["4"]}]': finished 2023-09-23T20:53:28.883 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:53:28 smithi026 ceph-mon[175151]: osdmap e121: 8 total, 8 up, 6 in 2023-09-23T20:53:28.883 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:53:28 smithi026 ceph-mon[175151]: failed to encode map e121 with expected crc 2023-09-23T20:53:28.883 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:53:28 smithi026 ceph-mon[175151]: failed to encode map e121 with expected crc 2023-09-23T20:53:28.883 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:53:28 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/249712225' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:53:28.883 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:53:28 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:53:28.890 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 69 16 89292 89276 80.7917 3.25 0.00412522 0.0122067 2023-09-23T20:53:28.900 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:53:28.935 INFO:teuthology.orchestra.run.smithi026.stderr:nodeep-scrub is set 2023-09-23T20:53:28.959 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:53:28.962 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:53:28.962 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:53:29.063 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:53:29.087 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:53:28 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/2921442280' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["4"]}]: dispatch 2023-09-23T20:53:29.088 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:53:28 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["4"]}]: dispatch 2023-09-23T20:53:29.088 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:53:28 smithi070 ceph-mon[168282]: Client client.admin marked osd.4 out, while it was still marked up 2023-09-23T20:53:29.088 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:53:28 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["4"]}]': finished 2023-09-23T20:53:29.088 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:53:28 smithi070 ceph-mon[168282]: osdmap e121: 8 total, 8 up, 6 in 2023-09-23T20:53:29.088 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:53:28 smithi070 ceph-mon[168282]: failed to encode map e121 with expected crc 2023-09-23T20:53:29.088 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:53:28 smithi070 ceph-mon[168282]: failed to encode map e121 with expected crc 2023-09-23T20:53:29.088 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:53:28 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/249712225' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:53:29.088 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:53:28 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:53:29.090 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:53:29.123 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:53:29.125 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:53:29.125 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:53:29.132 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0: {} 2023-09-23T20:53:29.140 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:53:29.152 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1: {} 2023-09-23T20:53:29.171 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2: {} 2023-09-23T20:53:29.190 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3: {} 2023-09-23T20:53:29.210 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4: {} 2023-09-23T20:53:29.226 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:53:29.229 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T20:53:29.248 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6: {} 2023-09-23T20:53:29.256 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:53:29.269 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T20:53:29.271 INFO:teuthology.orchestra.run.smithi026.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T20:53:29.271 INFO:teuthology.orchestra.run.smithi026.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T20:53:29.271 INFO:teuthology.orchestra.run.smithi026.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T20:53:29.271 INFO:teuthology.orchestra.run.smithi026.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T20:53:29.271 INFO:teuthology.orchestra.run.smithi026.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T20:53:29.271 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T20:53:29.271 INFO:teuthology.orchestra.run.smithi026.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T20:53:29.272 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T20:53:29.345 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:53:29.348 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:53:29.349 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:53:29.450 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:53:29.478 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:53:29.657 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:53:29.659 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:53:29.660 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:53:29.735 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:53:29 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[164029]: ::ffff:172.21.15.70 - - [23/Sep/2023:20:53:29] "GET /metrics HTTP/1.1" 200 36309 "" "Prometheus/2.43.0" 2023-09-23T20:53:29.761 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:53:29.787 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:53:29.820 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:53:29.823 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:53:29.823 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:53:29.891 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 70 16 89292 89276 79.6384 0 - 0.0122067 2023-09-23T20:53:29.924 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:53:29.954 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:53:29 smithi070 ceph-mon[168282]: pgmap v291: 89 pgs: 89 active+clean; 5.3 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 56 MiB/s wr, 903 op/s; 2 B/s, 0 objects/s recovering 2023-09-23T20:53:29.954 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:53:29 smithi070 ceph-mon[168282]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:53:29.954 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:53:29 smithi070 ceph-mon[168282]: failed to encode map e122 with expected crc 2023-09-23T20:53:29.955 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:53:29 smithi070 ceph-mon[168282]: failed to encode map e122 with expected crc 2023-09-23T20:53:29.955 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:53:29 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T20:53:29.955 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:53:29 smithi070 ceph-mon[168282]: osdmap e122: 8 total, 8 up, 6 in 2023-09-23T20:53:29.955 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:53:29 smithi070 ceph-mon[168282]: failed to encode map e122 with expected crc 2023-09-23T20:53:29.955 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:53:29 smithi070 ceph-mon[168282]: failed to encode map e122 with expected crc 2023-09-23T20:53:29.955 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:53:29 smithi070 ceph-mon[168282]: failed to encode map e122 with expected crc 2023-09-23T20:53:29.955 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:53:29 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/3457334267' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:53:29.960 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:53:30.053 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:53:30.057 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:53:30.058 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:53:30.131 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:53:29 smithi026 ceph-mon[175151]: pgmap v291: 89 pgs: 89 active+clean; 5.3 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 56 MiB/s wr, 903 op/s; 2 B/s, 0 objects/s recovering 2023-09-23T20:53:30.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:53:29 smithi026 ceph-mon[175151]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:53:30.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:53:29 smithi026 ceph-mon[175151]: failed to encode map e122 with expected crc 2023-09-23T20:53:30.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:53:29 smithi026 ceph-mon[175151]: failed to encode map e122 with expected crc 2023-09-23T20:53:30.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:53:29 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T20:53:30.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:53:29 smithi026 ceph-mon[175151]: osdmap e122: 8 total, 8 up, 6 in 2023-09-23T20:53:30.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:53:29 smithi026 ceph-mon[175151]: failed to encode map e122 with expected crc 2023-09-23T20:53:30.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:53:29 smithi026 ceph-mon[175151]: failed to encode map e122 with expected crc 2023-09-23T20:53:30.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:53:29 smithi026 ceph-mon[175151]: failed to encode map e122 with expected crc 2023-09-23T20:53:30.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:53:29 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/3457334267' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:53:30.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:53:29 smithi026 ceph-mon[179688]: pgmap v291: 89 pgs: 89 active+clean; 5.3 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 56 MiB/s wr, 903 op/s; 2 B/s, 0 objects/s recovering 2023-09-23T20:53:30.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:53:29 smithi026 ceph-mon[179688]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:53:30.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:53:29 smithi026 ceph-mon[179688]: failed to encode map e122 with expected crc 2023-09-23T20:53:30.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:53:29 smithi026 ceph-mon[179688]: failed to encode map e122 with expected crc 2023-09-23T20:53:30.134 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:53:29 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T20:53:30.134 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:53:29 smithi026 ceph-mon[179688]: osdmap e122: 8 total, 8 up, 6 in 2023-09-23T20:53:30.134 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:53:29 smithi026 ceph-mon[179688]: failed to encode map e122 with expected crc 2023-09-23T20:53:30.134 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:53:29 smithi026 ceph-mon[179688]: failed to encode map e122 with expected crc 2023-09-23T20:53:30.134 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:53:29 smithi026 ceph-mon[179688]: failed to encode map e122 with expected crc 2023-09-23T20:53:30.134 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:53:29 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/3457334267' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:53:30.159 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:53:30.203 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:53:30.298 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:53:30.301 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:53:30.301 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:53:30.402 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:53:30.426 INFO:teuthology.orchestra.run.smithi070.stdout:ERROR: (22) Invalid argument 2023-09-23T20:53:30.426 INFO:teuthology.orchestra.run.smithi070.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:53:30.447 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:53:30.507 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:53:30.510 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:53:30.510 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:53:30.611 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:53:30.648 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:53:30.765 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:53:30.769 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:53:30.769 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:53:30.807 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_ops_in_flight 2023-09-23T20:53:30.870 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:53:30.890 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 71 16 89750 89734 78.9204 14.3125 0.0145095 0.0126652 2023-09-23T20:53:30.928 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:53:30.979 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:53:30.983 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:53:30.983 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:53:31.084 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:53:31.178 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:53:31.253 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:53:31.263 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:53:31.264 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:53:30 smithi026 ceph-mon[175151]: failed to encode map e123 with expected crc 2023-09-23T20:53:31.264 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:53:30 smithi026 ceph-mon[175151]: failed to encode map e123 with expected crc 2023-09-23T20:53:31.264 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:53:30 smithi026 ceph-mon[175151]: osdmap e123: 8 total, 8 up, 6 in 2023-09-23T20:53:31.264 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:53:30 smithi026 ceph-mon[179688]: failed to encode map e123 with expected crc 2023-09-23T20:53:31.264 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:53:30 smithi026 ceph-mon[179688]: failed to encode map e123 with expected crc 2023-09-23T20:53:31.264 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:53:30 smithi026 ceph-mon[179688]: osdmap e123: 8 total, 8 up, 6 in 2023-09-23T20:53:31.265 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:53:31.365 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:53:31.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:53:30 smithi070 ceph-mon[168282]: failed to encode map e123 with expected crc 2023-09-23T20:53:31.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:53:30 smithi070 ceph-mon[168282]: failed to encode map e123 with expected crc 2023-09-23T20:53:31.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:53:30 smithi070 ceph-mon[168282]: osdmap e123: 8 total, 8 up, 6 in 2023-09-23T20:53:31.397 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T20:53:31.399 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:53:31.448 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:53:31.451 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:53:31.451 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:53:31.552 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:53:31.562 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:53:31.673 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:53:31.737 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T20:53:31.755 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:53:31.769 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:53:31.770 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:53:31.870 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:53:31.900 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 72 16 90398 90382 78.3861 40.5 0.0252674 0.0127524 2023-09-23T20:53:31.906 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:53:31.956 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:53:31 smithi026 ceph-mon[175151]: failed to encode map e123 with expected crc 2023-09-23T20:53:31.956 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:53:31 smithi026 ceph-mon[175151]: pgmap v294: 89 pgs: 2 unknown, 15 peering, 72 active+clean; 5.4 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 43 MiB/s wr, 692 op/s 2023-09-23T20:53:31.956 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:53:31 smithi026 ceph-mon[175151]: failed to encode map e123 with expected crc 2023-09-23T20:53:31.956 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:53:31 smithi026 ceph-mon[175151]: Health check failed: Reduced data availability: 5 pgs inactive, 6 pgs peering (PG_AVAILABILITY) 2023-09-23T20:53:31.956 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:53:31 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:53:31.956 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:53:31 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:53:31.962 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:53:31 smithi026 ceph-mon[179688]: failed to encode map e123 with expected crc 2023-09-23T20:53:31.963 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:53:31 smithi026 ceph-mon[179688]: pgmap v294: 89 pgs: 2 unknown, 15 peering, 72 active+clean; 5.4 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 43 MiB/s wr, 692 op/s 2023-09-23T20:53:31.963 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:53:31 smithi026 ceph-mon[179688]: failed to encode map e123 with expected crc 2023-09-23T20:53:32.050 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:53:32.052 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:53:31 smithi070 ceph-mon[168282]: failed to encode map e123 with expected crc 2023-09-23T20:53:32.052 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:53:31 smithi070 ceph-mon[168282]: pgmap v294: 89 pgs: 2 unknown, 15 peering, 72 active+clean; 5.4 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 43 MiB/s wr, 692 op/s 2023-09-23T20:53:32.053 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:53:31 smithi070 ceph-mon[168282]: failed to encode map e123 with expected crc 2023-09-23T20:53:32.053 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:53:31 smithi070 ceph-mon[168282]: Health check failed: Reduced data availability: 5 pgs inactive, 6 pgs peering (PG_AVAILABILITY) 2023-09-23T20:53:32.053 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:53:31 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:53:32.053 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:53:31 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:53:32.053 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:53:31 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:53:32.053 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:53:32.054 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:53:32.155 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:53:32.235 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:53:31 smithi026 ceph-mon[179688]: Health check failed: Reduced data availability: 5 pgs inactive, 6 pgs peering (PG_AVAILABILITY) 2023-09-23T20:53:32.235 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:53:31 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:53:32.235 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:53:31 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:53:32.235 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:53:31 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:53:32.235 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:53:32.237 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:53:31 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:53:32.260 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:53:32.321 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:53:32.328 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:53:32.328 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:53:32.429 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:53:32.480 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:53:32.583 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:53:32.585 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:53:32.586 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:53:32.686 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:53:32.780 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:53:32.819 INFO:teuthology.orchestra.run.smithi070.stdout:ERROR: (22) Invalid argument 2023-09-23T20:53:32.819 INFO:teuthology.orchestra.run.smithi070.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:53:32.892 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 73 16 91037 91021 77.8598 39.9375 0.0230237 0.0128376 2023-09-23T20:53:33.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:53:32 smithi070 ceph-mon[168282]: pgmap v295: 89 pgs: 1 active+recovering+undersized+remapped, 2 unknown, 15 peering, 71 active+clean; 5.5 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 43 MiB/s wr, 684 op/s; 5507/269127 objects misplaced (2.046%); 267 KiB/s, 1 keys/s, 5 objects/s recovering 2023-09-23T20:53:33.127 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:53:33.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:53:32 smithi026 ceph-mon[179688]: pgmap v295: 89 pgs: 1 active+recovering+undersized+remapped, 2 unknown, 15 peering, 71 active+clean; 5.5 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 43 MiB/s wr, 684 op/s; 5507/269127 objects misplaced (2.046%); 267 KiB/s, 1 keys/s, 5 objects/s recovering 2023-09-23T20:53:33.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:53:32 smithi026 ceph-mon[175151]: pgmap v295: 89 pgs: 1 active+recovering+undersized+remapped, 2 unknown, 15 peering, 71 active+clean; 5.5 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 43 MiB/s wr, 684 op/s; 5507/269127 objects misplaced (2.046%); 267 KiB/s, 1 keys/s, 5 objects/s recovering 2023-09-23T20:53:33.133 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:53:33.134 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:53:33.235 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:53:33.241 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_blocked_ops 2023-09-23T20:53:33.270 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:53:33.328 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:53:33.336 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:53:33.336 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:53:33.435 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 6, 7] out_osds: [5, 4] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-09-23T20:53:33.435 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T20:53:33.435 INFO:tasks.thrashosds.thrasher:inject_pause on osd.4 2023-09-23T20:53:33.435 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-09-23T20:53:33.436 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-09-23T20:53:33.436 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 config set filestore_inject_stall 3 2023-09-23T20:53:33.437 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:53:33.481 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:53:33.546 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:53:33.549 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:53:33.550 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:53:33.650 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:53:33.723 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:53:33.892 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 74 16 91716 91700 77.3815 42.4375 0.012832 0.0129178 2023-09-23T20:53:34.004 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:53:34.113 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:53:34.123 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:53:34.123 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:53:34.224 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:53:34.266 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:53:34.306 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:53:34.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:53:34 smithi026 ceph-mon[175151]: Health check failed: Degraded data redundancy: 16604/273180 objects degraded (6.078%), 3 pgs degraded (PG_DEGRADED) 2023-09-23T20:53:34.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:53:34 smithi026 ceph-mon[175151]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 5 pgs inactive, 6 pgs peering) 2023-09-23T20:53:34.385 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:53:34 smithi026 ceph-mon[179688]: Health check failed: Degraded data redundancy: 16604/273180 objects degraded (6.078%), 3 pgs degraded (PG_DEGRADED) 2023-09-23T20:53:34.385 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:53:34 smithi026 ceph-mon[179688]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 5 pgs inactive, 6 pgs peering) 2023-09-23T20:53:34.505 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:53:34.563 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:53:34 smithi070 ceph-mon[168282]: Health check failed: Degraded data redundancy: 16604/273180 objects degraded (6.078%), 3 pgs degraded (PG_DEGRADED) 2023-09-23T20:53:34.563 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:53:34 smithi070 ceph-mon[168282]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 5 pgs inactive, 6 pgs peering) 2023-09-23T20:53:34.629 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:53:34.633 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:53:34.634 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:53:34.735 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:53:34.782 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:53:34.892 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 75 16 92364 92348 76.89 40.5 0.0533344 0.0129993 2023-09-23T20:53:34.986 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:53:34.988 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:53:34.988 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:53:35.089 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:53:35.124 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:53:35.273 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:53:35.278 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:53:35.278 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:53:35.379 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:53:35.420 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:53:35.486 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:53:35.493 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:53:35.494 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:53:35.595 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:53:35.686 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:53:35.731 INFO:teuthology.orchestra.run.smithi070.stdout:ERROR: (22) Invalid argument 2023-09-23T20:53:35.731 INFO:teuthology.orchestra.run.smithi070.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:53:35.808 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:53:35.816 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:53:35.816 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:53:35.895 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 76 16 92995 92979 76.3946 39.4375 0.0375167 0.0130832 2023-09-23T20:53:35.917 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:53:35.957 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:53:35.977 INFO:teuthology.orchestra.run.smithi070.stdout:{ 2023-09-23T20:53:35.977 INFO:teuthology.orchestra.run.smithi070.stdout: "success": "filestore_inject_stall = '3' (not observed, change may require restart) " 2023-09-23T20:53:35.977 INFO:teuthology.orchestra.run.smithi070.stdout:} 2023-09-23T20:53:35.988 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:53:35 smithi026 ceph-mon[175151]: pgmap v296: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 84 active+clean; 5.6 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 37 MiB/s wr, 597 op/s; 16604/273180 objects degraded (6.078%); 11007/273180 objects misplaced (4.029%); 1.7 MiB/s, 1 keys/s, 28 objects/s recovering 2023-09-23T20:53:35.990 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:53:35 smithi026 ceph-mon[179688]: pgmap v296: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 84 active+clean; 5.6 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 37 MiB/s wr, 597 op/s; 16604/273180 objects degraded (6.078%); 11007/273180 objects misplaced (4.029%); 1.7 MiB/s, 1 keys/s, 28 objects/s recovering 2023-09-23T20:53:36.001 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:53:36.004 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:53:35 smithi070 ceph-mon[168282]: pgmap v296: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 84 active+clean; 5.6 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 37 MiB/s wr, 597 op/s; 16604/273180 objects degraded (6.078%); 11007/273180 objects misplaced (4.029%); 1.7 MiB/s, 1 keys/s, 28 objects/s recovering 2023-09-23T20:53:36.007 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:53:36.007 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:53:36.108 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:53:36.141 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:53:36.542 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:53:36.545 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:53:36.546 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:53:36.565 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_historic_ops 2023-09-23T20:53:36.644 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0: {} 2023-09-23T20:53:36.647 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:53:36.699 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1: {} 2023-09-23T20:53:36.746 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:53:36.765 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2: {} 2023-09-23T20:53:36.802 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3: {} 2023-09-23T20:53:36.859 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4: {} 2023-09-23T20:53:36.878 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:53:36.885 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:53:36.886 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:53:36.895 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 77 16 93590 93574 75.8858 37.1875 0.0309584 0.0131711 2023-09-23T20:53:36.900 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T20:53:36.925 INFO:teuthology.orchestra.run.smithi026.stderr:noscrub is unset 2023-09-23T20:53:36.930 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6: {} 2023-09-23T20:53:36.968 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T20:53:36.973 INFO:teuthology.orchestra.run.smithi026.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T20:53:36.973 INFO:teuthology.orchestra.run.smithi026.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T20:53:36.973 INFO:teuthology.orchestra.run.smithi026.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T20:53:36.973 INFO:teuthology.orchestra.run.smithi026.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T20:53:36.973 INFO:teuthology.orchestra.run.smithi026.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T20:53:36.973 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T20:53:36.974 INFO:teuthology.orchestra.run.smithi026.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T20:53:36.974 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T20:53:36.986 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:53:37.006 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:53:36 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/2715451942' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:53:37.007 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:53:36 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:53:37.007 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:53:36 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/3358483901' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:53:37.007 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:53:36 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/2715451942' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:53:37.007 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:53:36 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:53:37.007 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:53:36 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/3358483901' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:53:37.022 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:53:37.065 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:53:37.071 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:53:37.072 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:53:37.172 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:53:37.213 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:53:36 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/2715451942' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:53:37.213 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:53:36 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:53:37.213 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:53:36 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/3358483901' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:53:37.214 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:53:37.252 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:53:37.256 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:53:37.256 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:53:37.273 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:53:37.356 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:53:37.476 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:53:37.663 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:53:37.668 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:53:37.668 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:53:37.769 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:53:37.808 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:53:37.895 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 78 16 94234 94218 75.4292 40.25 0.0212733 0.0132516 2023-09-23T20:53:38.021 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:53:38.024 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:53:38.024 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:53:38.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:53:37 smithi070 ceph-mon[168282]: pgmap v297: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 84 active+clean; 5.6 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 34 MiB/s wr, 545 op/s; 16604/274998 objects degraded (6.038%); 10836/274998 objects misplaced (3.940%); 2.7 MiB/s, 1 keys/s, 43 objects/s recovering 2023-09-23T20:53:38.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:53:37 smithi070 ceph-mon[168282]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:53:38.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:53:37 smithi070 ceph-mon[168282]: failed to encode map e124 with expected crc 2023-09-23T20:53:38.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:53:37 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T20:53:38.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:53:37 smithi070 ceph-mon[168282]: osdmap e124: 8 total, 8 up, 6 in 2023-09-23T20:53:38.125 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:53:37 smithi070 ceph-mon[168282]: failed to encode map e124 with expected crc 2023-09-23T20:53:38.125 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:53:37 smithi070 ceph-mon[168282]: failed to encode map e124 with expected crc 2023-09-23T20:53:38.125 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:53:38.131 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:53:37 smithi026 ceph-mon[175151]: pgmap v297: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 84 active+clean; 5.6 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 34 MiB/s wr, 545 op/s; 16604/274998 objects degraded (6.038%); 10836/274998 objects misplaced (3.940%); 2.7 MiB/s, 1 keys/s, 43 objects/s recovering 2023-09-23T20:53:38.131 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:53:37 smithi026 ceph-mon[175151]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:53:38.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:53:37 smithi026 ceph-mon[175151]: failed to encode map e124 with expected crc 2023-09-23T20:53:38.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:53:37 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T20:53:38.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:53:37 smithi026 ceph-mon[175151]: osdmap e124: 8 total, 8 up, 6 in 2023-09-23T20:53:38.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:53:37 smithi026 ceph-mon[175151]: failed to encode map e124 with expected crc 2023-09-23T20:53:38.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:53:37 smithi026 ceph-mon[175151]: failed to encode map e124 with expected crc 2023-09-23T20:53:38.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:53:37 smithi026 ceph-mon[179688]: pgmap v297: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 84 active+clean; 5.6 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 34 MiB/s wr, 545 op/s; 16604/274998 objects degraded (6.038%); 10836/274998 objects misplaced (3.940%); 2.7 MiB/s, 1 keys/s, 43 objects/s recovering 2023-09-23T20:53:38.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:53:37 smithi026 ceph-mon[179688]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:53:38.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:53:37 smithi026 ceph-mon[179688]: failed to encode map e124 with expected crc 2023-09-23T20:53:38.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:53:37 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T20:53:38.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:53:37 smithi026 ceph-mon[179688]: osdmap e124: 8 total, 8 up, 6 in 2023-09-23T20:53:38.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:53:37 smithi026 ceph-mon[179688]: failed to encode map e124 with expected crc 2023-09-23T20:53:38.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:53:37 smithi026 ceph-mon[179688]: failed to encode map e124 with expected crc 2023-09-23T20:53:38.223 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:53:38.279 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:53:38.292 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:53:38.292 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:53:38.393 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:53:38.501 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:53:38.566 INFO:teuthology.orchestra.run.smithi070.stdout:{ 2023-09-23T20:53:38.566 INFO:teuthology.orchestra.run.smithi070.stdout: "size": 20, 2023-09-23T20:53:38.566 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 600, 2023-09-23T20:53:38.566 INFO:teuthology.orchestra.run.smithi070.stdout: "ops": [ 2023-09-23T20:53:38.567 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:38.567 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.34797.0:8389 6.a 6:5e21d836:::benchmark_data_smithi026_223868_object8388:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e107)", 2023-09-23T20:53:38.567 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T20:52:23.076717+0000", 2023-09-23T20:53:38.567 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 75.485971301999996, 2023-09-23T20:53:38.567 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.031400683999999998, 2023-09-23T20:53:38.567 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T20:53:38.567 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:53:38.567 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T20:53:38.567 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.34797", 2023-09-23T20:53:38.567 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T20:53:38.567 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 8389 2023-09-23T20:53:38.567 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:38.567 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T20:53:38.568 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:38.568 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T20:53:38.568 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:52:23.076717+0000", 2023-09-23T20:53:38.568 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T20:53:38.568 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:38.568 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:38.568 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T20:53:38.568 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:52:23.076717+0000", 2023-09-23T20:53:38.568 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T20:53:38.568 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:38.568 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:38.568 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T20:53:38.569 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:52:23.076717+0000", 2023-09-23T20:53:38.569 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.999999 2023-09-23T20:53:38.569 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:38.569 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:38.569 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T20:53:38.569 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:52:23.076785+0000", 2023-09-23T20:53:38.569 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.8429999999999994e-05 2023-09-23T20:53:38.569 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:38.569 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:38.569 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T20:53:38.569 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:52:23.076795+0000", 2023-09-23T20:53:38.569 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.6549999999999999e-06 2023-09-23T20:53:38.569 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:38.570 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:38.570 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T20:53:38.570 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:52:23.076799+0000", 2023-09-23T20:53:38.570 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.2880000000000003e-06 2023-09-23T20:53:38.570 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:38.570 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:38.570 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T20:53:38.570 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:52:23.076832+0000", 2023-09-23T20:53:38.570 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.3049000000000002e-05 2023-09-23T20:53:38.570 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:38.570 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:38.570 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T20:53:38.571 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:52:23.076909+0000", 2023-09-23T20:53:38.571 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.6923999999999997e-05 2023-09-23T20:53:38.571 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:38.571 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:38.571 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 0,2", 2023-09-23T20:53:38.571 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:52:23.077003+0000", 2023-09-23T20:53:38.571 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.4267999999999999e-05 2023-09-23T20:53:38.571 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:38.571 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:38.571 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T20:53:38.571 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:52:23.077489+0000", 2023-09-23T20:53:38.571 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00048602500000000001 2023-09-23T20:53:38.571 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:38.572 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:38.572 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:53:38.572 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:52:23.079834+0000", 2023-09-23T20:53:38.572 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0023447960000000001 2023-09-23T20:53:38.572 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:38.572 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:38.572 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:53:38.572 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:52:23.108054+0000", 2023-09-23T20:53:38.572 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.028219579000000002 2023-09-23T20:53:38.572 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:38.572 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:38.572 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T20:53:38.572 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:52:23.108091+0000", 2023-09-23T20:53:38.572 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.7472999999999998e-05 2023-09-23T20:53:38.573 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:38.573 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:38.573 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T20:53:38.573 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:52:23.108118+0000", 2023-09-23T20:53:38.573 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.694e-05 2023-09-23T20:53:38.573 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T20:53:38.573 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T20:53:38.573 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T20:53:38.573 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:38.573 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:38.573 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.34797.0:72742 6.a 6:5d8c85af:::benchmark_data_smithi026_223868_object72741:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e116)", 2023-09-23T20:53:38.573 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T20:53:09.769350+0000", 2023-09-23T20:53:38.573 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 28.793339137, 2023-09-23T20:53:38.573 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.029335693999999999, 2023-09-23T20:53:38.574 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T20:53:38.574 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:53:38.574 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T20:53:38.574 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.34797", 2023-09-23T20:53:38.574 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T20:53:38.574 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 72742 2023-09-23T20:53:38.574 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:38.574 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T20:53:38.574 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:38.574 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T20:53:38.574 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:09.769350+0000", 2023-09-23T20:53:38.574 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T20:53:38.574 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:38.575 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:38.575 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T20:53:38.575 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:09.769350+0000", 2023-09-23T20:53:38.575 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T20:53:38.575 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:38.575 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:38.575 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T20:53:38.575 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:09.769348+0000", 2023-09-23T20:53:38.575 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.999999 2023-09-23T20:53:38.575 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:38.575 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:38.575 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T20:53:38.576 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:09.769433+0000", 2023-09-23T20:53:38.576 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.4167999999999998e-05 2023-09-23T20:53:38.576 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:38.576 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:38.576 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T20:53:38.576 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:09.769449+0000", 2023-09-23T20:53:38.576 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.6265000000000001e-05 2023-09-23T20:53:38.576 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:38.576 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:38.576 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T20:53:38.576 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:09.769456+0000", 2023-09-23T20:53:38.576 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.221e-06 2023-09-23T20:53:38.577 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:38.577 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:38.577 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T20:53:38.577 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:09.769491+0000", 2023-09-23T20:53:38.577 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.5318999999999998e-05 2023-09-23T20:53:38.577 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:38.577 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:38.577 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T20:53:38.577 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:09.769605+0000", 2023-09-23T20:53:38.577 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00011369100000000001 2023-09-23T20:53:38.577 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:38.577 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:38.577 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 0,2", 2023-09-23T20:53:38.578 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:09.769686+0000", 2023-09-23T20:53:38.578 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.1012000000000003e-05 2023-09-23T20:53:38.578 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:38.578 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:38.578 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T20:53:38.578 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:09.772011+0000", 2023-09-23T20:53:38.578 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0023246460000000001 2023-09-23T20:53:38.578 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:38.578 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:38.578 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:53:38.578 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:09.773532+0000", 2023-09-23T20:53:38.578 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0015216959999999999 2023-09-23T20:53:38.579 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:38.579 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:38.579 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:53:38.579 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:09.798614+0000", 2023-09-23T20:53:38.579 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.025082009999999998 2023-09-23T20:53:38.579 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:38.579 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:38.579 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T20:53:38.579 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:09.798651+0000", 2023-09-23T20:53:38.579 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.6174000000000003e-05 2023-09-23T20:53:38.579 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:38.579 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:38.579 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T20:53:38.580 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:09.798685+0000", 2023-09-23T20:53:38.580 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.4619000000000002e-05 2023-09-23T20:53:38.580 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T20:53:38.580 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T20:53:38.580 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T20:53:38.580 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:38.580 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:38.580 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.34797.0:72957 6.a 6:5cba590d:::benchmark_data_smithi026_223868_object72956:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e116)", 2023-09-23T20:53:38.580 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T20:53:09.978018+0000", 2023-09-23T20:53:38.580 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 28.584670488, 2023-09-23T20:53:38.580 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.039033500999999998, 2023-09-23T20:53:38.580 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T20:53:38.580 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:53:38.581 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T20:53:38.581 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.34797", 2023-09-23T20:53:38.581 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T20:53:38.581 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 72957 2023-09-23T20:53:38.581 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:38.581 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T20:53:38.581 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:38.581 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T20:53:38.581 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:09.978018+0000", 2023-09-23T20:53:38.581 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T20:53:38.581 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:38.581 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:38.582 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T20:53:38.582 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:09.978018+0000", 2023-09-23T20:53:38.582 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T20:53:38.582 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:38.582 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:38.582 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T20:53:38.582 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:09.978017+0000", 2023-09-23T20:53:38.582 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T20:53:38.582 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:38.582 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:38.582 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T20:53:38.582 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:09.978042+0000", 2023-09-23T20:53:38.583 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.5701e-05 2023-09-23T20:53:38.583 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:38.583 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:38.583 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T20:53:38.583 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:09.978056+0000", 2023-09-23T20:53:38.583 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.4088e-05 2023-09-23T20:53:38.583 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:38.583 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:38.583 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T20:53:38.583 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:09.978063+0000", 2023-09-23T20:53:38.583 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.1070000000000001e-06 2023-09-23T20:53:38.583 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:38.584 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:38.584 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T20:53:38.584 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:09.978106+0000", 2023-09-23T20:53:38.584 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.3333e-05 2023-09-23T20:53:38.584 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:38.584 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:38.584 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T20:53:38.584 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:09.978247+0000", 2023-09-23T20:53:38.584 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00014116000000000001 2023-09-23T20:53:38.584 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:38.584 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:38.584 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 0,2", 2023-09-23T20:53:38.584 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:09.978378+0000", 2023-09-23T20:53:38.585 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00013071900000000001 2023-09-23T20:53:38.585 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:38.585 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:38.585 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T20:53:38.585 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:09.980729+0000", 2023-09-23T20:53:38.585 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0023507649999999999 2023-09-23T20:53:38.585 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:38.585 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:38.585 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:53:38.585 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:09.985367+0000", 2023-09-23T20:53:38.585 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0046387750000000004 2023-09-23T20:53:38.585 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:38.586 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:38.586 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:53:38.586 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:10.016974+0000", 2023-09-23T20:53:38.586 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.031606550999999997 2023-09-23T20:53:38.586 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:38.586 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:38.586 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T20:53:38.586 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:10.017010+0000", 2023-09-23T20:53:38.586 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.6304999999999999e-05 2023-09-23T20:53:38.586 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:38.586 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:38.586 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T20:53:38.587 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:10.017052+0000", 2023-09-23T20:53:38.587 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.1544000000000001e-05 2023-09-23T20:53:38.587 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T20:53:38.587 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T20:53:38.587 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T20:53:38.587 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:38.587 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:38.587 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.34797.0:72964 6.a 6:59542dc4:::benchmark_data_smithi026_223868_object72963:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e116)", 2023-09-23T20:53:38.587 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T20:53:09.987059+0000", 2023-09-23T20:53:38.587 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 28.575629521, 2023-09-23T20:53:38.587 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.030086519999999999, 2023-09-23T20:53:38.587 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T20:53:38.588 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:53:38.588 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T20:53:38.588 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.34797", 2023-09-23T20:53:38.588 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T20:53:38.588 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 72964 2023-09-23T20:53:38.588 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:38.588 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T20:53:38.588 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:38.588 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T20:53:38.588 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:09.987059+0000", 2023-09-23T20:53:38.588 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T20:53:38.588 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:38.588 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:38.589 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T20:53:38.589 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:09.987059+0000", 2023-09-23T20:53:38.589 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T20:53:38.589 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:38.589 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:38.589 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T20:53:38.589 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:09.987058+0000", 2023-09-23T20:53:38.589 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T20:53:38.589 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:38.589 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:38.589 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T20:53:38.589 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:09.987126+0000", 2023-09-23T20:53:38.589 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.8286000000000003e-05 2023-09-23T20:53:38.590 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:38.590 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:38.590 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T20:53:38.590 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:09.987143+0000", 2023-09-23T20:53:38.590 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.6735000000000002e-05 2023-09-23T20:53:38.590 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:38.590 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:38.590 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T20:53:38.590 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:09.987149+0000", 2023-09-23T20:53:38.590 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.7050000000000002e-06 2023-09-23T20:53:38.590 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:38.590 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:38.590 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T20:53:38.591 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:09.987184+0000", 2023-09-23T20:53:38.591 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.481e-05 2023-09-23T20:53:38.591 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:38.591 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:38.591 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T20:53:38.591 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:09.987297+0000", 2023-09-23T20:53:38.591 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000112581 2023-09-23T20:53:38.591 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:38.591 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:38.591 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 0,2", 2023-09-23T20:53:38.591 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:09.987453+0000", 2023-09-23T20:53:38.591 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00015579499999999999 2023-09-23T20:53:38.592 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:38.592 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:38.592 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T20:53:38.592 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:09.991943+0000", 2023-09-23T20:53:38.592 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0044900199999999999 2023-09-23T20:53:38.592 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:38.592 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:38.592 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:53:38.592 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:09.992639+0000", 2023-09-23T20:53:38.592 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00069652400000000001 2023-09-23T20:53:38.592 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:38.592 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:38.593 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:53:38.593 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:10.017089+0000", 2023-09-23T20:53:38.593 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.024449624 2023-09-23T20:53:38.593 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:38.593 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:38.593 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T20:53:38.593 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:10.017121+0000", 2023-09-23T20:53:38.593 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.2079999999999998e-05 2023-09-23T20:53:38.593 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:38.593 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:38.593 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T20:53:38.593 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:10.017146+0000", 2023-09-23T20:53:38.593 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.4842e-05 2023-09-23T20:53:38.594 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T20:53:38.594 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T20:53:38.594 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T20:53:38.594 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:38.594 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:38.594 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.34797.0:72969 6.a 6:5d3ebd6a:::benchmark_data_smithi026_223868_object72968:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e116)", 2023-09-23T20:53:38.594 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T20:53:09.987160+0000", 2023-09-23T20:53:38.594 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 28.575528515999999, 2023-09-23T20:53:38.594 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.036290307000000001, 2023-09-23T20:53:38.594 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T20:53:38.594 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:53:38.594 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T20:53:38.595 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.34797", 2023-09-23T20:53:38.595 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T20:53:38.595 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 72969 2023-09-23T20:53:38.595 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:38.595 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T20:53:38.595 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:38.595 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T20:53:38.595 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:09.987160+0000", 2023-09-23T20:53:38.595 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T20:53:38.595 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:38.595 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:38.595 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T20:53:38.595 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:09.987160+0000", 2023-09-23T20:53:38.596 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T20:53:38.596 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:38.596 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:38.596 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T20:53:38.596 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:09.987160+0000", 2023-09-23T20:53:38.596 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999995 2023-09-23T20:53:38.596 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:38.596 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:38.596 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T20:53:38.596 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:09.987192+0000", 2023-09-23T20:53:38.596 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.2817000000000001e-05 2023-09-23T20:53:38.596 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:38.596 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:38.597 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T20:53:38.597 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:09.987200+0000", 2023-09-23T20:53:38.597 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.8590000000000005e-06 2023-09-23T20:53:38.597 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:38.597 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:38.597 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T20:53:38.597 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:09.987205+0000", 2023-09-23T20:53:38.597 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.1549999999999998e-06 2023-09-23T20:53:38.597 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:38.597 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:38.597 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T20:53:38.597 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:09.989736+0000", 2023-09-23T20:53:38.598 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0025301730000000001 2023-09-23T20:53:38.598 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:38.598 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:38.598 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T20:53:38.598 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:09.989856+0000", 2023-09-23T20:53:38.598 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000120802 2023-09-23T20:53:38.598 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:38.598 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:38.598 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 0,2", 2023-09-23T20:53:38.598 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:09.989991+0000", 2023-09-23T20:53:38.598 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00013460699999999999 2023-09-23T20:53:38.598 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:38.598 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:38.598 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T20:53:38.599 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:09.992077+0000", 2023-09-23T20:53:38.599 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0020855610000000001 2023-09-23T20:53:38.599 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:38.599 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:38.599 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:53:38.599 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:09.992717+0000", 2023-09-23T20:53:38.599 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00064030800000000002 2023-09-23T20:53:38.599 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:38.599 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:38.599 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:53:38.599 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:10.023382+0000", 2023-09-23T20:53:38.599 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.030664945999999998 2023-09-23T20:53:38.599 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:38.600 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:38.600 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T20:53:38.600 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:10.023414+0000", 2023-09-23T20:53:38.600 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.2505999999999997e-05 2023-09-23T20:53:38.600 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:38.600 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:38.600 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T20:53:38.600 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:10.023450+0000", 2023-09-23T20:53:38.600 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.6130999999999997e-05 2023-09-23T20:53:38.600 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T20:53:38.600 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T20:53:38.600 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T20:53:38.601 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:38.601 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:38.601 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.34797.0:74420 6.a 6:5a2e8e47:::benchmark_data_smithi026_223868_object74419:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e116)", 2023-09-23T20:53:38.601 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T20:53:11.446326+0000", 2023-09-23T20:53:38.601 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 27.116362248000002, 2023-09-23T20:53:38.601 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.029879958000000002, 2023-09-23T20:53:38.601 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T20:53:38.601 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:53:38.601 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T20:53:38.601 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.34797", 2023-09-23T20:53:38.601 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T20:53:38.601 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 74420 2023-09-23T20:53:38.601 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:38.602 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T20:53:38.602 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:38.602 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T20:53:38.602 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:11.446326+0000", 2023-09-23T20:53:38.602 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T20:53:38.602 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:38.602 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:38.602 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T20:53:38.602 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:11.446326+0000", 2023-09-23T20:53:38.602 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T20:53:38.602 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:38.603 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:38.603 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T20:53:38.603 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:11.446325+0000", 2023-09-23T20:53:38.603 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T20:53:38.603 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:38.603 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:38.603 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T20:53:38.603 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:11.446381+0000", 2023-09-23T20:53:38.603 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.5998000000000002e-05 2023-09-23T20:53:38.603 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:38.603 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:38.603 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T20:53:38.603 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:11.446395+0000", 2023-09-23T20:53:38.603 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.4248e-05 2023-09-23T20:53:38.604 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:38.604 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:38.604 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T20:53:38.604 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:11.446400+0000", 2023-09-23T20:53:38.604 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.955e-06 2023-09-23T20:53:38.604 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:38.604 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:38.604 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T20:53:38.604 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:11.446437+0000", 2023-09-23T20:53:38.604 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.6869999999999998e-05 2023-09-23T20:53:38.604 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:38.604 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:38.604 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T20:53:38.604 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:11.446526+0000", 2023-09-23T20:53:38.605 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.9376999999999995e-05 2023-09-23T20:53:38.605 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:38.605 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:38.605 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 0,2", 2023-09-23T20:53:38.605 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:11.446607+0000", 2023-09-23T20:53:38.605 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.0490999999999999e-05 2023-09-23T20:53:38.605 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:38.605 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:38.605 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T20:53:38.605 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:11.448477+0000", 2023-09-23T20:53:38.605 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0018698180000000001 2023-09-23T20:53:38.606 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:38.606 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:38.606 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:53:38.606 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:11.461749+0000", 2023-09-23T20:53:38.606 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0132721 2023-09-23T20:53:38.606 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:38.606 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:38.606 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:53:38.606 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:11.476114+0000", 2023-09-23T20:53:38.606 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.014365380000000001 2023-09-23T20:53:38.606 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:38.607 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:38.607 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T20:53:38.607 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:11.476166+0000", 2023-09-23T20:53:38.607 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.1366000000000003e-05 2023-09-23T20:53:38.607 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:38.607 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:38.607 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T20:53:38.607 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:11.476206+0000", 2023-09-23T20:53:38.607 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.0871999999999998e-05 2023-09-23T20:53:38.607 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T20:53:38.607 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T20:53:38.607 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T20:53:38.607 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:38.608 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:38.608 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.34797.0:75204 6.a 6:5363edde:::benchmark_data_smithi026_223868_object75203:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e116)", 2023-09-23T20:53:38.608 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T20:53:12.297657+0000", 2023-09-23T20:53:38.608 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 26.265031669999999, 2023-09-23T20:53:38.608 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.034514603999999997, 2023-09-23T20:53:38.608 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T20:53:38.608 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:53:38.608 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T20:53:38.608 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.34797", 2023-09-23T20:53:38.608 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T20:53:38.608 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 75204 2023-09-23T20:53:38.608 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:38.608 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T20:53:38.609 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:38.609 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T20:53:38.609 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:12.297657+0000", 2023-09-23T20:53:38.609 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T20:53:38.609 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:38.609 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:38.609 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T20:53:38.609 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:12.297657+0000", 2023-09-23T20:53:38.609 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T20:53:38.609 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:38.609 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:38.609 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T20:53:38.609 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:12.297656+0000", 2023-09-23T20:53:38.609 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T20:53:38.610 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:38.610 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:38.610 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T20:53:38.610 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:12.297713+0000", 2023-09-23T20:53:38.610 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.7549000000000001e-05 2023-09-23T20:53:38.610 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:38.610 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:38.610 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T20:53:38.610 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:12.297729+0000", 2023-09-23T20:53:38.610 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.5332000000000002e-05 2023-09-23T20:53:38.610 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:38.611 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:38.611 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T20:53:38.611 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:12.297734+0000", 2023-09-23T20:53:38.611 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.8710000000000002e-06 2023-09-23T20:53:38.611 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:38.611 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:38.611 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T20:53:38.611 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:12.297782+0000", 2023-09-23T20:53:38.611 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.7843999999999998e-05 2023-09-23T20:53:38.611 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:38.611 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:38.611 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T20:53:38.611 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:12.297920+0000", 2023-09-23T20:53:38.612 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00013765699999999999 2023-09-23T20:53:38.612 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:38.612 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:38.612 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 0,2", 2023-09-23T20:53:38.612 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:12.298065+0000", 2023-09-23T20:53:38.612 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000145251 2023-09-23T20:53:38.612 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:38.612 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:38.612 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T20:53:38.612 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:12.302437+0000", 2023-09-23T20:53:38.612 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0043714970000000002 2023-09-23T20:53:38.612 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:38.612 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:38.612 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:53:38.613 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:12.317453+0000", 2023-09-23T20:53:38.613 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.015016432999999999 2023-09-23T20:53:38.613 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:38.613 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:38.613 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:53:38.613 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:12.332069+0000", 2023-09-23T20:53:38.613 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.014615594000000001 2023-09-23T20:53:38.613 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:38.613 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:38.613 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T20:53:38.613 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:12.332126+0000", 2023-09-23T20:53:38.613 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.7439999999999999e-05 2023-09-23T20:53:38.613 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:38.614 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:38.614 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T20:53:38.614 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:12.332172+0000", 2023-09-23T20:53:38.614 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.5482999999999998e-05 2023-09-23T20:53:38.614 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T20:53:38.614 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T20:53:38.614 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T20:53:38.614 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:38.614 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:38.614 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.34797.0:75205 6.a 6:5792abdb:::benchmark_data_smithi026_223868_object75204:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e116)", 2023-09-23T20:53:38.614 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T20:53:12.297746+0000", 2023-09-23T20:53:38.614 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 26.264942808000001, 2023-09-23T20:53:38.614 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.034562254000000001, 2023-09-23T20:53:38.614 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T20:53:38.615 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:53:38.615 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T20:53:38.615 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.34797", 2023-09-23T20:53:38.615 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T20:53:38.615 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 75205 2023-09-23T20:53:38.615 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:38.615 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T20:53:38.615 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:38.615 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T20:53:38.615 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:12.297746+0000", 2023-09-23T20:53:38.615 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T20:53:38.615 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:38.615 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:38.616 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T20:53:38.616 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:12.297746+0000", 2023-09-23T20:53:38.616 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T20:53:38.616 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:38.616 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:38.616 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T20:53:38.616 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:12.297745+0000", 2023-09-23T20:53:38.616 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999995 2023-09-23T20:53:38.616 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:38.616 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:38.616 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T20:53:38.616 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:12.297777+0000", 2023-09-23T20:53:38.616 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.1483000000000003e-05 2023-09-23T20:53:38.616 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:38.617 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:38.617 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T20:53:38.617 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:12.297784+0000", 2023-09-23T20:53:38.617 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.8190000000000003e-06 2023-09-23T20:53:38.617 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:38.617 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:38.617 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T20:53:38.617 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:12.297788+0000", 2023-09-23T20:53:38.617 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.2860000000000001e-06 2023-09-23T20:53:38.617 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:38.617 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:38.617 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T20:53:38.617 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:12.300002+0000", 2023-09-23T20:53:38.617 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.002213905 2023-09-23T20:53:38.618 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:38.618 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:38.618 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T20:53:38.618 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:12.300135+0000", 2023-09-23T20:53:38.618 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000133335 2023-09-23T20:53:38.618 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:38.618 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:38.618 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 0,2", 2023-09-23T20:53:38.618 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:12.300290+0000", 2023-09-23T20:53:38.618 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00015493399999999999 2023-09-23T20:53:38.618 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:38.618 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:38.618 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T20:53:38.619 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:12.302565+0000", 2023-09-23T20:53:38.619 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.002274688 2023-09-23T20:53:38.619 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:38.619 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:38.619 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:53:38.619 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:12.317548+0000", 2023-09-23T20:53:38.619 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.014983616999999999 2023-09-23T20:53:38.619 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:38.619 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:38.619 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:53:38.619 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:12.332217+0000", 2023-09-23T20:53:38.619 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.014668524 2023-09-23T20:53:38.620 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:38.620 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:38.620 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T20:53:38.620 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:12.332273+0000", 2023-09-23T20:53:38.620 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.5800000000000001e-05 2023-09-23T20:53:38.620 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:38.620 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:38.620 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T20:53:38.620 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:12.332308+0000", 2023-09-23T20:53:38.620 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.5348000000000001e-05 2023-09-23T20:53:38.620 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T20:53:38.620 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T20:53:38.620 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T20:53:38.620 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:38.621 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:38.621 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.34797.0:75209 6.a 6:5ec44f2a:::benchmark_data_smithi026_223868_object75208:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e116)", 2023-09-23T20:53:38.621 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T20:53:12.303954+0000", 2023-09-23T20:53:38.621 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 26.258734381, 2023-09-23T20:53:38.621 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.028513582999999999, 2023-09-23T20:53:38.621 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T20:53:38.621 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:53:38.621 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T20:53:38.621 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.34797", 2023-09-23T20:53:38.621 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T20:53:38.621 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 75209 2023-09-23T20:53:38.621 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:38.622 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T20:53:38.622 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:38.622 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T20:53:38.622 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:12.303954+0000", 2023-09-23T20:53:38.622 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T20:53:38.622 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:38.622 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:38.622 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T20:53:38.622 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:12.303954+0000", 2023-09-23T20:53:38.622 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T20:53:38.622 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:38.622 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:38.623 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T20:53:38.623 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:12.303953+0000", 2023-09-23T20:53:38.623 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T20:53:38.623 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:38.623 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:38.623 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T20:53:38.623 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:12.303982+0000", 2023-09-23T20:53:38.623 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.883e-05 2023-09-23T20:53:38.623 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:38.623 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:38.623 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T20:53:38.623 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:12.303993+0000", 2023-09-23T20:53:38.623 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.1586999999999999e-05 2023-09-23T20:53:38.624 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:38.624 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:38.624 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T20:53:38.624 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:12.303998+0000", 2023-09-23T20:53:38.624 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.8740000000000003e-06 2023-09-23T20:53:38.624 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:38.624 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:38.624 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T20:53:38.624 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:12.304040+0000", 2023-09-23T20:53:38.624 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.1359000000000002e-05 2023-09-23T20:53:38.624 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:38.624 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:38.624 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T20:53:38.624 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:12.304146+0000", 2023-09-23T20:53:38.625 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000106536 2023-09-23T20:53:38.625 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:38.625 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:38.625 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 0,2", 2023-09-23T20:53:38.625 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:12.304229+0000", 2023-09-23T20:53:38.625 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.2964000000000001e-05 2023-09-23T20:53:38.625 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:38.625 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:38.625 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T20:53:38.625 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:12.306462+0000", 2023-09-23T20:53:38.625 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0022332020000000001 2023-09-23T20:53:38.625 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:38.625 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:38.626 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:53:38.626 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:12.317568+0000", 2023-09-23T20:53:38.626 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.011105643 2023-09-23T20:53:38.626 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:38.626 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:38.626 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:53:38.626 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:12.332372+0000", 2023-09-23T20:53:38.626 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.014804473 2023-09-23T20:53:38.626 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:38.626 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:38.626 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T20:53:38.626 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:12.332436+0000", 2023-09-23T20:53:38.626 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.4017999999999996e-05 2023-09-23T20:53:38.627 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:38.627 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:38.627 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T20:53:38.627 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:12.332468+0000", 2023-09-23T20:53:38.627 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.1467999999999998e-05 2023-09-23T20:53:38.627 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T20:53:38.627 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T20:53:38.627 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T20:53:38.627 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:38.627 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:38.627 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.34797.0:75212 6.a 6:5b28eee2:::benchmark_data_smithi026_223868_object75211:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e116)", 2023-09-23T20:53:38.627 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T20:53:12.304006+0000", 2023-09-23T20:53:38.627 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 26.258683049999998, 2023-09-23T20:53:38.627 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.028584656, 2023-09-23T20:53:38.628 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T20:53:38.628 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:53:38.628 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T20:53:38.628 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.34797", 2023-09-23T20:53:38.628 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T20:53:38.628 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 75212 2023-09-23T20:53:38.628 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:38.628 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T20:53:38.628 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:38.628 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T20:53:38.628 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:12.304006+0000", 2023-09-23T20:53:38.628 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T20:53:38.628 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:38.628 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:38.629 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T20:53:38.629 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:12.304006+0000", 2023-09-23T20:53:38.629 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T20:53:38.629 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:38.629 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:38.629 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T20:53:38.629 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:12.304005+0000", 2023-09-23T20:53:38.629 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999995 2023-09-23T20:53:38.629 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:38.629 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:38.629 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T20:53:38.629 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:12.304029+0000", 2023-09-23T20:53:38.629 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.4029999999999999e-05 2023-09-23T20:53:38.630 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:38.630 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:38.630 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T20:53:38.630 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:12.304040+0000", 2023-09-23T20:53:38.630 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.084e-05 2023-09-23T20:53:38.630 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:38.630 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:38.630 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T20:53:38.630 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:12.304043+0000", 2023-09-23T20:53:38.630 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.9749999999999999e-06 2023-09-23T20:53:38.630 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:38.630 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:38.630 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T20:53:38.630 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:12.305251+0000", 2023-09-23T20:53:38.631 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.001207685 2023-09-23T20:53:38.631 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:38.631 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:38.631 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T20:53:38.631 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:12.305348+0000", 2023-09-23T20:53:38.631 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.7610999999999998e-05 2023-09-23T20:53:38.631 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:38.631 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:38.631 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 0,2", 2023-09-23T20:53:38.631 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:12.305441+0000", 2023-09-23T20:53:38.631 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.2380000000000007e-05 2023-09-23T20:53:38.631 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:38.631 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:38.631 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T20:53:38.632 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:12.306546+0000", 2023-09-23T20:53:38.632 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0011053530000000001 2023-09-23T20:53:38.632 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:38.632 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:38.632 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:53:38.632 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:12.317602+0000", 2023-09-23T20:53:38.632 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.011056194 2023-09-23T20:53:38.632 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:38.632 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:38.632 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:53:38.632 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:12.332510+0000", 2023-09-23T20:53:38.632 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.014907620999999999 2023-09-23T20:53:38.632 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:38.633 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:38.633 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T20:53:38.633 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:12.332557+0000", 2023-09-23T20:53:38.633 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.7052000000000003e-05 2023-09-23T20:53:38.633 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:38.633 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:38.633 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T20:53:38.633 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:12.332590+0000", 2023-09-23T20:53:38.633 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.3237e-05 2023-09-23T20:53:38.633 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T20:53:38.633 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T20:53:38.633 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T20:53:38.634 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:38.634 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:38.634 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.34797.0:75218 6.a 6:5c3267d6:::benchmark_data_smithi026_223868_object75217:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e116)", 2023-09-23T20:53:38.634 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T20:53:12.308218+0000", 2023-09-23T20:53:38.634 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 26.254470881, 2023-09-23T20:53:38.634 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.030264914, 2023-09-23T20:53:38.634 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T20:53:38.634 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:53:38.634 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T20:53:38.634 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.34797", 2023-09-23T20:53:38.634 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T20:53:38.634 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 75218 2023-09-23T20:53:38.635 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:38.635 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T20:53:38.635 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:38.635 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T20:53:38.635 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:12.308218+0000", 2023-09-23T20:53:38.635 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T20:53:38.635 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:38.635 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:38.635 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T20:53:38.635 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:12.308218+0000", 2023-09-23T20:53:38.635 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T20:53:38.636 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:38.636 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:38.636 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T20:53:38.636 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:12.308217+0000", 2023-09-23T20:53:38.636 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999995 2023-09-23T20:53:38.636 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:38.636 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:38.636 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T20:53:38.636 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:12.308250+0000", 2023-09-23T20:53:38.636 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.2546000000000003e-05 2023-09-23T20:53:38.636 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:38.636 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:38.637 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T20:53:38.637 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:12.308263+0000", 2023-09-23T20:53:38.637 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.2965e-05 2023-09-23T20:53:38.637 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:38.637 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:38.637 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T20:53:38.637 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:12.308267+0000", 2023-09-23T20:53:38.637 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.6550000000000003e-06 2023-09-23T20:53:38.637 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:38.637 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:38.637 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T20:53:38.637 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:12.308300+0000", 2023-09-23T20:53:38.637 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.222e-05 2023-09-23T20:53:38.638 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:38.638 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:38.638 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T20:53:38.638 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:12.308384+0000", 2023-09-23T20:53:38.638 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.4586000000000006e-05 2023-09-23T20:53:38.638 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:38.638 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:38.638 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 0,2", 2023-09-23T20:53:38.638 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:12.308470+0000", 2023-09-23T20:53:38.638 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.6058999999999997e-05 2023-09-23T20:53:38.638 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:38.638 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:38.638 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T20:53:38.639 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:12.309707+0000", 2023-09-23T20:53:38.639 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.001237073 2023-09-23T20:53:38.639 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:38.639 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:38.639 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:53:38.639 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:12.317624+0000", 2023-09-23T20:53:38.639 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.007916991 2023-09-23T20:53:38.639 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:38.639 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:38.639 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:53:38.639 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:12.338395+0000", 2023-09-23T20:53:38.639 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.020771049999999999 2023-09-23T20:53:38.639 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:38.640 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:38.640 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T20:53:38.640 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:12.338448+0000", 2023-09-23T20:53:38.640 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.2420999999999998e-05 2023-09-23T20:53:38.640 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:38.640 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:38.640 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T20:53:38.640 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:12.338483+0000", 2023-09-23T20:53:38.640 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.5027999999999997e-05 2023-09-23T20:53:38.640 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T20:53:38.640 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T20:53:38.640 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T20:53:38.641 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:38.641 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:38.641 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.34797.0:76950 6.a 6:5b00453b:::benchmark_data_smithi026_223868_object76949:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e116)", 2023-09-23T20:53:38.641 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T20:53:14.232946+0000", 2023-09-23T20:53:38.641 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 24.329742275000001, 2023-09-23T20:53:38.641 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.035521771000000001, 2023-09-23T20:53:38.641 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T20:53:38.641 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:53:38.641 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T20:53:38.641 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.34797", 2023-09-23T20:53:38.641 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T20:53:38.641 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 76950 2023-09-23T20:53:38.642 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:38.642 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T20:53:38.642 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:38.642 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T20:53:38.642 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:14.232946+0000", 2023-09-23T20:53:38.642 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T20:53:38.642 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:38.642 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:38.642 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T20:53:38.642 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:14.232946+0000", 2023-09-23T20:53:38.642 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T20:53:38.642 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:38.643 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:38.643 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T20:53:38.643 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:14.232946+0000", 2023-09-23T20:53:38.643 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999995 2023-09-23T20:53:38.643 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:38.643 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:38.643 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T20:53:38.643 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:14.232973+0000", 2023-09-23T20:53:38.643 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.7124999999999999e-05 2023-09-23T20:53:38.643 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:38.643 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:38.643 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T20:53:38.644 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:14.232984+0000", 2023-09-23T20:53:38.644 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.0909e-05 2023-09-23T20:53:38.644 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:38.644 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:38.644 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T20:53:38.644 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:14.232988+0000", 2023-09-23T20:53:38.644 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.4159999999999997e-06 2023-09-23T20:53:38.644 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:38.644 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:38.644 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T20:53:38.644 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:14.235270+0000", 2023-09-23T20:53:38.644 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0022813159999999998 2023-09-23T20:53:38.644 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:38.645 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:38.645 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T20:53:38.645 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:14.235402+0000", 2023-09-23T20:53:38.645 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00013255600000000001 2023-09-23T20:53:38.645 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:38.645 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:38.645 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 0,2", 2023-09-23T20:53:38.645 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:14.235548+0000", 2023-09-23T20:53:38.645 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00014592300000000001 2023-09-23T20:53:38.645 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:38.645 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:38.645 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T20:53:38.645 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:14.237575+0000", 2023-09-23T20:53:38.646 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0020270010000000001 2023-09-23T20:53:38.646 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:38.646 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:38.646 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:53:38.646 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:14.242404+0000", 2023-09-23T20:53:38.646 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.004828555 2023-09-23T20:53:38.646 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:38.646 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:38.646 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:53:38.646 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:14.268366+0000", 2023-09-23T20:53:38.646 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.025962697 2023-09-23T20:53:38.646 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:38.647 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:38.647 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T20:53:38.647 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:14.268422+0000", 2023-09-23T20:53:38.647 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.5537e-05 2023-09-23T20:53:38.647 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:38.647 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:38.647 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T20:53:38.647 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:14.268468+0000", 2023-09-23T20:53:38.647 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.6261999999999999e-05 2023-09-23T20:53:38.647 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T20:53:38.647 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T20:53:38.647 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T20:53:38.648 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:38.648 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:38.648 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.34797.0:78942 6.a 6:5fcb0a14:::benchmark_data_smithi026_223868_object78941:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e117)", 2023-09-23T20:53:38.648 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T20:53:16.389369+0000", 2023-09-23T20:53:38.648 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 22.173319336999999, 2023-09-23T20:53:38.648 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.029304034999999999, 2023-09-23T20:53:38.648 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T20:53:38.648 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:53:38.648 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T20:53:38.648 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.34797", 2023-09-23T20:53:38.648 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T20:53:38.648 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 78942 2023-09-23T20:53:38.649 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:38.649 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T20:53:38.649 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:38.649 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T20:53:38.649 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:16.389369+0000", 2023-09-23T20:53:38.649 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T20:53:38.649 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:38.649 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:38.649 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T20:53:38.649 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:16.389369+0000", 2023-09-23T20:53:38.649 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T20:53:38.649 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:38.650 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:38.650 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T20:53:38.650 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:16.389368+0000", 2023-09-23T20:53:38.650 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T20:53:38.650 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:38.650 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:38.650 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T20:53:38.650 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:16.389415+0000", 2023-09-23T20:53:38.650 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.7268000000000002e-05 2023-09-23T20:53:38.650 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:38.650 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:38.651 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T20:53:38.651 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:16.389429+0000", 2023-09-23T20:53:38.651 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.348e-05 2023-09-23T20:53:38.651 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:38.651 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:38.651 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T20:53:38.651 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:16.389434+0000", 2023-09-23T20:53:38.651 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.9710000000000003e-06 2023-09-23T20:53:38.651 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:38.651 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:38.651 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T20:53:38.651 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:16.389467+0000", 2023-09-23T20:53:38.652 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.3074999999999998e-05 2023-09-23T20:53:38.652 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:38.652 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:38.652 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T20:53:38.652 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:16.389558+0000", 2023-09-23T20:53:38.652 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.1131999999999994e-05 2023-09-23T20:53:38.652 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:38.652 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:38.652 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 0,2", 2023-09-23T20:53:38.652 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:16.389687+0000", 2023-09-23T20:53:38.652 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000129185 2023-09-23T20:53:38.652 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:38.653 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:38.653 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T20:53:38.653 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:16.394376+0000", 2023-09-23T20:53:38.653 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0046892349999999999 2023-09-23T20:53:38.653 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:38.653 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:38.653 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:53:38.653 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:16.400639+0000", 2023-09-23T20:53:38.653 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.006262297 2023-09-23T20:53:38.653 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:38.653 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:38.653 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:53:38.654 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:16.418584+0000", 2023-09-23T20:53:38.654 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.017945243999999999 2023-09-23T20:53:38.654 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:38.654 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:38.654 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T20:53:38.654 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:16.418632+0000", 2023-09-23T20:53:38.654 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.8606999999999997e-05 2023-09-23T20:53:38.654 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:38.654 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:38.654 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T20:53:38.654 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:16.418673+0000", 2023-09-23T20:53:38.654 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.0908000000000002e-05 2023-09-23T20:53:38.655 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T20:53:38.655 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T20:53:38.655 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T20:53:38.655 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:38.655 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:38.655 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.34797.0:78943 6.a 6:588b302d:::benchmark_data_smithi026_223868_object78942:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e117)", 2023-09-23T20:53:38.655 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T20:53:16.389442+0000", 2023-09-23T20:53:38.655 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 22.173246840000001, 2023-09-23T20:53:38.655 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.029384449, 2023-09-23T20:53:38.655 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T20:53:38.655 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:53:38.656 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T20:53:38.656 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.34797", 2023-09-23T20:53:38.656 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T20:53:38.656 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 78943 2023-09-23T20:53:38.656 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:38.656 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T20:53:38.656 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:38.656 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T20:53:38.656 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:16.389442+0000", 2023-09-23T20:53:38.656 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T20:53:38.656 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:38.656 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:38.656 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T20:53:38.657 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:16.389442+0000", 2023-09-23T20:53:38.657 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T20:53:38.657 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:38.657 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:38.657 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T20:53:38.657 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:16.389441+0000", 2023-09-23T20:53:38.657 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999995 2023-09-23T20:53:38.657 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:38.657 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:38.657 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T20:53:38.657 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:16.389474+0000", 2023-09-23T20:53:38.657 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.2889000000000003e-05 2023-09-23T20:53:38.658 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:38.658 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:38.658 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T20:53:38.658 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:16.389479+0000", 2023-09-23T20:53:38.658 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.5959999999999997e-06 2023-09-23T20:53:38.658 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:38.658 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:38.658 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T20:53:38.658 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:16.389482+0000", 2023-09-23T20:53:38.658 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.9340000000000002e-06 2023-09-23T20:53:38.658 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:38.659 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:38.659 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T20:53:38.659 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:16.391683+0000", 2023-09-23T20:53:38.659 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0022009780000000001 2023-09-23T20:53:38.659 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:38.659 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:38.659 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T20:53:38.659 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:16.391787+0000", 2023-09-23T20:53:38.659 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00010405599999999999 2023-09-23T20:53:38.659 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:38.659 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:38.659 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 0,2", 2023-09-23T20:53:38.659 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:16.391930+0000", 2023-09-23T20:53:38.660 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000142824 2023-09-23T20:53:38.660 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:38.660 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:38.660 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T20:53:38.660 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:16.394447+0000", 2023-09-23T20:53:38.660 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0025173800000000001 2023-09-23T20:53:38.660 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:38.660 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:38.660 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:53:38.660 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:16.400668+0000", 2023-09-23T20:53:38.660 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0062205289999999998 2023-09-23T20:53:38.660 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:38.660 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:38.661 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:53:38.661 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:16.418741+0000", 2023-09-23T20:53:38.661 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.018073545 2023-09-23T20:53:38.661 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:38.661 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:38.661 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T20:53:38.661 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:16.418793+0000", 2023-09-23T20:53:38.661 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.1694999999999999e-05 2023-09-23T20:53:38.661 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:38.661 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:38.661 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T20:53:38.661 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:16.418826+0000", 2023-09-23T20:53:38.662 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.3429000000000001e-05 2023-09-23T20:53:38.662 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T20:53:38.662 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T20:53:38.662 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T20:53:38.662 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:38.662 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:38.662 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.34797.0:88177 6.a 6:5927ede6:::benchmark_data_smithi026_223868_object88176:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e120)", 2023-09-23T20:53:38.662 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T20:53:26.482958+0000", 2023-09-23T20:53:38.662 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 12.079731042000001, 2023-09-23T20:53:38.662 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.030756308999999999, 2023-09-23T20:53:38.662 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T20:53:38.662 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:53:38.663 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T20:53:38.663 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.34797", 2023-09-23T20:53:38.663 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T20:53:38.663 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 88177 2023-09-23T20:53:38.663 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:38.663 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T20:53:38.663 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:38.663 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T20:53:38.663 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:26.482958+0000", 2023-09-23T20:53:38.663 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T20:53:38.663 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:38.663 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:38.663 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T20:53:38.664 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:26.482958+0000", 2023-09-23T20:53:38.664 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T20:53:38.664 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:38.664 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:38.664 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T20:53:38.664 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:26.482956+0000", 2023-09-23T20:53:38.664 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T20:53:38.664 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:38.664 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:38.664 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T20:53:38.664 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:26.482983+0000", 2023-09-23T20:53:38.664 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.6174e-05 2023-09-23T20:53:38.665 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:38.665 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:38.665 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T20:53:38.665 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:26.482995+0000", 2023-09-23T20:53:38.665 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.2584e-05 2023-09-23T20:53:38.665 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:38.665 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:38.665 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T20:53:38.665 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:26.483000+0000", 2023-09-23T20:53:38.665 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.3090000000000003e-06 2023-09-23T20:53:38.665 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:38.666 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:38.666 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T20:53:38.666 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:26.483045+0000", 2023-09-23T20:53:38.666 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.4535e-05 2023-09-23T20:53:38.666 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:38.666 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:38.666 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T20:53:38.666 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:26.483191+0000", 2023-09-23T20:53:38.666 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00014648999999999999 2023-09-23T20:53:38.666 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:38.666 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:38.666 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 0,2", 2023-09-23T20:53:38.667 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:26.483349+0000", 2023-09-23T20:53:38.667 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00015785199999999999 2023-09-23T20:53:38.667 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:38.667 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:38.667 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T20:53:38.667 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:26.485508+0000", 2023-09-23T20:53:38.667 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0021590139999999999 2023-09-23T20:53:38.667 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:38.667 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:38.667 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:53:38.667 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:26.489280+0000", 2023-09-23T20:53:38.667 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0037712890000000002 2023-09-23T20:53:38.667 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:38.668 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:38.668 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:53:38.668 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:26.513645+0000", 2023-09-23T20:53:38.668 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.024365817000000001 2023-09-23T20:53:38.668 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:38.668 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:38.668 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T20:53:38.668 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:26.513679+0000", 2023-09-23T20:53:38.668 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.3924999999999998e-05 2023-09-23T20:53:38.668 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:38.668 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:38.668 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T20:53:38.669 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:26.513714+0000", 2023-09-23T20:53:38.669 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.4591999999999997e-05 2023-09-23T20:53:38.669 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T20:53:38.669 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T20:53:38.669 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T20:53:38.669 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:38.669 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:38.669 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.34797.0:89247 6.a 6:57eb65b1:::benchmark_data_smithi026_223868_object89246:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e120)", 2023-09-23T20:53:38.669 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T20:53:27.905585+0000", 2023-09-23T20:53:38.669 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 10.657103731999999, 2023-09-23T20:53:38.669 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.047211367999999997, 2023-09-23T20:53:38.669 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T20:53:38.669 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:53:38.669 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T20:53:38.670 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.34797", 2023-09-23T20:53:38.670 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T20:53:38.670 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 89247 2023-09-23T20:53:38.670 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:38.670 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T20:53:38.670 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:38.670 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T20:53:38.670 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:27.905585+0000", 2023-09-23T20:53:38.670 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T20:53:38.670 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:38.670 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:38.670 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T20:53:38.671 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:27.905585+0000", 2023-09-23T20:53:38.671 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T20:53:38.671 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:38.671 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:38.671 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T20:53:38.671 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:27.905583+0000", 2023-09-23T20:53:38.671 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T20:53:38.671 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:38.671 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:38.671 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T20:53:38.671 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:27.905618+0000", 2023-09-23T20:53:38.671 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.4968000000000001e-05 2023-09-23T20:53:38.672 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:38.672 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:38.672 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T20:53:38.672 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:27.905635+0000", 2023-09-23T20:53:38.672 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.6433e-05 2023-09-23T20:53:38.672 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:38.672 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:38.672 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T20:53:38.672 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:27.905641+0000", 2023-09-23T20:53:38.672 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.5690000000000001e-06 2023-09-23T20:53:38.672 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:38.672 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:38.673 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T20:53:38.673 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:27.905691+0000", 2023-09-23T20:53:38.673 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.9895000000000003e-05 2023-09-23T20:53:38.673 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:38.673 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:38.673 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T20:53:38.673 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:27.905839+0000", 2023-09-23T20:53:38.673 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00014760500000000001 2023-09-23T20:53:38.673 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:38.673 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:38.673 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 0,2", 2023-09-23T20:53:38.673 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:27.905962+0000", 2023-09-23T20:53:38.673 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00012323999999999999 2023-09-23T20:53:38.673 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:38.674 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:38.674 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T20:53:38.674 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:27.908263+0000", 2023-09-23T20:53:38.674 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0023006929999999999 2023-09-23T20:53:38.674 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:38.674 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:38.674 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:53:38.674 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:27.939672+0000", 2023-09-23T20:53:38.674 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.031409434 2023-09-23T20:53:38.674 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:38.674 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:38.674 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:53:38.674 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:27.952714+0000", 2023-09-23T20:53:38.675 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.013041799 2023-09-23T20:53:38.675 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:38.675 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:38.675 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T20:53:38.675 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:27.952757+0000", 2023-09-23T20:53:38.675 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.2922e-05 2023-09-23T20:53:38.675 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:38.675 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:38.675 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T20:53:38.675 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:27.952796+0000", 2023-09-23T20:53:38.675 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.9298999999999998e-05 2023-09-23T20:53:38.675 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T20:53:38.675 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T20:53:38.676 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T20:53:38.676 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:38.676 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:38.676 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.34797.0:93604 6.a 6:51edbc5c:::benchmark_data_smithi026_223868_object93603:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e124)", 2023-09-23T20:53:38.676 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T20:53:36.938586+0000", 2023-09-23T20:53:38.676 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 1.6241027369999999, 2023-09-23T20:53:38.676 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.036723890000000002, 2023-09-23T20:53:38.676 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T20:53:38.676 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:53:38.676 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T20:53:38.676 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.34797", 2023-09-23T20:53:38.676 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T20:53:38.676 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 93604 2023-09-23T20:53:38.677 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:38.677 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T20:53:38.677 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:38.677 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T20:53:38.677 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:36.938586+0000", 2023-09-23T20:53:38.677 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T20:53:38.677 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:38.677 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:38.677 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T20:53:38.677 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:36.938586+0000", 2023-09-23T20:53:38.677 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T20:53:38.677 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:38.677 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:38.677 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T20:53:38.677 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:36.938584+0000", 2023-09-23T20:53:38.678 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T20:53:38.678 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:38.678 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:38.678 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T20:53:38.678 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:36.938622+0000", 2023-09-23T20:53:38.678 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.7982999999999999e-05 2023-09-23T20:53:38.678 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:38.678 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:38.678 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T20:53:38.678 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:36.938637+0000", 2023-09-23T20:53:38.678 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.4401e-05 2023-09-23T20:53:38.678 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:38.678 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:38.678 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T20:53:38.679 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:36.938660+0000", 2023-09-23T20:53:38.679 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.3181999999999999e-05 2023-09-23T20:53:38.679 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:38.679 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:38.679 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T20:53:38.679 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:36.938702+0000", 2023-09-23T20:53:38.679 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.2357999999999997e-05 2023-09-23T20:53:38.679 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:38.679 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:38.679 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T20:53:38.679 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:36.938812+0000", 2023-09-23T20:53:38.679 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000109237 2023-09-23T20:53:38.680 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:38.680 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:38.680 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 0,2", 2023-09-23T20:53:38.680 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:36.938955+0000", 2023-09-23T20:53:38.680 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00014382500000000001 2023-09-23T20:53:38.680 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:38.680 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:38.680 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T20:53:38.680 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:36.942018+0000", 2023-09-23T20:53:38.680 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0030625029999999998 2023-09-23T20:53:38.680 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:38.680 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:38.680 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:53:38.680 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:36.943416+0000", 2023-09-23T20:53:38.681 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0013978529999999999 2023-09-23T20:53:38.681 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:38.681 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:38.681 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:53:38.681 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:36.975204+0000", 2023-09-23T20:53:38.681 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.031787937000000002 2023-09-23T20:53:38.681 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:38.681 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:38.681 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T20:53:38.681 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:36.975260+0000", 2023-09-23T20:53:38.681 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.6301999999999997e-05 2023-09-23T20:53:38.681 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:38.681 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:38.682 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T20:53:38.682 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:36.975310+0000", 2023-09-23T20:53:38.682 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.9867000000000002e-05 2023-09-23T20:53:38.682 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T20:53:38.682 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T20:53:38.682 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T20:53:38.682 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:38.682 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:38.682 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_repop(client.34797.0:94310 6.2 e124/122 6:483b8287:::benchmark_data_smithi026_223868_object94309:head v 124'5727, mlcod=0'0)", 2023-09-23T20:53:38.682 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T20:53:38.004082+0000", 2023-09-23T20:53:38.682 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 0.55860705399999999, 2023-09-23T20:53:38.683 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.028562300999999998, 2023-09-23T20:53:38.683 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T20:53:38.683 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:53:38.683 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T20:53:38.683 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:38.683 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T20:53:38.683 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:38.004082+0000", 2023-09-23T20:53:38.683 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T20:53:38.683 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:38.683 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:38.683 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T20:53:38.683 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:38.004082+0000", 2023-09-23T20:53:38.683 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T20:53:38.683 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:38.684 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:38.684 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T20:53:38.684 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:38.004081+0000", 2023-09-23T20:53:38.684 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999995 2023-09-23T20:53:38.684 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:38.684 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:38.684 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T20:53:38.684 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:38.004113+0000", 2023-09-23T20:53:38.684 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.1222e-05 2023-09-23T20:53:38.684 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:38.684 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:38.684 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T20:53:38.685 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:38.004117+0000", 2023-09-23T20:53:38.685 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.1500000000000001e-06 2023-09-23T20:53:38.685 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:38.685 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:38.685 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T20:53:38.685 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:38.004119+0000", 2023-09-23T20:53:38.685 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.6479999999999999e-06 2023-09-23T20:53:38.685 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:38.685 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:38.685 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T20:53:38.685 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:38.014159+0000", 2023-09-23T20:53:38.685 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.010039497 2023-09-23T20:53:38.685 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:38.686 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:38.686 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T20:53:38.686 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:38.014189+0000", 2023-09-23T20:53:38.686 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.97e-05 2023-09-23T20:53:38.686 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:38.686 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:38.686 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T20:53:38.686 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:38.032602+0000", 2023-09-23T20:53:38.686 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.018413374999999999 2023-09-23T20:53:38.686 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:38.686 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:38.687 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T20:53:38.687 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:38.032644+0000", 2023-09-23T20:53:38.687 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.2008000000000002e-05 2023-09-23T20:53:38.687 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T20:53:38.687 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T20:53:38.687 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T20:53:38.687 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:38.687 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:38.687 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_repop(client.34797.0:94634 6.6 e124/122 6:696d4147:::benchmark_data_smithi026_223868_object94633:head v 124'5850, mlcod=0'0)", 2023-09-23T20:53:38.687 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T20:53:38.506278+0000", 2023-09-23T20:53:38.687 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 0.056411103999999997, 2023-09-23T20:53:38.688 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.027863038, 2023-09-23T20:53:38.688 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T20:53:38.688 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:53:38.688 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T20:53:38.688 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:38.688 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T20:53:38.688 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:38.506278+0000", 2023-09-23T20:53:38.688 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T20:53:38.688 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:38.688 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:38.688 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T20:53:38.689 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:38.506278+0000", 2023-09-23T20:53:38.689 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T20:53:38.689 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:38.689 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:38.689 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T20:53:38.689 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:38.506277+0000", 2023-09-23T20:53:38.689 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.999999 2023-09-23T20:53:38.689 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:38.689 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:38.689 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T20:53:38.689 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:38.506325+0000", 2023-09-23T20:53:38.690 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.8090000000000002e-05 2023-09-23T20:53:38.690 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:38.690 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:38.690 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T20:53:38.690 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:38.506342+0000", 2023-09-23T20:53:38.690 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.7719000000000001e-05 2023-09-23T20:53:38.690 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:38.690 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:38.690 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T20:53:38.690 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:38.506350+0000", 2023-09-23T20:53:38.690 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.2470000000000002e-06 2023-09-23T20:53:38.691 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:38.691 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:38.691 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T20:53:38.691 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:38.506589+0000", 2023-09-23T20:53:38.691 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00023953599999999999 2023-09-23T20:53:38.691 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:38.691 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:38.691 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T20:53:38.691 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:38.506614+0000", 2023-09-23T20:53:38.691 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.4740999999999999e-05 2023-09-23T20:53:38.691 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:38.691 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:38.691 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T20:53:38.692 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:38.534098+0000", 2023-09-23T20:53:38.692 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.027484150999999998 2023-09-23T20:53:38.692 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:38.692 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:38.692 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T20:53:38.692 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:38.534141+0000", 2023-09-23T20:53:38.692 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.2565999999999999e-05 2023-09-23T20:53:38.692 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T20:53:38.692 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T20:53:38.692 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T20:53:38.692 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:38.692 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:38.692 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_repop(client.34797.0:94645 6.6 e124/122 6:6cbc37c5:::benchmark_data_smithi026_223868_object94644:head v 124'5851, mlcod=0'0)", 2023-09-23T20:53:38.693 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T20:53:38.506368+0000", 2023-09-23T20:53:38.693 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 0.056320910000000002, 2023-09-23T20:53:38.693 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.027810346999999999, 2023-09-23T20:53:38.693 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T20:53:38.693 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:53:38.693 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T20:53:38.693 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:38.693 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T20:53:38.693 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:38.506368+0000", 2023-09-23T20:53:38.693 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T20:53:38.693 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:38.693 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:38.694 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T20:53:38.694 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:38.506368+0000", 2023-09-23T20:53:38.694 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T20:53:38.694 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:38.694 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:38.694 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T20:53:38.694 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:38.506368+0000", 2023-09-23T20:53:38.694 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999995 2023-09-23T20:53:38.694 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:38.694 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:38.694 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T20:53:38.695 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:38.506412+0000", 2023-09-23T20:53:38.695 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.4971e-05 2023-09-23T20:53:38.695 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:38.695 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:38.695 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T20:53:38.695 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:38.506424+0000", 2023-09-23T20:53:38.695 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.1062e-05 2023-09-23T20:53:38.695 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:38.695 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:38.695 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T20:53:38.695 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:38.506429+0000", 2023-09-23T20:53:38.695 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.3109999999999996e-06 2023-09-23T20:53:38.696 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:38.696 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:38.696 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T20:53:38.696 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:38.512508+0000", 2023-09-23T20:53:38.696 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0060795320000000003 2023-09-23T20:53:38.696 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:38.696 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:38.696 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T20:53:38.696 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:38.512532+0000", 2023-09-23T20:53:38.696 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.3521000000000001e-05 2023-09-23T20:53:38.696 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:38.696 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:38.697 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T20:53:38.697 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:38.534144+0000", 2023-09-23T20:53:38.697 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.021611872000000001 2023-09-23T20:53:38.697 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:38.697 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:38.697 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T20:53:38.697 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:38.534178+0000", 2023-09-23T20:53:38.697 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.4338000000000002e-05 2023-09-23T20:53:38.697 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T20:53:38.697 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T20:53:38.697 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T20:53:38.697 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T20:53:38.697 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T20:53:38.698 INFO:teuthology.orchestra.run.smithi070.stdout:} 2023-09-23T20:53:38.698 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:53:38.698 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:53:38.699 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:53:38.799 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:53:38.839 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:53:38.895 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 79 16 94931 94915 75.0261 43.5625 0.0307246 0.0133233 2023-09-23T20:53:38.957 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:53:38.963 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:53:38.963 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:53:39.064 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:53:39.078 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_ops_in_flight 2023-09-23T20:53:39.106 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:53:39.158 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:53:38 smithi070 ceph-mon[168282]: failed to encode map e124 with expected crc 2023-09-23T20:53:39.158 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:53:39.163 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:53:39.163 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:53:39.264 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:53:39.307 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:53:39.374 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:53:39.378 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:53:39.379 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:53:39.381 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:53:38 smithi026 ceph-mon[179688]: failed to encode map e124 with expected crc 2023-09-23T20:53:39.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:53:38 smithi026 ceph-mon[175151]: failed to encode map e124 with expected crc 2023-09-23T20:53:39.479 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:53:39.486 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T20:53:39.561 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:53:39.637 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:53:39.642 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:53:39.642 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:53:39.743 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:53:39.743 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:53:39.747 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T20:53:39.782 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:53:39.861 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:53:39 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[164029]: ::ffff:172.21.15.70 - - [23/Sep/2023:20:53:39] "GET /metrics HTTP/1.1" 200 36311 "" "Prometheus/2.43.0" 2023-09-23T20:53:39.884 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:53:39.886 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:53:39.887 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:53:39.895 INFO:tasks.radosbench.radosbench.0.smithi026.stdout:2023-09-23T20:53:39.896153+0000 min lat: 0.00125417 max lat: 2.40799 avg lat: 0.013398 2023-09-23T20:53:39.895 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T20:53:39.895 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 80 16 95587 95571 74.6011 41 0.0307232 0.013398 2023-09-23T20:53:39.988 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:53:40.026 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:53:40.035 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:53:39 smithi070 ceph-mon[168282]: pgmap v299: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 84 active+clean; 5.6 GiB data, 16 GiB used, 521 GiB / 536 GiB avail; 27 MiB/s wr, 436 op/s; 16604/277389 objects degraded (5.986%); 10836/277389 objects misplaced (3.906%); 2.6 MiB/s, 1 keys/s, 42 objects/s recovering 2023-09-23T20:53:40.036 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:53:39 smithi070 ceph-mon[168282]: Health check update: Degraded data redundancy: 16604/277389 objects degraded (5.986%), 3 pgs degraded (PG_DEGRADED) 2023-09-23T20:53:40.068 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:53:40.071 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:53:40.071 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:53:40.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:53:39 smithi026 ceph-mon[179688]: pgmap v299: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 84 active+clean; 5.6 GiB data, 16 GiB used, 521 GiB / 536 GiB avail; 27 MiB/s wr, 436 op/s; 16604/277389 objects degraded (5.986%); 10836/277389 objects misplaced (3.906%); 2.6 MiB/s, 1 keys/s, 42 objects/s recovering 2023-09-23T20:53:40.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:53:39 smithi026 ceph-mon[179688]: Health check update: Degraded data redundancy: 16604/277389 objects degraded (5.986%), 3 pgs degraded (PG_DEGRADED) 2023-09-23T20:53:40.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:53:39 smithi026 ceph-mon[175151]: pgmap v299: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 84 active+clean; 5.6 GiB data, 16 GiB used, 521 GiB / 536 GiB avail; 27 MiB/s wr, 436 op/s; 16604/277389 objects degraded (5.986%); 10836/277389 objects misplaced (3.906%); 2.6 MiB/s, 1 keys/s, 42 objects/s recovering 2023-09-23T20:53:40.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:53:39 smithi026 ceph-mon[175151]: Health check update: Degraded data redundancy: 16604/277389 objects degraded (5.986%), 3 pgs degraded (PG_DEGRADED) 2023-09-23T20:53:40.172 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:53:40.277 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:53:40.366 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:53:40.376 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:53:40.376 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:53:40.456 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:53:40.477 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:53:40.583 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:53:40.722 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:53:40.728 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:53:40.728 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:53:40.737 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:53:40.829 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:53:40.870 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:53:40.895 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 81 16 96144 96128 74.1102 34.8125 0.0363831 0.0134884 2023-09-23T20:53:40.920 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:53:40.924 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:53:40.924 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:53:41.024 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:53:41.029 INFO:teuthology.orchestra.run.smithi070.stdout:{ 2023-09-23T20:53:41.029 INFO:teuthology.orchestra.run.smithi070.stdout: "ops": [ 2023-09-23T20:53:41.029 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:41.029 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.34797.0:96223 6.3 6:c733bdf6:::benchmark_data_smithi026_223868_object96222:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e124)", 2023-09-23T20:53:41.029 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T20:53:41.018224+0000", 2023-09-23T20:53:41.029 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 0.0097088680000000007, 2023-09-23T20:53:41.029 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.009751437, 2023-09-23T20:53:41.029 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T20:53:41.029 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "waiting for sub ops", 2023-09-23T20:53:41.030 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T20:53:41.030 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.34797", 2023-09-23T20:53:41.030 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T20:53:41.030 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 96223 2023-09-23T20:53:41.030 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:41.030 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T20:53:41.030 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:41.030 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T20:53:41.030 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:41.018224+0000", 2023-09-23T20:53:41.030 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T20:53:41.030 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:41.030 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:41.031 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T20:53:41.031 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:41.018224+0000", 2023-09-23T20:53:41.031 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T20:53:41.031 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:41.031 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:41.031 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T20:53:41.031 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:41.018222+0000", 2023-09-23T20:53:41.031 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T20:53:41.031 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:41.031 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:41.031 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T20:53:41.031 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:41.018262+0000", 2023-09-23T20:53:41.031 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.9867999999999998e-05 2023-09-23T20:53:41.031 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:41.032 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:41.032 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T20:53:41.032 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:41.018278+0000", 2023-09-23T20:53:41.032 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.5719e-05 2023-09-23T20:53:41.032 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:41.032 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:41.032 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T20:53:41.032 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:41.018283+0000", 2023-09-23T20:53:41.032 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.536e-06 2023-09-23T20:53:41.032 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:41.032 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:41.032 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T20:53:41.032 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:41.018337+0000", 2023-09-23T20:53:41.033 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.3477999999999999e-05 2023-09-23T20:53:41.033 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:41.033 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:41.033 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T20:53:41.033 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:41.018480+0000", 2023-09-23T20:53:41.033 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00014368000000000001 2023-09-23T20:53:41.033 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:41.033 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:41.033 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 2,6", 2023-09-23T20:53:41.033 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:41.018610+0000", 2023-09-23T20:53:41.033 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00012969300000000001 2023-09-23T20:53:41.033 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:41.033 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:41.034 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:53:41.034 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:41.021458+0000", 2023-09-23T20:53:41.034 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0028483279999999998 2023-09-23T20:53:41.034 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:41.034 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:41.034 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T20:53:41.034 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:41.021560+0000", 2023-09-23T20:53:41.034 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000101671 2023-09-23T20:53:41.034 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T20:53:41.034 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T20:53:41.034 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T20:53:41.034 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:41.035 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:41.035 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.34797.0:96226 6.5 6:a7fff790:::benchmark_data_smithi026_223868_object96225:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e124)", 2023-09-23T20:53:41.035 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T20:53:41.018296+0000", 2023-09-23T20:53:41.035 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 0.0096367190000000002, 2023-09-23T20:53:41.035 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0097761129999999995, 2023-09-23T20:53:41.035 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T20:53:41.035 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "waiting for sub ops", 2023-09-23T20:53:41.035 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T20:53:41.035 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.34797", 2023-09-23T20:53:41.035 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T20:53:41.035 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 96226 2023-09-23T20:53:41.035 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:41.035 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T20:53:41.036 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:41.036 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T20:53:41.036 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:41.018296+0000", 2023-09-23T20:53:41.036 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T20:53:41.036 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:41.036 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:41.036 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T20:53:41.036 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:41.018296+0000", 2023-09-23T20:53:41.036 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T20:53:41.036 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:41.036 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:41.036 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T20:53:41.036 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:41.018295+0000", 2023-09-23T20:53:41.037 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999995 2023-09-23T20:53:41.037 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:41.037 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:41.037 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T20:53:41.037 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:41.018321+0000", 2023-09-23T20:53:41.037 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.5967999999999999e-05 2023-09-23T20:53:41.037 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:41.037 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:41.037 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T20:53:41.037 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:41.018327+0000", 2023-09-23T20:53:41.037 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.1820000000000003e-06 2023-09-23T20:53:41.037 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:41.037 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:41.038 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T20:53:41.038 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:41.018332+0000", 2023-09-23T20:53:41.038 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.6190000000000002e-06 2023-09-23T20:53:41.038 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:41.038 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:41.038 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T20:53:41.038 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:41.018373+0000", 2023-09-23T20:53:41.038 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.1099000000000001e-05 2023-09-23T20:53:41.038 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:41.038 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:41.038 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T20:53:41.038 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:41.018469+0000", 2023-09-23T20:53:41.039 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.5537000000000004e-05 2023-09-23T20:53:41.039 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:41.039 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:41.039 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 3,6", 2023-09-23T20:53:41.039 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:41.018567+0000", 2023-09-23T20:53:41.039 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.8317000000000001e-05 2023-09-23T20:53:41.039 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:41.039 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:41.039 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T20:53:41.039 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:41.021061+0000", 2023-09-23T20:53:41.039 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0024941270000000001 2023-09-23T20:53:41.039 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:41.040 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:41.040 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:53:41.040 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:41.022248+0000", 2023-09-23T20:53:41.040 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.001186704 2023-09-23T20:53:41.040 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T20:53:41.040 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T20:53:41.040 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T20:53:41.040 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:41.040 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:41.040 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_repop(client.34797.0:96220 6.6 e124/122 6:6d4da729:::benchmark_data_smithi026_223868_object96219:head v 124'5959, mlcod=0'0)", 2023-09-23T20:53:41.040 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T20:53:41.022017+0000", 2023-09-23T20:53:41.040 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 0.0059155700000000002, 2023-09-23T20:53:41.040 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0061534850000000002, 2023-09-23T20:53:41.040 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T20:53:41.041 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "started", 2023-09-23T20:53:41.041 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T20:53:41.041 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:41.041 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T20:53:41.041 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:41.022017+0000", 2023-09-23T20:53:41.041 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T20:53:41.041 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:41.041 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:41.041 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T20:53:41.041 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:41.022017+0000", 2023-09-23T20:53:41.041 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T20:53:41.041 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:41.041 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:41.042 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T20:53:41.042 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:41.022016+0000", 2023-09-23T20:53:41.042 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999995 2023-09-23T20:53:41.042 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:41.042 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:41.042 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T20:53:41.042 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:41.022071+0000", 2023-09-23T20:53:41.042 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.4843000000000003e-05 2023-09-23T20:53:41.042 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:41.042 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:41.042 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T20:53:41.042 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:41.022082+0000", 2023-09-23T20:53:41.043 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.0652000000000001e-05 2023-09-23T20:53:41.043 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:41.043 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:41.043 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T20:53:41.043 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:41.022087+0000", 2023-09-23T20:53:41.043 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.5690000000000004e-06 2023-09-23T20:53:41.043 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:41.043 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:41.043 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T20:53:41.043 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:41.022149+0000", 2023-09-23T20:53:41.043 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.1445999999999994e-05 2023-09-23T20:53:41.043 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:41.043 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:41.044 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T20:53:41.044 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:41.022176+0000", 2023-09-23T20:53:41.044 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.7076e-05 2023-09-23T20:53:41.044 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T20:53:41.044 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T20:53:41.044 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T20:53:41.044 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:41.044 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:41.044 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_repop(client.34797.0:96225 6.b e124/102 6:d7f18ab6:::benchmark_data_smithi026_223868_object96224:head v 124'5892, mlcod=124'5889)", 2023-09-23T20:53:41.044 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T20:53:41.022102+0000", 2023-09-23T20:53:41.044 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 0.0058309010000000003, 2023-09-23T20:53:41.044 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0061295819999999997, 2023-09-23T20:53:41.044 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T20:53:41.045 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "started", 2023-09-23T20:53:41.045 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T20:53:41.045 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:41.045 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T20:53:41.045 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:41.022102+0000", 2023-09-23T20:53:41.045 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T20:53:41.045 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:41.045 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:41.045 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T20:53:41.045 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:41.022102+0000", 2023-09-23T20:53:41.045 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T20:53:41.045 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:41.045 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:41.046 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T20:53:41.046 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:41.022101+0000", 2023-09-23T20:53:41.046 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999995 2023-09-23T20:53:41.046 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:41.046 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:41.046 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T20:53:41.046 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:41.022136+0000", 2023-09-23T20:53:41.046 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.5009000000000003e-05 2023-09-23T20:53:41.046 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:41.046 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:41.046 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T20:53:41.046 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:41.022152+0000", 2023-09-23T20:53:41.047 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.5347000000000001e-05 2023-09-23T20:53:41.047 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:41.047 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:41.047 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T20:53:41.047 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:41.022157+0000", 2023-09-23T20:53:41.047 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.7819999999999999e-06 2023-09-23T20:53:41.047 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:41.047 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:41.047 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T20:53:41.047 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:41.022197+0000", 2023-09-23T20:53:41.047 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.9805999999999998e-05 2023-09-23T20:53:41.047 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:41.047 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:41.048 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T20:53:41.048 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:41.022213+0000", 2023-09-23T20:53:41.048 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.5866999999999999e-05 2023-09-23T20:53:41.048 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T20:53:41.048 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T20:53:41.048 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T20:53:41.048 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:41.048 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:41.048 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_repop(client.34797.0:96228 6.b e124/102 6:dc6179e3:::benchmark_data_smithi026_223868_object96227:head v 124'5893, mlcod=124'5889)", 2023-09-23T20:53:41.048 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T20:53:41.022742+0000", 2023-09-23T20:53:41.048 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 0.005190241, 2023-09-23T20:53:41.048 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0055591140000000004, 2023-09-23T20:53:41.049 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T20:53:41.049 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "started", 2023-09-23T20:53:41.049 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T20:53:41.049 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:41.049 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T20:53:41.049 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:41.022742+0000", 2023-09-23T20:53:41.049 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T20:53:41.049 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:41.049 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:41.049 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T20:53:41.049 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:41.022742+0000", 2023-09-23T20:53:41.049 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T20:53:41.049 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:41.050 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:41.050 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T20:53:41.050 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:41.022742+0000", 2023-09-23T20:53:41.050 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999995 2023-09-23T20:53:41.050 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:41.050 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:41.050 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T20:53:41.050 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:41.022784+0000", 2023-09-23T20:53:41.050 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.2020000000000001e-05 2023-09-23T20:53:41.050 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:41.050 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:41.050 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T20:53:41.050 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:41.022796+0000", 2023-09-23T20:53:41.051 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.2726e-05 2023-09-23T20:53:41.051 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:41.051 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:41.051 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T20:53:41.051 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:41.022802+0000", 2023-09-23T20:53:41.051 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.4430000000000002e-06 2023-09-23T20:53:41.051 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:41.051 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:41.051 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T20:53:41.051 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:41.025098+0000", 2023-09-23T20:53:41.051 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0022963720000000001 2023-09-23T20:53:41.051 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:41.051 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:41.052 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T20:53:41.052 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:41.025118+0000", 2023-09-23T20:53:41.052 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.9381999999999998e-05 2023-09-23T20:53:41.052 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T20:53:41.052 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T20:53:41.052 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T20:53:41.052 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:41.052 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:41.052 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_repop(client.34797.0:96222 6.c e124/102 6:306909a7:::benchmark_data_smithi026_223868_object96221:head v 124'5960, mlcod=124'5958)", 2023-09-23T20:53:41.052 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T20:53:41.024017+0000", 2023-09-23T20:53:41.052 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 0.0039158810000000004, 2023-09-23T20:53:41.052 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.004343169, 2023-09-23T20:53:41.052 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T20:53:41.052 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "started", 2023-09-23T20:53:41.053 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T20:53:41.053 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:41.053 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T20:53:41.053 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:41.024017+0000", 2023-09-23T20:53:41.053 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T20:53:41.053 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:41.053 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:41.053 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T20:53:41.053 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:41.024017+0000", 2023-09-23T20:53:41.053 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T20:53:41.053 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:41.053 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:41.053 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T20:53:41.054 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:41.024016+0000", 2023-09-23T20:53:41.054 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999995 2023-09-23T20:53:41.054 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:41.054 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:41.054 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T20:53:41.054 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:41.024080+0000", 2023-09-23T20:53:41.054 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.4164000000000004e-05 2023-09-23T20:53:41.054 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:41.054 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:41.054 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T20:53:41.054 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:41.024093+0000", 2023-09-23T20:53:41.054 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.2904999999999999e-05 2023-09-23T20:53:41.054 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:41.055 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:41.055 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T20:53:41.055 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:41.024098+0000", 2023-09-23T20:53:41.055 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.9989999999999999e-06 2023-09-23T20:53:41.055 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:41.055 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:41.055 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T20:53:41.055 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:41.024149+0000", 2023-09-23T20:53:41.055 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.0445000000000002e-05 2023-09-23T20:53:41.055 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:41.055 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:41.055 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T20:53:41.055 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:41.024166+0000", 2023-09-23T20:53:41.056 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.7733000000000001e-05 2023-09-23T20:53:41.056 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T20:53:41.056 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T20:53:41.056 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T20:53:41.056 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:41.056 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:41.056 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_repop(client.34797.0:96230 6.6 e124/122 6:6ab528a4:::benchmark_data_smithi026_223868_object96229:head v 124'5960, mlcod=0'0)", 2023-09-23T20:53:41.056 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T20:53:41.026525+0000", 2023-09-23T20:53:41.056 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 0.001407007, 2023-09-23T20:53:41.056 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0018916359999999999, 2023-09-23T20:53:41.056 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T20:53:41.056 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "started", 2023-09-23T20:53:41.056 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T20:53:41.057 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:41.057 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T20:53:41.057 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:41.026525+0000", 2023-09-23T20:53:41.057 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T20:53:41.057 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:41.057 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:41.057 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T20:53:41.057 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:41.026525+0000", 2023-09-23T20:53:41.057 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T20:53:41.057 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:41.057 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:41.057 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T20:53:41.057 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:41.026525+0000", 2023-09-23T20:53:41.058 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999995 2023-09-23T20:53:41.058 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:41.058 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:41.058 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T20:53:41.058 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:41.026587+0000", 2023-09-23T20:53:41.058 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.1906e-05 2023-09-23T20:53:41.058 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:41.058 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:41.058 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T20:53:41.058 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:41.026597+0000", 2023-09-23T20:53:41.058 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.066e-05 2023-09-23T20:53:41.059 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:41.059 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:41.059 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T20:53:41.059 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:41.026603+0000", 2023-09-23T20:53:41.059 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.0000000000000002e-06 2023-09-23T20:53:41.059 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:41.059 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:41.059 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T20:53:41.059 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:41.027232+0000", 2023-09-23T20:53:41.059 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00062826199999999998 2023-09-23T20:53:41.059 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:41.059 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:41.059 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T20:53:41.060 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:41.027263+0000", 2023-09-23T20:53:41.060 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.0889999999999997e-05 2023-09-23T20:53:41.060 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T20:53:41.060 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T20:53:41.060 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T20:53:41.060 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:41.060 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:41.060 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_repop(client.34797.0:96231 6.c e124/102 6:3080ae48:::benchmark_data_smithi026_223868_object96230:head v 124'5961, mlcod=124'5958)", 2023-09-23T20:53:41.060 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T20:53:41.027672+0000", 2023-09-23T20:53:41.060 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 0.00026055400000000003, 2023-09-23T20:53:41.060 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00080489699999999995, 2023-09-23T20:53:41.060 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T20:53:41.061 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "started", 2023-09-23T20:53:41.061 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T20:53:41.061 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:41.061 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T20:53:41.061 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:41.027672+0000", 2023-09-23T20:53:41.061 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T20:53:41.061 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:41.061 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:41.061 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T20:53:41.061 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:41.027672+0000", 2023-09-23T20:53:41.061 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T20:53:41.061 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:41.061 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:41.062 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T20:53:41.062 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:41.027671+0000", 2023-09-23T20:53:41.062 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999995 2023-09-23T20:53:41.062 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:41.062 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:41.062 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T20:53:41.062 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:41.027705+0000", 2023-09-23T20:53:41.062 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.3793e-05 2023-09-23T20:53:41.062 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:41.062 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:41.062 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T20:53:41.062 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:41.027718+0000", 2023-09-23T20:53:41.063 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.2588e-05 2023-09-23T20:53:41.063 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:41.063 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:41.063 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T20:53:41.063 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:41.027724+0000", 2023-09-23T20:53:41.063 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.0390000000000001e-06 2023-09-23T20:53:41.063 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:41.063 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:41.063 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T20:53:41.063 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:41.027771+0000", 2023-09-23T20:53:41.063 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.778e-05 2023-09-23T20:53:41.063 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:41.063 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:41.064 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T20:53:41.064 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:41.027789+0000", 2023-09-23T20:53:41.064 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.7937000000000001e-05 2023-09-23T20:53:41.064 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T20:53:41.064 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T20:53:41.064 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T20:53:41.064 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:41.064 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:41.064 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_repop_reply(client.34797.0:96226 6.5 e124/102)", 2023-09-23T20:53:41.064 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T20:53:41.028094+0000", 2023-09-23T20:53:41.064 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 4294967295.9998384, 2023-09-23T20:53:41.064 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00043788399999999998, 2023-09-23T20:53:41.065 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T20:53:41.065 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "queued for pg", 2023-09-23T20:53:41.065 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T20:53:41.065 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:41.065 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T20:53:41.065 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:41.028094+0000", 2023-09-23T20:53:41.065 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T20:53:41.065 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:41.065 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:41.065 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T20:53:41.065 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:41.028094+0000", 2023-09-23T20:53:41.065 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T20:53:41.065 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:41.066 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:41.066 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T20:53:41.066 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:41.028094+0000", 2023-09-23T20:53:41.066 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999995 2023-09-23T20:53:41.066 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:41.066 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:41.066 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T20:53:41.066 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:41.028098+0000", 2023-09-23T20:53:41.066 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.5929999999999999e-06 2023-09-23T20:53:41.066 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:41.066 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:41.066 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T20:53:41.067 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:41.028112+0000", 2023-09-23T20:53:41.067 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.3974999999999999e-05 2023-09-23T20:53:41.067 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:53:41.067 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:53:41.067 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T20:53:41.067 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:41.028118+0000", 2023-09-23T20:53:41.067 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.9959999999999999e-06 2023-09-23T20:53:41.067 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T20:53:41.067 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T20:53:41.067 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T20:53:41.067 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T20:53:41.067 INFO:teuthology.orchestra.run.smithi070.stdout: ], 2023-09-23T20:53:41.068 INFO:teuthology.orchestra.run.smithi070.stdout: "num_ops": 9 2023-09-23T20:53:41.068 INFO:teuthology.orchestra.run.smithi070.stdout:} 2023-09-23T20:53:41.069 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:53:41.384 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:53:41.387 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:53:41.388 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:53:41.432 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_blocked_ops 2023-09-23T20:53:41.489 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:53:41.530 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:53:41.585 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:53:41.588 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:53:41.589 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:53:41.623 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 6, 7] out_osds: [5, 4] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-09-23T20:53:41.623 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T20:53:41.624 INFO:tasks.thrashosds.thrasher:inject_pause on osd.5 2023-09-23T20:53:41.624 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-09-23T20:53:41.624 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-09-23T20:53:41.624 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 config set filestore_inject_stall 3 2023-09-23T20:53:41.689 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:53:41.730 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:53:41.790 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:53:41.796 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:53:41.796 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:53:41.896 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 82 16 96720 96704 73.6457 36 0.00860525 0.0135719 2023-09-23T20:53:41.897 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:53:41.979 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:53:42.095 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:53:42.104 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:53:42.104 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:53:41 smithi026 ceph-mon[175151]: pgmap v300: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 84 active+clean; 5.8 GiB data, 16 GiB used, 521 GiB / 536 GiB avail; 34 MiB/s wr, 544 op/s; 16604/282873 objects degraded (5.870%); 10653/282873 objects misplaced (3.766%); 3.3 MiB/s, 0 keys/s, 53 objects/s recovering 2023-09-23T20:53:42.104 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:53:41 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:53:42.104 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:53:41 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:53:42.105 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:53:41 smithi026 ceph-mon[179688]: pgmap v300: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 84 active+clean; 5.8 GiB data, 16 GiB used, 521 GiB / 536 GiB avail; 34 MiB/s wr, 544 op/s; 16604/282873 objects degraded (5.870%); 10653/282873 objects misplaced (3.766%); 3.3 MiB/s, 0 keys/s, 53 objects/s recovering 2023-09-23T20:53:42.105 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:53:41 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:53:42.105 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:53:41 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:53:42.105 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:53:42.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:53:41 smithi070 ceph-mon[168282]: pgmap v300: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 84 active+clean; 5.8 GiB data, 16 GiB used, 521 GiB / 536 GiB avail; 34 MiB/s wr, 544 op/s; 16604/282873 objects degraded (5.870%); 10653/282873 objects misplaced (3.766%); 3.3 MiB/s, 0 keys/s, 53 objects/s recovering 2023-09-23T20:53:42.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:53:41 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:53:42.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:53:41 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:53:42.159 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:53:42.206 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:53:42.297 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:53:42.337 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:53:42.501 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:53:42.511 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:53:42.511 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:53:42.612 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:53:42.659 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:53:42.848 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:53:42.853 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:53:42.854 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:53:42.895 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 83 16 97258 97242 73.1639 33.625 0.0444536 0.0136611 2023-09-23T20:53:42.954 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:53:43.036 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:53:43.127 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0: {} 2023-09-23T20:53:43.184 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1: {} 2023-09-23T20:53:43.191 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:53:43.204 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:53:43.204 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:53:43.244 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2: {} 2023-09-23T20:53:43.305 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:53:43.305 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3: {} 2023-09-23T20:53:43.345 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:53:43.367 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4: {} 2023-09-23T20:53:43.395 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:53:43.401 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:53:43.402 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:53:43.415 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T20:53:43.481 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6: {} 2023-09-23T20:53:43.502 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:53:43.525 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T20:53:43.535 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:53:43.552 INFO:teuthology.orchestra.run.smithi026.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T20:53:43.553 INFO:teuthology.orchestra.run.smithi026.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T20:53:43.553 INFO:teuthology.orchestra.run.smithi026.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T20:53:43.553 INFO:teuthology.orchestra.run.smithi026.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T20:53:43.553 INFO:teuthology.orchestra.run.smithi026.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T20:53:43.553 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T20:53:43.553 INFO:teuthology.orchestra.run.smithi026.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T20:53:43.553 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T20:53:43.668 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:53:43.671 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:53:43.672 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:53:43.772 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:53:43.829 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:53:43.899 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 84 16 97809 97793 72.7031 34.4375 0.0589918 0.0137496 2023-09-23T20:53:44.001 INFO:teuthology.orchestra.run.smithi026.stderr:nodeep-scrub is unset 2023-09-23T20:53:44.121 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:53:44.129 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:53:44.129 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:53:44.214 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:53:43 smithi070 ceph-mon[168282]: pgmap v301: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 84 active+clean; 5.8 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 37 MiB/s wr, 594 op/s; 16604/286968 objects degraded (5.786%); 10480/286968 objects misplaced (3.652%); 4.2 MiB/s, 67 objects/s recovering 2023-09-23T20:53:44.215 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:53:43 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/3698293633' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:53:44.215 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:53:43 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:53:44.215 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:53:43 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/2997379849' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:53:44.230 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:53:44.282 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:53:44.388 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:53:43 smithi026 ceph-mon[175151]: pgmap v301: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 84 active+clean; 5.8 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 37 MiB/s wr, 594 op/s; 16604/286968 objects degraded (5.786%); 10480/286968 objects misplaced (3.652%); 4.2 MiB/s, 67 objects/s recovering 2023-09-23T20:53:44.388 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:53:43 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/3698293633' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:53:44.388 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:53:43 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:53:44.388 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:53:43 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/2997379849' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:53:44.389 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:53:43 smithi026 ceph-mon[179688]: pgmap v301: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 84 active+clean; 5.8 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 37 MiB/s wr, 594 op/s; 16604/286968 objects degraded (5.786%); 10480/286968 objects misplaced (3.652%); 4.2 MiB/s, 67 objects/s recovering 2023-09-23T20:53:44.389 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:53:43 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/3698293633' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:53:44.389 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:53:43 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:53:44.389 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:53:43 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/2997379849' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:53:44.394 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:53:44.401 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:53:44.401 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:53:44.432 INFO:teuthology.orchestra.run.smithi070.stdout:ERROR: (22) Invalid argument 2023-09-23T20:53:44.432 INFO:teuthology.orchestra.run.smithi070.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:53:44.502 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:53:44.607 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:53:44.654 INFO:teuthology.orchestra.run.smithi070.stdout:{ 2023-09-23T20:53:44.654 INFO:teuthology.orchestra.run.smithi070.stdout: "success": "filestore_inject_stall = '3' (not observed, change may require restart) " 2023-09-23T20:53:44.654 INFO:teuthology.orchestra.run.smithi070.stdout:} 2023-09-23T20:53:44.693 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:53:44.711 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:53:44.712 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:53:44.812 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:53:44.896 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 85 16 98393 98377 72.2775 36.5 0.0379272 0.0138291 2023-09-23T20:53:44.930 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:53:44.996 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:53:44 smithi070 ceph-mon[168282]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T20:53:44.997 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:53:44 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T20:53:44.997 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:53:44 smithi070 ceph-mon[168282]: osdmap e125: 8 total, 8 up, 6 in 2023-09-23T20:53:44.997 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:53:44 smithi070 ceph-mon[168282]: failed to encode map e125 with expected crc 2023-09-23T20:53:44.997 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:53:44 smithi070 ceph-mon[168282]: failed to encode map e125 with expected crc 2023-09-23T20:53:44.997 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:53:44 smithi070 ceph-mon[168282]: Health check update: Degraded data redundancy: 16604/286968 objects degraded (5.786%), 3 pgs degraded (PG_DEGRADED) 2023-09-23T20:53:45.066 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_historic_ops 2023-09-23T20:53:45.071 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:53:45.076 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:53:45.076 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:53:45.081 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:53:44 smithi026 ceph-mon[175151]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T20:53:45.081 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:53:44 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T20:53:45.081 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:53:44 smithi026 ceph-mon[175151]: osdmap e125: 8 total, 8 up, 6 in 2023-09-23T20:53:45.081 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:53:44 smithi026 ceph-mon[175151]: failed to encode map e125 with expected crc 2023-09-23T20:53:45.081 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:53:44 smithi026 ceph-mon[175151]: failed to encode map e125 with expected crc 2023-09-23T20:53:45.081 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:53:44 smithi026 ceph-mon[175151]: Health check update: Degraded data redundancy: 16604/286968 objects degraded (5.786%), 3 pgs degraded (PG_DEGRADED) 2023-09-23T20:53:45.081 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:53:44 smithi026 ceph-mon[179688]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T20:53:45.081 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:53:44 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T20:53:45.081 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:53:44 smithi026 ceph-mon[179688]: osdmap e125: 8 total, 8 up, 6 in 2023-09-23T20:53:45.081 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:53:44 smithi026 ceph-mon[179688]: failed to encode map e125 with expected crc 2023-09-23T20:53:45.082 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:53:44 smithi026 ceph-mon[179688]: failed to encode map e125 with expected crc 2023-09-23T20:53:45.082 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:53:44 smithi026 ceph-mon[179688]: Health check update: Degraded data redundancy: 16604/286968 objects degraded (5.786%), 3 pgs degraded (PG_DEGRADED) 2023-09-23T20:53:45.177 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:53:45.213 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:53:45.263 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:53:45.270 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:53:45.270 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:53:45.371 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:53:45.474 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:53:45.537 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:53:45.542 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:53:45.543 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:53:45.643 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:53:45.726 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:53:45.896 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 86 16 98997 98981 71.8762 37.75 0.0256925 0.0139073 2023-09-23T20:53:46.016 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:53:46.090 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T20:53:46.136 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:53:45 smithi026 ceph-mon[179688]: failed to encode map e125 with expected crc 2023-09-23T20:53:46.136 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:53:45 smithi026 ceph-mon[179688]: pgmap v303: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 84 active+clean; 5.9 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 44 MiB/s wr, 708 op/s; 16604/292023 objects degraded (5.686%); 10302/292023 objects misplaced (3.528%); 4.2 MiB/s, 66 objects/s recovering 2023-09-23T20:53:46.136 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:53:45 smithi026 ceph-mon[179688]: failed to encode map e125 with expected crc 2023-09-23T20:53:46.136 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:53:45 smithi026 ceph-mon[175151]: failed to encode map e125 with expected crc 2023-09-23T20:53:46.136 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:53:45 smithi026 ceph-mon[175151]: pgmap v303: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 84 active+clean; 5.9 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 44 MiB/s wr, 708 op/s; 16604/292023 objects degraded (5.686%); 10302/292023 objects misplaced (3.528%); 4.2 MiB/s, 66 objects/s recovering 2023-09-23T20:53:46.136 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:53:45 smithi026 ceph-mon[175151]: failed to encode map e125 with expected crc 2023-09-23T20:53:46.236 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:53:45 smithi070 ceph-mon[168282]: failed to encode map e125 with expected crc 2023-09-23T20:53:46.237 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:53:45 smithi070 ceph-mon[168282]: pgmap v303: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 84 active+clean; 5.9 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 44 MiB/s wr, 708 op/s; 16604/292023 objects degraded (5.686%); 10302/292023 objects misplaced (3.528%); 4.2 MiB/s, 66 objects/s recovering 2023-09-23T20:53:46.237 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:53:45 smithi070 ceph-mon[168282]: failed to encode map e125 with expected crc 2023-09-23T20:53:46.283 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:53:46.287 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:53:46.287 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:53:46.388 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:53:46.426 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:53:46.468 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T20:53:46.469 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:53:46.472 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:53:46.473 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:53:46.574 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:53:46.608 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:53:46.897 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 87 16 99562 99546 71.4557 35.3125 0.0376675 0.0139898 2023-09-23T20:53:46.917 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:53:46.926 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:53:46.926 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:53:47.021 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:53:47.027 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:53:47.060 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:53:47.111 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:53:47.115 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:53:47.115 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:53:47.216 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:53:47.354 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:53:47.455 INFO:teuthology.orchestra.run.smithi070.stdout:ERROR: (22) Invalid argument 2023-09-23T20:53:47.455 INFO:teuthology.orchestra.run.smithi070.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:53:47.456 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:53:47.463 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:53:47.463 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:53:47.464 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:53:47 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:53:47.464 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:53:47 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:53:47.464 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:53:47 smithi026 ceph-mon[175151]: pgmap v304: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 84 active+clean; 6.0 GiB data, 17 GiB used, 520 GiB / 536 GiB avail; 41 MiB/s wr, 663 op/s; 16604/293595 objects degraded (5.655%); 10132/293595 objects misplaced (3.451%); 4.7 MiB/s, 75 objects/s recovering 2023-09-23T20:53:47.466 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:53:47 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:53:47.466 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:53:47 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:53:47.466 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:53:47 smithi026 ceph-mon[179688]: pgmap v304: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 84 active+clean; 6.0 GiB data, 17 GiB used, 520 GiB / 536 GiB avail; 41 MiB/s wr, 663 op/s; 16604/293595 objects degraded (5.655%); 10132/293595 objects misplaced (3.451%); 4.7 MiB/s, 75 objects/s recovering 2023-09-23T20:53:47.478 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:53:47 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:53:47.478 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:53:47 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:53:47.478 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:53:47 smithi070 ceph-mon[168282]: pgmap v304: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 84 active+clean; 6.0 GiB data, 17 GiB used, 520 GiB / 536 GiB avail; 41 MiB/s wr, 663 op/s; 16604/293595 objects degraded (5.655%); 10132/293595 objects misplaced (3.451%); 4.7 MiB/s, 75 objects/s recovering 2023-09-23T20:53:47.565 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:53:47.644 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:53:47.663 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:53:47.740 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:53:47.749 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:53:47.750 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:53:47.851 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:53:47.865 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_ops_in_flight 2023-09-23T20:53:47.899 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 88 16 100109 100093 71.0308 34.1875 0.0262304 0.0140726 2023-09-23T20:53:47.903 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:53:48.012 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:53:48.016 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:53:48.016 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:53:48.117 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:53:48.221 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:53:48.327 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:53:48.339 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:53:48.340 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:53:48.440 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:53:48.470 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:53:48.519 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:53:48.522 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:53:48.522 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:53:48.623 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:53:48.656 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:53:48.700 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:53:48.703 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:53:48.704 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:53:48.805 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:53:48.833 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:53:48.874 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:53:48.879 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:53:48.879 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:53:48.899 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 89 16 100638 100622 70.6044 33.0625 0.0224955 0.0141577 2023-09-23T20:53:48.979 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:53:49.015 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:53:49.059 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:53:49.065 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:53:49.066 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:53:49.167 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:53:49.267 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:53:49.291 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:53:49.371 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:53:49.386 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:53:49.387 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:53:49.488 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:53:49.503 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:53:49 smithi026 ceph-mon[175151]: pgmap v305: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 84 active+clean; 6.0 GiB data, 17 GiB used, 520 GiB / 536 GiB avail; 38 MiB/s wr, 604 op/s; 16604/295551 objects degraded (5.618%); 10132/295551 objects misplaced (3.428%); 4.4 MiB/s, 70 objects/s recovering 2023-09-23T20:53:49.503 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:53:49 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:53:49.504 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:53:49 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:53:49.518 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:53:49.554 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:53:49.558 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:53:49.558 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:53:49.659 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:53:49.691 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:53:49.702 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0: {} 2023-09-23T20:53:49.728 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:53:49.734 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:53:49.734 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:53:49.751 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1: {} 2023-09-23T20:53:49.785 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2: {} 2023-09-23T20:53:49.819 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3: {} 2023-09-23T20:53:49.835 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:53:49.854 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4: {} 2023-09-23T20:53:49.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:53:49 smithi070 ceph-mon[168282]: pgmap v305: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 84 active+clean; 6.0 GiB data, 17 GiB used, 520 GiB / 536 GiB avail; 38 MiB/s wr, 604 op/s; 16604/295551 objects degraded (5.618%); 10132/295551 objects misplaced (3.428%); 4.4 MiB/s, 70 objects/s recovering 2023-09-23T20:53:49.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:53:49 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:53:49.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:53:49 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:53:49.875 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T20:53:49.876 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:53:49 smithi026 ceph-mon[179688]: pgmap v305: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 84 active+clean; 6.0 GiB data, 17 GiB used, 520 GiB / 536 GiB avail; 38 MiB/s wr, 604 op/s; 16604/295551 objects degraded (5.618%); 10132/295551 objects misplaced (3.428%); 4.4 MiB/s, 70 objects/s recovering 2023-09-23T20:53:49.876 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:53:49 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:53:49.876 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:53:49 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:53:49.877 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:53:49 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[164029]: ::ffff:172.21.15.70 - - [23/Sep/2023:20:53:49] "GET /metrics HTTP/1.1" 200 36331 "" "Prometheus/2.43.0" 2023-09-23T20:53:49.882 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:53:49.900 INFO:tasks.radosbench.radosbench.0.smithi026.stdout:Total time run: 90.0179 2023-09-23T20:53:49.900 INFO:tasks.radosbench.radosbench.0.smithi026.stdout:Total writes made: 101123 2023-09-23T20:53:49.900 INFO:tasks.radosbench.radosbench.0.smithi026.stdout:Write size: 65536 2023-09-23T20:53:49.900 INFO:tasks.radosbench.radosbench.0.smithi026.stdout:Object size: 65536 2023-09-23T20:53:49.900 INFO:tasks.radosbench.radosbench.0.smithi026.stdout:Bandwidth (MB/sec): 70.2103 2023-09-23T20:53:49.900 INFO:tasks.radosbench.radosbench.0.smithi026.stdout:Stddev Bandwidth: 33.9128 2023-09-23T20:53:49.900 INFO:tasks.radosbench.radosbench.0.smithi026.stdout:Max bandwidth (MB/sec): 196.5 2023-09-23T20:53:49.901 INFO:tasks.radosbench.radosbench.0.smithi026.stdout:Min bandwidth (MB/sec): 0 2023-09-23T20:53:49.901 INFO:tasks.radosbench.radosbench.0.smithi026.stdout:Average IOPS: 1123 2023-09-23T20:53:49.901 INFO:tasks.radosbench.radosbench.0.smithi026.stdout:Stddev IOPS: 542.604 2023-09-23T20:53:49.901 INFO:tasks.radosbench.radosbench.0.smithi026.stdout:Max IOPS: 3144 2023-09-23T20:53:49.901 INFO:tasks.radosbench.radosbench.0.smithi026.stdout:Min IOPS: 0 2023-09-23T20:53:49.901 INFO:tasks.radosbench.radosbench.0.smithi026.stdout:Average Latency(s): 0.0142375 2023-09-23T20:53:49.901 INFO:tasks.radosbench.radosbench.0.smithi026.stdout:Stddev Latency(s): 0.0305806 2023-09-23T20:53:49.901 INFO:tasks.radosbench.radosbench.0.smithi026.stdout:Max latency(s): 2.40799 2023-09-23T20:53:49.901 INFO:tasks.radosbench.radosbench.0.smithi026.stdout:Min latency(s): 0.00125417 2023-09-23T20:53:49.916 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6: {} 2023-09-23T20:53:49.921 INFO:tasks.radosbench.radosbench.0.smithi026.stdout:Cleaning up (deleting benchmark objects) 2023-09-23T20:53:49.949 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T20:53:49.954 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:53:49.970 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:53:49.970 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:53:49.980 INFO:teuthology.orchestra.run.smithi026.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T20:53:49.981 INFO:teuthology.orchestra.run.smithi026.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T20:53:49.981 INFO:teuthology.orchestra.run.smithi026.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T20:53:49.981 INFO:teuthology.orchestra.run.smithi026.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T20:53:49.981 INFO:teuthology.orchestra.run.smithi026.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T20:53:49.981 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T20:53:49.981 INFO:teuthology.orchestra.run.smithi026.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T20:53:49.981 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T20:53:50.072 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:53:50.185 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:53:50.349 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:53:50.369 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:53:50.369 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:53:50.372 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 6, 7] out_osds: [5, 4] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-09-23T20:53:50.372 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T20:53:50.373 INFO:tasks.thrashosds.thrasher:Growing pool unique_pool_0 2023-09-23T20:53:50.373 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T20:53:50.470 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:53:50.498 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:53:50.537 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:53:50.539 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:53:50.540 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:53:50.640 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:53:50.677 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:53:50.714 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:53:50.716 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:53:50.717 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:53:50.788 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:53:50 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/2604021592' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:53:50.789 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:53:50 smithi026 ceph-mon[175151]: Health check update: Degraded data redundancy: 16604/295551 objects degraded (5.618%), 3 pgs degraded (PG_DEGRADED) 2023-09-23T20:53:50.789 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:53:50 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:53:50.789 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:53:50 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:53:50.789 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:53:50 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/2604021592' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:53:50.789 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:53:50 smithi026 ceph-mon[179688]: Health check update: Degraded data redundancy: 16604/295551 objects degraded (5.618%), 3 pgs degraded (PG_DEGRADED) 2023-09-23T20:53:50.789 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:53:50 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:53:50.789 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:53:50 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:53:50.818 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:53:50.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:53:50 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/2604021592' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:53:50.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:53:50 smithi070 ceph-mon[168282]: Health check update: Degraded data redundancy: 16604/295551 objects degraded (5.618%), 3 pgs degraded (PG_DEGRADED) 2023-09-23T20:53:50.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:53:50 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:53:50.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:53:50 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:53:50.931 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:53:51.034 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:53:51.045 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:53:51.045 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:53:51.146 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:53:51.246 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:53:51.315 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:53:51.332 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:53:51.332 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:53:51.433 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:53:51.462 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:53:51.501 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:53:51.507 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:53:51.507 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:53:51.553 INFO:teuthology.orchestra.run.smithi026.stderr:noscrub is set 2023-09-23T20:53:51.597 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:53:51 smithi026 ceph-mon[179688]: pgmap v306: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 84 active+clean; 6.1 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 37 MiB/s wr, 586 op/s; 16604/300495 objects degraded (5.526%); 9951/300495 objects misplaced (3.312%); 4.4 MiB/s, 70 objects/s recovering 2023-09-23T20:53:51.597 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:53:51 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/555987444' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:53:51.597 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:53:51 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:53:51.597 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:53:51 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:53:51.597 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:53:51 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:53:51.598 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:53:51 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:53:51.598 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:53:51 smithi026 ceph-mon[175151]: pgmap v306: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 84 active+clean; 6.1 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 37 MiB/s wr, 586 op/s; 16604/300495 objects degraded (5.526%); 9951/300495 objects misplaced (3.312%); 4.4 MiB/s, 70 objects/s recovering 2023-09-23T20:53:51.598 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:53:51 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/555987444' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:53:51.598 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:53:51 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:53:51.598 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:53:51 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:53:51.598 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:53:51 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:53:51.598 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:53:51 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:53:51.609 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:53:51.732 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:53:51.804 INFO:teuthology.orchestra.run.smithi026.stdout:{ 2023-09-23T20:53:51.804 INFO:teuthology.orchestra.run.smithi026.stdout: "ops": [ 2023-09-23T20:53:51.804 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:53:51.804 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34797.0:102053 6.0 6:0f404d96:::benchmark_data_smithi026_223868_object927:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e126)", 2023-09-23T20:53:51.804 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T20:53:51.781777+0000", 2023-09-23T20:53:51.804 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 0.020077733, 2023-09-23T20:53:51.804 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.020114733999999999, 2023-09-23T20:53:51.804 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T20:53:51.804 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "waiting for sub ops", 2023-09-23T20:53:51.804 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T20:53:51.805 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34797", 2023-09-23T20:53:51.805 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T20:53:51.805 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 102053 2023-09-23T20:53:51.805 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:53:51.805 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T20:53:51.805 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:53:51.805 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T20:53:51.805 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:51.781777+0000", 2023-09-23T20:53:51.805 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:53:51.805 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:53:51.805 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:53:51.805 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T20:53:51.806 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:51.781777+0000", 2023-09-23T20:53:51.806 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:53:51.806 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:53:51.806 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:53:51.806 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T20:53:51.806 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:51.781776+0000", 2023-09-23T20:53:51.806 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999981 2023-09-23T20:53:51.806 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:53:51.806 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:53:51.806 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T20:53:51.806 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:51.781785+0000", 2023-09-23T20:53:51.807 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.6709999999999994e-06 2023-09-23T20:53:51.807 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:53:51.807 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:53:51.807 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T20:53:51.807 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:51.781807+0000", 2023-09-23T20:53:51.807 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.1809000000000001e-05 2023-09-23T20:53:51.807 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:53:51.807 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:53:51.807 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T20:53:51.807 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:51.781817+0000", 2023-09-23T20:53:51.807 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.6949999999999995e-06 2023-09-23T20:53:51.807 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:53:51.808 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:53:51.808 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:53:51.808 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:51.781897+0000", 2023-09-23T20:53:51.808 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.0624999999999995e-05 2023-09-23T20:53:51.808 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:53:51.808 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:53:51.808 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T20:53:51.808 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:51.782013+0000", 2023-09-23T20:53:51.808 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000115296 2023-09-23T20:53:51.808 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:53:51.808 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:53:51.808 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 2,3", 2023-09-23T20:53:51.809 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:51.782126+0000", 2023-09-23T20:53:51.809 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000113749 2023-09-23T20:53:51.809 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:53:51.809 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:53:51.809 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T20:53:51.809 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:51.790812+0000", 2023-09-23T20:53:51.809 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0086859350000000005 2023-09-23T20:53:51.809 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:53:51.809 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T20:53:51.809 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:53:51.810 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:53:51.810 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:53:51.810 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34797.0:102060 6.9 6:93961326:::benchmark_data_smithi026_223868_object934:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e126)", 2023-09-23T20:53:51.810 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T20:53:51.781833+0000", 2023-09-23T20:53:51.810 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 0.020021780999999999, 2023-09-23T20:53:51.810 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.020158505, 2023-09-23T20:53:51.810 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T20:53:51.810 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "waiting for sub ops", 2023-09-23T20:53:51.810 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T20:53:51.810 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34797", 2023-09-23T20:53:51.811 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T20:53:51.811 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 102060 2023-09-23T20:53:51.811 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:53:51.811 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T20:53:51.811 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:53:51.811 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T20:53:51.811 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:51.781833+0000", 2023-09-23T20:53:51.811 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:53:51.811 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:53:51.811 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:53:51.811 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T20:53:51.811 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:51.781833+0000", 2023-09-23T20:53:51.812 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:53:51.812 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:53:51.812 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:53:51.812 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T20:53:51.812 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:51.781833+0000", 2023-09-23T20:53:51.812 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T20:53:51.812 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:53:51.812 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:53:51.812 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T20:53:51.812 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:51.781836+0000", 2023-09-23T20:53:51.812 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.2820000000000001e-06 2023-09-23T20:53:51.813 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:53:51.813 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:53:51.813 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T20:53:51.813 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:51.781843+0000", 2023-09-23T20:53:51.813 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.6150000000000002e-06 2023-09-23T20:53:51.813 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:53:51.813 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:53:51.813 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T20:53:51.813 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:51.781848+0000", 2023-09-23T20:53:51.813 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.1900000000000003e-06 2023-09-23T20:53:51.814 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:53:51.814 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:53:51.814 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:53:51.814 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:51.789374+0000", 2023-09-23T20:53:51.814 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0075261879999999996 2023-09-23T20:53:51.814 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:53:51.814 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:53:51.814 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T20:53:51.814 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:51.789502+0000", 2023-09-23T20:53:51.814 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000128124 2023-09-23T20:53:51.814 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:53:51.815 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:53:51.815 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 2,7", 2023-09-23T20:53:51.815 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:51.789609+0000", 2023-09-23T20:53:51.815 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000106202 2023-09-23T20:53:51.815 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:53:51.815 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T20:53:51.815 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:53:51.815 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:53:51.815 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:53:51.815 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_repop(client.34797.0:102057 6.d e126/102 6:b19ef308:::benchmark_data_smithi026_223868_object931:head v 126'6424, mlcod=125'6421)", 2023-09-23T20:53:51.815 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T20:53:51.794210+0000", 2023-09-23T20:53:51.816 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 0.0076450889999999999, 2023-09-23T20:53:51.816 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0078647330000000005, 2023-09-23T20:53:51.816 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T20:53:51.816 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "started", 2023-09-23T20:53:51.816 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T20:53:51.816 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:53:51.816 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T20:53:51.816 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:51.794210+0000", 2023-09-23T20:53:51.816 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:53:51.816 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:53:51.816 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:53:51.816 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T20:53:51.816 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:51.794210+0000", 2023-09-23T20:53:51.817 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:53:51.817 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:53:51.817 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:53:51.817 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T20:53:51.817 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:51.794209+0000", 2023-09-23T20:53:51.817 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.999999 2023-09-23T20:53:51.817 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:53:51.817 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:53:51.817 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T20:53:51.817 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:51.794218+0000", 2023-09-23T20:53:51.817 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.3720000000000005e-06 2023-09-23T20:53:51.817 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:53:51.818 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:53:51.818 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T20:53:51.818 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:51.794232+0000", 2023-09-23T20:53:51.818 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.4148000000000001e-05 2023-09-23T20:53:51.818 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:53:51.818 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:53:51.818 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T20:53:51.818 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:51.794240+0000", 2023-09-23T20:53:51.818 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.4950000000000008e-06 2023-09-23T20:53:51.818 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:53:51.818 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:53:51.818 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:53:51.819 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:51.795956+0000", 2023-09-23T20:53:51.819 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.001715432 2023-09-23T20:53:51.819 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:53:51.819 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:53:51.819 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T20:53:51.819 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:51.795980+0000", 2023-09-23T20:53:51.819 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.4760999999999999e-05 2023-09-23T20:53:51.819 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:53:51.819 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T20:53:51.819 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:53:51.819 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:53:51.819 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:53:51.819 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_repop(client.34797.0:102059 6.d e126/102 6:b8c6846e:::benchmark_data_smithi026_223868_object933:head v 126'6425, mlcod=125'6421)", 2023-09-23T20:53:51.819 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T20:53:51.794395+0000", 2023-09-23T20:53:51.820 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 0.0074597980000000001, 2023-09-23T20:53:51.820 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0077453920000000002, 2023-09-23T20:53:51.820 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T20:53:51.820 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "started", 2023-09-23T20:53:51.820 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T20:53:51.820 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:53:51.820 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T20:53:51.820 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:51.794395+0000", 2023-09-23T20:53:51.821 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:53:51.821 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:53:51.821 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:53:51.821 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T20:53:51.821 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:51.794395+0000", 2023-09-23T20:53:51.821 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:53:51.821 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:53:51.821 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:53:51.821 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T20:53:51.821 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:51.794395+0000", 2023-09-23T20:53:51.821 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T20:53:51.822 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:53:51.822 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:53:51.822 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T20:53:51.822 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:51.794401+0000", 2023-09-23T20:53:51.822 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.1319999999999999e-06 2023-09-23T20:53:51.822 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:53:51.822 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:53:51.822 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T20:53:51.822 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:51.794411+0000", 2023-09-23T20:53:51.822 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0030999999999999e-05 2023-09-23T20:53:51.822 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:53:51.822 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:53:51.823 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T20:53:51.823 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:51.794418+0000", 2023-09-23T20:53:51.823 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.6730000000000003e-06 2023-09-23T20:53:51.823 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:53:51.823 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:53:51.823 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:53:51.823 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:51.799013+0000", 2023-09-23T20:53:51.823 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0045949449999999996 2023-09-23T20:53:51.823 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:53:51.823 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:53:51.823 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T20:53:51.823 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:51.799031+0000", 2023-09-23T20:53:51.824 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.8587999999999999e-05 2023-09-23T20:53:51.824 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:53:51.824 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T20:53:51.824 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:53:51.824 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:53:51.824 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:53:51.824 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_repop_reply(client.34797.0:102060 6.9 e126/102)", 2023-09-23T20:53:51.824 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T20:53:51.794498+0000", 2023-09-23T20:53:51.824 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 0.0073571449999999998, 2023-09-23T20:53:51.824 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0077066260000000003, 2023-09-23T20:53:51.824 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T20:53:51.825 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "queued for pg", 2023-09-23T20:53:51.825 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T20:53:51.825 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:53:51.825 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T20:53:51.825 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:51.794498+0000", 2023-09-23T20:53:51.825 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:53:51.825 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:53:51.825 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:53:51.825 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T20:53:51.825 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:51.794498+0000", 2023-09-23T20:53:51.825 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:53:51.826 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:53:51.826 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:53:51.826 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T20:53:51.826 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:51.794497+0000", 2023-09-23T20:53:51.826 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T20:53:51.826 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:53:51.826 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:53:51.826 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T20:53:51.826 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:51.794502+0000", 2023-09-23T20:53:51.826 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.702e-06 2023-09-23T20:53:51.826 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:53:51.826 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:53:51.827 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T20:53:51.827 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:51.794513+0000", 2023-09-23T20:53:51.827 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0608000000000001e-05 2023-09-23T20:53:51.827 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:53:51.827 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:53:51.827 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T20:53:51.827 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:51.794520+0000", 2023-09-23T20:53:51.827 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.3000000000000004e-06 2023-09-23T20:53:51.827 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:53:51.827 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T20:53:51.827 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:53:51.827 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:53:51.828 INFO:teuthology.orchestra.run.smithi026.stdout: ], 2023-09-23T20:53:51.828 INFO:teuthology.orchestra.run.smithi026.stdout: "num_ops": 5 2023-09-23T20:53:51.828 INFO:teuthology.orchestra.run.smithi026.stdout:} 2023-09-23T20:53:51.829 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:53:51.829 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:53:51.829 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:53:51.862 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:53:51.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:53:51 smithi070 ceph-mon[168282]: pgmap v306: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 84 active+clean; 6.1 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 37 MiB/s wr, 586 op/s; 16604/300495 objects degraded (5.526%); 9951/300495 objects misplaced (3.312%); 4.4 MiB/s, 70 objects/s recovering 2023-09-23T20:53:51.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:53:51 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/555987444' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:53:51.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:53:51 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:53:51.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:53:51 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:53:51.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:53:51 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:53:51.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:53:51 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:53:51.930 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:53:52.010 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:53:52.102 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:53:52.115 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:53:52.116 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:53:52.217 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:53:52.298 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:53:52.514 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:53:52.524 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:53:52.525 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:53:52.582 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_blocked_ops 2023-09-23T20:53:52.626 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:53:52.659 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:53:52.695 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:53:52.698 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:53:52.699 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:53:52.784 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:53:52 smithi026 ceph-mon[175151]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:53:52.784 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:53:52 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T20:53:52.784 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:53:52 smithi026 ceph-mon[175151]: osdmap e126: 8 total, 8 up, 6 in 2023-09-23T20:53:52.785 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:53:52 smithi026 ceph-mon[175151]: failed to encode map e126 with expected crc 2023-09-23T20:53:52.785 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:53:52 smithi026 ceph-mon[175151]: failed to encode map e126 with expected crc 2023-09-23T20:53:52.785 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:53:52 smithi026 ceph-mon[179688]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:53:52.785 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:53:52 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T20:53:52.785 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:53:52 smithi026 ceph-mon[179688]: osdmap e126: 8 total, 8 up, 6 in 2023-09-23T20:53:52.785 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:53:52 smithi026 ceph-mon[179688]: failed to encode map e126 with expected crc 2023-09-23T20:53:52.785 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:53:52 smithi026 ceph-mon[179688]: failed to encode map e126 with expected crc 2023-09-23T20:53:52.800 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:53:52.828 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:53:52 smithi070 ceph-mon[168282]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:53:52.829 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:53:52 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T20:53:52.829 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:53:52 smithi070 ceph-mon[168282]: osdmap e126: 8 total, 8 up, 6 in 2023-09-23T20:53:52.829 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:53:52 smithi070 ceph-mon[168282]: failed to encode map e126 with expected crc 2023-09-23T20:53:52.829 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:53:52 smithi070 ceph-mon[168282]: failed to encode map e126 with expected crc 2023-09-23T20:53:52.829 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:53:52.870 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:53:52.873 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:53:52.873 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:53:52.974 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:53:53.016 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T20:53:53.070 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:53:53.455 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:53:53.477 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:53:53.477 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:53:53.578 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:53:53.667 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:53:53 smithi026 ceph-mon[175151]: pgmap v308: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 84 active+clean; 6.1 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 32 MiB/s wr, 558 op/s; 16604/301773 objects degraded (5.502%); 9772/301773 objects misplaced (3.238%); 5.3 MiB/s, 85 objects/s recovering 2023-09-23T20:53:53.667 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:53:53 smithi026 ceph-mon[179688]: pgmap v308: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 84 active+clean; 6.1 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 32 MiB/s wr, 558 op/s; 16604/301773 objects degraded (5.502%); 9772/301773 objects misplaced (3.238%); 5.3 MiB/s, 85 objects/s recovering 2023-09-23T20:53:53.687 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:53:53.778 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:53:53.795 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:53:53.812 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:53:53.812 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:53:53.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:53:53 smithi070 ceph-mon[168282]: pgmap v308: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 84 active+clean; 6.1 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 32 MiB/s wr, 558 op/s; 16604/301773 objects degraded (5.502%); 9772/301773 objects misplaced (3.238%); 5.3 MiB/s, 85 objects/s recovering 2023-09-23T20:53:53.914 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:53:54.012 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:53:54.086 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:53:54.098 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:53:54.098 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:53:54.199 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:53:54.202 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T20:53:54.296 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:53:54.302 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:53:54.574 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:53:54.582 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:53:54.583 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:53:54.683 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:53:54.746 INFO:teuthology.orchestra.run.smithi026.stdout: 2023-09-23T20:53:54.751 INFO:teuthology.orchestra.run.smithi026.stderr:dumped all 2023-09-23T20:53:54.838 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:53:55.285 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:53:55.303 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:53:55.303 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:53:55.355 INFO:teuthology.orchestra.run.smithi026.stdout:{"pg_ready":true,"pg_map":{"version":309,"stamp":"2023-09-23T20:53:54.223435+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":6526408632,"num_objects":99601,"num_object_clones":0,"num_object_copies":298803,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":16604,"num_objects_misplaced":9585,"num_objects_unfound":0,"num_objects_dirty":99601,"num_whiteouts":0,"num_read":501,"num_read_kb":925,"num_write":102971,"num_write_kb":6474318,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1619,"num_bytes_recovered":106044406,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":98687,"ondisk_log_size":98687,"up":267,"acting":262,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":267,"num_osds":6,"num_per_pool_osds":6,"num_per_pool_omap_osds":6,"kb":562470912,"kb_used":18031784,"kb_used_data":17561272,"kb_used_omap":126,"kb_used_meta":470081,"kb_avail":544439128,"statfs":{"total":575970213888,"available":557505667072,"internally_reserved":0,"allocated":17982742528,"data_stored":17979446571,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":129413,"internal_metadata":481363579},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,1,5,2,4,3,1],"upper_bound":128},"perf_stat":{"commit_latency_ms":13,"apply_latency_ms":13,"commit_latency_ns":13000000,"apply_latency_ns":13000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":148045848,"num_objects":2260,"num_object_clones":0,"num_object_copies":6780,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":-717,"num_objects_unfound":0,"num_objects_dirty":2260,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":5354,"num_write_kb":243585,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":717,"num_bytes_recovered":46989312,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.009154"},"pg_stats":[{"pgid":"3.1a","version":"68'1","reported_seq":16,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-09-23T20:53:43.958459+0000","last_change":"2023-09-23T20:53:28.954203+0000","last_active":"2023-09-23T20:53:43.958459+0000","last_peered":"2023-09-23T20:53:43.958459+0000","last_clean":"2023-09-23T20:53:43.958459+0000","last_became_active":"2023-09-23T20:53:28.953842+0000","last_became_peered":"2023-09-23T20:53:28.953842+0000","last_unstale":"2023-09-23T20:53:43.958459+0000","last_undegraded":"2023-09-23T20:53:43.958459+0000","last_fullsized":"2023-09-23T20:53:43.958459+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:02:28.665664+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":107,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-09-23T20:53:43.961013+0000","last_change":"2023-09-23T20:53:28.942934+0000","last_active":"2023-09-23T20:53:43.961013+0000","last_peered":"2023-09-23T20:53:43.961013+0000","last_clean":"2023-09-23T20:53:43.961013+0000","last_became_active":"2023-09-23T20:53:28.942618+0000","last_became_peered":"2023-09-23T20:53:28.942618+0000","last_unstale":"2023-09-23T20:53:43.961013+0000","last_undegraded":"2023-09-23T20:53:43.961013+0000","last_fullsized":"2023-09-23T20:53:43.961013+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:11:53.011927+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":84,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-09-23T20:53:43.954634+0000","last_change":"2023-09-23T20:53:28.956930+0000","last_active":"2023-09-23T20:53:43.954634+0000","last_peered":"2023-09-23T20:53:43.954634+0000","last_clean":"2023-09-23T20:53:43.954634+0000","last_became_active":"2023-09-23T20:53:28.956607+0000","last_became_peered":"2023-09-23T20:53:28.956607+0000","last_unstale":"2023-09-23T20:53:43.954634+0000","last_undegraded":"2023-09-23T20:53:43.954634+0000","last_fullsized":"2023-09-23T20:53:43.954634+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:53:02.963454+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":80,"reported_epoch":124,"state":"active+clean","last_fresh":"2023-09-23T20:53:36.915096+0000","last_change":"2023-09-23T20:51:10.841420+0000","last_active":"2023-09-23T20:53:36.915096+0000","last_peered":"2023-09-23T20:53:36.915096+0000","last_clean":"2023-09-23T20:53:36.915096+0000","last_became_active":"2023-09-23T20:51:10.841112+0000","last_became_peered":"2023-09-23T20:51:10.841112+0000","last_unstale":"2023-09-23T20:53:36.915096+0000","last_undegraded":"2023-09-23T20:53:36.915096+0000","last_fullsized":"2023-09-23T20:53:36.915096+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:44:02.236650+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":107,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-09-23T20:53:43.959702+0000","last_change":"2023-09-23T20:51:10.833613+0000","last_active":"2023-09-23T20:53:43.959702+0000","last_peered":"2023-09-23T20:53:43.959702+0000","last_clean":"2023-09-23T20:53:43.959702+0000","last_became_active":"2023-09-23T20:51:10.833208+0000","last_became_peered":"2023-09-23T20:51:10.833208+0000","last_unstale":"2023-09-23T20:53:43.959702+0000","last_undegraded":"2023-09-23T20:53:43.959702+0000","last_fullsized":"2023-09-23T20:53:43.959702+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:51:28.246014+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":95,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-09-23T20:53:43.972441+0000","last_change":"2023-09-23T20:52:12.862262+0000","last_active":"2023-09-23T20:53:43.972441+0000","last_peered":"2023-09-23T20:53:43.972441+0000","last_clean":"2023-09-23T20:53:43.972441+0000","last_became_active":"2023-09-23T20:52:12.861972+0000","last_became_peered":"2023-09-23T20:52:12.861972+0000","last_unstale":"2023-09-23T20:53:43.972441+0000","last_undegraded":"2023-09-23T20:53:43.972441+0000","last_fullsized":"2023-09-23T20:53:43.972441+0000","mapping_epoch":100,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":101,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:10:42.887918+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":88,"reported_epoch":124,"state":"active+clean","last_fresh":"2023-09-23T20:53:36.914652+0000","last_change":"2023-09-23T20:52:12.862076+0000","last_active":"2023-09-23T20:53:36.914652+0000","last_peered":"2023-09-23T20:53:36.914652+0000","last_clean":"2023-09-23T20:53:36.914652+0000","last_became_active":"2023-09-23T20:52:12.861753+0000","last_became_peered":"2023-09-23T20:52:12.861753+0000","last_unstale":"2023-09-23T20:53:36.914652+0000","last_undegraded":"2023-09-23T20:53:36.914652+0000","last_fullsized":"2023-09-23T20:53:36.914652+0000","mapping_epoch":100,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":101,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:15:39.708595+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,6],"acting":[0,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":89,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-09-23T20:53:43.959773+0000","last_change":"2023-09-23T20:51:10.833424+0000","last_active":"2023-09-23T20:53:43.959773+0000","last_peered":"2023-09-23T20:53:43.959773+0000","last_clean":"2023-09-23T20:53:43.959773+0000","last_became_active":"2023-09-23T20:51:10.832808+0000","last_became_peered":"2023-09-23T20:51:10.832808+0000","last_unstale":"2023-09-23T20:53:43.959773+0000","last_undegraded":"2023-09-23T20:53:43.959773+0000","last_fullsized":"2023-09-23T20:53:43.959773+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:32:42.819583+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":78,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-09-23T20:53:43.956013+0000","last_change":"2023-09-23T20:52:12.860277+0000","last_active":"2023-09-23T20:53:43.956013+0000","last_peered":"2023-09-23T20:53:43.956013+0000","last_clean":"2023-09-23T20:53:43.956013+0000","last_became_active":"2023-09-23T20:52:12.859866+0000","last_became_peered":"2023-09-23T20:52:12.859866+0000","last_unstale":"2023-09-23T20:53:43.956013+0000","last_undegraded":"2023-09-23T20:53:43.956013+0000","last_fullsized":"2023-09-23T20:53:43.956013+0000","mapping_epoch":100,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":101,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:07:45.523326+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,3],"acting":[7,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":50,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-09-23T20:53:43.959671+0000","last_change":"2023-09-23T20:51:10.833790+0000","last_active":"2023-09-23T20:53:43.959671+0000","last_peered":"2023-09-23T20:53:43.959671+0000","last_clean":"2023-09-23T20:53:43.959671+0000","last_became_active":"2023-09-23T20:51:10.832804+0000","last_became_peered":"2023-09-23T20:51:10.832804+0000","last_unstale":"2023-09-23T20:53:43.959671+0000","last_undegraded":"2023-09-23T20:53:43.959671+0000","last_fullsized":"2023-09-23T20:53:43.959671+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:27:42.601674+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":105,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-09-23T20:53:43.953912+0000","last_change":"2023-09-23T20:53:28.971325+0000","last_active":"2023-09-23T20:53:43.953912+0000","last_peered":"2023-09-23T20:53:43.953912+0000","last_clean":"2023-09-23T20:53:43.953912+0000","last_became_active":"2023-09-23T20:53:28.951531+0000","last_became_peered":"2023-09-23T20:53:28.951531+0000","last_unstale":"2023-09-23T20:53:43.953912+0000","last_undegraded":"2023-09-23T20:53:43.953912+0000","last_fullsized":"2023-09-23T20:53:43.953912+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:32:16.755096+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":75,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-09-23T20:53:43.958087+0000","last_change":"2023-09-23T20:50:37.982103+0000","last_active":"2023-09-23T20:53:43.958087+0000","last_peered":"2023-09-23T20:53:43.958087+0000","last_clean":"2023-09-23T20:53:43.958087+0000","last_became_active":"2023-09-23T20:50:37.981791+0000","last_became_peered":"2023-09-23T20:50:37.981791+0000","last_unstale":"2023-09-23T20:53:43.958087+0000","last_undegraded":"2023-09-23T20:53:43.958087+0000","last_fullsized":"2023-09-23T20:53:43.958087+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:32:52.286186+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"79'22","reported_seq":69,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-09-23T20:53:43.954045+0000","last_change":"2023-09-23T20:53:28.940511+0000","last_active":"2023-09-23T20:53:43.954045+0000","last_peered":"2023-09-23T20:53:43.954045+0000","last_clean":"2023-09-23T20:53:43.954045+0000","last_became_active":"2023-09-23T20:53:28.929266+0000","last_became_peered":"2023-09-23T20:53:28.929266+0000","last_unstale":"2023-09-23T20:53:43.954045+0000","last_undegraded":"2023-09-23T20:53:43.954045+0000","last_fullsized":"2023-09-23T20:53:43.954045+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:00:45.427978+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4734,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,2],"acting":[7,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":56,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-09-23T20:53:43.971526+0000","last_change":"2023-09-23T20:53:28.942759+0000","last_active":"2023-09-23T20:53:43.971526+0000","last_peered":"2023-09-23T20:53:43.971526+0000","last_clean":"2023-09-23T20:53:43.971526+0000","last_became_active":"2023-09-23T20:53:28.942530+0000","last_became_peered":"2023-09-23T20:53:28.942530+0000","last_unstale":"2023-09-23T20:53:43.971526+0000","last_undegraded":"2023-09-23T20:53:43.971526+0000","last_fullsized":"2023-09-23T20:53:43.971526+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:29:19.573895+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,2],"acting":[6,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":104,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-09-23T20:53:43.953825+0000","last_change":"2023-09-23T20:53:28.945122+0000","last_active":"2023-09-23T20:53:43.953825+0000","last_peered":"2023-09-23T20:53:43.953825+0000","last_clean":"2023-09-23T20:53:43.953825+0000","last_became_active":"2023-09-23T20:53:28.933243+0000","last_became_peered":"2023-09-23T20:53:28.933243+0000","last_unstale":"2023-09-23T20:53:43.953825+0000","last_undegraded":"2023-09-23T20:53:43.953825+0000","last_fullsized":"2023-09-23T20:53:43.953825+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:25:30.855922+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,6],"acting":[7,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":89,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-09-23T20:53:43.959967+0000","last_change":"2023-09-23T20:51:10.831262+0000","last_active":"2023-09-23T20:53:43.959967+0000","last_peered":"2023-09-23T20:53:43.959967+0000","last_clean":"2023-09-23T20:53:43.959967+0000","last_became_active":"2023-09-23T20:51:10.830931+0000","last_became_peered":"2023-09-23T20:51:10.830931+0000","last_unstale":"2023-09-23T20:53:43.959967+0000","last_undegraded":"2023-09-23T20:53:43.959967+0000","last_fullsized":"2023-09-23T20:53:43.959967+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:51:54.982505+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":79,"reported_epoch":124,"state":"active+clean","last_fresh":"2023-09-23T20:53:36.920067+0000","last_change":"2023-09-23T20:51:10.840590+0000","last_active":"2023-09-23T20:53:36.920067+0000","last_peered":"2023-09-23T20:53:36.920067+0000","last_clean":"2023-09-23T20:53:36.920067+0000","last_became_active":"2023-09-23T20:51:10.840221+0000","last_became_peered":"2023-09-23T20:51:10.840221+0000","last_unstale":"2023-09-23T20:53:36.920067+0000","last_undegraded":"2023-09-23T20:53:36.920067+0000","last_fullsized":"2023-09-23T20:53:36.920067+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:03:39.138638+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":87,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-09-23T20:53:43.953845+0000","last_change":"2023-09-23T20:52:12.860214+0000","last_active":"2023-09-23T20:53:43.953845+0000","last_peered":"2023-09-23T20:53:43.953845+0000","last_clean":"2023-09-23T20:53:43.953845+0000","last_became_active":"2023-09-23T20:52:12.859837+0000","last_became_peered":"2023-09-23T20:52:12.859837+0000","last_unstale":"2023-09-23T20:53:43.953845+0000","last_undegraded":"2023-09-23T20:53:43.953845+0000","last_fullsized":"2023-09-23T20:53:43.953845+0000","mapping_epoch":100,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":101,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:01:38.149335+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,6],"acting":[7,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":79,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-09-23T20:53:43.953604+0000","last_change":"2023-09-23T20:53:28.951600+0000","last_active":"2023-09-23T20:53:43.953604+0000","last_peered":"2023-09-23T20:53:43.953604+0000","last_clean":"2023-09-23T20:53:43.953604+0000","last_became_active":"2023-09-23T20:53:28.951288+0000","last_became_peered":"2023-09-23T20:53:28.951288+0000","last_unstale":"2023-09-23T20:53:43.953604+0000","last_undegraded":"2023-09-23T20:53:43.953604+0000","last_fullsized":"2023-09-23T20:53:43.953604+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:10:05.697814+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,0],"acting":[7,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":79,"reported_epoch":124,"state":"active+clean","last_fresh":"2023-09-23T20:53:36.915008+0000","last_change":"2023-09-23T20:51:10.840849+0000","last_active":"2023-09-23T20:53:36.915008+0000","last_peered":"2023-09-23T20:53:36.915008+0000","last_clean":"2023-09-23T20:53:36.915008+0000","last_became_active":"2023-09-23T20:51:10.840593+0000","last_became_peered":"2023-09-23T20:51:10.840593+0000","last_unstale":"2023-09-23T20:53:36.915008+0000","last_undegraded":"2023-09-23T20:53:36.915008+0000","last_fullsized":"2023-09-23T20:53:36.915008+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:30:52.297625+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":114,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-09-23T20:53:43.970876+0000","last_change":"2023-09-23T20:52:12.870672+0000","last_active":"2023-09-23T20:53:43.970876+0000","last_peered":"2023-09-23T20:53:43.970876+0000","last_clean":"2023-09-23T20:53:43.970876+0000","last_became_active":"2023-09-23T20:52:12.862084+0000","last_became_peered":"2023-09-23T20:52:12.862084+0000","last_unstale":"2023-09-23T20:53:43.970876+0000","last_undegraded":"2023-09-23T20:53:43.970876+0000","last_fullsized":"2023-09-23T20:53:43.970876+0000","mapping_epoch":100,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":101,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:32:58.322893+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":89,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-09-23T20:53:43.969050+0000","last_change":"2023-09-23T20:51:10.833234+0000","last_active":"2023-09-23T20:53:43.969050+0000","last_peered":"2023-09-23T20:53:43.969050+0000","last_clean":"2023-09-23T20:53:43.969050+0000","last_became_active":"2023-09-23T20:51:10.832763+0000","last_became_peered":"2023-09-23T20:51:10.832763+0000","last_unstale":"2023-09-23T20:53:43.969050+0000","last_undegraded":"2023-09-23T20:53:43.969050+0000","last_fullsized":"2023-09-23T20:53:43.969050+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:24:08.283578+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.a","version":"126'6419","reported_seq":6451,"reported_epoch":126,"state":"active+clean","last_fresh":"2023-09-23T20:53:52.049843+0000","last_change":"2023-09-23T20:52:14.865442+0000","last_active":"2023-09-23T20:53:52.049843+0000","last_peered":"2023-09-23T20:53:52.049843+0000","last_clean":"2023-09-23T20:53:52.049843+0000","last_became_active":"2023-09-23T20:52:14.865018+0000","last_became_peered":"2023-09-23T20:52:14.865018+0000","last_unstale":"2023-09-23T20:53:52.049843+0000","last_undegraded":"2023-09-23T20:53:52.049843+0000","last_fullsized":"2023-09-23T20:53:52.049843+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":102,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:52:13.843048+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:52:13.843048+0000","last_clean_scrub_stamp":"2023-09-23T20:52:13.843048+0000","objects_scrubbed":0,"log_size":6419,"log_dups_size":0,"ondisk_log_size":6419,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:32:46.511747+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":412811264,"num_objects":6299,"num_object_clones":0,"num_object_copies":18897,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6299,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6419,"num_write_kb":406976,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,2],"acting":[6,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":115,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-09-23T20:53:43.953951+0000","last_change":"2023-09-23T20:53:28.965170+0000","last_active":"2023-09-23T20:53:43.953951+0000","last_peered":"2023-09-23T20:53:43.953951+0000","last_clean":"2023-09-23T20:53:43.953951+0000","last_became_active":"2023-09-23T20:53:28.951216+0000","last_became_peered":"2023-09-23T20:53:28.951216+0000","last_unstale":"2023-09-23T20:53:43.953951+0000","last_undegraded":"2023-09-23T20:53:43.953951+0000","last_fullsized":"2023-09-23T20:53:43.953951+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:32:59.435617+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":48,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-09-23T20:53:43.953365+0000","last_change":"2023-09-23T20:53:28.941715+0000","last_active":"2023-09-23T20:53:43.953365+0000","last_peered":"2023-09-23T20:53:43.953365+0000","last_clean":"2023-09-23T20:53:43.953365+0000","last_became_active":"2023-09-23T20:53:28.941447+0000","last_became_peered":"2023-09-23T20:53:28.941447+0000","last_unstale":"2023-09-23T20:53:43.953365+0000","last_undegraded":"2023-09-23T20:53:43.953365+0000","last_fullsized":"2023-09-23T20:53:43.953365+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:25:14.407363+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,3],"acting":[7,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"6.b","version":"126'6322","reported_seq":6355,"reported_epoch":126,"state":"active+clean","last_fresh":"2023-09-23T20:53:53.829744+0000","last_change":"2023-09-23T20:52:14.868057+0000","last_active":"2023-09-23T20:53:53.829744+0000","last_peered":"2023-09-23T20:53:53.829744+0000","last_clean":"2023-09-23T20:53:53.829744+0000","last_became_active":"2023-09-23T20:52:14.867734+0000","last_became_peered":"2023-09-23T20:52:14.867734+0000","last_unstale":"2023-09-23T20:53:53.829744+0000","last_undegraded":"2023-09-23T20:53:53.829744+0000","last_fullsized":"2023-09-23T20:53:53.829744+0000","mapping_epoch":102,"log_start":"107'500","ondisk_log_start":"107'500","created":102,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:52:13.843048+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:52:13.843048+0000","last_clean_scrub_stamp":"2023-09-23T20:52:13.843048+0000","objects_scrubbed":0,"log_size":5822,"log_dups_size":0,"ondisk_log_size":5822,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:40:31.506977+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":397672448,"num_objects":6068,"num_object_clones":0,"num_object_copies":18204,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6068,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6322,"num_write_kb":396480,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":102,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-09-23T20:53:43.956135+0000","last_change":"2023-09-23T20:53:28.968818+0000","last_active":"2023-09-23T20:53:43.956135+0000","last_peered":"2023-09-23T20:53:43.956135+0000","last_clean":"2023-09-23T20:53:43.956135+0000","last_became_active":"2023-09-23T20:53:28.951289+0000","last_became_peered":"2023-09-23T20:53:28.951289+0000","last_unstale":"2023-09-23T20:53:43.956135+0000","last_undegraded":"2023-09-23T20:53:43.956135+0000","last_fullsized":"2023-09-23T20:53:43.956135+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:03:04.353343+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":82,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-09-23T20:53:43.954564+0000","last_change":"2023-09-23T20:53:28.956879+0000","last_active":"2023-09-23T20:53:43.954564+0000","last_peered":"2023-09-23T20:53:43.954564+0000","last_clean":"2023-09-23T20:53:43.954564+0000","last_became_active":"2023-09-23T20:53:28.956554+0000","last_became_peered":"2023-09-23T20:53:28.956554+0000","last_unstale":"2023-09-23T20:53:43.954564+0000","last_undegraded":"2023-09-23T20:53:43.954564+0000","last_fullsized":"2023-09-23T20:53:43.954564+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:38:18.079214+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.8","version":"126'6424","reported_seq":6456,"reported_epoch":126,"state":"active+clean","last_fresh":"2023-09-23T20:53:51.959991+0000","last_change":"2023-09-23T20:52:14.869562+0000","last_active":"2023-09-23T20:53:51.959991+0000","last_peered":"2023-09-23T20:53:51.959991+0000","last_clean":"2023-09-23T20:53:51.959991+0000","last_became_active":"2023-09-23T20:52:14.869204+0000","last_became_peered":"2023-09-23T20:52:14.869204+0000","last_unstale":"2023-09-23T20:53:51.959991+0000","last_undegraded":"2023-09-23T20:53:51.959991+0000","last_fullsized":"2023-09-23T20:53:51.959991+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":102,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:52:13.843048+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:52:13.843048+0000","last_clean_scrub_stamp":"2023-09-23T20:52:13.843048+0000","objects_scrubbed":0,"log_size":6424,"log_dups_size":0,"ondisk_log_size":6424,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:32:14.355022+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":412483584,"num_objects":6294,"num_object_clones":0,"num_object_copies":18882,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6294,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6424,"num_write_kb":406976,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,3],"acting":[7,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":84,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-09-23T20:53:43.954000+0000","last_change":"2023-09-23T20:52:12.866062+0000","last_active":"2023-09-23T20:53:43.954000+0000","last_peered":"2023-09-23T20:53:43.954000+0000","last_clean":"2023-09-23T20:53:43.954000+0000","last_became_active":"2023-09-23T20:52:12.861064+0000","last_became_peered":"2023-09-23T20:52:12.861064+0000","last_unstale":"2023-09-23T20:53:43.954000+0000","last_undegraded":"2023-09-23T20:53:43.954000+0000","last_fullsized":"2023-09-23T20:53:43.954000+0000","mapping_epoch":100,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":101,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:42:14.367193+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,3],"acting":[7,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":85,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-09-23T20:53:43.971380+0000","last_change":"2023-09-23T20:50:05.867973+0000","last_active":"2023-09-23T20:53:43.971380+0000","last_peered":"2023-09-23T20:53:43.971380+0000","last_clean":"2023-09-23T20:53:43.971380+0000","last_became_active":"2023-09-23T20:50:05.867622+0000","last_became_peered":"2023-09-23T20:50:05.867622+0000","last_unstale":"2023-09-23T20:53:43.971380+0000","last_undegraded":"2023-09-23T20:53:43.971380+0000","last_fullsized":"2023-09-23T20:53:43.971380+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:10:12.617597+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.9","version":"125'6424","reported_seq":6456,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-09-23T20:53:50.825981+0000","last_change":"2023-09-23T20:52:14.864297+0000","last_active":"2023-09-23T20:53:50.825981+0000","last_peered":"2023-09-23T20:53:50.825981+0000","last_clean":"2023-09-23T20:53:50.825981+0000","last_became_active":"2023-09-23T20:52:14.863986+0000","last_became_peered":"2023-09-23T20:52:14.863986+0000","last_unstale":"2023-09-23T20:53:50.825981+0000","last_undegraded":"2023-09-23T20:53:50.825981+0000","last_fullsized":"2023-09-23T20:53:50.825981+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":102,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:52:13.843048+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:52:13.843048+0000","last_clean_scrub_stamp":"2023-09-23T20:52:13.843048+0000","objects_scrubbed":0,"log_size":6424,"log_dups_size":0,"ondisk_log_size":6424,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:37:59.995892+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":417071104,"num_objects":6364,"num_object_clones":0,"num_object_copies":19092,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6364,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6424,"num_write_kb":409216,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,2],"acting":[0,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":87,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-09-23T20:53:43.960237+0000","last_change":"2023-09-23T20:52:12.858198+0000","last_active":"2023-09-23T20:53:43.960237+0000","last_peered":"2023-09-23T20:53:43.960237+0000","last_clean":"2023-09-23T20:53:43.960237+0000","last_became_active":"2023-09-23T20:52:12.857937+0000","last_became_peered":"2023-09-23T20:52:12.857937+0000","last_unstale":"2023-09-23T20:53:43.960237+0000","last_undegraded":"2023-09-23T20:53:43.960237+0000","last_fullsized":"2023-09-23T20:53:43.960237+0000","mapping_epoch":100,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":101,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:18:42.976887+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":101,"reported_epoch":124,"state":"active+clean","last_fresh":"2023-09-23T20:53:36.921782+0000","last_change":"2023-09-23T20:53:28.940363+0000","last_active":"2023-09-23T20:53:36.921782+0000","last_peered":"2023-09-23T20:53:36.921782+0000","last_clean":"2023-09-23T20:53:36.921782+0000","last_became_active":"2023-09-23T20:53:28.940032+0000","last_became_peered":"2023-09-23T20:53:28.940032+0000","last_unstale":"2023-09-23T20:53:36.921782+0000","last_undegraded":"2023-09-23T20:53:36.921782+0000","last_fullsized":"2023-09-23T20:53:36.921782+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:42:09.135039+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,3],"acting":[0,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.e","version":"126'6553","reported_seq":8338,"reported_epoch":126,"state":"active+recovering+undersized+remapped","last_fresh":"2023-09-23T20:53:53.996940+0000","last_change":"2023-09-23T20:53:30.118281+0000","last_active":"2023-09-23T20:53:53.996940+0000","last_peered":"2023-09-23T20:53:53.996940+0000","last_clean":"2023-09-23T20:53:27.848968+0000","last_became_active":"2023-09-23T20:53:30.095741+0000","last_became_peered":"2023-09-23T20:53:30.095741+0000","last_unstale":"2023-09-23T20:53:53.996940+0000","last_undegraded":"2023-09-23T20:53:53.996940+0000","last_fullsized":"2023-09-23T20:53:29.919588+0000","mapping_epoch":122,"log_start":"107'400","ondisk_log_start":"107'400","created":102,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:52:13.843048+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:52:13.843048+0000","last_clean_scrub_stamp":"2023-09-23T20:52:13.843048+0000","objects_scrubbed":0,"log_size":6153,"log_dups_size":0,"ondisk_log_size":6153,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:03:06.666044+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":411893760,"num_objects":6285,"num_object_clones":0,"num_object_copies":18855,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":4771,"num_objects_unfound":0,"num_objects_dirty":6285,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6553,"num_write_kb":410816,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":873,"num_bytes_recovered":57212928,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,7],"acting":[1,2],"avail_no_missing":["1","2","4"],"object_location_counts":[{"shards":"1,2","objects":1514},{"shards":"1,2,4","objects":4771}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":100,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-09-23T20:53:43.960058+0000","last_change":"2023-09-23T20:53:28.966957+0000","last_active":"2023-09-23T20:53:43.960058+0000","last_peered":"2023-09-23T20:53:43.960058+0000","last_clean":"2023-09-23T20:53:43.960058+0000","last_became_active":"2023-09-23T20:53:28.952861+0000","last_became_peered":"2023-09-23T20:53:28.952861+0000","last_unstale":"2023-09-23T20:53:43.960058+0000","last_undegraded":"2023-09-23T20:53:43.960058+0000","last_fullsized":"2023-09-23T20:53:43.960058+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:37:05.229099+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":65,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-09-23T20:53:43.960164+0000","last_change":"2023-09-23T20:53:28.953389+0000","last_active":"2023-09-23T20:53:43.960164+0000","last_peered":"2023-09-23T20:53:43.960164+0000","last_clean":"2023-09-23T20:53:43.960164+0000","last_became_active":"2023-09-23T20:53:28.953095+0000","last_became_peered":"2023-09-23T20:53:28.953095+0000","last_unstale":"2023-09-23T20:53:43.960164+0000","last_undegraded":"2023-09-23T20:53:43.960164+0000","last_fullsized":"2023-09-23T20:53:43.960164+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:01:21.803090+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.f","version":"125'6278","reported_seq":7753,"reported_epoch":125,"state":"active+recovering+undersized+remapped","last_fresh":"2023-09-23T20:53:51.163642+0000","last_change":"2023-09-23T20:53:30.409338+0000","last_active":"2023-09-23T20:53:51.163642+0000","last_peered":"2023-09-23T20:53:51.163642+0000","last_clean":"2023-09-23T20:53:27.873169+0000","last_became_active":"2023-09-23T20:53:30.309646+0000","last_became_peered":"2023-09-23T20:53:30.309646+0000","last_unstale":"2023-09-23T20:53:51.163642+0000","last_undegraded":"2023-09-23T20:53:51.163642+0000","last_fullsized":"2023-09-23T20:53:29.922158+0000","mapping_epoch":122,"log_start":"0'0","ondisk_log_start":"0'0","created":102,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:52:13.843048+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:52:13.843048+0000","last_clean_scrub_stamp":"2023-09-23T20:52:13.843048+0000","objects_scrubbed":0,"log_size":6278,"log_dups_size":0,"ondisk_log_size":6278,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:34:44.669919+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":407502848,"num_objects":6218,"num_object_clones":0,"num_object_copies":18654,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":4814,"num_objects_unfound":0,"num_objects_dirty":6218,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6278,"num_write_kb":399872,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":718,"num_bytes_recovered":47054848,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,6],"acting":[2,3],"avail_no_missing":["2","3","4"],"object_location_counts":[{"shards":"2,3","objects":1404},{"shards":"2,3,4","objects":4814}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":96,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-09-23T20:53:43.971250+0000","last_change":"2023-09-23T20:53:28.953461+0000","last_active":"2023-09-23T20:53:43.971250+0000","last_peered":"2023-09-23T20:53:43.971250+0000","last_clean":"2023-09-23T20:53:43.971250+0000","last_became_active":"2023-09-23T20:53:28.953170+0000","last_became_peered":"2023-09-23T20:53:28.953170+0000","last_unstale":"2023-09-23T20:53:43.971250+0000","last_undegraded":"2023-09-23T20:53:43.971250+0000","last_fullsized":"2023-09-23T20:53:43.971250+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:40:55.097468+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":76,"reported_epoch":124,"state":"active+clean","last_fresh":"2023-09-23T20:53:36.896857+0000","last_change":"2023-09-23T20:53:28.947600+0000","last_active":"2023-09-23T20:53:36.896857+0000","last_peered":"2023-09-23T20:53:36.896857+0000","last_clean":"2023-09-23T20:53:36.896857+0000","last_became_active":"2023-09-23T20:53:28.947295+0000","last_became_peered":"2023-09-23T20:53:28.947295+0000","last_unstale":"2023-09-23T20:53:36.896857+0000","last_undegraded":"2023-09-23T20:53:36.896857+0000","last_fullsized":"2023-09-23T20:53:36.896857+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:13:14.396149+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"6.c","version":"126'6429","reported_seq":6462,"reported_epoch":126,"state":"active+clean","last_fresh":"2023-09-23T20:53:53.904340+0000","last_change":"2023-09-23T20:52:14.865953+0000","last_active":"2023-09-23T20:53:53.904340+0000","last_peered":"2023-09-23T20:53:53.904340+0000","last_clean":"2023-09-23T20:53:53.904340+0000","last_became_active":"2023-09-23T20:52:14.865335+0000","last_became_peered":"2023-09-23T20:52:14.865335+0000","last_unstale":"2023-09-23T20:53:53.904340+0000","last_undegraded":"2023-09-23T20:53:53.904340+0000","last_fullsized":"2023-09-23T20:53:53.904340+0000","mapping_epoch":102,"log_start":"107'600","ondisk_log_start":"107'600","created":102,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:52:13.843048+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:52:13.843048+0000","last_clean_scrub_stamp":"2023-09-23T20:52:13.843048+0000","objects_scrubbed":0,"log_size":5829,"log_dups_size":0,"ondisk_log_size":5829,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:20:00.174949+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":402587648,"num_objects":6143,"num_object_clones":0,"num_object_copies":18429,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6143,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6429,"num_write_kb":402304,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":60,"reported_epoch":124,"state":"active+clean","last_fresh":"2023-09-23T20:53:36.896647+0000","last_change":"2023-09-23T20:53:28.937531+0000","last_active":"2023-09-23T20:53:36.896647+0000","last_peered":"2023-09-23T20:53:36.896647+0000","last_clean":"2023-09-23T20:53:36.896647+0000","last_became_active":"2023-09-23T20:53:28.937179+0000","last_became_peered":"2023-09-23T20:53:28.937179+0000","last_unstale":"2023-09-23T20:53:36.896647+0000","last_undegraded":"2023-09-23T20:53:36.896647+0000","last_fullsized":"2023-09-23T20:53:36.896647+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:31:56.855048+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,1],"acting":[2,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":56,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-09-23T20:53:43.971635+0000","last_change":"2023-09-23T20:53:28.953413+0000","last_active":"2023-09-23T20:53:43.971635+0000","last_peered":"2023-09-23T20:53:43.971635+0000","last_clean":"2023-09-23T20:53:43.971635+0000","last_became_active":"2023-09-23T20:53:28.953103+0000","last_became_peered":"2023-09-23T20:53:28.953103+0000","last_unstale":"2023-09-23T20:53:43.971635+0000","last_undegraded":"2023-09-23T20:53:43.971635+0000","last_fullsized":"2023-09-23T20:53:43.971635+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:13:32.759527+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.d","version":"126'6512","reported_seq":6546,"reported_epoch":126,"state":"active+clean","last_fresh":"2023-09-23T20:53:53.973725+0000","last_change":"2023-09-23T20:52:14.867585+0000","last_active":"2023-09-23T20:53:53.973725+0000","last_peered":"2023-09-23T20:53:53.973725+0000","last_clean":"2023-09-23T20:53:53.973725+0000","last_became_active":"2023-09-23T20:52:14.867310+0000","last_became_peered":"2023-09-23T20:52:14.867310+0000","last_unstale":"2023-09-23T20:53:53.973725+0000","last_undegraded":"2023-09-23T20:53:53.973725+0000","last_fullsized":"2023-09-23T20:53:53.973725+0000","mapping_epoch":102,"log_start":"106'300","ondisk_log_start":"106'300","created":102,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:52:13.843048+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:52:13.843048+0000","last_clean_scrub_stamp":"2023-09-23T20:52:13.843048+0000","objects_scrubbed":0,"log_size":6212,"log_dups_size":0,"ondisk_log_size":6212,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:19:26.814547+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":409141272,"num_objects":6244,"num_object_clones":0,"num_object_copies":18732,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6244,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":6512,"num_write_kb":408129,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,2],"acting":[1,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":90,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-09-23T20:53:43.959552+0000","last_change":"2023-09-23T20:51:10.831197+0000","last_active":"2023-09-23T20:53:43.959552+0000","last_peered":"2023-09-23T20:53:43.959552+0000","last_clean":"2023-09-23T20:53:43.959552+0000","last_became_active":"2023-09-23T20:51:10.830878+0000","last_became_peered":"2023-09-23T20:51:10.830878+0000","last_unstale":"2023-09-23T20:53:43.959552+0000","last_undegraded":"2023-09-23T20:53:43.959552+0000","last_fullsized":"2023-09-23T20:53:43.959552+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:05:24.314478+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":84,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-09-23T20:53:43.954319+0000","last_change":"2023-09-23T20:53:28.956849+0000","last_active":"2023-09-23T20:53:43.954319+0000","last_peered":"2023-09-23T20:53:43.954319+0000","last_clean":"2023-09-23T20:53:43.954319+0000","last_became_active":"2023-09-23T20:53:28.956527+0000","last_became_peered":"2023-09-23T20:53:28.956527+0000","last_unstale":"2023-09-23T20:53:43.954319+0000","last_undegraded":"2023-09-23T20:53:43.954319+0000","last_fullsized":"2023-09-23T20:53:43.954319+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:50:18.677782+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":98,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-09-23T20:53:43.955502+0000","last_change":"2023-09-23T20:50:05.867130+0000","last_active":"2023-09-23T20:53:43.955502+0000","last_peered":"2023-09-23T20:53:43.955502+0000","last_clean":"2023-09-23T20:53:43.955502+0000","last_became_active":"2023-09-23T20:50:05.866758+0000","last_became_peered":"2023-09-23T20:50:05.866758+0000","last_unstale":"2023-09-23T20:53:43.955502+0000","last_undegraded":"2023-09-23T20:53:43.955502+0000","last_fullsized":"2023-09-23T20:53:43.955502+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:55:07.548209+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":104,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-09-23T20:53:43.961161+0000","last_change":"2023-09-23T20:52:12.857372+0000","last_active":"2023-09-23T20:53:43.961161+0000","last_peered":"2023-09-23T20:53:43.961161+0000","last_clean":"2023-09-23T20:53:43.961161+0000","last_became_active":"2023-09-23T20:52:12.857135+0000","last_became_peered":"2023-09-23T20:52:12.857135+0000","last_unstale":"2023-09-23T20:53:43.961161+0000","last_undegraded":"2023-09-23T20:53:43.961161+0000","last_fullsized":"2023-09-23T20:53:43.961161+0000","mapping_epoch":100,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":101,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:50:17.844004+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":78,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-09-23T20:53:43.958169+0000","last_change":"2023-09-23T20:53:28.953913+0000","last_active":"2023-09-23T20:53:43.958169+0000","last_peered":"2023-09-23T20:53:43.958169+0000","last_clean":"2023-09-23T20:53:43.958169+0000","last_became_active":"2023-09-23T20:53:28.953361+0000","last_became_peered":"2023-09-23T20:53:28.953361+0000","last_unstale":"2023-09-23T20:53:43.958169+0000","last_undegraded":"2023-09-23T20:53:43.958169+0000","last_fullsized":"2023-09-23T20:53:43.958169+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:39:34.719764+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,2],"acting":[1,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.0","version":"125'6385","reported_seq":6417,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-09-23T20:53:50.766450+0000","last_change":"2023-09-23T20:52:14.866690+0000","last_active":"2023-09-23T20:53:50.766450+0000","last_peered":"2023-09-23T20:53:50.766450+0000","last_clean":"2023-09-23T20:53:50.766450+0000","last_became_active":"2023-09-23T20:52:14.866408+0000","last_became_peered":"2023-09-23T20:52:14.866408+0000","last_unstale":"2023-09-23T20:53:50.766450+0000","last_undegraded":"2023-09-23T20:53:50.766450+0000","last_fullsized":"2023-09-23T20:53:50.766450+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":102,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:52:13.843048+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:52:13.843048+0000","last_clean_scrub_stamp":"2023-09-23T20:52:13.843048+0000","objects_scrubbed":0,"log_size":6385,"log_dups_size":0,"ondisk_log_size":6385,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:29:07.905971+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":415170560,"num_objects":6335,"num_object_clones":0,"num_object_copies":19005,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6335,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6385,"num_write_kb":407040,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.6","version":"126'6417","reported_seq":6456,"reported_epoch":126,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T20:53:53.915853+0000","last_change":"2023-09-23T20:53:30.315049+0000","last_active":"2023-09-23T20:53:53.915853+0000","last_peered":"2023-09-23T20:53:53.915853+0000","last_clean":"2023-09-23T20:53:27.922544+0000","last_became_active":"2023-09-23T20:53:30.303440+0000","last_became_peered":"2023-09-23T20:53:30.303440+0000","last_unstale":"2023-09-23T20:53:53.915853+0000","last_undegraded":"2023-09-23T20:53:29.963994+0000","last_fullsized":"2023-09-23T20:53:29.929801+0000","mapping_epoch":122,"log_start":"107'600","ondisk_log_start":"107'600","created":102,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:52:13.843048+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:52:13.843048+0000","last_clean_scrub_stamp":"2023-09-23T20:52:13.843048+0000","objects_scrubbed":0,"log_size":5817,"log_dups_size":0,"ondisk_log_size":5817,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:18:26.756786+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":401408000,"num_objects":6125,"num_object_clones":0,"num_object_copies":18375,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5527,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6125,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6417,"num_write_kb":401344,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7],"avail_no_missing":["3","4","7"],"object_location_counts":[{"shards":"3,4,7","objects":5380},{"shards":"3,7","objects":598}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":81,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-09-23T20:53:43.958314+0000","last_change":"2023-09-23T20:52:12.860753+0000","last_active":"2023-09-23T20:53:43.958314+0000","last_peered":"2023-09-23T20:53:43.958314+0000","last_clean":"2023-09-23T20:53:43.958314+0000","last_became_active":"2023-09-23T20:52:12.860394+0000","last_became_peered":"2023-09-23T20:52:12.860394+0000","last_unstale":"2023-09-23T20:53:43.958314+0000","last_undegraded":"2023-09-23T20:53:43.958314+0000","last_fullsized":"2023-09-23T20:53:43.958314+0000","mapping_epoch":100,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":101,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:44:12.690612+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":50,"reported_epoch":124,"state":"active+clean","last_fresh":"2023-09-23T20:53:36.921564+0000","last_change":"2023-09-23T20:53:28.950222+0000","last_active":"2023-09-23T20:53:36.921564+0000","last_peered":"2023-09-23T20:53:36.921564+0000","last_clean":"2023-09-23T20:53:36.921564+0000","last_became_active":"2023-09-23T20:53:28.949364+0000","last_became_peered":"2023-09-23T20:53:28.949364+0000","last_unstale":"2023-09-23T20:53:36.921564+0000","last_undegraded":"2023-09-23T20:53:36.921564+0000","last_fullsized":"2023-09-23T20:53:36.921564+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:08:16.202646+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":90,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-09-23T20:53:43.955306+0000","last_change":"2023-09-23T20:51:10.829889+0000","last_active":"2023-09-23T20:53:43.955306+0000","last_peered":"2023-09-23T20:53:43.955306+0000","last_clean":"2023-09-23T20:53:43.955306+0000","last_became_active":"2023-09-23T20:51:10.829533+0000","last_became_peered":"2023-09-23T20:51:10.829533+0000","last_unstale":"2023-09-23T20:53:43.955306+0000","last_undegraded":"2023-09-23T20:53:43.955306+0000","last_fullsized":"2023-09-23T20:53:43.955306+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:43:04.603460+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.5","version":"126'6351","reported_seq":6383,"reported_epoch":126,"state":"active+clean","last_fresh":"2023-09-23T20:53:51.906474+0000","last_change":"2023-09-23T20:52:14.869287+0000","last_active":"2023-09-23T20:53:51.906474+0000","last_peered":"2023-09-23T20:53:51.906474+0000","last_clean":"2023-09-23T20:53:51.906474+0000","last_became_active":"2023-09-23T20:52:14.868955+0000","last_became_peered":"2023-09-23T20:52:14.868955+0000","last_unstale":"2023-09-23T20:53:51.906474+0000","last_undegraded":"2023-09-23T20:53:51.906474+0000","last_fullsized":"2023-09-23T20:53:51.906474+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":102,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:52:13.843048+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:52:13.843048+0000","last_clean_scrub_stamp":"2023-09-23T20:52:13.843048+0000","objects_scrubbed":0,"log_size":6351,"log_dups_size":0,"ondisk_log_size":6351,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:05:41.908990+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":407306240,"num_objects":6215,"num_object_clones":0,"num_object_copies":18645,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6215,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6351,"num_write_kb":402112,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,3],"acting":[7,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":108,"reported_epoch":124,"state":"active+clean","last_fresh":"2023-09-23T20:53:36.896565+0000","last_change":"2023-09-23T20:51:10.838054+0000","last_active":"2023-09-23T20:53:36.896565+0000","last_peered":"2023-09-23T20:53:36.896565+0000","last_clean":"2023-09-23T20:53:36.896565+0000","last_became_active":"2023-09-23T20:51:10.837750+0000","last_became_peered":"2023-09-23T20:51:10.837750+0000","last_unstale":"2023-09-23T20:53:36.896565+0000","last_undegraded":"2023-09-23T20:53:36.896565+0000","last_fullsized":"2023-09-23T20:53:36.896565+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:15:51.648085+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":75,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-09-23T20:53:43.947793+0000","last_change":"2023-09-23T20:50:37.983072+0000","last_active":"2023-09-23T20:53:43.947793+0000","last_peered":"2023-09-23T20:53:43.947793+0000","last_clean":"2023-09-23T20:53:43.947793+0000","last_became_active":"2023-09-23T20:50:37.982777+0000","last_became_peered":"2023-09-23T20:50:37.982777+0000","last_unstale":"2023-09-23T20:53:43.947793+0000","last_undegraded":"2023-09-23T20:53:43.947793+0000","last_fullsized":"2023-09-23T20:53:43.947793+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:58:11.972144+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":102,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-09-23T20:53:43.954401+0000","last_change":"2023-09-23T20:52:12.864016+0000","last_active":"2023-09-23T20:53:43.954401+0000","last_peered":"2023-09-23T20:53:43.954401+0000","last_clean":"2023-09-23T20:53:43.954401+0000","last_became_active":"2023-09-23T20:52:12.863714+0000","last_became_peered":"2023-09-23T20:52:12.863714+0000","last_unstale":"2023-09-23T20:53:43.954401+0000","last_undegraded":"2023-09-23T20:53:43.954401+0000","last_fullsized":"2023-09-23T20:53:43.954401+0000","mapping_epoch":100,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":101,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:20:44.035420+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.7","version":"126'6505","reported_seq":6545,"reported_epoch":126,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T20:53:53.865767+0000","last_change":"2023-09-23T20:53:30.117114+0000","last_active":"2023-09-23T20:53:53.865767+0000","last_peered":"2023-09-23T20:53:53.865767+0000","last_clean":"2023-09-23T20:53:27.924842+0000","last_became_active":"2023-09-23T20:53:30.109967+0000","last_became_peered":"2023-09-23T20:53:30.109967+0000","last_unstale":"2023-09-23T20:53:53.865767+0000","last_undegraded":"2023-09-23T20:53:29.968654+0000","last_fullsized":"2023-09-23T20:53:29.930999+0000","mapping_epoch":122,"log_start":"107'700","ondisk_log_start":"107'700","created":102,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:52:13.843048+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:52:13.843048+0000","last_clean_scrub_stamp":"2023-09-23T20:52:13.843048+0000","objects_scrubbed":0,"log_size":5805,"log_dups_size":0,"ondisk_log_size":5805,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:34:34.580505+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":409796608,"num_objects":6253,"num_object_clones":0,"num_object_copies":18759,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5672,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6253,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6505,"num_write_kb":408256,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1],"avail_no_missing":["3","1","4"],"object_location_counts":[{"shards":"1,3","objects":581},{"shards":"1,3,4","objects":5545}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":85,"reported_epoch":124,"state":"active+clean","last_fresh":"2023-09-23T20:53:36.897989+0000","last_change":"2023-09-23T20:52:12.853881+0000","last_active":"2023-09-23T20:53:36.897989+0000","last_peered":"2023-09-23T20:53:36.897989+0000","last_clean":"2023-09-23T20:53:36.897989+0000","last_became_active":"2023-09-23T20:52:12.853545+0000","last_became_peered":"2023-09-23T20:52:12.853545+0000","last_unstale":"2023-09-23T20:53:36.897989+0000","last_undegraded":"2023-09-23T20:53:36.897989+0000","last_fullsized":"2023-09-23T20:53:36.897989+0000","mapping_epoch":100,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":101,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:18:05.760976+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,3],"acting":[2,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"79'161","reported_seq":467,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-09-23T20:53:43.953437+0000","last_change":"2023-09-23T20:49:32.777246+0000","last_active":"2023-09-23T20:53:43.953437+0000","last_peered":"2023-09-23T20:53:43.953437+0000","last_clean":"2023-09-23T20:53:43.953437+0000","last_became_active":"2023-09-23T20:49:32.776822+0000","last_became_peered":"2023-09-23T20:49:32.776822+0000","last_unstale":"2023-09-23T20:53:43.953437+0000","last_undegraded":"2023-09-23T20:53:43.953437+0000","last_fullsized":"2023-09-23T20:53:43.953437+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:38:57.324224+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:05:46.881275+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":68,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-09-23T20:53:43.971866+0000","last_change":"2023-09-23T20:53:28.953465+0000","last_active":"2023-09-23T20:53:43.971866+0000","last_peered":"2023-09-23T20:53:43.971866+0000","last_clean":"2023-09-23T20:53:43.971866+0000","last_became_active":"2023-09-23T20:53:28.952966+0000","last_became_peered":"2023-09-23T20:53:28.952966+0000","last_unstale":"2023-09-23T20:53:43.971866+0000","last_undegraded":"2023-09-23T20:53:43.971866+0000","last_fullsized":"2023-09-23T20:53:43.971866+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:54:09.011044+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":78,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-09-23T20:53:43.972893+0000","last_change":"2023-09-23T20:53:28.951452+0000","last_active":"2023-09-23T20:53:43.972893+0000","last_peered":"2023-09-23T20:53:43.972893+0000","last_clean":"2023-09-23T20:53:43.972893+0000","last_became_active":"2023-09-23T20:53:28.951126+0000","last_became_peered":"2023-09-23T20:53:28.951126+0000","last_unstale":"2023-09-23T20:53:43.972893+0000","last_undegraded":"2023-09-23T20:53:43.972893+0000","last_fullsized":"2023-09-23T20:53:43.972893+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:00:18.163076+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,0],"acting":[6,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":107,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-09-23T20:53:43.953448+0000","last_change":"2023-09-23T20:50:05.866898+0000","last_active":"2023-09-23T20:53:43.953448+0000","last_peered":"2023-09-23T20:53:43.953448+0000","last_clean":"2023-09-23T20:53:43.953448+0000","last_became_active":"2023-09-23T20:50:05.866541+0000","last_became_peered":"2023-09-23T20:50:05.866541+0000","last_unstale":"2023-09-23T20:53:43.953448+0000","last_undegraded":"2023-09-23T20:53:43.953448+0000","last_fullsized":"2023-09-23T20:53:43.953448+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:17:02.353010+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":89,"reported_epoch":124,"state":"active+clean","last_fresh":"2023-09-23T20:53:40.856823+0000","last_change":"2023-09-23T20:53:28.940444+0000","last_active":"2023-09-23T20:53:40.856823+0000","last_peered":"2023-09-23T20:53:40.856823+0000","last_clean":"2023-09-23T20:53:40.856823+0000","last_became_active":"2023-09-23T20:53:28.939999+0000","last_became_peered":"2023-09-23T20:53:28.939999+0000","last_unstale":"2023-09-23T20:53:40.856823+0000","last_undegraded":"2023-09-23T20:53:40.856823+0000","last_fullsized":"2023-09-23T20:53:40.856823+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:11:25.486989+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":95,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-09-23T20:53:43.969518+0000","last_change":"2023-09-23T20:50:37.982235+0000","last_active":"2023-09-23T20:53:43.969518+0000","last_peered":"2023-09-23T20:53:43.969518+0000","last_clean":"2023-09-23T20:53:43.969518+0000","last_became_active":"2023-09-23T20:50:37.981913+0000","last_became_peered":"2023-09-23T20:50:37.981913+0000","last_unstale":"2023-09-23T20:53:43.969518+0000","last_undegraded":"2023-09-23T20:53:43.969518+0000","last_fullsized":"2023-09-23T20:53:43.969518+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:59:57.122584+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.4","version":"126'6442","reported_seq":6475,"reported_epoch":126,"state":"active+clean","last_fresh":"2023-09-23T20:53:53.935646+0000","last_change":"2023-09-23T20:52:14.869618+0000","last_active":"2023-09-23T20:53:53.935646+0000","last_peered":"2023-09-23T20:53:53.935646+0000","last_clean":"2023-09-23T20:53:53.935646+0000","last_became_active":"2023-09-23T20:52:14.869288+0000","last_became_peered":"2023-09-23T20:52:14.869288+0000","last_unstale":"2023-09-23T20:53:53.935646+0000","last_undegraded":"2023-09-23T20:53:53.935646+0000","last_fullsized":"2023-09-23T20:53:53.935646+0000","mapping_epoch":102,"log_start":"106'300","ondisk_log_start":"106'300","created":102,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:52:13.843048+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:52:13.843048+0000","last_clean_scrub_stamp":"2023-09-23T20:52:13.843048+0000","objects_scrubbed":0,"log_size":6142,"log_dups_size":0,"ondisk_log_size":6142,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:48:22.341973+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":402915328,"num_objects":6148,"num_object_clones":0,"num_object_copies":18444,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6148,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6442,"num_write_kb":402880,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":100,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-09-23T20:53:43.953940+0000","last_change":"2023-09-23T20:53:28.949747+0000","last_active":"2023-09-23T20:53:43.953940+0000","last_peered":"2023-09-23T20:53:43.953940+0000","last_clean":"2023-09-23T20:53:43.953940+0000","last_became_active":"2023-09-23T20:53:28.949418+0000","last_became_peered":"2023-09-23T20:53:28.949418+0000","last_unstale":"2023-09-23T20:53:43.953940+0000","last_undegraded":"2023-09-23T20:53:43.953940+0000","last_fullsized":"2023-09-23T20:53:43.953940+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:19:30.438122+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":84,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-09-23T20:53:43.955427+0000","last_change":"2023-09-23T20:52:12.853904+0000","last_active":"2023-09-23T20:53:43.955427+0000","last_peered":"2023-09-23T20:53:43.955427+0000","last_clean":"2023-09-23T20:53:43.955427+0000","last_became_active":"2023-09-23T20:52:12.853589+0000","last_became_peered":"2023-09-23T20:52:12.853589+0000","last_unstale":"2023-09-23T20:53:43.955427+0000","last_undegraded":"2023-09-23T20:53:43.955427+0000","last_fullsized":"2023-09-23T20:53:43.955427+0000","mapping_epoch":100,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":101,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:36:58.772320+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":74,"reported_epoch":124,"state":"active+clean","last_fresh":"2023-09-23T20:53:40.856872+0000","last_change":"2023-09-23T20:52:12.861675+0000","last_active":"2023-09-23T20:53:40.856872+0000","last_peered":"2023-09-23T20:53:40.856872+0000","last_clean":"2023-09-23T20:53:40.856872+0000","last_became_active":"2023-09-23T20:52:12.861342+0000","last_became_peered":"2023-09-23T20:52:12.861342+0000","last_unstale":"2023-09-23T20:53:40.856872+0000","last_undegraded":"2023-09-23T20:53:40.856872+0000","last_fullsized":"2023-09-23T20:53:40.856872+0000","mapping_epoch":100,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":101,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:37:54.207520+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.1","version":"126'6513","reported_seq":6546,"reported_epoch":126,"state":"active+clean","last_fresh":"2023-09-23T20:53:53.942158+0000","last_change":"2023-09-23T20:52:14.866659+0000","last_active":"2023-09-23T20:53:53.942158+0000","last_peered":"2023-09-23T20:53:53.942158+0000","last_clean":"2023-09-23T20:53:53.942158+0000","last_became_active":"2023-09-23T20:52:14.866348+0000","last_became_peered":"2023-09-23T20:52:14.866348+0000","last_unstale":"2023-09-23T20:53:53.942158+0000","last_undegraded":"2023-09-23T20:53:53.942158+0000","last_fullsized":"2023-09-23T20:53:53.942158+0000","mapping_epoch":102,"log_start":"106'300","ondisk_log_start":"106'300","created":102,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:52:13.843048+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:52:13.843048+0000","last_clean_scrub_stamp":"2023-09-23T20:52:13.843048+0000","objects_scrubbed":0,"log_size":6213,"log_dups_size":0,"ondisk_log_size":6213,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:51:07.595313+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":411369472,"num_objects":6277,"num_object_clones":0,"num_object_copies":18831,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6277,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6513,"num_write_kb":409280,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":87,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-09-23T20:53:43.969138+0000","last_change":"2023-09-23T20:51:10.836910+0000","last_active":"2023-09-23T20:53:43.969138+0000","last_peered":"2023-09-23T20:53:43.969138+0000","last_clean":"2023-09-23T20:53:43.969138+0000","last_became_active":"2023-09-23T20:51:10.836557+0000","last_became_peered":"2023-09-23T20:51:10.836557+0000","last_unstale":"2023-09-23T20:53:43.969138+0000","last_undegraded":"2023-09-23T20:53:43.969138+0000","last_fullsized":"2023-09-23T20:53:43.969138+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:51:54.019969+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":69,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-09-23T20:53:43.957995+0000","last_change":"2023-09-23T20:53:28.955886+0000","last_active":"2023-09-23T20:53:43.957995+0000","last_peered":"2023-09-23T20:53:43.957995+0000","last_clean":"2023-09-23T20:53:43.957995+0000","last_became_active":"2023-09-23T20:53:28.955151+0000","last_became_peered":"2023-09-23T20:53:28.955151+0000","last_unstale":"2023-09-23T20:53:43.957995+0000","last_undegraded":"2023-09-23T20:53:43.957995+0000","last_fullsized":"2023-09-23T20:53:43.957995+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:26:26.758782+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,3],"acting":[1,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":84,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-09-23T20:53:43.959629+0000","last_change":"2023-09-23T20:51:10.833546+0000","last_active":"2023-09-23T20:53:43.959629+0000","last_peered":"2023-09-23T20:53:43.959629+0000","last_clean":"2023-09-23T20:53:43.959629+0000","last_became_active":"2023-09-23T20:51:10.833137+0000","last_became_peered":"2023-09-23T20:51:10.833137+0000","last_unstale":"2023-09-23T20:53:43.959629+0000","last_undegraded":"2023-09-23T20:53:43.959629+0000","last_fullsized":"2023-09-23T20:53:43.959629+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:16:42.840695+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.2","version":"126'6335","reported_seq":6374,"reported_epoch":126,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T20:53:53.923100+0000","last_change":"2023-09-23T20:53:30.302152+0000","last_active":"2023-09-23T20:53:53.923100+0000","last_peered":"2023-09-23T20:53:53.923100+0000","last_clean":"2023-09-23T20:53:27.905324+0000","last_became_active":"2023-09-23T20:53:30.292966+0000","last_became_peered":"2023-09-23T20:53:30.292966+0000","last_unstale":"2023-09-23T20:53:53.923100+0000","last_undegraded":"2023-09-23T20:53:29.955817+0000","last_fullsized":"2023-09-23T20:53:29.922518+0000","mapping_epoch":122,"log_start":"107'500","ondisk_log_start":"107'500","created":102,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:52:13.843048+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:52:13.843048+0000","last_clean_scrub_stamp":"2023-09-23T20:52:13.843048+0000","objects_scrubbed":0,"log_size":5835,"log_dups_size":0,"ondisk_log_size":5835,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:06:25.220461+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":396296192,"num_objects":6047,"num_object_clones":0,"num_object_copies":18141,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5405,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6047,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6335,"num_write_kb":396224,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2],"avail_no_missing":["3","2","4"],"object_location_counts":[{"shards":"2,3","objects":642},{"shards":"2,3,4","objects":5261}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":90,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-09-23T20:53:43.972286+0000","last_change":"2023-09-23T20:50:37.985663+0000","last_active":"2023-09-23T20:53:43.972286+0000","last_peered":"2023-09-23T20:53:43.972286+0000","last_clean":"2023-09-23T20:53:43.972286+0000","last_became_active":"2023-09-23T20:50:37.985325+0000","last_became_peered":"2023-09-23T20:50:37.985325+0000","last_unstale":"2023-09-23T20:53:43.972286+0000","last_undegraded":"2023-09-23T20:53:43.972286+0000","last_fullsized":"2023-09-23T20:53:43.972286+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:24:30.838843+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"70'1","reported_seq":98,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-09-23T20:53:43.969405+0000","last_change":"2023-09-23T20:53:28.961856+0000","last_active":"2023-09-23T20:53:43.969405+0000","last_peered":"2023-09-23T20:53:43.969405+0000","last_clean":"2023-09-23T20:53:43.969405+0000","last_became_active":"2023-09-23T20:53:28.946066+0000","last_became_peered":"2023-09-23T20:53:28.946066+0000","last_unstale":"2023-09-23T20:53:43.969405+0000","last_undegraded":"2023-09-23T20:53:43.969405+0000","last_fullsized":"2023-09-23T20:53:43.969405+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:49:30.379802+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,7],"acting":[0,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":61,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-09-23T20:53:43.971482+0000","last_change":"2023-09-23T20:53:28.949603+0000","last_active":"2023-09-23T20:53:43.971482+0000","last_peered":"2023-09-23T20:53:43.971482+0000","last_clean":"2023-09-23T20:53:43.971482+0000","last_became_active":"2023-09-23T20:53:28.949363+0000","last_became_peered":"2023-09-23T20:53:28.949363+0000","last_unstale":"2023-09-23T20:53:43.971482+0000","last_undegraded":"2023-09-23T20:53:43.971482+0000","last_fullsized":"2023-09-23T20:53:43.971482+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:10:37.833801+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,7],"acting":[6,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.3","version":"126'6362","reported_seq":6394,"reported_epoch":126,"state":"active+clean","last_fresh":"2023-09-23T20:53:52.045494+0000","last_change":"2023-09-23T20:52:14.865640+0000","last_active":"2023-09-23T20:53:52.045494+0000","last_peered":"2023-09-23T20:53:52.045494+0000","last_clean":"2023-09-23T20:53:52.045494+0000","last_became_active":"2023-09-23T20:52:14.865328+0000","last_became_peered":"2023-09-23T20:52:14.865328+0000","last_unstale":"2023-09-23T20:53:52.045494+0000","last_undegraded":"2023-09-23T20:53:52.045494+0000","last_fullsized":"2023-09-23T20:53:52.045494+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":102,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:52:13.843048+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:52:13.843048+0000","last_clean_scrub_stamp":"2023-09-23T20:52:13.843048+0000","objects_scrubbed":0,"log_size":6362,"log_dups_size":0,"ondisk_log_size":6362,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:04:22.193022+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":410386432,"num_objects":6262,"num_object_clones":0,"num_object_copies":18786,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6262,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6362,"num_write_kb":403968,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,2],"acting":[7,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":76,"reported_epoch":124,"state":"active+clean","last_fresh":"2023-09-23T20:53:36.921859+0000","last_change":"2023-09-23T20:53:28.950434+0000","last_active":"2023-09-23T20:53:36.921859+0000","last_peered":"2023-09-23T20:53:36.921859+0000","last_clean":"2023-09-23T20:53:36.921859+0000","last_became_active":"2023-09-23T20:53:28.949881+0000","last_became_peered":"2023-09-23T20:53:28.949881+0000","last_unstale":"2023-09-23T20:53:36.921859+0000","last_undegraded":"2023-09-23T20:53:36.921859+0000","last_fullsized":"2023-09-23T20:53:36.921859+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:16:12.454130+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":91,"reported_epoch":124,"state":"active+clean","last_fresh":"2023-09-23T20:53:36.895981+0000","last_change":"2023-09-23T20:52:12.855430+0000","last_active":"2023-09-23T20:53:36.895981+0000","last_peered":"2023-09-23T20:53:36.895981+0000","last_clean":"2023-09-23T20:53:36.895981+0000","last_became_active":"2023-09-23T20:52:12.855115+0000","last_became_peered":"2023-09-23T20:52:12.855115+0000","last_unstale":"2023-09-23T20:53:36.895981+0000","last_undegraded":"2023-09-23T20:53:36.895981+0000","last_fullsized":"2023-09-23T20:53:36.895981+0000","mapping_epoch":100,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":101,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:33:23.802214+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":30,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-09-23T20:53:43.955587+0000","last_change":"2023-09-23T20:53:28.960050+0000","last_active":"2023-09-23T20:53:43.955587+0000","last_peered":"2023-09-23T20:53:43.955587+0000","last_clean":"2023-09-23T20:53:43.955587+0000","last_became_active":"2023-09-23T20:53:28.946852+0000","last_became_peered":"2023-09-23T20:53:28.946852+0000","last_unstale":"2023-09-23T20:53:43.955587+0000","last_undegraded":"2023-09-23T20:53:43.955587+0000","last_fullsized":"2023-09-23T20:53:43.955587+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:12:07.341775+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,7],"acting":[1,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":82,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-09-23T20:53:43.959871+0000","last_change":"2023-09-23T20:53:28.953812+0000","last_active":"2023-09-23T20:53:43.959871+0000","last_peered":"2023-09-23T20:53:43.959871+0000","last_clean":"2023-09-23T20:53:43.959871+0000","last_became_active":"2023-09-23T20:53:28.953358+0000","last_became_peered":"2023-09-23T20:53:28.953358+0000","last_unstale":"2023-09-23T20:53:43.959871+0000","last_undegraded":"2023-09-23T20:53:43.959871+0000","last_fullsized":"2023-09-23T20:53:43.959871+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:38:25.570753+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":31,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-09-23T20:53:43.972261+0000","last_change":"2023-09-23T20:53:28.963615+0000","last_active":"2023-09-23T20:53:43.972261+0000","last_peered":"2023-09-23T20:53:43.972261+0000","last_clean":"2023-09-23T20:53:43.972261+0000","last_became_active":"2023-09-23T20:53:28.954909+0000","last_became_peered":"2023-09-23T20:53:28.954909+0000","last_unstale":"2023-09-23T20:53:43.972261+0000","last_undegraded":"2023-09-23T20:53:43.972261+0000","last_fullsized":"2023-09-23T20:53:43.972261+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:17:06.802938+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":82,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-09-23T20:53:43.972009+0000","last_change":"2023-09-23T20:53:28.953991+0000","last_active":"2023-09-23T20:53:43.972009+0000","last_peered":"2023-09-23T20:53:43.972009+0000","last_clean":"2023-09-23T20:53:43.972009+0000","last_became_active":"2023-09-23T20:53:28.953659+0000","last_became_peered":"2023-09-23T20:53:28.953659+0000","last_unstale":"2023-09-23T20:53:43.972009+0000","last_undegraded":"2023-09-23T20:53:43.972009+0000","last_fullsized":"2023-09-23T20:53:43.972009+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:38:42.937247+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,2],"acting":[3,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":95,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-09-23T20:53:43.961296+0000","last_change":"2023-09-23T20:51:10.833849+0000","last_active":"2023-09-23T20:53:43.961296+0000","last_peered":"2023-09-23T20:53:43.961296+0000","last_clean":"2023-09-23T20:53:43.961296+0000","last_became_active":"2023-09-23T20:51:10.833524+0000","last_became_peered":"2023-09-23T20:51:10.833524+0000","last_unstale":"2023-09-23T20:53:43.961296+0000","last_undegraded":"2023-09-23T20:53:43.961296+0000","last_fullsized":"2023-09-23T20:53:43.961296+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:23:12.501924+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":110,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-09-23T20:53:43.959701+0000","last_change":"2023-09-23T20:51:10.838334+0000","last_active":"2023-09-23T20:53:43.959701+0000","last_peered":"2023-09-23T20:53:43.959701+0000","last_clean":"2023-09-23T20:53:43.959701+0000","last_became_active":"2023-09-23T20:51:10.838026+0000","last_became_peered":"2023-09-23T20:51:10.838026+0000","last_unstale":"2023-09-23T20:53:43.959701+0000","last_undegraded":"2023-09-23T20:53:43.959701+0000","last_fullsized":"2023-09-23T20:53:43.959701+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:04:37.009457+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":98,"reported_epoch":124,"state":"active+clean","last_fresh":"2023-09-23T20:53:40.856941+0000","last_change":"2023-09-23T20:52:12.868379+0000","last_active":"2023-09-23T20:53:40.856941+0000","last_peered":"2023-09-23T20:53:40.856941+0000","last_clean":"2023-09-23T20:53:40.856941+0000","last_became_active":"2023-09-23T20:52:12.861899+0000","last_became_peered":"2023-09-23T20:52:12.861899+0000","last_unstale":"2023-09-23T20:53:40.856941+0000","last_undegraded":"2023-09-23T20:53:40.856941+0000","last_fullsized":"2023-09-23T20:53:40.856941+0000","mapping_epoch":100,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":101,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:17:39.308666+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":554,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,6],"acting":[0,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":71,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-09-23T20:53:43.970967+0000","last_change":"2023-09-23T20:53:28.941786+0000","last_active":"2023-09-23T20:53:43.970967+0000","last_peered":"2023-09-23T20:53:43.970967+0000","last_clean":"2023-09-23T20:53:43.970967+0000","last_became_active":"2023-09-23T20:53:28.941484+0000","last_became_peered":"2023-09-23T20:53:28.941484+0000","last_unstale":"2023-09-23T20:53:43.970967+0000","last_undegraded":"2023-09-23T20:53:43.970967+0000","last_fullsized":"2023-09-23T20:53:43.970967+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:22:05.701786+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,3],"acting":[6,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]}],"pool_stats":[{"poolid":6,"num_pg":16,"stat_sum":{"num_bytes":6525812760,"num_objects":99577,"num_object_clones":0,"num_object_copies":298731,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":16604,"num_objects_misplaced":9585,"num_objects_unfound":0,"num_objects_dirty":99577,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":102671,"num_write_kb":6471873,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1591,"num_bytes_recovered":104267776,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":19742535680,"data_stored":19742523464,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":98471,"ondisk_log_size":98471,"up":48,"acting":43,"num_store_stats":7},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":22,"num_bytes_recovered":5526,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":126976,"data_stored":21994,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":161,"ondisk_log_size":161,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201441,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3007252,"kb_used_data":2935948,"kb_used_omap":0,"kb_used_meta":71296,"kb_avail":90737900,"statfs":{"total":95995035648,"available":92915609600,"internally_reserved":0,"allocated":3006410752,"data_stored":3005910275,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":73007104},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:53:38 2023","interfaces":[{"interface":"back","average":{"1min":0.82099999999999995,"5min":0.66900000000000004,"15min":0.55000000000000004},"min":{"1min":0.223,"5min":0.223,"15min":0.223},"max":{"1min":5.8760000000000003,"5min":5.8760000000000003,"15min":5.8760000000000003},"last":0.311},{"interface":"front","average":{"1min":0.90400000000000003,"5min":0.65800000000000003,"15min":0.59399999999999997},"min":{"1min":0.218,"5min":0.218,"15min":0.218},"max":{"1min":5.8700000000000001,"5min":5.8700000000000001,"15min":5.8700000000000001},"last":0.377}]},{"osd":1,"last update":"Sat Sep 23 20:53:07 2023","interfaces":[{"interface":"back","average":{"1min":2.5640000000000001,"5min":0.94999999999999996,"15min":0.68200000000000005},"min":{"1min":0.26100000000000001,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":22.591999999999999,"5min":22.591999999999999,"15min":22.591999999999999},"last":0.29599999999999999},{"interface":"front","average":{"1min":1.992,"5min":0.83999999999999997,"15min":0.64200000000000002},"min":{"1min":0.25600000000000001,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":21.437000000000001,"5min":21.437000000000001,"15min":21.437000000000001},"last":2.4329999999999998}]},{"osd":2,"last update":"Sat Sep 23 20:53:45 2023","interfaces":[{"interface":"back","average":{"1min":1.8700000000000001,"5min":0.88300000000000001,"15min":0.66100000000000003},"min":{"1min":0.26200000000000001,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":5.7249999999999996,"5min":5.7249999999999996,"15min":5.7249999999999996},"last":1.579},{"interface":"front","average":{"1min":1.526,"5min":0.84199999999999997,"15min":0.67000000000000004},"min":{"1min":0.30499999999999999,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":3.7909999999999999,"5min":3.7909999999999999,"15min":3.7909999999999999},"last":0.495}]},{"osd":3,"last update":"Sat Sep 23 20:53:14 2023","interfaces":[{"interface":"back","average":{"1min":2.6459999999999999,"5min":0.97699999999999998,"15min":0.69899999999999995},"min":{"1min":0.39900000000000002,"5min":0.38700000000000001,"15min":0.38700000000000001},"max":{"1min":21.504999999999999,"5min":21.504999999999999,"15min":21.504999999999999},"last":20.138000000000002},{"interface":"front","average":{"1min":2.3319999999999999,"5min":0.92500000000000004,"15min":0.69099999999999995},"min":{"1min":0.36099999999999999,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":21.452000000000002,"5min":21.452000000000002,"15min":21.452000000000002},"last":1.3109999999999999}]},{"osd":4,"last update":"Sat Sep 23 20:53:04 2023","interfaces":[{"interface":"back","average":{"1min":0.50800000000000001,"5min":0.52900000000000003,"15min":0.49099999999999999},"min":{"1min":0.23699999999999999,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":1.3879999999999999,"5min":1.3879999999999999,"15min":1.3879999999999999},"last":0.34899999999999998},{"interface":"front","average":{"1min":0.69199999999999995,"5min":0.57699999999999996,"15min":0.53600000000000003},"min":{"1min":0.32100000000000001,"5min":0.32100000000000001,"15min":0.29599999999999999},"max":{"1min":3.2349999999999999,"5min":3.2349999999999999,"15min":3.2349999999999999},"last":0.44}]},{"osd":5,"last update":"Sat Sep 23 20:53:04 2023","interfaces":[{"interface":"back","average":{"1min":0.52700000000000002,"5min":0.56399999999999995,"15min":0.53800000000000003},"min":{"1min":0.29999999999999999,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":1.0620000000000001,"5min":1.0620000000000001,"15min":1.4610000000000001},"last":0.40600000000000003},{"interface":"front","average":{"1min":0.59099999999999997,"5min":0.60399999999999998,"15min":0.56100000000000005},"min":{"1min":0.29199999999999998,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":1.319,"5min":1.319,"15min":1.319},"last":0.45000000000000001}]},{"osd":6,"last update":"Sat Sep 23 20:53:04 2023","interfaces":[{"interface":"back","average":{"1min":0.58799999999999997,"5min":0.62,"15min":0.57899999999999996},"min":{"1min":0.27400000000000002,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":1.363,"5min":1.363,"15min":1.363},"last":0.39600000000000002},{"interface":"front","average":{"1min":0.61299999999999999,"5min":0.61599999999999999,"15min":0.59199999999999997},"min":{"1min":0.376,"5min":0.36299999999999999,"15min":0.316},"max":{"1min":1.4419999999999999,"5min":1.4419999999999999,"15min":1.4419999999999999},"last":0.41999999999999998}]}]},{"osd":6,"up_from":47,"seq":201863463078,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2669440,"kb_used_data":2599544,"kb_used_omap":0,"kb_used_meta":69888,"kb_avail":91075712,"statfs":{"total":95995035648,"available":93261529088,"internally_reserved":0,"allocated":2661933056,"data_stored":2661444993,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":71565312},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:53:39 2023","interfaces":[{"interface":"back","average":{"1min":1.7989999999999999,"5min":0.80800000000000005,"15min":0.65500000000000003},"min":{"1min":0.245,"5min":0.245,"15min":0.245},"max":{"1min":7.2060000000000004,"5min":7.2060000000000004,"15min":7.2060000000000004},"last":4.3979999999999997},{"interface":"front","average":{"1min":2.3540000000000001,"5min":0.96999999999999997,"15min":0.71599999999999997},"min":{"1min":0.27100000000000002,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":10.699999999999999,"5min":10.699999999999999,"15min":10.699999999999999},"last":1.5649999999999999}]},{"osd":1,"last update":"Sat Sep 23 20:53:13 2023","interfaces":[{"interface":"back","average":{"1min":1.6879999999999999,"5min":0.76800000000000002,"15min":0.61199999999999999},"min":{"1min":0.252,"5min":0.252,"15min":0.252},"max":{"1min":7.194,"5min":7.194,"15min":7.194},"last":0.78100000000000003},{"interface":"front","average":{"1min":1.3620000000000001,"5min":0.72599999999999998,"15min":0.622},"min":{"1min":0.27800000000000002,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":3.8260000000000001,"5min":3.8260000000000001,"15min":3.8260000000000001},"last":0.51300000000000001}]},{"osd":2,"last update":"Sat Sep 23 20:53:47 2023","interfaces":[{"interface":"back","average":{"1min":2.8199999999999998,"5min":1.0820000000000001,"15min":0.72599999999999998},"min":{"1min":0.29999999999999999,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":11.539999999999999,"5min":11.539999999999999,"15min":11.539999999999999},"last":3.7480000000000002},{"interface":"front","average":{"1min":2.6640000000000001,"5min":1.089,"15min":0.753},"min":{"1min":0.40500000000000003,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":11.583,"5min":11.583,"15min":11.583},"last":3.7370000000000001}]},{"osd":3,"last update":"Sat Sep 23 20:53:13 2023","interfaces":[{"interface":"back","average":{"1min":1.0509999999999999,"5min":0.68300000000000005,"15min":0.622},"min":{"1min":0.307,"5min":0.307,"15min":0.307},"max":{"1min":3.7200000000000002,"5min":3.7200000000000002,"15min":3.7200000000000002},"last":3.1779999999999999},{"interface":"front","average":{"1min":1.7709999999999999,"5min":0.81899999999999995,"15min":0.66000000000000003},"min":{"1min":0.315,"5min":0.315,"15min":0.315},"max":{"1min":7.4569999999999999,"5min":7.4569999999999999,"15min":7.4569999999999999},"last":4.6669999999999998}]},{"osd":4,"last update":"Sat Sep 23 20:53:47 2023","interfaces":[{"interface":"back","average":{"1min":0.64100000000000001,"5min":0.51100000000000001,"15min":0.47399999999999998},"min":{"1min":0.27600000000000002,"5min":0.248,"15min":0.248},"max":{"1min":4.952,"5min":4.952,"15min":4.952},"last":0.30299999999999999},{"interface":"front","average":{"1min":0.61799999999999999,"5min":0.54100000000000004,"15min":0.502},"min":{"1min":0.29899999999999999,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":2.8300000000000001,"5min":2.8300000000000001,"15min":2.8300000000000001},"last":0.495}]},{"osd":5,"last update":"Sat Sep 23 20:53:47 2023","interfaces":[{"interface":"back","average":{"1min":0.86199999999999999,"5min":0.59499999999999997,"15min":0.52500000000000002},"min":{"1min":0.30399999999999999,"5min":0.30399999999999999,"15min":0.28399999999999997},"max":{"1min":8.423,"5min":8.423,"15min":8.423},"last":0.34200000000000003},{"interface":"front","average":{"1min":0.56299999999999994,"5min":0.56299999999999994,"15min":0.53000000000000003},"min":{"1min":0.371,"5min":0.33100000000000002,"15min":0.28499999999999998},"max":{"1min":2.8620000000000001,"5min":2.8620000000000001,"15min":2.8620000000000001},"last":0.53000000000000003}]},{"osd":7,"last update":"Sat Sep 23 20:53:04 2023","interfaces":[{"interface":"back","average":{"1min":0.48999999999999999,"5min":0.55600000000000005,"15min":0.54000000000000004},"min":{"1min":0.34799999999999998,"5min":0.33000000000000002,"15min":0.32500000000000001},"max":{"1min":0.94599999999999995,"5min":0.94599999999999995,"15min":1.119},"last":0.379},{"interface":"front","average":{"1min":0.497,"5min":0.60099999999999998,"15min":0.58199999999999996},"min":{"1min":0.39000000000000001,"5min":0.38900000000000001,"15min":0.36099999999999999},"max":{"1min":0.84599999999999997,"5min":0.86899999999999999,"15min":0.94099999999999995},"last":0.45400000000000001}]}]},{"osd":1,"up_from":88,"seq":377957122096,"num_pgs":49,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2464880,"kb_used_data":2397032,"kb_used_omap":9,"kb_used_meta":67638,"kb_avail":91280272,"statfs":{"total":95995035648,"available":93470998528,"internally_reserved":0,"allocated":2454560768,"data_stored":2453977294,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9239,"internal_metadata":69262313},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,3,0,2,2,1],"upper_bound":128},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:53:07 2023","interfaces":[{"interface":"back","average":{"1min":0.98299999999999998,"5min":0.50900000000000001,"15min":0.442},"min":{"1min":0.216,"5min":0.216,"15min":0.216},"max":{"1min":5.0860000000000003,"5min":5.0860000000000003,"15min":5.0860000000000003},"last":0.65500000000000003},{"interface":"front","average":{"1min":1.071,"5min":0.54900000000000004,"15min":0.47299999999999998},"min":{"1min":0.31900000000000001,"5min":0.31,"15min":0.31},"max":{"1min":4.9059999999999997,"5min":4.9059999999999997,"15min":4.9059999999999997},"last":0.53300000000000003}]},{"osd":2,"last update":"Sat Sep 23 20:53:42 2023","interfaces":[{"interface":"back","average":{"1min":1.3,"5min":0.73899999999999999,"15min":0.504},"min":{"1min":0.20300000000000001,"5min":0.20300000000000001,"15min":0.20300000000000001},"max":{"1min":4.1050000000000004,"5min":6.6379999999999999,"15min":6.6379999999999999},"last":0.68799999999999994},{"interface":"front","average":{"1min":4.556,"5min":1.419,"15min":0.86299999999999999},"min":{"1min":0.38900000000000001,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":38.951000000000001,"5min":38.951000000000001,"15min":38.951000000000001},"last":0.69799999999999995}]},{"osd":3,"last update":"Sat Sep 23 20:53:13 2023","interfaces":[{"interface":"back","average":{"1min":2.1179999999999999,"5min":0.70999999999999996,"15min":0.47599999999999998},"min":{"1min":0.27700000000000002,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":5.6710000000000003,"5min":5.6710000000000003,"15min":5.6710000000000003},"last":2.5150000000000001},{"interface":"front","average":{"1min":1.7210000000000001,"5min":0.67200000000000004,"15min":0.497},"min":{"1min":0.32500000000000001,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":4.3780000000000001,"5min":4.3780000000000001,"15min":4.3780000000000001},"last":1.5669999999999999}]},{"osd":4,"last update":"Sat Sep 23 20:53:07 2023","interfaces":[{"interface":"back","average":{"1min":1.448,"5min":0.79300000000000004,"15min":0.69499999999999995},"min":{"1min":0.35199999999999998,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":5.7279999999999998,"5min":5.7279999999999998,"15min":5.7279999999999998},"last":2.7120000000000002},{"interface":"front","average":{"1min":0.86799999999999999,"5min":0.68999999999999995,"15min":0.67800000000000005},"min":{"1min":0.30099999999999999,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":4.4290000000000003,"5min":4.4290000000000003,"15min":4.4290000000000003},"last":0.90700000000000003}]},{"osd":5,"last update":"Sat Sep 23 20:53:07 2023","interfaces":[{"interface":"back","average":{"1min":0.91500000000000004,"5min":0.71099999999999997,"15min":0.70199999999999996},"min":{"1min":0.371,"5min":0.371,"15min":0.371},"max":{"1min":3.234,"5min":3.234,"15min":3.234},"last":0.77800000000000002},{"interface":"front","average":{"1min":1.4670000000000001,"5min":0.79600000000000004,"15min":0.69599999999999995},"min":{"1min":0.377,"5min":0.377,"15min":0.377},"max":{"1min":5.766,"5min":5.766,"15min":5.766},"last":2.6459999999999999}]},{"osd":6,"last update":"Sat Sep 23 20:53:07 2023","interfaces":[{"interface":"back","average":{"1min":0.91200000000000003,"5min":0.71399999999999997,"15min":0.69599999999999995},"min":{"1min":0.36899999999999999,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":3.278,"5min":3.278,"15min":3.278},"last":0.753},{"interface":"front","average":{"1min":1.476,"5min":0.80600000000000005,"15min":0.70699999999999996},"min":{"1min":0.41199999999999998,"5min":0.41199999999999998,"15min":0.41199999999999998},"max":{"1min":5.7489999999999997,"5min":5.7489999999999997,"15min":5.7489999999999997},"last":2.6800000000000002}]},{"osd":7,"last update":"Sat Sep 23 20:53:07 2023","interfaces":[{"interface":"back","average":{"1min":1.1140000000000001,"5min":0.74199999999999999,"15min":0.69599999999999995},"min":{"1min":0.36599999999999999,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":4.46,"5min":4.46,"15min":4.46},"last":0.874},{"interface":"front","average":{"1min":1.0189999999999999,"5min":0.73499999999999999,"15min":0.70199999999999996},"min":{"1min":0.38300000000000001,"5min":0.38300000000000001,"15min":0.38300000000000001},"max":{"1min":4.8700000000000001,"5min":4.8700000000000001,"15min":4.8700000000000001},"last":0.73099999999999998}]}]},{"osd":0,"up_from":83,"seq":356482285622,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1680308,"kb_used_data":1630636,"kb_used_omap":28,"kb_used_meta":49635,"kb_avail":92064844,"statfs":{"total":95995035648,"available":94274400256,"internally_reserved":0,"allocated":1669771264,"data_stored":1669177589,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":28848,"internal_metadata":50827088},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 20:53:10 2023","interfaces":[{"interface":"back","average":{"1min":1.286,"5min":0.52900000000000003,"15min":0.39800000000000002},"min":{"1min":0.249,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":6.3710000000000004,"5min":6.3710000000000004,"15min":6.3710000000000004},"last":0.65500000000000003},{"interface":"front","average":{"1min":1.2709999999999999,"5min":0.56200000000000006,"15min":0.44500000000000001},"min":{"1min":0.255,"5min":0.255,"15min":0.255},"max":{"1min":8.4969999999999999,"5min":8.4969999999999999,"15min":8.4969999999999999},"last":1.637}]},{"osd":2,"last update":"Sat Sep 23 20:53:42 2023","interfaces":[{"interface":"back","average":{"1min":3.7330000000000001,"5min":1.238,"15min":0.68899999999999995},"min":{"1min":0.23799999999999999,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":17.753,"5min":17.753,"15min":17.753},"last":1.7070000000000001},{"interface":"front","average":{"1min":2.7850000000000001,"5min":0.97099999999999997,"15min":0.58299999999999996},"min":{"1min":0.21099999999999999,"5min":0.21099999999999999,"15min":0.21099999999999999},"max":{"1min":11.314,"5min":11.314,"15min":11.314},"last":1.6930000000000001}]},{"osd":3,"last update":"Sat Sep 23 20:53:11 2023","interfaces":[{"interface":"back","average":{"1min":1.405,"5min":0.60099999999999998,"15min":0.46700000000000003},"min":{"1min":0.33100000000000002,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":6.0730000000000004,"5min":6.0730000000000004,"15min":6.0730000000000004},"last":0.38400000000000001},{"interface":"front","average":{"1min":1.7250000000000001,"5min":0.72699999999999998,"15min":0.56100000000000005},"min":{"1min":0.23300000000000001,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":8.5329999999999995,"5min":8.5329999999999995,"15min":8.5329999999999995},"last":6.4029999999999996}]},{"osd":4,"last update":"Sat Sep 23 20:53:42 2023","interfaces":[{"interface":"back","average":{"1min":1.407,"5min":0.84399999999999997,"15min":0.71299999999999997},"min":{"1min":0.311,"5min":0.311,"15min":0.311},"max":{"1min":4.6189999999999998,"5min":4.6189999999999998,"15min":4.6189999999999998},"last":1.7689999999999999},{"interface":"front","average":{"1min":0.97499999999999998,"5min":0.754,"15min":0.68999999999999995},"min":{"1min":0.28100000000000003,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":3.1230000000000002,"5min":4.5890000000000004,"15min":4.5890000000000004},"last":0.436}]},{"osd":5,"last update":"Sat Sep 23 20:53:42 2023","interfaces":[{"interface":"back","average":{"1min":2.2999999999999998,"5min":1.077,"15min":0.79100000000000004},"min":{"1min":0.51100000000000001,"5min":0.41499999999999998,"15min":0.41499999999999998},"max":{"1min":7.1589999999999998,"5min":8.4559999999999995,"15min":8.4559999999999995},"last":1.7809999999999999},{"interface":"front","average":{"1min":1.55,"5min":0.88700000000000001,"15min":0.76700000000000002},"min":{"1min":0.35899999999999999,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":4.6859999999999999,"5min":4.6859999999999999,"15min":4.6859999999999999},"last":1.7370000000000001}]},{"osd":6,"last update":"Sat Sep 23 20:53:42 2023","interfaces":[{"interface":"back","average":{"1min":1.4710000000000001,"5min":0.876,"15min":0.752},"min":{"1min":0.33500000000000002,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":4.6580000000000004,"5min":4.6580000000000004,"15min":4.6580000000000004},"last":1.7989999999999999},{"interface":"front","average":{"1min":1.0249999999999999,"5min":0.79500000000000004,"15min":0.71899999999999997},"min":{"1min":0.38800000000000001,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":3.1829999999999998,"5min":4.6200000000000001,"15min":4.6200000000000001},"last":0.41399999999999998}]},{"osd":7,"last update":"Sat Sep 23 20:53:42 2023","interfaces":[{"interface":"back","average":{"1min":2.2080000000000002,"5min":1.087,"15min":0.82899999999999996},"min":{"1min":0.45000000000000001,"5min":0.42799999999999999,"15min":0.42799999999999999},"max":{"1min":7.1079999999999997,"5min":8.3979999999999997,"15min":8.3979999999999997},"last":1.75},{"interface":"front","average":{"1min":1.03,"5min":0.78100000000000003,"15min":0.70899999999999996},"min":{"1min":0.42899999999999999,"5min":0.41899999999999998,"15min":0.41899999999999998},"max":{"1min":3.157,"5min":4.5599999999999996,"15min":4.5599999999999996},"last":0.45600000000000002}]}]},{"osd":2,"up_from":93,"seq":399431958569,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":4154272,"kb_used_data":4046232,"kb_used_omap":81,"kb_used_meta":107886,"kb_avail":89590880,"statfs":{"total":95995035648,"available":91741061120,"internally_reserved":0,"allocated":4143341568,"data_stored":4142767856,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":83750,"internal_metadata":110475482},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,2,2,1,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":4,"apply_latency_ms":4,"commit_latency_ns":4000000,"apply_latency_ns":4000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:53:39 2023","interfaces":[{"interface":"back","average":{"1min":2.4660000000000002,"5min":0.81299999999999994,"15min":0.50900000000000001},"min":{"1min":0.28799999999999998,"5min":0.20499999999999999,"15min":0.20499999999999999},"max":{"1min":6.3470000000000004,"5min":6.3470000000000004,"15min":6.3470000000000004},"last":1.861},{"interface":"front","average":{"1min":2.0369999999999999,"5min":0.75700000000000001,"15min":0.52300000000000002},"min":{"1min":0.222,"5min":0.222,"15min":0.222},"max":{"1min":6.3520000000000003,"5min":6.3520000000000003,"15min":6.3520000000000003},"last":6.923}]},{"osd":1,"last update":"Sat Sep 23 20:53:39 2023","interfaces":[{"interface":"back","average":{"1min":4.6200000000000001,"5min":1.3049999999999999,"15min":0.71299999999999997},"min":{"1min":0.32100000000000001,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":16.081,"5min":16.081,"15min":16.081},"last":1.8979999999999999},{"interface":"front","average":{"1min":4.2809999999999997,"5min":1.2490000000000001,"15min":0.69699999999999995},"min":{"1min":0.27400000000000002,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":16.041,"5min":16.041,"15min":16.041},"last":8.9550000000000001}]},{"osd":3,"last update":"Sat Sep 23 20:53:13 2023","interfaces":[{"interface":"back","average":{"1min":2.7370000000000001,"5min":0.878,"15min":0.56799999999999995},"min":{"1min":0.26800000000000002,"5min":0.255,"15min":0.255},"max":{"1min":11.423999999999999,"5min":11.423999999999999,"15min":11.423999999999999},"last":6.9470000000000001},{"interface":"front","average":{"1min":2.8969999999999998,"5min":0.88300000000000001,"15min":0.54700000000000004},"min":{"1min":0.35499999999999998,"5min":0.221,"15min":0.221},"max":{"1min":11.510999999999999,"5min":11.510999999999999,"15min":11.510999999999999},"last":11.013999999999999}]},{"osd":4,"last update":"Sat Sep 23 20:53:39 2023","interfaces":[{"interface":"back","average":{"1min":1.554,"5min":0.82199999999999995,"15min":0.68600000000000005},"min":{"1min":0.26400000000000001,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":6.2919999999999998,"5min":6.2919999999999998,"15min":6.2919999999999998},"last":6.8760000000000003},{"interface":"front","average":{"1min":1.4550000000000001,"5min":0.85599999999999998,"15min":0.73199999999999998},"min":{"1min":0.34100000000000003,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":8.4450000000000003,"5min":8.4450000000000003,"15min":8.4450000000000003},"last":0.60799999999999998}]},{"osd":5,"last update":"Sat Sep 23 20:53:39 2023","interfaces":[{"interface":"back","average":{"1min":1.5920000000000001,"5min":0.85199999999999998,"15min":0.71799999999999997},"min":{"1min":0.29399999999999998,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":6.3239999999999998,"5min":6.3239999999999998,"15min":6.3239999999999998},"last":6.9000000000000004},{"interface":"front","average":{"1min":1.8540000000000001,"5min":0.92100000000000004,"15min":0.73099999999999998},"min":{"1min":0.35099999999999998,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":8.4939999999999998,"5min":8.4939999999999998,"15min":8.4939999999999998},"last":0.56599999999999995}]},{"osd":6,"last update":"Sat Sep 23 20:53:39 2023","interfaces":[{"interface":"back","average":{"1min":1.847,"5min":0.95799999999999996,"15min":0.79000000000000004},"min":{"1min":0.34300000000000003,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":8.5340000000000007,"5min":8.5340000000000007,"15min":8.5340000000000007},"last":0.63900000000000001},{"interface":"front","average":{"1min":1.9219999999999999,"5min":0.95099999999999996,"15min":0.77400000000000002},"min":{"1min":0.40100000000000002,"5min":0.40100000000000002,"15min":0.40100000000000002},"max":{"1min":6.431,"5min":6.431,"15min":6.431},"last":1.758}]},{"osd":7,"last update":"Sat Sep 23 20:53:39 2023","interfaces":[{"interface":"back","average":{"1min":1.9350000000000001,"5min":0.91700000000000004,"15min":0.72899999999999998},"min":{"1min":0.434,"5min":0.42299999999999999,"15min":0.42299999999999999},"max":{"1min":6.4080000000000004,"5min":6.4080000000000004,"15min":6.4080000000000004},"last":1.8340000000000001},{"interface":"front","average":{"1min":1.8300000000000001,"5min":0.91300000000000003,"15min":0.745},"min":{"1min":0.34499999999999997,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":6.3780000000000001,"5min":6.3780000000000001,"15min":6.3780000000000001},"last":6.835}]}]},{"osd":3,"up_from":98,"seq":420906795043,"num_pgs":52,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":4055632,"kb_used_data":3951880,"kb_used_omap":7,"kb_used_meta":103736,"kb_avail":89689520,"statfs":{"total":95995035648,"available":91842068480,"internally_reserved":0,"allocated":4046725120,"data_stored":4046168564,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7576,"internal_metadata":106226280},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,1,0,0,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:53:15 2023","interfaces":[{"interface":"back","average":{"1min":3.903,"5min":1.077,"15min":0.60599999999999998},"min":{"1min":0.249,"5min":0.249,"15min":0.249},"max":{"1min":12.249000000000001,"5min":12.249000000000001,"15min":12.249000000000001},"last":0.249},{"interface":"front","average":{"1min":2.3660000000000001,"5min":0.75900000000000001,"15min":0.49099999999999999},"min":{"1min":0.21299999999999999,"5min":0.21299999999999999,"15min":0.21299999999999999},"max":{"1min":9.3170000000000002,"5min":9.3170000000000002,"15min":9.3170000000000002},"last":0.22700000000000001}]},{"osd":1,"last update":"Sat Sep 23 20:53:15 2023","interfaces":[{"interface":"back","average":{"1min":2.5369999999999999,"5min":0.80100000000000005,"15min":0.51100000000000001},"min":{"1min":0.26700000000000002,"5min":0.253,"15min":0.253},"max":{"1min":11.680999999999999,"5min":11.680999999999999,"15min":11.680999999999999},"last":0.46800000000000003},{"interface":"front","average":{"1min":2.9580000000000002,"5min":0.90200000000000002,"15min":0.56000000000000005},"min":{"1min":0.28299999999999997,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":9.5329999999999995,"5min":9.5329999999999995,"15min":9.5329999999999995},"last":2.4780000000000002}]},{"osd":2,"last update":"Sat Sep 23 20:53:15 2023","interfaces":[{"interface":"back","average":{"1min":2.5840000000000001,"5min":0.85599999999999998,"15min":0.56799999999999995},"min":{"1min":0.32800000000000001,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":9.4100000000000001,"5min":9.4100000000000001,"15min":9.4100000000000001},"last":0.628},{"interface":"front","average":{"1min":3.0339999999999998,"5min":0.95799999999999996,"15min":0.61199999999999999},"min":{"1min":0.34999999999999998,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":9.4920000000000009,"5min":9.4920000000000009,"15min":9.4920000000000009},"last":0.50700000000000001}]},{"osd":4,"last update":"Sat Sep 23 20:53:15 2023","interfaces":[{"interface":"back","average":{"1min":2.3149999999999999,"5min":0.96799999999999997,"15min":0.74399999999999999},"min":{"1min":0.40899999999999997,"5min":0.40899999999999997,"15min":0.40899999999999997},"max":{"1min":9.5449999999999999,"5min":9.5449999999999999,"15min":9.5449999999999999},"last":0.55400000000000005},{"interface":"front","average":{"1min":2.617,"5min":1.0249999999999999,"15min":0.75900000000000001},"min":{"1min":0.32700000000000001,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":9.7349999999999994,"5min":9.7349999999999994,"15min":9.7349999999999994},"last":1.198}]},{"osd":5,"last update":"Sat Sep 23 20:53:15 2023","interfaces":[{"interface":"back","average":{"1min":1.875,"5min":0.89100000000000001,"15min":0.72699999999999998},"min":{"1min":0.49299999999999999,"5min":0.42599999999999999,"15min":0.42599999999999999},"max":{"1min":9.4440000000000008,"5min":9.4440000000000008,"15min":9.4440000000000008},"last":1.2130000000000001},{"interface":"front","average":{"1min":2.1160000000000001,"5min":0.93200000000000005,"15min":0.73499999999999999},"min":{"1min":0.441,"5min":0.441,"15min":0.441},"max":{"1min":8.9730000000000008,"5min":8.9730000000000008,"15min":8.9730000000000008},"last":0.439}]},{"osd":6,"last update":"Sat Sep 23 20:53:15 2023","interfaces":[{"interface":"back","average":{"1min":2.1200000000000001,"5min":0.97599999999999998,"15min":0.78600000000000003},"min":{"1min":0.38100000000000001,"5min":0.38100000000000001,"15min":0.38100000000000001},"max":{"1min":8.9410000000000007,"5min":8.9410000000000007,"15min":8.9410000000000007},"last":0.56899999999999995},{"interface":"front","average":{"1min":2.3010000000000002,"5min":1.0349999999999999,"15min":0.82399999999999995},"min":{"1min":0.40699999999999997,"5min":0.40699999999999997,"15min":0.40699999999999997},"max":{"1min":9.4809999999999999,"5min":9.4809999999999999,"15min":9.4809999999999999},"last":0.59099999999999997}]},{"osd":7,"last update":"Sat Sep 23 20:53:15 2023","interfaces":[{"interface":"back","average":{"1min":1.8759999999999999,"5min":0.93100000000000005,"15min":0.77300000000000002},"min":{"1min":0.46300000000000002,"5min":0.46300000000000002,"15min":0.46300000000000002},"max":{"1min":9.5169999999999995,"5min":9.5169999999999995,"15min":9.5169999999999995},"last":0.51800000000000002},{"interface":"front","average":{"1min":2.1379999999999999,"5min":0.98199999999999998,"15min":0.79000000000000004},"min":{"1min":0.41899999999999998,"5min":0.41899999999999998,"15min":0.41899999999999998},"max":{"1min":8.8949999999999996,"5min":8.8949999999999996,"15min":8.8949999999999996},"last":0.53000000000000003}]}]},{"osd":4,"up_from":0,"seq":133143986350,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":5,"up_from":0,"seq":167503724715,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":666,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5322,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":4858,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5392,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1664028672,"data_stored":1664024600,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":2442072064,"data_stored":2442067992,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":4132507648,"data_stored":4132503576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4028956672,"data_stored":4028956672,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1820459008,"data_stored":1820459008,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":2651193344,"data_stored":2651193344,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":3003318272,"data_stored":3003318272,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T20:53:55.361 INFO:tasks.cephadm.ceph_manager.ceph:increase pool size by 10 2023-09-23T20:53:55.361 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph osd pool set unique_pool_0 pg_num 26 2023-09-23T20:53:55.404 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:53:55.435 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:53:55.474 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:53:55.477 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:53:55.477 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:53:55.578 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:53:55.704 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:53:55 smithi026 ceph-mon[175151]: pgmap v309: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 84 active+clean; 6.1 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 102 B/s rd, 24 MiB/s wr, 535 op/s; 16604/298803 objects degraded (5.557%); 9585/298803 objects misplaced (3.208%); 4.5 MiB/s, 71 objects/s recovering 2023-09-23T20:53:55.704 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:53:55 smithi026 ceph-mon[175151]: from='client.54712 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:53:55.704 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:53:55 smithi026 ceph-mon[175151]: Health check update: Degraded data redundancy: 16604/298803 objects degraded (5.557%), 3 pgs degraded (PG_DEGRADED) 2023-09-23T20:53:55.706 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:53:55 smithi026 ceph-mon[179688]: pgmap v309: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 84 active+clean; 6.1 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 102 B/s rd, 24 MiB/s wr, 535 op/s; 16604/298803 objects degraded (5.557%); 9585/298803 objects misplaced (3.208%); 4.5 MiB/s, 71 objects/s recovering 2023-09-23T20:53:55.707 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:53:55 smithi026 ceph-mon[179688]: from='client.54712 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:53:55.707 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:53:55 smithi026 ceph-mon[179688]: Health check update: Degraded data redundancy: 16604/298803 objects degraded (5.557%), 3 pgs degraded (PG_DEGRADED) 2023-09-23T20:53:55.710 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:53:55.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:53:55 smithi070 ceph-mon[168282]: pgmap v309: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 84 active+clean; 6.1 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 102 B/s rd, 24 MiB/s wr, 535 op/s; 16604/298803 objects degraded (5.557%); 9585/298803 objects misplaced (3.208%); 4.5 MiB/s, 71 objects/s recovering 2023-09-23T20:53:55.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:53:55 smithi070 ceph-mon[168282]: from='client.54712 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:53:55.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:53:55 smithi070 ceph-mon[168282]: Health check update: Degraded data redundancy: 16604/298803 objects degraded (5.557%), 3 pgs degraded (PG_DEGRADED) 2023-09-23T20:53:55.870 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:53:55.882 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:53:55.883 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:53:55.984 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:53:56.015 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:53:56.052 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:53:56.056 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:53:56.056 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:53:56.073 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:53:56.099 INFO:teuthology.orchestra.run.smithi026.stdout:{ 2023-09-23T20:53:56.100 INFO:teuthology.orchestra.run.smithi026.stdout: "ops": [], 2023-09-23T20:53:56.100 INFO:teuthology.orchestra.run.smithi026.stdout: "complaint_time": 30, 2023-09-23T20:53:56.100 INFO:teuthology.orchestra.run.smithi026.stdout: "num_blocked_ops": 0 2023-09-23T20:53:56.100 INFO:teuthology.orchestra.run.smithi026.stdout:} 2023-09-23T20:53:56.157 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:53:56.186 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:53:56.221 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:53:56.226 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:53:56.226 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:53:56.327 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:53:56.358 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:53:56.404 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:53:56.407 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:53:56.407 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:53:56.508 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:53:56.605 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:53:56.835 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:53:56.919 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_historic_ops 2023-09-23T20:53:57.504 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:53:57.516 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:53:57.516 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:53:57.617 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:53:57.733 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:53:57.747 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0: {} 2023-09-23T20:53:57.805 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:53:57 smithi026 ceph-mon[179688]: pgmap v310: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 84 active+clean; 6.0 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 102 B/s rd, 21 MiB/s wr, 536 op/s; 16604/297210 objects degraded (5.587%); 9406/297210 objects misplaced (3.165%); 4.5 MiB/s, 72 objects/s recovering 2023-09-23T20:53:57.806 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:53:57 smithi026 ceph-mon[175151]: pgmap v310: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 84 active+clean; 6.0 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 102 B/s rd, 21 MiB/s wr, 536 op/s; 16604/297210 objects degraded (5.587%); 9406/297210 objects misplaced (3.165%); 4.5 MiB/s, 72 objects/s recovering 2023-09-23T20:53:57.816 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1: {} 2023-09-23T20:53:57.853 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2: {} 2023-09-23T20:53:57.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:53:57 smithi070 ceph-mon[168282]: pgmap v310: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 84 active+clean; 6.0 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 102 B/s rd, 21 MiB/s wr, 536 op/s; 16604/297210 objects degraded (5.587%); 9406/297210 objects misplaced (3.165%); 4.5 MiB/s, 72 objects/s recovering 2023-09-23T20:53:57.902 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3: {} 2023-09-23T20:53:57.953 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4: {} 2023-09-23T20:53:57.991 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T20:53:58.030 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6: {} 2023-09-23T20:53:58.067 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T20:53:58.096 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:53:58.114 INFO:teuthology.orchestra.run.smithi026.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T20:53:58.114 INFO:teuthology.orchestra.run.smithi026.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T20:53:58.114 INFO:teuthology.orchestra.run.smithi026.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T20:53:58.114 INFO:teuthology.orchestra.run.smithi026.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T20:53:58.114 INFO:teuthology.orchestra.run.smithi026.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T20:53:58.114 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T20:53:58.114 INFO:teuthology.orchestra.run.smithi026.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T20:53:58.114 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T20:53:58.120 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:53:58.120 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:53:58.221 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:53:58.317 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:53:58.582 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:53:58.590 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:53:58.590 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:53:58.691 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:53:58.758 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:53:58.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:53:58 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/678337086' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:53:58.885 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:53:58 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/678337086' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:53:58.885 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:53:58 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/678337086' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:53:59.016 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:53:59.022 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:53:59.022 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:53:59.035 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:53:59.123 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:53:59.238 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:53:59.642 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:53:59.659 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:53:59.659 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:53:59.760 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:53:59.790 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:53:59.826 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:53:59.828 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:53:59 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[164029]: ::ffff:172.21.15.70 - - [23/Sep/2023:20:53:59] "GET /metrics HTTP/1.1" 200 36331 "" "Prometheus/2.43.0" 2023-09-23T20:53:59.830 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:53:59.830 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:53:59.831 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:53:59 smithi026 ceph-mon[175151]: pgmap v311: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 84 active+clean; 6.0 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 102 B/s rd, 16 MiB/s wr, 542 op/s; 16604/295065 objects degraded (5.627%); 9406/295065 objects misplaced (3.188%); 4.5 MiB/s, 72 objects/s recovering 2023-09-23T20:53:59.832 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:53:59 smithi026 ceph-mon[179688]: pgmap v311: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 84 active+clean; 6.0 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 102 B/s rd, 16 MiB/s wr, 542 op/s; 16604/295065 objects degraded (5.627%); 9406/295065 objects misplaced (3.188%); 4.5 MiB/s, 72 objects/s recovering 2023-09-23T20:53:59.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:53:59 smithi070 ceph-mon[168282]: pgmap v311: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 84 active+clean; 6.0 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 102 B/s rd, 16 MiB/s wr, 542 op/s; 16604/295065 objects degraded (5.627%); 9406/295065 objects misplaced (3.188%); 4.5 MiB/s, 72 objects/s recovering 2023-09-23T20:53:59.931 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:54:00.050 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:54:00.233 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:54:00.249 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:54:00.250 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:54:00.350 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:54:00.465 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:54:00.522 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:54:00.533 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:54:00.533 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:54:00.634 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:54:00.641 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T20:54:00.658 INFO:teuthology.orchestra.run.smithi026.stderr:nodeep-scrub is set 2023-09-23T20:54:00.661 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:54:00.663 INFO:teuthology.orchestra.run.smithi026.stderr:set pool 6 pg_num to 26 2023-09-23T20:54:00.701 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:54:00 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/2539664248' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:54:00.702 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:54:00 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:54:00.702 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:54:00 smithi026 ceph-mon[179688]: Health check update: Degraded data redundancy: 16604/295065 objects degraded (5.627%), 3 pgs degraded (PG_DEGRADED) 2023-09-23T20:54:00.702 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:54:00 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/3460351127' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pg_num", "val": "26"}]: dispatch 2023-09-23T20:54:00.702 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:54:00 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pg_num", "val": "26"}]: dispatch 2023-09-23T20:54:00.705 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:54:00.708 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:54:00 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/2539664248' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:54:00.708 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:54:00 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:54:00.708 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:54:00 smithi026 ceph-mon[175151]: Health check update: Degraded data redundancy: 16604/295065 objects degraded (5.627%), 3 pgs degraded (PG_DEGRADED) 2023-09-23T20:54:00.708 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:54:00 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/3460351127' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pg_num", "val": "26"}]: dispatch 2023-09-23T20:54:00.708 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:54:00 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pg_num", "val": "26"}]: dispatch 2023-09-23T20:54:00.710 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:54:00.711 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:54:00.811 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:54:00.898 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:54:01.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:54:00 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/2539664248' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:54:01.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:54:00 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:54:01.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:54:00 smithi070 ceph-mon[168282]: Health check update: Degraded data redundancy: 16604/295065 objects degraded (5.627%), 3 pgs degraded (PG_DEGRADED) 2023-09-23T20:54:01.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:54:00 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/3460351127' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pg_num", "val": "26"}]: dispatch 2023-09-23T20:54:01.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:54:00 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pg_num", "val": "26"}]: dispatch 2023-09-23T20:54:01.396 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:54:01.408 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:54:01.408 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:54:01.510 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:54:01.598 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:54:01.677 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:54:01.688 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:54:01.688 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:54:01.789 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:54:01.821 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:54:01.843 INFO:teuthology.orchestra.run.smithi026.stdout:ERROR: (22) Invalid argument 2023-09-23T20:54:01.843 INFO:teuthology.orchestra.run.smithi026.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:54:01.859 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:54:01.862 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:54:01.862 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:54:01.871 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:54:01 smithi026 ceph-mon[175151]: pgmap v312: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 84 active+clean; 5.9 GiB data, 17 GiB used, 520 GiB / 536 GiB avail; 102 B/s rd, 6.1 MiB/s wr, 536 op/s; 16604/290289 objects degraded (5.720%); 9203/290289 objects misplaced (3.170%); 4.7 MiB/s, 74 objects/s recovering 2023-09-23T20:54:01.872 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:54:01 smithi026 ceph-mon[175151]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:54:01.872 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:54:01 smithi026 ceph-mon[175151]: failed to encode map e127 with expected crc 2023-09-23T20:54:01.872 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:54:01 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T20:54:01.872 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:54:01 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pg_num", "val": "26"}]': finished 2023-09-23T20:54:01.872 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:54:01 smithi026 ceph-mon[175151]: osdmap e127: 8 total, 8 up, 6 in 2023-09-23T20:54:01.872 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:54:01 smithi026 ceph-mon[175151]: failed to encode map e127 with expected crc 2023-09-23T20:54:01.872 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:54:01 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:54:01.872 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:54:01 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:54:01.874 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:54:01 smithi026 ceph-mon[179688]: pgmap v312: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 84 active+clean; 5.9 GiB data, 17 GiB used, 520 GiB / 536 GiB avail; 102 B/s rd, 6.1 MiB/s wr, 536 op/s; 16604/290289 objects degraded (5.720%); 9203/290289 objects misplaced (3.170%); 4.7 MiB/s, 74 objects/s recovering 2023-09-23T20:54:01.874 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:54:01 smithi026 ceph-mon[179688]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:54:01.874 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:54:01 smithi026 ceph-mon[179688]: failed to encode map e127 with expected crc 2023-09-23T20:54:01.874 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:54:01 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T20:54:01.875 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:54:01 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pg_num", "val": "26"}]': finished 2023-09-23T20:54:01.875 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:54:01 smithi026 ceph-mon[179688]: osdmap e127: 8 total, 8 up, 6 in 2023-09-23T20:54:01.875 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:54:01 smithi026 ceph-mon[179688]: failed to encode map e127 with expected crc 2023-09-23T20:54:01.875 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:54:01 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:54:01.875 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:54:01 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:54:01.954 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:54:01.963 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:54:02.067 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:54:02.121 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:54:01 smithi070 ceph-mon[168282]: pgmap v312: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 84 active+clean; 5.9 GiB data, 17 GiB used, 520 GiB / 536 GiB avail; 102 B/s rd, 6.1 MiB/s wr, 536 op/s; 16604/290289 objects degraded (5.720%); 9203/290289 objects misplaced (3.170%); 4.7 MiB/s, 74 objects/s recovering 2023-09-23T20:54:02.121 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:54:01 smithi070 ceph-mon[168282]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:54:02.121 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:54:01 smithi070 ceph-mon[168282]: failed to encode map e127 with expected crc 2023-09-23T20:54:02.121 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:54:01 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T20:54:02.121 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:54:01 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pg_num", "val": "26"}]': finished 2023-09-23T20:54:02.121 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:54:01 smithi070 ceph-mon[168282]: osdmap e127: 8 total, 8 up, 6 in 2023-09-23T20:54:02.122 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:54:01 smithi070 ceph-mon[168282]: failed to encode map e127 with expected crc 2023-09-23T20:54:02.122 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:54:01 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:54:02.122 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:54:01 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:54:02.270 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:54:02.281 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:54:02.281 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:54:02.325 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_ops_in_flight 2023-09-23T20:54:02.382 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:54:02.412 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:54:02.452 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:54:02.457 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:54:02.457 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:54:02.558 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:54:02.595 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:54:02.637 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:54:02.641 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:54:02.641 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:54:02.742 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:54:02.834 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:54:02 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pg_num_actual", "val": "26"}]: dispatch 2023-09-23T20:54:02.838 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:54:02.918 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:54:02.923 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:54:02.923 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:54:03.001 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:54:02 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pg_num_actual", "val": "26"}]: dispatch 2023-09-23T20:54:03.001 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:54:02 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pg_num_actual", "val": "26"}]: dispatch 2023-09-23T20:54:03.024 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:54:03.049 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:54:03.082 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:54:03.084 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:54:03.084 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:54:03.185 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:54:03.211 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:54:03.244 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:54:03.246 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:54:03.246 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:54:03.347 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:54:03.373 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:54:03.408 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:54:03.410 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:54:03.411 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:54:03.504 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T20:54:03.511 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:54:03.586 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:54:03.873 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:54:03 smithi026 ceph-mon[179688]: pgmap v314: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 84 active+clean; 5.8 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 102 B/s rd, 1.4 MiB/s wr, 550 op/s; 16604/286620 objects degraded (5.793%); 9004/286620 objects misplaced (3.141%); 4.8 MiB/s, 76 objects/s recovering 2023-09-23T20:54:03.873 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:54:03 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pg_num_actual", "val": "26"}]': finished 2023-09-23T20:54:03.873 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:54:03 smithi026 ceph-mon[179688]: osdmap e128: 8 total, 8 up, 6 in 2023-09-23T20:54:03.874 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:54:03 smithi026 ceph-mon[179688]: failed to encode map e128 with expected crc 2023-09-23T20:54:03.874 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:54:03 smithi026 ceph-mon[179688]: failed to encode map e128 with expected crc 2023-09-23T20:54:03.874 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:54:03 smithi026 ceph-mon[179688]: failed to encode map e128 with expected crc 2023-09-23T20:54:03.874 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:54:03 smithi026 ceph-mon[175151]: pgmap v314: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 84 active+clean; 5.8 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 102 B/s rd, 1.4 MiB/s wr, 550 op/s; 16604/286620 objects degraded (5.793%); 9004/286620 objects misplaced (3.141%); 4.8 MiB/s, 76 objects/s recovering 2023-09-23T20:54:03.874 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:54:03 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pg_num_actual", "val": "26"}]': finished 2023-09-23T20:54:03.874 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:54:03 smithi026 ceph-mon[175151]: osdmap e128: 8 total, 8 up, 6 in 2023-09-23T20:54:03.874 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:54:03 smithi026 ceph-mon[175151]: failed to encode map e128 with expected crc 2023-09-23T20:54:03.874 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:54:03 smithi026 ceph-mon[175151]: failed to encode map e128 with expected crc 2023-09-23T20:54:03.874 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:54:03 smithi026 ceph-mon[175151]: failed to encode map e128 with expected crc 2023-09-23T20:54:04.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:54:03 smithi070 ceph-mon[168282]: pgmap v314: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 84 active+clean; 5.8 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 102 B/s rd, 1.4 MiB/s wr, 550 op/s; 16604/286620 objects degraded (5.793%); 9004/286620 objects misplaced (3.141%); 4.8 MiB/s, 76 objects/s recovering 2023-09-23T20:54:04.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:54:03 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pg_num_actual", "val": "26"}]': finished 2023-09-23T20:54:04.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:54:03 smithi070 ceph-mon[168282]: osdmap e128: 8 total, 8 up, 6 in 2023-09-23T20:54:04.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:54:03 smithi070 ceph-mon[168282]: failed to encode map e128 with expected crc 2023-09-23T20:54:04.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:54:03 smithi070 ceph-mon[168282]: failed to encode map e128 with expected crc 2023-09-23T20:54:04.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:54:03 smithi070 ceph-mon[168282]: failed to encode map e128 with expected crc 2023-09-23T20:54:05.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:54:04 smithi070 ceph-mon[168282]: failed to encode map e129 with expected crc 2023-09-23T20:54:05.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:54:04 smithi070 ceph-mon[168282]: osdmap e129: 8 total, 8 up, 6 in 2023-09-23T20:54:05.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:54:04 smithi070 ceph-mon[168282]: failed to encode map e129 with expected crc 2023-09-23T20:54:05.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:54:04 smithi070 ceph-mon[168282]: failed to encode map e129 with expected crc 2023-09-23T20:54:05.131 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:54:04 smithi026 ceph-mon[175151]: failed to encode map e129 with expected crc 2023-09-23T20:54:05.131 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:54:04 smithi026 ceph-mon[175151]: osdmap e129: 8 total, 8 up, 6 in 2023-09-23T20:54:05.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:54:04 smithi026 ceph-mon[175151]: failed to encode map e129 with expected crc 2023-09-23T20:54:05.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:54:04 smithi026 ceph-mon[175151]: failed to encode map e129 with expected crc 2023-09-23T20:54:05.134 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:54:04 smithi026 ceph-mon[179688]: failed to encode map e129 with expected crc 2023-09-23T20:54:05.134 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:54:04 smithi026 ceph-mon[179688]: osdmap e129: 8 total, 8 up, 6 in 2023-09-23T20:54:05.134 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:54:04 smithi026 ceph-mon[179688]: failed to encode map e129 with expected crc 2023-09-23T20:54:05.134 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:54:04 smithi026 ceph-mon[179688]: failed to encode map e129 with expected crc 2023-09-23T20:54:05.808 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:54:05.821 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:54:05.821 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:54:05.923 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:54:05.948 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:54:05 smithi070 ceph-mon[168282]: pgmap v317: 99 pgs: 2 active+recovery_wait+degraded, 85 active+clean, 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 1 active+recovering, 1 active+recovery_wait+undersized+degraded+remapped, 5 unknown, 3 peering; 5.8 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 682 op/s; 9855/282771 objects degraded (3.485%); 6905/282771 objects misplaced (2.442%); 6.7 MiB/s, 107 objects/s recovering 2023-09-23T20:54:05.948 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:54:05 smithi070 ceph-mon[168282]: failed to encode map e129 with expected crc 2023-09-23T20:54:05.948 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:54:05 smithi070 ceph-mon[168282]: failed to encode map e130 with expected crc 2023-09-23T20:54:05.948 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:54:05 smithi070 ceph-mon[168282]: failed to encode map e130 with expected crc 2023-09-23T20:54:05.948 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:54:05 smithi070 ceph-mon[168282]: osdmap e130: 8 total, 8 up, 6 in 2023-09-23T20:54:05.948 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:54:05 smithi070 ceph-mon[168282]: failed to encode map e130 with expected crc 2023-09-23T20:54:05.948 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:54:05 smithi070 ceph-mon[168282]: failed to encode map e130 with expected crc 2023-09-23T20:54:05.948 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:54:05 smithi070 ceph-mon[168282]: Health check update: Degraded data redundancy: 9855/282771 objects degraded (3.485%), 3 pgs degraded (PG_DEGRADED) 2023-09-23T20:54:05.951 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:54:05.990 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:54:05.996 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:54:05.998 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:54:06.099 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:54:06.124 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:54:05 smithi026 ceph-mon[175151]: pgmap v317: 99 pgs: 2 active+recovery_wait+degraded, 85 active+clean, 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 1 active+recovering, 1 active+recovery_wait+undersized+degraded+remapped, 5 unknown, 3 peering; 5.8 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 682 op/s; 9855/282771 objects degraded (3.485%); 6905/282771 objects misplaced (2.442%); 6.7 MiB/s, 107 objects/s recovering 2023-09-23T20:54:06.125 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:54:05 smithi026 ceph-mon[175151]: failed to encode map e129 with expected crc 2023-09-23T20:54:06.125 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:54:05 smithi026 ceph-mon[175151]: failed to encode map e130 with expected crc 2023-09-23T20:54:06.125 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:54:05 smithi026 ceph-mon[175151]: failed to encode map e130 with expected crc 2023-09-23T20:54:06.125 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:54:05 smithi026 ceph-mon[175151]: osdmap e130: 8 total, 8 up, 6 in 2023-09-23T20:54:06.125 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:54:05 smithi026 ceph-mon[175151]: failed to encode map e130 with expected crc 2023-09-23T20:54:06.125 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:54:05 smithi026 ceph-mon[175151]: failed to encode map e130 with expected crc 2023-09-23T20:54:06.125 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:54:05 smithi026 ceph-mon[175151]: Health check update: Degraded data redundancy: 9855/282771 objects degraded (3.485%), 3 pgs degraded (PG_DEGRADED) 2023-09-23T20:54:06.125 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:54:05 smithi026 ceph-mon[179688]: pgmap v317: 99 pgs: 2 active+recovery_wait+degraded, 85 active+clean, 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 1 active+recovering, 1 active+recovery_wait+undersized+degraded+remapped, 5 unknown, 3 peering; 5.8 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 682 op/s; 9855/282771 objects degraded (3.485%); 6905/282771 objects misplaced (2.442%); 6.7 MiB/s, 107 objects/s recovering 2023-09-23T20:54:06.125 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:54:05 smithi026 ceph-mon[179688]: failed to encode map e129 with expected crc 2023-09-23T20:54:06.125 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:54:05 smithi026 ceph-mon[179688]: failed to encode map e130 with expected crc 2023-09-23T20:54:06.126 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:54:05 smithi026 ceph-mon[179688]: failed to encode map e130 with expected crc 2023-09-23T20:54:06.126 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:54:05 smithi026 ceph-mon[179688]: osdmap e130: 8 total, 8 up, 6 in 2023-09-23T20:54:06.126 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:54:05 smithi026 ceph-mon[179688]: failed to encode map e130 with expected crc 2023-09-23T20:54:06.126 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:54:05 smithi026 ceph-mon[179688]: failed to encode map e130 with expected crc 2023-09-23T20:54:06.126 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:54:05 smithi026 ceph-mon[179688]: Health check update: Degraded data redundancy: 9855/282771 objects degraded (3.485%), 3 pgs degraded (PG_DEGRADED) 2023-09-23T20:54:06.127 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:54:06.455 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 6, 7] out_osds: [5, 4] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-09-23T20:54:06.456 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T20:54:06.456 INFO:tasks.thrashosds.thrasher:Adding osd 4 2023-09-23T20:54:06.456 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph osd in 4 2023-09-23T20:54:07.100 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:54:07.109 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:54:07.109 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:54:07.109 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:54:07.210 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:54:07.291 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:54:07.388 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:54:08.014 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:54:07 smithi026 ceph-mon[175151]: pgmap v319: 99 pgs: 2 active+recovery_wait+degraded, 87 active+clean, 1 active+recovery_wait, 1 active+recovering, 2 active+recovery_wait+undersized+degraded+remapped, 3 unknown, 3 peering; 5.7 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 524 op/s; 14163/281469 objects degraded (5.032%); 2469/281469 objects misplaced (0.877%); 6.3 MiB/s, 102 objects/s recovering 2023-09-23T20:54:08.014 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:54:07 smithi026 ceph-mon[179688]: pgmap v319: 99 pgs: 2 active+recovery_wait+degraded, 87 active+clean, 1 active+recovery_wait, 1 active+recovering, 2 active+recovery_wait+undersized+degraded+remapped, 3 unknown, 3 peering; 5.7 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 524 op/s; 14163/281469 objects degraded (5.032%); 2469/281469 objects misplaced (0.877%); 6.3 MiB/s, 102 objects/s recovering 2023-09-23T20:54:08.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:54:07 smithi070 ceph-mon[168282]: pgmap v319: 99 pgs: 2 active+recovery_wait+degraded, 87 active+clean, 1 active+recovery_wait, 1 active+recovering, 2 active+recovery_wait+undersized+degraded+remapped, 3 unknown, 3 peering; 5.7 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 524 op/s; 14163/281469 objects degraded (5.032%); 2469/281469 objects misplaced (0.877%); 6.3 MiB/s, 102 objects/s recovering 2023-09-23T20:54:08.502 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0: {} 2023-09-23T20:54:08.519 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:54:08.527 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:54:08.528 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:54:08.544 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1: {} 2023-09-23T20:54:08.593 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2: {} 2023-09-23T20:54:08.629 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:54:08.656 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3: {} 2023-09-23T20:54:08.703 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4: {} 2023-09-23T20:54:08.734 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:54:08.756 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T20:54:08.792 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6: {} 2023-09-23T20:54:08.820 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:54:08 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/2176796395' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:54:08.820 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:54:08 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/2176796395' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:54:08.841 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T20:54:08.890 INFO:teuthology.orchestra.run.smithi026.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T20:54:08.890 INFO:teuthology.orchestra.run.smithi026.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T20:54:08.891 INFO:teuthology.orchestra.run.smithi026.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T20:54:08.891 INFO:teuthology.orchestra.run.smithi026.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T20:54:08.891 INFO:teuthology.orchestra.run.smithi026.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T20:54:08.891 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T20:54:08.891 INFO:teuthology.orchestra.run.smithi026.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T20:54:08.891 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T20:54:08.927 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:54:08.937 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:54:08.937 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:54:09.038 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:54:09.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:54:08 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/2176796395' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:54:09.150 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:54:09.534 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:54:09.539 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:54:09.539 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:54:09.640 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:54:09.669 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:54:09.705 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:54:09.710 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:54:09.710 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:54:09.811 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:54:09.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:54:09 smithi026 ceph-mon[175151]: pgmap v320: 99 pgs: 2 active+recovery_wait+degraded, 90 active+clean, 1 active+recovery_wait, 1 active+recovering, 2 active+recovery_wait+undersized+degraded+remapped, 3 peering; 5.7 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 392 op/s; 14163/279552 objects degraded (5.066%); 2469/279552 objects misplaced (0.883%); 3.8 MiB/s, 62 objects/s recovering 2023-09-23T20:54:09.882 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:54:09 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[164029]: ::ffff:172.21.15.70 - - [23/Sep/2023:20:54:09] "GET /metrics HTTP/1.1" 200 36337 "" "Prometheus/2.43.0" 2023-09-23T20:54:09.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:54:09 smithi026 ceph-mon[179688]: pgmap v320: 99 pgs: 2 active+recovery_wait+degraded, 90 active+clean, 1 active+recovery_wait, 1 active+recovering, 2 active+recovery_wait+undersized+degraded+remapped, 3 peering; 5.7 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 392 op/s; 14163/279552 objects degraded (5.066%); 2469/279552 objects misplaced (0.883%); 3.8 MiB/s, 62 objects/s recovering 2023-09-23T20:54:09.887 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:54:09.899 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:54:10.065 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:54:10.070 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:54:10.070 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:54:10.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:54:09 smithi070 ceph-mon[168282]: pgmap v320: 99 pgs: 2 active+recovery_wait+degraded, 90 active+clean, 1 active+recovery_wait, 1 active+recovering, 2 active+recovery_wait+undersized+degraded+remapped, 3 peering; 5.7 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 392 op/s; 14163/279552 objects degraded (5.066%); 2469/279552 objects misplaced (0.883%); 3.8 MiB/s, 62 objects/s recovering 2023-09-23T20:54:10.171 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:54:10.202 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:54:10.242 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:54:10.245 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:54:10.246 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:54:10.346 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:54:10.373 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:54:10.413 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:54:10.416 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:54:10.416 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:54:10.442 INFO:teuthology.orchestra.run.smithi026.stdout:{ 2023-09-23T20:54:10.442 INFO:teuthology.orchestra.run.smithi026.stdout: "ops": [ 2023-09-23T20:54:10.442 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:54:10.442 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_repop(client.34797.0:113131 6.b e130/102 6:d2bc34fa:::benchmark_data_smithi026_223868_object12005:head v 130'6918, mlcod=130'6916)", 2023-09-23T20:54:10.442 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T20:54:10.428723+0000", 2023-09-23T20:54:10.443 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 0.0091088850000000006, 2023-09-23T20:54:10.443 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0091414359999999993, 2023-09-23T20:54:10.443 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T20:54:10.443 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "started", 2023-09-23T20:54:10.443 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T20:54:10.443 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:54:10.443 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T20:54:10.443 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:10.428723+0000", 2023-09-23T20:54:10.443 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:54:10.443 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:54:10.443 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:54:10.443 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T20:54:10.443 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:10.428723+0000", 2023-09-23T20:54:10.444 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:54:10.444 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:54:10.444 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:54:10.444 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T20:54:10.444 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:10.428722+0000", 2023-09-23T20:54:10.444 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T20:54:10.444 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:54:10.444 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:54:10.444 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T20:54:10.444 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:10.428730+0000", 2023-09-23T20:54:10.444 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.8869999999999992e-06 2023-09-23T20:54:10.444 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:54:10.444 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:54:10.445 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T20:54:10.445 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:10.428741+0000", 2023-09-23T20:54:10.445 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1731e-05 2023-09-23T20:54:10.445 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:54:10.445 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:54:10.445 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T20:54:10.445 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:10.428749+0000", 2023-09-23T20:54:10.445 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.0940000000000008e-06 2023-09-23T20:54:10.445 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:54:10.445 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:54:10.445 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:54:10.445 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:10.428788+0000", 2023-09-23T20:54:10.445 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.8795999999999999e-05 2023-09-23T20:54:10.445 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:54:10.445 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:54:10.446 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T20:54:10.446 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:10.428808+0000", 2023-09-23T20:54:10.446 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.9443e-05 2023-09-23T20:54:10.446 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:54:10.446 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T20:54:10.446 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:54:10.446 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:54:10.446 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:54:10.446 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34797.0:113137 6.4 6:214570f0:::benchmark_data_smithi026_223868_object12011:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e130)", 2023-09-23T20:54:10.446 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T20:54:10.431295+0000", 2023-09-23T20:54:10.446 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 0.0065369119999999998, 2023-09-23T20:54:10.446 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00664834, 2023-09-23T20:54:10.446 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T20:54:10.446 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "waiting for sub ops", 2023-09-23T20:54:10.447 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T20:54:10.447 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34797", 2023-09-23T20:54:10.447 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T20:54:10.447 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 113137 2023-09-23T20:54:10.447 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:54:10.447 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T20:54:10.447 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:54:10.447 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T20:54:10.447 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:10.431295+0000", 2023-09-23T20:54:10.447 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:54:10.447 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:54:10.447 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:54:10.447 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T20:54:10.447 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:10.431295+0000", 2023-09-23T20:54:10.448 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:54:10.448 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:54:10.448 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:54:10.448 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T20:54:10.448 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:10.431293+0000", 2023-09-23T20:54:10.448 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999981 2023-09-23T20:54:10.448 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:54:10.448 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:54:10.448 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T20:54:10.448 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:10.431304+0000", 2023-09-23T20:54:10.448 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0567999999999999e-05 2023-09-23T20:54:10.448 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:54:10.448 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:54:10.448 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T20:54:10.449 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:10.431321+0000", 2023-09-23T20:54:10.449 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.7810999999999999e-05 2023-09-23T20:54:10.449 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:54:10.449 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:54:10.449 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T20:54:10.449 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:10.431332+0000", 2023-09-23T20:54:10.449 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0399e-05 2023-09-23T20:54:10.449 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:54:10.449 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:54:10.449 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:54:10.449 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:10.432492+0000", 2023-09-23T20:54:10.449 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0011602260000000001 2023-09-23T20:54:10.449 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:54:10.449 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:54:10.450 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T20:54:10.450 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:10.432615+0000", 2023-09-23T20:54:10.450 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00012325699999999999 2023-09-23T20:54:10.450 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:54:10.450 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:54:10.450 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 3,6", 2023-09-23T20:54:10.450 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:10.432992+0000", 2023-09-23T20:54:10.450 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000376497 2023-09-23T20:54:10.450 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:54:10.450 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T20:54:10.450 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:54:10.450 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:54:10.450 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:54:10.450 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34797.0:113138 6.4 6:2670c76c:::benchmark_data_smithi026_223868_object12012:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e130)", 2023-09-23T20:54:10.451 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T20:54:10.431347+0000", 2023-09-23T20:54:10.451 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 0.0064843949999999996, 2023-09-23T20:54:10.451 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0066819640000000003, 2023-09-23T20:54:10.451 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T20:54:10.451 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "waiting for sub ops", 2023-09-23T20:54:10.451 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T20:54:10.451 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34797", 2023-09-23T20:54:10.451 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T20:54:10.451 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 113138 2023-09-23T20:54:10.451 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:54:10.451 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T20:54:10.451 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:54:10.451 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T20:54:10.451 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:10.431347+0000", 2023-09-23T20:54:10.452 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:54:10.452 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:54:10.452 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:54:10.452 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T20:54:10.452 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:10.431347+0000", 2023-09-23T20:54:10.452 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:54:10.452 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:54:10.452 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:54:10.452 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T20:54:10.452 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:10.431347+0000", 2023-09-23T20:54:10.452 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T20:54:10.452 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:54:10.452 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:54:10.453 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T20:54:10.453 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:10.431350+0000", 2023-09-23T20:54:10.453 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.88e-06 2023-09-23T20:54:10.453 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:54:10.453 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:54:10.453 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T20:54:10.453 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:10.431356+0000", 2023-09-23T20:54:10.453 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.7769999999999999e-06 2023-09-23T20:54:10.453 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:54:10.453 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:54:10.453 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T20:54:10.453 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:10.431361+0000", 2023-09-23T20:54:10.453 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.9219999999999996e-06 2023-09-23T20:54:10.453 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:54:10.454 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:54:10.454 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:54:10.454 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:10.436194+0000", 2023-09-23T20:54:10.454 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0048329109999999996 2023-09-23T20:54:10.454 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:54:10.454 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:54:10.454 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T20:54:10.454 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:10.436324+0000", 2023-09-23T20:54:10.454 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00013024299999999999 2023-09-23T20:54:10.454 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:54:10.454 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:54:10.454 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 3,6", 2023-09-23T20:54:10.454 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:10.436654+0000", 2023-09-23T20:54:10.454 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00033037399999999998 2023-09-23T20:54:10.454 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:54:10.455 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T20:54:10.455 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:54:10.455 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:54:10.455 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:54:10.455 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34797.0:113140 6.d 6:bd0ea723:::benchmark_data_smithi026_223868_object12014:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e130)", 2023-09-23T20:54:10.455 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T20:54:10.431365+0000", 2023-09-23T20:54:10.455 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 0.0064670120000000003, 2023-09-23T20:54:10.455 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0067462499999999996, 2023-09-23T20:54:10.455 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T20:54:10.455 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "waiting for sub ops", 2023-09-23T20:54:10.455 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T20:54:10.455 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34797", 2023-09-23T20:54:10.455 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T20:54:10.456 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 113140 2023-09-23T20:54:10.456 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:54:10.456 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T20:54:10.456 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:54:10.456 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T20:54:10.456 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:10.431365+0000", 2023-09-23T20:54:10.456 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:54:10.456 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:54:10.456 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:54:10.456 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T20:54:10.456 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:10.431365+0000", 2023-09-23T20:54:10.456 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:54:10.456 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:54:10.457 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:54:10.457 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T20:54:10.457 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:10.431365+0000", 2023-09-23T20:54:10.457 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T20:54:10.457 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:54:10.457 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:54:10.457 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T20:54:10.457 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:10.431367+0000", 2023-09-23T20:54:10.457 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.0810000000000001e-06 2023-09-23T20:54:10.457 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:54:10.457 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:54:10.457 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T20:54:10.457 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:10.431372+0000", 2023-09-23T20:54:10.457 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.9139999999999999e-06 2023-09-23T20:54:10.457 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:54:10.458 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:54:10.458 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T20:54:10.458 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:10.431376+0000", 2023-09-23T20:54:10.458 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.5499999999999996e-06 2023-09-23T20:54:10.458 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:54:10.458 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:54:10.458 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:54:10.458 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:10.433001+0000", 2023-09-23T20:54:10.458 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.001625086 2023-09-23T20:54:10.458 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:54:10.458 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:54:10.458 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T20:54:10.458 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:10.433104+0000", 2023-09-23T20:54:10.459 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00010331 2023-09-23T20:54:10.459 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:54:10.459 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:54:10.459 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 0,2", 2023-09-23T20:54:10.459 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:10.433214+0000", 2023-09-23T20:54:10.459 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00010939599999999999 2023-09-23T20:54:10.459 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:54:10.459 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T20:54:10.459 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:54:10.459 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:54:10.459 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:54:10.459 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_repop_reply(client.34797.0:113132 6.d e130/102)", 2023-09-23T20:54:10.459 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T20:54:10.437772+0000", 2023-09-23T20:54:10.459 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 5.9558999999999997e-05, 2023-09-23T20:54:10.460 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000417592, 2023-09-23T20:54:10.460 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T20:54:10.460 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "queued for pg", 2023-09-23T20:54:10.460 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T20:54:10.460 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:54:10.460 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T20:54:10.460 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:10.437772+0000", 2023-09-23T20:54:10.460 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:54:10.460 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:54:10.460 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:54:10.460 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T20:54:10.460 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:10.437772+0000", 2023-09-23T20:54:10.460 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:54:10.460 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:54:10.461 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:54:10.461 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T20:54:10.461 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:10.437771+0000", 2023-09-23T20:54:10.461 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.999999 2023-09-23T20:54:10.461 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:54:10.461 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:54:10.461 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T20:54:10.461 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:10.437778+0000", 2023-09-23T20:54:10.461 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.0419999999999997e-06 2023-09-23T20:54:10.461 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:54:10.461 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:54:10.461 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T20:54:10.461 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:10.437792+0000", 2023-09-23T20:54:10.462 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.3454e-05 2023-09-23T20:54:10.462 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:54:10.462 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:54:10.462 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T20:54:10.462 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:10.437800+0000", 2023-09-23T20:54:10.462 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.2630000000000003e-06 2023-09-23T20:54:10.462 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:54:10.462 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T20:54:10.462 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:54:10.462 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:54:10.462 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:54:10.462 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34797.0:113130 6.e 6:7dbaa99a:::benchmark_data_smithi026_223868_object12004:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e130)", 2023-09-23T20:54:10.463 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T20:54:10.425321+0000", 2023-09-23T20:54:10.463 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 0.012510752, 2023-09-23T20:54:10.463 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.012926959999999999, 2023-09-23T20:54:10.463 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T20:54:10.463 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "waiting for sub ops", 2023-09-23T20:54:10.463 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T20:54:10.463 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34797", 2023-09-23T20:54:10.463 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T20:54:10.463 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 113130 2023-09-23T20:54:10.463 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:54:10.463 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T20:54:10.463 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:54:10.463 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T20:54:10.464 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:10.425321+0000", 2023-09-23T20:54:10.464 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:54:10.464 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:54:10.464 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:54:10.464 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T20:54:10.464 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:10.425321+0000", 2023-09-23T20:54:10.464 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:54:10.464 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:54:10.464 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:54:10.464 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T20:54:10.464 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:10.425320+0000", 2023-09-23T20:54:10.464 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.999999 2023-09-23T20:54:10.464 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:54:10.465 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:54:10.465 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T20:54:10.465 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:10.425327+0000", 2023-09-23T20:54:10.465 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.9090000000000003e-06 2023-09-23T20:54:10.465 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:54:10.465 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:54:10.465 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T20:54:10.465 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:10.425337+0000", 2023-09-23T20:54:10.465 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0234999999999999e-05 2023-09-23T20:54:10.465 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:54:10.465 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:54:10.465 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T20:54:10.465 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:10.425344+0000", 2023-09-23T20:54:10.465 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.3699999999999997e-06 2023-09-23T20:54:10.466 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:54:10.466 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:54:10.466 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:54:10.466 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:10.425393+0000", 2023-09-23T20:54:10.466 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.8519000000000001e-05 2023-09-23T20:54:10.466 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:54:10.466 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:54:10.466 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T20:54:10.466 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:10.425545+0000", 2023-09-23T20:54:10.466 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00015159400000000001 2023-09-23T20:54:10.466 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:54:10.466 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:54:10.466 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 2,7", 2023-09-23T20:54:10.466 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:10.425703+0000", 2023-09-23T20:54:10.467 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00015808400000000001 2023-09-23T20:54:10.467 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:54:10.467 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:54:10.467 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T20:54:10.467 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:10.428951+0000", 2023-09-23T20:54:10.467 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.003248478 2023-09-23T20:54:10.467 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:54:10.467 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:54:10.467 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:54:10.467 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:10.431664+0000", 2023-09-23T20:54:10.467 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.002712702 2023-09-23T20:54:10.467 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:54:10.467 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T20:54:10.468 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:54:10.468 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:54:10.468 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:54:10.468 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34797.0:113132 6.d 6:bd522745:::benchmark_data_smithi026_223868_object12006:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e130)", 2023-09-23T20:54:10.468 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T20:54:10.425359+0000", 2023-09-23T20:54:10.468 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 0.012472515, 2023-09-23T20:54:10.468 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.013004920999999999, 2023-09-23T20:54:10.468 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T20:54:10.468 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "waiting for sub ops", 2023-09-23T20:54:10.468 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T20:54:10.468 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34797", 2023-09-23T20:54:10.468 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T20:54:10.468 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 113132 2023-09-23T20:54:10.469 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:54:10.469 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T20:54:10.469 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:54:10.469 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T20:54:10.469 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:10.425359+0000", 2023-09-23T20:54:10.469 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:54:10.469 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:54:10.469 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:54:10.469 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T20:54:10.469 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:10.425359+0000", 2023-09-23T20:54:10.469 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:54:10.469 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:54:10.469 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:54:10.469 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T20:54:10.470 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:10.425359+0000", 2023-09-23T20:54:10.470 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T20:54:10.470 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:54:10.470 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:54:10.470 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T20:54:10.470 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:10.425362+0000", 2023-09-23T20:54:10.470 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.3840000000000001e-06 2023-09-23T20:54:10.470 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:54:10.470 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:54:10.470 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T20:54:10.470 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:10.425369+0000", 2023-09-23T20:54:10.470 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.336e-06 2023-09-23T20:54:10.470 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:54:10.471 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:54:10.471 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T20:54:10.471 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:10.425374+0000", 2023-09-23T20:54:10.471 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.8159999999999999e-06 2023-09-23T20:54:10.471 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:54:10.471 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:54:10.471 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:54:10.471 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:10.425453+0000", 2023-09-23T20:54:10.471 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.8291e-05 2023-09-23T20:54:10.471 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:54:10.471 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:54:10.471 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T20:54:10.471 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:10.425600+0000", 2023-09-23T20:54:10.472 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00014765899999999999 2023-09-23T20:54:10.472 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:54:10.472 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:54:10.472 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 0,2", 2023-09-23T20:54:10.472 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:10.425719+0000", 2023-09-23T20:54:10.472 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00011897599999999999 2023-09-23T20:54:10.472 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:54:10.472 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:54:10.472 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T20:54:10.472 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:10.428917+0000", 2023-09-23T20:54:10.472 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0031978179999999998 2023-09-23T20:54:10.472 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:54:10.472 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T20:54:10.472 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:54:10.473 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:54:10.473 INFO:teuthology.orchestra.run.smithi026.stdout: ], 2023-09-23T20:54:10.473 INFO:teuthology.orchestra.run.smithi026.stdout: "num_ops": 7 2023-09-23T20:54:10.473 INFO:teuthology.orchestra.run.smithi026.stdout:} 2023-09-23T20:54:10.517 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:54:10.546 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:54:10.583 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:54:10.586 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:54:10.587 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:54:10.688 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:54:10.716 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:54:10.754 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:54:10.758 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:54:10.758 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:54:10.859 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:54:10.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:54:10 smithi070 ceph-mon[168282]: Health check update: Degraded data redundancy: 14163/279552 objects degraded (5.066%), 4 pgs degraded (PG_DEGRADED) 2023-09-23T20:54:10.891 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:54:10.930 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:54:10.933 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:54:10.933 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:54:11.034 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:54:11.068 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:54:11.109 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:54:11.112 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:54:11.112 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:54:11.131 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:54:10 smithi026 ceph-mon[175151]: Health check update: Degraded data redundancy: 14163/279552 objects degraded (5.066%), 4 pgs degraded (PG_DEGRADED) 2023-09-23T20:54:11.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:54:10 smithi026 ceph-mon[179688]: Health check update: Degraded data redundancy: 14163/279552 objects degraded (5.066%), 4 pgs degraded (PG_DEGRADED) 2023-09-23T20:54:11.213 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:54:11.241 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:54:11.258 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_blocked_ops 2023-09-23T20:54:11.282 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:54:11.286 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:54:11.287 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:54:11.387 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:54:11.491 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:54:11.620 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:54:11.626 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:54:11.626 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:54:11.703 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T20:54:11.727 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:54:11.763 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:54:11.799 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:54:11.801 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:54:11.802 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:54:11.902 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:54:11.914 INFO:teuthology.orchestra.run.smithi026.stderr:noscrub is unset 2023-09-23T20:54:12.091 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:54:12.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:54:11 smithi070 ceph-mon[168282]: pgmap v321: 99 pgs: 5 active+recovery_wait+degraded, 90 active+clean, 1 active+recovery_wait, 1 active+recovering, 2 active+recovery_wait+undersized+degraded+remapped; 5.6 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 580 op/s; 21948/273444 objects degraded (8.027%); 2045/273444 objects misplaced (0.748%); 11 MiB/s, 184 objects/s recovering 2023-09-23T20:54:12.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:54:11 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/3771601664' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:54:12.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:54:11 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:54:12.342 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:54:12.353 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:54:12.353 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:54:11 smithi026 ceph-mon[175151]: pgmap v321: 99 pgs: 5 active+recovery_wait+degraded, 90 active+clean, 1 active+recovery_wait, 1 active+recovering, 2 active+recovery_wait+undersized+degraded+remapped; 5.6 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 580 op/s; 21948/273444 objects degraded (8.027%); 2045/273444 objects misplaced (0.748%); 11 MiB/s, 184 objects/s recovering 2023-09-23T20:54:12.354 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:54:11 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/3771601664' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:54:12.354 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:54:11 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:54:12.354 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:54:11 smithi026 ceph-mon[179688]: pgmap v321: 99 pgs: 5 active+recovery_wait+degraded, 90 active+clean, 1 active+recovery_wait, 1 active+recovering, 2 active+recovery_wait+undersized+degraded+remapped; 5.6 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 580 op/s; 21948/273444 objects degraded (8.027%); 2045/273444 objects misplaced (0.748%); 11 MiB/s, 184 objects/s recovering 2023-09-23T20:54:12.354 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:54:11 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/3771601664' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:54:12.354 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:54:11 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:54:12.355 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:54:12.445 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:54:12.456 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:54:12.541 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:54:12.624 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:54:12.628 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:54:12.629 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:54:12.629 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:54:12 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mon-a[175127]: 2023-09-23T20:54:12.390+0000 7f1e2d487700 -1 mon.a@0(leader).osd e131 definitely_dead 0 2023-09-23T20:54:12.730 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:54:12.763 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:54:12.802 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:54:12.805 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:54:12.805 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:54:12.906 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:54:12.959 INFO:teuthology.orchestra.run.smithi026.stderr:marked in osd.4. 2023-09-23T20:54:12.985 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:54:12.993 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:54:13.032 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:54:13.035 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:54:13.035 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:54:13.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:54:12 smithi070 ceph-mon[168282]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:54:13.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:54:12 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T20:54:13.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:54:12 smithi070 ceph-mon[168282]: osdmap e131: 8 total, 8 up, 6 in 2023-09-23T20:54:13.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:54:12 smithi070 ceph-mon[168282]: failed to encode map e131 with expected crc 2023-09-23T20:54:13.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:54:12 smithi070 ceph-mon[168282]: failed to encode map e131 with expected crc 2023-09-23T20:54:13.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:54:12 smithi070 ceph-mon[168282]: 6.0 scrub starts 2023-09-23T20:54:13.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:54:12 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/1207806592' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["4"]}]: dispatch 2023-09-23T20:54:13.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:54:12 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["4"]}]: dispatch 2023-09-23T20:54:13.136 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:54:13.161 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:54:13.194 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:54:13.196 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:54:13.196 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:54:13.225 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:54:12 smithi026 ceph-mon[175151]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:54:13.225 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:54:12 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T20:54:13.225 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:54:12 smithi026 ceph-mon[175151]: osdmap e131: 8 total, 8 up, 6 in 2023-09-23T20:54:13.225 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:54:12 smithi026 ceph-mon[175151]: failed to encode map e131 with expected crc 2023-09-23T20:54:13.225 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:54:12 smithi026 ceph-mon[175151]: failed to encode map e131 with expected crc 2023-09-23T20:54:13.225 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:54:12 smithi026 ceph-mon[175151]: 6.0 scrub starts 2023-09-23T20:54:13.225 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:54:12 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/1207806592' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["4"]}]: dispatch 2023-09-23T20:54:13.225 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:54:12 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["4"]}]: dispatch 2023-09-23T20:54:13.226 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:54:12 smithi026 ceph-mon[179688]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:54:13.226 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:54:12 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T20:54:13.226 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:54:12 smithi026 ceph-mon[179688]: osdmap e131: 8 total, 8 up, 6 in 2023-09-23T20:54:13.226 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:54:12 smithi026 ceph-mon[179688]: failed to encode map e131 with expected crc 2023-09-23T20:54:13.226 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:54:12 smithi026 ceph-mon[179688]: failed to encode map e131 with expected crc 2023-09-23T20:54:13.226 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:54:12 smithi026 ceph-mon[179688]: 6.0 scrub starts 2023-09-23T20:54:13.226 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:54:12 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/1207806592' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["4"]}]: dispatch 2023-09-23T20:54:13.226 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:54:12 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["4"]}]: dispatch 2023-09-23T20:54:13.297 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:54:13.322 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:54:13.353 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:54:13.356 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:54:13.356 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:54:13.457 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:54:13.483 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:54:13.514 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:54:13.516 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:54:13.517 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:54:13.618 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:54:13.643 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:54:13.674 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:54:13.676 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:54:13.676 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:54:13.777 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:54:13.812 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:54:14.103 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:54:14.111 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:54:13 smithi026 ceph-mon[179688]: pgmap v323: 99 pgs: 5 active+recovery_wait+degraded, 90 active+clean, 1 active+recovery_wait, 1 active+recovering, 2 active+recovery_wait+undersized+degraded+remapped; 5.5 GiB data, 16 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 607 op/s; 21948/268179 objects degraded (8.184%); 2045/268179 objects misplaced (0.763%); 8.9 MiB/s, 143 objects/s recovering 2023-09-23T20:54:14.112 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:54:13 smithi026 ceph-mon[179688]: failed to encode map e132 with expected crc 2023-09-23T20:54:14.112 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:54:13 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["4"]}]': finished 2023-09-23T20:54:14.112 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:54:13 smithi026 ceph-mon[179688]: osdmap e132: 8 total, 8 up, 7 in 2023-09-23T20:54:14.112 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:54:13 smithi026 ceph-mon[179688]: failed to encode map e132 with expected crc 2023-09-23T20:54:14.112 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:54:13 smithi026 ceph-mon[179688]: failed to encode map e132 with expected crc 2023-09-23T20:54:14.112 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:54:13 smithi026 ceph-mon[179688]: failed to encode map e132 with expected crc 2023-09-23T20:54:14.112 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:54:13 smithi026 ceph-mon[179688]: failed to encode map e132 with expected crc 2023-09-23T20:54:14.112 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:54:13 smithi026 ceph-mon[179688]: failed to encode map e132 with expected crc 2023-09-23T20:54:14.112 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:54:13 smithi026 ceph-mon[175151]: pgmap v323: 99 pgs: 5 active+recovery_wait+degraded, 90 active+clean, 1 active+recovery_wait, 1 active+recovering, 2 active+recovery_wait+undersized+degraded+remapped; 5.5 GiB data, 16 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 607 op/s; 21948/268179 objects degraded (8.184%); 2045/268179 objects misplaced (0.763%); 8.9 MiB/s, 143 objects/s recovering 2023-09-23T20:54:14.113 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:54:13 smithi026 ceph-mon[175151]: failed to encode map e132 with expected crc 2023-09-23T20:54:14.113 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:54:13 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["4"]}]': finished 2023-09-23T20:54:14.113 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:54:13 smithi026 ceph-mon[175151]: osdmap e132: 8 total, 8 up, 7 in 2023-09-23T20:54:14.113 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:54:13 smithi026 ceph-mon[175151]: failed to encode map e132 with expected crc 2023-09-23T20:54:14.113 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:54:13 smithi026 ceph-mon[175151]: failed to encode map e132 with expected crc 2023-09-23T20:54:14.113 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:54:13 smithi026 ceph-mon[175151]: failed to encode map e132 with expected crc 2023-09-23T20:54:14.113 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:54:13 smithi026 ceph-mon[175151]: failed to encode map e132 with expected crc 2023-09-23T20:54:14.113 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:54:13 smithi026 ceph-mon[175151]: failed to encode map e132 with expected crc 2023-09-23T20:54:14.114 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:54:14.114 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:54:14.135 INFO:tasks.thrashosds.thrasher:Added osd 4 2023-09-23T20:54:14.215 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:54:14.245 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:54:14.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:54:13 smithi070 ceph-mon[168282]: pgmap v323: 99 pgs: 5 active+recovery_wait+degraded, 90 active+clean, 1 active+recovery_wait, 1 active+recovering, 2 active+recovery_wait+undersized+degraded+remapped; 5.5 GiB data, 16 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 607 op/s; 21948/268179 objects degraded (8.184%); 2045/268179 objects misplaced (0.763%); 8.9 MiB/s, 143 objects/s recovering 2023-09-23T20:54:14.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:54:13 smithi070 ceph-mon[168282]: failed to encode map e132 with expected crc 2023-09-23T20:54:14.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:54:13 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["4"]}]': finished 2023-09-23T20:54:14.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:54:13 smithi070 ceph-mon[168282]: osdmap e132: 8 total, 8 up, 7 in 2023-09-23T20:54:14.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:54:13 smithi070 ceph-mon[168282]: failed to encode map e132 with expected crc 2023-09-23T20:54:14.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:54:13 smithi070 ceph-mon[168282]: failed to encode map e132 with expected crc 2023-09-23T20:54:14.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:54:13 smithi070 ceph-mon[168282]: failed to encode map e132 with expected crc 2023-09-23T20:54:14.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:54:13 smithi070 ceph-mon[168282]: failed to encode map e132 with expected crc 2023-09-23T20:54:14.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:54:13 smithi070 ceph-mon[168282]: failed to encode map e132 with expected crc 2023-09-23T20:54:14.455 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T20:54:14.538 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:54:14.540 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:54:14.541 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:54:14.641 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:54:14.667 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:54:14.699 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:54:14.701 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:54:14.702 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:54:14.802 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:54:14.828 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:54:14.861 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:54:14.863 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:54:14.864 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:54:14.964 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:54:15.045 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:54:15.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:54:14 smithi026 ceph-mon[175151]: failed to encode map e132 with expected crc 2023-09-23T20:54:15.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:54:14 smithi026 ceph-mon[175151]: failed to encode map e133 with expected crc 2023-09-23T20:54:15.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:54:14 smithi026 ceph-mon[175151]: failed to encode map e133 with expected crc 2023-09-23T20:54:15.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:54:14 smithi026 ceph-mon[175151]: failed to encode map e133 with expected crc 2023-09-23T20:54:15.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:54:14 smithi026 ceph-mon[175151]: osdmap e133: 8 total, 8 up, 7 in 2023-09-23T20:54:15.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:54:14 smithi026 ceph-mon[175151]: failed to encode map e133 with expected crc 2023-09-23T20:54:15.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:54:14 smithi026 ceph-mon[175151]: failed to encode map e133 with expected crc 2023-09-23T20:54:15.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:54:14 smithi026 ceph-mon[175151]: failed to encode map e133 with expected crc 2023-09-23T20:54:15.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:54:14 smithi026 ceph-mon[175151]: failed to encode map e133 with expected crc 2023-09-23T20:54:15.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:54:14 smithi026 ceph-mon[175151]: failed to encode map e133 with expected crc 2023-09-23T20:54:15.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:54:14 smithi026 ceph-mon[175151]: failed to encode map e133 with expected crc 2023-09-23T20:54:15.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:54:14 smithi026 ceph-mon[175151]: 6.9 scrub starts 2023-09-23T20:54:15.134 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:54:14 smithi026 ceph-mon[175151]: Health check update: Degraded data redundancy: 21948/268179 objects degraded (8.184%), 7 pgs degraded (PG_DEGRADED) 2023-09-23T20:54:15.134 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:54:14 smithi026 ceph-mon[175151]: osdmap e134: 8 total, 8 up, 7 in 2023-09-23T20:54:15.134 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:54:14 smithi026 ceph-mon[175151]: failed to encode map e134 with expected crc 2023-09-23T20:54:15.134 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:54:14 smithi026 ceph-mon[179688]: failed to encode map e132 with expected crc 2023-09-23T20:54:15.134 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:54:14 smithi026 ceph-mon[179688]: failed to encode map e133 with expected crc 2023-09-23T20:54:15.134 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:54:14 smithi026 ceph-mon[179688]: failed to encode map e133 with expected crc 2023-09-23T20:54:15.134 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:54:14 smithi026 ceph-mon[179688]: failed to encode map e133 with expected crc 2023-09-23T20:54:15.134 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:54:14 smithi026 ceph-mon[179688]: osdmap e133: 8 total, 8 up, 7 in 2023-09-23T20:54:15.134 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:54:14 smithi026 ceph-mon[179688]: failed to encode map e133 with expected crc 2023-09-23T20:54:15.134 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:54:14 smithi026 ceph-mon[179688]: failed to encode map e133 with expected crc 2023-09-23T20:54:15.135 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:54:14 smithi026 ceph-mon[179688]: failed to encode map e133 with expected crc 2023-09-23T20:54:15.135 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:54:14 smithi026 ceph-mon[179688]: failed to encode map e133 with expected crc 2023-09-23T20:54:15.135 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:54:14 smithi026 ceph-mon[179688]: failed to encode map e133 with expected crc 2023-09-23T20:54:15.135 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:54:14 smithi026 ceph-mon[179688]: failed to encode map e133 with expected crc 2023-09-23T20:54:15.135 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:54:14 smithi026 ceph-mon[179688]: 6.9 scrub starts 2023-09-23T20:54:15.135 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:54:14 smithi026 ceph-mon[179688]: Health check update: Degraded data redundancy: 21948/268179 objects degraded (8.184%), 7 pgs degraded (PG_DEGRADED) 2023-09-23T20:54:15.135 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:54:14 smithi026 ceph-mon[179688]: osdmap e134: 8 total, 8 up, 7 in 2023-09-23T20:54:15.135 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:54:14 smithi026 ceph-mon[179688]: failed to encode map e134 with expected crc 2023-09-23T20:54:15.145 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:54:15.160 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:54:15.161 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:54:15.192 INFO:teuthology.orchestra.run.smithi026.stdout:{ 2023-09-23T20:54:15.198 INFO:teuthology.orchestra.run.smithi026.stdout: "ops": [], 2023-09-23T20:54:15.198 INFO:teuthology.orchestra.run.smithi026.stdout: "complaint_time": 30, 2023-09-23T20:54:15.198 INFO:teuthology.orchestra.run.smithi026.stdout: "num_blocked_ops": 0 2023-09-23T20:54:15.198 INFO:teuthology.orchestra.run.smithi026.stdout:} 2023-09-23T20:54:15.262 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:54:15.287 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:54:14 smithi070 ceph-mon[168282]: failed to encode map e132 with expected crc 2023-09-23T20:54:15.287 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:54:14 smithi070 ceph-mon[168282]: failed to encode map e133 with expected crc 2023-09-23T20:54:15.287 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:54:14 smithi070 ceph-mon[168282]: failed to encode map e133 with expected crc 2023-09-23T20:54:15.287 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:54:14 smithi070 ceph-mon[168282]: failed to encode map e133 with expected crc 2023-09-23T20:54:15.287 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:54:14 smithi070 ceph-mon[168282]: osdmap e133: 8 total, 8 up, 7 in 2023-09-23T20:54:15.287 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:54:14 smithi070 ceph-mon[168282]: failed to encode map e133 with expected crc 2023-09-23T20:54:15.287 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:54:14 smithi070 ceph-mon[168282]: failed to encode map e133 with expected crc 2023-09-23T20:54:15.287 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:54:14 smithi070 ceph-mon[168282]: failed to encode map e133 with expected crc 2023-09-23T20:54:15.288 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:54:14 smithi070 ceph-mon[168282]: failed to encode map e133 with expected crc 2023-09-23T20:54:15.288 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:54:14 smithi070 ceph-mon[168282]: failed to encode map e133 with expected crc 2023-09-23T20:54:15.288 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:54:14 smithi070 ceph-mon[168282]: failed to encode map e133 with expected crc 2023-09-23T20:54:15.288 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:54:14 smithi070 ceph-mon[168282]: 6.9 scrub starts 2023-09-23T20:54:15.288 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:54:14 smithi070 ceph-mon[168282]: Health check update: Degraded data redundancy: 21948/268179 objects degraded (8.184%), 7 pgs degraded (PG_DEGRADED) 2023-09-23T20:54:15.288 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:54:14 smithi070 ceph-mon[168282]: osdmap e134: 8 total, 8 up, 7 in 2023-09-23T20:54:15.288 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:54:14 smithi070 ceph-mon[168282]: failed to encode map e134 with expected crc 2023-09-23T20:54:15.290 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:54:15.327 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:54:15.329 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:54:15.330 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:54:15.430 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:54:15.463 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:54:15.499 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0: {} 2023-09-23T20:54:15.504 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:54:15.507 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:54:15.508 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:54:15.539 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1: {} 2023-09-23T20:54:15.577 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2: {} 2023-09-23T20:54:15.608 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:54:15.621 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3: {} 2023-09-23T20:54:15.661 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4: {} 2023-09-23T20:54:15.686 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:54:15.714 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_historic_ops 2023-09-23T20:54:15.715 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T20:54:15.738 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6: {} 2023-09-23T20:54:15.768 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:54:15.774 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T20:54:15.776 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:54:15.776 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:54:15.787 INFO:teuthology.orchestra.run.smithi026.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T20:54:15.787 INFO:teuthology.orchestra.run.smithi026.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T20:54:15.787 INFO:teuthology.orchestra.run.smithi026.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T20:54:15.787 INFO:teuthology.orchestra.run.smithi026.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T20:54:15.787 INFO:teuthology.orchestra.run.smithi026.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T20:54:15.787 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T20:54:15.787 INFO:teuthology.orchestra.run.smithi026.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T20:54:15.787 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T20:54:15.833 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:54:15.877 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:54:15.905 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:54:15.944 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:54:15.948 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:54:15.948 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:54:16.049 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:54:16.134 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:54:15 smithi026 ceph-mon[175151]: pgmap v326: 99 pgs: 4 remapped+peering, 1 active+recovery_wait+degraded, 91 active+clean, 1 active+recovering, 2 active+recovery_wait+undersized+degraded+remapped; 5.4 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 878 op/s; 8543/263724 objects degraded (3.239%); 2045/263724 objects misplaced (0.775%); 14 MiB/s, 230 objects/s recovering 2023-09-23T20:54:16.134 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:54:15 smithi026 ceph-mon[175151]: failed to encode map e134 with expected crc 2023-09-23T20:54:16.134 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:54:15 smithi026 ceph-mon[175151]: failed to encode map e134 with expected crc 2023-09-23T20:54:16.134 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:54:15 smithi026 ceph-mon[175151]: failed to encode map e134 with expected crc 2023-09-23T20:54:16.135 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:54:15 smithi026 ceph-mon[175151]: failed to encode map e134 with expected crc 2023-09-23T20:54:16.135 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:54:15 smithi026 ceph-mon[175151]: 6.5 scrub starts 2023-09-23T20:54:16.135 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:54:15 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/2962203030' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:54:16.135 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:54:15 smithi026 ceph-mon[179688]: pgmap v326: 99 pgs: 4 remapped+peering, 1 active+recovery_wait+degraded, 91 active+clean, 1 active+recovering, 2 active+recovery_wait+undersized+degraded+remapped; 5.4 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 878 op/s; 8543/263724 objects degraded (3.239%); 2045/263724 objects misplaced (0.775%); 14 MiB/s, 230 objects/s recovering 2023-09-23T20:54:16.135 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:54:15 smithi026 ceph-mon[179688]: failed to encode map e134 with expected crc 2023-09-23T20:54:16.135 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:54:15 smithi026 ceph-mon[179688]: failed to encode map e134 with expected crc 2023-09-23T20:54:16.135 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:54:15 smithi026 ceph-mon[179688]: failed to encode map e134 with expected crc 2023-09-23T20:54:16.135 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:54:15 smithi026 ceph-mon[179688]: failed to encode map e134 with expected crc 2023-09-23T20:54:16.135 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:54:15 smithi026 ceph-mon[179688]: 6.5 scrub starts 2023-09-23T20:54:16.136 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:54:15 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/2962203030' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:54:16.146 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:54:16.292 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:54:16.305 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:54:16.305 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:54:16.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:54:15 smithi070 ceph-mon[168282]: pgmap v326: 99 pgs: 4 remapped+peering, 1 active+recovery_wait+degraded, 91 active+clean, 1 active+recovering, 2 active+recovery_wait+undersized+degraded+remapped; 5.4 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 878 op/s; 8543/263724 objects degraded (3.239%); 2045/263724 objects misplaced (0.775%); 14 MiB/s, 230 objects/s recovering 2023-09-23T20:54:16.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:54:15 smithi070 ceph-mon[168282]: failed to encode map e134 with expected crc 2023-09-23T20:54:16.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:54:15 smithi070 ceph-mon[168282]: failed to encode map e134 with expected crc 2023-09-23T20:54:16.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:54:15 smithi070 ceph-mon[168282]: failed to encode map e134 with expected crc 2023-09-23T20:54:16.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:54:15 smithi070 ceph-mon[168282]: failed to encode map e134 with expected crc 2023-09-23T20:54:16.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:54:15 smithi070 ceph-mon[168282]: 6.5 scrub starts 2023-09-23T20:54:16.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:54:15 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/2962203030' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:54:16.406 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:54:16.507 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:54:16.972 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:54:16.987 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:54:16.987 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:54:17.032 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:54:17 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:54:17.033 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:54:17 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:54:17.041 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:54:17.089 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:54:17.117 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:54:17.158 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:54:17.162 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:54:17.163 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:54:17.263 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:54:17.353 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:54:17.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:54:17 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:54:17.639 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:54:17.639 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:54:17.640 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:54:17.740 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:54:17.771 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:54:17.816 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:54:17.821 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:54:17.822 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:54:17.923 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:54:17.951 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:54:17.989 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:54:17.993 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:54:17.993 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:54:18.094 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:54:18.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:54:18 smithi070 ceph-mon[168282]: pgmap v328: 99 pgs: 1 active+recovering+undersized+remapped, 4 remapped+peering, 91 active+clean, 3 active+recovery_wait+undersized+degraded+remapped; 5.3 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 732 op/s; 3300/261546 objects degraded (1.262%); 478/261546 objects misplaced (0.183%); 14 MiB/s, 220 objects/s recovering 2023-09-23T20:54:18.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:54:18 smithi070 ceph-mon[168282]: 6.19 scrub starts 2023-09-23T20:54:18.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:54:18 smithi070 ceph-mon[168282]: 6.8 scrub starts 2023-09-23T20:54:18.126 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:54:18.167 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:54:18.170 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:54:18.170 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:54:18.271 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:54:18.313 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:54:18 smithi026 ceph-mon[175151]: pgmap v328: 99 pgs: 1 active+recovering+undersized+remapped, 4 remapped+peering, 91 active+clean, 3 active+recovery_wait+undersized+degraded+remapped; 5.3 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 732 op/s; 3300/261546 objects degraded (1.262%); 478/261546 objects misplaced (0.183%); 14 MiB/s, 220 objects/s recovering 2023-09-23T20:54:18.313 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:54:18 smithi026 ceph-mon[175151]: 6.19 scrub starts 2023-09-23T20:54:18.313 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:54:18 smithi026 ceph-mon[175151]: 6.8 scrub starts 2023-09-23T20:54:18.314 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:54:18 smithi026 ceph-mon[179688]: pgmap v328: 99 pgs: 1 active+recovering+undersized+remapped, 4 remapped+peering, 91 active+clean, 3 active+recovery_wait+undersized+degraded+remapped; 5.3 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 732 op/s; 3300/261546 objects degraded (1.262%); 478/261546 objects misplaced (0.183%); 14 MiB/s, 220 objects/s recovering 2023-09-23T20:54:18.314 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:54:18 smithi026 ceph-mon[179688]: 6.19 scrub starts 2023-09-23T20:54:18.314 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:54:18 smithi026 ceph-mon[179688]: 6.8 scrub starts 2023-09-23T20:54:18.335 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:54:18.342 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T20:54:18.398 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:54:18.410 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:54:18.411 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:54:18.511 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:54:18.540 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:54:18.578 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:54:18.580 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:54:18.581 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:54:18.659 INFO:teuthology.orchestra.run.smithi026.stdout:ERROR: (22) Invalid argument 2023-09-23T20:54:18.660 INFO:teuthology.orchestra.run.smithi026.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:54:18.681 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:54:18.710 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:54:18.746 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:54:18.749 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:54:18.749 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:54:18.850 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:54:18.887 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:54:18.930 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:54:18.937 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:54:18.937 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:54:19.038 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:54:19.039 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:54:19 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/1006994921' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:54:19.039 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:54:19 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:54:19.039 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:54:19 smithi026 ceph-mon[175151]: pgmap v329: 99 pgs: 1 active+recovering+undersized+remapped, 4 remapped+peering, 91 active+clean, 3 active+recovery_wait+undersized+degraded+remapped; 5.3 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 479 op/s; 3300/259542 objects degraded (1.271%); 478/259542 objects misplaced (0.184%); 9.9 MiB/s, 1 keys/s, 160 objects/s recovering 2023-09-23T20:54:19.050 INFO:teuthology.orchestra.run.smithi026.stderr:nodeep-scrub is unset 2023-09-23T20:54:19.076 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:54:19.121 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:54:19.124 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:54:19.124 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:54:19.136 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 6, 7, 4] out_osds: [5] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-09-23T20:54:19.136 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T20:54:19.136 INFO:tasks.thrashosds.thrasher:inject_pause on osd.6 2023-09-23T20:54:19.136 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-09-23T20:54:19.136 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-09-23T20:54:19.136 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 config set filestore_inject_stall 3 2023-09-23T20:54:19.189 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:54:19 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/1006994921' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:54:19.189 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:54:19 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:54:19.189 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:54:19 smithi070 ceph-mon[168282]: pgmap v329: 99 pgs: 1 active+recovering+undersized+remapped, 4 remapped+peering, 91 active+clean, 3 active+recovery_wait+undersized+degraded+remapped; 5.3 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 479 op/s; 3300/259542 objects degraded (1.271%); 478/259542 objects misplaced (0.184%); 9.9 MiB/s, 1 keys/s, 160 objects/s recovering 2023-09-23T20:54:19.225 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:54:19.258 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:54:19.311 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:54:19.315 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:54:19.316 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:54:19.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:54:19 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/1006994921' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:54:19.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:54:19 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:54:19.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:54:19 smithi026 ceph-mon[179688]: pgmap v329: 99 pgs: 1 active+recovering+undersized+remapped, 4 remapped+peering, 91 active+clean, 3 active+recovery_wait+undersized+degraded+remapped; 5.3 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 479 op/s; 3300/259542 objects degraded (1.271%); 478/259542 objects misplaced (0.184%); 9.9 MiB/s, 1 keys/s, 160 objects/s recovering 2023-09-23T20:54:19.416 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:54:19.461 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:54:19.511 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:54:19.517 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:54:19.517 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:54:19.618 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:54:19.672 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:54:19.714 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:54:19.718 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:54:19.719 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:54:19.772 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:54:19.820 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:54:19.856 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:54:19.883 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:54:19 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[164029]: ::ffff:172.21.15.70 - - [23/Sep/2023:20:54:19] "GET /metrics HTTP/1.1" 200 36335 "" "Prometheus/2.43.0" 2023-09-23T20:54:19.905 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:54:19.909 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:54:19.910 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:54:20.011 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:54:20.046 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:54:20.327 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:54:20 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[153130]: logger=cleanup t=2023-09-23T20:54:20.213047222Z level=info msg="Completed cleanup jobs" duration=12.715182ms 2023-09-23T20:54:20.327 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:54:20 smithi070 ceph-mon[168282]: 6.3 scrub starts 2023-09-23T20:54:20.327 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:54:20 smithi070 ceph-mon[168282]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T20:54:20.327 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:54:20 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T20:54:20.327 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:54:20 smithi070 ceph-mon[168282]: osdmap e135: 8 total, 8 up, 7 in 2023-09-23T20:54:20.328 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:54:20 smithi070 ceph-mon[168282]: failed to encode map e135 with expected crc 2023-09-23T20:54:20.328 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:54:20 smithi070 ceph-mon[168282]: failed to encode map e135 with expected crc 2023-09-23T20:54:20.328 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:54:20 smithi070 ceph-mon[168282]: failed to encode map e135 with expected crc 2023-09-23T20:54:20.328 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:54:20 smithi070 ceph-mon[168282]: failed to encode map e135 with expected crc 2023-09-23T20:54:20.328 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:54:20 smithi070 ceph-mon[168282]: failed to encode map e135 with expected crc 2023-09-23T20:54:20.328 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:54:20 smithi070 ceph-mon[168282]: failed to encode map e135 with expected crc 2023-09-23T20:54:20.328 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:54:20 smithi070 ceph-mon[168282]: 6.10 scrub starts 2023-09-23T20:54:20.328 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:54:20 smithi070 ceph-mon[168282]: Health check update: Degraded data redundancy: 3300/259542 objects degraded (1.271%), 3 pgs degraded (PG_DEGRADED) 2023-09-23T20:54:20.383 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:54:20 smithi026 ceph-mon[179688]: 6.3 scrub starts 2023-09-23T20:54:20.383 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:54:20 smithi026 ceph-mon[179688]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T20:54:20.383 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:54:20 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T20:54:20.383 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:54:20 smithi026 ceph-mon[179688]: osdmap e135: 8 total, 8 up, 7 in 2023-09-23T20:54:20.383 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:54:20 smithi026 ceph-mon[179688]: failed to encode map e135 with expected crc 2023-09-23T20:54:20.383 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:54:20 smithi026 ceph-mon[179688]: failed to encode map e135 with expected crc 2023-09-23T20:54:20.383 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:54:20 smithi026 ceph-mon[179688]: failed to encode map e135 with expected crc 2023-09-23T20:54:20.384 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:54:20 smithi026 ceph-mon[179688]: failed to encode map e135 with expected crc 2023-09-23T20:54:20.384 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:54:20 smithi026 ceph-mon[179688]: failed to encode map e135 with expected crc 2023-09-23T20:54:20.384 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:54:20 smithi026 ceph-mon[179688]: failed to encode map e135 with expected crc 2023-09-23T20:54:20.384 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:54:20 smithi026 ceph-mon[179688]: 6.10 scrub starts 2023-09-23T20:54:20.384 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:54:20 smithi026 ceph-mon[179688]: Health check update: Degraded data redundancy: 3300/259542 objects degraded (1.271%), 3 pgs degraded (PG_DEGRADED) 2023-09-23T20:54:20.384 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:54:20 smithi026 ceph-mon[175151]: 6.3 scrub starts 2023-09-23T20:54:20.384 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:54:20 smithi026 ceph-mon[175151]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T20:54:20.384 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:54:20 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T20:54:20.384 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:54:20 smithi026 ceph-mon[175151]: osdmap e135: 8 total, 8 up, 7 in 2023-09-23T20:54:20.385 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:54:20 smithi026 ceph-mon[175151]: failed to encode map e135 with expected crc 2023-09-23T20:54:20.385 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:54:20 smithi026 ceph-mon[175151]: failed to encode map e135 with expected crc 2023-09-23T20:54:20.385 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:54:20 smithi026 ceph-mon[175151]: failed to encode map e135 with expected crc 2023-09-23T20:54:20.385 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:54:20 smithi026 ceph-mon[175151]: failed to encode map e135 with expected crc 2023-09-23T20:54:20.385 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:54:20 smithi026 ceph-mon[175151]: failed to encode map e135 with expected crc 2023-09-23T20:54:20.385 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:54:20 smithi026 ceph-mon[175151]: failed to encode map e135 with expected crc 2023-09-23T20:54:20.385 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:54:20 smithi026 ceph-mon[175151]: 6.10 scrub starts 2023-09-23T20:54:20.385 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:54:20 smithi026 ceph-mon[175151]: Health check update: Degraded data redundancy: 3300/259542 objects degraded (1.271%), 3 pgs degraded (PG_DEGRADED) 2023-09-23T20:54:20.608 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:54:20.613 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:54:20.613 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:54:20.694 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_ops_in_flight 2023-09-23T20:54:20.714 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:54:20.846 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:54:21.057 INFO:teuthology.orchestra.run.smithi070.stdout:{ 2023-09-23T20:54:21.058 INFO:teuthology.orchestra.run.smithi070.stdout: "success": "filestore_inject_stall = '3' (not observed, change may require restart) " 2023-09-23T20:54:21.058 INFO:teuthology.orchestra.run.smithi070.stdout:} 2023-09-23T20:54:21.061 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:54:21 smithi026 ceph-mon[175151]: 6.18 scrub starts 2023-09-23T20:54:21.061 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:54:21 smithi026 ceph-mon[175151]: 6.0 scrub starts 2023-09-23T20:54:21.061 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:54:21 smithi026 ceph-mon[175151]: pgmap v331: 99 pgs: 1 active+recovering+undersized+remapped, 91 active+clean, 7 active+recovery_wait+undersized+degraded+remapped; 5.2 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 760 op/s; 6500/253749 objects degraded (2.562%); 478/253749 objects misplaced (0.188%); 9.5 MiB/s, 1 keys/s, 154 objects/s recovering 2023-09-23T20:54:21.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:54:21 smithi070 ceph-mon[168282]: 6.18 scrub starts 2023-09-23T20:54:21.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:54:21 smithi070 ceph-mon[168282]: 6.0 scrub starts 2023-09-23T20:54:21.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:54:21 smithi070 ceph-mon[168282]: pgmap v331: 99 pgs: 1 active+recovering+undersized+remapped, 91 active+clean, 7 active+recovery_wait+undersized+degraded+remapped; 5.2 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 760 op/s; 6500/253749 objects degraded (2.562%); 478/253749 objects misplaced (0.188%); 9.5 MiB/s, 1 keys/s, 154 objects/s recovering 2023-09-23T20:54:21.387 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:54:21 smithi026 ceph-mon[179688]: 6.18 scrub starts 2023-09-23T20:54:21.387 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:54:21 smithi026 ceph-mon[179688]: 6.0 scrub starts 2023-09-23T20:54:21.387 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:54:21 smithi026 ceph-mon[179688]: pgmap v331: 99 pgs: 1 active+recovering+undersized+remapped, 91 active+clean, 7 active+recovery_wait+undersized+degraded+remapped; 5.2 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 760 op/s; 6500/253749 objects degraded (2.562%); 478/253749 objects misplaced (0.188%); 9.5 MiB/s, 1 keys/s, 154 objects/s recovering 2023-09-23T20:54:22.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:54:22 smithi070 ceph-mon[168282]: 6.15 scrub starts 2023-09-23T20:54:22.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:54:22 smithi070 ceph-mon[168282]: 6.9 scrub starts 2023-09-23T20:54:22.381 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:54:22 smithi026 ceph-mon[175151]: 6.15 scrub starts 2023-09-23T20:54:22.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:54:22 smithi026 ceph-mon[175151]: 6.9 scrub starts 2023-09-23T20:54:22.384 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:54:22 smithi026 ceph-mon[179688]: 6.15 scrub starts 2023-09-23T20:54:22.384 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:54:22 smithi026 ceph-mon[179688]: 6.9 scrub starts 2023-09-23T20:54:22.701 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:54:22.711 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:54:22.711 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:54:22.813 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:54:22.851 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:54:22.892 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:54:22.895 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:54:22.896 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:54:22.942 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:54:22.985 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:54:22.997 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:54:23.087 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:54:23.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:54:23 smithi070 ceph-mon[168282]: pgmap v332: 99 pgs: 1 active+recovering+undersized+remapped, 91 active+clean, 7 active+recovery_wait+undersized+degraded+remapped; 5.0 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 734 op/s; 6500/246078 objects degraded (2.641%); 115/246078 objects misplaced (0.047%); 7.5 MiB/s, 1 keys/s, 121 objects/s recovering 2023-09-23T20:54:23.381 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:54:23 smithi026 ceph-mon[175151]: pgmap v332: 99 pgs: 1 active+recovering+undersized+remapped, 91 active+clean, 7 active+recovery_wait+undersized+degraded+remapped; 5.0 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 734 op/s; 6500/246078 objects degraded (2.641%); 115/246078 objects misplaced (0.047%); 7.5 MiB/s, 1 keys/s, 121 objects/s recovering 2023-09-23T20:54:23.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:54:23 smithi026 ceph-mon[179688]: pgmap v332: 99 pgs: 1 active+recovering+undersized+remapped, 91 active+clean, 7 active+recovery_wait+undersized+degraded+remapped; 5.0 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 734 op/s; 6500/246078 objects degraded (2.641%); 115/246078 objects misplaced (0.047%); 7.5 MiB/s, 1 keys/s, 121 objects/s recovering 2023-09-23T20:54:23.859 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:54:23.862 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:54:23.863 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:54:23.964 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:54:23.997 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:54:24.044 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:54:24.050 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:54:24.051 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:54:24.131 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:54:24 smithi026 ceph-mon[179688]: 6.13 scrub starts 2023-09-23T20:54:24.140 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:54:24 smithi026 ceph-mon[175151]: 6.13 scrub starts 2023-09-23T20:54:24.152 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:54:24.181 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:54:24.228 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:54:24.236 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:54:24.236 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:54:24.337 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:54:24.364 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:54:24 smithi070 ceph-mon[168282]: 6.13 scrub starts 2023-09-23T20:54:24.367 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:54:24.414 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:54:24.417 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:54:24.417 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:54:24.518 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:54:24.587 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:54:24.764 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T20:54:25.120 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:54:25.129 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:54:25.130 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:54:25.230 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:54:25.261 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:54:25.308 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:54:25.313 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:54:25.313 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:54:25.371 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:54:25 smithi070 ceph-mon[168282]: 6.5 scrub starts 2023-09-23T20:54:25.371 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:54:25 smithi070 ceph-mon[168282]: 6.19 scrub starts 2023-09-23T20:54:25.371 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:54:25 smithi070 ceph-mon[168282]: pgmap v333: 99 pgs: 1 active+recovering+undersized+remapped, 91 active+clean, 7 active+recovery_wait+undersized+degraded+remapped; 4.8 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 908 op/s; 6500/238287 objects degraded (2.728%); 115/238287 objects misplaced (0.048%); 6.4 MiB/s, 1 keys/s, 104 objects/s recovering 2023-09-23T20:54:25.371 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:54:25 smithi070 ceph-mon[168282]: Health check update: Degraded data redundancy: 6500/246078 objects degraded (2.641%), 7 pgs degraded (PG_DEGRADED) 2023-09-23T20:54:25.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:54:25 smithi026 ceph-mon[179688]: 6.5 scrub starts 2023-09-23T20:54:25.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:54:25 smithi026 ceph-mon[179688]: 6.19 scrub starts 2023-09-23T20:54:25.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:54:25 smithi026 ceph-mon[179688]: pgmap v333: 99 pgs: 1 active+recovering+undersized+remapped, 91 active+clean, 7 active+recovery_wait+undersized+degraded+remapped; 4.8 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 908 op/s; 6500/238287 objects degraded (2.728%); 115/238287 objects misplaced (0.048%); 6.4 MiB/s, 1 keys/s, 104 objects/s recovering 2023-09-23T20:54:25.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:54:25 smithi026 ceph-mon[179688]: Health check update: Degraded data redundancy: 6500/246078 objects degraded (2.641%), 7 pgs degraded (PG_DEGRADED) 2023-09-23T20:54:25.384 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:54:25 smithi026 ceph-mon[175151]: 6.5 scrub starts 2023-09-23T20:54:25.385 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:54:25 smithi026 ceph-mon[175151]: 6.19 scrub starts 2023-09-23T20:54:25.385 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:54:25 smithi026 ceph-mon[175151]: pgmap v333: 99 pgs: 1 active+recovering+undersized+remapped, 91 active+clean, 7 active+recovery_wait+undersized+degraded+remapped; 4.8 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 908 op/s; 6500/238287 objects degraded (2.728%); 115/238287 objects misplaced (0.048%); 6.4 MiB/s, 1 keys/s, 104 objects/s recovering 2023-09-23T20:54:25.385 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:54:25 smithi026 ceph-mon[175151]: Health check update: Degraded data redundancy: 6500/246078 objects degraded (2.641%), 7 pgs degraded (PG_DEGRADED) 2023-09-23T20:54:25.414 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:54:25.450 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:54:25.499 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:54:25.502 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:54:25.502 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:54:25.603 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:54:25.643 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:54:25.686 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:54:25.688 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:54:25.689 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:54:25.790 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:54:25.910 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:54:26.080 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:54:26.086 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:54:26.087 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:54:26.187 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:54:26.255 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:54:26.341 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:54:26 smithi026 ceph-mon[179688]: 6.10 scrub starts 2023-09-23T20:54:26.342 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:54:26 smithi026 ceph-mon[175151]: 6.10 scrub starts 2023-09-23T20:54:26.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:54:26 smithi070 ceph-mon[168282]: 6.10 scrub starts 2023-09-23T20:54:26.473 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 6, 7, 4] out_osds: [5] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-09-23T20:54:26.473 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T20:54:26.473 INFO:tasks.thrashosds.thrasher:Adding osd 5 2023-09-23T20:54:26.473 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph osd in 5 2023-09-23T20:54:26.537 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:54:26.550 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:54:26.551 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:54:26.652 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:54:26.684 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:54:26.722 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:54:26.726 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:54:26.727 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:54:26.827 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:54:26.863 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:54:26.902 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:54:26.909 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:54:26.910 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:54:27.010 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:54:27.116 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:54:27.127 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:54:27.199 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:54:27.211 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:54:27.212 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:54:27.213 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0: {} 2023-09-23T20:54:27.256 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1: {} 2023-09-23T20:54:27.312 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:54:27.325 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2: {} 2023-09-23T20:54:27.344 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:54:27.366 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3: {} 2023-09-23T20:54:27.381 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:54:27.384 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:54:27 smithi026 ceph-mon[175151]: 6.8 scrub starts 2023-09-23T20:54:27.384 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:54:27 smithi026 ceph-mon[175151]: pgmap v334: 99 pgs: 1 active+recovering+undersized+remapped, 91 active+clean, 7 active+recovery_wait+undersized+degraded+remapped; 4.8 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 842 op/s; 6500/236250 objects degraded (2.751%); 115/236250 objects misplaced (0.049%); 2.3 MiB/s, 0 keys/s, 37 objects/s recovering 2023-09-23T20:54:27.385 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:54:27 smithi026 ceph-mon[179688]: 6.8 scrub starts 2023-09-23T20:54:27.385 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:54:27 smithi026 ceph-mon[179688]: pgmap v334: 99 pgs: 1 active+recovering+undersized+remapped, 91 active+clean, 7 active+recovery_wait+undersized+degraded+remapped; 4.8 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 842 op/s; 6500/236250 objects degraded (2.751%); 115/236250 objects misplaced (0.049%); 2.3 MiB/s, 0 keys/s, 37 objects/s recovering 2023-09-23T20:54:27.385 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:54:27 smithi070 ceph-mon[168282]: 6.8 scrub starts 2023-09-23T20:54:27.385 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:54:27 smithi070 ceph-mon[168282]: pgmap v334: 99 pgs: 1 active+recovering+undersized+remapped, 91 active+clean, 7 active+recovery_wait+undersized+degraded+remapped; 4.8 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 842 op/s; 6500/236250 objects degraded (2.751%); 115/236250 objects misplaced (0.049%); 2.3 MiB/s, 0 keys/s, 37 objects/s recovering 2023-09-23T20:54:27.386 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:54:27.386 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:54:27.404 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4: {} 2023-09-23T20:54:27.461 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T20:54:27.472 INFO:teuthology.orchestra.run.smithi026.stdout:{ 2023-09-23T20:54:27.472 INFO:teuthology.orchestra.run.smithi026.stdout: "ops": [ 2023-09-23T20:54:27.472 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:54:27.472 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_repop(client.34797.0:127481 6.e e135/132 6:7a2906d1:::benchmark_data_smithi026_223868_object26355:head v 135'8123, mlcod=123'5652)", 2023-09-23T20:54:27.472 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T20:54:27.468121+0000", 2023-09-23T20:54:27.473 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 0.0028771420000000001, 2023-09-23T20:54:27.473 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.002906605, 2023-09-23T20:54:27.473 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T20:54:27.473 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "started", 2023-09-23T20:54:27.473 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T20:54:27.473 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:54:27.473 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T20:54:27.473 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:27.468121+0000", 2023-09-23T20:54:27.473 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:54:27.473 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:54:27.473 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:54:27.473 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T20:54:27.473 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:27.468121+0000", 2023-09-23T20:54:27.474 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:54:27.474 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:54:27.474 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:54:27.474 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T20:54:27.474 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:27.468120+0000", 2023-09-23T20:54:27.474 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.999999 2023-09-23T20:54:27.474 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:54:27.474 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:54:27.474 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T20:54:27.474 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:27.468130+0000", 2023-09-23T20:54:27.474 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.3619999999999995e-06 2023-09-23T20:54:27.474 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:54:27.475 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:54:27.475 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T20:54:27.475 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:27.468142+0000", 2023-09-23T20:54:27.475 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.186e-05 2023-09-23T20:54:27.475 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:54:27.475 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:54:27.475 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T20:54:27.475 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:27.468149+0000", 2023-09-23T20:54:27.475 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.6000000000000001e-06 2023-09-23T20:54:27.475 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:54:27.475 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:54:27.475 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:54:27.475 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:27.468248+0000", 2023-09-23T20:54:27.476 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.8715999999999995e-05 2023-09-23T20:54:27.476 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:54:27.476 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:54:27.476 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T20:54:27.476 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:27.468275+0000", 2023-09-23T20:54:27.476 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.7124999999999999e-05 2023-09-23T20:54:27.476 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:54:27.476 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T20:54:27.476 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:54:27.476 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:54:27.476 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:54:27.476 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_repop(client.34797.0:127487 6.1 e135/128 6:83e9da06:::benchmark_data_smithi026_223868_object26361:head v 135'7421, mlcod=135'7418)", 2023-09-23T20:54:27.476 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T20:54:27.468165+0000", 2023-09-23T20:54:27.477 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 0.0028335019999999999, 2023-09-23T20:54:27.477 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0029366869999999999, 2023-09-23T20:54:27.477 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T20:54:27.477 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "started", 2023-09-23T20:54:27.477 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T20:54:27.477 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:54:27.477 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T20:54:27.477 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:27.468165+0000", 2023-09-23T20:54:27.477 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:54:27.477 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:54:27.477 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:54:27.477 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T20:54:27.477 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:27.468165+0000", 2023-09-23T20:54:27.478 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:54:27.478 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:54:27.478 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:54:27.478 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T20:54:27.478 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:27.468165+0000", 2023-09-23T20:54:27.478 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T20:54:27.478 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:54:27.478 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:54:27.478 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T20:54:27.478 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:27.468169+0000", 2023-09-23T20:54:27.478 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.0330000000000002e-06 2023-09-23T20:54:27.478 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:54:27.478 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:54:27.479 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T20:54:27.479 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:27.468175+0000", 2023-09-23T20:54:27.479 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.1800000000000001e-06 2023-09-23T20:54:27.479 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:54:27.479 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:54:27.479 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T20:54:27.479 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:27.468179+0000", 2023-09-23T20:54:27.479 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.5959999999999997e-06 2023-09-23T20:54:27.479 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:54:27.479 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:54:27.479 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:54:27.479 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:27.470819+0000", 2023-09-23T20:54:27.480 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.002639195 2023-09-23T20:54:27.480 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:54:27.480 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:54:27.480 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T20:54:27.480 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:27.470841+0000", 2023-09-23T20:54:27.480 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.2150999999999998e-05 2023-09-23T20:54:27.480 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:54:27.480 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T20:54:27.480 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:54:27.480 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:54:27.480 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:54:27.481 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_repop(client.34797.0:127486 6.e e135/132)", 2023-09-23T20:54:27.481 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T20:54:27.470597+0000", 2023-09-23T20:54:27.481 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 0.000401703, 2023-09-23T20:54:27.481 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00056716100000000005, 2023-09-23T20:54:27.481 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T20:54:27.481 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "queued for pg", 2023-09-23T20:54:27.481 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T20:54:27.481 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:54:27.482 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T20:54:27.482 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:27.470597+0000", 2023-09-23T20:54:27.482 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:54:27.482 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:54:27.482 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:54:27.482 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T20:54:27.482 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:27.470597+0000", 2023-09-23T20:54:27.483 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:54:27.483 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:54:27.483 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:54:27.483 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T20:54:27.483 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:27.470596+0000", 2023-09-23T20:54:27.483 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.999999 2023-09-23T20:54:27.484 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:54:27.484 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:54:27.484 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T20:54:27.484 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:27.470606+0000", 2023-09-23T20:54:27.484 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.8900000000000002e-06 2023-09-23T20:54:27.484 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:54:27.485 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:54:27.485 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T20:54:27.485 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:27.470626+0000", 2023-09-23T20:54:27.485 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.9831e-05 2023-09-23T20:54:27.485 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:54:27.485 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:54:27.486 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T20:54:27.486 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:27.470647+0000", 2023-09-23T20:54:27.486 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.1605000000000001e-05 2023-09-23T20:54:27.486 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:54:27.486 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T20:54:27.486 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:54:27.486 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:54:27.487 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:54:27.487 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34797.0:127471 6.f 6:f570ce77:::benchmark_data_smithi026_223868_object26345:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e135)", 2023-09-23T20:54:27.487 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T20:54:27.450305+0000", 2023-09-23T20:54:27.487 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 0.020693525000000001, 2023-09-23T20:54:27.487 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.020915313000000001, 2023-09-23T20:54:27.487 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T20:54:27.487 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "waiting for sub ops", 2023-09-23T20:54:27.487 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T20:54:27.488 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34797", 2023-09-23T20:54:27.488 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T20:54:27.488 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 127471 2023-09-23T20:54:27.488 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:54:27.488 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T20:54:27.488 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:54:27.488 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T20:54:27.489 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:27.450305+0000", 2023-09-23T20:54:27.489 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:54:27.489 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:54:27.489 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:54:27.489 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T20:54:27.489 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:27.450305+0000", 2023-09-23T20:54:27.490 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:54:27.490 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:54:27.490 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:54:27.490 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T20:54:27.490 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:27.450303+0000", 2023-09-23T20:54:27.490 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999981 2023-09-23T20:54:27.491 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:54:27.491 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:54:27.491 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T20:54:27.491 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:27.450311+0000", 2023-09-23T20:54:27.491 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.6510000000000001e-06 2023-09-23T20:54:27.491 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:54:27.491 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:54:27.491 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T20:54:27.491 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:27.450325+0000", 2023-09-23T20:54:27.491 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.4263999999999999e-05 2023-09-23T20:54:27.491 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:54:27.491 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:54:27.492 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T20:54:27.492 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:27.450334+0000", 2023-09-23T20:54:27.492 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.6969999999999999e-06 2023-09-23T20:54:27.492 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:54:27.492 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:54:27.492 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:54:27.492 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:27.450517+0000", 2023-09-23T20:54:27.492 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00018316599999999999 2023-09-23T20:54:27.492 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:54:27.492 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:54:27.492 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T20:54:27.492 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:27.450656+0000", 2023-09-23T20:54:27.492 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00013925 2023-09-23T20:54:27.493 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:54:27.493 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:54:27.493 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 3,4", 2023-09-23T20:54:27.493 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:27.450788+0000", 2023-09-23T20:54:27.493 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00013180399999999999 2023-09-23T20:54:27.493 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:54:27.493 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:54:27.493 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:54:27.493 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:27.468739+0000", 2023-09-23T20:54:27.493 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.017951359 2023-09-23T20:54:27.493 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:54:27.493 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:54:27.494 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T20:54:27.494 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:27.468747+0000", 2023-09-23T20:54:27.494 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.9510000000000006e-06 2023-09-23T20:54:27.494 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:54:27.494 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T20:54:27.494 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:54:27.494 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:54:27.494 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:54:27.494 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34797.0:127474 6.f 6:f9decd08:::benchmark_data_smithi026_223868_object26348:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e135)", 2023-09-23T20:54:27.494 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T20:54:27.450351+0000", 2023-09-23T20:54:27.494 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 0.020647454999999999, 2023-09-23T20:54:27.495 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.020922762000000001, 2023-09-23T20:54:27.495 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T20:54:27.495 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "waiting for sub ops", 2023-09-23T20:54:27.495 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T20:54:27.495 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34797", 2023-09-23T20:54:27.495 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T20:54:27.495 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 127474 2023-09-23T20:54:27.495 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:54:27.495 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T20:54:27.495 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:54:27.495 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T20:54:27.495 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:27.450351+0000", 2023-09-23T20:54:27.496 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:54:27.496 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:54:27.496 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:54:27.496 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T20:54:27.496 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:27.450351+0000", 2023-09-23T20:54:27.496 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:54:27.496 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:54:27.496 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:54:27.496 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T20:54:27.496 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:27.450350+0000", 2023-09-23T20:54:27.496 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T20:54:27.496 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:54:27.497 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:54:27.497 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T20:54:27.497 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:27.450354+0000", 2023-09-23T20:54:27.497 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.1300000000000001e-06 2023-09-23T20:54:27.497 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:54:27.497 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:54:27.497 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T20:54:27.497 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:27.450361+0000", 2023-09-23T20:54:27.497 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.5519999999999999e-06 2023-09-23T20:54:27.497 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:54:27.497 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:54:27.497 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T20:54:27.498 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:27.450367+0000", 2023-09-23T20:54:27.498 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.5670000000000002e-06 2023-09-23T20:54:27.498 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:54:27.498 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:54:27.498 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:54:27.498 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:27.455138+0000", 2023-09-23T20:54:27.498 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0047717749999999998 2023-09-23T20:54:27.498 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:54:27.498 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:54:27.498 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T20:54:27.498 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:27.455274+0000", 2023-09-23T20:54:27.498 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00013522799999999999 2023-09-23T20:54:27.499 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:54:27.499 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:54:27.499 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 3,4", 2023-09-23T20:54:27.499 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:27.455403+0000", 2023-09-23T20:54:27.499 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00012939700000000001 2023-09-23T20:54:27.499 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:54:27.499 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:54:27.499 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T20:54:27.499 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:27.468753+0000", 2023-09-23T20:54:27.499 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.013349849 2023-09-23T20:54:27.499 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:54:27.499 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:54:27.499 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:54:27.500 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:27.469107+0000", 2023-09-23T20:54:27.500 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00035374199999999999 2023-09-23T20:54:27.500 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:54:27.500 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T20:54:27.500 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:54:27.500 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:54:27.500 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:54:27.500 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34797.0:127475 6.12 6:4e389c4d:::benchmark_data_smithi026_223868_object26349:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e135)", 2023-09-23T20:54:27.500 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T20:54:27.450372+0000", 2023-09-23T20:54:27.500 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 0.020626743999999999, 2023-09-23T20:54:27.500 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.020959299000000001, 2023-09-23T20:54:27.500 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T20:54:27.500 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "waiting for sub ops", 2023-09-23T20:54:27.501 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T20:54:27.501 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34797", 2023-09-23T20:54:27.501 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T20:54:27.501 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 127475 2023-09-23T20:54:27.501 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:54:27.501 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T20:54:27.501 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:54:27.501 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T20:54:27.501 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:27.450372+0000", 2023-09-23T20:54:27.501 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:54:27.501 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:54:27.501 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:54:27.502 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T20:54:27.502 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:27.450372+0000", 2023-09-23T20:54:27.502 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:54:27.502 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:54:27.502 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:54:27.502 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T20:54:27.502 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:27.450371+0000", 2023-09-23T20:54:27.502 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T20:54:27.502 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:54:27.502 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:54:27.502 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T20:54:27.502 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:27.450374+0000", 2023-09-23T20:54:27.503 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.413e-06 2023-09-23T20:54:27.503 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:54:27.503 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:54:27.503 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T20:54:27.503 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:27.450380+0000", 2023-09-23T20:54:27.503 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.3690000000000003e-06 2023-09-23T20:54:27.503 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:54:27.503 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:54:27.503 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T20:54:27.503 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:27.450386+0000", 2023-09-23T20:54:27.503 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.6080000000000003e-06 2023-09-23T20:54:27.503 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:54:27.503 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:54:27.504 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:54:27.504 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:27.450438+0000", 2023-09-23T20:54:27.504 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.2577999999999997e-05 2023-09-23T20:54:27.504 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:54:27.504 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:54:27.504 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T20:54:27.504 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:27.450596+0000", 2023-09-23T20:54:27.504 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000158052 2023-09-23T20:54:27.504 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:54:27.504 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:54:27.504 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 3,4", 2023-09-23T20:54:27.504 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:27.450730+0000", 2023-09-23T20:54:27.505 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00013396400000000001 2023-09-23T20:54:27.505 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:54:27.505 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:54:27.505 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T20:54:27.505 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:27.455229+0000", 2023-09-23T20:54:27.505 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0044984029999999998 2023-09-23T20:54:27.505 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:54:27.505 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:54:27.505 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:54:27.505 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:27.455899+0000", 2023-09-23T20:54:27.505 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00067040800000000005 2023-09-23T20:54:27.505 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:54:27.505 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T20:54:27.506 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:54:27.506 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:54:27.506 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:54:27.506 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34797.0:127484 6.f 6:f8ec66fe:::benchmark_data_smithi026_223868_object26358:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e135)", 2023-09-23T20:54:27.506 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T20:54:27.458178+0000", 2023-09-23T20:54:27.506 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 0.012820450000000001, 2023-09-23T20:54:27.506 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.013214218, 2023-09-23T20:54:27.506 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T20:54:27.506 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "waiting for sub ops", 2023-09-23T20:54:27.506 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T20:54:27.506 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34797", 2023-09-23T20:54:27.506 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T20:54:27.507 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 127484 2023-09-23T20:54:27.507 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:54:27.507 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T20:54:27.507 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:54:27.507 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T20:54:27.507 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:27.458178+0000", 2023-09-23T20:54:27.507 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:54:27.507 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:54:27.507 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:54:27.507 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T20:54:27.507 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:27.458178+0000", 2023-09-23T20:54:27.507 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:54:27.507 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:54:27.508 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:54:27.508 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T20:54:27.508 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:27.458176+0000", 2023-09-23T20:54:27.508 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T20:54:27.508 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:54:27.508 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:54:27.508 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T20:54:27.508 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:27.458184+0000", 2023-09-23T20:54:27.508 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.515e-06 2023-09-23T20:54:27.508 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:54:27.508 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:54:27.508 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T20:54:27.509 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:27.458203+0000", 2023-09-23T20:54:27.509 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.8909000000000001e-05 2023-09-23T20:54:27.509 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:54:27.509 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:54:27.509 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T20:54:27.509 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:27.458212+0000", 2023-09-23T20:54:27.509 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.003e-06 2023-09-23T20:54:27.509 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:54:27.509 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:54:27.510 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:54:27.510 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:27.464486+0000", 2023-09-23T20:54:27.510 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.006274272 2023-09-23T20:54:27.510 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:54:27.510 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:54:27.510 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T20:54:27.510 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:27.464626+0000", 2023-09-23T20:54:27.510 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00014043399999999999 2023-09-23T20:54:27.510 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:54:27.510 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:54:27.510 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 3,4", 2023-09-23T20:54:27.510 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:27.464759+0000", 2023-09-23T20:54:27.510 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00013231500000000001 2023-09-23T20:54:27.511 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:54:27.511 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:54:27.511 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T20:54:27.511 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:27.470398+0000", 2023-09-23T20:54:27.511 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.005638784 2023-09-23T20:54:27.511 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:54:27.511 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:54:27.511 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:54:27.511 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:27.470880+0000", 2023-09-23T20:54:27.511 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00048198099999999998 2023-09-23T20:54:27.511 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:54:27.511 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T20:54:27.512 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:54:27.512 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:54:27.512 INFO:teuthology.orchestra.run.smithi026.stdout: ], 2023-09-23T20:54:27.512 INFO:teuthology.orchestra.run.smithi026.stdout: "num_ops": 7 2023-09-23T20:54:27.512 INFO:teuthology.orchestra.run.smithi026.stdout:} 2023-09-23T20:54:27.512 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:54:27.516 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6: {} 2023-09-23T20:54:27.516 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:54:27.557 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T20:54:27.585 INFO:teuthology.orchestra.run.smithi026.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T20:54:27.585 INFO:teuthology.orchestra.run.smithi026.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T20:54:27.585 INFO:teuthology.orchestra.run.smithi026.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T20:54:27.585 INFO:teuthology.orchestra.run.smithi026.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T20:54:27.585 INFO:teuthology.orchestra.run.smithi026.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T20:54:27.585 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T20:54:27.585 INFO:teuthology.orchestra.run.smithi026.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T20:54:27.585 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T20:54:27.634 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:54:27.914 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_blocked_ops 2023-09-23T20:54:27.921 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:54:27.930 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:54:27.931 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:54:28.032 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:54:28.142 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:54:28.384 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:54:28.389 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:54:28.389 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:54:28 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/3057426062' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:54:28.389 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:54:28 smithi026 ceph-mon[175151]: 6.0 scrub starts 2023-09-23T20:54:28.389 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:54:28 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/3057426062' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:54:28.389 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:54:28 smithi026 ceph-mon[179688]: 6.0 scrub starts 2023-09-23T20:54:28.390 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:54:28.491 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:54:28.518 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:54:28 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/3057426062' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:54:28.518 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:54:28 smithi070 ceph-mon[168282]: 6.0 scrub starts 2023-09-23T20:54:28.521 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:54:28.559 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:54:28.563 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:54:28.563 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:54:28.664 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:54:28.754 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:54:28.815 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:54:28.822 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:54:28.822 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:54:28.923 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:54:28.998 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:54:29.208 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:54:29.216 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:54:29.217 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:54:29.318 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:54:29.399 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:54:29 smithi026 ceph-mon[179688]: pgmap v335: 99 pgs: 1 active+recovering+undersized+remapped, 91 active+clean, 7 active+recovery_wait+undersized+degraded+remapped; 4.7 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 975 op/s; 6500/230268 objects degraded (2.823%); 3.0 MiB/s, 81 objects/s recovering 2023-09-23T20:54:29.399 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:54:29 smithi026 ceph-mon[175151]: pgmap v335: 99 pgs: 1 active+recovering+undersized+remapped, 91 active+clean, 7 active+recovery_wait+undersized+degraded+remapped; 4.7 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 975 op/s; 6500/230268 objects degraded (2.823%); 3.0 MiB/s, 81 objects/s recovering 2023-09-23T20:54:29.401 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:54:29.506 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:54:29.518 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:54:29.519 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:54:29.598 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:54:29.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:54:29 smithi070 ceph-mon[168282]: pgmap v335: 99 pgs: 1 active+recovering+undersized+remapped, 91 active+clean, 7 active+recovery_wait+undersized+degraded+remapped; 4.7 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 975 op/s; 6500/230268 objects degraded (2.823%); 3.0 MiB/s, 81 objects/s recovering 2023-09-23T20:54:29.619 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:54:29.693 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:54:29 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[164029]: ::ffff:172.21.15.70 - - [23/Sep/2023:20:54:29] "GET /metrics HTTP/1.1" 200 36335 "" "Prometheus/2.43.0" 2023-09-23T20:54:29.705 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:54:29.842 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:54:29.852 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:54:29.853 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:54:29.954 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:54:30.026 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:54:30.109 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:54:30.114 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:54:30.114 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:54:30.215 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:54:30.306 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:54:30 smithi026 ceph-mon[175151]: 6.9 scrub starts 2023-09-23T20:54:30.306 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:54:30 smithi026 ceph-mon[175151]: Health check update: Degraded data redundancy: 6500/230268 objects degraded (2.823%), 7 pgs degraded (PG_DEGRADED) 2023-09-23T20:54:30.306 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:54:30 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/2164638975' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:54:30.306 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:54:30 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:54:30.306 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:54:30 smithi026 ceph-mon[179688]: 6.9 scrub starts 2023-09-23T20:54:30.307 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:54:30 smithi026 ceph-mon[179688]: Health check update: Degraded data redundancy: 6500/230268 objects degraded (2.823%), 7 pgs degraded (PG_DEGRADED) 2023-09-23T20:54:30.307 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:54:30 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/2164638975' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:54:30.307 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:54:30 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:54:30.319 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:54:30.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:54:30 smithi070 ceph-mon[168282]: 6.9 scrub starts 2023-09-23T20:54:30.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:54:30 smithi070 ceph-mon[168282]: Health check update: Degraded data redundancy: 6500/230268 objects degraded (2.823%), 7 pgs degraded (PG_DEGRADED) 2023-09-23T20:54:30.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:54:30 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/2164638975' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:54:30.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:54:30 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:54:30.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:54:30 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mon-a[175127]: 2023-09-23T20:54:30.494+0000 7f1e2d487700 -1 mon.a@0(leader).osd e135 definitely_dead 0 2023-09-23T20:54:30.651 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:54:30.661 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:54:30.661 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:54:30.762 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:54:30.796 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:54:30.840 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:54:30.845 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:54:30.845 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:54:30.905 INFO:teuthology.orchestra.run.smithi026.stderr:noscrub is set 2023-09-23T20:54:30.905 INFO:teuthology.orchestra.run.smithi026.stderr:marked in osd.5. 2023-09-23T20:54:30.946 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:54:30.978 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T20:54:30.983 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:54:31.086 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:54:31.089 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:54:31.089 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:54:31.190 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:54:31.216 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:54:31 smithi026 ceph-mon[175151]: 6.3 scrub starts 2023-09-23T20:54:31.216 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:54:31 smithi026 ceph-mon[175151]: pgmap v336: 99 pgs: 1 active+recovering+undersized+remapped, 91 active+clean, 7 active+recovery_wait+undersized+degraded+remapped; 4.5 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 1.07k op/s; 6500/223311 objects degraded (2.911%); 2.7 MiB/s, 72 objects/s recovering 2023-09-23T20:54:31.216 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:54:31 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/2955432326' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["5"]}]: dispatch 2023-09-23T20:54:31.216 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:54:31 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["5"]}]: dispatch 2023-09-23T20:54:31.216 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:54:31 smithi026 ceph-mon[175151]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:54:31.216 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:54:31 smithi026 ceph-mon[175151]: failed to encode map e136 with expected crc 2023-09-23T20:54:31.217 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:54:31 smithi026 ceph-mon[175151]: failed to encode map e136 with expected crc 2023-09-23T20:54:31.217 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:54:31 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T20:54:31.217 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:54:31 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["5"]}]': finished 2023-09-23T20:54:31.217 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:54:31 smithi026 ceph-mon[175151]: osdmap e136: 8 total, 8 up, 8 in 2023-09-23T20:54:31.217 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:54:31 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:54:31.217 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:54:31 smithi026 ceph-mon[179688]: 6.3 scrub starts 2023-09-23T20:54:31.217 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:54:31 smithi026 ceph-mon[179688]: pgmap v336: 99 pgs: 1 active+recovering+undersized+remapped, 91 active+clean, 7 active+recovery_wait+undersized+degraded+remapped; 4.5 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 1.07k op/s; 6500/223311 objects degraded (2.911%); 2.7 MiB/s, 72 objects/s recovering 2023-09-23T20:54:31.217 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:54:31 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/2955432326' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["5"]}]: dispatch 2023-09-23T20:54:31.217 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:54:31 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["5"]}]: dispatch 2023-09-23T20:54:31.217 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:54:31 smithi026 ceph-mon[179688]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:54:31.218 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:54:31 smithi026 ceph-mon[179688]: failed to encode map e136 with expected crc 2023-09-23T20:54:31.218 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:54:31 smithi026 ceph-mon[179688]: failed to encode map e136 with expected crc 2023-09-23T20:54:31.218 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:54:31 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T20:54:31.218 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:54:31 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["5"]}]': finished 2023-09-23T20:54:31.218 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:54:31 smithi026 ceph-mon[179688]: osdmap e136: 8 total, 8 up, 8 in 2023-09-23T20:54:31.218 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:54:31 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:54:31.218 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:54:31.249 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:54:31.252 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:54:31.252 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:54:31.353 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:54:31.385 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:54:31.420 INFO:teuthology.orchestra.run.smithi026.stdout:{ 2023-09-23T20:54:31.420 INFO:teuthology.orchestra.run.smithi026.stdout: "ops": [], 2023-09-23T20:54:31.420 INFO:teuthology.orchestra.run.smithi026.stdout: "complaint_time": 30, 2023-09-23T20:54:31.421 INFO:teuthology.orchestra.run.smithi026.stdout: "num_blocked_ops": 0 2023-09-23T20:54:31.421 INFO:teuthology.orchestra.run.smithi026.stdout:} 2023-09-23T20:54:31.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:54:31 smithi070 ceph-mon[168282]: 6.3 scrub starts 2023-09-23T20:54:31.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:54:31 smithi070 ceph-mon[168282]: pgmap v336: 99 pgs: 1 active+recovering+undersized+remapped, 91 active+clean, 7 active+recovery_wait+undersized+degraded+remapped; 4.5 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 1.07k op/s; 6500/223311 objects degraded (2.911%); 2.7 MiB/s, 72 objects/s recovering 2023-09-23T20:54:31.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:54:31 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/2955432326' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["5"]}]: dispatch 2023-09-23T20:54:31.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:54:31 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["5"]}]: dispatch 2023-09-23T20:54:31.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:54:31 smithi070 ceph-mon[168282]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:54:31.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:54:31 smithi070 ceph-mon[168282]: failed to encode map e136 with expected crc 2023-09-23T20:54:31.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:54:31 smithi070 ceph-mon[168282]: failed to encode map e136 with expected crc 2023-09-23T20:54:31.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:54:31 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T20:54:31.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:54:31 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["5"]}]': finished 2023-09-23T20:54:31.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:54:31 smithi070 ceph-mon[168282]: osdmap e136: 8 total, 8 up, 8 in 2023-09-23T20:54:31.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:54:31 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:54:31.921 INFO:tasks.thrashosds.thrasher:Added osd 5 2023-09-23T20:54:31.921 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:54:31.923 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:54:31.923 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:54:31.964 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_historic_ops 2023-09-23T20:54:32.024 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:54:32.050 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:54:32.082 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:54:32.085 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:54:32.085 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:54:32.114 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:54:32.187 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:54:32.214 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:54:32.255 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:54:32.257 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:54:32.258 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:54:32.358 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:54:32.394 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:54:32.433 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:54:32.435 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:54:32.436 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:54:32.537 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:54:32.562 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:54:32.594 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:54:32.596 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:54:32.597 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:54:32.698 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:54:32.727 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:54:32.878 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:54:32 smithi026 ceph-mon[179688]: failed to encode map e137 with expected crc 2023-09-23T20:54:32.879 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:54:32 smithi026 ceph-mon[179688]: osdmap e137: 8 total, 8 up, 8 in 2023-09-23T20:54:32.879 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:54:32 smithi026 ceph-mon[179688]: failed to encode map e137 with expected crc 2023-09-23T20:54:32.879 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:54:32 smithi026 ceph-mon[179688]: failed to encode map e137 with expected crc 2023-09-23T20:54:32.879 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:54:32 smithi026 ceph-mon[179688]: failed to encode map e137 with expected crc 2023-09-23T20:54:32.879 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:54:32 smithi026 ceph-mon[179688]: failed to encode map e137 with expected crc 2023-09-23T20:54:32.879 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:54:32 smithi026 ceph-mon[179688]: failed to encode map e137 with expected crc 2023-09-23T20:54:32.879 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:54:32 smithi026 ceph-mon[179688]: failed to encode map e137 with expected crc 2023-09-23T20:54:32.879 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:54:32 smithi026 ceph-mon[175151]: failed to encode map e137 with expected crc 2023-09-23T20:54:32.879 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:54:32 smithi026 ceph-mon[175151]: osdmap e137: 8 total, 8 up, 8 in 2023-09-23T20:54:32.879 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:54:32 smithi026 ceph-mon[175151]: failed to encode map e137 with expected crc 2023-09-23T20:54:32.880 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:54:32 smithi026 ceph-mon[175151]: failed to encode map e137 with expected crc 2023-09-23T20:54:32.880 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:54:32 smithi026 ceph-mon[175151]: failed to encode map e137 with expected crc 2023-09-23T20:54:32.880 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:54:32 smithi026 ceph-mon[175151]: failed to encode map e137 with expected crc 2023-09-23T20:54:32.880 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:54:32 smithi026 ceph-mon[175151]: failed to encode map e137 with expected crc 2023-09-23T20:54:32.880 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:54:32 smithi026 ceph-mon[175151]: failed to encode map e137 with expected crc 2023-09-23T20:54:32.891 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:54:32.895 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:54:32.896 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:54:32.930 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:54:32.996 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:54:33.061 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:54:33.279 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:54:33.296 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:54:33.297 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:54:33.371 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:54:32 smithi070 ceph-mon[168282]: failed to encode map e137 with expected crc 2023-09-23T20:54:33.371 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:54:32 smithi070 ceph-mon[168282]: osdmap e137: 8 total, 8 up, 8 in 2023-09-23T20:54:33.371 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:54:32 smithi070 ceph-mon[168282]: failed to encode map e137 with expected crc 2023-09-23T20:54:33.371 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:54:32 smithi070 ceph-mon[168282]: failed to encode map e137 with expected crc 2023-09-23T20:54:33.371 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:54:32 smithi070 ceph-mon[168282]: failed to encode map e137 with expected crc 2023-09-23T20:54:33.371 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:54:32 smithi070 ceph-mon[168282]: failed to encode map e137 with expected crc 2023-09-23T20:54:33.371 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:54:32 smithi070 ceph-mon[168282]: failed to encode map e137 with expected crc 2023-09-23T20:54:33.372 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:54:32 smithi070 ceph-mon[168282]: failed to encode map e137 with expected crc 2023-09-23T20:54:33.398 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:54:33.475 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:54:33.635 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:54:33.641 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:54:33.641 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:54:33.663 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T20:54:33.742 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:54:33.797 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0: {} 2023-09-23T20:54:33.840 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1: {} 2023-09-23T20:54:33.869 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:54:33.879 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2: {} 2023-09-23T20:54:33.942 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3: {} 2023-09-23T20:54:33.950 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:54:33.980 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:54:33.980 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:54:33.998 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4: {} 2023-09-23T20:54:34.046 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T20:54:34.081 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:54:34.100 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6: {} 2023-09-23T20:54:34.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:54:33 smithi026 ceph-mon[175151]: pgmap v339: 99 pgs: 91 active+clean, 1 active+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 4.4 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 831 op/s; 6500/218316 objects degraded (2.977%); 920 KiB/s, 123 objects/s recovering 2023-09-23T20:54:34.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:54:33 smithi026 ceph-mon[175151]: failed to encode map e138 with expected crc 2023-09-23T20:54:34.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:54:33 smithi026 ceph-mon[175151]: failed to encode map e138 with expected crc 2023-09-23T20:54:34.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:54:33 smithi026 ceph-mon[175151]: osdmap e138: 8 total, 8 up, 8 in 2023-09-23T20:54:34.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:54:33 smithi026 ceph-mon[175151]: 6.14 starting backfill to osd.5 from (0'0,0'0] MAX to 135'7459 2023-09-23T20:54:34.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:54:33 smithi026 ceph-mon[175151]: 6.a starting backfill to osd.5 from (0'0,0'0] MAX to 135'8219 2023-09-23T20:54:34.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:54:33 smithi026 ceph-mon[175151]: 6.17 starting backfill to osd.5 from (0'0,0'0] MAX to 135'7498 2023-09-23T20:54:34.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:54:33 smithi026 ceph-mon[175151]: 6.d starting backfill to osd.5 from (0'0,0'0] MAX to 135'8202 2023-09-23T20:54:34.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:54:33 smithi026 ceph-mon[175151]: 6.7 starting backfill to osd.5 from (0'0,0'0] MAX to 135'7552 2023-09-23T20:54:34.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:54:33 smithi026 ceph-mon[175151]: 6.4 starting backfill to osd.5 from (0'0,0'0] MAX to 135'7439 2023-09-23T20:54:34.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:54:33 smithi026 ceph-mon[175151]: 6.c starting backfill to osd.5 from (0'0,0'0] MAX to 135'8046 2023-09-23T20:54:34.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:54:33 smithi026 ceph-mon[175151]: failed to encode map e138 with expected crc 2023-09-23T20:54:34.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:54:33 smithi026 ceph-mon[175151]: failed to encode map e138 with expected crc 2023-09-23T20:54:34.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:54:33 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/21776306' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:54:34.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:54:33 smithi026 ceph-mon[179688]: pgmap v339: 99 pgs: 91 active+clean, 1 active+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 4.4 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 831 op/s; 6500/218316 objects degraded (2.977%); 920 KiB/s, 123 objects/s recovering 2023-09-23T20:54:34.134 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:54:33 smithi026 ceph-mon[179688]: failed to encode map e138 with expected crc 2023-09-23T20:54:34.134 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:54:33 smithi026 ceph-mon[179688]: failed to encode map e138 with expected crc 2023-09-23T20:54:34.134 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:54:33 smithi026 ceph-mon[179688]: osdmap e138: 8 total, 8 up, 8 in 2023-09-23T20:54:34.134 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:54:33 smithi026 ceph-mon[179688]: 6.14 starting backfill to osd.5 from (0'0,0'0] MAX to 135'7459 2023-09-23T20:54:34.134 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:54:33 smithi026 ceph-mon[179688]: 6.a starting backfill to osd.5 from (0'0,0'0] MAX to 135'8219 2023-09-23T20:54:34.134 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:54:33 smithi026 ceph-mon[179688]: 6.17 starting backfill to osd.5 from (0'0,0'0] MAX to 135'7498 2023-09-23T20:54:34.134 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:54:33 smithi026 ceph-mon[179688]: 6.d starting backfill to osd.5 from (0'0,0'0] MAX to 135'8202 2023-09-23T20:54:34.134 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:54:33 smithi026 ceph-mon[179688]: 6.7 starting backfill to osd.5 from (0'0,0'0] MAX to 135'7552 2023-09-23T20:54:34.134 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:54:33 smithi026 ceph-mon[179688]: 6.4 starting backfill to osd.5 from (0'0,0'0] MAX to 135'7439 2023-09-23T20:54:34.134 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:54:33 smithi026 ceph-mon[179688]: 6.c starting backfill to osd.5 from (0'0,0'0] MAX to 135'8046 2023-09-23T20:54:34.134 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:54:33 smithi026 ceph-mon[179688]: failed to encode map e138 with expected crc 2023-09-23T20:54:34.134 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:54:33 smithi026 ceph-mon[179688]: failed to encode map e138 with expected crc 2023-09-23T20:54:34.135 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:54:33 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/21776306' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:54:34.139 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T20:54:34.172 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:54:34.190 INFO:teuthology.orchestra.run.smithi026.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T20:54:34.190 INFO:teuthology.orchestra.run.smithi026.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T20:54:34.190 INFO:teuthology.orchestra.run.smithi026.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T20:54:34.190 INFO:teuthology.orchestra.run.smithi026.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T20:54:34.190 INFO:teuthology.orchestra.run.smithi026.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T20:54:34.190 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T20:54:34.190 INFO:teuthology.orchestra.run.smithi026.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T20:54:34.190 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T20:54:34.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:54:33 smithi070 ceph-mon[168282]: pgmap v339: 99 pgs: 91 active+clean, 1 active+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 4.4 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 831 op/s; 6500/218316 objects degraded (2.977%); 920 KiB/s, 123 objects/s recovering 2023-09-23T20:54:34.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:54:33 smithi070 ceph-mon[168282]: failed to encode map e138 with expected crc 2023-09-23T20:54:34.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:54:33 smithi070 ceph-mon[168282]: failed to encode map e138 with expected crc 2023-09-23T20:54:34.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:54:33 smithi070 ceph-mon[168282]: osdmap e138: 8 total, 8 up, 8 in 2023-09-23T20:54:34.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:54:33 smithi070 ceph-mon[168282]: 6.14 starting backfill to osd.5 from (0'0,0'0] MAX to 135'7459 2023-09-23T20:54:34.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:54:33 smithi070 ceph-mon[168282]: 6.a starting backfill to osd.5 from (0'0,0'0] MAX to 135'8219 2023-09-23T20:54:34.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:54:33 smithi070 ceph-mon[168282]: 6.17 starting backfill to osd.5 from (0'0,0'0] MAX to 135'7498 2023-09-23T20:54:34.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:54:33 smithi070 ceph-mon[168282]: 6.d starting backfill to osd.5 from (0'0,0'0] MAX to 135'8202 2023-09-23T20:54:34.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:54:33 smithi070 ceph-mon[168282]: 6.7 starting backfill to osd.5 from (0'0,0'0] MAX to 135'7552 2023-09-23T20:54:34.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:54:33 smithi070 ceph-mon[168282]: 6.4 starting backfill to osd.5 from (0'0,0'0] MAX to 135'7439 2023-09-23T20:54:34.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:54:33 smithi070 ceph-mon[168282]: 6.c starting backfill to osd.5 from (0'0,0'0] MAX to 135'8046 2023-09-23T20:54:34.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:54:33 smithi070 ceph-mon[168282]: failed to encode map e138 with expected crc 2023-09-23T20:54:34.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:54:33 smithi070 ceph-mon[168282]: failed to encode map e138 with expected crc 2023-09-23T20:54:34.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:54:33 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/21776306' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:54:34.427 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:54:34.433 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:54:34.434 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:54:34.534 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:54:34.596 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:54:34.730 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:54:34.739 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:54:34.740 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:54:34.803 INFO:teuthology.orchestra.run.smithi026.stdout:ERROR: (22) Invalid argument 2023-09-23T20:54:34.803 INFO:teuthology.orchestra.run.smithi026.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:54:34.841 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:54:34.939 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:54:35.135 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:54:34 smithi026 ceph-mon[175151]: failed to encode map e138 with expected crc 2023-09-23T20:54:35.135 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:54:34 smithi026 ceph-mon[175151]: Health check update: Degraded data redundancy: 6500/218316 objects degraded (2.977%), 7 pgs degraded (PG_DEGRADED) 2023-09-23T20:54:35.135 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:54:34 smithi026 ceph-mon[179688]: failed to encode map e138 with expected crc 2023-09-23T20:54:35.136 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:54:34 smithi026 ceph-mon[179688]: Health check update: Degraded data redundancy: 6500/218316 objects degraded (2.977%), 7 pgs degraded (PG_DEGRADED) 2023-09-23T20:54:35.228 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:54:35.234 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:54:35.234 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:54:35.239 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:54:35.265 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_ops_in_flight 2023-09-23T20:54:35.335 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:54:35.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:54:34 smithi070 ceph-mon[168282]: failed to encode map e138 with expected crc 2023-09-23T20:54:35.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:54:34 smithi070 ceph-mon[168282]: Health check update: Degraded data redundancy: 6500/218316 objects degraded (2.977%), 7 pgs degraded (PG_DEGRADED) 2023-09-23T20:54:35.372 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:54:35.420 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:54:35.427 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:54:35.427 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:54:35.528 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:54:35.686 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:54:36.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:54:35 smithi070 ceph-mon[168282]: pgmap v341: 99 pgs: 2 active+recovery_wait+degraded+remapped, 88 active+clean, 3 active+remapped+backfill_wait, 1 active+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped; 4.4 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 840 op/s; 6347/215133 objects degraded (2.950%); 13340/215133 objects misplaced (6.201%); 1.6 MiB/s, 116 objects/s recovering 2023-09-23T20:54:36.376 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:54:36.380 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:54:36.380 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:54:36.381 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:54:35 smithi026 ceph-mon[175151]: pgmap v341: 99 pgs: 2 active+recovery_wait+degraded+remapped, 88 active+clean, 3 active+remapped+backfill_wait, 1 active+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped; 4.4 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 840 op/s; 6347/215133 objects degraded (2.950%); 13340/215133 objects misplaced (6.201%); 1.6 MiB/s, 116 objects/s recovering 2023-09-23T20:54:36.381 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:54:35 smithi026 ceph-mon[179688]: pgmap v341: 99 pgs: 2 active+recovery_wait+degraded+remapped, 88 active+clean, 3 active+remapped+backfill_wait, 1 active+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped; 4.4 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 840 op/s; 6347/215133 objects degraded (2.950%); 13340/215133 objects misplaced (6.201%); 1.6 MiB/s, 116 objects/s recovering 2023-09-23T20:54:36.481 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:54:36.517 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:54:36.557 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:54:36.563 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:54:36.563 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:54:36.664 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:54:36.701 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:54:36.758 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:54:36.761 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:54:36.762 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:54:36.863 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:54:36.879 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T20:54:36.921 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 6, 7, 4, 5] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-09-23T20:54:36.922 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T20:54:36.922 INFO:tasks.thrashosds.thrasher:Reweighting osd 3 to 0.8004770756021918 2023-09-23T20:54:36.922 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph osd reweight 3 0.8004770756021918 2023-09-23T20:54:36.946 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:54:38.203 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:54:38.217 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:54:38.217 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:54:37 smithi026 ceph-mon[179688]: pgmap v342: 99 pgs: 1 active+remapped+backfilling, 2 active+recovery_wait+degraded+remapped, 86 active+clean, 4 active+remapped+backfill_wait, 1 active+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped; 4.3 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 603 op/s; 6347/212436 objects degraded (2.988%); 21583/212436 objects misplaced (10.160%); 6.8 MiB/s, 201 objects/s recovering 2023-09-23T20:54:38.218 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:54:37 smithi026 ceph-mon[179688]: osdmap e139: 8 total, 8 up, 8 in 2023-09-23T20:54:38.218 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:54:37 smithi026 ceph-mon[179688]: failed to encode map e139 with expected crc 2023-09-23T20:54:38.218 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:54:37 smithi026 ceph-mon[179688]: failed to encode map e139 with expected crc 2023-09-23T20:54:38.218 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:54:38.219 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:54:37 smithi026 ceph-mon[175151]: pgmap v342: 99 pgs: 1 active+remapped+backfilling, 2 active+recovery_wait+degraded+remapped, 86 active+clean, 4 active+remapped+backfill_wait, 1 active+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped; 4.3 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 603 op/s; 6347/212436 objects degraded (2.988%); 21583/212436 objects misplaced (10.160%); 6.8 MiB/s, 201 objects/s recovering 2023-09-23T20:54:38.219 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:54:37 smithi026 ceph-mon[175151]: osdmap e139: 8 total, 8 up, 8 in 2023-09-23T20:54:38.219 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:54:37 smithi026 ceph-mon[175151]: failed to encode map e139 with expected crc 2023-09-23T20:54:38.219 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:54:37 smithi026 ceph-mon[175151]: failed to encode map e139 with expected crc 2023-09-23T20:54:38.319 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:54:38.348 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:54:37 smithi070 ceph-mon[168282]: pgmap v342: 99 pgs: 1 active+remapped+backfilling, 2 active+recovery_wait+degraded+remapped, 86 active+clean, 4 active+remapped+backfill_wait, 1 active+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped; 4.3 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 603 op/s; 6347/212436 objects degraded (2.988%); 21583/212436 objects misplaced (10.160%); 6.8 MiB/s, 201 objects/s recovering 2023-09-23T20:54:38.348 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:54:37 smithi070 ceph-mon[168282]: osdmap e139: 8 total, 8 up, 8 in 2023-09-23T20:54:38.348 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:54:37 smithi070 ceph-mon[168282]: failed to encode map e139 with expected crc 2023-09-23T20:54:38.348 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:54:37 smithi070 ceph-mon[168282]: failed to encode map e139 with expected crc 2023-09-23T20:54:38.351 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:54:38.391 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:54:38.393 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:54:38.394 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:54:38.494 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:54:38.519 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:54:38.527 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:54:38.553 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:54:38.572 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:54:38.575 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:54:38.575 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:54:38.621 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:54:38.676 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:54:38.705 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:54:38.746 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:54:38.751 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:54:38.752 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:54:38.853 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:54:38.954 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:54:39.285 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:54:39.292 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:54:39.292 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:54:38 smithi026 ceph-mon[175151]: osdmap e140: 8 total, 8 up, 8 in 2023-09-23T20:54:39.292 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:54:38 smithi026 ceph-mon[175151]: failed to encode map e140 with expected crc 2023-09-23T20:54:39.292 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:54:38 smithi026 ceph-mon[175151]: failed to encode map e140 with expected crc 2023-09-23T20:54:39.292 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:54:38 smithi026 ceph-mon[175151]: failed to encode map e140 with expected crc 2023-09-23T20:54:39.293 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:54:39.294 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:54:38 smithi026 ceph-mon[179688]: osdmap e140: 8 total, 8 up, 8 in 2023-09-23T20:54:39.294 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:54:38 smithi026 ceph-mon[179688]: failed to encode map e140 with expected crc 2023-09-23T20:54:39.295 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:54:38 smithi026 ceph-mon[179688]: failed to encode map e140 with expected crc 2023-09-23T20:54:39.295 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:54:38 smithi026 ceph-mon[179688]: failed to encode map e140 with expected crc 2023-09-23T20:54:39.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:54:38 smithi070 ceph-mon[168282]: osdmap e140: 8 total, 8 up, 8 in 2023-09-23T20:54:39.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:54:38 smithi070 ceph-mon[168282]: failed to encode map e140 with expected crc 2023-09-23T20:54:39.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:54:38 smithi070 ceph-mon[168282]: failed to encode map e140 with expected crc 2023-09-23T20:54:39.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:54:38 smithi070 ceph-mon[168282]: failed to encode map e140 with expected crc 2023-09-23T20:54:39.394 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:54:39.428 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:54:39.472 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:54:39.474 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:54:39.475 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:54:39.575 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:54:39.605 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:54:39.650 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:54:39.653 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:54:39.654 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:54:39.754 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:54:39.839 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:54:39 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[164029]: ::ffff:172.21.15.70 - - [23/Sep/2023:20:54:39] "GET /metrics HTTP/1.1" 200 36330 "" "Prometheus/2.43.0" 2023-09-23T20:54:39.848 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:54:40.026 INFO:teuthology.orchestra.run.smithi026.stderr:nodeep-scrub is set 2023-09-23T20:54:40.094 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:54:40.109 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:54:39 smithi026 ceph-mon[175151]: failed to encode map e140 with expected crc 2023-09-23T20:54:40.110 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:54:39 smithi026 ceph-mon[175151]: pgmap v345: 99 pgs: 1 peering, 1 active+remapped+backfilling, 2 active+recovery_wait+degraded+remapped, 87 active+clean, 4 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped; 4.3 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 497 op/s; 5517/209358 objects degraded (2.635%); 21583/209358 objects misplaced (10.309%); 11 MiB/s, 248 objects/s recovering 2023-09-23T20:54:40.110 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:54:39 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/400483747' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:54:40.110 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:54:39 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:54:40.110 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:54:39 smithi026 ceph-mon[175151]: Health check update: Degraded data redundancy: 5517/209358 objects degraded (2.635%), 6 pgs degraded (PG_DEGRADED) 2023-09-23T20:54:40.111 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:54:39 smithi026 ceph-mon[179688]: failed to encode map e140 with expected crc 2023-09-23T20:54:40.111 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:54:39 smithi026 ceph-mon[179688]: pgmap v345: 99 pgs: 1 peering, 1 active+remapped+backfilling, 2 active+recovery_wait+degraded+remapped, 87 active+clean, 4 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped; 4.3 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 497 op/s; 5517/209358 objects degraded (2.635%); 21583/209358 objects misplaced (10.309%); 11 MiB/s, 248 objects/s recovering 2023-09-23T20:54:40.111 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:54:39 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/400483747' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:54:40.111 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:54:39 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:54:40.111 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:54:39 smithi026 ceph-mon[179688]: Health check update: Degraded data redundancy: 5517/209358 objects degraded (2.635%), 6 pgs degraded (PG_DEGRADED) 2023-09-23T20:54:40.111 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:54:40.112 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:54:40.213 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:54:40.330 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:54:40.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:54:39 smithi070 ceph-mon[168282]: failed to encode map e140 with expected crc 2023-09-23T20:54:40.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:54:39 smithi070 ceph-mon[168282]: pgmap v345: 99 pgs: 1 peering, 1 active+remapped+backfilling, 2 active+recovery_wait+degraded+remapped, 87 active+clean, 4 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped; 4.3 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 497 op/s; 5517/209358 objects degraded (2.635%); 21583/209358 objects misplaced (10.309%); 11 MiB/s, 248 objects/s recovering 2023-09-23T20:54:40.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:54:39 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/400483747' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:54:40.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:54:39 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:54:40.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:54:39 smithi070 ceph-mon[168282]: Health check update: Degraded data redundancy: 5517/209358 objects degraded (2.635%), 6 pgs degraded (PG_DEGRADED) 2023-09-23T20:54:40.529 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:54:40.542 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:54:40.542 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:54:40.643 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:54:40.675 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:54:40.716 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:54:40.720 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:54:40.721 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:54:40.821 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:54:40.858 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:54:40.903 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:54:40.906 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:54:40.906 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:54:40.979 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:54:40 smithi070 ceph-mon[168282]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:54:40.979 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:54:40 smithi070 ceph-mon[168282]: failed to encode map e141 with expected crc 2023-09-23T20:54:40.979 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:54:40 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T20:54:40.979 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:54:40 smithi070 ceph-mon[168282]: osdmap e141: 8 total, 8 up, 8 in 2023-09-23T20:54:40.980 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:54:40 smithi070 ceph-mon[168282]: failed to encode map e141 with expected crc 2023-09-23T20:54:40.980 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:54:40 smithi070 ceph-mon[168282]: failed to encode map e141 with expected crc 2023-09-23T20:54:40.980 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:54:40 smithi070 ceph-mon[168282]: failed to encode map e141 with expected crc 2023-09-23T20:54:40.980 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:54:40 smithi070 ceph-mon[168282]: failed to encode map e141 with expected crc 2023-09-23T20:54:40.980 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:54:40 smithi070 ceph-mon[168282]: failed to encode map e141 with expected crc 2023-09-23T20:54:41.007 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:54:41.045 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:54:41.086 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:54:41.089 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:54:41.090 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:54:41.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:54:40 smithi026 ceph-mon[175151]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:54:41.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:54:40 smithi026 ceph-mon[175151]: failed to encode map e141 with expected crc 2023-09-23T20:54:41.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:54:40 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T20:54:41.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:54:40 smithi026 ceph-mon[175151]: osdmap e141: 8 total, 8 up, 8 in 2023-09-23T20:54:41.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:54:40 smithi026 ceph-mon[175151]: failed to encode map e141 with expected crc 2023-09-23T20:54:41.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:54:40 smithi026 ceph-mon[175151]: failed to encode map e141 with expected crc 2023-09-23T20:54:41.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:54:40 smithi026 ceph-mon[175151]: failed to encode map e141 with expected crc 2023-09-23T20:54:41.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:54:40 smithi026 ceph-mon[175151]: failed to encode map e141 with expected crc 2023-09-23T20:54:41.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:54:40 smithi026 ceph-mon[175151]: failed to encode map e141 with expected crc 2023-09-23T20:54:41.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:54:40 smithi026 ceph-mon[179688]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:54:41.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:54:40 smithi026 ceph-mon[179688]: failed to encode map e141 with expected crc 2023-09-23T20:54:41.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:54:40 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T20:54:41.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:54:40 smithi026 ceph-mon[179688]: osdmap e141: 8 total, 8 up, 8 in 2023-09-23T20:54:41.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:54:40 smithi026 ceph-mon[179688]: failed to encode map e141 with expected crc 2023-09-23T20:54:41.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:54:40 smithi026 ceph-mon[179688]: failed to encode map e141 with expected crc 2023-09-23T20:54:41.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:54:40 smithi026 ceph-mon[179688]: failed to encode map e141 with expected crc 2023-09-23T20:54:41.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:54:40 smithi026 ceph-mon[179688]: failed to encode map e141 with expected crc 2023-09-23T20:54:41.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:54:40 smithi026 ceph-mon[179688]: failed to encode map e141 with expected crc 2023-09-23T20:54:41.191 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:54:41.231 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:54:41.275 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:54:41.278 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:54:41.278 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:54:41.347 INFO:teuthology.orchestra.run.smithi026.stdout:ERROR: (22) Invalid argument 2023-09-23T20:54:41.347 INFO:teuthology.orchestra.run.smithi026.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:54:41.379 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:54:41.489 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:54:41.578 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:54:41.581 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:54:41.582 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:54:41.682 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:54:41.854 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0: {} 2023-09-23T20:54:41.859 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:54:41.907 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1: {} 2023-09-23T20:54:41.950 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2: {} 2023-09-23T20:54:41.959 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:54:41.967 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:54:41.967 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:54:41.994 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3: {} 2023-09-23T20:54:42.035 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4: {} 2023-09-23T20:54:42.036 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_blocked_ops 2023-09-23T20:54:42.068 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:54:42.087 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T20:54:42.104 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:54:42.127 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6: {} 2023-09-23T20:54:42.131 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:54:41 smithi026 ceph-mon[179688]: pgmap v347: 99 pgs: 1 active+recovering+undersized+remapped, 1 peering, 1 active+remapped+backfilling, 2 active+recovery_wait+degraded+remapped, 87 active+clean, 4 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped; 4.2 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 610 op/s; 4702/204129 objects degraded (2.303%); 20850/204129 objects misplaced (10.214%); 13 MiB/s, 284 objects/s recovering 2023-09-23T20:54:42.131 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:54:41 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/1341693666' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:54:42.131 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:54:41 smithi026 ceph-mon[175151]: pgmap v347: 99 pgs: 1 active+recovering+undersized+remapped, 1 peering, 1 active+remapped+backfilling, 2 active+recovery_wait+degraded+remapped, 87 active+clean, 4 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped; 4.2 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 610 op/s; 4702/204129 objects degraded (2.303%); 20850/204129 objects misplaced (10.214%); 13 MiB/s, 284 objects/s recovering 2023-09-23T20:54:42.131 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:54:41 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/1341693666' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:54:42.149 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:54:42.152 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:54:42.153 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:54:42.164 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T20:54:42.165 INFO:teuthology.orchestra.run.smithi026.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T20:54:42.165 INFO:teuthology.orchestra.run.smithi026.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T20:54:42.165 INFO:teuthology.orchestra.run.smithi026.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T20:54:42.166 INFO:teuthology.orchestra.run.smithi026.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T20:54:42.166 INFO:teuthology.orchestra.run.smithi026.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T20:54:42.166 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T20:54:42.166 INFO:teuthology.orchestra.run.smithi026.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T20:54:42.166 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T20:54:42.253 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:54:42.283 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:54:41 smithi070 ceph-mon[168282]: pgmap v347: 99 pgs: 1 active+recovering+undersized+remapped, 1 peering, 1 active+remapped+backfilling, 2 active+recovery_wait+degraded+remapped, 87 active+clean, 4 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped; 4.2 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 610 op/s; 4702/204129 objects degraded (2.303%); 20850/204129 objects misplaced (10.214%); 13 MiB/s, 284 objects/s recovering 2023-09-23T20:54:42.283 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:54:41 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/1341693666' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:54:42.286 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:54:42.325 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:54:42.329 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:54:42.329 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:54:42.430 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:54:42.512 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:54:42.602 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:54:42.609 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:54:42.609 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:54:42.611 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T20:54:42.711 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:54:42.740 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:54:42.784 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:54:42.786 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:54:42.787 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:54:42.887 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:54:42.916 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:54:42.966 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:54:42.970 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:54:42.971 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:54:43.046 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:54:43.070 INFO:teuthology.orchestra.run.smithi026.stderr:reweighted osd.3 to 0.800477 (ccec) 2023-09-23T20:54:43.072 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:54:43.131 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:54:43.293 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:54:43.296 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:54:43 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/2053185417' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 0.8004770756021918}]: dispatch 2023-09-23T20:54:43.296 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:54:43 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 0.8004770756021918}]: dispatch 2023-09-23T20:54:43.296 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:54:43 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/2053185417' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 0.8004770756021918}]: dispatch 2023-09-23T20:54:43.296 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:54:43 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 0.8004770756021918}]: dispatch 2023-09-23T20:54:43.297 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:54:43.297 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:54:43.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:54:43 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/2053185417' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 0.8004770756021918}]: dispatch 2023-09-23T20:54:43.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:54:43 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 0.8004770756021918}]: dispatch 2023-09-23T20:54:43.397 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:54:43.450 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:54:43.851 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:54:43.855 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:54:43.855 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:54:43.956 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:54:43.988 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:54:44.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:54:44 smithi026 ceph-mon[175151]: pgmap v348: 99 pgs: 1 active+remapped+backfilling, 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded+remapped, 88 active+clean, 4 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped; 4.0 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 819 op/s; 4702/197679 objects degraded (2.379%); 19280/197679 objects misplaced (9.753%); 20 MiB/s, 392 objects/s recovering 2023-09-23T20:54:44.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:54:44 smithi026 ceph-mon[175151]: failed to encode map e142 with expected crc 2023-09-23T20:54:44.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:54:44 smithi026 ceph-mon[175151]: failed to encode map e142 with expected crc 2023-09-23T20:54:44.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:54:44 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 0.8004770756021918}]': finished 2023-09-23T20:54:44.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:54:44 smithi026 ceph-mon[175151]: osdmap e142: 8 total, 8 up, 8 in 2023-09-23T20:54:44.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:54:44 smithi026 ceph-mon[179688]: pgmap v348: 99 pgs: 1 active+remapped+backfilling, 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded+remapped, 88 active+clean, 4 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped; 4.0 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 819 op/s; 4702/197679 objects degraded (2.379%); 19280/197679 objects misplaced (9.753%); 20 MiB/s, 392 objects/s recovering 2023-09-23T20:54:44.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:54:44 smithi026 ceph-mon[179688]: failed to encode map e142 with expected crc 2023-09-23T20:54:44.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:54:44 smithi026 ceph-mon[179688]: failed to encode map e142 with expected crc 2023-09-23T20:54:44.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:54:44 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 0.8004770756021918}]': finished 2023-09-23T20:54:44.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:54:44 smithi026 ceph-mon[179688]: osdmap e142: 8 total, 8 up, 8 in 2023-09-23T20:54:44.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:54:44 smithi070 ceph-mon[168282]: pgmap v348: 99 pgs: 1 active+remapped+backfilling, 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded+remapped, 88 active+clean, 4 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped; 4.0 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 819 op/s; 4702/197679 objects degraded (2.379%); 19280/197679 objects misplaced (9.753%); 20 MiB/s, 392 objects/s recovering 2023-09-23T20:54:44.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:54:44 smithi070 ceph-mon[168282]: failed to encode map e142 with expected crc 2023-09-23T20:54:44.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:54:44 smithi070 ceph-mon[168282]: failed to encode map e142 with expected crc 2023-09-23T20:54:44.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:54:44 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 0.8004770756021918}]': finished 2023-09-23T20:54:44.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:54:44 smithi070 ceph-mon[168282]: osdmap e142: 8 total, 8 up, 8 in 2023-09-23T20:54:44.614 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T20:54:45.040 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:54:45 smithi026 ceph-mon[179688]: failed to encode map e143 with expected crc 2023-09-23T20:54:45.040 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:54:45 smithi026 ceph-mon[179688]: failed to encode map e143 with expected crc 2023-09-23T20:54:45.040 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:54:45 smithi026 ceph-mon[179688]: osdmap e143: 8 total, 8 up, 8 in 2023-09-23T20:54:45.040 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:54:45 smithi026 ceph-mon[179688]: failed to encode map e143 with expected crc 2023-09-23T20:54:45.040 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:54:45 smithi026 ceph-mon[179688]: failed to encode map e143 with expected crc 2023-09-23T20:54:45.040 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:54:45 smithi026 ceph-mon[179688]: failed to encode map e143 with expected crc 2023-09-23T20:54:45.040 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:54:45 smithi026 ceph-mon[179688]: failed to encode map e143 with expected crc 2023-09-23T20:54:45.040 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:54:45 smithi026 ceph-mon[179688]: pgmap v351: 99 pgs: 1 active+remapped+backfilling, 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded+remapped, 89 active+clean, 4 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped; 3.9 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 908 op/s; 3210/193005 objects degraded (1.663%); 18971/193005 objects misplaced (9.829%); 18 MiB/s, 366 objects/s recovering 2023-09-23T20:54:45.040 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:54:45 smithi026 ceph-mon[179688]: Health check update: Degraded data redundancy: 4702/197679 objects degraded (2.379%), 5 pgs degraded (PG_DEGRADED) 2023-09-23T20:54:45.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:54:45 smithi070 ceph-mon[168282]: failed to encode map e143 with expected crc 2023-09-23T20:54:45.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:54:45 smithi070 ceph-mon[168282]: failed to encode map e143 with expected crc 2023-09-23T20:54:45.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:54:45 smithi070 ceph-mon[168282]: osdmap e143: 8 total, 8 up, 8 in 2023-09-23T20:54:45.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:54:45 smithi070 ceph-mon[168282]: failed to encode map e143 with expected crc 2023-09-23T20:54:45.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:54:45 smithi070 ceph-mon[168282]: failed to encode map e143 with expected crc 2023-09-23T20:54:45.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:54:45 smithi070 ceph-mon[168282]: failed to encode map e143 with expected crc 2023-09-23T20:54:45.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:54:45 smithi070 ceph-mon[168282]: failed to encode map e143 with expected crc 2023-09-23T20:54:45.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:54:45 smithi070 ceph-mon[168282]: pgmap v351: 99 pgs: 1 active+remapped+backfilling, 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded+remapped, 89 active+clean, 4 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped; 3.9 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 908 op/s; 3210/193005 objects degraded (1.663%); 18971/193005 objects misplaced (9.829%); 18 MiB/s, 366 objects/s recovering 2023-09-23T20:54:45.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:54:45 smithi070 ceph-mon[168282]: Health check update: Degraded data redundancy: 4702/197679 objects degraded (2.379%), 5 pgs degraded (PG_DEGRADED) 2023-09-23T20:54:45.381 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:54:45 smithi026 ceph-mon[175151]: failed to encode map e143 with expected crc 2023-09-23T20:54:45.381 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:54:45 smithi026 ceph-mon[175151]: failed to encode map e143 with expected crc 2023-09-23T20:54:45.381 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:54:45 smithi026 ceph-mon[175151]: osdmap e143: 8 total, 8 up, 8 in 2023-09-23T20:54:45.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:54:45 smithi026 ceph-mon[175151]: failed to encode map e143 with expected crc 2023-09-23T20:54:45.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:54:45 smithi026 ceph-mon[175151]: failed to encode map e143 with expected crc 2023-09-23T20:54:45.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:54:45 smithi026 ceph-mon[175151]: failed to encode map e143 with expected crc 2023-09-23T20:54:45.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:54:45 smithi026 ceph-mon[175151]: failed to encode map e143 with expected crc 2023-09-23T20:54:45.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:54:45 smithi026 ceph-mon[175151]: pgmap v351: 99 pgs: 1 active+remapped+backfilling, 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded+remapped, 89 active+clean, 4 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped; 3.9 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 908 op/s; 3210/193005 objects degraded (1.663%); 18971/193005 objects misplaced (9.829%); 18 MiB/s, 366 objects/s recovering 2023-09-23T20:54:45.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:54:45 smithi026 ceph-mon[175151]: Health check update: Degraded data redundancy: 4702/197679 objects degraded (2.379%), 5 pgs degraded (PG_DEGRADED) 2023-09-23T20:54:46.621 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:54:46 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:54:46.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:54:46 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:54:46.635 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:54:46 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:54:47.622 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:54:47 smithi070 ceph-mon[168282]: pgmap v352: 99 pgs: 1 active+remapped+backfilling, 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded+remapped, 89 active+clean, 4 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped; 3.9 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 740 op/s; 3210/190227 objects degraded (1.687%); 17426/190227 objects misplaced (9.161%); 26 MiB/s, 482 objects/s recovering 2023-09-23T20:54:47.631 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:54:47 smithi026 ceph-mon[175151]: pgmap v352: 99 pgs: 1 active+remapped+backfilling, 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded+remapped, 89 active+clean, 4 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped; 3.9 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 740 op/s; 3210/190227 objects degraded (1.687%); 17426/190227 objects misplaced (9.161%); 26 MiB/s, 482 objects/s recovering 2023-09-23T20:54:47.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:54:47 smithi026 ceph-mon[179688]: pgmap v352: 99 pgs: 1 active+remapped+backfilling, 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded+remapped, 89 active+clean, 4 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped; 3.9 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 740 op/s; 3210/190227 objects degraded (1.687%); 17426/190227 objects misplaced (9.161%); 26 MiB/s, 482 objects/s recovering 2023-09-23T20:54:48.385 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:54:48.394 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:54:48.395 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:54:48.496 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:54:48.530 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:54:48.561 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:54:48.572 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:54:48.574 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:54:48.575 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:54:48.652 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:54:48.675 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:54:48.766 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:54:48.870 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:54:48.879 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:54:48.879 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:54:48.980 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:54:49.045 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:54:49.132 INFO:teuthology.orchestra.run.smithi026.stdout:{ 2023-09-23T20:54:49.132 INFO:teuthology.orchestra.run.smithi026.stdout: "ops": [], 2023-09-23T20:54:49.132 INFO:teuthology.orchestra.run.smithi026.stdout: "complaint_time": 30, 2023-09-23T20:54:49.133 INFO:teuthology.orchestra.run.smithi026.stdout: "num_blocked_ops": 0 2023-09-23T20:54:49.133 INFO:teuthology.orchestra.run.smithi026.stdout:} 2023-09-23T20:54:49.286 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:54:49.296 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:54:49.297 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:54:49.397 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:54:49.491 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:54:49.579 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:54:49 smithi026 ceph-mon[175151]: pgmap v353: 99 pgs: 1 active+remapped+backfilling, 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded+remapped, 89 active+clean, 4 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped; 3.8 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 725 op/s; 3210/186711 objects degraded (1.719%); 17426/186711 objects misplaced (9.333%); 20 MiB/s, 377 objects/s recovering 2023-09-23T20:54:49.579 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:54:49 smithi026 ceph-mon[179688]: pgmap v353: 99 pgs: 1 active+remapped+backfilling, 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded+remapped, 89 active+clean, 4 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped; 3.8 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 725 op/s; 3210/186711 objects degraded (1.719%); 17426/186711 objects misplaced (9.333%); 20 MiB/s, 377 objects/s recovering 2023-09-23T20:54:49.618 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:54:49.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:54:49 smithi070 ceph-mon[168282]: pgmap v353: 99 pgs: 1 active+remapped+backfilling, 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded+remapped, 89 active+clean, 4 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped; 3.8 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 725 op/s; 3210/186711 objects degraded (1.719%); 17426/186711 objects misplaced (9.333%); 20 MiB/s, 377 objects/s recovering 2023-09-23T20:54:49.626 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:54:49.627 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:54:49.728 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:54:49.767 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:54:49.818 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:54:49.825 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:54:49.826 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:54:49.887 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:54:49 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[164029]: ::ffff:172.21.15.70 - - [23/Sep/2023:20:54:49] "GET /metrics HTTP/1.1" 200 36352 "" "Prometheus/2.43.0" 2023-09-23T20:54:49.926 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:54:50.005 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:54:50.248 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:54:50.255 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:54:50.255 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:54:50.317 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_historic_ops 2023-09-23T20:54:50.356 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:54:50.466 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:54:50.587 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:54:50.605 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:54:50.606 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:54:50.610 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:54:50 smithi026 ceph-mon[175151]: Health check update: Degraded data redundancy: 3210/186711 objects degraded (1.719%), 4 pgs degraded (PG_DEGRADED) 2023-09-23T20:54:50.610 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:54:50 smithi026 ceph-mon[179688]: Health check update: Degraded data redundancy: 3210/186711 objects degraded (1.719%), 4 pgs degraded (PG_DEGRADED) 2023-09-23T20:54:50.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:54:50 smithi070 ceph-mon[168282]: Health check update: Degraded data redundancy: 3210/186711 objects degraded (1.719%), 4 pgs degraded (PG_DEGRADED) 2023-09-23T20:54:50.707 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:54:50.795 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:54:50.841 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:54:50.847 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:54:50.847 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:54:50.948 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:54:50.979 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:54:51.028 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:54:51.032 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:54:51.032 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:54:51.107 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0: {} 2023-09-23T20:54:51.133 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:54:51.171 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1: {} 2023-09-23T20:54:51.173 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:54:51.200 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2: {} 2023-09-23T20:54:51.215 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:54:51.217 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:54:51.218 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:54:51.239 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3: {} 2023-09-23T20:54:51.280 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4: {} 2023-09-23T20:54:51.318 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:54:51.319 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T20:54:51.350 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6: {} 2023-09-23T20:54:51.402 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:54:51.402 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T20:54:51.420 INFO:teuthology.orchestra.run.smithi026.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T20:54:51.420 INFO:teuthology.orchestra.run.smithi026.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T20:54:51.420 INFO:teuthology.orchestra.run.smithi026.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T20:54:51.420 INFO:teuthology.orchestra.run.smithi026.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T20:54:51.420 INFO:teuthology.orchestra.run.smithi026.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T20:54:51.420 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T20:54:51.420 INFO:teuthology.orchestra.run.smithi026.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T20:54:51.420 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T20:54:51.434 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:54:51.499 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:54:51.515 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:54:51.515 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:54:51.616 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:54:51.650 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:54:51.694 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:54:51.697 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:54:51.697 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:54:51.798 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:54:51.847 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:54:51.894 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:54:51.898 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:54:51.898 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:54:51.936 INFO:teuthology.orchestra.run.smithi026.stderr:noscrub is unset 2023-09-23T20:54:51.999 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:54:52.030 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:54:52.076 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:54:52.080 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:54:52.081 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:54:52.112 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:54:51 smithi026 ceph-mon[179688]: pgmap v354: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfilling, 2 active+recovery_wait+degraded+remapped, 89 active+clean, 4 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped; 3.7 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 702 op/s; 3210/180804 objects degraded (1.775%); 16214/180804 objects misplaced (8.968%); 14 MiB/s, 289 objects/s recovering 2023-09-23T20:54:52.112 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:54:51 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/3835162788' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:54:52.112 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:54:51 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:54:52.112 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:54:51 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/3800990424' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:54:52.112 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:54:51 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:54:52.112 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:54:51 smithi026 ceph-mon[175151]: pgmap v354: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfilling, 2 active+recovery_wait+degraded+remapped, 89 active+clean, 4 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped; 3.7 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 702 op/s; 3210/180804 objects degraded (1.775%); 16214/180804 objects misplaced (8.968%); 14 MiB/s, 289 objects/s recovering 2023-09-23T20:54:52.112 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:54:51 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/3835162788' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:54:52.113 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:54:51 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:54:52.113 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:54:51 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/3800990424' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:54:52.113 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:54:51 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:54:52.182 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:54:52.211 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 6, 7, 4, 5] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-09-23T20:54:52.211 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T20:54:52.211 INFO:tasks.thrashosds.thrasher:Growing pool .mgr 2023-09-23T20:54:52.212 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T20:54:52.286 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:54:52.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:54:51 smithi070 ceph-mon[168282]: pgmap v354: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfilling, 2 active+recovery_wait+degraded+remapped, 89 active+clean, 4 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped; 3.7 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 702 op/s; 3210/180804 objects degraded (1.775%); 16214/180804 objects misplaced (8.968%); 14 MiB/s, 289 objects/s recovering 2023-09-23T20:54:52.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:54:51 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/3835162788' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:54:52.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:54:51 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:54:52.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:54:51 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/3800990424' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:54:52.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:54:51 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:54:52.518 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:54:52.533 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:54:52.533 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:54:52.634 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:54:52.682 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:54:52.850 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:54:52.854 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:54:52.855 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:54:52.956 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:54:53.073 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:54:53.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:54:52 smithi070 ceph-mon[168282]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:54:53.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:54:52 smithi070 ceph-mon[168282]: failed to encode map e144 with expected crc 2023-09-23T20:54:53.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:54:52 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T20:54:53.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:54:52 smithi070 ceph-mon[168282]: osdmap e144: 8 total, 8 up, 8 in 2023-09-23T20:54:53.131 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:54:53.135 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:54:53.135 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:54:53.236 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:54:53.293 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:54:53.338 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:54:53.343 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:54:53.343 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:54:53.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:54:52 smithi026 ceph-mon[179688]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:54:53.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:54:52 smithi026 ceph-mon[179688]: failed to encode map e144 with expected crc 2023-09-23T20:54:53.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:54:52 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T20:54:53.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:54:52 smithi026 ceph-mon[179688]: osdmap e144: 8 total, 8 up, 8 in 2023-09-23T20:54:53.383 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:54:52 smithi026 ceph-mon[175151]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:54:53.383 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:54:52 smithi026 ceph-mon[175151]: failed to encode map e144 with expected crc 2023-09-23T20:54:53.383 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:54:52 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T20:54:53.383 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:54:52 smithi026 ceph-mon[175151]: osdmap e144: 8 total, 8 up, 8 in 2023-09-23T20:54:53.444 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:54:53.528 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:54:53.925 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:54:53.937 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T20:54:54.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:54:53 smithi070 ceph-mon[168282]: pgmap v356: 99 pgs: 1 active+remapped+backfilling, 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded+remapped, 89 active+clean, 4 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped; 3.5 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 970 op/s; 3210/173700 objects degraded (1.848%); 15063/173700 objects misplaced (8.672%); 20 MiB/s, 381 objects/s recovering 2023-09-23T20:54:54.381 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:54:53 smithi026 ceph-mon[175151]: pgmap v356: 99 pgs: 1 active+remapped+backfilling, 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded+remapped, 89 active+clean, 4 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped; 3.5 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 970 op/s; 3210/173700 objects degraded (1.848%); 15063/173700 objects misplaced (8.672%); 20 MiB/s, 381 objects/s recovering 2023-09-23T20:54:54.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:54:53 smithi026 ceph-mon[179688]: pgmap v356: 99 pgs: 1 active+remapped+backfilling, 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded+remapped, 89 active+clean, 4 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped; 3.5 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 970 op/s; 3210/173700 objects degraded (1.848%); 15063/173700 objects misplaced (8.672%); 20 MiB/s, 381 objects/s recovering 2023-09-23T20:54:54.473 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T20:54:54.885 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:54:54.888 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:54:54.889 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:54:54.989 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:54:55.028 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:54:55.086 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:54:55.094 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:54:55.094 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:54:55.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:54:54 smithi026 ceph-mon[179688]: osdmap e145: 8 total, 8 up, 8 in 2023-09-23T20:54:55.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:54:54 smithi026 ceph-mon[179688]: failed to encode map e145 with expected crc 2023-09-23T20:54:55.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:54:54 smithi026 ceph-mon[179688]: failed to encode map e145 with expected crc 2023-09-23T20:54:55.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:54:54 smithi026 ceph-mon[179688]: failed to encode map e145 with expected crc 2023-09-23T20:54:55.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:54:54 smithi026 ceph-mon[179688]: failed to encode map e145 with expected crc 2023-09-23T20:54:55.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:54:54 smithi026 ceph-mon[179688]: failed to encode map e145 with expected crc 2023-09-23T20:54:55.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:54:54 smithi026 ceph-mon[179688]: failed to encode map e145 with expected crc 2023-09-23T20:54:55.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:54:54 smithi026 ceph-mon[179688]: Health check update: Degraded data redundancy: 3210/173700 objects degraded (1.848%), 4 pgs degraded (PG_DEGRADED) 2023-09-23T20:54:55.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:54:54 smithi026 ceph-mon[179688]: osdmap e146: 8 total, 8 up, 8 in 2023-09-23T20:54:55.134 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:54:54 smithi026 ceph-mon[175151]: osdmap e145: 8 total, 8 up, 8 in 2023-09-23T20:54:55.134 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:54:54 smithi026 ceph-mon[175151]: failed to encode map e145 with expected crc 2023-09-23T20:54:55.134 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:54:54 smithi026 ceph-mon[175151]: failed to encode map e145 with expected crc 2023-09-23T20:54:55.134 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:54:54 smithi026 ceph-mon[175151]: failed to encode map e145 with expected crc 2023-09-23T20:54:55.134 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:54:54 smithi026 ceph-mon[175151]: failed to encode map e145 with expected crc 2023-09-23T20:54:55.134 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:54:54 smithi026 ceph-mon[175151]: failed to encode map e145 with expected crc 2023-09-23T20:54:55.134 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:54:54 smithi026 ceph-mon[175151]: failed to encode map e145 with expected crc 2023-09-23T20:54:55.134 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:54:54 smithi026 ceph-mon[175151]: Health check update: Degraded data redundancy: 3210/173700 objects degraded (1.848%), 4 pgs degraded (PG_DEGRADED) 2023-09-23T20:54:55.135 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:54:54 smithi026 ceph-mon[175151]: osdmap e146: 8 total, 8 up, 8 in 2023-09-23T20:54:55.195 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:54:55.222 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:54:54 smithi070 ceph-mon[168282]: osdmap e145: 8 total, 8 up, 8 in 2023-09-23T20:54:55.222 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:54:54 smithi070 ceph-mon[168282]: failed to encode map e145 with expected crc 2023-09-23T20:54:55.222 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:54:54 smithi070 ceph-mon[168282]: failed to encode map e145 with expected crc 2023-09-23T20:54:55.223 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:54:54 smithi070 ceph-mon[168282]: failed to encode map e145 with expected crc 2023-09-23T20:54:55.223 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:54:54 smithi070 ceph-mon[168282]: failed to encode map e145 with expected crc 2023-09-23T20:54:55.223 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:54:54 smithi070 ceph-mon[168282]: failed to encode map e145 with expected crc 2023-09-23T20:54:55.223 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:54:54 smithi070 ceph-mon[168282]: failed to encode map e145 with expected crc 2023-09-23T20:54:55.223 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:54:54 smithi070 ceph-mon[168282]: Health check update: Degraded data redundancy: 3210/173700 objects degraded (1.848%), 4 pgs degraded (PG_DEGRADED) 2023-09-23T20:54:55.223 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:54:54 smithi070 ceph-mon[168282]: osdmap e146: 8 total, 8 up, 8 in 2023-09-23T20:54:55.227 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:54:55.265 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:54:55.268 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:54:55.268 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:54:55.369 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:54:55.398 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:54:55.484 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:54:55.486 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:54:55.486 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:54:55.587 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:54:55.614 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:54:55.747 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:54:55.750 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:54:55.750 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:54:55.851 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:54:55.879 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:54:55.915 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:54:55.918 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:54:55.918 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:54:55.997 INFO:teuthology.orchestra.run.smithi026.stdout:ERROR: (22) Invalid argument 2023-09-23T20:54:55.997 INFO:teuthology.orchestra.run.smithi026.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:54:56.019 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:54:56.059 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:54:55 smithi070 ceph-mon[168282]: pgmap v358: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfilling, 2 active+recovery_wait+degraded+remapped, 89 active+clean, 4 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped; 3.4 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 905 op/s; 3210/168492 objects degraded (1.905%); 14295/168492 objects misplaced (8.484%); 9.4 MiB/s, 231 objects/s recovering 2023-09-23T20:54:56.060 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:54:55 smithi070 ceph-mon[168282]: failed to encode map e146 with expected crc 2023-09-23T20:54:56.060 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:54:55 smithi070 ceph-mon[168282]: failed to encode map e146 with expected crc 2023-09-23T20:54:56.060 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:54:55 smithi070 ceph-mon[168282]: failed to encode map e146 with expected crc 2023-09-23T20:54:56.060 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:54:55 smithi070 ceph-mon[168282]: osdmap e147: 8 total, 8 up, 8 in 2023-09-23T20:54:56.062 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:54:56.093 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:54:56.095 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:54:56.096 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:54:56.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:54:55 smithi026 ceph-mon[175151]: pgmap v358: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfilling, 2 active+recovery_wait+degraded+remapped, 89 active+clean, 4 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped; 3.4 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 905 op/s; 3210/168492 objects degraded (1.905%); 14295/168492 objects misplaced (8.484%); 9.4 MiB/s, 231 objects/s recovering 2023-09-23T20:54:56.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:54:55 smithi026 ceph-mon[175151]: failed to encode map e146 with expected crc 2023-09-23T20:54:56.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:54:55 smithi026 ceph-mon[175151]: failed to encode map e146 with expected crc 2023-09-23T20:54:56.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:54:55 smithi026 ceph-mon[175151]: failed to encode map e146 with expected crc 2023-09-23T20:54:56.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:54:55 smithi026 ceph-mon[175151]: osdmap e147: 8 total, 8 up, 8 in 2023-09-23T20:54:56.137 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:54:55 smithi026 ceph-mon[179688]: pgmap v358: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfilling, 2 active+recovery_wait+degraded+remapped, 89 active+clean, 4 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped; 3.4 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 905 op/s; 3210/168492 objects degraded (1.905%); 14295/168492 objects misplaced (8.484%); 9.4 MiB/s, 231 objects/s recovering 2023-09-23T20:54:56.137 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:54:55 smithi026 ceph-mon[179688]: failed to encode map e146 with expected crc 2023-09-23T20:54:56.137 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:54:55 smithi026 ceph-mon[179688]: failed to encode map e146 with expected crc 2023-09-23T20:54:56.137 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:54:55 smithi026 ceph-mon[179688]: failed to encode map e146 with expected crc 2023-09-23T20:54:56.137 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:54:55 smithi026 ceph-mon[179688]: osdmap e147: 8 total, 8 up, 8 in 2023-09-23T20:54:56.197 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:54:56.252 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:54:56.374 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:54:56.381 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:54:56.381 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:54:56.482 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:54:56.572 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:54:57.135 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:54:56 smithi026 ceph-mon[175151]: failed to encode map e147 with expected crc 2023-09-23T20:54:57.135 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:54:56 smithi026 ceph-mon[175151]: failed to encode map e147 with expected crc 2023-09-23T20:54:57.135 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:54:56 smithi026 ceph-mon[175151]: failed to encode map e147 with expected crc 2023-09-23T20:54:57.135 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:54:56 smithi026 ceph-mon[175151]: failed to encode map e147 with expected crc 2023-09-23T20:54:57.135 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:54:56 smithi026 ceph-mon[175151]: failed to encode map e147 with expected crc 2023-09-23T20:54:57.136 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:54:56 smithi026 ceph-mon[175151]: failed to encode map e147 with expected crc 2023-09-23T20:54:57.136 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:54:56 smithi026 ceph-mon[179688]: failed to encode map e147 with expected crc 2023-09-23T20:54:57.136 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:54:56 smithi026 ceph-mon[179688]: failed to encode map e147 with expected crc 2023-09-23T20:54:57.136 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:54:56 smithi026 ceph-mon[179688]: failed to encode map e147 with expected crc 2023-09-23T20:54:57.136 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:54:56 smithi026 ceph-mon[179688]: failed to encode map e147 with expected crc 2023-09-23T20:54:57.136 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:54:56 smithi026 ceph-mon[179688]: failed to encode map e147 with expected crc 2023-09-23T20:54:57.136 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:54:56 smithi026 ceph-mon[179688]: failed to encode map e147 with expected crc 2023-09-23T20:54:57.262 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:54:57.268 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:54:57.269 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:54:57.363 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_ops_in_flight 2023-09-23T20:54:57.370 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:54:57.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:54:56 smithi070 ceph-mon[168282]: failed to encode map e147 with expected crc 2023-09-23T20:54:57.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:54:56 smithi070 ceph-mon[168282]: failed to encode map e147 with expected crc 2023-09-23T20:54:57.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:54:56 smithi070 ceph-mon[168282]: failed to encode map e147 with expected crc 2023-09-23T20:54:57.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:54:56 smithi070 ceph-mon[168282]: failed to encode map e147 with expected crc 2023-09-23T20:54:57.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:54:56 smithi070 ceph-mon[168282]: failed to encode map e147 with expected crc 2023-09-23T20:54:57.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:54:56 smithi070 ceph-mon[168282]: failed to encode map e147 with expected crc 2023-09-23T20:54:57.459 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:54:57.491 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:54:57.524 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:54:57.528 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:54:57.529 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:54:57.629 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:54:57.723 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:54:57.795 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:54:57.805 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:54:57.806 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:54:57.859 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:54:57.907 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:54:57.972 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:54:57 smithi026 ceph-mon[175151]: pgmap v361: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+remapped+backfilling, 2 active+recovery_wait+degraded+remapped, 89 active+clean, 3 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped; 3.4 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 724 op/s; 3210/166902 objects degraded (1.923%); 13028/166902 objects misplaced (7.806%); 16 MiB/s, 384 objects/s recovering 2023-09-23T20:54:57.972 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:54:57 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:54:57.972 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:54:57 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:54:58.017 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:54:58.083 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:54:58.300 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:54:57 smithi070 ceph-mon[168282]: pgmap v361: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+remapped+backfilling, 2 active+recovery_wait+degraded+remapped, 89 active+clean, 3 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped; 3.4 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 724 op/s; 3210/166902 objects degraded (1.923%); 13028/166902 objects misplaced (7.806%); 16 MiB/s, 384 objects/s recovering 2023-09-23T20:54:58.300 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:54:57 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:54:58.300 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:54:57 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:54:58.332 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:54:57 smithi026 ceph-mon[179688]: pgmap v361: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+remapped+backfilling, 2 active+recovery_wait+degraded+remapped, 89 active+clean, 3 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped; 3.4 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 724 op/s; 3210/166902 objects degraded (1.923%); 13028/166902 objects misplaced (7.806%); 16 MiB/s, 384 objects/s recovering 2023-09-23T20:54:58.332 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:54:57 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:54:58.333 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:54:57 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:54:58.458 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:54:58.480 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:54:58.480 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:54:58.581 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:54:58.663 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:54:58.738 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:54:58.749 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:54:58.749 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:54:58.850 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:54:58.893 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:54:59.051 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:54:59.054 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:54:59.054 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:54:59.155 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:54:59.196 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:54:59.256 INFO:teuthology.orchestra.run.smithi026.stdout: 2023-09-23T20:54:59.269 INFO:teuthology.orchestra.run.smithi026.stderr:dumped all 2023-09-23T20:54:59.297 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:54:59.301 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:54:59.302 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:54:59.402 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:54:59.429 INFO:teuthology.orchestra.run.smithi070.stdout:ERROR: (22) Invalid argument 2023-09-23T20:54:59.430 INFO:teuthology.orchestra.run.smithi070.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:54:59.449 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:54:59.511 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:54:59.518 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:54:59.519 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:54:59.616 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:54:59 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[164029]: ::ffff:172.21.15.70 - - [23/Sep/2023:20:54:59] "GET /metrics HTTP/1.1" 200 36352 "" "Prometheus/2.43.0" 2023-09-23T20:54:59.619 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:54:59.702 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:54:59.884 INFO:teuthology.orchestra.run.smithi026.stdout:{"pg_ready":true,"pg_map":{"version":362,"stamp":"2023-09-23T20:54:58.266559+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":3564574648,"num_objects":54407,"num_object_clones":0,"num_object_copies":163221,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3210,"num_objects_misplaced":12659,"num_objects_unfound":0,"num_objects_dirty":54407,"num_whiteouts":0,"num_read":501,"num_read_kb":925,"num_write":148165,"num_write_kb":6474318,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":14344,"num_bytes_recovered":765599504,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":194710,"ondisk_log_size":194710,"up":297,"acting":295,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":325,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":11834904,"kb_used_data":11045592,"kb_used_omap":126,"kb_used_meta":788865,"kb_avail":738126312,"statfs":{"total":767960285184,"available":755841343488,"internally_reserved":0,"allocated":11310686208,"data_stored":11305730092,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":129413,"internal_metadata":807798395},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":7,"apply_latency_ms":7,"commit_latency_ns":7000000,"apply_latency_ns":7000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-228917248,"num_objects":-3493,"num_object_clones":0,"num_object_copies":-10479,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":-2404,"num_objects_unfound":0,"num_objects_dirty":-3493,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3493,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2268,"num_bytes_recovered":95551488,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"6.004282"},"pg_stats":[{"pgid":"3.1a","version":"68'1","reported_seq":24,"reported_epoch":146,"state":"active+clean","last_fresh":"2023-09-23T20:54:54.891507+0000","last_change":"2023-09-23T20:54:13.946695+0000","last_active":"2023-09-23T20:54:54.891507+0000","last_peered":"2023-09-23T20:54:54.891507+0000","last_clean":"2023-09-23T20:54:54.891507+0000","last_became_active":"2023-09-23T20:54:13.945399+0000","last_became_peered":"2023-09-23T20:54:13.945399+0000","last_unstale":"2023-09-23T20:54:54.891507+0000","last_undegraded":"2023-09-23T20:54:54.891507+0000","last_fullsized":"2023-09-23T20:54:54.891507+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:32:52.365464+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":135,"reported_epoch":144,"state":"active+clean","last_fresh":"2023-09-23T20:54:51.911554+0000","last_change":"2023-09-23T20:54:13.942019+0000","last_active":"2023-09-23T20:54:51.911554+0000","last_peered":"2023-09-23T20:54:51.911554+0000","last_clean":"2023-09-23T20:54:51.911554+0000","last_became_active":"2023-09-23T20:54:13.941647+0000","last_became_peered":"2023-09-23T20:54:13.941647+0000","last_unstale":"2023-09-23T20:54:51.911554+0000","last_undegraded":"2023-09-23T20:54:51.911554+0000","last_fullsized":"2023-09-23T20:54:51.911554+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:30:54.400944+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":113,"reported_epoch":145,"state":"active+clean","last_fresh":"2023-09-23T20:54:53.951256+0000","last_change":"2023-09-23T20:54:13.941394+0000","last_active":"2023-09-23T20:54:53.951256+0000","last_peered":"2023-09-23T20:54:53.951256+0000","last_clean":"2023-09-23T20:54:53.951256+0000","last_became_active":"2023-09-23T20:54:13.941042+0000","last_became_peered":"2023-09-23T20:54:13.941042+0000","last_unstale":"2023-09-23T20:54:53.951256+0000","last_undegraded":"2023-09-23T20:54:53.951256+0000","last_fullsized":"2023-09-23T20:54:53.951256+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:19:41.401184+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":110,"reported_epoch":145,"state":"active+clean","last_fresh":"2023-09-23T20:54:53.953083+0000","last_change":"2023-09-23T20:54:44.037156+0000","last_active":"2023-09-23T20:54:53.953083+0000","last_peered":"2023-09-23T20:54:53.953083+0000","last_clean":"2023-09-23T20:54:53.953083+0000","last_became_active":"2023-09-23T20:54:44.036840+0000","last_became_peered":"2023-09-23T20:54:44.036840+0000","last_unstale":"2023-09-23T20:54:53.953083+0000","last_undegraded":"2023-09-23T20:54:53.953083+0000","last_fullsized":"2023-09-23T20:54:53.953083+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:39:01.707474+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":126,"reported_epoch":144,"state":"active+clean","last_fresh":"2023-09-23T20:54:51.909692+0000","last_change":"2023-09-23T20:51:10.833613+0000","last_active":"2023-09-23T20:54:51.909692+0000","last_peered":"2023-09-23T20:54:51.909692+0000","last_clean":"2023-09-23T20:54:51.909692+0000","last_became_active":"2023-09-23T20:51:10.833208+0000","last_became_peered":"2023-09-23T20:51:10.833208+0000","last_unstale":"2023-09-23T20:54:51.909692+0000","last_undegraded":"2023-09-23T20:54:51.909692+0000","last_fullsized":"2023-09-23T20:54:51.909692+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:51:28.246014+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":126,"reported_epoch":146,"state":"active+clean","last_fresh":"2023-09-23T20:54:54.889178+0000","last_change":"2023-09-23T20:54:31.892079+0000","last_active":"2023-09-23T20:54:54.889178+0000","last_peered":"2023-09-23T20:54:54.889178+0000","last_clean":"2023-09-23T20:54:54.889178+0000","last_became_active":"2023-09-23T20:54:31.891755+0000","last_became_peered":"2023-09-23T20:54:31.891755+0000","last_unstale":"2023-09-23T20:54:54.889178+0000","last_undegraded":"2023-09-23T20:54:54.889178+0000","last_fullsized":"2023-09-23T20:54:54.889178+0000","mapping_epoch":136,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:05:35.271620+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.12","version":"147'7821","reported_seq":9255,"reported_epoch":147,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T20:54:56.155484+0000","last_change":"2023-09-23T20:54:31.904491+0000","last_active":"2023-09-23T20:54:56.155484+0000","last_peered":"2023-09-23T20:54:56.155484+0000","last_clean":"2023-09-23T20:53:27.905324+0000","last_became_active":"2023-09-23T20:54:14.975981+0000","last_became_peered":"2023-09-23T20:54:14.975981+0000","last_unstale":"2023-09-23T20:54:56.155484+0000","last_undegraded":"2023-09-23T20:54:31.903975+0000","last_fullsized":"2023-09-23T20:54:14.921351+0000","mapping_epoch":133,"log_start":"107'800","ondisk_log_start":"107'800","created":128,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:52:13.843048+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:52:13.843048+0000","last_clean_scrub_stamp":"2023-09-23T20:52:13.843048+0000","objects_scrubbed":0,"log_size":7021,"log_dups_size":0,"ondisk_log_size":7021,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":110886912,"num_objects":1692,"num_object_clones":0,"num_object_copies":5076,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":818,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1692,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4499,"num_write_kb":198112,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":678,"num_bytes_recovered":44433408,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[2,3],"avail_no_missing":["2","3"],"object_location_counts":[{"shards":"2,3","objects":1284},{"shards":"2,3,6","objects":408}],"blocked_by":[],"up_primary":4,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":119,"reported_epoch":145,"state":"active+clean","last_fresh":"2023-09-23T20:54:53.953920+0000","last_change":"2023-09-23T20:54:31.885514+0000","last_active":"2023-09-23T20:54:53.953920+0000","last_peered":"2023-09-23T20:54:53.953920+0000","last_clean":"2023-09-23T20:54:53.953920+0000","last_became_active":"2023-09-23T20:54:31.884986+0000","last_became_peered":"2023-09-23T20:54:31.884986+0000","last_unstale":"2023-09-23T20:54:53.953920+0000","last_undegraded":"2023-09-23T20:54:53.953920+0000","last_fullsized":"2023-09-23T20:54:53.953920+0000","mapping_epoch":136,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:26:08.100220+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":108,"reported_epoch":144,"state":"active+clean","last_fresh":"2023-09-23T20:54:51.909759+0000","last_change":"2023-09-23T20:51:10.833424+0000","last_active":"2023-09-23T20:54:51.909759+0000","last_peered":"2023-09-23T20:54:51.909759+0000","last_clean":"2023-09-23T20:54:51.909759+0000","last_became_active":"2023-09-23T20:51:10.832808+0000","last_became_peered":"2023-09-23T20:51:10.832808+0000","last_unstale":"2023-09-23T20:54:51.909759+0000","last_undegraded":"2023-09-23T20:54:51.909759+0000","last_fullsized":"2023-09-23T20:54:51.909759+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:32:42.819583+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.13","version":"147'8083","reported_seq":8149,"reported_epoch":147,"state":"active+clean","last_fresh":"2023-09-23T20:54:57.098041+0000","last_change":"2023-09-23T20:54:23.034745+0000","last_active":"2023-09-23T20:54:57.098041+0000","last_peered":"2023-09-23T20:54:57.098041+0000","last_clean":"2023-09-23T20:54:57.098041+0000","last_became_active":"2023-09-23T20:54:03.761198+0000","last_became_peered":"2023-09-23T20:54:03.761198+0000","last_unstale":"2023-09-23T20:54:57.098041+0000","last_undegraded":"2023-09-23T20:54:57.098041+0000","last_fullsized":"2023-09-23T20:54:57.098041+0000","mapping_epoch":128,"log_start":"0'0","ondisk_log_start":"0'0","created":128,"last_epoch_clean":129,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:52:13.843048+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:52:13.843048+0000","last_clean_scrub_stamp":"2023-09-23T20:52:13.843048+0000","objects_scrubbed":0,"log_size":8083,"log_dups_size":0,"ondisk_log_size":8083,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":104759296,"num_objects":1599,"num_object_clones":0,"num_object_copies":4797,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1599,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4714,"num_write_kb":201984,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,2],"acting":[7,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":20,"reported_epoch":146,"state":"active+clean","last_fresh":"2023-09-23T20:54:54.892459+0000","last_change":"2023-09-23T20:54:31.890678+0000","last_active":"2023-09-23T20:54:54.892459+0000","last_peered":"2023-09-23T20:54:54.892459+0000","last_clean":"2023-09-23T20:54:54.892459+0000","last_became_active":"2023-09-23T20:54:31.890469+0000","last_became_peered":"2023-09-23T20:54:31.890469+0000","last_unstale":"2023-09-23T20:54:54.892459+0000","last_undegraded":"2023-09-23T20:54:54.892459+0000","last_fullsized":"2023-09-23T20:54:54.892459+0000","mapping_epoch":136,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:55:08.790146+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":69,"reported_epoch":144,"state":"active+clean","last_fresh":"2023-09-23T20:54:51.909508+0000","last_change":"2023-09-23T20:51:10.833790+0000","last_active":"2023-09-23T20:54:51.909508+0000","last_peered":"2023-09-23T20:54:51.909508+0000","last_clean":"2023-09-23T20:54:51.909508+0000","last_became_active":"2023-09-23T20:51:10.832804+0000","last_became_peered":"2023-09-23T20:51:10.832804+0000","last_unstale":"2023-09-23T20:54:51.909508+0000","last_undegraded":"2023-09-23T20:54:51.909508+0000","last_fullsized":"2023-09-23T20:54:51.909508+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:27:42.601674+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.10","version":"146'8100","reported_seq":8168,"reported_epoch":146,"state":"active+clean","last_fresh":"2023-09-23T20:54:55.155875+0000","last_change":"2023-09-23T20:54:25.189758+0000","last_active":"2023-09-23T20:54:55.155875+0000","last_peered":"2023-09-23T20:54:55.155875+0000","last_clean":"2023-09-23T20:54:55.155875+0000","last_became_active":"2023-09-23T20:54:03.824384+0000","last_became_peered":"2023-09-23T20:54:03.824384+0000","last_unstale":"2023-09-23T20:54:55.155875+0000","last_undegraded":"2023-09-23T20:54:55.155875+0000","last_fullsized":"2023-09-23T20:54:55.155875+0000","mapping_epoch":128,"log_start":"0'0","ondisk_log_start":"0'0","created":128,"last_epoch_clean":129,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:52:13.843048+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:52:13.843048+0000","last_clean_scrub_stamp":"2023-09-23T20:52:13.843048+0000","objects_scrubbed":0,"log_size":8100,"log_dups_size":0,"ondisk_log_size":8100,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":109740032,"num_objects":1675,"num_object_clones":0,"num_object_copies":5025,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1675,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4686,"num_write_kb":203520,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":142,"reported_epoch":146,"state":"active+clean","last_fresh":"2023-09-23T20:54:54.894598+0000","last_change":"2023-09-23T20:54:13.949180+0000","last_active":"2023-09-23T20:54:54.894598+0000","last_peered":"2023-09-23T20:54:54.894598+0000","last_clean":"2023-09-23T20:54:54.894598+0000","last_became_active":"2023-09-23T20:54:13.936669+0000","last_became_peered":"2023-09-23T20:54:13.936669+0000","last_unstale":"2023-09-23T20:54:54.894598+0000","last_undegraded":"2023-09-23T20:54:54.894598+0000","last_fullsized":"2023-09-23T20:54:54.894598+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:59:22.400515+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":95,"reported_epoch":144,"state":"active+clean","last_fresh":"2023-09-23T20:54:51.907831+0000","last_change":"2023-09-23T20:50:37.982103+0000","last_active":"2023-09-23T20:54:51.907831+0000","last_peered":"2023-09-23T20:54:51.907831+0000","last_clean":"2023-09-23T20:54:51.907831+0000","last_became_active":"2023-09-23T20:50:37.981791+0000","last_became_peered":"2023-09-23T20:50:37.981791+0000","last_unstale":"2023-09-23T20:54:51.907831+0000","last_undegraded":"2023-09-23T20:54:51.907831+0000","last_fullsized":"2023-09-23T20:54:51.907831+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:32:52.286186+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.11","version":"145'8044","reported_seq":8105,"reported_epoch":145,"state":"active+clean","last_fresh":"2023-09-23T20:54:54.030099+0000","last_change":"2023-09-23T20:54:03.779005+0000","last_active":"2023-09-23T20:54:54.030099+0000","last_peered":"2023-09-23T20:54:54.030099+0000","last_clean":"2023-09-23T20:54:54.030099+0000","last_became_active":"2023-09-23T20:54:03.774821+0000","last_became_peered":"2023-09-23T20:54:03.774821+0000","last_unstale":"2023-09-23T20:54:54.030099+0000","last_undegraded":"2023-09-23T20:54:54.030099+0000","last_fullsized":"2023-09-23T20:54:54.030099+0000","mapping_epoch":128,"log_start":"107'700","ondisk_log_start":"107'700","created":128,"last_epoch_clean":129,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:52:13.843048+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:52:13.843048+0000","last_clean_scrub_stamp":"2023-09-23T20:52:13.843048+0000","objects_scrubbed":0,"log_size":7344,"log_dups_size":0,"ondisk_log_size":7344,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":115703808,"num_objects":1766,"num_object_clones":0,"num_object_copies":5298,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1766,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4630,"num_write_kb":204640,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"79'22","reported_seq":34,"reported_epoch":146,"state":"active+clean","last_fresh":"2023-09-23T20:54:54.891353+0000","last_change":"2023-09-23T20:54:13.965683+0000","last_active":"2023-09-23T20:54:54.891353+0000","last_peered":"2023-09-23T20:54:54.891353+0000","last_clean":"2023-09-23T20:54:54.891353+0000","last_became_active":"2023-09-23T20:54:13.935921+0000","last_became_peered":"2023-09-23T20:54:13.935921+0000","last_unstale":"2023-09-23T20:54:54.891353+0000","last_undegraded":"2023-09-23T20:54:54.891353+0000","last_fullsized":"2023-09-23T20:54:54.891353+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:43:31.562279+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4734,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":24,"reported_epoch":146,"state":"active+clean","last_fresh":"2023-09-23T20:54:54.891840+0000","last_change":"2023-09-23T20:54:13.951899+0000","last_active":"2023-09-23T20:54:54.891840+0000","last_peered":"2023-09-23T20:54:54.891840+0000","last_clean":"2023-09-23T20:54:54.891840+0000","last_became_active":"2023-09-23T20:54:13.950493+0000","last_became_peered":"2023-09-23T20:54:13.950493+0000","last_unstale":"2023-09-23T20:54:54.891840+0000","last_undegraded":"2023-09-23T20:54:54.891840+0000","last_fullsized":"2023-09-23T20:54:54.891840+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:47:43.168104+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.16","version":"144'7888","reported_seq":10011,"reported_epoch":144,"state":"active+clean","last_fresh":"2023-09-23T20:54:53.789153+0000","last_change":"2023-09-23T20:54:44.045342+0000","last_active":"2023-09-23T20:54:53.789153+0000","last_peered":"2023-09-23T20:54:53.789153+0000","last_clean":"2023-09-23T20:54:53.789153+0000","last_became_active":"2023-09-23T20:54:44.031790+0000","last_became_peered":"2023-09-23T20:54:44.031790+0000","last_unstale":"2023-09-23T20:54:53.789153+0000","last_undegraded":"2023-09-23T20:54:53.789153+0000","last_fullsized":"2023-09-23T20:54:53.789153+0000","mapping_epoch":142,"log_start":"107'900","ondisk_log_start":"107'900","created":128,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:52:13.843048+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:52:13.843048+0000","last_clean_scrub_stamp":"2023-09-23T20:52:13.843048+0000","objects_scrubbed":0,"log_size":6988,"log_dups_size":0,"ondisk_log_size":6988,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":114163712,"num_objects":1742,"num_object_clones":0,"num_object_copies":5226,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1742,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4529,"num_write_kb":200672,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1019,"num_bytes_recovered":38404096,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":141,"reported_epoch":146,"state":"active+clean","last_fresh":"2023-09-23T20:54:54.894316+0000","last_change":"2023-09-23T20:54:13.957131+0000","last_active":"2023-09-23T20:54:54.894316+0000","last_peered":"2023-09-23T20:54:54.894316+0000","last_clean":"2023-09-23T20:54:54.894316+0000","last_became_active":"2023-09-23T20:54:13.939561+0000","last_became_peered":"2023-09-23T20:54:13.939561+0000","last_unstale":"2023-09-23T20:54:54.894316+0000","last_undegraded":"2023-09-23T20:54:54.894316+0000","last_fullsized":"2023-09-23T20:54:54.894316+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:56:43.943284+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":108,"reported_epoch":144,"state":"active+clean","last_fresh":"2023-09-23T20:54:51.912683+0000","last_change":"2023-09-23T20:51:10.831262+0000","last_active":"2023-09-23T20:54:51.912683+0000","last_peered":"2023-09-23T20:54:51.912683+0000","last_clean":"2023-09-23T20:54:51.912683+0000","last_became_active":"2023-09-23T20:51:10.830931+0000","last_became_peered":"2023-09-23T20:51:10.830931+0000","last_unstale":"2023-09-23T20:54:51.912683+0000","last_undegraded":"2023-09-23T20:54:51.912683+0000","last_fullsized":"2023-09-23T20:54:51.912683+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:51:54.982505+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.17","version":"145'8016","reported_seq":8543,"reported_epoch":145,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2023-09-23T20:54:54.037552+0000","last_change":"2023-09-23T20:54:33.016927+0000","last_active":"2023-09-23T20:54:54.037552+0000","last_peered":"2023-09-23T20:54:54.037552+0000","last_clean":"2023-09-23T20:53:27.924842+0000","last_became_active":"2023-09-23T20:54:32.987997+0000","last_became_peered":"2023-09-23T20:54:32.987997+0000","last_unstale":"2023-09-23T20:54:54.037552+0000","last_undegraded":"2023-09-23T20:54:32.932600+0000","last_fullsized":"2023-09-23T20:54:54.037552+0000","mapping_epoch":137,"log_start":"107'1000","ondisk_log_start":"107'1000","created":128,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:52:13.843048+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:52:13.843048+0000","last_clean_scrub_stamp":"2023-09-23T20:52:13.843048+0000","objects_scrubbed":0,"log_size":7016,"log_dups_size":0,"ondisk_log_size":7016,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":116850688,"num_objects":1783,"num_object_clones":0,"num_object_copies":5349,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":781,"num_objects_misplaced":1783,"num_objects_unfound":0,"num_objects_dirty":1783,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4596,"num_write_kb":204128,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":222,"num_bytes_recovered":14417920,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[3,4,1],"avail_no_missing":["3","1"],"object_location_counts":[{"shards":"1,3,4","objects":1453},{"shards":"1,3,7","objects":330}],"blocked_by":[],"up_primary":5,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":110,"reported_epoch":145,"state":"active+clean","last_fresh":"2023-09-23T20:54:53.953082+0000","last_change":"2023-09-23T20:54:44.037431+0000","last_active":"2023-09-23T20:54:53.953082+0000","last_peered":"2023-09-23T20:54:53.953082+0000","last_clean":"2023-09-23T20:54:53.953082+0000","last_became_active":"2023-09-23T20:54:44.037102+0000","last_became_peered":"2023-09-23T20:54:44.037102+0000","last_unstale":"2023-09-23T20:54:53.953082+0000","last_undegraded":"2023-09-23T20:54:53.953082+0000","last_fullsized":"2023-09-23T20:54:53.953082+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:12:11.074632+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":20,"reported_epoch":146,"state":"active+clean","last_fresh":"2023-09-23T20:54:54.893183+0000","last_change":"2023-09-23T20:54:31.887814+0000","last_active":"2023-09-23T20:54:54.893183+0000","last_peered":"2023-09-23T20:54:54.893183+0000","last_clean":"2023-09-23T20:54:54.893183+0000","last_became_active":"2023-09-23T20:54:31.887195+0000","last_became_peered":"2023-09-23T20:54:31.887195+0000","last_unstale":"2023-09-23T20:54:54.893183+0000","last_undegraded":"2023-09-23T20:54:54.893183+0000","last_fullsized":"2023-09-23T20:54:54.893183+0000","mapping_epoch":136,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:10:52.089553+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"6.14","version":"145'7958","reported_seq":8032,"reported_epoch":145,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-23T20:54:54.045396+0000","last_change":"2023-09-23T20:54:33.034052+0000","last_active":"2023-09-23T20:54:54.045396+0000","last_peered":"2023-09-23T20:54:54.045396+0000","last_clean":"2023-09-23T20:54:30.881380+0000","last_became_active":"2023-09-23T20:54:33.017063+0000","last_became_peered":"2023-09-23T20:54:33.017063+0000","last_unstale":"2023-09-23T20:54:54.045396+0000","last_undegraded":"2023-09-23T20:54:54.045396+0000","last_fullsized":"2023-09-23T20:54:54.045396+0000","mapping_epoch":137,"log_start":"107'600","ondisk_log_start":"107'600","created":128,"last_epoch_clean":132,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:52:13.843048+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:52:13.843048+0000","last_clean_scrub_stamp":"2023-09-23T20:52:13.843048+0000","objects_scrubbed":0,"log_size":7358,"log_dups_size":0,"ondisk_log_size":7358,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":111837184,"num_objects":1707,"num_object_clones":0,"num_object_copies":5121,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1707,"num_objects_unfound":0,"num_objects_dirty":1707,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4589,"num_write_kb":201440,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,3],"acting":[1,3,6],"avail_no_missing":["1","3","6"],"object_location_counts":[{"shards":"1,3,6","objects":1707}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":110,"reported_epoch":146,"state":"active+clean","last_fresh":"2023-09-23T20:54:54.894752+0000","last_change":"2023-09-23T20:54:13.937044+0000","last_active":"2023-09-23T20:54:54.894752+0000","last_peered":"2023-09-23T20:54:54.894752+0000","last_clean":"2023-09-23T20:54:54.894752+0000","last_became_active":"2023-09-23T20:54:13.936730+0000","last_became_peered":"2023-09-23T20:54:13.936730+0000","last_unstale":"2023-09-23T20:54:54.894752+0000","last_undegraded":"2023-09-23T20:54:54.894752+0000","last_fullsized":"2023-09-23T20:54:54.894752+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:48:19.881616+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":101,"reported_epoch":145,"state":"active+clean","last_fresh":"2023-09-23T20:54:53.953007+0000","last_change":"2023-09-23T20:51:10.840849+0000","last_active":"2023-09-23T20:54:53.953007+0000","last_peered":"2023-09-23T20:54:53.953007+0000","last_clean":"2023-09-23T20:54:53.953007+0000","last_became_active":"2023-09-23T20:51:10.840593+0000","last_became_peered":"2023-09-23T20:51:10.840593+0000","last_unstale":"2023-09-23T20:54:53.953007+0000","last_undegraded":"2023-09-23T20:54:53.953007+0000","last_fullsized":"2023-09-23T20:54:53.953007+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:30:52.297625+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.15","version":"147'7985","reported_seq":8051,"reported_epoch":147,"state":"active+clean","last_fresh":"2023-09-23T20:54:57.103852+0000","last_change":"2023-09-23T20:54:20.982582+0000","last_active":"2023-09-23T20:54:57.103852+0000","last_peered":"2023-09-23T20:54:57.103852+0000","last_clean":"2023-09-23T20:54:57.103852+0000","last_became_active":"2023-09-23T20:54:03.787047+0000","last_became_peered":"2023-09-23T20:54:03.787047+0000","last_unstale":"2023-09-23T20:54:57.103852+0000","last_undegraded":"2023-09-23T20:54:57.103852+0000","last_fullsized":"2023-09-23T20:54:57.103852+0000","mapping_epoch":128,"log_start":"0'0","ondisk_log_start":"0'0","created":128,"last_epoch_clean":129,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:52:13.843048+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:52:13.843048+0000","last_clean_scrub_stamp":"2023-09-23T20:52:13.843048+0000","objects_scrubbed":0,"log_size":7985,"log_dups_size":0,"ondisk_log_size":7985,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":107872256,"num_objects":1646,"num_object_clones":0,"num_object_copies":4938,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1646,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4637,"num_write_kb":201056,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,3],"acting":[7,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":155,"reported_epoch":146,"state":"active+clean","last_fresh":"2023-09-23T20:54:54.888595+0000","last_change":"2023-09-23T20:54:31.911149+0000","last_active":"2023-09-23T20:54:54.888595+0000","last_peered":"2023-09-23T20:54:54.888595+0000","last_clean":"2023-09-23T20:54:54.888595+0000","last_became_active":"2023-09-23T20:54:31.891490+0000","last_became_peered":"2023-09-23T20:54:31.891490+0000","last_unstale":"2023-09-23T20:54:54.888595+0000","last_undegraded":"2023-09-23T20:54:54.888595+0000","last_fullsized":"2023-09-23T20:54:54.888595+0000","mapping_epoch":136,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:13:09.636158+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":69,"reported_epoch":146,"state":"active+clean","last_fresh":"2023-09-23T20:54:54.894641+0000","last_change":"2023-09-23T20:54:44.051624+0000","last_active":"2023-09-23T20:54:54.894641+0000","last_peered":"2023-09-23T20:54:54.894641+0000","last_clean":"2023-09-23T20:54:54.894641+0000","last_became_active":"2023-09-23T20:54:44.051318+0000","last_became_peered":"2023-09-23T20:54:44.051318+0000","last_unstale":"2023-09-23T20:54:54.894641+0000","last_undegraded":"2023-09-23T20:54:54.894641+0000","last_fullsized":"2023-09-23T20:54:54.894641+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:39:44.612145+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,2],"acting":[7,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"6.a","version":"146'9285","reported_seq":11546,"reported_epoch":146,"state":"active+remapped+backfilling","last_fresh":"2023-09-23T20:54:55.923091+0000","last_change":"2023-09-23T20:54:53.279130+0000","last_active":"2023-09-23T20:54:55.923091+0000","last_peered":"2023-09-23T20:54:55.923091+0000","last_clean":"2023-09-23T20:54:30.834075+0000","last_became_active":"2023-09-23T20:54:32.996301+0000","last_became_peered":"2023-09-23T20:54:32.996301+0000","last_unstale":"2023-09-23T20:54:55.923091+0000","last_undegraded":"2023-09-23T20:54:55.923091+0000","last_fullsized":"2023-09-23T20:54:55.923091+0000","mapping_epoch":137,"log_start":"108'1739","ondisk_log_start":"108'1739","created":102,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:52:13.843048+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:52:13.843048+0000","last_clean_scrub_stamp":"2023-09-23T20:52:13.843048+0000","objects_scrubbed":0,"log_size":7546,"log_dups_size":0,"ondisk_log_size":7546,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:01:13.104682+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":224985088,"num_objects":3433,"num_object_clones":0,"num_object_copies":10299,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":2341,"num_objects_unfound":0,"num_objects_dirty":3433,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9285,"num_write_kb":406976,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1100,"num_bytes_recovered":72089600,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,0],"acting":[0,6,2],"avail_no_missing":["0","2","6"],"object_location_counts":[{"shards":"0,2,6","objects":3433}],"blocked_by":[],"up_primary":5,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":150,"reported_epoch":146,"state":"active+clean","last_fresh":"2023-09-23T20:54:54.894490+0000","last_change":"2023-09-23T20:54:14.007167+0000","last_active":"2023-09-23T20:54:54.894490+0000","last_peered":"2023-09-23T20:54:54.894490+0000","last_clean":"2023-09-23T20:54:54.894490+0000","last_became_active":"2023-09-23T20:54:13.939827+0000","last_became_peered":"2023-09-23T20:54:13.939827+0000","last_unstale":"2023-09-23T20:54:54.894490+0000","last_undegraded":"2023-09-23T20:54:54.894490+0000","last_fullsized":"2023-09-23T20:54:54.894490+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:12:48.477909+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":66,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":20,"reported_epoch":146,"state":"active+clean","last_fresh":"2023-09-23T20:54:54.891652+0000","last_change":"2023-09-23T20:54:31.887139+0000","last_active":"2023-09-23T20:54:54.891652+0000","last_peered":"2023-09-23T20:54:54.891652+0000","last_clean":"2023-09-23T20:54:54.891652+0000","last_became_active":"2023-09-23T20:54:31.886814+0000","last_became_peered":"2023-09-23T20:54:31.886814+0000","last_unstale":"2023-09-23T20:54:54.891652+0000","last_undegraded":"2023-09-23T20:54:54.891652+0000","last_fullsized":"2023-09-23T20:54:54.891652+0000","mapping_epoch":136,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:03:16.402782+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"6.b","version":"145'9024","reported_seq":9077,"reported_epoch":145,"state":"active+clean","last_fresh":"2023-09-23T20:54:54.060183+0000","last_change":"2023-09-23T20:52:14.868057+0000","last_active":"2023-09-23T20:54:54.060183+0000","last_peered":"2023-09-23T20:54:54.060183+0000","last_clean":"2023-09-23T20:54:54.060183+0000","last_became_active":"2023-09-23T20:52:14.867734+0000","last_became_peered":"2023-09-23T20:52:14.867734+0000","last_unstale":"2023-09-23T20:54:54.060183+0000","last_undegraded":"2023-09-23T20:54:54.060183+0000","last_fullsized":"2023-09-23T20:54:54.060183+0000","mapping_epoch":102,"log_start":"113'3246","ondisk_log_start":"113'3246","created":102,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:52:13.843048+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:52:13.843048+0000","last_clean_scrub_stamp":"2023-09-23T20:52:13.843048+0000","objects_scrubbed":0,"log_size":5778,"log_dups_size":0,"ondisk_log_size":5778,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:40:31.506977+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":220594176,"num_objects":3366,"num_object_clones":0,"num_object_copies":10098,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3366,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9024,"num_write_kb":396480,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":137,"reported_epoch":146,"state":"active+clean","last_fresh":"2023-09-23T20:54:54.894387+0000","last_change":"2023-09-23T20:54:14.024448+0000","last_active":"2023-09-23T20:54:54.894387+0000","last_peered":"2023-09-23T20:54:54.894387+0000","last_clean":"2023-09-23T20:54:54.894387+0000","last_became_active":"2023-09-23T20:54:13.944740+0000","last_became_peered":"2023-09-23T20:54:13.944740+0000","last_unstale":"2023-09-23T20:54:54.894387+0000","last_undegraded":"2023-09-23T20:54:54.894387+0000","last_fullsized":"2023-09-23T20:54:54.894387+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:24:20.877376+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":34,"reported_epoch":146,"state":"active+clean","last_fresh":"2023-09-23T20:54:54.891294+0000","last_change":"2023-09-23T20:54:44.061343+0000","last_active":"2023-09-23T20:54:54.891294+0000","last_peered":"2023-09-23T20:54:54.891294+0000","last_clean":"2023-09-23T20:54:54.891294+0000","last_became_active":"2023-09-23T20:54:44.061028+0000","last_became_peered":"2023-09-23T20:54:44.061028+0000","last_unstale":"2023-09-23T20:54:54.891294+0000","last_undegraded":"2023-09-23T20:54:54.891294+0000","last_fullsized":"2023-09-23T20:54:54.891294+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:12:40.019218+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.8","version":"147'8124","reported_seq":8193,"reported_epoch":147,"state":"active+clean","last_fresh":"2023-09-23T20:54:57.097906+0000","last_change":"2023-09-23T20:54:26.039442+0000","last_active":"2023-09-23T20:54:57.097906+0000","last_peered":"2023-09-23T20:54:57.097906+0000","last_clean":"2023-09-23T20:54:57.097906+0000","last_became_active":"2023-09-23T20:54:03.758672+0000","last_became_peered":"2023-09-23T20:54:03.758672+0000","last_unstale":"2023-09-23T20:54:57.097906+0000","last_undegraded":"2023-09-23T20:54:57.097906+0000","last_fullsized":"2023-09-23T20:54:57.097906+0000","mapping_epoch":102,"log_start":"106'100","ondisk_log_start":"106'100","created":102,"last_epoch_clean":129,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:52:13.843048+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:52:13.843048+0000","last_clean_scrub_stamp":"2023-09-23T20:52:13.843048+0000","objects_scrubbed":0,"log_size":8024,"log_dups_size":0,"ondisk_log_size":8024,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":107872256,"num_objects":1646,"num_object_clones":0,"num_object_copies":4938,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1646,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4713,"num_write_kb":203488,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,3],"acting":[7,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":119,"reported_epoch":146,"state":"active+clean","last_fresh":"2023-09-23T20:54:54.894299+0000","last_change":"2023-09-23T20:54:31.907775+0000","last_active":"2023-09-23T20:54:54.894299+0000","last_peered":"2023-09-23T20:54:54.894299+0000","last_clean":"2023-09-23T20:54:54.894299+0000","last_became_active":"2023-09-23T20:54:31.892001+0000","last_became_peered":"2023-09-23T20:54:31.892001+0000","last_unstale":"2023-09-23T20:54:54.894299+0000","last_undegraded":"2023-09-23T20:54:54.894299+0000","last_fullsized":"2023-09-23T20:54:54.894299+0000","mapping_epoch":136,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:31:57.847309+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":140,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":106,"reported_epoch":146,"state":"active+clean","last_fresh":"2023-09-23T20:54:54.888817+0000","last_change":"2023-09-23T20:50:05.867973+0000","last_active":"2023-09-23T20:54:54.888817+0000","last_peered":"2023-09-23T20:54:54.888817+0000","last_clean":"2023-09-23T20:54:54.888817+0000","last_became_active":"2023-09-23T20:50:05.867622+0000","last_became_peered":"2023-09-23T20:50:05.867622+0000","last_unstale":"2023-09-23T20:54:54.888817+0000","last_undegraded":"2023-09-23T20:54:54.888817+0000","last_fullsized":"2023-09-23T20:54:54.888817+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:10:12.617597+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.9","version":"146'8083","reported_seq":8150,"reported_epoch":146,"state":"active+clean","last_fresh":"2023-09-23T20:54:55.093119+0000","last_change":"2023-09-23T20:54:29.140827+0000","last_active":"2023-09-23T20:54:55.093119+0000","last_peered":"2023-09-23T20:54:55.093119+0000","last_clean":"2023-09-23T20:54:55.093119+0000","last_became_active":"2023-09-23T20:54:03.701452+0000","last_became_peered":"2023-09-23T20:54:03.701452+0000","last_unstale":"2023-09-23T20:54:55.093119+0000","last_undegraded":"2023-09-23T20:54:55.093119+0000","last_fullsized":"2023-09-23T20:54:55.093119+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":102,"last_epoch_clean":129,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:52:13.843048+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:52:13.843048+0000","last_clean_scrub_stamp":"2023-09-23T20:52:13.843048+0000","objects_scrubbed":0,"log_size":8083,"log_dups_size":0,"ondisk_log_size":8083,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":113344512,"num_objects":1729,"num_object_clones":0,"num_object_copies":5187,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1729,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4664,"num_write_kb":204608,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,2],"acting":[0,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":20,"reported_epoch":146,"state":"active+clean","last_fresh":"2023-09-23T20:54:54.892701+0000","last_change":"2023-09-23T20:54:31.892932+0000","last_active":"2023-09-23T20:54:54.892701+0000","last_peered":"2023-09-23T20:54:54.892701+0000","last_clean":"2023-09-23T20:54:54.892701+0000","last_became_active":"2023-09-23T20:54:31.892631+0000","last_became_peered":"2023-09-23T20:54:31.892631+0000","last_unstale":"2023-09-23T20:54:54.892701+0000","last_undegraded":"2023-09-23T20:54:54.892701+0000","last_fullsized":"2023-09-23T20:54:54.892701+0000","mapping_epoch":136,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:55:40.297988+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":132,"reported_epoch":145,"state":"active+clean","last_fresh":"2023-09-23T20:54:53.953457+0000","last_change":"2023-09-23T20:54:13.928715+0000","last_active":"2023-09-23T20:54:53.953457+0000","last_peered":"2023-09-23T20:54:53.953457+0000","last_clean":"2023-09-23T20:54:53.953457+0000","last_became_active":"2023-09-23T20:54:13.928200+0000","last_became_peered":"2023-09-23T20:54:13.928200+0000","last_unstale":"2023-09-23T20:54:53.953457+0000","last_undegraded":"2023-09-23T20:54:53.953457+0000","last_fullsized":"2023-09-23T20:54:53.953457+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:41:16.927513+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.e","version":"147'9418","reported_seq":14984,"reported_epoch":147,"state":"active+clean","last_fresh":"2023-09-23T20:54:57.089449+0000","last_change":"2023-09-23T20:54:55.922739+0000","last_active":"2023-09-23T20:54:57.089449+0000","last_peered":"2023-09-23T20:54:57.089449+0000","last_clean":"2023-09-23T20:54:57.089449+0000","last_became_active":"2023-09-23T20:54:55.902420+0000","last_became_peered":"2023-09-23T20:54:55.902420+0000","last_unstale":"2023-09-23T20:54:57.089449+0000","last_undegraded":"2023-09-23T20:54:57.089449+0000","last_fullsized":"2023-09-23T20:54:57.089449+0000","mapping_epoch":146,"log_start":"112'2928","ondisk_log_start":"112'2928","created":102,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:52:13.843048+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:52:13.843048+0000","last_clean_scrub_stamp":"2023-09-23T20:52:13.843048+0000","objects_scrubbed":0,"log_size":6490,"log_dups_size":0,"ondisk_log_size":6490,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:34:13.390956+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":224133120,"num_objects":3420,"num_object_clones":0,"num_object_copies":10260,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3420,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9418,"num_write_kb":410816,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2950,"num_bytes_recovered":135856128,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":132,"reported_epoch":144,"state":"active+clean","last_fresh":"2023-09-23T20:54:51.910810+0000","last_change":"2023-09-23T20:54:13.961788+0000","last_active":"2023-09-23T20:54:51.910810+0000","last_peered":"2023-09-23T20:54:51.910810+0000","last_clean":"2023-09-23T20:54:51.910810+0000","last_became_active":"2023-09-23T20:54:13.940937+0000","last_became_peered":"2023-09-23T20:54:13.940937+0000","last_unstale":"2023-09-23T20:54:51.910810+0000","last_undegraded":"2023-09-23T20:54:51.910810+0000","last_fullsized":"2023-09-23T20:54:51.910810+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:11:56.785299+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":24,"reported_epoch":146,"state":"active+clean","last_fresh":"2023-09-23T20:54:54.891497+0000","last_change":"2023-09-23T20:54:13.955668+0000","last_active":"2023-09-23T20:54:54.891497+0000","last_peered":"2023-09-23T20:54:54.891497+0000","last_clean":"2023-09-23T20:54:54.891497+0000","last_became_active":"2023-09-23T20:54:13.951783+0000","last_became_peered":"2023-09-23T20:54:13.951783+0000","last_unstale":"2023-09-23T20:54:54.891497+0000","last_undegraded":"2023-09-23T20:54:54.891497+0000","last_fullsized":"2023-09-23T20:54:54.891497+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:48:35.853472+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.f","version":"147'9097","reported_seq":14805,"reported_epoch":147,"state":"active+clean","last_fresh":"2023-09-23T20:54:56.262809+0000","last_change":"2023-09-23T20:54:32.961255+0000","last_active":"2023-09-23T20:54:56.262809+0000","last_peered":"2023-09-23T20:54:56.262809+0000","last_clean":"2023-09-23T20:54:56.262809+0000","last_became_active":"2023-09-23T20:54:32.944166+0000","last_became_peered":"2023-09-23T20:54:32.944166+0000","last_unstale":"2023-09-23T20:54:56.262809+0000","last_undegraded":"2023-09-23T20:54:56.262809+0000","last_fullsized":"2023-09-23T20:54:56.262809+0000","mapping_epoch":137,"log_start":"107'941","ondisk_log_start":"107'941","created":102,"last_epoch_clean":138,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:52:13.843048+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:52:13.843048+0000","last_clean_scrub_stamp":"2023-09-23T20:52:13.843048+0000","objects_scrubbed":0,"log_size":8156,"log_dups_size":0,"ondisk_log_size":8156,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:06:07.274753+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":222756864,"num_objects":3399,"num_object_clones":0,"num_object_copies":10197,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3399,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9097,"num_write_kb":399872,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2813,"num_bytes_recovered":127139840,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":127,"reported_epoch":146,"state":"active+clean","last_fresh":"2023-09-23T20:54:54.888745+0000","last_change":"2023-09-23T20:54:13.942028+0000","last_active":"2023-09-23T20:54:54.888745+0000","last_peered":"2023-09-23T20:54:54.888745+0000","last_clean":"2023-09-23T20:54:54.888745+0000","last_became_active":"2023-09-23T20:54:13.941636+0000","last_became_peered":"2023-09-23T20:54:13.941636+0000","last_unstale":"2023-09-23T20:54:54.888745+0000","last_undegraded":"2023-09-23T20:54:54.888745+0000","last_fullsized":"2023-09-23T20:54:54.888745+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:48:18.822879+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":24,"reported_epoch":146,"state":"active+clean","last_fresh":"2023-09-23T20:54:54.891621+0000","last_change":"2023-09-23T20:54:13.951459+0000","last_active":"2023-09-23T20:54:54.891621+0000","last_peered":"2023-09-23T20:54:54.891621+0000","last_clean":"2023-09-23T20:54:54.891621+0000","last_became_active":"2023-09-23T20:54:13.948409+0000","last_became_peered":"2023-09-23T20:54:13.948409+0000","last_unstale":"2023-09-23T20:54:54.891621+0000","last_undegraded":"2023-09-23T20:54:54.891621+0000","last_fullsized":"2023-09-23T20:54:54.891621+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:38:10.148061+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.c","version":"145'9162","reported_seq":9231,"reported_epoch":145,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-23T20:54:54.062468+0000","last_change":"2023-09-23T20:54:33.059349+0000","last_active":"2023-09-23T20:54:54.062468+0000","last_peered":"2023-09-23T20:54:54.062468+0000","last_clean":"2023-09-23T20:54:30.879229+0000","last_became_active":"2023-09-23T20:54:33.019925+0000","last_became_peered":"2023-09-23T20:54:33.019925+0000","last_unstale":"2023-09-23T20:54:54.062468+0000","last_undegraded":"2023-09-23T20:54:54.062468+0000","last_fullsized":"2023-09-23T20:54:54.062468+0000","mapping_epoch":137,"log_start":"113'3363","ondisk_log_start":"113'3363","created":102,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:52:13.843048+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:52:13.843048+0000","last_clean_scrub_stamp":"2023-09-23T20:52:13.843048+0000","objects_scrubbed":0,"log_size":5799,"log_dups_size":0,"ondisk_log_size":5799,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:48:58.224746+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":223477760,"num_objects":3410,"num_object_clones":0,"num_object_copies":10230,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":3410,"num_objects_unfound":0,"num_objects_dirty":3410,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9162,"num_write_kb":402304,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,7],"avail_no_missing":["3","6","7"],"object_location_counts":[{"shards":"3,6,7","objects":3410}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":24,"reported_epoch":146,"state":"active+clean","last_fresh":"2023-09-23T20:54:54.891182+0000","last_change":"2023-09-23T20:54:13.950254+0000","last_active":"2023-09-23T20:54:54.891182+0000","last_peered":"2023-09-23T20:54:54.891182+0000","last_clean":"2023-09-23T20:54:54.891182+0000","last_became_active":"2023-09-23T20:54:13.949934+0000","last_became_peered":"2023-09-23T20:54:13.949934+0000","last_unstale":"2023-09-23T20:54:54.891182+0000","last_undegraded":"2023-09-23T20:54:54.891182+0000","last_fullsized":"2023-09-23T20:54:54.891182+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:30:32.847167+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":24,"reported_epoch":146,"state":"active+clean","last_fresh":"2023-09-23T20:54:54.893101+0000","last_change":"2023-09-23T20:54:13.974842+0000","last_active":"2023-09-23T20:54:54.893101+0000","last_peered":"2023-09-23T20:54:54.893101+0000","last_clean":"2023-09-23T20:54:54.893101+0000","last_became_active":"2023-09-23T20:54:13.974538+0000","last_became_peered":"2023-09-23T20:54:13.974538+0000","last_unstale":"2023-09-23T20:54:54.893101+0000","last_undegraded":"2023-09-23T20:54:54.893101+0000","last_fullsized":"2023-09-23T20:54:54.893101+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:08:13.017823+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.d","version":"147'9342","reported_seq":17478,"reported_epoch":147,"state":"active+clean","last_fresh":"2023-09-23T20:54:57.057024+0000","last_change":"2023-09-23T20:54:54.922926+0000","last_active":"2023-09-23T20:54:57.057024+0000","last_peered":"2023-09-23T20:54:57.057024+0000","last_clean":"2023-09-23T20:54:57.057024+0000","last_became_active":"2023-09-23T20:54:54.898748+0000","last_became_peered":"2023-09-23T20:54:54.898748+0000","last_unstale":"2023-09-23T20:54:57.057024+0000","last_undegraded":"2023-09-23T20:54:57.057024+0000","last_fullsized":"2023-09-23T20:54:57.057024+0000","mapping_epoch":145,"log_start":"108'1447","ondisk_log_start":"108'1447","created":102,"last_epoch_clean":146,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:52:13.843048+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:52:13.843048+0000","last_clean_scrub_stamp":"2023-09-23T20:52:13.843048+0000","objects_scrubbed":0,"log_size":7895,"log_dups_size":0,"ondisk_log_size":7895,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:31:42.763337+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":223674392,"num_objects":3414,"num_object_clones":0,"num_object_copies":10242,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3414,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":9342,"num_write_kb":408129,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4030,"num_bytes_recovered":264044568,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":67,"reported_epoch":144,"state":"active+clean","last_fresh":"2023-09-23T20:54:51.908743+0000","last_change":"2023-09-23T20:54:44.034235+0000","last_active":"2023-09-23T20:54:51.908743+0000","last_peered":"2023-09-23T20:54:51.908743+0000","last_clean":"2023-09-23T20:54:51.908743+0000","last_became_active":"2023-09-23T20:54:44.033911+0000","last_became_peered":"2023-09-23T20:54:44.033911+0000","last_unstale":"2023-09-23T20:54:51.908743+0000","last_undegraded":"2023-09-23T20:54:51.908743+0000","last_fullsized":"2023-09-23T20:54:51.908743+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:12:48.442227+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,4],"acting":[1,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":113,"reported_epoch":144,"state":"active+clean","last_fresh":"2023-09-23T20:54:51.916536+0000","last_change":"2023-09-23T20:54:13.940748+0000","last_active":"2023-09-23T20:54:51.916536+0000","last_peered":"2023-09-23T20:54:51.916536+0000","last_clean":"2023-09-23T20:54:51.916536+0000","last_became_active":"2023-09-23T20:54:13.940419+0000","last_became_peered":"2023-09-23T20:54:13.940419+0000","last_unstale":"2023-09-23T20:54:51.916536+0000","last_undegraded":"2023-09-23T20:54:51.916536+0000","last_fullsized":"2023-09-23T20:54:51.916536+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:15:01.923816+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":118,"reported_epoch":144,"state":"active+clean","last_fresh":"2023-09-23T20:54:51.916728+0000","last_change":"2023-09-23T20:50:05.867130+0000","last_active":"2023-09-23T20:54:51.916728+0000","last_peered":"2023-09-23T20:54:51.916728+0000","last_clean":"2023-09-23T20:54:51.916728+0000","last_became_active":"2023-09-23T20:50:05.866758+0000","last_became_peered":"2023-09-23T20:50:05.866758+0000","last_unstale":"2023-09-23T20:54:51.916728+0000","last_undegraded":"2023-09-23T20:54:51.916728+0000","last_fullsized":"2023-09-23T20:54:51.916728+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:55:07.548209+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":20,"reported_epoch":146,"state":"active+clean","last_fresh":"2023-09-23T20:54:54.893417+0000","last_change":"2023-09-23T20:54:31.887604+0000","last_active":"2023-09-23T20:54:54.893417+0000","last_peered":"2023-09-23T20:54:54.893417+0000","last_clean":"2023-09-23T20:54:54.893417+0000","last_became_active":"2023-09-23T20:54:31.886752+0000","last_became_peered":"2023-09-23T20:54:31.886752+0000","last_unstale":"2023-09-23T20:54:54.893417+0000","last_undegraded":"2023-09-23T20:54:54.893417+0000","last_fullsized":"2023-09-23T20:54:54.893417+0000","mapping_epoch":136,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:13:58.872176+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":116,"reported_epoch":144,"state":"active+clean","last_fresh":"2023-09-23T20:54:51.907931+0000","last_change":"2023-09-23T20:54:31.899082+0000","last_active":"2023-09-23T20:54:51.907931+0000","last_peered":"2023-09-23T20:54:51.907931+0000","last_clean":"2023-09-23T20:54:51.907931+0000","last_became_active":"2023-09-23T20:54:31.898521+0000","last_became_peered":"2023-09-23T20:54:31.898521+0000","last_unstale":"2023-09-23T20:54:51.907931+0000","last_undegraded":"2023-09-23T20:54:51.907931+0000","last_fullsized":"2023-09-23T20:54:51.907931+0000","mapping_epoch":136,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:25:36.059711+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.0","version":"146'8078","reported_seq":8144,"reported_epoch":146,"state":"active+clean","last_fresh":"2023-09-23T20:54:55.140930+0000","last_change":"2023-09-23T20:54:27.195490+0000","last_active":"2023-09-23T20:54:55.140930+0000","last_peered":"2023-09-23T20:54:55.140930+0000","last_clean":"2023-09-23T20:54:55.140930+0000","last_became_active":"2023-09-23T20:54:03.763183+0000","last_became_peered":"2023-09-23T20:54:03.763183+0000","last_unstale":"2023-09-23T20:54:55.140930+0000","last_undegraded":"2023-09-23T20:54:55.140930+0000","last_fullsized":"2023-09-23T20:54:55.140930+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":102,"last_epoch_clean":129,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:52:13.843048+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:52:13.843048+0000","last_clean_scrub_stamp":"2023-09-23T20:52:13.843048+0000","objects_scrubbed":0,"log_size":8078,"log_dups_size":0,"ondisk_log_size":8078,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":111181824,"num_objects":1696,"num_object_clones":0,"num_object_copies":5088,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1696,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4663,"num_write_kb":203520,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.6","version":"145'7955","reported_seq":8458,"reported_epoch":145,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T20:54:54.049432+0000","last_change":"2023-09-23T20:54:31.905333+0000","last_active":"2023-09-23T20:54:54.049432+0000","last_peered":"2023-09-23T20:54:54.049432+0000","last_clean":"2023-09-23T20:53:27.922544+0000","last_became_active":"2023-09-23T20:54:14.984750+0000","last_became_peered":"2023-09-23T20:54:14.984750+0000","last_unstale":"2023-09-23T20:54:54.049432+0000","last_undegraded":"2023-09-23T20:54:31.904885+0000","last_fullsized":"2023-09-23T20:54:14.923524+0000","mapping_epoch":133,"log_start":"107'900","ondisk_log_start":"107'900","created":102,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:52:13.843048+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:52:13.843048+0000","last_clean_scrub_stamp":"2023-09-23T20:52:13.843048+0000","objects_scrubbed":0,"log_size":7055,"log_dups_size":0,"ondisk_log_size":7055,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":109772800,"num_objects":1675,"num_object_clones":0,"num_object_copies":5025,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":789,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1675,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4596,"num_write_kb":200672,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":212,"num_bytes_recovered":13828096,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,7],"avail_no_missing":["3","7"],"object_location_counts":[{"shards":"3,6,7","objects":361},{"shards":"3,7","objects":1314}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":20,"reported_epoch":146,"state":"active+clean","last_fresh":"2023-09-23T20:54:54.892251+0000","last_change":"2023-09-23T20:54:31.893226+0000","last_active":"2023-09-23T20:54:54.892251+0000","last_peered":"2023-09-23T20:54:54.892251+0000","last_clean":"2023-09-23T20:54:54.892251+0000","last_became_active":"2023-09-23T20:54:31.892843+0000","last_became_peered":"2023-09-23T20:54:31.892843+0000","last_unstale":"2023-09-23T20:54:54.892251+0000","last_undegraded":"2023-09-23T20:54:54.892251+0000","last_fullsized":"2023-09-23T20:54:54.892251+0000","mapping_epoch":136,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:12:37.172191+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":24,"reported_epoch":146,"state":"active+clean","last_fresh":"2023-09-23T20:54:54.891972+0000","last_change":"2023-09-23T20:54:13.936037+0000","last_active":"2023-09-23T20:54:54.891972+0000","last_peered":"2023-09-23T20:54:54.891972+0000","last_clean":"2023-09-23T20:54:54.891972+0000","last_became_active":"2023-09-23T20:54:13.935690+0000","last_became_peered":"2023-09-23T20:54:13.935690+0000","last_unstale":"2023-09-23T20:54:54.891972+0000","last_undegraded":"2023-09-23T20:54:54.891972+0000","last_fullsized":"2023-09-23T20:54:54.891972+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:36:31.397606+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":110,"reported_epoch":144,"state":"active+clean","last_fresh":"2023-09-23T20:54:51.916624+0000","last_change":"2023-09-23T20:51:10.829889+0000","last_active":"2023-09-23T20:54:51.916624+0000","last_peered":"2023-09-23T20:54:51.916624+0000","last_clean":"2023-09-23T20:54:51.916624+0000","last_became_active":"2023-09-23T20:51:10.829533+0000","last_became_peered":"2023-09-23T20:51:10.829533+0000","last_unstale":"2023-09-23T20:54:51.916624+0000","last_undegraded":"2023-09-23T20:54:51.916624+0000","last_fullsized":"2023-09-23T20:54:51.916624+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:43:04.603460+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.5","version":"147'7933","reported_seq":8003,"reported_epoch":147,"state":"active+clean","last_fresh":"2023-09-23T20:54:57.130172+0000","last_change":"2023-09-23T20:54:24.028468+0000","last_active":"2023-09-23T20:54:57.130172+0000","last_peered":"2023-09-23T20:54:57.130172+0000","last_clean":"2023-09-23T20:54:57.130172+0000","last_became_active":"2023-09-23T20:54:03.700333+0000","last_became_peered":"2023-09-23T20:54:03.700333+0000","last_unstale":"2023-09-23T20:54:57.130172+0000","last_undegraded":"2023-09-23T20:54:57.130172+0000","last_fullsized":"2023-09-23T20:54:57.130172+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":102,"last_epoch_clean":129,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:52:13.843048+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:52:13.843048+0000","last_clean_scrub_stamp":"2023-09-23T20:52:13.843048+0000","objects_scrubbed":0,"log_size":7933,"log_dups_size":0,"ondisk_log_size":7933,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":111280128,"num_objects":1698,"num_object_clones":0,"num_object_copies":5094,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1698,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4585,"num_write_kb":201056,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,3],"acting":[7,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":131,"reported_epoch":146,"state":"active+clean","last_fresh":"2023-09-23T20:54:54.894615+0000","last_change":"2023-09-23T20:51:10.838054+0000","last_active":"2023-09-23T20:54:54.894615+0000","last_peered":"2023-09-23T20:54:54.894615+0000","last_clean":"2023-09-23T20:54:54.894615+0000","last_became_active":"2023-09-23T20:51:10.837750+0000","last_became_peered":"2023-09-23T20:51:10.837750+0000","last_unstale":"2023-09-23T20:54:54.894615+0000","last_undegraded":"2023-09-23T20:54:54.894615+0000","last_fullsized":"2023-09-23T20:54:54.894615+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:15:51.648085+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":95,"reported_epoch":144,"state":"active+clean","last_fresh":"2023-09-23T20:54:51.908599+0000","last_change":"2023-09-23T20:50:37.983072+0000","last_active":"2023-09-23T20:54:51.908599+0000","last_peered":"2023-09-23T20:54:51.908599+0000","last_clean":"2023-09-23T20:54:51.908599+0000","last_became_active":"2023-09-23T20:50:37.982777+0000","last_became_peered":"2023-09-23T20:50:37.982777+0000","last_unstale":"2023-09-23T20:54:51.908599+0000","last_undegraded":"2023-09-23T20:54:51.908599+0000","last_fullsized":"2023-09-23T20:54:51.908599+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:58:11.972144+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":131,"reported_epoch":144,"state":"active+clean","last_fresh":"2023-09-23T20:54:51.916449+0000","last_change":"2023-09-23T20:54:31.898870+0000","last_active":"2023-09-23T20:54:51.916449+0000","last_peered":"2023-09-23T20:54:51.916449+0000","last_clean":"2023-09-23T20:54:51.916449+0000","last_became_active":"2023-09-23T20:54:31.898467+0000","last_became_peered":"2023-09-23T20:54:31.898467+0000","last_unstale":"2023-09-23T20:54:51.916449+0000","last_undegraded":"2023-09-23T20:54:51.916449+0000","last_fullsized":"2023-09-23T20:54:51.916449+0000","mapping_epoch":136,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:56:22.483408+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.7","version":"145'8094","reported_seq":8611,"reported_epoch":145,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2023-09-23T20:54:54.038254+0000","last_change":"2023-09-23T20:54:33.042299+0000","last_active":"2023-09-23T20:54:54.038254+0000","last_peered":"2023-09-23T20:54:54.038254+0000","last_clean":"2023-09-23T20:53:27.924842+0000","last_became_active":"2023-09-23T20:54:33.018857+0000","last_became_peered":"2023-09-23T20:54:33.018857+0000","last_unstale":"2023-09-23T20:54:54.038254+0000","last_undegraded":"2023-09-23T20:54:32.942306+0000","last_fullsized":"2023-09-23T20:54:54.038254+0000","mapping_epoch":137,"log_start":"107'1000","ondisk_log_start":"107'1000","created":102,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:52:13.843048+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:52:13.843048+0000","last_clean_scrub_stamp":"2023-09-23T20:52:13.843048+0000","objects_scrubbed":0,"log_size":7094,"log_dups_size":0,"ondisk_log_size":7094,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":111738880,"num_objects":1705,"num_object_clones":0,"num_object_copies":5115,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":822,"num_objects_misplaced":1705,"num_objects_unfound":0,"num_objects_dirty":1705,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4674,"num_write_kb":204128,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":215,"num_bytes_recovered":14090240,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[3,4,1],"avail_no_missing":["3","1"],"object_location_counts":[{"shards":"1,3","objects":1},{"shards":"1,3,4","objects":1327},{"shards":"1,3,7","objects":377}],"blocked_by":[],"up_primary":5,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":20,"reported_epoch":146,"state":"active+clean","last_fresh":"2023-09-23T20:54:54.892367+0000","last_change":"2023-09-23T20:54:31.883504+0000","last_active":"2023-09-23T20:54:54.892367+0000","last_peered":"2023-09-23T20:54:54.892367+0000","last_clean":"2023-09-23T20:54:54.892367+0000","last_became_active":"2023-09-23T20:54:31.883172+0000","last_became_peered":"2023-09-23T20:54:31.883172+0000","last_unstale":"2023-09-23T20:54:54.892367+0000","last_undegraded":"2023-09-23T20:54:54.892367+0000","last_fullsized":"2023-09-23T20:54:54.892367+0000","mapping_epoch":136,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:45:40.289120+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"79'161","reported_seq":488,"reported_epoch":146,"state":"active+clean","last_fresh":"2023-09-23T20:54:54.894026+0000","last_change":"2023-09-23T20:49:32.777246+0000","last_active":"2023-09-23T20:54:54.894026+0000","last_peered":"2023-09-23T20:54:54.894026+0000","last_clean":"2023-09-23T20:54:54.894026+0000","last_became_active":"2023-09-23T20:49:32.776822+0000","last_became_peered":"2023-09-23T20:49:32.776822+0000","last_unstale":"2023-09-23T20:54:54.894026+0000","last_undegraded":"2023-09-23T20:54:54.894026+0000","last_fullsized":"2023-09-23T20:54:54.894026+0000","mapping_epoch":83,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":84,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:38:57.324224+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:05:46.881275+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":105,"reported_epoch":144,"state":"active+clean","last_fresh":"2023-09-23T20:54:51.910862+0000","last_change":"2023-09-23T20:54:31.892130+0000","last_active":"2023-09-23T20:54:51.910862+0000","last_peered":"2023-09-23T20:54:51.910862+0000","last_clean":"2023-09-23T20:54:51.910862+0000","last_became_active":"2023-09-23T20:54:31.891695+0000","last_became_peered":"2023-09-23T20:54:31.891695+0000","last_unstale":"2023-09-23T20:54:51.910862+0000","last_undegraded":"2023-09-23T20:54:51.910862+0000","last_fullsized":"2023-09-23T20:54:51.910862+0000","mapping_epoch":136,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:26:00.839541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":109,"reported_epoch":146,"state":"active+clean","last_fresh":"2023-09-23T20:54:54.889123+0000","last_change":"2023-09-23T20:54:13.935747+0000","last_active":"2023-09-23T20:54:54.889123+0000","last_peered":"2023-09-23T20:54:54.889123+0000","last_clean":"2023-09-23T20:54:54.889123+0000","last_became_active":"2023-09-23T20:54:13.935318+0000","last_became_peered":"2023-09-23T20:54:13.935318+0000","last_unstale":"2023-09-23T20:54:54.889123+0000","last_undegraded":"2023-09-23T20:54:54.889123+0000","last_fullsized":"2023-09-23T20:54:54.889123+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:34:02.843714+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":128,"reported_epoch":146,"state":"active+clean","last_fresh":"2023-09-23T20:54:54.894014+0000","last_change":"2023-09-23T20:50:05.866898+0000","last_active":"2023-09-23T20:54:54.894014+0000","last_peered":"2023-09-23T20:54:54.894014+0000","last_clean":"2023-09-23T20:54:54.894014+0000","last_became_active":"2023-09-23T20:50:05.866541+0000","last_became_peered":"2023-09-23T20:50:05.866541+0000","last_unstale":"2023-09-23T20:54:54.894014+0000","last_undegraded":"2023-09-23T20:54:54.894014+0000","last_fullsized":"2023-09-23T20:54:54.894014+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:17:02.353010+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":128,"reported_epoch":145,"state":"active+clean","last_fresh":"2023-09-23T20:54:53.954894+0000","last_change":"2023-09-23T20:54:44.035149+0000","last_active":"2023-09-23T20:54:53.954894+0000","last_peered":"2023-09-23T20:54:53.954894+0000","last_clean":"2023-09-23T20:54:53.954894+0000","last_became_active":"2023-09-23T20:54:44.034729+0000","last_became_peered":"2023-09-23T20:54:44.034729+0000","last_unstale":"2023-09-23T20:54:53.954894+0000","last_undegraded":"2023-09-23T20:54:53.954894+0000","last_fullsized":"2023-09-23T20:54:53.954894+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:35:26.645773+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,5],"acting":[0,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":116,"reported_epoch":145,"state":"active+clean","last_fresh":"2023-09-23T20:54:53.953171+0000","last_change":"2023-09-23T20:50:37.982235+0000","last_active":"2023-09-23T20:54:53.953171+0000","last_peered":"2023-09-23T20:54:53.953171+0000","last_clean":"2023-09-23T20:54:53.953171+0000","last_became_active":"2023-09-23T20:50:37.981913+0000","last_became_peered":"2023-09-23T20:50:37.981913+0000","last_unstale":"2023-09-23T20:54:53.953171+0000","last_undegraded":"2023-09-23T20:54:53.953171+0000","last_fullsized":"2023-09-23T20:54:53.953171+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:59:57.122584+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.4","version":"145'7951","reported_seq":8022,"reported_epoch":145,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-23T20:54:53.957200+0000","last_change":"2023-09-23T20:54:33.039086+0000","last_active":"2023-09-23T20:54:53.957200+0000","last_peered":"2023-09-23T20:54:53.957200+0000","last_clean":"2023-09-23T20:54:30.866550+0000","last_became_active":"2023-09-23T20:54:33.018911+0000","last_became_peered":"2023-09-23T20:54:33.018911+0000","last_unstale":"2023-09-23T20:54:53.957200+0000","last_undegraded":"2023-09-23T20:54:53.957200+0000","last_fullsized":"2023-09-23T20:54:53.957200+0000","mapping_epoch":137,"log_start":"107'600","ondisk_log_start":"107'600","created":102,"last_epoch_clean":129,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:52:13.843048+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:52:13.843048+0000","last_clean_scrub_stamp":"2023-09-23T20:52:13.843048+0000","objects_scrubbed":0,"log_size":7351,"log_dups_size":0,"ondisk_log_size":7351,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":112295936,"num_objects":1713,"num_object_clones":0,"num_object_copies":5139,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1713,"num_objects_unfound":0,"num_objects_dirty":1713,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4581,"num_write_kb":201440,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,3],"acting":[1,3,6],"avail_no_missing":["1","3","6"],"object_location_counts":[{"shards":"1,3,6","objects":1713}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":131,"reported_epoch":146,"state":"active+clean","last_fresh":"2023-09-23T20:54:54.894385+0000","last_change":"2023-09-23T20:54:13.937566+0000","last_active":"2023-09-23T20:54:54.894385+0000","last_peered":"2023-09-23T20:54:54.894385+0000","last_clean":"2023-09-23T20:54:54.894385+0000","last_became_active":"2023-09-23T20:54:13.937107+0000","last_became_peered":"2023-09-23T20:54:13.937107+0000","last_unstale":"2023-09-23T20:54:54.894385+0000","last_undegraded":"2023-09-23T20:54:54.894385+0000","last_fullsized":"2023-09-23T20:54:54.894385+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:20:26.511292+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":113,"reported_epoch":144,"state":"active+clean","last_fresh":"2023-09-23T20:54:51.916709+0000","last_change":"2023-09-23T20:54:31.898965+0000","last_active":"2023-09-23T20:54:51.916709+0000","last_peered":"2023-09-23T20:54:51.916709+0000","last_clean":"2023-09-23T20:54:51.916709+0000","last_became_active":"2023-09-23T20:54:31.898663+0000","last_became_peered":"2023-09-23T20:54:31.898663+0000","last_unstale":"2023-09-23T20:54:51.916709+0000","last_undegraded":"2023-09-23T20:54:51.916709+0000","last_fullsized":"2023-09-23T20:54:51.916709+0000","mapping_epoch":136,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:53:05.058087+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":104,"reported_epoch":145,"state":"active+clean","last_fresh":"2023-09-23T20:54:53.953318+0000","last_change":"2023-09-23T20:54:31.884717+0000","last_active":"2023-09-23T20:54:53.953318+0000","last_peered":"2023-09-23T20:54:53.953318+0000","last_clean":"2023-09-23T20:54:53.953318+0000","last_became_active":"2023-09-23T20:54:31.884345+0000","last_became_peered":"2023-09-23T20:54:31.884345+0000","last_unstale":"2023-09-23T20:54:53.953318+0000","last_undegraded":"2023-09-23T20:54:53.953318+0000","last_fullsized":"2023-09-23T20:54:53.953318+0000","mapping_epoch":136,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:02:26.393868+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.1","version":"145'8101","reported_seq":8167,"reported_epoch":145,"state":"active+clean","last_fresh":"2023-09-23T20:54:54.030099+0000","last_change":"2023-09-23T20:54:03.859011+0000","last_active":"2023-09-23T20:54:54.030099+0000","last_peered":"2023-09-23T20:54:54.030099+0000","last_clean":"2023-09-23T20:54:54.030099+0000","last_became_active":"2023-09-23T20:54:03.724159+0000","last_became_peered":"2023-09-23T20:54:03.724159+0000","last_unstale":"2023-09-23T20:54:54.030099+0000","last_undegraded":"2023-09-23T20:54:54.030099+0000","last_fullsized":"2023-09-23T20:54:54.030099+0000","mapping_epoch":102,"log_start":"107'700","ondisk_log_start":"107'700","created":102,"last_epoch_clean":129,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:52:13.843048+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:52:13.843048+0000","last_clean_scrub_stamp":"2023-09-23T20:52:13.843048+0000","objects_scrubbed":0,"log_size":7401,"log_dups_size":0,"ondisk_log_size":7401,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":111968256,"num_objects":1708,"num_object_clones":0,"num_object_copies":5124,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1708,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4686,"num_write_kb":204640,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":106,"reported_epoch":144,"state":"active+clean","last_fresh":"2023-09-23T20:54:51.911679+0000","last_change":"2023-09-23T20:51:10.836910+0000","last_active":"2023-09-23T20:54:51.911679+0000","last_peered":"2023-09-23T20:54:51.911679+0000","last_clean":"2023-09-23T20:54:51.911679+0000","last_became_active":"2023-09-23T20:51:10.836557+0000","last_became_peered":"2023-09-23T20:51:10.836557+0000","last_unstale":"2023-09-23T20:54:51.911679+0000","last_undegraded":"2023-09-23T20:54:51.911679+0000","last_fullsized":"2023-09-23T20:54:51.911679+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:51:54.019969+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":98,"reported_epoch":144,"state":"active+clean","last_fresh":"2023-09-23T20:54:51.919532+0000","last_change":"2023-09-23T20:54:13.943183+0000","last_active":"2023-09-23T20:54:51.919532+0000","last_peered":"2023-09-23T20:54:51.919532+0000","last_clean":"2023-09-23T20:54:51.919532+0000","last_became_active":"2023-09-23T20:54:13.941729+0000","last_became_peered":"2023-09-23T20:54:13.941729+0000","last_unstale":"2023-09-23T20:54:51.919532+0000","last_undegraded":"2023-09-23T20:54:51.919532+0000","last_fullsized":"2023-09-23T20:54:51.919532+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:28:22.632255+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":64,"reported_epoch":146,"state":"active+clean","last_fresh":"2023-09-23T20:54:54.894185+0000","last_change":"2023-09-23T20:54:44.035747+0000","last_active":"2023-09-23T20:54:54.894185+0000","last_peered":"2023-09-23T20:54:54.894185+0000","last_clean":"2023-09-23T20:54:54.894185+0000","last_became_active":"2023-09-23T20:54:44.035433+0000","last_became_peered":"2023-09-23T20:54:44.035433+0000","last_unstale":"2023-09-23T20:54:54.894185+0000","last_undegraded":"2023-09-23T20:54:54.894185+0000","last_fullsized":"2023-09-23T20:54:54.894185+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:42:25.184541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"6.2","version":"147'7904","reported_seq":10102,"reported_epoch":147,"state":"active+clean","last_fresh":"2023-09-23T20:54:57.028365+0000","last_change":"2023-09-23T20:54:37.971783+0000","last_active":"2023-09-23T20:54:57.028365+0000","last_peered":"2023-09-23T20:54:57.028365+0000","last_clean":"2023-09-23T20:54:57.028365+0000","last_became_active":"2023-09-23T20:54:37.961980+0000","last_became_peered":"2023-09-23T20:54:37.961980+0000","last_unstale":"2023-09-23T20:54:57.028365+0000","last_undegraded":"2023-09-23T20:54:57.028365+0000","last_fullsized":"2023-09-23T20:54:57.028365+0000","mapping_epoch":139,"log_start":"0'0","ondisk_log_start":"0'0","created":102,"last_epoch_clean":140,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:52:13.843048+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:52:13.843048+0000","last_clean_scrub_stamp":"2023-09-23T20:52:13.843048+0000","objects_scrubbed":0,"log_size":7904,"log_dups_size":0,"ondisk_log_size":7904,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":105447424,"num_objects":1609,"num_object_clones":0,"num_object_copies":4827,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1609,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4582,"num_write_kb":198112,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1055,"num_bytes_recovered":39518208,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":111,"reported_epoch":146,"state":"active+clean","last_fresh":"2023-09-23T20:54:54.889125+0000","last_change":"2023-09-23T20:50:37.985663+0000","last_active":"2023-09-23T20:54:54.889125+0000","last_peered":"2023-09-23T20:54:54.889125+0000","last_clean":"2023-09-23T20:54:54.889125+0000","last_became_active":"2023-09-23T20:50:37.985325+0000","last_became_peered":"2023-09-23T20:50:37.985325+0000","last_unstale":"2023-09-23T20:54:54.889125+0000","last_undegraded":"2023-09-23T20:54:54.889125+0000","last_fullsized":"2023-09-23T20:54:54.889125+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:24:30.838843+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"70'1","reported_seq":132,"reported_epoch":145,"state":"active+clean","last_fresh":"2023-09-23T20:54:53.952901+0000","last_change":"2023-09-23T20:54:14.009723+0000","last_active":"2023-09-23T20:54:53.952901+0000","last_peered":"2023-09-23T20:54:53.952901+0000","last_clean":"2023-09-23T20:54:53.952901+0000","last_became_active":"2023-09-23T20:54:13.941771+0000","last_became_peered":"2023-09-23T20:54:13.941771+0000","last_unstale":"2023-09-23T20:54:53.952901+0000","last_undegraded":"2023-09-23T20:54:53.952901+0000","last_fullsized":"2023-09-23T20:54:53.952901+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:48:25.588116+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":34,"reported_epoch":146,"state":"active+clean","last_fresh":"2023-09-23T20:54:54.891202+0000","last_change":"2023-09-23T20:54:31.899991+0000","last_active":"2023-09-23T20:54:54.891202+0000","last_peered":"2023-09-23T20:54:54.891202+0000","last_clean":"2023-09-23T20:54:54.891202+0000","last_became_active":"2023-09-23T20:54:31.899577+0000","last_became_peered":"2023-09-23T20:54:31.899577+0000","last_unstale":"2023-09-23T20:54:54.891202+0000","last_undegraded":"2023-09-23T20:54:54.891202+0000","last_fullsized":"2023-09-23T20:54:54.891202+0000","mapping_epoch":136,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:55:10.251935+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.3","version":"147'8000","reported_seq":8070,"reported_epoch":147,"state":"active+clean","last_fresh":"2023-09-23T20:54:57.108655+0000","last_change":"2023-09-23T20:54:30.046207+0000","last_active":"2023-09-23T20:54:57.108655+0000","last_peered":"2023-09-23T20:54:57.108655+0000","last_clean":"2023-09-23T20:54:57.108655+0000","last_became_active":"2023-09-23T20:54:03.693286+0000","last_became_peered":"2023-09-23T20:54:03.693286+0000","last_unstale":"2023-09-23T20:54:57.108655+0000","last_undegraded":"2023-09-23T20:54:57.108655+0000","last_fullsized":"2023-09-23T20:54:57.108655+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":102,"last_epoch_clean":129,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:52:13.843048+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:52:13.843048+0000","last_clean_scrub_stamp":"2023-09-23T20:52:13.843048+0000","objects_scrubbed":0,"log_size":8000,"log_dups_size":0,"ondisk_log_size":8000,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":110198784,"num_objects":1681,"num_object_clones":0,"num_object_copies":5043,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1681,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4630,"num_write_kb":201984,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,2],"acting":[7,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":107,"reported_epoch":145,"state":"active+clean","last_fresh":"2023-09-23T20:54:53.953554+0000","last_change":"2023-09-23T20:54:13.929196+0000","last_active":"2023-09-23T20:54:53.953554+0000","last_peered":"2023-09-23T20:54:53.953554+0000","last_clean":"2023-09-23T20:54:53.953554+0000","last_became_active":"2023-09-23T20:54:13.928831+0000","last_became_peered":"2023-09-23T20:54:13.928831+0000","last_unstale":"2023-09-23T20:54:53.953554+0000","last_undegraded":"2023-09-23T20:54:53.953554+0000","last_fullsized":"2023-09-23T20:54:53.953554+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:28:44.316851+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":113,"reported_epoch":146,"state":"active+clean","last_fresh":"2023-09-23T20:54:54.894718+0000","last_change":"2023-09-23T20:52:12.855430+0000","last_active":"2023-09-23T20:54:54.894718+0000","last_peered":"2023-09-23T20:54:54.894718+0000","last_clean":"2023-09-23T20:54:54.894718+0000","last_became_active":"2023-09-23T20:52:12.855115+0000","last_became_peered":"2023-09-23T20:52:12.855115+0000","last_unstale":"2023-09-23T20:54:54.894718+0000","last_undegraded":"2023-09-23T20:54:54.894718+0000","last_fullsized":"2023-09-23T20:54:54.894718+0000","mapping_epoch":100,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":101,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:33:23.802214+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"6.19","version":"146'8045","reported_seq":8110,"reported_epoch":146,"state":"active+clean","last_fresh":"2023-09-23T20:54:55.131211+0000","last_change":"2023-09-23T20:54:24.179065+0000","last_active":"2023-09-23T20:54:55.131211+0000","last_peered":"2023-09-23T20:54:55.131211+0000","last_clean":"2023-09-23T20:54:55.131211+0000","last_became_active":"2023-09-23T20:54:03.763084+0000","last_became_peered":"2023-09-23T20:54:03.763084+0000","last_unstale":"2023-09-23T20:54:55.131211+0000","last_undegraded":"2023-09-23T20:54:55.131211+0000","last_fullsized":"2023-09-23T20:54:55.131211+0000","mapping_epoch":128,"log_start":"0'0","ondisk_log_start":"0'0","created":128,"last_epoch_clean":129,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:52:13.843048+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:52:13.843048+0000","last_clean_scrub_stamp":"2023-09-23T20:52:13.843048+0000","objects_scrubbed":0,"log_size":8045,"log_dups_size":0,"ondisk_log_size":8045,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":115834880,"num_objects":1768,"num_object_clones":0,"num_object_copies":5304,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1768,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4627,"num_write_kb":204608,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,2],"acting":[0,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":24,"reported_epoch":146,"state":"active+clean","last_fresh":"2023-09-23T20:54:54.892617+0000","last_change":"2023-09-23T20:54:31.902946+0000","last_active":"2023-09-23T20:54:54.892617+0000","last_peered":"2023-09-23T20:54:54.892617+0000","last_clean":"2023-09-23T20:54:54.892617+0000","last_became_active":"2023-09-23T20:54:31.889484+0000","last_became_peered":"2023-09-23T20:54:31.889484+0000","last_unstale":"2023-09-23T20:54:54.892617+0000","last_undegraded":"2023-09-23T20:54:54.892617+0000","last_fullsized":"2023-09-23T20:54:54.892617+0000","mapping_epoch":136,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:44:44.278398+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":24,"reported_epoch":146,"state":"active+clean","last_fresh":"2023-09-23T20:54:54.892085+0000","last_change":"2023-09-23T20:54:13.950351+0000","last_active":"2023-09-23T20:54:54.892085+0000","last_peered":"2023-09-23T20:54:54.892085+0000","last_clean":"2023-09-23T20:54:54.892085+0000","last_became_active":"2023-09-23T20:54:13.949987+0000","last_became_peered":"2023-09-23T20:54:13.949987+0000","last_unstale":"2023-09-23T20:54:54.892085+0000","last_undegraded":"2023-09-23T20:54:54.892085+0000","last_fullsized":"2023-09-23T20:54:54.892085+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:46:17.765134+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.18","version":"147'8067","reported_seq":8137,"reported_epoch":147,"state":"active+clean","last_fresh":"2023-09-23T20:54:57.122413+0000","last_change":"2023-09-23T20:54:19.991392+0000","last_active":"2023-09-23T20:54:57.122413+0000","last_peered":"2023-09-23T20:54:57.122413+0000","last_clean":"2023-09-23T20:54:57.122413+0000","last_became_active":"2023-09-23T20:54:03.787013+0000","last_became_peered":"2023-09-23T20:54:03.787013+0000","last_unstale":"2023-09-23T20:54:57.122413+0000","last_undegraded":"2023-09-23T20:54:57.122413+0000","last_fullsized":"2023-09-23T20:54:57.122413+0000","mapping_epoch":128,"log_start":"106'100","ondisk_log_start":"106'100","created":128,"last_epoch_clean":129,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:52:13.843048+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:52:13.843048+0000","last_clean_scrub_stamp":"2023-09-23T20:52:13.843048+0000","objects_scrubbed":0,"log_size":7967,"log_dups_size":0,"ondisk_log_size":7967,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":111607808,"num_objects":1703,"num_object_clones":0,"num_object_copies":5109,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1703,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4656,"num_write_kb":203488,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,3],"acting":[7,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":24,"reported_epoch":146,"state":"active+clean","last_fresh":"2023-09-23T20:54:54.893309+0000","last_change":"2023-09-23T20:54:31.905017+0000","last_active":"2023-09-23T20:54:54.893309+0000","last_peered":"2023-09-23T20:54:54.893309+0000","last_clean":"2023-09-23T20:54:54.893309+0000","last_became_active":"2023-09-23T20:54:31.892625+0000","last_became_peered":"2023-09-23T20:54:31.892625+0000","last_unstale":"2023-09-23T20:54:54.893309+0000","last_undegraded":"2023-09-23T20:54:54.893309+0000","last_fullsized":"2023-09-23T20:54:54.893309+0000","mapping_epoch":136,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:39:20.387716+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":24,"reported_epoch":146,"state":"active+clean","last_fresh":"2023-09-23T20:54:54.891640+0000","last_change":"2023-09-23T20:54:13.935307+0000","last_active":"2023-09-23T20:54:54.891640+0000","last_peered":"2023-09-23T20:54:54.891640+0000","last_clean":"2023-09-23T20:54:54.891640+0000","last_became_active":"2023-09-23T20:54:13.935000+0000","last_became_peered":"2023-09-23T20:54:13.935000+0000","last_unstale":"2023-09-23T20:54:54.891640+0000","last_undegraded":"2023-09-23T20:54:54.891640+0000","last_fullsized":"2023-09-23T20:54:54.891640+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:47:40.974082+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":114,"reported_epoch":144,"state":"active+clean","last_fresh":"2023-09-23T20:54:51.911106+0000","last_change":"2023-09-23T20:51:10.833849+0000","last_active":"2023-09-23T20:54:51.911106+0000","last_peered":"2023-09-23T20:54:51.911106+0000","last_clean":"2023-09-23T20:54:51.911106+0000","last_became_active":"2023-09-23T20:51:10.833524+0000","last_became_peered":"2023-09-23T20:51:10.833524+0000","last_unstale":"2023-09-23T20:54:51.911106+0000","last_undegraded":"2023-09-23T20:54:51.911106+0000","last_fullsized":"2023-09-23T20:54:51.911106+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:23:12.501924+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":129,"reported_epoch":144,"state":"active+clean","last_fresh":"2023-09-23T20:54:51.909538+0000","last_change":"2023-09-23T20:51:10.838334+0000","last_active":"2023-09-23T20:54:51.909538+0000","last_peered":"2023-09-23T20:54:51.909538+0000","last_clean":"2023-09-23T20:54:51.909538+0000","last_became_active":"2023-09-23T20:51:10.838026+0000","last_became_peered":"2023-09-23T20:51:10.838026+0000","last_unstale":"2023-09-23T20:54:51.909538+0000","last_undegraded":"2023-09-23T20:54:51.909538+0000","last_fullsized":"2023-09-23T20:54:51.909538+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:04:37.009457+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":134,"reported_epoch":145,"state":"active+clean","last_fresh":"2023-09-23T20:54:53.953833+0000","last_change":"2023-09-23T20:54:31.903139+0000","last_active":"2023-09-23T20:54:53.953833+0000","last_peered":"2023-09-23T20:54:53.953833+0000","last_clean":"2023-09-23T20:54:53.953833+0000","last_became_active":"2023-09-23T20:54:31.884520+0000","last_became_peered":"2023-09-23T20:54:31.884520+0000","last_unstale":"2023-09-23T20:54:53.953833+0000","last_undegraded":"2023-09-23T20:54:53.953833+0000","last_fullsized":"2023-09-23T20:54:53.953833+0000","mapping_epoch":136,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:39:34.269705+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1108,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":24,"reported_epoch":146,"state":"active+clean","last_fresh":"2023-09-23T20:54:54.891062+0000","last_change":"2023-09-23T20:54:13.946937+0000","last_active":"2023-09-23T20:54:54.891062+0000","last_peered":"2023-09-23T20:54:54.891062+0000","last_clean":"2023-09-23T20:54:54.891062+0000","last_became_active":"2023-09-23T20:54:13.945496+0000","last_became_peered":"2023-09-23T20:54:13.945496+0000","last_unstale":"2023-09-23T20:54:54.891062+0000","last_undegraded":"2023-09-23T20:54:54.891062+0000","last_fullsized":"2023-09-23T20:54:54.891062+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:30:08.345600+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":6,"num_pg":26,"stat_sum":{"num_bytes":3563978776,"num_objects":54383,"num_object_clones":0,"num_object_copies":163149,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3210,"num_objects_misplaced":12659,"num_objects_unfound":0,"num_objects_dirty":54383,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":147865,"num_write_kb":6471873,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":14300,"num_bytes_recovered":763822104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":11253727232,"data_stored":11253710944,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":194494,"ondisk_log_size":194494,"up":78,"acting":76,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":38,"num_bytes_recovered":6296,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":102400,"data_stored":16582,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":161,"ondisk_log_size":161,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201454,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1723712,"kb_used_data":1615928,"kb_used_omap":0,"kb_used_meta":107584,"kb_avail":92021440,"statfs":{"total":95995035648,"available":94229954560,"internally_reserved":0,"allocated":1654710272,"data_stored":1654143191,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":110166016},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:54:40 2023","interfaces":[{"interface":"back","average":{"1min":1.9419999999999999,"5min":0.95899999999999996,"15min":0.64700000000000002},"min":{"1min":0.253,"5min":0.223,"15min":0.223},"max":{"1min":5.8029999999999999,"5min":5.8760000000000003,"15min":5.8760000000000003},"last":0.39400000000000002},{"interface":"front","average":{"1min":2.4359999999999999,"5min":1.0329999999999999,"15min":0.71899999999999997},"min":{"1min":0.34000000000000002,"5min":0.218,"15min":0.218},"max":{"1min":8.7690000000000001,"5min":8.7690000000000001,"15min":8.7690000000000001},"last":0.433}]},{"osd":1,"last update":"Sat Sep 23 20:54:11 2023","interfaces":[{"interface":"back","average":{"1min":2.23,"5min":1.2869999999999999,"15min":0.79500000000000004},"min":{"1min":0.27200000000000002,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":6.9980000000000002,"5min":22.591999999999999,"15min":22.591999999999999},"last":0.41899999999999998},{"interface":"front","average":{"1min":1.724,"5min":1.0760000000000001,"15min":0.72099999999999997},"min":{"1min":0.26000000000000001,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":7.375,"5min":21.437000000000001,"15min":21.437000000000001},"last":0.56599999999999995}]},{"osd":2,"last update":"Sat Sep 23 20:54:46 2023","interfaces":[{"interface":"back","average":{"1min":1.798,"5min":1.1319999999999999,"15min":0.74399999999999999},"min":{"1min":0.26600000000000001,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":5.9770000000000003,"5min":5.9770000000000003,"15min":5.9770000000000003},"last":2.2210000000000001},{"interface":"front","average":{"1min":1.7789999999999999,"5min":1.081,"15min":0.749},"min":{"1min":0.32300000000000001,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":5.1500000000000004,"5min":5.1500000000000004,"15min":5.1500000000000004},"last":0.59199999999999997}]},{"osd":3,"last update":"Sat Sep 23 20:54:16 2023","interfaces":[{"interface":"back","average":{"1min":6.21,"5min":2.1070000000000002,"15min":1.075},"min":{"1min":0.314,"5min":0.314,"15min":0.314},"max":{"1min":36.408000000000001,"5min":36.408000000000001,"15min":36.408000000000001},"last":0.51000000000000001},{"interface":"front","average":{"1min":4.8159999999999998,"5min":1.774,"15min":0.97399999999999998},"min":{"1min":0.45800000000000002,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":35.593000000000004,"5min":35.593000000000004,"15min":35.593000000000004},"last":0.61499999999999999}]},{"osd":4,"last update":"Sat Sep 23 20:54:06 2023","interfaces":[{"interface":"back","average":{"1min":0.623,"5min":0.55700000000000005,"15min":0.502},"min":{"1min":0.29599999999999999,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":4.5410000000000004,"5min":4.5410000000000004,"15min":4.5410000000000004},"last":0.30499999999999999},{"interface":"front","average":{"1min":0.44500000000000001,"5min":0.56399999999999995,"15min":0.53100000000000003},"min":{"1min":0.28799999999999998,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":0.60399999999999998,"5min":3.2349999999999999,"15min":3.2349999999999999},"last":0.496}]},{"osd":5,"last update":"Sat Sep 23 20:54:06 2023","interfaces":[{"interface":"back","average":{"1min":0.67800000000000005,"5min":0.59799999999999998,"15min":0.54700000000000004},"min":{"1min":0.34399999999999997,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":3.2130000000000001,"5min":3.2130000000000001,"15min":3.2130000000000001},"last":0.34399999999999997},{"interface":"front","average":{"1min":0.47099999999999997,"5min":0.58799999999999997,"15min":0.55600000000000005},"min":{"1min":0.35399999999999998,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":0.81999999999999995,"5min":1.319,"15min":1.319},"last":0.52300000000000002}]},{"osd":6,"last update":"Sat Sep 23 20:54:06 2023","interfaces":[{"interface":"back","average":{"1min":0.54300000000000004,"5min":0.61299999999999999,"15min":0.57699999999999996},"min":{"1min":0.32700000000000001,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":1.6759999999999999,"5min":1.6759999999999999,"15min":1.6759999999999999},"last":0.47699999999999998},{"interface":"front","average":{"1min":0.63100000000000001,"5min":0.63300000000000001,"15min":0.59399999999999997},"min":{"1min":0.33800000000000002,"5min":0.33800000000000002,"15min":0.316},"max":{"1min":3.3170000000000002,"5min":3.3170000000000002,"15min":3.3170000000000002},"last":0.53900000000000003}]}]},{"osd":6,"up_from":47,"seq":201863463091,"num_pgs":42,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1500272,"kb_used_data":1408872,"kb_used_omap":0,"kb_used_meta":91392,"kb_avail":92244880,"statfs":{"total":95995035648,"available":94458757120,"internally_reserved":0,"allocated":1442684928,"data_stored":1442133820,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":93585408},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:54:39 2023","interfaces":[{"interface":"back","average":{"1min":1.5820000000000001,"5min":1.0089999999999999,"15min":0.72199999999999998},"min":{"1min":0.222,"5min":0.222,"15min":0.222},"max":{"1min":7.7249999999999996,"5min":7.7249999999999996,"15min":7.7249999999999996},"last":1.8620000000000001},{"interface":"front","average":{"1min":1.278,"5min":1.1080000000000001,"15min":0.76200000000000001},"min":{"1min":0.252,"5min":0.252,"15min":0.252},"max":{"1min":5.4509999999999996,"5min":10.699999999999999,"15min":10.699999999999999},"last":0.255}]},{"osd":1,"last update":"Sat Sep 23 20:54:19 2023","interfaces":[{"interface":"back","average":{"1min":3.1150000000000002,"5min":1.284,"15min":0.78400000000000003},"min":{"1min":0.39800000000000002,"5min":0.252,"15min":0.252},"max":{"1min":19.024000000000001,"5min":19.024000000000001,"15min":19.024000000000001},"last":1.448},{"interface":"front","average":{"1min":2.5659999999999998,"5min":1.125,"15min":0.755},"min":{"1min":0.255,"5min":0.255,"15min":0.255},"max":{"1min":16.536000000000001,"5min":16.536000000000001,"15min":16.536000000000001},"last":0.63800000000000001}]},{"osd":2,"last update":"Sat Sep 23 20:54:50 2023","interfaces":[{"interface":"back","average":{"1min":2.4900000000000002,"5min":1.4710000000000001,"15min":0.85499999999999998},"min":{"1min":0.36399999999999999,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":18.780999999999999,"5min":18.780999999999999,"15min":18.780999999999999},"last":1.5680000000000001},{"interface":"front","average":{"1min":2.504,"5min":1.4730000000000001,"15min":0.88100000000000001},"min":{"1min":0.32800000000000001,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":18.765999999999998,"5min":18.765999999999998,"15min":18.765999999999998},"last":1.583}]},{"osd":3,"last update":"Sat Sep 23 20:54:19 2023","interfaces":[{"interface":"back","average":{"1min":3.089,"5min":1.1830000000000001,"15min":0.78900000000000003},"min":{"1min":0.27200000000000002,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":16.524999999999999,"5min":16.524999999999999,"15min":16.524999999999999},"last":0.42299999999999999},{"interface":"front","average":{"1min":2.8900000000000001,"5min":1.28,"15min":0.81399999999999995},"min":{"1min":0.35899999999999999,"5min":0.315,"15min":0.315},"max":{"1min":16.617999999999999,"5min":16.617999999999999,"15min":16.617999999999999},"last":2.4550000000000001}]},{"osd":4,"last update":"Sat Sep 23 20:54:50 2023","interfaces":[{"interface":"back","average":{"1min":0.38700000000000001,"5min":0.501,"15min":0.47099999999999997},"min":{"1min":0.27500000000000002,"5min":0.248,"15min":0.248},"max":{"1min":0.65200000000000002,"5min":4.952,"15min":4.952},"last":3.4470000000000001},{"interface":"front","average":{"1min":0.48099999999999998,"5min":0.53900000000000003,"15min":0.502},"min":{"1min":0.32300000000000001,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":1.734,"5min":2.8300000000000001,"15min":2.8300000000000001},"last":0.36099999999999999}]},{"osd":5,"last update":"Sat Sep 23 20:54:50 2023","interfaces":[{"interface":"back","average":{"1min":0.45800000000000002,"5min":0.58899999999999997,"15min":0.52400000000000002},"min":{"1min":0.28599999999999998,"5min":0.28599999999999998,"15min":0.28399999999999997},"max":{"1min":0.71799999999999997,"5min":8.423,"15min":8.423},"last":3.2559999999999998},{"interface":"front","average":{"1min":0.40899999999999997,"5min":0.54000000000000004,"15min":0.52400000000000002},"min":{"1min":0.30099999999999999,"5min":0.30099999999999999,"15min":0.28499999999999998},"max":{"1min":0.621,"5min":2.8620000000000001,"15min":2.8620000000000001},"last":0.39500000000000002}]},{"osd":7,"last update":"Sat Sep 23 20:54:07 2023","interfaces":[{"interface":"back","average":{"1min":0.88900000000000001,"5min":0.625,"15min":0.56599999999999995},"min":{"1min":0.379,"5min":0.34799999999999998,"15min":0.32500000000000001},"max":{"1min":5.282,"5min":5.282,"15min":5.282},"last":0.70799999999999996},{"interface":"front","average":{"1min":0.752,"5min":0.63100000000000001,"15min":0.59499999999999997},"min":{"1min":0.309,"5min":0.309,"15min":0.309},"max":{"1min":3.0590000000000002,"5min":3.0590000000000002,"15min":3.0590000000000002},"last":0.69699999999999995}]}]},{"osd":1,"up_from":88,"seq":377957122108,"num_pgs":47,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1444004,"kb_used_data":1332060,"kb_used_omap":9,"kb_used_meta":111926,"kb_avail":92301148,"statfs":{"total":95995035648,"available":94516375552,"internally_reserved":0,"allocated":1364029440,"data_stored":1363351722,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9239,"internal_metadata":114613225},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:54:09 2023","interfaces":[{"interface":"back","average":{"1min":3.3740000000000001,"5min":1.1020000000000001,"15min":0.64000000000000001},"min":{"1min":0.22800000000000001,"5min":0.216,"15min":0.216},"max":{"1min":11.266,"5min":11.266,"15min":11.266},"last":10.255000000000001},{"interface":"front","average":{"1min":4.0679999999999996,"5min":1.276,"15min":0.71499999999999997},"min":{"1min":0.22500000000000001,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":18.457999999999998,"5min":18.457999999999998,"15min":18.457999999999998},"last":11.266}]},{"osd":2,"last update":"Sat Sep 23 20:54:43 2023","interfaces":[{"interface":"back","average":{"1min":2.9449999999999998,"5min":1.2509999999999999,"15min":0.67500000000000004},"min":{"1min":0.222,"5min":0.20300000000000001,"15min":0.20300000000000001},"max":{"1min":11.391999999999999,"5min":11.391999999999999,"15min":11.391999999999999},"last":3.641},{"interface":"front","average":{"1min":3.0390000000000001,"5min":1.909,"15min":1.026},"min":{"1min":0.26400000000000001,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":11.552,"5min":38.951000000000001,"15min":38.951000000000001},"last":3.7919999999999998}]},{"osd":3,"last update":"Sat Sep 23 20:54:15 2023","interfaces":[{"interface":"back","average":{"1min":5.25,"5min":1.6890000000000001,"15min":0.80200000000000005},"min":{"1min":0.25700000000000001,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":25.314,"5min":25.314,"15min":25.314},"last":1.635},{"interface":"front","average":{"1min":3.734,"5min":1.337,"15min":0.71899999999999997},"min":{"1min":0.28000000000000003,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":19.881,"5min":19.881,"15min":19.881},"last":3.597}]},{"osd":4,"last update":"Sat Sep 23 20:54:09 2023","interfaces":[{"interface":"back","average":{"1min":3.4260000000000002,"5min":1.349,"15min":0.88100000000000001},"min":{"1min":0.317,"5min":0.317,"15min":0.317},"max":{"1min":12.006,"5min":12.006,"15min":12.006},"last":0.436},{"interface":"front","average":{"1min":1.681,"5min":0.89200000000000002,"15min":0.746},"min":{"1min":0.29899999999999999,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":11.234999999999999,"5min":11.234999999999999,"15min":11.234999999999999},"last":3.6190000000000002}]},{"osd":5,"last update":"Sat Sep 23 20:54:09 2023","interfaces":[{"interface":"back","average":{"1min":2.2999999999999998,"5min":1.032,"15min":0.80900000000000005},"min":{"1min":0.36799999999999999,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":12.222,"5min":12.222,"15min":12.222},"last":3.5169999999999999},{"interface":"front","average":{"1min":3.2429999999999999,"5min":1.3149999999999999,"15min":0.86899999999999999},"min":{"1min":0.39300000000000002,"5min":0.377,"15min":0.377},"max":{"1min":12.08,"5min":12.08,"15min":12.08},"last":0.504}]},{"osd":6,"last update":"Sat Sep 23 20:54:09 2023","interfaces":[{"interface":"back","average":{"1min":2.661,"5min":1.109,"15min":0.82799999999999996},"min":{"1min":0.307,"5min":0.307,"15min":0.307},"max":{"1min":12.276999999999999,"5min":12.276999999999999,"15min":12.276999999999999},"last":3.4809999999999999},{"interface":"front","average":{"1min":3.4089999999999998,"5min":1.3560000000000001,"15min":0.89000000000000001},"min":{"1min":0.38,"5min":0.38,"15min":0.38},"max":{"1min":12.054,"5min":12.054,"15min":12.054},"last":0.67900000000000005}]},{"osd":7,"last update":"Sat Sep 23 20:54:09 2023","interfaces":[{"interface":"back","average":{"1min":1.6910000000000001,"5min":0.94599999999999995,"15min":0.76400000000000001},"min":{"1min":0.35699999999999998,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":11.186999999999999,"5min":11.186999999999999,"15min":11.186999999999999},"last":3.5720000000000001},{"interface":"front","average":{"1min":2.4470000000000001,"5min":1.087,"15min":0.81899999999999995},"min":{"1min":0.34599999999999997,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":12.252000000000001,"5min":12.252000000000001,"15min":12.252000000000001},"last":3.54}]}]},{"osd":0,"up_from":83,"seq":356482285635,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":959568,"kb_used_data":883656,"kb_used_omap":28,"kb_used_meta":75875,"kb_avail":92785584,"statfs":{"total":95995035648,"available":95012438016,"internally_reserved":0,"allocated":904863744,"data_stored":904160448,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":28848,"internal_metadata":77696848},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 20:54:13 2023","interfaces":[{"interface":"back","average":{"1min":2.6989999999999998,"5min":1.002,"15min":0.55600000000000005},"min":{"1min":0.20100000000000001,"5min":0.20100000000000001,"15min":0.20100000000000001},"max":{"1min":18.863,"5min":18.863,"15min":18.863},"last":2.8919999999999999},{"interface":"front","average":{"1min":3.5619999999999998,"5min":1.1970000000000001,"15min":0.65700000000000003},"min":{"1min":0.22600000000000001,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":17.704999999999998,"5min":17.704999999999998,"15min":17.704999999999998},"last":1.087}]},{"osd":2,"last update":"Sat Sep 23 20:54:43 2023","interfaces":[{"interface":"back","average":{"1min":3.0720000000000001,"5min":1.7689999999999999,"15min":0.86599999999999999},"min":{"1min":0.24299999999999999,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":13.052,"5min":17.753,"15min":17.753},"last":0.36699999999999999},{"interface":"front","average":{"1min":4.0199999999999996,"5min":1.6970000000000001,"15min":0.82499999999999996},"min":{"1min":0.311,"5min":0.21099999999999999,"15min":0.21099999999999999},"max":{"1min":13.218999999999999,"5min":13.218999999999999,"15min":13.218999999999999},"last":0.35099999999999998}]},{"osd":3,"last update":"Sat Sep 23 20:54:13 2023","interfaces":[{"interface":"back","average":{"1min":3.9500000000000002,"5min":1.3109999999999999,"15min":0.70399999999999996},"min":{"1min":0.309,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":18.893999999999998,"5min":18.893999999999998,"15min":18.893999999999998},"last":1.49},{"interface":"front","average":{"1min":3.617,"5min":1.355,"15min":0.77000000000000002},"min":{"1min":0.34999999999999998,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":17.472000000000001,"5min":17.472000000000001,"15min":17.472000000000001},"last":0.40999999999999998}]},{"osd":4,"last update":"Sat Sep 23 20:54:43 2023","interfaces":[{"interface":"back","average":{"1min":2.4620000000000002,"5min":1.2070000000000001,"15min":0.83399999999999996},"min":{"1min":0.32700000000000001,"5min":0.311,"15min":0.311},"max":{"1min":12.807,"5min":12.807,"15min":12.807},"last":0.443},{"interface":"front","average":{"1min":2.843,"5min":1.1910000000000001,"15min":0.83599999999999997},"min":{"1min":0.38500000000000001,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":18.724,"5min":18.724,"15min":18.724},"last":1.387}]},{"osd":5,"last update":"Sat Sep 23 20:54:43 2023","interfaces":[{"interface":"back","average":{"1min":1.4510000000000001,"5min":1.2370000000000001,"15min":0.84499999999999997},"min":{"1min":0.33400000000000002,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":4.8310000000000004,"5min":8.4559999999999995,"15min":8.4559999999999995},"last":0.46300000000000002},{"interface":"front","average":{"1min":2.5600000000000001,"5min":1.258,"15min":0.89000000000000001},"min":{"1min":0.35499999999999998,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":13.154999999999999,"5min":13.154999999999999,"15min":13.154999999999999},"last":4.157}]},{"osd":6,"last update":"Sat Sep 23 20:54:43 2023","interfaces":[{"interface":"back","average":{"1min":2.492,"5min":1.2370000000000001,"15min":0.873},"min":{"1min":0.35899999999999999,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":13.19,"5min":13.19,"15min":13.19},"last":4.2770000000000001},{"interface":"front","average":{"1min":2.718,"5min":1.202,"15min":0.85399999999999998},"min":{"1min":0.39600000000000002,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":18.829000000000001,"5min":18.829000000000001,"15min":18.829000000000001},"last":1.423}]},{"osd":7,"last update":"Sat Sep 23 20:54:43 2023","interfaces":[{"interface":"back","average":{"1min":1.454,"5min":1.2370000000000001,"15min":0.879},"min":{"1min":0.38400000000000001,"5min":0.38400000000000001,"15min":0.38400000000000001},"max":{"1min":4.7830000000000004,"5min":8.3979999999999997,"15min":8.3979999999999997},"last":1.4530000000000001},{"interface":"front","average":{"1min":2.681,"5min":1.1830000000000001,"15min":0.84299999999999997},"min":{"1min":0.38500000000000001,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":18.673999999999999,"5min":18.673999999999999,"15min":18.673999999999999},"last":1.5329999999999999}]}]},{"osd":2,"up_from":93,"seq":399431958582,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2344000,"kb_used_data":2195832,"kb_used_omap":81,"kb_used_meta":148078,"kb_avail":91401152,"statfs":{"total":95995035648,"available":93594779648,"internally_reserved":0,"allocated":2248531968,"data_stored":2247847938,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":83750,"internal_metadata":151632090},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:54:40 2023","interfaces":[{"interface":"back","average":{"1min":1.7110000000000001,"5min":1.0840000000000001,"15min":0.59999999999999998},"min":{"1min":0.249,"5min":0.20499999999999999,"15min":0.20499999999999999},"max":{"1min":11.183,"5min":11.183,"15min":11.183},"last":0.753},{"interface":"front","average":{"1min":3.0139999999999998,"5min":1.278,"15min":0.69699999999999995},"min":{"1min":0.29799999999999999,"5min":0.222,"15min":0.222},"max":{"1min":11.282999999999999,"5min":11.282999999999999,"15min":11.282999999999999},"last":0.68300000000000005}]},{"osd":1,"last update":"Sat Sep 23 20:54:40 2023","interfaces":[{"interface":"back","average":{"1min":3.3799999999999999,"5min":1.8979999999999999,"15min":0.91000000000000003},"min":{"1min":0.34000000000000002,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":11.215999999999999,"5min":16.081,"15min":16.081},"last":6.0309999999999997},{"interface":"front","average":{"1min":5.3369999999999997,"5min":2.2320000000000002,"15min":1.024},"min":{"1min":0.374,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":12.07,"5min":16.041,"15min":16.041},"last":5.9930000000000003}]},{"osd":3,"last update":"Sat Sep 23 20:54:16 2023","interfaces":[{"interface":"back","average":{"1min":4.0650000000000004,"5min":1.6080000000000001,"15min":0.81200000000000006},"min":{"1min":0.48299999999999998,"5min":0.255,"15min":0.255},"max":{"1min":14.18,"5min":14.18,"15min":14.18},"last":5.9610000000000003},{"interface":"front","average":{"1min":5.7880000000000003,"5min":1.9650000000000001,"15min":0.90800000000000003},"min":{"1min":0.29499999999999998,"5min":0.221,"15min":0.221},"max":{"1min":17.138999999999999,"5min":17.138999999999999,"15min":17.138999999999999},"last":5.8620000000000001}]},{"osd":4,"last update":"Sat Sep 23 20:54:40 2023","interfaces":[{"interface":"back","average":{"1min":2.653,"5min":1.228,"15min":0.82199999999999995},"min":{"1min":0.35499999999999998,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":9.8119999999999994,"5min":9.8119999999999994,"15min":9.8119999999999994},"last":0.59499999999999997},{"interface":"front","average":{"1min":2.6829999999999998,"5min":1.258,"15min":0.86599999999999999},"min":{"1min":0.34100000000000003,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":8.6750000000000007,"5min":8.6750000000000007,"15min":8.6750000000000007},"last":5.1440000000000001}]},{"osd":5,"last update":"Sat Sep 23 20:54:40 2023","interfaces":[{"interface":"back","average":{"1min":2.6600000000000001,"5min":1.2529999999999999,"15min":0.85199999999999998},"min":{"1min":0.40300000000000002,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":9.7629999999999999,"5min":9.7629999999999999,"15min":9.7629999999999999},"last":0.626},{"interface":"front","average":{"1min":2.7229999999999999,"5min":1.3380000000000001,"15min":0.87},"min":{"1min":0.39700000000000002,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":8.7509999999999994,"5min":8.7509999999999994,"15min":8.7509999999999994},"last":5.2220000000000004}]},{"osd":6,"last update":"Sat Sep 23 20:54:40 2023","interfaces":[{"interface":"back","average":{"1min":2.7040000000000002,"5min":1.3580000000000001,"15min":0.92300000000000004},"min":{"1min":0.38800000000000001,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":8.6609999999999996,"5min":8.6609999999999996,"15min":8.6609999999999996},"last":5.1900000000000004},{"interface":"front","average":{"1min":1.167,"5min":1.048,"15min":0.80600000000000005},"min":{"1min":0.41099999999999998,"5min":0.40100000000000002,"15min":0.40100000000000002},"max":{"1min":3.3639999999999999,"5min":6.431,"15min":6.431},"last":0.72799999999999998}]},{"osd":7,"last update":"Sat Sep 23 20:54:40 2023","interfaces":[{"interface":"back","average":{"1min":1.139,"5min":1.018,"15min":0.76200000000000001},"min":{"1min":0.371,"5min":0.371,"15min":0.371},"max":{"1min":3.4020000000000001,"5min":6.4080000000000004,"15min":6.4080000000000004},"last":0.78000000000000003},{"interface":"front","average":{"1min":2.8119999999999998,"5min":1.343,"15min":0.88900000000000001},"min":{"1min":0.40899999999999997,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":9.7859999999999996,"5min":9.7859999999999996,"15min":9.7859999999999996},"last":0.64900000000000002}]}]},{"osd":3,"up_from":98,"seq":420906795055,"num_pgs":52,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2337928,"kb_used_data":2193920,"kb_used_omap":7,"kb_used_meta":143928,"kb_avail":91407224,"statfs":{"total":95995035648,"available":93600997376,"internally_reserved":0,"allocated":2246574080,"data_stored":2245919693,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7576,"internal_metadata":147382888},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:54:18 2023","interfaces":[{"interface":"back","average":{"1min":7.1630000000000003,"5min":2.4350000000000001,"15min":1.0589999999999999},"min":{"1min":0.249,"5min":0.249,"15min":0.249},"max":{"1min":23.123999999999999,"5min":23.123999999999999,"15min":23.123999999999999},"last":1.554},{"interface":"front","average":{"1min":3.609,"5min":1.409,"15min":0.70799999999999996},"min":{"1min":0.20000000000000001,"5min":0.20000000000000001,"15min":0.20000000000000001},"max":{"1min":11.598000000000001,"5min":11.598000000000001,"15min":11.598000000000001},"last":15.079000000000001}]},{"osd":1,"last update":"Sat Sep 23 20:54:18 2023","interfaces":[{"interface":"back","average":{"1min":4.5949999999999998,"5min":1.6459999999999999,"15min":0.79300000000000004},"min":{"1min":0.22900000000000001,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":21.385000000000002,"5min":21.385000000000002,"15min":21.385000000000002},"last":13.683999999999999},{"interface":"front","average":{"1min":8.0389999999999997,"5min":2.4319999999999999,"15min":1.0700000000000001},"min":{"1min":0.34000000000000002,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":25.353000000000002,"5min":25.353000000000002,"15min":25.353000000000002},"last":10.321}]},{"osd":2,"last update":"Sat Sep 23 20:54:18 2023","interfaces":[{"interface":"back","average":{"1min":4.25,"5min":1.621,"15min":0.82299999999999995},"min":{"1min":0.30099999999999999,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":24.154,"5min":24.154,"15min":24.154},"last":1.4279999999999999},{"interface":"front","average":{"1min":7.9420000000000002,"5min":2.4580000000000002,"15min":1.1120000000000001},"min":{"1min":0.313,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":25.305,"5min":25.305,"15min":25.305},"last":1.5820000000000001}]},{"osd":4,"last update":"Sat Sep 23 20:54:18 2023","interfaces":[{"interface":"back","average":{"1min":1.9750000000000001,"5min":1.2370000000000001,"15min":0.83299999999999996},"min":{"1min":0.39700000000000002,"5min":0.39700000000000002,"15min":0.39700000000000002},"max":{"1min":11.861000000000001,"5min":11.861000000000001,"15min":11.861000000000001},"last":0.64300000000000002},{"interface":"front","average":{"1min":5.5970000000000004,"5min":2.0190000000000001,"15min":1.091},"min":{"1min":0.41699999999999998,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":22.853000000000002,"5min":22.853000000000002,"15min":22.853000000000002},"last":0.45400000000000001}]},{"osd":5,"last update":"Sat Sep 23 20:54:18 2023","interfaces":[{"interface":"back","average":{"1min":2.0950000000000002,"5min":1.181,"15min":0.82399999999999995},"min":{"1min":0.375,"5min":0.375,"15min":0.375},"max":{"1min":11.891999999999999,"5min":11.891999999999999,"15min":11.891999999999999},"last":0.56999999999999995},{"interface":"front","average":{"1min":2.1200000000000001,"5min":1.2290000000000001,"15min":0.83399999999999996},"min":{"1min":0.42899999999999999,"5min":0.42899999999999999,"15min":0.42899999999999999},"max":{"1min":9.2870000000000008,"5min":9.2870000000000008,"15min":9.2870000000000008},"last":0.52100000000000002}]},{"osd":6,"last update":"Sat Sep 23 20:54:18 2023","interfaces":[{"interface":"back","average":{"1min":2.2229999999999999,"5min":1.2829999999999999,"15min":0.88800000000000001},"min":{"1min":0.40899999999999997,"5min":0.38100000000000001,"15min":0.38100000000000001},"max":{"1min":9.2360000000000007,"5min":9.2360000000000007,"15min":9.2360000000000007},"last":0.623},{"interface":"front","average":{"1min":2.524,"5min":1.3959999999999999,"15min":0.94399999999999995},"min":{"1min":0.34899999999999998,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":11.824,"5min":11.824,"15min":11.824},"last":0.53300000000000003}]},{"osd":7,"last update":"Sat Sep 23 20:54:18 2023","interfaces":[{"interface":"back","average":{"1min":2.6509999999999998,"5min":1.3220000000000001,"15min":0.90400000000000003},"min":{"1min":0.42099999999999999,"5min":0.42099999999999999,"15min":0.42099999999999999},"max":{"1min":12.41,"5min":12.41,"15min":12.41},"last":0.60999999999999999},{"interface":"front","average":{"1min":2.0830000000000002,"5min":1.26,"15min":0.88200000000000001},"min":{"1min":0.39300000000000002,"5min":0.39300000000000002,"15min":0.39300000000000002},"max":{"1min":9.3179999999999996,"5min":9.3179999999999996,"15min":9.3179999999999996},"last":0.55600000000000005}]}]},{"osd":4,"up_from":31,"seq":133143986363,"num_pgs":42,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1188476,"kb_used_data":1120244,"kb_used_omap":0,"kb_used_meta":68224,"kb_avail":92556676,"statfs":{"total":95995035648,"available":94778036224,"internally_reserved":0,"allocated":1147129856,"data_stored":1146566580,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":69861376},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:54:46 2023","interfaces":[{"interface":"back","average":{"1min":1.1539999999999999,"5min":0.88,"15min":0.71599999999999997},"min":{"1min":0.32700000000000001,"5min":0.249,"15min":0.249},"max":{"1min":3.169,"5min":6.6260000000000003,"15min":6.6260000000000003},"last":8.7189999999999994},{"interface":"front","average":{"1min":1.2769999999999999,"5min":1.0569999999999999,"15min":0.96299999999999997},"min":{"1min":0.25800000000000001,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":3.4380000000000002,"5min":8.327,"15min":8.327},"last":8.7050000000000001}]},{"osd":1,"last update":"Sat Sep 23 20:54:12 2023","interfaces":[{"interface":"back","average":{"1min":2.149,"5min":1.1080000000000001,"15min":0.74199999999999999},"min":{"1min":0.26600000000000001,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":7.2009999999999996,"5min":7.9859999999999998,"15min":7.9859999999999998},"last":2.6309999999999998},{"interface":"front","average":{"1min":1.464,"5min":0.81599999999999995,"15min":0.67100000000000004},"min":{"1min":0.309,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":6.3540000000000001,"5min":6.3540000000000001,"15min":6.3540000000000001},"last":1.532}]},{"osd":2,"last update":"Sat Sep 23 20:54:46 2023","interfaces":[{"interface":"back","average":{"1min":2.6760000000000002,"5min":1.294,"15min":0.88800000000000001},"min":{"1min":0.39100000000000001,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":15.288,"5min":15.288,"15min":15.288},"last":2.0819999999999999},{"interface":"front","average":{"1min":3.5470000000000002,"5min":1.595,"15min":1.0469999999999999},"min":{"1min":0.41799999999999998,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":15.262,"5min":15.262,"15min":15.262},"last":1.4370000000000001}]},{"osd":3,"last update":"Sat Sep 23 20:54:12 2023","interfaces":[{"interface":"back","average":{"1min":3.8069999999999999,"5min":2.0179999999999998,"15min":1.107},"min":{"1min":0.42199999999999999,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":15.916,"5min":31.562999999999999,"15min":31.562999999999999},"last":6.4800000000000004},{"interface":"front","average":{"1min":3.5070000000000001,"5min":2.0099999999999998,"15min":1.1779999999999999},"min":{"1min":0.47599999999999998,"5min":0.41899999999999998,"15min":0.41899999999999998},"max":{"1min":15.882,"5min":31.574000000000002,"15min":31.574000000000002},"last":6.4930000000000003}]},{"osd":5,"last update":"Sat Sep 23 20:54:26 2023","interfaces":[{"interface":"back","average":{"1min":0.42299999999999999,"5min":0.499,"15min":0.46999999999999997},"min":{"1min":0.30599999999999999,"5min":0.30499999999999999,"15min":0.251},"max":{"1min":0.69899999999999995,"5min":0.995,"15min":0.995},"last":0.29099999999999998},{"interface":"front","average":{"1min":0.53300000000000003,"5min":0.57999999999999996,"15min":0.504},"min":{"1min":0.29199999999999998,"5min":0.29199999999999998,"15min":0.28799999999999998},"max":{"1min":1.4399999999999999,"5min":4.4249999999999998,"15min":4.4249999999999998},"last":0.92600000000000005}]},{"osd":6,"last update":"Sat Sep 23 20:53:57 2023","interfaces":[{"interface":"back","average":{"1min":1.159,"5min":0.64800000000000002,"15min":0.55300000000000005},"min":{"1min":0.27500000000000002,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":6.8289999999999997,"5min":6.8289999999999997,"15min":6.8289999999999997},"last":0.34100000000000003},{"interface":"front","average":{"1min":0.97999999999999998,"5min":0.68300000000000005,"15min":0.59899999999999998},"min":{"1min":0.31,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":5.8179999999999996,"5min":5.8179999999999996,"15min":5.8179999999999996},"last":0.32700000000000001}]},{"osd":7,"last update":"Sat Sep 23 20:54:12 2023","interfaces":[{"interface":"back","average":{"1min":1.23,"5min":0.754,"15min":0.626},"min":{"1min":0.29999999999999999,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":7.383,"5min":7.383,"15min":7.383},"last":0.38500000000000001},{"interface":"front","average":{"1min":0.86199999999999999,"5min":0.68200000000000005,"15min":0.59899999999999998},"min":{"1min":0.33400000000000002,"5min":0.33400000000000002,"15min":0.32700000000000001},"max":{"1min":3.363,"5min":3.9420000000000002,"15min":3.9420000000000002},"last":0.95299999999999996}]}]},{"osd":5,"up_from":39,"seq":167503724728,"num_pgs":28,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":336944,"kb_used_data":295080,"kb_used_omap":0,"kb_used_meta":41856,"kb_avail":93408208,"statfs":{"total":95995035648,"available":95650004992,"internally_reserved":0,"allocated":302161920,"data_stored":301606700,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":42860544},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:54:35 2023","interfaces":[{"interface":"back","average":{"1min":1.0900000000000001,"5min":0.77400000000000002,"15min":0.58799999999999997},"min":{"1min":0.23499999999999999,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":7.766,"5min":7.766,"15min":7.766},"last":0.36699999999999999},{"interface":"front","average":{"1min":1.385,"5min":0.85099999999999998,"15min":0.66700000000000004},"min":{"1min":0.26000000000000001,"5min":0.251,"15min":0.251},"max":{"1min":6.1059999999999999,"5min":6.1059999999999999,"15min":6.1059999999999999},"last":0.223}]},{"osd":1,"last update":"Sat Sep 23 20:54:10 2023","interfaces":[{"interface":"back","average":{"1min":2.0019999999999998,"5min":0.97399999999999998,"15min":0.70199999999999996},"min":{"1min":0.309,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":7.2990000000000004,"5min":7.2990000000000004,"15min":7.2990000000000004},"last":0.24399999999999999},{"interface":"front","average":{"1min":1.968,"5min":0.92300000000000004,"15min":0.67300000000000004},"min":{"1min":0.26200000000000001,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":7.4850000000000003,"5min":7.4850000000000003,"15min":7.4850000000000003},"last":0.32800000000000001}]},{"osd":2,"last update":"Sat Sep 23 20:54:44 2023","interfaces":[{"interface":"back","average":{"1min":2.3340000000000001,"5min":1.151,"15min":0.745},"min":{"1min":0.26800000000000002,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":12.651,"5min":12.651,"15min":12.651},"last":0.41799999999999998},{"interface":"front","average":{"1min":1.357,"5min":0.98099999999999998,"15min":0.71599999999999997},"min":{"1min":0.28100000000000003,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":3.4950000000000001,"5min":7.8109999999999999,"15min":7.8109999999999999},"last":0.318}]},{"osd":3,"last update":"Sat Sep 23 20:54:11 2023","interfaces":[{"interface":"back","average":{"1min":2.8740000000000001,"5min":1.222,"15min":0.82199999999999995},"min":{"1min":0.36899999999999999,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":9.5489999999999995,"5min":9.5489999999999995,"15min":9.5489999999999995},"last":0.307},{"interface":"front","average":{"1min":2.0680000000000001,"5min":0.97499999999999998,"15min":0.747},"min":{"1min":0.42799999999999999,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":6.2220000000000004,"5min":6.2220000000000004,"15min":6.2220000000000004},"last":0.436}]},{"osd":4,"last update":"Sat Sep 23 20:54:23 2023","interfaces":[{"interface":"back","average":{"1min":0.60699999999999998,"5min":0.54600000000000004,"15min":0.48999999999999999},"min":{"1min":0.29099999999999998,"5min":0.29099999999999998,"15min":0.253},"max":{"1min":2.7090000000000001,"5min":2.7090000000000001,"15min":2.7090000000000001},"last":0.28599999999999998},{"interface":"front","average":{"1min":0.66500000000000004,"5min":0.54400000000000004,"15min":0.51100000000000001},"min":{"1min":0.35299999999999998,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":3.0179999999999998,"5min":3.0179999999999998,"15min":3.0179999999999998},"last":0.40000000000000002}]},{"osd":6,"last update":"Sat Sep 23 20:54:48 2023","interfaces":[{"interface":"back","average":{"1min":0.44500000000000001,"5min":0.56999999999999995,"15min":0.53200000000000003},"min":{"1min":0.308,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":0.92800000000000005,"5min":7.2080000000000002,"15min":7.2080000000000002},"last":0.26400000000000001},{"interface":"front","average":{"1min":0.624,"5min":0.61799999999999999,"15min":0.56299999999999994},"min":{"1min":0.35499999999999998,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":2.379,"5min":2.379,"15min":2.379},"last":0.39400000000000002}]},{"osd":7,"last update":"Sat Sep 23 20:54:05 2023","interfaces":[{"interface":"back","average":{"1min":0.53000000000000003,"5min":0.56299999999999994,"15min":0.56100000000000005},"min":{"1min":0.30199999999999999,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":0.85399999999999998,"5min":0.91800000000000004,"15min":0.91800000000000004},"last":0.34300000000000003},{"interface":"front","average":{"1min":0.72899999999999998,"5min":0.64600000000000002,"15min":0.59799999999999998},"min":{"1min":0.27700000000000002,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":2.4870000000000001,"5min":2.4870000000000001,"15min":2.4870000000000001},"last":0.35399999999999998}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":902631424,"data_stored":902627352,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1360728064,"data_stored":1360723992,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":2236944384,"data_stored":2236940312,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":2237136896,"data_stored":2237136896,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1138622464,"data_stored":1138622464,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":308744192,"data_stored":308740120,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1417281536,"data_stored":1417281536,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1651638272,"data_stored":1651638272,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T20:54:59.891 INFO:tasks.cephadm.ceph_manager.ceph:increase pool size by 10 2023-09-23T20:54:59.891 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph osd pool set .mgr pg_num 11 2023-09-23T20:54:59.896 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_blocked_ops 2023-09-23T20:55:00.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:54:59 smithi070 ceph-mon[168282]: pgmap v362: 99 pgs: 1 active+remapped+backfilling, 2 active+recovery_wait+degraded+remapped, 91 active+clean, 3 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped; 3.3 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 581 op/s; 3210/163221 objects degraded (1.967%); 12659/163221 objects misplaced (7.756%); 15 MiB/s, 377 objects/s recovering 2023-09-23T20:55:00.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:54:59 smithi070 ceph-mon[168282]: Health check update: Degraded data redundancy: 3210/163221 objects degraded (1.967%), 4 pgs degraded (PG_DEGRADED) 2023-09-23T20:55:00.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:54:59 smithi026 ceph-mon[179688]: pgmap v362: 99 pgs: 1 active+remapped+backfilling, 2 active+recovery_wait+degraded+remapped, 91 active+clean, 3 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped; 3.3 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 581 op/s; 3210/163221 objects degraded (1.967%); 12659/163221 objects misplaced (7.756%); 15 MiB/s, 377 objects/s recovering 2023-09-23T20:55:00.134 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:54:59 smithi026 ceph-mon[179688]: Health check update: Degraded data redundancy: 3210/163221 objects degraded (1.967%), 4 pgs degraded (PG_DEGRADED) 2023-09-23T20:55:00.134 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:54:59 smithi026 ceph-mon[175151]: pgmap v362: 99 pgs: 1 active+remapped+backfilling, 2 active+recovery_wait+degraded+remapped, 91 active+clean, 3 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped; 3.3 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 581 op/s; 3210/163221 objects degraded (1.967%); 12659/163221 objects misplaced (7.756%); 15 MiB/s, 377 objects/s recovering 2023-09-23T20:55:00.134 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:54:59 smithi026 ceph-mon[175151]: Health check update: Degraded data redundancy: 3210/163221 objects degraded (1.967%), 4 pgs degraded (PG_DEGRADED) 2023-09-23T20:55:00.257 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:55:00.262 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:55:00.262 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:55:00.363 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:55:00.393 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:55:00.440 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:55:00.443 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0: {} 2023-09-23T20:55:00.445 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:55:00.445 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:55:00.483 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1: {} 2023-09-23T20:55:00.546 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:55:00.547 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2: {} 2023-09-23T20:55:00.580 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:55:00.581 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:55:00.601 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3: {} 2023-09-23T20:55:00.630 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:55:00.634 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:55:00.635 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:55:00.661 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4: {} 2023-09-23T20:55:00.700 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T20:55:00.735 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:55:00.742 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6: {} 2023-09-23T20:55:00.790 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T20:55:00.810 INFO:teuthology.orchestra.run.smithi026.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T20:55:00.811 INFO:teuthology.orchestra.run.smithi026.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T20:55:00.811 INFO:teuthology.orchestra.run.smithi026.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T20:55:00.811 INFO:teuthology.orchestra.run.smithi026.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T20:55:00.811 INFO:teuthology.orchestra.run.smithi026.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T20:55:00.811 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T20:55:00.811 INFO:teuthology.orchestra.run.smithi026.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T20:55:00.811 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T20:55:00.836 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:55:01.121 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:55:01 smithi070 ceph-mon[168282]: from='client.54781 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:55:01.122 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:55:01 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/2516211615' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:55:01.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:55:01 smithi026 ceph-mon[179688]: from='client.54781 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:55:01.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:55:01 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/2516211615' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:55:01.136 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:55:01 smithi026 ceph-mon[175151]: from='client.54781 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:55:01.136 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:55:01 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/2516211615' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:55:01.228 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:55:01.233 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:55:01.234 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:55:01.334 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:55:01.396 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:55:01.863 INFO:teuthology.orchestra.run.smithi070.stdout:{ 2023-09-23T20:55:01.863 INFO:teuthology.orchestra.run.smithi070.stdout: "ops": [], 2023-09-23T20:55:01.863 INFO:teuthology.orchestra.run.smithi070.stdout: "complaint_time": 30, 2023-09-23T20:55:01.864 INFO:teuthology.orchestra.run.smithi070.stdout: "num_blocked_ops": 0 2023-09-23T20:55:01.864 INFO:teuthology.orchestra.run.smithi070.stdout:} 2023-09-23T20:55:02.037 INFO:teuthology.orchestra.run.smithi026.stderr:nodeep-scrub is unset 2023-09-23T20:55:02.252 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:55:02 smithi070 ceph-mon[168282]: pgmap v363: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfilling, 2 active+recovery_wait+degraded+remapped, 91 active+clean, 3 active+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped; 3.3 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 722 op/s; 2421/160005 objects degraded (1.513%); 12089/160005 objects misplaced (7.555%); 18 MiB/s, 453 objects/s recovering 2023-09-23T20:55:02.253 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:55:02 smithi070 ceph-mon[168282]: failed to encode map e148 with expected crc 2023-09-23T20:55:02.253 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:55:02 smithi070 ceph-mon[168282]: osdmap e148: 8 total, 8 up, 8 in 2023-09-23T20:55:02.253 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:55:02 smithi070 ceph-mon[168282]: failed to encode map e148 with expected crc 2023-09-23T20:55:02.253 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:55:02 smithi070 ceph-mon[168282]: failed to encode map e148 with expected crc 2023-09-23T20:55:02.253 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:55:02 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/643725192' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:55:02.253 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:55:02 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:55:02.253 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:55:02 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:55:02.290 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_historic_ops 2023-09-23T20:55:02.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:55:02 smithi026 ceph-mon[179688]: pgmap v363: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfilling, 2 active+recovery_wait+degraded+remapped, 91 active+clean, 3 active+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped; 3.3 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 722 op/s; 2421/160005 objects degraded (1.513%); 12089/160005 objects misplaced (7.555%); 18 MiB/s, 453 objects/s recovering 2023-09-23T20:55:02.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:55:02 smithi026 ceph-mon[179688]: failed to encode map e148 with expected crc 2023-09-23T20:55:02.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:55:02 smithi026 ceph-mon[179688]: osdmap e148: 8 total, 8 up, 8 in 2023-09-23T20:55:02.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:55:02 smithi026 ceph-mon[179688]: failed to encode map e148 with expected crc 2023-09-23T20:55:02.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:55:02 smithi026 ceph-mon[179688]: failed to encode map e148 with expected crc 2023-09-23T20:55:02.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:55:02 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/643725192' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:55:02.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:55:02 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:55:02.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:55:02 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:55:02.385 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:55:02 smithi026 ceph-mon[175151]: pgmap v363: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfilling, 2 active+recovery_wait+degraded+remapped, 91 active+clean, 3 active+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped; 3.3 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 722 op/s; 2421/160005 objects degraded (1.513%); 12089/160005 objects misplaced (7.555%); 18 MiB/s, 453 objects/s recovering 2023-09-23T20:55:02.385 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:55:02 smithi026 ceph-mon[175151]: failed to encode map e148 with expected crc 2023-09-23T20:55:02.385 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:55:02 smithi026 ceph-mon[175151]: osdmap e148: 8 total, 8 up, 8 in 2023-09-23T20:55:02.385 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:55:02 smithi026 ceph-mon[175151]: failed to encode map e148 with expected crc 2023-09-23T20:55:02.385 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:55:02 smithi026 ceph-mon[175151]: failed to encode map e148 with expected crc 2023-09-23T20:55:02.385 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:55:02 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/643725192' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:55:02.385 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:55:02 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:55:02.385 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:55:02 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:55:03.036 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:55:03.077 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:55:03.297 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:55:03 smithi070 ceph-mon[168282]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T20:55:03.297 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:55:03 smithi070 ceph-mon[168282]: failed to encode map e149 with expected crc 2023-09-23T20:55:03.297 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:55:03 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T20:55:03.297 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:55:03 smithi070 ceph-mon[168282]: osdmap e149: 8 total, 8 up, 8 in 2023-09-23T20:55:03.297 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:55:03 smithi070 ceph-mon[168282]: failed to encode map e149 with expected crc 2023-09-23T20:55:03.297 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:55:03 smithi070 ceph-mon[168282]: failed to encode map e149 with expected crc 2023-09-23T20:55:03.298 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:55:03 smithi070 ceph-mon[168282]: failed to encode map e149 with expected crc 2023-09-23T20:55:03.298 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:55:03 smithi070 ceph-mon[168282]: pgmap v366: 99 pgs: 1 active+remapped+backfilling, 2 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded+remapped, 91 active+clean, 3 active+remapped+backfill_wait; 3.1 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 782 op/s; 1603/153540 objects degraded (1.044%); 11458/153540 objects misplaced (7.463%); 27 MiB/s, 499 objects/s recovering 2023-09-23T20:55:03.349 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:55:03 smithi026 ceph-mon[179688]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T20:55:03.349 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:55:03 smithi026 ceph-mon[179688]: failed to encode map e149 with expected crc 2023-09-23T20:55:03.349 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:55:03 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T20:55:03.349 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:55:03 smithi026 ceph-mon[179688]: osdmap e149: 8 total, 8 up, 8 in 2023-09-23T20:55:03.350 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:55:03 smithi026 ceph-mon[179688]: failed to encode map e149 with expected crc 2023-09-23T20:55:03.350 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:55:03 smithi026 ceph-mon[179688]: failed to encode map e149 with expected crc 2023-09-23T20:55:03.350 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:55:03 smithi026 ceph-mon[179688]: failed to encode map e149 with expected crc 2023-09-23T20:55:03.350 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:55:03 smithi026 ceph-mon[179688]: pgmap v366: 99 pgs: 1 active+remapped+backfilling, 2 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded+remapped, 91 active+clean, 3 active+remapped+backfill_wait; 3.1 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 782 op/s; 1603/153540 objects degraded (1.044%); 11458/153540 objects misplaced (7.463%); 27 MiB/s, 499 objects/s recovering 2023-09-23T20:55:03.350 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:55:03 smithi026 ceph-mon[175151]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T20:55:03.350 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:55:03 smithi026 ceph-mon[175151]: failed to encode map e149 with expected crc 2023-09-23T20:55:03.350 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:55:03 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T20:55:03.350 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:55:03 smithi026 ceph-mon[175151]: osdmap e149: 8 total, 8 up, 8 in 2023-09-23T20:55:03.351 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:55:03 smithi026 ceph-mon[175151]: failed to encode map e149 with expected crc 2023-09-23T20:55:03.351 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:55:03 smithi026 ceph-mon[175151]: failed to encode map e149 with expected crc 2023-09-23T20:55:03.351 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:55:03 smithi026 ceph-mon[175151]: failed to encode map e149 with expected crc 2023-09-23T20:55:03.351 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:55:03 smithi026 ceph-mon[175151]: pgmap v366: 99 pgs: 1 active+remapped+backfilling, 2 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded+remapped, 91 active+clean, 3 active+remapped+backfill_wait; 3.1 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 782 op/s; 1603/153540 objects degraded (1.044%); 11458/153540 objects misplaced (7.463%); 27 MiB/s, 499 objects/s recovering 2023-09-23T20:55:04.318 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:55:04.324 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:55:04.324 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:55:04.425 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:55:04.483 INFO:teuthology.orchestra.run.smithi070.stdout:{ 2023-09-23T20:55:04.483 INFO:teuthology.orchestra.run.smithi070.stdout: "size": 20, 2023-09-23T20:55:04.483 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 600, 2023-09-23T20:55:04.483 INFO:teuthology.orchestra.run.smithi070.stdout: "ops": [ 2023-09-23T20:55:04.483 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:55:04.483 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.34797.0:5568 6.2 6:43edb851:::benchmark_data_smithi026_223868_object5567:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e106)", 2023-09-23T20:55:04.483 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T20:52:21.786587+0000", 2023-09-23T20:55:04.483 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 162.69096291, 2023-09-23T20:55:04.483 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.086354670999999994, 2023-09-23T20:55:04.484 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T20:55:04.484 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:55:04.484 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T20:55:04.484 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.34797", 2023-09-23T20:55:04.484 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T20:55:04.484 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 5568 2023-09-23T20:55:04.484 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:55:04.484 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T20:55:04.484 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:55:04.484 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T20:55:04.484 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:52:21.786587+0000", 2023-09-23T20:55:04.484 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T20:55:04.484 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:55:04.485 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:55:04.485 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T20:55:04.485 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:52:21.786587+0000", 2023-09-23T20:55:04.485 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T20:55:04.485 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:55:04.485 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:55:04.485 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T20:55:04.485 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:52:21.786585+0000", 2023-09-23T20:55:04.485 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.999999 2023-09-23T20:55:04.485 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:55:04.485 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:55:04.485 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T20:55:04.486 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:52:21.786626+0000", 2023-09-23T20:55:04.486 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.0077999999999998e-05 2023-09-23T20:55:04.486 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:55:04.486 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:55:04.486 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T20:55:04.486 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:52:21.786637+0000", 2023-09-23T20:55:04.486 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.1236000000000001e-05 2023-09-23T20:55:04.486 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:55:04.486 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:55:04.486 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T20:55:04.486 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:52:21.786641+0000", 2023-09-23T20:55:04.486 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.4530000000000004e-06 2023-09-23T20:55:04.487 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:55:04.487 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:55:04.487 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T20:55:04.487 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:52:21.786687+0000", 2023-09-23T20:55:04.487 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.6109000000000002e-05 2023-09-23T20:55:04.487 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:55:04.487 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:55:04.487 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T20:55:04.487 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:52:21.786795+0000", 2023-09-23T20:55:04.487 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00010764099999999999 2023-09-23T20:55:04.487 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:55:04.487 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:55:04.487 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 2,3", 2023-09-23T20:55:04.488 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:52:21.871252+0000", 2023-09-23T20:55:04.488 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.084457312000000007 2023-09-23T20:55:04.488 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:55:04.488 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:55:04.488 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T20:55:04.488 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:52:21.872733+0000", 2023-09-23T20:55:04.488 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.001480695 2023-09-23T20:55:04.488 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:55:04.488 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:55:04.488 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:55:04.488 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:52:21.872822+0000", 2023-09-23T20:55:04.488 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.8809999999999998e-05 2023-09-23T20:55:04.488 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:55:04.489 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:55:04.489 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:55:04.489 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:52:21.872857+0000", 2023-09-23T20:55:04.489 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.5611000000000002e-05 2023-09-23T20:55:04.489 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:55:04.489 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:55:04.489 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T20:55:04.489 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:52:21.872903+0000", 2023-09-23T20:55:04.489 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.5658000000000002e-05 2023-09-23T20:55:04.489 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:55:04.489 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:55:04.489 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T20:55:04.489 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:52:21.872941+0000", 2023-09-23T20:55:04.490 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.8114000000000001e-05 2023-09-23T20:55:04.490 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T20:55:04.490 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T20:55:04.490 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T20:55:04.490 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:55:04.490 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:55:04.490 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.34797.0:5569 6.2 6:4aaa764c:::benchmark_data_smithi026_223868_object5568:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e106)", 2023-09-23T20:55:04.490 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T20:52:21.786652+0000", 2023-09-23T20:55:04.490 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 162.69089744999999, 2023-09-23T20:55:04.490 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.087665349000000004, 2023-09-23T20:55:04.490 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T20:55:04.490 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:55:04.491 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T20:55:04.491 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.34797", 2023-09-23T20:55:04.491 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T20:55:04.491 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 5569 2023-09-23T20:55:04.491 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:55:04.491 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T20:55:04.491 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:55:04.491 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T20:55:04.491 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:52:21.786652+0000", 2023-09-23T20:55:04.491 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T20:55:04.491 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:55:04.491 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:55:04.491 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T20:55:04.492 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:52:21.786652+0000", 2023-09-23T20:55:04.492 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T20:55:04.492 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:55:04.492 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:55:04.492 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T20:55:04.492 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:52:21.786652+0000", 2023-09-23T20:55:04.492 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999995 2023-09-23T20:55:04.492 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:55:04.492 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:55:04.492 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T20:55:04.492 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:52:21.787270+0000", 2023-09-23T20:55:04.492 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00061800199999999998 2023-09-23T20:55:04.493 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:55:04.493 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:55:04.493 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T20:55:04.493 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:52:21.871139+0000", 2023-09-23T20:55:04.493 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.083869155000000001 2023-09-23T20:55:04.493 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:55:04.493 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:55:04.493 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T20:55:04.493 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:52:21.871150+0000", 2023-09-23T20:55:04.493 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.0723000000000001e-05 2023-09-23T20:55:04.493 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:55:04.493 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:55:04.493 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T20:55:04.493 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:52:21.871619+0000", 2023-09-23T20:55:04.494 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00046919699999999997 2023-09-23T20:55:04.494 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:55:04.494 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:55:04.494 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T20:55:04.494 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:52:21.871737+0000", 2023-09-23T20:55:04.494 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00011873499999999999 2023-09-23T20:55:04.494 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:55:04.494 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:55:04.494 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 2,3", 2023-09-23T20:55:04.494 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:52:21.871867+0000", 2023-09-23T20:55:04.494 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000129061 2023-09-23T20:55:04.495 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:55:04.495 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:55:04.495 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T20:55:04.495 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:52:21.872737+0000", 2023-09-23T20:55:04.495 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00087040900000000005 2023-09-23T20:55:04.495 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:55:04.495 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:55:04.495 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:55:04.495 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:52:21.872987+0000", 2023-09-23T20:55:04.495 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00024983299999999998 2023-09-23T20:55:04.495 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:55:04.495 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:55:04.495 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:55:04.496 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:52:21.874231+0000", 2023-09-23T20:55:04.496 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0012444800000000001 2023-09-23T20:55:04.496 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:55:04.496 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:55:04.496 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T20:55:04.496 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:52:21.874280+0000", 2023-09-23T20:55:04.496 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.8838999999999998e-05 2023-09-23T20:55:04.496 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:55:04.496 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:55:04.496 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T20:55:04.496 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:52:21.874317+0000", 2023-09-23T20:55:04.496 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.7255999999999997e-05 2023-09-23T20:55:04.496 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T20:55:04.497 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T20:55:04.497 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T20:55:04.497 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:55:04.497 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:55:04.497 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.34797.0:55054 6.e 6:70538c3f:::benchmark_data_smithi026_223868_object55053:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e113)", 2023-09-23T20:55:04.497 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T20:52:53.453038+0000", 2023-09-23T20:55:04.497 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 131.024511741, 2023-09-23T20:55:04.497 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.031692072000000002, 2023-09-23T20:55:04.497 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T20:55:04.497 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:55:04.497 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T20:55:04.497 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.34797", 2023-09-23T20:55:04.497 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T20:55:04.498 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 55054 2023-09-23T20:55:04.498 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:55:04.498 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T20:55:04.498 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:55:04.498 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T20:55:04.498 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:52:53.453038+0000", 2023-09-23T20:55:04.498 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T20:55:04.498 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:55:04.498 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:55:04.498 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T20:55:04.498 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:52:53.453038+0000", 2023-09-23T20:55:04.499 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T20:55:04.499 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:55:04.499 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:55:04.499 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T20:55:04.499 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:52:53.453037+0000", 2023-09-23T20:55:04.499 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.999999 2023-09-23T20:55:04.499 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:55:04.499 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:55:04.499 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T20:55:04.499 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:52:53.453100+0000", 2023-09-23T20:55:04.499 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.3552000000000004e-05 2023-09-23T20:55:04.499 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:55:04.499 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:55:04.500 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T20:55:04.500 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:52:53.453110+0000", 2023-09-23T20:55:04.500 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.0037999999999999e-05 2023-09-23T20:55:04.500 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:55:04.500 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:55:04.500 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T20:55:04.500 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:52:53.453114+0000", 2023-09-23T20:55:04.500 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.0840000000000002e-06 2023-09-23T20:55:04.500 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:55:04.500 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:55:04.500 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T20:55:04.500 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:52:53.453176+0000", 2023-09-23T20:55:04.500 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.1882000000000001e-05 2023-09-23T20:55:04.501 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:55:04.501 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:55:04.501 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T20:55:04.501 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:52:53.453365+0000", 2023-09-23T20:55:04.501 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000188488 2023-09-23T20:55:04.501 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:55:04.501 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:55:04.501 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 1,2", 2023-09-23T20:55:04.501 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:52:53.453471+0000", 2023-09-23T20:55:04.501 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00010610799999999999 2023-09-23T20:55:04.501 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:55:04.501 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:55:04.501 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T20:55:04.501 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:52:53.457890+0000", 2023-09-23T20:55:04.502 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.004419409 2023-09-23T20:55:04.502 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:55:04.502 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:55:04.502 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:55:04.502 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:52:53.461810+0000", 2023-09-23T20:55:04.502 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0039193450000000003 2023-09-23T20:55:04.502 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:55:04.502 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:55:04.502 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:55:04.502 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:52:53.484652+0000", 2023-09-23T20:55:04.502 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.022841932999999998 2023-09-23T20:55:04.502 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:55:04.503 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:55:04.503 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T20:55:04.503 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:52:53.484692+0000", 2023-09-23T20:55:04.503 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.0160999999999997e-05 2023-09-23T20:55:04.503 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:55:04.503 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:55:04.503 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T20:55:04.503 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:52:53.484730+0000", 2023-09-23T20:55:04.503 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.7975999999999997e-05 2023-09-23T20:55:04.503 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T20:55:04.503 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T20:55:04.503 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T20:55:04.503 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:55:04.504 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:55:04.504 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.34797.0:55062 6.e 6:7b197a78:::benchmark_data_smithi026_223868_object55061:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e113)", 2023-09-23T20:55:04.504 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T20:52:53.453257+0000", 2023-09-23T20:55:04.504 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 131.02429253299999, 2023-09-23T20:55:04.504 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.031600160000000002, 2023-09-23T20:55:04.504 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T20:55:04.504 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:55:04.504 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T20:55:04.504 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.34797", 2023-09-23T20:55:04.504 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T20:55:04.504 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 55062 2023-09-23T20:55:04.504 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:55:04.504 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T20:55:04.505 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:55:04.505 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T20:55:04.505 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:52:53.453257+0000", 2023-09-23T20:55:04.505 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T20:55:04.505 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:55:04.505 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:55:04.505 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T20:55:04.505 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:52:53.453257+0000", 2023-09-23T20:55:04.505 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T20:55:04.505 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:55:04.505 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:55:04.505 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T20:55:04.505 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:52:53.453256+0000", 2023-09-23T20:55:04.505 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999995 2023-09-23T20:55:04.506 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:55:04.506 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:55:04.506 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T20:55:04.506 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:52:53.453289+0000", 2023-09-23T20:55:04.506 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.2033000000000003e-05 2023-09-23T20:55:04.506 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:55:04.506 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:55:04.506 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T20:55:04.506 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:52:53.453295+0000", 2023-09-23T20:55:04.506 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.6900000000000003e-06 2023-09-23T20:55:04.506 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:55:04.506 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:55:04.507 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T20:55:04.507 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:52:53.453300+0000", 2023-09-23T20:55:04.507 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.5979999999999999e-06 2023-09-23T20:55:04.507 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:55:04.507 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:55:04.507 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T20:55:04.507 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:52:53.454826+0000", 2023-09-23T20:55:04.507 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0015264079999999999 2023-09-23T20:55:04.507 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:55:04.507 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:55:04.507 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T20:55:04.507 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:52:53.454930+0000", 2023-09-23T20:55:04.507 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000103725 2023-09-23T20:55:04.507 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:55:04.508 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:55:04.508 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 1,2", 2023-09-23T20:55:04.508 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:52:53.455012+0000", 2023-09-23T20:55:04.508 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.2065999999999997e-05 2023-09-23T20:55:04.508 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:55:04.508 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:55:04.508 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T20:55:04.508 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:52:53.457908+0000", 2023-09-23T20:55:04.508 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.002895519 2023-09-23T20:55:04.508 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:55:04.508 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:55:04.508 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:55:04.508 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:52:53.461849+0000", 2023-09-23T20:55:04.509 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0039413820000000002 2023-09-23T20:55:04.509 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:55:04.509 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:55:04.509 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:55:04.509 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:52:53.484771+0000", 2023-09-23T20:55:04.509 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.022922188 2023-09-23T20:55:04.509 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:55:04.509 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:55:04.509 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T20:55:04.509 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:52:53.484820+0000", 2023-09-23T20:55:04.509 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.8498999999999998e-05 2023-09-23T20:55:04.509 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:55:04.509 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:55:04.509 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T20:55:04.510 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:52:53.484857+0000", 2023-09-23T20:55:04.510 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.7500999999999999e-05 2023-09-23T20:55:04.510 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T20:55:04.510 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T20:55:04.510 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T20:55:04.510 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:55:04.510 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:55:04.510 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.34797.0:55064 6.e 6:78c5ee49:::benchmark_data_smithi026_223868_object55063:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e113)", 2023-09-23T20:55:04.510 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T20:52:53.453307+0000", 2023-09-23T20:55:04.510 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 131.02424293499999, 2023-09-23T20:55:04.510 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.033872233000000002, 2023-09-23T20:55:04.510 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T20:55:04.511 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:55:04.511 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T20:55:04.511 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.34797", 2023-09-23T20:55:04.511 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T20:55:04.511 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 55064 2023-09-23T20:55:04.511 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:55:04.511 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T20:55:04.511 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:55:04.511 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T20:55:04.511 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:52:53.453307+0000", 2023-09-23T20:55:04.511 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T20:55:04.511 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:55:04.511 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:55:04.512 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T20:55:04.512 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:52:53.453307+0000", 2023-09-23T20:55:04.512 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T20:55:04.512 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:55:04.512 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:55:04.512 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T20:55:04.512 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:52:53.453306+0000", 2023-09-23T20:55:04.512 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999995 2023-09-23T20:55:04.512 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:55:04.512 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:55:04.512 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T20:55:04.512 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:52:53.453384+0000", 2023-09-23T20:55:04.512 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.7793000000000005e-05 2023-09-23T20:55:04.513 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:55:04.513 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:55:04.513 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T20:55:04.513 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:52:53.453391+0000", 2023-09-23T20:55:04.513 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.3359999999999997e-06 2023-09-23T20:55:04.513 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:55:04.513 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:55:04.513 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T20:55:04.513 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:52:53.453396+0000", 2023-09-23T20:55:04.513 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.724e-06 2023-09-23T20:55:04.513 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:55:04.514 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:55:04.514 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T20:55:04.514 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:52:53.456170+0000", 2023-09-23T20:55:04.514 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0027739959999999999 2023-09-23T20:55:04.514 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:55:04.514 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:55:04.514 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T20:55:04.514 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:52:53.456311+0000", 2023-09-23T20:55:04.514 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00014076599999999999 2023-09-23T20:55:04.514 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:55:04.514 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:55:04.514 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 1,2", 2023-09-23T20:55:04.515 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:52:53.456458+0000", 2023-09-23T20:55:04.515 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000147076 2023-09-23T20:55:04.515 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:55:04.515 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:55:04.515 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T20:55:04.515 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:52:53.458002+0000", 2023-09-23T20:55:04.515 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.001544068 2023-09-23T20:55:04.515 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:55:04.515 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:55:04.515 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:55:04.515 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:52:53.462297+0000", 2023-09-23T20:55:04.515 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0042949310000000001 2023-09-23T20:55:04.515 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:55:04.515 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:55:04.516 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:55:04.516 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:52:53.487092+0000", 2023-09-23T20:55:04.516 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.024795620000000001 2023-09-23T20:55:04.516 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:55:04.516 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:55:04.516 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T20:55:04.516 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:52:53.487142+0000", 2023-09-23T20:55:04.516 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.9737999999999997e-05 2023-09-23T20:55:04.516 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:55:04.516 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:55:04.516 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T20:55:04.516 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:52:53.487179+0000", 2023-09-23T20:55:04.516 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.6653000000000003e-05 2023-09-23T20:55:04.517 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T20:55:04.517 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T20:55:04.517 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T20:55:04.517 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:55:04.517 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:55:04.517 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.34797.0:72728 6.e 6:7cb510f1:::benchmark_data_smithi026_223868_object72727:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e116)", 2023-09-23T20:55:04.517 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T20:53:09.759530+0000", 2023-09-23T20:55:04.517 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 114.718019082, 2023-09-23T20:55:04.517 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.037606031999999998, 2023-09-23T20:55:04.517 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T20:55:04.517 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:55:04.517 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T20:55:04.517 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.34797", 2023-09-23T20:55:04.517 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T20:55:04.518 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 72728 2023-09-23T20:55:04.518 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:55:04.518 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T20:55:04.518 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:55:04.518 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T20:55:04.518 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:09.759530+0000", 2023-09-23T20:55:04.518 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T20:55:04.518 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:55:04.518 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:55:04.518 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T20:55:04.518 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:09.759530+0000", 2023-09-23T20:55:04.518 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T20:55:04.519 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:55:04.519 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:55:04.519 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T20:55:04.519 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:09.759529+0000", 2023-09-23T20:55:04.519 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.999999 2023-09-23T20:55:04.519 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:55:04.519 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:55:04.519 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T20:55:04.519 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:09.759651+0000", 2023-09-23T20:55:04.519 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000121649 2023-09-23T20:55:04.519 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:55:04.519 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:55:04.519 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T20:55:04.519 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:09.759666+0000", 2023-09-23T20:55:04.520 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.5478e-05 2023-09-23T20:55:04.520 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:55:04.520 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:55:04.520 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T20:55:04.520 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:09.759673+0000", 2023-09-23T20:55:04.520 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.0090000000000001e-06 2023-09-23T20:55:04.520 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:55:04.520 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:55:04.520 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T20:55:04.520 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:09.759715+0000", 2023-09-23T20:55:04.520 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.1461999999999997e-05 2023-09-23T20:55:04.520 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:55:04.520 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:55:04.521 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T20:55:04.521 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:09.759875+0000", 2023-09-23T20:55:04.521 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00015992600000000001 2023-09-23T20:55:04.521 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:55:04.521 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:55:04.521 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 1,2", 2023-09-23T20:55:04.521 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:09.760001+0000", 2023-09-23T20:55:04.521 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000126138 2023-09-23T20:55:04.521 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:55:04.521 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:55:04.521 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T20:55:04.521 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:09.762342+0000", 2023-09-23T20:55:04.521 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0023405460000000002 2023-09-23T20:55:04.522 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:55:04.522 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:55:04.522 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:55:04.522 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:09.766772+0000", 2023-09-23T20:55:04.522 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0044303600000000004 2023-09-23T20:55:04.522 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:55:04.522 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:55:04.522 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:55:04.522 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:09.797043+0000", 2023-09-23T20:55:04.522 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.030270882999999998 2023-09-23T20:55:04.522 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:55:04.522 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:55:04.523 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T20:55:04.523 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:09.797097+0000", 2023-09-23T20:55:04.523 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.3746999999999999e-05 2023-09-23T20:55:04.523 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:55:04.523 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:55:04.523 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T20:55:04.523 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:09.797136+0000", 2023-09-23T20:55:04.523 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.9864999999999999e-05 2023-09-23T20:55:04.523 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T20:55:04.523 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T20:55:04.523 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T20:55:04.523 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:55:04.523 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:55:04.524 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.34797.0:72738 6.e 6:7c0b4fde:::benchmark_data_smithi026_223868_object72737:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e116)", 2023-09-23T20:55:04.524 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T20:53:09.767045+0000", 2023-09-23T20:55:04.524 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 114.710504802, 2023-09-23T20:55:04.524 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.035259154000000001, 2023-09-23T20:55:04.524 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T20:55:04.524 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:55:04.524 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T20:55:04.524 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.34797", 2023-09-23T20:55:04.524 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T20:55:04.524 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 72738 2023-09-23T20:55:04.524 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:55:04.524 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T20:55:04.525 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:55:04.525 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T20:55:04.525 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:09.767045+0000", 2023-09-23T20:55:04.525 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T20:55:04.525 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:55:04.525 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:55:04.525 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T20:55:04.525 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:09.767045+0000", 2023-09-23T20:55:04.525 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T20:55:04.525 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:55:04.525 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:55:04.525 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T20:55:04.525 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:09.767043+0000", 2023-09-23T20:55:04.526 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T20:55:04.526 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:55:04.526 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:55:04.526 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T20:55:04.526 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:09.767097+0000", 2023-09-23T20:55:04.526 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.3683000000000001e-05 2023-09-23T20:55:04.526 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:55:04.526 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:55:04.526 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T20:55:04.526 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:09.767109+0000", 2023-09-23T20:55:04.526 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.1878999999999999e-05 2023-09-23T20:55:04.527 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:55:04.527 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:55:04.527 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T20:55:04.527 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:09.767115+0000", 2023-09-23T20:55:04.527 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.1739999999999997e-06 2023-09-23T20:55:04.527 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:55:04.527 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:55:04.527 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T20:55:04.527 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:09.767155+0000", 2023-09-23T20:55:04.527 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.9614999999999999e-05 2023-09-23T20:55:04.527 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:55:04.527 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:55:04.528 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T20:55:04.528 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:09.767273+0000", 2023-09-23T20:55:04.528 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000118756 2023-09-23T20:55:04.528 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:55:04.528 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:55:04.528 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 1,2", 2023-09-23T20:55:04.528 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:09.767361+0000", 2023-09-23T20:55:04.528 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.7095999999999994e-05 2023-09-23T20:55:04.528 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:55:04.528 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:55:04.528 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T20:55:04.528 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:09.769272+0000", 2023-09-23T20:55:04.528 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.001911577 2023-09-23T20:55:04.529 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:55:04.529 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:55:04.529 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:55:04.529 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:09.772552+0000", 2023-09-23T20:55:04.529 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.003279456 2023-09-23T20:55:04.529 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:55:04.529 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:55:04.529 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:55:04.529 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:09.802217+0000", 2023-09-23T20:55:04.529 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.029665571000000002 2023-09-23T20:55:04.529 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:55:04.529 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:55:04.529 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T20:55:04.530 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:09.802269+0000", 2023-09-23T20:55:04.530 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.2151000000000003e-05 2023-09-23T20:55:04.530 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:55:04.530 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:55:04.530 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T20:55:04.530 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:09.802304+0000", 2023-09-23T20:55:04.530 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.4486999999999997e-05 2023-09-23T20:55:04.530 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T20:55:04.530 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T20:55:04.530 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T20:55:04.530 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:55:04.531 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:55:04.531 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.34797.0:72744 6.2 6:45dcf4ef:::benchmark_data_smithi026_223868_object72743:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e116)", 2023-09-23T20:55:04.531 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T20:53:09.770144+0000", 2023-09-23T20:55:04.531 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 114.70740496800001, 2023-09-23T20:55:04.531 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.031705863000000001, 2023-09-23T20:55:04.531 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T20:55:04.531 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:55:04.531 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T20:55:04.531 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.34797", 2023-09-23T20:55:04.531 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T20:55:04.531 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 72744 2023-09-23T20:55:04.531 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:55:04.531 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T20:55:04.532 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:55:04.532 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T20:55:04.532 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:09.770144+0000", 2023-09-23T20:55:04.532 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T20:55:04.532 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:55:04.532 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:55:04.532 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T20:55:04.532 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:09.770144+0000", 2023-09-23T20:55:04.532 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T20:55:04.532 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:55:04.532 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:55:04.532 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T20:55:04.533 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:09.770143+0000", 2023-09-23T20:55:04.533 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T20:55:04.533 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:55:04.533 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:55:04.533 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T20:55:04.533 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:09.770196+0000", 2023-09-23T20:55:04.533 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.2324999999999998e-05 2023-09-23T20:55:04.533 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:55:04.533 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:55:04.533 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T20:55:04.533 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:09.770211+0000", 2023-09-23T20:55:04.533 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.5143000000000001e-05 2023-09-23T20:55:04.534 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:55:04.534 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:55:04.534 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T20:55:04.534 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:09.770217+0000", 2023-09-23T20:55:04.534 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.0039999999999996e-06 2023-09-23T20:55:04.534 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:55:04.534 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:55:04.534 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T20:55:04.534 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:09.770272+0000", 2023-09-23T20:55:04.534 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.4743000000000001e-05 2023-09-23T20:55:04.534 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:55:04.534 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:55:04.535 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T20:55:04.535 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:09.770392+0000", 2023-09-23T20:55:04.535 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000120732 2023-09-23T20:55:04.535 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:55:04.535 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:55:04.535 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 2,3", 2023-09-23T20:55:04.535 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:09.770535+0000", 2023-09-23T20:55:04.535 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00014287600000000001 2023-09-23T20:55:04.535 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:55:04.535 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:55:04.535 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T20:55:04.535 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:09.772561+0000", 2023-09-23T20:55:04.536 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.002025855 2023-09-23T20:55:04.536 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:55:04.536 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:55:04.536 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:55:04.536 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:09.774736+0000", 2023-09-23T20:55:04.536 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0021751359999999998 2023-09-23T20:55:04.536 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:55:04.536 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:55:04.536 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:55:04.536 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:09.801770+0000", 2023-09-23T20:55:04.536 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.027033809999999998 2023-09-23T20:55:04.536 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:55:04.537 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:55:04.537 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T20:55:04.537 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:09.801815+0000", 2023-09-23T20:55:04.537 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.5246e-05 2023-09-23T20:55:04.537 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:55:04.537 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:55:04.537 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T20:55:04.537 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:09.801850+0000", 2023-09-23T20:55:04.537 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.5191999999999998e-05 2023-09-23T20:55:04.537 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T20:55:04.537 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T20:55:04.537 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T20:55:04.538 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:55:04.538 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:55:04.538 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.34797.0:72961 6.e 6:7a194f07:::benchmark_data_smithi026_223868_object72960:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e116)", 2023-09-23T20:55:04.538 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T20:53:09.984664+0000", 2023-09-23T20:55:04.538 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 114.49288564299999, 2023-09-23T20:55:04.538 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.034323724999999999, 2023-09-23T20:55:04.538 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T20:55:04.538 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:55:04.538 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T20:55:04.538 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.34797", 2023-09-23T20:55:04.538 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T20:55:04.538 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 72961 2023-09-23T20:55:04.539 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:55:04.539 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T20:55:04.539 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:55:04.539 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T20:55:04.539 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:09.984664+0000", 2023-09-23T20:55:04.539 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T20:55:04.539 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:55:04.539 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:55:04.539 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T20:55:04.539 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:09.984664+0000", 2023-09-23T20:55:04.539 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T20:55:04.539 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:55:04.539 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:55:04.540 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T20:55:04.540 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:09.984663+0000", 2023-09-23T20:55:04.540 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.999999 2023-09-23T20:55:04.540 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:55:04.540 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:55:04.540 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T20:55:04.540 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:09.984715+0000", 2023-09-23T20:55:04.540 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.2317000000000001e-05 2023-09-23T20:55:04.540 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:55:04.540 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:55:04.540 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T20:55:04.540 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:09.984727+0000", 2023-09-23T20:55:04.541 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.1674e-05 2023-09-23T20:55:04.541 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:55:04.541 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:55:04.541 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T20:55:04.541 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:09.984733+0000", 2023-09-23T20:55:04.541 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.1500000000000004e-06 2023-09-23T20:55:04.541 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:55:04.541 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:55:04.541 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T20:55:04.541 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:09.984765+0000", 2023-09-23T20:55:04.541 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.2384e-05 2023-09-23T20:55:04.541 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:55:04.541 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:55:04.542 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T20:55:04.542 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:09.984880+0000", 2023-09-23T20:55:04.542 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000114193 2023-09-23T20:55:04.542 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:55:04.542 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:55:04.542 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 1,2", 2023-09-23T20:55:04.542 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:09.985015+0000", 2023-09-23T20:55:04.542 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00013562900000000001 2023-09-23T20:55:04.542 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:55:04.542 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:55:04.542 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T20:55:04.543 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:09.987073+0000", 2023-09-23T20:55:04.543 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0020582389999999999 2023-09-23T20:55:04.543 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:55:04.543 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:55:04.543 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:55:04.543 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:10.014307+0000", 2023-09-23T20:55:04.543 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.027233482999999999 2023-09-23T20:55:04.543 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:55:04.543 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:55:04.543 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:55:04.543 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:10.018934+0000", 2023-09-23T20:55:04.543 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0046273460000000001 2023-09-23T20:55:04.544 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:55:04.544 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:55:04.544 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T20:55:04.544 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:10.018965+0000", 2023-09-23T20:55:04.544 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.0771000000000001e-05 2023-09-23T20:55:04.544 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:55:04.544 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:55:04.544 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T20:55:04.544 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:10.018988+0000", 2023-09-23T20:55:04.544 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.2558e-05 2023-09-23T20:55:04.544 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T20:55:04.544 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T20:55:04.545 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T20:55:04.545 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:55:04.545 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:55:04.545 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.34797.0:72968 6.e 6:707ac5ff:::benchmark_data_smithi026_223868_object72967:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e116)", 2023-09-23T20:55:04.545 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T20:53:09.987554+0000", 2023-09-23T20:55:04.545 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 114.489995689, 2023-09-23T20:55:04.545 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.035372645000000001, 2023-09-23T20:55:04.545 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T20:55:04.545 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:55:04.545 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T20:55:04.545 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.34797", 2023-09-23T20:55:04.545 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T20:55:04.546 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 72968 2023-09-23T20:55:04.546 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:55:04.546 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T20:55:04.546 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:55:04.546 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T20:55:04.546 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:09.987554+0000", 2023-09-23T20:55:04.546 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T20:55:04.546 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:55:04.546 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:55:04.546 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T20:55:04.546 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:09.987554+0000", 2023-09-23T20:55:04.546 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T20:55:04.547 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:55:04.547 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:55:04.547 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T20:55:04.547 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:09.987552+0000", 2023-09-23T20:55:04.547 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T20:55:04.547 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:55:04.547 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:55:04.547 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T20:55:04.547 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:09.987598+0000", 2023-09-23T20:55:04.547 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.5847999999999999e-05 2023-09-23T20:55:04.547 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:55:04.547 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:55:04.547 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T20:55:04.548 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:09.987613+0000", 2023-09-23T20:55:04.548 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.435e-05 2023-09-23T20:55:04.548 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:55:04.548 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:55:04.548 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T20:55:04.548 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:09.987619+0000", 2023-09-23T20:55:04.548 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.6699999999999997e-06 2023-09-23T20:55:04.548 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:55:04.548 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:55:04.548 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T20:55:04.548 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:09.987740+0000", 2023-09-23T20:55:04.548 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000120309 2023-09-23T20:55:04.549 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:55:04.549 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:55:04.549 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T20:55:04.549 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:09.987853+0000", 2023-09-23T20:55:04.549 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00011377700000000001 2023-09-23T20:55:04.549 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:55:04.549 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:55:04.549 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 1,2", 2023-09-23T20:55:04.549 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:09.987938+0000", 2023-09-23T20:55:04.549 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.4060000000000005e-05 2023-09-23T20:55:04.549 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:55:04.549 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:55:04.549 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T20:55:04.550 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:09.992139+0000", 2023-09-23T20:55:04.550 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0042017319999999997 2023-09-23T20:55:04.550 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:55:04.550 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:55:04.550 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:55:04.550 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:10.019016+0000", 2023-09-23T20:55:04.550 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.026876272 2023-09-23T20:55:04.550 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:55:04.550 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:55:04.550 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:55:04.550 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:10.022854+0000", 2023-09-23T20:55:04.550 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.003838111 2023-09-23T20:55:04.551 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:55:04.551 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:55:04.551 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T20:55:04.551 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:10.022894+0000", 2023-09-23T20:55:04.551 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.9898999999999999e-05 2023-09-23T20:55:04.551 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:55:04.551 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:55:04.551 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T20:55:04.551 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:10.022926+0000", 2023-09-23T20:55:04.551 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.2894e-05 2023-09-23T20:55:04.551 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T20:55:04.551 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T20:55:04.552 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T20:55:04.552 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:55:04.552 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:55:04.552 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.34797.0:72973 6.e 6:75909271:::benchmark_data_smithi026_223868_object72972:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e116)", 2023-09-23T20:55:04.552 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T20:53:09.988008+0000", 2023-09-23T20:55:04.552 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 114.48954146299999, 2023-09-23T20:55:04.552 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.035000083000000001, 2023-09-23T20:55:04.552 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T20:55:04.552 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:55:04.552 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T20:55:04.552 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.34797", 2023-09-23T20:55:04.552 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T20:55:04.552 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 72973 2023-09-23T20:55:04.553 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:55:04.553 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T20:55:04.553 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:55:04.553 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T20:55:04.553 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:09.988008+0000", 2023-09-23T20:55:04.553 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T20:55:04.553 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:55:04.553 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:55:04.553 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T20:55:04.553 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:09.988008+0000", 2023-09-23T20:55:04.553 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T20:55:04.553 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:55:04.553 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:55:04.554 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T20:55:04.554 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:09.988007+0000", 2023-09-23T20:55:04.554 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999995 2023-09-23T20:55:04.554 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:55:04.554 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:55:04.554 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T20:55:04.554 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:09.988054+0000", 2023-09-23T20:55:04.554 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.6285999999999997e-05 2023-09-23T20:55:04.554 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:55:04.554 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:55:04.554 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T20:55:04.554 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:09.988063+0000", 2023-09-23T20:55:04.555 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.5729999999999997e-06 2023-09-23T20:55:04.555 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:55:04.555 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:55:04.555 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T20:55:04.555 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:09.988069+0000", 2023-09-23T20:55:04.555 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.8850000000000004e-06 2023-09-23T20:55:04.555 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:55:04.555 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:55:04.555 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T20:55:04.555 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:09.990101+0000", 2023-09-23T20:55:04.555 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0020317529999999999 2023-09-23T20:55:04.555 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:55:04.556 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:55:04.556 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T20:55:04.556 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:09.990191+0000", 2023-09-23T20:55:04.556 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.0446999999999996e-05 2023-09-23T20:55:04.556 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:55:04.556 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:55:04.556 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 1,2", 2023-09-23T20:55:04.556 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:09.990303+0000", 2023-09-23T20:55:04.556 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00011171999999999999 2023-09-23T20:55:04.556 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:55:04.556 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:55:04.556 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T20:55:04.557 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:09.992301+0000", 2023-09-23T20:55:04.557 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00199797 2023-09-23T20:55:04.557 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:55:04.557 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:55:04.557 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:55:04.557 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:10.022618+0000", 2023-09-23T20:55:04.557 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.030316694000000002 2023-09-23T20:55:04.557 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:55:04.557 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:55:04.557 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:55:04.557 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:10.022947+0000", 2023-09-23T20:55:04.557 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00032924400000000002 2023-09-23T20:55:04.557 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:55:04.558 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:55:04.558 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T20:55:04.558 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:10.022978+0000", 2023-09-23T20:55:04.558 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.1210000000000001e-05 2023-09-23T20:55:04.558 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:55:04.558 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:55:04.558 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T20:55:04.558 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:10.023008+0000", 2023-09-23T20:55:04.558 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.9986999999999999e-05 2023-09-23T20:55:04.558 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T20:55:04.558 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T20:55:04.559 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T20:55:04.559 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:55:04.559 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:55:04.559 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.34797.0:72996 6.2 6:475aa694:::benchmark_data_smithi026_223868_object72995:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e116)", 2023-09-23T20:55:04.559 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T20:53:10.032222+0000", 2023-09-23T20:55:04.559 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 114.44532752000001, 2023-09-23T20:55:04.559 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.062433426, 2023-09-23T20:55:04.559 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T20:55:04.559 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:55:04.559 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T20:55:04.559 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.34797", 2023-09-23T20:55:04.559 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T20:55:04.559 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 72996 2023-09-23T20:55:04.560 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:55:04.560 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T20:55:04.560 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:55:04.560 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T20:55:04.560 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:10.032222+0000", 2023-09-23T20:55:04.560 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T20:55:04.560 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:55:04.560 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:55:04.560 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T20:55:04.560 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:10.032222+0000", 2023-09-23T20:55:04.560 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T20:55:04.560 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:55:04.560 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:55:04.560 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T20:55:04.561 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:10.032221+0000", 2023-09-23T20:55:04.561 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T20:55:04.561 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:55:04.561 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:55:04.561 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T20:55:04.561 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:10.032276+0000", 2023-09-23T20:55:04.561 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.5797000000000001e-05 2023-09-23T20:55:04.561 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:55:04.561 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:55:04.561 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T20:55:04.561 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:10.032299+0000", 2023-09-23T20:55:04.561 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.2894000000000001e-05 2023-09-23T20:55:04.562 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:55:04.562 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:55:04.562 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T20:55:04.562 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:10.032306+0000", 2023-09-23T20:55:04.562 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.4160000000000001e-06 2023-09-23T20:55:04.562 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:55:04.562 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:55:04.562 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T20:55:04.562 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:10.032362+0000", 2023-09-23T20:55:04.562 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.5823999999999999e-05 2023-09-23T20:55:04.562 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:55:04.562 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:55:04.563 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T20:55:04.563 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:10.032491+0000", 2023-09-23T20:55:04.563 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00012955599999999999 2023-09-23T20:55:04.563 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:55:04.563 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:55:04.563 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 2,3", 2023-09-23T20:55:04.563 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:10.032628+0000", 2023-09-23T20:55:04.563 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00013680699999999999 2023-09-23T20:55:04.563 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:55:04.563 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:55:04.563 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T20:55:04.563 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:10.034822+0000", 2023-09-23T20:55:04.563 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0021942860000000002 2023-09-23T20:55:04.564 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:55:04.564 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:55:04.564 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:55:04.564 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:10.038329+0000", 2023-09-23T20:55:04.564 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0035064929999999998 2023-09-23T20:55:04.564 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:55:04.564 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:55:04.564 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:55:04.564 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:10.094544+0000", 2023-09-23T20:55:04.564 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.056215767999999999 2023-09-23T20:55:04.564 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:55:04.564 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:55:04.564 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T20:55:04.565 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:10.094606+0000", 2023-09-23T20:55:04.565 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.1224999999999997e-05 2023-09-23T20:55:04.565 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:55:04.565 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:55:04.565 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T20:55:04.565 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:10.094655+0000", 2023-09-23T20:55:04.565 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.9653e-05 2023-09-23T20:55:04.565 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T20:55:04.565 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T20:55:04.565 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T20:55:04.565 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:55:04.565 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:55:04.565 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.34797.0:74402 6.2 6:4781b876:::benchmark_data_smithi026_223868_object74401:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e116)", 2023-09-23T20:55:04.566 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T20:53:11.422811+0000", 2023-09-23T20:55:04.566 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 113.05473803700001, 2023-09-23T20:55:04.566 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.044602429999999998, 2023-09-23T20:55:04.566 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T20:55:04.566 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:55:04.566 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T20:55:04.566 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.34797", 2023-09-23T20:55:04.566 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T20:55:04.566 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 74402 2023-09-23T20:55:04.566 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:55:04.566 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T20:55:04.567 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:55:04.567 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T20:55:04.567 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:11.422811+0000", 2023-09-23T20:55:04.567 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T20:55:04.567 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:55:04.567 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:55:04.567 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T20:55:04.567 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:11.422811+0000", 2023-09-23T20:55:04.567 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T20:55:04.567 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:55:04.567 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:55:04.567 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T20:55:04.567 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:11.422811+0000", 2023-09-23T20:55:04.568 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.999999 2023-09-23T20:55:04.568 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:55:04.568 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:55:04.568 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T20:55:04.568 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:11.422840+0000", 2023-09-23T20:55:04.568 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.9139999999999999e-05 2023-09-23T20:55:04.568 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:55:04.568 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:55:04.568 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T20:55:04.568 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:11.422852+0000", 2023-09-23T20:55:04.568 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.2506e-05 2023-09-23T20:55:04.568 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:55:04.568 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:55:04.569 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T20:55:04.569 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:11.422858+0000", 2023-09-23T20:55:04.569 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.1890000000000004e-06 2023-09-23T20:55:04.569 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:55:04.569 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:55:04.569 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T20:55:04.569 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:11.422905+0000", 2023-09-23T20:55:04.569 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.6990000000000002e-05 2023-09-23T20:55:04.569 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:55:04.569 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:55:04.569 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T20:55:04.569 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:11.423041+0000", 2023-09-23T20:55:04.569 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00013542799999999999 2023-09-23T20:55:04.570 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:55:04.570 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:55:04.570 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 2,3", 2023-09-23T20:55:04.570 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:11.423197+0000", 2023-09-23T20:55:04.570 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00015623899999999999 2023-09-23T20:55:04.570 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:55:04.570 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:55:04.570 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T20:55:04.570 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:11.425255+0000", 2023-09-23T20:55:04.570 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0020580379999999999 2023-09-23T20:55:04.570 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:55:04.570 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:55:04.571 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:55:04.571 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:11.442630+0000", 2023-09-23T20:55:04.571 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.01737474 2023-09-23T20:55:04.571 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:55:04.571 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:55:04.571 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:55:04.571 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:11.467350+0000", 2023-09-23T20:55:04.571 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.024719652000000002 2023-09-23T20:55:04.571 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:55:04.571 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:55:04.571 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T20:55:04.571 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:11.467384+0000", 2023-09-23T20:55:04.571 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.4526000000000001e-05 2023-09-23T20:55:04.572 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:55:04.572 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:55:04.572 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T20:55:04.572 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:11.467414+0000", 2023-09-23T20:55:04.572 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.9810000000000001e-05 2023-09-23T20:55:04.572 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T20:55:04.572 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T20:55:04.572 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T20:55:04.572 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:55:04.572 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:55:04.572 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.34797.0:74412 6.2 6:4a2fa80e:::benchmark_data_smithi026_223868_object74411:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e116)", 2023-09-23T20:55:04.572 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T20:53:11.436354+0000", 2023-09-23T20:55:04.573 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 113.04119554899999, 2023-09-23T20:55:04.573 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.031153067, 2023-09-23T20:55:04.573 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T20:55:04.573 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:55:04.573 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T20:55:04.573 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.34797", 2023-09-23T20:55:04.573 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T20:55:04.573 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 74412 2023-09-23T20:55:04.573 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:55:04.573 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T20:55:04.573 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:55:04.573 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T20:55:04.574 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:11.436354+0000", 2023-09-23T20:55:04.574 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T20:55:04.574 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:55:04.574 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:55:04.574 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T20:55:04.574 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:11.436354+0000", 2023-09-23T20:55:04.574 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T20:55:04.574 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:55:04.574 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:55:04.574 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T20:55:04.574 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:11.436353+0000", 2023-09-23T20:55:04.575 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999995 2023-09-23T20:55:04.575 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:55:04.575 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:55:04.575 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T20:55:04.575 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:11.436391+0000", 2023-09-23T20:55:04.575 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.8216000000000001e-05 2023-09-23T20:55:04.575 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:55:04.575 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:55:04.575 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T20:55:04.575 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:11.436400+0000", 2023-09-23T20:55:04.575 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.3410000000000003e-06 2023-09-23T20:55:04.575 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:55:04.576 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:55:04.576 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T20:55:04.576 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:11.436405+0000", 2023-09-23T20:55:04.576 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.4190000000000001e-06 2023-09-23T20:55:04.576 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:55:04.576 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:55:04.576 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T20:55:04.576 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:11.436441+0000", 2023-09-23T20:55:04.576 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.5565999999999998e-05 2023-09-23T20:55:04.576 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:55:04.576 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:55:04.576 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T20:55:04.576 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:11.436523+0000", 2023-09-23T20:55:04.577 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.2541999999999997e-05 2023-09-23T20:55:04.577 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:55:04.577 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:55:04.577 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 2,3", 2023-09-23T20:55:04.577 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:11.436610+0000", 2023-09-23T20:55:04.577 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.6447999999999996e-05 2023-09-23T20:55:04.577 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:55:04.577 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:55:04.577 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T20:55:04.577 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:11.438490+0000", 2023-09-23T20:55:04.577 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0018803540000000001 2023-09-23T20:55:04.577 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:55:04.577 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:55:04.578 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:55:04.578 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:11.448972+0000", 2023-09-23T20:55:04.578 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.010481564000000001 2023-09-23T20:55:04.578 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:55:04.578 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:55:04.578 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:55:04.578 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:11.467447+0000", 2023-09-23T20:55:04.578 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.018475140000000001 2023-09-23T20:55:04.578 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:55:04.578 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:55:04.578 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T20:55:04.579 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:11.467483+0000", 2023-09-23T20:55:04.579 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.6316999999999998e-05 2023-09-23T20:55:04.579 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:55:04.579 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:55:04.579 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T20:55:04.579 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:11.467507+0000", 2023-09-23T20:55:04.579 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.3785999999999999e-05 2023-09-23T20:55:04.579 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T20:55:04.579 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T20:55:04.579 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T20:55:04.579 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:55:04.579 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:55:04.579 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.34797.0:74742 6.e 6:7a23665c:::benchmark_data_smithi026_223868_object74741:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e116)", 2023-09-23T20:55:04.580 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T20:53:11.817326+0000", 2023-09-23T20:55:04.580 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 112.660223413, 2023-09-23T20:55:04.580 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.035540334, 2023-09-23T20:55:04.580 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T20:55:04.580 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:55:04.580 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T20:55:04.580 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.34797", 2023-09-23T20:55:04.580 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T20:55:04.580 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 74742 2023-09-23T20:55:04.580 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:55:04.580 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T20:55:04.580 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:55:04.581 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T20:55:04.581 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:11.817326+0000", 2023-09-23T20:55:04.581 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T20:55:04.581 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:55:04.581 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:55:04.581 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T20:55:04.581 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:11.817326+0000", 2023-09-23T20:55:04.581 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T20:55:04.581 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:55:04.581 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:55:04.581 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T20:55:04.581 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:11.817325+0000", 2023-09-23T20:55:04.581 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.999999 2023-09-23T20:55:04.582 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:55:04.582 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:55:04.582 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T20:55:04.582 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:11.817390+0000", 2023-09-23T20:55:04.582 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.5147999999999994e-05 2023-09-23T20:55:04.582 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:55:04.582 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:55:04.582 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T20:55:04.582 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:11.817400+0000", 2023-09-23T20:55:04.582 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.9990000000000003e-06 2023-09-23T20:55:04.582 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:55:04.583 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:55:04.583 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T20:55:04.583 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:11.817405+0000", 2023-09-23T20:55:04.583 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.515e-06 2023-09-23T20:55:04.583 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:55:04.583 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:55:04.583 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T20:55:04.583 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:11.817436+0000", 2023-09-23T20:55:04.583 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.1411000000000001e-05 2023-09-23T20:55:04.583 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:55:04.583 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:55:04.583 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T20:55:04.583 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:11.817525+0000", 2023-09-23T20:55:04.584 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.9116999999999994e-05 2023-09-23T20:55:04.584 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:55:04.584 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:55:04.584 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 1,2", 2023-09-23T20:55:04.584 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:11.817608+0000", 2023-09-23T20:55:04.584 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.2480999999999999e-05 2023-09-23T20:55:04.584 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:55:04.584 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:55:04.584 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T20:55:04.584 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:11.819981+0000", 2023-09-23T20:55:04.584 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0023728690000000001 2023-09-23T20:55:04.584 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:55:04.584 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:55:04.585 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:55:04.585 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:11.823806+0000", 2023-09-23T20:55:04.585 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0038258659999999998 2023-09-23T20:55:04.585 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:55:04.585 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:55:04.585 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:55:04.585 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:11.852810+0000", 2023-09-23T20:55:04.585 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.029003955000000001 2023-09-23T20:55:04.585 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:55:04.585 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:55:04.585 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T20:55:04.585 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:11.852842+0000", 2023-09-23T20:55:04.585 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.1964000000000001e-05 2023-09-23T20:55:04.586 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:55:04.586 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:55:04.586 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T20:55:04.586 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:11.852866+0000", 2023-09-23T20:55:04.586 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.4046e-05 2023-09-23T20:55:04.586 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T20:55:04.586 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T20:55:04.586 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T20:55:04.586 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:55:04.586 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:55:04.586 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.34797.0:76089 6.2 6:4c75ae3e:::benchmark_data_smithi026_223868_object76088:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e116)", 2023-09-23T20:55:04.586 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T20:53:13.255656+0000", 2023-09-23T20:55:04.587 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 111.221893726, 2023-09-23T20:55:04.587 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.032109897999999998, 2023-09-23T20:55:04.587 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T20:55:04.587 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:55:04.587 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T20:55:04.587 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.34797", 2023-09-23T20:55:04.587 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T20:55:04.587 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 76089 2023-09-23T20:55:04.587 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:55:04.587 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T20:55:04.587 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:55:04.587 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T20:55:04.588 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:13.255656+0000", 2023-09-23T20:55:04.588 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T20:55:04.588 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:55:04.588 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:55:04.588 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T20:55:04.588 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:13.255656+0000", 2023-09-23T20:55:04.588 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T20:55:04.588 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:55:04.588 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:55:04.588 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T20:55:04.588 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:13.255655+0000", 2023-09-23T20:55:04.588 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999995 2023-09-23T20:55:04.588 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:55:04.589 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:55:04.589 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T20:55:04.589 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:13.255731+0000", 2023-09-23T20:55:04.589 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.5731999999999999e-05 2023-09-23T20:55:04.589 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:55:04.589 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:55:04.589 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T20:55:04.589 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:13.255735+0000", 2023-09-23T20:55:04.589 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.3379999999999998e-06 2023-09-23T20:55:04.589 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:55:04.589 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:55:04.589 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T20:55:04.589 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:13.255738+0000", 2023-09-23T20:55:04.590 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.0359999999999998e-06 2023-09-23T20:55:04.590 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:55:04.590 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:55:04.590 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T20:55:04.590 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:13.259963+0000", 2023-09-23T20:55:04.590 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00422478 2023-09-23T20:55:04.590 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:55:04.590 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:55:04.590 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T20:55:04.590 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:13.260100+0000", 2023-09-23T20:55:04.590 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000136439 2023-09-23T20:55:04.591 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:55:04.591 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:55:04.591 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 2,3", 2023-09-23T20:55:04.591 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:13.260308+0000", 2023-09-23T20:55:04.591 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00020837500000000001 2023-09-23T20:55:04.591 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:55:04.591 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:55:04.591 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T20:55:04.591 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:13.261597+0000", 2023-09-23T20:55:04.591 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.001289206 2023-09-23T20:55:04.591 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:55:04.591 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:55:04.591 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:55:04.592 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:13.267034+0000", 2023-09-23T20:55:04.592 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0054363850000000002 2023-09-23T20:55:04.592 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:55:04.592 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:55:04.592 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:55:04.592 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:13.287676+0000", 2023-09-23T20:55:04.592 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.020642120999999999 2023-09-23T20:55:04.592 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:55:04.592 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:55:04.592 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T20:55:04.592 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:13.287724+0000", 2023-09-23T20:55:04.592 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.8084999999999998e-05 2023-09-23T20:55:04.592 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:55:04.593 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:55:04.593 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T20:55:04.593 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:13.287766+0000", 2023-09-23T20:55:04.593 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.1763e-05 2023-09-23T20:55:04.593 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T20:55:04.593 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T20:55:04.593 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T20:55:04.593 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:55:04.593 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:55:04.593 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.34797.0:79019 6.e 6:79b9a4a0:::benchmark_data_smithi026_223868_object79018:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e117)", 2023-09-23T20:55:04.593 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T20:53:16.491881+0000", 2023-09-23T20:55:04.593 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 107.985668037, 2023-09-23T20:55:04.593 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.034071830999999997, 2023-09-23T20:55:04.594 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T20:55:04.594 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:55:04.594 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T20:55:04.594 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.34797", 2023-09-23T20:55:04.594 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T20:55:04.594 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 79019 2023-09-23T20:55:04.594 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:55:04.594 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T20:55:04.594 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:55:04.594 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T20:55:04.594 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:16.491881+0000", 2023-09-23T20:55:04.594 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T20:55:04.595 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:55:04.595 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:55:04.595 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T20:55:04.595 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:16.491881+0000", 2023-09-23T20:55:04.595 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T20:55:04.595 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:55:04.595 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:55:04.595 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T20:55:04.595 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:16.491881+0000", 2023-09-23T20:55:04.595 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.999999 2023-09-23T20:55:04.595 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:55:04.595 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:55:04.596 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T20:55:04.596 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:16.491939+0000", 2023-09-23T20:55:04.596 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.8334000000000001e-05 2023-09-23T20:55:04.596 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:55:04.596 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:55:04.596 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T20:55:04.596 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:16.491952+0000", 2023-09-23T20:55:04.596 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.2605e-05 2023-09-23T20:55:04.596 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:55:04.596 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:55:04.596 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T20:55:04.596 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:16.491957+0000", 2023-09-23T20:55:04.596 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.9580000000000004e-06 2023-09-23T20:55:04.597 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:55:04.597 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:55:04.597 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T20:55:04.597 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:16.492004+0000", 2023-09-23T20:55:04.597 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.6946000000000001e-05 2023-09-23T20:55:04.597 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:55:04.597 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:55:04.597 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T20:55:04.597 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:16.492146+0000", 2023-09-23T20:55:04.597 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00014170599999999999 2023-09-23T20:55:04.597 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:55:04.597 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:55:04.597 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 1,2", 2023-09-23T20:55:04.598 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:16.492317+0000", 2023-09-23T20:55:04.598 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000171106 2023-09-23T20:55:04.598 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:55:04.598 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:55:04.598 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T20:55:04.598 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:16.494714+0000", 2023-09-23T20:55:04.598 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0023966640000000002 2023-09-23T20:55:04.598 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:55:04.598 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:55:04.598 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:55:04.598 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:16.496325+0000", 2023-09-23T20:55:04.598 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.001611061 2023-09-23T20:55:04.599 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:55:04.599 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:55:04.599 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:55:04.599 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:16.525889+0000", 2023-09-23T20:55:04.599 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.029563754000000001 2023-09-23T20:55:04.599 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:55:04.599 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:55:04.599 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T20:55:04.599 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:16.525925+0000", 2023-09-23T20:55:04.599 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.612e-05 2023-09-23T20:55:04.599 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:55:04.599 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:55:04.599 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T20:55:04.600 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:16.525953+0000", 2023-09-23T20:55:04.600 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.8404000000000001e-05 2023-09-23T20:55:04.600 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T20:55:04.600 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T20:55:04.600 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T20:55:04.600 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:55:04.600 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:55:04.600 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.34797.0:89045 6.2 6:469b1fad:::benchmark_data_smithi026_223868_object89044:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e120)", 2023-09-23T20:55:04.600 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T20:53:27.589165+0000", 2023-09-23T20:55:04.600 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 96.888384752999997, 2023-09-23T20:55:04.600 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.033925067000000003, 2023-09-23T20:55:04.600 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T20:55:04.601 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:55:04.601 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T20:55:04.601 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.34797", 2023-09-23T20:55:04.601 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T20:55:04.601 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 89045 2023-09-23T20:55:04.601 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:55:04.601 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T20:55:04.601 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:55:04.601 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T20:55:04.601 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:27.589165+0000", 2023-09-23T20:55:04.601 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T20:55:04.601 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:55:04.601 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:55:04.602 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T20:55:04.602 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:27.589165+0000", 2023-09-23T20:55:04.602 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T20:55:04.602 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:55:04.602 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:55:04.602 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T20:55:04.602 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:27.589163+0000", 2023-09-23T20:55:04.602 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T20:55:04.602 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:55:04.602 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:55:04.602 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T20:55:04.602 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:27.589200+0000", 2023-09-23T20:55:04.603 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.7212999999999998e-05 2023-09-23T20:55:04.603 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:55:04.603 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:55:04.603 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T20:55:04.603 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:27.589245+0000", 2023-09-23T20:55:04.603 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.4545000000000001e-05 2023-09-23T20:55:04.603 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:55:04.603 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:55:04.603 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T20:55:04.603 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:27.589256+0000", 2023-09-23T20:55:04.603 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.1163e-05 2023-09-23T20:55:04.603 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:55:04.603 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:55:04.604 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T20:55:04.604 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:27.589315+0000", 2023-09-23T20:55:04.604 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.8695e-05 2023-09-23T20:55:04.604 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:55:04.604 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:55:04.604 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T20:55:04.604 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:27.589448+0000", 2023-09-23T20:55:04.604 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00013358700000000001 2023-09-23T20:55:04.604 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:55:04.604 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:55:04.604 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 2,3", 2023-09-23T20:55:04.604 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:27.589595+0000", 2023-09-23T20:55:04.605 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00014690499999999999 2023-09-23T20:55:04.605 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:55:04.605 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:55:04.605 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T20:55:04.605 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:27.591942+0000", 2023-09-23T20:55:04.605 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.002346435 2023-09-23T20:55:04.605 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:55:04.605 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:55:04.605 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:55:04.605 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:27.604948+0000", 2023-09-23T20:55:04.605 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.013006442 2023-09-23T20:55:04.605 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:55:04.605 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:55:04.606 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:55:04.606 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:27.623008+0000", 2023-09-23T20:55:04.606 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.018060207000000002 2023-09-23T20:55:04.606 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:55:04.606 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:55:04.606 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T20:55:04.606 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:27.623050+0000", 2023-09-23T20:55:04.606 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.1573000000000003e-05 2023-09-23T20:55:04.606 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:55:04.606 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:55:04.606 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T20:55:04.607 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:53:27.623090+0000", 2023-09-23T20:55:04.607 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.9873999999999998e-05 2023-09-23T20:55:04.607 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T20:55:04.607 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T20:55:04.607 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T20:55:04.607 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:55:04.607 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:55:04.607 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.34797.0:114997 6.12 6.28c54f92 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e132)", 2023-09-23T20:55:04.607 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T20:54:12.985231+0000", 2023-09-23T20:55:04.607 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 51.492318695999998, 2023-09-23T20:55:04.607 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.939145275, 2023-09-23T20:55:04.607 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T20:55:04.607 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "reached pg", 2023-09-23T20:55:04.608 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T20:55:04.608 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.34797", 2023-09-23T20:55:04.608 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T20:55:04.608 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 114997 2023-09-23T20:55:04.608 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:55:04.608 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T20:55:04.608 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:55:04.608 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T20:55:04.608 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:54:12.985231+0000", 2023-09-23T20:55:04.608 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T20:55:04.608 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:55:04.608 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:55:04.608 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T20:55:04.609 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:54:12.985231+0000", 2023-09-23T20:55:04.609 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T20:55:04.609 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:55:04.609 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:55:04.609 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T20:55:04.609 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:54:12.985228+0000", 2023-09-23T20:55:04.609 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999976 2023-09-23T20:55:04.609 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:55:04.609 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:55:04.609 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T20:55:04.609 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:54:12.985282+0000", 2023-09-23T20:55:04.609 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.3285000000000002e-05 2023-09-23T20:55:04.609 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:55:04.609 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:55:04.610 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T20:55:04.610 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:54:12.985298+0000", 2023-09-23T20:55:04.610 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.6677e-05 2023-09-23T20:55:04.610 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:55:04.610 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:55:04.610 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T20:55:04.610 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:54:12.985306+0000", 2023-09-23T20:55:04.610 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.8590000000000005e-06 2023-09-23T20:55:04.610 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:55:04.610 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:55:04.610 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T20:55:04.611 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:54:12.985354+0000", 2023-09-23T20:55:04.611 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.8124000000000002e-05 2023-09-23T20:55:04.611 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:55:04.611 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:55:04.611 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T20:55:04.611 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:54:12.985364+0000", 2023-09-23T20:55:04.611 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.6500000000000008e-06 2023-09-23T20:55:04.611 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:55:04.611 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:55:04.611 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T20:55:04.611 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:54:13.924362+0000", 2023-09-23T20:55:04.611 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.93899794599999997 2023-09-23T20:55:04.611 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:55:04.612 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:55:04.612 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T20:55:04.612 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:54:13.924376+0000", 2023-09-23T20:55:04.612 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.4238e-05 2023-09-23T20:55:04.612 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T20:55:04.612 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T20:55:04.612 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T20:55:04.612 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:55:04.612 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:55:04.612 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.34797.0:151330 6.e 6:70d4666f:::benchmark_data_smithi026_223868_object50204:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e147)", 2023-09-23T20:55:04.612 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T20:55:00.767727+0000", 2023-09-23T20:55:04.612 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 3.7098220509999997, 2023-09-23T20:55:04.613 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.033173332999999999, 2023-09-23T20:55:04.613 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T20:55:04.613 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:55:04.613 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T20:55:04.613 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.34797", 2023-09-23T20:55:04.613 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T20:55:04.613 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 151330 2023-09-23T20:55:04.613 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:55:04.613 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T20:55:04.613 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:55:04.613 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T20:55:04.613 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:00.767727+0000", 2023-09-23T20:55:04.614 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T20:55:04.614 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:55:04.614 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:55:04.614 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T20:55:04.614 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:00.767727+0000", 2023-09-23T20:55:04.614 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T20:55:04.614 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:55:04.614 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:55:04.614 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T20:55:04.614 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:00.767726+0000", 2023-09-23T20:55:04.614 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.999999 2023-09-23T20:55:04.614 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:55:04.614 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:55:04.615 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T20:55:04.615 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:00.767731+0000", 2023-09-23T20:55:04.615 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.3880000000000002e-06 2023-09-23T20:55:04.615 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:55:04.615 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:55:04.615 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T20:55:04.615 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:00.767739+0000", 2023-09-23T20:55:04.615 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.2840000000000006e-06 2023-09-23T20:55:04.615 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:55:04.615 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:55:04.615 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T20:55:04.615 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:00.767743+0000", 2023-09-23T20:55:04.616 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.1200000000000004e-06 2023-09-23T20:55:04.616 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:55:04.616 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:55:04.616 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T20:55:04.616 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:00.767777+0000", 2023-09-23T20:55:04.616 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.4085999999999999e-05 2023-09-23T20:55:04.616 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:55:04.616 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:55:04.616 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T20:55:04.616 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:00.767863+0000", 2023-09-23T20:55:04.616 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.5678000000000002e-05 2023-09-23T20:55:04.616 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:55:04.616 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:55:04.617 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 1,2", 2023-09-23T20:55:04.617 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:00.767957+0000", 2023-09-23T20:55:04.617 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.3935000000000001e-05 2023-09-23T20:55:04.617 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:55:04.617 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:55:04.617 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T20:55:04.617 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:00.770336+0000", 2023-09-23T20:55:04.617 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0023787180000000002 2023-09-23T20:55:04.617 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:55:04.617 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:55:04.617 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:55:04.617 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:00.779621+0000", 2023-09-23T20:55:04.617 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0092851449999999999 2023-09-23T20:55:04.618 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:55:04.618 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:55:04.618 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:55:04.618 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:00.800847+0000", 2023-09-23T20:55:04.618 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.021225820999999999 2023-09-23T20:55:04.618 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:55:04.618 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:55:04.618 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T20:55:04.618 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:00.800876+0000", 2023-09-23T20:55:04.618 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.9896e-05 2023-09-23T20:55:04.618 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:55:04.618 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:55:04.619 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T20:55:04.619 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:00.800901+0000", 2023-09-23T20:55:04.619 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.4332999999999999e-05 2023-09-23T20:55:04.619 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T20:55:04.619 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T20:55:04.619 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T20:55:04.619 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T20:55:04.619 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T20:55:04.619 INFO:teuthology.orchestra.run.smithi070.stdout:} 2023-09-23T20:55:04.620 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:55:04.856 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_ops_in_flight 2023-09-23T20:55:05.130 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:55:05.138 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:55:05.139 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:55:05.240 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:55:05.272 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:55:05.305 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:55:05.308 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:55:05.308 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:55:05.408 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T20:55:05.409 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:55:05.469 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:55:05.476 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:55:05 smithi026 ceph-mon[175151]: pgmap v367: 99 pgs: 1 active+remapped+backfilling, 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded+remapped, 91 active+clean, 1 active+recovery_wait, 3 active+remapped+backfill_wait; 3.0 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 730 op/s; 1603/149367 objects degraded (1.073%); 10784/149367 objects misplaced (7.220%); 13 MiB/s, 284 objects/s recovering 2023-09-23T20:55:05.476 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:55:05 smithi026 ceph-mon[175151]: Health check update: Degraded data redundancy: 1603/149367 objects degraded (1.073%), 2 pgs degraded (PG_DEGRADED) 2023-09-23T20:55:05.476 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:55:05 smithi026 ceph-mon[179688]: pgmap v367: 99 pgs: 1 active+remapped+backfilling, 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded+remapped, 91 active+clean, 1 active+recovery_wait, 3 active+remapped+backfill_wait; 3.0 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 730 op/s; 1603/149367 objects degraded (1.073%); 10784/149367 objects misplaced (7.220%); 13 MiB/s, 284 objects/s recovering 2023-09-23T20:55:05.477 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:55:05 smithi026 ceph-mon[179688]: Health check update: Degraded data redundancy: 1603/149367 objects degraded (1.073%), 2 pgs degraded (PG_DEGRADED) 2023-09-23T20:55:05.517 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:55:05.522 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:55:05 smithi070 ceph-mon[168282]: pgmap v367: 99 pgs: 1 active+remapped+backfilling, 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded+remapped, 91 active+clean, 1 active+recovery_wait, 3 active+remapped+backfill_wait; 3.0 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 730 op/s; 1603/149367 objects degraded (1.073%); 10784/149367 objects misplaced (7.220%); 13 MiB/s, 284 objects/s recovering 2023-09-23T20:55:05.523 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:55:05 smithi070 ceph-mon[168282]: Health check update: Degraded data redundancy: 1603/149367 objects degraded (1.073%), 2 pgs degraded (PG_DEGRADED) 2023-09-23T20:55:05.523 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:55:05.524 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:55:05.569 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:55:05.624 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:55:05.657 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:55:05.713 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:55:05.717 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:55:05.718 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:55:05.818 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:55:05.861 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:55:05.923 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:55:05.929 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:55:05.929 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:55:06.030 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:55:06.088 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:55:06.993 INFO:teuthology.orchestra.run.smithi070.stdout:{ 2023-09-23T20:55:06.993 INFO:teuthology.orchestra.run.smithi070.stdout: "ops": [ 2023-09-23T20:55:06.993 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:55:06.993 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.34797.0:156425 6.d 6:b2acaa3e:::benchmark_data_smithi026_223868_object55299:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e149)", 2023-09-23T20:55:06.994 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T20:55:06.980893+0000", 2023-09-23T20:55:06.994 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 0.011771660999999999, 2023-09-23T20:55:06.994 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.011794450999999999, 2023-09-23T20:55:06.994 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T20:55:06.994 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "waiting for sub ops", 2023-09-23T20:55:06.994 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T20:55:06.994 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.34797", 2023-09-23T20:55:06.998 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T20:55:06.998 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 156425 2023-09-23T20:55:06.998 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:55:06.998 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T20:55:06.998 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:55:06.998 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T20:55:06.998 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:06.980893+0000", 2023-09-23T20:55:06.999 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T20:55:06.999 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:55:06.999 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:55:06.999 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T20:55:06.999 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:06.980893+0000", 2023-09-23T20:55:06.999 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T20:55:06.999 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:55:06.999 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:55:06.999 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T20:55:06.999 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:06.980892+0000", 2023-09-23T20:55:06.999 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999981 2023-09-23T20:55:06.999 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:55:07.000 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:55:07.000 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T20:55:07.000 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:06.980900+0000", 2023-09-23T20:55:07.000 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.2109999999999998e-06 2023-09-23T20:55:07.000 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:55:07.000 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:55:07.000 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T20:55:07.000 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:06.980918+0000", 2023-09-23T20:55:07.000 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.8405000000000001e-05 2023-09-23T20:55:07.000 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:55:07.000 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:55:07.000 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T20:55:07.001 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:06.980926+0000", 2023-09-23T20:55:07.001 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.2010000000000002e-06 2023-09-23T20:55:07.001 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:55:07.001 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:55:07.001 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T20:55:07.001 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:06.980987+0000", 2023-09-23T20:55:07.001 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.1001000000000001e-05 2023-09-23T20:55:07.001 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:55:07.001 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:55:07.001 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T20:55:07.001 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:06.981094+0000", 2023-09-23T20:55:07.001 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000106986 2023-09-23T20:55:07.001 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:55:07.002 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:55:07.002 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 0,1", 2023-09-23T20:55:07.002 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:06.981191+0000", 2023-09-23T20:55:07.002 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.7801999999999996e-05 2023-09-23T20:55:07.002 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:55:07.002 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:55:07.002 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T20:55:07.002 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:06.987114+0000", 2023-09-23T20:55:07.002 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.005922259 2023-09-23T20:55:07.002 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:55:07.002 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:55:07.003 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:55:07.003 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:06.987190+0000", 2023-09-23T20:55:07.003 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.6865000000000004e-05 2023-09-23T20:55:07.003 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T20:55:07.003 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T20:55:07.003 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T20:55:07.003 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:55:07.003 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:55:07.003 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.34797.0:156426 6.d 6:b9c99cb2:::benchmark_data_smithi026_223868_object55300:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e149)", 2023-09-23T20:55:07.003 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T20:55:06.980939+0000", 2023-09-23T20:55:07.003 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 0.011725922, 2023-09-23T20:55:07.003 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.011817256, 2023-09-23T20:55:07.003 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T20:55:07.004 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "waiting for sub ops", 2023-09-23T20:55:07.004 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T20:55:07.004 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.34797", 2023-09-23T20:55:07.004 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T20:55:07.004 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 156426 2023-09-23T20:55:07.004 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:55:07.004 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T20:55:07.004 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:55:07.004 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T20:55:07.004 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:06.980939+0000", 2023-09-23T20:55:07.004 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T20:55:07.004 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:55:07.005 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:55:07.005 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T20:55:07.005 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:06.980939+0000", 2023-09-23T20:55:07.005 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T20:55:07.005 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:55:07.005 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:55:07.005 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T20:55:07.005 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:06.980939+0000", 2023-09-23T20:55:07.005 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999995 2023-09-23T20:55:07.005 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:55:07.005 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:55:07.005 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T20:55:07.005 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:06.980941+0000", 2023-09-23T20:55:07.006 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.5639999999999999e-06 2023-09-23T20:55:07.006 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:55:07.006 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:55:07.006 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T20:55:07.006 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:06.980947+0000", 2023-09-23T20:55:07.006 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.4920000000000001e-06 2023-09-23T20:55:07.006 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:55:07.006 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:55:07.006 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T20:55:07.006 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:06.980951+0000", 2023-09-23T20:55:07.006 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.7419999999999999e-06 2023-09-23T20:55:07.007 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:55:07.007 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:55:07.007 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T20:55:07.007 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:06.983860+0000", 2023-09-23T20:55:07.007 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0029094500000000001 2023-09-23T20:55:07.007 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:55:07.007 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:55:07.007 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T20:55:07.007 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:06.983994+0000", 2023-09-23T20:55:07.007 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00013404399999999999 2023-09-23T20:55:07.007 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:55:07.007 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:55:07.007 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 0,1", 2023-09-23T20:55:07.008 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:06.984106+0000", 2023-09-23T20:55:07.008 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000111838 2023-09-23T20:55:07.008 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:55:07.008 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:55:07.008 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T20:55:07.008 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:06.987209+0000", 2023-09-23T20:55:07.008 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.003103106 2023-09-23T20:55:07.008 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T20:55:07.008 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T20:55:07.008 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T20:55:07.008 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T20:55:07.008 INFO:teuthology.orchestra.run.smithi070.stdout: ], 2023-09-23T20:55:07.008 INFO:teuthology.orchestra.run.smithi070.stdout: "num_ops": 2 2023-09-23T20:55:07.009 INFO:teuthology.orchestra.run.smithi070.stdout:} 2023-09-23T20:55:07.484 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_blocked_ops 2023-09-23T20:55:07.506 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:55:07 smithi026 ceph-mon[175151]: pgmap v368: 99 pgs: 1 active+remapped+backfilling, 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded+remapped, 91 active+clean, 1 active+recovery_wait, 3 active+remapped+backfill_wait; 3.0 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 670 op/s; 1603/147123 objects degraded (1.090%); 9600/147123 objects misplaced (6.525%); 19 MiB/s, 353 objects/s recovering 2023-09-23T20:55:07.507 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:55:07 smithi026 ceph-mon[179688]: pgmap v368: 99 pgs: 1 active+remapped+backfilling, 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded+remapped, 91 active+clean, 1 active+recovery_wait, 3 active+remapped+backfill_wait; 3.0 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 670 op/s; 1603/147123 objects degraded (1.090%); 9600/147123 objects misplaced (6.525%); 19 MiB/s, 353 objects/s recovering 2023-09-23T20:55:07.530 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:55:07.535 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:55:07.536 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:55:07.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:55:07 smithi070 ceph-mon[168282]: pgmap v368: 99 pgs: 1 active+remapped+backfilling, 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded+remapped, 91 active+clean, 1 active+recovery_wait, 3 active+remapped+backfill_wait; 3.0 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 670 op/s; 1603/147123 objects degraded (1.090%); 9600/147123 objects misplaced (6.525%); 19 MiB/s, 353 objects/s recovering 2023-09-23T20:55:07.636 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:55:07.698 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:55:07.752 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:55:07.758 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:55:07.758 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:55:07.859 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:55:07.917 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:55:07.976 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:55:07.984 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:55:07.985 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:55:08.085 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:55:08.116 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:55:08.155 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:55:08.158 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:55:08.158 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:55:08.166 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:55:08.260 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:55:08.288 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:55:08.324 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:55:08.326 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:55:08.327 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:55:08.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:55:08 smithi070 ceph-mon[168282]: failed to encode map e150 with expected crc 2023-09-23T20:55:08.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:55:08 smithi070 ceph-mon[168282]: failed to encode map e150 with expected crc 2023-09-23T20:55:08.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:55:08 smithi070 ceph-mon[168282]: osdmap e150: 8 total, 8 up, 8 in 2023-09-23T20:55:08.428 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:55:08.476 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:55:08 smithi026 ceph-mon[175151]: failed to encode map e150 with expected crc 2023-09-23T20:55:08.476 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:55:08 smithi026 ceph-mon[175151]: failed to encode map e150 with expected crc 2023-09-23T20:55:08.476 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:55:08 smithi026 ceph-mon[175151]: osdmap e150: 8 total, 8 up, 8 in 2023-09-23T20:55:08.482 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:55:08 smithi026 ceph-mon[179688]: failed to encode map e150 with expected crc 2023-09-23T20:55:08.482 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:55:08 smithi026 ceph-mon[179688]: failed to encode map e150 with expected crc 2023-09-23T20:55:08.482 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:55:08 smithi026 ceph-mon[179688]: osdmap e150: 8 total, 8 up, 8 in 2023-09-23T20:55:08.508 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:55:08.527 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:55:08.584 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:55:08.588 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:55:08.588 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:55:08.689 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:55:08.769 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:55:08.924 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:55:08.935 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:55:08.936 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:55:09.036 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:55:09.090 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:55:09.140 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:55:09.144 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:55:09.144 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:55:09.234 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T20:55:09.245 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:55:09.397 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:55:09.561 INFO:teuthology.orchestra.run.smithi070.stdout:{ 2023-09-23T20:55:09.561 INFO:teuthology.orchestra.run.smithi070.stdout: "ops": [], 2023-09-23T20:55:09.561 INFO:teuthology.orchestra.run.smithi070.stdout: "complaint_time": 30, 2023-09-23T20:55:09.561 INFO:teuthology.orchestra.run.smithi070.stdout: "num_blocked_ops": 0 2023-09-23T20:55:09.561 INFO:teuthology.orchestra.run.smithi070.stdout:} 2023-09-23T20:55:09.614 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:55:09 smithi026 ceph-mon[179688]: 6.5 scrub starts 2023-09-23T20:55:09.614 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:55:09 smithi026 ceph-mon[179688]: pgmap v370: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfilling, 2 active+recovery_wait+degraded+remapped, 91 active+clean, 1 active+recovery_wait, 3 active+remapped+backfill_wait; 2.9 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 769 op/s; 1603/141108 objects degraded (1.136%); 9273/141108 objects misplaced (6.572%); 13 MiB/s, 288 objects/s recovering 2023-09-23T20:55:09.615 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:55:09 smithi026 ceph-mon[179688]: failed to encode map e151 with expected crc 2023-09-23T20:55:09.615 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:55:09 smithi026 ceph-mon[179688]: osdmap e151: 8 total, 8 up, 8 in 2023-09-23T20:55:09.615 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:55:09 smithi026 ceph-mon[179688]: failed to encode map e151 with expected crc 2023-09-23T20:55:09.615 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:55:09 smithi026 ceph-mon[179688]: failed to encode map e151 with expected crc 2023-09-23T20:55:09.615 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:55:09 smithi026 ceph-mon[179688]: failed to encode map e151 with expected crc 2023-09-23T20:55:09.615 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:55:09 smithi026 ceph-mon[179688]: failed to encode map e151 with expected crc 2023-09-23T20:55:09.615 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:55:09 smithi026 ceph-mon[179688]: failed to encode map e151 with expected crc 2023-09-23T20:55:09.615 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:55:09 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:55:09.615 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:55:09 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:55:09.615 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:55:09 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:55:09.615 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:55:09 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:55:09.615 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:55:09 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:55:09.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:55:09 smithi070 ceph-mon[168282]: 6.5 scrub starts 2023-09-23T20:55:09.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:55:09 smithi070 ceph-mon[168282]: pgmap v370: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfilling, 2 active+recovery_wait+degraded+remapped, 91 active+clean, 1 active+recovery_wait, 3 active+remapped+backfill_wait; 2.9 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 769 op/s; 1603/141108 objects degraded (1.136%); 9273/141108 objects misplaced (6.572%); 13 MiB/s, 288 objects/s recovering 2023-09-23T20:55:09.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:55:09 smithi070 ceph-mon[168282]: failed to encode map e151 with expected crc 2023-09-23T20:55:09.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:55:09 smithi070 ceph-mon[168282]: osdmap e151: 8 total, 8 up, 8 in 2023-09-23T20:55:09.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:55:09 smithi070 ceph-mon[168282]: failed to encode map e151 with expected crc 2023-09-23T20:55:09.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:55:09 smithi070 ceph-mon[168282]: failed to encode map e151 with expected crc 2023-09-23T20:55:09.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:55:09 smithi070 ceph-mon[168282]: failed to encode map e151 with expected crc 2023-09-23T20:55:09.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:55:09 smithi070 ceph-mon[168282]: failed to encode map e151 with expected crc 2023-09-23T20:55:09.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:55:09 smithi070 ceph-mon[168282]: failed to encode map e151 with expected crc 2023-09-23T20:55:09.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:55:09 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:55:09.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:55:09 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:55:09.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:55:09 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:55:09.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:55:09 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:55:09.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:55:09 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:55:09.626 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:55:09 smithi026 ceph-mon[175151]: 6.5 scrub starts 2023-09-23T20:55:09.626 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:55:09 smithi026 ceph-mon[175151]: pgmap v370: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfilling, 2 active+recovery_wait+degraded+remapped, 91 active+clean, 1 active+recovery_wait, 3 active+remapped+backfill_wait; 2.9 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 769 op/s; 1603/141108 objects degraded (1.136%); 9273/141108 objects misplaced (6.572%); 13 MiB/s, 288 objects/s recovering 2023-09-23T20:55:09.626 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:55:09 smithi026 ceph-mon[175151]: failed to encode map e151 with expected crc 2023-09-23T20:55:09.626 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:55:09 smithi026 ceph-mon[175151]: osdmap e151: 8 total, 8 up, 8 in 2023-09-23T20:55:09.626 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:55:09 smithi026 ceph-mon[175151]: failed to encode map e151 with expected crc 2023-09-23T20:55:09.626 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:55:09 smithi026 ceph-mon[175151]: failed to encode map e151 with expected crc 2023-09-23T20:55:09.626 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:55:09 smithi026 ceph-mon[175151]: failed to encode map e151 with expected crc 2023-09-23T20:55:09.626 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:55:09 smithi026 ceph-mon[175151]: failed to encode map e151 with expected crc 2023-09-23T20:55:09.626 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:55:09 smithi026 ceph-mon[175151]: failed to encode map e151 with expected crc 2023-09-23T20:55:09.626 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:55:09 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:55:09.626 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:55:09 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:55:09.627 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:55:09 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:55:09.627 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:55:09 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:55:09.627 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:55:09 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:55:09.627 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:55:09 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[164029]: ::ffff:172.21.15.70 - - [23/Sep/2023:20:55:09] "GET /metrics HTTP/1.1" 200 36354 "" "Prometheus/2.43.0" 2023-09-23T20:55:09.661 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:55:09.668 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:55:09.668 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:55:09.769 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:55:09.802 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:55:09.904 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:55:09.910 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:55:09.911 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:55:09.987 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_historic_ops 2023-09-23T20:55:10.011 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:55:10.088 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:55:10.153 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:55:10.160 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:55:10.161 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:55:10.261 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:55:10.318 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:55:10.320 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:55:10.384 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:55:10.387 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:55:10.387 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:55:10.432 INFO:teuthology.orchestra.run.smithi026.stderr:set pool 1 pg_num to 11 2023-09-23T20:55:10.473 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0: {} 2023-09-23T20:55:10.488 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:55:10.534 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1: {} 2023-09-23T20:55:10.545 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:55:10.582 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2: {} 2023-09-23T20:55:10.587 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:55:10.592 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:55:10 smithi070 ceph-mon[168282]: 6.8 scrub starts 2023-09-23T20:55:10.592 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:55:10 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/2670853214' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "11"}]: dispatch 2023-09-23T20:55:10.592 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:55:10 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "11"}]: dispatch 2023-09-23T20:55:10.592 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:55:10 smithi070 ceph-mon[168282]: Health check update: Degraded data redundancy: 1603/141108 objects degraded (1.136%), 2 pgs degraded (PG_DEGRADED) 2023-09-23T20:55:10.593 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:55:10.593 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:55:10.625 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3: {} 2023-09-23T20:55:10.632 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:55:10.662 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4: {} 2023-09-23T20:55:10.695 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:55:10.697 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T20:55:10.736 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:55:10.739 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6: {} 2023-09-23T20:55:10.779 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T20:55:10.792 INFO:teuthology.orchestra.run.smithi026.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T20:55:10.792 INFO:teuthology.orchestra.run.smithi026.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T20:55:10.792 INFO:teuthology.orchestra.run.smithi026.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T20:55:10.792 INFO:teuthology.orchestra.run.smithi026.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T20:55:10.792 INFO:teuthology.orchestra.run.smithi026.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T20:55:10.792 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T20:55:10.792 INFO:teuthology.orchestra.run.smithi026.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T20:55:10.792 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T20:55:10.796 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:55:10.801 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:55:10.801 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:55:10.822 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:55:10 smithi026 ceph-mon[179688]: 6.8 scrub starts 2023-09-23T20:55:10.822 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:55:10 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/2670853214' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "11"}]: dispatch 2023-09-23T20:55:10.822 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:55:10 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "11"}]: dispatch 2023-09-23T20:55:10.822 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:55:10 smithi026 ceph-mon[179688]: Health check update: Degraded data redundancy: 1603/141108 objects degraded (1.136%), 2 pgs degraded (PG_DEGRADED) 2023-09-23T20:55:10.822 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:55:10 smithi026 ceph-mon[175151]: 6.8 scrub starts 2023-09-23T20:55:10.823 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:55:10 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/2670853214' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "11"}]: dispatch 2023-09-23T20:55:10.823 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:55:10 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "11"}]: dispatch 2023-09-23T20:55:10.823 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:55:10 smithi026 ceph-mon[175151]: Health check update: Degraded data redundancy: 1603/141108 objects degraded (1.136%), 2 pgs degraded (PG_DEGRADED) 2023-09-23T20:55:10.902 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:55:11.009 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:55:11.309 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:55:11.317 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:55:11.317 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:55:11.417 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:55:11.479 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:55:11 smithi070 ceph-mon[168282]: 6.3 scrub starts 2023-09-23T20:55:11.479 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:55:11 smithi070 ceph-mon[168282]: pgmap v372: 99 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+remapped, 1 active+remapped+backfilling, 1 active+recovery_wait+degraded+remapped, 91 active+clean, 1 active+recovery_wait, 3 active+remapped+backfill_wait; 2.8 GiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 713 op/s; 822/136413 objects degraded (0.603%); 7172/136413 objects misplaced (5.258%); 14 MiB/s, 283 objects/s recovering 2023-09-23T20:55:11.480 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:55:11 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "11"}]': finished 2023-09-23T20:55:11.480 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:55:11 smithi070 ceph-mon[168282]: osdmap e152: 8 total, 8 up, 8 in 2023-09-23T20:55:11.480 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:55:11 smithi070 ceph-mon[168282]: failed to encode map e152 with expected crc 2023-09-23T20:55:11.480 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:55:11 smithi070 ceph-mon[168282]: failed to encode map e152 with expected crc 2023-09-23T20:55:11.480 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:55:11 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/1576823065' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:55:11.480 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:55:11 smithi070 ceph-mon[168282]: 6.3 scrub ok 2023-09-23T20:55:11.511 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:55:11.751 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:55:11 smithi026 ceph-mon[179688]: 6.3 scrub starts 2023-09-23T20:55:11.752 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:55:11 smithi026 ceph-mon[179688]: pgmap v372: 99 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+remapped, 1 active+remapped+backfilling, 1 active+recovery_wait+degraded+remapped, 91 active+clean, 1 active+recovery_wait, 3 active+remapped+backfill_wait; 2.8 GiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 713 op/s; 822/136413 objects degraded (0.603%); 7172/136413 objects misplaced (5.258%); 14 MiB/s, 283 objects/s recovering 2023-09-23T20:55:11.752 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:55:11 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "11"}]': finished 2023-09-23T20:55:11.752 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:55:11 smithi026 ceph-mon[179688]: osdmap e152: 8 total, 8 up, 8 in 2023-09-23T20:55:11.752 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:55:11 smithi026 ceph-mon[179688]: failed to encode map e152 with expected crc 2023-09-23T20:55:11.752 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:55:11 smithi026 ceph-mon[179688]: failed to encode map e152 with expected crc 2023-09-23T20:55:11.752 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:55:11 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/1576823065' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:55:11.752 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:55:11 smithi026 ceph-mon[179688]: 6.3 scrub ok 2023-09-23T20:55:11.752 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:55:11 smithi026 ceph-mon[175151]: 6.3 scrub starts 2023-09-23T20:55:11.752 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:55:11 smithi026 ceph-mon[175151]: pgmap v372: 99 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+remapped, 1 active+remapped+backfilling, 1 active+recovery_wait+degraded+remapped, 91 active+clean, 1 active+recovery_wait, 3 active+remapped+backfill_wait; 2.8 GiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 713 op/s; 822/136413 objects degraded (0.603%); 7172/136413 objects misplaced (5.258%); 14 MiB/s, 283 objects/s recovering 2023-09-23T20:55:11.753 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:55:11 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "11"}]': finished 2023-09-23T20:55:11.753 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:55:11 smithi026 ceph-mon[175151]: osdmap e152: 8 total, 8 up, 8 in 2023-09-23T20:55:11.753 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:55:11 smithi026 ceph-mon[175151]: failed to encode map e152 with expected crc 2023-09-23T20:55:11.753 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:55:11 smithi026 ceph-mon[175151]: failed to encode map e152 with expected crc 2023-09-23T20:55:11.753 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:55:11 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/1576823065' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:55:11.753 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:55:11 smithi026 ceph-mon[175151]: 6.3 scrub ok 2023-09-23T20:55:11.766 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:55:11.777 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:55:11.778 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:55:11.878 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:55:11.939 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:55:11.959 INFO:teuthology.orchestra.run.smithi070.stdout:ERROR: (22) Invalid argument 2023-09-23T20:55:11.959 INFO:teuthology.orchestra.run.smithi070.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:55:12.000 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:55:12.003 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:55:12.003 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:55:12.104 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:55:12.185 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:55:12.269 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:55:12.282 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:55:12.283 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:55:12.340 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_ops_in_flight 2023-09-23T20:55:12.383 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:55:12.473 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:55:12.544 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:55:12.552 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:55:12.552 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:55:12.621 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:55:12 smithi070 ceph-mon[168282]: 6.18 scrub starts 2023-09-23T20:55:12.621 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:55:12 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "11"}]: dispatch 2023-09-23T20:55:12.652 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:55:12.693 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:55:12.737 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:55:12.741 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:55:12.741 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:55:12.842 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:55:12.881 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:55:12 smithi026 ceph-mon[179688]: 6.18 scrub starts 2023-09-23T20:55:12.881 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:55:12 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "11"}]: dispatch 2023-09-23T20:55:12.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:55:12 smithi026 ceph-mon[175151]: 6.18 scrub starts 2023-09-23T20:55:12.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:55:12 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "11"}]: dispatch 2023-09-23T20:55:12.882 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:55:12.944 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:55:12.949 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:55:12.949 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:55:13.050 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:55:13.091 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:55:13.098 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:55:13.165 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:55:13.172 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:55:13.173 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:55:13.273 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:55:13.355 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:55:13.430 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:55:13.439 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:55:13.439 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:55:13.455 INFO:teuthology.orchestra.run.smithi026.stderr:noscrub is set 2023-09-23T20:55:13.475 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T20:55:13.540 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:55:13.580 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:55:13.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:55:13 smithi070 ceph-mon[168282]: pgmap v374: 99 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+degraded+remapped, 92 active+clean, 1 active+recovery_wait, 4 active+remapped+backfill_wait; 2.6 GiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 184 op/s; 822/128724 objects degraded (0.639%); 7020/128724 objects misplaced (5.454%); 7.6 MiB/s, 190 objects/s recovering 2023-09-23T20:55:13.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:55:13 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "11"}]': finished 2023-09-23T20:55:13.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:55:13 smithi070 ceph-mon[168282]: osdmap e153: 8 total, 8 up, 8 in 2023-09-23T20:55:13.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:55:13 smithi070 ceph-mon[168282]: failed to encode map e153 with expected crc 2023-09-23T20:55:13.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:55:13 smithi070 ceph-mon[168282]: failed to encode map e153 with expected crc 2023-09-23T20:55:13.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:55:13 smithi070 ceph-mon[168282]: failed to encode map e153 with expected crc 2023-09-23T20:55:13.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:55:13 smithi070 ceph-mon[168282]: failed to encode map e153 with expected crc 2023-09-23T20:55:13.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:55:13 smithi070 ceph-mon[168282]: failed to encode map e153 with expected crc 2023-09-23T20:55:13.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:55:13 smithi070 ceph-mon[168282]: failed to encode map e153 with expected crc 2023-09-23T20:55:13.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:55:13 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/3617714484' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:55:13.635 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:55:13 smithi026 ceph-mon[175151]: pgmap v374: 99 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+degraded+remapped, 92 active+clean, 1 active+recovery_wait, 4 active+remapped+backfill_wait; 2.6 GiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 184 op/s; 822/128724 objects degraded (0.639%); 7020/128724 objects misplaced (5.454%); 7.6 MiB/s, 190 objects/s recovering 2023-09-23T20:55:13.635 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:55:13 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "11"}]': finished 2023-09-23T20:55:13.635 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:55:13 smithi026 ceph-mon[175151]: osdmap e153: 8 total, 8 up, 8 in 2023-09-23T20:55:13.635 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:55:13 smithi026 ceph-mon[175151]: failed to encode map e153 with expected crc 2023-09-23T20:55:13.635 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:55:13 smithi026 ceph-mon[175151]: failed to encode map e153 with expected crc 2023-09-23T20:55:13.635 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:55:13 smithi026 ceph-mon[175151]: failed to encode map e153 with expected crc 2023-09-23T20:55:13.635 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:55:13 smithi026 ceph-mon[175151]: failed to encode map e153 with expected crc 2023-09-23T20:55:13.635 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:55:13 smithi026 ceph-mon[175151]: failed to encode map e153 with expected crc 2023-09-23T20:55:13.635 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:55:13 smithi026 ceph-mon[175151]: failed to encode map e153 with expected crc 2023-09-23T20:55:13.635 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:55:13 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/3617714484' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:55:13.636 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:55:13 smithi026 ceph-mon[179688]: pgmap v374: 99 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+degraded+remapped, 92 active+clean, 1 active+recovery_wait, 4 active+remapped+backfill_wait; 2.6 GiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 184 op/s; 822/128724 objects degraded (0.639%); 7020/128724 objects misplaced (5.454%); 7.6 MiB/s, 190 objects/s recovering 2023-09-23T20:55:13.636 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:55:13 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "11"}]': finished 2023-09-23T20:55:13.636 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:55:13 smithi026 ceph-mon[179688]: osdmap e153: 8 total, 8 up, 8 in 2023-09-23T20:55:13.636 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:55:13 smithi026 ceph-mon[179688]: failed to encode map e153 with expected crc 2023-09-23T20:55:13.636 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:55:13 smithi026 ceph-mon[179688]: failed to encode map e153 with expected crc 2023-09-23T20:55:13.636 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:55:13 smithi026 ceph-mon[179688]: failed to encode map e153 with expected crc 2023-09-23T20:55:13.636 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:55:13 smithi026 ceph-mon[179688]: failed to encode map e153 with expected crc 2023-09-23T20:55:13.636 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:55:13 smithi026 ceph-mon[179688]: failed to encode map e153 with expected crc 2023-09-23T20:55:13.636 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:55:13 smithi026 ceph-mon[179688]: failed to encode map e153 with expected crc 2023-09-23T20:55:13.636 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:55:13 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/3617714484' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:55:13.933 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:55:13.937 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:55:13.938 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:55:14.038 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:55:14.087 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:55:14.188 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:55:14.192 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:55:14.192 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:55:14.293 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:55:14.357 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:55:14.415 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:55:14.421 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:55:14.426 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:55:14.427 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:55:14.527 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:55:14.552 INFO:teuthology.orchestra.run.smithi070.stdout:ERROR: (22) Invalid argument 2023-09-23T20:55:14.552 INFO:teuthology.orchestra.run.smithi070.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:55:14.591 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:55:14.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:55:14 smithi070 ceph-mon[168282]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:55:14.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:55:14 smithi070 ceph-mon[168282]: failed to encode map e154 with expected crc 2023-09-23T20:55:14.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:55:14 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/3617714484' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T20:55:14.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:55:14 smithi070 ceph-mon[168282]: osdmap e154: 8 total, 8 up, 8 in 2023-09-23T20:55:14.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:55:14 smithi070 ceph-mon[168282]: failed to encode map e154 with expected crc 2023-09-23T20:55:14.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:55:14 smithi070 ceph-mon[168282]: failed to encode map e154 with expected crc 2023-09-23T20:55:14.676 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:55:14.684 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:55:14.684 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:55:14.696 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:55:14 smithi026 ceph-mon[179688]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:55:14.697 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:55:14 smithi026 ceph-mon[179688]: failed to encode map e154 with expected crc 2023-09-23T20:55:14.697 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:55:14 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/3617714484' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T20:55:14.697 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:55:14 smithi026 ceph-mon[179688]: osdmap e154: 8 total, 8 up, 8 in 2023-09-23T20:55:14.697 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:55:14 smithi026 ceph-mon[179688]: failed to encode map e154 with expected crc 2023-09-23T20:55:14.697 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:55:14 smithi026 ceph-mon[179688]: failed to encode map e154 with expected crc 2023-09-23T20:55:14.701 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:55:14 smithi026 ceph-mon[175151]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:55:14.701 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:55:14 smithi026 ceph-mon[175151]: failed to encode map e154 with expected crc 2023-09-23T20:55:14.701 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:55:14 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/3617714484' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T20:55:14.701 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:55:14 smithi026 ceph-mon[175151]: osdmap e154: 8 total, 8 up, 8 in 2023-09-23T20:55:14.702 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:55:14 smithi026 ceph-mon[175151]: failed to encode map e154 with expected crc 2023-09-23T20:55:14.702 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:55:14 smithi026 ceph-mon[175151]: failed to encode map e154 with expected crc 2023-09-23T20:55:14.785 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:55:14.857 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:55:14.926 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_blocked_ops 2023-09-23T20:55:14.928 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:55:14.941 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:55:14.942 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:55:15.043 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:55:15.116 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:55:15.252 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:55:15.258 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:55:15.259 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:55:15.359 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:55:15.425 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:55:15.555 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:55:15.570 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:55:15.570 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:55:15 smithi026 ceph-mon[175151]: pgmap v377: 109 pgs: 10 unknown, 1 active+recovering+remapped, 92 active+clean, 1 active+recovery_wait, 5 active+remapped+backfill_wait; 2.5 GiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 116 op/s; 7041/123951 objects misplaced (5.680%); 5.2 MiB/s, 170 objects/s recovering 2023-09-23T20:55:15.570 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:55:15 smithi026 ceph-mon[175151]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 822/128724 objects degraded (0.639%), 1 pg degraded) 2023-09-23T20:55:15.570 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:55:15 smithi026 ceph-mon[179688]: pgmap v377: 109 pgs: 10 unknown, 1 active+recovering+remapped, 92 active+clean, 1 active+recovery_wait, 5 active+remapped+backfill_wait; 2.5 GiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 116 op/s; 7041/123951 objects misplaced (5.680%); 5.2 MiB/s, 170 objects/s recovering 2023-09-23T20:55:15.570 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:55:15 smithi026 ceph-mon[179688]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 822/128724 objects degraded (0.639%), 1 pg degraded) 2023-09-23T20:55:15.571 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:55:15.648 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:55:15.672 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:55:15.766 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:55:15.848 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:55:15.852 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:55:15.853 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:55:15.871 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:55:15 smithi070 ceph-mon[168282]: pgmap v377: 109 pgs: 10 unknown, 1 active+recovering+remapped, 92 active+clean, 1 active+recovery_wait, 5 active+remapped+backfill_wait; 2.5 GiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 116 op/s; 7041/123951 objects misplaced (5.680%); 5.2 MiB/s, 170 objects/s recovering 2023-09-23T20:55:15.871 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:55:15 smithi070 ceph-mon[168282]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 822/128724 objects degraded (0.639%), 1 pg degraded) 2023-09-23T20:55:15.953 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:55:16.021 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0: {} 2023-09-23T20:55:16.036 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:55:16.067 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1: {} 2023-09-23T20:55:16.086 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T20:55:16.108 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2: {} 2023-09-23T20:55:16.116 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 6, 7, 4, 5] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-09-23T20:55:16.117 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T20:55:16.117 INFO:tasks.thrashosds.thrasher:Killing osd 6, live_osds are [0, 1, 2, 3, 4, 5, 6, 7] 2023-09-23T20:55:16.117 INFO:tasks.cephadm.osd.6:Stopping osd.6... 2023-09-23T20:55:16.117 DEBUG:teuthology.orchestra.run.smithi070:> sudo systemctl stop ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7@osd.6 2023-09-23T20:55:16.118 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:55:16.127 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:55:16.128 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:55:16.150 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3: {} 2023-09-23T20:55:16.199 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4: {} 2023-09-23T20:55:16.228 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:55:16.260 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T20:55:16.313 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6: {} 2023-09-23T20:55:16.335 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:55:16.341 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T20:55:16.374 INFO:teuthology.orchestra.run.smithi026.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T20:55:16.374 INFO:teuthology.orchestra.run.smithi026.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T20:55:16.374 INFO:teuthology.orchestra.run.smithi026.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T20:55:16.375 INFO:teuthology.orchestra.run.smithi026.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T20:55:16.375 INFO:teuthology.orchestra.run.smithi026.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T20:55:16.375 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T20:55:16.375 INFO:teuthology.orchestra.run.smithi026.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T20:55:16.375 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T20:55:16.423 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:55:16.430 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:55:16.431 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:55:16.442 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:55:16 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/336502975' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:55:16.442 INFO:journalctl@ceph.osd.6.smithi070.stdout:Sep 23 20:55:16 smithi070 systemd[1]: Stopping Ceph osd.6 for b3f8e4ec-5a50-11ee-8db4-212e2dc638e7... 2023-09-23T20:55:16.531 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:55:16.609 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:55:16.737 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:55:16.737 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:55:16 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/336502975' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:55:16.737 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:55:16 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:55:16.738 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:55:16 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:55:16.738 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:55:16 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/336502975' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:55:16.738 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:55:16 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:55:16.738 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:55:16 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:55:16.739 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:55:16.739 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:55:16.840 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:55:16.867 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:55:16 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:55:16.867 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:55:16 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:55:16.867 INFO:journalctl@ceph.osd.6.smithi070.stdout:Sep 23 20:55:16 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd-6[121137]: 2023-09-23T20:55:16.678+0000 7ffa9e95c700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-osd -n osd.6 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2023-09-23T20:55:16.867 INFO:journalctl@ceph.osd.6.smithi070.stdout:Sep 23 20:55:16 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd-6[121137]: 2023-09-23T20:55:16.678+0000 7ffa9e95c700 -1 osd.6 154 *** Got signal Terminated *** 2023-09-23T20:55:16.867 INFO:journalctl@ceph.osd.6.smithi070.stdout:Sep 23 20:55:16 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd-6[121137]: 2023-09-23T20:55:16.678+0000 7ffa9e95c700 -1 osd.6 154 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-09-23T20:55:16.913 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:55:16.987 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:55:16.993 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:55:16.993 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:55:17.094 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:55:17.127 INFO:teuthology.orchestra.run.smithi070.stdout:{ 2023-09-23T20:55:17.127 INFO:teuthology.orchestra.run.smithi070.stdout: "ops": [], 2023-09-23T20:55:17.127 INFO:teuthology.orchestra.run.smithi070.stdout: "complaint_time": 30, 2023-09-23T20:55:17.127 INFO:teuthology.orchestra.run.smithi070.stdout: "num_blocked_ops": 0 2023-09-23T20:55:17.127 INFO:teuthology.orchestra.run.smithi070.stdout:} 2023-09-23T20:55:17.172 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:55:17.180 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:55:17.337 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:55:17.340 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:55:17.341 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:55:17.442 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:55:17.503 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:55:17.535 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_historic_ops 2023-09-23T20:55:17.541 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:55:17.544 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:55:17.544 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:55:17.631 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:55:17 smithi026 ceph-mon[179688]: pgmap v378: 109 pgs: 10 unknown, 1 active+recovering+remapped, 92 active+clean, 1 active+recovery_wait, 5 active+remapped+backfill_wait; 2.5 GiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 7039/121413 objects misplaced (5.798%); 2.4 MiB/s, 124 objects/s recovering 2023-09-23T20:55:17.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:55:17 smithi026 ceph-mon[179688]: osd.6 marked itself down and dead 2023-09-23T20:55:17.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:55:17 smithi026 ceph-mon[175151]: pgmap v378: 109 pgs: 10 unknown, 1 active+recovering+remapped, 92 active+clean, 1 active+recovery_wait, 5 active+remapped+backfill_wait; 2.5 GiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 7039/121413 objects misplaced (5.798%); 2.4 MiB/s, 124 objects/s recovering 2023-09-23T20:55:17.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:55:17 smithi026 ceph-mon[175151]: osd.6 marked itself down and dead 2023-09-23T20:55:17.644 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:55:17.671 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:55:17.706 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:55:17.707 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:55:17 smithi070 ceph-mon[168282]: pgmap v378: 109 pgs: 10 unknown, 1 active+recovering+remapped, 92 active+clean, 1 active+recovery_wait, 5 active+remapped+backfill_wait; 2.5 GiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 7039/121413 objects misplaced (5.798%); 2.4 MiB/s, 124 objects/s recovering 2023-09-23T20:55:17.708 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:55:17 smithi070 ceph-mon[168282]: osd.6 marked itself down and dead 2023-09-23T20:55:17.708 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:55:17.708 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:55:17.809 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:55:17.836 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:55:17.869 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:55:17.875 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:55:17.876 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:55:17.976 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:55:18.006 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:55:18.098 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:55:18.100 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:55:18.101 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:55:18.202 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:55:18.229 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:55:18.252 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:55:18.369 INFO:journalctl@ceph.osd.6.smithi070.stdout:Sep 23 20:55:18 smithi070 bash[191850]: ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd-6 2023-09-23T20:55:18.389 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:55:18.394 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:55:18.395 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:55:18.495 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:55:18.582 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:55:18.671 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:55:18.672 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:55:18.672 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:55:18.772 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:55:18.801 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:55:18 smithi070 ceph-mon[168282]: Health check failed: 1 osds down (OSD_DOWN) 2023-09-23T20:55:18.801 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:55:18 smithi070 ceph-mon[168282]: failed to encode map e155 with expected crc 2023-09-23T20:55:18.801 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:55:18 smithi070 ceph-mon[168282]: failed to encode map e155 with expected crc 2023-09-23T20:55:18.801 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:55:18 smithi070 ceph-mon[168282]: osdmap e155: 8 total, 7 up, 8 in 2023-09-23T20:55:18.814 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:55:18.847 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T20:55:18.881 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:55:18 smithi026 ceph-mon[179688]: Health check failed: 1 osds down (OSD_DOWN) 2023-09-23T20:55:18.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:55:18 smithi026 ceph-mon[179688]: failed to encode map e155 with expected crc 2023-09-23T20:55:18.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:55:18 smithi026 ceph-mon[179688]: failed to encode map e155 with expected crc 2023-09-23T20:55:18.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:55:18 smithi026 ceph-mon[179688]: osdmap e155: 8 total, 7 up, 8 in 2023-09-23T20:55:18.883 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:55:18 smithi026 ceph-mon[175151]: Health check failed: 1 osds down (OSD_DOWN) 2023-09-23T20:55:18.883 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:55:18 smithi026 ceph-mon[175151]: failed to encode map e155 with expected crc 2023-09-23T20:55:18.884 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:55:18 smithi026 ceph-mon[175151]: failed to encode map e155 with expected crc 2023-09-23T20:55:18.884 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:55:18 smithi026 ceph-mon[175151]: osdmap e155: 8 total, 7 up, 8 in 2023-09-23T20:55:19.400 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:55:19.405 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:55:19.405 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:55:19.488 INFO:teuthology.orchestra.run.smithi026.stderr:nodeep-scrub is set 2023-09-23T20:55:19.506 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:55:19.566 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:55:19.628 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:55:19.631 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:55:19.632 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:55:19.680 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:55:19.732 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:55:19.792 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:55:19 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[164029]: ::ffff:172.21.15.70 - - [23/Sep/2023:20:55:19] "GET /metrics HTTP/1.1" 200 36342 "" "Prometheus/2.43.0" 2023-09-23T20:55:19.792 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:55:19 smithi026 ceph-mon[179688]: pgmap v380: 109 pgs: 6 stale+active+clean, 1 active+recovering+remapped, 96 active+clean, 1 active+recovery_wait, 5 active+remapped+backfill_wait; 2.4 GiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 725 op/s; 7039/115665 objects misplaced (6.086%); 949 KiB/s, 89 objects/s recovering 2023-09-23T20:55:19.792 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:55:19 smithi026 ceph-mon[179688]: failed to encode map e156 with expected crc 2023-09-23T20:55:19.792 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:55:19 smithi026 ceph-mon[179688]: failed to encode map e156 with expected crc 2023-09-23T20:55:19.792 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:55:19 smithi026 ceph-mon[179688]: failed to encode map e156 with expected crc 2023-09-23T20:55:19.792 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:55:19 smithi026 ceph-mon[179688]: osdmap e156: 8 total, 7 up, 8 in 2023-09-23T20:55:19.792 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:55:19 smithi026 ceph-mon[179688]: failed to encode map e156 with expected crc 2023-09-23T20:55:19.792 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:55:19 smithi026 ceph-mon[179688]: 6.14 continuing backfill to osd.5 from (107'600,154'8532] MIN to 154'8532 2023-09-23T20:55:19.793 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:55:19 smithi026 ceph-mon[179688]: 6.4 continuing backfill to osd.5 from (107'600,154'8523] MIN to 154'8523 2023-09-23T20:55:19.793 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:55:19 smithi026 ceph-mon[179688]: failed to encode map e156 with expected crc 2023-09-23T20:55:19.793 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:55:19 smithi026 ceph-mon[179688]: 6.a continuing backfill to osd.5 from (113'3177,154'10393] 6:5fa81015:::benchmark_data_smithi026_223868_object87440:head to 154'10393 2023-09-23T20:55:19.793 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:55:19 smithi026 ceph-mon[179688]: failed to encode map e156 with expected crc 2023-09-23T20:55:19.793 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:55:19 smithi026 ceph-mon[179688]: 6.c continuing backfill to osd.5 from (116'4463,154'10239] MIN to 154'10239 2023-09-23T20:55:19.793 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:55:19 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/4017684694' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:55:19.794 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:55:19 smithi026 ceph-mon[175151]: pgmap v380: 109 pgs: 6 stale+active+clean, 1 active+recovering+remapped, 96 active+clean, 1 active+recovery_wait, 5 active+remapped+backfill_wait; 2.4 GiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 725 op/s; 7039/115665 objects misplaced (6.086%); 949 KiB/s, 89 objects/s recovering 2023-09-23T20:55:19.794 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:55:19 smithi026 ceph-mon[175151]: failed to encode map e156 with expected crc 2023-09-23T20:55:19.794 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:55:19 smithi026 ceph-mon[175151]: failed to encode map e156 with expected crc 2023-09-23T20:55:19.794 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:55:19 smithi026 ceph-mon[175151]: failed to encode map e156 with expected crc 2023-09-23T20:55:19.794 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:55:19 smithi026 ceph-mon[175151]: osdmap e156: 8 total, 7 up, 8 in 2023-09-23T20:55:19.794 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:55:19 smithi026 ceph-mon[175151]: failed to encode map e156 with expected crc 2023-09-23T20:55:19.794 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:55:19 smithi026 ceph-mon[175151]: 6.14 continuing backfill to osd.5 from (107'600,154'8532] MIN to 154'8532 2023-09-23T20:55:19.795 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:55:19 smithi026 ceph-mon[175151]: 6.4 continuing backfill to osd.5 from (107'600,154'8523] MIN to 154'8523 2023-09-23T20:55:19.795 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:55:19 smithi026 ceph-mon[175151]: failed to encode map e156 with expected crc 2023-09-23T20:55:19.795 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:55:19 smithi026 ceph-mon[175151]: 6.a continuing backfill to osd.5 from (113'3177,154'10393] 6:5fa81015:::benchmark_data_smithi026_223868_object87440:head to 154'10393 2023-09-23T20:55:19.795 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:55:19 smithi026 ceph-mon[175151]: failed to encode map e156 with expected crc 2023-09-23T20:55:19.795 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:55:19 smithi026 ceph-mon[175151]: 6.c continuing backfill to osd.5 from (116'4463,154'10239] MIN to 154'10239 2023-09-23T20:55:19.795 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:55:19 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/4017684694' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:55:19.797 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:55:19.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:55:19 smithi070 ceph-mon[168282]: pgmap v380: 109 pgs: 6 stale+active+clean, 1 active+recovering+remapped, 96 active+clean, 1 active+recovery_wait, 5 active+remapped+backfill_wait; 2.4 GiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 725 op/s; 7039/115665 objects misplaced (6.086%); 949 KiB/s, 89 objects/s recovering 2023-09-23T20:55:19.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:55:19 smithi070 ceph-mon[168282]: failed to encode map e156 with expected crc 2023-09-23T20:55:19.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:55:19 smithi070 ceph-mon[168282]: failed to encode map e156 with expected crc 2023-09-23T20:55:19.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:55:19 smithi070 ceph-mon[168282]: failed to encode map e156 with expected crc 2023-09-23T20:55:19.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:55:19 smithi070 ceph-mon[168282]: osdmap e156: 8 total, 7 up, 8 in 2023-09-23T20:55:19.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:55:19 smithi070 ceph-mon[168282]: failed to encode map e156 with expected crc 2023-09-23T20:55:19.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:55:19 smithi070 ceph-mon[168282]: 6.14 continuing backfill to osd.5 from (107'600,154'8532] MIN to 154'8532 2023-09-23T20:55:19.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:55:19 smithi070 ceph-mon[168282]: 6.4 continuing backfill to osd.5 from (107'600,154'8523] MIN to 154'8523 2023-09-23T20:55:19.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:55:19 smithi070 ceph-mon[168282]: failed to encode map e156 with expected crc 2023-09-23T20:55:19.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:55:19 smithi070 ceph-mon[168282]: 6.a continuing backfill to osd.5 from (113'3177,154'10393] 6:5fa81015:::benchmark_data_smithi026_223868_object87440:head to 154'10393 2023-09-23T20:55:19.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:55:19 smithi070 ceph-mon[168282]: failed to encode map e156 with expected crc 2023-09-23T20:55:19.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:55:19 smithi070 ceph-mon[168282]: 6.c continuing backfill to osd.5 from (116'4463,154'10239] MIN to 154'10239 2023-09-23T20:55:19.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:55:19 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/4017684694' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:55:20.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:55:20 smithi070 ceph-mon[168282]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:55:20.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:55:20 smithi070 ceph-mon[168282]: failed to encode map e157 with expected crc 2023-09-23T20:55:20.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:55:20 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/4017684694' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T20:55:20.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:55:20 smithi070 ceph-mon[168282]: osdmap e157: 8 total, 7 up, 8 in 2023-09-23T20:55:20.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:55:20 smithi026 ceph-mon[175151]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:55:20.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:55:20 smithi026 ceph-mon[175151]: failed to encode map e157 with expected crc 2023-09-23T20:55:20.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:55:20 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/4017684694' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T20:55:20.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:55:20 smithi026 ceph-mon[175151]: osdmap e157: 8 total, 7 up, 8 in 2023-09-23T20:55:20.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:55:20 smithi026 ceph-mon[179688]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:55:20.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:55:20 smithi026 ceph-mon[179688]: failed to encode map e157 with expected crc 2023-09-23T20:55:20.883 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:55:20 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/4017684694' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T20:55:20.883 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:55:20 smithi026 ceph-mon[179688]: osdmap e157: 8 total, 7 up, 8 in 2023-09-23T20:55:20.892 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:55:20.899 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:55:20.900 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:55:21.000 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:55:21.052 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:55:21.202 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:55:21.206 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:55:21.207 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:55:21.307 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:55:21.336 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:55:21.544 INFO:teuthology.orchestra.run.smithi070.stderr:admin_socket: exception getting command descriptions: [Errno 111] Connection refused 2023-09-23T20:55:21.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:55:21 smithi070 ceph-mon[168282]: pgmap v383: 109 pgs: 1 active+recovering+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 8 active+undersized, 5 stale+active+clean, 88 active+clean, 2 active+undersized+degraded, 2 active+remapped+backfill_wait; 2.3 GiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 669 op/s; 6850/111903 objects degraded (6.121%); 3551/111903 objects misplaced (3.173%); 3.7 MiB/s, 117 objects/s recovering 2023-09-23T20:55:21.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:55:21 smithi070 ceph-mon[168282]: Health check failed: Degraded data redundancy: 6850/111903 objects degraded (6.121%), 5 pgs degraded (PG_DEGRADED) 2023-09-23T20:55:21.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:55:21 smithi070 ceph-mon[168282]: failed to encode map e158 with expected crc 2023-09-23T20:55:21.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:55:21 smithi070 ceph-mon[168282]: osdmap e158: 8 total, 7 up, 8 in 2023-09-23T20:55:21.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:55:21 smithi070 ceph-mon[168282]: failed to encode map e158 with expected crc 2023-09-23T20:55:21.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:55:21 smithi070 ceph-mon[168282]: failed to encode map e158 with expected crc 2023-09-23T20:55:21.751 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:55:21 smithi026 ceph-mon[175151]: pgmap v383: 109 pgs: 1 active+recovering+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 8 active+undersized, 5 stale+active+clean, 88 active+clean, 2 active+undersized+degraded, 2 active+remapped+backfill_wait; 2.3 GiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 669 op/s; 6850/111903 objects degraded (6.121%); 3551/111903 objects misplaced (3.173%); 3.7 MiB/s, 117 objects/s recovering 2023-09-23T20:55:21.752 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:55:21 smithi026 ceph-mon[175151]: Health check failed: Degraded data redundancy: 6850/111903 objects degraded (6.121%), 5 pgs degraded (PG_DEGRADED) 2023-09-23T20:55:21.752 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:55:21 smithi026 ceph-mon[175151]: failed to encode map e158 with expected crc 2023-09-23T20:55:21.752 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:55:21 smithi026 ceph-mon[175151]: osdmap e158: 8 total, 7 up, 8 in 2023-09-23T20:55:21.752 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:55:21 smithi026 ceph-mon[175151]: failed to encode map e158 with expected crc 2023-09-23T20:55:21.752 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:55:21 smithi026 ceph-mon[175151]: failed to encode map e158 with expected crc 2023-09-23T20:55:21.752 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:55:21 smithi026 ceph-mon[179688]: pgmap v383: 109 pgs: 1 active+recovering+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 8 active+undersized, 5 stale+active+clean, 88 active+clean, 2 active+undersized+degraded, 2 active+remapped+backfill_wait; 2.3 GiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 669 op/s; 6850/111903 objects degraded (6.121%); 3551/111903 objects misplaced (3.173%); 3.7 MiB/s, 117 objects/s recovering 2023-09-23T20:55:21.752 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:55:21 smithi026 ceph-mon[179688]: Health check failed: Degraded data redundancy: 6850/111903 objects degraded (6.121%), 5 pgs degraded (PG_DEGRADED) 2023-09-23T20:55:21.752 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:55:21 smithi026 ceph-mon[179688]: failed to encode map e158 with expected crc 2023-09-23T20:55:21.752 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:55:21 smithi026 ceph-mon[179688]: osdmap e158: 8 total, 7 up, 8 in 2023-09-23T20:55:21.753 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:55:21 smithi026 ceph-mon[179688]: failed to encode map e158 with expected crc 2023-09-23T20:55:21.753 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:55:21 smithi026 ceph-mon[179688]: failed to encode map e158 with expected crc 2023-09-23T20:55:21.930 DEBUG:teuthology.orchestra.run.smithi070:> sudo pkill -f 'journalctl -f -n 0 -u ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7@osd.6.service' 2023-09-23T20:55:22.004 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-09-23T20:55:22.004 INFO:tasks.cephadm.osd.6:Stopped osd.6 2023-09-23T20:55:22.083 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:55:22.090 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:55:22.091 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:55:22.192 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:55:22.195 DEBUG:teuthology.orchestra.run:got remote process result: 22 2023-09-23T20:55:22.195 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_ops_in_flight 2023-09-23T20:55:22.228 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:55:22.277 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:55:22.279 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:55:22.280 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:55:22.380 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:55:22.487 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:55:22.716 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:55:22.723 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:55:22.724 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:55:22.825 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:55:22.857 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:55:22.898 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:55:22.901 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:55:22.902 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:55:23.003 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:55:23.032 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:55:23.077 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:55:23.080 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:55:23.080 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:55:23.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:55:22 smithi070 ceph-mon[168282]: failed to encode map e159 with expected crc 2023-09-23T20:55:23.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:55:22 smithi070 ceph-mon[168282]: failed to encode map e159 with expected crc 2023-09-23T20:55:23.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:55:22 smithi070 ceph-mon[168282]: osdmap e159: 8 total, 7 up, 8 in 2023-09-23T20:55:23.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:55:22 smithi070 ceph-mon[168282]: failed to encode map e159 with expected crc 2023-09-23T20:55:23.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:55:22 smithi070 ceph-mon[168282]: failed to encode map e159 with expected crc 2023-09-23T20:55:23.135 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:55:22 smithi026 ceph-mon[175151]: failed to encode map e159 with expected crc 2023-09-23T20:55:23.135 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:55:22 smithi026 ceph-mon[175151]: failed to encode map e159 with expected crc 2023-09-23T20:55:23.135 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:55:22 smithi026 ceph-mon[175151]: osdmap e159: 8 total, 7 up, 8 in 2023-09-23T20:55:23.136 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:55:22 smithi026 ceph-mon[175151]: failed to encode map e159 with expected crc 2023-09-23T20:55:23.136 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:55:22 smithi026 ceph-mon[175151]: failed to encode map e159 with expected crc 2023-09-23T20:55:23.136 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:55:22 smithi026 ceph-mon[179688]: failed to encode map e159 with expected crc 2023-09-23T20:55:23.136 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:55:22 smithi026 ceph-mon[179688]: failed to encode map e159 with expected crc 2023-09-23T20:55:23.136 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:55:22 smithi026 ceph-mon[179688]: osdmap e159: 8 total, 7 up, 8 in 2023-09-23T20:55:23.136 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:55:22 smithi026 ceph-mon[179688]: failed to encode map e159 with expected crc 2023-09-23T20:55:23.136 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:55:22 smithi026 ceph-mon[179688]: failed to encode map e159 with expected crc 2023-09-23T20:55:23.181 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:55:23.213 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:55:23.227 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0: {} 2023-09-23T20:55:23.238 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:55:23.252 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:55:23.255 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:55:23.256 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:55:23.264 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1: {} 2023-09-23T20:55:23.309 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2: {} 2023-09-23T20:55:23.348 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3: {} 2023-09-23T20:55:23.357 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:55:23.385 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4: {} 2023-09-23T20:55:23.392 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:55:23.433 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T20:55:23.434 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:55:23.437 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:55:23.437 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:55:23.475 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T20:55:23.475 INFO:teuthology.orchestra.run.smithi026.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T20:55:23.475 INFO:teuthology.orchestra.run.smithi026.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T20:55:23.476 INFO:teuthology.orchestra.run.smithi026.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T20:55:23.476 INFO:teuthology.orchestra.run.smithi026.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T20:55:23.476 INFO:teuthology.orchestra.run.smithi026.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T20:55:23.476 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T20:55:23.476 INFO:teuthology.orchestra.run.smithi026.stderr:Error ENXIO: problem getting command descriptions from osd.6 2023-09-23T20:55:23.476 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T20:55:23.538 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:55:23.645 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:55:23.977 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:55:23.984 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:55:23.984 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:55:24.044 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T20:55:24.045 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T20:55:24.085 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:55:24.138 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:55:23 smithi026 ceph-mon[175151]: pgmap v386: 109 pgs: 3 active+clean+remapped, 1 active+recovering+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 31 active+undersized, 59 active+clean, 11 active+undersized+degraded, 1 active+remapped+backfill_wait; 2.2 GiB data, 7.9 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 797 op/s; 11017/106098 objects degraded (10.384%); 5671/106098 objects misplaced (5.345%); 6.3 MiB/s, 185 objects/s recovering 2023-09-23T20:55:24.138 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:55:23 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/3458469278' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:55:24.138 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:55:23 smithi026 ceph-mon[179688]: pgmap v386: 109 pgs: 3 active+clean+remapped, 1 active+recovering+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 31 active+undersized, 59 active+clean, 11 active+undersized+degraded, 1 active+remapped+backfill_wait; 2.2 GiB data, 7.9 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 797 op/s; 11017/106098 objects degraded (10.384%); 5671/106098 objects misplaced (5.345%); 6.3 MiB/s, 185 objects/s recovering 2023-09-23T20:55:24.138 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:55:23 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/3458469278' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:55:24.139 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T20:55:24.144 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:55:24.276 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:55:24.283 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:55:24.283 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:55:24.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:55:23 smithi070 ceph-mon[168282]: pgmap v386: 109 pgs: 3 active+clean+remapped, 1 active+recovering+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 31 active+undersized, 59 active+clean, 11 active+undersized+degraded, 1 active+remapped+backfill_wait; 2.2 GiB data, 7.9 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 797 op/s; 11017/106098 objects degraded (10.384%); 5671/106098 objects misplaced (5.345%); 6.3 MiB/s, 185 objects/s recovering 2023-09-23T20:55:24.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:55:23 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/3458469278' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:55:24.384 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:55:24.414 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:55:24.452 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:55:24.455 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:55:24.455 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:55:24.556 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:55:24.586 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:55:24.625 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:55:24.630 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:55:24.631 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:55:24.731 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:55:24.759 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:55:24.795 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:55:24.799 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:55:24.799 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:55:24.900 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:55:24.947 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:55:25.003 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:55:25.013 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:55:25.014 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:55:25.018 INFO:teuthology.orchestra.run.smithi026.stdout:ERROR: (22) Invalid argument 2023-09-23T20:55:25.018 INFO:teuthology.orchestra.run.smithi026.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:55:25.052 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:55:25.115 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:55:25.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:55:24 smithi070 ceph-mon[168282]: failed to encode map e160 with expected crc 2023-09-23T20:55:25.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:55:24 smithi070 ceph-mon[168282]: osdmap e160: 8 total, 7 up, 8 in 2023-09-23T20:55:25.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:55:24 smithi070 ceph-mon[168282]: failed to encode map e160 with expected crc 2023-09-23T20:55:25.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:55:24 smithi070 ceph-mon[168282]: failed to encode map e160 with expected crc 2023-09-23T20:55:25.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:55:24 smithi070 ceph-mon[168282]: failed to encode map e160 with expected crc 2023-09-23T20:55:25.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:55:24 smithi070 ceph-mon[168282]: osdmap e161: 8 total, 7 up, 8 in 2023-09-23T20:55:25.195 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:55:24 smithi026 ceph-mon[175151]: failed to encode map e160 with expected crc 2023-09-23T20:55:25.195 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:55:24 smithi026 ceph-mon[175151]: osdmap e160: 8 total, 7 up, 8 in 2023-09-23T20:55:25.195 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:55:24 smithi026 ceph-mon[175151]: failed to encode map e160 with expected crc 2023-09-23T20:55:25.195 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:55:24 smithi026 ceph-mon[175151]: failed to encode map e160 with expected crc 2023-09-23T20:55:25.195 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:55:24 smithi026 ceph-mon[175151]: failed to encode map e160 with expected crc 2023-09-23T20:55:25.195 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:55:24 smithi026 ceph-mon[175151]: osdmap e161: 8 total, 7 up, 8 in 2023-09-23T20:55:25.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:55:24 smithi026 ceph-mon[179688]: failed to encode map e160 with expected crc 2023-09-23T20:55:25.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:55:24 smithi026 ceph-mon[179688]: osdmap e160: 8 total, 7 up, 8 in 2023-09-23T20:55:25.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:55:24 smithi026 ceph-mon[179688]: failed to encode map e160 with expected crc 2023-09-23T20:55:25.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:55:24 smithi026 ceph-mon[179688]: failed to encode map e160 with expected crc 2023-09-23T20:55:25.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:55:24 smithi026 ceph-mon[179688]: failed to encode map e160 with expected crc 2023-09-23T20:55:25.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:55:24 smithi026 ceph-mon[179688]: osdmap e161: 8 total, 7 up, 8 in 2023-09-23T20:55:25.215 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:55:25.548 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:55:25.560 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:55:25.560 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:55:25.661 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:55:25.674 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_blocked_ops 2023-09-23T20:55:25.697 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:55:25.744 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:55:25.748 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:55:25.749 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:55:25.849 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:55:25.879 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:55:25.919 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:55:25.922 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:55:25.922 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:55:26.023 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:55:26.046 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T20:55:26.114 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:55:26.182 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:55:26.187 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:55:26.188 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:55:26.190 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:55:25 smithi026 ceph-mon[179688]: pgmap v388: 109 pgs: 1 active+undersized+degraded+remapped+backfilling, 1 peering, 3 active+clean+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 31 active+undersized, 59 active+clean, 11 active+undersized+degraded, 2 active+remapped+backfill_wait; 2.1 GiB data, 7.7 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 662 op/s; 9137/102351 objects degraded (8.927%); 6344/102351 objects misplaced (6.198%); 17 MiB/s, 296 objects/s recovering 2023-09-23T20:55:26.190 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:55:25 smithi026 ceph-mon[179688]: failed to encode map e161 with expected crc 2023-09-23T20:55:26.190 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:55:25 smithi026 ceph-mon[179688]: failed to encode map e161 with expected crc 2023-09-23T20:55:26.190 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:55:25 smithi026 ceph-mon[179688]: failed to encode map e161 with expected crc 2023-09-23T20:55:26.191 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:55:25 smithi026 ceph-mon[175151]: pgmap v388: 109 pgs: 1 active+undersized+degraded+remapped+backfilling, 1 peering, 3 active+clean+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 31 active+undersized, 59 active+clean, 11 active+undersized+degraded, 2 active+remapped+backfill_wait; 2.1 GiB data, 7.7 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 662 op/s; 9137/102351 objects degraded (8.927%); 6344/102351 objects misplaced (6.198%); 17 MiB/s, 296 objects/s recovering 2023-09-23T20:55:26.191 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:55:25 smithi026 ceph-mon[175151]: failed to encode map e161 with expected crc 2023-09-23T20:55:26.191 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:55:25 smithi026 ceph-mon[175151]: failed to encode map e161 with expected crc 2023-09-23T20:55:26.191 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:55:25 smithi026 ceph-mon[175151]: failed to encode map e161 with expected crc 2023-09-23T20:55:26.288 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:55:26.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:55:25 smithi070 ceph-mon[168282]: pgmap v388: 109 pgs: 1 active+undersized+degraded+remapped+backfilling, 1 peering, 3 active+clean+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 31 active+undersized, 59 active+clean, 11 active+undersized+degraded, 2 active+remapped+backfill_wait; 2.1 GiB data, 7.7 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 662 op/s; 9137/102351 objects degraded (8.927%); 6344/102351 objects misplaced (6.198%); 17 MiB/s, 296 objects/s recovering 2023-09-23T20:55:26.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:55:25 smithi070 ceph-mon[168282]: failed to encode map e161 with expected crc 2023-09-23T20:55:26.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:55:25 smithi070 ceph-mon[168282]: failed to encode map e161 with expected crc 2023-09-23T20:55:26.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:55:25 smithi070 ceph-mon[168282]: failed to encode map e161 with expected crc 2023-09-23T20:55:26.445 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:55:26.555 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:55:26.562 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:55:26.562 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:55:26.663 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:55:26.785 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:55:26.860 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:55:26.869 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:55:26.869 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:55:26.970 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:55:26.982 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:55:27.005 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 6, 7, 4, 5] out_osds: [] dead_osds: [6] live_osds: [0, 1, 2, 3, 4, 5, 7] 2023-09-23T20:55:27.005 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T20:55:27.005 INFO:tasks.thrashosds.thrasher:Reviving osd 6 2023-09-23T20:55:27.005 INFO:tasks.cephadm.osd.6:Restarting osd.6 (starting--it wasn't running)... 2023-09-23T20:55:27.005 DEBUG:teuthology.orchestra.run.smithi070:osd.6> sudo journalctl -f -n 0 -u ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7@osd.6.service 2023-09-23T20:55:27.008 DEBUG:teuthology.orchestra.run.smithi070:> sudo systemctl start ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7@osd.6 2023-09-23T20:55:27.036 INFO:journalctl@ceph.osd.6.smithi070.stdout:-- Logs begin at Sat 2023-09-23 20:21:16 UTC. -- 2023-09-23T20:55:27.039 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:55:27 smithi026 ceph-mon[175151]: pgmap v390: 109 pgs: 1 active+undersized+degraded+remapped+backfilling, 1 peering, 3 active+clean+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 31 active+undersized, 59 active+clean, 11 active+undersized+degraded, 2 active+remapped+backfill_wait; 2.1 GiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 665 op/s; 9137/101193 objects degraded (9.029%); 6344/101193 objects misplaced (6.269%); 16 MiB/s, 265 objects/s recovering 2023-09-23T20:55:27.039 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:55:27 smithi026 ceph-mon[175151]: Health check update: Degraded data redundancy: 9137/101193 objects degraded (9.029%), 13 pgs degraded (PG_DEGRADED) 2023-09-23T20:55:27.050 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:55:27.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:55:27 smithi070 ceph-mon[168282]: pgmap v390: 109 pgs: 1 active+undersized+degraded+remapped+backfilling, 1 peering, 3 active+clean+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 31 active+undersized, 59 active+clean, 11 active+undersized+degraded, 2 active+remapped+backfill_wait; 2.1 GiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 665 op/s; 9137/101193 objects degraded (9.029%); 6344/101193 objects misplaced (6.269%); 16 MiB/s, 265 objects/s recovering 2023-09-23T20:55:27.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:55:27 smithi070 ceph-mon[168282]: Health check update: Degraded data redundancy: 9137/101193 objects degraded (9.029%), 13 pgs degraded (PG_DEGRADED) 2023-09-23T20:55:27.370 INFO:journalctl@ceph.osd.6.smithi070.stdout:Sep 23 20:55:27 smithi070 systemd[1]: Starting Ceph osd.6 for b3f8e4ec-5a50-11ee-8db4-212e2dc638e7... 2023-09-23T20:55:27.383 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:55:27 smithi026 ceph-mon[179688]: pgmap v390: 109 pgs: 1 active+undersized+degraded+remapped+backfilling, 1 peering, 3 active+clean+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 31 active+undersized, 59 active+clean, 11 active+undersized+degraded, 2 active+remapped+backfill_wait; 2.1 GiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 665 op/s; 9137/101193 objects degraded (9.029%); 6344/101193 objects misplaced (6.269%); 16 MiB/s, 265 objects/s recovering 2023-09-23T20:55:27.383 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:55:27 smithi026 ceph-mon[179688]: Health check update: Degraded data redundancy: 9137/101193 objects degraded (9.029%), 13 pgs degraded (PG_DEGRADED) 2023-09-23T20:55:27.938 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:55:27.941 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:55:27.952 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:55:27.952 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:55:28.053 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:55:28.075 INFO:teuthology.orchestra.run.smithi026.stderr:noscrub is unset 2023-09-23T20:55:28.122 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:55:28.321 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:55:28.329 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:55:28.330 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:55:28.333 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:55:28 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/1817837333' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:55:28.333 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:55:28 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:55:28.333 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:55:28 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/1817837333' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:55:28.334 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:55:28 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:55:28.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:55:28 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/1817837333' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:55:28.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:55:28 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:55:28.370 INFO:journalctl@ceph.osd.6.smithi070.stdout:Sep 23 20:55:28 smithi070 bash[192940]: --> Failed to activate via raw: did not find any matching OSD to activate 2023-09-23T20:55:28.370 INFO:journalctl@ceph.osd.6.smithi070.stdout:Sep 23 20:55:28 smithi070 bash[192940]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-6 2023-09-23T20:55:28.370 INFO:journalctl@ceph.osd.6.smithi070.stdout:Sep 23 20:55:28 smithi070 bash[192940]: Running command: /usr/bin/ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev /dev/vg_nvme/lv_2 --path /var/lib/ceph/osd/ceph-6 --no-mon-config 2023-09-23T20:55:28.370 INFO:journalctl@ceph.osd.6.smithi070.stdout:Sep 23 20:55:28 smithi070 bash[192940]: Running command: /usr/bin/ln -snf /dev/vg_nvme/lv_2 /var/lib/ceph/osd/ceph-6/block 2023-09-23T20:55:28.370 INFO:journalctl@ceph.osd.6.smithi070.stdout:Sep 23 20:55:28 smithi070 bash[192940]: Running command: /usr/bin/chown -h ceph:ceph /var/lib/ceph/osd/ceph-6/block 2023-09-23T20:55:28.370 INFO:journalctl@ceph.osd.6.smithi070.stdout:Sep 23 20:55:28 smithi070 bash[192940]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-1 2023-09-23T20:55:28.370 INFO:journalctl@ceph.osd.6.smithi070.stdout:Sep 23 20:55:28 smithi070 bash[192940]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-6 2023-09-23T20:55:28.370 INFO:journalctl@ceph.osd.6.smithi070.stdout:Sep 23 20:55:28 smithi070 bash[192940]: --> ceph-volume lvm activate successful for osd ID: 6 2023-09-23T20:55:28.431 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:55:28.527 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:55:28.774 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:55:28.783 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:55:28.783 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:55:28.884 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:55:28.934 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_ops_in_flight 2023-09-23T20:55:28.966 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:55:29.078 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:55:29 smithi070 ceph-mon[168282]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:55:29.078 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:55:29 smithi070 ceph-mon[168282]: failed to encode map e162 with expected crc 2023-09-23T20:55:29.079 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:55:29 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T20:55:29.079 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:55:29 smithi070 ceph-mon[168282]: osdmap e162: 8 total, 7 up, 8 in 2023-09-23T20:55:29.079 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:55:29 smithi070 ceph-mon[168282]: failed to encode map e162 with expected crc 2023-09-23T20:55:29.079 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:55:29 smithi070 ceph-mon[168282]: 6.0 scrub starts 2023-09-23T20:55:29.079 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:55:29 smithi070 ceph-mon[168282]: pgmap v392: 109 pgs: 1 active+undersized+degraded+remapped+backfilling, 1 peering, 3 active+clean+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 31 active+undersized, 59 active+clean, 11 active+undersized+degraded, 2 active+remapped+backfill_wait; 1.9 GiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 598 op/s; 8658/95313 objects degraded (9.084%); 6086/95313 objects misplaced (6.385%); 13 MiB/s, 232 objects/s recovering 2023-09-23T20:55:29.079 INFO:journalctl@ceph.osd.6.smithi070.stdout:Sep 23 20:55:28 smithi070 podman[193203]: 2023-09-23T20:55:29.079 INFO:journalctl@ceph.osd.6.smithi070.stdout:Sep 23 20:55:28 smithi070 ceph-osd[193244]: -- 172.21.15.70:0/196453683 <== mon.1 v2:172.21.15.70:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x559e3dadab60 con 0x559e3daa0c00 2023-09-23T20:55:29.079 INFO:journalctl@ceph.osd.6.smithi070.stdout:Sep 23 20:55:28 smithi070 bash[193203]: e0131cf73f50c7b1fba03f900a8b9eb5edfb79ab75796db6d981334e1736aea3 2023-09-23T20:55:29.079 INFO:journalctl@ceph.osd.6.smithi070.stdout:Sep 23 20:55:28 smithi070 systemd[1]: Started Ceph osd.6 for b3f8e4ec-5a50-11ee-8db4-212e2dc638e7. 2023-09-23T20:55:29.278 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:55:29.289 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:55:29.290 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:55:29.290 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:55:29 smithi026 ceph-mon[175151]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:55:29.290 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:55:29 smithi026 ceph-mon[175151]: failed to encode map e162 with expected crc 2023-09-23T20:55:29.290 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:55:29 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T20:55:29.290 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:55:29 smithi026 ceph-mon[175151]: osdmap e162: 8 total, 7 up, 8 in 2023-09-23T20:55:29.290 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:55:29 smithi026 ceph-mon[175151]: failed to encode map e162 with expected crc 2023-09-23T20:55:29.290 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:55:29 smithi026 ceph-mon[175151]: 6.0 scrub starts 2023-09-23T20:55:29.290 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:55:29 smithi026 ceph-mon[175151]: pgmap v392: 109 pgs: 1 active+undersized+degraded+remapped+backfilling, 1 peering, 3 active+clean+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 31 active+undersized, 59 active+clean, 11 active+undersized+degraded, 2 active+remapped+backfill_wait; 1.9 GiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 598 op/s; 8658/95313 objects degraded (9.084%); 6086/95313 objects misplaced (6.385%); 13 MiB/s, 232 objects/s recovering 2023-09-23T20:55:29.291 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:55:29 smithi026 ceph-mon[179688]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:55:29.291 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:55:29 smithi026 ceph-mon[179688]: failed to encode map e162 with expected crc 2023-09-23T20:55:29.291 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:55:29 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T20:55:29.291 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:55:29 smithi026 ceph-mon[179688]: osdmap e162: 8 total, 7 up, 8 in 2023-09-23T20:55:29.291 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:55:29 smithi026 ceph-mon[179688]: failed to encode map e162 with expected crc 2023-09-23T20:55:29.291 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:55:29 smithi026 ceph-mon[179688]: 6.0 scrub starts 2023-09-23T20:55:29.291 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:55:29 smithi026 ceph-mon[179688]: pgmap v392: 109 pgs: 1 active+undersized+degraded+remapped+backfilling, 1 peering, 3 active+clean+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 31 active+undersized, 59 active+clean, 11 active+undersized+degraded, 2 active+remapped+backfill_wait; 1.9 GiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 598 op/s; 8658/95313 objects degraded (9.084%); 6086/95313 objects misplaced (6.385%); 13 MiB/s, 232 objects/s recovering 2023-09-23T20:55:29.391 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:55:29.426 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:55:29.472 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:55:29.475 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:55:29.476 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:55:29.576 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:55:29.606 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:55:29.614 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:55:29 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[164029]: ::ffff:172.21.15.70 - - [23/Sep/2023:20:55:29] "GET /metrics HTTP/1.1" 200 36342 "" "Prometheus/2.43.0" 2023-09-23T20:55:29.633 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:55:29.658 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:55:29.662 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:55:29.663 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:55:29.763 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:55:29.796 INFO:journalctl@ceph.osd.6.smithi070.stdout:Sep 23 20:55:29 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd-6[193219]: 2023-09-23T20:55:29.440+0000 7f417cd5c540 -1 Falling back to public interface 2023-09-23T20:55:29.807 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:55:29.897 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:55:29.901 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:55:29.902 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:55:30.003 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:55:30.040 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:55:30.181 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:55:30.183 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:55:30.184 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:55:30.213 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0: {} 2023-09-23T20:55:30.275 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1: {} 2023-09-23T20:55:30.285 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:55:30.329 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2: {} 2023-09-23T20:55:30.393 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:55:30.393 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3: {} 2023-09-23T20:55:30.430 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4: {} 2023-09-23T20:55:30.469 INFO:teuthology.orchestra.run.smithi026.stdout:{ 2023-09-23T20:55:30.469 INFO:teuthology.orchestra.run.smithi026.stdout: "ops": [], 2023-09-23T20:55:30.469 INFO:teuthology.orchestra.run.smithi026.stdout: "complaint_time": 30, 2023-09-23T20:55:30.469 INFO:teuthology.orchestra.run.smithi026.stdout: "num_blocked_ops": 0 2023-09-23T20:55:30.470 INFO:teuthology.orchestra.run.smithi026.stdout:} 2023-09-23T20:55:30.485 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T20:55:30.503 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:55:30.514 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:55:30.514 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:55:30.524 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T20:55:30.524 INFO:teuthology.orchestra.run.smithi026.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T20:55:30.524 INFO:teuthology.orchestra.run.smithi026.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T20:55:30.524 INFO:teuthology.orchestra.run.smithi026.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T20:55:30.525 INFO:teuthology.orchestra.run.smithi026.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T20:55:30.525 INFO:teuthology.orchestra.run.smithi026.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T20:55:30.525 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T20:55:30.525 INFO:teuthology.orchestra.run.smithi026.stderr:Error ENXIO: problem getting command descriptions from osd.6 2023-09-23T20:55:30.525 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T20:55:30.615 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:55:30.719 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:55:30.722 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T20:55:30.900 INFO:teuthology.orchestra.run.smithi070.stderr:no valid command found; 10 closest matches: 2023-09-23T20:55:30.900 INFO:teuthology.orchestra.run.smithi070.stderr:0 2023-09-23T20:55:30.900 INFO:teuthology.orchestra.run.smithi070.stderr:1 2023-09-23T20:55:30.900 INFO:teuthology.orchestra.run.smithi070.stderr:2 2023-09-23T20:55:30.900 INFO:teuthology.orchestra.run.smithi070.stderr:abort 2023-09-23T20:55:30.900 INFO:teuthology.orchestra.run.smithi070.stderr:assert 2023-09-23T20:55:30.900 INFO:teuthology.orchestra.run.smithi070.stderr:config diff 2023-09-23T20:55:30.901 INFO:teuthology.orchestra.run.smithi070.stderr:config diff get 2023-09-23T20:55:30.901 INFO:teuthology.orchestra.run.smithi070.stderr:config get 2023-09-23T20:55:30.901 INFO:teuthology.orchestra.run.smithi070.stderr:config help [] 2023-09-23T20:55:30.901 INFO:teuthology.orchestra.run.smithi070.stderr:config set ... 2023-09-23T20:55:30.901 INFO:teuthology.orchestra.run.smithi070.stderr:admin_socket: invalid command 2023-09-23T20:55:30.904 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:55:30.919 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:55:30.919 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:55:30.967 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:55:30 smithi026 ceph-mon[175151]: 6.8 scrub starts 2023-09-23T20:55:30.968 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:55:30 smithi026 ceph-mon[175151]: 6.2 scrub starts 2023-09-23T20:55:30.968 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:55:30 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/2620465554' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:55:30.983 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_historic_ops 2023-09-23T20:55:31.020 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:55:31.057 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:55:31.234 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:55:30 smithi026 ceph-mon[179688]: 6.8 scrub starts 2023-09-23T20:55:31.234 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:55:30 smithi026 ceph-mon[179688]: 6.2 scrub starts 2023-09-23T20:55:31.234 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:55:30 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/2620465554' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:55:31.277 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T20:55:31.277 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T20:55:31.292 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:55:30 smithi070 ceph-mon[168282]: 6.8 scrub starts 2023-09-23T20:55:31.292 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:55:30 smithi070 ceph-mon[168282]: 6.2 scrub starts 2023-09-23T20:55:31.292 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:55:30 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/2620465554' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:55:31.293 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:55:31.298 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:55:31.299 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:55:31.318 DEBUG:teuthology.orchestra.run:got remote process result: 22 2023-09-23T20:55:31.318 INFO:tasks.cephadm.ceph_manager.ceph:waiting on admin_socket for osd-6, ['dump_ops_in_flight'] 2023-09-23T20:55:31.400 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:55:31.431 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:55:31.475 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:55:31.480 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:55:31.480 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:55:31.581 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:55:31.672 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:55:31.704 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:55:31.788 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:55:31.797 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:55:31.798 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:55:31.803 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:55:31.899 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:55:31.938 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:55:31.972 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:55:31 smithi026 ceph-mon[175151]: 6.9 scrub starts 2023-09-23T20:55:31.972 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:55:31 smithi026 ceph-mon[175151]: pgmap v393: 109 pgs: 1 active+undersized+degraded+remapped+backfilling, 3 active+clean+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 31 active+undersized, 60 active+clean, 11 active+undersized+degraded, 2 active+remapped+backfill_wait; 1.8 GiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 814 op/s; 7207/90609 objects degraded (7.954%); 5557/90609 objects misplaced (6.133%); 21 MiB/s, 356 objects/s recovering 2023-09-23T20:55:31.972 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:55:31 smithi026 ceph-mon[175151]: 6.9 scrub ok 2023-09-23T20:55:31.972 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:55:31 smithi026 ceph-mon[175151]: 6.12 scrub starts 2023-09-23T20:55:31.972 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:55:31 smithi026 ceph-mon[175151]: 6.19 scrub starts 2023-09-23T20:55:31.972 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:55:31 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:55:31.972 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:55:31 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:55:31.972 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:55:31 smithi026 ceph-mon[175151]: 6.19 scrub ok 2023-09-23T20:55:31.975 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:55:31 smithi026 ceph-mon[179688]: 6.9 scrub starts 2023-09-23T20:55:31.975 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:55:31 smithi026 ceph-mon[179688]: pgmap v393: 109 pgs: 1 active+undersized+degraded+remapped+backfilling, 3 active+clean+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 31 active+undersized, 60 active+clean, 11 active+undersized+degraded, 2 active+remapped+backfill_wait; 1.8 GiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 814 op/s; 7207/90609 objects degraded (7.954%); 5557/90609 objects misplaced (6.133%); 21 MiB/s, 356 objects/s recovering 2023-09-23T20:55:31.975 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:55:31 smithi026 ceph-mon[179688]: 6.9 scrub ok 2023-09-23T20:55:31.975 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:55:31 smithi026 ceph-mon[179688]: 6.12 scrub starts 2023-09-23T20:55:31.975 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:55:31 smithi026 ceph-mon[179688]: 6.19 scrub starts 2023-09-23T20:55:31.975 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:55:31 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:55:31.975 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:55:31 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:55:31.975 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:55:31 smithi026 ceph-mon[179688]: 6.19 scrub ok 2023-09-23T20:55:31.983 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:55:31.987 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:55:31.988 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:55:32.088 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:55:32.190 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:55:32.370 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:55:32.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:55:31 smithi070 ceph-mon[168282]: 6.9 scrub starts 2023-09-23T20:55:32.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:55:31 smithi070 ceph-mon[168282]: pgmap v393: 109 pgs: 1 active+undersized+degraded+remapped+backfilling, 3 active+clean+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 31 active+undersized, 60 active+clean, 11 active+undersized+degraded, 2 active+remapped+backfill_wait; 1.8 GiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 814 op/s; 7207/90609 objects degraded (7.954%); 5557/90609 objects misplaced (6.133%); 21 MiB/s, 356 objects/s recovering 2023-09-23T20:55:32.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:55:31 smithi070 ceph-mon[168282]: 6.9 scrub ok 2023-09-23T20:55:32.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:55:31 smithi070 ceph-mon[168282]: 6.12 scrub starts 2023-09-23T20:55:32.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:55:31 smithi070 ceph-mon[168282]: 6.19 scrub starts 2023-09-23T20:55:32.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:55:31 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:55:32.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:55:31 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:55:32.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:55:31 smithi070 ceph-mon[168282]: 6.19 scrub ok 2023-09-23T20:55:32.381 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:55:32.382 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:55:32.482 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:55:32.515 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:55:32.570 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:55:32.573 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:55:32.574 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:55:32.674 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:55:32.770 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:55:33.033 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:55:33.046 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:55:33.046 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:55:32 smithi026 ceph-mon[175151]: 6.18 scrub starts 2023-09-23T20:55:33.047 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:55:32 smithi026 ceph-mon[179688]: 6.18 scrub starts 2023-09-23T20:55:33.047 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:55:33.148 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:55:33.190 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:55:33.239 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:55:33.244 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:55:33.245 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:55:33.245 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:55:32 smithi070 ceph-mon[168282]: 6.18 scrub starts 2023-09-23T20:55:33.279 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T20:55:33.346 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:55:33.383 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:55:33.429 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:55:33.432 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:55:33.433 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:55:33.533 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:55:33.573 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:55:33.619 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:55:33.622 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:55:33.622 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:55:33.723 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:55:33.759 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:55:33.802 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:55:33.807 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:55:33.808 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:55:33.869 INFO:journalctl@ceph.osd.6.smithi070.stdout:Sep 23 20:55:33 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd-6[193219]: 2023-09-23T20:55:33.709+0000 7f417cd5c540 -1 osd.6 154 log_to_monitors true 2023-09-23T20:55:33.909 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:55:33.984 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:55:34.369 INFO:journalctl@ceph.osd.6.smithi070.stdout:Sep 23 20:55:34 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd-6[193219]: 2023-09-23T20:55:34.028+0000 7f416e33d700 -1 osd.6 154 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-09-23T20:55:34.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:55:33 smithi070 ceph-mon[168282]: pgmap v394: 109 pgs: 1 active+undersized+degraded+remapped+backfilling, 3 active+clean+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 31 active+undersized, 60 active+clean, 11 active+undersized+degraded, 2 active+remapped+backfill_wait; 1.7 GiB data, 6.7 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 115 op/s; 6738/83205 objects degraded (8.098%); 5309/83205 objects misplaced (6.381%); 6.8 MiB/s, 108 objects/s recovering 2023-09-23T20:55:34.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:55:33 smithi070 ceph-mon[168282]: 6.2 scrub starts 2023-09-23T20:55:34.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:55:33 smithi070 ceph-mon[168282]: Health check update: Degraded data redundancy: 6738/83205 objects degraded (8.098%), 13 pgs degraded (PG_DEGRADED) 2023-09-23T20:55:34.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:55:33 smithi070 ceph-mon[168282]: 6.8 scrub starts 2023-09-23T20:55:34.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:55:33 smithi070 ceph-mon[168282]: 6.10 scrub starts 2023-09-23T20:55:34.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:55:33 smithi070 ceph-mon[168282]: from='osd.6 [v2:172.21.15.70:6816/4141218814,v1:172.21.15.70:6817/4141218814]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2023-09-23T20:55:34.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:55:33 smithi070 ceph-mon[168282]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2023-09-23T20:55:34.378 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:55:34.384 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:55:33 smithi026 ceph-mon[175151]: pgmap v394: 109 pgs: 1 active+undersized+degraded+remapped+backfilling, 3 active+clean+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 31 active+undersized, 60 active+clean, 11 active+undersized+degraded, 2 active+remapped+backfill_wait; 1.7 GiB data, 6.7 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 115 op/s; 6738/83205 objects degraded (8.098%); 5309/83205 objects misplaced (6.381%); 6.8 MiB/s, 108 objects/s recovering 2023-09-23T20:55:34.384 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:55:33 smithi026 ceph-mon[175151]: 6.2 scrub starts 2023-09-23T20:55:34.384 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:55:33 smithi026 ceph-mon[175151]: Health check update: Degraded data redundancy: 6738/83205 objects degraded (8.098%), 13 pgs degraded (PG_DEGRADED) 2023-09-23T20:55:34.384 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:55:33 smithi026 ceph-mon[175151]: 6.8 scrub starts 2023-09-23T20:55:34.384 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:55:33 smithi026 ceph-mon[175151]: 6.10 scrub starts 2023-09-23T20:55:34.384 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:55:33 smithi026 ceph-mon[175151]: from='osd.6 [v2:172.21.15.70:6816/4141218814,v1:172.21.15.70:6817/4141218814]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2023-09-23T20:55:34.384 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:55:33 smithi026 ceph-mon[175151]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2023-09-23T20:55:34.385 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:55:33 smithi026 ceph-mon[179688]: pgmap v394: 109 pgs: 1 active+undersized+degraded+remapped+backfilling, 3 active+clean+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 31 active+undersized, 60 active+clean, 11 active+undersized+degraded, 2 active+remapped+backfill_wait; 1.7 GiB data, 6.7 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 115 op/s; 6738/83205 objects degraded (8.098%); 5309/83205 objects misplaced (6.381%); 6.8 MiB/s, 108 objects/s recovering 2023-09-23T20:55:34.385 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:55:33 smithi026 ceph-mon[179688]: 6.2 scrub starts 2023-09-23T20:55:34.385 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:55:33 smithi026 ceph-mon[179688]: Health check update: Degraded data redundancy: 6738/83205 objects degraded (8.098%), 13 pgs degraded (PG_DEGRADED) 2023-09-23T20:55:34.385 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:55:33 smithi026 ceph-mon[179688]: 6.8 scrub starts 2023-09-23T20:55:34.385 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:55:33 smithi026 ceph-mon[179688]: 6.10 scrub starts 2023-09-23T20:55:34.385 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:55:33 smithi026 ceph-mon[179688]: from='osd.6 [v2:172.21.15.70:6816/4141218814,v1:172.21.15.70:6817/4141218814]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2023-09-23T20:55:34.385 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:55:33 smithi026 ceph-mon[179688]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2023-09-23T20:55:34.385 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:55:34.386 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:55:34.487 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:55:34.579 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:55:34.684 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:55:34.689 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:55:34.689 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:55:34.790 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:55:34.856 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:55:35.301 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:55:35 smithi026 ceph-mon[179688]: failed to encode map e163 with expected crc 2023-09-23T20:55:35.301 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:55:35 smithi026 ceph-mon[179688]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2023-09-23T20:55:35.302 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:55:35 smithi026 ceph-mon[179688]: osdmap e163: 8 total, 7 up, 8 in 2023-09-23T20:55:35.302 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:55:35 smithi026 ceph-mon[179688]: from='osd.6 [v2:172.21.15.70:6816/4141218814,v1:172.21.15.70:6817/4141218814]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi070", "root=default"]}]: dispatch 2023-09-23T20:55:35.302 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:55:35 smithi026 ceph-mon[179688]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi070", "root=default"]}]: dispatch 2023-09-23T20:55:35.302 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:55:35 smithi026 ceph-mon[179688]: failed to encode map e163 with expected crc 2023-09-23T20:55:35.302 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:55:35 smithi026 ceph-mon[179688]: 6.0 scrub starts 2023-09-23T20:55:35.302 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:55:35 smithi026 ceph-mon[179688]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-09-23T20:55:35.302 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:55:35 smithi026 ceph-mon[179688]: osd.6 [v2:172.21.15.70:6816/4141218814,v1:172.21.15.70:6817/4141218814] boot 2023-09-23T20:55:35.302 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:55:35 smithi026 ceph-mon[179688]: osdmap e164: 8 total, 8 up, 8 in 2023-09-23T20:55:35.302 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:55:35 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-09-23T20:55:35.303 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:55:35 smithi026 ceph-mon[175151]: failed to encode map e163 with expected crc 2023-09-23T20:55:35.303 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:55:35 smithi026 ceph-mon[175151]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2023-09-23T20:55:35.303 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:55:35 smithi026 ceph-mon[175151]: osdmap e163: 8 total, 7 up, 8 in 2023-09-23T20:55:35.303 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:55:35 smithi026 ceph-mon[175151]: from='osd.6 [v2:172.21.15.70:6816/4141218814,v1:172.21.15.70:6817/4141218814]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi070", "root=default"]}]: dispatch 2023-09-23T20:55:35.303 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:55:35 smithi026 ceph-mon[175151]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi070", "root=default"]}]: dispatch 2023-09-23T20:55:35.303 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:55:35 smithi026 ceph-mon[175151]: failed to encode map e163 with expected crc 2023-09-23T20:55:35.303 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:55:35 smithi026 ceph-mon[175151]: 6.0 scrub starts 2023-09-23T20:55:35.303 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:55:35 smithi026 ceph-mon[175151]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-09-23T20:55:35.303 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:55:35 smithi026 ceph-mon[175151]: osd.6 [v2:172.21.15.70:6816/4141218814,v1:172.21.15.70:6817/4141218814] boot 2023-09-23T20:55:35.303 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:55:35 smithi026 ceph-mon[175151]: osdmap e164: 8 total, 8 up, 8 in 2023-09-23T20:55:35.304 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:55:35 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-09-23T20:55:35.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:55:35 smithi070 ceph-mon[168282]: failed to encode map e163 with expected crc 2023-09-23T20:55:35.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:55:35 smithi070 ceph-mon[168282]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2023-09-23T20:55:35.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:55:35 smithi070 ceph-mon[168282]: osdmap e163: 8 total, 7 up, 8 in 2023-09-23T20:55:35.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:55:35 smithi070 ceph-mon[168282]: from='osd.6 [v2:172.21.15.70:6816/4141218814,v1:172.21.15.70:6817/4141218814]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi070", "root=default"]}]: dispatch 2023-09-23T20:55:35.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:55:35 smithi070 ceph-mon[168282]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi070", "root=default"]}]: dispatch 2023-09-23T20:55:35.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:55:35 smithi070 ceph-mon[168282]: failed to encode map e163 with expected crc 2023-09-23T20:55:35.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:55:35 smithi070 ceph-mon[168282]: 6.0 scrub starts 2023-09-23T20:55:35.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:55:35 smithi070 ceph-mon[168282]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-09-23T20:55:35.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:55:35 smithi070 ceph-mon[168282]: osd.6 [v2:172.21.15.70:6816/4141218814,v1:172.21.15.70:6817/4141218814] boot 2023-09-23T20:55:35.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:55:35 smithi070 ceph-mon[168282]: osdmap e164: 8 total, 8 up, 8 in 2023-09-23T20:55:35.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:55:35 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-09-23T20:55:35.502 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:55:35.505 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:55:35.506 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:55:35.606 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:55:35.651 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:55:35.746 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:55:35.749 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:55:35.749 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:55:35.761 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:55:35.851 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:55:35.877 INFO:teuthology.orchestra.run.smithi026.stdout:{ 2023-09-23T20:55:35.877 INFO:teuthology.orchestra.run.smithi026.stdout: "size": 20, 2023-09-23T20:55:35.877 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 600, 2023-09-23T20:55:35.877 INFO:teuthology.orchestra.run.smithi026.stdout: "ops": [ 2023-09-23T20:55:35.877 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:35.877 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34797.0:72078 6.0 6:027b02cd:::benchmark_data_smithi026_223868_object72077:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e116)", 2023-09-23T20:55:35.877 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T20:53:09.072607+0000", 2023-09-23T20:55:35.878 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 146.80129401299999, 2023-09-23T20:55:35.878 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.113808721, 2023-09-23T20:55:35.878 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T20:55:35.878 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:55:35.878 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T20:55:35.878 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34797", 2023-09-23T20:55:35.878 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T20:55:35.878 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 72078 2023-09-23T20:55:35.878 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:35.878 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T20:55:35.878 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:35.879 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T20:55:35.879 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:09.072607+0000", 2023-09-23T20:55:35.879 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:55:35.879 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:35.879 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:35.879 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T20:55:35.879 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:09.072607+0000", 2023-09-23T20:55:35.879 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:55:35.879 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:35.879 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:35.879 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T20:55:35.879 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:09.072605+0000", 2023-09-23T20:55:35.880 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999981 2023-09-23T20:55:35.880 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:35.880 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:35.880 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T20:55:35.880 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:09.072637+0000", 2023-09-23T20:55:35.880 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.1724000000000003e-05 2023-09-23T20:55:35.880 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:35.880 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:35.880 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T20:55:35.880 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:09.072651+0000", 2023-09-23T20:55:35.880 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.453e-05 2023-09-23T20:55:35.880 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:35.881 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:35.881 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T20:55:35.881 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:09.072661+0000", 2023-09-23T20:55:35.881 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.2920000000000001e-06 2023-09-23T20:55:35.881 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:35.881 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:35.881 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:55:35.881 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:09.072847+0000", 2023-09-23T20:55:35.881 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00018689600000000001 2023-09-23T20:55:35.881 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:35.881 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:35.881 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T20:55:35.882 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:09.072990+0000", 2023-09-23T20:55:35.882 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00014295000000000001 2023-09-23T20:55:35.882 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:35.882 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:35.882 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 2,3", 2023-09-23T20:55:35.882 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:09.073143+0000", 2023-09-23T20:55:35.882 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00015277199999999999 2023-09-23T20:55:35.882 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:35.882 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:35.882 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T20:55:35.882 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:09.078624+0000", 2023-09-23T20:55:35.882 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0054807620000000001 2023-09-23T20:55:35.883 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:35.883 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:35.883 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:55:35.883 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:09.079394+0000", 2023-09-23T20:55:35.883 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00077003 2023-09-23T20:55:35.883 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:35.883 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:35.883 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:55:35.883 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:09.186289+0000", 2023-09-23T20:55:35.883 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.106894899 2023-09-23T20:55:35.883 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:35.883 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:35.884 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T20:55:35.884 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:09.186374+0000", 2023-09-23T20:55:35.884 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.5068000000000006e-05 2023-09-23T20:55:35.884 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:35.884 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:35.884 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T20:55:35.884 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:09.186416+0000", 2023-09-23T20:55:35.884 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.1674000000000001e-05 2023-09-23T20:55:35.884 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:55:35.884 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T20:55:35.884 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:55:35.884 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:35.885 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:35.885 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34797.0:72083 6.0 6:0bf8574b:::benchmark_data_smithi026_223868_object72082:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e116)", 2023-09-23T20:55:35.885 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T20:53:09.072728+0000", 2023-09-23T20:55:35.885 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 146.801172834, 2023-09-23T20:55:35.885 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.11381173999999999, 2023-09-23T20:55:35.885 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T20:55:35.885 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:55:35.885 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T20:55:35.885 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34797", 2023-09-23T20:55:35.885 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T20:55:35.885 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 72083 2023-09-23T20:55:35.885 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:35.886 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T20:55:35.886 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:35.886 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T20:55:35.886 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:09.072728+0000", 2023-09-23T20:55:35.886 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:55:35.886 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:35.886 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:35.886 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T20:55:35.886 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:09.072728+0000", 2023-09-23T20:55:35.886 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:55:35.886 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:35.887 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:35.887 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T20:55:35.887 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:09.072728+0000", 2023-09-23T20:55:35.887 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T20:55:35.887 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:35.887 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:35.887 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T20:55:35.887 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:09.072758+0000", 2023-09-23T20:55:35.887 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.0448999999999999e-05 2023-09-23T20:55:35.887 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:35.887 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:35.887 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T20:55:35.888 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:09.072766+0000", 2023-09-23T20:55:35.888 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.7880000000000007e-06 2023-09-23T20:55:35.888 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:35.888 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:35.888 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T20:55:35.888 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:09.072772+0000", 2023-09-23T20:55:35.888 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.1340000000000001e-06 2023-09-23T20:55:35.888 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:35.888 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:35.888 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:55:35.888 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:09.075701+0000", 2023-09-23T20:55:35.889 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0029294009999999999 2023-09-23T20:55:35.889 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:35.889 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:35.889 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T20:55:35.889 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:09.075856+0000", 2023-09-23T20:55:35.889 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00015489200000000001 2023-09-23T20:55:35.889 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:35.889 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:35.889 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 2,3", 2023-09-23T20:55:35.889 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:09.076012+0000", 2023-09-23T20:55:35.889 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00015609099999999999 2023-09-23T20:55:35.889 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:35.890 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:35.890 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T20:55:35.890 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:09.078678+0000", 2023-09-23T20:55:35.890 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0026658810000000002 2023-09-23T20:55:35.890 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:35.890 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:35.890 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:55:35.890 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:09.079801+0000", 2023-09-23T20:55:35.890 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00112319 2023-09-23T20:55:35.890 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:35.890 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:35.891 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:55:35.891 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:09.186456+0000", 2023-09-23T20:55:35.891 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.106654584 2023-09-23T20:55:35.891 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:35.891 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:35.891 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T20:55:35.891 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:09.186505+0000", 2023-09-23T20:55:35.891 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.8816999999999997e-05 2023-09-23T20:55:35.891 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:35.891 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:35.891 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T20:55:35.891 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:09.186540+0000", 2023-09-23T20:55:35.891 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.4941999999999999e-05 2023-09-23T20:55:35.892 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:55:35.892 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T20:55:35.892 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:55:35.892 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:35.892 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:35.892 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34797.0:130474 6.d 6:baf3ed90:::benchmark_data_smithi026_223868_object29348:head [delete] snapc 0=[] RETRY=2 ondisk+retry+write+known_if_redirected+supports_pool_eio e137)", 2023-09-23T20:55:35.892 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T20:54:31.883928+0000", 2023-09-23T20:55:35.892 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 63.989972371, 2023-09-23T20:55:35.892 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.143112095, 2023-09-23T20:55:35.892 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T20:55:35.892 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "reached pg", 2023-09-23T20:55:35.892 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T20:55:35.893 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34797", 2023-09-23T20:55:35.893 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T20:55:35.893 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 130474 2023-09-23T20:55:35.893 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:35.893 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T20:55:35.893 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:35.893 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T20:55:35.893 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:31.883928+0000", 2023-09-23T20:55:35.893 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:55:35.893 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:35.893 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:35.893 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T20:55:35.894 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:31.883928+0000", 2023-09-23T20:55:35.894 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:55:35.894 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:35.894 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:35.894 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T20:55:35.894 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:31.883927+0000", 2023-09-23T20:55:35.894 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999981 2023-09-23T20:55:35.894 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:35.894 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:35.894 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T20:55:35.894 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:31.883932+0000", 2023-09-23T20:55:35.894 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.2610000000000001e-06 2023-09-23T20:55:35.895 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:35.895 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:35.895 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T20:55:35.895 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:31.883943+0000", 2023-09-23T20:55:35.895 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0861e-05 2023-09-23T20:55:35.895 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:35.895 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:35.895 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T20:55:35.895 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:31.883954+0000", 2023-09-23T20:55:35.895 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0686999999999999e-05 2023-09-23T20:55:35.895 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:35.895 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:35.896 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:55:35.896 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:31.884281+0000", 2023-09-23T20:55:35.896 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00032779599999999999 2023-09-23T20:55:35.896 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:35.896 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:35.896 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T20:55:35.896 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:31.884291+0000", 2023-09-23T20:55:35.896 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.3000000000000007e-06 2023-09-23T20:55:35.896 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:35.896 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:35.896 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:55:35.896 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:33.025141+0000", 2023-09-23T20:55:35.897 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1408508580000001 2023-09-23T20:55:35.897 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:35.897 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:35.897 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T20:55:35.897 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:33.027041+0000", 2023-09-23T20:55:35.897 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0018991030000000001 2023-09-23T20:55:35.897 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:55:35.897 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T20:55:35.897 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:55:35.897 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:35.897 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:35.897 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34797.0:130489 6.d 6:bb89226e:::benchmark_data_smithi026_223868_object29363:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e137)", 2023-09-23T20:55:35.898 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T20:54:31.883974+0000", 2023-09-23T20:55:35.898 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 63.989926721000003, 2023-09-23T20:55:35.898 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.161869933, 2023-09-23T20:55:35.898 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T20:55:35.898 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:55:35.898 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T20:55:35.898 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34797", 2023-09-23T20:55:35.898 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T20:55:35.898 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 130489 2023-09-23T20:55:35.898 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:35.898 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T20:55:35.899 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:35.899 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T20:55:35.899 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:31.883974+0000", 2023-09-23T20:55:35.899 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:55:35.899 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:35.899 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:35.899 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T20:55:35.899 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:31.883974+0000", 2023-09-23T20:55:35.899 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:55:35.899 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:35.899 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:35.899 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T20:55:35.899 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:31.883974+0000", 2023-09-23T20:55:35.900 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T20:55:35.900 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:35.900 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:35.900 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T20:55:35.900 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:31.883977+0000", 2023-09-23T20:55:35.900 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.348e-06 2023-09-23T20:55:35.900 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:35.900 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:35.900 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T20:55:35.900 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:31.883985+0000", 2023-09-23T20:55:35.900 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.191e-06 2023-09-23T20:55:35.900 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:35.901 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:35.901 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T20:55:35.901 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:31.883992+0000", 2023-09-23T20:55:35.901 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.7610000000000002e-06 2023-09-23T20:55:35.901 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:35.901 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:35.901 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:55:35.901 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:31.884354+0000", 2023-09-23T20:55:35.901 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00036240399999999998 2023-09-23T20:55:35.901 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:35.901 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:35.901 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T20:55:35.901 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:31.884363+0000", 2023-09-23T20:55:35.902 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.9989999999999997e-06 2023-09-23T20:55:35.902 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:35.902 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:35.902 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:55:35.902 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:33.027448+0000", 2023-09-23T20:55:35.902 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.143084325 2023-09-23T20:55:35.902 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:35.902 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:35.902 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T20:55:35.902 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:33.028112+0000", 2023-09-23T20:55:35.902 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00066470099999999996 2023-09-23T20:55:35.902 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:35.902 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:35.903 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 1,2,5", 2023-09-23T20:55:35.903 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:33.028273+0000", 2023-09-23T20:55:35.903 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00016043599999999999 2023-09-23T20:55:35.903 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:35.903 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:35.903 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T20:55:35.903 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:33.035710+0000", 2023-09-23T20:55:35.903 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0074372700000000002 2023-09-23T20:55:35.903 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:35.903 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:35.903 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:55:35.903 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:33.038722+0000", 2023-09-23T20:55:35.903 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0030117439999999998 2023-09-23T20:55:35.904 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:35.904 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:35.904 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:55:35.904 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:33.040863+0000", 2023-09-23T20:55:35.904 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0021409070000000001 2023-09-23T20:55:35.904 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:35.904 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:35.904 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:55:35.904 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:33.045669+0000", 2023-09-23T20:55:35.904 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0048059039999999997 2023-09-23T20:55:35.904 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:35.904 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:35.905 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T20:55:35.905 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:33.045728+0000", 2023-09-23T20:55:35.905 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.9129999999999998e-05 2023-09-23T20:55:35.905 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:35.905 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:35.905 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T20:55:35.905 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:33.045844+0000", 2023-09-23T20:55:35.905 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000116372 2023-09-23T20:55:35.905 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:55:35.905 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T20:55:35.905 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:55:35.905 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:35.906 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:35.906 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34797.0:130501 6.a 6:577795ba:::benchmark_data_smithi026_223868_object29375:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e137)", 2023-09-23T20:55:35.906 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T20:54:31.883997+0000", 2023-09-23T20:55:35.906 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 63.989903794, 2023-09-23T20:55:35.906 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1379962809999999, 2023-09-23T20:55:35.906 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T20:55:35.906 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:55:35.906 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T20:55:35.906 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34797", 2023-09-23T20:55:35.906 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T20:55:35.906 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 130501 2023-09-23T20:55:35.906 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:35.907 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T20:55:35.907 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:35.907 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T20:55:35.907 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:31.883997+0000", 2023-09-23T20:55:35.907 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:55:35.907 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:35.907 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:35.907 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T20:55:35.907 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:31.883997+0000", 2023-09-23T20:55:35.907 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:55:35.907 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:35.907 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:35.907 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T20:55:35.908 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:31.883997+0000", 2023-09-23T20:55:35.908 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T20:55:35.908 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:35.908 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:35.908 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T20:55:35.908 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:31.883999+0000", 2023-09-23T20:55:35.908 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.8660000000000002e-06 2023-09-23T20:55:35.908 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:35.908 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:35.908 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T20:55:35.908 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:31.884006+0000", 2023-09-23T20:55:35.908 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.5950000000000004e-06 2023-09-23T20:55:35.909 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:35.909 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:35.909 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T20:55:35.909 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:31.884013+0000", 2023-09-23T20:55:35.909 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.9179999999999997e-06 2023-09-23T20:55:35.909 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:35.909 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:35.909 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:55:35.909 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:31.884335+0000", 2023-09-23T20:55:35.909 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00032235800000000002 2023-09-23T20:55:35.909 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:35.909 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:35.910 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T20:55:35.910 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:31.884344+0000", 2023-09-23T20:55:35.910 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.7930000000000003e-06 2023-09-23T20:55:35.910 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:35.910 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:35.910 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:55:35.910 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:33.012175+0000", 2023-09-23T20:55:35.910 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.127831298 2023-09-23T20:55:35.910 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:35.910 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:35.910 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T20:55:35.911 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:33.014065+0000", 2023-09-23T20:55:35.911 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.001889305 2023-09-23T20:55:35.911 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:35.911 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:35.911 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 2,5,6", 2023-09-23T20:55:35.911 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:33.014209+0000", 2023-09-23T20:55:35.911 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00014406099999999999 2023-09-23T20:55:35.911 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:35.911 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:35.911 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T20:55:35.911 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:33.021710+0000", 2023-09-23T20:55:35.911 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0075012999999999998 2023-09-23T20:55:35.912 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:35.912 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:35.912 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:55:35.912 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:33.021748+0000", 2023-09-23T20:55:35.912 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.782e-05 2023-09-23T20:55:35.912 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:35.912 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:35.912 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:55:35.912 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:33.021781+0000", 2023-09-23T20:55:35.912 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.2969999999999998e-05 2023-09-23T20:55:35.912 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:35.912 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:35.912 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:55:35.913 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:33.021834+0000", 2023-09-23T20:55:35.913 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.3510000000000001e-05 2023-09-23T20:55:35.913 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:35.913 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:35.913 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T20:55:35.913 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:33.021888+0000", 2023-09-23T20:55:35.913 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.3115000000000002e-05 2023-09-23T20:55:35.913 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:35.913 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:35.913 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T20:55:35.913 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:33.021993+0000", 2023-09-23T20:55:35.913 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000105827 2023-09-23T20:55:35.914 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:55:35.914 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T20:55:35.914 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:55:35.914 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:35.914 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:35.914 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34797.0:130513 6.a 6:5baf817b:::benchmark_data_smithi026_223868_object29387:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e137)", 2023-09-23T20:55:35.914 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T20:54:31.884022+0000", 2023-09-23T20:55:35.914 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 63.989879207999998, 2023-09-23T20:55:35.914 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1549781690000001, 2023-09-23T20:55:35.914 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T20:55:35.914 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:55:35.915 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T20:55:35.915 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34797", 2023-09-23T20:55:35.915 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T20:55:35.915 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 130513 2023-09-23T20:55:35.915 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:35.915 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T20:55:35.915 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:35.915 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T20:55:35.915 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:31.884022+0000", 2023-09-23T20:55:35.915 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:55:35.915 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:35.915 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:35.916 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T20:55:35.916 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:31.884022+0000", 2023-09-23T20:55:35.916 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:55:35.916 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:35.916 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:35.916 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T20:55:35.916 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:31.884021+0000", 2023-09-23T20:55:35.916 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T20:55:35.916 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:35.916 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:35.916 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T20:55:35.916 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:31.884024+0000", 2023-09-23T20:55:35.917 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.8720000000000002e-06 2023-09-23T20:55:35.917 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:35.917 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:35.917 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T20:55:35.917 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:31.884031+0000", 2023-09-23T20:55:35.917 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.1670000000000002e-06 2023-09-23T20:55:35.917 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:35.917 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:35.917 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T20:55:35.917 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:31.884038+0000", 2023-09-23T20:55:35.917 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.2600000000000002e-06 2023-09-23T20:55:35.917 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:35.918 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:35.918 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:55:35.918 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:31.884370+0000", 2023-09-23T20:55:35.918 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00033257499999999999 2023-09-23T20:55:35.918 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:35.918 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:35.918 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T20:55:35.918 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:31.884378+0000", 2023-09-23T20:55:35.918 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.1899999999999995e-06 2023-09-23T20:55:35.918 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:35.918 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:35.918 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:55:35.919 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:33.018253+0000", 2023-09-23T20:55:35.919 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.133874724 2023-09-23T20:55:35.919 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:35.919 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:35.919 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T20:55:35.919 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:33.018408+0000", 2023-09-23T20:55:35.919 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00015456199999999999 2023-09-23T20:55:35.919 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:35.919 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:35.919 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 2,5,6", 2023-09-23T20:55:35.919 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:33.018566+0000", 2023-09-23T20:55:35.919 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00015848300000000001 2023-09-23T20:55:35.920 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:35.920 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:35.920 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T20:55:35.920 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:33.022005+0000", 2023-09-23T20:55:35.920 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0034394249999999999 2023-09-23T20:55:35.920 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:35.920 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:35.920 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:55:35.920 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:33.025311+0000", 2023-09-23T20:55:35.920 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0033057469999999999 2023-09-23T20:55:35.920 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:35.920 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:35.921 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:55:35.921 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:33.025347+0000", 2023-09-23T20:55:35.921 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.5887999999999999e-05 2023-09-23T20:55:35.921 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:35.921 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:35.921 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:55:35.921 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:33.038801+0000", 2023-09-23T20:55:35.921 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.013454173 2023-09-23T20:55:35.921 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:35.921 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:35.921 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T20:55:35.921 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:33.038860+0000", 2023-09-23T20:55:35.922 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.8795999999999997e-05 2023-09-23T20:55:35.922 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:35.922 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:35.922 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T20:55:35.922 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:33.039000+0000", 2023-09-23T20:55:35.922 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00013974899999999999 2023-09-23T20:55:35.922 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:55:35.922 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T20:55:35.922 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:55:35.922 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:35.922 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:35.922 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34797.0:140388 6.a 6:54078b98:::benchmark_data_smithi026_223868_object39262:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e143)", 2023-09-23T20:55:35.923 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T20:54:45.774817+0000", 2023-09-23T20:55:35.923 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 50.099083557, 2023-09-23T20:55:35.923 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.120818231, 2023-09-23T20:55:35.923 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T20:55:35.923 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:55:35.923 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T20:55:35.923 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34797", 2023-09-23T20:55:35.923 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T20:55:35.923 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 140388 2023-09-23T20:55:35.923 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:35.923 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T20:55:35.924 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:35.924 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T20:55:35.924 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:45.774817+0000", 2023-09-23T20:55:35.924 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:55:35.924 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:35.924 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:35.924 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T20:55:35.924 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:45.774817+0000", 2023-09-23T20:55:35.924 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:55:35.924 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:35.924 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:35.925 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T20:55:35.925 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:45.774817+0000", 2023-09-23T20:55:35.925 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T20:55:35.925 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:35.925 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:35.925 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T20:55:35.925 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:45.774820+0000", 2023-09-23T20:55:35.925 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.0580000000000002e-06 2023-09-23T20:55:35.925 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:35.925 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:35.925 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T20:55:35.925 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:45.774828+0000", 2023-09-23T20:55:35.926 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.5820000000000004e-06 2023-09-23T20:55:35.926 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:35.926 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:35.926 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T20:55:35.926 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:45.774833+0000", 2023-09-23T20:55:35.926 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.8719999999999999e-06 2023-09-23T20:55:35.926 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:35.926 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:35.926 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:55:35.926 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:45.785927+0000", 2023-09-23T20:55:35.926 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.011093337999999999 2023-09-23T20:55:35.927 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:35.927 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:35.927 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T20:55:35.927 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:45.786065+0000", 2023-09-23T20:55:35.927 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00013858100000000001 2023-09-23T20:55:35.927 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:35.927 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:35.927 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 2,5,6", 2023-09-23T20:55:35.927 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:45.786195+0000", 2023-09-23T20:55:35.927 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00013017399999999999 2023-09-23T20:55:35.927 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:35.927 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:35.928 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:55:35.928 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:45.796250+0000", 2023-09-23T20:55:35.928 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.010054808 2023-09-23T20:55:35.928 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:35.928 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:35.928 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:55:35.928 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:45.796286+0000", 2023-09-23T20:55:35.928 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.5574999999999997e-05 2023-09-23T20:55:35.928 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:35.928 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:35.928 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:55:35.929 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:45.797057+0000", 2023-09-23T20:55:35.929 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00077090000000000004 2023-09-23T20:55:35.929 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:35.929 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:35.929 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T20:55:35.929 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:45.797071+0000", 2023-09-23T20:55:35.929 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.4654000000000001e-05 2023-09-23T20:55:35.929 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:35.929 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:35.929 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T20:55:35.929 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:45.895516+0000", 2023-09-23T20:55:35.929 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.098444990999999996 2023-09-23T20:55:35.930 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:35.930 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:35.930 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T20:55:35.930 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:45.895636+0000", 2023-09-23T20:55:35.930 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000119138 2023-09-23T20:55:35.930 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:55:35.930 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T20:55:35.930 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:55:35.930 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:35.930 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:35.930 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34797.0:140393 6.a 6:53dab1dd:::benchmark_data_smithi026_223868_object39267:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e143)", 2023-09-23T20:55:35.930 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T20:54:45.783409+0000", 2023-09-23T20:55:35.931 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 50.090492230999999, 2023-09-23T20:55:35.931 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.112445293, 2023-09-23T20:55:35.931 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T20:55:35.931 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:55:35.931 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T20:55:35.931 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34797", 2023-09-23T20:55:35.931 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T20:55:35.931 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 140393 2023-09-23T20:55:35.931 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:35.931 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T20:55:35.931 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:35.931 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T20:55:35.931 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:45.783409+0000", 2023-09-23T20:55:35.932 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:55:35.932 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:35.932 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:35.932 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T20:55:35.932 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:45.783409+0000", 2023-09-23T20:55:35.932 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:55:35.932 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:35.932 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:35.932 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T20:55:35.932 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:45.783408+0000", 2023-09-23T20:55:35.932 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T20:55:35.932 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:35.933 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:35.933 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T20:55:35.933 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:45.783411+0000", 2023-09-23T20:55:35.933 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.8040000000000002e-06 2023-09-23T20:55:35.933 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:35.933 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:35.933 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T20:55:35.933 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:45.783417+0000", 2023-09-23T20:55:35.933 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.195e-06 2023-09-23T20:55:35.933 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:35.933 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:35.933 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T20:55:35.934 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:45.783422+0000", 2023-09-23T20:55:35.934 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.9640000000000002e-06 2023-09-23T20:55:35.934 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:35.934 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:35.934 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:55:35.934 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:45.789794+0000", 2023-09-23T20:55:35.934 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.006371402 2023-09-23T20:55:35.934 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:35.934 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:35.934 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T20:55:35.934 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:45.789914+0000", 2023-09-23T20:55:35.934 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000120188 2023-09-23T20:55:35.935 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:35.935 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:35.935 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 2,5,6", 2023-09-23T20:55:35.935 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:45.790030+0000", 2023-09-23T20:55:35.935 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000115869 2023-09-23T20:55:35.935 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:35.935 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:35.935 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:55:35.935 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:45.796717+0000", 2023-09-23T20:55:35.935 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0066874689999999997 2023-09-23T20:55:35.935 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:35.935 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:35.936 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:55:35.936 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:45.796750+0000", 2023-09-23T20:55:35.936 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.2654999999999998e-05 2023-09-23T20:55:35.936 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:35.936 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:35.936 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:55:35.936 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:45.895692+0000", 2023-09-23T20:55:35.936 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.098942046000000006 2023-09-23T20:55:35.936 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:35.936 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:35.936 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T20:55:35.936 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:45.895704+0000", 2023-09-23T20:55:35.936 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.2595000000000001e-05 2023-09-23T20:55:35.937 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:35.937 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:35.937 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T20:55:35.937 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:45.895747+0000", 2023-09-23T20:55:35.937 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.2404999999999998e-05 2023-09-23T20:55:35.937 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:35.937 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:35.937 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T20:55:35.937 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:45.895854+0000", 2023-09-23T20:55:35.937 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000107089 2023-09-23T20:55:35.937 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:55:35.938 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T20:55:35.938 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:55:35.938 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:35.938 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:35.938 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34797.0:140395 6.d 6:b0af5d57:::benchmark_data_smithi026_223868_object39269:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e143)", 2023-09-23T20:55:35.938 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T20:54:45.786548+0000", 2023-09-23T20:55:35.938 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 50.087352973999998, 2023-09-23T20:55:35.938 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.109415499, 2023-09-23T20:55:35.938 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T20:55:35.938 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:55:35.938 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T20:55:35.938 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34797", 2023-09-23T20:55:35.939 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T20:55:35.939 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 140395 2023-09-23T20:55:35.939 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:35.939 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T20:55:35.939 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:35.939 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T20:55:35.939 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:45.786548+0000", 2023-09-23T20:55:35.939 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:55:35.939 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:35.939 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:35.939 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T20:55:35.939 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:45.786548+0000", 2023-09-23T20:55:35.939 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:55:35.940 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:35.940 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:35.940 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T20:55:35.940 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:45.786546+0000", 2023-09-23T20:55:35.940 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T20:55:35.940 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:35.940 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:35.940 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T20:55:35.940 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:45.786553+0000", 2023-09-23T20:55:35.940 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.8689999999999998e-06 2023-09-23T20:55:35.940 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:35.940 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:35.941 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T20:55:35.941 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:45.786565+0000", 2023-09-23T20:55:35.941 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.2061e-05 2023-09-23T20:55:35.941 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:35.941 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:35.941 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T20:55:35.941 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:45.786573+0000", 2023-09-23T20:55:35.941 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.0600000000000008e-06 2023-09-23T20:55:35.941 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:35.941 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:35.941 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:55:35.941 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:45.786622+0000", 2023-09-23T20:55:35.941 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.8930000000000001e-05 2023-09-23T20:55:35.942 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:35.942 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:35.942 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T20:55:35.942 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:45.786762+0000", 2023-09-23T20:55:35.942 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00013947499999999999 2023-09-23T20:55:35.942 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:35.942 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:35.942 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 1,2,5", 2023-09-23T20:55:35.942 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:45.786898+0000", 2023-09-23T20:55:35.942 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000136293 2023-09-23T20:55:35.942 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:35.942 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:35.943 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:55:35.943 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:45.793986+0000", 2023-09-23T20:55:35.943 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0070876940000000003 2023-09-23T20:55:35.943 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:35.943 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:35.943 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:55:35.943 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:45.891876+0000", 2023-09-23T20:55:35.943 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.097890061 2023-09-23T20:55:35.943 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:35.943 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:35.943 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T20:55:35.943 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:45.891888+0000", 2023-09-23T20:55:35.944 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.2461e-05 2023-09-23T20:55:35.944 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:35.944 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:35.944 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:55:35.944 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:45.895808+0000", 2023-09-23T20:55:35.944 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.003919624 2023-09-23T20:55:35.944 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:35.944 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:35.944 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T20:55:35.944 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:45.895852+0000", 2023-09-23T20:55:35.944 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.3849e-05 2023-09-23T20:55:35.944 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:35.945 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:35.945 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T20:55:35.945 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:45.895963+0000", 2023-09-23T20:55:35.945 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000111512 2023-09-23T20:55:35.945 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:55:35.945 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T20:55:35.945 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:55:35.945 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:35.945 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:35.945 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34797.0:140398 6.0 6:04d5d8ea:::benchmark_data_smithi026_223868_object39272:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e143)", 2023-09-23T20:55:35.945 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T20:54:45.786600+0000", 2023-09-23T20:55:35.945 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 50.087300947000003, 2023-09-23T20:55:35.946 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.107835421, 2023-09-23T20:55:35.946 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T20:55:35.946 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:55:35.946 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T20:55:35.946 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34797", 2023-09-23T20:55:35.946 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T20:55:35.946 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 140398 2023-09-23T20:55:35.946 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:35.946 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T20:55:35.946 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:35.946 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T20:55:35.946 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:45.786600+0000", 2023-09-23T20:55:35.947 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:55:35.947 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:35.947 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:35.947 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T20:55:35.947 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:45.786600+0000", 2023-09-23T20:55:35.947 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:55:35.947 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:35.947 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:35.947 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T20:55:35.947 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:45.786599+0000", 2023-09-23T20:55:35.947 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T20:55:35.947 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:35.948 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:35.948 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T20:55:35.948 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:45.786603+0000", 2023-09-23T20:55:35.948 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.7400000000000002e-06 2023-09-23T20:55:35.948 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:35.948 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:35.948 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T20:55:35.948 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:45.786611+0000", 2023-09-23T20:55:35.948 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.011e-06 2023-09-23T20:55:35.948 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:35.948 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:35.948 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T20:55:35.948 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:45.786617+0000", 2023-09-23T20:55:35.949 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.8359999999999997e-06 2023-09-23T20:55:35.949 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:35.949 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:35.949 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:55:35.949 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:45.786684+0000", 2023-09-23T20:55:35.949 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.6851000000000001e-05 2023-09-23T20:55:35.949 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:35.949 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:35.949 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T20:55:35.949 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:45.786797+0000", 2023-09-23T20:55:35.949 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000113176 2023-09-23T20:55:35.950 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:35.950 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:35.950 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 2,3", 2023-09-23T20:55:35.950 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:45.786899+0000", 2023-09-23T20:55:35.950 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000101882 2023-09-23T20:55:35.950 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:35.950 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:35.950 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T20:55:35.950 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:45.791274+0000", 2023-09-23T20:55:35.950 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0043755479999999999 2023-09-23T20:55:35.950 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:35.950 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:35.950 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:55:35.951 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:45.795699+0000", 2023-09-23T20:55:35.951 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0044250419999999997 2023-09-23T20:55:35.951 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:35.951 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:35.951 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:55:35.951 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:45.894358+0000", 2023-09-23T20:55:35.951 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.098658750000000003 2023-09-23T20:55:35.951 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:35.951 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:35.951 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T20:55:35.951 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:45.894397+0000", 2023-09-23T20:55:35.951 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.8659999999999999e-05 2023-09-23T20:55:35.952 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:35.952 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:35.952 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T20:55:35.952 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:45.894435+0000", 2023-09-23T20:55:35.952 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.8492999999999999e-05 2023-09-23T20:55:35.952 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:55:35.952 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T20:55:35.952 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:55:35.952 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:35.952 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:35.952 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34797.0:140400 6.10 6:0fb3a641:::benchmark_data_smithi026_223868_object39274:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e143)", 2023-09-23T20:55:35.952 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T20:54:45.786629+0000", 2023-09-23T20:55:35.952 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 50.087272173000002, 2023-09-23T20:55:35.953 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.10810805900000001, 2023-09-23T20:55:35.953 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T20:55:35.953 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:55:35.953 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T20:55:35.953 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34797", 2023-09-23T20:55:35.953 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T20:55:35.953 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 140400 2023-09-23T20:55:35.953 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:35.953 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T20:55:35.953 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:35.953 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T20:55:35.954 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:45.786629+0000", 2023-09-23T20:55:35.954 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:55:35.954 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:35.954 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:35.954 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T20:55:35.954 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:45.786629+0000", 2023-09-23T20:55:35.954 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:55:35.954 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:35.954 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:35.954 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T20:55:35.954 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:45.786628+0000", 2023-09-23T20:55:35.954 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T20:55:35.955 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:35.955 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:35.955 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T20:55:35.955 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:45.786631+0000", 2023-09-23T20:55:35.955 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.5950000000000001e-06 2023-09-23T20:55:35.955 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:35.955 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:35.955 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T20:55:35.955 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:45.786637+0000", 2023-09-23T20:55:35.955 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.2439999999999998e-06 2023-09-23T20:55:35.955 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:35.955 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:35.955 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T20:55:35.956 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:45.786642+0000", 2023-09-23T20:55:35.956 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.0019999999999997e-06 2023-09-23T20:55:35.956 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:35.956 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:35.956 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:55:35.956 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:45.789078+0000", 2023-09-23T20:55:35.956 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.002436096 2023-09-23T20:55:35.956 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:35.956 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:35.956 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T20:55:35.956 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:45.789184+0000", 2023-09-23T20:55:35.956 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00010568 2023-09-23T20:55:35.957 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:35.957 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:35.957 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 2,3", 2023-09-23T20:55:35.957 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:45.789256+0000", 2023-09-23T20:55:35.957 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.1936999999999999e-05 2023-09-23T20:55:35.957 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:35.957 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:35.957 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:55:35.957 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:45.795720+0000", 2023-09-23T20:55:35.957 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0064636900000000002 2023-09-23T20:55:35.957 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:35.957 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:35.958 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:55:35.958 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:45.894646+0000", 2023-09-23T20:55:35.958 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.098926401999999997 2023-09-23T20:55:35.958 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:35.958 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:35.958 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T20:55:35.958 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:45.894659+0000", 2023-09-23T20:55:35.958 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.2986000000000001e-05 2023-09-23T20:55:35.958 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:35.958 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:35.958 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T20:55:35.958 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:45.894699+0000", 2023-09-23T20:55:35.959 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.0263999999999999e-05 2023-09-23T20:55:35.959 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:35.959 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:35.959 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T20:55:35.959 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:45.894737+0000", 2023-09-23T20:55:35.959 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.7533000000000001e-05 2023-09-23T20:55:35.959 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:55:35.959 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T20:55:35.959 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:55:35.959 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:35.959 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:35.959 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_repop_reply(client.34797.0:140395 6.d e143/137 ondisk, result = 0)", 2023-09-23T20:55:35.960 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T20:54:45.797020+0000", 2023-09-23T20:55:35.960 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 50.076881098000001, 2023-09-23T20:55:35.960 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.094869516000000001, 2023-09-23T20:55:35.960 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T20:55:35.960 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "started", 2023-09-23T20:55:35.960 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T20:55:35.960 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:35.960 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T20:55:35.960 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:45.797020+0000", 2023-09-23T20:55:35.960 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:55:35.960 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:35.960 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:35.960 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T20:55:35.961 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:45.797020+0000", 2023-09-23T20:55:35.961 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:55:35.961 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:35.961 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:35.961 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T20:55:35.961 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:45.797019+0000", 2023-09-23T20:55:35.961 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T20:55:35.961 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:35.961 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:35.961 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T20:55:35.961 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:45.797024+0000", 2023-09-23T20:55:35.961 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.7469999999999996e-06 2023-09-23T20:55:35.961 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:35.962 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:35.962 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T20:55:35.962 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:45.797031+0000", 2023-09-23T20:55:35.962 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.3499999999999999e-06 2023-09-23T20:55:35.962 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:35.962 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:35.962 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T20:55:35.962 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:45.797036+0000", 2023-09-23T20:55:35.962 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.7829999999999998e-06 2023-09-23T20:55:35.962 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:35.962 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:35.962 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:55:35.963 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:45.891858+0000", 2023-09-23T20:55:35.963 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.094822080000000003 2023-09-23T20:55:35.963 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:35.963 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:35.963 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T20:55:35.963 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:45.891868+0000", 2023-09-23T20:55:35.963 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0237e-05 2023-09-23T20:55:35.963 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:35.963 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:35.963 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T20:55:35.963 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:45.891889+0000", 2023-09-23T20:55:35.963 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.09e-05 2023-09-23T20:55:35.963 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:55:35.964 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T20:55:35.964 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:55:35.964 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:35.964 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:35.964 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34797.0:140402 6.10 6:0f3ec742:::benchmark_data_smithi026_223868_object39276:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e143)", 2023-09-23T20:55:35.964 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T20:54:45.797052+0000", 2023-09-23T20:55:35.964 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 50.076849353, 2023-09-23T20:55:35.964 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.098551920000000001, 2023-09-23T20:55:35.964 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T20:55:35.964 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:55:35.964 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T20:55:35.964 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34797", 2023-09-23T20:55:35.964 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T20:55:35.965 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 140402 2023-09-23T20:55:35.965 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:35.965 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T20:55:35.965 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:35.965 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T20:55:35.965 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:45.797052+0000", 2023-09-23T20:55:35.965 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:55:35.965 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:35.965 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:35.965 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T20:55:35.965 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:45.797052+0000", 2023-09-23T20:55:35.965 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:55:35.966 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:35.966 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:35.966 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T20:55:35.966 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:45.797050+0000", 2023-09-23T20:55:35.966 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.999999 2023-09-23T20:55:35.966 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:35.966 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:35.966 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T20:55:35.966 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:45.797055+0000", 2023-09-23T20:55:35.966 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.9949999999999996e-06 2023-09-23T20:55:35.966 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:35.967 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:35.967 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T20:55:35.967 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:45.797061+0000", 2023-09-23T20:55:35.967 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.8699999999999997e-06 2023-09-23T20:55:35.967 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:35.967 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:35.967 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T20:55:35.967 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:45.797066+0000", 2023-09-23T20:55:35.967 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.3839999999999999e-06 2023-09-23T20:55:35.967 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:35.967 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:35.967 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:55:35.967 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:45.891773+0000", 2023-09-23T20:55:35.968 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.094706980999999996 2023-09-23T20:55:35.968 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:35.968 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:35.968 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T20:55:35.968 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:45.891916+0000", 2023-09-23T20:55:35.968 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000142991 2023-09-23T20:55:35.968 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:35.968 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:35.968 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 2,3", 2023-09-23T20:55:35.968 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:45.892041+0000", 2023-09-23T20:55:35.968 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00012508500000000001 2023-09-23T20:55:35.968 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:35.968 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:35.969 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T20:55:35.969 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:45.895034+0000", 2023-09-23T20:55:35.969 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0029935560000000001 2023-09-23T20:55:35.969 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:35.969 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:35.969 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:55:35.969 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:45.895336+0000", 2023-09-23T20:55:35.969 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00030145500000000001 2023-09-23T20:55:35.969 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:35.969 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:35.969 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:55:35.969 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:45.895529+0000", 2023-09-23T20:55:35.969 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000193656 2023-09-23T20:55:35.970 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:35.970 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:35.970 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T20:55:35.970 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:45.895569+0000", 2023-09-23T20:55:35.970 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.9227999999999998e-05 2023-09-23T20:55:35.970 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:35.970 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:35.970 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T20:55:35.970 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:45.895603+0000", 2023-09-23T20:55:35.970 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.481e-05 2023-09-23T20:55:35.970 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:55:35.970 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T20:55:35.970 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:55:35.971 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:35.971 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:35.971 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34797.0:140432 6.d 6:bb68ee42:::benchmark_data_smithi026_223868_object39306:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e143)", 2023-09-23T20:55:35.971 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T20:54:45.821356+0000", 2023-09-23T20:55:35.971 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 50.052544884, 2023-09-23T20:55:35.971 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.085177881999999996, 2023-09-23T20:55:35.971 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T20:55:35.971 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:55:35.971 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T20:55:35.971 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34797", 2023-09-23T20:55:35.971 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T20:55:35.971 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 140432 2023-09-23T20:55:35.972 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:35.972 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T20:55:35.972 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:35.972 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T20:55:35.972 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:45.821356+0000", 2023-09-23T20:55:35.972 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:55:35.972 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:35.972 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:35.972 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T20:55:35.972 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:45.821356+0000", 2023-09-23T20:55:35.972 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:55:35.972 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:35.972 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:35.973 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T20:55:35.973 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:45.821354+0000", 2023-09-23T20:55:35.973 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999981 2023-09-23T20:55:35.973 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:35.973 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:35.973 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T20:55:35.973 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:45.821364+0000", 2023-09-23T20:55:35.973 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.6579999999999997e-06 2023-09-23T20:55:35.973 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:35.973 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:35.973 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T20:55:35.973 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:45.891751+0000", 2023-09-23T20:55:35.974 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.070387363999999994 2023-09-23T20:55:35.974 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:35.974 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:35.974 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T20:55:35.974 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:45.891765+0000", 2023-09-23T20:55:35.974 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.4270999999999999e-05 2023-09-23T20:55:35.974 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:35.974 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:35.974 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:55:35.974 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:45.891952+0000", 2023-09-23T20:55:35.974 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00018618399999999999 2023-09-23T20:55:35.974 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:35.975 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:35.975 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T20:55:35.975 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:45.892077+0000", 2023-09-23T20:55:35.975 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00012585999999999999 2023-09-23T20:55:35.975 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:35.975 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:35.975 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 1,2,5", 2023-09-23T20:55:35.975 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:45.892202+0000", 2023-09-23T20:55:35.975 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00012434999999999999 2023-09-23T20:55:35.975 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:35.975 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:35.975 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:55:35.976 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:45.896005+0000", 2023-09-23T20:55:35.976 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.003803772 2023-09-23T20:55:35.976 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:35.976 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:35.976 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T20:55:35.976 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:45.900054+0000", 2023-09-23T20:55:35.976 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0040481950000000001 2023-09-23T20:55:35.976 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:35.976 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:35.976 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:55:35.976 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:45.903088+0000", 2023-09-23T20:55:35.976 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0030344130000000001 2023-09-23T20:55:35.977 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:35.977 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:35.977 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:55:35.977 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:45.906342+0000", 2023-09-23T20:55:35.977 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0032535680000000001 2023-09-23T20:55:35.977 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:35.977 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:35.977 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T20:55:35.977 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:45.906403+0000", 2023-09-23T20:55:35.977 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.1470999999999995e-05 2023-09-23T20:55:35.977 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:35.977 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:35.977 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T20:55:35.978 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:45.906534+0000", 2023-09-23T20:55:35.978 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00013072499999999999 2023-09-23T20:55:35.978 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:55:35.978 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T20:55:35.978 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:55:35.978 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:35.978 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:35.978 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34797.0:154830 6.19 6:9817a9bc:::benchmark_data_smithi026_223868_object53704:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e149)", 2023-09-23T20:55:35.978 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T20:55:05.245538+0000", 2023-09-23T20:55:35.978 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 30.628363145000002, 2023-09-23T20:55:35.978 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.082167192999999999, 2023-09-23T20:55:35.979 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T20:55:35.979 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:55:35.979 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T20:55:35.979 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34797", 2023-09-23T20:55:35.979 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T20:55:35.979 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 154830 2023-09-23T20:55:35.979 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:35.979 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T20:55:35.979 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:35.979 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T20:55:35.979 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:05.245538+0000", 2023-09-23T20:55:35.979 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:55:35.980 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:35.980 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:35.980 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T20:55:35.980 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:05.245538+0000", 2023-09-23T20:55:35.980 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:55:35.980 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:35.980 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:35.980 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T20:55:35.980 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:05.245536+0000", 2023-09-23T20:55:35.980 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T20:55:35.980 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:35.980 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:35.980 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T20:55:35.981 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:05.245543+0000", 2023-09-23T20:55:35.981 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.0809999999999996e-06 2023-09-23T20:55:35.981 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:35.981 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:35.981 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T20:55:35.981 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:05.245555+0000", 2023-09-23T20:55:35.981 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1521e-05 2023-09-23T20:55:35.981 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:35.981 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:35.981 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T20:55:35.981 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:05.245563+0000", 2023-09-23T20:55:35.981 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.7980000000000006e-06 2023-09-23T20:55:35.982 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:35.982 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:35.982 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:55:35.982 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:05.246372+0000", 2023-09-23T20:55:35.982 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00080991400000000001 2023-09-23T20:55:35.982 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:35.982 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:35.982 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T20:55:35.982 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:05.246489+0000", 2023-09-23T20:55:35.982 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000116039 2023-09-23T20:55:35.982 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:35.982 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:35.983 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 2,7", 2023-09-23T20:55:35.983 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:05.246604+0000", 2023-09-23T20:55:35.983 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00011507599999999999 2023-09-23T20:55:35.983 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:35.983 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:35.983 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T20:55:35.983 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:05.254816+0000", 2023-09-23T20:55:35.983 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0082124190000000003 2023-09-23T20:55:35.983 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:35.983 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:35.983 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:55:35.983 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:05.254868+0000", 2023-09-23T20:55:35.983 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.2172999999999997e-05 2023-09-23T20:55:35.984 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:35.984 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:35.984 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:55:35.984 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:05.327610+0000", 2023-09-23T20:55:35.984 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.072741537999999994 2023-09-23T20:55:35.984 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:35.984 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:35.984 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T20:55:35.984 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:05.327661+0000", 2023-09-23T20:55:35.984 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.1758000000000001e-05 2023-09-23T20:55:35.984 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:35.984 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:35.985 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T20:55:35.985 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:05.327705+0000", 2023-09-23T20:55:35.985 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.3427000000000003e-05 2023-09-23T20:55:35.985 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:55:35.985 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T20:55:35.985 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:55:35.985 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:35.985 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:35.985 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34797.0:154833 6.19 6:9f32b4c5:::benchmark_data_smithi026_223868_object53707:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e149)", 2023-09-23T20:55:35.985 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T20:55:05.245578+0000", 2023-09-23T20:55:35.985 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 30.628323122000001, 2023-09-23T20:55:35.985 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.082331802999999995, 2023-09-23T20:55:35.986 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T20:55:35.986 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:55:35.986 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T20:55:35.986 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34797", 2023-09-23T20:55:35.986 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T20:55:35.986 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 154833 2023-09-23T20:55:35.986 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:35.986 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T20:55:35.986 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:35.986 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T20:55:35.986 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:05.245578+0000", 2023-09-23T20:55:35.986 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:55:35.986 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:35.987 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:35.987 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T20:55:35.987 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:05.245578+0000", 2023-09-23T20:55:35.987 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:55:35.987 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:35.987 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:35.987 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T20:55:35.987 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:05.245577+0000", 2023-09-23T20:55:35.987 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T20:55:35.987 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:35.987 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:35.987 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T20:55:35.988 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:05.245581+0000", 2023-09-23T20:55:35.988 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.2789999999999999e-06 2023-09-23T20:55:35.988 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:35.988 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:35.988 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T20:55:35.988 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:05.245588+0000", 2023-09-23T20:55:35.988 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.2239999999999998e-06 2023-09-23T20:55:35.988 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:35.988 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:35.988 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T20:55:35.988 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:05.245593+0000", 2023-09-23T20:55:35.988 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.6969999999999998e-06 2023-09-23T20:55:35.989 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:35.989 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:35.989 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:55:35.989 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:05.248618+0000", 2023-09-23T20:55:35.989 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0030243689999999998 2023-09-23T20:55:35.989 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:35.989 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:35.989 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T20:55:35.989 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:05.248731+0000", 2023-09-23T20:55:35.989 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000113027 2023-09-23T20:55:35.989 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:35.989 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:35.989 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 2,7", 2023-09-23T20:55:35.990 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:05.248837+0000", 2023-09-23T20:55:35.990 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000105677 2023-09-23T20:55:35.990 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:35.990 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:35.990 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T20:55:35.990 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:05.254824+0000", 2023-09-23T20:55:35.990 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0059876299999999999 2023-09-23T20:55:35.990 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:35.990 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:35.990 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:55:35.990 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:05.327746+0000", 2023-09-23T20:55:35.990 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.072921447 2023-09-23T20:55:35.991 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:35.991 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:35.991 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:55:35.991 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:05.327837+0000", 2023-09-23T20:55:35.991 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.1847000000000003e-05 2023-09-23T20:55:35.991 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:35.991 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:35.991 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T20:55:35.991 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:05.327877+0000", 2023-09-23T20:55:35.991 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.9684000000000002e-05 2023-09-23T20:55:35.991 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:35.991 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:35.992 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T20:55:35.992 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:05.327910+0000", 2023-09-23T20:55:35.992 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.2379999999999998e-05 2023-09-23T20:55:35.992 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:55:35.992 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T20:55:35.992 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:55:35.992 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:35.992 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:35.992 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34797.0:154836 6.19 6:99a00d2c:::benchmark_data_smithi026_223868_object53710:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e149)", 2023-09-23T20:55:35.992 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T20:55:05.245598+0000", 2023-09-23T20:55:35.992 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 30.628302576999999, 2023-09-23T20:55:35.992 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.082404851000000001, 2023-09-23T20:55:35.993 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T20:55:35.993 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:55:35.993 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T20:55:35.993 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34797", 2023-09-23T20:55:35.993 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T20:55:35.993 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 154836 2023-09-23T20:55:35.993 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:35.993 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T20:55:35.993 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:35.993 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T20:55:35.993 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:05.245598+0000", 2023-09-23T20:55:35.993 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:55:35.994 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:35.994 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:35.994 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T20:55:35.994 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:05.245598+0000", 2023-09-23T20:55:35.994 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:55:35.994 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:35.994 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:35.994 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T20:55:35.994 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:05.245598+0000", 2023-09-23T20:55:35.994 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T20:55:35.994 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:35.994 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:35.995 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T20:55:35.995 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:05.245600+0000", 2023-09-23T20:55:35.995 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.5569999999999998e-06 2023-09-23T20:55:35.995 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:35.995 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:35.995 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T20:55:35.995 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:05.245606+0000", 2023-09-23T20:55:35.995 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.0159999999999997e-06 2023-09-23T20:55:35.995 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:35.995 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:35.995 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T20:55:35.995 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:05.245612+0000", 2023-09-23T20:55:35.996 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.2989999999999996e-06 2023-09-23T20:55:35.996 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:35.996 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:35.996 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:55:35.996 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:05.250688+0000", 2023-09-23T20:55:35.996 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0050764590000000002 2023-09-23T20:55:35.996 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:35.996 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:35.996 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T20:55:35.996 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:05.250805+0000", 2023-09-23T20:55:35.996 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000116446 2023-09-23T20:55:35.996 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:35.996 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:35.997 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 2,7", 2023-09-23T20:55:35.997 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:05.250896+0000", 2023-09-23T20:55:35.997 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.1044000000000004e-05 2023-09-23T20:55:35.997 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:35.997 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:35.997 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T20:55:35.997 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:05.254831+0000", 2023-09-23T20:55:35.997 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0039353390000000004 2023-09-23T20:55:35.997 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:35.997 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:35.997 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:55:35.997 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:05.327777+0000", 2023-09-23T20:55:35.997 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.072945514000000003 2023-09-23T20:55:35.998 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:35.998 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:35.998 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:55:35.998 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:05.327941+0000", 2023-09-23T20:55:35.998 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00016408800000000001 2023-09-23T20:55:35.998 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:35.998 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:35.998 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T20:55:35.998 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:05.327973+0000", 2023-09-23T20:55:35.998 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.252e-05 2023-09-23T20:55:35.998 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:35.998 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:35.999 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T20:55:35.999 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:05.328003+0000", 2023-09-23T20:55:35.999 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.9966000000000001e-05 2023-09-23T20:55:35.999 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:55:35.999 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T20:55:35.999 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:55:35.999 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:35.999 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:35.999 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34797.0:154842 6.a 6:5ae508f9:::benchmark_data_smithi026_223868_object53716:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e149)", 2023-09-23T20:55:35.999 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T20:55:05.248640+0000", 2023-09-23T20:55:35.999 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 30.625260745999999, 2023-09-23T20:55:35.999 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.081250508999999999, 2023-09-23T20:55:36.000 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T20:55:36.000 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:55:36.000 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T20:55:36.000 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34797", 2023-09-23T20:55:36.000 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T20:55:36.000 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 154842 2023-09-23T20:55:36.000 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:36.000 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T20:55:36.000 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:36.000 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T20:55:36.000 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:05.248640+0000", 2023-09-23T20:55:36.000 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:55:36.000 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:36.001 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:36.001 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T20:55:36.001 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:05.248640+0000", 2023-09-23T20:55:36.001 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:55:36.001 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:36.001 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:36.001 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T20:55:36.001 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:05.248640+0000", 2023-09-23T20:55:36.001 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T20:55:36.001 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:36.001 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:36.001 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T20:55:36.002 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:05.248642+0000", 2023-09-23T20:55:36.002 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.8100000000000002e-06 2023-09-23T20:55:36.002 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:36.002 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:36.002 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T20:55:36.002 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:05.248650+0000", 2023-09-23T20:55:36.002 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.221e-06 2023-09-23T20:55:36.002 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:36.002 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:36.002 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T20:55:36.002 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:05.248655+0000", 2023-09-23T20:55:36.003 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.592e-06 2023-09-23T20:55:36.003 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:36.003 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:36.003 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:55:36.003 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:05.248704+0000", 2023-09-23T20:55:36.003 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.8767000000000003e-05 2023-09-23T20:55:36.003 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:36.003 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:36.003 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T20:55:36.003 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:05.248860+0000", 2023-09-23T20:55:36.003 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00015572000000000001 2023-09-23T20:55:36.003 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:36.004 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:36.004 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 2,5,6", 2023-09-23T20:55:36.004 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:05.248974+0000", 2023-09-23T20:55:36.004 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000114257 2023-09-23T20:55:36.004 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:36.004 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:36.004 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:55:36.004 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:05.329609+0000", 2023-09-23T20:55:36.004 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.080635381000000006 2023-09-23T20:55:36.004 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:36.004 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:36.004 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T20:55:36.005 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:05.329660+0000", 2023-09-23T20:55:36.005 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.0334999999999998e-05 2023-09-23T20:55:36.005 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:36.005 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:36.005 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:55:36.005 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:05.329697+0000", 2023-09-23T20:55:36.005 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.7549999999999998e-05 2023-09-23T20:55:36.005 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:36.005 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:36.005 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:55:36.005 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:05.329729+0000", 2023-09-23T20:55:36.005 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.1829999999999998e-05 2023-09-23T20:55:36.005 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:36.006 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:36.006 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T20:55:36.006 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:05.329780+0000", 2023-09-23T20:55:36.006 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.1359000000000001e-05 2023-09-23T20:55:36.006 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:36.006 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:36.006 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T20:55:36.006 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:05.329891+0000", 2023-09-23T20:55:36.006 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00011021700000000001 2023-09-23T20:55:36.006 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:55:36.006 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T20:55:36.007 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:55:36.007 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:36.007 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:36.007 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34797.0:154843 6.10 6:0bfabc78:::benchmark_data_smithi026_223868_object53717:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e149)", 2023-09-23T20:55:36.007 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T20:55:05.248697+0000", 2023-09-23T20:55:36.007 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 30.625203959, 2023-09-23T20:55:36.007 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.091418342, 2023-09-23T20:55:36.007 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T20:55:36.007 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:55:36.007 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T20:55:36.007 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34797", 2023-09-23T20:55:36.007 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T20:55:36.007 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 154843 2023-09-23T20:55:36.008 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:36.008 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T20:55:36.008 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:36.008 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T20:55:36.008 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:05.248697+0000", 2023-09-23T20:55:36.008 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:55:36.008 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:36.008 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:36.008 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T20:55:36.008 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:05.248697+0000", 2023-09-23T20:55:36.008 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:55:36.008 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:36.009 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:36.009 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T20:55:36.009 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:05.248696+0000", 2023-09-23T20:55:36.009 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T20:55:36.009 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:36.009 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:36.009 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T20:55:36.009 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:05.248700+0000", 2023-09-23T20:55:36.009 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.0600000000000001e-06 2023-09-23T20:55:36.009 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:36.009 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:36.009 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T20:55:36.009 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:05.248709+0000", 2023-09-23T20:55:36.010 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.8599999999999999e-06 2023-09-23T20:55:36.010 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:36.010 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:36.010 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T20:55:36.010 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:05.248716+0000", 2023-09-23T20:55:36.010 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.5309999999999998e-06 2023-09-23T20:55:36.010 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:36.010 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:36.010 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:55:36.010 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:05.248773+0000", 2023-09-23T20:55:36.010 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.7587000000000003e-05 2023-09-23T20:55:36.010 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:36.011 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:36.011 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T20:55:36.011 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:05.248881+0000", 2023-09-23T20:55:36.011 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000107303 2023-09-23T20:55:36.011 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:36.011 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:36.011 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 2,3", 2023-09-23T20:55:36.011 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:05.248987+0000", 2023-09-23T20:55:36.011 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000106155 2023-09-23T20:55:36.011 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:36.011 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:36.011 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T20:55:36.012 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:05.251393+0000", 2023-09-23T20:55:36.012 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0024064569999999999 2023-09-23T20:55:36.012 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:36.012 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:36.012 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:55:36.012 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:05.253619+0000", 2023-09-23T20:55:36.012 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0022254990000000001 2023-09-23T20:55:36.012 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:36.012 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:36.012 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:55:36.012 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:05.340027+0000", 2023-09-23T20:55:36.012 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.086408251000000005 2023-09-23T20:55:36.012 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:36.013 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:36.013 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T20:55:36.013 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:05.340078+0000", 2023-09-23T20:55:36.013 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.1434999999999998e-05 2023-09-23T20:55:36.013 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:36.013 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:36.013 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T20:55:36.013 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:05.340115+0000", 2023-09-23T20:55:36.013 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.6831e-05 2023-09-23T20:55:36.013 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:55:36.013 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T20:55:36.013 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:55:36.013 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:36.014 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:36.014 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34797.0:165124 6.a 6:524acc8d:::benchmark_data_smithi026_223868_object63998:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e155)", 2023-09-23T20:55:36.014 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T20:55:17.472537+0000", 2023-09-23T20:55:36.014 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 18.401363641, 2023-09-23T20:55:36.014 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.066343695, 2023-09-23T20:55:36.014 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T20:55:36.014 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:55:36.014 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T20:55:36.014 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34797", 2023-09-23T20:55:36.014 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T20:55:36.014 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 165124 2023-09-23T20:55:36.015 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:36.015 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T20:55:36.015 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:36.015 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T20:55:36.015 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:17.472537+0000", 2023-09-23T20:55:36.015 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:55:36.015 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:36.015 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:36.015 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T20:55:36.015 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:17.472537+0000", 2023-09-23T20:55:36.015 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:55:36.015 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:36.015 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:36.016 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T20:55:36.016 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:17.472536+0000", 2023-09-23T20:55:36.016 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T20:55:36.016 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:36.016 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:36.016 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T20:55:36.016 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:17.472541+0000", 2023-09-23T20:55:36.016 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.9130000000000002e-06 2023-09-23T20:55:36.016 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:36.016 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:36.016 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T20:55:36.016 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:17.472550+0000", 2023-09-23T20:55:36.017 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.1980000000000007e-06 2023-09-23T20:55:36.017 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:36.017 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:36.017 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T20:55:36.017 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:17.472559+0000", 2023-09-23T20:55:36.017 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.3500000000000003e-06 2023-09-23T20:55:36.017 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:36.017 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:36.017 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:55:36.017 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:17.472925+0000", 2023-09-23T20:55:36.017 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00036605699999999998 2023-09-23T20:55:36.017 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:36.018 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:36.018 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op must wait for map", 2023-09-23T20:55:36.018 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:17.472933+0000", 2023-09-23T20:55:36.018 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.8269999999999998e-06 2023-09-23T20:55:36.018 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:36.018 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:36.018 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:55:36.018 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:17.481419+0000", 2023-09-23T20:55:36.018 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0084858759999999998 2023-09-23T20:55:36.018 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:36.018 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:36.018 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T20:55:36.019 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:17.481429+0000", 2023-09-23T20:55:36.019 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.005e-05 2023-09-23T20:55:36.019 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:36.019 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:36.019 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:55:36.019 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:18.520137+0000", 2023-09-23T20:55:36.019 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0387079619999999 2023-09-23T20:55:36.019 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:36.019 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:36.019 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T20:55:36.019 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:18.520295+0000", 2023-09-23T20:55:36.019 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000157474 2023-09-23T20:55:36.019 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:36.020 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:36.020 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 2,5", 2023-09-23T20:55:36.020 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:18.520450+0000", 2023-09-23T20:55:36.020 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00015505200000000001 2023-09-23T20:55:36.020 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:36.020 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:36.020 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:55:36.020 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:18.524869+0000", 2023-09-23T20:55:36.020 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0044197100000000003 2023-09-23T20:55:36.020 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:36.020 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:36.020 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T20:55:36.020 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:18.532212+0000", 2023-09-23T20:55:36.021 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0073423100000000003 2023-09-23T20:55:36.021 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:36.021 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:36.021 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:55:36.021 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:18.538723+0000", 2023-09-23T20:55:36.021 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0065114609999999996 2023-09-23T20:55:36.021 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:36.021 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:36.021 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T20:55:36.021 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:18.538778+0000", 2023-09-23T20:55:36.021 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.4873000000000001e-05 2023-09-23T20:55:36.021 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:36.022 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:36.022 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T20:55:36.022 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:18.538881+0000", 2023-09-23T20:55:36.022 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000102976 2023-09-23T20:55:36.022 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:55:36.022 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T20:55:36.022 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:55:36.022 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:55:36.022 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T20:55:36.022 INFO:teuthology.orchestra.run.smithi026.stdout:} 2023-09-23T20:55:36.023 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:55:36.024 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:55:36.024 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:55:36.024 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:55:36.024 INFO:teuthology.orchestra.run.smithi026.stderr:nodeep-scrub is unset 2023-09-23T20:55:36.125 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:55:36.131 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:55:36 smithi026 ceph-mon[179688]: pgmap v396: 109 pgs: 1 active+undersized+degraded+remapped+backfilling, 3 active+clean+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 31 active+undersized, 60 active+clean, 11 active+undersized+degraded, 2 active+remapped+backfill_wait; 1.6 GiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 258 op/s; 5586/78612 objects degraded (7.106%); 4814/78612 objects misplaced (6.124%); 12 MiB/s, 195 objects/s recovering 2023-09-23T20:55:36.131 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:55:36 smithi026 ceph-mon[179688]: failed to encode map e164 with expected crc 2023-09-23T20:55:36.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:55:36 smithi026 ceph-mon[179688]: failed to encode map e164 with expected crc 2023-09-23T20:55:36.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:55:36 smithi026 ceph-mon[179688]: 6.18 scrub starts 2023-09-23T20:55:36.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:55:36 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/3727760315' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:55:36.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:55:36 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:55:36.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:55:36 smithi026 ceph-mon[179688]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T20:55:36.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:55:36 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T20:55:36.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:55:36 smithi026 ceph-mon[179688]: osdmap e165: 8 total, 8 up, 8 in 2023-09-23T20:55:36.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:55:36 smithi026 ceph-mon[179688]: failed to encode map e165 with expected crc 2023-09-23T20:55:36.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:55:36 smithi026 ceph-mon[175151]: pgmap v396: 109 pgs: 1 active+undersized+degraded+remapped+backfilling, 3 active+clean+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 31 active+undersized, 60 active+clean, 11 active+undersized+degraded, 2 active+remapped+backfill_wait; 1.6 GiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 258 op/s; 5586/78612 objects degraded (7.106%); 4814/78612 objects misplaced (6.124%); 12 MiB/s, 195 objects/s recovering 2023-09-23T20:55:36.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:55:36 smithi026 ceph-mon[175151]: failed to encode map e164 with expected crc 2023-09-23T20:55:36.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:55:36 smithi026 ceph-mon[175151]: failed to encode map e164 with expected crc 2023-09-23T20:55:36.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:55:36 smithi026 ceph-mon[175151]: 6.18 scrub starts 2023-09-23T20:55:36.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:55:36 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/3727760315' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:55:36.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:55:36 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:55:36.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:55:36 smithi026 ceph-mon[175151]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T20:55:36.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:55:36 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T20:55:36.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:55:36 smithi026 ceph-mon[175151]: osdmap e165: 8 total, 8 up, 8 in 2023-09-23T20:55:36.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:55:36 smithi026 ceph-mon[175151]: failed to encode map e165 with expected crc 2023-09-23T20:55:36.157 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:55:36.320 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_ops_in_flight 2023-09-23T20:55:36.343 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:55:36 smithi070 ceph-mon[168282]: pgmap v396: 109 pgs: 1 active+undersized+degraded+remapped+backfilling, 3 active+clean+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 31 active+undersized, 60 active+clean, 11 active+undersized+degraded, 2 active+remapped+backfill_wait; 1.6 GiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 258 op/s; 5586/78612 objects degraded (7.106%); 4814/78612 objects misplaced (6.124%); 12 MiB/s, 195 objects/s recovering 2023-09-23T20:55:36.344 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:55:36 smithi070 ceph-mon[168282]: failed to encode map e164 with expected crc 2023-09-23T20:55:36.344 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:55:36 smithi070 ceph-mon[168282]: failed to encode map e164 with expected crc 2023-09-23T20:55:36.344 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:55:36 smithi070 ceph-mon[168282]: 6.18 scrub starts 2023-09-23T20:55:36.344 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:55:36 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/3727760315' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:55:36.344 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:55:36 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:55:36.344 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:55:36 smithi070 ceph-mon[168282]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T20:55:36.344 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:55:36 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T20:55:36.344 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:55:36 smithi070 ceph-mon[168282]: osdmap e165: 8 total, 8 up, 8 in 2023-09-23T20:55:36.344 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:55:36 smithi070 ceph-mon[168282]: failed to encode map e165 with expected crc 2023-09-23T20:55:36.547 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_ops_in_flight 2023-09-23T20:55:36.651 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:55:36.653 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:55:36.654 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:55:36.754 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:55:36.785 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:55:36.927 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:55:37.034 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:55:37 smithi026 ceph-mon[179688]: failed to encode map e155 with expected crc 2023-09-23T20:55:37.034 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:55:37 smithi026 ceph-mon[175151]: failed to encode map e155 with expected crc 2023-09-23T20:55:37.034 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:55:37 smithi026 ceph-mon[175151]: failed to encode map e164 with expected crc 2023-09-23T20:55:37.034 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:55:37 smithi026 ceph-mon[175151]: failed to encode map e164 with expected crc 2023-09-23T20:55:37.035 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:55:37 smithi026 ceph-mon[175151]: failed to encode map e165 with expected crc 2023-09-23T20:55:37.035 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:55:37 smithi026 ceph-mon[175151]: failed to encode map e165 with expected crc 2023-09-23T20:55:37.035 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:55:37 smithi026 ceph-mon[175151]: failed to encode map e165 with expected crc 2023-09-23T20:55:37.035 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:55:37 smithi026 ceph-mon[175151]: failed to encode map e165 with expected crc 2023-09-23T20:55:37.035 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:55:37 smithi026 ceph-mon[175151]: failed to encode map e165 with expected crc 2023-09-23T20:55:37.035 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:55:37 smithi026 ceph-mon[175151]: failed to encode map e165 with expected crc 2023-09-23T20:55:37.035 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:55:37 smithi026 ceph-mon[175151]: 6.10 deep-scrub starts 2023-09-23T20:55:37.035 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:55:37 smithi026 ceph-mon[175151]: osdmap e166: 8 total, 8 up, 8 in 2023-09-23T20:55:37.035 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:55:37 smithi026 ceph-mon[175151]: failed to encode map e166 with expected crc 2023-09-23T20:55:37.054 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:55:37.066 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:55:37.067 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:55:37.167 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:55:37.280 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:55:37.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:55:37 smithi070 ceph-mon[168282]: failed to encode map e155 with expected crc 2023-09-23T20:55:37.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:55:37 smithi070 ceph-mon[168282]: failed to encode map e164 with expected crc 2023-09-23T20:55:37.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:55:37 smithi070 ceph-mon[168282]: failed to encode map e164 with expected crc 2023-09-23T20:55:37.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:55:37 smithi070 ceph-mon[168282]: failed to encode map e165 with expected crc 2023-09-23T20:55:37.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:55:37 smithi070 ceph-mon[168282]: failed to encode map e165 with expected crc 2023-09-23T20:55:37.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:55:37 smithi070 ceph-mon[168282]: failed to encode map e165 with expected crc 2023-09-23T20:55:37.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:55:37 smithi070 ceph-mon[168282]: failed to encode map e165 with expected crc 2023-09-23T20:55:37.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:55:37 smithi070 ceph-mon[168282]: failed to encode map e165 with expected crc 2023-09-23T20:55:37.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:55:37 smithi070 ceph-mon[168282]: failed to encode map e165 with expected crc 2023-09-23T20:55:37.371 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:55:37 smithi070 ceph-mon[168282]: 6.10 deep-scrub starts 2023-09-23T20:55:37.371 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:55:37 smithi070 ceph-mon[168282]: osdmap e166: 8 total, 8 up, 8 in 2023-09-23T20:55:37.371 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:55:37 smithi070 ceph-mon[168282]: failed to encode map e166 with expected crc 2023-09-23T20:55:37.377 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:55:37 smithi026 ceph-mon[179688]: failed to encode map e164 with expected crc 2023-09-23T20:55:37.377 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:55:37 smithi026 ceph-mon[179688]: failed to encode map e164 with expected crc 2023-09-23T20:55:37.378 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:55:37 smithi026 ceph-mon[179688]: failed to encode map e165 with expected crc 2023-09-23T20:55:37.378 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:55:37 smithi026 ceph-mon[179688]: failed to encode map e165 with expected crc 2023-09-23T20:55:37.378 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:55:37 smithi026 ceph-mon[179688]: failed to encode map e165 with expected crc 2023-09-23T20:55:37.378 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:55:37 smithi026 ceph-mon[179688]: failed to encode map e165 with expected crc 2023-09-23T20:55:37.378 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:55:37 smithi026 ceph-mon[179688]: failed to encode map e165 with expected crc 2023-09-23T20:55:37.378 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:55:37 smithi026 ceph-mon[179688]: failed to encode map e165 with expected crc 2023-09-23T20:55:37.378 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:55:37 smithi026 ceph-mon[179688]: 6.10 deep-scrub starts 2023-09-23T20:55:37.378 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:55:37 smithi026 ceph-mon[179688]: osdmap e166: 8 total, 8 up, 8 in 2023-09-23T20:55:37.378 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:55:37 smithi026 ceph-mon[179688]: failed to encode map e166 with expected crc 2023-09-23T20:55:37.378 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:55:37.391 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:55:37.397 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:55:37.397 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:55:37.498 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:55:37.571 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:55:37.697 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:55:37.704 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:55:37.705 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:55:37.806 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:55:37.821 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0: {} 2023-09-23T20:55:37.868 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1: {} 2023-09-23T20:55:37.895 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:55:37.912 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2: {} 2023-09-23T20:55:37.957 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3: {} 2023-09-23T20:55:38.019 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4: {} 2023-09-23T20:55:38.051 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T20:55:38.087 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6: {} 2023-09-23T20:55:38.125 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T20:55:38.135 INFO:teuthology.orchestra.run.smithi026.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T20:55:38.135 INFO:teuthology.orchestra.run.smithi026.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T20:55:38.135 INFO:teuthology.orchestra.run.smithi026.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T20:55:38.135 INFO:teuthology.orchestra.run.smithi026.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T20:55:38.135 INFO:teuthology.orchestra.run.smithi026.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T20:55:38.135 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T20:55:38.135 INFO:teuthology.orchestra.run.smithi026.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T20:55:38.135 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T20:55:38.335 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:55:38 smithi026 ceph-mon[179688]: pgmap v399: 109 pgs: 1 active+undersized+degraded+remapped+backfilling, 3 active+clean+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 31 active+undersized, 60 active+clean, 11 active+undersized+degraded, 2 active+remapped+backfill_wait; 1.6 GiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 5586/77244 objects degraded (7.232%); 4814/77244 objects misplaced (6.232%); 7.2 MiB/s, 115 objects/s recovering 2023-09-23T20:55:38.335 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:55:38 smithi026 ceph-mon[179688]: failed to encode map e166 with expected crc 2023-09-23T20:55:38.335 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:55:38 smithi026 ceph-mon[179688]: failed to encode map e166 with expected crc 2023-09-23T20:55:38.335 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:55:38 smithi026 ceph-mon[179688]: failed to encode map e166 with expected crc 2023-09-23T20:55:38.336 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:55:38 smithi026 ceph-mon[179688]: failed to encode map e166 with expected crc 2023-09-23T20:55:38.336 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:55:38 smithi026 ceph-mon[179688]: failed to encode map e166 with expected crc 2023-09-23T20:55:38.336 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:55:38 smithi026 ceph-mon[179688]: failed to encode map e166 with expected crc 2023-09-23T20:55:38.336 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:55:38 smithi026 ceph-mon[179688]: failed to encode map e166 with expected crc 2023-09-23T20:55:38.336 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:55:38 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/1635373372' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:55:38.336 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:55:38 smithi026 ceph-mon[175151]: pgmap v399: 109 pgs: 1 active+undersized+degraded+remapped+backfilling, 3 active+clean+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 31 active+undersized, 60 active+clean, 11 active+undersized+degraded, 2 active+remapped+backfill_wait; 1.6 GiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 5586/77244 objects degraded (7.232%); 4814/77244 objects misplaced (6.232%); 7.2 MiB/s, 115 objects/s recovering 2023-09-23T20:55:38.336 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:55:38 smithi026 ceph-mon[175151]: failed to encode map e166 with expected crc 2023-09-23T20:55:38.336 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:55:38 smithi026 ceph-mon[175151]: failed to encode map e166 with expected crc 2023-09-23T20:55:38.336 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:55:38 smithi026 ceph-mon[175151]: failed to encode map e166 with expected crc 2023-09-23T20:55:38.336 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:55:38 smithi026 ceph-mon[175151]: failed to encode map e166 with expected crc 2023-09-23T20:55:38.336 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:55:38 smithi026 ceph-mon[175151]: failed to encode map e166 with expected crc 2023-09-23T20:55:38.337 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:55:38 smithi026 ceph-mon[175151]: failed to encode map e166 with expected crc 2023-09-23T20:55:38.337 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:55:38 smithi026 ceph-mon[175151]: failed to encode map e166 with expected crc 2023-09-23T20:55:38.337 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:55:38 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/1635373372' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:55:38.364 INFO:teuthology.orchestra.run.smithi070.stdout:ERROR: (22) Invalid argument 2023-09-23T20:55:38.364 INFO:teuthology.orchestra.run.smithi070.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:55:38.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:55:38 smithi070 ceph-mon[168282]: pgmap v399: 109 pgs: 1 active+undersized+degraded+remapped+backfilling, 3 active+clean+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 31 active+undersized, 60 active+clean, 11 active+undersized+degraded, 2 active+remapped+backfill_wait; 1.6 GiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 5586/77244 objects degraded (7.232%); 4814/77244 objects misplaced (6.232%); 7.2 MiB/s, 115 objects/s recovering 2023-09-23T20:55:38.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:55:38 smithi070 ceph-mon[168282]: failed to encode map e166 with expected crc 2023-09-23T20:55:38.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:55:38 smithi070 ceph-mon[168282]: failed to encode map e166 with expected crc 2023-09-23T20:55:38.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:55:38 smithi070 ceph-mon[168282]: failed to encode map e166 with expected crc 2023-09-23T20:55:38.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:55:38 smithi070 ceph-mon[168282]: failed to encode map e166 with expected crc 2023-09-23T20:55:38.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:55:38 smithi070 ceph-mon[168282]: failed to encode map e166 with expected crc 2023-09-23T20:55:38.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:55:38 smithi070 ceph-mon[168282]: failed to encode map e166 with expected crc 2023-09-23T20:55:38.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:55:38 smithi070 ceph-mon[168282]: failed to encode map e166 with expected crc 2023-09-23T20:55:38.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:55:38 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/1635373372' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:55:38.471 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:55:38.482 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:55:38.482 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:55:38.583 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:55:38.623 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:55:38.640 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T20:55:38.719 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:55:38.725 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:55:38.726 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:55:38.826 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:55:38.928 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:55:39.033 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:55:39.046 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:55:39.047 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:55:39.147 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:55:39.187 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:55:39.230 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:55:39.234 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:55:39.234 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:55:39.335 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:55:39.368 INFO:teuthology.orchestra.run.smithi026.stdout:ERROR: (22) Invalid argument 2023-09-23T20:55:39.368 INFO:teuthology.orchestra.run.smithi026.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:55:39.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:55:39 smithi070 ceph-mon[168282]: 6.12 deep-scrub starts 2023-09-23T20:55:39.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:55:39 smithi070 ceph-mon[168282]: pgmap v401: 109 pgs: 1 active+undersized+degraded+remapped+backfilling, 1 active+clean+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 12 active+undersized, 85 active+clean, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+degraded, 2 active+remapped+backfill_wait; 1.5 GiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 4722/73407 objects degraded (6.433%); 3159/73407 objects misplaced (4.303%); 10 MiB/s, 163 objects/s recovering 2023-09-23T20:55:39.375 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:55:39.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:55:39 smithi026 ceph-mon[175151]: 6.12 deep-scrub starts 2023-09-23T20:55:39.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:55:39 smithi026 ceph-mon[175151]: pgmap v401: 109 pgs: 1 active+undersized+degraded+remapped+backfilling, 1 active+clean+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 12 active+undersized, 85 active+clean, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+degraded, 2 active+remapped+backfill_wait; 1.5 GiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 4722/73407 objects degraded (6.433%); 3159/73407 objects misplaced (4.303%); 10 MiB/s, 163 objects/s recovering 2023-09-23T20:55:39.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:55:39 smithi026 ceph-mon[179688]: 6.12 deep-scrub starts 2023-09-23T20:55:39.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:55:39 smithi026 ceph-mon[179688]: pgmap v401: 109 pgs: 1 active+undersized+degraded+remapped+backfilling, 1 active+clean+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 12 active+undersized, 85 active+clean, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+degraded, 2 active+remapped+backfill_wait; 1.5 GiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 4722/73407 objects degraded (6.433%); 3159/73407 objects misplaced (4.303%); 10 MiB/s, 163 objects/s recovering 2023-09-23T20:55:39.429 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:55:39.435 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:55:39.435 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:55:39.537 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:55:39.583 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:55:39.590 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:55:39.636 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:55:39.643 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:55:39.643 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:55:39.744 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:55:39.790 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:55:39 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[164029]: ::ffff:172.21.15.70 - - [23/Sep/2023:20:55:39] "GET /metrics HTTP/1.1" 200 36354 "" "Prometheus/2.43.0" 2023-09-23T20:55:39.811 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:55:40.074 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:55:40.082 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:55:40.082 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:55:40.108 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_blocked_ops 2023-09-23T20:55:40.183 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:55:40.212 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:55:40.264 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:55:40.268 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:55:40.269 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:55:40.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:55:40 smithi070 ceph-mon[168282]: 6.2 scrub starts 2023-09-23T20:55:40.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:55:40 smithi070 ceph-mon[168282]: Health check update: Degraded data redundancy: 4722/73407 objects degraded (6.433%), 9 pgs degraded (PG_DEGRADED) 2023-09-23T20:55:40.370 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:55:40.387 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:55:40 smithi026 ceph-mon[175151]: 6.2 scrub starts 2023-09-23T20:55:40.387 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:55:40 smithi026 ceph-mon[175151]: Health check update: Degraded data redundancy: 4722/73407 objects degraded (6.433%), 9 pgs degraded (PG_DEGRADED) 2023-09-23T20:55:40.387 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:55:40 smithi026 ceph-mon[179688]: 6.2 scrub starts 2023-09-23T20:55:40.387 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:55:40 smithi026 ceph-mon[179688]: Health check update: Degraded data redundancy: 4722/73407 objects degraded (6.433%), 9 pgs degraded (PG_DEGRADED) 2023-09-23T20:55:40.404 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:55:40.448 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:55:40.452 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:55:40.452 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:55:40.553 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:55:40.588 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:55:40.636 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:55:40.638 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:55:40.639 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:55:40.658 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T20:55:40.740 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:55:40.849 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:55:40.950 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:55:40.958 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:55:40.958 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:55:41.059 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:55:41.085 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:55:41 smithi026 ceph-mon[175151]: 6.12 scrub starts 2023-09-23T20:55:41.086 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:55:41 smithi026 ceph-mon[175151]: pgmap v402: 109 pgs: 1 active+recovering+undersized+remapped, 99 active+clean, 7 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 1.4 GiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 3202/70563 objects degraded (4.538%); 679/70563 objects misplaced (0.962%); 17 MiB/s, 311 objects/s recovering 2023-09-23T20:55:41.086 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:55:41 smithi026 ceph-mon[179688]: 6.12 scrub starts 2023-09-23T20:55:41.086 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:55:41 smithi026 ceph-mon[179688]: pgmap v402: 109 pgs: 1 active+recovering+undersized+remapped, 99 active+clean, 7 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 1.4 GiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 3202/70563 objects degraded (4.538%); 679/70563 objects misplaced (0.962%); 17 MiB/s, 311 objects/s recovering 2023-09-23T20:55:41.150 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:55:41.224 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:55:41.231 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:55:41.231 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:55:41.332 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:55:41.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:55:41 smithi070 ceph-mon[168282]: 6.12 scrub starts 2023-09-23T20:55:41.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:55:41 smithi070 ceph-mon[168282]: pgmap v402: 109 pgs: 1 active+recovering+undersized+remapped, 99 active+clean, 7 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 1.4 GiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 3202/70563 objects degraded (4.538%); 679/70563 objects misplaced (0.962%); 17 MiB/s, 311 objects/s recovering 2023-09-23T20:55:41.423 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:55:41.508 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:55:41.547 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:55:41.551 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:55:41.551 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:55:41.652 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:55:41.694 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:55:41.732 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:55:41.734 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:55:41.735 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:55:41.735 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:55:41.836 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:55:41.878 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:55:41.923 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:55:41.930 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:55:41.931 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:55:42.032 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:55:42.144 INFO:teuthology.orchestra.run.smithi026.stderr:noscrub is set 2023-09-23T20:55:42.166 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:55:42.167 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:55:42 smithi026 ceph-mon[175151]: 6.2 scrub starts 2023-09-23T20:55:42.167 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:55:42 smithi026 ceph-mon[175151]: 6.12 scrub starts 2023-09-23T20:55:42.167 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:55:42 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/2763454643' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:55:42.167 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:55:42 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:55:42.167 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:55:42 smithi026 ceph-mon[179688]: 6.2 scrub starts 2023-09-23T20:55:42.167 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:55:42 smithi026 ceph-mon[179688]: 6.12 scrub starts 2023-09-23T20:55:42.167 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:55:42 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/2763454643' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:55:42.168 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:55:42 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:55:42.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:55:42 smithi070 ceph-mon[168282]: 6.2 scrub starts 2023-09-23T20:55:42.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:55:42 smithi070 ceph-mon[168282]: 6.12 scrub starts 2023-09-23T20:55:42.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:55:42 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/2763454643' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:55:42.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:55:42 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:55:42.801 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:55:42.805 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:55:42.805 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:55:42.906 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:55:42.982 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:55:43.084 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:55:43 smithi026 ceph-mon[175151]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:55:43.085 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:55:43 smithi026 ceph-mon[175151]: failed to encode map e167 with expected crc 2023-09-23T20:55:43.085 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:55:43 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T20:55:43.085 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:55:43 smithi026 ceph-mon[175151]: osdmap e167: 8 total, 8 up, 8 in 2023-09-23T20:55:43.085 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:55:43 smithi026 ceph-mon[175151]: failed to encode map e167 with expected crc 2023-09-23T20:55:43.085 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:55:43 smithi026 ceph-mon[175151]: failed to encode map e167 with expected crc 2023-09-23T20:55:43.085 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:55:43 smithi026 ceph-mon[175151]: failed to encode map e167 with expected crc 2023-09-23T20:55:43.085 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:55:43 smithi026 ceph-mon[175151]: pgmap v404: 109 pgs: 1 active+recovering+undersized+remapped, 99 active+clean, 7 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 1.3 GiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 3202/63603 objects degraded (5.034%); 679/63603 objects misplaced (1.068%); 16 MiB/s, 294 objects/s recovering 2023-09-23T20:55:43.200 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:55:43.220 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:55:43.221 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:55:43.322 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:55:43.354 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:55:43 smithi070 ceph-mon[168282]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:55:43.354 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:55:43 smithi070 ceph-mon[168282]: failed to encode map e167 with expected crc 2023-09-23T20:55:43.354 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:55:43 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T20:55:43.355 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:55:43 smithi070 ceph-mon[168282]: osdmap e167: 8 total, 8 up, 8 in 2023-09-23T20:55:43.355 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:55:43 smithi070 ceph-mon[168282]: failed to encode map e167 with expected crc 2023-09-23T20:55:43.355 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:55:43 smithi070 ceph-mon[168282]: failed to encode map e167 with expected crc 2023-09-23T20:55:43.355 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:55:43 smithi070 ceph-mon[168282]: failed to encode map e167 with expected crc 2023-09-23T20:55:43.355 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:55:43 smithi070 ceph-mon[168282]: pgmap v404: 109 pgs: 1 active+recovering+undersized+remapped, 99 active+clean, 7 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 1.3 GiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 3202/63603 objects degraded (5.034%); 679/63603 objects misplaced (1.068%); 16 MiB/s, 294 objects/s recovering 2023-09-23T20:55:43.358 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:55:43.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:55:43 smithi026 ceph-mon[179688]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:55:43.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:55:43 smithi026 ceph-mon[179688]: failed to encode map e167 with expected crc 2023-09-23T20:55:43.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:55:43 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T20:55:43.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:55:43 smithi026 ceph-mon[179688]: osdmap e167: 8 total, 8 up, 8 in 2023-09-23T20:55:43.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:55:43 smithi026 ceph-mon[179688]: failed to encode map e167 with expected crc 2023-09-23T20:55:43.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:55:43 smithi026 ceph-mon[179688]: failed to encode map e167 with expected crc 2023-09-23T20:55:43.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:55:43 smithi026 ceph-mon[179688]: failed to encode map e167 with expected crc 2023-09-23T20:55:43.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:55:43 smithi026 ceph-mon[179688]: pgmap v404: 109 pgs: 1 active+recovering+undersized+remapped, 99 active+clean, 7 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 1.3 GiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 3202/63603 objects degraded (5.034%); 679/63603 objects misplaced (1.068%); 16 MiB/s, 294 objects/s recovering 2023-09-23T20:55:43.400 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:55:43.408 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:55:43.408 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:55:43.509 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:55:43.546 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:55:43.601 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:55:43.604 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:55:43.604 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:55:43.705 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:55:43.735 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 6, 7, 4, 5] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 7, 6] 2023-09-23T20:55:43.735 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T20:55:43.735 INFO:tasks.thrashosds.thrasher:inject_pause on osd.6 2023-09-23T20:55:43.735 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-09-23T20:55:43.735 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-09-23T20:55:43.736 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 config set filestore_inject_stall 3 2023-09-23T20:55:43.745 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:55:43.801 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:55:43.807 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:55:43.807 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:55:43.812 INFO:teuthology.orchestra.run.smithi026.stdout:ERROR: (22) Invalid argument 2023-09-23T20:55:43.812 INFO:teuthology.orchestra.run.smithi026.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:55:43.908 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:55:43.998 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:55:44.081 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:55:44.094 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:55:44.094 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:55:44.195 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:55:44.205 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0: {} 2023-09-23T20:55:44.247 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1: {} 2023-09-23T20:55:44.280 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:55:44.297 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2: {} 2023-09-23T20:55:44.346 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:55:44 smithi026 ceph-mon[175151]: failed to encode map e167 with expected crc 2023-09-23T20:55:44.346 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:55:44 smithi026 ceph-mon[179688]: failed to encode map e167 with expected crc 2023-09-23T20:55:44.347 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3: {} 2023-09-23T20:55:44.368 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:55:44.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:55:44 smithi070 ceph-mon[168282]: failed to encode map e167 with expected crc 2023-09-23T20:55:44.375 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:55:44.376 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:55:44.391 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4: {} 2023-09-23T20:55:44.411 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_historic_ops 2023-09-23T20:55:44.432 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T20:55:44.458 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:55:44.469 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6: {} 2023-09-23T20:55:44.476 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:55:44.505 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T20:55:44.506 INFO:teuthology.orchestra.run.smithi026.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T20:55:44.507 INFO:teuthology.orchestra.run.smithi026.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T20:55:44.507 INFO:teuthology.orchestra.run.smithi026.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T20:55:44.507 INFO:teuthology.orchestra.run.smithi026.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T20:55:44.507 INFO:teuthology.orchestra.run.smithi026.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T20:55:44.507 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T20:55:44.507 INFO:teuthology.orchestra.run.smithi026.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T20:55:44.507 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T20:55:44.519 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:55:44.565 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:55:44.568 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:55:44.569 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:55:44.670 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:55:44.700 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:55:44.870 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:55:44.873 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:55:44.873 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:55:44.974 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:55:45.028 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:55:45.197 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:55:45.224 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T20:55:45.309 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:55:45.312 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:55:45 smithi070 ceph-mon[168282]: osdmap e168: 8 total, 8 up, 8 in 2023-09-23T20:55:45.312 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:55:45 smithi070 ceph-mon[168282]: failed to encode map e168 with expected crc 2023-09-23T20:55:45.312 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:55:45 smithi070 ceph-mon[168282]: failed to encode map e168 with expected crc 2023-09-23T20:55:45.312 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:55:45 smithi070 ceph-mon[168282]: failed to encode map e168 with expected crc 2023-09-23T20:55:45.312 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:55:45 smithi070 ceph-mon[168282]: failed to encode map e168 with expected crc 2023-09-23T20:55:45.312 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:55:45 smithi070 ceph-mon[168282]: failed to encode map e168 with expected crc 2023-09-23T20:55:45.312 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:55:45 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/4214683117' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:55:45.312 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:55:45 smithi070 ceph-mon[168282]: pgmap v406: 109 pgs: 1 active+remapped+backfilling, 1 peering, 99 active+clean, 7 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfill_wait; 1.2 GiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 795 op/s; 3202/59091 objects degraded (5.419%); 508/59091 objects misplaced (0.860%); 14 MiB/s, 337 objects/s recovering 2023-09-23T20:55:45.312 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:55:45 smithi070 ceph-mon[168282]: Health check update: Degraded data redundancy: 3202/63603 objects degraded (5.034%), 7 pgs degraded (PG_DEGRADED) 2023-09-23T20:55:45.312 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:55:45 smithi070 ceph-mon[168282]: failed to encode map e169 with expected crc 2023-09-23T20:55:45.313 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:55:45 smithi070 ceph-mon[168282]: failed to encode map e169 with expected crc 2023-09-23T20:55:45.313 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:55:45 smithi070 ceph-mon[168282]: osdmap e169: 8 total, 8 up, 8 in 2023-09-23T20:55:45.313 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:55:45 smithi070 ceph-mon[168282]: failed to encode map e169 with expected crc 2023-09-23T20:55:45.313 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:55:45 smithi070 ceph-mon[168282]: failed to encode map e169 with expected crc 2023-09-23T20:55:45.313 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:55:45.314 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:55:45.383 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:55:45 smithi026 ceph-mon[175151]: osdmap e168: 8 total, 8 up, 8 in 2023-09-23T20:55:45.383 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:55:45 smithi026 ceph-mon[175151]: failed to encode map e168 with expected crc 2023-09-23T20:55:45.383 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:55:45 smithi026 ceph-mon[175151]: failed to encode map e168 with expected crc 2023-09-23T20:55:45.383 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:55:45 smithi026 ceph-mon[175151]: failed to encode map e168 with expected crc 2023-09-23T20:55:45.384 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:55:45 smithi026 ceph-mon[175151]: failed to encode map e168 with expected crc 2023-09-23T20:55:45.384 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:55:45 smithi026 ceph-mon[175151]: failed to encode map e168 with expected crc 2023-09-23T20:55:45.384 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:55:45 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/4214683117' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:55:45.384 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:55:45 smithi026 ceph-mon[175151]: pgmap v406: 109 pgs: 1 active+remapped+backfilling, 1 peering, 99 active+clean, 7 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfill_wait; 1.2 GiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 795 op/s; 3202/59091 objects degraded (5.419%); 508/59091 objects misplaced (0.860%); 14 MiB/s, 337 objects/s recovering 2023-09-23T20:55:45.384 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:55:45 smithi026 ceph-mon[175151]: Health check update: Degraded data redundancy: 3202/63603 objects degraded (5.034%), 7 pgs degraded (PG_DEGRADED) 2023-09-23T20:55:45.384 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:55:45 smithi026 ceph-mon[175151]: failed to encode map e169 with expected crc 2023-09-23T20:55:45.384 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:55:45 smithi026 ceph-mon[175151]: failed to encode map e169 with expected crc 2023-09-23T20:55:45.384 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:55:45 smithi026 ceph-mon[175151]: osdmap e169: 8 total, 8 up, 8 in 2023-09-23T20:55:45.384 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:55:45 smithi026 ceph-mon[175151]: failed to encode map e169 with expected crc 2023-09-23T20:55:45.384 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:55:45 smithi026 ceph-mon[175151]: failed to encode map e169 with expected crc 2023-09-23T20:55:45.384 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:55:45 smithi026 ceph-mon[179688]: osdmap e168: 8 total, 8 up, 8 in 2023-09-23T20:55:45.385 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:55:45 smithi026 ceph-mon[179688]: failed to encode map e168 with expected crc 2023-09-23T20:55:45.385 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:55:45 smithi026 ceph-mon[179688]: failed to encode map e168 with expected crc 2023-09-23T20:55:45.385 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:55:45 smithi026 ceph-mon[179688]: failed to encode map e168 with expected crc 2023-09-23T20:55:45.385 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:55:45 smithi026 ceph-mon[179688]: failed to encode map e168 with expected crc 2023-09-23T20:55:45.385 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:55:45 smithi026 ceph-mon[179688]: failed to encode map e168 with expected crc 2023-09-23T20:55:45.385 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:55:45 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/4214683117' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:55:45.385 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:55:45 smithi026 ceph-mon[179688]: pgmap v406: 109 pgs: 1 active+remapped+backfilling, 1 peering, 99 active+clean, 7 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfill_wait; 1.2 GiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 795 op/s; 3202/59091 objects degraded (5.419%); 508/59091 objects misplaced (0.860%); 14 MiB/s, 337 objects/s recovering 2023-09-23T20:55:45.385 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:55:45 smithi026 ceph-mon[179688]: Health check update: Degraded data redundancy: 3202/63603 objects degraded (5.034%), 7 pgs degraded (PG_DEGRADED) 2023-09-23T20:55:45.385 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:55:45 smithi026 ceph-mon[179688]: failed to encode map e169 with expected crc 2023-09-23T20:55:45.385 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:55:45 smithi026 ceph-mon[179688]: failed to encode map e169 with expected crc 2023-09-23T20:55:45.385 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:55:45 smithi026 ceph-mon[179688]: osdmap e169: 8 total, 8 up, 8 in 2023-09-23T20:55:45.386 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:55:45 smithi026 ceph-mon[179688]: failed to encode map e169 with expected crc 2023-09-23T20:55:45.386 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:55:45 smithi026 ceph-mon[179688]: failed to encode map e169 with expected crc 2023-09-23T20:55:45.414 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:55:45.457 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:55:45.499 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:55:45.503 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:55:45.503 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:55:45.604 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:55:45.637 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:55:46.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:55:46 smithi070 ceph-mon[168282]: failed to encode map e169 with expected crc 2023-09-23T20:55:46.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:55:46 smithi070 ceph-mon[168282]: failed to encode map e169 with expected crc 2023-09-23T20:55:46.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:55:46 smithi070 ceph-mon[168282]: failed to encode map e170 with expected crc 2023-09-23T20:55:46.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:55:46 smithi070 ceph-mon[168282]: failed to encode map e170 with expected crc 2023-09-23T20:55:46.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:55:46 smithi070 ceph-mon[168282]: osdmap e170: 8 total, 8 up, 8 in 2023-09-23T20:55:46.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:55:46 smithi070 ceph-mon[168282]: failed to encode map e170 with expected crc 2023-09-23T20:55:46.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:55:46 smithi026 ceph-mon[175151]: failed to encode map e169 with expected crc 2023-09-23T20:55:46.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:55:46 smithi026 ceph-mon[175151]: failed to encode map e169 with expected crc 2023-09-23T20:55:46.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:55:46 smithi026 ceph-mon[175151]: failed to encode map e170 with expected crc 2023-09-23T20:55:46.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:55:46 smithi026 ceph-mon[175151]: failed to encode map e170 with expected crc 2023-09-23T20:55:46.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:55:46 smithi026 ceph-mon[175151]: osdmap e170: 8 total, 8 up, 8 in 2023-09-23T20:55:46.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:55:46 smithi026 ceph-mon[175151]: failed to encode map e170 with expected crc 2023-09-23T20:55:46.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:55:46 smithi026 ceph-mon[179688]: failed to encode map e169 with expected crc 2023-09-23T20:55:46.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:55:46 smithi026 ceph-mon[179688]: failed to encode map e169 with expected crc 2023-09-23T20:55:46.383 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:55:46 smithi026 ceph-mon[179688]: failed to encode map e170 with expected crc 2023-09-23T20:55:46.383 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:55:46 smithi026 ceph-mon[179688]: failed to encode map e170 with expected crc 2023-09-23T20:55:46.383 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:55:46 smithi026 ceph-mon[179688]: osdmap e170: 8 total, 8 up, 8 in 2023-09-23T20:55:46.383 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:55:46 smithi026 ceph-mon[179688]: failed to encode map e170 with expected crc 2023-09-23T20:55:46.831 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T20:55:47.359 INFO:teuthology.orchestra.run.smithi070.stdout:{ 2023-09-23T20:55:47.359 INFO:teuthology.orchestra.run.smithi070.stdout: "success": "filestore_inject_stall = '3' (not observed, change may require restart) " 2023-09-23T20:55:47.359 INFO:teuthology.orchestra.run.smithi070.stdout:} 2023-09-23T20:55:47.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:55:47 smithi070 ceph-mon[168282]: failed to encode map e170 with expected crc 2023-09-23T20:55:47.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:55:47 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:55:47.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:55:47 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:55:47.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:55:47 smithi070 ceph-mon[168282]: pgmap v409: 109 pgs: 1 active+remapped+backfilling, 1 peering, 99 active+clean, 7 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfill_wait; 1.2 GiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 873 op/s; 3202/57726 objects degraded (5.547%); 508/57726 objects misplaced (0.880%); 870 KiB/s, 146 objects/s recovering 2023-09-23T20:55:47.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:55:47 smithi070 ceph-mon[168282]: failed to encode map e171 with expected crc 2023-09-23T20:55:47.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:55:47 smithi070 ceph-mon[168282]: osdmap e171: 8 total, 8 up, 8 in 2023-09-23T20:55:47.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:55:47 smithi070 ceph-mon[168282]: failed to encode map e171 with expected crc 2023-09-23T20:55:47.593 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:55:47 smithi026 ceph-mon[179688]: failed to encode map e170 with expected crc 2023-09-23T20:55:47.593 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:55:47 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:55:47.593 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:55:47 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:55:47.593 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:55:47 smithi026 ceph-mon[179688]: pgmap v409: 109 pgs: 1 active+remapped+backfilling, 1 peering, 99 active+clean, 7 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfill_wait; 1.2 GiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 873 op/s; 3202/57726 objects degraded (5.547%); 508/57726 objects misplaced (0.880%); 870 KiB/s, 146 objects/s recovering 2023-09-23T20:55:47.593 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:55:47 smithi026 ceph-mon[179688]: failed to encode map e171 with expected crc 2023-09-23T20:55:47.594 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:55:47 smithi026 ceph-mon[179688]: osdmap e171: 8 total, 8 up, 8 in 2023-09-23T20:55:47.594 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:55:47 smithi026 ceph-mon[179688]: failed to encode map e171 with expected crc 2023-09-23T20:55:47.594 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:55:47 smithi026 ceph-mon[175151]: failed to encode map e170 with expected crc 2023-09-23T20:55:47.594 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:55:47 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:55:47.594 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:55:47 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:55:47.594 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:55:47 smithi026 ceph-mon[175151]: pgmap v409: 109 pgs: 1 active+remapped+backfilling, 1 peering, 99 active+clean, 7 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfill_wait; 1.2 GiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 873 op/s; 3202/57726 objects degraded (5.547%); 508/57726 objects misplaced (0.880%); 870 KiB/s, 146 objects/s recovering 2023-09-23T20:55:47.594 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:55:47 smithi026 ceph-mon[175151]: failed to encode map e171 with expected crc 2023-09-23T20:55:47.594 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:55:47 smithi026 ceph-mon[175151]: osdmap e171: 8 total, 8 up, 8 in 2023-09-23T20:55:47.594 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:55:47 smithi026 ceph-mon[175151]: failed to encode map e171 with expected crc 2023-09-23T20:55:47.634 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:55:47.641 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:55:47.642 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:55:47.742 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:55:47.780 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:55:47.825 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:55:47.829 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:55:47.830 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:55:47.930 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:55:48.025 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:55:48.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:55:48 smithi026 ceph-mon[179688]: failed to encode map e171 with expected crc 2023-09-23T20:55:48.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:55:48 smithi026 ceph-mon[179688]: failed to encode map e171 with expected crc 2023-09-23T20:55:48.383 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:55:48 smithi026 ceph-mon[175151]: failed to encode map e171 with expected crc 2023-09-23T20:55:48.383 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:55:48 smithi026 ceph-mon[175151]: failed to encode map e171 with expected crc 2023-09-23T20:55:48.563 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:55:48.566 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:55:48.567 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:55:48.567 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:55:48.567 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:55:48.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:55:48 smithi070 ceph-mon[168282]: failed to encode map e171 with expected crc 2023-09-23T20:55:48.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:55:48 smithi070 ceph-mon[168282]: failed to encode map e171 with expected crc 2023-09-23T20:55:48.668 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:55:48.695 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:55:48.729 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:55:48.732 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:55:48.732 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:55:48.833 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:55:48.859 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:55:48.890 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:55:48.893 INFO:teuthology.orchestra.run.smithi026.stdout:{ 2023-09-23T20:55:48.893 INFO:teuthology.orchestra.run.smithi026.stdout: "size": 20, 2023-09-23T20:55:48.893 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 600, 2023-09-23T20:55:48.893 INFO:teuthology.orchestra.run.smithi026.stdout: "ops": [ 2023-09-23T20:55:48.893 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:48.894 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34797.0:72087 6.4 6:28e23fe0:::benchmark_data_smithi026_223868_object72086:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e116)", 2023-09-23T20:55:48.894 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T20:53:09.079653+0000", 2023-09-23T20:55:48.894 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 159.81073458399999, 2023-09-23T20:55:48.894 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.10674932400000001, 2023-09-23T20:55:48.894 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T20:55:48.894 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:55:48.894 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T20:55:48.894 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34797", 2023-09-23T20:55:48.894 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T20:55:48.895 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 72087 2023-09-23T20:55:48.895 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:48.895 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T20:55:48.895 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:48.895 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T20:55:48.895 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:09.079653+0000", 2023-09-23T20:55:48.895 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:55:48.895 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:48.895 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:48.895 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T20:55:48.896 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:09.079653+0000", 2023-09-23T20:55:48.896 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:55:48.896 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:48.896 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:48.896 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T20:55:48.896 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:09.079652+0000", 2023-09-23T20:55:48.896 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.999999 2023-09-23T20:55:48.896 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:48.896 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:48.896 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T20:55:48.896 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:09.079676+0000", 2023-09-23T20:55:48.897 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.3323e-05 2023-09-23T20:55:48.897 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:48.897 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:48.897 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T20:55:48.897 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:09.079688+0000", 2023-09-23T20:55:48.897 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.2285e-05 2023-09-23T20:55:48.897 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:48.897 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:48.897 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T20:55:48.897 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:09.079694+0000", 2023-09-23T20:55:48.897 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.9320000000000001e-06 2023-09-23T20:55:48.898 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:48.898 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:48.898 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:55:48.898 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:09.079737+0000", 2023-09-23T20:55:48.898 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.3557999999999999e-05 2023-09-23T20:55:48.898 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:48.898 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:48.898 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T20:55:48.898 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:09.079856+0000", 2023-09-23T20:55:48.898 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000118836 2023-09-23T20:55:48.898 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:48.899 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:48.899 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 3,6", 2023-09-23T20:55:48.899 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:09.079954+0000", 2023-09-23T20:55:48.899 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.7942999999999994e-05 2023-09-23T20:55:48.899 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:48.899 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:48.899 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:55:48.899 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:09.082304+0000", 2023-09-23T20:55:48.899 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0023497370000000002 2023-09-23T20:55:48.899 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:48.899 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:48.899 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T20:55:48.900 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:09.082523+0000", 2023-09-23T20:55:48.900 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00021949 2023-09-23T20:55:48.900 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:48.900 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:48.900 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:55:48.900 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:09.186306+0000", 2023-09-23T20:55:48.900 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.103782889 2023-09-23T20:55:48.900 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:48.900 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:48.900 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T20:55:48.900 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:09.186363+0000", 2023-09-23T20:55:48.900 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.6691999999999999e-05 2023-09-23T20:55:48.900 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:48.901 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:48.901 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T20:55:48.901 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:09.186403+0000", 2023-09-23T20:55:48.901 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.9718000000000002e-05 2023-09-23T20:55:48.901 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:55:48.901 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T20:55:48.901 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:55:48.901 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:48.901 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:48.901 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34797.0:97285 6.e 6:7ae966fb:::benchmark_data_smithi026_223868_object97284:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e124)", 2023-09-23T20:55:48.901 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T20:53:42.972750+0000", 2023-09-23T20:55:48.901 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 125.917637677, 2023-09-23T20:55:48.902 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.084367157999999998, 2023-09-23T20:55:48.902 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T20:55:48.902 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:55:48.902 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T20:55:48.902 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34797", 2023-09-23T20:55:48.902 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T20:55:48.902 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 97285 2023-09-23T20:55:48.902 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:48.902 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T20:55:48.902 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:48.902 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T20:55:48.902 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:42.972750+0000", 2023-09-23T20:55:48.903 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:55:48.903 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:48.903 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:48.903 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T20:55:48.903 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:42.972750+0000", 2023-09-23T20:55:48.903 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:55:48.903 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:48.903 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:48.903 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T20:55:48.903 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:42.972748+0000", 2023-09-23T20:55:48.903 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999981 2023-09-23T20:55:48.903 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:48.904 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:48.904 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T20:55:48.904 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:42.972779+0000", 2023-09-23T20:55:48.904 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.0473000000000001e-05 2023-09-23T20:55:48.904 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:48.904 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:48.904 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T20:55:48.904 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:42.972793+0000", 2023-09-23T20:55:48.904 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.3985000000000001e-05 2023-09-23T20:55:48.904 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:48.904 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:48.905 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T20:55:48.905 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:42.972802+0000", 2023-09-23T20:55:48.905 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.7839999999999998e-06 2023-09-23T20:55:48.905 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:48.905 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:48.905 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:55:48.905 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:42.972957+0000", 2023-09-23T20:55:48.905 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00015432600000000001 2023-09-23T20:55:48.905 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:48.905 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:48.905 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T20:55:48.905 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:42.973139+0000", 2023-09-23T20:55:48.906 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00018189599999999999 2023-09-23T20:55:48.906 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:48.906 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:48.906 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 2,7", 2023-09-23T20:55:48.906 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:42.973354+0000", 2023-09-23T20:55:48.906 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00021514599999999999 2023-09-23T20:55:48.906 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:48.906 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:48.906 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:55:48.906 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:42.994742+0000", 2023-09-23T20:55:48.906 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.021388205 2023-09-23T20:55:48.907 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:48.907 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:48.907 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:55:48.907 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:43.031945+0000", 2023-09-23T20:55:48.907 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.037202950999999998 2023-09-23T20:55:48.907 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:48.907 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:48.907 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T20:55:48.907 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:43.055155+0000", 2023-09-23T20:55:48.907 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.023210193 2023-09-23T20:55:48.907 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:48.908 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:48.908 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T20:55:48.908 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:43.055230+0000", 2023-09-23T20:55:48.908 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.4369999999999994e-05 2023-09-23T20:55:48.908 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:48.908 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:48.908 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T20:55:48.908 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:43.057117+0000", 2023-09-23T20:55:48.908 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0018879019999999999 2023-09-23T20:55:48.908 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:55:48.908 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T20:55:48.909 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:55:48.909 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:48.909 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:48.909 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34797.0:101685 6.e 6:7f2c1984:::benchmark_data_smithi026_223868_object559:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e125)", 2023-09-23T20:55:48.909 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T20:53:51.089634+0000", 2023-09-23T20:55:48.909 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 117.80075379199999, 2023-09-23T20:55:48.909 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.080046844000000006, 2023-09-23T20:55:48.909 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T20:55:48.909 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:55:48.909 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T20:55:48.909 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34797", 2023-09-23T20:55:48.909 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T20:55:48.909 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 101685 2023-09-23T20:55:48.910 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:48.910 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T20:55:48.910 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:48.910 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T20:55:48.910 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:51.089634+0000", 2023-09-23T20:55:48.910 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:55:48.910 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:48.910 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:48.910 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T20:55:48.910 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:51.089634+0000", 2023-09-23T20:55:48.910 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:55:48.910 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:48.911 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:48.911 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T20:55:48.911 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:51.089634+0000", 2023-09-23T20:55:48.911 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T20:55:48.911 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:48.911 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:48.911 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T20:55:48.911 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:51.089636+0000", 2023-09-23T20:55:48.911 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.148e-06 2023-09-23T20:55:48.911 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:48.911 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:48.911 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T20:55:48.911 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:51.089641+0000", 2023-09-23T20:55:48.912 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.9520000000000002e-06 2023-09-23T20:55:48.912 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:48.912 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:48.912 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T20:55:48.912 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:51.089646+0000", 2023-09-23T20:55:48.912 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.4909999999999999e-06 2023-09-23T20:55:48.912 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:48.912 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:48.912 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:55:48.912 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:51.116613+0000", 2023-09-23T20:55:48.912 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.026967676999999999 2023-09-23T20:55:48.912 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:48.912 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:48.913 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T20:55:48.913 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:51.116761+0000", 2023-09-23T20:55:48.913 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00014803400000000001 2023-09-23T20:55:48.913 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:48.913 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:48.913 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 2,7", 2023-09-23T20:55:48.913 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:51.116906+0000", 2023-09-23T20:55:48.913 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00014433999999999999 2023-09-23T20:55:48.913 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:48.913 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:48.913 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:55:48.913 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:51.135282+0000", 2023-09-23T20:55:48.913 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.018376414000000001 2023-09-23T20:55:48.914 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:48.914 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:48.914 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T20:55:48.914 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:51.135296+0000", 2023-09-23T20:55:48.914 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.3905e-05 2023-09-23T20:55:48.914 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:48.914 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:48.914 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:55:48.914 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:51.167842+0000", 2023-09-23T20:55:48.914 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.032545835000000002 2023-09-23T20:55:48.914 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:48.914 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:48.915 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T20:55:48.915 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:51.167897+0000", 2023-09-23T20:55:48.915 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.5690999999999999e-05 2023-09-23T20:55:48.915 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:48.915 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:48.915 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T20:55:48.915 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:51.169681+0000", 2023-09-23T20:55:48.915 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0017836340000000001 2023-09-23T20:55:48.915 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:55:48.915 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T20:55:48.915 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:55:48.915 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:48.915 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:48.916 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34797.0:114984 6.e 6:77dbcdd0:::benchmark_data_smithi026_223868_object13858:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e132)", 2023-09-23T20:55:48.916 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T20:54:12.945909+0000", 2023-09-23T20:55:48.916 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 95.944479053999999, 2023-09-23T20:55:48.916 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.081195696, 2023-09-23T20:55:48.916 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T20:55:48.916 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:55:48.916 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T20:55:48.916 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34797", 2023-09-23T20:55:48.916 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T20:55:48.916 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 114984 2023-09-23T20:55:48.916 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:48.916 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T20:55:48.916 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:48.917 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T20:55:48.917 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:12.945909+0000", 2023-09-23T20:55:48.917 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:55:48.917 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:48.917 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:48.917 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T20:55:48.917 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:12.945909+0000", 2023-09-23T20:55:48.917 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:55:48.917 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:48.917 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:48.917 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T20:55:48.917 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:12.945908+0000", 2023-09-23T20:55:48.917 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T20:55:48.918 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:48.918 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:48.918 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T20:55:48.918 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:12.945913+0000", 2023-09-23T20:55:48.918 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.305e-06 2023-09-23T20:55:48.918 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:48.918 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:48.918 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T20:55:48.918 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:12.945924+0000", 2023-09-23T20:55:48.918 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.115e-05 2023-09-23T20:55:48.918 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:48.918 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:48.919 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T20:55:48.919 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:12.945933+0000", 2023-09-23T20:55:48.919 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.2969999999999992e-06 2023-09-23T20:55:48.919 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:48.919 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:48.919 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:55:48.919 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:12.950695+0000", 2023-09-23T20:55:48.919 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0047620850000000001 2023-09-23T20:55:48.919 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:48.919 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:48.919 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T20:55:48.919 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:12.950706+0000", 2023-09-23T20:55:48.919 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0858e-05 2023-09-23T20:55:48.920 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:48.920 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:48.920 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:55:48.920 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:14.019393+0000", 2023-09-23T20:55:48.920 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.068686851 2023-09-23T20:55:48.920 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:48.920 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:48.920 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T20:55:48.920 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:14.019595+0000", 2023-09-23T20:55:48.920 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000201988 2023-09-23T20:55:48.920 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:48.920 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:48.920 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 2,4", 2023-09-23T20:55:48.921 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:14.019783+0000", 2023-09-23T20:55:48.921 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000188104 2023-09-23T20:55:48.921 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:48.921 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:48.921 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:55:48.921 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:14.023500+0000", 2023-09-23T20:55:48.921 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00371645 2023-09-23T20:55:48.921 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:48.921 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:48.921 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T20:55:48.921 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:14.026242+0000", 2023-09-23T20:55:48.922 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.002742474 2023-09-23T20:55:48.922 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:48.922 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:48.922 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:55:48.922 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:14.026508+0000", 2023-09-23T20:55:48.922 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00026582400000000001 2023-09-23T20:55:48.922 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:48.922 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:48.922 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T20:55:48.922 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:14.026547+0000", 2023-09-23T20:55:48.922 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.9169999999999999e-05 2023-09-23T20:55:48.922 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:48.923 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:48.923 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T20:55:48.923 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:14.027105+0000", 2023-09-23T20:55:48.923 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00055742400000000005 2023-09-23T20:55:48.923 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:55:48.923 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T20:55:48.923 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:55:48.923 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:48.923 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:48.923 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34797.0:114994 6.e 6:7ff0892b:::benchmark_data_smithi026_223868_object13868:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e132)", 2023-09-23T20:55:48.923 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T20:54:12.968874+0000", 2023-09-23T20:55:48.923 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 95.921514309000003, 2023-09-23T20:55:48.923 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0589051819999999, 2023-09-23T20:55:48.923 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T20:55:48.924 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:55:48.924 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T20:55:48.924 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34797", 2023-09-23T20:55:48.924 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T20:55:48.924 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 114994 2023-09-23T20:55:48.924 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:48.924 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T20:55:48.924 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:48.924 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T20:55:48.924 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:12.968874+0000", 2023-09-23T20:55:48.924 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:55:48.924 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:48.925 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:48.925 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T20:55:48.925 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:12.968874+0000", 2023-09-23T20:55:48.925 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:55:48.925 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:48.925 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:48.925 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T20:55:48.925 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:12.968873+0000", 2023-09-23T20:55:48.925 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T20:55:48.925 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:48.925 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:48.925 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T20:55:48.925 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:12.968877+0000", 2023-09-23T20:55:48.926 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.1339999999999997e-06 2023-09-23T20:55:48.926 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:48.926 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:48.926 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T20:55:48.926 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:12.968886+0000", 2023-09-23T20:55:48.926 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.5050000000000007e-06 2023-09-23T20:55:48.926 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:48.926 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:48.926 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T20:55:48.926 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:12.968892+0000", 2023-09-23T20:55:48.926 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.8510000000000004e-06 2023-09-23T20:55:48.926 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:48.927 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:48.927 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:55:48.927 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:12.968950+0000", 2023-09-23T20:55:48.927 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.8746999999999998e-05 2023-09-23T20:55:48.927 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:48.927 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:48.927 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T20:55:48.927 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:12.968961+0000", 2023-09-23T20:55:48.927 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.115e-05 2023-09-23T20:55:48.927 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:48.927 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:48.927 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:55:48.927 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:14.023568+0000", 2023-09-23T20:55:48.928 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0546062759999999 2023-09-23T20:55:48.928 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:48.928 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:48.928 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T20:55:48.928 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:14.023704+0000", 2023-09-23T20:55:48.928 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000136188 2023-09-23T20:55:48.928 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:48.928 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:48.928 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 2,4", 2023-09-23T20:55:48.928 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:14.023794+0000", 2023-09-23T20:55:48.928 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.9986000000000002e-05 2023-09-23T20:55:48.928 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:48.928 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:48.929 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T20:55:48.929 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:14.027128+0000", 2023-09-23T20:55:48.929 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0033338510000000001 2023-09-23T20:55:48.929 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:48.929 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:48.929 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:55:48.929 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:14.027235+0000", 2023-09-23T20:55:48.929 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000106932 2023-09-23T20:55:48.929 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:48.929 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:48.929 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:55:48.929 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:14.027379+0000", 2023-09-23T20:55:48.929 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00014385 2023-09-23T20:55:48.930 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:48.930 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:48.930 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T20:55:48.930 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:14.027432+0000", 2023-09-23T20:55:48.930 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.3171999999999998e-05 2023-09-23T20:55:48.930 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:48.930 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:48.930 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T20:55:48.930 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:14.027779+0000", 2023-09-23T20:55:48.930 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00034717400000000001 2023-09-23T20:55:48.930 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:55:48.930 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T20:55:48.931 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:55:48.931 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:48.931 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:48.931 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34797.0:130503 6.4 6:236b9417:::benchmark_data_smithi026_223868_object29377:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e136)", 2023-09-23T20:55:48.931 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T20:54:30.930235+0000", 2023-09-23T20:55:48.931 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 77.960153325999997, 2023-09-23T20:55:48.931 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.1274586960000001, 2023-09-23T20:55:48.931 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T20:55:48.931 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:55:48.931 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T20:55:48.931 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34797", 2023-09-23T20:55:48.932 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T20:55:48.932 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 130503 2023-09-23T20:55:48.932 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:48.932 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T20:55:48.932 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:48.932 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T20:55:48.932 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:30.930235+0000", 2023-09-23T20:55:48.932 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:55:48.932 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:48.932 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:48.932 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T20:55:48.932 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:30.930235+0000", 2023-09-23T20:55:48.932 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:55:48.932 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:48.933 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:48.933 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T20:55:48.933 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:30.930233+0000", 2023-09-23T20:55:48.933 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T20:55:48.933 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:48.933 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:48.933 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T20:55:48.933 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:30.930242+0000", 2023-09-23T20:55:48.933 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.8769999999999999e-06 2023-09-23T20:55:48.933 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:48.933 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:48.933 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T20:55:48.933 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:30.930263+0000", 2023-09-23T20:55:48.934 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.1492999999999999e-05 2023-09-23T20:55:48.934 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:48.934 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:48.934 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T20:55:48.934 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:30.930273+0000", 2023-09-23T20:55:48.934 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.9350000000000006e-06 2023-09-23T20:55:48.934 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:48.934 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:48.934 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:55:48.934 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:30.930336+0000", 2023-09-23T20:55:48.934 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.2428999999999995e-05 2023-09-23T20:55:48.935 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:48.935 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:48.935 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T20:55:48.935 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:30.930348+0000", 2023-09-23T20:55:48.935 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.2551000000000001e-05 2023-09-23T20:55:48.935 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:48.935 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:48.935 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:55:48.935 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:31.883549+0000", 2023-09-23T20:55:48.935 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.95320037800000001 2023-09-23T20:55:48.935 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:48.935 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:48.935 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T20:55:48.936 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:31.883563+0000", 2023-09-23T20:55:48.936 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.384e-05 2023-09-23T20:55:48.936 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:48.936 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:48.936 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:55:48.936 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:33.031575+0000", 2023-09-23T20:55:48.936 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1480122770000001 2023-09-23T20:55:48.936 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:48.936 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:48.936 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T20:55:48.936 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:33.031710+0000", 2023-09-23T20:55:48.936 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00013495900000000001 2023-09-23T20:55:48.936 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:48.937 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:48.937 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 3,5,6", 2023-09-23T20:55:48.937 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:33.031848+0000", 2023-09-23T20:55:48.937 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000138135 2023-09-23T20:55:48.937 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:48.937 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:48.937 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T20:55:48.937 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:33.040431+0000", 2023-09-23T20:55:48.937 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0085826510000000002 2023-09-23T20:55:48.937 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:48.937 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:48.937 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:55:48.938 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:33.040561+0000", 2023-09-23T20:55:48.938 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00013077100000000001 2023-09-23T20:55:48.938 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:48.938 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:48.938 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:55:48.938 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:33.045365+0000", 2023-09-23T20:55:48.938 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0048039889999999998 2023-09-23T20:55:48.938 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:48.938 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:48.938 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:55:48.938 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:33.057544+0000", 2023-09-23T20:55:48.939 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.012178227 2023-09-23T20:55:48.939 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:48.939 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:48.939 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T20:55:48.939 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:33.057593+0000", 2023-09-23T20:55:48.939 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.9086999999999999e-05 2023-09-23T20:55:48.939 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:48.939 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:48.939 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T20:55:48.939 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:33.057693+0000", 2023-09-23T20:55:48.939 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000100697 2023-09-23T20:55:48.939 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:55:48.939 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T20:55:48.940 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:55:48.940 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:48.940 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:48.940 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34797.0:130517 6.14 6:2e6d4a50:::benchmark_data_smithi026_223868_object29391:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e136)", 2023-09-23T20:55:48.940 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T20:54:30.938770+0000", 2023-09-23T20:55:48.940 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 77.951617683999999, 2023-09-23T20:55:48.940 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.1191109109999999, 2023-09-23T20:55:48.940 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T20:55:48.940 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:55:48.940 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T20:55:48.940 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34797", 2023-09-23T20:55:48.940 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T20:55:48.940 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 130517 2023-09-23T20:55:48.941 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:48.941 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T20:55:48.941 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:48.941 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T20:55:48.941 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:30.938770+0000", 2023-09-23T20:55:48.941 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:55:48.941 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:48.941 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:48.941 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T20:55:48.941 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:30.938770+0000", 2023-09-23T20:55:48.941 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:55:48.941 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:48.942 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:48.942 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T20:55:48.942 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:30.938769+0000", 2023-09-23T20:55:48.942 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.999999 2023-09-23T20:55:48.942 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:48.942 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:48.942 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T20:55:48.942 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:30.938775+0000", 2023-09-23T20:55:48.942 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.0100000000000001e-06 2023-09-23T20:55:48.942 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:48.942 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:48.942 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T20:55:48.943 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:30.938790+0000", 2023-09-23T20:55:48.943 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.4391e-05 2023-09-23T20:55:48.943 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:48.943 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:48.943 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T20:55:48.943 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:30.938799+0000", 2023-09-23T20:55:48.943 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.2e-06 2023-09-23T20:55:48.943 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:48.943 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:48.943 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:55:48.943 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:30.938857+0000", 2023-09-23T20:55:48.943 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.8446999999999998e-05 2023-09-23T20:55:48.943 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:48.943 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:48.944 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T20:55:48.944 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:30.938871+0000", 2023-09-23T20:55:48.944 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.4042e-05 2023-09-23T20:55:48.944 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:48.944 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:48.944 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:55:48.944 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:31.884077+0000", 2023-09-23T20:55:48.944 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.94520575100000004 2023-09-23T20:55:48.944 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:48.944 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:48.944 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T20:55:48.944 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:31.884087+0000", 2023-09-23T20:55:48.945 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0135999999999999e-05 2023-09-23T20:55:48.945 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:48.945 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:48.945 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:55:48.945 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:33.034200+0000", 2023-09-23T20:55:48.945 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1501126020000001 2023-09-23T20:55:48.945 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:48.945 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:48.945 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T20:55:48.945 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:33.034313+0000", 2023-09-23T20:55:48.945 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000113507 2023-09-23T20:55:48.946 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:48.946 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:48.946 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 3,5,6", 2023-09-23T20:55:48.946 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:33.034465+0000", 2023-09-23T20:55:48.946 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00015155799999999999 2023-09-23T20:55:48.946 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:48.946 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:48.946 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:55:48.946 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:33.040587+0000", 2023-09-23T20:55:48.946 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0061222459999999996 2023-09-23T20:55:48.946 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:48.946 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:48.947 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T20:55:48.947 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:33.042787+0000", 2023-09-23T20:55:48.947 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0022003410000000002 2023-09-23T20:55:48.947 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:48.947 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:48.947 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:55:48.947 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:33.045400+0000", 2023-09-23T20:55:48.947 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0026128879999999998 2023-09-23T20:55:48.947 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:48.947 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:48.947 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:55:48.947 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:33.057734+0000", 2023-09-23T20:55:48.947 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.012333823000000001 2023-09-23T20:55:48.947 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:48.948 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:48.948 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T20:55:48.948 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:33.057774+0000", 2023-09-23T20:55:48.948 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.9455000000000001e-05 2023-09-23T20:55:48.948 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:48.948 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:48.948 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T20:55:48.948 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:33.057881+0000", 2023-09-23T20:55:48.948 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000107697 2023-09-23T20:55:48.948 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:55:48.948 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T20:55:48.948 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:55:48.948 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:48.949 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:48.949 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34797.0:130519 6.14 6:2fbfebb3:::benchmark_data_smithi026_223868_object29393:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e136)", 2023-09-23T20:55:48.949 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T20:54:30.939911+0000", 2023-09-23T20:55:48.949 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 77.950476585999994, 2023-09-23T20:55:48.949 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.1181728999999998, 2023-09-23T20:55:48.949 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T20:55:48.949 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:55:48.949 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T20:55:48.949 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34797", 2023-09-23T20:55:48.949 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T20:55:48.949 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 130519 2023-09-23T20:55:48.949 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:48.950 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T20:55:48.950 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:48.950 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T20:55:48.950 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:30.939911+0000", 2023-09-23T20:55:48.950 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:55:48.950 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:48.950 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:48.950 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T20:55:48.950 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:30.939911+0000", 2023-09-23T20:55:48.950 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:55:48.951 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:48.951 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:48.951 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T20:55:48.951 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:30.939911+0000", 2023-09-23T20:55:48.951 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.999999 2023-09-23T20:55:48.951 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:48.951 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:48.951 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T20:55:48.951 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:30.939916+0000", 2023-09-23T20:55:48.951 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.0220000000000004e-06 2023-09-23T20:55:48.951 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:48.951 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:48.951 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T20:55:48.952 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:30.939930+0000", 2023-09-23T20:55:48.952 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.4129e-05 2023-09-23T20:55:48.952 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:48.952 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:48.952 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T20:55:48.952 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:30.939938+0000", 2023-09-23T20:55:48.952 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.0909999999999993e-06 2023-09-23T20:55:48.952 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:48.952 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:48.952 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:55:48.952 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:30.939996+0000", 2023-09-23T20:55:48.952 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.8115999999999997e-05 2023-09-23T20:55:48.953 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:48.953 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:48.953 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T20:55:48.953 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:30.940008+0000", 2023-09-23T20:55:48.953 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1711e-05 2023-09-23T20:55:48.953 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:48.953 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:48.953 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:55:48.953 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:31.884115+0000", 2023-09-23T20:55:48.953 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.94410687100000001 2023-09-23T20:55:48.953 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:48.953 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:48.953 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T20:55:48.954 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:31.884123+0000", 2023-09-23T20:55:48.954 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.3990000000000004e-06 2023-09-23T20:55:48.954 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:48.954 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:48.954 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:55:48.954 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:33.036642+0000", 2023-09-23T20:55:48.954 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1525194350000001 2023-09-23T20:55:48.954 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:48.954 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:48.954 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T20:55:48.954 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:33.036778+0000", 2023-09-23T20:55:48.955 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000135609 2023-09-23T20:55:48.955 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:48.955 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:48.955 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 3,5,6", 2023-09-23T20:55:48.955 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:33.036924+0000", 2023-09-23T20:55:48.955 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000146198 2023-09-23T20:55:48.955 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:48.955 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:48.955 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T20:55:48.955 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:33.042794+0000", 2023-09-23T20:55:48.956 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0058697810000000001 2023-09-23T20:55:48.956 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:48.956 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:48.956 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:55:48.956 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:33.045426+0000", 2023-09-23T20:55:48.956 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0026324569999999999 2023-09-23T20:55:48.956 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:48.956 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:48.956 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:55:48.956 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:33.048941+0000", 2023-09-23T20:55:48.956 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.003514667 2023-09-23T20:55:48.956 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:48.956 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:48.957 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:55:48.957 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:33.057922+0000", 2023-09-23T20:55:48.957 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0089806850000000004 2023-09-23T20:55:48.957 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:48.957 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:48.957 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T20:55:48.957 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:33.057967+0000", 2023-09-23T20:55:48.957 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.5430999999999999e-05 2023-09-23T20:55:48.957 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:48.957 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:48.957 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T20:55:48.958 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:33.058084+0000", 2023-09-23T20:55:48.958 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000117081 2023-09-23T20:55:48.958 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:55:48.958 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T20:55:48.958 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:55:48.958 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:48.958 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:48.958 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34797.0:130536 6.14 6:2a329320:::benchmark_data_smithi026_223868_object29410:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e136)", 2023-09-23T20:55:48.958 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T20:54:30.980524+0000", 2023-09-23T20:55:48.958 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 77.909864206999998, 2023-09-23T20:55:48.958 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.0776815489999998, 2023-09-23T20:55:48.958 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T20:55:48.959 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:55:48.959 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T20:55:48.959 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34797", 2023-09-23T20:55:48.959 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T20:55:48.959 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 130536 2023-09-23T20:55:48.959 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:48.959 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T20:55:48.959 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:48.959 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T20:55:48.959 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:30.980524+0000", 2023-09-23T20:55:48.960 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:55:48.960 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:48.960 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:48.960 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T20:55:48.960 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:30.980524+0000", 2023-09-23T20:55:48.960 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:55:48.960 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:48.960 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:48.960 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T20:55:48.960 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:30.980522+0000", 2023-09-23T20:55:48.960 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999981 2023-09-23T20:55:48.960 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:48.961 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:48.961 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T20:55:48.961 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:30.980533+0000", 2023-09-23T20:55:48.961 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0721e-05 2023-09-23T20:55:48.961 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:48.961 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:48.961 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T20:55:48.961 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:30.980555+0000", 2023-09-23T20:55:48.961 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.1789000000000001e-05 2023-09-23T20:55:48.961 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:48.961 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:48.962 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T20:55:48.962 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:30.980565+0000", 2023-09-23T20:55:48.962 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0492999999999999e-05 2023-09-23T20:55:48.962 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:48.962 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:48.962 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:55:48.962 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:30.980633+0000", 2023-09-23T20:55:48.962 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.8361999999999994e-05 2023-09-23T20:55:48.962 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:48.962 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:48.962 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T20:55:48.963 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:30.980647+0000", 2023-09-23T20:55:48.963 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.3818999999999999e-05 2023-09-23T20:55:48.963 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:48.963 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:48.963 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:55:48.963 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:31.884138+0000", 2023-09-23T20:55:48.963 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.90349046700000002 2023-09-23T20:55:48.963 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:48.963 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:48.963 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T20:55:48.963 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:31.884142+0000", 2023-09-23T20:55:48.965 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.2660000000000003e-06 2023-09-23T20:55:48.965 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:48.965 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:48.966 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:55:48.966 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:33.038763+0000", 2023-09-23T20:55:48.966 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1546210180000001 2023-09-23T20:55:48.966 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:48.966 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:48.966 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T20:55:48.966 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:33.038886+0000", 2023-09-23T20:55:48.966 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00012294000000000001 2023-09-23T20:55:48.966 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:48.966 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:48.966 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 3,5,6", 2023-09-23T20:55:48.967 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:33.039020+0000", 2023-09-23T20:55:48.967 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000133621 2023-09-23T20:55:48.967 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:48.967 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:48.967 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T20:55:48.967 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:33.042797+0000", 2023-09-23T20:55:48.967 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0037779649999999999 2023-09-23T20:55:48.967 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:48.967 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:48.967 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:55:48.967 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:33.045454+0000", 2023-09-23T20:55:48.967 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.002656215 2023-09-23T20:55:48.968 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:48.968 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:48.968 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:55:48.968 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:33.048976+0000", 2023-09-23T20:55:48.968 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0035227209999999999 2023-09-23T20:55:48.968 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:48.968 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:48.968 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:55:48.968 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:33.058125+0000", 2023-09-23T20:55:48.968 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0091488279999999995 2023-09-23T20:55:48.968 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:48.968 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:48.969 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T20:55:48.969 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:33.058151+0000", 2023-09-23T20:55:48.969 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.5992000000000001e-05 2023-09-23T20:55:48.969 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:48.969 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:48.969 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T20:55:48.969 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:33.058205+0000", 2023-09-23T20:55:48.969 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.4104999999999998e-05 2023-09-23T20:55:48.969 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:55:48.969 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T20:55:48.969 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:55:48.969 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:48.970 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:48.970 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34797.0:165115 6.11 6:8a9c510c:::benchmark_data_smithi026_223868_object63989:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e154)", 2023-09-23T20:55:48.970 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T20:55:17.458662+0000", 2023-09-23T20:55:48.970 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 31.431725932999999, 2023-09-23T20:55:48.970 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0640936540000001, 2023-09-23T20:55:48.970 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T20:55:48.970 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "reached pg", 2023-09-23T20:55:48.970 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T20:55:48.970 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34797", 2023-09-23T20:55:48.971 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T20:55:48.971 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 165115 2023-09-23T20:55:48.971 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:48.971 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T20:55:48.971 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:48.971 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T20:55:48.971 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:17.458662+0000", 2023-09-23T20:55:48.971 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:55:48.971 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:48.971 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:48.971 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T20:55:48.971 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:17.458662+0000", 2023-09-23T20:55:48.972 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:55:48.972 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:48.972 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:48.972 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T20:55:48.972 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:17.458661+0000", 2023-09-23T20:55:48.972 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T20:55:48.972 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:48.972 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:48.972 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T20:55:48.972 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:17.458669+0000", 2023-09-23T20:55:48.972 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.2549999999999997e-06 2023-09-23T20:55:48.972 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:48.973 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:48.973 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T20:55:48.973 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:17.458686+0000", 2023-09-23T20:55:48.973 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.7163000000000002e-05 2023-09-23T20:55:48.973 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:48.973 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:48.973 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T20:55:48.973 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:17.458696+0000", 2023-09-23T20:55:48.973 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.6299999999999993e-06 2023-09-23T20:55:48.973 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:48.973 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:48.974 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:55:48.974 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:17.458754+0000", 2023-09-23T20:55:48.974 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.8409999999999998e-05 2023-09-23T20:55:48.974 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:48.974 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:48.974 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T20:55:48.974 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:17.458896+0000", 2023-09-23T20:55:48.974 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00014230699999999999 2023-09-23T20:55:48.974 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:48.974 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:48.975 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 2,6", 2023-09-23T20:55:48.975 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:17.459343+0000", 2023-09-23T20:55:48.975 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00044672499999999998 2023-09-23T20:55:48.975 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:48.975 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:48.975 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T20:55:48.975 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:17.465297+0000", 2023-09-23T20:55:48.975 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0059539279999999998 2023-09-23T20:55:48.975 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:48.975 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:48.975 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:55:48.976 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:17.473171+0000", 2023-09-23T20:55:48.976 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0078734059999999995 2023-09-23T20:55:48.976 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:48.976 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:48.976 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:55:48.976 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:17.487954+0000", 2023-09-23T20:55:48.976 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.014783889 2023-09-23T20:55:48.976 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:48.976 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:48.977 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T20:55:48.977 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:17.487966+0000", 2023-09-23T20:55:48.977 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1089e-05 2023-09-23T20:55:48.977 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:48.977 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:48.977 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:55:48.977 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:18.522668+0000", 2023-09-23T20:55:48.977 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0347028009999999 2023-09-23T20:55:48.977 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:48.977 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:48.977 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T20:55:48.977 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:18.522756+0000", 2023-09-23T20:55:48.978 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.7397000000000004e-05 2023-09-23T20:55:48.978 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:55:48.978 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T20:55:48.978 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:55:48.978 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:48.978 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:48.978 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34797.0:165119 6.11 6:888a8162:::benchmark_data_smithi026_223868_object63993:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e154)", 2023-09-23T20:55:48.978 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T20:55:17.470762+0000", 2023-09-23T20:55:48.978 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 31.419626126000001, 2023-09-23T20:55:48.978 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.071012407, 2023-09-23T20:55:48.978 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T20:55:48.979 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "reached pg", 2023-09-23T20:55:48.979 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T20:55:48.979 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34797", 2023-09-23T20:55:48.979 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T20:55:48.979 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 165119 2023-09-23T20:55:48.979 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:48.979 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T20:55:48.979 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:48.979 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T20:55:48.979 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:17.470762+0000", 2023-09-23T20:55:48.979 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:55:48.979 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:48.980 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:48.980 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T20:55:48.980 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:17.470762+0000", 2023-09-23T20:55:48.980 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:55:48.980 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:48.980 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:48.980 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T20:55:48.980 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:17.470760+0000", 2023-09-23T20:55:48.980 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999981 2023-09-23T20:55:48.980 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:48.980 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:48.980 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T20:55:48.981 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:17.470768+0000", 2023-09-23T20:55:48.981 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.3520000000000007e-06 2023-09-23T20:55:48.981 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:48.981 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:48.981 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T20:55:48.981 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:17.470789+0000", 2023-09-23T20:55:48.981 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.0230000000000001e-05 2023-09-23T20:55:48.981 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:48.981 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:48.981 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T20:55:48.981 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:17.470799+0000", 2023-09-23T20:55:48.981 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.083e-05 2023-09-23T20:55:48.982 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:48.982 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:48.982 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:55:48.982 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:17.470949+0000", 2023-09-23T20:55:48.982 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00014940900000000001 2023-09-23T20:55:48.982 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:48.982 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:48.982 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T20:55:48.982 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:17.471101+0000", 2023-09-23T20:55:48.982 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000152552 2023-09-23T20:55:48.982 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:48.983 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:48.983 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 2,6", 2023-09-23T20:55:48.983 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:17.471536+0000", 2023-09-23T20:55:48.983 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00043481999999999999 2023-09-23T20:55:48.983 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:48.983 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:48.983 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T20:55:48.983 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:17.479740+0000", 2023-09-23T20:55:48.983 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0082035990000000007 2023-09-23T20:55:48.983 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:48.983 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:48.984 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:55:48.984 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:17.488026+0000", 2023-09-23T20:55:48.984 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0082866199999999997 2023-09-23T20:55:48.984 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:48.984 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:48.984 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T20:55:48.984 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:17.488036+0000", 2023-09-23T20:55:48.984 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.3840000000000003e-06 2023-09-23T20:55:48.984 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:48.984 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:48.984 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:55:48.984 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:18.523188+0000", 2023-09-23T20:55:48.985 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.035152157 2023-09-23T20:55:48.985 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:48.985 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:48.985 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for missing object", 2023-09-23T20:55:48.985 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:18.523282+0000", 2023-09-23T20:55:48.985 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.4153000000000005e-05 2023-09-23T20:55:48.986 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:48.986 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:48.986 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:55:48.986 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:18.541721+0000", 2023-09-23T20:55:48.986 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.018438985000000001 2023-09-23T20:55:48.986 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:48.986 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:48.987 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T20:55:48.987 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:18.541774+0000", 2023-09-23T20:55:48.987 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.3115000000000002e-05 2023-09-23T20:55:48.987 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:55:48.987 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T20:55:48.987 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:55:48.987 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:48.987 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:48.987 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34797.0:165120 6.4 6:2711d346:::benchmark_data_smithi026_223868_object63994:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e154)", 2023-09-23T20:55:48.987 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T20:55:17.470823+0000", 2023-09-23T20:55:48.987 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 31.419565264999999, 2023-09-23T20:55:48.988 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.034766882, 2023-09-23T20:55:48.988 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T20:55:48.988 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "reached pg", 2023-09-23T20:55:48.988 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T20:55:48.988 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34797", 2023-09-23T20:55:48.988 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T20:55:48.988 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 165120 2023-09-23T20:55:48.988 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:48.988 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T20:55:48.989 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:48.989 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T20:55:48.989 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:17.470823+0000", 2023-09-23T20:55:48.989 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:55:48.989 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:48.989 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:48.989 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T20:55:48.989 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:17.470823+0000", 2023-09-23T20:55:48.989 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:55:48.990 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:48.990 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:48.990 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T20:55:48.990 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:17.470822+0000", 2023-09-23T20:55:48.990 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T20:55:48.990 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:48.990 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:48.990 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T20:55:48.990 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:17.470826+0000", 2023-09-23T20:55:48.991 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.7270000000000001e-06 2023-09-23T20:55:48.991 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:48.991 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:48.991 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T20:55:48.991 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:17.470834+0000", 2023-09-23T20:55:48.991 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.8679999999999999e-06 2023-09-23T20:55:48.991 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:48.991 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:48.991 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T20:55:48.991 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:17.470840+0000", 2023-09-23T20:55:48.991 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.2750000000000001e-06 2023-09-23T20:55:48.991 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:48.992 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:48.992 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:55:48.992 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:17.473228+0000", 2023-09-23T20:55:48.992 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0023883070000000001 2023-09-23T20:55:48.992 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:48.992 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:48.992 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T20:55:48.992 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:17.473373+0000", 2023-09-23T20:55:48.992 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00014514199999999999 2023-09-23T20:55:48.992 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:48.992 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:48.993 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 3,5,6", 2023-09-23T20:55:48.993 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:17.473855+0000", 2023-09-23T20:55:48.993 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00048128899999999998 2023-09-23T20:55:48.993 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:48.993 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:48.993 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:55:48.993 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:17.479648+0000", 2023-09-23T20:55:48.993 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0057931959999999996 2023-09-23T20:55:48.993 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:48.993 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:48.993 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T20:55:48.994 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:17.479658+0000", 2023-09-23T20:55:48.994 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0404999999999999e-05 2023-09-23T20:55:48.994 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:48.994 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:48.994 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:55:48.994 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:17.487974+0000", 2023-09-23T20:55:48.994 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.008316024 2023-09-23T20:55:48.994 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:48.994 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:48.994 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T20:55:48.994 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:17.487985+0000", 2023-09-23T20:55:48.995 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0193e-05 2023-09-23T20:55:48.995 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:48.995 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:48.995 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:55:48.995 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:18.505499+0000", 2023-09-23T20:55:48.995 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0175144899999999 2023-09-23T20:55:48.995 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:48.995 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:48.995 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T20:55:48.995 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:18.505590+0000", 2023-09-23T20:55:48.995 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.0523e-05 2023-09-23T20:55:48.995 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:55:48.996 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T20:55:48.996 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:55:48.996 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:48.996 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:48.996 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34797.0:165133 6.4 6:2151236b:::benchmark_data_smithi026_223868_object64007:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e155)", 2023-09-23T20:55:48.996 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T20:55:17.506170+0000", 2023-09-23T20:55:48.996 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 31.384217663000001, 2023-09-23T20:55:48.996 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.062355591, 2023-09-23T20:55:48.996 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T20:55:48.996 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:55:48.996 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T20:55:48.996 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34797", 2023-09-23T20:55:48.997 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T20:55:48.997 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 165133 2023-09-23T20:55:48.997 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:48.997 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T20:55:48.997 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:48.997 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T20:55:48.997 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:17.506170+0000", 2023-09-23T20:55:48.997 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:55:48.997 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:48.997 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:48.997 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T20:55:48.997 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:17.506170+0000", 2023-09-23T20:55:48.998 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:55:48.998 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:48.998 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:48.998 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T20:55:48.998 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:17.506169+0000", 2023-09-23T20:55:48.998 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999981 2023-09-23T20:55:48.998 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:48.998 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:48.998 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T20:55:48.998 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:17.506178+0000", 2023-09-23T20:55:48.998 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.3230000000000004e-06 2023-09-23T20:55:48.999 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:48.999 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:48.999 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T20:55:48.999 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:17.506202+0000", 2023-09-23T20:55:48.999 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.3856999999999999e-05 2023-09-23T20:55:48.999 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:48.999 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:48.999 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T20:55:48.999 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:17.506213+0000", 2023-09-23T20:55:48.999 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1454000000000001e-05 2023-09-23T20:55:48.999 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:48.999 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:48.999 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:55:49.000 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:17.506304+0000", 2023-09-23T20:55:49.000 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.1155000000000004e-05 2023-09-23T20:55:49.000 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:49.000 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:49.000 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T20:55:49.000 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:17.506314+0000", 2023-09-23T20:55:49.000 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.6509999999999996e-06 2023-09-23T20:55:49.000 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:49.000 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:49.000 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:55:49.000 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:18.505917+0000", 2023-09-23T20:55:49.001 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.99960311199999996 2023-09-23T20:55:49.001 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:49.001 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:49.001 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T20:55:49.001 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:18.506065+0000", 2023-09-23T20:55:49.001 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00014810299999999999 2023-09-23T20:55:49.002 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:49.002 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:49.002 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 3,5", 2023-09-23T20:55:49.002 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:18.506566+0000", 2023-09-23T20:55:49.002 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00050105199999999999 2023-09-23T20:55:49.002 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:49.002 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:49.002 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T20:55:49.002 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:18.508896+0000", 2023-09-23T20:55:49.002 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0023299129999999999 2023-09-23T20:55:49.003 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:49.003 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:49.003 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:55:49.003 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:18.509051+0000", 2023-09-23T20:55:49.003 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00015425899999999999 2023-09-23T20:55:49.003 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:49.003 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:49.003 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:55:49.003 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:18.568363+0000", 2023-09-23T20:55:49.003 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.059312294000000002 2023-09-23T20:55:49.003 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:49.004 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:49.004 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T20:55:49.004 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:18.568418+0000", 2023-09-23T20:55:49.004 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.5701000000000001e-05 2023-09-23T20:55:49.004 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:49.004 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:49.004 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T20:55:49.004 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:18.568526+0000", 2023-09-23T20:55:49.004 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000107416 2023-09-23T20:55:49.004 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:55:49.004 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T20:55:49.004 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:55:49.005 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:49.005 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:49.005 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34797.0:165136 6.1 6:8407df14:::benchmark_data_smithi026_223868_object64010:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e155)", 2023-09-23T20:55:49.005 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T20:55:17.506228+0000", 2023-09-23T20:55:49.005 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 31.384159802999999, 2023-09-23T20:55:49.005 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0166797729999999, 2023-09-23T20:55:49.005 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T20:55:49.005 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:55:49.005 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T20:55:49.005 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34797", 2023-09-23T20:55:49.005 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T20:55:49.005 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 165136 2023-09-23T20:55:49.006 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:49.006 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T20:55:49.006 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:49.006 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T20:55:49.006 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:17.506228+0000", 2023-09-23T20:55:49.006 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:55:49.006 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:49.006 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:49.006 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T20:55:49.006 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:17.506228+0000", 2023-09-23T20:55:49.007 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:55:49.007 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:49.007 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:49.007 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T20:55:49.007 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:17.506228+0000", 2023-09-23T20:55:49.007 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T20:55:49.007 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:49.007 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:49.007 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T20:55:49.007 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:17.506231+0000", 2023-09-23T20:55:49.007 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.0460000000000001e-06 2023-09-23T20:55:49.007 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:49.008 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:49.008 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T20:55:49.008 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:17.506237+0000", 2023-09-23T20:55:49.008 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.9440000000000002e-06 2023-09-23T20:55:49.008 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:49.008 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:49.008 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T20:55:49.008 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:17.506242+0000", 2023-09-23T20:55:49.008 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.1109999999999999e-06 2023-09-23T20:55:49.008 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:49.008 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:49.008 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:55:49.009 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:17.506295+0000", 2023-09-23T20:55:49.009 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.2828999999999999e-05 2023-09-23T20:55:49.009 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:49.009 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:49.009 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T20:55:49.009 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:17.506305+0000", 2023-09-23T20:55:49.009 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0373999999999999e-05 2023-09-23T20:55:49.009 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:49.009 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:49.009 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:55:49.009 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:18.494023+0000", 2023-09-23T20:55:49.009 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.98771804900000004 2023-09-23T20:55:49.009 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:49.010 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:49.010 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T20:55:49.010 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:18.494131+0000", 2023-09-23T20:55:49.010 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000107593 2023-09-23T20:55:49.010 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:49.010 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:49.010 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 2", 2023-09-23T20:55:49.010 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:18.494586+0000", 2023-09-23T20:55:49.010 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00045516799999999999 2023-09-23T20:55:49.010 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:49.011 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:49.011 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T20:55:49.011 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:18.522748+0000", 2023-09-23T20:55:49.011 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.028161951000000001 2023-09-23T20:55:49.011 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:49.011 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:49.011 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:55:49.011 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:18.522790+0000", 2023-09-23T20:55:49.011 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.1777999999999999e-05 2023-09-23T20:55:49.011 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:49.011 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:49.011 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T20:55:49.012 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:18.522828+0000", 2023-09-23T20:55:49.012 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.8386999999999997e-05 2023-09-23T20:55:49.012 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:49.012 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:49.012 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T20:55:49.012 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:18.522908+0000", 2023-09-23T20:55:49.012 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.9939999999999997e-05 2023-09-23T20:55:49.012 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:55:49.012 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T20:55:49.012 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:55:49.012 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:49.013 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:49.013 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34797.0:165140 6.1 6:804ea438:::benchmark_data_smithi026_223868_object64014:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e155)", 2023-09-23T20:55:49.013 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T20:55:17.506959+0000", 2023-09-23T20:55:49.013 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 31.383429202999999, 2023-09-23T20:55:49.013 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0315099489999999, 2023-09-23T20:55:49.013 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T20:55:49.013 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:55:49.013 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T20:55:49.013 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34797", 2023-09-23T20:55:49.013 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T20:55:49.013 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 165140 2023-09-23T20:55:49.014 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:49.014 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T20:55:49.014 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:49.014 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T20:55:49.014 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:17.506959+0000", 2023-09-23T20:55:49.014 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:55:49.014 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:49.014 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:49.014 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T20:55:49.015 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:17.506959+0000", 2023-09-23T20:55:49.015 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:55:49.015 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:49.015 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:49.015 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T20:55:49.015 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:17.506958+0000", 2023-09-23T20:55:49.015 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.999999 2023-09-23T20:55:49.015 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:49.015 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:49.015 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T20:55:49.015 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:17.506964+0000", 2023-09-23T20:55:49.015 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.9399999999999999e-06 2023-09-23T20:55:49.016 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:49.016 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:49.016 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T20:55:49.016 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:17.506979+0000", 2023-09-23T20:55:49.016 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.5164000000000001e-05 2023-09-23T20:55:49.016 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:49.016 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:49.016 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T20:55:49.016 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:17.506987+0000", 2023-09-23T20:55:49.016 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.0369999999999995e-06 2023-09-23T20:55:49.016 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:49.016 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:49.017 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:55:49.017 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:17.507038+0000", 2023-09-23T20:55:49.017 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.0719e-05 2023-09-23T20:55:49.017 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:49.017 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:49.017 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T20:55:49.017 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:17.507047+0000", 2023-09-23T20:55:49.017 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.5880000000000004e-06 2023-09-23T20:55:49.017 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:49.017 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:49.017 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:55:49.017 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:18.523352+0000", 2023-09-23T20:55:49.018 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0163044240000001 2023-09-23T20:55:49.018 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:49.018 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:49.018 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T20:55:49.018 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:18.523449+0000", 2023-09-23T20:55:49.018 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.7126999999999994e-05 2023-09-23T20:55:49.018 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:49.018 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:49.018 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 2", 2023-09-23T20:55:49.019 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:18.523867+0000", 2023-09-23T20:55:49.019 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00041800399999999998 2023-09-23T20:55:49.019 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:49.019 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:49.019 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T20:55:49.019 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:18.528419+0000", 2023-09-23T20:55:49.019 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0045518729999999997 2023-09-23T20:55:49.019 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:49.019 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:49.019 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:55:49.019 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:18.538351+0000", 2023-09-23T20:55:49.019 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0099326940000000006 2023-09-23T20:55:49.020 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:49.020 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:49.020 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T20:55:49.020 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:18.538403+0000", 2023-09-23T20:55:49.020 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.1396000000000001e-05 2023-09-23T20:55:49.020 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:49.020 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:49.020 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T20:55:49.020 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:18.538469+0000", 2023-09-23T20:55:49.020 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.5953999999999999e-05 2023-09-23T20:55:49.020 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:55:49.020 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T20:55:49.021 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:55:49.021 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:49.021 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:49.021 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34797.0:165141 6.11 6:8bd9c32d:::benchmark_data_smithi026_223868_object64015:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e155)", 2023-09-23T20:55:49.021 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T20:55:17.507141+0000", 2023-09-23T20:55:49.021 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 31.383247253, 2023-09-23T20:55:49.021 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.031517094, 2023-09-23T20:55:49.021 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T20:55:49.021 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:55:49.021 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T20:55:49.021 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34797", 2023-09-23T20:55:49.021 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T20:55:49.022 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 165141 2023-09-23T20:55:49.022 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:49.022 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T20:55:49.022 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:49.022 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T20:55:49.022 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:17.507141+0000", 2023-09-23T20:55:49.022 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:55:49.022 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:49.022 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:49.022 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T20:55:49.022 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:17.507141+0000", 2023-09-23T20:55:49.023 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:55:49.023 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:49.023 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:49.023 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T20:55:49.023 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:17.507140+0000", 2023-09-23T20:55:49.023 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.999999 2023-09-23T20:55:49.023 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:49.023 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:49.023 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T20:55:49.023 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:17.507146+0000", 2023-09-23T20:55:49.023 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.0249999999999999e-06 2023-09-23T20:55:49.024 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:49.024 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:49.024 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T20:55:49.024 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:17.507161+0000", 2023-09-23T20:55:49.024 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.541e-05 2023-09-23T20:55:49.024 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:49.024 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:49.024 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T20:55:49.024 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:17.507169+0000", 2023-09-23T20:55:49.024 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.9670000000000001e-06 2023-09-23T20:55:49.024 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:49.025 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:49.025 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:55:49.025 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:17.507225+0000", 2023-09-23T20:55:49.025 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.6239000000000001e-05 2023-09-23T20:55:49.025 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:49.025 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:49.025 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T20:55:49.025 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:17.507234+0000", 2023-09-23T20:55:49.025 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.8829999999999995e-06 2023-09-23T20:55:49.025 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:49.025 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:49.025 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:55:49.026 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:18.526050+0000", 2023-09-23T20:55:49.026 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0188153660000001 2023-09-23T20:55:49.026 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:49.026 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:49.026 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T20:55:49.026 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:18.526116+0000", 2023-09-23T20:55:49.026 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.6645999999999999e-05 2023-09-23T20:55:49.026 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:49.026 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:49.026 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 2", 2023-09-23T20:55:49.026 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:18.526440+0000", 2023-09-23T20:55:49.027 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00032320800000000001 2023-09-23T20:55:49.027 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:49.027 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:49.027 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T20:55:49.027 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:18.532621+0000", 2023-09-23T20:55:49.027 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0061811289999999996 2023-09-23T20:55:49.027 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:49.027 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:49.027 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:55:49.027 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:18.538524+0000", 2023-09-23T20:55:49.027 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0059037450000000002 2023-09-23T20:55:49.027 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:49.028 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:49.028 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T20:55:49.028 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:18.538569+0000", 2023-09-23T20:55:49.028 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.4977e-05 2023-09-23T20:55:49.028 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:49.028 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:49.028 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T20:55:49.028 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:18.538658+0000", 2023-09-23T20:55:49.028 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.8459000000000001e-05 2023-09-23T20:55:49.028 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:55:49.029 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T20:55:49.029 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:55:49.029 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:49.029 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:49.029 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_repop(client.34797.0:166758 6.d e157/145 6:bfb8f551:::benchmark_data_smithi026_223868_object65632:head v 157'10500, mlcod=157'10498)", 2023-09-23T20:55:49.029 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T20:55:20.358174+0000", 2023-09-23T20:55:49.029 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 28.532213595000002, 2023-09-23T20:55:49.029 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.083851205999999998, 2023-09-23T20:55:49.029 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T20:55:49.029 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:55:49.030 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T20:55:49.030 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:49.030 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T20:55:49.030 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:20.358174+0000", 2023-09-23T20:55:49.030 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:55:49.030 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:49.030 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:49.030 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T20:55:49.030 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:20.358174+0000", 2023-09-23T20:55:49.030 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:55:49.030 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:49.031 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:49.031 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T20:55:49.031 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:20.358173+0000", 2023-09-23T20:55:49.031 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.999999 2023-09-23T20:55:49.031 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:49.031 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:49.031 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T20:55:49.031 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:20.358187+0000", 2023-09-23T20:55:49.031 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.3152999999999999e-05 2023-09-23T20:55:49.031 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:49.031 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:49.031 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T20:55:49.031 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:20.421853+0000", 2023-09-23T20:55:49.032 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.063666658000000001 2023-09-23T20:55:49.032 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:49.032 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:49.032 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T20:55:49.032 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:20.421863+0000", 2023-09-23T20:55:49.032 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.5610000000000004e-06 2023-09-23T20:55:49.032 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:49.032 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:49.032 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:55:49.032 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:20.422963+0000", 2023-09-23T20:55:49.032 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0010996840000000001 2023-09-23T20:55:49.032 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:49.033 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:49.033 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T20:55:49.033 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:20.422983+0000", 2023-09-23T20:55:49.033 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.0040000000000001e-05 2023-09-23T20:55:49.033 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:49.033 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:49.033 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T20:55:49.033 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:20.441988+0000", 2023-09-23T20:55:49.033 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.019005594000000001 2023-09-23T20:55:49.033 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:49.033 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:49.033 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T20:55:49.034 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:20.442026+0000", 2023-09-23T20:55:49.034 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.7413000000000002e-05 2023-09-23T20:55:49.034 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:55:49.034 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T20:55:49.034 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:55:49.034 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:49.034 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:49.034 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34797.0:176594 6.14 6:2d0a8cc5:::benchmark_data_smithi026_223868_object75468:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e162)", 2023-09-23T20:55:49.034 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T20:55:33.391368+0000", 2023-09-23T20:55:49.034 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 15.499019540999999, 2023-09-23T20:55:49.035 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.076892294999999999, 2023-09-23T20:55:49.035 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T20:55:49.035 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:55:49.035 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T20:55:49.035 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34797", 2023-09-23T20:55:49.035 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T20:55:49.035 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 176594 2023-09-23T20:55:49.035 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:49.035 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T20:55:49.035 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:49.035 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T20:55:49.035 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:33.391368+0000", 2023-09-23T20:55:49.035 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:55:49.036 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:49.036 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:49.036 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T20:55:49.036 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:33.391368+0000", 2023-09-23T20:55:49.036 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:55:49.036 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:49.036 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:49.036 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T20:55:49.036 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:33.391367+0000", 2023-09-23T20:55:49.036 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T20:55:49.036 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:49.036 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:49.037 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T20:55:49.037 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:33.391383+0000", 2023-09-23T20:55:49.037 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.5832e-05 2023-09-23T20:55:49.037 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:49.037 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:49.037 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T20:55:49.037 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:33.391398+0000", 2023-09-23T20:55:49.037 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.4893e-05 2023-09-23T20:55:49.037 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:49.037 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:49.037 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T20:55:49.037 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:33.391408+0000", 2023-09-23T20:55:49.038 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.8179999999999999e-06 2023-09-23T20:55:49.038 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:49.038 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:49.038 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:55:49.038 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:33.391461+0000", 2023-09-23T20:55:49.038 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.2846999999999997e-05 2023-09-23T20:55:49.038 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:49.038 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:49.038 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T20:55:49.038 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:33.391614+0000", 2023-09-23T20:55:49.038 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000152998 2023-09-23T20:55:49.039 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:49.039 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:49.039 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 3,5", 2023-09-23T20:55:49.039 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:33.392083+0000", 2023-09-23T20:55:49.039 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00046953400000000002 2023-09-23T20:55:49.039 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:49.039 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:49.039 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T20:55:49.039 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:33.396690+0000", 2023-09-23T20:55:49.039 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0046064909999999999 2023-09-23T20:55:49.039 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:49.039 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:49.040 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:55:49.040 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:33.401562+0000", 2023-09-23T20:55:49.040 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0048721900000000002 2023-09-23T20:55:49.040 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:49.040 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:49.040 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:55:49.040 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:33.468102+0000", 2023-09-23T20:55:49.040 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.066540537999999996 2023-09-23T20:55:49.040 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:49.040 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:49.040 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T20:55:49.040 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:33.468155+0000", 2023-09-23T20:55:49.041 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.3015e-05 2023-09-23T20:55:49.041 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:49.041 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:49.041 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T20:55:49.041 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:33.468261+0000", 2023-09-23T20:55:49.041 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000105444 2023-09-23T20:55:49.041 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:55:49.041 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T20:55:49.041 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:55:49.041 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:49.041 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:49.041 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34797.0:177915 6.11 6:8818350e:::benchmark_data_smithi026_223868_object76789:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e164)", 2023-09-23T20:55:49.042 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T20:55:34.933656+0000", 2023-09-23T20:55:49.042 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 13.956732141, 2023-09-23T20:55:49.042 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.0860430320000001, 2023-09-23T20:55:49.042 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T20:55:49.042 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:55:49.042 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T20:55:49.042 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34797", 2023-09-23T20:55:49.042 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T20:55:49.043 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 177915 2023-09-23T20:55:49.043 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:49.043 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T20:55:49.043 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:49.043 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T20:55:49.043 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:34.933656+0000", 2023-09-23T20:55:49.043 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:55:49.043 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:49.043 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:49.044 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T20:55:49.044 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:34.933656+0000", 2023-09-23T20:55:49.044 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:55:49.044 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:49.044 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:49.044 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T20:55:49.044 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:34.933654+0000", 2023-09-23T20:55:49.044 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999981 2023-09-23T20:55:49.044 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:49.044 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:49.044 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T20:55:49.045 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:34.933665+0000", 2023-09-23T20:55:49.045 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0963e-05 2023-09-23T20:55:49.045 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:49.045 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:49.045 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T20:55:49.045 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:34.933690+0000", 2023-09-23T20:55:49.045 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.5219999999999999e-05 2023-09-23T20:55:49.045 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:49.045 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:49.045 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T20:55:49.045 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:34.933701+0000", 2023-09-23T20:55:49.045 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0944000000000001e-05 2023-09-23T20:55:49.046 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:49.046 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:49.046 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:55:49.046 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:34.933803+0000", 2023-09-23T20:55:49.046 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000101654 2023-09-23T20:55:49.046 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:49.046 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:49.046 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op must wait for map", 2023-09-23T20:55:49.046 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:34.933811+0000", 2023-09-23T20:55:49.046 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.2190000000000004e-06 2023-09-23T20:55:49.046 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:49.047 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:49.047 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:55:49.047 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:34.946283+0000", 2023-09-23T20:55:49.047 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.012472298 2023-09-23T20:55:49.047 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:49.047 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:49.047 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T20:55:49.047 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:34.946294+0000", 2023-09-23T20:55:49.047 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0832999999999999e-05 2023-09-23T20:55:49.047 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:49.047 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:49.048 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:55:49.048 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:35.931861+0000", 2023-09-23T20:55:49.048 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.98556678499999995 2023-09-23T20:55:49.048 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:49.048 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:49.048 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T20:55:49.048 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:35.931870+0000", 2023-09-23T20:55:49.048 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.5799999999999992e-06 2023-09-23T20:55:49.048 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:49.048 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:49.048 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:55:49.049 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:37.013529+0000", 2023-09-23T20:55:49.049 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0816588920000001 2023-09-23T20:55:49.049 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:49.049 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:49.049 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T20:55:49.049 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:37.013686+0000", 2023-09-23T20:55:49.049 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000157825 2023-09-23T20:55:49.049 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:49.049 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:49.049 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 2,6", 2023-09-23T20:55:49.049 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:37.014148+0000", 2023-09-23T20:55:49.049 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00046177499999999999 2023-09-23T20:55:49.049 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:49.050 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:49.050 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T20:55:49.050 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:37.019388+0000", 2023-09-23T20:55:49.050 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0052396250000000004 2023-09-23T20:55:49.050 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:49.050 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:49.050 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:55:49.050 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:37.019453+0000", 2023-09-23T20:55:49.050 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.5685000000000006e-05 2023-09-23T20:55:49.050 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:49.050 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:49.050 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:55:49.051 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:37.019503+0000", 2023-09-23T20:55:49.051 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.9861999999999998e-05 2023-09-23T20:55:49.051 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:49.051 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:49.051 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T20:55:49.051 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:37.019559+0000", 2023-09-23T20:55:49.051 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.5605e-05 2023-09-23T20:55:49.051 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:49.051 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:49.051 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T20:55:49.051 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:37.019699+0000", 2023-09-23T20:55:49.051 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00013997099999999999 2023-09-23T20:55:49.052 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:55:49.052 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T20:55:49.052 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:55:49.052 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:49.052 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:49.052 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34797.0:177930 6.1 6:872a742b:::benchmark_data_smithi026_223868_object76804:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e164)", 2023-09-23T20:55:49.052 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T20:55:34.965696+0000", 2023-09-23T20:55:49.052 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 13.924691962000001, 2023-09-23T20:55:49.052 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.0561283210000001, 2023-09-23T20:55:49.052 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T20:55:49.052 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:55:49.052 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T20:55:49.053 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34797", 2023-09-23T20:55:49.053 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T20:55:49.053 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 177930 2023-09-23T20:55:49.053 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:49.053 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T20:55:49.053 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:49.053 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T20:55:49.053 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:34.965696+0000", 2023-09-23T20:55:49.053 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:55:49.053 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:49.053 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:49.053 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T20:55:49.054 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:34.965696+0000", 2023-09-23T20:55:49.054 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:55:49.054 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:49.054 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:49.054 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T20:55:49.054 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:34.965694+0000", 2023-09-23T20:55:49.054 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999981 2023-09-23T20:55:49.054 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:49.054 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:49.054 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T20:55:49.054 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:34.965702+0000", 2023-09-23T20:55:49.055 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.1000000000000004e-06 2023-09-23T20:55:49.055 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:49.055 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:49.055 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T20:55:49.055 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:34.965716+0000", 2023-09-23T20:55:49.055 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.3190000000000001e-05 2023-09-23T20:55:49.055 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:49.055 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:49.055 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T20:55:49.055 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:34.965724+0000", 2023-09-23T20:55:49.055 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.5699999999999993e-06 2023-09-23T20:55:49.055 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:49.056 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:49.056 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:55:49.056 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:34.965780+0000", 2023-09-23T20:55:49.056 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.5807999999999998e-05 2023-09-23T20:55:49.056 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:49.056 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:49.056 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T20:55:49.056 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:34.965791+0000", 2023-09-23T20:55:49.056 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0597e-05 2023-09-23T20:55:49.056 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:49.056 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:49.056 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:55:49.056 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:35.931819+0000", 2023-09-23T20:55:49.057 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.96602831899999997 2023-09-23T20:55:49.057 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:49.057 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:49.057 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T20:55:49.057 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:35.931833+0000", 2023-09-23T20:55:49.057 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.4409999999999999e-05 2023-09-23T20:55:49.057 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:49.057 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:49.057 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:55:49.057 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:37.016222+0000", 2023-09-23T20:55:49.057 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.084388186 2023-09-23T20:55:49.058 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:49.058 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:49.058 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T20:55:49.058 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:37.016369+0000", 2023-09-23T20:55:49.058 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00014728699999999999 2023-09-23T20:55:49.058 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:49.058 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:49.058 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 2,6", 2023-09-23T20:55:49.058 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:37.016878+0000", 2023-09-23T20:55:49.058 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000509068 2023-09-23T20:55:49.058 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:49.059 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:49.059 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T20:55:49.059 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:37.020205+0000", 2023-09-23T20:55:49.059 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0033272660000000002 2023-09-23T20:55:49.059 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:49.059 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:49.059 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:55:49.059 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:37.020577+0000", 2023-09-23T20:55:49.059 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00037149500000000001 2023-09-23T20:55:49.059 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:49.059 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:49.059 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:55:49.059 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:37.021628+0000", 2023-09-23T20:55:49.060 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0010512239999999999 2023-09-23T20:55:49.060 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:49.060 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:49.060 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T20:55:49.060 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:37.021685+0000", 2023-09-23T20:55:49.060 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.6715999999999997e-05 2023-09-23T20:55:49.060 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:55:49.060 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:55:49.060 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T20:55:49.060 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:37.021824+0000", 2023-09-23T20:55:49.060 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00013976799999999999 2023-09-23T20:55:49.060 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:55:49.061 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T20:55:49.061 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:55:49.061 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:55:49.061 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T20:55:49.061 INFO:teuthology.orchestra.run.smithi026.stdout:} 2023-09-23T20:55:49.062 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:55:49.062 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:55:49.163 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:55:49.194 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:55:49.613 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:55:49 smithi026 ceph-mon[175151]: failed to encode map e172 with expected crc 2023-09-23T20:55:49.613 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:55:49 smithi026 ceph-mon[175151]: osdmap e172: 8 total, 8 up, 8 in 2023-09-23T20:55:49.613 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:55:49 smithi026 ceph-mon[175151]: failed to encode map e172 with expected crc 2023-09-23T20:55:49.613 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:55:49 smithi026 ceph-mon[175151]: failed to encode map e172 with expected crc 2023-09-23T20:55:49.613 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:55:49 smithi026 ceph-mon[175151]: failed to encode map e172 with expected crc 2023-09-23T20:55:49.613 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:55:49 smithi026 ceph-mon[175151]: pgmap v412: 109 pgs: 1 active+recovering+undersized+remapped, 1 peering, 102 active+clean, 5 active+recovery_wait+undersized+degraded+remapped; 1.1 GiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 477 op/s; 2022/53355 objects degraded (3.790%); 6.5 MiB/s, 362 objects/s recovering 2023-09-23T20:55:49.613 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:55:49 smithi026 ceph-mon[179688]: failed to encode map e172 with expected crc 2023-09-23T20:55:49.613 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:55:49 smithi026 ceph-mon[179688]: osdmap e172: 8 total, 8 up, 8 in 2023-09-23T20:55:49.614 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:55:49 smithi026 ceph-mon[179688]: failed to encode map e172 with expected crc 2023-09-23T20:55:49.614 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:55:49 smithi026 ceph-mon[179688]: failed to encode map e172 with expected crc 2023-09-23T20:55:49.614 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:55:49 smithi026 ceph-mon[179688]: failed to encode map e172 with expected crc 2023-09-23T20:55:49.614 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:55:49 smithi026 ceph-mon[179688]: pgmap v412: 109 pgs: 1 active+recovering+undersized+remapped, 1 peering, 102 active+clean, 5 active+recovery_wait+undersized+degraded+remapped; 1.1 GiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 477 op/s; 2022/53355 objects degraded (3.790%); 6.5 MiB/s, 362 objects/s recovering 2023-09-23T20:55:49.614 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:55:49 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[164029]: ::ffff:172.21.15.70 - - [23/Sep/2023:20:55:49] "GET /metrics HTTP/1.1" 200 36346 "" "Prometheus/2.43.0" 2023-09-23T20:55:49.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:55:49 smithi070 ceph-mon[168282]: failed to encode map e172 with expected crc 2023-09-23T20:55:49.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:55:49 smithi070 ceph-mon[168282]: osdmap e172: 8 total, 8 up, 8 in 2023-09-23T20:55:49.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:55:49 smithi070 ceph-mon[168282]: failed to encode map e172 with expected crc 2023-09-23T20:55:49.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:55:49 smithi070 ceph-mon[168282]: failed to encode map e172 with expected crc 2023-09-23T20:55:49.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:55:49 smithi070 ceph-mon[168282]: failed to encode map e172 with expected crc 2023-09-23T20:55:49.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:55:49 smithi070 ceph-mon[168282]: pgmap v412: 109 pgs: 1 active+recovering+undersized+remapped, 1 peering, 102 active+clean, 5 active+recovery_wait+undersized+degraded+remapped; 1.1 GiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 477 op/s; 2022/53355 objects degraded (3.790%); 6.5 MiB/s, 362 objects/s recovering 2023-09-23T20:55:49.962 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:55:49.966 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:55:49.967 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:55:50.067 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:55:50.095 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:55:50.129 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:55:50.131 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:55:50.132 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:55:50.232 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:55:50.233 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:55:50 smithi026 ceph-mon[179688]: failed to encode map e173 with expected crc 2023-09-23T20:55:50.233 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:55:50 smithi026 ceph-mon[179688]: failed to encode map e173 with expected crc 2023-09-23T20:55:50.234 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:55:50 smithi026 ceph-mon[179688]: failed to encode map e173 with expected crc 2023-09-23T20:55:50.234 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:55:50 smithi026 ceph-mon[179688]: osdmap e173: 8 total, 8 up, 8 in 2023-09-23T20:55:50.234 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:55:50 smithi026 ceph-mon[179688]: failed to encode map e173 with expected crc 2023-09-23T20:55:50.234 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:55:50 smithi026 ceph-mon[179688]: failed to encode map e173 with expected crc 2023-09-23T20:55:50.234 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:55:50 smithi026 ceph-mon[179688]: Health check update: Degraded data redundancy: 2022/53355 objects degraded (3.790%), 5 pgs degraded (PG_DEGRADED) 2023-09-23T20:55:50.234 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:55:50 smithi026 ceph-mon[175151]: failed to encode map e173 with expected crc 2023-09-23T20:55:50.234 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:55:50 smithi026 ceph-mon[175151]: failed to encode map e173 with expected crc 2023-09-23T20:55:50.235 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:55:50 smithi026 ceph-mon[175151]: failed to encode map e173 with expected crc 2023-09-23T20:55:50.235 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:55:50 smithi026 ceph-mon[175151]: osdmap e173: 8 total, 8 up, 8 in 2023-09-23T20:55:50.235 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:55:50 smithi026 ceph-mon[175151]: failed to encode map e173 with expected crc 2023-09-23T20:55:50.235 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:55:50 smithi026 ceph-mon[175151]: failed to encode map e173 with expected crc 2023-09-23T20:55:50.235 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:55:50 smithi026 ceph-mon[175151]: Health check update: Degraded data redundancy: 2022/53355 objects degraded (3.790%), 5 pgs degraded (PG_DEGRADED) 2023-09-23T20:55:50.236 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_ops_in_flight 2023-09-23T20:55:50.287 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:55:50.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:55:50 smithi070 ceph-mon[168282]: failed to encode map e173 with expected crc 2023-09-23T20:55:50.371 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:55:50 smithi070 ceph-mon[168282]: failed to encode map e173 with expected crc 2023-09-23T20:55:50.371 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:55:50 smithi070 ceph-mon[168282]: failed to encode map e173 with expected crc 2023-09-23T20:55:50.371 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:55:50 smithi070 ceph-mon[168282]: osdmap e173: 8 total, 8 up, 8 in 2023-09-23T20:55:50.371 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:55:50 smithi070 ceph-mon[168282]: failed to encode map e173 with expected crc 2023-09-23T20:55:50.371 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:55:50 smithi070 ceph-mon[168282]: failed to encode map e173 with expected crc 2023-09-23T20:55:50.371 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:55:50 smithi070 ceph-mon[168282]: Health check update: Degraded data redundancy: 2022/53355 objects degraded (3.790%), 5 pgs degraded (PG_DEGRADED) 2023-09-23T20:55:50.411 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:55:50.420 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:55:50.421 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:55:50.521 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:55:50.605 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:55:50.701 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:55:50.708 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:55:50.709 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:55:50.809 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:55:50.815 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0: {} 2023-09-23T20:55:50.842 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:55:50.857 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1: {} 2023-09-23T20:55:50.884 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:55:50.887 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:55:50.887 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:55:50.898 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2: {} 2023-09-23T20:55:50.942 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3: {} 2023-09-23T20:55:50.977 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4: {} 2023-09-23T20:55:50.987 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:55:51.008 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T20:55:51.020 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:55:51.040 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6: {} 2023-09-23T20:55:51.043 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:55:51.059 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:55:51.062 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:55:51.062 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:55:51.077 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T20:55:51.085 INFO:teuthology.orchestra.run.smithi026.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T20:55:51.085 INFO:teuthology.orchestra.run.smithi026.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T20:55:51.085 INFO:teuthology.orchestra.run.smithi026.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T20:55:51.085 INFO:teuthology.orchestra.run.smithi026.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T20:55:51.085 INFO:teuthology.orchestra.run.smithi026.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T20:55:51.085 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T20:55:51.086 INFO:teuthology.orchestra.run.smithi026.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T20:55:51.086 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T20:55:51.163 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:55:51.191 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:55:51.225 INFO:teuthology.orchestra.run.smithi026.stderr:nodeep-scrub is set 2023-09-23T20:55:51.233 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:55:51.235 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:55:51.236 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:55:51.337 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:55:51.377 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:55:51.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:55:51 smithi026 ceph-mon[175151]: failed to encode map e174 with expected crc 2023-09-23T20:55:51.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:55:51 smithi026 ceph-mon[175151]: failed to encode map e174 with expected crc 2023-09-23T20:55:51.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:55:51 smithi026 ceph-mon[175151]: failed to encode map e174 with expected crc 2023-09-23T20:55:51.383 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:55:51 smithi026 ceph-mon[175151]: osdmap e174: 8 total, 8 up, 8 in 2023-09-23T20:55:51.383 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:55:51 smithi026 ceph-mon[175151]: failed to encode map e174 with expected crc 2023-09-23T20:55:51.383 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:55:51 smithi026 ceph-mon[175151]: failed to encode map e174 with expected crc 2023-09-23T20:55:51.383 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:55:51 smithi026 ceph-mon[175151]: pgmap v415: 109 pgs: 2 active+recovering+undersized+remapped, 103 active+clean, 4 active+recovery_wait+undersized+degraded+remapped; 1.0 GiB data, 4.6 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 551 op/s; 1605/51108 objects degraded (3.140%); 6.5 MiB/s, 463 objects/s recovering 2023-09-23T20:55:51.383 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:55:51 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/719237254' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:55:51.384 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:55:51 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/3124098045' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:55:51.384 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:55:51 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:55:51.384 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:55:51 smithi026 ceph-mon[179688]: failed to encode map e174 with expected crc 2023-09-23T20:55:51.384 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:55:51 smithi026 ceph-mon[179688]: failed to encode map e174 with expected crc 2023-09-23T20:55:51.385 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:55:51 smithi026 ceph-mon[179688]: failed to encode map e174 with expected crc 2023-09-23T20:55:51.385 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:55:51 smithi026 ceph-mon[179688]: osdmap e174: 8 total, 8 up, 8 in 2023-09-23T20:55:51.385 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:55:51 smithi026 ceph-mon[179688]: failed to encode map e174 with expected crc 2023-09-23T20:55:51.385 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:55:51 smithi026 ceph-mon[179688]: failed to encode map e174 with expected crc 2023-09-23T20:55:51.385 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:55:51 smithi026 ceph-mon[179688]: pgmap v415: 109 pgs: 2 active+recovering+undersized+remapped, 103 active+clean, 4 active+recovery_wait+undersized+degraded+remapped; 1.0 GiB data, 4.6 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 551 op/s; 1605/51108 objects degraded (3.140%); 6.5 MiB/s, 463 objects/s recovering 2023-09-23T20:55:51.386 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:55:51 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/719237254' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:55:51.386 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:55:51 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/3124098045' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:55:51.386 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:55:51 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:55:51.435 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:55:51.436 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:55:51.437 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:55:51.537 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:55:51.574 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:55:51 smithi070 ceph-mon[168282]: failed to encode map e174 with expected crc 2023-09-23T20:55:51.574 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:55:51 smithi070 ceph-mon[168282]: failed to encode map e174 with expected crc 2023-09-23T20:55:51.574 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:55:51 smithi070 ceph-mon[168282]: failed to encode map e174 with expected crc 2023-09-23T20:55:51.574 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:55:51 smithi070 ceph-mon[168282]: osdmap e174: 8 total, 8 up, 8 in 2023-09-23T20:55:51.574 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:55:51 smithi070 ceph-mon[168282]: failed to encode map e174 with expected crc 2023-09-23T20:55:51.574 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:55:51 smithi070 ceph-mon[168282]: failed to encode map e174 with expected crc 2023-09-23T20:55:51.574 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:55:51 smithi070 ceph-mon[168282]: pgmap v415: 109 pgs: 2 active+recovering+undersized+remapped, 103 active+clean, 4 active+recovery_wait+undersized+degraded+remapped; 1.0 GiB data, 4.6 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 551 op/s; 1605/51108 objects degraded (3.140%); 6.5 MiB/s, 463 objects/s recovering 2023-09-23T20:55:51.574 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:55:51 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/719237254' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:55:51.575 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:55:51 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/3124098045' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:55:51.575 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:55:51 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:55:51.577 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:55:51.616 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:55:51.619 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:55:51.619 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:55:51.720 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:55:51.763 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:55:51.810 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:55:51.814 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:55:51.815 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:55:51.915 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:55:52.016 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:55:52.321 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:55:52.328 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:55:52.329 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:55:52.430 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:55:52.474 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:55:52.526 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:55:52.528 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:55:52 smithi070 ceph-mon[168282]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:55:52.528 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:55:52 smithi070 ceph-mon[168282]: failed to encode map e175 with expected crc 2023-09-23T20:55:52.529 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:55:52 smithi070 ceph-mon[168282]: failed to encode map e175 with expected crc 2023-09-23T20:55:52.529 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:55:52 smithi070 ceph-mon[168282]: failed to encode map e175 with expected crc 2023-09-23T20:55:52.529 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:55:52 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T20:55:52.529 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:55:52 smithi070 ceph-mon[168282]: osdmap e175: 8 total, 8 up, 8 in 2023-09-23T20:55:52.529 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:55:52 smithi070 ceph-mon[168282]: failed to encode map e175 with expected crc 2023-09-23T20:55:52.529 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:55:52.530 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:55:52.631 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:55:52.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:55:52 smithi026 ceph-mon[179688]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:55:52.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:55:52 smithi026 ceph-mon[179688]: failed to encode map e175 with expected crc 2023-09-23T20:55:52.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:55:52 smithi026 ceph-mon[179688]: failed to encode map e175 with expected crc 2023-09-23T20:55:52.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:55:52 smithi026 ceph-mon[179688]: failed to encode map e175 with expected crc 2023-09-23T20:55:52.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:55:52 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T20:55:52.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:55:52 smithi026 ceph-mon[179688]: osdmap e175: 8 total, 8 up, 8 in 2023-09-23T20:55:52.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:55:52 smithi026 ceph-mon[179688]: failed to encode map e175 with expected crc 2023-09-23T20:55:52.633 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:55:52 smithi026 ceph-mon[175151]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:55:52.633 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:55:52 smithi026 ceph-mon[175151]: failed to encode map e175 with expected crc 2023-09-23T20:55:52.633 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:55:52 smithi026 ceph-mon[175151]: failed to encode map e175 with expected crc 2023-09-23T20:55:52.633 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:55:52 smithi026 ceph-mon[175151]: failed to encode map e175 with expected crc 2023-09-23T20:55:52.633 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:55:52 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T20:55:52.633 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:55:52 smithi026 ceph-mon[175151]: osdmap e175: 8 total, 8 up, 8 in 2023-09-23T20:55:52.633 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:55:52 smithi026 ceph-mon[175151]: failed to encode map e175 with expected crc 2023-09-23T20:55:52.690 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:55:52.736 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 6, 7, 4, 5] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 7, 6] 2023-09-23T20:55:52.736 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T20:55:52.737 INFO:tasks.thrashosds.thrasher:fixing pg num pool unique_pool_0 2023-09-23T20:55:52.737 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T20:55:52.747 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:55:52.754 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:55:52.755 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:55:52.855 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:55:52.948 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:55:52.993 INFO:teuthology.orchestra.run.smithi026.stdout:ERROR: (22) Invalid argument 2023-09-23T20:55:52.994 INFO:teuthology.orchestra.run.smithi026.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:55:53.019 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:55:53.019 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:55:53.019 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:55:53.120 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:55:53.160 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:55:53.210 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:55:53.218 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:55:53.219 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:55:53.239 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:55:53 smithi026 ceph-mon[179688]: pgmap v417: 109 pgs: 1 active+recovering+undersized+remapped, 105 active+clean, 3 active+recovery_wait+undersized+degraded+remapped; 934 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 692 op/s; 1236/44859 objects degraded (2.755%); 0 B/s, 225 objects/s recovering 2023-09-23T20:55:53.239 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:55:53 smithi026 ceph-mon[175151]: pgmap v417: 109 pgs: 1 active+recovering+undersized+remapped, 105 active+clean, 3 active+recovery_wait+undersized+degraded+remapped; 934 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 692 op/s; 1236/44859 objects degraded (2.755%); 0 B/s, 225 objects/s recovering 2023-09-23T20:55:53.320 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:55:53.337 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_blocked_ops 2023-09-23T20:55:53.359 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:55:53.407 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:55:53.409 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:55:53 smithi070 ceph-mon[168282]: pgmap v417: 109 pgs: 1 active+recovering+undersized+remapped, 105 active+clean, 3 active+recovery_wait+undersized+degraded+remapped; 934 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 692 op/s; 1236/44859 objects degraded (2.755%); 0 B/s, 225 objects/s recovering 2023-09-23T20:55:53.410 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:55:53.410 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:55:53.512 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:55:53.593 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:55:53.652 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T20:55:53.667 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:55:53.673 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:55:53.674 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:55:53.747 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:55:53.775 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:55:53.811 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:55:53.852 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:55:53.855 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T20:55:53.857 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:55:53.857 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:55:53.959 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:55:54.006 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:55:54.240 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:55:54 smithi026 ceph-mon[175151]: failed to encode map e176 with expected crc 2023-09-23T20:55:54.241 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:55:54 smithi026 ceph-mon[175151]: failed to encode map e176 with expected crc 2023-09-23T20:55:54.241 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:55:54 smithi026 ceph-mon[175151]: osdmap e176: 8 total, 8 up, 8 in 2023-09-23T20:55:54.241 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:55:54 smithi026 ceph-mon[175151]: failed to encode map e176 with expected crc 2023-09-23T20:55:54.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:55:54 smithi070 ceph-mon[168282]: failed to encode map e176 with expected crc 2023-09-23T20:55:54.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:55:54 smithi070 ceph-mon[168282]: failed to encode map e176 with expected crc 2023-09-23T20:55:54.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:55:54 smithi070 ceph-mon[168282]: osdmap e176: 8 total, 8 up, 8 in 2023-09-23T20:55:54.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:55:54 smithi070 ceph-mon[168282]: failed to encode map e176 with expected crc 2023-09-23T20:55:54.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:55:54 smithi026 ceph-mon[179688]: failed to encode map e176 with expected crc 2023-09-23T20:55:54.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:55:54 smithi026 ceph-mon[179688]: failed to encode map e176 with expected crc 2023-09-23T20:55:54.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:55:54 smithi026 ceph-mon[179688]: osdmap e176: 8 total, 8 up, 8 in 2023-09-23T20:55:54.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:55:54 smithi026 ceph-mon[179688]: failed to encode map e176 with expected crc 2023-09-23T20:55:54.905 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:55:54.908 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:55:54.909 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:55:55.009 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:55:55.082 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:55:55.255 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:55:55.260 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:55:55.260 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:55:55.361 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:55:55.418 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:55:55.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:55:55 smithi070 ceph-mon[168282]: failed to encode map e177 with expected crc 2023-09-23T20:55:55.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:55:55 smithi070 ceph-mon[168282]: failed to encode map e177 with expected crc 2023-09-23T20:55:55.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:55:55 smithi070 ceph-mon[168282]: osdmap e177: 8 total, 8 up, 8 in 2023-09-23T20:55:55.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:55:55 smithi070 ceph-mon[168282]: failed to encode map e177 with expected crc 2023-09-23T20:55:55.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:55:55 smithi070 ceph-mon[168282]: failed to encode map e177 with expected crc 2023-09-23T20:55:55.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:55:55 smithi070 ceph-mon[168282]: pgmap v420: 109 pgs: 1 peering, 106 active+clean, 2 active+recovery_wait+undersized+degraded+remapped; 870 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 756 op/s; 806/41826 objects degraded (1.927%); 0 B/s, 234 objects/s recovering 2023-09-23T20:55:55.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:55:55 smithi070 ceph-mon[168282]: Health check update: Degraded data redundancy: 1236/44859 objects degraded (2.755%), 3 pgs degraded (PG_DEGRADED) 2023-09-23T20:55:55.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:55:55 smithi070 ceph-mon[168282]: failed to encode map e178 with expected crc 2023-09-23T20:55:55.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:55:55 smithi070 ceph-mon[168282]: failed to encode map e178 with expected crc 2023-09-23T20:55:55.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:55:55 smithi070 ceph-mon[168282]: osdmap e178: 8 total, 8 up, 8 in 2023-09-23T20:55:55.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:55:55 smithi070 ceph-mon[168282]: failed to encode map e178 with expected crc 2023-09-23T20:55:55.621 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:55:55 smithi070 ceph-mon[168282]: failed to encode map e178 with expected crc 2023-09-23T20:55:55.621 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:55:55 smithi070 ceph-mon[168282]: failed to encode map e178 with expected crc 2023-09-23T20:55:55.631 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:55:55 smithi026 ceph-mon[175151]: failed to encode map e177 with expected crc 2023-09-23T20:55:55.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:55:55 smithi026 ceph-mon[175151]: failed to encode map e177 with expected crc 2023-09-23T20:55:55.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:55:55 smithi026 ceph-mon[175151]: osdmap e177: 8 total, 8 up, 8 in 2023-09-23T20:55:55.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:55:55 smithi026 ceph-mon[175151]: failed to encode map e177 with expected crc 2023-09-23T20:55:55.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:55:55 smithi026 ceph-mon[175151]: failed to encode map e177 with expected crc 2023-09-23T20:55:55.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:55:55 smithi026 ceph-mon[175151]: pgmap v420: 109 pgs: 1 peering, 106 active+clean, 2 active+recovery_wait+undersized+degraded+remapped; 870 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 756 op/s; 806/41826 objects degraded (1.927%); 0 B/s, 234 objects/s recovering 2023-09-23T20:55:55.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:55:55 smithi026 ceph-mon[175151]: Health check update: Degraded data redundancy: 1236/44859 objects degraded (2.755%), 3 pgs degraded (PG_DEGRADED) 2023-09-23T20:55:55.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:55:55 smithi026 ceph-mon[175151]: failed to encode map e178 with expected crc 2023-09-23T20:55:55.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:55:55 smithi026 ceph-mon[175151]: failed to encode map e178 with expected crc 2023-09-23T20:55:55.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:55:55 smithi026 ceph-mon[175151]: osdmap e178: 8 total, 8 up, 8 in 2023-09-23T20:55:55.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:55:55 smithi026 ceph-mon[175151]: failed to encode map e178 with expected crc 2023-09-23T20:55:55.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:55:55 smithi026 ceph-mon[175151]: failed to encode map e178 with expected crc 2023-09-23T20:55:55.633 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:55:55 smithi026 ceph-mon[175151]: failed to encode map e178 with expected crc 2023-09-23T20:55:55.633 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:55:55 smithi026 ceph-mon[179688]: failed to encode map e177 with expected crc 2023-09-23T20:55:55.633 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:55:55 smithi026 ceph-mon[179688]: failed to encode map e177 with expected crc 2023-09-23T20:55:55.633 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:55:55 smithi026 ceph-mon[179688]: osdmap e177: 8 total, 8 up, 8 in 2023-09-23T20:55:55.633 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:55:55 smithi026 ceph-mon[179688]: failed to encode map e177 with expected crc 2023-09-23T20:55:55.633 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:55:55 smithi026 ceph-mon[179688]: failed to encode map e177 with expected crc 2023-09-23T20:55:55.633 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:55:55 smithi026 ceph-mon[179688]: pgmap v420: 109 pgs: 1 peering, 106 active+clean, 2 active+recovery_wait+undersized+degraded+remapped; 870 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 756 op/s; 806/41826 objects degraded (1.927%); 0 B/s, 234 objects/s recovering 2023-09-23T20:55:55.633 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:55:55 smithi026 ceph-mon[179688]: Health check update: Degraded data redundancy: 1236/44859 objects degraded (2.755%), 3 pgs degraded (PG_DEGRADED) 2023-09-23T20:55:55.633 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:55:55 smithi026 ceph-mon[179688]: failed to encode map e178 with expected crc 2023-09-23T20:55:55.633 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:55:55 smithi026 ceph-mon[179688]: failed to encode map e178 with expected crc 2023-09-23T20:55:55.633 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:55:55 smithi026 ceph-mon[179688]: osdmap e178: 8 total, 8 up, 8 in 2023-09-23T20:55:55.634 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:55:55 smithi026 ceph-mon[179688]: failed to encode map e178 with expected crc 2023-09-23T20:55:55.634 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:55:55 smithi026 ceph-mon[179688]: failed to encode map e178 with expected crc 2023-09-23T20:55:55.634 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:55:55 smithi026 ceph-mon[179688]: failed to encode map e178 with expected crc 2023-09-23T20:55:55.823 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:55:55.825 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:55:55.825 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:55:55.926 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:55:55.955 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:55:55.999 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:55:56.004 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:55:56.005 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:55:56.105 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:55:56.151 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:55:56.207 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:55:56.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:55:56 smithi026 ceph-mon[179688]: failed to encode map e178 with expected crc 2023-09-23T20:55:56.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:55:56 smithi026 ceph-mon[179688]: failed to encode map e179 with expected crc 2023-09-23T20:55:56.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:55:56 smithi026 ceph-mon[179688]: osdmap e179: 8 total, 8 up, 8 in 2023-09-23T20:55:56.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:55:56 smithi026 ceph-mon[179688]: failed to encode map e179 with expected crc 2023-09-23T20:55:56.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:55:56 smithi026 ceph-mon[179688]: failed to encode map e179 with expected crc 2023-09-23T20:55:56.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:55:56 smithi026 ceph-mon[175151]: failed to encode map e178 with expected crc 2023-09-23T20:55:56.383 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:55:56 smithi026 ceph-mon[175151]: failed to encode map e179 with expected crc 2023-09-23T20:55:56.383 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:55:56 smithi026 ceph-mon[175151]: osdmap e179: 8 total, 8 up, 8 in 2023-09-23T20:55:56.383 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:55:56 smithi026 ceph-mon[175151]: failed to encode map e179 with expected crc 2023-09-23T20:55:56.383 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:55:56 smithi026 ceph-mon[175151]: failed to encode map e179 with expected crc 2023-09-23T20:55:56.405 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:55:56.408 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:55:56.409 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:55:56.509 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:55:56.573 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:55:56.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:55:56 smithi070 ceph-mon[168282]: failed to encode map e178 with expected crc 2023-09-23T20:55:56.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:55:56 smithi070 ceph-mon[168282]: failed to encode map e179 with expected crc 2023-09-23T20:55:56.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:55:56 smithi070 ceph-mon[168282]: osdmap e179: 8 total, 8 up, 8 in 2023-09-23T20:55:56.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:55:56 smithi070 ceph-mon[168282]: failed to encode map e179 with expected crc 2023-09-23T20:55:56.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:55:56 smithi070 ceph-mon[168282]: failed to encode map e179 with expected crc 2023-09-23T20:55:56.686 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:55:56.731 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:55:56.740 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:55:56.741 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:55:56.841 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:55:56.874 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:55:56.891 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:55:56.917 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:55:56.921 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:55:56.921 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:55:57.022 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:55:57.124 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:55:57.306 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:55:57.313 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:55:57.313 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:55:57.414 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:55:57.448 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:55:57.501 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:55:57.508 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:55:57.508 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:55:57.531 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:55:57 smithi026 ceph-mon[179688]: failed to encode map e179 with expected crc 2023-09-23T20:55:57.531 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:55:57 smithi026 ceph-mon[179688]: pgmap v423: 109 pgs: 1 peering, 106 active+clean, 2 active+recovery_wait+undersized+degraded+remapped; 845 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 352 op/s; 806/40620 objects degraded (1.984%); 0 B/s, 110 objects/s recovering 2023-09-23T20:55:57.531 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:55:57 smithi026 ceph-mon[175151]: failed to encode map e179 with expected crc 2023-09-23T20:55:57.532 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:55:57 smithi026 ceph-mon[175151]: pgmap v423: 109 pgs: 1 peering, 106 active+clean, 2 active+recovery_wait+undersized+degraded+remapped; 845 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 352 op/s; 806/40620 objects degraded (1.984%); 0 B/s, 110 objects/s recovering 2023-09-23T20:55:57.609 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:55:57.683 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:55:57 smithi070 ceph-mon[168282]: failed to encode map e179 with expected crc 2023-09-23T20:55:57.683 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:55:57 smithi070 ceph-mon[168282]: pgmap v423: 109 pgs: 1 peering, 106 active+clean, 2 active+recovery_wait+undersized+degraded+remapped; 845 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 352 op/s; 806/40620 objects degraded (1.984%); 0 B/s, 110 objects/s recovering 2023-09-23T20:55:57.782 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:55:58.114 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:55:58.123 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:55:58.123 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:55:58.225 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:55:58.346 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:55:58.593 INFO:teuthology.orchestra.run.smithi026.stdout: 2023-09-23T20:55:58.624 INFO:teuthology.orchestra.run.smithi026.stderr:dumped all 2023-09-23T20:55:58.635 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:55:58.645 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:55:58.645 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:55:58.746 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:55:58.774 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:55:58 smithi070 ceph-mon[168282]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 806/40620 objects degraded (1.984%), 2 pgs degraded) 2023-09-23T20:55:58.776 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:55:58.814 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:55:58.819 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:55:58.819 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:55:58.832 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:55:58 smithi026 ceph-mon[175151]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 806/40620 objects degraded (1.984%), 2 pgs degraded) 2023-09-23T20:55:58.832 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:55:58 smithi026 ceph-mon[179688]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 806/40620 objects degraded (1.984%), 2 pgs degraded) 2023-09-23T20:55:58.920 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:55:59.001 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:55:59.191 INFO:teuthology.orchestra.run.smithi026.stdout:{"pg_ready":true,"pg_map":{"version":424,"stamp":"2023-09-23T20:55:58.303792+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":762025912,"num_objects":11644,"num_object_clones":0,"num_object_copies":34932,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":11644,"num_whiteouts":0,"num_read":501,"num_read_kb":925,"num_write":174910,"num_write_kb":5863118,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":28659,"num_bytes_recovered":1328946960,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":225880,"ondisk_log_size":225880,"up":327,"acting":327,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":337,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":3755028,"kb_used_data":2883028,"kb_used_omap":136,"kb_used_meta":871799,"kb_avail":746206188,"statfs":{"total":767960285184,"available":764115136512,"internally_reserved":0,"allocated":2952220672,"data_stored":2946186853,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":139741,"internal_metadata":892722723},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[5],"upper_bound":2},"perf_stat":{"commit_latency_ms":4,"apply_latency_ms":4,"commit_latency_ns":4000000,"apply_latency_ns":4000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-216858624,"num_objects":-3309,"num_object_clones":0,"num_object_copies":-9927,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":-1236,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":-3309,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3309,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1247,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"5.068726"},"pg_stats":[{"pgid":"3.1a","version":"68'1","reported_seq":56,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-09-23T20:55:54.925875+0000","last_change":"2023-09-23T20:54:13.946695+0000","last_active":"2023-09-23T20:55:54.925875+0000","last_peered":"2023-09-23T20:55:54.925875+0000","last_clean":"2023-09-23T20:55:54.925875+0000","last_became_active":"2023-09-23T20:54:13.945399+0000","last_became_peered":"2023-09-23T20:54:13.945399+0000","last_unstale":"2023-09-23T20:55:54.925875+0000","last_undegraded":"2023-09-23T20:55:54.925875+0000","last_fullsized":"2023-09-23T20:55:54.925875+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:32:52.365464+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":166,"reported_epoch":175,"state":"active+clean","last_fresh":"2023-09-23T20:55:51.231741+0000","last_change":"2023-09-23T20:54:13.942019+0000","last_active":"2023-09-23T20:55:51.231741+0000","last_peered":"2023-09-23T20:55:51.231741+0000","last_clean":"2023-09-23T20:55:51.231741+0000","last_became_active":"2023-09-23T20:54:13.941647+0000","last_became_peered":"2023-09-23T20:54:13.941647+0000","last_unstale":"2023-09-23T20:55:51.231741+0000","last_undegraded":"2023-09-23T20:55:51.231741+0000","last_fullsized":"2023-09-23T20:55:51.231741+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:30:54.400944+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":144,"reported_epoch":175,"state":"active+clean","last_fresh":"2023-09-23T20:55:51.235832+0000","last_change":"2023-09-23T20:54:13.941394+0000","last_active":"2023-09-23T20:55:51.235832+0000","last_peered":"2023-09-23T20:55:51.235832+0000","last_clean":"2023-09-23T20:55:51.235832+0000","last_became_active":"2023-09-23T20:54:13.941042+0000","last_became_peered":"2023-09-23T20:54:13.941042+0000","last_unstale":"2023-09-23T20:55:51.235832+0000","last_undegraded":"2023-09-23T20:55:51.235832+0000","last_fullsized":"2023-09-23T20:55:51.235832+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:19:41.401184+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":143,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-09-23T20:55:54.364037+0000","last_change":"2023-09-23T20:54:44.037156+0000","last_active":"2023-09-23T20:55:54.364037+0000","last_peered":"2023-09-23T20:55:54.364037+0000","last_clean":"2023-09-23T20:55:54.364037+0000","last_became_active":"2023-09-23T20:54:44.036840+0000","last_became_peered":"2023-09-23T20:54:44.036840+0000","last_unstale":"2023-09-23T20:55:54.364037+0000","last_undegraded":"2023-09-23T20:55:54.364037+0000","last_fullsized":"2023-09-23T20:55:54.364037+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:39:01.707474+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":157,"reported_epoch":175,"state":"active+clean","last_fresh":"2023-09-23T20:55:51.229718+0000","last_change":"2023-09-23T20:51:10.833613+0000","last_active":"2023-09-23T20:55:51.229718+0000","last_peered":"2023-09-23T20:55:51.229718+0000","last_clean":"2023-09-23T20:55:51.229718+0000","last_became_active":"2023-09-23T20:51:10.833208+0000","last_became_peered":"2023-09-23T20:51:10.833208+0000","last_unstale":"2023-09-23T20:55:51.229718+0000","last_undegraded":"2023-09-23T20:55:51.229718+0000","last_fullsized":"2023-09-23T20:55:51.229718+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:51:28.246014+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":141,"reported_epoch":175,"state":"active+clean","last_fresh":"2023-09-23T20:55:51.230481+0000","last_change":"2023-09-23T20:55:35.958384+0000","last_active":"2023-09-23T20:55:51.230481+0000","last_peered":"2023-09-23T20:55:51.230481+0000","last_clean":"2023-09-23T20:55:51.230481+0000","last_became_active":"2023-09-23T20:55:35.954563+0000","last_became_peered":"2023-09-23T20:55:35.954563+0000","last_unstale":"2023-09-23T20:55:51.230481+0000","last_undegraded":"2023-09-23T20:55:51.230481+0000","last_fullsized":"2023-09-23T20:55:51.230481+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:37:48.332259+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.12","version":"179'9149","reported_seq":12276,"reported_epoch":179,"state":"active+clean","last_fresh":"2023-09-23T20:55:56.947162+0000","last_change":"2023-09-23T20:55:41.633557+0000","last_active":"2023-09-23T20:55:56.947162+0000","last_peered":"2023-09-23T20:55:56.947162+0000","last_clean":"2023-09-23T20:55:56.947162+0000","last_became_active":"2023-09-23T20:55:08.352401+0000","last_became_peered":"2023-09-23T20:55:08.352401+0000","last_unstale":"2023-09-23T20:55:56.947162+0000","last_undegraded":"2023-09-23T20:55:56.947162+0000","last_fullsized":"2023-09-23T20:55:56.947162+0000","mapping_epoch":150,"log_start":"0'0","ondisk_log_start":"0'0","created":128,"last_epoch_clean":151,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:52:13.843048+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:52:13.843048+0000","last_clean_scrub_stamp":"2023-09-23T20:52:13.843048+0000","objects_scrubbed":0,"log_size":9149,"log_dups_size":0,"ondisk_log_size":9149,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":23855104,"num_objects":364,"num_object_clones":0,"num_object_copies":1092,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":364,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5827,"num_write_kb":198112,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1496,"num_bytes_recovered":71172096,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":151,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-09-23T20:55:54.364280+0000","last_change":"2023-09-23T20:54:31.885514+0000","last_active":"2023-09-23T20:55:54.364280+0000","last_peered":"2023-09-23T20:55:54.364280+0000","last_clean":"2023-09-23T20:55:54.364280+0000","last_became_active":"2023-09-23T20:54:31.884986+0000","last_became_peered":"2023-09-23T20:54:31.884986+0000","last_unstale":"2023-09-23T20:55:54.364280+0000","last_undegraded":"2023-09-23T20:55:54.364280+0000","last_fullsized":"2023-09-23T20:55:54.364280+0000","mapping_epoch":136,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:26:08.100220+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":155,"reported_epoch":175,"state":"active+clean","last_fresh":"2023-09-23T20:55:51.229817+0000","last_change":"2023-09-23T20:55:35.951889+0000","last_active":"2023-09-23T20:55:51.229817+0000","last_peered":"2023-09-23T20:55:51.229817+0000","last_clean":"2023-09-23T20:55:51.229817+0000","last_became_active":"2023-09-23T20:55:35.951563+0000","last_became_peered":"2023-09-23T20:55:35.951563+0000","last_unstale":"2023-09-23T20:55:51.229817+0000","last_undegraded":"2023-09-23T20:55:51.229817+0000","last_fullsized":"2023-09-23T20:55:51.229817+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:41:53.098781+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.13","version":"179'9437","reported_seq":10413,"reported_epoch":179,"state":"active+clean","last_fresh":"2023-09-23T20:55:57.150423+0000","last_change":"2023-09-23T20:55:55.989037+0000","last_active":"2023-09-23T20:55:57.150423+0000","last_peered":"2023-09-23T20:55:57.150423+0000","last_clean":"2023-09-23T20:55:57.150423+0000","last_became_active":"2023-09-23T20:55:55.933959+0000","last_became_peered":"2023-09-23T20:55:55.933959+0000","last_unstale":"2023-09-23T20:55:57.150423+0000","last_undegraded":"2023-09-23T20:55:57.150423+0000","last_fullsized":"2023-09-23T20:55:57.150423+0000","mapping_epoch":178,"log_start":"107'1195","ondisk_log_start":"107'1195","created":128,"last_epoch_clean":179,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:52:13.843048+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:52:13.843048+0000","last_clean_scrub_stamp":"2023-09-23T20:52:13.843048+0000","objects_scrubbed":0,"log_size":8242,"log_dups_size":0,"ondisk_log_size":8242,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":16023552,"num_objects":245,"num_object_clones":0,"num_object_copies":735,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":245,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6068,"num_write_kb":201984,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":424,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,2],"acting":[7,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":52,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-09-23T20:55:54.975660+0000","last_change":"2023-09-23T20:54:31.890678+0000","last_active":"2023-09-23T20:55:54.975660+0000","last_peered":"2023-09-23T20:55:54.975660+0000","last_clean":"2023-09-23T20:55:54.975660+0000","last_became_active":"2023-09-23T20:54:31.890469+0000","last_became_peered":"2023-09-23T20:54:31.890469+0000","last_unstale":"2023-09-23T20:55:54.975660+0000","last_undegraded":"2023-09-23T20:55:54.975660+0000","last_fullsized":"2023-09-23T20:55:54.975660+0000","mapping_epoch":136,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:55:08.790146+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":116,"reported_epoch":175,"state":"active+clean","last_fresh":"2023-09-23T20:55:51.229819+0000","last_change":"2023-09-23T20:55:35.952661+0000","last_active":"2023-09-23T20:55:51.229819+0000","last_peered":"2023-09-23T20:55:51.229819+0000","last_clean":"2023-09-23T20:55:51.229819+0000","last_became_active":"2023-09-23T20:55:35.952349+0000","last_became_peered":"2023-09-23T20:55:35.952349+0000","last_unstale":"2023-09-23T20:55:51.229819+0000","last_undegraded":"2023-09-23T20:55:51.229819+0000","last_fullsized":"2023-09-23T20:55:51.229819+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:00:03.110126+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.10","version":"178'9456","reported_seq":9558,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-09-23T20:55:55.541024+0000","last_change":"2023-09-23T20:55:36.935871+0000","last_active":"2023-09-23T20:55:55.541024+0000","last_peered":"2023-09-23T20:55:55.541024+0000","last_clean":"2023-09-23T20:55:55.541024+0000","last_became_active":"2023-09-23T20:54:03.824384+0000","last_became_peered":"2023-09-23T20:54:03.824384+0000","last_unstale":"2023-09-23T20:55:55.541024+0000","last_undegraded":"2023-09-23T20:55:55.541024+0000","last_fullsized":"2023-09-23T20:55:55.541024+0000","mapping_epoch":128,"log_start":"0'0","ondisk_log_start":"0'0","created":128,"last_epoch_clean":129,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:52:13.843048+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:52:13.843048+0000","last_clean_scrub_stamp":"2023-09-23T20:52:13.843048+0000","objects_scrubbed":0,"log_size":9456,"log_dups_size":0,"ondisk_log_size":9456,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":20873216,"num_objects":319,"num_object_clones":0,"num_object_copies":957,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":319,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6042,"num_write_kb":203520,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":174,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-09-23T20:55:54.927340+0000","last_change":"2023-09-23T20:54:13.949180+0000","last_active":"2023-09-23T20:55:54.927340+0000","last_peered":"2023-09-23T20:55:54.927340+0000","last_clean":"2023-09-23T20:55:54.927340+0000","last_became_active":"2023-09-23T20:54:13.936669+0000","last_became_peered":"2023-09-23T20:54:13.936669+0000","last_unstale":"2023-09-23T20:55:54.927340+0000","last_undegraded":"2023-09-23T20:55:54.927340+0000","last_fullsized":"2023-09-23T20:55:54.927340+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:59:22.400515+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":141,"reported_epoch":175,"state":"active+clean","last_fresh":"2023-09-23T20:55:51.234304+0000","last_change":"2023-09-23T20:55:35.951763+0000","last_active":"2023-09-23T20:55:51.234304+0000","last_peered":"2023-09-23T20:55:51.234304+0000","last_clean":"2023-09-23T20:55:51.234304+0000","last_became_active":"2023-09-23T20:55:35.951499+0000","last_became_peered":"2023-09-23T20:55:35.951499+0000","last_unstale":"2023-09-23T20:55:51.234304+0000","last_undegraded":"2023-09-23T20:55:51.234304+0000","last_fullsized":"2023-09-23T20:55:51.234304+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:31:30.704792+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.11","version":"175'9396","reported_seq":10369,"reported_epoch":176,"state":"peering","last_fresh":"2023-09-23T20:55:53.252414+0000","last_change":"2023-09-23T20:55:53.252578+0000","last_active":"2023-09-23T20:55:53.252414+0000","last_peered":"2023-09-23T20:55:53.221675+0000","last_clean":"2023-09-23T20:55:17.456733+0000","last_became_active":"2023-09-23T20:55:36.992610+0000","last_became_peered":"2023-09-23T20:55:36.992610+0000","last_unstale":"2023-09-23T20:55:53.252414+0000","last_undegraded":"2023-09-23T20:55:53.252414+0000","last_fullsized":"2023-09-23T20:55:53.252414+0000","mapping_epoch":176,"log_start":"107'700","ondisk_log_start":"107'700","created":128,"last_epoch_clean":129,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:52:13.843048+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:52:13.843048+0000","last_clean_scrub_stamp":"2023-09-23T20:52:13.843048+0000","objects_scrubbed":0,"log_size":8696,"log_dups_size":0,"ondisk_log_size":8696,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":27099136,"num_objects":414,"num_object_clones":0,"num_object_copies":1242,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":414,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5982,"num_write_kb":204640,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":431,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[2,6],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"79'22","reported_seq":83,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-09-23T20:55:54.925763+0000","last_change":"2023-09-23T20:55:35.949418+0000","last_active":"2023-09-23T20:55:54.925763+0000","last_peered":"2023-09-23T20:55:54.925763+0000","last_clean":"2023-09-23T20:55:54.925763+0000","last_became_active":"2023-09-23T20:55:35.948566+0000","last_became_peered":"2023-09-23T20:55:35.948566+0000","last_unstale":"2023-09-23T20:55:54.925763+0000","last_undegraded":"2023-09-23T20:55:54.925763+0000","last_fullsized":"2023-09-23T20:55:54.925763+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:59:37.433128+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4734,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":73,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-09-23T20:55:54.925393+0000","last_change":"2023-09-23T20:55:35.949178+0000","last_active":"2023-09-23T20:55:54.925393+0000","last_peered":"2023-09-23T20:55:54.925393+0000","last_clean":"2023-09-23T20:55:54.925393+0000","last_became_active":"2023-09-23T20:55:35.948795+0000","last_became_peered":"2023-09-23T20:55:35.948795+0000","last_unstale":"2023-09-23T20:55:54.925393+0000","last_undegraded":"2023-09-23T20:55:54.925393+0000","last_fullsized":"2023-09-23T20:55:54.925393+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:13:08.996932+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.16","version":"176'9214","reported_seq":11368,"reported_epoch":176,"state":"active+clean","last_fresh":"2023-09-23T20:55:53.977152+0000","last_change":"2023-09-23T20:54:44.045342+0000","last_active":"2023-09-23T20:55:53.977152+0000","last_peered":"2023-09-23T20:55:53.977152+0000","last_clean":"2023-09-23T20:55:53.977152+0000","last_became_active":"2023-09-23T20:54:44.031790+0000","last_became_peered":"2023-09-23T20:54:44.031790+0000","last_unstale":"2023-09-23T20:55:53.977152+0000","last_undegraded":"2023-09-23T20:55:53.977152+0000","last_fullsized":"2023-09-23T20:55:53.977152+0000","mapping_epoch":142,"log_start":"107'900","ondisk_log_start":"107'900","created":128,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:52:13.843048+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:52:13.843048+0000","last_clean_scrub_stamp":"2023-09-23T20:52:13.843048+0000","objects_scrubbed":0,"log_size":8314,"log_dups_size":0,"ondisk_log_size":8314,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":27262976,"num_objects":416,"num_object_clones":0,"num_object_copies":1248,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":416,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5855,"num_write_kb":200672,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1019,"num_bytes_recovered":38404096,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":173,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-09-23T20:55:54.928061+0000","last_change":"2023-09-23T20:54:13.957131+0000","last_active":"2023-09-23T20:55:54.928061+0000","last_peered":"2023-09-23T20:55:54.928061+0000","last_clean":"2023-09-23T20:55:54.928061+0000","last_became_active":"2023-09-23T20:54:13.939561+0000","last_became_peered":"2023-09-23T20:54:13.939561+0000","last_unstale":"2023-09-23T20:55:54.928061+0000","last_undegraded":"2023-09-23T20:55:54.928061+0000","last_fullsized":"2023-09-23T20:55:54.928061+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:56:43.943284+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":140,"reported_epoch":175,"state":"active+clean","last_fresh":"2023-09-23T20:55:51.229892+0000","last_change":"2023-09-23T20:51:10.831262+0000","last_active":"2023-09-23T20:55:51.229892+0000","last_peered":"2023-09-23T20:55:51.229892+0000","last_clean":"2023-09-23T20:55:51.229892+0000","last_became_active":"2023-09-23T20:51:10.830931+0000","last_became_peered":"2023-09-23T20:51:10.830931+0000","last_unstale":"2023-09-23T20:55:51.229892+0000","last_undegraded":"2023-09-23T20:55:51.229892+0000","last_fullsized":"2023-09-23T20:55:51.229892+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:51:54.982505+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.17","version":"179'9404","reported_seq":12668,"reported_epoch":179,"state":"active+clean","last_fresh":"2023-09-23T20:55:57.060668+0000","last_change":"2023-09-23T20:55:46.955287+0000","last_active":"2023-09-23T20:55:57.060668+0000","last_peered":"2023-09-23T20:55:57.060668+0000","last_clean":"2023-09-23T20:55:57.060668+0000","last_became_active":"2023-09-23T20:55:46.932584+0000","last_became_peered":"2023-09-23T20:55:46.932584+0000","last_unstale":"2023-09-23T20:55:57.060668+0000","last_undegraded":"2023-09-23T20:55:57.060668+0000","last_fullsized":"2023-09-23T20:55:57.060668+0000","mapping_epoch":170,"log_start":"107'1000","ondisk_log_start":"107'1000","created":128,"last_epoch_clean":171,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:52:13.843048+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:52:13.843048+0000","last_clean_scrub_stamp":"2023-09-23T20:52:13.843048+0000","objects_scrubbed":0,"log_size":8404,"log_dups_size":0,"ondisk_log_size":8404,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":25886720,"num_objects":395,"num_object_clones":0,"num_object_copies":1185,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":395,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5984,"num_write_kb":204128,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1560,"num_bytes_recovered":72548352,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":142,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-09-23T20:55:54.284134+0000","last_change":"2023-09-23T20:54:44.037431+0000","last_active":"2023-09-23T20:55:54.284134+0000","last_peered":"2023-09-23T20:55:54.284134+0000","last_clean":"2023-09-23T20:55:54.284134+0000","last_became_active":"2023-09-23T20:54:44.037102+0000","last_became_peered":"2023-09-23T20:54:44.037102+0000","last_unstale":"2023-09-23T20:55:54.284134+0000","last_undegraded":"2023-09-23T20:55:54.284134+0000","last_fullsized":"2023-09-23T20:55:54.284134+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:12:11.074632+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":52,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-09-23T20:55:54.975424+0000","last_change":"2023-09-23T20:54:31.887814+0000","last_active":"2023-09-23T20:55:54.975424+0000","last_peered":"2023-09-23T20:55:54.975424+0000","last_clean":"2023-09-23T20:55:54.975424+0000","last_became_active":"2023-09-23T20:54:31.887195+0000","last_became_peered":"2023-09-23T20:54:31.887195+0000","last_unstale":"2023-09-23T20:55:54.975424+0000","last_undegraded":"2023-09-23T20:55:54.975424+0000","last_fullsized":"2023-09-23T20:55:54.975424+0000","mapping_epoch":136,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:10:52.089553+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"6.14","version":"176'9273","reported_seq":10806,"reported_epoch":176,"state":"active+clean","last_fresh":"2023-09-23T20:55:53.987644+0000","last_change":"2023-09-23T20:55:36.958390+0000","last_active":"2023-09-23T20:55:53.987644+0000","last_peered":"2023-09-23T20:55:53.987644+0000","last_clean":"2023-09-23T20:55:53.987644+0000","last_became_active":"2023-09-23T20:55:36.942420+0000","last_became_peered":"2023-09-23T20:55:36.942420+0000","last_unstale":"2023-09-23T20:55:53.987644+0000","last_undegraded":"2023-09-23T20:55:53.987644+0000","last_fullsized":"2023-09-23T20:55:53.987644+0000","mapping_epoch":165,"log_start":"107'600","ondisk_log_start":"107'600","created":128,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:52:13.843048+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:52:13.843048+0000","last_clean_scrub_stamp":"2023-09-23T20:52:13.843048+0000","objects_scrubbed":0,"log_size":8673,"log_dups_size":0,"ondisk_log_size":8673,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":25657344,"num_objects":392,"num_object_clones":0,"num_object_copies":1176,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":392,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5904,"num_write_kb":201440,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":702,"num_bytes_recovered":45940736,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,3],"acting":[1,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":159,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-09-23T20:55:54.926881+0000","last_change":"2023-09-23T20:55:35.947101+0000","last_active":"2023-09-23T20:55:54.926881+0000","last_peered":"2023-09-23T20:55:54.926881+0000","last_clean":"2023-09-23T20:55:54.926881+0000","last_became_active":"2023-09-23T20:55:35.946002+0000","last_became_peered":"2023-09-23T20:55:35.946002+0000","last_unstale":"2023-09-23T20:55:54.926881+0000","last_undegraded":"2023-09-23T20:55:54.926881+0000","last_fullsized":"2023-09-23T20:55:54.926881+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:31:31.122303+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":133,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-09-23T20:55:54.284622+0000","last_change":"2023-09-23T20:51:10.840849+0000","last_active":"2023-09-23T20:55:54.284622+0000","last_peered":"2023-09-23T20:55:54.284622+0000","last_clean":"2023-09-23T20:55:54.284622+0000","last_became_active":"2023-09-23T20:51:10.840593+0000","last_became_peered":"2023-09-23T20:51:10.840593+0000","last_unstale":"2023-09-23T20:55:54.284622+0000","last_undegraded":"2023-09-23T20:55:54.284622+0000","last_fullsized":"2023-09-23T20:55:54.284622+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:30:52.297625+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.15","version":"179'9330","reported_seq":10221,"reported_epoch":179,"state":"active+clean","last_fresh":"2023-09-23T20:55:57.067641+0000","last_change":"2023-09-23T20:55:54.967491+0000","last_active":"2023-09-23T20:55:57.067641+0000","last_peered":"2023-09-23T20:55:57.067641+0000","last_clean":"2023-09-23T20:55:57.067641+0000","last_became_active":"2023-09-23T20:55:54.930143+0000","last_became_peered":"2023-09-23T20:55:54.930143+0000","last_unstale":"2023-09-23T20:55:57.067641+0000","last_undegraded":"2023-09-23T20:55:57.067641+0000","last_fullsized":"2023-09-23T20:55:57.067641+0000","mapping_epoch":177,"log_start":"107'998","ondisk_log_start":"107'998","created":128,"last_epoch_clean":178,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:52:13.843048+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:52:13.843048+0000","last_clean_scrub_stamp":"2023-09-23T20:52:13.843048+0000","objects_scrubbed":0,"log_size":8332,"log_dups_size":0,"ondisk_log_size":8332,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19726336,"num_objects":301,"num_object_clones":0,"num_object_copies":903,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":301,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5982,"num_write_kb":201056,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":382,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,3],"acting":[7,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":168,"reported_epoch":175,"state":"active+clean","last_fresh":"2023-09-23T20:55:51.229426+0000","last_change":"2023-09-23T20:55:36.943969+0000","last_active":"2023-09-23T20:55:51.229426+0000","last_peered":"2023-09-23T20:55:51.229426+0000","last_clean":"2023-09-23T20:55:51.229426+0000","last_became_active":"2023-09-23T20:55:36.943580+0000","last_became_peered":"2023-09-23T20:55:36.943580+0000","last_unstale":"2023-09-23T20:55:51.229426+0000","last_undegraded":"2023-09-23T20:55:51.229426+0000","last_fullsized":"2023-09-23T20:55:51.229426+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:49:06.969396+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":101,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-09-23T20:55:54.927419+0000","last_change":"2023-09-23T20:54:44.051624+0000","last_active":"2023-09-23T20:55:54.927419+0000","last_peered":"2023-09-23T20:55:54.927419+0000","last_clean":"2023-09-23T20:55:54.927419+0000","last_became_active":"2023-09-23T20:54:44.051318+0000","last_became_peered":"2023-09-23T20:54:44.051318+0000","last_unstale":"2023-09-23T20:55:54.927419+0000","last_undegraded":"2023-09-23T20:55:54.927419+0000","last_fullsized":"2023-09-23T20:55:54.927419+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:39:44.612145+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,2],"acting":[7,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"6.a","version":"179'12020","reported_seq":20279,"reported_epoch":179,"state":"active+clean","last_fresh":"2023-09-23T20:55:57.056211+0000","last_change":"2023-09-23T20:55:49.252831+0000","last_active":"2023-09-23T20:55:57.056211+0000","last_peered":"2023-09-23T20:55:57.056211+0000","last_clean":"2023-09-23T20:55:57.056211+0000","last_became_active":"2023-09-23T20:55:49.213594+0000","last_became_peered":"2023-09-23T20:55:49.213594+0000","last_unstale":"2023-09-23T20:55:57.056211+0000","last_undegraded":"2023-09-23T20:55:57.056211+0000","last_fullsized":"2023-09-23T20:55:57.056211+0000","mapping_epoch":172,"log_start":"113'3377","ondisk_log_start":"113'3377","created":102,"last_epoch_clean":173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:52:13.843048+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:52:13.843048+0000","last_clean_scrub_stamp":"2023-09-23T20:52:13.843048+0000","objects_scrubbed":0,"log_size":8643,"log_dups_size":0,"ondisk_log_size":8643,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:44:19.565193+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":45744128,"num_objects":698,"num_object_clones":0,"num_object_copies":2094,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":698,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12020,"num_write_kb":406976,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4061,"num_bytes_recovered":212729856,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,0],"acting":[5,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":182,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-09-23T20:55:54.927319+0000","last_change":"2023-09-23T20:54:14.007167+0000","last_active":"2023-09-23T20:55:54.927319+0000","last_peered":"2023-09-23T20:55:54.927319+0000","last_clean":"2023-09-23T20:55:54.927319+0000","last_became_active":"2023-09-23T20:54:13.939827+0000","last_became_peered":"2023-09-23T20:54:13.939827+0000","last_unstale":"2023-09-23T20:55:54.927319+0000","last_undegraded":"2023-09-23T20:55:54.927319+0000","last_fullsized":"2023-09-23T20:55:54.927319+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:12:48.477909+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":66,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":69,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-09-23T20:55:54.975120+0000","last_change":"2023-09-23T20:55:35.957768+0000","last_active":"2023-09-23T20:55:54.975120+0000","last_peered":"2023-09-23T20:55:54.975120+0000","last_clean":"2023-09-23T20:55:54.975120+0000","last_became_active":"2023-09-23T20:55:35.957421+0000","last_became_peered":"2023-09-23T20:55:35.957421+0000","last_unstale":"2023-09-23T20:55:54.975120+0000","last_undegraded":"2023-09-23T20:55:54.975120+0000","last_fullsized":"2023-09-23T20:55:54.975120+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:49:41.585961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"6.b","version":"176'11554","reported_seq":11638,"reported_epoch":176,"state":"active+clean","last_fresh":"2023-09-23T20:55:54.083114+0000","last_change":"2023-09-23T20:52:14.868057+0000","last_active":"2023-09-23T20:55:54.083114+0000","last_peered":"2023-09-23T20:55:54.083114+0000","last_clean":"2023-09-23T20:55:54.083114+0000","last_became_active":"2023-09-23T20:52:14.867734+0000","last_became_peered":"2023-09-23T20:52:14.867734+0000","last_unstale":"2023-09-23T20:55:54.083114+0000","last_undegraded":"2023-09-23T20:55:54.083114+0000","last_fullsized":"2023-09-23T20:55:54.083114+0000","mapping_epoch":102,"log_start":"116'4346","ondisk_log_start":"116'4346","created":102,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:52:13.843048+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:52:13.843048+0000","last_clean_scrub_stamp":"2023-09-23T20:52:13.843048+0000","objects_scrubbed":0,"log_size":7208,"log_dups_size":0,"ondisk_log_size":7208,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:40:31.506977+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":54788096,"num_objects":836,"num_object_clones":0,"num_object_copies":2508,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":836,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11554,"num_write_kb":396480,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":169,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-09-23T20:55:54.927169+0000","last_change":"2023-09-23T20:54:14.024448+0000","last_active":"2023-09-23T20:55:54.927169+0000","last_peered":"2023-09-23T20:55:54.927169+0000","last_clean":"2023-09-23T20:55:54.927169+0000","last_became_active":"2023-09-23T20:54:13.944740+0000","last_became_peered":"2023-09-23T20:54:13.944740+0000","last_unstale":"2023-09-23T20:55:54.927169+0000","last_undegraded":"2023-09-23T20:55:54.927169+0000","last_fullsized":"2023-09-23T20:55:54.927169+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:24:20.877376+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":90,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-09-23T20:55:54.925166+0000","last_change":"2023-09-23T20:55:36.943955+0000","last_active":"2023-09-23T20:55:54.925166+0000","last_peered":"2023-09-23T20:55:54.925166+0000","last_clean":"2023-09-23T20:55:54.925166+0000","last_became_active":"2023-09-23T20:55:36.943639+0000","last_became_peered":"2023-09-23T20:55:36.943639+0000","last_unstale":"2023-09-23T20:55:54.925166+0000","last_undegraded":"2023-09-23T20:55:54.925166+0000","last_fullsized":"2023-09-23T20:55:54.925166+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:06:13.742269+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.8","version":"179'9506","reported_seq":9616,"reported_epoch":179,"state":"active+clean","last_fresh":"2023-09-23T20:55:57.138897+0000","last_change":"2023-09-23T20:55:33.124955+0000","last_active":"2023-09-23T20:55:57.138897+0000","last_peered":"2023-09-23T20:55:57.138897+0000","last_clean":"2023-09-23T20:55:57.138897+0000","last_became_active":"2023-09-23T20:54:03.758672+0000","last_became_peered":"2023-09-23T20:54:03.758672+0000","last_unstale":"2023-09-23T20:55:57.138897+0000","last_undegraded":"2023-09-23T20:55:57.138897+0000","last_fullsized":"2023-09-23T20:55:57.138897+0000","mapping_epoch":102,"log_start":"107'1141","ondisk_log_start":"107'1141","created":102,"last_epoch_clean":129,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:52:13.843048+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:52:13.843048+0000","last_clean_scrub_stamp":"2023-09-23T20:52:13.843048+0000","objects_scrubbed":0,"log_size":8365,"log_dups_size":0,"ondisk_log_size":8365,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17301504,"num_objects":264,"num_object_clones":0,"num_object_copies":792,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":264,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6095,"num_write_kb":203488,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,3],"acting":[7,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":168,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-09-23T20:55:54.927178+0000","last_change":"2023-09-23T20:55:35.981413+0000","last_active":"2023-09-23T20:55:54.927178+0000","last_peered":"2023-09-23T20:55:54.927178+0000","last_clean":"2023-09-23T20:55:54.927178+0000","last_became_active":"2023-09-23T20:55:35.980678+0000","last_became_peered":"2023-09-23T20:55:35.980678+0000","last_unstale":"2023-09-23T20:55:54.927178+0000","last_undegraded":"2023-09-23T20:55:54.927178+0000","last_fullsized":"2023-09-23T20:55:54.927178+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:36:58.620275+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":140,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":115,"reported_epoch":175,"state":"active+clean","last_fresh":"2023-09-23T20:55:51.229691+0000","last_change":"2023-09-23T20:55:35.965427+0000","last_active":"2023-09-23T20:55:51.229691+0000","last_peered":"2023-09-23T20:55:51.229691+0000","last_clean":"2023-09-23T20:55:51.229691+0000","last_became_active":"2023-09-23T20:55:35.963969+0000","last_became_peered":"2023-09-23T20:55:35.963969+0000","last_unstale":"2023-09-23T20:55:51.229691+0000","last_undegraded":"2023-09-23T20:55:51.229691+0000","last_fullsized":"2023-09-23T20:55:51.229691+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:38:40.612083+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.9","version":"178'9481","reported_seq":9612,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-09-23T20:55:55.511200+0000","last_change":"2023-09-23T20:55:30.500052+0000","last_active":"2023-09-23T20:55:55.511200+0000","last_peered":"2023-09-23T20:55:55.511200+0000","last_clean":"2023-09-23T20:55:55.511200+0000","last_became_active":"2023-09-23T20:54:03.701452+0000","last_became_peered":"2023-09-23T20:54:03.701452+0000","last_unstale":"2023-09-23T20:55:55.511200+0000","last_undegraded":"2023-09-23T20:55:55.511200+0000","last_fullsized":"2023-09-23T20:55:55.511200+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":102,"last_epoch_clean":129,"parent":"0.0","parent_split_bits":0,"last_scrub":"162'8928","last_scrub_stamp":"2023-09-23T20:55:30.499752+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:52:13.843048+0000","last_clean_scrub_stamp":"2023-09-23T20:55:30.499752+0000","objects_scrubbed":935,"log_size":9481,"log_dups_size":0,"ondisk_log_size":9481,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:25:43.038827+0000","scrub_duration":0.367003197,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":24707072,"num_objects":377,"num_object_clones":0,"num_object_copies":1131,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":377,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":558,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,2],"acting":[0,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":69,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-09-23T20:55:54.978223+0000","last_change":"2023-09-23T20:55:35.962005+0000","last_active":"2023-09-23T20:55:54.978223+0000","last_peered":"2023-09-23T20:55:54.978223+0000","last_clean":"2023-09-23T20:55:54.978223+0000","last_became_active":"2023-09-23T20:55:35.961695+0000","last_became_peered":"2023-09-23T20:55:35.961695+0000","last_unstale":"2023-09-23T20:55:54.978223+0000","last_undegraded":"2023-09-23T20:55:54.978223+0000","last_fullsized":"2023-09-23T20:55:54.978223+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:28:22.876201+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":164,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-09-23T20:55:54.284276+0000","last_change":"2023-09-23T20:54:13.928715+0000","last_active":"2023-09-23T20:55:54.284276+0000","last_peered":"2023-09-23T20:55:54.284276+0000","last_clean":"2023-09-23T20:55:54.284276+0000","last_became_active":"2023-09-23T20:54:13.928200+0000","last_became_peered":"2023-09-23T20:54:13.928200+0000","last_unstale":"2023-09-23T20:55:54.284276+0000","last_undegraded":"2023-09-23T20:55:54.284276+0000","last_fullsized":"2023-09-23T20:55:54.284276+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:41:16.927513+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.9","version":"79'161","reported_seq":547,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-09-23T20:55:54.926668+0000","last_change":"2023-09-23T20:55:35.947484+0000","last_active":"2023-09-23T20:55:54.926668+0000","last_peered":"2023-09-23T20:55:54.926668+0000","last_clean":"2023-09-23T20:55:54.926668+0000","last_became_active":"2023-09-23T20:55:35.946805+0000","last_became_peered":"2023-09-23T20:55:35.946805+0000","last_unstale":"2023-09-23T20:55:54.926668+0000","last_undegraded":"2023-09-23T20:55:54.926668+0000","last_fullsized":"2023-09-23T20:55:54.926668+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:38:57.324224+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":45,"num_read_kb":84,"num_write":21,"num_write_kb":219,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":161009,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"6.e","version":"179'12110","reported_seq":17709,"reported_epoch":179,"state":"active+clean","last_fresh":"2023-09-23T20:55:57.097308+0000","last_change":"2023-09-23T20:54:55.922739+0000","last_active":"2023-09-23T20:55:57.097308+0000","last_peered":"2023-09-23T20:55:57.097308+0000","last_clean":"2023-09-23T20:55:57.097308+0000","last_became_active":"2023-09-23T20:54:55.902420+0000","last_became_peered":"2023-09-23T20:54:55.902420+0000","last_unstale":"2023-09-23T20:55:57.097308+0000","last_undegraded":"2023-09-23T20:55:57.097308+0000","last_fullsized":"2023-09-23T20:55:57.097308+0000","mapping_epoch":146,"log_start":"117'4928","ondisk_log_start":"117'4928","created":102,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:52:13.843048+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:52:13.843048+0000","last_clean_scrub_stamp":"2023-09-23T20:52:13.843048+0000","objects_scrubbed":0,"log_size":7182,"log_dups_size":0,"ondisk_log_size":7182,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:34:13.390956+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47710208,"num_objects":728,"num_object_clones":0,"num_object_copies":2184,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":728,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12110,"num_write_kb":410816,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2950,"num_bytes_recovered":135856128,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":164,"reported_epoch":175,"state":"active+clean","last_fresh":"2023-09-23T20:55:51.230336+0000","last_change":"2023-09-23T20:54:13.961788+0000","last_active":"2023-09-23T20:55:51.230336+0000","last_peered":"2023-09-23T20:55:51.230336+0000","last_clean":"2023-09-23T20:55:51.230336+0000","last_became_active":"2023-09-23T20:54:13.940937+0000","last_became_peered":"2023-09-23T20:54:13.940937+0000","last_unstale":"2023-09-23T20:55:51.230336+0000","last_undegraded":"2023-09-23T20:55:51.230336+0000","last_fullsized":"2023-09-23T20:55:51.230336+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:11:56.785299+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":73,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-09-23T20:55:54.925960+0000","last_change":"2023-09-23T20:55:35.949670+0000","last_active":"2023-09-23T20:55:54.925960+0000","last_peered":"2023-09-23T20:55:54.925960+0000","last_clean":"2023-09-23T20:55:54.925960+0000","last_became_active":"2023-09-23T20:55:35.949140+0000","last_became_peered":"2023-09-23T20:55:35.949140+0000","last_unstale":"2023-09-23T20:55:54.925960+0000","last_undegraded":"2023-09-23T20:55:54.925960+0000","last_fullsized":"2023-09-23T20:55:54.925960+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:55:59.340303+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.8","version":"79'161","reported_seq":547,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-09-23T20:55:54.926592+0000","last_change":"2023-09-23T20:55:35.952566+0000","last_active":"2023-09-23T20:55:54.926592+0000","last_peered":"2023-09-23T20:55:54.926592+0000","last_clean":"2023-09-23T20:55:54.926592+0000","last_became_active":"2023-09-23T20:55:35.952149+0000","last_became_peered":"2023-09-23T20:55:35.952149+0000","last_unstale":"2023-09-23T20:55:54.926592+0000","last_undegraded":"2023-09-23T20:55:54.926592+0000","last_fullsized":"2023-09-23T20:55:54.926592+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:38:57.324224+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":45,"num_read_kb":84,"num_write":22,"num_write_kb":219,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":161009,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"6.f","version":"179'11788","reported_seq":17528,"reported_epoch":179,"state":"active+clean","last_fresh":"2023-09-23T20:55:56.318078+0000","last_change":"2023-09-23T20:54:32.961255+0000","last_active":"2023-09-23T20:55:56.318078+0000","last_peered":"2023-09-23T20:55:56.318078+0000","last_clean":"2023-09-23T20:55:56.318078+0000","last_became_active":"2023-09-23T20:54:32.944166+0000","last_became_peered":"2023-09-23T20:54:32.944166+0000","last_unstale":"2023-09-23T20:55:56.318078+0000","last_undegraded":"2023-09-23T20:55:56.318078+0000","last_fullsized":"2023-09-23T20:55:56.318078+0000","mapping_epoch":137,"log_start":"114'3641","ondisk_log_start":"114'3641","created":102,"last_epoch_clean":138,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:52:13.843048+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:52:13.843048+0000","last_clean_scrub_stamp":"2023-09-23T20:52:13.843048+0000","objects_scrubbed":0,"log_size":8147,"log_dups_size":0,"ondisk_log_size":8147,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:06:07.274753+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":46399488,"num_objects":708,"num_object_clones":0,"num_object_copies":2124,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":708,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11788,"num_write_kb":399872,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2813,"num_bytes_recovered":127139840,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":138,"reported_epoch":175,"state":"active+clean","last_fresh":"2023-09-23T20:55:51.230066+0000","last_change":"2023-09-23T20:55:35.959942+0000","last_active":"2023-09-23T20:55:51.230066+0000","last_peered":"2023-09-23T20:55:51.230066+0000","last_clean":"2023-09-23T20:55:51.230066+0000","last_became_active":"2023-09-23T20:55:35.957071+0000","last_became_peered":"2023-09-23T20:55:35.957071+0000","last_unstale":"2023-09-23T20:55:51.230066+0000","last_undegraded":"2023-09-23T20:55:51.230066+0000","last_fullsized":"2023-09-23T20:55:51.230066+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:44:30.308036+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":56,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-09-23T20:55:54.926281+0000","last_change":"2023-09-23T20:54:13.951459+0000","last_active":"2023-09-23T20:55:54.926281+0000","last_peered":"2023-09-23T20:55:54.926281+0000","last_clean":"2023-09-23T20:55:54.926281+0000","last_became_active":"2023-09-23T20:54:13.948409+0000","last_became_peered":"2023-09-23T20:54:13.948409+0000","last_unstale":"2023-09-23T20:55:54.926281+0000","last_undegraded":"2023-09-23T20:55:54.926281+0000","last_fullsized":"2023-09-23T20:55:54.926281+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:38:10.148061+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.c","version":"176'11740","reported_seq":17168,"reported_epoch":176,"state":"active+clean","last_fresh":"2023-09-23T20:55:54.144266+0000","last_change":"2023-09-23T20:55:44.960762+0000","last_active":"2023-09-23T20:55:54.144266+0000","last_peered":"2023-09-23T20:55:54.144266+0000","last_clean":"2023-09-23T20:55:54.144266+0000","last_became_active":"2023-09-23T20:55:44.927799+0000","last_became_peered":"2023-09-23T20:55:44.927799+0000","last_unstale":"2023-09-23T20:55:54.144266+0000","last_undegraded":"2023-09-23T20:55:54.144266+0000","last_fullsized":"2023-09-23T20:55:54.144266+0000","mapping_epoch":168,"log_start":"116'4463","ondisk_log_start":"116'4463","created":102,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:52:13.843048+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:52:13.843048+0000","last_clean_scrub_stamp":"2023-09-23T20:52:13.843048+0000","objects_scrubbed":0,"log_size":7277,"log_dups_size":0,"ondisk_log_size":7277,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:39:28.324907+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":54525952,"num_objects":832,"num_object_clones":0,"num_object_copies":2496,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":832,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11740,"num_write_kb":402304,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2649,"num_bytes_recovered":121831424,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":56,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-09-23T20:55:54.924954+0000","last_change":"2023-09-23T20:54:13.950254+0000","last_active":"2023-09-23T20:55:54.924954+0000","last_peered":"2023-09-23T20:55:54.924954+0000","last_clean":"2023-09-23T20:55:54.924954+0000","last_became_active":"2023-09-23T20:54:13.949934+0000","last_became_peered":"2023-09-23T20:54:13.949934+0000","last_unstale":"2023-09-23T20:55:54.924954+0000","last_undegraded":"2023-09-23T20:55:54.924954+0000","last_fullsized":"2023-09-23T20:55:54.924954+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:30:32.847167+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":73,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-09-23T20:55:54.925168+0000","last_change":"2023-09-23T20:55:35.956153+0000","last_active":"2023-09-23T20:55:54.925168+0000","last_peered":"2023-09-23T20:55:54.925168+0000","last_clean":"2023-09-23T20:55:54.925168+0000","last_became_active":"2023-09-23T20:55:35.955630+0000","last_became_peered":"2023-09-23T20:55:35.955630+0000","last_unstale":"2023-09-23T20:55:54.925168+0000","last_undegraded":"2023-09-23T20:55:54.925168+0000","last_fullsized":"2023-09-23T20:55:54.925168+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:45:20.282929+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.a","version":"79'161","reported_seq":547,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-09-23T20:55:54.927654+0000","last_change":"2023-09-23T20:55:35.952596+0000","last_active":"2023-09-23T20:55:54.927654+0000","last_peered":"2023-09-23T20:55:54.927654+0000","last_clean":"2023-09-23T20:55:54.927654+0000","last_became_active":"2023-09-23T20:55:35.952276+0000","last_became_peered":"2023-09-23T20:55:35.952276+0000","last_unstale":"2023-09-23T20:55:54.927654+0000","last_undegraded":"2023-09-23T20:55:54.927654+0000","last_fullsized":"2023-09-23T20:55:54.927654+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:38:57.324224+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53669,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":45,"num_read_kb":84,"num_write":21,"num_write_kb":219,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":161009,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"6.d","version":"179'12061","reported_seq":20229,"reported_epoch":179,"state":"active+clean","last_fresh":"2023-09-23T20:55:57.088713+0000","last_change":"2023-09-23T20:54:54.922926+0000","last_active":"2023-09-23T20:55:57.088713+0000","last_peered":"2023-09-23T20:55:57.088713+0000","last_clean":"2023-09-23T20:55:57.088713+0000","last_became_active":"2023-09-23T20:54:54.898748+0000","last_became_peered":"2023-09-23T20:54:54.898748+0000","last_unstale":"2023-09-23T20:55:57.088713+0000","last_undegraded":"2023-09-23T20:55:57.088713+0000","last_fullsized":"2023-09-23T20:55:57.088713+0000","mapping_epoch":145,"log_start":"109'2047","ondisk_log_start":"109'2047","created":102,"last_epoch_clean":146,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:52:13.843048+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:52:13.843048+0000","last_clean_scrub_stamp":"2023-09-23T20:52:13.843048+0000","objects_scrubbed":0,"log_size":10014,"log_dups_size":0,"ondisk_log_size":10014,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:31:42.763337+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":45482008,"num_objects":695,"num_object_clones":0,"num_object_copies":2085,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":695,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":12061,"num_write_kb":408129,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4030,"num_bytes_recovered":264044568,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":98,"reported_epoch":175,"state":"active+clean","last_fresh":"2023-09-23T20:55:51.234208+0000","last_change":"2023-09-23T20:54:44.034235+0000","last_active":"2023-09-23T20:55:51.234208+0000","last_peered":"2023-09-23T20:55:51.234208+0000","last_clean":"2023-09-23T20:55:51.234208+0000","last_became_active":"2023-09-23T20:54:44.033911+0000","last_became_peered":"2023-09-23T20:54:44.033911+0000","last_unstale":"2023-09-23T20:55:51.234208+0000","last_undegraded":"2023-09-23T20:55:51.234208+0000","last_fullsized":"2023-09-23T20:55:51.234208+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:12:48.442227+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,4],"acting":[1,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":144,"reported_epoch":175,"state":"active+clean","last_fresh":"2023-09-23T20:55:51.234813+0000","last_change":"2023-09-23T20:54:13.940748+0000","last_active":"2023-09-23T20:55:51.234813+0000","last_peered":"2023-09-23T20:55:51.234813+0000","last_clean":"2023-09-23T20:55:51.234813+0000","last_became_active":"2023-09-23T20:54:13.940419+0000","last_became_peered":"2023-09-23T20:54:13.940419+0000","last_unstale":"2023-09-23T20:55:51.234813+0000","last_undegraded":"2023-09-23T20:55:51.234813+0000","last_fullsized":"2023-09-23T20:55:51.234813+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:15:01.923816+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.7","version":"79'161","reported_seq":547,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-09-23T20:55:54.927096+0000","last_change":"2023-09-23T20:55:35.952070+0000","last_active":"2023-09-23T20:55:54.927096+0000","last_peered":"2023-09-23T20:55:54.927096+0000","last_clean":"2023-09-23T20:55:54.927096+0000","last_became_active":"2023-09-23T20:55:35.951681+0000","last_became_peered":"2023-09-23T20:55:35.951681+0000","last_unstale":"2023-09-23T20:55:54.927096+0000","last_undegraded":"2023-09-23T20:55:54.927096+0000","last_fullsized":"2023-09-23T20:55:54.927096+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:38:57.324224+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":45,"num_read_kb":84,"num_write":22,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":161009,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":149,"reported_epoch":175,"state":"active+clean","last_fresh":"2023-09-23T20:55:51.234592+0000","last_change":"2023-09-23T20:50:05.867130+0000","last_active":"2023-09-23T20:55:51.234592+0000","last_peered":"2023-09-23T20:55:51.234592+0000","last_clean":"2023-09-23T20:55:51.234592+0000","last_became_active":"2023-09-23T20:50:05.866758+0000","last_became_peered":"2023-09-23T20:50:05.866758+0000","last_unstale":"2023-09-23T20:55:51.234592+0000","last_undegraded":"2023-09-23T20:55:51.234592+0000","last_fullsized":"2023-09-23T20:55:51.234592+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:55:07.548209+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":52,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-09-23T20:55:54.975387+0000","last_change":"2023-09-23T20:54:31.887604+0000","last_active":"2023-09-23T20:55:54.975387+0000","last_peered":"2023-09-23T20:55:54.975387+0000","last_clean":"2023-09-23T20:55:54.975387+0000","last_became_active":"2023-09-23T20:54:31.886752+0000","last_became_peered":"2023-09-23T20:54:31.886752+0000","last_unstale":"2023-09-23T20:55:54.975387+0000","last_undegraded":"2023-09-23T20:55:54.975387+0000","last_fullsized":"2023-09-23T20:55:54.975387+0000","mapping_epoch":136,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:13:58.872176+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":147,"reported_epoch":175,"state":"active+clean","last_fresh":"2023-09-23T20:55:51.234377+0000","last_change":"2023-09-23T20:54:31.899082+0000","last_active":"2023-09-23T20:55:51.234377+0000","last_peered":"2023-09-23T20:55:51.234377+0000","last_clean":"2023-09-23T20:55:51.234377+0000","last_became_active":"2023-09-23T20:54:31.898521+0000","last_became_peered":"2023-09-23T20:54:31.898521+0000","last_unstale":"2023-09-23T20:55:51.234377+0000","last_undegraded":"2023-09-23T20:55:51.234377+0000","last_fullsized":"2023-09-23T20:55:51.234377+0000","mapping_epoch":136,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:25:36.059711+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.0","version":"178'9396","reported_seq":9496,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-09-23T20:55:55.516156+0000","last_change":"2023-09-23T20:55:34.164543+0000","last_active":"2023-09-23T20:55:55.516156+0000","last_peered":"2023-09-23T20:55:55.516156+0000","last_clean":"2023-09-23T20:55:55.516156+0000","last_became_active":"2023-09-23T20:54:03.763183+0000","last_became_peered":"2023-09-23T20:54:03.763183+0000","last_unstale":"2023-09-23T20:55:55.516156+0000","last_undegraded":"2023-09-23T20:55:55.516156+0000","last_fullsized":"2023-09-23T20:55:55.516156+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":102,"last_epoch_clean":129,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:52:13.843048+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:52:13.843048+0000","last_clean_scrub_stamp":"2023-09-23T20:52:13.843048+0000","objects_scrubbed":0,"log_size":9396,"log_dups_size":0,"ondisk_log_size":9396,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":24805376,"num_objects":378,"num_object_clones":0,"num_object_copies":1134,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":378,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5981,"num_write_kb":203520,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1","version":"79'161","reported_seq":546,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-09-23T20:55:54.926688+0000","last_change":"2023-09-23T20:55:35.947295+0000","last_active":"2023-09-23T20:55:54.926688+0000","last_peered":"2023-09-23T20:55:54.926688+0000","last_clean":"2023-09-23T20:55:54.926688+0000","last_became_active":"2023-09-23T20:55:35.946395+0000","last_became_peered":"2023-09-23T20:55:35.946395+0000","last_unstale":"2023-09-23T20:55:54.926688+0000","last_undegraded":"2023-09-23T20:55:54.926688+0000","last_fullsized":"2023-09-23T20:55:54.926688+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:38:57.324224+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":46,"num_read_kb":84,"num_write":22,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":161010,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"6.6","version":"176'9252","reported_seq":11379,"reported_epoch":176,"state":"active+clean","last_fresh":"2023-09-23T20:55:53.968693+0000","last_change":"2023-09-23T20:55:18.611897+0000","last_active":"2023-09-23T20:55:53.968693+0000","last_peered":"2023-09-23T20:55:53.968693+0000","last_clean":"2023-09-23T20:55:53.968693+0000","last_became_active":"2023-09-23T20:55:02.041495+0000","last_became_peered":"2023-09-23T20:55:02.041495+0000","last_unstale":"2023-09-23T20:55:53.968693+0000","last_undegraded":"2023-09-23T20:55:53.968693+0000","last_fullsized":"2023-09-23T20:55:53.968693+0000","mapping_epoch":148,"log_start":"107'900","ondisk_log_start":"107'900","created":102,"last_epoch_clean":156,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:52:13.843048+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:52:13.843048+0000","last_clean_scrub_stamp":"2023-09-23T20:52:13.843048+0000","objects_scrubbed":0,"log_size":8352,"log_dups_size":0,"ondisk_log_size":8352,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":24772608,"num_objects":378,"num_object_clones":0,"num_object_copies":1134,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":378,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5893,"num_write_kb":200672,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1002,"num_bytes_recovered":37486592,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":69,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-09-23T20:55:54.975587+0000","last_change":"2023-09-23T20:55:35.959514+0000","last_active":"2023-09-23T20:55:54.975587+0000","last_peered":"2023-09-23T20:55:54.975587+0000","last_clean":"2023-09-23T20:55:54.975587+0000","last_became_active":"2023-09-23T20:55:35.959244+0000","last_became_peered":"2023-09-23T20:55:35.959244+0000","last_unstale":"2023-09-23T20:55:54.975587+0000","last_undegraded":"2023-09-23T20:55:54.975587+0000","last_fullsized":"2023-09-23T20:55:54.975587+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:47:32.665316+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":73,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-09-23T20:55:54.925542+0000","last_change":"2023-09-23T20:55:35.949000+0000","last_active":"2023-09-23T20:55:54.925542+0000","last_peered":"2023-09-23T20:55:54.925542+0000","last_clean":"2023-09-23T20:55:54.925542+0000","last_became_active":"2023-09-23T20:55:35.948410+0000","last_became_peered":"2023-09-23T20:55:35.948410+0000","last_unstale":"2023-09-23T20:55:54.925542+0000","last_undegraded":"2023-09-23T20:55:54.925542+0000","last_fullsized":"2023-09-23T20:55:54.925542+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:26:50.477289+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":141,"reported_epoch":175,"state":"active+clean","last_fresh":"2023-09-23T20:55:51.234381+0000","last_change":"2023-09-23T20:51:10.829889+0000","last_active":"2023-09-23T20:55:51.234381+0000","last_peered":"2023-09-23T20:55:51.234381+0000","last_clean":"2023-09-23T20:55:51.234381+0000","last_became_active":"2023-09-23T20:51:10.829533+0000","last_became_peered":"2023-09-23T20:51:10.829533+0000","last_unstale":"2023-09-23T20:55:51.234381+0000","last_undegraded":"2023-09-23T20:55:51.234381+0000","last_fullsized":"2023-09-23T20:55:51.234381+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:43:04.603460+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"79'161","reported_seq":547,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-09-23T20:55:54.927261+0000","last_change":"2023-09-23T20:55:35.952265+0000","last_active":"2023-09-23T20:55:54.927261+0000","last_peered":"2023-09-23T20:55:54.927261+0000","last_clean":"2023-09-23T20:55:54.927261+0000","last_became_active":"2023-09-23T20:55:35.951898+0000","last_became_peered":"2023-09-23T20:55:35.951898+0000","last_unstale":"2023-09-23T20:55:54.927261+0000","last_undegraded":"2023-09-23T20:55:54.927261+0000","last_fullsized":"2023-09-23T20:55:54.927261+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:38:57.324224+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":46,"num_read_kb":84,"num_write":22,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":161010,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"6.5","version":"179'9189","reported_seq":10062,"reported_epoch":179,"state":"active+clean","last_fresh":"2023-09-23T20:55:57.140069+0000","last_change":"2023-09-23T20:55:51.280500+0000","last_active":"2023-09-23T20:55:57.140069+0000","last_peered":"2023-09-23T20:55:57.140069+0000","last_clean":"2023-09-23T20:55:57.140069+0000","last_became_active":"2023-09-23T20:55:51.232779+0000","last_became_peered":"2023-09-23T20:55:51.232779+0000","last_unstale":"2023-09-23T20:55:57.140069+0000","last_undegraded":"2023-09-23T20:55:57.140069+0000","last_fullsized":"2023-09-23T20:55:57.140069+0000","mapping_epoch":174,"log_start":"107'397","ondisk_log_start":"107'397","created":102,"last_epoch_clean":175,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:52:13.843048+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:52:13.843048+0000","last_clean_scrub_stamp":"2023-09-23T20:52:13.843048+0000","objects_scrubbed":0,"log_size":8792,"log_dups_size":0,"ondisk_log_size":8792,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":28966912,"num_objects":442,"num_object_clones":0,"num_object_copies":1326,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":442,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5841,"num_write_kb":201056,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":369,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,3],"acting":[7,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":163,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-09-23T20:55:54.930895+0000","last_change":"2023-09-23T20:51:10.838054+0000","last_active":"2023-09-23T20:55:54.930895+0000","last_peered":"2023-09-23T20:55:54.930895+0000","last_clean":"2023-09-23T20:55:54.930895+0000","last_became_active":"2023-09-23T20:51:10.837750+0000","last_became_peered":"2023-09-23T20:51:10.837750+0000","last_unstale":"2023-09-23T20:55:54.930895+0000","last_undegraded":"2023-09-23T20:55:54.930895+0000","last_fullsized":"2023-09-23T20:55:54.930895+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:15:51.648085+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":141,"reported_epoch":175,"state":"active+clean","last_fresh":"2023-09-23T20:55:51.234067+0000","last_change":"2023-09-23T20:55:35.951146+0000","last_active":"2023-09-23T20:55:51.234067+0000","last_peered":"2023-09-23T20:55:51.234067+0000","last_clean":"2023-09-23T20:55:51.234067+0000","last_became_active":"2023-09-23T20:55:35.950874+0000","last_became_peered":"2023-09-23T20:55:35.950874+0000","last_unstale":"2023-09-23T20:55:51.234067+0000","last_undegraded":"2023-09-23T20:55:51.234067+0000","last_fullsized":"2023-09-23T20:55:51.234067+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:58:13.957677+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":162,"reported_epoch":175,"state":"active+clean","last_fresh":"2023-09-23T20:55:51.234774+0000","last_change":"2023-09-23T20:54:31.898870+0000","last_active":"2023-09-23T20:55:51.234774+0000","last_peered":"2023-09-23T20:55:51.234774+0000","last_clean":"2023-09-23T20:55:51.234774+0000","last_became_active":"2023-09-23T20:54:31.898467+0000","last_became_peered":"2023-09-23T20:54:31.898467+0000","last_unstale":"2023-09-23T20:55:51.234774+0000","last_undegraded":"2023-09-23T20:55:51.234774+0000","last_fullsized":"2023-09-23T20:55:51.234774+0000","mapping_epoch":136,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:56:22.483408+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.7","version":"179'9540","reported_seq":13300,"reported_epoch":179,"state":"active+clean","last_fresh":"2023-09-23T20:55:57.067619+0000","last_change":"2023-09-23T20:55:46.955291+0000","last_active":"2023-09-23T20:55:57.067619+0000","last_peered":"2023-09-23T20:55:57.067619+0000","last_clean":"2023-09-23T20:55:57.067619+0000","last_became_active":"2023-09-23T20:55:46.932571+0000","last_became_peered":"2023-09-23T20:55:46.932571+0000","last_unstale":"2023-09-23T20:55:57.067619+0000","last_undegraded":"2023-09-23T20:55:57.067619+0000","last_fullsized":"2023-09-23T20:55:57.067619+0000","mapping_epoch":170,"log_start":"107'1000","ondisk_log_start":"107'1000","created":102,"last_epoch_clean":171,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:52:13.843048+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:52:13.843048+0000","last_clean_scrub_stamp":"2023-09-23T20:52:13.843048+0000","objects_scrubbed":0,"log_size":8540,"log_dups_size":0,"ondisk_log_size":8540,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":16973824,"num_objects":259,"num_object_clones":0,"num_object_copies":777,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":259,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6120,"num_write_kb":204128,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1805,"num_bytes_recovered":89194496,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":52,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-09-23T20:55:54.975354+0000","last_change":"2023-09-23T20:54:31.883504+0000","last_active":"2023-09-23T20:55:54.975354+0000","last_peered":"2023-09-23T20:55:54.975354+0000","last_clean":"2023-09-23T20:55:54.975354+0000","last_became_active":"2023-09-23T20:54:31.883172+0000","last_became_peered":"2023-09-23T20:54:31.883172+0000","last_unstale":"2023-09-23T20:55:54.975354+0000","last_undegraded":"2023-09-23T20:55:54.975354+0000","last_fullsized":"2023-09-23T20:55:54.975354+0000","mapping_epoch":136,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:45:40.289120+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"79'161","reported_seq":548,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-09-23T20:55:54.926695+0000","last_change":"2023-09-23T20:55:35.952651+0000","last_active":"2023-09-23T20:55:54.926695+0000","last_peered":"2023-09-23T20:55:54.926695+0000","last_clean":"2023-09-23T20:55:54.926695+0000","last_became_active":"2023-09-23T20:55:35.952398+0000","last_became_peered":"2023-09-23T20:55:35.952398+0000","last_unstale":"2023-09-23T20:55:54.926695+0000","last_undegraded":"2023-09-23T20:55:54.926695+0000","last_fullsized":"2023-09-23T20:55:54.926695+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:38:57.324224+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53669,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":45,"num_read_kb":84,"num_write":21,"num_write_kb":219,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":161009,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":152,"reported_epoch":175,"state":"active+clean","last_fresh":"2023-09-23T20:55:51.230160+0000","last_change":"2023-09-23T20:55:35.950777+0000","last_active":"2023-09-23T20:55:51.230160+0000","last_peered":"2023-09-23T20:55:51.230160+0000","last_clean":"2023-09-23T20:55:51.230160+0000","last_became_active":"2023-09-23T20:55:35.950427+0000","last_became_peered":"2023-09-23T20:55:35.950427+0000","last_unstale":"2023-09-23T20:55:51.230160+0000","last_undegraded":"2023-09-23T20:55:51.230160+0000","last_fullsized":"2023-09-23T20:55:51.230160+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:20:10.371751+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":120,"reported_epoch":175,"state":"active+clean","last_fresh":"2023-09-23T20:55:51.232205+0000","last_change":"2023-09-23T20:55:35.978140+0000","last_active":"2023-09-23T20:55:51.232205+0000","last_peered":"2023-09-23T20:55:51.232205+0000","last_clean":"2023-09-23T20:55:51.232205+0000","last_became_active":"2023-09-23T20:55:35.977831+0000","last_became_peered":"2023-09-23T20:55:35.977831+0000","last_unstale":"2023-09-23T20:55:51.232205+0000","last_undegraded":"2023-09-23T20:55:51.232205+0000","last_fullsized":"2023-09-23T20:55:51.232205+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:25:47.678023+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.3","version":"79'161","reported_seq":547,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-09-23T20:55:54.927792+0000","last_change":"2023-09-23T20:55:35.945053+0000","last_active":"2023-09-23T20:55:54.927792+0000","last_peered":"2023-09-23T20:55:54.927792+0000","last_clean":"2023-09-23T20:55:54.927792+0000","last_became_active":"2023-09-23T20:55:35.944737+0000","last_became_peered":"2023-09-23T20:55:35.944737+0000","last_unstale":"2023-09-23T20:55:54.927792+0000","last_undegraded":"2023-09-23T20:55:54.927792+0000","last_fullsized":"2023-09-23T20:55:54.927792+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:38:57.324224+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":46,"num_read_kb":84,"num_write":22,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":161010,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":160,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-09-23T20:55:54.926689+0000","last_change":"2023-09-23T20:50:05.866898+0000","last_active":"2023-09-23T20:55:54.926689+0000","last_peered":"2023-09-23T20:55:54.926689+0000","last_clean":"2023-09-23T20:55:54.926689+0000","last_became_active":"2023-09-23T20:50:05.866541+0000","last_became_peered":"2023-09-23T20:50:05.866541+0000","last_unstale":"2023-09-23T20:55:54.926689+0000","last_undegraded":"2023-09-23T20:55:54.926689+0000","last_fullsized":"2023-09-23T20:55:54.926689+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:17:02.353010+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":161,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-09-23T20:55:54.284059+0000","last_change":"2023-09-23T20:54:44.035149+0000","last_active":"2023-09-23T20:55:54.284059+0000","last_peered":"2023-09-23T20:55:54.284059+0000","last_clean":"2023-09-23T20:55:54.284059+0000","last_became_active":"2023-09-23T20:54:44.034729+0000","last_became_peered":"2023-09-23T20:54:44.034729+0000","last_unstale":"2023-09-23T20:55:54.284059+0000","last_undegraded":"2023-09-23T20:55:54.284059+0000","last_fullsized":"2023-09-23T20:55:54.284059+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:35:26.645773+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,5],"acting":[0,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":148,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-09-23T20:55:54.364205+0000","last_change":"2023-09-23T20:50:37.982235+0000","last_active":"2023-09-23T20:55:54.364205+0000","last_peered":"2023-09-23T20:55:54.364205+0000","last_clean":"2023-09-23T20:55:54.364205+0000","last_became_active":"2023-09-23T20:50:37.981913+0000","last_became_peered":"2023-09-23T20:50:37.981913+0000","last_unstale":"2023-09-23T20:55:54.364205+0000","last_undegraded":"2023-09-23T20:55:54.364205+0000","last_fullsized":"2023-09-23T20:55:54.364205+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:59:57.122584+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.4","version":"176'9279","reported_seq":11577,"reported_epoch":176,"state":"active+clean","last_fresh":"2023-09-23T20:55:54.016319+0000","last_change":"2023-09-23T20:55:24.972678+0000","last_active":"2023-09-23T20:55:54.016319+0000","last_peered":"2023-09-23T20:55:54.016319+0000","last_clean":"2023-09-23T20:55:54.016319+0000","last_became_active":"2023-09-23T20:55:24.950897+0000","last_became_peered":"2023-09-23T20:55:24.950897+0000","last_unstale":"2023-09-23T20:55:54.016319+0000","last_undegraded":"2023-09-23T20:55:54.016319+0000","last_fullsized":"2023-09-23T20:55:54.016319+0000","mapping_epoch":160,"log_start":"107'600","ondisk_log_start":"107'600","created":102,"last_epoch_clean":161,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:52:13.843048+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:52:13.843048+0000","last_clean_scrub_stamp":"2023-09-23T20:52:13.843048+0000","objects_scrubbed":0,"log_size":8679,"log_dups_size":0,"ondisk_log_size":8679,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":25264128,"num_objects":385,"num_object_clones":0,"num_object_copies":1155,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":385,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5909,"num_write_kb":201440,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1088,"num_bytes_recovered":71303168,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,3],"acting":[1,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"79'161","reported_seq":547,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-09-23T20:55:54.927017+0000","last_change":"2023-09-23T20:55:35.953634+0000","last_active":"2023-09-23T20:55:54.927017+0000","last_peered":"2023-09-23T20:55:54.927017+0000","last_clean":"2023-09-23T20:55:54.927017+0000","last_became_active":"2023-09-23T20:55:35.953387+0000","last_became_peered":"2023-09-23T20:55:35.953387+0000","last_unstale":"2023-09-23T20:55:54.927017+0000","last_undegraded":"2023-09-23T20:55:54.927017+0000","last_fullsized":"2023-09-23T20:55:54.927017+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:38:57.324224+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":45,"num_read_kb":84,"num_write":22,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":161009,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":163,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-09-23T20:55:54.927261+0000","last_change":"2023-09-23T20:54:13.937566+0000","last_active":"2023-09-23T20:55:54.927261+0000","last_peered":"2023-09-23T20:55:54.927261+0000","last_clean":"2023-09-23T20:55:54.927261+0000","last_became_active":"2023-09-23T20:54:13.937107+0000","last_became_peered":"2023-09-23T20:54:13.937107+0000","last_unstale":"2023-09-23T20:55:54.927261+0000","last_undegraded":"2023-09-23T20:55:54.927261+0000","last_fullsized":"2023-09-23T20:55:54.927261+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:20:26.511292+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":144,"reported_epoch":175,"state":"active+clean","last_fresh":"2023-09-23T20:55:51.234519+0000","last_change":"2023-09-23T20:54:31.898965+0000","last_active":"2023-09-23T20:55:51.234519+0000","last_peered":"2023-09-23T20:55:51.234519+0000","last_clean":"2023-09-23T20:55:51.234519+0000","last_became_active":"2023-09-23T20:54:31.898663+0000","last_became_peered":"2023-09-23T20:54:31.898663+0000","last_unstale":"2023-09-23T20:55:51.234519+0000","last_undegraded":"2023-09-23T20:55:51.234519+0000","last_fullsized":"2023-09-23T20:55:51.234519+0000","mapping_epoch":136,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:53:05.058087+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":137,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-09-23T20:55:54.284150+0000","last_change":"2023-09-23T20:54:31.884717+0000","last_active":"2023-09-23T20:55:54.284150+0000","last_peered":"2023-09-23T20:55:54.284150+0000","last_clean":"2023-09-23T20:55:54.284150+0000","last_became_active":"2023-09-23T20:54:31.884345+0000","last_became_peered":"2023-09-23T20:54:31.884345+0000","last_unstale":"2023-09-23T20:55:54.284150+0000","last_undegraded":"2023-09-23T20:55:54.284150+0000","last_fullsized":"2023-09-23T20:55:54.284150+0000","mapping_epoch":136,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:02:26.393868+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.1","version":"176'9447","reported_seq":10404,"reported_epoch":176,"state":"active+clean","last_fresh":"2023-09-23T20:55:54.061178+0000","last_change":"2023-09-23T20:55:50.242453+0000","last_active":"2023-09-23T20:55:54.061178+0000","last_peered":"2023-09-23T20:55:54.061178+0000","last_clean":"2023-09-23T20:55:54.061178+0000","last_became_active":"2023-09-23T20:55:50.214075+0000","last_became_peered":"2023-09-23T20:55:50.214075+0000","last_unstale":"2023-09-23T20:55:54.061178+0000","last_undegraded":"2023-09-23T20:55:54.061178+0000","last_fullsized":"2023-09-23T20:55:54.061178+0000","mapping_epoch":173,"log_start":"107'700","ondisk_log_start":"107'700","created":102,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:52:13.843048+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:52:13.843048+0000","last_clean_scrub_stamp":"2023-09-23T20:52:13.843048+0000","objects_scrubbed":0,"log_size":8747,"log_dups_size":0,"ondisk_log_size":8747,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":23756800,"num_objects":362,"num_object_clones":0,"num_object_copies":1086,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":362,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6032,"num_write_kb":204640,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":418,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"79'161","reported_seq":547,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-09-23T20:55:54.927063+0000","last_change":"2023-09-23T20:55:35.981211+0000","last_active":"2023-09-23T20:55:54.927063+0000","last_peered":"2023-09-23T20:55:54.927063+0000","last_clean":"2023-09-23T20:55:54.927063+0000","last_became_active":"2023-09-23T20:55:35.980338+0000","last_became_peered":"2023-09-23T20:55:35.980338+0000","last_unstale":"2023-09-23T20:55:54.927063+0000","last_undegraded":"2023-09-23T20:55:54.927063+0000","last_fullsized":"2023-09-23T20:55:54.927063+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:38:57.324224+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":46,"num_read_kb":84,"num_write":22,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":161010,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":138,"reported_epoch":175,"state":"active+clean","last_fresh":"2023-09-23T20:55:51.230448+0000","last_change":"2023-09-23T20:51:10.836910+0000","last_active":"2023-09-23T20:55:51.230448+0000","last_peered":"2023-09-23T20:55:51.230448+0000","last_clean":"2023-09-23T20:55:51.230448+0000","last_became_active":"2023-09-23T20:51:10.836557+0000","last_became_peered":"2023-09-23T20:51:10.836557+0000","last_unstale":"2023-09-23T20:55:51.230448+0000","last_undegraded":"2023-09-23T20:55:51.230448+0000","last_fullsized":"2023-09-23T20:55:51.230448+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:51:54.019969+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":144,"reported_epoch":175,"state":"active+clean","last_fresh":"2023-09-23T20:55:51.235063+0000","last_change":"2023-09-23T20:55:35.953444+0000","last_active":"2023-09-23T20:55:51.235063+0000","last_peered":"2023-09-23T20:55:51.235063+0000","last_clean":"2023-09-23T20:55:51.235063+0000","last_became_active":"2023-09-23T20:55:35.953151+0000","last_became_peered":"2023-09-23T20:55:35.953151+0000","last_unstale":"2023-09-23T20:55:51.235063+0000","last_undegraded":"2023-09-23T20:55:51.235063+0000","last_fullsized":"2023-09-23T20:55:51.235063+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:39:12.404391+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":96,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-09-23T20:55:54.926925+0000","last_change":"2023-09-23T20:54:44.035747+0000","last_active":"2023-09-23T20:55:54.926925+0000","last_peered":"2023-09-23T20:55:54.926925+0000","last_clean":"2023-09-23T20:55:54.926925+0000","last_became_active":"2023-09-23T20:54:44.035433+0000","last_became_peered":"2023-09-23T20:54:44.035433+0000","last_unstale":"2023-09-23T20:55:54.926925+0000","last_undegraded":"2023-09-23T20:55:54.926925+0000","last_fullsized":"2023-09-23T20:55:54.926925+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:42:25.184541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"6.2","version":"179'9150","reported_seq":11392,"reported_epoch":179,"state":"active+clean","last_fresh":"2023-09-23T20:55:57.006325+0000","last_change":"2023-09-23T20:55:40.629163+0000","last_active":"2023-09-23T20:55:57.006325+0000","last_peered":"2023-09-23T20:55:57.006325+0000","last_clean":"2023-09-23T20:55:57.006325+0000","last_became_active":"2023-09-23T20:54:37.961980+0000","last_became_peered":"2023-09-23T20:54:37.961980+0000","last_unstale":"2023-09-23T20:55:57.006325+0000","last_undegraded":"2023-09-23T20:55:57.006325+0000","last_fullsized":"2023-09-23T20:55:57.006325+0000","mapping_epoch":139,"log_start":"0'0","ondisk_log_start":"0'0","created":102,"last_epoch_clean":140,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:52:13.843048+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:52:13.843048+0000","last_clean_scrub_stamp":"2023-09-23T20:52:13.843048+0000","objects_scrubbed":0,"log_size":9150,"log_dups_size":0,"ondisk_log_size":9150,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":23789568,"num_objects":363,"num_object_clones":0,"num_object_copies":1089,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":363,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5828,"num_write_kb":198112,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1055,"num_bytes_recovered":39518208,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.4","version":"79'161","reported_seq":547,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-09-23T20:55:54.927642+0000","last_change":"2023-09-23T20:55:35.947560+0000","last_active":"2023-09-23T20:55:54.927642+0000","last_peered":"2023-09-23T20:55:54.927642+0000","last_clean":"2023-09-23T20:55:54.927642+0000","last_became_active":"2023-09-23T20:55:35.947180+0000","last_became_peered":"2023-09-23T20:55:35.947180+0000","last_unstale":"2023-09-23T20:55:54.927642+0000","last_undegraded":"2023-09-23T20:55:54.927642+0000","last_fullsized":"2023-09-23T20:55:54.927642+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:38:57.324224+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":46,"num_read_kb":84,"num_write":22,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":161010,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":125,"reported_epoch":175,"state":"active+clean","last_fresh":"2023-09-23T20:55:51.230412+0000","last_change":"2023-09-23T20:55:35.949541+0000","last_active":"2023-09-23T20:55:51.230412+0000","last_peered":"2023-09-23T20:55:51.230412+0000","last_clean":"2023-09-23T20:55:51.230412+0000","last_became_active":"2023-09-23T20:55:35.944635+0000","last_became_peered":"2023-09-23T20:55:35.944635+0000","last_unstale":"2023-09-23T20:55:51.230412+0000","last_undegraded":"2023-09-23T20:55:51.230412+0000","last_fullsized":"2023-09-23T20:55:51.230412+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:30:36.033585+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"70'1","reported_seq":164,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-09-23T20:55:54.364147+0000","last_change":"2023-09-23T20:54:14.009723+0000","last_active":"2023-09-23T20:55:54.364147+0000","last_peered":"2023-09-23T20:55:54.364147+0000","last_clean":"2023-09-23T20:55:54.364147+0000","last_became_active":"2023-09-23T20:54:13.941771+0000","last_became_peered":"2023-09-23T20:54:13.941771+0000","last_unstale":"2023-09-23T20:55:54.364147+0000","last_undegraded":"2023-09-23T20:55:54.364147+0000","last_fullsized":"2023-09-23T20:55:54.364147+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:48:25.588116+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":83,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-09-23T20:55:54.925038+0000","last_change":"2023-09-23T20:55:35.949780+0000","last_active":"2023-09-23T20:55:54.925038+0000","last_peered":"2023-09-23T20:55:54.925038+0000","last_clean":"2023-09-23T20:55:54.925038+0000","last_became_active":"2023-09-23T20:55:35.949497+0000","last_became_peered":"2023-09-23T20:55:35.949497+0000","last_unstale":"2023-09-23T20:55:54.925038+0000","last_undegraded":"2023-09-23T20:55:54.925038+0000","last_fullsized":"2023-09-23T20:55:54.925038+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:09:37.420882+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.3","version":"179'9262","reported_seq":10189,"reported_epoch":179,"state":"active+clean","last_fresh":"2023-09-23T20:55:57.121661+0000","last_change":"2023-09-23T20:55:45.992553+0000","last_active":"2023-09-23T20:55:57.121661+0000","last_peered":"2023-09-23T20:55:57.121661+0000","last_clean":"2023-09-23T20:55:57.121661+0000","last_became_active":"2023-09-23T20:55:45.933467+0000","last_became_peered":"2023-09-23T20:55:45.933467+0000","last_unstale":"2023-09-23T20:55:57.121661+0000","last_undegraded":"2023-09-23T20:55:57.121661+0000","last_fullsized":"2023-09-23T20:55:57.121661+0000","mapping_epoch":169,"log_start":"107'482","ondisk_log_start":"107'482","created":102,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"152'8320","last_scrub_stamp":"2023-09-23T20:55:10.620678+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:52:13.843048+0000","last_clean_scrub_stamp":"2023-09-23T20:55:10.620678+0000","objects_scrubbed":1283,"log_size":8780,"log_dups_size":0,"ondisk_log_size":8780,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:32:12.962603+0000","scrub_duration":0.432808627,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":21692416,"num_objects":331,"num_object_clones":0,"num_object_copies":993,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":331,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":952,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":367,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,2],"acting":[7,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":139,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-09-23T20:55:54.284351+0000","last_change":"2023-09-23T20:54:13.929196+0000","last_active":"2023-09-23T20:55:54.284351+0000","last_peered":"2023-09-23T20:55:54.284351+0000","last_clean":"2023-09-23T20:55:54.284351+0000","last_became_active":"2023-09-23T20:54:13.928831+0000","last_became_peered":"2023-09-23T20:54:13.928831+0000","last_unstale":"2023-09-23T20:55:54.284351+0000","last_undegraded":"2023-09-23T20:55:54.284351+0000","last_fullsized":"2023-09-23T20:55:54.284351+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:28:44.316851+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":146,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-09-23T20:55:54.931006+0000","last_change":"2023-09-23T20:52:12.855430+0000","last_active":"2023-09-23T20:55:54.931006+0000","last_peered":"2023-09-23T20:55:54.931006+0000","last_clean":"2023-09-23T20:55:54.931006+0000","last_became_active":"2023-09-23T20:52:12.855115+0000","last_became_peered":"2023-09-23T20:52:12.855115+0000","last_unstale":"2023-09-23T20:55:54.931006+0000","last_undegraded":"2023-09-23T20:55:54.931006+0000","last_fullsized":"2023-09-23T20:55:54.931006+0000","mapping_epoch":100,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":101,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:33:23.802214+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"6.19","version":"178'9408","reported_seq":9533,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-09-23T20:55:55.507780+0000","last_change":"2023-09-23T20:55:31.410310+0000","last_active":"2023-09-23T20:55:55.507780+0000","last_peered":"2023-09-23T20:55:55.507780+0000","last_clean":"2023-09-23T20:55:55.507780+0000","last_became_active":"2023-09-23T20:54:03.763084+0000","last_became_peered":"2023-09-23T20:54:03.763084+0000","last_unstale":"2023-09-23T20:55:55.507780+0000","last_undegraded":"2023-09-23T20:55:55.507780+0000","last_fullsized":"2023-09-23T20:55:55.507780+0000","mapping_epoch":128,"log_start":"0'0","ondisk_log_start":"0'0","created":128,"last_epoch_clean":129,"parent":"0.0","parent_split_bits":5,"last_scrub":"162'8912","last_scrub_stamp":"2023-09-23T20:55:31.410071+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:52:13.843048+0000","last_clean_scrub_stamp":"2023-09-23T20:55:31.410071+0000","objects_scrubbed":863,"log_size":9408,"log_dups_size":0,"ondisk_log_size":9408,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:20:28.379233+0000","scrub_duration":0.32384137200000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":23527424,"num_objects":359,"num_object_clones":0,"num_object_copies":1077,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":359,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":504,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,2],"acting":[0,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":56,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-09-23T20:55:54.978244+0000","last_change":"2023-09-23T20:54:31.902946+0000","last_active":"2023-09-23T20:55:54.978244+0000","last_peered":"2023-09-23T20:55:54.978244+0000","last_clean":"2023-09-23T20:55:54.978244+0000","last_became_active":"2023-09-23T20:54:31.889484+0000","last_became_peered":"2023-09-23T20:54:31.889484+0000","last_unstale":"2023-09-23T20:55:54.978244+0000","last_undegraded":"2023-09-23T20:55:54.978244+0000","last_fullsized":"2023-09-23T20:55:54.978244+0000","mapping_epoch":136,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:44:44.278398+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":56,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-09-23T20:55:54.925669+0000","last_change":"2023-09-23T20:54:13.950351+0000","last_active":"2023-09-23T20:55:54.925669+0000","last_peered":"2023-09-23T20:55:54.925669+0000","last_clean":"2023-09-23T20:55:54.925669+0000","last_became_active":"2023-09-23T20:54:13.949987+0000","last_became_peered":"2023-09-23T20:54:13.949987+0000","last_unstale":"2023-09-23T20:55:54.925669+0000","last_undegraded":"2023-09-23T20:55:54.925669+0000","last_fullsized":"2023-09-23T20:55:54.925669+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:46:17.765134+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.18","version":"179'9391","reported_seq":9502,"reported_epoch":179,"state":"active+clean","last_fresh":"2023-09-23T20:55:57.138734+0000","last_change":"2023-09-23T20:55:35.096729+0000","last_active":"2023-09-23T20:55:57.138734+0000","last_peered":"2023-09-23T20:55:57.138734+0000","last_clean":"2023-09-23T20:55:57.138734+0000","last_became_active":"2023-09-23T20:54:03.787013+0000","last_became_peered":"2023-09-23T20:54:03.787013+0000","last_unstale":"2023-09-23T20:55:57.138734+0000","last_undegraded":"2023-09-23T20:55:57.138734+0000","last_fullsized":"2023-09-23T20:55:57.138734+0000","mapping_epoch":128,"log_start":"107'764","ondisk_log_start":"107'764","created":128,"last_epoch_clean":129,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:52:13.843048+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:52:13.843048+0000","last_clean_scrub_stamp":"2023-09-23T20:52:13.843048+0000","objects_scrubbed":0,"log_size":8627,"log_dups_size":0,"ondisk_log_size":8627,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":24838144,"num_objects":379,"num_object_clones":0,"num_object_copies":1137,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":379,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5980,"num_write_kb":203488,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,3],"acting":[7,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":73,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-09-23T20:55:54.975565+0000","last_change":"2023-09-23T20:55:35.978731+0000","last_active":"2023-09-23T20:55:54.975565+0000","last_peered":"2023-09-23T20:55:54.975565+0000","last_clean":"2023-09-23T20:55:54.975565+0000","last_became_active":"2023-09-23T20:55:35.978268+0000","last_became_peered":"2023-09-23T20:55:35.978268+0000","last_unstale":"2023-09-23T20:55:54.975565+0000","last_undegraded":"2023-09-23T20:55:54.975565+0000","last_fullsized":"2023-09-23T20:55:54.975565+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:37:21.317686+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":56,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-09-23T20:55:54.925941+0000","last_change":"2023-09-23T20:54:13.935307+0000","last_active":"2023-09-23T20:55:54.925941+0000","last_peered":"2023-09-23T20:55:54.925941+0000","last_clean":"2023-09-23T20:55:54.925941+0000","last_became_active":"2023-09-23T20:54:13.935000+0000","last_became_peered":"2023-09-23T20:54:13.935000+0000","last_unstale":"2023-09-23T20:55:54.925941+0000","last_undegraded":"2023-09-23T20:55:54.925941+0000","last_fullsized":"2023-09-23T20:55:54.925941+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:47:40.974082+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":161,"reported_epoch":175,"state":"active+clean","last_fresh":"2023-09-23T20:55:51.230097+0000","last_change":"2023-09-23T20:55:35.954870+0000","last_active":"2023-09-23T20:55:51.230097+0000","last_peered":"2023-09-23T20:55:51.230097+0000","last_clean":"2023-09-23T20:55:51.230097+0000","last_became_active":"2023-09-23T20:55:35.954568+0000","last_became_peered":"2023-09-23T20:55:35.954568+0000","last_unstale":"2023-09-23T20:55:51.230097+0000","last_undegraded":"2023-09-23T20:55:51.230097+0000","last_fullsized":"2023-09-23T20:55:51.230097+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:06:22.423566+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":160,"reported_epoch":175,"state":"active+clean","last_fresh":"2023-09-23T20:55:51.229891+0000","last_change":"2023-09-23T20:51:10.838334+0000","last_active":"2023-09-23T20:55:51.229891+0000","last_peered":"2023-09-23T20:55:51.229891+0000","last_clean":"2023-09-23T20:55:51.229891+0000","last_became_active":"2023-09-23T20:51:10.838026+0000","last_became_peered":"2023-09-23T20:51:10.838026+0000","last_unstale":"2023-09-23T20:55:51.229891+0000","last_undegraded":"2023-09-23T20:55:51.229891+0000","last_fullsized":"2023-09-23T20:55:51.229891+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:04:37.009457+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":167,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-09-23T20:55:54.364061+0000","last_change":"2023-09-23T20:54:31.903139+0000","last_active":"2023-09-23T20:55:54.364061+0000","last_peered":"2023-09-23T20:55:54.364061+0000","last_clean":"2023-09-23T20:55:54.364061+0000","last_became_active":"2023-09-23T20:54:31.884520+0000","last_became_peered":"2023-09-23T20:54:31.884520+0000","last_unstale":"2023-09-23T20:55:54.364061+0000","last_undegraded":"2023-09-23T20:55:54.364061+0000","last_fullsized":"2023-09-23T20:55:54.364061+0000","mapping_epoch":136,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:39:34.269705+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1108,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":73,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-09-23T20:55:54.924671+0000","last_change":"2023-09-23T20:55:35.951572+0000","last_active":"2023-09-23T20:55:54.924671+0000","last_peered":"2023-09-23T20:55:54.924671+0000","last_clean":"2023-09-23T20:55:54.924671+0000","last_became_active":"2023-09-23T20:55:35.951294+0000","last_became_peered":"2023-09-23T20:55:35.951294+0000","last_unstale":"2023-09-23T20:55:54.924671+0000","last_undegraded":"2023-09-23T20:55:54.924671+0000","last_fullsized":"2023-09-23T20:55:54.924671+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:36:32.447210+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":6,"num_pg":26,"stat_sum":{"num_bytes":761430040,"num_objects":11620,"num_object_clones":0,"num_object_copies":34860,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":11620,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":174610,"num_write_kb":5860673,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":28623,"num_bytes_recovered":1327169560,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2855944192,"data_stored":2855927904,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":224054,"ondisk_log_size":224054,"up":78,"acting":78,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":30,"num_bytes_recovered":6296,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":11,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1771,"ondisk_log_size":1771,"up":33,"acting":33,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201466,"num_pgs":54,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":545064,"kb_used_data":421216,"kb_used_omap":0,"kb_used_meta":123840,"kb_avail":93200088,"statfs":{"total":95995035648,"available":95436890112,"internally_reserved":0,"allocated":431325184,"data_stored":430654729,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":126812160},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:55:41 2023","interfaces":[{"interface":"back","average":{"1min":1.1479999999999999,"5min":1.0900000000000001,"15min":0.68999999999999995},"min":{"1min":0.21099999999999999,"5min":0.21099999999999999,"15min":0.21099999999999999},"max":{"1min":5.694,"5min":5.8760000000000003,"15min":5.8760000000000003},"last":0.45800000000000002},{"interface":"front","average":{"1min":1.6539999999999999,"5min":1.2509999999999999,"15min":0.79200000000000004},"min":{"1min":0.31900000000000001,"5min":0.218,"15min":0.218},"max":{"1min":5.7060000000000004,"5min":8.7690000000000001,"15min":8.7690000000000001},"last":0.40999999999999998}]},{"osd":1,"last update":"Sat Sep 23 20:55:12 2023","interfaces":[{"interface":"back","average":{"1min":2.3359999999999999,"5min":1.6439999999999999,"15min":0.91400000000000003},"min":{"1min":0.26900000000000002,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":8.1999999999999993,"5min":22.591999999999999,"15min":22.591999999999999},"last":0.41699999999999998},{"interface":"front","average":{"1min":1.4650000000000001,"5min":1.26,"15min":0.78200000000000003},"min":{"1min":0.249,"5min":0.249,"15min":0.249},"max":{"1min":6.6950000000000003,"5min":21.437000000000001,"15min":21.437000000000001},"last":0.42699999999999999}]},{"osd":2,"last update":"Sat Sep 23 20:55:47 2023","interfaces":[{"interface":"back","average":{"1min":2.5310000000000001,"5min":1.528,"15min":0.876},"min":{"1min":0.32400000000000001,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":9.4290000000000003,"5min":9.4290000000000003,"15min":9.4290000000000003},"last":0.44800000000000001},{"interface":"front","average":{"1min":2.2999999999999998,"5min":1.4239999999999999,"15min":0.86399999999999999},"min":{"1min":0.32100000000000001,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":9.0030000000000001,"5min":9.0030000000000001,"15min":9.0030000000000001},"last":0.36099999999999999}]},{"osd":3,"last update":"Sat Sep 23 20:55:20 2023","interfaces":[{"interface":"back","average":{"1min":3.6629999999999998,"5min":2.7269999999999999,"15min":1.282},"min":{"1min":0.40100000000000002,"5min":0.314,"15min":0.314},"max":{"1min":13.007,"5min":36.408000000000001,"15min":36.408000000000001},"last":0.38300000000000001},{"interface":"front","average":{"1min":2.5179999999999998,"5min":2.1619999999999999,"15min":1.103},"min":{"1min":0.36099999999999999,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":7.1130000000000004,"5min":35.593000000000004,"15min":35.593000000000004},"last":0.35099999999999998}]},{"osd":4,"last update":"Sat Sep 23 20:55:08 2023","interfaces":[{"interface":"back","average":{"1min":1.2809999999999999,"5min":0.71099999999999997,"15min":0.55600000000000005},"min":{"1min":0.30499999999999999,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":5.8920000000000003,"5min":5.8920000000000003,"15min":5.8920000000000003},"last":0.311},{"interface":"front","average":{"1min":1.1040000000000001,"5min":0.67600000000000005,"15min":0.56999999999999995},"min":{"1min":0.30099999999999999,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":5.3200000000000003,"5min":5.3200000000000003,"15min":5.3200000000000003},"last":0.29299999999999998}]},{"osd":5,"last update":"Sat Sep 23 20:55:08 2023","interfaces":[{"interface":"back","average":{"1min":1.018,"5min":0.68799999999999994,"15min":0.57999999999999996},"min":{"1min":0.308,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":5.258,"5min":5.258,"15min":5.258},"last":0.33500000000000002},{"interface":"front","average":{"1min":0.72299999999999998,"5min":0.61499999999999999,"15min":0.56799999999999995},"min":{"1min":0.27800000000000002,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":3.3530000000000002,"5min":3.3530000000000002,"15min":3.3530000000000002},"last":0.39500000000000002}]},{"osd":6,"last update":"Sat Sep 23 20:55:08 2023","interfaces":[{"interface":"back","average":{"1min":1.079,"5min":0.70399999999999996,"15min":0.61199999999999999},"min":{"1min":0.317,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":5.3399999999999999,"5min":5.3399999999999999,"15min":5.3399999999999999},"last":0.32600000000000001},{"interface":"front","average":{"1min":1.1890000000000001,"5min":0.747,"15min":0.63400000000000001},"min":{"1min":0.32600000000000001,"5min":0.32600000000000001,"15min":0.316},"max":{"1min":5.1580000000000004,"5min":5.1580000000000004,"15min":5.1580000000000004},"last":0.372}]}]},{"osd":6,"up_from":164,"seq":704374636549,"num_pgs":49,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":422016,"kb_used_data":369912,"kb_used_omap":10,"kb_used_meta":52085,"kb_avail":93323136,"statfs":{"total":95995035648,"available":95562891264,"internally_reserved":0,"allocated":378789888,"data_stored":378135659,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":10328,"internal_metadata":53335976},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":3.7629999999999999}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.27700000000000002}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":2.415}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.93700000000000006}]},{"osd":4,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":9.0079999999999991}]},{"osd":5,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.72999999999999998}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":3.847}]}]},{"osd":1,"up_from":88,"seq":377957122120,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":450972,"kb_used_data":333268,"kb_used_omap":9,"kb_used_meta":117686,"kb_avail":93294180,"statfs":{"total":95995035648,"available":95533240320,"internally_reserved":0,"allocated":341266432,"data_stored":340412303,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9239,"internal_metadata":120511465},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:55:12 2023","interfaces":[{"interface":"back","average":{"1min":3.2850000000000001,"5min":1.677,"15min":0.83099999999999996},"min":{"1min":0.216,"5min":0.216,"15min":0.216},"max":{"1min":14.372,"5min":14.372,"15min":14.372},"last":2.7519999999999998},{"interface":"front","average":{"1min":3.5369999999999999,"5min":1.8959999999999999,"15min":0.92200000000000004},"min":{"1min":0.249,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":11.266,"5min":18.457999999999998,"15min":18.457999999999998},"last":2.2530000000000001}]},{"osd":2,"last update":"Sat Sep 23 20:55:46 2023","interfaces":[{"interface":"back","average":{"1min":2.9729999999999999,"5min":1.768,"15min":0.84699999999999998},"min":{"1min":0.28999999999999998,"5min":0.20300000000000001,"15min":0.20300000000000001},"max":{"1min":14.335000000000001,"5min":14.335000000000001,"15min":14.335000000000001},"last":2.823},{"interface":"front","average":{"1min":2.3330000000000002,"5min":2.2589999999999999,"15min":1.143},"min":{"1min":0.248,"5min":0.248,"15min":0.248},"max":{"1min":7.7199999999999998,"5min":38.951000000000001,"15min":38.951000000000001},"last":2.585}]},{"osd":3,"last update":"Sat Sep 23 20:55:17 2023","interfaces":[{"interface":"back","average":{"1min":2.9199999999999999,"5min":2.2010000000000001,"15min":0.97299999999999998},"min":{"1min":0.25600000000000001,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":7.3970000000000002,"5min":25.314,"15min":25.314},"last":5.7140000000000004},{"interface":"front","average":{"1min":2.8559999999999999,"5min":1.8260000000000001,"15min":0.88200000000000001},"min":{"1min":0.33400000000000002,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":14.225,"5min":19.881,"15min":19.881},"last":6.5410000000000004}]},{"osd":4,"last update":"Sat Sep 23 20:55:12 2023","interfaces":[{"interface":"back","average":{"1min":2.0830000000000002,"5min":1.6359999999999999,"15min":0.97599999999999998},"min":{"1min":0.28499999999999998,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":6.6630000000000003,"5min":12.006,"15min":12.006},"last":4.444},{"interface":"front","average":{"1min":2.2290000000000001,"5min":1.2030000000000001,"15min":0.84899999999999998},"min":{"1min":0.32500000000000001,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":7.3079999999999998,"5min":11.234999999999999,"15min":11.234999999999999},"last":2.7949999999999999}]},{"osd":5,"last update":"Sat Sep 23 20:55:12 2023","interfaces":[{"interface":"back","average":{"1min":1.843,"5min":1.2609999999999999,"15min":0.88500000000000001},"min":{"1min":0.36199999999999999,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":5.577,"5min":12.222,"15min":12.222},"last":6.3970000000000002},{"interface":"front","average":{"1min":2.3879999999999999,"5min":1.6639999999999999,"15min":0.98599999999999999},"min":{"1min":0.32200000000000001,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":6.702,"5min":12.08,"15min":12.08},"last":2.8839999999999999}]},{"osd":6,"last update":"Sat Sep 23 20:55:12 2023","interfaces":[{"interface":"back","average":{"1min":1.8440000000000001,"5min":1.3400000000000001,"15min":0.90500000000000003},"min":{"1min":0.39300000000000002,"5min":0.307,"15min":0.307},"max":{"1min":5.5389999999999997,"5min":12.276999999999999,"15min":12.276999999999999},"last":2.919},{"interface":"front","average":{"1min":2.5110000000000001,"5min":1.726,"15min":1.014},"min":{"1min":0.34499999999999997,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":6.7439999999999998,"5min":12.054,"15min":12.054},"last":6.5670000000000002}]},{"osd":7,"last update":"Sat Sep 23 20:55:12 2023","interfaces":[{"interface":"back","average":{"1min":2.2549999999999999,"5min":1.262,"15min":0.87},"min":{"1min":0.42699999999999999,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":7.3440000000000003,"5min":11.186999999999999,"15min":11.186999999999999},"last":6.4699999999999998},{"interface":"front","average":{"1min":1.833,"5min":1.3160000000000001,"15min":0.89600000000000002},"min":{"1min":0.48099999999999998,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":5.609,"5min":12.252000000000001,"15min":12.252000000000001},"last":0.42999999999999999}]}]},{"osd":0,"up_from":83,"seq":356482285647,"num_pgs":42,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":278392,"kb_used_data":192624,"kb_used_omap":28,"kb_used_meta":85731,"kb_avail":93466760,"statfs":{"total":95995035648,"available":95709962240,"internally_reserved":0,"allocated":197246976,"data_stored":196382019,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":28848,"internal_metadata":87789392},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 20:55:13 2023","interfaces":[{"interface":"back","average":{"1min":3.4929999999999999,"5min":1.6339999999999999,"15min":0.76600000000000001},"min":{"1min":0.22700000000000001,"5min":0.20100000000000001,"15min":0.20100000000000001},"max":{"1min":16.484999999999999,"5min":18.863,"15min":18.863},"last":3.7290000000000001},{"interface":"front","average":{"1min":2.2130000000000001,"5min":1.5620000000000001,"15min":0.77800000000000002},"min":{"1min":0.26300000000000001,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":13.013999999999999,"5min":17.704999999999998,"15min":17.704999999999998},"last":10.156000000000001}]},{"osd":2,"last update":"Sat Sep 23 20:55:43 2023","interfaces":[{"interface":"back","average":{"1min":2.004,"5min":2.0870000000000002,"15min":0.97199999999999998},"min":{"1min":0.28299999999999997,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":7.4569999999999999,"5min":17.753,"15min":17.753},"last":3.891},{"interface":"front","average":{"1min":2.6619999999999999,"5min":2.1509999999999998,"15min":0.97699999999999998},"min":{"1min":0.26800000000000002,"5min":0.21099999999999999,"15min":0.21099999999999999},"max":{"1min":9.5440000000000005,"5min":13.218999999999999,"15min":13.218999999999999},"last":6.2750000000000004}]},{"osd":3,"last update":"Sat Sep 23 20:55:13 2023","interfaces":[{"interface":"back","average":{"1min":2.4809999999999999,"5min":1.7270000000000001,"15min":0.84299999999999997},"min":{"1min":0.32400000000000001,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":9.7370000000000001,"5min":18.893999999999998,"15min":18.893999999999998},"last":0.27500000000000002},{"interface":"front","average":{"1min":2.621,"5min":1.7829999999999999,"15min":0.91300000000000003},"min":{"1min":0.40999999999999998,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":16.977,"5min":17.472000000000001,"15min":17.472000000000001},"last":4.9039999999999999}]},{"osd":4,"last update":"Sat Sep 23 20:55:43 2023","interfaces":[{"interface":"back","average":{"1min":2.2610000000000001,"5min":1.5289999999999999,"15min":0.94099999999999995},"min":{"1min":0.32000000000000001,"5min":0.311,"15min":0.311},"max":{"1min":7.4710000000000001,"5min":12.807,"15min":12.807},"last":6.3109999999999999},{"interface":"front","average":{"1min":1.6930000000000001,"5min":1.3979999999999999,"15min":0.90500000000000003},"min":{"1min":0.41199999999999998,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":7.1790000000000003,"5min":18.724,"15min":18.724},"last":0.31}]},{"osd":5,"last update":"Sat Sep 23 20:55:43 2023","interfaces":[{"interface":"back","average":{"1min":1.5680000000000001,"5min":1.421,"15min":0.90600000000000003},"min":{"1min":0.35799999999999998,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":6.0629999999999997,"5min":8.4559999999999995,"15min":8.4559999999999995},"last":3.827},{"interface":"front","average":{"1min":2.6360000000000001,"5min":1.6439999999999999,"15min":1.0189999999999999},"min":{"1min":0.34999999999999998,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":7.4139999999999997,"5min":13.154999999999999,"15min":13.154999999999999},"last":6.3390000000000004}]},{"osd":6,"last update":"Sat Sep 23 20:54:43 2023","interfaces":[{"interface":"back","average":{"1min":2.492,"5min":1.2370000000000001,"15min":0.873},"min":{"1min":0.35899999999999999,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":13.19,"5min":13.19,"15min":13.19},"last":0.52200000000000002},{"interface":"front","average":{"1min":2.718,"5min":1.202,"15min":0.85399999999999998},"min":{"1min":0.39600000000000002,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":18.829000000000001,"5min":18.829000000000001,"15min":18.829000000000001},"last":0.55000000000000004}]},{"osd":7,"last update":"Sat Sep 23 20:55:43 2023","interfaces":[{"interface":"back","average":{"1min":1.593,"5min":1.4159999999999999,"15min":0.93799999999999994},"min":{"1min":0.39400000000000002,"5min":0.38400000000000001,"15min":0.38400000000000001},"max":{"1min":6.008,"5min":8.3979999999999997,"15min":8.3979999999999997},"last":3.8580000000000001},{"interface":"front","average":{"1min":1.7150000000000001,"5min":1.391,"15min":0.91300000000000003},"min":{"1min":0.46000000000000002,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":7.1070000000000002,"5min":18.673999999999999,"15min":18.673999999999999},"last":0.498}]}]},{"osd":2,"up_from":93,"seq":399431958594,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":747300,"kb_used_data":585180,"kb_used_omap":81,"kb_used_meta":162030,"kb_avail":92997852,"statfs":{"total":95995035648,"available":95229800448,"internally_reserved":0,"allocated":599224320,"data_stored":598382534,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":83750,"internal_metadata":165918938},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[5],"upper_bound":2},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:55:41 2023","interfaces":[{"interface":"back","average":{"1min":1.9490000000000001,"5min":1.4019999999999999,"15min":0.70599999999999996},"min":{"1min":0.26600000000000001,"5min":0.20499999999999999,"15min":0.20499999999999999},"max":{"1min":9.5129999999999999,"5min":11.183,"15min":11.183},"last":0.36199999999999999},{"interface":"front","average":{"1min":2.5579999999999998,"5min":1.708,"15min":0.83999999999999997},"min":{"1min":0.20999999999999999,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":12.641,"5min":12.641,"15min":12.641},"last":6.9829999999999997}]},{"osd":1,"last update":"Sat Sep 23 20:55:41 2023","interfaces":[{"interface":"back","average":{"1min":2.2679999999999998,"5min":2.2679999999999998,"15min":1.034},"min":{"1min":0.20200000000000001,"5min":0.20200000000000001,"15min":0.20200000000000001},"max":{"1min":12.130000000000001,"5min":16.081,"15min":16.081},"last":3.6459999999999999},{"interface":"front","average":{"1min":2.5870000000000002,"5min":2.665,"15min":1.169},"min":{"1min":0.255,"5min":0.255,"15min":0.255},"max":{"1min":8.1300000000000008,"5min":16.041,"15min":16.041},"last":7.1660000000000004}]},{"osd":3,"last update":"Sat Sep 23 20:55:17 2023","interfaces":[{"interface":"back","average":{"1min":3.774,"5min":2.2799999999999998,"15min":1.036},"min":{"1min":0.29699999999999999,"5min":0.255,"15min":0.255},"max":{"1min":16.611000000000001,"5min":16.611000000000001,"15min":16.611000000000001},"last":7.1390000000000002},{"interface":"front","average":{"1min":5.3120000000000003,"5min":2.9510000000000001,"15min":1.2370000000000001},"min":{"1min":0.22800000000000001,"5min":0.221,"15min":0.221},"max":{"1min":16.094999999999999,"5min":17.138999999999999,"15min":17.138999999999999},"last":7.2080000000000002}]},{"osd":4,"last update":"Sat Sep 23 20:55:41 2023","interfaces":[{"interface":"back","average":{"1min":1.786,"5min":1.462,"15min":0.90000000000000002},"min":{"1min":0.29999999999999999,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":6.8150000000000004,"5min":9.8119999999999994,"15min":9.8119999999999994},"last":7.077},{"interface":"front","average":{"1min":2.8759999999999999,"5min":1.6990000000000001,"15min":1.0129999999999999},"min":{"1min":0.33100000000000002,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":10.609,"5min":10.609,"15min":10.609},"last":1.222}]},{"osd":5,"last update":"Sat Sep 23 20:55:41 2023","interfaces":[{"interface":"back","average":{"1min":2.778,"5min":1.679,"15min":0.99399999999999999},"min":{"1min":0.307,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":9.0329999999999995,"5min":9.7629999999999999,"15min":9.7629999999999999},"last":7.109},{"interface":"front","average":{"1min":3.1269999999999998,"5min":1.8360000000000001,"15min":1.036},"min":{"1min":0.30299999999999999,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":10.574,"5min":10.574,"15min":10.574},"last":1.29}]},{"osd":6,"last update":"Sat Sep 23 20:54:40 2023","interfaces":[{"interface":"back","average":{"1min":2.7040000000000002,"5min":1.3580000000000001,"15min":0.92300000000000004},"min":{"1min":0.38800000000000001,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":8.6609999999999996,"5min":8.6609999999999996,"15min":8.6609999999999996},"last":1.2609999999999999},{"interface":"front","average":{"1min":1.167,"5min":1.048,"15min":0.80600000000000005},"min":{"1min":0.41099999999999998,"5min":0.40100000000000002,"15min":0.40100000000000002},"max":{"1min":3.3639999999999999,"5min":6.431,"15min":6.431},"last":1.319}]},{"osd":7,"last update":"Sat Sep 23 20:55:41 2023","interfaces":[{"interface":"back","average":{"1min":2.0019999999999998,"5min":1.2909999999999999,"15min":0.85299999999999998},"min":{"1min":0.39500000000000002,"5min":0.371,"15min":0.371},"max":{"1min":8.0820000000000007,"5min":8.0820000000000007,"15min":8.0820000000000007},"last":0.39800000000000002},{"interface":"front","average":{"1min":2.1230000000000002,"5min":1.635,"15min":0.98599999999999999},"min":{"1min":0.35899999999999999,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":6.79,"5min":9.7859999999999996,"15min":9.7859999999999996},"last":7.0419999999999998}]}]},{"osd":3,"up_from":98,"seq":420906795067,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":678968,"kb_used_data":516976,"kb_used_omap":7,"kb_used_meta":161976,"kb_avail":93066184,"statfs":{"total":95995035648,"available":95299772416,"internally_reserved":0,"allocated":529383424,"data_stored":528560405,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7576,"internal_metadata":165864040},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:55:20 2023","interfaces":[{"interface":"back","average":{"1min":2.3700000000000001,"5min":2.835,"15min":1.1919999999999999},"min":{"1min":0.313,"5min":0.249,"15min":0.249},"max":{"1min":8.3420000000000005,"5min":23.123999999999999,"15min":23.123999999999999},"last":5.0750000000000002},{"interface":"front","average":{"1min":2.9500000000000002,"5min":1.9279999999999999,"15min":0.88100000000000001},"min":{"1min":0.221,"5min":0.20000000000000001,"15min":0.20000000000000001},"max":{"1min":15.079000000000001,"5min":15.079000000000001,"15min":15.079000000000001},"last":5.1760000000000002}]},{"osd":1,"last update":"Sat Sep 23 20:55:20 2023","interfaces":[{"interface":"back","average":{"1min":3.1040000000000001,"5min":2.194,"15min":0.97599999999999998},"min":{"1min":0.24199999999999999,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":13.683999999999999,"5min":21.385000000000002,"15min":21.385000000000002},"last":1.175},{"interface":"front","average":{"1min":2.6400000000000001,"5min":2.883,"15min":1.22},"min":{"1min":0.28000000000000003,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":10.321,"5min":25.353000000000002,"15min":25.353000000000002},"last":0.218}]},{"osd":2,"last update":"Sat Sep 23 20:55:20 2023","interfaces":[{"interface":"back","average":{"1min":2.8780000000000001,"5min":2.1120000000000001,"15min":0.98699999999999999},"min":{"1min":0.34899999999999998,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":10.051,"5min":24.154,"15min":24.154},"last":0.54900000000000004},{"interface":"front","average":{"1min":2.4239999999999999,"5min":2.855,"15min":1.244},"min":{"1min":0.22600000000000001,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":7.9320000000000004,"5min":25.305,"15min":25.305},"last":0.32600000000000001}]},{"osd":4,"last update":"Sat Sep 23 20:55:20 2023","interfaces":[{"interface":"back","average":{"1min":1.9730000000000001,"5min":1.5049999999999999,"15min":0.92300000000000004},"min":{"1min":0.29299999999999998,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":8.3550000000000004,"5min":11.861000000000001,"15min":11.861000000000001},"last":1.7370000000000001},{"interface":"front","average":{"1min":1.587,"5min":2.2109999999999999,"15min":1.155},"min":{"1min":0.32200000000000001,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":4.8650000000000002,"5min":22.853000000000002,"15min":22.853000000000002},"last":1.119}]},{"osd":5,"last update":"Sat Sep 23 20:55:20 2023","interfaces":[{"interface":"back","average":{"1min":1.8500000000000001,"5min":1.4219999999999999,"15min":0.90400000000000003},"min":{"1min":0.41199999999999998,"5min":0.375,"15min":0.375},"max":{"1min":8.4640000000000004,"5min":11.891999999999999,"15min":11.891999999999999},"last":1.3779999999999999},{"interface":"front","average":{"1min":1.0309999999999999,"5min":1.3080000000000001,"15min":0.85999999999999999},"min":{"1min":0.33100000000000002,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":3.3570000000000002,"5min":9.2870000000000008,"15min":9.2870000000000008},"last":5.258}]},{"osd":6,"last update":"Sat Sep 23 20:54:18 2023","interfaces":[{"interface":"back","average":{"1min":2.2229999999999999,"5min":1.2829999999999999,"15min":0.88800000000000001},"min":{"1min":0.40899999999999997,"5min":0.38100000000000001,"15min":0.38100000000000001},"max":{"1min":9.2360000000000007,"5min":9.2360000000000007,"15min":9.2360000000000007},"last":0.628},{"interface":"front","average":{"1min":2.524,"5min":1.3959999999999999,"15min":0.94399999999999995},"min":{"1min":0.34899999999999998,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":11.824,"5min":11.824,"15min":11.824},"last":5.133}]},{"osd":7,"last update":"Sat Sep 23 20:55:20 2023","interfaces":[{"interface":"back","average":{"1min":2.383,"5min":1.6599999999999999,"15min":1.016},"min":{"1min":0.435,"5min":0.42099999999999999,"15min":0.42099999999999999},"max":{"1min":8.5090000000000003,"5min":12.41,"15min":12.41},"last":1.698},{"interface":"front","average":{"1min":1.516,"5min":1.425,"15min":0.93700000000000006},"min":{"1min":0.379,"5min":0.379,"15min":0.379},"max":{"1min":7.782,"5min":9.3179999999999996,"15min":9.3179999999999996},"last":5.2850000000000001}]}]},{"osd":4,"up_from":31,"seq":133143986375,"num_pgs":42,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":317284,"kb_used_data":229916,"kb_used_omap":0,"kb_used_meta":87360,"kb_avail":93427868,"statfs":{"total":95995035648,"available":95670136832,"internally_reserved":0,"allocated":235433984,"data_stored":234767334,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":89456640},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:55:47 2023","interfaces":[{"interface":"back","average":{"1min":1.921,"5min":1.137,"15min":0.80100000000000005},"min":{"1min":0.22500000000000001,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":8.7189999999999994,"5min":8.7189999999999994,"15min":8.7189999999999994},"last":5.0949999999999998},{"interface":"front","average":{"1min":2.3730000000000002,"5min":1.349,"15min":1.0600000000000001},"min":{"1min":0.26700000000000002,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":8.7050000000000001,"5min":8.7050000000000001,"15min":8.7050000000000001},"last":5.1139999999999999}]},{"osd":1,"last update":"Sat Sep 23 20:55:12 2023","interfaces":[{"interface":"back","average":{"1min":1.534,"5min":1.3029999999999999,"15min":0.80700000000000005},"min":{"1min":0.26100000000000001,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":5.5439999999999996,"5min":7.9859999999999998,"15min":7.9859999999999998},"last":0.56899999999999995},{"interface":"front","average":{"1min":1.2809999999999999,"5min":0.95299999999999996,"15min":0.71699999999999997},"min":{"1min":0.28199999999999997,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":5.093,"5min":6.3540000000000001,"15min":6.3540000000000001},"last":0.33300000000000002}]},{"osd":2,"last update":"Sat Sep 23 20:55:47 2023","interfaces":[{"interface":"back","average":{"1min":1.899,"5min":1.5369999999999999,"15min":0.96899999999999997},"min":{"1min":0.311,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":5.8940000000000001,"5min":15.288,"15min":15.288},"last":0.434},{"interface":"front","average":{"1min":1.8580000000000001,"5min":1.8109999999999999,"15min":1.119},"min":{"1min":0.28799999999999998,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":4.593,"5min":15.262,"15min":15.262},"last":0.36099999999999999}]},{"osd":3,"last update":"Sat Sep 23 20:55:12 2023","interfaces":[{"interface":"back","average":{"1min":2.7389999999999999,"5min":2.4359999999999999,"15min":1.246},"min":{"1min":0.314,"5min":0.314,"15min":0.314},"max":{"1min":8.1449999999999996,"5min":31.562999999999999,"15min":31.562999999999999},"last":1.0900000000000001},{"interface":"front","average":{"1min":2.7330000000000001,"5min":2.4039999999999999,"15min":1.3100000000000001},"min":{"1min":0.50700000000000001,"5min":0.41899999999999998,"15min":0.41899999999999998},"max":{"1min":8.157,"5min":31.574000000000002,"15min":31.574000000000002},"last":1.0780000000000001}]},{"osd":5,"last update":"Sat Sep 23 20:55:30 2023","interfaces":[{"interface":"back","average":{"1min":0.54500000000000004,"5min":0.51100000000000001,"15min":0.48099999999999998},"min":{"1min":0.251,"5min":0.251,"15min":0.251},"max":{"1min":3.3279999999999998,"5min":3.3279999999999998,"15min":3.3279999999999998},"last":0.89000000000000001},{"interface":"front","average":{"1min":0.88900000000000001,"5min":0.65100000000000002,"15min":0.53700000000000003},"min":{"1min":0.251,"5min":0.251,"15min":0.251},"max":{"1min":7.7619999999999996,"5min":7.7619999999999996,"15min":7.7619999999999996},"last":0.72199999999999998}]},{"osd":6,"last update":"Sat Sep 23 20:54:59 2023","interfaces":[{"interface":"back","average":{"1min":0.64800000000000002,"5min":0.67700000000000005,"15min":0.56100000000000005},"min":{"1min":0.34100000000000003,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":2.169,"5min":6.8289999999999997,"15min":6.8289999999999997},"last":0.60599999999999998},{"interface":"front","average":{"1min":0.56100000000000005,"5min":0.68100000000000005,"15min":0.59799999999999998},"min":{"1min":0.32700000000000001,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":1.413,"5min":5.8179999999999996,"15min":5.8179999999999996},"last":0.40999999999999998}]},{"osd":7,"last update":"Sat Sep 23 20:55:12 2023","interfaces":[{"interface":"back","average":{"1min":0.51400000000000001,"5min":0.74399999999999999,"15min":0.623},"min":{"1min":0.316,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":0.88100000000000001,"5min":7.383,"15min":7.383},"last":0.38300000000000001},{"interface":"front","average":{"1min":0.60599999999999998,"5min":0.68899999999999995,"15min":0.60199999999999998},"min":{"1min":0.31900000000000001,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":1.075,"5min":3.9420000000000002,"15min":3.9420000000000002},"last":0.755}]}]},{"osd":5,"up_from":39,"seq":167503724740,"num_pgs":28,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":315032,"kb_used_data":233936,"kb_used_omap":0,"kb_used_meta":81088,"kb_avail":93430120,"statfs":{"total":95995035648,"available":95672442880,"internally_reserved":0,"allocated":239550464,"data_stored":238891870,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":83034112},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:55:36 2023","interfaces":[{"interface":"back","average":{"1min":0.97599999999999998,"5min":0.87,"15min":0.62},"min":{"1min":0.22600000000000001,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":3,"5min":7.766,"15min":7.766},"last":1.2989999999999999},{"interface":"front","average":{"1min":2.29,"5min":1.194,"15min":0.78200000000000003},"min":{"1min":0.223,"5min":0.223,"15min":0.223},"max":{"1min":15.037000000000001,"5min":15.037000000000001,"15min":15.037000000000001},"last":7.5250000000000004}]},{"osd":1,"last update":"Sat Sep 23 20:55:12 2023","interfaces":[{"interface":"back","average":{"1min":2.278,"5min":1.3169999999999999,"15min":0.81599999999999995},"min":{"1min":0.24399999999999999,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":8.3520000000000003,"5min":8.3520000000000003,"15min":8.3520000000000003},"last":3.1619999999999999},{"interface":"front","average":{"1min":1.8220000000000001,"5min":1.177,"15min":0.75800000000000001},"min":{"1min":0.252,"5min":0.252,"15min":0.252},"max":{"1min":7.1230000000000002,"5min":7.4850000000000003,"15min":7.4850000000000003},"last":0.30199999999999999}]},{"osd":2,"last update":"Sat Sep 23 20:55:45 2023","interfaces":[{"interface":"back","average":{"1min":1.595,"5min":1.361,"15min":0.81499999999999995},"min":{"1min":0.36899999999999999,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":6.141,"5min":12.651,"15min":12.651},"last":1.274},{"interface":"front","average":{"1min":2.492,"5min":1.363,"15min":0.84299999999999997},"min":{"1min":0.318,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":8.3209999999999997,"5min":8.3209999999999997,"15min":8.3209999999999997},"last":0.51400000000000001}]},{"osd":3,"last update":"Sat Sep 23 20:55:12 2023","interfaces":[{"interface":"back","average":{"1min":1.6839999999999999,"5min":1.4339999999999999,"15min":0.89200000000000002},"min":{"1min":0.28599999999999998,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":8.4079999999999995,"5min":9.5489999999999995,"15min":9.5489999999999995},"last":2.0819999999999999},{"interface":"front","average":{"1min":1.1279999999999999,"5min":1.0740000000000001,"15min":0.78000000000000003},"min":{"1min":0.32100000000000001,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":4.6559999999999997,"5min":6.2220000000000004,"15min":6.2220000000000004},"last":1.3140000000000001}]},{"osd":4,"last update":"Sat Sep 23 20:55:25 2023","interfaces":[{"interface":"back","average":{"1min":0.74099999999999999,"5min":0.59599999999999997,"15min":0.51200000000000001},"min":{"1min":0.27600000000000002,"5min":0.27600000000000002,"15min":0.253},"max":{"1min":3.8479999999999999,"5min":3.8479999999999999,"15min":3.8479999999999999},"last":0.38600000000000001},{"interface":"front","average":{"1min":0.68100000000000005,"5min":0.57099999999999995,"15min":0.52500000000000002},"min":{"1min":0.29099999999999998,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":2.4550000000000001,"5min":3.0179999999999998,"15min":3.0179999999999998},"last":1.3779999999999999}]},{"osd":6,"last update":"Sat Sep 23 20:54:48 2023","interfaces":[{"interface":"back","average":{"1min":0.44500000000000001,"5min":0.56999999999999995,"15min":0.53200000000000003},"min":{"1min":0.308,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":0.92800000000000005,"5min":7.2080000000000002,"15min":7.2080000000000002},"last":0.44900000000000001},{"interface":"front","average":{"1min":0.624,"5min":0.61799999999999999,"15min":0.56299999999999994},"min":{"1min":0.35499999999999998,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":2.379,"5min":2.379,"15min":2.379},"last":0.36499999999999999}]},{"osd":7,"last update":"Sat Sep 23 20:55:07 2023","interfaces":[{"interface":"back","average":{"1min":0.97799999999999998,"5min":0.64100000000000001,"15min":0.58999999999999997},"min":{"1min":0.312,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":8.5649999999999995,"5min":8.5649999999999995,"15min":8.5649999999999995},"last":0.48399999999999999},{"interface":"front","average":{"1min":0.90500000000000003,"5min":0.69999999999999996,"15min":0.62},"min":{"1min":0.35099999999999998,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":4.5430000000000001,"5min":4.5430000000000001,"15min":4.5430000000000001},"last":0.33700000000000002}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":192679936,"data_stored":192675864,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":326963200,"data_stored":326959128,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":592187392,"data_stored":592183320,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":523698176,"data_stored":523698176,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":227147776,"data_stored":227147776,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":225316864,"data_stored":225312792,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":367460352,"data_stored":367460352,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":400490496,"data_stored":400490496,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T20:55:59.200 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph osd pool set unique_pool_0 pgp_num 26 2023-09-23T20:55:59.231 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:55:59.237 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:55:59.238 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:55:59.338 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:55:59.371 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:55:59.411 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:55:59.414 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:55:59.414 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:55:59.515 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:55:59.545 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:55:59.585 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:55:59.588 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:55:59.588 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:55:59.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:55:59 smithi070 ceph-mon[168282]: pgmap v424: 109 pgs: 1 peering, 108 active+clean; 727 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 652 op/s; 0 B/s, 246 objects/s recovering 2023-09-23T20:55:59.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:55:59 smithi070 ceph-mon[168282]: from='client.35037 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:55:59.690 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:55:59.724 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:55:59 smithi026 ceph-mon[175151]: pgmap v424: 109 pgs: 1 peering, 108 active+clean; 727 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 652 op/s; 0 B/s, 246 objects/s recovering 2023-09-23T20:55:59.724 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:55:59 smithi026 ceph-mon[175151]: from='client.35037 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:55:59.725 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:55:59 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[164029]: ::ffff:172.21.15.70 - - [23/Sep/2023:20:55:59] "GET /metrics HTTP/1.1" 200 36346 "" "Prometheus/2.43.0" 2023-09-23T20:55:59.725 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:55:59 smithi026 ceph-mon[179688]: pgmap v424: 109 pgs: 1 peering, 108 active+clean; 727 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 652 op/s; 0 B/s, 246 objects/s recovering 2023-09-23T20:55:59.725 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:55:59 smithi026 ceph-mon[179688]: from='client.35037 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:55:59.768 INFO:teuthology.orchestra.run.smithi026.stdout:ERROR: (22) Invalid argument 2023-09-23T20:55:59.768 INFO:teuthology.orchestra.run.smithi026.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:55:59.821 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:56:00.174 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:56:00.178 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:56:00.179 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:56:00.279 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:56:00.328 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0: {} 2023-09-23T20:56:00.367 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1: {} 2023-09-23T20:56:00.372 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_historic_ops 2023-09-23T20:56:00.385 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:56:00.411 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2: {} 2023-09-23T20:56:00.461 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3: {} 2023-09-23T20:56:00.489 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:56:00.501 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:56:00.501 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:56:00.502 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4: {} 2023-09-23T20:56:00.532 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T20:56:00.586 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:56:00.586 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6: {} 2023-09-23T20:56:00.603 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:56:00.631 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:56:00.654 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T20:56:00.678 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:56:00.679 INFO:teuthology.orchestra.run.smithi026.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T20:56:00.679 INFO:teuthology.orchestra.run.smithi026.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T20:56:00.680 INFO:teuthology.orchestra.run.smithi026.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T20:56:00.680 INFO:teuthology.orchestra.run.smithi026.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T20:56:00.680 INFO:teuthology.orchestra.run.smithi026.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T20:56:00.680 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T20:56:00.680 INFO:teuthology.orchestra.run.smithi026.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T20:56:00.680 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T20:56:00.681 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:56:00.682 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:56:00.783 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:56:00.816 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:56:00.857 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:56:00.862 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:56:00.863 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:56:00.964 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:56:00.983 INFO:teuthology.orchestra.run.smithi026.stderr:noscrub is unset 2023-09-23T20:56:00.996 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:56:01.040 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:56:01.044 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:56:01.044 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:56:01.145 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:56:01.179 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:56:01.222 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:56:01.226 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:56:00 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/4022167016' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:56:01.226 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:56:00 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:56:01.226 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:56:00 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/3181250421' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:56:01.226 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:56:00 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "11"}]: dispatch 2023-09-23T20:56:01.226 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:56:00 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num_actual", "val": "17"}]: dispatch 2023-09-23T20:56:01.227 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:56:01.227 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:56:01.244 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:56:00 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/4022167016' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:56:01.244 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:56:00 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:56:01.245 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:56:00 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/3181250421' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:56:01.245 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:56:00 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "11"}]: dispatch 2023-09-23T20:56:01.245 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:56:00 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num_actual", "val": "17"}]: dispatch 2023-09-23T20:56:01.245 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:56:00 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/4022167016' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:56:01.245 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:56:00 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:56:01.245 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:56:00 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/3181250421' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:56:01.245 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:56:00 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "11"}]: dispatch 2023-09-23T20:56:01.245 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:56:00 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num_actual", "val": "17"}]: dispatch 2023-09-23T20:56:01.328 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:56:01.394 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:56:02.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:56:01 smithi026 ceph-mon[175151]: pgmap v425: 109 pgs: 109 active+clean; 660 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 553 op/s; 0 B/s, 132 objects/s recovering 2023-09-23T20:56:02.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:56:01 smithi026 ceph-mon[175151]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:56:02.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:56:01 smithi026 ceph-mon[175151]: failed to encode map e180 with expected crc 2023-09-23T20:56:02.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:56:01 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T20:56:02.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:56:01 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "11"}]': finished 2023-09-23T20:56:02.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:56:01 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num_actual", "val": "17"}]': finished 2023-09-23T20:56:02.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:56:01 smithi026 ceph-mon[175151]: osdmap e180: 8 total, 8 up, 8 in 2023-09-23T20:56:02.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:56:01 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:56:02.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:56:01 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:56:02.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:56:01 smithi026 ceph-mon[179688]: pgmap v425: 109 pgs: 109 active+clean; 660 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 553 op/s; 0 B/s, 132 objects/s recovering 2023-09-23T20:56:02.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:56:01 smithi026 ceph-mon[179688]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:56:02.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:56:01 smithi026 ceph-mon[179688]: failed to encode map e180 with expected crc 2023-09-23T20:56:02.134 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:56:01 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T20:56:02.134 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:56:01 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "11"}]': finished 2023-09-23T20:56:02.134 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:56:01 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num_actual", "val": "17"}]': finished 2023-09-23T20:56:02.134 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:56:01 smithi026 ceph-mon[179688]: osdmap e180: 8 total, 8 up, 8 in 2023-09-23T20:56:02.134 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:56:01 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:56:02.134 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:56:01 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:56:02.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:56:01 smithi070 ceph-mon[168282]: pgmap v425: 109 pgs: 109 active+clean; 660 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 553 op/s; 0 B/s, 132 objects/s recovering 2023-09-23T20:56:02.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:56:01 smithi070 ceph-mon[168282]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:56:02.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:56:01 smithi070 ceph-mon[168282]: failed to encode map e180 with expected crc 2023-09-23T20:56:02.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:56:01 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T20:56:02.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:56:01 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "11"}]': finished 2023-09-23T20:56:02.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:56:01 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num_actual", "val": "17"}]': finished 2023-09-23T20:56:02.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:56:01 smithi070 ceph-mon[168282]: osdmap e180: 8 total, 8 up, 8 in 2023-09-23T20:56:02.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:56:01 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:56:02.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:56:01 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:56:03.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:56:02 smithi070 ceph-mon[168282]: 6.7 scrub starts 2023-09-23T20:56:03.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:56:02 smithi070 ceph-mon[168282]: 6.7 scrub ok 2023-09-23T20:56:03.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:56:02 smithi070 ceph-mon[168282]: 6.16 scrub starts 2023-09-23T20:56:03.371 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:56:02 smithi070 ceph-mon[168282]: failed to encode map e181 with expected crc 2023-09-23T20:56:03.371 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:56:02 smithi070 ceph-mon[168282]: osdmap e181: 8 total, 8 up, 8 in 2023-09-23T20:56:03.371 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:56:02 smithi070 ceph-mon[168282]: failed to encode map e181 with expected crc 2023-09-23T20:56:03.371 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:56:02 smithi070 ceph-mon[168282]: failed to encode map e181 with expected crc 2023-09-23T20:56:03.371 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:56:02 smithi070 ceph-mon[168282]: failed to encode map e181 with expected crc 2023-09-23T20:56:03.371 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:56:02 smithi070 ceph-mon[168282]: failed to encode map e181 with expected crc 2023-09-23T20:56:03.371 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:56:02 smithi070 ceph-mon[168282]: failed to encode map e181 with expected crc 2023-09-23T20:56:03.371 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:56:02 smithi070 ceph-mon[168282]: failed to encode map e181 with expected crc 2023-09-23T20:56:03.371 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:56:02 smithi070 ceph-mon[168282]: 6.16 scrub ok 2023-09-23T20:56:03.383 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:56:02 smithi026 ceph-mon[179688]: 6.7 scrub starts 2023-09-23T20:56:03.383 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:56:02 smithi026 ceph-mon[179688]: 6.7 scrub ok 2023-09-23T20:56:03.383 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:56:02 smithi026 ceph-mon[179688]: 6.16 scrub starts 2023-09-23T20:56:03.383 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:56:02 smithi026 ceph-mon[179688]: failed to encode map e181 with expected crc 2023-09-23T20:56:03.383 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:56:02 smithi026 ceph-mon[179688]: osdmap e181: 8 total, 8 up, 8 in 2023-09-23T20:56:03.383 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:56:02 smithi026 ceph-mon[179688]: failed to encode map e181 with expected crc 2023-09-23T20:56:03.384 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:56:02 smithi026 ceph-mon[179688]: failed to encode map e181 with expected crc 2023-09-23T20:56:03.384 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:56:02 smithi026 ceph-mon[179688]: failed to encode map e181 with expected crc 2023-09-23T20:56:03.384 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:56:02 smithi026 ceph-mon[179688]: failed to encode map e181 with expected crc 2023-09-23T20:56:03.384 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:56:02 smithi026 ceph-mon[179688]: failed to encode map e181 with expected crc 2023-09-23T20:56:03.384 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:56:02 smithi026 ceph-mon[179688]: failed to encode map e181 with expected crc 2023-09-23T20:56:03.384 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:56:02 smithi026 ceph-mon[179688]: 6.16 scrub ok 2023-09-23T20:56:03.388 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:56:02 smithi026 ceph-mon[175151]: 6.7 scrub starts 2023-09-23T20:56:03.388 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:56:02 smithi026 ceph-mon[175151]: 6.7 scrub ok 2023-09-23T20:56:03.388 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:56:02 smithi026 ceph-mon[175151]: 6.16 scrub starts 2023-09-23T20:56:03.388 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:56:02 smithi026 ceph-mon[175151]: failed to encode map e181 with expected crc 2023-09-23T20:56:03.388 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:56:02 smithi026 ceph-mon[175151]: osdmap e181: 8 total, 8 up, 8 in 2023-09-23T20:56:03.388 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:56:02 smithi026 ceph-mon[175151]: failed to encode map e181 with expected crc 2023-09-23T20:56:03.388 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:56:02 smithi026 ceph-mon[175151]: failed to encode map e181 with expected crc 2023-09-23T20:56:03.388 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:56:02 smithi026 ceph-mon[175151]: failed to encode map e181 with expected crc 2023-09-23T20:56:03.388 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:56:02 smithi026 ceph-mon[175151]: failed to encode map e181 with expected crc 2023-09-23T20:56:03.388 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:56:02 smithi026 ceph-mon[175151]: failed to encode map e181 with expected crc 2023-09-23T20:56:03.389 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:56:02 smithi026 ceph-mon[175151]: failed to encode map e181 with expected crc 2023-09-23T20:56:03.389 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:56:02 smithi026 ceph-mon[175151]: 6.16 scrub ok 2023-09-23T20:56:03.834 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:56:03.842 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:56:03.842 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:56:03.943 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:56:04.045 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:56:04.106 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:56:04.124 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:56:04.141 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:56:04.141 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:56:03 smithi026 ceph-mon[175151]: pgmap v428: 109 pgs: 2 remapped+peering, 1 peering, 106 active+clean; 500 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail 2023-09-23T20:56:04.141 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:56:03 smithi026 ceph-mon[175151]: 6.0 scrub starts 2023-09-23T20:56:04.141 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:56:03 smithi026 ceph-mon[175151]: 6.0 scrub ok 2023-09-23T20:56:04.141 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:56:03 smithi026 ceph-mon[175151]: failed to encode map e182 with expected crc 2023-09-23T20:56:04.142 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:56:03 smithi026 ceph-mon[175151]: failed to encode map e182 with expected crc 2023-09-23T20:56:04.142 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:56:03 smithi026 ceph-mon[175151]: osdmap e182: 8 total, 8 up, 8 in 2023-09-23T20:56:04.142 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:56:03 smithi026 ceph-mon[175151]: failed to encode map e182 with expected crc 2023-09-23T20:56:04.142 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:56:03 smithi026 ceph-mon[179688]: pgmap v428: 109 pgs: 2 remapped+peering, 1 peering, 106 active+clean; 500 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail 2023-09-23T20:56:04.142 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:56:03 smithi026 ceph-mon[179688]: 6.0 scrub starts 2023-09-23T20:56:04.142 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:56:03 smithi026 ceph-mon[179688]: 6.0 scrub ok 2023-09-23T20:56:04.142 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:56:03 smithi026 ceph-mon[179688]: failed to encode map e182 with expected crc 2023-09-23T20:56:04.143 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:56:03 smithi026 ceph-mon[179688]: failed to encode map e182 with expected crc 2023-09-23T20:56:04.143 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:56:03 smithi026 ceph-mon[179688]: osdmap e182: 8 total, 8 up, 8 in 2023-09-23T20:56:04.143 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:56:03 smithi026 ceph-mon[179688]: failed to encode map e182 with expected crc 2023-09-23T20:56:04.143 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:56:04.244 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:56:04.276 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:56:03 smithi070 ceph-mon[168282]: pgmap v428: 109 pgs: 2 remapped+peering, 1 peering, 106 active+clean; 500 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail 2023-09-23T20:56:04.277 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:56:03 smithi070 ceph-mon[168282]: 6.0 scrub starts 2023-09-23T20:56:04.277 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:56:03 smithi070 ceph-mon[168282]: 6.0 scrub ok 2023-09-23T20:56:04.277 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:56:03 smithi070 ceph-mon[168282]: failed to encode map e182 with expected crc 2023-09-23T20:56:04.277 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:56:03 smithi070 ceph-mon[168282]: failed to encode map e182 with expected crc 2023-09-23T20:56:04.277 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:56:03 smithi070 ceph-mon[168282]: osdmap e182: 8 total, 8 up, 8 in 2023-09-23T20:56:04.277 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:56:03 smithi070 ceph-mon[168282]: failed to encode map e182 with expected crc 2023-09-23T20:56:04.280 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:56:04.324 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:56:04.326 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:56:04.327 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:56:04.427 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:56:04.518 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:56:04.622 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:56:04.628 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:56:04.628 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:56:04.729 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:56:04.827 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:56:05.136 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:56:04 smithi026 ceph-mon[175151]: failed to encode map e182 with expected crc 2023-09-23T20:56:05.136 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:56:04 smithi026 ceph-mon[175151]: 1.0 scrub starts 2023-09-23T20:56:05.136 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:56:04 smithi026 ceph-mon[175151]: 1.0 scrub ok 2023-09-23T20:56:05.136 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:56:04 smithi026 ceph-mon[175151]: failed to encode map e183 with expected crc 2023-09-23T20:56:05.137 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:56:04 smithi026 ceph-mon[175151]: osdmap e183: 8 total, 8 up, 8 in 2023-09-23T20:56:05.137 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:56:04 smithi026 ceph-mon[175151]: failed to encode map e183 with expected crc 2023-09-23T20:56:05.137 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:56:04 smithi026 ceph-mon[175151]: failed to encode map e183 with expected crc 2023-09-23T20:56:05.137 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:56:04 smithi026 ceph-mon[175151]: failed to encode map e183 with expected crc 2023-09-23T20:56:05.137 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:56:04 smithi026 ceph-mon[175151]: osdmap e184: 8 total, 8 up, 8 in 2023-09-23T20:56:05.137 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:56:04 smithi026 ceph-mon[179688]: failed to encode map e182 with expected crc 2023-09-23T20:56:05.137 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:56:04 smithi026 ceph-mon[179688]: 1.0 scrub starts 2023-09-23T20:56:05.137 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:56:04 smithi026 ceph-mon[179688]: 1.0 scrub ok 2023-09-23T20:56:05.137 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:56:04 smithi026 ceph-mon[179688]: failed to encode map e183 with expected crc 2023-09-23T20:56:05.137 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:56:04 smithi026 ceph-mon[179688]: osdmap e183: 8 total, 8 up, 8 in 2023-09-23T20:56:05.137 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:56:04 smithi026 ceph-mon[179688]: failed to encode map e183 with expected crc 2023-09-23T20:56:05.138 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:56:04 smithi026 ceph-mon[179688]: failed to encode map e183 with expected crc 2023-09-23T20:56:05.138 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:56:04 smithi026 ceph-mon[179688]: failed to encode map e183 with expected crc 2023-09-23T20:56:05.138 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:56:04 smithi026 ceph-mon[179688]: osdmap e184: 8 total, 8 up, 8 in 2023-09-23T20:56:05.224 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:56:05.233 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:56:05.233 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:56:05.334 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:56:05.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:56:04 smithi070 ceph-mon[168282]: failed to encode map e182 with expected crc 2023-09-23T20:56:05.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:56:04 smithi070 ceph-mon[168282]: 1.0 scrub starts 2023-09-23T20:56:05.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:56:04 smithi070 ceph-mon[168282]: 1.0 scrub ok 2023-09-23T20:56:05.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:56:04 smithi070 ceph-mon[168282]: failed to encode map e183 with expected crc 2023-09-23T20:56:05.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:56:04 smithi070 ceph-mon[168282]: osdmap e183: 8 total, 8 up, 8 in 2023-09-23T20:56:05.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:56:04 smithi070 ceph-mon[168282]: failed to encode map e183 with expected crc 2023-09-23T20:56:05.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:56:04 smithi070 ceph-mon[168282]: failed to encode map e183 with expected crc 2023-09-23T20:56:05.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:56:04 smithi070 ceph-mon[168282]: failed to encode map e183 with expected crc 2023-09-23T20:56:05.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:56:04 smithi070 ceph-mon[168282]: osdmap e184: 8 total, 8 up, 8 in 2023-09-23T20:56:05.446 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:56:05.566 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:56:05.571 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:56:05.572 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:56:05.648 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T20:56:05.673 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:56:05.711 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T20:56:05.712 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:56:05.754 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:56:05.760 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:56:05.760 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:56:05.861 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:56:05.891 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:56:05.930 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:56:05.933 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:56:05.933 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:56:06.034 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:56:06.070 INFO:teuthology.orchestra.run.smithi026.stderr:set pool 6 pgp_num to 26 2023-09-23T20:56:06.070 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:56:06.106 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:56:06.108 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:56:06.109 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:56:06.113 INFO:teuthology.orchestra.run.smithi026.stdout:{ 2023-09-23T20:56:06.114 INFO:teuthology.orchestra.run.smithi026.stdout: "size": 20, 2023-09-23T20:56:06.114 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 600, 2023-09-23T20:56:06.114 INFO:teuthology.orchestra.run.smithi026.stdout: "ops": [ 2023-09-23T20:56:06.114 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:06.114 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34797.0:5581 6.f 6:f9876e9a:::benchmark_data_smithi026_223868_object5580:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e106)", 2023-09-23T20:56:06.114 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T20:52:21.793639+0000", 2023-09-23T20:56:06.114 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 224.31484377800001, 2023-09-23T20:56:06.114 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.080631881000000002, 2023-09-23T20:56:06.114 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T20:56:06.114 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:56:06.115 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T20:56:06.115 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34797", 2023-09-23T20:56:06.115 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T20:56:06.115 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 5581 2023-09-23T20:56:06.115 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:06.115 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T20:56:06.115 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:06.115 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T20:56:06.115 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:21.793639+0000", 2023-09-23T20:56:06.115 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:56:06.115 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:06.115 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:06.115 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T20:56:06.116 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:21.793639+0000", 2023-09-23T20:56:06.116 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:56:06.116 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:06.116 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:06.116 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T20:56:06.116 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:21.793637+0000", 2023-09-23T20:56:06.116 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T20:56:06.116 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:06.116 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:06.116 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T20:56:06.116 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:21.793661+0000", 2023-09-23T20:56:06.116 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.4215000000000001e-05 2023-09-23T20:56:06.116 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:06.117 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:06.117 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T20:56:06.117 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:21.793673+0000", 2023-09-23T20:56:06.117 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.2011000000000001e-05 2023-09-23T20:56:06.117 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:06.117 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:06.117 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T20:56:06.117 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:21.793681+0000", 2023-09-23T20:56:06.117 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.2069999999999995e-06 2023-09-23T20:56:06.117 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:06.117 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:06.117 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:56:06.117 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:21.793739+0000", 2023-09-23T20:56:06.118 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.7176000000000003e-05 2023-09-23T20:56:06.118 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:06.118 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:06.118 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T20:56:06.118 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:21.793869+0000", 2023-09-23T20:56:06.118 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00012998400000000001 2023-09-23T20:56:06.118 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:06.118 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:06.118 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 3,4", 2023-09-23T20:56:06.118 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:21.794008+0000", 2023-09-23T20:56:06.118 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00013893900000000001 2023-09-23T20:56:06.118 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:06.118 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:06.119 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T20:56:06.119 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:21.794668+0000", 2023-09-23T20:56:06.119 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00066021500000000002 2023-09-23T20:56:06.119 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:06.119 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:06.119 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:56:06.119 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:21.797987+0000", 2023-09-23T20:56:06.119 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.003319383 2023-09-23T20:56:06.119 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:06.119 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:06.119 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:56:06.119 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:21.874167+0000", 2023-09-23T20:56:06.119 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.076179809000000001 2023-09-23T20:56:06.120 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:06.120 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:06.120 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T20:56:06.120 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:21.874227+0000", 2023-09-23T20:56:06.120 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.9590999999999999e-05 2023-09-23T20:56:06.120 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:06.120 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:06.120 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T20:56:06.120 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:52:21.874270+0000", 2023-09-23T20:56:06.120 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.3908e-05 2023-09-23T20:56:06.120 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:56:06.120 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T20:56:06.121 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:56:06.121 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:06.121 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:06.121 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34797.0:93586 6.f 6:f877714a:::benchmark_data_smithi026_223868_object93585:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e123)", 2023-09-23T20:56:06.121 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T20:53:36.886990+0000", 2023-09-23T20:56:06.121 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 149.22149229499999, 2023-09-23T20:56:06.121 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.087266787999999998, 2023-09-23T20:56:06.121 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T20:56:06.121 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:56:06.121 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T20:56:06.121 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34797", 2023-09-23T20:56:06.121 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T20:56:06.121 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 93586 2023-09-23T20:56:06.122 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:06.122 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T20:56:06.122 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:06.122 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T20:56:06.122 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:36.886990+0000", 2023-09-23T20:56:06.122 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:56:06.122 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:06.122 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:06.122 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T20:56:06.122 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:36.886990+0000", 2023-09-23T20:56:06.122 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:56:06.122 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:06.122 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:06.123 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T20:56:06.123 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:36.886989+0000", 2023-09-23T20:56:06.123 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T20:56:06.123 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:06.123 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:06.123 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T20:56:06.123 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:36.887011+0000", 2023-09-23T20:56:06.123 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.2776e-05 2023-09-23T20:56:06.123 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:06.123 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:06.123 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T20:56:06.123 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:36.887024+0000", 2023-09-23T20:56:06.123 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.2454e-05 2023-09-23T20:56:06.124 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:06.124 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:06.124 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T20:56:06.124 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:36.887033+0000", 2023-09-23T20:56:06.124 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.5939999999999994e-06 2023-09-23T20:56:06.124 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:06.124 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:06.124 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:56:06.124 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:36.893492+0000", 2023-09-23T20:56:06.124 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0064594630000000004 2023-09-23T20:56:06.124 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:06.124 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:06.124 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T20:56:06.125 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:36.893632+0000", 2023-09-23T20:56:06.125 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00013976 2023-09-23T20:56:06.125 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:06.125 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:06.125 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 3,6", 2023-09-23T20:56:06.125 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:36.893786+0000", 2023-09-23T20:56:06.125 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00015381999999999999 2023-09-23T20:56:06.125 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:06.125 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:06.125 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T20:56:06.125 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:36.901222+0000", 2023-09-23T20:56:06.126 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0074365880000000001 2023-09-23T20:56:06.126 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:06.126 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:06.126 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:56:06.126 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:36.907301+0000", 2023-09-23T20:56:06.126 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.006078511 2023-09-23T20:56:06.126 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:06.126 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:06.126 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:56:06.126 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:36.971427+0000", 2023-09-23T20:56:06.126 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.064126341000000003 2023-09-23T20:56:06.126 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:06.126 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:06.127 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T20:56:06.127 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:36.971493+0000", 2023-09-23T20:56:06.127 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.5703000000000004e-05 2023-09-23T20:56:06.127 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:06.127 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:06.127 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T20:56:06.127 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:36.974257+0000", 2023-09-23T20:56:06.127 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0027641639999999999 2023-09-23T20:56:06.127 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:56:06.127 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T20:56:06.127 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:56:06.127 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:06.127 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:06.128 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34797.0:95323 6.f 6:fbdee395:::benchmark_data_smithi026_223868_object95322:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e124)", 2023-09-23T20:56:06.128 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T20:53:39.434938+0000", 2023-09-23T20:56:06.128 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 146.67354462200001, 2023-09-23T20:56:06.128 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.078830418999999999, 2023-09-23T20:56:06.128 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T20:56:06.128 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:56:06.128 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T20:56:06.128 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34797", 2023-09-23T20:56:06.128 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T20:56:06.128 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 95323 2023-09-23T20:56:06.128 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:06.129 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T20:56:06.129 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:06.129 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T20:56:06.129 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:39.434938+0000", 2023-09-23T20:56:06.129 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:56:06.129 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:06.129 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:06.129 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T20:56:06.129 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:39.434938+0000", 2023-09-23T20:56:06.129 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:56:06.129 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:06.129 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:06.129 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T20:56:06.130 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:39.434936+0000", 2023-09-23T20:56:06.130 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999981 2023-09-23T20:56:06.130 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:06.130 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:06.130 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T20:56:06.130 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:39.434962+0000", 2023-09-23T20:56:06.130 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.5474999999999999e-05 2023-09-23T20:56:06.130 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:06.130 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:06.130 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T20:56:06.130 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:39.434975+0000", 2023-09-23T20:56:06.130 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.3251000000000001e-05 2023-09-23T20:56:06.130 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:06.131 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:06.131 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T20:56:06.131 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:39.434983+0000", 2023-09-23T20:56:06.131 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.6459999999999999e-06 2023-09-23T20:56:06.131 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:06.131 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:06.131 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:56:06.131 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:39.457725+0000", 2023-09-23T20:56:06.131 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.022741447000000001 2023-09-23T20:56:06.131 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:06.131 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:06.131 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T20:56:06.132 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:39.457923+0000", 2023-09-23T20:56:06.132 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00019830499999999999 2023-09-23T20:56:06.132 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:06.132 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:06.132 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 3,6", 2023-09-23T20:56:06.132 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:39.458411+0000", 2023-09-23T20:56:06.132 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00048770299999999998 2023-09-23T20:56:06.132 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:06.132 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:06.132 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T20:56:06.132 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:39.474580+0000", 2023-09-23T20:56:06.132 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.016168775999999999 2023-09-23T20:56:06.132 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:06.133 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:06.133 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:56:06.133 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:39.482933+0000", 2023-09-23T20:56:06.133 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0083536540000000003 2023-09-23T20:56:06.133 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:06.133 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:06.133 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:56:06.133 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:39.511050+0000", 2023-09-23T20:56:06.133 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.028116253000000001 2023-09-23T20:56:06.133 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:06.133 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:06.133 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T20:56:06.133 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:39.511110+0000", 2023-09-23T20:56:06.133 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.0050000000000003e-05 2023-09-23T20:56:06.134 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:06.134 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:06.134 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T20:56:06.134 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:39.513768+0000", 2023-09-23T20:56:06.134 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0026585340000000002 2023-09-23T20:56:06.134 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:56:06.134 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T20:56:06.134 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:56:06.134 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:06.134 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:06.134 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34797.0:95576 6.f 6:f421f915:::benchmark_data_smithi026_223868_object95575:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e124)", 2023-09-23T20:56:06.134 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T20:53:39.878342+0000", 2023-09-23T20:56:06.135 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 146.23013999200001, 2023-09-23T20:56:06.135 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.095662639999999993, 2023-09-23T20:56:06.135 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T20:56:06.135 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:56:06.135 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T20:56:06.135 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34797", 2023-09-23T20:56:06.135 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T20:56:06.135 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 95576 2023-09-23T20:56:06.135 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:06.135 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T20:56:06.135 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:06.135 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T20:56:06.136 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:39.878342+0000", 2023-09-23T20:56:06.136 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:56:06.136 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:06.136 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:06.136 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T20:56:06.136 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:39.878342+0000", 2023-09-23T20:56:06.136 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:56:06.136 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:06.136 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:06.136 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T20:56:06.136 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:39.878341+0000", 2023-09-23T20:56:06.136 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999981 2023-09-23T20:56:06.136 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:06.137 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:06.137 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T20:56:06.137 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:39.878367+0000", 2023-09-23T20:56:06.137 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.65e-05 2023-09-23T20:56:06.137 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:06.137 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:06.137 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T20:56:06.137 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:39.878389+0000", 2023-09-23T20:56:06.137 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.1730999999999999e-05 2023-09-23T20:56:06.137 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:06.137 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:06.138 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T20:56:06.138 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:39.878400+0000", 2023-09-23T20:56:06.138 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0917999999999999e-05 2023-09-23T20:56:06.138 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:06.138 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:06.138 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:56:06.138 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:39.895274+0000", 2023-09-23T20:56:06.138 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.016874739 2023-09-23T20:56:06.138 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:06.138 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:06.138 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T20:56:06.138 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:39.895428+0000", 2023-09-23T20:56:06.138 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000153793 2023-09-23T20:56:06.139 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:06.139 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:06.139 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 3,6", 2023-09-23T20:56:06.139 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:39.895606+0000", 2023-09-23T20:56:06.139 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00017741999999999999 2023-09-23T20:56:06.139 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:06.139 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:06.139 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:56:06.139 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:39.928658+0000", 2023-09-23T20:56:06.139 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.033052444 2023-09-23T20:56:06.139 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:06.139 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:06.140 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T20:56:06.140 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:39.937104+0000", 2023-09-23T20:56:06.140 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0084460769999999998 2023-09-23T20:56:06.140 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:06.140 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:06.140 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:56:06.140 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:39.971312+0000", 2023-09-23T20:56:06.140 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.034208239000000001 2023-09-23T20:56:06.140 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:06.140 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:06.140 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T20:56:06.140 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:39.971372+0000", 2023-09-23T20:56:06.140 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.94e-05 2023-09-23T20:56:06.141 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:06.141 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:06.141 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T20:56:06.141 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:39.974005+0000", 2023-09-23T20:56:06.141 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.002633242 2023-09-23T20:56:06.141 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:56:06.141 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T20:56:06.141 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:56:06.141 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:06.141 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:06.141 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34797.0:95584 6.f 6:f94fbf5d:::benchmark_data_smithi026_223868_object95583:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e124)", 2023-09-23T20:56:06.141 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T20:53:39.889495+0000", 2023-09-23T20:56:06.141 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 146.21898714599999, 2023-09-23T20:56:06.142 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.086974098999999999, 2023-09-23T20:56:06.142 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T20:56:06.142 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:56:06.142 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T20:56:06.142 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34797", 2023-09-23T20:56:06.142 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T20:56:06.142 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 95584 2023-09-23T20:56:06.142 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:06.142 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T20:56:06.142 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:06.142 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T20:56:06.142 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:39.889495+0000", 2023-09-23T20:56:06.143 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:56:06.143 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:06.143 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:06.143 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T20:56:06.143 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:39.889495+0000", 2023-09-23T20:56:06.143 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:56:06.143 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:06.143 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:06.143 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T20:56:06.143 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:39.889493+0000", 2023-09-23T20:56:06.143 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999981 2023-09-23T20:56:06.143 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:06.144 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:06.144 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T20:56:06.144 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:39.889529+0000", 2023-09-23T20:56:06.144 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.5772999999999998e-05 2023-09-23T20:56:06.144 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:06.144 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:06.144 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T20:56:06.144 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:39.889563+0000", 2023-09-23T20:56:06.144 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.4406999999999998e-05 2023-09-23T20:56:06.144 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:06.144 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:06.144 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T20:56:06.144 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:39.889578+0000", 2023-09-23T20:56:06.145 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.4581e-05 2023-09-23T20:56:06.145 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:06.145 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:06.145 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:56:06.145 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:39.898894+0000", 2023-09-23T20:56:06.145 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0093163740000000005 2023-09-23T20:56:06.145 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:06.145 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:06.145 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T20:56:06.145 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:39.899035+0000", 2023-09-23T20:56:06.145 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00014104499999999999 2023-09-23T20:56:06.146 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:06.146 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:06.146 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 3,6", 2023-09-23T20:56:06.146 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:39.899192+0000", 2023-09-23T20:56:06.146 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000156983 2023-09-23T20:56:06.146 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:06.146 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:06.146 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:56:06.146 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:39.931031+0000", 2023-09-23T20:56:06.146 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.031838311000000001 2023-09-23T20:56:06.146 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:06.146 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:06.147 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T20:56:06.147 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:39.937111+0000", 2023-09-23T20:56:06.147 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0060799560000000001 2023-09-23T20:56:06.147 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:06.147 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:06.147 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:56:06.147 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:39.974076+0000", 2023-09-23T20:56:06.147 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.036965022 2023-09-23T20:56:06.147 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:06.147 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:06.147 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T20:56:06.147 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:39.974145+0000", 2023-09-23T20:56:06.148 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.9210999999999999e-05 2023-09-23T20:56:06.148 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:06.148 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:06.148 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T20:56:06.148 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:39.976469+0000", 2023-09-23T20:56:06.148 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0023243640000000002 2023-09-23T20:56:06.148 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:56:06.148 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T20:56:06.148 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:56:06.148 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:06.148 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:06.149 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34797.0:102101 6.f 6:fb23e207:::benchmark_data_smithi026_223868_object975:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e126)", 2023-09-23T20:56:06.149 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T20:53:51.863446+0000", 2023-09-23T20:56:06.149 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 134.24503600200001, 2023-09-23T20:56:06.149 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.073157818999999999, 2023-09-23T20:56:06.149 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T20:56:06.149 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:56:06.149 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T20:56:06.149 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34797", 2023-09-23T20:56:06.149 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T20:56:06.149 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 102101 2023-09-23T20:56:06.149 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:06.149 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T20:56:06.149 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:06.150 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T20:56:06.150 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:51.863446+0000", 2023-09-23T20:56:06.150 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:56:06.150 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:06.150 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:06.150 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T20:56:06.150 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:51.863446+0000", 2023-09-23T20:56:06.150 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:56:06.150 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:06.150 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:06.150 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T20:56:06.150 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:51.863446+0000", 2023-09-23T20:56:06.151 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T20:56:06.151 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:06.151 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:06.151 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T20:56:06.151 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:51.863448+0000", 2023-09-23T20:56:06.151 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.9180000000000001e-06 2023-09-23T20:56:06.151 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:06.151 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:06.151 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T20:56:06.151 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:51.863452+0000", 2023-09-23T20:56:06.151 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.0010000000000003e-06 2023-09-23T20:56:06.151 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:06.151 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:06.152 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T20:56:06.152 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:51.863455+0000", 2023-09-23T20:56:06.152 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.9689999999999999e-06 2023-09-23T20:56:06.152 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:06.152 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:06.152 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:56:06.152 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:51.872125+0000", 2023-09-23T20:56:06.152 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0086700519999999993 2023-09-23T20:56:06.152 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:06.152 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:06.152 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T20:56:06.152 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:51.872301+0000", 2023-09-23T20:56:06.152 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000176219 2023-09-23T20:56:06.153 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:06.153 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:06.153 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 3,6", 2023-09-23T20:56:06.153 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:51.872462+0000", 2023-09-23T20:56:06.153 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00016080900000000001 2023-09-23T20:56:06.153 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:06.153 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:06.153 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:56:06.153 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:51.893483+0000", 2023-09-23T20:56:06.153 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.021020575 2023-09-23T20:56:06.153 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:06.153 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:06.153 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:56:06.154 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:51.896008+0000", 2023-09-23T20:56:06.154 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0025256250000000001 2023-09-23T20:56:06.154 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:06.154 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:06.154 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T20:56:06.154 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:51.934560+0000", 2023-09-23T20:56:06.154 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.038551821999999999 2023-09-23T20:56:06.154 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:06.154 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:06.154 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T20:56:06.154 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:51.934605+0000", 2023-09-23T20:56:06.155 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.4814000000000002e-05 2023-09-23T20:56:06.155 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:06.155 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:06.155 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T20:56:06.155 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:53:51.936604+0000", 2023-09-23T20:56:06.155 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0019993440000000001 2023-09-23T20:56:06.155 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:56:06.155 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T20:56:06.155 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:56:06.155 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:06.155 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:06.155 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34797.0:114968 6.f 6:fe20b540:::benchmark_data_smithi026_223868_object13842:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e131)", 2023-09-23T20:56:06.156 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T20:54:12.903762+0000", 2023-09-23T20:56:06.156 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 113.204720444, 2023-09-23T20:56:06.156 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1643464619999999, 2023-09-23T20:56:06.156 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T20:56:06.156 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "reached pg", 2023-09-23T20:56:06.156 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T20:56:06.156 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34797", 2023-09-23T20:56:06.156 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T20:56:06.156 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 114968 2023-09-23T20:56:06.156 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:06.156 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T20:56:06.156 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:06.156 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T20:56:06.156 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:12.903762+0000", 2023-09-23T20:56:06.157 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:56:06.157 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:06.157 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:06.157 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T20:56:06.157 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:12.903762+0000", 2023-09-23T20:56:06.157 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:56:06.157 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:06.157 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:06.157 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T20:56:06.157 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:12.903760+0000", 2023-09-23T20:56:06.157 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999981 2023-09-23T20:56:06.157 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:06.157 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:06.157 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T20:56:06.158 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:12.903769+0000", 2023-09-23T20:56:06.158 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.772e-06 2023-09-23T20:56:06.158 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:06.158 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:06.158 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T20:56:06.158 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:12.903784+0000", 2023-09-23T20:56:06.158 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.5063e-05 2023-09-23T20:56:06.158 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:06.158 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:06.158 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T20:56:06.158 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:12.903794+0000", 2023-09-23T20:56:06.158 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0274e-05 2023-09-23T20:56:06.159 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:06.159 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:06.159 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:56:06.159 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:12.903882+0000", 2023-09-23T20:56:06.159 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.7639999999999994e-05 2023-09-23T20:56:06.159 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:06.159 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:06.159 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T20:56:06.159 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:12.904022+0000", 2023-09-23T20:56:06.159 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000140724 2023-09-23T20:56:06.159 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:06.159 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:06.160 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 3,6", 2023-09-23T20:56:06.160 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:12.904139+0000", 2023-09-23T20:56:06.160 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000116762 2023-09-23T20:56:06.160 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:06.160 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:06.160 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T20:56:06.160 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:12.907761+0000", 2023-09-23T20:56:06.160 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0036214770000000001 2023-09-23T20:56:06.160 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:06.160 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:06.160 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:56:06.160 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:12.921765+0000", 2023-09-23T20:56:06.161 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.014004312 2023-09-23T20:56:06.161 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:06.161 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:06.161 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:56:06.161 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:12.950178+0000", 2023-09-23T20:56:06.161 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.028413560000000001 2023-09-23T20:56:06.161 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:06.161 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:06.161 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T20:56:06.161 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:12.950192+0000", 2023-09-23T20:56:06.161 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.3509000000000001e-05 2023-09-23T20:56:06.161 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:06.162 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:06.162 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:56:06.162 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:14.018502+0000", 2023-09-23T20:56:06.162 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0683096089999999 2023-09-23T20:56:06.162 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:06.162 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:06.162 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for missing object", 2023-09-23T20:56:06.162 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:14.018606+0000", 2023-09-23T20:56:06.162 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000104041 2023-09-23T20:56:06.162 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:06.162 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:06.162 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:56:06.163 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:14.068012+0000", 2023-09-23T20:56:06.163 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.049406377000000001 2023-09-23T20:56:06.163 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:06.163 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:06.163 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T20:56:06.163 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:14.068108+0000", 2023-09-23T20:56:06.163 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.6364000000000001e-05 2023-09-23T20:56:06.163 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:56:06.163 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T20:56:06.163 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:56:06.163 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:06.163 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:06.164 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34797.0:114973 6.f 6:f527467a:::benchmark_data_smithi026_223868_object13847:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e131)", 2023-09-23T20:56:06.164 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T20:54:12.913308+0000", 2023-09-23T20:56:06.164 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 113.195174698, 2023-09-23T20:56:06.164 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1662446850000001, 2023-09-23T20:56:06.164 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T20:56:06.164 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "reached pg", 2023-09-23T20:56:06.164 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T20:56:06.164 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34797", 2023-09-23T20:56:06.164 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T20:56:06.164 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 114973 2023-09-23T20:56:06.164 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:06.164 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T20:56:06.164 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:06.165 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T20:56:06.165 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:12.913308+0000", 2023-09-23T20:56:06.165 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:56:06.165 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:06.165 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:06.165 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T20:56:06.165 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:12.913308+0000", 2023-09-23T20:56:06.165 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:56:06.165 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:06.165 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:06.165 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T20:56:06.165 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:12.913306+0000", 2023-09-23T20:56:06.165 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T20:56:06.166 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:06.166 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:06.166 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T20:56:06.166 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:12.913314+0000", 2023-09-23T20:56:06.166 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.9820000000000008e-06 2023-09-23T20:56:06.166 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:06.166 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:06.166 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T20:56:06.166 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:12.913326+0000", 2023-09-23T20:56:06.166 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1749999999999999e-05 2023-09-23T20:56:06.166 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:06.166 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:06.166 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T20:56:06.167 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:12.913334+0000", 2023-09-23T20:56:06.167 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.7640000000000005e-06 2023-09-23T20:56:06.167 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:06.167 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:06.167 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:56:06.167 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:12.913407+0000", 2023-09-23T20:56:06.167 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.3134999999999997e-05 2023-09-23T20:56:06.167 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:06.167 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:06.167 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T20:56:06.167 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:12.913576+0000", 2023-09-23T20:56:06.167 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00016879199999999999 2023-09-23T20:56:06.167 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:06.168 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:06.168 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 3,6", 2023-09-23T20:56:06.168 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:12.913707+0000", 2023-09-23T20:56:06.168 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000131168 2023-09-23T20:56:06.168 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:06.168 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:06.168 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T20:56:06.168 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:12.918380+0000", 2023-09-23T20:56:06.168 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0046727549999999998 2023-09-23T20:56:06.168 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:06.168 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:06.168 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:56:06.169 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:12.927220+0000", 2023-09-23T20:56:06.169 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0088402120000000001 2023-09-23T20:56:06.169 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:06.169 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:06.169 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:56:06.169 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:12.950224+0000", 2023-09-23T20:56:06.169 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.023004 2023-09-23T20:56:06.169 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:06.169 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:06.169 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T20:56:06.169 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:12.950233+0000", 2023-09-23T20:56:06.169 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.4199999999999996e-06 2023-09-23T20:56:06.169 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:06.170 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:06.170 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:56:06.170 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:14.068138+0000", 2023-09-23T20:56:06.170 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.117904504 2023-09-23T20:56:06.170 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:06.170 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:06.170 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for missing object", 2023-09-23T20:56:06.170 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:14.068256+0000", 2023-09-23T20:56:06.170 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000117847 2023-09-23T20:56:06.170 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:06.170 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:06.170 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:56:06.171 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:14.079477+0000", 2023-09-23T20:56:06.171 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.011221287 2023-09-23T20:56:06.171 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:06.171 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:06.171 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T20:56:06.171 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:14.079552+0000", 2023-09-23T20:56:06.171 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.5480000000000002e-05 2023-09-23T20:56:06.171 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:56:06.171 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T20:56:06.171 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:56:06.171 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:06.171 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:06.171 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34797.0:114979 6.f 6:f96f8492:::benchmark_data_smithi026_223868_object13853:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e131)", 2023-09-23T20:56:06.171 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T20:54:12.932449+0000", 2023-09-23T20:56:06.172 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 113.17603343099999, 2023-09-23T20:56:06.172 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.152965391, 2023-09-23T20:56:06.172 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T20:56:06.172 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "reached pg", 2023-09-23T20:56:06.172 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T20:56:06.172 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34797", 2023-09-23T20:56:06.172 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T20:56:06.172 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 114979 2023-09-23T20:56:06.172 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:06.172 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T20:56:06.172 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:06.173 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T20:56:06.173 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:12.932449+0000", 2023-09-23T20:56:06.173 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:56:06.173 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:06.173 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:06.173 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T20:56:06.173 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:12.932449+0000", 2023-09-23T20:56:06.173 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:56:06.173 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:06.173 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:06.173 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T20:56:06.173 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:12.932447+0000", 2023-09-23T20:56:06.173 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999981 2023-09-23T20:56:06.174 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:06.174 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:06.174 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T20:56:06.174 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:12.932457+0000", 2023-09-23T20:56:06.174 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0331e-05 2023-09-23T20:56:06.174 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:06.174 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:06.174 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T20:56:06.174 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:12.932489+0000", 2023-09-23T20:56:06.174 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.2135999999999999e-05 2023-09-23T20:56:06.174 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:06.174 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:06.175 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T20:56:06.175 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:12.932501+0000", 2023-09-23T20:56:06.175 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1969e-05 2023-09-23T20:56:06.175 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:06.175 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:06.175 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:56:06.175 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:12.932581+0000", 2023-09-23T20:56:06.175 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.9510000000000003e-05 2023-09-23T20:56:06.175 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:06.175 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:06.175 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T20:56:06.175 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:12.932696+0000", 2023-09-23T20:56:06.175 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00011554599999999999 2023-09-23T20:56:06.176 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:06.176 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:06.176 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 3,6", 2023-09-23T20:56:06.176 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:12.932783+0000", 2023-09-23T20:56:06.176 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.6712000000000006e-05 2023-09-23T20:56:06.176 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:06.176 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:06.176 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:56:06.176 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:12.950261+0000", 2023-09-23T20:56:06.176 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.017477724 2023-09-23T20:56:06.176 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:06.176 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:06.176 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T20:56:06.177 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:12.950270+0000", 2023-09-23T20:56:06.177 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.2900000000000008e-06 2023-09-23T20:56:06.177 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:06.177 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:06.177 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:56:06.177 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:14.068290+0000", 2023-09-23T20:56:06.177 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1180193949999999 2023-09-23T20:56:06.177 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:06.177 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:06.177 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for missing object", 2023-09-23T20:56:06.177 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:14.068456+0000", 2023-09-23T20:56:06.177 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000166627 2023-09-23T20:56:06.177 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:06.178 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:06.178 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:56:06.178 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:14.085291+0000", 2023-09-23T20:56:06.178 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.016835249 2023-09-23T20:56:06.178 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:06.178 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:06.178 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T20:56:06.178 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:14.085414+0000", 2023-09-23T20:56:06.178 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000122853 2023-09-23T20:56:06.178 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:56:06.178 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T20:56:06.178 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:56:06.178 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:06.179 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:06.179 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34797.0:114981 6.f 6:f98e4163:::benchmark_data_smithi026_223868_object13855:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e131)", 2023-09-23T20:56:06.179 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T20:54:12.932519+0000", 2023-09-23T20:56:06.179 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 113.175963638, 2023-09-23T20:56:06.179 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.154672455, 2023-09-23T20:56:06.179 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T20:56:06.179 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "reached pg", 2023-09-23T20:56:06.179 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T20:56:06.179 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34797", 2023-09-23T20:56:06.179 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T20:56:06.179 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 114981 2023-09-23T20:56:06.179 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:06.180 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T20:56:06.180 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:06.180 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T20:56:06.180 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:12.932519+0000", 2023-09-23T20:56:06.180 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:56:06.180 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:06.180 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:06.180 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T20:56:06.180 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:12.932519+0000", 2023-09-23T20:56:06.180 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:56:06.180 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:06.180 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:06.180 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T20:56:06.181 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:12.932518+0000", 2023-09-23T20:56:06.181 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T20:56:06.181 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:06.181 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:06.181 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T20:56:06.181 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:12.932522+0000", 2023-09-23T20:56:06.181 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.5219999999999999e-06 2023-09-23T20:56:06.181 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:06.181 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:06.181 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T20:56:06.181 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:12.932529+0000", 2023-09-23T20:56:06.181 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.8940000000000004e-06 2023-09-23T20:56:06.182 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:06.182 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:06.182 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T20:56:06.182 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:12.932534+0000", 2023-09-23T20:56:06.182 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.2190000000000003e-06 2023-09-23T20:56:06.182 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:06.182 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:06.182 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:56:06.182 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:12.936410+0000", 2023-09-23T20:56:06.182 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0038765879999999998 2023-09-23T20:56:06.182 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:06.182 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:06.182 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T20:56:06.183 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:12.936579+0000", 2023-09-23T20:56:06.183 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000168844 2023-09-23T20:56:06.183 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:06.183 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:06.183 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 3,6", 2023-09-23T20:56:06.183 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:12.936731+0000", 2023-09-23T20:56:06.183 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00015202799999999999 2023-09-23T20:56:06.183 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:06.183 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:06.183 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:56:06.183 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:12.950298+0000", 2023-09-23T20:56:06.183 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.013566686 2023-09-23T20:56:06.184 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:06.184 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:06.184 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T20:56:06.184 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:12.950307+0000", 2023-09-23T20:56:06.184 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.8610000000000004e-06 2023-09-23T20:56:06.184 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:06.184 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:06.184 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:56:06.184 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:14.068540+0000", 2023-09-23T20:56:06.184 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1182336260000001 2023-09-23T20:56:06.184 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:06.184 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:06.184 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for missing object", 2023-09-23T20:56:06.185 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:14.068707+0000", 2023-09-23T20:56:06.185 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00016695499999999999 2023-09-23T20:56:06.185 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:06.185 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:06.185 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:56:06.185 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:14.087135+0000", 2023-09-23T20:56:06.185 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.018427507999999999 2023-09-23T20:56:06.185 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:06.185 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:06.185 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T20:56:06.185 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:14.087191+0000", 2023-09-23T20:56:06.185 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.6329000000000002e-05 2023-09-23T20:56:06.185 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:56:06.186 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T20:56:06.186 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:56:06.186 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:06.186 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:06.186 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34797.0:115000 6.f 6:feaa8694:::benchmark_data_smithi026_223868_object13874:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e132)", 2023-09-23T20:56:06.186 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T20:54:12.985062+0000", 2023-09-23T20:56:06.186 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 113.123419906, 2023-09-23T20:56:06.186 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.100759915, 2023-09-23T20:56:06.186 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T20:56:06.186 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:56:06.186 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T20:56:06.186 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34797", 2023-09-23T20:56:06.187 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T20:56:06.187 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 115000 2023-09-23T20:56:06.187 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:06.187 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T20:56:06.187 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:06.187 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T20:56:06.187 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:12.985062+0000", 2023-09-23T20:56:06.187 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:56:06.187 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:06.187 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:06.187 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T20:56:06.188 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:12.985062+0000", 2023-09-23T20:56:06.188 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:56:06.188 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:06.188 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:06.188 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T20:56:06.188 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:12.985060+0000", 2023-09-23T20:56:06.188 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999976 2023-09-23T20:56:06.188 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:06.188 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:06.188 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T20:56:06.188 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:12.985071+0000", 2023-09-23T20:56:06.189 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0922e-05 2023-09-23T20:56:06.189 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:06.189 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:06.189 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T20:56:06.189 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:12.985091+0000", 2023-09-23T20:56:06.189 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.0228e-05 2023-09-23T20:56:06.189 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:06.189 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:06.189 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T20:56:06.189 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:12.985103+0000", 2023-09-23T20:56:06.189 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1773000000000001e-05 2023-09-23T20:56:06.189 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:06.190 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:06.190 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:56:06.190 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:12.986211+0000", 2023-09-23T20:56:06.190 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0011084770000000001 2023-09-23T20:56:06.190 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:06.190 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:06.190 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T20:56:06.190 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:12.986223+0000", 2023-09-23T20:56:06.190 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1134999999999999e-05 2023-09-23T20:56:06.190 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:06.190 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:06.190 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:56:06.191 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:14.068739+0000", 2023-09-23T20:56:06.191 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0825159499999999 2023-09-23T20:56:06.191 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:06.191 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:06.191 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T20:56:06.191 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:14.068895+0000", 2023-09-23T20:56:06.191 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00015601699999999999 2023-09-23T20:56:06.191 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:06.191 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:06.191 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 3,4", 2023-09-23T20:56:06.191 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:14.069072+0000", 2023-09-23T20:56:06.191 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00017741799999999999 2023-09-23T20:56:06.192 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:06.192 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:06.192 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T20:56:06.192 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:14.079431+0000", 2023-09-23T20:56:06.192 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.010358539999999999 2023-09-23T20:56:06.192 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:06.192 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:06.192 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:56:06.192 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:14.084053+0000", 2023-09-23T20:56:06.192 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0046225789999999999 2023-09-23T20:56:06.192 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:06.192 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:06.193 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:56:06.193 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:14.085446+0000", 2023-09-23T20:56:06.193 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.001392403 2023-09-23T20:56:06.193 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:06.193 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:06.193 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T20:56:06.193 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:14.085514+0000", 2023-09-23T20:56:06.193 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.8056e-05 2023-09-23T20:56:06.193 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:06.193 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:06.193 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T20:56:06.193 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:14.085822+0000", 2023-09-23T20:56:06.193 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000308795 2023-09-23T20:56:06.194 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:56:06.194 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T20:56:06.194 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:56:06.194 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:06.194 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:06.194 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34797.0:115007 6.f 6:f1e2704f:::benchmark_data_smithi026_223868_object13881:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e132)", 2023-09-23T20:56:06.194 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T20:54:13.004902+0000", 2023-09-23T20:56:06.194 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 113.10358018300001, 2023-09-23T20:56:06.194 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0814330700000001, 2023-09-23T20:56:06.194 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T20:56:06.194 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:56:06.194 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T20:56:06.195 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34797", 2023-09-23T20:56:06.195 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T20:56:06.195 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 115007 2023-09-23T20:56:06.195 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:06.195 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T20:56:06.195 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:06.195 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T20:56:06.195 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:13.004902+0000", 2023-09-23T20:56:06.195 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:56:06.195 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:06.195 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:06.195 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T20:56:06.196 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:13.004902+0000", 2023-09-23T20:56:06.196 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:56:06.196 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:06.196 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:06.196 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T20:56:06.196 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:13.004901+0000", 2023-09-23T20:56:06.196 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T20:56:06.196 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:06.196 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:06.196 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T20:56:06.196 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:13.004908+0000", 2023-09-23T20:56:06.196 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.8599999999999993e-06 2023-09-23T20:56:06.196 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:06.197 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:06.197 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T20:56:06.197 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:13.004922+0000", 2023-09-23T20:56:06.197 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.3678e-05 2023-09-23T20:56:06.197 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:06.197 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:06.197 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T20:56:06.197 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:13.004929+0000", 2023-09-23T20:56:06.197 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.8770000000000004e-06 2023-09-23T20:56:06.197 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:06.197 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:06.197 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:56:06.197 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:13.004992+0000", 2023-09-23T20:56:06.197 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.2823000000000005e-05 2023-09-23T20:56:06.198 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:06.198 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:06.198 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T20:56:06.198 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:13.005003+0000", 2023-09-23T20:56:06.198 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1345999999999999e-05 2023-09-23T20:56:06.198 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:06.198 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:06.198 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:56:06.198 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:14.072238+0000", 2023-09-23T20:56:06.198 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.067234351 2023-09-23T20:56:06.198 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:06.199 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:06.199 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T20:56:06.199 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:14.072395+0000", 2023-09-23T20:56:06.199 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00015737999999999999 2023-09-23T20:56:06.199 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:06.199 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:06.199 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 3,4", 2023-09-23T20:56:06.199 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:14.072533+0000", 2023-09-23T20:56:06.199 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000137758 2023-09-23T20:56:06.199 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:06.199 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:06.200 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T20:56:06.200 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:14.079442+0000", 2023-09-23T20:56:06.200 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0069094589999999997 2023-09-23T20:56:06.200 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:06.200 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:06.200 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:56:06.200 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:14.084446+0000", 2023-09-23T20:56:06.200 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0050034479999999997 2023-09-23T20:56:06.200 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:06.200 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:06.200 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:56:06.200 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:14.085889+0000", 2023-09-23T20:56:06.200 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.001443819 2023-09-23T20:56:06.201 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:06.201 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:06.201 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T20:56:06.201 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:14.085956+0000", 2023-09-23T20:56:06.201 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.6622e-05 2023-09-23T20:56:06.201 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:06.201 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:06.201 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T20:56:06.201 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:14.086335+0000", 2023-09-23T20:56:06.201 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00037926700000000003 2023-09-23T20:56:06.201 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:56:06.201 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T20:56:06.201 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:56:06.201 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:06.202 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:06.202 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34797.0:115008 6.f 6:f2635b96:::benchmark_data_smithi026_223868_object13882:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e132)", 2023-09-23T20:56:06.202 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T20:54:13.004945+0000", 2023-09-23T20:56:06.202 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 113.103537272, 2023-09-23T20:56:06.202 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0825950639999999, 2023-09-23T20:56:06.202 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T20:56:06.202 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:56:06.202 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T20:56:06.202 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34797", 2023-09-23T20:56:06.202 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T20:56:06.202 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 115008 2023-09-23T20:56:06.202 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:06.202 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T20:56:06.203 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:06.203 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T20:56:06.203 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:13.004945+0000", 2023-09-23T20:56:06.203 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:56:06.203 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:06.203 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:06.203 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T20:56:06.203 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:13.004945+0000", 2023-09-23T20:56:06.203 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:56:06.203 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:06.203 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:06.203 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T20:56:06.203 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:13.004945+0000", 2023-09-23T20:56:06.204 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T20:56:06.204 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:06.204 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:06.204 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T20:56:06.204 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:13.004947+0000", 2023-09-23T20:56:06.204 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.7949999999999999e-06 2023-09-23T20:56:06.204 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:06.204 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:06.204 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T20:56:06.204 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:13.004954+0000", 2023-09-23T20:56:06.204 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.8190000000000003e-06 2023-09-23T20:56:06.204 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:06.204 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:06.204 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T20:56:06.205 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:13.004960+0000", 2023-09-23T20:56:06.205 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.5609999999999998e-06 2023-09-23T20:56:06.205 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:06.205 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:06.205 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:56:06.205 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:13.005034+0000", 2023-09-23T20:56:06.205 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.3818000000000004e-05 2023-09-23T20:56:06.205 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:06.205 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:06.205 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T20:56:06.205 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:13.005043+0000", 2023-09-23T20:56:06.205 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.9930000000000001e-06 2023-09-23T20:56:06.206 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:06.206 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:06.206 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:56:06.206 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:14.075534+0000", 2023-09-23T20:56:06.206 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0704913330000001 2023-09-23T20:56:06.206 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:06.206 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:06.206 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T20:56:06.206 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:14.075709+0000", 2023-09-23T20:56:06.206 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00017482899999999999 2023-09-23T20:56:06.206 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:06.206 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:06.207 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 3,4", 2023-09-23T20:56:06.207 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:14.075860+0000", 2023-09-23T20:56:06.207 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00015149599999999999 2023-09-23T20:56:06.207 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:06.207 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:06.207 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T20:56:06.207 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:14.079734+0000", 2023-09-23T20:56:06.207 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.003873816 2023-09-23T20:56:06.207 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:06.207 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:06.207 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:56:06.207 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:14.084488+0000", 2023-09-23T20:56:06.207 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0047534150000000004 2023-09-23T20:56:06.207 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:06.208 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:06.208 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:56:06.208 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:14.087241+0000", 2023-09-23T20:56:06.208 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.002753288 2023-09-23T20:56:06.208 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:06.208 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:06.208 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T20:56:06.208 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:14.087280+0000", 2023-09-23T20:56:06.208 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.9647000000000002e-05 2023-09-23T20:56:06.208 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:06.208 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:06.209 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T20:56:06.209 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:14.087540+0000", 2023-09-23T20:56:06.209 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00025969900000000001 2023-09-23T20:56:06.209 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:56:06.209 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T20:56:06.209 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:56:06.209 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:06.209 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:06.209 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34797.0:114997 6.12 6:49f2a314:::benchmark_data_smithi026_223868_object13871:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e133)", 2023-09-23T20:56:06.209 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T20:54:13.930512+0000", 2023-09-23T20:56:06.209 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 112.177970049, 2023-09-23T20:56:06.209 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0903023789999999, 2023-09-23T20:56:06.209 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T20:56:06.210 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:56:06.210 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T20:56:06.210 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34797", 2023-09-23T20:56:06.210 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T20:56:06.210 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 114997 2023-09-23T20:56:06.210 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:06.210 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T20:56:06.210 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:06.210 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T20:56:06.210 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:13.930512+0000", 2023-09-23T20:56:06.210 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:56:06.210 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:06.211 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:06.211 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T20:56:06.211 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:13.930512+0000", 2023-09-23T20:56:06.211 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:56:06.211 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:06.211 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:06.211 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T20:56:06.211 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:13.930510+0000", 2023-09-23T20:56:06.211 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999981 2023-09-23T20:56:06.211 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:06.211 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:06.211 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T20:56:06.211 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:13.930520+0000", 2023-09-23T20:56:06.212 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.6630000000000005e-06 2023-09-23T20:56:06.212 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:06.212 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:06.212 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T20:56:06.212 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:13.930540+0000", 2023-09-23T20:56:06.212 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.0469e-05 2023-09-23T20:56:06.212 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:06.212 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:06.212 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T20:56:06.212 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:13.930556+0000", 2023-09-23T20:56:06.212 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.5702999999999999e-05 2023-09-23T20:56:06.212 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:06.213 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:06.213 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:56:06.213 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:13.936376+0000", 2023-09-23T20:56:06.213 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0058194709999999997 2023-09-23T20:56:06.213 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:06.213 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:06.213 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T20:56:06.213 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:13.936391+0000", 2023-09-23T20:56:06.213 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.5276000000000001e-05 2023-09-23T20:56:06.213 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:06.213 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:06.213 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:56:06.213 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:14.999262+0000", 2023-09-23T20:56:06.213 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.062871597 2023-09-23T20:56:06.214 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:06.214 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:06.214 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T20:56:06.214 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:14.999394+0000", 2023-09-23T20:56:06.214 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000132008 2023-09-23T20:56:06.214 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:06.214 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:06.214 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 3,4", 2023-09-23T20:56:06.214 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:14.999530+0000", 2023-09-23T20:56:06.214 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000135718 2023-09-23T20:56:06.214 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:06.215 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:06.215 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:56:06.215 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:15.010731+0000", 2023-09-23T20:56:06.215 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.01120052 2023-09-23T20:56:06.215 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:06.215 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:06.215 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:56:06.215 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:15.012838+0000", 2023-09-23T20:56:06.215 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0021071369999999998 2023-09-23T20:56:06.215 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:06.215 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:06.215 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T20:56:06.215 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:15.020280+0000", 2023-09-23T20:56:06.216 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0074424410000000002 2023-09-23T20:56:06.216 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:06.216 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:06.216 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T20:56:06.216 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:15.020331+0000", 2023-09-23T20:56:06.216 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.0407e-05 2023-09-23T20:56:06.216 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:06.216 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:06.216 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T20:56:06.216 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:15.020815+0000", 2023-09-23T20:56:06.216 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00048407799999999998 2023-09-23T20:56:06.217 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:56:06.217 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T20:56:06.217 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:56:06.217 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:06.217 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:06.217 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34797.0:115054 6.2 6:477c9927:::benchmark_data_smithi026_223868_object13928:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e133)", 2023-09-23T20:56:06.217 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T20:54:14.131922+0000", 2023-09-23T20:56:06.217 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 111.976559915, 2023-09-23T20:56:06.217 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.90045666199999996, 2023-09-23T20:56:06.217 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T20:56:06.217 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:56:06.217 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T20:56:06.217 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34797", 2023-09-23T20:56:06.218 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T20:56:06.218 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 115054 2023-09-23T20:56:06.218 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:06.218 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T20:56:06.218 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:06.218 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T20:56:06.218 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:14.131922+0000", 2023-09-23T20:56:06.218 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:56:06.218 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:06.218 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:06.218 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T20:56:06.218 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:14.131922+0000", 2023-09-23T20:56:06.219 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:56:06.219 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:06.219 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:06.219 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T20:56:06.219 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:14.131921+0000", 2023-09-23T20:56:06.219 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999981 2023-09-23T20:56:06.219 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:06.219 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:06.219 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T20:56:06.219 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:14.131928+0000", 2023-09-23T20:56:06.219 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.2259999999999999e-06 2023-09-23T20:56:06.219 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:06.219 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:06.220 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T20:56:06.220 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:14.131944+0000", 2023-09-23T20:56:06.220 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.5557e-05 2023-09-23T20:56:06.220 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:06.220 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:06.220 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T20:56:06.220 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:14.131951+0000", 2023-09-23T20:56:06.220 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.7800000000000001e-06 2023-09-23T20:56:06.220 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:06.220 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:06.220 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:56:06.220 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:14.132008+0000", 2023-09-23T20:56:06.221 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.6441999999999999e-05 2023-09-23T20:56:06.221 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:06.221 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:06.221 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T20:56:06.221 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:14.132019+0000", 2023-09-23T20:56:06.221 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1345999999999999e-05 2023-09-23T20:56:06.221 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:06.221 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:06.221 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:56:06.221 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:15.010763+0000", 2023-09-23T20:56:06.221 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.87874358900000005 2023-09-23T20:56:06.221 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:06.222 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:06.222 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T20:56:06.222 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:15.010901+0000", 2023-09-23T20:56:06.222 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00013785100000000001 2023-09-23T20:56:06.222 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:06.222 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:06.222 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 3,4", 2023-09-23T20:56:06.222 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:15.011033+0000", 2023-09-23T20:56:06.222 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00013242000000000001 2023-09-23T20:56:06.222 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:06.222 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:06.222 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:56:06.222 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:15.018341+0000", 2023-09-23T20:56:06.223 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0073078370000000002 2023-09-23T20:56:06.223 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:06.223 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:06.223 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T20:56:06.223 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:15.020828+0000", 2023-09-23T20:56:06.223 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.002487586 2023-09-23T20:56:06.223 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:06.223 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:06.223 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:56:06.223 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:15.031849+0000", 2023-09-23T20:56:06.223 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.011020416 2023-09-23T20:56:06.223 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:06.224 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:06.224 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T20:56:06.224 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:15.031918+0000", 2023-09-23T20:56:06.224 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.9061999999999997e-05 2023-09-23T20:56:06.224 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:06.224 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:06.224 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T20:56:06.224 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:15.032379+0000", 2023-09-23T20:56:06.224 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00046126700000000001 2023-09-23T20:56:06.224 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:56:06.224 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T20:56:06.224 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:56:06.225 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:06.225 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:06.225 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34797.0:115060 6.12 6:4c652c33:::benchmark_data_smithi026_223868_object13934:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e133)", 2023-09-23T20:56:06.225 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T20:54:14.144100+0000", 2023-09-23T20:56:06.225 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 111.964382376, 2023-09-23T20:56:06.225 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.88882830499999999, 2023-09-23T20:56:06.225 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T20:56:06.225 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:56:06.225 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T20:56:06.225 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34797", 2023-09-23T20:56:06.225 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T20:56:06.225 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 115060 2023-09-23T20:56:06.225 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:06.226 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T20:56:06.226 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:06.226 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T20:56:06.226 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:14.144100+0000", 2023-09-23T20:56:06.226 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:56:06.226 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:06.226 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:06.226 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T20:56:06.226 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:14.144100+0000", 2023-09-23T20:56:06.226 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:56:06.226 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:06.226 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:06.227 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T20:56:06.227 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:14.144099+0000", 2023-09-23T20:56:06.227 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T20:56:06.227 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:06.227 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:06.227 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T20:56:06.227 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:14.144105+0000", 2023-09-23T20:56:06.227 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.409e-06 2023-09-23T20:56:06.227 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:06.227 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:06.227 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T20:56:06.227 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:14.144121+0000", 2023-09-23T20:56:06.227 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.5832e-05 2023-09-23T20:56:06.227 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:06.228 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:06.228 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T20:56:06.228 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:14.144129+0000", 2023-09-23T20:56:06.228 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.3480000000000004e-06 2023-09-23T20:56:06.228 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:06.228 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:06.228 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:56:06.228 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:14.144193+0000", 2023-09-23T20:56:06.228 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.3899999999999995e-05 2023-09-23T20:56:06.228 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:06.228 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:06.228 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T20:56:06.228 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:14.144207+0000", 2023-09-23T20:56:06.228 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.393e-05 2023-09-23T20:56:06.229 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:06.229 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:06.229 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:56:06.229 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:15.012864+0000", 2023-09-23T20:56:06.229 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.868656596 2023-09-23T20:56:06.229 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:06.229 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:06.229 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T20:56:06.229 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:15.012999+0000", 2023-09-23T20:56:06.229 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00013505900000000001 2023-09-23T20:56:06.229 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:06.229 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:06.229 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 3,4", 2023-09-23T20:56:06.229 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:15.013127+0000", 2023-09-23T20:56:06.230 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000128689 2023-09-23T20:56:06.230 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:06.230 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:06.230 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T20:56:06.230 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:15.020836+0000", 2023-09-23T20:56:06.230 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0077085089999999997 2023-09-23T20:56:06.230 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:06.230 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:06.230 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:56:06.230 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:15.022545+0000", 2023-09-23T20:56:06.230 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0017095280000000001 2023-09-23T20:56:06.230 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:06.231 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:06.231 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:56:06.231 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:15.032424+0000", 2023-09-23T20:56:06.231 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0098784930000000003 2023-09-23T20:56:06.231 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:06.231 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:06.231 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T20:56:06.231 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:15.032483+0000", 2023-09-23T20:56:06.231 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.8903000000000002e-05 2023-09-23T20:56:06.231 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:06.231 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:06.231 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T20:56:06.232 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:15.032928+0000", 2023-09-23T20:56:06.232 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00044556499999999998 2023-09-23T20:56:06.232 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:56:06.232 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T20:56:06.232 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:56:06.232 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:06.232 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:06.232 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34797.0:115081 6.12 6:4a57480c:::benchmark_data_smithi026_223868_object13955:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e133)", 2023-09-23T20:56:06.232 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T20:54:14.203254+0000", 2023-09-23T20:56:06.232 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 111.905228417, 2023-09-23T20:56:06.232 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.82998758699999997, 2023-09-23T20:56:06.233 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T20:56:06.233 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:56:06.233 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T20:56:06.233 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34797", 2023-09-23T20:56:06.233 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T20:56:06.233 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 115081 2023-09-23T20:56:06.233 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:06.233 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T20:56:06.233 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:06.233 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T20:56:06.233 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:14.203254+0000", 2023-09-23T20:56:06.233 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:56:06.234 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:06.234 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:06.234 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T20:56:06.234 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:14.203254+0000", 2023-09-23T20:56:06.234 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:56:06.234 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:06.234 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:06.234 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T20:56:06.234 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:14.203253+0000", 2023-09-23T20:56:06.234 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T20:56:06.234 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:06.234 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:06.234 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T20:56:06.235 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:14.203258+0000", 2023-09-23T20:56:06.235 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.3709999999999999e-06 2023-09-23T20:56:06.235 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:06.235 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:06.235 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T20:56:06.235 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:14.203271+0000", 2023-09-23T20:56:06.235 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.344e-05 2023-09-23T20:56:06.235 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:06.235 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:06.235 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T20:56:06.235 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:56:06.237 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:14.203277+0000", 2023-09-23T20:56:06.237 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.9039999999999997e-06 2023-09-23T20:56:06.237 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:06.237 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:06.237 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:56:06.237 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:14.203330+0000", 2023-09-23T20:56:06.237 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.3086e-05 2023-09-23T20:56:06.237 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:06.237 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:06.237 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T20:56:06.238 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:14.203340+0000", 2023-09-23T20:56:06.238 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.7089999999999997e-06 2023-09-23T20:56:06.238 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:06.238 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:06.238 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:56:06.238 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:15.014667+0000", 2023-09-23T20:56:06.238 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.81132689999999996 2023-09-23T20:56:06.238 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:06.238 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:06.238 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T20:56:06.238 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:15.014782+0000", 2023-09-23T20:56:06.239 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000115203 2023-09-23T20:56:06.239 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:06.239 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:06.239 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 3,4", 2023-09-23T20:56:06.239 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:15.014903+0000", 2023-09-23T20:56:06.239 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000120617 2023-09-23T20:56:06.239 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:06.239 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:06.239 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T20:56:06.239 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:15.020843+0000", 2023-09-23T20:56:06.239 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0059400110000000002 2023-09-23T20:56:06.239 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:06.239 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:06.239 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:56:06.240 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:15.022580+0000", 2023-09-23T20:56:06.240 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.001737335 2023-09-23T20:56:06.240 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:06.240 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:06.240 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:56:06.240 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:15.032969+0000", 2023-09-23T20:56:06.240 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.010388413000000001 2023-09-23T20:56:06.240 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:06.240 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:06.240 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T20:56:06.240 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:15.033021+0000", 2023-09-23T20:56:06.240 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.1974000000000001e-05 2023-09-23T20:56:06.241 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:06.241 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:06.241 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T20:56:06.241 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:15.033242+0000", 2023-09-23T20:56:06.241 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000220919 2023-09-23T20:56:06.241 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:56:06.241 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T20:56:06.241 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:56:06.241 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:06.241 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:06.241 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34797.0:154519 6.f 6:ffed1803:::benchmark_data_smithi026_223868_object53393:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e149)", 2023-09-23T20:56:06.241 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T20:55:04.852085+0000", 2023-09-23T20:56:06.242 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 61.256397773000003, 2023-09-23T20:56:06.242 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.074300168999999999, 2023-09-23T20:56:06.242 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T20:56:06.242 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:56:06.242 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T20:56:06.242 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34797", 2023-09-23T20:56:06.242 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T20:56:06.242 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 154519 2023-09-23T20:56:06.242 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:06.242 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T20:56:06.242 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:06.242 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T20:56:06.243 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:04.852085+0000", 2023-09-23T20:56:06.243 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:56:06.243 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:06.243 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:06.243 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T20:56:06.243 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:04.852085+0000", 2023-09-23T20:56:06.243 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:56:06.243 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:06.243 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:06.243 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T20:56:06.243 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:04.852084+0000", 2023-09-23T20:56:06.243 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.999999 2023-09-23T20:56:06.243 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:06.244 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:06.244 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T20:56:06.244 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:04.852089+0000", 2023-09-23T20:56:06.244 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.9860000000000002e-06 2023-09-23T20:56:06.244 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:06.244 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:06.244 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T20:56:06.244 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:04.852099+0000", 2023-09-23T20:56:06.244 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.7850000000000003e-06 2023-09-23T20:56:06.244 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:06.244 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:06.244 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T20:56:06.244 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:04.852105+0000", 2023-09-23T20:56:06.245 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.7100000000000001e-06 2023-09-23T20:56:06.245 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:06.245 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:06.245 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:56:06.245 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:04.908982+0000", 2023-09-23T20:56:06.245 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.056876610000000001 2023-09-23T20:56:06.245 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:06.245 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:06.245 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T20:56:06.245 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:04.909111+0000", 2023-09-23T20:56:06.245 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000129076 2023-09-23T20:56:06.245 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:06.246 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:06.246 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 3,4", 2023-09-23T20:56:06.246 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:04.909227+0000", 2023-09-23T20:56:06.246 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00011630500000000001 2023-09-23T20:56:06.246 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:06.246 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:06.246 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T20:56:06.246 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:04.916868+0000", 2023-09-23T20:56:06.246 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0076405839999999997 2023-09-23T20:56:06.246 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:06.246 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:06.246 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:56:06.247 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:04.917806+0000", 2023-09-23T20:56:06.247 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000938253 2023-09-23T20:56:06.247 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:06.247 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:06.247 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:56:06.247 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:04.926307+0000", 2023-09-23T20:56:06.247 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0085010580000000006 2023-09-23T20:56:06.247 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:06.247 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:06.247 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T20:56:06.247 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:04.926351+0000", 2023-09-23T20:56:06.247 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.3504000000000002e-05 2023-09-23T20:56:06.248 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:06.248 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:06.248 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T20:56:06.248 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:04.926385+0000", 2023-09-23T20:56:06.248 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.4035000000000002e-05 2023-09-23T20:56:06.248 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:56:06.248 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T20:56:06.248 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:56:06.248 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:06.248 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:06.248 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34797.0:154838 6.12 6:4d84fe3c:::benchmark_data_smithi026_223868_object53712:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e149)", 2023-09-23T20:56:06.248 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T20:55:05.248478+0000", 2023-09-23T20:56:06.249 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 60.860004326999999, 2023-09-23T20:56:06.249 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.092226136, 2023-09-23T20:56:06.249 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T20:56:06.249 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:56:06.249 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T20:56:06.249 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34797", 2023-09-23T20:56:06.249 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T20:56:06.249 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 154838 2023-09-23T20:56:06.249 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:06.249 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T20:56:06.249 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:06.249 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T20:56:06.250 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:05.248478+0000", 2023-09-23T20:56:06.250 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:56:06.250 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:06.250 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:06.250 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T20:56:06.250 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:05.248478+0000", 2023-09-23T20:56:06.250 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:56:06.250 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:06.250 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:06.250 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T20:56:06.250 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:05.248472+0000", 2023-09-23T20:56:06.250 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999938 2023-09-23T20:56:06.251 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:06.251 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:06.251 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T20:56:06.251 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:05.248483+0000", 2023-09-23T20:56:06.251 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1756999999999999e-05 2023-09-23T20:56:06.251 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:06.251 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:06.251 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T20:56:06.251 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:05.248496+0000", 2023-09-23T20:56:06.251 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.2918999999999999e-05 2023-09-23T20:56:06.251 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:06.251 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:06.251 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T20:56:06.251 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:05.248505+0000", 2023-09-23T20:56:06.252 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.0119999999999994e-06 2023-09-23T20:56:06.252 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:06.252 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:06.252 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:56:06.252 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:05.250879+0000", 2023-09-23T20:56:06.252 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0023737530000000001 2023-09-23T20:56:06.252 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:06.252 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:06.252 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T20:56:06.252 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:05.251017+0000", 2023-09-23T20:56:06.252 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000137871 2023-09-23T20:56:06.252 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:06.252 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:06.252 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 3,4", 2023-09-23T20:56:06.253 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:05.251139+0000", 2023-09-23T20:56:06.253 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000122421 2023-09-23T20:56:06.253 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:06.253 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:06.253 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:56:06.253 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:05.256902+0000", 2023-09-23T20:56:06.253 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.005762461 2023-09-23T20:56:06.253 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:06.253 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:06.253 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T20:56:06.253 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:05.256911+0000", 2023-09-23T20:56:06.253 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.7010000000000002e-06 2023-09-23T20:56:06.253 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:06.253 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:06.254 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:56:06.254 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:05.340551+0000", 2023-09-23T20:56:06.254 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.083640754999999997 2023-09-23T20:56:06.254 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:06.254 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:06.254 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T20:56:06.254 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:05.340601+0000", 2023-09-23T20:56:06.254 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.9908999999999999e-05 2023-09-23T20:56:06.254 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:06.254 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:06.254 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T20:56:06.254 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:05.340704+0000", 2023-09-23T20:56:06.255 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00010300499999999999 2023-09-23T20:56:06.255 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:56:06.255 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T20:56:06.255 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:56:06.255 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:06.255 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:06.255 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_repop(client.34797.0:154846 6.a e149/137 6:5c295e0d:::benchmark_data_smithi026_223868_object53720:head v 149'9747, mlcod=149'9744)", 2023-09-23T20:56:06.255 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T20:55:05.253838+0000", 2023-09-23T20:56:06.255 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 60.854644209, 2023-09-23T20:56:06.255 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.075777259, 2023-09-23T20:56:06.255 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T20:56:06.255 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:56:06.255 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T20:56:06.256 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:06.256 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T20:56:06.256 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:05.253838+0000", 2023-09-23T20:56:06.256 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:56:06.256 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:06.256 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:06.256 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T20:56:06.256 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:05.253838+0000", 2023-09-23T20:56:06.256 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:56:06.256 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:06.256 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:06.256 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T20:56:06.256 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:05.253837+0000", 2023-09-23T20:56:06.256 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T20:56:06.257 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:06.257 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:06.257 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T20:56:06.257 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:05.253843+0000", 2023-09-23T20:56:06.257 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.3809999999999998e-06 2023-09-23T20:56:06.257 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:06.257 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:06.257 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T20:56:06.257 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:05.253853+0000", 2023-09-23T20:56:06.257 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0441e-05 2023-09-23T20:56:06.257 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:06.257 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:06.257 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T20:56:06.257 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:05.253860+0000", 2023-09-23T20:56:06.258 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.5479999999999998e-06 2023-09-23T20:56:06.258 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:06.258 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:06.258 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:56:06.258 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:05.256897+0000", 2023-09-23T20:56:06.258 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.003037363 2023-09-23T20:56:06.258 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:06.258 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:06.258 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T20:56:06.258 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:05.256921+0000", 2023-09-23T20:56:06.258 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.3985000000000002e-05 2023-09-23T20:56:06.258 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:06.259 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:06.259 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T20:56:06.259 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:05.259848+0000", 2023-09-23T20:56:06.259 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.002926826 2023-09-23T20:56:06.259 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:06.259 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:06.259 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T20:56:06.259 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:05.329615+0000", 2023-09-23T20:56:06.259 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.069767398999999994 2023-09-23T20:56:06.259 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:56:06.259 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T20:56:06.259 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:56:06.260 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:56:06.260 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T20:56:06.260 INFO:teuthology.orchestra.run.smithi026.stdout:} 2023-09-23T20:56:06.267 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:56:06 smithi070 ceph-mon[168282]: pgmap v431: 109 pgs: 2 remapped+peering, 5 peering, 102 active+clean; 442 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-09-23T20:56:06.268 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:56:06 smithi070 ceph-mon[168282]: failed to encode map e184 with expected crc 2023-09-23T20:56:06.268 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:56:06 smithi070 ceph-mon[168282]: failed to encode map e184 with expected crc 2023-09-23T20:56:06.268 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:56:06 smithi070 ceph-mon[168282]: failed to encode map e184 with expected crc 2023-09-23T20:56:06.268 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:56:06 smithi070 ceph-mon[168282]: failed to encode map e184 with expected crc 2023-09-23T20:56:06.268 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:56:06 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/1446668024' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num", "val": "26"}]: dispatch 2023-09-23T20:56:06.268 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:56:06 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num", "val": "26"}]: dispatch 2023-09-23T20:56:06.268 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:56:06.318 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:56:06.324 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:56:06.324 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:56:06.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:56:06 smithi026 ceph-mon[179688]: pgmap v431: 109 pgs: 2 remapped+peering, 5 peering, 102 active+clean; 442 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-09-23T20:56:06.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:56:06 smithi026 ceph-mon[179688]: failed to encode map e184 with expected crc 2023-09-23T20:56:06.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:56:06 smithi026 ceph-mon[179688]: failed to encode map e184 with expected crc 2023-09-23T20:56:06.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:56:06 smithi026 ceph-mon[179688]: failed to encode map e184 with expected crc 2023-09-23T20:56:06.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:56:06 smithi026 ceph-mon[179688]: failed to encode map e184 with expected crc 2023-09-23T20:56:06.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:56:06 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/1446668024' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num", "val": "26"}]: dispatch 2023-09-23T20:56:06.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:56:06 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num", "val": "26"}]: dispatch 2023-09-23T20:56:06.383 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:56:06 smithi026 ceph-mon[175151]: pgmap v431: 109 pgs: 2 remapped+peering, 5 peering, 102 active+clean; 442 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-09-23T20:56:06.383 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:56:06 smithi026 ceph-mon[175151]: failed to encode map e184 with expected crc 2023-09-23T20:56:06.383 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:56:06 smithi026 ceph-mon[175151]: failed to encode map e184 with expected crc 2023-09-23T20:56:06.383 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:56:06 smithi026 ceph-mon[175151]: failed to encode map e184 with expected crc 2023-09-23T20:56:06.383 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:56:06 smithi026 ceph-mon[175151]: failed to encode map e184 with expected crc 2023-09-23T20:56:06.384 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:56:06 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/1446668024' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num", "val": "26"}]: dispatch 2023-09-23T20:56:06.384 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:56:06 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num", "val": "26"}]: dispatch 2023-09-23T20:56:06.425 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:56:06.545 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:56:06.762 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:56:06.767 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:56:06.768 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:56:06.835 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_ops_in_flight 2023-09-23T20:56:06.869 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:56:06.987 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:56:07.048 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:56:07.059 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:56:07.060 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:56:07 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num", "val": "26"}]': finished 2023-09-23T20:56:07.060 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:56:07 smithi026 ceph-mon[179688]: osdmap e185: 8 total, 8 up, 8 in 2023-09-23T20:56:07.060 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:56:07 smithi026 ceph-mon[179688]: failed to encode map e185 with expected crc 2023-09-23T20:56:07.060 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:56:07 smithi026 ceph-mon[179688]: failed to encode map e185 with expected crc 2023-09-23T20:56:07.060 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:56:07 smithi026 ceph-mon[179688]: failed to encode map e185 with expected crc 2023-09-23T20:56:07.060 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:56:07 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:56:07.072 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:56:07.073 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:56:07.106 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:56:07.174 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:56:07.210 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:56:07.254 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:56:07.257 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:56:07.258 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:56:07.358 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:56:07.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:56:07 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num", "val": "26"}]': finished 2023-09-23T20:56:07.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:56:07 smithi070 ceph-mon[168282]: osdmap e185: 8 total, 8 up, 8 in 2023-09-23T20:56:07.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:56:07 smithi070 ceph-mon[168282]: failed to encode map e185 with expected crc 2023-09-23T20:56:07.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:56:07 smithi070 ceph-mon[168282]: failed to encode map e185 with expected crc 2023-09-23T20:56:07.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:56:07 smithi070 ceph-mon[168282]: failed to encode map e185 with expected crc 2023-09-23T20:56:07.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:56:07 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:56:07.387 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:56:07 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num", "val": "26"}]': finished 2023-09-23T20:56:07.387 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:56:07 smithi026 ceph-mon[175151]: osdmap e185: 8 total, 8 up, 8 in 2023-09-23T20:56:07.387 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:56:07 smithi026 ceph-mon[175151]: failed to encode map e185 with expected crc 2023-09-23T20:56:07.387 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:56:07 smithi026 ceph-mon[175151]: failed to encode map e185 with expected crc 2023-09-23T20:56:07.387 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:56:07 smithi026 ceph-mon[175151]: failed to encode map e185 with expected crc 2023-09-23T20:56:07.387 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:56:07 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:56:07.449 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:56:07.589 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:56:07.605 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:56:07.606 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:56:07.706 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:56:07.734 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:56:07.775 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:56:07.780 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:56:07.780 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:56:07.881 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:56:07.920 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:56:07.926 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:56:07.957 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:56:07.962 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:56:07.962 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:56:08.056 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:56:08 smithi026 ceph-mon[175151]: pgmap v434: 109 pgs: 1 active+recovering+undersized+remapped, 2 remapped+peering, 4 peering, 102 active+clean; 420 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 248/20208 objects misplaced (1.227%) 2023-09-23T20:56:08.056 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:56:08 smithi026 ceph-mon[175151]: 1.7 scrub starts 2023-09-23T20:56:08.056 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:56:08 smithi026 ceph-mon[175151]: 6.6 scrub starts 2023-09-23T20:56:08.056 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:56:08 smithi026 ceph-mon[175151]: 6.6 scrub ok 2023-09-23T20:56:08.057 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:56:08 smithi026 ceph-mon[175151]: 6.5 scrub starts 2023-09-23T20:56:08.057 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:56:08 smithi026 ceph-mon[175151]: 6.5 scrub ok 2023-09-23T20:56:08.057 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:56:08 smithi070 ceph-mon[168282]: pgmap v434: 109 pgs: 1 active+recovering+undersized+remapped, 2 remapped+peering, 4 peering, 102 active+clean; 420 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 248/20208 objects misplaced (1.227%) 2023-09-23T20:56:08.057 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:56:08 smithi070 ceph-mon[168282]: 1.7 scrub starts 2023-09-23T20:56:08.057 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:56:08 smithi070 ceph-mon[168282]: 6.6 scrub starts 2023-09-23T20:56:08.057 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:56:08 smithi070 ceph-mon[168282]: 6.6 scrub ok 2023-09-23T20:56:08.058 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:56:08 smithi070 ceph-mon[168282]: 6.5 scrub starts 2023-09-23T20:56:08.058 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:56:08 smithi070 ceph-mon[168282]: 6.5 scrub ok 2023-09-23T20:56:08.060 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:56:08 smithi026 ceph-mon[179688]: pgmap v434: 109 pgs: 1 active+recovering+undersized+remapped, 2 remapped+peering, 4 peering, 102 active+clean; 420 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 248/20208 objects misplaced (1.227%) 2023-09-23T20:56:08.060 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:56:08 smithi026 ceph-mon[179688]: 1.7 scrub starts 2023-09-23T20:56:08.060 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:56:08 smithi026 ceph-mon[179688]: 6.6 scrub starts 2023-09-23T20:56:08.060 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:56:08 smithi026 ceph-mon[179688]: 6.6 scrub ok 2023-09-23T20:56:08.060 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:56:08 smithi026 ceph-mon[179688]: 6.5 scrub starts 2023-09-23T20:56:08.060 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:56:08 smithi026 ceph-mon[179688]: 6.5 scrub ok 2023-09-23T20:56:08.064 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:56:08.134 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:56:08.218 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:56:08.225 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:56:08.225 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:56:08.326 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:56:08.414 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:56:08.495 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:56:08.507 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:56:08.507 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:56:08.608 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:56:08.641 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:56:08.685 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:56:08.690 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:56:08.691 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:56:08.791 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:56:08.889 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:56:09.166 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:56:09.177 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:56:09.177 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:56:09.278 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:56:09.361 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:56:09 smithi026 ceph-mon[179688]: 6.2 scrub starts 2023-09-23T20:56:09.361 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:56:09 smithi026 ceph-mon[179688]: 6.2 scrub ok 2023-09-23T20:56:09.361 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:56:09 smithi026 ceph-mon[179688]: 1.4 scrub starts 2023-09-23T20:56:09.361 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:56:09 smithi026 ceph-mon[175151]: 6.2 scrub starts 2023-09-23T20:56:09.362 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:56:09 smithi026 ceph-mon[175151]: 6.2 scrub ok 2023-09-23T20:56:09.362 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:56:09 smithi026 ceph-mon[175151]: 1.4 scrub starts 2023-09-23T20:56:09.367 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:56:09.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:56:09 smithi070 ceph-mon[168282]: 6.2 scrub starts 2023-09-23T20:56:09.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:56:09 smithi070 ceph-mon[168282]: 6.2 scrub ok 2023-09-23T20:56:09.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:56:09 smithi070 ceph-mon[168282]: 1.4 scrub starts 2023-09-23T20:56:09.885 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:56:09 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[164029]: ::ffff:172.21.15.70 - - [23/Sep/2023:20:56:09] "GET /metrics HTTP/1.1" 200 36323 "" "Prometheus/2.43.0" 2023-09-23T20:56:10.126 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:56:10.132 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:56:10.133 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:56:10.234 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:56:10.258 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:56:10 smithi026 ceph-mon[175151]: pgmap v435: 109 pgs: 1 active+recovering+undersized+remapped, 4 peering, 104 active+clean; 312 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 248/15048 objects misplaced (1.648%) 2023-09-23T20:56:10.258 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:56:10 smithi026 ceph-mon[175151]: 6.12 scrub starts 2023-09-23T20:56:10.258 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:56:10 smithi026 ceph-mon[175151]: 6.12 scrub ok 2023-09-23T20:56:10.258 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:56:10 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:56:10.258 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:56:10 smithi026 ceph-mon[179688]: pgmap v435: 109 pgs: 1 active+recovering+undersized+remapped, 4 peering, 104 active+clean; 312 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 248/15048 objects misplaced (1.648%) 2023-09-23T20:56:10.258 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:56:10 smithi026 ceph-mon[179688]: 6.12 scrub starts 2023-09-23T20:56:10.258 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:56:10 smithi026 ceph-mon[179688]: 6.12 scrub ok 2023-09-23T20:56:10.258 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:56:10 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:56:10.318 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:56:10.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:56:10 smithi070 ceph-mon[168282]: pgmap v435: 109 pgs: 1 active+recovering+undersized+remapped, 4 peering, 104 active+clean; 312 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 248/15048 objects misplaced (1.648%) 2023-09-23T20:56:10.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:56:10 smithi070 ceph-mon[168282]: 6.12 scrub starts 2023-09-23T20:56:10.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:56:10 smithi070 ceph-mon[168282]: 6.12 scrub ok 2023-09-23T20:56:10.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:56:10 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:56:10.384 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0: {} 2023-09-23T20:56:10.394 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:56:10.404 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1: {} 2023-09-23T20:56:10.406 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:56:10.407 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:56:10.449 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2: {} 2023-09-23T20:56:10.501 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3: {} 2023-09-23T20:56:10.508 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:56:10.542 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4: {} 2023-09-23T20:56:10.551 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:56:10.591 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:56:10.592 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T20:56:10.593 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:56:10.594 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:56:10.639 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6: {} 2023-09-23T20:56:10.668 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T20:56:10.685 INFO:teuthology.orchestra.run.smithi026.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T20:56:10.685 INFO:teuthology.orchestra.run.smithi026.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T20:56:10.685 INFO:teuthology.orchestra.run.smithi026.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T20:56:10.685 INFO:teuthology.orchestra.run.smithi026.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T20:56:10.685 INFO:teuthology.orchestra.run.smithi026.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T20:56:10.685 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T20:56:10.685 INFO:teuthology.orchestra.run.smithi026.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T20:56:10.685 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T20:56:10.695 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:56:10.736 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:56:10.788 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:56:10.797 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:56:10.797 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:56:10.898 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:56:11.023 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:56:11.200 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:56:11.208 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:56:11.209 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:56:11.310 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:56:11.331 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:56:11 smithi026 ceph-mon[179688]: 1.6 scrub starts 2023-09-23T20:56:11.331 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:56:11 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/1806291845' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:56:11.331 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:56:11 smithi026 ceph-mon[179688]: pgmap v436: 109 pgs: 1 active+clean+scrubbing, 1 active+recovering+undersized+remapped, 107 active+clean; 236 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 248/11388 objects misplaced (2.178%) 2023-09-23T20:56:11.331 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:56:11 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num_actual", "val": "18"}]: dispatch 2023-09-23T20:56:11.331 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:56:11 smithi026 ceph-mon[179688]: 1.5 scrub starts 2023-09-23T20:56:11.331 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:56:11 smithi026 ceph-mon[179688]: 1.5 scrub ok 2023-09-23T20:56:11.332 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:56:11 smithi026 ceph-mon[175151]: 1.6 scrub starts 2023-09-23T20:56:11.332 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:56:11 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/1806291845' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:56:11.332 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:56:11 smithi026 ceph-mon[175151]: pgmap v436: 109 pgs: 1 active+clean+scrubbing, 1 active+recovering+undersized+remapped, 107 active+clean; 236 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 248/11388 objects misplaced (2.178%) 2023-09-23T20:56:11.332 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:56:11 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num_actual", "val": "18"}]: dispatch 2023-09-23T20:56:11.332 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:56:11 smithi026 ceph-mon[175151]: 1.5 scrub starts 2023-09-23T20:56:11.332 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:56:11 smithi026 ceph-mon[175151]: 1.5 scrub ok 2023-09-23T20:56:11.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:56:11 smithi070 ceph-mon[168282]: 1.6 scrub starts 2023-09-23T20:56:11.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:56:11 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/1806291845' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:56:11.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:56:11 smithi070 ceph-mon[168282]: pgmap v436: 109 pgs: 1 active+clean+scrubbing, 1 active+recovering+undersized+remapped, 107 active+clean; 236 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 248/11388 objects misplaced (2.178%) 2023-09-23T20:56:11.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:56:11 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num_actual", "val": "18"}]: dispatch 2023-09-23T20:56:11.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:56:11 smithi070 ceph-mon[168282]: 1.5 scrub starts 2023-09-23T20:56:11.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:56:11 smithi070 ceph-mon[168282]: 1.5 scrub ok 2023-09-23T20:56:11.377 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:56:11.440 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:56:11.446 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:56:11.447 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:56:11.497 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 6, 7, 4, 5] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 7, 6] 2023-09-23T20:56:11.497 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T20:56:11.498 INFO:tasks.thrashosds.thrasher:inject_pause on osd.1 2023-09-23T20:56:11.498 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-09-23T20:56:11.498 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-09-23T20:56:11.498 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 config set filestore_inject_stall 3 2023-09-23T20:56:11.548 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:56:11.584 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:56:11.731 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:56:11.733 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:56:11.734 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:56:11.780 INFO:teuthology.orchestra.run.smithi026.stdout:ERROR: (22) Invalid argument 2023-09-23T20:56:11.780 INFO:teuthology.orchestra.run.smithi026.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:56:11.834 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:56:11.876 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:56:11.969 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:56:11.972 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:56:11.972 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:56:12.073 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:56:12.103 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:56:12 smithi026 ceph-mon[175151]: failed to encode map e186 with expected crc 2023-09-23T20:56:12.103 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:56:12 smithi026 ceph-mon[175151]: failed to encode map e186 with expected crc 2023-09-23T20:56:12.103 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:56:12 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/2910417629' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:56:12.103 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:56:12 smithi026 ceph-mon[175151]: failed to encode map e186 with expected crc 2023-09-23T20:56:12.103 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:56:12 smithi026 ceph-mon[175151]: failed to encode map e186 with expected crc 2023-09-23T20:56:12.103 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:56:12 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num_actual", "val": "18"}]': finished 2023-09-23T20:56:12.103 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:56:12 smithi026 ceph-mon[175151]: osdmap e186: 8 total, 8 up, 8 in 2023-09-23T20:56:12.103 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:56:12 smithi026 ceph-mon[175151]: failed to encode map e186 with expected crc 2023-09-23T20:56:12.103 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:56:12 smithi026 ceph-mon[175151]: failed to encode map e186 with expected crc 2023-09-23T20:56:12.103 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:56:12 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:56:12.107 INFO:teuthology.orchestra.run.smithi026.stderr:nodeep-scrub is unset 2023-09-23T20:56:12.117 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:56:12.204 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:56:12.207 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:56:12.208 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:56:12.284 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_blocked_ops 2023-09-23T20:56:12.308 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:56:12.355 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:56:12 smithi026 ceph-mon[179688]: failed to encode map e186 with expected crc 2023-09-23T20:56:12.355 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:56:12 smithi026 ceph-mon[179688]: failed to encode map e186 with expected crc 2023-09-23T20:56:12.355 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:56:12 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/2910417629' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:56:12.355 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:56:12 smithi026 ceph-mon[179688]: failed to encode map e186 with expected crc 2023-09-23T20:56:12.356 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:56:12 smithi026 ceph-mon[179688]: failed to encode map e186 with expected crc 2023-09-23T20:56:12.356 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:56:12 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num_actual", "val": "18"}]': finished 2023-09-23T20:56:12.356 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:56:12 smithi026 ceph-mon[179688]: osdmap e186: 8 total, 8 up, 8 in 2023-09-23T20:56:12.356 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:56:12 smithi026 ceph-mon[179688]: failed to encode map e186 with expected crc 2023-09-23T20:56:12.356 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:56:12 smithi026 ceph-mon[179688]: failed to encode map e186 with expected crc 2023-09-23T20:56:12.356 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:56:12 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:56:12.358 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:56:12.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:56:12 smithi070 ceph-mon[168282]: failed to encode map e186 with expected crc 2023-09-23T20:56:12.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:56:12 smithi070 ceph-mon[168282]: failed to encode map e186 with expected crc 2023-09-23T20:56:12.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:56:12 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/2910417629' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:56:12.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:56:12 smithi070 ceph-mon[168282]: failed to encode map e186 with expected crc 2023-09-23T20:56:12.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:56:12 smithi070 ceph-mon[168282]: failed to encode map e186 with expected crc 2023-09-23T20:56:12.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:56:12 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num_actual", "val": "18"}]': finished 2023-09-23T20:56:12.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:56:12 smithi070 ceph-mon[168282]: osdmap e186: 8 total, 8 up, 8 in 2023-09-23T20:56:12.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:56:12 smithi070 ceph-mon[168282]: failed to encode map e186 with expected crc 2023-09-23T20:56:12.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:56:12 smithi070 ceph-mon[168282]: failed to encode map e186 with expected crc 2023-09-23T20:56:12.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:56:12 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:56:12.396 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:56:12.400 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:56:12.400 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:56:12.501 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:56:12.533 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:56:12.938 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:56:12.942 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:56:12.942 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:56:13.005 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:56:13.043 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:56:13.088 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:56:13.245 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T20:56:13.318 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:56:13 smithi026 ceph-mon[179688]: 1.7 scrub starts 2023-09-23T20:56:13.318 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:56:13 smithi026 ceph-mon[179688]: 1.4 scrub starts 2023-09-23T20:56:13.318 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:56:13 smithi026 ceph-mon[179688]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T20:56:13.319 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:56:13 smithi026 ceph-mon[179688]: Cluster is now healthy 2023-09-23T20:56:13.319 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:56:13 smithi026 ceph-mon[179688]: failed to encode map e187 with expected crc 2023-09-23T20:56:13.319 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:56:13 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T20:56:13.319 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:56:13 smithi026 ceph-mon[179688]: osdmap e187: 8 total, 8 up, 8 in 2023-09-23T20:56:13.319 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:56:13 smithi026 ceph-mon[179688]: failed to encode map e187 with expected crc 2023-09-23T20:56:13.319 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:56:13 smithi026 ceph-mon[179688]: failed to encode map e187 with expected crc 2023-09-23T20:56:13.320 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:56:13 smithi026 ceph-mon[179688]: pgmap v439: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+clean+scrubbing, 107 active+clean; 93 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-09-23T20:56:13.320 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:56:13 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num_actual", "val": "19"}]: dispatch 2023-09-23T20:56:13.320 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:56:13 smithi026 ceph-mon[175151]: 1.7 scrub starts 2023-09-23T20:56:13.320 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:56:13 smithi026 ceph-mon[175151]: 1.4 scrub starts 2023-09-23T20:56:13.320 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:56:13 smithi026 ceph-mon[175151]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T20:56:13.320 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:56:13 smithi026 ceph-mon[175151]: Cluster is now healthy 2023-09-23T20:56:13.321 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:56:13 smithi026 ceph-mon[175151]: failed to encode map e187 with expected crc 2023-09-23T20:56:13.321 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:56:13 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T20:56:13.321 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:56:13 smithi026 ceph-mon[175151]: osdmap e187: 8 total, 8 up, 8 in 2023-09-23T20:56:13.321 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:56:13 smithi026 ceph-mon[175151]: failed to encode map e187 with expected crc 2023-09-23T20:56:13.321 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:56:13 smithi026 ceph-mon[175151]: failed to encode map e187 with expected crc 2023-09-23T20:56:13.321 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:56:13 smithi026 ceph-mon[175151]: pgmap v439: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+clean+scrubbing, 107 active+clean; 93 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-09-23T20:56:13.321 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:56:13 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num_actual", "val": "19"}]: dispatch 2023-09-23T20:56:13.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:56:13 smithi070 ceph-mon[168282]: 1.7 scrub starts 2023-09-23T20:56:13.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:56:13 smithi070 ceph-mon[168282]: 1.4 scrub starts 2023-09-23T20:56:13.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:56:13 smithi070 ceph-mon[168282]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T20:56:13.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:56:13 smithi070 ceph-mon[168282]: Cluster is now healthy 2023-09-23T20:56:13.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:56:13 smithi070 ceph-mon[168282]: failed to encode map e187 with expected crc 2023-09-23T20:56:13.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:56:13 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T20:56:13.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:56:13 smithi070 ceph-mon[168282]: osdmap e187: 8 total, 8 up, 8 in 2023-09-23T20:56:13.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:56:13 smithi070 ceph-mon[168282]: failed to encode map e187 with expected crc 2023-09-23T20:56:13.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:56:13 smithi070 ceph-mon[168282]: failed to encode map e187 with expected crc 2023-09-23T20:56:13.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:56:13 smithi070 ceph-mon[168282]: pgmap v439: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+clean+scrubbing, 107 active+clean; 93 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-09-23T20:56:13.371 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:56:13 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num_actual", "val": "19"}]: dispatch 2023-09-23T20:56:13.954 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:56:13.958 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:56:13.959 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:56:14.059 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:56:14.087 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:56:14.133 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:56:14.136 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:56:14.136 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:56:14.200 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:56:14.237 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:56:14.265 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:56:14.301 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:56:14.303 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:56:14.304 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:56:14.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:56:14 smithi070 ceph-mon[168282]: 1.4 scrub starts 2023-09-23T20:56:14.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:56:14 smithi070 ceph-mon[168282]: failed to encode map e188 with expected crc 2023-09-23T20:56:14.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:56:14 smithi070 ceph-mon[168282]: failed to encode map e188 with expected crc 2023-09-23T20:56:14.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:56:14 smithi070 ceph-mon[168282]: failed to encode map e188 with expected crc 2023-09-23T20:56:14.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:56:14 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num_actual", "val": "19"}]': finished 2023-09-23T20:56:14.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:56:14 smithi070 ceph-mon[168282]: osdmap e188: 8 total, 8 up, 8 in 2023-09-23T20:56:14.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:56:14 smithi070 ceph-mon[168282]: 6.11 starting backfill to osd.0 from (0'0,0'0] MAX to 185'9768 2023-09-23T20:56:14.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:56:14 smithi070 ceph-mon[168282]: 6.11 starting backfill to osd.5 from (0'0,0'0] MAX to 185'9768 2023-09-23T20:56:14.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:56:14 smithi026 ceph-mon[179688]: 1.4 scrub starts 2023-09-23T20:56:14.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:56:14 smithi026 ceph-mon[179688]: failed to encode map e188 with expected crc 2023-09-23T20:56:14.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:56:14 smithi026 ceph-mon[179688]: failed to encode map e188 with expected crc 2023-09-23T20:56:14.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:56:14 smithi026 ceph-mon[179688]: failed to encode map e188 with expected crc 2023-09-23T20:56:14.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:56:14 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num_actual", "val": "19"}]': finished 2023-09-23T20:56:14.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:56:14 smithi026 ceph-mon[179688]: osdmap e188: 8 total, 8 up, 8 in 2023-09-23T20:56:14.383 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:56:14 smithi026 ceph-mon[179688]: 6.11 starting backfill to osd.0 from (0'0,0'0] MAX to 185'9768 2023-09-23T20:56:14.383 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:56:14 smithi026 ceph-mon[179688]: 6.11 starting backfill to osd.5 from (0'0,0'0] MAX to 185'9768 2023-09-23T20:56:14.383 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:56:14 smithi026 ceph-mon[175151]: 1.4 scrub starts 2023-09-23T20:56:14.383 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:56:14 smithi026 ceph-mon[175151]: failed to encode map e188 with expected crc 2023-09-23T20:56:14.384 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:56:14 smithi026 ceph-mon[175151]: failed to encode map e188 with expected crc 2023-09-23T20:56:14.384 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:56:14 smithi026 ceph-mon[175151]: failed to encode map e188 with expected crc 2023-09-23T20:56:14.384 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:56:14 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num_actual", "val": "19"}]': finished 2023-09-23T20:56:14.384 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:56:14 smithi026 ceph-mon[175151]: osdmap e188: 8 total, 8 up, 8 in 2023-09-23T20:56:14.384 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:56:14 smithi026 ceph-mon[175151]: 6.11 starting backfill to osd.0 from (0'0,0'0] MAX to 185'9768 2023-09-23T20:56:14.384 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:56:14 smithi026 ceph-mon[175151]: 6.11 starting backfill to osd.5 from (0'0,0'0] MAX to 185'9768 2023-09-23T20:56:14.405 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:56:14.435 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:56:14.575 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:56:14.579 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:56:14.579 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:56:14.680 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:56:14.711 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:56:14.940 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T20:56:15.375 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:56:15 smithi070 ceph-mon[168282]: 1.4 scrub starts 2023-09-23T20:56:15.375 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:56:15 smithi070 ceph-mon[168282]: failed to encode map e189 with expected crc 2023-09-23T20:56:15.375 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:56:15 smithi070 ceph-mon[168282]: failed to encode map e189 with expected crc 2023-09-23T20:56:15.375 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:56:15 smithi070 ceph-mon[168282]: failed to encode map e189 with expected crc 2023-09-23T20:56:15.375 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:56:15 smithi070 ceph-mon[168282]: osdmap e189: 8 total, 8 up, 8 in 2023-09-23T20:56:15.375 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:56:15 smithi070 ceph-mon[168282]: failed to encode map e189 with expected crc 2023-09-23T20:56:15.376 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:56:15 smithi070 ceph-mon[168282]: pgmap v442: 109 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+remapped, 107 active+clean; 48 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 26/2352 objects misplaced (1.105%) 2023-09-23T20:56:15.376 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:56:15 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num_actual", "val": "20"}]: dispatch 2023-09-23T20:56:15.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:56:15 smithi026 ceph-mon[179688]: 1.4 scrub starts 2023-09-23T20:56:15.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:56:15 smithi026 ceph-mon[179688]: failed to encode map e189 with expected crc 2023-09-23T20:56:15.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:56:15 smithi026 ceph-mon[179688]: failed to encode map e189 with expected crc 2023-09-23T20:56:15.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:56:15 smithi026 ceph-mon[179688]: failed to encode map e189 with expected crc 2023-09-23T20:56:15.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:56:15 smithi026 ceph-mon[179688]: osdmap e189: 8 total, 8 up, 8 in 2023-09-23T20:56:15.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:56:15 smithi026 ceph-mon[179688]: failed to encode map e189 with expected crc 2023-09-23T20:56:15.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:56:15 smithi026 ceph-mon[179688]: pgmap v442: 109 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+remapped, 107 active+clean; 48 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 26/2352 objects misplaced (1.105%) 2023-09-23T20:56:15.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:56:15 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num_actual", "val": "20"}]: dispatch 2023-09-23T20:56:15.383 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:56:15 smithi026 ceph-mon[175151]: 1.4 scrub starts 2023-09-23T20:56:15.383 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:56:15 smithi026 ceph-mon[175151]: failed to encode map e189 with expected crc 2023-09-23T20:56:15.383 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:56:15 smithi026 ceph-mon[175151]: failed to encode map e189 with expected crc 2023-09-23T20:56:15.383 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:56:15 smithi026 ceph-mon[175151]: failed to encode map e189 with expected crc 2023-09-23T20:56:15.383 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:56:15 smithi026 ceph-mon[175151]: osdmap e189: 8 total, 8 up, 8 in 2023-09-23T20:56:15.383 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:56:15 smithi026 ceph-mon[175151]: failed to encode map e189 with expected crc 2023-09-23T20:56:15.383 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:56:15 smithi026 ceph-mon[175151]: pgmap v442: 109 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+remapped, 107 active+clean; 48 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 26/2352 objects misplaced (1.105%) 2023-09-23T20:56:15.383 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:56:15 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num_actual", "val": "20"}]: dispatch 2023-09-23T20:56:15.681 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:56:15.687 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:56:15.687 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:56:15.788 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:56:15.817 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:56:16.117 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:56:16 smithi026 ceph-mon[175151]: 1.6 scrub starts 2023-09-23T20:56:16.117 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:56:16 smithi026 ceph-mon[175151]: 1.4 scrub starts 2023-09-23T20:56:16.117 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:56:16 smithi026 ceph-mon[175151]: failed to encode map e190 with expected crc 2023-09-23T20:56:16.117 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:56:16 smithi026 ceph-mon[175151]: failed to encode map e190 with expected crc 2023-09-23T20:56:16.117 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:56:16 smithi026 ceph-mon[175151]: failed to encode map e190 with expected crc 2023-09-23T20:56:16.118 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:56:16 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num_actual", "val": "20"}]': finished 2023-09-23T20:56:16.118 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:56:16 smithi026 ceph-mon[175151]: osdmap e190: 8 total, 8 up, 8 in 2023-09-23T20:56:16.118 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:56:16 smithi026 ceph-mon[175151]: 6.15 scrub starts 2023-09-23T20:56:16.118 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:56:16 smithi026 ceph-mon[175151]: 6.15 scrub ok 2023-09-23T20:56:16.118 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:56:16 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:56:16.118 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:56:16 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:56:16.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:56:16 smithi070 ceph-mon[168282]: 1.6 scrub starts 2023-09-23T20:56:16.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:56:16 smithi070 ceph-mon[168282]: 1.4 scrub starts 2023-09-23T20:56:16.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:56:16 smithi070 ceph-mon[168282]: failed to encode map e190 with expected crc 2023-09-23T20:56:16.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:56:16 smithi070 ceph-mon[168282]: failed to encode map e190 with expected crc 2023-09-23T20:56:16.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:56:16 smithi070 ceph-mon[168282]: failed to encode map e190 with expected crc 2023-09-23T20:56:16.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:56:16 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num_actual", "val": "20"}]': finished 2023-09-23T20:56:16.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:56:16 smithi070 ceph-mon[168282]: osdmap e190: 8 total, 8 up, 8 in 2023-09-23T20:56:16.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:56:16 smithi070 ceph-mon[168282]: 6.15 scrub starts 2023-09-23T20:56:16.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:56:16 smithi070 ceph-mon[168282]: 6.15 scrub ok 2023-09-23T20:56:16.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:56:16 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:56:16.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:56:16 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:56:16.381 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:56:16 smithi026 ceph-mon[179688]: 1.6 scrub starts 2023-09-23T20:56:16.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:56:16 smithi026 ceph-mon[179688]: 1.4 scrub starts 2023-09-23T20:56:16.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:56:16 smithi026 ceph-mon[179688]: failed to encode map e190 with expected crc 2023-09-23T20:56:16.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:56:16 smithi026 ceph-mon[179688]: failed to encode map e190 with expected crc 2023-09-23T20:56:16.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:56:16 smithi026 ceph-mon[179688]: failed to encode map e190 with expected crc 2023-09-23T20:56:16.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:56:16 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num_actual", "val": "20"}]': finished 2023-09-23T20:56:16.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:56:16 smithi026 ceph-mon[179688]: osdmap e190: 8 total, 8 up, 8 in 2023-09-23T20:56:16.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:56:16 smithi026 ceph-mon[179688]: 6.15 scrub starts 2023-09-23T20:56:16.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:56:16 smithi026 ceph-mon[179688]: 6.15 scrub ok 2023-09-23T20:56:16.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:56:16 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:56:16.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:56:16 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:56:17.346 INFO:tasks.radosbench.radosbench.0.smithi026.stdout:Removed 101123 objects 2023-09-23T20:56:17.347 INFO:tasks.radosbench.radosbench.0.smithi026.stdout:Clean up completed and total clean up time :147.425 2023-09-23T20:56:17.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:56:17 smithi026 ceph-mon[175151]: 1.4 scrub starts 2023-09-23T20:56:17.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:56:17 smithi026 ceph-mon[175151]: failed to encode map e191 with expected crc 2023-09-23T20:56:17.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:56:17 smithi026 ceph-mon[175151]: failed to encode map e191 with expected crc 2023-09-23T20:56:17.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:56:17 smithi026 ceph-mon[175151]: failed to encode map e191 with expected crc 2023-09-23T20:56:17.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:56:17 smithi026 ceph-mon[175151]: osdmap e191: 8 total, 8 up, 8 in 2023-09-23T20:56:17.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:56:17 smithi026 ceph-mon[175151]: failed to encode map e191 with expected crc 2023-09-23T20:56:17.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:56:17 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:56:17.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:56:17 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:56:17.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:56:17 smithi026 ceph-mon[175151]: pgmap v445: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 107 active+clean; 40 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 0 B/s wr, 223 op/s; 26/1983 objects misplaced (1.311%); 0 B/s, 130 objects/s recovering 2023-09-23T20:56:17.383 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:56:17 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num_actual", "val": "21"}]: dispatch 2023-09-23T20:56:17.383 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:56:17 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:56:17.383 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:56:17 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.1d", "id": [4, 2]}]: dispatch 2023-09-23T20:56:17.383 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:56:17 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.12", "id": [1, 5]}]: dispatch 2023-09-23T20:56:17.383 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:56:17 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1e", "id": [1, 5]}]: dispatch 2023-09-23T20:56:17.383 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:56:17 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "6.5", "id": [3, 1]}]: dispatch 2023-09-23T20:56:17.383 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:56:17 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "6.b", "id": [3, 6]}]: dispatch 2023-09-23T20:56:17.383 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:56:17 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "6.14", "id": [3, 0]}]: dispatch 2023-09-23T20:56:17.383 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:56:17 smithi026 ceph-mon[179688]: 1.4 scrub starts 2023-09-23T20:56:17.383 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:56:17 smithi026 ceph-mon[179688]: failed to encode map e191 with expected crc 2023-09-23T20:56:17.384 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:56:17 smithi026 ceph-mon[179688]: failed to encode map e191 with expected crc 2023-09-23T20:56:17.384 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:56:17 smithi026 ceph-mon[179688]: failed to encode map e191 with expected crc 2023-09-23T20:56:17.384 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:56:17 smithi026 ceph-mon[179688]: osdmap e191: 8 total, 8 up, 8 in 2023-09-23T20:56:17.384 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:56:17 smithi026 ceph-mon[179688]: failed to encode map e191 with expected crc 2023-09-23T20:56:17.384 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:56:17 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:56:17.384 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:56:17 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:56:17.384 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:56:17 smithi026 ceph-mon[179688]: pgmap v445: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 107 active+clean; 40 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 0 B/s wr, 223 op/s; 26/1983 objects misplaced (1.311%); 0 B/s, 130 objects/s recovering 2023-09-23T20:56:17.384 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:56:17 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num_actual", "val": "21"}]: dispatch 2023-09-23T20:56:17.384 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:56:17 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:56:17.384 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:56:17 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.1d", "id": [4, 2]}]: dispatch 2023-09-23T20:56:17.384 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:56:17 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.12", "id": [1, 5]}]: dispatch 2023-09-23T20:56:17.385 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:56:17 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1e", "id": [1, 5]}]: dispatch 2023-09-23T20:56:17.385 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:56:17 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "6.5", "id": [3, 1]}]: dispatch 2023-09-23T20:56:17.385 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:56:17 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "6.b", "id": [3, 6]}]: dispatch 2023-09-23T20:56:17.385 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:56:17 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "6.14", "id": [3, 0]}]: dispatch 2023-09-23T20:56:17.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:56:17 smithi070 ceph-mon[168282]: 1.4 scrub starts 2023-09-23T20:56:17.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:56:17 smithi070 ceph-mon[168282]: failed to encode map e191 with expected crc 2023-09-23T20:56:17.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:56:17 smithi070 ceph-mon[168282]: failed to encode map e191 with expected crc 2023-09-23T20:56:17.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:56:17 smithi070 ceph-mon[168282]: failed to encode map e191 with expected crc 2023-09-23T20:56:17.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:56:17 smithi070 ceph-mon[168282]: osdmap e191: 8 total, 8 up, 8 in 2023-09-23T20:56:17.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:56:17 smithi070 ceph-mon[168282]: failed to encode map e191 with expected crc 2023-09-23T20:56:17.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:56:17 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:56:17.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:56:17 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:56:17.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:56:17 smithi070 ceph-mon[168282]: pgmap v445: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 107 active+clean; 40 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 0 B/s wr, 223 op/s; 26/1983 objects misplaced (1.311%); 0 B/s, 130 objects/s recovering 2023-09-23T20:56:17.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:56:17 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num_actual", "val": "21"}]: dispatch 2023-09-23T20:56:17.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:56:17 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:56:17.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:56:17 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.1d", "id": [4, 2]}]: dispatch 2023-09-23T20:56:17.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:56:17 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.12", "id": [1, 5]}]: dispatch 2023-09-23T20:56:17.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:56:17 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1e", "id": [1, 5]}]: dispatch 2023-09-23T20:56:17.621 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:56:17 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "6.5", "id": [3, 1]}]: dispatch 2023-09-23T20:56:17.621 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:56:17 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "6.b", "id": [3, 6]}]: dispatch 2023-09-23T20:56:17.621 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:56:17 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "6.14", "id": [3, 0]}]: dispatch 2023-09-23T20:56:17.705 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:56:17.708 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:56:17.708 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:56:17.793 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:56:17.809 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:56:17.835 INFO:tasks.cephadm.ceph_manager.ceph:removing pool_name unique_pool_0 2023-09-23T20:56:17.835 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph osd pool rm unique_pool_0 unique_pool_0 --yes-i-really-really-mean-it 2023-09-23T20:56:17.841 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:56:17.876 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:56:17.880 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:56:17.880 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:56:17.981 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:56:18.011 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:56:18.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:56:18 smithi026 ceph-mon[175151]: 1.4 scrub starts 2023-09-23T20:56:18.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:56:18 smithi026 ceph-mon[175151]: failed to encode map e192 with expected crc 2023-09-23T20:56:18.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:56:18 smithi026 ceph-mon[175151]: failed to encode map e192 with expected crc 2023-09-23T20:56:18.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:56:18 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num_actual", "val": "21"}]': finished 2023-09-23T20:56:18.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:56:18 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.1d", "id": [4, 2]}]': finished 2023-09-23T20:56:18.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:56:18 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.12", "id": [1, 5]}]': finished 2023-09-23T20:56:18.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:56:18 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1e", "id": [1, 5]}]': finished 2023-09-23T20:56:18.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:56:18 smithi026 ceph-mon[175151]: failed to encode map e192 with expected crc 2023-09-23T20:56:18.383 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:56:18 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "6.5", "id": [3, 1]}]': finished 2023-09-23T20:56:18.383 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:56:18 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "6.b", "id": [3, 6]}]': finished 2023-09-23T20:56:18.383 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:56:18 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "6.14", "id": [3, 0]}]': finished 2023-09-23T20:56:18.383 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:56:18 smithi026 ceph-mon[175151]: osdmap e192: 8 total, 8 up, 8 in 2023-09-23T20:56:18.383 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:56:18 smithi026 ceph-mon[175151]: 6.13 starting backfill to osd.0 from (0'0,0'0] MAX to 188'9759 2023-09-23T20:56:18.383 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:56:18 smithi026 ceph-mon[175151]: Health check failed: Degraded data redundancy: 1 pg degraded (PG_DEGRADED) 2023-09-23T20:56:18.383 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:56:18 smithi026 ceph-mon[175151]: 6.13 starting backfill to osd.3 from (0'0,0'0] MAX to 188'9759 2023-09-23T20:56:18.383 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:56:18 smithi026 ceph-mon[179688]: 1.4 scrub starts 2023-09-23T20:56:18.383 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:56:18 smithi026 ceph-mon[179688]: failed to encode map e192 with expected crc 2023-09-23T20:56:18.383 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:56:18 smithi026 ceph-mon[179688]: failed to encode map e192 with expected crc 2023-09-23T20:56:18.384 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:56:18 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num_actual", "val": "21"}]': finished 2023-09-23T20:56:18.384 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:56:18 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.1d", "id": [4, 2]}]': finished 2023-09-23T20:56:18.384 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:56:18 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.12", "id": [1, 5]}]': finished 2023-09-23T20:56:18.384 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:56:18 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1e", "id": [1, 5]}]': finished 2023-09-23T20:56:18.384 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:56:18 smithi026 ceph-mon[179688]: failed to encode map e192 with expected crc 2023-09-23T20:56:18.384 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:56:18 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "6.5", "id": [3, 1]}]': finished 2023-09-23T20:56:18.384 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:56:18 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "6.b", "id": [3, 6]}]': finished 2023-09-23T20:56:18.384 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:56:18 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "6.14", "id": [3, 0]}]': finished 2023-09-23T20:56:18.384 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:56:18 smithi026 ceph-mon[179688]: osdmap e192: 8 total, 8 up, 8 in 2023-09-23T20:56:18.384 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:56:18 smithi026 ceph-mon[179688]: 6.13 starting backfill to osd.0 from (0'0,0'0] MAX to 188'9759 2023-09-23T20:56:18.384 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:56:18 smithi026 ceph-mon[179688]: Health check failed: Degraded data redundancy: 1 pg degraded (PG_DEGRADED) 2023-09-23T20:56:18.385 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:56:18 smithi026 ceph-mon[179688]: 6.13 starting backfill to osd.3 from (0'0,0'0] MAX to 188'9759 2023-09-23T20:56:18.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:56:18 smithi070 ceph-mon[168282]: 1.4 scrub starts 2023-09-23T20:56:18.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:56:18 smithi070 ceph-mon[168282]: failed to encode map e192 with expected crc 2023-09-23T20:56:18.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:56:18 smithi070 ceph-mon[168282]: failed to encode map e192 with expected crc 2023-09-23T20:56:18.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:56:18 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num_actual", "val": "21"}]': finished 2023-09-23T20:56:18.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:56:18 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.1d", "id": [4, 2]}]': finished 2023-09-23T20:56:18.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:56:18 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.12", "id": [1, 5]}]': finished 2023-09-23T20:56:18.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:56:18 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1e", "id": [1, 5]}]': finished 2023-09-23T20:56:18.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:56:18 smithi070 ceph-mon[168282]: failed to encode map e192 with expected crc 2023-09-23T20:56:18.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:56:18 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "6.5", "id": [3, 1]}]': finished 2023-09-23T20:56:18.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:56:18 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "6.b", "id": [3, 6]}]': finished 2023-09-23T20:56:18.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:56:18 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "6.14", "id": [3, 0]}]': finished 2023-09-23T20:56:18.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:56:18 smithi070 ceph-mon[168282]: osdmap e192: 8 total, 8 up, 8 in 2023-09-23T20:56:18.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:56:18 smithi070 ceph-mon[168282]: 6.13 starting backfill to osd.0 from (0'0,0'0] MAX to 188'9759 2023-09-23T20:56:18.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:56:18 smithi070 ceph-mon[168282]: Health check failed: Degraded data redundancy: 1 pg degraded (PG_DEGRADED) 2023-09-23T20:56:18.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:56:18 smithi070 ceph-mon[168282]: 6.13 starting backfill to osd.3 from (0'0,0'0] MAX to 188'9759 2023-09-23T20:56:18.870 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:56:18.874 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:56:18.875 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:56:18.976 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:56:19.006 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:56:19.043 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:56:19.045 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:56:19.046 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:56:19.134 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:56:19 smithi026 ceph-mon[179688]: failed to encode map e193 with expected crc 2023-09-23T20:56:19.135 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:56:19 smithi026 ceph-mon[179688]: osdmap e193: 8 total, 8 up, 8 in 2023-09-23T20:56:19.135 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:56:19 smithi026 ceph-mon[179688]: failed to encode map e193 with expected crc 2023-09-23T20:56:19.135 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:56:19 smithi026 ceph-mon[179688]: failed to encode map e193 with expected crc 2023-09-23T20:56:19.135 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:56:19 smithi026 ceph-mon[179688]: failed to encode map e193 with expected crc 2023-09-23T20:56:19.135 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:56:19 smithi026 ceph-mon[179688]: failed to encode map e193 with expected crc 2023-09-23T20:56:19.135 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:56:19 smithi026 ceph-mon[179688]: failed to encode map e193 with expected crc 2023-09-23T20:56:19.135 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:56:19 smithi026 ceph-mon[179688]: pgmap v448: 109 pgs: 1 peering, 1 remapped+peering, 2 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 104 active+clean; 15 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 17/759 objects degraded (2.240%); 26/759 objects misplaced (3.426%); 192 KiB/s, 136 objects/s recovering 2023-09-23T20:56:19.135 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:56:19 smithi026 ceph-mon[175151]: failed to encode map e193 with expected crc 2023-09-23T20:56:19.136 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:56:19 smithi026 ceph-mon[175151]: osdmap e193: 8 total, 8 up, 8 in 2023-09-23T20:56:19.136 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:56:19 smithi026 ceph-mon[175151]: failed to encode map e193 with expected crc 2023-09-23T20:56:19.136 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:56:19 smithi026 ceph-mon[175151]: failed to encode map e193 with expected crc 2023-09-23T20:56:19.136 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:56:19 smithi026 ceph-mon[175151]: failed to encode map e193 with expected crc 2023-09-23T20:56:19.136 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:56:19 smithi026 ceph-mon[175151]: failed to encode map e193 with expected crc 2023-09-23T20:56:19.136 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:56:19 smithi026 ceph-mon[175151]: failed to encode map e193 with expected crc 2023-09-23T20:56:19.136 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:56:19 smithi026 ceph-mon[175151]: pgmap v448: 109 pgs: 1 peering, 1 remapped+peering, 2 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 104 active+clean; 15 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 17/759 objects degraded (2.240%); 26/759 objects misplaced (3.426%); 192 KiB/s, 136 objects/s recovering 2023-09-23T20:56:19.146 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:56:19.184 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:56:19.246 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:56:19.249 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:56:19.250 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:56:19.351 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:56:19.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:56:19 smithi070 ceph-mon[168282]: failed to encode map e193 with expected crc 2023-09-23T20:56:19.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:56:19 smithi070 ceph-mon[168282]: osdmap e193: 8 total, 8 up, 8 in 2023-09-23T20:56:19.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:56:19 smithi070 ceph-mon[168282]: failed to encode map e193 with expected crc 2023-09-23T20:56:19.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:56:19 smithi070 ceph-mon[168282]: failed to encode map e193 with expected crc 2023-09-23T20:56:19.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:56:19 smithi070 ceph-mon[168282]: failed to encode map e193 with expected crc 2023-09-23T20:56:19.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:56:19 smithi070 ceph-mon[168282]: failed to encode map e193 with expected crc 2023-09-23T20:56:19.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:56:19 smithi070 ceph-mon[168282]: failed to encode map e193 with expected crc 2023-09-23T20:56:19.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:56:19 smithi070 ceph-mon[168282]: pgmap v448: 109 pgs: 1 peering, 1 remapped+peering, 2 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 104 active+clean; 15 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 17/759 objects degraded (2.240%); 26/759 objects misplaced (3.426%); 192 KiB/s, 136 objects/s recovering 2023-09-23T20:56:19.377 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:56:19.412 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:56:19.414 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:56:19.415 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:56:19.515 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:56:19.543 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:56:19.578 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:56:19.581 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:56:19.582 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:56:19.682 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:56:19.710 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:56:19.746 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:56:19.749 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:56:19.749 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:56:19.850 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:56:19.877 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:56:19.881 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:56:19 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[164029]: ::ffff:172.21.15.70 - - [23/Sep/2023:20:56:19] "GET /metrics HTTP/1.1" 200 36321 "" "Prometheus/2.43.0" 2023-09-23T20:56:19.912 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:56:19.915 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:56:19.915 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:56:20.016 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:56:20.049 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:56:20.376 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:56:20.380 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:56:20.380 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:56:20.381 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:56:20 smithi026 ceph-mon[179688]: failed to encode map e194 with expected crc 2023-09-23T20:56:20.381 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:56:20 smithi026 ceph-mon[179688]: failed to encode map e194 with expected crc 2023-09-23T20:56:20.381 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:56:20 smithi026 ceph-mon[179688]: failed to encode map e194 with expected crc 2023-09-23T20:56:20.381 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:56:20 smithi026 ceph-mon[179688]: failed to encode map e194 with expected crc 2023-09-23T20:56:20.381 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:56:20 smithi026 ceph-mon[179688]: osdmap e194: 8 total, 8 up, 8 in 2023-09-23T20:56:20.381 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:56:20 smithi026 ceph-mon[179688]: 6.b starting backfill to osd.6 from (0'0,0'0] MAX to 190'12390 2023-09-23T20:56:20.381 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:56:20 smithi026 ceph-mon[179688]: 6.14 starting backfill to osd.2 from (0'0,0'0] MAX to 190'9651 2023-09-23T20:56:20.381 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:56:20 smithi026 ceph-mon[179688]: 6.5 starting backfill to osd.1 from (0'0,0'0] MAX to 190'9562 2023-09-23T20:56:20.381 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:56:20 smithi026 ceph-mon[179688]: 6.14 starting backfill to osd.4 from (0'0,0'0] MAX to 190'9651 2023-09-23T20:56:20.381 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:56:20 smithi026 ceph-mon[179688]: failed to encode map e194 with expected crc 2023-09-23T20:56:20.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:56:20 smithi026 ceph-mon[179688]: 6.14 starting backfill to osd.7 from (0'0,0'0] MAX to 190'9651 2023-09-23T20:56:20.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:56:20 smithi026 ceph-mon[175151]: failed to encode map e194 with expected crc 2023-09-23T20:56:20.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:56:20 smithi026 ceph-mon[175151]: failed to encode map e194 with expected crc 2023-09-23T20:56:20.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:56:20 smithi026 ceph-mon[175151]: failed to encode map e194 with expected crc 2023-09-23T20:56:20.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:56:20 smithi026 ceph-mon[175151]: failed to encode map e194 with expected crc 2023-09-23T20:56:20.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:56:20 smithi026 ceph-mon[175151]: osdmap e194: 8 total, 8 up, 8 in 2023-09-23T20:56:20.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:56:20 smithi026 ceph-mon[175151]: 6.b starting backfill to osd.6 from (0'0,0'0] MAX to 190'12390 2023-09-23T20:56:20.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:56:20 smithi026 ceph-mon[175151]: 6.14 starting backfill to osd.2 from (0'0,0'0] MAX to 190'9651 2023-09-23T20:56:20.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:56:20 smithi026 ceph-mon[175151]: 6.5 starting backfill to osd.1 from (0'0,0'0] MAX to 190'9562 2023-09-23T20:56:20.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:56:20 smithi026 ceph-mon[175151]: 6.14 starting backfill to osd.4 from (0'0,0'0] MAX to 190'9651 2023-09-23T20:56:20.383 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:56:20 smithi026 ceph-mon[175151]: failed to encode map e194 with expected crc 2023-09-23T20:56:20.383 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:56:20 smithi026 ceph-mon[175151]: 6.14 starting backfill to osd.7 from (0'0,0'0] MAX to 190'9651 2023-09-23T20:56:20.481 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:56:20.506 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:56:20 smithi070 ceph-mon[168282]: failed to encode map e194 with expected crc 2023-09-23T20:56:20.506 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:56:20 smithi070 ceph-mon[168282]: failed to encode map e194 with expected crc 2023-09-23T20:56:20.506 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:56:20 smithi070 ceph-mon[168282]: failed to encode map e194 with expected crc 2023-09-23T20:56:20.506 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:56:20 smithi070 ceph-mon[168282]: failed to encode map e194 with expected crc 2023-09-23T20:56:20.506 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:56:20 smithi070 ceph-mon[168282]: osdmap e194: 8 total, 8 up, 8 in 2023-09-23T20:56:20.506 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:56:20 smithi070 ceph-mon[168282]: 6.b starting backfill to osd.6 from (0'0,0'0] MAX to 190'12390 2023-09-23T20:56:20.506 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:56:20 smithi070 ceph-mon[168282]: 6.14 starting backfill to osd.2 from (0'0,0'0] MAX to 190'9651 2023-09-23T20:56:20.506 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:56:20 smithi070 ceph-mon[168282]: 6.5 starting backfill to osd.1 from (0'0,0'0] MAX to 190'9562 2023-09-23T20:56:20.507 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:56:20 smithi070 ceph-mon[168282]: 6.14 starting backfill to osd.4 from (0'0,0'0] MAX to 190'9651 2023-09-23T20:56:20.507 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:56:20 smithi070 ceph-mon[168282]: failed to encode map e194 with expected crc 2023-09-23T20:56:20.507 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:56:20 smithi070 ceph-mon[168282]: 6.14 starting backfill to osd.7 from (0'0,0'0] MAX to 190'9651 2023-09-23T20:56:20.509 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:56:20.544 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:56:20.546 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:56:20.547 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:56:20.647 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:56:20.675 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:56:20.710 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:56:20.714 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:56:20.714 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:56:20.815 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:56:20.847 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:56:21.381 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:56:21 smithi026 ceph-mon[175151]: 1.4 scrub starts 2023-09-23T20:56:21.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:56:21 smithi026 ceph-mon[175151]: pgmap v450: 109 pgs: 2 activating+remapped, 1 peering, 1 remapped+peering, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 102 active+clean; 10 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 17/546 objects degraded (3.114%); 64/546 objects misplaced (11.722%); 183 KiB/s, 5 objects/s recovering 2023-09-23T20:56:21.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:56:21 smithi026 ceph-mon[179688]: 1.4 scrub starts 2023-09-23T20:56:21.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:56:21 smithi026 ceph-mon[179688]: pgmap v450: 109 pgs: 2 activating+remapped, 1 peering, 1 remapped+peering, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 102 active+clean; 10 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 17/546 objects degraded (3.114%); 64/546 objects misplaced (11.722%); 183 KiB/s, 5 objects/s recovering 2023-09-23T20:56:21.492 INFO:teuthology.orchestra.run.smithi026.stdout:{ 2023-09-23T20:56:21.492 INFO:teuthology.orchestra.run.smithi026.stdout: "success": "filestore_inject_stall = '3' (not observed, change may require restart) " 2023-09-23T20:56:21.492 INFO:teuthology.orchestra.run.smithi026.stdout:} 2023-09-23T20:56:21.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:56:21 smithi070 ceph-mon[168282]: 1.4 scrub starts 2023-09-23T20:56:21.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:56:21 smithi070 ceph-mon[168282]: pgmap v450: 109 pgs: 2 activating+remapped, 1 peering, 1 remapped+peering, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 102 active+clean; 10 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 17/546 objects degraded (3.114%); 64/546 objects misplaced (11.722%); 183 KiB/s, 5 objects/s recovering 2023-09-23T20:56:22.469 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:56:22 smithi026 ceph-mon[175151]: 1.4 scrub starts 2023-09-23T20:56:22.469 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:56:22 smithi026 ceph-mon[179688]: 1.4 scrub starts 2023-09-23T20:56:22.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:56:22 smithi070 ceph-mon[168282]: 1.4 scrub starts 2023-09-23T20:56:23.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:56:23 smithi070 ceph-mon[168282]: 1.4 scrub starts 2023-09-23T20:56:23.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:56:23 smithi070 ceph-mon[168282]: pgmap v451: 109 pgs: 2 activating+remapped, 3 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 102 active+clean; 10 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 17/543 objects degraded (3.131%); 64/543 objects misplaced (11.786%); 128 KiB/s, 375 objects/s recovering 2023-09-23T20:56:23.631 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:56:23 smithi026 ceph-mon[179688]: 1.4 scrub starts 2023-09-23T20:56:23.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:56:23 smithi026 ceph-mon[179688]: pgmap v451: 109 pgs: 2 activating+remapped, 3 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 102 active+clean; 10 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 17/543 objects degraded (3.131%); 64/543 objects misplaced (11.786%); 128 KiB/s, 375 objects/s recovering 2023-09-23T20:56:23.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:56:23 smithi026 ceph-mon[175151]: 1.4 scrub starts 2023-09-23T20:56:23.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:56:23 smithi026 ceph-mon[175151]: pgmap v451: 109 pgs: 2 activating+remapped, 3 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 102 active+clean; 10 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 17/543 objects degraded (3.131%); 64/543 objects misplaced (11.786%); 128 KiB/s, 375 objects/s recovering 2023-09-23T20:56:23.897 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:56:23.902 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:56:23.903 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:56:24.004 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:56:24.031 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:56:24.068 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:56:24.071 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:56:24.071 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:56:24.172 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:56:24.199 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:56:24.206 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:56:24.238 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:56:24.241 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:56:24.242 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:56:24.343 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:56:24.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:56:24 smithi070 ceph-mon[168282]: Health check update: Degraded data redundancy: 17/543 objects degraded (3.131%), 2 pgs degraded (PG_DEGRADED) 2023-09-23T20:56:24.378 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:56:24.469 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:56:24.472 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:56:24 smithi026 ceph-mon[179688]: Health check update: Degraded data redundancy: 17/543 objects degraded (3.131%), 2 pgs degraded (PG_DEGRADED) 2023-09-23T20:56:24.472 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:56:24 smithi026 ceph-mon[175151]: Health check update: Degraded data redundancy: 17/543 objects degraded (3.131%), 2 pgs degraded (PG_DEGRADED) 2023-09-23T20:56:24.473 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:56:24.474 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:56:24.484 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:56:24.554 INFO:teuthology.orchestra.run.smithi026.stdout:ERROR: (22) Invalid argument 2023-09-23T20:56:24.554 INFO:teuthology.orchestra.run.smithi026.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:56:24.575 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:56:24.604 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:56:24.640 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:56:24.642 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:56:24.643 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:56:24.744 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:56:24.772 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:56:24.809 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:56:24.811 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:56:24.811 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:56:24.912 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:56:24.940 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:56:24.976 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:56:24.980 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:56:24.980 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:56:25.081 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:56:25.085 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0: {} 2023-09-23T20:56:25.106 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1: {} 2023-09-23T20:56:25.116 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:56:25.130 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2: {} 2023-09-23T20:56:25.158 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3: {} 2023-09-23T20:56:25.184 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4: {} 2023-09-23T20:56:25.204 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T20:56:25.229 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6: {} 2023-09-23T20:56:25.255 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T20:56:25.258 INFO:teuthology.orchestra.run.smithi026.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T20:56:25.258 INFO:teuthology.orchestra.run.smithi026.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T20:56:25.258 INFO:teuthology.orchestra.run.smithi026.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T20:56:25.258 INFO:teuthology.orchestra.run.smithi026.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T20:56:25.258 INFO:teuthology.orchestra.run.smithi026.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T20:56:25.259 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T20:56:25.259 INFO:teuthology.orchestra.run.smithi026.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T20:56:25.259 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T20:56:25.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:56:25 smithi070 ceph-mon[168282]: 1.7 scrub starts 2023-09-23T20:56:25.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:56:25 smithi070 ceph-mon[168282]: pgmap v452: 109 pgs: 1 active+clean+scrubbing, 5 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 101 active+clean; 10 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 17/543 objects degraded (3.131%); 64/543 objects misplaced (11.786%); 107 KiB/s, 313 objects/s recovering 2023-09-23T20:56:25.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:56:25 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/2885345924' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:56:25.381 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:56:25 smithi026 ceph-mon[175151]: 1.7 scrub starts 2023-09-23T20:56:25.381 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:56:25 smithi026 ceph-mon[175151]: pgmap v452: 109 pgs: 1 active+clean+scrubbing, 5 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 101 active+clean; 10 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 17/543 objects degraded (3.131%); 64/543 objects misplaced (11.786%); 107 KiB/s, 313 objects/s recovering 2023-09-23T20:56:25.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:56:25 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/2885345924' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:56:25.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:56:25 smithi026 ceph-mon[179688]: 1.7 scrub starts 2023-09-23T20:56:25.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:56:25 smithi026 ceph-mon[179688]: pgmap v452: 109 pgs: 1 active+clean+scrubbing, 5 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 101 active+clean; 10 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 17/543 objects degraded (3.131%); 64/543 objects misplaced (11.786%); 107 KiB/s, 313 objects/s recovering 2023-09-23T20:56:25.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:56:25 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/2885345924' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:56:26.420 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:56:26 smithi026 ceph-mon[179688]: 1.4 scrub starts 2023-09-23T20:56:26.420 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:56:26 smithi026 ceph-mon[179688]: 1.6 scrub starts 2023-09-23T20:56:26.420 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:56:26 smithi026 ceph-mon[175151]: 1.4 scrub starts 2023-09-23T20:56:26.420 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:56:26 smithi026 ceph-mon[175151]: 1.6 scrub starts 2023-09-23T20:56:26.576 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_historic_ops 2023-09-23T20:56:26.616 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:56:26.618 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:56:26.619 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:56:26.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:56:26 smithi070 ceph-mon[168282]: 1.4 scrub starts 2023-09-23T20:56:26.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:56:26 smithi070 ceph-mon[168282]: 1.6 scrub starts 2023-09-23T20:56:26.720 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:56:26.752 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:56:27.517 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:56:27 smithi026 ceph-mon[175151]: failed to encode map e195 with expected crc 2023-09-23T20:56:27.518 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:56:27 smithi026 ceph-mon[175151]: failed to encode map e195 with expected crc 2023-09-23T20:56:27.518 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:56:27 smithi026 ceph-mon[175151]: failed to encode map e195 with expected crc 2023-09-23T20:56:27.518 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:56:27 smithi026 ceph-mon[175151]: osdmap e195: 8 total, 8 up, 8 in 2023-09-23T20:56:27.518 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:56:27 smithi026 ceph-mon[175151]: failed to encode map e195 with expected crc 2023-09-23T20:56:27.518 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:56:27 smithi026 ceph-mon[175151]: failed to encode map e195 with expected crc 2023-09-23T20:56:27.518 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:56:27 smithi026 ceph-mon[175151]: failed to encode map e195 with expected crc 2023-09-23T20:56:27.518 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:56:27 smithi026 ceph-mon[175151]: pgmap v454: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+clean+scrubbing, 5 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 101 active+clean; 10 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 0 B/s wr, 12 op/s; 17/543 objects degraded (3.131%); 64/543 objects misplaced (11.786%); 0 B/s, 427 objects/s recovering 2023-09-23T20:56:27.518 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:56:27 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:56:27.518 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:56:27 smithi026 ceph-mon[179688]: failed to encode map e195 with expected crc 2023-09-23T20:56:27.519 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:56:27 smithi026 ceph-mon[179688]: failed to encode map e195 with expected crc 2023-09-23T20:56:27.519 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:56:27 smithi026 ceph-mon[179688]: failed to encode map e195 with expected crc 2023-09-23T20:56:27.519 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:56:27 smithi026 ceph-mon[179688]: osdmap e195: 8 total, 8 up, 8 in 2023-09-23T20:56:27.519 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:56:27 smithi026 ceph-mon[179688]: failed to encode map e195 with expected crc 2023-09-23T20:56:27.519 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:56:27 smithi026 ceph-mon[179688]: failed to encode map e195 with expected crc 2023-09-23T20:56:27.519 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:56:27 smithi026 ceph-mon[179688]: failed to encode map e195 with expected crc 2023-09-23T20:56:27.519 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:56:27 smithi026 ceph-mon[179688]: pgmap v454: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+clean+scrubbing, 5 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 101 active+clean; 10 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 0 B/s wr, 12 op/s; 17/543 objects degraded (3.131%); 64/543 objects misplaced (11.786%); 0 B/s, 427 objects/s recovering 2023-09-23T20:56:27.519 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:56:27 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:56:27.568 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:56:27.570 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:56:27.570 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:56:27.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:56:27 smithi070 ceph-mon[168282]: failed to encode map e195 with expected crc 2023-09-23T20:56:27.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:56:27 smithi070 ceph-mon[168282]: failed to encode map e195 with expected crc 2023-09-23T20:56:27.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:56:27 smithi070 ceph-mon[168282]: failed to encode map e195 with expected crc 2023-09-23T20:56:27.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:56:27 smithi070 ceph-mon[168282]: osdmap e195: 8 total, 8 up, 8 in 2023-09-23T20:56:27.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:56:27 smithi070 ceph-mon[168282]: failed to encode map e195 with expected crc 2023-09-23T20:56:27.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:56:27 smithi070 ceph-mon[168282]: failed to encode map e195 with expected crc 2023-09-23T20:56:27.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:56:27 smithi070 ceph-mon[168282]: failed to encode map e195 with expected crc 2023-09-23T20:56:27.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:56:27 smithi070 ceph-mon[168282]: pgmap v454: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+clean+scrubbing, 5 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 101 active+clean; 10 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 0 B/s wr, 12 op/s; 17/543 objects degraded (3.131%); 64/543 objects misplaced (11.786%); 0 B/s, 427 objects/s recovering 2023-09-23T20:56:27.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:56:27 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:56:27.671 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:56:27.702 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:56:27.737 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:56:27.739 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:56:27.740 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:56:27.840 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:56:27.866 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:56:27.898 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:56:27.900 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:56:27.901 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:56:28.002 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:56:28.043 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:56:28.136 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:56:28.138 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:56:28.139 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:56:28.240 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:56:28.265 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:56:28.297 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:56:28.334 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:56:28.334 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:56:28.381 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:56:28 smithi026 ceph-mon[179688]: 1.7 scrub starts 2023-09-23T20:56:28.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:56:28 smithi026 ceph-mon[179688]: 1.4 scrub starts 2023-09-23T20:56:28.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:56:28 smithi026 ceph-mon[179688]: failed to encode map e196 with expected crc 2023-09-23T20:56:28.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:56:28 smithi026 ceph-mon[179688]: osdmap e196: 8 total, 8 up, 8 in 2023-09-23T20:56:28.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:56:28 smithi026 ceph-mon[179688]: failed to encode map e196 with expected crc 2023-09-23T20:56:28.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:56:28 smithi026 ceph-mon[179688]: failed to encode map e196 with expected crc 2023-09-23T20:56:28.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:56:28 smithi026 ceph-mon[179688]: failed to encode map e196 with expected crc 2023-09-23T20:56:28.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:56:28 smithi026 ceph-mon[179688]: failed to encode map e196 with expected crc 2023-09-23T20:56:28.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:56:28 smithi026 ceph-mon[179688]: failed to encode map e196 with expected crc 2023-09-23T20:56:28.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:56:28 smithi026 ceph-mon[179688]: failed to encode map e196 with expected crc 2023-09-23T20:56:28.383 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:56:28 smithi026 ceph-mon[179688]: failed to encode map e196 with expected crc 2023-09-23T20:56:28.383 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:56:28 smithi026 ceph-mon[179688]: failed to encode map e196 with expected crc 2023-09-23T20:56:28.383 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:56:28 smithi026 ceph-mon[179688]: failed to encode map e196 with expected crc 2023-09-23T20:56:28.383 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:56:28 smithi026 ceph-mon[179688]: osdmap e197: 8 total, 8 up, 8 in 2023-09-23T20:56:28.383 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:56:28 smithi026 ceph-mon[175151]: 1.7 scrub starts 2023-09-23T20:56:28.383 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:56:28 smithi026 ceph-mon[175151]: 1.4 scrub starts 2023-09-23T20:56:28.383 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:56:28 smithi026 ceph-mon[175151]: failed to encode map e196 with expected crc 2023-09-23T20:56:28.383 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:56:28 smithi026 ceph-mon[175151]: osdmap e196: 8 total, 8 up, 8 in 2023-09-23T20:56:28.383 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:56:28 smithi026 ceph-mon[175151]: failed to encode map e196 with expected crc 2023-09-23T20:56:28.383 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:56:28 smithi026 ceph-mon[175151]: failed to encode map e196 with expected crc 2023-09-23T20:56:28.383 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:56:28 smithi026 ceph-mon[175151]: failed to encode map e196 with expected crc 2023-09-23T20:56:28.384 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:56:28 smithi026 ceph-mon[175151]: failed to encode map e196 with expected crc 2023-09-23T20:56:28.384 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:56:28 smithi026 ceph-mon[175151]: failed to encode map e196 with expected crc 2023-09-23T20:56:28.384 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:56:28 smithi026 ceph-mon[175151]: failed to encode map e196 with expected crc 2023-09-23T20:56:28.384 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:56:28 smithi026 ceph-mon[175151]: failed to encode map e196 with expected crc 2023-09-23T20:56:28.384 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:56:28 smithi026 ceph-mon[175151]: failed to encode map e196 with expected crc 2023-09-23T20:56:28.384 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:56:28 smithi026 ceph-mon[175151]: failed to encode map e196 with expected crc 2023-09-23T20:56:28.384 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:56:28 smithi026 ceph-mon[175151]: osdmap e197: 8 total, 8 up, 8 in 2023-09-23T20:56:28.407 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:56:28.436 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:56:28.464 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:56:28.516 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:56:28.518 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:56:28.519 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:56:28.619 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:56:28.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:56:28 smithi070 ceph-mon[168282]: 1.7 scrub starts 2023-09-23T20:56:28.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:56:28 smithi070 ceph-mon[168282]: 1.4 scrub starts 2023-09-23T20:56:28.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:56:28 smithi070 ceph-mon[168282]: failed to encode map e196 with expected crc 2023-09-23T20:56:28.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:56:28 smithi070 ceph-mon[168282]: osdmap e196: 8 total, 8 up, 8 in 2023-09-23T20:56:28.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:56:28 smithi070 ceph-mon[168282]: failed to encode map e196 with expected crc 2023-09-23T20:56:28.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:56:28 smithi070 ceph-mon[168282]: failed to encode map e196 with expected crc 2023-09-23T20:56:28.621 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:56:28 smithi070 ceph-mon[168282]: failed to encode map e196 with expected crc 2023-09-23T20:56:28.621 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:56:28 smithi070 ceph-mon[168282]: failed to encode map e196 with expected crc 2023-09-23T20:56:28.621 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:56:28 smithi070 ceph-mon[168282]: failed to encode map e196 with expected crc 2023-09-23T20:56:28.621 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:56:28 smithi070 ceph-mon[168282]: failed to encode map e196 with expected crc 2023-09-23T20:56:28.621 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:56:28 smithi070 ceph-mon[168282]: failed to encode map e196 with expected crc 2023-09-23T20:56:28.621 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:56:28 smithi070 ceph-mon[168282]: failed to encode map e196 with expected crc 2023-09-23T20:56:28.621 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:56:28 smithi070 ceph-mon[168282]: failed to encode map e196 with expected crc 2023-09-23T20:56:28.621 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:56:28 smithi070 ceph-mon[168282]: osdmap e197: 8 total, 8 up, 8 in 2023-09-23T20:56:28.647 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:56:28.681 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:56:28.684 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:56:28.684 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:56:28.785 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:56:28.816 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:56:28.852 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:56:28.854 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:56:28.855 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:56:28.956 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:56:28.983 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 6, 7, 4, 5] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 7, 6] 2023-09-23T20:56:28.984 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T20:56:28.984 INFO:tasks.thrashosds.thrasher:inject_pause on osd.7 2023-09-23T20:56:28.984 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-09-23T20:56:28.984 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-09-23T20:56:28.984 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 config set filestore_inject_stall 3 2023-09-23T20:56:28.985 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:56:29.176 INFO:teuthology.orchestra.run.smithi026.stderr:pool 'unique_pool_0' removed 2023-09-23T20:56:29.177 INFO:teuthology.orchestra.run.smithi026.stderr:noscrub is set 2023-09-23T20:56:29.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:56:29 smithi070 ceph-mon[168282]: 1.9 scrub starts 2023-09-23T20:56:29.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:56:29 smithi070 ceph-mon[168282]: 1.9 scrub ok 2023-09-23T20:56:29.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:56:29 smithi070 ceph-mon[168282]: failed to encode map e197 with expected crc 2023-09-23T20:56:29.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:56:29 smithi070 ceph-mon[168282]: failed to encode map e197 with expected crc 2023-09-23T20:56:29.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:56:29 smithi070 ceph-mon[168282]: failed to encode map e197 with expected crc 2023-09-23T20:56:29.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:56:29 smithi070 ceph-mon[168282]: failed to encode map e197 with expected crc 2023-09-23T20:56:29.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:56:29 smithi070 ceph-mon[168282]: failed to encode map e197 with expected crc 2023-09-23T20:56:29.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:56:29 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/1819052987' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:56:29.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:56:29 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:56:29.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:56:29 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/712149148' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-23T20:56:29.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:56:29 smithi070 ceph-mon[168282]: pgmap v457: 109 pgs: 1 active, 2 peering, 1 active+recovering+undersized+remapped, 1 active+clean+scrubbing, 3 active+remapped+backfill_wait, 101 active+clean; 10 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 22/543 objects misplaced (4.052%); 0 B/s, 412 objects/s recovering 2023-09-23T20:56:29.379 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:56:29.382 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:56:29.382 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:56:29.459 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:56:29 smithi026 ceph-mon[179688]: 1.9 scrub starts 2023-09-23T20:56:29.459 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:56:29 smithi026 ceph-mon[179688]: 1.9 scrub ok 2023-09-23T20:56:29.459 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:56:29 smithi026 ceph-mon[179688]: failed to encode map e197 with expected crc 2023-09-23T20:56:29.459 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:56:29 smithi026 ceph-mon[179688]: failed to encode map e197 with expected crc 2023-09-23T20:56:29.459 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:56:29 smithi026 ceph-mon[179688]: failed to encode map e197 with expected crc 2023-09-23T20:56:29.460 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:56:29 smithi026 ceph-mon[179688]: failed to encode map e197 with expected crc 2023-09-23T20:56:29.460 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:56:29 smithi026 ceph-mon[179688]: failed to encode map e197 with expected crc 2023-09-23T20:56:29.460 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:56:29 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/1819052987' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:56:29.460 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:56:29 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:56:29.460 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:56:29 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/712149148' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-23T20:56:29.460 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:56:29 smithi026 ceph-mon[179688]: pgmap v457: 109 pgs: 1 active, 2 peering, 1 active+recovering+undersized+remapped, 1 active+clean+scrubbing, 3 active+remapped+backfill_wait, 101 active+clean; 10 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 22/543 objects misplaced (4.052%); 0 B/s, 412 objects/s recovering 2023-09-23T20:56:29.460 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:56:29 smithi026 ceph-mon[175151]: 1.9 scrub starts 2023-09-23T20:56:29.460 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:56:29 smithi026 ceph-mon[175151]: 1.9 scrub ok 2023-09-23T20:56:29.460 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:56:29 smithi026 ceph-mon[175151]: failed to encode map e197 with expected crc 2023-09-23T20:56:29.460 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:56:29 smithi026 ceph-mon[175151]: failed to encode map e197 with expected crc 2023-09-23T20:56:29.460 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:56:29 smithi026 ceph-mon[175151]: failed to encode map e197 with expected crc 2023-09-23T20:56:29.461 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:56:29 smithi026 ceph-mon[175151]: failed to encode map e197 with expected crc 2023-09-23T20:56:29.461 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:56:29 smithi026 ceph-mon[175151]: failed to encode map e197 with expected crc 2023-09-23T20:56:29.461 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:56:29 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/1819052987' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:56:29.461 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:56:29 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:56:29.461 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:56:29 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/712149148' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-23T20:56:29.461 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:56:29 smithi026 ceph-mon[175151]: pgmap v457: 109 pgs: 1 active, 2 peering, 1 active+recovering+undersized+remapped, 1 active+clean+scrubbing, 3 active+remapped+backfill_wait, 101 active+clean; 10 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 22/543 objects misplaced (4.052%); 0 B/s, 412 objects/s recovering 2023-09-23T20:56:29.483 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:56:29.514 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:56:29.541 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T20:56:29.626 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:56:29.751 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:56:29 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[164029]: ::ffff:172.21.15.70 - - [23/Sep/2023:20:56:29] "GET /metrics HTTP/1.1" 200 36321 "" "Prometheus/2.43.0" 2023-09-23T20:56:30.006 INFO:teuthology.task.full_sequential:In full_sequential, running task radosbench... 2023-09-23T20:56:30.006 INFO:tasks.radosbench:Beginning radosbench... 2023-09-23T20:56:30.007 INFO:tasks.cephadm.ceph_manager.ceph:creating pool_name unique_pool_1 2023-09-23T20:56:30.007 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph osd pool create unique_pool_1 16 2023-09-23T20:56:30.019 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:56:30.023 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:56:30.023 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:56:30.124 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:56:30.152 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:56:30.280 INFO:teuthology.orchestra.run.smithi026.stdout:{ 2023-09-23T20:56:30.280 INFO:teuthology.orchestra.run.smithi026.stdout: "size": 20, 2023-09-23T20:56:30.280 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 600, 2023-09-23T20:56:30.280 INFO:teuthology.orchestra.run.smithi026.stdout: "ops": [ 2023-09-23T20:56:30.280 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.280 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34797.0:114960 6.17 6:edae14e8:::benchmark_data_smithi026_223868_object13834:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e131)", 2023-09-23T20:56:30.281 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T20:54:12.901858+0000", 2023-09-23T20:56:30.281 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 137.37475573899999, 2023-09-23T20:56:30.281 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.1321539249999999, 2023-09-23T20:56:30.281 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T20:56:30.281 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "reached pg", 2023-09-23T20:56:30.281 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T20:56:30.281 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34797", 2023-09-23T20:56:30.282 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T20:56:30.282 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 114960 2023-09-23T20:56:30.282 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.282 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T20:56:30.282 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.282 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T20:56:30.283 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:12.901858+0000", 2023-09-23T20:56:30.283 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:56:30.283 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.283 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.283 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T20:56:30.283 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:12.901858+0000", 2023-09-23T20:56:30.283 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:56:30.284 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.284 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.284 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T20:56:30.284 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:12.901857+0000", 2023-09-23T20:56:30.284 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T20:56:30.284 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.285 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.285 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T20:56:30.285 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:12.901863+0000", 2023-09-23T20:56:30.285 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.3720000000000001e-06 2023-09-23T20:56:30.285 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.285 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.285 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T20:56:30.286 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:12.901875+0000", 2023-09-23T20:56:30.286 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1924e-05 2023-09-23T20:56:30.286 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.286 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.286 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T20:56:30.286 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:12.901883+0000", 2023-09-23T20:56:30.287 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.038e-06 2023-09-23T20:56:30.287 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.287 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.287 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:56:30.287 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:12.904218+0000", 2023-09-23T20:56:30.287 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0023356050000000001 2023-09-23T20:56:30.287 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.287 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.288 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for degraded object", 2023-09-23T20:56:30.288 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:12.909234+0000", 2023-09-23T20:56:30.288 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0050151680000000004 2023-09-23T20:56:30.288 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.288 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.288 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:56:30.288 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:12.927966+0000", 2023-09-23T20:56:30.289 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.018731945999999999 2023-09-23T20:56:30.289 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.289 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.289 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T20:56:30.289 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:12.928078+0000", 2023-09-23T20:56:30.289 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000112485 2023-09-23T20:56:30.290 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.290 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.290 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 1,7", 2023-09-23T20:56:30.290 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:12.928391+0000", 2023-09-23T20:56:30.290 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00031247000000000002 2023-09-23T20:56:30.290 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.291 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.291 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:56:30.291 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:12.951689+0000", 2023-09-23T20:56:30.291 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.023298762000000001 2023-09-23T20:56:30.291 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.291 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.291 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T20:56:30.291 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:12.951697+0000", 2023-09-23T20:56:30.291 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.8189999999999992e-06 2023-09-23T20:56:30.291 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.291 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.292 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:56:30.292 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:13.925895+0000", 2023-09-23T20:56:30.292 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.97419815399999998 2023-09-23T20:56:30.292 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.292 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.292 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T20:56:30.292 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:13.925906+0000", 2023-09-23T20:56:30.292 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0319000000000001e-05 2023-09-23T20:56:30.292 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.292 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.292 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:56:30.292 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:15.017793+0000", 2023-09-23T20:56:30.292 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0918876260000001 2023-09-23T20:56:30.292 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.293 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.293 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for missing object", 2023-09-23T20:56:30.293 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:15.017905+0000", 2023-09-23T20:56:30.293 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000112256 2023-09-23T20:56:30.293 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.293 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.293 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:56:30.293 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:15.033919+0000", 2023-09-23T20:56:30.293 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.016013263999999999 2023-09-23T20:56:30.293 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.293 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.293 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T20:56:30.293 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:15.034012+0000", 2023-09-23T20:56:30.294 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.3101000000000002e-05 2023-09-23T20:56:30.294 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:56:30.294 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T20:56:30.294 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:56:30.294 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.294 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.294 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34797.0:114965 6.17 6:eed136d7:::benchmark_data_smithi026_223868_object13839:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e131)", 2023-09-23T20:56:30.294 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T20:54:12.907032+0000", 2023-09-23T20:56:30.294 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 137.36958182999999, 2023-09-23T20:56:30.294 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.1278061450000001, 2023-09-23T20:56:30.294 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T20:56:30.294 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:56:30.295 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T20:56:30.295 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34797", 2023-09-23T20:56:30.295 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T20:56:30.295 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 114965 2023-09-23T20:56:30.295 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.295 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T20:56:30.295 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.295 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T20:56:30.295 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:12.907032+0000", 2023-09-23T20:56:30.296 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:56:30.296 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.296 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.296 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T20:56:30.296 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:12.907032+0000", 2023-09-23T20:56:30.296 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:56:30.297 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.297 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.297 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T20:56:30.297 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:12.907030+0000", 2023-09-23T20:56:30.297 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T20:56:30.297 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.297 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.297 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T20:56:30.297 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:12.907038+0000", 2023-09-23T20:56:30.297 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.3350000000000001e-06 2023-09-23T20:56:30.297 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.297 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.298 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T20:56:30.298 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:12.907050+0000", 2023-09-23T20:56:30.298 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.2398999999999999e-05 2023-09-23T20:56:30.298 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.298 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.298 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T20:56:30.298 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:12.907058+0000", 2023-09-23T20:56:30.298 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.1820000000000006e-06 2023-09-23T20:56:30.298 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.298 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.298 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:56:30.298 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:12.909340+0000", 2023-09-23T20:56:30.299 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.002281731 2023-09-23T20:56:30.299 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.299 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.299 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for degraded object", 2023-09-23T20:56:30.299 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:12.918086+0000", 2023-09-23T20:56:30.299 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.008745971 2023-09-23T20:56:30.299 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.299 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.299 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:56:30.299 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:12.952195+0000", 2023-09-23T20:56:30.299 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.034108861999999997 2023-09-23T20:56:30.299 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.299 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.300 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T20:56:30.300 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:12.952202+0000", 2023-09-23T20:56:30.300 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.686e-06 2023-09-23T20:56:30.300 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.300 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.300 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:56:30.300 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:13.925935+0000", 2023-09-23T20:56:30.300 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.97373368500000002 2023-09-23T20:56:30.300 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.300 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.300 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T20:56:30.300 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:13.925944+0000", 2023-09-23T20:56:30.301 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.9530000000000005e-06 2023-09-23T20:56:30.301 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.301 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.301 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:56:30.301 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:15.017933+0000", 2023-09-23T20:56:30.301 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0919890029999999 2023-09-23T20:56:30.301 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.301 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.302 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T20:56:30.302 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:15.018041+0000", 2023-09-23T20:56:30.302 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000107551 2023-09-23T20:56:30.302 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.302 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.302 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 1,4", 2023-09-23T20:56:30.302 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:15.018366+0000", 2023-09-23T20:56:30.303 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00032571500000000002 2023-09-23T20:56:30.303 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.303 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.303 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:56:30.303 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:15.031011+0000", 2023-09-23T20:56:30.303 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.012644479 2023-09-23T20:56:30.303 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.304 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.304 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:56:30.304 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:15.031048+0000", 2023-09-23T20:56:30.304 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.6822000000000001e-05 2023-09-23T20:56:30.304 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.304 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.304 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T20:56:30.304 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:15.034520+0000", 2023-09-23T20:56:30.305 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0034718459999999998 2023-09-23T20:56:30.305 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.305 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.305 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T20:56:30.305 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:15.034571+0000", 2023-09-23T20:56:30.305 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.1898999999999999e-05 2023-09-23T20:56:30.305 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.305 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.305 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T20:56:30.306 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:15.034838+0000", 2023-09-23T20:56:30.306 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00026648299999999998 2023-09-23T20:56:30.306 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:56:30.306 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T20:56:30.306 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:56:30.306 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.306 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.306 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34797.0:114977 6.16 6:6abc4ce0:::benchmark_data_smithi026_223868_object13851:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e131)", 2023-09-23T20:56:30.307 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T20:54:12.931076+0000", 2023-09-23T20:56:30.307 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 137.34553807699999, 2023-09-23T20:56:30.307 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.0790598340000002, 2023-09-23T20:56:30.307 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T20:56:30.307 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:56:30.307 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T20:56:30.307 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34797", 2023-09-23T20:56:30.307 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T20:56:30.307 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 114977 2023-09-23T20:56:30.308 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.308 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T20:56:30.308 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.308 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T20:56:30.308 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:12.931076+0000", 2023-09-23T20:56:30.308 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:56:30.308 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.308 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.308 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T20:56:30.309 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:12.931076+0000", 2023-09-23T20:56:30.309 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:56:30.309 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.309 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.309 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T20:56:30.309 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:12.931075+0000", 2023-09-23T20:56:30.309 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T20:56:30.309 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.310 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.310 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T20:56:30.310 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:12.931079+0000", 2023-09-23T20:56:30.310 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.5200000000000002e-06 2023-09-23T20:56:30.310 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.310 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.310 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T20:56:30.311 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:12.931086+0000", 2023-09-23T20:56:30.311 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.0299999999999996e-06 2023-09-23T20:56:30.311 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.311 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.311 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T20:56:30.311 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:12.931108+0000", 2023-09-23T20:56:30.311 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.2226999999999999e-05 2023-09-23T20:56:30.311 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.312 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.312 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:56:30.312 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:12.931257+0000", 2023-09-23T20:56:30.312 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00014939900000000001 2023-09-23T20:56:30.312 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.312 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.312 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T20:56:30.312 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:12.931281+0000", 2023-09-23T20:56:30.313 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.4057e-05 2023-09-23T20:56:30.313 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.313 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.313 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:56:30.313 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:13.925102+0000", 2023-09-23T20:56:30.313 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.993820284 2023-09-23T20:56:30.313 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.313 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.314 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T20:56:30.314 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:13.925116+0000", 2023-09-23T20:56:30.314 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.3886000000000001e-05 2023-09-23T20:56:30.314 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.314 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.314 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:56:30.314 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:15.000774+0000", 2023-09-23T20:56:30.314 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0756585869999999 2023-09-23T20:56:30.315 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.315 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.315 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T20:56:30.315 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:15.000859+0000", 2023-09-23T20:56:30.315 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.5370000000000004e-05 2023-09-23T20:56:30.315 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.315 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.315 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 4,7", 2023-09-23T20:56:30.316 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:15.001678+0000", 2023-09-23T20:56:30.316 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000818913 2023-09-23T20:56:30.316 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.316 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.316 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:56:30.316 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:15.008665+0000", 2023-09-23T20:56:30.316 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0069865600000000002 2023-09-23T20:56:30.316 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.316 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.317 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:56:30.317 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:15.008706+0000", 2023-09-23T20:56:30.317 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.1285999999999997e-05 2023-09-23T20:56:30.317 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.317 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.317 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T20:56:30.317 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:15.009602+0000", 2023-09-23T20:56:30.317 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00089599299999999998 2023-09-23T20:56:30.317 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.318 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.318 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T20:56:30.318 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:15.009659+0000", 2023-09-23T20:56:30.318 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.6289999999999998e-05 2023-09-23T20:56:30.318 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.318 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.318 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T20:56:30.318 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:15.010135+0000", 2023-09-23T20:56:30.319 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00047690699999999999 2023-09-23T20:56:30.319 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:56:30.319 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T20:56:30.319 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:56:30.319 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.319 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.319 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34797.0:114989 6.6 6:60ed2ddf:::benchmark_data_smithi026_223868_object13863:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e132)", 2023-09-23T20:56:30.319 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T20:54:12.964779+0000", 2023-09-23T20:56:30.320 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 137.31183437799999, 2023-09-23T20:56:30.320 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.0443830630000002, 2023-09-23T20:56:30.320 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T20:56:30.320 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:56:30.320 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T20:56:30.320 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34797", 2023-09-23T20:56:30.320 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T20:56:30.320 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 114989 2023-09-23T20:56:30.320 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.321 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T20:56:30.321 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.321 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T20:56:30.321 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:12.964779+0000", 2023-09-23T20:56:30.321 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:56:30.321 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.321 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.321 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T20:56:30.321 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:12.964779+0000", 2023-09-23T20:56:30.321 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:56:30.321 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.321 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.322 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T20:56:30.322 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:12.964778+0000", 2023-09-23T20:56:30.322 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T20:56:30.322 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.322 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.322 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T20:56:30.322 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:12.964784+0000", 2023-09-23T20:56:30.322 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.8989999999999998e-06 2023-09-23T20:56:30.322 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.322 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.323 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T20:56:30.323 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:12.964795+0000", 2023-09-23T20:56:30.323 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1214e-05 2023-09-23T20:56:30.323 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.323 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.323 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T20:56:30.323 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:12.964805+0000", 2023-09-23T20:56:30.323 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0346999999999999e-05 2023-09-23T20:56:30.323 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.323 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.323 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:56:30.324 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:12.964859+0000", 2023-09-23T20:56:30.324 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.3591000000000002e-05 2023-09-23T20:56:30.324 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.324 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.324 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T20:56:30.324 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:12.964871+0000", 2023-09-23T20:56:30.324 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.2109999999999999e-05 2023-09-23T20:56:30.324 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.324 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.324 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:56:30.324 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:13.925234+0000", 2023-09-23T20:56:30.324 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.96036344399999996 2023-09-23T20:56:30.324 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.325 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.325 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T20:56:30.325 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:13.925245+0000", 2023-09-23T20:56:30.325 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0054999999999999e-05 2023-09-23T20:56:30.325 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.325 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.325 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:56:30.325 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:14.996272+0000", 2023-09-23T20:56:30.325 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0710279009999999 2023-09-23T20:56:30.325 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.326 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.326 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T20:56:30.326 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:14.996437+0000", 2023-09-23T20:56:30.326 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00016484299999999999 2023-09-23T20:56:30.326 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.326 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.326 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 4,7", 2023-09-23T20:56:30.326 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:14.996837+0000", 2023-09-23T20:56:30.326 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00039951599999999999 2023-09-23T20:56:30.327 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.327 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.327 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:56:30.327 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:14.998787+0000", 2023-09-23T20:56:30.327 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0019506770000000001 2023-09-23T20:56:30.327 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.327 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.327 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:56:30.327 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:15.001358+0000", 2023-09-23T20:56:30.327 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0025709750000000001 2023-09-23T20:56:30.327 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.327 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.327 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T20:56:30.328 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:15.008625+0000", 2023-09-23T20:56:30.328 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0072665230000000004 2023-09-23T20:56:30.328 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.328 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.328 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T20:56:30.328 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:15.008685+0000", 2023-09-23T20:56:30.328 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.0244000000000001e-05 2023-09-23T20:56:30.328 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.328 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.328 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T20:56:30.328 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:15.009162+0000", 2023-09-23T20:56:30.328 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00047711299999999998 2023-09-23T20:56:30.328 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:56:30.329 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T20:56:30.329 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:56:30.329 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.329 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.329 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34797.0:114991 6.17 6:ed158f03:::benchmark_data_smithi026_223868_object13865:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e132)", 2023-09-23T20:56:30.329 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T20:54:12.968317+0000", 2023-09-23T20:56:30.329 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 137.30829616599999, 2023-09-23T20:56:30.329 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.0674853450000001, 2023-09-23T20:56:30.329 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T20:56:30.329 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:56:30.329 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T20:56:30.329 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34797", 2023-09-23T20:56:30.329 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T20:56:30.330 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 114991 2023-09-23T20:56:30.330 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.330 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T20:56:30.330 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.330 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T20:56:30.330 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:12.968317+0000", 2023-09-23T20:56:30.330 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:56:30.330 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.330 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.330 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T20:56:30.331 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:12.968317+0000", 2023-09-23T20:56:30.331 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:56:30.331 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.331 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.331 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T20:56:30.331 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:12.968316+0000", 2023-09-23T20:56:30.331 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T20:56:30.331 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.331 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.331 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T20:56:30.331 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:12.968321+0000", 2023-09-23T20:56:30.331 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.2669999999999997e-06 2023-09-23T20:56:30.331 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.332 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.332 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T20:56:30.332 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:12.968334+0000", 2023-09-23T20:56:30.332 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.3101999999999999e-05 2023-09-23T20:56:30.332 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.332 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.332 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T20:56:30.332 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:12.968342+0000", 2023-09-23T20:56:30.332 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.5089999999999996e-06 2023-09-23T20:56:30.332 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.332 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.333 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:56:30.333 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:12.968406+0000", 2023-09-23T20:56:30.333 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.4132000000000002e-05 2023-09-23T20:56:30.333 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.333 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.333 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T20:56:30.333 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:12.968418+0000", 2023-09-23T20:56:30.333 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.2459999999999999e-05 2023-09-23T20:56:30.333 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.333 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.333 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:56:30.333 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:13.925969+0000", 2023-09-23T20:56:30.333 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.95755082800000002 2023-09-23T20:56:30.334 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.334 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.334 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T20:56:30.334 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:13.925978+0000", 2023-09-23T20:56:30.334 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.0149999999999992e-06 2023-09-23T20:56:30.334 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.334 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.334 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:56:30.334 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:15.019699+0000", 2023-09-23T20:56:30.334 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0937209290000001 2023-09-23T20:56:30.334 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.334 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.335 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T20:56:30.335 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:15.019848+0000", 2023-09-23T20:56:30.335 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00014907899999999999 2023-09-23T20:56:30.335 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.335 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.335 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 1,4", 2023-09-23T20:56:30.335 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:15.020197+0000", 2023-09-23T20:56:30.335 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000348964 2023-09-23T20:56:30.335 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.335 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.335 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:56:30.335 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:15.033098+0000", 2023-09-23T20:56:30.335 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.012900327 2023-09-23T20:56:30.336 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.336 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.336 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T20:56:30.336 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:15.034847+0000", 2023-09-23T20:56:30.336 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0017495 2023-09-23T20:56:30.336 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.336 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.336 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:56:30.336 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:15.035482+0000", 2023-09-23T20:56:30.337 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00063472799999999998 2023-09-23T20:56:30.337 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.337 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.337 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T20:56:30.337 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:15.035528+0000", 2023-09-23T20:56:30.337 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.5874000000000002e-05 2023-09-23T20:56:30.337 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.337 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.337 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T20:56:30.338 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:15.035803+0000", 2023-09-23T20:56:30.338 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00027521799999999998 2023-09-23T20:56:30.338 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:56:30.338 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T20:56:30.338 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:56:30.338 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.338 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.339 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34797.0:114998 6.7 6:e21330c6:::benchmark_data_smithi026_223868_object13872:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e132)", 2023-09-23T20:56:30.339 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T20:54:12.985057+0000", 2023-09-23T20:56:30.339 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 137.291556464, 2023-09-23T20:56:30.339 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.0503685319999998, 2023-09-23T20:56:30.339 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T20:56:30.339 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:56:30.339 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T20:56:30.339 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34797", 2023-09-23T20:56:30.339 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T20:56:30.340 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 114998 2023-09-23T20:56:30.340 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.340 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T20:56:30.340 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.340 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T20:56:30.340 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:12.985057+0000", 2023-09-23T20:56:30.340 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:56:30.340 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.341 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.341 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T20:56:30.341 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:12.985057+0000", 2023-09-23T20:56:30.341 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:56:30.341 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.341 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.341 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T20:56:30.341 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:12.985054+0000", 2023-09-23T20:56:30.341 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999971 2023-09-23T20:56:30.342 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.342 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.342 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T20:56:30.342 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:12.985066+0000", 2023-09-23T20:56:30.342 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.2031000000000001e-05 2023-09-23T20:56:30.342 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.342 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.342 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T20:56:30.343 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:12.985089+0000", 2023-09-23T20:56:30.343 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.2501e-05 2023-09-23T20:56:30.343 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.343 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.343 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T20:56:30.343 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:12.985103+0000", 2023-09-23T20:56:30.344 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.4392e-05 2023-09-23T20:56:30.344 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.344 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.344 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:56:30.344 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:12.988980+0000", 2023-09-23T20:56:30.344 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0038764509999999999 2023-09-23T20:56:30.344 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.345 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.345 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T20:56:30.345 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:12.988996+0000", 2023-09-23T20:56:30.345 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.5953000000000001e-05 2023-09-23T20:56:30.345 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.345 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.345 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:56:30.346 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:13.925849+0000", 2023-09-23T20:56:30.346 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.93685326800000002 2023-09-23T20:56:30.346 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.346 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.346 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T20:56:30.346 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:13.925860+0000", 2023-09-23T20:56:30.346 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1090000000000001e-05 2023-09-23T20:56:30.347 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.347 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.347 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:56:30.347 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:15.019676+0000", 2023-09-23T20:56:30.347 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0938154149999999 2023-09-23T20:56:30.347 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.347 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.348 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T20:56:30.348 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:15.019809+0000", 2023-09-23T20:56:30.348 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00013316599999999999 2023-09-23T20:56:30.348 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.348 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.348 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 1,4", 2023-09-23T20:56:30.348 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:15.020197+0000", 2023-09-23T20:56:30.348 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00038824999999999998 2023-09-23T20:56:30.349 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.349 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.349 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:56:30.349 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:15.033893+0000", 2023-09-23T20:56:30.349 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.013696442 2023-09-23T20:56:30.349 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.349 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.350 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T20:56:30.350 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:15.034854+0000", 2023-09-23T20:56:30.350 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00096095700000000002 2023-09-23T20:56:30.350 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.350 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.350 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:56:30.350 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:15.034911+0000", 2023-09-23T20:56:30.351 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.6915e-05 2023-09-23T20:56:30.351 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.351 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.351 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T20:56:30.351 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:15.034967+0000", 2023-09-23T20:56:30.351 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.6249000000000003e-05 2023-09-23T20:56:30.351 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.352 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.352 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T20:56:30.352 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:15.035426+0000", 2023-09-23T20:56:30.352 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00045822899999999999 2023-09-23T20:56:30.352 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:56:30.352 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T20:56:30.352 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:56:30.353 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.353 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.353 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34797.0:115009 6.7 6:e365786c:::benchmark_data_smithi026_223868_object13883:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e133)", 2023-09-23T20:56:30.353 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T20:54:14.026745+0000", 2023-09-23T20:56:30.353 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 136.24986849300001, 2023-09-23T20:56:30.353 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0094194009999999, 2023-09-23T20:56:30.353 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T20:56:30.353 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:56:30.354 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T20:56:30.354 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34797", 2023-09-23T20:56:30.354 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T20:56:30.354 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 115009 2023-09-23T20:56:30.354 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.354 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T20:56:30.354 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.354 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T20:56:30.354 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:14.026745+0000", 2023-09-23T20:56:30.354 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:56:30.354 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.355 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.355 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T20:56:30.355 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:14.026745+0000", 2023-09-23T20:56:30.355 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:56:30.355 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.355 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.355 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T20:56:30.355 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:14.026743+0000", 2023-09-23T20:56:30.355 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999981 2023-09-23T20:56:30.355 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.355 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.355 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T20:56:30.355 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:14.026752+0000", 2023-09-23T20:56:30.356 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.1670000000000005e-06 2023-09-23T20:56:30.356 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.356 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.356 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T20:56:30.356 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:14.026774+0000", 2023-09-23T20:56:30.356 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.1203e-05 2023-09-23T20:56:30.356 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.356 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.356 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T20:56:30.356 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:14.026782+0000", 2023-09-23T20:56:30.356 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.4370000000000007e-06 2023-09-23T20:56:30.356 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.356 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.357 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:56:30.357 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:14.026855+0000", 2023-09-23T20:56:30.357 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.2911999999999996e-05 2023-09-23T20:56:30.357 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.357 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.357 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T20:56:30.357 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:14.026870+0000", 2023-09-23T20:56:30.357 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.4718e-05 2023-09-23T20:56:30.357 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.357 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.357 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:56:30.357 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:15.023554+0000", 2023-09-23T20:56:30.357 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.99668419500000005 2023-09-23T20:56:30.358 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.358 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.358 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T20:56:30.358 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:15.023665+0000", 2023-09-23T20:56:30.358 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000110841 2023-09-23T20:56:30.358 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.358 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.358 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 1,4", 2023-09-23T20:56:30.358 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:15.024005+0000", 2023-09-23T20:56:30.358 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00034030399999999999 2023-09-23T20:56:30.358 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.359 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.359 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:56:30.359 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:15.034505+0000", 2023-09-23T20:56:30.359 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.010499812000000001 2023-09-23T20:56:30.359 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.359 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.359 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T20:56:30.359 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:15.034861+0000", 2023-09-23T20:56:30.359 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000355991 2023-09-23T20:56:30.359 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.359 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.359 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:56:30.359 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:15.035843+0000", 2023-09-23T20:56:30.360 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00098196299999999998 2023-09-23T20:56:30.360 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.360 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.360 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T20:56:30.360 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:15.035888+0000", 2023-09-23T20:56:30.360 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.4678000000000002e-05 2023-09-23T20:56:30.360 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.360 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.360 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T20:56:30.360 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:15.036165+0000", 2023-09-23T20:56:30.360 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00027704800000000001 2023-09-23T20:56:30.360 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:56:30.360 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T20:56:30.361 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:56:30.361 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.361 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.361 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34797.0:130469 6.c 6:38e631ce:::benchmark_data_smithi026_223868_object29343:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e135)", 2023-09-23T20:56:30.361 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T20:54:30.880090+0000", 2023-09-23T20:56:30.361 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 119.3965232, 2023-09-23T20:56:30.361 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.1674490940000002, 2023-09-23T20:56:30.361 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T20:56:30.361 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "reached pg", 2023-09-23T20:56:30.361 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T20:56:30.361 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34797", 2023-09-23T20:56:30.361 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T20:56:30.361 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 130469 2023-09-23T20:56:30.362 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.362 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T20:56:30.362 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.362 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T20:56:30.362 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:30.880090+0000", 2023-09-23T20:56:30.362 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:56:30.362 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.362 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.362 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T20:56:30.362 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:30.880090+0000", 2023-09-23T20:56:30.362 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:56:30.363 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.363 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.363 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T20:56:30.363 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:30.880088+0000", 2023-09-23T20:56:30.363 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999981 2023-09-23T20:56:30.363 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.363 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.363 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T20:56:30.363 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:30.880097+0000", 2023-09-23T20:56:30.364 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.4610000000000008e-06 2023-09-23T20:56:30.364 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.364 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.364 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T20:56:30.364 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:30.880115+0000", 2023-09-23T20:56:30.364 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.7901e-05 2023-09-23T20:56:30.364 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.364 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.364 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T20:56:30.364 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:30.880124+0000", 2023-09-23T20:56:30.364 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.3379999999999994e-06 2023-09-23T20:56:30.364 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.364 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.365 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:56:30.365 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:30.880185+0000", 2023-09-23T20:56:30.365 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.1262999999999999e-05 2023-09-23T20:56:30.365 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.365 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.365 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T20:56:30.365 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:30.880348+0000", 2023-09-23T20:56:30.365 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00016296799999999999 2023-09-23T20:56:30.365 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.365 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.365 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 6,7", 2023-09-23T20:56:30.365 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:30.880434+0000", 2023-09-23T20:56:30.365 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.5793999999999999e-05 2023-09-23T20:56:30.366 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.366 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.366 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T20:56:30.366 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:30.888682+0000", 2023-09-23T20:56:30.366 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0082474290000000006 2023-09-23T20:56:30.366 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.366 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.366 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:56:30.366 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:30.915595+0000", 2023-09-23T20:56:30.366 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.026913060999999999 2023-09-23T20:56:30.366 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.366 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.367 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T20:56:30.367 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:30.915608+0000", 2023-09-23T20:56:30.367 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.3742e-05 2023-09-23T20:56:30.367 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.367 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.367 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:56:30.367 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:31.887950+0000", 2023-09-23T20:56:30.367 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.97234175 2023-09-23T20:56:30.367 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.367 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.367 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T20:56:30.367 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:31.887964+0000", 2023-09-23T20:56:30.367 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.3711e-05 2023-09-23T20:56:30.367 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.368 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.368 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:56:30.368 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:33.036111+0000", 2023-09-23T20:56:30.368 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1481468340000001 2023-09-23T20:56:30.368 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.368 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.368 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for missing object", 2023-09-23T20:56:30.368 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:33.036258+0000", 2023-09-23T20:56:30.368 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00014725900000000001 2023-09-23T20:56:30.368 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.368 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.368 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:56:30.368 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:33.047449+0000", 2023-09-23T20:56:30.369 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.011190752 2023-09-23T20:56:30.369 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.369 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.369 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T20:56:30.369 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:33.047540+0000", 2023-09-23T20:56:30.369 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.0873999999999998e-05 2023-09-23T20:56:30.369 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:56:30.369 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T20:56:30.369 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:56:30.369 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.369 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.369 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34797.0:130480 6.c 6:3c6c8e98:::benchmark_data_smithi026_223868_object29354:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e135)", 2023-09-23T20:56:30.369 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T20:54:30.892126+0000", 2023-09-23T20:56:30.370 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 119.384487347, 2023-09-23T20:56:30.370 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.1590397989999999, 2023-09-23T20:56:30.370 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T20:56:30.370 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "reached pg", 2023-09-23T20:56:30.370 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T20:56:30.370 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34797", 2023-09-23T20:56:30.370 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T20:56:30.370 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 130480 2023-09-23T20:56:30.370 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.370 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T20:56:30.370 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.370 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T20:56:30.371 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:30.892126+0000", 2023-09-23T20:56:30.371 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:56:30.371 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.371 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.371 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T20:56:30.371 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:30.892126+0000", 2023-09-23T20:56:30.371 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:56:30.371 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.371 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.371 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T20:56:30.371 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:30.892125+0000", 2023-09-23T20:56:30.371 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T20:56:30.371 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.372 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.372 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T20:56:30.372 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:30.892131+0000", 2023-09-23T20:56:30.372 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.7209999999999997e-06 2023-09-23T20:56:30.372 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.372 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.372 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T20:56:30.372 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:30.892144+0000", 2023-09-23T20:56:30.372 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.2488e-05 2023-09-23T20:56:30.372 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.372 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.372 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T20:56:30.372 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:30.892152+0000", 2023-09-23T20:56:30.372 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.7330000000000003e-06 2023-09-23T20:56:30.373 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.373 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.373 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:56:30.373 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:30.892201+0000", 2023-09-23T20:56:30.373 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.9678000000000001e-05 2023-09-23T20:56:30.373 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.373 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.373 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T20:56:30.373 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:30.892324+0000", 2023-09-23T20:56:30.373 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00012292299999999999 2023-09-23T20:56:30.373 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.373 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.374 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 6,7", 2023-09-23T20:56:30.374 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:30.892438+0000", 2023-09-23T20:56:30.374 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000113915 2023-09-23T20:56:30.374 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.374 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.374 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T20:56:30.374 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:30.907285+0000", 2023-09-23T20:56:30.374 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.014847057 2023-09-23T20:56:30.374 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.375 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.375 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:56:30.375 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:30.915644+0000", 2023-09-23T20:56:30.375 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0083591480000000003 2023-09-23T20:56:30.375 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.375 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.375 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T20:56:30.375 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:30.915653+0000", 2023-09-23T20:56:30.375 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.4300000000000006e-06 2023-09-23T20:56:30.375 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.375 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.375 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:56:30.375 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:31.888064+0000", 2023-09-23T20:56:30.376 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.97241095300000002 2023-09-23T20:56:30.376 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.376 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.376 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T20:56:30.376 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:31.888069+0000", 2023-09-23T20:56:30.376 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.6450000000000002e-06 2023-09-23T20:56:30.376 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.376 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.376 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:56:30.376 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:33.037086+0000", 2023-09-23T20:56:30.376 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1490164059999999 2023-09-23T20:56:30.376 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.376 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.376 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for missing object", 2023-09-23T20:56:30.377 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:33.037184+0000", 2023-09-23T20:56:30.377 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.8024999999999997e-05 2023-09-23T20:56:30.377 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.377 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.377 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:56:30.377 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:33.051071+0000", 2023-09-23T20:56:30.377 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.013886895999999999 2023-09-23T20:56:30.377 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.377 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.377 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T20:56:30.377 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:33.051166+0000", 2023-09-23T20:56:30.377 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.5426000000000004e-05 2023-09-23T20:56:30.377 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:56:30.378 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T20:56:30.378 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:56:30.378 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.378 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.378 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34797.0:130484 6.17 6:ebb95d79:::benchmark_data_smithi026_223868_object29358:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e136)", 2023-09-23T20:56:30.378 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T20:54:30.909236+0000", 2023-09-23T20:56:30.378 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 119.367377695, 2023-09-23T20:56:30.378 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.1373689279999999, 2023-09-23T20:56:30.378 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T20:56:30.378 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:56:30.378 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T20:56:30.379 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34797", 2023-09-23T20:56:30.379 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T20:56:30.379 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 130484 2023-09-23T20:56:30.379 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.379 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T20:56:30.379 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.379 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T20:56:30.379 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:30.909236+0000", 2023-09-23T20:56:30.379 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:56:30.379 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.379 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.379 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T20:56:30.379 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:30.909236+0000", 2023-09-23T20:56:30.380 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:56:30.380 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.380 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.380 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T20:56:30.380 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:30.909234+0000", 2023-09-23T20:56:30.380 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999981 2023-09-23T20:56:30.380 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.380 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.380 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T20:56:30.380 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:30.909241+0000", 2023-09-23T20:56:30.380 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.9630000000000001e-06 2023-09-23T20:56:30.380 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.381 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.381 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T20:56:30.381 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:30.909258+0000", 2023-09-23T20:56:30.381 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.7105999999999999e-05 2023-09-23T20:56:30.381 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.381 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.381 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T20:56:30.381 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:30.909267+0000", 2023-09-23T20:56:30.381 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.0440000000000001e-06 2023-09-23T20:56:30.381 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.381 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.381 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:56:30.381 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:30.914297+0000", 2023-09-23T20:56:30.382 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0050297720000000001 2023-09-23T20:56:30.382 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.382 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.382 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T20:56:30.382 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:30.914307+0000", 2023-09-23T20:56:30.382 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0166000000000001e-05 2023-09-23T20:56:30.382 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.382 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.382 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:56:30.382 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:31.880008+0000", 2023-09-23T20:56:30.382 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.96570099399999998 2023-09-23T20:56:30.382 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.383 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.383 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T20:56:30.383 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:31.880021+0000", 2023-09-23T20:56:30.383 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.2700999999999999e-05 2023-09-23T20:56:30.383 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.383 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.383 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:56:30.383 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:33.017285+0000", 2023-09-23T20:56:30.383 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1372641450000001 2023-09-23T20:56:30.383 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.383 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.383 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T20:56:30.383 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:33.017380+0000", 2023-09-23T20:56:30.384 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.5118000000000006e-05 2023-09-23T20:56:30.384 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.384 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.384 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 1,4,5", 2023-09-23T20:56:30.384 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:33.017728+0000", 2023-09-23T20:56:30.384 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000348 2023-09-23T20:56:30.384 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.384 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.384 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:56:30.384 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:33.021064+0000", 2023-09-23T20:56:30.385 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0033353430000000002 2023-09-23T20:56:30.385 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.385 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.385 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:56:30.385 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:33.042239+0000", 2023-09-23T20:56:30.385 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.021175225999999998 2023-09-23T20:56:30.386 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.386 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.386 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:56:30.386 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:33.042839+0000", 2023-09-23T20:56:30.386 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00060004999999999998 2023-09-23T20:56:30.386 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.386 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.386 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T20:56:30.386 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:33.046070+0000", 2023-09-23T20:56:30.386 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0032314000000000002 2023-09-23T20:56:30.387 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.387 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.387 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T20:56:30.387 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:33.046122+0000", 2023-09-23T20:56:30.387 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.1583999999999999e-05 2023-09-23T20:56:30.387 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.387 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.387 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T20:56:30.387 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:33.046605+0000", 2023-09-23T20:56:30.387 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000483005 2023-09-23T20:56:30.387 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:56:30.387 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T20:56:30.387 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:56:30.388 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.388 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.388 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34797.0:130487 6.c 6:3110d152:::benchmark_data_smithi026_223868_object29361:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e136)", 2023-09-23T20:56:30.388 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T20:54:30.913625+0000", 2023-09-23T20:56:30.388 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 119.362988981, 2023-09-23T20:56:30.388 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.1412422530000002, 2023-09-23T20:56:30.388 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T20:56:30.388 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:56:30.388 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T20:56:30.388 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34797", 2023-09-23T20:56:30.388 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T20:56:30.388 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 130487 2023-09-23T20:56:30.389 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.389 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T20:56:30.389 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.389 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T20:56:30.389 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:30.913625+0000", 2023-09-23T20:56:30.389 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:56:30.389 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.389 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.389 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T20:56:30.389 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:30.913625+0000", 2023-09-23T20:56:30.389 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:56:30.389 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.390 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.390 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T20:56:30.390 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:30.913623+0000", 2023-09-23T20:56:30.390 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T20:56:30.390 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.390 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.390 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T20:56:30.390 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:30.913631+0000", 2023-09-23T20:56:30.390 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.5979999999999999e-06 2023-09-23T20:56:30.390 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.390 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.390 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T20:56:30.391 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:30.913643+0000", 2023-09-23T20:56:30.391 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.223e-05 2023-09-23T20:56:30.391 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.391 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.391 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T20:56:30.391 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:30.913665+0000", 2023-09-23T20:56:30.391 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.1792000000000001e-05 2023-09-23T20:56:30.391 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.391 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.391 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:56:30.391 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:30.915686+0000", 2023-09-23T20:56:30.391 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0020213150000000001 2023-09-23T20:56:30.391 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.391 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.392 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T20:56:30.392 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:30.915699+0000", 2023-09-23T20:56:30.392 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.2496e-05 2023-09-23T20:56:30.392 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.392 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.392 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:56:30.392 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:31.888085+0000", 2023-09-23T20:56:30.392 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.97238664500000005 2023-09-23T20:56:30.392 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.392 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.392 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T20:56:30.392 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:31.888090+0000", 2023-09-23T20:56:30.392 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.4839999999999998e-06 2023-09-23T20:56:30.392 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.393 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.393 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:56:30.393 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:33.037214+0000", 2023-09-23T20:56:30.393 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1491243719999999 2023-09-23T20:56:30.393 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.393 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.393 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T20:56:30.393 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:33.037330+0000", 2023-09-23T20:56:30.393 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000115977 2023-09-23T20:56:30.393 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.393 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.393 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 5,6,7", 2023-09-23T20:56:30.393 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:33.037423+0000", 2023-09-23T20:56:30.394 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.2723e-05 2023-09-23T20:56:30.394 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.394 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.394 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:56:30.394 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:33.046955+0000", 2023-09-23T20:56:30.394 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0095323809999999995 2023-09-23T20:56:30.394 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.394 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.394 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:56:30.394 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:33.047055+0000", 2023-09-23T20:56:30.394 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.9870000000000004e-05 2023-09-23T20:56:30.395 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.395 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.395 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:56:30.395 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:33.051207+0000", 2023-09-23T20:56:30.395 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0041521850000000001 2023-09-23T20:56:30.395 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.395 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.395 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T20:56:30.395 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:33.054703+0000", 2023-09-23T20:56:30.395 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0034956599999999998 2023-09-23T20:56:30.395 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.395 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.395 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T20:56:30.395 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:33.054753+0000", 2023-09-23T20:56:30.396 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.0430999999999999e-05 2023-09-23T20:56:30.396 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.396 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.396 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T20:56:30.396 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:33.054867+0000", 2023-09-23T20:56:30.396 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000113626 2023-09-23T20:56:30.396 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:56:30.396 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T20:56:30.396 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:56:30.396 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.396 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.397 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34797.0:130508 6.c 6:33de9cb1:::benchmark_data_smithi026_223868_object29382:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e136)", 2023-09-23T20:56:30.397 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T20:54:30.927850+0000", 2023-09-23T20:56:30.397 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 119.348763166, 2023-09-23T20:56:30.397 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.1274186720000001, 2023-09-23T20:56:30.397 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T20:56:30.397 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:56:30.397 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T20:56:30.397 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34797", 2023-09-23T20:56:30.397 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T20:56:30.398 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 130508 2023-09-23T20:56:30.398 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.398 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T20:56:30.398 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.398 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T20:56:30.398 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:30.927850+0000", 2023-09-23T20:56:30.398 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:56:30.399 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.399 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.399 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T20:56:30.399 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:30.927850+0000", 2023-09-23T20:56:30.399 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:56:30.399 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.399 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.399 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T20:56:30.399 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:30.927850+0000", 2023-09-23T20:56:30.399 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T20:56:30.400 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.400 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.400 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T20:56:30.400 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:30.927854+0000", 2023-09-23T20:56:30.400 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.0949999999999998e-06 2023-09-23T20:56:30.400 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.400 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.400 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T20:56:30.400 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:30.927866+0000", 2023-09-23T20:56:30.400 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1843e-05 2023-09-23T20:56:30.401 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.401 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.401 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T20:56:30.401 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:30.927871+0000", 2023-09-23T20:56:30.401 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.9490000000000004e-06 2023-09-23T20:56:30.401 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.401 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.401 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:56:30.401 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:30.927913+0000", 2023-09-23T20:56:30.402 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.2367999999999998e-05 2023-09-23T20:56:30.402 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.402 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.402 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T20:56:30.402 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:30.927921+0000", 2023-09-23T20:56:30.402 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.2339999999999994e-06 2023-09-23T20:56:30.402 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.402 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.402 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:56:30.402 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:31.888102+0000", 2023-09-23T20:56:30.402 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.960181116 2023-09-23T20:56:30.403 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.403 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.403 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T20:56:30.403 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:31.888106+0000", 2023-09-23T20:56:30.403 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.0969999999999999e-06 2023-09-23T20:56:30.403 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.403 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.403 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:56:30.403 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:33.041530+0000", 2023-09-23T20:56:30.403 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1534237329999999 2023-09-23T20:56:30.403 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.403 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.403 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T20:56:30.404 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:33.041653+0000", 2023-09-23T20:56:30.404 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00012306099999999999 2023-09-23T20:56:30.404 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.404 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.404 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 5,6,7", 2023-09-23T20:56:30.404 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:33.041780+0000", 2023-09-23T20:56:30.404 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000126978 2023-09-23T20:56:30.404 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.404 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.404 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:56:30.404 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:33.047098+0000", 2023-09-23T20:56:30.404 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0053179400000000002 2023-09-23T20:56:30.404 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.405 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.405 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:56:30.405 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:33.047424+0000", 2023-09-23T20:56:30.405 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00032540299999999999 2023-09-23T20:56:30.405 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.405 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.405 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:56:30.405 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:33.051248+0000", 2023-09-23T20:56:30.405 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.003824858 2023-09-23T20:56:30.405 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.405 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.405 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T20:56:30.406 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:33.055098+0000", 2023-09-23T20:56:30.406 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0038491419999999998 2023-09-23T20:56:30.406 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.406 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.406 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T20:56:30.406 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:33.055154+0000", 2023-09-23T20:56:30.406 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.6585000000000001e-05 2023-09-23T20:56:30.406 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.406 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.407 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T20:56:30.407 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:33.055269+0000", 2023-09-23T20:56:30.407 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000114967 2023-09-23T20:56:30.407 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:56:30.407 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T20:56:30.407 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:56:30.407 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.407 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.407 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34797.0:130522 6.c 6:3f85e3c7:::benchmark_data_smithi026_223868_object29396:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e136)", 2023-09-23T20:56:30.407 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T20:54:30.945451+0000", 2023-09-23T20:56:30.407 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 119.33116312, 2023-09-23T20:56:30.407 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.1100408050000001, 2023-09-23T20:56:30.407 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T20:56:30.408 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:56:30.408 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T20:56:30.408 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34797", 2023-09-23T20:56:30.408 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T20:56:30.408 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 130522 2023-09-23T20:56:30.408 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.408 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T20:56:30.408 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.408 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T20:56:30.408 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:30.945451+0000", 2023-09-23T20:56:30.408 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:56:30.408 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.408 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.408 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T20:56:30.409 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:30.945451+0000", 2023-09-23T20:56:30.409 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:56:30.409 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.409 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.409 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T20:56:30.409 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:30.945449+0000", 2023-09-23T20:56:30.409 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T20:56:30.409 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.409 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.409 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T20:56:30.409 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:30.945456+0000", 2023-09-23T20:56:30.409 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.9360000000000002e-06 2023-09-23T20:56:30.409 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.410 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.410 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T20:56:30.410 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:30.945469+0000", 2023-09-23T20:56:30.410 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.3597e-05 2023-09-23T20:56:30.410 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.410 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.410 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T20:56:30.410 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:30.945490+0000", 2023-09-23T20:56:30.410 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.0840999999999999e-05 2023-09-23T20:56:30.410 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.410 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.410 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:56:30.411 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:30.945602+0000", 2023-09-23T20:56:30.411 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000112007 2023-09-23T20:56:30.411 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.411 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.411 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T20:56:30.411 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:30.945610+0000", 2023-09-23T20:56:30.411 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.9149999999999996e-06 2023-09-23T20:56:30.411 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.411 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.411 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:56:30.411 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:31.888125+0000", 2023-09-23T20:56:30.411 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.94251427099999996 2023-09-23T20:56:30.411 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.411 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.412 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T20:56:30.412 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:31.888132+0000", 2023-09-23T20:56:30.412 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.4599999999999997e-06 2023-09-23T20:56:30.412 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.412 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.412 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:56:30.412 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:33.047604+0000", 2023-09-23T20:56:30.412 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.159472442 2023-09-23T20:56:30.412 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.412 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.412 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T20:56:30.412 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:33.047716+0000", 2023-09-23T20:56:30.413 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00011191700000000001 2023-09-23T20:56:30.413 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.413 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.413 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 5,6,7", 2023-09-23T20:56:30.413 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:33.047852+0000", 2023-09-23T20:56:30.413 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000136124 2023-09-23T20:56:30.413 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.413 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.413 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:56:30.413 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:33.054959+0000", 2023-09-23T20:56:30.413 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0071068900000000003 2023-09-23T20:56:30.413 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.413 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.414 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:56:30.414 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:33.055005+0000", 2023-09-23T20:56:30.414 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.5657e-05 2023-09-23T20:56:30.414 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.414 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.414 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:56:30.414 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:33.055041+0000", 2023-09-23T20:56:30.414 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.5620000000000001e-05 2023-09-23T20:56:30.414 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.414 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.414 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T20:56:30.414 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:33.055333+0000", 2023-09-23T20:56:30.415 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00029220899999999998 2023-09-23T20:56:30.415 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.415 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.415 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T20:56:30.415 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:33.055376+0000", 2023-09-23T20:56:30.415 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.2719999999999998e-05 2023-09-23T20:56:30.415 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.415 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.415 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T20:56:30.415 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:33.055491+0000", 2023-09-23T20:56:30.415 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000115767 2023-09-23T20:56:30.415 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:56:30.415 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T20:56:30.416 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:56:30.416 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.416 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.416 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34797.0:130523 6.7 6:e5f3342f:::benchmark_data_smithi026_223868_object29397:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e136)", 2023-09-23T20:56:30.416 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T20:54:30.945619+0000", 2023-09-23T20:56:30.416 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 119.33099460299999, 2023-09-23T20:56:30.416 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.103502862, 2023-09-23T20:56:30.416 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T20:56:30.416 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:56:30.416 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T20:56:30.416 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34797", 2023-09-23T20:56:30.416 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T20:56:30.416 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 130523 2023-09-23T20:56:30.417 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.417 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T20:56:30.417 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.417 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T20:56:30.417 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:30.945619+0000", 2023-09-23T20:56:30.417 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:56:30.417 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.417 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.417 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T20:56:30.417 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:30.945619+0000", 2023-09-23T20:56:30.418 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:56:30.418 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.418 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.418 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T20:56:30.418 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:30.945618+0000", 2023-09-23T20:56:30.418 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T20:56:30.418 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.418 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.418 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T20:56:30.418 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:30.945623+0000", 2023-09-23T20:56:30.418 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.8640000000000004e-06 2023-09-23T20:56:30.418 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.418 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.419 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T20:56:30.419 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:30.945633+0000", 2023-09-23T20:56:30.419 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0064999999999999e-05 2023-09-23T20:56:30.419 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.419 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.419 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T20:56:30.419 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:30.945641+0000", 2023-09-23T20:56:30.419 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.5599999999999996e-06 2023-09-23T20:56:30.419 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.419 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.419 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:56:30.419 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:30.945689+0000", 2023-09-23T20:56:30.420 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.7846999999999998e-05 2023-09-23T20:56:30.420 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.420 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.420 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T20:56:30.420 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:30.945699+0000", 2023-09-23T20:56:30.420 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0176999999999999e-05 2023-09-23T20:56:30.420 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.420 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.420 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:56:30.420 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:31.884383+0000", 2023-09-23T20:56:30.420 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.93868386199999998 2023-09-23T20:56:30.420 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.420 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.421 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T20:56:30.421 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:31.884394+0000", 2023-09-23T20:56:30.421 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1119e-05 2023-09-23T20:56:30.421 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.421 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.421 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:56:30.421 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:33.034107+0000", 2023-09-23T20:56:30.421 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1497131439999999 2023-09-23T20:56:30.421 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.421 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.421 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T20:56:30.421 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:33.034256+0000", 2023-09-23T20:56:30.421 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00014860700000000001 2023-09-23T20:56:30.421 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.422 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.422 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 1,4,5", 2023-09-23T20:56:30.422 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:33.036344+0000", 2023-09-23T20:56:30.422 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.002088679 2023-09-23T20:56:30.422 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.422 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.422 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:56:30.422 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:33.042876+0000", 2023-09-23T20:56:30.422 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0065314140000000001 2023-09-23T20:56:30.422 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.422 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.422 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:56:30.422 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:33.043075+0000", 2023-09-23T20:56:30.423 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00019934600000000001 2023-09-23T20:56:30.423 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.423 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.423 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:56:30.423 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:33.046043+0000", 2023-09-23T20:56:30.423 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0029680100000000001 2023-09-23T20:56:30.423 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.423 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.423 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T20:56:30.423 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:33.048588+0000", 2023-09-23T20:56:30.423 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0025450569999999999 2023-09-23T20:56:30.423 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.423 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.424 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T20:56:30.424 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:33.048637+0000", 2023-09-23T20:56:30.424 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.8881000000000002e-05 2023-09-23T20:56:30.424 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.424 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.424 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T20:56:30.424 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:33.049122+0000", 2023-09-23T20:56:30.424 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00048494100000000002 2023-09-23T20:56:30.424 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:56:30.424 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T20:56:30.424 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:56:30.424 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.424 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.425 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34797.0:130528 6.c 6:3991ff63:::benchmark_data_smithi026_223868_object29402:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e136)", 2023-09-23T20:56:30.425 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T20:54:30.953130+0000", 2023-09-23T20:56:30.425 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 119.323484071, 2023-09-23T20:56:30.425 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.1058828260000002, 2023-09-23T20:56:30.425 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T20:56:30.425 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:56:30.425 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T20:56:30.425 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34797", 2023-09-23T20:56:30.425 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T20:56:30.425 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 130528 2023-09-23T20:56:30.425 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.425 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T20:56:30.425 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.425 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T20:56:30.426 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:30.953130+0000", 2023-09-23T20:56:30.426 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:56:30.426 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.426 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.426 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T20:56:30.426 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:30.953130+0000", 2023-09-23T20:56:30.426 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:56:30.426 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.426 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.426 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T20:56:30.427 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:30.953129+0000", 2023-09-23T20:56:30.427 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.999999 2023-09-23T20:56:30.427 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.427 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.427 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T20:56:30.427 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:30.953133+0000", 2023-09-23T20:56:30.427 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.5739999999999997e-06 2023-09-23T20:56:30.427 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.427 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.427 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T20:56:30.427 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:30.953142+0000", 2023-09-23T20:56:30.428 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.9309999999999997e-06 2023-09-23T20:56:30.428 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.428 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.428 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T20:56:30.428 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:30.953147+0000", 2023-09-23T20:56:30.428 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.1329999999999998e-06 2023-09-23T20:56:30.428 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.428 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.428 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:56:30.428 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:30.953195+0000", 2023-09-23T20:56:30.428 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.8015e-05 2023-09-23T20:56:30.428 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.428 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.428 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T20:56:30.429 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:30.953202+0000", 2023-09-23T20:56:30.429 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.9480000000000002e-06 2023-09-23T20:56:30.429 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.429 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.429 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:56:30.429 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:31.888161+0000", 2023-09-23T20:56:30.429 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.934958497 2023-09-23T20:56:30.429 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.429 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.429 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T20:56:30.429 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:31.888169+0000", 2023-09-23T20:56:30.429 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.4349999999999997e-06 2023-09-23T20:56:30.429 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.430 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.430 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:56:30.430 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:33.051353+0000", 2023-09-23T20:56:30.430 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.163184022 2023-09-23T20:56:30.430 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.430 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.430 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T20:56:30.430 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:33.051488+0000", 2023-09-23T20:56:30.430 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00013468200000000001 2023-09-23T20:56:30.430 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.430 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.430 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 5,6,7", 2023-09-23T20:56:30.430 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:33.051620+0000", 2023-09-23T20:56:30.431 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000131524 2023-09-23T20:56:30.431 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.431 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.431 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T20:56:30.431 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:33.056424+0000", 2023-09-23T20:56:30.431 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0048044180000000004 2023-09-23T20:56:30.431 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.431 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.431 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:56:30.431 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:33.058674+0000", 2023-09-23T20:56:30.431 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.002250326 2023-09-23T20:56:30.431 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.431 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.431 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:56:30.432 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:33.058712+0000", 2023-09-23T20:56:30.432 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.7469999999999999e-05 2023-09-23T20:56:30.432 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.432 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.432 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:56:30.432 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:33.058880+0000", 2023-09-23T20:56:30.432 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00016785699999999999 2023-09-23T20:56:30.432 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.432 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.432 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T20:56:30.432 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:33.058932+0000", 2023-09-23T20:56:30.432 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.2051e-05 2023-09-23T20:56:30.432 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.433 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.433 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T20:56:30.433 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:33.059012+0000", 2023-09-23T20:56:30.433 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.0708e-05 2023-09-23T20:56:30.433 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:56:30.433 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T20:56:30.433 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:56:30.433 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.433 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.433 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34797.0:151487 6.6 6:657607da:::benchmark_data_smithi026_223868_object50361:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e147)", 2023-09-23T20:56:30.433 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T20:55:01.023782+0000", 2023-09-23T20:56:30.433 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 89.252831981, 2023-09-23T20:56:30.433 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0492650269999999, 2023-09-23T20:56:30.434 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T20:56:30.434 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "reached pg", 2023-09-23T20:56:30.434 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T20:56:30.434 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34797", 2023-09-23T20:56:30.434 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T20:56:30.434 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 151487 2023-09-23T20:56:30.434 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.434 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T20:56:30.434 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.434 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T20:56:30.434 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:01.023782+0000", 2023-09-23T20:56:30.434 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:56:30.435 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.435 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.435 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T20:56:30.435 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:01.023782+0000", 2023-09-23T20:56:30.435 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:56:30.435 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.435 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.435 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T20:56:30.435 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:01.023781+0000", 2023-09-23T20:56:30.435 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T20:56:30.435 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.435 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.435 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T20:56:30.435 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:01.023784+0000", 2023-09-23T20:56:30.436 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.96e-06 2023-09-23T20:56:30.436 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.436 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.436 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T20:56:30.436 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:01.023790+0000", 2023-09-23T20:56:30.436 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.8259999999999998e-06 2023-09-23T20:56:30.436 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.436 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.436 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T20:56:30.436 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:01.023795+0000", 2023-09-23T20:56:30.436 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.7269999999999998e-06 2023-09-23T20:56:30.436 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.436 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.437 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:56:30.437 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:01.024359+0000", 2023-09-23T20:56:30.437 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00056414799999999999 2023-09-23T20:56:30.437 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.437 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.437 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T20:56:30.437 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:01.024503+0000", 2023-09-23T20:56:30.437 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00014404000000000001 2023-09-23T20:56:30.437 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.437 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.437 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 4,7", 2023-09-23T20:56:30.438 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:01.024955+0000", 2023-09-23T20:56:30.438 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000451736 2023-09-23T20:56:30.438 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.438 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.438 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:56:30.438 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:01.029855+0000", 2023-09-23T20:56:30.438 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0049002569999999999 2023-09-23T20:56:30.438 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.438 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.438 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T20:56:30.438 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:01.035683+0000", 2023-09-23T20:56:30.439 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0058282439999999998 2023-09-23T20:56:30.439 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.439 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.439 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:56:30.439 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:01.044471+0000", 2023-09-23T20:56:30.439 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0087875460000000002 2023-09-23T20:56:30.439 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.439 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.439 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T20:56:30.439 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:01.044492+0000", 2023-09-23T20:56:30.439 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.1117000000000001e-05 2023-09-23T20:56:30.439 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.440 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.440 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:56:30.440 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:02.054661+0000", 2023-09-23T20:56:30.440 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.010168873 2023-09-23T20:56:30.440 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.440 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.440 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for missing object", 2023-09-23T20:56:30.440 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:02.054793+0000", 2023-09-23T20:56:30.440 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00013191400000000001 2023-09-23T20:56:30.440 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.440 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.440 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:56:30.440 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:02.072945+0000", 2023-09-23T20:56:30.441 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.01815288 2023-09-23T20:56:30.441 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.441 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.441 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T20:56:30.441 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:02.073047+0000", 2023-09-23T20:56:30.441 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000101245 2023-09-23T20:56:30.441 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:56:30.441 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T20:56:30.441 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:56:30.441 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.441 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.441 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34797.0:151498 6.6 6:62751f79:::benchmark_data_smithi026_223868_object50372:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e148)", 2023-09-23T20:56:30.442 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T20:55:01.044337+0000", 2023-09-23T20:56:30.442 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 89.232276627999994, 2023-09-23T20:56:30.442 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0348560959999999, 2023-09-23T20:56:30.442 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T20:56:30.442 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:56:30.442 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T20:56:30.442 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34797", 2023-09-23T20:56:30.442 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T20:56:30.442 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 151498 2023-09-23T20:56:30.442 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.442 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T20:56:30.442 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.443 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T20:56:30.443 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:01.044337+0000", 2023-09-23T20:56:30.443 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:56:30.443 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.443 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.443 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T20:56:30.443 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:01.044337+0000", 2023-09-23T20:56:30.443 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:56:30.443 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.443 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.443 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T20:56:30.443 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:01.044335+0000", 2023-09-23T20:56:30.443 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999981 2023-09-23T20:56:30.444 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.444 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.444 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T20:56:30.444 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:01.044343+0000", 2023-09-23T20:56:30.444 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.6490000000000008e-06 2023-09-23T20:56:30.444 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.444 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.444 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T20:56:30.444 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:01.045000+0000", 2023-09-23T20:56:30.444 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00065786399999999997 2023-09-23T20:56:30.444 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.444 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.444 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T20:56:30.445 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:01.045010+0000", 2023-09-23T20:56:30.445 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0009e-05 2023-09-23T20:56:30.445 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.445 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.445 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:56:30.445 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:01.046191+0000", 2023-09-23T20:56:30.445 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.001180843 2023-09-23T20:56:30.445 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.445 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.446 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T20:56:30.446 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:01.046209+0000", 2023-09-23T20:56:30.446 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.7980000000000001e-05 2023-09-23T20:56:30.446 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.446 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.446 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:56:30.446 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:02.073061+0000", 2023-09-23T20:56:30.446 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0268516350000001 2023-09-23T20:56:30.446 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.447 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.447 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T20:56:30.447 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:02.073135+0000", 2023-09-23T20:56:30.447 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.4067999999999996e-05 2023-09-23T20:56:30.447 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.447 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.447 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 4,7", 2023-09-23T20:56:30.447 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:02.073539+0000", 2023-09-23T20:56:30.447 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00040385099999999999 2023-09-23T20:56:30.447 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.447 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.447 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:56:30.447 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:02.079023+0000", 2023-09-23T20:56:30.448 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00548461 2023-09-23T20:56:30.448 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.448 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.448 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T20:56:30.448 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:02.079040+0000", 2023-09-23T20:56:30.448 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.6812000000000001e-05 2023-09-23T20:56:30.448 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.448 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.448 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:56:30.448 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:02.079070+0000", 2023-09-23T20:56:30.449 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.9459e-05 2023-09-23T20:56:30.449 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.449 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.449 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T20:56:30.449 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:02.079108+0000", 2023-09-23T20:56:30.449 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.8037999999999997e-05 2023-09-23T20:56:30.449 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.449 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.449 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T20:56:30.449 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:02.079193+0000", 2023-09-23T20:56:30.449 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.5344000000000001e-05 2023-09-23T20:56:30.449 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:56:30.450 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T20:56:30.450 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:56:30.450 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.450 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.450 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34797.0:165117 6.c 6:3504fcb2:::benchmark_data_smithi026_223868_object63991:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e154)", 2023-09-23T20:56:30.450 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T20:55:17.462057+0000", 2023-09-23T20:56:30.450 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 72.814556371999998, 2023-09-23T20:56:30.450 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.091172386, 2023-09-23T20:56:30.450 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T20:56:30.450 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "reached pg", 2023-09-23T20:56:30.450 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T20:56:30.450 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34797", 2023-09-23T20:56:30.451 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T20:56:30.451 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 165117 2023-09-23T20:56:30.451 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.451 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T20:56:30.451 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.451 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T20:56:30.451 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:17.462057+0000", 2023-09-23T20:56:30.451 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:56:30.451 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.451 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.451 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T20:56:30.451 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:17.462057+0000", 2023-09-23T20:56:30.452 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:56:30.452 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.452 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.452 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T20:56:30.452 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:17.462056+0000", 2023-09-23T20:56:30.452 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999981 2023-09-23T20:56:30.452 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.452 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.452 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T20:56:30.452 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:17.462063+0000", 2023-09-23T20:56:30.452 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.7610000000000007e-06 2023-09-23T20:56:30.452 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.452 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.452 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T20:56:30.453 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:17.462077+0000", 2023-09-23T20:56:30.453 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.3570000000000001e-05 2023-09-23T20:56:30.453 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.453 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.453 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T20:56:30.453 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:17.462085+0000", 2023-09-23T20:56:30.453 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.2470000000000008e-06 2023-09-23T20:56:30.453 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.453 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.453 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:56:30.453 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:17.462146+0000", 2023-09-23T20:56:30.453 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.1288e-05 2023-09-23T20:56:30.453 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.454 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.454 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T20:56:30.454 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:17.462301+0000", 2023-09-23T20:56:30.454 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000154971 2023-09-23T20:56:30.454 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.454 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.454 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 5,6,7", 2023-09-23T20:56:30.454 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:17.462508+0000", 2023-09-23T20:56:30.454 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000206691 2023-09-23T20:56:30.454 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.454 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.454 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T20:56:30.455 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:17.467494+0000", 2023-09-23T20:56:30.455 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0049860490000000002 2023-09-23T20:56:30.455 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.455 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.455 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:56:30.455 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:17.473605+0000", 2023-09-23T20:56:30.455 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0061112429999999997 2023-09-23T20:56:30.455 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.455 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.455 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:56:30.455 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:17.486250+0000", 2023-09-23T20:56:30.455 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.012644931999999999 2023-09-23T20:56:30.455 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.455 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.456 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T20:56:30.456 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:17.486261+0000", 2023-09-23T20:56:30.456 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0638999999999999e-05 2023-09-23T20:56:30.456 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.456 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.456 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:56:30.456 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:18.548375+0000", 2023-09-23T20:56:30.456 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0621140440000001 2023-09-23T20:56:30.456 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.456 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.456 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for missing object", 2023-09-23T20:56:30.456 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:18.548503+0000", 2023-09-23T20:56:30.456 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000127668 2023-09-23T20:56:30.457 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.457 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.457 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:56:30.457 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:18.553178+0000", 2023-09-23T20:56:30.457 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.004675289 2023-09-23T20:56:30.457 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.457 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.457 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T20:56:30.457 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:18.553230+0000", 2023-09-23T20:56:30.457 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.1743999999999998e-05 2023-09-23T20:56:30.457 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:56:30.457 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T20:56:30.457 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:56:30.458 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.458 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.458 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34797.0:177908 6.c 6:3bd0ad3d:::benchmark_data_smithi026_223868_object76782:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e164)", 2023-09-23T20:56:30.458 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T20:55:34.934464+0000", 2023-09-23T20:56:30.458 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 55.342150079, 2023-09-23T20:56:30.458 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.0970397529999998, 2023-09-23T20:56:30.458 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T20:56:30.458 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:56:30.458 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T20:56:30.458 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34797", 2023-09-23T20:56:30.459 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T20:56:30.459 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 177908 2023-09-23T20:56:30.459 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.459 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T20:56:30.459 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.459 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T20:56:30.459 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:34.934464+0000", 2023-09-23T20:56:30.459 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:56:30.459 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.459 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.459 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T20:56:30.459 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:34.934464+0000", 2023-09-23T20:56:30.459 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:56:30.460 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.460 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.460 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T20:56:30.460 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:34.934463+0000", 2023-09-23T20:56:30.460 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T20:56:30.460 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.460 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.460 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T20:56:30.460 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:34.934467+0000", 2023-09-23T20:56:30.460 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.4979999999999998e-06 2023-09-23T20:56:30.460 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.460 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.460 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T20:56:30.461 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:34.934487+0000", 2023-09-23T20:56:30.461 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.0602e-05 2023-09-23T20:56:30.461 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.461 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.461 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T20:56:30.461 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:34.934494+0000", 2023-09-23T20:56:30.461 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.4010000000000002e-06 2023-09-23T20:56:30.461 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.461 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.461 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:56:30.461 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:34.950588+0000", 2023-09-23T20:56:30.461 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.016094285 2023-09-23T20:56:30.461 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.461 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.462 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T20:56:30.462 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:34.950600+0000", 2023-09-23T20:56:30.462 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.2224e-05 2023-09-23T20:56:30.462 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.462 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.462 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:56:30.462 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:35.937538+0000", 2023-09-23T20:56:30.462 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.98693769200000003 2023-09-23T20:56:30.462 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.462 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.462 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T20:56:30.462 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:35.937552+0000", 2023-09-23T20:56:30.463 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.3811e-05 2023-09-23T20:56:30.463 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.463 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.463 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:56:30.463 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:37.017091+0000", 2023-09-23T20:56:30.463 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.079539322 2023-09-23T20:56:30.463 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.463 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.463 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T20:56:30.463 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:37.017258+0000", 2023-09-23T20:56:30.463 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00016676500000000001 2023-09-23T20:56:30.463 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.463 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.464 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 5,6", 2023-09-23T20:56:30.464 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:37.017367+0000", 2023-09-23T20:56:30.464 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000108909 2023-09-23T20:56:30.464 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.464 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.464 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T20:56:30.464 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:37.027299+0000", 2023-09-23T20:56:30.464 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0099326350000000004 2023-09-23T20:56:30.464 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.464 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.464 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:56:30.464 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:37.028919+0000", 2023-09-23T20:56:30.464 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.001619756 2023-09-23T20:56:30.464 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.465 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.465 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:56:30.465 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:37.031159+0000", 2023-09-23T20:56:30.465 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0022400330000000002 2023-09-23T20:56:30.465 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.465 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.465 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T20:56:30.465 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:37.031241+0000", 2023-09-23T20:56:30.465 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.2249000000000005e-05 2023-09-23T20:56:30.465 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.465 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.465 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T20:56:30.465 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:37.031503+0000", 2023-09-23T20:56:30.466 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00026212400000000003 2023-09-23T20:56:30.466 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:56:30.466 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T20:56:30.466 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:56:30.466 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.466 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.466 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34797.0:177953 6.c 6:3867b4fa:::benchmark_data_smithi026_223868_object76827:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e164)", 2023-09-23T20:56:30.466 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T20:55:35.005631+0000", 2023-09-23T20:56:30.466 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 55.270982988999997, 2023-09-23T20:56:30.466 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.0261709309999998, 2023-09-23T20:56:30.466 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T20:56:30.466 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:56:30.467 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T20:56:30.467 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34797", 2023-09-23T20:56:30.467 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T20:56:30.467 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 177953 2023-09-23T20:56:30.467 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.467 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T20:56:30.467 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.467 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T20:56:30.467 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:35.005631+0000", 2023-09-23T20:56:30.467 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:56:30.467 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.467 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.467 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T20:56:30.467 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:35.005631+0000", 2023-09-23T20:56:30.468 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:56:30.468 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.468 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.468 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T20:56:30.468 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:35.005629+0000", 2023-09-23T20:56:30.468 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T20:56:30.468 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.468 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.468 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T20:56:30.468 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:35.005635+0000", 2023-09-23T20:56:30.468 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.0809999999999999e-06 2023-09-23T20:56:30.468 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.469 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.469 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T20:56:30.469 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:35.005651+0000", 2023-09-23T20:56:30.469 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.5191000000000001e-05 2023-09-23T20:56:30.469 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.469 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.469 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T20:56:30.469 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:35.005659+0000", 2023-09-23T20:56:30.469 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.6999999999999997e-06 2023-09-23T20:56:30.469 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.470 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.470 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:56:30.470 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:35.005721+0000", 2023-09-23T20:56:30.470 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.1610000000000001e-05 2023-09-23T20:56:30.470 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.470 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.470 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T20:56:30.470 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:35.005734+0000", 2023-09-23T20:56:30.470 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.3137e-05 2023-09-23T20:56:30.471 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.471 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.471 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:56:30.471 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:35.937576+0000", 2023-09-23T20:56:30.471 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.931841582 2023-09-23T20:56:30.471 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.471 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.471 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T20:56:30.471 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:35.937584+0000", 2023-09-23T20:56:30.471 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.123e-06 2023-09-23T20:56:30.471 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.471 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.472 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:56:30.472 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:37.020962+0000", 2023-09-23T20:56:30.472 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.083378282 2023-09-23T20:56:30.472 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.472 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.472 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T20:56:30.472 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:37.021075+0000", 2023-09-23T20:56:30.472 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000112896 2023-09-23T20:56:30.472 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.472 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.472 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 5,6", 2023-09-23T20:56:30.472 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:37.021209+0000", 2023-09-23T20:56:30.472 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00013392100000000001 2023-09-23T20:56:30.473 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.473 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.473 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T20:56:30.473 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:37.027309+0000", 2023-09-23T20:56:30.473 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.006100482 2023-09-23T20:56:30.473 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.473 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.473 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:56:30.473 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:37.028967+0000", 2023-09-23T20:56:30.473 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0016573969999999999 2023-09-23T20:56:30.473 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.473 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.473 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:56:30.474 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:37.031581+0000", 2023-09-23T20:56:30.474 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0026142130000000002 2023-09-23T20:56:30.474 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.474 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.474 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T20:56:30.474 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:37.031645+0000", 2023-09-23T20:56:30.474 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.3997000000000004e-05 2023-09-23T20:56:30.474 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:56:30.474 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:56:30.474 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T20:56:30.474 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:37.031802+0000", 2023-09-23T20:56:30.474 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00015662 2023-09-23T20:56:30.475 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:56:30.475 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T20:56:30.475 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:56:30.475 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:56:30.475 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T20:56:30.475 INFO:teuthology.orchestra.run.smithi026.stdout:} 2023-09-23T20:56:30.476 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:56:30 smithi026 ceph-mon[175151]: 1.6 scrub starts 2023-09-23T20:56:30.476 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:56:30 smithi026 ceph-mon[175151]: 1.6 scrub ok 2023-09-23T20:56:30.476 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:56:30 smithi026 ceph-mon[175151]: 6.13 scrub starts 2023-09-23T20:56:30.476 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:56:30 smithi026 ceph-mon[175151]: 6.13 scrub ok 2023-09-23T20:56:30.476 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:56:30 smithi026 ceph-mon[175151]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 17/543 objects degraded (3.131%), 1 pg degraded) 2023-09-23T20:56:30.476 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:56:30 smithi026 ceph-mon[175151]: Cluster is now healthy 2023-09-23T20:56:30.476 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:56:30 smithi026 ceph-mon[175151]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:56:30.476 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:56:30 smithi026 ceph-mon[175151]: failed to encode map e198 with expected crc 2023-09-23T20:56:30.476 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:56:30 smithi026 ceph-mon[175151]: failed to encode map e198 with expected crc 2023-09-23T20:56:30.476 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:56:30 smithi026 ceph-mon[175151]: failed to encode map e198 with expected crc 2023-09-23T20:56:30.476 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:56:30 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T20:56:30.477 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:56:30 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/712149148' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true}]': finished 2023-09-23T20:56:30.477 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:56:30 smithi026 ceph-mon[175151]: osdmap e198: 8 total, 8 up, 8 in 2023-09-23T20:56:30.477 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:56:30 smithi026 ceph-mon[175151]: 6.14 deep-scrub starts 2023-09-23T20:56:30.477 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:56:30 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:56:30.477 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:56:30 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:56:30.477 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:56:30 smithi026 ceph-mon[179688]: 1.6 scrub starts 2023-09-23T20:56:30.477 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:56:30 smithi026 ceph-mon[179688]: 1.6 scrub ok 2023-09-23T20:56:30.477 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:56:30 smithi026 ceph-mon[179688]: 6.13 scrub starts 2023-09-23T20:56:30.477 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:56:30 smithi026 ceph-mon[179688]: 6.13 scrub ok 2023-09-23T20:56:30.477 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:56:30 smithi026 ceph-mon[179688]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 17/543 objects degraded (3.131%), 1 pg degraded) 2023-09-23T20:56:30.477 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:56:30 smithi026 ceph-mon[179688]: Cluster is now healthy 2023-09-23T20:56:30.478 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:56:30 smithi026 ceph-mon[179688]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:56:30.478 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:56:30 smithi026 ceph-mon[179688]: failed to encode map e198 with expected crc 2023-09-23T20:56:30.478 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:56:30 smithi026 ceph-mon[179688]: failed to encode map e198 with expected crc 2023-09-23T20:56:30.478 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:56:30 smithi026 ceph-mon[179688]: failed to encode map e198 with expected crc 2023-09-23T20:56:30.478 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:56:30 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T20:56:30.478 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:56:30 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/712149148' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true}]': finished 2023-09-23T20:56:30.478 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:56:30 smithi026 ceph-mon[179688]: osdmap e198: 8 total, 8 up, 8 in 2023-09-23T20:56:30.478 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:56:30 smithi026 ceph-mon[179688]: 6.14 deep-scrub starts 2023-09-23T20:56:30.478 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:56:30 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:56:30.478 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:56:30 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:56:30.533 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:56:30.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:56:30 smithi070 ceph-mon[168282]: 1.6 scrub starts 2023-09-23T20:56:30.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:56:30 smithi070 ceph-mon[168282]: 1.6 scrub ok 2023-09-23T20:56:30.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:56:30 smithi070 ceph-mon[168282]: 6.13 scrub starts 2023-09-23T20:56:30.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:56:30 smithi070 ceph-mon[168282]: 6.13 scrub ok 2023-09-23T20:56:30.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:56:30 smithi070 ceph-mon[168282]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 17/543 objects degraded (3.131%), 1 pg degraded) 2023-09-23T20:56:30.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:56:30 smithi070 ceph-mon[168282]: Cluster is now healthy 2023-09-23T20:56:30.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:56:30 smithi070 ceph-mon[168282]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:56:30.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:56:30 smithi070 ceph-mon[168282]: failed to encode map e198 with expected crc 2023-09-23T20:56:30.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:56:30 smithi070 ceph-mon[168282]: failed to encode map e198 with expected crc 2023-09-23T20:56:30.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:56:30 smithi070 ceph-mon[168282]: failed to encode map e198 with expected crc 2023-09-23T20:56:30.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:56:30 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T20:56:30.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:56:30 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/712149148' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true}]': finished 2023-09-23T20:56:30.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:56:30 smithi070 ceph-mon[168282]: osdmap e198: 8 total, 8 up, 8 in 2023-09-23T20:56:30.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:56:30 smithi070 ceph-mon[168282]: 6.14 deep-scrub starts 2023-09-23T20:56:30.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:56:30 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:56:30.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:56:30 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:56:30.795 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_ops_in_flight 2023-09-23T20:56:31.122 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:56:31.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:56:31 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:56:31.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:56:31 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:56:31.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:56:31 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:56:31.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:56:31 smithi026 ceph-mon[175151]: pgmap v459: 83 pgs: 83 active+clean; 425 KiB data, 1.1 GiB used, 714 GiB / 715 GiB avail 2023-09-23T20:56:31.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:56:31 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:56:31.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:56:31 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:56:31.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:56:31 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:56:31.383 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:56:31 smithi026 ceph-mon[179688]: pgmap v459: 83 pgs: 83 active+clean; 425 KiB data, 1.1 GiB used, 714 GiB / 715 GiB avail 2023-09-23T20:56:31.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:56:31 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:56:31.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:56:31 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:56:31.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:56:31 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:56:31.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:56:31 smithi070 ceph-mon[168282]: pgmap v459: 83 pgs: 83 active+clean; 425 KiB data, 1.1 GiB used, 714 GiB / 715 GiB avail 2023-09-23T20:56:31.967 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T20:56:32.147 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:56:32.151 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:56:32.151 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:56:32.177 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:56:32 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:56:32.177 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:56:32 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:56:32.252 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:56:32.288 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:56:32.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:56:32 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:56:32.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:56:32 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:56:32.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:56:32 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:56:32.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:56:32 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:56:33.071 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:56:33.074 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:56:33.075 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:56:33.175 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:56:33.205 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:56:33.293 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:56:33 smithi026 ceph-mon[175151]: pgmap v460: 83 pgs: 83 active+clean; 372 KiB data, 1.1 GiB used, 714 GiB / 715 GiB avail 2023-09-23T20:56:33.293 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:56:33 smithi026 ceph-mon[179688]: pgmap v460: 83 pgs: 83 active+clean; 372 KiB data, 1.1 GiB used, 714 GiB / 715 GiB avail 2023-09-23T20:56:33.300 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0: {} 2023-09-23T20:56:33.321 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1: {} 2023-09-23T20:56:33.341 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2: {} 2023-09-23T20:56:33.363 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3: {} 2023-09-23T20:56:33.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:56:33 smithi070 ceph-mon[168282]: pgmap v460: 83 pgs: 83 active+clean; 372 KiB data, 1.1 GiB used, 714 GiB / 715 GiB avail 2023-09-23T20:56:33.384 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4: {} 2023-09-23T20:56:33.404 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T20:56:33.423 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6: {} 2023-09-23T20:56:33.442 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T20:56:33.443 INFO:teuthology.orchestra.run.smithi026.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T20:56:33.443 INFO:teuthology.orchestra.run.smithi026.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T20:56:33.443 INFO:teuthology.orchestra.run.smithi026.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T20:56:33.444 INFO:teuthology.orchestra.run.smithi026.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T20:56:33.444 INFO:teuthology.orchestra.run.smithi026.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T20:56:33.444 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T20:56:33.444 INFO:teuthology.orchestra.run.smithi026.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T20:56:33.444 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T20:56:33.552 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:56:34.097 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:56:34.100 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:56:34.101 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:56:34.101 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:56:34.201 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:56:34.230 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:56:34.449 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:56:34.453 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:56:34.454 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:56:34.542 INFO:teuthology.orchestra.run.smithi026.stderr:pool 'unique_pool_1' created 2023-09-23T20:56:34.555 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:56:34.585 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:56:34.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:56:34 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/310464554' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:56:34.881 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:56:34 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/310464554' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:56:34.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:56:34 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/310464554' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:56:34.887 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph osd pool application enable unique_pool_1 rados --yes-i-really-mean-it || true 2023-09-23T20:56:35.022 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:56:35.026 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:56:35.027 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:56:35.127 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:56:35.157 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:56:35.416 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:56:35.419 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:56:35.419 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:56:35.492 INFO:teuthology.orchestra.run.smithi070.stdout:{ 2023-09-23T20:56:35.492 INFO:teuthology.orchestra.run.smithi070.stdout: "success": "filestore_inject_stall = '3' (not observed, change may require restart) " 2023-09-23T20:56:35.492 INFO:teuthology.orchestra.run.smithi070.stdout:} 2023-09-23T20:56:35.520 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:56:35.550 INFO:teuthology.orchestra.run.smithi026.stderr:nodeep-scrub is set 2023-09-23T20:56:35.559 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:56:35.567 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:56:35.595 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:56:35.600 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:56:35.600 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:56:35.701 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:56:35.727 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:56:35 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/959488176' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_1", "pg_num": 16}]: dispatch 2023-09-23T20:56:35.727 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:56:35 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_1", "pg_num": 16}]: dispatch 2023-09-23T20:56:35.727 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:56:35 smithi070 ceph-mon[168282]: pgmap v461: 83 pgs: 83 active+clean; 372 KiB data, 1.1 GiB used, 714 GiB / 715 GiB avail 2023-09-23T20:56:35.727 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:56:35 smithi070 ceph-mon[168282]: failed to encode map e199 with expected crc 2023-09-23T20:56:35.727 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:56:35 smithi070 ceph-mon[168282]: failed to encode map e199 with expected crc 2023-09-23T20:56:35.727 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:56:35 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_1", "pg_num": 16}]': finished 2023-09-23T20:56:35.728 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:56:35 smithi070 ceph-mon[168282]: osdmap e199: 8 total, 8 up, 8 in 2023-09-23T20:56:35.728 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:56:35 smithi070 ceph-mon[168282]: failed to encode map e199 with expected crc 2023-09-23T20:56:35.728 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:56:35 smithi070 ceph-mon[168282]: failed to encode map e199 with expected crc 2023-09-23T20:56:35.728 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:56:35 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/3321023545' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:56:35.728 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:56:35 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:56:35.730 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:56:35.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:56:35 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/959488176' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_1", "pg_num": 16}]: dispatch 2023-09-23T20:56:35.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:56:35 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_1", "pg_num": 16}]: dispatch 2023-09-23T20:56:35.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:56:35 smithi026 ceph-mon[175151]: pgmap v461: 83 pgs: 83 active+clean; 372 KiB data, 1.1 GiB used, 714 GiB / 715 GiB avail 2023-09-23T20:56:35.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:56:35 smithi026 ceph-mon[175151]: failed to encode map e199 with expected crc 2023-09-23T20:56:35.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:56:35 smithi026 ceph-mon[175151]: failed to encode map e199 with expected crc 2023-09-23T20:56:35.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:56:35 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_1", "pg_num": 16}]': finished 2023-09-23T20:56:35.883 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:56:35 smithi026 ceph-mon[175151]: osdmap e199: 8 total, 8 up, 8 in 2023-09-23T20:56:35.883 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:56:35 smithi026 ceph-mon[175151]: failed to encode map e199 with expected crc 2023-09-23T20:56:35.883 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:56:35 smithi026 ceph-mon[175151]: failed to encode map e199 with expected crc 2023-09-23T20:56:35.883 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:56:35 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/3321023545' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:56:35.883 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:56:35 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:56:35.883 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:56:35 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/959488176' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_1", "pg_num": 16}]: dispatch 2023-09-23T20:56:35.883 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:56:35 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_1", "pg_num": 16}]: dispatch 2023-09-23T20:56:35.883 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:56:35 smithi026 ceph-mon[179688]: pgmap v461: 83 pgs: 83 active+clean; 372 KiB data, 1.1 GiB used, 714 GiB / 715 GiB avail 2023-09-23T20:56:35.883 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:56:35 smithi026 ceph-mon[179688]: failed to encode map e199 with expected crc 2023-09-23T20:56:35.883 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:56:35 smithi026 ceph-mon[179688]: failed to encode map e199 with expected crc 2023-09-23T20:56:35.884 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:56:35 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_1", "pg_num": 16}]': finished 2023-09-23T20:56:35.884 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:56:35 smithi026 ceph-mon[179688]: osdmap e199: 8 total, 8 up, 8 in 2023-09-23T20:56:35.884 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:56:35 smithi026 ceph-mon[179688]: failed to encode map e199 with expected crc 2023-09-23T20:56:35.884 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:56:35 smithi026 ceph-mon[179688]: failed to encode map e199 with expected crc 2023-09-23T20:56:35.884 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:56:35 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/3321023545' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:56:35.884 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:56:35 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:56:36.065 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:56:36.067 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:56:36.068 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:56:36.164 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T20:56:36.168 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:56:36.196 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:56:36.278 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:56:36.280 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:56:36.281 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:56:36.381 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:56:36.411 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:56:36.527 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:56:36 smithi070 ceph-mon[168282]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:56:36.527 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:56:36 smithi070 ceph-mon[168282]: failed to encode map e200 with expected crc 2023-09-23T20:56:36.527 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:56:36 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T20:56:36.527 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:56:36 smithi070 ceph-mon[168282]: osdmap e200: 8 total, 8 up, 8 in 2023-09-23T20:56:36.527 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:56:36 smithi070 ceph-mon[168282]: failed to encode map e200 with expected crc 2023-09-23T20:56:36.527 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:56:36 smithi070 ceph-mon[168282]: failed to encode map e200 with expected crc 2023-09-23T20:56:36.528 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:56:36 smithi070 ceph-mon[168282]: failed to encode map e200 with expected crc 2023-09-23T20:56:36.528 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:56:36 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:56:36.566 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:56:36.569 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:56:36 smithi026 ceph-mon[179688]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:56:36.569 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:56:36 smithi026 ceph-mon[179688]: failed to encode map e200 with expected crc 2023-09-23T20:56:36.569 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:56:36 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T20:56:36.569 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:56:36 smithi026 ceph-mon[179688]: osdmap e200: 8 total, 8 up, 8 in 2023-09-23T20:56:36.569 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:56:36 smithi026 ceph-mon[179688]: failed to encode map e200 with expected crc 2023-09-23T20:56:36.569 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:56:36 smithi026 ceph-mon[179688]: failed to encode map e200 with expected crc 2023-09-23T20:56:36.569 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:56:36 smithi026 ceph-mon[179688]: failed to encode map e200 with expected crc 2023-09-23T20:56:36.569 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:56:36 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:56:36.570 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:56:36 smithi026 ceph-mon[175151]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:56:36.570 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:56:36 smithi026 ceph-mon[175151]: failed to encode map e200 with expected crc 2023-09-23T20:56:36.570 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:56:36 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T20:56:36.570 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:56:36 smithi026 ceph-mon[175151]: osdmap e200: 8 total, 8 up, 8 in 2023-09-23T20:56:36.570 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:56:36 smithi026 ceph-mon[175151]: failed to encode map e200 with expected crc 2023-09-23T20:56:36.570 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:56:36 smithi026 ceph-mon[175151]: failed to encode map e200 with expected crc 2023-09-23T20:56:36.570 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:56:36 smithi026 ceph-mon[175151]: failed to encode map e200 with expected crc 2023-09-23T20:56:36.570 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:56:36 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:56:36.571 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:56:36.571 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:56:36.672 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:56:36.701 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:56:36.718 INFO:teuthology.orchestra.run.smithi070.stdout:ERROR: (22) Invalid argument 2023-09-23T20:56:36.718 INFO:teuthology.orchestra.run.smithi070.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:56:36.739 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:56:36.742 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:56:36.742 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:56:36.843 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:56:36.870 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:56:37.053 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_blocked_ops 2023-09-23T20:56:37.083 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:56:37.086 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:56:37.086 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:56:37.087 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:56:37.187 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:56:37.213 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:56:37.246 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:56:37.249 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:56:37.249 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:56:37.350 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:56:37.380 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:56:37.414 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:56:37.418 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:56:37.419 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:56:37.519 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:56:37.547 INFO:teuthology.orchestra.run.smithi026.stderr:enabled application 'rados' on pool 'unique_pool_1' 2023-09-23T20:56:37.549 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:56:37.575 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:56:37 smithi026 ceph-mon[175151]: pgmap v464: 99 pgs: 12 unknown, 87 active+clean; 372 KiB data, 1.1 GiB used, 714 GiB / 715 GiB avail 2023-09-23T20:56:37.575 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:56:37 smithi026 ceph-mon[175151]: failed to encode map e201 with expected crc 2023-09-23T20:56:37.575 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:56:37 smithi026 ceph-mon[175151]: failed to encode map e201 with expected crc 2023-09-23T20:56:37.575 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:56:37 smithi026 ceph-mon[175151]: osdmap e201: 8 total, 8 up, 8 in 2023-09-23T20:56:37.575 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:56:37 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/3968727277' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_1", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T20:56:37.575 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:56:37 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_1", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T20:56:37.575 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:56:37 smithi026 ceph-mon[179688]: pgmap v464: 99 pgs: 12 unknown, 87 active+clean; 372 KiB data, 1.1 GiB used, 714 GiB / 715 GiB avail 2023-09-23T20:56:37.576 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:56:37 smithi026 ceph-mon[179688]: failed to encode map e201 with expected crc 2023-09-23T20:56:37.576 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:56:37 smithi026 ceph-mon[179688]: failed to encode map e201 with expected crc 2023-09-23T20:56:37.576 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:56:37 smithi026 ceph-mon[179688]: osdmap e201: 8 total, 8 up, 8 in 2023-09-23T20:56:37.576 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:56:37 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/3968727277' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_1", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T20:56:37.576 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:56:37 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_1", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T20:56:37.633 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:56:37.636 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:56:37.636 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:56:37.683 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:56:37.737 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:56:37.766 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:56:37.802 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:56:37.804 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:56:37.805 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:56:37.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:56:37 smithi070 ceph-mon[168282]: pgmap v464: 99 pgs: 12 unknown, 87 active+clean; 372 KiB data, 1.1 GiB used, 714 GiB / 715 GiB avail 2023-09-23T20:56:37.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:56:37 smithi070 ceph-mon[168282]: failed to encode map e201 with expected crc 2023-09-23T20:56:37.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:56:37 smithi070 ceph-mon[168282]: failed to encode map e201 with expected crc 2023-09-23T20:56:37.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:56:37 smithi070 ceph-mon[168282]: osdmap e201: 8 total, 8 up, 8 in 2023-09-23T20:56:37.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:56:37 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/3968727277' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_1", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T20:56:37.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:56:37 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_1", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T20:56:37.906 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:56:37.933 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:56:38.004 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T20:56:38.019 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:56:38.022 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:56:38.022 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:56:38.125 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:56:38.152 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:56:38.462 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:56:38.465 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:56:38.465 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:56:38.566 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:56:38.598 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:56:38.688 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:56:38 smithi070 ceph-mon[168282]: failed to encode map e202 with expected crc 2023-09-23T20:56:38.688 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:56:38 smithi070 ceph-mon[168282]: failed to encode map e202 with expected crc 2023-09-23T20:56:38.688 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:56:38 smithi070 ceph-mon[168282]: failed to encode map e202 with expected crc 2023-09-23T20:56:38.688 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:56:38 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_1", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-09-23T20:56:38.688 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:56:38 smithi070 ceph-mon[168282]: osdmap e202: 8 total, 8 up, 8 in 2023-09-23T20:56:38.837 DEBUG:teuthology.orchestra.run.smithi026:> /bin/sh -c 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados --no-log-to-stderr --name client.0 -b 65536 --object-size 65536 -p unique_pool_1 bench 90 write' 2023-09-23T20:56:38.840 INFO:tasks.radosbench:joining radosbench (timing out after 3000s) 2023-09-23T20:56:38.840 INFO:teuthology.orchestra.run:waiting for 3000 2023-09-23T20:56:38.843 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:56:38.846 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:56:38 smithi026 ceph-mon[179688]: failed to encode map e202 with expected crc 2023-09-23T20:56:38.846 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:56:38 smithi026 ceph-mon[179688]: failed to encode map e202 with expected crc 2023-09-23T20:56:38.846 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:56:38 smithi026 ceph-mon[179688]: failed to encode map e202 with expected crc 2023-09-23T20:56:38.846 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:56:38 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_1", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-09-23T20:56:38.846 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:56:38 smithi026 ceph-mon[179688]: osdmap e202: 8 total, 8 up, 8 in 2023-09-23T20:56:38.847 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:56:38 smithi026 ceph-mon[175151]: failed to encode map e202 with expected crc 2023-09-23T20:56:38.847 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:56:38 smithi026 ceph-mon[175151]: failed to encode map e202 with expected crc 2023-09-23T20:56:38.847 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:56:38 smithi026 ceph-mon[175151]: failed to encode map e202 with expected crc 2023-09-23T20:56:38.847 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:56:38 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_1", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-09-23T20:56:38.847 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:56:38 smithi026 ceph-mon[175151]: osdmap e202: 8 total, 8 up, 8 in 2023-09-23T20:56:38.847 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:56:38.848 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:56:38.893 INFO:tasks.radosbench.radosbench.0.smithi026.stdout:hints = 1 2023-09-23T20:56:38.893 INFO:tasks.radosbench.radosbench.0.smithi026.stdout:Maintaining 16 concurrent writes of 65536 bytes to objects of size 65536 for up to 90 seconds or 0 objects 2023-09-23T20:56:38.893 INFO:tasks.radosbench.radosbench.0.smithi026.stdout:Object prefix: benchmark_data_smithi026_260489 2023-09-23T20:56:38.893 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T20:56:38.894 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 0 0 0 0 0 0 - 0 2023-09-23T20:56:38.905 INFO:teuthology.orchestra.run.smithi070.stdout:ERROR: (22) Invalid argument 2023-09-23T20:56:38.905 INFO:teuthology.orchestra.run.smithi070.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:56:38.948 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:56:39.021 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:56:39.275 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:56:39.280 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:56:39.281 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:56:39.350 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_historic_ops 2023-09-23T20:56:39.381 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:56:39.441 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:56:39.501 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:56:39.507 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:56:39.508 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:56:39.608 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:56:39.668 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:56:39.781 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:56:39.785 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:56:39.785 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:56:39.847 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:56:39 smithi026 ceph-mon[175151]: pgmap v467: 99 pgs: 7 unknown, 92 active+clean; 372 KiB data, 1.1 GiB used, 714 GiB / 715 GiB avail 2023-09-23T20:56:39.847 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:56:39 smithi026 ceph-mon[179688]: pgmap v467: 99 pgs: 7 unknown, 92 active+clean; 372 KiB data, 1.1 GiB used, 714 GiB / 715 GiB avail 2023-09-23T20:56:39.849 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:56:39 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[164029]: ::ffff:172.21.15.70 - - [23/Sep/2023:20:56:39] "GET /metrics HTTP/1.1" 200 33968 "" "Prometheus/2.43.0" 2023-09-23T20:56:39.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:56:39 smithi070 ceph-mon[168282]: pgmap v467: 99 pgs: 7 unknown, 92 active+clean; 372 KiB data, 1.1 GiB used, 714 GiB / 715 GiB avail 2023-09-23T20:56:39.886 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:56:39.895 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 1 16 3451 3435 214.303 214.688 0.00363012 0.00464142 2023-09-23T20:56:39.943 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:56:40.003 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:56:40.009 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:56:40.010 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:56:40.107 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:56:40.111 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:56:40.172 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:56:40.257 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:56:40.261 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:56:40.262 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:56:40.340 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0: {} 2023-09-23T20:56:40.363 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:56:40.379 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1: {} 2023-09-23T20:56:40.379 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:56:40.405 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:56:40.418 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2: {} 2023-09-23T20:56:40.448 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:56:40.454 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:56:40.454 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:56:40.462 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3: {} 2023-09-23T20:56:40.512 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4: {} 2023-09-23T20:56:40.555 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:56:40.559 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T20:56:40.601 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6: {} 2023-09-23T20:56:40.610 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:56:40.645 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T20:56:40.650 INFO:teuthology.orchestra.run.smithi026.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T20:56:40.650 INFO:teuthology.orchestra.run.smithi026.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T20:56:40.651 INFO:teuthology.orchestra.run.smithi026.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T20:56:40.651 INFO:teuthology.orchestra.run.smithi026.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T20:56:40.651 INFO:teuthology.orchestra.run.smithi026.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T20:56:40.651 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T20:56:40.651 INFO:teuthology.orchestra.run.smithi026.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T20:56:40.651 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T20:56:40.675 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:56:40.685 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:56:40.685 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:56:40.786 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:56:40.848 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:56:40.895 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 2 16 6043 6027 188.166 162 0.00265132 0.00528638 2023-09-23T20:56:40.909 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:56:40.913 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:56:40.914 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:56:41.000 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 6, 7, 4, 5] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 7, 6] 2023-09-23T20:56:41.000 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T20:56:41.001 INFO:tasks.thrashosds.thrasher:primary_affinity 2023-09-23T20:56:41.001 INFO:tasks.thrashosds.thrasher:Setting osd 7 primary_affinity to 0.043541 2023-09-23T20:56:41.001 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph osd primary-affinity 7 0.043541329996032774 2023-09-23T20:56:41.015 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:56:41.065 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:56:41.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:56:40 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/935885806' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:56:41.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:56:40 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/935885806' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:56:41.190 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:56:41.193 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:56:41.193 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:56:41.284 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:56:40 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/935885806' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:56:41.294 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:56:41.342 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:56:41.384 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:56:41.387 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:56:41.388 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:56:41.489 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:56:41.557 INFO:teuthology.orchestra.run.smithi070.stdout:{ 2023-09-23T20:56:41.557 INFO:teuthology.orchestra.run.smithi070.stdout: "size": 20, 2023-09-23T20:56:41.557 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 600, 2023-09-23T20:56:41.557 INFO:teuthology.orchestra.run.smithi070.stdout: "ops": [ 2023-09-23T20:56:41.557 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:41.557 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.34797.0:5568 6.2 6:43edb851:::benchmark_data_smithi026_223868_object5567:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e106)", 2023-09-23T20:56:41.558 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T20:52:21.786587+0000", 2023-09-23T20:56:41.558 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 259.76694408600002, 2023-09-23T20:56:41.558 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.086354670999999994, 2023-09-23T20:56:41.558 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T20:56:41.558 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:56:41.558 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T20:56:41.558 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.34797", 2023-09-23T20:56:41.558 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T20:56:41.558 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 5568 2023-09-23T20:56:41.558 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:41.558 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T20:56:41.558 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:41.558 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T20:56:41.559 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:52:21.786587+0000", 2023-09-23T20:56:41.559 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T20:56:41.559 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:41.559 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:41.559 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T20:56:41.559 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:52:21.786587+0000", 2023-09-23T20:56:41.559 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T20:56:41.559 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:41.559 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:41.559 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T20:56:41.559 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:52:21.786585+0000", 2023-09-23T20:56:41.559 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.999999 2023-09-23T20:56:41.560 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:41.560 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:41.560 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T20:56:41.560 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:52:21.786626+0000", 2023-09-23T20:56:41.560 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.0077999999999998e-05 2023-09-23T20:56:41.560 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:41.560 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:41.560 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T20:56:41.560 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:52:21.786637+0000", 2023-09-23T20:56:41.560 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.1236000000000001e-05 2023-09-23T20:56:41.560 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:41.560 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:41.561 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T20:56:41.561 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:52:21.786641+0000", 2023-09-23T20:56:41.561 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.4530000000000004e-06 2023-09-23T20:56:41.561 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:41.561 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:41.561 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T20:56:41.561 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:52:21.786687+0000", 2023-09-23T20:56:41.561 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.6109000000000002e-05 2023-09-23T20:56:41.561 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:41.561 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:41.561 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T20:56:41.561 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:52:21.786795+0000", 2023-09-23T20:56:41.561 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00010764099999999999 2023-09-23T20:56:41.561 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:41.562 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:41.562 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 2,3", 2023-09-23T20:56:41.562 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:52:21.871252+0000", 2023-09-23T20:56:41.562 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.084457312000000007 2023-09-23T20:56:41.562 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:41.562 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:41.562 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T20:56:41.562 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:52:21.872733+0000", 2023-09-23T20:56:41.562 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.001480695 2023-09-23T20:56:41.562 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:41.563 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:41.563 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:56:41.563 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:52:21.872822+0000", 2023-09-23T20:56:41.563 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.8809999999999998e-05 2023-09-23T20:56:41.563 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:41.563 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:41.563 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:56:41.563 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:52:21.872857+0000", 2023-09-23T20:56:41.563 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.5611000000000002e-05 2023-09-23T20:56:41.563 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:41.563 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:41.563 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T20:56:41.564 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:52:21.872903+0000", 2023-09-23T20:56:41.564 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.5658000000000002e-05 2023-09-23T20:56:41.564 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:41.564 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:41.564 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T20:56:41.564 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:52:21.872941+0000", 2023-09-23T20:56:41.564 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.8114000000000001e-05 2023-09-23T20:56:41.564 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T20:56:41.564 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T20:56:41.564 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T20:56:41.564 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:41.564 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:41.564 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.34797.0:5569 6.2 6:4aaa764c:::benchmark_data_smithi026_223868_object5568:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e106)", 2023-09-23T20:56:41.565 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T20:52:21.786652+0000", 2023-09-23T20:56:41.565 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 259.76687862599999, 2023-09-23T20:56:41.565 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.087665349000000004, 2023-09-23T20:56:41.565 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T20:56:41.565 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:56:41.565 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T20:56:41.565 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.34797", 2023-09-23T20:56:41.565 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T20:56:41.565 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 5569 2023-09-23T20:56:41.565 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:41.565 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T20:56:41.565 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:41.566 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T20:56:41.566 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:52:21.786652+0000", 2023-09-23T20:56:41.566 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T20:56:41.566 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:41.566 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:41.566 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T20:56:41.566 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:52:21.786652+0000", 2023-09-23T20:56:41.566 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T20:56:41.566 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:41.566 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:41.566 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T20:56:41.566 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:52:21.786652+0000", 2023-09-23T20:56:41.567 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999995 2023-09-23T20:56:41.567 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:41.567 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:41.567 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T20:56:41.567 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:52:21.787270+0000", 2023-09-23T20:56:41.567 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00061800199999999998 2023-09-23T20:56:41.567 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:41.567 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:41.567 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T20:56:41.567 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:52:21.871139+0000", 2023-09-23T20:56:41.567 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.083869155000000001 2023-09-23T20:56:41.567 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:41.567 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:41.567 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T20:56:41.568 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:52:21.871150+0000", 2023-09-23T20:56:41.568 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.0723000000000001e-05 2023-09-23T20:56:41.568 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:41.568 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:41.568 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T20:56:41.568 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:52:21.871619+0000", 2023-09-23T20:56:41.568 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00046919699999999997 2023-09-23T20:56:41.568 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:41.568 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:41.568 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T20:56:41.568 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:52:21.871737+0000", 2023-09-23T20:56:41.568 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00011873499999999999 2023-09-23T20:56:41.568 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:41.569 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:41.569 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 2,3", 2023-09-23T20:56:41.569 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:52:21.871867+0000", 2023-09-23T20:56:41.569 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000129061 2023-09-23T20:56:41.569 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:41.569 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:41.569 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T20:56:41.569 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:52:21.872737+0000", 2023-09-23T20:56:41.569 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00087040900000000005 2023-09-23T20:56:41.569 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:41.569 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:41.569 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:56:41.569 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:52:21.872987+0000", 2023-09-23T20:56:41.570 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00024983299999999998 2023-09-23T20:56:41.570 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:41.570 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:41.570 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:56:41.570 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:52:21.874231+0000", 2023-09-23T20:56:41.570 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0012444800000000001 2023-09-23T20:56:41.570 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:41.570 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:41.570 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T20:56:41.570 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:52:21.874280+0000", 2023-09-23T20:56:41.570 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.8838999999999998e-05 2023-09-23T20:56:41.570 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:41.571 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:41.571 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T20:56:41.571 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:52:21.874317+0000", 2023-09-23T20:56:41.571 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.7255999999999997e-05 2023-09-23T20:56:41.571 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T20:56:41.571 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T20:56:41.571 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T20:56:41.571 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:41.571 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:41.571 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.34797.0:114997 6.12 6.28c54f92 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e132)", 2023-09-23T20:56:41.571 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T20:54:12.985231+0000", 2023-09-23T20:56:41.571 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 148.56829987200001, 2023-09-23T20:56:41.571 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.939145275, 2023-09-23T20:56:41.572 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T20:56:41.572 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "reached pg", 2023-09-23T20:56:41.572 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T20:56:41.572 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.34797", 2023-09-23T20:56:41.572 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T20:56:41.572 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 114997 2023-09-23T20:56:41.572 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:41.572 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T20:56:41.572 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:41.572 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T20:56:41.572 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:54:12.985231+0000", 2023-09-23T20:56:41.572 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T20:56:41.572 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:41.573 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:41.573 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T20:56:41.573 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:54:12.985231+0000", 2023-09-23T20:56:41.573 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T20:56:41.573 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:41.573 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:41.573 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T20:56:41.573 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:54:12.985228+0000", 2023-09-23T20:56:41.573 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999976 2023-09-23T20:56:41.573 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:41.573 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:41.573 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T20:56:41.574 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:54:12.985282+0000", 2023-09-23T20:56:41.574 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.3285000000000002e-05 2023-09-23T20:56:41.574 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:41.574 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:41.574 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T20:56:41.574 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:54:12.985298+0000", 2023-09-23T20:56:41.574 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.6677e-05 2023-09-23T20:56:41.574 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:41.574 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:41.574 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T20:56:41.574 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:54:12.985306+0000", 2023-09-23T20:56:41.574 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.8590000000000005e-06 2023-09-23T20:56:41.575 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:41.575 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:41.575 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T20:56:41.575 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:54:12.985354+0000", 2023-09-23T20:56:41.575 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.8124000000000002e-05 2023-09-23T20:56:41.575 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:41.575 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:41.575 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T20:56:41.575 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:54:12.985364+0000", 2023-09-23T20:56:41.575 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.6500000000000008e-06 2023-09-23T20:56:41.575 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:41.575 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:41.575 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T20:56:41.576 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:54:13.924362+0000", 2023-09-23T20:56:41.576 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.93899794599999997 2023-09-23T20:56:41.576 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:41.576 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:41.576 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T20:56:41.576 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:54:13.924376+0000", 2023-09-23T20:56:41.576 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.4238e-05 2023-09-23T20:56:41.576 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T20:56:41.576 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T20:56:41.576 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T20:56:41.576 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:41.576 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:41.577 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.34797.0:156782 6.12 6:4ab9e603:::benchmark_data_smithi026_223868_object55656:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e150)", 2023-09-23T20:56:41.577 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T20:55:07.352049+0000", 2023-09-23T20:56:41.577 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 94.201482010999996, 2023-09-23T20:56:41.577 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.014989538, 2023-09-23T20:56:41.577 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T20:56:41.577 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:56:41.577 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T20:56:41.577 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.34797", 2023-09-23T20:56:41.577 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T20:56:41.577 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 156782 2023-09-23T20:56:41.577 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:41.577 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T20:56:41.577 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:41.578 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T20:56:41.578 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:07.352049+0000", 2023-09-23T20:56:41.578 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T20:56:41.578 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:41.578 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:41.578 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T20:56:41.578 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:07.352049+0000", 2023-09-23T20:56:41.578 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T20:56:41.578 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:41.578 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:41.578 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T20:56:41.578 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:07.352047+0000", 2023-09-23T20:56:41.578 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T20:56:41.579 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:41.579 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:41.579 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T20:56:41.579 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:07.352052+0000", 2023-09-23T20:56:41.579 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.9280000000000001e-06 2023-09-23T20:56:41.579 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:41.579 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:41.579 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T20:56:41.579 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:07.352064+0000", 2023-09-23T20:56:41.579 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.1902999999999999e-05 2023-09-23T20:56:41.579 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:41.579 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:41.580 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T20:56:41.580 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:07.352070+0000", 2023-09-23T20:56:41.580 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.8459999999999996e-06 2023-09-23T20:56:41.580 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:41.580 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:41.580 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T20:56:41.580 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:07.352358+0000", 2023-09-23T20:56:41.580 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00028820300000000001 2023-09-23T20:56:41.580 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:41.580 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:41.580 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T20:56:41.580 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:07.352372+0000", 2023-09-23T20:56:41.580 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.3669000000000001e-05 2023-09-23T20:56:41.581 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:41.581 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:41.581 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T20:56:41.581 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.361399+0000", 2023-09-23T20:56:41.581 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.0090268060000001 2023-09-23T20:56:41.581 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:41.581 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:41.581 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T20:56:41.581 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.361509+0000", 2023-09-23T20:56:41.581 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000110451 2023-09-23T20:56:41.581 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:41.581 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:41.581 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 2,3", 2023-09-23T20:56:41.582 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.361803+0000", 2023-09-23T20:56:41.582 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00029413900000000001 2023-09-23T20:56:41.582 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:41.582 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:41.582 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T20:56:41.582 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.363432+0000", 2023-09-23T20:56:41.582 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.001628188 2023-09-23T20:56:41.582 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:41.582 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:41.582 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:56:41.582 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.366824+0000", 2023-09-23T20:56:41.582 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0033921910000000001 2023-09-23T20:56:41.583 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:41.583 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:41.583 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:56:41.583 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.366953+0000", 2023-09-23T20:56:41.583 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00012964399999999999 2023-09-23T20:56:41.583 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:41.583 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:41.583 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T20:56:41.583 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.367000+0000", 2023-09-23T20:56:41.583 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.6319999999999997e-05 2023-09-23T20:56:41.583 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:41.583 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:41.583 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T20:56:41.584 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.367038+0000", 2023-09-23T20:56:41.584 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.8498000000000003e-05 2023-09-23T20:56:41.584 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T20:56:41.584 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T20:56:41.584 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T20:56:41.584 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:41.584 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:41.584 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.34797.0:156812 6.12 6:4c46565a:::benchmark_data_smithi026_223868_object55686:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e150)", 2023-09-23T20:56:41.584 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T20:55:07.386674+0000", 2023-09-23T20:56:41.584 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 94.166856852999999, 2023-09-23T20:56:41.584 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.98619251500000005, 2023-09-23T20:56:41.584 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T20:56:41.585 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:56:41.585 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T20:56:41.585 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.34797", 2023-09-23T20:56:41.585 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T20:56:41.585 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 156812 2023-09-23T20:56:41.585 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:41.585 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T20:56:41.585 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:41.585 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T20:56:41.585 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:07.386674+0000", 2023-09-23T20:56:41.585 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T20:56:41.585 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:41.586 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:41.586 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T20:56:41.586 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:07.386674+0000", 2023-09-23T20:56:41.586 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T20:56:41.586 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:41.586 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:41.586 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T20:56:41.586 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:07.386672+0000", 2023-09-23T20:56:41.586 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T20:56:41.586 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:41.586 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:41.586 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T20:56:41.586 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:07.386681+0000", 2023-09-23T20:56:41.587 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.2679999999999994e-06 2023-09-23T20:56:41.587 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:41.587 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:41.587 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T20:56:41.587 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:07.386702+0000", 2023-09-23T20:56:41.587 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.173e-05 2023-09-23T20:56:41.587 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:41.587 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:41.587 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T20:56:41.587 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:07.386710+0000", 2023-09-23T20:56:41.587 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.0069999999999997e-06 2023-09-23T20:56:41.588 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:41.588 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:41.588 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T20:56:41.588 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:07.386770+0000", 2023-09-23T20:56:41.588 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.9499000000000001e-05 2023-09-23T20:56:41.588 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:41.588 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:41.588 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T20:56:41.588 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:07.386780+0000", 2023-09-23T20:56:41.588 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.5619999999999993e-06 2023-09-23T20:56:41.588 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:41.588 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:41.588 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T20:56:41.589 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.363453+0000", 2023-09-23T20:56:41.589 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.97667318999999997 2023-09-23T20:56:41.589 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:41.589 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:41.589 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T20:56:41.589 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.363538+0000", 2023-09-23T20:56:41.589 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.5484999999999999e-05 2023-09-23T20:56:41.589 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:41.589 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:41.589 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 2,3", 2023-09-23T20:56:41.589 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.363717+0000", 2023-09-23T20:56:41.589 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00017910699999999999 2023-09-23T20:56:41.590 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:41.590 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:41.590 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T20:56:41.590 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.366914+0000", 2023-09-23T20:56:41.590 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0031969059999999998 2023-09-23T20:56:41.590 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:41.590 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:41.590 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:56:41.590 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.372737+0000", 2023-09-23T20:56:41.590 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00582243 2023-09-23T20:56:41.590 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:41.590 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:41.591 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:56:41.591 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.372790+0000", 2023-09-23T20:56:41.591 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.3454e-05 2023-09-23T20:56:41.591 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:41.591 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:41.591 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T20:56:41.591 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.372821+0000", 2023-09-23T20:56:41.591 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.0725999999999997e-05 2023-09-23T20:56:41.591 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:41.591 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:41.591 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T20:56:41.591 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.372866+0000", 2023-09-23T20:56:41.591 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.5433999999999999e-05 2023-09-23T20:56:41.592 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T20:56:41.592 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T20:56:41.592 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T20:56:41.592 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:41.592 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:41.592 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.34797.0:156863 6.12 6:48481dbe:::benchmark_data_smithi026_223868_object55737:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e150)", 2023-09-23T20:56:41.592 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T20:55:07.442400+0000", 2023-09-23T20:56:41.592 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 94.111130482999997, 2023-09-23T20:56:41.592 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.93060301300000003, 2023-09-23T20:56:41.592 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T20:56:41.592 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:56:41.592 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T20:56:41.592 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.34797", 2023-09-23T20:56:41.593 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T20:56:41.593 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 156863 2023-09-23T20:56:41.593 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:41.593 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T20:56:41.593 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:41.593 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T20:56:41.593 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:07.442400+0000", 2023-09-23T20:56:41.593 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T20:56:41.593 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:41.593 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:41.593 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T20:56:41.593 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:07.442400+0000", 2023-09-23T20:56:41.593 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T20:56:41.594 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:41.594 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:41.594 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T20:56:41.594 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:07.442399+0000", 2023-09-23T20:56:41.594 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T20:56:41.594 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:41.594 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:41.594 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T20:56:41.594 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:07.442405+0000", 2023-09-23T20:56:41.595 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.7490000000000001e-06 2023-09-23T20:56:41.595 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:41.595 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:41.595 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T20:56:41.595 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:07.442421+0000", 2023-09-23T20:56:41.595 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.6110999999999999e-05 2023-09-23T20:56:41.595 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:41.595 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:41.595 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T20:56:41.595 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:07.442428+0000", 2023-09-23T20:56:41.595 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.9349999999999997e-06 2023-09-23T20:56:41.595 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:41.595 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:41.596 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T20:56:41.596 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:07.442498+0000", 2023-09-23T20:56:41.596 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.9488999999999998e-05 2023-09-23T20:56:41.596 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:41.596 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:41.596 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T20:56:41.596 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:07.442508+0000", 2023-09-23T20:56:41.596 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.0329000000000001e-05 2023-09-23T20:56:41.596 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:41.596 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:41.596 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T20:56:41.596 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.364821+0000", 2023-09-23T20:56:41.596 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.92231251999999997 2023-09-23T20:56:41.597 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:41.597 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:41.597 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T20:56:41.597 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.364917+0000", 2023-09-23T20:56:41.597 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.5866000000000007e-05 2023-09-23T20:56:41.597 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:41.597 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:41.597 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 2,3", 2023-09-23T20:56:41.597 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.365161+0000", 2023-09-23T20:56:41.597 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00024452300000000002 2023-09-23T20:56:41.597 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:41.597 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:41.597 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T20:56:41.598 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.367047+0000", 2023-09-23T20:56:41.598 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.001885947 2023-09-23T20:56:41.598 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:41.598 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:41.598 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:56:41.598 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.372916+0000", 2023-09-23T20:56:41.598 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0058686110000000001 2023-09-23T20:56:41.598 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:41.598 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:41.598 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:56:41.598 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.372948+0000", 2023-09-23T20:56:41.598 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.2159000000000002e-05 2023-09-23T20:56:41.599 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:41.599 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:41.599 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T20:56:41.599 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.372972+0000", 2023-09-23T20:56:41.599 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.4244999999999999e-05 2023-09-23T20:56:41.599 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:41.599 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:41.599 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T20:56:41.599 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.373003+0000", 2023-09-23T20:56:41.599 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.112e-05 2023-09-23T20:56:41.599 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T20:56:41.599 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T20:56:41.599 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T20:56:41.600 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:41.600 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:41.600 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.34797.0:156879 6.12 6:4b4fed44:::benchmark_data_smithi026_223868_object55753:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e150)", 2023-09-23T20:56:41.600 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T20:55:07.459292+0000", 2023-09-23T20:56:41.600 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 94.094238993999994, 2023-09-23T20:56:41.600 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.91741719200000005, 2023-09-23T20:56:41.600 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T20:56:41.600 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:56:41.600 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T20:56:41.600 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.34797", 2023-09-23T20:56:41.601 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T20:56:41.601 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 156879 2023-09-23T20:56:41.601 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:41.601 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T20:56:41.601 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:41.601 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T20:56:41.601 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:07.459292+0000", 2023-09-23T20:56:41.602 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T20:56:41.602 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:41.602 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:41.602 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T20:56:41.602 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:07.459292+0000", 2023-09-23T20:56:41.602 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T20:56:41.602 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:41.602 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:41.602 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T20:56:41.602 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:07.459290+0000", 2023-09-23T20:56:41.602 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999981 2023-09-23T20:56:41.603 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:41.603 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:41.603 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T20:56:41.603 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:07.459299+0000", 2023-09-23T20:56:41.603 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.1679999999999993e-06 2023-09-23T20:56:41.603 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:41.603 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:41.603 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T20:56:41.603 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:07.459319+0000", 2023-09-23T20:56:41.603 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.9945999999999998e-05 2023-09-23T20:56:41.603 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:41.603 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:41.604 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T20:56:41.604 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:07.459328+0000", 2023-09-23T20:56:41.604 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.5439999999999999e-06 2023-09-23T20:56:41.604 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:41.604 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:41.604 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T20:56:41.604 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:07.459397+0000", 2023-09-23T20:56:41.604 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.9640000000000004e-05 2023-09-23T20:56:41.604 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:41.604 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:41.604 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T20:56:41.604 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:07.459408+0000", 2023-09-23T20:56:41.604 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.1228e-05 2023-09-23T20:56:41.605 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:41.605 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:41.605 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T20:56:41.605 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.367075+0000", 2023-09-23T20:56:41.605 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.90766636599999995 2023-09-23T20:56:41.605 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:41.605 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:41.605 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T20:56:41.605 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.367196+0000", 2023-09-23T20:56:41.605 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000121274 2023-09-23T20:56:41.605 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:41.605 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:41.605 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 2,3", 2023-09-23T20:56:41.606 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.367465+0000", 2023-09-23T20:56:41.606 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00026886999999999999 2023-09-23T20:56:41.606 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:41.606 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:41.606 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T20:56:41.606 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.372639+0000", 2023-09-23T20:56:41.606 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0051736430000000003 2023-09-23T20:56:41.606 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:41.606 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:41.606 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:56:41.606 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.376593+0000", 2023-09-23T20:56:41.606 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0039539919999999999 2023-09-23T20:56:41.606 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:41.607 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:41.607 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:56:41.607 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.376628+0000", 2023-09-23T20:56:41.607 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.5759000000000001e-05 2023-09-23T20:56:41.607 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:41.607 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:41.607 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T20:56:41.607 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.376671+0000", 2023-09-23T20:56:41.607 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.2969999999999997e-05 2023-09-23T20:56:41.607 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:41.607 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:41.607 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T20:56:41.608 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.376709+0000", 2023-09-23T20:56:41.608 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.7540000000000003e-05 2023-09-23T20:56:41.608 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T20:56:41.608 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T20:56:41.608 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T20:56:41.608 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:41.608 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:41.608 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.34797.0:156990 6.12 6:491f6f4b:::benchmark_data_smithi026_223868_object55864:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e150)", 2023-09-23T20:56:41.608 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T20:55:07.581755+0000", 2023-09-23T20:56:41.608 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 93.971775566000005, 2023-09-23T20:56:41.608 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.79863560899999997, 2023-09-23T20:56:41.608 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T20:56:41.608 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:56:41.609 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T20:56:41.609 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.34797", 2023-09-23T20:56:41.609 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T20:56:41.609 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 156990 2023-09-23T20:56:41.609 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:41.609 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T20:56:41.609 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:41.609 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T20:56:41.609 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:07.581755+0000", 2023-09-23T20:56:41.609 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T20:56:41.609 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:41.609 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:41.609 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T20:56:41.610 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:07.581755+0000", 2023-09-23T20:56:41.610 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T20:56:41.610 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:41.610 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:41.610 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T20:56:41.610 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:07.581754+0000", 2023-09-23T20:56:41.610 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T20:56:41.610 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:41.610 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:41.610 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T20:56:41.611 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:07.581760+0000", 2023-09-23T20:56:41.611 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.02e-06 2023-09-23T20:56:41.611 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:41.611 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:41.611 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T20:56:41.611 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:07.581773+0000", 2023-09-23T20:56:41.611 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.2921e-05 2023-09-23T20:56:41.611 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:41.611 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:41.611 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T20:56:41.611 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:07.581779+0000", 2023-09-23T20:56:41.611 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.7889999999999997e-06 2023-09-23T20:56:41.612 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:41.612 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:41.612 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T20:56:41.612 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:07.581838+0000", 2023-09-23T20:56:41.612 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.8850000000000001e-05 2023-09-23T20:56:41.612 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:41.612 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:41.612 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T20:56:41.612 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:07.581848+0000", 2023-09-23T20:56:41.612 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.9340000000000001e-06 2023-09-23T20:56:41.612 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:41.612 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:41.612 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T20:56:41.613 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.368883+0000", 2023-09-23T20:56:41.613 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.78703539700000003 2023-09-23T20:56:41.613 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:41.613 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:41.613 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T20:56:41.613 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.368984+0000", 2023-09-23T20:56:41.613 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000100717 2023-09-23T20:56:41.613 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:41.613 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:41.614 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 2,3", 2023-09-23T20:56:41.614 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.369202+0000", 2023-09-23T20:56:41.614 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000218304 2023-09-23T20:56:41.614 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:41.614 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:41.614 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T20:56:41.614 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.372876+0000", 2023-09-23T20:56:41.614 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.003673511 2023-09-23T20:56:41.614 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:41.614 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:41.614 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:56:41.614 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.380243+0000", 2023-09-23T20:56:41.614 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0073674049999999996 2023-09-23T20:56:41.615 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:41.615 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:41.615 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:56:41.615 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.380308+0000", 2023-09-23T20:56:41.615 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.4309999999999999e-05 2023-09-23T20:56:41.615 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:41.615 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:41.615 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T20:56:41.615 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.380357+0000", 2023-09-23T20:56:41.615 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.9095999999999999e-05 2023-09-23T20:56:41.615 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:41.615 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:41.615 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T20:56:41.616 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.380391+0000", 2023-09-23T20:56:41.616 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.3850999999999998e-05 2023-09-23T20:56:41.616 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T20:56:41.616 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T20:56:41.616 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T20:56:41.616 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:41.616 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:41.616 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.34797.0:157031 6.12 6:48e5af49:::benchmark_data_smithi026_223868_object55905:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e150)", 2023-09-23T20:56:41.616 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T20:55:07.628542+0000", 2023-09-23T20:56:41.616 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 93.924988253999999, 2023-09-23T20:56:41.616 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.75194783899999995, 2023-09-23T20:56:41.616 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T20:56:41.616 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:56:41.617 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T20:56:41.617 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.34797", 2023-09-23T20:56:41.617 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T20:56:41.617 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 157031 2023-09-23T20:56:41.617 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:41.617 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T20:56:41.617 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:41.617 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T20:56:41.617 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:07.628542+0000", 2023-09-23T20:56:41.617 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T20:56:41.617 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:41.617 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:41.617 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T20:56:41.618 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:07.628542+0000", 2023-09-23T20:56:41.618 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T20:56:41.618 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:41.618 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:41.618 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T20:56:41.618 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:07.628541+0000", 2023-09-23T20:56:41.618 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.999999 2023-09-23T20:56:41.618 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:41.618 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:41.618 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T20:56:41.618 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:07.628549+0000", 2023-09-23T20:56:41.618 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.9019999999999999e-06 2023-09-23T20:56:41.618 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:41.619 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:41.619 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T20:56:41.619 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:07.628562+0000", 2023-09-23T20:56:41.619 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.2986999999999999e-05 2023-09-23T20:56:41.619 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:41.619 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:41.619 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T20:56:41.619 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:07.628567+0000", 2023-09-23T20:56:41.619 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.322e-06 2023-09-23T20:56:41.619 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:41.619 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:41.619 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T20:56:41.619 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:07.628615+0000", 2023-09-23T20:56:41.620 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.7564000000000001e-05 2023-09-23T20:56:41.620 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:41.620 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:41.620 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T20:56:41.620 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:07.628622+0000", 2023-09-23T20:56:41.620 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.6440000000000003e-06 2023-09-23T20:56:41.620 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:41.620 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:41.620 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T20:56:41.620 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.370771+0000", 2023-09-23T20:56:41.620 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.74214981700000004 2023-09-23T20:56:41.620 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:41.620 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:41.621 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T20:56:41.621 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.370876+0000", 2023-09-23T20:56:41.621 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00010476000000000001 2023-09-23T20:56:41.621 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:41.621 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:41.621 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 2,3", 2023-09-23T20:56:41.621 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.371134+0000", 2023-09-23T20:56:41.621 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00025788100000000001 2023-09-23T20:56:41.621 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:41.621 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:41.621 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T20:56:41.621 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.373009+0000", 2023-09-23T20:56:41.621 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.001875334 2023-09-23T20:56:41.622 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:41.622 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:41.622 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:56:41.622 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.380281+0000", 2023-09-23T20:56:41.622 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0072716710000000004 2023-09-23T20:56:41.622 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:41.622 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:41.622 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:56:41.622 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.380422+0000", 2023-09-23T20:56:41.622 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00014123299999999999 2023-09-23T20:56:41.622 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:41.622 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:41.623 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T20:56:41.623 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.380460+0000", 2023-09-23T20:56:41.623 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.7372000000000001e-05 2023-09-23T20:56:41.623 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:41.623 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:41.623 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T20:56:41.623 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.380490+0000", 2023-09-23T20:56:41.623 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.0493000000000001e-05 2023-09-23T20:56:41.623 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T20:56:41.623 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T20:56:41.623 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T20:56:41.623 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:41.623 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:41.624 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.34797.0:157050 6.12 6:4be05e13:::benchmark_data_smithi026_223868_object55924:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e150)", 2023-09-23T20:56:41.624 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T20:55:07.649612+0000", 2023-09-23T20:56:41.624 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 93.903919055000003, 2023-09-23T20:56:41.624 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.73668807700000005, 2023-09-23T20:56:41.624 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T20:56:41.624 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:56:41.624 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T20:56:41.624 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.34797", 2023-09-23T20:56:41.624 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T20:56:41.624 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 157050 2023-09-23T20:56:41.624 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:41.624 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T20:56:41.624 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:41.624 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T20:56:41.625 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:07.649612+0000", 2023-09-23T20:56:41.625 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T20:56:41.625 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:41.625 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:41.625 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T20:56:41.625 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:07.649612+0000", 2023-09-23T20:56:41.625 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T20:56:41.625 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:41.625 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:41.625 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T20:56:41.625 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:07.649610+0000", 2023-09-23T20:56:41.625 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T20:56:41.625 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:41.626 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:41.626 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T20:56:41.626 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:07.649619+0000", 2023-09-23T20:56:41.626 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.7439999999999996e-06 2023-09-23T20:56:41.626 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:41.626 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:41.626 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T20:56:41.626 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:07.649633+0000", 2023-09-23T20:56:41.626 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.4056000000000001e-05 2023-09-23T20:56:41.626 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:41.626 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:41.626 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T20:56:41.626 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:07.649641+0000", 2023-09-23T20:56:41.627 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.5290000000000003e-06 2023-09-23T20:56:41.627 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:41.627 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:41.627 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T20:56:41.627 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:07.649701+0000", 2023-09-23T20:56:41.627 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.0715999999999999e-05 2023-09-23T20:56:41.627 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:41.627 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:41.627 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T20:56:41.627 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:07.649711+0000", 2023-09-23T20:56:41.627 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.5529999999999998e-06 2023-09-23T20:56:41.627 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:41.627 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:41.628 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T20:56:41.628 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.373038+0000", 2023-09-23T20:56:41.628 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.72332733999999999 2023-09-23T20:56:41.628 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:41.628 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:41.628 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T20:56:41.628 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.373159+0000", 2023-09-23T20:56:41.628 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000120704 2023-09-23T20:56:41.628 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:41.628 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:41.628 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 2,3", 2023-09-23T20:56:41.628 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.373420+0000", 2023-09-23T20:56:41.628 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00026133599999999998 2023-09-23T20:56:41.629 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:41.629 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:41.629 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T20:56:41.629 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.376599+0000", 2023-09-23T20:56:41.629 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0031784790000000001 2023-09-23T20:56:41.629 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:41.629 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:41.629 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:56:41.629 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.380519+0000", 2023-09-23T20:56:41.629 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0039201569999999996 2023-09-23T20:56:41.629 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:41.629 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:41.629 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:56:41.630 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.386243+0000", 2023-09-23T20:56:41.630 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0057240770000000002 2023-09-23T20:56:41.630 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:41.630 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:41.630 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T20:56:41.630 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.386271+0000", 2023-09-23T20:56:41.630 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.8518e-05 2023-09-23T20:56:41.630 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:41.630 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:41.630 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T20:56:41.630 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.386300+0000", 2023-09-23T20:56:41.630 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.8211000000000001e-05 2023-09-23T20:56:41.630 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T20:56:41.631 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T20:56:41.631 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T20:56:41.631 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:41.631 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:41.631 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.34797.0:157117 6.12 6:4d4c4893:::benchmark_data_smithi026_223868_object55991:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e150)", 2023-09-23T20:56:41.631 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T20:55:07.747668+0000", 2023-09-23T20:56:41.631 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 93.805862288, 2023-09-23T20:56:41.631 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.63872341099999996, 2023-09-23T20:56:41.631 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T20:56:41.631 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:56:41.631 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T20:56:41.631 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.34797", 2023-09-23T20:56:41.631 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T20:56:41.632 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 157117 2023-09-23T20:56:41.632 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:41.632 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T20:56:41.632 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:41.632 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T20:56:41.632 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:07.747668+0000", 2023-09-23T20:56:41.632 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T20:56:41.632 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:41.632 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:41.632 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T20:56:41.632 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:07.747668+0000", 2023-09-23T20:56:41.632 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T20:56:41.632 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:41.633 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:41.633 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T20:56:41.633 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:07.747667+0000", 2023-09-23T20:56:41.633 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T20:56:41.633 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:41.633 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:41.633 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T20:56:41.633 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:07.747673+0000", 2023-09-23T20:56:41.633 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.2580000000000001e-06 2023-09-23T20:56:41.633 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:41.633 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:41.633 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T20:56:41.633 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:07.747689+0000", 2023-09-23T20:56:41.634 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.5837000000000001e-05 2023-09-23T20:56:41.634 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:41.634 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:41.634 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T20:56:41.634 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:07.747698+0000", 2023-09-23T20:56:41.634 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.6039999999999993e-06 2023-09-23T20:56:41.634 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:41.634 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:41.634 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T20:56:41.634 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:07.747754+0000", 2023-09-23T20:56:41.634 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.6487000000000003e-05 2023-09-23T20:56:41.634 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:41.635 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:41.635 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T20:56:41.635 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:07.747764+0000", 2023-09-23T20:56:41.635 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.0108e-05 2023-09-23T20:56:41.635 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:41.635 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:41.635 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T20:56:41.635 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.374677+0000", 2023-09-23T20:56:41.635 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.62691302900000001 2023-09-23T20:56:41.635 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:41.635 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:41.635 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T20:56:41.635 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.374761+0000", 2023-09-23T20:56:41.635 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.3807999999999994e-05 2023-09-23T20:56:41.636 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:41.636 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:41.636 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 2,3", 2023-09-23T20:56:41.636 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.374943+0000", 2023-09-23T20:56:41.636 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00018220399999999999 2023-09-23T20:56:41.636 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:41.636 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:41.636 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T20:56:41.636 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.378533+0000", 2023-09-23T20:56:41.636 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0035901190000000001 2023-09-23T20:56:41.636 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:41.636 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:41.637 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:56:41.637 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.380547+0000", 2023-09-23T20:56:41.637 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.002013379 2023-09-23T20:56:41.637 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:41.637 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:41.637 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:56:41.637 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.386342+0000", 2023-09-23T20:56:41.637 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0057952480000000002 2023-09-23T20:56:41.637 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:41.637 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:41.637 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T20:56:41.637 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.386365+0000", 2023-09-23T20:56:41.637 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.3411e-05 2023-09-23T20:56:41.638 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:41.638 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:41.638 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T20:56:41.638 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.386392+0000", 2023-09-23T20:56:41.638 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.6301000000000001e-05 2023-09-23T20:56:41.638 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T20:56:41.638 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T20:56:41.638 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T20:56:41.638 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:41.638 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:41.638 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.34797.0:157161 6.12 6:4e589f92:::benchmark_data_smithi026_223868_object56035:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e150)", 2023-09-23T20:56:41.638 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T20:55:07.800631+0000", 2023-09-23T20:56:41.639 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 93.752899693000003, 2023-09-23T20:56:41.639 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.58585551000000002, 2023-09-23T20:56:41.639 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T20:56:41.639 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:56:41.639 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T20:56:41.639 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.34797", 2023-09-23T20:56:41.639 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T20:56:41.639 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 157161 2023-09-23T20:56:41.639 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:41.639 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T20:56:41.639 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:41.639 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T20:56:41.639 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:07.800631+0000", 2023-09-23T20:56:41.640 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T20:56:41.640 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:41.640 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:41.640 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T20:56:41.640 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:07.800631+0000", 2023-09-23T20:56:41.640 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T20:56:41.640 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:41.640 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:41.640 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T20:56:41.640 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:07.800630+0000", 2023-09-23T20:56:41.640 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T20:56:41.640 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:41.641 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:41.641 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T20:56:41.641 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:07.800636+0000", 2023-09-23T20:56:41.641 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.9100000000000002e-06 2023-09-23T20:56:41.641 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:41.641 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:41.641 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T20:56:41.641 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:07.800650+0000", 2023-09-23T20:56:41.641 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.4409999999999999e-05 2023-09-23T20:56:41.641 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:41.641 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:41.641 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T20:56:41.641 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:07.800657+0000", 2023-09-23T20:56:41.642 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.0829999999999998e-06 2023-09-23T20:56:41.642 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:41.642 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:41.642 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T20:56:41.642 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:07.800709+0000", 2023-09-23T20:56:41.642 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.1907999999999998e-05 2023-09-23T20:56:41.642 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:41.642 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:41.642 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T20:56:41.642 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:07.800721+0000", 2023-09-23T20:56:41.642 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.2529e-05 2023-09-23T20:56:41.642 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:41.643 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:41.643 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T20:56:41.643 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.376733+0000", 2023-09-23T20:56:41.643 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.57601174499999996 2023-09-23T20:56:41.643 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:41.643 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:41.643 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T20:56:41.643 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.376824+0000", 2023-09-23T20:56:41.643 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.1108999999999998e-05 2023-09-23T20:56:41.643 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:41.643 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:41.643 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 2,3", 2023-09-23T20:56:41.643 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.377089+0000", 2023-09-23T20:56:41.644 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000265128 2023-09-23T20:56:41.644 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:41.644 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:41.644 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T20:56:41.644 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.380251+0000", 2023-09-23T20:56:41.644 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0031611230000000001 2023-09-23T20:56:41.644 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:41.644 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:41.644 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:56:41.644 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.386220+0000", 2023-09-23T20:56:41.644 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0059696009999999997 2023-09-23T20:56:41.644 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:41.644 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:41.645 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:56:41.645 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.386427+0000", 2023-09-23T20:56:41.645 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000206951 2023-09-23T20:56:41.645 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:41.645 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:41.645 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T20:56:41.645 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.386466+0000", 2023-09-23T20:56:41.645 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.8874000000000001e-05 2023-09-23T20:56:41.645 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:41.645 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:41.645 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T20:56:41.645 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.386486+0000", 2023-09-23T20:56:41.646 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.0457000000000001e-05 2023-09-23T20:56:41.646 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T20:56:41.646 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T20:56:41.646 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T20:56:41.646 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:41.646 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:41.646 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.34797.0:157227 6.12 6:4d4ca5fd:::benchmark_data_smithi026_223868_object56101:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e150)", 2023-09-23T20:56:41.646 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T20:55:07.900387+0000", 2023-09-23T20:56:41.646 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 93.653143417999999, 2023-09-23T20:56:41.646 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.49162882600000002, 2023-09-23T20:56:41.646 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T20:56:41.646 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:56:41.646 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T20:56:41.647 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.34797", 2023-09-23T20:56:41.647 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T20:56:41.647 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 157227 2023-09-23T20:56:41.647 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:41.647 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T20:56:41.647 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:41.647 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T20:56:41.647 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:07.900387+0000", 2023-09-23T20:56:41.647 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T20:56:41.647 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:41.647 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:41.647 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T20:56:41.647 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:07.900387+0000", 2023-09-23T20:56:41.648 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T20:56:41.648 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:41.648 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:41.648 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T20:56:41.648 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:07.900386+0000", 2023-09-23T20:56:41.648 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T20:56:41.648 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:41.648 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:41.648 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T20:56:41.648 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:07.900392+0000", 2023-09-23T20:56:41.648 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.5899999999999996e-06 2023-09-23T20:56:41.648 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:41.649 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:41.649 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T20:56:41.649 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:07.900408+0000", 2023-09-23T20:56:41.649 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.5430999999999998e-05 2023-09-23T20:56:41.649 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:41.649 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:41.649 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T20:56:41.649 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:07.900415+0000", 2023-09-23T20:56:41.649 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.7739999999999999e-06 2023-09-23T20:56:41.649 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:41.649 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:41.649 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T20:56:41.650 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:07.900476+0000", 2023-09-23T20:56:41.650 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.1452999999999996e-05 2023-09-23T20:56:41.650 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:41.650 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:41.650 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T20:56:41.650 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:07.900485+0000", 2023-09-23T20:56:41.650 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.3260000000000002e-06 2023-09-23T20:56:41.650 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:41.650 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:41.650 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T20:56:41.650 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.378561+0000", 2023-09-23T20:56:41.650 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.47807563199999997 2023-09-23T20:56:41.650 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:41.651 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:41.651 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T20:56:41.651 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.378655+0000", 2023-09-23T20:56:41.651 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.3793000000000001e-05 2023-09-23T20:56:41.651 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:41.651 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:41.651 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 2,3", 2023-09-23T20:56:41.651 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.378841+0000", 2023-09-23T20:56:41.651 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00018573600000000001 2023-09-23T20:56:41.651 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:41.651 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:41.651 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:56:41.651 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.386326+0000", 2023-09-23T20:56:41.652 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0074849620000000004 2023-09-23T20:56:41.652 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:41.652 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:41.652 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:56:41.652 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.386517+0000", 2023-09-23T20:56:41.652 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00019191999999999999 2023-09-23T20:56:41.652 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:41.652 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:41.652 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T20:56:41.652 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.391931+0000", 2023-09-23T20:56:41.652 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0054132310000000001 2023-09-23T20:56:41.652 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:41.652 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:41.653 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T20:56:41.653 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.391976+0000", 2023-09-23T20:56:41.653 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.5697e-05 2023-09-23T20:56:41.653 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:41.653 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:41.653 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T20:56:41.653 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.392016+0000", 2023-09-23T20:56:41.653 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.9644000000000002e-05 2023-09-23T20:56:41.653 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T20:56:41.653 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T20:56:41.653 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T20:56:41.653 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:41.653 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:41.654 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.34797.0:157259 6.12 6:4e9f8804:::benchmark_data_smithi026_223868_object56133:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e150)", 2023-09-23T20:56:41.654 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T20:55:07.947142+0000", 2023-09-23T20:56:41.654 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 93.606388186000004, 2023-09-23T20:56:41.654 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.44774977900000001, 2023-09-23T20:56:41.654 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T20:56:41.654 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:56:41.654 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T20:56:41.654 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.34797", 2023-09-23T20:56:41.654 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T20:56:41.654 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 157259 2023-09-23T20:56:41.654 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:41.654 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T20:56:41.655 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:41.655 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T20:56:41.655 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:07.947142+0000", 2023-09-23T20:56:41.655 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T20:56:41.655 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:41.655 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:41.655 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T20:56:41.655 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:07.947142+0000", 2023-09-23T20:56:41.655 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T20:56:41.655 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:41.655 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:41.655 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T20:56:41.655 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:07.947142+0000", 2023-09-23T20:56:41.655 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999995 2023-09-23T20:56:41.656 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:41.656 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:41.656 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T20:56:41.656 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:07.947146+0000", 2023-09-23T20:56:41.656 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.0500000000000002e-06 2023-09-23T20:56:41.656 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:41.656 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:41.656 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T20:56:41.656 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:07.947160+0000", 2023-09-23T20:56:41.656 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.4662e-05 2023-09-23T20:56:41.656 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:41.656 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:41.657 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T20:56:41.657 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:07.947166+0000", 2023-09-23T20:56:41.657 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.784e-06 2023-09-23T20:56:41.657 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:41.657 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:41.657 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T20:56:41.657 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:07.947223+0000", 2023-09-23T20:56:41.657 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.6854000000000001e-05 2023-09-23T20:56:41.657 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:41.657 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:41.657 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T20:56:41.657 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:07.947238+0000", 2023-09-23T20:56:41.657 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.4952e-05 2023-09-23T20:56:41.658 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:41.658 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:41.658 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T20:56:41.658 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.380572+0000", 2023-09-23T20:56:41.658 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.43333436400000003 2023-09-23T20:56:41.658 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:41.658 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:41.658 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T20:56:41.658 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.380689+0000", 2023-09-23T20:56:41.658 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00011646600000000001 2023-09-23T20:56:41.658 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:41.658 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:41.659 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 2,3", 2023-09-23T20:56:41.659 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.380934+0000", 2023-09-23T20:56:41.659 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00024519199999999998 2023-09-23T20:56:41.659 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:41.659 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:41.659 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:56:41.659 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.390072+0000", 2023-09-23T20:56:41.659 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0091384410000000006 2023-09-23T20:56:41.659 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:41.659 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:41.659 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T20:56:41.659 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.392023+0000", 2023-09-23T20:56:41.659 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0019504170000000001 2023-09-23T20:56:41.660 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:41.660 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:41.660 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:56:41.660 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.394809+0000", 2023-09-23T20:56:41.660 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0027863760000000001 2023-09-23T20:56:41.660 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:41.660 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:41.660 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T20:56:41.660 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.394852+0000", 2023-09-23T20:56:41.660 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.3130999999999998e-05 2023-09-23T20:56:41.660 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:41.660 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:41.660 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T20:56:41.661 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.394892+0000", 2023-09-23T20:56:41.661 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.9801000000000001e-05 2023-09-23T20:56:41.661 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T20:56:41.661 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T20:56:41.661 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T20:56:41.661 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:41.661 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:41.661 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.34797.0:157328 6.12 6:4811798e:::benchmark_data_smithi026_223868_object56202:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e150)", 2023-09-23T20:56:41.661 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T20:55:08.060527+0000", 2023-09-23T20:56:41.661 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 93.493003459999997, 2023-09-23T20:56:41.661 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.33450326899999999, 2023-09-23T20:56:41.661 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T20:56:41.661 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:56:41.662 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T20:56:41.662 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.34797", 2023-09-23T20:56:41.662 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T20:56:41.662 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 157328 2023-09-23T20:56:41.662 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:41.662 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T20:56:41.662 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:41.662 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T20:56:41.662 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.060527+0000", 2023-09-23T20:56:41.662 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T20:56:41.662 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:41.662 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:41.662 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T20:56:41.663 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.060527+0000", 2023-09-23T20:56:41.663 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T20:56:41.663 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:41.663 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:41.663 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T20:56:41.663 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.060526+0000", 2023-09-23T20:56:41.663 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.999999 2023-09-23T20:56:41.663 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:41.663 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:41.663 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T20:56:41.663 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.060531+0000", 2023-09-23T20:56:41.663 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.25e-06 2023-09-23T20:56:41.663 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:41.664 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:41.664 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T20:56:41.664 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.060540+0000", 2023-09-23T20:56:41.664 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.0259999999999996e-06 2023-09-23T20:56:41.664 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:41.664 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:41.664 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T20:56:41.664 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.060544+0000", 2023-09-23T20:56:41.664 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.1370000000000004e-06 2023-09-23T20:56:41.664 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:41.664 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:41.664 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T20:56:41.665 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.060605+0000", 2023-09-23T20:56:41.665 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.0939e-05 2023-09-23T20:56:41.665 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:41.665 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:41.665 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T20:56:41.665 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.060616+0000", 2023-09-23T20:56:41.665 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.1824999999999999e-05 2023-09-23T20:56:41.665 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:41.665 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:41.665 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T20:56:41.665 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.382773+0000", 2023-09-23T20:56:41.665 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.32215636800000003 2023-09-23T20:56:41.665 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:41.666 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:41.666 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T20:56:41.666 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.382925+0000", 2023-09-23T20:56:41.666 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00015193499999999999 2023-09-23T20:56:41.666 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:41.666 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:41.666 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 2,3", 2023-09-23T20:56:41.666 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.383267+0000", 2023-09-23T20:56:41.666 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00034205500000000002 2023-09-23T20:56:41.666 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:41.666 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:41.666 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:56:41.667 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.390103+0000", 2023-09-23T20:56:41.667 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0068357510000000002 2023-09-23T20:56:41.667 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:41.667 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:41.667 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T20:56:41.667 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.392028+0000", 2023-09-23T20:56:41.667 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.001925775 2023-09-23T20:56:41.667 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:41.667 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:41.667 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:56:41.667 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.394957+0000", 2023-09-23T20:56:41.667 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0029286030000000001 2023-09-23T20:56:41.667 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:41.668 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:41.668 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T20:56:41.668 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.394996+0000", 2023-09-23T20:56:41.668 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.9410999999999999e-05 2023-09-23T20:56:41.668 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:41.668 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:41.668 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T20:56:41.668 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.395030+0000", 2023-09-23T20:56:41.668 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.4112000000000001e-05 2023-09-23T20:56:41.668 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T20:56:41.668 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T20:56:41.668 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T20:56:41.668 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:41.669 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:41.669 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.34797.0:157346 6.12 6:4c4003c4:::benchmark_data_smithi026_223868_object56220:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e150)", 2023-09-23T20:56:41.669 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T20:55:08.089574+0000", 2023-09-23T20:56:41.669 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 93.463957015999995, 2023-09-23T20:56:41.669 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.305631714, 2023-09-23T20:56:41.669 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T20:56:41.669 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:56:41.669 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T20:56:41.669 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.34797", 2023-09-23T20:56:41.669 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T20:56:41.669 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 157346 2023-09-23T20:56:41.669 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:41.669 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T20:56:41.670 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:41.670 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T20:56:41.670 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.089574+0000", 2023-09-23T20:56:41.670 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T20:56:41.670 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:41.670 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:41.670 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T20:56:41.670 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.089574+0000", 2023-09-23T20:56:41.670 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T20:56:41.670 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:41.670 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:41.670 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T20:56:41.670 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.089572+0000", 2023-09-23T20:56:41.671 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999981 2023-09-23T20:56:41.671 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:41.671 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:41.671 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T20:56:41.671 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.089579+0000", 2023-09-23T20:56:41.671 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.6290000000000001e-06 2023-09-23T20:56:41.671 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:41.671 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:41.671 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T20:56:41.671 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.089596+0000", 2023-09-23T20:56:41.671 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.6511000000000002e-05 2023-09-23T20:56:41.671 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:41.671 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:41.672 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T20:56:41.672 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.089605+0000", 2023-09-23T20:56:41.672 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.5390000000000007e-06 2023-09-23T20:56:41.672 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:41.672 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:41.672 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T20:56:41.672 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.089665+0000", 2023-09-23T20:56:41.672 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.0859000000000001e-05 2023-09-23T20:56:41.672 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:41.672 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:41.672 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T20:56:41.672 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.089678+0000", 2023-09-23T20:56:41.673 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.2133000000000001e-05 2023-09-23T20:56:41.673 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:41.673 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:41.673 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T20:56:41.673 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.384720+0000", 2023-09-23T20:56:41.673 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.29504224499999998 2023-09-23T20:56:41.673 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:41.673 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:41.673 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T20:56:41.673 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.384814+0000", 2023-09-23T20:56:41.673 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.4611999999999995e-05 2023-09-23T20:56:41.673 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:41.673 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:41.674 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 2,3", 2023-09-23T20:56:41.674 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.385022+0000", 2023-09-23T20:56:41.674 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00020736999999999999 2023-09-23T20:56:41.674 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:41.674 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:41.674 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T20:56:41.674 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.392033+0000", 2023-09-23T20:56:41.674 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0070114649999999997 2023-09-23T20:56:41.674 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:41.674 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:41.674 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:56:41.674 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.395074+0000", 2023-09-23T20:56:41.675 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.003040297 2023-09-23T20:56:41.675 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:41.675 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:41.675 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:56:41.675 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.395133+0000", 2023-09-23T20:56:41.675 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.9700000000000001e-05 2023-09-23T20:56:41.675 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:41.675 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:41.675 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T20:56:41.675 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.395173+0000", 2023-09-23T20:56:41.675 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.9956999999999997e-05 2023-09-23T20:56:41.675 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:41.675 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:41.676 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T20:56:41.676 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.395205+0000", 2023-09-23T20:56:41.676 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.2146e-05 2023-09-23T20:56:41.676 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T20:56:41.676 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T20:56:41.676 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T20:56:41.676 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:41.676 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:41.676 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.34797.0:157407 6.12 6:494044ed:::benchmark_data_smithi026_223868_object56281:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e150)", 2023-09-23T20:56:41.676 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T20:55:08.202324+0000", 2023-09-23T20:56:41.676 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 93.351206876000006, 2023-09-23T20:56:41.676 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.19300698499999999, 2023-09-23T20:56:41.676 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T20:56:41.677 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:56:41.677 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T20:56:41.677 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.34797", 2023-09-23T20:56:41.677 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T20:56:41.677 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 157407 2023-09-23T20:56:41.677 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:41.677 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T20:56:41.677 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:41.677 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T20:56:41.677 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.202324+0000", 2023-09-23T20:56:41.677 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T20:56:41.677 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:41.678 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:41.678 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T20:56:41.678 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.202324+0000", 2023-09-23T20:56:41.678 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T20:56:41.678 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:41.678 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:41.678 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T20:56:41.678 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.202322+0000", 2023-09-23T20:56:41.678 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T20:56:41.678 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:41.678 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:41.678 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T20:56:41.679 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.202330+0000", 2023-09-23T20:56:41.679 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.8730000000000007e-06 2023-09-23T20:56:41.679 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:41.679 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:41.679 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T20:56:41.679 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.202346+0000", 2023-09-23T20:56:41.679 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.5807e-05 2023-09-23T20:56:41.679 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:41.679 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:41.679 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T20:56:41.679 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.202352+0000", 2023-09-23T20:56:41.679 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.4609999999999996e-06 2023-09-23T20:56:41.679 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:41.680 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:41.680 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T20:56:41.680 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.202412+0000", 2023-09-23T20:56:41.680 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.9930999999999999e-05 2023-09-23T20:56:41.680 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:41.680 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:41.680 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T20:56:41.680 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.202425+0000", 2023-09-23T20:56:41.680 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.2272e-05 2023-09-23T20:56:41.680 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:41.680 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:41.680 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T20:56:41.680 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.386530+0000", 2023-09-23T20:56:41.680 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.18410531799999999 2023-09-23T20:56:41.681 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:41.681 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:41.681 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T20:56:41.681 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.386605+0000", 2023-09-23T20:56:41.681 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.4880000000000001e-05 2023-09-23T20:56:41.681 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:41.681 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:41.681 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 2,3", 2023-09-23T20:56:41.681 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.386802+0000", 2023-09-23T20:56:41.681 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00019716199999999999 2023-09-23T20:56:41.681 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:41.681 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:41.681 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T20:56:41.682 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.392038+0000", 2023-09-23T20:56:41.682 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.005235941 2023-09-23T20:56:41.682 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:41.682 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:41.682 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:56:41.682 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.395108+0000", 2023-09-23T20:56:41.682 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0030700570000000002 2023-09-23T20:56:41.682 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:41.682 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:41.682 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:56:41.682 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.395266+0000", 2023-09-23T20:56:41.682 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00015852499999999999 2023-09-23T20:56:41.683 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:41.683 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:41.683 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T20:56:41.683 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.395302+0000", 2023-09-23T20:56:41.683 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.5374999999999999e-05 2023-09-23T20:56:41.683 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:41.683 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:41.683 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T20:56:41.683 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.395331+0000", 2023-09-23T20:56:41.683 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.8892999999999999e-05 2023-09-23T20:56:41.683 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T20:56:41.683 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T20:56:41.683 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T20:56:41.683 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:41.684 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:41.684 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.34797.0:157436 6.12 6:4be31d9d:::benchmark_data_smithi026_223868_object56310:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e150)", 2023-09-23T20:56:41.684 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T20:55:08.273211+0000", 2023-09-23T20:56:41.684 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 93.280319481000006, 2023-09-23T20:56:41.684 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.12224776599999999, 2023-09-23T20:56:41.684 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T20:56:41.684 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:56:41.684 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T20:56:41.684 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.34797", 2023-09-23T20:56:41.684 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T20:56:41.684 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 157436 2023-09-23T20:56:41.684 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:41.684 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T20:56:41.685 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:41.685 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T20:56:41.685 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.273211+0000", 2023-09-23T20:56:41.685 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T20:56:41.685 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:41.685 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:41.685 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T20:56:41.685 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.273211+0000", 2023-09-23T20:56:41.685 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T20:56:41.685 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:41.685 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:41.685 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T20:56:41.685 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.273210+0000", 2023-09-23T20:56:41.686 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.999999 2023-09-23T20:56:41.686 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:41.686 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:41.686 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T20:56:41.686 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.273214+0000", 2023-09-23T20:56:41.686 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.1080000000000003e-06 2023-09-23T20:56:41.686 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:41.686 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:41.686 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T20:56:41.686 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.273226+0000", 2023-09-23T20:56:41.686 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.168e-05 2023-09-23T20:56:41.686 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:41.686 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:41.687 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T20:56:41.687 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.273231+0000", 2023-09-23T20:56:41.687 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.6670000000000004e-06 2023-09-23T20:56:41.687 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:41.687 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:41.687 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T20:56:41.687 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.273295+0000", 2023-09-23T20:56:41.687 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.4342000000000002e-05 2023-09-23T20:56:41.687 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:41.687 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:41.687 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T20:56:41.687 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.273305+0000", 2023-09-23T20:56:41.687 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.0458e-05 2023-09-23T20:56:41.688 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:41.688 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:41.688 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T20:56:41.688 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.388109+0000", 2023-09-23T20:56:41.688 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.114803402 2023-09-23T20:56:41.688 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:41.688 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:41.688 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T20:56:41.688 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.388202+0000", 2023-09-23T20:56:41.688 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.3173999999999999e-05 2023-09-23T20:56:41.688 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:41.688 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:41.688 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 2,3", 2023-09-23T20:56:41.689 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.388429+0000", 2023-09-23T20:56:41.689 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00022721899999999999 2023-09-23T20:56:41.689 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:41.689 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:41.689 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T20:56:41.689 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.392043+0000", 2023-09-23T20:56:41.689 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0036135849999999999 2023-09-23T20:56:41.689 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:41.689 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:41.689 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:56:41.689 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.395241+0000", 2023-09-23T20:56:41.689 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0031982820000000002 2023-09-23T20:56:41.690 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:41.690 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:41.690 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:56:41.690 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.395384+0000", 2023-09-23T20:56:41.690 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00014287100000000001 2023-09-23T20:56:41.690 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:41.690 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:41.690 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T20:56:41.690 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.395420+0000", 2023-09-23T20:56:41.690 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.6174000000000003e-05 2023-09-23T20:56:41.690 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:41.690 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:41.691 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T20:56:41.691 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.395459+0000", 2023-09-23T20:56:41.691 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.8887999999999998e-05 2023-09-23T20:56:41.691 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T20:56:41.691 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T20:56:41.691 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T20:56:41.691 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:41.691 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:41.691 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.34797.0:166752 6.e 6:75b65df6:::benchmark_data_smithi026_223868_object65626:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e157)", 2023-09-23T20:56:41.691 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T20:55:20.347421+0000", 2023-09-23T20:56:41.691 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 81.206109850000004, 2023-09-23T20:56:41.691 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.074907219999999997, 2023-09-23T20:56:41.691 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T20:56:41.691 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:56:41.692 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T20:56:41.692 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.34797", 2023-09-23T20:56:41.692 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T20:56:41.692 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 166752 2023-09-23T20:56:41.692 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:41.692 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T20:56:41.692 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:41.692 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T20:56:41.692 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:20.347421+0000", 2023-09-23T20:56:41.692 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T20:56:41.692 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:41.692 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:41.693 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T20:56:41.693 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:20.347421+0000", 2023-09-23T20:56:41.693 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T20:56:41.693 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:41.693 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:41.693 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T20:56:41.693 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:20.347420+0000", 2023-09-23T20:56:41.693 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.999999 2023-09-23T20:56:41.693 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:41.693 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:41.693 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T20:56:41.693 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:20.347424+0000", 2023-09-23T20:56:41.693 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.0720000000000001e-06 2023-09-23T20:56:41.694 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:41.694 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:41.694 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T20:56:41.694 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:20.347433+0000", 2023-09-23T20:56:41.694 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.67e-06 2023-09-23T20:56:41.694 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:41.694 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:41.694 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T20:56:41.694 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:20.347438+0000", 2023-09-23T20:56:41.694 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.4729999999999999e-06 2023-09-23T20:56:41.694 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:41.694 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:41.695 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T20:56:41.695 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:20.347483+0000", 2023-09-23T20:56:41.695 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.5136999999999998e-05 2023-09-23T20:56:41.695 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:41.695 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:41.695 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T20:56:41.695 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:20.347605+0000", 2023-09-23T20:56:41.695 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000121839 2023-09-23T20:56:41.695 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:41.695 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:41.695 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 1,2", 2023-09-23T20:56:41.695 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:20.347725+0000", 2023-09-23T20:56:41.695 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00012022400000000001 2023-09-23T20:56:41.696 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:41.696 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:41.696 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T20:56:41.696 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:20.350407+0000", 2023-09-23T20:56:41.696 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.002681372 2023-09-23T20:56:41.696 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:41.696 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:41.696 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:56:41.696 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:20.355912+0000", 2023-09-23T20:56:41.696 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0055056430000000002 2023-09-23T20:56:41.696 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:41.696 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:41.696 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:56:41.697 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:20.422214+0000", 2023-09-23T20:56:41.697 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.066301386000000004 2023-09-23T20:56:41.697 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:41.697 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:41.697 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T20:56:41.697 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:20.422278+0000", 2023-09-23T20:56:41.697 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.4422000000000001e-05 2023-09-23T20:56:41.697 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:41.697 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:41.697 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T20:56:41.697 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:20.422328+0000", 2023-09-23T20:56:41.697 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.9917999999999999e-05 2023-09-23T20:56:41.697 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T20:56:41.698 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T20:56:41.698 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T20:56:41.698 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:41.698 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:41.698 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.34797.0:199468 6.2 6:40e90325:::benchmark_data_smithi026_223868_object98342:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e185)", 2023-09-23T20:56:41.698 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T20:56:09.094590+0000", 2023-09-23T20:56:41.698 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 32.458940816000002, 2023-09-23T20:56:41.698 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.079871018000000002, 2023-09-23T20:56:41.698 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T20:56:41.698 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:56:41.698 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T20:56:41.698 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.34797", 2023-09-23T20:56:41.698 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T20:56:41.699 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 199468 2023-09-23T20:56:41.699 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:41.699 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T20:56:41.699 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:41.699 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T20:56:41.699 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:56:09.094590+0000", 2023-09-23T20:56:41.699 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T20:56:41.699 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:41.699 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:41.699 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T20:56:41.699 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:56:09.094590+0000", 2023-09-23T20:56:41.699 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T20:56:41.700 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:41.700 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:41.700 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T20:56:41.700 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:56:09.094589+0000", 2023-09-23T20:56:41.700 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T20:56:41.700 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:41.700 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:41.700 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T20:56:41.700 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:56:09.094595+0000", 2023-09-23T20:56:41.700 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.3300000000000004e-06 2023-09-23T20:56:41.700 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:41.700 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:41.700 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T20:56:41.701 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:56:09.094610+0000", 2023-09-23T20:56:41.701 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.5057e-05 2023-09-23T20:56:41.701 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:41.701 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:41.701 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T20:56:41.701 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:56:09.094617+0000", 2023-09-23T20:56:41.701 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.6000000000000003e-06 2023-09-23T20:56:41.701 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:41.701 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:41.701 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T20:56:41.701 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:56:09.094663+0000", 2023-09-23T20:56:41.701 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.6746000000000003e-05 2023-09-23T20:56:41.701 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:41.702 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:41.702 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T20:56:41.702 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:56:09.094783+0000", 2023-09-23T20:56:41.702 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000119911 2023-09-23T20:56:41.702 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:41.702 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:41.702 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 2,3", 2023-09-23T20:56:41.702 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:56:09.095287+0000", 2023-09-23T20:56:41.702 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00050388699999999995 2023-09-23T20:56:41.702 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:41.702 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:41.702 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T20:56:41.703 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:56:09.099414+0000", 2023-09-23T20:56:41.703 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0041268870000000001 2023-09-23T20:56:41.703 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:41.703 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:41.703 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:56:41.703 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:56:09.106758+0000", 2023-09-23T20:56:41.703 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.007343965 2023-09-23T20:56:41.703 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:41.703 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:41.703 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:56:41.703 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:56:09.174397+0000", 2023-09-23T20:56:41.703 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.067638989999999996 2023-09-23T20:56:41.703 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:41.704 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:41.704 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T20:56:41.704 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:56:09.174431+0000", 2023-09-23T20:56:41.704 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.3859999999999998e-05 2023-09-23T20:56:41.704 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:41.704 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:41.704 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T20:56:41.704 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:56:09.174461+0000", 2023-09-23T20:56:41.704 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.0022000000000002e-05 2023-09-23T20:56:41.704 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T20:56:41.704 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T20:56:41.704 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T20:56:41.704 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T20:56:41.705 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T20:56:41.705 INFO:teuthology.orchestra.run.smithi070.stdout:} 2023-09-23T20:56:41.705 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:56:41.706 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:56:41.707 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:56:41.707 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:56:41.808 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:56:41.869 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:56:41.896 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 3 16 8581 8565 178.321 158.625 0.00565649 0.00559473 2023-09-23T20:56:41.923 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:56:41.927 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_ops_in_flight 2023-09-23T20:56:41.930 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:56:41.936 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:56:41.936 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:56:42.037 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:56:42.102 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:56:42.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:56:41 smithi026 ceph-mon[179688]: pgmap v468: 99 pgs: 99 active+clean; 27 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 4.6 MiB/s wr, 74 op/s 2023-09-23T20:56:42.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:56:41 smithi026 ceph-mon[175151]: pgmap v468: 99 pgs: 99 active+clean; 27 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 4.6 MiB/s wr, 74 op/s 2023-09-23T20:56:42.270 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:56:42.276 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:56:42.277 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:56:42.371 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:56:41 smithi070 ceph-mon[168282]: pgmap v468: 99 pgs: 99 active+clean; 27 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 4.6 MiB/s wr, 74 op/s 2023-09-23T20:56:42.378 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:56:42.413 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:56:42.455 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:56:42.458 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:56:42.458 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:56:42.559 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:56:42.597 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:56:42.638 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:56:42.642 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:56:42.647 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:56:42.647 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:56:42.748 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:56:42.807 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:56:42.879 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:56:42.888 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:56:42.888 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:56:42.896 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 4 16 10925 10909 170.356 146.5 0.00733512 0.00585809 2023-09-23T20:56:42.983 INFO:teuthology.orchestra.run.smithi026.stderr:noscrub is unset 2023-09-23T20:56:42.990 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:56:43.053 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:56:42 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/4220211468' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:56:43.053 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:56:42 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/4220211468' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:56:43.057 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:56:43.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:56:42 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/4220211468' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:56:43.332 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:56:43.340 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:56:43.341 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:56:43.441 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:56:43.481 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:56:43.607 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:56:43.612 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:56:43.612 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:56:43.707 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T20:56:43.713 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:56:43.759 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:56:43.855 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:56:43.860 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:56:43.860 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:56:43.896 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 5 16 13292 13276 165.871 147.938 0.0063459 0.00602078 2023-09-23T20:56:43.961 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:56:43.969 INFO:teuthology.orchestra.run.smithi070.stdout:{ 2023-09-23T20:56:43.969 INFO:teuthology.orchestra.run.smithi070.stdout: "ops": [ 2023-09-23T20:56:43.969 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:43.969 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.35100.0:13442 7.0 7:0a7e0ce6:::benchmark_data_smithi026_260489_object13441:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e203)", 2023-09-23T20:56:43.969 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T20:56:43.965117+0000", 2023-09-23T20:56:43.970 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 0.0043836459999999997, 2023-09-23T20:56:43.970 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.004408103, 2023-09-23T20:56:43.970 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T20:56:43.970 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "waiting for sub ops", 2023-09-23T20:56:43.970 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T20:56:43.970 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.35100", 2023-09-23T20:56:43.970 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/4108972961", 2023-09-23T20:56:43.970 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 13442 2023-09-23T20:56:43.970 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:43.970 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T20:56:43.970 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:43.970 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T20:56:43.971 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:56:43.965117+0000", 2023-09-23T20:56:43.971 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T20:56:43.971 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:43.971 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:43.971 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T20:56:43.971 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:56:43.965117+0000", 2023-09-23T20:56:43.971 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T20:56:43.971 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:43.971 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:43.971 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T20:56:43.971 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:56:43.965115+0000", 2023-09-23T20:56:43.971 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999981 2023-09-23T20:56:43.971 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:43.972 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:43.972 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T20:56:43.972 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:56:43.965167+0000", 2023-09-23T20:56:43.972 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.1836000000000003e-05 2023-09-23T20:56:43.972 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:43.972 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:43.972 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T20:56:43.972 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:56:43.965188+0000", 2023-09-23T20:56:43.972 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.0642999999999999e-05 2023-09-23T20:56:43.972 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:43.972 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:43.972 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T20:56:43.972 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:56:43.965195+0000", 2023-09-23T20:56:43.973 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.6029999999999999e-06 2023-09-23T20:56:43.973 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:43.973 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:43.973 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T20:56:43.973 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:56:43.965273+0000", 2023-09-23T20:56:43.973 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.7299000000000006e-05 2023-09-23T20:56:43.973 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:43.973 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:43.973 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T20:56:43.973 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:56:43.965416+0000", 2023-09-23T20:56:43.973 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00014319199999999999 2023-09-23T20:56:43.974 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:43.974 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:43.974 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 1,3", 2023-09-23T20:56:43.974 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:56:43.965552+0000", 2023-09-23T20:56:43.974 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00013586800000000001 2023-09-23T20:56:43.974 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:43.974 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:43.974 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T20:56:43.974 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:56:43.967345+0000", 2023-09-23T20:56:43.974 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0017935920000000001 2023-09-23T20:56:43.974 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:43.974 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:43.975 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:56:43.975 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:56:43.967381+0000", 2023-09-23T20:56:43.975 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.5120000000000003e-05 2023-09-23T20:56:43.975 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T20:56:43.975 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T20:56:43.975 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T20:56:43.975 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T20:56:43.975 INFO:teuthology.orchestra.run.smithi070.stdout: ], 2023-09-23T20:56:43.975 INFO:teuthology.orchestra.run.smithi070.stdout: "num_ops": 1 2023-09-23T20:56:43.975 INFO:teuthology.orchestra.run.smithi070.stdout:} 2023-09-23T20:56:44.038 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:56:43 smithi026 ceph-mon[179688]: pgmap v469: 99 pgs: 99 active+clean; 291 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 43 MiB/s wr, 685 op/s 2023-09-23T20:56:44.038 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:56:43 smithi026 ceph-mon[179688]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:56:44.038 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:56:43 smithi026 ceph-mon[179688]: failed to encode map e203 with expected crc 2023-09-23T20:56:44.038 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:56:43 smithi026 ceph-mon[179688]: failed to encode map e203 with expected crc 2023-09-23T20:56:44.039 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:56:43 smithi026 ceph-mon[179688]: failed to encode map e203 with expected crc 2023-09-23T20:56:44.039 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:56:43 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/4220211468' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T20:56:44.039 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:56:43 smithi026 ceph-mon[179688]: osdmap e203: 8 total, 8 up, 8 in 2023-09-23T20:56:44.039 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:56:43 smithi026 ceph-mon[175151]: pgmap v469: 99 pgs: 99 active+clean; 291 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 43 MiB/s wr, 685 op/s 2023-09-23T20:56:44.039 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:56:43 smithi026 ceph-mon[175151]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:56:44.039 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:56:43 smithi026 ceph-mon[175151]: failed to encode map e203 with expected crc 2023-09-23T20:56:44.039 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:56:43 smithi026 ceph-mon[175151]: failed to encode map e203 with expected crc 2023-09-23T20:56:44.040 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:56:43 smithi026 ceph-mon[175151]: failed to encode map e203 with expected crc 2023-09-23T20:56:44.040 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:56:43 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/4220211468' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T20:56:44.040 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:56:43 smithi026 ceph-mon[175151]: osdmap e203: 8 total, 8 up, 8 in 2023-09-23T20:56:44.055 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:56:44.104 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:56:43 smithi070 ceph-mon[168282]: pgmap v469: 99 pgs: 99 active+clean; 291 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 43 MiB/s wr, 685 op/s 2023-09-23T20:56:44.104 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:56:43 smithi070 ceph-mon[168282]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:56:44.104 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:56:43 smithi070 ceph-mon[168282]: failed to encode map e203 with expected crc 2023-09-23T20:56:44.104 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:56:43 smithi070 ceph-mon[168282]: failed to encode map e203 with expected crc 2023-09-23T20:56:44.104 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:56:43 smithi070 ceph-mon[168282]: failed to encode map e203 with expected crc 2023-09-23T20:56:44.105 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:56:43 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/4220211468' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T20:56:44.105 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:56:43 smithi070 ceph-mon[168282]: osdmap e203: 8 total, 8 up, 8 in 2023-09-23T20:56:44.123 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:56:44.128 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:56:44.129 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:56:44.229 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:56:44.291 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:56:44.353 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:56:44.357 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:56:44.358 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:56:44.454 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_blocked_ops 2023-09-23T20:56:44.458 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:56:44.522 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:56:44.584 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:56:44.591 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:56:44.592 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:56:44.609 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:56:44.693 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:56:44.765 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:56:44.877 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:56:44.885 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:56:44.886 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:56:44.896 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 6 16 15568 15552 161.925 142.25 0.00700531 0.00616753 2023-09-23T20:56:44.987 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:56:44.994 INFO:teuthology.orchestra.run.smithi026.stderr:set osd.7 primary-affinity to 0.0435413 (828532) 2023-09-23T20:56:45.056 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:56:45.131 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:56:44 smithi026 ceph-mon[175151]: 1.2 scrub starts 2023-09-23T20:56:45.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:56:44 smithi026 ceph-mon[175151]: 1.2 scrub ok 2023-09-23T20:56:45.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:56:44 smithi026 ceph-mon[175151]: 1.4 scrub starts 2023-09-23T20:56:45.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:56:44 smithi026 ceph-mon[175151]: 1.4 scrub ok 2023-09-23T20:56:45.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:56:44 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/1455245605' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 7, "weight": 0.043541329996032774}]: dispatch 2023-09-23T20:56:45.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:56:44 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 7, "weight": 0.043541329996032774}]: dispatch 2023-09-23T20:56:45.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:56:44 smithi026 ceph-mon[175151]: 1.3 scrub starts 2023-09-23T20:56:45.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:56:44 smithi026 ceph-mon[175151]: 1.3 scrub ok 2023-09-23T20:56:45.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:56:44 smithi026 ceph-mon[179688]: 1.2 scrub starts 2023-09-23T20:56:45.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:56:44 smithi026 ceph-mon[179688]: 1.2 scrub ok 2023-09-23T20:56:45.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:56:44 smithi026 ceph-mon[179688]: 1.4 scrub starts 2023-09-23T20:56:45.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:56:44 smithi026 ceph-mon[179688]: 1.4 scrub ok 2023-09-23T20:56:45.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:56:44 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/1455245605' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 7, "weight": 0.043541329996032774}]: dispatch 2023-09-23T20:56:45.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:56:44 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 7, "weight": 0.043541329996032774}]: dispatch 2023-09-23T20:56:45.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:56:44 smithi026 ceph-mon[179688]: 1.3 scrub starts 2023-09-23T20:56:45.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:56:44 smithi026 ceph-mon[179688]: 1.3 scrub ok 2023-09-23T20:56:45.172 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:56:45.269 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:56:45.271 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:56:45.272 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:56:45.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:56:44 smithi070 ceph-mon[168282]: 1.2 scrub starts 2023-09-23T20:56:45.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:56:44 smithi070 ceph-mon[168282]: 1.2 scrub ok 2023-09-23T20:56:45.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:56:44 smithi070 ceph-mon[168282]: 1.4 scrub starts 2023-09-23T20:56:45.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:56:44 smithi070 ceph-mon[168282]: 1.4 scrub ok 2023-09-23T20:56:45.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:56:44 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/1455245605' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 7, "weight": 0.043541329996032774}]: dispatch 2023-09-23T20:56:45.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:56:44 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 7, "weight": 0.043541329996032774}]: dispatch 2023-09-23T20:56:45.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:56:44 smithi070 ceph-mon[168282]: 1.3 scrub starts 2023-09-23T20:56:45.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:56:44 smithi070 ceph-mon[168282]: 1.3 scrub ok 2023-09-23T20:56:45.372 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:56:45.400 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:56:45.481 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T20:56:45.897 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 7 16 16050 16034 143.101 30.125 0.00157408 0.00615079 2023-09-23T20:56:46.252 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:56:45 smithi070 ceph-mon[168282]: pgmap v471: 99 pgs: 99 active+clean; 642 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 82 MiB/s wr, 1.31k op/s 2023-09-23T20:56:46.252 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:56:45 smithi070 ceph-mon[168282]: 1.a scrub starts 2023-09-23T20:56:46.252 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:56:45 smithi070 ceph-mon[168282]: 1.a scrub ok 2023-09-23T20:56:46.252 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:56:45 smithi070 ceph-mon[168282]: failed to encode map e204 with expected crc 2023-09-23T20:56:46.252 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:56:45 smithi070 ceph-mon[168282]: failed to encode map e204 with expected crc 2023-09-23T20:56:46.252 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:56:45 smithi070 ceph-mon[168282]: failed to encode map e204 with expected crc 2023-09-23T20:56:46.252 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:56:45 smithi070 ceph-mon[168282]: failed to encode map e204 with expected crc 2023-09-23T20:56:46.253 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:56:45 smithi070 ceph-mon[168282]: failed to encode map e204 with expected crc 2023-09-23T20:56:46.253 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:56:45 smithi070 ceph-mon[168282]: failed to encode map e204 with expected crc 2023-09-23T20:56:46.253 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:56:45 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 7, "weight": 0.043541329996032774}]': finished 2023-09-23T20:56:46.253 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:56:45 smithi070 ceph-mon[168282]: osdmap e204: 8 total, 8 up, 8 in 2023-09-23T20:56:46.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:56:45 smithi026 ceph-mon[175151]: pgmap v471: 99 pgs: 99 active+clean; 642 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 82 MiB/s wr, 1.31k op/s 2023-09-23T20:56:46.383 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:56:45 smithi026 ceph-mon[175151]: 1.a scrub starts 2023-09-23T20:56:46.383 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:56:45 smithi026 ceph-mon[175151]: 1.a scrub ok 2023-09-23T20:56:46.383 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:56:45 smithi026 ceph-mon[175151]: failed to encode map e204 with expected crc 2023-09-23T20:56:46.383 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:56:45 smithi026 ceph-mon[175151]: failed to encode map e204 with expected crc 2023-09-23T20:56:46.383 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:56:45 smithi026 ceph-mon[175151]: failed to encode map e204 with expected crc 2023-09-23T20:56:46.383 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:56:45 smithi026 ceph-mon[175151]: failed to encode map e204 with expected crc 2023-09-23T20:56:46.383 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:56:45 smithi026 ceph-mon[175151]: failed to encode map e204 with expected crc 2023-09-23T20:56:46.383 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:56:45 smithi026 ceph-mon[175151]: failed to encode map e204 with expected crc 2023-09-23T20:56:46.383 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:56:45 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 7, "weight": 0.043541329996032774}]': finished 2023-09-23T20:56:46.383 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:56:45 smithi026 ceph-mon[175151]: osdmap e204: 8 total, 8 up, 8 in 2023-09-23T20:56:46.384 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:56:45 smithi026 ceph-mon[179688]: pgmap v471: 99 pgs: 99 active+clean; 642 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 82 MiB/s wr, 1.31k op/s 2023-09-23T20:56:46.384 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:56:45 smithi026 ceph-mon[179688]: 1.a scrub starts 2023-09-23T20:56:46.384 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:56:45 smithi026 ceph-mon[179688]: 1.a scrub ok 2023-09-23T20:56:46.384 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:56:45 smithi026 ceph-mon[179688]: failed to encode map e204 with expected crc 2023-09-23T20:56:46.384 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:56:45 smithi026 ceph-mon[179688]: failed to encode map e204 with expected crc 2023-09-23T20:56:46.384 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:56:45 smithi026 ceph-mon[179688]: failed to encode map e204 with expected crc 2023-09-23T20:56:46.384 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:56:45 smithi026 ceph-mon[179688]: failed to encode map e204 with expected crc 2023-09-23T20:56:46.384 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:56:45 smithi026 ceph-mon[179688]: failed to encode map e204 with expected crc 2023-09-23T20:56:46.384 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:56:45 smithi026 ceph-mon[179688]: failed to encode map e204 with expected crc 2023-09-23T20:56:46.384 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:56:45 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 7, "weight": 0.043541329996032774}]': finished 2023-09-23T20:56:46.385 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:56:45 smithi026 ceph-mon[179688]: osdmap e204: 8 total, 8 up, 8 in 2023-09-23T20:56:46.479 INFO:teuthology.orchestra.run.smithi070.stdout:{ 2023-09-23T20:56:46.479 INFO:teuthology.orchestra.run.smithi070.stdout: "ops": [], 2023-09-23T20:56:46.479 INFO:teuthology.orchestra.run.smithi070.stdout: "complaint_time": 30, 2023-09-23T20:56:46.479 INFO:teuthology.orchestra.run.smithi070.stdout: "num_blocked_ops": 0 2023-09-23T20:56:46.479 INFO:teuthology.orchestra.run.smithi070.stdout:} 2023-09-23T20:56:46.786 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:56:46.793 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:56:46.793 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:56:46.836 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_historic_ops 2023-09-23T20:56:46.894 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:56:46.900 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 8 16 18253 18237 142.351 137.688 0.00546355 0.00701782 2023-09-23T20:56:46.946 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:56:47.001 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:56:47.006 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:56:47.006 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:56:47.107 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:56:47.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:56:46 smithi070 ceph-mon[168282]: failed to encode map e205 with expected crc 2023-09-23T20:56:47.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:56:46 smithi070 ceph-mon[168282]: failed to encode map e205 with expected crc 2023-09-23T20:56:47.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:56:46 smithi070 ceph-mon[168282]: failed to encode map e205 with expected crc 2023-09-23T20:56:47.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:56:46 smithi070 ceph-mon[168282]: failed to encode map e205 with expected crc 2023-09-23T20:56:47.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:56:46 smithi070 ceph-mon[168282]: osdmap e205: 8 total, 8 up, 8 in 2023-09-23T20:56:47.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:56:46 smithi070 ceph-mon[168282]: failed to encode map e205 with expected crc 2023-09-23T20:56:47.121 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:56:46 smithi070 ceph-mon[168282]: failed to encode map e205 with expected crc 2023-09-23T20:56:47.121 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:56:46 smithi070 ceph-mon[168282]: failed to encode map e205 with expected crc 2023-09-23T20:56:47.121 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:56:46 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:56:47.121 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:56:46 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:56:47.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:56:46 smithi026 ceph-mon[179688]: failed to encode map e205 with expected crc 2023-09-23T20:56:47.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:56:46 smithi026 ceph-mon[179688]: failed to encode map e205 with expected crc 2023-09-23T20:56:47.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:56:46 smithi026 ceph-mon[179688]: failed to encode map e205 with expected crc 2023-09-23T20:56:47.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:56:46 smithi026 ceph-mon[179688]: failed to encode map e205 with expected crc 2023-09-23T20:56:47.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:56:46 smithi026 ceph-mon[179688]: osdmap e205: 8 total, 8 up, 8 in 2023-09-23T20:56:47.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:56:46 smithi026 ceph-mon[179688]: failed to encode map e205 with expected crc 2023-09-23T20:56:47.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:56:46 smithi026 ceph-mon[179688]: failed to encode map e205 with expected crc 2023-09-23T20:56:47.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:56:46 smithi026 ceph-mon[179688]: failed to encode map e205 with expected crc 2023-09-23T20:56:47.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:56:46 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:56:47.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:56:46 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:56:47.134 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:56:46 smithi026 ceph-mon[175151]: failed to encode map e205 with expected crc 2023-09-23T20:56:47.134 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:56:46 smithi026 ceph-mon[175151]: failed to encode map e205 with expected crc 2023-09-23T20:56:47.134 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:56:46 smithi026 ceph-mon[175151]: failed to encode map e205 with expected crc 2023-09-23T20:56:47.134 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:56:46 smithi026 ceph-mon[175151]: failed to encode map e205 with expected crc 2023-09-23T20:56:47.134 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:56:46 smithi026 ceph-mon[175151]: osdmap e205: 8 total, 8 up, 8 in 2023-09-23T20:56:47.135 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:56:46 smithi026 ceph-mon[175151]: failed to encode map e205 with expected crc 2023-09-23T20:56:47.135 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:56:46 smithi026 ceph-mon[175151]: failed to encode map e205 with expected crc 2023-09-23T20:56:47.135 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:56:46 smithi026 ceph-mon[175151]: failed to encode map e205 with expected crc 2023-09-23T20:56:47.135 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:56:46 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:56:47.135 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:56:46 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:56:47.158 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:56:47.206 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:56:47.210 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:56:47.211 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:56:47.311 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:56:47.356 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:56:47.404 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:56:47.408 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:56:47.409 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:56:47.510 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:56:47.548 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:56:47.592 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:56:47.596 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:56:47.603 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:56:47.604 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:56:47.705 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:56:47.761 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:56:47.901 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 9 16 20582 20566 142.698 145.562 0.00769342 0.00700188 2023-09-23T20:56:48.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:56:47 smithi070 ceph-mon[168282]: pgmap v474: 99 pgs: 99 active+clean; 793 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 128 MiB/s wr, 2.04k op/s 2023-09-23T20:56:48.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:56:47 smithi070 ceph-mon[168282]: 1.9 scrub starts 2023-09-23T20:56:48.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:56:47 smithi070 ceph-mon[168282]: 1.9 scrub ok 2023-09-23T20:56:48.131 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:56:47 smithi026 ceph-mon[175151]: pgmap v474: 99 pgs: 99 active+clean; 793 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 128 MiB/s wr, 2.04k op/s 2023-09-23T20:56:48.131 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:56:47 smithi026 ceph-mon[175151]: 1.9 scrub starts 2023-09-23T20:56:48.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:56:47 smithi026 ceph-mon[175151]: 1.9 scrub ok 2023-09-23T20:56:48.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:56:47 smithi026 ceph-mon[179688]: pgmap v474: 99 pgs: 99 active+clean; 793 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 128 MiB/s wr, 2.04k op/s 2023-09-23T20:56:48.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:56:47 smithi026 ceph-mon[179688]: 1.9 scrub starts 2023-09-23T20:56:48.134 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:56:47 smithi026 ceph-mon[179688]: 1.9 scrub ok 2023-09-23T20:56:48.723 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:56:48.732 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:56:48.732 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:56:48.833 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:56:48.870 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:56:48.890 INFO:teuthology.orchestra.run.smithi070.stdout:{ 2023-09-23T20:56:48.890 INFO:teuthology.orchestra.run.smithi070.stdout: "size": 20, 2023-09-23T20:56:48.890 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 600, 2023-09-23T20:56:48.890 INFO:teuthology.orchestra.run.smithi070.stdout: "ops": [ 2023-09-23T20:56:48.890 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:48.890 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.34797.0:130474 6.d 6.9b7cf5d (undecoded) ondisk+retry+write+known_if_redirected+supports_pool_eio e136)", 2023-09-23T20:56:48.890 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T20:54:30.903072+0000", 2023-09-23T20:56:48.890 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 137.98359524400001, 2023-09-23T20:56:48.891 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.97617976500000003, 2023-09-23T20:56:48.891 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T20:56:48.891 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "reached pg", 2023-09-23T20:56:48.891 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T20:56:48.891 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.34797", 2023-09-23T20:56:48.891 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T20:56:48.891 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 130474 2023-09-23T20:56:48.891 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:48.891 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T20:56:48.892 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:48.892 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T20:56:48.892 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:54:30.903072+0000", 2023-09-23T20:56:48.892 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T20:56:48.892 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:48.892 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:48.892 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T20:56:48.892 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:54:30.903072+0000", 2023-09-23T20:56:48.892 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T20:56:48.892 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:48.892 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:48.893 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T20:56:48.893 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:54:30.903071+0000", 2023-09-23T20:56:48.893 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.999999 2023-09-23T20:56:48.893 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:48.893 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:48.893 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T20:56:48.893 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:54:30.903074+0000", 2023-09-23T20:56:48.893 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.7100000000000001e-06 2023-09-23T20:56:48.893 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:48.893 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:48.894 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T20:56:48.894 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:54:30.903083+0000", 2023-09-23T20:56:48.894 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.3240000000000003e-06 2023-09-23T20:56:48.894 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:48.894 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:48.894 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T20:56:48.894 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:54:30.903088+0000", 2023-09-23T20:56:48.894 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.4280000000000004e-06 2023-09-23T20:56:48.894 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:48.894 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:48.895 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T20:56:48.895 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:54:30.903139+0000", 2023-09-23T20:56:48.895 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.0769000000000001e-05 2023-09-23T20:56:48.895 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:48.895 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:48.895 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T20:56:48.895 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:54:30.903144+0000", 2023-09-23T20:56:48.895 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.2320000000000001e-06 2023-09-23T20:56:48.895 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:48.895 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:48.896 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T20:56:48.896 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:54:31.879231+0000", 2023-09-23T20:56:48.896 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.97608724099999999 2023-09-23T20:56:48.896 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:48.896 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:48.896 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T20:56:48.896 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:54:31.879251+0000", 2023-09-23T20:56:48.896 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.0156000000000001e-05 2023-09-23T20:56:48.896 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T20:56:48.896 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T20:56:48.897 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T20:56:48.897 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:48.897 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:48.897 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.34797.0:167244 6.a 6:5fa8594b:::benchmark_data_smithi026_223868_object66118:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e158)", 2023-09-23T20:56:48.897 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T20:55:20.930710+0000", 2023-09-23T20:56:48.897 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 87.955956861999994, 2023-09-23T20:56:48.897 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.0357931300000001, 2023-09-23T20:56:48.897 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T20:56:48.897 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:56:48.897 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T20:56:48.898 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.34797", 2023-09-23T20:56:48.898 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T20:56:48.898 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 167244 2023-09-23T20:56:48.898 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:48.898 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T20:56:48.898 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:48.898 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T20:56:48.898 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:20.930710+0000", 2023-09-23T20:56:48.898 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T20:56:48.898 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:48.899 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:48.899 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T20:56:48.899 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:20.930710+0000", 2023-09-23T20:56:48.899 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T20:56:48.899 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:48.899 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:48.899 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T20:56:48.899 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:20.930709+0000", 2023-09-23T20:56:48.899 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.999999 2023-09-23T20:56:48.899 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:48.899 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:48.900 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T20:56:48.900 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:20.930716+0000", 2023-09-23T20:56:48.900 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.2579999999999998e-06 2023-09-23T20:56:48.900 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:48.900 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:48.900 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T20:56:48.900 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:20.930726+0000", 2023-09-23T20:56:48.900 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.0087e-05 2023-09-23T20:56:48.900 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:48.900 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:48.901 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T20:56:48.901 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:20.930731+0000", 2023-09-23T20:56:48.901 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.7140000000000001e-06 2023-09-23T20:56:48.901 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:48.901 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:48.901 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T20:56:48.901 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:20.930770+0000", 2023-09-23T20:56:48.901 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.9199999999999997e-05 2023-09-23T20:56:48.901 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:48.901 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:48.902 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T20:56:48.902 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:20.930779+0000", 2023-09-23T20:56:48.902 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.3380000000000005e-06 2023-09-23T20:56:48.902 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:48.902 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:48.902 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T20:56:48.902 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:21.956800+0000", 2023-09-23T20:56:48.902 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.026021538 2023-09-23T20:56:48.902 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:48.902 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:48.902 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T20:56:48.903 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:21.956900+0000", 2023-09-23T20:56:48.903 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.9709000000000004e-05 2023-09-23T20:56:48.903 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:48.903 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:48.903 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 0,2", 2023-09-23T20:56:48.903 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:21.957010+0000", 2023-09-23T20:56:48.903 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000110504 2023-09-23T20:56:48.903 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:48.903 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:48.903 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T20:56:48.904 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:21.959547+0000", 2023-09-23T20:56:48.904 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0025366799999999999 2023-09-23T20:56:48.904 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:48.904 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:48.904 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:56:48.904 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:21.966355+0000", 2023-09-23T20:56:48.904 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.006807848 2023-09-23T20:56:48.904 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:48.904 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:48.904 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:56:48.905 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:21.966407+0000", 2023-09-23T20:56:48.905 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.1666999999999998e-05 2023-09-23T20:56:48.905 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:48.905 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:48.905 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T20:56:48.905 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:21.966442+0000", 2023-09-23T20:56:48.905 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.5602000000000002e-05 2023-09-23T20:56:48.905 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:48.905 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:48.905 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T20:56:48.906 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:21.966503+0000", 2023-09-23T20:56:48.906 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.1019e-05 2023-09-23T20:56:48.906 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T20:56:48.906 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T20:56:48.906 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T20:56:48.906 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:48.906 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:48.906 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.34797.0:167247 6.a 6:5b80d451:::benchmark_data_smithi026_223868_object66121:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e158)", 2023-09-23T20:56:48.906 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T20:55:20.930743+0000", 2023-09-23T20:56:48.907 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 87.955923815999995, 2023-09-23T20:56:48.907 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.0409182340000001, 2023-09-23T20:56:48.907 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T20:56:48.907 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:56:48.907 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T20:56:48.907 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.34797", 2023-09-23T20:56:48.907 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T20:56:48.907 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 167247 2023-09-23T20:56:48.907 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:48.907 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T20:56:48.907 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:48.907 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T20:56:48.908 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:20.930743+0000", 2023-09-23T20:56:48.908 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T20:56:48.908 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:48.908 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:48.908 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T20:56:48.908 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:20.930743+0000", 2023-09-23T20:56:48.908 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T20:56:48.908 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:48.908 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:48.908 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T20:56:48.908 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:20.930742+0000", 2023-09-23T20:56:48.909 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999995 2023-09-23T20:56:48.909 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:48.909 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:48.909 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T20:56:48.909 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:20.930746+0000", 2023-09-23T20:56:48.909 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.1879999999999998e-06 2023-09-23T20:56:48.909 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:48.909 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:48.909 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T20:56:48.909 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:20.930752+0000", 2023-09-23T20:56:48.909 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.0379999999999996e-06 2023-09-23T20:56:48.910 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:48.910 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:48.910 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T20:56:48.910 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:20.930756+0000", 2023-09-23T20:56:48.910 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.1030000000000004e-06 2023-09-23T20:56:48.910 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:48.910 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:48.910 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T20:56:48.910 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:20.930808+0000", 2023-09-23T20:56:48.910 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.2305999999999997e-05 2023-09-23T20:56:48.911 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:48.911 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:48.911 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T20:56:48.911 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:20.930813+0000", 2023-09-23T20:56:48.911 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.9910000000000002e-06 2023-09-23T20:56:48.911 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:48.911 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:48.911 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T20:56:48.911 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:21.959580+0000", 2023-09-23T20:56:48.911 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.0287672910000001 2023-09-23T20:56:48.911 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:48.912 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:48.912 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T20:56:48.912 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:21.959671+0000", 2023-09-23T20:56:48.912 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.0928999999999996e-05 2023-09-23T20:56:48.912 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:48.912 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:48.912 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 0,2", 2023-09-23T20:56:48.912 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:21.959739+0000", 2023-09-23T20:56:48.912 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.7663000000000006e-05 2023-09-23T20:56:48.912 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:48.912 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:48.912 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T20:56:48.913 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:21.969123+0000", 2023-09-23T20:56:48.913 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0093844800000000006 2023-09-23T20:56:48.913 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:48.913 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:48.913 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:56:48.913 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:21.971537+0000", 2023-09-23T20:56:48.913 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.002413157 2023-09-23T20:56:48.913 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:48.913 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:48.913 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:56:48.913 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:21.971574+0000", 2023-09-23T20:56:48.914 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.7681000000000001e-05 2023-09-23T20:56:48.914 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:48.914 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:48.914 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T20:56:48.914 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:21.971604+0000", 2023-09-23T20:56:48.914 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.0176999999999999e-05 2023-09-23T20:56:48.914 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:48.914 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:48.914 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T20:56:48.914 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:21.971661+0000", 2023-09-23T20:56:48.915 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.6857999999999997e-05 2023-09-23T20:56:48.915 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T20:56:48.915 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T20:56:48.915 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T20:56:48.915 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:48.915 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:48.915 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.34797.0:167248 6.a 6:59b345d1:::benchmark_data_smithi026_223868_object66122:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e158)", 2023-09-23T20:56:48.915 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T20:55:20.930761+0000", 2023-09-23T20:56:48.915 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 87.955906236000004, 2023-09-23T20:56:48.915 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.055093563, 2023-09-23T20:56:48.915 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T20:56:48.916 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:56:48.916 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T20:56:48.916 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.34797", 2023-09-23T20:56:48.916 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T20:56:48.916 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 167248 2023-09-23T20:56:48.916 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:48.916 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T20:56:48.916 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:48.916 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T20:56:48.916 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:20.930761+0000", 2023-09-23T20:56:48.916 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T20:56:48.917 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:48.917 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:48.917 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T20:56:48.917 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:20.930761+0000", 2023-09-23T20:56:48.917 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T20:56:48.917 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:48.917 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:48.917 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T20:56:48.917 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:20.930760+0000", 2023-09-23T20:56:48.917 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999995 2023-09-23T20:56:48.917 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:48.918 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:48.918 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T20:56:48.918 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:20.930762+0000", 2023-09-23T20:56:48.918 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.1119999999999999e-06 2023-09-23T20:56:48.918 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:48.918 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:48.918 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T20:56:48.918 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:20.930768+0000", 2023-09-23T20:56:48.918 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.4539999999999998e-06 2023-09-23T20:56:48.918 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:48.918 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:48.918 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T20:56:48.919 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:20.930771+0000", 2023-09-23T20:56:48.919 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.5899999999999999e-06 2023-09-23T20:56:48.919 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:48.919 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:48.919 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T20:56:48.919 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:20.930827+0000", 2023-09-23T20:56:48.919 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.5949000000000003e-05 2023-09-23T20:56:48.919 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:48.919 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:48.919 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T20:56:48.919 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:20.930831+0000", 2023-09-23T20:56:48.919 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.4199999999999999e-06 2023-09-23T20:56:48.920 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:48.920 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:48.920 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T20:56:48.920 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:21.961980+0000", 2023-09-23T20:56:48.920 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.0311488959999999 2023-09-23T20:56:48.920 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:48.920 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:48.920 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T20:56:48.920 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:21.962078+0000", 2023-09-23T20:56:48.920 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.8317000000000001e-05 2023-09-23T20:56:48.920 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:48.920 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:48.921 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 0,2", 2023-09-23T20:56:48.921 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:21.962194+0000", 2023-09-23T20:56:48.921 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000115528 2023-09-23T20:56:48.921 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:48.921 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:48.921 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T20:56:48.921 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:21.969134+0000", 2023-09-23T20:56:48.921 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0069402370000000001 2023-09-23T20:56:48.921 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:48.921 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:48.921 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:56:48.921 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:21.976008+0000", 2023-09-23T20:56:48.922 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.006874326 2023-09-23T20:56:48.922 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:48.922 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:48.922 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:56:48.922 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:21.985762+0000", 2023-09-23T20:56:48.922 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0097536199999999993 2023-09-23T20:56:48.922 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:48.922 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:48.922 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T20:56:48.922 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:21.985793+0000", 2023-09-23T20:56:48.922 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.1628999999999998e-05 2023-09-23T20:56:48.922 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:48.923 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:48.923 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T20:56:48.923 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:21.985854+0000", 2023-09-23T20:56:48.923 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.0838000000000003e-05 2023-09-23T20:56:48.923 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T20:56:48.923 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T20:56:48.923 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T20:56:48.923 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:48.923 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:48.923 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.34797.0:167255 6.a 6:574da207:::benchmark_data_smithi026_223868_object66129:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e158)", 2023-09-23T20:56:48.923 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T20:55:20.936722+0000", 2023-09-23T20:56:48.923 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 87.949944967999997, 2023-09-23T20:56:48.924 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.053378076, 2023-09-23T20:56:48.924 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T20:56:48.924 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:56:48.924 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T20:56:48.924 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.34797", 2023-09-23T20:56:48.924 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T20:56:48.924 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 167255 2023-09-23T20:56:48.924 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:48.924 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T20:56:48.924 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:48.924 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T20:56:48.924 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:20.936722+0000", 2023-09-23T20:56:48.925 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T20:56:48.925 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:48.925 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:48.925 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T20:56:48.925 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:20.936722+0000", 2023-09-23T20:56:48.925 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T20:56:48.925 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:48.925 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:48.925 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T20:56:48.925 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:20.936721+0000", 2023-09-23T20:56:48.925 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.999999 2023-09-23T20:56:48.926 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:48.926 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:48.926 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T20:56:48.926 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:20.936725+0000", 2023-09-23T20:56:48.926 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.3150000000000002e-06 2023-09-23T20:56:48.926 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:48.926 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:48.926 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T20:56:48.926 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:20.936734+0000", 2023-09-23T20:56:48.926 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.2199999999999998e-06 2023-09-23T20:56:48.927 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:48.927 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:48.927 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T20:56:48.927 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:20.936739+0000", 2023-09-23T20:56:48.927 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.4309999999999996e-06 2023-09-23T20:56:48.927 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:48.927 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:48.927 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T20:56:48.927 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:20.936768+0000", 2023-09-23T20:56:48.927 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.9567999999999999e-05 2023-09-23T20:56:48.927 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:48.928 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:48.928 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T20:56:48.928 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:20.936774+0000", 2023-09-23T20:56:48.928 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.485e-06 2023-09-23T20:56:48.928 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:48.928 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:48.928 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T20:56:48.928 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:21.964309+0000", 2023-09-23T20:56:48.928 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.027534704 2023-09-23T20:56:48.928 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:48.928 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:48.928 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T20:56:48.929 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:21.964430+0000", 2023-09-23T20:56:48.929 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00012104000000000001 2023-09-23T20:56:48.929 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:48.929 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:48.929 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 0,2", 2023-09-23T20:56:48.929 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:21.964551+0000", 2023-09-23T20:56:48.929 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00012153 2023-09-23T20:56:48.929 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:48.929 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:48.929 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T20:56:48.929 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:21.971489+0000", 2023-09-23T20:56:48.930 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0069378219999999997 2023-09-23T20:56:48.930 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:48.930 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:48.930 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:56:48.930 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:21.981600+0000", 2023-09-23T20:56:48.930 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.010111063 2023-09-23T20:56:48.930 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:48.930 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:48.930 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:56:48.930 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:21.989954+0000", 2023-09-23T20:56:48.930 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0083543060000000006 2023-09-23T20:56:48.931 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:48.931 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:48.931 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T20:56:48.931 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:21.990003+0000", 2023-09-23T20:56:48.931 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.8662999999999998e-05 2023-09-23T20:56:48.931 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:48.931 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:48.931 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T20:56:48.931 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:21.990100+0000", 2023-09-23T20:56:48.931 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.6912999999999999e-05 2023-09-23T20:56:48.932 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T20:56:48.932 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T20:56:48.932 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T20:56:48.932 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:48.932 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:48.932 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.34797.0:167281 6.a 6:5de3ccd5:::benchmark_data_smithi026_223868_object66155:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e158)", 2023-09-23T20:56:48.932 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T20:55:20.963939+0000", 2023-09-23T20:56:48.932 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 87.922727553000001, 2023-09-23T20:56:48.932 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.026322833, 2023-09-23T20:56:48.932 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T20:56:48.932 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:56:48.933 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T20:56:48.933 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.34797", 2023-09-23T20:56:48.933 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T20:56:48.933 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 167281 2023-09-23T20:56:48.933 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:48.933 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T20:56:48.933 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:48.933 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T20:56:48.933 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:20.963939+0000", 2023-09-23T20:56:48.933 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T20:56:48.933 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:48.933 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:48.933 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T20:56:48.934 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:20.963939+0000", 2023-09-23T20:56:48.934 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T20:56:48.934 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:48.934 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:48.934 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T20:56:48.934 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:20.963938+0000", 2023-09-23T20:56:48.934 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T20:56:48.934 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:48.934 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:48.934 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T20:56:48.934 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:20.963943+0000", 2023-09-23T20:56:48.934 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.2739999999999998e-06 2023-09-23T20:56:48.935 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:48.935 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:48.935 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T20:56:48.935 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:20.963958+0000", 2023-09-23T20:56:48.935 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.5096000000000001e-05 2023-09-23T20:56:48.935 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:48.935 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:48.935 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T20:56:48.935 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:20.963966+0000", 2023-09-23T20:56:48.935 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.5140000000000004e-06 2023-09-23T20:56:48.935 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:48.935 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:48.936 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T20:56:48.936 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:20.964006+0000", 2023-09-23T20:56:48.936 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.0504999999999999e-05 2023-09-23T20:56:48.936 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:48.936 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:48.936 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T20:56:48.936 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:20.964015+0000", 2023-09-23T20:56:48.936 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.8070000000000006e-06 2023-09-23T20:56:48.936 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:48.936 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:48.936 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T20:56:48.936 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:21.966539+0000", 2023-09-23T20:56:48.937 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.002524204 2023-09-23T20:56:48.937 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:48.937 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:48.937 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T20:56:48.937 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:21.966674+0000", 2023-09-23T20:56:48.937 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00013492699999999999 2023-09-23T20:56:48.937 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:48.937 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:48.937 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 0,2", 2023-09-23T20:56:48.937 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:21.967167+0000", 2023-09-23T20:56:48.937 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000492549 2023-09-23T20:56:48.938 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:48.938 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:48.938 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T20:56:48.938 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:21.971671+0000", 2023-09-23T20:56:48.938 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0045045529999999997 2023-09-23T20:56:48.938 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:48.938 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:48.938 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:56:48.938 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:21.981643+0000", 2023-09-23T20:56:48.938 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0099714829999999997 2023-09-23T20:56:48.939 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:48.939 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:48.939 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:56:48.939 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:21.990135+0000", 2023-09-23T20:56:48.939 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0084919009999999996 2023-09-23T20:56:48.939 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:48.939 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:48.939 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T20:56:48.939 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:21.990164+0000", 2023-09-23T20:56:48.939 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.9549000000000001e-05 2023-09-23T20:56:48.939 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:48.939 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:48.940 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T20:56:48.940 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:21.990262+0000", 2023-09-23T20:56:48.940 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.781e-05 2023-09-23T20:56:48.940 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T20:56:48.940 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T20:56:48.940 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T20:56:48.940 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:48.940 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:48.940 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.34797.0:167305 6.a 6:5183351c:::benchmark_data_smithi026_223868_object66179:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e158)", 2023-09-23T20:56:48.940 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T20:55:20.991170+0000", 2023-09-23T20:56:48.940 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 87.895496397000002, 2023-09-23T20:56:48.941 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.999303524, 2023-09-23T20:56:48.941 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T20:56:48.941 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:56:48.941 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T20:56:48.941 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.34797", 2023-09-23T20:56:48.941 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T20:56:48.941 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 167305 2023-09-23T20:56:48.941 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:48.941 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T20:56:48.941 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:48.941 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T20:56:48.942 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:20.991170+0000", 2023-09-23T20:56:48.942 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T20:56:48.942 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:48.942 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:48.942 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T20:56:48.942 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:20.991170+0000", 2023-09-23T20:56:48.942 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T20:56:48.942 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:48.942 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:48.942 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T20:56:48.942 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:20.991169+0000", 2023-09-23T20:56:48.943 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T20:56:48.943 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:48.943 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:48.943 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T20:56:48.943 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:20.991176+0000", 2023-09-23T20:56:48.943 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.2219999999999996e-06 2023-09-23T20:56:48.943 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:48.943 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:48.943 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T20:56:48.943 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:20.991192+0000", 2023-09-23T20:56:48.944 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.5798e-05 2023-09-23T20:56:48.944 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:48.944 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:48.944 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T20:56:48.944 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:20.991199+0000", 2023-09-23T20:56:48.944 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.516e-06 2023-09-23T20:56:48.944 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:48.944 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:48.944 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T20:56:48.944 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:20.991264+0000", 2023-09-23T20:56:48.944 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.5135000000000006e-05 2023-09-23T20:56:48.944 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:48.945 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:48.945 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T20:56:48.945 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:20.991272+0000", 2023-09-23T20:56:48.945 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.7250000000000003e-06 2023-09-23T20:56:48.945 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:48.945 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:48.945 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T20:56:48.945 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:21.969162+0000", 2023-09-23T20:56:48.945 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.97788947999999998 2023-09-23T20:56:48.945 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:48.945 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:48.945 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T20:56:48.946 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:21.969309+0000", 2023-09-23T20:56:48.946 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00014732499999999999 2023-09-23T20:56:48.946 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:48.946 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:48.946 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 0,2", 2023-09-23T20:56:48.946 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:21.969431+0000", 2023-09-23T20:56:48.946 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000121797 2023-09-23T20:56:48.946 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:48.946 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:48.946 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T20:56:48.946 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:21.976014+0000", 2023-09-23T20:56:48.947 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0065832219999999997 2023-09-23T20:56:48.947 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:48.947 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:48.947 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:56:48.947 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:21.985730+0000", 2023-09-23T20:56:48.947 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0097158490000000004 2023-09-23T20:56:48.947 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:48.947 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:48.947 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:56:48.947 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:21.990318+0000", 2023-09-23T20:56:48.947 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0045880269999999997 2023-09-23T20:56:48.947 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:48.948 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:48.948 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T20:56:48.948 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:21.990372+0000", 2023-09-23T20:56:48.948 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.3658e-05 2023-09-23T20:56:48.948 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:48.948 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:48.948 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T20:56:48.948 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:21.990474+0000", 2023-09-23T20:56:48.948 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00010228799999999999 2023-09-23T20:56:48.948 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T20:56:48.948 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T20:56:48.948 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T20:56:48.948 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:48.949 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:48.949 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.34797.0:167315 6.a 6:5232d899:::benchmark_data_smithi026_223868_object66189:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e158)", 2023-09-23T20:56:48.949 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T20:55:21.002335+0000", 2023-09-23T20:56:48.949 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 87.884331668000002, 2023-09-23T20:56:48.949 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.988365296, 2023-09-23T20:56:48.949 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T20:56:48.949 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:56:48.949 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T20:56:48.949 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.34797", 2023-09-23T20:56:48.949 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T20:56:48.949 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 167315 2023-09-23T20:56:48.949 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:48.950 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T20:56:48.950 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:48.950 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T20:56:48.950 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:21.002335+0000", 2023-09-23T20:56:48.950 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T20:56:48.950 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:48.950 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:48.950 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T20:56:48.950 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:21.002335+0000", 2023-09-23T20:56:48.950 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T20:56:48.951 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:48.951 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:48.951 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T20:56:48.951 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:21.002333+0000", 2023-09-23T20:56:48.951 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999981 2023-09-23T20:56:48.951 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:48.951 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:48.951 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T20:56:48.951 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:21.002342+0000", 2023-09-23T20:56:48.951 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.0149999999999992e-06 2023-09-23T20:56:48.951 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:48.951 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:48.952 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T20:56:48.952 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:21.002357+0000", 2023-09-23T20:56:48.952 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.4859e-05 2023-09-23T20:56:48.952 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:48.952 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:48.952 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T20:56:48.952 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:21.002366+0000", 2023-09-23T20:56:48.952 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.7099999999999996e-06 2023-09-23T20:56:48.952 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:48.952 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:48.952 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T20:56:48.952 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:21.002492+0000", 2023-09-23T20:56:48.953 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00012614200000000001 2023-09-23T20:56:48.953 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:48.953 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:48.953 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T20:56:48.953 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:21.002502+0000", 2023-09-23T20:56:48.953 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.0079e-05 2023-09-23T20:56:48.953 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:48.953 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:48.953 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T20:56:48.953 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:21.971710+0000", 2023-09-23T20:56:48.953 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.96920738699999998 2023-09-23T20:56:48.953 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:48.953 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:48.954 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T20:56:48.954 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:21.971831+0000", 2023-09-23T20:56:48.954 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00012119499999999999 2023-09-23T20:56:48.954 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:48.954 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:48.954 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 0,2", 2023-09-23T20:56:48.954 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:21.971946+0000", 2023-09-23T20:56:48.954 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000115135 2023-09-23T20:56:48.954 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:48.954 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:48.954 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T20:56:48.954 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:21.979629+0000", 2023-09-23T20:56:48.955 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0076829059999999998 2023-09-23T20:56:48.955 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:48.955 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:48.955 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:56:48.955 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:21.990521+0000", 2023-09-23T20:56:48.955 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.010892013000000001 2023-09-23T20:56:48.955 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:48.955 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:48.955 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:56:48.955 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:21.990565+0000", 2023-09-23T20:56:48.955 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.4579000000000001e-05 2023-09-23T20:56:48.955 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:48.955 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:48.956 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T20:56:48.956 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:21.990605+0000", 2023-09-23T20:56:48.956 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.9854000000000002e-05 2023-09-23T20:56:48.956 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:48.956 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:48.956 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T20:56:48.956 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:21.990701+0000", 2023-09-23T20:56:48.956 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.5227000000000002e-05 2023-09-23T20:56:48.956 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T20:56:48.956 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T20:56:48.956 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T20:56:48.956 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:48.957 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:48.957 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.34797.0:167338 6.a 6:571cce03:::benchmark_data_smithi026_223868_object66212:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e158)", 2023-09-23T20:56:48.957 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T20:55:21.027265+0000", 2023-09-23T20:56:48.957 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 87.859401903999995, 2023-09-23T20:56:48.957 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.976460098, 2023-09-23T20:56:48.957 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T20:56:48.957 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:56:48.957 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T20:56:48.957 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.34797", 2023-09-23T20:56:48.957 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T20:56:48.958 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 167338 2023-09-23T20:56:48.958 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:48.958 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T20:56:48.958 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:48.958 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T20:56:48.958 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:21.027265+0000", 2023-09-23T20:56:48.958 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T20:56:48.958 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:48.958 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:48.958 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T20:56:48.959 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:21.027265+0000", 2023-09-23T20:56:48.959 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T20:56:48.959 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:48.959 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:48.959 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T20:56:48.959 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:21.027263+0000", 2023-09-23T20:56:48.959 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999981 2023-09-23T20:56:48.959 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:48.959 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:48.959 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T20:56:48.959 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:21.027270+0000", 2023-09-23T20:56:48.959 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.5329999999999997e-06 2023-09-23T20:56:48.959 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:48.960 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:48.960 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T20:56:48.960 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:21.027283+0000", 2023-09-23T20:56:48.960 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.2516e-05 2023-09-23T20:56:48.960 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:48.960 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:48.960 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T20:56:48.960 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:21.027290+0000", 2023-09-23T20:56:48.960 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.4989999999999999e-06 2023-09-23T20:56:48.960 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:48.960 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:48.960 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T20:56:48.961 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:21.027342+0000", 2023-09-23T20:56:48.961 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.2244000000000003e-05 2023-09-23T20:56:48.961 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:48.961 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:48.961 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T20:56:48.961 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:21.027351+0000", 2023-09-23T20:56:48.961 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.8410000000000006e-06 2023-09-23T20:56:48.961 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:48.961 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:48.961 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T20:56:48.961 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:21.974196+0000", 2023-09-23T20:56:48.961 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.94684518299999998 2023-09-23T20:56:48.962 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:48.962 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:48.962 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T20:56:48.962 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:21.974318+0000", 2023-09-23T20:56:48.962 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00012264 2023-09-23T20:56:48.962 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:48.962 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:48.962 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 0,2", 2023-09-23T20:56:48.962 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:21.974384+0000", 2023-09-23T20:56:48.962 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.5337000000000001e-05 2023-09-23T20:56:48.962 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:48.962 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:48.962 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T20:56:48.963 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:21.981609+0000", 2023-09-23T20:56:48.963 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00722553 2023-09-23T20:56:48.963 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:48.963 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:48.963 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:56:48.963 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:21.995381+0000", 2023-09-23T20:56:48.963 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.013771781 2023-09-23T20:56:48.963 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:48.963 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:48.963 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:56:48.963 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:22.003516+0000", 2023-09-23T20:56:48.963 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0081353399999999996 2023-09-23T20:56:48.964 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:48.964 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:48.964 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T20:56:48.964 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:22.003568+0000", 2023-09-23T20:56:48.964 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.1354000000000003e-05 2023-09-23T20:56:48.964 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:48.964 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:48.964 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T20:56:48.964 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:22.003725+0000", 2023-09-23T20:56:48.964 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00015731400000000001 2023-09-23T20:56:48.964 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T20:56:48.964 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T20:56:48.965 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T20:56:48.965 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:48.965 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:48.965 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.34797.0:177896 6.a 6:540ceadc:::benchmark_data_smithi026_223868_object76770:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e163)", 2023-09-23T20:56:48.965 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T20:55:34.908670+0000", 2023-09-23T20:56:48.965 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 73.977997349000006, 2023-09-23T20:56:48.965 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.1466390440000001, 2023-09-23T20:56:48.965 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T20:56:48.965 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "reached pg", 2023-09-23T20:56:48.965 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T20:56:48.965 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.34797", 2023-09-23T20:56:48.965 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T20:56:48.966 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 177896 2023-09-23T20:56:48.966 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:48.966 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T20:56:48.966 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:48.966 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T20:56:48.966 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:34.908670+0000", 2023-09-23T20:56:48.966 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T20:56:48.966 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:48.966 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:48.966 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T20:56:48.966 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:34.908670+0000", 2023-09-23T20:56:48.966 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T20:56:48.967 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:48.967 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:48.967 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T20:56:48.967 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:34.908668+0000", 2023-09-23T20:56:48.967 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.999999 2023-09-23T20:56:48.967 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:48.967 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:48.967 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T20:56:48.967 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:34.908673+0000", 2023-09-23T20:56:48.967 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.4769999999999997e-06 2023-09-23T20:56:48.967 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:48.967 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:48.968 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T20:56:48.968 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:34.908683+0000", 2023-09-23T20:56:48.968 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.0254e-05 2023-09-23T20:56:48.968 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:48.968 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:48.968 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T20:56:48.968 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:34.908687+0000", 2023-09-23T20:56:48.968 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.1219999999999997e-06 2023-09-23T20:56:48.968 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:48.968 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:48.968 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T20:56:48.968 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:34.908728+0000", 2023-09-23T20:56:48.968 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.0781999999999997e-05 2023-09-23T20:56:48.969 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:48.969 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:48.969 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T20:56:48.969 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:34.908820+0000", 2023-09-23T20:56:48.969 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.2058999999999994e-05 2023-09-23T20:56:48.969 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:48.969 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:48.969 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 0,2", 2023-09-23T20:56:48.969 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:34.908888+0000", 2023-09-23T20:56:48.969 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.8287000000000005e-05 2023-09-23T20:56:48.970 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:48.970 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:48.970 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T20:56:48.970 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:34.911386+0000", 2023-09-23T20:56:48.970 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0024977459999999999 2023-09-23T20:56:48.970 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:48.970 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:48.970 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:56:48.970 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:34.923103+0000", 2023-09-23T20:56:48.970 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.011717293 2023-09-23T20:56:48.970 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:48.970 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:48.971 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T20:56:48.971 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:34.926697+0000", 2023-09-23T20:56:48.971 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0035937220000000001 2023-09-23T20:56:48.971 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:48.971 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:48.971 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T20:56:48.971 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:34.926708+0000", 2023-09-23T20:56:48.971 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.0339e-05 2023-09-23T20:56:48.971 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:48.971 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:48.971 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T20:56:48.972 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:35.946870+0000", 2023-09-23T20:56:48.972 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.0201623930000001 2023-09-23T20:56:48.972 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:48.972 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:48.972 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T20:56:48.972 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:35.946882+0000", 2023-09-23T20:56:48.972 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.1936999999999999e-05 2023-09-23T20:56:48.972 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:48.972 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:48.972 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T20:56:48.973 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:37.039221+0000", 2023-09-23T20:56:48.973 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.092338928 2023-09-23T20:56:48.973 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:48.973 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:48.973 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for missing object", 2023-09-23T20:56:48.973 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:37.039314+0000", 2023-09-23T20:56:48.973 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.3096000000000004e-05 2023-09-23T20:56:48.973 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:48.973 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:48.973 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T20:56:48.973 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:37.055200+0000", 2023-09-23T20:56:48.973 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.015886493000000002 2023-09-23T20:56:48.974 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:48.974 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:48.974 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T20:56:48.974 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:37.055309+0000", 2023-09-23T20:56:48.974 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00010818 2023-09-23T20:56:48.974 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T20:56:48.974 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T20:56:48.974 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T20:56:48.974 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:48.974 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:48.974 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.34797.0:177903 6.a 6:5c2bf67a:::benchmark_data_smithi026_223868_object76777:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e163)", 2023-09-23T20:56:48.974 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T20:55:34.915862+0000", 2023-09-23T20:56:48.974 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 73.970804767000004, 2023-09-23T20:56:48.975 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.1423018539999998, 2023-09-23T20:56:48.975 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T20:56:48.975 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "reached pg", 2023-09-23T20:56:48.975 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T20:56:48.975 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.34797", 2023-09-23T20:56:48.975 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T20:56:48.975 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 177903 2023-09-23T20:56:48.975 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:48.975 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T20:56:48.975 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:48.975 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T20:56:48.975 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:34.915862+0000", 2023-09-23T20:56:48.976 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T20:56:48.976 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:48.976 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:48.976 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T20:56:48.976 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:34.915862+0000", 2023-09-23T20:56:48.976 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T20:56:48.976 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:48.976 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:48.976 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T20:56:48.976 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:34.915861+0000", 2023-09-23T20:56:48.976 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.999999 2023-09-23T20:56:48.976 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:48.976 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:48.977 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T20:56:48.977 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:34.915866+0000", 2023-09-23T20:56:48.977 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.78e-06 2023-09-23T20:56:48.977 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:48.977 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:48.977 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T20:56:48.977 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:34.915881+0000", 2023-09-23T20:56:48.977 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.552e-05 2023-09-23T20:56:48.977 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:48.977 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:48.977 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T20:56:48.977 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:34.915889+0000", 2023-09-23T20:56:48.977 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.1960000000000002e-06 2023-09-23T20:56:48.977 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:48.978 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:48.978 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T20:56:48.978 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:34.915932+0000", 2023-09-23T20:56:48.978 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.3272000000000002e-05 2023-09-23T20:56:48.978 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:48.978 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:48.978 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T20:56:48.978 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:34.916061+0000", 2023-09-23T20:56:48.978 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00012904999999999999 2023-09-23T20:56:48.978 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:48.978 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:48.978 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 0,2", 2023-09-23T20:56:48.978 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:34.916135+0000", 2023-09-23T20:56:48.979 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.4144e-05 2023-09-23T20:56:48.979 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:48.979 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:48.979 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T20:56:48.979 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:34.918475+0000", 2023-09-23T20:56:48.979 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.002339921 2023-09-23T20:56:48.979 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:48.979 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:48.979 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T20:56:48.979 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:34.926898+0000", 2023-09-23T20:56:48.979 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0084229939999999996 2023-09-23T20:56:48.979 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:48.979 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:48.980 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T20:56:48.980 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:34.926907+0000", 2023-09-23T20:56:48.980 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.9590000000000001e-06 2023-09-23T20:56:48.980 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:48.980 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:48.980 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T20:56:48.980 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:35.947668+0000", 2023-09-23T20:56:48.980 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.0207606359999999 2023-09-23T20:56:48.980 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:48.980 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:48.980 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T20:56:48.980 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:35.947676+0000", 2023-09-23T20:56:48.980 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.7029999999999995e-06 2023-09-23T20:56:48.980 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:48.981 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:48.981 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T20:56:48.981 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:37.039586+0000", 2023-09-23T20:56:48.981 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.0919099109999999 2023-09-23T20:56:48.981 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:48.981 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:48.981 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for missing object", 2023-09-23T20:56:48.981 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:37.039684+0000", 2023-09-23T20:56:48.981 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.7732999999999994e-05 2023-09-23T20:56:48.981 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:48.981 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:48.981 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T20:56:48.982 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:37.058038+0000", 2023-09-23T20:56:48.982 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.018353669999999999 2023-09-23T20:56:48.982 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:48.982 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:48.982 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T20:56:48.982 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:37.058164+0000", 2023-09-23T20:56:48.982 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00012637400000000001 2023-09-23T20:56:48.982 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T20:56:48.982 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T20:56:48.982 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T20:56:48.982 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:48.982 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:48.982 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.34797.0:177914 6.a 6:5c0a39bf:::benchmark_data_smithi026_223868_object76788:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e164)", 2023-09-23T20:56:48.983 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T20:55:34.934127+0000", 2023-09-23T20:56:48.983 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 73.952540166000006, 2023-09-23T20:56:48.983 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.1279153630000001, 2023-09-23T20:56:48.983 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T20:56:48.983 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:56:48.983 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T20:56:48.983 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.34797", 2023-09-23T20:56:48.983 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T20:56:48.983 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 177914 2023-09-23T20:56:48.983 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:48.983 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T20:56:48.983 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:48.983 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T20:56:48.984 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:34.934127+0000", 2023-09-23T20:56:48.984 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T20:56:48.984 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:48.984 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:48.984 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T20:56:48.984 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:34.934127+0000", 2023-09-23T20:56:48.984 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T20:56:48.984 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:48.984 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:48.984 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T20:56:48.984 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:34.934126+0000", 2023-09-23T20:56:48.984 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999995 2023-09-23T20:56:48.985 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:48.985 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:48.985 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T20:56:48.985 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:34.934129+0000", 2023-09-23T20:56:48.985 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.1690000000000001e-06 2023-09-23T20:56:48.985 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:48.985 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:48.985 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T20:56:48.985 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:34.934136+0000", 2023-09-23T20:56:48.985 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.8120000000000002e-06 2023-09-23T20:56:48.985 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:48.986 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:48.986 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T20:56:48.986 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:34.934141+0000", 2023-09-23T20:56:48.986 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.7090000000000001e-06 2023-09-23T20:56:48.986 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:48.986 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:48.986 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T20:56:48.986 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:34.935261+0000", 2023-09-23T20:56:48.986 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0011200369999999999 2023-09-23T20:56:48.986 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:48.986 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:48.987 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T20:56:48.987 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:34.935272+0000", 2023-09-23T20:56:48.987 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.1202999999999999e-05 2023-09-23T20:56:48.987 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:48.987 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:48.987 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T20:56:48.987 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:35.947706+0000", 2023-09-23T20:56:48.987 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.012434187 2023-09-23T20:56:48.987 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:48.987 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:48.987 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T20:56:48.987 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:35.947714+0000", 2023-09-23T20:56:48.987 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.8990000000000001e-06 2023-09-23T20:56:48.988 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:48.988 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:48.988 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T20:56:48.988 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:37.039714+0000", 2023-09-23T20:56:48.988 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.091999768 2023-09-23T20:56:48.988 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:48.988 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:48.988 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T20:56:48.988 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:37.039838+0000", 2023-09-23T20:56:48.988 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00012372300000000001 2023-09-23T20:56:48.988 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:48.988 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:48.988 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 0,6", 2023-09-23T20:56:48.989 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:37.039921+0000", 2023-09-23T20:56:48.989 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.3489e-05 2023-09-23T20:56:48.989 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:48.989 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:48.989 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T20:56:48.989 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:37.053854+0000", 2023-09-23T20:56:48.989 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.013933164 2023-09-23T20:56:48.989 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:48.989 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:48.989 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:56:48.989 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:37.054159+0000", 2023-09-23T20:56:48.989 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00030508299999999999 2023-09-23T20:56:48.989 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:48.989 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:48.990 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:56:48.990 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:37.061887+0000", 2023-09-23T20:56:48.990 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0077278579999999998 2023-09-23T20:56:48.990 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:48.990 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:48.990 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T20:56:48.990 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:37.061930+0000", 2023-09-23T20:56:48.990 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.2454999999999999e-05 2023-09-23T20:56:48.990 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:48.990 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:48.990 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T20:56:48.990 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:37.062042+0000", 2023-09-23T20:56:48.991 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000112333 2023-09-23T20:56:48.991 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T20:56:48.991 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T20:56:48.991 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T20:56:48.991 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:48.991 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:48.991 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.34797.0:177920 6.a 6:5d69f263:::benchmark_data_smithi026_223868_object76794:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e164)", 2023-09-23T20:56:48.991 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T20:55:34.954622+0000", 2023-09-23T20:56:48.991 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 73.932044575999996, 2023-09-23T20:56:48.991 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.1076863440000002, 2023-09-23T20:56:48.991 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T20:56:48.991 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:56:48.991 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T20:56:48.992 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.34797", 2023-09-23T20:56:48.992 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T20:56:48.992 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 177920 2023-09-23T20:56:48.992 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:48.992 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T20:56:48.992 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:48.992 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T20:56:48.992 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:34.954622+0000", 2023-09-23T20:56:48.992 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T20:56:48.992 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:48.992 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:48.992 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T20:56:48.993 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:34.954622+0000", 2023-09-23T20:56:48.993 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T20:56:48.993 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:48.993 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:48.993 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T20:56:48.993 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:34.954621+0000", 2023-09-23T20:56:48.993 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T20:56:48.993 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:48.993 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:48.993 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T20:56:48.993 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:34.954627+0000", 2023-09-23T20:56:48.993 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.7440000000000004e-06 2023-09-23T20:56:48.993 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:48.994 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:48.994 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T20:56:48.994 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:34.954638+0000", 2023-09-23T20:56:48.994 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.1487e-05 2023-09-23T20:56:48.994 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:48.994 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:48.994 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T20:56:48.994 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:34.954644+0000", 2023-09-23T20:56:48.994 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.0519999999999998e-06 2023-09-23T20:56:48.994 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:48.994 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:48.994 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T20:56:48.995 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:34.954700+0000", 2023-09-23T20:56:48.995 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.5337999999999998e-05 2023-09-23T20:56:48.995 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:48.995 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:48.995 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T20:56:48.995 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:34.954713+0000", 2023-09-23T20:56:48.995 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.3003000000000001e-05 2023-09-23T20:56:48.995 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:48.995 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:48.995 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T20:56:48.995 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:35.947751+0000", 2023-09-23T20:56:48.995 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.99303834199999996 2023-09-23T20:56:48.995 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:48.996 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:48.996 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T20:56:48.996 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:35.947761+0000", 2023-09-23T20:56:48.996 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.4979999999999995e-06 2023-09-23T20:56:48.996 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:48.996 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:48.996 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T20:56:48.996 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:37.042465+0000", 2023-09-23T20:56:48.996 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.0947049710000001 2023-09-23T20:56:48.996 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:48.996 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:48.996 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T20:56:48.996 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:37.042599+0000", 2023-09-23T20:56:48.997 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00013388199999999999 2023-09-23T20:56:48.997 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:48.997 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:48.997 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 0,6", 2023-09-23T20:56:48.997 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:37.042722+0000", 2023-09-23T20:56:48.997 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000122893 2023-09-23T20:56:48.997 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:48.997 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:48.997 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T20:56:48.997 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:37.053862+0000", 2023-09-23T20:56:48.997 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.011139540999999999 2023-09-23T20:56:48.997 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:48.997 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:48.998 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:56:48.998 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:37.057624+0000", 2023-09-23T20:56:48.998 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.003762661 2023-09-23T20:56:48.998 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:48.998 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:48.998 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:56:48.998 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:37.062109+0000", 2023-09-23T20:56:48.998 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0044847259999999996 2023-09-23T20:56:48.998 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:48.998 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:48.998 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T20:56:48.998 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:37.062163+0000", 2023-09-23T20:56:48.998 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.346e-05 2023-09-23T20:56:48.999 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:48.999 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:48.999 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T20:56:48.999 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:37.062309+0000", 2023-09-23T20:56:48.999 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00014601300000000001 2023-09-23T20:56:48.999 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T20:56:48.999 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T20:56:48.999 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T20:56:48.999 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:48.999 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:48.999 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.34797.0:177923 6.a 6:53679877:::benchmark_data_smithi026_223868_object76797:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e164)", 2023-09-23T20:56:48.999 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T20:55:34.956263+0000", 2023-09-23T20:56:48.999 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 73.930403701000003, 2023-09-23T20:56:49.000 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.1062447940000002, 2023-09-23T20:56:49.000 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T20:56:49.000 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:56:49.000 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T20:56:49.000 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.34797", 2023-09-23T20:56:49.000 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T20:56:49.000 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 177923 2023-09-23T20:56:49.000 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:49.000 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T20:56:49.000 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:49.000 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T20:56:49.000 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:34.956263+0000", 2023-09-23T20:56:49.000 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T20:56:49.000 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:49.001 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:49.001 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T20:56:49.001 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:34.956263+0000", 2023-09-23T20:56:49.001 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T20:56:49.001 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:49.001 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:49.001 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T20:56:49.001 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:34.956262+0000", 2023-09-23T20:56:49.001 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T20:56:49.001 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:49.001 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:49.001 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T20:56:49.001 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:34.956268+0000", 2023-09-23T20:56:49.001 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.6189999999999996e-06 2023-09-23T20:56:49.002 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:49.002 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:49.002 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T20:56:49.002 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:34.956283+0000", 2023-09-23T20:56:49.002 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.4725999999999999e-05 2023-09-23T20:56:49.002 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:49.002 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:49.002 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T20:56:49.002 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:34.956289+0000", 2023-09-23T20:56:49.002 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.1380000000000004e-06 2023-09-23T20:56:49.002 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:49.002 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:49.003 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T20:56:49.003 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:34.956345+0000", 2023-09-23T20:56:49.003 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.5288000000000003e-05 2023-09-23T20:56:49.003 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:49.003 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:49.003 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T20:56:49.003 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:34.956354+0000", 2023-09-23T20:56:49.003 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.9150000000000002e-06 2023-09-23T20:56:49.003 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:49.003 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:49.003 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T20:56:49.003 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:35.947803+0000", 2023-09-23T20:56:49.003 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.99144918699999995 2023-09-23T20:56:49.003 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:49.004 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:49.004 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T20:56:49.004 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:35.947811+0000", 2023-09-23T20:56:49.004 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.5129999999999996e-06 2023-09-23T20:56:49.004 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:49.004 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:49.004 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T20:56:49.004 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:37.047285+0000", 2023-09-23T20:56:49.004 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.099473857 2023-09-23T20:56:49.004 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:49.004 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:49.004 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T20:56:49.004 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:37.047408+0000", 2023-09-23T20:56:49.004 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00012244 2023-09-23T20:56:49.005 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:49.005 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:49.005 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 0,6", 2023-09-23T20:56:49.005 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:37.047526+0000", 2023-09-23T20:56:49.005 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000118539 2023-09-23T20:56:49.005 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:49.005 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:49.005 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T20:56:49.005 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:37.053868+0000", 2023-09-23T20:56:49.005 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0063422560000000001 2023-09-23T20:56:49.005 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:49.005 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:49.005 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:56:49.006 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:37.058203+0000", 2023-09-23T20:56:49.006 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0043349929999999997 2023-09-23T20:56:49.006 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:49.006 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:49.006 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:56:49.006 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:37.062353+0000", 2023-09-23T20:56:49.006 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0041494890000000001 2023-09-23T20:56:49.006 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:49.006 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:49.006 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T20:56:49.006 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:37.062398+0000", 2023-09-23T20:56:49.006 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.5639999999999997e-05 2023-09-23T20:56:49.007 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:49.007 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:49.007 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T20:56:49.007 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:37.062508+0000", 2023-09-23T20:56:49.007 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00010949400000000001 2023-09-23T20:56:49.007 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T20:56:49.007 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T20:56:49.007 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T20:56:49.007 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:49.007 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:49.007 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.34797.0:177948 6.a 6:597c0f94:::benchmark_data_smithi026_223868_object76822:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e164)", 2023-09-23T20:56:49.007 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T20:55:34.996671+0000", 2023-09-23T20:56:49.007 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 73.889995853000002, 2023-09-23T20:56:49.007 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.06610118, 2023-09-23T20:56:49.008 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T20:56:49.008 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:56:49.008 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T20:56:49.008 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.34797", 2023-09-23T20:56:49.008 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T20:56:49.008 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 177948 2023-09-23T20:56:49.008 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:49.008 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T20:56:49.008 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:49.008 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T20:56:49.008 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:34.996671+0000", 2023-09-23T20:56:49.008 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T20:56:49.008 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:49.008 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:49.009 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T20:56:49.009 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:34.996671+0000", 2023-09-23T20:56:49.009 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T20:56:49.009 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:49.009 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:49.009 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T20:56:49.009 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:34.996670+0000", 2023-09-23T20:56:49.009 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.999999 2023-09-23T20:56:49.009 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:49.009 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:49.009 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T20:56:49.009 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:34.996675+0000", 2023-09-23T20:56:49.009 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.6249999999999998e-06 2023-09-23T20:56:49.009 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:49.010 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:49.010 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T20:56:49.010 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:34.996684+0000", 2023-09-23T20:56:49.010 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.2450000000000004e-06 2023-09-23T20:56:49.010 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:49.010 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:49.010 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T20:56:49.010 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:34.996687+0000", 2023-09-23T20:56:49.010 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.563e-06 2023-09-23T20:56:49.010 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:49.010 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:49.010 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T20:56:49.010 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:34.996739+0000", 2023-09-23T20:56:49.010 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.1579000000000002e-05 2023-09-23T20:56:49.011 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:49.011 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:49.011 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T20:56:49.011 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:34.996746+0000", 2023-09-23T20:56:49.011 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.2690000000000002e-06 2023-09-23T20:56:49.011 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:49.011 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:49.011 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T20:56:49.011 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:35.947834+0000", 2023-09-23T20:56:49.011 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.95108812399999998 2023-09-23T20:56:49.011 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:49.011 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:49.011 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T20:56:49.011 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:35.947842+0000", 2023-09-23T20:56:49.012 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.7370000000000006e-06 2023-09-23T20:56:49.012 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:49.012 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:49.012 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T20:56:49.012 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:37.050299+0000", 2023-09-23T20:56:49.012 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.1024569989999999 2023-09-23T20:56:49.012 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:49.012 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:49.012 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T20:56:49.012 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:37.050434+0000", 2023-09-23T20:56:49.012 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00013435199999999999 2023-09-23T20:56:49.013 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:49.013 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:49.013 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 0,6", 2023-09-23T20:56:49.013 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:37.050565+0000", 2023-09-23T20:56:49.013 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00013146300000000001 2023-09-23T20:56:49.013 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:49.013 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:49.013 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T20:56:49.013 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:37.053877+0000", 2023-09-23T20:56:49.013 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0033121589999999998 2023-09-23T20:56:49.013 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:49.013 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:49.013 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:56:49.013 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:37.058255+0000", 2023-09-23T20:56:49.014 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0043776930000000002 2023-09-23T20:56:49.014 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:49.014 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:49.014 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:56:49.014 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:37.062560+0000", 2023-09-23T20:56:49.014 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0043055719999999997 2023-09-23T20:56:49.014 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:49.014 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:49.014 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T20:56:49.014 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:37.062636+0000", 2023-09-23T20:56:49.014 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.5417e-05 2023-09-23T20:56:49.014 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:49.015 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:49.015 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T20:56:49.015 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:37.062772+0000", 2023-09-23T20:56:49.015 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00013637499999999999 2023-09-23T20:56:49.015 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T20:56:49.015 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T20:56:49.015 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T20:56:49.015 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:49.015 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:49.015 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.34797.0:184464 6.17 6:ec4eeb9e:::benchmark_data_smithi026_223868_object83338:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e170)", 2023-09-23T20:56:49.015 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T20:55:46.018382+0000", 2023-09-23T20:56:49.015 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 62.868284443999997, 2023-09-23T20:56:49.016 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.97416476299999999, 2023-09-23T20:56:49.016 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T20:56:49.016 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:56:49.016 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T20:56:49.016 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.34797", 2023-09-23T20:56:49.016 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T20:56:49.016 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 184464 2023-09-23T20:56:49.016 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:49.016 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T20:56:49.016 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:49.016 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T20:56:49.017 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:46.018382+0000", 2023-09-23T20:56:49.017 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T20:56:49.017 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:49.017 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:49.017 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T20:56:49.017 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:46.018382+0000", 2023-09-23T20:56:49.017 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T20:56:49.017 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:49.017 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:49.017 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T20:56:49.017 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:46.018382+0000", 2023-09-23T20:56:49.017 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999995 2023-09-23T20:56:49.017 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:49.018 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:49.018 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T20:56:49.018 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:46.018385+0000", 2023-09-23T20:56:49.018 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.571e-06 2023-09-23T20:56:49.018 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:49.018 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:49.018 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T20:56:49.018 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:46.018390+0000", 2023-09-23T20:56:49.018 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.6629999999999998e-06 2023-09-23T20:56:49.018 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:49.018 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:49.018 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T20:56:49.019 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:46.018394+0000", 2023-09-23T20:56:49.019 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.8659999999999999e-06 2023-09-23T20:56:49.019 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:49.019 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:49.019 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T20:56:49.019 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:46.018524+0000", 2023-09-23T20:56:49.019 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00012990000000000001 2023-09-23T20:56:49.019 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:49.019 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:49.019 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T20:56:49.019 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:46.018529+0000", 2023-09-23T20:56:49.019 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.1680000000000003e-06 2023-09-23T20:56:49.019 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:49.020 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:49.020 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T20:56:49.020 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:46.969211+0000", 2023-09-23T20:56:49.020 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.95068135499999995 2023-09-23T20:56:49.020 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:49.020 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:49.020 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T20:56:49.020 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:46.969338+0000", 2023-09-23T20:56:49.020 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000127699 2023-09-23T20:56:49.020 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:49.020 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:49.020 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 3,4", 2023-09-23T20:56:49.020 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:46.969463+0000", 2023-09-23T20:56:49.021 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000124833 2023-09-23T20:56:49.021 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:49.021 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:49.021 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:56:49.021 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:46.975159+0000", 2023-09-23T20:56:49.021 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0056955929999999997 2023-09-23T20:56:49.021 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:49.021 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:49.021 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T20:56:49.021 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:46.976924+0000", 2023-09-23T20:56:49.021 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.001765567 2023-09-23T20:56:49.021 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:49.021 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:49.021 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:56:49.022 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:46.992475+0000", 2023-09-23T20:56:49.022 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.015550402 2023-09-23T20:56:49.022 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:49.022 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:49.022 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T20:56:49.022 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:46.992514+0000", 2023-09-23T20:56:49.022 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.9075000000000001e-05 2023-09-23T20:56:49.022 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:49.022 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:49.022 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T20:56:49.022 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:46.992547+0000", 2023-09-23T20:56:49.022 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.3463999999999997e-05 2023-09-23T20:56:49.022 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T20:56:49.022 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T20:56:49.023 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T20:56:49.023 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:49.023 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:49.023 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.34797.0:185827 6.a 6:51461ffe:::benchmark_data_smithi026_223868_object84701:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e172)", 2023-09-23T20:56:49.023 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T20:55:48.221799+0000", 2023-09-23T20:56:49.023 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 60.664867741000002, 2023-09-23T20:56:49.023 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.038071078, 2023-09-23T20:56:49.023 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T20:56:49.023 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:56:49.023 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T20:56:49.023 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.34797", 2023-09-23T20:56:49.023 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T20:56:49.023 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 185827 2023-09-23T20:56:49.024 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:49.024 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T20:56:49.024 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:49.024 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T20:56:49.024 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:48.221799+0000", 2023-09-23T20:56:49.024 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T20:56:49.024 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:49.024 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:49.024 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T20:56:49.024 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:48.221799+0000", 2023-09-23T20:56:49.024 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T20:56:49.024 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:49.024 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:49.024 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T20:56:49.025 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:48.221799+0000", 2023-09-23T20:56:49.025 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999995 2023-09-23T20:56:49.025 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:49.025 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:49.025 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T20:56:49.025 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:48.221802+0000", 2023-09-23T20:56:49.025 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.2049999999999998e-06 2023-09-23T20:56:49.025 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:49.025 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:49.025 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T20:56:49.025 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:48.221808+0000", 2023-09-23T20:56:49.025 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.19e-06 2023-09-23T20:56:49.025 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:49.026 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:49.026 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T20:56:49.026 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:48.221811+0000", 2023-09-23T20:56:49.026 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.52e-06 2023-09-23T20:56:49.026 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:49.026 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:49.026 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T20:56:49.026 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:48.221859+0000", 2023-09-23T20:56:49.026 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.8857999999999999e-05 2023-09-23T20:56:49.026 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:49.026 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:49.026 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T20:56:49.026 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:48.221871+0000", 2023-09-23T20:56:49.027 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.1174e-05 2023-09-23T20:56:49.027 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:49.027 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:49.027 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T20:56:49.027 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:49.249764+0000", 2023-09-23T20:56:49.027 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.027893758 2023-09-23T20:56:49.027 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:49.027 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:49.027 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T20:56:49.027 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:49.249884+0000", 2023-09-23T20:56:49.027 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000119438 2023-09-23T20:56:49.027 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:49.027 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:49.027 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 0,6", 2023-09-23T20:56:49.028 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:49.249965+0000", 2023-09-23T20:56:49.028 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.1518000000000001e-05 2023-09-23T20:56:49.028 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:49.028 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:49.028 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:56:49.028 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:49.252936+0000", 2023-09-23T20:56:49.028 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0029711009999999999 2023-09-23T20:56:49.028 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:49.028 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:49.028 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T20:56:49.028 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:49.256217+0000", 2023-09-23T20:56:49.028 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0032802489999999998 2023-09-23T20:56:49.028 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:49.029 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:49.029 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:56:49.029 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:49.259769+0000", 2023-09-23T20:56:49.029 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0035519509999999998 2023-09-23T20:56:49.029 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:49.029 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:49.029 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T20:56:49.029 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:49.259825+0000", 2023-09-23T20:56:49.029 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.6854999999999997e-05 2023-09-23T20:56:49.029 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:49.029 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:49.029 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T20:56:49.029 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:49.259870+0000", 2023-09-23T20:56:49.029 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.4774999999999997e-05 2023-09-23T20:56:49.030 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T20:56:49.030 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T20:56:49.030 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T20:56:49.030 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:49.030 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:49.030 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.34797.0:185834 6.a 6:59f98fb0:::benchmark_data_smithi026_223868_object84708:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e172)", 2023-09-23T20:56:49.030 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T20:55:48.229707+0000", 2023-09-23T20:56:49.030 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 60.656959913000001, 2023-09-23T20:56:49.030 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.0303501639999999, 2023-09-23T20:56:49.030 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T20:56:49.030 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:56:49.030 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T20:56:49.030 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.34797", 2023-09-23T20:56:49.030 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T20:56:49.031 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 185834 2023-09-23T20:56:49.031 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:49.031 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T20:56:49.031 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:49.031 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T20:56:49.031 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:48.229707+0000", 2023-09-23T20:56:49.031 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T20:56:49.031 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:49.031 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:49.031 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T20:56:49.031 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:48.229707+0000", 2023-09-23T20:56:49.031 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T20:56:49.031 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:49.031 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:49.032 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T20:56:49.032 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:48.229706+0000", 2023-09-23T20:56:49.032 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.999999 2023-09-23T20:56:49.032 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:49.032 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:49.032 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T20:56:49.032 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:48.229712+0000", 2023-09-23T20:56:49.032 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.7400000000000001e-06 2023-09-23T20:56:49.032 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:49.032 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:49.032 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T20:56:49.032 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:48.229724+0000", 2023-09-23T20:56:49.032 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.2797e-05 2023-09-23T20:56:49.032 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:49.032 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:49.033 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T20:56:49.033 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:48.229731+0000", 2023-09-23T20:56:49.033 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.5080000000000002e-06 2023-09-23T20:56:49.033 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:49.033 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:49.033 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T20:56:49.033 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:48.229783+0000", 2023-09-23T20:56:49.033 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.1953000000000002e-05 2023-09-23T20:56:49.033 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:49.033 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:49.033 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T20:56:49.033 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:48.229794+0000", 2023-09-23T20:56:49.033 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.1061e-05 2023-09-23T20:56:49.034 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:49.034 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:49.034 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T20:56:49.034 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:49.253001+0000", 2023-09-23T20:56:49.034 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.023207209 2023-09-23T20:56:49.034 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:49.034 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:49.034 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T20:56:49.034 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:49.253151+0000", 2023-09-23T20:56:49.034 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00014949599999999999 2023-09-23T20:56:49.034 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:49.034 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:49.034 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 0,6", 2023-09-23T20:56:49.035 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:49.253282+0000", 2023-09-23T20:56:49.035 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000131922 2023-09-23T20:56:49.035 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:49.035 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:49.035 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:56:49.035 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:49.259649+0000", 2023-09-23T20:56:49.035 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0063662659999999998 2023-09-23T20:56:49.035 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:49.035 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:49.035 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T20:56:49.035 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:49.259934+0000", 2023-09-23T20:56:49.035 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00028488500000000002 2023-09-23T20:56:49.035 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:49.035 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:49.036 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:56:49.036 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:49.259975+0000", 2023-09-23T20:56:49.036 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.1811000000000003e-05 2023-09-23T20:56:49.036 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:49.036 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:49.036 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T20:56:49.036 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:49.260024+0000", 2023-09-23T20:56:49.036 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.8155000000000002e-05 2023-09-23T20:56:49.036 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:49.036 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:49.036 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T20:56:49.036 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:49.260057+0000", 2023-09-23T20:56:49.036 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.3522000000000002e-05 2023-09-23T20:56:49.036 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T20:56:49.037 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T20:56:49.037 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T20:56:49.037 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:49.037 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:49.037 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.34797.0:185844 6.a 6:5b27a6f2:::benchmark_data_smithi026_223868_object84718:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e172)", 2023-09-23T20:56:49.037 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T20:55:48.237898+0000", 2023-09-23T20:56:49.037 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 60.648768687999997, 2023-09-23T20:56:49.037 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.022291493, 2023-09-23T20:56:49.037 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T20:56:49.037 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:56:49.037 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T20:56:49.037 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.34797", 2023-09-23T20:56:49.037 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T20:56:49.037 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 185844 2023-09-23T20:56:49.038 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:49.038 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T20:56:49.038 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:49.038 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T20:56:49.038 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:48.237898+0000", 2023-09-23T20:56:49.038 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T20:56:49.038 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:49.038 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:49.038 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T20:56:49.038 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:48.237898+0000", 2023-09-23T20:56:49.038 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T20:56:49.038 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:49.038 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:49.038 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T20:56:49.039 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:48.237897+0000", 2023-09-23T20:56:49.039 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.999999 2023-09-23T20:56:49.039 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:49.039 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:49.039 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T20:56:49.039 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:48.237903+0000", 2023-09-23T20:56:49.039 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.2179999999999998e-06 2023-09-23T20:56:49.039 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:49.039 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:49.039 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T20:56:49.039 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:48.237916+0000", 2023-09-23T20:56:49.039 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.3753e-05 2023-09-23T20:56:49.039 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:49.039 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:49.040 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T20:56:49.040 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:48.237923+0000", 2023-09-23T20:56:49.040 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.6200000000000001e-06 2023-09-23T20:56:49.040 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:49.040 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:49.040 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T20:56:49.040 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:48.237984+0000", 2023-09-23T20:56:49.040 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.1314000000000003e-05 2023-09-23T20:56:49.040 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:49.040 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:49.040 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T20:56:49.040 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:48.237994+0000", 2023-09-23T20:56:49.040 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.8050000000000001e-06 2023-09-23T20:56:49.040 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:49.040 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:49.041 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T20:56:49.041 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:49.256263+0000", 2023-09-23T20:56:49.041 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.018269149 2023-09-23T20:56:49.041 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:49.041 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:49.041 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T20:56:49.041 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:49.256351+0000", 2023-09-23T20:56:49.041 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.8261000000000001e-05 2023-09-23T20:56:49.041 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:49.041 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:49.041 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 0,6", 2023-09-23T20:56:49.041 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:49.256430+0000", 2023-09-23T20:56:49.041 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.8651000000000003e-05 2023-09-23T20:56:49.041 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:49.042 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:49.042 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:56:49.042 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:49.259924+0000", 2023-09-23T20:56:49.042 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0034939390000000002 2023-09-23T20:56:49.042 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:49.042 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:49.042 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T20:56:49.042 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:49.259942+0000", 2023-09-23T20:56:49.042 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.8261999999999999e-05 2023-09-23T20:56:49.042 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:49.042 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:49.042 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:56:49.043 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:49.260103+0000", 2023-09-23T20:56:49.043 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000160679 2023-09-23T20:56:49.043 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:49.043 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:49.043 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T20:56:49.043 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:49.260148+0000", 2023-09-23T20:56:49.043 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.5330999999999997e-05 2023-09-23T20:56:49.043 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:49.043 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:49.043 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T20:56:49.043 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:49.260190+0000", 2023-09-23T20:56:49.043 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.1409999999999998e-05 2023-09-23T20:56:49.043 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T20:56:49.044 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T20:56:49.044 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T20:56:49.044 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:49.044 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:49.044 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.34797.0:185884 6.a 6:5db88532:::benchmark_data_smithi026_223868_object84758:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e172)", 2023-09-23T20:56:49.044 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T20:55:48.282714+0000", 2023-09-23T20:56:49.044 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 60.603952864999997, 2023-09-23T20:56:49.044 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.99305554799999995, 2023-09-23T20:56:49.044 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T20:56:49.044 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:56:49.044 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T20:56:49.044 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.34797", 2023-09-23T20:56:49.044 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T20:56:49.044 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 185884 2023-09-23T20:56:49.045 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:49.045 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T20:56:49.045 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:49.045 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T20:56:49.045 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:48.282714+0000", 2023-09-23T20:56:49.045 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T20:56:49.045 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:49.045 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:49.045 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T20:56:49.045 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:48.282714+0000", 2023-09-23T20:56:49.045 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T20:56:49.045 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:49.045 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:49.045 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T20:56:49.046 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:48.282714+0000", 2023-09-23T20:56:49.046 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999995 2023-09-23T20:56:49.046 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:49.046 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:49.046 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T20:56:49.046 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:48.282717+0000", 2023-09-23T20:56:49.046 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.1760000000000002e-06 2023-09-23T20:56:49.046 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:49.046 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:49.046 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T20:56:49.046 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:48.282727+0000", 2023-09-23T20:56:49.046 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.8079999999999999e-06 2023-09-23T20:56:49.046 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:49.047 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:49.047 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T20:56:49.047 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:48.282731+0000", 2023-09-23T20:56:49.047 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.4129999999999999e-06 2023-09-23T20:56:49.047 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:49.047 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:49.047 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T20:56:49.047 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:48.282781+0000", 2023-09-23T20:56:49.047 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.9870999999999997e-05 2023-09-23T20:56:49.047 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:49.047 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:49.047 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T20:56:49.047 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:48.282789+0000", 2023-09-23T20:56:49.048 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.1689999999999992e-06 2023-09-23T20:56:49.048 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:49.048 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:49.048 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T20:56:49.048 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:49.260248+0000", 2023-09-23T20:56:49.048 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.97745879800000002 2023-09-23T20:56:49.048 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:49.048 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:49.048 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T20:56:49.048 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:49.260330+0000", 2023-09-23T20:56:49.048 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.2013000000000002e-05 2023-09-23T20:56:49.049 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:49.049 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:49.049 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 0,6", 2023-09-23T20:56:49.049 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:49.260394+0000", 2023-09-23T20:56:49.049 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.4393000000000005e-05 2023-09-23T20:56:49.049 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:49.049 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:49.049 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:56:49.049 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:49.269250+0000", 2023-09-23T20:56:49.049 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0088559889999999999 2023-09-23T20:56:49.049 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:49.049 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:49.049 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:56:49.050 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:49.269282+0000", 2023-09-23T20:56:49.050 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.1733999999999998e-05 2023-09-23T20:56:49.050 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:49.050 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:49.050 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T20:56:49.050 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:49.275670+0000", 2023-09-23T20:56:49.050 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0063884220000000004 2023-09-23T20:56:49.050 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:49.050 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:49.050 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T20:56:49.050 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:49.275729+0000", 2023-09-23T20:56:49.050 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.8371000000000001e-05 2023-09-23T20:56:49.051 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:56:49.051 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:56:49.051 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T20:56:49.051 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:49.275770+0000", 2023-09-23T20:56:49.051 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.0884999999999999e-05 2023-09-23T20:56:49.051 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T20:56:49.051 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T20:56:49.051 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T20:56:49.051 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T20:56:49.051 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T20:56:49.051 INFO:teuthology.orchestra.run.smithi070.stdout:} 2023-09-23T20:56:49.052 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 10 16 22552 22536 140.741 123.125 0.00229215 0.00706044 2023-09-23T20:56:49.209 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:56:49 smithi070 ceph-mon[168282]: 1.1 scrub starts 2023-09-23T20:56:49.209 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:56:49 smithi070 ceph-mon[168282]: 1.1 scrub ok 2023-09-23T20:56:49.209 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:56:49 smithi070 ceph-mon[168282]: 1.7 scrub starts 2023-09-23T20:56:49.210 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:56:49 smithi070 ceph-mon[168282]: 1.7 scrub ok 2023-09-23T20:56:49.281 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_ops_in_flight 2023-09-23T20:56:49.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:56:49 smithi026 ceph-mon[175151]: 1.1 scrub starts 2023-09-23T20:56:49.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:56:49 smithi026 ceph-mon[175151]: 1.1 scrub ok 2023-09-23T20:56:49.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:56:49 smithi026 ceph-mon[175151]: 1.7 scrub starts 2023-09-23T20:56:49.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:56:49 smithi026 ceph-mon[175151]: 1.7 scrub ok 2023-09-23T20:56:49.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:56:49 smithi026 ceph-mon[179688]: 1.1 scrub starts 2023-09-23T20:56:49.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:56:49 smithi026 ceph-mon[179688]: 1.1 scrub ok 2023-09-23T20:56:49.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:56:49 smithi026 ceph-mon[179688]: 1.7 scrub starts 2023-09-23T20:56:49.383 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:56:49 smithi026 ceph-mon[179688]: 1.7 scrub ok 2023-09-23T20:56:49.667 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:56:49.673 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:56:49.673 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:56:49.774 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:56:49.813 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:56:49.861 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:56:49.868 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:56:49.868 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:56:49.883 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:56:49 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[164029]: ::ffff:172.21.15.70 - - [23/Sep/2023:20:56:49] "GET /metrics HTTP/1.1" 200 36318 "" "Prometheus/2.43.0" 2023-09-23T20:56:49.901 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 11 16 24526 24510 139.162 123.375 0.00423827 0.00718156 2023-09-23T20:56:49.969 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:56:49.988 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:56:50.005 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:56:50.012 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:56:50.061 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:56:50.065 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:56:50.066 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:56:50.066 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:56:50 smithi070 ceph-mon[168282]: pgmap v475: 99 pgs: 99 active+clean; 975 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 114 MiB/s wr, 1.82k op/s 2023-09-23T20:56:50.167 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:56:50.208 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:56:50.304 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:56:50.307 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:56:50.308 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:56:50.385 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:56:50 smithi026 ceph-mon[175151]: pgmap v475: 99 pgs: 99 active+clean; 975 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 114 MiB/s wr, 1.82k op/s 2023-09-23T20:56:50.385 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:56:50 smithi026 ceph-mon[179688]: pgmap v475: 99 pgs: 99 active+clean; 975 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 114 MiB/s wr, 1.82k op/s 2023-09-23T20:56:50.408 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:56:50.470 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:56:50.902 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 12 16 26521 26505 137.956 124.688 0.00654742 0.00724314 2023-09-23T20:56:51.063 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:56:51.074 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:56:51.074 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:56:51.175 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:56:51.212 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:56:51.255 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:56:51.258 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:56:51.258 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:56:51.306 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0: {} 2023-09-23T20:56:51.344 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1: {} 2023-09-23T20:56:51.359 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:56:51.386 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2: {} 2023-09-23T20:56:51.400 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:56:51.431 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3: {} 2023-09-23T20:56:51.463 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:56:51.466 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:56:51.466 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:56:51.470 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4: {} 2023-09-23T20:56:51.505 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T20:56:51.543 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6: {} 2023-09-23T20:56:51.567 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:56:51.592 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T20:56:51.613 INFO:teuthology.orchestra.run.smithi026.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T20:56:51.613 INFO:teuthology.orchestra.run.smithi026.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T20:56:51.613 INFO:teuthology.orchestra.run.smithi026.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T20:56:51.613 INFO:teuthology.orchestra.run.smithi026.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T20:56:51.613 INFO:teuthology.orchestra.run.smithi026.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T20:56:51.613 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T20:56:51.613 INFO:teuthology.orchestra.run.smithi026.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T20:56:51.613 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T20:56:51.631 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:56:51.848 INFO:teuthology.orchestra.run.smithi070.stdout:ERROR: (22) Invalid argument 2023-09-23T20:56:51.848 INFO:teuthology.orchestra.run.smithi070.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:56:51.901 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 13 16 28217 28201 135.498 106 0.00673281 0.00737441 2023-09-23T20:56:52.107 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:56:52.113 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:56:52.114 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:56:52.215 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:56:52.276 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:56:52 smithi026 ceph-mon[179688]: pgmap v476: 99 pgs: 99 active+clean; 1.2 GiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 127 MiB/s wr, 2.02k op/s 2023-09-23T20:56:52.277 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:56:52 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/1566426108' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:56:52.278 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:56:52 smithi026 ceph-mon[175151]: pgmap v476: 99 pgs: 99 active+clean; 1.2 GiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 127 MiB/s wr, 2.02k op/s 2023-09-23T20:56:52.278 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:56:52 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/1566426108' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:56:52.282 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:56:52.309 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:56:52 smithi070 ceph-mon[168282]: pgmap v476: 99 pgs: 99 active+clean; 1.2 GiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 127 MiB/s wr, 2.02k op/s 2023-09-23T20:56:52.309 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:56:52 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/1566426108' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:56:52.324 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_blocked_ops 2023-09-23T20:56:52.375 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:56:52.378 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:56:52.379 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:56:52.479 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:56:52.542 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:56:52.670 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:56:52.675 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:56:52.676 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:56:52.777 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:56:52.816 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:56:52.861 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:56:52.864 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:56:52.864 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:56:52.902 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 14 16 30003 29987 133.793 111.625 0.0102845 0.00746866 2023-09-23T20:56:52.965 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:56:53.027 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:56:53.036 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:56:53.096 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:56:53.100 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:56:53.101 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:56:53.202 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:56:53.270 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:56:53.339 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:56:53.347 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:56:53.348 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:56:53.449 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:56:53.495 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:56:53.796 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:56:53.801 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:56:53.801 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:56:53.902 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:56:53.903 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 15 16 31838 31822 132.518 114.688 0.00415409 0.00754029 2023-09-23T20:56:53.941 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:56:53.996 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:56:54.001 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:56:54.001 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:56:54.082 INFO:teuthology.orchestra.run.smithi026.stderr:nodeep-scrub is unset 2023-09-23T20:56:54.102 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:56:54.161 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:56:54.198 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T20:56:54.334 INFO:teuthology.orchestra.run.smithi070.stdout:ERROR: (22) Invalid argument 2023-09-23T20:56:54.334 INFO:teuthology.orchestra.run.smithi070.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:56:54.359 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:56:54 smithi070 ceph-mon[168282]: pgmap v477: 99 pgs: 99 active+clean; 1.6 GiB data, 5.7 GiB used, 710 GiB / 715 GiB avail; 118 MiB/s wr, 1.88k op/s 2023-09-23T20:56:54.359 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:56:54 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/2903031504' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:56:54.359 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:56:54 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:56:54.383 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:56:54 smithi026 ceph-mon[175151]: pgmap v477: 99 pgs: 99 active+clean; 1.6 GiB data, 5.7 GiB used, 710 GiB / 715 GiB avail; 118 MiB/s wr, 1.88k op/s 2023-09-23T20:56:54.383 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:56:54 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/2903031504' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:56:54.383 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:56:54 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:56:54.384 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:56:54 smithi026 ceph-mon[179688]: pgmap v477: 99 pgs: 99 active+clean; 1.6 GiB data, 5.7 GiB used, 710 GiB / 715 GiB avail; 118 MiB/s wr, 1.88k op/s 2023-09-23T20:56:54.384 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:56:54 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/2903031504' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:56:54.384 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:56:54 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:56:54.462 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:56:54.466 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:56:54.467 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:56:54.567 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:56:54.664 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:56:54.712 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 6, 7, 4, 5] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 7, 6] 2023-09-23T20:56:54.712 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T20:56:54.712 INFO:tasks.thrashosds.thrasher:Growing pool .mgr 2023-09-23T20:56:54.712 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T20:56:54.730 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_historic_ops 2023-09-23T20:56:54.730 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:56:54.737 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:56:54.737 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:56:54.838 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:56:54.874 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:56:54.903 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 16 16 33414 33398 130.387 98.5 0.0194915 0.0076639 2023-09-23T20:56:54.932 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:56:54.938 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:56:54.939 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:56:54.992 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:56:55.039 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:56:55.048 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:56:55 smithi026 ceph-mon[175151]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T20:56:55.048 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:56:55 smithi026 ceph-mon[175151]: Cluster is now healthy 2023-09-23T20:56:55.048 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:56:55 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T20:56:55.048 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:56:55 smithi026 ceph-mon[175151]: osdmap e206: 8 total, 8 up, 8 in 2023-09-23T20:56:55.049 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:56:55 smithi026 ceph-mon[175151]: failed to encode map e206 with expected crc 2023-09-23T20:56:55.049 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:56:55 smithi026 ceph-mon[175151]: failed to encode map e206 with expected crc 2023-09-23T20:56:55.049 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:56:55 smithi026 ceph-mon[175151]: failed to encode map e206 with expected crc 2023-09-23T20:56:55.049 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:56:55 smithi026 ceph-mon[175151]: failed to encode map e206 with expected crc 2023-09-23T20:56:55.049 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:56:55 smithi026 ceph-mon[175151]: failed to encode map e206 with expected crc 2023-09-23T20:56:55.049 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:56:55 smithi026 ceph-mon[175151]: failed to encode map e206 with expected crc 2023-09-23T20:56:55.120 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:56:55.199 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:56:55.205 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:56:55.206 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:56:55.298 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:56:55 smithi026 ceph-mon[179688]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T20:56:55.298 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:56:55 smithi026 ceph-mon[179688]: Cluster is now healthy 2023-09-23T20:56:55.298 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:56:55 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T20:56:55.298 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:56:55 smithi026 ceph-mon[179688]: osdmap e206: 8 total, 8 up, 8 in 2023-09-23T20:56:55.298 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:56:55 smithi026 ceph-mon[179688]: failed to encode map e206 with expected crc 2023-09-23T20:56:55.298 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:56:55 smithi026 ceph-mon[179688]: failed to encode map e206 with expected crc 2023-09-23T20:56:55.298 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:56:55 smithi026 ceph-mon[179688]: failed to encode map e206 with expected crc 2023-09-23T20:56:55.298 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:56:55 smithi026 ceph-mon[179688]: failed to encode map e206 with expected crc 2023-09-23T20:56:55.298 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:56:55 smithi026 ceph-mon[179688]: failed to encode map e206 with expected crc 2023-09-23T20:56:55.298 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:56:55 smithi026 ceph-mon[179688]: failed to encode map e206 with expected crc 2023-09-23T20:56:55.307 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:56:55.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:56:55 smithi070 ceph-mon[168282]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T20:56:55.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:56:55 smithi070 ceph-mon[168282]: Cluster is now healthy 2023-09-23T20:56:55.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:56:55 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T20:56:55.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:56:55 smithi070 ceph-mon[168282]: osdmap e206: 8 total, 8 up, 8 in 2023-09-23T20:56:55.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:56:55 smithi070 ceph-mon[168282]: failed to encode map e206 with expected crc 2023-09-23T20:56:55.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:56:55 smithi070 ceph-mon[168282]: failed to encode map e206 with expected crc 2023-09-23T20:56:55.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:56:55 smithi070 ceph-mon[168282]: failed to encode map e206 with expected crc 2023-09-23T20:56:55.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:56:55 smithi070 ceph-mon[168282]: failed to encode map e206 with expected crc 2023-09-23T20:56:55.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:56:55 smithi070 ceph-mon[168282]: failed to encode map e206 with expected crc 2023-09-23T20:56:55.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:56:55 smithi070 ceph-mon[168282]: failed to encode map e206 with expected crc 2023-09-23T20:56:55.385 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:56:55.435 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:56:55.445 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:56:55.456 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:56:55.456 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:56:55.557 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:56:55.592 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:56:55.683 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:56:55.686 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:56:55.686 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:56:55.787 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:56:55.829 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:56:55.876 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:56:55.880 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:56:55.880 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:56:55.903 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 17 16 35001 34985 128.552 99.1875 0.00535464 0.00777352 2023-09-23T20:56:55.936 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:56:55.981 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:56:56.063 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:56:56.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:56:56 smithi070 ceph-mon[168282]: pgmap v479: 99 pgs: 99 active+clean; 1.8 GiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 143 MiB/s wr, 2.29k op/s 2023-09-23T20:56:56.379 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:56:56.383 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:56:56 smithi026 ceph-mon[175151]: pgmap v479: 99 pgs: 99 active+clean; 1.8 GiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 143 MiB/s wr, 2.29k op/s 2023-09-23T20:56:56.383 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:56:56 smithi026 ceph-mon[179688]: pgmap v479: 99 pgs: 99 active+clean; 1.8 GiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 143 MiB/s wr, 2.29k op/s 2023-09-23T20:56:56.384 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:56:56.384 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:56:56.485 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:56:56.546 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:56:56.565 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T20:56:56.903 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 18 16 36494 36478 126.595 93.3125 0.00452462 0.00789188 2023-09-23T20:56:57.418 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:56:57.424 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:56:57.424 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:56:57.525 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:56:57.592 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:56:57.903 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 19 16 37828 37812 124.32 83.375 0.00167065 0.00798532 2023-09-23T20:56:58.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:56:58 smithi070 ceph-mon[168282]: pgmap v480: 99 pgs: 99 active+clean; 1.9 GiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 118 MiB/s wr, 1.88k op/s 2023-09-23T20:56:58.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:56:58 smithi026 ceph-mon[175151]: pgmap v480: 99 pgs: 99 active+clean; 1.9 GiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 118 MiB/s wr, 1.88k op/s 2023-09-23T20:56:58.387 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:56:58 smithi026 ceph-mon[179688]: pgmap v480: 99 pgs: 99 active+clean; 1.9 GiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 118 MiB/s wr, 1.88k op/s 2023-09-23T20:56:58.405 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:56:58.412 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:56:58.413 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:56:58.514 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:56:58.580 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:56:58.903 INFO:tasks.radosbench.radosbench.0.smithi026.stdout:2023-09-23T20:56:58.903861+0000 min lat: 0.0011341 max lat: 1.0132 avg lat: 0.00815713 2023-09-23T20:56:58.903 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T20:56:58.903 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 20 16 39237 39221 122.508 88.0625 0.00563008 0.00815713 2023-09-23T20:56:59.231 INFO:teuthology.orchestra.run.smithi070.stdout:ERROR: (22) Invalid argument 2023-09-23T20:56:59.232 INFO:teuthology.orchestra.run.smithi070.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:56:59.258 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:56:59 smithi070 ceph-mon[168282]: 1.8 scrub starts 2023-09-23T20:56:59.258 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:56:59 smithi070 ceph-mon[168282]: 1.8 scrub ok 2023-09-23T20:56:59.381 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:56:59 smithi026 ceph-mon[179688]: 1.8 scrub starts 2023-09-23T20:56:59.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:56:59 smithi026 ceph-mon[179688]: 1.8 scrub ok 2023-09-23T20:56:59.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:56:59 smithi026 ceph-mon[175151]: 1.8 scrub starts 2023-09-23T20:56:59.383 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:56:59 smithi026 ceph-mon[175151]: 1.8 scrub ok 2023-09-23T20:56:59.637 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_ops_in_flight 2023-09-23T20:56:59.720 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:56:59 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[164029]: ::ffff:172.21.15.70 - - [23/Sep/2023:20:56:59] "GET /metrics HTTP/1.1" 200 36318 "" "Prometheus/2.43.0" 2023-09-23T20:56:59.905 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 21 16 40929 40913 121.7 105.75 0.0054398 0.00821211 2023-09-23T20:57:00.283 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:57:00.290 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:57:00.291 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:57:00.297 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:57:00.361 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:57:00.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:57:00 smithi070 ceph-mon[168282]: pgmap v481: 99 pgs: 99 active+clean; 2.1 GiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 116 MiB/s wr, 1.85k op/s 2023-09-23T20:57:00.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:57:00 smithi026 ceph-mon[175151]: pgmap v481: 99 pgs: 99 active+clean; 2.1 GiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 116 MiB/s wr, 1.85k op/s 2023-09-23T20:57:00.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:57:00 smithi026 ceph-mon[179688]: pgmap v481: 99 pgs: 99 active+clean; 2.1 GiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 116 MiB/s wr, 1.85k op/s 2023-09-23T20:57:00.391 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:57:00.428 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0: {} 2023-09-23T20:57:00.463 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:57:00.466 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1: {} 2023-09-23T20:57:00.510 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2: {} 2023-09-23T20:57:00.541 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:57:00.544 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:57:00.544 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:57:00.549 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3: {} 2023-09-23T20:57:00.590 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4: {} 2023-09-23T20:57:00.623 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T20:57:00.646 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:57:00.661 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6: {} 2023-09-23T20:57:00.701 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T20:57:00.716 INFO:teuthology.orchestra.run.smithi026.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T20:57:00.716 INFO:teuthology.orchestra.run.smithi026.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T20:57:00.717 INFO:teuthology.orchestra.run.smithi026.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T20:57:00.717 INFO:teuthology.orchestra.run.smithi026.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T20:57:00.717 INFO:teuthology.orchestra.run.smithi026.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T20:57:00.717 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T20:57:00.717 INFO:teuthology.orchestra.run.smithi026.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T20:57:00.717 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T20:57:00.722 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:57:00.905 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 22 16 42231 42215 119.867 81.375 0.0198284 0.00833666 2023-09-23T20:57:01.018 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:57:01.021 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:57:01.022 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:57:01.122 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:57:01.162 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:57:01.208 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:57:01.211 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:57:01.211 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:57:01.312 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:57:01.327 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:57:01 smithi026 ceph-mon[179688]: pgmap v482: 99 pgs: 99 active+clean; 2.3 GiB data, 7.8 GiB used, 707 GiB / 715 GiB avail; 111 MiB/s wr, 1.77k op/s 2023-09-23T20:57:01.327 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:57:01 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/325421753' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:57:01.327 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:57:01 smithi026 ceph-mon[175151]: pgmap v482: 99 pgs: 99 active+clean; 2.3 GiB data, 7.8 GiB used, 707 GiB / 715 GiB avail; 111 MiB/s wr, 1.77k op/s 2023-09-23T20:57:01.328 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:57:01 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/325421753' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:57:01.347 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:57:01 smithi070 ceph-mon[168282]: pgmap v482: 99 pgs: 99 active+clean; 2.3 GiB data, 7.8 GiB used, 707 GiB / 715 GiB avail; 111 MiB/s wr, 1.77k op/s 2023-09-23T20:57:01.347 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:57:01 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/325421753' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:57:01.354 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:57:01.403 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:57:01.406 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:57:01.407 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:57:01.507 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:57:01.551 INFO:teuthology.orchestra.run.smithi026.stdout: 2023-09-23T20:57:01.554 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:57:01.554 INFO:teuthology.orchestra.run.smithi026.stderr:dumped all 2023-09-23T20:57:01.653 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:57:01.656 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:57:01.657 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:57:01.751 INFO:teuthology.orchestra.run.smithi070.stdout:{ 2023-09-23T20:57:01.751 INFO:teuthology.orchestra.run.smithi070.stdout: "ops": [], 2023-09-23T20:57:01.751 INFO:teuthology.orchestra.run.smithi070.stdout: "num_ops": 0 2023-09-23T20:57:01.752 INFO:teuthology.orchestra.run.smithi070.stdout:} 2023-09-23T20:57:01.758 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:57:01.795 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:57:01.846 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:57:01.860 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:57:01.861 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:57:01.905 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 23 16 43591 43575 118.351 85 0.0106097 0.00844341 2023-09-23T20:57:01.962 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:57:02.044 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:57:02.147 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_blocked_ops 2023-09-23T20:57:02.161 INFO:teuthology.orchestra.run.smithi026.stdout:{"pg_ready":true,"pg_map":{"version":482,"stamp":"2023-09-23T20:57:00.323284+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":2450033265,"num_objects":37401,"num_object_clones":0,"num_object_copies":112203,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":37401,"num_whiteouts":0,"num_read":133,"num_read_kb":248,"num_write":37503,"num_write_kb":2392815,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":751706,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":39203,"ondisk_log_size":39203,"up":297,"acting":297,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":298,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":8223420,"kb_used_data":7093564,"kb_used_omap":136,"kb_used_meta":1129655,"kb_avail":741737796,"statfs":{"total":767960285184,"available":759539503104,"internally_reserved":0,"allocated":7263809536,"data_stored":7257139453,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":139741,"internal_metadata":1156767267},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":4,"apply_latency_ms":4,"commit_latency_ns":4000000,"apply_latency_ns":4000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":1165569626,"num_objects":17785,"num_object_clones":0,"num_object_copies":53355,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":17785,"num_whiteouts":0,"num_read":-45,"num_read_kb":-83,"num_write":17764,"num_write_kb":1138084,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":-1,"num_bytes_recovered":-161010,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.004386"},"pg_stats":[{"pgid":"3.1a","version":"68'1","reported_seq":83,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-09-23T20:56:45.987015+0000","last_change":"2023-09-23T20:54:13.946695+0000","last_active":"2023-09-23T20:56:45.987015+0000","last_peered":"2023-09-23T20:56:45.987015+0000","last_clean":"2023-09-23T20:56:45.987015+0000","last_became_active":"2023-09-23T20:54:13.945399+0000","last_became_peered":"2023-09-23T20:54:13.945399+0000","last_unstale":"2023-09-23T20:56:45.987015+0000","last_undegraded":"2023-09-23T20:56:45.987015+0000","last_fullsized":"2023-09-23T20:56:45.987015+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:32:52.365464+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":197,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-09-23T20:56:45.977830+0000","last_change":"2023-09-23T20:54:13.942019+0000","last_active":"2023-09-23T20:56:45.977830+0000","last_peered":"2023-09-23T20:56:45.977830+0000","last_clean":"2023-09-23T20:56:45.977830+0000","last_became_active":"2023-09-23T20:54:13.941647+0000","last_became_peered":"2023-09-23T20:54:13.941647+0000","last_unstale":"2023-09-23T20:56:45.977830+0000","last_undegraded":"2023-09-23T20:56:45.977830+0000","last_fullsized":"2023-09-23T20:56:45.977830+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:30:54.400944+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":174,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-09-23T20:56:45.982867+0000","last_change":"2023-09-23T20:54:13.941394+0000","last_active":"2023-09-23T20:56:45.982867+0000","last_peered":"2023-09-23T20:56:45.982867+0000","last_clean":"2023-09-23T20:56:45.982867+0000","last_became_active":"2023-09-23T20:54:13.941042+0000","last_became_peered":"2023-09-23T20:54:13.941042+0000","last_unstale":"2023-09-23T20:56:45.982867+0000","last_undegraded":"2023-09-23T20:56:45.982867+0000","last_fullsized":"2023-09-23T20:56:45.982867+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:19:41.401184+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":180,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-09-23T20:56:45.982254+0000","last_change":"2023-09-23T20:56:18.138025+0000","last_active":"2023-09-23T20:56:45.982254+0000","last_peered":"2023-09-23T20:56:45.982254+0000","last_clean":"2023-09-23T20:56:45.982254+0000","last_became_active":"2023-09-23T20:56:18.137732+0000","last_became_peered":"2023-09-23T20:56:18.137732+0000","last_unstale":"2023-09-23T20:56:45.982254+0000","last_undegraded":"2023-09-23T20:56:45.982254+0000","last_fullsized":"2023-09-23T20:56:45.982254+0000","mapping_epoch":192,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":193,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:35:41.410004+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,5],"acting":[0,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":188,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-09-23T20:56:45.977635+0000","last_change":"2023-09-23T20:51:10.833613+0000","last_active":"2023-09-23T20:56:45.977635+0000","last_peered":"2023-09-23T20:56:45.977635+0000","last_clean":"2023-09-23T20:56:45.977635+0000","last_became_active":"2023-09-23T20:51:10.833208+0000","last_became_peered":"2023-09-23T20:51:10.833208+0000","last_unstale":"2023-09-23T20:56:45.977635+0000","last_undegraded":"2023-09-23T20:56:45.977635+0000","last_fullsized":"2023-09-23T20:56:45.977635+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:51:28.246014+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":171,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-09-23T20:56:45.981513+0000","last_change":"2023-09-23T20:55:35.958384+0000","last_active":"2023-09-23T20:56:45.981513+0000","last_peered":"2023-09-23T20:56:45.981513+0000","last_clean":"2023-09-23T20:56:45.981513+0000","last_became_active":"2023-09-23T20:55:35.954563+0000","last_became_peered":"2023-09-23T20:55:35.954563+0000","last_unstale":"2023-09-23T20:56:45.981513+0000","last_undegraded":"2023-09-23T20:56:45.981513+0000","last_fullsized":"2023-09-23T20:56:45.981513+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:37:48.332259+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":180,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-09-23T20:56:46.040247+0000","last_change":"2023-09-23T20:54:31.885514+0000","last_active":"2023-09-23T20:56:46.040247+0000","last_peered":"2023-09-23T20:56:46.040247+0000","last_clean":"2023-09-23T20:56:46.040247+0000","last_became_active":"2023-09-23T20:54:31.884986+0000","last_became_peered":"2023-09-23T20:54:31.884986+0000","last_unstale":"2023-09-23T20:56:46.040247+0000","last_undegraded":"2023-09-23T20:56:46.040247+0000","last_fullsized":"2023-09-23T20:56:46.040247+0000","mapping_epoch":136,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:26:08.100220+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":185,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-09-23T20:56:45.977634+0000","last_change":"2023-09-23T20:55:35.951889+0000","last_active":"2023-09-23T20:56:45.977634+0000","last_peered":"2023-09-23T20:56:45.977634+0000","last_clean":"2023-09-23T20:56:45.977634+0000","last_became_active":"2023-09-23T20:55:35.951563+0000","last_became_peered":"2023-09-23T20:55:35.951563+0000","last_unstale":"2023-09-23T20:56:45.977634+0000","last_undegraded":"2023-09-23T20:56:45.977634+0000","last_fullsized":"2023-09-23T20:56:45.977634+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:41:53.098781+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":79,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-09-23T20:56:45.983856+0000","last_change":"2023-09-23T20:54:31.890678+0000","last_active":"2023-09-23T20:56:45.983856+0000","last_peered":"2023-09-23T20:56:45.983856+0000","last_clean":"2023-09-23T20:56:45.983856+0000","last_became_active":"2023-09-23T20:54:31.890469+0000","last_became_peered":"2023-09-23T20:54:31.890469+0000","last_unstale":"2023-09-23T20:56:45.983856+0000","last_undegraded":"2023-09-23T20:56:45.983856+0000","last_fullsized":"2023-09-23T20:56:45.983856+0000","mapping_epoch":136,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:55:08.790146+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":146,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-09-23T20:56:45.977673+0000","last_change":"2023-09-23T20:55:35.952661+0000","last_active":"2023-09-23T20:56:45.977673+0000","last_peered":"2023-09-23T20:56:45.977673+0000","last_clean":"2023-09-23T20:56:45.977673+0000","last_became_active":"2023-09-23T20:55:35.952349+0000","last_became_peered":"2023-09-23T20:55:35.952349+0000","last_unstale":"2023-09-23T20:56:45.977673+0000","last_undegraded":"2023-09-23T20:56:45.977673+0000","last_fullsized":"2023-09-23T20:56:45.977673+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:00:03.110126+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":23,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-09-23T20:56:49.254365+0000","last_change":"2023-09-23T20:56:45.988607+0000","last_active":"2023-09-23T20:56:49.254365+0000","last_peered":"2023-09-23T20:56:49.254365+0000","last_clean":"2023-09-23T20:56:49.254365+0000","last_became_active":"2023-09-23T20:56:45.988206+0000","last_became_peered":"2023-09-23T20:56:45.988206+0000","last_unstale":"2023-09-23T20:56:49.254365+0000","last_undegraded":"2023-09-23T20:56:49.254365+0000","last_fullsized":"2023-09-23T20:56:49.254365+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":205,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:01:19.783688+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,4],"acting":[3,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":23,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-09-23T20:56:45.984194+0000","last_change":"2023-09-23T20:56:18.139142+0000","last_active":"2023-09-23T20:56:45.984194+0000","last_peered":"2023-09-23T20:56:45.984194+0000","last_clean":"2023-09-23T20:56:45.984194+0000","last_became_active":"2023-09-23T20:56:18.138844+0000","last_became_peered":"2023-09-23T20:56:18.138844+0000","last_unstale":"2023-09-23T20:56:45.984194+0000","last_undegraded":"2023-09-23T20:56:45.984194+0000","last_fullsized":"2023-09-23T20:56:45.984194+0000","mapping_epoch":192,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":193,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:09:02.217683+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,2],"acting":[5,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.14","version":"79'22","reported_seq":110,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-09-23T20:56:45.987002+0000","last_change":"2023-09-23T20:55:35.949418+0000","last_active":"2023-09-23T20:56:45.987002+0000","last_peered":"2023-09-23T20:56:45.987002+0000","last_clean":"2023-09-23T20:56:45.987002+0000","last_became_active":"2023-09-23T20:55:35.948566+0000","last_became_peered":"2023-09-23T20:55:35.948566+0000","last_unstale":"2023-09-23T20:56:45.987002+0000","last_undegraded":"2023-09-23T20:56:45.987002+0000","last_fullsized":"2023-09-23T20:56:45.987002+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:59:37.433128+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4734,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":100,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-09-23T20:56:45.987794+0000","last_change":"2023-09-23T20:55:35.949178+0000","last_active":"2023-09-23T20:56:45.987794+0000","last_peered":"2023-09-23T20:56:45.987794+0000","last_clean":"2023-09-23T20:56:45.987794+0000","last_became_active":"2023-09-23T20:55:35.948795+0000","last_became_peered":"2023-09-23T20:55:35.948795+0000","last_unstale":"2023-09-23T20:56:45.987794+0000","last_undegraded":"2023-09-23T20:56:45.987794+0000","last_fullsized":"2023-09-23T20:56:45.987794+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:13:08.996932+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":126,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-09-23T20:56:45.993493+0000","last_change":"2023-09-23T20:56:45.992577+0000","last_active":"2023-09-23T20:56:45.993493+0000","last_peered":"2023-09-23T20:56:45.993493+0000","last_clean":"2023-09-23T20:56:45.993493+0000","last_became_active":"2023-09-23T20:56:45.992297+0000","last_became_peered":"2023-09-23T20:56:45.992297+0000","last_unstale":"2023-09-23T20:56:45.993493+0000","last_undegraded":"2023-09-23T20:56:45.993493+0000","last_fullsized":"2023-09-23T20:56:45.993493+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":205,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:00:45.547202+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,2],"acting":[4,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":170,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-09-23T20:56:45.977861+0000","last_change":"2023-09-23T20:51:10.831262+0000","last_active":"2023-09-23T20:56:45.977861+0000","last_peered":"2023-09-23T20:56:45.977861+0000","last_clean":"2023-09-23T20:56:45.977861+0000","last_became_active":"2023-09-23T20:51:10.830931+0000","last_became_peered":"2023-09-23T20:51:10.830931+0000","last_unstale":"2023-09-23T20:56:45.977861+0000","last_undegraded":"2023-09-23T20:56:45.977861+0000","last_fullsized":"2023-09-23T20:56:45.977861+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:51:54.982505+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":171,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-09-23T20:56:45.982357+0000","last_change":"2023-09-23T20:54:44.037431+0000","last_active":"2023-09-23T20:56:45.982357+0000","last_peered":"2023-09-23T20:56:45.982357+0000","last_clean":"2023-09-23T20:56:45.982357+0000","last_became_active":"2023-09-23T20:54:44.037102+0000","last_became_peered":"2023-09-23T20:54:44.037102+0000","last_unstale":"2023-09-23T20:56:45.982357+0000","last_undegraded":"2023-09-23T20:56:45.982357+0000","last_fullsized":"2023-09-23T20:56:45.982357+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:12:11.074632+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":79,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-09-23T20:56:45.983971+0000","last_change":"2023-09-23T20:54:31.887814+0000","last_active":"2023-09-23T20:56:45.983971+0000","last_peered":"2023-09-23T20:56:45.983971+0000","last_clean":"2023-09-23T20:56:45.983971+0000","last_became_active":"2023-09-23T20:54:31.887195+0000","last_became_peered":"2023-09-23T20:54:31.887195+0000","last_unstale":"2023-09-23T20:56:45.983971+0000","last_undegraded":"2023-09-23T20:56:45.983971+0000","last_fullsized":"2023-09-23T20:56:45.983971+0000","mapping_epoch":136,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:10:52.089553+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":150,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-09-23T20:56:45.992093+0000","last_change":"2023-09-23T20:56:45.991822+0000","last_active":"2023-09-23T20:56:45.992093+0000","last_peered":"2023-09-23T20:56:45.992093+0000","last_clean":"2023-09-23T20:56:45.992093+0000","last_became_active":"2023-09-23T20:56:45.991460+0000","last_became_peered":"2023-09-23T20:56:45.991460+0000","last_unstale":"2023-09-23T20:56:45.992093+0000","last_undegraded":"2023-09-23T20:56:45.992093+0000","last_fullsized":"2023-09-23T20:56:45.992093+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":205,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:20:37.503317+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":162,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-09-23T20:56:46.040215+0000","last_change":"2023-09-23T20:51:10.840849+0000","last_active":"2023-09-23T20:56:46.040215+0000","last_peered":"2023-09-23T20:56:46.040215+0000","last_clean":"2023-09-23T20:56:46.040215+0000","last_became_active":"2023-09-23T20:51:10.840593+0000","last_became_peered":"2023-09-23T20:51:10.840593+0000","last_unstale":"2023-09-23T20:56:46.040215+0000","last_undegraded":"2023-09-23T20:56:46.040215+0000","last_fullsized":"2023-09-23T20:56:46.040215+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:30:52.297625+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":198,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-09-23T20:56:45.980674+0000","last_change":"2023-09-23T20:55:36.943969+0000","last_active":"2023-09-23T20:56:45.980674+0000","last_peered":"2023-09-23T20:56:45.980674+0000","last_clean":"2023-09-23T20:56:45.980674+0000","last_became_active":"2023-09-23T20:55:36.943580+0000","last_became_peered":"2023-09-23T20:55:36.943580+0000","last_unstale":"2023-09-23T20:56:45.980674+0000","last_undegraded":"2023-09-23T20:56:45.980674+0000","last_fullsized":"2023-09-23T20:56:45.980674+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:49:06.969396+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":69,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-09-23T20:56:49.223407+0000","last_change":"2023-09-23T20:56:45.991736+0000","last_active":"2023-09-23T20:56:49.223407+0000","last_peered":"2023-09-23T20:56:49.223407+0000","last_clean":"2023-09-23T20:56:49.223407+0000","last_became_active":"2023-09-23T20:56:45.991390+0000","last_became_peered":"2023-09-23T20:56:45.991390+0000","last_unstale":"2023-09-23T20:56:49.223407+0000","last_undegraded":"2023-09-23T20:56:49.223407+0000","last_fullsized":"2023-09-23T20:56:49.223407+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":205,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:33:12.371872+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,2],"acting":[1,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.b","version":"206'2427","reported_seq":2443,"reported_epoch":206,"state":"active+clean","last_fresh":"2023-09-23T20:56:59.180425+0000","last_change":"2023-09-23T20:56:35.543994+0000","last_active":"2023-09-23T20:56:59.180425+0000","last_peered":"2023-09-23T20:56:59.180425+0000","last_clean":"2023-09-23T20:56:59.180425+0000","last_became_active":"2023-09-23T20:56:35.543758+0000","last_became_peered":"2023-09-23T20:56:35.543758+0000","last_unstale":"2023-09-23T20:56:59.180425+0000","last_undegraded":"2023-09-23T20:56:59.180425+0000","last_fullsized":"2023-09-23T20:56:59.180425+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":199,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_clean_scrub_stamp":"2023-09-23T20:56:34.529103+0000","objects_scrubbed":0,"log_size":2427,"log_dups_size":0,"ondisk_log_size":2427,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:21:29.041484+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":159055872,"num_objects":2427,"num_object_clones":0,"num_object_copies":7281,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2427,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2427,"num_write_kb":155328,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":137,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-09-23T20:56:45.992641+0000","last_change":"2023-09-23T20:56:45.991841+0000","last_active":"2023-09-23T20:56:45.992641+0000","last_peered":"2023-09-23T20:56:45.992641+0000","last_clean":"2023-09-23T20:56:45.992641+0000","last_became_active":"2023-09-23T20:56:45.991491+0000","last_became_peered":"2023-09-23T20:56:45.991491+0000","last_unstale":"2023-09-23T20:56:45.992641+0000","last_undegraded":"2023-09-23T20:56:45.992641+0000","last_fullsized":"2023-09-23T20:56:45.992641+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":205,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:30:37.405319+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":96,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-09-23T20:56:45.983791+0000","last_change":"2023-09-23T20:55:35.957768+0000","last_active":"2023-09-23T20:56:45.983791+0000","last_peered":"2023-09-23T20:56:45.983791+0000","last_clean":"2023-09-23T20:56:45.983791+0000","last_became_active":"2023-09-23T20:55:35.957421+0000","last_became_peered":"2023-09-23T20:55:35.957421+0000","last_unstale":"2023-09-23T20:56:45.983791+0000","last_undegraded":"2023-09-23T20:56:45.983791+0000","last_fullsized":"2023-09-23T20:56:45.983791+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:49:41.585961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"7.a","version":"206'2495","reported_seq":2511,"reported_epoch":206,"state":"active+clean","last_fresh":"2023-09-23T20:56:59.253382+0000","last_change":"2023-09-23T20:56:35.547120+0000","last_active":"2023-09-23T20:56:59.253382+0000","last_peered":"2023-09-23T20:56:59.253382+0000","last_clean":"2023-09-23T20:56:59.253382+0000","last_became_active":"2023-09-23T20:56:35.546838+0000","last_became_peered":"2023-09-23T20:56:35.546838+0000","last_unstale":"2023-09-23T20:56:59.253382+0000","last_undegraded":"2023-09-23T20:56:59.253382+0000","last_fullsized":"2023-09-23T20:56:59.253382+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":199,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_clean_scrub_stamp":"2023-09-23T20:56:34.529103+0000","objects_scrubbed":0,"log_size":2495,"log_dups_size":0,"ondisk_log_size":2495,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:48:24.132768+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":163512320,"num_objects":2495,"num_object_clones":0,"num_object_copies":7485,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2495,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2495,"num_write_kb":159680,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,5],"acting":[3,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":124,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-09-23T20:56:45.992120+0000","last_change":"2023-09-23T20:56:45.991821+0000","last_active":"2023-09-23T20:56:45.992120+0000","last_peered":"2023-09-23T20:56:45.992120+0000","last_clean":"2023-09-23T20:56:45.992120+0000","last_became_active":"2023-09-23T20:56:45.991447+0000","last_became_peered":"2023-09-23T20:56:45.991447+0000","last_unstale":"2023-09-23T20:56:45.992120+0000","last_undegraded":"2023-09-23T20:56:45.992120+0000","last_fullsized":"2023-09-23T20:56:45.992120+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":205,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:14:25.525222+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":117,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-09-23T20:56:45.985756+0000","last_change":"2023-09-23T20:55:36.943955+0000","last_active":"2023-09-23T20:56:45.985756+0000","last_peered":"2023-09-23T20:56:45.985756+0000","last_clean":"2023-09-23T20:56:45.985756+0000","last_became_active":"2023-09-23T20:55:36.943639+0000","last_became_peered":"2023-09-23T20:55:36.943639+0000","last_unstale":"2023-09-23T20:56:45.985756+0000","last_undegraded":"2023-09-23T20:56:45.985756+0000","last_fullsized":"2023-09-23T20:56:45.985756+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:06:13.742269+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.9","version":"206'2198","reported_seq":2214,"reported_epoch":206,"state":"active+clean","last_fresh":"2023-09-23T20:56:56.031843+0000","last_change":"2023-09-23T20:56:35.549387+0000","last_active":"2023-09-23T20:56:56.031843+0000","last_peered":"2023-09-23T20:56:56.031843+0000","last_clean":"2023-09-23T20:56:56.031843+0000","last_became_active":"2023-09-23T20:56:35.549087+0000","last_became_peered":"2023-09-23T20:56:35.549087+0000","last_unstale":"2023-09-23T20:56:56.031843+0000","last_undegraded":"2023-09-23T20:56:56.031843+0000","last_fullsized":"2023-09-23T20:56:56.031843+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":199,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_clean_scrub_stamp":"2023-09-23T20:56:34.529103+0000","objects_scrubbed":0,"log_size":2198,"log_dups_size":0,"ondisk_log_size":2198,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:04:51.547362+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":144048128,"num_objects":2198,"num_object_clones":0,"num_object_copies":6594,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2198,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2198,"num_write_kb":140672,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":110,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-09-23T20:56:45.989614+0000","last_change":"2023-09-23T20:56:45.989153+0000","last_active":"2023-09-23T20:56:45.989614+0000","last_peered":"2023-09-23T20:56:45.989614+0000","last_clean":"2023-09-23T20:56:45.989614+0000","last_became_active":"2023-09-23T20:56:45.988800+0000","last_became_peered":"2023-09-23T20:56:45.988800+0000","last_unstale":"2023-09-23T20:56:45.989614+0000","last_undegraded":"2023-09-23T20:56:45.989614+0000","last_fullsized":"2023-09-23T20:56:45.989614+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":205,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:04:41.130194+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":145,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-09-23T20:56:45.980595+0000","last_change":"2023-09-23T20:55:35.965427+0000","last_active":"2023-09-23T20:56:45.980595+0000","last_peered":"2023-09-23T20:56:45.980595+0000","last_clean":"2023-09-23T20:56:45.980595+0000","last_became_active":"2023-09-23T20:55:35.963969+0000","last_became_peered":"2023-09-23T20:55:35.963969+0000","last_unstale":"2023-09-23T20:56:45.980595+0000","last_undegraded":"2023-09-23T20:56:45.980595+0000","last_fullsized":"2023-09-23T20:56:45.980595+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:38:40.612083+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.8","version":"206'2242","reported_seq":2258,"reported_epoch":206,"state":"active+clean","last_fresh":"2023-09-23T20:56:56.322387+0000","last_change":"2023-09-23T20:56:35.548119+0000","last_active":"2023-09-23T20:56:56.322387+0000","last_peered":"2023-09-23T20:56:56.322387+0000","last_clean":"2023-09-23T20:56:56.322387+0000","last_became_active":"2023-09-23T20:56:35.547914+0000","last_became_peered":"2023-09-23T20:56:35.547914+0000","last_unstale":"2023-09-23T20:56:56.322387+0000","last_undegraded":"2023-09-23T20:56:56.322387+0000","last_fullsized":"2023-09-23T20:56:56.322387+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":199,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_clean_scrub_stamp":"2023-09-23T20:56:34.529103+0000","objects_scrubbed":0,"log_size":2242,"log_dups_size":0,"ondisk_log_size":2242,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:42:54.255023+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":146931712,"num_objects":2242,"num_object_clones":0,"num_object_copies":6726,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2242,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2242,"num_write_kb":143488,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":96,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-09-23T20:56:45.983646+0000","last_change":"2023-09-23T20:55:35.962005+0000","last_active":"2023-09-23T20:56:45.983646+0000","last_peered":"2023-09-23T20:56:45.983646+0000","last_clean":"2023-09-23T20:56:45.983646+0000","last_became_active":"2023-09-23T20:55:35.961695+0000","last_became_peered":"2023-09-23T20:55:35.961695+0000","last_unstale":"2023-09-23T20:56:45.983646+0000","last_undegraded":"2023-09-23T20:56:45.983646+0000","last_fullsized":"2023-09-23T20:56:45.983646+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:28:22.876201+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":193,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-09-23T20:56:45.982586+0000","last_change":"2023-09-23T20:54:13.928715+0000","last_active":"2023-09-23T20:56:45.982586+0000","last_peered":"2023-09-23T20:56:45.982586+0000","last_clean":"2023-09-23T20:56:45.982586+0000","last_became_active":"2023-09-23T20:54:13.928200+0000","last_became_peered":"2023-09-23T20:54:13.928200+0000","last_unstale":"2023-09-23T20:56:45.982586+0000","last_undegraded":"2023-09-23T20:56:45.982586+0000","last_fullsized":"2023-09-23T20:56:45.982586+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:41:16.927513+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.f","version":"206'2277","reported_seq":2293,"reported_epoch":206,"state":"active+clean","last_fresh":"2023-09-23T20:56:56.030065+0000","last_change":"2023-09-23T20:56:35.545128+0000","last_active":"2023-09-23T20:56:56.030065+0000","last_peered":"2023-09-23T20:56:56.030065+0000","last_clean":"2023-09-23T20:56:56.030065+0000","last_became_active":"2023-09-23T20:56:35.544838+0000","last_became_peered":"2023-09-23T20:56:35.544838+0000","last_unstale":"2023-09-23T20:56:56.030065+0000","last_undegraded":"2023-09-23T20:56:56.030065+0000","last_fullsized":"2023-09-23T20:56:56.030065+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":199,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_clean_scrub_stamp":"2023-09-23T20:56:34.529103+0000","objects_scrubbed":0,"log_size":2277,"log_dups_size":0,"ondisk_log_size":2277,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:30:43.388717+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":149225472,"num_objects":2277,"num_object_clones":0,"num_object_copies":6831,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2277,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2277,"num_write_kb":145728,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.9","version":"79'161","reported_seq":578,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-09-23T20:56:49.254258+0000","last_change":"2023-09-23T20:56:46.921939+0000","last_active":"2023-09-23T20:56:49.254258+0000","last_peered":"2023-09-23T20:56:49.254258+0000","last_clean":"2023-09-23T20:56:49.254258+0000","last_became_active":"2023-09-23T20:56:45.989908+0000","last_became_peered":"2023-09-23T20:56:45.989908+0000","last_unstale":"2023-09-23T20:56:49.254258+0000","last_undegraded":"2023-09-23T20:56:49.254258+0000","last_fullsized":"2023-09-23T20:56:49.254258+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":205,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:56:46.921739+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:56:46.921739+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:20:42.635110+0000","scrub_duration":0.0071954109999999996,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":194,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-09-23T20:56:45.978388+0000","last_change":"2023-09-23T20:54:13.961788+0000","last_active":"2023-09-23T20:56:45.978388+0000","last_peered":"2023-09-23T20:56:45.978388+0000","last_clean":"2023-09-23T20:56:45.978388+0000","last_became_active":"2023-09-23T20:54:13.940937+0000","last_became_peered":"2023-09-23T20:54:13.940937+0000","last_unstale":"2023-09-23T20:56:45.978388+0000","last_undegraded":"2023-09-23T20:56:45.978388+0000","last_fullsized":"2023-09-23T20:56:45.978388+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:11:56.785299+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":100,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-09-23T20:56:45.987398+0000","last_change":"2023-09-23T20:55:35.949670+0000","last_active":"2023-09-23T20:56:45.987398+0000","last_peered":"2023-09-23T20:56:45.987398+0000","last_clean":"2023-09-23T20:56:45.987398+0000","last_became_active":"2023-09-23T20:55:35.949140+0000","last_became_peered":"2023-09-23T20:55:35.949140+0000","last_unstale":"2023-09-23T20:56:45.987398+0000","last_undegraded":"2023-09-23T20:56:45.987398+0000","last_fullsized":"2023-09-23T20:56:45.987398+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:55:59.340303+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.e","version":"206'2220","reported_seq":2236,"reported_epoch":206,"state":"active+clean","last_fresh":"2023-09-23T20:56:56.307235+0000","last_change":"2023-09-23T20:56:35.547184+0000","last_active":"2023-09-23T20:56:56.307235+0000","last_peered":"2023-09-23T20:56:56.307235+0000","last_clean":"2023-09-23T20:56:56.307235+0000","last_became_active":"2023-09-23T20:56:35.546912+0000","last_became_peered":"2023-09-23T20:56:35.546912+0000","last_unstale":"2023-09-23T20:56:56.307235+0000","last_undegraded":"2023-09-23T20:56:56.307235+0000","last_fullsized":"2023-09-23T20:56:56.307235+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":199,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_clean_scrub_stamp":"2023-09-23T20:56:34.529103+0000","objects_scrubbed":0,"log_size":2220,"log_dups_size":0,"ondisk_log_size":2220,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:57:07.422792+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":145489920,"num_objects":2220,"num_object_clones":0,"num_object_copies":6660,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2220,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2220,"num_write_kb":142080,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,0],"acting":[2,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.8","version":"79'161","reported_seq":439,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-09-23T20:56:45.983537+0000","last_change":"2023-09-23T20:56:04.982780+0000","last_active":"2023-09-23T20:56:45.983537+0000","last_peered":"2023-09-23T20:56:45.983537+0000","last_clean":"2023-09-23T20:56:45.983537+0000","last_became_active":"2023-09-23T20:56:04.982345+0000","last_became_peered":"2023-09-23T20:56:04.982345+0000","last_unstale":"2023-09-23T20:56:45.983537+0000","last_undegraded":"2023-09-23T20:56:45.983537+0000","last_fullsized":"2023-09-23T20:56:45.983537+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:38:57.324224+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":44,"num_read_kb":83,"num_write":22,"num_write_kb":219,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":423697,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":168,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-09-23T20:56:45.980774+0000","last_change":"2023-09-23T20:55:35.959942+0000","last_active":"2023-09-23T20:56:45.980774+0000","last_peered":"2023-09-23T20:56:45.980774+0000","last_clean":"2023-09-23T20:56:45.980774+0000","last_became_active":"2023-09-23T20:55:35.957071+0000","last_became_peered":"2023-09-23T20:55:35.957071+0000","last_unstale":"2023-09-23T20:56:45.980774+0000","last_undegraded":"2023-09-23T20:56:45.980774+0000","last_fullsized":"2023-09-23T20:56:45.980774+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:44:30.308036+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":83,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-09-23T20:56:45.985923+0000","last_change":"2023-09-23T20:54:13.951459+0000","last_active":"2023-09-23T20:56:45.985923+0000","last_peered":"2023-09-23T20:56:45.985923+0000","last_clean":"2023-09-23T20:56:45.985923+0000","last_became_active":"2023-09-23T20:54:13.948409+0000","last_became_peered":"2023-09-23T20:54:13.948409+0000","last_unstale":"2023-09-23T20:56:45.985923+0000","last_undegraded":"2023-09-23T20:56:45.985923+0000","last_fullsized":"2023-09-23T20:56:45.985923+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:38:10.148061+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.d","version":"206'2517","reported_seq":2532,"reported_epoch":206,"state":"active+clean","last_fresh":"2023-09-23T20:56:59.012252+0000","last_change":"2023-09-23T20:56:35.544132+0000","last_active":"2023-09-23T20:56:59.012252+0000","last_peered":"2023-09-23T20:56:59.012252+0000","last_clean":"2023-09-23T20:56:59.012252+0000","last_became_active":"2023-09-23T20:56:35.543836+0000","last_became_peered":"2023-09-23T20:56:35.543836+0000","last_unstale":"2023-09-23T20:56:59.012252+0000","last_undegraded":"2023-09-23T20:56:59.012252+0000","last_fullsized":"2023-09-23T20:56:59.012252+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":199,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_clean_scrub_stamp":"2023-09-23T20:56:34.529103+0000","objects_scrubbed":0,"log_size":2517,"log_dups_size":0,"ondisk_log_size":2517,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:27:14.636991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":164954112,"num_objects":2517,"num_object_clones":0,"num_object_copies":7551,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2517,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2517,"num_write_kb":161088,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":83,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-09-23T20:56:45.985951+0000","last_change":"2023-09-23T20:54:13.950254+0000","last_active":"2023-09-23T20:56:45.985951+0000","last_peered":"2023-09-23T20:56:45.985951+0000","last_clean":"2023-09-23T20:56:45.985951+0000","last_became_active":"2023-09-23T20:54:13.949934+0000","last_became_peered":"2023-09-23T20:54:13.949934+0000","last_unstale":"2023-09-23T20:56:45.985951+0000","last_undegraded":"2023-09-23T20:56:45.985951+0000","last_fullsized":"2023-09-23T20:56:45.985951+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:30:32.847167+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":100,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-09-23T20:56:45.986564+0000","last_change":"2023-09-23T20:55:35.956153+0000","last_active":"2023-09-23T20:56:45.986564+0000","last_peered":"2023-09-23T20:56:45.986564+0000","last_clean":"2023-09-23T20:56:45.986564+0000","last_became_active":"2023-09-23T20:55:35.955630+0000","last_became_peered":"2023-09-23T20:55:35.955630+0000","last_unstale":"2023-09-23T20:56:45.986564+0000","last_undegraded":"2023-09-23T20:56:45.986564+0000","last_fullsized":"2023-09-23T20:56:45.986564+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:45:20.282929+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.c","version":"206'2460","reported_seq":2475,"reported_epoch":206,"state":"active+clean","last_fresh":"2023-09-23T20:56:59.031208+0000","last_change":"2023-09-23T20:56:35.550136+0000","last_active":"2023-09-23T20:56:59.031208+0000","last_peered":"2023-09-23T20:56:59.031208+0000","last_clean":"2023-09-23T20:56:59.031208+0000","last_became_active":"2023-09-23T20:56:35.549806+0000","last_became_peered":"2023-09-23T20:56:35.549806+0000","last_unstale":"2023-09-23T20:56:59.031208+0000","last_undegraded":"2023-09-23T20:56:59.031208+0000","last_fullsized":"2023-09-23T20:56:59.031208+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":199,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_clean_scrub_stamp":"2023-09-23T20:56:34.529103+0000","objects_scrubbed":0,"log_size":2460,"log_dups_size":0,"ondisk_log_size":2460,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:39:43.734172+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":161218560,"num_objects":2460,"num_object_clones":0,"num_object_copies":7380,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2460,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2460,"num_write_kb":157440,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.a","version":"79'161","reported_seq":449,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-09-23T20:56:45.984331+0000","last_change":"2023-09-23T20:56:44.487778+0000","last_active":"2023-09-23T20:56:45.984331+0000","last_peered":"2023-09-23T20:56:45.984331+0000","last_clean":"2023-09-23T20:56:45.984331+0000","last_became_active":"2023-09-23T20:56:05.006740+0000","last_became_peered":"2023-09-23T20:56:05.006740+0000","last_unstale":"2023-09-23T20:56:45.984331+0000","last_undegraded":"2023-09-23T20:56:45.984331+0000","last_fullsized":"2023-09-23T20:56:45.984331+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:56:44.487678+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:56:44.487678+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:04:50.086749+0000","scrub_duration":0.0017408899999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":129,"reported_epoch":206,"state":"active+clean","last_fresh":"2023-09-23T20:56:54.057677+0000","last_change":"2023-09-23T20:54:44.034235+0000","last_active":"2023-09-23T20:56:54.057677+0000","last_peered":"2023-09-23T20:56:54.057677+0000","last_clean":"2023-09-23T20:56:54.057677+0000","last_became_active":"2023-09-23T20:54:44.033911+0000","last_became_peered":"2023-09-23T20:54:44.033911+0000","last_unstale":"2023-09-23T20:56:54.057677+0000","last_undegraded":"2023-09-23T20:56:54.057677+0000","last_fullsized":"2023-09-23T20:56:54.057677+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:12:48.442227+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,4],"acting":[1,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":175,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-09-23T20:56:49.223396+0000","last_change":"2023-09-23T20:54:13.940748+0000","last_active":"2023-09-23T20:56:49.223396+0000","last_peered":"2023-09-23T20:56:49.223396+0000","last_clean":"2023-09-23T20:56:49.223396+0000","last_became_active":"2023-09-23T20:54:13.940419+0000","last_became_peered":"2023-09-23T20:54:13.940419+0000","last_unstale":"2023-09-23T20:56:49.223396+0000","last_undegraded":"2023-09-23T20:56:49.223396+0000","last_fullsized":"2023-09-23T20:56:49.223396+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:15:01.923816+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.1","version":"206'2344","reported_seq":2358,"reported_epoch":206,"state":"active+clean","last_fresh":"2023-09-23T20:56:57.144206+0000","last_change":"2023-09-23T20:56:35.547590+0000","last_active":"2023-09-23T20:56:57.144206+0000","last_peered":"2023-09-23T20:56:57.144206+0000","last_clean":"2023-09-23T20:56:57.144206+0000","last_became_active":"2023-09-23T20:56:35.547333+0000","last_became_peered":"2023-09-23T20:56:35.547333+0000","last_unstale":"2023-09-23T20:56:57.144206+0000","last_undegraded":"2023-09-23T20:56:57.144206+0000","last_fullsized":"2023-09-23T20:56:57.144206+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":199,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_clean_scrub_stamp":"2023-09-23T20:56:34.529103+0000","objects_scrubbed":0,"log_size":2344,"log_dups_size":0,"ondisk_log_size":2344,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:29:45.487816+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":153616384,"num_objects":2344,"num_object_clones":0,"num_object_copies":7032,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2344,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2344,"num_write_kb":150016,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.7","version":"79'161","reported_seq":471,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-09-23T20:56:48.000246+0000","last_change":"2023-09-23T20:56:48.000182+0000","last_active":"2023-09-23T20:56:48.000246+0000","last_peered":"2023-09-23T20:56:48.000246+0000","last_clean":"2023-09-23T20:56:48.000246+0000","last_became_active":"2023-09-23T20:56:04.971335+0000","last_became_peered":"2023-09-23T20:56:04.971335+0000","last_unstale":"2023-09-23T20:56:48.000246+0000","last_undegraded":"2023-09-23T20:56:48.000246+0000","last_fullsized":"2023-09-23T20:56:48.000246+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":3,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:56:48.000109+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:56:48.000109+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:34:58.237605+0000","scrub_duration":0.001514086,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,5],"acting":[6,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":180,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-09-23T20:56:45.983464+0000","last_change":"2023-09-23T20:50:05.867130+0000","last_active":"2023-09-23T20:56:45.983464+0000","last_peered":"2023-09-23T20:56:45.983464+0000","last_clean":"2023-09-23T20:56:45.983464+0000","last_became_active":"2023-09-23T20:50:05.866758+0000","last_became_peered":"2023-09-23T20:50:05.866758+0000","last_unstale":"2023-09-23T20:56:45.983464+0000","last_undegraded":"2023-09-23T20:56:45.983464+0000","last_fullsized":"2023-09-23T20:56:45.983464+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:55:07.548209+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":79,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-09-23T20:56:45.984052+0000","last_change":"2023-09-23T20:54:31.887604+0000","last_active":"2023-09-23T20:56:45.984052+0000","last_peered":"2023-09-23T20:56:45.984052+0000","last_clean":"2023-09-23T20:56:45.984052+0000","last_became_active":"2023-09-23T20:54:31.886752+0000","last_became_peered":"2023-09-23T20:54:31.886752+0000","last_unstale":"2023-09-23T20:56:45.984052+0000","last_undegraded":"2023-09-23T20:56:45.984052+0000","last_fullsized":"2023-09-23T20:56:45.984052+0000","mapping_epoch":136,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:13:58.872176+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":178,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-09-23T20:56:45.982846+0000","last_change":"2023-09-23T20:54:31.899082+0000","last_active":"2023-09-23T20:56:45.982846+0000","last_peered":"2023-09-23T20:56:45.982846+0000","last_clean":"2023-09-23T20:56:45.982846+0000","last_became_active":"2023-09-23T20:54:31.898521+0000","last_became_peered":"2023-09-23T20:54:31.898521+0000","last_unstale":"2023-09-23T20:56:45.982846+0000","last_undegraded":"2023-09-23T20:56:45.982846+0000","last_fullsized":"2023-09-23T20:56:45.982846+0000","mapping_epoch":136,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:25:36.059711+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.7","version":"206'2478","reported_seq":2493,"reported_epoch":206,"state":"active+clean","last_fresh":"2023-09-23T20:56:59.022949+0000","last_change":"2023-09-23T20:56:35.541695+0000","last_active":"2023-09-23T20:56:59.022949+0000","last_peered":"2023-09-23T20:56:59.022949+0000","last_clean":"2023-09-23T20:56:59.022949+0000","last_became_active":"2023-09-23T20:56:35.541392+0000","last_became_peered":"2023-09-23T20:56:35.541392+0000","last_unstale":"2023-09-23T20:56:59.022949+0000","last_undegraded":"2023-09-23T20:56:59.022949+0000","last_fullsized":"2023-09-23T20:56:59.022949+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":199,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_clean_scrub_stamp":"2023-09-23T20:56:34.529103+0000","objects_scrubbed":0,"log_size":2478,"log_dups_size":0,"ondisk_log_size":2478,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:57:22.478253+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":162398208,"num_objects":2478,"num_object_clones":0,"num_object_copies":7434,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2478,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2478,"num_write_kb":158592,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.1","version":"79'161","reported_seq":449,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-09-23T20:56:47.563110+0000","last_change":"2023-09-23T20:56:47.563038+0000","last_active":"2023-09-23T20:56:47.563110+0000","last_peered":"2023-09-23T20:56:47.563110+0000","last_clean":"2023-09-23T20:56:47.563110+0000","last_became_active":"2023-09-23T20:56:05.000176+0000","last_became_peered":"2023-09-23T20:56:05.000176+0000","last_unstale":"2023-09-23T20:56:47.563110+0000","last_undegraded":"2023-09-23T20:56:47.563110+0000","last_fullsized":"2023-09-23T20:56:47.563110+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:56:47.562966+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:56:47.562966+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:04:07.052728+0000","scrub_duration":0.001898065,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":96,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-09-23T20:56:45.984061+0000","last_change":"2023-09-23T20:55:35.959514+0000","last_active":"2023-09-23T20:56:45.984061+0000","last_peered":"2023-09-23T20:56:45.984061+0000","last_clean":"2023-09-23T20:56:45.984061+0000","last_became_active":"2023-09-23T20:55:35.959244+0000","last_became_peered":"2023-09-23T20:55:35.959244+0000","last_unstale":"2023-09-23T20:56:45.984061+0000","last_undegraded":"2023-09-23T20:56:45.984061+0000","last_fullsized":"2023-09-23T20:56:45.984061+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:47:32.665316+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":100,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-09-23T20:56:45.987933+0000","last_change":"2023-09-23T20:55:35.949000+0000","last_active":"2023-09-23T20:56:45.987933+0000","last_peered":"2023-09-23T20:56:45.987933+0000","last_clean":"2023-09-23T20:56:45.987933+0000","last_became_active":"2023-09-23T20:55:35.948410+0000","last_became_peered":"2023-09-23T20:55:35.948410+0000","last_unstale":"2023-09-23T20:56:45.987933+0000","last_undegraded":"2023-09-23T20:56:45.987933+0000","last_fullsized":"2023-09-23T20:56:45.987933+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:26:50.477289+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":172,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-09-23T20:56:45.983316+0000","last_change":"2023-09-23T20:51:10.829889+0000","last_active":"2023-09-23T20:56:45.983316+0000","last_peered":"2023-09-23T20:56:45.983316+0000","last_clean":"2023-09-23T20:56:45.983316+0000","last_became_active":"2023-09-23T20:51:10.829533+0000","last_became_peered":"2023-09-23T20:51:10.829533+0000","last_unstale":"2023-09-23T20:56:45.983316+0000","last_undegraded":"2023-09-23T20:56:45.983316+0000","last_fullsized":"2023-09-23T20:56:45.983316+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:43:04.603460+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.4","version":"206'2178","reported_seq":2194,"reported_epoch":206,"state":"active+clean","last_fresh":"2023-09-23T20:56:56.018006+0000","last_change":"2023-09-23T20:56:35.549909+0000","last_active":"2023-09-23T20:56:56.018006+0000","last_peered":"2023-09-23T20:56:56.018006+0000","last_clean":"2023-09-23T20:56:56.018006+0000","last_became_active":"2023-09-23T20:56:35.549618+0000","last_became_peered":"2023-09-23T20:56:35.549618+0000","last_unstale":"2023-09-23T20:56:56.018006+0000","last_undegraded":"2023-09-23T20:56:56.018006+0000","last_fullsized":"2023-09-23T20:56:56.018006+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":199,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_clean_scrub_stamp":"2023-09-23T20:56:34.529103+0000","objects_scrubbed":0,"log_size":2178,"log_dups_size":0,"ondisk_log_size":2178,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:24:25.918475+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":142737408,"num_objects":2178,"num_object_clones":0,"num_object_copies":6534,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2178,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2178,"num_write_kb":139392,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.2","version":"79'161","reported_seq":414,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-09-23T20:56:46.040087+0000","last_change":"2023-09-23T20:56:45.989430+0000","last_active":"2023-09-23T20:56:46.040087+0000","last_peered":"2023-09-23T20:56:46.040087+0000","last_clean":"2023-09-23T20:56:46.040087+0000","last_became_active":"2023-09-23T20:56:45.989084+0000","last_became_peered":"2023-09-23T20:56:45.989084+0000","last_unstale":"2023-09-23T20:56:46.040087+0000","last_undegraded":"2023-09-23T20:56:46.040087+0000","last_fullsized":"2023-09-23T20:56:46.040087+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":205,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:56:43.267444+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:56:43.267444+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":45,"num_read_kb":83,"num_write":22,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":161010,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,4],"acting":[0,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":191,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-09-23T20:56:51.331787+0000","last_change":"2023-09-23T20:51:10.838054+0000","last_active":"2023-09-23T20:56:51.331787+0000","last_peered":"2023-09-23T20:56:51.331787+0000","last_clean":"2023-09-23T20:56:51.331787+0000","last_became_active":"2023-09-23T20:51:10.837750+0000","last_became_peered":"2023-09-23T20:51:10.837750+0000","last_unstale":"2023-09-23T20:56:51.331787+0000","last_undegraded":"2023-09-23T20:56:51.331787+0000","last_fullsized":"2023-09-23T20:56:51.331787+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:15:51.648085+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":172,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-09-23T20:56:45.982798+0000","last_change":"2023-09-23T20:55:35.951146+0000","last_active":"2023-09-23T20:56:45.982798+0000","last_peered":"2023-09-23T20:56:45.982798+0000","last_clean":"2023-09-23T20:56:45.982798+0000","last_became_active":"2023-09-23T20:55:35.950874+0000","last_became_peered":"2023-09-23T20:55:35.950874+0000","last_unstale":"2023-09-23T20:56:45.982798+0000","last_undegraded":"2023-09-23T20:56:45.982798+0000","last_fullsized":"2023-09-23T20:56:45.982798+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:58:13.957677+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":193,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-09-23T20:56:45.983127+0000","last_change":"2023-09-23T20:54:31.898870+0000","last_active":"2023-09-23T20:56:45.983127+0000","last_peered":"2023-09-23T20:56:45.983127+0000","last_clean":"2023-09-23T20:56:45.983127+0000","last_became_active":"2023-09-23T20:54:31.898467+0000","last_became_peered":"2023-09-23T20:54:31.898467+0000","last_unstale":"2023-09-23T20:56:45.983127+0000","last_undegraded":"2023-09-23T20:56:45.983127+0000","last_fullsized":"2023-09-23T20:56:45.983127+0000","mapping_epoch":136,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:56:22.483408+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.6","version":"206'2183","reported_seq":2199,"reported_epoch":206,"state":"active+clean","last_fresh":"2023-09-23T20:56:56.030992+0000","last_change":"2023-09-23T20:56:35.545911+0000","last_active":"2023-09-23T20:56:56.030992+0000","last_peered":"2023-09-23T20:56:56.030992+0000","last_clean":"2023-09-23T20:56:56.030992+0000","last_became_active":"2023-09-23T20:56:35.545582+0000","last_became_peered":"2023-09-23T20:56:35.545582+0000","last_unstale":"2023-09-23T20:56:56.030992+0000","last_undegraded":"2023-09-23T20:56:56.030992+0000","last_fullsized":"2023-09-23T20:56:56.030992+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":199,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_clean_scrub_stamp":"2023-09-23T20:56:34.529103+0000","objects_scrubbed":0,"log_size":2183,"log_dups_size":0,"ondisk_log_size":2183,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:17:16.998361+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":143065088,"num_objects":2183,"num_object_clones":0,"num_object_copies":6549,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2183,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2183,"num_write_kb":139712,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,5],"acting":[0,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":79,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-09-23T20:56:45.983686+0000","last_change":"2023-09-23T20:54:31.883504+0000","last_active":"2023-09-23T20:56:45.983686+0000","last_peered":"2023-09-23T20:56:45.983686+0000","last_clean":"2023-09-23T20:56:45.983686+0000","last_became_active":"2023-09-23T20:54:31.883172+0000","last_became_peered":"2023-09-23T20:54:31.883172+0000","last_unstale":"2023-09-23T20:56:45.983686+0000","last_undegraded":"2023-09-23T20:56:45.983686+0000","last_fullsized":"2023-09-23T20:56:45.983686+0000","mapping_epoch":136,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:45:40.289120+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"79'161","reported_seq":414,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-09-23T20:56:46.040044+0000","last_change":"2023-09-23T20:56:45.990284+0000","last_active":"2023-09-23T20:56:46.040044+0000","last_peered":"2023-09-23T20:56:46.040044+0000","last_clean":"2023-09-23T20:56:46.040044+0000","last_became_active":"2023-09-23T20:56:45.989891+0000","last_became_peered":"2023-09-23T20:56:45.989891+0000","last_unstale":"2023-09-23T20:56:46.040044+0000","last_undegraded":"2023-09-23T20:56:46.040044+0000","last_fullsized":"2023-09-23T20:56:46.040044+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":205,"parent":"0.0","parent_split_bits":0,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:56:03.192713+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:56:03.192713+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53669,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":44,"num_read_kb":82,"num_write":21,"num_write_kb":219,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":161009,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":182,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-09-23T20:56:45.978305+0000","last_change":"2023-09-23T20:55:35.950777+0000","last_active":"2023-09-23T20:56:45.978305+0000","last_peered":"2023-09-23T20:56:45.978305+0000","last_clean":"2023-09-23T20:56:45.978305+0000","last_became_active":"2023-09-23T20:55:35.950427+0000","last_became_peered":"2023-09-23T20:55:35.950427+0000","last_unstale":"2023-09-23T20:56:45.978305+0000","last_undegraded":"2023-09-23T20:56:45.978305+0000","last_fullsized":"2023-09-23T20:56:45.978305+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:20:10.371751+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":150,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-09-23T20:56:45.981648+0000","last_change":"2023-09-23T20:55:35.978140+0000","last_active":"2023-09-23T20:56:45.981648+0000","last_peered":"2023-09-23T20:56:45.981648+0000","last_clean":"2023-09-23T20:56:45.981648+0000","last_became_active":"2023-09-23T20:55:35.977831+0000","last_became_peered":"2023-09-23T20:55:35.977831+0000","last_unstale":"2023-09-23T20:56:45.981648+0000","last_undegraded":"2023-09-23T20:56:45.981648+0000","last_fullsized":"2023-09-23T20:56:45.981648+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:25:47.678023+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.5","version":"206'2183","reported_seq":2206,"reported_epoch":206,"state":"active+clean","last_fresh":"2023-09-23T20:56:56.311726+0000","last_change":"2023-09-23T20:56:45.990110+0000","last_active":"2023-09-23T20:56:56.311726+0000","last_peered":"2023-09-23T20:56:56.311726+0000","last_clean":"2023-09-23T20:56:56.311726+0000","last_became_active":"2023-09-23T20:56:45.988394+0000","last_became_peered":"2023-09-23T20:56:45.988394+0000","last_unstale":"2023-09-23T20:56:56.311726+0000","last_undegraded":"2023-09-23T20:56:56.311726+0000","last_fullsized":"2023-09-23T20:56:56.311726+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":199,"last_epoch_clean":205,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_clean_scrub_stamp":"2023-09-23T20:56:34.529103+0000","objects_scrubbed":0,"log_size":2183,"log_dups_size":0,"ondisk_log_size":2183,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:22:03.814210+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":143065088,"num_objects":2183,"num_object_clones":0,"num_object_copies":6549,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2183,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2183,"num_write_kb":139712,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,0],"acting":[2,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"79'161","reported_seq":450,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-09-23T20:56:45.983008+0000","last_change":"2023-09-23T20:56:44.376650+0000","last_active":"2023-09-23T20:56:45.983008+0000","last_peered":"2023-09-23T20:56:45.983008+0000","last_clean":"2023-09-23T20:56:45.983008+0000","last_became_active":"2023-09-23T20:56:04.978312+0000","last_became_peered":"2023-09-23T20:56:04.978312+0000","last_unstale":"2023-09-23T20:56:45.983008+0000","last_undegraded":"2023-09-23T20:56:45.983008+0000","last_fullsized":"2023-09-23T20:56:45.983008+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":3,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:56:44.376461+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:56:44.376461+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:20:56.927907+0000","scrub_duration":0.003668386,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":74,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-09-23T20:56:49.223300+0000","last_change":"2023-09-23T20:56:45.991884+0000","last_active":"2023-09-23T20:56:49.223300+0000","last_peered":"2023-09-23T20:56:49.223300+0000","last_clean":"2023-09-23T20:56:49.223300+0000","last_became_active":"2023-09-23T20:56:45.991568+0000","last_became_peered":"2023-09-23T20:56:45.991568+0000","last_unstale":"2023-09-23T20:56:49.223300+0000","last_undegraded":"2023-09-23T20:56:49.223300+0000","last_fullsized":"2023-09-23T20:56:49.223300+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":205,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:21:20.775511+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,0],"acting":[1,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":189,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-09-23T20:56:45.981996+0000","last_change":"2023-09-23T20:54:44.035149+0000","last_active":"2023-09-23T20:56:45.981996+0000","last_peered":"2023-09-23T20:56:45.981996+0000","last_clean":"2023-09-23T20:56:45.981996+0000","last_became_active":"2023-09-23T20:54:44.034729+0000","last_became_peered":"2023-09-23T20:54:44.034729+0000","last_unstale":"2023-09-23T20:56:45.981996+0000","last_undegraded":"2023-09-23T20:56:45.981996+0000","last_fullsized":"2023-09-23T20:56:45.981996+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:35:26.645773+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,5],"acting":[0,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":177,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-09-23T20:56:45.982007+0000","last_change":"2023-09-23T20:50:37.982235+0000","last_active":"2023-09-23T20:56:45.982007+0000","last_peered":"2023-09-23T20:56:45.982007+0000","last_clean":"2023-09-23T20:56:45.982007+0000","last_became_active":"2023-09-23T20:50:37.981913+0000","last_became_peered":"2023-09-23T20:50:37.981913+0000","last_unstale":"2023-09-23T20:56:45.982007+0000","last_undegraded":"2023-09-23T20:56:45.982007+0000","last_fullsized":"2023-09-23T20:56:45.982007+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:59:57.122584+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.0","version":"206'2299","reported_seq":2314,"reported_epoch":206,"state":"active+clean","last_fresh":"2023-09-23T20:56:57.102117+0000","last_change":"2023-09-23T20:56:35.550635+0000","last_active":"2023-09-23T20:56:57.102117+0000","last_peered":"2023-09-23T20:56:57.102117+0000","last_clean":"2023-09-23T20:56:57.102117+0000","last_became_active":"2023-09-23T20:56:35.550376+0000","last_became_peered":"2023-09-23T20:56:35.550376+0000","last_unstale":"2023-09-23T20:56:57.102117+0000","last_undegraded":"2023-09-23T20:56:57.102117+0000","last_fullsized":"2023-09-23T20:56:57.102117+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":199,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_clean_scrub_stamp":"2023-09-23T20:56:34.529103+0000","objects_scrubbed":0,"log_size":2299,"log_dups_size":0,"ondisk_log_size":2299,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:40:55.020049+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":150667264,"num_objects":2299,"num_object_clones":0,"num_object_copies":6897,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2299,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2299,"num_write_kb":147136,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.6","version":"79'161","reported_seq":468,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-09-23T20:56:45.982442+0000","last_change":"2023-09-23T20:56:28.806845+0000","last_active":"2023-09-23T20:56:45.982442+0000","last_peered":"2023-09-23T20:56:45.982442+0000","last_clean":"2023-09-23T20:56:45.982442+0000","last_became_active":"2023-09-23T20:56:04.980132+0000","last_became_peered":"2023-09-23T20:56:04.980132+0000","last_unstale":"2023-09-23T20:56:45.982442+0000","last_undegraded":"2023-09-23T20:56:45.982442+0000","last_fullsized":"2023-09-23T20:56:45.982442+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":3,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:56:28.806732+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:56:28.806732+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:45:12.588417+0000","scrub_duration":0.0020791540000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":121,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-09-23T20:56:46.040164+0000","last_change":"2023-09-23T20:56:45.988566+0000","last_active":"2023-09-23T20:56:46.040164+0000","last_peered":"2023-09-23T20:56:46.040164+0000","last_clean":"2023-09-23T20:56:46.040164+0000","last_became_active":"2023-09-23T20:56:45.988249+0000","last_became_peered":"2023-09-23T20:56:45.988249+0000","last_unstale":"2023-09-23T20:56:46.040164+0000","last_undegraded":"2023-09-23T20:56:46.040164+0000","last_fullsized":"2023-09-23T20:56:46.040164+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":205,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:51:36.644764+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,4],"acting":[0,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":175,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-09-23T20:56:45.983468+0000","last_change":"2023-09-23T20:54:31.898965+0000","last_active":"2023-09-23T20:56:45.983468+0000","last_peered":"2023-09-23T20:56:45.983468+0000","last_clean":"2023-09-23T20:56:45.983468+0000","last_became_active":"2023-09-23T20:54:31.898663+0000","last_became_peered":"2023-09-23T20:54:31.898663+0000","last_unstale":"2023-09-23T20:56:45.983468+0000","last_undegraded":"2023-09-23T20:56:45.983468+0000","last_fullsized":"2023-09-23T20:56:45.983468+0000","mapping_epoch":136,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:53:05.058087+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":165,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-09-23T20:56:45.982830+0000","last_change":"2023-09-23T20:54:31.884717+0000","last_active":"2023-09-23T20:56:45.982830+0000","last_peered":"2023-09-23T20:56:45.982830+0000","last_clean":"2023-09-23T20:56:45.982830+0000","last_became_active":"2023-09-23T20:54:31.884345+0000","last_became_peered":"2023-09-23T20:54:31.884345+0000","last_unstale":"2023-09-23T20:56:45.982830+0000","last_undegraded":"2023-09-23T20:56:45.982830+0000","last_fullsized":"2023-09-23T20:56:45.982830+0000","mapping_epoch":136,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:02:26.393868+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.3","version":"206'2398","reported_seq":2414,"reported_epoch":206,"state":"active+clean","last_fresh":"2023-09-23T20:56:59.245465+0000","last_change":"2023-09-23T20:56:35.549655+0000","last_active":"2023-09-23T20:56:59.245465+0000","last_peered":"2023-09-23T20:56:59.245465+0000","last_clean":"2023-09-23T20:56:59.245465+0000","last_became_active":"2023-09-23T20:56:35.549357+0000","last_became_peered":"2023-09-23T20:56:35.549357+0000","last_unstale":"2023-09-23T20:56:59.245465+0000","last_undegraded":"2023-09-23T20:56:59.245465+0000","last_fullsized":"2023-09-23T20:56:59.245465+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":199,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_clean_scrub_stamp":"2023-09-23T20:56:34.529103+0000","objects_scrubbed":0,"log_size":2398,"log_dups_size":0,"ondisk_log_size":2398,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:37:58.752518+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":157155328,"num_objects":2398,"num_object_clones":0,"num_object_copies":7194,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2398,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2398,"num_write_kb":153472,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.5","version":"79'161","reported_seq":449,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-09-23T20:56:45.985772+0000","last_change":"2023-09-23T20:56:10.732852+0000","last_active":"2023-09-23T20:56:45.985772+0000","last_peered":"2023-09-23T20:56:45.985772+0000","last_clean":"2023-09-23T20:56:45.985772+0000","last_became_active":"2023-09-23T20:56:04.990415+0000","last_became_peered":"2023-09-23T20:56:04.990415+0000","last_unstale":"2023-09-23T20:56:45.985772+0000","last_undegraded":"2023-09-23T20:56:45.985772+0000","last_fullsized":"2023-09-23T20:56:45.985772+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":3,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:56:10.732760+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:56:10.732760+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:38:09.709945+0000","scrub_duration":0.014169637000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":168,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-09-23T20:56:45.978071+0000","last_change":"2023-09-23T20:51:10.836910+0000","last_active":"2023-09-23T20:56:45.978071+0000","last_peered":"2023-09-23T20:56:45.978071+0000","last_clean":"2023-09-23T20:56:45.978071+0000","last_became_active":"2023-09-23T20:51:10.836557+0000","last_became_peered":"2023-09-23T20:51:10.836557+0000","last_unstale":"2023-09-23T20:56:45.978071+0000","last_undegraded":"2023-09-23T20:56:45.978071+0000","last_fullsized":"2023-09-23T20:56:45.978071+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:51:54.019969+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":175,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-09-23T20:56:45.983055+0000","last_change":"2023-09-23T20:55:35.953444+0000","last_active":"2023-09-23T20:56:45.983055+0000","last_peered":"2023-09-23T20:56:45.983055+0000","last_clean":"2023-09-23T20:56:45.983055+0000","last_became_active":"2023-09-23T20:55:35.953151+0000","last_became_peered":"2023-09-23T20:55:35.953151+0000","last_unstale":"2023-09-23T20:56:45.983055+0000","last_undegraded":"2023-09-23T20:56:45.983055+0000","last_fullsized":"2023-09-23T20:56:45.983055+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:39:12.404391+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":64,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-09-23T20:56:46.040020+0000","last_change":"2023-09-23T20:56:45.990141+0000","last_active":"2023-09-23T20:56:46.040020+0000","last_peered":"2023-09-23T20:56:46.040020+0000","last_clean":"2023-09-23T20:56:46.040020+0000","last_became_active":"2023-09-23T20:56:45.989830+0000","last_became_peered":"2023-09-23T20:56:45.989830+0000","last_unstale":"2023-09-23T20:56:46.040020+0000","last_undegraded":"2023-09-23T20:56:46.040020+0000","last_fullsized":"2023-09-23T20:56:46.040020+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":205,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:57:33.938485+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.2","version":"206'2478","reported_seq":2492,"reported_epoch":206,"state":"active+clean","last_fresh":"2023-09-23T20:56:59.222847+0000","last_change":"2023-09-23T20:56:35.543020+0000","last_active":"2023-09-23T20:56:59.222847+0000","last_peered":"2023-09-23T20:56:59.222847+0000","last_clean":"2023-09-23T20:56:59.222847+0000","last_became_active":"2023-09-23T20:56:35.542719+0000","last_became_peered":"2023-09-23T20:56:35.542719+0000","last_unstale":"2023-09-23T20:56:59.222847+0000","last_undegraded":"2023-09-23T20:56:59.222847+0000","last_fullsized":"2023-09-23T20:56:59.222847+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":199,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_clean_scrub_stamp":"2023-09-23T20:56:34.529103+0000","objects_scrubbed":0,"log_size":2478,"log_dups_size":0,"ondisk_log_size":2478,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:52:21.088628+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":162398208,"num_objects":2478,"num_object_clones":0,"num_object_copies":7434,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2478,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2478,"num_write_kb":158592,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.4","version":"79'161","reported_seq":462,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-09-23T20:56:45.977769+0000","last_change":"2023-09-23T20:56:43.900550+0000","last_active":"2023-09-23T20:56:45.977769+0000","last_peered":"2023-09-23T20:56:45.977769+0000","last_clean":"2023-09-23T20:56:45.977769+0000","last_became_active":"2023-09-23T20:56:04.986356+0000","last_became_peered":"2023-09-23T20:56:04.986356+0000","last_unstale":"2023-09-23T20:56:45.977769+0000","last_undegraded":"2023-09-23T20:56:45.977769+0000","last_fullsized":"2023-09-23T20:56:45.977769+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":3,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:56:43.900230+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:56:43.900230+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:48:32.518160+0000","scrub_duration":0.0034600859999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":155,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-09-23T20:56:45.981441+0000","last_change":"2023-09-23T20:55:35.949541+0000","last_active":"2023-09-23T20:56:45.981441+0000","last_peered":"2023-09-23T20:56:45.981441+0000","last_clean":"2023-09-23T20:56:45.981441+0000","last_became_active":"2023-09-23T20:55:35.944635+0000","last_became_peered":"2023-09-23T20:55:35.944635+0000","last_unstale":"2023-09-23T20:56:45.981441+0000","last_undegraded":"2023-09-23T20:56:45.981441+0000","last_fullsized":"2023-09-23T20:56:45.981441+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:30:36.033585+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"70'1","reported_seq":193,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-09-23T20:56:45.982102+0000","last_change":"2023-09-23T20:54:14.009723+0000","last_active":"2023-09-23T20:56:45.982102+0000","last_peered":"2023-09-23T20:56:45.982102+0000","last_clean":"2023-09-23T20:56:45.982102+0000","last_became_active":"2023-09-23T20:54:13.941771+0000","last_became_peered":"2023-09-23T20:54:13.941771+0000","last_unstale":"2023-09-23T20:56:45.982102+0000","last_undegraded":"2023-09-23T20:56:45.982102+0000","last_fullsized":"2023-09-23T20:56:45.982102+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:48:25.588116+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":110,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-09-23T20:56:45.985619+0000","last_change":"2023-09-23T20:55:35.949780+0000","last_active":"2023-09-23T20:56:45.985619+0000","last_peered":"2023-09-23T20:56:45.985619+0000","last_clean":"2023-09-23T20:56:45.985619+0000","last_became_active":"2023-09-23T20:55:35.949497+0000","last_became_peered":"2023-09-23T20:55:35.949497+0000","last_unstale":"2023-09-23T20:56:45.985619+0000","last_undegraded":"2023-09-23T20:56:45.985619+0000","last_fullsized":"2023-09-23T20:56:45.985619+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:09:37.420882+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":167,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-09-23T20:56:45.982750+0000","last_change":"2023-09-23T20:54:13.929196+0000","last_active":"2023-09-23T20:56:45.982750+0000","last_peered":"2023-09-23T20:56:45.982750+0000","last_clean":"2023-09-23T20:56:45.982750+0000","last_became_active":"2023-09-23T20:54:13.928831+0000","last_became_peered":"2023-09-23T20:54:13.928831+0000","last_unstale":"2023-09-23T20:56:45.982750+0000","last_undegraded":"2023-09-23T20:56:45.982750+0000","last_fullsized":"2023-09-23T20:56:45.982750+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:28:44.316851+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":173,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-09-23T20:56:45.979847+0000","last_change":"2023-09-23T20:52:12.855430+0000","last_active":"2023-09-23T20:56:45.979847+0000","last_peered":"2023-09-23T20:56:45.979847+0000","last_clean":"2023-09-23T20:56:45.979847+0000","last_became_active":"2023-09-23T20:52:12.855115+0000","last_became_peered":"2023-09-23T20:52:12.855115+0000","last_unstale":"2023-09-23T20:56:45.979847+0000","last_undegraded":"2023-09-23T20:56:45.979847+0000","last_fullsized":"2023-09-23T20:56:45.979847+0000","mapping_epoch":100,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":101,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:33:23.802214+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":83,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-09-23T20:56:45.983722+0000","last_change":"2023-09-23T20:54:31.902946+0000","last_active":"2023-09-23T20:56:45.983722+0000","last_peered":"2023-09-23T20:56:45.983722+0000","last_clean":"2023-09-23T20:56:45.983722+0000","last_became_active":"2023-09-23T20:54:31.889484+0000","last_became_peered":"2023-09-23T20:54:31.889484+0000","last_unstale":"2023-09-23T20:56:45.983722+0000","last_undegraded":"2023-09-23T20:56:45.983722+0000","last_fullsized":"2023-09-23T20:56:45.983722+0000","mapping_epoch":136,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:44:44.278398+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":83,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-09-23T20:56:45.988049+0000","last_change":"2023-09-23T20:54:13.950351+0000","last_active":"2023-09-23T20:56:45.988049+0000","last_peered":"2023-09-23T20:56:45.988049+0000","last_clean":"2023-09-23T20:56:45.988049+0000","last_became_active":"2023-09-23T20:54:13.949987+0000","last_became_peered":"2023-09-23T20:54:13.949987+0000","last_unstale":"2023-09-23T20:56:45.988049+0000","last_undegraded":"2023-09-23T20:56:45.988049+0000","last_fullsized":"2023-09-23T20:56:45.988049+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:46:17.765134+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":114,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-09-23T20:56:45.984139+0000","last_change":"2023-09-23T20:56:18.141728+0000","last_active":"2023-09-23T20:56:45.984139+0000","last_peered":"2023-09-23T20:56:45.984139+0000","last_clean":"2023-09-23T20:56:45.984139+0000","last_became_active":"2023-09-23T20:56:18.137979+0000","last_became_peered":"2023-09-23T20:56:18.137979+0000","last_unstale":"2023-09-23T20:56:45.984139+0000","last_undegraded":"2023-09-23T20:56:45.984139+0000","last_fullsized":"2023-09-23T20:56:45.984139+0000","mapping_epoch":192,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":193,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:58:05.658443+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":83,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-09-23T20:56:45.987074+0000","last_change":"2023-09-23T20:54:13.935307+0000","last_active":"2023-09-23T20:56:45.987074+0000","last_peered":"2023-09-23T20:56:45.987074+0000","last_clean":"2023-09-23T20:56:45.987074+0000","last_became_active":"2023-09-23T20:54:13.935000+0000","last_became_peered":"2023-09-23T20:54:13.935000+0000","last_unstale":"2023-09-23T20:56:45.987074+0000","last_undegraded":"2023-09-23T20:56:45.987074+0000","last_fullsized":"2023-09-23T20:56:45.987074+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:47:40.974082+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":191,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-09-23T20:56:45.978359+0000","last_change":"2023-09-23T20:55:35.954870+0000","last_active":"2023-09-23T20:56:45.978359+0000","last_peered":"2023-09-23T20:56:45.978359+0000","last_clean":"2023-09-23T20:56:45.978359+0000","last_became_active":"2023-09-23T20:55:35.954568+0000","last_became_peered":"2023-09-23T20:55:35.954568+0000","last_unstale":"2023-09-23T20:56:45.978359+0000","last_undegraded":"2023-09-23T20:56:45.978359+0000","last_fullsized":"2023-09-23T20:56:45.978359+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:06:22.423566+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":191,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-09-23T20:56:45.977761+0000","last_change":"2023-09-23T20:51:10.838334+0000","last_active":"2023-09-23T20:56:45.977761+0000","last_peered":"2023-09-23T20:56:45.977761+0000","last_clean":"2023-09-23T20:56:45.977761+0000","last_became_active":"2023-09-23T20:51:10.838026+0000","last_became_peered":"2023-09-23T20:51:10.838026+0000","last_unstale":"2023-09-23T20:56:45.977761+0000","last_undegraded":"2023-09-23T20:56:45.977761+0000","last_fullsized":"2023-09-23T20:56:45.977761+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:04:37.009457+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":195,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-09-23T20:56:45.984961+0000","last_change":"2023-09-23T20:54:31.903139+0000","last_active":"2023-09-23T20:56:45.984961+0000","last_peered":"2023-09-23T20:56:45.984961+0000","last_clean":"2023-09-23T20:56:45.984961+0000","last_became_active":"2023-09-23T20:54:31.884520+0000","last_became_peered":"2023-09-23T20:54:31.884520+0000","last_unstale":"2023-09-23T20:56:45.984961+0000","last_undegraded":"2023-09-23T20:56:45.984961+0000","last_fullsized":"2023-09-23T20:56:45.984961+0000","mapping_epoch":136,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:39:34.269705+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1108,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":100,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-09-23T20:56:45.986772+0000","last_change":"2023-09-23T20:55:35.951572+0000","last_active":"2023-09-23T20:56:45.986772+0000","last_peered":"2023-09-23T20:56:45.986772+0000","last_clean":"2023-09-23T20:56:45.986772+0000","last_became_active":"2023-09-23T20:55:35.951294+0000","last_became_peered":"2023-09-23T20:55:35.951294+0000","last_unstale":"2023-09-23T20:56:45.986772+0000","last_undegraded":"2023-09-23T20:56:45.986772+0000","last_fullsized":"2023-09-23T20:56:45.986772+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:36:32.447210+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":7,"num_pg":16,"stat_sum":{"num_bytes":2449539072,"num_objects":37377,"num_object_clones":0,"num_object_copies":112131,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":37377,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":37377,"num_write_kb":2392128,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":7364476928,"data_stored":7364476928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":37377,"ondisk_log_size":37377,"up":48,"acting":48,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":22,"num_bytes_recovered":5990,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":11,"stat_sum":{"num_bytes":488689,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":133,"num_read_kb":248,"num_write":65,"num_write_kb":658,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":745716,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1771,"ondisk_log_size":1771,"up":33,"acting":33,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201478,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":853360,"kb_used_data":714728,"kb_used_omap":0,"kb_used_meta":138624,"kb_avail":92891792,"statfs":{"total":95995035648,"available":95121195008,"internally_reserved":0,"allocated":731881472,"data_stored":731162395,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":141950976},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:56:43 2023","interfaces":[{"interface":"back","average":{"1min":0.83299999999999996,"5min":1.1319999999999999,"15min":0.71299999999999997},"min":{"1min":0.216,"5min":0.21099999999999999,"15min":0.21099999999999999},"max":{"1min":2.8279999999999998,"5min":5.8760000000000003,"15min":5.8760000000000003},"last":0.86599999999999999},{"interface":"front","average":{"1min":0.79500000000000004,"5min":1.3,"15min":0.80800000000000005},"min":{"1min":0.28199999999999997,"5min":0.218,"15min":0.218},"max":{"1min":2.8439999999999999,"5min":8.7690000000000001,"15min":8.7690000000000001},"last":0.83099999999999996}]},{"osd":1,"last update":"Sat Sep 23 20:56:13 2023","interfaces":[{"interface":"back","average":{"1min":2.2799999999999998,"5min":1.99,"15min":1.0289999999999999},"min":{"1min":0.23300000000000001,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":8.5909999999999993,"5min":22.591999999999999,"15min":22.591999999999999},"last":0.26500000000000001},{"interface":"front","average":{"1min":2.032,"5min":1.5580000000000001,"15min":0.88200000000000001},"min":{"1min":0.26900000000000002,"5min":0.249,"15min":0.249},"max":{"1min":10.097,"5min":21.437000000000001,"15min":21.437000000000001},"last":0.314}]},{"osd":2,"last update":"Sat Sep 23 20:56:48 2023","interfaces":[{"interface":"back","average":{"1min":0.626,"5min":1.5429999999999999,"15min":0.88100000000000001},"min":{"1min":0.26500000000000001,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":1.49,"5min":9.4290000000000003,"15min":9.4290000000000003},"last":0.30199999999999999},{"interface":"front","average":{"1min":0.94299999999999995,"5min":1.496,"15min":0.88800000000000001},"min":{"1min":0.36099999999999999,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":2.6640000000000001,"5min":9.0030000000000001,"15min":9.0030000000000001},"last":0.374}]},{"osd":3,"last update":"Sat Sep 23 20:56:21 2023","interfaces":[{"interface":"back","average":{"1min":1.6180000000000001,"5min":2.9390000000000001,"15min":1.353},"min":{"1min":0.32100000000000001,"5min":0.314,"15min":0.314},"max":{"1min":6.2329999999999997,"5min":36.408000000000001,"15min":36.408000000000001},"last":1.456},{"interface":"front","average":{"1min":2.327,"5min":2.5129999999999999,"15min":1.22},"min":{"1min":0.26000000000000001,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":14.220000000000001,"5min":35.593000000000004,"15min":35.593000000000004},"last":0.50600000000000001}]},{"osd":4,"last update":"Sat Sep 23 20:56:10 2023","interfaces":[{"interface":"back","average":{"1min":0.91500000000000004,"5min":0.78400000000000003,"15min":0.58499999999999996},"min":{"1min":0.23799999999999999,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":3.839,"5min":5.8920000000000003,"15min":5.8920000000000003},"last":0.28100000000000003},{"interface":"front","average":{"1min":1.034,"5min":0.76900000000000002,"15min":0.60499999999999998},"min":{"1min":0.26300000000000001,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":4.9900000000000002,"5min":5.3200000000000003,"15min":5.3200000000000003},"last":0.42299999999999999}]},{"osd":5,"last update":"Sat Sep 23 20:56:10 2023","interfaces":[{"interface":"back","average":{"1min":1.153,"5min":0.80000000000000004,"15min":0.621},"min":{"1min":0.28999999999999998,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":7.0389999999999997,"5min":7.0389999999999997,"15min":7.0389999999999997},"last":0.33300000000000002},{"interface":"front","average":{"1min":1.0029999999999999,"5min":0.68899999999999995,"15min":0.59799999999999998},"min":{"1min":0.29999999999999999,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":3.3079999999999998,"5min":3.3530000000000002,"15min":3.3530000000000002},"last":0.44700000000000001}]},{"osd":6,"last update":"Sat Sep 23 20:56:37 2023","interfaces":[{"interface":"back","average":{"1min":0.66700000000000004,"5min":0.66700000000000004,"15min":0.66700000000000004},"min":{"1min":0.31,"5min":0.31,"15min":0.31},"max":{"1min":3.1760000000000002,"5min":3.1760000000000002,"15min":3.1760000000000002},"last":0.57699999999999996},{"interface":"front","average":{"1min":0.80500000000000005,"5min":0.80500000000000005,"15min":0.80500000000000005},"min":{"1min":0.35899999999999999,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":3.129,"5min":3.129,"15min":3.129},"last":0.63}]}]},{"osd":6,"up_from":164,"seq":704374636562,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":864096,"kb_used_data":790680,"kb_used_omap":10,"kb_used_meta":73397,"kb_avail":92881056,"statfs":{"total":95995035648,"available":95110201344,"internally_reserved":0,"allocated":809656320,"data_stored":808953216,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":10328,"internal_metadata":75159464},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:56:37 2023","interfaces":[{"interface":"back","average":{"1min":0.86499999999999999,"5min":0.86499999999999999,"15min":0.86499999999999999},"min":{"1min":0.26300000000000001,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":3.7629999999999999,"5min":3.7629999999999999,"15min":3.7629999999999999},"last":1.853},{"interface":"front","average":{"1min":0.98299999999999998,"5min":0.98299999999999998,"15min":0.98299999999999998},"min":{"1min":0.22700000000000001,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":5.0519999999999996,"5min":5.0519999999999996,"15min":5.0519999999999996},"last":2.1120000000000001}]},{"osd":1,"last update":"Sat Sep 23 20:56:37 2023","interfaces":[{"interface":"back","average":{"1min":0.78900000000000003,"5min":0.78900000000000003,"15min":0.78900000000000003},"min":{"1min":0.25600000000000001,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":4.5549999999999997,"5min":4.5549999999999997,"15min":4.5549999999999997},"last":0.53200000000000003},{"interface":"front","average":{"1min":0.86799999999999999,"5min":0.86799999999999999,"15min":0.86799999999999999},"min":{"1min":0.247,"5min":0.247,"15min":0.247},"max":{"1min":3.5870000000000002,"5min":3.5870000000000002,"15min":3.5870000000000002},"last":1.3540000000000001}]},{"osd":2,"last update":"Sat Sep 23 20:56:37 2023","interfaces":[{"interface":"back","average":{"1min":1.647,"5min":1.647,"15min":1.647},"min":{"1min":0.27700000000000002,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":6.6829999999999998,"5min":6.6829999999999998,"15min":6.6829999999999998},"last":0.42799999999999999},{"interface":"front","average":{"1min":1.107,"5min":1.107,"15min":1.107},"min":{"1min":0.30199999999999999,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":3.9580000000000002,"5min":3.9580000000000002,"15min":3.9580000000000002},"last":1.6779999999999999}]},{"osd":3,"last update":"Sat Sep 23 20:56:37 2023","interfaces":[{"interface":"back","average":{"1min":1.117,"5min":1.117,"15min":1.117},"min":{"1min":0.32000000000000001,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":5.8739999999999997,"5min":5.8739999999999997,"15min":5.8739999999999997},"last":1.8779999999999999},{"interface":"front","average":{"1min":1.603,"5min":1.603,"15min":1.603},"min":{"1min":0.374,"5min":0.374,"15min":0.374},"max":{"1min":7.234,"5min":7.234,"15min":7.234},"last":1.7350000000000001}]},{"osd":4,"last update":"Sat Sep 23 20:56:37 2023","interfaces":[{"interface":"back","average":{"1min":1.052,"5min":1.052,"15min":1.052},"min":{"1min":0.27200000000000002,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":9.0079999999999991,"5min":9.0079999999999991,"15min":9.0079999999999991},"last":1.3},{"interface":"front","average":{"1min":0.53100000000000003,"5min":0.53100000000000003,"15min":0.53100000000000003},"min":{"1min":0.31,"5min":0.31,"15min":0.31},"max":{"1min":0.998,"5min":0.998,"15min":0.998},"last":0.441}]},{"osd":5,"last update":"Sat Sep 23 20:56:37 2023","interfaces":[{"interface":"back","average":{"1min":0.54100000000000004,"5min":0.54100000000000004,"15min":0.54100000000000004},"min":{"1min":0.28599999999999998,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":1.1599999999999999,"5min":1.1599999999999999,"15min":1.1599999999999999},"last":1.3360000000000001},{"interface":"front","average":{"1min":0.81000000000000005,"5min":0.81000000000000005,"15min":0.81000000000000005},"min":{"1min":0.40100000000000002,"5min":0.40100000000000002,"15min":0.40100000000000002},"max":{"1min":2.274,"5min":2.274,"15min":2.274},"last":0.39100000000000001}]},{"osd":7,"last update":"Sat Sep 23 20:56:37 2023","interfaces":[{"interface":"back","average":{"1min":0.92900000000000005,"5min":0.92900000000000005,"15min":0.92900000000000005},"min":{"1min":0.34499999999999997,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":3.847,"5min":3.847,"15min":3.847},"last":7.8570000000000002},{"interface":"front","average":{"1min":0.53000000000000003,"5min":0.53000000000000003,"15min":0.53000000000000003},"min":{"1min":0.37,"5min":0.37,"15min":0.37},"max":{"1min":0.93899999999999995,"5min":0.93899999999999995,"15min":0.93899999999999995},"last":8.282}]}]},{"osd":1,"up_from":88,"seq":377957122133,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1244816,"kb_used_data":1078728,"kb_used_omap":9,"kb_used_meta":166070,"kb_avail":92500336,"statfs":{"total":95995035648,"available":94720344064,"internally_reserved":0,"allocated":1104617472,"data_stored":1103639954,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9239,"internal_metadata":170056681},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:56:15 2023","interfaces":[{"interface":"back","average":{"1min":2.8239999999999998,"5min":2.1600000000000001,"15min":0.99199999999999999},"min":{"1min":0.221,"5min":0.216,"15min":0.216},"max":{"1min":9.9700000000000006,"5min":14.372,"15min":14.372},"last":1.159},{"interface":"front","average":{"1min":3.044,"5min":2.4180000000000001,"15min":1.0960000000000001},"min":{"1min":0.26600000000000001,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":9.2240000000000002,"5min":18.457999999999998,"15min":18.457999999999998},"last":1.76}]},{"osd":2,"last update":"Sat Sep 23 20:56:49 2023","interfaces":[{"interface":"back","average":{"1min":0.86499999999999999,"5min":1.8640000000000001,"15min":0.879},"min":{"1min":0.223,"5min":0.20300000000000001,"15min":0.20300000000000001},"max":{"1min":3.9510000000000001,"5min":14.335000000000001,"15min":14.335000000000001},"last":1.4890000000000001},{"interface":"front","average":{"1min":1.901,"5min":2.5219999999999998,"15min":1.23},"min":{"1min":0.29099999999999998,"5min":0.248,"15min":0.248},"max":{"1min":8.6129999999999995,"5min":38.951000000000001,"15min":38.951000000000001},"last":4.2190000000000003}]},{"osd":3,"last update":"Sat Sep 23 20:56:20 2023","interfaces":[{"interface":"back","average":{"1min":3.0339999999999998,"5min":2.7360000000000002,"15min":1.151},"min":{"1min":0.316,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":16.055,"5min":25.314,"15min":25.314},"last":0.95799999999999996},{"interface":"front","average":{"1min":2.2719999999999998,"5min":2.198,"15min":1.006},"min":{"1min":0.20599999999999999,"5min":0.20599999999999999,"15min":0.20599999999999999},"max":{"1min":7.9409999999999998,"5min":19.881,"15min":19.881},"last":1.2030000000000001}]},{"osd":4,"last update":"Sat Sep 23 20:56:15 2023","interfaces":[{"interface":"back","average":{"1min":1.8660000000000001,"5min":1.8799999999999999,"15min":1.0580000000000001},"min":{"1min":0.28899999999999998,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":8.8369999999999997,"5min":12.006,"15min":12.006},"last":1.0329999999999999},{"interface":"front","average":{"1min":1.2470000000000001,"5min":1.3180000000000001,"15min":0.88800000000000001},"min":{"1min":0.27600000000000002,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":5.7160000000000002,"5min":11.234999999999999,"15min":11.234999999999999},"last":1.232}]},{"osd":5,"last update":"Sat Sep 23 20:56:15 2023","interfaces":[{"interface":"back","average":{"1min":2.399,"5min":1.601,"15min":0.999},"min":{"1min":0.36299999999999999,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":7.694,"5min":12.222,"15min":12.222},"last":1.958},{"interface":"front","average":{"1min":2.0609999999999999,"5min":1.946,"15min":1.0800000000000001},"min":{"1min":0.32600000000000001,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":8.8729999999999993,"5min":12.08,"15min":12.08},"last":1.0609999999999999}]},{"osd":6,"last update":"Sat Sep 23 20:56:35 2023","interfaces":[{"interface":"back","average":{"1min":1.3149999999999999,"5min":1.3149999999999999,"15min":1.3149999999999999},"min":{"1min":0.36799999999999999,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":4.0910000000000002,"5min":4.0910000000000002,"15min":4.0910000000000002},"last":0.995},{"interface":"front","average":{"1min":1.2529999999999999,"5min":1.2529999999999999,"15min":1.2529999999999999},"min":{"1min":0.309,"5min":0.309,"15min":0.309},"max":{"1min":6.5670000000000002,"5min":6.5670000000000002,"15min":6.5670000000000002},"last":1.254}]},{"osd":7,"last update":"Sat Sep 23 20:56:15 2023","interfaces":[{"interface":"back","average":{"1min":1.7230000000000001,"5min":1.472,"15min":0.93999999999999995},"min":{"1min":0.36199999999999999,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":6.4699999999999998,"5min":11.186999999999999,"15min":11.186999999999999},"last":1.181},{"interface":"front","average":{"1min":2.1739999999999999,"5min":1.6140000000000001,"15min":0.995},"min":{"1min":0.40899999999999997,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":7.6589999999999998,"5min":12.252000000000001,"15min":12.252000000000001},"last":2.0009999999999999}]}]},{"osd":0,"up_from":83,"seq":356482285659,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1248256,"kb_used_data":1118200,"kb_used_omap":28,"kb_used_meta":130019,"kb_avail":92496896,"statfs":{"total":95995035648,"available":94716821504,"internally_reserved":0,"allocated":1145036800,"data_stored":1144074137,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":28848,"internal_metadata":133140304},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 20:56:17 2023","interfaces":[{"interface":"back","average":{"1min":2.6160000000000001,"5min":2.0910000000000002,"15min":0.91900000000000004},"min":{"1min":0.191,"5min":0.191,"15min":0.191},"max":{"1min":8.6440000000000001,"5min":18.863,"15min":18.863},"last":0.34300000000000003},{"interface":"front","average":{"1min":3.2170000000000001,"5min":2.1280000000000001,"15min":0.96699999999999997},"min":{"1min":0.27600000000000002,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":11.098000000000001,"5min":17.704999999999998,"15min":17.704999999999998},"last":1.401}]},{"osd":2,"last update":"Sat Sep 23 20:56:44 2023","interfaces":[{"interface":"back","average":{"1min":3.742,"5min":2.7519999999999998,"15min":1.194},"min":{"1min":0.30599999999999999,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":14.976000000000001,"5min":17.753,"15min":17.753},"last":1.4610000000000001},{"interface":"front","average":{"1min":1.792,"5min":2.4319999999999999,"15min":1.0700000000000001},"min":{"1min":0.25800000000000001,"5min":0.21099999999999999,"15min":0.21099999999999999},"max":{"1min":6.2750000000000004,"5min":13.218999999999999,"15min":13.218999999999999},"last":0.63400000000000001}]},{"osd":3,"last update":"Sat Sep 23 20:56:17 2023","interfaces":[{"interface":"back","average":{"1min":2.6120000000000001,"5min":2.169,"15min":0.98999999999999999},"min":{"1min":0.25800000000000001,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":8.8810000000000002,"5min":18.893999999999998,"15min":18.893999999999998},"last":0.27400000000000002},{"interface":"front","average":{"1min":4.0919999999999996,"5min":2.5059999999999998,"15min":1.1539999999999999},"min":{"1min":0.28000000000000003,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":14.923999999999999,"5min":17.472000000000001,"15min":17.472000000000001},"last":1.506}]},{"osd":4,"last update":"Sat Sep 23 20:56:44 2023","interfaces":[{"interface":"back","average":{"1min":1.3109999999999999,"5min":1.6619999999999999,"15min":0.98599999999999999},"min":{"1min":0.29399999999999998,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":6.3109999999999999,"5min":12.807,"15min":12.807},"last":0.55300000000000005},{"interface":"front","average":{"1min":1.5389999999999999,"5min":1.5680000000000001,"15min":0.96399999999999997},"min":{"1min":0.31,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":8.7029999999999994,"5min":18.724,"15min":18.724},"last":0.311}]},{"osd":5,"last update":"Sat Sep 23 20:56:44 2023","interfaces":[{"interface":"back","average":{"1min":1.9790000000000001,"5min":1.6970000000000001,"15min":0.995},"min":{"1min":0.318,"5min":0.318,"15min":0.318},"max":{"1min":8.4990000000000006,"5min":8.4990000000000006,"15min":8.4990000000000006},"last":1.5389999999999999},{"interface":"front","average":{"1min":1.3100000000000001,"5min":1.784,"15min":1.0589999999999999},"min":{"1min":0.39300000000000002,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":6.3390000000000004,"5min":13.154999999999999,"15min":13.154999999999999},"last":0.58499999999999996}]},{"osd":6,"last update":"Sat Sep 23 20:56:39 2023","interfaces":[{"interface":"back","average":{"1min":1.657,"5min":1.657,"15min":1.657},"min":{"1min":0.41799999999999998,"5min":0.41799999999999998,"15min":0.41799999999999998},"max":{"1min":8.7309999999999999,"5min":8.7309999999999999,"15min":8.7309999999999999},"last":0.374},{"interface":"front","average":{"1min":1.746,"5min":1.746,"15min":1.746},"min":{"1min":0.45900000000000002,"5min":0.45900000000000002,"15min":0.45900000000000002},"max":{"1min":8.6760000000000002,"5min":8.6760000000000002,"15min":8.6760000000000002},"last":0.44}]},{"osd":7,"last update":"Sat Sep 23 20:56:44 2023","interfaces":[{"interface":"back","average":{"1min":2.0699999999999998,"5min":1.702,"15min":1.03},"min":{"1min":0.379,"5min":0.379,"15min":0.379},"max":{"1min":8.4339999999999993,"5min":8.4339999999999993,"15min":8.4339999999999993},"last":1.5700000000000001},{"interface":"front","average":{"1min":1.673,"5min":1.5980000000000001,"15min":0.97899999999999998},"min":{"1min":0.42699999999999999,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":8.7539999999999996,"5min":18.673999999999999,"15min":18.673999999999999},"last":0.40100000000000002}]}]},{"osd":2,"up_from":93,"seq":399431958606,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1191668,"kb_used_data":1000364,"kb_used_omap":81,"kb_used_meta":191214,"kb_avail":92553484,"statfs":{"total":95995035648,"available":94774767616,"internally_reserved":0,"allocated":1024372736,"data_stored":1023419243,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":83750,"internal_metadata":195803354},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:56:42 2023","interfaces":[{"interface":"back","average":{"1min":1.3260000000000001,"5min":1.5960000000000001,"15min":0.77000000000000002},"min":{"1min":0.17999999999999999,"5min":0.17999999999999999,"15min":0.17999999999999999},"max":{"1min":7.3890000000000002,"5min":11.183,"15min":11.183},"last":0.85799999999999998},{"interface":"front","average":{"1min":1.9039999999999999,"5min":2.008,"15min":0.93999999999999995},"min":{"1min":0.26600000000000001,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":7.4039999999999999,"5min":12.641,"15min":12.641},"last":0.60199999999999998}]},{"osd":1,"last update":"Sat Sep 23 20:56:42 2023","interfaces":[{"interface":"back","average":{"1min":1.5660000000000001,"5min":2.4980000000000002,"15min":1.1100000000000001},"min":{"1min":0.28499999999999998,"5min":0.20200000000000001,"15min":0.20200000000000001},"max":{"1min":5.0869999999999997,"5min":16.081,"15min":16.081},"last":0.308},{"interface":"front","average":{"1min":1.927,"5min":2.9670000000000001,"15min":1.2689999999999999},"min":{"1min":0.23599999999999999,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":11.717000000000001,"5min":16.041,"15min":16.041},"last":0.35599999999999998}]},{"osd":3,"last update":"Sat Sep 23 20:56:20 2023","interfaces":[{"interface":"back","average":{"1min":2.4700000000000002,"5min":2.6920000000000002,"15min":1.173},"min":{"1min":0.28699999999999998,"5min":0.255,"15min":0.255},"max":{"1min":9.5600000000000005,"5min":16.611000000000001,"15min":16.611000000000001},"last":4.077},{"interface":"front","average":{"1min":2.5489999999999999,"5min":3.3849999999999998,"15min":1.381},"min":{"1min":0.23899999999999999,"5min":0.221,"15min":0.221},"max":{"1min":7.9050000000000002,"5min":17.138999999999999,"15min":17.138999999999999},"last":2.0379999999999998}]},{"osd":4,"last update":"Sat Sep 23 20:56:42 2023","interfaces":[{"interface":"back","average":{"1min":1.399,"5min":1.6180000000000001,"15min":0.95199999999999996},"min":{"1min":0.33500000000000002,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":7.2539999999999996,"5min":9.8119999999999994,"15min":9.8119999999999994},"last":0.42399999999999999},{"interface":"front","average":{"1min":1.1080000000000001,"5min":1.786,"15min":1.042},"min":{"1min":0.313,"5min":0.313,"15min":0.313},"max":{"1min":3.7530000000000001,"5min":10.609,"15min":10.609},"last":0.57799999999999996}]},{"osd":5,"last update":"Sat Sep 23 20:56:42 2023","interfaces":[{"interface":"back","average":{"1min":1.754,"5min":1.899,"15min":1.0669999999999999},"min":{"1min":0.36399999999999999,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":7.3470000000000004,"5min":9.7629999999999999,"15min":9.7629999999999999},"last":0.45000000000000001},{"interface":"front","average":{"1min":0.91500000000000004,"5min":1.8919999999999999,"15min":1.054},"min":{"1min":0.33700000000000002,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":3.4900000000000002,"5min":10.574,"15min":10.574},"last":0.52900000000000003}]},{"osd":6,"last update":"Sat Sep 23 20:56:37 2023","interfaces":[{"interface":"back","average":{"1min":1.6679999999999999,"5min":1.6679999999999999,"15min":1.6679999999999999},"min":{"1min":0.42499999999999999,"5min":0.42499999999999999,"15min":0.42499999999999999},"max":{"1min":14.244999999999999,"5min":14.244999999999999,"15min":14.244999999999999},"last":0.50600000000000001},{"interface":"front","average":{"1min":1.03,"5min":1.03,"15min":1.03},"min":{"1min":0.40999999999999998,"5min":0.40999999999999998,"15min":0.40999999999999998},"max":{"1min":3.516,"5min":3.516,"15min":3.516},"last":0.65900000000000003}]},{"osd":7,"last update":"Sat Sep 23 20:56:42 2023","interfaces":[{"interface":"back","average":{"1min":1.149,"5min":1.3939999999999999,"15min":0.88800000000000001},"min":{"1min":0.38900000000000001,"5min":0.371,"15min":0.371},"max":{"1min":4.9119999999999999,"5min":8.0820000000000007,"15min":8.0820000000000007},"last":0.82899999999999996},{"interface":"front","average":{"1min":2.1110000000000002,"5min":1.925,"15min":1.083},"min":{"1min":0.35699999999999998,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":7.9119999999999999,"5min":9.7859999999999996,"15min":9.7859999999999996},"last":0.56399999999999995}]}]},{"osd":3,"up_from":98,"seq":420906795080,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1171668,"kb_used_data":944204,"kb_used_omap":7,"kb_used_meta":227448,"kb_avail":92573484,"statfs":{"total":95995035648,"available":94795247616,"internally_reserved":0,"allocated":966864896,"data_stored":965939740,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7576,"internal_metadata":232907368},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:56:21 2023","interfaces":[{"interface":"back","average":{"1min":1.2589999999999999,"5min":3.0129999999999999,"15min":1.2509999999999999},"min":{"1min":0.20599999999999999,"5min":0.20599999999999999,"15min":0.20599999999999999},"max":{"1min":5.0750000000000002,"5min":23.123999999999999,"15min":23.123999999999999},"last":0.71399999999999997},{"interface":"front","average":{"1min":2.8359999999999999,"5min":2.423,"15min":1.046},"min":{"1min":0.309,"5min":0.20000000000000001,"15min":0.20000000000000001},"max":{"1min":13.323,"5min":15.079000000000001,"15min":15.079000000000001},"last":0.82799999999999996}]},{"osd":1,"last update":"Sat Sep 23 20:56:21 2023","interfaces":[{"interface":"back","average":{"1min":2.4430000000000001,"5min":2.609,"15min":1.1140000000000001},"min":{"1min":0.249,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":16.318000000000001,"5min":21.385000000000002,"15min":21.385000000000002},"last":0.68000000000000005},{"interface":"front","average":{"1min":0.88600000000000001,"5min":2.9820000000000002,"15min":1.2529999999999999},"min":{"1min":0.218,"5min":0.218,"15min":0.218},"max":{"1min":3.2509999999999999,"5min":25.353000000000002,"15min":25.353000000000002},"last":0.45900000000000002}]},{"osd":2,"last update":"Sat Sep 23 20:56:21 2023","interfaces":[{"interface":"back","average":{"1min":1.1870000000000001,"5min":2.2639999999999998,"15min":1.038},"min":{"1min":0.33900000000000002,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":4.5060000000000002,"5min":24.154,"15min":24.154},"last":0.88300000000000001},{"interface":"front","average":{"1min":1.8660000000000001,"5min":3.141,"15min":1.339},"min":{"1min":0.32600000000000001,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":16.032,"5min":25.305,"15min":25.305},"last":0.93500000000000005}]},{"osd":4,"last update":"Sat Sep 23 20:56:21 2023","interfaces":[{"interface":"back","average":{"1min":0.73399999999999999,"5min":1.5249999999999999,"15min":0.92900000000000005},"min":{"1min":0.29899999999999999,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":2.1840000000000002,"5min":11.861000000000001,"15min":11.861000000000001},"last":0.63900000000000001},{"interface":"front","average":{"1min":0.90200000000000002,"5min":2.266,"15min":1.173},"min":{"1min":0.316,"5min":0.316,"15min":0.316},"max":{"1min":3.2970000000000002,"5min":22.853000000000002,"15min":22.853000000000002},"last":0.48999999999999999}]},{"osd":5,"last update":"Sat Sep 23 20:56:21 2023","interfaces":[{"interface":"back","average":{"1min":0.78300000000000003,"5min":1.4490000000000001,"15min":0.91300000000000003},"min":{"1min":0.32500000000000001,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":2.1400000000000001,"5min":11.891999999999999,"15min":11.891999999999999},"last":0.73999999999999999},{"interface":"front","average":{"1min":1.762,"5min":1.5329999999999999,"15min":0.93500000000000005},"min":{"1min":0.376,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":7.1479999999999997,"5min":9.2870000000000008,"15min":9.2870000000000008},"last":0.72599999999999998}]},{"osd":6,"last update":"Sat Sep 23 20:56:35 2023","interfaces":[{"interface":"back","average":{"1min":0.94999999999999996,"5min":0.94999999999999996,"15min":0.94999999999999996},"min":{"1min":0.36799999999999999,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":3.3279999999999998,"5min":3.3279999999999998,"15min":3.3279999999999998},"last":0.51700000000000002},{"interface":"front","average":{"1min":1.4119999999999999,"5min":1.4119999999999999,"15min":1.4119999999999999},"min":{"1min":0.46800000000000003,"5min":0.46800000000000003,"15min":0.46800000000000003},"max":{"1min":7.234,"5min":7.234,"15min":7.234},"last":0.77200000000000002}]},{"osd":7,"last update":"Sat Sep 23 20:56:21 2023","interfaces":[{"interface":"back","average":{"1min":1.7190000000000001,"5min":1.8640000000000001,"15min":1.0840000000000001},"min":{"1min":0.36099999999999999,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":16.364999999999998,"5min":16.364999999999998,"15min":16.364999999999998},"last":0.79000000000000004},{"interface":"front","average":{"1min":1.819,"5min":1.6499999999999999,"15min":1.012},"min":{"1min":0.45800000000000002,"5min":0.379,"15min":0.379},"max":{"1min":7.2619999999999996,"5min":9.3179999999999996,"15min":9.3179999999999996},"last":0.80300000000000005}]}]},{"osd":4,"up_from":31,"seq":133143986387,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":830256,"kb_used_data":733160,"kb_used_omap":0,"kb_used_meta":97088,"kb_avail":92914896,"statfs":{"total":95995035648,"available":95144853504,"internally_reserved":0,"allocated":750755840,"data_stored":750037016,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":99418112},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:56:49 2023","interfaces":[{"interface":"back","average":{"1min":1.157,"5min":1.22,"15min":0.83599999999999997},"min":{"1min":0.23300000000000001,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":5.5090000000000003,"5min":8.7189999999999994,"15min":8.7189999999999994},"last":0.622},{"interface":"front","average":{"1min":1.1859999999999999,"5min":1.4359999999999999,"15min":1.0780000000000001},"min":{"1min":0.33000000000000002,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":5.4989999999999997,"5min":8.7050000000000001,"15min":8.7050000000000001},"last":0.63100000000000001}]},{"osd":1,"last update":"Sat Sep 23 20:56:16 2023","interfaces":[{"interface":"back","average":{"1min":1.5620000000000001,"5min":1.5029999999999999,"15min":0.873},"min":{"1min":0.25900000000000001,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":5.952,"5min":7.9859999999999998,"15min":7.9859999999999998},"last":1.085},{"interface":"front","average":{"1min":1.897,"5min":1.212,"15min":0.80300000000000005},"min":{"1min":0.33300000000000002,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":5.7839999999999998,"5min":6.3540000000000001,"15min":6.3540000000000001},"last":0.35999999999999999}]},{"osd":2,"last update":"Sat Sep 23 20:56:49 2023","interfaces":[{"interface":"back","average":{"1min":0.79300000000000004,"5min":1.5580000000000001,"15min":0.97599999999999998},"min":{"1min":0.313,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":2.778,"5min":15.288,"15min":15.288},"last":1.5289999999999999},{"interface":"front","average":{"1min":0.94099999999999995,"5min":1.845,"15min":1.131},"min":{"1min":0.28299999999999997,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":4.3579999999999997,"5min":15.262,"15min":15.262},"last":0.46200000000000002}]},{"osd":3,"last update":"Sat Sep 23 20:56:16 2023","interfaces":[{"interface":"back","average":{"1min":2.4660000000000002,"5min":2.7989999999999999,"15min":1.367},"min":{"1min":0.308,"5min":0.308,"15min":0.308},"max":{"1min":14.824999999999999,"5min":31.562999999999999,"15min":31.562999999999999},"last":1.0600000000000001},{"interface":"front","average":{"1min":2.714,"5min":2.794,"15min":1.4399999999999999},"min":{"1min":0.441,"5min":0.41899999999999998,"15min":0.41899999999999998},"max":{"1min":14.839,"5min":31.574000000000002,"15min":31.574000000000002},"last":0.77300000000000002}]},{"osd":5,"last update":"Sat Sep 23 20:56:32 2023","interfaces":[{"interface":"back","average":{"1min":0.54700000000000004,"5min":0.505,"15min":0.49199999999999999},"min":{"1min":0.22700000000000001,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":2.1779999999999999,"5min":3.3279999999999998,"15min":3.3279999999999998},"last":0.32900000000000001},{"interface":"front","average":{"1min":0.56100000000000005,"5min":0.65400000000000003,"15min":0.54900000000000004},"min":{"1min":0.25600000000000001,"5min":0.251,"15min":0.251},"max":{"1min":2.2949999999999999,"5min":7.7619999999999996,"15min":7.7619999999999996},"last":0.38900000000000001}]},{"osd":6,"last update":"Sat Sep 23 20:56:38 2023","interfaces":[{"interface":"back","average":{"1min":0.48399999999999999,"5min":0.48399999999999999,"15min":0.48399999999999999},"min":{"1min":0.26500000000000001,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":0.92200000000000004,"5min":0.92200000000000004,"15min":0.92200000000000004},"last":1.0349999999999999},{"interface":"front","average":{"1min":0.64400000000000002,"5min":0.64400000000000002,"15min":0.64400000000000002},"min":{"1min":0.36199999999999999,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":2.419,"5min":2.419,"15min":2.419},"last":0.40300000000000002}]},{"osd":7,"last update":"Sat Sep 23 20:56:16 2023","interfaces":[{"interface":"back","average":{"1min":0.96899999999999997,"5min":0.81100000000000005,"15min":0.65100000000000002},"min":{"1min":0.29499999999999998,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":6.2240000000000002,"5min":7.383,"15min":7.383},"last":0.434},{"interface":"front","average":{"1min":1.0409999999999999,"5min":0.77300000000000002,"15min":0.63300000000000001},"min":{"1min":0.28199999999999997,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":7.8310000000000004,"5min":7.8310000000000004,"15min":7.8310000000000004},"last":0.52700000000000002}]}]},{"osd":5,"up_from":39,"seq":167503724752,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":819300,"kb_used_data":713500,"kb_used_omap":0,"kb_used_meta":105792,"kb_avail":92925852,"statfs":{"total":95995035648,"available":95156072448,"internally_reserved":0,"allocated":730624000,"data_stored":729913752,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":108331008},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:56:36 2023","interfaces":[{"interface":"back","average":{"1min":1.1559999999999999,"5min":0.999,"15min":0.66400000000000003},"min":{"1min":0.22900000000000001,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":4.7949999999999999,"5min":7.766,"15min":7.766},"last":1.45},{"interface":"front","average":{"1min":1.266,"5min":1.3320000000000001,"15min":0.82799999999999996},"min":{"1min":0.251,"5min":0.223,"15min":0.223},"max":{"1min":7.5250000000000004,"5min":15.037000000000001,"15min":15.037000000000001},"last":0.378}]},{"osd":1,"last update":"Sat Sep 23 20:56:13 2023","interfaces":[{"interface":"back","average":{"1min":1.4990000000000001,"5min":1.5029999999999999,"15min":0.878},"min":{"1min":0.23300000000000001,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":4.5990000000000002,"5min":8.3520000000000003,"15min":8.3520000000000003},"last":0.67600000000000005},{"interface":"front","average":{"1min":1.9710000000000001,"5min":1.462,"15min":0.85299999999999998},"min":{"1min":0.22800000000000001,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":9.2100000000000009,"5min":9.2100000000000009,"15min":9.2100000000000009},"last":0.53700000000000003}]},{"osd":2,"last update":"Sat Sep 23 20:56:47 2023","interfaces":[{"interface":"back","average":{"1min":0.90400000000000003,"5min":1.4339999999999999,"15min":0.83899999999999997},"min":{"1min":0.26200000000000001,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":3.161,"5min":12.651,"15min":12.651},"last":0.84999999999999998},{"interface":"front","average":{"1min":1.2310000000000001,"5min":1.492,"15min":0.88600000000000001},"min":{"1min":0.35799999999999998,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":4.6870000000000003,"5min":8.3209999999999997,"15min":8.3209999999999997},"last":0.41799999999999998}]},{"osd":3,"last update":"Sat Sep 23 20:56:13 2023","interfaces":[{"interface":"back","average":{"1min":2.169,"5min":1.744,"15min":0.996},"min":{"1min":0.46100000000000002,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":7.6120000000000001,"5min":9.5489999999999995,"15min":9.5489999999999995},"last":0.60399999999999998},{"interface":"front","average":{"1min":2.367,"5min":1.421,"15min":0.89500000000000002},"min":{"1min":0.373,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":6.673,"5min":6.673,"15min":6.673},"last":1.4950000000000001}]},{"osd":4,"last update":"Sat Sep 23 20:56:29 2023","interfaces":[{"interface":"back","average":{"1min":0.629,"5min":0.61499999999999999,"15min":0.52700000000000002},"min":{"1min":0.29099999999999998,"5min":0.27600000000000002,"15min":0.253},"max":{"1min":3.625,"5min":3.8479999999999999,"15min":3.8479999999999999},"last":0.81799999999999995},{"interface":"front","average":{"1min":0.88,"5min":0.63800000000000001,"15min":0.55300000000000005},"min":{"1min":0.309,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":3.0680000000000001,"5min":3.0680000000000001,"15min":3.0680000000000001},"last":0.90100000000000002}]},{"osd":6,"last update":"Sat Sep 23 20:56:36 2023","interfaces":[{"interface":"back","average":{"1min":0.56100000000000005,"5min":0.56100000000000005,"15min":0.56100000000000005},"min":{"1min":0.28499999999999998,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":1.403,"5min":1.403,"15min":1.403},"last":0.63500000000000001},{"interface":"front","average":{"1min":0.74399999999999999,"5min":0.74399999999999999,"15min":0.74399999999999999},"min":{"1min":0.35999999999999999,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":3.6560000000000001,"5min":3.6560000000000001,"15min":3.6560000000000001},"last":0.94299999999999995}]},{"osd":7,"last update":"Sat Sep 23 20:56:10 2023","interfaces":[{"interface":"back","average":{"1min":0.90600000000000003,"5min":0.70699999999999996,"15min":0.61299999999999999},"min":{"1min":0.32200000000000001,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":4.3920000000000003,"5min":8.5649999999999995,"15min":8.5649999999999995},"last":0.92100000000000004},{"interface":"front","average":{"1min":0.72899999999999998,"5min":0.72099999999999997,"15min":0.63100000000000001},"min":{"1min":0.29799999999999999,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":3.8149999999999999,"5min":4.5430000000000001,"15min":4.5430000000000001},"last":1.0009999999999999}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1158938624,"data_stored":1158938624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1138753536,"data_stored":1138753536,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1013710848,"data_stored":1013710848,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":980484096,"data_stored":980484096,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":753205248,"data_stored":753205248,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":749731840,"data_stored":749731840,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":808910848,"data_stored":808910848,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":760741888,"data_stored":760741888,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T20:57:02.168 INFO:tasks.cephadm.ceph_manager.ceph:increase pool size by 10 2023-09-23T20:57:02.168 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph osd pool set .mgr pg_num 21 2023-09-23T20:57:02.222 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:57:02.226 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:57:02.226 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:57:02.327 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:57:02.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:57:02 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:57:02.371 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:57:02 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:57:02.371 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:57:02 smithi070 ceph-mon[168282]: from='client.45116 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:57:02.381 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:57:02 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:57:02.381 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:57:02 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:57:02.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:57:02 smithi026 ceph-mon[175151]: from='client.45116 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:57:02.385 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:57:02 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:57:02.385 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:57:02 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:57:02.385 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:57:02 smithi026 ceph-mon[179688]: from='client.45116 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:57:02.425 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:57:02.498 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:57:02.506 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:57:02.507 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:57:02.608 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:57:02.645 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:57:02.696 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:57:02.701 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:57:02.701 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:57:02.802 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:57:02.830 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:57:02.871 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:57:02.905 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 24 16 44866 44850 116.74 79.6875 0.00293697 0.00855949 2023-09-23T20:57:02.938 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:57:02.948 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:57:02.948 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:57:03.049 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:57:03.083 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:57:03.112 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:57:03.169 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:57:03.175 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:57:03.175 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:57:03.208 INFO:teuthology.orchestra.run.smithi026.stderr:noscrub is set 2023-09-23T20:57:03.276 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:57:03.325 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:57:03.361 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:57:03 smithi070 ceph-mon[168282]: pgmap v483: 99 pgs: 99 active+clean; 2.5 GiB data, 8.7 GiB used, 707 GiB / 715 GiB avail; 101 MiB/s wr, 1.61k op/s 2023-09-23T20:57:03.362 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:57:03 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/4193819682' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:57:03.442 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:57:03 smithi026 ceph-mon[175151]: pgmap v483: 99 pgs: 99 active+clean; 2.5 GiB data, 8.7 GiB used, 707 GiB / 715 GiB avail; 101 MiB/s wr, 1.61k op/s 2023-09-23T20:57:03.442 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:57:03 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/4193819682' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:57:03.445 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:57:03 smithi026 ceph-mon[179688]: pgmap v483: 99 pgs: 99 active+clean; 2.5 GiB data, 8.7 GiB used, 707 GiB / 715 GiB avail; 101 MiB/s wr, 1.61k op/s 2023-09-23T20:57:03.445 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:57:03 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/4193819682' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:57:03.563 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:57:03.566 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:57:03.566 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:57:03.667 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:57:03.716 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:57:03.748 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T20:57:03.781 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:57:03.785 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:57:03.785 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:57:03.886 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:57:03.905 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 25 16 46219 46203 115.453 84.5625 0.0223408 0.00865458 2023-09-23T20:57:03.920 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:57:03.973 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:57:03.978 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:57:03.979 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:57:04.079 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:57:04.117 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:57:04.176 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:57:04.180 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:57:04.180 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:57:04.198 INFO:teuthology.orchestra.run.smithi070.stdout:{ 2023-09-23T20:57:04.198 INFO:teuthology.orchestra.run.smithi070.stdout: "ops": [], 2023-09-23T20:57:04.198 INFO:teuthology.orchestra.run.smithi070.stdout: "complaint_time": 30, 2023-09-23T20:57:04.198 INFO:teuthology.orchestra.run.smithi070.stdout: "num_blocked_ops": 0 2023-09-23T20:57:04.199 INFO:teuthology.orchestra.run.smithi070.stdout:} 2023-09-23T20:57:04.281 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:57:04.362 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:57:04 smithi026 ceph-mon[179688]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:57:04.363 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:57:04 smithi026 ceph-mon[179688]: failed to encode map e207 with expected crc 2023-09-23T20:57:04.363 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:57:04 smithi026 ceph-mon[179688]: failed to encode map e207 with expected crc 2023-09-23T20:57:04.363 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:57:04 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/4193819682' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T20:57:04.363 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:57:04 smithi026 ceph-mon[179688]: osdmap e207: 8 total, 8 up, 8 in 2023-09-23T20:57:04.363 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:57:04 smithi026 ceph-mon[179688]: failed to encode map e207 with expected crc 2023-09-23T20:57:04.363 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:57:04 smithi026 ceph-mon[179688]: failed to encode map e207 with expected crc 2023-09-23T20:57:04.363 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:57:04 smithi026 ceph-mon[175151]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:57:04.363 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:57:04 smithi026 ceph-mon[175151]: failed to encode map e207 with expected crc 2023-09-23T20:57:04.364 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:57:04 smithi026 ceph-mon[175151]: failed to encode map e207 with expected crc 2023-09-23T20:57:04.364 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:57:04 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/4193819682' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T20:57:04.364 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:57:04 smithi026 ceph-mon[175151]: osdmap e207: 8 total, 8 up, 8 in 2023-09-23T20:57:04.364 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:57:04 smithi026 ceph-mon[175151]: failed to encode map e207 with expected crc 2023-09-23T20:57:04.364 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:57:04 smithi026 ceph-mon[175151]: failed to encode map e207 with expected crc 2023-09-23T20:57:04.377 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:57:04.460 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:57:04.464 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:57:04.465 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:57:04.509 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:57:04 smithi070 ceph-mon[168282]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:57:04.509 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:57:04 smithi070 ceph-mon[168282]: failed to encode map e207 with expected crc 2023-09-23T20:57:04.510 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:57:04 smithi070 ceph-mon[168282]: failed to encode map e207 with expected crc 2023-09-23T20:57:04.510 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:57:04 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/4193819682' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T20:57:04.510 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:57:04 smithi070 ceph-mon[168282]: osdmap e207: 8 total, 8 up, 8 in 2023-09-23T20:57:04.510 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:57:04 smithi070 ceph-mon[168282]: failed to encode map e207 with expected crc 2023-09-23T20:57:04.510 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:57:04 smithi070 ceph-mon[168282]: failed to encode map e207 with expected crc 2023-09-23T20:57:04.543 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_historic_ops 2023-09-23T20:57:04.565 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:57:04.622 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:57:04.673 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:57:04.678 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:57:04.678 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:57:04.779 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:57:04.819 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:57:04.882 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:57:04.885 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:57:04.886 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:57:04.905 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 26 16 47544 47528 114.198 82.8125 0.0123359 0.00874978 2023-09-23T20:57:04.923 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:57:04.987 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:57:05.068 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:57:05.160 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:57:05.165 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:57:05.166 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:57:05.266 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:57:05.297 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:57:05.302 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:57:05.435 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:57:05.438 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:57:05.438 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:57:05.539 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:57:05.589 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:57:05.645 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:57:05.650 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:57:05.650 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:57:05.751 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:57:05.796 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:57:05.858 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T20:57:05.898 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:57:05.901 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:57:05.901 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:57:05.905 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 27 16 48880 48864 113.061 83.5 0.00810688 0.00883899 2023-09-23T20:57:06.002 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:57:06.128 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:57:06.232 INFO:teuthology.orchestra.run.smithi026.stderr:set pool 1 pg_num to 21 2023-09-23T20:57:06.255 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:57:05 smithi026 ceph-mon[179688]: pgmap v485: 99 pgs: 99 active+clean; 2.7 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 94 MiB/s wr, 1.51k op/s 2023-09-23T20:57:06.256 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:57:05 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/3413087043' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "21"}]: dispatch 2023-09-23T20:57:06.256 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:57:05 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "21"}]: dispatch 2023-09-23T20:57:06.256 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:57:05 smithi026 ceph-mon[175151]: pgmap v485: 99 pgs: 99 active+clean; 2.7 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 94 MiB/s wr, 1.51k op/s 2023-09-23T20:57:06.256 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:57:05 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/3413087043' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "21"}]: dispatch 2023-09-23T20:57:06.256 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:57:05 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "21"}]: dispatch 2023-09-23T20:57:06.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:57:05 smithi070 ceph-mon[168282]: pgmap v485: 99 pgs: 99 active+clean; 2.7 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 94 MiB/s wr, 1.51k op/s 2023-09-23T20:57:06.371 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:57:05 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/3413087043' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "21"}]: dispatch 2023-09-23T20:57:06.371 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:57:05 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "21"}]: dispatch 2023-09-23T20:57:06.405 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:57:06.410 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:57:06.411 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:57:06.511 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:57:06.599 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:57:06.687 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:57:06.693 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:57:06.693 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:57:06.729 INFO:teuthology.orchestra.run.smithi070.stdout:{ 2023-09-23T20:57:06.729 INFO:teuthology.orchestra.run.smithi070.stdout: "size": 20, 2023-09-23T20:57:06.729 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 600, 2023-09-23T20:57:06.730 INFO:teuthology.orchestra.run.smithi070.stdout: "ops": [ 2023-09-23T20:57:06.730 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:57:06.730 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.35100.0:20313 7.c 7:3978b553:::benchmark_data_smithi026_260489_object20312:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e205)", 2023-09-23T20:57:06.730 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T20:56:47.772969+0000", 2023-09-23T20:57:06.730 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 18.953810969999999, 2023-09-23T20:57:06.730 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.047451477999999998, 2023-09-23T20:57:06.730 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T20:57:06.730 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:57:06.730 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T20:57:06.730 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.35100", 2023-09-23T20:57:06.730 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/4108972961", 2023-09-23T20:57:06.730 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 20313 2023-09-23T20:57:06.731 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:57:06.731 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T20:57:06.731 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:57:06.731 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T20:57:06.731 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:56:47.772969+0000", 2023-09-23T20:57:06.731 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T20:57:06.731 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:57:06.731 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:57:06.731 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T20:57:06.731 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:56:47.772969+0000", 2023-09-23T20:57:06.731 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T20:57:06.731 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:57:06.732 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:57:06.732 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T20:57:06.732 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:56:47.772968+0000", 2023-09-23T20:57:06.732 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.999999 2023-09-23T20:57:06.732 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:57:06.732 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:57:06.732 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T20:57:06.732 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:56:47.773001+0000", 2023-09-23T20:57:06.732 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.2453999999999998e-05 2023-09-23T20:57:06.732 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:57:06.732 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:57:06.732 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T20:57:06.733 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:56:47.773016+0000", 2023-09-23T20:57:06.733 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.5082999999999999e-05 2023-09-23T20:57:06.733 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:57:06.733 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:57:06.733 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T20:57:06.733 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:56:47.773026+0000", 2023-09-23T20:57:06.733 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.825e-06 2023-09-23T20:57:06.733 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:57:06.733 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:57:06.733 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T20:57:06.733 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:56:47.773078+0000", 2023-09-23T20:57:06.733 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.2754999999999999e-05 2023-09-23T20:57:06.733 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:57:06.734 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:57:06.734 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T20:57:06.734 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:56:47.773212+0000", 2023-09-23T20:57:06.734 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000133557 2023-09-23T20:57:06.734 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:57:06.734 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:57:06.734 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 2,7", 2023-09-23T20:57:06.734 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:56:47.773371+0000", 2023-09-23T20:57:06.734 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00015934900000000001 2023-09-23T20:57:06.734 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:57:06.734 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:57:06.734 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T20:57:06.735 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:56:47.774378+0000", 2023-09-23T20:57:06.735 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.001006715 2023-09-23T20:57:06.735 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:57:06.735 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:57:06.735 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:57:06.735 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:56:47.774878+0000", 2023-09-23T20:57:06.735 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00050008699999999997 2023-09-23T20:57:06.735 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:57:06.735 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:57:06.735 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:57:06.735 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:56:47.820327+0000", 2023-09-23T20:57:06.735 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.045449151 2023-09-23T20:57:06.736 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:57:06.736 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:57:06.736 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T20:57:06.736 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:56:47.820381+0000", 2023-09-23T20:57:06.736 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.3978999999999999e-05 2023-09-23T20:57:06.736 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:57:06.736 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:57:06.736 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T20:57:06.736 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:56:47.820421+0000", 2023-09-23T20:57:06.736 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.9706999999999998e-05 2023-09-23T20:57:06.736 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T20:57:06.736 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T20:57:06.737 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T20:57:06.737 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:57:06.737 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:57:06.737 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.35100.0:20730 7.d 7:b947c047:::benchmark_data_smithi026_260489_object20729:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e205)", 2023-09-23T20:57:06.737 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T20:56:47.965582+0000", 2023-09-23T20:57:06.737 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 18.761198036, 2023-09-23T20:57:06.737 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.046620743999999999, 2023-09-23T20:57:06.737 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T20:57:06.737 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:57:06.737 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T20:57:06.737 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.35100", 2023-09-23T20:57:06.737 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/4108972961", 2023-09-23T20:57:06.737 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 20730 2023-09-23T20:57:06.738 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:57:06.738 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T20:57:06.738 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:57:06.738 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T20:57:06.738 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:56:47.965582+0000", 2023-09-23T20:57:06.738 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T20:57:06.738 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:57:06.738 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:57:06.738 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T20:57:06.738 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:56:47.965582+0000", 2023-09-23T20:57:06.738 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T20:57:06.738 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:57:06.739 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:57:06.739 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T20:57:06.739 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:56:47.965581+0000", 2023-09-23T20:57:06.739 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999981 2023-09-23T20:57:06.739 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:57:06.739 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:57:06.739 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T20:57:06.739 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:56:47.965616+0000", 2023-09-23T20:57:06.739 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.5865999999999998e-05 2023-09-23T20:57:06.739 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:57:06.739 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:57:06.739 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T20:57:06.739 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:56:47.965630+0000", 2023-09-23T20:57:06.740 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.3309000000000001e-05 2023-09-23T20:57:06.740 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:57:06.740 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:57:06.740 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T20:57:06.740 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:56:47.965637+0000", 2023-09-23T20:57:06.740 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.9890000000000003e-06 2023-09-23T20:57:06.740 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:57:06.740 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:57:06.740 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T20:57:06.740 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:56:47.965684+0000", 2023-09-23T20:57:06.740 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.6832000000000002e-05 2023-09-23T20:57:06.740 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:57:06.741 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:57:06.741 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T20:57:06.741 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:56:47.965825+0000", 2023-09-23T20:57:06.741 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00014091400000000001 2023-09-23T20:57:06.741 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:57:06.741 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:57:06.741 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 0,1", 2023-09-23T20:57:06.741 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:56:47.965982+0000", 2023-09-23T20:57:06.741 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00015793900000000001 2023-09-23T20:57:06.741 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:57:06.741 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:57:06.741 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T20:57:06.741 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:56:47.967846+0000", 2023-09-23T20:57:06.742 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0018633020000000001 2023-09-23T20:57:06.742 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:57:06.742 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:57:06.742 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:57:06.742 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:56:47.968904+0000", 2023-09-23T20:57:06.742 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0010585970000000001 2023-09-23T20:57:06.742 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:57:06.742 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:57:06.742 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:57:06.742 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:56:48.012120+0000", 2023-09-23T20:57:06.742 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.043215569000000002 2023-09-23T20:57:06.742 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:57:06.743 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:57:06.743 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T20:57:06.743 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:56:48.012163+0000", 2023-09-23T20:57:06.743 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.3445999999999997e-05 2023-09-23T20:57:06.743 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:57:06.743 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:57:06.743 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T20:57:06.743 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:56:48.012203+0000", 2023-09-23T20:57:06.743 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.9690000000000001e-05 2023-09-23T20:57:06.743 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T20:57:06.743 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T20:57:06.743 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T20:57:06.743 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:57:06.744 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:57:06.744 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.35100.0:20734 7.d 7:b14b67d3:::benchmark_data_smithi026_260489_object20733:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e205)", 2023-09-23T20:57:06.744 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T20:56:47.966934+0000", 2023-09-23T20:57:06.744 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 18.759846203999999, 2023-09-23T20:57:06.744 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.045384169000000002, 2023-09-23T20:57:06.744 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T20:57:06.744 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:57:06.744 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T20:57:06.744 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.35100", 2023-09-23T20:57:06.744 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/4108972961", 2023-09-23T20:57:06.744 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 20734 2023-09-23T20:57:06.744 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:57:06.745 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T20:57:06.745 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:57:06.745 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T20:57:06.745 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:56:47.966934+0000", 2023-09-23T20:57:06.745 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T20:57:06.745 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:57:06.745 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:57:06.745 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T20:57:06.745 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:56:47.966934+0000", 2023-09-23T20:57:06.745 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T20:57:06.745 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:57:06.745 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:57:06.745 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T20:57:06.746 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:56:47.966934+0000", 2023-09-23T20:57:06.746 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999995 2023-09-23T20:57:06.746 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:57:06.746 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:57:06.746 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T20:57:06.746 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:56:47.966961+0000", 2023-09-23T20:57:06.746 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.7013999999999999e-05 2023-09-23T20:57:06.746 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:57:06.746 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:57:06.746 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T20:57:06.746 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:56:47.966969+0000", 2023-09-23T20:57:06.746 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.0800000000000006e-06 2023-09-23T20:57:06.747 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:57:06.747 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:57:06.747 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T20:57:06.747 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:56:47.966974+0000", 2023-09-23T20:57:06.747 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.3959999999999997e-06 2023-09-23T20:57:06.747 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:57:06.747 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:57:06.747 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T20:57:06.747 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:56:47.967018+0000", 2023-09-23T20:57:06.747 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.4249000000000003e-05 2023-09-23T20:57:06.747 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:57:06.747 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:57:06.748 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T20:57:06.748 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:56:47.967135+0000", 2023-09-23T20:57:06.748 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000116797 2023-09-23T20:57:06.748 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:57:06.748 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:57:06.748 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 0,1", 2023-09-23T20:57:06.748 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:56:47.967272+0000", 2023-09-23T20:57:06.748 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000136661 2023-09-23T20:57:06.748 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:57:06.748 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:57:06.748 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T20:57:06.748 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:56:47.967997+0000", 2023-09-23T20:57:06.748 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00072486899999999995 2023-09-23T20:57:06.749 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:57:06.749 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:57:06.749 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:57:06.749 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:56:47.969930+0000", 2023-09-23T20:57:06.749 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0019338269999999999 2023-09-23T20:57:06.749 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:57:06.749 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:57:06.749 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:57:06.749 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:56:48.012249+0000", 2023-09-23T20:57:06.749 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.042319015000000001 2023-09-23T20:57:06.749 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:57:06.749 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:57:06.750 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T20:57:06.750 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:56:48.012287+0000", 2023-09-23T20:57:06.750 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.8043999999999997e-05 2023-09-23T20:57:06.750 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:57:06.750 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:57:06.750 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T20:57:06.750 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:56:48.012318+0000", 2023-09-23T20:57:06.750 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.0865999999999999e-05 2023-09-23T20:57:06.750 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T20:57:06.750 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T20:57:06.750 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T20:57:06.750 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:57:06.751 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:57:06.751 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.35100.0:20739 7.7 7:e41605b7:::benchmark_data_smithi026_260489_object20738:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e205)", 2023-09-23T20:57:06.751 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T20:56:47.968554+0000", 2023-09-23T20:57:06.751 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 18.758226270000002, 2023-09-23T20:57:06.751 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.045104374000000003, 2023-09-23T20:57:06.751 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T20:57:06.751 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:57:06.751 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T20:57:06.751 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.35100", 2023-09-23T20:57:06.751 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/4108972961", 2023-09-23T20:57:06.751 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 20739 2023-09-23T20:57:06.751 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:57:06.752 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T20:57:06.752 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:57:06.752 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T20:57:06.752 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:56:47.968554+0000", 2023-09-23T20:57:06.752 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T20:57:06.752 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:57:06.752 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:57:06.752 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T20:57:06.752 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:56:47.968554+0000", 2023-09-23T20:57:06.752 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T20:57:06.752 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:57:06.752 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:57:06.752 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T20:57:06.753 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:56:47.968553+0000", 2023-09-23T20:57:06.753 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.999999 2023-09-23T20:57:06.753 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:57:06.753 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:57:06.753 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T20:57:06.753 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:56:47.968615+0000", 2023-09-23T20:57:06.753 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.1462000000000002e-05 2023-09-23T20:57:06.753 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:57:06.753 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:57:06.753 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T20:57:06.753 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:56:47.968630+0000", 2023-09-23T20:57:06.754 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.4932999999999999e-05 2023-09-23T20:57:06.754 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:57:06.754 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:57:06.754 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T20:57:06.754 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:56:47.968636+0000", 2023-09-23T20:57:06.754 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.5629999999999997e-06 2023-09-23T20:57:06.754 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:57:06.754 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:57:06.754 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T20:57:06.754 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:56:47.968712+0000", 2023-09-23T20:57:06.754 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.5817999999999998e-05 2023-09-23T20:57:06.755 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:57:06.755 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:57:06.755 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T20:57:06.755 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:56:47.968877+0000", 2023-09-23T20:57:06.755 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00016510799999999999 2023-09-23T20:57:06.755 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:57:06.755 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:57:06.755 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 0,3", 2023-09-23T20:57:06.755 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:56:47.969095+0000", 2023-09-23T20:57:06.755 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000217999 2023-09-23T20:57:06.755 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:57:06.755 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:57:06.755 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T20:57:06.756 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:56:47.970244+0000", 2023-09-23T20:57:06.756 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0011492589999999999 2023-09-23T20:57:06.756 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:57:06.756 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:57:06.756 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:57:06.756 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:56:48.012142+0000", 2023-09-23T20:57:06.756 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.041897397000000003 2023-09-23T20:57:06.756 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:57:06.756 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:57:06.756 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:57:06.756 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:56:48.013580+0000", 2023-09-23T20:57:06.756 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0014379110000000001 2023-09-23T20:57:06.757 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:57:06.757 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:57:06.757 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T20:57:06.757 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:56:48.013624+0000", 2023-09-23T20:57:06.757 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.4159000000000002e-05 2023-09-23T20:57:06.757 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:57:06.757 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:57:06.757 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T20:57:06.757 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:56:48.013658+0000", 2023-09-23T20:57:06.757 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.4635999999999998e-05 2023-09-23T20:57:06.757 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T20:57:06.757 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T20:57:06.758 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T20:57:06.758 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:57:06.758 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:57:06.758 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.35100.0:21553 7.7 7:eb31b534:::benchmark_data_smithi026_260489_object21552:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e205)", 2023-09-23T20:57:06.758 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T20:56:48.353043+0000", 2023-09-23T20:57:06.758 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 18.373737591000001, 2023-09-23T20:57:06.758 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.06098083, 2023-09-23T20:57:06.758 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T20:57:06.758 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:57:06.758 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T20:57:06.758 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.35100", 2023-09-23T20:57:06.758 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/4108972961", 2023-09-23T20:57:06.759 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 21553 2023-09-23T20:57:06.759 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:57:06.759 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T20:57:06.759 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:57:06.759 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T20:57:06.759 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:56:48.353043+0000", 2023-09-23T20:57:06.759 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T20:57:06.759 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:57:06.759 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:57:06.759 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T20:57:06.759 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:56:48.353043+0000", 2023-09-23T20:57:06.759 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T20:57:06.760 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:57:06.760 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:57:06.760 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T20:57:06.760 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:56:48.353041+0000", 2023-09-23T20:57:06.760 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T20:57:06.760 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:57:06.760 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:57:06.760 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T20:57:06.760 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:56:48.353074+0000", 2023-09-23T20:57:06.760 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.2121e-05 2023-09-23T20:57:06.760 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:57:06.760 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:57:06.760 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T20:57:06.761 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:56:48.353083+0000", 2023-09-23T20:57:06.761 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.3989999999999993e-06 2023-09-23T20:57:06.761 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:57:06.761 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:57:06.761 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T20:57:06.761 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:56:48.353088+0000", 2023-09-23T20:57:06.761 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.2089999999999996e-06 2023-09-23T20:57:06.761 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:57:06.761 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:57:06.761 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T20:57:06.761 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:56:48.353313+0000", 2023-09-23T20:57:06.761 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00022465699999999999 2023-09-23T20:57:06.762 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:57:06.762 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:57:06.762 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T20:57:06.762 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:56:48.353425+0000", 2023-09-23T20:57:06.762 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000111772 2023-09-23T20:57:06.762 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:57:06.762 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:57:06.762 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 0,3", 2023-09-23T20:57:06.762 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:56:48.353552+0000", 2023-09-23T20:57:06.762 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000127217 2023-09-23T20:57:06.762 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:57:06.762 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:57:06.763 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:57:06.763 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:56:48.358830+0000", 2023-09-23T20:57:06.763 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0052780450000000003 2023-09-23T20:57:06.763 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:57:06.763 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:57:06.763 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T20:57:06.763 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:56:48.359897+0000", 2023-09-23T20:57:06.763 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0010672850000000001 2023-09-23T20:57:06.763 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:57:06.763 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:57:06.763 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:57:06.763 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:56:48.413953+0000", 2023-09-23T20:57:06.763 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.054055379000000001 2023-09-23T20:57:06.764 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:57:06.764 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:57:06.764 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T20:57:06.764 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:56:48.413995+0000", 2023-09-23T20:57:06.764 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.2256999999999999e-05 2023-09-23T20:57:06.764 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:57:06.764 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:57:06.764 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T20:57:06.764 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:56:48.414024+0000", 2023-09-23T20:57:06.764 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.8759e-05 2023-09-23T20:57:06.764 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T20:57:06.764 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T20:57:06.764 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T20:57:06.765 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:57:06.765 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:57:06.765 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.35100.0:21563 7.d 7:b7bd2ce2:::benchmark_data_smithi026_260489_object21562:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e205)", 2023-09-23T20:57:06.765 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T20:56:48.357536+0000", 2023-09-23T20:57:06.765 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 18.369244555000002, 2023-09-23T20:57:06.765 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.058882570000000002, 2023-09-23T20:57:06.765 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T20:57:06.765 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:57:06.765 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T20:57:06.765 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.35100", 2023-09-23T20:57:06.765 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/4108972961", 2023-09-23T20:57:06.765 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 21563 2023-09-23T20:57:06.765 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:57:06.766 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T20:57:06.766 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:57:06.766 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T20:57:06.766 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:56:48.357536+0000", 2023-09-23T20:57:06.766 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T20:57:06.766 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:57:06.766 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:57:06.766 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T20:57:06.766 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:56:48.357536+0000", 2023-09-23T20:57:06.766 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T20:57:06.766 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:57:06.766 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:57:06.767 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T20:57:06.767 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:56:48.357534+0000", 2023-09-23T20:57:06.767 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T20:57:06.767 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:57:06.767 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:57:06.767 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T20:57:06.767 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:56:48.357570+0000", 2023-09-23T20:57:06.767 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.5639999999999998e-05 2023-09-23T20:57:06.767 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:57:06.767 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:57:06.767 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T20:57:06.767 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:56:48.357580+0000", 2023-09-23T20:57:06.768 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.9539999999999999e-06 2023-09-23T20:57:06.768 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:57:06.768 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:57:06.768 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T20:57:06.768 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:56:48.357586+0000", 2023-09-23T20:57:06.768 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.6910000000000002e-06 2023-09-23T20:57:06.768 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:57:06.768 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:57:06.768 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T20:57:06.768 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:56:48.358864+0000", 2023-09-23T20:57:06.768 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.001278485 2023-09-23T20:57:06.768 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:57:06.768 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:57:06.768 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T20:57:06.769 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:56:48.358958+0000", 2023-09-23T20:57:06.769 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.3580999999999997e-05 2023-09-23T20:57:06.769 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:57:06.769 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:57:06.769 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 0,1", 2023-09-23T20:57:06.769 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:56:48.359042+0000", 2023-09-23T20:57:06.769 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.4276000000000004e-05 2023-09-23T20:57:06.769 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:57:06.769 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:57:06.769 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:57:06.769 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:56:48.360514+0000", 2023-09-23T20:57:06.769 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0014723469999999999 2023-09-23T20:57:06.769 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:57:06.769 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:57:06.770 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T20:57:06.770 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:56:48.360528+0000", 2023-09-23T20:57:06.770 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.3927000000000001e-05 2023-09-23T20:57:06.770 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:57:06.770 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:57:06.770 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:57:06.770 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:56:48.416357+0000", 2023-09-23T20:57:06.770 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.055828228000000001 2023-09-23T20:57:06.770 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:57:06.770 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:57:06.770 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T20:57:06.770 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:56:48.416392+0000", 2023-09-23T20:57:06.770 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.5487000000000001e-05 2023-09-23T20:57:06.771 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:57:06.771 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:57:06.771 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T20:57:06.771 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:56:48.416418+0000", 2023-09-23T20:57:06.771 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.6293000000000001e-05 2023-09-23T20:57:06.771 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T20:57:06.771 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T20:57:06.771 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T20:57:06.771 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:57:06.771 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:57:06.771 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.35100.0:21566 7.7 7:e1aec70a:::benchmark_data_smithi026_260489_object21565:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e205)", 2023-09-23T20:57:06.771 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T20:56:48.357600+0000", 2023-09-23T20:57:06.771 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 18.369180574000001, 2023-09-23T20:57:06.771 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.060568535999999999, 2023-09-23T20:57:06.772 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T20:57:06.772 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:57:06.772 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T20:57:06.772 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.35100", 2023-09-23T20:57:06.772 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/4108972961", 2023-09-23T20:57:06.772 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 21566 2023-09-23T20:57:06.772 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:57:06.772 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T20:57:06.772 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:57:06.772 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T20:57:06.772 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:56:48.357600+0000", 2023-09-23T20:57:06.772 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T20:57:06.772 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:57:06.772 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:57:06.772 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T20:57:06.773 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:56:48.357600+0000", 2023-09-23T20:57:06.773 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T20:57:06.773 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:57:06.773 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:57:06.773 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T20:57:06.773 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:56:48.357599+0000", 2023-09-23T20:57:06.773 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999995 2023-09-23T20:57:06.773 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:57:06.773 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:57:06.773 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T20:57:06.773 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:56:48.357630+0000", 2023-09-23T20:57:06.773 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.0457999999999999e-05 2023-09-23T20:57:06.773 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:57:06.774 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:57:06.774 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T20:57:06.774 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:56:48.357638+0000", 2023-09-23T20:57:06.774 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.9100000000000005e-06 2023-09-23T20:57:06.774 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:57:06.774 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:57:06.774 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T20:57:06.774 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:56:48.357643+0000", 2023-09-23T20:57:06.774 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.101e-06 2023-09-23T20:57:06.774 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:57:06.774 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:57:06.774 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T20:57:06.775 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:56:48.357721+0000", 2023-09-23T20:57:06.775 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.7927000000000001e-05 2023-09-23T20:57:06.775 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:57:06.775 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:57:06.775 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T20:57:06.775 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:56:48.357856+0000", 2023-09-23T20:57:06.775 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00013521299999999999 2023-09-23T20:57:06.775 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:57:06.775 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:57:06.775 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 0,3", 2023-09-23T20:57:06.775 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:56:48.358002+0000", 2023-09-23T20:57:06.776 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00014609900000000001 2023-09-23T20:57:06.776 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:57:06.776 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:57:06.776 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:57:06.776 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:56:48.359890+0000", 2023-09-23T20:57:06.776 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0018881760000000001 2023-09-23T20:57:06.776 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:57:06.776 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:57:06.776 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T20:57:06.776 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:56:48.359905+0000", 2023-09-23T20:57:06.776 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.4671999999999999e-05 2023-09-23T20:57:06.776 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:57:06.776 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:57:06.777 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:57:06.777 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:56:48.418049+0000", 2023-09-23T20:57:06.777 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.058143976 2023-09-23T20:57:06.777 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:57:06.777 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:57:06.777 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T20:57:06.777 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:56:48.418117+0000", 2023-09-23T20:57:06.777 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.8357999999999998e-05 2023-09-23T20:57:06.777 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:57:06.777 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:57:06.777 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T20:57:06.777 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:56:48.418168+0000", 2023-09-23T20:57:06.777 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.1109000000000001e-05 2023-09-23T20:57:06.777 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T20:57:06.778 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T20:57:06.778 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T20:57:06.778 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:57:06.778 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:57:06.778 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.35100.0:21570 7.c 7:3c6a3895:::benchmark_data_smithi026_260489_object21569:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e205)", 2023-09-23T20:57:06.778 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T20:56:48.359940+0000", 2023-09-23T20:57:06.778 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 18.366840565, 2023-09-23T20:57:06.778 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.055508808, 2023-09-23T20:57:06.778 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T20:57:06.778 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:57:06.778 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T20:57:06.778 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.35100", 2023-09-23T20:57:06.779 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/4108972961", 2023-09-23T20:57:06.779 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 21570 2023-09-23T20:57:06.779 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:57:06.779 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T20:57:06.779 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:57:06.779 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T20:57:06.779 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:56:48.359940+0000", 2023-09-23T20:57:06.779 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T20:57:06.779 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:57:06.779 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:57:06.779 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T20:57:06.779 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:56:48.359940+0000", 2023-09-23T20:57:06.779 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T20:57:06.779 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:57:06.780 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:57:06.780 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T20:57:06.780 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:56:48.359935+0000", 2023-09-23T20:57:06.780 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999948 2023-09-23T20:57:06.780 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:57:06.780 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:57:06.780 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T20:57:06.780 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:56:48.359992+0000", 2023-09-23T20:57:06.780 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.7370000000000001e-05 2023-09-23T20:57:06.780 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:57:06.780 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:57:06.780 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T20:57:06.780 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:56:48.360005+0000", 2023-09-23T20:57:06.780 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.2857000000000001e-05 2023-09-23T20:57:06.781 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:57:06.781 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:57:06.781 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T20:57:06.781 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:56:48.360012+0000", 2023-09-23T20:57:06.781 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.6610000000000003e-06 2023-09-23T20:57:06.781 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:57:06.781 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:57:06.781 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T20:57:06.781 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:56:48.360205+0000", 2023-09-23T20:57:06.781 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00019341599999999999 2023-09-23T20:57:06.781 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:57:06.781 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:57:06.782 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T20:57:06.782 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:56:48.360358+0000", 2023-09-23T20:57:06.782 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00015248700000000001 2023-09-23T20:57:06.782 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:57:06.782 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:57:06.782 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 2,7", 2023-09-23T20:57:06.782 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:56:48.360540+0000", 2023-09-23T20:57:06.782 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000182184 2023-09-23T20:57:06.782 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:57:06.782 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:57:06.782 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T20:57:06.782 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:56:48.361656+0000", 2023-09-23T20:57:06.782 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0011158559999999999 2023-09-23T20:57:06.783 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:57:06.783 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:57:06.783 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:57:06.783 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:56:48.361969+0000", 2023-09-23T20:57:06.783 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000313491 2023-09-23T20:57:06.783 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:57:06.783 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:57:06.783 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:57:06.783 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:56:48.415352+0000", 2023-09-23T20:57:06.783 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.053382563000000001 2023-09-23T20:57:06.784 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:57:06.784 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:57:06.784 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T20:57:06.784 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:56:48.415406+0000", 2023-09-23T20:57:06.784 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.4392999999999999e-05 2023-09-23T20:57:06.784 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:57:06.784 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:57:06.784 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T20:57:06.784 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:56:48.415449+0000", 2023-09-23T20:57:06.784 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.2549000000000002e-05 2023-09-23T20:57:06.784 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T20:57:06.784 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T20:57:06.785 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T20:57:06.785 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:57:06.785 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:57:06.785 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.35100.0:21575 7.7 7:ea424bbd:::benchmark_data_smithi026_260489_object21574:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e205)", 2023-09-23T20:57:06.785 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T20:56:48.361064+0000", 2023-09-23T20:57:06.785 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 18.365716126999999, 2023-09-23T20:57:06.785 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.057484973000000002, 2023-09-23T20:57:06.785 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T20:57:06.785 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:57:06.785 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T20:57:06.785 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.35100", 2023-09-23T20:57:06.785 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/4108972961", 2023-09-23T20:57:06.785 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 21575 2023-09-23T20:57:06.786 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:57:06.786 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T20:57:06.786 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:57:06.786 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T20:57:06.786 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:56:48.361064+0000", 2023-09-23T20:57:06.786 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T20:57:06.786 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:57:06.786 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:57:06.786 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T20:57:06.786 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:56:48.361064+0000", 2023-09-23T20:57:06.786 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T20:57:06.786 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:57:06.787 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:57:06.787 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T20:57:06.787 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:56:48.361063+0000", 2023-09-23T20:57:06.787 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.999999 2023-09-23T20:57:06.787 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:57:06.787 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:57:06.787 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T20:57:06.787 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:56:48.361133+0000", 2023-09-23T20:57:06.787 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.9453000000000001e-05 2023-09-23T20:57:06.787 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:57:06.787 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:57:06.787 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T20:57:06.787 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:56:48.361146+0000", 2023-09-23T20:57:06.787 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.3516000000000001e-05 2023-09-23T20:57:06.788 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:57:06.788 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:57:06.788 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T20:57:06.788 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:56:48.361152+0000", 2023-09-23T20:57:06.788 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.2389999999999999e-06 2023-09-23T20:57:06.788 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:57:06.788 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:57:06.788 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T20:57:06.788 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:56:48.361205+0000", 2023-09-23T20:57:06.788 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.2351000000000001e-05 2023-09-23T20:57:06.788 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:57:06.788 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:57:06.789 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T20:57:06.789 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:56:48.361360+0000", 2023-09-23T20:57:06.789 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00015498399999999999 2023-09-23T20:57:06.789 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:57:06.789 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:57:06.789 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 0,3", 2023-09-23T20:57:06.789 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:56:48.361534+0000", 2023-09-23T20:57:06.789 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00017449499999999999 2023-09-23T20:57:06.789 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:57:06.789 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:57:06.789 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T20:57:06.789 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:56:48.362502+0000", 2023-09-23T20:57:06.789 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00096804400000000004 2023-09-23T20:57:06.790 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:57:06.790 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:57:06.790 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:57:06.790 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:56:48.416521+0000", 2023-09-23T20:57:06.790 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.054018726000000003 2023-09-23T20:57:06.790 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:57:06.790 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:57:06.790 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:57:06.790 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:56:48.418449+0000", 2023-09-23T20:57:06.790 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0019280129999999999 2023-09-23T20:57:06.790 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:57:06.790 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:57:06.791 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T20:57:06.791 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:56:48.418504+0000", 2023-09-23T20:57:06.791 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.4892000000000002e-05 2023-09-23T20:57:06.791 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:57:06.791 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:57:06.791 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T20:57:06.791 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:56:48.418549+0000", 2023-09-23T20:57:06.791 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.5361000000000002e-05 2023-09-23T20:57:06.791 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T20:57:06.791 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T20:57:06.791 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T20:57:06.791 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:57:06.792 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:57:06.792 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.35100.0:22529 7.7 7:ed834f3b:::benchmark_data_smithi026_260489_object22528:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e205)", 2023-09-23T20:57:06.792 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T20:56:48.838043+0000", 2023-09-23T20:57:06.792 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 17.888737245000002, 2023-09-23T20:57:06.792 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.075851958999999997, 2023-09-23T20:57:06.792 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T20:57:06.792 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:57:06.792 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T20:57:06.792 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.35100", 2023-09-23T20:57:06.792 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/4108972961", 2023-09-23T20:57:06.792 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 22529 2023-09-23T20:57:06.793 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:57:06.793 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T20:57:06.793 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:57:06.793 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T20:57:06.793 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:56:48.838043+0000", 2023-09-23T20:57:06.793 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T20:57:06.793 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:57:06.793 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:57:06.793 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T20:57:06.793 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:56:48.838043+0000", 2023-09-23T20:57:06.793 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T20:57:06.793 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:57:06.794 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:57:06.794 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T20:57:06.794 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:56:48.838043+0000", 2023-09-23T20:57:06.794 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999995 2023-09-23T20:57:06.794 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:57:06.794 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:57:06.794 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T20:57:06.794 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:56:48.838074+0000", 2023-09-23T20:57:06.794 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.1732e-05 2023-09-23T20:57:06.794 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:57:06.794 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:57:06.794 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T20:57:06.795 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:56:48.838083+0000", 2023-09-23T20:57:06.795 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.4770000000000003e-06 2023-09-23T20:57:06.795 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:57:06.795 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:57:06.795 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T20:57:06.795 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:56:48.838088+0000", 2023-09-23T20:57:06.795 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.1089999999999997e-06 2023-09-23T20:57:06.795 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:57:06.795 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:57:06.795 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T20:57:06.795 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:56:48.838134+0000", 2023-09-23T20:57:06.796 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.5738999999999997e-05 2023-09-23T20:57:06.796 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:57:06.796 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:57:06.796 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T20:57:06.796 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:56:48.839369+0000", 2023-09-23T20:57:06.796 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0012354639999999999 2023-09-23T20:57:06.796 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:57:06.796 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:57:06.796 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 0,3", 2023-09-23T20:57:06.796 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:56:48.839483+0000", 2023-09-23T20:57:06.796 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00011385800000000001 2023-09-23T20:57:06.796 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:57:06.796 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:57:06.797 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T20:57:06.797 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:56:48.840406+0000", 2023-09-23T20:57:06.797 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00092304799999999999 2023-09-23T20:57:06.797 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:57:06.797 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:57:06.797 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:57:06.797 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:56:48.845954+0000", 2023-09-23T20:57:06.797 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.005548458 2023-09-23T20:57:06.797 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:57:06.797 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:57:06.797 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:57:06.797 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:56:48.913803+0000", 2023-09-23T20:57:06.798 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.067848156000000007 2023-09-23T20:57:06.798 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:57:06.798 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:57:06.798 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T20:57:06.798 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:56:48.913849+0000", 2023-09-23T20:57:06.798 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.5952999999999999e-05 2023-09-23T20:57:06.798 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:57:06.798 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:57:06.798 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T20:57:06.798 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:56:48.913895+0000", 2023-09-23T20:57:06.798 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.6542999999999998e-05 2023-09-23T20:57:06.799 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T20:57:06.799 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T20:57:06.799 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T20:57:06.799 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:57:06.799 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:57:06.799 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.35100.0:22537 7.7 7:ec22c751:::benchmark_data_smithi026_260489_object22536:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e205)", 2023-09-23T20:57:06.799 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T20:56:48.842227+0000", 2023-09-23T20:57:06.799 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 17.884553069999999, 2023-09-23T20:57:06.799 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.071782594000000005, 2023-09-23T20:57:06.799 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T20:57:06.799 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:57:06.799 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T20:57:06.799 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.35100", 2023-09-23T20:57:06.799 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/4108972961", 2023-09-23T20:57:06.800 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 22537 2023-09-23T20:57:06.800 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:57:06.800 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T20:57:06.800 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:57:06.800 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T20:57:06.800 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:56:48.842227+0000", 2023-09-23T20:57:06.800 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T20:57:06.800 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:57:06.800 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:57:06.800 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T20:57:06.800 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:56:48.842227+0000", 2023-09-23T20:57:06.801 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T20:57:06.801 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:57:06.801 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:57:06.801 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T20:57:06.801 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:56:48.842226+0000", 2023-09-23T20:57:06.801 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T20:57:06.801 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:57:06.801 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:57:06.801 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T20:57:06.801 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:56:48.842276+0000", 2023-09-23T20:57:06.801 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.9931e-05 2023-09-23T20:57:06.801 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:57:06.801 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:57:06.802 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T20:57:06.802 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:56:48.842288+0000", 2023-09-23T20:57:06.802 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.2587e-05 2023-09-23T20:57:06.802 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:57:06.802 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:57:06.802 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T20:57:06.802 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:56:48.842295+0000", 2023-09-23T20:57:06.802 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.7700000000000004e-06 2023-09-23T20:57:06.802 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:57:06.802 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:57:06.802 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T20:57:06.802 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:56:48.842348+0000", 2023-09-23T20:57:06.802 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.2540000000000002e-05 2023-09-23T20:57:06.802 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:57:06.803 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:57:06.803 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T20:57:06.803 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:56:48.842494+0000", 2023-09-23T20:57:06.803 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00014688599999999999 2023-09-23T20:57:06.803 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:57:06.803 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:57:06.803 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 0,3", 2023-09-23T20:57:06.803 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:56:48.842639+0000", 2023-09-23T20:57:06.803 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00014424500000000001 2023-09-23T20:57:06.803 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:57:06.803 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:57:06.803 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T20:57:06.803 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:56:48.844755+0000", 2023-09-23T20:57:06.804 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0021167109999999999 2023-09-23T20:57:06.804 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:57:06.804 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:57:06.804 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:57:06.804 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:56:48.847919+0000", 2023-09-23T20:57:06.804 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.003163281 2023-09-23T20:57:06.804 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:57:06.804 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:57:06.804 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:57:06.804 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:56:48.913938+0000", 2023-09-23T20:57:06.804 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.066019790999999994 2023-09-23T20:57:06.804 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:57:06.805 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:57:06.805 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T20:57:06.805 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:56:48.913979+0000", 2023-09-23T20:57:06.805 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.0708999999999999e-05 2023-09-23T20:57:06.805 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:57:06.805 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:57:06.805 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T20:57:06.805 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:56:48.914010+0000", 2023-09-23T20:57:06.806 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.0675e-05 2023-09-23T20:57:06.806 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T20:57:06.806 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T20:57:06.806 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T20:57:06.806 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:57:06.806 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:57:06.806 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.35100.0:22538 7.7 7:e9ad8298:::benchmark_data_smithi026_260489_object22537:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e205)", 2023-09-23T20:57:06.806 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T20:56:48.842308+0000", 2023-09-23T20:57:06.806 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 17.884472088999999, 2023-09-23T20:57:06.806 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.071939473000000004, 2023-09-23T20:57:06.806 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T20:57:06.806 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:57:06.807 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T20:57:06.807 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.35100", 2023-09-23T20:57:06.807 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/4108972961", 2023-09-23T20:57:06.807 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 22538 2023-09-23T20:57:06.807 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:57:06.807 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T20:57:06.807 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:57:06.807 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T20:57:06.807 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:56:48.842308+0000", 2023-09-23T20:57:06.807 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T20:57:06.807 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:57:06.808 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:57:06.808 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T20:57:06.808 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:56:48.842308+0000", 2023-09-23T20:57:06.808 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T20:57:06.808 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:57:06.808 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:57:06.808 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T20:57:06.808 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:56:48.842308+0000", 2023-09-23T20:57:06.808 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999995 2023-09-23T20:57:06.808 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:57:06.808 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:57:06.808 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T20:57:06.809 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:56:48.842336+0000", 2023-09-23T20:57:06.809 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.8646999999999999e-05 2023-09-23T20:57:06.809 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:57:06.809 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:57:06.809 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T20:57:06.809 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:56:48.842343+0000", 2023-09-23T20:57:06.809 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.1040000000000001e-06 2023-09-23T20:57:06.809 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:57:06.809 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:57:06.809 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T20:57:06.809 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:56:48.842348+0000", 2023-09-23T20:57:06.809 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.9860000000000002e-06 2023-09-23T20:57:06.809 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:57:06.810 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:57:06.810 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T20:57:06.810 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:56:48.843650+0000", 2023-09-23T20:57:06.810 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0013012620000000001 2023-09-23T20:57:06.810 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:57:06.810 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:57:06.810 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T20:57:06.810 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:56:48.843776+0000", 2023-09-23T20:57:06.810 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00012584900000000001 2023-09-23T20:57:06.810 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:57:06.810 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:57:06.810 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 0,3", 2023-09-23T20:57:06.810 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:56:48.843929+0000", 2023-09-23T20:57:06.810 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000153354 2023-09-23T20:57:06.811 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:57:06.811 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:57:06.811 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T20:57:06.811 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:56:48.844882+0000", 2023-09-23T20:57:06.811 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00095270800000000003 2023-09-23T20:57:06.811 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:57:06.811 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:57:06.811 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:57:06.811 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:56:48.848066+0000", 2023-09-23T20:57:06.811 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0031848879999999999 2023-09-23T20:57:06.811 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:57:06.811 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:57:06.811 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:57:06.812 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:56:48.914142+0000", 2023-09-23T20:57:06.812 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.066075421999999995 2023-09-23T20:57:06.812 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:57:06.812 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:57:06.812 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T20:57:06.812 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:56:48.914195+0000", 2023-09-23T20:57:06.812 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.3035000000000003e-05 2023-09-23T20:57:06.812 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:57:06.812 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:57:06.812 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T20:57:06.812 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:56:48.914248+0000", 2023-09-23T20:57:06.812 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.2802e-05 2023-09-23T20:57:06.812 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T20:57:06.813 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T20:57:06.813 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T20:57:06.813 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:57:06.813 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:57:06.813 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.35100.0:22550 7.d 7:bf6b452c:::benchmark_data_smithi026_260489_object22549:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e205)", 2023-09-23T20:57:06.813 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T20:56:48.853075+0000", 2023-09-23T20:57:06.813 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 17.873704914000001, 2023-09-23T20:57:06.813 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.063515895000000003, 2023-09-23T20:57:06.813 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T20:57:06.813 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:57:06.813 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T20:57:06.813 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.35100", 2023-09-23T20:57:06.813 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/4108972961", 2023-09-23T20:57:06.814 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 22550 2023-09-23T20:57:06.814 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:57:06.814 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T20:57:06.814 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:57:06.814 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T20:57:06.814 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:56:48.853075+0000", 2023-09-23T20:57:06.814 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T20:57:06.814 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:57:06.814 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:57:06.814 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T20:57:06.814 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:56:48.853075+0000", 2023-09-23T20:57:06.814 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T20:57:06.815 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:57:06.815 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:57:06.815 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T20:57:06.815 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:56:48.853075+0000", 2023-09-23T20:57:06.815 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.999999 2023-09-23T20:57:06.815 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:57:06.815 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:57:06.815 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T20:57:06.815 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:56:48.853174+0000", 2023-09-23T20:57:06.815 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.9712999999999999e-05 2023-09-23T20:57:06.815 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:57:06.815 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:57:06.815 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T20:57:06.816 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:56:48.853190+0000", 2023-09-23T20:57:06.816 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.575e-05 2023-09-23T20:57:06.816 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:57:06.816 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:57:06.816 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T20:57:06.816 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:56:48.853196+0000", 2023-09-23T20:57:06.816 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.8549999999999998e-06 2023-09-23T20:57:06.816 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:57:06.816 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:57:06.816 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T20:57:06.816 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:56:48.853248+0000", 2023-09-23T20:57:06.816 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.2200000000000002e-05 2023-09-23T20:57:06.816 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:57:06.817 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:57:06.817 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T20:57:06.817 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:56:48.853382+0000", 2023-09-23T20:57:06.817 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00013374400000000001 2023-09-23T20:57:06.817 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:57:06.817 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:57:06.817 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 0,1", 2023-09-23T20:57:06.817 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:56:48.853532+0000", 2023-09-23T20:57:06.817 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000150354 2023-09-23T20:57:06.817 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:57:06.817 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:57:06.817 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T20:57:06.817 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:56:48.854452+0000", 2023-09-23T20:57:06.817 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00091986599999999998 2023-09-23T20:57:06.818 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:57:06.818 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:57:06.818 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:57:06.818 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:56:48.854740+0000", 2023-09-23T20:57:06.818 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0002876 2023-09-23T20:57:06.818 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:57:06.818 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:57:06.818 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:57:06.818 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:56:48.916490+0000", 2023-09-23T20:57:06.818 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.061750459000000001 2023-09-23T20:57:06.818 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:57:06.818 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:57:06.819 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T20:57:06.819 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:56:48.916544+0000", 2023-09-23T20:57:06.819 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.4351000000000002e-05 2023-09-23T20:57:06.819 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:57:06.819 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:57:06.819 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T20:57:06.819 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:56:48.916591+0000", 2023-09-23T20:57:06.819 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.6845000000000003e-05 2023-09-23T20:57:06.819 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T20:57:06.819 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T20:57:06.819 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T20:57:06.819 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:57:06.819 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:57:06.820 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.35100.0:23179 7.d 7:b696d16b:::benchmark_data_smithi026_260489_object23178:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e205)", 2023-09-23T20:57:06.820 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T20:56:49.203610+0000", 2023-09-23T20:57:06.820 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 17.523170567000001, 2023-09-23T20:57:06.820 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.050249323999999998, 2023-09-23T20:57:06.820 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T20:57:06.820 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:57:06.820 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T20:57:06.820 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.35100", 2023-09-23T20:57:06.820 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/4108972961", 2023-09-23T20:57:06.820 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 23179 2023-09-23T20:57:06.820 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:57:06.820 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T20:57:06.821 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:57:06.821 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T20:57:06.821 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:56:49.203610+0000", 2023-09-23T20:57:06.821 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T20:57:06.821 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:57:06.821 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:57:06.821 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T20:57:06.821 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:56:49.203610+0000", 2023-09-23T20:57:06.821 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T20:57:06.821 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:57:06.821 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:57:06.822 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T20:57:06.822 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:56:49.203608+0000", 2023-09-23T20:57:06.822 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T20:57:06.822 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:57:06.822 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:57:06.822 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T20:57:06.822 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:56:49.203660+0000", 2023-09-23T20:57:06.822 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.1767000000000001e-05 2023-09-23T20:57:06.822 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:57:06.822 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:57:06.822 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T20:57:06.822 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:56:49.203674+0000", 2023-09-23T20:57:06.823 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.4229e-05 2023-09-23T20:57:06.823 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:57:06.823 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:57:06.823 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T20:57:06.823 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:56:49.203680+0000", 2023-09-23T20:57:06.823 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.1419999999999998e-06 2023-09-23T20:57:06.823 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:57:06.823 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:57:06.823 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T20:57:06.823 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:56:49.203738+0000", 2023-09-23T20:57:06.823 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.7652999999999998e-05 2023-09-23T20:57:06.823 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:57:06.824 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:57:06.824 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T20:57:06.824 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:56:49.203890+0000", 2023-09-23T20:57:06.824 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000151633 2023-09-23T20:57:06.824 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:57:06.824 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:57:06.824 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 0,1", 2023-09-23T20:57:06.824 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:56:49.204042+0000", 2023-09-23T20:57:06.824 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000152039 2023-09-23T20:57:06.824 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:57:06.824 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:57:06.824 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T20:57:06.824 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:56:49.205144+0000", 2023-09-23T20:57:06.825 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0011018549999999999 2023-09-23T20:57:06.825 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:57:06.825 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:57:06.825 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:57:06.825 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:56:49.209577+0000", 2023-09-23T20:57:06.825 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0044335879999999996 2023-09-23T20:57:06.825 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:57:06.825 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:57:06.825 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:57:06.825 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:56:49.253780+0000", 2023-09-23T20:57:06.825 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.044203078 2023-09-23T20:57:06.825 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:57:06.826 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:57:06.826 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T20:57:06.826 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:56:49.253830+0000", 2023-09-23T20:57:06.826 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.9833000000000002e-05 2023-09-23T20:57:06.826 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:57:06.826 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:57:06.826 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T20:57:06.826 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:56:49.253859+0000", 2023-09-23T20:57:06.826 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.9088e-05 2023-09-23T20:57:06.826 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T20:57:06.826 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T20:57:06.827 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T20:57:06.827 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:57:06.827 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:57:06.827 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.35100.0:23180 7.d 7:b81c8cf4:::benchmark_data_smithi026_260489_object23179:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e205)", 2023-09-23T20:57:06.827 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T20:56:49.205375+0000", 2023-09-23T20:57:06.827 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 17.521405189999999, 2023-09-23T20:57:06.827 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.048620045000000001, 2023-09-23T20:57:06.827 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T20:57:06.827 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:57:06.827 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T20:57:06.827 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.35100", 2023-09-23T20:57:06.827 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/4108972961", 2023-09-23T20:57:06.827 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 23180 2023-09-23T20:57:06.828 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:57:06.828 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T20:57:06.828 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:57:06.828 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T20:57:06.828 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:56:49.205375+0000", 2023-09-23T20:57:06.828 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T20:57:06.828 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:57:06.828 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:57:06.828 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T20:57:06.828 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:56:49.205375+0000", 2023-09-23T20:57:06.828 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T20:57:06.828 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:57:06.828 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:57:06.829 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T20:57:06.829 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:56:49.205374+0000", 2023-09-23T20:57:06.829 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T20:57:06.829 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:57:06.829 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:57:06.829 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T20:57:06.829 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:56:49.205441+0000", 2023-09-23T20:57:06.829 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.7294999999999999e-05 2023-09-23T20:57:06.829 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:57:06.829 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:57:06.829 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T20:57:06.829 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:56:49.205464+0000", 2023-09-23T20:57:06.829 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.3311000000000001e-05 2023-09-23T20:57:06.830 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:57:06.830 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:57:06.830 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T20:57:06.830 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:56:49.205471+0000", 2023-09-23T20:57:06.830 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.0149999999999998e-06 2023-09-23T20:57:06.830 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:57:06.830 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:57:06.830 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T20:57:06.830 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:56:49.205508+0000", 2023-09-23T20:57:06.830 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.6980000000000002e-05 2023-09-23T20:57:06.830 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:57:06.830 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:57:06.830 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T20:57:06.831 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:56:49.205644+0000", 2023-09-23T20:57:06.831 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000136136 2023-09-23T20:57:06.831 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:57:06.831 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:57:06.831 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 0,1", 2023-09-23T20:57:06.831 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:56:49.205802+0000", 2023-09-23T20:57:06.831 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000157341 2023-09-23T20:57:06.831 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:57:06.831 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:57:06.831 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T20:57:06.831 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:56:49.206960+0000", 2023-09-23T20:57:06.831 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.001158201 2023-09-23T20:57:06.831 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:57:06.831 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:57:06.832 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:57:06.832 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:56:49.209607+0000", 2023-09-23T20:57:06.832 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0026470420000000001 2023-09-23T20:57:06.832 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:57:06.832 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:57:06.832 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:57:06.832 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:56:49.253907+0000", 2023-09-23T20:57:06.832 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.044299566999999998 2023-09-23T20:57:06.832 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:57:06.832 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:57:06.832 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T20:57:06.832 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:56:49.253959+0000", 2023-09-23T20:57:06.832 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.1950999999999998e-05 2023-09-23T20:57:06.832 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:57:06.833 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:57:06.833 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T20:57:06.833 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:56:49.253995+0000", 2023-09-23T20:57:06.833 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.6668999999999997e-05 2023-09-23T20:57:06.833 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T20:57:06.833 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T20:57:06.833 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T20:57:06.833 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:57:06.833 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:57:06.833 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.35100.0:23182 7.7 7:e2589c5f:::benchmark_data_smithi026_260489_object23181:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e205)", 2023-09-23T20:57:06.833 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T20:56:49.205484+0000", 2023-09-23T20:57:06.833 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 17.521296789000001, 2023-09-23T20:57:06.834 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.049168392999999998, 2023-09-23T20:57:06.834 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T20:57:06.834 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:57:06.834 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T20:57:06.834 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.35100", 2023-09-23T20:57:06.834 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/4108972961", 2023-09-23T20:57:06.834 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 23182 2023-09-23T20:57:06.834 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:57:06.834 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T20:57:06.834 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:57:06.834 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T20:57:06.835 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:56:49.205484+0000", 2023-09-23T20:57:06.835 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T20:57:06.835 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:57:06.835 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:57:06.835 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T20:57:06.835 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:56:49.205484+0000", 2023-09-23T20:57:06.835 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T20:57:06.835 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:57:06.835 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:57:06.835 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T20:57:06.835 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:56:49.205483+0000", 2023-09-23T20:57:06.836 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.999999 2023-09-23T20:57:06.836 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:57:06.836 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:57:06.836 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T20:57:06.836 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:56:49.205517+0000", 2023-09-23T20:57:06.836 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.3779000000000003e-05 2023-09-23T20:57:06.836 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:57:06.836 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:57:06.836 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T20:57:06.836 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:56:49.205525+0000", 2023-09-23T20:57:06.836 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.1440000000000003e-06 2023-09-23T20:57:06.836 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:57:06.836 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:57:06.837 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T20:57:06.837 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:56:49.205530+0000", 2023-09-23T20:57:06.837 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.6879999999999996e-06 2023-09-23T20:57:06.837 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:57:06.837 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:57:06.837 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T20:57:06.837 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:56:49.205576+0000", 2023-09-23T20:57:06.837 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.5938e-05 2023-09-23T20:57:06.837 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:57:06.837 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:57:06.837 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T20:57:06.837 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:56:49.205682+0000", 2023-09-23T20:57:06.838 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00010605 2023-09-23T20:57:06.838 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:57:06.838 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:57:06.838 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 0,3", 2023-09-23T20:57:06.838 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:56:49.205802+0000", 2023-09-23T20:57:06.838 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000119375 2023-09-23T20:57:06.838 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:57:06.838 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:57:06.838 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T20:57:06.838 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:56:49.207038+0000", 2023-09-23T20:57:06.838 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.001236214 2023-09-23T20:57:06.838 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:57:06.838 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:57:06.839 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:57:06.839 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:56:49.207498+0000", 2023-09-23T20:57:06.839 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00046014599999999998 2023-09-23T20:57:06.839 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:57:06.839 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:57:06.839 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:57:06.839 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:56:49.254563+0000", 2023-09-23T20:57:06.839 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.047065221999999997 2023-09-23T20:57:06.839 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:57:06.839 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:57:06.839 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T20:57:06.839 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:56:49.254615+0000", 2023-09-23T20:57:06.839 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.1143000000000001e-05 2023-09-23T20:57:06.840 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:57:06.840 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:57:06.840 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T20:57:06.840 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:56:49.254652+0000", 2023-09-23T20:57:06.840 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.7447000000000002e-05 2023-09-23T20:57:06.840 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T20:57:06.840 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T20:57:06.840 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T20:57:06.840 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:57:06.840 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:57:06.840 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.35100.0:23188 7.d 7:b2968f6e:::benchmark_data_smithi026_260489_object23187:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e205)", 2023-09-23T20:57:06.840 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T20:56:49.207224+0000", 2023-09-23T20:57:06.840 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 17.519556478999998, 2023-09-23T20:57:06.840 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.049438218999999999, 2023-09-23T20:57:06.841 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T20:57:06.841 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:57:06.841 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T20:57:06.841 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.35100", 2023-09-23T20:57:06.841 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/4108972961", 2023-09-23T20:57:06.841 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 23188 2023-09-23T20:57:06.841 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:57:06.841 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T20:57:06.841 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:57:06.841 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T20:57:06.841 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:56:49.207224+0000", 2023-09-23T20:57:06.841 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T20:57:06.842 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:57:06.842 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:57:06.842 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T20:57:06.842 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:56:49.207224+0000", 2023-09-23T20:57:06.842 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T20:57:06.842 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:57:06.842 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:57:06.842 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T20:57:06.842 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:56:49.207223+0000", 2023-09-23T20:57:06.842 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T20:57:06.842 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:57:06.842 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:57:06.842 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T20:57:06.843 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:56:49.207284+0000", 2023-09-23T20:57:06.843 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.1248999999999996e-05 2023-09-23T20:57:06.843 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:57:06.843 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:57:06.843 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T20:57:06.843 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:56:49.207298+0000", 2023-09-23T20:57:06.843 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.398e-05 2023-09-23T20:57:06.843 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:57:06.843 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:57:06.843 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T20:57:06.843 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:56:49.207305+0000", 2023-09-23T20:57:06.843 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.6699999999999997e-06 2023-09-23T20:57:06.843 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:57:06.844 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:57:06.844 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T20:57:06.844 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:56:49.207348+0000", 2023-09-23T20:57:06.844 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.2994999999999997e-05 2023-09-23T20:57:06.844 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:57:06.844 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:57:06.844 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T20:57:06.844 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:56:49.207486+0000", 2023-09-23T20:57:06.844 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00013896500000000001 2023-09-23T20:57:06.844 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:57:06.844 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:57:06.844 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 0,1", 2023-09-23T20:57:06.845 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:56:49.207636+0000", 2023-09-23T20:57:06.845 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00014980000000000001 2023-09-23T20:57:06.845 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:57:06.845 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:57:06.845 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T20:57:06.845 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:56:49.210528+0000", 2023-09-23T20:57:06.845 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0028918400000000001 2023-09-23T20:57:06.845 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:57:06.845 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:57:06.845 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:57:06.845 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:56:49.210765+0000", 2023-09-23T20:57:06.845 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00023676200000000001 2023-09-23T20:57:06.845 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:57:06.846 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:57:06.846 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:57:06.846 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:56:49.256567+0000", 2023-09-23T20:57:06.846 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.045802401999999999 2023-09-23T20:57:06.846 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:57:06.846 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:57:06.846 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T20:57:06.846 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:56:49.256623+0000", 2023-09-23T20:57:06.846 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.5560000000000003e-05 2023-09-23T20:57:06.846 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:57:06.846 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:57:06.846 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T20:57:06.847 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:56:49.256662+0000", 2023-09-23T20:57:06.847 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.9239000000000002e-05 2023-09-23T20:57:06.847 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T20:57:06.847 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T20:57:06.847 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T20:57:06.847 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:57:06.847 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:57:06.847 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.35100.0:26542 7.c 7:3f766fa7:::benchmark_data_smithi026_260489_object26541:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e205)", 2023-09-23T20:57:06.847 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T20:56:50.914710+0000", 2023-09-23T20:57:06.847 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 15.812070729, 2023-09-23T20:57:06.847 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.049076518, 2023-09-23T20:57:06.847 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T20:57:06.847 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:57:06.847 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T20:57:06.848 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.35100", 2023-09-23T20:57:06.848 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/4108972961", 2023-09-23T20:57:06.848 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 26542 2023-09-23T20:57:06.848 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:57:06.848 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T20:57:06.848 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:57:06.848 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T20:57:06.848 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:56:50.914710+0000", 2023-09-23T20:57:06.848 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T20:57:06.848 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:57:06.848 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:57:06.848 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T20:57:06.848 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:56:50.914710+0000", 2023-09-23T20:57:06.849 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T20:57:06.849 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:57:06.849 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:57:06.849 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T20:57:06.849 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:56:50.914709+0000", 2023-09-23T20:57:06.849 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.999999 2023-09-23T20:57:06.849 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:57:06.849 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:57:06.849 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T20:57:06.849 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:56:50.914734+0000", 2023-09-23T20:57:06.849 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.5579e-05 2023-09-23T20:57:06.849 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:57:06.849 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:57:06.849 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T20:57:06.850 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:56:50.914745+0000", 2023-09-23T20:57:06.850 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.1211e-05 2023-09-23T20:57:06.850 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:57:06.850 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:57:06.850 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T20:57:06.850 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:56:50.914750+0000", 2023-09-23T20:57:06.850 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.8550000000000001e-06 2023-09-23T20:57:06.850 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:57:06.850 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:57:06.850 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T20:57:06.850 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:56:50.914789+0000", 2023-09-23T20:57:06.850 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.9050000000000001e-05 2023-09-23T20:57:06.850 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:57:06.850 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:57:06.851 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T20:57:06.851 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:56:50.914921+0000", 2023-09-23T20:57:06.851 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00013121300000000001 2023-09-23T20:57:06.851 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:57:06.851 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:57:06.851 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 2,7", 2023-09-23T20:57:06.851 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:56:50.915075+0000", 2023-09-23T20:57:06.851 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00015417999999999999 2023-09-23T20:57:06.851 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:57:06.851 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:57:06.851 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:57:06.851 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:56:50.918696+0000", 2023-09-23T20:57:06.851 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0036214670000000002 2023-09-23T20:57:06.851 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:57:06.852 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:57:06.852 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T20:57:06.852 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:56:50.918711+0000", 2023-09-23T20:57:06.852 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.4628e-05 2023-09-23T20:57:06.852 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:57:06.852 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:57:06.852 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:57:06.852 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:56:50.963686+0000", 2023-09-23T20:57:06.852 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.044975213999999999 2023-09-23T20:57:06.852 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:57:06.852 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:57:06.852 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T20:57:06.852 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:56:50.963739+0000", 2023-09-23T20:57:06.853 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.2984e-05 2023-09-23T20:57:06.853 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:57:06.853 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:57:06.853 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T20:57:06.853 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:56:50.963786+0000", 2023-09-23T20:57:06.853 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.7117999999999998e-05 2023-09-23T20:57:06.853 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T20:57:06.853 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T20:57:06.853 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T20:57:06.853 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:57:06.853 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:57:06.853 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.35100.0:26554 7.d 7:ba2e384b:::benchmark_data_smithi026_260489_object26553:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e205)", 2023-09-23T20:57:06.853 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T20:56:50.914764+0000", 2023-09-23T20:57:06.854 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 15.812016552999999, 2023-09-23T20:57:06.854 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.049151763000000001, 2023-09-23T20:57:06.854 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T20:57:06.854 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:57:06.854 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T20:57:06.854 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.35100", 2023-09-23T20:57:06.854 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/4108972961", 2023-09-23T20:57:06.854 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 26554 2023-09-23T20:57:06.854 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:57:06.854 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T20:57:06.854 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:57:06.854 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T20:57:06.854 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:56:50.914764+0000", 2023-09-23T20:57:06.855 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T20:57:06.855 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:57:06.855 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:57:06.855 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T20:57:06.855 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:56:50.914764+0000", 2023-09-23T20:57:06.855 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T20:57:06.855 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:57:06.855 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:57:06.855 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T20:57:06.855 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:56:50.914763+0000", 2023-09-23T20:57:06.855 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999995 2023-09-23T20:57:06.856 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:57:06.856 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:57:06.856 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T20:57:06.856 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:56:50.914801+0000", 2023-09-23T20:57:06.856 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.7317999999999997e-05 2023-09-23T20:57:06.856 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:57:06.856 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:57:06.856 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T20:57:06.856 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:56:50.914806+0000", 2023-09-23T20:57:06.856 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.9969999999999998e-06 2023-09-23T20:57:06.856 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:57:06.856 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:57:06.856 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T20:57:06.857 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:56:50.914809+0000", 2023-09-23T20:57:06.857 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.1829999999999998e-06 2023-09-23T20:57:06.857 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:57:06.857 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:57:06.857 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T20:57:06.857 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:56:50.914855+0000", 2023-09-23T20:57:06.857 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.6248000000000002e-05 2023-09-23T20:57:06.857 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:57:06.857 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:57:06.857 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T20:57:06.857 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:56:50.914963+0000", 2023-09-23T20:57:06.858 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000108232 2023-09-23T20:57:06.858 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:57:06.858 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:57:06.858 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 0,1", 2023-09-23T20:57:06.858 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:56:50.915075+0000", 2023-09-23T20:57:06.858 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000111699 2023-09-23T20:57:06.858 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:57:06.858 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:57:06.858 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T20:57:06.858 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:56:50.917498+0000", 2023-09-23T20:57:06.858 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.002423129 2023-09-23T20:57:06.858 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:57:06.858 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:57:06.859 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:57:06.859 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:56:50.917877+0000", 2023-09-23T20:57:06.859 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00037874300000000002 2023-09-23T20:57:06.859 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:57:06.859 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:57:06.859 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:57:06.859 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:56:50.963822+0000", 2023-09-23T20:57:06.859 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.045945380000000001 2023-09-23T20:57:06.859 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:57:06.859 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:57:06.859 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T20:57:06.859 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:56:50.963875+0000", 2023-09-23T20:57:06.859 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.2731e-05 2023-09-23T20:57:06.859 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:57:06.860 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:57:06.860 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T20:57:06.860 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:56:50.963916+0000", 2023-09-23T20:57:06.860 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.0451000000000003e-05 2023-09-23T20:57:06.860 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T20:57:06.860 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T20:57:06.860 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T20:57:06.860 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:57:06.860 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:57:06.860 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.35100.0:26556 7.c 7:32517113:::benchmark_data_smithi026_260489_object26555:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e205)", 2023-09-23T20:57:06.860 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T20:56:50.917319+0000", 2023-09-23T20:57:06.860 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 15.809461278000001, 2023-09-23T20:57:06.860 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.048036126999999998, 2023-09-23T20:57:06.860 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T20:57:06.861 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:57:06.861 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T20:57:06.861 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.35100", 2023-09-23T20:57:06.861 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/4108972961", 2023-09-23T20:57:06.861 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 26556 2023-09-23T20:57:06.861 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:57:06.861 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T20:57:06.861 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:57:06.861 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T20:57:06.861 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:56:50.917319+0000", 2023-09-23T20:57:06.861 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T20:57:06.861 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:57:06.861 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:57:06.862 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T20:57:06.862 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:56:50.917319+0000", 2023-09-23T20:57:06.862 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T20:57:06.862 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:57:06.862 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:57:06.862 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T20:57:06.862 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:56:50.917317+0000", 2023-09-23T20:57:06.862 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T20:57:06.862 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:57:06.862 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:57:06.862 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T20:57:06.862 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:56:50.917349+0000", 2023-09-23T20:57:06.862 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.1538000000000002e-05 2023-09-23T20:57:06.863 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:57:06.863 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:57:06.863 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T20:57:06.863 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:56:50.917367+0000", 2023-09-23T20:57:06.863 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.7487e-05 2023-09-23T20:57:06.863 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:57:06.863 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:57:06.863 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T20:57:06.863 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:56:50.917374+0000", 2023-09-23T20:57:06.863 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.5000000000000002e-06 2023-09-23T20:57:06.863 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:57:06.863 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:57:06.863 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T20:57:06.863 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:56:50.917416+0000", 2023-09-23T20:57:06.864 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.1967e-05 2023-09-23T20:57:06.864 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:57:06.864 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:57:06.864 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T20:57:06.864 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:56:50.917557+0000", 2023-09-23T20:57:06.864 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00014108600000000001 2023-09-23T20:57:06.864 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:57:06.864 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:57:06.864 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 2,7", 2023-09-23T20:57:06.864 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:56:50.917705+0000", 2023-09-23T20:57:06.864 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00014808900000000001 2023-09-23T20:57:06.864 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:57:06.864 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:57:06.865 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T20:57:06.865 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:56:50.918878+0000", 2023-09-23T20:57:06.865 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.001172483 2023-09-23T20:57:06.865 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:57:06.865 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:57:06.865 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:57:06.865 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:56:50.918980+0000", 2023-09-23T20:57:06.865 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000102138 2023-09-23T20:57:06.865 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:57:06.865 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:57:06.865 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:57:06.865 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:56:50.965302+0000", 2023-09-23T20:57:06.866 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.046321767 2023-09-23T20:57:06.866 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:57:06.866 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:57:06.866 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T20:57:06.866 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:56:50.965333+0000", 2023-09-23T20:57:06.866 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.1111000000000001e-05 2023-09-23T20:57:06.866 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:57:06.866 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:57:06.866 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T20:57:06.866 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:56:50.965355+0000", 2023-09-23T20:57:06.866 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.2546999999999999e-05 2023-09-23T20:57:06.866 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T20:57:06.866 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T20:57:06.866 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T20:57:06.867 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T20:57:06.867 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T20:57:06.867 INFO:teuthology.orchestra.run.smithi070.stdout:} 2023-09-23T20:57:06.867 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:57:06.905 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 28 16 50101 50085 111.749 76.3125 0.00621149 0.0089428 2023-09-23T20:57:06.936 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:57:07.094 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_ops_in_flight 2023-09-23T20:57:07.098 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:57:07.103 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:57:07.103 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:57:07.204 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:57:07.243 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:57:07.275 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0: {} 2023-09-23T20:57:07.290 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:57:07.293 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:57:07.294 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:57:07.319 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1: {} 2023-09-23T20:57:07.364 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2: {} 2023-09-23T20:57:07.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:57:07 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "21"}]': finished 2023-09-23T20:57:07.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:57:07 smithi070 ceph-mon[168282]: osdmap e208: 8 total, 8 up, 8 in 2023-09-23T20:57:07.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:57:07 smithi070 ceph-mon[168282]: failed to encode map e208 with expected crc 2023-09-23T20:57:07.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:57:07 smithi070 ceph-mon[168282]: failed to encode map e208 with expected crc 2023-09-23T20:57:07.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:57:07 smithi070 ceph-mon[168282]: failed to encode map e208 with expected crc 2023-09-23T20:57:07.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:57:07 smithi070 ceph-mon[168282]: pgmap v487: 99 pgs: 99 active+clean; 2.8 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 93 MiB/s wr, 1.49k op/s 2023-09-23T20:57:07.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:57:07 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "21"}]: dispatch 2023-09-23T20:57:07.395 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:57:07.407 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3: {} 2023-09-23T20:57:07.428 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:57:07.439 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4: {} 2023-09-23T20:57:07.474 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T20:57:07.520 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:57:07 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "21"}]': finished 2023-09-23T20:57:07.520 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:57:07 smithi026 ceph-mon[179688]: osdmap e208: 8 total, 8 up, 8 in 2023-09-23T20:57:07.520 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:57:07 smithi026 ceph-mon[179688]: failed to encode map e208 with expected crc 2023-09-23T20:57:07.520 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:57:07 smithi026 ceph-mon[179688]: failed to encode map e208 with expected crc 2023-09-23T20:57:07.520 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:57:07 smithi026 ceph-mon[179688]: failed to encode map e208 with expected crc 2023-09-23T20:57:07.520 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:57:07 smithi026 ceph-mon[179688]: pgmap v487: 99 pgs: 99 active+clean; 2.8 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 93 MiB/s wr, 1.49k op/s 2023-09-23T20:57:07.520 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:57:07 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "21"}]: dispatch 2023-09-23T20:57:07.521 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6: {} 2023-09-23T20:57:07.522 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:57:07 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "21"}]': finished 2023-09-23T20:57:07.522 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:57:07 smithi026 ceph-mon[175151]: osdmap e208: 8 total, 8 up, 8 in 2023-09-23T20:57:07.522 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:57:07 smithi026 ceph-mon[175151]: failed to encode map e208 with expected crc 2023-09-23T20:57:07.522 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:57:07 smithi026 ceph-mon[175151]: failed to encode map e208 with expected crc 2023-09-23T20:57:07.522 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:57:07 smithi026 ceph-mon[175151]: failed to encode map e208 with expected crc 2023-09-23T20:57:07.522 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:57:07 smithi026 ceph-mon[175151]: pgmap v487: 99 pgs: 99 active+clean; 2.8 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 93 MiB/s wr, 1.49k op/s 2023-09-23T20:57:07.522 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:57:07 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "21"}]: dispatch 2023-09-23T20:57:07.526 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:57:07.567 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T20:57:07.577 INFO:teuthology.orchestra.run.smithi026.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T20:57:07.578 INFO:teuthology.orchestra.run.smithi026.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T20:57:07.578 INFO:teuthology.orchestra.run.smithi026.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T20:57:07.578 INFO:teuthology.orchestra.run.smithi026.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T20:57:07.578 INFO:teuthology.orchestra.run.smithi026.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T20:57:07.578 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T20:57:07.578 INFO:teuthology.orchestra.run.smithi026.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T20:57:07.578 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T20:57:07.626 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:57:07.631 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:57:07.631 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:57:07.732 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:57:07.763 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:57:07.806 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:57:07.809 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:57:07.809 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:57:07.906 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 29 16 51228 51212 110.325 70.4375 0.00589702 0.00905715 2023-09-23T20:57:07.910 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:57:07.949 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:57:07.997 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:57:08.000 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:57:08.001 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:57:08.101 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:57:08.170 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:57:08.381 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:57:08 smithi026 ceph-mon[175151]: failed to encode map e209 with expected crc 2023-09-23T20:57:08.381 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:57:08 smithi026 ceph-mon[175151]: failed to encode map e209 with expected crc 2023-09-23T20:57:08.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:57:08 smithi026 ceph-mon[175151]: failed to encode map e209 with expected crc 2023-09-23T20:57:08.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:57:08 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "21"}]': finished 2023-09-23T20:57:08.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:57:08 smithi026 ceph-mon[175151]: osdmap e209: 8 total, 8 up, 8 in 2023-09-23T20:57:08.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:57:08 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/2667551449' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:57:08.384 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:57:08 smithi026 ceph-mon[179688]: failed to encode map e209 with expected crc 2023-09-23T20:57:08.384 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:57:08 smithi026 ceph-mon[179688]: failed to encode map e209 with expected crc 2023-09-23T20:57:08.384 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:57:08 smithi026 ceph-mon[179688]: failed to encode map e209 with expected crc 2023-09-23T20:57:08.384 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:57:08 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "21"}]': finished 2023-09-23T20:57:08.384 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:57:08 smithi026 ceph-mon[179688]: osdmap e209: 8 total, 8 up, 8 in 2023-09-23T20:57:08.384 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:57:08 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/2667551449' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:57:08.490 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:57:08.494 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:57:08.507 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:57:08.507 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:57:08.608 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:57:08.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:57:08 smithi070 ceph-mon[168282]: failed to encode map e209 with expected crc 2023-09-23T20:57:08.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:57:08 smithi070 ceph-mon[168282]: failed to encode map e209 with expected crc 2023-09-23T20:57:08.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:57:08 smithi070 ceph-mon[168282]: failed to encode map e209 with expected crc 2023-09-23T20:57:08.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:57:08 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "21"}]': finished 2023-09-23T20:57:08.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:57:08 smithi070 ceph-mon[168282]: osdmap e209: 8 total, 8 up, 8 in 2023-09-23T20:57:08.621 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:57:08 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/2667551449' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:57:08.684 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:57:08.866 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:57:08.873 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:57:08.873 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:57:08.908 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 30 16 52464 52448 109.215 77.25 0.00447914 0.00915003 2023-09-23T20:57:08.974 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:57:09.027 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:57:09.075 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:57:09.079 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:57:09.079 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:57:09.179 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:57:09.274 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:57:09.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:57:09 smithi070 ceph-mon[168282]: failed to encode map e210 with expected crc 2023-09-23T20:57:09.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:57:09 smithi070 ceph-mon[168282]: failed to encode map e210 with expected crc 2023-09-23T20:57:09.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:57:09 smithi070 ceph-mon[168282]: failed to encode map e210 with expected crc 2023-09-23T20:57:09.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:57:09 smithi070 ceph-mon[168282]: osdmap e210: 8 total, 8 up, 8 in 2023-09-23T20:57:09.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:57:09 smithi070 ceph-mon[168282]: failed to encode map e210 with expected crc 2023-09-23T20:57:09.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:57:09 smithi070 ceph-mon[168282]: failed to encode map e210 with expected crc 2023-09-23T20:57:09.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:57:09 smithi070 ceph-mon[168282]: failed to encode map e210 with expected crc 2023-09-23T20:57:09.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:57:09 smithi070 ceph-mon[168282]: pgmap v490: 109 pgs: 10 unknown, 99 active+clean; 2.9 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 81 MiB/s wr, 1.30k op/s 2023-09-23T20:57:09.445 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:57:09.451 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:57:09.451 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:57:09.507 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:57:09 smithi026 ceph-mon[175151]: failed to encode map e210 with expected crc 2023-09-23T20:57:09.507 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:57:09 smithi026 ceph-mon[175151]: failed to encode map e210 with expected crc 2023-09-23T20:57:09.507 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:57:09 smithi026 ceph-mon[175151]: failed to encode map e210 with expected crc 2023-09-23T20:57:09.507 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:57:09 smithi026 ceph-mon[175151]: osdmap e210: 8 total, 8 up, 8 in 2023-09-23T20:57:09.507 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:57:09 smithi026 ceph-mon[175151]: failed to encode map e210 with expected crc 2023-09-23T20:57:09.507 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:57:09 smithi026 ceph-mon[175151]: failed to encode map e210 with expected crc 2023-09-23T20:57:09.507 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:57:09 smithi026 ceph-mon[175151]: failed to encode map e210 with expected crc 2023-09-23T20:57:09.507 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:57:09 smithi026 ceph-mon[175151]: pgmap v490: 109 pgs: 10 unknown, 99 active+clean; 2.9 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 81 MiB/s wr, 1.30k op/s 2023-09-23T20:57:09.511 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:57:09 smithi026 ceph-mon[179688]: failed to encode map e210 with expected crc 2023-09-23T20:57:09.511 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:57:09 smithi026 ceph-mon[179688]: failed to encode map e210 with expected crc 2023-09-23T20:57:09.511 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:57:09 smithi026 ceph-mon[179688]: failed to encode map e210 with expected crc 2023-09-23T20:57:09.511 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:57:09 smithi026 ceph-mon[179688]: osdmap e210: 8 total, 8 up, 8 in 2023-09-23T20:57:09.511 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:57:09 smithi026 ceph-mon[179688]: failed to encode map e210 with expected crc 2023-09-23T20:57:09.511 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:57:09 smithi026 ceph-mon[179688]: failed to encode map e210 with expected crc 2023-09-23T20:57:09.511 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:57:09 smithi026 ceph-mon[179688]: failed to encode map e210 with expected crc 2023-09-23T20:57:09.511 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:57:09 smithi026 ceph-mon[179688]: pgmap v490: 109 pgs: 10 unknown, 99 active+clean; 2.9 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 81 MiB/s wr, 1.30k op/s 2023-09-23T20:57:09.553 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:57:09.582 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:57:09.622 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:57:09.625 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:57:09.626 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:57:09.726 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:57:09.763 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:57:09.805 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:57:09.815 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:57:09.815 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:57:09.840 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:57:09 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[164029]: ::ffff:172.21.15.70 - - [23/Sep/2023:20:57:09] "GET /metrics HTTP/1.1" 200 36316 "" "Prometheus/2.43.0" 2023-09-23T20:57:09.908 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 31 16 53674 53658 108.132 75.625 0.019303 0.0092442 2023-09-23T20:57:09.916 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:57:09.999 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:57:10.042 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T20:57:10.067 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:57:10.073 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:57:10.074 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:57:10.175 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:57:10.281 INFO:teuthology.orchestra.run.smithi026.stdout:ERROR: (22) Invalid argument 2023-09-23T20:57:10.281 INFO:teuthology.orchestra.run.smithi026.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:57:10.287 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:57:10.306 INFO:teuthology.orchestra.run.smithi026.stderr:nodeep-scrub is set 2023-09-23T20:57:10.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:57:10 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/2300686501' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:57:10.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:57:10 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:57:10.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:57:10 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/2300686501' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:57:10.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:57:10 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:57:10.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:57:10 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/2300686501' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:57:10.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:57:10 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:57:10.724 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:57:10.729 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:57:10.729 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:57:10.817 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_blocked_ops 2023-09-23T20:57:10.830 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:57:10.912 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 32 16 54817 54801 106.974 71.4375 0.0175484 0.00934336 2023-09-23T20:57:10.979 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:57:11.597 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:57:11.611 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:57:11.611 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:57:11 smithi026 ceph-mon[175151]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:57:11.611 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:57:11 smithi026 ceph-mon[175151]: failed to encode map e211 with expected crc 2023-09-23T20:57:11.611 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:57:11 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T20:57:11.612 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:57:11 smithi026 ceph-mon[175151]: failed to encode map e211 with expected crc 2023-09-23T20:57:11.612 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:57:11 smithi026 ceph-mon[175151]: osdmap e211: 8 total, 8 up, 8 in 2023-09-23T20:57:11.612 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:57:11 smithi026 ceph-mon[175151]: pgmap v492: 109 pgs: 4 unknown, 105 active+clean; 3.1 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 73 MiB/s wr, 1.17k op/s 2023-09-23T20:57:11.612 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:57:11 smithi026 ceph-mon[179688]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:57:11.612 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:57:11 smithi026 ceph-mon[179688]: failed to encode map e211 with expected crc 2023-09-23T20:57:11.612 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:57:11 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T20:57:11.612 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:57:11 smithi026 ceph-mon[179688]: failed to encode map e211 with expected crc 2023-09-23T20:57:11.612 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:57:11 smithi026 ceph-mon[179688]: osdmap e211: 8 total, 8 up, 8 in 2023-09-23T20:57:11.612 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:57:11 smithi026 ceph-mon[179688]: pgmap v492: 109 pgs: 4 unknown, 105 active+clean; 3.1 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 73 MiB/s wr, 1.17k op/s 2023-09-23T20:57:11.613 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:57:11.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:57:11 smithi070 ceph-mon[168282]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:57:11.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:57:11 smithi070 ceph-mon[168282]: failed to encode map e211 with expected crc 2023-09-23T20:57:11.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:57:11 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T20:57:11.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:57:11 smithi070 ceph-mon[168282]: failed to encode map e211 with expected crc 2023-09-23T20:57:11.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:57:11 smithi070 ceph-mon[168282]: osdmap e211: 8 total, 8 up, 8 in 2023-09-23T20:57:11.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:57:11 smithi070 ceph-mon[168282]: pgmap v492: 109 pgs: 4 unknown, 105 active+clean; 3.1 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 73 MiB/s wr, 1.17k op/s 2023-09-23T20:57:11.714 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:57:11.748 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:57:11.784 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:57:11.788 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:57:11.788 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:57:11.889 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:57:11.897 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:57:11.912 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 33 15 56040 56025 106.048 76.5 0.0160949 0.00942339 2023-09-23T20:57:11.921 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:57:11.948 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:57:12.210 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 6, 7, 4, 5] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 7, 6] 2023-09-23T20:57:12.210 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph osd reweight 0 1 2023-09-23T20:57:12.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:57:12 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "21"}]: dispatch 2023-09-23T20:57:12.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:57:12 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "21"}]: dispatch 2023-09-23T20:57:12.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:57:12 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "21"}]: dispatch 2023-09-23T20:57:12.917 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 34 16 57268 57252 105.171 76.6875 0.0130938 0.00950325 2023-09-23T20:57:13.029 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:57:13.032 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:57:13.033 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:57:13.134 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:57:13.192 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:57:13.276 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:57:13.285 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:57:13.286 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:57:13.387 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:57:13.438 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:57:13.494 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:57:13.501 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:57:13.501 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:57:13.603 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:57:13.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:57:13 smithi026 ceph-mon[175151]: pgmap v493: 109 pgs: 109 active+clean; 3.3 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 85 MiB/s wr, 1.35k op/s 2023-09-23T20:57:13.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:57:13 smithi026 ceph-mon[175151]: failed to encode map e212 with expected crc 2023-09-23T20:57:13.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:57:13 smithi026 ceph-mon[175151]: failed to encode map e212 with expected crc 2023-09-23T20:57:13.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:57:13 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "21"}]': finished 2023-09-23T20:57:13.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:57:13 smithi026 ceph-mon[175151]: osdmap e212: 8 total, 8 up, 8 in 2023-09-23T20:57:13.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:57:13 smithi026 ceph-mon[175151]: failed to encode map e212 with expected crc 2023-09-23T20:57:13.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:57:13 smithi026 ceph-mon[175151]: failed to encode map e212 with expected crc 2023-09-23T20:57:13.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:57:13 smithi026 ceph-mon[175151]: failed to encode map e212 with expected crc 2023-09-23T20:57:13.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:57:13 smithi026 ceph-mon[175151]: failed to encode map e212 with expected crc 2023-09-23T20:57:13.633 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:57:13 smithi026 ceph-mon[175151]: failed to encode map e212 with expected crc 2023-09-23T20:57:13.633 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:57:13 smithi026 ceph-mon[179688]: pgmap v493: 109 pgs: 109 active+clean; 3.3 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 85 MiB/s wr, 1.35k op/s 2023-09-23T20:57:13.633 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:57:13 smithi026 ceph-mon[179688]: failed to encode map e212 with expected crc 2023-09-23T20:57:13.633 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:57:13 smithi026 ceph-mon[179688]: failed to encode map e212 with expected crc 2023-09-23T20:57:13.633 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:57:13 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "21"}]': finished 2023-09-23T20:57:13.633 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:57:13 smithi026 ceph-mon[179688]: osdmap e212: 8 total, 8 up, 8 in 2023-09-23T20:57:13.633 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:57:13 smithi026 ceph-mon[179688]: failed to encode map e212 with expected crc 2023-09-23T20:57:13.633 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:57:13 smithi026 ceph-mon[179688]: failed to encode map e212 with expected crc 2023-09-23T20:57:13.633 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:57:13 smithi026 ceph-mon[179688]: failed to encode map e212 with expected crc 2023-09-23T20:57:13.633 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:57:13 smithi026 ceph-mon[179688]: failed to encode map e212 with expected crc 2023-09-23T20:57:13.634 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:57:13 smithi026 ceph-mon[179688]: failed to encode map e212 with expected crc 2023-09-23T20:57:13.648 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:57:13.659 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T20:57:13.691 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:57:13.695 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:57:13 smithi070 ceph-mon[168282]: pgmap v493: 109 pgs: 109 active+clean; 3.3 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 85 MiB/s wr, 1.35k op/s 2023-09-23T20:57:13.695 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:57:13 smithi070 ceph-mon[168282]: failed to encode map e212 with expected crc 2023-09-23T20:57:13.695 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:57:13 smithi070 ceph-mon[168282]: failed to encode map e212 with expected crc 2023-09-23T20:57:13.695 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:57:13 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "21"}]': finished 2023-09-23T20:57:13.695 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:57:13 smithi070 ceph-mon[168282]: osdmap e212: 8 total, 8 up, 8 in 2023-09-23T20:57:13.695 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:57:13 smithi070 ceph-mon[168282]: failed to encode map e212 with expected crc 2023-09-23T20:57:13.696 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:57:13 smithi070 ceph-mon[168282]: failed to encode map e212 with expected crc 2023-09-23T20:57:13.696 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:57:13 smithi070 ceph-mon[168282]: failed to encode map e212 with expected crc 2023-09-23T20:57:13.696 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:57:13 smithi070 ceph-mon[168282]: failed to encode map e212 with expected crc 2023-09-23T20:57:13.696 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:57:13 smithi070 ceph-mon[168282]: failed to encode map e212 with expected crc 2023-09-23T20:57:13.696 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:57:13.697 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:57:13.797 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:57:13.917 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 35 16 58506 58490 104.377 77.375 0.0180028 0.00957433 2023-09-23T20:57:13.920 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:57:14.415 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:57:14.423 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:57:14.423 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:57:14.524 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:57:14.592 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:57:14.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:57:14 smithi070 ceph-mon[168282]: osdmap e213: 8 total, 8 up, 8 in 2023-09-23T20:57:14.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:57:14 smithi070 ceph-mon[168282]: failed to encode map e213 with expected crc 2023-09-23T20:57:14.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:57:14 smithi070 ceph-mon[168282]: failed to encode map e213 with expected crc 2023-09-23T20:57:14.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:57:14 smithi070 ceph-mon[168282]: failed to encode map e213 with expected crc 2023-09-23T20:57:14.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:57:14 smithi070 ceph-mon[168282]: failed to encode map e213 with expected crc 2023-09-23T20:57:14.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:57:14 smithi070 ceph-mon[168282]: failed to encode map e213 with expected crc 2023-09-23T20:57:14.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:57:14 smithi070 ceph-mon[168282]: failed to encode map e213 with expected crc 2023-09-23T20:57:14.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:57:14 smithi070 ceph-mon[168282]: failed to encode map e213 with expected crc 2023-09-23T20:57:14.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:57:14 smithi070 ceph-mon[168282]: failed to encode map e213 with expected crc 2023-09-23T20:57:14.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:57:14 smithi070 ceph-mon[168282]: failed to encode map e213 with expected crc 2023-09-23T20:57:14.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:57:14 smithi070 ceph-mon[168282]: osdmap e214: 8 total, 8 up, 8 in 2023-09-23T20:57:14.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:57:14 smithi026 ceph-mon[179688]: osdmap e213: 8 total, 8 up, 8 in 2023-09-23T20:57:14.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:57:14 smithi026 ceph-mon[179688]: failed to encode map e213 with expected crc 2023-09-23T20:57:14.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:57:14 smithi026 ceph-mon[179688]: failed to encode map e213 with expected crc 2023-09-23T20:57:14.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:57:14 smithi026 ceph-mon[179688]: failed to encode map e213 with expected crc 2023-09-23T20:57:14.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:57:14 smithi026 ceph-mon[179688]: failed to encode map e213 with expected crc 2023-09-23T20:57:14.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:57:14 smithi026 ceph-mon[179688]: failed to encode map e213 with expected crc 2023-09-23T20:57:14.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:57:14 smithi026 ceph-mon[179688]: failed to encode map e213 with expected crc 2023-09-23T20:57:14.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:57:14 smithi026 ceph-mon[179688]: failed to encode map e213 with expected crc 2023-09-23T20:57:14.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:57:14 smithi026 ceph-mon[179688]: failed to encode map e213 with expected crc 2023-09-23T20:57:14.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:57:14 smithi026 ceph-mon[179688]: failed to encode map e213 with expected crc 2023-09-23T20:57:14.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:57:14 smithi026 ceph-mon[179688]: osdmap e214: 8 total, 8 up, 8 in 2023-09-23T20:57:14.883 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:57:14 smithi026 ceph-mon[175151]: osdmap e213: 8 total, 8 up, 8 in 2023-09-23T20:57:14.883 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:57:14 smithi026 ceph-mon[175151]: failed to encode map e213 with expected crc 2023-09-23T20:57:14.883 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:57:14 smithi026 ceph-mon[175151]: failed to encode map e213 with expected crc 2023-09-23T20:57:14.883 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:57:14 smithi026 ceph-mon[175151]: failed to encode map e213 with expected crc 2023-09-23T20:57:14.883 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:57:14 smithi026 ceph-mon[175151]: failed to encode map e213 with expected crc 2023-09-23T20:57:14.883 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:57:14 smithi026 ceph-mon[175151]: failed to encode map e213 with expected crc 2023-09-23T20:57:14.883 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:57:14 smithi026 ceph-mon[175151]: failed to encode map e213 with expected crc 2023-09-23T20:57:14.883 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:57:14 smithi026 ceph-mon[175151]: failed to encode map e213 with expected crc 2023-09-23T20:57:14.883 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:57:14 smithi026 ceph-mon[175151]: failed to encode map e213 with expected crc 2023-09-23T20:57:14.883 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:57:14 smithi026 ceph-mon[175151]: failed to encode map e213 with expected crc 2023-09-23T20:57:14.883 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:57:14 smithi026 ceph-mon[175151]: osdmap e214: 8 total, 8 up, 8 in 2023-09-23T20:57:14.917 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 36 16 59636 59620 103.44 70.625 0.0146263 0.00966195 2023-09-23T20:57:15.075 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:57:15.078 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:57:15.079 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:57:15.179 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:57:15.215 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:57:15.261 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:57:15.264 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:57:15.264 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:57:15.365 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:57:15.452 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:57:15.564 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:57:15 smithi026 ceph-mon[175151]: pgmap v496: 109 pgs: 3 remapped+peering, 106 active+clean; 3.4 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 79 MiB/s wr, 1.26k op/s 2023-09-23T20:57:15.564 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:57:15 smithi026 ceph-mon[175151]: failed to encode map e214 with expected crc 2023-09-23T20:57:15.564 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:57:15 smithi026 ceph-mon[175151]: failed to encode map e214 with expected crc 2023-09-23T20:57:15.564 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:57:15 smithi026 ceph-mon[175151]: failed to encode map e214 with expected crc 2023-09-23T20:57:15.564 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:57:15 smithi026 ceph-mon[175151]: failed to encode map e214 with expected crc 2023-09-23T20:57:15.564 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:57:15 smithi026 ceph-mon[175151]: failed to encode map e214 with expected crc 2023-09-23T20:57:15.564 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:57:15 smithi026 ceph-mon[175151]: osdmap e215: 8 total, 8 up, 8 in 2023-09-23T20:57:15.564 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:57:15 smithi026 ceph-mon[175151]: failed to encode map e215 with expected crc 2023-09-23T20:57:15.565 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:57:15 smithi026 ceph-mon[179688]: pgmap v496: 109 pgs: 3 remapped+peering, 106 active+clean; 3.4 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 79 MiB/s wr, 1.26k op/s 2023-09-23T20:57:15.565 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:57:15 smithi026 ceph-mon[179688]: failed to encode map e214 with expected crc 2023-09-23T20:57:15.565 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:57:15 smithi026 ceph-mon[179688]: failed to encode map e214 with expected crc 2023-09-23T20:57:15.565 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:57:15 smithi026 ceph-mon[179688]: failed to encode map e214 with expected crc 2023-09-23T20:57:15.565 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:57:15 smithi026 ceph-mon[179688]: failed to encode map e214 with expected crc 2023-09-23T20:57:15.565 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:57:15 smithi026 ceph-mon[179688]: failed to encode map e214 with expected crc 2023-09-23T20:57:15.565 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:57:15 smithi026 ceph-mon[179688]: osdmap e215: 8 total, 8 up, 8 in 2023-09-23T20:57:15.565 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:57:15 smithi026 ceph-mon[179688]: failed to encode map e215 with expected crc 2023-09-23T20:57:15.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:57:15 smithi070 ceph-mon[168282]: pgmap v496: 109 pgs: 3 remapped+peering, 106 active+clean; 3.4 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 79 MiB/s wr, 1.26k op/s 2023-09-23T20:57:15.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:57:15 smithi070 ceph-mon[168282]: failed to encode map e214 with expected crc 2023-09-23T20:57:15.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:57:15 smithi070 ceph-mon[168282]: failed to encode map e214 with expected crc 2023-09-23T20:57:15.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:57:15 smithi070 ceph-mon[168282]: failed to encode map e214 with expected crc 2023-09-23T20:57:15.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:57:15 smithi070 ceph-mon[168282]: failed to encode map e214 with expected crc 2023-09-23T20:57:15.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:57:15 smithi070 ceph-mon[168282]: failed to encode map e214 with expected crc 2023-09-23T20:57:15.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:57:15 smithi070 ceph-mon[168282]: osdmap e215: 8 total, 8 up, 8 in 2023-09-23T20:57:15.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:57:15 smithi070 ceph-mon[168282]: failed to encode map e215 with expected crc 2023-09-23T20:57:15.851 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:57:15.852 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:57:15.853 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:57:15.871 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:57:15.917 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 37 16 60781 60765 102.577 71.5625 0.00937644 0.00974157 2023-09-23T20:57:15.954 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:57:15.992 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:57:16.034 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:57:16.038 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:57:16.039 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:57:16.056 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:57:16.139 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:57:16.219 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:57:16.605 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:57:16.611 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:57:16.612 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:57:16.678 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:57:16 smithi026 ceph-mon[175151]: failed to encode map e215 with expected crc 2023-09-23T20:57:16.678 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:57:16 smithi026 ceph-mon[175151]: osdmap e216: 8 total, 8 up, 8 in 2023-09-23T20:57:16.678 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:57:16 smithi026 ceph-mon[175151]: failed to encode map e216 with expected crc 2023-09-23T20:57:16.678 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:57:16 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:57:16.678 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:57:16 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:57:16.679 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:57:16 smithi026 ceph-mon[179688]: failed to encode map e215 with expected crc 2023-09-23T20:57:16.679 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:57:16 smithi026 ceph-mon[179688]: osdmap e216: 8 total, 8 up, 8 in 2023-09-23T20:57:16.679 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:57:16 smithi026 ceph-mon[179688]: failed to encode map e216 with expected crc 2023-09-23T20:57:16.679 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:57:16 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:57:16.679 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:57:16 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:57:16.712 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:57:16.805 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:57:16.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:57:16 smithi070 ceph-mon[168282]: failed to encode map e215 with expected crc 2023-09-23T20:57:16.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:57:16 smithi070 ceph-mon[168282]: osdmap e216: 8 total, 8 up, 8 in 2023-09-23T20:57:16.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:57:16 smithi070 ceph-mon[168282]: failed to encode map e216 with expected crc 2023-09-23T20:57:16.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:57:16 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:57:16.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:57:16 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:57:16.918 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 38 16 61867 61851 101.664 67.875 0.0155389 0.00983153 2023-09-23T20:57:17.149 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:57:17.170 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:57:17.170 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:57:17.271 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:57:17.313 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0: {} 2023-09-23T20:57:17.364 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1: {} 2023-09-23T20:57:17.382 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:57:17.415 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2: {} 2023-09-23T20:57:17.456 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3: {} 2023-09-23T20:57:17.492 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4: {} 2023-09-23T20:57:17.542 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T20:57:17.583 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6: {} 2023-09-23T20:57:17.609 INFO:teuthology.orchestra.run.smithi026.stdout:{ 2023-09-23T20:57:17.609 INFO:teuthology.orchestra.run.smithi026.stdout: "ops": [], 2023-09-23T20:57:17.609 INFO:teuthology.orchestra.run.smithi026.stdout: "complaint_time": 30, 2023-09-23T20:57:17.610 INFO:teuthology.orchestra.run.smithi026.stdout: "num_blocked_ops": 0 2023-09-23T20:57:17.610 INFO:teuthology.orchestra.run.smithi026.stdout:} 2023-09-23T20:57:17.617 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T20:57:17.630 INFO:teuthology.orchestra.run.smithi026.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T20:57:17.630 INFO:teuthology.orchestra.run.smithi026.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T20:57:17.630 INFO:teuthology.orchestra.run.smithi026.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T20:57:17.630 INFO:teuthology.orchestra.run.smithi026.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T20:57:17.630 INFO:teuthology.orchestra.run.smithi026.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T20:57:17.630 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T20:57:17.630 INFO:teuthology.orchestra.run.smithi026.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T20:57:17.630 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T20:57:17.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:57:17 smithi026 ceph-mon[179688]: failed to encode map e216 with expected crc 2023-09-23T20:57:17.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:57:17 smithi026 ceph-mon[179688]: pgmap v500: 109 pgs: 5 remapped+peering, 104 active+clean; 3.6 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 11 KiB/s rd, 66 MiB/s wr, 1.06k op/s; 20 KiB/s, 0 objects/s recovering 2023-09-23T20:57:17.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:57:17 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/3384382502' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:57:17.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:57:17 smithi026 ceph-mon[175151]: failed to encode map e216 with expected crc 2023-09-23T20:57:17.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:57:17 smithi026 ceph-mon[175151]: pgmap v500: 109 pgs: 5 remapped+peering, 104 active+clean; 3.6 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 11 KiB/s rd, 66 MiB/s wr, 1.06k op/s; 20 KiB/s, 0 objects/s recovering 2023-09-23T20:57:17.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:57:17 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/3384382502' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:57:17.695 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:57:17.704 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:57:17.704 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:57:17.805 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:57:17.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:57:17 smithi070 ceph-mon[168282]: failed to encode map e216 with expected crc 2023-09-23T20:57:17.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:57:17 smithi070 ceph-mon[168282]: pgmap v500: 109 pgs: 5 remapped+peering, 104 active+clean; 3.6 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 11 KiB/s rd, 66 MiB/s wr, 1.06k op/s; 20 KiB/s, 0 objects/s recovering 2023-09-23T20:57:17.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:57:17 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/3384382502' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:57:17.881 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:57:17.919 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 39 16 62864 62848 100.652 62.3125 0.00786357 0.0099282 2023-09-23T20:57:18.391 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_historic_ops 2023-09-23T20:57:18.548 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:57:18.552 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:57:18.552 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:57:18.653 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:57:18.782 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:57:18.876 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:57:18.883 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:57:18.883 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:57:18.919 INFO:tasks.radosbench.radosbench.0.smithi026.stdout:2023-09-23T20:57:18.919876+0000 min lat: 0.0011341 max lat: 1.0132 avg lat: 0.010032 2023-09-23T20:57:18.919 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T20:57:18.919 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 40 16 63822 63806 99.6334 59.875 0.00540037 0.010032 2023-09-23T20:57:18.984 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:57:19.070 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:57:19.148 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:57:19.165 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:57:19.165 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:57:19.266 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:57:19.361 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:57:19.409 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:57:19.416 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:57:19.428 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:57:19.428 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:57:19.502 INFO:teuthology.orchestra.run.smithi026.stderr:reweighted osd.0 to 1 (10000) 2023-09-23T20:57:19.521 INFO:teuthology.orchestra.run.smithi026.stderr:noscrub is unset 2023-09-23T20:57:19.529 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:57:19.564 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:57:19.604 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:57:19.609 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:57:19.609 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:57:19.618 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:57:19 smithi026 ceph-mon[175151]: pgmap v501: 109 pgs: 4 remapped+peering, 105 active+clean; 3.7 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 17 KiB/s rd, 76 MiB/s wr, 1.22k op/s; 32 KiB/s, 0 objects/s recovering 2023-09-23T20:57:19.618 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:57:19 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/2254012978' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-09-23T20:57:19.618 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:57:19 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/2125366986' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:57:19.618 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:57:19 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:57:19.618 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:57:19 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[164029]: ::ffff:172.21.15.70 - - [23/Sep/2023:20:57:19] "GET /metrics HTTP/1.1" 200 36315 "" "Prometheus/2.43.0" 2023-09-23T20:57:19.618 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:57:19 smithi026 ceph-mon[179688]: pgmap v501: 109 pgs: 4 remapped+peering, 105 active+clean; 3.7 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 17 KiB/s rd, 76 MiB/s wr, 1.22k op/s; 32 KiB/s, 0 objects/s recovering 2023-09-23T20:57:19.619 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:57:19 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/2254012978' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-09-23T20:57:19.619 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:57:19 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/2125366986' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:57:19.619 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:57:19 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:57:19.710 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:57:19.742 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:57:19 smithi070 ceph-mon[168282]: pgmap v501: 109 pgs: 4 remapped+peering, 105 active+clean; 3.7 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 17 KiB/s rd, 76 MiB/s wr, 1.22k op/s; 32 KiB/s, 0 objects/s recovering 2023-09-23T20:57:19.742 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:57:19 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/2254012978' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-09-23T20:57:19.742 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:57:19 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/2125366986' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:57:19.742 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:57:19 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:57:19.745 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:57:19.786 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:57:19.789 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:57:19.789 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:57:19.890 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:57:19.919 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 41 16 64908 64892 98.859 67.875 0.0136989 0.0101108 2023-09-23T20:57:19.930 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:57:19.974 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:57:19.978 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:57:19.978 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:57:20.026 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph osd reweight 1 1 2023-09-23T20:57:20.079 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:57:20.150 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:57:20.731 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T20:57:20.806 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:57:20 smithi026 ceph-mon[179688]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:57:20.807 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:57:20 smithi026 ceph-mon[179688]: failed to encode map e217 with expected crc 2023-09-23T20:57:20.807 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:57:20 smithi026 ceph-mon[179688]: failed to encode map e217 with expected crc 2023-09-23T20:57:20.807 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:57:20 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/2254012978' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-09-23T20:57:20.807 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:57:20 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T20:57:20.807 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:57:20 smithi026 ceph-mon[179688]: osdmap e217: 8 total, 8 up, 8 in 2023-09-23T20:57:20.807 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:57:20 smithi026 ceph-mon[179688]: failed to encode map e217 with expected crc 2023-09-23T20:57:20.807 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:57:20 smithi026 ceph-mon[179688]: 1.1 scrub starts 2023-09-23T20:57:20.807 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:57:20 smithi026 ceph-mon[179688]: 1.1 scrub ok 2023-09-23T20:57:20.807 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:57:20 smithi026 ceph-mon[175151]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:57:20.808 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:57:20 smithi026 ceph-mon[175151]: failed to encode map e217 with expected crc 2023-09-23T20:57:20.808 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:57:20 smithi026 ceph-mon[175151]: failed to encode map e217 with expected crc 2023-09-23T20:57:20.808 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:57:20 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/2254012978' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-09-23T20:57:20.808 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:57:20 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T20:57:20.808 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:57:20 smithi026 ceph-mon[175151]: osdmap e217: 8 total, 8 up, 8 in 2023-09-23T20:57:20.808 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:57:20 smithi026 ceph-mon[175151]: failed to encode map e217 with expected crc 2023-09-23T20:57:20.808 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:57:20 smithi026 ceph-mon[175151]: 1.1 scrub starts 2023-09-23T20:57:20.808 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:57:20 smithi026 ceph-mon[175151]: 1.1 scrub ok 2023-09-23T20:57:20.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:57:20 smithi070 ceph-mon[168282]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:57:20.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:57:20 smithi070 ceph-mon[168282]: failed to encode map e217 with expected crc 2023-09-23T20:57:20.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:57:20 smithi070 ceph-mon[168282]: failed to encode map e217 with expected crc 2023-09-23T20:57:20.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:57:20 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/2254012978' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-09-23T20:57:20.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:57:20 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T20:57:20.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:57:20 smithi070 ceph-mon[168282]: osdmap e217: 8 total, 8 up, 8 in 2023-09-23T20:57:20.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:57:20 smithi070 ceph-mon[168282]: failed to encode map e217 with expected crc 2023-09-23T20:57:20.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:57:20 smithi070 ceph-mon[168282]: 1.1 scrub starts 2023-09-23T20:57:20.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:57:20 smithi070 ceph-mon[168282]: 1.1 scrub ok 2023-09-23T20:57:20.920 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 42 16 65979 65963 98.0975 66.9375 0.0120085 0.0101877 2023-09-23T20:57:21.109 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:57:21.113 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:57:21.114 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:57:21.215 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:57:21.287 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:57:21.522 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:57:21.537 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:57:21.538 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:57:21.638 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:57:21.676 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:57:21.721 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:57:21.727 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:57:21.727 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:57:21.828 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:57:21.859 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:57:21 smithi026 ceph-mon[179688]: 1.4 scrub starts 2023-09-23T20:57:21.859 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:57:21 smithi026 ceph-mon[179688]: 1.4 scrub ok 2023-09-23T20:57:21.860 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:57:21 smithi026 ceph-mon[179688]: pgmap v503: 109 pgs: 109 active+clean; 3.8 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 14 KiB/s rd, 59 MiB/s wr, 955 op/s; 81 KiB/s, 0 objects/s recovering 2023-09-23T20:57:21.860 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:57:21 smithi026 ceph-mon[179688]: 1.3 scrub starts 2023-09-23T20:57:21.860 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:57:21 smithi026 ceph-mon[179688]: 1.3 scrub ok 2023-09-23T20:57:21.860 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:57:21 smithi026 ceph-mon[179688]: 1.10 scrub starts 2023-09-23T20:57:21.860 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:57:21 smithi026 ceph-mon[179688]: 1.10 scrub ok 2023-09-23T20:57:21.860 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:57:21 smithi026 ceph-mon[179688]: 1.0 scrub starts 2023-09-23T20:57:21.860 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:57:21 smithi026 ceph-mon[179688]: 1.0 scrub ok 2023-09-23T20:57:21.860 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:57:21 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:57:21.860 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:57:21 smithi026 ceph-mon[175151]: 1.4 scrub starts 2023-09-23T20:57:21.860 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:57:21 smithi026 ceph-mon[175151]: 1.4 scrub ok 2023-09-23T20:57:21.861 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:57:21 smithi026 ceph-mon[175151]: pgmap v503: 109 pgs: 109 active+clean; 3.8 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 14 KiB/s rd, 59 MiB/s wr, 955 op/s; 81 KiB/s, 0 objects/s recovering 2023-09-23T20:57:21.861 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:57:21 smithi026 ceph-mon[175151]: 1.3 scrub starts 2023-09-23T20:57:21.861 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:57:21 smithi026 ceph-mon[175151]: 1.3 scrub ok 2023-09-23T20:57:21.861 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:57:21 smithi026 ceph-mon[175151]: 1.10 scrub starts 2023-09-23T20:57:21.861 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:57:21 smithi026 ceph-mon[175151]: 1.10 scrub ok 2023-09-23T20:57:21.861 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:57:21 smithi026 ceph-mon[175151]: 1.0 scrub starts 2023-09-23T20:57:21.861 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:57:21 smithi026 ceph-mon[175151]: 1.0 scrub ok 2023-09-23T20:57:21.861 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:57:21 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:57:21.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:57:21 smithi070 ceph-mon[168282]: 1.4 scrub starts 2023-09-23T20:57:21.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:57:21 smithi070 ceph-mon[168282]: 1.4 scrub ok 2023-09-23T20:57:21.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:57:21 smithi070 ceph-mon[168282]: pgmap v503: 109 pgs: 109 active+clean; 3.8 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 14 KiB/s rd, 59 MiB/s wr, 955 op/s; 81 KiB/s, 0 objects/s recovering 2023-09-23T20:57:21.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:57:21 smithi070 ceph-mon[168282]: 1.3 scrub starts 2023-09-23T20:57:21.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:57:21 smithi070 ceph-mon[168282]: 1.3 scrub ok 2023-09-23T20:57:21.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:57:21 smithi070 ceph-mon[168282]: 1.10 scrub starts 2023-09-23T20:57:21.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:57:21 smithi070 ceph-mon[168282]: 1.10 scrub ok 2023-09-23T20:57:21.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:57:21 smithi070 ceph-mon[168282]: 1.0 scrub starts 2023-09-23T20:57:21.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:57:21 smithi070 ceph-mon[168282]: 1.0 scrub ok 2023-09-23T20:57:21.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:57:21 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:57:21.880 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:57:21.921 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 43 16 67084 67068 97.4202 69.0625 0.0181364 0.0102593 2023-09-23T20:57:22.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:57:22 smithi070 ceph-mon[168282]: 1.12 scrub starts 2023-09-23T20:57:22.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:57:22 smithi070 ceph-mon[168282]: 1.12 scrub ok 2023-09-23T20:57:22.883 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:57:22 smithi026 ceph-mon[175151]: 1.12 scrub starts 2023-09-23T20:57:22.883 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:57:22 smithi026 ceph-mon[175151]: 1.12 scrub ok 2023-09-23T20:57:22.883 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:57:22 smithi026 ceph-mon[179688]: 1.12 scrub starts 2023-09-23T20:57:22.883 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:57:22 smithi026 ceph-mon[179688]: 1.12 scrub ok 2023-09-23T20:57:22.921 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 44 16 68308 68292 96.9448 76.5 0.0130523 0.01031 2023-09-23T20:57:23.166 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T20:57:23.433 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:57:23.441 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:57:23.441 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:57:23.542 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:57:23.653 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:57:23.780 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:57:23.784 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:57:23 smithi026 ceph-mon[179688]: 1.c scrub starts 2023-09-23T20:57:23.784 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:57:23 smithi026 ceph-mon[179688]: 1.c scrub ok 2023-09-23T20:57:23.784 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:57:23 smithi026 ceph-mon[179688]: pgmap v504: 109 pgs: 109 active+clean; 4.0 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 72 MiB/s wr, 1.15k op/s; 33 KiB/s, 0 objects/s recovering 2023-09-23T20:57:23.784 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:57:23 smithi026 ceph-mon[175151]: 1.c scrub starts 2023-09-23T20:57:23.785 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:57:23 smithi026 ceph-mon[175151]: 1.c scrub ok 2023-09-23T20:57:23.785 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:57:23 smithi026 ceph-mon[175151]: pgmap v504: 109 pgs: 109 active+clean; 4.0 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 72 MiB/s wr, 1.15k op/s; 33 KiB/s, 0 objects/s recovering 2023-09-23T20:57:23.785 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:57:23.786 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:57:23.786 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:57:23.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:57:23 smithi070 ceph-mon[168282]: 1.c scrub starts 2023-09-23T20:57:23.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:57:23 smithi070 ceph-mon[168282]: 1.c scrub ok 2023-09-23T20:57:23.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:57:23 smithi070 ceph-mon[168282]: pgmap v504: 109 pgs: 109 active+clean; 4.0 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 72 MiB/s wr, 1.15k op/s; 33 KiB/s, 0 objects/s recovering 2023-09-23T20:57:23.887 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:57:23.888 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:57:23.916 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:57:23.921 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 45 16 69331 69315 96.2116 63.9375 0.0314556 0.0103883 2023-09-23T20:57:23.956 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:57:23.960 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:57:23.960 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:57:24.061 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:57:24.096 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:57:24.141 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:57:24.146 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:57:24.147 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:57:24.248 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:57:24.303 INFO:teuthology.orchestra.run.smithi026.stdout:{ 2023-09-23T20:57:24.303 INFO:teuthology.orchestra.run.smithi026.stdout: "size": 20, 2023-09-23T20:57:24.303 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 600, 2023-09-23T20:57:24.303 INFO:teuthology.orchestra.run.smithi026.stdout: "ops": [ 2023-09-23T20:57:24.303 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.304 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34797.0:130474 6.d 6:baf3ed90:::benchmark_data_smithi026_223868_object29348:head [delete] snapc 0=[] RETRY=2 ondisk+retry+write+known_if_redirected+supports_pool_eio e137)", 2023-09-23T20:57:24.304 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T20:54:31.883928+0000", 2023-09-23T20:57:24.304 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 172.406419755, 2023-09-23T20:57:24.304 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.143112095, 2023-09-23T20:57:24.304 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T20:57:24.304 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "reached pg", 2023-09-23T20:57:24.304 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T20:57:24.304 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34797", 2023-09-23T20:57:24.304 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T20:57:24.304 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 130474 2023-09-23T20:57:24.304 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.305 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T20:57:24.305 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.305 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T20:57:24.305 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:31.883928+0000", 2023-09-23T20:57:24.305 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:57:24.305 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.305 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.305 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T20:57:24.305 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:31.883928+0000", 2023-09-23T20:57:24.305 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:57:24.305 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.305 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.306 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T20:57:24.306 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:31.883927+0000", 2023-09-23T20:57:24.306 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999981 2023-09-23T20:57:24.306 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.306 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.306 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T20:57:24.306 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:31.883932+0000", 2023-09-23T20:57:24.306 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.2610000000000001e-06 2023-09-23T20:57:24.306 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.306 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.306 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T20:57:24.306 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:31.883943+0000", 2023-09-23T20:57:24.307 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0861e-05 2023-09-23T20:57:24.307 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.307 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.307 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T20:57:24.307 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:31.883954+0000", 2023-09-23T20:57:24.307 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0686999999999999e-05 2023-09-23T20:57:24.307 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.307 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.307 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:57:24.307 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:31.884281+0000", 2023-09-23T20:57:24.307 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00032779599999999999 2023-09-23T20:57:24.307 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.307 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.308 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T20:57:24.308 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:31.884291+0000", 2023-09-23T20:57:24.308 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.3000000000000007e-06 2023-09-23T20:57:24.308 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.308 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.308 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:57:24.308 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:33.025141+0000", 2023-09-23T20:57:24.308 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1408508580000001 2023-09-23T20:57:24.308 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.308 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.308 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T20:57:24.308 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:33.027041+0000", 2023-09-23T20:57:24.309 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0018991030000000001 2023-09-23T20:57:24.309 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:57:24.309 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T20:57:24.309 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:57:24.309 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.309 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.309 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34797.0:130489 6.d 6:bb89226e:::benchmark_data_smithi026_223868_object29363:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e137)", 2023-09-23T20:57:24.309 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T20:54:31.883974+0000", 2023-09-23T20:57:24.309 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 172.406374105, 2023-09-23T20:57:24.309 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.161869933, 2023-09-23T20:57:24.309 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T20:57:24.309 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:57:24.310 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T20:57:24.310 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34797", 2023-09-23T20:57:24.310 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T20:57:24.310 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 130489 2023-09-23T20:57:24.310 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.310 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T20:57:24.310 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.310 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T20:57:24.310 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:31.883974+0000", 2023-09-23T20:57:24.310 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:57:24.310 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.310 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.311 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T20:57:24.311 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:31.883974+0000", 2023-09-23T20:57:24.311 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:57:24.311 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.311 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.311 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T20:57:24.311 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:31.883974+0000", 2023-09-23T20:57:24.311 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T20:57:24.311 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.311 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.311 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T20:57:24.311 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:31.883977+0000", 2023-09-23T20:57:24.312 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.348e-06 2023-09-23T20:57:24.312 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.312 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.312 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T20:57:24.312 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:31.883985+0000", 2023-09-23T20:57:24.312 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.191e-06 2023-09-23T20:57:24.312 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.312 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.312 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T20:57:24.312 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:31.883992+0000", 2023-09-23T20:57:24.312 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.7610000000000002e-06 2023-09-23T20:57:24.312 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.313 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.313 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:57:24.313 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:31.884354+0000", 2023-09-23T20:57:24.313 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00036240399999999998 2023-09-23T20:57:24.313 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.313 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.313 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T20:57:24.313 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:31.884363+0000", 2023-09-23T20:57:24.313 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.9989999999999997e-06 2023-09-23T20:57:24.313 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.313 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.313 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:57:24.313 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:33.027448+0000", 2023-09-23T20:57:24.314 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.143084325 2023-09-23T20:57:24.314 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.314 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.314 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T20:57:24.314 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:33.028112+0000", 2023-09-23T20:57:24.314 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00066470099999999996 2023-09-23T20:57:24.314 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.314 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.314 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 1,2,5", 2023-09-23T20:57:24.314 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:33.028273+0000", 2023-09-23T20:57:24.314 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00016043599999999999 2023-09-23T20:57:24.314 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.315 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.315 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T20:57:24.315 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:33.035710+0000", 2023-09-23T20:57:24.315 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0074372700000000002 2023-09-23T20:57:24.315 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.315 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.315 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:57:24.315 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:33.038722+0000", 2023-09-23T20:57:24.315 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0030117439999999998 2023-09-23T20:57:24.315 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.315 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.315 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:57:24.316 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:33.040863+0000", 2023-09-23T20:57:24.316 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0021409070000000001 2023-09-23T20:57:24.316 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.316 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.316 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:57:24.316 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:33.045669+0000", 2023-09-23T20:57:24.316 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0048059039999999997 2023-09-23T20:57:24.316 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.316 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.316 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T20:57:24.316 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:33.045728+0000", 2023-09-23T20:57:24.316 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.9129999999999998e-05 2023-09-23T20:57:24.316 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.317 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.317 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T20:57:24.317 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:33.045844+0000", 2023-09-23T20:57:24.317 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000116372 2023-09-23T20:57:24.317 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:57:24.317 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T20:57:24.317 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:57:24.317 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.317 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.317 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34797.0:130501 6.a 6:577795ba:::benchmark_data_smithi026_223868_object29375:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e137)", 2023-09-23T20:57:24.317 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T20:54:31.883997+0000", 2023-09-23T20:57:24.317 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 172.40635117799999, 2023-09-23T20:57:24.317 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1379962809999999, 2023-09-23T20:57:24.318 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T20:57:24.318 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:57:24.318 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T20:57:24.318 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34797", 2023-09-23T20:57:24.318 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T20:57:24.318 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 130501 2023-09-23T20:57:24.318 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.318 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T20:57:24.318 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.318 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T20:57:24.318 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:31.883997+0000", 2023-09-23T20:57:24.318 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:57:24.319 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.319 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.319 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T20:57:24.319 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:31.883997+0000", 2023-09-23T20:57:24.319 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:57:24.319 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.319 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.319 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T20:57:24.319 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:31.883997+0000", 2023-09-23T20:57:24.319 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T20:57:24.319 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.319 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.320 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T20:57:24.320 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:31.883999+0000", 2023-09-23T20:57:24.320 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.8660000000000002e-06 2023-09-23T20:57:24.320 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.320 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.320 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T20:57:24.320 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:31.884006+0000", 2023-09-23T20:57:24.320 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.5950000000000004e-06 2023-09-23T20:57:24.320 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.320 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.320 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T20:57:24.320 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:31.884013+0000", 2023-09-23T20:57:24.321 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.9179999999999997e-06 2023-09-23T20:57:24.321 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.321 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.321 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:57:24.321 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:31.884335+0000", 2023-09-23T20:57:24.321 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00032235800000000002 2023-09-23T20:57:24.321 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.321 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.321 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T20:57:24.321 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:31.884344+0000", 2023-09-23T20:57:24.321 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.7930000000000003e-06 2023-09-23T20:57:24.321 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.321 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.322 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:57:24.322 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:33.012175+0000", 2023-09-23T20:57:24.322 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.127831298 2023-09-23T20:57:24.322 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.322 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.322 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T20:57:24.322 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:33.014065+0000", 2023-09-23T20:57:24.322 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.001889305 2023-09-23T20:57:24.322 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.322 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.322 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 2,5,6", 2023-09-23T20:57:24.322 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:33.014209+0000", 2023-09-23T20:57:24.323 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00014406099999999999 2023-09-23T20:57:24.323 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.323 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.323 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T20:57:24.323 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:33.021710+0000", 2023-09-23T20:57:24.323 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0075012999999999998 2023-09-23T20:57:24.323 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.323 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.323 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:57:24.323 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:33.021748+0000", 2023-09-23T20:57:24.323 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.782e-05 2023-09-23T20:57:24.324 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.324 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.324 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:57:24.324 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:33.021781+0000", 2023-09-23T20:57:24.324 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.2969999999999998e-05 2023-09-23T20:57:24.324 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.324 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.324 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:57:24.324 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:33.021834+0000", 2023-09-23T20:57:24.324 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.3510000000000001e-05 2023-09-23T20:57:24.324 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.324 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.324 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T20:57:24.325 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:33.021888+0000", 2023-09-23T20:57:24.325 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.3115000000000002e-05 2023-09-23T20:57:24.325 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.325 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.325 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T20:57:24.325 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:33.021993+0000", 2023-09-23T20:57:24.325 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000105827 2023-09-23T20:57:24.325 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:57:24.325 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T20:57:24.325 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:57:24.325 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.325 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.326 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34797.0:130513 6.a 6:5baf817b:::benchmark_data_smithi026_223868_object29387:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e137)", 2023-09-23T20:57:24.326 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T20:54:31.884022+0000", 2023-09-23T20:57:24.326 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 172.406326592, 2023-09-23T20:57:24.326 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1549781690000001, 2023-09-23T20:57:24.326 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T20:57:24.326 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:57:24.326 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T20:57:24.326 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34797", 2023-09-23T20:57:24.326 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T20:57:24.326 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 130513 2023-09-23T20:57:24.326 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.326 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T20:57:24.327 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.327 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T20:57:24.327 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:31.884022+0000", 2023-09-23T20:57:24.327 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:57:24.327 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.327 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.327 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T20:57:24.327 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:31.884022+0000", 2023-09-23T20:57:24.327 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:57:24.327 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.327 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.327 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T20:57:24.327 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:31.884021+0000", 2023-09-23T20:57:24.328 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T20:57:24.328 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.328 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.328 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T20:57:24.328 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:31.884024+0000", 2023-09-23T20:57:24.328 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.8720000000000002e-06 2023-09-23T20:57:24.328 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.328 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.328 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T20:57:24.328 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:31.884031+0000", 2023-09-23T20:57:24.328 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.1670000000000002e-06 2023-09-23T20:57:24.328 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.329 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.329 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T20:57:24.329 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:31.884038+0000", 2023-09-23T20:57:24.329 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.2600000000000002e-06 2023-09-23T20:57:24.329 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.329 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.329 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:57:24.329 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:31.884370+0000", 2023-09-23T20:57:24.329 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00033257499999999999 2023-09-23T20:57:24.329 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.329 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.329 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T20:57:24.329 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:31.884378+0000", 2023-09-23T20:57:24.330 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.1899999999999995e-06 2023-09-23T20:57:24.330 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.330 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.330 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:57:24.330 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:33.018253+0000", 2023-09-23T20:57:24.330 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.133874724 2023-09-23T20:57:24.330 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.330 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.330 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T20:57:24.330 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:33.018408+0000", 2023-09-23T20:57:24.330 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00015456199999999999 2023-09-23T20:57:24.330 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.331 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.331 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 2,5,6", 2023-09-23T20:57:24.331 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:33.018566+0000", 2023-09-23T20:57:24.331 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00015848300000000001 2023-09-23T20:57:24.331 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.331 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.331 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T20:57:24.331 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:33.022005+0000", 2023-09-23T20:57:24.331 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0034394249999999999 2023-09-23T20:57:24.331 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.331 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.331 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:57:24.332 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:33.025311+0000", 2023-09-23T20:57:24.332 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0033057469999999999 2023-09-23T20:57:24.332 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.332 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.332 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:57:24.332 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:33.025347+0000", 2023-09-23T20:57:24.332 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.5887999999999999e-05 2023-09-23T20:57:24.332 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.332 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.332 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:57:24.332 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:33.038801+0000", 2023-09-23T20:57:24.332 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.013454173 2023-09-23T20:57:24.333 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.333 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.333 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T20:57:24.333 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:33.038860+0000", 2023-09-23T20:57:24.333 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.8795999999999997e-05 2023-09-23T20:57:24.333 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.333 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.333 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T20:57:24.333 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:33.039000+0000", 2023-09-23T20:57:24.333 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00013974899999999999 2023-09-23T20:57:24.333 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:57:24.333 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T20:57:24.333 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:57:24.334 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.334 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.334 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34797.0:194304 6.10 6:0f79980e:::benchmark_data_smithi026_223868_object93178:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e179)", 2023-09-23T20:57:24.334 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T20:56:00.936645+0000", 2023-09-23T20:57:24.334 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 83.353703052, 2023-09-23T20:57:24.334 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.3260833879999998, 2023-09-23T20:57:24.334 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T20:57:24.334 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "reached pg", 2023-09-23T20:57:24.334 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T20:57:24.334 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34797", 2023-09-23T20:57:24.334 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T20:57:24.335 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 194304 2023-09-23T20:57:24.335 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.335 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T20:57:24.335 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.335 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T20:57:24.335 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:00.936645+0000", 2023-09-23T20:57:24.335 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:57:24.335 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.335 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.335 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T20:57:24.335 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:00.936645+0000", 2023-09-23T20:57:24.336 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:57:24.336 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.336 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.336 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T20:57:24.336 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:00.936643+0000", 2023-09-23T20:57:24.336 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999976 2023-09-23T20:57:24.336 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.336 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.336 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T20:57:24.336 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:00.936653+0000", 2023-09-23T20:57:24.336 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.5619999999999993e-06 2023-09-23T20:57:24.336 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.336 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.336 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T20:57:24.337 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:00.936672+0000", 2023-09-23T20:57:24.337 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.9179000000000001e-05 2023-09-23T20:57:24.337 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.337 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.337 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T20:57:24.337 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:00.936680+0000", 2023-09-23T20:57:24.337 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.2009999999999999e-06 2023-09-23T20:57:24.337 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.337 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.337 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:57:24.337 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:00.936746+0000", 2023-09-23T20:57:24.337 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.6188999999999999e-05 2023-09-23T20:57:24.337 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.338 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.338 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T20:57:24.338 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:00.936882+0000", 2023-09-23T20:57:24.338 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000135795 2023-09-23T20:57:24.338 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.338 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.338 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 2,3", 2023-09-23T20:57:24.338 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:00.937429+0000", 2023-09-23T20:57:24.338 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00054749499999999995 2023-09-23T20:57:24.338 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.338 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.338 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T20:57:24.339 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:00.940535+0000", 2023-09-23T20:57:24.339 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0031056090000000001 2023-09-23T20:57:24.339 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.339 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.339 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:57:24.339 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:00.964290+0000", 2023-09-23T20:57:24.339 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.023754941000000002 2023-09-23T20:57:24.339 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.339 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.339 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T20:57:24.339 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:00.964301+0000", 2023-09-23T20:57:24.339 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1242e-05 2023-09-23T20:57:24.339 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.340 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.340 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:57:24.340 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.950552+0000", 2023-09-23T20:57:24.340 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.98625030400000002 2023-09-23T20:57:24.340 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.340 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.340 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T20:57:24.340 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.950566+0000", 2023-09-23T20:57:24.340 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.488e-05 2023-09-23T20:57:24.340 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.340 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.340 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:57:24.340 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.254843+0000", 2023-09-23T20:57:24.341 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.3042760799999999 2023-09-23T20:57:24.341 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.341 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.341 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for missing object", 2023-09-23T20:57:24.341 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.254977+0000", 2023-09-23T20:57:24.341 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00013490299999999999 2023-09-23T20:57:24.341 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.341 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.341 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:57:24.341 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.262666+0000", 2023-09-23T20:57:24.341 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0076884479999999996 2023-09-23T20:57:24.341 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.341 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.342 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T20:57:24.342 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.262729+0000", 2023-09-23T20:57:24.342 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.2722e-05 2023-09-23T20:57:24.342 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:57:24.342 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T20:57:24.342 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:57:24.342 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.342 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.342 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34797.0:194354 6.10 6:0825a30e:::benchmark_data_smithi026_223868_object93228:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e180)", 2023-09-23T20:57:24.342 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T20:56:01.013619+0000", 2023-09-23T20:57:24.342 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 83.276728855000002, 2023-09-23T20:57:24.342 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.3123553139999999, 2023-09-23T20:57:24.343 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T20:57:24.343 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:57:24.343 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T20:57:24.343 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34797", 2023-09-23T20:57:24.343 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T20:57:24.343 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 194354 2023-09-23T20:57:24.343 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.343 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T20:57:24.343 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.343 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T20:57:24.343 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.013619+0000", 2023-09-23T20:57:24.343 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:57:24.344 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.344 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.344 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T20:57:24.344 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.013619+0000", 2023-09-23T20:57:24.344 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:57:24.344 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.344 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.344 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T20:57:24.344 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.013618+0000", 2023-09-23T20:57:24.344 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T20:57:24.344 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.344 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.345 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T20:57:24.345 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.013625+0000", 2023-09-23T20:57:24.345 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.6410000000000002e-06 2023-09-23T20:57:24.345 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.345 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.345 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T20:57:24.345 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.013649+0000", 2023-09-23T20:57:24.345 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.3589000000000001e-05 2023-09-23T20:57:24.345 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.345 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.345 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T20:57:24.345 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.013661+0000", 2023-09-23T20:57:24.345 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1861e-05 2023-09-23T20:57:24.346 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.346 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.346 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:57:24.346 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.013731+0000", 2023-09-23T20:57:24.346 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.9720000000000003e-05 2023-09-23T20:57:24.346 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.346 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.346 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T20:57:24.346 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.013745+0000", 2023-09-23T20:57:24.346 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.4389000000000001e-05 2023-09-23T20:57:24.346 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.346 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.347 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:57:24.347 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.950782+0000", 2023-09-23T20:57:24.347 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.93703733700000003 2023-09-23T20:57:24.347 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.347 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.347 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T20:57:24.347 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.950792+0000", 2023-09-23T20:57:24.347 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.8649999999999996e-06 2023-09-23T20:57:24.347 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.347 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.347 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:57:24.347 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.255412+0000", 2023-09-23T20:57:24.347 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.3046194689999999 2023-09-23T20:57:24.347 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.348 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.348 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for degraded object", 2023-09-23T20:57:24.348 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.255792+0000", 2023-09-23T20:57:24.348 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00038020699999999997 2023-09-23T20:57:24.348 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.348 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.348 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:57:24.348 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.263497+0000", 2023-09-23T20:57:24.348 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0077052049999999997 2023-09-23T20:57:24.348 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.348 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.348 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T20:57:24.348 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.263609+0000", 2023-09-23T20:57:24.349 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000111499 2023-09-23T20:57:24.349 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.349 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.349 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 1,5", 2023-09-23T20:57:24.349 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.264170+0000", 2023-09-23T20:57:24.349 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00056130799999999995 2023-09-23T20:57:24.349 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.349 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.349 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T20:57:24.349 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.270530+0000", 2023-09-23T20:57:24.349 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0063601439999999999 2023-09-23T20:57:24.349 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.349 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.350 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:57:24.350 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.321515+0000", 2023-09-23T20:57:24.350 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.050984578000000003 2023-09-23T20:57:24.350 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.350 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.350 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:57:24.350 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.325525+0000", 2023-09-23T20:57:24.350 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0040102269999999999 2023-09-23T20:57:24.350 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.350 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.350 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T20:57:24.350 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.325562+0000", 2023-09-23T20:57:24.351 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.7540999999999999e-05 2023-09-23T20:57:24.351 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.351 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.351 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T20:57:24.351 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.325975+0000", 2023-09-23T20:57:24.351 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00041231800000000001 2023-09-23T20:57:24.351 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:57:24.351 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T20:57:24.351 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:57:24.351 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.351 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.351 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34797.0:194386 6.10 6:0e6bbf83:::benchmark_data_smithi026_223868_object93260:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e180)", 2023-09-23T20:57:24.351 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T20:56:01.046640+0000", 2023-09-23T20:57:24.352 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 83.243708075000001, 2023-09-23T20:57:24.352 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.2797770470000001, 2023-09-23T20:57:24.352 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T20:57:24.352 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:57:24.352 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T20:57:24.352 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34797", 2023-09-23T20:57:24.352 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T20:57:24.352 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 194386 2023-09-23T20:57:24.352 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.352 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T20:57:24.352 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.352 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T20:57:24.352 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.046640+0000", 2023-09-23T20:57:24.353 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:57:24.353 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.353 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.353 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T20:57:24.353 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.046640+0000", 2023-09-23T20:57:24.353 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:57:24.353 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.353 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.353 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T20:57:24.353 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.046639+0000", 2023-09-23T20:57:24.353 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T20:57:24.353 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.353 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.354 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T20:57:24.354 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.046647+0000", 2023-09-23T20:57:24.354 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.9270000000000005e-06 2023-09-23T20:57:24.354 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.354 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.354 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T20:57:24.354 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.046667+0000", 2023-09-23T20:57:24.354 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.0699e-05 2023-09-23T20:57:24.354 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.354 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.354 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T20:57:24.354 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.046678+0000", 2023-09-23T20:57:24.355 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0336000000000001e-05 2023-09-23T20:57:24.355 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.355 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.355 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:57:24.355 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.046740+0000", 2023-09-23T20:57:24.355 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.2643000000000003e-05 2023-09-23T20:57:24.355 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.355 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.355 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T20:57:24.355 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.046757+0000", 2023-09-23T20:57:24.355 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.6673000000000001e-05 2023-09-23T20:57:24.355 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.356 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.356 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:57:24.356 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.950820+0000", 2023-09-23T20:57:24.356 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.90406270899999996 2023-09-23T20:57:24.356 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.356 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.356 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T20:57:24.356 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.950827+0000", 2023-09-23T20:57:24.356 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.3799999999999996e-06 2023-09-23T20:57:24.356 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.356 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.356 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:57:24.356 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.255844+0000", 2023-09-23T20:57:24.357 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.305017112 2023-09-23T20:57:24.357 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.357 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.357 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for degraded object", 2023-09-23T20:57:24.357 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.256176+0000", 2023-09-23T20:57:24.357 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00033179999999999999 2023-09-23T20:57:24.357 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.357 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.357 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:57:24.357 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.267675+0000", 2023-09-23T20:57:24.357 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.011498759000000001 2023-09-23T20:57:24.357 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.357 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.358 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T20:57:24.358 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.267748+0000", 2023-09-23T20:57:24.358 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.3232000000000006e-05 2023-09-23T20:57:24.358 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.358 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.358 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 1,5", 2023-09-23T20:57:24.358 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.268150+0000", 2023-09-23T20:57:24.358 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000402113 2023-09-23T20:57:24.358 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.358 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.358 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T20:57:24.358 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.273531+0000", 2023-09-23T20:57:24.358 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.005380625 2023-09-23T20:57:24.359 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.359 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.359 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:57:24.359 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.321574+0000", 2023-09-23T20:57:24.359 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.048043172000000002 2023-09-23T20:57:24.359 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.359 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.359 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:57:24.359 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.326045+0000", 2023-09-23T20:57:24.359 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0044713649999999997 2023-09-23T20:57:24.359 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.359 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.359 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T20:57:24.360 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.326080+0000", 2023-09-23T20:57:24.360 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.4918e-05 2023-09-23T20:57:24.360 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.360 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.360 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T20:57:24.360 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.326417+0000", 2023-09-23T20:57:24.360 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00033714999999999998 2023-09-23T20:57:24.360 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:57:24.360 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T20:57:24.360 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:57:24.360 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.360 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.360 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34797.0:194397 6.10 6:0cf77e90:::benchmark_data_smithi026_223868_object93271:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e180)", 2023-09-23T20:57:24.360 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T20:56:01.071586+0000", 2023-09-23T20:57:24.361 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 83.218762557000005, 2023-09-23T20:57:24.361 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.255280086, 2023-09-23T20:57:24.361 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T20:57:24.361 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:57:24.361 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T20:57:24.361 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34797", 2023-09-23T20:57:24.361 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T20:57:24.361 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 194397 2023-09-23T20:57:24.361 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.361 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T20:57:24.361 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.361 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T20:57:24.361 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.071586+0000", 2023-09-23T20:57:24.362 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:57:24.362 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.362 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.362 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T20:57:24.362 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.071586+0000", 2023-09-23T20:57:24.362 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:57:24.362 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.362 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.362 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T20:57:24.362 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.071585+0000", 2023-09-23T20:57:24.362 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T20:57:24.362 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.362 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.363 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T20:57:24.363 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.071588+0000", 2023-09-23T20:57:24.363 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.1950000000000002e-06 2023-09-23T20:57:24.363 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.363 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.363 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T20:57:24.363 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.071593+0000", 2023-09-23T20:57:24.363 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.7830000000000004e-06 2023-09-23T20:57:24.363 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.363 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.363 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T20:57:24.363 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.071599+0000", 2023-09-23T20:57:24.364 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.8900000000000004e-06 2023-09-23T20:57:24.364 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.364 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.364 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:57:24.364 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.071767+0000", 2023-09-23T20:57:24.364 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00016748500000000001 2023-09-23T20:57:24.364 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.364 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.364 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T20:57:24.364 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.071779+0000", 2023-09-23T20:57:24.364 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.2464e-05 2023-09-23T20:57:24.364 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.364 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.365 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:57:24.365 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.950851+0000", 2023-09-23T20:57:24.365 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.87907165899999995 2023-09-23T20:57:24.365 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.365 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.365 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T20:57:24.365 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.950859+0000", 2023-09-23T20:57:24.365 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.2940000000000006e-06 2023-09-23T20:57:24.365 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.365 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.365 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:57:24.365 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.256209+0000", 2023-09-23T20:57:24.366 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.3053493780000001 2023-09-23T20:57:24.366 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.366 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.366 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for degraded object", 2023-09-23T20:57:24.366 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.256530+0000", 2023-09-23T20:57:24.366 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00032157699999999997 2023-09-23T20:57:24.366 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.366 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.366 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:57:24.366 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.270992+0000", 2023-09-23T20:57:24.366 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.014462299 2023-09-23T20:57:24.366 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.367 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.367 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T20:57:24.367 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.271067+0000", 2023-09-23T20:57:24.367 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.4938000000000007e-05 2023-09-23T20:57:24.367 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.367 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.367 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 1,5", 2023-09-23T20:57:24.367 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.271446+0000", 2023-09-23T20:57:24.367 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00037825400000000001 2023-09-23T20:57:24.367 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.367 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.367 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T20:57:24.367 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.277790+0000", 2023-09-23T20:57:24.368 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0063448840000000003 2023-09-23T20:57:24.368 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.368 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.368 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:57:24.368 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.321615+0000", 2023-09-23T20:57:24.368 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.043824241999999999 2023-09-23T20:57:24.368 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.368 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.368 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:57:24.368 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.326572+0000", 2023-09-23T20:57:24.368 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0049568980000000004 2023-09-23T20:57:24.368 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.368 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.369 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T20:57:24.369 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.326624+0000", 2023-09-23T20:57:24.369 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.2061000000000002e-05 2023-09-23T20:57:24.369 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.369 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.369 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T20:57:24.369 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.326866+0000", 2023-09-23T20:57:24.369 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00024209700000000001 2023-09-23T20:57:24.369 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:57:24.369 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T20:57:24.369 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:57:24.369 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.369 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.370 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34797.0:194462 6.10 6:0e070208:::benchmark_data_smithi026_223868_object93336:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e180)", 2023-09-23T20:57:24.370 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T20:56:01.136635+0000", 2023-09-23T20:57:24.370 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 83.153713224000001, 2023-09-23T20:57:24.370 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.1905668330000001, 2023-09-23T20:57:24.370 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T20:57:24.370 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:57:24.370 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T20:57:24.370 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34797", 2023-09-23T20:57:24.370 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T20:57:24.370 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 194462 2023-09-23T20:57:24.370 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.370 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T20:57:24.370 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.371 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T20:57:24.371 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.136635+0000", 2023-09-23T20:57:24.371 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:57:24.371 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.371 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.371 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T20:57:24.371 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.136635+0000", 2023-09-23T20:57:24.371 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:57:24.371 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.371 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.371 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T20:57:24.371 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.136633+0000", 2023-09-23T20:57:24.371 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999981 2023-09-23T20:57:24.372 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.372 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.372 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T20:57:24.372 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.136640+0000", 2023-09-23T20:57:24.372 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.5989999999999998e-06 2023-09-23T20:57:24.372 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.372 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.372 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T20:57:24.372 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.136656+0000", 2023-09-23T20:57:24.372 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.6595e-05 2023-09-23T20:57:24.372 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.372 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.372 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T20:57:24.373 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.136665+0000", 2023-09-23T20:57:24.373 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.1540000000000002e-06 2023-09-23T20:57:24.373 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.373 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.373 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:57:24.373 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.136719+0000", 2023-09-23T20:57:24.373 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.4160000000000003e-05 2023-09-23T20:57:24.373 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.373 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.373 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T20:57:24.373 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.136726+0000", 2023-09-23T20:57:24.373 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.4429999999999997e-06 2023-09-23T20:57:24.373 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.374 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.374 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:57:24.374 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.950961+0000", 2023-09-23T20:57:24.374 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.81423474299999998 2023-09-23T20:57:24.374 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.374 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.374 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T20:57:24.374 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.950970+0000", 2023-09-23T20:57:24.374 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.2690000000000005e-06 2023-09-23T20:57:24.374 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.374 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.374 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:57:24.375 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.256671+0000", 2023-09-23T20:57:24.375 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.305701193 2023-09-23T20:57:24.375 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.375 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.375 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for degraded object", 2023-09-23T20:57:24.375 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.257010+0000", 2023-09-23T20:57:24.375 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00033869800000000002 2023-09-23T20:57:24.375 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.375 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.375 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:57:24.375 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.274991+0000", 2023-09-23T20:57:24.375 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.017981115999999998 2023-09-23T20:57:24.375 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.376 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.376 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T20:57:24.376 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.275118+0000", 2023-09-23T20:57:24.376 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00012721400000000001 2023-09-23T20:57:24.376 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.376 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.376 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 1,5", 2023-09-23T20:57:24.376 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.275667+0000", 2023-09-23T20:57:24.376 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00054888699999999996 2023-09-23T20:57:24.376 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.376 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.376 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T20:57:24.376 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.283025+0000", 2023-09-23T20:57:24.376 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0073580700000000004 2023-09-23T20:57:24.377 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.377 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.377 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:57:24.377 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.321667+0000", 2023-09-23T20:57:24.377 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.038641406000000003 2023-09-23T20:57:24.377 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.377 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.377 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:57:24.377 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.326925+0000", 2023-09-23T20:57:24.377 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0052579829999999999 2023-09-23T20:57:24.377 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.377 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.377 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T20:57:24.378 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.326974+0000", 2023-09-23T20:57:24.378 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.9079999999999998e-05 2023-09-23T20:57:24.378 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.378 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.378 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T20:57:24.378 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.327202+0000", 2023-09-23T20:57:24.378 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000228063 2023-09-23T20:57:24.378 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:57:24.378 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T20:57:24.378 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:57:24.378 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.378 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.379 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34797.0:194568 6.10 6:0af43279:::benchmark_data_smithi026_223868_object93442:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e180)", 2023-09-23T20:57:24.379 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T20:56:01.267334+0000", 2023-09-23T20:57:24.379 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 83.023014564999997, 2023-09-23T20:57:24.379 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.0602084349999998, 2023-09-23T20:57:24.379 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T20:57:24.379 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:57:24.379 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T20:57:24.379 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34797", 2023-09-23T20:57:24.379 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T20:57:24.379 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 194568 2023-09-23T20:57:24.379 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.379 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T20:57:24.379 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.380 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T20:57:24.380 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.267334+0000", 2023-09-23T20:57:24.380 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:57:24.380 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.380 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.380 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T20:57:24.380 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.267334+0000", 2023-09-23T20:57:24.380 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:57:24.380 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.380 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.380 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T20:57:24.380 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.267332+0000", 2023-09-23T20:57:24.381 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T20:57:24.381 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.381 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.381 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T20:57:24.381 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.267340+0000", 2023-09-23T20:57:24.381 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.6410000000000002e-06 2023-09-23T20:57:24.381 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.381 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.381 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T20:57:24.381 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.267356+0000", 2023-09-23T20:57:24.381 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.6409000000000002e-05 2023-09-23T20:57:24.381 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.382 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.382 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T20:57:24.382 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.267364+0000", 2023-09-23T20:57:24.382 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.6580000000000002e-06 2023-09-23T20:57:24.382 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.382 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.382 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:57:24.382 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.267423+0000", 2023-09-23T20:57:24.382 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.9392000000000003e-05 2023-09-23T20:57:24.382 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.382 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.382 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T20:57:24.383 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.267435+0000", 2023-09-23T20:57:24.383 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1653e-05 2023-09-23T20:57:24.383 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.383 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.383 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:57:24.383 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.950996+0000", 2023-09-23T20:57:24.383 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.68356135900000004 2023-09-23T20:57:24.383 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.383 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.383 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T20:57:24.383 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.951005+0000", 2023-09-23T20:57:24.383 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.5660000000000007e-06 2023-09-23T20:57:24.384 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.384 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.384 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:57:24.384 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.257054+0000", 2023-09-23T20:57:24.384 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.306049228 2023-09-23T20:57:24.384 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.384 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.384 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for degraded object", 2023-09-23T20:57:24.384 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.257353+0000", 2023-09-23T20:57:24.384 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00029845000000000002 2023-09-23T20:57:24.384 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.384 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.385 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:57:24.385 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.278511+0000", 2023-09-23T20:57:24.385 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.021158713999999999 2023-09-23T20:57:24.385 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.385 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.385 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T20:57:24.385 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.278633+0000", 2023-09-23T20:57:24.385 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00012163999999999999 2023-09-23T20:57:24.385 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.385 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.385 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 1,5", 2023-09-23T20:57:24.385 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.279079+0000", 2023-09-23T20:57:24.385 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000445807 2023-09-23T20:57:24.386 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.386 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.386 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T20:57:24.386 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.286413+0000", 2023-09-23T20:57:24.386 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0073338509999999997 2023-09-23T20:57:24.386 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.386 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.386 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:57:24.386 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.321694+0000", 2023-09-23T20:57:24.386 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.035281333999999998 2023-09-23T20:57:24.386 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.386 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.386 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:57:24.387 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.327259+0000", 2023-09-23T20:57:24.387 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0055648479999999998 2023-09-23T20:57:24.387 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.387 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.387 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T20:57:24.387 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.327308+0000", 2023-09-23T20:57:24.387 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.9527999999999997e-05 2023-09-23T20:57:24.387 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.387 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.387 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T20:57:24.387 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.327542+0000", 2023-09-23T20:57:24.387 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00023375499999999999 2023-09-23T20:57:24.387 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:57:24.388 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T20:57:24.388 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:57:24.388 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.388 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.388 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34797.0:194611 6.10 6:0de287e4:::benchmark_data_smithi026_223868_object93485:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e180)", 2023-09-23T20:57:24.388 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T20:56:01.323196+0000", 2023-09-23T20:57:24.388 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 82.967151951999995, 2023-09-23T20:57:24.388 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.0047136079999999, 2023-09-23T20:57:24.388 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T20:57:24.388 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:57:24.388 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T20:57:24.388 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34797", 2023-09-23T20:57:24.389 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T20:57:24.389 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 194611 2023-09-23T20:57:24.389 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.389 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T20:57:24.389 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.389 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T20:57:24.389 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.323196+0000", 2023-09-23T20:57:24.389 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:57:24.389 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.389 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.389 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T20:57:24.389 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.323196+0000", 2023-09-23T20:57:24.390 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:57:24.390 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.390 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.390 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T20:57:24.390 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.323196+0000", 2023-09-23T20:57:24.390 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T20:57:24.390 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.390 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.390 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T20:57:24.390 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.323199+0000", 2023-09-23T20:57:24.390 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.4470000000000002e-06 2023-09-23T20:57:24.390 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.390 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.391 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T20:57:24.391 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.323209+0000", 2023-09-23T20:57:24.391 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.4700000000000008e-06 2023-09-23T20:57:24.391 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.391 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.391 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T20:57:24.391 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.323216+0000", 2023-09-23T20:57:24.391 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.3900000000000004e-06 2023-09-23T20:57:24.391 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.391 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.391 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:57:24.391 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.323262+0000", 2023-09-23T20:57:24.391 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.5785000000000003e-05 2023-09-23T20:57:24.391 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.392 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.392 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T20:57:24.392 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.323273+0000", 2023-09-23T20:57:24.392 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0888e-05 2023-09-23T20:57:24.392 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.392 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.392 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:57:24.392 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.951032+0000", 2023-09-23T20:57:24.392 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.62775908800000002 2023-09-23T20:57:24.392 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.392 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.392 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T20:57:24.392 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.951039+0000", 2023-09-23T20:57:24.393 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.3100000000000003e-06 2023-09-23T20:57:24.393 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.393 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.393 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:57:24.393 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.257402+0000", 2023-09-23T20:57:24.393 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.306362724 2023-09-23T20:57:24.393 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.393 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.393 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for degraded object", 2023-09-23T20:57:24.393 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.257769+0000", 2023-09-23T20:57:24.393 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00036691000000000002 2023-09-23T20:57:24.393 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.393 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.393 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:57:24.394 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.283087+0000", 2023-09-23T20:57:24.394 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.025317925000000002 2023-09-23T20:57:24.394 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.394 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.394 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T20:57:24.394 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.283213+0000", 2023-09-23T20:57:24.394 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000126773 2023-09-23T20:57:24.394 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.394 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.394 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 1,5", 2023-09-23T20:57:24.394 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.283761+0000", 2023-09-23T20:57:24.394 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00054742500000000004 2023-09-23T20:57:24.395 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.395 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.395 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T20:57:24.395 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.290057+0000", 2023-09-23T20:57:24.395 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0062964869999999999 2023-09-23T20:57:24.395 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.395 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.395 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:57:24.395 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.321745+0000", 2023-09-23T20:57:24.395 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.031687651999999997 2023-09-23T20:57:24.395 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.395 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.395 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:57:24.396 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.327648+0000", 2023-09-23T20:57:24.396 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0059028600000000002 2023-09-23T20:57:24.396 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.396 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.396 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T20:57:24.396 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.327694+0000", 2023-09-23T20:57:24.396 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.6081000000000001e-05 2023-09-23T20:57:24.396 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.396 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.396 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T20:57:24.396 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.327910+0000", 2023-09-23T20:57:24.396 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00021589499999999999 2023-09-23T20:57:24.396 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:57:24.397 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T20:57:24.397 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:57:24.397 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.397 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.397 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34797.0:194684 6.10 6:0c8ebf5c:::benchmark_data_smithi026_223868_object93558:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e180)", 2023-09-23T20:57:24.397 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T20:56:01.410674+0000", 2023-09-23T20:57:24.397 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 82.879673760000003, 2023-09-23T20:57:24.397 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.9175628630000001, 2023-09-23T20:57:24.397 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T20:57:24.397 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:57:24.397 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T20:57:24.397 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34797", 2023-09-23T20:57:24.397 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T20:57:24.397 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 194684 2023-09-23T20:57:24.398 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.398 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T20:57:24.398 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.398 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T20:57:24.398 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.410674+0000", 2023-09-23T20:57:24.398 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:57:24.398 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.398 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.398 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T20:57:24.398 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.410674+0000", 2023-09-23T20:57:24.398 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:57:24.398 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.399 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.399 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T20:57:24.399 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.410674+0000", 2023-09-23T20:57:24.399 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T20:57:24.399 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.399 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.399 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T20:57:24.399 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.410677+0000", 2023-09-23T20:57:24.399 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.9579999999999999e-06 2023-09-23T20:57:24.399 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.399 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.399 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T20:57:24.399 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.410692+0000", 2023-09-23T20:57:24.399 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.539e-05 2023-09-23T20:57:24.400 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.400 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.400 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T20:57:24.400 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.410698+0000", 2023-09-23T20:57:24.400 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.2830000000000001e-06 2023-09-23T20:57:24.400 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.400 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.400 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:57:24.400 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.410771+0000", 2023-09-23T20:57:24.400 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.2899999999999997e-05 2023-09-23T20:57:24.400 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.400 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.401 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T20:57:24.401 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.410778+0000", 2023-09-23T20:57:24.401 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.4200000000000001e-06 2023-09-23T20:57:24.401 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.401 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.401 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:57:24.401 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.951065+0000", 2023-09-23T20:57:24.401 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.54028672700000002 2023-09-23T20:57:24.401 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.401 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.401 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T20:57:24.402 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.951073+0000", 2023-09-23T20:57:24.402 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.551e-06 2023-09-23T20:57:24.402 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.402 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.402 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:57:24.402 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.257812+0000", 2023-09-23T20:57:24.402 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.3067387959999999 2023-09-23T20:57:24.402 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.402 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.402 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for degraded object", 2023-09-23T20:57:24.402 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.258109+0000", 2023-09-23T20:57:24.402 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00029674299999999997 2023-09-23T20:57:24.403 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.403 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.403 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:57:24.403 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.286926+0000", 2023-09-23T20:57:24.403 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.028817470000000001 2023-09-23T20:57:24.403 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.403 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.403 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T20:57:24.403 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.286999+0000", 2023-09-23T20:57:24.403 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.3141999999999999e-05 2023-09-23T20:57:24.403 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.403 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.403 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 1,5", 2023-09-23T20:57:24.404 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.287434+0000", 2023-09-23T20:57:24.404 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00043483299999999998 2023-09-23T20:57:24.404 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.404 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.404 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T20:57:24.404 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.294642+0000", 2023-09-23T20:57:24.404 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0072078960000000001 2023-09-23T20:57:24.404 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.404 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.404 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:57:24.404 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.321776+0000", 2023-09-23T20:57:24.405 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.027134098999999998 2023-09-23T20:57:24.405 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.405 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.405 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:57:24.405 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.327964+0000", 2023-09-23T20:57:24.405 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0061875960000000001 2023-09-23T20:57:24.405 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.405 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.405 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T20:57:24.405 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.328014+0000", 2023-09-23T20:57:24.405 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.007e-05 2023-09-23T20:57:24.405 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.406 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.406 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T20:57:24.406 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.328237+0000", 2023-09-23T20:57:24.406 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000223391 2023-09-23T20:57:24.406 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:57:24.406 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T20:57:24.406 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:57:24.406 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.406 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.406 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34797.0:194713 6.10 6:0a64bb83:::benchmark_data_smithi026_223868_object93587:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e180)", 2023-09-23T20:57:24.406 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T20:56:01.445241+0000", 2023-09-23T20:57:24.406 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 82.845107666999994, 2023-09-23T20:57:24.407 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.8833663199999999, 2023-09-23T20:57:24.407 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T20:57:24.407 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:57:24.407 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T20:57:24.407 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34797", 2023-09-23T20:57:24.407 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T20:57:24.407 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 194713 2023-09-23T20:57:24.407 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.407 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T20:57:24.407 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.407 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T20:57:24.407 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.445241+0000", 2023-09-23T20:57:24.407 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:57:24.407 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.408 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.408 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T20:57:24.408 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.445241+0000", 2023-09-23T20:57:24.408 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:57:24.408 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.408 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.408 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T20:57:24.408 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.445239+0000", 2023-09-23T20:57:24.408 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T20:57:24.408 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.408 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.408 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T20:57:24.408 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.445246+0000", 2023-09-23T20:57:24.409 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.6950000000000003e-06 2023-09-23T20:57:24.409 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.409 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.409 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T20:57:24.409 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.445262+0000", 2023-09-23T20:57:24.409 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.5654999999999998e-05 2023-09-23T20:57:24.409 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.409 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.409 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T20:57:24.409 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.445271+0000", 2023-09-23T20:57:24.409 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.679e-06 2023-09-23T20:57:24.409 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.410 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.410 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:57:24.410 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.445322+0000", 2023-09-23T20:57:24.410 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.0516000000000002e-05 2023-09-23T20:57:24.410 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.410 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.410 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T20:57:24.410 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.445332+0000", 2023-09-23T20:57:24.410 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.9699999999999994e-06 2023-09-23T20:57:24.410 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.410 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.410 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:57:24.410 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.951102+0000", 2023-09-23T20:57:24.411 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.50577048800000002 2023-09-23T20:57:24.411 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.411 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.411 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T20:57:24.411 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.951110+0000", 2023-09-23T20:57:24.411 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.208e-06 2023-09-23T20:57:24.411 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.411 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.411 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:57:24.411 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.258157+0000", 2023-09-23T20:57:24.411 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.3070464900000001 2023-09-23T20:57:24.411 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.412 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.412 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for degraded object", 2023-09-23T20:57:24.412 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.258517+0000", 2023-09-23T20:57:24.412 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00035973599999999999 2023-09-23T20:57:24.412 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.412 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.412 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:57:24.412 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.291101+0000", 2023-09-23T20:57:24.412 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.032584136999999999 2023-09-23T20:57:24.412 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.412 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.412 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T20:57:24.413 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.291239+0000", 2023-09-23T20:57:24.413 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000137761 2023-09-23T20:57:24.413 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.413 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.413 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 1,5", 2023-09-23T20:57:24.413 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.291789+0000", 2023-09-23T20:57:24.413 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00055038199999999998 2023-09-23T20:57:24.413 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.413 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.413 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T20:57:24.413 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.295524+0000", 2023-09-23T20:57:24.413 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0037345519999999999 2023-09-23T20:57:24.413 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.414 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.414 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:57:24.414 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.321831+0000", 2023-09-23T20:57:24.414 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.026307456 2023-09-23T20:57:24.414 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.414 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.414 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:57:24.414 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.328336+0000", 2023-09-23T20:57:24.414 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0065046009999999996 2023-09-23T20:57:24.414 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.414 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.415 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T20:57:24.415 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.328382+0000", 2023-09-23T20:57:24.415 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.617e-05 2023-09-23T20:57:24.415 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.415 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.415 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T20:57:24.415 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.328607+0000", 2023-09-23T20:57:24.415 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00022515100000000001 2023-09-23T20:57:24.415 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:57:24.415 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T20:57:24.415 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:57:24.415 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.415 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.416 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34797.0:194727 6.10 6:0b60e6f5:::benchmark_data_smithi026_223868_object93601:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e180)", 2023-09-23T20:57:24.416 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T20:56:01.462928+0000", 2023-09-23T20:57:24.416 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 82.827419937000002, 2023-09-23T20:57:24.416 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.866058437, 2023-09-23T20:57:24.416 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T20:57:24.416 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:57:24.416 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T20:57:24.416 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34797", 2023-09-23T20:57:24.416 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T20:57:24.416 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 194727 2023-09-23T20:57:24.416 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.416 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T20:57:24.416 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.416 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T20:57:24.417 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.462928+0000", 2023-09-23T20:57:24.417 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:57:24.417 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.417 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.417 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T20:57:24.417 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.462928+0000", 2023-09-23T20:57:24.417 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:57:24.417 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.417 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.417 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T20:57:24.417 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.462927+0000", 2023-09-23T20:57:24.417 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.999999 2023-09-23T20:57:24.418 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.418 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.418 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T20:57:24.418 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.462933+0000", 2023-09-23T20:57:24.418 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.3730000000000001e-06 2023-09-23T20:57:24.418 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.418 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.418 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T20:57:24.418 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.462946+0000", 2023-09-23T20:57:24.418 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.3747000000000001e-05 2023-09-23T20:57:24.418 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.418 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.419 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T20:57:24.419 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.462955+0000", 2023-09-23T20:57:24.419 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.0000000000000002e-06 2023-09-23T20:57:24.419 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.419 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.419 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:57:24.419 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.463029+0000", 2023-09-23T20:57:24.419 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.3559000000000005e-05 2023-09-23T20:57:24.419 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.419 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.419 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T20:57:24.419 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.463042+0000", 2023-09-23T20:57:24.419 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.2661e-05 2023-09-23T20:57:24.420 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.420 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.420 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:57:24.420 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.951135+0000", 2023-09-23T20:57:24.420 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.48809338699999999 2023-09-23T20:57:24.420 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.420 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.420 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T20:57:24.420 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.951143+0000", 2023-09-23T20:57:24.420 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.7319999999999998e-06 2023-09-23T20:57:24.420 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.420 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.420 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:57:24.421 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.258558+0000", 2023-09-23T20:57:24.421 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.307415357 2023-09-23T20:57:24.421 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.421 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.421 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for degraded object", 2023-09-23T20:57:24.421 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.258901+0000", 2023-09-23T20:57:24.421 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000342515 2023-09-23T20:57:24.421 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.421 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.421 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:57:24.421 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.297191+0000", 2023-09-23T20:57:24.422 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.038290826 2023-09-23T20:57:24.422 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.422 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.422 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T20:57:24.422 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.297316+0000", 2023-09-23T20:57:24.422 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00012508899999999999 2023-09-23T20:57:24.422 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.422 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.422 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 1,5", 2023-09-23T20:57:24.422 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.297920+0000", 2023-09-23T20:57:24.422 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00060379499999999996 2023-09-23T20:57:24.422 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.423 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.423 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T20:57:24.423 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.304770+0000", 2023-09-23T20:57:24.423 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0068497499999999999 2023-09-23T20:57:24.423 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.423 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.423 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:57:24.423 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.325484+0000", 2023-09-23T20:57:24.423 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.020714521999999999 2023-09-23T20:57:24.423 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.423 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.423 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:57:24.424 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.328690+0000", 2023-09-23T20:57:24.424 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.003205875 2023-09-23T20:57:24.424 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.424 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.424 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T20:57:24.424 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.328740+0000", 2023-09-23T20:57:24.424 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.9492e-05 2023-09-23T20:57:24.424 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.424 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.424 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T20:57:24.424 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.328987+0000", 2023-09-23T20:57:24.424 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00024687500000000002 2023-09-23T20:57:24.425 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:57:24.425 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T20:57:24.425 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:57:24.425 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.425 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.425 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34797.0:194750 6.10 6:091e1945:::benchmark_data_smithi026_223868_object93624:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e180)", 2023-09-23T20:57:24.425 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T20:56:01.496055+0000", 2023-09-23T20:57:24.425 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 82.794293206999996, 2023-09-23T20:57:24.425 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.8333552829999999, 2023-09-23T20:57:24.425 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T20:57:24.425 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:57:24.425 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T20:57:24.426 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34797", 2023-09-23T20:57:24.426 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T20:57:24.426 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 194750 2023-09-23T20:57:24.426 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.426 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T20:57:24.426 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.426 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T20:57:24.426 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.496055+0000", 2023-09-23T20:57:24.426 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:57:24.426 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.426 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.426 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T20:57:24.427 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.496055+0000", 2023-09-23T20:57:24.427 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:57:24.427 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.427 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.427 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T20:57:24.427 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.496054+0000", 2023-09-23T20:57:24.427 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T20:57:24.427 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.427 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.427 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T20:57:24.427 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.496061+0000", 2023-09-23T20:57:24.427 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.5009999999999998e-06 2023-09-23T20:57:24.427 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.428 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.428 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T20:57:24.428 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.496078+0000", 2023-09-23T20:57:24.428 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.6917000000000001e-05 2023-09-23T20:57:24.428 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.428 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.428 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T20:57:24.428 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.496085+0000", 2023-09-23T20:57:24.428 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.1729999999999998e-06 2023-09-23T20:57:24.428 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.428 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.428 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:57:24.428 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.496150+0000", 2023-09-23T20:57:24.428 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.5225e-05 2023-09-23T20:57:24.429 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.429 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.429 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T20:57:24.429 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.496163+0000", 2023-09-23T20:57:24.429 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.2187e-05 2023-09-23T20:57:24.429 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.429 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.429 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:57:24.429 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.951168+0000", 2023-09-23T20:57:24.429 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.455005037 2023-09-23T20:57:24.429 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.429 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.429 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T20:57:24.430 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.951176+0000", 2023-09-23T20:57:24.430 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.9750000000000007e-06 2023-09-23T20:57:24.430 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.430 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.430 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:57:24.430 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.258954+0000", 2023-09-23T20:57:24.430 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.3077780589999999 2023-09-23T20:57:24.430 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.430 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.430 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for degraded object", 2023-09-23T20:57:24.430 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.259370+0000", 2023-09-23T20:57:24.430 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000416632 2023-09-23T20:57:24.430 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.430 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.431 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:57:24.431 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.301427+0000", 2023-09-23T20:57:24.431 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.042056291000000003 2023-09-23T20:57:24.431 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.431 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.431 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T20:57:24.431 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.301568+0000", 2023-09-23T20:57:24.431 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00014118900000000001 2023-09-23T20:57:24.431 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.431 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.431 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 1,5", 2023-09-23T20:57:24.431 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.302113+0000", 2023-09-23T20:57:24.431 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00054519200000000001 2023-09-23T20:57:24.432 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.432 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.432 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T20:57:24.432 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.309443+0000", 2023-09-23T20:57:24.432 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0073298130000000001 2023-09-23T20:57:24.432 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.432 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.432 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:57:24.432 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.326494+0000", 2023-09-23T20:57:24.432 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.017051667999999999 2023-09-23T20:57:24.432 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.432 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.432 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:57:24.432 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.329098+0000", 2023-09-23T20:57:24.433 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0026040550000000001 2023-09-23T20:57:24.433 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.433 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.433 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T20:57:24.433 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.329155+0000", 2023-09-23T20:57:24.433 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.6276000000000001e-05 2023-09-23T20:57:24.433 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.433 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.433 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T20:57:24.433 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.329410+0000", 2023-09-23T20:57:24.433 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00025554400000000002 2023-09-23T20:57:24.433 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:57:24.433 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T20:57:24.434 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:57:24.434 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.434 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.434 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34797.0:194898 6.10 6:0ab6bac4:::benchmark_data_smithi026_223868_object93772:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e180)", 2023-09-23T20:57:24.434 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T20:56:01.737608+0000", 2023-09-23T20:57:24.434 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 82.552740494000005, 2023-09-23T20:57:24.434 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.605071608, 2023-09-23T20:57:24.434 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T20:57:24.434 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:57:24.434 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T20:57:24.434 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34797", 2023-09-23T20:57:24.434 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T20:57:24.434 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 194898 2023-09-23T20:57:24.435 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.435 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T20:57:24.435 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.435 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T20:57:24.435 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.737608+0000", 2023-09-23T20:57:24.435 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:57:24.435 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.435 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.435 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T20:57:24.435 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.737608+0000", 2023-09-23T20:57:24.435 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:57:24.435 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.436 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.436 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T20:57:24.436 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.737606+0000", 2023-09-23T20:57:24.436 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999981 2023-09-23T20:57:24.436 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.436 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.436 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T20:57:24.436 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.737613+0000", 2023-09-23T20:57:24.436 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.5390000000000002e-06 2023-09-23T20:57:24.436 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.436 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.436 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T20:57:24.437 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.737626+0000", 2023-09-23T20:57:24.437 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.2232000000000001e-05 2023-09-23T20:57:24.437 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.437 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.437 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T20:57:24.437 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.737634+0000", 2023-09-23T20:57:24.437 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.0960000000000001e-06 2023-09-23T20:57:24.437 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.437 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.437 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:57:24.437 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.737715+0000", 2023-09-23T20:57:24.437 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.1599000000000003e-05 2023-09-23T20:57:24.437 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.437 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.438 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T20:57:24.438 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.737730+0000", 2023-09-23T20:57:24.438 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.4363e-05 2023-09-23T20:57:24.438 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.438 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.438 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:57:24.438 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.951200+0000", 2023-09-23T20:57:24.438 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.21347070400000001 2023-09-23T20:57:24.438 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.438 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.438 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T20:57:24.438 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.951208+0000", 2023-09-23T20:57:24.439 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.5599999999999996e-06 2023-09-23T20:57:24.439 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.439 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.439 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:57:24.439 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.259413+0000", 2023-09-23T20:57:24.439 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.3082054949999999 2023-09-23T20:57:24.439 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.439 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.439 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for degraded object", 2023-09-23T20:57:24.439 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.259853+0000", 2023-09-23T20:57:24.439 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00043965399999999999 2023-09-23T20:57:24.439 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.440 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.440 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:57:24.440 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.305598+0000", 2023-09-23T20:57:24.440 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.045744765999999999 2023-09-23T20:57:24.440 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.440 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.440 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T20:57:24.440 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.305723+0000", 2023-09-23T20:57:24.440 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000124807 2023-09-23T20:57:24.440 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.440 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.440 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 1,5", 2023-09-23T20:57:24.441 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.306293+0000", 2023-09-23T20:57:24.441 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00057089399999999996 2023-09-23T20:57:24.441 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.441 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.441 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T20:57:24.441 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.317574+0000", 2023-09-23T20:57:24.441 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.011280434000000001 2023-09-23T20:57:24.441 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.441 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.441 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:57:24.441 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.327600+0000", 2023-09-23T20:57:24.441 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.010026037999999999 2023-09-23T20:57:24.441 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.441 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.442 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:57:24.442 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.342119+0000", 2023-09-23T20:57:24.442 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.014519139 2023-09-23T20:57:24.442 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.442 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.442 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T20:57:24.442 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.342171+0000", 2023-09-23T20:57:24.442 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.1925000000000002e-05 2023-09-23T20:57:24.442 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.442 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.442 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T20:57:24.442 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.342679+0000", 2023-09-23T20:57:24.442 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00050835799999999999 2023-09-23T20:57:24.442 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:57:24.443 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T20:57:24.443 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:57:24.443 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.443 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.443 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34797.0:194908 6.10 6:0f877dc5:::benchmark_data_smithi026_223868_object93782:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e180)", 2023-09-23T20:57:24.443 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T20:56:01.753586+0000", 2023-09-23T20:57:24.443 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 82.536762288000006, 2023-09-23T20:57:24.443 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.5894383169999999, 2023-09-23T20:57:24.443 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T20:57:24.443 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:57:24.443 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T20:57:24.443 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34797", 2023-09-23T20:57:24.443 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T20:57:24.444 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 194908 2023-09-23T20:57:24.444 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.444 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T20:57:24.444 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.444 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T20:57:24.444 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.753586+0000", 2023-09-23T20:57:24.444 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:57:24.444 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.444 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.444 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T20:57:24.444 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.753586+0000", 2023-09-23T20:57:24.444 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:57:24.445 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.445 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.445 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T20:57:24.445 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.753584+0000", 2023-09-23T20:57:24.445 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T20:57:24.445 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.445 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.445 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T20:57:24.445 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.753592+0000", 2023-09-23T20:57:24.445 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.5349999999999999e-06 2023-09-23T20:57:24.445 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.445 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.445 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T20:57:24.446 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.753609+0000", 2023-09-23T20:57:24.446 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.696e-05 2023-09-23T20:57:24.446 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.446 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.446 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T20:57:24.446 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.753617+0000", 2023-09-23T20:57:24.446 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.2009999999999999e-06 2023-09-23T20:57:24.446 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.446 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.446 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:57:24.446 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.753675+0000", 2023-09-23T20:57:24.446 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.7856999999999998e-05 2023-09-23T20:57:24.447 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.447 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.447 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T20:57:24.447 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.753690+0000", 2023-09-23T20:57:24.447 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.5528000000000001e-05 2023-09-23T20:57:24.447 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.448 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.448 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:57:24.448 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.951234+0000", 2023-09-23T20:57:24.448 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.19754403500000001 2023-09-23T20:57:24.448 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.448 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.448 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T20:57:24.448 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.951242+0000", 2023-09-23T20:57:24.448 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.9330000000000001e-06 2023-09-23T20:57:24.449 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.449 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.449 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:57:24.449 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.259898+0000", 2023-09-23T20:57:24.449 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.308655608 2023-09-23T20:57:24.449 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.449 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.449 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for degraded object", 2023-09-23T20:57:24.449 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.260281+0000", 2023-09-23T20:57:24.449 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00038326300000000001 2023-09-23T20:57:24.449 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.450 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.450 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:57:24.450 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.309484+0000", 2023-09-23T20:57:24.450 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.049202391999999998 2023-09-23T20:57:24.450 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.450 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.450 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T20:57:24.450 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.309599+0000", 2023-09-23T20:57:24.450 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000115642 2023-09-23T20:57:24.450 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.450 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.451 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 1,5", 2023-09-23T20:57:24.451 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.310127+0000", 2023-09-23T20:57:24.451 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00052754099999999997 2023-09-23T20:57:24.451 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.451 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.451 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T20:57:24.451 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.320913+0000", 2023-09-23T20:57:24.451 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.010786568999999999 2023-09-23T20:57:24.451 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.451 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.451 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:57:24.451 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.328297+0000", 2023-09-23T20:57:24.451 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0073830889999999998 2023-09-23T20:57:24.452 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.452 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.452 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:57:24.452 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.342727+0000", 2023-09-23T20:57:24.452 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.014430493000000001 2023-09-23T20:57:24.452 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.452 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.452 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T20:57:24.452 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.342775+0000", 2023-09-23T20:57:24.452 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.7593000000000003e-05 2023-09-23T20:57:24.452 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.452 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.452 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T20:57:24.453 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.343024+0000", 2023-09-23T20:57:24.453 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000249652 2023-09-23T20:57:24.453 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:57:24.453 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T20:57:24.453 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:57:24.453 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.453 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.453 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34797.0:194925 6.10 6:0af4fd48:::benchmark_data_smithi026_223868_object93799:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e180)", 2023-09-23T20:57:24.453 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T20:56:01.778450+0000", 2023-09-23T20:57:24.453 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 82.511898602000002, 2023-09-23T20:57:24.453 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.5649251790000001, 2023-09-23T20:57:24.453 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T20:57:24.453 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:57:24.454 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T20:57:24.454 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34797", 2023-09-23T20:57:24.454 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T20:57:24.454 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 194925 2023-09-23T20:57:24.454 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.454 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T20:57:24.454 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.454 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T20:57:24.454 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.778450+0000", 2023-09-23T20:57:24.454 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:57:24.454 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.455 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.455 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T20:57:24.455 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.778450+0000", 2023-09-23T20:57:24.455 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:57:24.455 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.455 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.455 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T20:57:24.455 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.778448+0000", 2023-09-23T20:57:24.455 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T20:57:24.455 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.455 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.455 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T20:57:24.456 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.778456+0000", 2023-09-23T20:57:24.456 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.5270000000000001e-06 2023-09-23T20:57:24.456 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.456 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.456 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T20:57:24.456 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.778484+0000", 2023-09-23T20:57:24.456 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.8629999999999999e-05 2023-09-23T20:57:24.456 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.456 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.456 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T20:57:24.456 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.778496+0000", 2023-09-23T20:57:24.456 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.203e-05 2023-09-23T20:57:24.456 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.457 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.457 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:57:24.457 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.778564+0000", 2023-09-23T20:57:24.457 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.8099e-05 2023-09-23T20:57:24.457 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.457 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.457 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T20:57:24.457 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.778579+0000", 2023-09-23T20:57:24.457 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.4397e-05 2023-09-23T20:57:24.457 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.457 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.457 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:57:24.457 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.951272+0000", 2023-09-23T20:57:24.458 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.172692926 2023-09-23T20:57:24.458 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.458 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.458 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T20:57:24.458 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.951281+0000", 2023-09-23T20:57:24.458 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.0180000000000007e-06 2023-09-23T20:57:24.458 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.458 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.458 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:57:24.458 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.260332+0000", 2023-09-23T20:57:24.458 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.3090507920000001 2023-09-23T20:57:24.458 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.459 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.459 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for degraded object", 2023-09-23T20:57:24.459 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.260659+0000", 2023-09-23T20:57:24.459 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000327466 2023-09-23T20:57:24.459 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.459 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.459 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:57:24.459 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.313663+0000", 2023-09-23T20:57:24.459 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.053004308 2023-09-23T20:57:24.459 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.459 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.459 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T20:57:24.460 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.313780+0000", 2023-09-23T20:57:24.460 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00011642999999999999 2023-09-23T20:57:24.460 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.460 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.460 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 1,5", 2023-09-23T20:57:24.460 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.314321+0000", 2023-09-23T20:57:24.460 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00054174099999999999 2023-09-23T20:57:24.460 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.460 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.460 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T20:57:24.460 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.320923+0000", 2023-09-23T20:57:24.460 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0066012759999999997 2023-09-23T20:57:24.460 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.461 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.461 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:57:24.461 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.328654+0000", 2023-09-23T20:57:24.461 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0077311619999999998 2023-09-23T20:57:24.461 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.461 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.461 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:57:24.461 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.343069+0000", 2023-09-23T20:57:24.461 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.014414797 2023-09-23T20:57:24.461 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.461 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.461 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T20:57:24.461 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.343121+0000", 2023-09-23T20:57:24.462 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.2475999999999997e-05 2023-09-23T20:57:24.462 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.462 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.462 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T20:57:24.462 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.343375+0000", 2023-09-23T20:57:24.462 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00025362900000000001 2023-09-23T20:57:24.462 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:57:24.462 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T20:57:24.462 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:57:24.462 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.462 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.462 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34797.0:194948 6.10 6:0b718fd8:::benchmark_data_smithi026_223868_object93822:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e180)", 2023-09-23T20:57:24.463 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T20:56:01.821311+0000", 2023-09-23T20:57:24.463 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 82.469036767999995, 2023-09-23T20:57:24.463 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.5226795950000001, 2023-09-23T20:57:24.463 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T20:57:24.463 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:57:24.463 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T20:57:24.463 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34797", 2023-09-23T20:57:24.463 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T20:57:24.463 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 194948 2023-09-23T20:57:24.463 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.463 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T20:57:24.463 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.463 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T20:57:24.464 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.821311+0000", 2023-09-23T20:57:24.464 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:57:24.464 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.464 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.464 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T20:57:24.464 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.821311+0000", 2023-09-23T20:57:24.464 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:57:24.464 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.464 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.464 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T20:57:24.464 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.821310+0000", 2023-09-23T20:57:24.464 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T20:57:24.465 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.465 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.465 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T20:57:24.465 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.821318+0000", 2023-09-23T20:57:24.465 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.7640000000000005e-06 2023-09-23T20:57:24.465 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.465 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.465 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T20:57:24.465 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.821335+0000", 2023-09-23T20:57:24.465 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.7572999999999999e-05 2023-09-23T20:57:24.465 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.465 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.465 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T20:57:24.465 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.821344+0000", 2023-09-23T20:57:24.466 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.4009999999999997e-06 2023-09-23T20:57:24.466 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.466 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.466 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:57:24.466 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.821399+0000", 2023-09-23T20:57:24.466 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.5770000000000003e-05 2023-09-23T20:57:24.466 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.466 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.466 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T20:57:24.466 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.821410+0000", 2023-09-23T20:57:24.466 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0789e-05 2023-09-23T20:57:24.466 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.467 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.467 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:57:24.467 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.951308+0000", 2023-09-23T20:57:24.467 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.12989732600000001 2023-09-23T20:57:24.467 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.467 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.467 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T20:57:24.467 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.951315+0000", 2023-09-23T20:57:24.467 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.9449999999999993e-06 2023-09-23T20:57:24.467 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.467 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.467 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:57:24.468 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.260707+0000", 2023-09-23T20:57:24.468 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.309391529 2023-09-23T20:57:24.468 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.468 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.468 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for degraded object", 2023-09-23T20:57:24.468 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.261067+0000", 2023-09-23T20:57:24.468 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00035980499999999999 2023-09-23T20:57:24.468 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.468 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.468 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:57:24.468 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.317608+0000", 2023-09-23T20:57:24.468 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.056540900999999998 2023-09-23T20:57:24.468 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.469 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.469 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T20:57:24.469 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.317709+0000", 2023-09-23T20:57:24.469 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000101371 2023-09-23T20:57:24.469 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.469 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.469 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 1,5", 2023-09-23T20:57:24.469 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.318249+0000", 2023-09-23T20:57:24.469 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00054033200000000003 2023-09-23T20:57:24.469 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.469 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.469 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T20:57:24.469 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.321587+0000", 2023-09-23T20:57:24.470 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.003337904 2023-09-23T20:57:24.470 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.470 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.470 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:57:24.470 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.329047+0000", 2023-09-23T20:57:24.470 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0074594570000000001 2023-09-23T20:57:24.470 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.470 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.470 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:57:24.470 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.343443+0000", 2023-09-23T20:57:24.470 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.014395984000000001 2023-09-23T20:57:24.470 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.471 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.471 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T20:57:24.471 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.343740+0000", 2023-09-23T20:57:24.471 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000296886 2023-09-23T20:57:24.471 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.471 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.471 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T20:57:24.471 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.343991+0000", 2023-09-23T20:57:24.471 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00025143799999999999 2023-09-23T20:57:24.471 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:57:24.471 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T20:57:24.471 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:57:24.472 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.472 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.472 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34797.0:195032 6.10 6:08dbcd4e:::benchmark_data_smithi026_223868_object93906:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e181)", 2023-09-23T20:57:24.472 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T20:56:02.127089+0000", 2023-09-23T20:57:24.472 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 82.163259210999996, 2023-09-23T20:57:24.472 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.224242249, 2023-09-23T20:57:24.472 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T20:57:24.472 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:57:24.472 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T20:57:24.472 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34797", 2023-09-23T20:57:24.472 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T20:57:24.472 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 195032 2023-09-23T20:57:24.472 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.473 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T20:57:24.473 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.473 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T20:57:24.473 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:02.127089+0000", 2023-09-23T20:57:24.473 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:57:24.473 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.473 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.473 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T20:57:24.473 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:02.127089+0000", 2023-09-23T20:57:24.473 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:57:24.473 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.473 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.473 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T20:57:24.474 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:02.127088+0000", 2023-09-23T20:57:24.474 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T20:57:24.474 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.474 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.474 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T20:57:24.474 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:02.127094+0000", 2023-09-23T20:57:24.474 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.2330000000000003e-06 2023-09-23T20:57:24.474 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.474 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.474 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T20:57:24.474 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:02.127104+0000", 2023-09-23T20:57:24.474 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0019e-05 2023-09-23T20:57:24.474 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.475 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.475 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T20:57:24.475 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:02.127109+0000", 2023-09-23T20:57:24.475 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.4960000000000004e-06 2023-09-23T20:57:24.475 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.475 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.475 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:57:24.475 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:02.127158+0000", 2023-09-23T20:57:24.475 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.8550000000000001e-05 2023-09-23T20:57:24.475 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.475 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.475 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T20:57:24.475 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:02.127168+0000", 2023-09-23T20:57:24.476 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.8009999999999998e-06 2023-09-23T20:57:24.476 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.476 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.476 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:57:24.476 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.261102+0000", 2023-09-23T20:57:24.476 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1339344730000001 2023-09-23T20:57:24.476 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.476 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.476 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for degraded object", 2023-09-23T20:57:24.476 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.261501+0000", 2023-09-23T20:57:24.476 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00039857100000000002 2023-09-23T20:57:24.476 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.476 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.477 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:57:24.477 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.322489+0000", 2023-09-23T20:57:24.477 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.060988570999999998 2023-09-23T20:57:24.477 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.477 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.477 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T20:57:24.477 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.322601+0000", 2023-09-23T20:57:24.477 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000111908 2023-09-23T20:57:24.477 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.477 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.477 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 1,5", 2023-09-23T20:57:24.477 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.323185+0000", 2023-09-23T20:57:24.477 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00058331099999999999 2023-09-23T20:57:24.478 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.478 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.478 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T20:57:24.478 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.326510+0000", 2023-09-23T20:57:24.478 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.003325578 2023-09-23T20:57:24.478 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.478 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.478 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:57:24.478 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.329495+0000", 2023-09-23T20:57:24.478 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0029846249999999999 2023-09-23T20:57:24.478 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.478 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.479 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:57:24.479 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.350780+0000", 2023-09-23T20:57:24.479 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.021285182999999999 2023-09-23T20:57:24.479 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.479 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.479 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T20:57:24.479 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.350846+0000", 2023-09-23T20:57:24.479 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.5630999999999996e-05 2023-09-23T20:57:24.479 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:57:24.479 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:57:24.479 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T20:57:24.479 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.351331+0000", 2023-09-23T20:57:24.479 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00048559200000000002 2023-09-23T20:57:24.480 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:57:24.480 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T20:57:24.480 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:57:24.480 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:57:24.480 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T20:57:24.480 INFO:teuthology.orchestra.run.smithi026.stdout:} 2023-09-23T20:57:24.481 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:57:24.481 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:57:24.481 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:57:24.481 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:57:24.582 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:57:24.688 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:57:24.922 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 46 15 70340 70325 95.4922 63.125 0.0127155 0.0104665 2023-09-23T20:57:25.288 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_ops_in_flight 2023-09-23T20:57:25.566 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:57:25.580 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:57:25.581 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:57:25.616 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:57:25.682 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:57:25.795 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:57:25.922 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 47 16 71324 71308 94.7678 61.4375 0.0161277 0.0105465 2023-09-23T20:57:26.066 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:57:26.072 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:57:26.072 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:57:26.173 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:57:26.184 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:57:25 smithi026 ceph-mon[179688]: pgmap v505: 109 pgs: 109 active+clean; 4.1 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 74 MiB/s wr, 1.18k op/s; 29 KiB/s, 0 objects/s recovering 2023-09-23T20:57:26.184 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:57:25 smithi026 ceph-mon[179688]: 1.2 scrub starts 2023-09-23T20:57:26.184 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:57:25 smithi026 ceph-mon[179688]: 1.2 scrub ok 2023-09-23T20:57:26.184 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:57:25 smithi026 ceph-mon[179688]: 1.14 scrub starts 2023-09-23T20:57:26.184 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:57:25 smithi026 ceph-mon[179688]: 1.14 scrub ok 2023-09-23T20:57:26.185 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:57:25 smithi026 ceph-mon[175151]: pgmap v505: 109 pgs: 109 active+clean; 4.1 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 74 MiB/s wr, 1.18k op/s; 29 KiB/s, 0 objects/s recovering 2023-09-23T20:57:26.185 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:57:25 smithi026 ceph-mon[175151]: 1.2 scrub starts 2023-09-23T20:57:26.185 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:57:25 smithi026 ceph-mon[175151]: 1.2 scrub ok 2023-09-23T20:57:26.185 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:57:25 smithi026 ceph-mon[175151]: 1.14 scrub starts 2023-09-23T20:57:26.185 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:57:25 smithi026 ceph-mon[175151]: 1.14 scrub ok 2023-09-23T20:57:26.209 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:57:26.250 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:57:26.251 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:57:25 smithi070 ceph-mon[168282]: pgmap v505: 109 pgs: 109 active+clean; 4.1 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 74 MiB/s wr, 1.18k op/s; 29 KiB/s, 0 objects/s recovering 2023-09-23T20:57:26.252 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:57:25 smithi070 ceph-mon[168282]: 1.2 scrub starts 2023-09-23T20:57:26.252 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:57:25 smithi070 ceph-mon[168282]: 1.2 scrub ok 2023-09-23T20:57:26.252 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:57:25 smithi070 ceph-mon[168282]: 1.14 scrub starts 2023-09-23T20:57:26.252 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:57:25 smithi070 ceph-mon[168282]: 1.14 scrub ok 2023-09-23T20:57:26.252 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:57:26.253 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:57:26.353 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:57:26.430 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:57:26.511 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:57:26.519 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:57:26.519 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:57:26.620 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:57:26.654 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:57:26.699 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:57:26.703 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:57:26.704 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:57:26.741 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0: {} 2023-09-23T20:57:26.785 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1: {} 2023-09-23T20:57:26.804 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:57:26.831 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2: {} 2023-09-23T20:57:26.838 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:57:26.870 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3: {} 2023-09-23T20:57:26.870 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:57:26.877 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:57:26.879 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:57:26.880 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:57:26.918 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4: {} 2023-09-23T20:57:26.923 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 48 16 72249 72233 93.9982 57.8125 0.0127723 0.0106325 2023-09-23T20:57:26.959 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T20:57:26.981 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:57:27.015 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6: {} 2023-09-23T20:57:27.018 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:57:27.058 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:57:27.064 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T20:57:27.064 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:57:27.065 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:57:27.081 INFO:teuthology.orchestra.run.smithi026.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T20:57:27.081 INFO:teuthology.orchestra.run.smithi026.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T20:57:27.081 INFO:teuthology.orchestra.run.smithi026.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T20:57:27.081 INFO:teuthology.orchestra.run.smithi026.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T20:57:27.081 INFO:teuthology.orchestra.run.smithi026.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T20:57:27.081 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T20:57:27.081 INFO:teuthology.orchestra.run.smithi026.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T20:57:27.081 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T20:57:27.115 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:57:26 smithi026 ceph-mon[179688]: 1.7 scrub starts 2023-09-23T20:57:27.115 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:57:26 smithi026 ceph-mon[179688]: 1.7 scrub ok 2023-09-23T20:57:27.115 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:57:26 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/2836670469' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:57:27.115 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:57:26 smithi026 ceph-mon[175151]: 1.7 scrub starts 2023-09-23T20:57:27.115 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:57:26 smithi026 ceph-mon[175151]: 1.7 scrub ok 2023-09-23T20:57:27.115 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:57:26 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/2836670469' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:57:27.166 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:57:27.263 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:57:27.371 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:57:26 smithi070 ceph-mon[168282]: 1.7 scrub starts 2023-09-23T20:57:27.371 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:57:26 smithi070 ceph-mon[168282]: 1.7 scrub ok 2023-09-23T20:57:27.371 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:57:26 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/2836670469' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:57:27.584 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:57:27.588 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:57:27.588 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:57:27.689 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:57:27.729 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:57:27.775 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:57:27.778 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:57:27.779 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:57:27.880 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:57:27.919 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:57:27.923 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 49 16 73236 73220 93.3359 61.6875 0.0269336 0.0107086 2023-09-23T20:57:27.963 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:57:27.966 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:57:27.966 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:57:28.004 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:57:28 smithi026 ceph-mon[175151]: pgmap v506: 109 pgs: 109 active+clean; 4.2 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 67 MiB/s wr, 1.07k op/s 2023-09-23T20:57:28.004 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:57:28 smithi026 ceph-mon[175151]: 1.6 scrub starts 2023-09-23T20:57:28.004 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:57:28 smithi026 ceph-mon[175151]: 1.6 scrub ok 2023-09-23T20:57:28.004 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:57:28 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/3103427594' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-09-23T20:57:28.004 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:57:28 smithi026 ceph-mon[175151]: 1.b scrub starts 2023-09-23T20:57:28.004 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:57:28 smithi026 ceph-mon[175151]: 1.b scrub ok 2023-09-23T20:57:28.024 INFO:teuthology.orchestra.run.smithi026.stderr:reweighted osd.1 to 1 (10000) 2023-09-23T20:57:28.067 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:57:28.097 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:57:28.132 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:57:28.134 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:57:28.135 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:57:28.235 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:57:28.266 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:57:28 smithi070 ceph-mon[168282]: pgmap v506: 109 pgs: 109 active+clean; 4.2 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 67 MiB/s wr, 1.07k op/s 2023-09-23T20:57:28.266 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:57:28 smithi070 ceph-mon[168282]: 1.6 scrub starts 2023-09-23T20:57:28.266 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:57:28 smithi070 ceph-mon[168282]: 1.6 scrub ok 2023-09-23T20:57:28.266 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:57:28 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/3103427594' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-09-23T20:57:28.267 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:57:28 smithi070 ceph-mon[168282]: 1.b scrub starts 2023-09-23T20:57:28.267 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:57:28 smithi070 ceph-mon[168282]: 1.b scrub ok 2023-09-23T20:57:28.273 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:57:28.317 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:57:28.321 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:57:28.321 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:57:28.386 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:57:28 smithi026 ceph-mon[179688]: pgmap v506: 109 pgs: 109 active+clean; 4.2 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 67 MiB/s wr, 1.07k op/s 2023-09-23T20:57:28.386 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:57:28 smithi026 ceph-mon[179688]: 1.6 scrub starts 2023-09-23T20:57:28.387 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:57:28 smithi026 ceph-mon[179688]: 1.6 scrub ok 2023-09-23T20:57:28.387 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:57:28 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/3103427594' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-09-23T20:57:28.387 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:57:28 smithi026 ceph-mon[179688]: 1.b scrub starts 2023-09-23T20:57:28.387 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:57:28 smithi026 ceph-mon[179688]: 1.b scrub ok 2023-09-23T20:57:28.422 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:57:28.478 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:57:28.923 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 50 16 74229 74213 92.7106 62.0625 0.00780378 0.0107812 2023-09-23T20:57:29.035 INFO:teuthology.orchestra.run.smithi026.stderr:nodeep-scrub is unset 2023-09-23T20:57:29.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:57:29 smithi070 ceph-mon[168282]: 1.f scrub starts 2023-09-23T20:57:29.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:57:29 smithi070 ceph-mon[168282]: 1.f scrub ok 2023-09-23T20:57:29.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:57:29 smithi070 ceph-mon[168282]: failed to encode map e218 with expected crc 2023-09-23T20:57:29.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:57:29 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/3103427594' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-09-23T20:57:29.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:57:29 smithi070 ceph-mon[168282]: osdmap e218: 8 total, 8 up, 8 in 2023-09-23T20:57:29.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:57:29 smithi070 ceph-mon[168282]: failed to encode map e218 with expected crc 2023-09-23T20:57:29.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:57:29 smithi070 ceph-mon[168282]: failed to encode map e218 with expected crc 2023-09-23T20:57:29.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:57:29 smithi070 ceph-mon[168282]: failed to encode map e218 with expected crc 2023-09-23T20:57:29.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:57:29 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/2384118176' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:57:29.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:57:29 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:57:29.383 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:57:29 smithi026 ceph-mon[175151]: 1.f scrub starts 2023-09-23T20:57:29.384 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:57:29 smithi026 ceph-mon[175151]: 1.f scrub ok 2023-09-23T20:57:29.384 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:57:29 smithi026 ceph-mon[175151]: failed to encode map e218 with expected crc 2023-09-23T20:57:29.384 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:57:29 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/3103427594' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-09-23T20:57:29.384 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:57:29 smithi026 ceph-mon[175151]: osdmap e218: 8 total, 8 up, 8 in 2023-09-23T20:57:29.384 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:57:29 smithi026 ceph-mon[175151]: failed to encode map e218 with expected crc 2023-09-23T20:57:29.384 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:57:29 smithi026 ceph-mon[175151]: failed to encode map e218 with expected crc 2023-09-23T20:57:29.384 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:57:29 smithi026 ceph-mon[175151]: failed to encode map e218 with expected crc 2023-09-23T20:57:29.384 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:57:29 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/2384118176' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:57:29.384 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:57:29 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:57:29.385 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:57:29 smithi026 ceph-mon[179688]: 1.f scrub starts 2023-09-23T20:57:29.385 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:57:29 smithi026 ceph-mon[179688]: 1.f scrub ok 2023-09-23T20:57:29.385 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:57:29 smithi026 ceph-mon[179688]: failed to encode map e218 with expected crc 2023-09-23T20:57:29.385 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:57:29 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/3103427594' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-09-23T20:57:29.385 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:57:29 smithi026 ceph-mon[179688]: osdmap e218: 8 total, 8 up, 8 in 2023-09-23T20:57:29.385 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:57:29 smithi026 ceph-mon[179688]: failed to encode map e218 with expected crc 2023-09-23T20:57:29.385 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:57:29 smithi026 ceph-mon[179688]: failed to encode map e218 with expected crc 2023-09-23T20:57:29.385 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:57:29 smithi026 ceph-mon[179688]: failed to encode map e218 with expected crc 2023-09-23T20:57:29.385 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:57:29 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/2384118176' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:57:29.385 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:57:29 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:57:29.477 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph osd reweight 2 1 2023-09-23T20:57:29.628 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T20:57:29.742 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:57:29 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[164029]: ::ffff:172.21.15.70 - - [23/Sep/2023:20:57:29] "GET /metrics HTTP/1.1" 200 36315 "" "Prometheus/2.43.0" 2023-09-23T20:57:29.927 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 51 16 75099 75083 91.9574 54.375 0.0186283 0.0108687 2023-09-23T20:57:30.018 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:57:30 smithi026 ceph-mon[175151]: pgmap v508: 109 pgs: 109 active+clean; 4.3 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 58 MiB/s wr, 930 op/s 2023-09-23T20:57:30.018 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:57:30 smithi026 ceph-mon[175151]: 1.11 scrub starts 2023-09-23T20:57:30.019 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:57:30 smithi026 ceph-mon[175151]: 1.11 scrub ok 2023-09-23T20:57:30.019 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:57:30 smithi026 ceph-mon[175151]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T20:57:30.019 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:57:30 smithi026 ceph-mon[175151]: Cluster is now healthy 2023-09-23T20:57:30.019 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:57:30 smithi026 ceph-mon[175151]: failed to encode map e219 with expected crc 2023-09-23T20:57:30.019 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:57:30 smithi026 ceph-mon[175151]: failed to encode map e219 with expected crc 2023-09-23T20:57:30.019 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:57:30 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T20:57:30.019 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:57:30 smithi026 ceph-mon[175151]: osdmap e219: 8 total, 8 up, 8 in 2023-09-23T20:57:30.019 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:57:30 smithi026 ceph-mon[175151]: failed to encode map e219 with expected crc 2023-09-23T20:57:30.019 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:57:30 smithi026 ceph-mon[179688]: pgmap v508: 109 pgs: 109 active+clean; 4.3 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 58 MiB/s wr, 930 op/s 2023-09-23T20:57:30.020 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:57:30 smithi026 ceph-mon[179688]: 1.11 scrub starts 2023-09-23T20:57:30.020 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:57:30 smithi026 ceph-mon[179688]: 1.11 scrub ok 2023-09-23T20:57:30.020 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:57:30 smithi026 ceph-mon[179688]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T20:57:30.020 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:57:30 smithi026 ceph-mon[179688]: Cluster is now healthy 2023-09-23T20:57:30.020 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:57:30 smithi026 ceph-mon[179688]: failed to encode map e219 with expected crc 2023-09-23T20:57:30.020 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:57:30 smithi026 ceph-mon[179688]: failed to encode map e219 with expected crc 2023-09-23T20:57:30.020 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:57:30 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T20:57:30.020 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:57:30 smithi026 ceph-mon[179688]: osdmap e219: 8 total, 8 up, 8 in 2023-09-23T20:57:30.020 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:57:30 smithi026 ceph-mon[179688]: failed to encode map e219 with expected crc 2023-09-23T20:57:30.211 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:57:30.215 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:57:30.216 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:57:30.317 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:57:30.357 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:57:30 smithi070 ceph-mon[168282]: pgmap v508: 109 pgs: 109 active+clean; 4.3 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 58 MiB/s wr, 930 op/s 2023-09-23T20:57:30.357 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:57:30 smithi070 ceph-mon[168282]: 1.11 scrub starts 2023-09-23T20:57:30.357 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:57:30 smithi070 ceph-mon[168282]: 1.11 scrub ok 2023-09-23T20:57:30.357 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:57:30 smithi070 ceph-mon[168282]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T20:57:30.358 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:57:30 smithi070 ceph-mon[168282]: Cluster is now healthy 2023-09-23T20:57:30.358 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:57:30 smithi070 ceph-mon[168282]: failed to encode map e219 with expected crc 2023-09-23T20:57:30.358 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:57:30 smithi070 ceph-mon[168282]: failed to encode map e219 with expected crc 2023-09-23T20:57:30.358 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:57:30 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T20:57:30.358 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:57:30 smithi070 ceph-mon[168282]: osdmap e219: 8 total, 8 up, 8 in 2023-09-23T20:57:30.358 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:57:30 smithi070 ceph-mon[168282]: failed to encode map e219 with expected crc 2023-09-23T20:57:30.360 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:57:30.414 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:57:30.419 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:57:30.419 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:57:30.520 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:57:30.565 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:57:30.614 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:57:30.619 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:57:30.619 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:57:30.720 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:57:30.822 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:57:30.925 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 52 16 76141 76125 91.441 65.125 0.0180329 0.0109303 2023-09-23T20:57:31.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:57:31 smithi026 ceph-mon[175151]: 1.5 scrub starts 2023-09-23T20:57:31.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:57:31 smithi026 ceph-mon[175151]: 1.5 scrub ok 2023-09-23T20:57:31.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:57:31 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:57:31.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:57:31 smithi026 ceph-mon[179688]: 1.5 scrub starts 2023-09-23T20:57:31.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:57:31 smithi026 ceph-mon[179688]: 1.5 scrub ok 2023-09-23T20:57:31.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:57:31 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:57:31.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:57:31 smithi070 ceph-mon[168282]: 1.5 scrub starts 2023-09-23T20:57:31.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:57:31 smithi070 ceph-mon[168282]: 1.5 scrub ok 2023-09-23T20:57:31.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:57:31 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:57:31.925 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 53 16 77242 77226 91.0141 68.8125 0.0151642 0.0109827 2023-09-23T20:57:32.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:57:32 smithi070 ceph-mon[168282]: pgmap v510: 109 pgs: 109 active+clean; 4.5 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 59 MiB/s wr, 937 op/s 2023-09-23T20:57:32.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:57:32 smithi070 ceph-mon[168282]: 1.d scrub starts 2023-09-23T20:57:32.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:57:32 smithi070 ceph-mon[168282]: 1.d scrub ok 2023-09-23T20:57:32.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:57:32 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:57:32.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:57:32 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:57:32.381 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:57:32 smithi026 ceph-mon[175151]: pgmap v510: 109 pgs: 109 active+clean; 4.5 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 59 MiB/s wr, 937 op/s 2023-09-23T20:57:32.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:57:32 smithi026 ceph-mon[175151]: 1.d scrub starts 2023-09-23T20:57:32.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:57:32 smithi026 ceph-mon[175151]: 1.d scrub ok 2023-09-23T20:57:32.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:57:32 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:57:32.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:57:32 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:57:32.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:57:32 smithi026 ceph-mon[179688]: pgmap v510: 109 pgs: 109 active+clean; 4.5 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 59 MiB/s wr, 937 op/s 2023-09-23T20:57:32.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:57:32 smithi026 ceph-mon[179688]: 1.d scrub starts 2023-09-23T20:57:32.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:57:32 smithi026 ceph-mon[179688]: 1.d scrub ok 2023-09-23T20:57:32.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:57:32 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:57:32.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:57:32 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:57:32.925 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 54 16 78214 78198 90.4538 60.75 0.0149632 0.0110511 2023-09-23T20:57:33.039 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:57:33.045 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:57:33.046 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:57:33.147 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:57:33.217 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:57:33.276 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:57:33.286 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:57:33.287 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:57:33.387 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:57:33.434 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:57:33.481 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:57:33.483 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:57:33.484 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:57:33.584 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:57:33.616 INFO:teuthology.orchestra.run.smithi026.stdout:ERROR: (22) Invalid argument 2023-09-23T20:57:33.616 INFO:teuthology.orchestra.run.smithi026.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:57:33.711 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:57:33.711 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:57:33.792 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:57:33.792 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:57:33.806 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:57:33.807 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:57:33.807 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:57:33 smithi026 ceph-mon[179688]: pgmap v511: 109 pgs: 109 active+clean; 4.6 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 69 MiB/s wr, 1.10k op/s 2023-09-23T20:57:33.807 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:57:33 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:57:33.808 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:57:33 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:57:33.810 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:57:33 smithi026 ceph-mon[175151]: pgmap v511: 109 pgs: 109 active+clean; 4.6 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 69 MiB/s wr, 1.10k op/s 2023-09-23T20:57:33.811 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:57:33 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:57:33.811 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:57:33 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:57:33.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:57:33 smithi070 ceph-mon[168282]: pgmap v511: 109 pgs: 109 active+clean; 4.6 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 69 MiB/s wr, 1.10k op/s 2023-09-23T20:57:33.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:57:33 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:57:33.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:57:33 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:57:33.908 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:57:33.925 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 55 16 79116 79100 89.8344 56.375 0.0163895 0.0111255 2023-09-23T20:57:33.944 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:57:34.000 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:57:34.003 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:57:34.004 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:57:34.104 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:57:34.140 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:57:34.182 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:57:34.188 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:57:34.188 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:57:34.289 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:57:34.399 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:57:34.472 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:57:34.483 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:57:34.483 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:57:34.556 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_blocked_ops 2023-09-23T20:57:34.584 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:57:34.694 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:57:34.758 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T20:57:34.932 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 56 16 79989 79973 89.2012 54.5625 0.0135299 0.0112042 2023-09-23T20:57:35.053 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:57:35.066 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:57:35.066 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:57:35.167 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:57:35.200 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:57:35.250 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:57:35.257 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:57:35.258 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:57:35.359 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:57:35.406 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:57:35.506 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:57:35.509 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:57:35.509 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:57:35.610 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:57:35.641 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:57:35.681 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:57:35.684 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:57:35.684 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:57:35.785 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:57:35.861 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:57:35.928 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 57 16 80826 80810 88.5543 52.3125 0.0235167 0.0112863 2023-09-23T20:57:36.111 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:57:35 smithi026 ceph-mon[179688]: 1.e scrub starts 2023-09-23T20:57:36.112 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:57:35 smithi026 ceph-mon[179688]: 1.e scrub ok 2023-09-23T20:57:36.112 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:57:35 smithi026 ceph-mon[179688]: pgmap v512: 109 pgs: 109 active+clean; 4.7 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 61 MiB/s wr, 970 op/s 2023-09-23T20:57:36.112 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:57:35 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:57:36.112 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:57:35 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:57:36.112 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:57:35 smithi026 ceph-mon[175151]: 1.e scrub starts 2023-09-23T20:57:36.112 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:57:35 smithi026 ceph-mon[175151]: 1.e scrub ok 2023-09-23T20:57:36.112 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:57:35 smithi026 ceph-mon[175151]: pgmap v512: 109 pgs: 109 active+clean; 4.7 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 61 MiB/s wr, 970 op/s 2023-09-23T20:57:36.112 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:57:35 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:57:36.113 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:57:35 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:57:36.158 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:57:36.163 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:57:36.164 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:57:36.264 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:57:36.304 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:57:35 smithi070 ceph-mon[168282]: 1.e scrub starts 2023-09-23T20:57:36.304 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:57:35 smithi070 ceph-mon[168282]: 1.e scrub ok 2023-09-23T20:57:36.304 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:57:35 smithi070 ceph-mon[168282]: pgmap v512: 109 pgs: 109 active+clean; 4.7 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 61 MiB/s wr, 970 op/s 2023-09-23T20:57:36.305 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:57:35 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:57:36.305 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:57:35 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:57:36.310 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:57:36.321 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:57:36.356 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:57:36.360 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:57:36.361 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:57:36.461 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:57:36.491 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:57:36.493 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0: {} 2023-09-23T20:57:36.530 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:57:36.532 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:57:36.533 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:57:36.536 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1: {} 2023-09-23T20:57:36.577 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2: {} 2023-09-23T20:57:36.617 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3: {} 2023-09-23T20:57:36.621 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:57:36.634 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:57:36.661 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:57:36.700 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:57:36.701 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4: {} 2023-09-23T20:57:36.704 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:57:36.704 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:57:36.749 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T20:57:36.793 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6: {} 2023-09-23T20:57:36.806 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:57:36.825 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T20:57:36.827 INFO:teuthology.orchestra.run.smithi026.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T20:57:36.827 INFO:teuthology.orchestra.run.smithi026.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T20:57:36.828 INFO:teuthology.orchestra.run.smithi026.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T20:57:36.828 INFO:teuthology.orchestra.run.smithi026.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T20:57:36.828 INFO:teuthology.orchestra.run.smithi026.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T20:57:36.828 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T20:57:36.828 INFO:teuthology.orchestra.run.smithi026.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T20:57:36.828 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T20:57:36.837 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:57:36.886 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:57:36.889 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:57:36.890 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:57:36.930 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 58 16 81575 81559 87.8317 46.8125 0.0165148 0.0113802 2023-09-23T20:57:36.990 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:57:37.012 INFO:teuthology.orchestra.run.smithi026.stderr:reweighted osd.2 to 1 (10000) 2023-09-23T20:57:37.020 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:57:37.059 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:57:37.063 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:57:37.063 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:57:37.164 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:57:37.246 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:57:36 smithi026 ceph-mon[175151]: 1.13 scrub starts 2023-09-23T20:57:37.246 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:57:36 smithi026 ceph-mon[175151]: 1.13 scrub ok 2023-09-23T20:57:37.246 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:57:36 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/644524185' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:57:37.246 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:57:36 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/1999731377' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T20:57:37.249 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:57:36 smithi026 ceph-mon[179688]: 1.13 scrub starts 2023-09-23T20:57:37.249 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:57:36 smithi026 ceph-mon[179688]: 1.13 scrub ok 2023-09-23T20:57:37.249 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:57:36 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/644524185' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:57:37.249 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:57:36 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/1999731377' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T20:57:37.260 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:57:37.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:57:36 smithi070 ceph-mon[168282]: 1.13 scrub starts 2023-09-23T20:57:37.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:57:36 smithi070 ceph-mon[168282]: 1.13 scrub ok 2023-09-23T20:57:37.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:57:36 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/644524185' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:57:37.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:57:36 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/1999731377' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T20:57:37.716 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:57:37.722 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:57:37.723 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:57:37.823 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:57:37.859 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:57:37.900 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:57:37.903 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:57:37.903 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:57:37.930 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 59 16 82470 82454 87.2913 55.9375 0.020848 0.0114503 2023-09-23T20:57:37.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:57:37 smithi026 ceph-mon[175151]: pgmap v513: 109 pgs: 109 active+clean; 4.8 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 61 MiB/s wr, 979 op/s 2023-09-23T20:57:37.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:57:37 smithi026 ceph-mon[175151]: failed to encode map e220 with expected crc 2023-09-23T20:57:37.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:57:37 smithi026 ceph-mon[175151]: failed to encode map e220 with expected crc 2023-09-23T20:57:37.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:57:37 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/1999731377' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-09-23T20:57:37.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:57:37 smithi026 ceph-mon[175151]: osdmap e220: 8 total, 8 up, 8 in 2023-09-23T20:57:37.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:57:37 smithi026 ceph-mon[175151]: failed to encode map e220 with expected crc 2023-09-23T20:57:37.988 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:57:37 smithi026 ceph-mon[179688]: pgmap v513: 109 pgs: 109 active+clean; 4.8 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 61 MiB/s wr, 979 op/s 2023-09-23T20:57:37.988 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:57:37 smithi026 ceph-mon[179688]: failed to encode map e220 with expected crc 2023-09-23T20:57:37.988 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:57:37 smithi026 ceph-mon[179688]: failed to encode map e220 with expected crc 2023-09-23T20:57:37.988 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:57:37 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/1999731377' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-09-23T20:57:37.988 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:57:37 smithi026 ceph-mon[179688]: osdmap e220: 8 total, 8 up, 8 in 2023-09-23T20:57:37.988 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:57:37 smithi026 ceph-mon[179688]: failed to encode map e220 with expected crc 2023-09-23T20:57:38.004 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:57:38.042 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:57:38.095 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:57:38.098 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:57:38.099 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:57:38.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:57:37 smithi070 ceph-mon[168282]: pgmap v513: 109 pgs: 109 active+clean; 4.8 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 61 MiB/s wr, 979 op/s 2023-09-23T20:57:38.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:57:37 smithi070 ceph-mon[168282]: failed to encode map e220 with expected crc 2023-09-23T20:57:38.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:57:37 smithi070 ceph-mon[168282]: failed to encode map e220 with expected crc 2023-09-23T20:57:38.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:57:37 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/1999731377' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-09-23T20:57:38.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:57:37 smithi070 ceph-mon[168282]: osdmap e220: 8 total, 8 up, 8 in 2023-09-23T20:57:38.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:57:37 smithi070 ceph-mon[168282]: failed to encode map e220 with expected crc 2023-09-23T20:57:38.199 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:57:38.271 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph osd reweight 3 1 2023-09-23T20:57:38.290 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:57:38.526 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:57:38.530 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:57:38.530 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:57:38.631 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:57:38.721 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:57:38.791 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:57:38.805 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:57:38.805 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:57:38.906 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:57:38.930 INFO:tasks.radosbench.radosbench.0.smithi026.stdout:2023-09-23T20:57:38.930994+0000 min lat: 0.0011341 max lat: 1.0132 avg lat: 0.0115293 2023-09-23T20:57:38.930 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T20:57:38.930 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 60 16 83283 83267 86.6836 50.8125 0.0210892 0.0115293 2023-09-23T20:57:38.932 INFO:teuthology.orchestra.run.smithi026.stdout:{ 2023-09-23T20:57:38.932 INFO:teuthology.orchestra.run.smithi026.stdout: "ops": [], 2023-09-23T20:57:38.932 INFO:teuthology.orchestra.run.smithi026.stdout: "complaint_time": 30, 2023-09-23T20:57:38.932 INFO:teuthology.orchestra.run.smithi026.stdout: "num_blocked_ops": 0 2023-09-23T20:57:38.932 INFO:teuthology.orchestra.run.smithi026.stdout:} 2023-09-23T20:57:38.936 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:57:38.984 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:57:38.988 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:57:38.989 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:57:39.089 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:57:39.168 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:57:39.258 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:57:39.265 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:57:39.266 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:57:39.366 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:57:39.381 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T20:57:39.451 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:57:39.741 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:57:39.749 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:57:39.749 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:57:39.750 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:57:39 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[164029]: ::ffff:172.21.15.70 - - [23/Sep/2023:20:57:39] "GET /metrics HTTP/1.1" 200 36321 "" "Prometheus/2.43.0" 2023-09-23T20:57:39.772 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:57:39.789 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_historic_ops 2023-09-23T20:57:39.851 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:57:39.931 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 61 16 84042 84026 86.0399 47.4375 0.0354482 0.011617 2023-09-23T20:57:39.990 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:57:40.101 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:57:39 smithi026 ceph-mon[179688]: pgmap v515: 109 pgs: 109 active+clean; 4.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 64 MiB/s wr, 1.02k op/s 2023-09-23T20:57:40.104 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:57:39 smithi026 ceph-mon[175151]: pgmap v515: 109 pgs: 109 active+clean; 4.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 64 MiB/s wr, 1.02k op/s 2023-09-23T20:57:40.138 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:57:40.144 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:57:40.144 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:57:40.245 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:57:40.276 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:57:39 smithi070 ceph-mon[168282]: pgmap v515: 109 pgs: 109 active+clean; 4.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 64 MiB/s wr, 1.02k op/s 2023-09-23T20:57:40.279 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:57:40.328 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:57:40.331 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:57:40.331 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:57:40.428 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:57:40.432 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:57:40.461 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:57:40.501 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:57:40.507 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:57:40.508 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:57:40.608 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:57:40.693 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:57:40.931 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 62 16 84844 84828 85.4608 50.125 0.0152276 0.0116955 2023-09-23T20:57:41.039 INFO:teuthology.orchestra.run.smithi026.stderr:noscrub is set 2023-09-23T20:57:41.150 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:57:41.159 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:57:41.159 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:57:41.260 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:57:41.337 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:57:41 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/2837640381' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:57:41.337 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:57:41 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:57:41.338 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:57:41 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/2837640381' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:57:41.338 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:57:41 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:57:41.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:57:41 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/2837640381' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:57:41.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:57:41 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:57:41.376 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:57:41.860 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:57:41.866 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:57:41.866 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:57:41.933 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 63 16 85760 85744 85.0099 57.25 0.0250763 0.0117579 2023-09-23T20:57:41.967 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:57:42.074 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:57:42.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:57:42 smithi026 ceph-mon[179688]: pgmap v516: 109 pgs: 109 active+clean; 5.0 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 58 MiB/s wr, 921 op/s 2023-09-23T20:57:42.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:57:42 smithi026 ceph-mon[179688]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:57:42.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:57:42 smithi026 ceph-mon[179688]: failed to encode map e221 with expected crc 2023-09-23T20:57:42.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:57:42 smithi026 ceph-mon[179688]: failed to encode map e221 with expected crc 2023-09-23T20:57:42.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:57:42 smithi026 ceph-mon[179688]: failed to encode map e221 with expected crc 2023-09-23T20:57:42.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:57:42 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T20:57:42.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:57:42 smithi026 ceph-mon[179688]: osdmap e221: 8 total, 8 up, 8 in 2023-09-23T20:57:42.141 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:57:42 smithi026 ceph-mon[175151]: pgmap v516: 109 pgs: 109 active+clean; 5.0 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 58 MiB/s wr, 921 op/s 2023-09-23T20:57:42.141 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:57:42 smithi026 ceph-mon[175151]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:57:42.141 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:57:42 smithi026 ceph-mon[175151]: failed to encode map e221 with expected crc 2023-09-23T20:57:42.141 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:57:42 smithi026 ceph-mon[175151]: failed to encode map e221 with expected crc 2023-09-23T20:57:42.141 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:57:42 smithi026 ceph-mon[175151]: failed to encode map e221 with expected crc 2023-09-23T20:57:42.141 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:57:42 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T20:57:42.141 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:57:42 smithi026 ceph-mon[175151]: osdmap e221: 8 total, 8 up, 8 in 2023-09-23T20:57:42.199 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:57:42.216 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:57:42.216 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:57:42.317 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:57:42.329 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:57:42.343 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:57:42 smithi070 ceph-mon[168282]: pgmap v516: 109 pgs: 109 active+clean; 5.0 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 58 MiB/s wr, 921 op/s 2023-09-23T20:57:42.343 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:57:42 smithi070 ceph-mon[168282]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:57:42.343 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:57:42 smithi070 ceph-mon[168282]: failed to encode map e221 with expected crc 2023-09-23T20:57:42.344 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:57:42 smithi070 ceph-mon[168282]: failed to encode map e221 with expected crc 2023-09-23T20:57:42.344 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:57:42 smithi070 ceph-mon[168282]: failed to encode map e221 with expected crc 2023-09-23T20:57:42.344 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:57:42 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T20:57:42.344 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:57:42 smithi070 ceph-mon[168282]: osdmap e221: 8 total, 8 up, 8 in 2023-09-23T20:57:42.347 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:57:42.386 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:57:42.390 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:57:42.391 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:57:42.491 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:57:42.528 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:57:42.566 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:57:42.569 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:57:42.569 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:57:42.670 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:57:42.760 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:57:42.937 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 64 16 86502 86486 84.4035 46.375 0.0278537 0.0118417 2023-09-23T20:57:42.958 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:57:42.987 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:57:42.988 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:57:43.089 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:57:43.183 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:57:43.353 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0: {} 2023-09-23T20:57:43.397 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1: {} 2023-09-23T20:57:43.436 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2: {} 2023-09-23T20:57:43.485 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:57:43.486 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3: {} 2023-09-23T20:57:43.499 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:57:43.499 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:57:43.528 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4: {} 2023-09-23T20:57:43.559 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T20:57:43.600 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:57:43.600 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6: {} 2023-09-23T20:57:43.600 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:57:43 smithi026 ceph-mon[175151]: pgmap v518: 109 pgs: 109 active+clean; 5.2 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 59 MiB/s wr, 943 op/s 2023-09-23T20:57:43.601 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:57:43 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:57:43.601 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:57:43 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:57:43.601 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:57:43 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/2998060834' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-09-23T20:57:43.601 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:57:43 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-09-23T20:57:43.601 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:57:43 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/1878422783' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:57:43.657 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T20:57:43.676 INFO:teuthology.orchestra.run.smithi026.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T20:57:43.676 INFO:teuthology.orchestra.run.smithi026.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T20:57:43.676 INFO:teuthology.orchestra.run.smithi026.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T20:57:43.676 INFO:teuthology.orchestra.run.smithi026.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T20:57:43.676 INFO:teuthology.orchestra.run.smithi026.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T20:57:43.676 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T20:57:43.676 INFO:teuthology.orchestra.run.smithi026.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T20:57:43.676 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T20:57:43.692 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:57:43.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:57:43 smithi070 ceph-mon[168282]: pgmap v518: 109 pgs: 109 active+clean; 5.2 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 59 MiB/s wr, 943 op/s 2023-09-23T20:57:43.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:57:43 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:57:43.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:57:43 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:57:43.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:57:43 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/2998060834' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-09-23T20:57:43.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:57:43 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-09-23T20:57:43.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:57:43 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/1878422783' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:57:43.881 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:57:43 smithi026 ceph-mon[179688]: pgmap v518: 109 pgs: 109 active+clean; 5.2 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 59 MiB/s wr, 943 op/s 2023-09-23T20:57:43.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:57:43 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:57:43.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:57:43 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:57:43.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:57:43 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/2998060834' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-09-23T20:57:43.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:57:43 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-09-23T20:57:43.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:57:43 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/1878422783' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:57:43.938 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 65 16 87284 87268 83.8533 48.875 0.0212744 0.0119205 2023-09-23T20:57:44.033 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T20:57:44.097 INFO:teuthology.orchestra.run.smithi026.stderr:reweighted osd.3 to 1 (10000) 2023-09-23T20:57:44.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:57:44 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:57:44.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:57:44 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:57:44.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:57:44 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:57:44.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:57:44 smithi070 ceph-mon[168282]: failed to encode map e222 with expected crc 2023-09-23T20:57:44.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:57:44 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-09-23T20:57:44.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:57:44 smithi070 ceph-mon[168282]: osdmap e222: 8 total, 8 up, 8 in 2023-09-23T20:57:44.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:57:44 smithi070 ceph-mon[168282]: failed to encode map e222 with expected crc 2023-09-23T20:57:44.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:57:44 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:57:44.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:57:44 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:57:44.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:57:44 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:57:44.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:57:44 smithi026 ceph-mon[175151]: failed to encode map e222 with expected crc 2023-09-23T20:57:44.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:57:44 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-09-23T20:57:44.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:57:44 smithi026 ceph-mon[175151]: osdmap e222: 8 total, 8 up, 8 in 2023-09-23T20:57:44.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:57:44 smithi026 ceph-mon[175151]: failed to encode map e222 with expected crc 2023-09-23T20:57:44.883 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:57:44 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:57:44.883 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:57:44 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:57:44.883 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:57:44 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:57:44.883 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:57:44 smithi026 ceph-mon[179688]: failed to encode map e222 with expected crc 2023-09-23T20:57:44.883 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:57:44 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-09-23T20:57:44.883 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:57:44 smithi026 ceph-mon[179688]: osdmap e222: 8 total, 8 up, 8 in 2023-09-23T20:57:44.883 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:57:44 smithi026 ceph-mon[179688]: failed to encode map e222 with expected crc 2023-09-23T20:57:44.939 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 66 16 87489 87473 82.7776 12.8125 0.00383053 0.0119263 2023-09-23T20:57:45.463 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:57:45.467 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:57:45.467 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:57:45.568 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:57:45.603 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:57:45.613 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph osd reweight 6 1 2023-09-23T20:57:45.693 INFO:teuthology.orchestra.run.smithi026.stdout:ERROR: (22) Invalid argument 2023-09-23T20:57:45.694 INFO:teuthology.orchestra.run.smithi026.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:57:45.855 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:57:45 smithi026 ceph-mon[175151]: failed to encode map e222 with expected crc 2023-09-23T20:57:45.855 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:57:45 smithi026 ceph-mon[175151]: pgmap v520: 109 pgs: 6 peering, 103 active+clean; 5.3 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 54 MiB/s wr, 868 op/s 2023-09-23T20:57:45.855 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:57:45 smithi026 ceph-mon[175151]: failed to encode map e223 with expected crc 2023-09-23T20:57:45.855 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:57:45 smithi026 ceph-mon[175151]: failed to encode map e223 with expected crc 2023-09-23T20:57:45.855 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:57:45 smithi026 ceph-mon[175151]: osdmap e223: 8 total, 8 up, 8 in 2023-09-23T20:57:45.855 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:57:45 smithi026 ceph-mon[175151]: failed to encode map e223 with expected crc 2023-09-23T20:57:45.855 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:57:45 smithi026 ceph-mon[175151]: failed to encode map e223 with expected crc 2023-09-23T20:57:45.855 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:57:45 smithi026 ceph-mon[179688]: failed to encode map e222 with expected crc 2023-09-23T20:57:45.855 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:57:45 smithi026 ceph-mon[179688]: pgmap v520: 109 pgs: 6 peering, 103 active+clean; 5.3 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 54 MiB/s wr, 868 op/s 2023-09-23T20:57:45.856 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:57:45 smithi026 ceph-mon[179688]: failed to encode map e223 with expected crc 2023-09-23T20:57:45.856 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:57:45 smithi026 ceph-mon[179688]: failed to encode map e223 with expected crc 2023-09-23T20:57:45.856 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:57:45 smithi026 ceph-mon[179688]: osdmap e223: 8 total, 8 up, 8 in 2023-09-23T20:57:45.856 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:57:45 smithi026 ceph-mon[179688]: failed to encode map e223 with expected crc 2023-09-23T20:57:45.856 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:57:45 smithi026 ceph-mon[179688]: failed to encode map e223 with expected crc 2023-09-23T20:57:45.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:57:45 smithi070 ceph-mon[168282]: failed to encode map e222 with expected crc 2023-09-23T20:57:45.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:57:45 smithi070 ceph-mon[168282]: pgmap v520: 109 pgs: 6 peering, 103 active+clean; 5.3 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 54 MiB/s wr, 868 op/s 2023-09-23T20:57:45.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:57:45 smithi070 ceph-mon[168282]: failed to encode map e223 with expected crc 2023-09-23T20:57:45.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:57:45 smithi070 ceph-mon[168282]: failed to encode map e223 with expected crc 2023-09-23T20:57:45.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:57:45 smithi070 ceph-mon[168282]: osdmap e223: 8 total, 8 up, 8 in 2023-09-23T20:57:45.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:57:45 smithi070 ceph-mon[168282]: failed to encode map e223 with expected crc 2023-09-23T20:57:45.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:57:45 smithi070 ceph-mon[168282]: failed to encode map e223 with expected crc 2023-09-23T20:57:45.939 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 67 16 87489 87473 81.5428 0 - 0.0119263 2023-09-23T20:57:46.102 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:57:46.105 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:57:46.106 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:57:46.158 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_ops_in_flight 2023-09-23T20:57:46.206 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:57:46.232 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:57:46.246 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:57:46.264 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:57:46.267 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:57:46.267 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:57:46.368 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:57:46.425 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:57:46.473 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:57:46.476 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:57:46.477 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:57:46.577 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:57:46.608 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:57:46.644 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:57:46.652 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:57:46.652 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:57:46.728 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:57:46 smithi026 ceph-mon[175151]: failed to encode map e223 with expected crc 2023-09-23T20:57:46.728 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:57:46 smithi026 ceph-mon[175151]: failed to encode map e223 with expected crc 2023-09-23T20:57:46.728 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:57:46 smithi026 ceph-mon[175151]: failed to encode map e224 with expected crc 2023-09-23T20:57:46.728 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:57:46 smithi026 ceph-mon[175151]: failed to encode map e224 with expected crc 2023-09-23T20:57:46.728 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:57:46 smithi026 ceph-mon[175151]: osdmap e224: 8 total, 8 up, 8 in 2023-09-23T20:57:46.728 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:57:46 smithi026 ceph-mon[175151]: failed to encode map e224 with expected crc 2023-09-23T20:57:46.728 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:57:46 smithi026 ceph-mon[175151]: failed to encode map e224 with expected crc 2023-09-23T20:57:46.729 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:57:46 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:57:46.729 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:57:46 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:57:46.729 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:57:46 smithi026 ceph-mon[179688]: failed to encode map e223 with expected crc 2023-09-23T20:57:46.729 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:57:46 smithi026 ceph-mon[179688]: failed to encode map e223 with expected crc 2023-09-23T20:57:46.729 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:57:46 smithi026 ceph-mon[179688]: failed to encode map e224 with expected crc 2023-09-23T20:57:46.729 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:57:46 smithi026 ceph-mon[179688]: failed to encode map e224 with expected crc 2023-09-23T20:57:46.729 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:57:46 smithi026 ceph-mon[179688]: osdmap e224: 8 total, 8 up, 8 in 2023-09-23T20:57:46.729 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:57:46 smithi026 ceph-mon[179688]: failed to encode map e224 with expected crc 2023-09-23T20:57:46.729 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:57:46 smithi026 ceph-mon[179688]: failed to encode map e224 with expected crc 2023-09-23T20:57:46.729 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:57:46 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:57:46.729 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:57:46 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:57:46.753 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:57:46.761 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:57:46.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:57:46 smithi070 ceph-mon[168282]: failed to encode map e223 with expected crc 2023-09-23T20:57:46.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:57:46 smithi070 ceph-mon[168282]: failed to encode map e223 with expected crc 2023-09-23T20:57:46.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:57:46 smithi070 ceph-mon[168282]: failed to encode map e224 with expected crc 2023-09-23T20:57:46.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:57:46 smithi070 ceph-mon[168282]: failed to encode map e224 with expected crc 2023-09-23T20:57:46.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:57:46 smithi070 ceph-mon[168282]: osdmap e224: 8 total, 8 up, 8 in 2023-09-23T20:57:46.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:57:46 smithi070 ceph-mon[168282]: failed to encode map e224 with expected crc 2023-09-23T20:57:46.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:57:46 smithi070 ceph-mon[168282]: failed to encode map e224 with expected crc 2023-09-23T20:57:46.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:57:46 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:57:46.871 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:57:46 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:57:46.875 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:57:46.943 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 68 16 87776 87760 80.6032 8.96875 0.047545 0.0123991 2023-09-23T20:57:47.151 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:57:47.169 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:57:47.170 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:57:47.260 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:57:47.271 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:57:47.299 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:57:47.337 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:57:47.340 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:57:47.340 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:57:47.441 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:57:47.569 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:57:47.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:57:47 smithi070 ceph-mon[168282]: failed to encode map e224 with expected crc 2023-09-23T20:57:47.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:57:47 smithi070 ceph-mon[168282]: failed to encode map e224 with expected crc 2023-09-23T20:57:47.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:57:47 smithi070 ceph-mon[168282]: pgmap v523: 109 pgs: 1 activating+undersized+degraded+remapped, 6 peering, 102 active+clean; 5.3 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 55 MiB/s wr, 877 op/s; 5398/260640 objects degraded (2.071%) 2023-09-23T20:57:47.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:57:47 smithi070 ceph-mon[168282]: Health check failed: Degraded data redundancy: 5398/260640 objects degraded (2.071%), 1 pg degraded (PG_DEGRADED) 2023-09-23T20:57:47.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:57:47 smithi070 ceph-mon[168282]: osdmap e225: 8 total, 8 up, 8 in 2023-09-23T20:57:47.668 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:57:47.672 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:57:47.672 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:57:47.773 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:57:47.850 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:57:47 smithi026 ceph-mon[179688]: failed to encode map e224 with expected crc 2023-09-23T20:57:47.850 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:57:47 smithi026 ceph-mon[179688]: failed to encode map e224 with expected crc 2023-09-23T20:57:47.850 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:57:47 smithi026 ceph-mon[179688]: pgmap v523: 109 pgs: 1 activating+undersized+degraded+remapped, 6 peering, 102 active+clean; 5.3 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 55 MiB/s wr, 877 op/s; 5398/260640 objects degraded (2.071%) 2023-09-23T20:57:47.850 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:57:47 smithi026 ceph-mon[179688]: Health check failed: Degraded data redundancy: 5398/260640 objects degraded (2.071%), 1 pg degraded (PG_DEGRADED) 2023-09-23T20:57:47.850 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:57:47 smithi026 ceph-mon[179688]: osdmap e225: 8 total, 8 up, 8 in 2023-09-23T20:57:47.851 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:57:47 smithi026 ceph-mon[175151]: failed to encode map e224 with expected crc 2023-09-23T20:57:47.851 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:57:47 smithi026 ceph-mon[175151]: failed to encode map e224 with expected crc 2023-09-23T20:57:47.851 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:57:47 smithi026 ceph-mon[175151]: pgmap v523: 109 pgs: 1 activating+undersized+degraded+remapped, 6 peering, 102 active+clean; 5.3 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 55 MiB/s wr, 877 op/s; 5398/260640 objects degraded (2.071%) 2023-09-23T20:57:47.851 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:57:47 smithi026 ceph-mon[175151]: Health check failed: Degraded data redundancy: 5398/260640 objects degraded (2.071%), 1 pg degraded (PG_DEGRADED) 2023-09-23T20:57:47.851 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:57:47 smithi026 ceph-mon[175151]: osdmap e225: 8 total, 8 up, 8 in 2023-09-23T20:57:47.857 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:57:47.952 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 69 16 88444 88428 80.0304 41.75 0.0437921 0.012489 2023-09-23T20:57:48.083 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T20:57:48.210 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:57:48.232 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:57:48.233 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:57:48.334 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:57:48.440 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:57:48.682 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:57:48.690 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:57:48.691 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:57:48 smithi026 ceph-mon[179688]: failed to encode map e225 with expected crc 2023-09-23T20:57:48.691 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:57:48 smithi026 ceph-mon[179688]: failed to encode map e225 with expected crc 2023-09-23T20:57:48.691 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:57:48 smithi026 ceph-mon[179688]: osdmap e226: 8 total, 8 up, 8 in 2023-09-23T20:57:48.691 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:57:48.692 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:57:48 smithi026 ceph-mon[175151]: failed to encode map e225 with expected crc 2023-09-23T20:57:48.692 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:57:48 smithi026 ceph-mon[175151]: failed to encode map e225 with expected crc 2023-09-23T20:57:48.692 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:57:48 smithi026 ceph-mon[175151]: osdmap e226: 8 total, 8 up, 8 in 2023-09-23T20:57:48.793 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:57:48.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:57:48 smithi070 ceph-mon[168282]: failed to encode map e225 with expected crc 2023-09-23T20:57:48.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:57:48 smithi070 ceph-mon[168282]: failed to encode map e225 with expected crc 2023-09-23T20:57:48.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:57:48 smithi070 ceph-mon[168282]: osdmap e226: 8 total, 8 up, 8 in 2023-09-23T20:57:48.884 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:57:48.952 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 70 16 89021 89005 79.4027 36.0625 0.0216316 0.0125871 2023-09-23T20:57:49.088 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:57:49.093 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:57:49.094 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:57:49.194 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:57:49.319 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:57:49.474 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:57:49.479 INFO:teuthology.orchestra.run.smithi026.stdout:ERROR: (22) Invalid argument 2023-09-23T20:57:49.479 INFO:teuthology.orchestra.run.smithi026.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:57:49.479 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:57:49.480 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:57:49.552 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:57:49.581 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:57:49.678 INFO:teuthology.orchestra.run.smithi026.stderr:nodeep-scrub is set 2023-09-23T20:57:49.727 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:57:49.872 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:57:49 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[164029]: ::ffff:172.21.15.70 - - [23/Sep/2023:20:57:49] "GET /metrics HTTP/1.1" 200 36293 "" "Prometheus/2.43.0" 2023-09-23T20:57:49.872 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:57:49 smithi026 ceph-mon[179688]: failed to encode map e226 with expected crc 2023-09-23T20:57:49.872 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:57:49 smithi026 ceph-mon[179688]: failed to encode map e226 with expected crc 2023-09-23T20:57:49.872 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:57:49 smithi026 ceph-mon[179688]: failed to encode map e226 with expected crc 2023-09-23T20:57:49.872 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:57:49 smithi026 ceph-mon[179688]: failed to encode map e226 with expected crc 2023-09-23T20:57:49.873 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:57:49 smithi026 ceph-mon[179688]: pgmap v526: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 activating+undersized+degraded+remapped, 6 peering, 101 active+clean; 5.3 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 20 MiB/s wr, 321 op/s; 10868/262653 objects degraded (4.138%); 16 KiB/s, 0 objects/s recovering 2023-09-23T20:57:49.873 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:57:49 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/1705760440' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:57:49.873 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:57:49 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:57:49.873 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:57:49 smithi026 ceph-mon[175151]: failed to encode map e226 with expected crc 2023-09-23T20:57:49.873 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:57:49 smithi026 ceph-mon[175151]: failed to encode map e226 with expected crc 2023-09-23T20:57:49.873 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:57:49 smithi026 ceph-mon[175151]: failed to encode map e226 with expected crc 2023-09-23T20:57:49.873 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:57:49 smithi026 ceph-mon[175151]: failed to encode map e226 with expected crc 2023-09-23T20:57:49.873 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:57:49 smithi026 ceph-mon[175151]: pgmap v526: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 activating+undersized+degraded+remapped, 6 peering, 101 active+clean; 5.3 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 20 MiB/s wr, 321 op/s; 10868/262653 objects degraded (4.138%); 16 KiB/s, 0 objects/s recovering 2023-09-23T20:57:49.874 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:57:49 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/1705760440' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:57:49.874 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:57:49 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:57:49.952 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 71 15 89586 89571 78.7829 35.375 0.014447 0.0126873 2023-09-23T20:57:50.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:57:49 smithi070 ceph-mon[168282]: failed to encode map e226 with expected crc 2023-09-23T20:57:50.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:57:49 smithi070 ceph-mon[168282]: failed to encode map e226 with expected crc 2023-09-23T20:57:50.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:57:49 smithi070 ceph-mon[168282]: failed to encode map e226 with expected crc 2023-09-23T20:57:50.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:57:49 smithi070 ceph-mon[168282]: failed to encode map e226 with expected crc 2023-09-23T20:57:50.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:57:49 smithi070 ceph-mon[168282]: pgmap v526: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 activating+undersized+degraded+remapped, 6 peering, 101 active+clean; 5.3 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 20 MiB/s wr, 321 op/s; 10868/262653 objects degraded (4.138%); 16 KiB/s, 0 objects/s recovering 2023-09-23T20:57:50.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:57:49 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/1705760440' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:57:50.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:57:49 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:57:50.121 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:57:50.131 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:57:50.131 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:57:50.149 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_blocked_ops 2023-09-23T20:57:50.232 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:57:50.264 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:57:50.305 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:57:50.310 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:57:50.311 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:57:50.411 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:57:50.522 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:57:50.723 INFO:teuthology.orchestra.run.smithi026.stderr:reweighted osd.6 to 1 (10000) 2023-09-23T20:57:50.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:57:50 smithi026 ceph-mon[175151]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:57:50.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:57:50 smithi026 ceph-mon[175151]: failed to encode map e227 with expected crc 2023-09-23T20:57:50.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:57:50 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T20:57:50.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:57:50 smithi026 ceph-mon[175151]: osdmap e227: 8 total, 8 up, 8 in 2023-09-23T20:57:50.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:57:50 smithi026 ceph-mon[175151]: failed to encode map e227 with expected crc 2023-09-23T20:57:50.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:57:50 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/144658171' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-09-23T20:57:50.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:57:50 smithi026 ceph-mon[179688]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:57:50.883 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:57:50 smithi026 ceph-mon[179688]: failed to encode map e227 with expected crc 2023-09-23T20:57:50.883 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:57:50 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T20:57:50.883 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:57:50 smithi026 ceph-mon[179688]: osdmap e227: 8 total, 8 up, 8 in 2023-09-23T20:57:50.883 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:57:50 smithi026 ceph-mon[179688]: failed to encode map e227 with expected crc 2023-09-23T20:57:50.883 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:57:50 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/144658171' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-09-23T20:57:50.956 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 72 16 90214 90198 78.2294 39.1875 0.0229363 0.0127777 2023-09-23T20:57:51.019 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:57:51.026 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:57:51.027 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:57:51.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:57:50 smithi070 ceph-mon[168282]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:57:51.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:57:50 smithi070 ceph-mon[168282]: failed to encode map e227 with expected crc 2023-09-23T20:57:51.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:57:50 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T20:57:51.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:57:50 smithi070 ceph-mon[168282]: osdmap e227: 8 total, 8 up, 8 in 2023-09-23T20:57:51.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:57:50 smithi070 ceph-mon[168282]: failed to encode map e227 with expected crc 2023-09-23T20:57:51.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:57:50 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/144658171' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-09-23T20:57:51.128 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:57:51.244 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:57:51.867 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:57:51.876 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:57:51.876 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:57:51.935 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:57:51 smithi026 ceph-mon[175151]: pgmap v528: 109 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 activating+undersized+degraded+remapped, 106 active+clean; 5.4 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 29 MiB/s wr, 462 op/s; 16341/264870 objects degraded (6.169%); 15 KiB/s, 0 objects/s recovering 2023-09-23T20:57:51.935 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:57:51 smithi026 ceph-mon[175151]: failed to encode map e228 with expected crc 2023-09-23T20:57:51.935 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:57:51 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/144658171' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-09-23T20:57:51.936 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:57:51 smithi026 ceph-mon[175151]: osdmap e228: 8 total, 8 up, 8 in 2023-09-23T20:57:51.936 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:57:51 smithi026 ceph-mon[175151]: failed to encode map e228 with expected crc 2023-09-23T20:57:51.936 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:57:51 smithi026 ceph-mon[175151]: failed to encode map e228 with expected crc 2023-09-23T20:57:51.936 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:57:51 smithi026 ceph-mon[179688]: pgmap v528: 109 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 activating+undersized+degraded+remapped, 106 active+clean; 5.4 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 29 MiB/s wr, 462 op/s; 16341/264870 objects degraded (6.169%); 15 KiB/s, 0 objects/s recovering 2023-09-23T20:57:51.936 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:57:51 smithi026 ceph-mon[179688]: failed to encode map e228 with expected crc 2023-09-23T20:57:51.936 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:57:51 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/144658171' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-09-23T20:57:51.936 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:57:51 smithi026 ceph-mon[179688]: osdmap e228: 8 total, 8 up, 8 in 2023-09-23T20:57:51.936 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:57:51 smithi026 ceph-mon[179688]: failed to encode map e228 with expected crc 2023-09-23T20:57:51.936 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:57:51 smithi026 ceph-mon[179688]: failed to encode map e228 with expected crc 2023-09-23T20:57:51.958 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 73 16 90914 90898 77.7545 43.75 0.018433 0.0128551 2023-09-23T20:57:51.965 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph osd reweight 7 1 2023-09-23T20:57:51.978 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:57:52.003 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:57:51 smithi070 ceph-mon[168282]: pgmap v528: 109 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 activating+undersized+degraded+remapped, 106 active+clean; 5.4 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 29 MiB/s wr, 462 op/s; 16341/264870 objects degraded (6.169%); 15 KiB/s, 0 objects/s recovering 2023-09-23T20:57:52.003 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:57:51 smithi070 ceph-mon[168282]: failed to encode map e228 with expected crc 2023-09-23T20:57:52.003 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:57:51 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/144658171' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-09-23T20:57:52.003 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:57:51 smithi070 ceph-mon[168282]: osdmap e228: 8 total, 8 up, 8 in 2023-09-23T20:57:52.003 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:57:51 smithi070 ceph-mon[168282]: failed to encode map e228 with expected crc 2023-09-23T20:57:52.003 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:57:51 smithi070 ceph-mon[168282]: failed to encode map e228 with expected crc 2023-09-23T20:57:52.005 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:57:52.041 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:57:52.044 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:57:52.044 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:57:52.145 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:57:52.226 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:57:52.272 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:57:52.352 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T20:57:52.486 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:57:52.489 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:57:52.490 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:57:52.590 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:57:52.621 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:57:52.656 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:57:52.658 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:57:52.659 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:57:52.751 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0: {} 2023-09-23T20:57:52.759 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:57:52.809 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1: {} 2023-09-23T20:57:52.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:57:52 smithi070 ceph-mon[168282]: Health check update: Degraded data redundancy: 10943/269823 objects degraded (4.056%), 2 pgs degraded (PG_DEGRADED) 2023-09-23T20:57:52.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:57:52 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/1456546558' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:57:52.872 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2: {} 2023-09-23T20:57:52.898 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:57:52.919 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3: {} 2023-09-23T20:57:52.958 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 74 16 91463 91447 77.1679 34.3125 0.0100962 0.0129515 2023-09-23T20:57:52.959 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4: {} 2023-09-23T20:57:52.999 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:57:53.005 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T20:57:53.009 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:57:53.009 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:57:52 smithi026 ceph-mon[175151]: Health check update: Degraded data redundancy: 10943/269823 objects degraded (4.056%), 2 pgs degraded (PG_DEGRADED) 2023-09-23T20:57:53.009 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:57:52 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/1456546558' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:57:53.009 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:57:52 smithi026 ceph-mon[179688]: Health check update: Degraded data redundancy: 10943/269823 objects degraded (4.056%), 2 pgs degraded (PG_DEGRADED) 2023-09-23T20:57:53.009 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:57:52 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/1456546558' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:57:53.010 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:57:53.053 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6: {} 2023-09-23T20:57:53.110 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:57:53.130 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T20:57:53.174 INFO:teuthology.orchestra.run.smithi026.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T20:57:53.174 INFO:teuthology.orchestra.run.smithi026.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T20:57:53.174 INFO:teuthology.orchestra.run.smithi026.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T20:57:53.174 INFO:teuthology.orchestra.run.smithi026.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T20:57:53.174 INFO:teuthology.orchestra.run.smithi026.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T20:57:53.174 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T20:57:53.174 INFO:teuthology.orchestra.run.smithi026.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T20:57:53.174 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T20:57:53.215 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:57:53.626 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:57:53.632 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:57:53.633 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:57:53.734 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:57:53.764 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:57:53.793 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:57:53.801 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:57:53.803 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:57:53.803 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:57:53.804 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:57:53.904 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:57:53.931 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:57:53.959 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 75 16 92101 92085 76.671 39.875 0.00698287 0.013037 2023-09-23T20:57:53.978 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:57:53.982 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:57:53.982 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:57:54.083 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:57:54.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:57:53 smithi070 ceph-mon[168282]: pgmap v530: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 5.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 29 MiB/s wr, 464 op/s; 10943/269823 objects degraded (4.056%); 5309/269823 objects misplaced (1.968%); 1.1 MiB/s, 17 objects/s recovering 2023-09-23T20:57:54.143 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:57:53 smithi026 ceph-mon[175151]: pgmap v530: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 5.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 29 MiB/s wr, 464 op/s; 10943/269823 objects degraded (4.056%); 5309/269823 objects misplaced (1.968%); 1.1 MiB/s, 17 objects/s recovering 2023-09-23T20:57:54.143 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:57:53 smithi026 ceph-mon[179688]: pgmap v530: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 5.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 29 MiB/s wr, 464 op/s; 10943/269823 objects degraded (4.056%); 5309/269823 objects misplaced (1.968%); 1.1 MiB/s, 17 objects/s recovering 2023-09-23T20:57:54.182 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:57:54.271 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:57:54.275 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:57:54.276 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:57:54.376 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:57:54.458 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:57:54.529 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:57:54.539 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:57:54.540 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:57:54.640 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:57:54.673 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:57:54.718 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:57:54.721 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:57:54.721 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:57:54.822 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:57:54.855 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:57:54.894 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:57:54.900 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:57:54.901 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:57:54.961 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 76 16 92764 92748 76.2049 41.4375 0.0240057 0.0131162 2023-09-23T20:57:55.002 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:57:55.109 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:57:55.142 INFO:teuthology.orchestra.run.smithi026.stdout:{ 2023-09-23T20:57:55.142 INFO:teuthology.orchestra.run.smithi026.stdout: "ops": [], 2023-09-23T20:57:55.142 INFO:teuthology.orchestra.run.smithi026.stdout: "complaint_time": 30, 2023-09-23T20:57:55.142 INFO:teuthology.orchestra.run.smithi026.stdout: "num_blocked_ops": 0 2023-09-23T20:57:55.142 INFO:teuthology.orchestra.run.smithi026.stdout:} 2023-09-23T20:57:55.521 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:57:55.535 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:57:55.536 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:57:55.637 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:57:55.696 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T20:57:55.735 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:57:55.889 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:57:55.915 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:57:55.916 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:57:55.942 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_historic_ops 2023-09-23T20:57:55.965 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 77 16 93408 93392 75.7369 40.25 0.0348326 0.0131986 2023-09-23T20:57:56.017 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:57:56.056 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:57:55 smithi026 ceph-mon[175151]: pgmap v531: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 5.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 33 MiB/s wr, 521 op/s; 10943/272232 objects degraded (4.020%); 5309/272232 objects misplaced (1.950%); 930 KiB/s, 14 objects/s recovering 2023-09-23T20:57:56.056 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:57:55 smithi026 ceph-mon[179688]: pgmap v531: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 5.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 33 MiB/s wr, 521 op/s; 10943/272232 objects degraded (4.020%); 5309/272232 objects misplaced (1.950%); 930 KiB/s, 14 objects/s recovering 2023-09-23T20:57:56.126 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:57:56.267 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:57:56.283 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:57:56.284 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:57:56.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:57:55 smithi070 ceph-mon[168282]: pgmap v531: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 5.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 33 MiB/s wr, 521 op/s; 10943/272232 objects degraded (4.020%); 5309/272232 objects misplaced (1.950%); 930 KiB/s, 14 objects/s recovering 2023-09-23T20:57:56.384 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:57:56.412 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:57:56.447 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:57:56.451 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:57:56.451 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:57:56.552 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:57:56.663 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:57:56.964 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 78 16 93949 93933 75.1986 33.8125 0.0205286 0.0132919 2023-09-23T20:57:56.974 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:57:56.978 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:57:56.978 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:57:57.079 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:57:57.145 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:57:57.183 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:57:57.248 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:57:57.261 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:57:57.261 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:57:57.362 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:57:57.421 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:57:57.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:57:57 smithi026 ceph-mon[179688]: pgmap v532: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 5.6 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 36 MiB/s wr, 571 op/s; 10943/276372 objects degraded (3.960%); 5208/276372 objects misplaced (1.884%); 1.5 MiB/s, 23 objects/s recovering 2023-09-23T20:57:57.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:57:57 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:57:57.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:57:57 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/4019830360' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-09-23T20:57:57.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:57:57 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-09-23T20:57:57.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:57:57 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/4140345139' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:57:57.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:57:57 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:57:57.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:57:57 smithi026 ceph-mon[175151]: pgmap v532: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 5.6 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 36 MiB/s wr, 571 op/s; 10943/276372 objects degraded (3.960%); 5208/276372 objects misplaced (1.884%); 1.5 MiB/s, 23 objects/s recovering 2023-09-23T20:57:57.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:57:57 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:57:57.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:57:57 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/4019830360' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-09-23T20:57:57.633 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:57:57 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-09-23T20:57:57.633 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:57:57 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/4140345139' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:57:57.633 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:57:57 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:57:57.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:57:57 smithi070 ceph-mon[168282]: pgmap v532: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 5.6 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 36 MiB/s wr, 571 op/s; 10943/276372 objects degraded (3.960%); 5208/276372 objects misplaced (1.884%); 1.5 MiB/s, 23 objects/s recovering 2023-09-23T20:57:57.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:57:57 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:57:57.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:57:57 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/4019830360' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-09-23T20:57:57.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:57:57 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-09-23T20:57:57.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:57:57 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/4140345139' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:57:57.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:57:57 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:57:57.964 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 79 16 94562 94546 74.7319 38.3125 0.0184401 0.0133754 2023-09-23T20:57:58.034 INFO:teuthology.orchestra.run.smithi026.stderr:reweighted osd.7 to 1 (10000) 2023-09-23T20:57:58.044 INFO:teuthology.orchestra.run.smithi026.stderr:noscrub is unset 2023-09-23T20:57:58.204 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:57:58.211 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:57:58.216 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:57:58.217 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:57:58.317 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:57:58.451 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:57:58.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:57:58 smithi026 ceph-mon[179688]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:57:58.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:57:58 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-09-23T20:57:58.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:57:58 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T20:57:58.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:57:58 smithi026 ceph-mon[179688]: osdmap e229: 8 total, 8 up, 8 in 2023-09-23T20:57:58.633 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:57:58 smithi026 ceph-mon[179688]: failed to encode map e229 with expected crc 2023-09-23T20:57:58.633 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:57:58 smithi026 ceph-mon[179688]: failed to encode map e229 with expected crc 2023-09-23T20:57:58.633 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:57:58 smithi026 ceph-mon[179688]: failed to encode map e229 with expected crc 2023-09-23T20:57:58.633 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:57:58 smithi026 ceph-mon[175151]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:57:58.633 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:57:58 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-09-23T20:57:58.633 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:57:58 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T20:57:58.633 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:57:58 smithi026 ceph-mon[175151]: osdmap e229: 8 total, 8 up, 8 in 2023-09-23T20:57:58.633 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:57:58 smithi026 ceph-mon[175151]: failed to encode map e229 with expected crc 2023-09-23T20:57:58.634 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:57:58 smithi026 ceph-mon[175151]: failed to encode map e229 with expected crc 2023-09-23T20:57:58.634 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:57:58 smithi026 ceph-mon[175151]: failed to encode map e229 with expected crc 2023-09-23T20:57:58.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:57:58 smithi070 ceph-mon[168282]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:57:58.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:57:58 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-09-23T20:57:58.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:57:58 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T20:57:58.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:57:58 smithi070 ceph-mon[168282]: osdmap e229: 8 total, 8 up, 8 in 2023-09-23T20:57:58.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:57:58 smithi070 ceph-mon[168282]: failed to encode map e229 with expected crc 2023-09-23T20:57:58.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:57:58 smithi070 ceph-mon[168282]: failed to encode map e229 with expected crc 2023-09-23T20:57:58.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:57:58 smithi070 ceph-mon[168282]: failed to encode map e229 with expected crc 2023-09-23T20:57:58.965 INFO:tasks.radosbench.radosbench.0.smithi026.stdout:2023-09-23T20:57:58.965493+0000 min lat: 0.0011341 max lat: 2.507 avg lat: 0.0134507 2023-09-23T20:57:58.965 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T20:57:58.965 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 80 16 95217 95201 74.3097 40.9375 0.0206163 0.0134507 2023-09-23T20:57:59.109 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:57:59.112 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:57:59.113 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:57:59.213 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:57:59.304 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:57:59.316 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph osd reweight 4 1 2023-09-23T20:57:59.480 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:57:59.488 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:57:59.489 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:57:59.590 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:57:59.736 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:57:59 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[164029]: ::ffff:172.21.15.70 - - [23/Sep/2023:20:57:59] "GET /metrics HTTP/1.1" 200 36293 "" "Prometheus/2.43.0" 2023-09-23T20:57:59.739 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:57:59 smithi026 ceph-mon[175151]: failed to encode map e229 with expected crc 2023-09-23T20:57:59.739 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:57:59 smithi026 ceph-mon[175151]: failed to encode map e229 with expected crc 2023-09-23T20:57:59.739 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:57:59 smithi026 ceph-mon[175151]: pgmap v534: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 5.7 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 37 MiB/s wr, 594 op/s; 10943/279159 objects degraded (3.920%); 5208/279159 objects misplaced (1.866%); 1.5 MiB/s, 23 objects/s recovering 2023-09-23T20:57:59.739 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:57:59 smithi026 ceph-mon[175151]: Health check update: Degraded data redundancy: 10943/279159 objects degraded (3.920%), 2 pgs degraded (PG_DEGRADED) 2023-09-23T20:57:59.739 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:57:59 smithi026 ceph-mon[179688]: failed to encode map e229 with expected crc 2023-09-23T20:57:59.739 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:57:59 smithi026 ceph-mon[179688]: failed to encode map e229 with expected crc 2023-09-23T20:57:59.740 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:57:59 smithi026 ceph-mon[179688]: pgmap v534: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 5.7 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 37 MiB/s wr, 594 op/s; 10943/279159 objects degraded (3.920%); 5208/279159 objects misplaced (1.866%); 1.5 MiB/s, 23 objects/s recovering 2023-09-23T20:57:59.740 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:57:59 smithi026 ceph-mon[179688]: Health check update: Degraded data redundancy: 10943/279159 objects degraded (3.920%), 2 pgs degraded (PG_DEGRADED) 2023-09-23T20:57:59.744 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:57:59.844 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:57:59.851 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:57:59.851 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:57:59.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:57:59 smithi070 ceph-mon[168282]: failed to encode map e229 with expected crc 2023-09-23T20:57:59.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:57:59 smithi070 ceph-mon[168282]: failed to encode map e229 with expected crc 2023-09-23T20:57:59.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:57:59 smithi070 ceph-mon[168282]: pgmap v534: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 5.7 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 37 MiB/s wr, 594 op/s; 10943/279159 objects degraded (3.920%); 5208/279159 objects misplaced (1.866%); 1.5 MiB/s, 23 objects/s recovering 2023-09-23T20:57:59.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:57:59 smithi070 ceph-mon[168282]: Health check update: Degraded data redundancy: 10943/279159 objects degraded (3.920%), 2 pgs degraded (PG_DEGRADED) 2023-09-23T20:57:59.952 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:57:59.974 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 81 16 95835 95819 73.8614 38.625 0.0111632 0.0135328 2023-09-23T20:57:59.983 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:58:00.021 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:58:00.025 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:58:00.025 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:58:00.126 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:58:00.156 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:58:00.195 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:58:00.199 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:58:00.200 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:58:00.300 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:58:00.333 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:58:00.361 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:58:00.370 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:58:00.374 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:58:00.375 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:58:00.475 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:58:00.565 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0: {} 2023-09-23T20:58:00.592 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:58:00.592 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1: {} 2023-09-23T20:58:00.636 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2: {} 2023-09-23T20:58:00.679 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:58:00.683 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:58:00.684 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:58:00.684 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3: {} 2023-09-23T20:58:00.741 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4: {} 2023-09-23T20:58:00.785 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:58:00.801 INFO:teuthology.orchestra.run.smithi026.stdout:ERROR: (22) Invalid argument 2023-09-23T20:58:00.801 INFO:teuthology.orchestra.run.smithi026.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:58:00.809 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T20:58:00.864 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6: {} 2023-09-23T20:58:00.918 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T20:58:00.921 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:58:00.939 INFO:teuthology.orchestra.run.smithi026.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T20:58:00.940 INFO:teuthology.orchestra.run.smithi026.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T20:58:00.940 INFO:teuthology.orchestra.run.smithi026.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T20:58:00.940 INFO:teuthology.orchestra.run.smithi026.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T20:58:00.940 INFO:teuthology.orchestra.run.smithi026.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T20:58:00.940 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T20:58:00.940 INFO:teuthology.orchestra.run.smithi026.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T20:58:00.940 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T20:58:00.974 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 82 16 96407 96391 73.3967 35.75 0.0293828 0.0136189 2023-09-23T20:58:01.098 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:58:01 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/3354028950' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:58:01.099 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:58:01 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/3354028950' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:58:01.218 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T20:58:01.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:58:01 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/3354028950' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:58:01.499 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_ops_in_flight 2023-09-23T20:58:01.688 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:58:01.695 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:58:01.696 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:58:01.797 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:58:01.888 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:58:01.974 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 83 16 97017 97001 72.9721 38.125 0.0269994 0.0136976 2023-09-23T20:58:02.023 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:58:02 smithi026 ceph-mon[175151]: pgmap v535: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 5.8 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 40 MiB/s wr, 634 op/s; 10943/283305 objects degraded (3.863%); 5208/283305 objects misplaced (1.838%); 1.2 MiB/s, 19 objects/s recovering 2023-09-23T20:58:02.023 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:58:02 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:58:02.023 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:58:02 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:58:02.023 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:58:02 smithi026 ceph-mon[179688]: pgmap v535: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 5.8 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 40 MiB/s wr, 634 op/s; 10943/283305 objects degraded (3.863%); 5208/283305 objects misplaced (1.838%); 1.2 MiB/s, 19 objects/s recovering 2023-09-23T20:58:02.023 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:58:02 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:58:02.023 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:58:02 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:58:02.027 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:58:02.033 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:58:02.034 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:58:02.135 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:58:02.163 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:58:02.203 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:58:02.206 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:58:02.206 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:58:02.307 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:58:02.333 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:58:02 smithi070 ceph-mon[168282]: pgmap v535: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 5.8 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 40 MiB/s wr, 634 op/s; 10943/283305 objects degraded (3.863%); 5208/283305 objects misplaced (1.838%); 1.2 MiB/s, 19 objects/s recovering 2023-09-23T20:58:02.334 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:58:02 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:58:02.334 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:58:02 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:58:02.336 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:58:02.378 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:58:02.380 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:58:02.380 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:58:02.481 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:58:02.586 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:58:02.686 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:58:02.696 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:58:02.697 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:58:02.697 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:58:02.798 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:58:02.805 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:58:02.826 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:58:02.866 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:58:02.869 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:58:02.870 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:58:02.970 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:58:02.974 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 84 16 97613 97597 72.5472 37.25 0.00856329 0.013779 2023-09-23T20:58:02.998 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:58:03.041 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:58:03.045 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:58:03.045 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:58:03.047 INFO:teuthology.orchestra.run.smithi026.stderr:reweighted osd.4 to 1 (10000) 2023-09-23T20:58:03.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:58:03 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/255860523' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-09-23T20:58:03.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:58:03 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-09-23T20:58:03.145 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:58:03.180 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:58:03.226 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:58:03.229 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:58:03.230 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:58:03.315 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:58:03 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/255860523' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-09-23T20:58:03.315 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:58:03 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-09-23T20:58:03.315 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:58:03 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/255860523' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-09-23T20:58:03.316 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:58:03 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-09-23T20:58:03.331 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:58:03.428 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:58:03.980 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 85 16 98272 98256 72.1765 41.1875 0.0198047 0.0138494 2023-09-23T20:58:04.082 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T20:58:04.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:58:04 smithi070 ceph-mon[168282]: pgmap v536: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 5.9 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 39 MiB/s wr, 620 op/s; 10943/288465 objects degraded (3.794%); 5104/288465 objects misplaced (1.769%); 1.3 MiB/s, 20 objects/s recovering 2023-09-23T20:58:04.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:58:04 smithi070 ceph-mon[168282]: failed to encode map e230 with expected crc 2023-09-23T20:58:04.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:58:04 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-09-23T20:58:04.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:58:04 smithi070 ceph-mon[168282]: osdmap e230: 8 total, 8 up, 8 in 2023-09-23T20:58:04.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:58:04 smithi070 ceph-mon[168282]: failed to encode map e230 with expected crc 2023-09-23T20:58:04.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:58:04 smithi070 ceph-mon[168282]: failed to encode map e230 with expected crc 2023-09-23T20:58:04.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:58:04 smithi070 ceph-mon[168282]: failed to encode map e230 with expected crc 2023-09-23T20:58:04.381 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:58:04 smithi026 ceph-mon[175151]: pgmap v536: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 5.9 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 39 MiB/s wr, 620 op/s; 10943/288465 objects degraded (3.794%); 5104/288465 objects misplaced (1.769%); 1.3 MiB/s, 20 objects/s recovering 2023-09-23T20:58:04.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:58:04 smithi026 ceph-mon[175151]: failed to encode map e230 with expected crc 2023-09-23T20:58:04.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:58:04 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-09-23T20:58:04.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:58:04 smithi026 ceph-mon[175151]: osdmap e230: 8 total, 8 up, 8 in 2023-09-23T20:58:04.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:58:04 smithi026 ceph-mon[175151]: failed to encode map e230 with expected crc 2023-09-23T20:58:04.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:58:04 smithi026 ceph-mon[175151]: failed to encode map e230 with expected crc 2023-09-23T20:58:04.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:58:04 smithi026 ceph-mon[175151]: failed to encode map e230 with expected crc 2023-09-23T20:58:04.383 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:58:04 smithi026 ceph-mon[179688]: pgmap v536: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 5.9 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 39 MiB/s wr, 620 op/s; 10943/288465 objects degraded (3.794%); 5104/288465 objects misplaced (1.769%); 1.3 MiB/s, 20 objects/s recovering 2023-09-23T20:58:04.383 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:58:04 smithi026 ceph-mon[179688]: failed to encode map e230 with expected crc 2023-09-23T20:58:04.383 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:58:04 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-09-23T20:58:04.383 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:58:04 smithi026 ceph-mon[179688]: osdmap e230: 8 total, 8 up, 8 in 2023-09-23T20:58:04.383 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:58:04 smithi026 ceph-mon[179688]: failed to encode map e230 with expected crc 2023-09-23T20:58:04.383 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:58:04 smithi026 ceph-mon[179688]: failed to encode map e230 with expected crc 2023-09-23T20:58:04.383 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:58:04 smithi026 ceph-mon[179688]: failed to encode map e230 with expected crc 2023-09-23T20:58:04.981 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 86 16 98979 98963 71.848 44.1875 0.0143166 0.0139133 2023-09-23T20:58:05.684 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:58:05.690 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:58:05.690 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:58:05.791 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:58:05.881 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:58:05.984 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 87 16 99700 99684 71.5379 45.0625 0.0174018 0.0139731 2023-09-23T20:58:05.996 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:58:05 smithi026 ceph-mon[179688]: pgmap v538: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 5.9 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 38 MiB/s wr, 601 op/s; 10943/290832 objects degraded (3.763%); 5104/290832 objects misplaced (1.755%); 831 KiB/s, 12 objects/s recovering 2023-09-23T20:58:05.996 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:58:05 smithi026 ceph-mon[179688]: Health check update: Degraded data redundancy: 10943/288465 objects degraded (3.794%), 2 pgs degraded (PG_DEGRADED) 2023-09-23T20:58:06.347 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:58:05 smithi026 ceph-mon[175151]: pgmap v538: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 5.9 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 38 MiB/s wr, 601 op/s; 10943/290832 objects degraded (3.763%); 5104/290832 objects misplaced (1.755%); 831 KiB/s, 12 objects/s recovering 2023-09-23T20:58:06.347 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:58:05 smithi026 ceph-mon[175151]: Health check update: Degraded data redundancy: 10943/288465 objects degraded (3.794%), 2 pgs degraded (PG_DEGRADED) 2023-09-23T20:58:06.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:58:05 smithi070 ceph-mon[168282]: pgmap v538: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 5.9 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 38 MiB/s wr, 601 op/s; 10943/290832 objects degraded (3.763%); 5104/290832 objects misplaced (1.755%); 831 KiB/s, 12 objects/s recovering 2023-09-23T20:58:06.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:58:05 smithi070 ceph-mon[168282]: Health check update: Degraded data redundancy: 10943/288465 objects degraded (3.794%), 2 pgs degraded (PG_DEGRADED) 2023-09-23T20:58:06.986 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 88 16 100412 100396 71.2309 44.5 0.0248133 0.0140329 2023-09-23T20:58:07.987 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 89 16 101133 101117 70.9344 45.0625 0.00589721 0.0140909 2023-09-23T20:58:08.004 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph osd reweight 5 1 2023-09-23T20:58:08.324 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:58:08.336 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:58:08.336 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:58:08.337 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:58:07 smithi026 ceph-mon[175151]: pgmap v539: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 6.0 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 39 MiB/s wr, 628 op/s; 10943/294912 objects degraded (3.711%); 5002/294912 objects misplaced (1.696%); 1.5 MiB/s, 24 objects/s recovering 2023-09-23T20:58:08.338 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:58:07 smithi026 ceph-mon[179688]: pgmap v539: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 6.0 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 39 MiB/s wr, 628 op/s; 10943/294912 objects degraded (3.711%); 5002/294912 objects misplaced (1.696%); 1.5 MiB/s, 24 objects/s recovering 2023-09-23T20:58:08.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:58:07 smithi070 ceph-mon[168282]: pgmap v539: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 6.0 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 39 MiB/s wr, 628 op/s; 10943/294912 objects degraded (3.711%); 5002/294912 objects misplaced (1.696%); 1.5 MiB/s, 24 objects/s recovering 2023-09-23T20:58:08.437 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:58:08.470 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:58:08.511 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:58:08.513 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:58:08.514 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:58:08.615 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:58:08.705 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:58:08.988 INFO:tasks.radosbench.radosbench.0.smithi026.stdout:Total time run: 90.0145 2023-09-23T20:58:08.988 INFO:tasks.radosbench.radosbench.0.smithi026.stdout:Total writes made: 101759 2023-09-23T20:58:08.988 INFO:tasks.radosbench.radosbench.0.smithi026.stdout:Write size: 65536 2023-09-23T20:58:08.988 INFO:tasks.radosbench.radosbench.0.smithi026.stdout:Object size: 65536 2023-09-23T20:58:08.988 INFO:tasks.radosbench.radosbench.0.smithi026.stdout:Bandwidth (MB/sec): 70.6546 2023-09-23T20:58:08.988 INFO:tasks.radosbench.radosbench.0.smithi026.stdout:Stddev Bandwidth: 37.1498 2023-09-23T20:58:08.988 INFO:tasks.radosbench.radosbench.0.smithi026.stdout:Max bandwidth (MB/sec): 214.688 2023-09-23T20:58:08.988 INFO:tasks.radosbench.radosbench.0.smithi026.stdout:Min bandwidth (MB/sec): 0 2023-09-23T20:58:08.988 INFO:tasks.radosbench.radosbench.0.smithi026.stdout:Average IOPS: 1130 2023-09-23T20:58:08.988 INFO:tasks.radosbench.radosbench.0.smithi026.stdout:Stddev IOPS: 594.406 2023-09-23T20:58:08.988 INFO:tasks.radosbench.radosbench.0.smithi026.stdout:Max IOPS: 3435 2023-09-23T20:58:08.989 INFO:tasks.radosbench.radosbench.0.smithi026.stdout:Min IOPS: 0 2023-09-23T20:58:08.989 INFO:tasks.radosbench.radosbench.0.smithi026.stdout:Average Latency(s): 0.0141493 2023-09-23T20:58:08.989 INFO:tasks.radosbench.radosbench.0.smithi026.stdout:Stddev Latency(s): 0.0337388 2023-09-23T20:58:08.989 INFO:tasks.radosbench.radosbench.0.smithi026.stdout:Max latency(s): 2.507 2023-09-23T20:58:08.989 INFO:tasks.radosbench.radosbench.0.smithi026.stdout:Min latency(s): 0.0011341 2023-09-23T20:58:08.992 INFO:tasks.radosbench.radosbench.0.smithi026.stdout:Cleaning up (deleting benchmark objects) 2023-09-23T20:58:09.074 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:58:09.081 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:58:09.082 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:58:09.182 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:58:09.263 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:58:09.453 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:58:09.882 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:58:09 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[164029]: ::ffff:172.21.15.70 - - [23/Sep/2023:20:58:09] "GET /metrics HTTP/1.1" 200 36317 "" "Prometheus/2.43.0" 2023-09-23T20:58:10.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:58:10 smithi070 ceph-mon[168282]: pgmap v540: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 6.1 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 40 MiB/s wr, 633 op/s; 10943/298167 objects degraded (3.670%); 5002/298167 objects misplaced (1.678%); 1.3 MiB/s, 20 objects/s recovering 2023-09-23T20:58:10.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:58:10 smithi070 ceph-mon[168282]: Health check update: Degraded data redundancy: 10943/298167 objects degraded (3.670%), 2 pgs degraded (PG_DEGRADED) 2023-09-23T20:58:10.384 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:58:10 smithi026 ceph-mon[175151]: pgmap v540: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 6.1 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 40 MiB/s wr, 633 op/s; 10943/298167 objects degraded (3.670%); 5002/298167 objects misplaced (1.678%); 1.3 MiB/s, 20 objects/s recovering 2023-09-23T20:58:10.384 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:58:10 smithi026 ceph-mon[175151]: Health check update: Degraded data redundancy: 10943/298167 objects degraded (3.670%), 2 pgs degraded (PG_DEGRADED) 2023-09-23T20:58:10.386 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:58:10 smithi026 ceph-mon[179688]: pgmap v540: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 6.1 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 40 MiB/s wr, 633 op/s; 10943/298167 objects degraded (3.670%); 5002/298167 objects misplaced (1.678%); 1.3 MiB/s, 20 objects/s recovering 2023-09-23T20:58:10.386 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:58:10 smithi026 ceph-mon[179688]: Health check update: Degraded data redundancy: 10943/298167 objects degraded (3.670%), 2 pgs degraded (PG_DEGRADED) 2023-09-23T20:58:10.405 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:58:10.418 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:58:10.418 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:58:10.519 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:58:10.613 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:58:10.703 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:58:10.709 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:58:10.710 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:58:10.810 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:58:10.933 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:58:10.961 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:58:10.991 INFO:teuthology.orchestra.run.smithi026.stdout:ERROR: (22) Invalid argument 2023-09-23T20:58:10.991 INFO:teuthology.orchestra.run.smithi026.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:58:11.105 INFO:teuthology.orchestra.run.smithi026.stderr:nodeep-scrub is unset 2023-09-23T20:58:11.277 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:58:11 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/2839856534' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:58:11.277 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:58:11 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:58:11.277 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:58:11 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/2839856534' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:58:11.277 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:58:11 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:58:11.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:58:11 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/2839856534' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:58:11.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:58:11 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:58:11.847 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_blocked_ops 2023-09-23T20:58:12.109 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:58:12.116 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:58:12.117 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:58:12.118 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:58:12 smithi026 ceph-mon[175151]: pgmap v541: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 6.1 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 102 B/s rd, 40 MiB/s wr, 645 op/s; 10943/302229 objects degraded (3.621%); 5002/302229 objects misplaced (1.655%); 1.3 MiB/s, 20 objects/s recovering 2023-09-23T20:58:12.118 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:58:12 smithi026 ceph-mon[175151]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T20:58:12.118 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:58:12 smithi026 ceph-mon[175151]: failed to encode map e231 with expected crc 2023-09-23T20:58:12.118 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:58:12 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T20:58:12.118 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:58:12 smithi026 ceph-mon[175151]: osdmap e231: 8 total, 8 up, 8 in 2023-09-23T20:58:12.118 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:58:12 smithi026 ceph-mon[175151]: failed to encode map e231 with expected crc 2023-09-23T20:58:12.119 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:58:12 smithi026 ceph-mon[175151]: failed to encode map e231 with expected crc 2023-09-23T20:58:12.119 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:58:12 smithi026 ceph-mon[179688]: pgmap v541: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 6.1 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 102 B/s rd, 40 MiB/s wr, 645 op/s; 10943/302229 objects degraded (3.621%); 5002/302229 objects misplaced (1.655%); 1.3 MiB/s, 20 objects/s recovering 2023-09-23T20:58:12.119 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:58:12 smithi026 ceph-mon[179688]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T20:58:12.119 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:58:12 smithi026 ceph-mon[179688]: failed to encode map e231 with expected crc 2023-09-23T20:58:12.119 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:58:12 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T20:58:12.119 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:58:12 smithi026 ceph-mon[179688]: osdmap e231: 8 total, 8 up, 8 in 2023-09-23T20:58:12.119 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:58:12 smithi026 ceph-mon[179688]: failed to encode map e231 with expected crc 2023-09-23T20:58:12.119 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:58:12 smithi026 ceph-mon[179688]: failed to encode map e231 with expected crc 2023-09-23T20:58:12.218 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:58:12.245 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:58:12.283 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:58:12.287 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:58:12.287 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:58:12.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:58:12 smithi070 ceph-mon[168282]: pgmap v541: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 6.1 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 102 B/s rd, 40 MiB/s wr, 645 op/s; 10943/302229 objects degraded (3.621%); 5002/302229 objects misplaced (1.655%); 1.3 MiB/s, 20 objects/s recovering 2023-09-23T20:58:12.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:58:12 smithi070 ceph-mon[168282]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T20:58:12.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:58:12 smithi070 ceph-mon[168282]: failed to encode map e231 with expected crc 2023-09-23T20:58:12.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:58:12 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T20:58:12.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:58:12 smithi070 ceph-mon[168282]: osdmap e231: 8 total, 8 up, 8 in 2023-09-23T20:58:12.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:58:12 smithi070 ceph-mon[168282]: failed to encode map e231 with expected crc 2023-09-23T20:58:12.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:58:12 smithi070 ceph-mon[168282]: failed to encode map e231 with expected crc 2023-09-23T20:58:12.388 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:58:12.497 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:58:13.002 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:58:13.009 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:58:13.009 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:58:13.110 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:58:13.138 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:58:13.173 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:58:13.183 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:58:13.183 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:58:13.183 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0: {} 2023-09-23T20:58:13.230 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1: {} 2023-09-23T20:58:13.268 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2: {} 2023-09-23T20:58:13.283 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:58:13.294 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:58:13.308 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3: {} 2023-09-23T20:58:13.312 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:58:13.354 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:58:13.356 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:58:13.357 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:58:13.357 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4: {} 2023-09-23T20:58:13.396 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T20:58:13.447 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6: {} 2023-09-23T20:58:13.458 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:58:13.483 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T20:58:13.491 INFO:teuthology.orchestra.run.smithi026.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T20:58:13.491 INFO:teuthology.orchestra.run.smithi026.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T20:58:13.491 INFO:teuthology.orchestra.run.smithi026.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T20:58:13.492 INFO:teuthology.orchestra.run.smithi026.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T20:58:13.492 INFO:teuthology.orchestra.run.smithi026.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T20:58:13.492 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T20:58:13.492 INFO:teuthology.orchestra.run.smithi026.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T20:58:13.492 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T20:58:13.575 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:58:13.816 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:58:13.823 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:58:13.824 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:58:13.924 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:58:14.010 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:58:14.152 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:58:14.161 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:58:14.161 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:58:14.262 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:58:14 smithi026 ceph-mon[175151]: pgmap v543: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 6.2 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 109 B/s rd, 38 MiB/s wr, 706 op/s; 10943/302430 objects degraded (3.618%); 4882/302430 objects misplaced (1.614%); 1.5 MiB/s, 23 objects/s recovering 2023-09-23T20:58:14.262 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:58:14 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/2088749624' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:58:14.262 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:58:14 smithi026 ceph-mon[179688]: pgmap v543: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 6.2 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 109 B/s rd, 38 MiB/s wr, 706 op/s; 10943/302430 objects degraded (3.618%); 4882/302430 objects misplaced (1.614%); 1.5 MiB/s, 23 objects/s recovering 2023-09-23T20:58:14.262 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:58:14 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/2088749624' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:58:14.262 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:58:14.348 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:58:14.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:58:14 smithi070 ceph-mon[168282]: pgmap v543: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 6.2 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 109 B/s rd, 38 MiB/s wr, 706 op/s; 10943/302430 objects degraded (3.618%); 4882/302430 objects misplaced (1.614%); 1.5 MiB/s, 23 objects/s recovering 2023-09-23T20:58:14.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:58:14 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/2088749624' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:58:14.378 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T20:58:14.543 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:58:14.560 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:58:14.560 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:58:14.661 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:58:14.788 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:58:14.924 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:58:14.930 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:58:14.930 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:58:15.031 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:58:15.057 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:58:15.094 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:58:15.097 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:58:15.097 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:58:15.142 INFO:teuthology.orchestra.run.smithi026.stderr:reweighted osd.5 to 1 (10000) 2023-09-23T20:58:15.180 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:58:15 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/2471809530' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-09-23T20:58:15.180 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:58:15 smithi026 ceph-mon[179688]: Health check update: Degraded data redundancy: 10943/302430 objects degraded (3.618%), 2 pgs degraded (PG_DEGRADED) 2023-09-23T20:58:15.181 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:58:15 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/2471809530' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-09-23T20:58:15.181 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:58:15 smithi026 ceph-mon[175151]: Health check update: Degraded data redundancy: 10943/302430 objects degraded (3.618%), 2 pgs degraded (PG_DEGRADED) 2023-09-23T20:58:15.198 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:58:15.229 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:58:15.269 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:58:15.272 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:58:15.272 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:58:15.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:58:15 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/2471809530' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-09-23T20:58:15.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:58:15 smithi070 ceph-mon[168282]: Health check update: Degraded data redundancy: 10943/302430 objects degraded (3.618%), 2 pgs degraded (PG_DEGRADED) 2023-09-23T20:58:15.373 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:58:15.435 INFO:teuthology.orchestra.run.smithi026.stdout:{ 2023-09-23T20:58:15.435 INFO:teuthology.orchestra.run.smithi026.stdout: "ops": [], 2023-09-23T20:58:15.435 INFO:teuthology.orchestra.run.smithi026.stdout: "complaint_time": 30, 2023-09-23T20:58:15.435 INFO:teuthology.orchestra.run.smithi026.stdout: "num_blocked_ops": 0 2023-09-23T20:58:15.435 INFO:teuthology.orchestra.run.smithi026.stdout:} 2023-09-23T20:58:15.463 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:58:15.728 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:58:15.735 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:58:15.736 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:58:15.836 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:58:15.864 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:58:15.905 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:58:15.908 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:58:15.909 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:58:15.978 INFO:tasks.cephadm.ceph_manager.ceph:waiting for recovery to complete 2023-09-23T20:58:15.979 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T20:58:16.009 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:58:16.038 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:58:16.084 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:58:16.087 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:58:16.087 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:58:16.098 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:58:16.188 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:58:16.195 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_historic_ops 2023-09-23T20:58:16.278 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:58:16 smithi026 ceph-mon[175151]: pgmap v544: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 6.1 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 102 B/s rd, 30 MiB/s wr, 662 op/s; 10943/299991 objects degraded (3.648%); 4882/299991 objects misplaced (1.627%); 1.4 MiB/s, 22 objects/s recovering 2023-09-23T20:58:16.279 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:58:16 smithi026 ceph-mon[175151]: failed to encode map e232 with expected crc 2023-09-23T20:58:16.279 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:58:16 smithi026 ceph-mon[175151]: failed to encode map e232 with expected crc 2023-09-23T20:58:16.279 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:58:16 smithi026 ceph-mon[175151]: failed to encode map e232 with expected crc 2023-09-23T20:58:16.279 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:58:16 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/2471809530' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-09-23T20:58:16.279 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:58:16 smithi026 ceph-mon[175151]: osdmap e232: 8 total, 8 up, 8 in 2023-09-23T20:58:16.279 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:58:16 smithi026 ceph-mon[179688]: pgmap v544: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 6.1 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 102 B/s rd, 30 MiB/s wr, 662 op/s; 10943/299991 objects degraded (3.648%); 4882/299991 objects misplaced (1.627%); 1.4 MiB/s, 22 objects/s recovering 2023-09-23T20:58:16.279 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:58:16 smithi026 ceph-mon[179688]: failed to encode map e232 with expected crc 2023-09-23T20:58:16.279 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:58:16 smithi026 ceph-mon[179688]: failed to encode map e232 with expected crc 2023-09-23T20:58:16.279 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:58:16 smithi026 ceph-mon[179688]: failed to encode map e232 with expected crc 2023-09-23T20:58:16.280 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:58:16 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/2471809530' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-09-23T20:58:16.280 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:58:16 smithi026 ceph-mon[179688]: osdmap e232: 8 total, 8 up, 8 in 2023-09-23T20:58:16.295 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:58:16.430 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:58:16.438 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:58:16.438 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:58:16.539 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:58:16.547 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T20:58:16.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:58:16 smithi070 ceph-mon[168282]: pgmap v544: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 6.1 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 102 B/s rd, 30 MiB/s wr, 662 op/s; 10943/299991 objects degraded (3.648%); 4882/299991 objects misplaced (1.627%); 1.4 MiB/s, 22 objects/s recovering 2023-09-23T20:58:16.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:58:16 smithi070 ceph-mon[168282]: failed to encode map e232 with expected crc 2023-09-23T20:58:16.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:58:16 smithi070 ceph-mon[168282]: failed to encode map e232 with expected crc 2023-09-23T20:58:16.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:58:16 smithi070 ceph-mon[168282]: failed to encode map e232 with expected crc 2023-09-23T20:58:16.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:58:16 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/2471809530' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-09-23T20:58:16.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:58:16 smithi070 ceph-mon[168282]: osdmap e232: 8 total, 8 up, 8 in 2023-09-23T20:58:16.674 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:58:16.837 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:58:16.856 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:58:16.857 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:58:16.958 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:58:16.984 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:58:17.023 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:58:17.028 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:58:17.028 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:58:17.129 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:58:17.157 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:58:17.170 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:58:17.200 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:58:17.206 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:58:17.206 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:58:17.224 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:58:17 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:58:17.224 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:58:17 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:58:17.224 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:58:17 smithi026 ceph-mon[175151]: pgmap v546: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 6.0 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 127 B/s rd, 19 MiB/s wr, 692 op/s; 10943/295896 objects degraded (3.698%); 4771/295896 objects misplaced (1.612%); 1.8 MiB/s, 28 objects/s recovering 2023-09-23T20:58:17.307 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:58:17.332 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:58:17 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:58:17.332 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:58:17 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:58:17.332 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:58:17 smithi070 ceph-mon[168282]: pgmap v546: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 6.0 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 127 B/s rd, 19 MiB/s wr, 692 op/s; 10943/295896 objects degraded (3.698%); 4771/295896 objects misplaced (1.612%); 1.8 MiB/s, 28 objects/s recovering 2023-09-23T20:58:17.334 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:58:17.376 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:58:17.379 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:58:17.379 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:58:17.480 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:58:17.505 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:58:17 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:58:17.505 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:58:17 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:58:17.505 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:58:17 smithi026 ceph-mon[179688]: pgmap v546: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 6.0 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 127 B/s rd, 19 MiB/s wr, 692 op/s; 10943/295896 objects degraded (3.698%); 4771/295896 objects misplaced (1.612%); 1.8 MiB/s, 28 objects/s recovering 2023-09-23T20:58:17.510 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:58:17.547 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:58:17.547 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:58:17.551 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:58:17.552 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:58:17.652 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:58:17.681 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:58:17.721 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:58:17.724 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:58:17.724 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:58:17.825 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:58:17.857 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:58:17.895 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:58:17.899 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:58:17.899 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:58:17.969 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:58:18.000 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:58:18.030 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:58:18.072 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:58:18.076 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:58:18.076 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:58:18.177 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:58:18.208 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:58:18.248 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:58:18.252 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:58:18.252 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:58:18.353 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:58:18.474 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:58:18.588 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:58:18.596 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:58:18.596 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:58:18.697 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:58:18.808 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:58:18.938 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:58:18.942 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:58:18.943 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:58:19.043 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:58:19.127 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:58:19.417 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:58:19.431 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:58:19.432 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:58:19.520 INFO:teuthology.orchestra.run.smithi026.stderr:noscrub is set 2023-09-23T20:58:19.533 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:58:19.658 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:58:19.797 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:58:19 smithi026 ceph-mon[175151]: pgmap v547: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 6.0 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 7.5 MiB/s wr, 624 op/s; 10943/293028 objects degraded (3.734%); 4771/293028 objects misplaced (1.628%); 1.8 MiB/s, 28 objects/s recovering 2023-09-23T20:58:19.797 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:58:19 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/1029142514' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:58:19.797 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:58:19 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:58:19.797 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:58:19 smithi026 ceph-mon[179688]: pgmap v547: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 6.0 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 7.5 MiB/s wr, 624 op/s; 10943/293028 objects degraded (3.734%); 4771/293028 objects misplaced (1.628%); 1.8 MiB/s, 28 objects/s recovering 2023-09-23T20:58:19.797 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:58:19 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/1029142514' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:58:19.797 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:58:19 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:58:19.798 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:58:19 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[164029]: ::ffff:172.21.15.70 - - [23/Sep/2023:20:58:19] "GET /metrics HTTP/1.1" 200 36321 "" "Prometheus/2.43.0" 2023-09-23T20:58:19.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:58:19 smithi070 ceph-mon[168282]: pgmap v547: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 6.0 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 7.5 MiB/s wr, 624 op/s; 10943/293028 objects degraded (3.734%); 4771/293028 objects misplaced (1.628%); 1.8 MiB/s, 28 objects/s recovering 2023-09-23T20:58:19.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:58:19 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/1029142514' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:58:19.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:58:19 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:58:19.926 INFO:teuthology.orchestra.run.smithi026.stdout: 2023-09-23T20:58:19.955 INFO:teuthology.orchestra.run.smithi026.stderr:dumped all 2023-09-23T20:58:20.040 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:58:20.049 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:58:20.050 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:58:20.150 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:58:20.284 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:58:20.528 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:58:20.541 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:58:20.542 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:58:20.642 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:58:20.737 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:58:20 smithi026 ceph-mon[175151]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:58:20.737 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:58:20 smithi026 ceph-mon[175151]: failed to encode map e233 with expected crc 2023-09-23T20:58:20.737 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:58:20 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T20:58:20.737 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:58:20 smithi026 ceph-mon[175151]: osdmap e233: 8 total, 8 up, 8 in 2023-09-23T20:58:20.738 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:58:20 smithi026 ceph-mon[175151]: failed to encode map e233 with expected crc 2023-09-23T20:58:20.738 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:58:20 smithi026 ceph-mon[175151]: from='client.35238 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:58:20.738 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:58:20 smithi026 ceph-mon[175151]: Health check update: Degraded data redundancy: 10943/293028 objects degraded (3.734%), 2 pgs degraded (PG_DEGRADED) 2023-09-23T20:58:20.738 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:58:20 smithi026 ceph-mon[179688]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:58:20.738 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:58:20 smithi026 ceph-mon[179688]: failed to encode map e233 with expected crc 2023-09-23T20:58:20.738 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:58:20 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T20:58:20.738 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:58:20 smithi026 ceph-mon[179688]: osdmap e233: 8 total, 8 up, 8 in 2023-09-23T20:58:20.739 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:58:20 smithi026 ceph-mon[179688]: failed to encode map e233 with expected crc 2023-09-23T20:58:20.739 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:58:20 smithi026 ceph-mon[179688]: from='client.35238 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:58:20.739 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:58:20 smithi026 ceph-mon[179688]: Health check update: Degraded data redundancy: 10943/293028 objects degraded (3.734%), 2 pgs degraded (PG_DEGRADED) 2023-09-23T20:58:20.739 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:58:20.848 INFO:teuthology.orchestra.run.smithi026.stdout:{"pg_ready":true,"pg_map":{"version":548,"stamp":"2023-09-23T20:58:19.466765+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":6317152184,"num_objects":96408,"num_object_clones":0,"num_object_copies":289224,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":10943,"num_objects_misplaced":4771,"num_objects_unfound":0,"num_objects_dirty":96408,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":107197,"num_write_kb":6512606,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":650,"num_bytes_recovered":41162598,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":108972,"ondisk_log_size":108972,"up":327,"acting":324,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":331,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":20087752,"kb_used_data":18659720,"kb_used_omap":136,"kb_used_meta":1427703,"kb_avail":729873464,"statfs":{"total":767960285184,"available":747390427136,"internally_reserved":0,"allocated":19107553280,"data_stored":19099865629,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":139741,"internal_metadata":1461968419},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[3,4,4,2,8,1,2],"upper_bound":128},"perf_stat":{"commit_latency_ms":10,"apply_latency_ms":10,"commit_latency_ns":10000000,"apply_latency_ns":10000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-284098560,"num_objects":-4335,"num_object_clones":0,"num_object_copies":-13005,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":-231,"num_objects_unfound":0,"num_objects_dirty":-4335,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6267,"num_write_kb":61824,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":231,"num_bytes_recovered":15138816,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"8.412561"},"pg_stats":[{"pgid":"3.1a","version":"68'1","reported_seq":109,"reported_epoch":231,"state":"active+clean","last_fresh":"2023-09-23T20:58:11.067772+0000","last_change":"2023-09-23T20:54:13.946695+0000","last_active":"2023-09-23T20:58:11.067772+0000","last_peered":"2023-09-23T20:58:11.067772+0000","last_clean":"2023-09-23T20:58:11.067772+0000","last_became_active":"2023-09-23T20:54:13.945399+0000","last_became_peered":"2023-09-23T20:54:13.945399+0000","last_unstale":"2023-09-23T20:58:11.067772+0000","last_undegraded":"2023-09-23T20:58:11.067772+0000","last_fullsized":"2023-09-23T20:58:11.067772+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:32:52.365464+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":223,"reported_epoch":231,"state":"active+clean","last_fresh":"2023-09-23T20:58:11.109618+0000","last_change":"2023-09-23T20:54:13.942019+0000","last_active":"2023-09-23T20:58:11.109618+0000","last_peered":"2023-09-23T20:58:11.109618+0000","last_clean":"2023-09-23T20:58:11.109618+0000","last_became_active":"2023-09-23T20:54:13.941647+0000","last_became_peered":"2023-09-23T20:54:13.941647+0000","last_unstale":"2023-09-23T20:58:11.109618+0000","last_undegraded":"2023-09-23T20:58:11.109618+0000","last_fullsized":"2023-09-23T20:58:11.109618+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:30:54.400944+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":201,"reported_epoch":231,"state":"active+clean","last_fresh":"2023-09-23T20:58:11.082249+0000","last_change":"2023-09-23T20:54:13.941394+0000","last_active":"2023-09-23T20:58:11.082249+0000","last_peered":"2023-09-23T20:58:11.082249+0000","last_clean":"2023-09-23T20:58:11.082249+0000","last_became_active":"2023-09-23T20:54:13.941042+0000","last_became_peered":"2023-09-23T20:54:13.941042+0000","last_unstale":"2023-09-23T20:58:11.082249+0000","last_undegraded":"2023-09-23T20:58:11.082249+0000","last_fullsized":"2023-09-23T20:58:11.082249+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:19:41.401184+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":216,"reported_epoch":231,"state":"active+clean","last_fresh":"2023-09-23T20:58:11.083615+0000","last_change":"2023-09-23T20:57:44.979811+0000","last_active":"2023-09-23T20:58:11.083615+0000","last_peered":"2023-09-23T20:58:11.083615+0000","last_clean":"2023-09-23T20:58:11.083615+0000","last_became_active":"2023-09-23T20:57:44.979515+0000","last_became_peered":"2023-09-23T20:57:44.979515+0000","last_unstale":"2023-09-23T20:58:11.083615+0000","last_undegraded":"2023-09-23T20:58:11.083615+0000","last_fullsized":"2023-09-23T20:58:11.083615+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:38:22.162140+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":214,"reported_epoch":231,"state":"active+clean","last_fresh":"2023-09-23T20:58:11.088098+0000","last_change":"2023-09-23T20:51:10.833613+0000","last_active":"2023-09-23T20:58:11.088098+0000","last_peered":"2023-09-23T20:58:11.088098+0000","last_clean":"2023-09-23T20:58:11.088098+0000","last_became_active":"2023-09-23T20:51:10.833208+0000","last_became_peered":"2023-09-23T20:51:10.833208+0000","last_unstale":"2023-09-23T20:58:11.088098+0000","last_undegraded":"2023-09-23T20:58:11.088098+0000","last_fullsized":"2023-09-23T20:58:11.088098+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:51:28.246014+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":197,"reported_epoch":231,"state":"active+clean","last_fresh":"2023-09-23T20:58:11.058162+0000","last_change":"2023-09-23T20:55:35.958384+0000","last_active":"2023-09-23T20:58:11.058162+0000","last_peered":"2023-09-23T20:58:11.058162+0000","last_clean":"2023-09-23T20:58:11.058162+0000","last_became_active":"2023-09-23T20:55:35.954563+0000","last_became_peered":"2023-09-23T20:55:35.954563+0000","last_unstale":"2023-09-23T20:58:11.058162+0000","last_undegraded":"2023-09-23T20:58:11.058162+0000","last_fullsized":"2023-09-23T20:58:11.058162+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:37:48.332259+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":206,"reported_epoch":231,"state":"active+clean","last_fresh":"2023-09-23T20:58:11.084389+0000","last_change":"2023-09-23T20:54:31.885514+0000","last_active":"2023-09-23T20:58:11.084389+0000","last_peered":"2023-09-23T20:58:11.084389+0000","last_clean":"2023-09-23T20:58:11.084389+0000","last_became_active":"2023-09-23T20:54:31.884986+0000","last_became_peered":"2023-09-23T20:54:31.884986+0000","last_unstale":"2023-09-23T20:58:11.084389+0000","last_undegraded":"2023-09-23T20:58:11.084389+0000","last_fullsized":"2023-09-23T20:58:11.084389+0000","mapping_epoch":136,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:26:08.100220+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":212,"reported_epoch":231,"state":"active+clean","last_fresh":"2023-09-23T20:58:11.088020+0000","last_change":"2023-09-23T20:55:35.951889+0000","last_active":"2023-09-23T20:58:11.088020+0000","last_peered":"2023-09-23T20:58:11.088020+0000","last_clean":"2023-09-23T20:58:11.088020+0000","last_became_active":"2023-09-23T20:55:35.951563+0000","last_became_peered":"2023-09-23T20:55:35.951563+0000","last_unstale":"2023-09-23T20:58:11.088020+0000","last_undegraded":"2023-09-23T20:58:11.088020+0000","last_fullsized":"2023-09-23T20:58:11.088020+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:41:53.098781+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"79'161","reported_seq":518,"reported_epoch":231,"state":"active+clean","last_fresh":"2023-09-23T20:58:11.092844+0000","last_change":"2023-09-23T20:57:25.096559+0000","last_active":"2023-09-23T20:58:11.092844+0000","last_peered":"2023-09-23T20:58:11.092844+0000","last_clean":"2023-09-23T20:58:11.092844+0000","last_became_active":"2023-09-23T20:57:15.975674+0000","last_became_peered":"2023-09-23T20:57:15.975674+0000","last_unstale":"2023-09-23T20:58:11.092844+0000","last_undegraded":"2023-09-23T20:58:11.092844+0000","last_fullsized":"2023-09-23T20:58:11.092844+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:25.096401+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:25.096401+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:05:05.448404+0000","scrub_duration":0.002859393,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":105,"reported_epoch":231,"state":"active+clean","last_fresh":"2023-09-23T20:58:11.069727+0000","last_change":"2023-09-23T20:54:31.890678+0000","last_active":"2023-09-23T20:58:11.069727+0000","last_peered":"2023-09-23T20:58:11.069727+0000","last_clean":"2023-09-23T20:58:11.069727+0000","last_became_active":"2023-09-23T20:54:31.890469+0000","last_became_peered":"2023-09-23T20:54:31.890469+0000","last_unstale":"2023-09-23T20:58:11.069727+0000","last_undegraded":"2023-09-23T20:58:11.069727+0000","last_fullsized":"2023-09-23T20:58:11.069727+0000","mapping_epoch":136,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:55:08.790146+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":173,"reported_epoch":231,"state":"active+clean","last_fresh":"2023-09-23T20:58:11.088206+0000","last_change":"2023-09-23T20:55:35.952661+0000","last_active":"2023-09-23T20:58:11.088206+0000","last_peered":"2023-09-23T20:58:11.088206+0000","last_clean":"2023-09-23T20:58:11.088206+0000","last_became_active":"2023-09-23T20:55:35.952349+0000","last_became_peered":"2023-09-23T20:55:35.952349+0000","last_unstale":"2023-09-23T20:58:11.088206+0000","last_undegraded":"2023-09-23T20:58:11.088206+0000","last_fullsized":"2023-09-23T20:58:11.088206+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:00:03.110126+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":49,"reported_epoch":231,"state":"active+clean","last_fresh":"2023-09-23T20:58:11.109876+0000","last_change":"2023-09-23T20:56:45.988607+0000","last_active":"2023-09-23T20:58:11.109876+0000","last_peered":"2023-09-23T20:58:11.109876+0000","last_clean":"2023-09-23T20:58:11.109876+0000","last_became_active":"2023-09-23T20:56:45.988206+0000","last_became_peered":"2023-09-23T20:56:45.988206+0000","last_unstale":"2023-09-23T20:58:11.109876+0000","last_undegraded":"2023-09-23T20:58:11.109876+0000","last_fullsized":"2023-09-23T20:58:11.109876+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":205,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:01:19.783688+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,4],"acting":[3,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":49,"reported_epoch":231,"state":"active+clean","last_fresh":"2023-09-23T20:58:11.069211+0000","last_change":"2023-09-23T20:56:18.139142+0000","last_active":"2023-09-23T20:58:11.069211+0000","last_peered":"2023-09-23T20:58:11.069211+0000","last_clean":"2023-09-23T20:58:11.069211+0000","last_became_active":"2023-09-23T20:56:18.138844+0000","last_became_peered":"2023-09-23T20:56:18.138844+0000","last_unstale":"2023-09-23T20:58:11.069211+0000","last_undegraded":"2023-09-23T20:58:11.069211+0000","last_fullsized":"2023-09-23T20:58:11.069211+0000","mapping_epoch":192,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":193,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:09:02.217683+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,2],"acting":[5,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.14","version":"79'22","reported_seq":136,"reported_epoch":231,"state":"active+clean","last_fresh":"2023-09-23T20:58:11.067955+0000","last_change":"2023-09-23T20:55:35.949418+0000","last_active":"2023-09-23T20:58:11.067955+0000","last_peered":"2023-09-23T20:58:11.067955+0000","last_clean":"2023-09-23T20:58:11.067955+0000","last_became_active":"2023-09-23T20:55:35.948566+0000","last_became_peered":"2023-09-23T20:55:35.948566+0000","last_unstale":"2023-09-23T20:58:11.067955+0000","last_undegraded":"2023-09-23T20:58:11.067955+0000","last_fullsized":"2023-09-23T20:58:11.067955+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:59:37.433128+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4734,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":126,"reported_epoch":231,"state":"active+clean","last_fresh":"2023-09-23T20:58:11.067990+0000","last_change":"2023-09-23T20:55:35.949178+0000","last_active":"2023-09-23T20:58:11.067990+0000","last_peered":"2023-09-23T20:58:11.067990+0000","last_clean":"2023-09-23T20:58:11.067990+0000","last_became_active":"2023-09-23T20:55:35.948795+0000","last_became_peered":"2023-09-23T20:55:35.948795+0000","last_unstale":"2023-09-23T20:58:11.067990+0000","last_undegraded":"2023-09-23T20:58:11.067990+0000","last_fullsized":"2023-09-23T20:58:11.067990+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:13:08.996932+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.11","version":"79'161","reported_seq":460,"reported_epoch":231,"state":"active+clean","last_fresh":"2023-09-23T20:58:11.084395+0000","last_change":"2023-09-23T20:57:28.756796+0000","last_active":"2023-09-23T20:58:11.084395+0000","last_peered":"2023-09-23T20:58:11.084395+0000","last_clean":"2023-09-23T20:58:11.084395+0000","last_became_active":"2023-09-23T20:57:15.980606+0000","last_became_peered":"2023-09-23T20:57:15.980606+0000","last_unstale":"2023-09-23T20:58:11.084395+0000","last_undegraded":"2023-09-23T20:58:11.084395+0000","last_fullsized":"2023-09-23T20:58:11.084395+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:28.756634+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:28.756634+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:09:19.150791+0000","scrub_duration":0.0064925119999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":152,"reported_epoch":231,"state":"active+clean","last_fresh":"2023-09-23T20:58:11.067849+0000","last_change":"2023-09-23T20:56:45.992577+0000","last_active":"2023-09-23T20:58:11.067849+0000","last_peered":"2023-09-23T20:58:11.067849+0000","last_clean":"2023-09-23T20:58:11.067849+0000","last_became_active":"2023-09-23T20:56:45.992297+0000","last_became_peered":"2023-09-23T20:56:45.992297+0000","last_unstale":"2023-09-23T20:58:11.067849+0000","last_undegraded":"2023-09-23T20:58:11.067849+0000","last_fullsized":"2023-09-23T20:58:11.067849+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":205,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:00:45.547202+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,2],"acting":[4,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":197,"reported_epoch":231,"state":"active+clean","last_fresh":"2023-09-23T20:58:11.100257+0000","last_change":"2023-09-23T20:51:10.831262+0000","last_active":"2023-09-23T20:58:11.100257+0000","last_peered":"2023-09-23T20:58:11.100257+0000","last_clean":"2023-09-23T20:58:11.100257+0000","last_became_active":"2023-09-23T20:51:10.830931+0000","last_became_peered":"2023-09-23T20:51:10.830931+0000","last_unstale":"2023-09-23T20:58:11.100257+0000","last_undegraded":"2023-09-23T20:58:11.100257+0000","last_fullsized":"2023-09-23T20:58:11.100257+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:51:54.982505+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.10","version":"79'161","reported_seq":469,"reported_epoch":231,"state":"active+clean","last_fresh":"2023-09-23T20:58:11.068804+0000","last_change":"2023-09-23T20:57:20.708003+0000","last_active":"2023-09-23T20:58:11.068804+0000","last_peered":"2023-09-23T20:58:11.068804+0000","last_clean":"2023-09-23T20:58:11.068804+0000","last_became_active":"2023-09-23T20:57:15.975302+0000","last_became_peered":"2023-09-23T20:57:15.975302+0000","last_unstale":"2023-09-23T20:58:11.068804+0000","last_undegraded":"2023-09-23T20:58:11.068804+0000","last_fullsized":"2023-09-23T20:58:11.068804+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:20.707939+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:20.707939+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:16:51.721994+0000","scrub_duration":0.0017733409999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":206,"reported_epoch":231,"state":"active+clean","last_fresh":"2023-09-23T20:58:11.083124+0000","last_change":"2023-09-23T20:57:44.979736+0000","last_active":"2023-09-23T20:58:11.083124+0000","last_peered":"2023-09-23T20:58:11.083124+0000","last_clean":"2023-09-23T20:58:11.083124+0000","last_became_active":"2023-09-23T20:57:44.979356+0000","last_became_peered":"2023-09-23T20:57:44.979356+0000","last_unstale":"2023-09-23T20:58:11.083124+0000","last_undegraded":"2023-09-23T20:58:11.083124+0000","last_fullsized":"2023-09-23T20:58:11.083124+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:14:43.848832+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":105,"reported_epoch":231,"state":"active+clean","last_fresh":"2023-09-23T20:58:11.069227+0000","last_change":"2023-09-23T20:54:31.887814+0000","last_active":"2023-09-23T20:58:11.069227+0000","last_peered":"2023-09-23T20:58:11.069227+0000","last_clean":"2023-09-23T20:58:11.069227+0000","last_became_active":"2023-09-23T20:54:31.887195+0000","last_became_peered":"2023-09-23T20:54:31.887195+0000","last_unstale":"2023-09-23T20:58:11.069227+0000","last_undegraded":"2023-09-23T20:58:11.069227+0000","last_fullsized":"2023-09-23T20:58:11.069227+0000","mapping_epoch":136,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:10:52.089553+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.13","version":"79'161","reported_seq":531,"reported_epoch":231,"state":"active+clean","last_fresh":"2023-09-23T20:58:11.082845+0000","last_change":"2023-09-23T20:57:48.262970+0000","last_active":"2023-09-23T20:58:11.082845+0000","last_peered":"2023-09-23T20:58:11.082845+0000","last_clean":"2023-09-23T20:58:11.082845+0000","last_became_active":"2023-09-23T20:57:48.262663+0000","last_became_peered":"2023-09-23T20:57:48.262663+0000","last_unstale":"2023-09-23T20:58:11.082845+0000","last_undegraded":"2023-09-23T20:58:11.082845+0000","last_fullsized":"2023-09-23T20:58:11.082845+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:36.420362+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:36.420362+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:28:28.613492+0000","scrub_duration":0.028838444000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":176,"reported_epoch":231,"state":"active+clean","last_fresh":"2023-09-23T20:58:11.067679+0000","last_change":"2023-09-23T20:56:45.991822+0000","last_active":"2023-09-23T20:58:11.067679+0000","last_peered":"2023-09-23T20:58:11.067679+0000","last_clean":"2023-09-23T20:58:11.067679+0000","last_became_active":"2023-09-23T20:56:45.991460+0000","last_became_peered":"2023-09-23T20:56:45.991460+0000","last_unstale":"2023-09-23T20:58:11.067679+0000","last_undegraded":"2023-09-23T20:58:11.067679+0000","last_fullsized":"2023-09-23T20:58:11.067679+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":205,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:20:37.503317+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":188,"reported_epoch":231,"state":"active+clean","last_fresh":"2023-09-23T20:58:11.083757+0000","last_change":"2023-09-23T20:51:10.840849+0000","last_active":"2023-09-23T20:58:11.083757+0000","last_peered":"2023-09-23T20:58:11.083757+0000","last_clean":"2023-09-23T20:58:11.083757+0000","last_became_active":"2023-09-23T20:51:10.840593+0000","last_became_peered":"2023-09-23T20:51:10.840593+0000","last_unstale":"2023-09-23T20:58:11.083757+0000","last_undegraded":"2023-09-23T20:58:11.083757+0000","last_fullsized":"2023-09-23T20:58:11.083757+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:30:52.297625+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.12","version":"79'161","reported_seq":469,"reported_epoch":231,"state":"active+clean","last_fresh":"2023-09-23T20:58:11.069062+0000","last_change":"2023-09-23T20:57:21.750765+0000","last_active":"2023-09-23T20:58:11.069062+0000","last_peered":"2023-09-23T20:58:11.069062+0000","last_clean":"2023-09-23T20:58:11.069062+0000","last_became_active":"2023-09-23T20:57:15.979195+0000","last_became_peered":"2023-09-23T20:57:15.979195+0000","last_unstale":"2023-09-23T20:58:11.069062+0000","last_undegraded":"2023-09-23T20:58:11.069062+0000","last_fullsized":"2023-09-23T20:58:11.069062+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:21.750680+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:21.750680+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:03:41.915962+0000","scrub_duration":0.0092391680000000007,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":224,"reported_epoch":231,"state":"active+clean","last_fresh":"2023-09-23T20:58:11.058005+0000","last_change":"2023-09-23T20:55:36.943969+0000","last_active":"2023-09-23T20:58:11.058005+0000","last_peered":"2023-09-23T20:58:11.058005+0000","last_clean":"2023-09-23T20:58:11.058005+0000","last_became_active":"2023-09-23T20:55:36.943580+0000","last_became_peered":"2023-09-23T20:55:36.943580+0000","last_unstale":"2023-09-23T20:58:11.058005+0000","last_undegraded":"2023-09-23T20:58:11.058005+0000","last_fullsized":"2023-09-23T20:58:11.058005+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:49:06.969396+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":20,"reported_epoch":231,"state":"active+clean","last_fresh":"2023-09-23T20:58:11.111710+0000","last_change":"2023-09-23T20:57:44.980498+0000","last_active":"2023-09-23T20:58:11.111710+0000","last_peered":"2023-09-23T20:58:11.111710+0000","last_clean":"2023-09-23T20:58:11.111710+0000","last_became_active":"2023-09-23T20:57:44.980182+0000","last_became_peered":"2023-09-23T20:57:44.980182+0000","last_unstale":"2023-09-23T20:58:11.111710+0000","last_undegraded":"2023-09-23T20:58:11.111710+0000","last_fullsized":"2023-09-23T20:58:11.111710+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:34:51.620544+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.d","version":"79'161","reported_seq":514,"reported_epoch":231,"state":"active+clean","last_fresh":"2023-09-23T20:58:11.067870+0000","last_change":"2023-09-23T20:57:31.075213+0000","last_active":"2023-09-23T20:58:11.067870+0000","last_peered":"2023-09-23T20:58:11.067870+0000","last_clean":"2023-09-23T20:58:11.067870+0000","last_became_active":"2023-09-23T20:57:15.988188+0000","last_became_peered":"2023-09-23T20:57:15.988188+0000","last_unstale":"2023-09-23T20:58:11.067870+0000","last_undegraded":"2023-09-23T20:58:11.067870+0000","last_fullsized":"2023-09-23T20:58:11.067870+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:31.075098+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:31.075098+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:41:52.142362+0000","scrub_duration":0.010304610000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.b","version":"232'6703","reported_seq":6745,"reported_epoch":232,"state":"active+clean","last_fresh":"2023-09-23T20:58:19.226895+0000","last_change":"2023-09-23T20:56:35.543994+0000","last_active":"2023-09-23T20:58:19.226895+0000","last_peered":"2023-09-23T20:58:19.226895+0000","last_clean":"2023-09-23T20:58:19.226895+0000","last_became_active":"2023-09-23T20:56:35.543758+0000","last_became_peered":"2023-09-23T20:56:35.543758+0000","last_unstale":"2023-09-23T20:58:19.226895+0000","last_undegraded":"2023-09-23T20:58:19.226895+0000","last_fullsized":"2023-09-23T20:58:19.226895+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":199,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_clean_scrub_stamp":"2023-09-23T20:56:34.529103+0000","objects_scrubbed":0,"log_size":6703,"log_dups_size":0,"ondisk_log_size":6703,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:21:29.041484+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":391708672,"num_objects":5977,"num_object_clones":0,"num_object_copies":17931,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5977,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6703,"num_write_kb":405760,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":163,"reported_epoch":231,"state":"active+clean","last_fresh":"2023-09-23T20:58:11.068141+0000","last_change":"2023-09-23T20:56:45.991841+0000","last_active":"2023-09-23T20:58:11.068141+0000","last_peered":"2023-09-23T20:58:11.068141+0000","last_clean":"2023-09-23T20:58:11.068141+0000","last_became_active":"2023-09-23T20:56:45.991491+0000","last_became_peered":"2023-09-23T20:56:45.991491+0000","last_unstale":"2023-09-23T20:58:11.068141+0000","last_undegraded":"2023-09-23T20:58:11.068141+0000","last_fullsized":"2023-09-23T20:58:11.068141+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":205,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:30:37.405319+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":122,"reported_epoch":231,"state":"active+clean","last_fresh":"2023-09-23T20:58:11.068952+0000","last_change":"2023-09-23T20:55:35.957768+0000","last_active":"2023-09-23T20:58:11.068952+0000","last_peered":"2023-09-23T20:58:11.068952+0000","last_clean":"2023-09-23T20:58:11.068952+0000","last_became_active":"2023-09-23T20:55:35.957421+0000","last_became_peered":"2023-09-23T20:55:35.957421+0000","last_unstale":"2023-09-23T20:58:11.068952+0000","last_undegraded":"2023-09-23T20:58:11.068952+0000","last_fullsized":"2023-09-23T20:58:11.068952+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:49:41.585961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.c","version":"79'161","reported_seq":532,"reported_epoch":231,"state":"active+clean","last_fresh":"2023-09-23T20:58:11.088408+0000","last_change":"2023-09-23T20:57:22.041071+0000","last_active":"2023-09-23T20:58:11.088408+0000","last_peered":"2023-09-23T20:58:11.088408+0000","last_clean":"2023-09-23T20:58:11.088408+0000","last_became_active":"2023-09-23T20:57:15.973204+0000","last_became_peered":"2023-09-23T20:57:15.973204+0000","last_unstale":"2023-09-23T20:58:11.088408+0000","last_undegraded":"2023-09-23T20:58:11.088408+0000","last_fullsized":"2023-09-23T20:58:11.088408+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:22.040907+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:22.040907+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:47:02.019978+0000","scrub_duration":0.013021176000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.a","version":"232'6773","reported_seq":6815,"reported_epoch":232,"state":"active+clean","last_fresh":"2023-09-23T20:58:19.379850+0000","last_change":"2023-09-23T20:56:35.547120+0000","last_active":"2023-09-23T20:58:19.379850+0000","last_peered":"2023-09-23T20:58:19.379850+0000","last_clean":"2023-09-23T20:58:19.379850+0000","last_became_active":"2023-09-23T20:56:35.546838+0000","last_became_peered":"2023-09-23T20:56:35.546838+0000","last_unstale":"2023-09-23T20:58:19.379850+0000","last_undegraded":"2023-09-23T20:58:19.379850+0000","last_fullsized":"2023-09-23T20:58:19.379850+0000","mapping_epoch":199,"log_start":"202'600","ondisk_log_start":"202'600","created":199,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_clean_scrub_stamp":"2023-09-23T20:56:34.529103+0000","objects_scrubbed":0,"log_size":6173,"log_dups_size":0,"ondisk_log_size":6173,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:48:24.132768+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":390791168,"num_objects":5963,"num_object_clones":0,"num_object_copies":17889,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5963,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6773,"num_write_kb":407552,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,5],"acting":[3,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":150,"reported_epoch":231,"state":"active+clean","last_fresh":"2023-09-23T20:58:11.068243+0000","last_change":"2023-09-23T20:56:45.991821+0000","last_active":"2023-09-23T20:58:11.068243+0000","last_peered":"2023-09-23T20:58:11.068243+0000","last_clean":"2023-09-23T20:58:11.068243+0000","last_became_active":"2023-09-23T20:56:45.991447+0000","last_became_peered":"2023-09-23T20:56:45.991447+0000","last_unstale":"2023-09-23T20:58:11.068243+0000","last_undegraded":"2023-09-23T20:58:11.068243+0000","last_fullsized":"2023-09-23T20:58:11.068243+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":205,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:14:25.525222+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":153,"reported_epoch":231,"state":"active+clean","last_fresh":"2023-09-23T20:58:11.067996+0000","last_change":"2023-09-23T20:57:44.982059+0000","last_active":"2023-09-23T20:58:11.067996+0000","last_peered":"2023-09-23T20:58:11.067996+0000","last_clean":"2023-09-23T20:58:11.067996+0000","last_became_active":"2023-09-23T20:57:44.981742+0000","last_became_peered":"2023-09-23T20:57:44.981742+0000","last_unstale":"2023-09-23T20:58:11.067996+0000","last_undegraded":"2023-09-23T20:58:11.067996+0000","last_fullsized":"2023-09-23T20:58:11.067996+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:41:24.042916+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.f","version":"79'161","reported_seq":536,"reported_epoch":231,"state":"active+clean","last_fresh":"2023-09-23T20:58:11.057914+0000","last_change":"2023-09-23T20:57:27.880841+0000","last_active":"2023-09-23T20:58:11.057914+0000","last_peered":"2023-09-23T20:58:11.057914+0000","last_clean":"2023-09-23T20:58:11.057914+0000","last_became_active":"2023-09-23T20:57:15.970925+0000","last_became_peered":"2023-09-23T20:57:15.970925+0000","last_unstale":"2023-09-23T20:58:11.057914+0000","last_undegraded":"2023-09-23T20:58:11.057914+0000","last_fullsized":"2023-09-23T20:58:11.057914+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:27.880776+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:27.880776+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:15:13.438014+0000","scrub_duration":0.0014004899999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.9","version":"232'6645","reported_seq":6687,"reported_epoch":232,"state":"active+clean","last_fresh":"2023-09-23T20:58:16.198778+0000","last_change":"2023-09-23T20:56:35.549387+0000","last_active":"2023-09-23T20:58:16.198778+0000","last_peered":"2023-09-23T20:58:16.198778+0000","last_clean":"2023-09-23T20:58:16.198778+0000","last_became_active":"2023-09-23T20:56:35.549087+0000","last_became_peered":"2023-09-23T20:56:35.549087+0000","last_unstale":"2023-09-23T20:58:16.198778+0000","last_undegraded":"2023-09-23T20:58:16.198778+0000","last_fullsized":"2023-09-23T20:58:16.198778+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":199,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_clean_scrub_stamp":"2023-09-23T20:56:34.529103+0000","objects_scrubbed":0,"log_size":6645,"log_dups_size":0,"ondisk_log_size":6645,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:04:51.547362+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":397213696,"num_objects":6061,"num_object_clones":0,"num_object_copies":18183,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6061,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6645,"num_write_kb":406592,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":136,"reported_epoch":231,"state":"active+clean","last_fresh":"2023-09-23T20:58:11.069147+0000","last_change":"2023-09-23T20:56:45.989153+0000","last_active":"2023-09-23T20:58:11.069147+0000","last_peered":"2023-09-23T20:58:11.069147+0000","last_clean":"2023-09-23T20:58:11.069147+0000","last_became_active":"2023-09-23T20:56:45.988800+0000","last_became_peered":"2023-09-23T20:56:45.988800+0000","last_unstale":"2023-09-23T20:58:11.069147+0000","last_undegraded":"2023-09-23T20:58:11.069147+0000","last_fullsized":"2023-09-23T20:58:11.069147+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":205,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:04:41.130194+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":171,"reported_epoch":231,"state":"active+clean","last_fresh":"2023-09-23T20:58:11.057849+0000","last_change":"2023-09-23T20:55:35.965427+0000","last_active":"2023-09-23T20:58:11.057849+0000","last_peered":"2023-09-23T20:58:11.057849+0000","last_clean":"2023-09-23T20:58:11.057849+0000","last_became_active":"2023-09-23T20:55:35.963969+0000","last_became_peered":"2023-09-23T20:55:35.963969+0000","last_unstale":"2023-09-23T20:58:11.057849+0000","last_undegraded":"2023-09-23T20:58:11.057849+0000","last_fullsized":"2023-09-23T20:58:11.057849+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:38:40.612083+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.e","version":"79'161","reported_seq":443,"reported_epoch":231,"state":"active+clean","last_fresh":"2023-09-23T20:58:11.068026+0000","last_change":"2023-09-23T20:57:34.132738+0000","last_active":"2023-09-23T20:58:11.068026+0000","last_peered":"2023-09-23T20:58:11.068026+0000","last_clean":"2023-09-23T20:58:11.068026+0000","last_became_active":"2023-09-23T20:57:15.974318+0000","last_became_peered":"2023-09-23T20:57:15.974318+0000","last_unstale":"2023-09-23T20:58:11.068026+0000","last_undegraded":"2023-09-23T20:58:11.068026+0000","last_fullsized":"2023-09-23T20:58:11.068026+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:34.132660+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:34.132660+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:07:07.739438+0000","scrub_duration":0.005229694,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.8","version":"232'6742","reported_seq":6784,"reported_epoch":232,"state":"active+clean","last_fresh":"2023-09-23T20:58:16.270662+0000","last_change":"2023-09-23T20:56:35.548119+0000","last_active":"2023-09-23T20:58:16.270662+0000","last_peered":"2023-09-23T20:58:16.270662+0000","last_clean":"2023-09-23T20:58:16.270662+0000","last_became_active":"2023-09-23T20:56:35.547914+0000","last_became_peered":"2023-09-23T20:56:35.547914+0000","last_unstale":"2023-09-23T20:58:16.270662+0000","last_undegraded":"2023-09-23T20:58:16.270662+0000","last_fullsized":"2023-09-23T20:58:16.270662+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":199,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_clean_scrub_stamp":"2023-09-23T20:56:34.529103+0000","objects_scrubbed":0,"log_size":6742,"log_dups_size":0,"ondisk_log_size":6742,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:42:54.255023+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":400556032,"num_objects":6112,"num_object_clones":0,"num_object_copies":18336,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6112,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6742,"num_write_kb":411328,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":122,"reported_epoch":231,"state":"active+clean","last_fresh":"2023-09-23T20:58:11.069065+0000","last_change":"2023-09-23T20:55:35.962005+0000","last_active":"2023-09-23T20:58:11.069065+0000","last_peered":"2023-09-23T20:58:11.069065+0000","last_clean":"2023-09-23T20:58:11.069065+0000","last_became_active":"2023-09-23T20:55:35.961695+0000","last_became_peered":"2023-09-23T20:55:35.961695+0000","last_unstale":"2023-09-23T20:58:11.069065+0000","last_undegraded":"2023-09-23T20:58:11.069065+0000","last_fullsized":"2023-09-23T20:58:11.069065+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:28:22.876201+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":219,"reported_epoch":231,"state":"active+clean","last_fresh":"2023-09-23T20:58:11.083400+0000","last_change":"2023-09-23T20:54:13.928715+0000","last_active":"2023-09-23T20:58:11.083400+0000","last_peered":"2023-09-23T20:58:11.083400+0000","last_clean":"2023-09-23T20:58:11.083400+0000","last_became_active":"2023-09-23T20:54:13.928200+0000","last_became_peered":"2023-09-23T20:54:13.928200+0000","last_unstale":"2023-09-23T20:58:11.083400+0000","last_undegraded":"2023-09-23T20:58:11.083400+0000","last_fullsized":"2023-09-23T20:58:11.083400+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:41:16.927513+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.f","version":"232'6706","reported_seq":6748,"reported_epoch":232,"state":"active+clean","last_fresh":"2023-09-23T20:58:16.287628+0000","last_change":"2023-09-23T20:56:35.545128+0000","last_active":"2023-09-23T20:58:16.287628+0000","last_peered":"2023-09-23T20:58:16.287628+0000","last_clean":"2023-09-23T20:58:16.287628+0000","last_became_active":"2023-09-23T20:56:35.544838+0000","last_became_peered":"2023-09-23T20:56:35.544838+0000","last_unstale":"2023-09-23T20:58:16.287628+0000","last_undegraded":"2023-09-23T20:58:16.287628+0000","last_fullsized":"2023-09-23T20:58:16.287628+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":199,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_clean_scrub_stamp":"2023-09-23T20:56:34.529103+0000","objects_scrubbed":0,"log_size":6706,"log_dups_size":0,"ondisk_log_size":6706,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:30:43.388717+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":403701760,"num_objects":6160,"num_object_clones":0,"num_object_copies":18480,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6160,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6706,"num_write_kb":411712,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.9","version":"79'161","reported_seq":604,"reported_epoch":231,"state":"active+clean","last_fresh":"2023-09-23T20:58:11.088132+0000","last_change":"2023-09-23T20:56:46.921939+0000","last_active":"2023-09-23T20:58:11.088132+0000","last_peered":"2023-09-23T20:58:11.088132+0000","last_clean":"2023-09-23T20:58:11.088132+0000","last_became_active":"2023-09-23T20:56:45.989908+0000","last_became_peered":"2023-09-23T20:56:45.989908+0000","last_unstale":"2023-09-23T20:58:11.088132+0000","last_undegraded":"2023-09-23T20:58:11.088132+0000","last_fullsized":"2023-09-23T20:58:11.088132+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":205,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:56:46.921739+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:56:46.921739+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:20:42.635110+0000","scrub_duration":0.0071954109999999996,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":221,"reported_epoch":231,"state":"active+clean","last_fresh":"2023-09-23T20:58:11.088287+0000","last_change":"2023-09-23T20:54:13.961788+0000","last_active":"2023-09-23T20:58:11.088287+0000","last_peered":"2023-09-23T20:58:11.088287+0000","last_clean":"2023-09-23T20:58:11.088287+0000","last_became_active":"2023-09-23T20:54:13.940937+0000","last_became_peered":"2023-09-23T20:54:13.940937+0000","last_unstale":"2023-09-23T20:58:11.088287+0000","last_undegraded":"2023-09-23T20:58:11.088287+0000","last_fullsized":"2023-09-23T20:58:11.088287+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:11:56.785299+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":126,"reported_epoch":231,"state":"active+clean","last_fresh":"2023-09-23T20:58:11.068149+0000","last_change":"2023-09-23T20:55:35.949670+0000","last_active":"2023-09-23T20:58:11.068149+0000","last_peered":"2023-09-23T20:58:11.068149+0000","last_clean":"2023-09-23T20:58:11.068149+0000","last_became_active":"2023-09-23T20:55:35.949140+0000","last_became_peered":"2023-09-23T20:55:35.949140+0000","last_unstale":"2023-09-23T20:58:11.068149+0000","last_undegraded":"2023-09-23T20:58:11.068149+0000","last_fullsized":"2023-09-23T20:58:11.068149+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:55:59.340303+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.e","version":"232'6685","reported_seq":6727,"reported_epoch":232,"state":"active+clean","last_fresh":"2023-09-23T20:58:16.331356+0000","last_change":"2023-09-23T20:56:35.547184+0000","last_active":"2023-09-23T20:58:16.331356+0000","last_peered":"2023-09-23T20:58:16.331356+0000","last_clean":"2023-09-23T20:58:16.331356+0000","last_became_active":"2023-09-23T20:56:35.546912+0000","last_became_peered":"2023-09-23T20:56:35.546912+0000","last_unstale":"2023-09-23T20:58:16.331356+0000","last_undegraded":"2023-09-23T20:58:16.331356+0000","last_fullsized":"2023-09-23T20:58:16.331356+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":199,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_clean_scrub_stamp":"2023-09-23T20:56:34.529103+0000","objects_scrubbed":0,"log_size":6685,"log_dups_size":0,"ondisk_log_size":6685,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:57:07.422792+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":399441920,"num_objects":6095,"num_object_clones":0,"num_object_copies":18285,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6095,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6685,"num_write_kb":408960,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,0],"acting":[2,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.8","version":"79'161","reported_seq":475,"reported_epoch":231,"state":"active+clean","last_fresh":"2023-09-23T20:58:11.068888+0000","last_change":"2023-09-23T20:56:57.505737+0000","last_active":"2023-09-23T20:58:11.068888+0000","last_peered":"2023-09-23T20:58:11.068888+0000","last_clean":"2023-09-23T20:58:11.068888+0000","last_became_active":"2023-09-23T20:56:04.982345+0000","last_became_peered":"2023-09-23T20:56:04.982345+0000","last_unstale":"2023-09-23T20:58:11.068888+0000","last_undegraded":"2023-09-23T20:58:11.068888+0000","last_fullsized":"2023-09-23T20:58:11.068888+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:56:57.505663+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:56:57.505663+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:34:30.109724+0000","scrub_duration":0.026284108,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":194,"reported_epoch":231,"state":"active+clean","last_fresh":"2023-09-23T20:58:11.058039+0000","last_change":"2023-09-23T20:55:35.959942+0000","last_active":"2023-09-23T20:58:11.058039+0000","last_peered":"2023-09-23T20:58:11.058039+0000","last_clean":"2023-09-23T20:58:11.058039+0000","last_became_active":"2023-09-23T20:55:35.957071+0000","last_became_peered":"2023-09-23T20:55:35.957071+0000","last_unstale":"2023-09-23T20:58:11.058039+0000","last_undegraded":"2023-09-23T20:58:11.058039+0000","last_fullsized":"2023-09-23T20:58:11.058039+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:44:30.308036+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":109,"reported_epoch":231,"state":"active+clean","last_fresh":"2023-09-23T20:58:11.068241+0000","last_change":"2023-09-23T20:54:13.951459+0000","last_active":"2023-09-23T20:58:11.068241+0000","last_peered":"2023-09-23T20:58:11.068241+0000","last_clean":"2023-09-23T20:58:11.068241+0000","last_became_active":"2023-09-23T20:54:13.948409+0000","last_became_peered":"2023-09-23T20:54:13.948409+0000","last_unstale":"2023-09-23T20:58:11.068241+0000","last_undegraded":"2023-09-23T20:58:11.068241+0000","last_fullsized":"2023-09-23T20:58:11.068241+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:38:10.148061+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.b","version":"79'161","reported_seq":514,"reported_epoch":231,"state":"active+clean","last_fresh":"2023-09-23T20:58:11.082929+0000","last_change":"2023-09-23T20:57:27.320172+0000","last_active":"2023-09-23T20:58:11.082929+0000","last_peered":"2023-09-23T20:58:11.082929+0000","last_clean":"2023-09-23T20:58:11.082929+0000","last_became_active":"2023-09-23T20:57:16.009630+0000","last_became_peered":"2023-09-23T20:57:16.009630+0000","last_unstale":"2023-09-23T20:58:11.082929+0000","last_undegraded":"2023-09-23T20:58:11.082929+0000","last_fullsized":"2023-09-23T20:58:11.082929+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:27.320022+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:27.320022+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:31:35.221558+0000","scrub_duration":0.0043894679999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,5],"acting":[1,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.d","version":"232'6783","reported_seq":6825,"reported_epoch":232,"state":"active+clean","last_fresh":"2023-09-23T20:58:19.039457+0000","last_change":"2023-09-23T20:56:35.544132+0000","last_active":"2023-09-23T20:58:19.039457+0000","last_peered":"2023-09-23T20:58:19.039457+0000","last_clean":"2023-09-23T20:58:19.039457+0000","last_became_active":"2023-09-23T20:56:35.543836+0000","last_became_peered":"2023-09-23T20:56:35.543836+0000","last_unstale":"2023-09-23T20:58:19.039457+0000","last_undegraded":"2023-09-23T20:58:19.039457+0000","last_fullsized":"2023-09-23T20:58:19.039457+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":199,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_clean_scrub_stamp":"2023-09-23T20:56:34.529103+0000","objects_scrubbed":0,"log_size":6783,"log_dups_size":0,"ondisk_log_size":6783,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:27:14.636991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":393609240,"num_objects":6007,"num_object_clones":0,"num_object_copies":18021,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6007,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":6783,"num_write_kb":409217,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":109,"reported_epoch":231,"state":"active+clean","last_fresh":"2023-09-23T20:58:11.067759+0000","last_change":"2023-09-23T20:54:13.950254+0000","last_active":"2023-09-23T20:58:11.067759+0000","last_peered":"2023-09-23T20:58:11.067759+0000","last_clean":"2023-09-23T20:58:11.067759+0000","last_became_active":"2023-09-23T20:54:13.949934+0000","last_became_peered":"2023-09-23T20:54:13.949934+0000","last_unstale":"2023-09-23T20:58:11.067759+0000","last_undegraded":"2023-09-23T20:58:11.067759+0000","last_fullsized":"2023-09-23T20:58:11.067759+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:30:32.847167+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":126,"reported_epoch":231,"state":"active+clean","last_fresh":"2023-09-23T20:58:11.068243+0000","last_change":"2023-09-23T20:55:35.956153+0000","last_active":"2023-09-23T20:58:11.068243+0000","last_peered":"2023-09-23T20:58:11.068243+0000","last_clean":"2023-09-23T20:58:11.068243+0000","last_became_active":"2023-09-23T20:55:35.955630+0000","last_became_peered":"2023-09-23T20:55:35.955630+0000","last_unstale":"2023-09-23T20:58:11.068243+0000","last_undegraded":"2023-09-23T20:58:11.068243+0000","last_fullsized":"2023-09-23T20:58:11.068243+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:45:20.282929+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.c","version":"232'6798","reported_seq":6849,"reported_epoch":232,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T20:58:19.019873+0000","last_change":"2023-09-23T20:57:46.463005+0000","last_active":"2023-09-23T20:58:19.019873+0000","last_peered":"2023-09-23T20:58:19.019873+0000","last_clean":"2023-09-23T20:57:44.015119+0000","last_became_active":"2023-09-23T20:57:46.448702+0000","last_became_peered":"2023-09-23T20:57:46.448702+0000","last_unstale":"2023-09-23T20:58:19.019873+0000","last_undegraded":"2023-09-23T20:57:46.010874+0000","last_fullsized":"2023-09-23T20:57:45.978434+0000","mapping_epoch":223,"log_start":"0'0","ondisk_log_start":"0'0","created":199,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_clean_scrub_stamp":"2023-09-23T20:56:34.529103+0000","objects_scrubbed":0,"log_size":6798,"log_dups_size":0,"ondisk_log_size":6798,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:08:34.180651+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":393871360,"num_objects":6010,"num_object_clones":0,"num_object_copies":18030,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5473,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6010,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6798,"num_write_kb":409856,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,7],"avail_no_missing":["6","2","7"],"object_location_counts":[{"shards":"2,6,7","objects":5079},{"shards":"6,7","objects":537}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.a","version":"79'161","reported_seq":475,"reported_epoch":231,"state":"active+clean","last_fresh":"2023-09-23T20:58:11.069308+0000","last_change":"2023-09-23T20:56:44.487778+0000","last_active":"2023-09-23T20:58:11.069308+0000","last_peered":"2023-09-23T20:58:11.069308+0000","last_clean":"2023-09-23T20:58:11.069308+0000","last_became_active":"2023-09-23T20:56:05.006740+0000","last_became_peered":"2023-09-23T20:56:05.006740+0000","last_unstale":"2023-09-23T20:58:11.069308+0000","last_undegraded":"2023-09-23T20:58:11.069308+0000","last_fullsized":"2023-09-23T20:58:11.069308+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:56:44.487678+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:56:44.487678+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:04:50.086749+0000","scrub_duration":0.0017408899999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":20,"reported_epoch":231,"state":"active+clean","last_fresh":"2023-09-23T20:58:11.087850+0000","last_change":"2023-09-23T20:57:44.982513+0000","last_active":"2023-09-23T20:58:11.087850+0000","last_peered":"2023-09-23T20:58:11.087850+0000","last_clean":"2023-09-23T20:58:11.087850+0000","last_became_active":"2023-09-23T20:57:44.982112+0000","last_became_peered":"2023-09-23T20:57:44.982112+0000","last_unstale":"2023-09-23T20:58:11.087850+0000","last_undegraded":"2023-09-23T20:58:11.087850+0000","last_fullsized":"2023-09-23T20:58:11.087850+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:09:05.630041+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":201,"reported_epoch":231,"state":"active+clean","last_fresh":"2023-09-23T20:58:11.081867+0000","last_change":"2023-09-23T20:54:13.940748+0000","last_active":"2023-09-23T20:58:11.081867+0000","last_peered":"2023-09-23T20:58:11.081867+0000","last_clean":"2023-09-23T20:58:11.081867+0000","last_became_active":"2023-09-23T20:54:13.940419+0000","last_became_peered":"2023-09-23T20:54:13.940419+0000","last_unstale":"2023-09-23T20:58:11.081867+0000","last_undegraded":"2023-09-23T20:58:11.081867+0000","last_fullsized":"2023-09-23T20:58:11.081867+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:15:01.923816+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.1","version":"232'6718","reported_seq":6772,"reported_epoch":232,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T20:58:17.136400+0000","last_change":"2023-09-23T20:57:46.483248+0000","last_active":"2023-09-23T20:58:17.136400+0000","last_peered":"2023-09-23T20:58:17.136400+0000","last_clean":"2023-09-23T20:57:44.047957+0000","last_became_active":"2023-09-23T20:57:46.475568+0000","last_became_peered":"2023-09-23T20:57:46.475568+0000","last_unstale":"2023-09-23T20:58:17.136400+0000","last_undegraded":"2023-09-23T20:57:46.083591+0000","last_fullsized":"2023-09-23T20:57:45.978436+0000","mapping_epoch":223,"log_start":"0'0","ondisk_log_start":"0'0","created":199,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_clean_scrub_stamp":"2023-09-23T20:56:34.529103+0000","objects_scrubbed":0,"log_size":6718,"log_dups_size":0,"ondisk_log_size":6718,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:22:45.067269+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":397672448,"num_objects":6068,"num_object_clones":0,"num_object_copies":18204,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5470,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6068,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6718,"num_write_kb":409152,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,6],"avail_no_missing":["4","1","6"],"object_location_counts":[{"shards":"1,4,6","objects":5145},{"shards":"4,6","objects":598}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.7","version":"79'161","reported_seq":537,"reported_epoch":231,"state":"active+clean","last_fresh":"2023-09-23T20:58:11.058248+0000","last_change":"2023-09-23T20:57:48.243572+0000","last_active":"2023-09-23T20:58:11.058248+0000","last_peered":"2023-09-23T20:58:11.058248+0000","last_clean":"2023-09-23T20:58:11.058248+0000","last_became_active":"2023-09-23T20:57:48.243229+0000","last_became_peered":"2023-09-23T20:57:48.243229+0000","last_unstale":"2023-09-23T20:58:11.058248+0000","last_undegraded":"2023-09-23T20:58:11.058248+0000","last_fullsized":"2023-09-23T20:58:11.058248+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":3,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:25.916867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:25.916867+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:23:19.385831+0000","scrub_duration":0.001665323,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":206,"reported_epoch":231,"state":"active+clean","last_fresh":"2023-09-23T20:58:11.082649+0000","last_change":"2023-09-23T20:50:05.867130+0000","last_active":"2023-09-23T20:58:11.082649+0000","last_peered":"2023-09-23T20:58:11.082649+0000","last_clean":"2023-09-23T20:58:11.082649+0000","last_became_active":"2023-09-23T20:50:05.866758+0000","last_became_peered":"2023-09-23T20:50:05.866758+0000","last_unstale":"2023-09-23T20:58:11.082649+0000","last_undegraded":"2023-09-23T20:58:11.082649+0000","last_fullsized":"2023-09-23T20:58:11.082649+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:55:07.548209+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":105,"reported_epoch":231,"state":"active+clean","last_fresh":"2023-09-23T20:58:11.069310+0000","last_change":"2023-09-23T20:54:31.887604+0000","last_active":"2023-09-23T20:58:11.069310+0000","last_peered":"2023-09-23T20:58:11.069310+0000","last_clean":"2023-09-23T20:58:11.069310+0000","last_became_active":"2023-09-23T20:54:31.886752+0000","last_became_peered":"2023-09-23T20:54:31.886752+0000","last_unstale":"2023-09-23T20:58:11.069310+0000","last_undegraded":"2023-09-23T20:58:11.069310+0000","last_fullsized":"2023-09-23T20:58:11.069310+0000","mapping_epoch":136,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:13:58.872176+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":204,"reported_epoch":231,"state":"active+clean","last_fresh":"2023-09-23T20:58:11.082938+0000","last_change":"2023-09-23T20:54:31.899082+0000","last_active":"2023-09-23T20:58:11.082938+0000","last_peered":"2023-09-23T20:58:11.082938+0000","last_clean":"2023-09-23T20:58:11.082938+0000","last_became_active":"2023-09-23T20:54:31.898521+0000","last_became_peered":"2023-09-23T20:54:31.898521+0000","last_unstale":"2023-09-23T20:58:11.082938+0000","last_undegraded":"2023-09-23T20:58:11.082938+0000","last_fullsized":"2023-09-23T20:58:11.082938+0000","mapping_epoch":136,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:25:36.059711+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.7","version":"232'6794","reported_seq":6835,"reported_epoch":232,"state":"active+clean","last_fresh":"2023-09-23T20:58:19.052228+0000","last_change":"2023-09-23T20:56:35.541695+0000","last_active":"2023-09-23T20:58:19.052228+0000","last_peered":"2023-09-23T20:58:19.052228+0000","last_clean":"2023-09-23T20:58:19.052228+0000","last_became_active":"2023-09-23T20:56:35.541392+0000","last_became_peered":"2023-09-23T20:56:35.541392+0000","last_unstale":"2023-09-23T20:58:19.052228+0000","last_undegraded":"2023-09-23T20:58:19.052228+0000","last_fullsized":"2023-09-23T20:58:19.052228+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":199,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_clean_scrub_stamp":"2023-09-23T20:56:34.529103+0000","objects_scrubbed":0,"log_size":6794,"log_dups_size":0,"ondisk_log_size":6794,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:57:22.478253+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":394264576,"num_objects":6016,"num_object_clones":0,"num_object_copies":18048,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6016,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6794,"num_write_kb":409920,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.1","version":"79'161","reported_seq":495,"reported_epoch":231,"state":"active+clean","last_fresh":"2023-09-23T20:58:11.068887+0000","last_change":"2023-09-23T20:57:19.669346+0000","last_active":"2023-09-23T20:58:11.068887+0000","last_peered":"2023-09-23T20:58:11.068887+0000","last_clean":"2023-09-23T20:58:11.068887+0000","last_became_active":"2023-09-23T20:57:08.246046+0000","last_became_peered":"2023-09-23T20:57:08.246046+0000","last_unstale":"2023-09-23T20:58:11.068887+0000","last_undegraded":"2023-09-23T20:58:11.068887+0000","last_fullsized":"2023-09-23T20:58:11.068887+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:19.669259+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:19.669259+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:14:03.435531+0000","scrub_duration":0.0045759559999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":122,"reported_epoch":231,"state":"active+clean","last_fresh":"2023-09-23T20:58:11.069182+0000","last_change":"2023-09-23T20:55:35.959514+0000","last_active":"2023-09-23T20:58:11.069182+0000","last_peered":"2023-09-23T20:58:11.069182+0000","last_clean":"2023-09-23T20:58:11.069182+0000","last_became_active":"2023-09-23T20:55:35.959244+0000","last_became_peered":"2023-09-23T20:55:35.959244+0000","last_unstale":"2023-09-23T20:58:11.069182+0000","last_undegraded":"2023-09-23T20:58:11.069182+0000","last_fullsized":"2023-09-23T20:58:11.069182+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:47:32.665316+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":126,"reported_epoch":231,"state":"active+clean","last_fresh":"2023-09-23T20:58:11.068168+0000","last_change":"2023-09-23T20:55:35.949000+0000","last_active":"2023-09-23T20:58:11.068168+0000","last_peered":"2023-09-23T20:58:11.068168+0000","last_clean":"2023-09-23T20:58:11.068168+0000","last_became_active":"2023-09-23T20:55:35.948410+0000","last_became_peered":"2023-09-23T20:55:35.948410+0000","last_unstale":"2023-09-23T20:58:11.068168+0000","last_undegraded":"2023-09-23T20:58:11.068168+0000","last_fullsized":"2023-09-23T20:58:11.068168+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:26:50.477289+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":198,"reported_epoch":231,"state":"active+clean","last_fresh":"2023-09-23T20:58:11.082438+0000","last_change":"2023-09-23T20:51:10.829889+0000","last_active":"2023-09-23T20:58:11.082438+0000","last_peered":"2023-09-23T20:58:11.082438+0000","last_clean":"2023-09-23T20:58:11.082438+0000","last_became_active":"2023-09-23T20:51:10.829533+0000","last_became_peered":"2023-09-23T20:51:10.829533+0000","last_unstale":"2023-09-23T20:58:11.082438+0000","last_undegraded":"2023-09-23T20:58:11.082438+0000","last_fullsized":"2023-09-23T20:58:11.082438+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:43:04.603460+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.4","version":"232'6632","reported_seq":6674,"reported_epoch":232,"state":"active+clean","last_fresh":"2023-09-23T20:58:16.263486+0000","last_change":"2023-09-23T20:56:35.549909+0000","last_active":"2023-09-23T20:58:16.263486+0000","last_peered":"2023-09-23T20:58:16.263486+0000","last_clean":"2023-09-23T20:58:16.263486+0000","last_became_active":"2023-09-23T20:56:35.549618+0000","last_became_peered":"2023-09-23T20:56:35.549618+0000","last_unstale":"2023-09-23T20:58:16.263486+0000","last_undegraded":"2023-09-23T20:58:16.263486+0000","last_fullsized":"2023-09-23T20:58:16.263486+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":199,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_clean_scrub_stamp":"2023-09-23T20:56:34.529103+0000","objects_scrubbed":0,"log_size":6632,"log_dups_size":0,"ondisk_log_size":6632,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:24:25.918475+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":399114240,"num_objects":6090,"num_object_clones":0,"num_object_copies":18270,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6090,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6632,"num_write_kb":407104,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.2","version":"79'161","reported_seq":461,"reported_epoch":231,"state":"active+clean","last_fresh":"2023-09-23T20:58:11.083070+0000","last_change":"2023-09-23T20:57:24.802586+0000","last_active":"2023-09-23T20:58:11.083070+0000","last_peered":"2023-09-23T20:58:11.083070+0000","last_clean":"2023-09-23T20:58:11.083070+0000","last_became_active":"2023-09-23T20:57:08.236114+0000","last_became_peered":"2023-09-23T20:57:08.236114+0000","last_unstale":"2023-09-23T20:58:11.083070+0000","last_undegraded":"2023-09-23T20:58:11.083070+0000","last_fullsized":"2023-09-23T20:58:11.083070+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:24.802451+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:24.802451+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:06:22.309980+0000","scrub_duration":0.0090861250000000005,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,4],"acting":[0,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":217,"reported_epoch":231,"state":"active+clean","last_fresh":"2023-09-23T20:58:11.093166+0000","last_change":"2023-09-23T20:51:10.838054+0000","last_active":"2023-09-23T20:58:11.093166+0000","last_peered":"2023-09-23T20:58:11.093166+0000","last_clean":"2023-09-23T20:58:11.093166+0000","last_became_active":"2023-09-23T20:51:10.837750+0000","last_became_peered":"2023-09-23T20:51:10.837750+0000","last_unstale":"2023-09-23T20:58:11.093166+0000","last_undegraded":"2023-09-23T20:58:11.093166+0000","last_fullsized":"2023-09-23T20:58:11.093166+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:15:51.648085+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":199,"reported_epoch":231,"state":"active+clean","last_fresh":"2023-09-23T20:58:11.083370+0000","last_change":"2023-09-23T20:55:35.951146+0000","last_active":"2023-09-23T20:58:11.083370+0000","last_peered":"2023-09-23T20:58:11.083370+0000","last_clean":"2023-09-23T20:58:11.083370+0000","last_became_active":"2023-09-23T20:55:35.950874+0000","last_became_peered":"2023-09-23T20:55:35.950874+0000","last_unstale":"2023-09-23T20:58:11.083370+0000","last_undegraded":"2023-09-23T20:58:11.083370+0000","last_fullsized":"2023-09-23T20:58:11.083370+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:58:13.957677+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":219,"reported_epoch":231,"state":"active+clean","last_fresh":"2023-09-23T20:58:11.081798+0000","last_change":"2023-09-23T20:54:31.898870+0000","last_active":"2023-09-23T20:58:11.081798+0000","last_peered":"2023-09-23T20:58:11.081798+0000","last_clean":"2023-09-23T20:58:11.081798+0000","last_became_active":"2023-09-23T20:54:31.898467+0000","last_became_peered":"2023-09-23T20:54:31.898467+0000","last_unstale":"2023-09-23T20:58:11.081798+0000","last_undegraded":"2023-09-23T20:58:11.081798+0000","last_fullsized":"2023-09-23T20:58:11.081798+0000","mapping_epoch":136,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:56:22.483408+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.6","version":"232'6546","reported_seq":7850,"reported_epoch":232,"state":"active+recovering+undersized+remapped","last_fresh":"2023-09-23T20:58:16.298317+0000","last_change":"2023-09-23T20:57:46.535681+0000","last_active":"2023-09-23T20:58:16.298317+0000","last_peered":"2023-09-23T20:58:16.298317+0000","last_clean":"2023-09-23T20:57:44.031597+0000","last_became_active":"2023-09-23T20:57:46.443927+0000","last_became_peered":"2023-09-23T20:57:46.443927+0000","last_unstale":"2023-09-23T20:58:16.298317+0000","last_undegraded":"2023-09-23T20:58:16.298317+0000","last_fullsized":"2023-09-23T20:57:45.977227+0000","mapping_epoch":223,"log_start":"0'0","ondisk_log_start":"0'0","created":199,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_clean_scrub_stamp":"2023-09-23T20:56:34.529103+0000","objects_scrubbed":0,"log_size":6546,"log_dups_size":0,"ondisk_log_size":6546,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:37:23.911131+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":395968512,"num_objects":6042,"num_object_clones":0,"num_object_copies":18126,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":4771,"num_objects_unfound":0,"num_objects_dirty":6042,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6546,"num_write_kb":402816,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":627,"num_bytes_recovered":41091072,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7],"avail_no_missing":["0","5","7"],"object_location_counts":[{"shards":"0,5,7","objects":4771},{"shards":"0,7","objects":1271}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":105,"reported_epoch":231,"state":"active+clean","last_fresh":"2023-09-23T20:58:11.069455+0000","last_change":"2023-09-23T20:54:31.883504+0000","last_active":"2023-09-23T20:58:11.069455+0000","last_peered":"2023-09-23T20:58:11.069455+0000","last_clean":"2023-09-23T20:58:11.069455+0000","last_became_active":"2023-09-23T20:54:31.883172+0000","last_became_peered":"2023-09-23T20:54:31.883172+0000","last_unstale":"2023-09-23T20:58:11.069455+0000","last_undegraded":"2023-09-23T20:58:11.069455+0000","last_fullsized":"2023-09-23T20:58:11.069455+0000","mapping_epoch":136,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:45:40.289120+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"79'161","reported_seq":461,"reported_epoch":231,"state":"active+clean","last_fresh":"2023-09-23T20:58:11.083865+0000","last_change":"2023-09-23T20:57:20.738417+0000","last_active":"2023-09-23T20:58:11.083865+0000","last_peered":"2023-09-23T20:58:11.083865+0000","last_clean":"2023-09-23T20:58:11.083865+0000","last_became_active":"2023-09-23T20:57:08.236842+0000","last_became_peered":"2023-09-23T20:57:08.236842+0000","last_unstale":"2023-09-23T20:58:11.083865+0000","last_undegraded":"2023-09-23T20:58:11.083865+0000","last_fullsized":"2023-09-23T20:58:11.083865+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":0,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:20.738273+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:20.738273+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:18:28.327990+0000","scrub_duration":0.0088336339999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":209,"reported_epoch":231,"state":"active+clean","last_fresh":"2023-09-23T20:58:11.102041+0000","last_change":"2023-09-23T20:55:35.950777+0000","last_active":"2023-09-23T20:58:11.102041+0000","last_peered":"2023-09-23T20:58:11.102041+0000","last_clean":"2023-09-23T20:58:11.102041+0000","last_became_active":"2023-09-23T20:55:35.950427+0000","last_became_peered":"2023-09-23T20:55:35.950427+0000","last_unstale":"2023-09-23T20:58:11.102041+0000","last_undegraded":"2023-09-23T20:58:11.102041+0000","last_fullsized":"2023-09-23T20:58:11.102041+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:20:10.371751+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":176,"reported_epoch":231,"state":"active+clean","last_fresh":"2023-09-23T20:58:11.058400+0000","last_change":"2023-09-23T20:55:35.978140+0000","last_active":"2023-09-23T20:58:11.058400+0000","last_peered":"2023-09-23T20:58:11.058400+0000","last_clean":"2023-09-23T20:58:11.058400+0000","last_became_active":"2023-09-23T20:55:35.977831+0000","last_became_peered":"2023-09-23T20:55:35.977831+0000","last_unstale":"2023-09-23T20:58:11.058400+0000","last_undegraded":"2023-09-23T20:58:11.058400+0000","last_fullsized":"2023-09-23T20:58:11.058400+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:25:47.678023+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.5","version":"232'6620","reported_seq":6669,"reported_epoch":232,"state":"active+clean","last_fresh":"2023-09-23T20:58:16.388128+0000","last_change":"2023-09-23T20:56:45.990110+0000","last_active":"2023-09-23T20:58:16.388128+0000","last_peered":"2023-09-23T20:58:16.388128+0000","last_clean":"2023-09-23T20:58:16.388128+0000","last_became_active":"2023-09-23T20:56:45.988394+0000","last_became_peered":"2023-09-23T20:56:45.988394+0000","last_unstale":"2023-09-23T20:58:16.388128+0000","last_undegraded":"2023-09-23T20:58:16.388128+0000","last_fullsized":"2023-09-23T20:58:16.388128+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":199,"last_epoch_clean":205,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_clean_scrub_stamp":"2023-09-23T20:56:34.529103+0000","objects_scrubbed":0,"log_size":6620,"log_dups_size":0,"ondisk_log_size":6620,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:22:03.814210+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":397017088,"num_objects":6058,"num_object_clones":0,"num_object_copies":18174,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6058,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6620,"num_write_kb":405696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,0],"acting":[2,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"79'161","reported_seq":498,"reported_epoch":231,"state":"active+clean","last_fresh":"2023-09-23T20:58:11.083095+0000","last_change":"2023-09-23T20:57:20.378531+0000","last_active":"2023-09-23T20:58:11.083095+0000","last_peered":"2023-09-23T20:58:11.083095+0000","last_clean":"2023-09-23T20:58:11.083095+0000","last_became_active":"2023-09-23T20:57:08.238019+0000","last_became_peered":"2023-09-23T20:57:08.238019+0000","last_unstale":"2023-09-23T20:58:11.083095+0000","last_undegraded":"2023-09-23T20:58:11.083095+0000","last_fullsized":"2023-09-23T20:58:11.083095+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":3,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:20.377714+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:20.377714+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:00:08.305602+0000","scrub_duration":0.003015228,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":100,"reported_epoch":231,"state":"active+clean","last_fresh":"2023-09-23T20:58:11.083505+0000","last_change":"2023-09-23T20:56:45.991884+0000","last_active":"2023-09-23T20:58:11.083505+0000","last_peered":"2023-09-23T20:58:11.083505+0000","last_clean":"2023-09-23T20:58:11.083505+0000","last_became_active":"2023-09-23T20:56:45.991568+0000","last_became_peered":"2023-09-23T20:56:45.991568+0000","last_unstale":"2023-09-23T20:58:11.083505+0000","last_undegraded":"2023-09-23T20:58:11.083505+0000","last_fullsized":"2023-09-23T20:58:11.083505+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":205,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:21:20.775511+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,0],"acting":[1,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":225,"reported_epoch":231,"state":"active+clean","last_fresh":"2023-09-23T20:58:11.104008+0000","last_change":"2023-09-23T20:57:44.979950+0000","last_active":"2023-09-23T20:58:11.104008+0000","last_peered":"2023-09-23T20:58:11.104008+0000","last_clean":"2023-09-23T20:58:11.104008+0000","last_became_active":"2023-09-23T20:57:44.979346+0000","last_became_peered":"2023-09-23T20:57:44.979346+0000","last_unstale":"2023-09-23T20:58:11.104008+0000","last_undegraded":"2023-09-23T20:58:11.104008+0000","last_fullsized":"2023-09-23T20:58:11.104008+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:10:24.372800+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":204,"reported_epoch":231,"state":"active+clean","last_fresh":"2023-09-23T20:58:11.294658+0000","last_change":"2023-09-23T20:50:37.982235+0000","last_active":"2023-09-23T20:58:11.294658+0000","last_peered":"2023-09-23T20:58:11.294658+0000","last_clean":"2023-09-23T20:58:11.294658+0000","last_became_active":"2023-09-23T20:50:37.981913+0000","last_became_peered":"2023-09-23T20:50:37.981913+0000","last_unstale":"2023-09-23T20:58:11.294658+0000","last_undegraded":"2023-09-23T20:58:11.294658+0000","last_fullsized":"2023-09-23T20:58:11.294658+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:59:57.122584+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.0","version":"232'6658","reported_seq":6699,"reported_epoch":232,"state":"active+clean","last_fresh":"2023-09-23T20:58:17.068032+0000","last_change":"2023-09-23T20:56:35.550635+0000","last_active":"2023-09-23T20:58:17.068032+0000","last_peered":"2023-09-23T20:58:17.068032+0000","last_clean":"2023-09-23T20:58:17.068032+0000","last_became_active":"2023-09-23T20:56:35.550376+0000","last_became_peered":"2023-09-23T20:56:35.550376+0000","last_unstale":"2023-09-23T20:58:17.068032+0000","last_undegraded":"2023-09-23T20:58:17.068032+0000","last_fullsized":"2023-09-23T20:58:17.068032+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":199,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_clean_scrub_stamp":"2023-09-23T20:56:34.529103+0000","objects_scrubbed":0,"log_size":6658,"log_dups_size":0,"ondisk_log_size":6658,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:40:55.020049+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":394788864,"num_objects":6024,"num_object_clones":0,"num_object_copies":18072,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6024,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6658,"num_write_kb":405824,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.6","version":"79'161","reported_seq":514,"reported_epoch":231,"state":"active+clean","last_fresh":"2023-09-23T20:58:11.057967+0000","last_change":"2023-09-23T20:57:26.907505+0000","last_active":"2023-09-23T20:58:11.057967+0000","last_peered":"2023-09-23T20:58:11.057967+0000","last_clean":"2023-09-23T20:58:11.057967+0000","last_became_active":"2023-09-23T20:57:08.240321+0000","last_became_peered":"2023-09-23T20:57:08.240321+0000","last_unstale":"2023-09-23T20:58:11.057967+0000","last_undegraded":"2023-09-23T20:58:11.057967+0000","last_fullsized":"2023-09-23T20:58:11.057967+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":3,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:26.907408+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:26.907408+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:34:17.392928+0000","scrub_duration":0.010337038,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":147,"reported_epoch":231,"state":"active+clean","last_fresh":"2023-09-23T20:58:11.084287+0000","last_change":"2023-09-23T20:56:45.988566+0000","last_active":"2023-09-23T20:58:11.084287+0000","last_peered":"2023-09-23T20:58:11.084287+0000","last_clean":"2023-09-23T20:58:11.084287+0000","last_became_active":"2023-09-23T20:56:45.988249+0000","last_became_peered":"2023-09-23T20:56:45.988249+0000","last_unstale":"2023-09-23T20:58:11.084287+0000","last_undegraded":"2023-09-23T20:58:11.084287+0000","last_fullsized":"2023-09-23T20:58:11.084287+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":205,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:51:36.644764+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,4],"acting":[0,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":201,"reported_epoch":231,"state":"active+clean","last_fresh":"2023-09-23T20:58:11.082576+0000","last_change":"2023-09-23T20:54:31.898965+0000","last_active":"2023-09-23T20:58:11.082576+0000","last_peered":"2023-09-23T20:58:11.082576+0000","last_clean":"2023-09-23T20:58:11.082576+0000","last_became_active":"2023-09-23T20:54:31.898663+0000","last_became_peered":"2023-09-23T20:54:31.898663+0000","last_unstale":"2023-09-23T20:58:11.082576+0000","last_undegraded":"2023-09-23T20:58:11.082576+0000","last_fullsized":"2023-09-23T20:58:11.082576+0000","mapping_epoch":136,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:53:05.058087+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":192,"reported_epoch":231,"state":"active+clean","last_fresh":"2023-09-23T20:58:11.083677+0000","last_change":"2023-09-23T20:54:31.884717+0000","last_active":"2023-09-23T20:58:11.083677+0000","last_peered":"2023-09-23T20:58:11.083677+0000","last_clean":"2023-09-23T20:58:11.083677+0000","last_became_active":"2023-09-23T20:54:31.884345+0000","last_became_peered":"2023-09-23T20:54:31.884345+0000","last_unstale":"2023-09-23T20:58:11.083677+0000","last_undegraded":"2023-09-23T20:58:11.083677+0000","last_fullsized":"2023-09-23T20:58:11.083677+0000","mapping_epoch":136,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:02:26.393868+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.3","version":"232'6678","reported_seq":6720,"reported_epoch":232,"state":"active+clean","last_fresh":"2023-09-23T20:58:19.347713+0000","last_change":"2023-09-23T20:56:35.549655+0000","last_active":"2023-09-23T20:58:19.347713+0000","last_peered":"2023-09-23T20:58:19.347713+0000","last_clean":"2023-09-23T20:58:19.347713+0000","last_became_active":"2023-09-23T20:56:35.549357+0000","last_became_peered":"2023-09-23T20:56:35.549357+0000","last_unstale":"2023-09-23T20:58:19.347713+0000","last_undegraded":"2023-09-23T20:58:19.347713+0000","last_fullsized":"2023-09-23T20:58:19.347713+0000","mapping_epoch":199,"log_start":"202'500","ondisk_log_start":"202'500","created":199,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_clean_scrub_stamp":"2023-09-23T20:56:34.529103+0000","objects_scrubbed":0,"log_size":6178,"log_dups_size":0,"ondisk_log_size":6178,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:37:58.752518+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":382992384,"num_objects":5844,"num_object_clones":0,"num_object_copies":17532,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5844,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6678,"num_write_kb":400704,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.5","version":"79'161","reported_seq":495,"reported_epoch":231,"state":"active+clean","last_fresh":"2023-09-23T20:58:11.067966+0000","last_change":"2023-09-23T20:57:30.029479+0000","last_active":"2023-09-23T20:58:11.067966+0000","last_peered":"2023-09-23T20:58:11.067966+0000","last_clean":"2023-09-23T20:58:11.067966+0000","last_became_active":"2023-09-23T20:57:08.249909+0000","last_became_peered":"2023-09-23T20:57:08.249909+0000","last_unstale":"2023-09-23T20:58:11.067966+0000","last_undegraded":"2023-09-23T20:58:11.067966+0000","last_fullsized":"2023-09-23T20:58:11.067966+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":3,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:30.029396+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:30.029396+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:16:46.083145+0000","scrub_duration":0.012914682,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":195,"reported_epoch":231,"state":"active+clean","last_fresh":"2023-09-23T20:58:11.112028+0000","last_change":"2023-09-23T20:51:10.836910+0000","last_active":"2023-09-23T20:58:11.112028+0000","last_peered":"2023-09-23T20:58:11.112028+0000","last_clean":"2023-09-23T20:58:11.112028+0000","last_became_active":"2023-09-23T20:51:10.836557+0000","last_became_peered":"2023-09-23T20:51:10.836557+0000","last_unstale":"2023-09-23T20:58:11.112028+0000","last_undegraded":"2023-09-23T20:58:11.112028+0000","last_fullsized":"2023-09-23T20:58:11.112028+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:51:54.019969+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":201,"reported_epoch":231,"state":"active+clean","last_fresh":"2023-09-23T20:58:11.081537+0000","last_change":"2023-09-23T20:55:35.953444+0000","last_active":"2023-09-23T20:58:11.081537+0000","last_peered":"2023-09-23T20:58:11.081537+0000","last_clean":"2023-09-23T20:58:11.081537+0000","last_became_active":"2023-09-23T20:55:35.953151+0000","last_became_peered":"2023-09-23T20:55:35.953151+0000","last_unstale":"2023-09-23T20:58:11.081537+0000","last_undegraded":"2023-09-23T20:58:11.081537+0000","last_fullsized":"2023-09-23T20:58:11.081537+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:39:12.404391+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":20,"reported_epoch":231,"state":"active+clean","last_fresh":"2023-09-23T20:58:11.087841+0000","last_change":"2023-09-23T20:57:44.982509+0000","last_active":"2023-09-23T20:58:11.087841+0000","last_peered":"2023-09-23T20:58:11.087841+0000","last_clean":"2023-09-23T20:58:11.087841+0000","last_became_active":"2023-09-23T20:57:44.982175+0000","last_became_peered":"2023-09-23T20:57:44.982175+0000","last_unstale":"2023-09-23T20:58:11.087841+0000","last_undegraded":"2023-09-23T20:58:11.087841+0000","last_fullsized":"2023-09-23T20:58:11.087841+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:37:49.215488+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.2","version":"232'6655","reported_seq":6697,"reported_epoch":232,"state":"active+clean","last_fresh":"2023-09-23T20:58:19.357620+0000","last_change":"2023-09-23T20:56:35.543020+0000","last_active":"2023-09-23T20:58:19.357620+0000","last_peered":"2023-09-23T20:58:19.357620+0000","last_clean":"2023-09-23T20:58:19.357620+0000","last_became_active":"2023-09-23T20:56:35.542719+0000","last_became_peered":"2023-09-23T20:56:35.542719+0000","last_unstale":"2023-09-23T20:58:19.357620+0000","last_undegraded":"2023-09-23T20:58:19.357620+0000","last_fullsized":"2023-09-23T20:58:19.357620+0000","mapping_epoch":199,"log_start":"202'500","ondisk_log_start":"202'500","created":199,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_clean_scrub_stamp":"2023-09-23T20:56:34.529103+0000","objects_scrubbed":0,"log_size":6155,"log_dups_size":0,"ondisk_log_size":6155,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:52:21.088628+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":383844352,"num_objects":5857,"num_object_clones":0,"num_object_copies":17571,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5857,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6655,"num_write_kb":400384,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.4","version":"79'161","reported_seq":510,"reported_epoch":231,"state":"active+clean","last_fresh":"2023-09-23T20:58:11.100644+0000","last_change":"2023-09-23T20:57:20.012904+0000","last_active":"2023-09-23T20:58:11.100644+0000","last_peered":"2023-09-23T20:58:11.100644+0000","last_clean":"2023-09-23T20:58:11.100644+0000","last_became_active":"2023-09-23T20:57:08.237996+0000","last_became_peered":"2023-09-23T20:57:08.237996+0000","last_unstale":"2023-09-23T20:58:11.100644+0000","last_undegraded":"2023-09-23T20:58:11.100644+0000","last_fullsized":"2023-09-23T20:58:11.100644+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":3,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:20.012764+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:20.012764+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:51:28.796946+0000","scrub_duration":0.0045251919999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":181,"reported_epoch":231,"state":"active+clean","last_fresh":"2023-09-23T20:58:11.058089+0000","last_change":"2023-09-23T20:55:35.949541+0000","last_active":"2023-09-23T20:58:11.058089+0000","last_peered":"2023-09-23T20:58:11.058089+0000","last_clean":"2023-09-23T20:58:11.058089+0000","last_became_active":"2023-09-23T20:55:35.944635+0000","last_became_peered":"2023-09-23T20:55:35.944635+0000","last_unstale":"2023-09-23T20:58:11.058089+0000","last_undegraded":"2023-09-23T20:58:11.058089+0000","last_fullsized":"2023-09-23T20:58:11.058089+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:30:36.033585+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"70'1","reported_seq":220,"reported_epoch":231,"state":"active+clean","last_fresh":"2023-09-23T20:58:11.294674+0000","last_change":"2023-09-23T20:54:14.009723+0000","last_active":"2023-09-23T20:58:11.294674+0000","last_peered":"2023-09-23T20:58:11.294674+0000","last_clean":"2023-09-23T20:58:11.294674+0000","last_became_active":"2023-09-23T20:54:13.941771+0000","last_became_peered":"2023-09-23T20:54:13.941771+0000","last_unstale":"2023-09-23T20:58:11.294674+0000","last_undegraded":"2023-09-23T20:58:11.294674+0000","last_fullsized":"2023-09-23T20:58:11.294674+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:48:25.588116+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":136,"reported_epoch":231,"state":"active+clean","last_fresh":"2023-09-23T20:58:11.067847+0000","last_change":"2023-09-23T20:55:35.949780+0000","last_active":"2023-09-23T20:58:11.067847+0000","last_peered":"2023-09-23T20:58:11.067847+0000","last_clean":"2023-09-23T20:58:11.067847+0000","last_became_active":"2023-09-23T20:55:35.949497+0000","last_became_peered":"2023-09-23T20:55:35.949497+0000","last_unstale":"2023-09-23T20:58:11.067847+0000","last_undegraded":"2023-09-23T20:58:11.067847+0000","last_fullsized":"2023-09-23T20:58:11.067847+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:09:37.420882+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":194,"reported_epoch":231,"state":"active+clean","last_fresh":"2023-09-23T20:58:11.083588+0000","last_change":"2023-09-23T20:54:13.929196+0000","last_active":"2023-09-23T20:58:11.083588+0000","last_peered":"2023-09-23T20:58:11.083588+0000","last_clean":"2023-09-23T20:58:11.083588+0000","last_became_active":"2023-09-23T20:54:13.928831+0000","last_became_peered":"2023-09-23T20:54:13.928831+0000","last_unstale":"2023-09-23T20:58:11.083588+0000","last_undegraded":"2023-09-23T20:58:11.083588+0000","last_fullsized":"2023-09-23T20:58:11.083588+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:28:44.316851+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":200,"reported_epoch":231,"state":"active+clean","last_fresh":"2023-09-23T20:58:11.092755+0000","last_change":"2023-09-23T20:52:12.855430+0000","last_active":"2023-09-23T20:58:11.092755+0000","last_peered":"2023-09-23T20:58:11.092755+0000","last_clean":"2023-09-23T20:58:11.092755+0000","last_became_active":"2023-09-23T20:52:12.855115+0000","last_became_peered":"2023-09-23T20:52:12.855115+0000","last_unstale":"2023-09-23T20:58:11.092755+0000","last_undegraded":"2023-09-23T20:58:11.092755+0000","last_fullsized":"2023-09-23T20:58:11.092755+0000","mapping_epoch":100,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":101,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:33:23.802214+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":109,"reported_epoch":231,"state":"active+clean","last_fresh":"2023-09-23T20:58:11.069151+0000","last_change":"2023-09-23T20:54:31.902946+0000","last_active":"2023-09-23T20:58:11.069151+0000","last_peered":"2023-09-23T20:58:11.069151+0000","last_clean":"2023-09-23T20:58:11.069151+0000","last_became_active":"2023-09-23T20:54:31.889484+0000","last_became_peered":"2023-09-23T20:54:31.889484+0000","last_unstale":"2023-09-23T20:58:11.069151+0000","last_undegraded":"2023-09-23T20:58:11.069151+0000","last_fullsized":"2023-09-23T20:58:11.069151+0000","mapping_epoch":136,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:44:44.278398+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":109,"reported_epoch":231,"state":"active+clean","last_fresh":"2023-09-23T20:58:11.068242+0000","last_change":"2023-09-23T20:54:13.950351+0000","last_active":"2023-09-23T20:58:11.068242+0000","last_peered":"2023-09-23T20:58:11.068242+0000","last_clean":"2023-09-23T20:58:11.068242+0000","last_became_active":"2023-09-23T20:54:13.949987+0000","last_became_peered":"2023-09-23T20:54:13.949987+0000","last_unstale":"2023-09-23T20:58:11.068242+0000","last_undegraded":"2023-09-23T20:58:11.068242+0000","last_fullsized":"2023-09-23T20:58:11.068242+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:46:17.765134+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":140,"reported_epoch":231,"state":"active+clean","last_fresh":"2023-09-23T20:58:11.069392+0000","last_change":"2023-09-23T20:56:18.141728+0000","last_active":"2023-09-23T20:58:11.069392+0000","last_peered":"2023-09-23T20:58:11.069392+0000","last_clean":"2023-09-23T20:58:11.069392+0000","last_became_active":"2023-09-23T20:56:18.137979+0000","last_became_peered":"2023-09-23T20:56:18.137979+0000","last_unstale":"2023-09-23T20:58:11.069392+0000","last_undegraded":"2023-09-23T20:58:11.069392+0000","last_fullsized":"2023-09-23T20:58:11.069392+0000","mapping_epoch":192,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":193,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:58:05.658443+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":109,"reported_epoch":231,"state":"active+clean","last_fresh":"2023-09-23T20:58:11.067832+0000","last_change":"2023-09-23T20:54:13.935307+0000","last_active":"2023-09-23T20:58:11.067832+0000","last_peered":"2023-09-23T20:58:11.067832+0000","last_clean":"2023-09-23T20:58:11.067832+0000","last_became_active":"2023-09-23T20:54:13.935000+0000","last_became_peered":"2023-09-23T20:54:13.935000+0000","last_unstale":"2023-09-23T20:58:11.067832+0000","last_undegraded":"2023-09-23T20:58:11.067832+0000","last_fullsized":"2023-09-23T20:58:11.067832+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:47:40.974082+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":218,"reported_epoch":231,"state":"active+clean","last_fresh":"2023-09-23T20:58:11.112405+0000","last_change":"2023-09-23T20:55:35.954870+0000","last_active":"2023-09-23T20:58:11.112405+0000","last_peered":"2023-09-23T20:58:11.112405+0000","last_clean":"2023-09-23T20:58:11.112405+0000","last_became_active":"2023-09-23T20:55:35.954568+0000","last_became_peered":"2023-09-23T20:55:35.954568+0000","last_unstale":"2023-09-23T20:58:11.112405+0000","last_undegraded":"2023-09-23T20:58:11.112405+0000","last_fullsized":"2023-09-23T20:58:11.112405+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:06:22.423566+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":217,"reported_epoch":231,"state":"active+clean","last_fresh":"2023-09-23T20:58:11.088049+0000","last_change":"2023-09-23T20:51:10.838334+0000","last_active":"2023-09-23T20:58:11.088049+0000","last_peered":"2023-09-23T20:58:11.088049+0000","last_clean":"2023-09-23T20:58:11.088049+0000","last_became_active":"2023-09-23T20:51:10.838026+0000","last_became_peered":"2023-09-23T20:51:10.838026+0000","last_unstale":"2023-09-23T20:58:11.088049+0000","last_undegraded":"2023-09-23T20:58:11.088049+0000","last_fullsized":"2023-09-23T20:58:11.088049+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:04:37.009457+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":222,"reported_epoch":231,"state":"active+clean","last_fresh":"2023-09-23T20:58:11.084470+0000","last_change":"2023-09-23T20:54:31.903139+0000","last_active":"2023-09-23T20:58:11.084470+0000","last_peered":"2023-09-23T20:58:11.084470+0000","last_clean":"2023-09-23T20:58:11.084470+0000","last_became_active":"2023-09-23T20:54:31.884520+0000","last_became_peered":"2023-09-23T20:54:31.884520+0000","last_unstale":"2023-09-23T20:58:11.084470+0000","last_undegraded":"2023-09-23T20:58:11.084470+0000","last_fullsized":"2023-09-23T20:58:11.084470+0000","mapping_epoch":136,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:39:34.269705+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1108,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":126,"reported_epoch":231,"state":"active+clean","last_fresh":"2023-09-23T20:58:11.067582+0000","last_change":"2023-09-23T20:55:35.951572+0000","last_active":"2023-09-23T20:58:11.067582+0000","last_peered":"2023-09-23T20:58:11.067582+0000","last_clean":"2023-09-23T20:58:11.067582+0000","last_became_active":"2023-09-23T20:55:35.951294+0000","last_became_peered":"2023-09-23T20:55:35.951294+0000","last_unstale":"2023-09-23T20:58:11.067582+0000","last_undegraded":"2023-09-23T20:58:11.067582+0000","last_fullsized":"2023-09-23T20:58:11.067582+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:36:32.447210+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":7,"num_pg":16,"stat_sum":{"num_bytes":6316556312,"num_objects":96384,"num_object_clones":0,"num_object_copies":289152,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":10943,"num_objects_misplaced":4771,"num_objects_unfound":0,"num_objects_dirty":96384,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":107136,"num_write_kb":6512577,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":628,"num_bytes_recovered":41156608,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":19034812416,"data_stored":19034800200,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":105536,"ondisk_log_size":105536,"up":48,"acting":45,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":22,"num_bytes_recovered":5990,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":21,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3381,"ondisk_log_size":3381,"up":63,"acting":63,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201494,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2121124,"kb_used_data":1951260,"kb_used_omap":0,"kb_used_meta":169856,"kb_avail":91624028,"statfs":{"total":95995035648,"available":93823004672,"internally_reserved":0,"allocated":1998090240,"data_stored":1997270676,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":173932544},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:57:44 2023","interfaces":[{"interface":"back","average":{"1min":3.3239999999999998,"5min":1.613,"15min":0.90200000000000002},"min":{"1min":0.23100000000000001,"5min":0.21099999999999999,"15min":0.21099999999999999},"max":{"1min":43.994999999999997,"5min":43.994999999999997,"15min":43.994999999999997},"last":4.5810000000000004},{"interface":"front","average":{"1min":3.448,"5min":1.847,"15min":1},"min":{"1min":0.35299999999999998,"5min":0.218,"15min":0.218},"max":{"1min":44.206000000000003,"5min":44.206000000000003,"15min":44.206000000000003},"last":1.1599999999999999}]},{"osd":1,"last update":"Sat Sep 23 20:57:14 2023","interfaces":[{"interface":"back","average":{"1min":0.92400000000000004,"5min":2.0659999999999998,"15min":1.054},"min":{"1min":0.26500000000000001,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":3.7290000000000001,"5min":22.591999999999999,"15min":22.591999999999999},"last":10.622},{"interface":"front","average":{"1min":0.81000000000000005,"5min":1.6040000000000001,"15min":0.89900000000000002},"min":{"1min":0.23699999999999999,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":2.9990000000000001,"5min":21.437000000000001,"15min":21.437000000000001},"last":7.5629999999999997}]},{"osd":2,"last update":"Sat Sep 23 20:57:50 2023","interfaces":[{"interface":"back","average":{"1min":1.3919999999999999,"5min":1.643,"15min":0.93799999999999994},"min":{"1min":0.26800000000000002,"5min":0.26200000000000001,"15min":0.24399999999999999},"max":{"1min":5.3929999999999998,"5min":9.4290000000000003,"15min":9.4290000000000003},"last":0.27600000000000002},{"interface":"front","average":{"1min":3.3420000000000001,"5min":1.978,"15min":1.071},"min":{"1min":0.29899999999999999,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":45.25,"5min":45.25,"15min":45.25},"last":4.6159999999999997}]},{"osd":3,"last update":"Sat Sep 23 20:57:25 2023","interfaces":[{"interface":"back","average":{"1min":1.286,"5min":3.0840000000000001,"15min":1.401},"min":{"1min":0.39700000000000002,"5min":0.314,"15min":0.314},"max":{"1min":3.9220000000000002,"5min":36.408000000000001,"15min":36.408000000000001},"last":1.546},{"interface":"front","average":{"1min":1.3540000000000001,"5min":2.669,"15min":1.272},"min":{"1min":0.42199999999999999,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":5.7969999999999997,"5min":35.593000000000004,"15min":35.593000000000004},"last":1.3959999999999999}]},{"osd":4,"last update":"Sat Sep 23 20:57:13 2023","interfaces":[{"interface":"back","average":{"1min":0.61899999999999999,"5min":0.78900000000000003,"15min":0.59499999999999997},"min":{"1min":0.28100000000000003,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":2.931,"5min":5.8920000000000003,"15min":5.8920000000000003},"last":4.5410000000000004},{"interface":"front","average":{"1min":0.52000000000000002,"5min":0.75900000000000001,"15min":0.60599999999999998},"min":{"1min":0.28499999999999998,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":1.931,"5min":5.3200000000000003,"15min":5.3200000000000003},"last":1.337}]},{"osd":5,"last update":"Sat Sep 23 20:57:13 2023","interfaces":[{"interface":"back","average":{"1min":0.59999999999999998,"5min":0.79500000000000004,"15min":0.626},"min":{"1min":0.31,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":2.774,"5min":7.0389999999999997,"15min":7.0389999999999997},"last":0.40999999999999998},{"interface":"front","average":{"1min":0.61599999999999999,"5min":0.68000000000000005,"15min":0.60199999999999998},"min":{"1min":0.32800000000000001,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":1.6359999999999999,"5min":3.3530000000000002,"15min":3.3530000000000002},"last":4.5999999999999996}]},{"osd":6,"last update":"Sat Sep 23 20:57:38 2023","interfaces":[{"interface":"back","average":{"1min":0.72099999999999997,"5min":0.67700000000000005,"15min":0.67000000000000004},"min":{"1min":0.34799999999999998,"5min":0.31,"15min":0.31},"max":{"1min":1.9039999999999999,"5min":3.1760000000000002,"15min":3.1760000000000002},"last":1.3700000000000001},{"interface":"front","average":{"1min":0.753,"5min":0.79400000000000004,"15min":0.80100000000000005},"min":{"1min":0.32300000000000001,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":1.96,"5min":3.129,"15min":3.129},"last":1.427}]}]},{"osd":6,"up_from":164,"seq":704374636578,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2021592,"kb_used_data":1925904,"kb_used_omap":10,"kb_used_meta":95669,"kb_avail":91723560,"statfs":{"total":95995035648,"available":93924925440,"internally_reserved":0,"allocated":1972125696,"data_stored":1971318033,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":10328,"internal_metadata":97965992},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,2],"upper_bound":16},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:57:40 2023","interfaces":[{"interface":"back","average":{"1min":1.8799999999999999,"5min":1.0680000000000001,"15min":0.93200000000000005},"min":{"1min":0.20999999999999999,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":8.7319999999999993,"5min":8.7319999999999993,"15min":8.7319999999999993},"last":5.3849999999999998},{"interface":"front","average":{"1min":1.393,"5min":1.0649999999999999,"15min":1.01},"min":{"1min":0.29299999999999998,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":6.7110000000000003,"5min":6.7110000000000003,"15min":6.7110000000000003},"last":6.1130000000000004}]},{"osd":1,"last update":"Sat Sep 23 20:57:40 2023","interfaces":[{"interface":"back","average":{"1min":1.575,"5min":0.94599999999999995,"15min":0.84099999999999997},"min":{"1min":0.34599999999999997,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":6.6029999999999998,"5min":6.6029999999999998,"15min":6.6029999999999998},"last":5.2350000000000003},{"interface":"front","average":{"1min":1.04,"5min":0.90200000000000002,"15min":0.879},"min":{"1min":0.255,"5min":0.247,"15min":0.247},"max":{"1min":2.7389999999999999,"5min":3.5870000000000002,"15min":3.5870000000000002},"last":0.88200000000000001}]},{"osd":2,"last update":"Sat Sep 23 20:57:40 2023","interfaces":[{"interface":"back","average":{"1min":2.6850000000000001,"5min":1.8540000000000001,"15min":1.716},"min":{"1min":0.371,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":13.750999999999999,"5min":13.750999999999999,"15min":13.750999999999999},"last":0.45600000000000002},{"interface":"front","average":{"1min":1.9079999999999999,"5min":1.2669999999999999,"15min":1.1599999999999999},"min":{"1min":0.39200000000000002,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":7.4279999999999999,"5min":7.4279999999999999,"15min":7.4279999999999999},"last":8.0890000000000004}]},{"osd":3,"last update":"Sat Sep 23 20:57:40 2023","interfaces":[{"interface":"back","average":{"1min":1.2070000000000001,"5min":1.135,"15min":1.123},"min":{"1min":0.33300000000000002,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":3.093,"5min":5.8739999999999997,"15min":5.8739999999999997},"last":0.33300000000000002},{"interface":"front","average":{"1min":1.7090000000000001,"5min":1.6240000000000001,"15min":1.6100000000000001},"min":{"1min":0.311,"5min":0.311,"15min":0.311},"max":{"1min":5.46,"5min":7.234,"15min":7.234},"last":2.8769999999999998}]},{"osd":4,"last update":"Sat Sep 23 20:57:40 2023","interfaces":[{"interface":"back","average":{"1min":0.65000000000000002,"5min":0.97099999999999997,"15min":1.0249999999999999},"min":{"1min":0.28299999999999997,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":1.9039999999999999,"5min":9.0079999999999991,"15min":9.0079999999999991},"last":0.315},{"interface":"front","average":{"1min":0.45700000000000002,"5min":0.51600000000000001,"15min":0.52600000000000002},"min":{"1min":0.25800000000000001,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":1.3120000000000001,"5min":1.3120000000000001,"15min":1.3120000000000001},"last":0.46999999999999997}]},{"osd":5,"last update":"Sat Sep 23 20:57:40 2023","interfaces":[{"interface":"back","average":{"1min":0.68600000000000005,"5min":0.56999999999999995,"15min":0.55000000000000004},"min":{"1min":0.27900000000000003,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":1.9399999999999999,"5min":1.9399999999999999,"15min":1.9399999999999999},"last":0.35299999999999998},{"interface":"front","average":{"1min":0.68899999999999995,"5min":0.78500000000000003,"15min":0.80100000000000005},"min":{"1min":0.30599999999999999,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":1.9159999999999999,"5min":2.274,"15min":2.274},"last":1.7589999999999999}]},{"osd":7,"last update":"Sat Sep 23 20:57:40 2023","interfaces":[{"interface":"back","average":{"1min":0.95799999999999996,"5min":0.93400000000000005,"15min":0.93000000000000005},"min":{"1min":0.33500000000000002,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":7.8570000000000002,"5min":7.8570000000000002,"15min":7.8570000000000002},"last":0.36499999999999999},{"interface":"front","average":{"1min":1.103,"5min":0.64400000000000002,"15min":0.56799999999999995},"min":{"1min":0.33400000000000002,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":8.282,"5min":8.282,"15min":8.282},"last":2.6989999999999998}]}]},{"osd":1,"up_from":88,"seq":377957122149,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2844552,"kb_used_data":2650496,"kb_used_omap":9,"kb_used_meta":193974,"kb_avail":90900600,"statfs":{"total":95995035648,"available":93082214400,"internally_reserved":0,"allocated":2714107904,"data_stored":2712968207,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9239,"internal_metadata":198630377},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,2,0,0,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:57:18 2023","interfaces":[{"interface":"back","average":{"1min":2.1829999999999998,"5min":2.5289999999999999,"15min":1.111},"min":{"1min":0.22800000000000001,"5min":0.216,"15min":0.216},"max":{"1min":16.646999999999998,"5min":16.646999999999998,"15min":16.646999999999998},"last":7.2409999999999997},{"interface":"front","average":{"1min":2.109,"5min":2.7650000000000001,"15min":1.2070000000000001},"min":{"1min":0.246,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":16.664999999999999,"5min":18.457999999999998,"15min":18.457999999999998},"last":7.4729999999999999}]},{"osd":2,"last update":"Sat Sep 23 20:57:50 2023","interfaces":[{"interface":"back","average":{"1min":2.5760000000000001,"5min":2.1309999999999998,"15min":1.0249999999999999},"min":{"1min":0.221,"5min":0.20300000000000001,"15min":0.20300000000000001},"max":{"1min":8.6500000000000004,"5min":14.335000000000001,"15min":14.335000000000001},"last":2.9969999999999999},{"interface":"front","average":{"1min":3.2770000000000001,"5min":3.0209999999999999,"15min":1.4099999999999999},"min":{"1min":0.35699999999999998,"5min":0.248,"15min":0.248},"max":{"1min":10.095000000000001,"5min":38.951000000000001,"15min":38.951000000000001},"last":10.884}]},{"osd":3,"last update":"Sat Sep 23 20:57:20 2023","interfaces":[{"interface":"back","average":{"1min":0.85499999999999998,"5min":2.835,"15min":1.1839999999999999},"min":{"1min":0.248,"5min":0.248,"15min":0.248},"max":{"1min":4.1589999999999998,"5min":25.314,"15min":25.314},"last":2.5960000000000001},{"interface":"front","average":{"1min":1.3700000000000001,"5min":2.3900000000000001,"15min":1.0700000000000001},"min":{"1min":0.30499999999999999,"5min":0.20599999999999999,"15min":0.20599999999999999},"max":{"1min":8.4719999999999995,"5min":19.881,"15min":19.881},"last":3.0449999999999999}]},{"osd":4,"last update":"Sat Sep 23 20:57:18 2023","interfaces":[{"interface":"back","average":{"1min":0.69599999999999995,"5min":1.903,"15min":1.0609999999999999},"min":{"1min":0.308,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":1.8100000000000001,"5min":12.006,"15min":12.006},"last":0.89100000000000001},{"interface":"front","average":{"1min":1.1759999999999999,"5min":1.4399999999999999,"15min":0.92100000000000004},"min":{"1min":0.311,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":5.4539999999999997,"5min":11.234999999999999,"15min":11.234999999999999},"last":2.9340000000000002}]},{"osd":5,"last update":"Sat Sep 23 20:57:18 2023","interfaces":[{"interface":"back","average":{"1min":1.0640000000000001,"5min":1.704,"15min":1.0229999999999999},"min":{"1min":0.372,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":3.7160000000000002,"5min":12.222,"15min":12.222},"last":2.5539999999999998},{"interface":"front","average":{"1min":0.72799999999999998,"5min":1.9770000000000001,"15min":1.085},"min":{"1min":0.35699999999999998,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":1.869,"5min":12.08,"15min":12.08},"last":0.94999999999999996}]},{"osd":6,"last update":"Sat Sep 23 20:57:35 2023","interfaces":[{"interface":"back","average":{"1min":1.2929999999999999,"5min":1.3100000000000001,"15min":1.3129999999999999},"min":{"1min":0.39000000000000001,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":5.3810000000000002,"5min":5.3810000000000002,"15min":5.3810000000000002},"last":0.92500000000000004},{"interface":"front","average":{"1min":1.9530000000000001,"5min":1.393,"15min":1.2989999999999999},"min":{"1min":0.439,"5min":0.309,"15min":0.309},"max":{"1min":6.0860000000000003,"5min":6.5670000000000002,"15min":6.5670000000000002},"last":2.9710000000000001}]},{"osd":7,"last update":"Sat Sep 23 20:57:18 2023","interfaces":[{"interface":"back","average":{"1min":1.2230000000000001,"5min":1.601,"15min":0.97599999999999998},"min":{"1min":0.39600000000000002,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":5.524,"5min":11.186999999999999,"15min":11.186999999999999},"last":3.02},{"interface":"front","average":{"1min":1.3109999999999999,"5min":1.756,"15min":1.0369999999999999},"min":{"1min":0.443,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":3.7650000000000001,"5min":12.252000000000001,"15min":12.252000000000001},"last":2.6339999999999999}]}]},{"osd":0,"up_from":83,"seq":356482285675,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3281520,"kb_used_data":3127656,"kb_used_omap":28,"kb_used_meta":153827,"kb_avail":90463632,"statfs":{"total":95995035648,"available":92634759168,"internally_reserved":0,"allocated":3202719744,"data_stored":3201602256,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":28848,"internal_metadata":157519696},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 20:57:18 2023","interfaces":[{"interface":"back","average":{"1min":2.0249999999999999,"5min":2.423,"15min":1.0309999999999999},"min":{"1min":0.217,"5min":0.191,"15min":0.191},"max":{"1min":8.2989999999999995,"5min":18.863,"15min":18.863},"last":5.9139999999999997},{"interface":"front","average":{"1min":1.3759999999999999,"5min":2.327,"15min":1.0329999999999999},"min":{"1min":0.25800000000000001,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":7.6639999999999997,"5min":17.704999999999998,"15min":17.704999999999998},"last":4.6619999999999999}]},{"osd":2,"last update":"Sat Sep 23 20:57:47 2023","interfaces":[{"interface":"back","average":{"1min":2.6150000000000002,"5min":3.0329999999999999,"15min":1.3400000000000001},"min":{"1min":0.35599999999999998,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":8.4949999999999992,"5min":17.753,"15min":17.753},"last":8.9469999999999992},{"interface":"front","average":{"1min":2.843,"5min":2.8199999999999998,"15min":1.234},"min":{"1min":0.308,"5min":0.21099999999999999,"15min":0.21099999999999999},"max":{"1min":9.9469999999999992,"5min":13.218999999999999,"15min":13.218999999999999},"last":5.3470000000000004}]},{"osd":3,"last update":"Sat Sep 23 20:57:18 2023","interfaces":[{"interface":"back","average":{"1min":1.6990000000000001,"5min":2.4289999999999998,"15min":1.077},"min":{"1min":0.27400000000000002,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":7.1749999999999998,"5min":18.893999999999998,"15min":18.893999999999998},"last":5.944},{"interface":"front","average":{"1min":1.2050000000000001,"5min":2.6520000000000001,"15min":1.202},"min":{"1min":0.28199999999999997,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":4.5759999999999996,"5min":17.472000000000001,"15min":17.472000000000001},"last":3.1299999999999999}]},{"osd":4,"last update":"Sat Sep 23 20:57:47 2023","interfaces":[{"interface":"back","average":{"1min":1.968,"5min":1.881,"15min":1.0740000000000001},"min":{"1min":0.35099999999999998,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":9.9079999999999995,"5min":12.807,"15min":12.807},"last":5.3179999999999996},{"interface":"front","average":{"1min":1.1659999999999999,"5min":1.643,"15min":0.997},"min":{"1min":0.311,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":4.7850000000000001,"5min":18.724,"15min":18.724},"last":5.4950000000000001}]},{"osd":5,"last update":"Sat Sep 23 20:57:47 2023","interfaces":[{"interface":"back","average":{"1min":1.7669999999999999,"5min":1.8129999999999999,"15min":1.069},"min":{"1min":0.33600000000000002,"5min":0.318,"15min":0.318},"max":{"1min":6.673,"5min":8.4990000000000006,"15min":8.4990000000000006},"last":3.0750000000000002},{"interface":"front","average":{"1min":2.2440000000000002,"5min":2.0600000000000001,"15min":1.1619999999999999},"min":{"1min":0.39000000000000001,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":9.8599999999999994,"5min":13.154999999999999,"15min":13.154999999999999},"last":5.274}]},{"osd":6,"last update":"Sat Sep 23 20:57:41 2023","interfaces":[{"interface":"back","average":{"1min":1.8160000000000001,"5min":1.6879999999999999,"15min":1.667},"min":{"1min":0.37,"5min":0.37,"15min":0.37},"max":{"1min":6.8470000000000004,"5min":8.7309999999999999,"15min":8.7309999999999999},"last":5.5289999999999999},{"interface":"front","average":{"1min":1.875,"5min":1.7709999999999999,"15min":1.754},"min":{"1min":0.377,"5min":0.377,"15min":0.377},"max":{"1min":5.492,"5min":8.6760000000000002,"15min":8.6760000000000002},"last":5.5510000000000002}]},{"osd":7,"last update":"Sat Sep 23 20:57:47 2023","interfaces":[{"interface":"back","average":{"1min":1.786,"5min":1.8220000000000001,"15min":1.1020000000000001},"min":{"1min":0.41599999999999998,"5min":0.379,"15min":0.379},"max":{"1min":6.6390000000000002,"5min":8.4339999999999993,"15min":8.4339999999999993},"last":3.161},{"interface":"front","average":{"1min":1.393,"5min":1.698,"15min":1.0269999999999999},"min":{"1min":0.40100000000000002,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":4.6799999999999997,"5min":18.673999999999999,"15min":18.673999999999999},"last":5.5730000000000004}]}]},{"osd":2,"up_from":93,"seq":399431958622,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2951196,"kb_used_data":2677780,"kb_used_omap":81,"kb_used_meta":273262,"kb_avail":90793956,"statfs":{"total":95995035648,"available":92973010944,"internally_reserved":0,"allocated":2742046720,"data_stored":2740945687,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":83750,"internal_metadata":279820506},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,1,0,0,3],"upper_bound":32},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:57:43 2023","interfaces":[{"interface":"back","average":{"1min":2.2090000000000001,"5min":1.9319999999999999,"15min":0.89400000000000002},"min":{"1min":0.27100000000000002,"5min":0.17999999999999999,"15min":0.17999999999999999},"max":{"1min":9.7460000000000004,"5min":11.183,"15min":11.183},"last":0.497},{"interface":"front","average":{"1min":3.0449999999999999,"5min":2.5110000000000001,"15min":1.1160000000000001},"min":{"1min":0.216,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":10.891999999999999,"5min":12.641,"15min":12.641},"last":0.54700000000000004}]},{"osd":1,"last update":"Sat Sep 23 20:57:43 2023","interfaces":[{"interface":"back","average":{"1min":2.23,"5min":2.8119999999999998,"15min":1.2310000000000001},"min":{"1min":0.21199999999999999,"5min":0.20200000000000001,"15min":0.20200000000000001},"max":{"1min":8.2889999999999997,"5min":16.081,"15min":16.081},"last":0.89100000000000001},{"interface":"front","average":{"1min":2.9660000000000002,"5min":3.419,"15min":1.4390000000000001},"min":{"1min":0.26300000000000001,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":8.8960000000000008,"5min":16.041,"15min":16.041},"last":0.92000000000000004}]},{"osd":3,"last update":"Sat Sep 23 20:57:22 2023","interfaces":[{"interface":"back","average":{"1min":1.891,"5min":2.9870000000000001,"15min":1.2709999999999999},"min":{"1min":0.36299999999999999,"5min":0.26800000000000002,"15min":0.255},"max":{"1min":10.488,"5min":16.611000000000001,"15min":16.611000000000001},"last":4.1420000000000003},{"interface":"front","average":{"1min":1.835,"5min":3.6760000000000002,"15min":1.478},"min":{"1min":0.28000000000000003,"5min":0.22800000000000001,"15min":0.221},"max":{"1min":6.6059999999999999,"5min":17.138999999999999,"15min":17.138999999999999},"last":4.2080000000000002}]},{"osd":4,"last update":"Sat Sep 23 20:57:43 2023","interfaces":[{"interface":"back","average":{"1min":1.8660000000000001,"5min":1.851,"15min":1.0349999999999999},"min":{"1min":0.318,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":8.657,"5min":9.8119999999999994,"15min":9.8119999999999994},"last":0.58199999999999996},{"interface":"front","average":{"1min":1.9770000000000001,"5min":2.0190000000000001,"15min":1.1299999999999999},"min":{"1min":0.34300000000000003,"5min":0.313,"15min":0.313},"max":{"1min":6.7850000000000001,"5min":10.609,"15min":10.609},"last":0.97799999999999998}]},{"osd":5,"last update":"Sat Sep 23 20:57:43 2023","interfaces":[{"interface":"back","average":{"1min":1.823,"5min":2.121,"15min":1.145},"min":{"1min":0.29299999999999998,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":8.5700000000000003,"5min":9.7629999999999999,"15min":9.7629999999999999},"last":0.63300000000000001},{"interface":"front","average":{"1min":2.0259999999999998,"5min":2.129,"15min":1.147},"min":{"1min":0.32600000000000001,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":6.7510000000000003,"5min":10.574,"15min":10.574},"last":1.01}]},{"osd":6,"last update":"Sat Sep 23 20:57:41 2023","interfaces":[{"interface":"back","average":{"1min":1.9450000000000001,"5min":1.7230000000000001,"15min":1.6859999999999999},"min":{"1min":0.312,"5min":0.312,"15min":0.312},"max":{"1min":6.8109999999999999,"5min":14.244999999999999,"15min":14.244999999999999},"last":1.04},{"interface":"front","average":{"1min":1.9390000000000001,"5min":1.2110000000000001,"15min":1.0900000000000001},"min":{"1min":0.374,"5min":0.374,"15min":0.374},"max":{"1min":6.835,"5min":6.835,"15min":6.835},"last":1.0649999999999999}]},{"osd":7,"last update":"Sat Sep 23 20:57:43 2023","interfaces":[{"interface":"back","average":{"1min":1.397,"5min":1.524,"15min":0.93799999999999994},"min":{"1min":0.32500000000000001,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":4.9820000000000002,"5min":8.0820000000000007,"15min":8.0820000000000007},"last":0.46000000000000002},{"interface":"front","average":{"1min":1.804,"5min":2.1360000000000001,"15min":1.159},"min":{"1min":0.35699999999999998,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":8.6219999999999999,"5min":9.7859999999999996,"15min":9.7859999999999996},"last":0.60599999999999998}]}]},{"osd":3,"up_from":98,"seq":420906795096,"num_pgs":49,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2775804,"kb_used_data":2484340,"kb_used_omap":7,"kb_used_meta":291448,"kb_avail":90969348,"statfs":{"total":95995035648,"available":93152612352,"internally_reserved":0,"allocated":2543964160,"data_stored":2542892000,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7576,"internal_metadata":298443368},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,1,0,0,2,1,2],"upper_bound":128},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:57:22 2023","interfaces":[{"interface":"back","average":{"1min":1.2130000000000001,"5min":3.181,"15min":1.3069999999999999},"min":{"1min":0.23100000000000001,"5min":0.20599999999999999,"15min":0.20599999999999999},"max":{"1min":8.5289999999999999,"5min":23.123999999999999,"15min":23.123999999999999},"last":8.4260000000000002},{"interface":"front","average":{"1min":1.552,"5min":2.6619999999999999,"15min":1.1259999999999999},"min":{"1min":0.217,"5min":0.20000000000000001,"15min":0.20000000000000001},"max":{"1min":6.3819999999999997,"5min":15.079000000000001,"15min":15.079000000000001},"last":4.5590000000000002}]},{"osd":1,"last update":"Sat Sep 23 20:57:22 2023","interfaces":[{"interface":"back","average":{"1min":1.833,"5min":2.9020000000000001,"15min":1.212},"min":{"1min":0.27500000000000002,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":7.6879999999999997,"5min":21.385000000000002,"15min":21.385000000000002},"last":3.6549999999999998},{"interface":"front","average":{"1min":1.5609999999999999,"5min":3.2160000000000002,"15min":1.331},"min":{"1min":0.22600000000000001,"5min":0.218,"15min":0.218},"max":{"1min":9.9190000000000005,"5min":25.353000000000002,"15min":25.353000000000002},"last":4.2439999999999998}]},{"osd":2,"last update":"Sat Sep 23 20:57:22 2023","interfaces":[{"interface":"back","average":{"1min":2.0939999999999999,"5min":2.5979999999999999,"15min":1.149},"min":{"1min":0.34300000000000003,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":6.7679999999999998,"5min":24.154,"15min":24.154},"last":8.1940000000000008},{"interface":"front","average":{"1min":1.591,"5min":3.371,"15min":1.4159999999999999},"min":{"1min":0.32900000000000001,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":6.2359999999999998,"5min":25.305,"15min":25.305},"last":4.742}]},{"osd":4,"last update":"Sat Sep 23 20:57:22 2023","interfaces":[{"interface":"back","average":{"1min":1.786,"5min":1.756,"15min":1.006},"min":{"1min":0.38600000000000001,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":6.6790000000000003,"5min":11.861000000000001,"15min":11.861000000000001},"last":3.7280000000000002},{"interface":"front","average":{"1min":0.995,"5min":2.339,"15min":1.1970000000000001},"min":{"1min":0.29499999999999998,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":6.2800000000000002,"5min":22.853000000000002,"15min":22.853000000000002},"last":4.165}]},{"osd":5,"last update":"Sat Sep 23 20:57:22 2023","interfaces":[{"interface":"back","average":{"1min":1.819,"5min":1.6839999999999999,"15min":0.99199999999999999},"min":{"1min":0.41299999999999998,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":6.7169999999999996,"5min":11.891999999999999,"15min":11.891999999999999},"last":3.6949999999999998},{"interface":"front","average":{"1min":1.337,"5min":1.673,"15min":0.98199999999999998},"min":{"1min":0.35099999999999998,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":6.3419999999999996,"5min":9.2870000000000008,"15min":9.2870000000000008},"last":4.6070000000000002}]},{"osd":6,"last update":"Sat Sep 23 20:57:39 2023","interfaces":[{"interface":"back","average":{"1min":1.173,"5min":0.99399999999999999,"15min":0.96399999999999997},"min":{"1min":0.40999999999999998,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":6.3079999999999998,"5min":6.3079999999999998,"15min":6.3079999999999998},"last":4.2110000000000003},{"interface":"front","average":{"1min":1.6879999999999999,"5min":1.4670000000000001,"15min":1.4299999999999999},"min":{"1min":0.40100000000000002,"5min":0.40100000000000002,"15min":0.40100000000000002},"max":{"1min":5.8230000000000004,"5min":7.234,"15min":7.234},"last":4.6449999999999996}]},{"osd":7,"last update":"Sat Sep 23 20:57:22 2023","interfaces":[{"interface":"back","average":{"1min":1.9379999999999999,"5min":2.113,"15min":1.167},"min":{"1min":0.47099999999999997,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":6.7450000000000001,"5min":16.364999999999998,"15min":16.364999999999998},"last":3.7570000000000001},{"interface":"front","average":{"1min":1.2,"5min":1.7509999999999999,"15min":1.046},"min":{"1min":0.442,"5min":0.379,"15min":0.379},"max":{"1min":2.9249999999999998,"5min":9.3179999999999996,"15min":9.3179999999999996},"last":4.6749999999999998}]}]},{"osd":4,"up_from":31,"seq":133143986403,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2063844,"kb_used_data":1940572,"kb_used_omap":0,"kb_used_meta":123264,"kb_avail":91681308,"statfs":{"total":95995035648,"available":93881659392,"internally_reserved":0,"allocated":1987145728,"data_stored":1986326417,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":126222336},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,2,0,2],"upper_bound":32},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:57:52 2023","interfaces":[{"interface":"back","average":{"1min":2.4359999999999999,"5min":1.593,"15min":0.95599999999999996},"min":{"1min":0.214,"5min":0.214,"15min":0.214},"max":{"1min":11.784000000000001,"5min":11.784000000000001,"15min":11.784000000000001},"last":0.39500000000000002},{"interface":"front","average":{"1min":2.2970000000000002,"5min":1.782,"15min":1.1699999999999999},"min":{"1min":0.28100000000000003,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":11.766,"5min":11.766,"15min":11.766},"last":0.40699999999999997}]},{"osd":1,"last update":"Sat Sep 23 20:57:17 2023","interfaces":[{"interface":"back","average":{"1min":1.2230000000000001,"5min":1.583,"15min":0.91800000000000004},"min":{"1min":0.252,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":6.6550000000000002,"5min":7.9859999999999998,"15min":7.9859999999999998},"last":0.318},{"interface":"front","average":{"1min":0.96199999999999997,"5min":1.274,"15min":0.82699999999999996},"min":{"1min":0.26600000000000001,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":3.2629999999999999,"5min":6.3540000000000001,"15min":6.3540000000000001},"last":0.47899999999999998}]},{"osd":2,"last update":"Sat Sep 23 20:57:52 2023","interfaces":[{"interface":"back","average":{"1min":1.5840000000000001,"5min":1.746,"15min":1.036},"min":{"1min":0.35699999999999998,"5min":0.311,"15min":0.29099999999999998},"max":{"1min":5.21,"5min":15.288,"15min":15.288},"last":1.22},{"interface":"front","average":{"1min":1.8939999999999999,"5min":2.0720000000000001,"15min":1.2050000000000001},"min":{"1min":0.39500000000000002,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":3.8300000000000001,"5min":15.262,"15min":15.262},"last":1.3500000000000001}]},{"osd":3,"last update":"Sat Sep 23 20:57:17 2023","interfaces":[{"interface":"back","average":{"1min":1.49,"5min":2.9660000000000002,"15min":1.423},"min":{"1min":0.317,"5min":0.308,"15min":0.308},"max":{"1min":6.0270000000000001,"5min":31.562999999999999,"15min":31.562999999999999},"last":1.665},{"interface":"front","average":{"1min":1.5640000000000001,"5min":2.9540000000000002,"15min":1.4930000000000001},"min":{"1min":0.44800000000000001,"5min":0.441,"15min":0.41899999999999998},"max":{"1min":6.0410000000000004,"5min":31.574000000000002,"15min":31.574000000000002},"last":1.6599999999999999}]},{"osd":5,"last update":"Sat Sep 23 20:57:34 2023","interfaces":[{"interface":"back","average":{"1min":0.74199999999999999,"5min":0.53700000000000003,"15min":0.50600000000000001},"min":{"1min":0.248,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":3.008,"5min":3.3279999999999998,"15min":3.3279999999999998},"last":0.42399999999999999},{"interface":"front","average":{"1min":0.66300000000000003,"5min":0.68300000000000005,"15min":0.55800000000000005},"min":{"1min":0.27300000000000002,"5min":0.251,"15min":0.251},"max":{"1min":3.1280000000000001,"5min":7.7619999999999996,"15min":7.7619999999999996},"last":0.33200000000000002}]},{"osd":6,"last update":"Sat Sep 23 20:57:40 2023","interfaces":[{"interface":"back","average":{"1min":1.032,"5min":0.59299999999999997,"15min":0.52000000000000002},"min":{"1min":0.26900000000000002,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":4.6909999999999998,"5min":4.6909999999999998,"15min":4.6909999999999998},"last":0.34999999999999998},{"interface":"front","average":{"1min":0.82499999999999996,"5min":0.68000000000000005,"15min":0.65600000000000003},"min":{"1min":0.38900000000000001,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":2.7120000000000002,"5min":2.7120000000000002,"15min":2.7120000000000002},"last":0.46200000000000002}]},{"osd":7,"last update":"Sat Sep 23 20:57:17 2023","interfaces":[{"interface":"back","average":{"1min":0.60399999999999998,"5min":0.79200000000000004,"15min":0.65400000000000003},"min":{"1min":0.32400000000000001,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":2.3639999999999999,"5min":7.383,"15min":7.383},"last":0.44600000000000001},{"interface":"front","average":{"1min":0.59499999999999997,"5min":0.77000000000000002,"15min":0.63500000000000001},"min":{"1min":0.34999999999999998,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":2.032,"5min":7.8310000000000004,"15min":7.8310000000000004},"last":0.372}]}]},{"osd":5,"up_from":39,"seq":167503724768,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2028120,"kb_used_data":1901712,"kb_used_omap":0,"kb_used_meta":126400,"kb_avail":91717032,"statfs":{"total":95995035648,"available":93918240768,"internally_reserved":0,"allocated":1947353088,"data_stored":1946542353,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":129433600},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:57:36 2023","interfaces":[{"interface":"back","average":{"1min":1.145,"5min":1.1200000000000001,"15min":0.70799999999999996},"min":{"1min":0.20399999999999999,"5min":0.20399999999999999,"15min":0.20399999999999999},"max":{"1min":7.5019999999999998,"5min":7.766,"15min":7.766},"last":1.2789999999999999},{"interface":"front","average":{"1min":1.647,"5min":1.556,"15min":0.89900000000000002},"min":{"1min":0.25600000000000001,"5min":0.223,"15min":0.223},"max":{"1min":11.378,"5min":15.037000000000001,"15min":15.037000000000001},"last":3.879}]},{"osd":1,"last update":"Sat Sep 23 20:58:16 2023","interfaces":[{"interface":"back","average":{"1min":1.5549999999999999,"5min":1.603,"15min":0.95199999999999996},"min":{"1min":0.27100000000000002,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":6.7409999999999997,"5min":8.3520000000000003,"15min":8.3520000000000003},"last":6.7409999999999997},{"interface":"front","average":{"1min":1.9590000000000001,"5min":1.7450000000000001,"15min":0.97699999999999998},"min":{"1min":0.28100000000000003,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":7.4669999999999996,"5min":10.285,"15min":10.285},"last":2.762}]},{"osd":2,"last update":"Sat Sep 23 20:57:50 2023","interfaces":[{"interface":"back","average":{"1min":1.2410000000000001,"5min":1.575,"15min":0.88500000000000001},"min":{"1min":0.26200000000000001,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":10.579000000000001,"5min":12.651,"15min":12.651},"last":1.2410000000000001},{"interface":"front","average":{"1min":1.0669999999999999,"5min":1.5449999999999999,"15min":0.91900000000000004},"min":{"1min":0.26100000000000001,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":5.2949999999999999,"5min":8.3209999999999997,"15min":8.3209999999999997},"last":1.0720000000000001}]},{"osd":3,"last update":"Sat Sep 23 20:58:16 2023","interfaces":[{"interface":"back","average":{"1min":2.331,"5min":2.0670000000000002,"15min":1.153},"min":{"1min":0.39200000000000002,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":8.6579999999999995,"5min":9.5489999999999995,"15min":9.5489999999999995},"last":0.67300000000000004},{"interface":"front","average":{"1min":1.6000000000000001,"5min":1.6040000000000001,"15min":0.97499999999999998},"min":{"1min":0.28899999999999998,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":10.457000000000001,"5min":10.457000000000001,"15min":10.457000000000001},"last":3.9969999999999999}]},{"osd":4,"last update":"Sat Sep 23 20:57:29 2023","interfaces":[{"interface":"back","average":{"1min":0.49099999999999999,"5min":0.60199999999999998,"15min":0.52800000000000002},"min":{"1min":0.29699999999999999,"5min":0.27600000000000002,"15min":0.253},"max":{"1min":1.4239999999999999,"5min":3.8479999999999999,"15min":3.8479999999999999},"last":0.40899999999999997},{"interface":"front","average":{"1min":0.56299999999999994,"5min":0.63900000000000001,"15min":0.55600000000000005},"min":{"1min":0.23599999999999999,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":1.389,"5min":3.0680000000000001,"15min":3.0680000000000001},"last":0.29899999999999999}]},{"osd":6,"last update":"Sat Sep 23 20:57:36 2023","interfaces":[{"interface":"back","average":{"1min":0.80800000000000005,"5min":0.60999999999999999,"15min":0.57699999999999996},"min":{"1min":0.28000000000000003,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":2.9820000000000002,"5min":2.9820000000000002,"15min":2.9820000000000002},"last":0.34000000000000002},{"interface":"front","average":{"1min":0.76400000000000001,"5min":0.748,"15min":0.745},"min":{"1min":0.33200000000000002,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":2.6579999999999999,"5min":3.6560000000000001,"15min":3.6560000000000001},"last":0.44400000000000001}]},{"osd":7,"last update":"Sat Sep 23 20:58:13 2023","interfaces":[{"interface":"back","average":{"1min":0.66500000000000004,"5min":0.76600000000000001,"15min":0.63200000000000001},"min":{"1min":0.316,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":2.4809999999999999,"5min":8.5649999999999995,"15min":8.5649999999999995},"last":0.48699999999999999},{"interface":"front","average":{"1min":0.60299999999999998,"5min":0.74099999999999999,"15min":0.64400000000000002},"min":{"1min":0.35399999999999998,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":2.7109999999999999,"5min":4.5430000000000001,"15min":4.5430000000000001},"last":0.51800000000000002}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":3193507840,"data_stored":3193503768,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":2699038720,"data_stored":2699034648,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":2734751744,"data_stored":2734751744,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":2537226240,"data_stored":2537226240,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1977024512,"data_stored":1977024512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1939668992,"data_stored":1939668992,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1966411776,"data_stored":1966407704,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1987182592,"data_stored":1987182592,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T20:58:20.854 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T20:58:20.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:58:20 smithi070 ceph-mon[168282]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:58:20.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:58:20 smithi070 ceph-mon[168282]: failed to encode map e233 with expected crc 2023-09-23T20:58:20.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:58:20 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T20:58:20.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:58:20 smithi070 ceph-mon[168282]: osdmap e233: 8 total, 8 up, 8 in 2023-09-23T20:58:20.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:58:20 smithi070 ceph-mon[168282]: failed to encode map e233 with expected crc 2023-09-23T20:58:20.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:58:20 smithi070 ceph-mon[168282]: from='client.35238 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:58:20.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:58:20 smithi070 ceph-mon[168282]: Health check update: Degraded data redundancy: 10943/293028 objects degraded (3.734%), 2 pgs degraded (PG_DEGRADED) 2023-09-23T20:58:20.890 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:58:20.906 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:58:20.906 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:58:21.007 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:58:21.130 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:58:21.476 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:58:21.483 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:58:21.484 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:58:21.584 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:58:21.612 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:58:21.649 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:58:21.653 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:58:21.653 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:58:21.754 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:58:21.777 INFO:teuthology.orchestra.run.smithi026.stdout:{ 2023-09-23T20:58:21.778 INFO:teuthology.orchestra.run.smithi026.stdout: "size": 20, 2023-09-23T20:58:21.778 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 600, 2023-09-23T20:58:21.778 INFO:teuthology.orchestra.run.smithi026.stdout: "ops": [ 2023-09-23T20:58:21.778 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.778 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34797.0:114960 6.17 6:edae14e8:::benchmark_data_smithi026_223868_object13834:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e131)", 2023-09-23T20:58:21.778 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T20:54:12.901858+0000", 2023-09-23T20:58:21.778 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 248.858977194, 2023-09-23T20:58:21.778 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.1321539249999999, 2023-09-23T20:58:21.778 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T20:58:21.779 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "reached pg", 2023-09-23T20:58:21.779 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T20:58:21.779 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34797", 2023-09-23T20:58:21.779 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T20:58:21.779 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 114960 2023-09-23T20:58:21.779 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.779 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T20:58:21.779 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.779 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T20:58:21.779 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:12.901858+0000", 2023-09-23T20:58:21.779 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:58:21.780 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.780 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.780 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T20:58:21.780 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:12.901858+0000", 2023-09-23T20:58:21.780 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:58:21.780 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.780 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.780 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T20:58:21.780 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:12.901857+0000", 2023-09-23T20:58:21.780 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T20:58:21.780 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.780 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.780 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T20:58:21.781 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:12.901863+0000", 2023-09-23T20:58:21.781 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.3720000000000001e-06 2023-09-23T20:58:21.781 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.781 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.781 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T20:58:21.781 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:12.901875+0000", 2023-09-23T20:58:21.781 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1924e-05 2023-09-23T20:58:21.781 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.781 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.781 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T20:58:21.781 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:12.901883+0000", 2023-09-23T20:58:21.781 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.038e-06 2023-09-23T20:58:21.781 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.782 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.782 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:58:21.782 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:12.904218+0000", 2023-09-23T20:58:21.782 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0023356050000000001 2023-09-23T20:58:21.782 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.782 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.782 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for degraded object", 2023-09-23T20:58:21.782 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:12.909234+0000", 2023-09-23T20:58:21.782 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0050151680000000004 2023-09-23T20:58:21.782 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.782 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.782 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:58:21.783 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:12.927966+0000", 2023-09-23T20:58:21.783 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.018731945999999999 2023-09-23T20:58:21.783 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.783 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.783 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T20:58:21.783 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:12.928078+0000", 2023-09-23T20:58:21.783 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000112485 2023-09-23T20:58:21.783 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.783 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.783 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 1,7", 2023-09-23T20:58:21.783 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:12.928391+0000", 2023-09-23T20:58:21.783 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00031247000000000002 2023-09-23T20:58:21.783 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.784 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.784 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:58:21.784 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:12.951689+0000", 2023-09-23T20:58:21.784 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.023298762000000001 2023-09-23T20:58:21.784 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.784 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.784 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T20:58:21.784 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:12.951697+0000", 2023-09-23T20:58:21.784 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.8189999999999992e-06 2023-09-23T20:58:21.784 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.784 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.784 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:58:21.784 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:13.925895+0000", 2023-09-23T20:58:21.785 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.97419815399999998 2023-09-23T20:58:21.785 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.785 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.785 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T20:58:21.785 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:13.925906+0000", 2023-09-23T20:58:21.785 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0319000000000001e-05 2023-09-23T20:58:21.785 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.785 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.785 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:58:21.785 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:15.017793+0000", 2023-09-23T20:58:21.785 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0918876260000001 2023-09-23T20:58:21.785 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.785 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.786 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for missing object", 2023-09-23T20:58:21.786 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:15.017905+0000", 2023-09-23T20:58:21.786 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000112256 2023-09-23T20:58:21.786 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.786 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.786 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:58:21.786 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:15.033919+0000", 2023-09-23T20:58:21.786 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.016013263999999999 2023-09-23T20:58:21.786 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.786 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.786 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T20:58:21.786 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:15.034012+0000", 2023-09-23T20:58:21.787 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.3101000000000002e-05 2023-09-23T20:58:21.787 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:58:21.787 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T20:58:21.787 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:58:21.787 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.787 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.787 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34797.0:114965 6.17 6:eed136d7:::benchmark_data_smithi026_223868_object13839:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e131)", 2023-09-23T20:58:21.787 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T20:54:12.907032+0000", 2023-09-23T20:58:21.787 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 248.853803285, 2023-09-23T20:58:21.787 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.1278061450000001, 2023-09-23T20:58:21.787 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T20:58:21.787 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:58:21.787 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T20:58:21.788 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34797", 2023-09-23T20:58:21.788 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T20:58:21.788 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 114965 2023-09-23T20:58:21.788 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.788 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T20:58:21.788 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.788 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T20:58:21.788 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:12.907032+0000", 2023-09-23T20:58:21.788 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:58:21.788 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.788 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.788 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T20:58:21.788 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:12.907032+0000", 2023-09-23T20:58:21.789 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:58:21.789 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.789 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.789 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T20:58:21.789 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:12.907030+0000", 2023-09-23T20:58:21.789 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T20:58:21.789 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.789 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.789 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T20:58:21.789 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:12.907038+0000", 2023-09-23T20:58:21.789 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.3350000000000001e-06 2023-09-23T20:58:21.789 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.789 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.789 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T20:58:21.790 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:12.907050+0000", 2023-09-23T20:58:21.790 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.2398999999999999e-05 2023-09-23T20:58:21.790 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.790 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.790 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T20:58:21.790 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:12.907058+0000", 2023-09-23T20:58:21.790 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.1820000000000006e-06 2023-09-23T20:58:21.790 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.790 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.790 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:58:21.790 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:12.909340+0000", 2023-09-23T20:58:21.790 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.002281731 2023-09-23T20:58:21.791 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.791 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.791 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for degraded object", 2023-09-23T20:58:21.791 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:12.918086+0000", 2023-09-23T20:58:21.791 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.008745971 2023-09-23T20:58:21.791 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.791 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.791 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:58:21.791 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:12.952195+0000", 2023-09-23T20:58:21.791 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.034108861999999997 2023-09-23T20:58:21.791 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.791 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.791 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T20:58:21.792 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:12.952202+0000", 2023-09-23T20:58:21.792 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.686e-06 2023-09-23T20:58:21.792 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.792 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.792 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:58:21.792 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:13.925935+0000", 2023-09-23T20:58:21.792 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.97373368500000002 2023-09-23T20:58:21.792 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.792 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.792 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T20:58:21.792 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:13.925944+0000", 2023-09-23T20:58:21.792 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.9530000000000005e-06 2023-09-23T20:58:21.793 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.793 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.793 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:58:21.793 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:15.017933+0000", 2023-09-23T20:58:21.793 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0919890029999999 2023-09-23T20:58:21.793 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.793 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.793 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T20:58:21.793 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:15.018041+0000", 2023-09-23T20:58:21.793 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000107551 2023-09-23T20:58:21.793 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.793 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.793 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 1,4", 2023-09-23T20:58:21.794 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:15.018366+0000", 2023-09-23T20:58:21.794 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00032571500000000002 2023-09-23T20:58:21.794 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.794 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.794 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:58:21.794 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:15.031011+0000", 2023-09-23T20:58:21.794 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.012644479 2023-09-23T20:58:21.794 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.794 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.794 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:58:21.794 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:15.031048+0000", 2023-09-23T20:58:21.795 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.6822000000000001e-05 2023-09-23T20:58:21.795 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.795 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.795 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T20:58:21.795 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:15.034520+0000", 2023-09-23T20:58:21.795 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0034718459999999998 2023-09-23T20:58:21.795 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.795 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.795 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T20:58:21.795 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:15.034571+0000", 2023-09-23T20:58:21.795 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.1898999999999999e-05 2023-09-23T20:58:21.795 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.795 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.796 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T20:58:21.796 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:15.034838+0000", 2023-09-23T20:58:21.796 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00026648299999999998 2023-09-23T20:58:21.796 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:58:21.796 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T20:58:21.796 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:58:21.796 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.796 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.796 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34797.0:114977 6.16 6:6abc4ce0:::benchmark_data_smithi026_223868_object13851:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e131)", 2023-09-23T20:58:21.796 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T20:54:12.931076+0000", 2023-09-23T20:58:21.796 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 248.829759532, 2023-09-23T20:58:21.796 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.0790598340000002, 2023-09-23T20:58:21.796 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T20:58:21.797 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:58:21.797 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T20:58:21.797 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34797", 2023-09-23T20:58:21.797 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T20:58:21.797 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 114977 2023-09-23T20:58:21.797 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.797 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T20:58:21.797 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.797 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T20:58:21.797 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:12.931076+0000", 2023-09-23T20:58:21.797 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:58:21.797 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.798 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.798 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T20:58:21.798 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:12.931076+0000", 2023-09-23T20:58:21.798 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:58:21.798 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.798 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.798 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T20:58:21.798 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:12.931075+0000", 2023-09-23T20:58:21.798 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T20:58:21.799 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.799 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.799 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T20:58:21.799 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:12.931079+0000", 2023-09-23T20:58:21.799 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.5200000000000002e-06 2023-09-23T20:58:21.799 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.799 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.799 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T20:58:21.799 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:12.931086+0000", 2023-09-23T20:58:21.799 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.0299999999999996e-06 2023-09-23T20:58:21.799 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.799 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.799 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T20:58:21.800 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:12.931108+0000", 2023-09-23T20:58:21.800 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.2226999999999999e-05 2023-09-23T20:58:21.800 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.800 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.800 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:58:21.800 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:12.931257+0000", 2023-09-23T20:58:21.800 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00014939900000000001 2023-09-23T20:58:21.800 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.800 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.800 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T20:58:21.800 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:12.931281+0000", 2023-09-23T20:58:21.800 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.4057e-05 2023-09-23T20:58:21.800 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.800 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.801 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:58:21.801 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:13.925102+0000", 2023-09-23T20:58:21.801 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.993820284 2023-09-23T20:58:21.801 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.801 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.801 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T20:58:21.801 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:13.925116+0000", 2023-09-23T20:58:21.801 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.3886000000000001e-05 2023-09-23T20:58:21.801 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.801 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.801 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:58:21.801 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:15.000774+0000", 2023-09-23T20:58:21.802 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0756585869999999 2023-09-23T20:58:21.802 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.802 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.802 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T20:58:21.802 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:15.000859+0000", 2023-09-23T20:58:21.802 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.5370000000000004e-05 2023-09-23T20:58:21.802 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.802 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.802 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 4,7", 2023-09-23T20:58:21.802 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:15.001678+0000", 2023-09-23T20:58:21.802 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000818913 2023-09-23T20:58:21.802 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.803 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.803 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:58:21.803 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:15.008665+0000", 2023-09-23T20:58:21.803 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0069865600000000002 2023-09-23T20:58:21.803 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.803 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.803 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:58:21.803 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:15.008706+0000", 2023-09-23T20:58:21.803 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.1285999999999997e-05 2023-09-23T20:58:21.803 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.803 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.803 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T20:58:21.804 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:15.009602+0000", 2023-09-23T20:58:21.804 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00089599299999999998 2023-09-23T20:58:21.804 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.804 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.804 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T20:58:21.804 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:15.009659+0000", 2023-09-23T20:58:21.804 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.6289999999999998e-05 2023-09-23T20:58:21.804 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.804 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.804 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T20:58:21.804 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:15.010135+0000", 2023-09-23T20:58:21.804 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00047690699999999999 2023-09-23T20:58:21.805 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:58:21.805 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T20:58:21.805 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:58:21.805 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.805 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.805 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34797.0:114989 6.6 6:60ed2ddf:::benchmark_data_smithi026_223868_object13863:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e132)", 2023-09-23T20:58:21.805 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T20:54:12.964779+0000", 2023-09-23T20:58:21.805 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 248.796055833, 2023-09-23T20:58:21.805 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.0443830630000002, 2023-09-23T20:58:21.805 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T20:58:21.805 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:58:21.805 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T20:58:21.806 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34797", 2023-09-23T20:58:21.806 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T20:58:21.806 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 114989 2023-09-23T20:58:21.806 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.806 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T20:58:21.806 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.806 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T20:58:21.806 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:12.964779+0000", 2023-09-23T20:58:21.806 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:58:21.806 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.806 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.806 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T20:58:21.807 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:12.964779+0000", 2023-09-23T20:58:21.807 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:58:21.807 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.807 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.807 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T20:58:21.807 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:12.964778+0000", 2023-09-23T20:58:21.807 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T20:58:21.807 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.807 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.807 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T20:58:21.807 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:12.964784+0000", 2023-09-23T20:58:21.807 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.8989999999999998e-06 2023-09-23T20:58:21.807 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.808 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.808 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T20:58:21.808 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:12.964795+0000", 2023-09-23T20:58:21.808 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1214e-05 2023-09-23T20:58:21.808 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.808 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.808 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T20:58:21.808 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:12.964805+0000", 2023-09-23T20:58:21.808 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0346999999999999e-05 2023-09-23T20:58:21.808 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.808 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.808 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:58:21.808 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:12.964859+0000", 2023-09-23T20:58:21.809 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.3591000000000002e-05 2023-09-23T20:58:21.809 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.809 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.809 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T20:58:21.809 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:12.964871+0000", 2023-09-23T20:58:21.809 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.2109999999999999e-05 2023-09-23T20:58:21.809 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.809 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.809 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:58:21.809 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:13.925234+0000", 2023-09-23T20:58:21.809 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.96036344399999996 2023-09-23T20:58:21.809 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.809 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.809 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T20:58:21.810 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:13.925245+0000", 2023-09-23T20:58:21.810 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0054999999999999e-05 2023-09-23T20:58:21.810 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.810 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.810 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:58:21.810 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:14.996272+0000", 2023-09-23T20:58:21.810 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0710279009999999 2023-09-23T20:58:21.810 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.810 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.810 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T20:58:21.810 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:14.996437+0000", 2023-09-23T20:58:21.810 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00016484299999999999 2023-09-23T20:58:21.811 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.811 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.811 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 4,7", 2023-09-23T20:58:21.811 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:14.996837+0000", 2023-09-23T20:58:21.811 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00039951599999999999 2023-09-23T20:58:21.811 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.811 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.811 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:58:21.811 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:14.998787+0000", 2023-09-23T20:58:21.811 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0019506770000000001 2023-09-23T20:58:21.811 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.811 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.811 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:58:21.812 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:15.001358+0000", 2023-09-23T20:58:21.812 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0025709750000000001 2023-09-23T20:58:21.812 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.812 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.812 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T20:58:21.812 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:15.008625+0000", 2023-09-23T20:58:21.812 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0072665230000000004 2023-09-23T20:58:21.812 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.812 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.812 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T20:58:21.812 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:15.008685+0000", 2023-09-23T20:58:21.812 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.0244000000000001e-05 2023-09-23T20:58:21.813 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.813 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.813 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T20:58:21.813 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:15.009162+0000", 2023-09-23T20:58:21.813 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00047711299999999998 2023-09-23T20:58:21.813 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:58:21.813 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T20:58:21.813 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:58:21.813 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.813 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.813 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34797.0:114991 6.17 6:ed158f03:::benchmark_data_smithi026_223868_object13865:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e132)", 2023-09-23T20:58:21.813 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T20:54:12.968317+0000", 2023-09-23T20:58:21.813 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 248.792517621, 2023-09-23T20:58:21.814 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.0674853450000001, 2023-09-23T20:58:21.814 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T20:58:21.814 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:58:21.814 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T20:58:21.814 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34797", 2023-09-23T20:58:21.814 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T20:58:21.814 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 114991 2023-09-23T20:58:21.814 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.814 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T20:58:21.814 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.814 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T20:58:21.815 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:12.968317+0000", 2023-09-23T20:58:21.815 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:58:21.815 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.815 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.815 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T20:58:21.815 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:12.968317+0000", 2023-09-23T20:58:21.815 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:58:21.815 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.815 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.815 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T20:58:21.815 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:12.968316+0000", 2023-09-23T20:58:21.815 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T20:58:21.815 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.816 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.816 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T20:58:21.816 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:12.968321+0000", 2023-09-23T20:58:21.816 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.2669999999999997e-06 2023-09-23T20:58:21.816 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.816 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.816 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T20:58:21.816 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:12.968334+0000", 2023-09-23T20:58:21.816 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.3101999999999999e-05 2023-09-23T20:58:21.816 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.816 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.816 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T20:58:21.816 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:12.968342+0000", 2023-09-23T20:58:21.817 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.5089999999999996e-06 2023-09-23T20:58:21.817 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.817 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.817 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:58:21.817 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:12.968406+0000", 2023-09-23T20:58:21.817 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.4132000000000002e-05 2023-09-23T20:58:21.817 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.817 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.817 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T20:58:21.817 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:12.968418+0000", 2023-09-23T20:58:21.817 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.2459999999999999e-05 2023-09-23T20:58:21.817 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.817 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.818 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:58:21.818 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:13.925969+0000", 2023-09-23T20:58:21.818 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.95755082800000002 2023-09-23T20:58:21.818 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.818 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.818 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T20:58:21.818 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:13.925978+0000", 2023-09-23T20:58:21.818 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.0149999999999992e-06 2023-09-23T20:58:21.818 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.818 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.818 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:58:21.818 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:15.019699+0000", 2023-09-23T20:58:21.819 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0937209290000001 2023-09-23T20:58:21.819 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.819 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.819 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T20:58:21.819 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:15.019848+0000", 2023-09-23T20:58:21.819 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00014907899999999999 2023-09-23T20:58:21.819 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.819 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.819 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 1,4", 2023-09-23T20:58:21.819 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:15.020197+0000", 2023-09-23T20:58:21.819 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000348964 2023-09-23T20:58:21.819 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.819 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.820 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:58:21.820 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:15.033098+0000", 2023-09-23T20:58:21.820 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.012900327 2023-09-23T20:58:21.820 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.820 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.820 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T20:58:21.820 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:15.034847+0000", 2023-09-23T20:58:21.820 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0017495 2023-09-23T20:58:21.820 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.820 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.820 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:58:21.820 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:15.035482+0000", 2023-09-23T20:58:21.820 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00063472799999999998 2023-09-23T20:58:21.821 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.821 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.821 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T20:58:21.821 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:15.035528+0000", 2023-09-23T20:58:21.821 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.5874000000000002e-05 2023-09-23T20:58:21.821 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.821 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.821 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T20:58:21.821 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:15.035803+0000", 2023-09-23T20:58:21.821 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00027521799999999998 2023-09-23T20:58:21.821 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:58:21.821 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T20:58:21.821 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:58:21.822 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.822 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.822 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34797.0:114998 6.7 6:e21330c6:::benchmark_data_smithi026_223868_object13872:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e132)", 2023-09-23T20:58:21.822 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T20:54:12.985057+0000", 2023-09-23T20:58:21.822 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 248.77577791900001, 2023-09-23T20:58:21.822 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.0503685319999998, 2023-09-23T20:58:21.822 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T20:58:21.822 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:58:21.822 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T20:58:21.822 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34797", 2023-09-23T20:58:21.822 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T20:58:21.822 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 114998 2023-09-23T20:58:21.823 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.823 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T20:58:21.823 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.823 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T20:58:21.823 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:12.985057+0000", 2023-09-23T20:58:21.823 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:58:21.823 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.823 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.823 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T20:58:21.823 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:12.985057+0000", 2023-09-23T20:58:21.823 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:58:21.823 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.824 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.824 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T20:58:21.824 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:12.985054+0000", 2023-09-23T20:58:21.824 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999971 2023-09-23T20:58:21.824 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.824 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.824 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T20:58:21.824 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:12.985066+0000", 2023-09-23T20:58:21.824 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.2031000000000001e-05 2023-09-23T20:58:21.824 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.824 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.824 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T20:58:21.825 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:12.985089+0000", 2023-09-23T20:58:21.825 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.2501e-05 2023-09-23T20:58:21.825 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.825 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.825 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T20:58:21.825 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:12.985103+0000", 2023-09-23T20:58:21.825 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.4392e-05 2023-09-23T20:58:21.825 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.825 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.825 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:58:21.825 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:12.988980+0000", 2023-09-23T20:58:21.826 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0038764509999999999 2023-09-23T20:58:21.826 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.826 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.826 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T20:58:21.826 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:12.988996+0000", 2023-09-23T20:58:21.826 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.5953000000000001e-05 2023-09-23T20:58:21.826 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.826 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.826 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:58:21.826 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:13.925849+0000", 2023-09-23T20:58:21.826 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.93685326800000002 2023-09-23T20:58:21.826 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.827 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.827 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T20:58:21.827 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:13.925860+0000", 2023-09-23T20:58:21.827 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1090000000000001e-05 2023-09-23T20:58:21.827 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.827 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.827 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:58:21.827 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:15.019676+0000", 2023-09-23T20:58:21.827 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0938154149999999 2023-09-23T20:58:21.827 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.827 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.827 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T20:58:21.827 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:15.019809+0000", 2023-09-23T20:58:21.828 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00013316599999999999 2023-09-23T20:58:21.828 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.828 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.828 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 1,4", 2023-09-23T20:58:21.828 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:15.020197+0000", 2023-09-23T20:58:21.828 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00038824999999999998 2023-09-23T20:58:21.828 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.828 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.828 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:58:21.828 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:15.033893+0000", 2023-09-23T20:58:21.828 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.013696442 2023-09-23T20:58:21.828 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.829 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.829 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T20:58:21.829 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:15.034854+0000", 2023-09-23T20:58:21.829 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00096095700000000002 2023-09-23T20:58:21.829 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.829 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.829 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:58:21.829 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:15.034911+0000", 2023-09-23T20:58:21.829 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.6915e-05 2023-09-23T20:58:21.829 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.829 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.829 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T20:58:21.829 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:15.034967+0000", 2023-09-23T20:58:21.830 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.6249000000000003e-05 2023-09-23T20:58:21.830 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.830 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.830 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T20:58:21.830 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:15.035426+0000", 2023-09-23T20:58:21.830 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00045822899999999999 2023-09-23T20:58:21.830 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:58:21.830 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T20:58:21.830 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:58:21.830 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.830 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.830 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34797.0:115009 6.7 6:e365786c:::benchmark_data_smithi026_223868_object13883:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e133)", 2023-09-23T20:58:21.831 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T20:54:14.026745+0000", 2023-09-23T20:58:21.831 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 247.73408994799999, 2023-09-23T20:58:21.831 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0094194009999999, 2023-09-23T20:58:21.831 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T20:58:21.831 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:58:21.831 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T20:58:21.831 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34797", 2023-09-23T20:58:21.831 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T20:58:21.831 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 115009 2023-09-23T20:58:21.831 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.831 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T20:58:21.831 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.831 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T20:58:21.832 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:14.026745+0000", 2023-09-23T20:58:21.832 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:58:21.832 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.832 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.832 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T20:58:21.832 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:14.026745+0000", 2023-09-23T20:58:21.832 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:58:21.832 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.832 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.832 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T20:58:21.832 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:14.026743+0000", 2023-09-23T20:58:21.832 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999981 2023-09-23T20:58:21.832 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.833 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.833 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T20:58:21.833 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:14.026752+0000", 2023-09-23T20:58:21.833 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.1670000000000005e-06 2023-09-23T20:58:21.833 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.833 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.833 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T20:58:21.833 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:14.026774+0000", 2023-09-23T20:58:21.833 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.1203e-05 2023-09-23T20:58:21.833 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.833 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.833 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T20:58:21.834 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:14.026782+0000", 2023-09-23T20:58:21.834 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.4370000000000007e-06 2023-09-23T20:58:21.834 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.834 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.834 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:58:21.834 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:14.026855+0000", 2023-09-23T20:58:21.834 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.2911999999999996e-05 2023-09-23T20:58:21.834 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.834 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.834 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T20:58:21.834 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:14.026870+0000", 2023-09-23T20:58:21.834 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.4718e-05 2023-09-23T20:58:21.835 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.835 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.835 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:58:21.835 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:15.023554+0000", 2023-09-23T20:58:21.835 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.99668419500000005 2023-09-23T20:58:21.835 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.835 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.835 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T20:58:21.835 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:15.023665+0000", 2023-09-23T20:58:21.835 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000110841 2023-09-23T20:58:21.835 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.835 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.835 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 1,4", 2023-09-23T20:58:21.836 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:15.024005+0000", 2023-09-23T20:58:21.836 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00034030399999999999 2023-09-23T20:58:21.836 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.836 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.836 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:58:21.836 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:15.034505+0000", 2023-09-23T20:58:21.836 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.010499812000000001 2023-09-23T20:58:21.836 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.836 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.836 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T20:58:21.836 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:15.034861+0000", 2023-09-23T20:58:21.836 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000355991 2023-09-23T20:58:21.837 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.837 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.837 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:58:21.837 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:15.035843+0000", 2023-09-23T20:58:21.837 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00098196299999999998 2023-09-23T20:58:21.837 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.837 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.837 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T20:58:21.837 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:15.035888+0000", 2023-09-23T20:58:21.837 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.4678000000000002e-05 2023-09-23T20:58:21.837 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.837 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.837 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T20:58:21.838 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:15.036165+0000", 2023-09-23T20:58:21.838 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00027704800000000001 2023-09-23T20:58:21.838 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:58:21.838 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T20:58:21.838 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:58:21.838 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.838 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.838 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34797.0:130469 6.c 6:38e631ce:::benchmark_data_smithi026_223868_object29343:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e135)", 2023-09-23T20:58:21.838 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T20:54:30.880090+0000", 2023-09-23T20:58:21.838 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 230.880744655, 2023-09-23T20:58:21.838 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.1674490940000002, 2023-09-23T20:58:21.838 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T20:58:21.839 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "reached pg", 2023-09-23T20:58:21.839 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T20:58:21.839 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34797", 2023-09-23T20:58:21.839 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T20:58:21.839 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 130469 2023-09-23T20:58:21.839 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.839 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T20:58:21.839 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.839 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T20:58:21.839 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:30.880090+0000", 2023-09-23T20:58:21.839 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:58:21.839 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.839 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.840 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T20:58:21.840 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:30.880090+0000", 2023-09-23T20:58:21.840 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:58:21.840 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.840 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.840 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T20:58:21.840 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:30.880088+0000", 2023-09-23T20:58:21.840 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999981 2023-09-23T20:58:21.840 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.840 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.840 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T20:58:21.840 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:30.880097+0000", 2023-09-23T20:58:21.840 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.4610000000000008e-06 2023-09-23T20:58:21.841 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.841 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.841 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T20:58:21.841 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:30.880115+0000", 2023-09-23T20:58:21.841 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.7901e-05 2023-09-23T20:58:21.841 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.841 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.841 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T20:58:21.841 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:30.880124+0000", 2023-09-23T20:58:21.841 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.3379999999999994e-06 2023-09-23T20:58:21.841 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.841 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.841 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:58:21.842 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:30.880185+0000", 2023-09-23T20:58:21.842 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.1262999999999999e-05 2023-09-23T20:58:21.842 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.842 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.842 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T20:58:21.842 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:30.880348+0000", 2023-09-23T20:58:21.842 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00016296799999999999 2023-09-23T20:58:21.842 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.842 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.842 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 6,7", 2023-09-23T20:58:21.842 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:30.880434+0000", 2023-09-23T20:58:21.843 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.5793999999999999e-05 2023-09-23T20:58:21.843 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.843 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.843 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T20:58:21.843 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:30.888682+0000", 2023-09-23T20:58:21.843 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0082474290000000006 2023-09-23T20:58:21.843 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.843 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.843 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:58:21.843 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:30.915595+0000", 2023-09-23T20:58:21.843 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.026913060999999999 2023-09-23T20:58:21.843 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.843 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.844 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T20:58:21.844 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:30.915608+0000", 2023-09-23T20:58:21.844 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.3742e-05 2023-09-23T20:58:21.844 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.844 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.844 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:58:21.844 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:31.887950+0000", 2023-09-23T20:58:21.844 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.97234175 2023-09-23T20:58:21.844 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.844 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.844 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T20:58:21.844 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:31.887964+0000", 2023-09-23T20:58:21.845 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.3711e-05 2023-09-23T20:58:21.845 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.845 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.845 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:58:21.845 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:33.036111+0000", 2023-09-23T20:58:21.845 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1481468340000001 2023-09-23T20:58:21.845 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.845 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.845 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for missing object", 2023-09-23T20:58:21.845 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:33.036258+0000", 2023-09-23T20:58:21.845 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00014725900000000001 2023-09-23T20:58:21.845 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.846 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.846 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:58:21.846 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:33.047449+0000", 2023-09-23T20:58:21.846 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.011190752 2023-09-23T20:58:21.846 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.846 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.846 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T20:58:21.846 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:33.047540+0000", 2023-09-23T20:58:21.846 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.0873999999999998e-05 2023-09-23T20:58:21.846 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:58:21.846 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T20:58:21.846 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:58:21.847 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.847 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.847 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34797.0:130480 6.c 6:3c6c8e98:::benchmark_data_smithi026_223868_object29354:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e135)", 2023-09-23T20:58:21.847 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T20:54:30.892126+0000", 2023-09-23T20:58:21.847 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 230.86870880199999, 2023-09-23T20:58:21.847 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.1590397989999999, 2023-09-23T20:58:21.847 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T20:58:21.847 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "reached pg", 2023-09-23T20:58:21.847 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T20:58:21.847 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34797", 2023-09-23T20:58:21.847 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T20:58:21.847 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 130480 2023-09-23T20:58:21.848 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.848 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T20:58:21.848 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.848 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T20:58:21.848 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:30.892126+0000", 2023-09-23T20:58:21.848 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:58:21.848 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.848 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.848 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T20:58:21.848 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:30.892126+0000", 2023-09-23T20:58:21.848 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:58:21.848 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.848 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.849 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T20:58:21.849 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:30.892125+0000", 2023-09-23T20:58:21.849 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T20:58:21.849 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.849 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.849 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T20:58:21.849 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:30.892131+0000", 2023-09-23T20:58:21.849 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.7209999999999997e-06 2023-09-23T20:58:21.849 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.849 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.849 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T20:58:21.849 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:30.892144+0000", 2023-09-23T20:58:21.849 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.2488e-05 2023-09-23T20:58:21.850 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.850 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.850 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T20:58:21.850 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:30.892152+0000", 2023-09-23T20:58:21.850 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.7330000000000003e-06 2023-09-23T20:58:21.850 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.850 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.850 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:58:21.850 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:30.892201+0000", 2023-09-23T20:58:21.850 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.9678000000000001e-05 2023-09-23T20:58:21.850 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.851 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.851 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T20:58:21.851 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:30.892324+0000", 2023-09-23T20:58:21.851 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00012292299999999999 2023-09-23T20:58:21.851 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.851 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.851 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 6,7", 2023-09-23T20:58:21.851 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:30.892438+0000", 2023-09-23T20:58:21.851 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000113915 2023-09-23T20:58:21.851 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.851 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.851 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T20:58:21.851 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:30.907285+0000", 2023-09-23T20:58:21.852 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.014847057 2023-09-23T20:58:21.852 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.852 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.852 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:58:21.852 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:30.915644+0000", 2023-09-23T20:58:21.852 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0083591480000000003 2023-09-23T20:58:21.852 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.852 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.852 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T20:58:21.852 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:30.915653+0000", 2023-09-23T20:58:21.852 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.4300000000000006e-06 2023-09-23T20:58:21.852 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.852 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.853 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:58:21.853 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:31.888064+0000", 2023-09-23T20:58:21.853 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.97241095300000002 2023-09-23T20:58:21.853 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.853 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.853 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T20:58:21.853 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:31.888069+0000", 2023-09-23T20:58:21.853 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.6450000000000002e-06 2023-09-23T20:58:21.853 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.853 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.853 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:58:21.853 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:33.037086+0000", 2023-09-23T20:58:21.853 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1490164059999999 2023-09-23T20:58:21.854 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.854 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.854 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for missing object", 2023-09-23T20:58:21.854 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:33.037184+0000", 2023-09-23T20:58:21.854 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.8024999999999997e-05 2023-09-23T20:58:21.854 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.854 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.854 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:58:21.854 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:33.051071+0000", 2023-09-23T20:58:21.854 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.013886895999999999 2023-09-23T20:58:21.854 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.854 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.855 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T20:58:21.855 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:33.051166+0000", 2023-09-23T20:58:21.855 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.5426000000000004e-05 2023-09-23T20:58:21.855 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:58:21.855 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T20:58:21.855 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:58:21.855 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.855 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.855 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34797.0:130484 6.17 6:ebb95d79:::benchmark_data_smithi026_223868_object29358:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e136)", 2023-09-23T20:58:21.855 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T20:54:30.909236+0000", 2023-09-23T20:58:21.855 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 230.85159915, 2023-09-23T20:58:21.855 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.1373689279999999, 2023-09-23T20:58:21.856 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T20:58:21.856 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:58:21.856 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T20:58:21.856 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34797", 2023-09-23T20:58:21.856 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T20:58:21.856 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 130484 2023-09-23T20:58:21.856 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.856 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T20:58:21.856 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.856 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T20:58:21.856 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:30.909236+0000", 2023-09-23T20:58:21.856 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:58:21.857 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.857 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.857 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T20:58:21.857 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:30.909236+0000", 2023-09-23T20:58:21.857 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:58:21.857 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.857 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.857 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T20:58:21.857 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:30.909234+0000", 2023-09-23T20:58:21.857 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999981 2023-09-23T20:58:21.857 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.857 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.857 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T20:58:21.858 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:30.909241+0000", 2023-09-23T20:58:21.858 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.9630000000000001e-06 2023-09-23T20:58:21.858 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.858 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.858 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T20:58:21.858 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:30.909258+0000", 2023-09-23T20:58:21.858 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.7105999999999999e-05 2023-09-23T20:58:21.858 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.858 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.858 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T20:58:21.858 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:30.909267+0000", 2023-09-23T20:58:21.858 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.0440000000000001e-06 2023-09-23T20:58:21.859 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.859 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.859 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:58:21.859 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:30.914297+0000", 2023-09-23T20:58:21.859 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0050297720000000001 2023-09-23T20:58:21.859 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.859 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.859 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T20:58:21.859 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:30.914307+0000", 2023-09-23T20:58:21.859 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0166000000000001e-05 2023-09-23T20:58:21.859 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.859 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.859 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:58:21.860 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:31.880008+0000", 2023-09-23T20:58:21.860 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.96570099399999998 2023-09-23T20:58:21.860 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.860 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.860 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T20:58:21.860 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:31.880021+0000", 2023-09-23T20:58:21.860 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.2700999999999999e-05 2023-09-23T20:58:21.860 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.860 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.860 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:58:21.860 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:33.017285+0000", 2023-09-23T20:58:21.860 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1372641450000001 2023-09-23T20:58:21.860 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.861 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.861 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T20:58:21.861 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:33.017380+0000", 2023-09-23T20:58:21.861 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.5118000000000006e-05 2023-09-23T20:58:21.861 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.861 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.861 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 1,4,5", 2023-09-23T20:58:21.861 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:33.017728+0000", 2023-09-23T20:58:21.861 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000348 2023-09-23T20:58:21.861 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.861 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.861 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:58:21.861 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:33.021064+0000", 2023-09-23T20:58:21.862 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0033353430000000002 2023-09-23T20:58:21.862 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.862 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.862 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:58:21.862 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:33.042239+0000", 2023-09-23T20:58:21.862 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.021175225999999998 2023-09-23T20:58:21.862 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.862 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.862 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:58:21.862 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:33.042839+0000", 2023-09-23T20:58:21.862 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00060004999999999998 2023-09-23T20:58:21.862 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.863 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.863 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T20:58:21.863 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:33.046070+0000", 2023-09-23T20:58:21.863 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0032314000000000002 2023-09-23T20:58:21.863 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.863 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.863 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T20:58:21.863 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:33.046122+0000", 2023-09-23T20:58:21.863 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.1583999999999999e-05 2023-09-23T20:58:21.863 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.863 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.863 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T20:58:21.863 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:33.046605+0000", 2023-09-23T20:58:21.864 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000483005 2023-09-23T20:58:21.864 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:58:21.864 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T20:58:21.864 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:58:21.864 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.864 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.864 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34797.0:130487 6.c 6:3110d152:::benchmark_data_smithi026_223868_object29361:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e136)", 2023-09-23T20:58:21.864 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T20:54:30.913625+0000", 2023-09-23T20:58:21.864 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 230.84721043600001, 2023-09-23T20:58:21.864 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.1412422530000002, 2023-09-23T20:58:21.864 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T20:58:21.864 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:58:21.864 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T20:58:21.865 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34797", 2023-09-23T20:58:21.865 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T20:58:21.865 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 130487 2023-09-23T20:58:21.865 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.865 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T20:58:21.865 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.865 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T20:58:21.865 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:30.913625+0000", 2023-09-23T20:58:21.865 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:58:21.865 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.865 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.865 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T20:58:21.865 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:30.913625+0000", 2023-09-23T20:58:21.866 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:58:21.866 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.866 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.866 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T20:58:21.866 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:30.913623+0000", 2023-09-23T20:58:21.866 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T20:58:21.866 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.866 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.866 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T20:58:21.866 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:30.913631+0000", 2023-09-23T20:58:21.866 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.5979999999999999e-06 2023-09-23T20:58:21.866 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.867 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.867 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T20:58:21.867 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:30.913643+0000", 2023-09-23T20:58:21.867 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.223e-05 2023-09-23T20:58:21.867 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.867 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.867 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T20:58:21.867 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:30.913665+0000", 2023-09-23T20:58:21.867 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.1792000000000001e-05 2023-09-23T20:58:21.867 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.867 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.867 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:58:21.867 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:30.915686+0000", 2023-09-23T20:58:21.868 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0020213150000000001 2023-09-23T20:58:21.868 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.868 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.868 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T20:58:21.868 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:30.915699+0000", 2023-09-23T20:58:21.868 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.2496e-05 2023-09-23T20:58:21.868 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.868 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.868 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:58:21.868 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:31.888085+0000", 2023-09-23T20:58:21.868 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.97238664500000005 2023-09-23T20:58:21.868 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.869 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.869 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T20:58:21.869 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:31.888090+0000", 2023-09-23T20:58:21.869 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.4839999999999998e-06 2023-09-23T20:58:21.869 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.869 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.869 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:58:21.869 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:33.037214+0000", 2023-09-23T20:58:21.869 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1491243719999999 2023-09-23T20:58:21.869 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.869 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.869 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T20:58:21.870 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:33.037330+0000", 2023-09-23T20:58:21.870 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000115977 2023-09-23T20:58:21.870 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.870 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.870 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 5,6,7", 2023-09-23T20:58:21.870 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:33.037423+0000", 2023-09-23T20:58:21.870 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.2723e-05 2023-09-23T20:58:21.870 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.870 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.870 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:58:21.870 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:33.046955+0000", 2023-09-23T20:58:21.870 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0095323809999999995 2023-09-23T20:58:21.871 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.871 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.871 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:58:21.871 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:33.047055+0000", 2023-09-23T20:58:21.871 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.9870000000000004e-05 2023-09-23T20:58:21.871 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.871 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.871 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:58:21.871 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:33.051207+0000", 2023-09-23T20:58:21.871 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0041521850000000001 2023-09-23T20:58:21.871 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.871 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.871 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T20:58:21.872 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:33.054703+0000", 2023-09-23T20:58:21.872 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0034956599999999998 2023-09-23T20:58:21.872 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.872 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.872 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T20:58:21.872 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:33.054753+0000", 2023-09-23T20:58:21.872 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.0430999999999999e-05 2023-09-23T20:58:21.872 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.872 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.872 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T20:58:21.872 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:33.054867+0000", 2023-09-23T20:58:21.872 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000113626 2023-09-23T20:58:21.873 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:58:21.873 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T20:58:21.873 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:58:21.873 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.873 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.873 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34797.0:130508 6.c 6:33de9cb1:::benchmark_data_smithi026_223868_object29382:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e136)", 2023-09-23T20:58:21.873 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T20:54:30.927850+0000", 2023-09-23T20:58:21.873 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 230.83298462100001, 2023-09-23T20:58:21.873 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.1274186720000001, 2023-09-23T20:58:21.873 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T20:58:21.873 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:58:21.873 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T20:58:21.873 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34797", 2023-09-23T20:58:21.874 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T20:58:21.874 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 130508 2023-09-23T20:58:21.874 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.874 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T20:58:21.874 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.874 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T20:58:21.874 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:30.927850+0000", 2023-09-23T20:58:21.874 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:58:21.874 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.874 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.874 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T20:58:21.874 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:30.927850+0000", 2023-09-23T20:58:21.875 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:58:21.875 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.875 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.875 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T20:58:21.875 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:30.927850+0000", 2023-09-23T20:58:21.875 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T20:58:21.875 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.875 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.875 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T20:58:21.875 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:30.927854+0000", 2023-09-23T20:58:21.875 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.0949999999999998e-06 2023-09-23T20:58:21.875 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.875 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.876 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T20:58:21.876 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:30.927866+0000", 2023-09-23T20:58:21.876 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1843e-05 2023-09-23T20:58:21.876 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.876 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.876 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T20:58:21.876 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:30.927871+0000", 2023-09-23T20:58:21.876 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.9490000000000004e-06 2023-09-23T20:58:21.876 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.876 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.876 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:58:21.876 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:30.927913+0000", 2023-09-23T20:58:21.876 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.2367999999999998e-05 2023-09-23T20:58:21.877 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.877 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.877 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T20:58:21.877 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:30.927921+0000", 2023-09-23T20:58:21.877 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.2339999999999994e-06 2023-09-23T20:58:21.877 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.877 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.877 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:58:21.877 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:31.888102+0000", 2023-09-23T20:58:21.877 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.960181116 2023-09-23T20:58:21.877 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.877 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.877 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T20:58:21.878 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:31.888106+0000", 2023-09-23T20:58:21.878 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.0969999999999999e-06 2023-09-23T20:58:21.878 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.878 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.878 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:58:21.878 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:33.041530+0000", 2023-09-23T20:58:21.878 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1534237329999999 2023-09-23T20:58:21.878 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.878 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.878 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T20:58:21.878 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:33.041653+0000", 2023-09-23T20:58:21.879 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00012306099999999999 2023-09-23T20:58:21.879 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.879 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.879 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 5,6,7", 2023-09-23T20:58:21.879 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:33.041780+0000", 2023-09-23T20:58:21.879 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000126978 2023-09-23T20:58:21.879 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.879 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.879 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:58:21.879 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:33.047098+0000", 2023-09-23T20:58:21.879 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0053179400000000002 2023-09-23T20:58:21.879 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.880 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.880 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:58:21.880 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:33.047424+0000", 2023-09-23T20:58:21.880 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00032540299999999999 2023-09-23T20:58:21.880 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.880 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.880 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:58:21.880 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:33.051248+0000", 2023-09-23T20:58:21.880 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.003824858 2023-09-23T20:58:21.880 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.880 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.880 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T20:58:21.880 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:33.055098+0000", 2023-09-23T20:58:21.881 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0038491419999999998 2023-09-23T20:58:21.881 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.881 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.881 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T20:58:21.881 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:33.055154+0000", 2023-09-23T20:58:21.881 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.6585000000000001e-05 2023-09-23T20:58:21.881 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.881 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.881 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T20:58:21.881 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:33.055269+0000", 2023-09-23T20:58:21.881 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000114967 2023-09-23T20:58:21.881 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:58:21.881 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T20:58:21.882 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:58:21.882 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.882 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.882 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34797.0:130522 6.c 6:3f85e3c7:::benchmark_data_smithi026_223868_object29396:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e136)", 2023-09-23T20:58:21.882 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T20:54:30.945451+0000", 2023-09-23T20:58:21.882 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 230.815384575, 2023-09-23T20:58:21.882 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.1100408050000001, 2023-09-23T20:58:21.882 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T20:58:21.882 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:58:21.882 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T20:58:21.882 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34797", 2023-09-23T20:58:21.882 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T20:58:21.883 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 130522 2023-09-23T20:58:21.883 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.883 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T20:58:21.883 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.883 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T20:58:21.883 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:30.945451+0000", 2023-09-23T20:58:21.883 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:58:21.883 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.883 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.883 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T20:58:21.883 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:30.945451+0000", 2023-09-23T20:58:21.883 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:58:21.884 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.884 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.884 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T20:58:21.884 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:30.945449+0000", 2023-09-23T20:58:21.884 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T20:58:21.884 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.884 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.884 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T20:58:21.884 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:30.945456+0000", 2023-09-23T20:58:21.884 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.9360000000000002e-06 2023-09-23T20:58:21.884 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.884 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.885 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T20:58:21.885 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:30.945469+0000", 2023-09-23T20:58:21.885 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.3597e-05 2023-09-23T20:58:21.885 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.885 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.885 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T20:58:21.885 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:30.945490+0000", 2023-09-23T20:58:21.885 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.0840999999999999e-05 2023-09-23T20:58:21.885 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.885 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.885 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:58:21.885 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:30.945602+0000", 2023-09-23T20:58:21.886 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000112007 2023-09-23T20:58:21.886 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.886 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.886 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T20:58:21.886 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:30.945610+0000", 2023-09-23T20:58:21.886 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.9149999999999996e-06 2023-09-23T20:58:21.886 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.886 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.886 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:58:21.886 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:31.888125+0000", 2023-09-23T20:58:21.886 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.94251427099999996 2023-09-23T20:58:21.886 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.886 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.887 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T20:58:21.887 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:31.888132+0000", 2023-09-23T20:58:21.887 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.4599999999999997e-06 2023-09-23T20:58:21.887 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.887 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.887 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:58:21.887 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:33.047604+0000", 2023-09-23T20:58:21.887 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.159472442 2023-09-23T20:58:21.887 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.887 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.887 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T20:58:21.887 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:33.047716+0000", 2023-09-23T20:58:21.888 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00011191700000000001 2023-09-23T20:58:21.888 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.888 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.888 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 5,6,7", 2023-09-23T20:58:21.888 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:33.047852+0000", 2023-09-23T20:58:21.888 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000136124 2023-09-23T20:58:21.888 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.888 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.888 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:58:21.888 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:33.054959+0000", 2023-09-23T20:58:21.888 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0071068900000000003 2023-09-23T20:58:21.888 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.889 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.889 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:58:21.889 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:33.055005+0000", 2023-09-23T20:58:21.889 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.5657e-05 2023-09-23T20:58:21.889 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.889 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.889 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:58:21.889 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:33.055041+0000", 2023-09-23T20:58:21.889 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.5620000000000001e-05 2023-09-23T20:58:21.889 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.889 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.889 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T20:58:21.890 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:33.055333+0000", 2023-09-23T20:58:21.890 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00029220899999999998 2023-09-23T20:58:21.890 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.890 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.890 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T20:58:21.890 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:33.055376+0000", 2023-09-23T20:58:21.890 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.2719999999999998e-05 2023-09-23T20:58:21.890 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.890 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.890 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T20:58:21.890 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:33.055491+0000", 2023-09-23T20:58:21.890 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000115767 2023-09-23T20:58:21.891 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:58:21.891 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T20:58:21.891 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:58:21.891 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.891 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.891 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34797.0:130523 6.7 6:e5f3342f:::benchmark_data_smithi026_223868_object29397:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e136)", 2023-09-23T20:58:21.891 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T20:54:30.945619+0000", 2023-09-23T20:58:21.891 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 230.815216058, 2023-09-23T20:58:21.891 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.103502862, 2023-09-23T20:58:21.891 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T20:58:21.891 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:58:21.891 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T20:58:21.891 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34797", 2023-09-23T20:58:21.892 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T20:58:21.892 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 130523 2023-09-23T20:58:21.892 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.892 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T20:58:21.892 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.892 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T20:58:21.892 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:30.945619+0000", 2023-09-23T20:58:21.892 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:58:21.892 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.892 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.892 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T20:58:21.892 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:30.945619+0000", 2023-09-23T20:58:21.892 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:58:21.893 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.893 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.893 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T20:58:21.893 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:30.945618+0000", 2023-09-23T20:58:21.893 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T20:58:21.893 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.893 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.893 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T20:58:21.893 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:30.945623+0000", 2023-09-23T20:58:21.893 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.8640000000000004e-06 2023-09-23T20:58:21.893 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.893 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.893 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T20:58:21.894 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:30.945633+0000", 2023-09-23T20:58:21.894 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0064999999999999e-05 2023-09-23T20:58:21.894 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.894 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.894 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T20:58:21.894 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:30.945641+0000", 2023-09-23T20:58:21.894 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.5599999999999996e-06 2023-09-23T20:58:21.894 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.894 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.894 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:58:21.894 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:30.945689+0000", 2023-09-23T20:58:21.895 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.7846999999999998e-05 2023-09-23T20:58:21.895 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.895 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.895 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T20:58:21.895 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:30.945699+0000", 2023-09-23T20:58:21.895 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0176999999999999e-05 2023-09-23T20:58:21.895 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.895 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.895 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:58:21.895 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:31.884383+0000", 2023-09-23T20:58:21.895 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.93868386199999998 2023-09-23T20:58:21.895 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.895 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.896 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T20:58:21.896 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:31.884394+0000", 2023-09-23T20:58:21.896 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1119e-05 2023-09-23T20:58:21.896 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.896 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.896 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:58:21.896 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:33.034107+0000", 2023-09-23T20:58:21.896 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1497131439999999 2023-09-23T20:58:21.896 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.896 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.896 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T20:58:21.896 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:33.034256+0000", 2023-09-23T20:58:21.897 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00014860700000000001 2023-09-23T20:58:21.897 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.897 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.897 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 1,4,5", 2023-09-23T20:58:21.897 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:33.036344+0000", 2023-09-23T20:58:21.897 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.002088679 2023-09-23T20:58:21.897 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.897 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.897 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:58:21.897 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:33.042876+0000", 2023-09-23T20:58:21.897 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0065314140000000001 2023-09-23T20:58:21.897 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.898 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.898 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:58:21.898 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:33.043075+0000", 2023-09-23T20:58:21.898 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00019934600000000001 2023-09-23T20:58:21.898 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.898 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.898 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:58:21.898 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:33.046043+0000", 2023-09-23T20:58:21.898 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0029680100000000001 2023-09-23T20:58:21.898 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.898 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.898 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T20:58:21.899 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:33.048588+0000", 2023-09-23T20:58:21.899 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0025450569999999999 2023-09-23T20:58:21.899 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.899 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.899 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T20:58:21.899 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:33.048637+0000", 2023-09-23T20:58:21.899 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.8881000000000002e-05 2023-09-23T20:58:21.899 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.899 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.899 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T20:58:21.899 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:33.049122+0000", 2023-09-23T20:58:21.899 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00048494100000000002 2023-09-23T20:58:21.899 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:58:21.901 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T20:58:21.901 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:58:21.901 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.901 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.902 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34797.0:130528 6.c 6:3991ff63:::benchmark_data_smithi026_223868_object29402:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e136)", 2023-09-23T20:58:21.902 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T20:54:30.953130+0000", 2023-09-23T20:58:21.902 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 230.80770552600001, 2023-09-23T20:58:21.902 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.1058828260000002, 2023-09-23T20:58:21.902 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T20:58:21.902 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:58:21.903 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T20:58:21.903 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34797", 2023-09-23T20:58:21.903 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T20:58:21.903 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 130528 2023-09-23T20:58:21.903 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.903 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T20:58:21.903 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.903 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T20:58:21.903 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:30.953130+0000", 2023-09-23T20:58:21.903 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:58:21.903 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.903 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.903 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T20:58:21.904 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:30.953130+0000", 2023-09-23T20:58:21.904 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:58:21.904 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.904 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.904 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T20:58:21.904 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:30.953129+0000", 2023-09-23T20:58:21.904 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.999999 2023-09-23T20:58:21.904 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.904 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.904 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T20:58:21.904 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:30.953133+0000", 2023-09-23T20:58:21.904 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.5739999999999997e-06 2023-09-23T20:58:21.904 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.905 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.905 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T20:58:21.905 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:30.953142+0000", 2023-09-23T20:58:21.905 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.9309999999999997e-06 2023-09-23T20:58:21.905 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.905 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.905 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T20:58:21.905 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:30.953147+0000", 2023-09-23T20:58:21.905 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.1329999999999998e-06 2023-09-23T20:58:21.905 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.905 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.905 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:58:21.906 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:30.953195+0000", 2023-09-23T20:58:21.906 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.8015e-05 2023-09-23T20:58:21.906 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.906 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.906 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T20:58:21.906 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:30.953202+0000", 2023-09-23T20:58:21.906 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.9480000000000002e-06 2023-09-23T20:58:21.906 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.906 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.906 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:58:21.906 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:31.888161+0000", 2023-09-23T20:58:21.906 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.934958497 2023-09-23T20:58:21.906 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.907 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.907 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T20:58:21.907 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:31.888169+0000", 2023-09-23T20:58:21.907 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.4349999999999997e-06 2023-09-23T20:58:21.907 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.907 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.907 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:58:21.907 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:33.051353+0000", 2023-09-23T20:58:21.907 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.163184022 2023-09-23T20:58:21.907 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.907 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.907 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T20:58:21.907 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:33.051488+0000", 2023-09-23T20:58:21.908 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00013468200000000001 2023-09-23T20:58:21.908 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.908 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.908 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 5,6,7", 2023-09-23T20:58:21.908 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:33.051620+0000", 2023-09-23T20:58:21.908 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000131524 2023-09-23T20:58:21.908 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.908 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.908 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T20:58:21.908 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:33.056424+0000", 2023-09-23T20:58:21.908 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0048044180000000004 2023-09-23T20:58:21.908 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.908 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.909 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:58:21.909 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:33.058674+0000", 2023-09-23T20:58:21.909 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.002250326 2023-09-23T20:58:21.909 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.909 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.909 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:58:21.909 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:33.058712+0000", 2023-09-23T20:58:21.909 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.7469999999999999e-05 2023-09-23T20:58:21.909 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.909 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.909 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:58:21.909 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:33.058880+0000", 2023-09-23T20:58:21.909 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00016785699999999999 2023-09-23T20:58:21.910 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.910 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.910 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T20:58:21.910 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:33.058932+0000", 2023-09-23T20:58:21.910 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.2051e-05 2023-09-23T20:58:21.910 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.910 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.910 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T20:58:21.910 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:33.059012+0000", 2023-09-23T20:58:21.910 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.0708e-05 2023-09-23T20:58:21.910 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:58:21.910 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T20:58:21.910 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:58:21.911 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.911 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.911 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34797.0:151487 6.6 6:657607da:::benchmark_data_smithi026_223868_object50361:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e147)", 2023-09-23T20:58:21.911 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T20:55:01.023782+0000", 2023-09-23T20:58:21.911 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 200.737053436, 2023-09-23T20:58:21.911 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0492650269999999, 2023-09-23T20:58:21.911 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T20:58:21.911 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "reached pg", 2023-09-23T20:58:21.911 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T20:58:21.911 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34797", 2023-09-23T20:58:21.911 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T20:58:21.911 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 151487 2023-09-23T20:58:21.911 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.912 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T20:58:21.912 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.912 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T20:58:21.912 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:01.023782+0000", 2023-09-23T20:58:21.912 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:58:21.912 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.912 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.912 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T20:58:21.912 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:01.023782+0000", 2023-09-23T20:58:21.912 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:58:21.912 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.912 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.912 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T20:58:21.913 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:01.023781+0000", 2023-09-23T20:58:21.913 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T20:58:21.913 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.913 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.913 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T20:58:21.913 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:01.023784+0000", 2023-09-23T20:58:21.913 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.96e-06 2023-09-23T20:58:21.913 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.913 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.913 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T20:58:21.913 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:01.023790+0000", 2023-09-23T20:58:21.913 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.8259999999999998e-06 2023-09-23T20:58:21.913 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.913 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.914 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T20:58:21.914 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:01.023795+0000", 2023-09-23T20:58:21.914 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.7269999999999998e-06 2023-09-23T20:58:21.914 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.914 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.914 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:58:21.914 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:01.024359+0000", 2023-09-23T20:58:21.914 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00056414799999999999 2023-09-23T20:58:21.914 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.914 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.914 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T20:58:21.915 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:01.024503+0000", 2023-09-23T20:58:21.915 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00014404000000000001 2023-09-23T20:58:21.915 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.915 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.915 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 4,7", 2023-09-23T20:58:21.915 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:01.024955+0000", 2023-09-23T20:58:21.915 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000451736 2023-09-23T20:58:21.915 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.915 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.915 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:58:21.916 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:01.029855+0000", 2023-09-23T20:58:21.916 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0049002569999999999 2023-09-23T20:58:21.916 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.916 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.916 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T20:58:21.916 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:01.035683+0000", 2023-09-23T20:58:21.916 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0058282439999999998 2023-09-23T20:58:21.916 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.916 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.916 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:58:21.916 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:01.044471+0000", 2023-09-23T20:58:21.917 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0087875460000000002 2023-09-23T20:58:21.917 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.917 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.917 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T20:58:21.917 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:01.044492+0000", 2023-09-23T20:58:21.917 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.1117000000000001e-05 2023-09-23T20:58:21.917 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.917 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.917 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:58:21.917 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:02.054661+0000", 2023-09-23T20:58:21.917 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.010168873 2023-09-23T20:58:21.918 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.918 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.918 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for missing object", 2023-09-23T20:58:21.918 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:02.054793+0000", 2023-09-23T20:58:21.918 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00013191400000000001 2023-09-23T20:58:21.918 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.918 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.918 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:58:21.918 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:02.072945+0000", 2023-09-23T20:58:21.918 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.01815288 2023-09-23T20:58:21.918 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.918 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.918 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T20:58:21.919 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:02.073047+0000", 2023-09-23T20:58:21.919 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000101245 2023-09-23T20:58:21.919 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:58:21.919 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T20:58:21.919 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:58:21.919 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.919 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.919 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34797.0:151498 6.6 6:62751f79:::benchmark_data_smithi026_223868_object50372:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e148)", 2023-09-23T20:58:21.919 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T20:55:01.044337+0000", 2023-09-23T20:58:21.919 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 200.716498083, 2023-09-23T20:58:21.919 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0348560959999999, 2023-09-23T20:58:21.919 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T20:58:21.920 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:58:21.920 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T20:58:21.920 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34797", 2023-09-23T20:58:21.920 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T20:58:21.920 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 151498 2023-09-23T20:58:21.920 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.920 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T20:58:21.920 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.920 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T20:58:21.920 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:01.044337+0000", 2023-09-23T20:58:21.920 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:58:21.920 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.920 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.921 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T20:58:21.921 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:01.044337+0000", 2023-09-23T20:58:21.921 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:58:21.921 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.921 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.921 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T20:58:21.921 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:01.044335+0000", 2023-09-23T20:58:21.921 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999981 2023-09-23T20:58:21.921 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.921 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.921 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T20:58:21.921 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:01.044343+0000", 2023-09-23T20:58:21.921 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.6490000000000008e-06 2023-09-23T20:58:21.922 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.922 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.922 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T20:58:21.922 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:01.045000+0000", 2023-09-23T20:58:21.922 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00065786399999999997 2023-09-23T20:58:21.922 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.922 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.922 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T20:58:21.922 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:01.045010+0000", 2023-09-23T20:58:21.922 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0009e-05 2023-09-23T20:58:21.922 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.923 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.923 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:58:21.923 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:01.046191+0000", 2023-09-23T20:58:21.923 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.001180843 2023-09-23T20:58:21.923 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.923 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.923 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T20:58:21.923 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:01.046209+0000", 2023-09-23T20:58:21.923 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.7980000000000001e-05 2023-09-23T20:58:21.923 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.923 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.923 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:58:21.923 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:02.073061+0000", 2023-09-23T20:58:21.924 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0268516350000001 2023-09-23T20:58:21.924 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.924 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.924 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T20:58:21.924 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:02.073135+0000", 2023-09-23T20:58:21.924 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.4067999999999996e-05 2023-09-23T20:58:21.924 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.924 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.924 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 4,7", 2023-09-23T20:58:21.924 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:02.073539+0000", 2023-09-23T20:58:21.924 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00040385099999999999 2023-09-23T20:58:21.924 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.924 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.925 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:58:21.925 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:02.079023+0000", 2023-09-23T20:58:21.925 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00548461 2023-09-23T20:58:21.925 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.925 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.925 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T20:58:21.925 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:02.079040+0000", 2023-09-23T20:58:21.925 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.6812000000000001e-05 2023-09-23T20:58:21.925 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.925 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.925 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:58:21.925 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:02.079070+0000", 2023-09-23T20:58:21.926 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.9459e-05 2023-09-23T20:58:21.926 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.926 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.926 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T20:58:21.926 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:02.079108+0000", 2023-09-23T20:58:21.926 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.8037999999999997e-05 2023-09-23T20:58:21.926 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.926 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.926 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T20:58:21.926 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:02.079193+0000", 2023-09-23T20:58:21.926 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.5344000000000001e-05 2023-09-23T20:58:21.926 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:58:21.927 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T20:58:21.927 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:58:21.927 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.927 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.927 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34797.0:165117 6.c 6:3504fcb2:::benchmark_data_smithi026_223868_object63991:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e154)", 2023-09-23T20:58:21.927 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T20:55:17.462057+0000", 2023-09-23T20:58:21.927 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 184.29877782700001, 2023-09-23T20:58:21.927 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.091172386, 2023-09-23T20:58:21.927 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T20:58:21.927 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "reached pg", 2023-09-23T20:58:21.927 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T20:58:21.927 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34797", 2023-09-23T20:58:21.927 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T20:58:21.928 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 165117 2023-09-23T20:58:21.928 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.928 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T20:58:21.928 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.928 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T20:58:21.928 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:17.462057+0000", 2023-09-23T20:58:21.928 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:58:21.928 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.928 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.928 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T20:58:21.928 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:17.462057+0000", 2023-09-23T20:58:21.928 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:58:21.928 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.929 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.929 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T20:58:21.929 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:17.462056+0000", 2023-09-23T20:58:21.929 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999981 2023-09-23T20:58:21.929 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.929 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.929 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T20:58:21.929 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:17.462063+0000", 2023-09-23T20:58:21.929 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.7610000000000007e-06 2023-09-23T20:58:21.929 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.929 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.929 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T20:58:21.930 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:17.462077+0000", 2023-09-23T20:58:21.930 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.3570000000000001e-05 2023-09-23T20:58:21.930 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.930 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.930 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T20:58:21.930 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:17.462085+0000", 2023-09-23T20:58:21.930 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.2470000000000008e-06 2023-09-23T20:58:21.930 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.930 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.930 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:58:21.930 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:17.462146+0000", 2023-09-23T20:58:21.931 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.1288e-05 2023-09-23T20:58:21.931 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.931 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.931 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T20:58:21.931 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:17.462301+0000", 2023-09-23T20:58:21.931 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000154971 2023-09-23T20:58:21.931 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.931 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.931 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 5,6,7", 2023-09-23T20:58:21.931 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:17.462508+0000", 2023-09-23T20:58:21.931 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000206691 2023-09-23T20:58:21.931 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.931 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.932 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T20:58:21.932 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:17.467494+0000", 2023-09-23T20:58:21.932 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0049860490000000002 2023-09-23T20:58:21.932 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.932 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.932 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:58:21.932 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:17.473605+0000", 2023-09-23T20:58:21.932 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0061112429999999997 2023-09-23T20:58:21.932 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.932 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.932 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:58:21.932 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:17.486250+0000", 2023-09-23T20:58:21.932 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.012644931999999999 2023-09-23T20:58:21.933 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.933 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.933 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T20:58:21.933 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:17.486261+0000", 2023-09-23T20:58:21.933 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0638999999999999e-05 2023-09-23T20:58:21.933 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.933 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.933 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:58:21.933 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:18.548375+0000", 2023-09-23T20:58:21.933 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0621140440000001 2023-09-23T20:58:21.933 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.933 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.934 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for missing object", 2023-09-23T20:58:21.934 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:18.548503+0000", 2023-09-23T20:58:21.934 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000127668 2023-09-23T20:58:21.934 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.934 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.934 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:58:21.934 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:18.553178+0000", 2023-09-23T20:58:21.934 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.004675289 2023-09-23T20:58:21.934 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.934 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.935 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T20:58:21.935 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:18.553230+0000", 2023-09-23T20:58:21.935 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.1743999999999998e-05 2023-09-23T20:58:21.935 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:58:21.935 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T20:58:21.935 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:58:21.935 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.935 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.935 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34797.0:177908 6.c 6:3bd0ad3d:::benchmark_data_smithi026_223868_object76782:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e164)", 2023-09-23T20:58:21.935 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T20:55:34.934464+0000", 2023-09-23T20:58:21.935 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 166.826371534, 2023-09-23T20:58:21.935 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.0970397529999998, 2023-09-23T20:58:21.936 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T20:58:21.936 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:58:21.936 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T20:58:21.936 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34797", 2023-09-23T20:58:21.936 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T20:58:21.936 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 177908 2023-09-23T20:58:21.936 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.936 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T20:58:21.936 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.936 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T20:58:21.936 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:34.934464+0000", 2023-09-23T20:58:21.936 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:58:21.936 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.937 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.937 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T20:58:21.937 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:34.934464+0000", 2023-09-23T20:58:21.937 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:58:21.937 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.937 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.937 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T20:58:21.937 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:34.934463+0000", 2023-09-23T20:58:21.937 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T20:58:21.937 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.937 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.938 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T20:58:21.938 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:34.934467+0000", 2023-09-23T20:58:21.938 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.4979999999999998e-06 2023-09-23T20:58:21.938 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.938 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.938 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T20:58:21.938 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:34.934487+0000", 2023-09-23T20:58:21.938 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.0602e-05 2023-09-23T20:58:21.938 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.938 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.938 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T20:58:21.939 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:34.934494+0000", 2023-09-23T20:58:21.939 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.4010000000000002e-06 2023-09-23T20:58:21.939 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.939 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.939 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:58:21.939 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:34.950588+0000", 2023-09-23T20:58:21.939 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.016094285 2023-09-23T20:58:21.939 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.939 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.939 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T20:58:21.939 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:34.950600+0000", 2023-09-23T20:58:21.940 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.2224e-05 2023-09-23T20:58:21.940 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.940 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.940 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:58:21.940 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:35.937538+0000", 2023-09-23T20:58:21.940 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.98693769200000003 2023-09-23T20:58:21.940 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.940 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.940 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T20:58:21.940 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:35.937552+0000", 2023-09-23T20:58:21.940 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.3811e-05 2023-09-23T20:58:21.940 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.940 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.941 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:58:21.941 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:37.017091+0000", 2023-09-23T20:58:21.941 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.079539322 2023-09-23T20:58:21.941 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.941 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.941 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T20:58:21.941 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:37.017258+0000", 2023-09-23T20:58:21.941 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00016676500000000001 2023-09-23T20:58:21.941 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.941 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.941 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 5,6", 2023-09-23T20:58:21.941 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:37.017367+0000", 2023-09-23T20:58:21.941 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000108909 2023-09-23T20:58:21.941 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.942 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.942 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T20:58:21.942 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:37.027299+0000", 2023-09-23T20:58:21.942 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0099326350000000004 2023-09-23T20:58:21.942 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.942 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.942 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:58:21.942 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:37.028919+0000", 2023-09-23T20:58:21.942 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.001619756 2023-09-23T20:58:21.942 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.942 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.943 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:58:21.943 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:37.031159+0000", 2023-09-23T20:58:21.943 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0022400330000000002 2023-09-23T20:58:21.943 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.943 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.943 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T20:58:21.943 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:37.031241+0000", 2023-09-23T20:58:21.943 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.2249000000000005e-05 2023-09-23T20:58:21.943 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.943 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.943 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T20:58:21.943 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:37.031503+0000", 2023-09-23T20:58:21.944 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00026212400000000003 2023-09-23T20:58:21.944 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:58:21.944 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T20:58:21.944 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:58:21.944 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.944 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.944 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34797.0:177953 6.c 6:3867b4fa:::benchmark_data_smithi026_223868_object76827:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e164)", 2023-09-23T20:58:21.944 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T20:55:35.005631+0000", 2023-09-23T20:58:21.944 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 166.75520444399999, 2023-09-23T20:58:21.944 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.0261709309999998, 2023-09-23T20:58:21.944 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T20:58:21.944 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:58:21.944 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T20:58:21.945 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34797", 2023-09-23T20:58:21.945 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T20:58:21.945 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 177953 2023-09-23T20:58:21.945 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.945 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T20:58:21.945 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.945 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T20:58:21.945 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:35.005631+0000", 2023-09-23T20:58:21.945 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:58:21.945 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.945 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.945 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T20:58:21.945 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:35.005631+0000", 2023-09-23T20:58:21.946 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:58:21.946 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.946 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.946 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T20:58:21.946 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:35.005629+0000", 2023-09-23T20:58:21.946 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T20:58:21.946 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.946 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.946 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T20:58:21.946 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:35.005635+0000", 2023-09-23T20:58:21.946 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.0809999999999999e-06 2023-09-23T20:58:21.946 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.947 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.947 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T20:58:21.947 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:35.005651+0000", 2023-09-23T20:58:21.947 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.5191000000000001e-05 2023-09-23T20:58:21.947 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.947 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.947 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T20:58:21.947 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:35.005659+0000", 2023-09-23T20:58:21.947 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.6999999999999997e-06 2023-09-23T20:58:21.947 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.947 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.947 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:58:21.948 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:35.005721+0000", 2023-09-23T20:58:21.948 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.1610000000000001e-05 2023-09-23T20:58:21.948 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.948 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.948 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T20:58:21.948 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:35.005734+0000", 2023-09-23T20:58:21.948 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.3137e-05 2023-09-23T20:58:21.948 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.948 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.948 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:58:21.948 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:35.937576+0000", 2023-09-23T20:58:21.948 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.931841582 2023-09-23T20:58:21.949 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.949 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.949 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T20:58:21.949 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:35.937584+0000", 2023-09-23T20:58:21.949 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.123e-06 2023-09-23T20:58:21.949 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.949 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.949 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:58:21.949 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:37.020962+0000", 2023-09-23T20:58:21.949 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.083378282 2023-09-23T20:58:21.949 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.950 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.950 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T20:58:21.950 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:37.021075+0000", 2023-09-23T20:58:21.950 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000112896 2023-09-23T20:58:21.950 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.950 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.950 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 5,6", 2023-09-23T20:58:21.950 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:37.021209+0000", 2023-09-23T20:58:21.950 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00013392100000000001 2023-09-23T20:58:21.950 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.950 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.950 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T20:58:21.951 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:37.027309+0000", 2023-09-23T20:58:21.951 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.006100482 2023-09-23T20:58:21.951 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.951 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.951 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:58:21.951 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:37.028967+0000", 2023-09-23T20:58:21.951 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0016573969999999999 2023-09-23T20:58:21.951 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.951 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.951 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:58:21.951 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:37.031581+0000", 2023-09-23T20:58:21.951 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0026142130000000002 2023-09-23T20:58:21.952 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.952 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.952 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T20:58:21.952 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:37.031645+0000", 2023-09-23T20:58:21.952 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.3997000000000004e-05 2023-09-23T20:58:21.952 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:58:21.952 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:58:21.952 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T20:58:21.952 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:37.031802+0000", 2023-09-23T20:58:21.952 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00015662 2023-09-23T20:58:21.952 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:58:21.953 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T20:58:21.953 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:58:21.953 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:58:21.953 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T20:58:21.953 INFO:teuthology.orchestra.run.smithi026.stdout:} 2023-09-23T20:58:21.953 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:58:21.954 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:58:21.954 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:58:21.954 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:58:21.961 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0: {} 2023-09-23T20:58:21.997 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1: {} 2023-09-23T20:58:22.040 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2: {} 2023-09-23T20:58:22.055 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:58:22.082 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3: {} 2023-09-23T20:58:22.106 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T20:58:22.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:58:22 smithi026 ceph-mon[179688]: pgmap v549: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 5.9 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 549 op/s; 10943/289224 objects degraded (3.784%); 4771/289224 objects misplaced (1.650%); 887 KiB/s, 13 objects/s recovering 2023-09-23T20:58:22.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:58:22 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/596036064' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:58:22.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:58:22 smithi026 ceph-mon[175151]: pgmap v549: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 5.9 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 549 op/s; 10943/289224 objects degraded (3.784%); 4771/289224 objects misplaced (1.650%); 887 KiB/s, 13 objects/s recovering 2023-09-23T20:58:22.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:58:22 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/596036064' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:58:22.143 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4: {} 2023-09-23T20:58:22.154 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:58:22.199 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T20:58:22.238 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6: {} 2023-09-23T20:58:22.250 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:58:22.253 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:58:22.254 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:58:22.257 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:58:22.278 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T20:58:22.289 INFO:teuthology.orchestra.run.smithi026.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T20:58:22.289 INFO:teuthology.orchestra.run.smithi026.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T20:58:22.289 INFO:teuthology.orchestra.run.smithi026.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T20:58:22.289 INFO:teuthology.orchestra.run.smithi026.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T20:58:22.289 INFO:teuthology.orchestra.run.smithi026.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T20:58:22.289 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T20:58:22.289 INFO:teuthology.orchestra.run.smithi026.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T20:58:22.289 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T20:58:22.323 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_ops_in_flight 2023-09-23T20:58:22.349 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:58:22 smithi070 ceph-mon[168282]: pgmap v549: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 5.9 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 549 op/s; 10943/289224 objects degraded (3.784%); 4771/289224 objects misplaced (1.650%); 887 KiB/s, 13 objects/s recovering 2023-09-23T20:58:22.349 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:58:22 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/596036064' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:58:22.354 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:58:22.505 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:58:22.718 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:58:22.729 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:58:22.730 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:58:22.831 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:58:22.900 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:58:22.975 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:58:22.982 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:58:22.986 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:58:22.986 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:58:23.087 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:58:23.124 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:58:23.220 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:58:23.223 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:58:23.224 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:58:23.324 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:58:23.418 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:58:23.616 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:58:23.616 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:58:23.625 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:58:23.626 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:58:23.727 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:58:23.836 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:58:23.917 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:58:23.929 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:58:23.929 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:58:24.030 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:58:24.136 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:58:24.199 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:58:24 smithi026 ceph-mon[179688]: pgmap v550: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 5.8 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 660 op/s; 10943/284124 objects degraded (3.851%); 4671/284124 objects misplaced (1.644%); 1.6 MiB/s, 26 objects/s recovering 2023-09-23T20:58:24.199 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:58:24 smithi026 ceph-mon[175151]: pgmap v550: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 5.8 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 660 op/s; 10943/284124 objects degraded (3.851%); 4671/284124 objects misplaced (1.644%); 1.6 MiB/s, 26 objects/s recovering 2023-09-23T20:58:24.308 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:58:24 smithi070 ceph-mon[168282]: pgmap v550: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 5.8 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 660 op/s; 10943/284124 objects degraded (3.851%); 4671/284124 objects misplaced (1.644%); 1.6 MiB/s, 26 objects/s recovering 2023-09-23T20:58:24.541 INFO:teuthology.orchestra.run.smithi070.stdout:ERROR: (22) Invalid argument 2023-09-23T20:58:24.541 INFO:teuthology.orchestra.run.smithi070.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:58:24.622 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:58:24.633 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:58:24.634 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:58:24.734 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:58:24.770 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T20:58:24.770 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:58:24.921 INFO:teuthology.orchestra.run.smithi026.stdout: 2023-09-23T20:58:24.923 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:58:24.928 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:58:24.928 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:58:24.955 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_blocked_ops 2023-09-23T20:58:24.958 INFO:teuthology.orchestra.run.smithi026.stderr:dumped all 2023-09-23T20:58:25.029 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:58:25.069 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:58:25.114 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:58:25.118 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:58:25.118 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:58:25.185 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:58:25 smithi026 ceph-mon[175151]: Health check update: Degraded data redundancy: 10943/284124 objects degraded (3.851%), 2 pgs degraded (PG_DEGRADED) 2023-09-23T20:58:25.185 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:58:25 smithi026 ceph-mon[179688]: Health check update: Degraded data redundancy: 10943/284124 objects degraded (3.851%), 2 pgs degraded (PG_DEGRADED) 2023-09-23T20:58:25.219 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:58:25.312 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:58:25.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:58:25 smithi070 ceph-mon[168282]: Health check update: Degraded data redundancy: 10943/284124 objects degraded (3.851%), 2 pgs degraded (PG_DEGRADED) 2023-09-23T20:58:25.666 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:58:25.680 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:58:25.688 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:58:25.689 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:58:25.706 INFO:teuthology.orchestra.run.smithi026.stdout:{"pg_ready":true,"pg_map":{"version":551,"stamp":"2023-09-23T20:58:24.380105+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":6152198072,"num_objects":93891,"num_object_clones":0,"num_object_copies":281673,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":10943,"num_objects_misplaced":4671,"num_objects_unfound":0,"num_objects_dirty":93891,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":109714,"num_write_kb":6512606,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":750,"num_bytes_recovered":47716198,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":111289,"ondisk_log_size":111289,"up":327,"acting":324,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":331,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":19661856,"kb_used_data":18188576,"kb_used_omap":136,"kb_used_meta":1472951,"kb_avail":730299360,"statfs":{"total":767960285184,"available":747826544640,"internally_reserved":0,"allocated":18625101824,"data_stored":18617382375,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":139741,"internal_metadata":1508302371},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[3,1,1,1,8,5,3],"upper_bound":128},"perf_stat":{"commit_latency_ms":10,"apply_latency_ms":10,"commit_latency_ns":10000000,"apply_latency_ns":10000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-360906752,"num_objects":-5507,"num_object_clones":0,"num_object_copies":-16521,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":-211,"num_objects_unfound":0,"num_objects_dirty":-5507,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5507,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":211,"num_bytes_recovered":13828096,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"9.269551"},"pg_stats":[{"pgid":"3.1a","version":"68'1","reported_seq":110,"reported_epoch":232,"state":"active+clean","last_fresh":"2023-09-23T20:58:15.117346+0000","last_change":"2023-09-23T20:54:13.946695+0000","last_active":"2023-09-23T20:58:15.117346+0000","last_peered":"2023-09-23T20:58:15.117346+0000","last_clean":"2023-09-23T20:58:15.117346+0000","last_became_active":"2023-09-23T20:54:13.945399+0000","last_became_peered":"2023-09-23T20:54:13.945399+0000","last_unstale":"2023-09-23T20:58:15.117346+0000","last_undegraded":"2023-09-23T20:58:15.117346+0000","last_fullsized":"2023-09-23T20:58:15.117346+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:32:52.365464+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":223,"reported_epoch":231,"state":"active+clean","last_fresh":"2023-09-23T20:58:11.109618+0000","last_change":"2023-09-23T20:54:13.942019+0000","last_active":"2023-09-23T20:58:11.109618+0000","last_peered":"2023-09-23T20:58:11.109618+0000","last_clean":"2023-09-23T20:58:11.109618+0000","last_became_active":"2023-09-23T20:54:13.941647+0000","last_became_peered":"2023-09-23T20:54:13.941647+0000","last_unstale":"2023-09-23T20:58:11.109618+0000","last_undegraded":"2023-09-23T20:58:11.109618+0000","last_fullsized":"2023-09-23T20:58:11.109618+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:30:54.400944+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":202,"reported_epoch":232,"state":"active+clean","last_fresh":"2023-09-23T20:58:15.132178+0000","last_change":"2023-09-23T20:54:13.941394+0000","last_active":"2023-09-23T20:58:15.132178+0000","last_peered":"2023-09-23T20:58:15.132178+0000","last_clean":"2023-09-23T20:58:15.132178+0000","last_became_active":"2023-09-23T20:54:13.941042+0000","last_became_peered":"2023-09-23T20:54:13.941042+0000","last_unstale":"2023-09-23T20:58:15.132178+0000","last_undegraded":"2023-09-23T20:58:15.132178+0000","last_fullsized":"2023-09-23T20:58:15.132178+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:19:41.401184+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":217,"reported_epoch":232,"state":"active+clean","last_fresh":"2023-09-23T20:58:15.136875+0000","last_change":"2023-09-23T20:57:44.979811+0000","last_active":"2023-09-23T20:58:15.136875+0000","last_peered":"2023-09-23T20:58:15.136875+0000","last_clean":"2023-09-23T20:58:15.136875+0000","last_became_active":"2023-09-23T20:57:44.979515+0000","last_became_peered":"2023-09-23T20:57:44.979515+0000","last_unstale":"2023-09-23T20:58:15.136875+0000","last_undegraded":"2023-09-23T20:58:15.136875+0000","last_fullsized":"2023-09-23T20:58:15.136875+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:38:22.162140+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":214,"reported_epoch":231,"state":"active+clean","last_fresh":"2023-09-23T20:58:11.088098+0000","last_change":"2023-09-23T20:51:10.833613+0000","last_active":"2023-09-23T20:58:11.088098+0000","last_peered":"2023-09-23T20:58:11.088098+0000","last_clean":"2023-09-23T20:58:11.088098+0000","last_became_active":"2023-09-23T20:51:10.833208+0000","last_became_peered":"2023-09-23T20:51:10.833208+0000","last_unstale":"2023-09-23T20:58:11.088098+0000","last_undegraded":"2023-09-23T20:58:11.088098+0000","last_fullsized":"2023-09-23T20:58:11.088098+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:51:28.246014+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":198,"reported_epoch":232,"state":"active+clean","last_fresh":"2023-09-23T20:58:15.116229+0000","last_change":"2023-09-23T20:55:35.958384+0000","last_active":"2023-09-23T20:58:15.116229+0000","last_peered":"2023-09-23T20:58:15.116229+0000","last_clean":"2023-09-23T20:58:15.116229+0000","last_became_active":"2023-09-23T20:55:35.954563+0000","last_became_peered":"2023-09-23T20:55:35.954563+0000","last_unstale":"2023-09-23T20:58:15.116229+0000","last_undegraded":"2023-09-23T20:58:15.116229+0000","last_fullsized":"2023-09-23T20:58:15.116229+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:37:48.332259+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":207,"reported_epoch":232,"state":"active+clean","last_fresh":"2023-09-23T20:58:15.145781+0000","last_change":"2023-09-23T20:54:31.885514+0000","last_active":"2023-09-23T20:58:15.145781+0000","last_peered":"2023-09-23T20:58:15.145781+0000","last_clean":"2023-09-23T20:58:15.145781+0000","last_became_active":"2023-09-23T20:54:31.884986+0000","last_became_peered":"2023-09-23T20:54:31.884986+0000","last_unstale":"2023-09-23T20:58:15.145781+0000","last_undegraded":"2023-09-23T20:58:15.145781+0000","last_fullsized":"2023-09-23T20:58:15.145781+0000","mapping_epoch":136,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:26:08.100220+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":212,"reported_epoch":231,"state":"active+clean","last_fresh":"2023-09-23T20:58:11.088020+0000","last_change":"2023-09-23T20:55:35.951889+0000","last_active":"2023-09-23T20:58:11.088020+0000","last_peered":"2023-09-23T20:58:11.088020+0000","last_clean":"2023-09-23T20:58:11.088020+0000","last_became_active":"2023-09-23T20:55:35.951563+0000","last_became_peered":"2023-09-23T20:55:35.951563+0000","last_unstale":"2023-09-23T20:58:11.088020+0000","last_undegraded":"2023-09-23T20:58:11.088020+0000","last_fullsized":"2023-09-23T20:58:11.088020+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:41:53.098781+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"79'161","reported_seq":519,"reported_epoch":232,"state":"active+clean","last_fresh":"2023-09-23T20:58:15.124121+0000","last_change":"2023-09-23T20:57:25.096559+0000","last_active":"2023-09-23T20:58:15.124121+0000","last_peered":"2023-09-23T20:58:15.124121+0000","last_clean":"2023-09-23T20:58:15.124121+0000","last_became_active":"2023-09-23T20:57:15.975674+0000","last_became_peered":"2023-09-23T20:57:15.975674+0000","last_unstale":"2023-09-23T20:58:15.124121+0000","last_undegraded":"2023-09-23T20:58:15.124121+0000","last_fullsized":"2023-09-23T20:58:15.124121+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:25.096401+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:25.096401+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:05:05.448404+0000","scrub_duration":0.002859393,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":106,"reported_epoch":232,"state":"active+clean","last_fresh":"2023-09-23T20:58:15.116961+0000","last_change":"2023-09-23T20:54:31.890678+0000","last_active":"2023-09-23T20:58:15.116961+0000","last_peered":"2023-09-23T20:58:15.116961+0000","last_clean":"2023-09-23T20:58:15.116961+0000","last_became_active":"2023-09-23T20:54:31.890469+0000","last_became_peered":"2023-09-23T20:54:31.890469+0000","last_unstale":"2023-09-23T20:58:15.116961+0000","last_undegraded":"2023-09-23T20:58:15.116961+0000","last_fullsized":"2023-09-23T20:58:15.116961+0000","mapping_epoch":136,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:55:08.790146+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":173,"reported_epoch":231,"state":"active+clean","last_fresh":"2023-09-23T20:58:11.088206+0000","last_change":"2023-09-23T20:55:35.952661+0000","last_active":"2023-09-23T20:58:11.088206+0000","last_peered":"2023-09-23T20:58:11.088206+0000","last_clean":"2023-09-23T20:58:11.088206+0000","last_became_active":"2023-09-23T20:55:35.952349+0000","last_became_peered":"2023-09-23T20:55:35.952349+0000","last_unstale":"2023-09-23T20:58:11.088206+0000","last_undegraded":"2023-09-23T20:58:11.088206+0000","last_fullsized":"2023-09-23T20:58:11.088206+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:00:03.110126+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":49,"reported_epoch":231,"state":"active+clean","last_fresh":"2023-09-23T20:58:11.109876+0000","last_change":"2023-09-23T20:56:45.988607+0000","last_active":"2023-09-23T20:58:11.109876+0000","last_peered":"2023-09-23T20:58:11.109876+0000","last_clean":"2023-09-23T20:58:11.109876+0000","last_became_active":"2023-09-23T20:56:45.988206+0000","last_became_peered":"2023-09-23T20:56:45.988206+0000","last_unstale":"2023-09-23T20:58:11.109876+0000","last_undegraded":"2023-09-23T20:58:11.109876+0000","last_fullsized":"2023-09-23T20:58:11.109876+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":205,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:01:19.783688+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,4],"acting":[3,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":50,"reported_epoch":232,"state":"active+clean","last_fresh":"2023-09-23T20:58:15.116878+0000","last_change":"2023-09-23T20:56:18.139142+0000","last_active":"2023-09-23T20:58:15.116878+0000","last_peered":"2023-09-23T20:58:15.116878+0000","last_clean":"2023-09-23T20:58:15.116878+0000","last_became_active":"2023-09-23T20:56:18.138844+0000","last_became_peered":"2023-09-23T20:56:18.138844+0000","last_unstale":"2023-09-23T20:58:15.116878+0000","last_undegraded":"2023-09-23T20:58:15.116878+0000","last_fullsized":"2023-09-23T20:58:15.116878+0000","mapping_epoch":192,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":193,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:09:02.217683+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,2],"acting":[5,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.14","version":"79'22","reported_seq":137,"reported_epoch":232,"state":"active+clean","last_fresh":"2023-09-23T20:58:15.116903+0000","last_change":"2023-09-23T20:55:35.949418+0000","last_active":"2023-09-23T20:58:15.116903+0000","last_peered":"2023-09-23T20:58:15.116903+0000","last_clean":"2023-09-23T20:58:15.116903+0000","last_became_active":"2023-09-23T20:55:35.948566+0000","last_became_peered":"2023-09-23T20:55:35.948566+0000","last_unstale":"2023-09-23T20:58:15.116903+0000","last_undegraded":"2023-09-23T20:58:15.116903+0000","last_fullsized":"2023-09-23T20:58:15.116903+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:59:37.433128+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4734,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":127,"reported_epoch":232,"state":"active+clean","last_fresh":"2023-09-23T20:58:15.117837+0000","last_change":"2023-09-23T20:55:35.949178+0000","last_active":"2023-09-23T20:58:15.117837+0000","last_peered":"2023-09-23T20:58:15.117837+0000","last_clean":"2023-09-23T20:58:15.117837+0000","last_became_active":"2023-09-23T20:55:35.948795+0000","last_became_peered":"2023-09-23T20:55:35.948795+0000","last_unstale":"2023-09-23T20:58:15.117837+0000","last_undegraded":"2023-09-23T20:58:15.117837+0000","last_fullsized":"2023-09-23T20:58:15.117837+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:13:08.996932+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.11","version":"79'161","reported_seq":461,"reported_epoch":232,"state":"active+clean","last_fresh":"2023-09-23T20:58:15.134374+0000","last_change":"2023-09-23T20:57:28.756796+0000","last_active":"2023-09-23T20:58:15.134374+0000","last_peered":"2023-09-23T20:58:15.134374+0000","last_clean":"2023-09-23T20:58:15.134374+0000","last_became_active":"2023-09-23T20:57:15.980606+0000","last_became_peered":"2023-09-23T20:57:15.980606+0000","last_unstale":"2023-09-23T20:58:15.134374+0000","last_undegraded":"2023-09-23T20:58:15.134374+0000","last_fullsized":"2023-09-23T20:58:15.134374+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:28.756634+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:28.756634+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:09:19.150791+0000","scrub_duration":0.0064925119999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":153,"reported_epoch":232,"state":"active+clean","last_fresh":"2023-09-23T20:58:15.117761+0000","last_change":"2023-09-23T20:56:45.992577+0000","last_active":"2023-09-23T20:58:15.117761+0000","last_peered":"2023-09-23T20:58:15.117761+0000","last_clean":"2023-09-23T20:58:15.117761+0000","last_became_active":"2023-09-23T20:56:45.992297+0000","last_became_peered":"2023-09-23T20:56:45.992297+0000","last_unstale":"2023-09-23T20:58:15.117761+0000","last_undegraded":"2023-09-23T20:58:15.117761+0000","last_fullsized":"2023-09-23T20:58:15.117761+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":205,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:00:45.547202+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,2],"acting":[4,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":197,"reported_epoch":231,"state":"active+clean","last_fresh":"2023-09-23T20:58:11.100257+0000","last_change":"2023-09-23T20:51:10.831262+0000","last_active":"2023-09-23T20:58:11.100257+0000","last_peered":"2023-09-23T20:58:11.100257+0000","last_clean":"2023-09-23T20:58:11.100257+0000","last_became_active":"2023-09-23T20:51:10.830931+0000","last_became_peered":"2023-09-23T20:51:10.830931+0000","last_unstale":"2023-09-23T20:58:11.100257+0000","last_undegraded":"2023-09-23T20:58:11.100257+0000","last_fullsized":"2023-09-23T20:58:11.100257+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:51:54.982505+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.10","version":"79'161","reported_seq":470,"reported_epoch":232,"state":"active+clean","last_fresh":"2023-09-23T20:58:15.115610+0000","last_change":"2023-09-23T20:57:20.708003+0000","last_active":"2023-09-23T20:58:15.115610+0000","last_peered":"2023-09-23T20:58:15.115610+0000","last_clean":"2023-09-23T20:58:15.115610+0000","last_became_active":"2023-09-23T20:57:15.975302+0000","last_became_peered":"2023-09-23T20:57:15.975302+0000","last_unstale":"2023-09-23T20:58:15.115610+0000","last_undegraded":"2023-09-23T20:58:15.115610+0000","last_fullsized":"2023-09-23T20:58:15.115610+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:20.707939+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:20.707939+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:16:51.721994+0000","scrub_duration":0.0017733409999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":207,"reported_epoch":232,"state":"active+clean","last_fresh":"2023-09-23T20:58:15.134965+0000","last_change":"2023-09-23T20:57:44.979736+0000","last_active":"2023-09-23T20:58:15.134965+0000","last_peered":"2023-09-23T20:58:15.134965+0000","last_clean":"2023-09-23T20:58:15.134965+0000","last_became_active":"2023-09-23T20:57:44.979356+0000","last_became_peered":"2023-09-23T20:57:44.979356+0000","last_unstale":"2023-09-23T20:58:15.134965+0000","last_undegraded":"2023-09-23T20:58:15.134965+0000","last_fullsized":"2023-09-23T20:58:15.134965+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:14:43.848832+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":106,"reported_epoch":232,"state":"active+clean","last_fresh":"2023-09-23T20:58:15.116165+0000","last_change":"2023-09-23T20:54:31.887814+0000","last_active":"2023-09-23T20:58:15.116165+0000","last_peered":"2023-09-23T20:58:15.116165+0000","last_clean":"2023-09-23T20:58:15.116165+0000","last_became_active":"2023-09-23T20:54:31.887195+0000","last_became_peered":"2023-09-23T20:54:31.887195+0000","last_unstale":"2023-09-23T20:58:15.116165+0000","last_undegraded":"2023-09-23T20:58:15.116165+0000","last_fullsized":"2023-09-23T20:58:15.116165+0000","mapping_epoch":136,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:10:52.089553+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.13","version":"79'161","reported_seq":532,"reported_epoch":232,"state":"active+clean","last_fresh":"2023-09-23T20:58:15.132266+0000","last_change":"2023-09-23T20:57:48.262970+0000","last_active":"2023-09-23T20:58:15.132266+0000","last_peered":"2023-09-23T20:58:15.132266+0000","last_clean":"2023-09-23T20:58:15.132266+0000","last_became_active":"2023-09-23T20:57:48.262663+0000","last_became_peered":"2023-09-23T20:57:48.262663+0000","last_unstale":"2023-09-23T20:58:15.132266+0000","last_undegraded":"2023-09-23T20:58:15.132266+0000","last_fullsized":"2023-09-23T20:58:15.132266+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:36.420362+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:36.420362+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:28:28.613492+0000","scrub_duration":0.028838444000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":177,"reported_epoch":232,"state":"active+clean","last_fresh":"2023-09-23T20:58:15.115990+0000","last_change":"2023-09-23T20:56:45.991822+0000","last_active":"2023-09-23T20:58:15.115990+0000","last_peered":"2023-09-23T20:58:15.115990+0000","last_clean":"2023-09-23T20:58:15.115990+0000","last_became_active":"2023-09-23T20:56:45.991460+0000","last_became_peered":"2023-09-23T20:56:45.991460+0000","last_unstale":"2023-09-23T20:58:15.115990+0000","last_undegraded":"2023-09-23T20:58:15.115990+0000","last_fullsized":"2023-09-23T20:58:15.115990+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":205,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:20:37.503317+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":189,"reported_epoch":232,"state":"active+clean","last_fresh":"2023-09-23T20:58:15.137002+0000","last_change":"2023-09-23T20:51:10.840849+0000","last_active":"2023-09-23T20:58:15.137002+0000","last_peered":"2023-09-23T20:58:15.137002+0000","last_clean":"2023-09-23T20:58:15.137002+0000","last_became_active":"2023-09-23T20:51:10.840593+0000","last_became_peered":"2023-09-23T20:51:10.840593+0000","last_unstale":"2023-09-23T20:58:15.137002+0000","last_undegraded":"2023-09-23T20:58:15.137002+0000","last_fullsized":"2023-09-23T20:58:15.137002+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:30:52.297625+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.12","version":"79'161","reported_seq":470,"reported_epoch":232,"state":"active+clean","last_fresh":"2023-09-23T20:58:15.116682+0000","last_change":"2023-09-23T20:57:21.750765+0000","last_active":"2023-09-23T20:58:15.116682+0000","last_peered":"2023-09-23T20:58:15.116682+0000","last_clean":"2023-09-23T20:58:15.116682+0000","last_became_active":"2023-09-23T20:57:15.979195+0000","last_became_peered":"2023-09-23T20:57:15.979195+0000","last_unstale":"2023-09-23T20:58:15.116682+0000","last_undegraded":"2023-09-23T20:58:15.116682+0000","last_fullsized":"2023-09-23T20:58:15.116682+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:21.750680+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:21.750680+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:03:41.915962+0000","scrub_duration":0.0092391680000000007,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":225,"reported_epoch":232,"state":"active+clean","last_fresh":"2023-09-23T20:58:15.115180+0000","last_change":"2023-09-23T20:55:36.943969+0000","last_active":"2023-09-23T20:58:15.115180+0000","last_peered":"2023-09-23T20:58:15.115180+0000","last_clean":"2023-09-23T20:58:15.115180+0000","last_became_active":"2023-09-23T20:55:36.943580+0000","last_became_peered":"2023-09-23T20:55:36.943580+0000","last_unstale":"2023-09-23T20:58:15.115180+0000","last_undegraded":"2023-09-23T20:58:15.115180+0000","last_fullsized":"2023-09-23T20:58:15.115180+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:49:06.969396+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":20,"reported_epoch":231,"state":"active+clean","last_fresh":"2023-09-23T20:58:11.111710+0000","last_change":"2023-09-23T20:57:44.980498+0000","last_active":"2023-09-23T20:58:11.111710+0000","last_peered":"2023-09-23T20:58:11.111710+0000","last_clean":"2023-09-23T20:58:11.111710+0000","last_became_active":"2023-09-23T20:57:44.980182+0000","last_became_peered":"2023-09-23T20:57:44.980182+0000","last_unstale":"2023-09-23T20:58:11.111710+0000","last_undegraded":"2023-09-23T20:58:11.111710+0000","last_fullsized":"2023-09-23T20:58:11.111710+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:34:51.620544+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.d","version":"79'161","reported_seq":515,"reported_epoch":232,"state":"active+clean","last_fresh":"2023-09-23T20:58:15.116149+0000","last_change":"2023-09-23T20:57:31.075213+0000","last_active":"2023-09-23T20:58:15.116149+0000","last_peered":"2023-09-23T20:58:15.116149+0000","last_clean":"2023-09-23T20:58:15.116149+0000","last_became_active":"2023-09-23T20:57:15.988188+0000","last_became_peered":"2023-09-23T20:57:15.988188+0000","last_unstale":"2023-09-23T20:58:15.116149+0000","last_undegraded":"2023-09-23T20:58:15.116149+0000","last_fullsized":"2023-09-23T20:58:15.116149+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:31.075098+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:31.075098+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:41:52.142362+0000","scrub_duration":0.010304610000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.b","version":"233'6891","reported_seq":6934,"reported_epoch":233,"state":"active+clean","last_fresh":"2023-09-23T20:58:24.236233+0000","last_change":"2023-09-23T20:56:35.543994+0000","last_active":"2023-09-23T20:58:24.236233+0000","last_peered":"2023-09-23T20:58:24.236233+0000","last_clean":"2023-09-23T20:58:24.236233+0000","last_became_active":"2023-09-23T20:56:35.543758+0000","last_became_peered":"2023-09-23T20:56:35.543758+0000","last_unstale":"2023-09-23T20:58:24.236233+0000","last_undegraded":"2023-09-23T20:58:24.236233+0000","last_fullsized":"2023-09-23T20:58:24.236233+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":199,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_clean_scrub_stamp":"2023-09-23T20:56:34.529103+0000","objects_scrubbed":0,"log_size":6891,"log_dups_size":0,"ondisk_log_size":6891,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:21:29.041484+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":379387904,"num_objects":5789,"num_object_clones":0,"num_object_copies":17367,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5789,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6891,"num_write_kb":405760,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":164,"reported_epoch":232,"state":"active+clean","last_fresh":"2023-09-23T20:58:15.117148+0000","last_change":"2023-09-23T20:56:45.991841+0000","last_active":"2023-09-23T20:58:15.117148+0000","last_peered":"2023-09-23T20:58:15.117148+0000","last_clean":"2023-09-23T20:58:15.117148+0000","last_became_active":"2023-09-23T20:56:45.991491+0000","last_became_peered":"2023-09-23T20:56:45.991491+0000","last_unstale":"2023-09-23T20:58:15.117148+0000","last_undegraded":"2023-09-23T20:58:15.117148+0000","last_fullsized":"2023-09-23T20:58:15.117148+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":205,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:30:37.405319+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":123,"reported_epoch":232,"state":"active+clean","last_fresh":"2023-09-23T20:58:15.116117+0000","last_change":"2023-09-23T20:55:35.957768+0000","last_active":"2023-09-23T20:58:15.116117+0000","last_peered":"2023-09-23T20:58:15.116117+0000","last_clean":"2023-09-23T20:58:15.116117+0000","last_became_active":"2023-09-23T20:55:35.957421+0000","last_became_peered":"2023-09-23T20:55:35.957421+0000","last_unstale":"2023-09-23T20:58:15.116117+0000","last_undegraded":"2023-09-23T20:58:15.116117+0000","last_fullsized":"2023-09-23T20:58:15.116117+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:49:41.585961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.c","version":"79'161","reported_seq":532,"reported_epoch":231,"state":"active+clean","last_fresh":"2023-09-23T20:58:11.088408+0000","last_change":"2023-09-23T20:57:22.041071+0000","last_active":"2023-09-23T20:58:11.088408+0000","last_peered":"2023-09-23T20:58:11.088408+0000","last_clean":"2023-09-23T20:58:11.088408+0000","last_became_active":"2023-09-23T20:57:15.973204+0000","last_became_peered":"2023-09-23T20:57:15.973204+0000","last_unstale":"2023-09-23T20:58:11.088408+0000","last_undegraded":"2023-09-23T20:58:11.088408+0000","last_fullsized":"2023-09-23T20:58:11.088408+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:22.040907+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:22.040907+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:47:02.019978+0000","scrub_duration":0.013021176000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.a","version":"232'6773","reported_seq":6815,"reported_epoch":232,"state":"active+clean","last_fresh":"2023-09-23T20:58:19.379850+0000","last_change":"2023-09-23T20:56:35.547120+0000","last_active":"2023-09-23T20:58:19.379850+0000","last_peered":"2023-09-23T20:58:19.379850+0000","last_clean":"2023-09-23T20:58:19.379850+0000","last_became_active":"2023-09-23T20:56:35.546838+0000","last_became_peered":"2023-09-23T20:56:35.546838+0000","last_unstale":"2023-09-23T20:58:19.379850+0000","last_undegraded":"2023-09-23T20:58:19.379850+0000","last_fullsized":"2023-09-23T20:58:19.379850+0000","mapping_epoch":199,"log_start":"202'600","ondisk_log_start":"202'600","created":199,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_clean_scrub_stamp":"2023-09-23T20:56:34.529103+0000","objects_scrubbed":0,"log_size":6173,"log_dups_size":0,"ondisk_log_size":6173,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:48:24.132768+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":390791168,"num_objects":5963,"num_object_clones":0,"num_object_copies":17889,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5963,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6773,"num_write_kb":407552,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,5],"acting":[3,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":151,"reported_epoch":232,"state":"active+clean","last_fresh":"2023-09-23T20:58:15.116785+0000","last_change":"2023-09-23T20:56:45.991821+0000","last_active":"2023-09-23T20:58:15.116785+0000","last_peered":"2023-09-23T20:58:15.116785+0000","last_clean":"2023-09-23T20:58:15.116785+0000","last_became_active":"2023-09-23T20:56:45.991447+0000","last_became_peered":"2023-09-23T20:56:45.991447+0000","last_unstale":"2023-09-23T20:58:15.116785+0000","last_undegraded":"2023-09-23T20:58:15.116785+0000","last_fullsized":"2023-09-23T20:58:15.116785+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":205,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:14:25.525222+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":154,"reported_epoch":232,"state":"active+clean","last_fresh":"2023-09-23T20:58:15.116283+0000","last_change":"2023-09-23T20:57:44.982059+0000","last_active":"2023-09-23T20:58:15.116283+0000","last_peered":"2023-09-23T20:58:15.116283+0000","last_clean":"2023-09-23T20:58:15.116283+0000","last_became_active":"2023-09-23T20:57:44.981742+0000","last_became_peered":"2023-09-23T20:57:44.981742+0000","last_unstale":"2023-09-23T20:58:15.116283+0000","last_undegraded":"2023-09-23T20:58:15.116283+0000","last_fullsized":"2023-09-23T20:58:15.116283+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:41:24.042916+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.f","version":"79'161","reported_seq":537,"reported_epoch":232,"state":"active+clean","last_fresh":"2023-09-23T20:58:15.115983+0000","last_change":"2023-09-23T20:57:27.880841+0000","last_active":"2023-09-23T20:58:15.115983+0000","last_peered":"2023-09-23T20:58:15.115983+0000","last_clean":"2023-09-23T20:58:15.115983+0000","last_became_active":"2023-09-23T20:57:15.970925+0000","last_became_peered":"2023-09-23T20:57:15.970925+0000","last_unstale":"2023-09-23T20:58:15.115983+0000","last_undegraded":"2023-09-23T20:58:15.115983+0000","last_fullsized":"2023-09-23T20:58:15.115983+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:27.880776+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:27.880776+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:15:13.438014+0000","scrub_duration":0.0014004899999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.9","version":"233'6819","reported_seq":6862,"reported_epoch":233,"state":"active+clean","last_fresh":"2023-09-23T20:58:21.285202+0000","last_change":"2023-09-23T20:56:35.549387+0000","last_active":"2023-09-23T20:58:21.285202+0000","last_peered":"2023-09-23T20:58:21.285202+0000","last_clean":"2023-09-23T20:58:21.285202+0000","last_became_active":"2023-09-23T20:56:35.549087+0000","last_became_peered":"2023-09-23T20:56:35.549087+0000","last_unstale":"2023-09-23T20:58:21.285202+0000","last_undegraded":"2023-09-23T20:58:21.285202+0000","last_fullsized":"2023-09-23T20:58:21.285202+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":199,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_clean_scrub_stamp":"2023-09-23T20:56:34.529103+0000","objects_scrubbed":0,"log_size":6819,"log_dups_size":0,"ondisk_log_size":6819,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:04:51.547362+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":385810432,"num_objects":5887,"num_object_clones":0,"num_object_copies":17661,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5887,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6819,"num_write_kb":406592,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":137,"reported_epoch":232,"state":"active+clean","last_fresh":"2023-09-23T20:58:15.116079+0000","last_change":"2023-09-23T20:56:45.989153+0000","last_active":"2023-09-23T20:58:15.116079+0000","last_peered":"2023-09-23T20:58:15.116079+0000","last_clean":"2023-09-23T20:58:15.116079+0000","last_became_active":"2023-09-23T20:56:45.988800+0000","last_became_peered":"2023-09-23T20:56:45.988800+0000","last_unstale":"2023-09-23T20:58:15.116079+0000","last_undegraded":"2023-09-23T20:58:15.116079+0000","last_fullsized":"2023-09-23T20:58:15.116079+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":205,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:04:41.130194+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":172,"reported_epoch":232,"state":"active+clean","last_fresh":"2023-09-23T20:58:15.115179+0000","last_change":"2023-09-23T20:55:35.965427+0000","last_active":"2023-09-23T20:58:15.115179+0000","last_peered":"2023-09-23T20:58:15.115179+0000","last_clean":"2023-09-23T20:58:15.115179+0000","last_became_active":"2023-09-23T20:55:35.963969+0000","last_became_peered":"2023-09-23T20:55:35.963969+0000","last_unstale":"2023-09-23T20:58:15.115179+0000","last_undegraded":"2023-09-23T20:58:15.115179+0000","last_fullsized":"2023-09-23T20:58:15.115179+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:38:40.612083+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.e","version":"79'161","reported_seq":444,"reported_epoch":232,"state":"active+clean","last_fresh":"2023-09-23T20:58:15.116644+0000","last_change":"2023-09-23T20:57:34.132738+0000","last_active":"2023-09-23T20:58:15.116644+0000","last_peered":"2023-09-23T20:58:15.116644+0000","last_clean":"2023-09-23T20:58:15.116644+0000","last_became_active":"2023-09-23T20:57:15.974318+0000","last_became_peered":"2023-09-23T20:57:15.974318+0000","last_unstale":"2023-09-23T20:58:15.116644+0000","last_undegraded":"2023-09-23T20:58:15.116644+0000","last_fullsized":"2023-09-23T20:58:15.116644+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:34.132660+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:34.132660+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:07:07.739438+0000","scrub_duration":0.005229694,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.8","version":"233'6918","reported_seq":6961,"reported_epoch":233,"state":"active+clean","last_fresh":"2023-09-23T20:58:21.314629+0000","last_change":"2023-09-23T20:56:35.548119+0000","last_active":"2023-09-23T20:58:21.314629+0000","last_peered":"2023-09-23T20:58:21.314629+0000","last_clean":"2023-09-23T20:58:21.314629+0000","last_became_active":"2023-09-23T20:56:35.547914+0000","last_became_peered":"2023-09-23T20:56:35.547914+0000","last_unstale":"2023-09-23T20:58:21.314629+0000","last_undegraded":"2023-09-23T20:58:21.314629+0000","last_fullsized":"2023-09-23T20:58:21.314629+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":199,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_clean_scrub_stamp":"2023-09-23T20:56:34.529103+0000","objects_scrubbed":0,"log_size":6918,"log_dups_size":0,"ondisk_log_size":6918,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:42:54.255023+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":389021696,"num_objects":5936,"num_object_clones":0,"num_object_copies":17808,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5936,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6918,"num_write_kb":411328,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":123,"reported_epoch":232,"state":"active+clean","last_fresh":"2023-09-23T20:58:15.117341+0000","last_change":"2023-09-23T20:55:35.962005+0000","last_active":"2023-09-23T20:58:15.117341+0000","last_peered":"2023-09-23T20:58:15.117341+0000","last_clean":"2023-09-23T20:58:15.117341+0000","last_became_active":"2023-09-23T20:55:35.961695+0000","last_became_peered":"2023-09-23T20:55:35.961695+0000","last_unstale":"2023-09-23T20:58:15.117341+0000","last_undegraded":"2023-09-23T20:58:15.117341+0000","last_fullsized":"2023-09-23T20:58:15.117341+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:28:22.876201+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":220,"reported_epoch":232,"state":"active+clean","last_fresh":"2023-09-23T20:58:15.135171+0000","last_change":"2023-09-23T20:54:13.928715+0000","last_active":"2023-09-23T20:58:15.135171+0000","last_peered":"2023-09-23T20:58:15.135171+0000","last_clean":"2023-09-23T20:58:15.135171+0000","last_became_active":"2023-09-23T20:54:13.928200+0000","last_became_peered":"2023-09-23T20:54:13.928200+0000","last_unstale":"2023-09-23T20:58:15.135171+0000","last_undegraded":"2023-09-23T20:58:15.135171+0000","last_fullsized":"2023-09-23T20:58:15.135171+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:41:16.927513+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.f","version":"233'6905","reported_seq":6948,"reported_epoch":233,"state":"active+clean","last_fresh":"2023-09-23T20:58:21.244833+0000","last_change":"2023-09-23T20:56:35.545128+0000","last_active":"2023-09-23T20:58:21.244833+0000","last_peered":"2023-09-23T20:58:21.244833+0000","last_clean":"2023-09-23T20:58:21.244833+0000","last_became_active":"2023-09-23T20:56:35.544838+0000","last_became_peered":"2023-09-23T20:56:35.544838+0000","last_unstale":"2023-09-23T20:58:21.244833+0000","last_undegraded":"2023-09-23T20:58:21.244833+0000","last_fullsized":"2023-09-23T20:58:21.244833+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":199,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_clean_scrub_stamp":"2023-09-23T20:56:34.529103+0000","objects_scrubbed":0,"log_size":6905,"log_dups_size":0,"ondisk_log_size":6905,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:30:43.388717+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":390660096,"num_objects":5961,"num_object_clones":0,"num_object_copies":17883,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5961,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6905,"num_write_kb":411712,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.9","version":"79'161","reported_seq":604,"reported_epoch":231,"state":"active+clean","last_fresh":"2023-09-23T20:58:11.088132+0000","last_change":"2023-09-23T20:56:46.921939+0000","last_active":"2023-09-23T20:58:11.088132+0000","last_peered":"2023-09-23T20:58:11.088132+0000","last_clean":"2023-09-23T20:58:11.088132+0000","last_became_active":"2023-09-23T20:56:45.989908+0000","last_became_peered":"2023-09-23T20:56:45.989908+0000","last_unstale":"2023-09-23T20:58:11.088132+0000","last_undegraded":"2023-09-23T20:58:11.088132+0000","last_fullsized":"2023-09-23T20:58:11.088132+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":205,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:56:46.921739+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:56:46.921739+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:20:42.635110+0000","scrub_duration":0.0071954109999999996,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":221,"reported_epoch":231,"state":"active+clean","last_fresh":"2023-09-23T20:58:11.088287+0000","last_change":"2023-09-23T20:54:13.961788+0000","last_active":"2023-09-23T20:58:11.088287+0000","last_peered":"2023-09-23T20:58:11.088287+0000","last_clean":"2023-09-23T20:58:11.088287+0000","last_became_active":"2023-09-23T20:54:13.940937+0000","last_became_peered":"2023-09-23T20:54:13.940937+0000","last_unstale":"2023-09-23T20:58:11.088287+0000","last_undegraded":"2023-09-23T20:58:11.088287+0000","last_fullsized":"2023-09-23T20:58:11.088287+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:11:56.785299+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":127,"reported_epoch":232,"state":"active+clean","last_fresh":"2023-09-23T20:58:15.117069+0000","last_change":"2023-09-23T20:55:35.949670+0000","last_active":"2023-09-23T20:58:15.117069+0000","last_peered":"2023-09-23T20:58:15.117069+0000","last_clean":"2023-09-23T20:58:15.117069+0000","last_became_active":"2023-09-23T20:55:35.949140+0000","last_became_peered":"2023-09-23T20:55:35.949140+0000","last_unstale":"2023-09-23T20:58:15.117069+0000","last_undegraded":"2023-09-23T20:58:15.117069+0000","last_fullsized":"2023-09-23T20:58:15.117069+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:55:59.340303+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.e","version":"233'6872","reported_seq":6915,"reported_epoch":233,"state":"active+clean","last_fresh":"2023-09-23T20:58:21.392317+0000","last_change":"2023-09-23T20:56:35.547184+0000","last_active":"2023-09-23T20:58:21.392317+0000","last_peered":"2023-09-23T20:58:21.392317+0000","last_clean":"2023-09-23T20:58:21.392317+0000","last_became_active":"2023-09-23T20:56:35.546912+0000","last_became_peered":"2023-09-23T20:56:35.546912+0000","last_unstale":"2023-09-23T20:58:21.392317+0000","last_undegraded":"2023-09-23T20:58:21.392317+0000","last_fullsized":"2023-09-23T20:58:21.392317+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":199,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_clean_scrub_stamp":"2023-09-23T20:56:34.529103+0000","objects_scrubbed":0,"log_size":6872,"log_dups_size":0,"ondisk_log_size":6872,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:57:07.422792+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":387186688,"num_objects":5908,"num_object_clones":0,"num_object_copies":17724,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5908,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6872,"num_write_kb":408960,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,0],"acting":[2,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.8","version":"79'161","reported_seq":476,"reported_epoch":232,"state":"active+clean","last_fresh":"2023-09-23T20:58:15.116039+0000","last_change":"2023-09-23T20:56:57.505737+0000","last_active":"2023-09-23T20:58:15.116039+0000","last_peered":"2023-09-23T20:58:15.116039+0000","last_clean":"2023-09-23T20:58:15.116039+0000","last_became_active":"2023-09-23T20:56:04.982345+0000","last_became_peered":"2023-09-23T20:56:04.982345+0000","last_unstale":"2023-09-23T20:58:15.116039+0000","last_undegraded":"2023-09-23T20:58:15.116039+0000","last_fullsized":"2023-09-23T20:58:15.116039+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:56:57.505663+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:56:57.505663+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:34:30.109724+0000","scrub_duration":0.026284108,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":195,"reported_epoch":232,"state":"active+clean","last_fresh":"2023-09-23T20:58:15.115376+0000","last_change":"2023-09-23T20:55:35.959942+0000","last_active":"2023-09-23T20:58:15.115376+0000","last_peered":"2023-09-23T20:58:15.115376+0000","last_clean":"2023-09-23T20:58:15.115376+0000","last_became_active":"2023-09-23T20:55:35.957071+0000","last_became_peered":"2023-09-23T20:55:35.957071+0000","last_unstale":"2023-09-23T20:58:15.115376+0000","last_undegraded":"2023-09-23T20:58:15.115376+0000","last_fullsized":"2023-09-23T20:58:15.115376+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:44:30.308036+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":110,"reported_epoch":232,"state":"active+clean","last_fresh":"2023-09-23T20:58:15.116544+0000","last_change":"2023-09-23T20:54:13.951459+0000","last_active":"2023-09-23T20:58:15.116544+0000","last_peered":"2023-09-23T20:58:15.116544+0000","last_clean":"2023-09-23T20:58:15.116544+0000","last_became_active":"2023-09-23T20:54:13.948409+0000","last_became_peered":"2023-09-23T20:54:13.948409+0000","last_unstale":"2023-09-23T20:58:15.116544+0000","last_undegraded":"2023-09-23T20:58:15.116544+0000","last_fullsized":"2023-09-23T20:58:15.116544+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:38:10.148061+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.b","version":"79'161","reported_seq":515,"reported_epoch":232,"state":"active+clean","last_fresh":"2023-09-23T20:58:15.132235+0000","last_change":"2023-09-23T20:57:27.320172+0000","last_active":"2023-09-23T20:58:15.132235+0000","last_peered":"2023-09-23T20:58:15.132235+0000","last_clean":"2023-09-23T20:58:15.132235+0000","last_became_active":"2023-09-23T20:57:16.009630+0000","last_became_peered":"2023-09-23T20:57:16.009630+0000","last_unstale":"2023-09-23T20:58:15.132235+0000","last_undegraded":"2023-09-23T20:58:15.132235+0000","last_fullsized":"2023-09-23T20:58:15.132235+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:27.320022+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:27.320022+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:31:35.221558+0000","scrub_duration":0.0043894679999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,5],"acting":[1,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.d","version":"233'7010","reported_seq":7053,"reported_epoch":233,"state":"active+clean","last_fresh":"2023-09-23T20:58:24.024241+0000","last_change":"2023-09-23T20:56:35.544132+0000","last_active":"2023-09-23T20:58:24.024241+0000","last_peered":"2023-09-23T20:58:24.024241+0000","last_clean":"2023-09-23T20:58:24.024241+0000","last_became_active":"2023-09-23T20:56:35.543836+0000","last_became_peered":"2023-09-23T20:56:35.543836+0000","last_unstale":"2023-09-23T20:58:24.024241+0000","last_undegraded":"2023-09-23T20:58:24.024241+0000","last_fullsized":"2023-09-23T20:58:24.024241+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":199,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_clean_scrub_stamp":"2023-09-23T20:56:34.529103+0000","objects_scrubbed":0,"log_size":7010,"log_dups_size":0,"ondisk_log_size":7010,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:27:14.636991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":378732568,"num_objects":5780,"num_object_clones":0,"num_object_copies":17340,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5780,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":7010,"num_write_kb":409217,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":110,"reported_epoch":232,"state":"active+clean","last_fresh":"2023-09-23T20:58:15.116071+0000","last_change":"2023-09-23T20:54:13.950254+0000","last_active":"2023-09-23T20:58:15.116071+0000","last_peered":"2023-09-23T20:58:15.116071+0000","last_clean":"2023-09-23T20:58:15.116071+0000","last_became_active":"2023-09-23T20:54:13.949934+0000","last_became_peered":"2023-09-23T20:54:13.949934+0000","last_unstale":"2023-09-23T20:58:15.116071+0000","last_undegraded":"2023-09-23T20:58:15.116071+0000","last_fullsized":"2023-09-23T20:58:15.116071+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:30:32.847167+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":127,"reported_epoch":232,"state":"active+clean","last_fresh":"2023-09-23T20:58:15.116867+0000","last_change":"2023-09-23T20:55:35.956153+0000","last_active":"2023-09-23T20:58:15.116867+0000","last_peered":"2023-09-23T20:58:15.116867+0000","last_clean":"2023-09-23T20:58:15.116867+0000","last_became_active":"2023-09-23T20:55:35.955630+0000","last_became_peered":"2023-09-23T20:55:35.955630+0000","last_unstale":"2023-09-23T20:58:15.116867+0000","last_undegraded":"2023-09-23T20:58:15.116867+0000","last_fullsized":"2023-09-23T20:58:15.116867+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:45:20.282929+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.c","version":"233'6987","reported_seq":7039,"reported_epoch":233,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T20:58:24.042417+0000","last_change":"2023-09-23T20:57:46.463005+0000","last_active":"2023-09-23T20:58:24.042417+0000","last_peered":"2023-09-23T20:58:24.042417+0000","last_clean":"2023-09-23T20:57:44.015119+0000","last_became_active":"2023-09-23T20:57:46.448702+0000","last_became_peered":"2023-09-23T20:57:46.448702+0000","last_unstale":"2023-09-23T20:58:24.042417+0000","last_undegraded":"2023-09-23T20:57:46.010874+0000","last_fullsized":"2023-09-23T20:57:45.978434+0000","mapping_epoch":223,"log_start":"0'0","ondisk_log_start":"0'0","created":199,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_clean_scrub_stamp":"2023-09-23T20:56:34.529103+0000","objects_scrubbed":0,"log_size":6987,"log_dups_size":0,"ondisk_log_size":6987,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:08:34.180651+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":381485056,"num_objects":5821,"num_object_clones":0,"num_object_copies":17463,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5473,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5821,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6987,"num_write_kb":409856,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,7],"avail_no_missing":["6","2","7"],"object_location_counts":[{"shards":"2,6,7","objects":4890},{"shards":"6,7","objects":348}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.a","version":"79'161","reported_seq":476,"reported_epoch":232,"state":"active+clean","last_fresh":"2023-09-23T20:58:15.116933+0000","last_change":"2023-09-23T20:56:44.487778+0000","last_active":"2023-09-23T20:58:15.116933+0000","last_peered":"2023-09-23T20:58:15.116933+0000","last_clean":"2023-09-23T20:58:15.116933+0000","last_became_active":"2023-09-23T20:56:05.006740+0000","last_became_peered":"2023-09-23T20:56:05.006740+0000","last_unstale":"2023-09-23T20:58:15.116933+0000","last_undegraded":"2023-09-23T20:58:15.116933+0000","last_fullsized":"2023-09-23T20:58:15.116933+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:56:44.487678+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:56:44.487678+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:04:50.086749+0000","scrub_duration":0.0017408899999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":20,"reported_epoch":231,"state":"active+clean","last_fresh":"2023-09-23T20:58:11.087850+0000","last_change":"2023-09-23T20:57:44.982513+0000","last_active":"2023-09-23T20:58:11.087850+0000","last_peered":"2023-09-23T20:58:11.087850+0000","last_clean":"2023-09-23T20:58:11.087850+0000","last_became_active":"2023-09-23T20:57:44.982112+0000","last_became_peered":"2023-09-23T20:57:44.982112+0000","last_unstale":"2023-09-23T20:58:11.087850+0000","last_undegraded":"2023-09-23T20:58:11.087850+0000","last_fullsized":"2023-09-23T20:58:11.087850+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:09:05.630041+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":202,"reported_epoch":232,"state":"active+clean","last_fresh":"2023-09-23T20:58:15.130434+0000","last_change":"2023-09-23T20:54:13.940748+0000","last_active":"2023-09-23T20:58:15.130434+0000","last_peered":"2023-09-23T20:58:15.130434+0000","last_clean":"2023-09-23T20:58:15.130434+0000","last_became_active":"2023-09-23T20:54:13.940419+0000","last_became_peered":"2023-09-23T20:54:13.940419+0000","last_unstale":"2023-09-23T20:58:15.130434+0000","last_undegraded":"2023-09-23T20:58:15.130434+0000","last_fullsized":"2023-09-23T20:58:15.130434+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:15:01.923816+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.1","version":"233'6926","reported_seq":6980,"reported_epoch":233,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T20:58:22.146505+0000","last_change":"2023-09-23T20:57:46.483248+0000","last_active":"2023-09-23T20:58:22.146505+0000","last_peered":"2023-09-23T20:58:22.146505+0000","last_clean":"2023-09-23T20:57:44.047957+0000","last_became_active":"2023-09-23T20:57:46.475568+0000","last_became_peered":"2023-09-23T20:57:46.475568+0000","last_unstale":"2023-09-23T20:58:22.146505+0000","last_undegraded":"2023-09-23T20:57:46.083591+0000","last_fullsized":"2023-09-23T20:57:45.978436+0000","mapping_epoch":223,"log_start":"202'200","ondisk_log_start":"202'200","created":199,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_clean_scrub_stamp":"2023-09-23T20:56:34.529103+0000","objects_scrubbed":0,"log_size":6726,"log_dups_size":0,"ondisk_log_size":6726,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:22:45.067269+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":384040960,"num_objects":5860,"num_object_clones":0,"num_object_copies":17580,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5470,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5860,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6926,"num_write_kb":409152,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,6],"avail_no_missing":["4","1","6"],"object_location_counts":[{"shards":"1,4,6","objects":4937},{"shards":"4,6","objects":390}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.7","version":"79'161","reported_seq":538,"reported_epoch":232,"state":"active+clean","last_fresh":"2023-09-23T20:58:15.116321+0000","last_change":"2023-09-23T20:57:48.243572+0000","last_active":"2023-09-23T20:58:15.116321+0000","last_peered":"2023-09-23T20:58:15.116321+0000","last_clean":"2023-09-23T20:58:15.116321+0000","last_became_active":"2023-09-23T20:57:48.243229+0000","last_became_peered":"2023-09-23T20:57:48.243229+0000","last_unstale":"2023-09-23T20:58:15.116321+0000","last_undegraded":"2023-09-23T20:58:15.116321+0000","last_fullsized":"2023-09-23T20:58:15.116321+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":3,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:25.916867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:25.916867+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:23:19.385831+0000","scrub_duration":0.001665323,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":208,"reported_epoch":233,"state":"active+clean","last_fresh":"2023-09-23T20:58:19.480495+0000","last_change":"2023-09-23T20:50:05.867130+0000","last_active":"2023-09-23T20:58:19.480495+0000","last_peered":"2023-09-23T20:58:19.480495+0000","last_clean":"2023-09-23T20:58:19.480495+0000","last_became_active":"2023-09-23T20:50:05.866758+0000","last_became_peered":"2023-09-23T20:50:05.866758+0000","last_unstale":"2023-09-23T20:58:19.480495+0000","last_undegraded":"2023-09-23T20:58:19.480495+0000","last_fullsized":"2023-09-23T20:58:19.480495+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:55:07.548209+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":106,"reported_epoch":232,"state":"active+clean","last_fresh":"2023-09-23T20:58:15.116250+0000","last_change":"2023-09-23T20:54:31.887604+0000","last_active":"2023-09-23T20:58:15.116250+0000","last_peered":"2023-09-23T20:58:15.116250+0000","last_clean":"2023-09-23T20:58:15.116250+0000","last_became_active":"2023-09-23T20:54:31.886752+0000","last_became_peered":"2023-09-23T20:54:31.886752+0000","last_unstale":"2023-09-23T20:58:15.116250+0000","last_undegraded":"2023-09-23T20:58:15.116250+0000","last_fullsized":"2023-09-23T20:58:15.116250+0000","mapping_epoch":136,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:13:58.872176+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":206,"reported_epoch":233,"state":"active+clean","last_fresh":"2023-09-23T20:58:19.478333+0000","last_change":"2023-09-23T20:54:31.899082+0000","last_active":"2023-09-23T20:58:19.478333+0000","last_peered":"2023-09-23T20:58:19.478333+0000","last_clean":"2023-09-23T20:58:19.478333+0000","last_became_active":"2023-09-23T20:54:31.898521+0000","last_became_peered":"2023-09-23T20:54:31.898521+0000","last_unstale":"2023-09-23T20:58:19.478333+0000","last_undegraded":"2023-09-23T20:58:19.478333+0000","last_fullsized":"2023-09-23T20:58:19.478333+0000","mapping_epoch":136,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:25:36.059711+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.7","version":"233'7007","reported_seq":7049,"reported_epoch":233,"state":"active+clean","last_fresh":"2023-09-23T20:58:24.042386+0000","last_change":"2023-09-23T20:56:35.541695+0000","last_active":"2023-09-23T20:58:24.042386+0000","last_peered":"2023-09-23T20:58:24.042386+0000","last_clean":"2023-09-23T20:58:24.042386+0000","last_became_active":"2023-09-23T20:56:35.541392+0000","last_became_peered":"2023-09-23T20:56:35.541392+0000","last_unstale":"2023-09-23T20:58:24.042386+0000","last_undegraded":"2023-09-23T20:58:24.042386+0000","last_fullsized":"2023-09-23T20:58:24.042386+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":199,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_clean_scrub_stamp":"2023-09-23T20:56:34.529103+0000","objects_scrubbed":0,"log_size":7007,"log_dups_size":0,"ondisk_log_size":7007,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:57:22.478253+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":380305408,"num_objects":5803,"num_object_clones":0,"num_object_copies":17409,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5803,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7007,"num_write_kb":409920,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.1","version":"79'161","reported_seq":496,"reported_epoch":232,"state":"active+clean","last_fresh":"2023-09-23T20:58:15.116628+0000","last_change":"2023-09-23T20:57:19.669346+0000","last_active":"2023-09-23T20:58:15.116628+0000","last_peered":"2023-09-23T20:58:15.116628+0000","last_clean":"2023-09-23T20:58:15.116628+0000","last_became_active":"2023-09-23T20:57:08.246046+0000","last_became_peered":"2023-09-23T20:57:08.246046+0000","last_unstale":"2023-09-23T20:58:15.116628+0000","last_undegraded":"2023-09-23T20:58:15.116628+0000","last_fullsized":"2023-09-23T20:58:15.116628+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:19.669259+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:19.669259+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:14:03.435531+0000","scrub_duration":0.0045759559999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":123,"reported_epoch":232,"state":"active+clean","last_fresh":"2023-09-23T20:58:15.116785+0000","last_change":"2023-09-23T20:55:35.959514+0000","last_active":"2023-09-23T20:58:15.116785+0000","last_peered":"2023-09-23T20:58:15.116785+0000","last_clean":"2023-09-23T20:58:15.116785+0000","last_became_active":"2023-09-23T20:55:35.959244+0000","last_became_peered":"2023-09-23T20:55:35.959244+0000","last_unstale":"2023-09-23T20:58:15.116785+0000","last_undegraded":"2023-09-23T20:58:15.116785+0000","last_fullsized":"2023-09-23T20:58:15.116785+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:47:32.665316+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":127,"reported_epoch":232,"state":"active+clean","last_fresh":"2023-09-23T20:58:15.117909+0000","last_change":"2023-09-23T20:55:35.949000+0000","last_active":"2023-09-23T20:58:15.117909+0000","last_peered":"2023-09-23T20:58:15.117909+0000","last_clean":"2023-09-23T20:58:15.117909+0000","last_became_active":"2023-09-23T20:55:35.948410+0000","last_became_peered":"2023-09-23T20:55:35.948410+0000","last_unstale":"2023-09-23T20:58:15.117909+0000","last_undegraded":"2023-09-23T20:58:15.117909+0000","last_fullsized":"2023-09-23T20:58:15.117909+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:26:50.477289+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":200,"reported_epoch":233,"state":"active+clean","last_fresh":"2023-09-23T20:58:19.480270+0000","last_change":"2023-09-23T20:51:10.829889+0000","last_active":"2023-09-23T20:58:19.480270+0000","last_peered":"2023-09-23T20:58:19.480270+0000","last_clean":"2023-09-23T20:58:19.480270+0000","last_became_active":"2023-09-23T20:51:10.829533+0000","last_became_peered":"2023-09-23T20:51:10.829533+0000","last_unstale":"2023-09-23T20:58:19.480270+0000","last_undegraded":"2023-09-23T20:58:19.480270+0000","last_fullsized":"2023-09-23T20:58:19.480270+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:43:04.603460+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.4","version":"233'6835","reported_seq":6876,"reported_epoch":233,"state":"active+clean","last_fresh":"2023-09-23T20:58:21.261466+0000","last_change":"2023-09-23T20:56:35.549909+0000","last_active":"2023-09-23T20:58:21.261466+0000","last_peered":"2023-09-23T20:58:21.261466+0000","last_clean":"2023-09-23T20:58:21.261466+0000","last_became_active":"2023-09-23T20:56:35.549618+0000","last_became_peered":"2023-09-23T20:56:35.549618+0000","last_unstale":"2023-09-23T20:58:21.261466+0000","last_undegraded":"2023-09-23T20:58:21.261466+0000","last_fullsized":"2023-09-23T20:58:21.261466+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":199,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_clean_scrub_stamp":"2023-09-23T20:56:34.529103+0000","objects_scrubbed":0,"log_size":6835,"log_dups_size":0,"ondisk_log_size":6835,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:24:25.918475+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":385810432,"num_objects":5887,"num_object_clones":0,"num_object_copies":17661,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5887,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6835,"num_write_kb":407104,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.2","version":"79'161","reported_seq":462,"reported_epoch":232,"state":"active+clean","last_fresh":"2023-09-23T20:58:15.134893+0000","last_change":"2023-09-23T20:57:24.802586+0000","last_active":"2023-09-23T20:58:15.134893+0000","last_peered":"2023-09-23T20:58:15.134893+0000","last_clean":"2023-09-23T20:58:15.134893+0000","last_became_active":"2023-09-23T20:57:08.236114+0000","last_became_peered":"2023-09-23T20:57:08.236114+0000","last_unstale":"2023-09-23T20:58:15.134893+0000","last_undegraded":"2023-09-23T20:58:15.134893+0000","last_fullsized":"2023-09-23T20:58:15.134893+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:24.802451+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:24.802451+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:06:22.309980+0000","scrub_duration":0.0090861250000000005,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,4],"acting":[0,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":218,"reported_epoch":232,"state":"active+clean","last_fresh":"2023-09-23T20:58:15.123876+0000","last_change":"2023-09-23T20:51:10.838054+0000","last_active":"2023-09-23T20:58:15.123876+0000","last_peered":"2023-09-23T20:58:15.123876+0000","last_clean":"2023-09-23T20:58:15.123876+0000","last_became_active":"2023-09-23T20:51:10.837750+0000","last_became_peered":"2023-09-23T20:51:10.837750+0000","last_unstale":"2023-09-23T20:58:15.123876+0000","last_undegraded":"2023-09-23T20:58:15.123876+0000","last_fullsized":"2023-09-23T20:58:15.123876+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:15:51.648085+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":200,"reported_epoch":232,"state":"active+clean","last_fresh":"2023-09-23T20:58:15.130040+0000","last_change":"2023-09-23T20:55:35.951146+0000","last_active":"2023-09-23T20:58:15.130040+0000","last_peered":"2023-09-23T20:58:15.130040+0000","last_clean":"2023-09-23T20:58:15.130040+0000","last_became_active":"2023-09-23T20:55:35.950874+0000","last_became_peered":"2023-09-23T20:55:35.950874+0000","last_unstale":"2023-09-23T20:58:15.130040+0000","last_undegraded":"2023-09-23T20:58:15.130040+0000","last_fullsized":"2023-09-23T20:58:15.130040+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:58:13.957677+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":220,"reported_epoch":232,"state":"active+clean","last_fresh":"2023-09-23T20:58:15.130361+0000","last_change":"2023-09-23T20:54:31.898870+0000","last_active":"2023-09-23T20:58:15.130361+0000","last_peered":"2023-09-23T20:58:15.130361+0000","last_clean":"2023-09-23T20:58:15.130361+0000","last_became_active":"2023-09-23T20:54:31.898467+0000","last_became_peered":"2023-09-23T20:54:31.898467+0000","last_unstale":"2023-09-23T20:58:15.130361+0000","last_undegraded":"2023-09-23T20:58:15.130361+0000","last_fullsized":"2023-09-23T20:58:15.130361+0000","mapping_epoch":136,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:56:22.483408+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.6","version":"233'6722","reported_seq":8228,"reported_epoch":233,"state":"active+recovering+undersized+remapped","last_fresh":"2023-09-23T20:58:21.288009+0000","last_change":"2023-09-23T20:57:46.535681+0000","last_active":"2023-09-23T20:58:21.288009+0000","last_peered":"2023-09-23T20:58:21.288009+0000","last_clean":"2023-09-23T20:57:44.031597+0000","last_became_active":"2023-09-23T20:57:46.443927+0000","last_became_peered":"2023-09-23T20:57:46.443927+0000","last_unstale":"2023-09-23T20:58:21.288009+0000","last_undegraded":"2023-09-23T20:58:21.288009+0000","last_fullsized":"2023-09-23T20:57:45.977227+0000","mapping_epoch":223,"log_start":"0'0","ondisk_log_start":"0'0","created":199,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_clean_scrub_stamp":"2023-09-23T20:56:34.529103+0000","objects_scrubbed":0,"log_size":6722,"log_dups_size":0,"ondisk_log_size":6722,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:37:23.911131+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":384434176,"num_objects":5866,"num_object_clones":0,"num_object_copies":17598,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":4671,"num_objects_unfound":0,"num_objects_dirty":5866,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6722,"num_write_kb":402816,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":727,"num_bytes_recovered":47644672,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7],"avail_no_missing":["0","5","7"],"object_location_counts":[{"shards":"0,5,7","objects":4671},{"shards":"0,7","objects":1195}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":106,"reported_epoch":232,"state":"active+clean","last_fresh":"2023-09-23T20:58:15.116531+0000","last_change":"2023-09-23T20:54:31.883504+0000","last_active":"2023-09-23T20:58:15.116531+0000","last_peered":"2023-09-23T20:58:15.116531+0000","last_clean":"2023-09-23T20:58:15.116531+0000","last_became_active":"2023-09-23T20:54:31.883172+0000","last_became_peered":"2023-09-23T20:54:31.883172+0000","last_unstale":"2023-09-23T20:58:15.116531+0000","last_undegraded":"2023-09-23T20:58:15.116531+0000","last_fullsized":"2023-09-23T20:58:15.116531+0000","mapping_epoch":136,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:45:40.289120+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"79'161","reported_seq":462,"reported_epoch":232,"state":"active+clean","last_fresh":"2023-09-23T20:58:15.134581+0000","last_change":"2023-09-23T20:57:20.738417+0000","last_active":"2023-09-23T20:58:15.134581+0000","last_peered":"2023-09-23T20:58:15.134581+0000","last_clean":"2023-09-23T20:58:15.134581+0000","last_became_active":"2023-09-23T20:57:08.236842+0000","last_became_peered":"2023-09-23T20:57:08.236842+0000","last_unstale":"2023-09-23T20:58:15.134581+0000","last_undegraded":"2023-09-23T20:58:15.134581+0000","last_fullsized":"2023-09-23T20:58:15.134581+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":0,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:20.738273+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:20.738273+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:18:28.327990+0000","scrub_duration":0.0088336339999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":209,"reported_epoch":231,"state":"active+clean","last_fresh":"2023-09-23T20:58:11.102041+0000","last_change":"2023-09-23T20:55:35.950777+0000","last_active":"2023-09-23T20:58:11.102041+0000","last_peered":"2023-09-23T20:58:11.102041+0000","last_clean":"2023-09-23T20:58:11.102041+0000","last_became_active":"2023-09-23T20:55:35.950427+0000","last_became_peered":"2023-09-23T20:55:35.950427+0000","last_unstale":"2023-09-23T20:58:11.102041+0000","last_undegraded":"2023-09-23T20:58:11.102041+0000","last_fullsized":"2023-09-23T20:58:11.102041+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:20:10.371751+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":177,"reported_epoch":232,"state":"active+clean","last_fresh":"2023-09-23T20:58:15.115532+0000","last_change":"2023-09-23T20:55:35.978140+0000","last_active":"2023-09-23T20:58:15.115532+0000","last_peered":"2023-09-23T20:58:15.115532+0000","last_clean":"2023-09-23T20:58:15.115532+0000","last_became_active":"2023-09-23T20:55:35.977831+0000","last_became_peered":"2023-09-23T20:55:35.977831+0000","last_unstale":"2023-09-23T20:58:15.115532+0000","last_undegraded":"2023-09-23T20:58:15.115532+0000","last_fullsized":"2023-09-23T20:58:15.115532+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:25:47.678023+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.5","version":"233'6814","reported_seq":6864,"reported_epoch":233,"state":"active+clean","last_fresh":"2023-09-23T20:58:21.350093+0000","last_change":"2023-09-23T20:56:45.990110+0000","last_active":"2023-09-23T20:58:21.350093+0000","last_peered":"2023-09-23T20:58:21.350093+0000","last_clean":"2023-09-23T20:58:21.350093+0000","last_became_active":"2023-09-23T20:56:45.988394+0000","last_became_peered":"2023-09-23T20:56:45.988394+0000","last_unstale":"2023-09-23T20:58:21.350093+0000","last_undegraded":"2023-09-23T20:58:21.350093+0000","last_fullsized":"2023-09-23T20:58:21.350093+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":199,"last_epoch_clean":205,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_clean_scrub_stamp":"2023-09-23T20:56:34.529103+0000","objects_scrubbed":0,"log_size":6814,"log_dups_size":0,"ondisk_log_size":6814,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:22:03.814210+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":384303104,"num_objects":5864,"num_object_clones":0,"num_object_copies":17592,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5864,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6814,"num_write_kb":405696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,0],"acting":[2,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"79'161","reported_seq":499,"reported_epoch":232,"state":"active+clean","last_fresh":"2023-09-23T20:58:15.132363+0000","last_change":"2023-09-23T20:57:20.378531+0000","last_active":"2023-09-23T20:58:15.132363+0000","last_peered":"2023-09-23T20:58:15.132363+0000","last_clean":"2023-09-23T20:58:15.132363+0000","last_became_active":"2023-09-23T20:57:08.238019+0000","last_became_peered":"2023-09-23T20:57:08.238019+0000","last_unstale":"2023-09-23T20:58:15.132363+0000","last_undegraded":"2023-09-23T20:58:15.132363+0000","last_fullsized":"2023-09-23T20:58:15.132363+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":3,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:20.377714+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:20.377714+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:00:08.305602+0000","scrub_duration":0.003015228,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":101,"reported_epoch":232,"state":"active+clean","last_fresh":"2023-09-23T20:58:15.130212+0000","last_change":"2023-09-23T20:56:45.991884+0000","last_active":"2023-09-23T20:58:15.130212+0000","last_peered":"2023-09-23T20:58:15.130212+0000","last_clean":"2023-09-23T20:58:15.130212+0000","last_became_active":"2023-09-23T20:56:45.991568+0000","last_became_peered":"2023-09-23T20:56:45.991568+0000","last_unstale":"2023-09-23T20:58:15.130212+0000","last_undegraded":"2023-09-23T20:58:15.130212+0000","last_fullsized":"2023-09-23T20:58:15.130212+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":205,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:21:20.775511+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,0],"acting":[1,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":226,"reported_epoch":232,"state":"active+clean","last_fresh":"2023-09-23T20:58:15.134708+0000","last_change":"2023-09-23T20:57:44.979950+0000","last_active":"2023-09-23T20:58:15.134708+0000","last_peered":"2023-09-23T20:58:15.134708+0000","last_clean":"2023-09-23T20:58:15.134708+0000","last_became_active":"2023-09-23T20:57:44.979346+0000","last_became_peered":"2023-09-23T20:57:44.979346+0000","last_unstale":"2023-09-23T20:58:15.134708+0000","last_undegraded":"2023-09-23T20:58:15.134708+0000","last_fullsized":"2023-09-23T20:58:15.134708+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:10:24.372800+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":205,"reported_epoch":232,"state":"active+clean","last_fresh":"2023-09-23T20:58:15.136659+0000","last_change":"2023-09-23T20:50:37.982235+0000","last_active":"2023-09-23T20:58:15.136659+0000","last_peered":"2023-09-23T20:58:15.136659+0000","last_clean":"2023-09-23T20:58:15.136659+0000","last_became_active":"2023-09-23T20:50:37.981913+0000","last_became_peered":"2023-09-23T20:50:37.981913+0000","last_unstale":"2023-09-23T20:58:15.136659+0000","last_undegraded":"2023-09-23T20:58:15.136659+0000","last_fullsized":"2023-09-23T20:58:15.136659+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:59:57.122584+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.0","version":"233'6841","reported_seq":6883,"reported_epoch":233,"state":"active+clean","last_fresh":"2023-09-23T20:58:22.077241+0000","last_change":"2023-09-23T20:56:35.550635+0000","last_active":"2023-09-23T20:58:22.077241+0000","last_peered":"2023-09-23T20:58:22.077241+0000","last_clean":"2023-09-23T20:58:22.077241+0000","last_became_active":"2023-09-23T20:56:35.550376+0000","last_became_peered":"2023-09-23T20:56:35.550376+0000","last_unstale":"2023-09-23T20:58:22.077241+0000","last_undegraded":"2023-09-23T20:58:22.077241+0000","last_fullsized":"2023-09-23T20:58:22.077241+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":199,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_clean_scrub_stamp":"2023-09-23T20:56:34.529103+0000","objects_scrubbed":0,"log_size":6841,"log_dups_size":0,"ondisk_log_size":6841,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:40:55.020049+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":382795776,"num_objects":5841,"num_object_clones":0,"num_object_copies":17523,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5841,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6841,"num_write_kb":405824,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.6","version":"79'161","reported_seq":515,"reported_epoch":232,"state":"active+clean","last_fresh":"2023-09-23T20:58:15.115802+0000","last_change":"2023-09-23T20:57:26.907505+0000","last_active":"2023-09-23T20:58:15.115802+0000","last_peered":"2023-09-23T20:58:15.115802+0000","last_clean":"2023-09-23T20:58:15.115802+0000","last_became_active":"2023-09-23T20:57:08.240321+0000","last_became_peered":"2023-09-23T20:57:08.240321+0000","last_unstale":"2023-09-23T20:58:15.115802+0000","last_undegraded":"2023-09-23T20:58:15.115802+0000","last_fullsized":"2023-09-23T20:58:15.115802+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":3,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:26.907408+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:26.907408+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:34:17.392928+0000","scrub_duration":0.010337038,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":148,"reported_epoch":232,"state":"active+clean","last_fresh":"2023-09-23T20:58:15.137159+0000","last_change":"2023-09-23T20:56:45.988566+0000","last_active":"2023-09-23T20:58:15.137159+0000","last_peered":"2023-09-23T20:58:15.137159+0000","last_clean":"2023-09-23T20:58:15.137159+0000","last_became_active":"2023-09-23T20:56:45.988249+0000","last_became_peered":"2023-09-23T20:56:45.988249+0000","last_unstale":"2023-09-23T20:58:15.137159+0000","last_undegraded":"2023-09-23T20:58:15.137159+0000","last_fullsized":"2023-09-23T20:58:15.137159+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":205,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:51:36.644764+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,4],"acting":[0,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":203,"reported_epoch":233,"state":"active+clean","last_fresh":"2023-09-23T20:58:19.480413+0000","last_change":"2023-09-23T20:54:31.898965+0000","last_active":"2023-09-23T20:58:19.480413+0000","last_peered":"2023-09-23T20:58:19.480413+0000","last_clean":"2023-09-23T20:58:19.480413+0000","last_became_active":"2023-09-23T20:54:31.898663+0000","last_became_peered":"2023-09-23T20:54:31.898663+0000","last_unstale":"2023-09-23T20:58:19.480413+0000","last_undegraded":"2023-09-23T20:58:19.480413+0000","last_fullsized":"2023-09-23T20:58:19.480413+0000","mapping_epoch":136,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:53:05.058087+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":193,"reported_epoch":232,"state":"active+clean","last_fresh":"2023-09-23T20:58:15.135369+0000","last_change":"2023-09-23T20:54:31.884717+0000","last_active":"2023-09-23T20:58:15.135369+0000","last_peered":"2023-09-23T20:58:15.135369+0000","last_clean":"2023-09-23T20:58:15.135369+0000","last_became_active":"2023-09-23T20:54:31.884345+0000","last_became_peered":"2023-09-23T20:54:31.884345+0000","last_unstale":"2023-09-23T20:58:15.135369+0000","last_undegraded":"2023-09-23T20:58:15.135369+0000","last_fullsized":"2023-09-23T20:58:15.135369+0000","mapping_epoch":136,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:02:26.393868+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.3","version":"232'6678","reported_seq":6720,"reported_epoch":232,"state":"active+clean","last_fresh":"2023-09-23T20:58:19.347713+0000","last_change":"2023-09-23T20:56:35.549655+0000","last_active":"2023-09-23T20:58:19.347713+0000","last_peered":"2023-09-23T20:58:19.347713+0000","last_clean":"2023-09-23T20:58:19.347713+0000","last_became_active":"2023-09-23T20:56:35.549357+0000","last_became_peered":"2023-09-23T20:56:35.549357+0000","last_unstale":"2023-09-23T20:58:19.347713+0000","last_undegraded":"2023-09-23T20:58:19.347713+0000","last_fullsized":"2023-09-23T20:58:19.347713+0000","mapping_epoch":199,"log_start":"202'500","ondisk_log_start":"202'500","created":199,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_clean_scrub_stamp":"2023-09-23T20:56:34.529103+0000","objects_scrubbed":0,"log_size":6178,"log_dups_size":0,"ondisk_log_size":6178,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:37:58.752518+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":382992384,"num_objects":5844,"num_object_clones":0,"num_object_copies":17532,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5844,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6678,"num_write_kb":400704,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.5","version":"79'161","reported_seq":496,"reported_epoch":232,"state":"active+clean","last_fresh":"2023-09-23T20:58:15.116247+0000","last_change":"2023-09-23T20:57:30.029479+0000","last_active":"2023-09-23T20:58:15.116247+0000","last_peered":"2023-09-23T20:58:15.116247+0000","last_clean":"2023-09-23T20:58:15.116247+0000","last_became_active":"2023-09-23T20:57:08.249909+0000","last_became_peered":"2023-09-23T20:57:08.249909+0000","last_unstale":"2023-09-23T20:58:15.116247+0000","last_undegraded":"2023-09-23T20:58:15.116247+0000","last_fullsized":"2023-09-23T20:58:15.116247+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":3,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:30.029396+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:30.029396+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:16:46.083145+0000","scrub_duration":0.012914682,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":195,"reported_epoch":231,"state":"active+clean","last_fresh":"2023-09-23T20:58:11.112028+0000","last_change":"2023-09-23T20:51:10.836910+0000","last_active":"2023-09-23T20:58:11.112028+0000","last_peered":"2023-09-23T20:58:11.112028+0000","last_clean":"2023-09-23T20:58:11.112028+0000","last_became_active":"2023-09-23T20:51:10.836557+0000","last_became_peered":"2023-09-23T20:51:10.836557+0000","last_unstale":"2023-09-23T20:58:11.112028+0000","last_undegraded":"2023-09-23T20:58:11.112028+0000","last_fullsized":"2023-09-23T20:58:11.112028+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:51:54.019969+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":202,"reported_epoch":232,"state":"active+clean","last_fresh":"2023-09-23T20:58:15.130466+0000","last_change":"2023-09-23T20:55:35.953444+0000","last_active":"2023-09-23T20:58:15.130466+0000","last_peered":"2023-09-23T20:58:15.130466+0000","last_clean":"2023-09-23T20:58:15.130466+0000","last_became_active":"2023-09-23T20:55:35.953151+0000","last_became_peered":"2023-09-23T20:55:35.953151+0000","last_unstale":"2023-09-23T20:58:15.130466+0000","last_undegraded":"2023-09-23T20:58:15.130466+0000","last_fullsized":"2023-09-23T20:58:15.130466+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:39:12.404391+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":20,"reported_epoch":231,"state":"active+clean","last_fresh":"2023-09-23T20:58:11.087841+0000","last_change":"2023-09-23T20:57:44.982509+0000","last_active":"2023-09-23T20:58:11.087841+0000","last_peered":"2023-09-23T20:58:11.087841+0000","last_clean":"2023-09-23T20:58:11.087841+0000","last_became_active":"2023-09-23T20:57:44.982175+0000","last_became_peered":"2023-09-23T20:57:44.982175+0000","last_unstale":"2023-09-23T20:58:11.087841+0000","last_undegraded":"2023-09-23T20:58:11.087841+0000","last_fullsized":"2023-09-23T20:58:11.087841+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:37:49.215488+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.2","version":"232'6655","reported_seq":6697,"reported_epoch":232,"state":"active+clean","last_fresh":"2023-09-23T20:58:19.357620+0000","last_change":"2023-09-23T20:56:35.543020+0000","last_active":"2023-09-23T20:58:19.357620+0000","last_peered":"2023-09-23T20:58:19.357620+0000","last_clean":"2023-09-23T20:58:19.357620+0000","last_became_active":"2023-09-23T20:56:35.542719+0000","last_became_peered":"2023-09-23T20:56:35.542719+0000","last_unstale":"2023-09-23T20:58:19.357620+0000","last_undegraded":"2023-09-23T20:58:19.357620+0000","last_fullsized":"2023-09-23T20:58:19.357620+0000","mapping_epoch":199,"log_start":"202'500","ondisk_log_start":"202'500","created":199,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_clean_scrub_stamp":"2023-09-23T20:56:34.529103+0000","objects_scrubbed":0,"log_size":6155,"log_dups_size":0,"ondisk_log_size":6155,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:52:21.088628+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":383844352,"num_objects":5857,"num_object_clones":0,"num_object_copies":17571,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5857,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6655,"num_write_kb":400384,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.4","version":"79'161","reported_seq":510,"reported_epoch":231,"state":"active+clean","last_fresh":"2023-09-23T20:58:11.100644+0000","last_change":"2023-09-23T20:57:20.012904+0000","last_active":"2023-09-23T20:58:11.100644+0000","last_peered":"2023-09-23T20:58:11.100644+0000","last_clean":"2023-09-23T20:58:11.100644+0000","last_became_active":"2023-09-23T20:57:08.237996+0000","last_became_peered":"2023-09-23T20:57:08.237996+0000","last_unstale":"2023-09-23T20:58:11.100644+0000","last_undegraded":"2023-09-23T20:58:11.100644+0000","last_fullsized":"2023-09-23T20:58:11.100644+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":3,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:20.012764+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:20.012764+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:51:28.796946+0000","scrub_duration":0.0045251919999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":182,"reported_epoch":232,"state":"active+clean","last_fresh":"2023-09-23T20:58:15.116158+0000","last_change":"2023-09-23T20:55:35.949541+0000","last_active":"2023-09-23T20:58:15.116158+0000","last_peered":"2023-09-23T20:58:15.116158+0000","last_clean":"2023-09-23T20:58:15.116158+0000","last_became_active":"2023-09-23T20:55:35.944635+0000","last_became_peered":"2023-09-23T20:55:35.944635+0000","last_unstale":"2023-09-23T20:58:15.116158+0000","last_undegraded":"2023-09-23T20:58:15.116158+0000","last_fullsized":"2023-09-23T20:58:15.116158+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:30:36.033585+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"70'1","reported_seq":221,"reported_epoch":232,"state":"active+clean","last_fresh":"2023-09-23T20:58:15.136748+0000","last_change":"2023-09-23T20:54:14.009723+0000","last_active":"2023-09-23T20:58:15.136748+0000","last_peered":"2023-09-23T20:58:15.136748+0000","last_clean":"2023-09-23T20:58:15.136748+0000","last_became_active":"2023-09-23T20:54:13.941771+0000","last_became_peered":"2023-09-23T20:54:13.941771+0000","last_unstale":"2023-09-23T20:58:15.136748+0000","last_undegraded":"2023-09-23T20:58:15.136748+0000","last_fullsized":"2023-09-23T20:58:15.136748+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:48:25.588116+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":137,"reported_epoch":232,"state":"active+clean","last_fresh":"2023-09-23T20:58:15.116143+0000","last_change":"2023-09-23T20:55:35.949780+0000","last_active":"2023-09-23T20:58:15.116143+0000","last_peered":"2023-09-23T20:58:15.116143+0000","last_clean":"2023-09-23T20:58:15.116143+0000","last_became_active":"2023-09-23T20:55:35.949497+0000","last_became_peered":"2023-09-23T20:55:35.949497+0000","last_unstale":"2023-09-23T20:58:15.116143+0000","last_undegraded":"2023-09-23T20:58:15.116143+0000","last_fullsized":"2023-09-23T20:58:15.116143+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:09:37.420882+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":195,"reported_epoch":232,"state":"active+clean","last_fresh":"2023-09-23T20:58:15.135302+0000","last_change":"2023-09-23T20:54:13.929196+0000","last_active":"2023-09-23T20:58:15.135302+0000","last_peered":"2023-09-23T20:58:15.135302+0000","last_clean":"2023-09-23T20:58:15.135302+0000","last_became_active":"2023-09-23T20:54:13.928831+0000","last_became_peered":"2023-09-23T20:54:13.928831+0000","last_unstale":"2023-09-23T20:58:15.135302+0000","last_undegraded":"2023-09-23T20:58:15.135302+0000","last_fullsized":"2023-09-23T20:58:15.135302+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:28:44.316851+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":201,"reported_epoch":232,"state":"active+clean","last_fresh":"2023-09-23T20:58:15.124073+0000","last_change":"2023-09-23T20:52:12.855430+0000","last_active":"2023-09-23T20:58:15.124073+0000","last_peered":"2023-09-23T20:58:15.124073+0000","last_clean":"2023-09-23T20:58:15.124073+0000","last_became_active":"2023-09-23T20:52:12.855115+0000","last_became_peered":"2023-09-23T20:52:12.855115+0000","last_unstale":"2023-09-23T20:58:15.124073+0000","last_undegraded":"2023-09-23T20:58:15.124073+0000","last_fullsized":"2023-09-23T20:58:15.124073+0000","mapping_epoch":100,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":101,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:33:23.802214+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":110,"reported_epoch":232,"state":"active+clean","last_fresh":"2023-09-23T20:58:15.117425+0000","last_change":"2023-09-23T20:54:31.902946+0000","last_active":"2023-09-23T20:58:15.117425+0000","last_peered":"2023-09-23T20:58:15.117425+0000","last_clean":"2023-09-23T20:58:15.117425+0000","last_became_active":"2023-09-23T20:54:31.889484+0000","last_became_peered":"2023-09-23T20:54:31.889484+0000","last_unstale":"2023-09-23T20:58:15.117425+0000","last_undegraded":"2023-09-23T20:58:15.117425+0000","last_fullsized":"2023-09-23T20:58:15.117425+0000","mapping_epoch":136,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:44:44.278398+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":110,"reported_epoch":232,"state":"active+clean","last_fresh":"2023-09-23T20:58:15.117975+0000","last_change":"2023-09-23T20:54:13.950351+0000","last_active":"2023-09-23T20:58:15.117975+0000","last_peered":"2023-09-23T20:58:15.117975+0000","last_clean":"2023-09-23T20:58:15.117975+0000","last_became_active":"2023-09-23T20:54:13.949987+0000","last_became_peered":"2023-09-23T20:54:13.949987+0000","last_unstale":"2023-09-23T20:58:15.117975+0000","last_undegraded":"2023-09-23T20:58:15.117975+0000","last_fullsized":"2023-09-23T20:58:15.117975+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:46:17.765134+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":141,"reported_epoch":232,"state":"active+clean","last_fresh":"2023-09-23T20:58:15.116343+0000","last_change":"2023-09-23T20:56:18.141728+0000","last_active":"2023-09-23T20:58:15.116343+0000","last_peered":"2023-09-23T20:58:15.116343+0000","last_clean":"2023-09-23T20:58:15.116343+0000","last_became_active":"2023-09-23T20:56:18.137979+0000","last_became_peered":"2023-09-23T20:56:18.137979+0000","last_unstale":"2023-09-23T20:58:15.116343+0000","last_undegraded":"2023-09-23T20:58:15.116343+0000","last_fullsized":"2023-09-23T20:58:15.116343+0000","mapping_epoch":192,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":193,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:58:05.658443+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":110,"reported_epoch":232,"state":"active+clean","last_fresh":"2023-09-23T20:58:15.117498+0000","last_change":"2023-09-23T20:54:13.935307+0000","last_active":"2023-09-23T20:58:15.117498+0000","last_peered":"2023-09-23T20:58:15.117498+0000","last_clean":"2023-09-23T20:58:15.117498+0000","last_became_active":"2023-09-23T20:54:13.935000+0000","last_became_peered":"2023-09-23T20:54:13.935000+0000","last_unstale":"2023-09-23T20:58:15.117498+0000","last_undegraded":"2023-09-23T20:58:15.117498+0000","last_fullsized":"2023-09-23T20:58:15.117498+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:47:40.974082+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":218,"reported_epoch":231,"state":"active+clean","last_fresh":"2023-09-23T20:58:11.112405+0000","last_change":"2023-09-23T20:55:35.954870+0000","last_active":"2023-09-23T20:58:11.112405+0000","last_peered":"2023-09-23T20:58:11.112405+0000","last_clean":"2023-09-23T20:58:11.112405+0000","last_became_active":"2023-09-23T20:55:35.954568+0000","last_became_peered":"2023-09-23T20:55:35.954568+0000","last_unstale":"2023-09-23T20:58:11.112405+0000","last_undegraded":"2023-09-23T20:58:11.112405+0000","last_fullsized":"2023-09-23T20:58:11.112405+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:06:22.423566+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":217,"reported_epoch":231,"state":"active+clean","last_fresh":"2023-09-23T20:58:11.088049+0000","last_change":"2023-09-23T20:51:10.838334+0000","last_active":"2023-09-23T20:58:11.088049+0000","last_peered":"2023-09-23T20:58:11.088049+0000","last_clean":"2023-09-23T20:58:11.088049+0000","last_became_active":"2023-09-23T20:51:10.838026+0000","last_became_peered":"2023-09-23T20:51:10.838026+0000","last_unstale":"2023-09-23T20:58:11.088049+0000","last_undegraded":"2023-09-23T20:58:11.088049+0000","last_fullsized":"2023-09-23T20:58:11.088049+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:04:37.009457+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":223,"reported_epoch":232,"state":"active+clean","last_fresh":"2023-09-23T20:58:15.145863+0000","last_change":"2023-09-23T20:54:31.903139+0000","last_active":"2023-09-23T20:58:15.145863+0000","last_peered":"2023-09-23T20:58:15.145863+0000","last_clean":"2023-09-23T20:58:15.145863+0000","last_became_active":"2023-09-23T20:54:31.884520+0000","last_became_peered":"2023-09-23T20:54:31.884520+0000","last_unstale":"2023-09-23T20:58:15.145863+0000","last_undegraded":"2023-09-23T20:58:15.145863+0000","last_fullsized":"2023-09-23T20:58:15.145863+0000","mapping_epoch":136,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:39:34.269705+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1108,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":127,"reported_epoch":232,"state":"active+clean","last_fresh":"2023-09-23T20:58:15.115821+0000","last_change":"2023-09-23T20:55:35.951572+0000","last_active":"2023-09-23T20:58:15.115821+0000","last_peered":"2023-09-23T20:58:15.115821+0000","last_clean":"2023-09-23T20:58:15.115821+0000","last_became_active":"2023-09-23T20:55:35.951294+0000","last_became_peered":"2023-09-23T20:55:35.951294+0000","last_unstale":"2023-09-23T20:58:15.115821+0000","last_undegraded":"2023-09-23T20:58:15.115821+0000","last_fullsized":"2023-09-23T20:58:15.115821+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:36:32.447210+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":7,"num_pg":16,"stat_sum":{"num_bytes":6151602200,"num_objects":93867,"num_object_clones":0,"num_object_copies":281601,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":10943,"num_objects_misplaced":4671,"num_objects_unfound":0,"num_objects_dirty":93867,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":109653,"num_write_kb":6512577,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":728,"num_bytes_recovered":47710208,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":18547355648,"data_stored":18547343432,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":107853,"ondisk_log_size":107853,"up":48,"acting":45,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":22,"num_bytes_recovered":5990,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":21,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3381,"ondisk_log_size":3381,"up":63,"acting":63,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201495,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2061808,"kb_used_data":1891176,"kb_used_omap":0,"kb_used_meta":170624,"kb_avail":91683344,"statfs":{"total":95995035648,"available":93883744256,"internally_reserved":0,"allocated":1936564224,"data_stored":1935740647,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":174718976},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:57:44 2023","interfaces":[{"interface":"back","average":{"1min":3.3239999999999998,"5min":1.613,"15min":0.90200000000000002},"min":{"1min":0.23100000000000001,"5min":0.21099999999999999,"15min":0.21099999999999999},"max":{"1min":43.994999999999997,"5min":43.994999999999997,"15min":43.994999999999997},"last":4.601},{"interface":"front","average":{"1min":3.448,"5min":1.847,"15min":1},"min":{"1min":0.35299999999999998,"5min":0.218,"15min":0.218},"max":{"1min":44.206000000000003,"5min":44.206000000000003,"15min":44.206000000000003},"last":4.5899999999999999}]},{"osd":1,"last update":"Sat Sep 23 20:58:18 2023","interfaces":[{"interface":"back","average":{"1min":1.895,"5min":1.9330000000000001,"15min":1.1439999999999999},"min":{"1min":0.29999999999999999,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":10.622,"5min":10.622,"15min":22.591999999999999},"last":0.40699999999999997},{"interface":"front","average":{"1min":1.8320000000000001,"5min":1.5720000000000001,"15min":0.98499999999999999},"min":{"1min":0.25800000000000001,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":7.5629999999999997,"5min":10.097,"15min":21.437000000000001},"last":0.32900000000000001}]},{"osd":2,"last update":"Sat Sep 23 20:57:50 2023","interfaces":[{"interface":"back","average":{"1min":1.3919999999999999,"5min":1.643,"15min":0.93799999999999994},"min":{"1min":0.26800000000000002,"5min":0.26200000000000001,"15min":0.24399999999999999},"max":{"1min":5.3929999999999998,"5min":9.4290000000000003,"15min":9.4290000000000003},"last":4.9930000000000003},{"interface":"front","average":{"1min":3.3420000000000001,"5min":1.978,"15min":1.071},"min":{"1min":0.29899999999999999,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":45.25,"5min":45.25,"15min":45.25},"last":2.1349999999999998}]},{"osd":3,"last update":"Sat Sep 23 20:57:25 2023","interfaces":[{"interface":"back","average":{"1min":1.286,"5min":3.0840000000000001,"15min":1.401},"min":{"1min":0.39700000000000002,"5min":0.314,"15min":0.314},"max":{"1min":3.9220000000000002,"5min":36.408000000000001,"15min":36.408000000000001},"last":9.9939999999999998},{"interface":"front","average":{"1min":1.3540000000000001,"5min":2.669,"15min":1.272},"min":{"1min":0.42199999999999999,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":5.7969999999999997,"5min":35.593000000000004,"15min":35.593000000000004},"last":2.3319999999999999}]},{"osd":4,"last update":"Sat Sep 23 20:58:18 2023","interfaces":[{"interface":"back","average":{"1min":0.69999999999999996,"5min":0.82699999999999996,"15min":0.60899999999999999},"min":{"1min":0.28299999999999997,"5min":0.23799999999999999,"15min":0.23699999999999999},"max":{"1min":4.5410000000000004,"5min":5.8920000000000003,"15min":5.8920000000000003},"last":0.36899999999999999},{"interface":"front","average":{"1min":0.70399999999999996,"5min":0.76100000000000001,"15min":0.61399999999999999},"min":{"1min":0.27100000000000002,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":4.4080000000000004,"5min":5.3200000000000003,"15min":5.3200000000000003},"last":0.38300000000000001}]},{"osd":5,"last update":"Sat Sep 23 20:58:18 2023","interfaces":[{"interface":"back","average":{"1min":0.69299999999999995,"5min":0.82799999999999996,"15min":0.63300000000000001},"min":{"1min":0.28499999999999998,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":2.5110000000000001,"5min":7.0389999999999997,"15min":7.0389999999999997},"last":0.34899999999999998},{"interface":"front","average":{"1min":0.72199999999999998,"5min":0.70699999999999996,"15min":0.61199999999999999},"min":{"1min":0.317,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":4.5999999999999996,"5min":4.5999999999999996,"15min":4.5999999999999996},"last":0.42299999999999999}]},{"osd":6,"last update":"Sat Sep 23 20:57:38 2023","interfaces":[{"interface":"back","average":{"1min":0.72099999999999997,"5min":0.67700000000000005,"15min":0.67000000000000004},"min":{"1min":0.34799999999999998,"5min":0.31,"15min":0.31},"max":{"1min":1.9039999999999999,"5min":3.1760000000000002,"15min":3.1760000000000002},"last":0.435},{"interface":"front","average":{"1min":0.753,"5min":0.79400000000000004,"15min":0.80100000000000005},"min":{"1min":0.32300000000000001,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":1.96,"5min":3.129,"15min":3.129},"last":0.45700000000000002}]}]},{"osd":6,"up_from":164,"seq":704374636579,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1961828,"kb_used_data":1861276,"kb_used_omap":10,"kb_used_meta":100533,"kb_avail":91783324,"statfs":{"total":95995035648,"available":93986123776,"internally_reserved":0,"allocated":1905946624,"data_stored":1905134948,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":10328,"internal_metadata":102946728},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:57:40 2023","interfaces":[{"interface":"back","average":{"1min":1.8799999999999999,"5min":1.0680000000000001,"15min":0.93200000000000005},"min":{"1min":0.20999999999999999,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":8.7319999999999993,"5min":8.7319999999999993,"15min":8.7319999999999993},"last":2.9740000000000002},{"interface":"front","average":{"1min":1.393,"5min":1.0649999999999999,"15min":1.01},"min":{"1min":0.29299999999999998,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":6.7110000000000003,"5min":6.7110000000000003,"15min":6.7110000000000003},"last":2.7469999999999999}]},{"osd":1,"last update":"Sat Sep 23 20:57:40 2023","interfaces":[{"interface":"back","average":{"1min":1.575,"5min":0.94599999999999995,"15min":0.84099999999999997},"min":{"1min":0.34599999999999997,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":6.6029999999999998,"5min":6.6029999999999998,"15min":6.6029999999999998},"last":2.1800000000000002},{"interface":"front","average":{"1min":1.04,"5min":0.90200000000000002,"15min":0.879},"min":{"1min":0.255,"5min":0.247,"15min":0.247},"max":{"1min":2.7389999999999999,"5min":3.5870000000000002,"15min":3.5870000000000002},"last":0.80800000000000005}]},{"osd":2,"last update":"Sat Sep 23 20:57:40 2023","interfaces":[{"interface":"back","average":{"1min":2.6850000000000001,"5min":1.8540000000000001,"15min":1.716},"min":{"1min":0.371,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":13.750999999999999,"5min":13.750999999999999,"15min":13.750999999999999},"last":4.694},{"interface":"front","average":{"1min":1.9079999999999999,"5min":1.2669999999999999,"15min":1.1599999999999999},"min":{"1min":0.39200000000000002,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":7.4279999999999999,"5min":7.4279999999999999,"15min":7.4279999999999999},"last":2.7679999999999998}]},{"osd":3,"last update":"Sat Sep 23 20:57:40 2023","interfaces":[{"interface":"back","average":{"1min":1.2070000000000001,"5min":1.135,"15min":1.123},"min":{"1min":0.33300000000000002,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":3.093,"5min":5.8739999999999997,"15min":5.8739999999999997},"last":0.36799999999999999},{"interface":"front","average":{"1min":1.7090000000000001,"5min":1.6240000000000001,"15min":1.6100000000000001},"min":{"1min":0.311,"5min":0.311,"15min":0.311},"max":{"1min":5.46,"5min":7.234,"15min":7.234},"last":1.9550000000000001}]},{"osd":4,"last update":"Sat Sep 23 20:57:40 2023","interfaces":[{"interface":"back","average":{"1min":0.65000000000000002,"5min":0.97099999999999997,"15min":1.0249999999999999},"min":{"1min":0.28299999999999997,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":1.9039999999999999,"5min":9.0079999999999991,"15min":9.0079999999999991},"last":0.41499999999999998},{"interface":"front","average":{"1min":0.45700000000000002,"5min":0.51600000000000001,"15min":0.52600000000000002},"min":{"1min":0.25800000000000001,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":1.3120000000000001,"5min":1.3120000000000001,"15min":1.3120000000000001},"last":0.40600000000000003}]},{"osd":5,"last update":"Sat Sep 23 20:57:40 2023","interfaces":[{"interface":"back","average":{"1min":0.68600000000000005,"5min":0.56999999999999995,"15min":0.55000000000000004},"min":{"1min":0.27900000000000003,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":1.9399999999999999,"5min":1.9399999999999999,"15min":1.9399999999999999},"last":0.438},{"interface":"front","average":{"1min":0.68899999999999995,"5min":0.78500000000000003,"15min":0.80100000000000005},"min":{"1min":0.30599999999999999,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":1.9159999999999999,"5min":2.274,"15min":2.274},"last":0.33400000000000002}]},{"osd":7,"last update":"Sat Sep 23 20:57:40 2023","interfaces":[{"interface":"back","average":{"1min":0.95799999999999996,"5min":0.93400000000000005,"15min":0.93000000000000005},"min":{"1min":0.33500000000000002,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":7.8570000000000002,"5min":7.8570000000000002,"15min":7.8570000000000002},"last":0.38300000000000001},{"interface":"front","average":{"1min":1.103,"5min":0.64400000000000002,"15min":0.56799999999999995},"min":{"1min":0.33400000000000002,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":8.282,"5min":8.282,"15min":8.282},"last":0.35099999999999998}]}]},{"osd":1,"up_from":88,"seq":377957122150,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2775764,"kb_used_data":2577676,"kb_used_omap":9,"kb_used_meta":198070,"kb_avail":90969388,"statfs":{"total":95995035648,"available":93152653312,"internally_reserved":0,"allocated":2639540224,"data_stored":2638396514,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9239,"internal_metadata":202824681},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:58:22 2023","interfaces":[{"interface":"back","average":{"1min":3.3290000000000002,"5min":2.9990000000000001,"15min":1.3049999999999999},"min":{"1min":0.23200000000000001,"5min":0.216,"15min":0.216},"max":{"1min":10.73,"5min":16.646999999999998,"15min":16.646999999999998},"last":8.4139999999999997},{"interface":"front","average":{"1min":4.2149999999999999,"5min":3.3940000000000001,"15min":1.4590000000000001},"min":{"1min":0.247,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":15.135999999999999,"5min":18.457999999999998,"15min":18.457999999999998},"last":4.4420000000000002}]},{"osd":2,"last update":"Sat Sep 23 20:57:50 2023","interfaces":[{"interface":"back","average":{"1min":2.5760000000000001,"5min":2.1309999999999998,"15min":1.0249999999999999},"min":{"1min":0.221,"5min":0.20300000000000001,"15min":0.20300000000000001},"max":{"1min":8.6500000000000004,"5min":14.335000000000001,"15min":14.335000000000001},"last":8.4550000000000001},{"interface":"front","average":{"1min":3.2770000000000001,"5min":3.0209999999999999,"15min":1.4099999999999999},"min":{"1min":0.35699999999999998,"5min":0.248,"15min":0.248},"max":{"1min":10.095000000000001,"5min":38.951000000000001,"15min":38.951000000000001},"last":5.2809999999999997}]},{"osd":3,"last update":"Sat Sep 23 20:58:22 2023","interfaces":[{"interface":"back","average":{"1min":3.1859999999999999,"5min":3.0489999999999999,"15min":1.3720000000000001},"min":{"1min":0.246,"5min":0.246,"15min":0.246},"max":{"1min":10.701000000000001,"5min":25.314,"15min":25.314},"last":2.5},{"interface":"front","average":{"1min":2.8450000000000002,"5min":2.6150000000000002,"15min":1.232},"min":{"1min":0.33800000000000002,"5min":0.20599999999999999,"15min":0.20599999999999999},"max":{"1min":7.8899999999999997,"5min":19.881,"15min":19.881},"last":2.468}]},{"osd":4,"last update":"Sat Sep 23 20:58:22 2023","interfaces":[{"interface":"back","average":{"1min":1.7050000000000001,"5min":1.9550000000000001,"15min":1.131},"min":{"1min":0.36099999999999999,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":10.598000000000001,"5min":12.006,"15min":12.006},"last":1.27},{"interface":"front","average":{"1min":2.0899999999999999,"5min":1.6839999999999999,"15min":1.016},"min":{"1min":0.31900000000000001,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":6.0460000000000003,"5min":11.234999999999999,"15min":11.234999999999999},"last":2.161}]},{"osd":5,"last update":"Sat Sep 23 20:58:22 2023","interfaces":[{"interface":"back","average":{"1min":3.0609999999999999,"5min":2.133,"15min":1.181},"min":{"1min":0.34399999999999997,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":15.041,"5min":15.041,"15min":15.041},"last":4.319},{"interface":"front","average":{"1min":1.669,"5min":2.0169999999999999,"15min":1.153},"min":{"1min":0.33200000000000002,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":10.647,"5min":12.08,"15min":12.08},"last":1.1950000000000001}]},{"osd":6,"last update":"Sat Sep 23 20:57:35 2023","interfaces":[{"interface":"back","average":{"1min":1.2929999999999999,"5min":1.3100000000000001,"15min":1.3129999999999999},"min":{"1min":0.39000000000000001,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":5.3810000000000002,"5min":5.3810000000000002,"15min":5.3810000000000002},"last":1.2370000000000001},{"interface":"front","average":{"1min":1.9530000000000001,"5min":1.393,"15min":1.2989999999999999},"min":{"1min":0.439,"5min":0.309,"15min":0.309},"max":{"1min":6.0860000000000003,"5min":6.5670000000000002,"15min":6.5670000000000002},"last":2.2029999999999998}]},{"osd":7,"last update":"Sat Sep 23 20:58:22 2023","interfaces":[{"interface":"back","average":{"1min":2.0659999999999998,"5min":1.7909999999999999,"15min":1.069},"min":{"1min":0.42499999999999999,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":6.149,"5min":11.186999999999999,"15min":11.186999999999999},"last":2.1219999999999999},{"interface":"front","average":{"1min":3.1520000000000001,"5min":2.1829999999999998,"15min":1.2010000000000001},"min":{"1min":0.41899999999999998,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":15.098000000000001,"5min":15.098000000000001,"15min":15.098000000000001},"last":4.3559999999999999}]}]},{"osd":0,"up_from":83,"seq":356482285676,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3190268,"kb_used_data":3032244,"kb_used_omap":28,"kb_used_meta":157923,"kb_avail":90554884,"statfs":{"total":95995035648,"available":92728201216,"internally_reserved":0,"allocated":3105017856,"data_stored":3103896355,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":28848,"internal_metadata":161714000},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,5,1,1],"upper_bound":128},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 20:58:19 2023","interfaces":[{"interface":"back","average":{"1min":3.7799999999999998,"5min":2.9220000000000002,"15min":1.2609999999999999},"min":{"1min":0.25800000000000001,"5min":0.191,"15min":0.191},"max":{"1min":15.946999999999999,"5min":18.863,"15min":18.863},"last":7.6310000000000002},{"interface":"front","average":{"1min":3.9060000000000001,"5min":2.8540000000000001,"15min":1.2669999999999999},"min":{"1min":0.247,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":15.766999999999999,"5min":17.704999999999998,"15min":17.704999999999998},"last":4.2809999999999997}]},{"osd":2,"last update":"Sat Sep 23 20:57:47 2023","interfaces":[{"interface":"back","average":{"1min":2.6150000000000002,"5min":3.0329999999999999,"15min":1.3400000000000001},"min":{"1min":0.35599999999999998,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":8.4949999999999992,"5min":17.753,"15min":17.753},"last":0.434},{"interface":"front","average":{"1min":2.843,"5min":2.8199999999999998,"15min":1.234},"min":{"1min":0.308,"5min":0.21099999999999999,"15min":0.21099999999999999},"max":{"1min":9.9469999999999992,"5min":13.218999999999999,"15min":13.218999999999999},"last":0.55000000000000004}]},{"osd":3,"last update":"Sat Sep 23 20:58:19 2023","interfaces":[{"interface":"back","average":{"1min":3.722,"5min":2.8919999999999999,"15min":1.298},"min":{"1min":0.39300000000000002,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":15.994,"5min":18.893999999999998,"15min":18.893999999999998},"last":0.88400000000000001},{"interface":"front","average":{"1min":3.7930000000000001,"5min":3.0649999999999999,"15min":1.423},"min":{"1min":0.29699999999999999,"5min":0.28000000000000003,"15min":0.23300000000000001},"max":{"1min":15.808,"5min":17.472000000000001,"15min":17.472000000000001},"last":0.5}]},{"osd":4,"last update":"Sat Sep 23 20:57:47 2023","interfaces":[{"interface":"back","average":{"1min":1.968,"5min":1.881,"15min":1.0740000000000001},"min":{"1min":0.35099999999999998,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":9.9079999999999995,"5min":12.807,"15min":12.807},"last":0.52200000000000002},{"interface":"front","average":{"1min":1.1659999999999999,"5min":1.643,"15min":0.997},"min":{"1min":0.311,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":4.7850000000000001,"5min":18.724,"15min":18.724},"last":0.93799999999999994}]},{"osd":5,"last update":"Sat Sep 23 20:57:47 2023","interfaces":[{"interface":"back","average":{"1min":1.7669999999999999,"5min":1.8129999999999999,"15min":1.069},"min":{"1min":0.33600000000000002,"5min":0.318,"15min":0.318},"max":{"1min":6.673,"5min":8.4990000000000006,"15min":8.4990000000000006},"last":0.46899999999999997},{"interface":"front","average":{"1min":2.2440000000000002,"5min":2.0600000000000001,"15min":1.1619999999999999},"min":{"1min":0.39000000000000001,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":9.8599999999999994,"5min":13.154999999999999,"15min":13.154999999999999},"last":0.57799999999999996}]},{"osd":6,"last update":"Sat Sep 23 20:57:41 2023","interfaces":[{"interface":"back","average":{"1min":1.8160000000000001,"5min":1.6879999999999999,"15min":1.667},"min":{"1min":0.37,"5min":0.37,"15min":0.37},"max":{"1min":6.8470000000000004,"5min":8.7309999999999999,"15min":8.7309999999999999},"last":0.97199999999999998},{"interface":"front","average":{"1min":1.875,"5min":1.7709999999999999,"15min":1.754},"min":{"1min":0.377,"5min":0.377,"15min":0.377},"max":{"1min":5.492,"5min":8.6760000000000002,"15min":8.6760000000000002},"last":1.032}]},{"osd":7,"last update":"Sat Sep 23 20:57:47 2023","interfaces":[{"interface":"back","average":{"1min":1.786,"5min":1.8220000000000001,"15min":1.1020000000000001},"min":{"1min":0.41599999999999998,"5min":0.379,"15min":0.379},"max":{"1min":6.6390000000000002,"5min":8.4339999999999993,"15min":8.4339999999999993},"last":0.65300000000000002},{"interface":"front","average":{"1min":1.393,"5min":1.698,"15min":1.0269999999999999},"min":{"1min":0.40100000000000002,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":4.6799999999999997,"5min":18.673999999999999,"15min":18.673999999999999},"last":0.999}]}]},{"osd":2,"up_from":93,"seq":399431958623,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2902828,"kb_used_data":2607268,"kb_used_omap":81,"kb_used_meta":295406,"kb_avail":90842324,"statfs":{"total":95995035648,"available":93022539776,"internally_reserved":0,"allocated":2669842432,"data_stored":2668733679,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":83750,"internal_metadata":302495962},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,1,0,3],"upper_bound":64},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:57:43 2023","interfaces":[{"interface":"back","average":{"1min":2.2090000000000001,"5min":1.9319999999999999,"15min":0.89400000000000002},"min":{"1min":0.27100000000000002,"5min":0.17999999999999999,"15min":0.17999999999999999},"max":{"1min":9.7460000000000004,"5min":11.183,"15min":11.183},"last":5.8380000000000001},{"interface":"front","average":{"1min":3.0449999999999999,"5min":2.5110000000000001,"15min":1.1160000000000001},"min":{"1min":0.216,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":10.891999999999999,"5min":12.641,"15min":12.641},"last":5.7699999999999996}]},{"osd":1,"last update":"Sat Sep 23 20:57:43 2023","interfaces":[{"interface":"back","average":{"1min":2.23,"5min":2.8119999999999998,"15min":1.2310000000000001},"min":{"1min":0.21199999999999999,"5min":0.20200000000000001,"15min":0.20200000000000001},"max":{"1min":8.2889999999999997,"5min":16.081,"15min":16.081},"last":5.9729999999999999},{"interface":"front","average":{"1min":2.9660000000000002,"5min":3.419,"15min":1.4390000000000001},"min":{"1min":0.26300000000000001,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":8.8960000000000008,"5min":16.041,"15min":16.041},"last":5.8090000000000002}]},{"osd":3,"last update":"Sat Sep 23 20:57:22 2023","interfaces":[{"interface":"back","average":{"1min":1.891,"5min":2.9870000000000001,"15min":1.2709999999999999},"min":{"1min":0.36299999999999999,"5min":0.26800000000000002,"15min":0.255},"max":{"1min":10.488,"5min":16.611000000000001,"15min":16.611000000000001},"last":5.7039999999999997},{"interface":"front","average":{"1min":1.835,"5min":3.6760000000000002,"15min":1.478},"min":{"1min":0.28000000000000003,"5min":0.22800000000000001,"15min":0.221},"max":{"1min":6.6059999999999999,"5min":17.138999999999999,"15min":17.138999999999999},"last":1.7350000000000001}]},{"osd":4,"last update":"Sat Sep 23 20:57:43 2023","interfaces":[{"interface":"back","average":{"1min":1.8660000000000001,"5min":1.851,"15min":1.0349999999999999},"min":{"1min":0.318,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":8.657,"5min":9.8119999999999994,"15min":9.8119999999999994},"last":5.6619999999999999},{"interface":"front","average":{"1min":1.9770000000000001,"5min":2.0190000000000001,"15min":1.1299999999999999},"min":{"1min":0.34300000000000003,"5min":0.313,"15min":0.313},"max":{"1min":6.7850000000000001,"5min":10.609,"15min":10.609},"last":1.7569999999999999}]},{"osd":5,"last update":"Sat Sep 23 20:57:43 2023","interfaces":[{"interface":"back","average":{"1min":1.823,"5min":2.121,"15min":1.145},"min":{"1min":0.29299999999999998,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":8.5700000000000003,"5min":9.7629999999999999,"15min":9.7629999999999999},"last":5.7389999999999999},{"interface":"front","average":{"1min":2.0259999999999998,"5min":2.129,"15min":1.147},"min":{"1min":0.32600000000000001,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":6.7510000000000003,"5min":10.574,"15min":10.574},"last":1.6759999999999999}]},{"osd":6,"last update":"Sat Sep 23 20:57:41 2023","interfaces":[{"interface":"back","average":{"1min":1.9450000000000001,"5min":1.7230000000000001,"15min":1.6859999999999999},"min":{"1min":0.312,"5min":0.312,"15min":0.312},"max":{"1min":6.8109999999999999,"5min":14.244999999999999,"15min":14.244999999999999},"last":1.7110000000000001},{"interface":"front","average":{"1min":1.9390000000000001,"5min":1.2110000000000001,"15min":1.0900000000000001},"min":{"1min":0.374,"5min":0.374,"15min":0.374},"max":{"1min":6.835,"5min":6.835,"15min":6.835},"last":1.7789999999999999}]},{"osd":7,"last update":"Sat Sep 23 20:57:43 2023","interfaces":[{"interface":"back","average":{"1min":1.397,"5min":1.524,"15min":0.93799999999999994},"min":{"1min":0.32500000000000001,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":4.9820000000000002,"5min":8.0820000000000007,"15min":8.0820000000000007},"last":5.8570000000000002},{"interface":"front","average":{"1min":1.804,"5min":2.1360000000000001,"15min":1.159},"min":{"1min":0.35699999999999998,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":8.6219999999999999,"5min":9.7859999999999996,"15min":9.7859999999999996},"last":5.7930000000000001}]}]},{"osd":3,"up_from":98,"seq":420906795096,"num_pgs":49,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2775804,"kb_used_data":2484340,"kb_used_omap":7,"kb_used_meta":291448,"kb_avail":90969348,"statfs":{"total":95995035648,"available":93152612352,"internally_reserved":0,"allocated":2543964160,"data_stored":2542892000,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7576,"internal_metadata":298443368},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,1,0,0,2,1,2],"upper_bound":128},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:57:22 2023","interfaces":[{"interface":"back","average":{"1min":1.2130000000000001,"5min":3.181,"15min":1.3069999999999999},"min":{"1min":0.23100000000000001,"5min":0.20599999999999999,"15min":0.20599999999999999},"max":{"1min":8.5289999999999999,"5min":23.123999999999999,"15min":23.123999999999999},"last":8.4260000000000002},{"interface":"front","average":{"1min":1.552,"5min":2.6619999999999999,"15min":1.1259999999999999},"min":{"1min":0.217,"5min":0.20000000000000001,"15min":0.20000000000000001},"max":{"1min":6.3819999999999997,"5min":15.079000000000001,"15min":15.079000000000001},"last":4.5590000000000002}]},{"osd":1,"last update":"Sat Sep 23 20:57:22 2023","interfaces":[{"interface":"back","average":{"1min":1.833,"5min":2.9020000000000001,"15min":1.212},"min":{"1min":0.27500000000000002,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":7.6879999999999997,"5min":21.385000000000002,"15min":21.385000000000002},"last":3.6549999999999998},{"interface":"front","average":{"1min":1.5609999999999999,"5min":3.2160000000000002,"15min":1.331},"min":{"1min":0.22600000000000001,"5min":0.218,"15min":0.218},"max":{"1min":9.9190000000000005,"5min":25.353000000000002,"15min":25.353000000000002},"last":4.2439999999999998}]},{"osd":2,"last update":"Sat Sep 23 20:57:22 2023","interfaces":[{"interface":"back","average":{"1min":2.0939999999999999,"5min":2.5979999999999999,"15min":1.149},"min":{"1min":0.34300000000000003,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":6.7679999999999998,"5min":24.154,"15min":24.154},"last":8.1940000000000008},{"interface":"front","average":{"1min":1.591,"5min":3.371,"15min":1.4159999999999999},"min":{"1min":0.32900000000000001,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":6.2359999999999998,"5min":25.305,"15min":25.305},"last":4.742}]},{"osd":4,"last update":"Sat Sep 23 20:57:22 2023","interfaces":[{"interface":"back","average":{"1min":1.786,"5min":1.756,"15min":1.006},"min":{"1min":0.38600000000000001,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":6.6790000000000003,"5min":11.861000000000001,"15min":11.861000000000001},"last":3.7280000000000002},{"interface":"front","average":{"1min":0.995,"5min":2.339,"15min":1.1970000000000001},"min":{"1min":0.29499999999999998,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":6.2800000000000002,"5min":22.853000000000002,"15min":22.853000000000002},"last":4.165}]},{"osd":5,"last update":"Sat Sep 23 20:57:22 2023","interfaces":[{"interface":"back","average":{"1min":1.819,"5min":1.6839999999999999,"15min":0.99199999999999999},"min":{"1min":0.41299999999999998,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":6.7169999999999996,"5min":11.891999999999999,"15min":11.891999999999999},"last":3.6949999999999998},{"interface":"front","average":{"1min":1.337,"5min":1.673,"15min":0.98199999999999998},"min":{"1min":0.35099999999999998,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":6.3419999999999996,"5min":9.2870000000000008,"15min":9.2870000000000008},"last":4.6070000000000002}]},{"osd":6,"last update":"Sat Sep 23 20:57:39 2023","interfaces":[{"interface":"back","average":{"1min":1.173,"5min":0.99399999999999999,"15min":0.96399999999999997},"min":{"1min":0.40999999999999998,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":6.3079999999999998,"5min":6.3079999999999998,"15min":6.3079999999999998},"last":4.2110000000000003},{"interface":"front","average":{"1min":1.6879999999999999,"5min":1.4670000000000001,"15min":1.4299999999999999},"min":{"1min":0.40100000000000002,"5min":0.40100000000000002,"15min":0.40100000000000002},"max":{"1min":5.8230000000000004,"5min":7.234,"15min":7.234},"last":4.6449999999999996}]},{"osd":7,"last update":"Sat Sep 23 20:57:22 2023","interfaces":[{"interface":"back","average":{"1min":1.9379999999999999,"5min":2.113,"15min":1.167},"min":{"1min":0.47099999999999997,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":6.7450000000000001,"5min":16.364999999999998,"15min":16.364999999999998},"last":3.7570000000000001},{"interface":"front","average":{"1min":1.2,"5min":1.7509999999999999,"15min":1.046},"min":{"1min":0.442,"5min":0.379,"15min":0.379},"max":{"1min":2.9249999999999998,"5min":9.3179999999999996,"15min":9.3179999999999996},"last":4.6749999999999998}]}]},{"osd":4,"up_from":31,"seq":133143986404,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2009072,"kb_used_data":1881064,"kb_used_omap":0,"kb_used_meta":128000,"kb_avail":91736080,"statfs":{"total":95995035648,"available":93937745920,"internally_reserved":0,"allocated":1926209536,"data_stored":1925386212,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":131072000},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,0,0,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:57:52 2023","interfaces":[{"interface":"back","average":{"1min":2.4359999999999999,"5min":1.593,"15min":0.95599999999999996},"min":{"1min":0.214,"5min":0.214,"15min":0.214},"max":{"1min":11.784000000000001,"5min":11.784000000000001,"15min":11.784000000000001},"last":0.28199999999999997},{"interface":"front","average":{"1min":2.2970000000000002,"5min":1.782,"15min":1.1699999999999999},"min":{"1min":0.28100000000000003,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":11.766,"5min":11.766,"15min":11.766},"last":0.314}]},{"osd":1,"last update":"Sat Sep 23 20:58:21 2023","interfaces":[{"interface":"back","average":{"1min":1.7370000000000001,"5min":1.641,"15min":0.996},"min":{"1min":0.246,"5min":0.246,"15min":0.23599999999999999},"max":{"1min":10.153,"5min":10.153,"15min":10.153},"last":3.645},{"interface":"front","average":{"1min":2.645,"5min":1.649,"15min":0.96399999999999997},"min":{"1min":0.26100000000000001,"5min":0.26100000000000001,"15min":0.23200000000000001},"max":{"1min":12.292999999999999,"5min":12.292999999999999,"15min":12.292999999999999},"last":3.8100000000000001}]},{"osd":2,"last update":"Sat Sep 23 20:57:52 2023","interfaces":[{"interface":"back","average":{"1min":1.5840000000000001,"5min":1.746,"15min":1.036},"min":{"1min":0.35699999999999998,"5min":0.311,"15min":0.29099999999999998},"max":{"1min":5.21,"5min":15.288,"15min":15.288},"last":1.095},{"interface":"front","average":{"1min":1.8939999999999999,"5min":2.0720000000000001,"15min":1.2050000000000001},"min":{"1min":0.39500000000000002,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":3.8300000000000001,"5min":15.262,"15min":15.262},"last":0.76900000000000002}]},{"osd":3,"last update":"Sat Sep 23 20:58:21 2023","interfaces":[{"interface":"back","average":{"1min":2.0209999999999999,"5min":2.504,"15min":1.514},"min":{"1min":0.28899999999999998,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":7.7549999999999999,"5min":15.916,"15min":31.562999999999999},"last":3.2850000000000001},{"interface":"front","average":{"1min":2.2080000000000002,"5min":2.5449999999999999,"15min":1.589},"min":{"1min":0.433,"5min":0.433,"15min":0.41899999999999998},"max":{"1min":7.7300000000000004,"5min":15.882,"15min":31.574000000000002},"last":3.3119999999999998}]},{"osd":5,"last update":"Sat Sep 23 20:57:34 2023","interfaces":[{"interface":"back","average":{"1min":0.74199999999999999,"5min":0.53700000000000003,"15min":0.50600000000000001},"min":{"1min":0.248,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":3.008,"5min":3.3279999999999998,"15min":3.3279999999999998},"last":0.372},{"interface":"front","average":{"1min":0.66300000000000003,"5min":0.68300000000000005,"15min":0.55800000000000005},"min":{"1min":0.27300000000000002,"5min":0.251,"15min":0.251},"max":{"1min":3.1280000000000001,"5min":7.7619999999999996,"15min":7.7619999999999996},"last":0.44500000000000001}]},{"osd":6,"last update":"Sat Sep 23 20:57:40 2023","interfaces":[{"interface":"back","average":{"1min":1.032,"5min":0.59299999999999997,"15min":0.52000000000000002},"min":{"1min":0.26900000000000002,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":4.6909999999999998,"5min":4.6909999999999998,"15min":4.6909999999999998},"last":0.66700000000000004},{"interface":"front","average":{"1min":0.82499999999999996,"5min":0.68000000000000005,"15min":0.65600000000000003},"min":{"1min":0.38900000000000001,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":2.7120000000000002,"5min":2.7120000000000002,"15min":2.7120000000000002},"last":0.64500000000000002}]},{"osd":7,"last update":"Sat Sep 23 20:58:20 2023","interfaces":[{"interface":"back","average":{"1min":0.84199999999999997,"5min":0.83099999999999996,"15min":0.67100000000000004},"min":{"1min":0.442,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":3.327,"5min":7.383,"15min":7.383},"last":0.47099999999999997},{"interface":"front","average":{"1min":0.93899999999999995,"5min":0.80800000000000005,"15min":0.66000000000000003},"min":{"1min":0.36399999999999999,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":3.883,"5min":7.8310000000000004,"15min":7.8310000000000004},"last":0.54400000000000004}]}]},{"osd":5,"up_from":39,"seq":167503724769,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1984484,"kb_used_data":1853532,"kb_used_omap":0,"kb_used_meta":130944,"kb_avail":91760668,"statfs":{"total":95995035648,"available":93962924032,"internally_reserved":0,"allocated":1898016768,"data_stored":1897202020,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":134086656},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:57:36 2023","interfaces":[{"interface":"back","average":{"1min":1.145,"5min":1.1200000000000001,"15min":0.70799999999999996},"min":{"1min":0.20399999999999999,"5min":0.20399999999999999,"15min":0.20399999999999999},"max":{"1min":7.5019999999999998,"5min":7.766,"15min":7.766},"last":1.2789999999999999},{"interface":"front","average":{"1min":1.647,"5min":1.556,"15min":0.89900000000000002},"min":{"1min":0.25600000000000001,"5min":0.223,"15min":0.223},"max":{"1min":11.378,"5min":15.037000000000001,"15min":15.037000000000001},"last":3.879}]},{"osd":1,"last update":"Sat Sep 23 20:58:16 2023","interfaces":[{"interface":"back","average":{"1min":1.5549999999999999,"5min":1.603,"15min":0.95199999999999996},"min":{"1min":0.27100000000000002,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":6.7409999999999997,"5min":8.3520000000000003,"15min":8.3520000000000003},"last":6.7409999999999997},{"interface":"front","average":{"1min":1.9590000000000001,"5min":1.7450000000000001,"15min":0.97699999999999998},"min":{"1min":0.28100000000000003,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":7.4669999999999996,"5min":10.285,"15min":10.285},"last":2.762}]},{"osd":2,"last update":"Sat Sep 23 20:57:50 2023","interfaces":[{"interface":"back","average":{"1min":1.2410000000000001,"5min":1.575,"15min":0.88500000000000001},"min":{"1min":0.26200000000000001,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":10.579000000000001,"5min":12.651,"15min":12.651},"last":1.2410000000000001},{"interface":"front","average":{"1min":1.0669999999999999,"5min":1.5449999999999999,"15min":0.91900000000000004},"min":{"1min":0.26100000000000001,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":5.2949999999999999,"5min":8.3209999999999997,"15min":8.3209999999999997},"last":1.0720000000000001}]},{"osd":3,"last update":"Sat Sep 23 20:58:16 2023","interfaces":[{"interface":"back","average":{"1min":2.331,"5min":2.0670000000000002,"15min":1.153},"min":{"1min":0.39200000000000002,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":8.6579999999999995,"5min":9.5489999999999995,"15min":9.5489999999999995},"last":0.67300000000000004},{"interface":"front","average":{"1min":1.6000000000000001,"5min":1.6040000000000001,"15min":0.97499999999999998},"min":{"1min":0.28899999999999998,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":10.457000000000001,"5min":10.457000000000001,"15min":10.457000000000001},"last":3.9969999999999999}]},{"osd":4,"last update":"Sat Sep 23 20:57:29 2023","interfaces":[{"interface":"back","average":{"1min":0.49099999999999999,"5min":0.60199999999999998,"15min":0.52800000000000002},"min":{"1min":0.29699999999999999,"5min":0.27600000000000002,"15min":0.253},"max":{"1min":1.4239999999999999,"5min":3.8479999999999999,"15min":3.8479999999999999},"last":0.40899999999999997},{"interface":"front","average":{"1min":0.56299999999999994,"5min":0.63900000000000001,"15min":0.55600000000000005},"min":{"1min":0.23599999999999999,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":1.389,"5min":3.0680000000000001,"15min":3.0680000000000001},"last":0.29899999999999999}]},{"osd":6,"last update":"Sat Sep 23 20:57:36 2023","interfaces":[{"interface":"back","average":{"1min":0.80800000000000005,"5min":0.60999999999999999,"15min":0.57699999999999996},"min":{"1min":0.28000000000000003,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":2.9820000000000002,"5min":2.9820000000000002,"15min":2.9820000000000002},"last":0.34000000000000002},{"interface":"front","average":{"1min":0.76400000000000001,"5min":0.748,"15min":0.745},"min":{"1min":0.33200000000000002,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":2.6579999999999999,"5min":3.6560000000000001,"15min":3.6560000000000001},"last":0.44400000000000001}]},{"osd":7,"last update":"Sat Sep 23 20:58:13 2023","interfaces":[{"interface":"back","average":{"1min":0.66500000000000004,"5min":0.76600000000000001,"15min":0.63200000000000001},"min":{"1min":0.316,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":2.4809999999999999,"5min":8.5649999999999995,"15min":8.5649999999999995},"last":0.48699999999999999},{"interface":"front","average":{"1min":0.60299999999999998,"5min":0.74099999999999999,"15min":0.64400000000000002},"min":{"1min":0.35399999999999998,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":2.7109999999999999,"5min":4.5430000000000001,"15min":4.5430000000000001},"last":0.51800000000000002}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":3093499904,"data_stored":3093495832,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":2622427136,"data_stored":2622423064,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":2662203392,"data_stored":2662203392,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":2537226240,"data_stored":2537226240,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1916469248,"data_stored":1916469248,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1889861632,"data_stored":1889861632,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1898713088,"data_stored":1898709016,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1926955008,"data_stored":1926955008,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T20:58:25.713 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph status --format=json 2023-09-23T20:58:25.789 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:58:25.890 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:58:25.953 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:58:26.076 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:58:26.085 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:58:26.085 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:58:26.128 INFO:teuthology.orchestra.run.smithi026.stderr:nodeep-scrub is set 2023-09-23T20:58:26.186 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:58:26.222 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:58:26 smithi026 ceph-mon[175151]: pgmap v551: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 5.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 594 op/s; 10943/281673 objects degraded (3.885%); 4671/281673 objects misplaced (1.658%); 1.4 MiB/s, 22 objects/s recovering 2023-09-23T20:58:26.222 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:58:26 smithi026 ceph-mon[175151]: from='client.55069 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:58:26.222 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:58:26 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/2472024391' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:58:26.222 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:58:26 smithi026 ceph-mon[179688]: pgmap v551: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 5.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 594 op/s; 10943/281673 objects degraded (3.885%); 4671/281673 objects misplaced (1.658%); 1.4 MiB/s, 22 objects/s recovering 2023-09-23T20:58:26.222 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:58:26 smithi026 ceph-mon[179688]: from='client.55069 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:58:26.223 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:58:26 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/2472024391' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:58:26.232 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:58:26 smithi070 ceph-mon[168282]: pgmap v551: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 5.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 594 op/s; 10943/281673 objects degraded (3.885%); 4671/281673 objects misplaced (1.658%); 1.4 MiB/s, 22 objects/s recovering 2023-09-23T20:58:26.232 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:58:26 smithi070 ceph-mon[168282]: from='client.55069 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:58:26.232 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:58:26 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/2472024391' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:58:26.287 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:58:26.543 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:58:26.546 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:58:26.547 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:58:26.648 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:58:26.692 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:58:26.793 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:58:26.798 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:58:26.798 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:58:26.899 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:58:27.013 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:58:27.053 INFO:teuthology.orchestra.run.smithi070.stdout:ERROR: (22) Invalid argument 2023-09-23T20:58:27.053 INFO:teuthology.orchestra.run.smithi070.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:58:27.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:58:27 smithi070 ceph-mon[168282]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:58:27.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:58:27 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/2472024391' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T20:58:27.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:58:27 smithi070 ceph-mon[168282]: osdmap e234: 8 total, 8 up, 8 in 2023-09-23T20:58:27.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:58:27 smithi070 ceph-mon[168282]: failed to encode map e234 with expected crc 2023-09-23T20:58:27.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:58:27 smithi070 ceph-mon[168282]: failed to encode map e234 with expected crc 2023-09-23T20:58:27.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:58:27 smithi070 ceph-mon[168282]: failed to encode map e234 with expected crc 2023-09-23T20:58:27.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:58:27 smithi070 ceph-mon[168282]: failed to encode map e234 with expected crc 2023-09-23T20:58:27.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:58:27 smithi026 ceph-mon[179688]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:58:27.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:58:27 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/2472024391' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T20:58:27.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:58:27 smithi026 ceph-mon[179688]: osdmap e234: 8 total, 8 up, 8 in 2023-09-23T20:58:27.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:58:27 smithi026 ceph-mon[179688]: failed to encode map e234 with expected crc 2023-09-23T20:58:27.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:58:27 smithi026 ceph-mon[179688]: failed to encode map e234 with expected crc 2023-09-23T20:58:27.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:58:27 smithi026 ceph-mon[179688]: failed to encode map e234 with expected crc 2023-09-23T20:58:27.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:58:27 smithi026 ceph-mon[179688]: failed to encode map e234 with expected crc 2023-09-23T20:58:27.383 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:58:27 smithi026 ceph-mon[175151]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:58:27.383 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:58:27 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/2472024391' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T20:58:27.383 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:58:27 smithi026 ceph-mon[175151]: osdmap e234: 8 total, 8 up, 8 in 2023-09-23T20:58:27.383 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:58:27 smithi026 ceph-mon[175151]: failed to encode map e234 with expected crc 2023-09-23T20:58:27.383 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:58:27 smithi026 ceph-mon[175151]: failed to encode map e234 with expected crc 2023-09-23T20:58:27.383 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:58:27 smithi026 ceph-mon[175151]: failed to encode map e234 with expected crc 2023-09-23T20:58:27.383 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:58:27 smithi026 ceph-mon[175151]: failed to encode map e234 with expected crc 2023-09-23T20:58:27.550 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_historic_ops 2023-09-23T20:58:28.239 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:58:28.458 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:58:28 smithi070 ceph-mon[168282]: pgmap v553: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 5.7 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 632 op/s; 10943/277836 objects degraded (3.939%); 4563/277836 objects misplaced (1.642%); 1.6 MiB/s, 25 objects/s recovering 2023-09-23T20:58:28.620 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:58:28.628 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:58:28.629 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:58:28.629 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:58:28 smithi026 ceph-mon[175151]: pgmap v553: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 5.7 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 632 op/s; 10943/277836 objects degraded (3.939%); 4563/277836 objects misplaced (1.642%); 1.6 MiB/s, 25 objects/s recovering 2023-09-23T20:58:28.629 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:58:28 smithi026 ceph-mon[179688]: pgmap v553: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 5.7 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 632 op/s; 10943/277836 objects degraded (3.939%); 4563/277836 objects misplaced (1.642%); 1.6 MiB/s, 25 objects/s recovering 2023-09-23T20:58:28.730 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:58:28.769 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:58:29.017 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:58:29.023 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:58:29.023 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:58:29.124 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:58:29.151 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:58:29.160 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:58:29.203 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:58:29.206 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:58:29.207 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:58:29.308 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:58:29.427 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:58:29.576 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:58:29.592 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:58:29.593 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:58:29.669 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0: {} 2023-09-23T20:58:29.694 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:58:29.707 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1: {} 2023-09-23T20:58:29.745 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2: {} 2023-09-23T20:58:29.768 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:58:29.799 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3: {} 2023-09-23T20:58:29.834 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4: {} 2023-09-23T20:58:29.880 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T20:58:29.882 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:58:29 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[164029]: ::ffff:172.21.15.70 - - [23/Sep/2023:20:58:29] "GET /metrics HTTP/1.1" 200 36321 "" "Prometheus/2.43.0" 2023-09-23T20:58:29.885 INFO:teuthology.orchestra.run.smithi070.stdout:{ 2023-09-23T20:58:29.885 INFO:teuthology.orchestra.run.smithi070.stdout: "size": 20, 2023-09-23T20:58:29.885 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 600, 2023-09-23T20:58:29.885 INFO:teuthology.orchestra.run.smithi070.stdout: "ops": [ 2023-09-23T20:58:29.886 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:58:29.886 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.34797.0:114997 6.12 6.28c54f92 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e132)", 2023-09-23T20:58:29.886 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T20:54:12.985231+0000", 2023-09-23T20:58:29.886 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 256.89641327599998, 2023-09-23T20:58:29.886 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.939145275, 2023-09-23T20:58:29.886 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T20:58:29.886 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "reached pg", 2023-09-23T20:58:29.886 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T20:58:29.886 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.34797", 2023-09-23T20:58:29.886 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T20:58:29.886 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 114997 2023-09-23T20:58:29.886 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:58:29.887 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T20:58:29.887 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:58:29.887 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T20:58:29.887 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:54:12.985231+0000", 2023-09-23T20:58:29.887 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T20:58:29.887 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:58:29.887 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:58:29.887 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T20:58:29.887 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:54:12.985231+0000", 2023-09-23T20:58:29.887 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T20:58:29.887 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:58:29.887 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:58:29.887 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T20:58:29.888 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:54:12.985228+0000", 2023-09-23T20:58:29.888 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999976 2023-09-23T20:58:29.888 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:58:29.888 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:58:29.888 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T20:58:29.888 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:54:12.985282+0000", 2023-09-23T20:58:29.888 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.3285000000000002e-05 2023-09-23T20:58:29.888 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:58:29.888 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:58:29.888 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T20:58:29.888 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:54:12.985298+0000", 2023-09-23T20:58:29.888 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.6677e-05 2023-09-23T20:58:29.888 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:58:29.889 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:58:29.889 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T20:58:29.889 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:54:12.985306+0000", 2023-09-23T20:58:29.889 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.8590000000000005e-06 2023-09-23T20:58:29.889 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:58:29.889 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:58:29.889 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T20:58:29.889 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:54:12.985354+0000", 2023-09-23T20:58:29.889 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.8124000000000002e-05 2023-09-23T20:58:29.889 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:58:29.889 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:58:29.889 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T20:58:29.890 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:54:12.985364+0000", 2023-09-23T20:58:29.890 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.6500000000000008e-06 2023-09-23T20:58:29.890 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:58:29.890 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:58:29.890 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T20:58:29.890 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:54:13.924362+0000", 2023-09-23T20:58:29.890 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.93899794599999997 2023-09-23T20:58:29.890 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:58:29.890 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:58:29.890 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T20:58:29.890 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:54:13.924376+0000", 2023-09-23T20:58:29.890 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.4238e-05 2023-09-23T20:58:29.890 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T20:58:29.891 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T20:58:29.891 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T20:58:29.891 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:58:29.891 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:58:29.891 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.34797.0:156782 6.12 6:4ab9e603:::benchmark_data_smithi026_223868_object55656:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e150)", 2023-09-23T20:58:29.891 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T20:55:07.352049+0000", 2023-09-23T20:58:29.891 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 202.52959541499999, 2023-09-23T20:58:29.891 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.014989538, 2023-09-23T20:58:29.891 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T20:58:29.891 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:58:29.891 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T20:58:29.891 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.34797", 2023-09-23T20:58:29.891 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T20:58:29.892 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 156782 2023-09-23T20:58:29.892 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:58:29.892 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T20:58:29.892 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:58:29.892 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T20:58:29.892 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:07.352049+0000", 2023-09-23T20:58:29.892 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T20:58:29.892 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:58:29.892 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:58:29.892 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T20:58:29.892 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:07.352049+0000", 2023-09-23T20:58:29.892 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T20:58:29.892 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:58:29.893 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:58:29.893 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T20:58:29.893 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:07.352047+0000", 2023-09-23T20:58:29.893 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T20:58:29.893 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:58:29.893 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:58:29.893 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T20:58:29.893 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:07.352052+0000", 2023-09-23T20:58:29.893 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.9280000000000001e-06 2023-09-23T20:58:29.893 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:58:29.893 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:58:29.893 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T20:58:29.894 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:07.352064+0000", 2023-09-23T20:58:29.894 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.1902999999999999e-05 2023-09-23T20:58:29.894 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:58:29.894 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:58:29.894 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T20:58:29.894 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:07.352070+0000", 2023-09-23T20:58:29.894 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.8459999999999996e-06 2023-09-23T20:58:29.894 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:58:29.894 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:58:29.894 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T20:58:29.894 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:07.352358+0000", 2023-09-23T20:58:29.894 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00028820300000000001 2023-09-23T20:58:29.894 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:58:29.895 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:58:29.895 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T20:58:29.895 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:07.352372+0000", 2023-09-23T20:58:29.895 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.3669000000000001e-05 2023-09-23T20:58:29.895 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:58:29.895 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:58:29.895 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T20:58:29.895 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.361399+0000", 2023-09-23T20:58:29.895 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.0090268060000001 2023-09-23T20:58:29.895 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:58:29.895 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:58:29.895 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T20:58:29.896 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.361509+0000", 2023-09-23T20:58:29.896 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000110451 2023-09-23T20:58:29.896 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:58:29.896 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:58:29.896 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 2,3", 2023-09-23T20:58:29.896 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.361803+0000", 2023-09-23T20:58:29.896 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00029413900000000001 2023-09-23T20:58:29.896 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:58:29.896 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:58:29.896 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T20:58:29.896 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.363432+0000", 2023-09-23T20:58:29.896 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.001628188 2023-09-23T20:58:29.896 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:58:29.897 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:58:29.897 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:58:29.897 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.366824+0000", 2023-09-23T20:58:29.897 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0033921910000000001 2023-09-23T20:58:29.897 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:58:29.897 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:58:29.897 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:58:29.897 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.366953+0000", 2023-09-23T20:58:29.897 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00012964399999999999 2023-09-23T20:58:29.897 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:58:29.897 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:58:29.897 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T20:58:29.897 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.367000+0000", 2023-09-23T20:58:29.898 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.6319999999999997e-05 2023-09-23T20:58:29.898 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:58:29.898 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:58:29.898 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T20:58:29.898 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.367038+0000", 2023-09-23T20:58:29.898 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.8498000000000003e-05 2023-09-23T20:58:29.898 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T20:58:29.898 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T20:58:29.898 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T20:58:29.898 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:58:29.898 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:58:29.899 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.34797.0:156812 6.12 6:4c46565a:::benchmark_data_smithi026_223868_object55686:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e150)", 2023-09-23T20:58:29.899 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T20:55:07.386674+0000", 2023-09-23T20:58:29.899 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 202.49497025700001, 2023-09-23T20:58:29.899 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.98619251500000005, 2023-09-23T20:58:29.899 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T20:58:29.899 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:58:29.899 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T20:58:29.899 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.34797", 2023-09-23T20:58:29.899 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T20:58:29.899 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 156812 2023-09-23T20:58:29.899 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:58:29.899 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T20:58:29.899 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:58:29.900 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T20:58:29.900 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:07.386674+0000", 2023-09-23T20:58:29.900 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T20:58:29.900 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:58:29.900 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:58:29.900 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T20:58:29.900 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:07.386674+0000", 2023-09-23T20:58:29.900 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T20:58:29.900 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:58:29.900 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:58:29.900 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T20:58:29.900 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:07.386672+0000", 2023-09-23T20:58:29.901 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T20:58:29.901 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:58:29.901 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:58:29.901 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T20:58:29.901 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:07.386681+0000", 2023-09-23T20:58:29.901 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.2679999999999994e-06 2023-09-23T20:58:29.902 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:58:29.902 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:58:29.902 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T20:58:29.902 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:07.386702+0000", 2023-09-23T20:58:29.902 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.173e-05 2023-09-23T20:58:29.902 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:58:29.902 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:58:29.902 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T20:58:29.902 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:07.386710+0000", 2023-09-23T20:58:29.902 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.0069999999999997e-06 2023-09-23T20:58:29.902 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:58:29.902 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:58:29.903 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T20:58:29.903 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:07.386770+0000", 2023-09-23T20:58:29.903 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.9499000000000001e-05 2023-09-23T20:58:29.903 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:58:29.903 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:58:29.903 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T20:58:29.903 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:07.386780+0000", 2023-09-23T20:58:29.903 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.5619999999999993e-06 2023-09-23T20:58:29.904 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:58:29.904 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:58:29.904 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T20:58:29.904 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.363453+0000", 2023-09-23T20:58:29.904 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.97667318999999997 2023-09-23T20:58:29.904 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:58:29.904 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:58:29.904 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T20:58:29.904 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.363538+0000", 2023-09-23T20:58:29.904 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.5484999999999999e-05 2023-09-23T20:58:29.904 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:58:29.904 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:58:29.905 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 2,3", 2023-09-23T20:58:29.905 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.363717+0000", 2023-09-23T20:58:29.905 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00017910699999999999 2023-09-23T20:58:29.905 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:58:29.905 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:58:29.905 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T20:58:29.905 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.366914+0000", 2023-09-23T20:58:29.905 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0031969059999999998 2023-09-23T20:58:29.905 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:58:29.905 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:58:29.905 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:58:29.905 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.372737+0000", 2023-09-23T20:58:29.905 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00582243 2023-09-23T20:58:29.906 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:58:29.906 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:58:29.906 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:58:29.906 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.372790+0000", 2023-09-23T20:58:29.906 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.3454e-05 2023-09-23T20:58:29.906 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:58:29.906 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:58:29.906 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T20:58:29.906 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.372821+0000", 2023-09-23T20:58:29.907 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.0725999999999997e-05 2023-09-23T20:58:29.907 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:58:29.907 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:58:29.907 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T20:58:29.907 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.372866+0000", 2023-09-23T20:58:29.907 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.5433999999999999e-05 2023-09-23T20:58:29.907 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T20:58:29.907 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T20:58:29.907 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T20:58:29.907 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:58:29.908 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:58:29.908 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.34797.0:156863 6.12 6:48481dbe:::benchmark_data_smithi026_223868_object55737:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e150)", 2023-09-23T20:58:29.908 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T20:55:07.442400+0000", 2023-09-23T20:58:29.908 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 202.439243887, 2023-09-23T20:58:29.908 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.93060301300000003, 2023-09-23T20:58:29.908 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T20:58:29.908 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:58:29.908 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T20:58:29.908 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.34797", 2023-09-23T20:58:29.908 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T20:58:29.908 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 156863 2023-09-23T20:58:29.908 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:58:29.909 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T20:58:29.909 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:58:29.909 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T20:58:29.909 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:07.442400+0000", 2023-09-23T20:58:29.909 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T20:58:29.909 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:58:29.909 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:58:29.909 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T20:58:29.909 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:07.442400+0000", 2023-09-23T20:58:29.909 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T20:58:29.909 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:58:29.909 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:58:29.909 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T20:58:29.910 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:07.442399+0000", 2023-09-23T20:58:29.910 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T20:58:29.910 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:58:29.910 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:58:29.910 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T20:58:29.910 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:07.442405+0000", 2023-09-23T20:58:29.910 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.7490000000000001e-06 2023-09-23T20:58:29.910 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:58:29.910 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:58:29.910 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T20:58:29.910 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:07.442421+0000", 2023-09-23T20:58:29.911 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.6110999999999999e-05 2023-09-23T20:58:29.911 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:58:29.911 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:58:29.911 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T20:58:29.911 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:07.442428+0000", 2023-09-23T20:58:29.911 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.9349999999999997e-06 2023-09-23T20:58:29.911 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:58:29.911 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:58:29.911 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T20:58:29.911 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:07.442498+0000", 2023-09-23T20:58:29.911 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.9488999999999998e-05 2023-09-23T20:58:29.911 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:58:29.912 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:58:29.912 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T20:58:29.912 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:07.442508+0000", 2023-09-23T20:58:29.912 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.0329000000000001e-05 2023-09-23T20:58:29.912 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:58:29.912 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:58:29.912 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T20:58:29.913 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.364821+0000", 2023-09-23T20:58:29.913 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.92231251999999997 2023-09-23T20:58:29.913 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:58:29.913 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:58:29.913 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T20:58:29.913 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.364917+0000", 2023-09-23T20:58:29.913 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.5866000000000007e-05 2023-09-23T20:58:29.914 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:58:29.914 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:58:29.914 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 2,3", 2023-09-23T20:58:29.914 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.365161+0000", 2023-09-23T20:58:29.914 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00024452300000000002 2023-09-23T20:58:29.914 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:58:29.914 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:58:29.914 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T20:58:29.914 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.367047+0000", 2023-09-23T20:58:29.915 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.001885947 2023-09-23T20:58:29.915 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:58:29.915 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:58:29.915 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:58:29.915 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.372916+0000", 2023-09-23T20:58:29.915 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0058686110000000001 2023-09-23T20:58:29.915 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:58:29.916 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:58:29.916 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:58:29.916 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.372948+0000", 2023-09-23T20:58:29.916 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.2159000000000002e-05 2023-09-23T20:58:29.916 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:58:29.916 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:58:29.916 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T20:58:29.916 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.372972+0000", 2023-09-23T20:58:29.916 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.4244999999999999e-05 2023-09-23T20:58:29.916 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:58:29.916 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:58:29.916 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T20:58:29.916 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.373003+0000", 2023-09-23T20:58:29.917 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.112e-05 2023-09-23T20:58:29.917 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T20:58:29.917 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T20:58:29.917 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T20:58:29.917 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:58:29.917 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:58:29.917 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.34797.0:156879 6.12 6:4b4fed44:::benchmark_data_smithi026_223868_object55753:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e150)", 2023-09-23T20:58:29.917 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T20:55:07.459292+0000", 2023-09-23T20:58:29.917 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 202.42235239799999, 2023-09-23T20:58:29.917 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.91741719200000005, 2023-09-23T20:58:29.917 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T20:58:29.917 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:58:29.918 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T20:58:29.918 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.34797", 2023-09-23T20:58:29.918 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T20:58:29.918 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 156879 2023-09-23T20:58:29.918 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:58:29.918 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T20:58:29.918 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:58:29.918 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T20:58:29.918 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:07.459292+0000", 2023-09-23T20:58:29.918 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T20:58:29.918 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:58:29.918 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:58:29.919 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T20:58:29.919 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:07.459292+0000", 2023-09-23T20:58:29.919 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T20:58:29.919 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:58:29.919 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:58:29.919 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T20:58:29.919 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:07.459290+0000", 2023-09-23T20:58:29.919 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999981 2023-09-23T20:58:29.919 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:58:29.919 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:58:29.919 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T20:58:29.919 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:07.459299+0000", 2023-09-23T20:58:29.920 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.1679999999999993e-06 2023-09-23T20:58:29.920 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:58:29.920 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:58:29.920 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T20:58:29.920 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:07.459319+0000", 2023-09-23T20:58:29.920 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.9945999999999998e-05 2023-09-23T20:58:29.920 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:58:29.920 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:58:29.920 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T20:58:29.920 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:07.459328+0000", 2023-09-23T20:58:29.920 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.5439999999999999e-06 2023-09-23T20:58:29.920 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:58:29.921 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:58:29.921 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T20:58:29.921 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:07.459397+0000", 2023-09-23T20:58:29.921 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.9640000000000004e-05 2023-09-23T20:58:29.921 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:58:29.921 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:58:29.921 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T20:58:29.921 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:07.459408+0000", 2023-09-23T20:58:29.921 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.1228e-05 2023-09-23T20:58:29.921 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:58:29.921 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:58:29.921 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T20:58:29.921 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.367075+0000", 2023-09-23T20:58:29.922 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.90766636599999995 2023-09-23T20:58:29.922 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:58:29.922 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:58:29.922 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T20:58:29.922 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.367196+0000", 2023-09-23T20:58:29.922 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000121274 2023-09-23T20:58:29.922 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:58:29.922 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:58:29.922 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 2,3", 2023-09-23T20:58:29.922 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.367465+0000", 2023-09-23T20:58:29.922 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00026886999999999999 2023-09-23T20:58:29.922 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:58:29.923 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:58:29.923 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T20:58:29.923 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.372639+0000", 2023-09-23T20:58:29.923 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0051736430000000003 2023-09-23T20:58:29.923 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:58:29.923 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:58:29.923 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:58:29.923 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.376593+0000", 2023-09-23T20:58:29.923 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0039539919999999999 2023-09-23T20:58:29.923 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:58:29.923 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:58:29.923 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:58:29.924 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.376628+0000", 2023-09-23T20:58:29.924 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.5759000000000001e-05 2023-09-23T20:58:29.924 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:58:29.924 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:58:29.924 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T20:58:29.924 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.376671+0000", 2023-09-23T20:58:29.924 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.2969999999999997e-05 2023-09-23T20:58:29.924 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:58:29.924 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:58:29.924 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T20:58:29.924 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.376709+0000", 2023-09-23T20:58:29.924 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.7540000000000003e-05 2023-09-23T20:58:29.924 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T20:58:29.925 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T20:58:29.925 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T20:58:29.925 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:58:29.925 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:58:29.925 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.34797.0:156990 6.12 6:491f6f4b:::benchmark_data_smithi026_223868_object55864:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e150)", 2023-09-23T20:58:29.925 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T20:55:07.581755+0000", 2023-09-23T20:58:29.925 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 202.29988897000001, 2023-09-23T20:58:29.925 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.79863560899999997, 2023-09-23T20:58:29.925 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T20:58:29.925 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:58:29.925 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T20:58:29.925 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.34797", 2023-09-23T20:58:29.925 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T20:58:29.926 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 156990 2023-09-23T20:58:29.926 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:58:29.926 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T20:58:29.926 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:58:29.926 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T20:58:29.926 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:07.581755+0000", 2023-09-23T20:58:29.926 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T20:58:29.926 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:58:29.926 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:58:29.926 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T20:58:29.926 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:07.581755+0000", 2023-09-23T20:58:29.926 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T20:58:29.927 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:58:29.927 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:58:29.927 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T20:58:29.927 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:07.581754+0000", 2023-09-23T20:58:29.927 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T20:58:29.927 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:58:29.927 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:58:29.927 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T20:58:29.927 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:07.581760+0000", 2023-09-23T20:58:29.927 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.02e-06 2023-09-23T20:58:29.927 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:58:29.927 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:58:29.927 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T20:58:29.928 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:07.581773+0000", 2023-09-23T20:58:29.928 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.2921e-05 2023-09-23T20:58:29.928 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:58:29.928 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:58:29.928 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T20:58:29.928 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:07.581779+0000", 2023-09-23T20:58:29.928 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.7889999999999997e-06 2023-09-23T20:58:29.928 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:58:29.928 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:58:29.928 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T20:58:29.929 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:07.581838+0000", 2023-09-23T20:58:29.929 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.8850000000000001e-05 2023-09-23T20:58:29.929 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:58:29.929 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:58:29.929 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T20:58:29.929 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:07.581848+0000", 2023-09-23T20:58:29.929 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.9340000000000001e-06 2023-09-23T20:58:29.929 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:58:29.929 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:58:29.929 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T20:58:29.929 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.368883+0000", 2023-09-23T20:58:29.930 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.78703539700000003 2023-09-23T20:58:29.930 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:58:29.930 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:58:29.930 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T20:58:29.930 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.368984+0000", 2023-09-23T20:58:29.930 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000100717 2023-09-23T20:58:29.930 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:58:29.930 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:58:29.930 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 2,3", 2023-09-23T20:58:29.930 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.369202+0000", 2023-09-23T20:58:29.930 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000218304 2023-09-23T20:58:29.930 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:58:29.931 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:58:29.931 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T20:58:29.931 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.372876+0000", 2023-09-23T20:58:29.931 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.003673511 2023-09-23T20:58:29.931 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:58:29.931 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:58:29.931 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:58:29.931 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.380243+0000", 2023-09-23T20:58:29.931 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0073674049999999996 2023-09-23T20:58:29.931 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:58:29.931 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:58:29.931 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:58:29.931 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.380308+0000", 2023-09-23T20:58:29.932 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.4309999999999999e-05 2023-09-23T20:58:29.932 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:58:29.932 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:58:29.932 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T20:58:29.932 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.380357+0000", 2023-09-23T20:58:29.932 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.9095999999999999e-05 2023-09-23T20:58:29.932 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:58:29.932 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:58:29.932 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T20:58:29.932 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.380391+0000", 2023-09-23T20:58:29.932 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.3850999999999998e-05 2023-09-23T20:58:29.932 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T20:58:29.933 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T20:58:29.933 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T20:58:29.933 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:58:29.933 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:58:29.933 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.34797.0:157031 6.12 6:48e5af49:::benchmark_data_smithi026_223868_object55905:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e150)", 2023-09-23T20:58:29.933 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T20:55:07.628542+0000", 2023-09-23T20:58:29.933 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 202.25310165799999, 2023-09-23T20:58:29.933 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.75194783899999995, 2023-09-23T20:58:29.933 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T20:58:29.933 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:58:29.933 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T20:58:29.933 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.34797", 2023-09-23T20:58:29.933 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T20:58:29.934 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 157031 2023-09-23T20:58:29.934 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:58:29.934 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T20:58:29.934 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:58:29.934 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T20:58:29.934 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:07.628542+0000", 2023-09-23T20:58:29.934 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T20:58:29.934 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:58:29.934 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:58:29.934 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T20:58:29.934 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:07.628542+0000", 2023-09-23T20:58:29.935 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T20:58:29.935 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:58:29.935 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:58:29.935 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T20:58:29.935 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:07.628541+0000", 2023-09-23T20:58:29.935 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.999999 2023-09-23T20:58:29.935 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:58:29.935 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:58:29.935 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T20:58:29.935 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:07.628549+0000", 2023-09-23T20:58:29.935 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.9019999999999999e-06 2023-09-23T20:58:29.935 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:58:29.936 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:58:29.936 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T20:58:29.936 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:07.628562+0000", 2023-09-23T20:58:29.936 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.2986999999999999e-05 2023-09-23T20:58:29.936 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:58:29.936 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:58:29.936 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T20:58:29.936 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:07.628567+0000", 2023-09-23T20:58:29.936 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.322e-06 2023-09-23T20:58:29.936 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:58:29.936 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:58:29.936 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T20:58:29.936 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:07.628615+0000", 2023-09-23T20:58:29.936 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.7564000000000001e-05 2023-09-23T20:58:29.937 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:58:29.937 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:58:29.937 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T20:58:29.937 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:07.628622+0000", 2023-09-23T20:58:29.937 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.6440000000000003e-06 2023-09-23T20:58:29.937 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:58:29.937 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:58:29.937 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T20:58:29.937 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.370771+0000", 2023-09-23T20:58:29.937 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.74214981700000004 2023-09-23T20:58:29.937 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:58:29.937 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:58:29.937 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T20:58:29.937 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.370876+0000", 2023-09-23T20:58:29.938 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00010476000000000001 2023-09-23T20:58:29.938 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:58:29.938 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:58:29.938 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 2,3", 2023-09-23T20:58:29.938 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.371134+0000", 2023-09-23T20:58:29.938 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00025788100000000001 2023-09-23T20:58:29.938 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:58:29.938 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:58:29.938 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T20:58:29.938 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.373009+0000", 2023-09-23T20:58:29.938 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.001875334 2023-09-23T20:58:29.939 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:58:29.939 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:58:29.939 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:58:29.939 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.380281+0000", 2023-09-23T20:58:29.939 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0072716710000000004 2023-09-23T20:58:29.939 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:58:29.939 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:58:29.939 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:58:29.939 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.380422+0000", 2023-09-23T20:58:29.939 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00014123299999999999 2023-09-23T20:58:29.940 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:58:29.940 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:58:29.940 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T20:58:29.940 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.380460+0000", 2023-09-23T20:58:29.940 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.7372000000000001e-05 2023-09-23T20:58:29.940 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:58:29.940 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:58:29.940 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T20:58:29.940 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.380490+0000", 2023-09-23T20:58:29.940 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.0493000000000001e-05 2023-09-23T20:58:29.940 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T20:58:29.940 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T20:58:29.940 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T20:58:29.941 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:58:29.941 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:58:29.941 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.34797.0:157050 6.12 6:4be05e13:::benchmark_data_smithi026_223868_object55924:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e150)", 2023-09-23T20:58:29.941 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T20:55:07.649612+0000", 2023-09-23T20:58:29.941 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 202.23203245900001, 2023-09-23T20:58:29.941 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.73668807700000005, 2023-09-23T20:58:29.941 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T20:58:29.942 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:58:29.942 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T20:58:29.942 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.34797", 2023-09-23T20:58:29.942 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T20:58:29.942 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 157050 2023-09-23T20:58:29.942 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:58:29.942 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T20:58:29.942 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:58:29.942 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T20:58:29.942 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:07.649612+0000", 2023-09-23T20:58:29.942 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T20:58:29.942 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:58:29.943 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:58:29.943 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T20:58:29.943 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:07.649612+0000", 2023-09-23T20:58:29.943 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T20:58:29.943 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:58:29.943 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:58:29.943 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T20:58:29.943 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:07.649610+0000", 2023-09-23T20:58:29.943 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T20:58:29.943 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:58:29.943 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:58:29.943 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T20:58:29.944 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:07.649619+0000", 2023-09-23T20:58:29.944 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.7439999999999996e-06 2023-09-23T20:58:29.944 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:58:29.944 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:58:29.944 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T20:58:29.944 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:07.649633+0000", 2023-09-23T20:58:29.944 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.4056000000000001e-05 2023-09-23T20:58:29.944 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:58:29.944 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:58:29.944 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T20:58:29.945 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:07.649641+0000", 2023-09-23T20:58:29.945 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.5290000000000003e-06 2023-09-23T20:58:29.945 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:58:29.945 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:58:29.945 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T20:58:29.945 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:07.649701+0000", 2023-09-23T20:58:29.946 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.0715999999999999e-05 2023-09-23T20:58:29.946 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:58:29.946 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:58:29.946 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T20:58:29.946 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:07.649711+0000", 2023-09-23T20:58:29.946 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.5529999999999998e-06 2023-09-23T20:58:29.946 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:58:29.946 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:58:29.946 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T20:58:29.947 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.373038+0000", 2023-09-23T20:58:29.947 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.72332733999999999 2023-09-23T20:58:29.947 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:58:29.947 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:58:29.947 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T20:58:29.947 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.373159+0000", 2023-09-23T20:58:29.947 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000120704 2023-09-23T20:58:29.947 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:58:29.948 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:58:29.948 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 2,3", 2023-09-23T20:58:29.948 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.373420+0000", 2023-09-23T20:58:29.948 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00026133599999999998 2023-09-23T20:58:29.948 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:58:29.948 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:58:29.948 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T20:58:29.948 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.376599+0000", 2023-09-23T20:58:29.948 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0031784790000000001 2023-09-23T20:58:29.949 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:58:29.949 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:58:29.949 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:58:29.949 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.380519+0000", 2023-09-23T20:58:29.949 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0039201569999999996 2023-09-23T20:58:29.949 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:58:29.949 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:58:29.949 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:58:29.949 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.386243+0000", 2023-09-23T20:58:29.949 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0057240770000000002 2023-09-23T20:58:29.949 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:58:29.949 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:58:29.949 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T20:58:29.950 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.386271+0000", 2023-09-23T20:58:29.950 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.8518e-05 2023-09-23T20:58:29.950 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:58:29.950 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:58:29.950 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T20:58:29.950 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.386300+0000", 2023-09-23T20:58:29.950 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.8211000000000001e-05 2023-09-23T20:58:29.950 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T20:58:29.950 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T20:58:29.950 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T20:58:29.951 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:58:29.951 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:58:29.951 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.34797.0:157117 6.12 6:4d4c4893:::benchmark_data_smithi026_223868_object55991:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e150)", 2023-09-23T20:58:29.951 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T20:55:07.747668+0000", 2023-09-23T20:58:29.951 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 202.13397569200001, 2023-09-23T20:58:29.951 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.63872341099999996, 2023-09-23T20:58:29.951 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T20:58:29.951 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:58:29.951 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T20:58:29.951 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.34797", 2023-09-23T20:58:29.951 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T20:58:29.952 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 157117 2023-09-23T20:58:29.952 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:58:29.952 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T20:58:29.952 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:58:29.952 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T20:58:29.952 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:07.747668+0000", 2023-09-23T20:58:29.952 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T20:58:29.952 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:58:29.952 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:58:29.952 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T20:58:29.952 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:07.747668+0000", 2023-09-23T20:58:29.952 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T20:58:29.953 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:58:29.953 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:58:29.953 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T20:58:29.953 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:07.747667+0000", 2023-09-23T20:58:29.953 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T20:58:29.953 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:58:29.953 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:58:29.953 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T20:58:29.953 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:07.747673+0000", 2023-09-23T20:58:29.953 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.2580000000000001e-06 2023-09-23T20:58:29.953 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:58:29.953 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:58:29.954 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T20:58:29.954 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:07.747689+0000", 2023-09-23T20:58:29.954 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.5837000000000001e-05 2023-09-23T20:58:29.954 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:58:29.954 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:58:29.954 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T20:58:29.954 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:07.747698+0000", 2023-09-23T20:58:29.954 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.6039999999999993e-06 2023-09-23T20:58:29.954 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:58:29.954 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:58:29.954 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T20:58:29.955 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:07.747754+0000", 2023-09-23T20:58:29.955 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.6487000000000003e-05 2023-09-23T20:58:29.955 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:58:29.955 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:58:29.955 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T20:58:29.955 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:07.747764+0000", 2023-09-23T20:58:29.955 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.0108e-05 2023-09-23T20:58:29.955 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:58:29.955 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:58:29.955 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T20:58:29.955 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.374677+0000", 2023-09-23T20:58:29.955 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.62691302900000001 2023-09-23T20:58:29.955 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:58:29.956 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:58:29.956 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T20:58:29.956 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.374761+0000", 2023-09-23T20:58:29.956 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.3807999999999994e-05 2023-09-23T20:58:29.956 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:58:29.956 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:58:29.956 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 2,3", 2023-09-23T20:58:29.956 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.374943+0000", 2023-09-23T20:58:29.956 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00018220399999999999 2023-09-23T20:58:29.956 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:58:29.956 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:58:29.956 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T20:58:29.956 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.378533+0000", 2023-09-23T20:58:29.957 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0035901190000000001 2023-09-23T20:58:29.957 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:58:29.957 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:58:29.957 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:58:29.957 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.380547+0000", 2023-09-23T20:58:29.957 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.002013379 2023-09-23T20:58:29.957 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:58:29.957 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:58:29.957 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:58:29.957 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.386342+0000", 2023-09-23T20:58:29.957 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0057952480000000002 2023-09-23T20:58:29.957 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:58:29.957 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:58:29.958 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T20:58:29.958 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.386365+0000", 2023-09-23T20:58:29.958 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.3411e-05 2023-09-23T20:58:29.958 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:58:29.958 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:58:29.958 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T20:58:29.958 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.386392+0000", 2023-09-23T20:58:29.958 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.6301000000000001e-05 2023-09-23T20:58:29.958 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T20:58:29.958 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T20:58:29.958 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T20:58:29.958 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:58:29.959 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:58:29.959 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.34797.0:157161 6.12 6:4e589f92:::benchmark_data_smithi026_223868_object56035:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e150)", 2023-09-23T20:58:29.959 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T20:55:07.800631+0000", 2023-09-23T20:58:29.959 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 202.08101309700001, 2023-09-23T20:58:29.959 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.58585551000000002, 2023-09-23T20:58:29.959 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T20:58:29.959 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:58:29.959 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T20:58:29.959 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.34797", 2023-09-23T20:58:29.959 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T20:58:29.959 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 157161 2023-09-23T20:58:29.959 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:58:29.959 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T20:58:29.960 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:58:29.960 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T20:58:29.960 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:07.800631+0000", 2023-09-23T20:58:29.960 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T20:58:29.960 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:58:29.960 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:58:29.960 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T20:58:29.960 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:07.800631+0000", 2023-09-23T20:58:29.960 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T20:58:29.960 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:58:29.960 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:58:29.960 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T20:58:29.960 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:07.800630+0000", 2023-09-23T20:58:29.961 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T20:58:29.961 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:58:29.961 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:58:29.961 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T20:58:29.961 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:07.800636+0000", 2023-09-23T20:58:29.961 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.9100000000000002e-06 2023-09-23T20:58:29.961 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:58:29.961 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:58:29.961 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T20:58:29.961 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:07.800650+0000", 2023-09-23T20:58:29.961 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.4409999999999999e-05 2023-09-23T20:58:29.961 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:58:29.962 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:58:29.962 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T20:58:29.962 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:07.800657+0000", 2023-09-23T20:58:29.962 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.0829999999999998e-06 2023-09-23T20:58:29.962 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:58:29.962 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:58:29.962 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T20:58:29.962 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:07.800709+0000", 2023-09-23T20:58:29.962 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.1907999999999998e-05 2023-09-23T20:58:29.962 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:58:29.962 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:58:29.962 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T20:58:29.963 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:07.800721+0000", 2023-09-23T20:58:29.963 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.2529e-05 2023-09-23T20:58:29.963 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:58:29.963 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:58:29.963 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T20:58:29.963 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.376733+0000", 2023-09-23T20:58:29.963 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.57601174499999996 2023-09-23T20:58:29.963 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:58:29.963 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:58:29.963 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T20:58:29.963 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.376824+0000", 2023-09-23T20:58:29.963 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.1108999999999998e-05 2023-09-23T20:58:29.964 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:58:29.964 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:58:29.964 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 2,3", 2023-09-23T20:58:29.964 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.377089+0000", 2023-09-23T20:58:29.964 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000265128 2023-09-23T20:58:29.964 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:58:29.964 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:58:29.964 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T20:58:29.964 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.380251+0000", 2023-09-23T20:58:29.964 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0031611230000000001 2023-09-23T20:58:29.964 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:58:29.964 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:58:29.965 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:58:29.965 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.386220+0000", 2023-09-23T20:58:29.965 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0059696009999999997 2023-09-23T20:58:29.965 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:58:29.965 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:58:29.965 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:58:29.965 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.386427+0000", 2023-09-23T20:58:29.965 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000206951 2023-09-23T20:58:29.965 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:58:29.965 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:58:29.965 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T20:58:29.965 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.386466+0000", 2023-09-23T20:58:29.965 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.8874000000000001e-05 2023-09-23T20:58:29.966 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:58:29.966 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:58:29.966 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T20:58:29.966 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.386486+0000", 2023-09-23T20:58:29.966 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.0457000000000001e-05 2023-09-23T20:58:29.966 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T20:58:29.966 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T20:58:29.966 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T20:58:29.966 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:58:29.966 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:58:29.966 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.34797.0:157227 6.12 6:4d4ca5fd:::benchmark_data_smithi026_223868_object56101:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e150)", 2023-09-23T20:58:29.966 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T20:55:07.900387+0000", 2023-09-23T20:58:29.967 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 201.98125682200001, 2023-09-23T20:58:29.967 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.49162882600000002, 2023-09-23T20:58:29.967 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T20:58:29.967 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:58:29.967 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T20:58:29.967 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.34797", 2023-09-23T20:58:29.967 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T20:58:29.967 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 157227 2023-09-23T20:58:29.967 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:58:29.967 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T20:58:29.967 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:58:29.968 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T20:58:29.968 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:07.900387+0000", 2023-09-23T20:58:29.968 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T20:58:29.968 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:58:29.968 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:58:29.968 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T20:58:29.968 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:07.900387+0000", 2023-09-23T20:58:29.968 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T20:58:29.968 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:58:29.968 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:58:29.968 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T20:58:29.968 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:07.900386+0000", 2023-09-23T20:58:29.969 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T20:58:29.969 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:58:29.969 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:58:29.969 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T20:58:29.969 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:07.900392+0000", 2023-09-23T20:58:29.969 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.5899999999999996e-06 2023-09-23T20:58:29.969 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:58:29.969 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:58:29.969 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T20:58:29.969 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:07.900408+0000", 2023-09-23T20:58:29.969 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.5430999999999998e-05 2023-09-23T20:58:29.970 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:58:29.970 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:58:29.970 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T20:58:29.970 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:07.900415+0000", 2023-09-23T20:58:29.970 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.7739999999999999e-06 2023-09-23T20:58:29.970 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:58:29.970 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:58:29.970 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T20:58:29.970 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:07.900476+0000", 2023-09-23T20:58:29.970 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.1452999999999996e-05 2023-09-23T20:58:29.970 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:58:29.971 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:58:29.971 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T20:58:29.971 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:07.900485+0000", 2023-09-23T20:58:29.971 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.3260000000000002e-06 2023-09-23T20:58:29.971 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:58:29.971 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:58:29.971 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T20:58:29.971 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.378561+0000", 2023-09-23T20:58:29.971 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.47807563199999997 2023-09-23T20:58:29.971 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:58:29.971 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:58:29.971 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T20:58:29.972 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.378655+0000", 2023-09-23T20:58:29.972 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.3793000000000001e-05 2023-09-23T20:58:29.972 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:58:29.972 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:58:29.972 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 2,3", 2023-09-23T20:58:29.972 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.378841+0000", 2023-09-23T20:58:29.972 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00018573600000000001 2023-09-23T20:58:29.972 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:58:29.972 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:58:29.972 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:58:29.972 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.386326+0000", 2023-09-23T20:58:29.972 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0074849620000000004 2023-09-23T20:58:29.972 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:58:29.973 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:58:29.973 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:58:29.973 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.386517+0000", 2023-09-23T20:58:29.973 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00019191999999999999 2023-09-23T20:58:29.973 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:58:29.973 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:58:29.973 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T20:58:29.973 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.391931+0000", 2023-09-23T20:58:29.973 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0054132310000000001 2023-09-23T20:58:29.973 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:58:29.973 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:58:29.973 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T20:58:29.974 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.391976+0000", 2023-09-23T20:58:29.974 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.5697e-05 2023-09-23T20:58:29.974 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:58:29.974 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:58:29.974 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T20:58:29.974 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.392016+0000", 2023-09-23T20:58:29.974 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.9644000000000002e-05 2023-09-23T20:58:29.974 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T20:58:29.974 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T20:58:29.975 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T20:58:29.975 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:58:29.975 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:58:29.975 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.34797.0:157259 6.12 6:4e9f8804:::benchmark_data_smithi026_223868_object56133:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e150)", 2023-09-23T20:58:29.975 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T20:55:07.947142+0000", 2023-09-23T20:58:29.975 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 201.93450159, 2023-09-23T20:58:29.975 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.44774977900000001, 2023-09-23T20:58:29.975 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T20:58:29.975 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:58:29.975 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T20:58:29.976 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.34797", 2023-09-23T20:58:29.976 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T20:58:29.976 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 157259 2023-09-23T20:58:29.976 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:58:29.976 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T20:58:29.976 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:58:29.976 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T20:58:29.976 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:07.947142+0000", 2023-09-23T20:58:29.976 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T20:58:29.976 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:58:29.976 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:58:29.976 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T20:58:29.976 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:07.947142+0000", 2023-09-23T20:58:29.977 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T20:58:29.977 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:58:29.977 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:58:29.977 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T20:58:29.977 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:07.947142+0000", 2023-09-23T20:58:29.977 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999995 2023-09-23T20:58:29.977 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:58:29.977 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:58:29.977 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T20:58:29.977 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:07.947146+0000", 2023-09-23T20:58:29.977 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.0500000000000002e-06 2023-09-23T20:58:29.977 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:58:29.978 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:58:29.978 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T20:58:29.978 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:07.947160+0000", 2023-09-23T20:58:29.978 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.4662e-05 2023-09-23T20:58:29.978 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:58:29.978 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:58:29.978 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T20:58:29.978 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:07.947166+0000", 2023-09-23T20:58:29.978 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.784e-06 2023-09-23T20:58:29.978 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:58:29.978 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:58:29.978 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T20:58:29.979 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:07.947223+0000", 2023-09-23T20:58:29.979 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.6854000000000001e-05 2023-09-23T20:58:29.979 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:58:29.979 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:58:29.979 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T20:58:29.979 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:07.947238+0000", 2023-09-23T20:58:29.979 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.4952e-05 2023-09-23T20:58:29.979 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:58:29.979 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:58:29.979 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T20:58:29.979 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.380572+0000", 2023-09-23T20:58:29.979 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.43333436400000003 2023-09-23T20:58:29.979 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:58:29.980 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:58:29.980 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T20:58:29.980 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.380689+0000", 2023-09-23T20:58:29.980 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00011646600000000001 2023-09-23T20:58:29.980 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:58:29.980 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:58:29.980 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 2,3", 2023-09-23T20:58:29.980 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.380934+0000", 2023-09-23T20:58:29.980 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00024519199999999998 2023-09-23T20:58:29.980 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:58:29.980 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:58:29.980 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:58:29.981 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.390072+0000", 2023-09-23T20:58:29.981 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0091384410000000006 2023-09-23T20:58:29.981 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:58:29.981 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:58:29.981 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T20:58:29.981 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.392023+0000", 2023-09-23T20:58:29.981 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0019504170000000001 2023-09-23T20:58:29.981 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:58:29.981 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:58:29.981 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:58:29.981 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.394809+0000", 2023-09-23T20:58:29.981 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0027863760000000001 2023-09-23T20:58:29.981 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:58:29.982 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:58:29.982 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T20:58:29.982 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.394852+0000", 2023-09-23T20:58:29.982 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.3130999999999998e-05 2023-09-23T20:58:29.982 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:58:29.982 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:58:29.982 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T20:58:29.982 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.394892+0000", 2023-09-23T20:58:29.982 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.9801000000000001e-05 2023-09-23T20:58:29.982 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T20:58:29.982 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T20:58:29.982 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T20:58:29.983 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:58:29.983 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:58:29.983 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.34797.0:157328 6.12 6:4811798e:::benchmark_data_smithi026_223868_object56202:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e150)", 2023-09-23T20:58:29.983 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T20:55:08.060527+0000", 2023-09-23T20:58:29.983 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 201.821116864, 2023-09-23T20:58:29.983 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.33450326899999999, 2023-09-23T20:58:29.983 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T20:58:29.983 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:58:29.983 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T20:58:29.983 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.34797", 2023-09-23T20:58:29.983 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T20:58:29.983 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 157328 2023-09-23T20:58:29.984 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:58:29.984 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T20:58:29.984 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:58:29.984 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T20:58:29.984 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.060527+0000", 2023-09-23T20:58:29.984 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T20:58:29.984 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:58:29.984 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:58:29.984 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T20:58:29.984 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.060527+0000", 2023-09-23T20:58:29.984 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T20:58:29.984 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:58:29.985 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:58:29.985 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T20:58:29.985 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.060526+0000", 2023-09-23T20:58:29.985 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.999999 2023-09-23T20:58:29.985 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:58:29.985 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:58:29.985 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T20:58:29.985 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.060531+0000", 2023-09-23T20:58:29.985 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.25e-06 2023-09-23T20:58:29.985 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:58:29.985 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:58:29.985 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T20:58:29.985 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.060540+0000", 2023-09-23T20:58:29.986 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.0259999999999996e-06 2023-09-23T20:58:29.986 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:58:29.986 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:58:29.986 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T20:58:29.986 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.060544+0000", 2023-09-23T20:58:29.986 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.1370000000000004e-06 2023-09-23T20:58:29.986 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:58:29.986 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:58:29.987 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T20:58:29.987 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.060605+0000", 2023-09-23T20:58:29.987 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.0939e-05 2023-09-23T20:58:29.987 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:58:29.987 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:58:29.987 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T20:58:29.987 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.060616+0000", 2023-09-23T20:58:29.987 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.1824999999999999e-05 2023-09-23T20:58:29.987 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:58:29.988 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:58:29.988 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T20:58:29.988 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.382773+0000", 2023-09-23T20:58:29.988 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.32215636800000003 2023-09-23T20:58:29.988 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:58:29.988 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:58:29.988 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T20:58:29.988 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.382925+0000", 2023-09-23T20:58:29.988 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00015193499999999999 2023-09-23T20:58:29.988 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:58:29.988 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:58:29.988 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 2,3", 2023-09-23T20:58:29.988 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.383267+0000", 2023-09-23T20:58:29.989 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00034205500000000002 2023-09-23T20:58:29.989 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:58:29.989 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:58:29.989 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:58:29.989 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.390103+0000", 2023-09-23T20:58:29.989 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0068357510000000002 2023-09-23T20:58:29.989 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:58:29.989 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:58:29.989 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T20:58:29.989 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.392028+0000", 2023-09-23T20:58:29.989 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.001925775 2023-09-23T20:58:29.989 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:58:29.989 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:58:29.990 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:58:29.990 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.394957+0000", 2023-09-23T20:58:29.990 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0029286030000000001 2023-09-23T20:58:29.990 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:58:29.990 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:58:29.990 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T20:58:29.990 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.394996+0000", 2023-09-23T20:58:29.990 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.9410999999999999e-05 2023-09-23T20:58:29.990 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:58:29.990 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:58:29.990 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T20:58:29.990 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.395030+0000", 2023-09-23T20:58:29.991 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.4112000000000001e-05 2023-09-23T20:58:29.991 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T20:58:29.991 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T20:58:29.991 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T20:58:29.991 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:58:29.991 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:58:29.991 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.34797.0:157346 6.12 6:4c4003c4:::benchmark_data_smithi026_223868_object56220:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e150)", 2023-09-23T20:58:29.991 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T20:55:08.089574+0000", 2023-09-23T20:58:29.991 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 201.79207041999999, 2023-09-23T20:58:29.991 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.305631714, 2023-09-23T20:58:29.991 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T20:58:29.991 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:58:29.991 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T20:58:29.992 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.34797", 2023-09-23T20:58:29.992 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T20:58:29.992 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 157346 2023-09-23T20:58:29.992 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:58:29.992 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T20:58:29.992 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:58:29.992 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T20:58:29.992 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.089574+0000", 2023-09-23T20:58:29.992 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T20:58:29.992 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:58:29.992 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:58:29.992 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T20:58:29.992 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.089574+0000", 2023-09-23T20:58:29.993 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T20:58:29.993 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:58:29.993 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:58:29.993 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T20:58:29.993 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.089572+0000", 2023-09-23T20:58:29.993 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999981 2023-09-23T20:58:29.993 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:58:29.993 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:58:29.993 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T20:58:29.993 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.089579+0000", 2023-09-23T20:58:29.993 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.6290000000000001e-06 2023-09-23T20:58:29.993 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:58:29.993 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:58:29.994 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T20:58:29.994 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.089596+0000", 2023-09-23T20:58:29.994 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.6511000000000002e-05 2023-09-23T20:58:29.994 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:58:29.994 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:58:29.994 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T20:58:29.994 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.089605+0000", 2023-09-23T20:58:29.994 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.5390000000000007e-06 2023-09-23T20:58:29.994 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:58:29.995 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:58:29.995 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T20:58:29.995 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.089665+0000", 2023-09-23T20:58:29.995 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.0859000000000001e-05 2023-09-23T20:58:29.995 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:58:29.995 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:58:29.995 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T20:58:29.995 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.089678+0000", 2023-09-23T20:58:29.995 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.2133000000000001e-05 2023-09-23T20:58:29.995 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:58:29.995 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:58:29.995 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T20:58:29.995 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.384720+0000", 2023-09-23T20:58:29.996 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.29504224499999998 2023-09-23T20:58:29.996 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:58:29.996 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:58:29.996 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T20:58:29.996 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.384814+0000", 2023-09-23T20:58:29.996 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.4611999999999995e-05 2023-09-23T20:58:29.996 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:58:29.996 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:58:29.996 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 2,3", 2023-09-23T20:58:29.996 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.385022+0000", 2023-09-23T20:58:29.996 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00020736999999999999 2023-09-23T20:58:29.996 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:58:29.996 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:58:29.997 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T20:58:29.997 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.392033+0000", 2023-09-23T20:58:29.997 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0070114649999999997 2023-09-23T20:58:29.997 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:58:29.997 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:58:29.997 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:58:29.997 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.395074+0000", 2023-09-23T20:58:29.997 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.003040297 2023-09-23T20:58:29.997 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:58:29.997 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:58:29.997 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:58:29.997 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.395133+0000", 2023-09-23T20:58:29.997 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.9700000000000001e-05 2023-09-23T20:58:29.998 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:58:29.998 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:58:29.998 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T20:58:29.998 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.395173+0000", 2023-09-23T20:58:29.998 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.9956999999999997e-05 2023-09-23T20:58:29.998 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:58:29.998 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:58:29.998 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T20:58:29.998 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.395205+0000", 2023-09-23T20:58:29.998 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.2146e-05 2023-09-23T20:58:29.998 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T20:58:29.999 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T20:58:29.999 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T20:58:29.999 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:58:29.999 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:58:29.999 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.34797.0:157407 6.12 6:494044ed:::benchmark_data_smithi026_223868_object56281:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e150)", 2023-09-23T20:58:29.999 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T20:55:08.202324+0000", 2023-09-23T20:58:29.999 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 201.67932028000001, 2023-09-23T20:58:29.999 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.19300698499999999, 2023-09-23T20:58:29.999 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T20:58:29.999 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:58:29.999 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T20:58:30.000 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.34797", 2023-09-23T20:58:30.000 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T20:58:30.000 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 157407 2023-09-23T20:58:30.000 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:58:30.000 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T20:58:30.000 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:58:30.000 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T20:58:30.000 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.202324+0000", 2023-09-23T20:58:30.000 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T20:58:30.000 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:58:30.000 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:58:30.000 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T20:58:30.001 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.202324+0000", 2023-09-23T20:58:30.001 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T20:58:30.001 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:58:30.001 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:58:30.001 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T20:58:30.001 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.202322+0000", 2023-09-23T20:58:30.001 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T20:58:30.001 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:58:30.001 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:58:30.001 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T20:58:30.001 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.202330+0000", 2023-09-23T20:58:30.001 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.8730000000000007e-06 2023-09-23T20:58:30.001 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:58:30.002 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:58:30.002 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T20:58:30.002 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.202346+0000", 2023-09-23T20:58:30.002 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.5807e-05 2023-09-23T20:58:30.002 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:58:30.002 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:58:30.002 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T20:58:30.002 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.202352+0000", 2023-09-23T20:58:30.002 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.4609999999999996e-06 2023-09-23T20:58:30.002 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:58:30.002 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:58:30.002 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T20:58:30.003 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.202412+0000", 2023-09-23T20:58:30.003 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.9930999999999999e-05 2023-09-23T20:58:30.003 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:58:30.003 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:58:30.003 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T20:58:30.003 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.202425+0000", 2023-09-23T20:58:30.003 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.2272e-05 2023-09-23T20:58:30.003 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:58:30.003 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:58:30.003 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T20:58:30.003 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.386530+0000", 2023-09-23T20:58:30.003 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.18410531799999999 2023-09-23T20:58:30.004 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:58:30.004 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:58:30.004 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T20:58:30.004 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.386605+0000", 2023-09-23T20:58:30.004 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.4880000000000001e-05 2023-09-23T20:58:30.004 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:58:30.004 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:58:30.004 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 2,3", 2023-09-23T20:58:30.004 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.386802+0000", 2023-09-23T20:58:30.004 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00019716199999999999 2023-09-23T20:58:30.005 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:58:30.005 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:58:30.005 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T20:58:30.005 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.392038+0000", 2023-09-23T20:58:30.005 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.005235941 2023-09-23T20:58:30.005 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:58:30.005 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:58:30.005 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:58:30.005 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.395108+0000", 2023-09-23T20:58:30.005 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0030700570000000002 2023-09-23T20:58:30.005 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:58:30.005 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:58:30.005 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:58:30.006 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.395266+0000", 2023-09-23T20:58:30.006 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00015852499999999999 2023-09-23T20:58:30.006 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:58:30.006 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:58:30.006 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T20:58:30.006 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.395302+0000", 2023-09-23T20:58:30.006 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.5374999999999999e-05 2023-09-23T20:58:30.006 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:58:30.006 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:58:30.006 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T20:58:30.006 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.395331+0000", 2023-09-23T20:58:30.007 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.8892999999999999e-05 2023-09-23T20:58:30.007 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T20:58:30.007 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T20:58:30.007 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T20:58:30.007 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:58:30.007 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:58:30.007 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.34797.0:157436 6.12 6:4be31d9d:::benchmark_data_smithi026_223868_object56310:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e150)", 2023-09-23T20:58:30.007 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T20:55:08.273211+0000", 2023-09-23T20:58:30.007 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 201.60843288500001, 2023-09-23T20:58:30.007 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.12224776599999999, 2023-09-23T20:58:30.007 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T20:58:30.007 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:58:30.007 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T20:58:30.008 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.34797", 2023-09-23T20:58:30.008 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T20:58:30.008 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 157436 2023-09-23T20:58:30.008 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:58:30.008 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T20:58:30.008 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:58:30.008 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T20:58:30.008 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.273211+0000", 2023-09-23T20:58:30.008 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T20:58:30.008 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:58:30.008 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:58:30.008 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T20:58:30.008 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.273211+0000", 2023-09-23T20:58:30.009 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T20:58:30.009 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:58:30.009 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:58:30.009 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T20:58:30.009 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.273210+0000", 2023-09-23T20:58:30.009 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.999999 2023-09-23T20:58:30.009 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:58:30.009 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:58:30.009 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T20:58:30.009 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.273214+0000", 2023-09-23T20:58:30.009 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.1080000000000003e-06 2023-09-23T20:58:30.009 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:58:30.010 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:58:30.010 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T20:58:30.010 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.273226+0000", 2023-09-23T20:58:30.010 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.168e-05 2023-09-23T20:58:30.010 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:58:30.010 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:58:30.010 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T20:58:30.010 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.273231+0000", 2023-09-23T20:58:30.010 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.6670000000000004e-06 2023-09-23T20:58:30.010 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:58:30.011 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:58:30.011 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T20:58:30.011 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.273295+0000", 2023-09-23T20:58:30.011 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.4342000000000002e-05 2023-09-23T20:58:30.011 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:58:30.011 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:58:30.011 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T20:58:30.011 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.273305+0000", 2023-09-23T20:58:30.011 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.0458e-05 2023-09-23T20:58:30.011 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:58:30.011 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:58:30.011 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T20:58:30.012 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.388109+0000", 2023-09-23T20:58:30.012 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.114803402 2023-09-23T20:58:30.012 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:58:30.012 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:58:30.012 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T20:58:30.012 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.388202+0000", 2023-09-23T20:58:30.012 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.3173999999999999e-05 2023-09-23T20:58:30.012 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:58:30.012 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:58:30.012 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 2,3", 2023-09-23T20:58:30.012 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.388429+0000", 2023-09-23T20:58:30.012 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00022721899999999999 2023-09-23T20:58:30.012 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:58:30.013 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:58:30.013 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T20:58:30.013 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.392043+0000", 2023-09-23T20:58:30.013 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0036135849999999999 2023-09-23T20:58:30.013 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:58:30.013 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:58:30.013 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:58:30.013 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.395241+0000", 2023-09-23T20:58:30.013 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0031982820000000002 2023-09-23T20:58:30.013 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:58:30.013 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:58:30.014 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:58:30.014 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.395384+0000", 2023-09-23T20:58:30.014 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00014287100000000001 2023-09-23T20:58:30.014 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:58:30.014 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:58:30.014 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T20:58:30.014 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.395420+0000", 2023-09-23T20:58:30.014 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.6174000000000003e-05 2023-09-23T20:58:30.014 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:58:30.014 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:58:30.014 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T20:58:30.015 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.395459+0000", 2023-09-23T20:58:30.015 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.8887999999999998e-05 2023-09-23T20:58:30.015 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T20:58:30.015 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T20:58:30.015 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T20:58:30.015 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:58:30.015 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:58:30.015 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.35100.0:94301 7.1 7:8f95fbda:::benchmark_data_smithi026_260489_object94300:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e228)", 2023-09-23T20:58:30.015 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T20:57:57.540024+0000", 2023-09-23T20:58:30.015 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 32.341619594000001, 2023-09-23T20:58:30.015 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.14579631700000001, 2023-09-23T20:58:30.015 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T20:58:30.016 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:58:30.016 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T20:58:30.016 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.35100", 2023-09-23T20:58:30.016 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/4108972961", 2023-09-23T20:58:30.016 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 94301 2023-09-23T20:58:30.016 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:58:30.016 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T20:58:30.016 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:58:30.016 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T20:58:30.016 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:57:57.540024+0000", 2023-09-23T20:58:30.016 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T20:58:30.016 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:58:30.017 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:58:30.017 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T20:58:30.017 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:57:57.540024+0000", 2023-09-23T20:58:30.017 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T20:58:30.017 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:58:30.017 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:58:30.017 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T20:58:30.017 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:57:57.540024+0000", 2023-09-23T20:58:30.017 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999995 2023-09-23T20:58:30.017 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:58:30.017 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:58:30.017 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T20:58:30.018 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:57:57.540060+0000", 2023-09-23T20:58:30.018 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.6013000000000002e-05 2023-09-23T20:58:30.018 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:58:30.018 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:58:30.018 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T20:58:30.018 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:57:57.540067+0000", 2023-09-23T20:58:30.018 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.866e-06 2023-09-23T20:58:30.018 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:58:30.018 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:58:30.018 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T20:58:30.018 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:57:57.540075+0000", 2023-09-23T20:58:30.019 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.7239999999999992e-06 2023-09-23T20:58:30.019 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:58:30.019 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:58:30.019 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T20:58:30.019 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:57:57.549778+0000", 2023-09-23T20:58:30.019 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0097031600000000006 2023-09-23T20:58:30.019 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:58:30.019 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:58:30.019 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T20:58:30.019 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:57:57.549889+0000", 2023-09-23T20:58:30.019 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000110818 2023-09-23T20:58:30.019 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:58:30.019 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:58:30.020 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 3,6", 2023-09-23T20:58:30.020 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:57:57.550024+0000", 2023-09-23T20:58:30.020 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00013545899999999999 2023-09-23T20:58:30.020 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:58:30.020 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:58:30.020 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T20:58:30.020 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:57:57.551983+0000", 2023-09-23T20:58:30.020 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.001958871 2023-09-23T20:58:30.020 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:58:30.020 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:58:30.020 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:58:30.020 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:57:57.552271+0000", 2023-09-23T20:58:30.021 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00028848799999999999 2023-09-23T20:58:30.021 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:58:30.021 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:58:30.021 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:58:30.021 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:57:57.572759+0000", 2023-09-23T20:58:30.021 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.020487701000000001 2023-09-23T20:58:30.021 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:58:30.021 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:58:30.021 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T20:58:30.021 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:57:57.572816+0000", 2023-09-23T20:58:30.021 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.7247999999999998e-05 2023-09-23T20:58:30.021 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:58:30.022 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:58:30.022 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T20:58:30.022 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:57:57.685821+0000", 2023-09-23T20:58:30.022 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.1130044 2023-09-23T20:58:30.022 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T20:58:30.022 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T20:58:30.022 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T20:58:30.022 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:58:30.022 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:58:30.022 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_repop(client.35100.0:94315 7.8 e228/199 7:126e5865:::benchmark_data_smithi026_260489_object94314:head v 228'5947, mlcod=228'5944)", 2023-09-23T20:58:30.022 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T20:57:57.570899+0000", 2023-09-23T20:58:30.023 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 32.310744786000001, 2023-09-23T20:58:30.023 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.11494793, 2023-09-23T20:58:30.023 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T20:58:30.023 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:58:30.023 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T20:58:30.023 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:58:30.023 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T20:58:30.023 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:57:57.570899+0000", 2023-09-23T20:58:30.023 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T20:58:30.023 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:58:30.023 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:58:30.023 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T20:58:30.024 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:57:57.570899+0000", 2023-09-23T20:58:30.024 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T20:58:30.024 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:58:30.024 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:58:30.024 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T20:58:30.024 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:57:57.570899+0000", 2023-09-23T20:58:30.024 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999995 2023-09-23T20:58:30.024 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:58:30.024 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:58:30.024 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T20:58:30.024 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:57:57.571002+0000", 2023-09-23T20:58:30.024 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00010277600000000001 2023-09-23T20:58:30.025 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:58:30.025 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:58:30.025 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T20:58:30.025 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:57:57.571010+0000", 2023-09-23T20:58:30.025 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.9029999999999993e-06 2023-09-23T20:58:30.025 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:58:30.025 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:58:30.025 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T20:58:30.025 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:57:57.571015+0000", 2023-09-23T20:58:30.025 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.702e-06 2023-09-23T20:58:30.025 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:58:30.025 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:58:30.025 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T20:58:30.026 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:57:57.571058+0000", 2023-09-23T20:58:30.026 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.2369e-05 2023-09-23T20:58:30.026 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:58:30.026 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:58:30.026 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T20:58:30.026 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:57:57.571076+0000", 2023-09-23T20:58:30.026 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.8805e-05 2023-09-23T20:58:30.026 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:58:30.026 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:58:30.026 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T20:58:30.026 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:57:57.685810+0000", 2023-09-23T20:58:30.027 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.114733351 2023-09-23T20:58:30.027 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:58:30.027 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:58:30.027 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T20:58:30.027 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:57:57.685847+0000", 2023-09-23T20:58:30.027 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.7502000000000001e-05 2023-09-23T20:58:30.027 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T20:58:30.027 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T20:58:30.027 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T20:58:30.027 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:58:30.027 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:58:30.028 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.35100.0:94323 7.1 7:86b69fc0:::benchmark_data_smithi026_260489_object94322:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e228)", 2023-09-23T20:58:30.028 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T20:57:57.573982+0000", 2023-09-23T20:58:30.028 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 32.307661668999998, 2023-09-23T20:58:30.028 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.144034897, 2023-09-23T20:58:30.028 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T20:58:30.028 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:58:30.028 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T20:58:30.028 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.35100", 2023-09-23T20:58:30.028 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/4108972961", 2023-09-23T20:58:30.028 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 94323 2023-09-23T20:58:30.028 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:58:30.028 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T20:58:30.028 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:58:30.029 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T20:58:30.029 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:57:57.573982+0000", 2023-09-23T20:58:30.029 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T20:58:30.029 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:58:30.029 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:58:30.029 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T20:58:30.029 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:57:57.573982+0000", 2023-09-23T20:58:30.029 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T20:58:30.029 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:58:30.029 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:58:30.029 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T20:58:30.029 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:57:57.573981+0000", 2023-09-23T20:58:30.030 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T20:58:30.030 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:58:30.030 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:58:30.030 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T20:58:30.030 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:57:57.574033+0000", 2023-09-23T20:58:30.030 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.2389999999999998e-05 2023-09-23T20:58:30.030 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:58:30.030 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:58:30.030 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T20:58:30.030 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:57:57.574044+0000", 2023-09-23T20:58:30.030 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.1079e-05 2023-09-23T20:58:30.030 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:58:30.031 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:58:30.031 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T20:58:30.031 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:57:57.574051+0000", 2023-09-23T20:58:30.031 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.2689999999999996e-06 2023-09-23T20:58:30.031 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:58:30.031 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:58:30.031 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T20:58:30.031 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:57:57.685883+0000", 2023-09-23T20:58:30.031 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.11183256599999999 2023-09-23T20:58:30.031 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:58:30.032 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:58:30.032 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T20:58:30.032 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:57:57.686002+0000", 2023-09-23T20:58:30.032 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00011926600000000001 2023-09-23T20:58:30.032 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:58:30.032 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:58:30.032 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 3,6", 2023-09-23T20:58:30.032 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:57:57.686143+0000", 2023-09-23T20:58:30.032 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00014049200000000001 2023-09-23T20:58:30.032 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:58:30.032 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:58:30.032 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:58:30.033 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:57:57.692125+0000", 2023-09-23T20:58:30.033 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0059817849999999999 2023-09-23T20:58:30.033 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:58:30.033 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:58:30.033 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T20:58:30.033 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:57:57.697824+0000", 2023-09-23T20:58:30.033 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00569959 2023-09-23T20:58:30.033 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:58:30.033 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:58:30.033 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:58:30.033 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:57:57.714832+0000", 2023-09-23T20:58:30.033 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.017007967999999998 2023-09-23T20:58:30.033 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:58:30.034 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:58:30.034 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T20:58:30.034 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:57:57.714872+0000", 2023-09-23T20:58:30.034 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.9638000000000002e-05 2023-09-23T20:58:30.034 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:58:30.034 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:58:30.034 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T20:58:30.034 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:57:57.718017+0000", 2023-09-23T20:58:30.034 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0031452939999999999 2023-09-23T20:58:30.034 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T20:58:30.035 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T20:58:30.035 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T20:58:30.035 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:58:30.035 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:58:30.035 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_repop(client.35100.0:94326 7.9 e228/199 7:94da320a:::benchmark_data_smithi026_260489_object94325:head v 228'5898, mlcod=228'5896)", 2023-09-23T20:58:30.035 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T20:57:57.578378+0000", 2023-09-23T20:58:30.035 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 32.303266080999997, 2023-09-23T20:58:30.035 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.11680721299999999, 2023-09-23T20:58:30.035 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T20:58:30.035 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:58:30.035 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T20:58:30.035 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:58:30.035 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T20:58:30.036 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:57:57.578378+0000", 2023-09-23T20:58:30.036 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T20:58:30.036 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:58:30.036 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:58:30.036 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T20:58:30.036 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:57:57.578378+0000", 2023-09-23T20:58:30.036 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T20:58:30.036 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:58:30.036 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:58:30.036 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T20:58:30.036 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:57:57.578377+0000", 2023-09-23T20:58:30.036 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.999999 2023-09-23T20:58:30.036 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:58:30.037 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:58:30.037 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T20:58:30.037 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:57:57.578434+0000", 2023-09-23T20:58:30.037 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.6963999999999999e-05 2023-09-23T20:58:30.037 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:58:30.037 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:58:30.037 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T20:58:30.037 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:57:57.685806+0000", 2023-09-23T20:58:30.037 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.10737245400000001 2023-09-23T20:58:30.037 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:58:30.037 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:58:30.037 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T20:58:30.037 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:57:57.685817+0000", 2023-09-23T20:58:30.038 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.0964e-05 2023-09-23T20:58:30.038 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:58:30.038 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:58:30.038 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T20:58:30.038 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:57:57.685869+0000", 2023-09-23T20:58:30.038 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.1773e-05 2023-09-23T20:58:30.038 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:58:30.038 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:58:30.038 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T20:58:30.038 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:57:57.685890+0000", 2023-09-23T20:58:30.038 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.0489999999999999e-05 2023-09-23T20:58:30.038 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:58:30.039 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:58:30.039 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T20:58:30.039 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:57:57.695157+0000", 2023-09-23T20:58:30.039 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0092676800000000004 2023-09-23T20:58:30.039 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:58:30.039 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:58:30.039 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T20:58:30.039 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:57:57.695185+0000", 2023-09-23T20:58:30.039 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.7773999999999999e-05 2023-09-23T20:58:30.039 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T20:58:30.040 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T20:58:30.040 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T20:58:30.040 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T20:58:30.040 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T20:58:30.040 INFO:teuthology.orchestra.run.smithi070.stdout:} 2023-09-23T20:58:30.041 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:58:30.041 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:58:30.041 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:58:30.042 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6: {} 2023-09-23T20:58:30.042 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T20:58:30.042 INFO:teuthology.orchestra.run.smithi026.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T20:58:30.042 INFO:teuthology.orchestra.run.smithi026.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T20:58:30.042 INFO:teuthology.orchestra.run.smithi026.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T20:58:30.042 INFO:teuthology.orchestra.run.smithi026.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T20:58:30.043 INFO:teuthology.orchestra.run.smithi026.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T20:58:30.043 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T20:58:30.043 INFO:teuthology.orchestra.run.smithi026.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T20:58:30.043 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T20:58:30.142 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:58:30.173 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:58:30 smithi070 ceph-mon[168282]: pgmap v554: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 5.6 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 530 op/s; 10943/275031 objects degraded (3.979%); 4563/275031 objects misplaced (1.659%); 1.5 MiB/s, 23 objects/s recovering 2023-09-23T20:58:30.173 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:58:30 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/1358145841' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:58:30.173 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:58:30 smithi070 ceph-mon[168282]: Health check update: Degraded data redundancy: 10943/275031 objects degraded (3.979%), 2 pgs degraded (PG_DEGRADED) 2023-09-23T20:58:30.177 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:58:30.305 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:58:30 smithi026 ceph-mon[175151]: pgmap v554: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 5.6 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 530 op/s; 10943/275031 objects degraded (3.979%); 4563/275031 objects misplaced (1.659%); 1.5 MiB/s, 23 objects/s recovering 2023-09-23T20:58:30.305 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:58:30 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/1358145841' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:58:30.306 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:58:30 smithi026 ceph-mon[175151]: Health check update: Degraded data redundancy: 10943/275031 objects degraded (3.979%), 2 pgs degraded (PG_DEGRADED) 2023-09-23T20:58:30.306 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:58:30 smithi026 ceph-mon[179688]: pgmap v554: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 5.6 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 530 op/s; 10943/275031 objects degraded (3.979%); 4563/275031 objects misplaced (1.659%); 1.5 MiB/s, 23 objects/s recovering 2023-09-23T20:58:30.306 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:58:30 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/1358145841' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:58:30.306 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:58:30 smithi026 ceph-mon[179688]: Health check update: Degraded data redundancy: 10943/275031 objects degraded (3.979%), 2 pgs degraded (PG_DEGRADED) 2023-09-23T20:58:30.313 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:58:30.316 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:58:30.317 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:58:30.327 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_ops_in_flight 2023-09-23T20:58:30.417 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:58:30.452 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:58:30.504 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:58:30.507 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:58:30.507 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:58:30.608 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:58:30.710 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:58:30.728 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T20:58:30.805 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:58:30.821 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:58:30.821 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:58:30.922 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:58:30.958 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:58:31.002 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:58:31.004 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:58:31.004 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:58:31.034 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:58:31.106 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:58:31.135 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:58:31.173 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:58:31.222 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:58:31.222 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:58:31.323 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:58:31.359 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:58:31.506 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:58:31.509 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:58:31.509 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:58:31.610 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:58:31.648 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:58:31.742 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:58:31.780 INFO:teuthology.orchestra.run.smithi026.stdout: 2023-09-23T20:58:31.780 INFO:teuthology.orchestra.run.smithi026.stdout:{"fsid":"b3f8e4ec-5a50-11ee-8db4-212e2dc638e7","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"noscrub,nodeep-scrub flag(s) set","count":20},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 10943/270978 objects degraded (4.038%), 2 pgs degraded","count":2},"muted":false}},"mutes":[]},"election_epoch":30,"quorum":[0,1,2],"quorum_names":["a","b","c"],"quorum_age":630,"monmap":{"epoch":4,"min_mon_release_name":"reef","num_mons":3},"osdmap":{"epoch":234,"num_osds":8,"num_up_osds":8,"osd_up_since":1695502534,"num_in_osds":8,"osd_in_since":1695502470,"num_remapped_pgs":3},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":106},{"state_name":"active+recovery_wait+undersized+degraded+remapped","count":2},{"state_name":"active+recovering+undersized+remapped","count":1}],"num_pgs":109,"num_pools":5,"num_objects":90326,"data_bytes":5918562232,"bytes_used":19503902720,"bytes_avail":748456382464,"bytes_total":767960285184,"degraded_objects":10943,"degraded_total":270978,"degraded_ratio":0.040383352154049409,"misplaced_objects":4563,"misplaced_total":270978,"misplaced_ratio":0.016839005380510596,"recovering_objects_per_sec":20,"recovering_bytes_per_sec":1361462,"recovering_keys_per_sec":0,"num_objects_recovered":208,"num_bytes_recovered":13631488,"num_keys_recovered":0,"write_bytes_sec":0,"read_op_per_sec":0,"write_op_per_sec":607},"fsmap":{"epoch":8,"id":1,"up":1,"in":1,"max":1,"by_rank":[{"filesystem_id":1,"rank":0,"name":"foo.smithi026.znfpnl","status":"up:active","gid":14658}],"up:standby":1},"mgrmap":{"available":true,"num_standbys":1,"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"services":{"dashboard":"https://172.21.15.26:8443/","prometheus":"http://172.21.15.26:9283/"}},"servicemap":{"epoch":21,"modified":"2023-09-23T20:56:34.314324+0000","services":{}},"progress_events":{"8b64cbfe-d367-468b-a773-0e71ee4ff645":{"message":"Global Recovery Event (0s)\n [............................] ","progress":0,"add_to_ceph_s":true},"fc746e9d-3f76-434d-b6d2-8fc9e8a72a5e":{"message":"Upgrade to 18.2.0-460-gbb0daeff (4m)\n [============................] (remaining: 5m)","progress":0.44999998807907104,"add_to_ceph_s":true}}} 2023-09-23T20:58:31.907 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:58:31.914 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:58:31.915 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:58:32.015 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:58:32.112 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:58:32.317 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:58:32.326 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:58:32.326 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:58:32 smithi026 ceph-mon[175151]: pgmap v555: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 5.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 607 op/s; 10943/270978 objects degraded (4.038%); 4563/270978 objects misplaced (1.684%); 1.3 MiB/s, 20 objects/s recovering 2023-09-23T20:58:32.326 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:58:32 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:58:32.326 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:58:32 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:58:32.326 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:58:32 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/3104114544' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-23T20:58:32.327 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:58:32.327 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:58:32 smithi026 ceph-mon[179688]: pgmap v555: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 5.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 607 op/s; 10943/270978 objects degraded (4.038%); 4563/270978 objects misplaced (1.684%); 1.3 MiB/s, 20 objects/s recovering 2023-09-23T20:58:32.327 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:58:32 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:58:32.328 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:58:32 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:58:32.328 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:58:32 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/3104114544' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-23T20:58:32.328 INFO:teuthology.orchestra.run.smithi070.stdout:{ 2023-09-23T20:58:32.328 INFO:teuthology.orchestra.run.smithi070.stdout: "ops": [], 2023-09-23T20:58:32.328 INFO:teuthology.orchestra.run.smithi070.stdout: "num_ops": 0 2023-09-23T20:58:32.328 INFO:teuthology.orchestra.run.smithi070.stdout:} 2023-09-23T20:58:32.386 INFO:tasks.cephadm.ceph_manager.ceph:making progress, resetting timeout 2023-09-23T20:58:32.386 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T20:58:32.428 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:58:32.445 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:58:32 smithi070 ceph-mon[168282]: pgmap v555: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 5.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 607 op/s; 10943/270978 objects degraded (4.038%); 4563/270978 objects misplaced (1.684%); 1.3 MiB/s, 20 objects/s recovering 2023-09-23T20:58:32.445 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:58:32 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:58:32.445 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:58:32 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:58:32.445 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:58:32 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/3104114544' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-23T20:58:32.465 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:58:32.624 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T20:58:32.663 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:58:32.665 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:58:32.666 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:58:32.728 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_blocked_ops 2023-09-23T20:58:32.767 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:58:32.901 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:58:33.071 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:58:33.081 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:58:33.082 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:58:33.183 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:58:33.211 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:58:33.253 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:58:33.256 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:58:33.257 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:58:33.357 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:58:33.385 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:58:33.448 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:58:33.544 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:58:33.605 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:58:33.616 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:58:33.616 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:58:33.717 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:58:33.763 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:58:33.857 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:58:33.905 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:58:33.908 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:58:33.909 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:58:34.009 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:58:34.110 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:58:34.354 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:58:34 smithi026 ceph-mon[175151]: pgmap v556: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 5.4 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 623 op/s; 10943/265383 objects degraded (4.123%); 4453/265383 objects misplaced (1.678%); 1.4 MiB/s, 21 objects/s recovering 2023-09-23T20:58:34.355 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:58:34 smithi026 ceph-mon[179688]: pgmap v556: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 5.4 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 623 op/s; 10943/265383 objects degraded (4.123%); 4453/265383 objects misplaced (1.678%); 1.4 MiB/s, 21 objects/s recovering 2023-09-23T20:58:34.415 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:58:34 smithi070 ceph-mon[168282]: pgmap v556: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 5.4 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 623 op/s; 10943/265383 objects degraded (4.123%); 4453/265383 objects misplaced (1.678%); 1.4 MiB/s, 21 objects/s recovering 2023-09-23T20:58:34.656 INFO:teuthology.orchestra.run.smithi070.stdout:{ 2023-09-23T20:58:34.656 INFO:teuthology.orchestra.run.smithi070.stdout: "ops": [], 2023-09-23T20:58:34.656 INFO:teuthology.orchestra.run.smithi070.stdout: "complaint_time": 30, 2023-09-23T20:58:34.656 INFO:teuthology.orchestra.run.smithi070.stdout: "num_blocked_ops": 0 2023-09-23T20:58:34.656 INFO:teuthology.orchestra.run.smithi070.stdout:} 2023-09-23T20:58:34.669 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:58:34.676 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:58:34.677 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:58:34.778 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:58:34.857 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:58:34.984 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:58:34.994 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:58:34.994 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:58:35.059 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_historic_ops 2023-09-23T20:58:35.096 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:58:35.141 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:58:35.197 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:58:35.204 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:58:35.205 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:58:35.219 INFO:teuthology.orchestra.run.smithi026.stderr:noscrub is unset 2023-09-23T20:58:35.253 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:58:35 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/515711627' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:58:35.253 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:58:35 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:58:35.253 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:58:35 smithi026 ceph-mon[179688]: pgmap v557: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 5.4 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 619 op/s; 10943/263034 objects degraded (4.160%); 4453/263034 objects misplaced (1.693%); 1.4 MiB/s, 21 objects/s recovering 2023-09-23T20:58:35.253 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:58:35 smithi026 ceph-mon[179688]: Health check update: Degraded data redundancy: 10943/265383 objects degraded (4.123%), 2 pgs degraded (PG_DEGRADED) 2023-09-23T20:58:35.253 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:58:35 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/515711627' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:58:35.253 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:58:35 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:58:35.254 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:58:35 smithi026 ceph-mon[175151]: pgmap v557: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 5.4 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 619 op/s; 10943/263034 objects degraded (4.160%); 4453/263034 objects misplaced (1.693%); 1.4 MiB/s, 21 objects/s recovering 2023-09-23T20:58:35.254 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:58:35 smithi026 ceph-mon[175151]: Health check update: Degraded data redundancy: 10943/265383 objects degraded (4.123%), 2 pgs degraded (PG_DEGRADED) 2023-09-23T20:58:35.305 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:58:35.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:58:35 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/515711627' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:58:35.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:58:35 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:58:35.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:58:35 smithi070 ceph-mon[168282]: pgmap v557: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 5.4 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 619 op/s; 10943/263034 objects degraded (4.160%); 4453/263034 objects misplaced (1.693%); 1.4 MiB/s, 21 objects/s recovering 2023-09-23T20:58:35.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:58:35 smithi070 ceph-mon[168282]: Health check update: Degraded data redundancy: 10943/265383 objects degraded (4.123%), 2 pgs degraded (PG_DEGRADED) 2023-09-23T20:58:35.402 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:58:35.744 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:58:35.994 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:58:35.999 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:58:36.000 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:58:36.100 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:58:36.132 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:58:36.171 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:58:36.174 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:58:36.175 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:58:36.254 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:58:36 smithi026 ceph-mon[179688]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:58:36.254 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:58:36 smithi026 ceph-mon[179688]: failed to encode map e235 with expected crc 2023-09-23T20:58:36.254 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:58:36 smithi026 ceph-mon[179688]: failed to encode map e235 with expected crc 2023-09-23T20:58:36.254 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:58:36 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T20:58:36.254 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:58:36 smithi026 ceph-mon[179688]: osdmap e235: 8 total, 8 up, 8 in 2023-09-23T20:58:36.254 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:58:36 smithi026 ceph-mon[179688]: failed to encode map e235 with expected crc 2023-09-23T20:58:36.255 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:58:36 smithi026 ceph-mon[175151]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:58:36.255 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:58:36 smithi026 ceph-mon[175151]: failed to encode map e235 with expected crc 2023-09-23T20:58:36.255 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:58:36 smithi026 ceph-mon[175151]: failed to encode map e235 with expected crc 2023-09-23T20:58:36.255 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:58:36 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T20:58:36.255 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:58:36 smithi026 ceph-mon[175151]: osdmap e235: 8 total, 8 up, 8 in 2023-09-23T20:58:36.255 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:58:36 smithi026 ceph-mon[175151]: failed to encode map e235 with expected crc 2023-09-23T20:58:36.263 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:58:36 smithi070 ceph-mon[168282]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:58:36.263 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:58:36 smithi070 ceph-mon[168282]: failed to encode map e235 with expected crc 2023-09-23T20:58:36.263 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:58:36 smithi070 ceph-mon[168282]: failed to encode map e235 with expected crc 2023-09-23T20:58:36.264 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:58:36 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T20:58:36.264 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:58:36 smithi070 ceph-mon[168282]: osdmap e235: 8 total, 8 up, 8 in 2023-09-23T20:58:36.264 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:58:36 smithi070 ceph-mon[168282]: failed to encode map e235 with expected crc 2023-09-23T20:58:36.276 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:58:36.309 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:58:36.526 INFO:teuthology.orchestra.run.smithi026.stdout: 2023-09-23T20:58:36.538 INFO:teuthology.orchestra.run.smithi026.stderr:dumped all 2023-09-23T20:58:36.544 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:58:36.548 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:58:36.548 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:58:36.599 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0: {} 2023-09-23T20:58:36.645 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1: {} 2023-09-23T20:58:36.649 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:58:36.691 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2: {} 2023-09-23T20:58:36.740 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:58:36.743 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3: {} 2023-09-23T20:58:36.780 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4: {} 2023-09-23T20:58:36.814 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T20:58:36.875 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6: {} 2023-09-23T20:58:36.915 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T20:58:36.941 INFO:teuthology.orchestra.run.smithi026.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T20:58:36.942 INFO:teuthology.orchestra.run.smithi026.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T20:58:36.942 INFO:teuthology.orchestra.run.smithi026.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T20:58:36.942 INFO:teuthology.orchestra.run.smithi026.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T20:58:36.942 INFO:teuthology.orchestra.run.smithi026.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T20:58:36.942 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T20:58:36.942 INFO:teuthology.orchestra.run.smithi026.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T20:58:36.942 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T20:58:37.012 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:58:37.021 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:58:37.021 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:58:37.065 INFO:teuthology.orchestra.run.smithi026.stdout:{"pg_ready":true,"pg_map":{"version":559,"stamp":"2023-09-23T20:58:36.405416+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":5661398968,"num_objects":86402,"num_object_clones":0,"num_object_copies":259206,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":10943,"num_objects_misplaced":4342,"num_objects_unfound":0,"num_objects_dirty":86402,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":117203,"num_write_kb":6512606,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1079,"num_bytes_recovered":69277542,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":116278,"ondisk_log_size":116278,"up":327,"acting":324,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":331,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":18389032,"kb_used_data":16894568,"kb_used_omap":136,"kb_used_meta":1494071,"kb_avail":731572184,"statfs":{"total":767960285184,"available":749129916416,"internally_reserved":0,"allocated":17300037632,"data_stored":17292263225,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":139741,"internal_metadata":1529929251},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,1,1,4,4,10],"upper_bound":64},"perf_stat":{"commit_latency_ms":10,"apply_latency_ms":10,"commit_latency_ns":10000000,"apply_latency_ns":10000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-406978560,"num_objects":-6210,"num_object_clones":0,"num_object_copies":-18630,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":-221,"num_objects_unfound":0,"num_objects_dirty":-6210,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6210,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":221,"num_bytes_recovered":14483456,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.022872"},"pg_stats":[{"pgid":"3.1a","version":"68'1","reported_seq":111,"reported_epoch":233,"state":"active+clean","last_fresh":"2023-09-23T20:58:19.485182+0000","last_change":"2023-09-23T20:54:13.946695+0000","last_active":"2023-09-23T20:58:19.485182+0000","last_peered":"2023-09-23T20:58:19.485182+0000","last_clean":"2023-09-23T20:58:19.485182+0000","last_became_active":"2023-09-23T20:54:13.945399+0000","last_became_peered":"2023-09-23T20:54:13.945399+0000","last_unstale":"2023-09-23T20:58:19.485182+0000","last_undegraded":"2023-09-23T20:58:19.485182+0000","last_fullsized":"2023-09-23T20:58:19.485182+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:32:52.365464+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":225,"reported_epoch":233,"state":"active+clean","last_fresh":"2023-09-23T20:58:19.489122+0000","last_change":"2023-09-23T20:54:13.942019+0000","last_active":"2023-09-23T20:58:19.489122+0000","last_peered":"2023-09-23T20:58:19.489122+0000","last_clean":"2023-09-23T20:58:19.489122+0000","last_became_active":"2023-09-23T20:54:13.941647+0000","last_became_peered":"2023-09-23T20:54:13.941647+0000","last_unstale":"2023-09-23T20:58:19.489122+0000","last_undegraded":"2023-09-23T20:58:19.489122+0000","last_fullsized":"2023-09-23T20:58:19.489122+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:30:54.400944+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":203,"reported_epoch":233,"state":"active+clean","last_fresh":"2023-09-23T20:58:19.477598+0000","last_change":"2023-09-23T20:54:13.941394+0000","last_active":"2023-09-23T20:58:19.477598+0000","last_peered":"2023-09-23T20:58:19.477598+0000","last_clean":"2023-09-23T20:58:19.477598+0000","last_became_active":"2023-09-23T20:54:13.941042+0000","last_became_peered":"2023-09-23T20:54:13.941042+0000","last_unstale":"2023-09-23T20:58:19.477598+0000","last_undegraded":"2023-09-23T20:58:19.477598+0000","last_fullsized":"2023-09-23T20:58:19.477598+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:19:41.401184+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":219,"reported_epoch":234,"state":"active+clean","last_fresh":"2023-09-23T20:58:26.130527+0000","last_change":"2023-09-23T20:57:44.979811+0000","last_active":"2023-09-23T20:58:26.130527+0000","last_peered":"2023-09-23T20:58:26.130527+0000","last_clean":"2023-09-23T20:58:26.130527+0000","last_became_active":"2023-09-23T20:57:44.979515+0000","last_became_peered":"2023-09-23T20:57:44.979515+0000","last_unstale":"2023-09-23T20:58:26.130527+0000","last_undegraded":"2023-09-23T20:58:26.130527+0000","last_fullsized":"2023-09-23T20:58:26.130527+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:38:22.162140+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":217,"reported_epoch":233,"state":"active+clean","last_fresh":"2023-09-23T20:58:24.408875+0000","last_change":"2023-09-23T20:51:10.833613+0000","last_active":"2023-09-23T20:58:24.408875+0000","last_peered":"2023-09-23T20:58:24.408875+0000","last_clean":"2023-09-23T20:58:24.408875+0000","last_became_active":"2023-09-23T20:51:10.833208+0000","last_became_peered":"2023-09-23T20:51:10.833208+0000","last_unstale":"2023-09-23T20:58:24.408875+0000","last_undegraded":"2023-09-23T20:58:24.408875+0000","last_fullsized":"2023-09-23T20:58:24.408875+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:51:28.246014+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":199,"reported_epoch":233,"state":"active+clean","last_fresh":"2023-09-23T20:58:19.469667+0000","last_change":"2023-09-23T20:55:35.958384+0000","last_active":"2023-09-23T20:58:19.469667+0000","last_peered":"2023-09-23T20:58:19.469667+0000","last_clean":"2023-09-23T20:58:19.469667+0000","last_became_active":"2023-09-23T20:55:35.954563+0000","last_became_peered":"2023-09-23T20:55:35.954563+0000","last_unstale":"2023-09-23T20:58:19.469667+0000","last_undegraded":"2023-09-23T20:58:19.469667+0000","last_fullsized":"2023-09-23T20:58:19.469667+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:37:48.332259+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":209,"reported_epoch":234,"state":"active+clean","last_fresh":"2023-09-23T20:58:26.136449+0000","last_change":"2023-09-23T20:54:31.885514+0000","last_active":"2023-09-23T20:58:26.136449+0000","last_peered":"2023-09-23T20:58:26.136449+0000","last_clean":"2023-09-23T20:58:26.136449+0000","last_became_active":"2023-09-23T20:54:31.884986+0000","last_became_peered":"2023-09-23T20:54:31.884986+0000","last_unstale":"2023-09-23T20:58:26.136449+0000","last_undegraded":"2023-09-23T20:58:26.136449+0000","last_fullsized":"2023-09-23T20:58:26.136449+0000","mapping_epoch":136,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:26:08.100220+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":214,"reported_epoch":233,"state":"active+clean","last_fresh":"2023-09-23T20:58:19.486290+0000","last_change":"2023-09-23T20:55:35.951889+0000","last_active":"2023-09-23T20:58:19.486290+0000","last_peered":"2023-09-23T20:58:19.486290+0000","last_clean":"2023-09-23T20:58:19.486290+0000","last_became_active":"2023-09-23T20:55:35.951563+0000","last_became_peered":"2023-09-23T20:55:35.951563+0000","last_unstale":"2023-09-23T20:58:19.486290+0000","last_undegraded":"2023-09-23T20:58:19.486290+0000","last_fullsized":"2023-09-23T20:58:19.486290+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:41:53.098781+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"79'161","reported_seq":520,"reported_epoch":233,"state":"active+clean","last_fresh":"2023-09-23T20:58:19.490945+0000","last_change":"2023-09-23T20:57:25.096559+0000","last_active":"2023-09-23T20:58:19.490945+0000","last_peered":"2023-09-23T20:58:19.490945+0000","last_clean":"2023-09-23T20:58:19.490945+0000","last_became_active":"2023-09-23T20:57:15.975674+0000","last_became_peered":"2023-09-23T20:57:15.975674+0000","last_unstale":"2023-09-23T20:58:19.490945+0000","last_undegraded":"2023-09-23T20:58:19.490945+0000","last_fullsized":"2023-09-23T20:58:19.490945+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:25.096401+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:25.096401+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:05:05.448404+0000","scrub_duration":0.002859393,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":107,"reported_epoch":233,"state":"active+clean","last_fresh":"2023-09-23T20:58:19.482714+0000","last_change":"2023-09-23T20:54:31.890678+0000","last_active":"2023-09-23T20:58:19.482714+0000","last_peered":"2023-09-23T20:58:19.482714+0000","last_clean":"2023-09-23T20:58:19.482714+0000","last_became_active":"2023-09-23T20:54:31.890469+0000","last_became_peered":"2023-09-23T20:54:31.890469+0000","last_unstale":"2023-09-23T20:58:19.482714+0000","last_undegraded":"2023-09-23T20:58:19.482714+0000","last_fullsized":"2023-09-23T20:58:19.482714+0000","mapping_epoch":136,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:55:08.790146+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":175,"reported_epoch":233,"state":"active+clean","last_fresh":"2023-09-23T20:58:19.484172+0000","last_change":"2023-09-23T20:55:35.952661+0000","last_active":"2023-09-23T20:58:19.484172+0000","last_peered":"2023-09-23T20:58:19.484172+0000","last_clean":"2023-09-23T20:58:19.484172+0000","last_became_active":"2023-09-23T20:55:35.952349+0000","last_became_peered":"2023-09-23T20:55:35.952349+0000","last_unstale":"2023-09-23T20:58:19.484172+0000","last_undegraded":"2023-09-23T20:58:19.484172+0000","last_fullsized":"2023-09-23T20:58:19.484172+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:00:03.110126+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":51,"reported_epoch":233,"state":"active+clean","last_fresh":"2023-09-23T20:58:19.489213+0000","last_change":"2023-09-23T20:56:45.988607+0000","last_active":"2023-09-23T20:58:19.489213+0000","last_peered":"2023-09-23T20:58:19.489213+0000","last_clean":"2023-09-23T20:58:19.489213+0000","last_became_active":"2023-09-23T20:56:45.988206+0000","last_became_peered":"2023-09-23T20:56:45.988206+0000","last_unstale":"2023-09-23T20:58:19.489213+0000","last_undegraded":"2023-09-23T20:58:19.489213+0000","last_fullsized":"2023-09-23T20:58:19.489213+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":205,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:01:19.783688+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,4],"acting":[3,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":51,"reported_epoch":233,"state":"active+clean","last_fresh":"2023-09-23T20:58:19.482495+0000","last_change":"2023-09-23T20:56:18.139142+0000","last_active":"2023-09-23T20:58:19.482495+0000","last_peered":"2023-09-23T20:58:19.482495+0000","last_clean":"2023-09-23T20:58:19.482495+0000","last_became_active":"2023-09-23T20:56:18.138844+0000","last_became_peered":"2023-09-23T20:56:18.138844+0000","last_unstale":"2023-09-23T20:58:19.482495+0000","last_undegraded":"2023-09-23T20:58:19.482495+0000","last_fullsized":"2023-09-23T20:58:19.482495+0000","mapping_epoch":192,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":193,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:09:02.217683+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,2],"acting":[5,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.14","version":"79'22","reported_seq":138,"reported_epoch":233,"state":"active+clean","last_fresh":"2023-09-23T20:58:19.485501+0000","last_change":"2023-09-23T20:55:35.949418+0000","last_active":"2023-09-23T20:58:19.485501+0000","last_peered":"2023-09-23T20:58:19.485501+0000","last_clean":"2023-09-23T20:58:19.485501+0000","last_became_active":"2023-09-23T20:55:35.948566+0000","last_became_peered":"2023-09-23T20:55:35.948566+0000","last_unstale":"2023-09-23T20:58:19.485501+0000","last_undegraded":"2023-09-23T20:58:19.485501+0000","last_fullsized":"2023-09-23T20:58:19.485501+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:59:37.433128+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4734,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":128,"reported_epoch":233,"state":"active+clean","last_fresh":"2023-09-23T20:58:19.485727+0000","last_change":"2023-09-23T20:55:35.949178+0000","last_active":"2023-09-23T20:58:19.485727+0000","last_peered":"2023-09-23T20:58:19.485727+0000","last_clean":"2023-09-23T20:58:19.485727+0000","last_became_active":"2023-09-23T20:55:35.948795+0000","last_became_peered":"2023-09-23T20:55:35.948795+0000","last_unstale":"2023-09-23T20:58:19.485727+0000","last_undegraded":"2023-09-23T20:58:19.485727+0000","last_fullsized":"2023-09-23T20:58:19.485727+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:13:08.996932+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.11","version":"79'161","reported_seq":463,"reported_epoch":234,"state":"active+clean","last_fresh":"2023-09-23T20:58:26.125646+0000","last_change":"2023-09-23T20:57:28.756796+0000","last_active":"2023-09-23T20:58:26.125646+0000","last_peered":"2023-09-23T20:58:26.125646+0000","last_clean":"2023-09-23T20:58:26.125646+0000","last_became_active":"2023-09-23T20:57:15.980606+0000","last_became_peered":"2023-09-23T20:57:15.980606+0000","last_unstale":"2023-09-23T20:58:26.125646+0000","last_undegraded":"2023-09-23T20:58:26.125646+0000","last_fullsized":"2023-09-23T20:58:26.125646+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:28.756634+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:28.756634+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:09:19.150791+0000","scrub_duration":0.0064925119999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":154,"reported_epoch":233,"state":"active+clean","last_fresh":"2023-09-23T20:58:19.485366+0000","last_change":"2023-09-23T20:56:45.992577+0000","last_active":"2023-09-23T20:58:19.485366+0000","last_peered":"2023-09-23T20:58:19.485366+0000","last_clean":"2023-09-23T20:58:19.485366+0000","last_became_active":"2023-09-23T20:56:45.992297+0000","last_became_peered":"2023-09-23T20:56:45.992297+0000","last_unstale":"2023-09-23T20:58:19.485366+0000","last_undegraded":"2023-09-23T20:58:19.485366+0000","last_fullsized":"2023-09-23T20:58:19.485366+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":205,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:00:45.547202+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,2],"acting":[4,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":199,"reported_epoch":233,"state":"active+clean","last_fresh":"2023-09-23T20:58:19.494497+0000","last_change":"2023-09-23T20:51:10.831262+0000","last_active":"2023-09-23T20:58:19.494497+0000","last_peered":"2023-09-23T20:58:19.494497+0000","last_clean":"2023-09-23T20:58:19.494497+0000","last_became_active":"2023-09-23T20:51:10.830931+0000","last_became_peered":"2023-09-23T20:51:10.830931+0000","last_unstale":"2023-09-23T20:58:19.494497+0000","last_undegraded":"2023-09-23T20:58:19.494497+0000","last_fullsized":"2023-09-23T20:58:19.494497+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:51:54.982505+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.10","version":"79'161","reported_seq":471,"reported_epoch":233,"state":"active+clean","last_fresh":"2023-09-23T20:58:19.482175+0000","last_change":"2023-09-23T20:57:20.708003+0000","last_active":"2023-09-23T20:58:19.482175+0000","last_peered":"2023-09-23T20:58:19.482175+0000","last_clean":"2023-09-23T20:58:19.482175+0000","last_became_active":"2023-09-23T20:57:15.975302+0000","last_became_peered":"2023-09-23T20:57:15.975302+0000","last_unstale":"2023-09-23T20:58:19.482175+0000","last_undegraded":"2023-09-23T20:58:19.482175+0000","last_fullsized":"2023-09-23T20:58:19.482175+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:20.707939+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:20.707939+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:16:51.721994+0000","scrub_duration":0.0017733409999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":209,"reported_epoch":234,"state":"active+clean","last_fresh":"2023-09-23T20:58:26.126213+0000","last_change":"2023-09-23T20:57:44.979736+0000","last_active":"2023-09-23T20:58:26.126213+0000","last_peered":"2023-09-23T20:58:26.126213+0000","last_clean":"2023-09-23T20:58:26.126213+0000","last_became_active":"2023-09-23T20:57:44.979356+0000","last_became_peered":"2023-09-23T20:57:44.979356+0000","last_unstale":"2023-09-23T20:58:26.126213+0000","last_undegraded":"2023-09-23T20:58:26.126213+0000","last_fullsized":"2023-09-23T20:58:26.126213+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:14:43.848832+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":107,"reported_epoch":233,"state":"active+clean","last_fresh":"2023-09-23T20:58:19.482690+0000","last_change":"2023-09-23T20:54:31.887814+0000","last_active":"2023-09-23T20:58:19.482690+0000","last_peered":"2023-09-23T20:58:19.482690+0000","last_clean":"2023-09-23T20:58:19.482690+0000","last_became_active":"2023-09-23T20:54:31.887195+0000","last_became_peered":"2023-09-23T20:54:31.887195+0000","last_unstale":"2023-09-23T20:58:19.482690+0000","last_undegraded":"2023-09-23T20:58:19.482690+0000","last_fullsized":"2023-09-23T20:58:19.482690+0000","mapping_epoch":136,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:10:52.089553+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.13","version":"79'161","reported_seq":533,"reported_epoch":233,"state":"active+clean","last_fresh":"2023-09-23T20:58:19.477756+0000","last_change":"2023-09-23T20:57:48.262970+0000","last_active":"2023-09-23T20:58:19.477756+0000","last_peered":"2023-09-23T20:58:19.477756+0000","last_clean":"2023-09-23T20:58:19.477756+0000","last_became_active":"2023-09-23T20:57:48.262663+0000","last_became_peered":"2023-09-23T20:57:48.262663+0000","last_unstale":"2023-09-23T20:58:19.477756+0000","last_undegraded":"2023-09-23T20:58:19.477756+0000","last_fullsized":"2023-09-23T20:58:19.477756+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:36.420362+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:36.420362+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:28:28.613492+0000","scrub_duration":0.028838444000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":178,"reported_epoch":233,"state":"active+clean","last_fresh":"2023-09-23T20:58:19.485180+0000","last_change":"2023-09-23T20:56:45.991822+0000","last_active":"2023-09-23T20:58:19.485180+0000","last_peered":"2023-09-23T20:58:19.485180+0000","last_clean":"2023-09-23T20:58:19.485180+0000","last_became_active":"2023-09-23T20:56:45.991460+0000","last_became_peered":"2023-09-23T20:56:45.991460+0000","last_unstale":"2023-09-23T20:58:19.485180+0000","last_undegraded":"2023-09-23T20:58:19.485180+0000","last_fullsized":"2023-09-23T20:58:19.485180+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":205,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:20:37.503317+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":191,"reported_epoch":234,"state":"active+clean","last_fresh":"2023-09-23T20:58:26.130680+0000","last_change":"2023-09-23T20:51:10.840849+0000","last_active":"2023-09-23T20:58:26.130680+0000","last_peered":"2023-09-23T20:58:26.130680+0000","last_clean":"2023-09-23T20:58:26.130680+0000","last_became_active":"2023-09-23T20:51:10.840593+0000","last_became_peered":"2023-09-23T20:51:10.840593+0000","last_unstale":"2023-09-23T20:58:26.130680+0000","last_undegraded":"2023-09-23T20:58:26.130680+0000","last_fullsized":"2023-09-23T20:58:26.130680+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:30:52.297625+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.12","version":"79'161","reported_seq":471,"reported_epoch":233,"state":"active+clean","last_fresh":"2023-09-23T20:58:19.482231+0000","last_change":"2023-09-23T20:57:21.750765+0000","last_active":"2023-09-23T20:58:19.482231+0000","last_peered":"2023-09-23T20:58:19.482231+0000","last_clean":"2023-09-23T20:58:19.482231+0000","last_became_active":"2023-09-23T20:57:15.979195+0000","last_became_peered":"2023-09-23T20:57:15.979195+0000","last_unstale":"2023-09-23T20:58:19.482231+0000","last_undegraded":"2023-09-23T20:58:19.482231+0000","last_fullsized":"2023-09-23T20:58:19.482231+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:21.750680+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:21.750680+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:03:41.915962+0000","scrub_duration":0.0092391680000000007,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":226,"reported_epoch":233,"state":"active+clean","last_fresh":"2023-09-23T20:58:19.469124+0000","last_change":"2023-09-23T20:55:36.943969+0000","last_active":"2023-09-23T20:58:19.469124+0000","last_peered":"2023-09-23T20:58:19.469124+0000","last_clean":"2023-09-23T20:58:19.469124+0000","last_became_active":"2023-09-23T20:55:36.943580+0000","last_became_peered":"2023-09-23T20:55:36.943580+0000","last_unstale":"2023-09-23T20:58:19.469124+0000","last_undegraded":"2023-09-23T20:58:19.469124+0000","last_fullsized":"2023-09-23T20:58:19.469124+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:49:06.969396+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":22,"reported_epoch":233,"state":"active+clean","last_fresh":"2023-09-23T20:58:19.493216+0000","last_change":"2023-09-23T20:57:44.980498+0000","last_active":"2023-09-23T20:58:19.493216+0000","last_peered":"2023-09-23T20:58:19.493216+0000","last_clean":"2023-09-23T20:58:19.493216+0000","last_became_active":"2023-09-23T20:57:44.980182+0000","last_became_peered":"2023-09-23T20:57:44.980182+0000","last_unstale":"2023-09-23T20:58:19.493216+0000","last_undegraded":"2023-09-23T20:58:19.493216+0000","last_fullsized":"2023-09-23T20:58:19.493216+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:34:51.620544+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.d","version":"79'161","reported_seq":516,"reported_epoch":233,"state":"active+clean","last_fresh":"2023-09-23T20:58:19.485398+0000","last_change":"2023-09-23T20:57:31.075213+0000","last_active":"2023-09-23T20:58:19.485398+0000","last_peered":"2023-09-23T20:58:19.485398+0000","last_clean":"2023-09-23T20:58:19.485398+0000","last_became_active":"2023-09-23T20:57:15.988188+0000","last_became_peered":"2023-09-23T20:57:15.988188+0000","last_unstale":"2023-09-23T20:58:19.485398+0000","last_undegraded":"2023-09-23T20:58:19.485398+0000","last_fullsized":"2023-09-23T20:58:19.485398+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:31.075098+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:31.075098+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:41:52.142362+0000","scrub_duration":0.010304610000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.b","version":"234'7273","reported_seq":7317,"reported_epoch":234,"state":"active+clean","last_fresh":"2023-09-23T20:58:34.251990+0000","last_change":"2023-09-23T20:56:35.543994+0000","last_active":"2023-09-23T20:58:34.251990+0000","last_peered":"2023-09-23T20:58:34.251990+0000","last_clean":"2023-09-23T20:58:34.251990+0000","last_became_active":"2023-09-23T20:56:35.543758+0000","last_became_peered":"2023-09-23T20:56:35.543758+0000","last_unstale":"2023-09-23T20:58:34.251990+0000","last_undegraded":"2023-09-23T20:58:34.251990+0000","last_fullsized":"2023-09-23T20:58:34.251990+0000","mapping_epoch":199,"log_start":"202'200","ondisk_log_start":"202'200","created":199,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_clean_scrub_stamp":"2023-09-23T20:56:34.529103+0000","objects_scrubbed":0,"log_size":7073,"log_dups_size":0,"ondisk_log_size":7073,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:21:29.041484+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":354353152,"num_objects":5407,"num_object_clones":0,"num_object_copies":16221,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5407,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7273,"num_write_kb":405760,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":165,"reported_epoch":233,"state":"active+clean","last_fresh":"2023-09-23T20:58:19.485515+0000","last_change":"2023-09-23T20:56:45.991841+0000","last_active":"2023-09-23T20:58:19.485515+0000","last_peered":"2023-09-23T20:58:19.485515+0000","last_clean":"2023-09-23T20:58:19.485515+0000","last_became_active":"2023-09-23T20:56:45.991491+0000","last_became_peered":"2023-09-23T20:56:45.991491+0000","last_unstale":"2023-09-23T20:58:19.485515+0000","last_undegraded":"2023-09-23T20:58:19.485515+0000","last_fullsized":"2023-09-23T20:58:19.485515+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":205,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:30:37.405319+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":124,"reported_epoch":233,"state":"active+clean","last_fresh":"2023-09-23T20:58:19.482290+0000","last_change":"2023-09-23T20:55:35.957768+0000","last_active":"2023-09-23T20:58:19.482290+0000","last_peered":"2023-09-23T20:58:19.482290+0000","last_clean":"2023-09-23T20:58:19.482290+0000","last_became_active":"2023-09-23T20:55:35.957421+0000","last_became_peered":"2023-09-23T20:55:35.957421+0000","last_unstale":"2023-09-23T20:58:19.482290+0000","last_undegraded":"2023-09-23T20:58:19.482290+0000","last_fullsized":"2023-09-23T20:58:19.482290+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:49:41.585961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.c","version":"79'161","reported_seq":534,"reported_epoch":233,"state":"active+clean","last_fresh":"2023-09-23T20:58:19.494345+0000","last_change":"2023-09-23T20:57:22.041071+0000","last_active":"2023-09-23T20:58:19.494345+0000","last_peered":"2023-09-23T20:58:19.494345+0000","last_clean":"2023-09-23T20:58:19.494345+0000","last_became_active":"2023-09-23T20:57:15.973204+0000","last_became_peered":"2023-09-23T20:57:15.973204+0000","last_unstale":"2023-09-23T20:58:19.494345+0000","last_undegraded":"2023-09-23T20:58:19.494345+0000","last_fullsized":"2023-09-23T20:58:19.494345+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:22.040907+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:22.040907+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:47:02.019978+0000","scrub_duration":0.013021176000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.a","version":"234'7346","reported_seq":7390,"reported_epoch":234,"state":"active+clean","last_fresh":"2023-09-23T20:58:34.475904+0000","last_change":"2023-09-23T20:56:35.547120+0000","last_active":"2023-09-23T20:58:34.475904+0000","last_peered":"2023-09-23T20:58:34.475904+0000","last_clean":"2023-09-23T20:58:34.475904+0000","last_became_active":"2023-09-23T20:56:35.546838+0000","last_became_peered":"2023-09-23T20:56:35.546838+0000","last_unstale":"2023-09-23T20:58:34.475904+0000","last_undegraded":"2023-09-23T20:58:34.475904+0000","last_fullsized":"2023-09-23T20:58:34.475904+0000","mapping_epoch":199,"log_start":"205'1200","ondisk_log_start":"205'1200","created":199,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_clean_scrub_stamp":"2023-09-23T20:56:34.529103+0000","objects_scrubbed":0,"log_size":6146,"log_dups_size":0,"ondisk_log_size":6146,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:48:24.132768+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":353239040,"num_objects":5390,"num_object_clones":0,"num_object_copies":16170,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5390,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7346,"num_write_kb":407552,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,5],"acting":[3,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":152,"reported_epoch":233,"state":"active+clean","last_fresh":"2023-09-23T20:58:19.485701+0000","last_change":"2023-09-23T20:56:45.991821+0000","last_active":"2023-09-23T20:58:19.485701+0000","last_peered":"2023-09-23T20:58:19.485701+0000","last_clean":"2023-09-23T20:58:19.485701+0000","last_became_active":"2023-09-23T20:56:45.991447+0000","last_became_peered":"2023-09-23T20:56:45.991447+0000","last_unstale":"2023-09-23T20:58:19.485701+0000","last_undegraded":"2023-09-23T20:58:19.485701+0000","last_fullsized":"2023-09-23T20:58:19.485701+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":205,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:14:25.525222+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":155,"reported_epoch":233,"state":"active+clean","last_fresh":"2023-09-23T20:58:19.485457+0000","last_change":"2023-09-23T20:57:44.982059+0000","last_active":"2023-09-23T20:58:19.485457+0000","last_peered":"2023-09-23T20:58:19.485457+0000","last_clean":"2023-09-23T20:58:19.485457+0000","last_became_active":"2023-09-23T20:57:44.981742+0000","last_became_peered":"2023-09-23T20:57:44.981742+0000","last_unstale":"2023-09-23T20:58:19.485457+0000","last_undegraded":"2023-09-23T20:58:19.485457+0000","last_fullsized":"2023-09-23T20:58:19.485457+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:41:24.042916+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.f","version":"79'161","reported_seq":538,"reported_epoch":233,"state":"active+clean","last_fresh":"2023-09-23T20:58:19.469433+0000","last_change":"2023-09-23T20:57:27.880841+0000","last_active":"2023-09-23T20:58:19.469433+0000","last_peered":"2023-09-23T20:58:19.469433+0000","last_clean":"2023-09-23T20:58:19.469433+0000","last_became_active":"2023-09-23T20:57:15.970925+0000","last_became_peered":"2023-09-23T20:57:15.970925+0000","last_unstale":"2023-09-23T20:58:19.469433+0000","last_undegraded":"2023-09-23T20:58:19.469433+0000","last_fullsized":"2023-09-23T20:58:19.469433+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:27.880776+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:27.880776+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:15:13.438014+0000","scrub_duration":0.0014004899999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.9","version":"235'7420","reported_seq":7464,"reported_epoch":235,"state":"active+clean","last_fresh":"2023-09-23T20:58:36.298465+0000","last_change":"2023-09-23T20:56:35.549387+0000","last_active":"2023-09-23T20:58:36.298465+0000","last_peered":"2023-09-23T20:58:36.298465+0000","last_clean":"2023-09-23T20:58:36.298465+0000","last_became_active":"2023-09-23T20:56:35.549087+0000","last_became_peered":"2023-09-23T20:56:35.549087+0000","last_unstale":"2023-09-23T20:58:36.298465+0000","last_undegraded":"2023-09-23T20:58:36.298465+0000","last_fullsized":"2023-09-23T20:58:36.298465+0000","mapping_epoch":199,"log_start":"202'100","ondisk_log_start":"202'100","created":199,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_clean_scrub_stamp":"2023-09-23T20:56:34.529103+0000","objects_scrubbed":0,"log_size":7320,"log_dups_size":0,"ondisk_log_size":7320,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:04:51.547362+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":346423296,"num_objects":5286,"num_object_clones":0,"num_object_copies":15858,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5286,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7420,"num_write_kb":406592,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":138,"reported_epoch":233,"state":"active+clean","last_fresh":"2023-09-23T20:58:19.482620+0000","last_change":"2023-09-23T20:56:45.989153+0000","last_active":"2023-09-23T20:58:19.482620+0000","last_peered":"2023-09-23T20:58:19.482620+0000","last_clean":"2023-09-23T20:58:19.482620+0000","last_became_active":"2023-09-23T20:56:45.988800+0000","last_became_peered":"2023-09-23T20:56:45.988800+0000","last_unstale":"2023-09-23T20:58:19.482620+0000","last_undegraded":"2023-09-23T20:58:19.482620+0000","last_fullsized":"2023-09-23T20:58:19.482620+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":205,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:04:41.130194+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":173,"reported_epoch":233,"state":"active+clean","last_fresh":"2023-09-23T20:58:19.469279+0000","last_change":"2023-09-23T20:55:35.965427+0000","last_active":"2023-09-23T20:58:19.469279+0000","last_peered":"2023-09-23T20:58:19.469279+0000","last_clean":"2023-09-23T20:58:19.469279+0000","last_became_active":"2023-09-23T20:55:35.963969+0000","last_became_peered":"2023-09-23T20:55:35.963969+0000","last_unstale":"2023-09-23T20:58:19.469279+0000","last_undegraded":"2023-09-23T20:58:19.469279+0000","last_fullsized":"2023-09-23T20:58:19.469279+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:38:40.612083+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.e","version":"79'161","reported_seq":445,"reported_epoch":233,"state":"active+clean","last_fresh":"2023-09-23T20:58:19.485554+0000","last_change":"2023-09-23T20:57:34.132738+0000","last_active":"2023-09-23T20:58:19.485554+0000","last_peered":"2023-09-23T20:58:19.485554+0000","last_clean":"2023-09-23T20:58:19.485554+0000","last_became_active":"2023-09-23T20:57:15.974318+0000","last_became_peered":"2023-09-23T20:57:15.974318+0000","last_unstale":"2023-09-23T20:58:19.485554+0000","last_undegraded":"2023-09-23T20:58:19.485554+0000","last_fullsized":"2023-09-23T20:58:19.485554+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:34.132660+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:34.132660+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:07:07.739438+0000","scrub_duration":0.005229694,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.8","version":"234'7304","reported_seq":7348,"reported_epoch":234,"state":"active+clean","last_fresh":"2023-09-23T20:58:31.365457+0000","last_change":"2023-09-23T20:56:35.548119+0000","last_active":"2023-09-23T20:58:31.365457+0000","last_peered":"2023-09-23T20:58:31.365457+0000","last_clean":"2023-09-23T20:58:31.365457+0000","last_became_active":"2023-09-23T20:56:35.547914+0000","last_became_peered":"2023-09-23T20:56:35.547914+0000","last_unstale":"2023-09-23T20:58:31.365457+0000","last_undegraded":"2023-09-23T20:58:31.365457+0000","last_fullsized":"2023-09-23T20:58:31.365457+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":199,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_clean_scrub_stamp":"2023-09-23T20:56:34.529103+0000","objects_scrubbed":0,"log_size":7304,"log_dups_size":0,"ondisk_log_size":7304,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:42:54.255023+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":363724800,"num_objects":5550,"num_object_clones":0,"num_object_copies":16650,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5550,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7304,"num_write_kb":411328,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":124,"reported_epoch":233,"state":"active+clean","last_fresh":"2023-09-23T20:58:19.482617+0000","last_change":"2023-09-23T20:55:35.962005+0000","last_active":"2023-09-23T20:58:19.482617+0000","last_peered":"2023-09-23T20:58:19.482617+0000","last_clean":"2023-09-23T20:58:19.482617+0000","last_became_active":"2023-09-23T20:55:35.961695+0000","last_became_peered":"2023-09-23T20:55:35.961695+0000","last_unstale":"2023-09-23T20:58:19.482617+0000","last_undegraded":"2023-09-23T20:58:19.482617+0000","last_fullsized":"2023-09-23T20:58:19.482617+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:28:22.876201+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":223,"reported_epoch":234,"state":"active+clean","last_fresh":"2023-09-23T20:58:31.339098+0000","last_change":"2023-09-23T20:54:13.928715+0000","last_active":"2023-09-23T20:58:31.339098+0000","last_peered":"2023-09-23T20:58:31.339098+0000","last_clean":"2023-09-23T20:58:31.339098+0000","last_became_active":"2023-09-23T20:54:13.928200+0000","last_became_peered":"2023-09-23T20:54:13.928200+0000","last_unstale":"2023-09-23T20:58:31.339098+0000","last_undegraded":"2023-09-23T20:58:31.339098+0000","last_fullsized":"2023-09-23T20:58:31.339098+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:41:16.927513+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.f","version":"235'7468","reported_seq":7513,"reported_epoch":235,"state":"active+clean","last_fresh":"2023-09-23T20:58:36.326557+0000","last_change":"2023-09-23T20:56:35.545128+0000","last_active":"2023-09-23T20:58:36.326557+0000","last_peered":"2023-09-23T20:58:36.326557+0000","last_clean":"2023-09-23T20:58:36.326557+0000","last_became_active":"2023-09-23T20:56:35.544838+0000","last_became_peered":"2023-09-23T20:56:35.544838+0000","last_unstale":"2023-09-23T20:58:36.326557+0000","last_undegraded":"2023-09-23T20:58:36.326557+0000","last_fullsized":"2023-09-23T20:58:36.326557+0000","mapping_epoch":199,"log_start":"202'100","ondisk_log_start":"202'100","created":199,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_clean_scrub_stamp":"2023-09-23T20:56:34.529103+0000","objects_scrubbed":0,"log_size":7368,"log_dups_size":0,"ondisk_log_size":7368,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:30:43.388717+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":353763328,"num_objects":5398,"num_object_clones":0,"num_object_copies":16194,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5398,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7468,"num_write_kb":411712,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.9","version":"79'161","reported_seq":606,"reported_epoch":233,"state":"active+clean","last_fresh":"2023-09-23T20:58:19.484068+0000","last_change":"2023-09-23T20:56:46.921939+0000","last_active":"2023-09-23T20:58:19.484068+0000","last_peered":"2023-09-23T20:58:19.484068+0000","last_clean":"2023-09-23T20:58:19.484068+0000","last_became_active":"2023-09-23T20:56:45.989908+0000","last_became_peered":"2023-09-23T20:56:45.989908+0000","last_unstale":"2023-09-23T20:58:19.484068+0000","last_undegraded":"2023-09-23T20:58:19.484068+0000","last_fullsized":"2023-09-23T20:58:19.484068+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":205,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:56:46.921739+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:56:46.921739+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:20:42.635110+0000","scrub_duration":0.0071954109999999996,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":223,"reported_epoch":233,"state":"active+clean","last_fresh":"2023-09-23T20:58:19.497740+0000","last_change":"2023-09-23T20:54:13.961788+0000","last_active":"2023-09-23T20:58:19.497740+0000","last_peered":"2023-09-23T20:58:19.497740+0000","last_clean":"2023-09-23T20:58:19.497740+0000","last_became_active":"2023-09-23T20:54:13.940937+0000","last_became_peered":"2023-09-23T20:54:13.940937+0000","last_unstale":"2023-09-23T20:58:19.497740+0000","last_undegraded":"2023-09-23T20:58:19.497740+0000","last_fullsized":"2023-09-23T20:58:19.497740+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:11:56.785299+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":128,"reported_epoch":233,"state":"active+clean","last_fresh":"2023-09-23T20:58:19.485686+0000","last_change":"2023-09-23T20:55:35.949670+0000","last_active":"2023-09-23T20:58:19.485686+0000","last_peered":"2023-09-23T20:58:19.485686+0000","last_clean":"2023-09-23T20:58:19.485686+0000","last_became_active":"2023-09-23T20:55:35.949140+0000","last_became_peered":"2023-09-23T20:55:35.949140+0000","last_unstale":"2023-09-23T20:58:19.485686+0000","last_undegraded":"2023-09-23T20:58:19.485686+0000","last_fullsized":"2023-09-23T20:58:19.485686+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:55:59.340303+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.e","version":"234'7271","reported_seq":7315,"reported_epoch":234,"state":"active+clean","last_fresh":"2023-09-23T20:58:31.411196+0000","last_change":"2023-09-23T20:56:35.547184+0000","last_active":"2023-09-23T20:58:31.411196+0000","last_peered":"2023-09-23T20:58:31.411196+0000","last_clean":"2023-09-23T20:58:31.411196+0000","last_became_active":"2023-09-23T20:56:35.546912+0000","last_became_peered":"2023-09-23T20:56:35.546912+0000","last_unstale":"2023-09-23T20:58:31.411196+0000","last_undegraded":"2023-09-23T20:58:31.411196+0000","last_fullsized":"2023-09-23T20:58:31.411196+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":199,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_clean_scrub_stamp":"2023-09-23T20:56:34.529103+0000","objects_scrubbed":0,"log_size":7271,"log_dups_size":0,"ondisk_log_size":7271,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:57:07.422792+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":361037824,"num_objects":5509,"num_object_clones":0,"num_object_copies":16527,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5509,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7271,"num_write_kb":408960,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,0],"acting":[2,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.8","version":"79'161","reported_seq":477,"reported_epoch":233,"state":"active+clean","last_fresh":"2023-09-23T20:58:19.482288+0000","last_change":"2023-09-23T20:56:57.505737+0000","last_active":"2023-09-23T20:58:19.482288+0000","last_peered":"2023-09-23T20:58:19.482288+0000","last_clean":"2023-09-23T20:58:19.482288+0000","last_became_active":"2023-09-23T20:56:04.982345+0000","last_became_peered":"2023-09-23T20:56:04.982345+0000","last_unstale":"2023-09-23T20:58:19.482288+0000","last_undegraded":"2023-09-23T20:58:19.482288+0000","last_fullsized":"2023-09-23T20:58:19.482288+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:56:57.505663+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:56:57.505663+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:34:30.109724+0000","scrub_duration":0.026284108,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":196,"reported_epoch":233,"state":"active+clean","last_fresh":"2023-09-23T20:58:19.469513+0000","last_change":"2023-09-23T20:55:35.959942+0000","last_active":"2023-09-23T20:58:19.469513+0000","last_peered":"2023-09-23T20:58:19.469513+0000","last_clean":"2023-09-23T20:58:19.469513+0000","last_became_active":"2023-09-23T20:55:35.957071+0000","last_became_peered":"2023-09-23T20:55:35.957071+0000","last_unstale":"2023-09-23T20:58:19.469513+0000","last_undegraded":"2023-09-23T20:58:19.469513+0000","last_fullsized":"2023-09-23T20:58:19.469513+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:44:30.308036+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":111,"reported_epoch":233,"state":"active+clean","last_fresh":"2023-09-23T20:58:19.485765+0000","last_change":"2023-09-23T20:54:13.951459+0000","last_active":"2023-09-23T20:58:19.485765+0000","last_peered":"2023-09-23T20:58:19.485765+0000","last_clean":"2023-09-23T20:58:19.485765+0000","last_became_active":"2023-09-23T20:54:13.948409+0000","last_became_peered":"2023-09-23T20:54:13.948409+0000","last_unstale":"2023-09-23T20:58:19.485765+0000","last_undegraded":"2023-09-23T20:58:19.485765+0000","last_fullsized":"2023-09-23T20:58:19.485765+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:38:10.148061+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.b","version":"79'161","reported_seq":516,"reported_epoch":233,"state":"active+clean","last_fresh":"2023-09-23T20:58:19.477840+0000","last_change":"2023-09-23T20:57:27.320172+0000","last_active":"2023-09-23T20:58:19.477840+0000","last_peered":"2023-09-23T20:58:19.477840+0000","last_clean":"2023-09-23T20:58:19.477840+0000","last_became_active":"2023-09-23T20:57:16.009630+0000","last_became_peered":"2023-09-23T20:57:16.009630+0000","last_unstale":"2023-09-23T20:58:19.477840+0000","last_undegraded":"2023-09-23T20:58:19.477840+0000","last_fullsized":"2023-09-23T20:58:19.477840+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:27.320022+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:27.320022+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:31:35.221558+0000","scrub_duration":0.0043894679999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,5],"acting":[1,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.d","version":"234'7411","reported_seq":7455,"reported_epoch":234,"state":"active+clean","last_fresh":"2023-09-23T20:58:34.040357+0000","last_change":"2023-09-23T20:56:35.544132+0000","last_active":"2023-09-23T20:58:34.040357+0000","last_peered":"2023-09-23T20:58:34.040357+0000","last_clean":"2023-09-23T20:58:34.040357+0000","last_became_active":"2023-09-23T20:56:35.543836+0000","last_became_peered":"2023-09-23T20:56:35.543836+0000","last_unstale":"2023-09-23T20:58:34.040357+0000","last_undegraded":"2023-09-23T20:58:34.040357+0000","last_fullsized":"2023-09-23T20:58:34.040357+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":199,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_clean_scrub_stamp":"2023-09-23T20:56:34.529103+0000","objects_scrubbed":0,"log_size":7411,"log_dups_size":0,"ondisk_log_size":7411,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:27:14.636991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":352452632,"num_objects":5379,"num_object_clones":0,"num_object_copies":16137,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5379,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":7411,"num_write_kb":409217,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":111,"reported_epoch":233,"state":"active+clean","last_fresh":"2023-09-23T20:58:19.485266+0000","last_change":"2023-09-23T20:54:13.950254+0000","last_active":"2023-09-23T20:58:19.485266+0000","last_peered":"2023-09-23T20:58:19.485266+0000","last_clean":"2023-09-23T20:58:19.485266+0000","last_became_active":"2023-09-23T20:54:13.949934+0000","last_became_peered":"2023-09-23T20:54:13.949934+0000","last_unstale":"2023-09-23T20:58:19.485266+0000","last_undegraded":"2023-09-23T20:58:19.485266+0000","last_fullsized":"2023-09-23T20:58:19.485266+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:30:32.847167+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":128,"reported_epoch":233,"state":"active+clean","last_fresh":"2023-09-23T20:58:19.485777+0000","last_change":"2023-09-23T20:55:35.956153+0000","last_active":"2023-09-23T20:58:19.485777+0000","last_peered":"2023-09-23T20:58:19.485777+0000","last_clean":"2023-09-23T20:58:19.485777+0000","last_became_active":"2023-09-23T20:55:35.955630+0000","last_became_peered":"2023-09-23T20:55:35.955630+0000","last_unstale":"2023-09-23T20:58:19.485777+0000","last_undegraded":"2023-09-23T20:58:19.485777+0000","last_fullsized":"2023-09-23T20:58:19.485777+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:45:20.282929+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.c","version":"234'7369","reported_seq":7422,"reported_epoch":234,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T20:58:34.049291+0000","last_change":"2023-09-23T20:57:46.463005+0000","last_active":"2023-09-23T20:58:34.049291+0000","last_peered":"2023-09-23T20:58:34.049291+0000","last_clean":"2023-09-23T20:57:44.015119+0000","last_became_active":"2023-09-23T20:57:46.448702+0000","last_became_peered":"2023-09-23T20:57:46.448702+0000","last_unstale":"2023-09-23T20:58:34.049291+0000","last_undegraded":"2023-09-23T20:57:46.010874+0000","last_fullsized":"2023-09-23T20:57:45.978434+0000","mapping_epoch":223,"log_start":"0'0","ondisk_log_start":"0'0","created":199,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_clean_scrub_stamp":"2023-09-23T20:56:34.529103+0000","objects_scrubbed":0,"log_size":7369,"log_dups_size":0,"ondisk_log_size":7369,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:08:34.180651+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":356450304,"num_objects":5439,"num_object_clones":0,"num_object_copies":16317,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5473,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5439,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7369,"num_write_kb":409856,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,7],"avail_no_missing":["6","2","7"],"object_location_counts":[{"shards":"2,6,7","objects":4508},{"shards":"6,7","objects":-34}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.a","version":"79'161","reported_seq":477,"reported_epoch":233,"state":"active+clean","last_fresh":"2023-09-23T20:58:19.482585+0000","last_change":"2023-09-23T20:56:44.487778+0000","last_active":"2023-09-23T20:58:19.482585+0000","last_peered":"2023-09-23T20:58:19.482585+0000","last_clean":"2023-09-23T20:58:19.482585+0000","last_became_active":"2023-09-23T20:56:05.006740+0000","last_became_peered":"2023-09-23T20:56:05.006740+0000","last_unstale":"2023-09-23T20:58:19.482585+0000","last_undegraded":"2023-09-23T20:58:19.482585+0000","last_fullsized":"2023-09-23T20:58:19.482585+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:56:44.487678+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:56:44.487678+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:04:50.086749+0000","scrub_duration":0.0017408899999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":22,"reported_epoch":233,"state":"active+clean","last_fresh":"2023-09-23T20:58:19.485925+0000","last_change":"2023-09-23T20:57:44.982513+0000","last_active":"2023-09-23T20:58:19.485925+0000","last_peered":"2023-09-23T20:58:19.485925+0000","last_clean":"2023-09-23T20:58:19.485925+0000","last_became_active":"2023-09-23T20:57:44.982112+0000","last_became_peered":"2023-09-23T20:57:44.982112+0000","last_unstale":"2023-09-23T20:58:19.485925+0000","last_undegraded":"2023-09-23T20:58:19.485925+0000","last_fullsized":"2023-09-23T20:58:19.485925+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:09:05.630041+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":203,"reported_epoch":233,"state":"active+clean","last_fresh":"2023-09-23T20:58:19.477735+0000","last_change":"2023-09-23T20:54:13.940748+0000","last_active":"2023-09-23T20:58:19.477735+0000","last_peered":"2023-09-23T20:58:19.477735+0000","last_clean":"2023-09-23T20:58:19.477735+0000","last_became_active":"2023-09-23T20:54:13.940419+0000","last_became_peered":"2023-09-23T20:54:13.940419+0000","last_unstale":"2023-09-23T20:58:19.477735+0000","last_undegraded":"2023-09-23T20:58:19.477735+0000","last_fullsized":"2023-09-23T20:58:19.477735+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:15:01.923816+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.1","version":"234'7340","reported_seq":7396,"reported_epoch":234,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T20:58:32.133199+0000","last_change":"2023-09-23T20:57:46.483248+0000","last_active":"2023-09-23T20:58:32.133199+0000","last_peered":"2023-09-23T20:58:32.133199+0000","last_clean":"2023-09-23T20:57:44.047957+0000","last_became_active":"2023-09-23T20:57:46.475568+0000","last_became_peered":"2023-09-23T20:57:46.475568+0000","last_unstale":"2023-09-23T20:58:32.133199+0000","last_undegraded":"2023-09-23T20:57:46.083591+0000","last_fullsized":"2023-09-23T20:57:45.978436+0000","mapping_epoch":223,"log_start":"202'600","ondisk_log_start":"202'600","created":199,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_clean_scrub_stamp":"2023-09-23T20:56:34.529103+0000","objects_scrubbed":0,"log_size":6740,"log_dups_size":0,"ondisk_log_size":6740,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:22:45.067269+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":356909056,"num_objects":5446,"num_object_clones":0,"num_object_copies":16338,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5470,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5446,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7340,"num_write_kb":409152,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,6],"avail_no_missing":["4","1","6"],"object_location_counts":[{"shards":"1,4,6","objects":4523},{"shards":"4,6","objects":-24}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.7","version":"79'161","reported_seq":539,"reported_epoch":233,"state":"active+clean","last_fresh":"2023-09-23T20:58:19.469744+0000","last_change":"2023-09-23T20:57:48.243572+0000","last_active":"2023-09-23T20:58:19.469744+0000","last_peered":"2023-09-23T20:58:19.469744+0000","last_clean":"2023-09-23T20:58:19.469744+0000","last_became_active":"2023-09-23T20:57:48.243229+0000","last_became_peered":"2023-09-23T20:57:48.243229+0000","last_unstale":"2023-09-23T20:58:19.469744+0000","last_undegraded":"2023-09-23T20:58:19.469744+0000","last_fullsized":"2023-09-23T20:58:19.469744+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":3,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:25.916867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:25.916867+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:23:19.385831+0000","scrub_duration":0.001665323,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":209,"reported_epoch":233,"state":"active+clean","last_fresh":"2023-09-23T20:58:24.275631+0000","last_change":"2023-09-23T20:50:05.867130+0000","last_active":"2023-09-23T20:58:24.275631+0000","last_peered":"2023-09-23T20:58:24.275631+0000","last_clean":"2023-09-23T20:58:24.275631+0000","last_became_active":"2023-09-23T20:50:05.866758+0000","last_became_peered":"2023-09-23T20:50:05.866758+0000","last_unstale":"2023-09-23T20:58:24.275631+0000","last_undegraded":"2023-09-23T20:58:24.275631+0000","last_fullsized":"2023-09-23T20:58:24.275631+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:55:07.548209+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":107,"reported_epoch":233,"state":"active+clean","last_fresh":"2023-09-23T20:58:19.482749+0000","last_change":"2023-09-23T20:54:31.887604+0000","last_active":"2023-09-23T20:58:19.482749+0000","last_peered":"2023-09-23T20:58:19.482749+0000","last_clean":"2023-09-23T20:58:19.482749+0000","last_became_active":"2023-09-23T20:54:31.886752+0000","last_became_peered":"2023-09-23T20:54:31.886752+0000","last_unstale":"2023-09-23T20:58:19.482749+0000","last_undegraded":"2023-09-23T20:58:19.482749+0000","last_fullsized":"2023-09-23T20:58:19.482749+0000","mapping_epoch":136,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:13:58.872176+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":207,"reported_epoch":233,"state":"active+clean","last_fresh":"2023-09-23T20:58:24.275549+0000","last_change":"2023-09-23T20:54:31.899082+0000","last_active":"2023-09-23T20:58:24.275549+0000","last_peered":"2023-09-23T20:58:24.275549+0000","last_clean":"2023-09-23T20:58:24.275549+0000","last_became_active":"2023-09-23T20:54:31.898521+0000","last_became_peered":"2023-09-23T20:54:31.898521+0000","last_unstale":"2023-09-23T20:58:24.275549+0000","last_undegraded":"2023-09-23T20:58:24.275549+0000","last_fullsized":"2023-09-23T20:58:24.275549+0000","mapping_epoch":136,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:25:36.059711+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.7","version":"234'7391","reported_seq":7434,"reported_epoch":234,"state":"active+clean","last_fresh":"2023-09-23T20:58:34.024805+0000","last_change":"2023-09-23T20:56:35.541695+0000","last_active":"2023-09-23T20:58:34.024805+0000","last_peered":"2023-09-23T20:58:34.024805+0000","last_clean":"2023-09-23T20:58:34.024805+0000","last_became_active":"2023-09-23T20:56:35.541392+0000","last_became_peered":"2023-09-23T20:56:35.541392+0000","last_unstale":"2023-09-23T20:58:34.024805+0000","last_undegraded":"2023-09-23T20:58:34.024805+0000","last_fullsized":"2023-09-23T20:58:34.024805+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":199,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_clean_scrub_stamp":"2023-09-23T20:56:34.529103+0000","objects_scrubbed":0,"log_size":7391,"log_dups_size":0,"ondisk_log_size":7391,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:57:22.478253+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":355139584,"num_objects":5419,"num_object_clones":0,"num_object_copies":16257,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5419,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7391,"num_write_kb":409920,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.1","version":"79'161","reported_seq":497,"reported_epoch":233,"state":"active+clean","last_fresh":"2023-09-23T20:58:19.482247+0000","last_change":"2023-09-23T20:57:19.669346+0000","last_active":"2023-09-23T20:58:19.482247+0000","last_peered":"2023-09-23T20:58:19.482247+0000","last_clean":"2023-09-23T20:58:19.482247+0000","last_became_active":"2023-09-23T20:57:08.246046+0000","last_became_peered":"2023-09-23T20:57:08.246046+0000","last_unstale":"2023-09-23T20:58:19.482247+0000","last_undegraded":"2023-09-23T20:58:19.482247+0000","last_fullsized":"2023-09-23T20:58:19.482247+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:19.669259+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:19.669259+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:14:03.435531+0000","scrub_duration":0.0045759559999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":124,"reported_epoch":233,"state":"active+clean","last_fresh":"2023-09-23T20:58:19.482412+0000","last_change":"2023-09-23T20:55:35.959514+0000","last_active":"2023-09-23T20:58:19.482412+0000","last_peered":"2023-09-23T20:58:19.482412+0000","last_clean":"2023-09-23T20:58:19.482412+0000","last_became_active":"2023-09-23T20:55:35.959244+0000","last_became_peered":"2023-09-23T20:55:35.959244+0000","last_unstale":"2023-09-23T20:58:19.482412+0000","last_undegraded":"2023-09-23T20:58:19.482412+0000","last_fullsized":"2023-09-23T20:58:19.482412+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:47:32.665316+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":128,"reported_epoch":233,"state":"active+clean","last_fresh":"2023-09-23T20:58:19.485868+0000","last_change":"2023-09-23T20:55:35.949000+0000","last_active":"2023-09-23T20:58:19.485868+0000","last_peered":"2023-09-23T20:58:19.485868+0000","last_clean":"2023-09-23T20:58:19.485868+0000","last_became_active":"2023-09-23T20:55:35.948410+0000","last_became_peered":"2023-09-23T20:55:35.948410+0000","last_unstale":"2023-09-23T20:58:19.485868+0000","last_undegraded":"2023-09-23T20:58:19.485868+0000","last_fullsized":"2023-09-23T20:58:19.485868+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:26:50.477289+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":201,"reported_epoch":233,"state":"active+clean","last_fresh":"2023-09-23T20:58:24.275599+0000","last_change":"2023-09-23T20:51:10.829889+0000","last_active":"2023-09-23T20:58:24.275599+0000","last_peered":"2023-09-23T20:58:24.275599+0000","last_clean":"2023-09-23T20:58:24.275599+0000","last_became_active":"2023-09-23T20:51:10.829533+0000","last_became_peered":"2023-09-23T20:51:10.829533+0000","last_unstale":"2023-09-23T20:58:24.275599+0000","last_undegraded":"2023-09-23T20:58:24.275599+0000","last_fullsized":"2023-09-23T20:58:24.275599+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:43:04.603460+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.4","version":"235'7405","reported_seq":7450,"reported_epoch":235,"state":"active+clean","last_fresh":"2023-09-23T20:58:36.279442+0000","last_change":"2023-09-23T20:56:35.549909+0000","last_active":"2023-09-23T20:58:36.279442+0000","last_peered":"2023-09-23T20:58:36.279442+0000","last_clean":"2023-09-23T20:58:36.279442+0000","last_became_active":"2023-09-23T20:56:35.549618+0000","last_became_peered":"2023-09-23T20:56:35.549618+0000","last_unstale":"2023-09-23T20:58:36.279442+0000","last_undegraded":"2023-09-23T20:58:36.279442+0000","last_fullsized":"2023-09-23T20:58:36.279442+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":199,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_clean_scrub_stamp":"2023-09-23T20:56:34.529103+0000","objects_scrubbed":0,"log_size":7405,"log_dups_size":0,"ondisk_log_size":7405,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:24:25.918475+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":348454912,"num_objects":5317,"num_object_clones":0,"num_object_copies":15951,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5317,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7405,"num_write_kb":407104,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.2","version":"79'161","reported_seq":464,"reported_epoch":234,"state":"active+clean","last_fresh":"2023-09-23T20:58:26.126129+0000","last_change":"2023-09-23T20:57:24.802586+0000","last_active":"2023-09-23T20:58:26.126129+0000","last_peered":"2023-09-23T20:58:26.126129+0000","last_clean":"2023-09-23T20:58:26.126129+0000","last_became_active":"2023-09-23T20:57:08.236114+0000","last_became_peered":"2023-09-23T20:57:08.236114+0000","last_unstale":"2023-09-23T20:58:26.126129+0000","last_undegraded":"2023-09-23T20:58:26.126129+0000","last_fullsized":"2023-09-23T20:58:26.126129+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:24.802451+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:24.802451+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:06:22.309980+0000","scrub_duration":0.0090861250000000005,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,4],"acting":[0,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":219,"reported_epoch":233,"state":"active+clean","last_fresh":"2023-09-23T20:58:19.490662+0000","last_change":"2023-09-23T20:51:10.838054+0000","last_active":"2023-09-23T20:58:19.490662+0000","last_peered":"2023-09-23T20:58:19.490662+0000","last_clean":"2023-09-23T20:58:19.490662+0000","last_became_active":"2023-09-23T20:51:10.837750+0000","last_became_peered":"2023-09-23T20:51:10.837750+0000","last_unstale":"2023-09-23T20:58:19.490662+0000","last_undegraded":"2023-09-23T20:58:19.490662+0000","last_fullsized":"2023-09-23T20:58:19.490662+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:15:51.648085+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":201,"reported_epoch":233,"state":"active+clean","last_fresh":"2023-09-23T20:58:19.481048+0000","last_change":"2023-09-23T20:55:35.951146+0000","last_active":"2023-09-23T20:58:19.481048+0000","last_peered":"2023-09-23T20:58:19.481048+0000","last_clean":"2023-09-23T20:58:19.481048+0000","last_became_active":"2023-09-23T20:55:35.950874+0000","last_became_peered":"2023-09-23T20:55:35.950874+0000","last_unstale":"2023-09-23T20:58:19.481048+0000","last_undegraded":"2023-09-23T20:58:19.481048+0000","last_fullsized":"2023-09-23T20:58:19.481048+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:58:13.957677+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":221,"reported_epoch":233,"state":"active+clean","last_fresh":"2023-09-23T20:58:19.477629+0000","last_change":"2023-09-23T20:54:31.898870+0000","last_active":"2023-09-23T20:58:19.477629+0000","last_peered":"2023-09-23T20:58:19.477629+0000","last_clean":"2023-09-23T20:58:19.477629+0000","last_became_active":"2023-09-23T20:54:31.898467+0000","last_became_peered":"2023-09-23T20:54:31.898467+0000","last_unstale":"2023-09-23T20:58:19.477629+0000","last_undegraded":"2023-09-23T20:58:19.477629+0000","last_fullsized":"2023-09-23T20:58:19.477629+0000","mapping_epoch":136,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:56:22.483408+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.6","version":"235'7289","reported_seq":9455,"reported_epoch":235,"state":"active+recovering+undersized+remapped","last_fresh":"2023-09-23T20:58:36.329949+0000","last_change":"2023-09-23T20:57:46.535681+0000","last_active":"2023-09-23T20:58:36.329949+0000","last_peered":"2023-09-23T20:58:36.329949+0000","last_clean":"2023-09-23T20:57:44.031597+0000","last_became_active":"2023-09-23T20:57:46.443927+0000","last_became_peered":"2023-09-23T20:57:46.443927+0000","last_unstale":"2023-09-23T20:58:36.329949+0000","last_undegraded":"2023-09-23T20:58:36.329949+0000","last_fullsized":"2023-09-23T20:57:45.977227+0000","mapping_epoch":223,"log_start":"0'0","ondisk_log_start":"0'0","created":199,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_clean_scrub_stamp":"2023-09-23T20:56:34.529103+0000","objects_scrubbed":0,"log_size":7289,"log_dups_size":0,"ondisk_log_size":7289,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:37:23.911131+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":347275264,"num_objects":5299,"num_object_clones":0,"num_object_copies":15897,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":4342,"num_objects_unfound":0,"num_objects_dirty":5299,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7289,"num_write_kb":402816,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1056,"num_bytes_recovered":69206016,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7],"avail_no_missing":["0","5","7"],"object_location_counts":[{"shards":"0,5,7","objects":4342},{"shards":"0,7","objects":957}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":107,"reported_epoch":233,"state":"active+clean","last_fresh":"2023-09-23T20:58:19.482564+0000","last_change":"2023-09-23T20:54:31.883504+0000","last_active":"2023-09-23T20:58:19.482564+0000","last_peered":"2023-09-23T20:58:19.482564+0000","last_clean":"2023-09-23T20:58:19.482564+0000","last_became_active":"2023-09-23T20:54:31.883172+0000","last_became_peered":"2023-09-23T20:54:31.883172+0000","last_unstale":"2023-09-23T20:58:19.482564+0000","last_undegraded":"2023-09-23T20:58:19.482564+0000","last_fullsized":"2023-09-23T20:58:19.482564+0000","mapping_epoch":136,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:45:40.289120+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"79'161","reported_seq":464,"reported_epoch":234,"state":"active+clean","last_fresh":"2023-09-23T20:58:26.125626+0000","last_change":"2023-09-23T20:57:20.738417+0000","last_active":"2023-09-23T20:58:26.125626+0000","last_peered":"2023-09-23T20:58:26.125626+0000","last_clean":"2023-09-23T20:58:26.125626+0000","last_became_active":"2023-09-23T20:57:08.236842+0000","last_became_peered":"2023-09-23T20:57:08.236842+0000","last_unstale":"2023-09-23T20:58:26.125626+0000","last_undegraded":"2023-09-23T20:58:26.125626+0000","last_fullsized":"2023-09-23T20:58:26.125626+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":0,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:20.738273+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:20.738273+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:18:28.327990+0000","scrub_duration":0.0088336339999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":211,"reported_epoch":233,"state":"active+clean","last_fresh":"2023-09-23T20:58:19.489034+0000","last_change":"2023-09-23T20:55:35.950777+0000","last_active":"2023-09-23T20:58:19.489034+0000","last_peered":"2023-09-23T20:58:19.489034+0000","last_clean":"2023-09-23T20:58:19.489034+0000","last_became_active":"2023-09-23T20:55:35.950427+0000","last_became_peered":"2023-09-23T20:55:35.950427+0000","last_unstale":"2023-09-23T20:58:19.489034+0000","last_undegraded":"2023-09-23T20:58:19.489034+0000","last_fullsized":"2023-09-23T20:58:19.489034+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:20:10.371751+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":178,"reported_epoch":233,"state":"active+clean","last_fresh":"2023-09-23T20:58:19.469926+0000","last_change":"2023-09-23T20:55:35.978140+0000","last_active":"2023-09-23T20:58:19.469926+0000","last_peered":"2023-09-23T20:58:19.469926+0000","last_clean":"2023-09-23T20:58:19.469926+0000","last_became_active":"2023-09-23T20:55:35.977831+0000","last_became_peered":"2023-09-23T20:55:35.977831+0000","last_unstale":"2023-09-23T20:58:19.469926+0000","last_undegraded":"2023-09-23T20:58:19.469926+0000","last_fullsized":"2023-09-23T20:58:19.469926+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:25:47.678023+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.5","version":"234'7196","reported_seq":7247,"reported_epoch":234,"state":"active+clean","last_fresh":"2023-09-23T20:58:31.342635+0000","last_change":"2023-09-23T20:56:45.990110+0000","last_active":"2023-09-23T20:58:31.342635+0000","last_peered":"2023-09-23T20:58:31.342635+0000","last_clean":"2023-09-23T20:58:31.342635+0000","last_became_active":"2023-09-23T20:56:45.988394+0000","last_became_peered":"2023-09-23T20:56:45.988394+0000","last_unstale":"2023-09-23T20:58:31.342635+0000","last_undegraded":"2023-09-23T20:58:31.342635+0000","last_fullsized":"2023-09-23T20:58:31.342635+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":199,"last_epoch_clean":205,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_clean_scrub_stamp":"2023-09-23T20:56:34.529103+0000","objects_scrubbed":0,"log_size":7196,"log_dups_size":0,"ondisk_log_size":7196,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:22:03.814210+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":359268352,"num_objects":5482,"num_object_clones":0,"num_object_copies":16446,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5482,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7196,"num_write_kb":405696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,0],"acting":[2,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"79'161","reported_seq":500,"reported_epoch":233,"state":"active+clean","last_fresh":"2023-09-23T20:58:19.477998+0000","last_change":"2023-09-23T20:57:20.378531+0000","last_active":"2023-09-23T20:58:19.477998+0000","last_peered":"2023-09-23T20:58:19.477998+0000","last_clean":"2023-09-23T20:58:19.477998+0000","last_became_active":"2023-09-23T20:57:08.238019+0000","last_became_peered":"2023-09-23T20:57:08.238019+0000","last_unstale":"2023-09-23T20:58:19.477998+0000","last_undegraded":"2023-09-23T20:58:19.477998+0000","last_fullsized":"2023-09-23T20:58:19.477998+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":3,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:20.377714+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:20.377714+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:00:08.305602+0000","scrub_duration":0.003015228,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":102,"reported_epoch":233,"state":"active+clean","last_fresh":"2023-09-23T20:58:19.481174+0000","last_change":"2023-09-23T20:56:45.991884+0000","last_active":"2023-09-23T20:58:19.481174+0000","last_peered":"2023-09-23T20:58:19.481174+0000","last_clean":"2023-09-23T20:58:19.481174+0000","last_became_active":"2023-09-23T20:56:45.991568+0000","last_became_peered":"2023-09-23T20:56:45.991568+0000","last_unstale":"2023-09-23T20:58:19.481174+0000","last_undegraded":"2023-09-23T20:58:19.481174+0000","last_fullsized":"2023-09-23T20:58:19.481174+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":205,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:21:20.775511+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,0],"acting":[1,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":228,"reported_epoch":234,"state":"active+clean","last_fresh":"2023-09-23T20:58:26.125989+0000","last_change":"2023-09-23T20:57:44.979950+0000","last_active":"2023-09-23T20:58:26.125989+0000","last_peered":"2023-09-23T20:58:26.125989+0000","last_clean":"2023-09-23T20:58:26.125989+0000","last_became_active":"2023-09-23T20:57:44.979346+0000","last_became_peered":"2023-09-23T20:57:44.979346+0000","last_unstale":"2023-09-23T20:58:26.125989+0000","last_undegraded":"2023-09-23T20:58:26.125989+0000","last_fullsized":"2023-09-23T20:58:26.125989+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:10:24.372800+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":207,"reported_epoch":234,"state":"active+clean","last_fresh":"2023-09-23T20:58:26.130246+0000","last_change":"2023-09-23T20:50:37.982235+0000","last_active":"2023-09-23T20:58:26.130246+0000","last_peered":"2023-09-23T20:58:26.130246+0000","last_clean":"2023-09-23T20:58:26.130246+0000","last_became_active":"2023-09-23T20:50:37.981913+0000","last_became_peered":"2023-09-23T20:50:37.981913+0000","last_unstale":"2023-09-23T20:58:26.130246+0000","last_undegraded":"2023-09-23T20:58:26.130246+0000","last_fullsized":"2023-09-23T20:58:26.130246+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:59:57.122584+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.0","version":"234'7222","reported_seq":7265,"reported_epoch":234,"state":"active+clean","last_fresh":"2023-09-23T20:58:32.090902+0000","last_change":"2023-09-23T20:56:35.550635+0000","last_active":"2023-09-23T20:58:32.090902+0000","last_peered":"2023-09-23T20:58:32.090902+0000","last_clean":"2023-09-23T20:58:32.090902+0000","last_became_active":"2023-09-23T20:56:35.550376+0000","last_became_peered":"2023-09-23T20:56:35.550376+0000","last_unstale":"2023-09-23T20:58:32.090902+0000","last_undegraded":"2023-09-23T20:58:32.090902+0000","last_fullsized":"2023-09-23T20:58:32.090902+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":199,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_clean_scrub_stamp":"2023-09-23T20:56:34.529103+0000","objects_scrubbed":0,"log_size":7222,"log_dups_size":0,"ondisk_log_size":7222,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:40:55.020049+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":357826560,"num_objects":5460,"num_object_clones":0,"num_object_copies":16380,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5460,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7222,"num_write_kb":405824,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.6","version":"79'161","reported_seq":516,"reported_epoch":233,"state":"active+clean","last_fresh":"2023-09-23T20:58:19.469398+0000","last_change":"2023-09-23T20:57:26.907505+0000","last_active":"2023-09-23T20:58:19.469398+0000","last_peered":"2023-09-23T20:58:19.469398+0000","last_clean":"2023-09-23T20:58:19.469398+0000","last_became_active":"2023-09-23T20:57:08.240321+0000","last_became_peered":"2023-09-23T20:57:08.240321+0000","last_unstale":"2023-09-23T20:58:19.469398+0000","last_undegraded":"2023-09-23T20:58:19.469398+0000","last_fullsized":"2023-09-23T20:58:19.469398+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":3,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:26.907408+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:26.907408+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:34:17.392928+0000","scrub_duration":0.010337038,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":150,"reported_epoch":234,"state":"active+clean","last_fresh":"2023-09-23T20:58:26.130877+0000","last_change":"2023-09-23T20:56:45.988566+0000","last_active":"2023-09-23T20:58:26.130877+0000","last_peered":"2023-09-23T20:58:26.130877+0000","last_clean":"2023-09-23T20:58:26.130877+0000","last_became_active":"2023-09-23T20:56:45.988249+0000","last_became_peered":"2023-09-23T20:56:45.988249+0000","last_unstale":"2023-09-23T20:58:26.130877+0000","last_undegraded":"2023-09-23T20:58:26.130877+0000","last_fullsized":"2023-09-23T20:58:26.130877+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":205,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:51:36.644764+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,4],"acting":[0,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":204,"reported_epoch":233,"state":"active+clean","last_fresh":"2023-09-23T20:58:24.275617+0000","last_change":"2023-09-23T20:54:31.898965+0000","last_active":"2023-09-23T20:58:24.275617+0000","last_peered":"2023-09-23T20:58:24.275617+0000","last_clean":"2023-09-23T20:58:24.275617+0000","last_became_active":"2023-09-23T20:54:31.898663+0000","last_became_peered":"2023-09-23T20:54:31.898663+0000","last_unstale":"2023-09-23T20:58:24.275617+0000","last_undegraded":"2023-09-23T20:58:24.275617+0000","last_fullsized":"2023-09-23T20:58:24.275617+0000","mapping_epoch":136,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:53:05.058087+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":195,"reported_epoch":234,"state":"active+clean","last_fresh":"2023-09-23T20:58:26.126650+0000","last_change":"2023-09-23T20:54:31.884717+0000","last_active":"2023-09-23T20:58:26.126650+0000","last_peered":"2023-09-23T20:58:26.126650+0000","last_clean":"2023-09-23T20:58:26.126650+0000","last_became_active":"2023-09-23T20:54:31.884345+0000","last_became_peered":"2023-09-23T20:54:31.884345+0000","last_unstale":"2023-09-23T20:58:26.126650+0000","last_undegraded":"2023-09-23T20:58:26.126650+0000","last_fullsized":"2023-09-23T20:58:26.126650+0000","mapping_epoch":136,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:02:26.393868+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.3","version":"234'7223","reported_seq":7266,"reported_epoch":234,"state":"active+clean","last_fresh":"2023-09-23T20:58:34.440685+0000","last_change":"2023-09-23T20:56:35.549655+0000","last_active":"2023-09-23T20:58:34.440685+0000","last_peered":"2023-09-23T20:58:34.440685+0000","last_clean":"2023-09-23T20:58:34.440685+0000","last_became_active":"2023-09-23T20:56:35.549357+0000","last_became_peered":"2023-09-23T20:56:35.549357+0000","last_unstale":"2023-09-23T20:58:34.440685+0000","last_undegraded":"2023-09-23T20:58:34.440685+0000","last_fullsized":"2023-09-23T20:58:34.440685+0000","mapping_epoch":199,"log_start":"205'1100","ondisk_log_start":"205'1100","created":199,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_clean_scrub_stamp":"2023-09-23T20:56:34.529103+0000","objects_scrubbed":0,"log_size":6123,"log_dups_size":0,"ondisk_log_size":6123,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:37:58.752518+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":347275264,"num_objects":5299,"num_object_clones":0,"num_object_copies":15897,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5299,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7223,"num_write_kb":400704,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.5","version":"79'161","reported_seq":497,"reported_epoch":233,"state":"active+clean","last_fresh":"2023-09-23T20:58:19.485481+0000","last_change":"2023-09-23T20:57:30.029479+0000","last_active":"2023-09-23T20:58:19.485481+0000","last_peered":"2023-09-23T20:58:19.485481+0000","last_clean":"2023-09-23T20:58:19.485481+0000","last_became_active":"2023-09-23T20:57:08.249909+0000","last_became_peered":"2023-09-23T20:57:08.249909+0000","last_unstale":"2023-09-23T20:58:19.485481+0000","last_undegraded":"2023-09-23T20:58:19.485481+0000","last_fullsized":"2023-09-23T20:58:19.485481+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":3,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:30.029396+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:30.029396+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:16:46.083145+0000","scrub_duration":0.012914682,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":197,"reported_epoch":233,"state":"active+clean","last_fresh":"2023-09-23T20:58:19.493504+0000","last_change":"2023-09-23T20:51:10.836910+0000","last_active":"2023-09-23T20:58:19.493504+0000","last_peered":"2023-09-23T20:58:19.493504+0000","last_clean":"2023-09-23T20:58:19.493504+0000","last_became_active":"2023-09-23T20:51:10.836557+0000","last_became_peered":"2023-09-23T20:51:10.836557+0000","last_unstale":"2023-09-23T20:58:19.493504+0000","last_undegraded":"2023-09-23T20:58:19.493504+0000","last_fullsized":"2023-09-23T20:58:19.493504+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:51:54.019969+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":203,"reported_epoch":233,"state":"active+clean","last_fresh":"2023-09-23T20:58:19.477324+0000","last_change":"2023-09-23T20:55:35.953444+0000","last_active":"2023-09-23T20:58:19.477324+0000","last_peered":"2023-09-23T20:58:19.477324+0000","last_clean":"2023-09-23T20:58:19.477324+0000","last_became_active":"2023-09-23T20:55:35.953151+0000","last_became_peered":"2023-09-23T20:55:35.953151+0000","last_unstale":"2023-09-23T20:58:19.477324+0000","last_undegraded":"2023-09-23T20:58:19.477324+0000","last_fullsized":"2023-09-23T20:58:19.477324+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:39:12.404391+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":22,"reported_epoch":233,"state":"active+clean","last_fresh":"2023-09-23T20:58:19.486011+0000","last_change":"2023-09-23T20:57:44.982509+0000","last_active":"2023-09-23T20:58:19.486011+0000","last_peered":"2023-09-23T20:58:19.486011+0000","last_clean":"2023-09-23T20:58:19.486011+0000","last_became_active":"2023-09-23T20:57:44.982175+0000","last_became_peered":"2023-09-23T20:57:44.982175+0000","last_unstale":"2023-09-23T20:58:19.486011+0000","last_undegraded":"2023-09-23T20:58:19.486011+0000","last_fullsized":"2023-09-23T20:58:19.486011+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:37:49.215488+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.2","version":"234'7214","reported_seq":7258,"reported_epoch":234,"state":"active+clean","last_fresh":"2023-09-23T20:58:34.462422+0000","last_change":"2023-09-23T20:56:35.543020+0000","last_active":"2023-09-23T20:58:34.462422+0000","last_peered":"2023-09-23T20:58:34.462422+0000","last_clean":"2023-09-23T20:58:34.462422+0000","last_became_active":"2023-09-23T20:56:35.542719+0000","last_became_peered":"2023-09-23T20:56:35.542719+0000","last_unstale":"2023-09-23T20:58:34.462422+0000","last_undegraded":"2023-09-23T20:58:34.462422+0000","last_fullsized":"2023-09-23T20:58:34.462422+0000","mapping_epoch":199,"log_start":"205'1000","ondisk_log_start":"205'1000","created":199,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_clean_scrub_stamp":"2023-09-23T20:56:34.529103+0000","objects_scrubbed":0,"log_size":6214,"log_dups_size":0,"ondisk_log_size":6214,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:52:21.088628+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":347209728,"num_objects":5298,"num_object_clones":0,"num_object_copies":15894,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5298,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7214,"num_write_kb":400384,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.4","version":"79'161","reported_seq":512,"reported_epoch":233,"state":"active+clean","last_fresh":"2023-09-23T20:58:19.494422+0000","last_change":"2023-09-23T20:57:20.012904+0000","last_active":"2023-09-23T20:58:19.494422+0000","last_peered":"2023-09-23T20:58:19.494422+0000","last_clean":"2023-09-23T20:58:19.494422+0000","last_became_active":"2023-09-23T20:57:08.237996+0000","last_became_peered":"2023-09-23T20:57:08.237996+0000","last_unstale":"2023-09-23T20:58:19.494422+0000","last_undegraded":"2023-09-23T20:58:19.494422+0000","last_fullsized":"2023-09-23T20:58:19.494422+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":3,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:20.012764+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:20.012764+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:51:28.796946+0000","scrub_duration":0.0045251919999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":183,"reported_epoch":233,"state":"active+clean","last_fresh":"2023-09-23T20:58:19.469595+0000","last_change":"2023-09-23T20:55:35.949541+0000","last_active":"2023-09-23T20:58:19.469595+0000","last_peered":"2023-09-23T20:58:19.469595+0000","last_clean":"2023-09-23T20:58:19.469595+0000","last_became_active":"2023-09-23T20:55:35.944635+0000","last_became_peered":"2023-09-23T20:55:35.944635+0000","last_unstale":"2023-09-23T20:58:19.469595+0000","last_undegraded":"2023-09-23T20:58:19.469595+0000","last_fullsized":"2023-09-23T20:58:19.469595+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:30:36.033585+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"70'1","reported_seq":223,"reported_epoch":234,"state":"active+clean","last_fresh":"2023-09-23T20:58:26.130358+0000","last_change":"2023-09-23T20:54:14.009723+0000","last_active":"2023-09-23T20:58:26.130358+0000","last_peered":"2023-09-23T20:58:26.130358+0000","last_clean":"2023-09-23T20:58:26.130358+0000","last_became_active":"2023-09-23T20:54:13.941771+0000","last_became_peered":"2023-09-23T20:54:13.941771+0000","last_unstale":"2023-09-23T20:58:26.130358+0000","last_undegraded":"2023-09-23T20:58:26.130358+0000","last_fullsized":"2023-09-23T20:58:26.130358+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:48:25.588116+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":138,"reported_epoch":233,"state":"active+clean","last_fresh":"2023-09-23T20:58:19.485332+0000","last_change":"2023-09-23T20:55:35.949780+0000","last_active":"2023-09-23T20:58:19.485332+0000","last_peered":"2023-09-23T20:58:19.485332+0000","last_clean":"2023-09-23T20:58:19.485332+0000","last_became_active":"2023-09-23T20:55:35.949497+0000","last_became_peered":"2023-09-23T20:55:35.949497+0000","last_unstale":"2023-09-23T20:58:19.485332+0000","last_undegraded":"2023-09-23T20:58:19.485332+0000","last_fullsized":"2023-09-23T20:58:19.485332+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:09:37.420882+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":197,"reported_epoch":234,"state":"active+clean","last_fresh":"2023-09-23T20:58:26.126584+0000","last_change":"2023-09-23T20:54:13.929196+0000","last_active":"2023-09-23T20:58:26.126584+0000","last_peered":"2023-09-23T20:58:26.126584+0000","last_clean":"2023-09-23T20:58:26.126584+0000","last_became_active":"2023-09-23T20:54:13.928831+0000","last_became_peered":"2023-09-23T20:54:13.928831+0000","last_unstale":"2023-09-23T20:58:26.126584+0000","last_undegraded":"2023-09-23T20:58:26.126584+0000","last_fullsized":"2023-09-23T20:58:26.126584+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:28:44.316851+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":202,"reported_epoch":233,"state":"active+clean","last_fresh":"2023-09-23T20:58:19.490867+0000","last_change":"2023-09-23T20:52:12.855430+0000","last_active":"2023-09-23T20:58:19.490867+0000","last_peered":"2023-09-23T20:58:19.490867+0000","last_clean":"2023-09-23T20:58:19.490867+0000","last_became_active":"2023-09-23T20:52:12.855115+0000","last_became_peered":"2023-09-23T20:52:12.855115+0000","last_unstale":"2023-09-23T20:58:19.490867+0000","last_undegraded":"2023-09-23T20:58:19.490867+0000","last_fullsized":"2023-09-23T20:58:19.490867+0000","mapping_epoch":100,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":101,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:33:23.802214+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":111,"reported_epoch":233,"state":"active+clean","last_fresh":"2023-09-23T20:58:19.482712+0000","last_change":"2023-09-23T20:54:31.902946+0000","last_active":"2023-09-23T20:58:19.482712+0000","last_peered":"2023-09-23T20:58:19.482712+0000","last_clean":"2023-09-23T20:58:19.482712+0000","last_became_active":"2023-09-23T20:54:31.889484+0000","last_became_peered":"2023-09-23T20:54:31.889484+0000","last_unstale":"2023-09-23T20:58:19.482712+0000","last_undegraded":"2023-09-23T20:58:19.482712+0000","last_fullsized":"2023-09-23T20:58:19.482712+0000","mapping_epoch":136,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:44:44.278398+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":111,"reported_epoch":233,"state":"active+clean","last_fresh":"2023-09-23T20:58:19.485961+0000","last_change":"2023-09-23T20:54:13.950351+0000","last_active":"2023-09-23T20:58:19.485961+0000","last_peered":"2023-09-23T20:58:19.485961+0000","last_clean":"2023-09-23T20:58:19.485961+0000","last_became_active":"2023-09-23T20:54:13.949987+0000","last_became_peered":"2023-09-23T20:54:13.949987+0000","last_unstale":"2023-09-23T20:58:19.485961+0000","last_undegraded":"2023-09-23T20:58:19.485961+0000","last_fullsized":"2023-09-23T20:58:19.485961+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:46:17.765134+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":142,"reported_epoch":233,"state":"active+clean","last_fresh":"2023-09-23T20:58:19.482829+0000","last_change":"2023-09-23T20:56:18.141728+0000","last_active":"2023-09-23T20:58:19.482829+0000","last_peered":"2023-09-23T20:58:19.482829+0000","last_clean":"2023-09-23T20:58:19.482829+0000","last_became_active":"2023-09-23T20:56:18.137979+0000","last_became_peered":"2023-09-23T20:56:18.137979+0000","last_unstale":"2023-09-23T20:58:19.482829+0000","last_undegraded":"2023-09-23T20:58:19.482829+0000","last_fullsized":"2023-09-23T20:58:19.482829+0000","mapping_epoch":192,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":193,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:58:05.658443+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":111,"reported_epoch":233,"state":"active+clean","last_fresh":"2023-09-23T20:58:19.485262+0000","last_change":"2023-09-23T20:54:13.935307+0000","last_active":"2023-09-23T20:58:19.485262+0000","last_peered":"2023-09-23T20:58:19.485262+0000","last_clean":"2023-09-23T20:58:19.485262+0000","last_became_active":"2023-09-23T20:54:13.935000+0000","last_became_peered":"2023-09-23T20:54:13.935000+0000","last_unstale":"2023-09-23T20:58:19.485262+0000","last_undegraded":"2023-09-23T20:58:19.485262+0000","last_fullsized":"2023-09-23T20:58:19.485262+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:47:40.974082+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":220,"reported_epoch":233,"state":"active+clean","last_fresh":"2023-09-23T20:58:19.493041+0000","last_change":"2023-09-23T20:55:35.954870+0000","last_active":"2023-09-23T20:58:19.493041+0000","last_peered":"2023-09-23T20:58:19.493041+0000","last_clean":"2023-09-23T20:58:19.493041+0000","last_became_active":"2023-09-23T20:55:35.954568+0000","last_became_peered":"2023-09-23T20:55:35.954568+0000","last_unstale":"2023-09-23T20:58:19.493041+0000","last_undegraded":"2023-09-23T20:58:19.493041+0000","last_fullsized":"2023-09-23T20:58:19.493041+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:06:22.423566+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":220,"reported_epoch":233,"state":"active+clean","last_fresh":"2023-09-23T20:58:24.408905+0000","last_change":"2023-09-23T20:51:10.838334+0000","last_active":"2023-09-23T20:58:24.408905+0000","last_peered":"2023-09-23T20:58:24.408905+0000","last_clean":"2023-09-23T20:58:24.408905+0000","last_became_active":"2023-09-23T20:51:10.838026+0000","last_became_peered":"2023-09-23T20:51:10.838026+0000","last_unstale":"2023-09-23T20:58:24.408905+0000","last_undegraded":"2023-09-23T20:58:24.408905+0000","last_fullsized":"2023-09-23T20:58:24.408905+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:04:37.009457+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":225,"reported_epoch":234,"state":"active+clean","last_fresh":"2023-09-23T20:58:26.136555+0000","last_change":"2023-09-23T20:54:31.903139+0000","last_active":"2023-09-23T20:58:26.136555+0000","last_peered":"2023-09-23T20:58:26.136555+0000","last_clean":"2023-09-23T20:58:26.136555+0000","last_became_active":"2023-09-23T20:54:31.884520+0000","last_became_peered":"2023-09-23T20:54:31.884520+0000","last_unstale":"2023-09-23T20:58:26.136555+0000","last_undegraded":"2023-09-23T20:58:26.136555+0000","last_fullsized":"2023-09-23T20:58:26.136555+0000","mapping_epoch":136,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:39:34.269705+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1108,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":128,"reported_epoch":233,"state":"active+clean","last_fresh":"2023-09-23T20:58:19.485043+0000","last_change":"2023-09-23T20:55:35.951572+0000","last_active":"2023-09-23T20:58:19.485043+0000","last_peered":"2023-09-23T20:58:19.485043+0000","last_clean":"2023-09-23T20:58:19.485043+0000","last_became_active":"2023-09-23T20:55:35.951294+0000","last_became_peered":"2023-09-23T20:55:35.951294+0000","last_unstale":"2023-09-23T20:58:19.485043+0000","last_undegraded":"2023-09-23T20:58:19.485043+0000","last_fullsized":"2023-09-23T20:58:19.485043+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:36:32.447210+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":7,"num_pg":16,"stat_sum":{"num_bytes":5660803096,"num_objects":86378,"num_object_clones":0,"num_object_copies":259134,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":10943,"num_objects_misplaced":4342,"num_objects_unfound":0,"num_objects_dirty":86378,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":117142,"num_write_kb":6512577,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1057,"num_bytes_recovered":69271552,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":17217564672,"data_stored":17217552456,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":112842,"ondisk_log_size":112842,"up":48,"acting":45,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":22,"num_bytes_recovered":5990,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":21,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3381,"ondisk_log_size":3381,"up":63,"acting":63,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201497,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1933308,"kb_used_data":1769332,"kb_used_omap":0,"kb_used_meta":163968,"kb_avail":91811844,"statfs":{"total":95995035648,"available":94015328256,"internally_reserved":0,"allocated":1811795968,"data_stored":1810968378,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":167903232},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:57:44 2023","interfaces":[{"interface":"back","average":{"1min":3.3239999999999998,"5min":1.613,"15min":0.90200000000000002},"min":{"1min":0.23100000000000001,"5min":0.21099999999999999,"15min":0.21099999999999999},"max":{"1min":43.994999999999997,"5min":43.994999999999997,"15min":43.994999999999997},"last":3.282},{"interface":"front","average":{"1min":3.448,"5min":1.847,"15min":1},"min":{"1min":0.35299999999999998,"5min":0.218,"15min":0.218},"max":{"1min":44.206000000000003,"5min":44.206000000000003,"15min":44.206000000000003},"last":3.2599999999999998}]},{"osd":1,"last update":"Sat Sep 23 20:58:18 2023","interfaces":[{"interface":"back","average":{"1min":1.895,"5min":1.9330000000000001,"15min":1.1439999999999999},"min":{"1min":0.29999999999999999,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":10.622,"5min":10.622,"15min":22.591999999999999},"last":2.6280000000000001},{"interface":"front","average":{"1min":1.8320000000000001,"5min":1.5720000000000001,"15min":0.98499999999999999},"min":{"1min":0.25800000000000001,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":7.5629999999999997,"5min":10.097,"15min":21.437000000000001},"last":0.84199999999999997}]},{"osd":2,"last update":"Sat Sep 23 20:57:50 2023","interfaces":[{"interface":"back","average":{"1min":1.3919999999999999,"5min":1.643,"15min":0.93799999999999994},"min":{"1min":0.26800000000000002,"5min":0.26200000000000001,"15min":0.24399999999999999},"max":{"1min":5.3929999999999998,"5min":9.4290000000000003,"15min":9.4290000000000003},"last":0.39700000000000002},{"interface":"front","average":{"1min":3.3420000000000001,"5min":1.978,"15min":1.071},"min":{"1min":0.29899999999999999,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":45.25,"5min":45.25,"15min":45.25},"last":0.27800000000000002}]},{"osd":3,"last update":"Sat Sep 23 20:58:26 2023","interfaces":[{"interface":"back","average":{"1min":3.355,"5min":3.226,"15min":1.587},"min":{"1min":0.32100000000000001,"5min":0.314,"15min":0.314},"max":{"1min":9.9939999999999998,"5min":36.408000000000001,"15min":36.408000000000001},"last":6.1360000000000001},{"interface":"front","average":{"1min":2.5649999999999999,"5min":2.7160000000000002,"15min":1.405},"min":{"1min":0.32000000000000001,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":9.4529999999999994,"5min":35.593000000000004,"15min":35.593000000000004},"last":6.3029999999999999}]},{"osd":4,"last update":"Sat Sep 23 20:58:18 2023","interfaces":[{"interface":"back","average":{"1min":0.69999999999999996,"5min":0.82699999999999996,"15min":0.60899999999999999},"min":{"1min":0.28299999999999997,"5min":0.23799999999999999,"15min":0.23699999999999999},"max":{"1min":4.5410000000000004,"5min":5.8920000000000003,"15min":5.8920000000000003},"last":0.35499999999999998},{"interface":"front","average":{"1min":0.70399999999999996,"5min":0.76100000000000001,"15min":0.61399999999999999},"min":{"1min":0.27100000000000002,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":4.4080000000000004,"5min":5.3200000000000003,"15min":5.3200000000000003},"last":2.952}]},{"osd":5,"last update":"Sat Sep 23 20:58:18 2023","interfaces":[{"interface":"back","average":{"1min":0.69299999999999995,"5min":0.82799999999999996,"15min":0.63300000000000001},"min":{"1min":0.28499999999999998,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":2.5110000000000001,"5min":7.0389999999999997,"15min":7.0389999999999997},"last":0.442},{"interface":"front","average":{"1min":0.72199999999999998,"5min":0.70699999999999996,"15min":0.61199999999999999},"min":{"1min":0.317,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":4.5999999999999996,"5min":4.5999999999999996,"15min":4.5999999999999996},"last":0.38600000000000001}]},{"osd":6,"last update":"Sat Sep 23 20:57:38 2023","interfaces":[{"interface":"back","average":{"1min":0.72099999999999997,"5min":0.67700000000000005,"15min":0.67000000000000004},"min":{"1min":0.34799999999999998,"5min":0.31,"15min":0.31},"max":{"1min":1.9039999999999999,"5min":3.1760000000000002,"15min":3.1760000000000002},"last":1.831},{"interface":"front","average":{"1min":0.753,"5min":0.79400000000000004,"15min":0.80100000000000005},"min":{"1min":0.32300000000000001,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":1.96,"5min":3.129,"15min":3.129},"last":1.929}]}]},{"osd":6,"up_from":164,"seq":704374636581,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1830896,"kb_used_data":1736936,"kb_used_omap":10,"kb_used_meta":93941,"kb_avail":91914256,"statfs":{"total":95995035648,"available":94120198144,"internally_reserved":0,"allocated":1778622464,"data_stored":1777806775,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":10328,"internal_metadata":96196520},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:57:40 2023","interfaces":[{"interface":"back","average":{"1min":1.8799999999999999,"5min":1.0680000000000001,"15min":0.93200000000000005},"min":{"1min":0.20999999999999999,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":8.7319999999999993,"5min":8.7319999999999993,"15min":8.7319999999999993},"last":0.49099999999999999},{"interface":"front","average":{"1min":1.393,"5min":1.0649999999999999,"15min":1.01},"min":{"1min":0.29299999999999998,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":6.7110000000000003,"5min":6.7110000000000003,"15min":6.7110000000000003},"last":0.25}]},{"osd":1,"last update":"Sat Sep 23 20:57:40 2023","interfaces":[{"interface":"back","average":{"1min":1.575,"5min":0.94599999999999995,"15min":0.84099999999999997},"min":{"1min":0.34599999999999997,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":6.6029999999999998,"5min":6.6029999999999998,"15min":6.6029999999999998},"last":0.753},{"interface":"front","average":{"1min":1.04,"5min":0.90200000000000002,"15min":0.879},"min":{"1min":0.255,"5min":0.247,"15min":0.247},"max":{"1min":2.7389999999999999,"5min":3.5870000000000002,"15min":3.5870000000000002},"last":1.3959999999999999}]},{"osd":2,"last update":"Sat Sep 23 20:57:40 2023","interfaces":[{"interface":"back","average":{"1min":2.6850000000000001,"5min":1.8540000000000001,"15min":1.716},"min":{"1min":0.371,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":13.750999999999999,"5min":13.750999999999999,"15min":13.750999999999999},"last":0.52100000000000002},{"interface":"front","average":{"1min":1.9079999999999999,"5min":1.2669999999999999,"15min":1.1599999999999999},"min":{"1min":0.39200000000000002,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":7.4279999999999999,"5min":7.4279999999999999,"15min":7.4279999999999999},"last":0.41599999999999998}]},{"osd":3,"last update":"Sat Sep 23 20:57:40 2023","interfaces":[{"interface":"back","average":{"1min":1.2070000000000001,"5min":1.135,"15min":1.123},"min":{"1min":0.33300000000000002,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":3.093,"5min":5.8739999999999997,"15min":5.8739999999999997},"last":0.86199999999999999},{"interface":"front","average":{"1min":1.7090000000000001,"5min":1.6240000000000001,"15min":1.6100000000000001},"min":{"1min":0.311,"5min":0.311,"15min":0.311},"max":{"1min":5.46,"5min":7.234,"15min":7.234},"last":1.1120000000000001}]},{"osd":4,"last update":"Sat Sep 23 20:57:40 2023","interfaces":[{"interface":"back","average":{"1min":0.65000000000000002,"5min":0.97099999999999997,"15min":1.0249999999999999},"min":{"1min":0.28299999999999997,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":1.9039999999999999,"5min":9.0079999999999991,"15min":9.0079999999999991},"last":0.33000000000000002},{"interface":"front","average":{"1min":0.45700000000000002,"5min":0.51600000000000001,"15min":0.52600000000000002},"min":{"1min":0.25800000000000001,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":1.3120000000000001,"5min":1.3120000000000001,"15min":1.3120000000000001},"last":0.432}]},{"osd":5,"last update":"Sat Sep 23 20:57:40 2023","interfaces":[{"interface":"back","average":{"1min":0.68600000000000005,"5min":0.56999999999999995,"15min":0.55000000000000004},"min":{"1min":0.27900000000000003,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":1.9399999999999999,"5min":1.9399999999999999,"15min":1.9399999999999999},"last":0.47799999999999998},{"interface":"front","average":{"1min":0.68899999999999995,"5min":0.78500000000000003,"15min":0.80100000000000005},"min":{"1min":0.30599999999999999,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":1.9159999999999999,"5min":2.274,"15min":2.274},"last":2.161}]},{"osd":7,"last update":"Sat Sep 23 20:57:40 2023","interfaces":[{"interface":"back","average":{"1min":0.95799999999999996,"5min":0.93400000000000005,"15min":0.93000000000000005},"min":{"1min":0.33500000000000002,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":7.8570000000000002,"5min":7.8570000000000002,"15min":7.8570000000000002},"last":0.46600000000000003},{"interface":"front","average":{"1min":1.103,"5min":0.64400000000000002,"15min":0.56799999999999995},"min":{"1min":0.33400000000000002,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":8.282,"5min":8.282,"15min":8.282},"last":0.40200000000000002}]}]},{"osd":1,"up_from":88,"seq":377957122152,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2631968,"kb_used_data":2425496,"kb_used_omap":9,"kb_used_meta":206262,"kb_avail":91113184,"statfs":{"total":95995035648,"available":93299900416,"internally_reserved":0,"allocated":2483707904,"data_stored":2482560181,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9239,"internal_metadata":211213289},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,0,1,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:58:22 2023","interfaces":[{"interface":"back","average":{"1min":3.3290000000000002,"5min":2.9990000000000001,"15min":1.3049999999999999},"min":{"1min":0.23200000000000001,"5min":0.216,"15min":0.216},"max":{"1min":10.73,"5min":16.646999999999998,"15min":16.646999999999998},"last":0.20699999999999999},{"interface":"front","average":{"1min":4.2149999999999999,"5min":3.3940000000000001,"15min":1.4590000000000001},"min":{"1min":0.247,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":15.135999999999999,"5min":18.457999999999998,"15min":18.457999999999998},"last":2.7130000000000001}]},{"osd":2,"last update":"Sat Sep 23 20:57:50 2023","interfaces":[{"interface":"back","average":{"1min":2.5760000000000001,"5min":2.1309999999999998,"15min":1.0249999999999999},"min":{"1min":0.221,"5min":0.20300000000000001,"15min":0.20300000000000001},"max":{"1min":8.6500000000000004,"5min":14.335000000000001,"15min":14.335000000000001},"last":0.35399999999999998},{"interface":"front","average":{"1min":3.2770000000000001,"5min":3.0209999999999999,"15min":1.4099999999999999},"min":{"1min":0.35699999999999998,"5min":0.248,"15min":0.248},"max":{"1min":10.095000000000001,"5min":38.951000000000001,"15min":38.951000000000001},"last":3.1549999999999998}]},{"osd":3,"last update":"Sat Sep 23 20:58:22 2023","interfaces":[{"interface":"back","average":{"1min":3.1859999999999999,"5min":3.0489999999999999,"15min":1.3720000000000001},"min":{"1min":0.246,"5min":0.246,"15min":0.246},"max":{"1min":10.701000000000001,"5min":25.314,"15min":25.314},"last":3.5579999999999998},{"interface":"front","average":{"1min":2.8450000000000002,"5min":2.6150000000000002,"15min":1.232},"min":{"1min":0.33800000000000002,"5min":0.20599999999999999,"15min":0.20599999999999999},"max":{"1min":7.8899999999999997,"5min":19.881,"15min":19.881},"last":3.5409999999999999}]},{"osd":4,"last update":"Sat Sep 23 20:58:22 2023","interfaces":[{"interface":"back","average":{"1min":1.7050000000000001,"5min":1.9550000000000001,"15min":1.131},"min":{"1min":0.36099999999999999,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":10.598000000000001,"5min":12.006,"15min":12.006},"last":2.6179999999999999},{"interface":"front","average":{"1min":2.0899999999999999,"5min":1.6839999999999999,"15min":1.016},"min":{"1min":0.31900000000000001,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":6.0460000000000003,"5min":11.234999999999999,"15min":11.234999999999999},"last":0.42099999999999999}]},{"osd":5,"last update":"Sat Sep 23 20:58:22 2023","interfaces":[{"interface":"back","average":{"1min":3.0609999999999999,"5min":2.133,"15min":1.181},"min":{"1min":0.34399999999999997,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":15.041,"5min":15.041,"15min":15.041},"last":2.7410000000000001},{"interface":"front","average":{"1min":1.669,"5min":2.0169999999999999,"15min":1.153},"min":{"1min":0.33200000000000002,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":10.647,"5min":12.08,"15min":12.08},"last":2.6560000000000001}]},{"osd":6,"last update":"Sat Sep 23 20:57:35 2023","interfaces":[{"interface":"back","average":{"1min":1.2929999999999999,"5min":1.3100000000000001,"15min":1.3129999999999999},"min":{"1min":0.39000000000000001,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":5.3810000000000002,"5min":5.3810000000000002,"15min":5.3810000000000002},"last":2.6789999999999998},{"interface":"front","average":{"1min":1.9530000000000001,"5min":1.393,"15min":1.2989999999999999},"min":{"1min":0.439,"5min":0.309,"15min":0.309},"max":{"1min":6.0860000000000003,"5min":6.5670000000000002,"15min":6.5670000000000002},"last":0.442}]},{"osd":7,"last update":"Sat Sep 23 20:58:22 2023","interfaces":[{"interface":"back","average":{"1min":2.0659999999999998,"5min":1.7909999999999999,"15min":1.069},"min":{"1min":0.42499999999999999,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":6.149,"5min":11.186999999999999,"15min":11.186999999999999},"last":0.45800000000000002},{"interface":"front","average":{"1min":3.1520000000000001,"5min":2.1829999999999998,"15min":1.2010000000000001},"min":{"1min":0.41899999999999998,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":15.098000000000001,"5min":15.098000000000001,"15min":15.098000000000001},"last":2.7919999999999998}]}]},{"osd":0,"up_from":83,"seq":356482285679,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2894168,"kb_used_data":2736208,"kb_used_omap":28,"kb_used_meta":157923,"kb_avail":90850984,"statfs":{"total":95995035648,"available":93031407616,"internally_reserved":0,"allocated":2801876992,"data_stored":2800743758,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":28848,"internal_metadata":161714000},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,0,4],"upper_bound":64},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 20:58:19 2023","interfaces":[{"interface":"back","average":{"1min":3.7799999999999998,"5min":2.9220000000000002,"15min":1.2609999999999999},"min":{"1min":0.25800000000000001,"5min":0.191,"15min":0.191},"max":{"1min":15.946999999999999,"5min":18.863,"15min":18.863},"last":0.36099999999999999},{"interface":"front","average":{"1min":3.9060000000000001,"5min":2.8540000000000001,"15min":1.2669999999999999},"min":{"1min":0.247,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":15.766999999999999,"5min":17.704999999999998,"15min":17.704999999999998},"last":8.0470000000000006}]},{"osd":2,"last update":"Sat Sep 23 20:57:47 2023","interfaces":[{"interface":"back","average":{"1min":2.6150000000000002,"5min":3.0329999999999999,"15min":1.3400000000000001},"min":{"1min":0.35599999999999998,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":8.4949999999999992,"5min":17.753,"15min":17.753},"last":3.5899999999999999},{"interface":"front","average":{"1min":2.843,"5min":2.8199999999999998,"15min":1.234},"min":{"1min":0.308,"5min":0.21099999999999999,"15min":0.21099999999999999},"max":{"1min":9.9469999999999992,"5min":13.218999999999999,"15min":13.218999999999999},"last":14.003}]},{"osd":3,"last update":"Sat Sep 23 20:58:19 2023","interfaces":[{"interface":"back","average":{"1min":3.722,"5min":2.8919999999999999,"15min":1.298},"min":{"1min":0.39300000000000002,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":15.994,"5min":18.893999999999998,"15min":18.893999999999998},"last":3.02},{"interface":"front","average":{"1min":3.7930000000000001,"5min":3.0649999999999999,"15min":1.423},"min":{"1min":0.29699999999999999,"5min":0.28000000000000003,"15min":0.23300000000000001},"max":{"1min":15.808,"5min":17.472000000000001,"15min":17.472000000000001},"last":3.0950000000000002}]},{"osd":4,"last update":"Sat Sep 23 20:57:47 2023","interfaces":[{"interface":"back","average":{"1min":1.968,"5min":1.881,"15min":1.0740000000000001},"min":{"1min":0.35099999999999998,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":9.9079999999999995,"5min":12.807,"15min":12.807},"last":14.044},{"interface":"front","average":{"1min":1.1659999999999999,"5min":1.643,"15min":0.997},"min":{"1min":0.311,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":4.7850000000000001,"5min":18.724,"15min":18.724},"last":0.41199999999999998}]},{"osd":5,"last update":"Sat Sep 23 20:57:47 2023","interfaces":[{"interface":"back","average":{"1min":1.7669999999999999,"5min":1.8129999999999999,"15min":1.069},"min":{"1min":0.33600000000000002,"5min":0.318,"15min":0.318},"max":{"1min":6.673,"5min":8.4990000000000006,"15min":8.4990000000000006},"last":2.9569999999999999},{"interface":"front","average":{"1min":2.2440000000000002,"5min":2.0600000000000001,"15min":1.1619999999999999},"min":{"1min":0.39000000000000001,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":9.8599999999999994,"5min":13.154999999999999,"15min":13.154999999999999},"last":14.076000000000001}]},{"osd":6,"last update":"Sat Sep 23 20:57:41 2023","interfaces":[{"interface":"back","average":{"1min":1.8160000000000001,"5min":1.6879999999999999,"15min":1.667},"min":{"1min":0.37,"5min":0.37,"15min":0.37},"max":{"1min":6.8470000000000004,"5min":8.7309999999999999,"15min":8.7309999999999999},"last":0.497},{"interface":"front","average":{"1min":1.875,"5min":1.7709999999999999,"15min":1.754},"min":{"1min":0.377,"5min":0.377,"15min":0.377},"max":{"1min":5.492,"5min":8.6760000000000002,"15min":8.6760000000000002},"last":0.441}]},{"osd":7,"last update":"Sat Sep 23 20:57:47 2023","interfaces":[{"interface":"back","average":{"1min":1.786,"5min":1.8220000000000001,"15min":1.1020000000000001},"min":{"1min":0.41599999999999998,"5min":0.379,"15min":0.379},"max":{"1min":6.6390000000000002,"5min":8.4339999999999993,"15min":8.4339999999999993},"last":2.984},{"interface":"front","average":{"1min":1.393,"5min":1.698,"15min":1.0269999999999999},"min":{"1min":0.40100000000000002,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":4.6799999999999997,"5min":18.673999999999999,"15min":18.673999999999999},"last":0.46800000000000003}]}]},{"osd":2,"up_from":93,"seq":399431958625,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2765308,"kb_used_data":2461620,"kb_used_omap":81,"kb_used_meta":303598,"kb_avail":90979844,"statfs":{"total":95995035648,"available":93163360256,"internally_reserved":0,"allocated":2520698880,"data_stored":2519582407,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":83750,"internal_metadata":310884570},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:57:43 2023","interfaces":[{"interface":"back","average":{"1min":2.2090000000000001,"5min":1.9319999999999999,"15min":0.89400000000000002},"min":{"1min":0.27100000000000002,"5min":0.17999999999999999,"15min":0.17999999999999999},"max":{"1min":9.7460000000000004,"5min":11.183,"15min":11.183},"last":2.6339999999999999},{"interface":"front","average":{"1min":3.0449999999999999,"5min":2.5110000000000001,"15min":1.1160000000000001},"min":{"1min":0.216,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":10.891999999999999,"5min":12.641,"15min":12.641},"last":4.6449999999999996}]},{"osd":1,"last update":"Sat Sep 23 20:57:43 2023","interfaces":[{"interface":"back","average":{"1min":2.23,"5min":2.8119999999999998,"15min":1.2310000000000001},"min":{"1min":0.21199999999999999,"5min":0.20200000000000001,"15min":0.20200000000000001},"max":{"1min":8.2889999999999997,"5min":16.081,"15min":16.081},"last":0.52300000000000002},{"interface":"front","average":{"1min":2.9660000000000002,"5min":3.419,"15min":1.4390000000000001},"min":{"1min":0.26300000000000001,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":8.8960000000000008,"5min":16.041,"15min":16.041},"last":0.41499999999999998}]},{"osd":3,"last update":"Sat Sep 23 20:58:26 2023","interfaces":[{"interface":"back","average":{"1min":5.1120000000000001,"5min":3.4620000000000002,"15min":1.585},"min":{"1min":0.23300000000000001,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":23.309999999999999,"5min":23.309999999999999,"15min":23.309999999999999},"last":4.5990000000000002},{"interface":"front","average":{"1min":5.5659999999999998,"5min":4.21,"15min":1.8240000000000001},"min":{"1min":0.34300000000000003,"5min":0.22800000000000001,"15min":0.221},"max":{"1min":23.324999999999999,"5min":23.324999999999999,"15min":23.324999999999999},"last":1.212}]},{"osd":4,"last update":"Sat Sep 23 20:57:43 2023","interfaces":[{"interface":"back","average":{"1min":1.8660000000000001,"5min":1.851,"15min":1.0349999999999999},"min":{"1min":0.318,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":8.657,"5min":9.8119999999999994,"15min":9.8119999999999994},"last":0.45200000000000001},{"interface":"front","average":{"1min":1.9770000000000001,"5min":2.0190000000000001,"15min":1.1299999999999999},"min":{"1min":0.34300000000000003,"5min":0.313,"15min":0.313},"max":{"1min":6.7850000000000001,"5min":10.609,"15min":10.609},"last":0.53800000000000003}]},{"osd":5,"last update":"Sat Sep 23 20:57:43 2023","interfaces":[{"interface":"back","average":{"1min":1.823,"5min":2.121,"15min":1.145},"min":{"1min":0.29299999999999998,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":8.5700000000000003,"5min":9.7629999999999999,"15min":9.7629999999999999},"last":0.47599999999999998},{"interface":"front","average":{"1min":2.0259999999999998,"5min":2.129,"15min":1.147},"min":{"1min":0.32600000000000001,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":6.7510000000000003,"5min":10.574,"15min":10.574},"last":0.57799999999999996}]},{"osd":6,"last update":"Sat Sep 23 20:57:41 2023","interfaces":[{"interface":"back","average":{"1min":1.9450000000000001,"5min":1.7230000000000001,"15min":1.6859999999999999},"min":{"1min":0.312,"5min":0.312,"15min":0.312},"max":{"1min":6.8109999999999999,"5min":14.244999999999999,"15min":14.244999999999999},"last":0.49199999999999999},{"interface":"front","average":{"1min":1.9390000000000001,"5min":1.2110000000000001,"15min":1.0900000000000001},"min":{"1min":0.374,"5min":0.374,"15min":0.374},"max":{"1min":6.835,"5min":6.835,"15min":6.835},"last":0.60599999999999998}]},{"osd":7,"last update":"Sat Sep 23 20:57:43 2023","interfaces":[{"interface":"back","average":{"1min":1.397,"5min":1.524,"15min":0.93799999999999994},"min":{"1min":0.32500000000000001,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":4.9820000000000002,"5min":8.0820000000000007,"15min":8.0820000000000007},"last":0.55300000000000005},{"interface":"front","average":{"1min":1.804,"5min":2.1360000000000001,"15min":1.159},"min":{"1min":0.35699999999999998,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":8.6219999999999999,"5min":9.7859999999999996,"15min":9.7859999999999996},"last":0.56599999999999995}]}]},{"osd":3,"up_from":98,"seq":420906795099,"num_pgs":49,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2580956,"kb_used_data":2253396,"kb_used_omap":7,"kb_used_meta":327544,"kb_avail":91164196,"statfs":{"total":95995035648,"available":93352136704,"internally_reserved":0,"allocated":2307477504,"data_stored":2306389904,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7576,"internal_metadata":335405672},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1,1,1,5],"upper_bound":64},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:58:25 2023","interfaces":[{"interface":"back","average":{"1min":3.097,"5min":3.02,"15min":1.4890000000000001},"min":{"1min":0.308,"5min":0.20599999999999999,"15min":0.20599999999999999},"max":{"1min":8.5549999999999997,"5min":23.123999999999999,"15min":23.123999999999999},"last":1.698},{"interface":"front","average":{"1min":2.9900000000000002,"5min":2.7869999999999999,"15min":1.3009999999999999},"min":{"1min":0.19800000000000001,"5min":0.19800000000000001,"15min":0.19800000000000001},"max":{"1min":9.8889999999999993,"5min":15.079000000000001,"15min":15.079000000000001},"last":0.81799999999999995}]},{"osd":1,"last update":"Sat Sep 23 20:58:25 2023","interfaces":[{"interface":"back","average":{"1min":3.7829999999999999,"5min":3.1509999999999998,"15min":1.4390000000000001},"min":{"1min":0.32400000000000001,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":13.173,"5min":21.385000000000002,"15min":21.385000000000002},"last":0.50700000000000001},{"interface":"front","average":{"1min":3.0880000000000001,"5min":3.242,"15min":1.5109999999999999},"min":{"1min":0.28799999999999998,"5min":0.218,"15min":0.218},"max":{"1min":10.683999999999999,"5min":25.353000000000002,"15min":25.353000000000002},"last":0.53800000000000003}]},{"osd":2,"last update":"Sat Sep 23 20:58:25 2023","interfaces":[{"interface":"back","average":{"1min":4.851,"5min":3.052,"15min":1.444},"min":{"1min":0.35899999999999999,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":15.476000000000001,"5min":24.154,"15min":24.154},"last":0.45600000000000002},{"interface":"front","average":{"1min":2.8519999999999999,"5min":3.335,"15min":1.577},"min":{"1min":0.23999999999999999,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":8.6069999999999993,"5min":25.305,"15min":25.305},"last":0.56399999999999995}]},{"osd":4,"last update":"Sat Sep 23 20:58:25 2023","interfaces":[{"interface":"back","average":{"1min":2.6539999999999999,"5min":1.8240000000000001,"15min":1.141},"min":{"1min":0.38900000000000001,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":9.6170000000000009,"5min":11.861000000000001,"15min":11.861000000000001},"last":0.48599999999999999},{"interface":"front","average":{"1min":1.3600000000000001,"5min":2.0880000000000001,"15min":1.246},"min":{"1min":0.313,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":4.8109999999999999,"5min":22.853000000000002,"15min":22.853000000000002},"last":0.60599999999999998}]},{"osd":5,"last update":"Sat Sep 23 20:58:25 2023","interfaces":[{"interface":"back","average":{"1min":2.9540000000000002,"5min":1.8999999999999999,"15min":1.1459999999999999},"min":{"1min":0.436,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":9.6430000000000007,"5min":11.891999999999999,"15min":11.891999999999999},"last":0.63500000000000001},{"interface":"front","average":{"1min":2.04,"5min":1.6579999999999999,"15min":1.075},"min":{"1min":0.33300000000000002,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":5.7690000000000001,"5min":9.2870000000000008,"15min":9.2870000000000008},"last":0.748}]},{"osd":6,"last update":"Sat Sep 23 20:57:39 2023","interfaces":[{"interface":"back","average":{"1min":1.173,"5min":0.99399999999999999,"15min":0.96399999999999997},"min":{"1min":0.40999999999999998,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":6.3079999999999998,"5min":6.3079999999999998,"15min":6.3079999999999998},"last":0.58599999999999997},{"interface":"front","average":{"1min":1.6879999999999999,"5min":1.4670000000000001,"15min":1.4299999999999999},"min":{"1min":0.40100000000000002,"5min":0.40100000000000002,"15min":0.40100000000000002},"max":{"1min":5.8230000000000004,"5min":7.234,"15min":7.234},"last":0.79300000000000004}]},{"osd":7,"last update":"Sat Sep 23 20:58:25 2023","interfaces":[{"interface":"back","average":{"1min":3.0099999999999998,"5min":2.3399999999999999,"15min":1.3220000000000001},"min":{"1min":0.46500000000000002,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":9.5749999999999993,"5min":16.364999999999998,"15min":16.364999999999998},"last":0.87},{"interface":"front","average":{"1min":2.214,"5min":1.766,"15min":1.147},"min":{"1min":0.42499999999999999,"5min":0.379,"15min":0.379},"max":{"1min":5.8090000000000002,"5min":9.3179999999999996,"15min":9.3179999999999996},"last":0.95099999999999996}]}]},{"osd":4,"up_from":31,"seq":133143986406,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1876988,"kb_used_data":1755764,"kb_used_omap":0,"kb_used_meta":121216,"kb_avail":91868164,"statfs":{"total":95995035648,"available":94072999936,"internally_reserved":0,"allocated":1797902336,"data_stored":1797074999,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":124125184},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,0,2,1,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:57:52 2023","interfaces":[{"interface":"back","average":{"1min":2.4359999999999999,"5min":1.593,"15min":0.95599999999999996},"min":{"1min":0.214,"5min":0.214,"15min":0.214},"max":{"1min":11.784000000000001,"5min":11.784000000000001,"15min":11.784000000000001},"last":1.0009999999999999},{"interface":"front","average":{"1min":2.2970000000000002,"5min":1.782,"15min":1.1699999999999999},"min":{"1min":0.28100000000000003,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":11.766,"5min":11.766,"15min":11.766},"last":1.008}]},{"osd":1,"last update":"Sat Sep 23 20:58:21 2023","interfaces":[{"interface":"back","average":{"1min":1.7370000000000001,"5min":1.641,"15min":0.996},"min":{"1min":0.246,"5min":0.246,"15min":0.23599999999999999},"max":{"1min":10.153,"5min":10.153,"15min":10.153},"last":2.5470000000000002},{"interface":"front","average":{"1min":2.645,"5min":1.649,"15min":0.96399999999999997},"min":{"1min":0.26100000000000001,"5min":0.26100000000000001,"15min":0.23200000000000001},"max":{"1min":12.292999999999999,"5min":12.292999999999999,"15min":12.292999999999999},"last":2.8530000000000002}]},{"osd":2,"last update":"Sat Sep 23 20:57:52 2023","interfaces":[{"interface":"back","average":{"1min":1.5840000000000001,"5min":1.746,"15min":1.036},"min":{"1min":0.35699999999999998,"5min":0.311,"15min":0.29099999999999998},"max":{"1min":5.21,"5min":15.288,"15min":15.288},"last":0.92900000000000005},{"interface":"front","average":{"1min":1.8939999999999999,"5min":2.0720000000000001,"15min":1.2050000000000001},"min":{"1min":0.39500000000000002,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":3.8300000000000001,"5min":15.262,"15min":15.262},"last":2.1000000000000001}]},{"osd":3,"last update":"Sat Sep 23 20:58:21 2023","interfaces":[{"interface":"back","average":{"1min":2.0209999999999999,"5min":2.504,"15min":1.514},"min":{"1min":0.28899999999999998,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":7.7549999999999999,"5min":15.916,"15min":31.562999999999999},"last":2.5659999999999998},{"interface":"front","average":{"1min":2.2080000000000002,"5min":2.5449999999999999,"15min":1.589},"min":{"1min":0.433,"5min":0.433,"15min":0.41899999999999998},"max":{"1min":7.7300000000000004,"5min":15.882,"15min":31.574000000000002},"last":0.30299999999999999}]},{"osd":5,"last update":"Sat Sep 23 20:57:34 2023","interfaces":[{"interface":"back","average":{"1min":0.74199999999999999,"5min":0.53700000000000003,"15min":0.50600000000000001},"min":{"1min":0.248,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":3.008,"5min":3.3279999999999998,"15min":3.3279999999999998},"last":0.496},{"interface":"front","average":{"1min":0.66300000000000003,"5min":0.68300000000000005,"15min":0.55800000000000005},"min":{"1min":0.27300000000000002,"5min":0.251,"15min":0.251},"max":{"1min":3.1280000000000001,"5min":7.7619999999999996,"15min":7.7619999999999996},"last":0.51000000000000001}]},{"osd":6,"last update":"Sat Sep 23 20:57:40 2023","interfaces":[{"interface":"back","average":{"1min":1.032,"5min":0.59299999999999997,"15min":0.52000000000000002},"min":{"1min":0.26900000000000002,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":4.6909999999999998,"5min":4.6909999999999998,"15min":4.6909999999999998},"last":2.4929999999999999},{"interface":"front","average":{"1min":0.82499999999999996,"5min":0.68000000000000005,"15min":0.65600000000000003},"min":{"1min":0.38900000000000001,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":2.7120000000000002,"5min":2.7120000000000002,"15min":2.7120000000000002},"last":1.5760000000000001}]},{"osd":7,"last update":"Sat Sep 23 20:58:20 2023","interfaces":[{"interface":"back","average":{"1min":0.84199999999999997,"5min":0.83099999999999996,"15min":0.67100000000000004},"min":{"1min":0.442,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":3.327,"5min":7.383,"15min":7.383},"last":0.42199999999999999},{"interface":"front","average":{"1min":0.93899999999999995,"5min":0.80800000000000005,"15min":0.66000000000000003},"min":{"1min":0.36399999999999999,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":3.883,"5min":7.8310000000000004,"15min":7.8310000000000004},"last":0.54200000000000004}]}]},{"osd":5,"up_from":39,"seq":167503724771,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1875440,"kb_used_data":1755816,"kb_used_omap":0,"kb_used_meta":119616,"kb_avail":91869712,"statfs":{"total":95995035648,"available":94074585088,"internally_reserved":0,"allocated":1797955584,"data_stored":1797136823,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":122486784},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:57:36 2023","interfaces":[{"interface":"back","average":{"1min":1.145,"5min":1.1200000000000001,"15min":0.70799999999999996},"min":{"1min":0.20399999999999999,"5min":0.20399999999999999,"15min":0.20399999999999999},"max":{"1min":7.5019999999999998,"5min":7.766,"15min":7.766},"last":3.6949999999999998},{"interface":"front","average":{"1min":1.647,"5min":1.556,"15min":0.89900000000000002},"min":{"1min":0.25600000000000001,"5min":0.223,"15min":0.223},"max":{"1min":11.378,"5min":15.037000000000001,"15min":15.037000000000001},"last":0.37}]},{"osd":1,"last update":"Sat Sep 23 20:58:16 2023","interfaces":[{"interface":"back","average":{"1min":1.5549999999999999,"5min":1.603,"15min":0.95199999999999996},"min":{"1min":0.27100000000000002,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":6.7409999999999997,"5min":8.3520000000000003,"15min":8.3520000000000003},"last":0.248},{"interface":"front","average":{"1min":1.9590000000000001,"5min":1.7450000000000001,"15min":0.97699999999999998},"min":{"1min":0.28100000000000003,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":7.4669999999999996,"5min":10.285,"15min":10.285},"last":2.4660000000000002}]},{"osd":2,"last update":"Sat Sep 23 20:57:50 2023","interfaces":[{"interface":"back","average":{"1min":1.2410000000000001,"5min":1.575,"15min":0.88500000000000001},"min":{"1min":0.26200000000000001,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":10.579000000000001,"5min":12.651,"15min":12.651},"last":1.8360000000000001},{"interface":"front","average":{"1min":1.0669999999999999,"5min":1.5449999999999999,"15min":0.91900000000000004},"min":{"1min":0.26100000000000001,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":5.2949999999999999,"5min":8.3209999999999997,"15min":8.3209999999999997},"last":5.7969999999999997}]},{"osd":3,"last update":"Sat Sep 23 20:58:16 2023","interfaces":[{"interface":"back","average":{"1min":2.331,"5min":2.0670000000000002,"15min":1.153},"min":{"1min":0.39200000000000002,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":8.6579999999999995,"5min":9.5489999999999995,"15min":9.5489999999999995},"last":3.722},{"interface":"front","average":{"1min":1.6000000000000001,"5min":1.6040000000000001,"15min":0.97499999999999998},"min":{"1min":0.28899999999999998,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":10.457000000000001,"5min":10.457000000000001,"15min":10.457000000000001},"last":1.764}]},{"osd":4,"last update":"Sat Sep 23 20:57:29 2023","interfaces":[{"interface":"back","average":{"1min":0.49099999999999999,"5min":0.60199999999999998,"15min":0.52800000000000002},"min":{"1min":0.29699999999999999,"5min":0.27600000000000002,"15min":0.253},"max":{"1min":1.4239999999999999,"5min":3.8479999999999999,"15min":3.8479999999999999},"last":0.32900000000000001},{"interface":"front","average":{"1min":0.56299999999999994,"5min":0.63900000000000001,"15min":0.55600000000000005},"min":{"1min":0.23599999999999999,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":1.389,"5min":3.0680000000000001,"15min":3.0680000000000001},"last":0.34399999999999997}]},{"osd":6,"last update":"Sat Sep 23 20:57:36 2023","interfaces":[{"interface":"back","average":{"1min":0.80800000000000005,"5min":0.60999999999999999,"15min":0.57699999999999996},"min":{"1min":0.28000000000000003,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":2.9820000000000002,"5min":2.9820000000000002,"15min":2.9820000000000002},"last":0.45900000000000002},{"interface":"front","average":{"1min":0.76400000000000001,"5min":0.748,"15min":0.745},"min":{"1min":0.33200000000000002,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":2.6579999999999999,"5min":3.6560000000000001,"15min":3.6560000000000001},"last":0.45100000000000001}]},{"osd":7,"last update":"Sat Sep 23 20:58:13 2023","interfaces":[{"interface":"back","average":{"1min":0.66500000000000004,"5min":0.76600000000000001,"15min":0.63200000000000001},"min":{"1min":0.316,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":2.4809999999999999,"5min":8.5649999999999995,"15min":8.5649999999999995},"last":0.48799999999999999},{"interface":"front","average":{"1min":0.60299999999999998,"5min":0.74099999999999999,"15min":0.64400000000000002},"min":{"1min":0.35399999999999998,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":2.7109999999999999,"5min":4.5430000000000001,"15min":4.5430000000000001},"last":0.42199999999999999}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":2790133760,"data_stored":2790129688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":2470776832,"data_stored":2470772760,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":2512191488,"data_stored":2512191488,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":2298806272,"data_stored":2298806272,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1785790464,"data_stored":1785790464,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1788280832,"data_stored":1788280832,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1770917888,"data_stored":1770913816,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1800667136,"data_stored":1800667136,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T20:58:37.123 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:58:37.149 INFO:teuthology.orchestra.run.smithi070.stdout:ERROR: (22) Invalid argument 2023-09-23T20:58:37.149 INFO:teuthology.orchestra.run.smithi070.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:58:37.168 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:58:37.269 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:58:37.272 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:58:37.273 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:58:37.373 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:58:37.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:58:37 smithi026 ceph-mon[179688]: pgmap v559: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 5.3 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 619 op/s; 10943/259206 objects degraded (4.222%); 4342/259206 objects misplaced (1.675%); 1.4 MiB/s, 22 objects/s recovering 2023-09-23T20:58:37.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:58:37 smithi026 ceph-mon[179688]: from='client.35265 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:58:37.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:58:37 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/3690204925' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:58:37.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:58:37 smithi026 ceph-mon[175151]: pgmap v559: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 5.3 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 619 op/s; 10943/259206 objects degraded (4.222%); 4342/259206 objects misplaced (1.675%); 1.4 MiB/s, 22 objects/s recovering 2023-09-23T20:58:37.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:58:37 smithi026 ceph-mon[175151]: from='client.35265 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:58:37.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:58:37 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/3690204925' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:58:37.403 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:58:37.567 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:58:37 smithi070 ceph-mon[168282]: pgmap v559: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 5.3 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 619 op/s; 10943/259206 objects degraded (4.222%); 4342/259206 objects misplaced (1.675%); 1.4 MiB/s, 22 objects/s recovering 2023-09-23T20:58:37.568 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:58:37 smithi070 ceph-mon[168282]: from='client.35265 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:58:37.568 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:58:37 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/3690204925' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:58:37.579 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:58:37.583 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:58:37.583 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:58:37.623 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_ops_in_flight 2023-09-23T20:58:37.684 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:58:37.744 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:58:37.797 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:58:37.804 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:58:37.805 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:58:37.905 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:58:37.944 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:58:37.987 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:58:37.995 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:58:37.995 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:58:38.096 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:58:38.160 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T20:58:38.196 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:58:38.286 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:58:38.292 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:58:38.293 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:58:38.342 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:58:38.394 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:58:38.527 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:58:38.646 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:58:38.659 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:58:38.660 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:58:38.761 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:58:38.802 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:58:38.867 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:58:38.870 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:58:38.871 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:58:38.971 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:58:39.034 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:58:39.096 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:58:39.111 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:58:39.115 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:58:39.115 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:58:39.216 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:58:39.265 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:58:39.393 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:58:39.398 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:58:39.398 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:58:39.409 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T20:58:39.499 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:58:39.545 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:58:39.641 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:58:39.643 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:58:39.644 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:58:39.744 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:58:39.752 INFO:teuthology.orchestra.run.smithi070.stdout:ERROR: (22) Invalid argument 2023-09-23T20:58:39.752 INFO:teuthology.orchestra.run.smithi070.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:58:39.773 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:58:39 smithi070 ceph-mon[168282]: pgmap v560: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 5.2 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 618 op/s; 10943/256443 objects degraded (4.267%); 4342/256443 objects misplaced (1.693%); 1.4 MiB/s, 22 objects/s recovering 2023-09-23T20:58:39.778 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:58:39.822 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:58:39.824 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:58:39.825 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:58:39.883 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:58:39 smithi026 ceph-mon[175151]: pgmap v560: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 5.2 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 618 op/s; 10943/256443 objects degraded (4.267%); 4342/256443 objects misplaced (1.693%); 1.4 MiB/s, 22 objects/s recovering 2023-09-23T20:58:39.884 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:58:39 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[164029]: ::ffff:172.21.15.70 - - [23/Sep/2023:20:58:39] "GET /metrics HTTP/1.1" 200 36324 "" "Prometheus/2.43.0" 2023-09-23T20:58:39.884 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:58:39 smithi026 ceph-mon[179688]: pgmap v560: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 5.2 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 618 op/s; 10943/256443 objects degraded (4.267%); 4342/256443 objects misplaced (1.693%); 1.4 MiB/s, 22 objects/s recovering 2023-09-23T20:58:39.925 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:58:39.953 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:58:40.073 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T20:58:40.182 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:58:40.185 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:58:40.185 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:58:40.240 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_blocked_ops 2023-09-23T20:58:40.286 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:58:40.375 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:58:40.489 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:58:40 smithi026 ceph-mon[175151]: Health check update: Degraded data redundancy: 10943/256443 objects degraded (4.267%), 2 pgs degraded (PG_DEGRADED) 2023-09-23T20:58:40.779 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:58:40.790 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:58:40.790 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:58:40 smithi026 ceph-mon[179688]: Health check update: Degraded data redundancy: 10943/256443 objects degraded (4.267%), 2 pgs degraded (PG_DEGRADED) 2023-09-23T20:58:40.791 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:58:40.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:58:40 smithi070 ceph-mon[168282]: Health check update: Degraded data redundancy: 10943/256443 objects degraded (4.267%), 2 pgs degraded (PG_DEGRADED) 2023-09-23T20:58:40.891 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:58:40.913 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:58:40.995 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:58:41.459 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:58:41.469 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:58:41.469 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:58:41.570 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:58:41.672 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:58:41.819 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:58:41.833 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:58:41.834 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:58:41.935 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:58:41.975 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:58:42.050 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:58:42 smithi070 ceph-mon[168282]: pgmap v561: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 5.1 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 614 op/s; 10943/252510 objects degraded (4.334%); 4342/252510 objects misplaced (1.720%); 1.4 MiB/s, 22 objects/s recovering 2023-09-23T20:58:42.079 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:58:42.082 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:58:42.083 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:58:42.183 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:58:42.196 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:58:42.256 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:58:42 smithi026 ceph-mon[175151]: pgmap v561: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 5.1 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 614 op/s; 10943/252510 objects degraded (4.334%); 4342/252510 objects misplaced (1.720%); 1.4 MiB/s, 22 objects/s recovering 2023-09-23T20:58:42.256 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:58:42 smithi026 ceph-mon[179688]: pgmap v561: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 5.1 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 614 op/s; 10943/252510 objects degraded (4.334%); 4342/252510 objects misplaced (1.720%); 1.4 MiB/s, 22 objects/s recovering 2023-09-23T20:58:42.273 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:58:42.295 INFO:teuthology.orchestra.run.smithi070.stdout:ERROR: (22) Invalid argument 2023-09-23T20:58:42.295 INFO:teuthology.orchestra.run.smithi070.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:58:42.382 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:58:42.387 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:58:42.388 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:58:42.388 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:58:42.489 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:58:42.605 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:58:42.702 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_historic_ops 2023-09-23T20:58:43.010 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:58:43.028 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:58:43.028 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:58:43.129 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:58:43.157 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:58:43.197 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:58:43.202 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:58:43.203 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:58:43.303 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:58:43.338 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:58:43.366 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:58:43.381 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:58:43.384 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:58:43.385 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:58:43.486 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:58:43.518 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:58:43.622 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:58:43.628 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:58:43.628 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:58:43.729 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:58:43.763 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:58:43.804 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:58:43.808 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:58:43.809 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:58:43.909 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:58:43.942 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:58:44.063 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:58:44 smithi070 ceph-mon[168282]: pgmap v562: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 5.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 610 op/s; 10981/247071 objects degraded (4.444%); 4186/247071 objects misplaced (1.694%); 1.4 MiB/s, 22 objects/s recovering 2023-09-23T20:58:44.063 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:58:44 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/2514513546' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:58:44.063 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:58:44 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:58:44.063 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:58:44 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:58:44.066 INFO:teuthology.orchestra.run.smithi026.stderr:nodeep-scrub is unset 2023-09-23T20:58:44.134 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:58:44 smithi026 ceph-mon[175151]: pgmap v562: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 5.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 610 op/s; 10981/247071 objects degraded (4.444%); 4186/247071 objects misplaced (1.694%); 1.4 MiB/s, 22 objects/s recovering 2023-09-23T20:58:44.135 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:58:44 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/2514513546' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:58:44.135 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:58:44 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:58:44.135 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:58:44 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:58:44.135 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:58:44 smithi026 ceph-mon[179688]: pgmap v562: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 5.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 610 op/s; 10981/247071 objects degraded (4.444%); 4186/247071 objects misplaced (1.694%); 1.4 MiB/s, 22 objects/s recovering 2023-09-23T20:58:44.135 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:58:44 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/2514513546' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:58:44.135 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:58:44 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:58:44.135 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:58:44 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:58:44.200 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:58:44.204 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:58:44.204 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:58:44.305 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:58:44.353 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:58:44.392 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:58:44.396 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:58:44.396 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:58:44.497 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:58:44.546 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:58:44.600 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:58:44.606 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:58:44.606 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:58:44.620 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0: {} 2023-09-23T20:58:44.694 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1: {} 2023-09-23T20:58:44.707 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:58:44.732 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2: {} 2023-09-23T20:58:44.747 INFO:teuthology.orchestra.run.smithi070.stdout:ERROR: (22) Invalid argument 2023-09-23T20:58:44.748 INFO:teuthology.orchestra.run.smithi070.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:58:44.795 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3: {} 2023-09-23T20:58:44.801 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:58:44.843 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4: {} 2023-09-23T20:58:44.883 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:58:44.898 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T20:58:44.904 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:58:44.904 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:58:44.943 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6: {} 2023-09-23T20:58:45.003 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T20:58:45.005 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:58:45.033 INFO:teuthology.orchestra.run.smithi026.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T20:58:45.033 INFO:teuthology.orchestra.run.smithi026.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T20:58:45.033 INFO:teuthology.orchestra.run.smithi026.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T20:58:45.033 INFO:teuthology.orchestra.run.smithi026.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T20:58:45.033 INFO:teuthology.orchestra.run.smithi026.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T20:58:45.033 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T20:58:45.034 INFO:teuthology.orchestra.run.smithi026.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T20:58:45.034 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T20:58:45.065 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:58:45 smithi026 ceph-mon[179688]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T20:58:45.065 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:58:45 smithi026 ceph-mon[179688]: failed to encode map e236 with expected crc 2023-09-23T20:58:45.065 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:58:45 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T20:58:45.065 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:58:45 smithi026 ceph-mon[179688]: osdmap e236: 8 total, 8 up, 8 in 2023-09-23T20:58:45.066 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:58:45 smithi026 ceph-mon[179688]: failed to encode map e236 with expected crc 2023-09-23T20:58:45.066 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:58:45 smithi026 ceph-mon[179688]: failed to encode map e236 with expected crc 2023-09-23T20:58:45.066 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:58:45 smithi026 ceph-mon[179688]: failed to encode map e236 with expected crc 2023-09-23T20:58:45.066 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:58:45 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:58:45.066 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:58:45 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:58:45.066 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:58:45 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/4218357479' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:58:45.066 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:58:45 smithi026 ceph-mon[179688]: Health check update: Degraded data redundancy: 10981/247071 objects degraded (4.444%), 3 pgs degraded (PG_DEGRADED) 2023-09-23T20:58:45.066 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:58:45 smithi026 ceph-mon[175151]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T20:58:45.066 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:58:45 smithi026 ceph-mon[175151]: failed to encode map e236 with expected crc 2023-09-23T20:58:45.067 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:58:45 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T20:58:45.067 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:58:45 smithi026 ceph-mon[175151]: osdmap e236: 8 total, 8 up, 8 in 2023-09-23T20:58:45.067 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:58:45 smithi026 ceph-mon[175151]: failed to encode map e236 with expected crc 2023-09-23T20:58:45.067 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:58:45 smithi026 ceph-mon[175151]: failed to encode map e236 with expected crc 2023-09-23T20:58:45.067 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:58:45 smithi026 ceph-mon[175151]: failed to encode map e236 with expected crc 2023-09-23T20:58:45.067 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:58:45 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:58:45.067 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:58:45 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:58:45.067 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:58:45 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/4218357479' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:58:45.067 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:58:45 smithi026 ceph-mon[175151]: Health check update: Degraded data redundancy: 10981/247071 objects degraded (4.444%), 3 pgs degraded (PG_DEGRADED) 2023-09-23T20:58:45.100 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:58:45.150 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_ops_in_flight 2023-09-23T20:58:45.179 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:58:45 smithi070 ceph-mon[168282]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T20:58:45.179 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:58:45 smithi070 ceph-mon[168282]: failed to encode map e236 with expected crc 2023-09-23T20:58:45.179 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:58:45 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T20:58:45.179 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:58:45 smithi070 ceph-mon[168282]: osdmap e236: 8 total, 8 up, 8 in 2023-09-23T20:58:45.179 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:58:45 smithi070 ceph-mon[168282]: failed to encode map e236 with expected crc 2023-09-23T20:58:45.179 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:58:45 smithi070 ceph-mon[168282]: failed to encode map e236 with expected crc 2023-09-23T20:58:45.179 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:58:45 smithi070 ceph-mon[168282]: failed to encode map e236 with expected crc 2023-09-23T20:58:45.180 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:58:45 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:58:45.180 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:58:45 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:58:45.180 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:58:45 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/4218357479' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:58:45.180 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:58:45 smithi070 ceph-mon[168282]: Health check update: Degraded data redundancy: 10981/247071 objects degraded (4.444%), 3 pgs degraded (PG_DEGRADED) 2023-09-23T20:58:45.423 INFO:teuthology.orchestra.run.smithi026.stdout: 2023-09-23T20:58:45.455 INFO:teuthology.orchestra.run.smithi026.stderr:dumped all 2023-09-23T20:58:45.470 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:58:45.487 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:58:45.488 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:58:45.589 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:58:45.621 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:58:45.661 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:58:45.664 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:58:45.664 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:58:45.765 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:58:45.867 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:58:45.878 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:58:45.979 INFO:teuthology.orchestra.run.smithi026.stdout:{"pg_ready":true,"pg_map":{"version":564,"stamp":"2023-09-23T20:58:44.410467+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":5345712056,"num_objects":81585,"num_object_clones":0,"num_object_copies":244755,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":10981,"num_objects_misplaced":4186,"num_objects_unfound":0,"num_objects_dirty":81585,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":122020,"num_write_kb":6512606,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1197,"num_bytes_recovered":77010790,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":117995,"ondisk_log_size":117995,"up":327,"acting":324,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":331,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":17513984,"kb_used_data":15956608,"kb_used_omap":136,"kb_used_meta":1557175,"kb_avail":732447232,"statfs":{"total":767960285184,"available":750025965568,"internally_reserved":0,"allocated":16339566592,"data_stored":16331760387,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":139741,"internal_metadata":1594547747},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":9,"apply_latency_ms":9,"commit_latency_ns":9000000,"apply_latency_ns":9000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-363069440,"num_objects":-5540,"num_object_clones":0,"num_object_copies":-16620,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":38,"num_objects_misplaced":-267,"num_objects_unfound":0,"num_objects_dirty":-5540,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5540,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":229,"num_bytes_recovered":15007744,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"9.229925"},"pg_stats":[{"pgid":"3.1a","version":"68'1","reported_seq":112,"reported_epoch":234,"state":"active+clean","last_fresh":"2023-09-23T20:58:26.117177+0000","last_change":"2023-09-23T20:54:13.946695+0000","last_active":"2023-09-23T20:58:26.117177+0000","last_peered":"2023-09-23T20:58:26.117177+0000","last_clean":"2023-09-23T20:58:26.117177+0000","last_became_active":"2023-09-23T20:54:13.945399+0000","last_became_peered":"2023-09-23T20:54:13.945399+0000","last_unstale":"2023-09-23T20:58:26.117177+0000","last_undegraded":"2023-09-23T20:58:26.117177+0000","last_fullsized":"2023-09-23T20:58:26.117177+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:32:52.365464+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":227,"reported_epoch":235,"state":"active+clean","last_fresh":"2023-09-23T20:58:35.203449+0000","last_change":"2023-09-23T20:54:13.942019+0000","last_active":"2023-09-23T20:58:35.203449+0000","last_peered":"2023-09-23T20:58:35.203449+0000","last_clean":"2023-09-23T20:58:35.203449+0000","last_became_active":"2023-09-23T20:54:13.941647+0000","last_became_peered":"2023-09-23T20:54:13.941647+0000","last_unstale":"2023-09-23T20:58:35.203449+0000","last_undegraded":"2023-09-23T20:58:35.203449+0000","last_fullsized":"2023-09-23T20:58:35.203449+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:30:54.400944+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":205,"reported_epoch":235,"state":"active+clean","last_fresh":"2023-09-23T20:58:35.201680+0000","last_change":"2023-09-23T20:54:13.941394+0000","last_active":"2023-09-23T20:58:35.201680+0000","last_peered":"2023-09-23T20:58:35.201680+0000","last_clean":"2023-09-23T20:58:35.201680+0000","last_became_active":"2023-09-23T20:54:13.941042+0000","last_became_peered":"2023-09-23T20:54:13.941042+0000","last_unstale":"2023-09-23T20:58:35.201680+0000","last_undegraded":"2023-09-23T20:58:35.201680+0000","last_fullsized":"2023-09-23T20:58:35.201680+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:19:41.401184+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":219,"reported_epoch":234,"state":"active+clean","last_fresh":"2023-09-23T20:58:26.130527+0000","last_change":"2023-09-23T20:57:44.979811+0000","last_active":"2023-09-23T20:58:26.130527+0000","last_peered":"2023-09-23T20:58:26.130527+0000","last_clean":"2023-09-23T20:58:26.130527+0000","last_became_active":"2023-09-23T20:57:44.979515+0000","last_became_peered":"2023-09-23T20:57:44.979515+0000","last_unstale":"2023-09-23T20:58:26.130527+0000","last_undegraded":"2023-09-23T20:58:26.130527+0000","last_fullsized":"2023-09-23T20:58:26.130527+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:38:22.162140+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":218,"reported_epoch":234,"state":"active+clean","last_fresh":"2023-09-23T20:58:26.130962+0000","last_change":"2023-09-23T20:51:10.833613+0000","last_active":"2023-09-23T20:58:26.130962+0000","last_peered":"2023-09-23T20:58:26.130962+0000","last_clean":"2023-09-23T20:58:26.130962+0000","last_became_active":"2023-09-23T20:51:10.833208+0000","last_became_peered":"2023-09-23T20:51:10.833208+0000","last_unstale":"2023-09-23T20:58:26.130962+0000","last_undegraded":"2023-09-23T20:58:26.130962+0000","last_fullsized":"2023-09-23T20:58:26.130962+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:51:28.246014+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":201,"reported_epoch":235,"state":"active+clean","last_fresh":"2023-09-23T20:58:35.190854+0000","last_change":"2023-09-23T20:55:35.958384+0000","last_active":"2023-09-23T20:58:35.190854+0000","last_peered":"2023-09-23T20:58:35.190854+0000","last_clean":"2023-09-23T20:58:35.190854+0000","last_became_active":"2023-09-23T20:55:35.954563+0000","last_became_peered":"2023-09-23T20:55:35.954563+0000","last_unstale":"2023-09-23T20:58:35.190854+0000","last_undegraded":"2023-09-23T20:58:35.190854+0000","last_fullsized":"2023-09-23T20:58:35.190854+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:37:48.332259+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":209,"reported_epoch":234,"state":"active+clean","last_fresh":"2023-09-23T20:58:26.136449+0000","last_change":"2023-09-23T20:54:31.885514+0000","last_active":"2023-09-23T20:58:26.136449+0000","last_peered":"2023-09-23T20:58:26.136449+0000","last_clean":"2023-09-23T20:58:26.136449+0000","last_became_active":"2023-09-23T20:54:31.884986+0000","last_became_peered":"2023-09-23T20:54:31.884986+0000","last_unstale":"2023-09-23T20:58:26.136449+0000","last_undegraded":"2023-09-23T20:58:26.136449+0000","last_fullsized":"2023-09-23T20:58:26.136449+0000","mapping_epoch":136,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:26:08.100220+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":215,"reported_epoch":234,"state":"active+clean","last_fresh":"2023-09-23T20:58:26.131036+0000","last_change":"2023-09-23T20:55:35.951889+0000","last_active":"2023-09-23T20:58:26.131036+0000","last_peered":"2023-09-23T20:58:26.131036+0000","last_clean":"2023-09-23T20:58:26.131036+0000","last_became_active":"2023-09-23T20:55:35.951563+0000","last_became_peered":"2023-09-23T20:55:35.951563+0000","last_unstale":"2023-09-23T20:58:26.131036+0000","last_undegraded":"2023-09-23T20:58:26.131036+0000","last_fullsized":"2023-09-23T20:58:26.131036+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:41:53.098781+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"79'161","reported_seq":521,"reported_epoch":234,"state":"active+clean","last_fresh":"2023-09-23T20:58:26.110892+0000","last_change":"2023-09-23T20:57:25.096559+0000","last_active":"2023-09-23T20:58:26.110892+0000","last_peered":"2023-09-23T20:58:26.110892+0000","last_clean":"2023-09-23T20:58:26.110892+0000","last_became_active":"2023-09-23T20:57:15.975674+0000","last_became_peered":"2023-09-23T20:57:15.975674+0000","last_unstale":"2023-09-23T20:58:26.110892+0000","last_undegraded":"2023-09-23T20:58:26.110892+0000","last_fullsized":"2023-09-23T20:58:26.110892+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:25.096401+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:25.096401+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:05:05.448404+0000","scrub_duration":0.002859393,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":108,"reported_epoch":234,"state":"active+clean","last_fresh":"2023-09-23T20:58:26.118569+0000","last_change":"2023-09-23T20:54:31.890678+0000","last_active":"2023-09-23T20:58:26.118569+0000","last_peered":"2023-09-23T20:58:26.118569+0000","last_clean":"2023-09-23T20:58:26.118569+0000","last_became_active":"2023-09-23T20:54:31.890469+0000","last_became_peered":"2023-09-23T20:54:31.890469+0000","last_unstale":"2023-09-23T20:58:26.118569+0000","last_undegraded":"2023-09-23T20:58:26.118569+0000","last_fullsized":"2023-09-23T20:58:26.118569+0000","mapping_epoch":136,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:55:08.790146+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":176,"reported_epoch":234,"state":"active+clean","last_fresh":"2023-09-23T20:58:26.131795+0000","last_change":"2023-09-23T20:55:35.952661+0000","last_active":"2023-09-23T20:58:26.131795+0000","last_peered":"2023-09-23T20:58:26.131795+0000","last_clean":"2023-09-23T20:58:26.131795+0000","last_became_active":"2023-09-23T20:55:35.952349+0000","last_became_peered":"2023-09-23T20:55:35.952349+0000","last_unstale":"2023-09-23T20:58:26.131795+0000","last_undegraded":"2023-09-23T20:58:26.131795+0000","last_fullsized":"2023-09-23T20:58:26.131795+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:00:03.110126+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":52,"reported_epoch":234,"state":"active+clean","last_fresh":"2023-09-23T20:58:26.135528+0000","last_change":"2023-09-23T20:56:45.988607+0000","last_active":"2023-09-23T20:58:26.135528+0000","last_peered":"2023-09-23T20:58:26.135528+0000","last_clean":"2023-09-23T20:58:26.135528+0000","last_became_active":"2023-09-23T20:56:45.988206+0000","last_became_peered":"2023-09-23T20:56:45.988206+0000","last_unstale":"2023-09-23T20:58:26.135528+0000","last_undegraded":"2023-09-23T20:58:26.135528+0000","last_fullsized":"2023-09-23T20:58:26.135528+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":205,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:01:19.783688+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,4],"acting":[3,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":52,"reported_epoch":234,"state":"active+clean","last_fresh":"2023-09-23T20:58:26.118869+0000","last_change":"2023-09-23T20:56:18.139142+0000","last_active":"2023-09-23T20:58:26.118869+0000","last_peered":"2023-09-23T20:58:26.118869+0000","last_clean":"2023-09-23T20:58:26.118869+0000","last_became_active":"2023-09-23T20:56:18.138844+0000","last_became_peered":"2023-09-23T20:56:18.138844+0000","last_unstale":"2023-09-23T20:58:26.118869+0000","last_undegraded":"2023-09-23T20:58:26.118869+0000","last_fullsized":"2023-09-23T20:58:26.118869+0000","mapping_epoch":192,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":193,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:09:02.217683+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,2],"acting":[5,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.14","version":"79'22","reported_seq":139,"reported_epoch":234,"state":"active+clean","last_fresh":"2023-09-23T20:58:26.117425+0000","last_change":"2023-09-23T20:55:35.949418+0000","last_active":"2023-09-23T20:58:26.117425+0000","last_peered":"2023-09-23T20:58:26.117425+0000","last_clean":"2023-09-23T20:58:26.117425+0000","last_became_active":"2023-09-23T20:55:35.948566+0000","last_became_peered":"2023-09-23T20:55:35.948566+0000","last_unstale":"2023-09-23T20:58:26.117425+0000","last_undegraded":"2023-09-23T20:58:26.117425+0000","last_fullsized":"2023-09-23T20:58:26.117425+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:59:37.433128+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4734,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":129,"reported_epoch":234,"state":"active+clean","last_fresh":"2023-09-23T20:58:26.117762+0000","last_change":"2023-09-23T20:55:35.949178+0000","last_active":"2023-09-23T20:58:26.117762+0000","last_peered":"2023-09-23T20:58:26.117762+0000","last_clean":"2023-09-23T20:58:26.117762+0000","last_became_active":"2023-09-23T20:55:35.948795+0000","last_became_peered":"2023-09-23T20:55:35.948795+0000","last_unstale":"2023-09-23T20:58:26.117762+0000","last_undegraded":"2023-09-23T20:58:26.117762+0000","last_fullsized":"2023-09-23T20:58:26.117762+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:13:08.996932+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.11","version":"79'161","reported_seq":463,"reported_epoch":234,"state":"active+clean","last_fresh":"2023-09-23T20:58:26.125646+0000","last_change":"2023-09-23T20:57:28.756796+0000","last_active":"2023-09-23T20:58:26.125646+0000","last_peered":"2023-09-23T20:58:26.125646+0000","last_clean":"2023-09-23T20:58:26.125646+0000","last_became_active":"2023-09-23T20:57:15.980606+0000","last_became_peered":"2023-09-23T20:57:15.980606+0000","last_unstale":"2023-09-23T20:58:26.125646+0000","last_undegraded":"2023-09-23T20:58:26.125646+0000","last_fullsized":"2023-09-23T20:58:26.125646+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:28.756634+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:28.756634+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:09:19.150791+0000","scrub_duration":0.0064925119999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":155,"reported_epoch":234,"state":"active+clean","last_fresh":"2023-09-23T20:58:26.117628+0000","last_change":"2023-09-23T20:56:45.992577+0000","last_active":"2023-09-23T20:58:26.117628+0000","last_peered":"2023-09-23T20:58:26.117628+0000","last_clean":"2023-09-23T20:58:26.117628+0000","last_became_active":"2023-09-23T20:56:45.992297+0000","last_became_peered":"2023-09-23T20:56:45.992297+0000","last_unstale":"2023-09-23T20:58:26.117628+0000","last_undegraded":"2023-09-23T20:58:26.117628+0000","last_fullsized":"2023-09-23T20:58:26.117628+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":205,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:00:45.547202+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,2],"acting":[4,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":200,"reported_epoch":234,"state":"active+clean","last_fresh":"2023-09-23T20:58:26.132503+0000","last_change":"2023-09-23T20:51:10.831262+0000","last_active":"2023-09-23T20:58:26.132503+0000","last_peered":"2023-09-23T20:58:26.132503+0000","last_clean":"2023-09-23T20:58:26.132503+0000","last_became_active":"2023-09-23T20:51:10.830931+0000","last_became_peered":"2023-09-23T20:51:10.830931+0000","last_unstale":"2023-09-23T20:58:26.132503+0000","last_undegraded":"2023-09-23T20:58:26.132503+0000","last_fullsized":"2023-09-23T20:58:26.132503+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:51:54.982505+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.10","version":"79'161","reported_seq":472,"reported_epoch":234,"state":"active+clean","last_fresh":"2023-09-23T20:58:26.117631+0000","last_change":"2023-09-23T20:57:20.708003+0000","last_active":"2023-09-23T20:58:26.117631+0000","last_peered":"2023-09-23T20:58:26.117631+0000","last_clean":"2023-09-23T20:58:26.117631+0000","last_became_active":"2023-09-23T20:57:15.975302+0000","last_became_peered":"2023-09-23T20:57:15.975302+0000","last_unstale":"2023-09-23T20:58:26.117631+0000","last_undegraded":"2023-09-23T20:58:26.117631+0000","last_fullsized":"2023-09-23T20:58:26.117631+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:20.707939+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:20.707939+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:16:51.721994+0000","scrub_duration":0.0017733409999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":209,"reported_epoch":234,"state":"active+clean","last_fresh":"2023-09-23T20:58:26.126213+0000","last_change":"2023-09-23T20:57:44.979736+0000","last_active":"2023-09-23T20:58:26.126213+0000","last_peered":"2023-09-23T20:58:26.126213+0000","last_clean":"2023-09-23T20:58:26.126213+0000","last_became_active":"2023-09-23T20:57:44.979356+0000","last_became_peered":"2023-09-23T20:57:44.979356+0000","last_unstale":"2023-09-23T20:58:26.126213+0000","last_undegraded":"2023-09-23T20:58:26.126213+0000","last_fullsized":"2023-09-23T20:58:26.126213+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:14:43.848832+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":108,"reported_epoch":234,"state":"active+clean","last_fresh":"2023-09-23T20:58:26.118064+0000","last_change":"2023-09-23T20:54:31.887814+0000","last_active":"2023-09-23T20:58:26.118064+0000","last_peered":"2023-09-23T20:58:26.118064+0000","last_clean":"2023-09-23T20:58:26.118064+0000","last_became_active":"2023-09-23T20:54:31.887195+0000","last_became_peered":"2023-09-23T20:54:31.887195+0000","last_unstale":"2023-09-23T20:58:26.118064+0000","last_undegraded":"2023-09-23T20:58:26.118064+0000","last_fullsized":"2023-09-23T20:58:26.118064+0000","mapping_epoch":136,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:10:52.089553+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.13","version":"79'161","reported_seq":535,"reported_epoch":235,"state":"active+clean","last_fresh":"2023-09-23T20:58:35.201804+0000","last_change":"2023-09-23T20:57:48.262970+0000","last_active":"2023-09-23T20:58:35.201804+0000","last_peered":"2023-09-23T20:58:35.201804+0000","last_clean":"2023-09-23T20:58:35.201804+0000","last_became_active":"2023-09-23T20:57:48.262663+0000","last_became_peered":"2023-09-23T20:57:48.262663+0000","last_unstale":"2023-09-23T20:58:35.201804+0000","last_undegraded":"2023-09-23T20:58:35.201804+0000","last_fullsized":"2023-09-23T20:58:35.201804+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:36.420362+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:36.420362+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:28:28.613492+0000","scrub_duration":0.028838444000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":179,"reported_epoch":234,"state":"active+clean","last_fresh":"2023-09-23T20:58:26.119188+0000","last_change":"2023-09-23T20:56:45.991822+0000","last_active":"2023-09-23T20:58:26.119188+0000","last_peered":"2023-09-23T20:58:26.119188+0000","last_clean":"2023-09-23T20:58:26.119188+0000","last_became_active":"2023-09-23T20:56:45.991460+0000","last_became_peered":"2023-09-23T20:56:45.991460+0000","last_unstale":"2023-09-23T20:58:26.119188+0000","last_undegraded":"2023-09-23T20:58:26.119188+0000","last_fullsized":"2023-09-23T20:58:26.119188+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":205,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:20:37.503317+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":191,"reported_epoch":234,"state":"active+clean","last_fresh":"2023-09-23T20:58:26.130680+0000","last_change":"2023-09-23T20:51:10.840849+0000","last_active":"2023-09-23T20:58:26.130680+0000","last_peered":"2023-09-23T20:58:26.130680+0000","last_clean":"2023-09-23T20:58:26.130680+0000","last_became_active":"2023-09-23T20:51:10.840593+0000","last_became_peered":"2023-09-23T20:51:10.840593+0000","last_unstale":"2023-09-23T20:58:26.130680+0000","last_undegraded":"2023-09-23T20:58:26.130680+0000","last_fullsized":"2023-09-23T20:58:26.130680+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:30:52.297625+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.12","version":"79'161","reported_seq":472,"reported_epoch":234,"state":"active+clean","last_fresh":"2023-09-23T20:58:26.118602+0000","last_change":"2023-09-23T20:57:21.750765+0000","last_active":"2023-09-23T20:58:26.118602+0000","last_peered":"2023-09-23T20:58:26.118602+0000","last_clean":"2023-09-23T20:58:26.118602+0000","last_became_active":"2023-09-23T20:57:15.979195+0000","last_became_peered":"2023-09-23T20:57:15.979195+0000","last_unstale":"2023-09-23T20:58:26.118602+0000","last_undegraded":"2023-09-23T20:58:26.118602+0000","last_fullsized":"2023-09-23T20:58:26.118602+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:21.750680+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:21.750680+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:03:41.915962+0000","scrub_duration":0.0092391680000000007,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":228,"reported_epoch":235,"state":"active+clean","last_fresh":"2023-09-23T20:58:35.186173+0000","last_change":"2023-09-23T20:55:36.943969+0000","last_active":"2023-09-23T20:58:35.186173+0000","last_peered":"2023-09-23T20:58:35.186173+0000","last_clean":"2023-09-23T20:58:35.186173+0000","last_became_active":"2023-09-23T20:55:36.943580+0000","last_became_peered":"2023-09-23T20:55:36.943580+0000","last_unstale":"2023-09-23T20:58:35.186173+0000","last_undegraded":"2023-09-23T20:58:35.186173+0000","last_fullsized":"2023-09-23T20:58:35.186173+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:49:06.969396+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":23,"reported_epoch":234,"state":"active+clean","last_fresh":"2023-09-23T20:58:26.132902+0000","last_change":"2023-09-23T20:57:44.980498+0000","last_active":"2023-09-23T20:58:26.132902+0000","last_peered":"2023-09-23T20:58:26.132902+0000","last_clean":"2023-09-23T20:58:26.132902+0000","last_became_active":"2023-09-23T20:57:44.980182+0000","last_became_peered":"2023-09-23T20:57:44.980182+0000","last_unstale":"2023-09-23T20:58:26.132902+0000","last_undegraded":"2023-09-23T20:58:26.132902+0000","last_fullsized":"2023-09-23T20:58:26.132902+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:34:51.620544+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.d","version":"79'161","reported_seq":517,"reported_epoch":234,"state":"active+clean","last_fresh":"2023-09-23T20:58:26.118121+0000","last_change":"2023-09-23T20:57:31.075213+0000","last_active":"2023-09-23T20:58:26.118121+0000","last_peered":"2023-09-23T20:58:26.118121+0000","last_clean":"2023-09-23T20:58:26.118121+0000","last_became_active":"2023-09-23T20:57:15.988188+0000","last_became_peered":"2023-09-23T20:57:15.988188+0000","last_unstale":"2023-09-23T20:58:26.118121+0000","last_undegraded":"2023-09-23T20:58:26.118121+0000","last_fullsized":"2023-09-23T20:58:26.118121+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:31.075098+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:31.075098+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:41:52.142362+0000","scrub_duration":0.010304610000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.b","version":"235'7655","reported_seq":7701,"reported_epoch":235,"state":"active+clean","last_fresh":"2023-09-23T20:58:44.079913+0000","last_change":"2023-09-23T20:56:35.543994+0000","last_active":"2023-09-23T20:58:44.079913+0000","last_peered":"2023-09-23T20:58:44.079913+0000","last_clean":"2023-09-23T20:58:44.079913+0000","last_became_active":"2023-09-23T20:56:35.543758+0000","last_became_peered":"2023-09-23T20:56:35.543758+0000","last_unstale":"2023-09-23T20:58:44.079913+0000","last_undegraded":"2023-09-23T20:58:44.079913+0000","last_fullsized":"2023-09-23T20:58:44.079913+0000","mapping_epoch":199,"log_start":"202'600","ondisk_log_start":"202'600","created":199,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_clean_scrub_stamp":"2023-09-23T20:56:34.529103+0000","objects_scrubbed":0,"log_size":7055,"log_dups_size":0,"ondisk_log_size":7055,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:21:29.041484+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":329318400,"num_objects":5025,"num_object_clones":0,"num_object_copies":15075,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5025,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7655,"num_write_kb":405760,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":166,"reported_epoch":234,"state":"active+clean","last_fresh":"2023-09-23T20:58:26.118990+0000","last_change":"2023-09-23T20:56:45.991841+0000","last_active":"2023-09-23T20:58:26.118990+0000","last_peered":"2023-09-23T20:58:26.118990+0000","last_clean":"2023-09-23T20:58:26.118990+0000","last_became_active":"2023-09-23T20:56:45.991491+0000","last_became_peered":"2023-09-23T20:56:45.991491+0000","last_unstale":"2023-09-23T20:58:26.118990+0000","last_undegraded":"2023-09-23T20:58:26.118990+0000","last_fullsized":"2023-09-23T20:58:26.118990+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":205,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:30:37.405319+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":125,"reported_epoch":234,"state":"active+clean","last_fresh":"2023-09-23T20:58:26.117888+0000","last_change":"2023-09-23T20:55:35.957768+0000","last_active":"2023-09-23T20:58:26.117888+0000","last_peered":"2023-09-23T20:58:26.117888+0000","last_clean":"2023-09-23T20:58:26.117888+0000","last_became_active":"2023-09-23T20:55:35.957421+0000","last_became_peered":"2023-09-23T20:55:35.957421+0000","last_unstale":"2023-09-23T20:58:26.117888+0000","last_undegraded":"2023-09-23T20:58:26.117888+0000","last_fullsized":"2023-09-23T20:58:26.117888+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:49:41.585961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.c","version":"79'161","reported_seq":535,"reported_epoch":234,"state":"active+clean","last_fresh":"2023-09-23T20:58:26.132330+0000","last_change":"2023-09-23T20:57:22.041071+0000","last_active":"2023-09-23T20:58:26.132330+0000","last_peered":"2023-09-23T20:58:26.132330+0000","last_clean":"2023-09-23T20:58:26.132330+0000","last_became_active":"2023-09-23T20:57:15.973204+0000","last_became_peered":"2023-09-23T20:57:15.973204+0000","last_unstale":"2023-09-23T20:58:26.132330+0000","last_undegraded":"2023-09-23T20:58:26.132330+0000","last_fullsized":"2023-09-23T20:58:26.132330+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:22.040907+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:22.040907+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:47:02.019978+0000","scrub_duration":0.013021176000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.a","version":"235'7535","reported_seq":7580,"reported_epoch":235,"state":"active+clean","last_fresh":"2023-09-23T20:58:39.438863+0000","last_change":"2023-09-23T20:56:35.547120+0000","last_active":"2023-09-23T20:58:39.438863+0000","last_peered":"2023-09-23T20:58:39.438863+0000","last_clean":"2023-09-23T20:58:39.438863+0000","last_became_active":"2023-09-23T20:56:35.546838+0000","last_became_peered":"2023-09-23T20:56:35.546838+0000","last_unstale":"2023-09-23T20:58:39.438863+0000","last_undegraded":"2023-09-23T20:58:39.438863+0000","last_fullsized":"2023-09-23T20:58:39.438863+0000","mapping_epoch":199,"log_start":"205'1400","ondisk_log_start":"205'1400","created":199,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_clean_scrub_stamp":"2023-09-23T20:56:34.529103+0000","objects_scrubbed":0,"log_size":6135,"log_dups_size":0,"ondisk_log_size":6135,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:48:24.132768+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":340852736,"num_objects":5201,"num_object_clones":0,"num_object_copies":15603,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5201,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7535,"num_write_kb":407552,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,5],"acting":[3,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":153,"reported_epoch":234,"state":"active+clean","last_fresh":"2023-09-23T20:58:26.118645+0000","last_change":"2023-09-23T20:56:45.991821+0000","last_active":"2023-09-23T20:58:26.118645+0000","last_peered":"2023-09-23T20:58:26.118645+0000","last_clean":"2023-09-23T20:58:26.118645+0000","last_became_active":"2023-09-23T20:56:45.991447+0000","last_became_peered":"2023-09-23T20:56:45.991447+0000","last_unstale":"2023-09-23T20:58:26.118645+0000","last_undegraded":"2023-09-23T20:58:26.118645+0000","last_fullsized":"2023-09-23T20:58:26.118645+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":205,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:14:25.525222+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":156,"reported_epoch":234,"state":"active+clean","last_fresh":"2023-09-23T20:58:26.119448+0000","last_change":"2023-09-23T20:57:44.982059+0000","last_active":"2023-09-23T20:58:26.119448+0000","last_peered":"2023-09-23T20:58:26.119448+0000","last_clean":"2023-09-23T20:58:26.119448+0000","last_became_active":"2023-09-23T20:57:44.981742+0000","last_became_peered":"2023-09-23T20:57:44.981742+0000","last_unstale":"2023-09-23T20:58:26.119448+0000","last_undegraded":"2023-09-23T20:58:26.119448+0000","last_fullsized":"2023-09-23T20:58:26.119448+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:41:24.042916+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.f","version":"79'161","reported_seq":540,"reported_epoch":235,"state":"active+clean","last_fresh":"2023-09-23T20:58:35.190624+0000","last_change":"2023-09-23T20:57:27.880841+0000","last_active":"2023-09-23T20:58:35.190624+0000","last_peered":"2023-09-23T20:58:35.190624+0000","last_clean":"2023-09-23T20:58:35.190624+0000","last_became_active":"2023-09-23T20:57:15.970925+0000","last_became_peered":"2023-09-23T20:57:15.970925+0000","last_unstale":"2023-09-23T20:58:35.190624+0000","last_undegraded":"2023-09-23T20:58:35.190624+0000","last_fullsized":"2023-09-23T20:58:35.190624+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:27.880776+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:27.880776+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:15:13.438014+0000","scrub_duration":0.0014004899999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.9","version":"235'7612","reported_seq":7657,"reported_epoch":235,"state":"active+clean","last_fresh":"2023-09-23T20:58:41.348349+0000","last_change":"2023-09-23T20:56:35.549387+0000","last_active":"2023-09-23T20:58:41.348349+0000","last_peered":"2023-09-23T20:58:41.348349+0000","last_clean":"2023-09-23T20:58:41.348349+0000","last_became_active":"2023-09-23T20:56:35.549087+0000","last_became_peered":"2023-09-23T20:56:35.549087+0000","last_unstale":"2023-09-23T20:58:41.348349+0000","last_undegraded":"2023-09-23T20:58:41.348349+0000","last_fullsized":"2023-09-23T20:58:41.348349+0000","mapping_epoch":199,"log_start":"202'200","ondisk_log_start":"202'200","created":199,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_clean_scrub_stamp":"2023-09-23T20:56:34.529103+0000","objects_scrubbed":0,"log_size":7412,"log_dups_size":0,"ondisk_log_size":7412,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:04:51.547362+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":333840384,"num_objects":5094,"num_object_clones":0,"num_object_copies":15282,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5094,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7612,"num_write_kb":406592,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":139,"reported_epoch":234,"state":"active+clean","last_fresh":"2023-09-23T20:58:26.118020+0000","last_change":"2023-09-23T20:56:45.989153+0000","last_active":"2023-09-23T20:58:26.118020+0000","last_peered":"2023-09-23T20:58:26.118020+0000","last_clean":"2023-09-23T20:58:26.118020+0000","last_became_active":"2023-09-23T20:56:45.988800+0000","last_became_peered":"2023-09-23T20:56:45.988800+0000","last_unstale":"2023-09-23T20:58:26.118020+0000","last_undegraded":"2023-09-23T20:58:26.118020+0000","last_fullsized":"2023-09-23T20:58:26.118020+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":205,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:04:41.130194+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":175,"reported_epoch":235,"state":"active+clean","last_fresh":"2023-09-23T20:58:35.186301+0000","last_change":"2023-09-23T20:55:35.965427+0000","last_active":"2023-09-23T20:58:35.186301+0000","last_peered":"2023-09-23T20:58:35.186301+0000","last_clean":"2023-09-23T20:58:35.186301+0000","last_became_active":"2023-09-23T20:55:35.963969+0000","last_became_peered":"2023-09-23T20:55:35.963969+0000","last_unstale":"2023-09-23T20:58:35.186301+0000","last_undegraded":"2023-09-23T20:58:35.186301+0000","last_fullsized":"2023-09-23T20:58:35.186301+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:38:40.612083+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.e","version":"79'161","reported_seq":446,"reported_epoch":234,"state":"active+clean","last_fresh":"2023-09-23T20:58:26.118507+0000","last_change":"2023-09-23T20:57:34.132738+0000","last_active":"2023-09-23T20:58:26.118507+0000","last_peered":"2023-09-23T20:58:26.118507+0000","last_clean":"2023-09-23T20:58:26.118507+0000","last_became_active":"2023-09-23T20:57:15.974318+0000","last_became_peered":"2023-09-23T20:57:15.974318+0000","last_unstale":"2023-09-23T20:58:26.118507+0000","last_undegraded":"2023-09-23T20:58:26.118507+0000","last_fullsized":"2023-09-23T20:58:26.118507+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:34.132660+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:34.132660+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:07:07.739438+0000","scrub_duration":0.005229694,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.8","version":"235'7691","reported_seq":7736,"reported_epoch":235,"state":"active+clean","last_fresh":"2023-09-23T20:58:41.317992+0000","last_change":"2023-09-23T20:56:35.548119+0000","last_active":"2023-09-23T20:58:41.317992+0000","last_peered":"2023-09-23T20:58:41.317992+0000","last_clean":"2023-09-23T20:58:41.317992+0000","last_became_active":"2023-09-23T20:56:35.547914+0000","last_became_peered":"2023-09-23T20:56:35.547914+0000","last_unstale":"2023-09-23T20:58:41.317992+0000","last_undegraded":"2023-09-23T20:58:41.317992+0000","last_fullsized":"2023-09-23T20:58:41.317992+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":199,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_clean_scrub_stamp":"2023-09-23T20:56:34.529103+0000","objects_scrubbed":0,"log_size":7691,"log_dups_size":0,"ondisk_log_size":7691,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:42:54.255023+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":338362368,"num_objects":5163,"num_object_clones":0,"num_object_copies":15489,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5163,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7691,"num_write_kb":411328,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":125,"reported_epoch":234,"state":"active+clean","last_fresh":"2023-09-23T20:58:26.117792+0000","last_change":"2023-09-23T20:55:35.962005+0000","last_active":"2023-09-23T20:58:26.117792+0000","last_peered":"2023-09-23T20:58:26.117792+0000","last_clean":"2023-09-23T20:58:26.117792+0000","last_became_active":"2023-09-23T20:55:35.961695+0000","last_became_peered":"2023-09-23T20:55:35.961695+0000","last_unstale":"2023-09-23T20:58:26.117792+0000","last_undegraded":"2023-09-23T20:58:26.117792+0000","last_fullsized":"2023-09-23T20:58:26.117792+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:28:22.876201+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":223,"reported_epoch":234,"state":"active+clean","last_fresh":"2023-09-23T20:58:31.339098+0000","last_change":"2023-09-23T20:54:13.928715+0000","last_active":"2023-09-23T20:58:31.339098+0000","last_peered":"2023-09-23T20:58:31.339098+0000","last_clean":"2023-09-23T20:58:31.339098+0000","last_became_active":"2023-09-23T20:54:13.928200+0000","last_became_peered":"2023-09-23T20:54:13.928200+0000","last_unstale":"2023-09-23T20:58:31.339098+0000","last_undegraded":"2023-09-23T20:58:31.339098+0000","last_fullsized":"2023-09-23T20:58:31.339098+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:41:16.927513+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.f","version":"235'7676","reported_seq":7721,"reported_epoch":235,"state":"active+clean","last_fresh":"2023-09-23T20:58:41.373780+0000","last_change":"2023-09-23T20:56:35.545128+0000","last_active":"2023-09-23T20:58:41.373780+0000","last_peered":"2023-09-23T20:58:41.373780+0000","last_clean":"2023-09-23T20:58:41.373780+0000","last_became_active":"2023-09-23T20:56:35.544838+0000","last_became_peered":"2023-09-23T20:56:35.544838+0000","last_unstale":"2023-09-23T20:58:41.373780+0000","last_undegraded":"2023-09-23T20:58:41.373780+0000","last_fullsized":"2023-09-23T20:58:41.373780+0000","mapping_epoch":199,"log_start":"202'300","ondisk_log_start":"202'300","created":199,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_clean_scrub_stamp":"2023-09-23T20:56:34.529103+0000","objects_scrubbed":0,"log_size":7376,"log_dups_size":0,"ondisk_log_size":7376,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:30:43.388717+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":340131840,"num_objects":5190,"num_object_clones":0,"num_object_copies":15570,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5190,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7676,"num_write_kb":411712,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.9","version":"79'161","reported_seq":607,"reported_epoch":234,"state":"active+clean","last_fresh":"2023-09-23T20:58:26.131716+0000","last_change":"2023-09-23T20:56:46.921939+0000","last_active":"2023-09-23T20:58:26.131716+0000","last_peered":"2023-09-23T20:58:26.131716+0000","last_clean":"2023-09-23T20:58:26.131716+0000","last_became_active":"2023-09-23T20:56:45.989908+0000","last_became_peered":"2023-09-23T20:56:45.989908+0000","last_unstale":"2023-09-23T20:58:26.131716+0000","last_undegraded":"2023-09-23T20:58:26.131716+0000","last_fullsized":"2023-09-23T20:58:26.131716+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":205,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:56:46.921739+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:56:46.921739+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:20:42.635110+0000","scrub_duration":0.0071954109999999996,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":224,"reported_epoch":234,"state":"active+clean","last_fresh":"2023-09-23T20:58:26.133847+0000","last_change":"2023-09-23T20:54:13.961788+0000","last_active":"2023-09-23T20:58:26.133847+0000","last_peered":"2023-09-23T20:58:26.133847+0000","last_clean":"2023-09-23T20:58:26.133847+0000","last_became_active":"2023-09-23T20:54:13.940937+0000","last_became_peered":"2023-09-23T20:54:13.940937+0000","last_unstale":"2023-09-23T20:58:26.133847+0000","last_undegraded":"2023-09-23T20:58:26.133847+0000","last_fullsized":"2023-09-23T20:58:26.133847+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:11:56.785299+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":129,"reported_epoch":234,"state":"active+clean","last_fresh":"2023-09-23T20:58:26.117626+0000","last_change":"2023-09-23T20:55:35.949670+0000","last_active":"2023-09-23T20:58:26.117626+0000","last_peered":"2023-09-23T20:58:26.117626+0000","last_clean":"2023-09-23T20:58:26.117626+0000","last_became_active":"2023-09-23T20:55:35.949140+0000","last_became_peered":"2023-09-23T20:55:35.949140+0000","last_unstale":"2023-09-23T20:58:26.117626+0000","last_undegraded":"2023-09-23T20:58:26.117626+0000","last_fullsized":"2023-09-23T20:58:26.117626+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:55:59.340303+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.e","version":"235'7630","reported_seq":7675,"reported_epoch":235,"state":"active+clean","last_fresh":"2023-09-23T20:58:41.392197+0000","last_change":"2023-09-23T20:56:35.547184+0000","last_active":"2023-09-23T20:58:41.392197+0000","last_peered":"2023-09-23T20:58:41.392197+0000","last_clean":"2023-09-23T20:58:41.392197+0000","last_became_active":"2023-09-23T20:56:35.546912+0000","last_became_peered":"2023-09-23T20:56:35.546912+0000","last_unstale":"2023-09-23T20:58:41.392197+0000","last_undegraded":"2023-09-23T20:58:41.392197+0000","last_fullsized":"2023-09-23T20:58:41.392197+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":199,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_clean_scrub_stamp":"2023-09-23T20:56:34.529103+0000","objects_scrubbed":0,"log_size":7630,"log_dups_size":0,"ondisk_log_size":7630,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:57:07.422792+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":337510400,"num_objects":5150,"num_object_clones":0,"num_object_copies":15450,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5150,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7630,"num_write_kb":408960,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,0],"acting":[2,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.8","version":"79'161","reported_seq":478,"reported_epoch":234,"state":"active+clean","last_fresh":"2023-09-23T20:58:26.117818+0000","last_change":"2023-09-23T20:56:57.505737+0000","last_active":"2023-09-23T20:58:26.117818+0000","last_peered":"2023-09-23T20:58:26.117818+0000","last_clean":"2023-09-23T20:58:26.117818+0000","last_became_active":"2023-09-23T20:56:04.982345+0000","last_became_peered":"2023-09-23T20:56:04.982345+0000","last_unstale":"2023-09-23T20:58:26.117818+0000","last_undegraded":"2023-09-23T20:58:26.117818+0000","last_fullsized":"2023-09-23T20:58:26.117818+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:56:57.505663+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:56:57.505663+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:34:30.109724+0000","scrub_duration":0.026284108,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":198,"reported_epoch":235,"state":"active+clean","last_fresh":"2023-09-23T20:58:35.186502+0000","last_change":"2023-09-23T20:55:35.959942+0000","last_active":"2023-09-23T20:58:35.186502+0000","last_peered":"2023-09-23T20:58:35.186502+0000","last_clean":"2023-09-23T20:58:35.186502+0000","last_became_active":"2023-09-23T20:55:35.957071+0000","last_became_peered":"2023-09-23T20:55:35.957071+0000","last_unstale":"2023-09-23T20:58:35.186502+0000","last_undegraded":"2023-09-23T20:58:35.186502+0000","last_fullsized":"2023-09-23T20:58:35.186502+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:44:30.308036+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":112,"reported_epoch":234,"state":"active+clean","last_fresh":"2023-09-23T20:58:26.118439+0000","last_change":"2023-09-23T20:54:13.951459+0000","last_active":"2023-09-23T20:58:26.118439+0000","last_peered":"2023-09-23T20:58:26.118439+0000","last_clean":"2023-09-23T20:58:26.118439+0000","last_became_active":"2023-09-23T20:54:13.948409+0000","last_became_peered":"2023-09-23T20:54:13.948409+0000","last_unstale":"2023-09-23T20:58:26.118439+0000","last_undegraded":"2023-09-23T20:58:26.118439+0000","last_fullsized":"2023-09-23T20:58:26.118439+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:38:10.148061+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.b","version":"79'161","reported_seq":518,"reported_epoch":235,"state":"active+clean","last_fresh":"2023-09-23T20:58:35.201881+0000","last_change":"2023-09-23T20:57:27.320172+0000","last_active":"2023-09-23T20:58:35.201881+0000","last_peered":"2023-09-23T20:58:35.201881+0000","last_clean":"2023-09-23T20:58:35.201881+0000","last_became_active":"2023-09-23T20:57:16.009630+0000","last_became_peered":"2023-09-23T20:57:16.009630+0000","last_unstale":"2023-09-23T20:58:35.201881+0000","last_undegraded":"2023-09-23T20:58:35.201881+0000","last_fullsized":"2023-09-23T20:58:35.201881+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:27.320022+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:27.320022+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:31:35.221558+0000","scrub_duration":0.0043894679999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,5],"acting":[1,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.d","version":"235'7778","reported_seq":7824,"reported_epoch":235,"state":"active+clean","last_fresh":"2023-09-23T20:58:44.009845+0000","last_change":"2023-09-23T20:56:35.544132+0000","last_active":"2023-09-23T20:58:44.009845+0000","last_peered":"2023-09-23T20:58:44.009845+0000","last_clean":"2023-09-23T20:58:44.009845+0000","last_became_active":"2023-09-23T20:56:35.543836+0000","last_became_peered":"2023-09-23T20:56:35.543836+0000","last_unstale":"2023-09-23T20:58:44.009845+0000","last_undegraded":"2023-09-23T20:58:44.009845+0000","last_fullsized":"2023-09-23T20:58:44.009845+0000","mapping_epoch":199,"log_start":"202'400","ondisk_log_start":"202'400","created":199,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_clean_scrub_stamp":"2023-09-23T20:56:34.529103+0000","objects_scrubbed":0,"log_size":7378,"log_dups_size":0,"ondisk_log_size":7378,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:27:14.636991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":328400920,"num_objects":5012,"num_object_clones":0,"num_object_copies":15036,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5012,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":7778,"num_write_kb":409217,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":112,"reported_epoch":234,"state":"active+clean","last_fresh":"2023-09-23T20:58:26.119271+0000","last_change":"2023-09-23T20:54:13.950254+0000","last_active":"2023-09-23T20:58:26.119271+0000","last_peered":"2023-09-23T20:58:26.119271+0000","last_clean":"2023-09-23T20:58:26.119271+0000","last_became_active":"2023-09-23T20:54:13.949934+0000","last_became_peered":"2023-09-23T20:54:13.949934+0000","last_unstale":"2023-09-23T20:58:26.119271+0000","last_undegraded":"2023-09-23T20:58:26.119271+0000","last_fullsized":"2023-09-23T20:58:26.119271+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:30:32.847167+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":129,"reported_epoch":234,"state":"active+clean","last_fresh":"2023-09-23T20:58:26.118721+0000","last_change":"2023-09-23T20:55:35.956153+0000","last_active":"2023-09-23T20:58:26.118721+0000","last_peered":"2023-09-23T20:58:26.118721+0000","last_clean":"2023-09-23T20:58:26.118721+0000","last_became_active":"2023-09-23T20:55:35.955630+0000","last_became_peered":"2023-09-23T20:55:35.955630+0000","last_unstale":"2023-09-23T20:58:26.118721+0000","last_undegraded":"2023-09-23T20:58:26.118721+0000","last_fullsized":"2023-09-23T20:58:26.118721+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:45:20.282929+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.c","version":"235'7775","reported_seq":7830,"reported_epoch":235,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T20:58:44.029342+0000","last_change":"2023-09-23T20:57:46.463005+0000","last_active":"2023-09-23T20:58:44.029342+0000","last_peered":"2023-09-23T20:58:44.029342+0000","last_clean":"2023-09-23T20:57:44.015119+0000","last_became_active":"2023-09-23T20:57:46.448702+0000","last_became_peered":"2023-09-23T20:57:46.448702+0000","last_unstale":"2023-09-23T20:58:44.029342+0000","last_undegraded":"2023-09-23T20:57:46.010874+0000","last_fullsized":"2023-09-23T20:57:45.978434+0000","mapping_epoch":223,"log_start":"202'400","ondisk_log_start":"202'400","created":199,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_clean_scrub_stamp":"2023-09-23T20:56:34.529103+0000","objects_scrubbed":0,"log_size":7375,"log_dups_size":0,"ondisk_log_size":7375,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:08:34.180651+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":329842688,"num_objects":5033,"num_object_clones":0,"num_object_copies":15099,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5473,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5033,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7775,"num_write_kb":409856,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,7],"avail_no_missing":["6","2","7"],"object_location_counts":[{"shards":"2,6,7","objects":4102},{"shards":"6,7","objects":-440}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.a","version":"79'161","reported_seq":478,"reported_epoch":234,"state":"active+clean","last_fresh":"2023-09-23T20:58:26.119046+0000","last_change":"2023-09-23T20:56:44.487778+0000","last_active":"2023-09-23T20:58:26.119046+0000","last_peered":"2023-09-23T20:58:26.119046+0000","last_clean":"2023-09-23T20:58:26.119046+0000","last_became_active":"2023-09-23T20:56:05.006740+0000","last_became_peered":"2023-09-23T20:56:05.006740+0000","last_unstale":"2023-09-23T20:58:26.119046+0000","last_undegraded":"2023-09-23T20:58:26.119046+0000","last_fullsized":"2023-09-23T20:58:26.119046+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:56:44.487678+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:56:44.487678+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:04:50.086749+0000","scrub_duration":0.0017408899999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":23,"reported_epoch":234,"state":"active+clean","last_fresh":"2023-09-23T20:58:26.130665+0000","last_change":"2023-09-23T20:57:44.982513+0000","last_active":"2023-09-23T20:58:26.130665+0000","last_peered":"2023-09-23T20:58:26.130665+0000","last_clean":"2023-09-23T20:58:26.130665+0000","last_became_active":"2023-09-23T20:57:44.982112+0000","last_became_peered":"2023-09-23T20:57:44.982112+0000","last_unstale":"2023-09-23T20:58:26.130665+0000","last_undegraded":"2023-09-23T20:58:26.130665+0000","last_fullsized":"2023-09-23T20:58:26.130665+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:09:05.630041+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":205,"reported_epoch":235,"state":"active+clean","last_fresh":"2023-09-23T20:58:35.200643+0000","last_change":"2023-09-23T20:54:13.940748+0000","last_active":"2023-09-23T20:58:35.200643+0000","last_peered":"2023-09-23T20:58:35.200643+0000","last_clean":"2023-09-23T20:58:35.200643+0000","last_became_active":"2023-09-23T20:54:13.940419+0000","last_became_peered":"2023-09-23T20:54:13.940419+0000","last_unstale":"2023-09-23T20:58:35.200643+0000","last_undegraded":"2023-09-23T20:58:35.200643+0000","last_fullsized":"2023-09-23T20:58:35.200643+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:15:01.923816+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.1","version":"235'7722","reported_seq":7779,"reported_epoch":235,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T20:58:42.158081+0000","last_change":"2023-09-23T20:57:46.483248+0000","last_active":"2023-09-23T20:58:42.158081+0000","last_peered":"2023-09-23T20:58:42.158081+0000","last_clean":"2023-09-23T20:57:44.047957+0000","last_became_active":"2023-09-23T20:57:46.475568+0000","last_became_peered":"2023-09-23T20:57:46.475568+0000","last_unstale":"2023-09-23T20:58:42.158081+0000","last_undegraded":"2023-09-23T20:57:46.083591+0000","last_fullsized":"2023-09-23T20:57:45.978436+0000","mapping_epoch":223,"log_start":"203'1000","ondisk_log_start":"203'1000","created":199,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_clean_scrub_stamp":"2023-09-23T20:56:34.529103+0000","objects_scrubbed":0,"log_size":6722,"log_dups_size":0,"ondisk_log_size":6722,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:22:45.067269+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":331874304,"num_objects":5064,"num_object_clones":0,"num_object_copies":15192,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5470,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5064,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7722,"num_write_kb":409152,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,6],"avail_no_missing":["4","1","6"],"object_location_counts":[{"shards":"1,4,6","objects":4141},{"shards":"4,6","objects":-406}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.7","version":"79'161","reported_seq":541,"reported_epoch":235,"state":"active+clean","last_fresh":"2023-09-23T20:58:35.190913+0000","last_change":"2023-09-23T20:57:48.243572+0000","last_active":"2023-09-23T20:58:35.190913+0000","last_peered":"2023-09-23T20:58:35.190913+0000","last_clean":"2023-09-23T20:58:35.190913+0000","last_became_active":"2023-09-23T20:57:48.243229+0000","last_became_peered":"2023-09-23T20:57:48.243229+0000","last_unstale":"2023-09-23T20:58:35.190913+0000","last_undegraded":"2023-09-23T20:58:35.190913+0000","last_fullsized":"2023-09-23T20:58:35.190913+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":3,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:25.916867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:25.916867+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:23:19.385831+0000","scrub_duration":0.001665323,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":211,"reported_epoch":235,"state":"active+clean","last_fresh":"2023-09-23T20:58:35.201891+0000","last_change":"2023-09-23T20:50:05.867130+0000","last_active":"2023-09-23T20:58:35.201891+0000","last_peered":"2023-09-23T20:58:35.201891+0000","last_clean":"2023-09-23T20:58:35.201891+0000","last_became_active":"2023-09-23T20:50:05.866758+0000","last_became_peered":"2023-09-23T20:50:05.866758+0000","last_unstale":"2023-09-23T20:58:35.201891+0000","last_undegraded":"2023-09-23T20:58:35.201891+0000","last_fullsized":"2023-09-23T20:58:35.201891+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:55:07.548209+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":108,"reported_epoch":234,"state":"active+clean","last_fresh":"2023-09-23T20:58:26.118103+0000","last_change":"2023-09-23T20:54:31.887604+0000","last_active":"2023-09-23T20:58:26.118103+0000","last_peered":"2023-09-23T20:58:26.118103+0000","last_clean":"2023-09-23T20:58:26.118103+0000","last_became_active":"2023-09-23T20:54:31.886752+0000","last_became_peered":"2023-09-23T20:54:31.886752+0000","last_unstale":"2023-09-23T20:58:26.118103+0000","last_undegraded":"2023-09-23T20:58:26.118103+0000","last_fullsized":"2023-09-23T20:58:26.118103+0000","mapping_epoch":136,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:13:58.872176+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":209,"reported_epoch":235,"state":"active+clean","last_fresh":"2023-09-23T20:58:35.209727+0000","last_change":"2023-09-23T20:54:31.899082+0000","last_active":"2023-09-23T20:58:35.209727+0000","last_peered":"2023-09-23T20:58:35.209727+0000","last_clean":"2023-09-23T20:58:35.209727+0000","last_became_active":"2023-09-23T20:54:31.898521+0000","last_became_peered":"2023-09-23T20:54:31.898521+0000","last_unstale":"2023-09-23T20:58:35.209727+0000","last_undegraded":"2023-09-23T20:58:35.209727+0000","last_fullsized":"2023-09-23T20:58:35.209727+0000","mapping_epoch":136,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:25:36.059711+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.7","version":"235'7761","reported_seq":7806,"reported_epoch":236,"state":"active+clean","last_fresh":"2023-09-23T20:58:44.049826+0000","last_change":"2023-09-23T20:56:35.541695+0000","last_active":"2023-09-23T20:58:44.049826+0000","last_peered":"2023-09-23T20:58:44.049826+0000","last_clean":"2023-09-23T20:58:44.049826+0000","last_became_active":"2023-09-23T20:56:35.541392+0000","last_became_peered":"2023-09-23T20:56:35.541392+0000","last_unstale":"2023-09-23T20:58:44.049826+0000","last_undegraded":"2023-09-23T20:58:44.049826+0000","last_fullsized":"2023-09-23T20:58:44.049826+0000","mapping_epoch":199,"log_start":"202'400","ondisk_log_start":"202'400","created":199,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_clean_scrub_stamp":"2023-09-23T20:56:34.529103+0000","objects_scrubbed":0,"log_size":7361,"log_dups_size":0,"ondisk_log_size":7361,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:57:22.478253+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":330891264,"num_objects":5049,"num_object_clones":0,"num_object_copies":15147,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5049,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7761,"num_write_kb":409920,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.1","version":"79'161","reported_seq":498,"reported_epoch":234,"state":"active+clean","last_fresh":"2023-09-23T20:58:26.117501+0000","last_change":"2023-09-23T20:57:19.669346+0000","last_active":"2023-09-23T20:58:26.117501+0000","last_peered":"2023-09-23T20:58:26.117501+0000","last_clean":"2023-09-23T20:58:26.117501+0000","last_became_active":"2023-09-23T20:57:08.246046+0000","last_became_peered":"2023-09-23T20:57:08.246046+0000","last_unstale":"2023-09-23T20:58:26.117501+0000","last_undegraded":"2023-09-23T20:58:26.117501+0000","last_fullsized":"2023-09-23T20:58:26.117501+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:19.669259+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:19.669259+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:14:03.435531+0000","scrub_duration":0.0045759559999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":125,"reported_epoch":234,"state":"active+clean","last_fresh":"2023-09-23T20:58:26.118742+0000","last_change":"2023-09-23T20:55:35.959514+0000","last_active":"2023-09-23T20:58:26.118742+0000","last_peered":"2023-09-23T20:58:26.118742+0000","last_clean":"2023-09-23T20:58:26.118742+0000","last_became_active":"2023-09-23T20:55:35.959244+0000","last_became_peered":"2023-09-23T20:55:35.959244+0000","last_unstale":"2023-09-23T20:58:26.118742+0000","last_undegraded":"2023-09-23T20:58:26.118742+0000","last_fullsized":"2023-09-23T20:58:26.118742+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:47:32.665316+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":129,"reported_epoch":234,"state":"active+clean","last_fresh":"2023-09-23T20:58:26.117897+0000","last_change":"2023-09-23T20:55:35.949000+0000","last_active":"2023-09-23T20:58:26.117897+0000","last_peered":"2023-09-23T20:58:26.117897+0000","last_clean":"2023-09-23T20:58:26.117897+0000","last_became_active":"2023-09-23T20:55:35.948410+0000","last_became_peered":"2023-09-23T20:55:35.948410+0000","last_unstale":"2023-09-23T20:58:26.117897+0000","last_undegraded":"2023-09-23T20:58:26.117897+0000","last_fullsized":"2023-09-23T20:58:26.117897+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:26:50.477289+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":203,"reported_epoch":235,"state":"active+clean","last_fresh":"2023-09-23T20:58:35.201663+0000","last_change":"2023-09-23T20:51:10.829889+0000","last_active":"2023-09-23T20:58:35.201663+0000","last_peered":"2023-09-23T20:58:35.201663+0000","last_clean":"2023-09-23T20:58:35.201663+0000","last_became_active":"2023-09-23T20:51:10.829533+0000","last_became_peered":"2023-09-23T20:51:10.829533+0000","last_unstale":"2023-09-23T20:58:35.201663+0000","last_undegraded":"2023-09-23T20:58:35.201663+0000","last_fullsized":"2023-09-23T20:58:35.201663+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:43:04.603460+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.4","version":"235'7595","reported_seq":7640,"reported_epoch":235,"state":"active+clean","last_fresh":"2023-09-23T20:58:41.395227+0000","last_change":"2023-09-23T20:56:35.549909+0000","last_active":"2023-09-23T20:58:41.395227+0000","last_peered":"2023-09-23T20:58:41.395227+0000","last_clean":"2023-09-23T20:58:41.395227+0000","last_became_active":"2023-09-23T20:56:35.549618+0000","last_became_peered":"2023-09-23T20:56:35.549618+0000","last_unstale":"2023-09-23T20:58:41.395227+0000","last_undegraded":"2023-09-23T20:58:41.395227+0000","last_fullsized":"2023-09-23T20:58:41.395227+0000","mapping_epoch":199,"log_start":"202'200","ondisk_log_start":"202'200","created":199,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_clean_scrub_stamp":"2023-09-23T20:56:34.529103+0000","objects_scrubbed":0,"log_size":7395,"log_dups_size":0,"ondisk_log_size":7395,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:24:25.918475+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":336003072,"num_objects":5127,"num_object_clones":0,"num_object_copies":15381,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5127,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7595,"num_write_kb":407104,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.2","version":"79'161","reported_seq":464,"reported_epoch":234,"state":"active+clean","last_fresh":"2023-09-23T20:58:26.126129+0000","last_change":"2023-09-23T20:57:24.802586+0000","last_active":"2023-09-23T20:58:26.126129+0000","last_peered":"2023-09-23T20:58:26.126129+0000","last_clean":"2023-09-23T20:58:26.126129+0000","last_became_active":"2023-09-23T20:57:08.236114+0000","last_became_peered":"2023-09-23T20:57:08.236114+0000","last_unstale":"2023-09-23T20:58:26.126129+0000","last_undegraded":"2023-09-23T20:58:26.126129+0000","last_fullsized":"2023-09-23T20:58:26.126129+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:24.802451+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:24.802451+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:06:22.309980+0000","scrub_duration":0.0090861250000000005,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,4],"acting":[0,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":220,"reported_epoch":234,"state":"active+clean","last_fresh":"2023-09-23T20:58:26.110601+0000","last_change":"2023-09-23T20:51:10.838054+0000","last_active":"2023-09-23T20:58:26.110601+0000","last_peered":"2023-09-23T20:58:26.110601+0000","last_clean":"2023-09-23T20:58:26.110601+0000","last_became_active":"2023-09-23T20:51:10.837750+0000","last_became_peered":"2023-09-23T20:51:10.837750+0000","last_unstale":"2023-09-23T20:58:26.110601+0000","last_undegraded":"2023-09-23T20:58:26.110601+0000","last_fullsized":"2023-09-23T20:58:26.110601+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:15:51.648085+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":203,"reported_epoch":235,"state":"active+clean","last_fresh":"2023-09-23T20:58:35.200936+0000","last_change":"2023-09-23T20:55:35.951146+0000","last_active":"2023-09-23T20:58:35.200936+0000","last_peered":"2023-09-23T20:58:35.200936+0000","last_clean":"2023-09-23T20:58:35.200936+0000","last_became_active":"2023-09-23T20:55:35.950874+0000","last_became_peered":"2023-09-23T20:55:35.950874+0000","last_unstale":"2023-09-23T20:58:35.200936+0000","last_undegraded":"2023-09-23T20:58:35.200936+0000","last_fullsized":"2023-09-23T20:58:35.200936+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:58:13.957677+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":224,"reported_epoch":236,"state":"active+clean","last_fresh":"2023-09-23T20:58:44.083623+0000","last_change":"2023-09-23T20:54:31.898870+0000","last_active":"2023-09-23T20:58:44.083623+0000","last_peered":"2023-09-23T20:58:44.083623+0000","last_clean":"2023-09-23T20:58:44.083623+0000","last_became_active":"2023-09-23T20:54:31.898467+0000","last_became_peered":"2023-09-23T20:54:31.898467+0000","last_unstale":"2023-09-23T20:58:44.083623+0000","last_undegraded":"2023-09-23T20:58:44.083623+0000","last_fullsized":"2023-09-23T20:58:44.083623+0000","mapping_epoch":136,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:56:22.483408+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.6","version":"235'7506","reported_seq":9908,"reported_epoch":235,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-09-23T20:58:41.405265+0000","last_change":"2023-09-23T20:58:39.453197+0000","last_active":"2023-09-23T20:58:41.405265+0000","last_peered":"2023-09-23T20:58:41.405265+0000","last_clean":"2023-09-23T20:57:44.031597+0000","last_became_active":"2023-09-23T20:57:46.443927+0000","last_became_peered":"2023-09-23T20:57:46.443927+0000","last_unstale":"2023-09-23T20:58:41.405265+0000","last_undegraded":"2023-09-23T20:58:39.450464+0000","last_fullsized":"2023-09-23T20:57:45.977227+0000","mapping_epoch":223,"log_start":"202'100","ondisk_log_start":"202'100","created":199,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_clean_scrub_stamp":"2023-09-23T20:56:34.529103+0000","objects_scrubbed":0,"log_size":7406,"log_dups_size":0,"ondisk_log_size":7406,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:37:23.911131+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":333053952,"num_objects":5082,"num_object_clones":0,"num_object_copies":15246,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":38,"num_objects_misplaced":4186,"num_objects_unfound":0,"num_objects_dirty":5082,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7506,"num_write_kb":402816,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1174,"num_bytes_recovered":76939264,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7],"avail_no_missing":["0","5","7"],"object_location_counts":[{"shards":"0,5,7","objects":4186},{"shards":"0,7","objects":858}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":108,"reported_epoch":234,"state":"active+clean","last_fresh":"2023-09-23T20:58:26.119351+0000","last_change":"2023-09-23T20:54:31.883504+0000","last_active":"2023-09-23T20:58:26.119351+0000","last_peered":"2023-09-23T20:58:26.119351+0000","last_clean":"2023-09-23T20:58:26.119351+0000","last_became_active":"2023-09-23T20:54:31.883172+0000","last_became_peered":"2023-09-23T20:54:31.883172+0000","last_unstale":"2023-09-23T20:58:26.119351+0000","last_undegraded":"2023-09-23T20:58:26.119351+0000","last_fullsized":"2023-09-23T20:58:26.119351+0000","mapping_epoch":136,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:45:40.289120+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"79'161","reported_seq":464,"reported_epoch":234,"state":"active+clean","last_fresh":"2023-09-23T20:58:26.125626+0000","last_change":"2023-09-23T20:57:20.738417+0000","last_active":"2023-09-23T20:58:26.125626+0000","last_peered":"2023-09-23T20:58:26.125626+0000","last_clean":"2023-09-23T20:58:26.125626+0000","last_became_active":"2023-09-23T20:57:08.236842+0000","last_became_peered":"2023-09-23T20:57:08.236842+0000","last_unstale":"2023-09-23T20:58:26.125626+0000","last_undegraded":"2023-09-23T20:58:26.125626+0000","last_fullsized":"2023-09-23T20:58:26.125626+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":0,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:20.738273+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:20.738273+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:18:28.327990+0000","scrub_duration":0.0088336339999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":212,"reported_epoch":234,"state":"active+clean","last_fresh":"2023-09-23T20:58:26.132830+0000","last_change":"2023-09-23T20:55:35.950777+0000","last_active":"2023-09-23T20:58:26.132830+0000","last_peered":"2023-09-23T20:58:26.132830+0000","last_clean":"2023-09-23T20:58:26.132830+0000","last_became_active":"2023-09-23T20:55:35.950427+0000","last_became_peered":"2023-09-23T20:55:35.950427+0000","last_unstale":"2023-09-23T20:58:26.132830+0000","last_undegraded":"2023-09-23T20:58:26.132830+0000","last_fullsized":"2023-09-23T20:58:26.132830+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:20:10.371751+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":180,"reported_epoch":235,"state":"active+clean","last_fresh":"2023-09-23T20:58:35.187409+0000","last_change":"2023-09-23T20:55:35.978140+0000","last_active":"2023-09-23T20:58:35.187409+0000","last_peered":"2023-09-23T20:58:35.187409+0000","last_clean":"2023-09-23T20:58:35.187409+0000","last_became_active":"2023-09-23T20:55:35.977831+0000","last_became_peered":"2023-09-23T20:55:35.977831+0000","last_unstale":"2023-09-23T20:58:35.187409+0000","last_undegraded":"2023-09-23T20:58:35.187409+0000","last_fullsized":"2023-09-23T20:58:35.187409+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:25:47.678023+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.5","version":"235'7604","reported_seq":7656,"reported_epoch":235,"state":"active+clean","last_fresh":"2023-09-23T20:58:41.398868+0000","last_change":"2023-09-23T20:56:45.990110+0000","last_active":"2023-09-23T20:58:41.398868+0000","last_peered":"2023-09-23T20:58:41.398868+0000","last_clean":"2023-09-23T20:58:41.398868+0000","last_became_active":"2023-09-23T20:56:45.988394+0000","last_became_peered":"2023-09-23T20:56:45.988394+0000","last_unstale":"2023-09-23T20:58:41.398868+0000","last_undegraded":"2023-09-23T20:58:41.398868+0000","last_fullsized":"2023-09-23T20:58:41.398868+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":199,"last_epoch_clean":205,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_clean_scrub_stamp":"2023-09-23T20:56:34.529103+0000","objects_scrubbed":0,"log_size":7604,"log_dups_size":0,"ondisk_log_size":7604,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:22:03.814210+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":332529664,"num_objects":5074,"num_object_clones":0,"num_object_copies":15222,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5074,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7604,"num_write_kb":405696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,0],"acting":[2,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"79'161","reported_seq":502,"reported_epoch":235,"state":"active+clean","last_fresh":"2023-09-23T20:58:35.202050+0000","last_change":"2023-09-23T20:57:20.378531+0000","last_active":"2023-09-23T20:58:35.202050+0000","last_peered":"2023-09-23T20:58:35.202050+0000","last_clean":"2023-09-23T20:58:35.202050+0000","last_became_active":"2023-09-23T20:57:08.238019+0000","last_became_peered":"2023-09-23T20:57:08.238019+0000","last_unstale":"2023-09-23T20:58:35.202050+0000","last_undegraded":"2023-09-23T20:58:35.202050+0000","last_fullsized":"2023-09-23T20:58:35.202050+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":3,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:20.377714+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:20.377714+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:00:08.305602+0000","scrub_duration":0.003015228,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":104,"reported_epoch":235,"state":"active+clean","last_fresh":"2023-09-23T20:58:35.201044+0000","last_change":"2023-09-23T20:56:45.991884+0000","last_active":"2023-09-23T20:58:35.201044+0000","last_peered":"2023-09-23T20:58:35.201044+0000","last_clean":"2023-09-23T20:58:35.201044+0000","last_became_active":"2023-09-23T20:56:45.991568+0000","last_became_peered":"2023-09-23T20:56:45.991568+0000","last_unstale":"2023-09-23T20:58:35.201044+0000","last_undegraded":"2023-09-23T20:58:35.201044+0000","last_fullsized":"2023-09-23T20:58:35.201044+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":205,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:21:20.775511+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,0],"acting":[1,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":228,"reported_epoch":234,"state":"active+clean","last_fresh":"2023-09-23T20:58:26.125989+0000","last_change":"2023-09-23T20:57:44.979950+0000","last_active":"2023-09-23T20:58:26.125989+0000","last_peered":"2023-09-23T20:58:26.125989+0000","last_clean":"2023-09-23T20:58:26.125989+0000","last_became_active":"2023-09-23T20:57:44.979346+0000","last_became_peered":"2023-09-23T20:57:44.979346+0000","last_unstale":"2023-09-23T20:58:26.125989+0000","last_undegraded":"2023-09-23T20:58:26.125989+0000","last_fullsized":"2023-09-23T20:58:26.125989+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:10:24.372800+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":207,"reported_epoch":234,"state":"active+clean","last_fresh":"2023-09-23T20:58:26.130246+0000","last_change":"2023-09-23T20:50:37.982235+0000","last_active":"2023-09-23T20:58:26.130246+0000","last_peered":"2023-09-23T20:58:26.130246+0000","last_clean":"2023-09-23T20:58:26.130246+0000","last_became_active":"2023-09-23T20:50:37.981913+0000","last_became_peered":"2023-09-23T20:50:37.981913+0000","last_unstale":"2023-09-23T20:58:26.130246+0000","last_undegraded":"2023-09-23T20:58:26.130246+0000","last_fullsized":"2023-09-23T20:58:26.130246+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:59:57.122584+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.0","version":"235'7613","reported_seq":7656,"reported_epoch":235,"state":"active+clean","last_fresh":"2023-09-23T20:58:42.120306+0000","last_change":"2023-09-23T20:56:35.550635+0000","last_active":"2023-09-23T20:58:42.120306+0000","last_peered":"2023-09-23T20:58:42.120306+0000","last_clean":"2023-09-23T20:58:42.120306+0000","last_became_active":"2023-09-23T20:56:35.550376+0000","last_became_peered":"2023-09-23T20:56:35.550376+0000","last_unstale":"2023-09-23T20:58:42.120306+0000","last_undegraded":"2023-09-23T20:58:42.120306+0000","last_fullsized":"2023-09-23T20:58:42.120306+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":199,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_clean_scrub_stamp":"2023-09-23T20:56:34.529103+0000","objects_scrubbed":0,"log_size":7613,"log_dups_size":0,"ondisk_log_size":7613,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:40:55.020049+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":332201984,"num_objects":5069,"num_object_clones":0,"num_object_copies":15207,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5069,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7613,"num_write_kb":405824,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.6","version":"79'161","reported_seq":518,"reported_epoch":235,"state":"active+clean","last_fresh":"2023-09-23T20:58:35.190487+0000","last_change":"2023-09-23T20:57:26.907505+0000","last_active":"2023-09-23T20:58:35.190487+0000","last_peered":"2023-09-23T20:58:35.190487+0000","last_clean":"2023-09-23T20:58:35.190487+0000","last_became_active":"2023-09-23T20:57:08.240321+0000","last_became_peered":"2023-09-23T20:57:08.240321+0000","last_unstale":"2023-09-23T20:58:35.190487+0000","last_undegraded":"2023-09-23T20:58:35.190487+0000","last_fullsized":"2023-09-23T20:58:35.190487+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":3,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:26.907408+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:26.907408+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:34:17.392928+0000","scrub_duration":0.010337038,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":150,"reported_epoch":234,"state":"active+clean","last_fresh":"2023-09-23T20:58:26.130877+0000","last_change":"2023-09-23T20:56:45.988566+0000","last_active":"2023-09-23T20:58:26.130877+0000","last_peered":"2023-09-23T20:58:26.130877+0000","last_clean":"2023-09-23T20:58:26.130877+0000","last_became_active":"2023-09-23T20:56:45.988249+0000","last_became_peered":"2023-09-23T20:56:45.988249+0000","last_unstale":"2023-09-23T20:58:26.130877+0000","last_undegraded":"2023-09-23T20:58:26.130877+0000","last_fullsized":"2023-09-23T20:58:26.130877+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":205,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:51:36.644764+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,4],"acting":[0,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":206,"reported_epoch":235,"state":"active+clean","last_fresh":"2023-09-23T20:58:35.201812+0000","last_change":"2023-09-23T20:54:31.898965+0000","last_active":"2023-09-23T20:58:35.201812+0000","last_peered":"2023-09-23T20:58:35.201812+0000","last_clean":"2023-09-23T20:58:35.201812+0000","last_became_active":"2023-09-23T20:54:31.898663+0000","last_became_peered":"2023-09-23T20:54:31.898663+0000","last_unstale":"2023-09-23T20:58:35.201812+0000","last_undegraded":"2023-09-23T20:58:35.201812+0000","last_fullsized":"2023-09-23T20:58:35.201812+0000","mapping_epoch":136,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:53:05.058087+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":195,"reported_epoch":234,"state":"active+clean","last_fresh":"2023-09-23T20:58:26.126650+0000","last_change":"2023-09-23T20:54:31.884717+0000","last_active":"2023-09-23T20:58:26.126650+0000","last_peered":"2023-09-23T20:58:26.126650+0000","last_clean":"2023-09-23T20:58:26.126650+0000","last_became_active":"2023-09-23T20:54:31.884345+0000","last_became_peered":"2023-09-23T20:54:31.884345+0000","last_unstale":"2023-09-23T20:58:26.126650+0000","last_undegraded":"2023-09-23T20:58:26.126650+0000","last_fullsized":"2023-09-23T20:58:26.126650+0000","mapping_epoch":136,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:02:26.393868+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.3","version":"235'7400","reported_seq":7445,"reported_epoch":235,"state":"active+clean","last_fresh":"2023-09-23T20:58:39.471576+0000","last_change":"2023-09-23T20:56:35.549655+0000","last_active":"2023-09-23T20:58:39.471576+0000","last_peered":"2023-09-23T20:58:39.471576+0000","last_clean":"2023-09-23T20:58:39.471576+0000","last_became_active":"2023-09-23T20:56:35.549357+0000","last_became_peered":"2023-09-23T20:56:35.549357+0000","last_unstale":"2023-09-23T20:58:39.471576+0000","last_undegraded":"2023-09-23T20:58:39.471576+0000","last_fullsized":"2023-09-23T20:58:39.471576+0000","mapping_epoch":199,"log_start":"205'1200","ondisk_log_start":"205'1200","created":199,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_clean_scrub_stamp":"2023-09-23T20:56:34.529103+0000","objects_scrubbed":0,"log_size":6200,"log_dups_size":0,"ondisk_log_size":6200,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:37:58.752518+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":335675392,"num_objects":5122,"num_object_clones":0,"num_object_copies":15366,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5122,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7400,"num_write_kb":400704,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.5","version":"79'161","reported_seq":498,"reported_epoch":234,"state":"active+clean","last_fresh":"2023-09-23T20:58:26.118177+0000","last_change":"2023-09-23T20:57:30.029479+0000","last_active":"2023-09-23T20:58:26.118177+0000","last_peered":"2023-09-23T20:58:26.118177+0000","last_clean":"2023-09-23T20:58:26.118177+0000","last_became_active":"2023-09-23T20:57:08.249909+0000","last_became_peered":"2023-09-23T20:57:08.249909+0000","last_unstale":"2023-09-23T20:58:26.118177+0000","last_undegraded":"2023-09-23T20:58:26.118177+0000","last_fullsized":"2023-09-23T20:58:26.118177+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":3,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:30.029396+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:30.029396+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:16:46.083145+0000","scrub_duration":0.012914682,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":198,"reported_epoch":234,"state":"active+clean","last_fresh":"2023-09-23T20:58:26.133232+0000","last_change":"2023-09-23T20:51:10.836910+0000","last_active":"2023-09-23T20:58:26.133232+0000","last_peered":"2023-09-23T20:58:26.133232+0000","last_clean":"2023-09-23T20:58:26.133232+0000","last_became_active":"2023-09-23T20:51:10.836557+0000","last_became_peered":"2023-09-23T20:51:10.836557+0000","last_unstale":"2023-09-23T20:58:26.133232+0000","last_undegraded":"2023-09-23T20:58:26.133232+0000","last_fullsized":"2023-09-23T20:58:26.133232+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:51:54.019969+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":206,"reported_epoch":236,"state":"active+clean","last_fresh":"2023-09-23T20:58:44.083967+0000","last_change":"2023-09-23T20:55:35.953444+0000","last_active":"2023-09-23T20:58:44.083967+0000","last_peered":"2023-09-23T20:58:44.083967+0000","last_clean":"2023-09-23T20:58:44.083967+0000","last_became_active":"2023-09-23T20:55:35.953151+0000","last_became_peered":"2023-09-23T20:55:35.953151+0000","last_unstale":"2023-09-23T20:58:44.083967+0000","last_undegraded":"2023-09-23T20:58:44.083967+0000","last_fullsized":"2023-09-23T20:58:44.083967+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:39:12.404391+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":23,"reported_epoch":234,"state":"active+clean","last_fresh":"2023-09-23T20:58:26.130753+0000","last_change":"2023-09-23T20:57:44.982509+0000","last_active":"2023-09-23T20:58:26.130753+0000","last_peered":"2023-09-23T20:58:26.130753+0000","last_clean":"2023-09-23T20:58:26.130753+0000","last_became_active":"2023-09-23T20:57:44.982175+0000","last_became_peered":"2023-09-23T20:57:44.982175+0000","last_unstale":"2023-09-23T20:58:26.130753+0000","last_undegraded":"2023-09-23T20:58:26.130753+0000","last_fullsized":"2023-09-23T20:58:26.130753+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:37:49.215488+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.2","version":"235'7406","reported_seq":7451,"reported_epoch":235,"state":"active+clean","last_fresh":"2023-09-23T20:58:39.431384+0000","last_change":"2023-09-23T20:56:35.543020+0000","last_active":"2023-09-23T20:58:39.431384+0000","last_peered":"2023-09-23T20:58:39.431384+0000","last_clean":"2023-09-23T20:58:39.431384+0000","last_became_active":"2023-09-23T20:56:35.542719+0000","last_became_peered":"2023-09-23T20:56:35.542719+0000","last_unstale":"2023-09-23T20:58:39.431384+0000","last_undegraded":"2023-09-23T20:58:39.431384+0000","last_fullsized":"2023-09-23T20:58:39.431384+0000","mapping_epoch":199,"log_start":"205'1200","ondisk_log_start":"205'1200","created":199,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_clean_scrub_stamp":"2023-09-23T20:56:34.529103+0000","objects_scrubbed":0,"log_size":6206,"log_dups_size":0,"ondisk_log_size":6206,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:52:21.088628+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":334626816,"num_objects":5106,"num_object_clones":0,"num_object_copies":15318,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5106,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7406,"num_write_kb":400384,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.4","version":"79'161","reported_seq":513,"reported_epoch":234,"state":"active+clean","last_fresh":"2023-09-23T20:58:26.132420+0000","last_change":"2023-09-23T20:57:20.012904+0000","last_active":"2023-09-23T20:58:26.132420+0000","last_peered":"2023-09-23T20:58:26.132420+0000","last_clean":"2023-09-23T20:58:26.132420+0000","last_became_active":"2023-09-23T20:57:08.237996+0000","last_became_peered":"2023-09-23T20:57:08.237996+0000","last_unstale":"2023-09-23T20:58:26.132420+0000","last_undegraded":"2023-09-23T20:58:26.132420+0000","last_fullsized":"2023-09-23T20:58:26.132420+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":3,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:20.012764+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:20.012764+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:51:28.796946+0000","scrub_duration":0.0045251919999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":185,"reported_epoch":235,"state":"active+clean","last_fresh":"2023-09-23T20:58:35.190775+0000","last_change":"2023-09-23T20:55:35.949541+0000","last_active":"2023-09-23T20:58:35.190775+0000","last_peered":"2023-09-23T20:58:35.190775+0000","last_clean":"2023-09-23T20:58:35.190775+0000","last_became_active":"2023-09-23T20:55:35.944635+0000","last_became_peered":"2023-09-23T20:55:35.944635+0000","last_unstale":"2023-09-23T20:58:35.190775+0000","last_undegraded":"2023-09-23T20:58:35.190775+0000","last_fullsized":"2023-09-23T20:58:35.190775+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:30:36.033585+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"70'1","reported_seq":223,"reported_epoch":234,"state":"active+clean","last_fresh":"2023-09-23T20:58:26.130358+0000","last_change":"2023-09-23T20:54:14.009723+0000","last_active":"2023-09-23T20:58:26.130358+0000","last_peered":"2023-09-23T20:58:26.130358+0000","last_clean":"2023-09-23T20:58:26.130358+0000","last_became_active":"2023-09-23T20:54:13.941771+0000","last_became_peered":"2023-09-23T20:54:13.941771+0000","last_unstale":"2023-09-23T20:58:26.130358+0000","last_undegraded":"2023-09-23T20:58:26.130358+0000","last_fullsized":"2023-09-23T20:58:26.130358+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:48:25.588116+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":139,"reported_epoch":234,"state":"active+clean","last_fresh":"2023-09-23T20:58:26.119337+0000","last_change":"2023-09-23T20:55:35.949780+0000","last_active":"2023-09-23T20:58:26.119337+0000","last_peered":"2023-09-23T20:58:26.119337+0000","last_clean":"2023-09-23T20:58:26.119337+0000","last_became_active":"2023-09-23T20:55:35.949497+0000","last_became_peered":"2023-09-23T20:55:35.949497+0000","last_unstale":"2023-09-23T20:58:26.119337+0000","last_undegraded":"2023-09-23T20:58:26.119337+0000","last_fullsized":"2023-09-23T20:58:26.119337+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:09:37.420882+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":197,"reported_epoch":234,"state":"active+clean","last_fresh":"2023-09-23T20:58:26.126584+0000","last_change":"2023-09-23T20:54:13.929196+0000","last_active":"2023-09-23T20:58:26.126584+0000","last_peered":"2023-09-23T20:58:26.126584+0000","last_clean":"2023-09-23T20:58:26.126584+0000","last_became_active":"2023-09-23T20:54:13.928831+0000","last_became_peered":"2023-09-23T20:54:13.928831+0000","last_unstale":"2023-09-23T20:58:26.126584+0000","last_undegraded":"2023-09-23T20:58:26.126584+0000","last_fullsized":"2023-09-23T20:58:26.126584+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:28:44.316851+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":203,"reported_epoch":234,"state":"active+clean","last_fresh":"2023-09-23T20:58:26.110811+0000","last_change":"2023-09-23T20:52:12.855430+0000","last_active":"2023-09-23T20:58:26.110811+0000","last_peered":"2023-09-23T20:58:26.110811+0000","last_clean":"2023-09-23T20:58:26.110811+0000","last_became_active":"2023-09-23T20:52:12.855115+0000","last_became_peered":"2023-09-23T20:52:12.855115+0000","last_unstale":"2023-09-23T20:58:26.110811+0000","last_undegraded":"2023-09-23T20:58:26.110811+0000","last_fullsized":"2023-09-23T20:58:26.110811+0000","mapping_epoch":100,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":101,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:33:23.802214+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":112,"reported_epoch":234,"state":"active+clean","last_fresh":"2023-09-23T20:58:26.117876+0000","last_change":"2023-09-23T20:54:31.902946+0000","last_active":"2023-09-23T20:58:26.117876+0000","last_peered":"2023-09-23T20:58:26.117876+0000","last_clean":"2023-09-23T20:58:26.117876+0000","last_became_active":"2023-09-23T20:54:31.889484+0000","last_became_peered":"2023-09-23T20:54:31.889484+0000","last_unstale":"2023-09-23T20:58:26.117876+0000","last_undegraded":"2023-09-23T20:58:26.117876+0000","last_fullsized":"2023-09-23T20:58:26.117876+0000","mapping_epoch":136,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:44:44.278398+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":112,"reported_epoch":234,"state":"active+clean","last_fresh":"2023-09-23T20:58:26.118025+0000","last_change":"2023-09-23T20:54:13.950351+0000","last_active":"2023-09-23T20:58:26.118025+0000","last_peered":"2023-09-23T20:58:26.118025+0000","last_clean":"2023-09-23T20:58:26.118025+0000","last_became_active":"2023-09-23T20:54:13.949987+0000","last_became_peered":"2023-09-23T20:54:13.949987+0000","last_unstale":"2023-09-23T20:58:26.118025+0000","last_undegraded":"2023-09-23T20:58:26.118025+0000","last_fullsized":"2023-09-23T20:58:26.118025+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:46:17.765134+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":143,"reported_epoch":234,"state":"active+clean","last_fresh":"2023-09-23T20:58:26.118158+0000","last_change":"2023-09-23T20:56:18.141728+0000","last_active":"2023-09-23T20:58:26.118158+0000","last_peered":"2023-09-23T20:58:26.118158+0000","last_clean":"2023-09-23T20:58:26.118158+0000","last_became_active":"2023-09-23T20:56:18.137979+0000","last_became_peered":"2023-09-23T20:56:18.137979+0000","last_unstale":"2023-09-23T20:58:26.118158+0000","last_undegraded":"2023-09-23T20:58:26.118158+0000","last_fullsized":"2023-09-23T20:58:26.118158+0000","mapping_epoch":192,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":193,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:58:05.658443+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":112,"reported_epoch":234,"state":"active+clean","last_fresh":"2023-09-23T20:58:26.117270+0000","last_change":"2023-09-23T20:54:13.935307+0000","last_active":"2023-09-23T20:58:26.117270+0000","last_peered":"2023-09-23T20:58:26.117270+0000","last_clean":"2023-09-23T20:58:26.117270+0000","last_became_active":"2023-09-23T20:54:13.935000+0000","last_became_peered":"2023-09-23T20:54:13.935000+0000","last_unstale":"2023-09-23T20:58:26.117270+0000","last_undegraded":"2023-09-23T20:58:26.117270+0000","last_fullsized":"2023-09-23T20:58:26.117270+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:47:40.974082+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":221,"reported_epoch":234,"state":"active+clean","last_fresh":"2023-09-23T20:58:26.145911+0000","last_change":"2023-09-23T20:55:35.954870+0000","last_active":"2023-09-23T20:58:26.145911+0000","last_peered":"2023-09-23T20:58:26.145911+0000","last_clean":"2023-09-23T20:58:26.145911+0000","last_became_active":"2023-09-23T20:55:35.954568+0000","last_became_peered":"2023-09-23T20:55:35.954568+0000","last_unstale":"2023-09-23T20:58:26.145911+0000","last_undegraded":"2023-09-23T20:58:26.145911+0000","last_fullsized":"2023-09-23T20:58:26.145911+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:06:22.423566+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":221,"reported_epoch":234,"state":"active+clean","last_fresh":"2023-09-23T20:58:26.131627+0000","last_change":"2023-09-23T20:51:10.838334+0000","last_active":"2023-09-23T20:58:26.131627+0000","last_peered":"2023-09-23T20:58:26.131627+0000","last_clean":"2023-09-23T20:58:26.131627+0000","last_became_active":"2023-09-23T20:51:10.838026+0000","last_became_peered":"2023-09-23T20:51:10.838026+0000","last_unstale":"2023-09-23T20:58:26.131627+0000","last_undegraded":"2023-09-23T20:58:26.131627+0000","last_fullsized":"2023-09-23T20:58:26.131627+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:04:37.009457+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":225,"reported_epoch":234,"state":"active+clean","last_fresh":"2023-09-23T20:58:26.136555+0000","last_change":"2023-09-23T20:54:31.903139+0000","last_active":"2023-09-23T20:58:26.136555+0000","last_peered":"2023-09-23T20:58:26.136555+0000","last_clean":"2023-09-23T20:58:26.136555+0000","last_became_active":"2023-09-23T20:54:31.884520+0000","last_became_peered":"2023-09-23T20:54:31.884520+0000","last_unstale":"2023-09-23T20:58:26.136555+0000","last_undegraded":"2023-09-23T20:58:26.136555+0000","last_fullsized":"2023-09-23T20:58:26.136555+0000","mapping_epoch":136,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:39:34.269705+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1108,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":129,"reported_epoch":234,"state":"active+clean","last_fresh":"2023-09-23T20:58:26.116950+0000","last_change":"2023-09-23T20:55:35.951572+0000","last_active":"2023-09-23T20:58:26.116950+0000","last_peered":"2023-09-23T20:58:26.116950+0000","last_clean":"2023-09-23T20:58:26.116950+0000","last_became_active":"2023-09-23T20:55:35.951294+0000","last_became_peered":"2023-09-23T20:55:35.951294+0000","last_unstale":"2023-09-23T20:58:26.116950+0000","last_undegraded":"2023-09-23T20:58:26.116950+0000","last_fullsized":"2023-09-23T20:58:26.116950+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:36:32.447210+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":7,"num_pg":16,"stat_sum":{"num_bytes":5345116184,"num_objects":81561,"num_object_clones":0,"num_object_copies":244683,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":10981,"num_objects_misplaced":4186,"num_objects_unfound":0,"num_objects_dirty":81561,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":121959,"num_write_kb":6512577,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1175,"num_bytes_recovered":77004800,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":16252809216,"data_stored":16252797000,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":114559,"ondisk_log_size":114559,"up":48,"acting":45,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":22,"num_bytes_recovered":5990,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":21,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3381,"ondisk_log_size":3381,"up":63,"acting":63,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201499,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1813960,"kb_used_data":1640192,"kb_used_omap":0,"kb_used_meta":173760,"kb_avail":91931192,"statfs":{"total":95995035648,"available":94137540608,"internally_reserved":0,"allocated":1679556608,"data_stored":1678725005,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":177930240},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:57:44 2023","interfaces":[{"interface":"back","average":{"1min":3.3239999999999998,"5min":1.613,"15min":0.90200000000000002},"min":{"1min":0.23100000000000001,"5min":0.21099999999999999,"15min":0.21099999999999999},"max":{"1min":43.994999999999997,"5min":43.994999999999997,"15min":43.994999999999997},"last":5.4290000000000003},{"interface":"front","average":{"1min":3.448,"5min":1.847,"15min":1},"min":{"1min":0.35299999999999998,"5min":0.218,"15min":0.218},"max":{"1min":44.206000000000003,"5min":44.206000000000003,"15min":44.206000000000003},"last":5.4509999999999996}]},{"osd":1,"last update":"Sat Sep 23 20:58:18 2023","interfaces":[{"interface":"back","average":{"1min":1.895,"5min":1.9330000000000001,"15min":1.1439999999999999},"min":{"1min":0.29999999999999999,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":10.622,"5min":10.622,"15min":22.591999999999999},"last":3.1080000000000001},{"interface":"front","average":{"1min":1.8320000000000001,"5min":1.5720000000000001,"15min":0.98499999999999999},"min":{"1min":0.25800000000000001,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":7.5629999999999997,"5min":10.097,"15min":21.437000000000001},"last":3.6099999999999999}]},{"osd":2,"last update":"Sat Sep 23 20:57:50 2023","interfaces":[{"interface":"back","average":{"1min":1.3919999999999999,"5min":1.643,"15min":0.93799999999999994},"min":{"1min":0.26800000000000002,"5min":0.26200000000000001,"15min":0.24399999999999999},"max":{"1min":5.3929999999999998,"5min":9.4290000000000003,"15min":9.4290000000000003},"last":4.3449999999999998},{"interface":"front","average":{"1min":3.3420000000000001,"5min":1.978,"15min":1.071},"min":{"1min":0.29899999999999999,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":45.25,"5min":45.25,"15min":45.25},"last":3.5899999999999999}]},{"osd":3,"last update":"Sat Sep 23 20:58:26 2023","interfaces":[{"interface":"back","average":{"1min":3.355,"5min":3.226,"15min":1.587},"min":{"1min":0.32100000000000001,"5min":0.314,"15min":0.314},"max":{"1min":9.9939999999999998,"5min":36.408000000000001,"15min":36.408000000000001},"last":1.383},{"interface":"front","average":{"1min":2.5649999999999999,"5min":2.7160000000000002,"15min":1.405},"min":{"1min":0.32000000000000001,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":9.4529999999999994,"5min":35.593000000000004,"15min":35.593000000000004},"last":5.024}]},{"osd":4,"last update":"Sat Sep 23 20:58:18 2023","interfaces":[{"interface":"back","average":{"1min":0.69999999999999996,"5min":0.82699999999999996,"15min":0.60899999999999999},"min":{"1min":0.28299999999999997,"5min":0.23799999999999999,"15min":0.23699999999999999},"max":{"1min":4.5410000000000004,"5min":5.8920000000000003,"15min":5.8920000000000003},"last":0.29699999999999999},{"interface":"front","average":{"1min":0.70399999999999996,"5min":0.76100000000000001,"15min":0.61399999999999999},"min":{"1min":0.27100000000000002,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":4.4080000000000004,"5min":5.3200000000000003,"15min":5.3200000000000003},"last":0.32700000000000001}]},{"osd":5,"last update":"Sat Sep 23 20:58:18 2023","interfaces":[{"interface":"back","average":{"1min":0.69299999999999995,"5min":0.82799999999999996,"15min":0.63300000000000001},"min":{"1min":0.28499999999999998,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":2.5110000000000001,"5min":7.0389999999999997,"15min":7.0389999999999997},"last":0.41099999999999998},{"interface":"front","average":{"1min":0.72199999999999998,"5min":0.70699999999999996,"15min":0.61199999999999999},"min":{"1min":0.317,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":4.5999999999999996,"5min":4.5999999999999996,"15min":4.5999999999999996},"last":0.34699999999999998}]},{"osd":6,"last update":"Sat Sep 23 20:58:40 2023","interfaces":[{"interface":"back","average":{"1min":0.86599999999999999,"5min":0.71699999999999997,"15min":0.68300000000000005},"min":{"1min":0.33700000000000002,"5min":0.31,"15min":0.31},"max":{"1min":3.5249999999999999,"5min":3.5249999999999999,"15min":3.5249999999999999},"last":0.39500000000000002},{"interface":"front","average":{"1min":0.90100000000000002,"5min":0.81299999999999994,"15min":0.80700000000000005},"min":{"1min":0.36899999999999999,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":3.5409999999999999,"5min":3.5409999999999999,"15min":3.5409999999999999},"last":0.36899999999999999}]}]},{"osd":6,"up_from":164,"seq":704374636583,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1717564,"kb_used_data":1613748,"kb_used_omap":10,"kb_used_meta":103797,"kb_avail":92027588,"statfs":{"total":95995035648,"available":94236250112,"internally_reserved":0,"allocated":1652477952,"data_stored":1651658250,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":10328,"internal_metadata":106289064},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:58:43 2023","interfaces":[{"interface":"back","average":{"1min":2.7330000000000001,"5min":1.4410000000000001,"15min":1.0569999999999999},"min":{"1min":0.25800000000000001,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":7.5179999999999998,"5min":8.7319999999999993,"15min":8.7319999999999993},"last":1.5620000000000001},{"interface":"front","average":{"1min":3.1709999999999998,"5min":1.502,"15min":1.1559999999999999},"min":{"1min":0.25,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":6.7140000000000004,"5min":6.7140000000000004,"15min":6.7140000000000004},"last":0.53800000000000003}]},{"osd":1,"last update":"Sat Sep 23 20:58:43 2023","interfaces":[{"interface":"back","average":{"1min":1.5449999999999999,"5min":1.097,"15min":0.89100000000000001},"min":{"1min":0.36799999999999999,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":5.2350000000000003,"5min":6.6029999999999998,"15min":6.6029999999999998},"last":0.48099999999999998},{"interface":"front","average":{"1min":2.5779999999999998,"5min":1.244,"15min":0.99299999999999999},"min":{"1min":0.23000000000000001,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":10.324999999999999,"5min":10.324999999999999,"15min":10.324999999999999},"last":2.7480000000000002}]},{"osd":2,"last update":"Sat Sep 23 20:58:43 2023","interfaces":[{"interface":"back","average":{"1min":2.286,"5min":1.982,"15min":1.758},"min":{"1min":0.28999999999999998,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":7.0519999999999996,"5min":13.750999999999999,"15min":13.750999999999999},"last":0.56200000000000006},{"interface":"front","average":{"1min":2.419,"5min":1.5289999999999999,"15min":1.2470000000000001},"min":{"1min":0.26200000000000001,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":8.0890000000000004,"5min":8.0890000000000004,"15min":8.0890000000000004},"last":4.1970000000000001}]},{"osd":3,"last update":"Sat Sep 23 20:58:43 2023","interfaces":[{"interface":"back","average":{"1min":1.8680000000000001,"5min":1.2849999999999999,"15min":1.173},"min":{"1min":0.32000000000000001,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":6.9000000000000004,"5min":6.9000000000000004,"15min":6.9000000000000004},"last":0.41899999999999998},{"interface":"front","average":{"1min":2.1819999999999999,"5min":1.74,"15min":1.6479999999999999},"min":{"1min":0.28599999999999998,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":11.021000000000001,"5min":11.021000000000001,"15min":11.021000000000001},"last":0.73599999999999999}]},{"osd":4,"last update":"Sat Sep 23 20:58:43 2023","interfaces":[{"interface":"back","average":{"1min":0.49099999999999999,"5min":0.85899999999999999,"15min":0.98699999999999999},"min":{"1min":0.23000000000000001,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":2.7919999999999998,"5min":9.0079999999999991,"15min":9.0079999999999991},"last":2.7919999999999998},{"interface":"front","average":{"1min":0.49299999999999999,"5min":0.50800000000000001,"15min":0.52300000000000002},"min":{"1min":0.27200000000000002,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":2.0059999999999998,"5min":2.0059999999999998,"15min":2.0059999999999998},"last":0.72199999999999998}]},{"osd":5,"last update":"Sat Sep 23 20:58:43 2023","interfaces":[{"interface":"back","average":{"1min":0.53700000000000003,"5min":0.56899999999999995,"15min":0.55000000000000004},"min":{"1min":0.27300000000000002,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":2.8239999999999998,"5min":2.8239999999999998,"15min":2.8239999999999998},"last":2.8239999999999998},{"interface":"front","average":{"1min":0.81000000000000005,"5min":0.78500000000000003,"15min":0.80100000000000005},"min":{"1min":0.33400000000000002,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":3.1379999999999999,"5min":3.1379999999999999,"15min":3.1379999999999999},"last":3.1379999999999999}]},{"osd":7,"last update":"Sat Sep 23 20:58:43 2023","interfaces":[{"interface":"back","average":{"1min":0.52700000000000002,"5min":0.85399999999999998,"15min":0.90400000000000003},"min":{"1min":0.312,"5min":0.312,"15min":0.312},"max":{"1min":2.073,"5min":7.8570000000000002,"15min":7.8570000000000002},"last":0.68700000000000006},{"interface":"front","average":{"1min":0.61899999999999999,"5min":0.66200000000000003,"15min":0.57399999999999995},"min":{"1min":0.30299999999999999,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":2.6989999999999998,"5min":8.282,"15min":8.282},"last":0.76200000000000001}]}]},{"osd":1,"up_from":88,"seq":377957122154,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2508464,"kb_used_data":2281512,"kb_used_omap":9,"kb_used_meta":226934,"kb_avail":91236688,"statfs":{"total":95995035648,"available":93426368512,"internally_reserved":0,"allocated":2336268288,"data_stored":2335112845,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9239,"internal_metadata":232381417},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:58:22 2023","interfaces":[{"interface":"back","average":{"1min":3.3290000000000002,"5min":2.9990000000000001,"15min":1.3049999999999999},"min":{"1min":0.23200000000000001,"5min":0.216,"15min":0.216},"max":{"1min":10.73,"5min":16.646999999999998,"15min":16.646999999999998},"last":7.6440000000000001},{"interface":"front","average":{"1min":4.2149999999999999,"5min":3.3940000000000001,"15min":1.4590000000000001},"min":{"1min":0.247,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":15.135999999999999,"5min":18.457999999999998,"15min":18.457999999999998},"last":7.827}]},{"osd":2,"last update":"Sat Sep 23 20:57:50 2023","interfaces":[{"interface":"back","average":{"1min":2.5760000000000001,"5min":2.1309999999999998,"15min":1.0249999999999999},"min":{"1min":0.221,"5min":0.20300000000000001,"15min":0.20300000000000001},"max":{"1min":8.6500000000000004,"5min":14.335000000000001,"15min":14.335000000000001},"last":7.5899999999999999},{"interface":"front","average":{"1min":3.2770000000000001,"5min":3.0209999999999999,"15min":1.4099999999999999},"min":{"1min":0.35699999999999998,"5min":0.248,"15min":0.248},"max":{"1min":10.095000000000001,"5min":38.951000000000001,"15min":38.951000000000001},"last":0.88400000000000001}]},{"osd":3,"last update":"Sat Sep 23 20:58:22 2023","interfaces":[{"interface":"back","average":{"1min":3.1859999999999999,"5min":3.0489999999999999,"15min":1.3720000000000001},"min":{"1min":0.246,"5min":0.246,"15min":0.246},"max":{"1min":10.701000000000001,"5min":25.314,"15min":25.314},"last":6.0259999999999998},{"interface":"front","average":{"1min":2.8450000000000002,"5min":2.6150000000000002,"15min":1.232},"min":{"1min":0.33800000000000002,"5min":0.20599999999999999,"15min":0.20599999999999999},"max":{"1min":7.8899999999999997,"5min":19.881,"15min":19.881},"last":7.6180000000000003}]},{"osd":4,"last update":"Sat Sep 23 20:58:22 2023","interfaces":[{"interface":"back","average":{"1min":1.7050000000000001,"5min":1.9550000000000001,"15min":1.131},"min":{"1min":0.36099999999999999,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":10.598000000000001,"5min":12.006,"15min":12.006},"last":5.056},{"interface":"front","average":{"1min":2.0899999999999999,"5min":1.6839999999999999,"15min":1.016},"min":{"1min":0.31900000000000001,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":6.0460000000000003,"5min":11.234999999999999,"15min":11.234999999999999},"last":7.532}]},{"osd":5,"last update":"Sat Sep 23 20:58:22 2023","interfaces":[{"interface":"back","average":{"1min":3.0609999999999999,"5min":2.133,"15min":1.181},"min":{"1min":0.34399999999999997,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":15.041,"5min":15.041,"15min":15.041},"last":0.85099999999999998},{"interface":"front","average":{"1min":1.669,"5min":2.0169999999999999,"15min":1.153},"min":{"1min":0.33200000000000002,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":10.647,"5min":12.08,"15min":12.08},"last":5.093}]},{"osd":6,"last update":"Sat Sep 23 20:58:39 2023","interfaces":[{"interface":"back","average":{"1min":1.6160000000000001,"5min":1.3700000000000001,"15min":1.333},"min":{"1min":0.30299999999999999,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":10.673999999999999,"5min":10.673999999999999,"15min":10.673999999999999},"last":5.133},{"interface":"front","average":{"1min":1.8280000000000001,"5min":1.508,"15min":1.3380000000000001},"min":{"1min":0.38100000000000001,"5min":0.309,"15min":0.309},"max":{"1min":8.7530000000000001,"5min":8.7530000000000001,"15min":8.7530000000000001},"last":7.6779999999999999}]},{"osd":7,"last update":"Sat Sep 23 20:58:22 2023","interfaces":[{"interface":"back","average":{"1min":2.0659999999999998,"5min":1.7909999999999999,"15min":1.069},"min":{"1min":0.42499999999999999,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":6.149,"5min":11.186999999999999,"15min":11.186999999999999},"last":7.6609999999999996},{"interface":"front","average":{"1min":3.1520000000000001,"5min":2.1829999999999998,"15min":1.2010000000000001},"min":{"1min":0.41899999999999998,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":15.098000000000001,"5min":15.098000000000001,"15min":15.098000000000001},"last":0.497}]}]},{"osd":0,"up_from":83,"seq":356482285680,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2799384,"kb_used_data":2637328,"kb_used_omap":28,"kb_used_meta":162019,"kb_avail":90945768,"statfs":{"total":95995035648,"available":93128466432,"internally_reserved":0,"allocated":2700623872,"data_stored":2699490638,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":28848,"internal_metadata":165908304},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 20:58:19 2023","interfaces":[{"interface":"back","average":{"1min":3.7799999999999998,"5min":2.9220000000000002,"15min":1.2609999999999999},"min":{"1min":0.25800000000000001,"5min":0.191,"15min":0.191},"max":{"1min":15.946999999999999,"5min":18.863,"15min":18.863},"last":0.22700000000000001},{"interface":"front","average":{"1min":3.9060000000000001,"5min":2.8540000000000001,"15min":1.2669999999999999},"min":{"1min":0.247,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":15.766999999999999,"5min":17.704999999999998,"15min":17.704999999999998},"last":5.3259999999999996}]},{"osd":2,"last update":"Sat Sep 23 20:57:47 2023","interfaces":[{"interface":"back","average":{"1min":2.6150000000000002,"5min":3.0329999999999999,"15min":1.3400000000000001},"min":{"1min":0.35599999999999998,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":8.4949999999999992,"5min":17.753,"15min":17.753},"last":5.8789999999999996},{"interface":"front","average":{"1min":2.843,"5min":2.8199999999999998,"15min":1.234},"min":{"1min":0.308,"5min":0.21099999999999999,"15min":0.21099999999999999},"max":{"1min":9.9469999999999992,"5min":13.218999999999999,"15min":13.218999999999999},"last":0.38500000000000001}]},{"osd":3,"last update":"Sat Sep 23 20:58:19 2023","interfaces":[{"interface":"back","average":{"1min":3.722,"5min":2.8919999999999999,"15min":1.298},"min":{"1min":0.39300000000000002,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":15.994,"5min":18.893999999999998,"15min":18.893999999999998},"last":5.859},{"interface":"front","average":{"1min":3.7930000000000001,"5min":3.0649999999999999,"15min":1.423},"min":{"1min":0.29699999999999999,"5min":0.28000000000000003,"15min":0.23300000000000001},"max":{"1min":15.808,"5min":17.472000000000001,"15min":17.472000000000001},"last":5.4530000000000003}]},{"osd":4,"last update":"Sat Sep 23 20:57:47 2023","interfaces":[{"interface":"back","average":{"1min":1.968,"5min":1.881,"15min":1.0740000000000001},"min":{"1min":0.35099999999999998,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":9.9079999999999995,"5min":12.807,"15min":12.807},"last":0.48999999999999999},{"interface":"front","average":{"1min":1.1659999999999999,"5min":1.643,"15min":0.997},"min":{"1min":0.311,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":4.7850000000000001,"5min":18.724,"15min":18.724},"last":0.57899999999999996}]},{"osd":5,"last update":"Sat Sep 23 20:57:47 2023","interfaces":[{"interface":"back","average":{"1min":1.7669999999999999,"5min":1.8129999999999999,"15min":1.069},"min":{"1min":0.33600000000000002,"5min":0.318,"15min":0.318},"max":{"1min":6.673,"5min":8.4990000000000006,"15min":8.4990000000000006},"last":5.4119999999999999},{"interface":"front","average":{"1min":2.2440000000000002,"5min":2.0600000000000001,"15min":1.1619999999999999},"min":{"1min":0.39000000000000001,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":9.8599999999999994,"5min":13.154999999999999,"15min":13.154999999999999},"last":0.45200000000000001}]},{"osd":6,"last update":"Sat Sep 23 20:57:41 2023","interfaces":[{"interface":"back","average":{"1min":1.8160000000000001,"5min":1.6879999999999999,"15min":1.667},"min":{"1min":0.37,"5min":0.37,"15min":0.37},"max":{"1min":6.8470000000000004,"5min":8.7309999999999999,"15min":8.7309999999999999},"last":0.52000000000000002},{"interface":"front","average":{"1min":1.875,"5min":1.7709999999999999,"15min":1.754},"min":{"1min":0.377,"5min":0.377,"15min":0.377},"max":{"1min":5.492,"5min":8.6760000000000002,"15min":8.6760000000000002},"last":0.54900000000000004}]},{"osd":7,"last update":"Sat Sep 23 20:57:47 2023","interfaces":[{"interface":"back","average":{"1min":1.786,"5min":1.8220000000000001,"15min":1.1020000000000001},"min":{"1min":0.41599999999999998,"5min":0.379,"15min":0.379},"max":{"1min":6.6390000000000002,"5min":8.4339999999999993,"15min":8.4339999999999993},"last":5.3739999999999997},{"interface":"front","average":{"1min":1.393,"5min":1.698,"15min":1.0269999999999999},"min":{"1min":0.40100000000000002,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":4.6799999999999997,"5min":18.673999999999999,"15min":18.673999999999999},"last":0.60999999999999999}]}]},{"osd":2,"up_from":93,"seq":399431958627,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2611336,"kb_used_data":2315840,"kb_used_omap":81,"kb_used_meta":295406,"kb_avail":91133816,"statfs":{"total":95995035648,"available":93321027584,"internally_reserved":0,"allocated":2371420160,"data_stored":2370299674,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":83750,"internal_metadata":302495962},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:57:43 2023","interfaces":[{"interface":"back","average":{"1min":2.2090000000000001,"5min":1.9319999999999999,"15min":0.89400000000000002},"min":{"1min":0.27100000000000002,"5min":0.17999999999999999,"15min":0.17999999999999999},"max":{"1min":9.7460000000000004,"5min":11.183,"15min":11.183},"last":4.641},{"interface":"front","average":{"1min":3.0449999999999999,"5min":2.5110000000000001,"15min":1.1160000000000001},"min":{"1min":0.216,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":10.891999999999999,"5min":12.641,"15min":12.641},"last":4.032}]},{"osd":1,"last update":"Sat Sep 23 20:57:43 2023","interfaces":[{"interface":"back","average":{"1min":2.23,"5min":2.8119999999999998,"15min":1.2310000000000001},"min":{"1min":0.21199999999999999,"5min":0.20200000000000001,"15min":0.20200000000000001},"max":{"1min":8.2889999999999997,"5min":16.081,"15min":16.081},"last":4.6120000000000001},{"interface":"front","average":{"1min":2.9660000000000002,"5min":3.419,"15min":1.4390000000000001},"min":{"1min":0.26300000000000001,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":8.8960000000000008,"5min":16.041,"15min":16.041},"last":7.218}]},{"osd":3,"last update":"Sat Sep 23 20:58:26 2023","interfaces":[{"interface":"back","average":{"1min":5.1120000000000001,"5min":3.4620000000000002,"15min":1.585},"min":{"1min":0.23300000000000001,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":23.309999999999999,"5min":23.309999999999999,"15min":23.309999999999999},"last":9.6760000000000002},{"interface":"front","average":{"1min":5.5659999999999998,"5min":4.21,"15min":1.8240000000000001},"min":{"1min":0.34300000000000003,"5min":0.22800000000000001,"15min":0.221},"max":{"1min":23.324999999999999,"5min":23.324999999999999,"15min":23.324999999999999},"last":9.6590000000000007}]},{"osd":4,"last update":"Sat Sep 23 20:57:43 2023","interfaces":[{"interface":"back","average":{"1min":1.8660000000000001,"5min":1.851,"15min":1.0349999999999999},"min":{"1min":0.318,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":8.657,"5min":9.8119999999999994,"15min":9.8119999999999994},"last":7.0759999999999996},{"interface":"front","average":{"1min":1.9770000000000001,"5min":2.0190000000000001,"15min":1.1299999999999999},"min":{"1min":0.34300000000000003,"5min":0.313,"15min":0.313},"max":{"1min":6.7850000000000001,"5min":10.609,"15min":10.609},"last":0.32700000000000001}]},{"osd":5,"last update":"Sat Sep 23 20:57:43 2023","interfaces":[{"interface":"back","average":{"1min":1.823,"5min":2.121,"15min":1.145},"min":{"1min":0.29299999999999998,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":8.5700000000000003,"5min":9.7629999999999999,"15min":9.7629999999999999},"last":4.0970000000000004},{"interface":"front","average":{"1min":2.0259999999999998,"5min":2.129,"15min":1.147},"min":{"1min":0.32600000000000001,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":6.7510000000000003,"5min":10.574,"15min":10.574},"last":0.36399999999999999}]},{"osd":6,"last update":"Sat Sep 23 20:57:41 2023","interfaces":[{"interface":"back","average":{"1min":1.9450000000000001,"5min":1.7230000000000001,"15min":1.6859999999999999},"min":{"1min":0.312,"5min":0.312,"15min":0.312},"max":{"1min":6.8109999999999999,"5min":14.244999999999999,"15min":14.244999999999999},"last":0.38800000000000001},{"interface":"front","average":{"1min":1.9390000000000001,"5min":1.2110000000000001,"15min":1.0900000000000001},"min":{"1min":0.374,"5min":0.374,"15min":0.374},"max":{"1min":6.835,"5min":6.835,"15min":6.835},"last":0.41199999999999998}]},{"osd":7,"last update":"Sat Sep 23 20:57:43 2023","interfaces":[{"interface":"back","average":{"1min":1.397,"5min":1.524,"15min":0.93799999999999994},"min":{"1min":0.32500000000000001,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":4.9820000000000002,"5min":8.0820000000000007,"15min":8.0820000000000007},"last":4.5730000000000004},{"interface":"front","average":{"1min":1.804,"5min":2.1360000000000001,"15min":1.159},"min":{"1min":0.35699999999999998,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":8.6219999999999999,"5min":9.7859999999999996,"15min":9.7859999999999996},"last":4.0679999999999996}]}]},{"osd":3,"up_from":98,"seq":420906795100,"num_pgs":49,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2509800,"kb_used_data":2174048,"kb_used_omap":7,"kb_used_meta":335736,"kb_avail":91235352,"statfs":{"total":95995035648,"available":93425000448,"internally_reserved":0,"allocated":2226225152,"data_stored":2225133539,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7576,"internal_metadata":343794280},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:58:25 2023","interfaces":[{"interface":"back","average":{"1min":3.097,"5min":3.02,"15min":1.4890000000000001},"min":{"1min":0.308,"5min":0.20599999999999999,"15min":0.20599999999999999},"max":{"1min":8.5549999999999997,"5min":23.123999999999999,"15min":23.123999999999999},"last":0.5},{"interface":"front","average":{"1min":2.9900000000000002,"5min":2.7869999999999999,"15min":1.3009999999999999},"min":{"1min":0.19800000000000001,"5min":0.19800000000000001,"15min":0.19800000000000001},"max":{"1min":9.8889999999999993,"5min":15.079000000000001,"15min":15.079000000000001},"last":0.253}]},{"osd":1,"last update":"Sat Sep 23 20:58:25 2023","interfaces":[{"interface":"back","average":{"1min":3.7829999999999999,"5min":3.1509999999999998,"15min":1.4390000000000001},"min":{"1min":0.32400000000000001,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":13.173,"5min":21.385000000000002,"15min":21.385000000000002},"last":0.23899999999999999},{"interface":"front","average":{"1min":3.0880000000000001,"5min":3.242,"15min":1.5109999999999999},"min":{"1min":0.28799999999999998,"5min":0.218,"15min":0.218},"max":{"1min":10.683999999999999,"5min":25.353000000000002,"15min":25.353000000000002},"last":0.30099999999999999}]},{"osd":2,"last update":"Sat Sep 23 20:58:25 2023","interfaces":[{"interface":"back","average":{"1min":4.851,"5min":3.052,"15min":1.444},"min":{"1min":0.35899999999999999,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":15.476000000000001,"5min":24.154,"15min":24.154},"last":3.4169999999999998},{"interface":"front","average":{"1min":2.8519999999999999,"5min":3.335,"15min":1.577},"min":{"1min":0.23999999999999999,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":8.6069999999999993,"5min":25.305,"15min":25.305},"last":7.2270000000000003}]},{"osd":4,"last update":"Sat Sep 23 20:58:25 2023","interfaces":[{"interface":"back","average":{"1min":2.6539999999999999,"5min":1.8240000000000001,"15min":1.141},"min":{"1min":0.38900000000000001,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":9.6170000000000009,"5min":11.861000000000001,"15min":11.861000000000001},"last":0.42199999999999999},{"interface":"front","average":{"1min":1.3600000000000001,"5min":2.0880000000000001,"15min":1.246},"min":{"1min":0.313,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":4.8109999999999999,"5min":22.853000000000002,"15min":22.853000000000002},"last":0.35499999999999998}]},{"osd":5,"last update":"Sat Sep 23 20:58:25 2023","interfaces":[{"interface":"back","average":{"1min":2.9540000000000002,"5min":1.8999999999999999,"15min":1.1459999999999999},"min":{"1min":0.436,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":9.6430000000000007,"5min":11.891999999999999,"15min":11.891999999999999},"last":0.39000000000000001},{"interface":"front","average":{"1min":2.04,"5min":1.6579999999999999,"15min":1.075},"min":{"1min":0.33300000000000002,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":5.7690000000000001,"5min":9.2870000000000008,"15min":9.2870000000000008},"last":1.212}]},{"osd":6,"last update":"Sat Sep 23 20:57:39 2023","interfaces":[{"interface":"back","average":{"1min":1.173,"5min":0.99399999999999999,"15min":0.96399999999999997},"min":{"1min":0.40999999999999998,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":6.3079999999999998,"5min":6.3079999999999998,"15min":6.3079999999999998},"last":0.55400000000000005},{"interface":"front","average":{"1min":1.6879999999999999,"5min":1.4670000000000001,"15min":1.4299999999999999},"min":{"1min":0.40100000000000002,"5min":0.40100000000000002,"15min":0.40100000000000002},"max":{"1min":5.8230000000000004,"5min":7.234,"15min":7.234},"last":1.2749999999999999}]},{"osd":7,"last update":"Sat Sep 23 20:58:25 2023","interfaces":[{"interface":"back","average":{"1min":3.0099999999999998,"5min":2.3399999999999999,"15min":1.3220000000000001},"min":{"1min":0.46500000000000002,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":9.5749999999999993,"5min":16.364999999999998,"15min":16.364999999999998},"last":0.51800000000000002},{"interface":"front","average":{"1min":2.214,"5min":1.766,"15min":1.147},"min":{"1min":0.42499999999999999,"5min":0.379,"15min":0.379},"max":{"1min":5.8090000000000002,"5min":9.3179999999999996,"15min":9.3179999999999996},"last":1.246}]}]},{"osd":4,"up_from":31,"seq":133143986408,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1765640,"kb_used_data":1634816,"kb_used_omap":0,"kb_used_meta":130816,"kb_avail":91979512,"statfs":{"total":95995035648,"available":94187020288,"internally_reserved":0,"allocated":1674051584,"data_stored":1673220234,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":133955584},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:57:52 2023","interfaces":[{"interface":"back","average":{"1min":2.4359999999999999,"5min":1.593,"15min":0.95599999999999996},"min":{"1min":0.214,"5min":0.214,"15min":0.214},"max":{"1min":11.784000000000001,"5min":11.784000000000001,"15min":11.784000000000001},"last":0.59099999999999997},{"interface":"front","average":{"1min":2.2970000000000002,"5min":1.782,"15min":1.1699999999999999},"min":{"1min":0.28100000000000003,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":11.766,"5min":11.766,"15min":11.766},"last":0.61399999999999999}]},{"osd":1,"last update":"Sat Sep 23 20:58:21 2023","interfaces":[{"interface":"back","average":{"1min":1.7370000000000001,"5min":1.641,"15min":0.996},"min":{"1min":0.246,"5min":0.246,"15min":0.23599999999999999},"max":{"1min":10.153,"5min":10.153,"15min":10.153},"last":1.194},{"interface":"front","average":{"1min":2.645,"5min":1.649,"15min":0.96399999999999997},"min":{"1min":0.26100000000000001,"5min":0.26100000000000001,"15min":0.23200000000000001},"max":{"1min":12.292999999999999,"5min":12.292999999999999,"15min":12.292999999999999},"last":5.8390000000000004}]},{"osd":2,"last update":"Sat Sep 23 20:57:52 2023","interfaces":[{"interface":"back","average":{"1min":1.5840000000000001,"5min":1.746,"15min":1.036},"min":{"1min":0.35699999999999998,"5min":0.311,"15min":0.29099999999999998},"max":{"1min":5.21,"5min":15.288,"15min":15.288},"last":1.179},{"interface":"front","average":{"1min":1.8939999999999999,"5min":2.0720000000000001,"15min":1.2050000000000001},"min":{"1min":0.39500000000000002,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":3.8300000000000001,"5min":15.262,"15min":15.262},"last":1.087}]},{"osd":3,"last update":"Sat Sep 23 20:58:21 2023","interfaces":[{"interface":"back","average":{"1min":2.0209999999999999,"5min":2.504,"15min":1.514},"min":{"1min":0.28899999999999998,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":7.7549999999999999,"5min":15.916,"15min":31.562999999999999},"last":0.81200000000000006},{"interface":"front","average":{"1min":2.2080000000000002,"5min":2.5449999999999999,"15min":1.589},"min":{"1min":0.433,"5min":0.433,"15min":0.41899999999999998},"max":{"1min":7.7300000000000004,"5min":15.882,"15min":31.574000000000002},"last":0.83299999999999996}]},{"osd":5,"last update":"Sat Sep 23 20:58:35 2023","interfaces":[{"interface":"back","average":{"1min":0.81399999999999995,"5min":0.61399999999999999,"15min":0.52900000000000003},"min":{"1min":0.29199999999999998,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":2.9470000000000001,"5min":3.3279999999999998,"15min":3.3279999999999998},"last":0.316},{"interface":"front","average":{"1min":0.77900000000000003,"5min":0.68500000000000005,"15min":0.57799999999999996},"min":{"1min":0.33200000000000002,"5min":0.251,"15min":0.251},"max":{"1min":1.9350000000000001,"5min":7.7619999999999996,"15min":7.7619999999999996},"last":0.34799999999999998}]},{"osd":6,"last update":"Sat Sep 23 20:58:41 2023","interfaces":[{"interface":"back","average":{"1min":0.90300000000000002,"5min":0.67700000000000005,"15min":0.54800000000000004},"min":{"1min":0.29899999999999999,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":3.4780000000000002,"5min":4.6909999999999998,"15min":4.6909999999999998},"last":0.434},{"interface":"front","average":{"1min":0.82299999999999995,"5min":0.71599999999999997,"15min":0.66800000000000004},"min":{"1min":0.36799999999999999,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":3.456,"5min":3.456,"15min":3.456},"last":0.42199999999999999}]},{"osd":7,"last update":"Sat Sep 23 20:58:20 2023","interfaces":[{"interface":"back","average":{"1min":0.84199999999999997,"5min":0.83099999999999996,"15min":0.67100000000000004},"min":{"1min":0.442,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":3.327,"5min":7.383,"15min":7.383},"last":0.45100000000000001},{"interface":"front","average":{"1min":0.93899999999999995,"5min":0.80800000000000005,"15min":0.66000000000000003},"min":{"1min":0.36399999999999999,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":3.883,"5min":7.8310000000000004,"15min":7.8310000000000004},"last":0.40999999999999998}]}]},{"osd":5,"up_from":39,"seq":167503724773,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1787836,"kb_used_data":1659124,"kb_used_omap":0,"kb_used_meta":128704,"kb_avail":91957316,"statfs":{"total":95995035648,"available":94164291584,"internally_reserved":0,"allocated":1698942976,"data_stored":1698120202,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":131792896},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:58:37 2023","interfaces":[{"interface":"back","average":{"1min":0.78600000000000003,"5min":1.03,"15min":0.72699999999999998},"min":{"1min":0.23899999999999999,"5min":0.20399999999999999,"15min":0.20399999999999999},"max":{"1min":3.6949999999999998,"5min":7.766,"15min":7.766},"last":12.56},{"interface":"front","average":{"1min":1.8049999999999999,"5min":1.6779999999999999,"15min":0.98099999999999998},"min":{"1min":0.25,"5min":0.223,"15min":0.223},"max":{"1min":7.8120000000000003,"5min":15.037000000000001,"15min":15.037000000000001},"last":16.219999999999999}]},{"osd":1,"last update":"Sat Sep 23 20:58:16 2023","interfaces":[{"interface":"back","average":{"1min":1.5549999999999999,"5min":1.603,"15min":0.95199999999999996},"min":{"1min":0.27100000000000002,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":6.7409999999999997,"5min":8.3520000000000003,"15min":8.3520000000000003},"last":4.1020000000000003},{"interface":"front","average":{"1min":1.9590000000000001,"5min":1.7450000000000001,"15min":0.97699999999999998},"min":{"1min":0.28100000000000003,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":7.4669999999999996,"5min":10.285,"15min":10.285},"last":4.1130000000000004}]},{"osd":2,"last update":"Sat Sep 23 20:57:50 2023","interfaces":[{"interface":"back","average":{"1min":1.2410000000000001,"5min":1.575,"15min":0.88500000000000001},"min":{"1min":0.26200000000000001,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":10.579000000000001,"5min":12.651,"15min":12.651},"last":4.3360000000000003},{"interface":"front","average":{"1min":1.0669999999999999,"5min":1.5449999999999999,"15min":0.91900000000000004},"min":{"1min":0.26100000000000001,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":5.2949999999999999,"5min":8.3209999999999997,"15min":8.3209999999999997},"last":12.300000000000001}]},{"osd":3,"last update":"Sat Sep 23 20:58:16 2023","interfaces":[{"interface":"back","average":{"1min":2.331,"5min":2.0670000000000002,"15min":1.153},"min":{"1min":0.39200000000000002,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":8.6579999999999995,"5min":9.5489999999999995,"15min":9.5489999999999995},"last":12.188000000000001},{"interface":"front","average":{"1min":1.6000000000000001,"5min":1.6040000000000001,"15min":0.97499999999999998},"min":{"1min":0.28899999999999998,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":10.457000000000001,"5min":10.457000000000001,"15min":10.457000000000001},"last":7.258}]},{"osd":4,"last update":"Sat Sep 23 20:58:33 2023","interfaces":[{"interface":"back","average":{"1min":0.48799999999999999,"5min":0.59099999999999997,"15min":0.52900000000000003},"min":{"1min":0.252,"5min":0.252,"15min":0.252},"max":{"1min":2.6909999999999998,"5min":3.8479999999999999,"15min":3.8479999999999999},"last":2.0179999999999998},{"interface":"front","average":{"1min":0.41899999999999998,"5min":0.64100000000000001,"15min":0.54900000000000004},"min":{"1min":0.29299999999999998,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":0.747,"5min":3.0680000000000001,"15min":3.0680000000000001},"last":3.3530000000000002}]},{"osd":6,"last update":"Sat Sep 23 20:58:37 2023","interfaces":[{"interface":"back","average":{"1min":0.46600000000000003,"5min":0.59099999999999997,"15min":0.57099999999999995},"min":{"1min":0.28499999999999998,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":1.0509999999999999,"5min":2.9820000000000002,"15min":2.9820000000000002},"last":2.476},{"interface":"front","average":{"1min":0.499,"5min":0.69899999999999995,"15min":0.72899999999999998},"min":{"1min":0.36299999999999999,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":0.78500000000000003,"5min":3.6560000000000001,"15min":3.6560000000000001},"last":1.9830000000000001}]},{"osd":7,"last update":"Sat Sep 23 20:58:13 2023","interfaces":[{"interface":"back","average":{"1min":0.66500000000000004,"5min":0.76600000000000001,"15min":0.63200000000000001},"min":{"1min":0.316,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":2.4809999999999999,"5min":8.5649999999999995,"15min":8.5649999999999995},"last":2.516},{"interface":"front","average":{"1min":0.60299999999999998,"5min":0.74099999999999999,"15min":0.64400000000000002},"min":{"1min":0.35399999999999998,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":2.7109999999999999,"5min":4.5430000000000001,"15min":4.5430000000000001},"last":2.0430000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":2685800448,"data_stored":2685796376,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":2321420288,"data_stored":2321416216,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":2361917440,"data_stored":2361917440,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":2217279488,"data_stored":2217279488,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1662058496,"data_stored":1662058496,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1689190400,"data_stored":1689190400,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1646071808,"data_stored":1646067736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1669070848,"data_stored":1669070848,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T20:58:45.986 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph status --format=json 2023-09-23T20:58:45.991 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:58:45.997 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:58:45.997 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:58:46.097 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:58:46.107 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:58:46 smithi026 ceph-mon[175151]: pgmap v564: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 5.0 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 600 op/s; 10981/244755 objects degraded (4.487%); 4186/244755 objects misplaced (1.710%); 1.6 MiB/s, 24 objects/s recovering 2023-09-23T20:58:46.107 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:58:46 smithi026 ceph-mon[179688]: pgmap v564: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 5.0 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 600 op/s; 10981/244755 objects degraded (4.487%); 4186/244755 objects misplaced (1.710%); 1.6 MiB/s, 24 objects/s recovering 2023-09-23T20:58:46.223 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:58:46.333 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:58:46.342 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:58:46.343 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:58:46.367 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:58:46 smithi070 ceph-mon[168282]: pgmap v564: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 5.0 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 600 op/s; 10981/244755 objects degraded (4.487%); 4186/244755 objects misplaced (1.710%); 1.6 MiB/s, 24 objects/s recovering 2023-09-23T20:58:46.444 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:58:46.533 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:58:46.658 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:58:46.658 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:58:46.659 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:58:46.691 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T20:58:46.760 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:58:46.809 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:58:46.846 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:58:46.858 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:58:46.862 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:58:46.862 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:58:46.963 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:58:47.010 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:58:47.317 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:58:47.320 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:58:47.321 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:58:47.321 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:58:47 smithi070 ceph-mon[168282]: from='client.55111 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:58:47.321 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:58:47 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:58:47.322 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:58:47 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:58:47.392 INFO:teuthology.orchestra.run.smithi070.stdout:{ 2023-09-23T20:58:47.392 INFO:teuthology.orchestra.run.smithi070.stdout: "ops": [ 2023-09-23T20:58:47.392 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:58:47.392 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.35100.0:125148 7.7 7:eaf53988:::benchmark_data_smithi026_260489_object23386:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e236)", 2023-09-23T20:58:47.392 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T20:58:47.381004+0000", 2023-09-23T20:58:47.392 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 0.010584557, 2023-09-23T20:58:47.392 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.010616749, 2023-09-23T20:58:47.392 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T20:58:47.392 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "waiting for sub ops", 2023-09-23T20:58:47.392 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T20:58:47.393 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.35100", 2023-09-23T20:58:47.393 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/4108972961", 2023-09-23T20:58:47.393 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 125148 2023-09-23T20:58:47.393 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:58:47.393 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T20:58:47.393 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:58:47.393 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T20:58:47.393 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:58:47.381004+0000", 2023-09-23T20:58:47.393 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T20:58:47.393 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:58:47.393 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:58:47.393 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T20:58:47.394 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:58:47.381004+0000", 2023-09-23T20:58:47.394 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T20:58:47.394 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:58:47.394 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:58:47.394 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T20:58:47.394 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:58:47.381003+0000", 2023-09-23T20:58:47.394 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.999999 2023-09-23T20:58:47.394 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:58:47.394 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:58:47.394 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T20:58:47.394 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:58:47.381008+0000", 2023-09-23T20:58:47.394 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.4859999999999997e-06 2023-09-23T20:58:47.395 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:58:47.395 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:58:47.395 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T20:58:47.395 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:58:47.381021+0000", 2023-09-23T20:58:47.395 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.259e-05 2023-09-23T20:58:47.395 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:58:47.395 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:58:47.395 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T20:58:47.395 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:58:47.381027+0000", 2023-09-23T20:58:47.395 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.4980000000000003e-06 2023-09-23T20:58:47.395 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:58:47.395 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:58:47.396 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T20:58:47.396 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:58:47.381088+0000", 2023-09-23T20:58:47.396 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.0897999999999999e-05 2023-09-23T20:58:47.396 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:58:47.396 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:58:47.396 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T20:58:47.396 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:58:47.381241+0000", 2023-09-23T20:58:47.396 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00015325500000000001 2023-09-23T20:58:47.396 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:58:47.396 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:58:47.396 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 0,3", 2023-09-23T20:58:47.396 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:58:47.381381+0000", 2023-09-23T20:58:47.396 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000140197 2023-09-23T20:58:47.397 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:58:47.397 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:58:47.397 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T20:58:47.397 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:58:47.388016+0000", 2023-09-23T20:58:47.397 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0066350710000000002 2023-09-23T20:58:47.397 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T20:58:47.397 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T20:58:47.397 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T20:58:47.397 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:58:47.397 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:58:47.397 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.35100.0:125152 7.7 7:e248e596:::benchmark_data_smithi026_260489_object23390:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e236)", 2023-09-23T20:58:47.397 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T20:58:47.381040+0000", 2023-09-23T20:58:47.398 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 0.010548567999999999, 2023-09-23T20:58:47.398 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.010671218999999999, 2023-09-23T20:58:47.398 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T20:58:47.398 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "waiting for sub ops", 2023-09-23T20:58:47.398 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T20:58:47.398 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.35100", 2023-09-23T20:58:47.398 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/4108972961", 2023-09-23T20:58:47.398 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 125152 2023-09-23T20:58:47.398 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:58:47.398 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T20:58:47.398 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:58:47.398 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T20:58:47.399 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:58:47.381040+0000", 2023-09-23T20:58:47.399 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T20:58:47.399 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:58:47.399 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:58:47.399 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T20:58:47.399 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:58:47.381040+0000", 2023-09-23T20:58:47.399 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T20:58:47.399 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:58:47.399 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:58:47.399 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T20:58:47.399 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:58:47.381039+0000", 2023-09-23T20:58:47.399 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999995 2023-09-23T20:58:47.400 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:58:47.400 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:58:47.400 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T20:58:47.400 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:58:47.381042+0000", 2023-09-23T20:58:47.400 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.6740000000000001e-06 2023-09-23T20:58:47.400 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:58:47.400 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:58:47.400 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T20:58:47.400 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:58:47.381051+0000", 2023-09-23T20:58:47.400 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.1700000000000003e-06 2023-09-23T20:58:47.400 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:58:47.401 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:58:47.401 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T20:58:47.401 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:58:47.381056+0000", 2023-09-23T20:58:47.401 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.198e-06 2023-09-23T20:58:47.401 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:58:47.401 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:58:47.401 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T20:58:47.401 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:58:47.385274+0000", 2023-09-23T20:58:47.401 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0042173810000000001 2023-09-23T20:58:47.401 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:58:47.401 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:58:47.401 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T20:58:47.401 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:58:47.385374+0000", 2023-09-23T20:58:47.402 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00010090399999999999 2023-09-23T20:58:47.402 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:58:47.402 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:58:47.402 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 0,3", 2023-09-23T20:58:47.402 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:58:47.385462+0000", 2023-09-23T20:58:47.402 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.7261000000000004e-05 2023-09-23T20:58:47.402 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:58:47.402 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:58:47.402 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T20:58:47.402 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:58:47.388130+0000", 2023-09-23T20:58:47.402 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0026682799999999999 2023-09-23T20:58:47.403 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T20:58:47.403 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T20:58:47.403 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T20:58:47.403 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:58:47.403 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:58:47.403 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.35100.0:125155 7.c 7:39f2884a:::benchmark_data_smithi026_260489_object23393:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e236)", 2023-09-23T20:58:47.403 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T20:58:47.388626+0000", 2023-09-23T20:58:47.403 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 0.0029620860000000001, 2023-09-23T20:58:47.403 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0031678819999999999, 2023-09-23T20:58:47.403 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T20:58:47.403 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "waiting for sub ops", 2023-09-23T20:58:47.403 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T20:58:47.403 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.35100", 2023-09-23T20:58:47.404 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/4108972961", 2023-09-23T20:58:47.404 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 125155 2023-09-23T20:58:47.404 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:58:47.404 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T20:58:47.404 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:58:47.404 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T20:58:47.404 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:58:47.388626+0000", 2023-09-23T20:58:47.404 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T20:58:47.404 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:58:47.404 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:58:47.404 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T20:58:47.404 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:58:47.388626+0000", 2023-09-23T20:58:47.404 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T20:58:47.404 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:58:47.405 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:58:47.405 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T20:58:47.405 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:58:47.388625+0000", 2023-09-23T20:58:47.405 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T20:58:47.405 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:58:47.405 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:58:47.405 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T20:58:47.405 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:58:47.388631+0000", 2023-09-23T20:58:47.405 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.4790000000000001e-06 2023-09-23T20:58:47.405 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:58:47.405 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:58:47.405 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T20:58:47.405 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:58:47.388645+0000", 2023-09-23T20:58:47.406 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.3889e-05 2023-09-23T20:58:47.406 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:58:47.406 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:58:47.406 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T20:58:47.406 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:58:47.388651+0000", 2023-09-23T20:58:47.406 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.6869999999999999e-06 2023-09-23T20:58:47.406 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:58:47.406 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:58:47.406 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T20:58:47.406 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:58:47.388711+0000", 2023-09-23T20:58:47.406 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.9898000000000001e-05 2023-09-23T20:58:47.407 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:58:47.407 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:58:47.407 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T20:58:47.407 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:58:47.388841+0000", 2023-09-23T20:58:47.407 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00013027500000000001 2023-09-23T20:58:47.407 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:58:47.407 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:58:47.407 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 3,7", 2023-09-23T20:58:47.407 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:58:47.388939+0000", 2023-09-23T20:58:47.407 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.7774000000000003e-05 2023-09-23T20:58:47.407 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T20:58:47.407 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T20:58:47.408 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T20:58:47.408 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:58:47.408 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:58:47.408 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.35100.0:125156 7.7 7:eadd354b:::benchmark_data_smithi026_260489_object23394:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e236)", 2023-09-23T20:58:47.408 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T20:58:47.388665+0000", 2023-09-23T20:58:47.408 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 0.0029231359999999998, 2023-09-23T20:58:47.408 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.003217709, 2023-09-23T20:58:47.408 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T20:58:47.408 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "waiting for sub ops", 2023-09-23T20:58:47.408 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T20:58:47.408 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.35100", 2023-09-23T20:58:47.408 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/4108972961", 2023-09-23T20:58:47.408 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 125156 2023-09-23T20:58:47.408 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:58:47.409 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T20:58:47.409 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:58:47.409 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T20:58:47.409 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:58:47.388665+0000", 2023-09-23T20:58:47.409 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T20:58:47.409 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:58:47.409 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:58:47.409 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T20:58:47.409 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:58:47.388665+0000", 2023-09-23T20:58:47.409 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T20:58:47.409 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:58:47.409 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:58:47.409 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T20:58:47.410 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:58:47.388665+0000", 2023-09-23T20:58:47.410 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999995 2023-09-23T20:58:47.410 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:58:47.410 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:58:47.410 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T20:58:47.410 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:58:47.388668+0000", 2023-09-23T20:58:47.410 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.0029999999999999e-06 2023-09-23T20:58:47.410 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:58:47.410 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:58:47.410 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T20:58:47.410 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:58:47.388679+0000", 2023-09-23T20:58:47.411 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.1168999999999999e-05 2023-09-23T20:58:47.411 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:58:47.411 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:58:47.411 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T20:58:47.411 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:58:47.388682+0000", 2023-09-23T20:58:47.411 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.3110000000000001e-06 2023-09-23T20:58:47.411 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:58:47.411 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:58:47.411 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T20:58:47.411 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:58:47.388717+0000", 2023-09-23T20:58:47.411 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.5370000000000002e-05 2023-09-23T20:58:47.411 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:58:47.412 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:58:47.412 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T20:58:47.412 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:58:47.388824+0000", 2023-09-23T20:58:47.412 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000106517 2023-09-23T20:58:47.412 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:58:47.412 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:58:47.412 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 0,3", 2023-09-23T20:58:47.412 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:58:47.388899+0000", 2023-09-23T20:58:47.412 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.4913999999999995e-05 2023-09-23T20:58:47.412 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T20:58:47.412 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T20:58:47.412 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T20:58:47.413 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:58:47.413 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:58:47.413 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_repop_reply(client.35100.0:125148 7.7 e236/199)", 2023-09-23T20:58:47.413 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T20:58:47.391734+0000", 2023-09-23T20:58:47.413 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 4294967295.9998541, 2023-09-23T20:58:47.413 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000218684, 2023-09-23T20:58:47.413 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T20:58:47.413 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "queued for pg", 2023-09-23T20:58:47.413 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T20:58:47.413 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:58:47.413 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T20:58:47.413 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:58:47.391734+0000", 2023-09-23T20:58:47.414 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T20:58:47.414 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:58:47.414 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:58:47.414 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T20:58:47.414 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:58:47.391734+0000", 2023-09-23T20:58:47.414 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T20:58:47.414 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:58:47.414 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:58:47.414 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T20:58:47.414 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:58:47.391733+0000", 2023-09-23T20:58:47.414 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.999999 2023-09-23T20:58:47.415 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:58:47.415 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:58:47.415 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T20:58:47.415 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:58:47.391738+0000", 2023-09-23T20:58:47.415 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.3650000000000003e-06 2023-09-23T20:58:47.415 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:58:47.415 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:58:47.415 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T20:58:47.415 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:58:47.391751+0000", 2023-09-23T20:58:47.415 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.2194999999999999e-05 2023-09-23T20:58:47.415 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T20:58:47.415 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T20:58:47.415 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T20:58:47.416 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:58:47.391757+0000", 2023-09-23T20:58:47.416 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.7089999999999996e-06 2023-09-23T20:58:47.416 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T20:58:47.416 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T20:58:47.416 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T20:58:47.416 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T20:58:47.416 INFO:teuthology.orchestra.run.smithi070.stdout: ], 2023-09-23T20:58:47.416 INFO:teuthology.orchestra.run.smithi070.stdout: "num_ops": 5 2023-09-23T20:58:47.416 INFO:teuthology.orchestra.run.smithi070.stdout:} 2023-09-23T20:58:47.422 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:58:47.480 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:58:47 smithi026 ceph-mon[179688]: from='client.55111 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:58:47.480 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:58:47 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:58:47.480 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:58:47 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:58:47.481 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:58:47 smithi026 ceph-mon[175151]: from='client.55111 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:58:47.481 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:58:47 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:58:47.481 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:58:47 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:58:47.489 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:58:47.582 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T20:58:47.826 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:58:47.834 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:58:47.835 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:58:47.887 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_blocked_ops 2023-09-23T20:58:47.935 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:58:48.025 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:58:48.571 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:58:48.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:58:48 smithi070 ceph-mon[168282]: pgmap v565: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 4.9 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 538 op/s; 10981/243042 objects degraded (4.518%); 4186/243042 objects misplaced (1.722%); 755 KiB/s, 11 objects/s recovering 2023-09-23T20:58:48.631 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:58:48 smithi026 ceph-mon[175151]: pgmap v565: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 4.9 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 538 op/s; 10981/243042 objects degraded (4.518%); 4186/243042 objects misplaced (1.722%); 755 KiB/s, 11 objects/s recovering 2023-09-23T20:58:48.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:58:48 smithi026 ceph-mon[179688]: pgmap v565: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 4.9 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 538 op/s; 10981/243042 objects degraded (4.518%); 4186/243042 objects misplaced (1.722%); 755 KiB/s, 11 objects/s recovering 2023-09-23T20:58:48.918 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:58:48.922 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:58:48.922 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:58:49.023 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:58:49.056 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:58:49.100 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:58:49.106 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:58:49.106 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:58:49.207 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:58:49.251 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:58:49.291 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:58:49.294 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:58:49.295 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:58:49.395 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:58:49.425 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:58:49.538 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:58:49.541 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:58:49 smithi070 ceph-mon[168282]: pgmap v566: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 4.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 606 op/s; 11078/238245 objects degraded (4.650%); 3985/238245 objects misplaced (1.673%); 1.4 MiB/s, 22 objects/s recovering 2023-09-23T20:58:49.541 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:58:49.542 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:58:49.600 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:58:49.603 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:58:49 smithi026 ceph-mon[175151]: pgmap v566: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 4.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 606 op/s; 11078/238245 objects degraded (4.650%); 3985/238245 objects misplaced (1.673%); 1.4 MiB/s, 22 objects/s recovering 2023-09-23T20:58:49.603 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:58:49 smithi026 ceph-mon[179688]: pgmap v566: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 4.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 606 op/s; 11078/238245 objects degraded (4.650%); 3985/238245 objects misplaced (1.673%); 1.4 MiB/s, 22 objects/s recovering 2023-09-23T20:58:49.643 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:58:49.677 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:58:49.715 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:58:49.723 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:58:49.723 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:58:49.789 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:58:49.824 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:58:49.884 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:58:49 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[164029]: ::ffff:172.21.15.70 - - [23/Sep/2023:20:58:49] "GET /metrics HTTP/1.1" 200 36320 "" "Prometheus/2.43.0" 2023-09-23T20:58:49.949 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:58:50.034 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:58:50.045 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:58:50.045 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:58:50.146 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:58:50.214 INFO:teuthology.orchestra.run.smithi070.stdout:{ 2023-09-23T20:58:50.214 INFO:teuthology.orchestra.run.smithi070.stdout: "ops": [], 2023-09-23T20:58:50.214 INFO:teuthology.orchestra.run.smithi070.stdout: "complaint_time": 30, 2023-09-23T20:58:50.214 INFO:teuthology.orchestra.run.smithi070.stdout: "num_blocked_ops": 0 2023-09-23T20:58:50.214 INFO:teuthology.orchestra.run.smithi070.stdout:} 2023-09-23T20:58:50.248 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:58:50.380 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:58:50.390 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:58:50.390 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:58:50.491 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:58:50.577 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:58:50 smithi026 ceph-mon[179688]: Health check update: Degraded data redundancy: 11078/238245 objects degraded (4.650%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T20:58:50.577 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:58:50 smithi026 ceph-mon[175151]: Health check update: Degraded data redundancy: 11078/238245 objects degraded (4.650%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T20:58:50.590 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:58:50.622 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:58:50 smithi070 ceph-mon[168282]: Health check update: Degraded data redundancy: 11078/238245 objects degraded (4.650%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T20:58:50.842 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:58:50.846 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:58:50.847 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:58:50.948 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:58:50.983 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:58:51.545 INFO:teuthology.orchestra.run.smithi026.stdout: 2023-09-23T20:58:51.545 INFO:teuthology.orchestra.run.smithi026.stdout:{"fsid":"b3f8e4ec-5a50-11ee-8db4-212e2dc638e7","health":{"status":"HEALTH_WARN","checks":{"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 11078/234468 objects degraded (4.725%), 3 pgs degraded, 3 pgs undersized","count":6},"muted":false}},"mutes":[]},"election_epoch":30,"quorum":[0,1,2],"quorum_names":["a","b","c"],"quorum_age":650,"monmap":{"epoch":4,"min_mon_release_name":"reef","num_mons":3},"osdmap":{"epoch":236,"num_osds":8,"num_up_osds":8,"osd_up_since":1695502534,"num_in_osds":8,"osd_in_since":1695502470,"num_remapped_pgs":3},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":106},{"state_name":"active+recovery_wait+undersized+degraded+remapped","count":2},{"state_name":"active+recovering+undersized+degraded+remapped","count":1}],"num_pgs":109,"num_pools":5,"num_objects":78156,"data_bytes":5120989112,"bytes_used":17341906944,"bytes_avail":750618378240,"bytes_total":767960285184,"degraded_objects":11078,"degraded_total":234468,"degraded_ratio":0.047247385570738862,"misplaced_objects":3985,"misplaced_total":234468,"misplaced_ratio":0.016995922684545438,"recovering_objects_per_sec":22,"recovering_bytes_per_sec":1454028,"recovering_keys_per_sec":0,"num_objects_recovered":222,"num_bytes_recovered":14548992,"num_keys_recovered":0,"write_bytes_sec":0,"read_op_per_sec":0,"write_op_per_sec":601},"fsmap":{"epoch":8,"id":1,"up":1,"in":1,"max":1,"by_rank":[{"filesystem_id":1,"rank":0,"name":"foo.smithi026.znfpnl","status":"up:active","gid":14658}],"up:standby":1},"mgrmap":{"available":true,"num_standbys":1,"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"services":{"dashboard":"https://172.21.15.26:8443/","prometheus":"http://172.21.15.26:9283/"}},"servicemap":{"epoch":21,"modified":"2023-09-23T20:56:34.314324+0000","services":{}},"progress_events":{"8b64cbfe-d367-468b-a773-0e71ee4ff645":{"message":"Global Recovery Event (0s)\n [............................] ","progress":0,"add_to_ceph_s":true},"fc746e9d-3f76-434d-b6d2-8fc9e8a72a5e":{"message":"Upgrade to 18.2.0-460-gbb0daeff (4m)\n [============................] (remaining: 5m)","progress":0.44999998807907104,"add_to_ceph_s":true}}} 2023-09-23T20:58:51.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:58:51 smithi070 ceph-mon[168282]: pgmap v567: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 4.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 601 op/s; 11078/234468 objects degraded (4.725%); 3985/234468 objects misplaced (1.700%); 1.4 MiB/s, 22 objects/s recovering 2023-09-23T20:58:51.630 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:58:51 smithi026 ceph-mon[179688]: pgmap v567: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 4.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 601 op/s; 11078/234468 objects degraded (4.725%); 3985/234468 objects misplaced (1.700%); 1.4 MiB/s, 22 objects/s recovering 2023-09-23T20:58:51.631 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:58:51 smithi026 ceph-mon[175151]: pgmap v567: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 4.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 601 op/s; 11078/234468 objects degraded (4.725%); 3985/234468 objects misplaced (1.700%); 1.4 MiB/s, 22 objects/s recovering 2023-09-23T20:58:52.295 INFO:tasks.cephadm.ceph_manager.ceph:making progress, resetting timeout 2023-09-23T20:58:52.296 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T20:58:52.371 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_historic_ops 2023-09-23T20:58:52.385 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0: {} 2023-09-23T20:58:52.434 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1: {} 2023-09-23T20:58:52.491 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2: {} 2023-09-23T20:58:52.538 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3: {} 2023-09-23T20:58:52.550 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:58:52 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/2298916434' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-23T20:58:52.550 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:58:52 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/2298916434' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-23T20:58:52.577 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4: {} 2023-09-23T20:58:52.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:58:52 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/2298916434' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-23T20:58:52.627 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T20:58:52.660 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6: {} 2023-09-23T20:58:52.711 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T20:58:52.747 INFO:teuthology.orchestra.run.smithi026.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T20:58:52.747 INFO:teuthology.orchestra.run.smithi026.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T20:58:52.747 INFO:teuthology.orchestra.run.smithi026.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T20:58:52.747 INFO:teuthology.orchestra.run.smithi026.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T20:58:52.747 INFO:teuthology.orchestra.run.smithi026.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T20:58:52.747 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T20:58:52.747 INFO:teuthology.orchestra.run.smithi026.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T20:58:52.748 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T20:58:52.876 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:58:52.878 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:58:52.879 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:58:52.980 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:58:53.013 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:58:53.117 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:58:53.123 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:58:53.123 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:58:53.224 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:58:53.282 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:58:53.315 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:58:53 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/2580640235' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:58:53.315 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:58:53 smithi026 ceph-mon[175151]: pgmap v568: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 4.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 585 op/s; 11166/229485 objects degraded (4.866%); 3791/229485 objects misplaced (1.652%); 1.3 MiB/s, 20 objects/s recovering 2023-09-23T20:58:53.320 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:58:53 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/2580640235' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:58:53.320 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:58:53 smithi070 ceph-mon[168282]: pgmap v568: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 4.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 585 op/s; 11166/229485 objects degraded (4.866%); 3791/229485 objects misplaced (1.652%); 1.3 MiB/s, 20 objects/s recovering 2023-09-23T20:58:53.361 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:58:53.446 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:58:53.454 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:58:53.454 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:58:53.555 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:58:53.631 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:58:53 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/2580640235' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:58:53.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:58:53 smithi026 ceph-mon[179688]: pgmap v568: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 4.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 585 op/s; 11166/229485 objects degraded (4.866%); 3791/229485 objects misplaced (1.652%); 1.3 MiB/s, 20 objects/s recovering 2023-09-23T20:58:53.660 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:58:53.681 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:58:53.745 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:58:53.750 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:58:53.750 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:58:53.851 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:58:53.964 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:58:54.038 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:58:54.047 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:58:54.048 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:58:54.149 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:58:54.269 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:58:54.423 INFO:teuthology.orchestra.run.smithi026.stderr:noscrub is set 2023-09-23T20:58:54.584 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:58:54.594 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:58:54.595 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:58:54.596 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:58:54 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/4171419575' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:58:54.596 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:58:54 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:58:54.596 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:58:54 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:58:54.596 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:58:54 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:58:54.597 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:58:54 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/4171419575' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:58:54.597 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:58:54 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:58:54.597 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:58:54 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:58:54.597 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:58:54 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:58:54.691 INFO:teuthology.orchestra.run.smithi070.stdout:ERROR: (22) Invalid argument 2023-09-23T20:58:54.692 INFO:teuthology.orchestra.run.smithi070.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:58:54.696 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:58:54.723 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:58:54 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/4171419575' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:58:54.723 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:58:54 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:58:54.723 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:58:54 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:58:54.723 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:58:54 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:58:54.735 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:58:55.015 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:58:55.019 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:58:55.020 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:58:55.070 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_ops_in_flight 2023-09-23T20:58:55.121 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:58:55.163 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:58:55.201 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:58:55.203 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:58:55.204 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:58:55.304 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:58:55.438 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:58:55.490 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T20:58:55.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:58:55 smithi070 ceph-mon[168282]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:58:55.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:58:55 smithi070 ceph-mon[168282]: failed to encode map e237 with expected crc 2023-09-23T20:58:55.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:58:55 smithi070 ceph-mon[168282]: failed to encode map e237 with expected crc 2023-09-23T20:58:55.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:58:55 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T20:58:55.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:58:55 smithi070 ceph-mon[168282]: osdmap e237: 8 total, 8 up, 8 in 2023-09-23T20:58:55.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:58:55 smithi070 ceph-mon[168282]: pgmap v570: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 4.6 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 580 op/s; 11166/227319 objects degraded (4.912%); 3791/227319 objects misplaced (1.668%); 1.3 MiB/s, 20 objects/s recovering 2023-09-23T20:58:55.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:58:55 smithi070 ceph-mon[168282]: Health check update: Degraded data redundancy: 11166/229485 objects degraded (4.866%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T20:58:55.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:58:55 smithi026 ceph-mon[175151]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:58:55.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:58:55 smithi026 ceph-mon[175151]: failed to encode map e237 with expected crc 2023-09-23T20:58:55.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:58:55 smithi026 ceph-mon[175151]: failed to encode map e237 with expected crc 2023-09-23T20:58:55.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:58:55 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T20:58:55.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:58:55 smithi026 ceph-mon[175151]: osdmap e237: 8 total, 8 up, 8 in 2023-09-23T20:58:55.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:58:55 smithi026 ceph-mon[175151]: pgmap v570: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 4.6 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 580 op/s; 11166/227319 objects degraded (4.912%); 3791/227319 objects misplaced (1.668%); 1.3 MiB/s, 20 objects/s recovering 2023-09-23T20:58:55.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:58:55 smithi026 ceph-mon[175151]: Health check update: Degraded data redundancy: 11166/229485 objects degraded (4.866%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T20:58:55.883 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:58:55 smithi026 ceph-mon[179688]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:58:55.883 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:58:55 smithi026 ceph-mon[179688]: failed to encode map e237 with expected crc 2023-09-23T20:58:55.883 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:58:55 smithi026 ceph-mon[179688]: failed to encode map e237 with expected crc 2023-09-23T20:58:55.883 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:58:55 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T20:58:55.883 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:58:55 smithi026 ceph-mon[179688]: osdmap e237: 8 total, 8 up, 8 in 2023-09-23T20:58:55.883 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:58:55 smithi026 ceph-mon[179688]: pgmap v570: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 4.6 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 580 op/s; 11166/227319 objects degraded (4.912%); 3791/227319 objects misplaced (1.668%); 1.3 MiB/s, 20 objects/s recovering 2023-09-23T20:58:55.883 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:58:55 smithi026 ceph-mon[179688]: Health check update: Degraded data redundancy: 11166/229485 objects degraded (4.866%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T20:58:57.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:58:57 smithi070 ceph-mon[168282]: pgmap v571: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 4.6 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 575 op/s; 11166/225768 objects degraded (4.946%); 3791/225768 objects misplaced (1.679%); 1.3 MiB/s, 20 objects/s recovering 2023-09-23T20:58:57.884 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:58:57 smithi026 ceph-mon[175151]: pgmap v571: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 4.6 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 575 op/s; 11166/225768 objects degraded (4.946%); 3791/225768 objects misplaced (1.679%); 1.3 MiB/s, 20 objects/s recovering 2023-09-23T20:58:57.884 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:58:57 smithi026 ceph-mon[179688]: pgmap v571: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 4.6 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 575 op/s; 11166/225768 objects degraded (4.946%); 3791/225768 objects misplaced (1.679%); 1.3 MiB/s, 20 objects/s recovering 2023-09-23T20:58:58.371 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:58:58.389 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:58:58.389 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:58:58.490 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:58:58.521 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:58:58.560 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:58:58.563 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:58:58.563 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:58:58.664 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:58:58.777 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:58:59.104 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:58:59 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:58:59.104 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:58:59 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:58:59.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:58:59 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:58:59.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:58:59 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:58:59.383 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:58:59 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:58:59.383 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:58:59 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:58:59.883 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:58:59 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[164029]: ::ffff:172.21.15.70 - - [23/Sep/2023:20:58:59] "GET /metrics HTTP/1.1" 200 36320 "" "Prometheus/2.43.0" 2023-09-23T20:59:00.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:59:00 smithi026 ceph-mon[175151]: pgmap v572: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 4.5 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 575 op/s; 11215/220950 objects degraded (5.076%); 3627/220950 objects misplaced (1.642%); 1.4 MiB/s, 22 objects/s recovering 2023-09-23T20:59:00.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:59:00 smithi026 ceph-mon[175151]: Health check update: Degraded data redundancy: 11215/220950 objects degraded (5.076%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T20:59:00.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:59:00 smithi026 ceph-mon[179688]: pgmap v572: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 4.5 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 575 op/s; 11215/220950 objects degraded (5.076%); 3627/220950 objects misplaced (1.642%); 1.4 MiB/s, 22 objects/s recovering 2023-09-23T20:59:00.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:59:00 smithi026 ceph-mon[179688]: Health check update: Degraded data redundancy: 11215/220950 objects degraded (5.076%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T20:59:00.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:59:00 smithi070 ceph-mon[168282]: pgmap v572: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 4.5 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 575 op/s; 11215/220950 objects degraded (5.076%); 3627/220950 objects misplaced (1.642%); 1.4 MiB/s, 22 objects/s recovering 2023-09-23T20:59:00.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:59:00 smithi070 ceph-mon[168282]: Health check update: Degraded data redundancy: 11215/220950 objects degraded (5.076%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T20:59:01.168 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:59:01.177 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:59:01.178 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:59:01.278 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:59:01.350 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:59:01.583 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:59:01.597 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:59:02.086 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:59:02.094 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:59:02.095 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:59:02.196 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:59:02.274 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:59:02.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:59:02 smithi026 ceph-mon[175151]: pgmap v573: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 4.4 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 584 op/s; 11215/216927 objects degraded (5.170%); 3627/216927 objects misplaced (1.672%); 1.4 MiB/s, 22 objects/s recovering 2023-09-23T20:59:02.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:59:02 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:59:02.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:59:02 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:59:02.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:59:02 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:59:02.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:59:02 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:59:02.383 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:59:02 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:59:02.387 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:59:02 smithi026 ceph-mon[179688]: pgmap v573: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 4.4 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 584 op/s; 11215/216927 objects degraded (5.170%); 3627/216927 objects misplaced (1.672%); 1.4 MiB/s, 22 objects/s recovering 2023-09-23T20:59:02.387 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:59:02 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:59:02.387 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:59:02 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:59:02.387 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:59:02 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:59:02.387 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:59:02 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:59:02.387 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:59:02 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:59:02.445 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:59:02.453 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:59:02.454 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:59:02.555 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:59:02.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:59:02 smithi070 ceph-mon[168282]: pgmap v573: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 4.4 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 584 op/s; 11215/216927 objects degraded (5.170%); 3627/216927 objects misplaced (1.672%); 1.4 MiB/s, 22 objects/s recovering 2023-09-23T20:59:02.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:59:02 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:59:02.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:59:02 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:59:02.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:59:02 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:59:02.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:59:02 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:59:02.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:59:02 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:59:02.640 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:59:02.764 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:59:02.772 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:59:02.772 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:59:02.873 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:59:02.901 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:59:02.946 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:59:02.951 INFO:teuthology.orchestra.run.smithi026.stdout: 2023-09-23T20:59:02.953 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:59:02.953 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:59:02.963 INFO:teuthology.orchestra.run.smithi026.stderr:dumped all 2023-09-23T20:59:03.054 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:59:03.132 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:59:03.217 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T20:59:03.993 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:59:04.005 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:59:04.006 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:59:04.052 INFO:teuthology.orchestra.run.smithi026.stdout:{"pg_ready":true,"pg_map":{"version":574,"stamp":"2023-09-23T20:59:02.424883+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4614199224,"num_objects":70423,"num_object_clones":0,"num_object_copies":211269,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":11300,"num_objects_misplaced":3418,"num_objects_unfound":0,"num_objects_dirty":70423,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":133182,"num_write_kb":6512606,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1646,"num_bytes_recovered":106436454,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":120757,"ondisk_log_size":120757,"up":327,"acting":324,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":331,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":15492420,"kb_used_data":13939332,"kb_used_omap":136,"kb_used_meta":1552567,"kb_avail":734468796,"statfs":{"total":767960285184,"available":752096047104,"internally_reserved":0,"allocated":14273875968,"data_stored":14266001848,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":139741,"internal_metadata":1589829155},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":8,"apply_latency_ms":8,"commit_latency_ns":8000000,"apply_latency_ns":8000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-397934592,"num_objects":-6072,"num_object_clones":0,"num_object_copies":-18216,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":134,"num_objects_misplaced":-373,"num_objects_unfound":0,"num_objects_dirty":-6072,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6072,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":239,"num_bytes_recovered":15663104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.007831"},"pg_stats":[{"pgid":"3.1a","version":"68'1","reported_seq":114,"reported_epoch":236,"state":"active+clean","last_fresh":"2023-09-23T20:58:44.072478+0000","last_change":"2023-09-23T20:54:13.946695+0000","last_active":"2023-09-23T20:58:44.072478+0000","last_peered":"2023-09-23T20:58:44.072478+0000","last_clean":"2023-09-23T20:58:44.072478+0000","last_became_active":"2023-09-23T20:54:13.945399+0000","last_became_peered":"2023-09-23T20:54:13.945399+0000","last_unstale":"2023-09-23T20:58:44.072478+0000","last_undegraded":"2023-09-23T20:58:44.072478+0000","last_fullsized":"2023-09-23T20:58:44.072478+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:32:52.365464+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":229,"reported_epoch":236,"state":"active+clean","last_fresh":"2023-09-23T20:58:44.063520+0000","last_change":"2023-09-23T20:54:13.942019+0000","last_active":"2023-09-23T20:58:44.063520+0000","last_peered":"2023-09-23T20:58:44.063520+0000","last_clean":"2023-09-23T20:58:44.063520+0000","last_became_active":"2023-09-23T20:54:13.941647+0000","last_became_peered":"2023-09-23T20:54:13.941647+0000","last_unstale":"2023-09-23T20:58:44.063520+0000","last_undegraded":"2023-09-23T20:58:44.063520+0000","last_fullsized":"2023-09-23T20:58:44.063520+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:30:54.400944+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":206,"reported_epoch":236,"state":"active+clean","last_fresh":"2023-09-23T20:58:44.070931+0000","last_change":"2023-09-23T20:54:13.941394+0000","last_active":"2023-09-23T20:58:44.070931+0000","last_peered":"2023-09-23T20:58:44.070931+0000","last_clean":"2023-09-23T20:58:44.070931+0000","last_became_active":"2023-09-23T20:54:13.941042+0000","last_became_peered":"2023-09-23T20:54:13.941042+0000","last_unstale":"2023-09-23T20:58:44.070931+0000","last_undegraded":"2023-09-23T20:58:44.070931+0000","last_fullsized":"2023-09-23T20:58:44.070931+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:19:41.401184+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":221,"reported_epoch":236,"state":"active+clean","last_fresh":"2023-09-23T20:58:44.074119+0000","last_change":"2023-09-23T20:57:44.979811+0000","last_active":"2023-09-23T20:58:44.074119+0000","last_peered":"2023-09-23T20:58:44.074119+0000","last_clean":"2023-09-23T20:58:44.074119+0000","last_became_active":"2023-09-23T20:57:44.979515+0000","last_became_peered":"2023-09-23T20:57:44.979515+0000","last_unstale":"2023-09-23T20:58:44.074119+0000","last_undegraded":"2023-09-23T20:58:44.074119+0000","last_fullsized":"2023-09-23T20:58:44.074119+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:38:22.162140+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":220,"reported_epoch":236,"state":"active+clean","last_fresh":"2023-09-23T20:58:44.062563+0000","last_change":"2023-09-23T20:51:10.833613+0000","last_active":"2023-09-23T20:58:44.062563+0000","last_peered":"2023-09-23T20:58:44.062563+0000","last_clean":"2023-09-23T20:58:44.062563+0000","last_became_active":"2023-09-23T20:51:10.833208+0000","last_became_peered":"2023-09-23T20:51:10.833208+0000","last_unstale":"2023-09-23T20:58:44.062563+0000","last_undegraded":"2023-09-23T20:58:44.062563+0000","last_fullsized":"2023-09-23T20:58:44.062563+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:51:28.246014+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":202,"reported_epoch":236,"state":"active+clean","last_fresh":"2023-09-23T20:58:44.049972+0000","last_change":"2023-09-23T20:55:35.958384+0000","last_active":"2023-09-23T20:58:44.049972+0000","last_peered":"2023-09-23T20:58:44.049972+0000","last_clean":"2023-09-23T20:58:44.049972+0000","last_became_active":"2023-09-23T20:55:35.954563+0000","last_became_peered":"2023-09-23T20:55:35.954563+0000","last_unstale":"2023-09-23T20:58:44.049972+0000","last_undegraded":"2023-09-23T20:58:44.049972+0000","last_fullsized":"2023-09-23T20:58:44.049972+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:37:48.332259+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":211,"reported_epoch":236,"state":"active+clean","last_fresh":"2023-09-23T20:58:44.068317+0000","last_change":"2023-09-23T20:54:31.885514+0000","last_active":"2023-09-23T20:58:44.068317+0000","last_peered":"2023-09-23T20:58:44.068317+0000","last_clean":"2023-09-23T20:58:44.068317+0000","last_became_active":"2023-09-23T20:54:31.884986+0000","last_became_peered":"2023-09-23T20:54:31.884986+0000","last_unstale":"2023-09-23T20:58:44.068317+0000","last_undegraded":"2023-09-23T20:58:44.068317+0000","last_fullsized":"2023-09-23T20:58:44.068317+0000","mapping_epoch":136,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:26:08.100220+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":217,"reported_epoch":236,"state":"active+clean","last_fresh":"2023-09-23T20:58:44.062633+0000","last_change":"2023-09-23T20:55:35.951889+0000","last_active":"2023-09-23T20:58:44.062633+0000","last_peered":"2023-09-23T20:58:44.062633+0000","last_clean":"2023-09-23T20:58:44.062633+0000","last_became_active":"2023-09-23T20:55:35.951563+0000","last_became_peered":"2023-09-23T20:55:35.951563+0000","last_unstale":"2023-09-23T20:58:44.062633+0000","last_undegraded":"2023-09-23T20:58:44.062633+0000","last_fullsized":"2023-09-23T20:58:44.062633+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:41:53.098781+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"79'161","reported_seq":523,"reported_epoch":236,"state":"active+clean","last_fresh":"2023-09-23T20:58:44.063148+0000","last_change":"2023-09-23T20:57:25.096559+0000","last_active":"2023-09-23T20:58:44.063148+0000","last_peered":"2023-09-23T20:58:44.063148+0000","last_clean":"2023-09-23T20:58:44.063148+0000","last_became_active":"2023-09-23T20:57:15.975674+0000","last_became_peered":"2023-09-23T20:57:15.975674+0000","last_unstale":"2023-09-23T20:58:44.063148+0000","last_undegraded":"2023-09-23T20:58:44.063148+0000","last_fullsized":"2023-09-23T20:58:44.063148+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:25.096401+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:25.096401+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:05:05.448404+0000","scrub_duration":0.002859393,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":110,"reported_epoch":236,"state":"active+clean","last_fresh":"2023-09-23T20:58:44.066959+0000","last_change":"2023-09-23T20:54:31.890678+0000","last_active":"2023-09-23T20:58:44.066959+0000","last_peered":"2023-09-23T20:58:44.066959+0000","last_clean":"2023-09-23T20:58:44.066959+0000","last_became_active":"2023-09-23T20:54:31.890469+0000","last_became_peered":"2023-09-23T20:54:31.890469+0000","last_unstale":"2023-09-23T20:58:44.066959+0000","last_undegraded":"2023-09-23T20:58:44.066959+0000","last_fullsized":"2023-09-23T20:58:44.066959+0000","mapping_epoch":136,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:55:08.790146+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":178,"reported_epoch":236,"state":"active+clean","last_fresh":"2023-09-23T20:58:44.063022+0000","last_change":"2023-09-23T20:55:35.952661+0000","last_active":"2023-09-23T20:58:44.063022+0000","last_peered":"2023-09-23T20:58:44.063022+0000","last_clean":"2023-09-23T20:58:44.063022+0000","last_became_active":"2023-09-23T20:55:35.952349+0000","last_became_peered":"2023-09-23T20:55:35.952349+0000","last_unstale":"2023-09-23T20:58:44.063022+0000","last_undegraded":"2023-09-23T20:58:44.063022+0000","last_fullsized":"2023-09-23T20:58:44.063022+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:00:03.110126+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":54,"reported_epoch":236,"state":"active+clean","last_fresh":"2023-09-23T20:58:44.063605+0000","last_change":"2023-09-23T20:56:45.988607+0000","last_active":"2023-09-23T20:58:44.063605+0000","last_peered":"2023-09-23T20:58:44.063605+0000","last_clean":"2023-09-23T20:58:44.063605+0000","last_became_active":"2023-09-23T20:56:45.988206+0000","last_became_peered":"2023-09-23T20:56:45.988206+0000","last_unstale":"2023-09-23T20:58:44.063605+0000","last_undegraded":"2023-09-23T20:58:44.063605+0000","last_fullsized":"2023-09-23T20:58:44.063605+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":205,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:01:19.783688+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,4],"acting":[3,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":54,"reported_epoch":236,"state":"active+clean","last_fresh":"2023-09-23T20:58:44.067149+0000","last_change":"2023-09-23T20:56:18.139142+0000","last_active":"2023-09-23T20:58:44.067149+0000","last_peered":"2023-09-23T20:58:44.067149+0000","last_clean":"2023-09-23T20:58:44.067149+0000","last_became_active":"2023-09-23T20:56:18.138844+0000","last_became_peered":"2023-09-23T20:56:18.138844+0000","last_unstale":"2023-09-23T20:58:44.067149+0000","last_undegraded":"2023-09-23T20:58:44.067149+0000","last_fullsized":"2023-09-23T20:58:44.067149+0000","mapping_epoch":192,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":193,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:09:02.217683+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,2],"acting":[5,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.14","version":"79'22","reported_seq":141,"reported_epoch":236,"state":"active+clean","last_fresh":"2023-09-23T20:58:44.072760+0000","last_change":"2023-09-23T20:55:35.949418+0000","last_active":"2023-09-23T20:58:44.072760+0000","last_peered":"2023-09-23T20:58:44.072760+0000","last_clean":"2023-09-23T20:58:44.072760+0000","last_became_active":"2023-09-23T20:55:35.948566+0000","last_became_peered":"2023-09-23T20:55:35.948566+0000","last_unstale":"2023-09-23T20:58:44.072760+0000","last_undegraded":"2023-09-23T20:58:44.072760+0000","last_fullsized":"2023-09-23T20:58:44.072760+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:59:37.433128+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4734,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":131,"reported_epoch":236,"state":"active+clean","last_fresh":"2023-09-23T20:58:44.072668+0000","last_change":"2023-09-23T20:55:35.949178+0000","last_active":"2023-09-23T20:58:44.072668+0000","last_peered":"2023-09-23T20:58:44.072668+0000","last_clean":"2023-09-23T20:58:44.072668+0000","last_became_active":"2023-09-23T20:55:35.948795+0000","last_became_peered":"2023-09-23T20:55:35.948795+0000","last_unstale":"2023-09-23T20:58:44.072668+0000","last_undegraded":"2023-09-23T20:58:44.072668+0000","last_fullsized":"2023-09-23T20:58:44.072668+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:13:08.996932+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.11","version":"79'161","reported_seq":465,"reported_epoch":236,"state":"active+clean","last_fresh":"2023-09-23T20:58:44.064612+0000","last_change":"2023-09-23T20:57:28.756796+0000","last_active":"2023-09-23T20:58:44.064612+0000","last_peered":"2023-09-23T20:58:44.064612+0000","last_clean":"2023-09-23T20:58:44.064612+0000","last_became_active":"2023-09-23T20:57:15.980606+0000","last_became_peered":"2023-09-23T20:57:15.980606+0000","last_unstale":"2023-09-23T20:58:44.064612+0000","last_undegraded":"2023-09-23T20:58:44.064612+0000","last_fullsized":"2023-09-23T20:58:44.064612+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:28.756634+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:28.756634+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:09:19.150791+0000","scrub_duration":0.0064925119999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":157,"reported_epoch":236,"state":"active+clean","last_fresh":"2023-09-23T20:58:44.072481+0000","last_change":"2023-09-23T20:56:45.992577+0000","last_active":"2023-09-23T20:58:44.072481+0000","last_peered":"2023-09-23T20:58:44.072481+0000","last_clean":"2023-09-23T20:58:44.072481+0000","last_became_active":"2023-09-23T20:56:45.992297+0000","last_became_peered":"2023-09-23T20:56:45.992297+0000","last_unstale":"2023-09-23T20:58:44.072481+0000","last_undegraded":"2023-09-23T20:58:44.072481+0000","last_fullsized":"2023-09-23T20:58:44.072481+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":205,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:00:45.547202+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,2],"acting":[4,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":202,"reported_epoch":236,"state":"active+clean","last_fresh":"2023-09-23T20:58:44.065039+0000","last_change":"2023-09-23T20:51:10.831262+0000","last_active":"2023-09-23T20:58:44.065039+0000","last_peered":"2023-09-23T20:58:44.065039+0000","last_clean":"2023-09-23T20:58:44.065039+0000","last_became_active":"2023-09-23T20:51:10.830931+0000","last_became_peered":"2023-09-23T20:51:10.830931+0000","last_unstale":"2023-09-23T20:58:44.065039+0000","last_undegraded":"2023-09-23T20:58:44.065039+0000","last_fullsized":"2023-09-23T20:58:44.065039+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:51:54.982505+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.10","version":"79'161","reported_seq":474,"reported_epoch":236,"state":"active+clean","last_fresh":"2023-09-23T20:58:44.066497+0000","last_change":"2023-09-23T20:57:20.708003+0000","last_active":"2023-09-23T20:58:44.066497+0000","last_peered":"2023-09-23T20:58:44.066497+0000","last_clean":"2023-09-23T20:58:44.066497+0000","last_became_active":"2023-09-23T20:57:15.975302+0000","last_became_peered":"2023-09-23T20:57:15.975302+0000","last_unstale":"2023-09-23T20:58:44.066497+0000","last_undegraded":"2023-09-23T20:58:44.066497+0000","last_fullsized":"2023-09-23T20:58:44.066497+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:20.707939+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:20.707939+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:16:51.721994+0000","scrub_duration":0.0017733409999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":212,"reported_epoch":236,"state":"active+clean","last_fresh":"2023-09-23T20:58:51.439460+0000","last_change":"2023-09-23T20:57:44.979736+0000","last_active":"2023-09-23T20:58:51.439460+0000","last_peered":"2023-09-23T20:58:51.439460+0000","last_clean":"2023-09-23T20:58:51.439460+0000","last_became_active":"2023-09-23T20:57:44.979356+0000","last_became_peered":"2023-09-23T20:57:44.979356+0000","last_unstale":"2023-09-23T20:58:51.439460+0000","last_undegraded":"2023-09-23T20:58:51.439460+0000","last_fullsized":"2023-09-23T20:58:51.439460+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:14:43.848832+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":110,"reported_epoch":236,"state":"active+clean","last_fresh":"2023-09-23T20:58:44.066731+0000","last_change":"2023-09-23T20:54:31.887814+0000","last_active":"2023-09-23T20:58:44.066731+0000","last_peered":"2023-09-23T20:58:44.066731+0000","last_clean":"2023-09-23T20:58:44.066731+0000","last_became_active":"2023-09-23T20:54:31.887195+0000","last_became_peered":"2023-09-23T20:54:31.887195+0000","last_unstale":"2023-09-23T20:58:44.066731+0000","last_undegraded":"2023-09-23T20:58:44.066731+0000","last_fullsized":"2023-09-23T20:58:44.066731+0000","mapping_epoch":136,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:10:52.089553+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.13","version":"79'161","reported_seq":536,"reported_epoch":236,"state":"active+clean","last_fresh":"2023-09-23T20:58:44.080049+0000","last_change":"2023-09-23T20:57:48.262970+0000","last_active":"2023-09-23T20:58:44.080049+0000","last_peered":"2023-09-23T20:58:44.080049+0000","last_clean":"2023-09-23T20:58:44.080049+0000","last_became_active":"2023-09-23T20:57:48.262663+0000","last_became_peered":"2023-09-23T20:57:48.262663+0000","last_unstale":"2023-09-23T20:58:44.080049+0000","last_undegraded":"2023-09-23T20:58:44.080049+0000","last_fullsized":"2023-09-23T20:58:44.080049+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:36.420362+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:36.420362+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:28:28.613492+0000","scrub_duration":0.028838444000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":181,"reported_epoch":236,"state":"active+clean","last_fresh":"2023-09-23T20:58:44.074738+0000","last_change":"2023-09-23T20:56:45.991822+0000","last_active":"2023-09-23T20:58:44.074738+0000","last_peered":"2023-09-23T20:58:44.074738+0000","last_clean":"2023-09-23T20:58:44.074738+0000","last_became_active":"2023-09-23T20:56:45.991460+0000","last_became_peered":"2023-09-23T20:56:45.991460+0000","last_unstale":"2023-09-23T20:58:44.074738+0000","last_undegraded":"2023-09-23T20:58:44.074738+0000","last_fullsized":"2023-09-23T20:58:44.074738+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":205,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:20:37.503317+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":193,"reported_epoch":236,"state":"active+clean","last_fresh":"2023-09-23T20:58:44.074243+0000","last_change":"2023-09-23T20:51:10.840849+0000","last_active":"2023-09-23T20:58:44.074243+0000","last_peered":"2023-09-23T20:58:44.074243+0000","last_clean":"2023-09-23T20:58:44.074243+0000","last_became_active":"2023-09-23T20:51:10.840593+0000","last_became_peered":"2023-09-23T20:51:10.840593+0000","last_unstale":"2023-09-23T20:58:44.074243+0000","last_undegraded":"2023-09-23T20:58:44.074243+0000","last_fullsized":"2023-09-23T20:58:44.074243+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:30:52.297625+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.12","version":"79'161","reported_seq":474,"reported_epoch":236,"state":"active+clean","last_fresh":"2023-09-23T20:58:44.066611+0000","last_change":"2023-09-23T20:57:21.750765+0000","last_active":"2023-09-23T20:58:44.066611+0000","last_peered":"2023-09-23T20:58:44.066611+0000","last_clean":"2023-09-23T20:58:44.066611+0000","last_became_active":"2023-09-23T20:57:15.979195+0000","last_became_peered":"2023-09-23T20:57:15.979195+0000","last_unstale":"2023-09-23T20:58:44.066611+0000","last_undegraded":"2023-09-23T20:58:44.066611+0000","last_fullsized":"2023-09-23T20:58:44.066611+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:21.750680+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:21.750680+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:03:41.915962+0000","scrub_duration":0.0092391680000000007,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":229,"reported_epoch":236,"state":"active+clean","last_fresh":"2023-09-23T20:58:44.049518+0000","last_change":"2023-09-23T20:55:36.943969+0000","last_active":"2023-09-23T20:58:44.049518+0000","last_peered":"2023-09-23T20:58:44.049518+0000","last_clean":"2023-09-23T20:58:44.049518+0000","last_became_active":"2023-09-23T20:55:36.943580+0000","last_became_peered":"2023-09-23T20:55:36.943580+0000","last_unstale":"2023-09-23T20:58:44.049518+0000","last_undegraded":"2023-09-23T20:58:44.049518+0000","last_fullsized":"2023-09-23T20:58:44.049518+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:49:06.969396+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":25,"reported_epoch":236,"state":"active+clean","last_fresh":"2023-09-23T20:58:44.064146+0000","last_change":"2023-09-23T20:57:44.980498+0000","last_active":"2023-09-23T20:58:44.064146+0000","last_peered":"2023-09-23T20:58:44.064146+0000","last_clean":"2023-09-23T20:58:44.064146+0000","last_became_active":"2023-09-23T20:57:44.980182+0000","last_became_peered":"2023-09-23T20:57:44.980182+0000","last_unstale":"2023-09-23T20:58:44.064146+0000","last_undegraded":"2023-09-23T20:58:44.064146+0000","last_fullsized":"2023-09-23T20:58:44.064146+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:34:51.620544+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.d","version":"79'161","reported_seq":519,"reported_epoch":236,"state":"active+clean","last_fresh":"2023-09-23T20:58:44.072827+0000","last_change":"2023-09-23T20:57:31.075213+0000","last_active":"2023-09-23T20:58:44.072827+0000","last_peered":"2023-09-23T20:58:44.072827+0000","last_clean":"2023-09-23T20:58:44.072827+0000","last_became_active":"2023-09-23T20:57:15.988188+0000","last_became_peered":"2023-09-23T20:57:15.988188+0000","last_unstale":"2023-09-23T20:58:44.072827+0000","last_undegraded":"2023-09-23T20:58:44.072827+0000","last_fullsized":"2023-09-23T20:58:44.072827+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:31.075098+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:31.075098+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:41:52.142362+0000","scrub_duration":0.010304610000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.b","version":"237'8197","reported_seq":8245,"reported_epoch":237,"state":"active+clean","last_fresh":"2023-09-23T20:58:59.296695+0000","last_change":"2023-09-23T20:56:35.543994+0000","last_active":"2023-09-23T20:58:59.296695+0000","last_peered":"2023-09-23T20:58:59.296695+0000","last_clean":"2023-09-23T20:58:59.296695+0000","last_became_active":"2023-09-23T20:56:35.543758+0000","last_became_peered":"2023-09-23T20:56:35.543758+0000","last_unstale":"2023-09-23T20:58:59.296695+0000","last_undegraded":"2023-09-23T20:58:59.296695+0000","last_fullsized":"2023-09-23T20:58:59.296695+0000","mapping_epoch":199,"log_start":"205'1200","ondisk_log_start":"205'1200","created":199,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_clean_scrub_stamp":"2023-09-23T20:56:34.529103+0000","objects_scrubbed":0,"log_size":6997,"log_dups_size":0,"ondisk_log_size":6997,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:21:29.041484+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":293797888,"num_objects":4483,"num_object_clones":0,"num_object_copies":13449,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4483,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8197,"num_write_kb":405760,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":168,"reported_epoch":236,"state":"active+clean","last_fresh":"2023-09-23T20:58:44.073643+0000","last_change":"2023-09-23T20:56:45.991841+0000","last_active":"2023-09-23T20:58:44.073643+0000","last_peered":"2023-09-23T20:58:44.073643+0000","last_clean":"2023-09-23T20:58:44.073643+0000","last_became_active":"2023-09-23T20:56:45.991491+0000","last_became_peered":"2023-09-23T20:56:45.991491+0000","last_unstale":"2023-09-23T20:58:44.073643+0000","last_undegraded":"2023-09-23T20:58:44.073643+0000","last_fullsized":"2023-09-23T20:58:44.073643+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":205,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:30:37.405319+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":127,"reported_epoch":236,"state":"active+clean","last_fresh":"2023-09-23T20:58:44.066584+0000","last_change":"2023-09-23T20:55:35.957768+0000","last_active":"2023-09-23T20:58:44.066584+0000","last_peered":"2023-09-23T20:58:44.066584+0000","last_clean":"2023-09-23T20:58:44.066584+0000","last_became_active":"2023-09-23T20:55:35.957421+0000","last_became_peered":"2023-09-23T20:55:35.957421+0000","last_unstale":"2023-09-23T20:58:44.066584+0000","last_undegraded":"2023-09-23T20:58:44.066584+0000","last_fullsized":"2023-09-23T20:58:44.066584+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:49:41.585961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.c","version":"79'161","reported_seq":537,"reported_epoch":236,"state":"active+clean","last_fresh":"2023-09-23T20:58:44.064869+0000","last_change":"2023-09-23T20:57:22.041071+0000","last_active":"2023-09-23T20:58:44.064869+0000","last_peered":"2023-09-23T20:58:44.064869+0000","last_clean":"2023-09-23T20:58:44.064869+0000","last_became_active":"2023-09-23T20:57:15.973204+0000","last_became_peered":"2023-09-23T20:57:15.973204+0000","last_unstale":"2023-09-23T20:58:44.064869+0000","last_undegraded":"2023-09-23T20:58:44.064869+0000","last_fullsized":"2023-09-23T20:58:44.064869+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:22.040907+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:22.040907+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:47:02.019978+0000","scrub_duration":0.013021176000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.a","version":"237'8303","reported_seq":8351,"reported_epoch":237,"state":"active+clean","last_fresh":"2023-09-23T20:58:59.431010+0000","last_change":"2023-09-23T20:56:35.547120+0000","last_active":"2023-09-23T20:58:59.431010+0000","last_peered":"2023-09-23T20:58:59.431010+0000","last_clean":"2023-09-23T20:58:59.431010+0000","last_became_active":"2023-09-23T20:56:35.546838+0000","last_became_peered":"2023-09-23T20:56:35.546838+0000","last_unstale":"2023-09-23T20:58:59.431010+0000","last_undegraded":"2023-09-23T20:58:59.431010+0000","last_fullsized":"2023-09-23T20:58:59.431010+0000","mapping_epoch":199,"log_start":"206'2100","ondisk_log_start":"206'2100","created":199,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_clean_scrub_stamp":"2023-09-23T20:56:34.529103+0000","objects_scrubbed":0,"log_size":6203,"log_dups_size":0,"ondisk_log_size":6203,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:48:24.132768+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":290521088,"num_objects":4433,"num_object_clones":0,"num_object_copies":13299,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4433,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8303,"num_write_kb":407552,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,5],"acting":[3,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":155,"reported_epoch":236,"state":"active+clean","last_fresh":"2023-09-23T20:58:44.073395+0000","last_change":"2023-09-23T20:56:45.991821+0000","last_active":"2023-09-23T20:58:44.073395+0000","last_peered":"2023-09-23T20:58:44.073395+0000","last_clean":"2023-09-23T20:58:44.073395+0000","last_became_active":"2023-09-23T20:56:45.991447+0000","last_became_peered":"2023-09-23T20:56:45.991447+0000","last_unstale":"2023-09-23T20:58:44.073395+0000","last_undegraded":"2023-09-23T20:58:44.073395+0000","last_fullsized":"2023-09-23T20:58:44.073395+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":205,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:14:25.525222+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":158,"reported_epoch":236,"state":"active+clean","last_fresh":"2023-09-23T20:58:44.075133+0000","last_change":"2023-09-23T20:57:44.982059+0000","last_active":"2023-09-23T20:58:44.075133+0000","last_peered":"2023-09-23T20:58:44.075133+0000","last_clean":"2023-09-23T20:58:44.075133+0000","last_became_active":"2023-09-23T20:57:44.981742+0000","last_became_peered":"2023-09-23T20:57:44.981742+0000","last_unstale":"2023-09-23T20:58:44.075133+0000","last_undegraded":"2023-09-23T20:58:44.075133+0000","last_fullsized":"2023-09-23T20:58:44.075133+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:41:24.042916+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.f","version":"79'161","reported_seq":541,"reported_epoch":236,"state":"active+clean","last_fresh":"2023-09-23T20:58:44.049718+0000","last_change":"2023-09-23T20:57:27.880841+0000","last_active":"2023-09-23T20:58:44.049718+0000","last_peered":"2023-09-23T20:58:44.049718+0000","last_clean":"2023-09-23T20:58:44.049718+0000","last_became_active":"2023-09-23T20:57:15.970925+0000","last_became_peered":"2023-09-23T20:57:15.970925+0000","last_unstale":"2023-09-23T20:58:44.049718+0000","last_undegraded":"2023-09-23T20:58:44.049718+0000","last_fullsized":"2023-09-23T20:58:44.049718+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:27.880776+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:27.880776+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:15:13.438014+0000","scrub_duration":0.0014004899999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.9","version":"237'8366","reported_seq":8414,"reported_epoch":237,"state":"active+clean","last_fresh":"2023-09-23T20:59:01.475967+0000","last_change":"2023-09-23T20:56:35.549387+0000","last_active":"2023-09-23T20:59:01.475967+0000","last_peered":"2023-09-23T20:59:01.475967+0000","last_clean":"2023-09-23T20:59:01.475967+0000","last_became_active":"2023-09-23T20:56:35.549087+0000","last_became_peered":"2023-09-23T20:56:35.549087+0000","last_unstale":"2023-09-23T20:59:01.475967+0000","last_undegraded":"2023-09-23T20:59:01.475967+0000","last_fullsized":"2023-09-23T20:59:01.475967+0000","mapping_epoch":199,"log_start":"204'1000","ondisk_log_start":"204'1000","created":199,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_clean_scrub_stamp":"2023-09-23T20:56:34.529103+0000","objects_scrubbed":0,"log_size":7366,"log_dups_size":0,"ondisk_log_size":7366,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:04:51.547362+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":284426240,"num_objects":4340,"num_object_clones":0,"num_object_copies":13020,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4340,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8366,"num_write_kb":406592,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":141,"reported_epoch":236,"state":"active+clean","last_fresh":"2023-09-23T20:58:44.066748+0000","last_change":"2023-09-23T20:56:45.989153+0000","last_active":"2023-09-23T20:58:44.066748+0000","last_peered":"2023-09-23T20:58:44.066748+0000","last_clean":"2023-09-23T20:58:44.066748+0000","last_became_active":"2023-09-23T20:56:45.988800+0000","last_became_peered":"2023-09-23T20:56:45.988800+0000","last_unstale":"2023-09-23T20:58:44.066748+0000","last_undegraded":"2023-09-23T20:58:44.066748+0000","last_fullsized":"2023-09-23T20:58:44.066748+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":205,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:04:41.130194+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":176,"reported_epoch":236,"state":"active+clean","last_fresh":"2023-09-23T20:58:44.049718+0000","last_change":"2023-09-23T20:55:35.965427+0000","last_active":"2023-09-23T20:58:44.049718+0000","last_peered":"2023-09-23T20:58:44.049718+0000","last_clean":"2023-09-23T20:58:44.049718+0000","last_became_active":"2023-09-23T20:55:35.963969+0000","last_became_peered":"2023-09-23T20:55:35.963969+0000","last_unstale":"2023-09-23T20:58:44.049718+0000","last_undegraded":"2023-09-23T20:58:44.049718+0000","last_fullsized":"2023-09-23T20:58:44.049718+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:38:40.612083+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.e","version":"79'161","reported_seq":448,"reported_epoch":236,"state":"active+clean","last_fresh":"2023-09-23T20:58:44.073258+0000","last_change":"2023-09-23T20:57:34.132738+0000","last_active":"2023-09-23T20:58:44.073258+0000","last_peered":"2023-09-23T20:58:44.073258+0000","last_clean":"2023-09-23T20:58:44.073258+0000","last_became_active":"2023-09-23T20:57:15.974318+0000","last_became_peered":"2023-09-23T20:57:15.974318+0000","last_unstale":"2023-09-23T20:58:44.073258+0000","last_undegraded":"2023-09-23T20:58:44.073258+0000","last_fullsized":"2023-09-23T20:58:44.073258+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:34.132660+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:34.132660+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:07:07.739438+0000","scrub_duration":0.005229694,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.8","version":"237'8431","reported_seq":8479,"reported_epoch":237,"state":"active+clean","last_fresh":"2023-09-23T20:59:01.217683+0000","last_change":"2023-09-23T20:56:35.548119+0000","last_active":"2023-09-23T20:59:01.217683+0000","last_peered":"2023-09-23T20:59:01.217683+0000","last_clean":"2023-09-23T20:59:01.217683+0000","last_became_active":"2023-09-23T20:56:35.547914+0000","last_became_peered":"2023-09-23T20:56:35.547914+0000","last_unstale":"2023-09-23T20:59:01.217683+0000","last_undegraded":"2023-09-23T20:59:01.217683+0000","last_fullsized":"2023-09-23T20:59:01.217683+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":199,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_clean_scrub_stamp":"2023-09-23T20:56:34.529103+0000","objects_scrubbed":0,"log_size":8431,"log_dups_size":0,"ondisk_log_size":8431,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:42:54.255023+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":289865728,"num_objects":4423,"num_object_clones":0,"num_object_copies":13269,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4423,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8431,"num_write_kb":411328,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":127,"reported_epoch":236,"state":"active+clean","last_fresh":"2023-09-23T20:58:44.066803+0000","last_change":"2023-09-23T20:55:35.962005+0000","last_active":"2023-09-23T20:58:44.066803+0000","last_peered":"2023-09-23T20:58:44.066803+0000","last_clean":"2023-09-23T20:58:44.066803+0000","last_became_active":"2023-09-23T20:55:35.961695+0000","last_became_peered":"2023-09-23T20:55:35.961695+0000","last_unstale":"2023-09-23T20:58:44.066803+0000","last_undegraded":"2023-09-23T20:58:44.066803+0000","last_fullsized":"2023-09-23T20:58:44.066803+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:28:22.876201+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":225,"reported_epoch":236,"state":"active+clean","last_fresh":"2023-09-23T20:58:44.061487+0000","last_change":"2023-09-23T20:54:13.928715+0000","last_active":"2023-09-23T20:58:44.061487+0000","last_peered":"2023-09-23T20:58:44.061487+0000","last_clean":"2023-09-23T20:58:44.061487+0000","last_became_active":"2023-09-23T20:54:13.928200+0000","last_became_peered":"2023-09-23T20:54:13.928200+0000","last_unstale":"2023-09-23T20:58:44.061487+0000","last_undegraded":"2023-09-23T20:58:44.061487+0000","last_fullsized":"2023-09-23T20:58:44.061487+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:41:16.927513+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.f","version":"237'8498","reported_seq":8546,"reported_epoch":237,"state":"active+clean","last_fresh":"2023-09-23T20:59:01.425181+0000","last_change":"2023-09-23T20:56:35.545128+0000","last_active":"2023-09-23T20:59:01.425181+0000","last_peered":"2023-09-23T20:59:01.425181+0000","last_clean":"2023-09-23T20:59:01.425181+0000","last_became_active":"2023-09-23T20:56:35.544838+0000","last_became_peered":"2023-09-23T20:56:35.544838+0000","last_unstale":"2023-09-23T20:59:01.425181+0000","last_undegraded":"2023-09-23T20:59:01.425181+0000","last_fullsized":"2023-09-23T20:59:01.425181+0000","mapping_epoch":199,"log_start":"205'1100","ondisk_log_start":"205'1100","created":199,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_clean_scrub_stamp":"2023-09-23T20:56:34.529103+0000","objects_scrubbed":0,"log_size":7398,"log_dups_size":0,"ondisk_log_size":7398,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:30:43.388717+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":286261248,"num_objects":4368,"num_object_clones":0,"num_object_copies":13104,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4368,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8498,"num_write_kb":411712,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.9","version":"79'161","reported_seq":610,"reported_epoch":237,"state":"active+clean","last_fresh":"2023-09-23T20:58:54.378732+0000","last_change":"2023-09-23T20:56:46.921939+0000","last_active":"2023-09-23T20:58:54.378732+0000","last_peered":"2023-09-23T20:58:54.378732+0000","last_clean":"2023-09-23T20:58:54.378732+0000","last_became_active":"2023-09-23T20:56:45.989908+0000","last_became_peered":"2023-09-23T20:56:45.989908+0000","last_unstale":"2023-09-23T20:58:54.378732+0000","last_undegraded":"2023-09-23T20:58:54.378732+0000","last_fullsized":"2023-09-23T20:58:54.378732+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":205,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:56:46.921739+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:56:46.921739+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:20:42.635110+0000","scrub_duration":0.0071954109999999996,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":226,"reported_epoch":236,"state":"active+clean","last_fresh":"2023-09-23T20:58:44.065812+0000","last_change":"2023-09-23T20:54:13.961788+0000","last_active":"2023-09-23T20:58:44.065812+0000","last_peered":"2023-09-23T20:58:44.065812+0000","last_clean":"2023-09-23T20:58:44.065812+0000","last_became_active":"2023-09-23T20:54:13.940937+0000","last_became_peered":"2023-09-23T20:54:13.940937+0000","last_unstale":"2023-09-23T20:58:44.065812+0000","last_undegraded":"2023-09-23T20:58:44.065812+0000","last_fullsized":"2023-09-23T20:58:44.065812+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:11:56.785299+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":131,"reported_epoch":236,"state":"active+clean","last_fresh":"2023-09-23T20:58:44.072847+0000","last_change":"2023-09-23T20:55:35.949670+0000","last_active":"2023-09-23T20:58:44.072847+0000","last_peered":"2023-09-23T20:58:44.072847+0000","last_clean":"2023-09-23T20:58:44.072847+0000","last_became_active":"2023-09-23T20:55:35.949140+0000","last_became_peered":"2023-09-23T20:55:35.949140+0000","last_unstale":"2023-09-23T20:58:44.072847+0000","last_undegraded":"2023-09-23T20:58:44.072847+0000","last_fullsized":"2023-09-23T20:58:44.072847+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:55:59.340303+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.e","version":"237'8382","reported_seq":8430,"reported_epoch":237,"state":"active+clean","last_fresh":"2023-09-23T20:59:01.365745+0000","last_change":"2023-09-23T20:56:35.547184+0000","last_active":"2023-09-23T20:59:01.365745+0000","last_peered":"2023-09-23T20:59:01.365745+0000","last_clean":"2023-09-23T20:59:01.365745+0000","last_became_active":"2023-09-23T20:56:35.546912+0000","last_became_peered":"2023-09-23T20:56:35.546912+0000","last_unstale":"2023-09-23T20:59:01.365745+0000","last_undegraded":"2023-09-23T20:59:01.365745+0000","last_fullsized":"2023-09-23T20:59:01.365745+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":199,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_clean_scrub_stamp":"2023-09-23T20:56:34.529103+0000","objects_scrubbed":0,"log_size":8382,"log_dups_size":0,"ondisk_log_size":8382,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:57:07.422792+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":288227328,"num_objects":4398,"num_object_clones":0,"num_object_copies":13194,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4398,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8382,"num_write_kb":408960,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,0],"acting":[2,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.8","version":"79'161","reported_seq":480,"reported_epoch":236,"state":"active+clean","last_fresh":"2023-09-23T20:58:44.066574+0000","last_change":"2023-09-23T20:56:57.505737+0000","last_active":"2023-09-23T20:58:44.066574+0000","last_peered":"2023-09-23T20:58:44.066574+0000","last_clean":"2023-09-23T20:58:44.066574+0000","last_became_active":"2023-09-23T20:56:04.982345+0000","last_became_peered":"2023-09-23T20:56:04.982345+0000","last_unstale":"2023-09-23T20:58:44.066574+0000","last_undegraded":"2023-09-23T20:58:44.066574+0000","last_fullsized":"2023-09-23T20:58:44.066574+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:56:57.505663+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:56:57.505663+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:34:30.109724+0000","scrub_duration":0.026284108,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":199,"reported_epoch":236,"state":"active+clean","last_fresh":"2023-09-23T20:58:44.050274+0000","last_change":"2023-09-23T20:55:35.959942+0000","last_active":"2023-09-23T20:58:44.050274+0000","last_peered":"2023-09-23T20:58:44.050274+0000","last_clean":"2023-09-23T20:58:44.050274+0000","last_became_active":"2023-09-23T20:55:35.957071+0000","last_became_peered":"2023-09-23T20:55:35.957071+0000","last_unstale":"2023-09-23T20:58:44.050274+0000","last_undegraded":"2023-09-23T20:58:44.050274+0000","last_fullsized":"2023-09-23T20:58:44.050274+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:44:30.308036+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":114,"reported_epoch":236,"state":"active+clean","last_fresh":"2023-09-23T20:58:44.073150+0000","last_change":"2023-09-23T20:54:13.951459+0000","last_active":"2023-09-23T20:58:44.073150+0000","last_peered":"2023-09-23T20:58:44.073150+0000","last_clean":"2023-09-23T20:58:44.073150+0000","last_became_active":"2023-09-23T20:54:13.948409+0000","last_became_peered":"2023-09-23T20:54:13.948409+0000","last_unstale":"2023-09-23T20:58:44.073150+0000","last_undegraded":"2023-09-23T20:58:44.073150+0000","last_fullsized":"2023-09-23T20:58:44.073150+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:38:10.148061+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.b","version":"79'161","reported_seq":519,"reported_epoch":236,"state":"active+clean","last_fresh":"2023-09-23T20:58:44.083460+0000","last_change":"2023-09-23T20:57:27.320172+0000","last_active":"2023-09-23T20:58:44.083460+0000","last_peered":"2023-09-23T20:58:44.083460+0000","last_clean":"2023-09-23T20:58:44.083460+0000","last_became_active":"2023-09-23T20:57:16.009630+0000","last_became_peered":"2023-09-23T20:57:16.009630+0000","last_unstale":"2023-09-23T20:58:44.083460+0000","last_undegraded":"2023-09-23T20:58:44.083460+0000","last_fullsized":"2023-09-23T20:58:44.083460+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:27.320022+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:27.320022+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:31:35.221558+0000","scrub_duration":0.0043894679999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,5],"acting":[1,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.d","version":"237'8335","reported_seq":8381,"reported_epoch":237,"state":"active+clean","last_fresh":"2023-09-23T20:58:59.085718+0000","last_change":"2023-09-23T20:56:35.544132+0000","last_active":"2023-09-23T20:58:59.085718+0000","last_peered":"2023-09-23T20:58:59.085718+0000","last_clean":"2023-09-23T20:58:59.085718+0000","last_became_active":"2023-09-23T20:56:35.543836+0000","last_became_peered":"2023-09-23T20:56:35.543836+0000","last_unstale":"2023-09-23T20:58:59.085718+0000","last_undegraded":"2023-09-23T20:58:59.085718+0000","last_fullsized":"2023-09-23T20:58:59.085718+0000","mapping_epoch":199,"log_start":"203'1000","ondisk_log_start":"203'1000","created":199,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_clean_scrub_stamp":"2023-09-23T20:56:34.529103+0000","objects_scrubbed":0,"log_size":7335,"log_dups_size":0,"ondisk_log_size":7335,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:27:14.636991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":291897368,"num_objects":4455,"num_object_clones":0,"num_object_copies":13365,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4455,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":8335,"num_write_kb":409217,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":114,"reported_epoch":236,"state":"active+clean","last_fresh":"2023-09-23T20:58:44.074840+0000","last_change":"2023-09-23T20:54:13.950254+0000","last_active":"2023-09-23T20:58:44.074840+0000","last_peered":"2023-09-23T20:58:44.074840+0000","last_clean":"2023-09-23T20:58:44.074840+0000","last_became_active":"2023-09-23T20:54:13.949934+0000","last_became_peered":"2023-09-23T20:54:13.949934+0000","last_unstale":"2023-09-23T20:58:44.074840+0000","last_undegraded":"2023-09-23T20:58:44.074840+0000","last_fullsized":"2023-09-23T20:58:44.074840+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:30:32.847167+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":131,"reported_epoch":236,"state":"active+clean","last_fresh":"2023-09-23T20:58:44.073467+0000","last_change":"2023-09-23T20:55:35.956153+0000","last_active":"2023-09-23T20:58:44.073467+0000","last_peered":"2023-09-23T20:58:44.073467+0000","last_clean":"2023-09-23T20:58:44.073467+0000","last_became_active":"2023-09-23T20:55:35.955630+0000","last_became_peered":"2023-09-23T20:55:35.955630+0000","last_unstale":"2023-09-23T20:58:44.073467+0000","last_undegraded":"2023-09-23T20:58:44.073467+0000","last_fullsized":"2023-09-23T20:58:44.073467+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:45:20.282929+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.c","version":"237'8297","reported_seq":8353,"reported_epoch":237,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T20:58:59.073318+0000","last_change":"2023-09-23T20:57:46.463005+0000","last_active":"2023-09-23T20:58:59.073318+0000","last_peered":"2023-09-23T20:58:59.073318+0000","last_clean":"2023-09-23T20:57:44.015119+0000","last_became_active":"2023-09-23T20:57:46.448702+0000","last_became_peered":"2023-09-23T20:57:46.448702+0000","last_unstale":"2023-09-23T20:58:59.073318+0000","last_undegraded":"2023-09-23T20:57:46.010874+0000","last_fullsized":"2023-09-23T20:57:45.978434+0000","mapping_epoch":223,"log_start":"203'900","ondisk_log_start":"203'900","created":199,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_clean_scrub_stamp":"2023-09-23T20:56:34.529103+0000","objects_scrubbed":0,"log_size":7397,"log_dups_size":0,"ondisk_log_size":7397,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:08:34.180651+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":295632896,"num_objects":4511,"num_object_clones":0,"num_object_copies":13533,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5473,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4511,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8297,"num_write_kb":409856,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,7],"avail_no_missing":["6","2","7"],"object_location_counts":[{"shards":"2,6,7","objects":3580},{"shards":"6,7","objects":-962}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.a","version":"79'161","reported_seq":480,"reported_epoch":236,"state":"active+clean","last_fresh":"2023-09-23T20:58:44.067163+0000","last_change":"2023-09-23T20:56:44.487778+0000","last_active":"2023-09-23T20:58:44.067163+0000","last_peered":"2023-09-23T20:58:44.067163+0000","last_clean":"2023-09-23T20:58:44.067163+0000","last_became_active":"2023-09-23T20:56:05.006740+0000","last_became_peered":"2023-09-23T20:56:05.006740+0000","last_unstale":"2023-09-23T20:58:44.067163+0000","last_undegraded":"2023-09-23T20:58:44.067163+0000","last_fullsized":"2023-09-23T20:58:44.067163+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:56:44.487678+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:56:44.487678+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:04:50.086749+0000","scrub_duration":0.0017408899999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":25,"reported_epoch":236,"state":"active+clean","last_fresh":"2023-09-23T20:58:44.062305+0000","last_change":"2023-09-23T20:57:44.982513+0000","last_active":"2023-09-23T20:58:44.062305+0000","last_peered":"2023-09-23T20:58:44.062305+0000","last_clean":"2023-09-23T20:58:44.062305+0000","last_became_active":"2023-09-23T20:57:44.982112+0000","last_became_peered":"2023-09-23T20:57:44.982112+0000","last_unstale":"2023-09-23T20:58:44.062305+0000","last_undegraded":"2023-09-23T20:58:44.062305+0000","last_fullsized":"2023-09-23T20:58:44.062305+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:09:05.630041+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":206,"reported_epoch":236,"state":"active+clean","last_fresh":"2023-09-23T20:58:44.083707+0000","last_change":"2023-09-23T20:54:13.940748+0000","last_active":"2023-09-23T20:58:44.083707+0000","last_peered":"2023-09-23T20:58:44.083707+0000","last_clean":"2023-09-23T20:58:44.083707+0000","last_became_active":"2023-09-23T20:54:13.940419+0000","last_became_peered":"2023-09-23T20:54:13.940419+0000","last_unstale":"2023-09-23T20:58:44.083707+0000","last_undegraded":"2023-09-23T20:58:44.083707+0000","last_fullsized":"2023-09-23T20:58:44.083707+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:15:01.923816+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.1","version":"237'8445","reported_seq":8504,"reported_epoch":237,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T20:59:02.126946+0000","last_change":"2023-09-23T20:57:46.483248+0000","last_active":"2023-09-23T20:59:02.126946+0000","last_peered":"2023-09-23T20:59:02.126946+0000","last_clean":"2023-09-23T20:57:44.047957+0000","last_became_active":"2023-09-23T20:57:46.475568+0000","last_became_peered":"2023-09-23T20:57:46.475568+0000","last_unstale":"2023-09-23T20:59:02.126946+0000","last_undegraded":"2023-09-23T20:57:46.083591+0000","last_fullsized":"2023-09-23T20:57:45.978436+0000","mapping_epoch":223,"log_start":"205'1700","ondisk_log_start":"205'1700","created":199,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_clean_scrub_stamp":"2023-09-23T20:56:34.529103+0000","objects_scrubbed":0,"log_size":6745,"log_dups_size":0,"ondisk_log_size":6745,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:22:45.067269+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":284491776,"num_objects":4341,"num_object_clones":0,"num_object_copies":13023,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5470,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4341,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8445,"num_write_kb":409152,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,6],"avail_no_missing":["4","1","6"],"object_location_counts":[{"shards":"1,4,6","objects":3418},{"shards":"4,6","objects":-1129}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.7","version":"79'161","reported_seq":542,"reported_epoch":236,"state":"active+clean","last_fresh":"2023-09-23T20:58:44.050055+0000","last_change":"2023-09-23T20:57:48.243572+0000","last_active":"2023-09-23T20:58:44.050055+0000","last_peered":"2023-09-23T20:58:44.050055+0000","last_clean":"2023-09-23T20:58:44.050055+0000","last_became_active":"2023-09-23T20:57:48.243229+0000","last_became_peered":"2023-09-23T20:57:48.243229+0000","last_unstale":"2023-09-23T20:58:44.050055+0000","last_undegraded":"2023-09-23T20:58:44.050055+0000","last_fullsized":"2023-09-23T20:58:44.050055+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":3,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:25.916867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:25.916867+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:23:19.385831+0000","scrub_duration":0.001665323,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":212,"reported_epoch":236,"state":"active+clean","last_fresh":"2023-09-23T20:58:44.084330+0000","last_change":"2023-09-23T20:50:05.867130+0000","last_active":"2023-09-23T20:58:44.084330+0000","last_peered":"2023-09-23T20:58:44.084330+0000","last_clean":"2023-09-23T20:58:44.084330+0000","last_became_active":"2023-09-23T20:50:05.866758+0000","last_became_peered":"2023-09-23T20:50:05.866758+0000","last_unstale":"2023-09-23T20:58:44.084330+0000","last_undegraded":"2023-09-23T20:58:44.084330+0000","last_fullsized":"2023-09-23T20:58:44.084330+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:55:07.548209+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":110,"reported_epoch":236,"state":"active+clean","last_fresh":"2023-09-23T20:58:44.066851+0000","last_change":"2023-09-23T20:54:31.887604+0000","last_active":"2023-09-23T20:58:44.066851+0000","last_peered":"2023-09-23T20:58:44.066851+0000","last_clean":"2023-09-23T20:58:44.066851+0000","last_became_active":"2023-09-23T20:54:31.886752+0000","last_became_peered":"2023-09-23T20:54:31.886752+0000","last_unstale":"2023-09-23T20:58:44.066851+0000","last_undegraded":"2023-09-23T20:58:44.066851+0000","last_fullsized":"2023-09-23T20:58:44.066851+0000","mapping_epoch":136,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:13:58.872176+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":210,"reported_epoch":236,"state":"active+clean","last_fresh":"2023-09-23T20:58:44.070268+0000","last_change":"2023-09-23T20:54:31.899082+0000","last_active":"2023-09-23T20:58:44.070268+0000","last_peered":"2023-09-23T20:58:44.070268+0000","last_clean":"2023-09-23T20:58:44.070268+0000","last_became_active":"2023-09-23T20:54:31.898521+0000","last_became_peered":"2023-09-23T20:54:31.898521+0000","last_unstale":"2023-09-23T20:58:44.070268+0000","last_undegraded":"2023-09-23T20:58:44.070268+0000","last_fullsized":"2023-09-23T20:58:44.070268+0000","mapping_epoch":136,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:25:36.059711+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.7","version":"237'8322","reported_seq":8368,"reported_epoch":237,"state":"active+clean","last_fresh":"2023-09-23T20:58:58.991028+0000","last_change":"2023-09-23T20:56:35.541695+0000","last_active":"2023-09-23T20:58:58.991028+0000","last_peered":"2023-09-23T20:58:58.991028+0000","last_clean":"2023-09-23T20:58:58.991028+0000","last_became_active":"2023-09-23T20:56:35.541392+0000","last_became_peered":"2023-09-23T20:56:35.541392+0000","last_unstale":"2023-09-23T20:58:58.991028+0000","last_undegraded":"2023-09-23T20:58:58.991028+0000","last_fullsized":"2023-09-23T20:58:58.991028+0000","mapping_epoch":199,"log_start":"203'1000","ondisk_log_start":"203'1000","created":199,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_clean_scrub_stamp":"2023-09-23T20:56:34.529103+0000","objects_scrubbed":0,"log_size":7322,"log_dups_size":0,"ondisk_log_size":7322,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:57:22.478253+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":294125568,"num_objects":4488,"num_object_clones":0,"num_object_copies":13464,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4488,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8322,"num_write_kb":409920,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.1","version":"79'161","reported_seq":500,"reported_epoch":236,"state":"active+clean","last_fresh":"2023-09-23T20:58:44.066608+0000","last_change":"2023-09-23T20:57:19.669346+0000","last_active":"2023-09-23T20:58:44.066608+0000","last_peered":"2023-09-23T20:58:44.066608+0000","last_clean":"2023-09-23T20:58:44.066608+0000","last_became_active":"2023-09-23T20:57:08.246046+0000","last_became_peered":"2023-09-23T20:57:08.246046+0000","last_unstale":"2023-09-23T20:58:44.066608+0000","last_undegraded":"2023-09-23T20:58:44.066608+0000","last_fullsized":"2023-09-23T20:58:44.066608+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:19.669259+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:19.669259+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:14:03.435531+0000","scrub_duration":0.0045759559999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":127,"reported_epoch":236,"state":"active+clean","last_fresh":"2023-09-23T20:58:44.067057+0000","last_change":"2023-09-23T20:55:35.959514+0000","last_active":"2023-09-23T20:58:44.067057+0000","last_peered":"2023-09-23T20:58:44.067057+0000","last_clean":"2023-09-23T20:58:44.067057+0000","last_became_active":"2023-09-23T20:55:35.959244+0000","last_became_peered":"2023-09-23T20:55:35.959244+0000","last_unstale":"2023-09-23T20:58:44.067057+0000","last_undegraded":"2023-09-23T20:58:44.067057+0000","last_fullsized":"2023-09-23T20:58:44.067057+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:47:32.665316+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":131,"reported_epoch":236,"state":"active+clean","last_fresh":"2023-09-23T20:58:44.072736+0000","last_change":"2023-09-23T20:55:35.949000+0000","last_active":"2023-09-23T20:58:44.072736+0000","last_peered":"2023-09-23T20:58:44.072736+0000","last_clean":"2023-09-23T20:58:44.072736+0000","last_became_active":"2023-09-23T20:55:35.948410+0000","last_became_peered":"2023-09-23T20:55:35.948410+0000","last_unstale":"2023-09-23T20:58:44.072736+0000","last_undegraded":"2023-09-23T20:58:44.072736+0000","last_fullsized":"2023-09-23T20:58:44.072736+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:26:50.477289+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":204,"reported_epoch":236,"state":"active+clean","last_fresh":"2023-09-23T20:58:44.083899+0000","last_change":"2023-09-23T20:51:10.829889+0000","last_active":"2023-09-23T20:58:44.083899+0000","last_peered":"2023-09-23T20:58:44.083899+0000","last_clean":"2023-09-23T20:58:44.083899+0000","last_became_active":"2023-09-23T20:51:10.829533+0000","last_became_peered":"2023-09-23T20:51:10.829533+0000","last_unstale":"2023-09-23T20:58:44.083899+0000","last_undegraded":"2023-09-23T20:58:44.083899+0000","last_fullsized":"2023-09-23T20:58:44.083899+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:43:04.603460+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.4","version":"237'8348","reported_seq":8396,"reported_epoch":237,"state":"active+clean","last_fresh":"2023-09-23T20:59:01.435743+0000","last_change":"2023-09-23T20:56:35.549909+0000","last_active":"2023-09-23T20:59:01.435743+0000","last_peered":"2023-09-23T20:59:01.435743+0000","last_clean":"2023-09-23T20:59:01.435743+0000","last_became_active":"2023-09-23T20:56:35.549618+0000","last_became_peered":"2023-09-23T20:56:35.549618+0000","last_unstale":"2023-09-23T20:59:01.435743+0000","last_undegraded":"2023-09-23T20:59:01.435743+0000","last_fullsized":"2023-09-23T20:59:01.435743+0000","mapping_epoch":199,"log_start":"204'1000","ondisk_log_start":"204'1000","created":199,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_clean_scrub_stamp":"2023-09-23T20:56:34.529103+0000","objects_scrubbed":0,"log_size":7348,"log_dups_size":0,"ondisk_log_size":7348,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:24:25.918475+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":286654464,"num_objects":4374,"num_object_clones":0,"num_object_copies":13122,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4374,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8348,"num_write_kb":407104,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.2","version":"79'161","reported_seq":467,"reported_epoch":236,"state":"active+clean","last_fresh":"2023-09-23T20:58:51.439439+0000","last_change":"2023-09-23T20:57:24.802586+0000","last_active":"2023-09-23T20:58:51.439439+0000","last_peered":"2023-09-23T20:58:51.439439+0000","last_clean":"2023-09-23T20:58:51.439439+0000","last_became_active":"2023-09-23T20:57:08.236114+0000","last_became_peered":"2023-09-23T20:57:08.236114+0000","last_unstale":"2023-09-23T20:58:51.439439+0000","last_undegraded":"2023-09-23T20:58:51.439439+0000","last_fullsized":"2023-09-23T20:58:51.439439+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:24.802451+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:24.802451+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:06:22.309980+0000","scrub_duration":0.0090861250000000005,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,4],"acting":[0,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":223,"reported_epoch":237,"state":"active+clean","last_fresh":"2023-09-23T20:58:54.388802+0000","last_change":"2023-09-23T20:51:10.838054+0000","last_active":"2023-09-23T20:58:54.388802+0000","last_peered":"2023-09-23T20:58:54.388802+0000","last_clean":"2023-09-23T20:58:54.388802+0000","last_became_active":"2023-09-23T20:51:10.837750+0000","last_became_peered":"2023-09-23T20:51:10.837750+0000","last_unstale":"2023-09-23T20:58:54.388802+0000","last_undegraded":"2023-09-23T20:58:54.388802+0000","last_fullsized":"2023-09-23T20:58:54.388802+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:15:51.648085+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":204,"reported_epoch":236,"state":"active+clean","last_fresh":"2023-09-23T20:58:44.070566+0000","last_change":"2023-09-23T20:55:35.951146+0000","last_active":"2023-09-23T20:58:44.070566+0000","last_peered":"2023-09-23T20:58:44.070566+0000","last_clean":"2023-09-23T20:58:44.070566+0000","last_became_active":"2023-09-23T20:55:35.950874+0000","last_became_peered":"2023-09-23T20:55:35.950874+0000","last_unstale":"2023-09-23T20:58:44.070566+0000","last_undegraded":"2023-09-23T20:58:44.070566+0000","last_fullsized":"2023-09-23T20:58:44.070566+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:58:13.957677+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":225,"reported_epoch":236,"state":"active+clean","last_fresh":"2023-09-23T20:58:44.295268+0000","last_change":"2023-09-23T20:54:31.898870+0000","last_active":"2023-09-23T20:58:44.295268+0000","last_peered":"2023-09-23T20:58:44.295268+0000","last_clean":"2023-09-23T20:58:44.295268+0000","last_became_active":"2023-09-23T20:54:31.898467+0000","last_became_peered":"2023-09-23T20:54:31.898467+0000","last_unstale":"2023-09-23T20:58:44.295268+0000","last_undegraded":"2023-09-23T20:58:44.295268+0000","last_fullsized":"2023-09-23T20:58:44.295268+0000","mapping_epoch":136,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:56:22.483408+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.6","version":"237'8270","reported_seq":11573,"reported_epoch":237,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-09-23T20:59:01.471449+0000","last_change":"2023-09-23T20:58:39.453197+0000","last_active":"2023-09-23T20:59:01.471449+0000","last_peered":"2023-09-23T20:59:01.471449+0000","last_clean":"2023-09-23T20:57:44.031597+0000","last_became_active":"2023-09-23T20:57:46.443927+0000","last_became_peered":"2023-09-23T20:57:46.443927+0000","last_unstale":"2023-09-23T20:59:01.471449+0000","last_undegraded":"2023-09-23T20:58:39.450464+0000","last_fullsized":"2023-09-23T20:57:45.977227+0000","mapping_epoch":223,"log_start":"203'900","ondisk_log_start":"203'900","created":199,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_clean_scrub_stamp":"2023-09-23T20:56:34.529103+0000","objects_scrubbed":0,"log_size":7370,"log_dups_size":0,"ondisk_log_size":7370,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:37:23.911131+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":282984448,"num_objects":4318,"num_object_clones":0,"num_object_copies":12954,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":357,"num_objects_misplaced":3418,"num_objects_unfound":0,"num_objects_dirty":4318,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8270,"num_write_kb":402816,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1623,"num_bytes_recovered":106364928,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7],"avail_no_missing":["0","5","7"],"object_location_counts":[{"shards":"0,5,7","objects":3418},{"shards":"0,7","objects":543}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":110,"reported_epoch":236,"state":"active+clean","last_fresh":"2023-09-23T20:58:44.066917+0000","last_change":"2023-09-23T20:54:31.883504+0000","last_active":"2023-09-23T20:58:44.066917+0000","last_peered":"2023-09-23T20:58:44.066917+0000","last_clean":"2023-09-23T20:58:44.066917+0000","last_became_active":"2023-09-23T20:54:31.883172+0000","last_became_peered":"2023-09-23T20:54:31.883172+0000","last_unstale":"2023-09-23T20:58:44.066917+0000","last_undegraded":"2023-09-23T20:58:44.066917+0000","last_fullsized":"2023-09-23T20:58:44.066917+0000","mapping_epoch":136,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:45:40.289120+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"79'161","reported_seq":467,"reported_epoch":236,"state":"active+clean","last_fresh":"2023-09-23T20:58:46.428020+0000","last_change":"2023-09-23T20:57:20.738417+0000","last_active":"2023-09-23T20:58:46.428020+0000","last_peered":"2023-09-23T20:58:46.428020+0000","last_clean":"2023-09-23T20:58:46.428020+0000","last_became_active":"2023-09-23T20:57:08.236842+0000","last_became_peered":"2023-09-23T20:57:08.236842+0000","last_unstale":"2023-09-23T20:58:46.428020+0000","last_undegraded":"2023-09-23T20:58:46.428020+0000","last_fullsized":"2023-09-23T20:58:46.428020+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":0,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:20.738273+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:20.738273+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:18:28.327990+0000","scrub_duration":0.0088336339999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":214,"reported_epoch":236,"state":"active+clean","last_fresh":"2023-09-23T20:58:44.064702+0000","last_change":"2023-09-23T20:55:35.950777+0000","last_active":"2023-09-23T20:58:44.064702+0000","last_peered":"2023-09-23T20:58:44.064702+0000","last_clean":"2023-09-23T20:58:44.064702+0000","last_became_active":"2023-09-23T20:55:35.950427+0000","last_became_peered":"2023-09-23T20:55:35.950427+0000","last_unstale":"2023-09-23T20:58:44.064702+0000","last_undegraded":"2023-09-23T20:58:44.064702+0000","last_fullsized":"2023-09-23T20:58:44.064702+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:20:10.371751+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":181,"reported_epoch":236,"state":"active+clean","last_fresh":"2023-09-23T20:58:44.049880+0000","last_change":"2023-09-23T20:55:35.978140+0000","last_active":"2023-09-23T20:58:44.049880+0000","last_peered":"2023-09-23T20:58:44.049880+0000","last_clean":"2023-09-23T20:58:44.049880+0000","last_became_active":"2023-09-23T20:55:35.977831+0000","last_became_peered":"2023-09-23T20:55:35.977831+0000","last_unstale":"2023-09-23T20:58:44.049880+0000","last_undegraded":"2023-09-23T20:58:44.049880+0000","last_fullsized":"2023-09-23T20:58:44.049880+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:25:47.678023+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.5","version":"237'8308","reported_seq":8362,"reported_epoch":237,"state":"active+clean","last_fresh":"2023-09-23T20:59:01.414626+0000","last_change":"2023-09-23T20:56:45.990110+0000","last_active":"2023-09-23T20:59:01.414626+0000","last_peered":"2023-09-23T20:59:01.414626+0000","last_clean":"2023-09-23T20:59:01.414626+0000","last_became_active":"2023-09-23T20:56:45.988394+0000","last_became_peered":"2023-09-23T20:56:45.988394+0000","last_unstale":"2023-09-23T20:59:01.414626+0000","last_undegraded":"2023-09-23T20:59:01.414626+0000","last_fullsized":"2023-09-23T20:59:01.414626+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":199,"last_epoch_clean":205,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_clean_scrub_stamp":"2023-09-23T20:56:34.529103+0000","objects_scrubbed":0,"log_size":8308,"log_dups_size":0,"ondisk_log_size":8308,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:22:03.814210+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":286392320,"num_objects":4370,"num_object_clones":0,"num_object_copies":13110,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4370,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8308,"num_write_kb":405696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,0],"acting":[2,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"79'161","reported_seq":503,"reported_epoch":236,"state":"active+clean","last_fresh":"2023-09-23T20:58:44.083672+0000","last_change":"2023-09-23T20:57:20.378531+0000","last_active":"2023-09-23T20:58:44.083672+0000","last_peered":"2023-09-23T20:58:44.083672+0000","last_clean":"2023-09-23T20:58:44.083672+0000","last_became_active":"2023-09-23T20:57:08.238019+0000","last_became_peered":"2023-09-23T20:57:08.238019+0000","last_unstale":"2023-09-23T20:58:44.083672+0000","last_undegraded":"2023-09-23T20:58:44.083672+0000","last_fullsized":"2023-09-23T20:58:44.083672+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":3,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:20.377714+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:20.377714+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:00:08.305602+0000","scrub_duration":0.003015228,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":106,"reported_epoch":236,"state":"active+clean","last_fresh":"2023-09-23T20:58:49.296543+0000","last_change":"2023-09-23T20:56:45.991884+0000","last_active":"2023-09-23T20:58:49.296543+0000","last_peered":"2023-09-23T20:58:49.296543+0000","last_clean":"2023-09-23T20:58:49.296543+0000","last_became_active":"2023-09-23T20:56:45.991568+0000","last_became_peered":"2023-09-23T20:56:45.991568+0000","last_unstale":"2023-09-23T20:58:49.296543+0000","last_undegraded":"2023-09-23T20:58:49.296543+0000","last_fullsized":"2023-09-23T20:58:49.296543+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":205,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:21:20.775511+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,0],"acting":[1,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":230,"reported_epoch":236,"state":"active+clean","last_fresh":"2023-09-23T20:58:44.064975+0000","last_change":"2023-09-23T20:57:44.979950+0000","last_active":"2023-09-23T20:58:44.064975+0000","last_peered":"2023-09-23T20:58:44.064975+0000","last_clean":"2023-09-23T20:58:44.064975+0000","last_became_active":"2023-09-23T20:57:44.979346+0000","last_became_peered":"2023-09-23T20:57:44.979346+0000","last_unstale":"2023-09-23T20:58:44.064975+0000","last_undegraded":"2023-09-23T20:58:44.064975+0000","last_fullsized":"2023-09-23T20:58:44.064975+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:10:24.372800+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":209,"reported_epoch":236,"state":"active+clean","last_fresh":"2023-09-23T20:58:44.073890+0000","last_change":"2023-09-23T20:50:37.982235+0000","last_active":"2023-09-23T20:58:44.073890+0000","last_peered":"2023-09-23T20:58:44.073890+0000","last_clean":"2023-09-23T20:58:44.073890+0000","last_became_active":"2023-09-23T20:50:37.981913+0000","last_became_peered":"2023-09-23T20:50:37.981913+0000","last_unstale":"2023-09-23T20:58:44.073890+0000","last_undegraded":"2023-09-23T20:58:44.073890+0000","last_fullsized":"2023-09-23T20:58:44.073890+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:59:57.122584+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.0","version":"237'8353","reported_seq":8399,"reported_epoch":237,"state":"active+clean","last_fresh":"2023-09-23T20:59:02.089705+0000","last_change":"2023-09-23T20:56:35.550635+0000","last_active":"2023-09-23T20:59:02.089705+0000","last_peered":"2023-09-23T20:59:02.089705+0000","last_clean":"2023-09-23T20:59:02.089705+0000","last_became_active":"2023-09-23T20:56:35.550376+0000","last_became_peered":"2023-09-23T20:56:35.550376+0000","last_unstale":"2023-09-23T20:59:02.089705+0000","last_undegraded":"2023-09-23T20:59:02.089705+0000","last_fullsized":"2023-09-23T20:59:02.089705+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":199,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_clean_scrub_stamp":"2023-09-23T20:56:34.529103+0000","objects_scrubbed":0,"log_size":8353,"log_dups_size":0,"ondisk_log_size":8353,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:40:55.020049+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":283705344,"num_objects":4329,"num_object_clones":0,"num_object_copies":12987,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4329,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8353,"num_write_kb":405824,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.6","version":"79'161","reported_seq":519,"reported_epoch":236,"state":"active+clean","last_fresh":"2023-09-23T20:58:44.050123+0000","last_change":"2023-09-23T20:57:26.907505+0000","last_active":"2023-09-23T20:58:44.050123+0000","last_peered":"2023-09-23T20:58:44.050123+0000","last_clean":"2023-09-23T20:58:44.050123+0000","last_became_active":"2023-09-23T20:57:08.240321+0000","last_became_peered":"2023-09-23T20:57:08.240321+0000","last_unstale":"2023-09-23T20:58:44.050123+0000","last_undegraded":"2023-09-23T20:58:44.050123+0000","last_fullsized":"2023-09-23T20:58:44.050123+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":3,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:26.907408+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:26.907408+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:34:17.392928+0000","scrub_duration":0.010337038,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":153,"reported_epoch":236,"state":"active+clean","last_fresh":"2023-09-23T20:58:51.439554+0000","last_change":"2023-09-23T20:56:45.988566+0000","last_active":"2023-09-23T20:58:51.439554+0000","last_peered":"2023-09-23T20:58:51.439554+0000","last_clean":"2023-09-23T20:58:51.439554+0000","last_became_active":"2023-09-23T20:56:45.988249+0000","last_became_peered":"2023-09-23T20:56:45.988249+0000","last_unstale":"2023-09-23T20:58:51.439554+0000","last_undegraded":"2023-09-23T20:58:51.439554+0000","last_fullsized":"2023-09-23T20:58:51.439554+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":205,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:51:36.644764+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,4],"acting":[0,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":207,"reported_epoch":236,"state":"active+clean","last_fresh":"2023-09-23T20:58:44.084263+0000","last_change":"2023-09-23T20:54:31.898965+0000","last_active":"2023-09-23T20:58:44.084263+0000","last_peered":"2023-09-23T20:58:44.084263+0000","last_clean":"2023-09-23T20:58:44.084263+0000","last_became_active":"2023-09-23T20:54:31.898663+0000","last_became_peered":"2023-09-23T20:54:31.898663+0000","last_unstale":"2023-09-23T20:58:44.084263+0000","last_undegraded":"2023-09-23T20:58:44.084263+0000","last_fullsized":"2023-09-23T20:58:44.084263+0000","mapping_epoch":136,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:53:05.058087+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":197,"reported_epoch":236,"state":"active+clean","last_fresh":"2023-09-23T20:58:44.067946+0000","last_change":"2023-09-23T20:54:31.884717+0000","last_active":"2023-09-23T20:58:44.067946+0000","last_peered":"2023-09-23T20:58:44.067946+0000","last_clean":"2023-09-23T20:58:44.067946+0000","last_became_active":"2023-09-23T20:54:31.884345+0000","last_became_peered":"2023-09-23T20:54:31.884345+0000","last_unstale":"2023-09-23T20:58:44.067946+0000","last_undegraded":"2023-09-23T20:58:44.067946+0000","last_fullsized":"2023-09-23T20:58:44.067946+0000","mapping_epoch":136,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:02:26.393868+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.3","version":"237'8122","reported_seq":8170,"reported_epoch":237,"state":"active+clean","last_fresh":"2023-09-23T20:58:59.518968+0000","last_change":"2023-09-23T20:56:35.549655+0000","last_active":"2023-09-23T20:58:59.518968+0000","last_peered":"2023-09-23T20:58:59.518968+0000","last_clean":"2023-09-23T20:58:59.518968+0000","last_became_active":"2023-09-23T20:56:35.549357+0000","last_became_peered":"2023-09-23T20:56:35.549357+0000","last_unstale":"2023-09-23T20:58:59.518968+0000","last_undegraded":"2023-09-23T20:58:59.518968+0000","last_fullsized":"2023-09-23T20:58:59.518968+0000","mapping_epoch":199,"log_start":"205'1900","ondisk_log_start":"205'1900","created":199,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_clean_scrub_stamp":"2023-09-23T20:56:34.529103+0000","objects_scrubbed":0,"log_size":6222,"log_dups_size":0,"ondisk_log_size":6222,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:37:58.752518+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":288358400,"num_objects":4400,"num_object_clones":0,"num_object_copies":13200,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4400,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8122,"num_write_kb":400704,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.5","version":"79'161","reported_seq":500,"reported_epoch":236,"state":"active+clean","last_fresh":"2023-09-23T20:58:44.072902+0000","last_change":"2023-09-23T20:57:30.029479+0000","last_active":"2023-09-23T20:58:44.072902+0000","last_peered":"2023-09-23T20:58:44.072902+0000","last_clean":"2023-09-23T20:58:44.072902+0000","last_became_active":"2023-09-23T20:57:08.249909+0000","last_became_peered":"2023-09-23T20:57:08.249909+0000","last_unstale":"2023-09-23T20:58:44.072902+0000","last_undegraded":"2023-09-23T20:58:44.072902+0000","last_fullsized":"2023-09-23T20:58:44.072902+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":3,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:30.029396+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:30.029396+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:16:46.083145+0000","scrub_duration":0.012914682,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":200,"reported_epoch":236,"state":"active+clean","last_fresh":"2023-09-23T20:58:44.064322+0000","last_change":"2023-09-23T20:51:10.836910+0000","last_active":"2023-09-23T20:58:44.064322+0000","last_peered":"2023-09-23T20:58:44.064322+0000","last_clean":"2023-09-23T20:58:44.064322+0000","last_became_active":"2023-09-23T20:51:10.836557+0000","last_became_peered":"2023-09-23T20:51:10.836557+0000","last_unstale":"2023-09-23T20:58:44.064322+0000","last_undegraded":"2023-09-23T20:58:44.064322+0000","last_fullsized":"2023-09-23T20:58:44.064322+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:51:54.019969+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":207,"reported_epoch":236,"state":"active+clean","last_fresh":"2023-09-23T20:58:44.295250+0000","last_change":"2023-09-23T20:55:35.953444+0000","last_active":"2023-09-23T20:58:44.295250+0000","last_peered":"2023-09-23T20:58:44.295250+0000","last_clean":"2023-09-23T20:58:44.295250+0000","last_became_active":"2023-09-23T20:55:35.953151+0000","last_became_peered":"2023-09-23T20:55:35.953151+0000","last_unstale":"2023-09-23T20:58:44.295250+0000","last_undegraded":"2023-09-23T20:58:44.295250+0000","last_fullsized":"2023-09-23T20:58:44.295250+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:39:12.404391+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":25,"reported_epoch":236,"state":"active+clean","last_fresh":"2023-09-23T20:58:44.062419+0000","last_change":"2023-09-23T20:57:44.982509+0000","last_active":"2023-09-23T20:58:44.062419+0000","last_peered":"2023-09-23T20:58:44.062419+0000","last_clean":"2023-09-23T20:58:44.062419+0000","last_became_active":"2023-09-23T20:57:44.982175+0000","last_became_peered":"2023-09-23T20:57:44.982175+0000","last_unstale":"2023-09-23T20:58:44.062419+0000","last_undegraded":"2023-09-23T20:58:44.062419+0000","last_fullsized":"2023-09-23T20:58:44.062419+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:37:49.215488+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.2","version":"237'8144","reported_seq":8192,"reported_epoch":237,"state":"active+clean","last_fresh":"2023-09-23T20:58:59.431266+0000","last_change":"2023-09-23T20:56:35.543020+0000","last_active":"2023-09-23T20:58:59.431266+0000","last_peered":"2023-09-23T20:58:59.431266+0000","last_clean":"2023-09-23T20:58:59.431266+0000","last_became_active":"2023-09-23T20:56:35.542719+0000","last_became_peered":"2023-09-23T20:56:35.542719+0000","last_unstale":"2023-09-23T20:58:59.431266+0000","last_undegraded":"2023-09-23T20:58:59.431266+0000","last_fullsized":"2023-09-23T20:58:59.431266+0000","mapping_epoch":199,"log_start":"206'2000","ondisk_log_start":"206'2000","created":199,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_clean_scrub_stamp":"2023-09-23T20:56:34.529103+0000","objects_scrubbed":0,"log_size":6144,"log_dups_size":0,"ondisk_log_size":6144,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:52:21.088628+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":286261248,"num_objects":4368,"num_object_clones":0,"num_object_copies":13104,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4368,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8144,"num_write_kb":400384,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.4","version":"79'161","reported_seq":515,"reported_epoch":236,"state":"active+clean","last_fresh":"2023-09-23T20:58:44.064961+0000","last_change":"2023-09-23T20:57:20.012904+0000","last_active":"2023-09-23T20:58:44.064961+0000","last_peered":"2023-09-23T20:58:44.064961+0000","last_clean":"2023-09-23T20:58:44.064961+0000","last_became_active":"2023-09-23T20:57:08.237996+0000","last_became_peered":"2023-09-23T20:57:08.237996+0000","last_unstale":"2023-09-23T20:58:44.064961+0000","last_undegraded":"2023-09-23T20:58:44.064961+0000","last_fullsized":"2023-09-23T20:58:44.064961+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":3,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:20.012764+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:20.012764+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:51:28.796946+0000","scrub_duration":0.0045251919999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":186,"reported_epoch":236,"state":"active+clean","last_fresh":"2023-09-23T20:58:44.049904+0000","last_change":"2023-09-23T20:55:35.949541+0000","last_active":"2023-09-23T20:58:44.049904+0000","last_peered":"2023-09-23T20:58:44.049904+0000","last_clean":"2023-09-23T20:58:44.049904+0000","last_became_active":"2023-09-23T20:55:35.944635+0000","last_became_peered":"2023-09-23T20:55:35.944635+0000","last_unstale":"2023-09-23T20:58:44.049904+0000","last_undegraded":"2023-09-23T20:58:44.049904+0000","last_fullsized":"2023-09-23T20:58:44.049904+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:30:36.033585+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"70'1","reported_seq":225,"reported_epoch":236,"state":"active+clean","last_fresh":"2023-09-23T20:58:44.073990+0000","last_change":"2023-09-23T20:54:14.009723+0000","last_active":"2023-09-23T20:58:44.073990+0000","last_peered":"2023-09-23T20:58:44.073990+0000","last_clean":"2023-09-23T20:58:44.073990+0000","last_became_active":"2023-09-23T20:54:13.941771+0000","last_became_peered":"2023-09-23T20:54:13.941771+0000","last_unstale":"2023-09-23T20:58:44.073990+0000","last_undegraded":"2023-09-23T20:58:44.073990+0000","last_fullsized":"2023-09-23T20:58:44.073990+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:48:25.588116+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":141,"reported_epoch":236,"state":"active+clean","last_fresh":"2023-09-23T20:58:44.074980+0000","last_change":"2023-09-23T20:55:35.949780+0000","last_active":"2023-09-23T20:58:44.074980+0000","last_peered":"2023-09-23T20:58:44.074980+0000","last_clean":"2023-09-23T20:58:44.074980+0000","last_became_active":"2023-09-23T20:55:35.949497+0000","last_became_peered":"2023-09-23T20:55:35.949497+0000","last_unstale":"2023-09-23T20:58:44.074980+0000","last_undegraded":"2023-09-23T20:58:44.074980+0000","last_fullsized":"2023-09-23T20:58:44.074980+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:09:37.420882+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":199,"reported_epoch":236,"state":"active+clean","last_fresh":"2023-09-23T20:58:44.067816+0000","last_change":"2023-09-23T20:54:13.929196+0000","last_active":"2023-09-23T20:58:44.067816+0000","last_peered":"2023-09-23T20:58:44.067816+0000","last_clean":"2023-09-23T20:58:44.067816+0000","last_became_active":"2023-09-23T20:54:13.928831+0000","last_became_peered":"2023-09-23T20:54:13.928831+0000","last_unstale":"2023-09-23T20:58:44.067816+0000","last_undegraded":"2023-09-23T20:58:44.067816+0000","last_fullsized":"2023-09-23T20:58:44.067816+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:28:44.316851+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":205,"reported_epoch":236,"state":"active+clean","last_fresh":"2023-09-23T20:58:44.063073+0000","last_change":"2023-09-23T20:52:12.855430+0000","last_active":"2023-09-23T20:58:44.063073+0000","last_peered":"2023-09-23T20:58:44.063073+0000","last_clean":"2023-09-23T20:58:44.063073+0000","last_became_active":"2023-09-23T20:52:12.855115+0000","last_became_peered":"2023-09-23T20:52:12.855115+0000","last_unstale":"2023-09-23T20:58:44.063073+0000","last_undegraded":"2023-09-23T20:58:44.063073+0000","last_fullsized":"2023-09-23T20:58:44.063073+0000","mapping_epoch":100,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":101,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:33:23.802214+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":114,"reported_epoch":236,"state":"active+clean","last_fresh":"2023-09-23T20:58:44.066902+0000","last_change":"2023-09-23T20:54:31.902946+0000","last_active":"2023-09-23T20:58:44.066902+0000","last_peered":"2023-09-23T20:58:44.066902+0000","last_clean":"2023-09-23T20:58:44.066902+0000","last_became_active":"2023-09-23T20:54:31.889484+0000","last_became_peered":"2023-09-23T20:54:31.889484+0000","last_unstale":"2023-09-23T20:58:44.066902+0000","last_undegraded":"2023-09-23T20:58:44.066902+0000","last_fullsized":"2023-09-23T20:58:44.066902+0000","mapping_epoch":136,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:44:44.278398+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":114,"reported_epoch":236,"state":"active+clean","last_fresh":"2023-09-23T20:58:44.072812+0000","last_change":"2023-09-23T20:54:13.950351+0000","last_active":"2023-09-23T20:58:44.072812+0000","last_peered":"2023-09-23T20:58:44.072812+0000","last_clean":"2023-09-23T20:58:44.072812+0000","last_became_active":"2023-09-23T20:54:13.949987+0000","last_became_peered":"2023-09-23T20:54:13.949987+0000","last_unstale":"2023-09-23T20:58:44.072812+0000","last_undegraded":"2023-09-23T20:58:44.072812+0000","last_fullsized":"2023-09-23T20:58:44.072812+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:46:17.765134+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":145,"reported_epoch":236,"state":"active+clean","last_fresh":"2023-09-23T20:58:44.066853+0000","last_change":"2023-09-23T20:56:18.141728+0000","last_active":"2023-09-23T20:58:44.066853+0000","last_peered":"2023-09-23T20:58:44.066853+0000","last_clean":"2023-09-23T20:58:44.066853+0000","last_became_active":"2023-09-23T20:56:18.137979+0000","last_became_peered":"2023-09-23T20:56:18.137979+0000","last_unstale":"2023-09-23T20:58:44.066853+0000","last_undegraded":"2023-09-23T20:58:44.066853+0000","last_fullsized":"2023-09-23T20:58:44.066853+0000","mapping_epoch":192,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":193,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:58:05.658443+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":114,"reported_epoch":236,"state":"active+clean","last_fresh":"2023-09-23T20:58:44.072583+0000","last_change":"2023-09-23T20:54:13.935307+0000","last_active":"2023-09-23T20:58:44.072583+0000","last_peered":"2023-09-23T20:58:44.072583+0000","last_clean":"2023-09-23T20:58:44.072583+0000","last_became_active":"2023-09-23T20:54:13.935000+0000","last_became_peered":"2023-09-23T20:54:13.935000+0000","last_unstale":"2023-09-23T20:58:44.072583+0000","last_undegraded":"2023-09-23T20:58:44.072583+0000","last_fullsized":"2023-09-23T20:58:44.072583+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:47:40.974082+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":223,"reported_epoch":236,"state":"active+clean","last_fresh":"2023-09-23T20:58:44.064020+0000","last_change":"2023-09-23T20:55:35.954870+0000","last_active":"2023-09-23T20:58:44.064020+0000","last_peered":"2023-09-23T20:58:44.064020+0000","last_clean":"2023-09-23T20:58:44.064020+0000","last_became_active":"2023-09-23T20:55:35.954568+0000","last_became_peered":"2023-09-23T20:55:35.954568+0000","last_unstale":"2023-09-23T20:58:44.064020+0000","last_undegraded":"2023-09-23T20:58:44.064020+0000","last_fullsized":"2023-09-23T20:58:44.064020+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:06:22.423566+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":223,"reported_epoch":236,"state":"active+clean","last_fresh":"2023-09-23T20:58:44.062878+0000","last_change":"2023-09-23T20:51:10.838334+0000","last_active":"2023-09-23T20:58:44.062878+0000","last_peered":"2023-09-23T20:58:44.062878+0000","last_clean":"2023-09-23T20:58:44.062878+0000","last_became_active":"2023-09-23T20:51:10.838026+0000","last_became_peered":"2023-09-23T20:51:10.838026+0000","last_unstale":"2023-09-23T20:58:44.062878+0000","last_undegraded":"2023-09-23T20:58:44.062878+0000","last_fullsized":"2023-09-23T20:58:44.062878+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:04:37.009457+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":227,"reported_epoch":236,"state":"active+clean","last_fresh":"2023-09-23T20:58:44.068405+0000","last_change":"2023-09-23T20:54:31.903139+0000","last_active":"2023-09-23T20:58:44.068405+0000","last_peered":"2023-09-23T20:58:44.068405+0000","last_clean":"2023-09-23T20:58:44.068405+0000","last_became_active":"2023-09-23T20:54:31.884520+0000","last_became_peered":"2023-09-23T20:54:31.884520+0000","last_unstale":"2023-09-23T20:58:44.068405+0000","last_undegraded":"2023-09-23T20:58:44.068405+0000","last_fullsized":"2023-09-23T20:58:44.068405+0000","mapping_epoch":136,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:39:34.269705+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1108,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":131,"reported_epoch":236,"state":"active+clean","last_fresh":"2023-09-23T20:58:44.072302+0000","last_change":"2023-09-23T20:55:35.951572+0000","last_active":"2023-09-23T20:58:44.072302+0000","last_peered":"2023-09-23T20:58:44.072302+0000","last_clean":"2023-09-23T20:58:44.072302+0000","last_became_active":"2023-09-23T20:55:35.951294+0000","last_became_peered":"2023-09-23T20:55:35.951294+0000","last_unstale":"2023-09-23T20:58:44.072302+0000","last_undegraded":"2023-09-23T20:58:44.072302+0000","last_fullsized":"2023-09-23T20:58:44.072302+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:36:32.447210+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":7,"num_pg":16,"stat_sum":{"num_bytes":4613603352,"num_objects":70399,"num_object_clones":0,"num_object_copies":211197,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":11300,"num_objects_misplaced":3418,"num_objects_unfound":0,"num_objects_dirty":70399,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":133121,"num_write_kb":6512577,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1624,"num_bytes_recovered":106430464,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":14197010432,"data_stored":14196998216,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":117321,"ondisk_log_size":117321,"up":48,"acting":45,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":22,"num_bytes_recovered":5990,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":21,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3381,"ondisk_log_size":3381,"up":63,"acting":63,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201503,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1578208,"kb_used_data":1401240,"kb_used_omap":0,"kb_used_meta":176960,"kb_avail":92166944,"statfs":{"total":95995035648,"available":94378950656,"internally_reserved":0,"allocated":1434869760,"data_stored":1434030131,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":181207040},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:58:44 2023","interfaces":[{"interface":"back","average":{"1min":2.052,"5min":1.859,"15min":1.006},"min":{"1min":0.248,"5min":0.21099999999999999,"15min":0.21099999999999999},"max":{"1min":5.5279999999999996,"5min":43.994999999999997,"15min":43.994999999999997},"last":0.78800000000000003},{"interface":"front","average":{"1min":2.0489999999999999,"5min":2.0760000000000001,"15min":1.099},"min":{"1min":0.34599999999999997,"5min":0.28199999999999997,"15min":0.218},"max":{"1min":5.5129999999999999,"5min":44.206000000000003,"15min":44.206000000000003},"last":0.73499999999999999}]},{"osd":1,"last update":"Sat Sep 23 20:58:18 2023","interfaces":[{"interface":"back","average":{"1min":1.895,"5min":1.9330000000000001,"15min":1.1439999999999999},"min":{"1min":0.29999999999999999,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":10.622,"5min":10.622,"15min":22.591999999999999},"last":5.4359999999999999},{"interface":"front","average":{"1min":1.8320000000000001,"5min":1.5720000000000001,"15min":0.98499999999999999},"min":{"1min":0.25800000000000001,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":7.5629999999999997,"5min":10.097,"15min":21.437000000000001},"last":0.95999999999999996}]},{"osd":2,"last update":"Sat Sep 23 20:58:54 2023","interfaces":[{"interface":"back","average":{"1min":3.7370000000000001,"5min":2.016,"15min":1.1499999999999999},"min":{"1min":0.27600000000000002,"5min":0.26500000000000001,"15min":0.24399999999999999},"max":{"1min":10.805999999999999,"5min":10.805999999999999,"15min":10.805999999999999},"last":0.91400000000000003},{"interface":"front","average":{"1min":2.3969999999999998,"5min":2.1520000000000001,"15min":1.1919999999999999},"min":{"1min":0.27800000000000002,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":9.4290000000000003,"5min":45.25,"15min":45.25},"last":0.33100000000000002}]},{"osd":3,"last update":"Sat Sep 23 20:58:26 2023","interfaces":[{"interface":"back","average":{"1min":3.355,"5min":3.226,"15min":1.587},"min":{"1min":0.32100000000000001,"5min":0.314,"15min":0.314},"max":{"1min":9.9939999999999998,"5min":36.408000000000001,"15min":36.408000000000001},"last":4.8650000000000002},{"interface":"front","average":{"1min":2.5649999999999999,"5min":2.7160000000000002,"15min":1.405},"min":{"1min":0.32000000000000001,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":9.4529999999999994,"5min":35.593000000000004,"15min":35.593000000000004},"last":1.256}]},{"osd":4,"last update":"Sat Sep 23 20:58:18 2023","interfaces":[{"interface":"back","average":{"1min":0.69999999999999996,"5min":0.82699999999999996,"15min":0.60899999999999999},"min":{"1min":0.28299999999999997,"5min":0.23799999999999999,"15min":0.23699999999999999},"max":{"1min":4.5410000000000004,"5min":5.8920000000000003,"15min":5.8920000000000003},"last":0.25700000000000001},{"interface":"front","average":{"1min":0.70399999999999996,"5min":0.76100000000000001,"15min":0.61399999999999999},"min":{"1min":0.27100000000000002,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":4.4080000000000004,"5min":5.3200000000000003,"15min":5.3200000000000003},"last":0.316}]},{"osd":5,"last update":"Sat Sep 23 20:58:18 2023","interfaces":[{"interface":"back","average":{"1min":0.69299999999999995,"5min":0.82799999999999996,"15min":0.63300000000000001},"min":{"1min":0.28499999999999998,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":2.5110000000000001,"5min":7.0389999999999997,"15min":7.0389999999999997},"last":0.35799999999999998},{"interface":"front","average":{"1min":0.72199999999999998,"5min":0.70699999999999996,"15min":0.61199999999999999},"min":{"1min":0.317,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":4.5999999999999996,"5min":4.5999999999999996,"15min":4.5999999999999996},"last":0.30199999999999999}]},{"osd":6,"last update":"Sat Sep 23 20:58:40 2023","interfaces":[{"interface":"back","average":{"1min":0.86599999999999999,"5min":0.71699999999999997,"15min":0.68300000000000005},"min":{"1min":0.33700000000000002,"5min":0.31,"15min":0.31},"max":{"1min":3.5249999999999999,"5min":3.5249999999999999,"15min":3.5249999999999999},"last":0.28899999999999998},{"interface":"front","average":{"1min":0.90100000000000002,"5min":0.81299999999999994,"15min":0.80700000000000005},"min":{"1min":0.36899999999999999,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":3.5409999999999999,"5min":3.5409999999999999,"15min":3.5409999999999999},"last":0.34699999999999998}]}]},{"osd":6,"up_from":164,"seq":704374636586,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1539412,"kb_used_data":1441484,"kb_used_omap":10,"kb_used_meta":97909,"kb_avail":92205740,"statfs":{"total":95995035648,"available":94418677760,"internally_reserved":0,"allocated":1476079616,"data_stored":1475251888,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":10328,"internal_metadata":100259752},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:58:43 2023","interfaces":[{"interface":"back","average":{"1min":2.7330000000000001,"5min":1.4410000000000001,"15min":1.0569999999999999},"min":{"1min":0.25800000000000001,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":7.5179999999999998,"5min":8.7319999999999993,"15min":8.7319999999999993},"last":0.47399999999999998},{"interface":"front","average":{"1min":3.1709999999999998,"5min":1.502,"15min":1.1559999999999999},"min":{"1min":0.25,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":6.7140000000000004,"5min":6.7140000000000004,"15min":6.7140000000000004},"last":0.45400000000000001}]},{"osd":1,"last update":"Sat Sep 23 20:58:43 2023","interfaces":[{"interface":"back","average":{"1min":1.5449999999999999,"5min":1.097,"15min":0.89100000000000001},"min":{"1min":0.36799999999999999,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":5.2350000000000003,"5min":6.6029999999999998,"15min":6.6029999999999998},"last":0.52500000000000002},{"interface":"front","average":{"1min":2.5779999999999998,"5min":1.244,"15min":0.99299999999999999},"min":{"1min":0.23000000000000001,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":10.324999999999999,"5min":10.324999999999999,"15min":10.324999999999999},"last":0.497}]},{"osd":2,"last update":"Sat Sep 23 20:58:43 2023","interfaces":[{"interface":"back","average":{"1min":2.286,"5min":1.982,"15min":1.758},"min":{"1min":0.28999999999999998,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":7.0519999999999996,"5min":13.750999999999999,"15min":13.750999999999999},"last":0.56100000000000005},{"interface":"front","average":{"1min":2.419,"5min":1.5289999999999999,"15min":1.2470000000000001},"min":{"1min":0.26200000000000001,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":8.0890000000000004,"5min":8.0890000000000004,"15min":8.0890000000000004},"last":0.57499999999999996}]},{"osd":3,"last update":"Sat Sep 23 20:58:43 2023","interfaces":[{"interface":"back","average":{"1min":1.8680000000000001,"5min":1.2849999999999999,"15min":1.173},"min":{"1min":0.32000000000000001,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":6.9000000000000004,"5min":6.9000000000000004,"15min":6.9000000000000004},"last":0.67400000000000004},{"interface":"front","average":{"1min":2.1819999999999999,"5min":1.74,"15min":1.6479999999999999},"min":{"1min":0.28599999999999998,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":11.021000000000001,"5min":11.021000000000001,"15min":11.021000000000001},"last":0.65800000000000003}]},{"osd":4,"last update":"Sat Sep 23 20:58:43 2023","interfaces":[{"interface":"back","average":{"1min":0.49099999999999999,"5min":0.85899999999999999,"15min":0.98699999999999999},"min":{"1min":0.23000000000000001,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":2.7919999999999998,"5min":9.0079999999999991,"15min":9.0079999999999991},"last":0.77200000000000002},{"interface":"front","average":{"1min":0.49299999999999999,"5min":0.50800000000000001,"15min":0.52300000000000002},"min":{"1min":0.27200000000000002,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":2.0059999999999998,"5min":2.0059999999999998,"15min":2.0059999999999998},"last":0.73599999999999999}]},{"osd":5,"last update":"Sat Sep 23 20:58:43 2023","interfaces":[{"interface":"back","average":{"1min":0.53700000000000003,"5min":0.56899999999999995,"15min":0.55000000000000004},"min":{"1min":0.27300000000000002,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":2.8239999999999998,"5min":2.8239999999999998,"15min":2.8239999999999998},"last":0.749},{"interface":"front","average":{"1min":0.81000000000000005,"5min":0.78500000000000003,"15min":0.80100000000000005},"min":{"1min":0.33400000000000002,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":3.1379999999999999,"5min":3.1379999999999999,"15min":3.1379999999999999},"last":0.76200000000000001}]},{"osd":7,"last update":"Sat Sep 23 20:58:43 2023","interfaces":[{"interface":"back","average":{"1min":0.52700000000000002,"5min":0.85399999999999998,"15min":0.90400000000000003},"min":{"1min":0.312,"5min":0.312,"15min":0.312},"max":{"1min":2.073,"5min":7.8570000000000002,"15min":7.8570000000000002},"last":0.70599999999999996},{"interface":"front","average":{"1min":0.61899999999999999,"5min":0.66200000000000003,"15min":0.57399999999999995},"min":{"1min":0.30299999999999999,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":2.6989999999999998,"5min":8.282,"15min":8.282},"last":0.63200000000000001}]}]},{"osd":1,"up_from":88,"seq":377957122157,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2293448,"kb_used_data":2070528,"kb_used_omap":9,"kb_used_meta":222838,"kb_avail":91451704,"statfs":{"total":95995035648,"available":93646544896,"internally_reserved":0,"allocated":2120220672,"data_stored":2119057203,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9239,"internal_metadata":228187113},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:58:22 2023","interfaces":[{"interface":"back","average":{"1min":3.3290000000000002,"5min":2.9990000000000001,"15min":1.3049999999999999},"min":{"1min":0.23200000000000001,"5min":0.216,"15min":0.216},"max":{"1min":10.73,"5min":16.646999999999998,"15min":16.646999999999998},"last":3.6280000000000001},{"interface":"front","average":{"1min":4.2149999999999999,"5min":3.3940000000000001,"15min":1.4590000000000001},"min":{"1min":0.247,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":15.135999999999999,"5min":18.457999999999998,"15min":18.457999999999998},"last":3.6419999999999999}]},{"osd":2,"last update":"Sat Sep 23 20:58:53 2023","interfaces":[{"interface":"back","average":{"1min":2.7130000000000001,"5min":2.4140000000000001,"15min":1.1799999999999999},"min":{"1min":0.24299999999999999,"5min":0.221,"15min":0.20300000000000001},"max":{"1min":8.6829999999999998,"5min":14.335000000000001,"15min":14.335000000000001},"last":3.2469999999999999},{"interface":"front","average":{"1min":5.8019999999999996,"5min":3.27,"15min":1.758},"min":{"1min":0.39500000000000002,"5min":0.248,"15min":0.248},"max":{"1min":16.048999999999999,"5min":16.048999999999999,"15min":38.951000000000001},"last":0.53000000000000003}]},{"osd":3,"last update":"Sat Sep 23 20:58:22 2023","interfaces":[{"interface":"back","average":{"1min":3.1859999999999999,"5min":3.0489999999999999,"15min":1.3720000000000001},"min":{"1min":0.246,"5min":0.246,"15min":0.246},"max":{"1min":10.701000000000001,"5min":25.314,"15min":25.314},"last":1.0649999999999999},{"interface":"front","average":{"1min":2.8450000000000002,"5min":2.6150000000000002,"15min":1.232},"min":{"1min":0.33800000000000002,"5min":0.20599999999999999,"15min":0.20599999999999999},"max":{"1min":7.8899999999999997,"5min":19.881,"15min":19.881},"last":0.872}]},{"osd":4,"last update":"Sat Sep 23 20:58:22 2023","interfaces":[{"interface":"back","average":{"1min":1.7050000000000001,"5min":1.9550000000000001,"15min":1.131},"min":{"1min":0.36099999999999999,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":10.598000000000001,"5min":12.006,"15min":12.006},"last":0.496},{"interface":"front","average":{"1min":2.0899999999999999,"5min":1.6839999999999999,"15min":1.016},"min":{"1min":0.31900000000000001,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":6.0460000000000003,"5min":11.234999999999999,"15min":11.234999999999999},"last":0.67200000000000004}]},{"osd":5,"last update":"Sat Sep 23 20:58:22 2023","interfaces":[{"interface":"back","average":{"1min":3.0609999999999999,"5min":2.133,"15min":1.181},"min":{"1min":0.34399999999999997,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":15.041,"5min":15.041,"15min":15.041},"last":0.55800000000000005},{"interface":"front","average":{"1min":1.669,"5min":2.0169999999999999,"15min":1.153},"min":{"1min":0.33200000000000002,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":10.647,"5min":12.08,"15min":12.08},"last":0.432}]},{"osd":6,"last update":"Sat Sep 23 20:58:39 2023","interfaces":[{"interface":"back","average":{"1min":1.6160000000000001,"5min":1.3700000000000001,"15min":1.333},"min":{"1min":0.30299999999999999,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":10.673999999999999,"5min":10.673999999999999,"15min":10.673999999999999},"last":0.46999999999999997},{"interface":"front","average":{"1min":1.8280000000000001,"5min":1.508,"15min":1.3380000000000001},"min":{"1min":0.38100000000000001,"5min":0.309,"15min":0.309},"max":{"1min":8.7530000000000001,"5min":8.7530000000000001,"15min":8.7530000000000001},"last":0.64600000000000002}]},{"osd":7,"last update":"Sat Sep 23 20:58:22 2023","interfaces":[{"interface":"back","average":{"1min":2.0659999999999998,"5min":1.7909999999999999,"15min":1.069},"min":{"1min":0.42499999999999999,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":6.149,"5min":11.186999999999999,"15min":11.186999999999999},"last":0.61799999999999999},{"interface":"front","average":{"1min":3.1520000000000001,"5min":2.1829999999999998,"15min":1.2010000000000001},"min":{"1min":0.41899999999999998,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":15.098000000000001,"5min":15.098000000000001,"15min":15.098000000000001},"last":0.58499999999999996}]}]},{"osd":0,"up_from":83,"seq":356482285684,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2437552,"kb_used_data":2253288,"kb_used_omap":28,"kb_used_meta":184163,"kb_avail":91307600,"statfs":{"total":95995035648,"available":93498982400,"internally_reserved":0,"allocated":2307366912,"data_stored":2306225652,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":28848,"internal_metadata":188583760},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 20:58:19 2023","interfaces":[{"interface":"back","average":{"1min":3.7799999999999998,"5min":2.9220000000000002,"15min":1.2609999999999999},"min":{"1min":0.25800000000000001,"5min":0.191,"15min":0.191},"max":{"1min":15.946999999999999,"5min":18.863,"15min":18.863},"last":2.988},{"interface":"front","average":{"1min":3.9060000000000001,"5min":2.8540000000000001,"15min":1.2669999999999999},"min":{"1min":0.247,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":15.766999999999999,"5min":17.704999999999998,"15min":17.704999999999998},"last":0.223}]},{"osd":2,"last update":"Sat Sep 23 20:58:50 2023","interfaces":[{"interface":"back","average":{"1min":4.843,"5min":3.2549999999999999,"15min":1.6359999999999999},"min":{"1min":0.434,"5min":0.24299999999999999,"15min":0.23799999999999999},"max":{"1min":14.942,"5min":14.976000000000001,"15min":17.753},"last":4.6180000000000003},{"interface":"front","average":{"1min":7.774,"5min":3.8180000000000001,"15min":1.726},"min":{"1min":0.38500000000000001,"5min":0.25800000000000001,"15min":0.21099999999999999},"max":{"1min":21.747,"5min":21.747,"15min":21.747},"last":7.9390000000000001}]},{"osd":3,"last update":"Sat Sep 23 20:58:19 2023","interfaces":[{"interface":"back","average":{"1min":3.722,"5min":2.8919999999999999,"15min":1.298},"min":{"1min":0.39300000000000002,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":15.994,"5min":18.893999999999998,"15min":18.893999999999998},"last":1.1579999999999999},{"interface":"front","average":{"1min":3.7930000000000001,"5min":3.0649999999999999,"15min":1.423},"min":{"1min":0.29699999999999999,"5min":0.28000000000000003,"15min":0.23300000000000001},"max":{"1min":15.808,"5min":17.472000000000001,"15min":17.472000000000001},"last":4.7050000000000001}]},{"osd":4,"last update":"Sat Sep 23 20:58:50 2023","interfaces":[{"interface":"back","average":{"1min":6.6399999999999997,"5min":2.9279999999999999,"15min":1.4730000000000001},"min":{"1min":0.36599999999999999,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":21.594999999999999,"5min":21.594999999999999,"15min":21.594999999999999},"last":0.40600000000000003},{"interface":"front","average":{"1min":2.657,"5min":1.9790000000000001,"15min":1.131},"min":{"1min":0.35199999999999998,"5min":0.31,"15min":0.28100000000000003},"max":{"1min":10.393000000000001,"5min":18.724,"15min":18.724},"last":0.82399999999999995}]},{"osd":5,"last update":"Sat Sep 23 20:58:50 2023","interfaces":[{"interface":"back","average":{"1min":2.2719999999999998,"5min":1.8069999999999999,"15min":1.177},"min":{"1min":0.34999999999999998,"5min":0.318,"15min":0.318},"max":{"1min":8.8970000000000002,"5min":8.8970000000000002,"15min":8.8970000000000002},"last":3.0070000000000001},{"interface":"front","average":{"1min":5.7110000000000003,"5min":2.8919999999999999,"15min":1.4950000000000001},"min":{"1min":0.40600000000000003,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":20.512,"5min":20.512,"15min":20.512},"last":0.434}]},{"osd":6,"last update":"Sat Sep 23 20:58:44 2023","interfaces":[{"interface":"back","average":{"1min":2.681,"5min":1.893,"15min":1.7350000000000001},"min":{"1min":0.39900000000000002,"5min":0.37,"15min":0.37},"max":{"1min":10.420999999999999,"5min":10.420999999999999,"15min":10.420999999999999},"last":0.85799999999999998},{"interface":"front","average":{"1min":2.7120000000000002,"5min":1.9650000000000001,"15min":1.819},"min":{"1min":0.42199999999999999,"5min":0.377,"15min":0.377},"max":{"1min":10.331,"5min":10.331,"15min":10.331},"last":0.90200000000000002}]},{"osd":7,"last update":"Sat Sep 23 20:58:50 2023","interfaces":[{"interface":"back","average":{"1min":2.4729999999999999,"5min":1.875,"15min":1.22},"min":{"1min":0.38100000000000001,"5min":0.379,"15min":0.379},"max":{"1min":8.7729999999999997,"5min":8.7729999999999997,"15min":8.7729999999999997},"last":3.044},{"interface":"front","average":{"1min":2.7010000000000001,"5min":2.032,"15min":1.1619999999999999},"min":{"1min":0.377,"5min":0.377,"15min":0.377},"max":{"1min":10.359999999999999,"5min":18.673999999999999,"15min":18.673999999999999},"last":0.88200000000000001}]}]},{"osd":2,"up_from":93,"seq":399431958631,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2320992,"kb_used_data":2029464,"kb_used_omap":81,"kb_used_meta":291310,"kb_avail":91424160,"statfs":{"total":95995035648,"available":93618339840,"internally_reserved":0,"allocated":2078171136,"data_stored":2077042624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":83750,"internal_metadata":298301658},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:58:47 2023","interfaces":[{"interface":"back","average":{"1min":3.1429999999999998,"5min":2.0670000000000002,"15min":1.079},"min":{"1min":0.22700000000000001,"5min":0.17999999999999999,"15min":0.17999999999999999},"max":{"1min":14.773999999999999,"5min":14.773999999999999,"15min":14.773999999999999},"last":0.189},{"interface":"front","average":{"1min":5.1459999999999999,"5min":3.133,"15min":1.4319999999999999},"min":{"1min":0.32400000000000001,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":17.045000000000002,"5min":17.045000000000002,"15min":17.045000000000002},"last":0.20599999999999999}]},{"osd":1,"last update":"Sat Sep 23 20:58:47 2023","interfaces":[{"interface":"back","average":{"1min":4.1470000000000002,"5min":2.718,"15min":1.48},"min":{"1min":0.29299999999999998,"5min":0.20200000000000001,"15min":0.20200000000000001},"max":{"1min":14.709,"5min":14.709,"15min":16.081},"last":0.27800000000000002},{"interface":"front","average":{"1min":5.3819999999999997,"5min":3.6389999999999998,"15min":1.77},"min":{"1min":0.41499999999999998,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":11.492000000000001,"5min":12.07,"15min":16.041},"last":0.26600000000000001}]},{"osd":3,"last update":"Sat Sep 23 20:58:26 2023","interfaces":[{"interface":"back","average":{"1min":5.1120000000000001,"5min":3.4620000000000002,"15min":1.585},"min":{"1min":0.23300000000000001,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":23.309999999999999,"5min":23.309999999999999,"15min":23.309999999999999},"last":27.614999999999998},{"interface":"front","average":{"1min":5.5659999999999998,"5min":4.21,"15min":1.8240000000000001},"min":{"1min":0.34300000000000003,"5min":0.22800000000000001,"15min":0.221},"max":{"1min":23.324999999999999,"5min":23.324999999999999,"15min":23.324999999999999},"last":27.562000000000001}]},{"osd":4,"last update":"Sat Sep 23 20:58:47 2023","interfaces":[{"interface":"back","average":{"1min":2.2519999999999998,"5min":1.9910000000000001,"15min":1.1439999999999999},"min":{"1min":0.28699999999999998,"5min":0.28699999999999998,"15min":0.26400000000000001},"max":{"1min":8.4580000000000002,"5min":9.8119999999999994,"15min":9.8119999999999994},"last":0.52200000000000002},{"interface":"front","average":{"1min":2.137,"5min":2.1560000000000001,"15min":1.2270000000000001},"min":{"1min":0.32400000000000001,"5min":0.313,"15min":0.313},"max":{"1min":8.1649999999999991,"5min":10.609,"15min":10.609},"last":0.67000000000000004}]},{"osd":5,"last update":"Sat Sep 23 20:58:47 2023","interfaces":[{"interface":"back","average":{"1min":2.1779999999999999,"5min":2.238,"15min":1.2470000000000001},"min":{"1min":0.32300000000000001,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":8.5,"5min":9.7629999999999999,"15min":9.7629999999999999},"last":0.48399999999999999},{"interface":"front","average":{"1min":2.0670000000000002,"5min":2.1709999999999998,"15min":1.242},"min":{"1min":0.36399999999999999,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":8.1170000000000009,"5min":10.574,"15min":10.574},"last":0.627}]},{"osd":6,"last update":"Sat Sep 23 20:58:42 2023","interfaces":[{"interface":"back","average":{"1min":2.1360000000000001,"5min":1.8169999999999999,"15min":1.7170000000000001},"min":{"1min":0.379,"5min":0.312,"15min":0.312},"max":{"1min":8.2270000000000003,"5min":14.244999999999999,"15min":14.244999999999999},"last":0.51100000000000001},{"interface":"front","average":{"1min":2.1949999999999998,"5min":1.444,"15min":1.1679999999999999},"min":{"1min":0.41199999999999998,"5min":0.374,"15min":0.374},"max":{"1min":8.1980000000000004,"5min":8.1980000000000004,"15min":8.1980000000000004},"last":0.65000000000000002}]},{"osd":7,"last update":"Sat Sep 23 20:58:47 2023","interfaces":[{"interface":"back","average":{"1min":1.4830000000000001,"5min":1.4339999999999999,"15min":0.995},"min":{"1min":0.315,"5min":0.315,"15min":0.315},"max":{"1min":5.8570000000000002,"5min":8.0820000000000007,"15min":8.0820000000000007},"last":0.63800000000000001},{"interface":"front","average":{"1min":2.2090000000000001,"5min":2.2109999999999999,"15min":1.262},"min":{"1min":0.35099999999999998,"5min":0.35099999999999998,"15min":0.34499999999999997},"max":{"1min":8.5250000000000004,"5min":9.7859999999999996,"15min":9.7859999999999996},"last":0.54300000000000004}]}]},{"osd":3,"up_from":98,"seq":420906795104,"num_pgs":49,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2203780,"kb_used_data":1880252,"kb_used_omap":7,"kb_used_meta":323448,"kb_avail":91541372,"statfs":{"total":95995035648,"available":93738364928,"internally_reserved":0,"allocated":1925378048,"data_stored":1924274702,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7576,"internal_metadata":331211368},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:58:25 2023","interfaces":[{"interface":"back","average":{"1min":3.097,"5min":3.02,"15min":1.4890000000000001},"min":{"1min":0.308,"5min":0.20599999999999999,"15min":0.20599999999999999},"max":{"1min":8.5549999999999997,"5min":23.123999999999999,"15min":23.123999999999999},"last":6.968},{"interface":"front","average":{"1min":2.9900000000000002,"5min":2.7869999999999999,"15min":1.3009999999999999},"min":{"1min":0.19800000000000001,"5min":0.19800000000000001,"15min":0.19800000000000001},"max":{"1min":9.8889999999999993,"5min":15.079000000000001,"15min":15.079000000000001},"last":1.554}]},{"osd":1,"last update":"Sat Sep 23 20:58:25 2023","interfaces":[{"interface":"back","average":{"1min":3.7829999999999999,"5min":3.1509999999999998,"15min":1.4390000000000001},"min":{"1min":0.32400000000000001,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":13.173,"5min":21.385000000000002,"15min":21.385000000000002},"last":0.48799999999999999},{"interface":"front","average":{"1min":3.0880000000000001,"5min":3.242,"15min":1.5109999999999999},"min":{"1min":0.28799999999999998,"5min":0.218,"15min":0.218},"max":{"1min":10.683999999999999,"5min":25.353000000000002,"15min":25.353000000000002},"last":2.056}]},{"osd":2,"last update":"Sat Sep 23 20:58:25 2023","interfaces":[{"interface":"back","average":{"1min":4.851,"5min":3.052,"15min":1.444},"min":{"1min":0.35899999999999999,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":15.476000000000001,"5min":24.154,"15min":24.154},"last":1.8700000000000001},{"interface":"front","average":{"1min":2.8519999999999999,"5min":3.335,"15min":1.577},"min":{"1min":0.23999999999999999,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":8.6069999999999993,"5min":25.305,"15min":25.305},"last":2.0750000000000002}]},{"osd":4,"last update":"Sat Sep 23 20:58:25 2023","interfaces":[{"interface":"back","average":{"1min":2.6539999999999999,"5min":1.8240000000000001,"15min":1.141},"min":{"1min":0.38900000000000001,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":9.6170000000000009,"5min":11.861000000000001,"15min":11.861000000000001},"last":0.51700000000000002},{"interface":"front","average":{"1min":1.3600000000000001,"5min":2.0880000000000001,"15min":1.246},"min":{"1min":0.313,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":4.8109999999999999,"5min":22.853000000000002,"15min":22.853000000000002},"last":2.0110000000000001}]},{"osd":5,"last update":"Sat Sep 23 20:58:25 2023","interfaces":[{"interface":"back","average":{"1min":2.9540000000000002,"5min":1.8999999999999999,"15min":1.1459999999999999},"min":{"1min":0.436,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":9.6430000000000007,"5min":11.891999999999999,"15min":11.891999999999999},"last":1.0760000000000001},{"interface":"front","average":{"1min":2.04,"5min":1.6579999999999999,"15min":1.075},"min":{"1min":0.33300000000000002,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":5.7690000000000001,"5min":9.2870000000000008,"15min":9.2870000000000008},"last":0.61099999999999999}]},{"osd":6,"last update":"Sat Sep 23 20:58:43 2023","interfaces":[{"interface":"back","average":{"1min":1.9450000000000001,"5min":1.1930000000000001,"15min":1.0309999999999999},"min":{"1min":0.33500000000000002,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":10.018000000000001,"5min":10.018000000000001,"15min":10.018000000000001},"last":2.0390000000000001},{"interface":"front","average":{"1min":1.4770000000000001,"5min":1.48,"15min":1.4339999999999999},"min":{"1min":0.38700000000000001,"5min":0.38700000000000001,"15min":0.38700000000000001},"max":{"1min":5.3239999999999998,"5min":7.234,"15min":7.234},"last":0.66400000000000003}]},{"osd":7,"last update":"Sat Sep 23 20:58:25 2023","interfaces":[{"interface":"back","average":{"1min":3.0099999999999998,"5min":2.3399999999999999,"15min":1.3220000000000001},"min":{"1min":0.46500000000000002,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":9.5749999999999993,"5min":16.364999999999998,"15min":16.364999999999998},"last":0.54100000000000004},{"interface":"front","average":{"1min":2.214,"5min":1.766,"15min":1.147},"min":{"1min":0.42499999999999999,"5min":0.379,"15min":0.379},"max":{"1min":5.8090000000000002,"5min":9.3179999999999996,"15min":9.3179999999999996},"last":0.58399999999999996}]}]},{"osd":4,"up_from":31,"seq":133143986412,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1521760,"kb_used_data":1392216,"kb_used_omap":0,"kb_used_meta":129536,"kb_avail":92223392,"statfs":{"total":95995035648,"available":94436753408,"internally_reserved":0,"allocated":1425629184,"data_stored":1424789808,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":132644864},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:58:53 2023","interfaces":[{"interface":"back","average":{"1min":1.917,"5min":1.7170000000000001,"15min":1.042},"min":{"1min":0.245,"5min":0.214,"15min":0.214},"max":{"1min":8.8439999999999994,"5min":11.784000000000001,"15min":11.784000000000001},"last":5.1390000000000002},{"interface":"front","average":{"1min":1.8149999999999999,"5min":1.7889999999999999,"15min":1.23},"min":{"1min":0.314,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":8.8300000000000001,"5min":11.766,"15min":11.766},"last":5.1529999999999996}]},{"osd":1,"last update":"Sat Sep 23 20:58:21 2023","interfaces":[{"interface":"back","average":{"1min":1.7370000000000001,"5min":1.641,"15min":0.996},"min":{"1min":0.246,"5min":0.246,"15min":0.23599999999999999},"max":{"1min":10.153,"5min":10.153,"15min":10.153},"last":6.4059999999999997},{"interface":"front","average":{"1min":2.645,"5min":1.649,"15min":0.96399999999999997},"min":{"1min":0.26100000000000001,"5min":0.26100000000000001,"15min":0.23200000000000001},"max":{"1min":12.292999999999999,"5min":12.292999999999999,"15min":12.292999999999999},"last":1.5669999999999999}]},{"osd":2,"last update":"Sat Sep 23 20:58:53 2023","interfaces":[{"interface":"back","average":{"1min":1.8700000000000001,"5min":1.764,"15min":1.115},"min":{"1min":0.34300000000000003,"5min":0.311,"15min":0.29099999999999998},"max":{"1min":7.3150000000000004,"5min":15.288,"15min":15.288},"last":1.6830000000000001},{"interface":"front","average":{"1min":1.633,"5min":1.974,"15min":1.2629999999999999},"min":{"1min":0.32600000000000001,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":8.7959999999999994,"5min":15.262,"15min":15.262},"last":4.1799999999999997}]},{"osd":3,"last update":"Sat Sep 23 20:58:21 2023","interfaces":[{"interface":"back","average":{"1min":2.0209999999999999,"5min":2.504,"15min":1.514},"min":{"1min":0.28899999999999998,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":7.7549999999999999,"5min":15.916,"15min":31.562999999999999},"last":4.2569999999999997},{"interface":"front","average":{"1min":2.2080000000000002,"5min":2.5449999999999999,"15min":1.589},"min":{"1min":0.433,"5min":0.433,"15min":0.41899999999999998},"max":{"1min":7.7300000000000004,"5min":15.882,"15min":31.574000000000002},"last":4.2460000000000004}]},{"osd":5,"last update":"Sat Sep 23 20:58:35 2023","interfaces":[{"interface":"back","average":{"1min":0.81399999999999995,"5min":0.61399999999999999,"15min":0.52900000000000003},"min":{"1min":0.29199999999999998,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":2.9470000000000001,"5min":3.3279999999999998,"15min":3.3279999999999998},"last":0.28899999999999998},{"interface":"front","average":{"1min":0.77900000000000003,"5min":0.68500000000000005,"15min":0.57799999999999996},"min":{"1min":0.33200000000000002,"5min":0.251,"15min":0.251},"max":{"1min":1.9350000000000001,"5min":7.7619999999999996,"15min":7.7619999999999996},"last":0.39400000000000002}]},{"osd":6,"last update":"Sat Sep 23 20:58:41 2023","interfaces":[{"interface":"back","average":{"1min":0.90300000000000002,"5min":0.67700000000000005,"15min":0.54800000000000004},"min":{"1min":0.29899999999999999,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":3.4780000000000002,"5min":4.6909999999999998,"15min":4.6909999999999998},"last":0.375},{"interface":"front","average":{"1min":0.82299999999999995,"5min":0.71599999999999997,"15min":0.66800000000000004},"min":{"1min":0.36799999999999999,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":3.456,"5min":3.456,"15min":3.456},"last":0.34499999999999997}]},{"osd":7,"last update":"Sat Sep 23 20:58:20 2023","interfaces":[{"interface":"back","average":{"1min":0.84199999999999997,"5min":0.83099999999999996,"15min":0.67100000000000004},"min":{"1min":0.442,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":3.327,"5min":7.383,"15min":7.383},"last":0.32600000000000001},{"interface":"front","average":{"1min":0.93899999999999995,"5min":0.80800000000000005,"15min":0.66000000000000003},"min":{"1min":0.36399999999999999,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":3.883,"5min":7.8310000000000004,"15min":7.8310000000000004},"last":0.50700000000000001}]}]},{"osd":5,"up_from":39,"seq":167503724777,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1597268,"kb_used_data":1470860,"kb_used_omap":0,"kb_used_meta":126400,"kb_avail":92147884,"statfs":{"total":95995035648,"available":94359433216,"internally_reserved":0,"allocated":1506160640,"data_stored":1505329840,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":129433600},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:58:37 2023","interfaces":[{"interface":"back","average":{"1min":0.78600000000000003,"5min":1.03,"15min":0.72699999999999998},"min":{"1min":0.23899999999999999,"5min":0.20399999999999999,"15min":0.20399999999999999},"max":{"1min":3.6949999999999998,"5min":7.766,"15min":7.766},"last":8.9120000000000008},{"interface":"front","average":{"1min":1.8049999999999999,"5min":1.6779999999999999,"15min":0.98099999999999998},"min":{"1min":0.25,"5min":0.223,"15min":0.223},"max":{"1min":7.8120000000000003,"5min":15.037000000000001,"15min":15.037000000000001},"last":0.71799999999999997}]},{"osd":1,"last update":"Sat Sep 23 20:58:16 2023","interfaces":[{"interface":"back","average":{"1min":1.5549999999999999,"5min":1.603,"15min":0.95199999999999996},"min":{"1min":0.27100000000000002,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":6.7409999999999997,"5min":8.3520000000000003,"15min":8.3520000000000003},"last":4.1280000000000001},{"interface":"front","average":{"1min":1.9590000000000001,"5min":1.7450000000000001,"15min":0.97699999999999998},"min":{"1min":0.28100000000000003,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":7.4669999999999996,"5min":10.285,"15min":10.285},"last":4.798}]},{"osd":2,"last update":"Sat Sep 23 20:58:52 2023","interfaces":[{"interface":"back","average":{"1min":1.9299999999999999,"5min":1.6000000000000001,"15min":0.97799999999999998},"min":{"1min":0.311,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":6,"5min":12.651,"15min":12.651},"last":9.1440000000000001},{"interface":"front","average":{"1min":3.077,"5min":1.8440000000000001,"15min":1.085},"min":{"1min":0.27100000000000002,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":12.300000000000001,"5min":12.300000000000001,"15min":12.300000000000001},"last":4.9989999999999997}]},{"osd":3,"last update":"Sat Sep 23 20:58:16 2023","interfaces":[{"interface":"back","average":{"1min":2.331,"5min":2.0670000000000002,"15min":1.153},"min":{"1min":0.39200000000000002,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":8.6579999999999995,"5min":9.5489999999999995,"15min":9.5489999999999995},"last":0.92900000000000005},{"interface":"front","average":{"1min":1.6000000000000001,"5min":1.6040000000000001,"15min":0.97499999999999998},"min":{"1min":0.28899999999999998,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":10.457000000000001,"5min":10.457000000000001,"15min":10.457000000000001},"last":0.32500000000000001}]},{"osd":4,"last update":"Sat Sep 23 20:58:33 2023","interfaces":[{"interface":"back","average":{"1min":0.48799999999999999,"5min":0.59099999999999997,"15min":0.52900000000000003},"min":{"1min":0.252,"5min":0.252,"15min":0.252},"max":{"1min":2.6909999999999998,"5min":3.8479999999999999,"15min":3.8479999999999999},"last":0.86899999999999999},{"interface":"front","average":{"1min":0.41899999999999998,"5min":0.64100000000000001,"15min":0.54900000000000004},"min":{"1min":0.29299999999999998,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":0.747,"5min":3.0680000000000001,"15min":3.0680000000000001},"last":0.27400000000000002}]},{"osd":6,"last update":"Sat Sep 23 20:58:37 2023","interfaces":[{"interface":"back","average":{"1min":0.46600000000000003,"5min":0.59099999999999997,"15min":0.57099999999999995},"min":{"1min":0.28499999999999998,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":1.0509999999999999,"5min":2.9820000000000002,"15min":2.9820000000000002},"last":0.308},{"interface":"front","average":{"1min":0.499,"5min":0.69899999999999995,"15min":0.72899999999999998},"min":{"1min":0.36299999999999999,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":0.78500000000000003,"5min":3.6560000000000001,"15min":3.6560000000000001},"last":0.90400000000000003}]},{"osd":7,"last update":"Sat Sep 23 20:58:13 2023","interfaces":[{"interface":"back","average":{"1min":0.66500000000000004,"5min":0.76600000000000001,"15min":0.63200000000000001},"min":{"1min":0.316,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":2.4809999999999999,"5min":8.5649999999999995,"15min":8.5649999999999995},"last":0.34799999999999998},{"interface":"front","average":{"1min":0.60299999999999998,"5min":0.74099999999999999,"15min":0.64400000000000002},"min":{"1min":0.35399999999999998,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":2.7109999999999999,"5min":4.5430000000000001,"15min":4.5430000000000001},"last":0.95199999999999996}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":2288193536,"data_stored":2288189464,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":2104889344,"data_stored":2104885272,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":2072444928,"data_stored":2072444928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1920991232,"data_stored":1920991232,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1422065664,"data_stored":1422065664,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1494155264,"data_stored":1494155264,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1468928000,"data_stored":1468923928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1425342464,"data_stored":1425342464,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T20:59:04.106 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:59:04.194 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:59:04.385 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:59:04 smithi026 ceph-mon[179688]: pgmap v574: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 4.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 606 op/s; 11300/211269 objects degraded (5.349%); 3418/211269 objects misplaced (1.618%); 1.5 MiB/s, 23 objects/s recovering 2023-09-23T20:59:04.386 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:59:04 smithi026 ceph-mon[179688]: from='client.55132 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:59:04.390 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:59:04 smithi026 ceph-mon[175151]: pgmap v574: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 4.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 606 op/s; 11300/211269 objects degraded (5.349%); 3418/211269 objects misplaced (1.618%); 1.5 MiB/s, 23 objects/s recovering 2023-09-23T20:59:04.390 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:59:04 smithi026 ceph-mon[175151]: from='client.55132 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:59:04.424 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:59:04.434 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:59:04.434 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:59:04.535 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:59:04.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:59:04 smithi070 ceph-mon[168282]: pgmap v574: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 4.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 606 op/s; 11300/211269 objects degraded (5.349%); 3418/211269 objects misplaced (1.618%); 1.5 MiB/s, 23 objects/s recovering 2023-09-23T20:59:04.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:59:04 smithi070 ceph-mon[168282]: from='client.55132 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:59:04.660 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:59:04.754 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:59:04.758 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:59:04.766 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:59:04.767 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:59:04.849 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0: {} 2023-09-23T20:59:04.868 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:59:04.891 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1: {} 2023-09-23T20:59:04.960 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2: {} 2023-09-23T20:59:04.967 INFO:teuthology.orchestra.run.smithi026.stdout:{ 2023-09-23T20:59:04.967 INFO:teuthology.orchestra.run.smithi026.stdout: "ops": [ 2023-09-23T20:59:04.967 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:59:04.967 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35100.0:135679 7.6 7:674c0c94:::benchmark_data_smithi026_260489_object33917:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e237)", 2023-09-23T20:59:04.967 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T20:59:04.950849+0000", 2023-09-23T20:59:04.968 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 0.015184977000000001, 2023-09-23T20:59:04.968 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.015220005999999999, 2023-09-23T20:59:04.968 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T20:59:04.968 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "waiting for sub ops", 2023-09-23T20:59:04.968 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T20:59:04.968 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35100", 2023-09-23T20:59:04.968 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/4108972961", 2023-09-23T20:59:04.968 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 135679 2023-09-23T20:59:04.968 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:59:04.968 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T20:59:04.968 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:59:04.968 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T20:59:04.968 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:59:04.950849+0000", 2023-09-23T20:59:04.969 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:59:04.969 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:59:04.969 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:59:04.969 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T20:59:04.969 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:59:04.950849+0000", 2023-09-23T20:59:04.969 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:59:04.969 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:59:04.969 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:59:04.969 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T20:59:04.969 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:59:04.950846+0000", 2023-09-23T20:59:04.969 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999976 2023-09-23T20:59:04.969 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:59:04.969 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:59:04.969 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T20:59:04.970 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:59:04.950856+0000", 2023-09-23T20:59:04.970 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0274e-05 2023-09-23T20:59:04.970 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:59:04.970 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:59:04.970 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T20:59:04.970 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:59:04.950883+0000", 2023-09-23T20:59:04.970 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.6917999999999999e-05 2023-09-23T20:59:04.970 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:59:04.970 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:59:04.970 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T20:59:04.970 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:59:04.950895+0000", 2023-09-23T20:59:04.970 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1260999999999999e-05 2023-09-23T20:59:04.971 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:59:04.971 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:59:04.971 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:59:04.971 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:59:04.953450+0000", 2023-09-23T20:59:04.971 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0025552230000000001 2023-09-23T20:59:04.971 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:59:04.971 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:59:04.971 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T20:59:04.971 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:59:04.953601+0000", 2023-09-23T20:59:04.971 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000151428 2023-09-23T20:59:04.971 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:59:04.971 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:59:04.971 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 3,7", 2023-09-23T20:59:04.972 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:59:04.953773+0000", 2023-09-23T20:59:04.972 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00017158700000000001 2023-09-23T20:59:04.972 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:59:04.972 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:59:04.972 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:59:04.972 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:59:04.958887+0000", 2023-09-23T20:59:04.972 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0051138080000000001 2023-09-23T20:59:04.972 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:59:04.972 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T20:59:04.972 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:59:04.972 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:59:04.972 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:59:04.972 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_repop(client.35100.0:135684 7.d e237/199 7:b4d25f09:::benchmark_data_smithi026_260489_object33922:head v 237'8560, mlcod=237'8558)", 2023-09-23T20:59:04.973 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T20:59:04.959482+0000", 2023-09-23T20:59:04.973 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 0.0065516050000000003, 2023-09-23T20:59:04.973 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0066946760000000001, 2023-09-23T20:59:04.973 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T20:59:04.973 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "started", 2023-09-23T20:59:04.973 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T20:59:04.973 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:59:04.973 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T20:59:04.973 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:59:04.959482+0000", 2023-09-23T20:59:04.973 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:59:04.973 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:59:04.973 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:59:04.974 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T20:59:04.974 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:59:04.959482+0000", 2023-09-23T20:59:04.974 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:59:04.974 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:59:04.974 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:59:04.974 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T20:59:04.974 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:59:04.959481+0000", 2023-09-23T20:59:04.974 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T20:59:04.974 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:59:04.974 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:59:04.974 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T20:59:04.974 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:59:04.959488+0000", 2023-09-23T20:59:04.974 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.9299999999999997e-06 2023-09-23T20:59:04.975 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:59:04.975 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:59:04.975 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T20:59:04.975 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:59:04.959499+0000", 2023-09-23T20:59:04.975 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1039000000000001e-05 2023-09-23T20:59:04.975 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:59:04.975 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:59:04.975 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T20:59:04.975 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:59:04.959506+0000", 2023-09-23T20:59:04.975 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.562e-06 2023-09-23T20:59:04.975 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:59:04.975 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:59:04.976 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:59:04.976 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:59:04.959575+0000", 2023-09-23T20:59:04.976 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.9376999999999997e-05 2023-09-23T20:59:04.976 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:59:04.976 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:59:04.976 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T20:59:04.976 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:59:04.959596+0000", 2023-09-23T20:59:04.976 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.0488000000000001e-05 2023-09-23T20:59:04.976 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:59:04.976 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T20:59:04.976 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:59:04.976 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:59:04.976 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:59:04.977 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35100.0:135681 7.4 7:2b11a46e:::benchmark_data_smithi026_260489_object33919:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e237)", 2023-09-23T20:59:04.977 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T20:59:04.960832+0000", 2023-09-23T20:59:04.977 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 0.0052013889999999998, 2023-09-23T20:59:04.977 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0054183520000000004, 2023-09-23T20:59:04.977 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T20:59:04.977 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "waiting for sub ops", 2023-09-23T20:59:04.977 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T20:59:04.977 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35100", 2023-09-23T20:59:04.977 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/4108972961", 2023-09-23T20:59:04.977 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 135681 2023-09-23T20:59:04.977 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:59:04.977 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T20:59:04.977 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:59:04.978 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T20:59:04.978 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:59:04.960832+0000", 2023-09-23T20:59:04.978 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:59:04.978 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:59:04.978 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:59:04.978 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T20:59:04.978 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:59:04.960832+0000", 2023-09-23T20:59:04.978 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:59:04.978 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:59:04.978 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:59:04.978 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T20:59:04.978 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:59:04.960830+0000", 2023-09-23T20:59:04.978 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999981 2023-09-23T20:59:04.979 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:59:04.979 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:59:04.979 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T20:59:04.979 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:59:04.960838+0000", 2023-09-23T20:59:04.979 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.7589999999999997e-06 2023-09-23T20:59:04.979 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:59:04.979 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:59:04.979 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T20:59:04.979 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:59:04.960852+0000", 2023-09-23T20:59:04.979 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.4276e-05 2023-09-23T20:59:04.979 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:59:04.979 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:59:04.979 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T20:59:04.980 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:59:04.960861+0000", 2023-09-23T20:59:04.980 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.6789999999999994e-06 2023-09-23T20:59:04.980 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:59:04.980 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:59:04.980 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:59:04.980 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:59:04.962539+0000", 2023-09-23T20:59:04.980 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.001677659 2023-09-23T20:59:04.980 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:59:04.980 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:59:04.980 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T20:59:04.980 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:59:04.962658+0000", 2023-09-23T20:59:04.980 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00011964399999999999 2023-09-23T20:59:04.980 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:59:04.981 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:59:04.981 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 1,4", 2023-09-23T20:59:04.981 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:59:04.962765+0000", 2023-09-23T20:59:04.981 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00010634800000000001 2023-09-23T20:59:04.981 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:59:04.981 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T20:59:04.981 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:59:04.981 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:59:04.981 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:59:04.981 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35100.0:135682 7.f 7:f8701433:::benchmark_data_smithi026_260489_object33920:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e237)", 2023-09-23T20:59:04.981 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T20:59:04.960884+0000", 2023-09-23T20:59:04.981 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 0.0051497629999999999, 2023-09-23T20:59:04.981 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0054541889999999999, 2023-09-23T20:59:04.982 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T20:59:04.982 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "waiting for sub ops", 2023-09-23T20:59:04.982 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T20:59:04.982 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35100", 2023-09-23T20:59:04.982 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/4108972961", 2023-09-23T20:59:04.982 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 135682 2023-09-23T20:59:04.982 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:59:04.982 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T20:59:04.982 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:59:04.982 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T20:59:04.982 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:59:04.960884+0000", 2023-09-23T20:59:04.982 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:59:04.983 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:59:04.983 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:59:04.983 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T20:59:04.983 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:59:04.960884+0000", 2023-09-23T20:59:04.983 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:59:04.983 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:59:04.983 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:59:04.983 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T20:59:04.983 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:59:04.960883+0000", 2023-09-23T20:59:04.983 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T20:59:04.983 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:59:04.983 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:59:04.984 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T20:59:04.984 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:59:04.960886+0000", 2023-09-23T20:59:04.984 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.9639999999999999e-06 2023-09-23T20:59:04.984 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:59:04.984 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:59:04.984 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T20:59:04.984 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:59:04.960893+0000", 2023-09-23T20:59:04.984 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.4969999999999998e-06 2023-09-23T20:59:04.984 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:59:04.984 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:59:04.984 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T20:59:04.984 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:59:04.960903+0000", 2023-09-23T20:59:04.984 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.6209999999999999e-06 2023-09-23T20:59:04.985 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:59:04.985 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:59:04.985 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:59:04.985 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:59:04.960976+0000", 2023-09-23T20:59:04.985 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.3910000000000002e-05 2023-09-23T20:59:04.985 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:59:04.985 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:59:04.985 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T20:59:04.985 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:59:04.961092+0000", 2023-09-23T20:59:04.985 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000115171 2023-09-23T20:59:04.985 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:59:04.985 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:59:04.985 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 3,7", 2023-09-23T20:59:04.985 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:59:04.961198+0000", 2023-09-23T20:59:04.986 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00010675699999999999 2023-09-23T20:59:04.986 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:59:04.986 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T20:59:04.986 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:59:04.986 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:59:04.986 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:59:04.986 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35100.0:135685 7.4 7.8166eaf4 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e237)", 2023-09-23T20:59:04.986 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T20:59:04.960914+0000", 2023-09-23T20:59:04.986 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 0.0051199360000000003, 2023-09-23T20:59:04.986 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0055109599999999996, 2023-09-23T20:59:04.986 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T20:59:04.986 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "queued for pg", 2023-09-23T20:59:04.986 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T20:59:04.987 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35100", 2023-09-23T20:59:04.987 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/4108972961", 2023-09-23T20:59:04.987 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 135685 2023-09-23T20:59:04.987 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:59:04.987 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T20:59:04.987 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:59:04.987 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T20:59:04.987 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:59:04.960914+0000", 2023-09-23T20:59:04.987 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:59:04.987 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:59:04.987 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:59:04.987 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T20:59:04.987 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:59:04.960914+0000", 2023-09-23T20:59:04.988 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:59:04.988 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:59:04.988 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:59:04.988 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T20:59:04.988 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:59:04.960913+0000", 2023-09-23T20:59:04.988 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T20:59:04.988 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:59:04.988 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:59:04.988 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T20:59:04.988 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:59:04.960916+0000", 2023-09-23T20:59:04.988 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.4229999999999999e-06 2023-09-23T20:59:04.988 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:59:04.989 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:59:04.989 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T20:59:04.989 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:59:04.960921+0000", 2023-09-23T20:59:04.989 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.626e-06 2023-09-23T20:59:04.989 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:59:04.989 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:59:04.989 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T20:59:04.989 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:59:04.960926+0000", 2023-09-23T20:59:04.989 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.5449999999999997e-06 2023-09-23T20:59:04.989 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:59:04.989 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T20:59:04.989 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:59:04.989 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:59:04.989 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:59:04.990 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35100.0:135689 7.6 7:675d62f9:::benchmark_data_smithi026_260489_object33927:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e237)", 2023-09-23T20:59:04.990 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T20:59:04.960930+0000", 2023-09-23T20:59:04.990 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 0.0051037590000000002, 2023-09-23T20:59:04.990 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0055655899999999996, 2023-09-23T20:59:04.990 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T20:59:04.990 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "waiting for sub ops", 2023-09-23T20:59:04.990 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T20:59:04.990 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35100", 2023-09-23T20:59:04.990 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/4108972961", 2023-09-23T20:59:04.990 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 135689 2023-09-23T20:59:04.990 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:59:04.990 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T20:59:04.991 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:59:04.991 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T20:59:04.991 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:59:04.960930+0000", 2023-09-23T20:59:04.991 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:59:04.991 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:59:04.991 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:59:04.991 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T20:59:04.991 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:59:04.960930+0000", 2023-09-23T20:59:04.991 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:59:04.991 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:59:04.991 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:59:04.991 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T20:59:04.991 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:59:04.960930+0000", 2023-09-23T20:59:04.992 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T20:59:04.992 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:59:04.992 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:59:04.992 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T20:59:04.992 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:59:04.960932+0000", 2023-09-23T20:59:04.992 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.193e-06 2023-09-23T20:59:04.992 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:59:04.992 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:59:04.992 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T20:59:04.992 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:59:04.960937+0000", 2023-09-23T20:59:04.992 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.0150000000000003e-06 2023-09-23T20:59:04.992 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:59:04.992 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:59:04.993 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T20:59:04.993 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:59:04.960941+0000", 2023-09-23T20:59:04.993 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.245e-06 2023-09-23T20:59:04.993 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:59:04.993 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:59:04.993 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:59:04.993 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:59:04.965352+0000", 2023-09-23T20:59:04.993 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0044110210000000002 2023-09-23T20:59:04.993 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:59:04.993 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:59:04.993 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T20:59:04.993 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:59:04.965470+0000", 2023-09-23T20:59:04.993 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00011784499999999999 2023-09-23T20:59:04.994 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:59:04.994 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:59:04.994 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 3,7", 2023-09-23T20:59:04.994 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:59:04.965640+0000", 2023-09-23T20:59:04.994 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000170151 2023-09-23T20:59:04.994 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:59:04.994 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T20:59:04.994 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:59:04.994 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:59:04.994 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:59:04.994 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35100.0:135690 7.4 7.eba9cd84 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e237)", 2023-09-23T20:59:04.994 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T20:59:04.965245+0000", 2023-09-23T20:59:04.994 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 0.00078892600000000002, 2023-09-23T20:59:04.995 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0013291399999999999, 2023-09-23T20:59:04.995 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T20:59:04.995 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "queued for pg", 2023-09-23T20:59:04.995 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T20:59:04.995 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35100", 2023-09-23T20:59:04.995 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/4108972961", 2023-09-23T20:59:04.995 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 135690 2023-09-23T20:59:04.995 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:59:04.995 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T20:59:04.995 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:59:04.995 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T20:59:04.995 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:59:04.965245+0000", 2023-09-23T20:59:04.995 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:59:04.996 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:59:04.996 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:59:04.996 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T20:59:04.996 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:59:04.965245+0000", 2023-09-23T20:59:04.996 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:59:04.996 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:59:04.996 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:59:04.996 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T20:59:04.996 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:59:04.965243+0000", 2023-09-23T20:59:04.996 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T20:59:04.996 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:59:04.996 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:59:04.996 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T20:59:04.997 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:59:04.965250+0000", 2023-09-23T20:59:04.997 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.319e-06 2023-09-23T20:59:04.997 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:59:04.997 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:59:04.997 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T20:59:04.997 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:59:04.965262+0000", 2023-09-23T20:59:04.997 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1984000000000001e-05 2023-09-23T20:59:04.997 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:59:04.997 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:59:04.997 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T20:59:04.997 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:59:04.965270+0000", 2023-09-23T20:59:04.997 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.5599999999999994e-06 2023-09-23T20:59:04.997 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:59:04.998 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T20:59:04.998 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:59:04.998 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:59:04.998 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:59:04.998 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35100.0:135691 7.f 7:ffd2dab3:::benchmark_data_smithi026_260489_object33929:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e237)", 2023-09-23T20:59:04.998 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T20:59:04.965277+0000", 2023-09-23T20:59:04.998 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 0.00075622000000000005, 2023-09-23T20:59:04.998 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0013649510000000001, 2023-09-23T20:59:04.998 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T20:59:04.998 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "waiting for sub ops", 2023-09-23T20:59:04.998 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T20:59:04.998 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35100", 2023-09-23T20:59:04.999 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/4108972961", 2023-09-23T20:59:04.999 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 135691 2023-09-23T20:59:04.999 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:59:04.999 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T20:59:04.999 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:59:04.999 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T20:59:04.999 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:59:04.965277+0000", 2023-09-23T20:59:04.999 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:59:04.999 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:59:04.999 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:59:04.999 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T20:59:04.999 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:59:04.965277+0000", 2023-09-23T20:59:04.999 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:59:04.999 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:59:05.000 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:59:05.000 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T20:59:05.000 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:59:04.965277+0000", 2023-09-23T20:59:05.000 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T20:59:05.000 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:59:05.000 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:59:05.000 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T20:59:05.000 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:59:04.965280+0000", 2023-09-23T20:59:05.000 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.7180000000000001e-06 2023-09-23T20:59:05.000 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:59:05.000 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:59:05.000 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T20:59:05.000 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:59:04.965286+0000", 2023-09-23T20:59:05.001 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.8109999999999999e-06 2023-09-23T20:59:05.001 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:59:05.001 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:59:05.001 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T20:59:05.001 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:59:04.965290+0000", 2023-09-23T20:59:05.001 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.7360000000000001e-06 2023-09-23T20:59:05.001 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:59:05.001 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:59:05.001 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:59:05.001 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:59:04.965817+0000", 2023-09-23T20:59:05.001 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00052674299999999998 2023-09-23T20:59:05.001 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:59:05.001 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:59:05.002 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T20:59:05.002 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:59:04.965931+0000", 2023-09-23T20:59:05.002 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00011377199999999999 2023-09-23T20:59:05.002 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:59:05.002 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:59:05.002 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 3,7", 2023-09-23T20:59:05.002 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:59:04.966034+0000", 2023-09-23T20:59:05.002 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000103566 2023-09-23T20:59:05.002 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:59:05.002 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T20:59:05.002 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:59:05.002 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:59:05.002 INFO:teuthology.orchestra.run.smithi026.stdout: ], 2023-09-23T20:59:05.003 INFO:teuthology.orchestra.run.smithi026.stdout: "num_ops": 8 2023-09-23T20:59:05.003 INFO:teuthology.orchestra.run.smithi026.stdout:} 2023-09-23T20:59:05.003 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:59:05.045 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3: {} 2023-09-23T20:59:05.077 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4: {} 2023-09-23T20:59:05.112 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T20:59:05.161 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6: {} 2023-09-23T20:59:05.206 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T20:59:05.225 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:59:05 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/3551850909' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:59:05.225 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:59:05 smithi026 ceph-mon[179688]: Health check update: Degraded data redundancy: 11300/211269 objects degraded (5.349%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T20:59:05.226 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:59:05 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/3551850909' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:59:05.226 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:59:05 smithi026 ceph-mon[175151]: Health check update: Degraded data redundancy: 11300/211269 objects degraded (5.349%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T20:59:05.229 INFO:teuthology.orchestra.run.smithi026.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T20:59:05.229 INFO:teuthology.orchestra.run.smithi026.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T20:59:05.230 INFO:teuthology.orchestra.run.smithi026.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T20:59:05.230 INFO:teuthology.orchestra.run.smithi026.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T20:59:05.230 INFO:teuthology.orchestra.run.smithi026.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T20:59:05.230 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T20:59:05.230 INFO:teuthology.orchestra.run.smithi026.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T20:59:05.230 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T20:59:05.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:59:05 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/3551850909' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:59:05.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:59:05 smithi070 ceph-mon[168282]: Health check update: Degraded data redundancy: 11300/211269 objects degraded (5.349%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T20:59:05.697 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:59:05.711 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:59:05.711 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:59:05.743 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_blocked_ops 2023-09-23T20:59:05.812 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:59:05.900 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:59:06.177 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:59:06.189 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:59:06.190 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:59:06.290 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:59:06.321 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:59:06.368 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:59:06.371 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:59:06.371 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:59:06.472 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:59:06.500 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:59:06 smithi070 ceph-mon[168282]: pgmap v575: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 4.3 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 608 op/s; 11300/208953 objects degraded (5.408%); 3418/208953 objects misplaced (1.636%); 1.5 MiB/s, 23 objects/s recovering 2023-09-23T20:59:06.503 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:59:06.548 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:59:06.551 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:59:06.552 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:59:06.572 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:59:06 smithi026 ceph-mon[179688]: pgmap v575: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 4.3 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 608 op/s; 11300/208953 objects degraded (5.408%); 3418/208953 objects misplaced (1.636%); 1.5 MiB/s, 23 objects/s recovering 2023-09-23T20:59:06.573 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:59:06 smithi026 ceph-mon[175151]: pgmap v575: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 4.3 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 608 op/s; 11300/208953 objects degraded (5.408%); 3418/208953 objects misplaced (1.636%); 1.5 MiB/s, 23 objects/s recovering 2023-09-23T20:59:06.653 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:59:06.758 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:59:06.836 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:59:06.848 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:59:06.849 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:59:06.873 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:59:06.950 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:59:06.980 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:59:07.021 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:59:07.031 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:59:07.032 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:59:07.060 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T20:59:07.132 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:59:07.259 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:59:07.395 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:59:07.410 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:59:07.411 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:59:07.411 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:59:07 smithi026 ceph-mon[179688]: pgmap v576: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 4.2 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 557 op/s; 11300/207219 objects degraded (5.453%); 3418/207219 objects misplaced (1.649%); 1.2 MiB/s, 19 objects/s recovering 2023-09-23T20:59:07.412 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:59:07 smithi026 ceph-mon[175151]: pgmap v576: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 4.2 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 557 op/s; 11300/207219 objects degraded (5.453%); 3418/207219 objects misplaced (1.649%); 1.2 MiB/s, 19 objects/s recovering 2023-09-23T20:59:07.511 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:59:07.537 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:59:07 smithi070 ceph-mon[168282]: pgmap v576: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 4.2 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 557 op/s; 11300/207219 objects degraded (5.453%); 3418/207219 objects misplaced (1.649%); 1.2 MiB/s, 19 objects/s recovering 2023-09-23T20:59:07.541 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:59:07.591 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:59:07.599 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:59:07.600 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:59:07.701 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:59:07.729 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:59:07.774 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:59:07.776 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:59:07.777 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:59:07.877 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:59:08.040 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T20:59:08.042 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:59:08.114 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:59:08.118 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:59:08.118 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:59:08.219 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:59:08.251 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:59:08.294 INFO:teuthology.orchestra.run.smithi026.stderr:nodeep-scrub is set 2023-09-23T20:59:08.296 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:59:08.306 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:59:08.306 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:59:08.330 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:59:08.407 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:59:08.434 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:59:08.468 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:59:08.472 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:59:08.472 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:59:08.522 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:59:08 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/3474463683' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:59:08.523 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:59:08 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/3474463683' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:59:08.573 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:59:08.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:59:08 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/3474463683' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:59:08.680 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:59:08.806 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:59:08.817 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:59:08.818 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:59:08.852 INFO:teuthology.orchestra.run.smithi026.stdout:{ 2023-09-23T20:59:08.852 INFO:teuthology.orchestra.run.smithi026.stdout: "ops": [], 2023-09-23T20:59:08.852 INFO:teuthology.orchestra.run.smithi026.stdout: "complaint_time": 30, 2023-09-23T20:59:08.852 INFO:teuthology.orchestra.run.smithi026.stdout: "num_blocked_ops": 0 2023-09-23T20:59:08.852 INFO:teuthology.orchestra.run.smithi026.stdout:} 2023-09-23T20:59:08.918 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:59:09.046 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:59:09.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:59:09 smithi026 ceph-mon[179688]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:59:09.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:59:09 smithi026 ceph-mon[179688]: failed to encode map e238 with expected crc 2023-09-23T20:59:09.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:59:09 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/3474463683' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T20:59:09.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:59:09 smithi026 ceph-mon[179688]: osdmap e238: 8 total, 8 up, 8 in 2023-09-23T20:59:09.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:59:09 smithi026 ceph-mon[179688]: failed to encode map e238 with expected crc 2023-09-23T20:59:09.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:59:09 smithi026 ceph-mon[179688]: failed to encode map e238 with expected crc 2023-09-23T20:59:09.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:59:09 smithi026 ceph-mon[179688]: pgmap v578: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 4.1 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 616 op/s; 11368/202437 objects degraded (5.616%); 3233/202437 objects misplaced (1.597%); 1.5 MiB/s, 24 objects/s recovering 2023-09-23T20:59:09.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:59:09 smithi026 ceph-mon[175151]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:59:09.383 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:59:09 smithi026 ceph-mon[175151]: failed to encode map e238 with expected crc 2023-09-23T20:59:09.383 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:59:09 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/3474463683' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T20:59:09.383 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:59:09 smithi026 ceph-mon[175151]: osdmap e238: 8 total, 8 up, 8 in 2023-09-23T20:59:09.383 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:59:09 smithi026 ceph-mon[175151]: failed to encode map e238 with expected crc 2023-09-23T20:59:09.383 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:59:09 smithi026 ceph-mon[175151]: failed to encode map e238 with expected crc 2023-09-23T20:59:09.383 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:59:09 smithi026 ceph-mon[175151]: pgmap v578: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 4.1 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 616 op/s; 11368/202437 objects degraded (5.616%); 3233/202437 objects misplaced (1.597%); 1.5 MiB/s, 24 objects/s recovering 2023-09-23T20:59:09.505 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:59:09.512 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:59:09.512 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:59:09.592 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_historic_ops 2023-09-23T20:59:09.613 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:59:09.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:59:09 smithi070 ceph-mon[168282]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:59:09.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:59:09 smithi070 ceph-mon[168282]: failed to encode map e238 with expected crc 2023-09-23T20:59:09.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:59:09 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/3474463683' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T20:59:09.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:59:09 smithi070 ceph-mon[168282]: osdmap e238: 8 total, 8 up, 8 in 2023-09-23T20:59:09.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:59:09 smithi070 ceph-mon[168282]: failed to encode map e238 with expected crc 2023-09-23T20:59:09.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:59:09 smithi070 ceph-mon[168282]: failed to encode map e238 with expected crc 2023-09-23T20:59:09.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:59:09 smithi070 ceph-mon[168282]: pgmap v578: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 4.1 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 616 op/s; 11368/202437 objects degraded (5.616%); 3233/202437 objects misplaced (1.597%); 1.5 MiB/s, 24 objects/s recovering 2023-09-23T20:59:09.760 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:59:09 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[164029]: ::ffff:172.21.15.70 - - [23/Sep/2023:20:59:09] "GET /metrics HTTP/1.1" 200 36320 "" "Prometheus/2.43.0" 2023-09-23T20:59:09.763 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:59:09.972 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:59:09.972 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:59:09.976 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:59:09.976 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:59:10.077 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:59:10.149 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:59:10.231 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:59:10.240 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:59:10.241 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:59:10.342 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:59:10.428 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:59:10 smithi026 ceph-mon[179688]: Health check update: Degraded data redundancy: 11368/202437 objects degraded (5.616%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T20:59:10.428 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:59:10 smithi026 ceph-mon[175151]: Health check update: Degraded data redundancy: 11368/202437 objects degraded (5.616%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T20:59:10.472 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:59:10.557 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:59:10.564 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:59:10.564 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:59:10.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:59:10 smithi070 ceph-mon[168282]: Health check update: Degraded data redundancy: 11368/202437 objects degraded (5.616%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T20:59:10.665 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:59:10.697 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:59:10.734 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:59:10.737 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:59:10.737 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:59:10.838 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:59:10.948 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:59:11.185 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:59:11.192 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:59:11.193 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:59:11.267 INFO:teuthology.orchestra.run.smithi026.stdout: 2023-09-23T20:59:11.290 INFO:teuthology.orchestra.run.smithi026.stderr:dumped all 2023-09-23T20:59:11.293 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:59:11.311 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:59:11.322 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:59:11.363 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:59:11.367 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:59:11.368 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:59:11.468 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:59:11.508 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:59:11 smithi026 ceph-mon[179688]: pgmap v579: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 4.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 604 op/s; 11368/198780 objects degraded (5.719%); 3233/198780 objects misplaced (1.626%); 1.5 MiB/s, 24 objects/s recovering 2023-09-23T20:59:11.508 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:59:11 smithi026 ceph-mon[175151]: pgmap v579: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 4.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 604 op/s; 11368/198780 objects degraded (5.719%); 3233/198780 objects misplaced (1.626%); 1.5 MiB/s, 24 objects/s recovering 2023-09-23T20:59:11.555 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:59:11.564 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T20:59:11.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:59:11 smithi070 ceph-mon[168282]: pgmap v579: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 4.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 604 op/s; 11368/198780 objects degraded (5.719%); 3233/198780 objects misplaced (1.626%); 1.5 MiB/s, 24 objects/s recovering 2023-09-23T20:59:11.881 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:59:11.887 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:59:11.887 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:59:11.988 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:59:12.061 INFO:teuthology.orchestra.run.smithi026.stdout:{"pg_ready":true,"pg_map":{"version":579,"stamp":"2023-09-23T20:59:10.433382+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4341372856,"num_objects":66260,"num_object_clones":0,"num_object_copies":198780,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":11368,"num_objects_misplaced":3233,"num_objects_unfound":0,"num_objects_dirty":66260,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":137345,"num_write_kb":6512606,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1763,"num_bytes_recovered":114104166,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":121520,"ondisk_log_size":121520,"up":327,"acting":324,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":331,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":14801644,"kb_used_data":13216620,"kb_used_omap":136,"kb_used_meta":1584631,"kb_avail":735159572,"statfs":{"total":767960285184,"available":752803401728,"internally_reserved":0,"allocated":13533818880,"data_stored":13525929014,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":139741,"internal_metadata":1622662691},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,1,2,3,4],"upper_bound":32},"perf_stat":{"commit_latency_ms":11,"apply_latency_ms":11,"commit_latency_ns":11000000,"apply_latency_ns":11000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-396427264,"num_objects":-6049,"num_object_clones":0,"num_object_copies":-18147,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":153,"num_objects_misplaced":-394,"num_objects_unfound":0,"num_objects_dirty":-6049,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6049,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":241,"num_bytes_recovered":15794176,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.009075"},"pg_stats":[{"pgid":"3.1a","version":"68'1","reported_seq":114,"reported_epoch":236,"state":"active+clean","last_fresh":"2023-09-23T20:58:44.072478+0000","last_change":"2023-09-23T20:54:13.946695+0000","last_active":"2023-09-23T20:58:44.072478+0000","last_peered":"2023-09-23T20:58:44.072478+0000","last_clean":"2023-09-23T20:58:44.072478+0000","last_became_active":"2023-09-23T20:54:13.945399+0000","last_became_peered":"2023-09-23T20:54:13.945399+0000","last_unstale":"2023-09-23T20:58:44.072478+0000","last_undegraded":"2023-09-23T20:58:44.072478+0000","last_fullsized":"2023-09-23T20:58:44.072478+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:32:52.365464+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":230,"reported_epoch":237,"state":"active+clean","last_fresh":"2023-09-23T20:58:54.378611+0000","last_change":"2023-09-23T20:54:13.942019+0000","last_active":"2023-09-23T20:58:54.378611+0000","last_peered":"2023-09-23T20:58:54.378611+0000","last_clean":"2023-09-23T20:58:54.378611+0000","last_became_active":"2023-09-23T20:54:13.941647+0000","last_became_peered":"2023-09-23T20:54:13.941647+0000","last_unstale":"2023-09-23T20:58:54.378611+0000","last_undegraded":"2023-09-23T20:58:54.378611+0000","last_fullsized":"2023-09-23T20:58:54.378611+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:30:54.400944+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":207,"reported_epoch":237,"state":"active+clean","last_fresh":"2023-09-23T20:58:54.383500+0000","last_change":"2023-09-23T20:54:13.941394+0000","last_active":"2023-09-23T20:58:54.383500+0000","last_peered":"2023-09-23T20:58:54.383500+0000","last_clean":"2023-09-23T20:58:54.383500+0000","last_became_active":"2023-09-23T20:54:13.941042+0000","last_became_peered":"2023-09-23T20:54:13.941042+0000","last_unstale":"2023-09-23T20:58:54.383500+0000","last_undegraded":"2023-09-23T20:58:54.383500+0000","last_fullsized":"2023-09-23T20:58:54.383500+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:19:41.401184+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":221,"reported_epoch":236,"state":"active+clean","last_fresh":"2023-09-23T20:58:44.074119+0000","last_change":"2023-09-23T20:57:44.979811+0000","last_active":"2023-09-23T20:58:44.074119+0000","last_peered":"2023-09-23T20:58:44.074119+0000","last_clean":"2023-09-23T20:58:44.074119+0000","last_became_active":"2023-09-23T20:57:44.979515+0000","last_became_peered":"2023-09-23T20:57:44.979515+0000","last_unstale":"2023-09-23T20:58:44.074119+0000","last_undegraded":"2023-09-23T20:58:44.074119+0000","last_fullsized":"2023-09-23T20:58:44.074119+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:38:22.162140+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":221,"reported_epoch":237,"state":"active+clean","last_fresh":"2023-09-23T20:58:54.379268+0000","last_change":"2023-09-23T20:51:10.833613+0000","last_active":"2023-09-23T20:58:54.379268+0000","last_peered":"2023-09-23T20:58:54.379268+0000","last_clean":"2023-09-23T20:58:54.379268+0000","last_became_active":"2023-09-23T20:51:10.833208+0000","last_became_peered":"2023-09-23T20:51:10.833208+0000","last_unstale":"2023-09-23T20:58:54.379268+0000","last_undegraded":"2023-09-23T20:58:54.379268+0000","last_fullsized":"2023-09-23T20:58:54.379268+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:51:28.246014+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":203,"reported_epoch":237,"state":"active+clean","last_fresh":"2023-09-23T20:58:54.372087+0000","last_change":"2023-09-23T20:55:35.958384+0000","last_active":"2023-09-23T20:58:54.372087+0000","last_peered":"2023-09-23T20:58:54.372087+0000","last_clean":"2023-09-23T20:58:54.372087+0000","last_became_active":"2023-09-23T20:55:35.954563+0000","last_became_peered":"2023-09-23T20:55:35.954563+0000","last_unstale":"2023-09-23T20:58:54.372087+0000","last_undegraded":"2023-09-23T20:58:54.372087+0000","last_fullsized":"2023-09-23T20:58:54.372087+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:37:48.332259+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":211,"reported_epoch":236,"state":"active+clean","last_fresh":"2023-09-23T20:58:44.068317+0000","last_change":"2023-09-23T20:54:31.885514+0000","last_active":"2023-09-23T20:58:44.068317+0000","last_peered":"2023-09-23T20:58:44.068317+0000","last_clean":"2023-09-23T20:58:44.068317+0000","last_became_active":"2023-09-23T20:54:31.884986+0000","last_became_peered":"2023-09-23T20:54:31.884986+0000","last_unstale":"2023-09-23T20:58:44.068317+0000","last_undegraded":"2023-09-23T20:58:44.068317+0000","last_fullsized":"2023-09-23T20:58:44.068317+0000","mapping_epoch":136,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:26:08.100220+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":218,"reported_epoch":237,"state":"active+clean","last_fresh":"2023-09-23T20:58:54.379340+0000","last_change":"2023-09-23T20:55:35.951889+0000","last_active":"2023-09-23T20:58:54.379340+0000","last_peered":"2023-09-23T20:58:54.379340+0000","last_clean":"2023-09-23T20:58:54.379340+0000","last_became_active":"2023-09-23T20:55:35.951563+0000","last_became_peered":"2023-09-23T20:55:35.951563+0000","last_unstale":"2023-09-23T20:58:54.379340+0000","last_undegraded":"2023-09-23T20:58:54.379340+0000","last_fullsized":"2023-09-23T20:58:54.379340+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:41:53.098781+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"79'161","reported_seq":523,"reported_epoch":236,"state":"active+clean","last_fresh":"2023-09-23T20:58:44.063148+0000","last_change":"2023-09-23T20:57:25.096559+0000","last_active":"2023-09-23T20:58:44.063148+0000","last_peered":"2023-09-23T20:58:44.063148+0000","last_clean":"2023-09-23T20:58:44.063148+0000","last_became_active":"2023-09-23T20:57:15.975674+0000","last_became_peered":"2023-09-23T20:57:15.975674+0000","last_unstale":"2023-09-23T20:58:44.063148+0000","last_undegraded":"2023-09-23T20:58:44.063148+0000","last_fullsized":"2023-09-23T20:58:44.063148+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:25.096401+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:25.096401+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:05:05.448404+0000","scrub_duration":0.002859393,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":110,"reported_epoch":236,"state":"active+clean","last_fresh":"2023-09-23T20:58:44.066959+0000","last_change":"2023-09-23T20:54:31.890678+0000","last_active":"2023-09-23T20:58:44.066959+0000","last_peered":"2023-09-23T20:58:44.066959+0000","last_clean":"2023-09-23T20:58:44.066959+0000","last_became_active":"2023-09-23T20:54:31.890469+0000","last_became_peered":"2023-09-23T20:54:31.890469+0000","last_unstale":"2023-09-23T20:58:44.066959+0000","last_undegraded":"2023-09-23T20:58:44.066959+0000","last_fullsized":"2023-09-23T20:58:44.066959+0000","mapping_epoch":136,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:55:08.790146+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":179,"reported_epoch":237,"state":"active+clean","last_fresh":"2023-09-23T20:58:54.378811+0000","last_change":"2023-09-23T20:55:35.952661+0000","last_active":"2023-09-23T20:58:54.378811+0000","last_peered":"2023-09-23T20:58:54.378811+0000","last_clean":"2023-09-23T20:58:54.378811+0000","last_became_active":"2023-09-23T20:55:35.952349+0000","last_became_peered":"2023-09-23T20:55:35.952349+0000","last_unstale":"2023-09-23T20:58:54.378811+0000","last_undegraded":"2023-09-23T20:58:54.378811+0000","last_fullsized":"2023-09-23T20:58:54.378811+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:00:03.110126+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":55,"reported_epoch":237,"state":"active+clean","last_fresh":"2023-09-23T20:58:54.378699+0000","last_change":"2023-09-23T20:56:45.988607+0000","last_active":"2023-09-23T20:58:54.378699+0000","last_peered":"2023-09-23T20:58:54.378699+0000","last_clean":"2023-09-23T20:58:54.378699+0000","last_became_active":"2023-09-23T20:56:45.988206+0000","last_became_peered":"2023-09-23T20:56:45.988206+0000","last_unstale":"2023-09-23T20:58:54.378699+0000","last_undegraded":"2023-09-23T20:58:54.378699+0000","last_fullsized":"2023-09-23T20:58:54.378699+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":205,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:01:19.783688+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,4],"acting":[3,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":54,"reported_epoch":236,"state":"active+clean","last_fresh":"2023-09-23T20:58:44.067149+0000","last_change":"2023-09-23T20:56:18.139142+0000","last_active":"2023-09-23T20:58:44.067149+0000","last_peered":"2023-09-23T20:58:44.067149+0000","last_clean":"2023-09-23T20:58:44.067149+0000","last_became_active":"2023-09-23T20:56:18.138844+0000","last_became_peered":"2023-09-23T20:56:18.138844+0000","last_unstale":"2023-09-23T20:58:44.067149+0000","last_undegraded":"2023-09-23T20:58:44.067149+0000","last_fullsized":"2023-09-23T20:58:44.067149+0000","mapping_epoch":192,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":193,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:09:02.217683+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,2],"acting":[5,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.14","version":"79'22","reported_seq":141,"reported_epoch":236,"state":"active+clean","last_fresh":"2023-09-23T20:58:44.072760+0000","last_change":"2023-09-23T20:55:35.949418+0000","last_active":"2023-09-23T20:58:44.072760+0000","last_peered":"2023-09-23T20:58:44.072760+0000","last_clean":"2023-09-23T20:58:44.072760+0000","last_became_active":"2023-09-23T20:55:35.948566+0000","last_became_peered":"2023-09-23T20:55:35.948566+0000","last_unstale":"2023-09-23T20:58:44.072760+0000","last_undegraded":"2023-09-23T20:58:44.072760+0000","last_fullsized":"2023-09-23T20:58:44.072760+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:59:37.433128+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4734,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":131,"reported_epoch":236,"state":"active+clean","last_fresh":"2023-09-23T20:58:44.072668+0000","last_change":"2023-09-23T20:55:35.949178+0000","last_active":"2023-09-23T20:58:44.072668+0000","last_peered":"2023-09-23T20:58:44.072668+0000","last_clean":"2023-09-23T20:58:44.072668+0000","last_became_active":"2023-09-23T20:55:35.948795+0000","last_became_peered":"2023-09-23T20:55:35.948795+0000","last_unstale":"2023-09-23T20:58:44.072668+0000","last_undegraded":"2023-09-23T20:58:44.072668+0000","last_fullsized":"2023-09-23T20:58:44.072668+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:13:08.996932+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.11","version":"79'161","reported_seq":465,"reported_epoch":236,"state":"active+clean","last_fresh":"2023-09-23T20:58:44.064612+0000","last_change":"2023-09-23T20:57:28.756796+0000","last_active":"2023-09-23T20:58:44.064612+0000","last_peered":"2023-09-23T20:58:44.064612+0000","last_clean":"2023-09-23T20:58:44.064612+0000","last_became_active":"2023-09-23T20:57:15.980606+0000","last_became_peered":"2023-09-23T20:57:15.980606+0000","last_unstale":"2023-09-23T20:58:44.064612+0000","last_undegraded":"2023-09-23T20:58:44.064612+0000","last_fullsized":"2023-09-23T20:58:44.064612+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:28.756634+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:28.756634+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:09:19.150791+0000","scrub_duration":0.0064925119999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":157,"reported_epoch":236,"state":"active+clean","last_fresh":"2023-09-23T20:58:44.072481+0000","last_change":"2023-09-23T20:56:45.992577+0000","last_active":"2023-09-23T20:58:44.072481+0000","last_peered":"2023-09-23T20:58:44.072481+0000","last_clean":"2023-09-23T20:58:44.072481+0000","last_became_active":"2023-09-23T20:56:45.992297+0000","last_became_peered":"2023-09-23T20:56:45.992297+0000","last_unstale":"2023-09-23T20:58:44.072481+0000","last_undegraded":"2023-09-23T20:58:44.072481+0000","last_fullsized":"2023-09-23T20:58:44.072481+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":205,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:00:45.547202+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,2],"acting":[4,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":203,"reported_epoch":237,"state":"active+clean","last_fresh":"2023-09-23T20:58:54.379466+0000","last_change":"2023-09-23T20:51:10.831262+0000","last_active":"2023-09-23T20:58:54.379466+0000","last_peered":"2023-09-23T20:58:54.379466+0000","last_clean":"2023-09-23T20:58:54.379466+0000","last_became_active":"2023-09-23T20:51:10.830931+0000","last_became_peered":"2023-09-23T20:51:10.830931+0000","last_unstale":"2023-09-23T20:58:54.379466+0000","last_undegraded":"2023-09-23T20:58:54.379466+0000","last_fullsized":"2023-09-23T20:58:54.379466+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:51:54.982505+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.10","version":"79'161","reported_seq":474,"reported_epoch":236,"state":"active+clean","last_fresh":"2023-09-23T20:58:44.066497+0000","last_change":"2023-09-23T20:57:20.708003+0000","last_active":"2023-09-23T20:58:44.066497+0000","last_peered":"2023-09-23T20:58:44.066497+0000","last_clean":"2023-09-23T20:58:44.066497+0000","last_became_active":"2023-09-23T20:57:15.975302+0000","last_became_peered":"2023-09-23T20:57:15.975302+0000","last_unstale":"2023-09-23T20:58:44.066497+0000","last_undegraded":"2023-09-23T20:58:44.066497+0000","last_fullsized":"2023-09-23T20:58:44.066497+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:20.707939+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:20.707939+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:16:51.721994+0000","scrub_duration":0.0017733409999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":212,"reported_epoch":236,"state":"active+clean","last_fresh":"2023-09-23T20:58:51.439460+0000","last_change":"2023-09-23T20:57:44.979736+0000","last_active":"2023-09-23T20:58:51.439460+0000","last_peered":"2023-09-23T20:58:51.439460+0000","last_clean":"2023-09-23T20:58:51.439460+0000","last_became_active":"2023-09-23T20:57:44.979356+0000","last_became_peered":"2023-09-23T20:57:44.979356+0000","last_unstale":"2023-09-23T20:58:51.439460+0000","last_undegraded":"2023-09-23T20:58:51.439460+0000","last_fullsized":"2023-09-23T20:58:51.439460+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:14:43.848832+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":110,"reported_epoch":236,"state":"active+clean","last_fresh":"2023-09-23T20:58:44.066731+0000","last_change":"2023-09-23T20:54:31.887814+0000","last_active":"2023-09-23T20:58:44.066731+0000","last_peered":"2023-09-23T20:58:44.066731+0000","last_clean":"2023-09-23T20:58:44.066731+0000","last_became_active":"2023-09-23T20:54:31.887195+0000","last_became_peered":"2023-09-23T20:54:31.887195+0000","last_unstale":"2023-09-23T20:58:44.066731+0000","last_undegraded":"2023-09-23T20:58:44.066731+0000","last_fullsized":"2023-09-23T20:58:44.066731+0000","mapping_epoch":136,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:10:52.089553+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.13","version":"79'161","reported_seq":537,"reported_epoch":237,"state":"active+clean","last_fresh":"2023-09-23T20:58:54.382914+0000","last_change":"2023-09-23T20:57:48.262970+0000","last_active":"2023-09-23T20:58:54.382914+0000","last_peered":"2023-09-23T20:58:54.382914+0000","last_clean":"2023-09-23T20:58:54.382914+0000","last_became_active":"2023-09-23T20:57:48.262663+0000","last_became_peered":"2023-09-23T20:57:48.262663+0000","last_unstale":"2023-09-23T20:58:54.382914+0000","last_undegraded":"2023-09-23T20:58:54.382914+0000","last_fullsized":"2023-09-23T20:58:54.382914+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:36.420362+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:36.420362+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:28:28.613492+0000","scrub_duration":0.028838444000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":181,"reported_epoch":236,"state":"active+clean","last_fresh":"2023-09-23T20:58:44.074738+0000","last_change":"2023-09-23T20:56:45.991822+0000","last_active":"2023-09-23T20:58:44.074738+0000","last_peered":"2023-09-23T20:58:44.074738+0000","last_clean":"2023-09-23T20:58:44.074738+0000","last_became_active":"2023-09-23T20:56:45.991460+0000","last_became_peered":"2023-09-23T20:56:45.991460+0000","last_unstale":"2023-09-23T20:58:44.074738+0000","last_undegraded":"2023-09-23T20:58:44.074738+0000","last_fullsized":"2023-09-23T20:58:44.074738+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":205,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:20:37.503317+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":193,"reported_epoch":236,"state":"active+clean","last_fresh":"2023-09-23T20:58:44.074243+0000","last_change":"2023-09-23T20:51:10.840849+0000","last_active":"2023-09-23T20:58:44.074243+0000","last_peered":"2023-09-23T20:58:44.074243+0000","last_clean":"2023-09-23T20:58:44.074243+0000","last_became_active":"2023-09-23T20:51:10.840593+0000","last_became_peered":"2023-09-23T20:51:10.840593+0000","last_unstale":"2023-09-23T20:58:44.074243+0000","last_undegraded":"2023-09-23T20:58:44.074243+0000","last_fullsized":"2023-09-23T20:58:44.074243+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:30:52.297625+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.12","version":"79'161","reported_seq":474,"reported_epoch":236,"state":"active+clean","last_fresh":"2023-09-23T20:58:44.066611+0000","last_change":"2023-09-23T20:57:21.750765+0000","last_active":"2023-09-23T20:58:44.066611+0000","last_peered":"2023-09-23T20:58:44.066611+0000","last_clean":"2023-09-23T20:58:44.066611+0000","last_became_active":"2023-09-23T20:57:15.979195+0000","last_became_peered":"2023-09-23T20:57:15.979195+0000","last_unstale":"2023-09-23T20:58:44.066611+0000","last_undegraded":"2023-09-23T20:58:44.066611+0000","last_fullsized":"2023-09-23T20:58:44.066611+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:21.750680+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:21.750680+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:03:41.915962+0000","scrub_duration":0.0092391680000000007,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":230,"reported_epoch":237,"state":"active+clean","last_fresh":"2023-09-23T20:58:54.371577+0000","last_change":"2023-09-23T20:55:36.943969+0000","last_active":"2023-09-23T20:58:54.371577+0000","last_peered":"2023-09-23T20:58:54.371577+0000","last_clean":"2023-09-23T20:58:54.371577+0000","last_became_active":"2023-09-23T20:55:36.943580+0000","last_became_peered":"2023-09-23T20:55:36.943580+0000","last_unstale":"2023-09-23T20:58:54.371577+0000","last_undegraded":"2023-09-23T20:58:54.371577+0000","last_fullsized":"2023-09-23T20:58:54.371577+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:49:06.969396+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":26,"reported_epoch":237,"state":"active+clean","last_fresh":"2023-09-23T20:58:54.379697+0000","last_change":"2023-09-23T20:57:44.980498+0000","last_active":"2023-09-23T20:58:54.379697+0000","last_peered":"2023-09-23T20:58:54.379697+0000","last_clean":"2023-09-23T20:58:54.379697+0000","last_became_active":"2023-09-23T20:57:44.980182+0000","last_became_peered":"2023-09-23T20:57:44.980182+0000","last_unstale":"2023-09-23T20:58:54.379697+0000","last_undegraded":"2023-09-23T20:58:54.379697+0000","last_fullsized":"2023-09-23T20:58:54.379697+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:34:51.620544+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.d","version":"79'161","reported_seq":519,"reported_epoch":236,"state":"active+clean","last_fresh":"2023-09-23T20:58:44.072827+0000","last_change":"2023-09-23T20:57:31.075213+0000","last_active":"2023-09-23T20:58:44.072827+0000","last_peered":"2023-09-23T20:58:44.072827+0000","last_clean":"2023-09-23T20:58:44.072827+0000","last_became_active":"2023-09-23T20:57:15.988188+0000","last_became_peered":"2023-09-23T20:57:15.988188+0000","last_unstale":"2023-09-23T20:58:44.072827+0000","last_undegraded":"2023-09-23T20:58:44.072827+0000","last_fullsized":"2023-09-23T20:58:44.072827+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:31.075098+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:31.075098+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:41:52.142362+0000","scrub_duration":0.010304610000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.b","version":"238'8560","reported_seq":8609,"reported_epoch":238,"state":"active+clean","last_fresh":"2023-09-23T20:59:09.265999+0000","last_change":"2023-09-23T20:56:35.543994+0000","last_active":"2023-09-23T20:59:09.265999+0000","last_peered":"2023-09-23T20:59:09.265999+0000","last_clean":"2023-09-23T20:59:09.265999+0000","last_became_active":"2023-09-23T20:56:35.543758+0000","last_became_peered":"2023-09-23T20:56:35.543758+0000","last_unstale":"2023-09-23T20:59:09.265999+0000","last_undegraded":"2023-09-23T20:59:09.265999+0000","last_fullsized":"2023-09-23T20:59:09.265999+0000","mapping_epoch":199,"log_start":"205'1500","ondisk_log_start":"205'1500","created":199,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_clean_scrub_stamp":"2023-09-23T20:56:34.529103+0000","objects_scrubbed":0,"log_size":7060,"log_dups_size":0,"ondisk_log_size":7060,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:21:29.041484+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":270008320,"num_objects":4120,"num_object_clones":0,"num_object_copies":12360,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4120,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8560,"num_write_kb":405760,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":168,"reported_epoch":236,"state":"active+clean","last_fresh":"2023-09-23T20:58:44.073643+0000","last_change":"2023-09-23T20:56:45.991841+0000","last_active":"2023-09-23T20:58:44.073643+0000","last_peered":"2023-09-23T20:58:44.073643+0000","last_clean":"2023-09-23T20:58:44.073643+0000","last_became_active":"2023-09-23T20:56:45.991491+0000","last_became_peered":"2023-09-23T20:56:45.991491+0000","last_unstale":"2023-09-23T20:58:44.073643+0000","last_undegraded":"2023-09-23T20:58:44.073643+0000","last_fullsized":"2023-09-23T20:58:44.073643+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":205,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:30:37.405319+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":127,"reported_epoch":236,"state":"active+clean","last_fresh":"2023-09-23T20:58:44.066584+0000","last_change":"2023-09-23T20:55:35.957768+0000","last_active":"2023-09-23T20:58:44.066584+0000","last_peered":"2023-09-23T20:58:44.066584+0000","last_clean":"2023-09-23T20:58:44.066584+0000","last_became_active":"2023-09-23T20:55:35.957421+0000","last_became_peered":"2023-09-23T20:55:35.957421+0000","last_unstale":"2023-09-23T20:58:44.066584+0000","last_undegraded":"2023-09-23T20:58:44.066584+0000","last_fullsized":"2023-09-23T20:58:44.066584+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:49:41.585961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.c","version":"79'161","reported_seq":538,"reported_epoch":237,"state":"active+clean","last_fresh":"2023-09-23T20:58:54.379295+0000","last_change":"2023-09-23T20:57:22.041071+0000","last_active":"2023-09-23T20:58:54.379295+0000","last_peered":"2023-09-23T20:58:54.379295+0000","last_clean":"2023-09-23T20:58:54.379295+0000","last_became_active":"2023-09-23T20:57:15.973204+0000","last_became_peered":"2023-09-23T20:57:15.973204+0000","last_unstale":"2023-09-23T20:58:54.379295+0000","last_undegraded":"2023-09-23T20:58:54.379295+0000","last_fullsized":"2023-09-23T20:58:54.379295+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:22.040907+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:22.040907+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:47:02.019978+0000","scrub_duration":0.013021176000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.a","version":"238'8645","reported_seq":8694,"reported_epoch":238,"state":"active+clean","last_fresh":"2023-09-23T20:59:09.489755+0000","last_change":"2023-09-23T20:56:35.547120+0000","last_active":"2023-09-23T20:59:09.489755+0000","last_peered":"2023-09-23T20:59:09.489755+0000","last_clean":"2023-09-23T20:59:09.489755+0000","last_became_active":"2023-09-23T20:56:35.546838+0000","last_became_peered":"2023-09-23T20:56:35.546838+0000","last_unstale":"2023-09-23T20:59:09.489755+0000","last_undegraded":"2023-09-23T20:59:09.489755+0000","last_fullsized":"2023-09-23T20:59:09.489755+0000","mapping_epoch":199,"log_start":"206'2500","ondisk_log_start":"206'2500","created":199,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_clean_scrub_stamp":"2023-09-23T20:56:34.529103+0000","objects_scrubbed":0,"log_size":6145,"log_dups_size":0,"ondisk_log_size":6145,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:48:24.132768+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":268107776,"num_objects":4091,"num_object_clones":0,"num_object_copies":12273,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4091,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8645,"num_write_kb":407552,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,5],"acting":[3,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":155,"reported_epoch":236,"state":"active+clean","last_fresh":"2023-09-23T20:58:44.073395+0000","last_change":"2023-09-23T20:56:45.991821+0000","last_active":"2023-09-23T20:58:44.073395+0000","last_peered":"2023-09-23T20:58:44.073395+0000","last_clean":"2023-09-23T20:58:44.073395+0000","last_became_active":"2023-09-23T20:56:45.991447+0000","last_became_peered":"2023-09-23T20:56:45.991447+0000","last_unstale":"2023-09-23T20:58:44.073395+0000","last_undegraded":"2023-09-23T20:58:44.073395+0000","last_fullsized":"2023-09-23T20:58:44.073395+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":205,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:14:25.525222+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":158,"reported_epoch":236,"state":"active+clean","last_fresh":"2023-09-23T20:58:44.075133+0000","last_change":"2023-09-23T20:57:44.982059+0000","last_active":"2023-09-23T20:58:44.075133+0000","last_peered":"2023-09-23T20:58:44.075133+0000","last_clean":"2023-09-23T20:58:44.075133+0000","last_became_active":"2023-09-23T20:57:44.981742+0000","last_became_peered":"2023-09-23T20:57:44.981742+0000","last_unstale":"2023-09-23T20:58:44.075133+0000","last_undegraded":"2023-09-23T20:58:44.075133+0000","last_fullsized":"2023-09-23T20:58:44.075133+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:41:24.042916+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.f","version":"79'161","reported_seq":542,"reported_epoch":237,"state":"active+clean","last_fresh":"2023-09-23T20:58:54.371923+0000","last_change":"2023-09-23T20:57:27.880841+0000","last_active":"2023-09-23T20:58:54.371923+0000","last_peered":"2023-09-23T20:58:54.371923+0000","last_clean":"2023-09-23T20:58:54.371923+0000","last_became_active":"2023-09-23T20:57:15.970925+0000","last_became_peered":"2023-09-23T20:57:15.970925+0000","last_unstale":"2023-09-23T20:58:54.371923+0000","last_undegraded":"2023-09-23T20:58:54.371923+0000","last_fullsized":"2023-09-23T20:58:54.371923+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:27.880776+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:27.880776+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:15:13.438014+0000","scrub_duration":0.0014004899999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.9","version":"237'8530","reported_seq":8578,"reported_epoch":237,"state":"active+clean","last_fresh":"2023-09-23T20:59:06.455437+0000","last_change":"2023-09-23T20:56:35.549387+0000","last_active":"2023-09-23T20:59:06.455437+0000","last_peered":"2023-09-23T20:59:06.455437+0000","last_clean":"2023-09-23T20:59:06.455437+0000","last_became_active":"2023-09-23T20:56:35.549087+0000","last_became_peered":"2023-09-23T20:56:35.549087+0000","last_unstale":"2023-09-23T20:59:06.455437+0000","last_undegraded":"2023-09-23T20:59:06.455437+0000","last_fullsized":"2023-09-23T20:59:06.455437+0000","mapping_epoch":199,"log_start":"205'1200","ondisk_log_start":"205'1200","created":199,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_clean_scrub_stamp":"2023-09-23T20:56:34.529103+0000","objects_scrubbed":0,"log_size":7330,"log_dups_size":0,"ondisk_log_size":7330,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:04:51.547362+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":273678336,"num_objects":4176,"num_object_clones":0,"num_object_copies":12528,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4176,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8530,"num_write_kb":406592,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":141,"reported_epoch":236,"state":"active+clean","last_fresh":"2023-09-23T20:58:44.066748+0000","last_change":"2023-09-23T20:56:45.989153+0000","last_active":"2023-09-23T20:58:44.066748+0000","last_peered":"2023-09-23T20:58:44.066748+0000","last_clean":"2023-09-23T20:58:44.066748+0000","last_became_active":"2023-09-23T20:56:45.988800+0000","last_became_peered":"2023-09-23T20:56:45.988800+0000","last_unstale":"2023-09-23T20:58:44.066748+0000","last_undegraded":"2023-09-23T20:58:44.066748+0000","last_fullsized":"2023-09-23T20:58:44.066748+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":205,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:04:41.130194+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":177,"reported_epoch":237,"state":"active+clean","last_fresh":"2023-09-23T20:58:54.371680+0000","last_change":"2023-09-23T20:55:35.965427+0000","last_active":"2023-09-23T20:58:54.371680+0000","last_peered":"2023-09-23T20:58:54.371680+0000","last_clean":"2023-09-23T20:58:54.371680+0000","last_became_active":"2023-09-23T20:55:35.963969+0000","last_became_peered":"2023-09-23T20:55:35.963969+0000","last_unstale":"2023-09-23T20:58:54.371680+0000","last_undegraded":"2023-09-23T20:58:54.371680+0000","last_fullsized":"2023-09-23T20:58:54.371680+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:38:40.612083+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.e","version":"79'161","reported_seq":448,"reported_epoch":236,"state":"active+clean","last_fresh":"2023-09-23T20:58:44.073258+0000","last_change":"2023-09-23T20:57:34.132738+0000","last_active":"2023-09-23T20:58:44.073258+0000","last_peered":"2023-09-23T20:58:44.073258+0000","last_clean":"2023-09-23T20:58:44.073258+0000","last_became_active":"2023-09-23T20:57:15.974318+0000","last_became_peered":"2023-09-23T20:57:15.974318+0000","last_unstale":"2023-09-23T20:58:44.073258+0000","last_undegraded":"2023-09-23T20:58:44.073258+0000","last_fullsized":"2023-09-23T20:58:44.073258+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:34.132660+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:34.132660+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:07:07.739438+0000","scrub_duration":0.005229694,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.8","version":"237'8616","reported_seq":8664,"reported_epoch":237,"state":"active+clean","last_fresh":"2023-09-23T20:59:06.424316+0000","last_change":"2023-09-23T20:56:35.548119+0000","last_active":"2023-09-23T20:59:06.424316+0000","last_peered":"2023-09-23T20:59:06.424316+0000","last_clean":"2023-09-23T20:59:06.424316+0000","last_became_active":"2023-09-23T20:56:35.547914+0000","last_became_peered":"2023-09-23T20:56:35.547914+0000","last_unstale":"2023-09-23T20:59:06.424316+0000","last_undegraded":"2023-09-23T20:59:06.424316+0000","last_fullsized":"2023-09-23T20:59:06.424316+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":199,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_clean_scrub_stamp":"2023-09-23T20:56:34.529103+0000","objects_scrubbed":0,"log_size":8616,"log_dups_size":0,"ondisk_log_size":8616,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:42:54.255023+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":277741568,"num_objects":4238,"num_object_clones":0,"num_object_copies":12714,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4238,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8616,"num_write_kb":411328,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":127,"reported_epoch":236,"state":"active+clean","last_fresh":"2023-09-23T20:58:44.066803+0000","last_change":"2023-09-23T20:55:35.962005+0000","last_active":"2023-09-23T20:58:44.066803+0000","last_peered":"2023-09-23T20:58:44.066803+0000","last_clean":"2023-09-23T20:58:44.066803+0000","last_became_active":"2023-09-23T20:55:35.961695+0000","last_became_peered":"2023-09-23T20:55:35.961695+0000","last_unstale":"2023-09-23T20:58:44.066803+0000","last_undegraded":"2023-09-23T20:58:44.066803+0000","last_fullsized":"2023-09-23T20:58:44.066803+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:28:22.876201+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":225,"reported_epoch":236,"state":"active+clean","last_fresh":"2023-09-23T20:58:44.061487+0000","last_change":"2023-09-23T20:54:13.928715+0000","last_active":"2023-09-23T20:58:44.061487+0000","last_peered":"2023-09-23T20:58:44.061487+0000","last_clean":"2023-09-23T20:58:44.061487+0000","last_became_active":"2023-09-23T20:54:13.928200+0000","last_became_peered":"2023-09-23T20:54:13.928200+0000","last_unstale":"2023-09-23T20:58:44.061487+0000","last_undegraded":"2023-09-23T20:58:44.061487+0000","last_fullsized":"2023-09-23T20:58:44.061487+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:41:16.927513+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.f","version":"237'8688","reported_seq":8734,"reported_epoch":237,"state":"active+clean","last_fresh":"2023-09-23T20:59:06.474022+0000","last_change":"2023-09-23T20:56:35.545128+0000","last_active":"2023-09-23T20:59:06.474022+0000","last_peered":"2023-09-23T20:59:06.474022+0000","last_clean":"2023-09-23T20:59:06.474022+0000","last_became_active":"2023-09-23T20:56:35.544838+0000","last_became_peered":"2023-09-23T20:56:35.544838+0000","last_unstale":"2023-09-23T20:59:06.474022+0000","last_undegraded":"2023-09-23T20:59:06.474022+0000","last_fullsized":"2023-09-23T20:59:06.474022+0000","mapping_epoch":199,"log_start":"205'1300","ondisk_log_start":"205'1300","created":199,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_clean_scrub_stamp":"2023-09-23T20:56:34.529103+0000","objects_scrubbed":0,"log_size":7388,"log_dups_size":0,"ondisk_log_size":7388,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:30:43.388717+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":273809408,"num_objects":4178,"num_object_clones":0,"num_object_copies":12534,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4178,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8688,"num_write_kb":411712,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.9","version":"79'161","reported_seq":611,"reported_epoch":237,"state":"active+clean","last_fresh":"2023-09-23T20:58:59.536889+0000","last_change":"2023-09-23T20:56:46.921939+0000","last_active":"2023-09-23T20:58:59.536889+0000","last_peered":"2023-09-23T20:58:59.536889+0000","last_clean":"2023-09-23T20:58:59.536889+0000","last_became_active":"2023-09-23T20:56:45.989908+0000","last_became_peered":"2023-09-23T20:56:45.989908+0000","last_unstale":"2023-09-23T20:58:59.536889+0000","last_undegraded":"2023-09-23T20:58:59.536889+0000","last_fullsized":"2023-09-23T20:58:59.536889+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":205,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:56:46.921739+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:56:46.921739+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:20:42.635110+0000","scrub_duration":0.0071954109999999996,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":227,"reported_epoch":237,"state":"active+clean","last_fresh":"2023-09-23T20:58:54.379093+0000","last_change":"2023-09-23T20:54:13.961788+0000","last_active":"2023-09-23T20:58:54.379093+0000","last_peered":"2023-09-23T20:58:54.379093+0000","last_clean":"2023-09-23T20:58:54.379093+0000","last_became_active":"2023-09-23T20:54:13.940937+0000","last_became_peered":"2023-09-23T20:54:13.940937+0000","last_unstale":"2023-09-23T20:58:54.379093+0000","last_undegraded":"2023-09-23T20:58:54.379093+0000","last_fullsized":"2023-09-23T20:58:54.379093+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:11:56.785299+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":131,"reported_epoch":236,"state":"active+clean","last_fresh":"2023-09-23T20:58:44.072847+0000","last_change":"2023-09-23T20:55:35.949670+0000","last_active":"2023-09-23T20:58:44.072847+0000","last_peered":"2023-09-23T20:58:44.072847+0000","last_clean":"2023-09-23T20:58:44.072847+0000","last_became_active":"2023-09-23T20:55:35.949140+0000","last_became_peered":"2023-09-23T20:55:35.949140+0000","last_unstale":"2023-09-23T20:58:44.072847+0000","last_undegraded":"2023-09-23T20:58:44.072847+0000","last_fullsized":"2023-09-23T20:58:44.072847+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:55:59.340303+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.e","version":"237'8563","reported_seq":8611,"reported_epoch":237,"state":"active+clean","last_fresh":"2023-09-23T20:59:06.324889+0000","last_change":"2023-09-23T20:56:35.547184+0000","last_active":"2023-09-23T20:59:06.324889+0000","last_peered":"2023-09-23T20:59:06.324889+0000","last_clean":"2023-09-23T20:59:06.324889+0000","last_became_active":"2023-09-23T20:56:35.546912+0000","last_became_peered":"2023-09-23T20:56:35.546912+0000","last_unstale":"2023-09-23T20:59:06.324889+0000","last_undegraded":"2023-09-23T20:59:06.324889+0000","last_fullsized":"2023-09-23T20:59:06.324889+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":199,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_clean_scrub_stamp":"2023-09-23T20:56:34.529103+0000","objects_scrubbed":0,"log_size":8563,"log_dups_size":0,"ondisk_log_size":8563,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:57:07.422792+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":276365312,"num_objects":4217,"num_object_clones":0,"num_object_copies":12651,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4217,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8563,"num_write_kb":408960,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,0],"acting":[2,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.8","version":"79'161","reported_seq":480,"reported_epoch":236,"state":"active+clean","last_fresh":"2023-09-23T20:58:44.066574+0000","last_change":"2023-09-23T20:56:57.505737+0000","last_active":"2023-09-23T20:58:44.066574+0000","last_peered":"2023-09-23T20:58:44.066574+0000","last_clean":"2023-09-23T20:58:44.066574+0000","last_became_active":"2023-09-23T20:56:04.982345+0000","last_became_peered":"2023-09-23T20:56:04.982345+0000","last_unstale":"2023-09-23T20:58:44.066574+0000","last_undegraded":"2023-09-23T20:58:44.066574+0000","last_fullsized":"2023-09-23T20:58:44.066574+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:56:57.505663+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:56:57.505663+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:34:30.109724+0000","scrub_duration":0.026284108,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":200,"reported_epoch":237,"state":"active+clean","last_fresh":"2023-09-23T20:58:54.371849+0000","last_change":"2023-09-23T20:55:35.959942+0000","last_active":"2023-09-23T20:58:54.371849+0000","last_peered":"2023-09-23T20:58:54.371849+0000","last_clean":"2023-09-23T20:58:54.371849+0000","last_became_active":"2023-09-23T20:55:35.957071+0000","last_became_peered":"2023-09-23T20:55:35.957071+0000","last_unstale":"2023-09-23T20:58:54.371849+0000","last_undegraded":"2023-09-23T20:58:54.371849+0000","last_fullsized":"2023-09-23T20:58:54.371849+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:44:30.308036+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":114,"reported_epoch":236,"state":"active+clean","last_fresh":"2023-09-23T20:58:44.073150+0000","last_change":"2023-09-23T20:54:13.951459+0000","last_active":"2023-09-23T20:58:44.073150+0000","last_peered":"2023-09-23T20:58:44.073150+0000","last_clean":"2023-09-23T20:58:44.073150+0000","last_became_active":"2023-09-23T20:54:13.948409+0000","last_became_peered":"2023-09-23T20:54:13.948409+0000","last_unstale":"2023-09-23T20:58:44.073150+0000","last_undegraded":"2023-09-23T20:58:44.073150+0000","last_fullsized":"2023-09-23T20:58:44.073150+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:38:10.148061+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.b","version":"79'161","reported_seq":520,"reported_epoch":237,"state":"active+clean","last_fresh":"2023-09-23T20:58:54.383005+0000","last_change":"2023-09-23T20:57:27.320172+0000","last_active":"2023-09-23T20:58:54.383005+0000","last_peered":"2023-09-23T20:58:54.383005+0000","last_clean":"2023-09-23T20:58:54.383005+0000","last_became_active":"2023-09-23T20:57:16.009630+0000","last_became_peered":"2023-09-23T20:57:16.009630+0000","last_unstale":"2023-09-23T20:58:54.383005+0000","last_undegraded":"2023-09-23T20:58:54.383005+0000","last_fullsized":"2023-09-23T20:58:54.383005+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:27.320022+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:27.320022+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:31:35.221558+0000","scrub_duration":0.0043894679999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,5],"acting":[1,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.d","version":"238'8710","reported_seq":8758,"reported_epoch":238,"state":"active+clean","last_fresh":"2023-09-23T20:59:09.053622+0000","last_change":"2023-09-23T20:56:35.544132+0000","last_active":"2023-09-23T20:59:09.053622+0000","last_peered":"2023-09-23T20:59:09.053622+0000","last_clean":"2023-09-23T20:59:09.053622+0000","last_became_active":"2023-09-23T20:56:35.543836+0000","last_became_peered":"2023-09-23T20:56:35.543836+0000","last_unstale":"2023-09-23T20:59:09.053622+0000","last_undegraded":"2023-09-23T20:59:09.053622+0000","last_fullsized":"2023-09-23T20:59:09.053622+0000","mapping_epoch":199,"log_start":"205'1300","ondisk_log_start":"205'1300","created":199,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_clean_scrub_stamp":"2023-09-23T20:56:34.529103+0000","objects_scrubbed":0,"log_size":7410,"log_dups_size":0,"ondisk_log_size":7410,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:27:14.636991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":267321368,"num_objects":4080,"num_object_clones":0,"num_object_copies":12240,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4080,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":8710,"num_write_kb":409217,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":114,"reported_epoch":236,"state":"active+clean","last_fresh":"2023-09-23T20:58:44.074840+0000","last_change":"2023-09-23T20:54:13.950254+0000","last_active":"2023-09-23T20:58:44.074840+0000","last_peered":"2023-09-23T20:58:44.074840+0000","last_clean":"2023-09-23T20:58:44.074840+0000","last_became_active":"2023-09-23T20:54:13.949934+0000","last_became_peered":"2023-09-23T20:54:13.949934+0000","last_unstale":"2023-09-23T20:58:44.074840+0000","last_undegraded":"2023-09-23T20:58:44.074840+0000","last_fullsized":"2023-09-23T20:58:44.074840+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:30:32.847167+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":131,"reported_epoch":236,"state":"active+clean","last_fresh":"2023-09-23T20:58:44.073467+0000","last_change":"2023-09-23T20:55:35.956153+0000","last_active":"2023-09-23T20:58:44.073467+0000","last_peered":"2023-09-23T20:58:44.073467+0000","last_clean":"2023-09-23T20:58:44.073467+0000","last_became_active":"2023-09-23T20:55:35.955630+0000","last_became_peered":"2023-09-23T20:55:35.955630+0000","last_unstale":"2023-09-23T20:58:44.073467+0000","last_undegraded":"2023-09-23T20:58:44.073467+0000","last_fullsized":"2023-09-23T20:58:44.073467+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:45:20.282929+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.c","version":"238'8674","reported_seq":8731,"reported_epoch":238,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T20:59:09.048499+0000","last_change":"2023-09-23T20:57:46.463005+0000","last_active":"2023-09-23T20:59:09.048499+0000","last_peered":"2023-09-23T20:59:09.048499+0000","last_clean":"2023-09-23T20:57:44.015119+0000","last_became_active":"2023-09-23T20:57:46.448702+0000","last_became_peered":"2023-09-23T20:57:46.448702+0000","last_unstale":"2023-09-23T20:59:09.048499+0000","last_undegraded":"2023-09-23T20:57:46.010874+0000","last_fullsized":"2023-09-23T20:57:45.978434+0000","mapping_epoch":223,"log_start":"205'1300","ondisk_log_start":"205'1300","created":199,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_clean_scrub_stamp":"2023-09-23T20:56:34.529103+0000","objects_scrubbed":0,"log_size":7374,"log_dups_size":0,"ondisk_log_size":7374,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:08:34.180651+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":270925824,"num_objects":4134,"num_object_clones":0,"num_object_copies":12402,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5473,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4134,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8674,"num_write_kb":409856,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,7],"avail_no_missing":["6","2","7"],"object_location_counts":[{"shards":"2,6,7","objects":3203},{"shards":"6,7","objects":-1339}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.a","version":"79'161","reported_seq":480,"reported_epoch":236,"state":"active+clean","last_fresh":"2023-09-23T20:58:44.067163+0000","last_change":"2023-09-23T20:56:44.487778+0000","last_active":"2023-09-23T20:58:44.067163+0000","last_peered":"2023-09-23T20:58:44.067163+0000","last_clean":"2023-09-23T20:58:44.067163+0000","last_became_active":"2023-09-23T20:56:05.006740+0000","last_became_peered":"2023-09-23T20:56:05.006740+0000","last_unstale":"2023-09-23T20:58:44.067163+0000","last_undegraded":"2023-09-23T20:58:44.067163+0000","last_fullsized":"2023-09-23T20:58:44.067163+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:56:44.487678+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:56:44.487678+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:04:50.086749+0000","scrub_duration":0.0017408899999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":26,"reported_epoch":237,"state":"active+clean","last_fresh":"2023-09-23T20:58:54.379018+0000","last_change":"2023-09-23T20:57:44.982513+0000","last_active":"2023-09-23T20:58:54.379018+0000","last_peered":"2023-09-23T20:58:54.379018+0000","last_clean":"2023-09-23T20:58:54.379018+0000","last_became_active":"2023-09-23T20:57:44.982112+0000","last_became_peered":"2023-09-23T20:57:44.982112+0000","last_unstale":"2023-09-23T20:58:54.379018+0000","last_undegraded":"2023-09-23T20:58:54.379018+0000","last_fullsized":"2023-09-23T20:58:54.379018+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:09:05.630041+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":208,"reported_epoch":237,"state":"active+clean","last_fresh":"2023-09-23T20:59:04.306152+0000","last_change":"2023-09-23T20:54:13.940748+0000","last_active":"2023-09-23T20:59:04.306152+0000","last_peered":"2023-09-23T20:59:04.306152+0000","last_clean":"2023-09-23T20:59:04.306152+0000","last_became_active":"2023-09-23T20:54:13.940419+0000","last_became_peered":"2023-09-23T20:54:13.940419+0000","last_unstale":"2023-09-23T20:59:04.306152+0000","last_undegraded":"2023-09-23T20:59:04.306152+0000","last_fullsized":"2023-09-23T20:59:04.306152+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:15:01.923816+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.1","version":"237'8625","reported_seq":8684,"reported_epoch":237,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T20:59:07.060760+0000","last_change":"2023-09-23T20:57:46.483248+0000","last_active":"2023-09-23T20:59:07.060760+0000","last_peered":"2023-09-23T20:59:07.060760+0000","last_clean":"2023-09-23T20:57:44.047957+0000","last_became_active":"2023-09-23T20:57:46.475568+0000","last_became_peered":"2023-09-23T20:57:46.475568+0000","last_unstale":"2023-09-23T20:59:07.060760+0000","last_undegraded":"2023-09-23T20:57:46.083591+0000","last_fullsized":"2023-09-23T20:57:45.978436+0000","mapping_epoch":223,"log_start":"205'1900","ondisk_log_start":"205'1900","created":199,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_clean_scrub_stamp":"2023-09-23T20:56:34.529103+0000","objects_scrubbed":0,"log_size":6725,"log_dups_size":0,"ondisk_log_size":6725,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:22:45.067269+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":272695296,"num_objects":4161,"num_object_clones":0,"num_object_copies":12483,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5470,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4161,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8625,"num_write_kb":409152,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,6],"avail_no_missing":["4","1","6"],"object_location_counts":[{"shards":"1,4,6","objects":3238},{"shards":"4,6","objects":-1309}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.7","version":"79'161","reported_seq":543,"reported_epoch":237,"state":"active+clean","last_fresh":"2023-09-23T20:58:54.372093+0000","last_change":"2023-09-23T20:57:48.243572+0000","last_active":"2023-09-23T20:58:54.372093+0000","last_peered":"2023-09-23T20:58:54.372093+0000","last_clean":"2023-09-23T20:58:54.372093+0000","last_became_active":"2023-09-23T20:57:48.243229+0000","last_became_peered":"2023-09-23T20:57:48.243229+0000","last_unstale":"2023-09-23T20:58:54.372093+0000","last_undegraded":"2023-09-23T20:58:54.372093+0000","last_fullsized":"2023-09-23T20:58:54.372093+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":3,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:25.916867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:25.916867+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:23:19.385831+0000","scrub_duration":0.001665323,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":213,"reported_epoch":237,"state":"active+clean","last_fresh":"2023-09-23T20:58:54.386256+0000","last_change":"2023-09-23T20:50:05.867130+0000","last_active":"2023-09-23T20:58:54.386256+0000","last_peered":"2023-09-23T20:58:54.386256+0000","last_clean":"2023-09-23T20:58:54.386256+0000","last_became_active":"2023-09-23T20:50:05.866758+0000","last_became_peered":"2023-09-23T20:50:05.866758+0000","last_unstale":"2023-09-23T20:58:54.386256+0000","last_undegraded":"2023-09-23T20:58:54.386256+0000","last_fullsized":"2023-09-23T20:58:54.386256+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:55:07.548209+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":110,"reported_epoch":236,"state":"active+clean","last_fresh":"2023-09-23T20:58:44.066851+0000","last_change":"2023-09-23T20:54:31.887604+0000","last_active":"2023-09-23T20:58:44.066851+0000","last_peered":"2023-09-23T20:58:44.066851+0000","last_clean":"2023-09-23T20:58:44.066851+0000","last_became_active":"2023-09-23T20:54:31.886752+0000","last_became_peered":"2023-09-23T20:54:31.886752+0000","last_unstale":"2023-09-23T20:58:44.066851+0000","last_undegraded":"2023-09-23T20:58:44.066851+0000","last_fullsized":"2023-09-23T20:58:44.066851+0000","mapping_epoch":136,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:13:58.872176+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":211,"reported_epoch":237,"state":"active+clean","last_fresh":"2023-09-23T20:58:54.383703+0000","last_change":"2023-09-23T20:54:31.899082+0000","last_active":"2023-09-23T20:58:54.383703+0000","last_peered":"2023-09-23T20:58:54.383703+0000","last_clean":"2023-09-23T20:58:54.383703+0000","last_became_active":"2023-09-23T20:54:31.898521+0000","last_became_peered":"2023-09-23T20:54:31.898521+0000","last_unstale":"2023-09-23T20:58:54.383703+0000","last_undegraded":"2023-09-23T20:58:54.383703+0000","last_fullsized":"2023-09-23T20:58:54.383703+0000","mapping_epoch":136,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:25:36.059711+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.7","version":"238'8693","reported_seq":8740,"reported_epoch":238,"state":"active+clean","last_fresh":"2023-09-23T20:59:09.000222+0000","last_change":"2023-09-23T20:56:35.541695+0000","last_active":"2023-09-23T20:59:09.000222+0000","last_peered":"2023-09-23T20:59:09.000222+0000","last_clean":"2023-09-23T20:59:09.000222+0000","last_became_active":"2023-09-23T20:56:35.541392+0000","last_became_peered":"2023-09-23T20:56:35.541392+0000","last_unstale":"2023-09-23T20:59:09.000222+0000","last_undegraded":"2023-09-23T20:59:09.000222+0000","last_fullsized":"2023-09-23T20:59:09.000222+0000","mapping_epoch":199,"log_start":"205'1300","ondisk_log_start":"205'1300","created":199,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_clean_scrub_stamp":"2023-09-23T20:56:34.529103+0000","objects_scrubbed":0,"log_size":7393,"log_dups_size":0,"ondisk_log_size":7393,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:57:22.478253+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":269811712,"num_objects":4117,"num_object_clones":0,"num_object_copies":12351,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4117,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8693,"num_write_kb":409920,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.1","version":"79'161","reported_seq":500,"reported_epoch":236,"state":"active+clean","last_fresh":"2023-09-23T20:58:44.066608+0000","last_change":"2023-09-23T20:57:19.669346+0000","last_active":"2023-09-23T20:58:44.066608+0000","last_peered":"2023-09-23T20:58:44.066608+0000","last_clean":"2023-09-23T20:58:44.066608+0000","last_became_active":"2023-09-23T20:57:08.246046+0000","last_became_peered":"2023-09-23T20:57:08.246046+0000","last_unstale":"2023-09-23T20:58:44.066608+0000","last_undegraded":"2023-09-23T20:58:44.066608+0000","last_fullsized":"2023-09-23T20:58:44.066608+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:19.669259+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:19.669259+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:14:03.435531+0000","scrub_duration":0.0045759559999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":127,"reported_epoch":236,"state":"active+clean","last_fresh":"2023-09-23T20:58:44.067057+0000","last_change":"2023-09-23T20:55:35.959514+0000","last_active":"2023-09-23T20:58:44.067057+0000","last_peered":"2023-09-23T20:58:44.067057+0000","last_clean":"2023-09-23T20:58:44.067057+0000","last_became_active":"2023-09-23T20:55:35.959244+0000","last_became_peered":"2023-09-23T20:55:35.959244+0000","last_unstale":"2023-09-23T20:58:44.067057+0000","last_undegraded":"2023-09-23T20:58:44.067057+0000","last_fullsized":"2023-09-23T20:58:44.067057+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:47:32.665316+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":131,"reported_epoch":236,"state":"active+clean","last_fresh":"2023-09-23T20:58:44.072736+0000","last_change":"2023-09-23T20:55:35.949000+0000","last_active":"2023-09-23T20:58:44.072736+0000","last_peered":"2023-09-23T20:58:44.072736+0000","last_clean":"2023-09-23T20:58:44.072736+0000","last_became_active":"2023-09-23T20:55:35.948410+0000","last_became_peered":"2023-09-23T20:55:35.948410+0000","last_unstale":"2023-09-23T20:58:44.072736+0000","last_undegraded":"2023-09-23T20:58:44.072736+0000","last_fullsized":"2023-09-23T20:58:44.072736+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:26:50.477289+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":205,"reported_epoch":237,"state":"active+clean","last_fresh":"2023-09-23T20:58:54.386071+0000","last_change":"2023-09-23T20:51:10.829889+0000","last_active":"2023-09-23T20:58:54.386071+0000","last_peered":"2023-09-23T20:58:54.386071+0000","last_clean":"2023-09-23T20:58:54.386071+0000","last_became_active":"2023-09-23T20:51:10.829533+0000","last_became_peered":"2023-09-23T20:51:10.829533+0000","last_unstale":"2023-09-23T20:58:54.386071+0000","last_undegraded":"2023-09-23T20:58:54.386071+0000","last_fullsized":"2023-09-23T20:58:54.386071+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:43:04.603460+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.4","version":"237'8514","reported_seq":8562,"reported_epoch":237,"state":"active+clean","last_fresh":"2023-09-23T20:59:06.454838+0000","last_change":"2023-09-23T20:56:35.549909+0000","last_active":"2023-09-23T20:59:06.454838+0000","last_peered":"2023-09-23T20:59:06.454838+0000","last_clean":"2023-09-23T20:59:06.454838+0000","last_became_active":"2023-09-23T20:56:35.549618+0000","last_became_peered":"2023-09-23T20:56:35.549618+0000","last_unstale":"2023-09-23T20:59:06.454838+0000","last_undegraded":"2023-09-23T20:59:06.454838+0000","last_fullsized":"2023-09-23T20:59:06.454838+0000","mapping_epoch":199,"log_start":"205'1100","ondisk_log_start":"205'1100","created":199,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_clean_scrub_stamp":"2023-09-23T20:56:34.529103+0000","objects_scrubbed":0,"log_size":7414,"log_dups_size":0,"ondisk_log_size":7414,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:24:25.918475+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":275775488,"num_objects":4208,"num_object_clones":0,"num_object_copies":12624,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4208,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8514,"num_write_kb":407104,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.2","version":"79'161","reported_seq":467,"reported_epoch":236,"state":"active+clean","last_fresh":"2023-09-23T20:58:51.439439+0000","last_change":"2023-09-23T20:57:24.802586+0000","last_active":"2023-09-23T20:58:51.439439+0000","last_peered":"2023-09-23T20:58:51.439439+0000","last_clean":"2023-09-23T20:58:51.439439+0000","last_became_active":"2023-09-23T20:57:08.236114+0000","last_became_peered":"2023-09-23T20:57:08.236114+0000","last_unstale":"2023-09-23T20:58:51.439439+0000","last_undegraded":"2023-09-23T20:58:51.439439+0000","last_fullsized":"2023-09-23T20:58:51.439439+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:24.802451+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:24.802451+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:06:22.309980+0000","scrub_duration":0.0090861250000000005,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,4],"acting":[0,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":223,"reported_epoch":237,"state":"active+clean","last_fresh":"2023-09-23T20:58:54.388802+0000","last_change":"2023-09-23T20:51:10.838054+0000","last_active":"2023-09-23T20:58:54.388802+0000","last_peered":"2023-09-23T20:58:54.388802+0000","last_clean":"2023-09-23T20:58:54.388802+0000","last_became_active":"2023-09-23T20:51:10.837750+0000","last_became_peered":"2023-09-23T20:51:10.837750+0000","last_unstale":"2023-09-23T20:58:54.388802+0000","last_undegraded":"2023-09-23T20:58:54.388802+0000","last_fullsized":"2023-09-23T20:58:54.388802+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:15:51.648085+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":205,"reported_epoch":237,"state":"active+clean","last_fresh":"2023-09-23T20:58:54.383096+0000","last_change":"2023-09-23T20:55:35.951146+0000","last_active":"2023-09-23T20:58:54.383096+0000","last_peered":"2023-09-23T20:58:54.383096+0000","last_clean":"2023-09-23T20:58:54.383096+0000","last_became_active":"2023-09-23T20:55:35.950874+0000","last_became_peered":"2023-09-23T20:55:35.950874+0000","last_unstale":"2023-09-23T20:58:54.383096+0000","last_undegraded":"2023-09-23T20:58:54.383096+0000","last_fullsized":"2023-09-23T20:58:54.383096+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:58:13.957677+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":226,"reported_epoch":237,"state":"active+clean","last_fresh":"2023-09-23T20:58:54.386652+0000","last_change":"2023-09-23T20:54:31.898870+0000","last_active":"2023-09-23T20:58:54.386652+0000","last_peered":"2023-09-23T20:58:54.386652+0000","last_clean":"2023-09-23T20:58:54.386652+0000","last_became_active":"2023-09-23T20:54:31.898467+0000","last_became_peered":"2023-09-23T20:54:31.898467+0000","last_unstale":"2023-09-23T20:58:54.386652+0000","last_undegraded":"2023-09-23T20:58:54.386652+0000","last_fullsized":"2023-09-23T20:58:54.386652+0000","mapping_epoch":136,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:56:22.483408+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.6","version":"237'8458","reported_seq":11995,"reported_epoch":237,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-09-23T20:59:06.492811+0000","last_change":"2023-09-23T20:58:39.453197+0000","last_active":"2023-09-23T20:59:06.492811+0000","last_peered":"2023-09-23T20:59:06.492811+0000","last_clean":"2023-09-23T20:57:44.031597+0000","last_became_active":"2023-09-23T20:57:46.443927+0000","last_became_peered":"2023-09-23T20:57:46.443927+0000","last_unstale":"2023-09-23T20:59:06.492811+0000","last_undegraded":"2023-09-23T20:58:39.450464+0000","last_fullsized":"2023-09-23T20:57:45.977227+0000","mapping_epoch":223,"log_start":"205'1100","ondisk_log_start":"205'1100","created":199,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_clean_scrub_stamp":"2023-09-23T20:56:34.529103+0000","objects_scrubbed":0,"log_size":7358,"log_dups_size":0,"ondisk_log_size":7358,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:37:23.911131+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":270663680,"num_objects":4130,"num_object_clones":0,"num_object_copies":12390,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":425,"num_objects_misplaced":3233,"num_objects_unfound":0,"num_objects_dirty":4130,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8458,"num_write_kb":402816,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1740,"num_bytes_recovered":114032640,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7],"avail_no_missing":["0","5","7"],"object_location_counts":[{"shards":"0,5,7","objects":3233},{"shards":"0,7","objects":472}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":110,"reported_epoch":236,"state":"active+clean","last_fresh":"2023-09-23T20:58:44.066917+0000","last_change":"2023-09-23T20:54:31.883504+0000","last_active":"2023-09-23T20:58:44.066917+0000","last_peered":"2023-09-23T20:58:44.066917+0000","last_clean":"2023-09-23T20:58:44.066917+0000","last_became_active":"2023-09-23T20:54:31.883172+0000","last_became_peered":"2023-09-23T20:54:31.883172+0000","last_unstale":"2023-09-23T20:58:44.066917+0000","last_undegraded":"2023-09-23T20:58:44.066917+0000","last_fullsized":"2023-09-23T20:58:44.066917+0000","mapping_epoch":136,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:45:40.289120+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"79'161","reported_seq":467,"reported_epoch":236,"state":"active+clean","last_fresh":"2023-09-23T20:58:46.428020+0000","last_change":"2023-09-23T20:57:20.738417+0000","last_active":"2023-09-23T20:58:46.428020+0000","last_peered":"2023-09-23T20:58:46.428020+0000","last_clean":"2023-09-23T20:58:46.428020+0000","last_became_active":"2023-09-23T20:57:08.236842+0000","last_became_peered":"2023-09-23T20:57:08.236842+0000","last_unstale":"2023-09-23T20:58:46.428020+0000","last_undegraded":"2023-09-23T20:58:46.428020+0000","last_fullsized":"2023-09-23T20:58:46.428020+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":0,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:20.738273+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:20.738273+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:18:28.327990+0000","scrub_duration":0.0088336339999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":215,"reported_epoch":237,"state":"active+clean","last_fresh":"2023-09-23T20:58:54.380793+0000","last_change":"2023-09-23T20:55:35.950777+0000","last_active":"2023-09-23T20:58:54.380793+0000","last_peered":"2023-09-23T20:58:54.380793+0000","last_clean":"2023-09-23T20:58:54.380793+0000","last_became_active":"2023-09-23T20:55:35.950427+0000","last_became_peered":"2023-09-23T20:55:35.950427+0000","last_unstale":"2023-09-23T20:58:54.380793+0000","last_undegraded":"2023-09-23T20:58:54.380793+0000","last_fullsized":"2023-09-23T20:58:54.380793+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:20:10.371751+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":182,"reported_epoch":237,"state":"active+clean","last_fresh":"2023-09-23T20:58:54.372144+0000","last_change":"2023-09-23T20:55:35.978140+0000","last_active":"2023-09-23T20:58:54.372144+0000","last_peered":"2023-09-23T20:58:54.372144+0000","last_clean":"2023-09-23T20:58:54.372144+0000","last_became_active":"2023-09-23T20:55:35.977831+0000","last_became_peered":"2023-09-23T20:55:35.977831+0000","last_unstale":"2023-09-23T20:58:54.372144+0000","last_undegraded":"2023-09-23T20:58:54.372144+0000","last_fullsized":"2023-09-23T20:58:54.372144+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:25:47.678023+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.5","version":"237'8471","reported_seq":8526,"reported_epoch":237,"state":"active+clean","last_fresh":"2023-09-23T20:59:06.431044+0000","last_change":"2023-09-23T20:56:45.990110+0000","last_active":"2023-09-23T20:59:06.431044+0000","last_peered":"2023-09-23T20:59:06.431044+0000","last_clean":"2023-09-23T20:59:06.431044+0000","last_became_active":"2023-09-23T20:56:45.988394+0000","last_became_peered":"2023-09-23T20:56:45.988394+0000","last_unstale":"2023-09-23T20:59:06.431044+0000","last_undegraded":"2023-09-23T20:59:06.431044+0000","last_fullsized":"2023-09-23T20:59:06.431044+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":199,"last_epoch_clean":205,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_clean_scrub_stamp":"2023-09-23T20:56:34.529103+0000","objects_scrubbed":0,"log_size":8471,"log_dups_size":0,"ondisk_log_size":8471,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:22:03.814210+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":275709952,"num_objects":4207,"num_object_clones":0,"num_object_copies":12621,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4207,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8471,"num_write_kb":405696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,0],"acting":[2,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"79'161","reported_seq":504,"reported_epoch":237,"state":"active+clean","last_fresh":"2023-09-23T20:58:54.386449+0000","last_change":"2023-09-23T20:57:20.378531+0000","last_active":"2023-09-23T20:58:54.386449+0000","last_peered":"2023-09-23T20:58:54.386449+0000","last_clean":"2023-09-23T20:58:54.386449+0000","last_became_active":"2023-09-23T20:57:08.238019+0000","last_became_peered":"2023-09-23T20:57:08.238019+0000","last_unstale":"2023-09-23T20:58:54.386449+0000","last_undegraded":"2023-09-23T20:58:54.386449+0000","last_fullsized":"2023-09-23T20:58:54.386449+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":3,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:20.377714+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:20.377714+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:00:08.305602+0000","scrub_duration":0.003015228,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":107,"reported_epoch":237,"state":"active+clean","last_fresh":"2023-09-23T20:58:54.383212+0000","last_change":"2023-09-23T20:56:45.991884+0000","last_active":"2023-09-23T20:58:54.383212+0000","last_peered":"2023-09-23T20:58:54.383212+0000","last_clean":"2023-09-23T20:58:54.383212+0000","last_became_active":"2023-09-23T20:56:45.991568+0000","last_became_peered":"2023-09-23T20:56:45.991568+0000","last_unstale":"2023-09-23T20:58:54.383212+0000","last_undegraded":"2023-09-23T20:58:54.383212+0000","last_fullsized":"2023-09-23T20:58:54.383212+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":205,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:21:20.775511+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,0],"acting":[1,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":230,"reported_epoch":236,"state":"active+clean","last_fresh":"2023-09-23T20:58:44.064975+0000","last_change":"2023-09-23T20:57:44.979950+0000","last_active":"2023-09-23T20:58:44.064975+0000","last_peered":"2023-09-23T20:58:44.064975+0000","last_clean":"2023-09-23T20:58:44.064975+0000","last_became_active":"2023-09-23T20:57:44.979346+0000","last_became_peered":"2023-09-23T20:57:44.979346+0000","last_unstale":"2023-09-23T20:58:44.064975+0000","last_undegraded":"2023-09-23T20:58:44.064975+0000","last_fullsized":"2023-09-23T20:58:44.064975+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:10:24.372800+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":209,"reported_epoch":236,"state":"active+clean","last_fresh":"2023-09-23T20:58:44.073890+0000","last_change":"2023-09-23T20:50:37.982235+0000","last_active":"2023-09-23T20:58:44.073890+0000","last_peered":"2023-09-23T20:58:44.073890+0000","last_clean":"2023-09-23T20:58:44.073890+0000","last_became_active":"2023-09-23T20:50:37.981913+0000","last_became_peered":"2023-09-23T20:50:37.981913+0000","last_unstale":"2023-09-23T20:58:44.073890+0000","last_undegraded":"2023-09-23T20:58:44.073890+0000","last_fullsized":"2023-09-23T20:58:44.073890+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:59:57.122584+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.0","version":"237'8530","reported_seq":8576,"reported_epoch":237,"state":"active+clean","last_fresh":"2023-09-23T20:59:07.118202+0000","last_change":"2023-09-23T20:56:35.550635+0000","last_active":"2023-09-23T20:59:07.118202+0000","last_peered":"2023-09-23T20:59:07.118202+0000","last_clean":"2023-09-23T20:59:07.118202+0000","last_became_active":"2023-09-23T20:56:35.550376+0000","last_became_peered":"2023-09-23T20:56:35.550376+0000","last_unstale":"2023-09-23T20:59:07.118202+0000","last_undegraded":"2023-09-23T20:59:07.118202+0000","last_fullsized":"2023-09-23T20:59:07.118202+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":199,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_clean_scrub_stamp":"2023-09-23T20:56:34.529103+0000","objects_scrubbed":0,"log_size":8530,"log_dups_size":0,"ondisk_log_size":8530,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:40:55.020049+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":272105472,"num_objects":4152,"num_object_clones":0,"num_object_copies":12456,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4152,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8530,"num_write_kb":405824,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.6","version":"79'161","reported_seq":520,"reported_epoch":237,"state":"active+clean","last_fresh":"2023-09-23T20:58:54.371906+0000","last_change":"2023-09-23T20:57:26.907505+0000","last_active":"2023-09-23T20:58:54.371906+0000","last_peered":"2023-09-23T20:58:54.371906+0000","last_clean":"2023-09-23T20:58:54.371906+0000","last_became_active":"2023-09-23T20:57:08.240321+0000","last_became_peered":"2023-09-23T20:57:08.240321+0000","last_unstale":"2023-09-23T20:58:54.371906+0000","last_undegraded":"2023-09-23T20:58:54.371906+0000","last_fullsized":"2023-09-23T20:58:54.371906+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":3,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:26.907408+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:26.907408+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:34:17.392928+0000","scrub_duration":0.010337038,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":153,"reported_epoch":236,"state":"active+clean","last_fresh":"2023-09-23T20:58:51.439554+0000","last_change":"2023-09-23T20:56:45.988566+0000","last_active":"2023-09-23T20:58:51.439554+0000","last_peered":"2023-09-23T20:58:51.439554+0000","last_clean":"2023-09-23T20:58:51.439554+0000","last_became_active":"2023-09-23T20:56:45.988249+0000","last_became_peered":"2023-09-23T20:56:45.988249+0000","last_unstale":"2023-09-23T20:58:51.439554+0000","last_undegraded":"2023-09-23T20:58:51.439554+0000","last_fullsized":"2023-09-23T20:58:51.439554+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":205,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:51:36.644764+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,4],"acting":[0,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":208,"reported_epoch":237,"state":"active+clean","last_fresh":"2023-09-23T20:58:54.386193+0000","last_change":"2023-09-23T20:54:31.898965+0000","last_active":"2023-09-23T20:58:54.386193+0000","last_peered":"2023-09-23T20:58:54.386193+0000","last_clean":"2023-09-23T20:58:54.386193+0000","last_became_active":"2023-09-23T20:54:31.898663+0000","last_became_peered":"2023-09-23T20:54:31.898663+0000","last_unstale":"2023-09-23T20:58:54.386193+0000","last_undegraded":"2023-09-23T20:58:54.386193+0000","last_fullsized":"2023-09-23T20:58:54.386193+0000","mapping_epoch":136,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:53:05.058087+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":197,"reported_epoch":236,"state":"active+clean","last_fresh":"2023-09-23T20:58:44.067946+0000","last_change":"2023-09-23T20:54:31.884717+0000","last_active":"2023-09-23T20:58:44.067946+0000","last_peered":"2023-09-23T20:58:44.067946+0000","last_clean":"2023-09-23T20:58:44.067946+0000","last_became_active":"2023-09-23T20:54:31.884345+0000","last_became_peered":"2023-09-23T20:54:31.884345+0000","last_unstale":"2023-09-23T20:58:44.067946+0000","last_undegraded":"2023-09-23T20:58:44.067946+0000","last_fullsized":"2023-09-23T20:58:44.067946+0000","mapping_epoch":136,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:02:26.393868+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.3","version":"238'8463","reported_seq":8512,"reported_epoch":238,"state":"active+clean","last_fresh":"2023-09-23T20:59:09.430764+0000","last_change":"2023-09-23T20:56:35.549655+0000","last_active":"2023-09-23T20:59:09.430764+0000","last_peered":"2023-09-23T20:59:09.430764+0000","last_clean":"2023-09-23T20:59:09.430764+0000","last_became_active":"2023-09-23T20:56:35.549357+0000","last_became_peered":"2023-09-23T20:56:35.549357+0000","last_unstale":"2023-09-23T20:59:09.430764+0000","last_undegraded":"2023-09-23T20:59:09.430764+0000","last_fullsized":"2023-09-23T20:59:09.430764+0000","mapping_epoch":199,"log_start":"206'2300","ondisk_log_start":"206'2300","created":199,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_clean_scrub_stamp":"2023-09-23T20:56:34.529103+0000","objects_scrubbed":0,"log_size":6163,"log_dups_size":0,"ondisk_log_size":6163,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:37:58.752518+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":266010624,"num_objects":4059,"num_object_clones":0,"num_object_copies":12177,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4059,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8463,"num_write_kb":400704,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.5","version":"79'161","reported_seq":500,"reported_epoch":236,"state":"active+clean","last_fresh":"2023-09-23T20:58:44.072902+0000","last_change":"2023-09-23T20:57:30.029479+0000","last_active":"2023-09-23T20:58:44.072902+0000","last_peered":"2023-09-23T20:58:44.072902+0000","last_clean":"2023-09-23T20:58:44.072902+0000","last_became_active":"2023-09-23T20:57:08.249909+0000","last_became_peered":"2023-09-23T20:57:08.249909+0000","last_unstale":"2023-09-23T20:58:44.072902+0000","last_undegraded":"2023-09-23T20:58:44.072902+0000","last_fullsized":"2023-09-23T20:58:44.072902+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":3,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:30.029396+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:30.029396+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:16:46.083145+0000","scrub_duration":0.012914682,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":201,"reported_epoch":237,"state":"active+clean","last_fresh":"2023-09-23T20:58:54.379948+0000","last_change":"2023-09-23T20:51:10.836910+0000","last_active":"2023-09-23T20:58:54.379948+0000","last_peered":"2023-09-23T20:58:54.379948+0000","last_clean":"2023-09-23T20:58:54.379948+0000","last_became_active":"2023-09-23T20:51:10.836557+0000","last_became_peered":"2023-09-23T20:51:10.836557+0000","last_unstale":"2023-09-23T20:58:54.379948+0000","last_undegraded":"2023-09-23T20:58:54.379948+0000","last_fullsized":"2023-09-23T20:58:54.379948+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:51:54.019969+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":208,"reported_epoch":237,"state":"active+clean","last_fresh":"2023-09-23T20:58:54.386077+0000","last_change":"2023-09-23T20:55:35.953444+0000","last_active":"2023-09-23T20:58:54.386077+0000","last_peered":"2023-09-23T20:58:54.386077+0000","last_clean":"2023-09-23T20:58:54.386077+0000","last_became_active":"2023-09-23T20:55:35.953151+0000","last_became_peered":"2023-09-23T20:55:35.953151+0000","last_unstale":"2023-09-23T20:58:54.386077+0000","last_undegraded":"2023-09-23T20:58:54.386077+0000","last_fullsized":"2023-09-23T20:58:54.386077+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:39:12.404391+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":26,"reported_epoch":237,"state":"active+clean","last_fresh":"2023-09-23T20:58:54.379086+0000","last_change":"2023-09-23T20:57:44.982509+0000","last_active":"2023-09-23T20:58:54.379086+0000","last_peered":"2023-09-23T20:58:54.379086+0000","last_clean":"2023-09-23T20:58:54.379086+0000","last_became_active":"2023-09-23T20:57:44.982175+0000","last_became_peered":"2023-09-23T20:57:44.982175+0000","last_unstale":"2023-09-23T20:58:54.379086+0000","last_undegraded":"2023-09-23T20:58:54.379086+0000","last_fullsized":"2023-09-23T20:58:54.379086+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:37:49.215488+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.2","version":"238'8544","reported_seq":8593,"reported_epoch":238,"state":"active+clean","last_fresh":"2023-09-23T20:59:09.516720+0000","last_change":"2023-09-23T20:56:35.543020+0000","last_active":"2023-09-23T20:59:09.516720+0000","last_peered":"2023-09-23T20:59:09.516720+0000","last_clean":"2023-09-23T20:59:09.516720+0000","last_became_active":"2023-09-23T20:56:35.542719+0000","last_became_peered":"2023-09-23T20:56:35.542719+0000","last_unstale":"2023-09-23T20:59:09.516720+0000","last_undegraded":"2023-09-23T20:59:09.516720+0000","last_fullsized":"2023-09-23T20:59:09.516720+0000","mapping_epoch":199,"log_start":"206'2400","ondisk_log_start":"206'2400","created":199,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_clean_scrub_stamp":"2023-09-23T20:56:34.529103+0000","objects_scrubbed":0,"log_size":6144,"log_dups_size":0,"ondisk_log_size":6144,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:52:21.088628+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":260046848,"num_objects":3968,"num_object_clones":0,"num_object_copies":11904,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3968,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8544,"num_write_kb":400384,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.4","version":"79'161","reported_seq":516,"reported_epoch":237,"state":"active+clean","last_fresh":"2023-09-23T20:58:54.379386+0000","last_change":"2023-09-23T20:57:20.012904+0000","last_active":"2023-09-23T20:58:54.379386+0000","last_peered":"2023-09-23T20:58:54.379386+0000","last_clean":"2023-09-23T20:58:54.379386+0000","last_became_active":"2023-09-23T20:57:08.237996+0000","last_became_peered":"2023-09-23T20:57:08.237996+0000","last_unstale":"2023-09-23T20:58:54.379386+0000","last_undegraded":"2023-09-23T20:58:54.379386+0000","last_fullsized":"2023-09-23T20:58:54.379386+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":3,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:20.012764+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:20.012764+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:51:28.796946+0000","scrub_duration":0.0045251919999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":187,"reported_epoch":237,"state":"active+clean","last_fresh":"2023-09-23T20:58:54.371999+0000","last_change":"2023-09-23T20:55:35.949541+0000","last_active":"2023-09-23T20:58:54.371999+0000","last_peered":"2023-09-23T20:58:54.371999+0000","last_clean":"2023-09-23T20:58:54.371999+0000","last_became_active":"2023-09-23T20:55:35.944635+0000","last_became_peered":"2023-09-23T20:55:35.944635+0000","last_unstale":"2023-09-23T20:58:54.371999+0000","last_undegraded":"2023-09-23T20:58:54.371999+0000","last_fullsized":"2023-09-23T20:58:54.371999+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:30:36.033585+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"70'1","reported_seq":225,"reported_epoch":236,"state":"active+clean","last_fresh":"2023-09-23T20:58:44.073990+0000","last_change":"2023-09-23T20:54:14.009723+0000","last_active":"2023-09-23T20:58:44.073990+0000","last_peered":"2023-09-23T20:58:44.073990+0000","last_clean":"2023-09-23T20:58:44.073990+0000","last_became_active":"2023-09-23T20:54:13.941771+0000","last_became_peered":"2023-09-23T20:54:13.941771+0000","last_unstale":"2023-09-23T20:58:44.073990+0000","last_undegraded":"2023-09-23T20:58:44.073990+0000","last_fullsized":"2023-09-23T20:58:44.073990+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:48:25.588116+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":141,"reported_epoch":236,"state":"active+clean","last_fresh":"2023-09-23T20:58:44.074980+0000","last_change":"2023-09-23T20:55:35.949780+0000","last_active":"2023-09-23T20:58:44.074980+0000","last_peered":"2023-09-23T20:58:44.074980+0000","last_clean":"2023-09-23T20:58:44.074980+0000","last_became_active":"2023-09-23T20:55:35.949497+0000","last_became_peered":"2023-09-23T20:55:35.949497+0000","last_unstale":"2023-09-23T20:58:44.074980+0000","last_undegraded":"2023-09-23T20:58:44.074980+0000","last_fullsized":"2023-09-23T20:58:44.074980+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:09:37.420882+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":199,"reported_epoch":236,"state":"active+clean","last_fresh":"2023-09-23T20:58:44.067816+0000","last_change":"2023-09-23T20:54:13.929196+0000","last_active":"2023-09-23T20:58:44.067816+0000","last_peered":"2023-09-23T20:58:44.067816+0000","last_clean":"2023-09-23T20:58:44.067816+0000","last_became_active":"2023-09-23T20:54:13.928831+0000","last_became_peered":"2023-09-23T20:54:13.928831+0000","last_unstale":"2023-09-23T20:58:44.067816+0000","last_undegraded":"2023-09-23T20:58:44.067816+0000","last_fullsized":"2023-09-23T20:58:44.067816+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:28:44.316851+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":205,"reported_epoch":236,"state":"active+clean","last_fresh":"2023-09-23T20:58:44.063073+0000","last_change":"2023-09-23T20:52:12.855430+0000","last_active":"2023-09-23T20:58:44.063073+0000","last_peered":"2023-09-23T20:58:44.063073+0000","last_clean":"2023-09-23T20:58:44.063073+0000","last_became_active":"2023-09-23T20:52:12.855115+0000","last_became_peered":"2023-09-23T20:52:12.855115+0000","last_unstale":"2023-09-23T20:58:44.063073+0000","last_undegraded":"2023-09-23T20:58:44.063073+0000","last_fullsized":"2023-09-23T20:58:44.063073+0000","mapping_epoch":100,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":101,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:33:23.802214+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":114,"reported_epoch":236,"state":"active+clean","last_fresh":"2023-09-23T20:58:44.066902+0000","last_change":"2023-09-23T20:54:31.902946+0000","last_active":"2023-09-23T20:58:44.066902+0000","last_peered":"2023-09-23T20:58:44.066902+0000","last_clean":"2023-09-23T20:58:44.066902+0000","last_became_active":"2023-09-23T20:54:31.889484+0000","last_became_peered":"2023-09-23T20:54:31.889484+0000","last_unstale":"2023-09-23T20:58:44.066902+0000","last_undegraded":"2023-09-23T20:58:44.066902+0000","last_fullsized":"2023-09-23T20:58:44.066902+0000","mapping_epoch":136,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:44:44.278398+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":114,"reported_epoch":236,"state":"active+clean","last_fresh":"2023-09-23T20:58:44.072812+0000","last_change":"2023-09-23T20:54:13.950351+0000","last_active":"2023-09-23T20:58:44.072812+0000","last_peered":"2023-09-23T20:58:44.072812+0000","last_clean":"2023-09-23T20:58:44.072812+0000","last_became_active":"2023-09-23T20:54:13.949987+0000","last_became_peered":"2023-09-23T20:54:13.949987+0000","last_unstale":"2023-09-23T20:58:44.072812+0000","last_undegraded":"2023-09-23T20:58:44.072812+0000","last_fullsized":"2023-09-23T20:58:44.072812+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:46:17.765134+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":145,"reported_epoch":236,"state":"active+clean","last_fresh":"2023-09-23T20:58:44.066853+0000","last_change":"2023-09-23T20:56:18.141728+0000","last_active":"2023-09-23T20:58:44.066853+0000","last_peered":"2023-09-23T20:58:44.066853+0000","last_clean":"2023-09-23T20:58:44.066853+0000","last_became_active":"2023-09-23T20:56:18.137979+0000","last_became_peered":"2023-09-23T20:56:18.137979+0000","last_unstale":"2023-09-23T20:58:44.066853+0000","last_undegraded":"2023-09-23T20:58:44.066853+0000","last_fullsized":"2023-09-23T20:58:44.066853+0000","mapping_epoch":192,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":193,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:58:05.658443+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":114,"reported_epoch":236,"state":"active+clean","last_fresh":"2023-09-23T20:58:44.072583+0000","last_change":"2023-09-23T20:54:13.935307+0000","last_active":"2023-09-23T20:58:44.072583+0000","last_peered":"2023-09-23T20:58:44.072583+0000","last_clean":"2023-09-23T20:58:44.072583+0000","last_became_active":"2023-09-23T20:54:13.935000+0000","last_became_peered":"2023-09-23T20:54:13.935000+0000","last_unstale":"2023-09-23T20:58:44.072583+0000","last_undegraded":"2023-09-23T20:58:44.072583+0000","last_fullsized":"2023-09-23T20:58:44.072583+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:47:40.974082+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":225,"reported_epoch":238,"state":"active+clean","last_fresh":"2023-09-23T20:59:08.297977+0000","last_change":"2023-09-23T20:55:35.954870+0000","last_active":"2023-09-23T20:59:08.297977+0000","last_peered":"2023-09-23T20:59:08.297977+0000","last_clean":"2023-09-23T20:59:08.297977+0000","last_became_active":"2023-09-23T20:55:35.954568+0000","last_became_peered":"2023-09-23T20:55:35.954568+0000","last_unstale":"2023-09-23T20:59:08.297977+0000","last_undegraded":"2023-09-23T20:59:08.297977+0000","last_fullsized":"2023-09-23T20:59:08.297977+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:06:22.423566+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":224,"reported_epoch":237,"state":"active+clean","last_fresh":"2023-09-23T20:58:54.378646+0000","last_change":"2023-09-23T20:51:10.838334+0000","last_active":"2023-09-23T20:58:54.378646+0000","last_peered":"2023-09-23T20:58:54.378646+0000","last_clean":"2023-09-23T20:58:54.378646+0000","last_became_active":"2023-09-23T20:51:10.838026+0000","last_became_peered":"2023-09-23T20:51:10.838026+0000","last_unstale":"2023-09-23T20:58:54.378646+0000","last_undegraded":"2023-09-23T20:58:54.378646+0000","last_fullsized":"2023-09-23T20:58:54.378646+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:04:37.009457+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":227,"reported_epoch":236,"state":"active+clean","last_fresh":"2023-09-23T20:58:44.068405+0000","last_change":"2023-09-23T20:54:31.903139+0000","last_active":"2023-09-23T20:58:44.068405+0000","last_peered":"2023-09-23T20:58:44.068405+0000","last_clean":"2023-09-23T20:58:44.068405+0000","last_became_active":"2023-09-23T20:54:31.884520+0000","last_became_peered":"2023-09-23T20:54:31.884520+0000","last_unstale":"2023-09-23T20:58:44.068405+0000","last_undegraded":"2023-09-23T20:58:44.068405+0000","last_fullsized":"2023-09-23T20:58:44.068405+0000","mapping_epoch":136,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:39:34.269705+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1108,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":131,"reported_epoch":236,"state":"active+clean","last_fresh":"2023-09-23T20:58:44.072302+0000","last_change":"2023-09-23T20:55:35.951572+0000","last_active":"2023-09-23T20:58:44.072302+0000","last_peered":"2023-09-23T20:58:44.072302+0000","last_clean":"2023-09-23T20:58:44.072302+0000","last_became_active":"2023-09-23T20:55:35.951294+0000","last_became_peered":"2023-09-23T20:55:35.951294+0000","last_unstale":"2023-09-23T20:58:44.072302+0000","last_undegraded":"2023-09-23T20:58:44.072302+0000","last_fullsized":"2023-09-23T20:58:44.072302+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:36:32.447210+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":7,"num_pg":16,"stat_sum":{"num_bytes":4340776984,"num_objects":66236,"num_object_clones":0,"num_object_copies":198708,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":11368,"num_objects_misplaced":3233,"num_objects_unfound":0,"num_objects_dirty":66236,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":137284,"num_write_kb":6512577,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1741,"num_bytes_recovered":114098176,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":13457436672,"data_stored":13457424456,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":118084,"ondisk_log_size":118084,"up":48,"acting":45,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":22,"num_bytes_recovered":5990,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":21,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3381,"ondisk_log_size":3381,"up":63,"acting":63,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201504,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1525856,"kb_used_data":1344088,"kb_used_omap":0,"kb_used_meta":181760,"kb_avail":92219296,"statfs":{"total":95995035648,"available":94432559104,"internally_reserved":0,"allocated":1376346112,"data_stored":1375506483,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":186122240},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:58:44 2023","interfaces":[{"interface":"back","average":{"1min":2.052,"5min":1.859,"15min":1.006},"min":{"1min":0.248,"5min":0.21099999999999999,"15min":0.21099999999999999},"max":{"1min":5.5279999999999996,"5min":43.994999999999997,"15min":43.994999999999997},"last":4.7359999999999998},{"interface":"front","average":{"1min":2.0489999999999999,"5min":2.0760000000000001,"15min":1.099},"min":{"1min":0.34599999999999997,"5min":0.28199999999999997,"15min":0.218},"max":{"1min":5.5129999999999999,"5min":44.206000000000003,"15min":44.206000000000003},"last":4.7229999999999999}]},{"osd":1,"last update":"Sat Sep 23 20:58:18 2023","interfaces":[{"interface":"back","average":{"1min":1.895,"5min":1.9330000000000001,"15min":1.1439999999999999},"min":{"1min":0.29999999999999999,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":10.622,"5min":10.622,"15min":22.591999999999999},"last":0.26600000000000001},{"interface":"front","average":{"1min":1.8320000000000001,"5min":1.5720000000000001,"15min":0.98499999999999999},"min":{"1min":0.25800000000000001,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":7.5629999999999997,"5min":10.097,"15min":21.437000000000001},"last":3.645}]},{"osd":2,"last update":"Sat Sep 23 20:58:54 2023","interfaces":[{"interface":"back","average":{"1min":3.7370000000000001,"5min":2.016,"15min":1.1499999999999999},"min":{"1min":0.27600000000000002,"5min":0.26500000000000001,"15min":0.24399999999999999},"max":{"1min":10.805999999999999,"5min":10.805999999999999,"15min":10.805999999999999},"last":8.5289999999999999},{"interface":"front","average":{"1min":2.3969999999999998,"5min":2.1520000000000001,"15min":1.1919999999999999},"min":{"1min":0.27800000000000002,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":9.4290000000000003,"5min":45.25,"15min":45.25},"last":3.6099999999999999}]},{"osd":3,"last update":"Sat Sep 23 20:58:26 2023","interfaces":[{"interface":"back","average":{"1min":3.355,"5min":3.226,"15min":1.587},"min":{"1min":0.32100000000000001,"5min":0.314,"15min":0.314},"max":{"1min":9.9939999999999998,"5min":36.408000000000001,"15min":36.408000000000001},"last":0.63900000000000001},{"interface":"front","average":{"1min":2.5649999999999999,"5min":2.7160000000000002,"15min":1.405},"min":{"1min":0.32000000000000001,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":9.4529999999999994,"5min":35.593000000000004,"15min":35.593000000000004},"last":3.4649999999999999}]},{"osd":4,"last update":"Sat Sep 23 20:58:18 2023","interfaces":[{"interface":"back","average":{"1min":0.69999999999999996,"5min":0.82699999999999996,"15min":0.60899999999999999},"min":{"1min":0.28299999999999997,"5min":0.23799999999999999,"15min":0.23699999999999999},"max":{"1min":4.5410000000000004,"5min":5.8920000000000003,"15min":5.8920000000000003},"last":0.38800000000000001},{"interface":"front","average":{"1min":0.70399999999999996,"5min":0.76100000000000001,"15min":0.61399999999999999},"min":{"1min":0.27100000000000002,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":4.4080000000000004,"5min":5.3200000000000003,"15min":5.3200000000000003},"last":0.44}]},{"osd":5,"last update":"Sat Sep 23 20:58:18 2023","interfaces":[{"interface":"back","average":{"1min":0.69299999999999995,"5min":0.82799999999999996,"15min":0.63300000000000001},"min":{"1min":0.28499999999999998,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":2.5110000000000001,"5min":7.0389999999999997,"15min":7.0389999999999997},"last":0.316},{"interface":"front","average":{"1min":0.72199999999999998,"5min":0.70699999999999996,"15min":0.61199999999999999},"min":{"1min":0.317,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":4.5999999999999996,"5min":4.5999999999999996,"15min":4.5999999999999996},"last":0.42399999999999999}]},{"osd":6,"last update":"Sat Sep 23 20:58:40 2023","interfaces":[{"interface":"back","average":{"1min":0.86599999999999999,"5min":0.71699999999999997,"15min":0.68300000000000005},"min":{"1min":0.33700000000000002,"5min":0.31,"15min":0.31},"max":{"1min":3.5249999999999999,"5min":3.5249999999999999,"15min":3.5249999999999999},"last":0.40799999999999997},{"interface":"front","average":{"1min":0.90100000000000002,"5min":0.81299999999999994,"15min":0.80700000000000005},"min":{"1min":0.36899999999999999,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":3.5409999999999999,"5min":3.5409999999999999,"15min":3.5409999999999999},"last":0.376}]}]},{"osd":6,"up_from":164,"seq":704374636588,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1431328,"kb_used_data":1323608,"kb_used_omap":10,"kb_used_meta":107701,"kb_avail":92313824,"statfs":{"total":95995035648,"available":94529355776,"internally_reserved":0,"allocated":1355374592,"data_stored":1354542851,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":10328,"internal_metadata":110286760},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:58:43 2023","interfaces":[{"interface":"back","average":{"1min":2.7330000000000001,"5min":1.4410000000000001,"15min":1.0569999999999999},"min":{"1min":0.25800000000000001,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":7.5179999999999998,"5min":8.7319999999999993,"15min":8.7319999999999993},"last":0.23599999999999999},{"interface":"front","average":{"1min":3.1709999999999998,"5min":1.502,"15min":1.1559999999999999},"min":{"1min":0.25,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":6.7140000000000004,"5min":6.7140000000000004,"15min":6.7140000000000004},"last":2.214}]},{"osd":1,"last update":"Sat Sep 23 20:58:43 2023","interfaces":[{"interface":"back","average":{"1min":1.5449999999999999,"5min":1.097,"15min":0.89100000000000001},"min":{"1min":0.36799999999999999,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":5.2350000000000003,"5min":6.6029999999999998,"15min":6.6029999999999998},"last":0.55900000000000005},{"interface":"front","average":{"1min":2.5779999999999998,"5min":1.244,"15min":0.99299999999999999},"min":{"1min":0.23000000000000001,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":10.324999999999999,"5min":10.324999999999999,"15min":10.324999999999999},"last":0.32100000000000001}]},{"osd":2,"last update":"Sat Sep 23 20:58:43 2023","interfaces":[{"interface":"back","average":{"1min":2.286,"5min":1.982,"15min":1.758},"min":{"1min":0.28999999999999998,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":7.0519999999999996,"5min":13.750999999999999,"15min":13.750999999999999},"last":0.61699999999999999},{"interface":"front","average":{"1min":2.419,"5min":1.5289999999999999,"15min":1.2470000000000001},"min":{"1min":0.26200000000000001,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":8.0890000000000004,"5min":8.0890000000000004,"15min":8.0890000000000004},"last":1.9730000000000001}]},{"osd":3,"last update":"Sat Sep 23 20:58:43 2023","interfaces":[{"interface":"back","average":{"1min":1.8680000000000001,"5min":1.2849999999999999,"15min":1.173},"min":{"1min":0.32000000000000001,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":6.9000000000000004,"5min":6.9000000000000004,"15min":6.9000000000000004},"last":2.6560000000000001},{"interface":"front","average":{"1min":2.1819999999999999,"5min":1.74,"15min":1.6479999999999999},"min":{"1min":0.28599999999999998,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":11.021000000000001,"5min":11.021000000000001,"15min":11.021000000000001},"last":0.58499999999999996}]},{"osd":4,"last update":"Sat Sep 23 20:58:43 2023","interfaces":[{"interface":"back","average":{"1min":0.49099999999999999,"5min":0.85899999999999999,"15min":0.98699999999999999},"min":{"1min":0.23000000000000001,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":2.7919999999999998,"5min":9.0079999999999991,"15min":9.0079999999999991},"last":0.29099999999999998},{"interface":"front","average":{"1min":0.49299999999999999,"5min":0.50800000000000001,"15min":0.52300000000000002},"min":{"1min":0.27200000000000002,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":2.0059999999999998,"5min":2.0059999999999998,"15min":2.0059999999999998},"last":0.35899999999999999}]},{"osd":5,"last update":"Sat Sep 23 20:58:43 2023","interfaces":[{"interface":"back","average":{"1min":0.53700000000000003,"5min":0.56899999999999995,"15min":0.55000000000000004},"min":{"1min":0.27300000000000002,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":2.8239999999999998,"5min":2.8239999999999998,"15min":2.8239999999999998},"last":0.50800000000000001},{"interface":"front","average":{"1min":0.81000000000000005,"5min":0.78500000000000003,"15min":0.80100000000000005},"min":{"1min":0.33400000000000002,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":3.1379999999999999,"5min":3.1379999999999999,"15min":3.1379999999999999},"last":0.39200000000000002}]},{"osd":7,"last update":"Sat Sep 23 20:58:43 2023","interfaces":[{"interface":"back","average":{"1min":0.52700000000000002,"5min":0.85399999999999998,"15min":0.90400000000000003},"min":{"1min":0.312,"5min":0.312,"15min":0.312},"max":{"1min":2.073,"5min":7.8570000000000002,"15min":7.8570000000000002},"last":0.41899999999999998},{"interface":"front","average":{"1min":0.61899999999999999,"5min":0.66200000000000003,"15min":0.57399999999999995},"min":{"1min":0.30299999999999999,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":2.6989999999999998,"5min":8.282,"15min":8.282},"last":0.52600000000000002}]}]},{"osd":1,"up_from":88,"seq":377957122159,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2158616,"kb_used_data":1927440,"kb_used_omap":9,"kb_used_meta":231030,"kb_avail":91586536,"statfs":{"total":95995035648,"available":93784612864,"internally_reserved":0,"allocated":1973698560,"data_stored":1972527371,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9239,"internal_metadata":236575721},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,0,1,2],"upper_bound":32},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:58:22 2023","interfaces":[{"interface":"back","average":{"1min":3.3290000000000002,"5min":2.9990000000000001,"15min":1.3049999999999999},"min":{"1min":0.23200000000000001,"5min":0.216,"15min":0.216},"max":{"1min":10.73,"5min":16.646999999999998,"15min":16.646999999999998},"last":3.4729999999999999},{"interface":"front","average":{"1min":4.2149999999999999,"5min":3.3940000000000001,"15min":1.4590000000000001},"min":{"1min":0.247,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":15.135999999999999,"5min":18.457999999999998,"15min":18.457999999999998},"last":2.508}]},{"osd":2,"last update":"Sat Sep 23 20:58:53 2023","interfaces":[{"interface":"back","average":{"1min":2.7130000000000001,"5min":2.4140000000000001,"15min":1.1799999999999999},"min":{"1min":0.24299999999999999,"5min":0.221,"15min":0.20300000000000001},"max":{"1min":8.6829999999999998,"5min":14.335000000000001,"15min":14.335000000000001},"last":3.4510000000000001},{"interface":"front","average":{"1min":5.8019999999999996,"5min":3.27,"15min":1.758},"min":{"1min":0.39500000000000002,"5min":0.248,"15min":0.248},"max":{"1min":16.048999999999999,"5min":16.048999999999999,"15min":38.951000000000001},"last":2.536}]},{"osd":3,"last update":"Sat Sep 23 20:58:22 2023","interfaces":[{"interface":"back","average":{"1min":3.1859999999999999,"5min":3.0489999999999999,"15min":1.3720000000000001},"min":{"1min":0.246,"5min":0.246,"15min":0.246},"max":{"1min":10.701000000000001,"5min":25.314,"15min":25.314},"last":5.9199999999999999},{"interface":"front","average":{"1min":2.8450000000000002,"5min":2.6150000000000002,"15min":1.232},"min":{"1min":0.33800000000000002,"5min":0.20599999999999999,"15min":0.20599999999999999},"max":{"1min":7.8899999999999997,"5min":19.881,"15min":19.881},"last":5.9459999999999997}]},{"osd":4,"last update":"Sat Sep 23 20:58:22 2023","interfaces":[{"interface":"back","average":{"1min":1.7050000000000001,"5min":1.9550000000000001,"15min":1.131},"min":{"1min":0.36099999999999999,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":10.598000000000001,"5min":12.006,"15min":12.006},"last":1.383},{"interface":"front","average":{"1min":2.0899999999999999,"5min":1.6839999999999999,"15min":1.016},"min":{"1min":0.31900000000000001,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":6.0460000000000003,"5min":11.234999999999999,"15min":11.234999999999999},"last":3.3599999999999999}]},{"osd":5,"last update":"Sat Sep 23 20:58:22 2023","interfaces":[{"interface":"back","average":{"1min":3.0609999999999999,"5min":2.133,"15min":1.181},"min":{"1min":0.34399999999999997,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":15.041,"5min":15.041,"15min":15.041},"last":2.0310000000000001},{"interface":"front","average":{"1min":1.669,"5min":2.0169999999999999,"15min":1.153},"min":{"1min":0.33200000000000002,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":10.647,"5min":12.08,"15min":12.08},"last":1.357}]},{"osd":6,"last update":"Sat Sep 23 20:58:39 2023","interfaces":[{"interface":"back","average":{"1min":1.6160000000000001,"5min":1.3700000000000001,"15min":1.333},"min":{"1min":0.30299999999999999,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":10.673999999999999,"5min":10.673999999999999,"15min":10.673999999999999},"last":1.319},{"interface":"front","average":{"1min":1.8280000000000001,"5min":1.508,"15min":1.3380000000000001},"min":{"1min":0.38100000000000001,"5min":0.309,"15min":0.309},"max":{"1min":8.7530000000000001,"5min":8.7530000000000001,"15min":8.7530000000000001},"last":3.4279999999999999}]},{"osd":7,"last update":"Sat Sep 23 20:58:22 2023","interfaces":[{"interface":"back","average":{"1min":2.0659999999999998,"5min":1.7909999999999999,"15min":1.069},"min":{"1min":0.42499999999999999,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":6.149,"5min":11.186999999999999,"15min":11.186999999999999},"last":3.399},{"interface":"front","average":{"1min":3.1520000000000001,"5min":2.1829999999999998,"15min":1.2010000000000001},"min":{"1min":0.41899999999999998,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":15.098000000000001,"5min":15.098000000000001,"15min":15.098000000000001},"last":1.994}]}]},{"osd":0,"up_from":83,"seq":356482285685,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2329840,"kb_used_data":2157928,"kb_used_omap":28,"kb_used_meta":171875,"kb_avail":91415312,"statfs":{"total":95995035648,"available":93609279488,"internally_reserved":0,"allocated":2209718272,"data_stored":2208577012,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":28848,"internal_metadata":176000848},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,2,0,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 20:58:19 2023","interfaces":[{"interface":"back","average":{"1min":3.7799999999999998,"5min":2.9220000000000002,"15min":1.2609999999999999},"min":{"1min":0.25800000000000001,"5min":0.191,"15min":0.191},"max":{"1min":15.946999999999999,"5min":18.863,"15min":18.863},"last":2.1419999999999999},{"interface":"front","average":{"1min":3.9060000000000001,"5min":2.8540000000000001,"15min":1.2669999999999999},"min":{"1min":0.247,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":15.766999999999999,"5min":17.704999999999998,"15min":17.704999999999998},"last":11.135999999999999}]},{"osd":2,"last update":"Sat Sep 23 20:58:50 2023","interfaces":[{"interface":"back","average":{"1min":4.843,"5min":3.2549999999999999,"15min":1.6359999999999999},"min":{"1min":0.434,"5min":0.24299999999999999,"15min":0.23799999999999999},"max":{"1min":14.942,"5min":14.976000000000001,"15min":17.753},"last":1.8759999999999999},{"interface":"front","average":{"1min":7.774,"5min":3.8180000000000001,"15min":1.726},"min":{"1min":0.38500000000000001,"5min":0.25800000000000001,"15min":0.21099999999999999},"max":{"1min":21.747,"5min":21.747,"15min":21.747},"last":8.0180000000000007}]},{"osd":3,"last update":"Sat Sep 23 20:58:19 2023","interfaces":[{"interface":"back","average":{"1min":3.722,"5min":2.8919999999999999,"15min":1.298},"min":{"1min":0.39300000000000002,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":15.994,"5min":18.893999999999998,"15min":18.893999999999998},"last":6.2110000000000003},{"interface":"front","average":{"1min":3.7930000000000001,"5min":3.0649999999999999,"15min":1.423},"min":{"1min":0.29699999999999999,"5min":0.28000000000000003,"15min":0.23300000000000001},"max":{"1min":15.808,"5min":17.472000000000001,"15min":17.472000000000001},"last":1.6819999999999999}]},{"osd":4,"last update":"Sat Sep 23 20:58:50 2023","interfaces":[{"interface":"back","average":{"1min":6.6399999999999997,"5min":2.9279999999999999,"15min":1.4730000000000001},"min":{"1min":0.36599999999999999,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":21.594999999999999,"5min":21.594999999999999,"15min":21.594999999999999},"last":7.931},{"interface":"front","average":{"1min":2.657,"5min":1.9790000000000001,"15min":1.131},"min":{"1min":0.35199999999999998,"5min":0.31,"15min":0.28100000000000003},"max":{"1min":10.393000000000001,"5min":18.724,"15min":18.724},"last":1.714}]},{"osd":5,"last update":"Sat Sep 23 20:58:50 2023","interfaces":[{"interface":"back","average":{"1min":2.2719999999999998,"5min":1.8069999999999999,"15min":1.177},"min":{"1min":0.34999999999999998,"5min":0.318,"15min":0.318},"max":{"1min":8.8970000000000002,"5min":8.8970000000000002,"15min":8.8970000000000002},"last":1.629},{"interface":"front","average":{"1min":5.7110000000000003,"5min":2.8919999999999999,"15min":1.4950000000000001},"min":{"1min":0.40600000000000003,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":20.512,"5min":20.512,"15min":20.512},"last":7.9820000000000002}]},{"osd":6,"last update":"Sat Sep 23 20:58:44 2023","interfaces":[{"interface":"back","average":{"1min":2.681,"5min":1.893,"15min":1.7350000000000001},"min":{"1min":0.39900000000000002,"5min":0.37,"15min":0.37},"max":{"1min":10.420999999999999,"5min":10.420999999999999,"15min":10.420999999999999},"last":1.7709999999999999},{"interface":"front","average":{"1min":2.7120000000000002,"5min":1.9650000000000001,"15min":1.819},"min":{"1min":0.42199999999999999,"5min":0.377,"15min":0.377},"max":{"1min":10.331,"5min":10.331,"15min":10.331},"last":1.79}]},{"osd":7,"last update":"Sat Sep 23 20:58:50 2023","interfaces":[{"interface":"back","average":{"1min":2.4729999999999999,"5min":1.875,"15min":1.22},"min":{"1min":0.38100000000000001,"5min":0.379,"15min":0.379},"max":{"1min":8.7729999999999997,"5min":8.7729999999999997,"15min":8.7729999999999997},"last":1.6579999999999999},{"interface":"front","average":{"1min":2.7010000000000001,"5min":2.032,"15min":1.1619999999999999},"min":{"1min":0.377,"5min":0.377,"15min":0.377},"max":{"1min":10.359999999999999,"5min":18.673999999999999,"15min":18.673999999999999},"last":1.734}]}]},{"osd":2,"up_from":93,"seq":399431958632,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2257504,"kb_used_data":1961944,"kb_used_omap":81,"kb_used_meta":295406,"kb_avail":91487648,"statfs":{"total":95995035648,"available":93683351552,"internally_reserved":0,"allocated":2009030656,"data_stored":2007902144,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":83750,"internal_metadata":302495962},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:58:47 2023","interfaces":[{"interface":"back","average":{"1min":3.1429999999999998,"5min":2.0670000000000002,"15min":1.079},"min":{"1min":0.22700000000000001,"5min":0.17999999999999999,"15min":0.17999999999999999},"max":{"1min":14.773999999999999,"5min":14.773999999999999,"15min":14.773999999999999},"last":3.8180000000000001},{"interface":"front","average":{"1min":5.1459999999999999,"5min":3.133,"15min":1.4319999999999999},"min":{"1min":0.32400000000000001,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":17.045000000000002,"5min":17.045000000000002,"15min":17.045000000000002},"last":3.9780000000000002}]},{"osd":1,"last update":"Sat Sep 23 20:58:47 2023","interfaces":[{"interface":"back","average":{"1min":4.1470000000000002,"5min":2.718,"15min":1.48},"min":{"1min":0.29299999999999998,"5min":0.20200000000000001,"15min":0.20200000000000001},"max":{"1min":14.709,"5min":14.709,"15min":16.081},"last":3.7610000000000001},{"interface":"front","average":{"1min":5.3819999999999997,"5min":3.6389999999999998,"15min":1.77},"min":{"1min":0.41499999999999998,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":11.492000000000001,"5min":12.07,"15min":16.041},"last":0.57899999999999996}]},{"osd":3,"last update":"Sat Sep 23 20:58:26 2023","interfaces":[{"interface":"back","average":{"1min":5.1120000000000001,"5min":3.4620000000000002,"15min":1.585},"min":{"1min":0.23300000000000001,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":23.309999999999999,"5min":23.309999999999999,"15min":23.309999999999999},"last":2.351},{"interface":"front","average":{"1min":5.5659999999999998,"5min":4.21,"15min":1.8240000000000001},"min":{"1min":0.34300000000000003,"5min":0.22800000000000001,"15min":0.221},"max":{"1min":23.324999999999999,"5min":23.324999999999999,"15min":23.324999999999999},"last":2.7629999999999999}]},{"osd":4,"last update":"Sat Sep 23 20:58:47 2023","interfaces":[{"interface":"back","average":{"1min":2.2519999999999998,"5min":1.9910000000000001,"15min":1.1439999999999999},"min":{"1min":0.28699999999999998,"5min":0.28699999999999998,"15min":0.26400000000000001},"max":{"1min":8.4580000000000002,"5min":9.8119999999999994,"15min":9.8119999999999994},"last":0.48899999999999999},{"interface":"front","average":{"1min":2.137,"5min":2.1560000000000001,"15min":1.2270000000000001},"min":{"1min":0.32400000000000001,"5min":0.313,"15min":0.313},"max":{"1min":8.1649999999999991,"5min":10.609,"15min":10.609},"last":2.8479999999999999}]},{"osd":5,"last update":"Sat Sep 23 20:58:47 2023","interfaces":[{"interface":"back","average":{"1min":2.1779999999999999,"5min":2.238,"15min":1.2470000000000001},"min":{"1min":0.32300000000000001,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":8.5,"5min":9.7629999999999999,"15min":9.7629999999999999},"last":0.51000000000000001},{"interface":"front","average":{"1min":2.0670000000000002,"5min":2.1709999999999998,"15min":1.242},"min":{"1min":0.36399999999999999,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":8.1170000000000009,"5min":10.574,"15min":10.574},"last":2.802}]},{"osd":6,"last update":"Sat Sep 23 20:58:42 2023","interfaces":[{"interface":"back","average":{"1min":2.1360000000000001,"5min":1.8169999999999999,"15min":1.7170000000000001},"min":{"1min":0.379,"5min":0.312,"15min":0.312},"max":{"1min":8.2270000000000003,"5min":14.244999999999999,"15min":14.244999999999999},"last":2.827},{"interface":"front","average":{"1min":2.1949999999999998,"5min":1.444,"15min":1.1679999999999999},"min":{"1min":0.41199999999999998,"5min":0.374,"15min":0.374},"max":{"1min":8.1980000000000004,"5min":8.1980000000000004,"15min":8.1980000000000004},"last":2.8719999999999999}]},{"osd":7,"last update":"Sat Sep 23 20:58:47 2023","interfaces":[{"interface":"back","average":{"1min":1.4830000000000001,"5min":1.4339999999999999,"15min":0.995},"min":{"1min":0.315,"5min":0.315,"15min":0.315},"max":{"1min":5.8570000000000002,"5min":8.0820000000000007,"15min":8.0820000000000007},"last":2.6970000000000001},{"interface":"front","average":{"1min":2.2090000000000001,"5min":2.2109999999999999,"15min":1.262},"min":{"1min":0.35099999999999998,"5min":0.35099999999999998,"15min":0.34499999999999997},"max":{"1min":8.5250000000000004,"5min":9.7859999999999996,"15min":9.7859999999999996},"last":0.45800000000000002}]}]},{"osd":3,"up_from":98,"seq":420906795106,"num_pgs":49,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2074832,"kb_used_data":1739080,"kb_used_omap":7,"kb_used_meta":335736,"kb_avail":91670320,"statfs":{"total":95995035648,"available":93870407680,"internally_reserved":0,"allocated":1780817920,"data_stored":1779710561,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7576,"internal_metadata":343794280},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,0,1,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:58:25 2023","interfaces":[{"interface":"back","average":{"1min":3.097,"5min":3.02,"15min":1.4890000000000001},"min":{"1min":0.308,"5min":0.20599999999999999,"15min":0.20599999999999999},"max":{"1min":8.5549999999999997,"5min":23.123999999999999,"15min":23.123999999999999},"last":5.6950000000000003},{"interface":"front","average":{"1min":2.9900000000000002,"5min":2.7869999999999999,"15min":1.3009999999999999},"min":{"1min":0.19800000000000001,"5min":0.19800000000000001,"15min":0.19800000000000001},"max":{"1min":9.8889999999999993,"5min":15.079000000000001,"15min":15.079000000000001},"last":5.085}]},{"osd":1,"last update":"Sat Sep 23 20:58:25 2023","interfaces":[{"interface":"back","average":{"1min":3.7829999999999999,"5min":3.1509999999999998,"15min":1.4390000000000001},"min":{"1min":0.32400000000000001,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":13.173,"5min":21.385000000000002,"15min":21.385000000000002},"last":0.46899999999999997},{"interface":"front","average":{"1min":3.0880000000000001,"5min":3.242,"15min":1.5109999999999999},"min":{"1min":0.28799999999999998,"5min":0.218,"15min":0.218},"max":{"1min":10.683999999999999,"5min":25.353000000000002,"15min":25.353000000000002},"last":0.376}]},{"osd":2,"last update":"Sat Sep 23 20:58:25 2023","interfaces":[{"interface":"back","average":{"1min":4.851,"5min":3.052,"15min":1.444},"min":{"1min":0.35899999999999999,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":15.476000000000001,"5min":24.154,"15min":24.154},"last":0.42399999999999999},{"interface":"front","average":{"1min":2.8519999999999999,"5min":3.335,"15min":1.577},"min":{"1min":0.23999999999999999,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":8.6069999999999993,"5min":25.305,"15min":25.305},"last":0.48399999999999999}]},{"osd":4,"last update":"Sat Sep 23 20:58:25 2023","interfaces":[{"interface":"back","average":{"1min":2.6539999999999999,"5min":1.8240000000000001,"15min":1.141},"min":{"1min":0.38900000000000001,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":9.6170000000000009,"5min":11.861000000000001,"15min":11.861000000000001},"last":4.1799999999999997},{"interface":"front","average":{"1min":1.3600000000000001,"5min":2.0880000000000001,"15min":1.246},"min":{"1min":0.313,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":4.8109999999999999,"5min":22.853000000000002,"15min":22.853000000000002},"last":0.45400000000000001}]},{"osd":5,"last update":"Sat Sep 23 20:58:25 2023","interfaces":[{"interface":"back","average":{"1min":2.9540000000000002,"5min":1.8999999999999999,"15min":1.1459999999999999},"min":{"1min":0.436,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":9.6430000000000007,"5min":11.891999999999999,"15min":11.891999999999999},"last":0.64600000000000002},{"interface":"front","average":{"1min":2.04,"5min":1.6579999999999999,"15min":1.075},"min":{"1min":0.33300000000000002,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":5.7690000000000001,"5min":9.2870000000000008,"15min":9.2870000000000008},"last":5.0449999999999999}]},{"osd":6,"last update":"Sat Sep 23 20:58:43 2023","interfaces":[{"interface":"back","average":{"1min":1.9450000000000001,"5min":1.1930000000000001,"15min":1.0309999999999999},"min":{"1min":0.33500000000000002,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":10.018000000000001,"5min":10.018000000000001,"15min":10.018000000000001},"last":0.51300000000000001},{"interface":"front","average":{"1min":1.4770000000000001,"5min":1.48,"15min":1.4339999999999999},"min":{"1min":0.38700000000000001,"5min":0.38700000000000001,"15min":0.38700000000000001},"max":{"1min":5.3239999999999998,"5min":7.234,"15min":7.234},"last":5.0090000000000003}]},{"osd":7,"last update":"Sat Sep 23 20:58:25 2023","interfaces":[{"interface":"back","average":{"1min":3.0099999999999998,"5min":2.3399999999999999,"15min":1.3220000000000001},"min":{"1min":0.46500000000000002,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":9.5749999999999993,"5min":16.364999999999998,"15min":16.364999999999998},"last":0.66500000000000004},{"interface":"front","average":{"1min":2.214,"5min":1.766,"15min":1.147},"min":{"1min":0.42499999999999999,"5min":0.379,"15min":0.379},"max":{"1min":5.8090000000000002,"5min":9.3179999999999996,"15min":9.3179999999999996},"last":5.6550000000000002}]}]},{"osd":4,"up_from":31,"seq":133143986413,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1470496,"kb_used_data":1336216,"kb_used_omap":0,"kb_used_meta":134272,"kb_avail":92274656,"statfs":{"total":95995035648,"available":94489247744,"internally_reserved":0,"allocated":1368285184,"data_stored":1367445808,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":137494528},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:58:53 2023","interfaces":[{"interface":"back","average":{"1min":1.917,"5min":1.7170000000000001,"15min":1.042},"min":{"1min":0.245,"5min":0.214,"15min":0.214},"max":{"1min":8.8439999999999994,"5min":11.784000000000001,"15min":11.784000000000001},"last":9.0749999999999993},{"interface":"front","average":{"1min":1.8149999999999999,"5min":1.7889999999999999,"15min":1.23},"min":{"1min":0.314,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":8.8300000000000001,"5min":11.766,"15min":11.766},"last":9.1029999999999998}]},{"osd":1,"last update":"Sat Sep 23 20:58:21 2023","interfaces":[{"interface":"back","average":{"1min":1.7370000000000001,"5min":1.641,"15min":0.996},"min":{"1min":0.246,"5min":0.246,"15min":0.23599999999999999},"max":{"1min":10.153,"5min":10.153,"15min":10.153},"last":3.0369999999999999},{"interface":"front","average":{"1min":2.645,"5min":1.649,"15min":0.96399999999999997},"min":{"1min":0.26100000000000001,"5min":0.26100000000000001,"15min":0.23200000000000001},"max":{"1min":12.292999999999999,"5min":12.292999999999999,"15min":12.292999999999999},"last":5.0469999999999997}]},{"osd":2,"last update":"Sat Sep 23 20:58:53 2023","interfaces":[{"interface":"back","average":{"1min":1.8700000000000001,"5min":1.764,"15min":1.115},"min":{"1min":0.34300000000000003,"5min":0.311,"15min":0.29099999999999998},"max":{"1min":7.3150000000000004,"5min":15.288,"15min":15.288},"last":4.9459999999999997},{"interface":"front","average":{"1min":1.633,"5min":1.974,"15min":1.2629999999999999},"min":{"1min":0.32600000000000001,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":8.7959999999999994,"5min":15.262,"15min":15.262},"last":0.49199999999999999}]},{"osd":3,"last update":"Sat Sep 23 20:58:21 2023","interfaces":[{"interface":"back","average":{"1min":2.0209999999999999,"5min":2.504,"15min":1.514},"min":{"1min":0.28899999999999998,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":7.7549999999999999,"5min":15.916,"15min":31.562999999999999},"last":2.0670000000000002},{"interface":"front","average":{"1min":2.2080000000000002,"5min":2.5449999999999999,"15min":1.589},"min":{"1min":0.433,"5min":0.433,"15min":0.41899999999999998},"max":{"1min":7.7300000000000004,"5min":15.882,"15min":31.574000000000002},"last":2.0790000000000002}]},{"osd":5,"last update":"Sat Sep 23 20:58:35 2023","interfaces":[{"interface":"back","average":{"1min":0.81399999999999995,"5min":0.61399999999999999,"15min":0.52900000000000003},"min":{"1min":0.29199999999999998,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":2.9470000000000001,"5min":3.3279999999999998,"15min":3.3279999999999998},"last":0.309},{"interface":"front","average":{"1min":0.77900000000000003,"5min":0.68500000000000005,"15min":0.57799999999999996},"min":{"1min":0.33200000000000002,"5min":0.251,"15min":0.251},"max":{"1min":1.9350000000000001,"5min":7.7619999999999996,"15min":7.7619999999999996},"last":1.46}]},{"osd":6,"last update":"Sat Sep 23 20:58:41 2023","interfaces":[{"interface":"back","average":{"1min":0.90300000000000002,"5min":0.67700000000000005,"15min":0.54800000000000004},"min":{"1min":0.29899999999999999,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":3.4780000000000002,"5min":4.6909999999999998,"15min":4.6909999999999998},"last":0.42899999999999999},{"interface":"front","average":{"1min":0.82299999999999995,"5min":0.71599999999999997,"15min":0.66800000000000004},"min":{"1min":0.36799999999999999,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":3.456,"5min":3.456,"15min":3.456},"last":0.41799999999999998}]},{"osd":7,"last update":"Sat Sep 23 20:58:20 2023","interfaces":[{"interface":"back","average":{"1min":0.84199999999999997,"5min":0.83099999999999996,"15min":0.67100000000000004},"min":{"1min":0.442,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":3.327,"5min":7.383,"15min":7.383},"last":0.38900000000000001},{"interface":"front","average":{"1min":0.93899999999999995,"5min":0.80800000000000005,"15min":0.66000000000000003},"min":{"1min":0.36399999999999999,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":3.883,"5min":7.8310000000000004,"15min":7.8310000000000004},"last":0.40300000000000002}]}]},{"osd":5,"up_from":39,"seq":167503724778,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1553172,"kb_used_data":1426316,"kb_used_omap":0,"kb_used_meta":126848,"kb_avail":92191980,"statfs":{"total":95995035648,"available":94404587520,"internally_reserved":0,"allocated":1460547584,"data_stored":1459716784,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":129892352},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:58:37 2023","interfaces":[{"interface":"back","average":{"1min":0.78600000000000003,"5min":1.03,"15min":0.72699999999999998},"min":{"1min":0.23899999999999999,"5min":0.20399999999999999,"15min":0.20399999999999999},"max":{"1min":3.6949999999999998,"5min":7.766,"15min":7.766},"last":0.34799999999999998},{"interface":"front","average":{"1min":1.8049999999999999,"5min":1.6779999999999999,"15min":0.98099999999999998},"min":{"1min":0.25,"5min":0.223,"15min":0.223},"max":{"1min":7.8120000000000003,"5min":15.037000000000001,"15min":15.037000000000001},"last":0.36199999999999999}]},{"osd":1,"last update":"Sat Sep 23 20:58:16 2023","interfaces":[{"interface":"back","average":{"1min":1.5549999999999999,"5min":1.603,"15min":0.95199999999999996},"min":{"1min":0.27100000000000002,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":6.7409999999999997,"5min":8.3520000000000003,"15min":8.3520000000000003},"last":8.1829999999999998},{"interface":"front","average":{"1min":1.9590000000000001,"5min":1.7450000000000001,"15min":0.97699999999999998},"min":{"1min":0.28100000000000003,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":7.4669999999999996,"5min":10.285,"15min":10.285},"last":0.436}]},{"osd":2,"last update":"Sat Sep 23 20:58:52 2023","interfaces":[{"interface":"back","average":{"1min":1.9299999999999999,"5min":1.6000000000000001,"15min":0.97799999999999998},"min":{"1min":0.311,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":6,"5min":12.651,"15min":12.651},"last":1.3540000000000001},{"interface":"front","average":{"1min":3.077,"5min":1.8440000000000001,"15min":1.085},"min":{"1min":0.27100000000000002,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":12.300000000000001,"5min":12.300000000000001,"15min":12.300000000000001},"last":3.4910000000000001}]},{"osd":3,"last update":"Sat Sep 23 20:58:16 2023","interfaces":[{"interface":"back","average":{"1min":2.331,"5min":2.0670000000000002,"15min":1.153},"min":{"1min":0.39200000000000002,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":8.6579999999999995,"5min":9.5489999999999995,"15min":9.5489999999999995},"last":0.48799999999999999},{"interface":"front","average":{"1min":1.6000000000000001,"5min":1.6040000000000001,"15min":0.97499999999999998},"min":{"1min":0.28899999999999998,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":10.457000000000001,"5min":10.457000000000001,"15min":10.457000000000001},"last":0.42599999999999999}]},{"osd":4,"last update":"Sat Sep 23 20:58:33 2023","interfaces":[{"interface":"back","average":{"1min":0.48799999999999999,"5min":0.59099999999999997,"15min":0.52900000000000003},"min":{"1min":0.252,"5min":0.252,"15min":0.252},"max":{"1min":2.6909999999999998,"5min":3.8479999999999999,"15min":3.8479999999999999},"last":0.32100000000000001},{"interface":"front","average":{"1min":0.41899999999999998,"5min":0.64100000000000001,"15min":0.54900000000000004},"min":{"1min":0.29299999999999998,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":0.747,"5min":3.0680000000000001,"15min":3.0680000000000001},"last":0.38500000000000001}]},{"osd":6,"last update":"Sat Sep 23 20:58:37 2023","interfaces":[{"interface":"back","average":{"1min":0.46600000000000003,"5min":0.59099999999999997,"15min":0.57099999999999995},"min":{"1min":0.28499999999999998,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":1.0509999999999999,"5min":2.9820000000000002,"15min":2.9820000000000002},"last":0.376},{"interface":"front","average":{"1min":0.499,"5min":0.69899999999999995,"15min":0.72899999999999998},"min":{"1min":0.36299999999999999,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":0.78500000000000003,"5min":3.6560000000000001,"15min":3.6560000000000001},"last":0.39400000000000002}]},{"osd":7,"last update":"Sat Sep 23 20:58:13 2023","interfaces":[{"interface":"back","average":{"1min":0.66500000000000004,"5min":0.76600000000000001,"15min":0.63200000000000001},"min":{"1min":0.316,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":2.4809999999999999,"5min":8.5649999999999995,"15min":8.5649999999999995},"last":0.33400000000000002},{"interface":"front","average":{"1min":0.60299999999999998,"5min":0.74099999999999999,"15min":0.64400000000000002},"min":{"1min":0.35399999999999998,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":2.7109999999999999,"5min":4.5430000000000001,"15min":4.5430000000000001},"last":0.45800000000000002}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":2193559552,"data_stored":2193555480,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1958088704,"data_stored":1958084632,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":2002976768,"data_stored":2002976768,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1774977024,"data_stored":1774977024,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1365442560,"data_stored":1365442560,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1449459712,"data_stored":1449459712,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1346899968,"data_stored":1346895896,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1366032384,"data_stored":1366032384,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T20:59:12.064 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:59:12.097 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph status --format=json 2023-09-23T20:59:12.097 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0: {} 2023-09-23T20:59:12.125 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1: {} 2023-09-23T20:59:12.135 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:59:12.152 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:59:12.152 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:59:12.169 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2: {} 2023-09-23T20:59:12.202 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3: {} 2023-09-23T20:59:12.244 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4: {} 2023-09-23T20:59:12.253 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:59:12.278 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T20:59:12.341 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6: {} 2023-09-23T20:59:12.380 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T20:59:12.387 INFO:teuthology.orchestra.run.smithi026.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T20:59:12.387 INFO:teuthology.orchestra.run.smithi026.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T20:59:12.387 INFO:teuthology.orchestra.run.smithi026.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T20:59:12.387 INFO:teuthology.orchestra.run.smithi026.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T20:59:12.387 INFO:teuthology.orchestra.run.smithi026.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T20:59:12.387 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T20:59:12.387 INFO:teuthology.orchestra.run.smithi026.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T20:59:12.388 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T20:59:12.412 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:59:12.601 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:59:12.614 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:59:12.614 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:59:12.615 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:59:12 smithi026 ceph-mon[175151]: from='client.45284 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:59:12.615 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:59:12 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/1932747092' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:59:12.616 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:59:12 smithi026 ceph-mon[179688]: from='client.45284 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:59:12.616 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:59:12 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/1932747092' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:59:12.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:59:12 smithi070 ceph-mon[168282]: from='client.45284 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:59:12.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:59:12 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/1932747092' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:59:12.715 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:59:12.742 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:59:12.785 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:59:12.788 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:59:12.789 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:59:12.889 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:59:12.927 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:59:12.971 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:59:12.973 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:59:12.974 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:59:13.074 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:59:13.104 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:59:13.148 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:59:13.151 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:59:13.151 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:59:13.252 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:59:13.280 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:59:13.335 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:59:13.340 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:59:13.340 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:59:13.441 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:59:13.479 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:59:13.529 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:59:13.532 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:59:13.532 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:59:13.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:59:13 smithi070 ceph-mon[168282]: pgmap v580: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 3.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 577 op/s; 11416/193926 objects degraded (5.887%); 3066/193926 objects misplaced (1.581%); 1.5 MiB/s, 23 objects/s recovering 2023-09-23T20:59:13.633 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:59:13.635 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:59:13 smithi026 ceph-mon[175151]: pgmap v580: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 3.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 577 op/s; 11416/193926 objects degraded (5.887%); 3066/193926 objects misplaced (1.581%); 1.5 MiB/s, 23 objects/s recovering 2023-09-23T20:59:13.635 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:59:13 smithi026 ceph-mon[179688]: pgmap v580: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 3.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 577 op/s; 11416/193926 objects degraded (5.887%); 3066/193926 objects misplaced (1.581%); 1.5 MiB/s, 23 objects/s recovering 2023-09-23T20:59:13.712 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:59:13.776 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:59:13.780 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:59:13.799 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:59:13.800 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:59:13.901 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:59:14.036 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:59:14.065 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:59:14.208 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:59:14.219 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:59:14.220 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:59:14.320 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:59:14.401 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:59:14.539 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:59:14.543 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:59:14.543 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:59:14.644 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:59:14.676 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:59:14.719 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:59:14.722 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:59:14.723 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:59:14.823 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:59:14.939 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:59:15.351 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:59:15.363 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:59:15.364 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:59:15.464 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:59:15.535 INFO:teuthology.orchestra.run.smithi026.stdout:ERROR: (22) Invalid argument 2023-09-23T20:59:15.535 INFO:teuthology.orchestra.run.smithi026.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:59:15.616 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:59:15.640 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:59:15 smithi026 ceph-mon[179688]: pgmap v581: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 3.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 571 op/s; 11416/191802 objects degraded (5.952%); 3066/191802 objects misplaced (1.599%); 1.5 MiB/s, 23 objects/s recovering 2023-09-23T20:59:15.640 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:59:15 smithi026 ceph-mon[179688]: Health check update: Degraded data redundancy: 11416/191802 objects degraded (5.952%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T20:59:15.640 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:59:15 smithi026 ceph-mon[175151]: pgmap v581: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 3.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 571 op/s; 11416/191802 objects degraded (5.952%); 3066/191802 objects misplaced (1.599%); 1.5 MiB/s, 23 objects/s recovering 2023-09-23T20:59:15.640 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:59:15 smithi026 ceph-mon[175151]: Health check update: Degraded data redundancy: 11416/191802 objects degraded (5.952%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T20:59:15.749 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:59:15.756 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:59:15.757 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:59:15.835 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T20:59:15.858 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:59:15.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:59:15 smithi070 ceph-mon[168282]: pgmap v581: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 3.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 571 op/s; 11416/191802 objects degraded (5.952%); 3066/191802 objects misplaced (1.599%); 1.5 MiB/s, 23 objects/s recovering 2023-09-23T20:59:15.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:59:15 smithi070 ceph-mon[168282]: Health check update: Degraded data redundancy: 11416/191802 objects degraded (5.952%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T20:59:15.890 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:59:15.933 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:59:15.935 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:59:15.936 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:59:16.037 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:59:16.139 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_ops_in_flight 2023-09-23T20:59:16.221 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:59:16.350 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:59:16.368 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:59:16.368 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:59:16.469 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:59:16.497 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:59:16.535 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:59:16.537 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:59:16.538 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:59:16.633 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:59:16 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:59:16.633 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:59:16 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:59:16.639 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:59:16.785 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:59:16.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:59:16 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:59:16.917 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:59:16.930 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:59:16.930 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:59:16.946 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:59:17.032 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:59:17.063 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:59:17.100 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:59:17.106 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:59:17.106 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:59:17.161 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:59:17.188 INFO:teuthology.orchestra.run.smithi026.stdout: 2023-09-23T20:59:17.188 INFO:teuthology.orchestra.run.smithi026.stdout:{"fsid":"b3f8e4ec-5a50-11ee-8db4-212e2dc638e7","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"noscrub,nodeep-scrub flag(s) set","count":20},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 11416/190236 objects degraded (6.001%), 3 pgs degraded, 3 pgs undersized","count":6},"muted":false}},"mutes":[]},"election_epoch":30,"quorum":[0,1,2],"quorum_names":["a","b","c"],"quorum_age":675,"monmap":{"epoch":4,"min_mon_release_name":"reef","num_mons":3},"osdmap":{"epoch":238,"num_osds":8,"num_up_osds":8,"osd_up_since":1695502534,"num_in_osds":8,"osd_in_since":1695502470,"num_remapped_pgs":3},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":106},{"state_name":"active+recovery_wait+undersized+degraded+remapped","count":2},{"state_name":"active+recovering+undersized+degraded+remapped","count":1}],"num_pgs":109,"num_pools":5,"num_objects":63412,"data_bytes":4154726328,"bytes_used":14654480384,"bytes_avail":753305804800,"bytes_total":767960285184,"degraded_objects":11416,"degraded_total":190236,"degraded_ratio":0.060009672196639968,"misplaced_objects":3066,"misplaced_total":190236,"misplaced_ratio":0.016116823314199204,"recovering_objects_per_sec":23,"recovering_bytes_per_sec":1545941,"recovering_keys_per_sec":0,"num_objects_recovered":236,"num_bytes_recovered":15466496,"num_keys_recovered":0,"write_bytes_sec":0,"read_op_per_sec":0,"write_op_per_sec":565},"fsmap":{"epoch":8,"id":1,"up":1,"in":1,"max":1,"by_rank":[{"filesystem_id":1,"rank":0,"name":"foo.smithi026.znfpnl","status":"up:active","gid":14658}],"up:standby":1},"mgrmap":{"available":true,"num_standbys":1,"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"services":{"dashboard":"https://172.21.15.26:8443/","prometheus":"http://172.21.15.26:9283/"}},"servicemap":{"epoch":21,"modified":"2023-09-23T20:56:34.314324+0000","services":{}},"progress_events":{"8b64cbfe-d367-468b-a773-0e71ee4ff645":{"message":"Global Recovery Event (0s)\n [............................] ","progress":0,"add_to_ceph_s":true},"fc746e9d-3f76-434d-b6d2-8fc9e8a72a5e":{"message":"Upgrade to 18.2.0-460-gbb0daeff (4m)\n [============................] (remaining: 5m)","progress":0.44999998807907104,"add_to_ceph_s":true}}} 2023-09-23T20:59:17.208 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:59:17.303 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:59:17.470 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:59:17.483 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:59:17.484 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:59:17.578 INFO:teuthology.orchestra.run.smithi026.stderr:noscrub is unset 2023-09-23T20:59:17.585 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:59:17.674 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:59:17.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:59:17 smithi070 ceph-mon[168282]: pgmap v582: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 3.9 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 565 op/s; 11416/190236 objects degraded (6.001%); 3066/190236 objects misplaced (1.612%); 1.5 MiB/s, 23 objects/s recovering 2023-09-23T20:59:17.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:59:17 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/4018040724' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:59:17.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:59:17 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:59:17.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:59:17 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/1313605713' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-23T20:59:17.881 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:59:17 smithi026 ceph-mon[175151]: pgmap v582: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 3.9 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 565 op/s; 11416/190236 objects degraded (6.001%); 3066/190236 objects misplaced (1.612%); 1.5 MiB/s, 23 objects/s recovering 2023-09-23T20:59:17.881 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:59:17 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/4018040724' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:59:17.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:59:17 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:59:17.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:59:17 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/1313605713' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-23T20:59:17.884 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:59:17 smithi026 ceph-mon[179688]: pgmap v582: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 3.9 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 565 op/s; 11416/190236 objects degraded (6.001%); 3066/190236 objects misplaced (1.612%); 1.5 MiB/s, 23 objects/s recovering 2023-09-23T20:59:17.884 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:59:17 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/4018040724' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:59:17.884 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:59:17 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:59:17.884 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:59:17 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/1313605713' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-23T20:59:17.945 INFO:tasks.cephadm.ceph_manager.ceph:making progress, resetting timeout 2023-09-23T20:59:17.946 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T20:59:17.973 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:59:17.984 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:59:17.985 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:59:18.086 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:59:18.112 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:59:18.152 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:59:18.155 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:59:18.155 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:59:18.256 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:59:18.358 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:59:18.582 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:59:18 smithi026 ceph-mon[179688]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:59:18.582 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:59:18 smithi026 ceph-mon[179688]: failed to encode map e239 with expected crc 2023-09-23T20:59:18.582 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:59:18 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T20:59:18.582 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:59:18 smithi026 ceph-mon[179688]: osdmap e239: 8 total, 8 up, 8 in 2023-09-23T20:59:18.583 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:59:18 smithi026 ceph-mon[179688]: failed to encode map e239 with expected crc 2023-09-23T20:59:18.583 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:59:18 smithi026 ceph-mon[179688]: failed to encode map e239 with expected crc 2023-09-23T20:59:18.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:59:18 smithi070 ceph-mon[168282]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:59:18.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:59:18 smithi070 ceph-mon[168282]: failed to encode map e239 with expected crc 2023-09-23T20:59:18.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:59:18 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T20:59:18.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:59:18 smithi070 ceph-mon[168282]: osdmap e239: 8 total, 8 up, 8 in 2023-09-23T20:59:18.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:59:18 smithi070 ceph-mon[168282]: failed to encode map e239 with expected crc 2023-09-23T20:59:18.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:59:18 smithi070 ceph-mon[168282]: failed to encode map e239 with expected crc 2023-09-23T20:59:18.886 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:59:18 smithi026 ceph-mon[175151]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:59:18.886 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:59:18 smithi026 ceph-mon[175151]: failed to encode map e239 with expected crc 2023-09-23T20:59:18.886 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:59:18 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T20:59:18.886 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:59:18 smithi026 ceph-mon[175151]: osdmap e239: 8 total, 8 up, 8 in 2023-09-23T20:59:18.886 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:59:18 smithi026 ceph-mon[175151]: failed to encode map e239 with expected crc 2023-09-23T20:59:18.886 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:59:18 smithi026 ceph-mon[175151]: failed to encode map e239 with expected crc 2023-09-23T20:59:19.341 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:59:19.345 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:59:19.345 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:59:19.446 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:59:19.480 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:59:19.521 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:59:19.524 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:59:19.524 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:59:19.625 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:59:19.655 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 20:59:19 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[153130]: logger=sqlstore.transactions t=2023-09-23T20:59:19.375243704Z level=info msg="Database locked, sleeping then retrying" error="database is locked" retry=0 code="database is locked" 2023-09-23T20:59:19.658 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:59:19 smithi070 ceph-mon[168282]: pgmap v584: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 3.8 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 570 op/s; 11463/185301 objects degraded (6.186%); 2897/185301 objects misplaced (1.563%); 1.5 MiB/s, 24 objects/s recovering 2023-09-23T20:59:19.659 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:59:19.706 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:59:19.711 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:59:19.712 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:59:19.754 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:59:19 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[164029]: ::ffff:172.21.15.70 - - [23/Sep/2023:20:59:19] "GET /metrics HTTP/1.1" 200 36318 "" "Prometheus/2.43.0" 2023-09-23T20:59:19.754 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:59:19 smithi026 ceph-mon[179688]: pgmap v584: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 3.8 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 570 op/s; 11463/185301 objects degraded (6.186%); 2897/185301 objects misplaced (1.563%); 1.5 MiB/s, 24 objects/s recovering 2023-09-23T20:59:19.755 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:59:19 smithi026 ceph-mon[175151]: pgmap v584: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 3.8 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 570 op/s; 11463/185301 objects degraded (6.186%); 2897/185301 objects misplaced (1.563%); 1.5 MiB/s, 24 objects/s recovering 2023-09-23T20:59:19.812 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:59:19.844 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:59:19.886 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:59:19.889 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:59:19.889 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:59:19.990 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:59:20.024 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:59:20.061 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:59:20.064 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:59:20.064 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:59:20.165 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:59:20.248 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:59:20.249 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:59:20.316 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0: {} 2023-09-23T20:59:20.346 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:59:20.365 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:59:20.366 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:59:20.385 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1: {} 2023-09-23T20:59:20.387 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T20:59:20.432 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2: {} 2023-09-23T20:59:20.467 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:59:20.485 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3: {} 2023-09-23T20:59:20.499 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:59:20.535 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:59:20.540 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:59:20.541 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:59:20.543 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4: {} 2023-09-23T20:59:20.577 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T20:59:20.618 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6: {} 2023-09-23T20:59:20.642 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:59:20.651 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T20:59:20.669 INFO:teuthology.orchestra.run.smithi026.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T20:59:20.669 INFO:teuthology.orchestra.run.smithi026.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T20:59:20.669 INFO:teuthology.orchestra.run.smithi026.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T20:59:20.670 INFO:teuthology.orchestra.run.smithi026.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T20:59:20.670 INFO:teuthology.orchestra.run.smithi026.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T20:59:20.670 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T20:59:20.670 INFO:teuthology.orchestra.run.smithi026.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T20:59:20.670 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T20:59:20.670 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:59:20.707 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:59:20.709 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:59:20.710 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:59:20.811 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:59:20.831 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:59:20 smithi026 ceph-mon[179688]: Health check update: Degraded data redundancy: 11463/185301 objects degraded (6.186%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T20:59:20.831 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:59:20 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/588113531' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:59:20.831 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:59:20 smithi026 ceph-mon[175151]: Health check update: Degraded data redundancy: 11463/185301 objects degraded (6.186%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T20:59:20.831 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:59:20 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/588113531' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:59:20.837 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:59:20 smithi070 ceph-mon[168282]: Health check update: Degraded data redundancy: 11463/185301 objects degraded (6.186%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T20:59:20.837 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:59:20 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/588113531' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:59:20.839 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:59:20.878 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:59:20.880 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:59:20.881 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:59:20.982 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:59:21.012 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:59:21.055 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:59:21.059 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:59:21.059 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:59:21.160 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:59:21.189 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:59:21.228 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:59:21.231 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:59:21.232 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:59:21.244 INFO:teuthology.orchestra.run.smithi026.stdout:{ 2023-09-23T20:59:21.244 INFO:teuthology.orchestra.run.smithi026.stdout: "ops": [ 2023-09-23T20:59:21.244 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:59:21.244 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35100.0:144862 7.b 7:de8aca88:::benchmark_data_smithi026_260489_object43100:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e239)", 2023-09-23T20:59:21.244 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T20:59:21.232579+0000", 2023-09-23T20:59:21.244 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 0.0071897999999999997, 2023-09-23T20:59:21.244 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0072276609999999998, 2023-09-23T20:59:21.245 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T20:59:21.245 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "waiting for sub ops", 2023-09-23T20:59:21.245 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T20:59:21.245 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35100", 2023-09-23T20:59:21.245 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/4108972961", 2023-09-23T20:59:21.245 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 144862 2023-09-23T20:59:21.245 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:59:21.245 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T20:59:21.245 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:59:21.245 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T20:59:21.245 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:59:21.232579+0000", 2023-09-23T20:59:21.245 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:59:21.246 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:59:21.246 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:59:21.246 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T20:59:21.246 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:59:21.232579+0000", 2023-09-23T20:59:21.246 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:59:21.246 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:59:21.246 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:59:21.246 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T20:59:21.246 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:59:21.232578+0000", 2023-09-23T20:59:21.246 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T20:59:21.246 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:59:21.246 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:59:21.247 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T20:59:21.247 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:59:21.232585+0000", 2023-09-23T20:59:21.247 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.7970000000000003e-06 2023-09-23T20:59:21.247 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:59:21.247 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:59:21.247 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T20:59:21.247 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:59:21.232601+0000", 2023-09-23T20:59:21.247 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.6259000000000001e-05 2023-09-23T20:59:21.247 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:59:21.247 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:59:21.247 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T20:59:21.247 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:59:21.232611+0000", 2023-09-23T20:59:21.248 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.5480000000000007e-06 2023-09-23T20:59:21.248 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:59:21.248 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:59:21.248 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:59:21.248 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:59:21.232662+0000", 2023-09-23T20:59:21.248 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.1032999999999997e-05 2023-09-23T20:59:21.248 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:59:21.248 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:59:21.248 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T20:59:21.248 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:59:21.232791+0000", 2023-09-23T20:59:21.248 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00012944399999999999 2023-09-23T20:59:21.248 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:59:21.248 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:59:21.249 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 5,6", 2023-09-23T20:59:21.249 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:59:21.232905+0000", 2023-09-23T20:59:21.249 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000114199 2023-09-23T20:59:21.249 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:59:21.249 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:59:21.249 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:59:21.249 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:59:21.239761+0000", 2023-09-23T20:59:21.249 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0068558239999999999 2023-09-23T20:59:21.249 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:59:21.249 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T20:59:21.249 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:59:21.249 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:59:21.250 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:59:21.250 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_repop(client.35100.0:144863 7.d e239/199 7:b91c66be:::benchmark_data_smithi026_260489_object43101:head v 239'9134, mlcod=239'9132)", 2023-09-23T20:59:21.250 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T20:59:21.236821+0000", 2023-09-23T20:59:21.250 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 0.0029486099999999999, 2023-09-23T20:59:21.250 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.003094678, 2023-09-23T20:59:21.250 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T20:59:21.250 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "started", 2023-09-23T20:59:21.250 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T20:59:21.250 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:59:21.250 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T20:59:21.250 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:59:21.236821+0000", 2023-09-23T20:59:21.250 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:59:21.251 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:59:21.251 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:59:21.251 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T20:59:21.251 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:59:21.236821+0000", 2023-09-23T20:59:21.251 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:59:21.251 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:59:21.251 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:59:21.251 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T20:59:21.251 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:59:21.236820+0000", 2023-09-23T20:59:21.251 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.999999 2023-09-23T20:59:21.251 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:59:21.251 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:59:21.251 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T20:59:21.252 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:59:21.236834+0000", 2023-09-23T20:59:21.252 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.4722000000000001e-05 2023-09-23T20:59:21.252 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:59:21.252 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:59:21.252 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T20:59:21.252 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:59:21.236846+0000", 2023-09-23T20:59:21.252 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1868000000000001e-05 2023-09-23T20:59:21.252 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:59:21.252 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:59:21.252 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T20:59:21.252 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:59:21.236855+0000", 2023-09-23T20:59:21.252 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.5620000000000004e-06 2023-09-23T20:59:21.253 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:59:21.253 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:59:21.253 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:59:21.253 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:59:21.237014+0000", 2023-09-23T20:59:21.253 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000159295 2023-09-23T20:59:21.253 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:59:21.253 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:59:21.253 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T20:59:21.253 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:59:21.237037+0000", 2023-09-23T20:59:21.253 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.2855999999999999e-05 2023-09-23T20:59:21.253 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:59:21.253 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T20:59:21.253 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:59:21.254 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:59:21.254 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:59:21.254 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_repop(client.35100.0:144865 7.4 e239/199 7:223372d3:::benchmark_data_smithi026_260489_object43103:head v 239'9043, mlcod=239'9040)", 2023-09-23T20:59:21.254 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T20:59:21.237681+0000", 2023-09-23T20:59:21.254 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 0.002087953, 2023-09-23T20:59:21.254 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.002301352, 2023-09-23T20:59:21.254 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T20:59:21.254 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "started", 2023-09-23T20:59:21.254 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T20:59:21.254 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:59:21.254 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T20:59:21.254 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:59:21.237681+0000", 2023-09-23T20:59:21.255 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:59:21.255 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:59:21.255 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:59:21.255 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T20:59:21.255 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:59:21.237681+0000", 2023-09-23T20:59:21.255 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:59:21.255 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:59:21.255 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:59:21.255 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T20:59:21.255 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:59:21.237681+0000", 2023-09-23T20:59:21.255 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.999999 2023-09-23T20:59:21.255 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:59:21.256 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:59:21.256 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T20:59:21.256 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:59:21.237687+0000", 2023-09-23T20:59:21.256 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.4960000000000001e-06 2023-09-23T20:59:21.256 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:59:21.256 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:59:21.256 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T20:59:21.256 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:59:21.237702+0000", 2023-09-23T20:59:21.256 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.4849e-05 2023-09-23T20:59:21.256 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:59:21.256 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:59:21.256 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T20:59:21.257 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:59:21.237710+0000", 2023-09-23T20:59:21.257 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.0369999999999995e-06 2023-09-23T20:59:21.257 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:59:21.257 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:59:21.257 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:59:21.257 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:59:21.237756+0000", 2023-09-23T20:59:21.257 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.5816999999999999e-05 2023-09-23T20:59:21.257 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:59:21.257 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:59:21.257 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T20:59:21.257 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:59:21.237776+0000", 2023-09-23T20:59:21.257 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.0633e-05 2023-09-23T20:59:21.258 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:59:21.258 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T20:59:21.258 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:59:21.258 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:59:21.258 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:59:21.258 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_repop(client.35100.0:144866 7.d e239/199)", 2023-09-23T20:59:21.258 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T20:59:21.239997+0000", 2023-09-23T20:59:21.258 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 4294967295.9997725, 2023-09-23T20:59:21.258 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.6030999999999999e-05, 2023-09-23T20:59:21.258 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T20:59:21.258 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "queued for pg", 2023-09-23T20:59:21.258 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T20:59:21.259 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:59:21.259 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T20:59:21.259 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:59:21.239997+0000", 2023-09-23T20:59:21.259 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:59:21.259 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:59:21.259 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:59:21.259 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T20:59:21.259 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:59:21.239997+0000", 2023-09-23T20:59:21.259 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:59:21.259 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:59:21.259 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:59:21.259 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T20:59:21.259 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:59:21.239996+0000", 2023-09-23T20:59:21.259 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T20:59:21.260 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:59:21.260 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:59:21.260 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T20:59:21.260 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:59:21.240004+0000", 2023-09-23T20:59:21.260 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.1580000000000005e-06 2023-09-23T20:59:21.260 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:59:21.260 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:59:21.260 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T20:59:21.260 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:59:21.240018+0000", 2023-09-23T20:59:21.260 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.3371e-05 2023-09-23T20:59:21.260 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:59:21.260 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:59:21.260 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T20:59:21.261 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:59:21.240024+0000", 2023-09-23T20:59:21.261 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.105e-06 2023-09-23T20:59:21.261 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:59:21.261 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T20:59:21.261 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:59:21.261 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:59:21.261 INFO:teuthology.orchestra.run.smithi026.stdout: ], 2023-09-23T20:59:21.261 INFO:teuthology.orchestra.run.smithi026.stdout: "num_ops": 4 2023-09-23T20:59:21.261 INFO:teuthology.orchestra.run.smithi026.stdout:} 2023-09-23T20:59:21.333 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:59:21.361 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:59:21.400 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:59:21.402 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:59:21.403 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:59:21.503 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:59:21.594 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:59:21.948 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:59:21.957 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:59:21.957 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:59:22.058 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:59:22.091 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:59:22.115 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:59:22 smithi026 ceph-mon[175151]: pgmap v585: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 3.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 564 op/s; 11463/181827 objects degraded (6.304%); 2897/181827 objects misplaced (1.593%); 1.5 MiB/s, 24 objects/s recovering 2023-09-23T20:59:22.116 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:59:22 smithi026 ceph-mon[179688]: pgmap v585: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 3.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 564 op/s; 11463/181827 objects degraded (6.304%); 2897/181827 objects misplaced (1.593%); 1.5 MiB/s, 24 objects/s recovering 2023-09-23T20:59:22.127 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:59:22.131 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_blocked_ops 2023-09-23T20:59:22.131 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:59:22.132 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:59:22.233 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:59:22.263 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:59:22.310 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:59:22.313 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:59:22 smithi070 ceph-mon[168282]: pgmap v585: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 3.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 564 op/s; 11463/181827 objects degraded (6.304%); 2897/181827 objects misplaced (1.593%); 1.5 MiB/s, 24 objects/s recovering 2023-09-23T20:59:22.314 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:59:22.314 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:59:22.415 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:59:22.574 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:59:22.677 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:59:22.684 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:59:22.684 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:59:22.723 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:59:22.785 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:59:22.818 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:59:22.857 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:59:22.859 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:59:22.860 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:59:22.960 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:59:22.988 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:59:23.024 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:59:23.026 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:59:23.027 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:59:23.128 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:59:23.174 INFO:teuthology.orchestra.run.smithi026.stdout: 2023-09-23T20:59:23.187 INFO:teuthology.orchestra.run.smithi026.stderr:dumped all 2023-09-23T20:59:23.217 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:59:23.297 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:59:23.304 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:59:23.305 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:59:23.361 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:59:23.405 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:59:23.434 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:59:23.471 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:59:23.474 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:59:23.475 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:59:23.576 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:59:23.606 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:59:23.653 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:59:23.655 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:59:23.655 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:59:23.756 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:59:23.886 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:59:24.138 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:59:24 smithi026 ceph-mon[175151]: pgmap v586: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 3.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 560 op/s; 11513/177099 objects degraded (6.501%); 2721/177099 objects misplaced (1.536%); 1.5 MiB/s, 24 objects/s recovering 2023-09-23T20:59:24.138 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:59:24 smithi026 ceph-mon[175151]: from='client.45305 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:59:24.138 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:59:24 smithi026 ceph-mon[179688]: pgmap v586: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 3.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 560 op/s; 11513/177099 objects degraded (6.501%); 2721/177099 objects misplaced (1.536%); 1.5 MiB/s, 24 objects/s recovering 2023-09-23T20:59:24.139 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:59:24 smithi026 ceph-mon[179688]: from='client.45305 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:59:24.172 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:59:24.173 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T20:59:24.177 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:59:24.178 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:59:24.252 INFO:teuthology.orchestra.run.smithi026.stdout:{"pg_ready":true,"pg_map":{"version":586,"stamp":"2023-09-23T20:59:22.438679+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":3867744184,"num_objects":59033,"num_object_clones":0,"num_object_copies":177099,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":11513,"num_objects_misplaced":2721,"num_objects_unfound":0,"num_objects_dirty":59033,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":144572,"num_write_kb":6512606,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2130,"num_bytes_recovered":138155878,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":122447,"ondisk_log_size":122447,"up":327,"acting":324,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":331,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":13455000,"kb_used_data":11863512,"kb_used_omap":136,"kb_used_meta":1591095,"kb_avail":736506216,"statfs":{"total":767960285184,"available":754182365184,"internally_reserved":0,"allocated":12148236288,"data_stored":12140279425,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":139741,"internal_metadata":1629281827},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,0,1,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":11,"apply_latency_ms":11,"commit_latency_ns":11000000,"apply_latency_ns":11000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-367591424,"num_objects":-5609,"num_object_clones":0,"num_object_copies":-16827,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":97,"num_objects_misplaced":-345,"num_objects_unfound":0,"num_objects_dirty":-5609,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5609,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":248,"num_bytes_recovered":16252928,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.004580"},"pg_stats":[{"pgid":"3.1a","version":"68'1","reported_seq":116,"reported_epoch":238,"state":"active+clean","last_fresh":"2023-09-23T20:59:08.299989+0000","last_change":"2023-09-23T20:54:13.946695+0000","last_active":"2023-09-23T20:59:08.299989+0000","last_peered":"2023-09-23T20:59:08.299989+0000","last_clean":"2023-09-23T20:59:08.299989+0000","last_became_active":"2023-09-23T20:54:13.945399+0000","last_became_peered":"2023-09-23T20:54:13.945399+0000","last_unstale":"2023-09-23T20:59:08.299989+0000","last_undegraded":"2023-09-23T20:59:08.299989+0000","last_fullsized":"2023-09-23T20:59:08.299989+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:32:52.365464+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":231,"reported_epoch":238,"state":"active+clean","last_fresh":"2023-09-23T20:59:08.296636+0000","last_change":"2023-09-23T20:54:13.942019+0000","last_active":"2023-09-23T20:59:08.296636+0000","last_peered":"2023-09-23T20:59:08.296636+0000","last_clean":"2023-09-23T20:59:08.296636+0000","last_became_active":"2023-09-23T20:54:13.941647+0000","last_became_peered":"2023-09-23T20:54:13.941647+0000","last_unstale":"2023-09-23T20:59:08.296636+0000","last_undegraded":"2023-09-23T20:59:08.296636+0000","last_fullsized":"2023-09-23T20:59:08.296636+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:30:54.400944+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":208,"reported_epoch":238,"state":"active+clean","last_fresh":"2023-09-23T20:59:08.287851+0000","last_change":"2023-09-23T20:54:13.941394+0000","last_active":"2023-09-23T20:59:08.287851+0000","last_peered":"2023-09-23T20:59:08.287851+0000","last_clean":"2023-09-23T20:59:08.287851+0000","last_became_active":"2023-09-23T20:54:13.941042+0000","last_became_peered":"2023-09-23T20:54:13.941042+0000","last_unstale":"2023-09-23T20:59:08.287851+0000","last_undegraded":"2023-09-23T20:59:08.287851+0000","last_fullsized":"2023-09-23T20:59:08.287851+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:19:41.401184+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":223,"reported_epoch":238,"state":"active+clean","last_fresh":"2023-09-23T20:59:08.299918+0000","last_change":"2023-09-23T20:57:44.979811+0000","last_active":"2023-09-23T20:59:08.299918+0000","last_peered":"2023-09-23T20:59:08.299918+0000","last_clean":"2023-09-23T20:59:08.299918+0000","last_became_active":"2023-09-23T20:57:44.979515+0000","last_became_peered":"2023-09-23T20:57:44.979515+0000","last_unstale":"2023-09-23T20:59:08.299918+0000","last_undegraded":"2023-09-23T20:59:08.299918+0000","last_fullsized":"2023-09-23T20:59:08.299918+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:38:22.162140+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":222,"reported_epoch":238,"state":"active+clean","last_fresh":"2023-09-23T20:59:08.297503+0000","last_change":"2023-09-23T20:51:10.833613+0000","last_active":"2023-09-23T20:59:08.297503+0000","last_peered":"2023-09-23T20:59:08.297503+0000","last_clean":"2023-09-23T20:59:08.297503+0000","last_became_active":"2023-09-23T20:51:10.833208+0000","last_became_peered":"2023-09-23T20:51:10.833208+0000","last_unstale":"2023-09-23T20:59:08.297503+0000","last_undegraded":"2023-09-23T20:59:08.297503+0000","last_fullsized":"2023-09-23T20:59:08.297503+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:51:28.246014+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":204,"reported_epoch":238,"state":"active+clean","last_fresh":"2023-09-23T20:59:08.287433+0000","last_change":"2023-09-23T20:55:35.958384+0000","last_active":"2023-09-23T20:59:08.287433+0000","last_peered":"2023-09-23T20:59:08.287433+0000","last_clean":"2023-09-23T20:59:08.287433+0000","last_became_active":"2023-09-23T20:55:35.954563+0000","last_became_peered":"2023-09-23T20:55:35.954563+0000","last_unstale":"2023-09-23T20:59:08.287433+0000","last_undegraded":"2023-09-23T20:59:08.287433+0000","last_fullsized":"2023-09-23T20:59:08.287433+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:37:48.332259+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":214,"reported_epoch":238,"state":"active+clean","last_fresh":"2023-09-23T20:59:11.533592+0000","last_change":"2023-09-23T20:54:31.885514+0000","last_active":"2023-09-23T20:59:11.533592+0000","last_peered":"2023-09-23T20:59:11.533592+0000","last_clean":"2023-09-23T20:59:11.533592+0000","last_became_active":"2023-09-23T20:54:31.884986+0000","last_became_peered":"2023-09-23T20:54:31.884986+0000","last_unstale":"2023-09-23T20:59:11.533592+0000","last_undegraded":"2023-09-23T20:59:11.533592+0000","last_fullsized":"2023-09-23T20:59:11.533592+0000","mapping_epoch":136,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:26:08.100220+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":219,"reported_epoch":238,"state":"active+clean","last_fresh":"2023-09-23T20:59:08.297583+0000","last_change":"2023-09-23T20:55:35.951889+0000","last_active":"2023-09-23T20:59:08.297583+0000","last_peered":"2023-09-23T20:59:08.297583+0000","last_clean":"2023-09-23T20:59:08.297583+0000","last_became_active":"2023-09-23T20:55:35.951563+0000","last_became_peered":"2023-09-23T20:55:35.951563+0000","last_unstale":"2023-09-23T20:59:08.297583+0000","last_undegraded":"2023-09-23T20:59:08.297583+0000","last_fullsized":"2023-09-23T20:59:08.297583+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:41:53.098781+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"79'161","reported_seq":525,"reported_epoch":238,"state":"active+clean","last_fresh":"2023-09-23T20:59:08.309758+0000","last_change":"2023-09-23T20:57:25.096559+0000","last_active":"2023-09-23T20:59:08.309758+0000","last_peered":"2023-09-23T20:59:08.309758+0000","last_clean":"2023-09-23T20:59:08.309758+0000","last_became_active":"2023-09-23T20:57:15.975674+0000","last_became_peered":"2023-09-23T20:57:15.975674+0000","last_unstale":"2023-09-23T20:59:08.309758+0000","last_undegraded":"2023-09-23T20:59:08.309758+0000","last_fullsized":"2023-09-23T20:59:08.309758+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:25.096401+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:25.096401+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:05:05.448404+0000","scrub_duration":0.002859393,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":112,"reported_epoch":238,"state":"active+clean","last_fresh":"2023-09-23T20:59:08.297894+0000","last_change":"2023-09-23T20:54:31.890678+0000","last_active":"2023-09-23T20:59:08.297894+0000","last_peered":"2023-09-23T20:59:08.297894+0000","last_clean":"2023-09-23T20:59:08.297894+0000","last_became_active":"2023-09-23T20:54:31.890469+0000","last_became_peered":"2023-09-23T20:54:31.890469+0000","last_unstale":"2023-09-23T20:59:08.297894+0000","last_undegraded":"2023-09-23T20:59:08.297894+0000","last_fullsized":"2023-09-23T20:59:08.297894+0000","mapping_epoch":136,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:55:08.790146+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":180,"reported_epoch":238,"state":"active+clean","last_fresh":"2023-09-23T20:59:08.298031+0000","last_change":"2023-09-23T20:55:35.952661+0000","last_active":"2023-09-23T20:59:08.298031+0000","last_peered":"2023-09-23T20:59:08.298031+0000","last_clean":"2023-09-23T20:59:08.298031+0000","last_became_active":"2023-09-23T20:55:35.952349+0000","last_became_peered":"2023-09-23T20:55:35.952349+0000","last_unstale":"2023-09-23T20:59:08.298031+0000","last_undegraded":"2023-09-23T20:59:08.298031+0000","last_fullsized":"2023-09-23T20:59:08.298031+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:00:03.110126+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":57,"reported_epoch":238,"state":"active+clean","last_fresh":"2023-09-23T20:59:14.570760+0000","last_change":"2023-09-23T20:56:45.988607+0000","last_active":"2023-09-23T20:59:14.570760+0000","last_peered":"2023-09-23T20:59:14.570760+0000","last_clean":"2023-09-23T20:59:14.570760+0000","last_became_active":"2023-09-23T20:56:45.988206+0000","last_became_peered":"2023-09-23T20:56:45.988206+0000","last_unstale":"2023-09-23T20:59:14.570760+0000","last_undegraded":"2023-09-23T20:59:14.570760+0000","last_fullsized":"2023-09-23T20:59:14.570760+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":205,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:01:19.783688+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,4],"acting":[3,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":56,"reported_epoch":238,"state":"active+clean","last_fresh":"2023-09-23T20:59:08.297659+0000","last_change":"2023-09-23T20:56:18.139142+0000","last_active":"2023-09-23T20:59:08.297659+0000","last_peered":"2023-09-23T20:59:08.297659+0000","last_clean":"2023-09-23T20:59:08.297659+0000","last_became_active":"2023-09-23T20:56:18.138844+0000","last_became_peered":"2023-09-23T20:56:18.138844+0000","last_unstale":"2023-09-23T20:59:08.297659+0000","last_undegraded":"2023-09-23T20:59:08.297659+0000","last_fullsized":"2023-09-23T20:59:08.297659+0000","mapping_epoch":192,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":193,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:09:02.217683+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,2],"acting":[5,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.14","version":"79'22","reported_seq":143,"reported_epoch":238,"state":"active+clean","last_fresh":"2023-09-23T20:59:08.300310+0000","last_change":"2023-09-23T20:55:35.949418+0000","last_active":"2023-09-23T20:59:08.300310+0000","last_peered":"2023-09-23T20:59:08.300310+0000","last_clean":"2023-09-23T20:59:08.300310+0000","last_became_active":"2023-09-23T20:55:35.948566+0000","last_became_peered":"2023-09-23T20:55:35.948566+0000","last_unstale":"2023-09-23T20:59:08.300310+0000","last_undegraded":"2023-09-23T20:59:08.300310+0000","last_fullsized":"2023-09-23T20:59:08.300310+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:59:37.433128+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4734,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":133,"reported_epoch":238,"state":"active+clean","last_fresh":"2023-09-23T20:59:08.301429+0000","last_change":"2023-09-23T20:55:35.949178+0000","last_active":"2023-09-23T20:59:08.301429+0000","last_peered":"2023-09-23T20:59:08.301429+0000","last_clean":"2023-09-23T20:59:08.301429+0000","last_became_active":"2023-09-23T20:55:35.948795+0000","last_became_peered":"2023-09-23T20:55:35.948795+0000","last_unstale":"2023-09-23T20:59:08.301429+0000","last_undegraded":"2023-09-23T20:59:08.301429+0000","last_fullsized":"2023-09-23T20:59:08.301429+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:13:08.996932+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.11","version":"79'161","reported_seq":467,"reported_epoch":238,"state":"active+clean","last_fresh":"2023-09-23T20:59:08.304588+0000","last_change":"2023-09-23T20:57:28.756796+0000","last_active":"2023-09-23T20:59:08.304588+0000","last_peered":"2023-09-23T20:59:08.304588+0000","last_clean":"2023-09-23T20:59:08.304588+0000","last_became_active":"2023-09-23T20:57:15.980606+0000","last_became_peered":"2023-09-23T20:57:15.980606+0000","last_unstale":"2023-09-23T20:59:08.304588+0000","last_undegraded":"2023-09-23T20:59:08.304588+0000","last_fullsized":"2023-09-23T20:59:08.304588+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:28.756634+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:28.756634+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:09:19.150791+0000","scrub_duration":0.0064925119999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":159,"reported_epoch":238,"state":"active+clean","last_fresh":"2023-09-23T20:59:08.301287+0000","last_change":"2023-09-23T20:56:45.992577+0000","last_active":"2023-09-23T20:59:08.301287+0000","last_peered":"2023-09-23T20:59:08.301287+0000","last_clean":"2023-09-23T20:59:08.301287+0000","last_became_active":"2023-09-23T20:56:45.992297+0000","last_became_peered":"2023-09-23T20:56:45.992297+0000","last_unstale":"2023-09-23T20:59:08.301287+0000","last_undegraded":"2023-09-23T20:59:08.301287+0000","last_fullsized":"2023-09-23T20:59:08.301287+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":205,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:00:45.547202+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,2],"acting":[4,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":204,"reported_epoch":238,"state":"active+clean","last_fresh":"2023-09-23T20:59:08.296739+0000","last_change":"2023-09-23T20:51:10.831262+0000","last_active":"2023-09-23T20:59:08.296739+0000","last_peered":"2023-09-23T20:59:08.296739+0000","last_clean":"2023-09-23T20:59:08.296739+0000","last_became_active":"2023-09-23T20:51:10.830931+0000","last_became_peered":"2023-09-23T20:51:10.830931+0000","last_unstale":"2023-09-23T20:59:08.296739+0000","last_undegraded":"2023-09-23T20:59:08.296739+0000","last_fullsized":"2023-09-23T20:59:08.296739+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:51:54.982505+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.10","version":"79'161","reported_seq":476,"reported_epoch":238,"state":"active+clean","last_fresh":"2023-09-23T20:59:08.297312+0000","last_change":"2023-09-23T20:57:20.708003+0000","last_active":"2023-09-23T20:59:08.297312+0000","last_peered":"2023-09-23T20:59:08.297312+0000","last_clean":"2023-09-23T20:59:08.297312+0000","last_became_active":"2023-09-23T20:57:15.975302+0000","last_became_peered":"2023-09-23T20:57:15.975302+0000","last_unstale":"2023-09-23T20:59:08.297312+0000","last_undegraded":"2023-09-23T20:59:08.297312+0000","last_fullsized":"2023-09-23T20:59:08.297312+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:20.707939+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:20.707939+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:16:51.721994+0000","scrub_duration":0.0017733409999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":214,"reported_epoch":238,"state":"active+clean","last_fresh":"2023-09-23T20:59:08.307987+0000","last_change":"2023-09-23T20:57:44.979736+0000","last_active":"2023-09-23T20:59:08.307987+0000","last_peered":"2023-09-23T20:59:08.307987+0000","last_clean":"2023-09-23T20:59:08.307987+0000","last_became_active":"2023-09-23T20:57:44.979356+0000","last_became_peered":"2023-09-23T20:57:44.979356+0000","last_unstale":"2023-09-23T20:59:08.307987+0000","last_undegraded":"2023-09-23T20:59:08.307987+0000","last_fullsized":"2023-09-23T20:59:08.307987+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:14:43.848832+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":112,"reported_epoch":238,"state":"active+clean","last_fresh":"2023-09-23T20:59:08.297950+0000","last_change":"2023-09-23T20:54:31.887814+0000","last_active":"2023-09-23T20:59:08.297950+0000","last_peered":"2023-09-23T20:59:08.297950+0000","last_clean":"2023-09-23T20:59:08.297950+0000","last_became_active":"2023-09-23T20:54:31.887195+0000","last_became_peered":"2023-09-23T20:54:31.887195+0000","last_unstale":"2023-09-23T20:59:08.297950+0000","last_undegraded":"2023-09-23T20:59:08.297950+0000","last_fullsized":"2023-09-23T20:59:08.297950+0000","mapping_epoch":136,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:10:52.089553+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.13","version":"79'161","reported_seq":538,"reported_epoch":238,"state":"active+clean","last_fresh":"2023-09-23T20:59:08.288138+0000","last_change":"2023-09-23T20:57:48.262970+0000","last_active":"2023-09-23T20:59:08.288138+0000","last_peered":"2023-09-23T20:59:08.288138+0000","last_clean":"2023-09-23T20:59:08.288138+0000","last_became_active":"2023-09-23T20:57:48.262663+0000","last_became_peered":"2023-09-23T20:57:48.262663+0000","last_unstale":"2023-09-23T20:59:08.288138+0000","last_undegraded":"2023-09-23T20:59:08.288138+0000","last_fullsized":"2023-09-23T20:59:08.288138+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:36.420362+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:36.420362+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:28:28.613492+0000","scrub_duration":0.028838444000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":183,"reported_epoch":238,"state":"active+clean","last_fresh":"2023-09-23T20:59:08.301819+0000","last_change":"2023-09-23T20:56:45.991822+0000","last_active":"2023-09-23T20:59:08.301819+0000","last_peered":"2023-09-23T20:59:08.301819+0000","last_clean":"2023-09-23T20:59:08.301819+0000","last_became_active":"2023-09-23T20:56:45.991460+0000","last_became_peered":"2023-09-23T20:56:45.991460+0000","last_unstale":"2023-09-23T20:59:08.301819+0000","last_undegraded":"2023-09-23T20:59:08.301819+0000","last_fullsized":"2023-09-23T20:59:08.301819+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":205,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:20:37.503317+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":196,"reported_epoch":238,"state":"active+clean","last_fresh":"2023-09-23T20:59:11.533559+0000","last_change":"2023-09-23T20:51:10.840849+0000","last_active":"2023-09-23T20:59:11.533559+0000","last_peered":"2023-09-23T20:59:11.533559+0000","last_clean":"2023-09-23T20:59:11.533559+0000","last_became_active":"2023-09-23T20:51:10.840593+0000","last_became_peered":"2023-09-23T20:51:10.840593+0000","last_unstale":"2023-09-23T20:59:11.533559+0000","last_undegraded":"2023-09-23T20:59:11.533559+0000","last_fullsized":"2023-09-23T20:59:11.533559+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:30:52.297625+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.12","version":"79'161","reported_seq":476,"reported_epoch":238,"state":"active+clean","last_fresh":"2023-09-23T20:59:08.297465+0000","last_change":"2023-09-23T20:57:21.750765+0000","last_active":"2023-09-23T20:59:08.297465+0000","last_peered":"2023-09-23T20:59:08.297465+0000","last_clean":"2023-09-23T20:59:08.297465+0000","last_became_active":"2023-09-23T20:57:15.979195+0000","last_became_peered":"2023-09-23T20:57:15.979195+0000","last_unstale":"2023-09-23T20:59:08.297465+0000","last_undegraded":"2023-09-23T20:59:08.297465+0000","last_fullsized":"2023-09-23T20:59:08.297465+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:21.750680+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:21.750680+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:03:41.915962+0000","scrub_duration":0.0092391680000000007,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":231,"reported_epoch":238,"state":"active+clean","last_fresh":"2023-09-23T20:59:08.286663+0000","last_change":"2023-09-23T20:55:36.943969+0000","last_active":"2023-09-23T20:59:08.286663+0000","last_peered":"2023-09-23T20:59:08.286663+0000","last_clean":"2023-09-23T20:59:08.286663+0000","last_became_active":"2023-09-23T20:55:36.943580+0000","last_became_peered":"2023-09-23T20:55:36.943580+0000","last_unstale":"2023-09-23T20:59:08.286663+0000","last_undegraded":"2023-09-23T20:59:08.286663+0000","last_fullsized":"2023-09-23T20:59:08.286663+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:49:06.969396+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":27,"reported_epoch":238,"state":"active+clean","last_fresh":"2023-09-23T20:59:08.297238+0000","last_change":"2023-09-23T20:57:44.980498+0000","last_active":"2023-09-23T20:59:08.297238+0000","last_peered":"2023-09-23T20:59:08.297238+0000","last_clean":"2023-09-23T20:59:08.297238+0000","last_became_active":"2023-09-23T20:57:44.980182+0000","last_became_peered":"2023-09-23T20:57:44.980182+0000","last_unstale":"2023-09-23T20:59:08.297238+0000","last_undegraded":"2023-09-23T20:59:08.297238+0000","last_fullsized":"2023-09-23T20:59:08.297238+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:34:51.620544+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.d","version":"79'161","reported_seq":521,"reported_epoch":238,"state":"active+clean","last_fresh":"2023-09-23T20:59:08.300198+0000","last_change":"2023-09-23T20:57:31.075213+0000","last_active":"2023-09-23T20:59:08.300198+0000","last_peered":"2023-09-23T20:59:08.300198+0000","last_clean":"2023-09-23T20:59:08.300198+0000","last_became_active":"2023-09-23T20:57:15.988188+0000","last_became_peered":"2023-09-23T20:57:15.988188+0000","last_unstale":"2023-09-23T20:59:08.300198+0000","last_undegraded":"2023-09-23T20:59:08.300198+0000","last_fullsized":"2023-09-23T20:59:08.300198+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:31.075098+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:31.075098+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:41:52.142362+0000","scrub_duration":0.010304610000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.b","version":"239'8898","reported_seq":8948,"reported_epoch":239,"state":"active+clean","last_fresh":"2023-09-23T20:59:19.143714+0000","last_change":"2023-09-23T20:56:35.543994+0000","last_active":"2023-09-23T20:59:19.143714+0000","last_peered":"2023-09-23T20:59:19.143714+0000","last_clean":"2023-09-23T20:59:19.143714+0000","last_became_active":"2023-09-23T20:56:35.543758+0000","last_became_peered":"2023-09-23T20:56:35.543758+0000","last_unstale":"2023-09-23T20:59:19.143714+0000","last_undegraded":"2023-09-23T20:59:19.143714+0000","last_fullsized":"2023-09-23T20:59:19.143714+0000","mapping_epoch":199,"log_start":"205'1900","ondisk_log_start":"205'1900","created":199,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_clean_scrub_stamp":"2023-09-23T20:56:34.529103+0000","objects_scrubbed":0,"log_size":6998,"log_dups_size":0,"ondisk_log_size":6998,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:21:29.041484+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":247857152,"num_objects":3782,"num_object_clones":0,"num_object_copies":11346,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3782,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8898,"num_write_kb":405760,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":170,"reported_epoch":238,"state":"active+clean","last_fresh":"2023-09-23T20:59:08.300678+0000","last_change":"2023-09-23T20:56:45.991841+0000","last_active":"2023-09-23T20:59:08.300678+0000","last_peered":"2023-09-23T20:59:08.300678+0000","last_clean":"2023-09-23T20:59:08.300678+0000","last_became_active":"2023-09-23T20:56:45.991491+0000","last_became_peered":"2023-09-23T20:56:45.991491+0000","last_unstale":"2023-09-23T20:59:08.300678+0000","last_undegraded":"2023-09-23T20:59:08.300678+0000","last_fullsized":"2023-09-23T20:59:08.300678+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":205,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:30:37.405319+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":129,"reported_epoch":238,"state":"active+clean","last_fresh":"2023-09-23T20:59:08.297423+0000","last_change":"2023-09-23T20:55:35.957768+0000","last_active":"2023-09-23T20:59:08.297423+0000","last_peered":"2023-09-23T20:59:08.297423+0000","last_clean":"2023-09-23T20:59:08.297423+0000","last_became_active":"2023-09-23T20:55:35.957421+0000","last_became_peered":"2023-09-23T20:55:35.957421+0000","last_unstale":"2023-09-23T20:59:08.297423+0000","last_undegraded":"2023-09-23T20:59:08.297423+0000","last_fullsized":"2023-09-23T20:59:08.297423+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:49:41.585961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.c","version":"79'161","reported_seq":539,"reported_epoch":238,"state":"active+clean","last_fresh":"2023-09-23T20:59:08.298061+0000","last_change":"2023-09-23T20:57:22.041071+0000","last_active":"2023-09-23T20:59:08.298061+0000","last_peered":"2023-09-23T20:59:08.298061+0000","last_clean":"2023-09-23T20:59:08.298061+0000","last_became_active":"2023-09-23T20:57:15.973204+0000","last_became_peered":"2023-09-23T20:57:15.973204+0000","last_unstale":"2023-09-23T20:59:08.298061+0000","last_undegraded":"2023-09-23T20:59:08.298061+0000","last_fullsized":"2023-09-23T20:59:08.298061+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:22.040907+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:22.040907+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:47:02.019978+0000","scrub_duration":0.013021176000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.a","version":"239'8992","reported_seq":9042,"reported_epoch":239,"state":"active+clean","last_fresh":"2023-09-23T20:59:19.456716+0000","last_change":"2023-09-23T20:56:35.547120+0000","last_active":"2023-09-23T20:59:19.456716+0000","last_peered":"2023-09-23T20:59:19.456716+0000","last_clean":"2023-09-23T20:59:19.456716+0000","last_became_active":"2023-09-23T20:56:35.546838+0000","last_became_peered":"2023-09-23T20:56:35.546838+0000","last_unstale":"2023-09-23T20:59:19.456716+0000","last_undegraded":"2023-09-23T20:59:19.456716+0000","last_fullsized":"2023-09-23T20:59:19.456716+0000","mapping_epoch":199,"log_start":"206'2800","ondisk_log_start":"206'2800","created":199,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_clean_scrub_stamp":"2023-09-23T20:56:34.529103+0000","objects_scrubbed":0,"log_size":6192,"log_dups_size":0,"ondisk_log_size":6192,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:48:24.132768+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":245366784,"num_objects":3744,"num_object_clones":0,"num_object_copies":11232,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3744,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8992,"num_write_kb":407552,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,5],"acting":[3,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":157,"reported_epoch":238,"state":"active+clean","last_fresh":"2023-09-23T20:59:08.300716+0000","last_change":"2023-09-23T20:56:45.991821+0000","last_active":"2023-09-23T20:59:08.300716+0000","last_peered":"2023-09-23T20:59:08.300716+0000","last_clean":"2023-09-23T20:59:08.300716+0000","last_became_active":"2023-09-23T20:56:45.991447+0000","last_became_peered":"2023-09-23T20:56:45.991447+0000","last_unstale":"2023-09-23T20:59:08.300716+0000","last_undegraded":"2023-09-23T20:59:08.300716+0000","last_fullsized":"2023-09-23T20:59:08.300716+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":205,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:14:25.525222+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":160,"reported_epoch":238,"state":"active+clean","last_fresh":"2023-09-23T20:59:08.302083+0000","last_change":"2023-09-23T20:57:44.982059+0000","last_active":"2023-09-23T20:59:08.302083+0000","last_peered":"2023-09-23T20:59:08.302083+0000","last_clean":"2023-09-23T20:59:08.302083+0000","last_became_active":"2023-09-23T20:57:44.981742+0000","last_became_peered":"2023-09-23T20:57:44.981742+0000","last_unstale":"2023-09-23T20:59:08.302083+0000","last_undegraded":"2023-09-23T20:59:08.302083+0000","last_fullsized":"2023-09-23T20:59:08.302083+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:41:24.042916+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.f","version":"79'161","reported_seq":543,"reported_epoch":238,"state":"active+clean","last_fresh":"2023-09-23T20:59:08.287183+0000","last_change":"2023-09-23T20:57:27.880841+0000","last_active":"2023-09-23T20:59:08.287183+0000","last_peered":"2023-09-23T20:59:08.287183+0000","last_clean":"2023-09-23T20:59:08.287183+0000","last_became_active":"2023-09-23T20:57:15.970925+0000","last_became_peered":"2023-09-23T20:57:15.970925+0000","last_unstale":"2023-09-23T20:59:08.287183+0000","last_undegraded":"2023-09-23T20:59:08.287183+0000","last_fullsized":"2023-09-23T20:59:08.287183+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:27.880776+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:27.880776+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:15:13.438014+0000","scrub_duration":0.0014004899999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.9","version":"239'9067","reported_seq":9117,"reported_epoch":239,"state":"active+clean","last_fresh":"2023-09-23T20:59:21.482997+0000","last_change":"2023-09-23T20:56:35.549387+0000","last_active":"2023-09-23T20:59:21.482997+0000","last_peered":"2023-09-23T20:59:21.482997+0000","last_clean":"2023-09-23T20:59:21.482997+0000","last_became_active":"2023-09-23T20:56:35.549087+0000","last_became_peered":"2023-09-23T20:56:35.549087+0000","last_unstale":"2023-09-23T20:59:21.482997+0000","last_undegraded":"2023-09-23T20:59:21.482997+0000","last_fullsized":"2023-09-23T20:59:21.482997+0000","mapping_epoch":199,"log_start":"205'1700","ondisk_log_start":"205'1700","created":199,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_clean_scrub_stamp":"2023-09-23T20:56:34.529103+0000","objects_scrubbed":0,"log_size":7367,"log_dups_size":0,"ondisk_log_size":7367,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:04:51.547362+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":238485504,"num_objects":3639,"num_object_clones":0,"num_object_copies":10917,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3639,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9067,"num_write_kb":406592,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":143,"reported_epoch":238,"state":"active+clean","last_fresh":"2023-09-23T20:59:08.297710+0000","last_change":"2023-09-23T20:56:45.989153+0000","last_active":"2023-09-23T20:59:08.297710+0000","last_peered":"2023-09-23T20:59:08.297710+0000","last_clean":"2023-09-23T20:59:08.297710+0000","last_became_active":"2023-09-23T20:56:45.988800+0000","last_became_peered":"2023-09-23T20:56:45.988800+0000","last_unstale":"2023-09-23T20:59:08.297710+0000","last_undegraded":"2023-09-23T20:59:08.297710+0000","last_fullsized":"2023-09-23T20:59:08.297710+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":205,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:04:41.130194+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":178,"reported_epoch":238,"state":"active+clean","last_fresh":"2023-09-23T20:59:08.286918+0000","last_change":"2023-09-23T20:55:35.965427+0000","last_active":"2023-09-23T20:59:08.286918+0000","last_peered":"2023-09-23T20:59:08.286918+0000","last_clean":"2023-09-23T20:59:08.286918+0000","last_became_active":"2023-09-23T20:55:35.963969+0000","last_became_peered":"2023-09-23T20:55:35.963969+0000","last_unstale":"2023-09-23T20:59:08.286918+0000","last_undegraded":"2023-09-23T20:59:08.286918+0000","last_fullsized":"2023-09-23T20:59:08.286918+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:38:40.612083+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.e","version":"79'161","reported_seq":450,"reported_epoch":238,"state":"active+clean","last_fresh":"2023-09-23T20:59:08.300601+0000","last_change":"2023-09-23T20:57:34.132738+0000","last_active":"2023-09-23T20:59:08.300601+0000","last_peered":"2023-09-23T20:59:08.300601+0000","last_clean":"2023-09-23T20:59:08.300601+0000","last_became_active":"2023-09-23T20:57:15.974318+0000","last_became_peered":"2023-09-23T20:57:15.974318+0000","last_unstale":"2023-09-23T20:59:08.300601+0000","last_undegraded":"2023-09-23T20:59:08.300601+0000","last_fullsized":"2023-09-23T20:59:08.300601+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:34.132660+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:34.132660+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:07:07.739438+0000","scrub_duration":0.005229694,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.8","version":"239'9168","reported_seq":9218,"reported_epoch":239,"state":"active+clean","last_fresh":"2023-09-23T20:59:21.386968+0000","last_change":"2023-09-23T20:56:35.548119+0000","last_active":"2023-09-23T20:59:21.386968+0000","last_peered":"2023-09-23T20:59:21.386968+0000","last_clean":"2023-09-23T20:59:21.386968+0000","last_became_active":"2023-09-23T20:56:35.547914+0000","last_became_peered":"2023-09-23T20:56:35.547914+0000","last_unstale":"2023-09-23T20:59:21.386968+0000","last_undegraded":"2023-09-23T20:59:21.386968+0000","last_fullsized":"2023-09-23T20:59:21.386968+0000","mapping_epoch":199,"log_start":"202'300","ondisk_log_start":"202'300","created":199,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_clean_scrub_stamp":"2023-09-23T20:56:34.529103+0000","objects_scrubbed":0,"log_size":8868,"log_dups_size":0,"ondisk_log_size":8868,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:42:54.255023+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":241565696,"num_objects":3686,"num_object_clones":0,"num_object_copies":11058,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3686,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9168,"num_write_kb":411328,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":129,"reported_epoch":238,"state":"active+clean","last_fresh":"2023-09-23T20:59:08.297759+0000","last_change":"2023-09-23T20:55:35.962005+0000","last_active":"2023-09-23T20:59:08.297759+0000","last_peered":"2023-09-23T20:59:08.297759+0000","last_clean":"2023-09-23T20:59:08.297759+0000","last_became_active":"2023-09-23T20:55:35.961695+0000","last_became_peered":"2023-09-23T20:55:35.961695+0000","last_unstale":"2023-09-23T20:59:08.297759+0000","last_undegraded":"2023-09-23T20:59:08.297759+0000","last_fullsized":"2023-09-23T20:59:08.297759+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:28:22.876201+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":227,"reported_epoch":238,"state":"active+clean","last_fresh":"2023-09-23T20:59:08.305542+0000","last_change":"2023-09-23T20:54:13.928715+0000","last_active":"2023-09-23T20:59:08.305542+0000","last_peered":"2023-09-23T20:59:08.305542+0000","last_clean":"2023-09-23T20:59:08.305542+0000","last_became_active":"2023-09-23T20:54:13.928200+0000","last_became_peered":"2023-09-23T20:54:13.928200+0000","last_unstale":"2023-09-23T20:59:08.305542+0000","last_undegraded":"2023-09-23T20:59:08.305542+0000","last_fullsized":"2023-09-23T20:59:08.305542+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:41:16.927513+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.f","version":"239'9248","reported_seq":9298,"reported_epoch":239,"state":"active+clean","last_fresh":"2023-09-23T20:59:21.453812+0000","last_change":"2023-09-23T20:56:35.545128+0000","last_active":"2023-09-23T20:59:21.453812+0000","last_peered":"2023-09-23T20:59:21.453812+0000","last_clean":"2023-09-23T20:59:21.453812+0000","last_became_active":"2023-09-23T20:56:35.544838+0000","last_became_peered":"2023-09-23T20:56:35.544838+0000","last_unstale":"2023-09-23T20:59:21.453812+0000","last_undegraded":"2023-09-23T20:59:21.453812+0000","last_fullsized":"2023-09-23T20:59:21.453812+0000","mapping_epoch":199,"log_start":"205'1900","ondisk_log_start":"205'1900","created":199,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_clean_scrub_stamp":"2023-09-23T20:56:34.529103+0000","objects_scrubbed":0,"log_size":7348,"log_dups_size":0,"ondisk_log_size":7348,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:30:43.388717+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":237109248,"num_objects":3618,"num_object_clones":0,"num_object_copies":10854,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3618,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9248,"num_write_kb":411712,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.9","version":"79'161","reported_seq":612,"reported_epoch":238,"state":"active+clean","last_fresh":"2023-09-23T20:59:08.297942+0000","last_change":"2023-09-23T20:56:46.921939+0000","last_active":"2023-09-23T20:59:08.297942+0000","last_peered":"2023-09-23T20:59:08.297942+0000","last_clean":"2023-09-23T20:59:08.297942+0000","last_became_active":"2023-09-23T20:56:45.989908+0000","last_became_peered":"2023-09-23T20:56:45.989908+0000","last_unstale":"2023-09-23T20:59:08.297942+0000","last_undegraded":"2023-09-23T20:59:08.297942+0000","last_fullsized":"2023-09-23T20:59:08.297942+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":205,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:56:46.921739+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:56:46.921739+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:20:42.635110+0000","scrub_duration":0.0071954109999999996,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":228,"reported_epoch":238,"state":"active+clean","last_fresh":"2023-09-23T20:59:08.296294+0000","last_change":"2023-09-23T20:54:13.961788+0000","last_active":"2023-09-23T20:59:08.296294+0000","last_peered":"2023-09-23T20:59:08.296294+0000","last_clean":"2023-09-23T20:59:08.296294+0000","last_became_active":"2023-09-23T20:54:13.940937+0000","last_became_peered":"2023-09-23T20:54:13.940937+0000","last_unstale":"2023-09-23T20:59:08.296294+0000","last_undegraded":"2023-09-23T20:59:08.296294+0000","last_fullsized":"2023-09-23T20:59:08.296294+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:11:56.785299+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":133,"reported_epoch":238,"state":"active+clean","last_fresh":"2023-09-23T20:59:08.300491+0000","last_change":"2023-09-23T20:55:35.949670+0000","last_active":"2023-09-23T20:59:08.300491+0000","last_peered":"2023-09-23T20:59:08.300491+0000","last_clean":"2023-09-23T20:59:08.300491+0000","last_became_active":"2023-09-23T20:55:35.949140+0000","last_became_peered":"2023-09-23T20:55:35.949140+0000","last_unstale":"2023-09-23T20:59:08.300491+0000","last_undegraded":"2023-09-23T20:59:08.300491+0000","last_fullsized":"2023-09-23T20:59:08.300491+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:55:59.340303+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.e","version":"239'9108","reported_seq":9158,"reported_epoch":239,"state":"active+clean","last_fresh":"2023-09-23T20:59:21.401743+0000","last_change":"2023-09-23T20:56:35.547184+0000","last_active":"2023-09-23T20:59:21.401743+0000","last_peered":"2023-09-23T20:59:21.401743+0000","last_clean":"2023-09-23T20:59:21.401743+0000","last_became_active":"2023-09-23T20:56:35.546912+0000","last_became_peered":"2023-09-23T20:56:35.546912+0000","last_unstale":"2023-09-23T20:59:21.401743+0000","last_undegraded":"2023-09-23T20:59:21.401743+0000","last_fullsized":"2023-09-23T20:59:21.401743+0000","mapping_epoch":199,"log_start":"202'200","ondisk_log_start":"202'200","created":199,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_clean_scrub_stamp":"2023-09-23T20:56:34.529103+0000","objects_scrubbed":0,"log_size":8908,"log_dups_size":0,"ondisk_log_size":8908,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:57:07.422792+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":240648192,"num_objects":3672,"num_object_clones":0,"num_object_copies":11016,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3672,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9108,"num_write_kb":408960,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,0],"acting":[2,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.8","version":"79'161","reported_seq":482,"reported_epoch":238,"state":"active+clean","last_fresh":"2023-09-23T20:59:08.297407+0000","last_change":"2023-09-23T20:56:57.505737+0000","last_active":"2023-09-23T20:59:08.297407+0000","last_peered":"2023-09-23T20:59:08.297407+0000","last_clean":"2023-09-23T20:59:08.297407+0000","last_became_active":"2023-09-23T20:56:04.982345+0000","last_became_peered":"2023-09-23T20:56:04.982345+0000","last_unstale":"2023-09-23T20:59:08.297407+0000","last_undegraded":"2023-09-23T20:59:08.297407+0000","last_fullsized":"2023-09-23T20:59:08.297407+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:56:57.505663+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:56:57.505663+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:34:30.109724+0000","scrub_duration":0.026284108,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":201,"reported_epoch":238,"state":"active+clean","last_fresh":"2023-09-23T20:59:08.287105+0000","last_change":"2023-09-23T20:55:35.959942+0000","last_active":"2023-09-23T20:59:08.287105+0000","last_peered":"2023-09-23T20:59:08.287105+0000","last_clean":"2023-09-23T20:59:08.287105+0000","last_became_active":"2023-09-23T20:55:35.957071+0000","last_became_peered":"2023-09-23T20:55:35.957071+0000","last_unstale":"2023-09-23T20:59:08.287105+0000","last_undegraded":"2023-09-23T20:59:08.287105+0000","last_fullsized":"2023-09-23T20:59:08.287105+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:44:30.308036+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":116,"reported_epoch":238,"state":"active+clean","last_fresh":"2023-09-23T20:59:08.300516+0000","last_change":"2023-09-23T20:54:13.951459+0000","last_active":"2023-09-23T20:59:08.300516+0000","last_peered":"2023-09-23T20:59:08.300516+0000","last_clean":"2023-09-23T20:59:08.300516+0000","last_became_active":"2023-09-23T20:54:13.948409+0000","last_became_peered":"2023-09-23T20:54:13.948409+0000","last_unstale":"2023-09-23T20:59:08.300516+0000","last_undegraded":"2023-09-23T20:59:08.300516+0000","last_fullsized":"2023-09-23T20:59:08.300516+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:38:10.148061+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.b","version":"79'161","reported_seq":521,"reported_epoch":238,"state":"active+clean","last_fresh":"2023-09-23T20:59:08.288160+0000","last_change":"2023-09-23T20:57:27.320172+0000","last_active":"2023-09-23T20:59:08.288160+0000","last_peered":"2023-09-23T20:59:08.288160+0000","last_clean":"2023-09-23T20:59:08.288160+0000","last_became_active":"2023-09-23T20:57:16.009630+0000","last_became_peered":"2023-09-23T20:57:16.009630+0000","last_unstale":"2023-09-23T20:59:08.288160+0000","last_undegraded":"2023-09-23T20:59:08.288160+0000","last_fullsized":"2023-09-23T20:59:08.288160+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:27.320022+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:27.320022+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:31:35.221558+0000","scrub_duration":0.0043894679999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,5],"acting":[1,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.d","version":"239'9064","reported_seq":9113,"reported_epoch":239,"state":"active+clean","last_fresh":"2023-09-23T20:59:19.057214+0000","last_change":"2023-09-23T20:56:35.544132+0000","last_active":"2023-09-23T20:59:19.057214+0000","last_peered":"2023-09-23T20:59:19.057214+0000","last_clean":"2023-09-23T20:59:19.057214+0000","last_became_active":"2023-09-23T20:56:35.543836+0000","last_became_peered":"2023-09-23T20:56:35.543836+0000","last_unstale":"2023-09-23T20:59:19.057214+0000","last_undegraded":"2023-09-23T20:59:19.057214+0000","last_fullsized":"2023-09-23T20:59:19.057214+0000","mapping_epoch":199,"log_start":"205'1700","ondisk_log_start":"205'1700","created":199,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_clean_scrub_stamp":"2023-09-23T20:56:34.529103+0000","objects_scrubbed":0,"log_size":7364,"log_dups_size":0,"ondisk_log_size":7364,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:27:14.636991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":244121624,"num_objects":3726,"num_object_clones":0,"num_object_copies":11178,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3726,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":9064,"num_write_kb":409217,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":116,"reported_epoch":238,"state":"active+clean","last_fresh":"2023-09-23T20:59:08.301893+0000","last_change":"2023-09-23T20:54:13.950254+0000","last_active":"2023-09-23T20:59:08.301893+0000","last_peered":"2023-09-23T20:59:08.301893+0000","last_clean":"2023-09-23T20:59:08.301893+0000","last_became_active":"2023-09-23T20:54:13.949934+0000","last_became_peered":"2023-09-23T20:54:13.949934+0000","last_unstale":"2023-09-23T20:59:08.301893+0000","last_undegraded":"2023-09-23T20:59:08.301893+0000","last_fullsized":"2023-09-23T20:59:08.301893+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:30:32.847167+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":133,"reported_epoch":238,"state":"active+clean","last_fresh":"2023-09-23T20:59:08.300801+0000","last_change":"2023-09-23T20:55:35.956153+0000","last_active":"2023-09-23T20:59:08.300801+0000","last_peered":"2023-09-23T20:59:08.300801+0000","last_clean":"2023-09-23T20:59:08.300801+0000","last_became_active":"2023-09-23T20:55:35.955630+0000","last_became_peered":"2023-09-23T20:55:35.955630+0000","last_unstale":"2023-09-23T20:59:08.300801+0000","last_undegraded":"2023-09-23T20:59:08.300801+0000","last_fullsized":"2023-09-23T20:59:08.300801+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:45:20.282929+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.c","version":"239'8993","reported_seq":9051,"reported_epoch":239,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T20:59:18.811465+0000","last_change":"2023-09-23T20:57:46.463005+0000","last_active":"2023-09-23T20:59:18.811465+0000","last_peered":"2023-09-23T20:59:18.811465+0000","last_clean":"2023-09-23T20:57:44.015119+0000","last_became_active":"2023-09-23T20:57:46.448702+0000","last_became_peered":"2023-09-23T20:57:46.448702+0000","last_unstale":"2023-09-23T20:59:18.811465+0000","last_undegraded":"2023-09-23T20:57:46.010874+0000","last_fullsized":"2023-09-23T20:57:45.978434+0000","mapping_epoch":223,"log_start":"205'1600","ondisk_log_start":"205'1600","created":199,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_clean_scrub_stamp":"2023-09-23T20:56:34.529103+0000","objects_scrubbed":0,"log_size":7393,"log_dups_size":0,"ondisk_log_size":7393,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:08:34.180651+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":250019840,"num_objects":3815,"num_object_clones":0,"num_object_copies":11445,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5473,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3815,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8993,"num_write_kb":409856,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,7],"avail_no_missing":["6","2","7"],"object_location_counts":[{"shards":"2,6,7","objects":2884},{"shards":"6,7","objects":-1658}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.a","version":"79'161","reported_seq":482,"reported_epoch":238,"state":"active+clean","last_fresh":"2023-09-23T20:59:08.297757+0000","last_change":"2023-09-23T20:56:44.487778+0000","last_active":"2023-09-23T20:59:08.297757+0000","last_peered":"2023-09-23T20:59:08.297757+0000","last_clean":"2023-09-23T20:59:08.297757+0000","last_became_active":"2023-09-23T20:56:05.006740+0000","last_became_peered":"2023-09-23T20:56:05.006740+0000","last_unstale":"2023-09-23T20:59:08.297757+0000","last_undegraded":"2023-09-23T20:59:08.297757+0000","last_fullsized":"2023-09-23T20:59:08.297757+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:56:44.487678+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:56:44.487678+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:04:50.086749+0000","scrub_duration":0.0017408899999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":27,"reported_epoch":238,"state":"active+clean","last_fresh":"2023-09-23T20:59:08.295435+0000","last_change":"2023-09-23T20:57:44.982513+0000","last_active":"2023-09-23T20:59:08.295435+0000","last_peered":"2023-09-23T20:59:08.295435+0000","last_clean":"2023-09-23T20:59:08.295435+0000","last_became_active":"2023-09-23T20:57:44.982112+0000","last_became_peered":"2023-09-23T20:57:44.982112+0000","last_unstale":"2023-09-23T20:59:08.295435+0000","last_undegraded":"2023-09-23T20:59:08.295435+0000","last_fullsized":"2023-09-23T20:59:08.295435+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:09:05.630041+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":209,"reported_epoch":238,"state":"active+clean","last_fresh":"2023-09-23T20:59:08.287922+0000","last_change":"2023-09-23T20:54:13.940748+0000","last_active":"2023-09-23T20:59:08.287922+0000","last_peered":"2023-09-23T20:59:08.287922+0000","last_clean":"2023-09-23T20:59:08.287922+0000","last_became_active":"2023-09-23T20:54:13.940419+0000","last_became_peered":"2023-09-23T20:54:13.940419+0000","last_unstale":"2023-09-23T20:59:08.287922+0000","last_undegraded":"2023-09-23T20:59:08.287922+0000","last_fullsized":"2023-09-23T20:59:08.287922+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:15:01.923816+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.1","version":"239'9118","reported_seq":9179,"reported_epoch":239,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T20:59:22.106422+0000","last_change":"2023-09-23T20:57:46.483248+0000","last_active":"2023-09-23T20:59:22.106422+0000","last_peered":"2023-09-23T20:59:22.106422+0000","last_clean":"2023-09-23T20:57:44.047957+0000","last_became_active":"2023-09-23T20:57:46.475568+0000","last_became_peered":"2023-09-23T20:57:46.475568+0000","last_unstale":"2023-09-23T20:59:22.106422+0000","last_undegraded":"2023-09-23T20:57:46.083591+0000","last_fullsized":"2023-09-23T20:57:45.978436+0000","mapping_epoch":223,"log_start":"206'2400","ondisk_log_start":"206'2400","created":199,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_clean_scrub_stamp":"2023-09-23T20:56:34.529103+0000","objects_scrubbed":0,"log_size":6718,"log_dups_size":0,"ondisk_log_size":6718,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:22:45.067269+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":240386048,"num_objects":3668,"num_object_clones":0,"num_object_copies":11004,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5470,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3668,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9118,"num_write_kb":409152,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,6],"avail_no_missing":["4","1","6"],"object_location_counts":[{"shards":"1,4,6","objects":2745},{"shards":"4,6","objects":-1802}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.7","version":"79'161","reported_seq":544,"reported_epoch":238,"state":"active+clean","last_fresh":"2023-09-23T20:59:08.287511+0000","last_change":"2023-09-23T20:57:48.243572+0000","last_active":"2023-09-23T20:59:08.287511+0000","last_peered":"2023-09-23T20:59:08.287511+0000","last_clean":"2023-09-23T20:59:08.287511+0000","last_became_active":"2023-09-23T20:57:48.243229+0000","last_became_peered":"2023-09-23T20:57:48.243229+0000","last_unstale":"2023-09-23T20:59:08.287511+0000","last_undegraded":"2023-09-23T20:59:08.287511+0000","last_fullsized":"2023-09-23T20:59:08.287511+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":3,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:25.916867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:25.916867+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:23:19.385831+0000","scrub_duration":0.001665323,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":214,"reported_epoch":238,"state":"active+clean","last_fresh":"2023-09-23T20:59:08.288660+0000","last_change":"2023-09-23T20:50:05.867130+0000","last_active":"2023-09-23T20:59:08.288660+0000","last_peered":"2023-09-23T20:59:08.288660+0000","last_clean":"2023-09-23T20:59:08.288660+0000","last_became_active":"2023-09-23T20:50:05.866758+0000","last_became_peered":"2023-09-23T20:50:05.866758+0000","last_unstale":"2023-09-23T20:59:08.288660+0000","last_undegraded":"2023-09-23T20:59:08.288660+0000","last_fullsized":"2023-09-23T20:59:08.288660+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:55:07.548209+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":112,"reported_epoch":238,"state":"active+clean","last_fresh":"2023-09-23T20:59:08.297804+0000","last_change":"2023-09-23T20:54:31.887604+0000","last_active":"2023-09-23T20:59:08.297804+0000","last_peered":"2023-09-23T20:59:08.297804+0000","last_clean":"2023-09-23T20:59:08.297804+0000","last_became_active":"2023-09-23T20:54:31.886752+0000","last_became_peered":"2023-09-23T20:54:31.886752+0000","last_unstale":"2023-09-23T20:59:08.297804+0000","last_undegraded":"2023-09-23T20:59:08.297804+0000","last_fullsized":"2023-09-23T20:59:08.297804+0000","mapping_epoch":136,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:13:58.872176+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":212,"reported_epoch":238,"state":"active+clean","last_fresh":"2023-09-23T20:59:08.287402+0000","last_change":"2023-09-23T20:54:31.899082+0000","last_active":"2023-09-23T20:59:08.287402+0000","last_peered":"2023-09-23T20:59:08.287402+0000","last_clean":"2023-09-23T20:59:08.287402+0000","last_became_active":"2023-09-23T20:54:31.898521+0000","last_became_peered":"2023-09-23T20:54:31.898521+0000","last_unstale":"2023-09-23T20:59:08.287402+0000","last_undegraded":"2023-09-23T20:59:08.287402+0000","last_fullsized":"2023-09-23T20:59:08.287402+0000","mapping_epoch":136,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:25:36.059711+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.7","version":"239'9051","reported_seq":9099,"reported_epoch":239,"state":"active+clean","last_fresh":"2023-09-23T20:59:18.821743+0000","last_change":"2023-09-23T20:56:35.541695+0000","last_active":"2023-09-23T20:59:18.821743+0000","last_peered":"2023-09-23T20:59:18.821743+0000","last_clean":"2023-09-23T20:59:18.821743+0000","last_became_active":"2023-09-23T20:56:35.541392+0000","last_became_peered":"2023-09-23T20:56:35.541392+0000","last_unstale":"2023-09-23T20:59:18.821743+0000","last_undegraded":"2023-09-23T20:59:18.821743+0000","last_fullsized":"2023-09-23T20:59:18.821743+0000","mapping_epoch":199,"log_start":"205'1700","ondisk_log_start":"205'1700","created":199,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_clean_scrub_stamp":"2023-09-23T20:56:34.529103+0000","objects_scrubbed":0,"log_size":7351,"log_dups_size":0,"ondisk_log_size":7351,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:57:22.478253+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":246349824,"num_objects":3759,"num_object_clones":0,"num_object_copies":11277,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3759,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9051,"num_write_kb":409920,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.1","version":"79'161","reported_seq":502,"reported_epoch":238,"state":"active+clean","last_fresh":"2023-09-23T20:59:08.297396+0000","last_change":"2023-09-23T20:57:19.669346+0000","last_active":"2023-09-23T20:59:08.297396+0000","last_peered":"2023-09-23T20:59:08.297396+0000","last_clean":"2023-09-23T20:59:08.297396+0000","last_became_active":"2023-09-23T20:57:08.246046+0000","last_became_peered":"2023-09-23T20:57:08.246046+0000","last_unstale":"2023-09-23T20:59:08.297396+0000","last_undegraded":"2023-09-23T20:59:08.297396+0000","last_fullsized":"2023-09-23T20:59:08.297396+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:19.669259+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:19.669259+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:14:03.435531+0000","scrub_duration":0.0045759559999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":129,"reported_epoch":238,"state":"active+clean","last_fresh":"2023-09-23T20:59:08.297548+0000","last_change":"2023-09-23T20:55:35.959514+0000","last_active":"2023-09-23T20:59:08.297548+0000","last_peered":"2023-09-23T20:59:08.297548+0000","last_clean":"2023-09-23T20:59:08.297548+0000","last_became_active":"2023-09-23T20:55:35.959244+0000","last_became_peered":"2023-09-23T20:55:35.959244+0000","last_unstale":"2023-09-23T20:59:08.297548+0000","last_undegraded":"2023-09-23T20:59:08.297548+0000","last_fullsized":"2023-09-23T20:59:08.297548+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:47:32.665316+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":133,"reported_epoch":238,"state":"active+clean","last_fresh":"2023-09-23T20:59:08.301561+0000","last_change":"2023-09-23T20:55:35.949000+0000","last_active":"2023-09-23T20:59:08.301561+0000","last_peered":"2023-09-23T20:59:08.301561+0000","last_clean":"2023-09-23T20:59:08.301561+0000","last_became_active":"2023-09-23T20:55:35.948410+0000","last_became_peered":"2023-09-23T20:55:35.948410+0000","last_unstale":"2023-09-23T20:59:08.301561+0000","last_undegraded":"2023-09-23T20:59:08.301561+0000","last_fullsized":"2023-09-23T20:59:08.301561+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:26:50.477289+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":206,"reported_epoch":238,"state":"active+clean","last_fresh":"2023-09-23T20:59:08.288431+0000","last_change":"2023-09-23T20:51:10.829889+0000","last_active":"2023-09-23T20:59:08.288431+0000","last_peered":"2023-09-23T20:59:08.288431+0000","last_clean":"2023-09-23T20:59:08.288431+0000","last_became_active":"2023-09-23T20:51:10.829533+0000","last_became_peered":"2023-09-23T20:51:10.829533+0000","last_unstale":"2023-09-23T20:59:08.288431+0000","last_undegraded":"2023-09-23T20:59:08.288431+0000","last_fullsized":"2023-09-23T20:59:08.288431+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:43:04.603460+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.4","version":"239'9054","reported_seq":9104,"reported_epoch":239,"state":"active+clean","last_fresh":"2023-09-23T20:59:21.538210+0000","last_change":"2023-09-23T20:56:35.549909+0000","last_active":"2023-09-23T20:59:21.538210+0000","last_peered":"2023-09-23T20:59:21.538210+0000","last_clean":"2023-09-23T20:59:21.538210+0000","last_became_active":"2023-09-23T20:56:35.549618+0000","last_became_peered":"2023-09-23T20:56:35.549618+0000","last_unstale":"2023-09-23T20:59:21.538210+0000","last_undegraded":"2023-09-23T20:59:21.538210+0000","last_fullsized":"2023-09-23T20:59:21.538210+0000","mapping_epoch":199,"log_start":"205'1700","ondisk_log_start":"205'1700","created":199,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_clean_scrub_stamp":"2023-09-23T20:56:34.529103+0000","objects_scrubbed":0,"log_size":7354,"log_dups_size":0,"ondisk_log_size":7354,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:24:25.918475+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":240386048,"num_objects":3668,"num_object_clones":0,"num_object_copies":11004,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3668,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9054,"num_write_kb":407104,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.2","version":"79'161","reported_seq":469,"reported_epoch":238,"state":"active+clean","last_fresh":"2023-09-23T20:59:08.307878+0000","last_change":"2023-09-23T20:57:24.802586+0000","last_active":"2023-09-23T20:59:08.307878+0000","last_peered":"2023-09-23T20:59:08.307878+0000","last_clean":"2023-09-23T20:59:08.307878+0000","last_became_active":"2023-09-23T20:57:08.236114+0000","last_became_peered":"2023-09-23T20:57:08.236114+0000","last_unstale":"2023-09-23T20:59:08.307878+0000","last_undegraded":"2023-09-23T20:59:08.307878+0000","last_fullsized":"2023-09-23T20:59:08.307878+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:24.802451+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:24.802451+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:06:22.309980+0000","scrub_duration":0.0090861250000000005,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,4],"acting":[0,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":225,"reported_epoch":238,"state":"active+clean","last_fresh":"2023-09-23T20:59:08.310754+0000","last_change":"2023-09-23T20:51:10.838054+0000","last_active":"2023-09-23T20:59:08.310754+0000","last_peered":"2023-09-23T20:59:08.310754+0000","last_clean":"2023-09-23T20:59:08.310754+0000","last_became_active":"2023-09-23T20:51:10.837750+0000","last_became_peered":"2023-09-23T20:51:10.837750+0000","last_unstale":"2023-09-23T20:59:08.310754+0000","last_undegraded":"2023-09-23T20:59:08.310754+0000","last_fullsized":"2023-09-23T20:59:08.310754+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:15:51.648085+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":206,"reported_epoch":238,"state":"active+clean","last_fresh":"2023-09-23T20:59:08.287287+0000","last_change":"2023-09-23T20:55:35.951146+0000","last_active":"2023-09-23T20:59:08.287287+0000","last_peered":"2023-09-23T20:59:08.287287+0000","last_clean":"2023-09-23T20:59:08.287287+0000","last_became_active":"2023-09-23T20:55:35.950874+0000","last_became_peered":"2023-09-23T20:55:35.950874+0000","last_unstale":"2023-09-23T20:59:08.287287+0000","last_undegraded":"2023-09-23T20:59:08.287287+0000","last_fullsized":"2023-09-23T20:59:08.287287+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:58:13.957677+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":227,"reported_epoch":238,"state":"active+clean","last_fresh":"2023-09-23T20:59:08.287843+0000","last_change":"2023-09-23T20:54:31.898870+0000","last_active":"2023-09-23T20:59:08.287843+0000","last_peered":"2023-09-23T20:59:08.287843+0000","last_clean":"2023-09-23T20:59:08.287843+0000","last_became_active":"2023-09-23T20:54:31.898467+0000","last_became_peered":"2023-09-23T20:54:31.898467+0000","last_unstale":"2023-09-23T20:59:08.287843+0000","last_undegraded":"2023-09-23T20:59:08.287843+0000","last_fullsized":"2023-09-23T20:59:08.287843+0000","mapping_epoch":136,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:56:22.483408+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.6","version":"239'8965","reported_seq":13238,"reported_epoch":239,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-09-23T20:59:21.547187+0000","last_change":"2023-09-23T20:58:39.453197+0000","last_active":"2023-09-23T20:59:21.547187+0000","last_peered":"2023-09-23T20:59:21.547187+0000","last_clean":"2023-09-23T20:57:44.031597+0000","last_became_active":"2023-09-23T20:57:46.443927+0000","last_became_peered":"2023-09-23T20:57:46.443927+0000","last_unstale":"2023-09-23T20:59:21.547187+0000","last_undegraded":"2023-09-23T20:58:39.450464+0000","last_fullsized":"2023-09-23T20:57:45.977227+0000","mapping_epoch":223,"log_start":"205'1600","ondisk_log_start":"205'1600","created":199,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_clean_scrub_stamp":"2023-09-23T20:56:34.529103+0000","objects_scrubbed":0,"log_size":7365,"log_dups_size":0,"ondisk_log_size":7365,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:37:23.911131+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":237436928,"num_objects":3623,"num_object_clones":0,"num_object_copies":10869,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":570,"num_objects_misplaced":2721,"num_objects_unfound":0,"num_objects_dirty":3623,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8965,"num_write_kb":402816,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2107,"num_bytes_recovered":138084352,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7],"avail_no_missing":["0","5","7"],"object_location_counts":[{"shards":"0,5,7","objects":2721},{"shards":"0,7","objects":332}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":112,"reported_epoch":238,"state":"active+clean","last_fresh":"2023-09-23T20:59:08.297769+0000","last_change":"2023-09-23T20:54:31.883504+0000","last_active":"2023-09-23T20:59:08.297769+0000","last_peered":"2023-09-23T20:59:08.297769+0000","last_clean":"2023-09-23T20:59:08.297769+0000","last_became_active":"2023-09-23T20:54:31.883172+0000","last_became_peered":"2023-09-23T20:54:31.883172+0000","last_unstale":"2023-09-23T20:59:08.297769+0000","last_undegraded":"2023-09-23T20:59:08.297769+0000","last_fullsized":"2023-09-23T20:59:08.297769+0000","mapping_epoch":136,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:45:40.289120+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"79'161","reported_seq":469,"reported_epoch":238,"state":"active+clean","last_fresh":"2023-09-23T20:59:08.298385+0000","last_change":"2023-09-23T20:57:20.738417+0000","last_active":"2023-09-23T20:59:08.298385+0000","last_peered":"2023-09-23T20:59:08.298385+0000","last_clean":"2023-09-23T20:59:08.298385+0000","last_became_active":"2023-09-23T20:57:08.236842+0000","last_became_peered":"2023-09-23T20:57:08.236842+0000","last_unstale":"2023-09-23T20:59:08.298385+0000","last_undegraded":"2023-09-23T20:59:08.298385+0000","last_fullsized":"2023-09-23T20:59:08.298385+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":0,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:20.738273+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:20.738273+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:18:28.327990+0000","scrub_duration":0.0088336339999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":216,"reported_epoch":238,"state":"active+clean","last_fresh":"2023-09-23T20:59:08.296334+0000","last_change":"2023-09-23T20:55:35.950777+0000","last_active":"2023-09-23T20:59:08.296334+0000","last_peered":"2023-09-23T20:59:08.296334+0000","last_clean":"2023-09-23T20:59:08.296334+0000","last_became_active":"2023-09-23T20:55:35.950427+0000","last_became_peered":"2023-09-23T20:55:35.950427+0000","last_unstale":"2023-09-23T20:59:08.296334+0000","last_undegraded":"2023-09-23T20:59:08.296334+0000","last_fullsized":"2023-09-23T20:59:08.296334+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:20:10.371751+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":183,"reported_epoch":238,"state":"active+clean","last_fresh":"2023-09-23T20:59:08.287055+0000","last_change":"2023-09-23T20:55:35.978140+0000","last_active":"2023-09-23T20:59:08.287055+0000","last_peered":"2023-09-23T20:59:08.287055+0000","last_clean":"2023-09-23T20:59:08.287055+0000","last_became_active":"2023-09-23T20:55:35.977831+0000","last_became_peered":"2023-09-23T20:55:35.977831+0000","last_unstale":"2023-09-23T20:59:08.287055+0000","last_undegraded":"2023-09-23T20:59:08.287055+0000","last_fullsized":"2023-09-23T20:59:08.287055+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:25:47.678023+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.5","version":"239'9026","reported_seq":9083,"reported_epoch":239,"state":"active+clean","last_fresh":"2023-09-23T20:59:21.449427+0000","last_change":"2023-09-23T20:56:45.990110+0000","last_active":"2023-09-23T20:59:21.449427+0000","last_peered":"2023-09-23T20:59:21.449427+0000","last_clean":"2023-09-23T20:59:21.449427+0000","last_became_active":"2023-09-23T20:56:45.988394+0000","last_became_peered":"2023-09-23T20:56:45.988394+0000","last_unstale":"2023-09-23T20:59:21.449427+0000","last_undegraded":"2023-09-23T20:59:21.449427+0000","last_fullsized":"2023-09-23T20:59:21.449427+0000","mapping_epoch":204,"log_start":"202'200","ondisk_log_start":"202'200","created":199,"last_epoch_clean":205,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_clean_scrub_stamp":"2023-09-23T20:56:34.529103+0000","objects_scrubbed":0,"log_size":8826,"log_dups_size":0,"ondisk_log_size":8826,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:22:03.814210+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":239337472,"num_objects":3652,"num_object_clones":0,"num_object_copies":10956,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3652,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9026,"num_write_kb":405696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,0],"acting":[2,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"79'161","reported_seq":505,"reported_epoch":238,"state":"active+clean","last_fresh":"2023-09-23T20:59:08.288309+0000","last_change":"2023-09-23T20:57:20.378531+0000","last_active":"2023-09-23T20:59:08.288309+0000","last_peered":"2023-09-23T20:59:08.288309+0000","last_clean":"2023-09-23T20:59:08.288309+0000","last_became_active":"2023-09-23T20:57:08.238019+0000","last_became_peered":"2023-09-23T20:57:08.238019+0000","last_unstale":"2023-09-23T20:59:08.288309+0000","last_undegraded":"2023-09-23T20:59:08.288309+0000","last_fullsized":"2023-09-23T20:59:08.288309+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":3,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:20.377714+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:20.377714+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:00:08.305602+0000","scrub_duration":0.003015228,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":108,"reported_epoch":238,"state":"active+clean","last_fresh":"2023-09-23T20:59:08.287417+0000","last_change":"2023-09-23T20:56:45.991884+0000","last_active":"2023-09-23T20:59:08.287417+0000","last_peered":"2023-09-23T20:59:08.287417+0000","last_clean":"2023-09-23T20:59:08.287417+0000","last_became_active":"2023-09-23T20:56:45.991568+0000","last_became_peered":"2023-09-23T20:56:45.991568+0000","last_unstale":"2023-09-23T20:59:08.287417+0000","last_undegraded":"2023-09-23T20:59:08.287417+0000","last_fullsized":"2023-09-23T20:59:08.287417+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":205,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:21:20.775511+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,0],"acting":[1,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":232,"reported_epoch":238,"state":"active+clean","last_fresh":"2023-09-23T20:59:08.308358+0000","last_change":"2023-09-23T20:57:44.979950+0000","last_active":"2023-09-23T20:59:08.308358+0000","last_peered":"2023-09-23T20:59:08.308358+0000","last_clean":"2023-09-23T20:59:08.308358+0000","last_became_active":"2023-09-23T20:57:44.979346+0000","last_became_peered":"2023-09-23T20:57:44.979346+0000","last_unstale":"2023-09-23T20:59:08.308358+0000","last_undegraded":"2023-09-23T20:59:08.308358+0000","last_fullsized":"2023-09-23T20:59:08.308358+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:10:24.372800+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":211,"reported_epoch":238,"state":"active+clean","last_fresh":"2023-09-23T20:59:08.299696+0000","last_change":"2023-09-23T20:50:37.982235+0000","last_active":"2023-09-23T20:59:08.299696+0000","last_peered":"2023-09-23T20:59:08.299696+0000","last_clean":"2023-09-23T20:59:08.299696+0000","last_became_active":"2023-09-23T20:50:37.981913+0000","last_became_peered":"2023-09-23T20:50:37.981913+0000","last_unstale":"2023-09-23T20:59:08.299696+0000","last_undegraded":"2023-09-23T20:59:08.299696+0000","last_fullsized":"2023-09-23T20:59:08.299696+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:59:57.122584+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.0","version":"239'9080","reported_seq":9128,"reported_epoch":239,"state":"active+clean","last_fresh":"2023-09-23T20:59:22.109661+0000","last_change":"2023-09-23T20:56:35.550635+0000","last_active":"2023-09-23T20:59:22.109661+0000","last_peered":"2023-09-23T20:59:22.109661+0000","last_clean":"2023-09-23T20:59:22.109661+0000","last_became_active":"2023-09-23T20:56:35.550376+0000","last_became_peered":"2023-09-23T20:56:35.550376+0000","last_unstale":"2023-09-23T20:59:22.109661+0000","last_undegraded":"2023-09-23T20:59:22.109661+0000","last_fullsized":"2023-09-23T20:59:22.109661+0000","mapping_epoch":199,"log_start":"202'500","ondisk_log_start":"202'500","created":199,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_clean_scrub_stamp":"2023-09-23T20:56:34.529103+0000","objects_scrubbed":0,"log_size":8580,"log_dups_size":0,"ondisk_log_size":8580,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:40:55.020049+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":236060672,"num_objects":3602,"num_object_clones":0,"num_object_copies":10806,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3602,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9080,"num_write_kb":405824,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.6","version":"79'161","reported_seq":521,"reported_epoch":238,"state":"active+clean","last_fresh":"2023-09-23T20:59:08.287031+0000","last_change":"2023-09-23T20:57:26.907505+0000","last_active":"2023-09-23T20:59:08.287031+0000","last_peered":"2023-09-23T20:59:08.287031+0000","last_clean":"2023-09-23T20:59:08.287031+0000","last_became_active":"2023-09-23T20:57:08.240321+0000","last_became_peered":"2023-09-23T20:57:08.240321+0000","last_unstale":"2023-09-23T20:59:08.287031+0000","last_undegraded":"2023-09-23T20:59:08.287031+0000","last_fullsized":"2023-09-23T20:59:08.287031+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":3,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:26.907408+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:26.907408+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:34:17.392928+0000","scrub_duration":0.010337038,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":155,"reported_epoch":238,"state":"active+clean","last_fresh":"2023-09-23T20:59:08.313670+0000","last_change":"2023-09-23T20:56:45.988566+0000","last_active":"2023-09-23T20:59:08.313670+0000","last_peered":"2023-09-23T20:59:08.313670+0000","last_clean":"2023-09-23T20:59:08.313670+0000","last_became_active":"2023-09-23T20:56:45.988249+0000","last_became_peered":"2023-09-23T20:56:45.988249+0000","last_unstale":"2023-09-23T20:59:08.313670+0000","last_undegraded":"2023-09-23T20:59:08.313670+0000","last_fullsized":"2023-09-23T20:59:08.313670+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":205,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:51:36.644764+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,4],"acting":[0,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":209,"reported_epoch":238,"state":"active+clean","last_fresh":"2023-09-23T20:59:08.288584+0000","last_change":"2023-09-23T20:54:31.898965+0000","last_active":"2023-09-23T20:59:08.288584+0000","last_peered":"2023-09-23T20:59:08.288584+0000","last_clean":"2023-09-23T20:59:08.288584+0000","last_became_active":"2023-09-23T20:54:31.898663+0000","last_became_peered":"2023-09-23T20:54:31.898663+0000","last_unstale":"2023-09-23T20:59:08.288584+0000","last_undegraded":"2023-09-23T20:59:08.288584+0000","last_fullsized":"2023-09-23T20:59:08.288584+0000","mapping_epoch":136,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:53:05.058087+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":199,"reported_epoch":238,"state":"active+clean","last_fresh":"2023-09-23T20:59:08.305786+0000","last_change":"2023-09-23T20:54:31.884717+0000","last_active":"2023-09-23T20:59:08.305786+0000","last_peered":"2023-09-23T20:59:08.305786+0000","last_clean":"2023-09-23T20:59:08.305786+0000","last_became_active":"2023-09-23T20:54:31.884345+0000","last_became_peered":"2023-09-23T20:54:31.884345+0000","last_unstale":"2023-09-23T20:59:08.305786+0000","last_undegraded":"2023-09-23T20:59:08.305786+0000","last_fullsized":"2023-09-23T20:59:08.305786+0000","mapping_epoch":136,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:02:26.393868+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.3","version":"239'8807","reported_seq":8857,"reported_epoch":239,"state":"active+clean","last_fresh":"2023-09-23T20:59:19.540260+0000","last_change":"2023-09-23T20:56:35.549655+0000","last_active":"2023-09-23T20:59:19.540260+0000","last_peered":"2023-09-23T20:59:19.540260+0000","last_clean":"2023-09-23T20:59:19.540260+0000","last_became_active":"2023-09-23T20:56:35.549357+0000","last_became_peered":"2023-09-23T20:56:35.549357+0000","last_unstale":"2023-09-23T20:59:19.540260+0000","last_undegraded":"2023-09-23T20:59:19.540260+0000","last_fullsized":"2023-09-23T20:59:19.540260+0000","mapping_epoch":199,"log_start":"206'2600","ondisk_log_start":"206'2600","created":199,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_clean_scrub_stamp":"2023-09-23T20:56:34.529103+0000","objects_scrubbed":0,"log_size":6207,"log_dups_size":0,"ondisk_log_size":6207,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:37:58.752518+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":243466240,"num_objects":3715,"num_object_clones":0,"num_object_copies":11145,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3715,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8807,"num_write_kb":400704,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.5","version":"79'161","reported_seq":502,"reported_epoch":238,"state":"active+clean","last_fresh":"2023-09-23T20:59:08.300272+0000","last_change":"2023-09-23T20:57:30.029479+0000","last_active":"2023-09-23T20:59:08.300272+0000","last_peered":"2023-09-23T20:59:08.300272+0000","last_clean":"2023-09-23T20:59:08.300272+0000","last_became_active":"2023-09-23T20:57:08.249909+0000","last_became_peered":"2023-09-23T20:57:08.249909+0000","last_unstale":"2023-09-23T20:59:08.300272+0000","last_undegraded":"2023-09-23T20:59:08.300272+0000","last_fullsized":"2023-09-23T20:59:08.300272+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":3,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:30.029396+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:30.029396+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:16:46.083145+0000","scrub_duration":0.012914682,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":202,"reported_epoch":238,"state":"active+clean","last_fresh":"2023-09-23T20:59:08.297563+0000","last_change":"2023-09-23T20:51:10.836910+0000","last_active":"2023-09-23T20:59:08.297563+0000","last_peered":"2023-09-23T20:59:08.297563+0000","last_clean":"2023-09-23T20:59:08.297563+0000","last_became_active":"2023-09-23T20:51:10.836557+0000","last_became_peered":"2023-09-23T20:51:10.836557+0000","last_unstale":"2023-09-23T20:59:08.297563+0000","last_undegraded":"2023-09-23T20:59:08.297563+0000","last_fullsized":"2023-09-23T20:59:08.297563+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:51:54.019969+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":209,"reported_epoch":238,"state":"active+clean","last_fresh":"2023-09-23T20:59:08.287915+0000","last_change":"2023-09-23T20:55:35.953444+0000","last_active":"2023-09-23T20:59:08.287915+0000","last_peered":"2023-09-23T20:59:08.287915+0000","last_clean":"2023-09-23T20:59:08.287915+0000","last_became_active":"2023-09-23T20:55:35.953151+0000","last_became_peered":"2023-09-23T20:55:35.953151+0000","last_unstale":"2023-09-23T20:59:08.287915+0000","last_undegraded":"2023-09-23T20:59:08.287915+0000","last_fullsized":"2023-09-23T20:59:08.287915+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:39:12.404391+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":27,"reported_epoch":238,"state":"active+clean","last_fresh":"2023-09-23T20:59:08.297281+0000","last_change":"2023-09-23T20:57:44.982509+0000","last_active":"2023-09-23T20:59:08.297281+0000","last_peered":"2023-09-23T20:59:08.297281+0000","last_clean":"2023-09-23T20:59:08.297281+0000","last_became_active":"2023-09-23T20:57:44.982175+0000","last_became_peered":"2023-09-23T20:57:44.982175+0000","last_unstale":"2023-09-23T20:59:08.297281+0000","last_undegraded":"2023-09-23T20:59:08.297281+0000","last_fullsized":"2023-09-23T20:59:08.297281+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:37:49.215488+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.2","version":"239'8872","reported_seq":8921,"reported_epoch":239,"state":"active+clean","last_fresh":"2023-09-23T20:59:19.540255+0000","last_change":"2023-09-23T20:56:35.543020+0000","last_active":"2023-09-23T20:59:19.540255+0000","last_peered":"2023-09-23T20:59:19.540255+0000","last_clean":"2023-09-23T20:59:19.540255+0000","last_became_active":"2023-09-23T20:56:35.542719+0000","last_became_peered":"2023-09-23T20:56:35.542719+0000","last_unstale":"2023-09-23T20:59:19.540255+0000","last_undegraded":"2023-09-23T20:59:19.540255+0000","last_fullsized":"2023-09-23T20:59:19.540255+0000","mapping_epoch":199,"log_start":"206'2700","ondisk_log_start":"206'2700","created":199,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_clean_scrub_stamp":"2023-09-23T20:56:34.529103+0000","objects_scrubbed":0,"log_size":6172,"log_dups_size":0,"ondisk_log_size":6172,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:52:21.088628+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":238551040,"num_objects":3640,"num_object_clones":0,"num_object_copies":10920,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3640,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8872,"num_write_kb":400384,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.4","version":"79'161","reported_seq":517,"reported_epoch":238,"state":"active+clean","last_fresh":"2023-09-23T20:59:08.296646+0000","last_change":"2023-09-23T20:57:20.012904+0000","last_active":"2023-09-23T20:59:08.296646+0000","last_peered":"2023-09-23T20:59:08.296646+0000","last_clean":"2023-09-23T20:59:08.296646+0000","last_became_active":"2023-09-23T20:57:08.237996+0000","last_became_peered":"2023-09-23T20:57:08.237996+0000","last_unstale":"2023-09-23T20:59:08.296646+0000","last_undegraded":"2023-09-23T20:59:08.296646+0000","last_fullsized":"2023-09-23T20:59:08.296646+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":3,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:20.012764+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:20.012764+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:51:28.796946+0000","scrub_duration":0.0045251919999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":188,"reported_epoch":238,"state":"active+clean","last_fresh":"2023-09-23T20:59:08.287364+0000","last_change":"2023-09-23T20:55:35.949541+0000","last_active":"2023-09-23T20:59:08.287364+0000","last_peered":"2023-09-23T20:59:08.287364+0000","last_clean":"2023-09-23T20:59:08.287364+0000","last_became_active":"2023-09-23T20:55:35.944635+0000","last_became_peered":"2023-09-23T20:55:35.944635+0000","last_unstale":"2023-09-23T20:59:08.287364+0000","last_undegraded":"2023-09-23T20:59:08.287364+0000","last_fullsized":"2023-09-23T20:59:08.287364+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:30:36.033585+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"70'1","reported_seq":227,"reported_epoch":238,"state":"active+clean","last_fresh":"2023-09-23T20:59:08.299785+0000","last_change":"2023-09-23T20:54:14.009723+0000","last_active":"2023-09-23T20:59:08.299785+0000","last_peered":"2023-09-23T20:59:08.299785+0000","last_clean":"2023-09-23T20:59:08.299785+0000","last_became_active":"2023-09-23T20:54:13.941771+0000","last_became_peered":"2023-09-23T20:54:13.941771+0000","last_unstale":"2023-09-23T20:59:08.299785+0000","last_undegraded":"2023-09-23T20:59:08.299785+0000","last_fullsized":"2023-09-23T20:59:08.299785+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:48:25.588116+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":143,"reported_epoch":238,"state":"active+clean","last_fresh":"2023-09-23T20:59:08.301962+0000","last_change":"2023-09-23T20:55:35.949780+0000","last_active":"2023-09-23T20:59:08.301962+0000","last_peered":"2023-09-23T20:59:08.301962+0000","last_clean":"2023-09-23T20:59:08.301962+0000","last_became_active":"2023-09-23T20:55:35.949497+0000","last_became_peered":"2023-09-23T20:55:35.949497+0000","last_unstale":"2023-09-23T20:59:08.301962+0000","last_undegraded":"2023-09-23T20:59:08.301962+0000","last_fullsized":"2023-09-23T20:59:08.301962+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:09:37.420882+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":201,"reported_epoch":238,"state":"active+clean","last_fresh":"2023-09-23T20:59:08.305703+0000","last_change":"2023-09-23T20:54:13.929196+0000","last_active":"2023-09-23T20:59:08.305703+0000","last_peered":"2023-09-23T20:59:08.305703+0000","last_clean":"2023-09-23T20:59:08.305703+0000","last_became_active":"2023-09-23T20:54:13.928831+0000","last_became_peered":"2023-09-23T20:54:13.928831+0000","last_unstale":"2023-09-23T20:59:08.305703+0000","last_undegraded":"2023-09-23T20:59:08.305703+0000","last_fullsized":"2023-09-23T20:59:08.305703+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:28:44.316851+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":207,"reported_epoch":238,"state":"active+clean","last_fresh":"2023-09-23T20:59:08.309670+0000","last_change":"2023-09-23T20:52:12.855430+0000","last_active":"2023-09-23T20:59:08.309670+0000","last_peered":"2023-09-23T20:59:08.309670+0000","last_clean":"2023-09-23T20:59:08.309670+0000","last_became_active":"2023-09-23T20:52:12.855115+0000","last_became_peered":"2023-09-23T20:52:12.855115+0000","last_unstale":"2023-09-23T20:59:08.309670+0000","last_undegraded":"2023-09-23T20:59:08.309670+0000","last_fullsized":"2023-09-23T20:59:08.309670+0000","mapping_epoch":100,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":101,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:33:23.802214+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":116,"reported_epoch":238,"state":"active+clean","last_fresh":"2023-09-23T20:59:08.297839+0000","last_change":"2023-09-23T20:54:31.902946+0000","last_active":"2023-09-23T20:59:08.297839+0000","last_peered":"2023-09-23T20:59:08.297839+0000","last_clean":"2023-09-23T20:59:08.297839+0000","last_became_active":"2023-09-23T20:54:31.889484+0000","last_became_peered":"2023-09-23T20:54:31.889484+0000","last_unstale":"2023-09-23T20:59:08.297839+0000","last_undegraded":"2023-09-23T20:59:08.297839+0000","last_fullsized":"2023-09-23T20:59:08.297839+0000","mapping_epoch":136,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:44:44.278398+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":116,"reported_epoch":238,"state":"active+clean","last_fresh":"2023-09-23T20:59:08.301677+0000","last_change":"2023-09-23T20:54:13.950351+0000","last_active":"2023-09-23T20:59:08.301677+0000","last_peered":"2023-09-23T20:59:08.301677+0000","last_clean":"2023-09-23T20:59:08.301677+0000","last_became_active":"2023-09-23T20:54:13.949987+0000","last_became_peered":"2023-09-23T20:54:13.949987+0000","last_unstale":"2023-09-23T20:59:08.301677+0000","last_undegraded":"2023-09-23T20:59:08.301677+0000","last_fullsized":"2023-09-23T20:59:08.301677+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:46:17.765134+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":147,"reported_epoch":238,"state":"active+clean","last_fresh":"2023-09-23T20:59:08.297891+0000","last_change":"2023-09-23T20:56:18.141728+0000","last_active":"2023-09-23T20:59:08.297891+0000","last_peered":"2023-09-23T20:59:08.297891+0000","last_clean":"2023-09-23T20:59:08.297891+0000","last_became_active":"2023-09-23T20:56:18.137979+0000","last_became_peered":"2023-09-23T20:56:18.137979+0000","last_unstale":"2023-09-23T20:59:08.297891+0000","last_undegraded":"2023-09-23T20:59:08.297891+0000","last_fullsized":"2023-09-23T20:59:08.297891+0000","mapping_epoch":192,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":193,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:58:05.658443+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":116,"reported_epoch":238,"state":"active+clean","last_fresh":"2023-09-23T20:59:08.300109+0000","last_change":"2023-09-23T20:54:13.935307+0000","last_active":"2023-09-23T20:59:08.300109+0000","last_peered":"2023-09-23T20:59:08.300109+0000","last_clean":"2023-09-23T20:59:08.300109+0000","last_became_active":"2023-09-23T20:54:13.935000+0000","last_became_peered":"2023-09-23T20:54:13.935000+0000","last_unstale":"2023-09-23T20:59:08.300109+0000","last_undegraded":"2023-09-23T20:59:08.300109+0000","last_fullsized":"2023-09-23T20:59:08.300109+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:47:40.974082+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":226,"reported_epoch":238,"state":"active+clean","last_fresh":"2023-09-23T20:59:09.562834+0000","last_change":"2023-09-23T20:55:35.954870+0000","last_active":"2023-09-23T20:59:09.562834+0000","last_peered":"2023-09-23T20:59:09.562834+0000","last_clean":"2023-09-23T20:59:09.562834+0000","last_became_active":"2023-09-23T20:55:35.954568+0000","last_became_peered":"2023-09-23T20:55:35.954568+0000","last_unstale":"2023-09-23T20:59:09.562834+0000","last_undegraded":"2023-09-23T20:59:09.562834+0000","last_fullsized":"2023-09-23T20:59:09.562834+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:06:22.423566+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":225,"reported_epoch":238,"state":"active+clean","last_fresh":"2023-09-23T20:59:08.297859+0000","last_change":"2023-09-23T20:51:10.838334+0000","last_active":"2023-09-23T20:59:08.297859+0000","last_peered":"2023-09-23T20:59:08.297859+0000","last_clean":"2023-09-23T20:59:08.297859+0000","last_became_active":"2023-09-23T20:51:10.838026+0000","last_became_peered":"2023-09-23T20:51:10.838026+0000","last_unstale":"2023-09-23T20:59:08.297859+0000","last_undegraded":"2023-09-23T20:59:08.297859+0000","last_fullsized":"2023-09-23T20:59:08.297859+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:04:37.009457+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":229,"reported_epoch":238,"state":"active+clean","last_fresh":"2023-09-23T20:59:08.298700+0000","last_change":"2023-09-23T20:54:31.903139+0000","last_active":"2023-09-23T20:59:08.298700+0000","last_peered":"2023-09-23T20:59:08.298700+0000","last_clean":"2023-09-23T20:59:08.298700+0000","last_became_active":"2023-09-23T20:54:31.884520+0000","last_became_peered":"2023-09-23T20:54:31.884520+0000","last_unstale":"2023-09-23T20:59:08.298700+0000","last_undegraded":"2023-09-23T20:59:08.298700+0000","last_fullsized":"2023-09-23T20:59:08.298700+0000","mapping_epoch":136,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:39:34.269705+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1108,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":133,"reported_epoch":238,"state":"active+clean","last_fresh":"2023-09-23T20:59:08.299760+0000","last_change":"2023-09-23T20:55:35.951572+0000","last_active":"2023-09-23T20:59:08.299760+0000","last_peered":"2023-09-23T20:59:08.299760+0000","last_clean":"2023-09-23T20:59:08.299760+0000","last_became_active":"2023-09-23T20:55:35.951294+0000","last_became_peered":"2023-09-23T20:55:35.951294+0000","last_unstale":"2023-09-23T20:59:08.299760+0000","last_undegraded":"2023-09-23T20:59:08.299760+0000","last_fullsized":"2023-09-23T20:59:08.299760+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:36:32.447210+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":7,"num_pg":16,"stat_sum":{"num_bytes":3867148312,"num_objects":59009,"num_object_clones":0,"num_object_copies":177027,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":11513,"num_objects_misplaced":2721,"num_objects_unfound":0,"num_objects_dirty":59009,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":144511,"num_write_kb":6512577,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2108,"num_bytes_recovered":138149888,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12086620160,"data_stored":12086607944,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":119011,"ondisk_log_size":119011,"up":48,"acting":45,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":22,"num_bytes_recovered":5990,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":21,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3381,"ondisk_log_size":3381,"up":63,"acting":63,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201507,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1351544,"kb_used_data":1175728,"kb_used_omap":0,"kb_used_meta":175808,"kb_avail":92393608,"statfs":{"total":95995035648,"available":94611054592,"internally_reserved":0,"allocated":1203945472,"data_stored":1203097817,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":180027392},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:58:44 2023","interfaces":[{"interface":"back","average":{"1min":2.052,"5min":1.859,"15min":1.006},"min":{"1min":0.248,"5min":0.21099999999999999,"15min":0.21099999999999999},"max":{"1min":5.5279999999999996,"5min":43.994999999999997,"15min":43.994999999999997},"last":0.39500000000000002},{"interface":"front","average":{"1min":2.0489999999999999,"5min":2.0760000000000001,"15min":1.099},"min":{"1min":0.34599999999999997,"5min":0.28199999999999997,"15min":0.218},"max":{"1min":5.5129999999999999,"5min":44.206000000000003,"15min":44.206000000000003},"last":0.40600000000000003}]},{"osd":1,"last update":"Sat Sep 23 20:59:19 2023","interfaces":[{"interface":"back","average":{"1min":1.74,"5min":1.835,"15min":1.2230000000000001},"min":{"1min":0.26600000000000001,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":5.4359999999999999,"5min":10.622,"15min":22.591999999999999},"last":2.5779999999999998},{"interface":"front","average":{"1min":2.0600000000000001,"5min":1.639,"15min":1.0860000000000001},"min":{"1min":0.32100000000000001,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":6.7709999999999999,"5min":10.097,"15min":21.437000000000001},"last":1.167}]},{"osd":2,"last update":"Sat Sep 23 20:58:54 2023","interfaces":[{"interface":"back","average":{"1min":3.7370000000000001,"5min":2.016,"15min":1.1499999999999999},"min":{"1min":0.27600000000000002,"5min":0.26500000000000001,"15min":0.24399999999999999},"max":{"1min":10.805999999999999,"5min":10.805999999999999,"15min":10.805999999999999},"last":3.907},{"interface":"front","average":{"1min":2.3969999999999998,"5min":2.1520000000000001,"15min":1.1919999999999999},"min":{"1min":0.27800000000000002,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":9.4290000000000003,"5min":45.25,"15min":45.25},"last":1.71}]},{"osd":3,"last update":"Sat Sep 23 20:58:26 2023","interfaces":[{"interface":"back","average":{"1min":3.355,"5min":3.226,"15min":1.587},"min":{"1min":0.32100000000000001,"5min":0.314,"15min":0.314},"max":{"1min":9.9939999999999998,"5min":36.408000000000001,"15min":36.408000000000001},"last":2.6669999999999998},{"interface":"front","average":{"1min":2.5649999999999999,"5min":2.7160000000000002,"15min":1.405},"min":{"1min":0.32000000000000001,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":9.4529999999999994,"5min":35.593000000000004,"15min":35.593000000000004},"last":0.46700000000000003}]},{"osd":4,"last update":"Sat Sep 23 20:59:19 2023","interfaces":[{"interface":"back","average":{"1min":0.41099999999999998,"5min":0.78500000000000003,"15min":0.60499999999999998},"min":{"1min":0.25700000000000001,"5min":0.23799999999999999,"15min":0.23699999999999999},"max":{"1min":1.1659999999999999,"5min":5.8920000000000003,"15min":5.8920000000000003},"last":0.30599999999999999},{"interface":"front","average":{"1min":0.51900000000000002,"5min":0.77600000000000002,"15min":0.61299999999999999},"min":{"1min":0.28599999999999998,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":2.952,"5min":5.3200000000000003,"15min":5.3200000000000003},"last":0.34100000000000003}]},{"osd":5,"last update":"Sat Sep 23 20:59:19 2023","interfaces":[{"interface":"back","average":{"1min":0.443,"5min":0.78100000000000003,"15min":0.628},"min":{"1min":0.316,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":0.64000000000000001,"5min":7.0389999999999997,"15min":7.0389999999999997},"last":0.377},{"interface":"front","average":{"1min":0.55200000000000005,"5min":0.72299999999999998,"15min":0.61399999999999999},"min":{"1min":0.30199999999999999,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":1.909,"5min":4.5999999999999996,"15min":4.5999999999999996},"last":0.36099999999999999}]},{"osd":6,"last update":"Sat Sep 23 20:58:40 2023","interfaces":[{"interface":"back","average":{"1min":0.86599999999999999,"5min":0.71699999999999997,"15min":0.68300000000000005},"min":{"1min":0.33700000000000002,"5min":0.31,"15min":0.31},"max":{"1min":3.5249999999999999,"5min":3.5249999999999999,"15min":3.5249999999999999},"last":0.496},{"interface":"front","average":{"1min":0.90100000000000002,"5min":0.81299999999999994,"15min":0.80700000000000005},"min":{"1min":0.36899999999999999,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":3.5409999999999999,"5min":3.5409999999999999,"15min":3.5409999999999999},"last":0.439}]}]},{"osd":6,"up_from":164,"seq":704374636590,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1312236,"kb_used_data":1211172,"kb_used_omap":10,"kb_used_meta":101045,"kb_avail":92432916,"statfs":{"total":95995035648,"available":94651305984,"internally_reserved":0,"allocated":1240240128,"data_stored":1239404374,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":10328,"internal_metadata":103471016},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:58:43 2023","interfaces":[{"interface":"back","average":{"1min":2.7330000000000001,"5min":1.4410000000000001,"15min":1.0569999999999999},"min":{"1min":0.25800000000000001,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":7.5179999999999998,"5min":8.7319999999999993,"15min":8.7319999999999993},"last":0.69599999999999995},{"interface":"front","average":{"1min":3.1709999999999998,"5min":1.502,"15min":1.1559999999999999},"min":{"1min":0.25,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":6.7140000000000004,"5min":6.7140000000000004,"15min":6.7140000000000004},"last":3.6150000000000002}]},{"osd":1,"last update":"Sat Sep 23 20:58:43 2023","interfaces":[{"interface":"back","average":{"1min":1.5449999999999999,"5min":1.097,"15min":0.89100000000000001},"min":{"1min":0.36799999999999999,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":5.2350000000000003,"5min":6.6029999999999998,"15min":6.6029999999999998},"last":0.34499999999999997},{"interface":"front","average":{"1min":2.5779999999999998,"5min":1.244,"15min":0.99299999999999999},"min":{"1min":0.23000000000000001,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":10.324999999999999,"5min":10.324999999999999,"15min":10.324999999999999},"last":1.9610000000000001}]},{"osd":2,"last update":"Sat Sep 23 20:58:43 2023","interfaces":[{"interface":"back","average":{"1min":2.286,"5min":1.982,"15min":1.758},"min":{"1min":0.28999999999999998,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":7.0519999999999996,"5min":13.750999999999999,"15min":13.750999999999999},"last":0.312},{"interface":"front","average":{"1min":2.419,"5min":1.5289999999999999,"15min":1.2470000000000001},"min":{"1min":0.26200000000000001,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":8.0890000000000004,"5min":8.0890000000000004,"15min":8.0890000000000004},"last":2.0419999999999998}]},{"osd":3,"last update":"Sat Sep 23 20:58:43 2023","interfaces":[{"interface":"back","average":{"1min":1.8680000000000001,"5min":1.2849999999999999,"15min":1.173},"min":{"1min":0.32000000000000001,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":6.9000000000000004,"5min":6.9000000000000004,"15min":6.9000000000000004},"last":0.45000000000000001},{"interface":"front","average":{"1min":2.1819999999999999,"5min":1.74,"15min":1.6479999999999999},"min":{"1min":0.28599999999999998,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":11.021000000000001,"5min":11.021000000000001,"15min":11.021000000000001},"last":3.3900000000000001}]},{"osd":4,"last update":"Sat Sep 23 20:58:43 2023","interfaces":[{"interface":"back","average":{"1min":0.49099999999999999,"5min":0.85899999999999999,"15min":0.98699999999999999},"min":{"1min":0.23000000000000001,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":2.7919999999999998,"5min":9.0079999999999991,"15min":9.0079999999999991},"last":1.9950000000000001},{"interface":"front","average":{"1min":0.49299999999999999,"5min":0.50800000000000001,"15min":0.52300000000000002},"min":{"1min":0.27200000000000002,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":2.0059999999999998,"5min":2.0059999999999998,"15min":2.0059999999999998},"last":0.40300000000000002}]},{"osd":5,"last update":"Sat Sep 23 20:58:43 2023","interfaces":[{"interface":"back","average":{"1min":0.53700000000000003,"5min":0.56899999999999995,"15min":0.55000000000000004},"min":{"1min":0.27300000000000002,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":2.8239999999999998,"5min":2.8239999999999998,"15min":2.8239999999999998},"last":2.0179999999999998},{"interface":"front","average":{"1min":0.81000000000000005,"5min":0.78500000000000003,"15min":0.80100000000000005},"min":{"1min":0.33400000000000002,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":3.1379999999999999,"5min":3.1379999999999999,"15min":3.1379999999999999},"last":0.42799999999999999}]},{"osd":7,"last update":"Sat Sep 23 20:58:43 2023","interfaces":[{"interface":"back","average":{"1min":0.52700000000000002,"5min":0.85399999999999998,"15min":0.90400000000000003},"min":{"1min":0.312,"5min":0.312,"15min":0.312},"max":{"1min":2.073,"5min":7.8570000000000002,"15min":7.8570000000000002},"last":0.46999999999999997},{"interface":"front","average":{"1min":0.61899999999999999,"5min":0.66200000000000003,"15min":0.57399999999999995},"min":{"1min":0.30299999999999999,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":2.6989999999999998,"5min":8.282,"15min":8.282},"last":0.58399999999999996}]}]},{"osd":1,"up_from":88,"seq":377957122161,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2010984,"kb_used_data":1788128,"kb_used_omap":9,"kb_used_meta":222838,"kb_avail":91734168,"statfs":{"total":95995035648,"available":93935788032,"internally_reserved":0,"allocated":1831043072,"data_stored":1829864163,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9239,"internal_metadata":228187113},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:58:22 2023","interfaces":[{"interface":"back","average":{"1min":3.3290000000000002,"5min":2.9990000000000001,"15min":1.3049999999999999},"min":{"1min":0.23200000000000001,"5min":0.216,"15min":0.216},"max":{"1min":10.73,"5min":16.646999999999998,"15min":16.646999999999998},"last":0.435},{"interface":"front","average":{"1min":4.2149999999999999,"5min":3.3940000000000001,"15min":1.4590000000000001},"min":{"1min":0.247,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":15.135999999999999,"5min":18.457999999999998,"15min":18.457999999999998},"last":4.8390000000000004}]},{"osd":2,"last update":"Sat Sep 23 20:58:53 2023","interfaces":[{"interface":"back","average":{"1min":2.7130000000000001,"5min":2.4140000000000001,"15min":1.1799999999999999},"min":{"1min":0.24299999999999999,"5min":0.221,"15min":0.20300000000000001},"max":{"1min":8.6829999999999998,"5min":14.335000000000001,"15min":14.335000000000001},"last":0.45900000000000002},{"interface":"front","average":{"1min":5.8019999999999996,"5min":3.27,"15min":1.758},"min":{"1min":0.39500000000000002,"5min":0.248,"15min":0.248},"max":{"1min":16.048999999999999,"5min":16.048999999999999,"15min":38.951000000000001},"last":6.7729999999999997}]},{"osd":3,"last update":"Sat Sep 23 20:58:22 2023","interfaces":[{"interface":"back","average":{"1min":3.1859999999999999,"5min":3.0489999999999999,"15min":1.3720000000000001},"min":{"1min":0.246,"5min":0.246,"15min":0.246},"max":{"1min":10.701000000000001,"5min":25.314,"15min":25.314},"last":2.6349999999999998},{"interface":"front","average":{"1min":2.8450000000000002,"5min":2.6150000000000002,"15min":1.232},"min":{"1min":0.33800000000000002,"5min":0.20599999999999999,"15min":0.20599999999999999},"max":{"1min":7.8899999999999997,"5min":19.881,"15min":19.881},"last":0.62}]},{"osd":4,"last update":"Sat Sep 23 20:58:22 2023","interfaces":[{"interface":"back","average":{"1min":1.7050000000000001,"5min":1.9550000000000001,"15min":1.131},"min":{"1min":0.36099999999999999,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":10.598000000000001,"5min":12.006,"15min":12.006},"last":2.0739999999999998},{"interface":"front","average":{"1min":2.0899999999999999,"5min":1.6839999999999999,"15min":1.016},"min":{"1min":0.31900000000000001,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":6.0460000000000003,"5min":11.234999999999999,"15min":11.234999999999999},"last":0.70199999999999996}]},{"osd":5,"last update":"Sat Sep 23 20:58:22 2023","interfaces":[{"interface":"back","average":{"1min":3.0609999999999999,"5min":2.133,"15min":1.181},"min":{"1min":0.34399999999999997,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":15.041,"5min":15.041,"15min":15.041},"last":6.6779999999999999},{"interface":"front","average":{"1min":1.669,"5min":2.0169999999999999,"15min":1.153},"min":{"1min":0.33200000000000002,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":10.647,"5min":12.08,"15min":12.08},"last":2.1099999999999999}]},{"osd":6,"last update":"Sat Sep 23 20:58:39 2023","interfaces":[{"interface":"back","average":{"1min":1.6160000000000001,"5min":1.3700000000000001,"15min":1.333},"min":{"1min":0.30299999999999999,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":10.673999999999999,"5min":10.673999999999999,"15min":10.673999999999999},"last":2.032},{"interface":"front","average":{"1min":1.8280000000000001,"5min":1.508,"15min":1.3380000000000001},"min":{"1min":0.38100000000000001,"5min":0.309,"15min":0.309},"max":{"1min":8.7530000000000001,"5min":8.7530000000000001,"15min":8.7530000000000001},"last":0.73599999999999999}]},{"osd":7,"last update":"Sat Sep 23 20:58:22 2023","interfaces":[{"interface":"back","average":{"1min":2.0659999999999998,"5min":1.7909999999999999,"15min":1.069},"min":{"1min":0.42499999999999999,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":6.149,"5min":11.186999999999999,"15min":11.186999999999999},"last":1.29},{"interface":"front","average":{"1min":3.1520000000000001,"5min":2.1829999999999998,"15min":1.2010000000000001},"min":{"1min":0.41899999999999998,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":15.098000000000001,"5min":15.098000000000001,"15min":15.098000000000001},"last":6.7309999999999999}]}]},{"osd":0,"up_from":83,"seq":356482285688,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2080716,"kb_used_data":1878468,"kb_used_omap":28,"kb_used_meta":202211,"kb_avail":91664436,"statfs":{"total":95995035648,"available":93864382464,"internally_reserved":0,"allocated":1923551232,"data_stored":1922398239,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":28848,"internal_metadata":207064912},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 20:59:19 2023","interfaces":[{"interface":"back","average":{"1min":3.694,"5min":3.121,"15min":1.4850000000000001},"min":{"1min":0.22700000000000001,"5min":0.191,"15min":0.191},"max":{"1min":15.577999999999999,"5min":16.484999999999999,"15min":18.863},"last":2.9329999999999998},{"interface":"front","average":{"1min":4.4610000000000003,"5min":3.0339999999999998,"15min":1.5389999999999999},"min":{"1min":0.20999999999999999,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":15.771000000000001,"5min":15.771000000000001,"15min":17.704999999999998},"last":1.8759999999999999}]},{"osd":2,"last update":"Sat Sep 23 20:58:50 2023","interfaces":[{"interface":"back","average":{"1min":4.843,"5min":3.2549999999999999,"15min":1.6359999999999999},"min":{"1min":0.434,"5min":0.24299999999999999,"15min":0.23799999999999999},"max":{"1min":14.942,"5min":14.976000000000001,"15min":17.753},"last":2.2160000000000002},{"interface":"front","average":{"1min":7.774,"5min":3.8180000000000001,"15min":1.726},"min":{"1min":0.38500000000000001,"5min":0.25800000000000001,"15min":0.21099999999999999},"max":{"1min":21.747,"5min":21.747,"15min":21.747},"last":2.8839999999999999}]},{"osd":3,"last update":"Sat Sep 23 20:59:19 2023","interfaces":[{"interface":"back","average":{"1min":4.7839999999999998,"5min":3.0590000000000002,"15min":1.5900000000000001},"min":{"1min":0.38300000000000001,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":21.184000000000001,"5min":21.184000000000001,"15min":21.184000000000001},"last":2.964},{"interface":"front","average":{"1min":6.0469999999999997,"5min":3.5510000000000002,"15min":1.794},"min":{"1min":0.30599999999999999,"5min":0.28000000000000003,"15min":0.23300000000000001},"max":{"1min":21.167000000000002,"5min":21.167000000000002,"15min":21.167000000000002},"last":7.6740000000000004}]},{"osd":4,"last update":"Sat Sep 23 20:58:50 2023","interfaces":[{"interface":"back","average":{"1min":6.6399999999999997,"5min":2.9279999999999999,"15min":1.4730000000000001},"min":{"1min":0.36599999999999999,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":21.594999999999999,"5min":21.594999999999999,"15min":21.594999999999999},"last":2.5819999999999999},{"interface":"front","average":{"1min":2.657,"5min":1.9790000000000001,"15min":1.131},"min":{"1min":0.35199999999999998,"5min":0.31,"15min":0.28100000000000003},"max":{"1min":10.393000000000001,"5min":18.724,"15min":18.724},"last":2.988}]},{"osd":5,"last update":"Sat Sep 23 20:58:50 2023","interfaces":[{"interface":"back","average":{"1min":2.2719999999999998,"5min":1.8069999999999999,"15min":1.177},"min":{"1min":0.34999999999999998,"5min":0.318,"15min":0.318},"max":{"1min":8.8970000000000002,"5min":8.8970000000000002,"15min":8.8970000000000002},"last":1.9419999999999999},{"interface":"front","average":{"1min":5.7110000000000003,"5min":2.8919999999999999,"15min":1.4950000000000001},"min":{"1min":0.40600000000000003,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":20.512,"5min":20.512,"15min":20.512},"last":2.6640000000000001}]},{"osd":6,"last update":"Sat Sep 23 20:58:44 2023","interfaces":[{"interface":"back","average":{"1min":2.681,"5min":1.893,"15min":1.7350000000000001},"min":{"1min":0.39900000000000002,"5min":0.37,"15min":0.37},"max":{"1min":10.420999999999999,"5min":10.420999999999999,"15min":10.420999999999999},"last":3.0369999999999999},{"interface":"front","average":{"1min":2.7120000000000002,"5min":1.9650000000000001,"15min":1.819},"min":{"1min":0.42199999999999999,"5min":0.377,"15min":0.377},"max":{"1min":10.331,"5min":10.331,"15min":10.331},"last":3.0600000000000001}]},{"osd":7,"last update":"Sat Sep 23 20:58:50 2023","interfaces":[{"interface":"back","average":{"1min":2.4729999999999999,"5min":1.875,"15min":1.22},"min":{"1min":0.38100000000000001,"5min":0.379,"15min":0.379},"max":{"1min":8.7729999999999997,"5min":8.7729999999999997,"15min":8.7729999999999997},"last":1.911},{"interface":"front","average":{"1min":2.7010000000000001,"5min":2.032,"15min":1.1619999999999999},"min":{"1min":0.377,"5min":0.377,"15min":0.377},"max":{"1min":10.359999999999999,"5min":18.673999999999999,"15min":18.673999999999999},"last":3.0139999999999998}]}]},{"osd":2,"up_from":93,"seq":399431958635,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2068668,"kb_used_data":1759028,"kb_used_omap":81,"kb_used_meta":309358,"kb_avail":91676484,"statfs":{"total":95995035648,"available":93876719616,"internally_reserved":0,"allocated":1801244672,"data_stored":1800104427,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":83750,"internal_metadata":316782810},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,0,0,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:58:47 2023","interfaces":[{"interface":"back","average":{"1min":3.1429999999999998,"5min":2.0670000000000002,"15min":1.079},"min":{"1min":0.22700000000000001,"5min":0.17999999999999999,"15min":0.17999999999999999},"max":{"1min":14.773999999999999,"5min":14.773999999999999,"15min":14.773999999999999},"last":0.56299999999999994},{"interface":"front","average":{"1min":5.1459999999999999,"5min":3.133,"15min":1.4319999999999999},"min":{"1min":0.32400000000000001,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":17.045000000000002,"5min":17.045000000000002,"15min":17.045000000000002},"last":0.57999999999999996}]},{"osd":1,"last update":"Sat Sep 23 20:58:47 2023","interfaces":[{"interface":"back","average":{"1min":4.1470000000000002,"5min":2.718,"15min":1.48},"min":{"1min":0.29299999999999998,"5min":0.20200000000000001,"15min":0.20200000000000001},"max":{"1min":14.709,"5min":14.709,"15min":16.081},"last":0.60099999999999998},{"interface":"front","average":{"1min":5.3819999999999997,"5min":3.6389999999999998,"15min":1.77},"min":{"1min":0.41499999999999998,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":11.492000000000001,"5min":12.07,"15min":16.041},"last":0.44600000000000001}]},{"osd":3,"last update":"Sat Sep 23 20:58:26 2023","interfaces":[{"interface":"back","average":{"1min":5.1120000000000001,"5min":3.4620000000000002,"15min":1.585},"min":{"1min":0.23300000000000001,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":23.309999999999999,"5min":23.309999999999999,"15min":23.309999999999999},"last":1.1020000000000001},{"interface":"front","average":{"1min":5.5659999999999998,"5min":4.21,"15min":1.8240000000000001},"min":{"1min":0.34300000000000003,"5min":0.22800000000000001,"15min":0.221},"max":{"1min":23.324999999999999,"5min":23.324999999999999,"15min":23.324999999999999},"last":6.1349999999999998}]},{"osd":4,"last update":"Sat Sep 23 20:58:47 2023","interfaces":[{"interface":"back","average":{"1min":2.2519999999999998,"5min":1.9910000000000001,"15min":1.1439999999999999},"min":{"1min":0.28699999999999998,"5min":0.28699999999999998,"15min":0.26400000000000001},"max":{"1min":8.4580000000000002,"5min":9.8119999999999994,"15min":9.8119999999999994},"last":0.52300000000000002},{"interface":"front","average":{"1min":2.137,"5min":2.1560000000000001,"15min":1.2270000000000001},"min":{"1min":0.32400000000000001,"5min":0.313,"15min":0.313},"max":{"1min":8.1649999999999991,"5min":10.609,"15min":10.609},"last":6.0679999999999996}]},{"osd":5,"last update":"Sat Sep 23 20:58:47 2023","interfaces":[{"interface":"back","average":{"1min":2.1779999999999999,"5min":2.238,"15min":1.2470000000000001},"min":{"1min":0.32300000000000001,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":8.5,"5min":9.7629999999999999,"15min":9.7629999999999999},"last":0.498},{"interface":"front","average":{"1min":2.0670000000000002,"5min":2.1709999999999998,"15min":1.242},"min":{"1min":0.36399999999999999,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":8.1170000000000009,"5min":10.574,"15min":10.574},"last":5.9880000000000004}]},{"osd":6,"last update":"Sat Sep 23 20:58:42 2023","interfaces":[{"interface":"back","average":{"1min":2.1360000000000001,"5min":1.8169999999999999,"15min":1.7170000000000001},"min":{"1min":0.379,"5min":0.312,"15min":0.312},"max":{"1min":8.2270000000000003,"5min":14.244999999999999,"15min":14.244999999999999},"last":6.0380000000000003},{"interface":"front","average":{"1min":2.1949999999999998,"5min":1.444,"15min":1.1679999999999999},"min":{"1min":0.41199999999999998,"5min":0.374,"15min":0.374},"max":{"1min":8.1980000000000004,"5min":8.1980000000000004,"15min":8.1980000000000004},"last":6.0990000000000002}]},{"osd":7,"last update":"Sat Sep 23 20:58:47 2023","interfaces":[{"interface":"back","average":{"1min":1.4830000000000001,"5min":1.4339999999999999,"15min":0.995},"min":{"1min":0.315,"5min":0.315,"15min":0.315},"max":{"1min":5.8570000000000002,"5min":8.0820000000000007,"15min":8.0820000000000007},"last":0.78400000000000003},{"interface":"front","average":{"1min":2.2090000000000001,"5min":2.2109999999999999,"15min":1.262},"min":{"1min":0.35099999999999998,"5min":0.35099999999999998,"15min":0.34499999999999997},"max":{"1min":8.5250000000000004,"5min":9.7859999999999996,"15min":9.7859999999999996},"last":0.54500000000000004}]}]},{"osd":3,"up_from":98,"seq":420906795108,"num_pgs":49,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1931616,"kb_used_data":1604056,"kb_used_omap":7,"kb_used_meta":327544,"kb_avail":91813536,"statfs":{"total":95995035648,"available":94017060864,"internally_reserved":0,"allocated":1642553344,"data_stored":1641438265,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7576,"internal_metadata":335405672},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:58:25 2023","interfaces":[{"interface":"back","average":{"1min":3.097,"5min":3.02,"15min":1.4890000000000001},"min":{"1min":0.308,"5min":0.20599999999999999,"15min":0.20599999999999999},"max":{"1min":8.5549999999999997,"5min":23.123999999999999,"15min":23.123999999999999},"last":2.0249999999999999},{"interface":"front","average":{"1min":2.9900000000000002,"5min":2.7869999999999999,"15min":1.3009999999999999},"min":{"1min":0.19800000000000001,"5min":0.19800000000000001,"15min":0.19800000000000001},"max":{"1min":9.8889999999999993,"5min":15.079000000000001,"15min":15.079000000000001},"last":0.51000000000000001}]},{"osd":1,"last update":"Sat Sep 23 20:58:25 2023","interfaces":[{"interface":"back","average":{"1min":3.7829999999999999,"5min":3.1509999999999998,"15min":1.4390000000000001},"min":{"1min":0.32400000000000001,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":13.173,"5min":21.385000000000002,"15min":21.385000000000002},"last":3.3029999999999999},{"interface":"front","average":{"1min":3.0880000000000001,"5min":3.242,"15min":1.5109999999999999},"min":{"1min":0.28799999999999998,"5min":0.218,"15min":0.218},"max":{"1min":10.683999999999999,"5min":25.353000000000002,"15min":25.353000000000002},"last":0.995}]},{"osd":2,"last update":"Sat Sep 23 20:58:25 2023","interfaces":[{"interface":"back","average":{"1min":4.851,"5min":3.052,"15min":1.444},"min":{"1min":0.35899999999999999,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":15.476000000000001,"5min":24.154,"15min":24.154},"last":3.1850000000000001},{"interface":"front","average":{"1min":2.8519999999999999,"5min":3.335,"15min":1.577},"min":{"1min":0.23999999999999999,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":8.6069999999999993,"5min":25.305,"15min":25.305},"last":2.2200000000000002}]},{"osd":4,"last update":"Sat Sep 23 20:58:25 2023","interfaces":[{"interface":"back","average":{"1min":2.6539999999999999,"5min":1.8240000000000001,"15min":1.141},"min":{"1min":0.38900000000000001,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":9.6170000000000009,"5min":11.861000000000001,"15min":11.861000000000001},"last":3.2280000000000002},{"interface":"front","average":{"1min":1.3600000000000001,"5min":2.0880000000000001,"15min":1.246},"min":{"1min":0.313,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":4.8109999999999999,"5min":22.853000000000002,"15min":22.853000000000002},"last":0.47899999999999998}]},{"osd":5,"last update":"Sat Sep 23 20:58:25 2023","interfaces":[{"interface":"back","average":{"1min":2.9540000000000002,"5min":1.8999999999999999,"15min":1.1459999999999999},"min":{"1min":0.436,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":9.6430000000000007,"5min":11.891999999999999,"15min":11.891999999999999},"last":3.254},{"interface":"front","average":{"1min":2.04,"5min":1.6579999999999999,"15min":1.075},"min":{"1min":0.33300000000000002,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":5.7690000000000001,"5min":9.2870000000000008,"15min":9.2870000000000008},"last":0.53200000000000003}]},{"osd":6,"last update":"Sat Sep 23 20:58:43 2023","interfaces":[{"interface":"back","average":{"1min":1.9450000000000001,"5min":1.1930000000000001,"15min":1.0309999999999999},"min":{"1min":0.33500000000000002,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":10.018000000000001,"5min":10.018000000000001,"15min":10.018000000000001},"last":0.44600000000000001},{"interface":"front","average":{"1min":1.4770000000000001,"5min":1.48,"15min":1.4339999999999999},"min":{"1min":0.38700000000000001,"5min":0.38700000000000001,"15min":0.38700000000000001},"max":{"1min":5.3239999999999998,"5min":7.234,"15min":7.234},"last":0.55300000000000005}]},{"osd":7,"last update":"Sat Sep 23 20:58:25 2023","interfaces":[{"interface":"back","average":{"1min":3.0099999999999998,"5min":2.3399999999999999,"15min":1.3220000000000001},"min":{"1min":0.46500000000000002,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":9.5749999999999993,"5min":16.364999999999998,"15min":16.364999999999998},"last":3.2770000000000001},{"interface":"front","average":{"1min":2.214,"5min":1.766,"15min":1.147},"min":{"1min":0.42499999999999999,"5min":0.379,"15min":0.379},"max":{"1min":5.8090000000000002,"5min":9.3179999999999996,"15min":9.3179999999999996},"last":0.58499999999999996}]}]},{"osd":4,"up_from":31,"seq":133143986416,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1298872,"kb_used_data":1166704,"kb_used_omap":0,"kb_used_meta":132160,"kb_avail":92446280,"statfs":{"total":95995035648,"available":94664990720,"internally_reserved":0,"allocated":1194704896,"data_stored":1193857494,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":135331840},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:58:53 2023","interfaces":[{"interface":"back","average":{"1min":1.917,"5min":1.7170000000000001,"15min":1.042},"min":{"1min":0.245,"5min":0.214,"15min":0.214},"max":{"1min":8.8439999999999994,"5min":11.784000000000001,"15min":11.784000000000001},"last":5.2640000000000002},{"interface":"front","average":{"1min":1.8149999999999999,"5min":1.7889999999999999,"15min":1.23},"min":{"1min":0.314,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":8.8300000000000001,"5min":11.766,"15min":11.766},"last":5.2789999999999999}]},{"osd":1,"last update":"Sat Sep 23 20:58:21 2023","interfaces":[{"interface":"back","average":{"1min":1.7370000000000001,"5min":1.641,"15min":0.996},"min":{"1min":0.246,"5min":0.246,"15min":0.23599999999999999},"max":{"1min":10.153,"5min":10.153,"15min":10.153},"last":1.3600000000000001},{"interface":"front","average":{"1min":2.645,"5min":1.649,"15min":0.96399999999999997},"min":{"1min":0.26100000000000001,"5min":0.26100000000000001,"15min":0.23200000000000001},"max":{"1min":12.292999999999999,"5min":12.292999999999999,"15min":12.292999999999999},"last":1.091}]},{"osd":2,"last update":"Sat Sep 23 20:58:53 2023","interfaces":[{"interface":"back","average":{"1min":1.8700000000000001,"5min":1.764,"15min":1.115},"min":{"1min":0.34300000000000003,"5min":0.311,"15min":0.29099999999999998},"max":{"1min":7.3150000000000004,"5min":15.288,"15min":15.288},"last":8.1509999999999998},{"interface":"front","average":{"1min":1.633,"5min":1.974,"15min":1.2629999999999999},"min":{"1min":0.32600000000000001,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":8.7959999999999994,"5min":15.262,"15min":15.262},"last":1.321}]},{"osd":3,"last update":"Sat Sep 23 20:58:21 2023","interfaces":[{"interface":"back","average":{"1min":2.0209999999999999,"5min":2.504,"15min":1.514},"min":{"1min":0.28899999999999998,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":7.7549999999999999,"5min":15.916,"15min":31.562999999999999},"last":1.0029999999999999},{"interface":"front","average":{"1min":2.2080000000000002,"5min":2.5449999999999999,"15min":1.589},"min":{"1min":0.433,"5min":0.433,"15min":0.41899999999999998},"max":{"1min":7.7300000000000004,"5min":15.882,"15min":31.574000000000002},"last":0.98999999999999999}]},{"osd":5,"last update":"Sat Sep 23 20:58:35 2023","interfaces":[{"interface":"back","average":{"1min":0.81399999999999995,"5min":0.61399999999999999,"15min":0.52900000000000003},"min":{"1min":0.29199999999999998,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":2.9470000000000001,"5min":3.3279999999999998,"15min":3.3279999999999998},"last":0.59499999999999997},{"interface":"front","average":{"1min":0.77900000000000003,"5min":0.68500000000000005,"15min":0.57799999999999996},"min":{"1min":0.33200000000000002,"5min":0.251,"15min":0.251},"max":{"1min":1.9350000000000001,"5min":7.7619999999999996,"15min":7.7619999999999996},"last":0.54000000000000004}]},{"osd":6,"last update":"Sat Sep 23 20:58:41 2023","interfaces":[{"interface":"back","average":{"1min":0.90300000000000002,"5min":0.67700000000000005,"15min":0.54800000000000004},"min":{"1min":0.29899999999999999,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":3.4780000000000002,"5min":4.6909999999999998,"15min":4.6909999999999998},"last":0.498},{"interface":"front","average":{"1min":0.82299999999999995,"5min":0.71599999999999997,"15min":0.66800000000000004},"min":{"1min":0.36799999999999999,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":3.456,"5min":3.456,"15min":3.456},"last":0.56399999999999995}]},{"osd":7,"last update":"Sat Sep 23 20:58:20 2023","interfaces":[{"interface":"back","average":{"1min":0.84199999999999997,"5min":0.83099999999999996,"15min":0.67100000000000004},"min":{"1min":0.442,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":3.327,"5min":7.383,"15min":7.383},"last":0.51700000000000002},{"interface":"front","average":{"1min":0.93899999999999995,"5min":0.80800000000000005,"15min":0.66000000000000003},"min":{"1min":0.36399999999999999,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":3.883,"5min":7.8310000000000004,"15min":7.8310000000000004},"last":0.58599999999999997}]}]},{"osd":5,"up_from":39,"seq":167503724781,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1400364,"kb_used_data":1280228,"kb_used_omap":0,"kb_used_meta":120128,"kb_avail":92344788,"statfs":{"total":95995035648,"available":94561062912,"internally_reserved":0,"allocated":1310953472,"data_stored":1310114646,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":123011072},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:58:37 2023","interfaces":[{"interface":"back","average":{"1min":0.78600000000000003,"5min":1.03,"15min":0.72699999999999998},"min":{"1min":0.23899999999999999,"5min":0.20399999999999999,"15min":0.20399999999999999},"max":{"1min":3.6949999999999998,"5min":7.766,"15min":7.766},"last":0.85199999999999998},{"interface":"front","average":{"1min":1.8049999999999999,"5min":1.6779999999999999,"15min":0.98099999999999998},"min":{"1min":0.25,"5min":0.223,"15min":0.223},"max":{"1min":7.8120000000000003,"5min":15.037000000000001,"15min":15.037000000000001},"last":0.63100000000000001}]},{"osd":1,"last update":"Sat Sep 23 20:59:17 2023","interfaces":[{"interface":"back","average":{"1min":2.7749999999999999,"5min":1.758,"15min":1.099},"min":{"1min":0.248,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":8.1829999999999998,"5min":8.3520000000000003,"15min":8.3520000000000003},"last":0.36399999999999999},{"interface":"front","average":{"1min":2.3690000000000002,"5min":1.825,"15min":1.099},"min":{"1min":0.27200000000000002,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":8.5399999999999991,"5min":10.285,"15min":10.285},"last":0.34200000000000003}]},{"osd":2,"last update":"Sat Sep 23 20:58:52 2023","interfaces":[{"interface":"back","average":{"1min":1.9299999999999999,"5min":1.6000000000000001,"15min":0.97799999999999998},"min":{"1min":0.311,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":6,"5min":12.651,"15min":12.651},"last":0.27400000000000002},{"interface":"front","average":{"1min":3.077,"5min":1.8440000000000001,"15min":1.085},"min":{"1min":0.27100000000000002,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":12.300000000000001,"5min":12.300000000000001,"15min":12.300000000000001},"last":0.28799999999999998}]},{"osd":3,"last update":"Sat Sep 23 20:59:17 2023","interfaces":[{"interface":"back","average":{"1min":3.5880000000000001,"5min":2.2090000000000001,"15min":1.351},"min":{"1min":0.46700000000000003,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":12.188000000000001,"5min":12.188000000000001,"15min":12.188000000000001},"last":1.4470000000000001},{"interface":"front","average":{"1min":2.0739999999999998,"5min":1.6060000000000001,"15min":1.071},"min":{"1min":0.32500000000000001,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":7.258,"5min":10.457000000000001,"15min":10.457000000000001},"last":0.39100000000000001}]},{"osd":4,"last update":"Sat Sep 23 20:58:33 2023","interfaces":[{"interface":"back","average":{"1min":0.48799999999999999,"5min":0.59099999999999997,"15min":0.52900000000000003},"min":{"1min":0.252,"5min":0.252,"15min":0.252},"max":{"1min":2.6909999999999998,"5min":3.8479999999999999,"15min":3.8479999999999999},"last":0.374},{"interface":"front","average":{"1min":0.41899999999999998,"5min":0.64100000000000001,"15min":0.54900000000000004},"min":{"1min":0.29299999999999998,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":0.747,"5min":3.0680000000000001,"15min":3.0680000000000001},"last":0.47199999999999998}]},{"osd":6,"last update":"Sat Sep 23 20:58:37 2023","interfaces":[{"interface":"back","average":{"1min":0.46600000000000003,"5min":0.59099999999999997,"15min":0.57099999999999995},"min":{"1min":0.28499999999999998,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":1.0509999999999999,"5min":2.9820000000000002,"15min":2.9820000000000002},"last":0.32800000000000001},{"interface":"front","average":{"1min":0.499,"5min":0.69899999999999995,"15min":0.72899999999999998},"min":{"1min":0.36299999999999999,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":0.78500000000000003,"5min":3.6560000000000001,"15min":3.6560000000000001},"last":0.52400000000000002}]},{"osd":7,"last update":"Sat Sep 23 20:59:17 2023","interfaces":[{"interface":"back","average":{"1min":0.71099999999999997,"5min":0.80200000000000005,"15min":0.64200000000000002},"min":{"1min":0.33400000000000002,"5min":0.307,"15min":0.30199999999999999},"max":{"1min":2.516,"5min":8.5649999999999995,"15min":8.5649999999999995},"last":0.41499999999999998},{"interface":"front","average":{"1min":0.77200000000000002,"5min":0.75,"15min":0.65700000000000003},"min":{"1min":0.34399999999999997,"5min":0.29799999999999999,"15min":0.27700000000000002},"max":{"1min":3.0150000000000001,"5min":4.5430000000000001,"15min":4.5430000000000001},"last":0.54600000000000004}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1912016896,"data_stored":1912012824,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1820397568,"data_stored":1820393496,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1792409600,"data_stored":1792409600,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1636892672,"data_stored":1636892672,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1191051264,"data_stored":1191051264,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1307967488,"data_stored":1307967488,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1235357696,"data_stored":1235353624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1190526976,"data_stored":1190526976,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T20:59:24.279 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:59:24.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:59:24 smithi070 ceph-mon[168282]: pgmap v586: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 3.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 560 op/s; 11513/177099 objects degraded (6.501%); 2721/177099 objects misplaced (1.536%); 1.5 MiB/s, 24 objects/s recovering 2023-09-23T20:59:24.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:59:24 smithi070 ceph-mon[168282]: from='client.45305 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:59:24.399 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:59:25.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:59:25 smithi070 ceph-mon[168282]: Health check update: Degraded data redundancy: 11513/177099 objects degraded (6.501%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T20:59:25.381 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:59:25 smithi026 ceph-mon[175151]: Health check update: Degraded data redundancy: 11513/177099 objects degraded (6.501%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T20:59:25.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:59:25 smithi026 ceph-mon[179688]: Health check update: Degraded data redundancy: 11513/177099 objects degraded (6.501%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T20:59:25.635 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:59:25.641 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:59:25.641 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:59:25.742 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:59:25.810 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:59:25.897 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:59:25.906 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:59:25.907 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:59:26.007 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:59:26.085 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:59:26.227 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:59:26.290 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:59:26 smithi026 ceph-mon[175151]: pgmap v587: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 3.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 560 op/s; 11513/174966 objects degraded (6.580%); 2721/174966 objects misplaced (1.555%); 1.5 MiB/s, 24 objects/s recovering 2023-09-23T20:59:26.290 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:59:26 smithi026 ceph-mon[179688]: pgmap v587: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 3.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 560 op/s; 11513/174966 objects degraded (6.580%); 2721/174966 objects misplaced (1.555%); 1.5 MiB/s, 24 objects/s recovering 2023-09-23T20:59:26.313 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:59:26.326 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:59:26.326 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:59:26.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:59:26 smithi070 ceph-mon[168282]: pgmap v587: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 3.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 560 op/s; 11513/174966 objects degraded (6.580%); 2721/174966 objects misplaced (1.555%); 1.5 MiB/s, 24 objects/s recovering 2023-09-23T20:59:26.427 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:59:26.529 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:59:26.669 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:59:26.681 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:59:26.681 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:59:26.689 INFO:teuthology.orchestra.run.smithi026.stdout:{ 2023-09-23T20:59:26.689 INFO:teuthology.orchestra.run.smithi026.stdout: "ops": [], 2023-09-23T20:59:26.689 INFO:teuthology.orchestra.run.smithi026.stdout: "complaint_time": 30, 2023-09-23T20:59:26.689 INFO:teuthology.orchestra.run.smithi026.stdout: "num_blocked_ops": 0 2023-09-23T20:59:26.689 INFO:teuthology.orchestra.run.smithi026.stdout:} 2023-09-23T20:59:26.783 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:59:26.865 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:59:27.095 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:59:27 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/2910643555' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:59:27.096 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:59:27 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:59:27.096 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:59:27 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/2910643555' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:59:27.096 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:59:27 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:59:27.141 INFO:teuthology.orchestra.run.smithi026.stderr:nodeep-scrub is unset 2023-09-23T20:59:27.260 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T20:59:27.301 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:59:27.320 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:59:27.321 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:59:27.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:59:27 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/2910643555' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:59:27.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:59:27 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:59:27.421 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:59:27.452 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_historic_ops 2023-09-23T20:59:27.453 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:59:27.502 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:59:27.505 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:59:27.505 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:59:27.606 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:59:27.754 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:59:27.883 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:59:27.893 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:59:27.894 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:59:27.994 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:59:28.120 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:59:28.252 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:59:28 smithi026 ceph-mon[175151]: pgmap v588: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 3.5 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 564 op/s; 11513/173295 objects degraded (6.644%); 2721/173295 objects misplaced (1.570%); 1.5 MiB/s, 24 objects/s recovering 2023-09-23T20:59:28.252 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:59:28 smithi026 ceph-mon[175151]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T20:59:28.252 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:59:28 smithi026 ceph-mon[175151]: failed to encode map e240 with expected crc 2023-09-23T20:59:28.252 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:59:28 smithi026 ceph-mon[175151]: failed to encode map e240 with expected crc 2023-09-23T20:59:28.252 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:59:28 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T20:59:28.253 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:59:28 smithi026 ceph-mon[175151]: osdmap e240: 8 total, 8 up, 8 in 2023-09-23T20:59:28.253 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:59:28 smithi026 ceph-mon[175151]: failed to encode map e240 with expected crc 2023-09-23T20:59:28.253 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:59:28 smithi026 ceph-mon[175151]: failed to encode map e240 with expected crc 2023-09-23T20:59:28.257 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:59:28 smithi026 ceph-mon[179688]: pgmap v588: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 3.5 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 564 op/s; 11513/173295 objects degraded (6.644%); 2721/173295 objects misplaced (1.570%); 1.5 MiB/s, 24 objects/s recovering 2023-09-23T20:59:28.257 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:59:28 smithi026 ceph-mon[179688]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T20:59:28.257 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:59:28 smithi026 ceph-mon[179688]: failed to encode map e240 with expected crc 2023-09-23T20:59:28.257 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:59:28 smithi026 ceph-mon[179688]: failed to encode map e240 with expected crc 2023-09-23T20:59:28.257 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:59:28 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T20:59:28.257 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:59:28 smithi026 ceph-mon[179688]: osdmap e240: 8 total, 8 up, 8 in 2023-09-23T20:59:28.257 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:59:28 smithi026 ceph-mon[179688]: failed to encode map e240 with expected crc 2023-09-23T20:59:28.257 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:59:28 smithi026 ceph-mon[179688]: failed to encode map e240 with expected crc 2023-09-23T20:59:28.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:59:28 smithi070 ceph-mon[168282]: pgmap v588: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 3.5 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 564 op/s; 11513/173295 objects degraded (6.644%); 2721/173295 objects misplaced (1.570%); 1.5 MiB/s, 24 objects/s recovering 2023-09-23T20:59:28.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:59:28 smithi070 ceph-mon[168282]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T20:59:28.371 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:59:28 smithi070 ceph-mon[168282]: failed to encode map e240 with expected crc 2023-09-23T20:59:28.371 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:59:28 smithi070 ceph-mon[168282]: failed to encode map e240 with expected crc 2023-09-23T20:59:28.371 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:59:28 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T20:59:28.371 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:59:28 smithi070 ceph-mon[168282]: osdmap e240: 8 total, 8 up, 8 in 2023-09-23T20:59:28.371 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:59:28 smithi070 ceph-mon[168282]: failed to encode map e240 with expected crc 2023-09-23T20:59:28.371 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:59:28 smithi070 ceph-mon[168282]: failed to encode map e240 with expected crc 2023-09-23T20:59:28.384 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:59:28.389 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:59:28.390 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:59:28.491 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:59:28.577 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0: {} 2023-09-23T20:59:28.597 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:59:28.626 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:59:28.630 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1: {} 2023-09-23T20:59:28.689 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2: {} 2023-09-23T20:59:28.692 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:59:28.696 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:59:28.713 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:59:28.714 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:59:28.727 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3: {} 2023-09-23T20:59:28.787 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4: {} 2023-09-23T20:59:28.815 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:59:28.837 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T20:59:28.844 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:59:28.884 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:59:28.887 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:59:28.888 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:59:28.900 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6: {} 2023-09-23T20:59:28.943 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T20:59:28.989 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:59:28.996 INFO:teuthology.orchestra.run.smithi026.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T20:59:28.996 INFO:teuthology.orchestra.run.smithi026.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T20:59:28.996 INFO:teuthology.orchestra.run.smithi026.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T20:59:28.996 INFO:teuthology.orchestra.run.smithi026.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T20:59:28.996 INFO:teuthology.orchestra.run.smithi026.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T20:59:28.996 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T20:59:28.996 INFO:teuthology.orchestra.run.smithi026.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T20:59:28.996 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T20:59:29.017 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:59:29.058 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:59:29.061 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:59:29.062 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:59:29.162 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:59:29.192 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:59:29.235 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:59:29.238 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:59:29.238 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:59:29.248 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:59:29 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/1110896064' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:59:29.248 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:59:29 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/1110896064' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:59:29.339 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:59:29.430 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:59:29.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:59:29 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/1110896064' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:59:29.856 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:59:29 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[164029]: ::ffff:172.21.15.70 - - [23/Sep/2023:20:59:29] "GET /metrics HTTP/1.1" 200 36318 "" "Prometheus/2.43.0" 2023-09-23T20:59:29.976 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T20:59:30.131 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:59:30 smithi026 ceph-mon[175151]: pgmap v590: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 3.4 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 565 op/s; 11576/168309 objects degraded (6.878%); 2533/168309 objects misplaced (1.505%); 1.6 MiB/s, 25 objects/s recovering 2023-09-23T20:59:30.131 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:59:30 smithi026 ceph-mon[175151]: Health check update: Degraded data redundancy: 11576/168309 objects degraded (6.878%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T20:59:30.141 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:59:30 smithi026 ceph-mon[179688]: pgmap v590: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 3.4 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 565 op/s; 11576/168309 objects degraded (6.878%); 2533/168309 objects misplaced (1.505%); 1.6 MiB/s, 25 objects/s recovering 2023-09-23T20:59:30.141 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:59:30 smithi026 ceph-mon[179688]: Health check update: Degraded data redundancy: 11576/168309 objects degraded (6.878%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T20:59:30.180 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:59:30.192 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:59:30.193 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:59:30.293 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:59:30.394 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:59:30.475 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:59:30.481 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:59:30.481 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:59:30.582 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:59:30.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:59:30 smithi070 ceph-mon[168282]: pgmap v590: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 3.4 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 565 op/s; 11576/168309 objects degraded (6.878%); 2533/168309 objects misplaced (1.505%); 1.6 MiB/s, 25 objects/s recovering 2023-09-23T20:59:30.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:59:30 smithi070 ceph-mon[168282]: Health check update: Degraded data redundancy: 11576/168309 objects degraded (6.878%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T20:59:30.681 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:59:31.449 INFO:teuthology.orchestra.run.smithi026.stdout:ERROR: (22) Invalid argument 2023-09-23T20:59:31.449 INFO:teuthology.orchestra.run.smithi026.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:59:32.355 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T20:59:32.469 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:59:32 smithi026 ceph-mon[175151]: pgmap v591: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 3.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 576 op/s; 11576/164508 objects degraded (7.037%); 2533/164508 objects misplaced (1.540%); 1.6 MiB/s, 25 objects/s recovering 2023-09-23T20:59:32.469 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:59:32 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:59:32.469 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:59:32 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:59:32.473 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:59:32 smithi026 ceph-mon[179688]: pgmap v591: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 3.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 576 op/s; 11576/164508 objects degraded (7.037%); 2533/164508 objects misplaced (1.540%); 1.6 MiB/s, 25 objects/s recovering 2023-09-23T20:59:32.473 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:59:32 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:59:32.473 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:59:32 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:59:32.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:59:32 smithi070 ceph-mon[168282]: pgmap v591: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 3.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 576 op/s; 11576/164508 objects degraded (7.037%); 2533/164508 objects misplaced (1.540%); 1.6 MiB/s, 25 objects/s recovering 2023-09-23T20:59:32.690 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:59:32 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:59:32.691 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:59:32 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:59:32.837 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:59:32.842 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:59:32.842 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:59:32.943 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:59:32.973 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:59:33.019 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:59:33.022 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:59:33.022 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:59:33.123 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:59:33.153 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:59:33.189 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:59:33.192 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:59:33.192 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:59:33.212 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_ops_in_flight 2023-09-23T20:59:33.292 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:59:33.370 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:59:33.371 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:59:33.574 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:59:33.583 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:59:33.583 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:59:33.646 INFO:teuthology.orchestra.run.smithi026.stdout: 2023-09-23T20:59:33.655 INFO:teuthology.orchestra.run.smithi026.stderr:dumped all 2023-09-23T20:59:33.684 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:59:33.684 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:59:33.713 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:59:33.751 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:59:33.755 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:59:33.756 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:59:33.856 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:59:33.950 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:59:34.173 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:59:34.198 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:59:34.198 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:59:34.299 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:59:34.331 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:59:34.358 INFO:teuthology.orchestra.run.smithi026.stdout:{"pg_ready":true,"pg_map":{"version":592,"stamp":"2023-09-23T20:59:32.446266+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":3483244472,"num_objects":53166,"num_object_clones":0,"num_object_copies":159498,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":11601,"num_objects_misplaced":2373,"num_objects_unfound":0,"num_objects_dirty":53166,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":150439,"num_write_kb":6512606,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2390,"num_bytes_recovered":155195238,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":122514,"ondisk_log_size":122514,"up":327,"acting":324,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":331,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":12483968,"kb_used_data":10819584,"kb_used_omap":136,"kb_used_meta":1663671,"kb_avail":737477248,"statfs":{"total":767960285184,"available":755176701952,"internally_reserved":0,"allocated":11079254016,"data_stored":11071257635,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":139741,"internal_metadata":1703599651},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,0,2,1,3,4],"upper_bound":64},"perf_stat":{"commit_latency_ms":11,"apply_latency_ms":11,"commit_latency_ns":11000000,"apply_latency_ns":11000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-384499712,"num_objects":-5867,"num_object_clones":0,"num_object_copies":-17601,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":88,"num_objects_misplaced":-348,"num_objects_unfound":0,"num_objects_dirty":-5867,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5867,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":260,"num_bytes_recovered":17039360,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.007586"},"pg_stats":[{"pgid":"3.1a","version":"68'1","reported_seq":117,"reported_epoch":239,"state":"active+clean","last_fresh":"2023-09-23T20:59:17.555280+0000","last_change":"2023-09-23T20:54:13.946695+0000","last_active":"2023-09-23T20:59:17.555280+0000","last_peered":"2023-09-23T20:59:17.555280+0000","last_clean":"2023-09-23T20:59:17.555280+0000","last_became_active":"2023-09-23T20:54:13.945399+0000","last_became_peered":"2023-09-23T20:54:13.945399+0000","last_unstale":"2023-09-23T20:59:17.555280+0000","last_undegraded":"2023-09-23T20:59:17.555280+0000","last_fullsized":"2023-09-23T20:59:17.555280+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:32:52.365464+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":232,"reported_epoch":239,"state":"active+clean","last_fresh":"2023-09-23T20:59:17.553751+0000","last_change":"2023-09-23T20:54:13.942019+0000","last_active":"2023-09-23T20:59:17.553751+0000","last_peered":"2023-09-23T20:59:17.553751+0000","last_clean":"2023-09-23T20:59:17.553751+0000","last_became_active":"2023-09-23T20:54:13.941647+0000","last_became_peered":"2023-09-23T20:54:13.941647+0000","last_unstale":"2023-09-23T20:59:17.553751+0000","last_undegraded":"2023-09-23T20:59:17.553751+0000","last_fullsized":"2023-09-23T20:59:17.553751+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:30:54.400944+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":209,"reported_epoch":239,"state":"active+clean","last_fresh":"2023-09-23T20:59:17.563445+0000","last_change":"2023-09-23T20:54:13.941394+0000","last_active":"2023-09-23T20:59:17.563445+0000","last_peered":"2023-09-23T20:59:17.563445+0000","last_clean":"2023-09-23T20:59:17.563445+0000","last_became_active":"2023-09-23T20:54:13.941042+0000","last_became_peered":"2023-09-23T20:54:13.941042+0000","last_unstale":"2023-09-23T20:59:17.563445+0000","last_undegraded":"2023-09-23T20:59:17.563445+0000","last_fullsized":"2023-09-23T20:59:17.563445+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:19:41.401184+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":225,"reported_epoch":240,"state":"active+clean","last_fresh":"2023-09-23T20:59:27.105947+0000","last_change":"2023-09-23T20:57:44.979811+0000","last_active":"2023-09-23T20:59:27.105947+0000","last_peered":"2023-09-23T20:59:27.105947+0000","last_clean":"2023-09-23T20:59:27.105947+0000","last_became_active":"2023-09-23T20:57:44.979515+0000","last_became_peered":"2023-09-23T20:57:44.979515+0000","last_unstale":"2023-09-23T20:59:27.105947+0000","last_undegraded":"2023-09-23T20:59:27.105947+0000","last_fullsized":"2023-09-23T20:59:27.105947+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:38:22.162140+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":223,"reported_epoch":239,"state":"active+clean","last_fresh":"2023-09-23T20:59:17.553274+0000","last_change":"2023-09-23T20:51:10.833613+0000","last_active":"2023-09-23T20:59:17.553274+0000","last_peered":"2023-09-23T20:59:17.553274+0000","last_clean":"2023-09-23T20:59:17.553274+0000","last_became_active":"2023-09-23T20:51:10.833208+0000","last_became_peered":"2023-09-23T20:51:10.833208+0000","last_unstale":"2023-09-23T20:59:17.553274+0000","last_undegraded":"2023-09-23T20:59:17.553274+0000","last_fullsized":"2023-09-23T20:59:17.553274+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:51:28.246014+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":205,"reported_epoch":239,"state":"active+clean","last_fresh":"2023-09-23T20:59:17.551603+0000","last_change":"2023-09-23T20:55:35.958384+0000","last_active":"2023-09-23T20:59:17.551603+0000","last_peered":"2023-09-23T20:59:17.551603+0000","last_clean":"2023-09-23T20:59:17.551603+0000","last_became_active":"2023-09-23T20:55:35.954563+0000","last_became_peered":"2023-09-23T20:55:35.954563+0000","last_unstale":"2023-09-23T20:59:17.551603+0000","last_undegraded":"2023-09-23T20:59:17.551603+0000","last_fullsized":"2023-09-23T20:59:17.551603+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:37:48.332259+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":215,"reported_epoch":239,"state":"active+clean","last_fresh":"2023-09-23T20:59:17.555740+0000","last_change":"2023-09-23T20:54:31.885514+0000","last_active":"2023-09-23T20:59:17.555740+0000","last_peered":"2023-09-23T20:59:17.555740+0000","last_clean":"2023-09-23T20:59:17.555740+0000","last_became_active":"2023-09-23T20:54:31.884986+0000","last_became_peered":"2023-09-23T20:54:31.884986+0000","last_unstale":"2023-09-23T20:59:17.555740+0000","last_undegraded":"2023-09-23T20:59:17.555740+0000","last_fullsized":"2023-09-23T20:59:17.555740+0000","mapping_epoch":136,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:26:08.100220+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":220,"reported_epoch":239,"state":"active+clean","last_fresh":"2023-09-23T20:59:17.553344+0000","last_change":"2023-09-23T20:55:35.951889+0000","last_active":"2023-09-23T20:59:17.553344+0000","last_peered":"2023-09-23T20:59:17.553344+0000","last_clean":"2023-09-23T20:59:17.553344+0000","last_became_active":"2023-09-23T20:55:35.951563+0000","last_became_peered":"2023-09-23T20:55:35.951563+0000","last_unstale":"2023-09-23T20:59:17.553344+0000","last_undegraded":"2023-09-23T20:59:17.553344+0000","last_fullsized":"2023-09-23T20:59:17.553344+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:41:53.098781+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"79'161","reported_seq":526,"reported_epoch":239,"state":"active+clean","last_fresh":"2023-09-23T20:59:17.565916+0000","last_change":"2023-09-23T20:57:25.096559+0000","last_active":"2023-09-23T20:59:17.565916+0000","last_peered":"2023-09-23T20:59:17.565916+0000","last_clean":"2023-09-23T20:59:17.565916+0000","last_became_active":"2023-09-23T20:57:15.975674+0000","last_became_peered":"2023-09-23T20:57:15.975674+0000","last_unstale":"2023-09-23T20:59:17.565916+0000","last_undegraded":"2023-09-23T20:59:17.565916+0000","last_fullsized":"2023-09-23T20:59:17.565916+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:25.096401+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:25.096401+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:05:05.448404+0000","scrub_duration":0.002859393,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":113,"reported_epoch":239,"state":"active+clean","last_fresh":"2023-09-23T20:59:17.572564+0000","last_change":"2023-09-23T20:54:31.890678+0000","last_active":"2023-09-23T20:59:17.572564+0000","last_peered":"2023-09-23T20:59:17.572564+0000","last_clean":"2023-09-23T20:59:17.572564+0000","last_became_active":"2023-09-23T20:54:31.890469+0000","last_became_peered":"2023-09-23T20:54:31.890469+0000","last_unstale":"2023-09-23T20:59:17.572564+0000","last_undegraded":"2023-09-23T20:59:17.572564+0000","last_fullsized":"2023-09-23T20:59:17.572564+0000","mapping_epoch":136,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:55:08.790146+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":181,"reported_epoch":239,"state":"active+clean","last_fresh":"2023-09-23T20:59:17.555093+0000","last_change":"2023-09-23T20:55:35.952661+0000","last_active":"2023-09-23T20:59:17.555093+0000","last_peered":"2023-09-23T20:59:17.555093+0000","last_clean":"2023-09-23T20:59:17.555093+0000","last_became_active":"2023-09-23T20:55:35.952349+0000","last_became_peered":"2023-09-23T20:55:35.952349+0000","last_unstale":"2023-09-23T20:59:17.555093+0000","last_undegraded":"2023-09-23T20:59:17.555093+0000","last_fullsized":"2023-09-23T20:59:17.555093+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:00:03.110126+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":58,"reported_epoch":239,"state":"active+clean","last_fresh":"2023-09-23T20:59:17.553825+0000","last_change":"2023-09-23T20:56:45.988607+0000","last_active":"2023-09-23T20:59:17.553825+0000","last_peered":"2023-09-23T20:59:17.553825+0000","last_clean":"2023-09-23T20:59:17.553825+0000","last_became_active":"2023-09-23T20:56:45.988206+0000","last_became_peered":"2023-09-23T20:56:45.988206+0000","last_unstale":"2023-09-23T20:59:17.553825+0000","last_undegraded":"2023-09-23T20:59:17.553825+0000","last_fullsized":"2023-09-23T20:59:17.553825+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":205,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:01:19.783688+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,4],"acting":[3,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":57,"reported_epoch":239,"state":"active+clean","last_fresh":"2023-09-23T20:59:17.572921+0000","last_change":"2023-09-23T20:56:18.139142+0000","last_active":"2023-09-23T20:59:17.572921+0000","last_peered":"2023-09-23T20:59:17.572921+0000","last_clean":"2023-09-23T20:59:17.572921+0000","last_became_active":"2023-09-23T20:56:18.138844+0000","last_became_peered":"2023-09-23T20:56:18.138844+0000","last_unstale":"2023-09-23T20:59:17.572921+0000","last_undegraded":"2023-09-23T20:59:17.572921+0000","last_fullsized":"2023-09-23T20:59:17.572921+0000","mapping_epoch":192,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":193,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:09:02.217683+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,2],"acting":[5,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.14","version":"79'22","reported_seq":144,"reported_epoch":239,"state":"active+clean","last_fresh":"2023-09-23T20:59:17.555406+0000","last_change":"2023-09-23T20:55:35.949418+0000","last_active":"2023-09-23T20:59:17.555406+0000","last_peered":"2023-09-23T20:59:17.555406+0000","last_clean":"2023-09-23T20:59:17.555406+0000","last_became_active":"2023-09-23T20:55:35.948566+0000","last_became_peered":"2023-09-23T20:55:35.948566+0000","last_unstale":"2023-09-23T20:59:17.555406+0000","last_undegraded":"2023-09-23T20:59:17.555406+0000","last_fullsized":"2023-09-23T20:59:17.555406+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:59:37.433128+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4734,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":134,"reported_epoch":239,"state":"active+clean","last_fresh":"2023-09-23T20:59:17.556299+0000","last_change":"2023-09-23T20:55:35.949178+0000","last_active":"2023-09-23T20:59:17.556299+0000","last_peered":"2023-09-23T20:59:17.556299+0000","last_clean":"2023-09-23T20:59:17.556299+0000","last_became_active":"2023-09-23T20:55:35.948795+0000","last_became_peered":"2023-09-23T20:55:35.948795+0000","last_unstale":"2023-09-23T20:59:17.556299+0000","last_undegraded":"2023-09-23T20:59:17.556299+0000","last_fullsized":"2023-09-23T20:59:17.556299+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:13:08.996932+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.11","version":"79'161","reported_seq":468,"reported_epoch":239,"state":"active+clean","last_fresh":"2023-09-23T20:59:17.556236+0000","last_change":"2023-09-23T20:57:28.756796+0000","last_active":"2023-09-23T20:59:17.556236+0000","last_peered":"2023-09-23T20:59:17.556236+0000","last_clean":"2023-09-23T20:59:17.556236+0000","last_became_active":"2023-09-23T20:57:15.980606+0000","last_became_peered":"2023-09-23T20:57:15.980606+0000","last_unstale":"2023-09-23T20:59:17.556236+0000","last_undegraded":"2023-09-23T20:59:17.556236+0000","last_fullsized":"2023-09-23T20:59:17.556236+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:28.756634+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:28.756634+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:09:19.150791+0000","scrub_duration":0.0064925119999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":160,"reported_epoch":239,"state":"active+clean","last_fresh":"2023-09-23T20:59:17.556163+0000","last_change":"2023-09-23T20:56:45.992577+0000","last_active":"2023-09-23T20:59:17.556163+0000","last_peered":"2023-09-23T20:59:17.556163+0000","last_clean":"2023-09-23T20:59:17.556163+0000","last_became_active":"2023-09-23T20:56:45.992297+0000","last_became_peered":"2023-09-23T20:56:45.992297+0000","last_unstale":"2023-09-23T20:59:17.556163+0000","last_undegraded":"2023-09-23T20:59:17.556163+0000","last_fullsized":"2023-09-23T20:59:17.556163+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":205,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:00:45.547202+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,2],"acting":[4,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":205,"reported_epoch":239,"state":"active+clean","last_fresh":"2023-09-23T20:59:17.553236+0000","last_change":"2023-09-23T20:51:10.831262+0000","last_active":"2023-09-23T20:59:17.553236+0000","last_peered":"2023-09-23T20:59:17.553236+0000","last_clean":"2023-09-23T20:59:17.553236+0000","last_became_active":"2023-09-23T20:51:10.830931+0000","last_became_peered":"2023-09-23T20:51:10.830931+0000","last_unstale":"2023-09-23T20:59:17.553236+0000","last_undegraded":"2023-09-23T20:59:17.553236+0000","last_fullsized":"2023-09-23T20:59:17.553236+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:51:54.982505+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.10","version":"79'161","reported_seq":477,"reported_epoch":239,"state":"active+clean","last_fresh":"2023-09-23T20:59:17.571807+0000","last_change":"2023-09-23T20:57:20.708003+0000","last_active":"2023-09-23T20:59:17.571807+0000","last_peered":"2023-09-23T20:59:17.571807+0000","last_clean":"2023-09-23T20:59:17.571807+0000","last_became_active":"2023-09-23T20:57:15.975302+0000","last_became_peered":"2023-09-23T20:57:15.975302+0000","last_unstale":"2023-09-23T20:59:17.571807+0000","last_undegraded":"2023-09-23T20:59:17.571807+0000","last_fullsized":"2023-09-23T20:59:17.571807+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:20.707939+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:20.707939+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:16:51.721994+0000","scrub_duration":0.0017733409999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":215,"reported_epoch":239,"state":"active+clean","last_fresh":"2023-09-23T20:59:17.555428+0000","last_change":"2023-09-23T20:57:44.979736+0000","last_active":"2023-09-23T20:59:17.555428+0000","last_peered":"2023-09-23T20:59:17.555428+0000","last_clean":"2023-09-23T20:59:17.555428+0000","last_became_active":"2023-09-23T20:57:44.979356+0000","last_became_peered":"2023-09-23T20:57:44.979356+0000","last_unstale":"2023-09-23T20:59:17.555428+0000","last_undegraded":"2023-09-23T20:59:17.555428+0000","last_fullsized":"2023-09-23T20:59:17.555428+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:14:43.848832+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":113,"reported_epoch":239,"state":"active+clean","last_fresh":"2023-09-23T20:59:17.572268+0000","last_change":"2023-09-23T20:54:31.887814+0000","last_active":"2023-09-23T20:59:17.572268+0000","last_peered":"2023-09-23T20:59:17.572268+0000","last_clean":"2023-09-23T20:59:17.572268+0000","last_became_active":"2023-09-23T20:54:31.887195+0000","last_became_peered":"2023-09-23T20:54:31.887195+0000","last_unstale":"2023-09-23T20:59:17.572268+0000","last_undegraded":"2023-09-23T20:59:17.572268+0000","last_fullsized":"2023-09-23T20:59:17.572268+0000","mapping_epoch":136,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:10:52.089553+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.13","version":"79'161","reported_seq":540,"reported_epoch":240,"state":"active+clean","last_fresh":"2023-09-23T20:59:27.113925+0000","last_change":"2023-09-23T20:57:48.262970+0000","last_active":"2023-09-23T20:59:27.113925+0000","last_peered":"2023-09-23T20:59:27.113925+0000","last_clean":"2023-09-23T20:59:27.113925+0000","last_became_active":"2023-09-23T20:57:48.262663+0000","last_became_peered":"2023-09-23T20:57:48.262663+0000","last_unstale":"2023-09-23T20:59:27.113925+0000","last_undegraded":"2023-09-23T20:59:27.113925+0000","last_fullsized":"2023-09-23T20:59:27.113925+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:36.420362+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:36.420362+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:28:28.613492+0000","scrub_duration":0.028838444000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":184,"reported_epoch":239,"state":"active+clean","last_fresh":"2023-09-23T20:59:17.556694+0000","last_change":"2023-09-23T20:56:45.991822+0000","last_active":"2023-09-23T20:59:17.556694+0000","last_peered":"2023-09-23T20:59:17.556694+0000","last_clean":"2023-09-23T20:59:17.556694+0000","last_became_active":"2023-09-23T20:56:45.991460+0000","last_became_peered":"2023-09-23T20:56:45.991460+0000","last_unstale":"2023-09-23T20:59:17.556694+0000","last_undegraded":"2023-09-23T20:59:17.556694+0000","last_fullsized":"2023-09-23T20:59:17.556694+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":205,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:20:37.503317+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":197,"reported_epoch":239,"state":"active+clean","last_fresh":"2023-09-23T20:59:17.556786+0000","last_change":"2023-09-23T20:51:10.840849+0000","last_active":"2023-09-23T20:59:17.556786+0000","last_peered":"2023-09-23T20:59:17.556786+0000","last_clean":"2023-09-23T20:59:17.556786+0000","last_became_active":"2023-09-23T20:51:10.840593+0000","last_became_peered":"2023-09-23T20:51:10.840593+0000","last_unstale":"2023-09-23T20:59:17.556786+0000","last_undegraded":"2023-09-23T20:59:17.556786+0000","last_fullsized":"2023-09-23T20:59:17.556786+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:30:52.297625+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.12","version":"79'161","reported_seq":477,"reported_epoch":239,"state":"active+clean","last_fresh":"2023-09-23T20:59:17.572739+0000","last_change":"2023-09-23T20:57:21.750765+0000","last_active":"2023-09-23T20:59:17.572739+0000","last_peered":"2023-09-23T20:59:17.572739+0000","last_clean":"2023-09-23T20:59:17.572739+0000","last_became_active":"2023-09-23T20:57:15.979195+0000","last_became_peered":"2023-09-23T20:57:15.979195+0000","last_unstale":"2023-09-23T20:59:17.572739+0000","last_undegraded":"2023-09-23T20:59:17.572739+0000","last_fullsized":"2023-09-23T20:59:17.572739+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:21.750680+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:21.750680+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:03:41.915962+0000","scrub_duration":0.0092391680000000007,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":232,"reported_epoch":239,"state":"active+clean","last_fresh":"2023-09-23T20:59:17.550981+0000","last_change":"2023-09-23T20:55:36.943969+0000","last_active":"2023-09-23T20:59:17.550981+0000","last_peered":"2023-09-23T20:59:17.550981+0000","last_clean":"2023-09-23T20:59:17.550981+0000","last_became_active":"2023-09-23T20:55:36.943580+0000","last_became_peered":"2023-09-23T20:55:36.943580+0000","last_unstale":"2023-09-23T20:59:17.550981+0000","last_undegraded":"2023-09-23T20:59:17.550981+0000","last_fullsized":"2023-09-23T20:59:17.550981+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:49:06.969396+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":28,"reported_epoch":239,"state":"active+clean","last_fresh":"2023-09-23T20:59:17.554354+0000","last_change":"2023-09-23T20:57:44.980498+0000","last_active":"2023-09-23T20:59:17.554354+0000","last_peered":"2023-09-23T20:59:17.554354+0000","last_clean":"2023-09-23T20:59:17.554354+0000","last_became_active":"2023-09-23T20:57:44.980182+0000","last_became_peered":"2023-09-23T20:57:44.980182+0000","last_unstale":"2023-09-23T20:59:17.554354+0000","last_undegraded":"2023-09-23T20:59:17.554354+0000","last_fullsized":"2023-09-23T20:59:17.554354+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:34:51.620544+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.d","version":"79'161","reported_seq":522,"reported_epoch":239,"state":"active+clean","last_fresh":"2023-09-23T20:59:17.555494+0000","last_change":"2023-09-23T20:57:31.075213+0000","last_active":"2023-09-23T20:59:17.555494+0000","last_peered":"2023-09-23T20:59:17.555494+0000","last_clean":"2023-09-23T20:59:17.555494+0000","last_became_active":"2023-09-23T20:57:15.988188+0000","last_became_peered":"2023-09-23T20:57:15.988188+0000","last_unstale":"2023-09-23T20:59:17.555494+0000","last_undegraded":"2023-09-23T20:59:17.555494+0000","last_fullsized":"2023-09-23T20:59:17.555494+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:31.075098+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:31.075098+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:41:52.142362+0000","scrub_duration":0.010304610000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.b","version":"240'9258","reported_seq":9309,"reported_epoch":240,"state":"active+clean","last_fresh":"2023-09-23T20:59:29.297945+0000","last_change":"2023-09-23T20:56:35.543994+0000","last_active":"2023-09-23T20:59:29.297945+0000","last_peered":"2023-09-23T20:59:29.297945+0000","last_clean":"2023-09-23T20:59:29.297945+0000","last_became_active":"2023-09-23T20:56:35.543758+0000","last_became_peered":"2023-09-23T20:56:35.543758+0000","last_unstale":"2023-09-23T20:59:29.297945+0000","last_undegraded":"2023-09-23T20:59:29.297945+0000","last_fullsized":"2023-09-23T20:59:29.297945+0000","mapping_epoch":199,"log_start":"206'2200","ondisk_log_start":"206'2200","created":199,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_clean_scrub_stamp":"2023-09-23T20:56:34.529103+0000","objects_scrubbed":0,"log_size":7058,"log_dups_size":0,"ondisk_log_size":7058,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:21:29.041484+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":224264192,"num_objects":3422,"num_object_clones":0,"num_object_copies":10266,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3422,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9258,"num_write_kb":405760,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":171,"reported_epoch":239,"state":"active+clean","last_fresh":"2023-09-23T20:59:17.556347+0000","last_change":"2023-09-23T20:56:45.991841+0000","last_active":"2023-09-23T20:59:17.556347+0000","last_peered":"2023-09-23T20:59:17.556347+0000","last_clean":"2023-09-23T20:59:17.556347+0000","last_became_active":"2023-09-23T20:56:45.991491+0000","last_became_peered":"2023-09-23T20:56:45.991491+0000","last_unstale":"2023-09-23T20:59:17.556347+0000","last_undegraded":"2023-09-23T20:59:17.556347+0000","last_fullsized":"2023-09-23T20:59:17.556347+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":205,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:30:37.405319+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":130,"reported_epoch":239,"state":"active+clean","last_fresh":"2023-09-23T20:59:17.571913+0000","last_change":"2023-09-23T20:55:35.957768+0000","last_active":"2023-09-23T20:59:17.571913+0000","last_peered":"2023-09-23T20:59:17.571913+0000","last_clean":"2023-09-23T20:59:17.571913+0000","last_became_active":"2023-09-23T20:55:35.957421+0000","last_became_peered":"2023-09-23T20:55:35.957421+0000","last_unstale":"2023-09-23T20:59:17.571913+0000","last_undegraded":"2023-09-23T20:59:17.571913+0000","last_fullsized":"2023-09-23T20:59:17.571913+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:49:41.585961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.c","version":"79'161","reported_seq":541,"reported_epoch":239,"state":"active+clean","last_fresh":"2023-09-23T20:59:24.584983+0000","last_change":"2023-09-23T20:57:22.041071+0000","last_active":"2023-09-23T20:59:24.584983+0000","last_peered":"2023-09-23T20:59:24.584983+0000","last_clean":"2023-09-23T20:59:24.584983+0000","last_became_active":"2023-09-23T20:57:15.973204+0000","last_became_peered":"2023-09-23T20:57:15.973204+0000","last_unstale":"2023-09-23T20:59:24.584983+0000","last_undegraded":"2023-09-23T20:59:24.584983+0000","last_fullsized":"2023-09-23T20:59:24.584983+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:22.040907+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:22.040907+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:47:02.019978+0000","scrub_duration":0.013021176000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.a","version":"240'9359","reported_seq":9407,"reported_epoch":240,"state":"active+clean","last_fresh":"2023-09-23T20:59:29.461942+0000","last_change":"2023-09-23T20:56:35.547120+0000","last_active":"2023-09-23T20:59:29.461942+0000","last_peered":"2023-09-23T20:59:29.461942+0000","last_clean":"2023-09-23T20:59:29.461942+0000","last_became_active":"2023-09-23T20:56:35.546838+0000","last_became_peered":"2023-09-23T20:56:35.546838+0000","last_unstale":"2023-09-23T20:59:29.461942+0000","last_undegraded":"2023-09-23T20:59:29.461942+0000","last_fullsized":"2023-09-23T20:59:29.461942+0000","mapping_epoch":199,"log_start":"209'3200","ondisk_log_start":"209'3200","created":199,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_clean_scrub_stamp":"2023-09-23T20:56:34.529103+0000","objects_scrubbed":0,"log_size":6159,"log_dups_size":0,"ondisk_log_size":6159,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:48:24.132768+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":221315072,"num_objects":3377,"num_object_clones":0,"num_object_copies":10131,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3377,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9359,"num_write_kb":407552,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,5],"acting":[3,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":158,"reported_epoch":239,"state":"active+clean","last_fresh":"2023-09-23T20:59:17.555998+0000","last_change":"2023-09-23T20:56:45.991821+0000","last_active":"2023-09-23T20:59:17.555998+0000","last_peered":"2023-09-23T20:59:17.555998+0000","last_clean":"2023-09-23T20:59:17.555998+0000","last_became_active":"2023-09-23T20:56:45.991447+0000","last_became_peered":"2023-09-23T20:56:45.991447+0000","last_unstale":"2023-09-23T20:59:17.555998+0000","last_undegraded":"2023-09-23T20:59:17.555998+0000","last_fullsized":"2023-09-23T20:59:17.555998+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":205,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:14:25.525222+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":161,"reported_epoch":239,"state":"active+clean","last_fresh":"2023-09-23T20:59:17.560888+0000","last_change":"2023-09-23T20:57:44.982059+0000","last_active":"2023-09-23T20:59:17.560888+0000","last_peered":"2023-09-23T20:59:17.560888+0000","last_clean":"2023-09-23T20:59:17.560888+0000","last_became_active":"2023-09-23T20:57:44.981742+0000","last_became_peered":"2023-09-23T20:57:44.981742+0000","last_unstale":"2023-09-23T20:59:17.560888+0000","last_undegraded":"2023-09-23T20:59:17.560888+0000","last_fullsized":"2023-09-23T20:59:17.560888+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:41:24.042916+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.f","version":"79'161","reported_seq":544,"reported_epoch":239,"state":"active+clean","last_fresh":"2023-09-23T20:59:17.551462+0000","last_change":"2023-09-23T20:57:27.880841+0000","last_active":"2023-09-23T20:59:17.551462+0000","last_peered":"2023-09-23T20:59:17.551462+0000","last_clean":"2023-09-23T20:59:17.551462+0000","last_became_active":"2023-09-23T20:57:15.970925+0000","last_became_peered":"2023-09-23T20:57:15.970925+0000","last_unstale":"2023-09-23T20:59:17.551462+0000","last_undegraded":"2023-09-23T20:59:17.551462+0000","last_fullsized":"2023-09-23T20:59:17.551462+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:27.880776+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:27.880776+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:15:13.438014+0000","scrub_duration":0.0014004899999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.9","version":"240'9431","reported_seq":9482,"reported_epoch":240,"state":"active+clean","last_fresh":"2023-09-23T20:59:31.533264+0000","last_change":"2023-09-23T20:56:35.549387+0000","last_active":"2023-09-23T20:59:31.533264+0000","last_peered":"2023-09-23T20:59:31.533264+0000","last_clean":"2023-09-23T20:59:31.533264+0000","last_became_active":"2023-09-23T20:56:35.549087+0000","last_became_peered":"2023-09-23T20:56:35.549087+0000","last_unstale":"2023-09-23T20:59:31.533264+0000","last_undegraded":"2023-09-23T20:59:31.533264+0000","last_fullsized":"2023-09-23T20:59:31.533264+0000","mapping_epoch":199,"log_start":"206'2100","ondisk_log_start":"206'2100","created":199,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_clean_scrub_stamp":"2023-09-23T20:56:34.529103+0000","objects_scrubbed":0,"log_size":7331,"log_dups_size":0,"ondisk_log_size":7331,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:04:51.547362+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":214630400,"num_objects":3275,"num_object_clones":0,"num_object_copies":9825,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3275,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9431,"num_write_kb":406592,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":144,"reported_epoch":239,"state":"active+clean","last_fresh":"2023-09-23T20:59:17.572215+0000","last_change":"2023-09-23T20:56:45.989153+0000","last_active":"2023-09-23T20:59:17.572215+0000","last_peered":"2023-09-23T20:59:17.572215+0000","last_clean":"2023-09-23T20:59:17.572215+0000","last_became_active":"2023-09-23T20:56:45.988800+0000","last_became_peered":"2023-09-23T20:56:45.988800+0000","last_unstale":"2023-09-23T20:59:17.572215+0000","last_undegraded":"2023-09-23T20:59:17.572215+0000","last_fullsized":"2023-09-23T20:59:17.572215+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":205,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:04:41.130194+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":179,"reported_epoch":239,"state":"active+clean","last_fresh":"2023-09-23T20:59:17.551089+0000","last_change":"2023-09-23T20:55:35.965427+0000","last_active":"2023-09-23T20:59:17.551089+0000","last_peered":"2023-09-23T20:59:17.551089+0000","last_clean":"2023-09-23T20:59:17.551089+0000","last_became_active":"2023-09-23T20:55:35.963969+0000","last_became_peered":"2023-09-23T20:55:35.963969+0000","last_unstale":"2023-09-23T20:59:17.551089+0000","last_undegraded":"2023-09-23T20:59:17.551089+0000","last_fullsized":"2023-09-23T20:59:17.551089+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:38:40.612083+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.e","version":"79'161","reported_seq":451,"reported_epoch":239,"state":"active+clean","last_fresh":"2023-09-23T20:59:17.555152+0000","last_change":"2023-09-23T20:57:34.132738+0000","last_active":"2023-09-23T20:59:17.555152+0000","last_peered":"2023-09-23T20:59:17.555152+0000","last_clean":"2023-09-23T20:59:17.555152+0000","last_became_active":"2023-09-23T20:57:15.974318+0000","last_became_peered":"2023-09-23T20:57:15.974318+0000","last_unstale":"2023-09-23T20:59:17.555152+0000","last_undegraded":"2023-09-23T20:59:17.555152+0000","last_fullsized":"2023-09-23T20:59:17.555152+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:34.132660+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:34.132660+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:07:07.739438+0000","scrub_duration":0.005229694,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.8","version":"240'9541","reported_seq":9592,"reported_epoch":240,"state":"active+clean","last_fresh":"2023-09-23T20:59:31.442088+0000","last_change":"2023-09-23T20:56:35.548119+0000","last_active":"2023-09-23T20:59:31.442088+0000","last_peered":"2023-09-23T20:59:31.442088+0000","last_clean":"2023-09-23T20:59:31.442088+0000","last_became_active":"2023-09-23T20:56:35.547914+0000","last_became_peered":"2023-09-23T20:56:35.547914+0000","last_unstale":"2023-09-23T20:59:31.442088+0000","last_undegraded":"2023-09-23T20:59:31.442088+0000","last_fullsized":"2023-09-23T20:59:31.442088+0000","mapping_epoch":199,"log_start":"202'700","ondisk_log_start":"202'700","created":199,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_clean_scrub_stamp":"2023-09-23T20:56:34.529103+0000","objects_scrubbed":0,"log_size":8841,"log_dups_size":0,"ondisk_log_size":8841,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:42:54.255023+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":217120768,"num_objects":3313,"num_object_clones":0,"num_object_copies":9939,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3313,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9541,"num_write_kb":411328,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":130,"reported_epoch":239,"state":"active+clean","last_fresh":"2023-09-23T20:59:17.571995+0000","last_change":"2023-09-23T20:55:35.962005+0000","last_active":"2023-09-23T20:59:17.571995+0000","last_peered":"2023-09-23T20:59:17.571995+0000","last_clean":"2023-09-23T20:59:17.571995+0000","last_became_active":"2023-09-23T20:55:35.961695+0000","last_became_peered":"2023-09-23T20:55:35.961695+0000","last_unstale":"2023-09-23T20:59:17.571995+0000","last_undegraded":"2023-09-23T20:59:17.571995+0000","last_fullsized":"2023-09-23T20:59:17.571995+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:28:22.876201+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":228,"reported_epoch":239,"state":"active+clean","last_fresh":"2023-09-23T20:59:17.555442+0000","last_change":"2023-09-23T20:54:13.928715+0000","last_active":"2023-09-23T20:59:17.555442+0000","last_peered":"2023-09-23T20:59:17.555442+0000","last_clean":"2023-09-23T20:59:17.555442+0000","last_became_active":"2023-09-23T20:54:13.928200+0000","last_became_peered":"2023-09-23T20:54:13.928200+0000","last_unstale":"2023-09-23T20:59:17.555442+0000","last_undegraded":"2023-09-23T20:59:17.555442+0000","last_fullsized":"2023-09-23T20:59:17.555442+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:41:16.927513+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.f","version":"240'9594","reported_seq":9645,"reported_epoch":240,"state":"active+clean","last_fresh":"2023-09-23T20:59:31.466421+0000","last_change":"2023-09-23T20:56:35.545128+0000","last_active":"2023-09-23T20:59:31.466421+0000","last_peered":"2023-09-23T20:59:31.466421+0000","last_clean":"2023-09-23T20:59:31.466421+0000","last_became_active":"2023-09-23T20:56:35.544838+0000","last_became_peered":"2023-09-23T20:56:35.544838+0000","last_unstale":"2023-09-23T20:59:31.466421+0000","last_undegraded":"2023-09-23T20:59:31.466421+0000","last_fullsized":"2023-09-23T20:59:31.466421+0000","mapping_epoch":199,"log_start":"206'2200","ondisk_log_start":"206'2200","created":199,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_clean_scrub_stamp":"2023-09-23T20:56:34.529103+0000","objects_scrubbed":0,"log_size":7394,"log_dups_size":0,"ondisk_log_size":7394,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:30:43.388717+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":214433792,"num_objects":3272,"num_object_clones":0,"num_object_copies":9816,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3272,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9594,"num_write_kb":411712,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.9","version":"79'161","reported_seq":613,"reported_epoch":239,"state":"active+clean","last_fresh":"2023-09-23T20:59:17.555025+0000","last_change":"2023-09-23T20:56:46.921939+0000","last_active":"2023-09-23T20:59:17.555025+0000","last_peered":"2023-09-23T20:59:17.555025+0000","last_clean":"2023-09-23T20:59:17.555025+0000","last_became_active":"2023-09-23T20:56:45.989908+0000","last_became_peered":"2023-09-23T20:56:45.989908+0000","last_unstale":"2023-09-23T20:59:17.555025+0000","last_undegraded":"2023-09-23T20:59:17.555025+0000","last_fullsized":"2023-09-23T20:59:17.555025+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":205,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:56:46.921739+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:56:46.921739+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:20:42.635110+0000","scrub_duration":0.0071954109999999996,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":229,"reported_epoch":239,"state":"active+clean","last_fresh":"2023-09-23T20:59:17.553626+0000","last_change":"2023-09-23T20:54:13.961788+0000","last_active":"2023-09-23T20:59:17.553626+0000","last_peered":"2023-09-23T20:59:17.553626+0000","last_clean":"2023-09-23T20:59:17.553626+0000","last_became_active":"2023-09-23T20:54:13.940937+0000","last_became_peered":"2023-09-23T20:54:13.940937+0000","last_unstale":"2023-09-23T20:59:17.553626+0000","last_undegraded":"2023-09-23T20:59:17.553626+0000","last_fullsized":"2023-09-23T20:59:17.553626+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:11:56.785299+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":134,"reported_epoch":239,"state":"active+clean","last_fresh":"2023-09-23T20:59:17.555609+0000","last_change":"2023-09-23T20:55:35.949670+0000","last_active":"2023-09-23T20:59:17.555609+0000","last_peered":"2023-09-23T20:59:17.555609+0000","last_clean":"2023-09-23T20:59:17.555609+0000","last_became_active":"2023-09-23T20:55:35.949140+0000","last_became_peered":"2023-09-23T20:55:35.949140+0000","last_unstale":"2023-09-23T20:59:17.555609+0000","last_undegraded":"2023-09-23T20:59:17.555609+0000","last_fullsized":"2023-09-23T20:59:17.555609+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:55:59.340303+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.e","version":"240'9479","reported_seq":9530,"reported_epoch":240,"state":"active+clean","last_fresh":"2023-09-23T20:59:31.460000+0000","last_change":"2023-09-23T20:56:35.547184+0000","last_active":"2023-09-23T20:59:31.460000+0000","last_peered":"2023-09-23T20:59:31.460000+0000","last_clean":"2023-09-23T20:59:31.460000+0000","last_became_active":"2023-09-23T20:56:35.546912+0000","last_became_peered":"2023-09-23T20:56:35.546912+0000","last_unstale":"2023-09-23T20:59:31.460000+0000","last_undegraded":"2023-09-23T20:59:31.460000+0000","last_fullsized":"2023-09-23T20:59:31.460000+0000","mapping_epoch":199,"log_start":"202'600","ondisk_log_start":"202'600","created":199,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_clean_scrub_stamp":"2023-09-23T20:56:34.529103+0000","objects_scrubbed":0,"log_size":8879,"log_dups_size":0,"ondisk_log_size":8879,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:57:07.422792+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":216334336,"num_objects":3301,"num_object_clones":0,"num_object_copies":9903,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3301,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9479,"num_write_kb":408960,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,0],"acting":[2,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.8","version":"79'161","reported_seq":483,"reported_epoch":239,"state":"active+clean","last_fresh":"2023-09-23T20:59:17.572095+0000","last_change":"2023-09-23T20:56:57.505737+0000","last_active":"2023-09-23T20:59:17.572095+0000","last_peered":"2023-09-23T20:59:17.572095+0000","last_clean":"2023-09-23T20:59:17.572095+0000","last_became_active":"2023-09-23T20:56:04.982345+0000","last_became_peered":"2023-09-23T20:56:04.982345+0000","last_unstale":"2023-09-23T20:59:17.572095+0000","last_undegraded":"2023-09-23T20:59:17.572095+0000","last_fullsized":"2023-09-23T20:59:17.572095+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:56:57.505663+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:56:57.505663+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:34:30.109724+0000","scrub_duration":0.026284108,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":202,"reported_epoch":239,"state":"active+clean","last_fresh":"2023-09-23T20:59:17.551175+0000","last_change":"2023-09-23T20:55:35.959942+0000","last_active":"2023-09-23T20:59:17.551175+0000","last_peered":"2023-09-23T20:59:17.551175+0000","last_clean":"2023-09-23T20:59:17.551175+0000","last_became_active":"2023-09-23T20:55:35.957071+0000","last_became_peered":"2023-09-23T20:55:35.957071+0000","last_unstale":"2023-09-23T20:59:17.551175+0000","last_undegraded":"2023-09-23T20:59:17.551175+0000","last_fullsized":"2023-09-23T20:59:17.551175+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:44:30.308036+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":117,"reported_epoch":239,"state":"active+clean","last_fresh":"2023-09-23T20:59:17.555821+0000","last_change":"2023-09-23T20:54:13.951459+0000","last_active":"2023-09-23T20:59:17.555821+0000","last_peered":"2023-09-23T20:59:17.555821+0000","last_clean":"2023-09-23T20:59:17.555821+0000","last_became_active":"2023-09-23T20:54:13.948409+0000","last_became_peered":"2023-09-23T20:54:13.948409+0000","last_unstale":"2023-09-23T20:59:17.555821+0000","last_undegraded":"2023-09-23T20:59:17.555821+0000","last_fullsized":"2023-09-23T20:59:17.555821+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:38:10.148061+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.b","version":"79'161","reported_seq":523,"reported_epoch":239,"state":"active+clean","last_fresh":"2023-09-23T20:59:24.316845+0000","last_change":"2023-09-23T20:57:27.320172+0000","last_active":"2023-09-23T20:59:24.316845+0000","last_peered":"2023-09-23T20:59:24.316845+0000","last_clean":"2023-09-23T20:59:24.316845+0000","last_became_active":"2023-09-23T20:57:16.009630+0000","last_became_peered":"2023-09-23T20:57:16.009630+0000","last_unstale":"2023-09-23T20:59:24.316845+0000","last_undegraded":"2023-09-23T20:59:24.316845+0000","last_fullsized":"2023-09-23T20:59:24.316845+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:27.320022+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:27.320022+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:31:35.221558+0000","scrub_duration":0.0043894679999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,5],"acting":[1,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.d","version":"240'9415","reported_seq":9465,"reported_epoch":240,"state":"active+clean","last_fresh":"2023-09-23T20:59:28.977778+0000","last_change":"2023-09-23T20:56:35.544132+0000","last_active":"2023-09-23T20:59:28.977778+0000","last_peered":"2023-09-23T20:59:28.977778+0000","last_clean":"2023-09-23T20:59:28.977778+0000","last_became_active":"2023-09-23T20:56:35.543836+0000","last_became_peered":"2023-09-23T20:56:35.543836+0000","last_unstale":"2023-09-23T20:59:28.977778+0000","last_undegraded":"2023-09-23T20:59:28.977778+0000","last_fullsized":"2023-09-23T20:59:28.977778+0000","mapping_epoch":199,"log_start":"205'2000","ondisk_log_start":"205'2000","created":199,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_clean_scrub_stamp":"2023-09-23T20:56:34.529103+0000","objects_scrubbed":0,"log_size":7415,"log_dups_size":0,"ondisk_log_size":7415,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:27:14.636991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":221118488,"num_objects":3375,"num_object_clones":0,"num_object_copies":10125,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3375,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":9415,"num_write_kb":409217,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":117,"reported_epoch":239,"state":"active+clean","last_fresh":"2023-09-23T20:59:17.556766+0000","last_change":"2023-09-23T20:54:13.950254+0000","last_active":"2023-09-23T20:59:17.556766+0000","last_peered":"2023-09-23T20:59:17.556766+0000","last_clean":"2023-09-23T20:59:17.556766+0000","last_became_active":"2023-09-23T20:54:13.949934+0000","last_became_peered":"2023-09-23T20:54:13.949934+0000","last_unstale":"2023-09-23T20:59:17.556766+0000","last_undegraded":"2023-09-23T20:59:17.556766+0000","last_fullsized":"2023-09-23T20:59:17.556766+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:30:32.847167+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":134,"reported_epoch":239,"state":"active+clean","last_fresh":"2023-09-23T20:59:17.556074+0000","last_change":"2023-09-23T20:55:35.956153+0000","last_active":"2023-09-23T20:59:17.556074+0000","last_peered":"2023-09-23T20:59:17.556074+0000","last_clean":"2023-09-23T20:59:17.556074+0000","last_became_active":"2023-09-23T20:55:35.955630+0000","last_became_peered":"2023-09-23T20:55:35.955630+0000","last_unstale":"2023-09-23T20:59:17.556074+0000","last_undegraded":"2023-09-23T20:59:17.556074+0000","last_fullsized":"2023-09-23T20:59:17.556074+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:45:20.282929+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.c","version":"240'9363","reported_seq":9422,"reported_epoch":240,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T20:59:29.056145+0000","last_change":"2023-09-23T20:57:46.463005+0000","last_active":"2023-09-23T20:59:29.056145+0000","last_peered":"2023-09-23T20:59:29.056145+0000","last_clean":"2023-09-23T20:57:44.015119+0000","last_became_active":"2023-09-23T20:57:46.448702+0000","last_became_peered":"2023-09-23T20:57:46.448702+0000","last_unstale":"2023-09-23T20:59:29.056145+0000","last_undegraded":"2023-09-23T20:57:46.010874+0000","last_fullsized":"2023-09-23T20:57:45.978434+0000","mapping_epoch":223,"log_start":"205'2000","ondisk_log_start":"205'2000","created":199,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_clean_scrub_stamp":"2023-09-23T20:56:34.529103+0000","objects_scrubbed":0,"log_size":7363,"log_dups_size":0,"ondisk_log_size":7363,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:08:34.180651+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":225771520,"num_objects":3445,"num_object_clones":0,"num_object_copies":10335,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5473,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3445,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9363,"num_write_kb":409856,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,7],"avail_no_missing":["6","2","7"],"object_location_counts":[{"shards":"2,6,7","objects":2514},{"shards":"6,7","objects":-2028}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.a","version":"79'161","reported_seq":483,"reported_epoch":239,"state":"active+clean","last_fresh":"2023-09-23T20:59:17.573004+0000","last_change":"2023-09-23T20:56:44.487778+0000","last_active":"2023-09-23T20:59:17.573004+0000","last_peered":"2023-09-23T20:59:17.573004+0000","last_clean":"2023-09-23T20:59:17.573004+0000","last_became_active":"2023-09-23T20:56:05.006740+0000","last_became_peered":"2023-09-23T20:56:05.006740+0000","last_unstale":"2023-09-23T20:59:17.573004+0000","last_undegraded":"2023-09-23T20:59:17.573004+0000","last_fullsized":"2023-09-23T20:59:17.573004+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:56:44.487678+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:56:44.487678+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:04:50.086749+0000","scrub_duration":0.0017408899999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":28,"reported_epoch":239,"state":"active+clean","last_fresh":"2023-09-23T20:59:17.553004+0000","last_change":"2023-09-23T20:57:44.982513+0000","last_active":"2023-09-23T20:59:17.553004+0000","last_peered":"2023-09-23T20:59:17.553004+0000","last_clean":"2023-09-23T20:59:17.553004+0000","last_became_active":"2023-09-23T20:57:44.982112+0000","last_became_peered":"2023-09-23T20:57:44.982112+0000","last_unstale":"2023-09-23T20:59:17.553004+0000","last_undegraded":"2023-09-23T20:59:17.553004+0000","last_fullsized":"2023-09-23T20:59:17.553004+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:09:05.630041+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":210,"reported_epoch":239,"state":"active+clean","last_fresh":"2023-09-23T20:59:17.564545+0000","last_change":"2023-09-23T20:54:13.940748+0000","last_active":"2023-09-23T20:59:17.564545+0000","last_peered":"2023-09-23T20:59:17.564545+0000","last_clean":"2023-09-23T20:59:17.564545+0000","last_became_active":"2023-09-23T20:54:13.940419+0000","last_became_peered":"2023-09-23T20:54:13.940419+0000","last_unstale":"2023-09-23T20:59:17.564545+0000","last_undegraded":"2023-09-23T20:59:17.564545+0000","last_fullsized":"2023-09-23T20:59:17.564545+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:15:01.923816+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.1","version":"240'9501","reported_seq":9563,"reported_epoch":240,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T20:59:32.174544+0000","last_change":"2023-09-23T20:57:46.483248+0000","last_active":"2023-09-23T20:59:32.174544+0000","last_peered":"2023-09-23T20:59:32.174544+0000","last_clean":"2023-09-23T20:57:44.047957+0000","last_became_active":"2023-09-23T20:57:46.475568+0000","last_became_peered":"2023-09-23T20:57:46.475568+0000","last_unstale":"2023-09-23T20:59:32.174544+0000","last_undegraded":"2023-09-23T20:57:46.083591+0000","last_fullsized":"2023-09-23T20:57:45.978436+0000","mapping_epoch":223,"log_start":"206'2800","ondisk_log_start":"206'2800","created":199,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_clean_scrub_stamp":"2023-09-23T20:56:34.529103+0000","objects_scrubbed":0,"log_size":6701,"log_dups_size":0,"ondisk_log_size":6701,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:22:45.067269+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":215285760,"num_objects":3285,"num_object_clones":0,"num_object_copies":9855,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5470,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3285,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9501,"num_write_kb":409152,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,6],"avail_no_missing":["4","1","6"],"object_location_counts":[{"shards":"1,4,6","objects":2362},{"shards":"4,6","objects":-2185}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.7","version":"79'161","reported_seq":545,"reported_epoch":239,"state":"active+clean","last_fresh":"2023-09-23T20:59:17.551623+0000","last_change":"2023-09-23T20:57:48.243572+0000","last_active":"2023-09-23T20:59:17.551623+0000","last_peered":"2023-09-23T20:59:17.551623+0000","last_clean":"2023-09-23T20:59:17.551623+0000","last_became_active":"2023-09-23T20:57:48.243229+0000","last_became_peered":"2023-09-23T20:57:48.243229+0000","last_unstale":"2023-09-23T20:59:17.551623+0000","last_undegraded":"2023-09-23T20:59:17.551623+0000","last_fullsized":"2023-09-23T20:59:17.551623+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":3,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:25.916867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:25.916867+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:23:19.385831+0000","scrub_duration":0.001665323,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":215,"reported_epoch":239,"state":"active+clean","last_fresh":"2023-09-23T20:59:17.564778+0000","last_change":"2023-09-23T20:50:05.867130+0000","last_active":"2023-09-23T20:59:17.564778+0000","last_peered":"2023-09-23T20:59:17.564778+0000","last_clean":"2023-09-23T20:59:17.564778+0000","last_became_active":"2023-09-23T20:50:05.866758+0000","last_became_peered":"2023-09-23T20:50:05.866758+0000","last_unstale":"2023-09-23T20:59:17.564778+0000","last_undegraded":"2023-09-23T20:59:17.564778+0000","last_fullsized":"2023-09-23T20:59:17.564778+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:55:07.548209+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":113,"reported_epoch":239,"state":"active+clean","last_fresh":"2023-09-23T20:59:17.572299+0000","last_change":"2023-09-23T20:54:31.887604+0000","last_active":"2023-09-23T20:59:17.572299+0000","last_peered":"2023-09-23T20:59:17.572299+0000","last_clean":"2023-09-23T20:59:17.572299+0000","last_became_active":"2023-09-23T20:54:31.886752+0000","last_became_peered":"2023-09-23T20:54:31.886752+0000","last_unstale":"2023-09-23T20:59:17.572299+0000","last_undegraded":"2023-09-23T20:59:17.572299+0000","last_fullsized":"2023-09-23T20:59:17.572299+0000","mapping_epoch":136,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:13:58.872176+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":213,"reported_epoch":239,"state":"active+clean","last_fresh":"2023-09-23T20:59:17.562883+0000","last_change":"2023-09-23T20:54:31.899082+0000","last_active":"2023-09-23T20:59:17.562883+0000","last_peered":"2023-09-23T20:59:17.562883+0000","last_clean":"2023-09-23T20:59:17.562883+0000","last_became_active":"2023-09-23T20:54:31.898521+0000","last_became_peered":"2023-09-23T20:54:31.898521+0000","last_unstale":"2023-09-23T20:59:17.562883+0000","last_undegraded":"2023-09-23T20:59:17.562883+0000","last_fullsized":"2023-09-23T20:59:17.562883+0000","mapping_epoch":136,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:25:36.059711+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.7","version":"240'9414","reported_seq":9463,"reported_epoch":240,"state":"active+clean","last_fresh":"2023-09-23T20:59:29.005031+0000","last_change":"2023-09-23T20:56:35.541695+0000","last_active":"2023-09-23T20:59:29.005031+0000","last_peered":"2023-09-23T20:59:29.005031+0000","last_clean":"2023-09-23T20:59:29.005031+0000","last_became_active":"2023-09-23T20:56:35.541392+0000","last_became_peered":"2023-09-23T20:56:35.541392+0000","last_unstale":"2023-09-23T20:59:29.005031+0000","last_undegraded":"2023-09-23T20:59:29.005031+0000","last_fullsized":"2023-09-23T20:59:29.005031+0000","mapping_epoch":199,"log_start":"205'2000","ondisk_log_start":"205'2000","created":199,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_clean_scrub_stamp":"2023-09-23T20:56:34.529103+0000","objects_scrubbed":0,"log_size":7414,"log_dups_size":0,"ondisk_log_size":7414,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:57:22.478253+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":222560256,"num_objects":3396,"num_object_clones":0,"num_object_copies":10188,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3396,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9414,"num_write_kb":409920,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.1","version":"79'161","reported_seq":503,"reported_epoch":239,"state":"active+clean","last_fresh":"2023-09-23T20:59:17.571869+0000","last_change":"2023-09-23T20:57:19.669346+0000","last_active":"2023-09-23T20:59:17.571869+0000","last_peered":"2023-09-23T20:59:17.571869+0000","last_clean":"2023-09-23T20:59:17.571869+0000","last_became_active":"2023-09-23T20:57:08.246046+0000","last_became_peered":"2023-09-23T20:57:08.246046+0000","last_unstale":"2023-09-23T20:59:17.571869+0000","last_undegraded":"2023-09-23T20:59:17.571869+0000","last_fullsized":"2023-09-23T20:59:17.571869+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:19.669259+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:19.669259+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:14:03.435531+0000","scrub_duration":0.0045759559999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":130,"reported_epoch":239,"state":"active+clean","last_fresh":"2023-09-23T20:59:17.572846+0000","last_change":"2023-09-23T20:55:35.959514+0000","last_active":"2023-09-23T20:59:17.572846+0000","last_peered":"2023-09-23T20:59:17.572846+0000","last_clean":"2023-09-23T20:59:17.572846+0000","last_became_active":"2023-09-23T20:55:35.959244+0000","last_became_peered":"2023-09-23T20:55:35.959244+0000","last_unstale":"2023-09-23T20:59:17.572846+0000","last_undegraded":"2023-09-23T20:59:17.572846+0000","last_fullsized":"2023-09-23T20:59:17.572846+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:47:32.665316+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":134,"reported_epoch":239,"state":"active+clean","last_fresh":"2023-09-23T20:59:17.556426+0000","last_change":"2023-09-23T20:55:35.949000+0000","last_active":"2023-09-23T20:59:17.556426+0000","last_peered":"2023-09-23T20:59:17.556426+0000","last_clean":"2023-09-23T20:59:17.556426+0000","last_became_active":"2023-09-23T20:55:35.948410+0000","last_became_peered":"2023-09-23T20:55:35.948410+0000","last_unstale":"2023-09-23T20:59:17.556426+0000","last_undegraded":"2023-09-23T20:59:17.556426+0000","last_fullsized":"2023-09-23T20:59:17.556426+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:26:50.477289+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":207,"reported_epoch":239,"state":"active+clean","last_fresh":"2023-09-23T20:59:17.564598+0000","last_change":"2023-09-23T20:51:10.829889+0000","last_active":"2023-09-23T20:59:17.564598+0000","last_peered":"2023-09-23T20:59:17.564598+0000","last_clean":"2023-09-23T20:59:17.564598+0000","last_became_active":"2023-09-23T20:51:10.829533+0000","last_became_peered":"2023-09-23T20:51:10.829533+0000","last_unstale":"2023-09-23T20:59:17.564598+0000","last_undegraded":"2023-09-23T20:59:17.564598+0000","last_fullsized":"2023-09-23T20:59:17.564598+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:43:04.603460+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.4","version":"240'9444","reported_seq":9495,"reported_epoch":240,"state":"active+clean","last_fresh":"2023-09-23T20:59:31.537589+0000","last_change":"2023-09-23T20:56:35.549909+0000","last_active":"2023-09-23T20:59:31.537589+0000","last_peered":"2023-09-23T20:59:31.537589+0000","last_clean":"2023-09-23T20:59:31.537589+0000","last_became_active":"2023-09-23T20:56:35.549618+0000","last_became_peered":"2023-09-23T20:56:35.549618+0000","last_unstale":"2023-09-23T20:59:31.537589+0000","last_undegraded":"2023-09-23T20:59:31.537589+0000","last_fullsized":"2023-09-23T20:59:31.537589+0000","mapping_epoch":199,"log_start":"206'2100","ondisk_log_start":"206'2100","created":199,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_clean_scrub_stamp":"2023-09-23T20:56:34.529103+0000","objects_scrubbed":0,"log_size":7344,"log_dups_size":0,"ondisk_log_size":7344,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:24:25.918475+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":214827008,"num_objects":3278,"num_object_clones":0,"num_object_copies":9834,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3278,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9444,"num_write_kb":407104,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.2","version":"79'161","reported_seq":470,"reported_epoch":239,"state":"active+clean","last_fresh":"2023-09-23T20:59:17.555337+0000","last_change":"2023-09-23T20:57:24.802586+0000","last_active":"2023-09-23T20:59:17.555337+0000","last_peered":"2023-09-23T20:59:17.555337+0000","last_clean":"2023-09-23T20:59:17.555337+0000","last_became_active":"2023-09-23T20:57:08.236114+0000","last_became_peered":"2023-09-23T20:57:08.236114+0000","last_unstale":"2023-09-23T20:59:17.555337+0000","last_undegraded":"2023-09-23T20:59:17.555337+0000","last_fullsized":"2023-09-23T20:59:17.555337+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:24.802451+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:24.802451+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:06:22.309980+0000","scrub_duration":0.0090861250000000005,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,4],"acting":[0,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":226,"reported_epoch":239,"state":"active+clean","last_fresh":"2023-09-23T20:59:17.564591+0000","last_change":"2023-09-23T20:51:10.838054+0000","last_active":"2023-09-23T20:59:17.564591+0000","last_peered":"2023-09-23T20:59:17.564591+0000","last_clean":"2023-09-23T20:59:17.564591+0000","last_became_active":"2023-09-23T20:51:10.837750+0000","last_became_peered":"2023-09-23T20:51:10.837750+0000","last_unstale":"2023-09-23T20:59:17.564591+0000","last_undegraded":"2023-09-23T20:59:17.564591+0000","last_fullsized":"2023-09-23T20:59:17.564591+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:15:51.648085+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":207,"reported_epoch":239,"state":"active+clean","last_fresh":"2023-09-23T20:59:17.563046+0000","last_change":"2023-09-23T20:55:35.951146+0000","last_active":"2023-09-23T20:59:17.563046+0000","last_peered":"2023-09-23T20:59:17.563046+0000","last_clean":"2023-09-23T20:59:17.563046+0000","last_became_active":"2023-09-23T20:55:35.950874+0000","last_became_peered":"2023-09-23T20:55:35.950874+0000","last_unstale":"2023-09-23T20:59:17.563046+0000","last_undegraded":"2023-09-23T20:59:17.563046+0000","last_fullsized":"2023-09-23T20:59:17.563046+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:58:13.957677+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":228,"reported_epoch":239,"state":"active+clean","last_fresh":"2023-09-23T20:59:17.564458+0000","last_change":"2023-09-23T20:54:31.898870+0000","last_active":"2023-09-23T20:59:17.564458+0000","last_peered":"2023-09-23T20:59:17.564458+0000","last_clean":"2023-09-23T20:59:17.564458+0000","last_became_active":"2023-09-23T20:54:31.898467+0000","last_became_peered":"2023-09-23T20:54:31.898467+0000","last_unstale":"2023-09-23T20:59:17.564458+0000","last_undegraded":"2023-09-23T20:59:17.564458+0000","last_fullsized":"2023-09-23T20:59:17.564458+0000","mapping_epoch":136,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:56:22.483408+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.6","version":"240'9317","reported_seq":14111,"reported_epoch":240,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-09-23T20:59:31.567542+0000","last_change":"2023-09-23T20:58:39.453197+0000","last_active":"2023-09-23T20:59:31.567542+0000","last_peered":"2023-09-23T20:59:31.567542+0000","last_clean":"2023-09-23T20:57:44.031597+0000","last_became_active":"2023-09-23T20:57:46.443927+0000","last_became_peered":"2023-09-23T20:57:46.443927+0000","last_unstale":"2023-09-23T20:59:31.567542+0000","last_undegraded":"2023-09-23T20:58:39.450464+0000","last_fullsized":"2023-09-23T20:57:45.977227+0000","mapping_epoch":223,"log_start":"205'1900","ondisk_log_start":"205'1900","created":199,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_clean_scrub_stamp":"2023-09-23T20:56:34.529103+0000","objects_scrubbed":0,"log_size":7417,"log_dups_size":0,"ondisk_log_size":7417,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:37:23.911131+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":214368256,"num_objects":3271,"num_object_clones":0,"num_object_copies":9813,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":658,"num_objects_misplaced":2373,"num_objects_unfound":0,"num_objects_dirty":3271,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9317,"num_write_kb":402816,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2367,"num_bytes_recovered":155123712,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7],"avail_no_missing":["0","5","7"],"object_location_counts":[{"shards":"0,5,7","objects":2373},{"shards":"0,7","objects":240}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":113,"reported_epoch":239,"state":"active+clean","last_fresh":"2023-09-23T20:59:17.572100+0000","last_change":"2023-09-23T20:54:31.883504+0000","last_active":"2023-09-23T20:59:17.572100+0000","last_peered":"2023-09-23T20:59:17.572100+0000","last_clean":"2023-09-23T20:59:17.572100+0000","last_became_active":"2023-09-23T20:54:31.883172+0000","last_became_peered":"2023-09-23T20:54:31.883172+0000","last_unstale":"2023-09-23T20:59:17.572100+0000","last_undegraded":"2023-09-23T20:59:17.572100+0000","last_fullsized":"2023-09-23T20:59:17.572100+0000","mapping_epoch":136,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:45:40.289120+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"79'161","reported_seq":470,"reported_epoch":239,"state":"active+clean","last_fresh":"2023-09-23T20:59:17.556640+0000","last_change":"2023-09-23T20:57:20.738417+0000","last_active":"2023-09-23T20:59:17.556640+0000","last_peered":"2023-09-23T20:59:17.556640+0000","last_clean":"2023-09-23T20:59:17.556640+0000","last_became_active":"2023-09-23T20:57:08.236842+0000","last_became_peered":"2023-09-23T20:57:08.236842+0000","last_unstale":"2023-09-23T20:59:17.556640+0000","last_undegraded":"2023-09-23T20:59:17.556640+0000","last_fullsized":"2023-09-23T20:59:17.556640+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":0,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:20.738273+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:20.738273+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:18:28.327990+0000","scrub_duration":0.0088336339999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":218,"reported_epoch":240,"state":"active+clean","last_fresh":"2023-09-23T20:59:27.107689+0000","last_change":"2023-09-23T20:55:35.950777+0000","last_active":"2023-09-23T20:59:27.107689+0000","last_peered":"2023-09-23T20:59:27.107689+0000","last_clean":"2023-09-23T20:59:27.107689+0000","last_became_active":"2023-09-23T20:55:35.950427+0000","last_became_peered":"2023-09-23T20:55:35.950427+0000","last_unstale":"2023-09-23T20:59:27.107689+0000","last_undegraded":"2023-09-23T20:59:27.107689+0000","last_fullsized":"2023-09-23T20:59:27.107689+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:20:10.371751+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":184,"reported_epoch":239,"state":"active+clean","last_fresh":"2023-09-23T20:59:17.551490+0000","last_change":"2023-09-23T20:55:35.978140+0000","last_active":"2023-09-23T20:59:17.551490+0000","last_peered":"2023-09-23T20:59:17.551490+0000","last_clean":"2023-09-23T20:59:17.551490+0000","last_became_active":"2023-09-23T20:55:35.977831+0000","last_became_peered":"2023-09-23T20:55:35.977831+0000","last_unstale":"2023-09-23T20:59:17.551490+0000","last_undegraded":"2023-09-23T20:59:17.551490+0000","last_fullsized":"2023-09-23T20:59:17.551490+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:25:47.678023+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.5","version":"240'9430","reported_seq":9488,"reported_epoch":240,"state":"active+clean","last_fresh":"2023-09-23T20:59:31.432012+0000","last_change":"2023-09-23T20:56:45.990110+0000","last_active":"2023-09-23T20:59:31.432012+0000","last_peered":"2023-09-23T20:59:31.432012+0000","last_clean":"2023-09-23T20:59:31.432012+0000","last_became_active":"2023-09-23T20:56:45.988394+0000","last_became_peered":"2023-09-23T20:56:45.988394+0000","last_unstale":"2023-09-23T20:59:31.432012+0000","last_undegraded":"2023-09-23T20:59:31.432012+0000","last_fullsized":"2023-09-23T20:59:31.432012+0000","mapping_epoch":204,"log_start":"202'600","ondisk_log_start":"202'600","created":199,"last_epoch_clean":205,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_clean_scrub_stamp":"2023-09-23T20:56:34.529103+0000","objects_scrubbed":0,"log_size":8830,"log_dups_size":0,"ondisk_log_size":8830,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:22:03.814210+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":212860928,"num_objects":3248,"num_object_clones":0,"num_object_copies":9744,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3248,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9430,"num_write_kb":405696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,0],"acting":[2,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"79'161","reported_seq":506,"reported_epoch":239,"state":"active+clean","last_fresh":"2023-09-23T20:59:17.563969+0000","last_change":"2023-09-23T20:57:20.378531+0000","last_active":"2023-09-23T20:59:17.563969+0000","last_peered":"2023-09-23T20:59:17.563969+0000","last_clean":"2023-09-23T20:59:17.563969+0000","last_became_active":"2023-09-23T20:57:08.238019+0000","last_became_peered":"2023-09-23T20:57:08.238019+0000","last_unstale":"2023-09-23T20:59:17.563969+0000","last_undegraded":"2023-09-23T20:59:17.563969+0000","last_fullsized":"2023-09-23T20:59:17.563969+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":3,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:20.377714+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:20.377714+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:00:08.305602+0000","scrub_duration":0.003015228,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":109,"reported_epoch":239,"state":"active+clean","last_fresh":"2023-09-23T20:59:17.563171+0000","last_change":"2023-09-23T20:56:45.991884+0000","last_active":"2023-09-23T20:59:17.563171+0000","last_peered":"2023-09-23T20:59:17.563171+0000","last_clean":"2023-09-23T20:59:17.563171+0000","last_became_active":"2023-09-23T20:56:45.991568+0000","last_became_peered":"2023-09-23T20:56:45.991568+0000","last_unstale":"2023-09-23T20:59:17.563171+0000","last_undegraded":"2023-09-23T20:59:17.563171+0000","last_fullsized":"2023-09-23T20:59:17.563171+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":205,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:21:20.775511+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,0],"acting":[1,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":233,"reported_epoch":239,"state":"active+clean","last_fresh":"2023-09-23T20:59:17.556605+0000","last_change":"2023-09-23T20:57:44.979950+0000","last_active":"2023-09-23T20:59:17.556605+0000","last_peered":"2023-09-23T20:59:17.556605+0000","last_clean":"2023-09-23T20:59:17.556605+0000","last_became_active":"2023-09-23T20:57:44.979346+0000","last_became_peered":"2023-09-23T20:57:44.979346+0000","last_unstale":"2023-09-23T20:59:17.556605+0000","last_undegraded":"2023-09-23T20:59:17.556605+0000","last_fullsized":"2023-09-23T20:59:17.556605+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:10:24.372800+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":212,"reported_epoch":239,"state":"active+clean","last_fresh":"2023-09-23T20:59:17.556232+0000","last_change":"2023-09-23T20:50:37.982235+0000","last_active":"2023-09-23T20:59:17.556232+0000","last_peered":"2023-09-23T20:59:17.556232+0000","last_clean":"2023-09-23T20:59:17.556232+0000","last_became_active":"2023-09-23T20:50:37.981913+0000","last_became_peered":"2023-09-23T20:50:37.981913+0000","last_unstale":"2023-09-23T20:59:17.556232+0000","last_undegraded":"2023-09-23T20:59:17.556232+0000","last_fullsized":"2023-09-23T20:59:17.556232+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:59:57.122584+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.0","version":"240'9429","reported_seq":9478,"reported_epoch":240,"state":"active+clean","last_fresh":"2023-09-23T20:59:32.093728+0000","last_change":"2023-09-23T20:56:35.550635+0000","last_active":"2023-09-23T20:59:32.093728+0000","last_peered":"2023-09-23T20:59:32.093728+0000","last_clean":"2023-09-23T20:59:32.093728+0000","last_became_active":"2023-09-23T20:56:35.550376+0000","last_became_peered":"2023-09-23T20:56:35.550376+0000","last_unstale":"2023-09-23T20:59:32.093728+0000","last_undegraded":"2023-09-23T20:59:32.093728+0000","last_fullsized":"2023-09-23T20:59:32.093728+0000","mapping_epoch":199,"log_start":"203'800","ondisk_log_start":"203'800","created":199,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_clean_scrub_stamp":"2023-09-23T20:56:34.529103+0000","objects_scrubbed":0,"log_size":8629,"log_dups_size":0,"ondisk_log_size":8629,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:40:55.020049+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":213188608,"num_objects":3253,"num_object_clones":0,"num_object_copies":9759,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3253,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9429,"num_write_kb":405824,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.6","version":"79'161","reported_seq":522,"reported_epoch":239,"state":"active+clean","last_fresh":"2023-09-23T20:59:17.551523+0000","last_change":"2023-09-23T20:57:26.907505+0000","last_active":"2023-09-23T20:59:17.551523+0000","last_peered":"2023-09-23T20:59:17.551523+0000","last_clean":"2023-09-23T20:59:17.551523+0000","last_became_active":"2023-09-23T20:57:08.240321+0000","last_became_peered":"2023-09-23T20:57:08.240321+0000","last_unstale":"2023-09-23T20:59:17.551523+0000","last_undegraded":"2023-09-23T20:59:17.551523+0000","last_fullsized":"2023-09-23T20:59:17.551523+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":3,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:26.907408+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:26.907408+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:34:17.392928+0000","scrub_duration":0.010337038,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":156,"reported_epoch":239,"state":"active+clean","last_fresh":"2023-09-23T20:59:17.556108+0000","last_change":"2023-09-23T20:56:45.988566+0000","last_active":"2023-09-23T20:59:17.556108+0000","last_peered":"2023-09-23T20:59:17.556108+0000","last_clean":"2023-09-23T20:59:17.556108+0000","last_became_active":"2023-09-23T20:56:45.988249+0000","last_became_peered":"2023-09-23T20:56:45.988249+0000","last_unstale":"2023-09-23T20:59:17.556108+0000","last_undegraded":"2023-09-23T20:59:17.556108+0000","last_fullsized":"2023-09-23T20:59:17.556108+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":205,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:51:36.644764+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,4],"acting":[0,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":210,"reported_epoch":239,"state":"active+clean","last_fresh":"2023-09-23T20:59:17.564699+0000","last_change":"2023-09-23T20:54:31.898965+0000","last_active":"2023-09-23T20:59:17.564699+0000","last_peered":"2023-09-23T20:59:17.564699+0000","last_clean":"2023-09-23T20:59:17.564699+0000","last_became_active":"2023-09-23T20:54:31.898663+0000","last_became_peered":"2023-09-23T20:54:31.898663+0000","last_unstale":"2023-09-23T20:59:17.564699+0000","last_undegraded":"2023-09-23T20:59:17.564699+0000","last_fullsized":"2023-09-23T20:59:17.564699+0000","mapping_epoch":136,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:53:05.058087+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":200,"reported_epoch":239,"state":"active+clean","last_fresh":"2023-09-23T20:59:17.555675+0000","last_change":"2023-09-23T20:54:31.884717+0000","last_active":"2023-09-23T20:59:17.555675+0000","last_peered":"2023-09-23T20:59:17.555675+0000","last_clean":"2023-09-23T20:59:17.555675+0000","last_became_active":"2023-09-23T20:54:31.884345+0000","last_became_peered":"2023-09-23T20:54:31.884345+0000","last_unstale":"2023-09-23T20:59:17.555675+0000","last_undegraded":"2023-09-23T20:59:17.555675+0000","last_fullsized":"2023-09-23T20:59:17.555675+0000","mapping_epoch":136,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:02:26.393868+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.3","version":"240'9176","reported_seq":9227,"reported_epoch":240,"state":"active+clean","last_fresh":"2023-09-23T20:59:29.533243+0000","last_change":"2023-09-23T20:56:35.549655+0000","last_active":"2023-09-23T20:59:29.533243+0000","last_peered":"2023-09-23T20:59:29.533243+0000","last_clean":"2023-09-23T20:59:29.533243+0000","last_became_active":"2023-09-23T20:56:35.549357+0000","last_became_peered":"2023-09-23T20:56:35.549357+0000","last_unstale":"2023-09-23T20:59:29.533243+0000","last_undegraded":"2023-09-23T20:59:29.533243+0000","last_fullsized":"2023-09-23T20:59:29.533243+0000","mapping_epoch":199,"log_start":"207'3000","ondisk_log_start":"207'3000","created":199,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_clean_scrub_stamp":"2023-09-23T20:56:34.529103+0000","objects_scrubbed":0,"log_size":6176,"log_dups_size":0,"ondisk_log_size":6176,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:37:58.752518+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":219283456,"num_objects":3346,"num_object_clones":0,"num_object_copies":10038,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3346,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9176,"num_write_kb":400704,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.5","version":"79'161","reported_seq":503,"reported_epoch":239,"state":"active+clean","last_fresh":"2023-09-23T20:59:17.555567+0000","last_change":"2023-09-23T20:57:30.029479+0000","last_active":"2023-09-23T20:59:17.555567+0000","last_peered":"2023-09-23T20:59:17.555567+0000","last_clean":"2023-09-23T20:59:17.555567+0000","last_became_active":"2023-09-23T20:57:08.249909+0000","last_became_peered":"2023-09-23T20:57:08.249909+0000","last_unstale":"2023-09-23T20:59:17.555567+0000","last_undegraded":"2023-09-23T20:59:17.555567+0000","last_fullsized":"2023-09-23T20:59:17.555567+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":3,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:30.029396+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:30.029396+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:16:46.083145+0000","scrub_duration":0.012914682,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":203,"reported_epoch":239,"state":"active+clean","last_fresh":"2023-09-23T20:59:17.554654+0000","last_change":"2023-09-23T20:51:10.836910+0000","last_active":"2023-09-23T20:59:17.554654+0000","last_peered":"2023-09-23T20:59:17.554654+0000","last_clean":"2023-09-23T20:59:17.554654+0000","last_became_active":"2023-09-23T20:51:10.836557+0000","last_became_peered":"2023-09-23T20:51:10.836557+0000","last_unstale":"2023-09-23T20:59:17.554654+0000","last_undegraded":"2023-09-23T20:59:17.554654+0000","last_fullsized":"2023-09-23T20:59:17.554654+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:51:54.019969+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":210,"reported_epoch":239,"state":"active+clean","last_fresh":"2023-09-23T20:59:17.564161+0000","last_change":"2023-09-23T20:55:35.953444+0000","last_active":"2023-09-23T20:59:17.564161+0000","last_peered":"2023-09-23T20:59:17.564161+0000","last_clean":"2023-09-23T20:59:17.564161+0000","last_became_active":"2023-09-23T20:55:35.953151+0000","last_became_peered":"2023-09-23T20:55:35.953151+0000","last_unstale":"2023-09-23T20:59:17.564161+0000","last_undegraded":"2023-09-23T20:59:17.564161+0000","last_fullsized":"2023-09-23T20:59:17.564161+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:39:12.404391+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":28,"reported_epoch":239,"state":"active+clean","last_fresh":"2023-09-23T20:59:17.553084+0000","last_change":"2023-09-23T20:57:44.982509+0000","last_active":"2023-09-23T20:59:17.553084+0000","last_peered":"2023-09-23T20:59:17.553084+0000","last_clean":"2023-09-23T20:59:17.553084+0000","last_became_active":"2023-09-23T20:57:44.982175+0000","last_became_peered":"2023-09-23T20:57:44.982175+0000","last_unstale":"2023-09-23T20:59:17.553084+0000","last_undegraded":"2023-09-23T20:59:17.553084+0000","last_fullsized":"2023-09-23T20:59:17.553084+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:37:49.215488+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.2","version":"240'9227","reported_seq":9278,"reported_epoch":240,"state":"active+clean","last_fresh":"2023-09-23T20:59:29.545276+0000","last_change":"2023-09-23T20:56:35.543020+0000","last_active":"2023-09-23T20:59:29.545276+0000","last_peered":"2023-09-23T20:59:29.545276+0000","last_clean":"2023-09-23T20:59:29.545276+0000","last_became_active":"2023-09-23T20:56:35.542719+0000","last_became_peered":"2023-09-23T20:56:35.542719+0000","last_unstale":"2023-09-23T20:59:29.545276+0000","last_undegraded":"2023-09-23T20:59:29.545276+0000","last_fullsized":"2023-09-23T20:59:29.545276+0000","mapping_epoch":199,"log_start":"208'3100","ondisk_log_start":"208'3100","created":199,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_clean_scrub_stamp":"2023-09-23T20:56:34.529103+0000","objects_scrubbed":0,"log_size":6127,"log_dups_size":0,"ondisk_log_size":6127,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:52:21.088628+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":215285760,"num_objects":3285,"num_object_clones":0,"num_object_copies":9855,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3285,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9227,"num_write_kb":400384,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.4","version":"79'161","reported_seq":518,"reported_epoch":239,"state":"active+clean","last_fresh":"2023-09-23T20:59:17.553154+0000","last_change":"2023-09-23T20:57:20.012904+0000","last_active":"2023-09-23T20:59:17.553154+0000","last_peered":"2023-09-23T20:59:17.553154+0000","last_clean":"2023-09-23T20:59:17.553154+0000","last_became_active":"2023-09-23T20:57:08.237996+0000","last_became_peered":"2023-09-23T20:57:08.237996+0000","last_unstale":"2023-09-23T20:59:17.553154+0000","last_undegraded":"2023-09-23T20:59:17.553154+0000","last_fullsized":"2023-09-23T20:59:17.553154+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":3,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:20.012764+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:20.012764+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:51:28.796946+0000","scrub_duration":0.0045251919999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":189,"reported_epoch":239,"state":"active+clean","last_fresh":"2023-09-23T20:59:17.551529+0000","last_change":"2023-09-23T20:55:35.949541+0000","last_active":"2023-09-23T20:59:17.551529+0000","last_peered":"2023-09-23T20:59:17.551529+0000","last_clean":"2023-09-23T20:59:17.551529+0000","last_became_active":"2023-09-23T20:55:35.944635+0000","last_became_peered":"2023-09-23T20:55:35.944635+0000","last_unstale":"2023-09-23T20:59:17.551529+0000","last_undegraded":"2023-09-23T20:59:17.551529+0000","last_fullsized":"2023-09-23T20:59:17.551529+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:30:36.033585+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"70'1","reported_seq":228,"reported_epoch":239,"state":"active+clean","last_fresh":"2023-09-23T20:59:17.556322+0000","last_change":"2023-09-23T20:54:14.009723+0000","last_active":"2023-09-23T20:59:17.556322+0000","last_peered":"2023-09-23T20:59:17.556322+0000","last_clean":"2023-09-23T20:59:17.556322+0000","last_became_active":"2023-09-23T20:54:13.941771+0000","last_became_peered":"2023-09-23T20:54:13.941771+0000","last_unstale":"2023-09-23T20:59:17.556322+0000","last_undegraded":"2023-09-23T20:59:17.556322+0000","last_fullsized":"2023-09-23T20:59:17.556322+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:48:25.588116+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":144,"reported_epoch":239,"state":"active+clean","last_fresh":"2023-09-23T20:59:17.556832+0000","last_change":"2023-09-23T20:55:35.949780+0000","last_active":"2023-09-23T20:59:17.556832+0000","last_peered":"2023-09-23T20:59:17.556832+0000","last_clean":"2023-09-23T20:59:17.556832+0000","last_became_active":"2023-09-23T20:55:35.949497+0000","last_became_peered":"2023-09-23T20:55:35.949497+0000","last_unstale":"2023-09-23T20:59:17.556832+0000","last_undegraded":"2023-09-23T20:59:17.556832+0000","last_fullsized":"2023-09-23T20:59:17.556832+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:09:37.420882+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":203,"reported_epoch":240,"state":"active+clean","last_fresh":"2023-09-23T20:59:27.103898+0000","last_change":"2023-09-23T20:54:13.929196+0000","last_active":"2023-09-23T20:59:27.103898+0000","last_peered":"2023-09-23T20:59:27.103898+0000","last_clean":"2023-09-23T20:59:27.103898+0000","last_became_active":"2023-09-23T20:54:13.928831+0000","last_became_peered":"2023-09-23T20:54:13.928831+0000","last_unstale":"2023-09-23T20:59:27.103898+0000","last_undegraded":"2023-09-23T20:59:27.103898+0000","last_fullsized":"2023-09-23T20:59:27.103898+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:28:44.316851+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":208,"reported_epoch":239,"state":"active+clean","last_fresh":"2023-09-23T20:59:17.565832+0000","last_change":"2023-09-23T20:52:12.855430+0000","last_active":"2023-09-23T20:59:17.565832+0000","last_peered":"2023-09-23T20:59:17.565832+0000","last_clean":"2023-09-23T20:59:17.565832+0000","last_became_active":"2023-09-23T20:52:12.855115+0000","last_became_peered":"2023-09-23T20:52:12.855115+0000","last_unstale":"2023-09-23T20:59:17.565832+0000","last_undegraded":"2023-09-23T20:59:17.565832+0000","last_fullsized":"2023-09-23T20:59:17.565832+0000","mapping_epoch":100,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":101,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:33:23.802214+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":117,"reported_epoch":239,"state":"active+clean","last_fresh":"2023-09-23T20:59:17.572109+0000","last_change":"2023-09-23T20:54:31.902946+0000","last_active":"2023-09-23T20:59:17.572109+0000","last_peered":"2023-09-23T20:59:17.572109+0000","last_clean":"2023-09-23T20:59:17.572109+0000","last_became_active":"2023-09-23T20:54:31.889484+0000","last_became_peered":"2023-09-23T20:54:31.889484+0000","last_unstale":"2023-09-23T20:59:17.572109+0000","last_undegraded":"2023-09-23T20:59:17.572109+0000","last_fullsized":"2023-09-23T20:59:17.572109+0000","mapping_epoch":136,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:44:44.278398+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":117,"reported_epoch":239,"state":"active+clean","last_fresh":"2023-09-23T20:59:17.556544+0000","last_change":"2023-09-23T20:54:13.950351+0000","last_active":"2023-09-23T20:59:17.556544+0000","last_peered":"2023-09-23T20:59:17.556544+0000","last_clean":"2023-09-23T20:59:17.556544+0000","last_became_active":"2023-09-23T20:54:13.949987+0000","last_became_peered":"2023-09-23T20:54:13.949987+0000","last_unstale":"2023-09-23T20:59:17.556544+0000","last_undegraded":"2023-09-23T20:59:17.556544+0000","last_fullsized":"2023-09-23T20:59:17.556544+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:46:17.765134+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":148,"reported_epoch":239,"state":"active+clean","last_fresh":"2023-09-23T20:59:17.572325+0000","last_change":"2023-09-23T20:56:18.141728+0000","last_active":"2023-09-23T20:59:17.572325+0000","last_peered":"2023-09-23T20:59:17.572325+0000","last_clean":"2023-09-23T20:59:17.572325+0000","last_became_active":"2023-09-23T20:56:18.137979+0000","last_became_peered":"2023-09-23T20:56:18.137979+0000","last_unstale":"2023-09-23T20:59:17.572325+0000","last_undegraded":"2023-09-23T20:59:17.572325+0000","last_fullsized":"2023-09-23T20:59:17.572325+0000","mapping_epoch":192,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":193,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:58:05.658443+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":117,"reported_epoch":239,"state":"active+clean","last_fresh":"2023-09-23T20:59:17.555405+0000","last_change":"2023-09-23T20:54:13.935307+0000","last_active":"2023-09-23T20:59:17.555405+0000","last_peered":"2023-09-23T20:59:17.555405+0000","last_clean":"2023-09-23T20:59:17.555405+0000","last_became_active":"2023-09-23T20:54:13.935000+0000","last_became_peered":"2023-09-23T20:54:13.935000+0000","last_unstale":"2023-09-23T20:59:17.555405+0000","last_undegraded":"2023-09-23T20:59:17.555405+0000","last_fullsized":"2023-09-23T20:59:17.555405+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:47:40.974082+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":227,"reported_epoch":239,"state":"active+clean","last_fresh":"2023-09-23T20:59:17.554219+0000","last_change":"2023-09-23T20:55:35.954870+0000","last_active":"2023-09-23T20:59:17.554219+0000","last_peered":"2023-09-23T20:59:17.554219+0000","last_clean":"2023-09-23T20:59:17.554219+0000","last_became_active":"2023-09-23T20:55:35.954568+0000","last_became_peered":"2023-09-23T20:55:35.954568+0000","last_unstale":"2023-09-23T20:59:17.554219+0000","last_undegraded":"2023-09-23T20:59:17.554219+0000","last_fullsized":"2023-09-23T20:59:17.554219+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:06:22.423566+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":226,"reported_epoch":239,"state":"active+clean","last_fresh":"2023-09-23T20:59:17.554949+0000","last_change":"2023-09-23T20:51:10.838334+0000","last_active":"2023-09-23T20:59:17.554949+0000","last_peered":"2023-09-23T20:59:17.554949+0000","last_clean":"2023-09-23T20:59:17.554949+0000","last_became_active":"2023-09-23T20:51:10.838026+0000","last_became_peered":"2023-09-23T20:51:10.838026+0000","last_unstale":"2023-09-23T20:59:17.554949+0000","last_undegraded":"2023-09-23T20:59:17.554949+0000","last_fullsized":"2023-09-23T20:59:17.554949+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:04:37.009457+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":230,"reported_epoch":239,"state":"active+clean","last_fresh":"2023-09-23T20:59:17.555827+0000","last_change":"2023-09-23T20:54:31.903139+0000","last_active":"2023-09-23T20:59:17.555827+0000","last_peered":"2023-09-23T20:59:17.555827+0000","last_clean":"2023-09-23T20:59:17.555827+0000","last_became_active":"2023-09-23T20:54:31.884520+0000","last_became_peered":"2023-09-23T20:54:31.884520+0000","last_unstale":"2023-09-23T20:59:17.555827+0000","last_undegraded":"2023-09-23T20:59:17.555827+0000","last_fullsized":"2023-09-23T20:59:17.555827+0000","mapping_epoch":136,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:39:34.269705+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1108,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":134,"reported_epoch":239,"state":"active+clean","last_fresh":"2023-09-23T20:59:17.555046+0000","last_change":"2023-09-23T20:55:35.951572+0000","last_active":"2023-09-23T20:59:17.555046+0000","last_peered":"2023-09-23T20:59:17.555046+0000","last_clean":"2023-09-23T20:59:17.555046+0000","last_became_active":"2023-09-23T20:55:35.951294+0000","last_became_peered":"2023-09-23T20:55:35.951294+0000","last_unstale":"2023-09-23T20:59:17.555046+0000","last_undegraded":"2023-09-23T20:59:17.555046+0000","last_fullsized":"2023-09-23T20:59:17.555046+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:36:32.447210+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":7,"num_pg":16,"stat_sum":{"num_bytes":3482648600,"num_objects":53142,"num_object_clones":0,"num_object_copies":159426,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":11601,"num_objects_misplaced":2373,"num_objects_unfound":0,"num_objects_dirty":53142,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":150378,"num_write_kb":6512577,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2368,"num_bytes_recovered":155189248,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":11006128128,"data_stored":11006115912,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":119078,"ondisk_log_size":119078,"up":48,"acting":45,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":22,"num_bytes_recovered":5990,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":21,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3381,"ondisk_log_size":3381,"up":63,"acting":63,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201509,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1243972,"kb_used_data":1058364,"kb_used_omap":0,"kb_used_meta":185600,"kb_avail":92501180,"statfs":{"total":95995035648,"available":94721208320,"internally_reserved":0,"allocated":1083764736,"data_stored":1082913068,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":190054400},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:58:44 2023","interfaces":[{"interface":"back","average":{"1min":2.052,"5min":1.859,"15min":1.006},"min":{"1min":0.248,"5min":0.21099999999999999,"15min":0.21099999999999999},"max":{"1min":5.5279999999999996,"5min":43.994999999999997,"15min":43.994999999999997},"last":0.92100000000000004},{"interface":"front","average":{"1min":2.0489999999999999,"5min":2.0760000000000001,"15min":1.099},"min":{"1min":0.34599999999999997,"5min":0.28199999999999997,"15min":0.218},"max":{"1min":5.5129999999999999,"5min":44.206000000000003,"15min":44.206000000000003},"last":0.93899999999999995}]},{"osd":1,"last update":"Sat Sep 23 20:59:19 2023","interfaces":[{"interface":"back","average":{"1min":1.74,"5min":1.835,"15min":1.2230000000000001},"min":{"1min":0.26600000000000001,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":5.4359999999999999,"5min":10.622,"15min":22.591999999999999},"last":0.373},{"interface":"front","average":{"1min":2.0600000000000001,"5min":1.639,"15min":1.0860000000000001},"min":{"1min":0.32100000000000001,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":6.7709999999999999,"5min":10.097,"15min":21.437000000000001},"last":0.36099999999999999}]},{"osd":2,"last update":"Sat Sep 23 20:58:54 2023","interfaces":[{"interface":"back","average":{"1min":3.7370000000000001,"5min":2.016,"15min":1.1499999999999999},"min":{"1min":0.27600000000000002,"5min":0.26500000000000001,"15min":0.24399999999999999},"max":{"1min":10.805999999999999,"5min":10.805999999999999,"15min":10.805999999999999},"last":3.593},{"interface":"front","average":{"1min":2.3969999999999998,"5min":2.1520000000000001,"15min":1.1919999999999999},"min":{"1min":0.27800000000000002,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":9.4290000000000003,"5min":45.25,"15min":45.25},"last":3.5209999999999999}]},{"osd":3,"last update":"Sat Sep 23 20:59:30 2023","interfaces":[{"interface":"back","average":{"1min":2.6200000000000001,"5min":2.508,"15min":1.7250000000000001},"min":{"1min":0.33500000000000002,"5min":0.32100000000000001,"15min":0.314},"max":{"1min":8.4390000000000001,"5min":13.007,"15min":36.408000000000001},"last":3.1379999999999999},{"interface":"front","average":{"1min":2.7789999999999999,"5min":2.3079999999999998,"15min":1.552},"min":{"1min":0.41599999999999998,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":13.198,"5min":14.220000000000001,"15min":35.593000000000004},"last":0.60599999999999998}]},{"osd":4,"last update":"Sat Sep 23 20:59:19 2023","interfaces":[{"interface":"back","average":{"1min":0.41099999999999998,"5min":0.78500000000000003,"15min":0.60499999999999998},"min":{"1min":0.25700000000000001,"5min":0.23799999999999999,"15min":0.23699999999999999},"max":{"1min":1.1659999999999999,"5min":5.8920000000000003,"15min":5.8920000000000003},"last":0.40899999999999997},{"interface":"front","average":{"1min":0.51900000000000002,"5min":0.77600000000000002,"15min":0.61299999999999999},"min":{"1min":0.28599999999999998,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":2.952,"5min":5.3200000000000003,"15min":5.3200000000000003},"last":0.63700000000000001}]},{"osd":5,"last update":"Sat Sep 23 20:59:19 2023","interfaces":[{"interface":"back","average":{"1min":0.443,"5min":0.78100000000000003,"15min":0.628},"min":{"1min":0.316,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":0.64000000000000001,"5min":7.0389999999999997,"15min":7.0389999999999997},"last":0.42499999999999999},{"interface":"front","average":{"1min":0.55200000000000005,"5min":0.72299999999999998,"15min":0.61399999999999999},"min":{"1min":0.30199999999999999,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":1.909,"5min":4.5999999999999996,"15min":4.5999999999999996},"last":0.51500000000000001}]},{"osd":6,"last update":"Sat Sep 23 20:58:40 2023","interfaces":[{"interface":"back","average":{"1min":0.86599999999999999,"5min":0.71699999999999997,"15min":0.68300000000000005},"min":{"1min":0.33700000000000002,"5min":0.31,"15min":0.31},"max":{"1min":3.5249999999999999,"5min":3.5249999999999999,"15min":3.5249999999999999},"last":0.39200000000000002},{"interface":"front","average":{"1min":0.90100000000000002,"5min":0.81299999999999994,"15min":0.80700000000000005},"min":{"1min":0.36899999999999999,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":3.5409999999999999,"5min":3.5409999999999999,"15min":3.5409999999999999},"last":0.441}]}]},{"osd":6,"up_from":164,"seq":704374636592,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1205880,"kb_used_data":1099248,"kb_used_omap":10,"kb_used_meta":106613,"kb_avail":92539272,"statfs":{"total":95995035648,"available":94760214528,"internally_reserved":0,"allocated":1125629952,"data_stored":1124790185,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":10328,"internal_metadata":109172648},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:58:43 2023","interfaces":[{"interface":"back","average":{"1min":2.7330000000000001,"5min":1.4410000000000001,"15min":1.0569999999999999},"min":{"1min":0.25800000000000001,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":7.5179999999999998,"5min":8.7319999999999993,"15min":8.7319999999999993},"last":2.226},{"interface":"front","average":{"1min":3.1709999999999998,"5min":1.502,"15min":1.1559999999999999},"min":{"1min":0.25,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":6.7140000000000004,"5min":6.7140000000000004,"15min":6.7140000000000004},"last":0.28100000000000003}]},{"osd":1,"last update":"Sat Sep 23 20:58:43 2023","interfaces":[{"interface":"back","average":{"1min":1.5449999999999999,"5min":1.097,"15min":0.89100000000000001},"min":{"1min":0.36799999999999999,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":5.2350000000000003,"5min":6.6029999999999998,"15min":6.6029999999999998},"last":2.4380000000000002},{"interface":"front","average":{"1min":2.5779999999999998,"5min":1.244,"15min":0.99299999999999999},"min":{"1min":0.23000000000000001,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":10.324999999999999,"5min":10.324999999999999,"15min":10.324999999999999},"last":0.26100000000000001}]},{"osd":2,"last update":"Sat Sep 23 20:58:43 2023","interfaces":[{"interface":"back","average":{"1min":2.286,"5min":1.982,"15min":1.758},"min":{"1min":0.28999999999999998,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":7.0519999999999996,"5min":13.750999999999999,"15min":13.750999999999999},"last":6.0670000000000002},{"interface":"front","average":{"1min":2.419,"5min":1.5289999999999999,"15min":1.2470000000000001},"min":{"1min":0.26200000000000001,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":8.0890000000000004,"5min":8.0890000000000004,"15min":8.0890000000000004},"last":1.5920000000000001}]},{"osd":3,"last update":"Sat Sep 23 20:58:43 2023","interfaces":[{"interface":"back","average":{"1min":1.8680000000000001,"5min":1.2849999999999999,"15min":1.173},"min":{"1min":0.32000000000000001,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":6.9000000000000004,"5min":6.9000000000000004,"15min":6.9000000000000004},"last":2.2000000000000002},{"interface":"front","average":{"1min":2.1819999999999999,"5min":1.74,"15min":1.6479999999999999},"min":{"1min":0.28599999999999998,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":11.021000000000001,"5min":11.021000000000001,"15min":11.021000000000001},"last":2.4049999999999998}]},{"osd":4,"last update":"Sat Sep 23 20:58:43 2023","interfaces":[{"interface":"back","average":{"1min":0.49099999999999999,"5min":0.85899999999999999,"15min":0.98699999999999999},"min":{"1min":0.23000000000000001,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":2.7919999999999998,"5min":9.0079999999999991,"15min":9.0079999999999991},"last":0.42599999999999999},{"interface":"front","average":{"1min":0.49299999999999999,"5min":0.50800000000000001,"15min":0.52300000000000002},"min":{"1min":0.27200000000000002,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":2.0059999999999998,"5min":2.0059999999999998,"15min":2.0059999999999998},"last":0.439}]},{"osd":5,"last update":"Sat Sep 23 20:58:43 2023","interfaces":[{"interface":"back","average":{"1min":0.53700000000000003,"5min":0.56899999999999995,"15min":0.55000000000000004},"min":{"1min":0.27300000000000002,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":2.8239999999999998,"5min":2.8239999999999998,"15min":2.8239999999999998},"last":0.45600000000000002},{"interface":"front","average":{"1min":0.81000000000000005,"5min":0.78500000000000003,"15min":0.80100000000000005},"min":{"1min":0.33400000000000002,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":3.1379999999999999,"5min":3.1379999999999999,"15min":3.1379999999999999},"last":0.52700000000000002}]},{"osd":7,"last update":"Sat Sep 23 20:58:43 2023","interfaces":[{"interface":"back","average":{"1min":0.52700000000000002,"5min":0.85399999999999998,"15min":0.90400000000000003},"min":{"1min":0.312,"5min":0.312,"15min":0.312},"max":{"1min":2.073,"5min":7.8570000000000002,"15min":7.8570000000000002},"last":0.47599999999999998},{"interface":"front","average":{"1min":0.61899999999999999,"5min":0.66200000000000003,"15min":0.57399999999999995},"min":{"1min":0.30299999999999999,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":2.6989999999999998,"5min":8.282,"15min":8.282},"last":0.48199999999999998}]}]},{"osd":1,"up_from":88,"seq":377957122163,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1885240,"kb_used_data":1654128,"kb_used_omap":9,"kb_used_meta":231030,"kb_avail":91859912,"statfs":{"total":95995035648,"available":94064549888,"internally_reserved":0,"allocated":1693827072,"data_stored":1692640443,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9239,"internal_metadata":236575721},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:59:23 2023","interfaces":[{"interface":"back","average":{"1min":3.9180000000000001,"5min":3.1070000000000002,"15min":1.5389999999999999},"min":{"1min":0.20699999999999999,"5min":0.20699999999999999,"15min":0.20699999999999999},"max":{"1min":14.813000000000001,"5min":16.646999999999998,"15min":16.646999999999998},"last":0.66200000000000003},{"interface":"front","average":{"1min":3.77,"5min":3.335,"15min":1.6819999999999999},"min":{"1min":0.33300000000000002,"5min":0.246,"15min":0.22500000000000001},"max":{"1min":9.484,"5min":16.664999999999999,"15min":18.457999999999998},"last":0.84299999999999997}]},{"osd":2,"last update":"Sat Sep 23 20:58:53 2023","interfaces":[{"interface":"back","average":{"1min":2.7130000000000001,"5min":2.4140000000000001,"15min":1.1799999999999999},"min":{"1min":0.24299999999999999,"5min":0.221,"15min":0.20300000000000001},"max":{"1min":8.6829999999999998,"5min":14.335000000000001,"15min":14.335000000000001},"last":0.80200000000000005},{"interface":"front","average":{"1min":5.8019999999999996,"5min":3.27,"15min":1.758},"min":{"1min":0.39500000000000002,"5min":0.248,"15min":0.248},"max":{"1min":16.048999999999999,"5min":16.048999999999999,"15min":38.951000000000001},"last":1.171}]},{"osd":3,"last update":"Sat Sep 23 20:59:23 2023","interfaces":[{"interface":"back","average":{"1min":3.1539999999999999,"5min":2.629,"15min":1.5589999999999999},"min":{"1min":0.42499999999999999,"5min":0.246,"15min":0.246},"max":{"1min":9.4450000000000003,"5min":16.055,"15min":25.314},"last":0.38500000000000001},{"interface":"front","average":{"1min":3.0870000000000002,"5min":2.4860000000000002,"15min":1.411},"min":{"1min":0.29799999999999999,"5min":0.20599999999999999,"15min":0.20599999999999999},"max":{"1min":10.204000000000001,"5min":14.225,"15min":19.881},"last":0.36699999999999999}]},{"osd":4,"last update":"Sat Sep 23 20:59:23 2023","interfaces":[{"interface":"back","average":{"1min":1.871,"5min":1.6439999999999999,"15min":1.2130000000000001},"min":{"1min":0.28299999999999997,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":9.3480000000000008,"5min":10.598000000000001,"15min":12.006},"last":0.48399999999999999},{"interface":"front","average":{"1min":1.633,"5min":1.675,"15min":1.0800000000000001},"min":{"1min":0.38400000000000001,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":7.532,"5min":7.532,"15min":11.234999999999999},"last":0.42999999999999999}]},{"osd":5,"last update":"Sat Sep 23 20:59:23 2023","interfaces":[{"interface":"back","average":{"1min":2.3700000000000001,"5min":2.1469999999999998,"15min":1.292},"min":{"1min":0.55800000000000005,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":8.4410000000000007,"5min":15.041,"15min":15.041},"last":0.78100000000000003},{"interface":"front","average":{"1min":1.9370000000000001,"5min":1.756,"15min":1.2390000000000001},"min":{"1min":0.432,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":9.3889999999999993,"5min":10.647,"15min":12.08},"last":0.44400000000000001}]},{"osd":6,"last update":"Sat Sep 23 20:58:39 2023","interfaces":[{"interface":"back","average":{"1min":1.6160000000000001,"5min":1.3700000000000001,"15min":1.333},"min":{"1min":0.30299999999999999,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":10.673999999999999,"5min":10.673999999999999,"15min":10.673999999999999},"last":0.51800000000000002},{"interface":"front","average":{"1min":1.8280000000000001,"5min":1.508,"15min":1.3380000000000001},"min":{"1min":0.38100000000000001,"5min":0.309,"15min":0.309},"max":{"1min":8.7530000000000001,"5min":8.7530000000000001,"15min":8.7530000000000001},"last":0.46400000000000002}]},{"osd":7,"last update":"Sat Sep 23 20:59:23 2023","interfaces":[{"interface":"back","average":{"1min":2.089,"5min":1.871,"15min":1.163},"min":{"1min":0.433,"5min":0.36199999999999999,"15min":0.35699999999999998},"max":{"1min":8.7769999999999992,"5min":8.7769999999999992,"15min":11.186999999999999},"last":0.497},{"interface":"front","average":{"1min":2.5430000000000001,"5min":2.202,"15min":1.325},"min":{"1min":0.497,"5min":0.40899999999999997,"15min":0.34599999999999997},"max":{"1min":8.4939999999999998,"5min":15.098000000000001,"15min":15.098000000000001},"last":0.81499999999999995}]}]},{"osd":0,"up_from":83,"seq":356482285690,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1883224,"kb_used_data":1689168,"kb_used_omap":28,"kb_used_meta":194019,"kb_avail":91861928,"statfs":{"total":95995035648,"available":94066614272,"internally_reserved":0,"allocated":1729708032,"data_stored":1728551026,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":28848,"internal_metadata":198676304},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 20:59:19 2023","interfaces":[{"interface":"back","average":{"1min":3.694,"5min":3.121,"15min":1.4850000000000001},"min":{"1min":0.22700000000000001,"5min":0.191,"15min":0.191},"max":{"1min":15.577999999999999,"5min":16.484999999999999,"15min":18.863},"last":3.3740000000000001},{"interface":"front","average":{"1min":4.4610000000000003,"5min":3.0339999999999998,"15min":1.5389999999999999},"min":{"1min":0.20999999999999999,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":15.771000000000001,"5min":15.771000000000001,"15min":17.704999999999998},"last":8.8759999999999994}]},{"osd":2,"last update":"Sat Sep 23 20:58:50 2023","interfaces":[{"interface":"back","average":{"1min":4.843,"5min":3.2549999999999999,"15min":1.6359999999999999},"min":{"1min":0.434,"5min":0.24299999999999999,"15min":0.23799999999999999},"max":{"1min":14.942,"5min":14.976000000000001,"15min":17.753},"last":1.02},{"interface":"front","average":{"1min":7.774,"5min":3.8180000000000001,"15min":1.726},"min":{"1min":0.38500000000000001,"5min":0.25800000000000001,"15min":0.21099999999999999},"max":{"1min":21.747,"5min":21.747,"15min":21.747},"last":1.1499999999999999}]},{"osd":3,"last update":"Sat Sep 23 20:59:19 2023","interfaces":[{"interface":"back","average":{"1min":4.7839999999999998,"5min":3.0590000000000002,"15min":1.5900000000000001},"min":{"1min":0.38300000000000001,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":21.184000000000001,"5min":21.184000000000001,"15min":21.184000000000001},"last":9.1389999999999993},{"interface":"front","average":{"1min":6.0469999999999997,"5min":3.5510000000000002,"15min":1.794},"min":{"1min":0.30599999999999999,"5min":0.28000000000000003,"15min":0.23300000000000001},"max":{"1min":21.167000000000002,"5min":21.167000000000002,"15min":21.167000000000002},"last":0.95999999999999996}]},{"osd":4,"last update":"Sat Sep 23 20:58:50 2023","interfaces":[{"interface":"back","average":{"1min":6.6399999999999997,"5min":2.9279999999999999,"15min":1.4730000000000001},"min":{"1min":0.36599999999999999,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":21.594999999999999,"5min":21.594999999999999,"15min":21.594999999999999},"last":1.085},{"interface":"front","average":{"1min":2.657,"5min":1.9790000000000001,"15min":1.131},"min":{"1min":0.35199999999999998,"5min":0.31,"15min":0.28100000000000003},"max":{"1min":10.393000000000001,"5min":18.724,"15min":18.724},"last":1.448}]},{"osd":5,"last update":"Sat Sep 23 20:58:50 2023","interfaces":[{"interface":"back","average":{"1min":2.2719999999999998,"5min":1.8069999999999999,"15min":1.177},"min":{"1min":0.34999999999999998,"5min":0.318,"15min":0.318},"max":{"1min":8.8970000000000002,"5min":8.8970000000000002,"15min":8.8970000000000002},"last":0.91200000000000003},{"interface":"front","average":{"1min":5.7110000000000003,"5min":2.8919999999999999,"15min":1.4950000000000001},"min":{"1min":0.40600000000000003,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":20.512,"5min":20.512,"15min":20.512},"last":1.119}]},{"osd":6,"last update":"Sat Sep 23 20:58:44 2023","interfaces":[{"interface":"back","average":{"1min":2.681,"5min":1.893,"15min":1.7350000000000001},"min":{"1min":0.39900000000000002,"5min":0.37,"15min":0.37},"max":{"1min":10.420999999999999,"5min":10.420999999999999,"15min":10.420999999999999},"last":1.4830000000000001},{"interface":"front","average":{"1min":2.7120000000000002,"5min":1.9650000000000001,"15min":1.819},"min":{"1min":0.42199999999999999,"5min":0.377,"15min":0.377},"max":{"1min":10.331,"5min":10.331,"15min":10.331},"last":1.5329999999999999}]},{"osd":7,"last update":"Sat Sep 23 20:58:50 2023","interfaces":[{"interface":"back","average":{"1min":2.4729999999999999,"5min":1.875,"15min":1.22},"min":{"1min":0.38100000000000001,"5min":0.379,"15min":0.379},"max":{"1min":8.7729999999999997,"5min":8.7729999999999997,"15min":8.7729999999999997},"last":0.99399999999999999},{"interface":"front","average":{"1min":2.7010000000000001,"5min":2.032,"15min":1.1619999999999999},"min":{"1min":0.377,"5min":0.377,"15min":0.377},"max":{"1min":10.359999999999999,"5min":18.673999999999999,"15min":18.673999999999999},"last":1.506}]}]},{"osd":2,"up_from":93,"seq":399431958637,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1935816,"kb_used_data":1618112,"kb_used_omap":81,"kb_used_meta":317550,"kb_avail":91809336,"statfs":{"total":95995035648,"available":94012760064,"internally_reserved":0,"allocated":1656946688,"data_stored":1655802430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":83750,"internal_metadata":325171418},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:58:47 2023","interfaces":[{"interface":"back","average":{"1min":3.1429999999999998,"5min":2.0670000000000002,"15min":1.079},"min":{"1min":0.22700000000000001,"5min":0.17999999999999999,"15min":0.17999999999999999},"max":{"1min":14.773999999999999,"5min":14.773999999999999,"15min":14.773999999999999},"last":29.498000000000001},{"interface":"front","average":{"1min":5.1459999999999999,"5min":3.133,"15min":1.4319999999999999},"min":{"1min":0.32400000000000001,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":17.045000000000002,"5min":17.045000000000002,"15min":17.045000000000002},"last":14.948}]},{"osd":1,"last update":"Sat Sep 23 20:58:47 2023","interfaces":[{"interface":"back","average":{"1min":4.1470000000000002,"5min":2.718,"15min":1.48},"min":{"1min":0.29299999999999998,"5min":0.20200000000000001,"15min":0.20200000000000001},"max":{"1min":14.709,"5min":14.709,"15min":16.081},"last":2.5419999999999998},{"interface":"front","average":{"1min":5.3819999999999997,"5min":3.6389999999999998,"15min":1.77},"min":{"1min":0.41499999999999998,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":11.492000000000001,"5min":12.07,"15min":16.041},"last":3.6789999999999998}]},{"osd":3,"last update":"Sat Sep 23 20:59:29 2023","interfaces":[{"interface":"back","average":{"1min":5.0449999999999999,"5min":3.6579999999999999,"15min":1.893},"min":{"1min":0.30299999999999999,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":27.614999999999998,"5min":27.614999999999998,"15min":27.614999999999998},"last":2.0299999999999998},{"interface":"front","average":{"1min":5.9119999999999999,"5min":4.234,"15min":2.1930000000000001},"min":{"1min":0.33000000000000002,"5min":0.22800000000000001,"15min":0.221},"max":{"1min":27.562000000000001,"5min":27.562000000000001,"15min":27.562000000000001},"last":1.9850000000000001}]},{"osd":4,"last update":"Sat Sep 23 20:58:47 2023","interfaces":[{"interface":"back","average":{"1min":2.2519999999999998,"5min":1.9910000000000001,"15min":1.1439999999999999},"min":{"1min":0.28699999999999998,"5min":0.28699999999999998,"15min":0.26400000000000001},"max":{"1min":8.4580000000000002,"5min":9.8119999999999994,"15min":9.8119999999999994},"last":0.46300000000000002},{"interface":"front","average":{"1min":2.137,"5min":2.1560000000000001,"15min":1.2270000000000001},"min":{"1min":0.32400000000000001,"5min":0.313,"15min":0.313},"max":{"1min":8.1649999999999991,"5min":10.609,"15min":10.609},"last":0.32400000000000001}]},{"osd":5,"last update":"Sat Sep 23 20:58:47 2023","interfaces":[{"interface":"back","average":{"1min":2.1779999999999999,"5min":2.238,"15min":1.2470000000000001},"min":{"1min":0.32300000000000001,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":8.5,"5min":9.7629999999999999,"15min":9.7629999999999999},"last":0.434},{"interface":"front","average":{"1min":2.0670000000000002,"5min":2.1709999999999998,"15min":1.242},"min":{"1min":0.36399999999999999,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":8.1170000000000009,"5min":10.574,"15min":10.574},"last":0.35599999999999998}]},{"osd":6,"last update":"Sat Sep 23 20:58:42 2023","interfaces":[{"interface":"back","average":{"1min":2.1360000000000001,"5min":1.8169999999999999,"15min":1.7170000000000001},"min":{"1min":0.379,"5min":0.312,"15min":0.312},"max":{"1min":8.2270000000000003,"5min":14.244999999999999,"15min":14.244999999999999},"last":0.39000000000000001},{"interface":"front","average":{"1min":2.1949999999999998,"5min":1.444,"15min":1.1679999999999999},"min":{"1min":0.41199999999999998,"5min":0.374,"15min":0.374},"max":{"1min":8.1980000000000004,"5min":8.1980000000000004,"15min":8.1980000000000004},"last":0.41599999999999998}]},{"osd":7,"last update":"Sat Sep 23 20:58:47 2023","interfaces":[{"interface":"back","average":{"1min":1.4830000000000001,"5min":1.4339999999999999,"15min":0.995},"min":{"1min":0.315,"5min":0.315,"15min":0.315},"max":{"1min":5.8570000000000002,"5min":8.0820000000000007,"15min":8.0820000000000007},"last":0.51400000000000001},{"interface":"front","average":{"1min":2.2090000000000001,"5min":2.2109999999999999,"15min":1.262},"min":{"1min":0.35099999999999998,"5min":0.35099999999999998,"15min":0.34499999999999997},"max":{"1min":8.5250000000000004,"5min":9.7859999999999996,"15min":9.7859999999999996},"last":0.5}]}]},{"osd":3,"up_from":98,"seq":420906795110,"num_pgs":49,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1825648,"kb_used_data":1467368,"kb_used_omap":7,"kb_used_meta":357880,"kb_avail":91919504,"statfs":{"total":95995035648,"available":94125572096,"internally_reserved":0,"allocated":1502584832,"data_stored":1501462033,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7576,"internal_metadata":366469736},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1,1,3,4],"upper_bound":64},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:59:28 2023","interfaces":[{"interface":"back","average":{"1min":4.056,"5min":2.399,"15min":1.7350000000000001},"min":{"1min":0.30199999999999999,"5min":0.20599999999999999,"15min":0.20599999999999999},"max":{"1min":13.566000000000001,"5min":13.566000000000001,"15min":23.123999999999999},"last":0.54600000000000004},{"interface":"front","average":{"1min":5.2229999999999999,"5min":3.1099999999999999,"15min":1.6259999999999999},"min":{"1min":0.23200000000000001,"5min":0.19800000000000001,"15min":0.19800000000000001},"max":{"1min":13.455,"5min":15.079000000000001,"15min":15.079000000000001},"last":0.23200000000000001}]},{"osd":1,"last update":"Sat Sep 23 20:59:28 2023","interfaces":[{"interface":"back","average":{"1min":2.9510000000000001,"5min":2.8220000000000001,"15min":1.6120000000000001},"min":{"1min":0.23899999999999999,"5min":0.23899999999999999,"15min":0.22900000000000001},"max":{"1min":11.343,"5min":16.318000000000001,"15min":21.385000000000002},"last":0.47699999999999998},{"interface":"front","average":{"1min":2.972,"5min":2.2290000000000001,"15min":1.6830000000000001},"min":{"1min":0.23400000000000001,"5min":0.218,"15min":0.218},"max":{"1min":11.782,"5min":11.782,"15min":25.353000000000002},"last":2.5939999999999999}]},{"osd":2,"last update":"Sat Sep 23 20:59:28 2023","interfaces":[{"interface":"back","average":{"1min":4.7210000000000001,"5min":3.1459999999999999,"15min":1.7310000000000001},"min":{"1min":0.33000000000000002,"5min":0.33000000000000002,"15min":0.30099999999999999},"max":{"1min":13.616,"5min":15.476000000000001,"15min":24.154},"last":0.42599999999999999},{"interface":"front","average":{"1min":3.665,"5min":2.4790000000000001,"15min":1.792},"min":{"1min":0.27300000000000002,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":13.125999999999999,"5min":16.032,"15min":25.305},"last":0.44600000000000001}]},{"osd":4,"last update":"Sat Sep 23 20:59:28 2023","interfaces":[{"interface":"back","average":{"1min":3.0070000000000001,"5min":2.0299999999999998,"15min":1.3},"min":{"1min":0.379,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":11.089,"5min":11.089,"15min":11.861000000000001},"last":0.55800000000000005},{"interface":"front","average":{"1min":1.954,"5min":1.359,"15min":1.335},"min":{"1min":0.309,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":9.9779999999999998,"5min":9.9779999999999998,"15min":22.853000000000002},"last":0.504}]},{"osd":5,"last update":"Sat Sep 23 20:59:28 2023","interfaces":[{"interface":"back","average":{"1min":2.8540000000000001,"5min":2.052,"15min":1.2929999999999999},"min":{"1min":0.39000000000000001,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":11.048,"5min":11.048,"15min":11.891999999999999},"last":0.51900000000000002},{"interface":"front","average":{"1min":3.585,"5min":1.9510000000000001,"15min":1.272},"min":{"1min":0.53200000000000003,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":13.43,"5min":13.43,"15min":13.43},"last":2.645}]},{"osd":6,"last update":"Sat Sep 23 20:58:43 2023","interfaces":[{"interface":"back","average":{"1min":1.9450000000000001,"5min":1.1930000000000001,"15min":1.0309999999999999},"min":{"1min":0.33500000000000002,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":10.018000000000001,"5min":10.018000000000001,"15min":10.018000000000001},"last":0.70199999999999996},{"interface":"front","average":{"1min":1.4770000000000001,"5min":1.48,"15min":1.4339999999999999},"min":{"1min":0.38700000000000001,"5min":0.38700000000000001,"15min":0.38700000000000001},"max":{"1min":5.3239999999999998,"5min":7.234,"15min":7.234},"last":2.6099999999999999}]},{"osd":7,"last update":"Sat Sep 23 20:59:28 2023","interfaces":[{"interface":"back","average":{"1min":2.8700000000000001,"5min":2.3839999999999999,"15min":1.4670000000000001},"min":{"1min":0.42799999999999999,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":11.114000000000001,"5min":16.364999999999998,"15min":16.364999999999998},"last":0.59799999999999998},{"interface":"front","average":{"1min":3.6400000000000001,"5min":2.077,"15min":1.3440000000000001},"min":{"1min":0.58399999999999996,"5min":0.379,"15min":0.379},"max":{"1min":13.355,"5min":13.355,"15min":13.355},"last":2.6699999999999999}]}]},{"osd":4,"up_from":31,"seq":133143986418,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1185796,"kb_used_data":1044028,"kb_used_omap":0,"kb_used_meta":141760,"kb_avail":92559356,"statfs":{"total":95995035648,"available":94780780544,"internally_reserved":0,"allocated":1069084672,"data_stored":1068233257,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":145162240},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1],"upper_bound":2},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:58:53 2023","interfaces":[{"interface":"back","average":{"1min":1.917,"5min":1.7170000000000001,"15min":1.042},"min":{"1min":0.245,"5min":0.214,"15min":0.214},"max":{"1min":8.8439999999999994,"5min":11.784000000000001,"15min":11.784000000000001},"last":0.255},{"interface":"front","average":{"1min":1.8149999999999999,"5min":1.7889999999999999,"15min":1.23},"min":{"1min":0.314,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":8.8300000000000001,"5min":11.766,"15min":11.766},"last":0.42499999999999999}]},{"osd":1,"last update":"Sat Sep 23 20:59:22 2023","interfaces":[{"interface":"back","average":{"1min":2.4119999999999999,"5min":1.6930000000000001,"15min":1.1200000000000001},"min":{"1min":0.35499999999999998,"5min":0.246,"15min":0.23599999999999999},"max":{"1min":7.3310000000000004,"5min":10.153,"15min":10.153},"last":5.5570000000000004},{"interface":"front","average":{"1min":3.2669999999999999,"5min":2.0099999999999998,"15min":1.1419999999999999},"min":{"1min":0.33000000000000002,"5min":0.26100000000000001,"15min":0.23200000000000001},"max":{"1min":9.7509999999999994,"5min":12.292999999999999,"15min":12.292999999999999},"last":2.351}]},{"osd":2,"last update":"Sat Sep 23 20:58:53 2023","interfaces":[{"interface":"back","average":{"1min":1.8700000000000001,"5min":1.764,"15min":1.115},"min":{"1min":0.34300000000000003,"5min":0.311,"15min":0.29099999999999998},"max":{"1min":7.3150000000000004,"5min":15.288,"15min":15.288},"last":3.9940000000000002},{"interface":"front","average":{"1min":1.633,"5min":1.974,"15min":1.2629999999999999},"min":{"1min":0.32600000000000001,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":8.7959999999999994,"5min":15.262,"15min":15.262},"last":3.2320000000000002}]},{"osd":3,"last update":"Sat Sep 23 20:59:22 2023","interfaces":[{"interface":"back","average":{"1min":2.21,"5min":2.1850000000000001,"15min":1.6180000000000001},"min":{"1min":0.30499999999999999,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":4.835,"5min":14.824999999999999,"15min":31.562999999999999},"last":4.6859999999999999},{"interface":"front","average":{"1min":2.2770000000000001,"5min":2.2989999999999999,"15min":1.6899999999999999},"min":{"1min":0.30299999999999999,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":4.8490000000000002,"5min":14.839,"15min":31.574000000000002},"last":4.6580000000000004}]},{"osd":5,"last update":"Sat Sep 23 20:58:35 2023","interfaces":[{"interface":"back","average":{"1min":0.81399999999999995,"5min":0.61399999999999999,"15min":0.52900000000000003},"min":{"1min":0.29199999999999998,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":2.9470000000000001,"5min":3.3279999999999998,"15min":3.3279999999999998},"last":0.315},{"interface":"front","average":{"1min":0.77900000000000003,"5min":0.68500000000000005,"15min":0.57799999999999996},"min":{"1min":0.33200000000000002,"5min":0.251,"15min":0.251},"max":{"1min":1.9350000000000001,"5min":7.7619999999999996,"15min":7.7619999999999996},"last":2.331}]},{"osd":6,"last update":"Sat Sep 23 20:58:41 2023","interfaces":[{"interface":"back","average":{"1min":0.90300000000000002,"5min":0.67700000000000005,"15min":0.54800000000000004},"min":{"1min":0.29899999999999999,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":3.4780000000000002,"5min":4.6909999999999998,"15min":4.6909999999999998},"last":0.38},{"interface":"front","average":{"1min":0.82299999999999995,"5min":0.71599999999999997,"15min":0.66800000000000004},"min":{"1min":0.36799999999999999,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":3.456,"5min":3.456,"15min":3.456},"last":0.36399999999999999}]},{"osd":7,"last update":"Sat Sep 23 20:59:22 2023","interfaces":[{"interface":"back","average":{"1min":0.83499999999999996,"5min":0.752,"15min":0.69399999999999995},"min":{"1min":0.32600000000000001,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":2.8889999999999998,"5min":6.2240000000000002,"15min":7.383},"last":0.45200000000000001},{"interface":"front","average":{"1min":0.73999999999999999,"5min":0.78400000000000003,"15min":0.67500000000000004},"min":{"1min":0.34200000000000003,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":2.8679999999999999,"5min":7.8310000000000004,"15min":7.8310000000000004},"last":0.39600000000000002}]}]},{"osd":5,"up_from":39,"seq":167503724783,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1318392,"kb_used_data":1189168,"kb_used_omap":0,"kb_used_meta":129216,"kb_avail":92426760,"statfs":{"total":95995035648,"available":94645002240,"internally_reserved":0,"allocated":1217708032,"data_stored":1216865193,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":132317184},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1],"upper_bound":2},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:58:37 2023","interfaces":[{"interface":"back","average":{"1min":0.78600000000000003,"5min":1.03,"15min":0.72699999999999998},"min":{"1min":0.23899999999999999,"5min":0.20399999999999999,"15min":0.20399999999999999},"max":{"1min":3.6949999999999998,"5min":7.766,"15min":7.766},"last":3.6240000000000001},{"interface":"front","average":{"1min":1.8049999999999999,"5min":1.6779999999999999,"15min":0.98099999999999998},"min":{"1min":0.25,"5min":0.223,"15min":0.223},"max":{"1min":7.8120000000000003,"5min":15.037000000000001,"15min":15.037000000000001},"last":0.24099999999999999}]},{"osd":1,"last update":"Sat Sep 23 20:59:17 2023","interfaces":[{"interface":"back","average":{"1min":2.7749999999999999,"5min":1.758,"15min":1.099},"min":{"1min":0.248,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":8.1829999999999998,"5min":8.3520000000000003,"15min":8.3520000000000003},"last":5.6619999999999999},{"interface":"front","average":{"1min":2.3690000000000002,"5min":1.825,"15min":1.099},"min":{"1min":0.27200000000000002,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":8.5399999999999991,"5min":10.285,"15min":10.285},"last":2.2429999999999999}]},{"osd":2,"last update":"Sat Sep 23 20:58:52 2023","interfaces":[{"interface":"back","average":{"1min":1.9299999999999999,"5min":1.6000000000000001,"15min":0.97799999999999998},"min":{"1min":0.311,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":6,"5min":12.651,"15min":12.651},"last":0.38500000000000001},{"interface":"front","average":{"1min":3.077,"5min":1.8440000000000001,"15min":1.085},"min":{"1min":0.27100000000000002,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":12.300000000000001,"5min":12.300000000000001,"15min":12.300000000000001},"last":2.98}]},{"osd":3,"last update":"Sat Sep 23 20:59:17 2023","interfaces":[{"interface":"back","average":{"1min":3.5880000000000001,"5min":2.2090000000000001,"15min":1.351},"min":{"1min":0.46700000000000003,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":12.188000000000001,"5min":12.188000000000001,"15min":12.188000000000001},"last":10.303000000000001},{"interface":"front","average":{"1min":2.0739999999999998,"5min":1.6060000000000001,"15min":1.071},"min":{"1min":0.32500000000000001,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":7.258,"5min":10.457000000000001,"15min":10.457000000000001},"last":3.964}]},{"osd":4,"last update":"Sat Sep 23 20:58:33 2023","interfaces":[{"interface":"back","average":{"1min":0.48799999999999999,"5min":0.59099999999999997,"15min":0.52900000000000003},"min":{"1min":0.252,"5min":0.252,"15min":0.252},"max":{"1min":2.6909999999999998,"5min":3.8479999999999999,"15min":3.8479999999999999},"last":0.32700000000000001},{"interface":"front","average":{"1min":0.41899999999999998,"5min":0.64100000000000001,"15min":0.54900000000000004},"min":{"1min":0.29299999999999998,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":0.747,"5min":3.0680000000000001,"15min":3.0680000000000001},"last":0.33900000000000002}]},{"osd":6,"last update":"Sat Sep 23 20:58:37 2023","interfaces":[{"interface":"back","average":{"1min":0.46600000000000003,"5min":0.59099999999999997,"15min":0.57099999999999995},"min":{"1min":0.28499999999999998,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":1.0509999999999999,"5min":2.9820000000000002,"15min":2.9820000000000002},"last":0.29199999999999998},{"interface":"front","average":{"1min":0.499,"5min":0.69899999999999995,"15min":0.72899999999999998},"min":{"1min":0.36299999999999999,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":0.78500000000000003,"5min":3.6560000000000001,"15min":3.6560000000000001},"last":0.36899999999999999}]},{"osd":7,"last update":"Sat Sep 23 20:59:17 2023","interfaces":[{"interface":"back","average":{"1min":0.71099999999999997,"5min":0.80200000000000005,"15min":0.64200000000000002},"min":{"1min":0.33400000000000002,"5min":0.307,"15min":0.30199999999999999},"max":{"1min":2.516,"5min":8.5649999999999995,"15min":8.5649999999999995},"last":0.35499999999999998},{"interface":"front","average":{"1min":0.77200000000000002,"5min":0.75,"15min":0.65700000000000003},"min":{"1min":0.34399999999999997,"5min":0.29799999999999999,"15min":0.27700000000000002},"max":{"1min":3.0150000000000001,"5min":4.5430000000000001,"15min":4.5430000000000001},"last":0.40200000000000002}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1716850688,"data_stored":1716846616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1678970880,"data_stored":1678966808,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1646133248,"data_stored":1646133248,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1496383488,"data_stored":1496383488,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1066074112,"data_stored":1066074112,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1213726720,"data_stored":1213726720,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1118769152,"data_stored":1118765080,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1069219840,"data_stored":1069219840,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T20:59:34.365 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph status --format=json 2023-09-23T20:59:34.370 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:59:34.374 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:59:34.374 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:59:34.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:59:34 smithi026 ceph-mon[179688]: pgmap v592: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 3.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 586 op/s; 11601/159498 objects degraded (7.273%); 2373/159498 objects misplaced (1.488%); 1.6 MiB/s, 25 objects/s recovering 2023-09-23T20:59:34.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:59:34 smithi026 ceph-mon[175151]: pgmap v592: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 3.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 586 op/s; 11601/159498 objects degraded (7.273%); 2373/159498 objects misplaced (1.488%); 1.6 MiB/s, 25 objects/s recovering 2023-09-23T20:59:34.475 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:59:34.572 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:59:34.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:59:34 smithi070 ceph-mon[168282]: pgmap v592: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 3.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 586 op/s; 11601/159498 objects degraded (7.273%); 2373/159498 objects misplaced (1.488%); 1.6 MiB/s, 25 objects/s recovering 2023-09-23T20:59:34.675 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:59:34.681 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:59:34.682 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:59:34.783 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:59:34.810 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:59:34.845 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:59:34.848 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:59:34.848 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:59:34.949 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:59:35.053 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:59:35.245 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:59:35.303 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:59:35 smithi026 ceph-mon[179688]: from='client.55171 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:59:35.303 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:59:35 smithi026 ceph-mon[179688]: pgmap v593: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 3.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 593 op/s; 11601/157155 objects degraded (7.382%); 2373/157155 objects misplaced (1.510%); 1.6 MiB/s, 25 objects/s recovering 2023-09-23T20:59:35.303 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:59:35 smithi026 ceph-mon[179688]: Health check update: Degraded data redundancy: 11601/159498 objects degraded (7.273%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T20:59:35.304 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:59:35 smithi026 ceph-mon[175151]: from='client.55171 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:59:35.304 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:59:35 smithi026 ceph-mon[175151]: pgmap v593: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 3.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 593 op/s; 11601/157155 objects degraded (7.382%); 2373/157155 objects misplaced (1.510%); 1.6 MiB/s, 25 objects/s recovering 2023-09-23T20:59:35.304 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:59:35 smithi026 ceph-mon[175151]: Health check update: Degraded data redundancy: 11601/159498 objects degraded (7.273%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T20:59:35.342 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:59:35.348 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:59:35.349 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:59:35.449 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:59:35.524 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:59:35.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:59:35 smithi070 ceph-mon[168282]: from='client.55171 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:59:35.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:59:35 smithi070 ceph-mon[168282]: pgmap v593: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 3.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 593 op/s; 11601/157155 objects degraded (7.382%); 2373/157155 objects misplaced (1.510%); 1.6 MiB/s, 25 objects/s recovering 2023-09-23T20:59:35.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:59:35 smithi070 ceph-mon[168282]: Health check update: Degraded data redundancy: 11601/159498 objects degraded (7.273%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T20:59:35.840 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:59:35.853 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:59:35.854 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:59:35.954 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:59:35.964 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:59:36.049 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:59:36.390 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:59:36.395 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:59:36.396 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:59:36.497 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:59:36.593 INFO:teuthology.orchestra.run.smithi026.stderr:noscrub is set 2023-09-23T20:59:36.620 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:59:36.651 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:59:36 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/1465118342' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:59:36.651 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:59:36 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:59:36.651 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:59:36 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/1465118342' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:59:36.652 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:59:36 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:59:36.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:59:36 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/1465118342' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:59:36.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:59:36 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:59:37.080 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:59:37.092 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:59:37.093 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:59:37.165 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0: {} 2023-09-23T20:59:37.193 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:59:37.214 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1: {} 2023-09-23T20:59:37.231 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:59:37.256 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2: {} 2023-09-23T20:59:37.275 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:59:37.279 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:59:37.279 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:59:37.297 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3: {} 2023-09-23T20:59:37.350 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4: {} 2023-09-23T20:59:37.380 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:59:37.400 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T20:59:37.410 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:59:37.439 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6: {} 2023-09-23T20:59:37.452 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:59:37.455 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:59:37.455 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:59:37.471 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T20:59:37.476 INFO:teuthology.orchestra.run.smithi026.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T20:59:37.476 INFO:teuthology.orchestra.run.smithi026.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T20:59:37.476 INFO:teuthology.orchestra.run.smithi026.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T20:59:37.476 INFO:teuthology.orchestra.run.smithi026.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T20:59:37.476 INFO:teuthology.orchestra.run.smithi026.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T20:59:37.476 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T20:59:37.477 INFO:teuthology.orchestra.run.smithi026.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T20:59:37.477 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T20:59:37.539 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:59:37 smithi070 ceph-mon[168282]: pgmap v594: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 3.2 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 593 op/s; 11601/155472 objects degraded (7.462%); 2373/155472 objects misplaced (1.526%); 1.6 MiB/s, 25 objects/s recovering 2023-09-23T20:59:37.539 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:59:37 smithi070 ceph-mon[168282]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:59:37.539 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:59:37 smithi070 ceph-mon[168282]: failed to encode map e241 with expected crc 2023-09-23T20:59:37.539 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:59:37 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T20:59:37.539 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:59:37 smithi070 ceph-mon[168282]: osdmap e241: 8 total, 8 up, 8 in 2023-09-23T20:59:37.540 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:59:37 smithi070 ceph-mon[168282]: failed to encode map e241 with expected crc 2023-09-23T20:59:37.540 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:59:37 smithi070 ceph-mon[168282]: failed to encode map e241 with expected crc 2023-09-23T20:59:37.540 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:59:37 smithi070 ceph-mon[168282]: failed to encode map e241 with expected crc 2023-09-23T20:59:37.540 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:59:37 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/4032629495' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:59:37.557 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:59:37.587 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:59:37.627 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:59:37.630 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:59:37.630 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:59:37.631 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:59:37 smithi026 ceph-mon[175151]: pgmap v594: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 3.2 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 593 op/s; 11601/155472 objects degraded (7.462%); 2373/155472 objects misplaced (1.526%); 1.6 MiB/s, 25 objects/s recovering 2023-09-23T20:59:37.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:59:37 smithi026 ceph-mon[175151]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:59:37.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:59:37 smithi026 ceph-mon[175151]: failed to encode map e241 with expected crc 2023-09-23T20:59:37.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:59:37 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T20:59:37.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:59:37 smithi026 ceph-mon[175151]: osdmap e241: 8 total, 8 up, 8 in 2023-09-23T20:59:37.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:59:37 smithi026 ceph-mon[175151]: failed to encode map e241 with expected crc 2023-09-23T20:59:37.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:59:37 smithi026 ceph-mon[175151]: failed to encode map e241 with expected crc 2023-09-23T20:59:37.633 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:59:37 smithi026 ceph-mon[175151]: failed to encode map e241 with expected crc 2023-09-23T20:59:37.633 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:59:37 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/4032629495' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:59:37.633 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:59:37 smithi026 ceph-mon[179688]: pgmap v594: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 3.2 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 593 op/s; 11601/155472 objects degraded (7.462%); 2373/155472 objects misplaced (1.526%); 1.6 MiB/s, 25 objects/s recovering 2023-09-23T20:59:37.633 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:59:37 smithi026 ceph-mon[179688]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:59:37.634 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:59:37 smithi026 ceph-mon[179688]: failed to encode map e241 with expected crc 2023-09-23T20:59:37.634 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:59:37 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T20:59:37.634 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:59:37 smithi026 ceph-mon[179688]: osdmap e241: 8 total, 8 up, 8 in 2023-09-23T20:59:37.634 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:59:37 smithi026 ceph-mon[179688]: failed to encode map e241 with expected crc 2023-09-23T20:59:37.634 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:59:37 smithi026 ceph-mon[179688]: failed to encode map e241 with expected crc 2023-09-23T20:59:37.634 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:59:37 smithi026 ceph-mon[179688]: failed to encode map e241 with expected crc 2023-09-23T20:59:37.635 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:59:37 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/4032629495' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:59:37.731 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:59:37.759 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:59:37.800 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:59:37.803 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:59:37.803 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:59:37.904 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:59:37.936 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:59:37.979 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:59:37.982 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:59:37.982 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:59:38.083 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:59:38.118 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:59:38.159 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:59:38.162 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:59:38.162 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:59:38.263 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:59:38.298 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:59:38.344 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:59:38.348 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:59:38.348 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:59:38.449 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:59:38.481 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:59:38.523 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:59:38.526 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:59:38.526 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:59:38.627 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:59:38.657 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:59:38.712 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:59:38.714 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:59:38.715 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:59:38.815 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:59:38.922 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:59:39.361 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T20:59:39.755 INFO:teuthology.orchestra.run.smithi026.stdout:{ 2023-09-23T20:59:39.755 INFO:teuthology.orchestra.run.smithi026.stdout: "ops": [ 2023-09-23T20:59:39.755 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:59:39.756 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35100.0:155446 7.5 7:a79fa071:::benchmark_data_smithi026_260489_object53684:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e241)", 2023-09-23T20:59:39.756 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T20:59:39.089611+0000", 2023-09-23T20:59:39.756 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 0.0056242779999999999, 2023-09-23T20:59:39.756 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0056590240000000003, 2023-09-23T20:59:39.756 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T20:59:39.756 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "waiting for sub ops", 2023-09-23T20:59:39.756 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T20:59:39.756 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35100", 2023-09-23T20:59:39.756 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/4108972961", 2023-09-23T20:59:39.756 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 155446 2023-09-23T20:59:39.756 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:59:39.756 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T20:59:39.757 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:59:39.757 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T20:59:39.757 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:59:39.089611+0000", 2023-09-23T20:59:39.757 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:59:39.757 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:59:39.757 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:59:39.757 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T20:59:39.757 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:59:39.089611+0000", 2023-09-23T20:59:39.757 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:59:39.757 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:59:39.757 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:59:39.757 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T20:59:39.757 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:59:39.089610+0000", 2023-09-23T20:59:39.758 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T20:59:39.758 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:59:39.758 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:59:39.758 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T20:59:39.758 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:59:39.089617+0000", 2023-09-23T20:59:39.758 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.255e-06 2023-09-23T20:59:39.758 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:59:39.758 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:59:39.758 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T20:59:39.758 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:59:39.089629+0000", 2023-09-23T20:59:39.758 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.2123000000000001e-05 2023-09-23T20:59:39.759 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:59:39.759 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:59:39.759 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T20:59:39.759 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:59:39.089639+0000", 2023-09-23T20:59:39.759 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.5610000000000004e-06 2023-09-23T20:59:39.759 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:59:39.759 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:59:39.759 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:59:39.759 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:59:39.093516+0000", 2023-09-23T20:59:39.759 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0038771190000000001 2023-09-23T20:59:39.759 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:59:39.759 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:59:39.760 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T20:59:39.760 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:59:39.093658+0000", 2023-09-23T20:59:39.760 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000142158 2023-09-23T20:59:39.760 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:59:39.760 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:59:39.760 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 0,7", 2023-09-23T20:59:39.760 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:59:39.093793+0000", 2023-09-23T20:59:39.760 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00013479 2023-09-23T20:59:39.760 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:59:39.760 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T20:59:39.760 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:59:39.760 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:59:39.760 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:59:39.761 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35100.0:155415 7.e 7:7f87df94:::benchmark_data_smithi026_260489_object53653:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e241)", 2023-09-23T20:59:39.761 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T20:59:39.037713+0000", 2023-09-23T20:59:39.761 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 0.057522359000000002, 2023-09-23T20:59:39.761 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.057642915000000003, 2023-09-23T20:59:39.761 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T20:59:39.761 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "waiting for sub ops", 2023-09-23T20:59:39.761 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T20:59:39.761 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35100", 2023-09-23T20:59:39.761 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/4108972961", 2023-09-23T20:59:39.761 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 155415 2023-09-23T20:59:39.761 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:59:39.761 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T20:59:39.762 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:59:39.762 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T20:59:39.762 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:59:39.037713+0000", 2023-09-23T20:59:39.762 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:59:39.762 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:59:39.762 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:59:39.762 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T20:59:39.762 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:59:39.037713+0000", 2023-09-23T20:59:39.762 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:59:39.762 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:59:39.762 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:59:39.762 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T20:59:39.763 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:59:39.037713+0000", 2023-09-23T20:59:39.763 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T20:59:39.763 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:59:39.763 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:59:39.763 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T20:59:39.763 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:59:39.037716+0000", 2023-09-23T20:59:39.763 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.5499999999999999e-06 2023-09-23T20:59:39.763 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:59:39.763 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:59:39.763 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T20:59:39.763 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:59:39.037724+0000", 2023-09-23T20:59:39.763 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.5109999999999997e-06 2023-09-23T20:59:39.763 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:59:39.764 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:59:39.764 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T20:59:39.764 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:59:39.037729+0000", 2023-09-23T20:59:39.764 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.57e-06 2023-09-23T20:59:39.764 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:59:39.764 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:59:39.764 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:59:39.764 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:59:39.042859+0000", 2023-09-23T20:59:39.764 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0051301769999999997 2023-09-23T20:59:39.764 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:59:39.764 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:59:39.764 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T20:59:39.765 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:59:39.043558+0000", 2023-09-23T20:59:39.765 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00069870499999999999 2023-09-23T20:59:39.765 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:59:39.765 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:59:39.765 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 0,5", 2023-09-23T20:59:39.765 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:59:39.043682+0000", 2023-09-23T20:59:39.765 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000123907 2023-09-23T20:59:39.765 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:59:39.765 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:59:39.765 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T20:59:39.765 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:59:39.055751+0000", 2023-09-23T20:59:39.765 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.012068898999999999 2023-09-23T20:59:39.765 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:59:39.766 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:59:39.766 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:59:39.766 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:59:39.060899+0000", 2023-09-23T20:59:39.766 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0051483129999999998 2023-09-23T20:59:39.766 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:59:39.766 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T20:59:39.766 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:59:39.766 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:59:39.766 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:59:39.766 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35100.0:155416 7.e 7:7b981ff8:::benchmark_data_smithi026_260489_object53654:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e241)", 2023-09-23T20:59:39.766 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T20:59:39.037808+0000", 2023-09-23T20:59:39.767 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 0.057426927000000003, 2023-09-23T20:59:39.767 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.057634800999999999, 2023-09-23T20:59:39.767 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T20:59:39.767 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "waiting for sub ops", 2023-09-23T20:59:39.767 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T20:59:39.767 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35100", 2023-09-23T20:59:39.767 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/4108972961", 2023-09-23T20:59:39.767 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 155416 2023-09-23T20:59:39.767 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:59:39.767 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T20:59:39.767 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:59:39.767 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T20:59:39.767 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:59:39.037808+0000", 2023-09-23T20:59:39.768 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:59:39.768 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:59:39.768 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:59:39.768 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T20:59:39.768 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:59:39.037808+0000", 2023-09-23T20:59:39.768 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:59:39.768 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:59:39.768 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:59:39.768 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T20:59:39.768 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:59:39.037808+0000", 2023-09-23T20:59:39.768 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T20:59:39.768 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:59:39.768 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:59:39.769 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T20:59:39.769 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:59:39.037812+0000", 2023-09-23T20:59:39.769 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.7340000000000002e-06 2023-09-23T20:59:39.769 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:59:39.769 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:59:39.769 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T20:59:39.769 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:59:39.037820+0000", 2023-09-23T20:59:39.769 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.5820000000000002e-06 2023-09-23T20:59:39.769 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:59:39.769 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:59:39.769 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T20:59:39.769 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:59:39.037827+0000", 2023-09-23T20:59:39.770 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.0840000000000003e-06 2023-09-23T20:59:39.770 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:59:39.770 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:59:39.770 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:59:39.770 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:59:39.047676+0000", 2023-09-23T20:59:39.770 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0098481929999999999 2023-09-23T20:59:39.770 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:59:39.770 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:59:39.770 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T20:59:39.770 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:59:39.047778+0000", 2023-09-23T20:59:39.770 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0001024 2023-09-23T20:59:39.770 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:59:39.770 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:59:39.771 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 0,5", 2023-09-23T20:59:39.771 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:59:39.047855+0000", 2023-09-23T20:59:39.771 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.6613999999999995e-05 2023-09-23T20:59:39.771 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:59:39.771 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:59:39.771 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T20:59:39.771 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:59:39.055754+0000", 2023-09-23T20:59:39.771 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0078992850000000007 2023-09-23T20:59:39.771 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:59:39.771 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:59:39.771 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:59:39.771 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:59:39.060931+0000", 2023-09-23T20:59:39.772 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0051770380000000001 2023-09-23T20:59:39.772 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:59:39.772 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T20:59:39.772 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:59:39.772 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:59:39.772 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:59:39.772 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35100.0:155438 7.5 7:a941b15d:::benchmark_data_smithi026_260489_object53676:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e241)", 2023-09-23T20:59:39.772 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T20:59:39.077930+0000", 2023-09-23T20:59:39.772 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 0.017305860999999999, 2023-09-23T20:59:39.772 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.017609972000000002, 2023-09-23T20:59:39.772 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T20:59:39.772 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "waiting for sub ops", 2023-09-23T20:59:39.772 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T20:59:39.773 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35100", 2023-09-23T20:59:39.773 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/4108972961", 2023-09-23T20:59:39.773 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 155438 2023-09-23T20:59:39.773 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:59:39.773 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T20:59:39.773 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:59:39.773 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T20:59:39.773 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:59:39.077930+0000", 2023-09-23T20:59:39.773 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:59:39.773 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:59:39.773 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:59:39.773 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T20:59:39.773 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:59:39.077930+0000", 2023-09-23T20:59:39.774 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:59:39.774 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:59:39.774 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:59:39.774 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T20:59:39.774 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:59:39.077929+0000", 2023-09-23T20:59:39.774 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T20:59:39.774 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:59:39.774 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:59:39.774 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T20:59:39.774 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:59:39.077932+0000", 2023-09-23T20:59:39.774 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.9019999999999999e-06 2023-09-23T20:59:39.774 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:59:39.775 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:59:39.775 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T20:59:39.775 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:59:39.077941+0000", 2023-09-23T20:59:39.775 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.6789999999999994e-06 2023-09-23T20:59:39.775 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:59:39.775 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:59:39.775 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T20:59:39.775 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:59:39.077946+0000", 2023-09-23T20:59:39.775 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.9089999999999999e-06 2023-09-23T20:59:39.775 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:59:39.775 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:59:39.775 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:59:39.775 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:59:39.078001+0000", 2023-09-23T20:59:39.775 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.5373e-05 2023-09-23T20:59:39.776 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:59:39.776 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:59:39.776 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T20:59:39.776 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:59:39.078114+0000", 2023-09-23T20:59:39.776 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000112905 2023-09-23T20:59:39.776 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:59:39.776 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:59:39.776 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 0,7", 2023-09-23T20:59:39.776 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:59:39.078225+0000", 2023-09-23T20:59:39.776 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000111598 2023-09-23T20:59:39.776 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:59:39.776 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:59:39.776 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:59:39.777 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:59:39.093432+0000", 2023-09-23T20:59:39.777 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.015206124 2023-09-23T20:59:39.777 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:59:39.777 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:59:39.777 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:59:39.777 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:59:39.093486+0000", 2023-09-23T20:59:39.777 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.4258999999999997e-05 2023-09-23T20:59:39.777 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:59:39.777 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T20:59:39.777 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:59:39.777 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:59:39.777 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:59:39.778 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35100.0:155439 7.5 7:a8e85312:::benchmark_data_smithi026_260489_object53677:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e241)", 2023-09-23T20:59:39.778 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T20:59:39.084288+0000", 2023-09-23T20:59:39.778 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 0.010947474, 2023-09-23T20:59:39.778 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.011337398, 2023-09-23T20:59:39.778 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T20:59:39.778 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "waiting for sub ops", 2023-09-23T20:59:39.778 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T20:59:39.778 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35100", 2023-09-23T20:59:39.778 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/4108972961", 2023-09-23T20:59:39.778 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 155439 2023-09-23T20:59:39.778 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:59:39.778 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T20:59:39.779 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:59:39.779 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T20:59:39.779 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:59:39.084288+0000", 2023-09-23T20:59:39.779 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:59:39.779 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:59:39.779 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:59:39.779 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T20:59:39.779 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:59:39.084288+0000", 2023-09-23T20:59:39.779 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:59:39.779 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:59:39.779 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:59:39.779 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T20:59:39.779 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:59:39.084287+0000", 2023-09-23T20:59:39.779 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T20:59:39.780 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:59:39.780 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:59:39.780 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T20:59:39.780 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:59:39.084293+0000", 2023-09-23T20:59:39.780 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.1990000000000003e-06 2023-09-23T20:59:39.780 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:59:39.780 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:59:39.780 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T20:59:39.780 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:59:39.084307+0000", 2023-09-23T20:59:39.780 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.4327e-05 2023-09-23T20:59:39.780 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:59:39.780 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:59:39.780 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T20:59:39.781 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:59:39.084316+0000", 2023-09-23T20:59:39.781 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.8570000000000001e-06 2023-09-23T20:59:39.781 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:59:39.781 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:59:39.781 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:59:39.781 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:59:39.084370+0000", 2023-09-23T20:59:39.781 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.3977000000000002e-05 2023-09-23T20:59:39.781 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:59:39.781 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:59:39.781 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T20:59:39.781 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:59:39.084557+0000", 2023-09-23T20:59:39.781 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000186696 2023-09-23T20:59:39.781 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:59:39.782 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:59:39.782 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 0,7", 2023-09-23T20:59:39.782 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:59:39.084712+0000", 2023-09-23T20:59:39.782 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00015544300000000001 2023-09-23T20:59:39.782 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:59:39.782 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T20:59:39.782 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:59:39.782 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:59:39.782 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:59:39.782 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35100.0:155440 7.5 7:ae6848ae:::benchmark_data_smithi026_260489_object53678:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e241)", 2023-09-23T20:59:39.782 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T20:59:39.087734+0000", 2023-09-23T20:59:39.782 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 0.0075017210000000003, 2023-09-23T20:59:39.783 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00796469, 2023-09-23T20:59:39.783 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T20:59:39.783 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "waiting for sub ops", 2023-09-23T20:59:39.783 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T20:59:39.783 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35100", 2023-09-23T20:59:39.783 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/4108972961", 2023-09-23T20:59:39.783 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 155440 2023-09-23T20:59:39.783 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:59:39.783 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T20:59:39.783 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:59:39.783 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T20:59:39.783 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:59:39.087734+0000", 2023-09-23T20:59:39.783 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:59:39.784 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:59:39.784 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:59:39.784 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T20:59:39.784 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:59:39.087734+0000", 2023-09-23T20:59:39.784 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:59:39.784 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:59:39.784 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:59:39.784 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T20:59:39.784 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:59:39.087731+0000", 2023-09-23T20:59:39.784 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999976 2023-09-23T20:59:39.784 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:59:39.784 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:59:39.785 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T20:59:39.785 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:59:39.087744+0000", 2023-09-23T20:59:39.785 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.2191999999999999e-05 2023-09-23T20:59:39.785 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:59:39.785 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:59:39.785 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T20:59:39.785 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:59:39.087763+0000", 2023-09-23T20:59:39.785 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.9406999999999999e-05 2023-09-23T20:59:39.785 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:59:39.785 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:59:39.785 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T20:59:39.785 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:59:39.087775+0000", 2023-09-23T20:59:39.785 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.2308e-05 2023-09-23T20:59:39.786 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:59:39.786 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:59:39.786 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:59:39.786 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:59:39.089463+0000", 2023-09-23T20:59:39.786 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0016879079999999999 2023-09-23T20:59:39.786 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:59:39.786 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:59:39.786 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T20:59:39.786 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:59:39.089601+0000", 2023-09-23T20:59:39.786 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00013750700000000001 2023-09-23T20:59:39.786 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:59:39.786 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:59:39.787 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 0,7", 2023-09-23T20:59:39.787 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:59:39.089694+0000", 2023-09-23T20:59:39.787 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.2887999999999996e-05 2023-09-23T20:59:39.787 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:59:39.787 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T20:59:39.787 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:59:39.787 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:59:39.787 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:59:39.787 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35100.0:155442 7.e 7:7170b5b2:::benchmark_data_smithi026_260489_object53680:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e241)", 2023-09-23T20:59:39.787 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T20:59:39.087791+0000", 2023-09-23T20:59:39.787 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 0.0074446310000000002, 2023-09-23T20:59:39.787 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0079837120000000004, 2023-09-23T20:59:39.787 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T20:59:39.788 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "waiting for sub ops", 2023-09-23T20:59:39.788 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T20:59:39.788 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35100", 2023-09-23T20:59:39.788 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/4108972961", 2023-09-23T20:59:39.788 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 155442 2023-09-23T20:59:39.788 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:59:39.788 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T20:59:39.788 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:59:39.788 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T20:59:39.788 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:59:39.087791+0000", 2023-09-23T20:59:39.788 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:59:39.788 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:59:39.788 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:59:39.789 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T20:59:39.789 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:59:39.087791+0000", 2023-09-23T20:59:39.789 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T20:59:39.789 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:59:39.789 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:59:39.789 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T20:59:39.789 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:59:39.087790+0000", 2023-09-23T20:59:39.789 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T20:59:39.789 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:59:39.789 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:59:39.789 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T20:59:39.789 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:59:39.087794+0000", 2023-09-23T20:59:39.789 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.9060000000000004e-06 2023-09-23T20:59:39.790 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:59:39.790 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:59:39.790 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T20:59:39.790 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:59:39.087808+0000", 2023-09-23T20:59:39.790 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.3529999999999999e-05 2023-09-23T20:59:39.790 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:59:39.790 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:59:39.790 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T20:59:39.790 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:59:39.087815+0000", 2023-09-23T20:59:39.790 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.1489999999999997e-06 2023-09-23T20:59:39.790 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:59:39.791 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:59:39.791 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T20:59:39.791 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:59:39.087864+0000", 2023-09-23T20:59:39.791 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.8738e-05 2023-09-23T20:59:39.791 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:59:39.791 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:59:39.791 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T20:59:39.791 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:59:39.088026+0000", 2023-09-23T20:59:39.791 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00016286299999999999 2023-09-23T20:59:39.791 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:59:39.791 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:59:39.791 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 0,5", 2023-09-23T20:59:39.791 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:59:39.088141+0000", 2023-09-23T20:59:39.792 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000114876 2023-09-23T20:59:39.792 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T20:59:39.792 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T20:59:39.792 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:59:39.792 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:59:39.092647+0000", 2023-09-23T20:59:39.792 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.004505645 2023-09-23T20:59:39.792 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:59:39.792 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T20:59:39.793 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:59:39.793 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T20:59:39.793 INFO:teuthology.orchestra.run.smithi026.stdout: ], 2023-09-23T20:59:39.793 INFO:teuthology.orchestra.run.smithi026.stdout: "num_ops": 7 2023-09-23T20:59:39.793 INFO:teuthology.orchestra.run.smithi026.stdout:} 2023-09-23T20:59:39.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:59:39 smithi070 ceph-mon[168282]: pgmap v596: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 3.1 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 590 op/s; 11655/150573 objects degraded (7.740%); 2185/150573 objects misplaced (1.451%); 1.7 MiB/s, 26 objects/s recovering 2023-09-23T20:59:39.881 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:59:39.881 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:59:39 smithi026 ceph-mon[179688]: pgmap v596: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 3.1 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 590 op/s; 11655/150573 objects degraded (7.740%); 2185/150573 objects misplaced (1.451%); 1.7 MiB/s, 26 objects/s recovering 2023-09-23T20:59:39.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:59:39 smithi026 ceph-mon[175151]: pgmap v596: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 3.1 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 590 op/s; 11655/150573 objects degraded (7.740%); 2185/150573 objects misplaced (1.451%); 1.7 MiB/s, 26 objects/s recovering 2023-09-23T20:59:39.883 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:59:39 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[164029]: ::ffff:172.21.15.70 - - [23/Sep/2023:20:59:39] "GET /metrics HTTP/1.1" 200 36318 "" "Prometheus/2.43.0" 2023-09-23T20:59:39.892 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:59:39.892 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:59:39.993 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:59:40.080 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:59:40.344 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_blocked_ops 2023-09-23T20:59:40.354 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:59:40.356 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:59:40.356 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:59:40.457 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:59:40.485 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:59:40.524 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:59:40.526 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:59:40.527 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:59:40.627 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:59:40.725 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:59:40 smithi026 ceph-mon[179688]: Health check update: Degraded data redundancy: 11655/150573 objects degraded (7.740%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T20:59:40.726 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:59:40 smithi026 ceph-mon[175151]: Health check update: Degraded data redundancy: 11655/150573 objects degraded (7.740%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T20:59:40.742 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:59:40.861 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:59:40.869 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:59:40.869 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:59:40.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:59:40 smithi070 ceph-mon[168282]: Health check update: Degraded data redundancy: 11655/150573 objects degraded (7.740%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T20:59:40.872 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:59:40.970 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:59:40.999 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:59:41.034 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:59:41.039 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:59:41.039 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:59:41.140 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:59:41.169 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:59:41.206 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:59:41.209 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:59:41.209 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:59:41.310 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:59:41.337 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:59:41.378 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:59:41.383 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:59:41.384 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:59:41.458 INFO:teuthology.orchestra.run.smithi026.stdout: 2023-09-23T20:59:41.458 INFO:teuthology.orchestra.run.smithi026.stdout:{"fsid":"b3f8e4ec-5a50-11ee-8db4-212e2dc638e7","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"noscrub flag(s) set","count":7},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 11655/146640 objects degraded (7.948%), 3 pgs degraded, 3 pgs undersized","count":6},"muted":false}},"mutes":[]},"election_epoch":30,"quorum":[0,1,2],"quorum_names":["a","b","c"],"quorum_age":700,"monmap":{"epoch":4,"min_mon_release_name":"reef","num_mons":3},"osdmap":{"epoch":241,"num_osds":8,"num_up_osds":8,"osd_up_since":1695502534,"num_in_osds":8,"osd_in_since":1695502470,"num_remapped_pgs":3},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":106},{"state_name":"active+recovery_wait+undersized+degraded+remapped","count":2},{"state_name":"active+recovering+undersized+degraded+remapped","count":1}],"num_pgs":109,"num_pools":5,"num_objects":48880,"data_bytes":3202357176,"bytes_used":12039737344,"bytes_avail":755920547840,"bytes_total":767960285184,"degraded_objects":11655,"degraded_total":146640,"degraded_ratio":0.079480360065466452,"misplaced_objects":2185,"misplaced_total":146640,"misplaced_ratio":0.014900436442989636,"recovering_objects_per_sec":26,"recovering_bytes_per_sec":1761985,"recovering_keys_per_sec":0,"num_objects_recovered":269,"num_bytes_recovered":17629184,"num_keys_recovered":0,"write_bytes_sec":0,"read_op_per_sec":0,"write_op_per_sec":595},"fsmap":{"epoch":8,"id":1,"up":1,"in":1,"max":1,"by_rank":[{"filesystem_id":1,"rank":0,"name":"foo.smithi026.znfpnl","status":"up:active","gid":14658}],"up:standby":1},"mgrmap":{"available":true,"num_standbys":1,"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"services":{"dashboard":"https://172.21.15.26:8443/","prometheus":"http://172.21.15.26:9283/"}},"servicemap":{"epoch":21,"modified":"2023-09-23T20:56:34.314324+0000","services":{}},"progress_events":{"8b64cbfe-d367-468b-a773-0e71ee4ff645":{"message":"Global Recovery Event (0s)\n [............................] ","progress":0,"add_to_ceph_s":true},"fc746e9d-3f76-434d-b6d2-8fc9e8a72a5e":{"message":"Upgrade to 18.2.0-460-gbb0daeff (4m)\n [============................] (remaining: 5m)","progress":0.44999998807907104,"add_to_ceph_s":true}}} 2023-09-23T20:59:41.485 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:59:41.516 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:59:41.556 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:59:41.559 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:59:41.559 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:59:41.573 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:59:41.660 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:59:41.779 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:59:42.047 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:59:42 smithi026 ceph-mon[175151]: pgmap v597: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 3.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 595 op/s; 11655/146640 objects degraded (7.948%); 2185/146640 objects misplaced (1.490%); 1.7 MiB/s, 26 objects/s recovering 2023-09-23T20:59:42.047 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:59:42 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/694114202' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-23T20:59:42.212 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:59:42.228 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:59:42.229 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:59:42.231 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T20:59:42.330 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:59:42.348 INFO:tasks.cephadm.ceph_manager.ceph:making progress, resetting timeout 2023-09-23T20:59:42.348 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T20:59:42.353 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:59:42 smithi026 ceph-mon[179688]: pgmap v597: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 3.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 595 op/s; 11655/146640 objects degraded (7.948%); 2185/146640 objects misplaced (1.490%); 1.7 MiB/s, 26 objects/s recovering 2023-09-23T20:59:42.353 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:59:42 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/694114202' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-23T20:59:42.354 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:59:42 smithi070 ceph-mon[168282]: pgmap v597: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 3.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 595 op/s; 11655/146640 objects degraded (7.948%); 2185/146640 objects misplaced (1.490%); 1.7 MiB/s, 26 objects/s recovering 2023-09-23T20:59:42.354 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:59:42 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/694114202' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-23T20:59:42.357 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:59:42.394 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:59:42.396 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:59:42.396 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:59:42.497 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:59:42.691 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:59:42.896 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:59:42.903 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:59:42.903 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:59:43.004 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:59:43.040 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:59:43.081 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:59:43.083 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:59:43.084 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:59:43.185 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:59:43.213 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:59:43.253 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:59:43.261 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:59:43.261 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:59:43.362 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:59:43.391 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:59:43.430 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:59:43.437 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:59:43.437 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:59:43.538 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:59:43.569 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:59:43.609 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:59:43.615 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:59:43.615 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:59:43.716 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:59:43.728 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:59:43.747 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:59:43.785 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:59:43.788 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:59:43.788 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:59:43.831 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:59:43.889 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:59:43.994 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:59:44.098 INFO:teuthology.orchestra.run.smithi026.stderr:nodeep-scrub is set 2023-09-23T20:59:44.135 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:59:44 smithi026 ceph-mon[179688]: pgmap v598: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 2.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 600 op/s; 11708/141477 objects degraded (8.276%); 1989/141477 objects misplaced (1.406%); 1.7 MiB/s, 27 objects/s recovering 2023-09-23T20:59:44.135 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:59:44 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/2469407140' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:59:44.135 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:59:44 smithi026 ceph-mon[175151]: pgmap v598: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 2.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 600 op/s; 11708/141477 objects degraded (8.276%); 1989/141477 objects misplaced (1.406%); 1.7 MiB/s, 27 objects/s recovering 2023-09-23T20:59:44.136 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:59:44 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/2469407140' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:59:44.176 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:59:44.193 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:59:44.193 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:59:44.294 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:59:44.319 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:59:44 smithi070 ceph-mon[168282]: pgmap v598: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 2.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 600 op/s; 11708/141477 objects degraded (8.276%); 1989/141477 objects misplaced (1.406%); 1.7 MiB/s, 27 objects/s recovering 2023-09-23T20:59:44.319 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:59:44 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/2469407140' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:59:44.321 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:59:44.357 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:59:44.360 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:59:44.361 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:59:44.395 INFO:teuthology.orchestra.run.smithi026.stdout:{ 2023-09-23T20:59:44.395 INFO:teuthology.orchestra.run.smithi026.stdout: "ops": [], 2023-09-23T20:59:44.395 INFO:teuthology.orchestra.run.smithi026.stdout: "complaint_time": 30, 2023-09-23T20:59:44.395 INFO:teuthology.orchestra.run.smithi026.stdout: "num_blocked_ops": 0 2023-09-23T20:59:44.395 INFO:teuthology.orchestra.run.smithi026.stdout:} 2023-09-23T20:59:44.462 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:59:44.570 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:59:44.843 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:59:44.856 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:59:44.856 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:59:44.957 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:59:45.061 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:59:45 smithi026 ceph-mon[175151]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:59:45.061 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:59:45 smithi026 ceph-mon[175151]: failed to encode map e242 with expected crc 2023-09-23T20:59:45.061 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:59:45 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/2469407140' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T20:59:45.061 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:59:45 smithi026 ceph-mon[175151]: osdmap e242: 8 total, 8 up, 8 in 2023-09-23T20:59:45.062 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:59:45 smithi026 ceph-mon[175151]: failed to encode map e242 with expected crc 2023-09-23T20:59:45.062 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:59:45 smithi026 ceph-mon[175151]: failed to encode map e242 with expected crc 2023-09-23T20:59:45.062 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:59:45 smithi026 ceph-mon[175151]: Health check update: Degraded data redundancy: 11708/141477 objects degraded (8.276%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T20:59:45.078 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:59:45.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:59:45 smithi070 ceph-mon[168282]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:59:45.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:59:45 smithi070 ceph-mon[168282]: failed to encode map e242 with expected crc 2023-09-23T20:59:45.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:59:45 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/2469407140' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T20:59:45.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:59:45 smithi070 ceph-mon[168282]: osdmap e242: 8 total, 8 up, 8 in 2023-09-23T20:59:45.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:59:45 smithi070 ceph-mon[168282]: failed to encode map e242 with expected crc 2023-09-23T20:59:45.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:59:45 smithi070 ceph-mon[168282]: failed to encode map e242 with expected crc 2023-09-23T20:59:45.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:59:45 smithi070 ceph-mon[168282]: Health check update: Degraded data redundancy: 11708/141477 objects degraded (8.276%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T20:59:45.387 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:59:45 smithi026 ceph-mon[179688]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:59:45.387 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:59:45 smithi026 ceph-mon[179688]: failed to encode map e242 with expected crc 2023-09-23T20:59:45.387 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:59:45 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/2469407140' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T20:59:45.387 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:59:45 smithi026 ceph-mon[179688]: osdmap e242: 8 total, 8 up, 8 in 2023-09-23T20:59:45.387 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:59:45 smithi026 ceph-mon[179688]: failed to encode map e242 with expected crc 2023-09-23T20:59:45.387 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:59:45 smithi026 ceph-mon[179688]: failed to encode map e242 with expected crc 2023-09-23T20:59:45.387 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:59:45 smithi026 ceph-mon[179688]: Health check update: Degraded data redundancy: 11708/141477 objects degraded (8.276%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T20:59:46.177 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:59:46 smithi026 ceph-mon[179688]: pgmap v600: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 2.8 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 668 op/s; 11708/139401 objects degraded (8.399%); 1989/139401 objects misplaced (1.427%); 2.2 MiB/s, 34 objects/s recovering 2023-09-23T20:59:46.177 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:59:46 smithi026 ceph-mon[175151]: pgmap v600: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 2.8 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 668 op/s; 11708/139401 objects degraded (8.399%); 1989/139401 objects misplaced (1.427%); 2.2 MiB/s, 34 objects/s recovering 2023-09-23T20:59:46.223 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_historic_ops 2023-09-23T20:59:46.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:59:46 smithi070 ceph-mon[168282]: pgmap v600: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 2.8 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 668 op/s; 11708/139401 objects degraded (8.399%); 1989/139401 objects misplaced (1.427%); 2.2 MiB/s, 34 objects/s recovering 2023-09-23T20:59:46.990 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:59:47.000 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:59:47.000 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:59:47.034 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T20:59:47.100 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:59:47.136 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:59:47.175 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:59:47.178 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:59:47.178 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:59:47.279 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:59:47.294 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:59:47 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:59:47.294 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:59:47 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:59:47.294 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:59:47 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:59:47.294 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:59:47 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:59:47.459 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:59:47.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:59:47 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T20:59:47.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:59:47 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:59:47.700 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:59:47.714 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:59:47.714 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:59:47.815 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:59:47.916 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:59:48.078 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:59:48.084 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:59:48.084 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:59:48.170 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:59:48.185 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:59:48.213 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:59:48.255 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:59:48.261 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:59:48.261 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:59:48.362 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:59:48.383 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:59:48 smithi026 ceph-mon[179688]: pgmap v601: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 2.8 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 541 op/s; 11708/137790 objects degraded (8.497%); 1989/137790 objects misplaced (1.444%); 1.7 MiB/s, 27 objects/s recovering 2023-09-23T20:59:48.383 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:59:48 smithi026 ceph-mon[175151]: pgmap v601: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 2.8 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 541 op/s; 11708/137790 objects degraded (8.497%); 1989/137790 objects misplaced (1.444%); 1.7 MiB/s, 27 objects/s recovering 2023-09-23T20:59:48.396 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:59:48.433 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:59:48.436 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:59:48.436 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:59:48.537 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:59:48.565 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:59:48 smithi070 ceph-mon[168282]: pgmap v601: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 2.8 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 541 op/s; 11708/137790 objects degraded (8.497%); 1989/137790 objects misplaced (1.444%); 1.7 MiB/s, 27 objects/s recovering 2023-09-23T20:59:48.567 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:59:48.604 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:59:48.611 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:59:48.611 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:59:48.661 INFO:teuthology.orchestra.run.smithi026.stdout: 2023-09-23T20:59:48.672 INFO:teuthology.orchestra.run.smithi026.stderr:dumped all 2023-09-23T20:59:48.684 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:59:48.685 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0: {} 2023-09-23T20:59:48.712 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:59:48.731 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1: {} 2023-09-23T20:59:48.742 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:59:48.773 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2: {} 2023-09-23T20:59:48.779 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:59:48.784 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:59:48.785 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:59:48.823 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3: {} 2023-09-23T20:59:48.861 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4: {} 2023-09-23T20:59:48.886 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:59:48.897 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T20:59:48.913 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:59:48.936 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6: {} 2023-09-23T20:59:48.955 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:59:48.961 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:59:48.961 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:59:48.974 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T20:59:48.991 INFO:teuthology.orchestra.run.smithi026.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T20:59:48.991 INFO:teuthology.orchestra.run.smithi026.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T20:59:48.991 INFO:teuthology.orchestra.run.smithi026.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T20:59:48.992 INFO:teuthology.orchestra.run.smithi026.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T20:59:48.992 INFO:teuthology.orchestra.run.smithi026.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T20:59:48.992 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T20:59:48.992 INFO:teuthology.orchestra.run.smithi026.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T20:59:48.992 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T20:59:49.061 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:59:49.137 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:59:49.381 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:59:49 smithi026 ceph-mon[179688]: pgmap v602: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 2.7 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 588 op/s; 11714/132909 objects degraded (8.814%); 1828/132909 objects misplaced (1.375%); 1.9 MiB/s, 29 objects/s recovering 2023-09-23T20:59:49.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:59:49 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/351552795' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:59:49.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:59:49 smithi026 ceph-mon[179688]: from='client.45335 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:59:49.384 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:59:49 smithi026 ceph-mon[175151]: pgmap v602: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 2.7 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 588 op/s; 11714/132909 objects degraded (8.814%); 1828/132909 objects misplaced (1.375%); 1.9 MiB/s, 29 objects/s recovering 2023-09-23T20:59:49.384 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:59:49 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/351552795' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:59:49.384 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:59:49 smithi026 ceph-mon[175151]: from='client.45335 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:59:49.439 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:59:49.454 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:59:49.455 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:59:49.555 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:59:49.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:59:49 smithi070 ceph-mon[168282]: pgmap v602: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 2.7 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 588 op/s; 11714/132909 objects degraded (8.814%); 1828/132909 objects misplaced (1.375%); 1.9 MiB/s, 29 objects/s recovering 2023-09-23T20:59:49.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:59:49 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/351552795' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:59:49.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:59:49 smithi070 ceph-mon[168282]: from='client.45335 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:59:49.647 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:59:49.742 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:59:49 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[164029]: ::ffff:172.21.15.70 - - [23/Sep/2023:20:59:49] "GET /metrics HTTP/1.1" 200 36323 "" "Prometheus/2.43.0" 2023-09-23T20:59:50.114 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:59:50.120 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:59:50.120 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:59:50.172 INFO:teuthology.orchestra.run.smithi026.stdout:{"pg_ready":true,"pg_map":{"version":602,"stamp":"2023-09-23T20:59:48.458631+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":2902398904,"num_objects":44303,"num_object_clones":0,"num_object_copies":132909,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":11714,"num_objects_misplaced":1828,"num_objects_unfound":0,"num_objects_dirty":44303,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":159302,"num_write_kb":6512606,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2822,"num_bytes_recovered":183506790,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":122577,"ondisk_log_size":122577,"up":327,"acting":324,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":331,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":10868596,"kb_used_data":9229108,"kb_used_omap":136,"kb_used_meta":1639031,"kb_avail":739092620,"statfs":{"total":767960285184,"available":756830842880,"internally_reserved":0,"allocated":9450606592,"data_stored":9442542908,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":139741,"internal_metadata":1678368291},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,5,4,3,8,4,6],"upper_bound":128},"perf_stat":{"commit_latency_ms":10,"apply_latency_ms":10,"commit_latency_ns":10000000,"apply_latency_ns":10000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-385875968,"num_objects":-5888,"num_object_clones":0,"num_object_copies":-17664,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":59,"num_objects_misplaced":-357,"num_objects_unfound":0,"num_objects_dirty":-5888,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5888,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":298,"num_bytes_recovered":19529728,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.008473"},"pg_stats":[{"pgid":"3.1a","version":"68'1","reported_seq":119,"reported_epoch":241,"state":"active+clean","last_fresh":"2023-09-23T20:59:36.546396+0000","last_change":"2023-09-23T20:54:13.946695+0000","last_active":"2023-09-23T20:59:36.546396+0000","last_peered":"2023-09-23T20:59:36.546396+0000","last_clean":"2023-09-23T20:59:36.546396+0000","last_became_active":"2023-09-23T20:54:13.945399+0000","last_became_peered":"2023-09-23T20:54:13.945399+0000","last_unstale":"2023-09-23T20:59:36.546396+0000","last_undegraded":"2023-09-23T20:59:36.546396+0000","last_fullsized":"2023-09-23T20:59:36.546396+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:32:52.365464+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":234,"reported_epoch":241,"state":"active+clean","last_fresh":"2023-09-23T20:59:36.567297+0000","last_change":"2023-09-23T20:54:13.942019+0000","last_active":"2023-09-23T20:59:36.567297+0000","last_peered":"2023-09-23T20:59:36.567297+0000","last_clean":"2023-09-23T20:59:36.567297+0000","last_became_active":"2023-09-23T20:54:13.941647+0000","last_became_peered":"2023-09-23T20:54:13.941647+0000","last_unstale":"2023-09-23T20:59:36.567297+0000","last_undegraded":"2023-09-23T20:59:36.567297+0000","last_fullsized":"2023-09-23T20:59:36.567297+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:30:54.400944+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":212,"reported_epoch":242,"state":"active+clean","last_fresh":"2023-09-23T20:59:44.087915+0000","last_change":"2023-09-23T20:54:13.941394+0000","last_active":"2023-09-23T20:59:44.087915+0000","last_peered":"2023-09-23T20:59:44.087915+0000","last_clean":"2023-09-23T20:59:44.087915+0000","last_became_active":"2023-09-23T20:54:13.941042+0000","last_became_peered":"2023-09-23T20:54:13.941042+0000","last_unstale":"2023-09-23T20:59:44.087915+0000","last_undegraded":"2023-09-23T20:59:44.087915+0000","last_fullsized":"2023-09-23T20:59:44.087915+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:19:41.401184+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":227,"reported_epoch":241,"state":"active+clean","last_fresh":"2023-09-23T20:59:36.548735+0000","last_change":"2023-09-23T20:57:44.979811+0000","last_active":"2023-09-23T20:59:36.548735+0000","last_peered":"2023-09-23T20:59:36.548735+0000","last_clean":"2023-09-23T20:59:36.548735+0000","last_became_active":"2023-09-23T20:57:44.979515+0000","last_became_peered":"2023-09-23T20:57:44.979515+0000","last_unstale":"2023-09-23T20:59:36.548735+0000","last_undegraded":"2023-09-23T20:59:36.548735+0000","last_fullsized":"2023-09-23T20:59:36.548735+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:38:22.162140+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":225,"reported_epoch":241,"state":"active+clean","last_fresh":"2023-09-23T20:59:36.567108+0000","last_change":"2023-09-23T20:51:10.833613+0000","last_active":"2023-09-23T20:59:36.567108+0000","last_peered":"2023-09-23T20:59:36.567108+0000","last_clean":"2023-09-23T20:59:36.567108+0000","last_became_active":"2023-09-23T20:51:10.833208+0000","last_became_peered":"2023-09-23T20:51:10.833208+0000","last_unstale":"2023-09-23T20:59:36.567108+0000","last_undegraded":"2023-09-23T20:59:36.567108+0000","last_fullsized":"2023-09-23T20:59:36.567108+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:51:28.246014+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":207,"reported_epoch":241,"state":"active+clean","last_fresh":"2023-09-23T20:59:36.548278+0000","last_change":"2023-09-23T20:55:35.958384+0000","last_active":"2023-09-23T20:59:36.548278+0000","last_peered":"2023-09-23T20:59:36.548278+0000","last_clean":"2023-09-23T20:59:36.548278+0000","last_became_active":"2023-09-23T20:55:35.954563+0000","last_became_peered":"2023-09-23T20:55:35.954563+0000","last_unstale":"2023-09-23T20:59:36.548278+0000","last_undegraded":"2023-09-23T20:59:36.548278+0000","last_fullsized":"2023-09-23T20:59:36.548278+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:37:48.332259+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":217,"reported_epoch":241,"state":"active+clean","last_fresh":"2023-09-23T20:59:36.547689+0000","last_change":"2023-09-23T20:54:31.885514+0000","last_active":"2023-09-23T20:59:36.547689+0000","last_peered":"2023-09-23T20:59:36.547689+0000","last_clean":"2023-09-23T20:59:36.547689+0000","last_became_active":"2023-09-23T20:54:31.884986+0000","last_became_peered":"2023-09-23T20:54:31.884986+0000","last_unstale":"2023-09-23T20:59:36.547689+0000","last_undegraded":"2023-09-23T20:59:36.547689+0000","last_fullsized":"2023-09-23T20:59:36.547689+0000","mapping_epoch":136,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:26:08.100220+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":222,"reported_epoch":241,"state":"active+clean","last_fresh":"2023-09-23T20:59:36.567050+0000","last_change":"2023-09-23T20:55:35.951889+0000","last_active":"2023-09-23T20:59:36.567050+0000","last_peered":"2023-09-23T20:59:36.567050+0000","last_clean":"2023-09-23T20:59:36.567050+0000","last_became_active":"2023-09-23T20:55:35.951563+0000","last_became_peered":"2023-09-23T20:55:35.951563+0000","last_unstale":"2023-09-23T20:59:36.567050+0000","last_undegraded":"2023-09-23T20:59:36.567050+0000","last_fullsized":"2023-09-23T20:59:36.567050+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:41:53.098781+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"79'161","reported_seq":529,"reported_epoch":241,"state":"active+clean","last_fresh":"2023-09-23T20:59:36.571037+0000","last_change":"2023-09-23T20:57:25.096559+0000","last_active":"2023-09-23T20:59:36.571037+0000","last_peered":"2023-09-23T20:59:36.571037+0000","last_clean":"2023-09-23T20:59:36.571037+0000","last_became_active":"2023-09-23T20:57:15.975674+0000","last_became_peered":"2023-09-23T20:57:15.975674+0000","last_unstale":"2023-09-23T20:59:36.571037+0000","last_undegraded":"2023-09-23T20:59:36.571037+0000","last_fullsized":"2023-09-23T20:59:36.571037+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:25.096401+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:25.096401+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:05:05.448404+0000","scrub_duration":0.002859393,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":115,"reported_epoch":241,"state":"active+clean","last_fresh":"2023-09-23T20:59:36.557518+0000","last_change":"2023-09-23T20:54:31.890678+0000","last_active":"2023-09-23T20:59:36.557518+0000","last_peered":"2023-09-23T20:59:36.557518+0000","last_clean":"2023-09-23T20:59:36.557518+0000","last_became_active":"2023-09-23T20:54:31.890469+0000","last_became_peered":"2023-09-23T20:54:31.890469+0000","last_unstale":"2023-09-23T20:59:36.557518+0000","last_undegraded":"2023-09-23T20:59:36.557518+0000","last_fullsized":"2023-09-23T20:59:36.557518+0000","mapping_epoch":136,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:55:08.790146+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":183,"reported_epoch":241,"state":"active+clean","last_fresh":"2023-09-23T20:59:36.567888+0000","last_change":"2023-09-23T20:55:35.952661+0000","last_active":"2023-09-23T20:59:36.567888+0000","last_peered":"2023-09-23T20:59:36.567888+0000","last_clean":"2023-09-23T20:59:36.567888+0000","last_became_active":"2023-09-23T20:55:35.952349+0000","last_became_peered":"2023-09-23T20:55:35.952349+0000","last_unstale":"2023-09-23T20:59:36.567888+0000","last_undegraded":"2023-09-23T20:59:36.567888+0000","last_fullsized":"2023-09-23T20:59:36.567888+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:00:03.110126+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":60,"reported_epoch":241,"state":"active+clean","last_fresh":"2023-09-23T20:59:36.567377+0000","last_change":"2023-09-23T20:56:45.988607+0000","last_active":"2023-09-23T20:59:36.567377+0000","last_peered":"2023-09-23T20:59:36.567377+0000","last_clean":"2023-09-23T20:59:36.567377+0000","last_became_active":"2023-09-23T20:56:45.988206+0000","last_became_peered":"2023-09-23T20:56:45.988206+0000","last_unstale":"2023-09-23T20:59:36.567377+0000","last_undegraded":"2023-09-23T20:59:36.567377+0000","last_fullsized":"2023-09-23T20:59:36.567377+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":205,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:01:19.783688+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,4],"acting":[3,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":59,"reported_epoch":241,"state":"active+clean","last_fresh":"2023-09-23T20:59:36.556172+0000","last_change":"2023-09-23T20:56:18.139142+0000","last_active":"2023-09-23T20:59:36.556172+0000","last_peered":"2023-09-23T20:59:36.556172+0000","last_clean":"2023-09-23T20:59:36.556172+0000","last_became_active":"2023-09-23T20:56:18.138844+0000","last_became_peered":"2023-09-23T20:56:18.138844+0000","last_unstale":"2023-09-23T20:59:36.556172+0000","last_undegraded":"2023-09-23T20:59:36.556172+0000","last_fullsized":"2023-09-23T20:59:36.556172+0000","mapping_epoch":192,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":193,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:09:02.217683+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,2],"acting":[5,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.14","version":"79'22","reported_seq":146,"reported_epoch":241,"state":"active+clean","last_fresh":"2023-09-23T20:59:36.546334+0000","last_change":"2023-09-23T20:55:35.949418+0000","last_active":"2023-09-23T20:59:36.546334+0000","last_peered":"2023-09-23T20:59:36.546334+0000","last_clean":"2023-09-23T20:59:36.546334+0000","last_became_active":"2023-09-23T20:55:35.948566+0000","last_became_peered":"2023-09-23T20:55:35.948566+0000","last_unstale":"2023-09-23T20:59:36.546334+0000","last_undegraded":"2023-09-23T20:59:36.546334+0000","last_fullsized":"2023-09-23T20:59:36.546334+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:59:37.433128+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4734,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":136,"reported_epoch":241,"state":"active+clean","last_fresh":"2023-09-23T20:59:36.546715+0000","last_change":"2023-09-23T20:55:35.949178+0000","last_active":"2023-09-23T20:59:36.546715+0000","last_peered":"2023-09-23T20:59:36.546715+0000","last_clean":"2023-09-23T20:59:36.546715+0000","last_became_active":"2023-09-23T20:55:35.948795+0000","last_became_peered":"2023-09-23T20:55:35.948795+0000","last_unstale":"2023-09-23T20:59:36.546715+0000","last_undegraded":"2023-09-23T20:59:36.546715+0000","last_fullsized":"2023-09-23T20:59:36.546715+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:13:08.996932+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.11","version":"79'161","reported_seq":471,"reported_epoch":241,"state":"active+clean","last_fresh":"2023-09-23T20:59:36.656374+0000","last_change":"2023-09-23T20:57:28.756796+0000","last_active":"2023-09-23T20:59:36.656374+0000","last_peered":"2023-09-23T20:59:36.656374+0000","last_clean":"2023-09-23T20:59:36.656374+0000","last_became_active":"2023-09-23T20:57:15.980606+0000","last_became_peered":"2023-09-23T20:57:15.980606+0000","last_unstale":"2023-09-23T20:59:36.656374+0000","last_undegraded":"2023-09-23T20:59:36.656374+0000","last_fullsized":"2023-09-23T20:59:36.656374+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:28.756634+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:28.756634+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:09:19.150791+0000","scrub_duration":0.0064925119999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":162,"reported_epoch":241,"state":"active+clean","last_fresh":"2023-09-23T20:59:36.546669+0000","last_change":"2023-09-23T20:56:45.992577+0000","last_active":"2023-09-23T20:59:36.546669+0000","last_peered":"2023-09-23T20:59:36.546669+0000","last_clean":"2023-09-23T20:59:36.546669+0000","last_became_active":"2023-09-23T20:56:45.992297+0000","last_became_peered":"2023-09-23T20:56:45.992297+0000","last_unstale":"2023-09-23T20:59:36.546669+0000","last_undegraded":"2023-09-23T20:59:36.546669+0000","last_fullsized":"2023-09-23T20:59:36.546669+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":205,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:00:45.547202+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,2],"acting":[4,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":207,"reported_epoch":241,"state":"active+clean","last_fresh":"2023-09-23T20:59:36.568255+0000","last_change":"2023-09-23T20:51:10.831262+0000","last_active":"2023-09-23T20:59:36.568255+0000","last_peered":"2023-09-23T20:59:36.568255+0000","last_clean":"2023-09-23T20:59:36.568255+0000","last_became_active":"2023-09-23T20:51:10.830931+0000","last_became_peered":"2023-09-23T20:51:10.830931+0000","last_unstale":"2023-09-23T20:59:36.568255+0000","last_undegraded":"2023-09-23T20:59:36.568255+0000","last_fullsized":"2023-09-23T20:59:36.568255+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:51:54.982505+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.10","version":"79'161","reported_seq":479,"reported_epoch":241,"state":"active+clean","last_fresh":"2023-09-23T20:59:36.557007+0000","last_change":"2023-09-23T20:57:20.708003+0000","last_active":"2023-09-23T20:59:36.557007+0000","last_peered":"2023-09-23T20:59:36.557007+0000","last_clean":"2023-09-23T20:59:36.557007+0000","last_became_active":"2023-09-23T20:57:15.975302+0000","last_became_peered":"2023-09-23T20:57:15.975302+0000","last_unstale":"2023-09-23T20:59:36.557007+0000","last_undegraded":"2023-09-23T20:59:36.557007+0000","last_fullsized":"2023-09-23T20:59:36.557007+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:20.707939+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:20.707939+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:16:51.721994+0000","scrub_duration":0.0017733409999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":217,"reported_epoch":241,"state":"active+clean","last_fresh":"2023-09-23T20:59:36.546636+0000","last_change":"2023-09-23T20:57:44.979736+0000","last_active":"2023-09-23T20:59:36.546636+0000","last_peered":"2023-09-23T20:59:36.546636+0000","last_clean":"2023-09-23T20:59:36.546636+0000","last_became_active":"2023-09-23T20:57:44.979356+0000","last_became_peered":"2023-09-23T20:57:44.979356+0000","last_unstale":"2023-09-23T20:59:36.546636+0000","last_undegraded":"2023-09-23T20:59:36.546636+0000","last_fullsized":"2023-09-23T20:59:36.546636+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:14:43.848832+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":115,"reported_epoch":241,"state":"active+clean","last_fresh":"2023-09-23T20:59:36.556746+0000","last_change":"2023-09-23T20:54:31.887814+0000","last_active":"2023-09-23T20:59:36.556746+0000","last_peered":"2023-09-23T20:59:36.556746+0000","last_clean":"2023-09-23T20:59:36.556746+0000","last_became_active":"2023-09-23T20:54:31.887195+0000","last_became_peered":"2023-09-23T20:54:31.887195+0000","last_unstale":"2023-09-23T20:59:36.556746+0000","last_undegraded":"2023-09-23T20:59:36.556746+0000","last_fullsized":"2023-09-23T20:59:36.556746+0000","mapping_epoch":136,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:10:52.089553+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.13","version":"79'161","reported_seq":542,"reported_epoch":241,"state":"active+clean","last_fresh":"2023-09-23T20:59:36.545643+0000","last_change":"2023-09-23T20:57:48.262970+0000","last_active":"2023-09-23T20:59:36.545643+0000","last_peered":"2023-09-23T20:59:36.545643+0000","last_clean":"2023-09-23T20:59:36.545643+0000","last_became_active":"2023-09-23T20:57:48.262663+0000","last_became_peered":"2023-09-23T20:57:48.262663+0000","last_unstale":"2023-09-23T20:59:36.545643+0000","last_undegraded":"2023-09-23T20:59:36.545643+0000","last_fullsized":"2023-09-23T20:59:36.545643+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:36.420362+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:36.420362+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:28:28.613492+0000","scrub_duration":0.028838444000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":186,"reported_epoch":241,"state":"active+clean","last_fresh":"2023-09-23T20:59:36.552324+0000","last_change":"2023-09-23T20:56:45.991822+0000","last_active":"2023-09-23T20:59:36.552324+0000","last_peered":"2023-09-23T20:59:36.552324+0000","last_clean":"2023-09-23T20:59:36.552324+0000","last_became_active":"2023-09-23T20:56:45.991460+0000","last_became_peered":"2023-09-23T20:56:45.991460+0000","last_unstale":"2023-09-23T20:59:36.552324+0000","last_undegraded":"2023-09-23T20:59:36.552324+0000","last_fullsized":"2023-09-23T20:59:36.552324+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":205,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:20:37.503317+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":199,"reported_epoch":241,"state":"active+clean","last_fresh":"2023-09-23T20:59:36.548852+0000","last_change":"2023-09-23T20:51:10.840849+0000","last_active":"2023-09-23T20:59:36.548852+0000","last_peered":"2023-09-23T20:59:36.548852+0000","last_clean":"2023-09-23T20:59:36.548852+0000","last_became_active":"2023-09-23T20:51:10.840593+0000","last_became_peered":"2023-09-23T20:51:10.840593+0000","last_unstale":"2023-09-23T20:59:36.548852+0000","last_undegraded":"2023-09-23T20:59:36.548852+0000","last_fullsized":"2023-09-23T20:59:36.548852+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:30:52.297625+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.12","version":"79'161","reported_seq":479,"reported_epoch":241,"state":"active+clean","last_fresh":"2023-09-23T20:59:36.555908+0000","last_change":"2023-09-23T20:57:21.750765+0000","last_active":"2023-09-23T20:59:36.555908+0000","last_peered":"2023-09-23T20:59:36.555908+0000","last_clean":"2023-09-23T20:59:36.555908+0000","last_became_active":"2023-09-23T20:57:15.979195+0000","last_became_peered":"2023-09-23T20:57:15.979195+0000","last_unstale":"2023-09-23T20:59:36.555908+0000","last_undegraded":"2023-09-23T20:59:36.555908+0000","last_fullsized":"2023-09-23T20:59:36.555908+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:21.750680+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:21.750680+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:03:41.915962+0000","scrub_duration":0.0092391680000000007,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":234,"reported_epoch":241,"state":"active+clean","last_fresh":"2023-09-23T20:59:36.547848+0000","last_change":"2023-09-23T20:55:36.943969+0000","last_active":"2023-09-23T20:59:36.547848+0000","last_peered":"2023-09-23T20:59:36.547848+0000","last_clean":"2023-09-23T20:59:36.547848+0000","last_became_active":"2023-09-23T20:55:36.943580+0000","last_became_peered":"2023-09-23T20:55:36.943580+0000","last_unstale":"2023-09-23T20:59:36.547848+0000","last_undegraded":"2023-09-23T20:59:36.547848+0000","last_fullsized":"2023-09-23T20:59:36.547848+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:49:06.969396+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":30,"reported_epoch":241,"state":"active+clean","last_fresh":"2023-09-23T20:59:36.574857+0000","last_change":"2023-09-23T20:57:44.980498+0000","last_active":"2023-09-23T20:59:36.574857+0000","last_peered":"2023-09-23T20:59:36.574857+0000","last_clean":"2023-09-23T20:59:36.574857+0000","last_became_active":"2023-09-23T20:57:44.980182+0000","last_became_peered":"2023-09-23T20:57:44.980182+0000","last_unstale":"2023-09-23T20:59:36.574857+0000","last_undegraded":"2023-09-23T20:59:36.574857+0000","last_fullsized":"2023-09-23T20:59:36.574857+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:34:51.620544+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.d","version":"79'161","reported_seq":524,"reported_epoch":241,"state":"active+clean","last_fresh":"2023-09-23T20:59:36.546379+0000","last_change":"2023-09-23T20:57:31.075213+0000","last_active":"2023-09-23T20:59:36.546379+0000","last_peered":"2023-09-23T20:59:36.546379+0000","last_clean":"2023-09-23T20:59:36.546379+0000","last_became_active":"2023-09-23T20:57:15.988188+0000","last_became_peered":"2023-09-23T20:57:15.988188+0000","last_unstale":"2023-09-23T20:59:36.546379+0000","last_undegraded":"2023-09-23T20:59:36.546379+0000","last_fullsized":"2023-09-23T20:59:36.546379+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:31.075098+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:31.075098+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:41:52.142362+0000","scrub_duration":0.010304610000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.b","version":"242'9776","reported_seq":9829,"reported_epoch":242,"state":"active+clean","last_fresh":"2023-09-23T20:59:44.283651+0000","last_change":"2023-09-23T20:56:35.543994+0000","last_active":"2023-09-23T20:59:44.283651+0000","last_peered":"2023-09-23T20:59:44.283651+0000","last_clean":"2023-09-23T20:59:44.283651+0000","last_became_active":"2023-09-23T20:56:35.543758+0000","last_became_peered":"2023-09-23T20:56:35.543758+0000","last_unstale":"2023-09-23T20:59:44.283651+0000","last_undegraded":"2023-09-23T20:59:44.283651+0000","last_fullsized":"2023-09-23T20:59:44.283651+0000","mapping_epoch":199,"log_start":"206'2700","ondisk_log_start":"206'2700","created":199,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_clean_scrub_stamp":"2023-09-23T20:56:34.529103+0000","objects_scrubbed":0,"log_size":7076,"log_dups_size":0,"ondisk_log_size":7076,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:21:29.041484+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":190316544,"num_objects":2904,"num_object_clones":0,"num_object_copies":8712,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2904,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9776,"num_write_kb":405760,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":173,"reported_epoch":241,"state":"active+clean","last_fresh":"2023-09-23T20:59:36.546690+0000","last_change":"2023-09-23T20:56:45.991841+0000","last_active":"2023-09-23T20:59:36.546690+0000","last_peered":"2023-09-23T20:59:36.546690+0000","last_clean":"2023-09-23T20:59:36.546690+0000","last_became_active":"2023-09-23T20:56:45.991491+0000","last_became_peered":"2023-09-23T20:56:45.991491+0000","last_unstale":"2023-09-23T20:59:36.546690+0000","last_undegraded":"2023-09-23T20:59:36.546690+0000","last_fullsized":"2023-09-23T20:59:36.546690+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":205,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:30:37.405319+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":132,"reported_epoch":241,"state":"active+clean","last_fresh":"2023-09-23T20:59:36.557164+0000","last_change":"2023-09-23T20:55:35.957768+0000","last_active":"2023-09-23T20:59:36.557164+0000","last_peered":"2023-09-23T20:59:36.557164+0000","last_clean":"2023-09-23T20:59:36.557164+0000","last_became_active":"2023-09-23T20:55:35.957421+0000","last_became_peered":"2023-09-23T20:55:35.957421+0000","last_unstale":"2023-09-23T20:59:36.557164+0000","last_undegraded":"2023-09-23T20:59:36.557164+0000","last_fullsized":"2023-09-23T20:59:36.557164+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:49:41.585961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.c","version":"79'161","reported_seq":543,"reported_epoch":241,"state":"active+clean","last_fresh":"2023-09-23T20:59:36.568130+0000","last_change":"2023-09-23T20:57:22.041071+0000","last_active":"2023-09-23T20:59:36.568130+0000","last_peered":"2023-09-23T20:59:36.568130+0000","last_clean":"2023-09-23T20:59:36.568130+0000","last_became_active":"2023-09-23T20:57:15.973204+0000","last_became_peered":"2023-09-23T20:57:15.973204+0000","last_unstale":"2023-09-23T20:59:36.568130+0000","last_undegraded":"2023-09-23T20:59:36.568130+0000","last_fullsized":"2023-09-23T20:59:36.568130+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:22.040907+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:22.040907+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:47:02.019978+0000","scrub_duration":0.013021176000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.a","version":"242'9882","reported_seq":9935,"reported_epoch":242,"state":"active+clean","last_fresh":"2023-09-23T20:59:44.533860+0000","last_change":"2023-09-23T20:56:35.547120+0000","last_active":"2023-09-23T20:59:44.533860+0000","last_peered":"2023-09-23T20:59:44.533860+0000","last_clean":"2023-09-23T20:59:44.533860+0000","last_became_active":"2023-09-23T20:56:35.546838+0000","last_became_peered":"2023-09-23T20:56:35.546838+0000","last_unstale":"2023-09-23T20:59:44.533860+0000","last_undegraded":"2023-09-23T20:59:44.533860+0000","last_fullsized":"2023-09-23T20:59:44.533860+0000","mapping_epoch":199,"log_start":"215'3700","ondisk_log_start":"215'3700","created":199,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_clean_scrub_stamp":"2023-09-23T20:56:34.529103+0000","objects_scrubbed":0,"log_size":6182,"log_dups_size":0,"ondisk_log_size":6182,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:48:24.132768+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":187039744,"num_objects":2854,"num_object_clones":0,"num_object_copies":8562,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2854,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9882,"num_write_kb":407552,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,5],"acting":[3,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":160,"reported_epoch":241,"state":"active+clean","last_fresh":"2023-09-23T20:59:36.546553+0000","last_change":"2023-09-23T20:56:45.991821+0000","last_active":"2023-09-23T20:59:36.546553+0000","last_peered":"2023-09-23T20:59:36.546553+0000","last_clean":"2023-09-23T20:59:36.546553+0000","last_became_active":"2023-09-23T20:56:45.991447+0000","last_became_peered":"2023-09-23T20:56:45.991447+0000","last_unstale":"2023-09-23T20:59:36.546553+0000","last_undegraded":"2023-09-23T20:59:36.546553+0000","last_fullsized":"2023-09-23T20:59:36.546553+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":205,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:14:25.525222+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":163,"reported_epoch":241,"state":"active+clean","last_fresh":"2023-09-23T20:59:36.552431+0000","last_change":"2023-09-23T20:57:44.982059+0000","last_active":"2023-09-23T20:59:36.552431+0000","last_peered":"2023-09-23T20:59:36.552431+0000","last_clean":"2023-09-23T20:59:36.552431+0000","last_became_active":"2023-09-23T20:57:44.981742+0000","last_became_peered":"2023-09-23T20:57:44.981742+0000","last_unstale":"2023-09-23T20:59:36.552431+0000","last_undegraded":"2023-09-23T20:59:36.552431+0000","last_fullsized":"2023-09-23T20:59:36.552431+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:41:24.042916+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.f","version":"79'161","reported_seq":546,"reported_epoch":241,"state":"active+clean","last_fresh":"2023-09-23T20:59:36.548022+0000","last_change":"2023-09-23T20:57:27.880841+0000","last_active":"2023-09-23T20:59:36.548022+0000","last_peered":"2023-09-23T20:59:36.548022+0000","last_clean":"2023-09-23T20:59:36.548022+0000","last_became_active":"2023-09-23T20:57:15.970925+0000","last_became_peered":"2023-09-23T20:57:15.970925+0000","last_unstale":"2023-09-23T20:59:36.548022+0000","last_undegraded":"2023-09-23T20:59:36.548022+0000","last_fullsized":"2023-09-23T20:59:36.548022+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:27.880776+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:27.880776+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:15:13.438014+0000","scrub_duration":0.0014004899999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.9","version":"242'9994","reported_seq":10047,"reported_epoch":242,"state":"active+clean","last_fresh":"2023-09-23T20:59:46.693624+0000","last_change":"2023-09-23T20:56:35.549387+0000","last_active":"2023-09-23T20:59:46.693624+0000","last_peered":"2023-09-23T20:59:46.693624+0000","last_clean":"2023-09-23T20:59:46.693624+0000","last_became_active":"2023-09-23T20:56:35.549087+0000","last_became_peered":"2023-09-23T20:56:35.549087+0000","last_unstale":"2023-09-23T20:59:46.693624+0000","last_undegraded":"2023-09-23T20:59:46.693624+0000","last_fullsized":"2023-09-23T20:59:46.693624+0000","mapping_epoch":199,"log_start":"206'2600","ondisk_log_start":"206'2600","created":199,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_clean_scrub_stamp":"2023-09-23T20:56:34.529103+0000","objects_scrubbed":0,"log_size":7394,"log_dups_size":0,"ondisk_log_size":7394,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:04:51.547362+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":177733632,"num_objects":2712,"num_object_clones":0,"num_object_copies":8136,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2712,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9994,"num_write_kb":406592,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":146,"reported_epoch":241,"state":"active+clean","last_fresh":"2023-09-23T20:59:36.556678+0000","last_change":"2023-09-23T20:56:45.989153+0000","last_active":"2023-09-23T20:59:36.556678+0000","last_peered":"2023-09-23T20:59:36.556678+0000","last_clean":"2023-09-23T20:59:36.556678+0000","last_became_active":"2023-09-23T20:56:45.988800+0000","last_became_peered":"2023-09-23T20:56:45.988800+0000","last_unstale":"2023-09-23T20:59:36.556678+0000","last_undegraded":"2023-09-23T20:59:36.556678+0000","last_fullsized":"2023-09-23T20:59:36.556678+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":205,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:04:41.130194+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":181,"reported_epoch":241,"state":"active+clean","last_fresh":"2023-09-23T20:59:36.547905+0000","last_change":"2023-09-23T20:55:35.965427+0000","last_active":"2023-09-23T20:59:36.547905+0000","last_peered":"2023-09-23T20:59:36.547905+0000","last_clean":"2023-09-23T20:59:36.547905+0000","last_became_active":"2023-09-23T20:55:35.963969+0000","last_became_peered":"2023-09-23T20:55:35.963969+0000","last_unstale":"2023-09-23T20:59:36.547905+0000","last_undegraded":"2023-09-23T20:59:36.547905+0000","last_fullsized":"2023-09-23T20:59:36.547905+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:38:40.612083+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.e","version":"79'161","reported_seq":453,"reported_epoch":241,"state":"active+clean","last_fresh":"2023-09-23T20:59:36.546404+0000","last_change":"2023-09-23T20:57:34.132738+0000","last_active":"2023-09-23T20:59:36.546404+0000","last_peered":"2023-09-23T20:59:36.546404+0000","last_clean":"2023-09-23T20:59:36.546404+0000","last_became_active":"2023-09-23T20:57:15.974318+0000","last_became_peered":"2023-09-23T20:57:15.974318+0000","last_unstale":"2023-09-23T20:59:36.546404+0000","last_undegraded":"2023-09-23T20:59:36.546404+0000","last_fullsized":"2023-09-23T20:59:36.546404+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:34.132660+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:34.132660+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:07:07.739438+0000","scrub_duration":0.005229694,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.8","version":"242'10089","reported_seq":10142,"reported_epoch":242,"state":"active+clean","last_fresh":"2023-09-23T20:59:46.437937+0000","last_change":"2023-09-23T20:56:35.548119+0000","last_active":"2023-09-23T20:59:46.437937+0000","last_peered":"2023-09-23T20:59:46.437937+0000","last_clean":"2023-09-23T20:59:46.437937+0000","last_became_active":"2023-09-23T20:56:35.547914+0000","last_became_peered":"2023-09-23T20:56:35.547914+0000","last_unstale":"2023-09-23T20:59:46.437937+0000","last_undegraded":"2023-09-23T20:59:46.437937+0000","last_fullsized":"2023-09-23T20:59:46.437937+0000","mapping_epoch":199,"log_start":"205'1200","ondisk_log_start":"205'1200","created":199,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_clean_scrub_stamp":"2023-09-23T20:56:34.529103+0000","objects_scrubbed":0,"log_size":8889,"log_dups_size":0,"ondisk_log_size":8889,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:42:54.255023+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":181207040,"num_objects":2765,"num_object_clones":0,"num_object_copies":8295,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2765,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10089,"num_write_kb":411328,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":132,"reported_epoch":241,"state":"active+clean","last_fresh":"2023-09-23T20:59:36.556446+0000","last_change":"2023-09-23T20:55:35.962005+0000","last_active":"2023-09-23T20:59:36.556446+0000","last_peered":"2023-09-23T20:59:36.556446+0000","last_clean":"2023-09-23T20:59:36.556446+0000","last_became_active":"2023-09-23T20:55:35.961695+0000","last_became_peered":"2023-09-23T20:55:35.961695+0000","last_unstale":"2023-09-23T20:59:36.556446+0000","last_undegraded":"2023-09-23T20:59:36.556446+0000","last_fullsized":"2023-09-23T20:59:36.556446+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:28:22.876201+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":230,"reported_epoch":241,"state":"active+clean","last_fresh":"2023-09-23T20:59:36.546356+0000","last_change":"2023-09-23T20:54:13.928715+0000","last_active":"2023-09-23T20:59:36.546356+0000","last_peered":"2023-09-23T20:59:36.546356+0000","last_clean":"2023-09-23T20:59:36.546356+0000","last_became_active":"2023-09-23T20:54:13.928200+0000","last_became_peered":"2023-09-23T20:54:13.928200+0000","last_unstale":"2023-09-23T20:59:36.546356+0000","last_undegraded":"2023-09-23T20:59:36.546356+0000","last_fullsized":"2023-09-23T20:59:36.546356+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:41:16.927513+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.f","version":"242'10161","reported_seq":10214,"reported_epoch":242,"state":"active+clean","last_fresh":"2023-09-23T20:59:46.709347+0000","last_change":"2023-09-23T20:56:35.545128+0000","last_active":"2023-09-23T20:59:46.709347+0000","last_peered":"2023-09-23T20:59:46.709347+0000","last_clean":"2023-09-23T20:59:46.709347+0000","last_became_active":"2023-09-23T20:56:35.544838+0000","last_became_peered":"2023-09-23T20:56:35.544838+0000","last_unstale":"2023-09-23T20:59:46.709347+0000","last_undegraded":"2023-09-23T20:59:46.709347+0000","last_fullsized":"2023-09-23T20:59:46.709347+0000","mapping_epoch":199,"log_start":"206'2800","ondisk_log_start":"206'2800","created":199,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_clean_scrub_stamp":"2023-09-23T20:56:34.529103+0000","objects_scrubbed":0,"log_size":7361,"log_dups_size":0,"ondisk_log_size":7361,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:30:43.388717+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":177274880,"num_objects":2705,"num_object_clones":0,"num_object_copies":8115,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2705,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10161,"num_write_kb":411712,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.9","version":"79'161","reported_seq":615,"reported_epoch":241,"state":"active+clean","last_fresh":"2023-09-23T20:59:36.567813+0000","last_change":"2023-09-23T20:56:46.921939+0000","last_active":"2023-09-23T20:59:36.567813+0000","last_peered":"2023-09-23T20:59:36.567813+0000","last_clean":"2023-09-23T20:59:36.567813+0000","last_became_active":"2023-09-23T20:56:45.989908+0000","last_became_peered":"2023-09-23T20:56:45.989908+0000","last_unstale":"2023-09-23T20:59:36.567813+0000","last_undegraded":"2023-09-23T20:59:36.567813+0000","last_fullsized":"2023-09-23T20:59:36.567813+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":205,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:56:46.921739+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:56:46.921739+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:20:42.635110+0000","scrub_duration":0.0071954109999999996,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":231,"reported_epoch":241,"state":"active+clean","last_fresh":"2023-09-23T20:59:36.567160+0000","last_change":"2023-09-23T20:54:13.961788+0000","last_active":"2023-09-23T20:59:36.567160+0000","last_peered":"2023-09-23T20:59:36.567160+0000","last_clean":"2023-09-23T20:59:36.567160+0000","last_became_active":"2023-09-23T20:54:13.940937+0000","last_became_peered":"2023-09-23T20:54:13.940937+0000","last_unstale":"2023-09-23T20:59:36.567160+0000","last_undegraded":"2023-09-23T20:59:36.567160+0000","last_fullsized":"2023-09-23T20:59:36.567160+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:11:56.785299+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":136,"reported_epoch":241,"state":"active+clean","last_fresh":"2023-09-23T20:59:36.546530+0000","last_change":"2023-09-23T20:55:35.949670+0000","last_active":"2023-09-23T20:59:36.546530+0000","last_peered":"2023-09-23T20:59:36.546530+0000","last_clean":"2023-09-23T20:59:36.546530+0000","last_became_active":"2023-09-23T20:55:35.949140+0000","last_became_peered":"2023-09-23T20:55:35.949140+0000","last_unstale":"2023-09-23T20:59:36.546530+0000","last_undegraded":"2023-09-23T20:59:36.546530+0000","last_fullsized":"2023-09-23T20:59:36.546530+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:55:59.340303+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.e","version":"242'10050","reported_seq":10103,"reported_epoch":242,"state":"active+clean","last_fresh":"2023-09-23T20:59:46.442726+0000","last_change":"2023-09-23T20:56:35.547184+0000","last_active":"2023-09-23T20:59:46.442726+0000","last_peered":"2023-09-23T20:59:46.442726+0000","last_clean":"2023-09-23T20:59:46.442726+0000","last_became_active":"2023-09-23T20:56:35.546912+0000","last_became_peered":"2023-09-23T20:56:35.546912+0000","last_unstale":"2023-09-23T20:59:46.442726+0000","last_undegraded":"2023-09-23T20:59:46.442726+0000","last_fullsized":"2023-09-23T20:59:46.442726+0000","mapping_epoch":199,"log_start":"205'1200","ondisk_log_start":"205'1200","created":199,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_clean_scrub_stamp":"2023-09-23T20:56:34.529103+0000","objects_scrubbed":0,"log_size":8850,"log_dups_size":0,"ondisk_log_size":8850,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:57:07.422792+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":178913280,"num_objects":2730,"num_object_clones":0,"num_object_copies":8190,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2730,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10050,"num_write_kb":408960,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,0],"acting":[2,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.8","version":"79'161","reported_seq":485,"reported_epoch":241,"state":"active+clean","last_fresh":"2023-09-23T20:59:36.557088+0000","last_change":"2023-09-23T20:56:57.505737+0000","last_active":"2023-09-23T20:59:36.557088+0000","last_peered":"2023-09-23T20:59:36.557088+0000","last_clean":"2023-09-23T20:59:36.557088+0000","last_became_active":"2023-09-23T20:56:04.982345+0000","last_became_peered":"2023-09-23T20:56:04.982345+0000","last_unstale":"2023-09-23T20:59:36.557088+0000","last_undegraded":"2023-09-23T20:59:36.557088+0000","last_fullsized":"2023-09-23T20:59:36.557088+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:56:57.505663+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:56:57.505663+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:34:30.109724+0000","scrub_duration":0.026284108,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":204,"reported_epoch":241,"state":"active+clean","last_fresh":"2023-09-23T20:59:36.548114+0000","last_change":"2023-09-23T20:55:35.959942+0000","last_active":"2023-09-23T20:59:36.548114+0000","last_peered":"2023-09-23T20:59:36.548114+0000","last_clean":"2023-09-23T20:59:36.548114+0000","last_became_active":"2023-09-23T20:55:35.957071+0000","last_became_peered":"2023-09-23T20:55:35.957071+0000","last_unstale":"2023-09-23T20:59:36.548114+0000","last_undegraded":"2023-09-23T20:59:36.548114+0000","last_fullsized":"2023-09-23T20:59:36.548114+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:44:30.308036+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":119,"reported_epoch":241,"state":"active+clean","last_fresh":"2023-09-23T20:59:36.546963+0000","last_change":"2023-09-23T20:54:13.951459+0000","last_active":"2023-09-23T20:59:36.546963+0000","last_peered":"2023-09-23T20:59:36.546963+0000","last_clean":"2023-09-23T20:59:36.546963+0000","last_became_active":"2023-09-23T20:54:13.948409+0000","last_became_peered":"2023-09-23T20:54:13.948409+0000","last_unstale":"2023-09-23T20:59:36.546963+0000","last_undegraded":"2023-09-23T20:59:36.546963+0000","last_fullsized":"2023-09-23T20:59:36.546963+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:38:10.148061+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.b","version":"79'161","reported_seq":525,"reported_epoch":241,"state":"active+clean","last_fresh":"2023-09-23T20:59:36.545640+0000","last_change":"2023-09-23T20:57:27.320172+0000","last_active":"2023-09-23T20:59:36.545640+0000","last_peered":"2023-09-23T20:59:36.545640+0000","last_clean":"2023-09-23T20:59:36.545640+0000","last_became_active":"2023-09-23T20:57:16.009630+0000","last_became_peered":"2023-09-23T20:57:16.009630+0000","last_unstale":"2023-09-23T20:59:36.545640+0000","last_undegraded":"2023-09-23T20:59:36.545640+0000","last_fullsized":"2023-09-23T20:59:36.545640+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:27.320022+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:27.320022+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:31:35.221558+0000","scrub_duration":0.0043894679999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,5],"acting":[1,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.d","version":"241'9996","reported_seq":10048,"reported_epoch":241,"state":"active+clean","last_fresh":"2023-09-23T20:59:43.985900+0000","last_change":"2023-09-23T20:56:35.544132+0000","last_active":"2023-09-23T20:59:43.985900+0000","last_peered":"2023-09-23T20:59:43.985900+0000","last_clean":"2023-09-23T20:59:43.985900+0000","last_became_active":"2023-09-23T20:56:35.543836+0000","last_became_peered":"2023-09-23T20:56:35.543836+0000","last_unstale":"2023-09-23T20:59:43.985900+0000","last_undegraded":"2023-09-23T20:59:43.985900+0000","last_fullsized":"2023-09-23T20:59:43.985900+0000","mapping_epoch":199,"log_start":"206'2600","ondisk_log_start":"206'2600","created":199,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_clean_scrub_stamp":"2023-09-23T20:56:34.529103+0000","objects_scrubbed":0,"log_size":7396,"log_dups_size":0,"ondisk_log_size":7396,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:27:14.636991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":183042072,"num_objects":2794,"num_object_clones":0,"num_object_copies":8382,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2794,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":9996,"num_write_kb":409217,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":119,"reported_epoch":241,"state":"active+clean","last_fresh":"2023-09-23T20:59:36.552319+0000","last_change":"2023-09-23T20:54:13.950254+0000","last_active":"2023-09-23T20:59:36.552319+0000","last_peered":"2023-09-23T20:59:36.552319+0000","last_clean":"2023-09-23T20:59:36.552319+0000","last_became_active":"2023-09-23T20:54:13.949934+0000","last_became_peered":"2023-09-23T20:54:13.949934+0000","last_unstale":"2023-09-23T20:59:36.552319+0000","last_undegraded":"2023-09-23T20:59:36.552319+0000","last_fullsized":"2023-09-23T20:59:36.552319+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:30:32.847167+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":136,"reported_epoch":241,"state":"active+clean","last_fresh":"2023-09-23T20:59:36.546628+0000","last_change":"2023-09-23T20:55:35.956153+0000","last_active":"2023-09-23T20:59:36.546628+0000","last_peered":"2023-09-23T20:59:36.546628+0000","last_clean":"2023-09-23T20:59:36.546628+0000","last_became_active":"2023-09-23T20:55:35.955630+0000","last_became_peered":"2023-09-23T20:55:35.955630+0000","last_unstale":"2023-09-23T20:59:36.546628+0000","last_undegraded":"2023-09-23T20:59:36.546628+0000","last_fullsized":"2023-09-23T20:59:36.546628+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:45:20.282929+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.c","version":"242'9919","reported_seq":9980,"reported_epoch":242,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T20:59:44.074660+0000","last_change":"2023-09-23T20:57:46.463005+0000","last_active":"2023-09-23T20:59:44.074660+0000","last_peered":"2023-09-23T20:59:44.074660+0000","last_clean":"2023-09-23T20:57:44.015119+0000","last_became_active":"2023-09-23T20:57:46.448702+0000","last_became_peered":"2023-09-23T20:57:46.448702+0000","last_unstale":"2023-09-23T20:59:44.074660+0000","last_undegraded":"2023-09-23T20:57:46.010874+0000","last_fullsized":"2023-09-23T20:57:45.978434+0000","mapping_epoch":223,"log_start":"206'2600","ondisk_log_start":"206'2600","created":199,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_clean_scrub_stamp":"2023-09-23T20:56:34.529103+0000","objects_scrubbed":0,"log_size":7319,"log_dups_size":0,"ondisk_log_size":7319,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:08:34.180651+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":189333504,"num_objects":2889,"num_object_clones":0,"num_object_copies":8667,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5473,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2889,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9919,"num_write_kb":409856,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,7],"avail_no_missing":["6","2","7"],"object_location_counts":[{"shards":"2,6,7","objects":1958},{"shards":"6,7","objects":-2584}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.a","version":"79'161","reported_seq":485,"reported_epoch":241,"state":"active+clean","last_fresh":"2023-09-23T20:59:36.556299+0000","last_change":"2023-09-23T20:56:44.487778+0000","last_active":"2023-09-23T20:59:36.556299+0000","last_peered":"2023-09-23T20:59:36.556299+0000","last_clean":"2023-09-23T20:59:36.556299+0000","last_became_active":"2023-09-23T20:56:05.006740+0000","last_became_peered":"2023-09-23T20:56:05.006740+0000","last_unstale":"2023-09-23T20:59:36.556299+0000","last_undegraded":"2023-09-23T20:59:36.556299+0000","last_fullsized":"2023-09-23T20:59:36.556299+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:56:44.487678+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:56:44.487678+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:04:50.086749+0000","scrub_duration":0.0017408899999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":30,"reported_epoch":241,"state":"active+clean","last_fresh":"2023-09-23T20:59:36.566771+0000","last_change":"2023-09-23T20:57:44.982513+0000","last_active":"2023-09-23T20:59:36.566771+0000","last_peered":"2023-09-23T20:59:36.566771+0000","last_clean":"2023-09-23T20:59:36.566771+0000","last_became_active":"2023-09-23T20:57:44.982112+0000","last_became_peered":"2023-09-23T20:57:44.982112+0000","last_unstale":"2023-09-23T20:59:36.566771+0000","last_undegraded":"2023-09-23T20:59:36.566771+0000","last_fullsized":"2023-09-23T20:59:36.566771+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:09:05.630041+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":212,"reported_epoch":241,"state":"active+clean","last_fresh":"2023-09-23T20:59:36.545345+0000","last_change":"2023-09-23T20:54:13.940748+0000","last_active":"2023-09-23T20:59:36.545345+0000","last_peered":"2023-09-23T20:59:36.545345+0000","last_clean":"2023-09-23T20:59:36.545345+0000","last_became_active":"2023-09-23T20:54:13.940419+0000","last_became_peered":"2023-09-23T20:54:13.940419+0000","last_unstale":"2023-09-23T20:59:36.545345+0000","last_undegraded":"2023-09-23T20:59:36.545345+0000","last_fullsized":"2023-09-23T20:59:36.545345+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:15:01.923816+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.1","version":"242'10026","reported_seq":10090,"reported_epoch":242,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T20:59:47.094470+0000","last_change":"2023-09-23T20:57:46.483248+0000","last_active":"2023-09-23T20:59:47.094470+0000","last_peered":"2023-09-23T20:59:47.094470+0000","last_clean":"2023-09-23T20:57:44.047957+0000","last_became_active":"2023-09-23T20:57:46.475568+0000","last_became_peered":"2023-09-23T20:57:46.475568+0000","last_unstale":"2023-09-23T20:59:47.094470+0000","last_undegraded":"2023-09-23T20:57:46.083591+0000","last_fullsized":"2023-09-23T20:57:45.978436+0000","mapping_epoch":223,"log_start":"210'3300","ondisk_log_start":"210'3300","created":199,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_clean_scrub_stamp":"2023-09-23T20:56:34.529103+0000","objects_scrubbed":0,"log_size":6726,"log_dups_size":0,"ondisk_log_size":6726,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:22:45.067269+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":180879360,"num_objects":2760,"num_object_clones":0,"num_object_copies":8280,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5470,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2760,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10026,"num_write_kb":409152,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,6],"avail_no_missing":["4","1","6"],"object_location_counts":[{"shards":"1,4,6","objects":1837},{"shards":"4,6","objects":-2710}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.7","version":"79'161","reported_seq":547,"reported_epoch":241,"state":"active+clean","last_fresh":"2023-09-23T20:59:36.548362+0000","last_change":"2023-09-23T20:57:48.243572+0000","last_active":"2023-09-23T20:59:36.548362+0000","last_peered":"2023-09-23T20:59:36.548362+0000","last_clean":"2023-09-23T20:59:36.548362+0000","last_became_active":"2023-09-23T20:57:48.243229+0000","last_became_peered":"2023-09-23T20:57:48.243229+0000","last_unstale":"2023-09-23T20:59:36.548362+0000","last_undegraded":"2023-09-23T20:59:36.548362+0000","last_fullsized":"2023-09-23T20:59:36.548362+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":3,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:25.916867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:25.916867+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:23:19.385831+0000","scrub_duration":0.001665323,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":217,"reported_epoch":241,"state":"active+clean","last_fresh":"2023-09-23T20:59:36.544022+0000","last_change":"2023-09-23T20:50:05.867130+0000","last_active":"2023-09-23T20:59:36.544022+0000","last_peered":"2023-09-23T20:59:36.544022+0000","last_clean":"2023-09-23T20:59:36.544022+0000","last_became_active":"2023-09-23T20:50:05.866758+0000","last_became_peered":"2023-09-23T20:50:05.866758+0000","last_unstale":"2023-09-23T20:59:36.544022+0000","last_undegraded":"2023-09-23T20:59:36.544022+0000","last_fullsized":"2023-09-23T20:59:36.544022+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:55:07.548209+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":115,"reported_epoch":241,"state":"active+clean","last_fresh":"2023-09-23T20:59:36.556816+0000","last_change":"2023-09-23T20:54:31.887604+0000","last_active":"2023-09-23T20:59:36.556816+0000","last_peered":"2023-09-23T20:59:36.556816+0000","last_clean":"2023-09-23T20:59:36.556816+0000","last_became_active":"2023-09-23T20:54:31.886752+0000","last_became_peered":"2023-09-23T20:54:31.886752+0000","last_unstale":"2023-09-23T20:59:36.556816+0000","last_undegraded":"2023-09-23T20:59:36.556816+0000","last_fullsized":"2023-09-23T20:59:36.556816+0000","mapping_epoch":136,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:13:58.872176+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":215,"reported_epoch":241,"state":"active+clean","last_fresh":"2023-09-23T20:59:36.544261+0000","last_change":"2023-09-23T20:54:31.899082+0000","last_active":"2023-09-23T20:59:36.544261+0000","last_peered":"2023-09-23T20:59:36.544261+0000","last_clean":"2023-09-23T20:59:36.544261+0000","last_became_active":"2023-09-23T20:54:31.898521+0000","last_became_peered":"2023-09-23T20:54:31.898521+0000","last_unstale":"2023-09-23T20:59:36.544261+0000","last_undegraded":"2023-09-23T20:59:36.544261+0000","last_fullsized":"2023-09-23T20:59:36.544261+0000","mapping_epoch":136,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:25:36.059711+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.7","version":"241'9986","reported_seq":10037,"reported_epoch":242,"state":"active+clean","last_fresh":"2023-09-23T20:59:44.080265+0000","last_change":"2023-09-23T20:56:35.541695+0000","last_active":"2023-09-23T20:59:44.080265+0000","last_peered":"2023-09-23T20:59:44.080265+0000","last_clean":"2023-09-23T20:59:44.080265+0000","last_became_active":"2023-09-23T20:56:35.541392+0000","last_became_peered":"2023-09-23T20:56:35.541392+0000","last_unstale":"2023-09-23T20:59:44.080265+0000","last_undegraded":"2023-09-23T20:59:44.080265+0000","last_fullsized":"2023-09-23T20:59:44.080265+0000","mapping_epoch":199,"log_start":"206'2600","ondisk_log_start":"206'2600","created":199,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_clean_scrub_stamp":"2023-09-23T20:56:34.529103+0000","objects_scrubbed":0,"log_size":7386,"log_dups_size":0,"ondisk_log_size":7386,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:57:22.478253+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":185073664,"num_objects":2824,"num_object_clones":0,"num_object_copies":8472,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2824,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9986,"num_write_kb":409920,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.1","version":"79'161","reported_seq":505,"reported_epoch":241,"state":"active+clean","last_fresh":"2023-09-23T20:59:36.555696+0000","last_change":"2023-09-23T20:57:19.669346+0000","last_active":"2023-09-23T20:59:36.555696+0000","last_peered":"2023-09-23T20:59:36.555696+0000","last_clean":"2023-09-23T20:59:36.555696+0000","last_became_active":"2023-09-23T20:57:08.246046+0000","last_became_peered":"2023-09-23T20:57:08.246046+0000","last_unstale":"2023-09-23T20:59:36.555696+0000","last_undegraded":"2023-09-23T20:59:36.555696+0000","last_fullsized":"2023-09-23T20:59:36.555696+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:19.669259+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:19.669259+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:14:03.435531+0000","scrub_duration":0.0045759559999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":132,"reported_epoch":241,"state":"active+clean","last_fresh":"2023-09-23T20:59:36.556044+0000","last_change":"2023-09-23T20:55:35.959514+0000","last_active":"2023-09-23T20:59:36.556044+0000","last_peered":"2023-09-23T20:59:36.556044+0000","last_clean":"2023-09-23T20:59:36.556044+0000","last_became_active":"2023-09-23T20:55:35.959244+0000","last_became_peered":"2023-09-23T20:55:35.959244+0000","last_unstale":"2023-09-23T20:59:36.556044+0000","last_undegraded":"2023-09-23T20:59:36.556044+0000","last_fullsized":"2023-09-23T20:59:36.556044+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:47:32.665316+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":136,"reported_epoch":241,"state":"active+clean","last_fresh":"2023-09-23T20:59:36.546761+0000","last_change":"2023-09-23T20:55:35.949000+0000","last_active":"2023-09-23T20:59:36.546761+0000","last_peered":"2023-09-23T20:59:36.546761+0000","last_clean":"2023-09-23T20:59:36.546761+0000","last_became_active":"2023-09-23T20:55:35.948410+0000","last_became_peered":"2023-09-23T20:55:35.948410+0000","last_unstale":"2023-09-23T20:59:36.546761+0000","last_undegraded":"2023-09-23T20:59:36.546761+0000","last_fullsized":"2023-09-23T20:59:36.546761+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:26:50.477289+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":209,"reported_epoch":241,"state":"active+clean","last_fresh":"2023-09-23T20:59:36.543793+0000","last_change":"2023-09-23T20:51:10.829889+0000","last_active":"2023-09-23T20:59:36.543793+0000","last_peered":"2023-09-23T20:59:36.543793+0000","last_clean":"2023-09-23T20:59:36.543793+0000","last_became_active":"2023-09-23T20:51:10.829533+0000","last_became_peered":"2023-09-23T20:51:10.829533+0000","last_unstale":"2023-09-23T20:59:36.543793+0000","last_undegraded":"2023-09-23T20:59:36.543793+0000","last_fullsized":"2023-09-23T20:59:36.543793+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:43:04.603460+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.4","version":"242'9973","reported_seq":10026,"reported_epoch":242,"state":"active+clean","last_fresh":"2023-09-23T20:59:46.693016+0000","last_change":"2023-09-23T20:56:35.549909+0000","last_active":"2023-09-23T20:59:46.693016+0000","last_peered":"2023-09-23T20:59:46.693016+0000","last_clean":"2023-09-23T20:59:46.693016+0000","last_became_active":"2023-09-23T20:56:35.549618+0000","last_became_peered":"2023-09-23T20:56:35.549618+0000","last_unstale":"2023-09-23T20:59:46.693016+0000","last_undegraded":"2023-09-23T20:59:46.693016+0000","last_fullsized":"2023-09-23T20:59:46.693016+0000","mapping_epoch":199,"log_start":"206'2600","ondisk_log_start":"206'2600","created":199,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_clean_scrub_stamp":"2023-09-23T20:56:34.529103+0000","objects_scrubbed":0,"log_size":7373,"log_dups_size":0,"ondisk_log_size":7373,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:24:25.918475+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":180158464,"num_objects":2749,"num_object_clones":0,"num_object_copies":8247,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2749,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9973,"num_write_kb":407104,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.2","version":"79'161","reported_seq":472,"reported_epoch":241,"state":"active+clean","last_fresh":"2023-09-23T20:59:36.546548+0000","last_change":"2023-09-23T20:57:24.802586+0000","last_active":"2023-09-23T20:59:36.546548+0000","last_peered":"2023-09-23T20:59:36.546548+0000","last_clean":"2023-09-23T20:59:36.546548+0000","last_became_active":"2023-09-23T20:57:08.236114+0000","last_became_peered":"2023-09-23T20:57:08.236114+0000","last_unstale":"2023-09-23T20:59:36.546548+0000","last_undegraded":"2023-09-23T20:59:36.546548+0000","last_fullsized":"2023-09-23T20:59:36.546548+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:24.802451+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:24.802451+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:06:22.309980+0000","scrub_duration":0.0090861250000000005,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,4],"acting":[0,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":228,"reported_epoch":241,"state":"active+clean","last_fresh":"2023-09-23T20:59:36.570723+0000","last_change":"2023-09-23T20:51:10.838054+0000","last_active":"2023-09-23T20:59:36.570723+0000","last_peered":"2023-09-23T20:59:36.570723+0000","last_clean":"2023-09-23T20:59:36.570723+0000","last_became_active":"2023-09-23T20:51:10.837750+0000","last_became_peered":"2023-09-23T20:51:10.837750+0000","last_unstale":"2023-09-23T20:59:36.570723+0000","last_undegraded":"2023-09-23T20:59:36.570723+0000","last_fullsized":"2023-09-23T20:59:36.570723+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:15:51.648085+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":209,"reported_epoch":241,"state":"active+clean","last_fresh":"2023-09-23T20:59:36.543689+0000","last_change":"2023-09-23T20:55:35.951146+0000","last_active":"2023-09-23T20:59:36.543689+0000","last_peered":"2023-09-23T20:59:36.543689+0000","last_clean":"2023-09-23T20:59:36.543689+0000","last_became_active":"2023-09-23T20:55:35.950874+0000","last_became_peered":"2023-09-23T20:55:35.950874+0000","last_unstale":"2023-09-23T20:59:36.543689+0000","last_undegraded":"2023-09-23T20:59:36.543689+0000","last_fullsized":"2023-09-23T20:59:36.543689+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:58:13.957677+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":230,"reported_epoch":241,"state":"active+clean","last_fresh":"2023-09-23T20:59:36.545248+0000","last_change":"2023-09-23T20:54:31.898870+0000","last_active":"2023-09-23T20:59:36.545248+0000","last_peered":"2023-09-23T20:59:36.545248+0000","last_clean":"2023-09-23T20:59:36.545248+0000","last_became_active":"2023-09-23T20:54:31.898467+0000","last_became_peered":"2023-09-23T20:54:31.898467+0000","last_unstale":"2023-09-23T20:59:36.545248+0000","last_undegraded":"2023-09-23T20:59:36.545248+0000","last_fullsized":"2023-09-23T20:59:36.545248+0000","mapping_epoch":136,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:56:22.483408+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.6","version":"242'9854","reported_seq":15512,"reported_epoch":242,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-09-23T20:59:46.739880+0000","last_change":"2023-09-23T20:58:39.453197+0000","last_active":"2023-09-23T20:59:46.739880+0000","last_peered":"2023-09-23T20:59:46.739880+0000","last_clean":"2023-09-23T20:57:44.031597+0000","last_became_active":"2023-09-23T20:57:46.443927+0000","last_became_peered":"2023-09-23T20:57:46.443927+0000","last_unstale":"2023-09-23T20:59:46.739880+0000","last_undegraded":"2023-09-23T20:58:39.450464+0000","last_fullsized":"2023-09-23T20:57:45.977227+0000","mapping_epoch":223,"log_start":"206'2500","ondisk_log_start":"206'2500","created":199,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_clean_scrub_stamp":"2023-09-23T20:56:34.529103+0000","objects_scrubbed":0,"log_size":7354,"log_dups_size":0,"ondisk_log_size":7354,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:37:23.911131+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":179175424,"num_objects":2734,"num_object_clones":0,"num_object_copies":8202,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":771,"num_objects_misplaced":1828,"num_objects_unfound":0,"num_objects_dirty":2734,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9854,"num_write_kb":402816,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2799,"num_bytes_recovered":183435264,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7],"avail_no_missing":["0","5","7"],"object_location_counts":[{"shards":"0,5,7","objects":1828},{"shards":"0,7","objects":135}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":115,"reported_epoch":241,"state":"active+clean","last_fresh":"2023-09-23T20:59:36.557393+0000","last_change":"2023-09-23T20:54:31.883504+0000","last_active":"2023-09-23T20:59:36.557393+0000","last_peered":"2023-09-23T20:59:36.557393+0000","last_clean":"2023-09-23T20:59:36.557393+0000","last_became_active":"2023-09-23T20:54:31.883172+0000","last_became_peered":"2023-09-23T20:54:31.883172+0000","last_unstale":"2023-09-23T20:59:36.557393+0000","last_undegraded":"2023-09-23T20:59:36.557393+0000","last_fullsized":"2023-09-23T20:59:36.557393+0000","mapping_epoch":136,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:45:40.289120+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"79'161","reported_seq":472,"reported_epoch":241,"state":"active+clean","last_fresh":"2023-09-23T20:59:36.546207+0000","last_change":"2023-09-23T20:57:20.738417+0000","last_active":"2023-09-23T20:59:36.546207+0000","last_peered":"2023-09-23T20:59:36.546207+0000","last_clean":"2023-09-23T20:59:36.546207+0000","last_became_active":"2023-09-23T20:57:08.236842+0000","last_became_peered":"2023-09-23T20:57:08.236842+0000","last_unstale":"2023-09-23T20:59:36.546207+0000","last_undegraded":"2023-09-23T20:59:36.546207+0000","last_fullsized":"2023-09-23T20:59:36.546207+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":0,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:20.738273+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:20.738273+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:18:28.327990+0000","scrub_duration":0.0088336339999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":220,"reported_epoch":241,"state":"active+clean","last_fresh":"2023-09-23T20:59:36.567290+0000","last_change":"2023-09-23T20:55:35.950777+0000","last_active":"2023-09-23T20:59:36.567290+0000","last_peered":"2023-09-23T20:59:36.567290+0000","last_clean":"2023-09-23T20:59:36.567290+0000","last_became_active":"2023-09-23T20:55:35.950427+0000","last_became_peered":"2023-09-23T20:55:35.950427+0000","last_unstale":"2023-09-23T20:59:36.567290+0000","last_undegraded":"2023-09-23T20:59:36.567290+0000","last_fullsized":"2023-09-23T20:59:36.567290+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:20:10.371751+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":186,"reported_epoch":241,"state":"active+clean","last_fresh":"2023-09-23T20:59:36.548388+0000","last_change":"2023-09-23T20:55:35.978140+0000","last_active":"2023-09-23T20:59:36.548388+0000","last_peered":"2023-09-23T20:59:36.548388+0000","last_clean":"2023-09-23T20:59:36.548388+0000","last_became_active":"2023-09-23T20:55:35.977831+0000","last_became_peered":"2023-09-23T20:55:35.977831+0000","last_unstale":"2023-09-23T20:59:36.548388+0000","last_undegraded":"2023-09-23T20:59:36.548388+0000","last_fullsized":"2023-09-23T20:59:36.548388+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:25:47.678023+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.5","version":"242'9988","reported_seq":10048,"reported_epoch":242,"state":"active+clean","last_fresh":"2023-09-23T20:59:46.379497+0000","last_change":"2023-09-23T20:56:45.990110+0000","last_active":"2023-09-23T20:59:46.379497+0000","last_peered":"2023-09-23T20:59:46.379497+0000","last_clean":"2023-09-23T20:59:46.379497+0000","last_became_active":"2023-09-23T20:56:45.988394+0000","last_became_peered":"2023-09-23T20:56:45.988394+0000","last_unstale":"2023-09-23T20:59:46.379497+0000","last_undegraded":"2023-09-23T20:59:46.379497+0000","last_fullsized":"2023-09-23T20:59:46.379497+0000","mapping_epoch":204,"log_start":"205'1100","ondisk_log_start":"205'1100","created":199,"last_epoch_clean":205,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_clean_scrub_stamp":"2023-09-23T20:56:34.529103+0000","objects_scrubbed":0,"log_size":8888,"log_dups_size":0,"ondisk_log_size":8888,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:22:03.814210+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":176291840,"num_objects":2690,"num_object_clones":0,"num_object_copies":8070,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2690,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9988,"num_write_kb":405696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,0],"acting":[2,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"79'161","reported_seq":508,"reported_epoch":241,"state":"active+clean","last_fresh":"2023-09-23T20:59:36.545780+0000","last_change":"2023-09-23T20:57:20.378531+0000","last_active":"2023-09-23T20:59:36.545780+0000","last_peered":"2023-09-23T20:59:36.545780+0000","last_clean":"2023-09-23T20:59:36.545780+0000","last_became_active":"2023-09-23T20:57:08.238019+0000","last_became_peered":"2023-09-23T20:57:08.238019+0000","last_unstale":"2023-09-23T20:59:36.545780+0000","last_undegraded":"2023-09-23T20:59:36.545780+0000","last_fullsized":"2023-09-23T20:59:36.545780+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":3,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:20.377714+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:20.377714+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:00:08.305602+0000","scrub_duration":0.003015228,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":111,"reported_epoch":241,"state":"active+clean","last_fresh":"2023-09-23T20:59:36.543829+0000","last_change":"2023-09-23T20:56:45.991884+0000","last_active":"2023-09-23T20:59:36.543829+0000","last_peered":"2023-09-23T20:59:36.543829+0000","last_clean":"2023-09-23T20:59:36.543829+0000","last_became_active":"2023-09-23T20:56:45.991568+0000","last_became_peered":"2023-09-23T20:56:45.991568+0000","last_unstale":"2023-09-23T20:59:36.543829+0000","last_undegraded":"2023-09-23T20:59:36.543829+0000","last_fullsized":"2023-09-23T20:59:36.543829+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":205,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:21:20.775511+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,0],"acting":[1,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":235,"reported_epoch":241,"state":"active+clean","last_fresh":"2023-09-23T20:59:36.547245+0000","last_change":"2023-09-23T20:57:44.979950+0000","last_active":"2023-09-23T20:59:36.547245+0000","last_peered":"2023-09-23T20:59:36.547245+0000","last_clean":"2023-09-23T20:59:36.547245+0000","last_became_active":"2023-09-23T20:57:44.979346+0000","last_became_peered":"2023-09-23T20:57:44.979346+0000","last_unstale":"2023-09-23T20:59:36.547245+0000","last_undegraded":"2023-09-23T20:59:36.547245+0000","last_fullsized":"2023-09-23T20:59:36.547245+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:10:24.372800+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":214,"reported_epoch":241,"state":"active+clean","last_fresh":"2023-09-23T20:59:36.548527+0000","last_change":"2023-09-23T20:50:37.982235+0000","last_active":"2023-09-23T20:59:36.548527+0000","last_peered":"2023-09-23T20:59:36.548527+0000","last_clean":"2023-09-23T20:59:36.548527+0000","last_became_active":"2023-09-23T20:50:37.981913+0000","last_became_peered":"2023-09-23T20:50:37.981913+0000","last_unstale":"2023-09-23T20:59:36.548527+0000","last_undegraded":"2023-09-23T20:59:36.548527+0000","last_fullsized":"2023-09-23T20:59:36.548527+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:59:57.122584+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.0","version":"242'10012","reported_seq":10063,"reported_epoch":242,"state":"active+clean","last_fresh":"2023-09-23T20:59:47.087685+0000","last_change":"2023-09-23T20:56:35.550635+0000","last_active":"2023-09-23T20:59:47.087685+0000","last_peered":"2023-09-23T20:59:47.087685+0000","last_clean":"2023-09-23T20:59:47.087685+0000","last_became_active":"2023-09-23T20:56:35.550376+0000","last_became_peered":"2023-09-23T20:56:35.550376+0000","last_unstale":"2023-09-23T20:59:47.087685+0000","last_undegraded":"2023-09-23T20:59:47.087685+0000","last_fullsized":"2023-09-23T20:59:47.087685+0000","mapping_epoch":199,"log_start":"205'1400","ondisk_log_start":"205'1400","created":199,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_clean_scrub_stamp":"2023-09-23T20:56:34.529103+0000","objects_scrubbed":0,"log_size":8612,"log_dups_size":0,"ondisk_log_size":8612,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:40:55.020049+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":174981120,"num_objects":2670,"num_object_clones":0,"num_object_copies":8010,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2670,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10012,"num_write_kb":405824,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.6","version":"79'161","reported_seq":524,"reported_epoch":241,"state":"active+clean","last_fresh":"2023-09-23T20:59:36.548096+0000","last_change":"2023-09-23T20:57:26.907505+0000","last_active":"2023-09-23T20:59:36.548096+0000","last_peered":"2023-09-23T20:59:36.548096+0000","last_clean":"2023-09-23T20:59:36.548096+0000","last_became_active":"2023-09-23T20:57:08.240321+0000","last_became_peered":"2023-09-23T20:57:08.240321+0000","last_unstale":"2023-09-23T20:59:36.548096+0000","last_undegraded":"2023-09-23T20:59:36.548096+0000","last_fullsized":"2023-09-23T20:59:36.548096+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":3,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:26.907408+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:26.907408+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:34:17.392928+0000","scrub_duration":0.010337038,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":158,"reported_epoch":241,"state":"active+clean","last_fresh":"2023-09-23T20:59:36.548093+0000","last_change":"2023-09-23T20:56:45.988566+0000","last_active":"2023-09-23T20:59:36.548093+0000","last_peered":"2023-09-23T20:59:36.548093+0000","last_clean":"2023-09-23T20:59:36.548093+0000","last_became_active":"2023-09-23T20:56:45.988249+0000","last_became_peered":"2023-09-23T20:56:45.988249+0000","last_unstale":"2023-09-23T20:59:36.548093+0000","last_undegraded":"2023-09-23T20:59:36.548093+0000","last_fullsized":"2023-09-23T20:59:36.548093+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":205,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:51:36.644764+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,4],"acting":[0,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":212,"reported_epoch":241,"state":"active+clean","last_fresh":"2023-09-23T20:59:36.543948+0000","last_change":"2023-09-23T20:54:31.898965+0000","last_active":"2023-09-23T20:59:36.543948+0000","last_peered":"2023-09-23T20:59:36.543948+0000","last_clean":"2023-09-23T20:59:36.543948+0000","last_became_active":"2023-09-23T20:54:31.898663+0000","last_became_peered":"2023-09-23T20:54:31.898663+0000","last_unstale":"2023-09-23T20:59:36.543948+0000","last_undegraded":"2023-09-23T20:59:36.543948+0000","last_fullsized":"2023-09-23T20:59:36.543948+0000","mapping_epoch":136,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:53:05.058087+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":202,"reported_epoch":241,"state":"active+clean","last_fresh":"2023-09-23T20:59:36.547635+0000","last_change":"2023-09-23T20:54:31.884717+0000","last_active":"2023-09-23T20:59:36.547635+0000","last_peered":"2023-09-23T20:59:36.547635+0000","last_clean":"2023-09-23T20:59:36.547635+0000","last_became_active":"2023-09-23T20:54:31.884345+0000","last_became_peered":"2023-09-23T20:54:31.884345+0000","last_unstale":"2023-09-23T20:59:36.547635+0000","last_undegraded":"2023-09-23T20:59:36.547635+0000","last_fullsized":"2023-09-23T20:59:36.547635+0000","mapping_epoch":136,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:02:26.393868+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.3","version":"242'9736","reported_seq":9789,"reported_epoch":242,"state":"active+clean","last_fresh":"2023-09-23T20:59:44.534639+0000","last_change":"2023-09-23T20:56:35.549655+0000","last_active":"2023-09-23T20:59:44.534639+0000","last_peered":"2023-09-23T20:59:44.534639+0000","last_clean":"2023-09-23T20:59:44.534639+0000","last_became_active":"2023-09-23T20:56:35.549357+0000","last_became_peered":"2023-09-23T20:56:35.549357+0000","last_unstale":"2023-09-23T20:59:44.534639+0000","last_undegraded":"2023-09-23T20:59:44.534639+0000","last_fullsized":"2023-09-23T20:59:44.534639+0000","mapping_epoch":199,"log_start":"213'3600","ondisk_log_start":"213'3600","created":199,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_clean_scrub_stamp":"2023-09-23T20:56:34.529103+0000","objects_scrubbed":0,"log_size":6136,"log_dups_size":0,"ondisk_log_size":6136,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:37:58.752518+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":182583296,"num_objects":2786,"num_object_clones":0,"num_object_copies":8358,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2786,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9736,"num_write_kb":400704,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.5","version":"79'161","reported_seq":505,"reported_epoch":241,"state":"active+clean","last_fresh":"2023-09-23T20:59:36.546473+0000","last_change":"2023-09-23T20:57:30.029479+0000","last_active":"2023-09-23T20:59:36.546473+0000","last_peered":"2023-09-23T20:59:36.546473+0000","last_clean":"2023-09-23T20:59:36.546473+0000","last_became_active":"2023-09-23T20:57:08.249909+0000","last_became_peered":"2023-09-23T20:57:08.249909+0000","last_unstale":"2023-09-23T20:59:36.546473+0000","last_undegraded":"2023-09-23T20:59:36.546473+0000","last_fullsized":"2023-09-23T20:59:36.546473+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":3,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:30.029396+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:30.029396+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:16:46.083145+0000","scrub_duration":0.012914682,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":205,"reported_epoch":241,"state":"active+clean","last_fresh":"2023-09-23T20:59:36.575122+0000","last_change":"2023-09-23T20:51:10.836910+0000","last_active":"2023-09-23T20:59:36.575122+0000","last_peered":"2023-09-23T20:59:36.575122+0000","last_clean":"2023-09-23T20:59:36.575122+0000","last_became_active":"2023-09-23T20:51:10.836557+0000","last_became_peered":"2023-09-23T20:51:10.836557+0000","last_unstale":"2023-09-23T20:59:36.575122+0000","last_undegraded":"2023-09-23T20:59:36.575122+0000","last_fullsized":"2023-09-23T20:59:36.575122+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:51:54.019969+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":212,"reported_epoch":241,"state":"active+clean","last_fresh":"2023-09-23T20:59:36.543013+0000","last_change":"2023-09-23T20:55:35.953444+0000","last_active":"2023-09-23T20:59:36.543013+0000","last_peered":"2023-09-23T20:59:36.543013+0000","last_clean":"2023-09-23T20:59:36.543013+0000","last_became_active":"2023-09-23T20:55:35.953151+0000","last_became_peered":"2023-09-23T20:55:35.953151+0000","last_unstale":"2023-09-23T20:59:36.543013+0000","last_undegraded":"2023-09-23T20:59:36.543013+0000","last_fullsized":"2023-09-23T20:59:36.543013+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:39:12.404391+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":31,"reported_epoch":242,"state":"active+clean","last_fresh":"2023-09-23T20:59:44.092647+0000","last_change":"2023-09-23T20:57:44.982509+0000","last_active":"2023-09-23T20:59:44.092647+0000","last_peered":"2023-09-23T20:59:44.092647+0000","last_clean":"2023-09-23T20:59:44.092647+0000","last_became_active":"2023-09-23T20:57:44.982175+0000","last_became_peered":"2023-09-23T20:57:44.982175+0000","last_unstale":"2023-09-23T20:59:44.092647+0000","last_undegraded":"2023-09-23T20:59:44.092647+0000","last_fullsized":"2023-09-23T20:59:44.092647+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:37:49.215488+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.2","version":"242'9799","reported_seq":9852,"reported_epoch":242,"state":"active+clean","last_fresh":"2023-09-23T20:59:44.589680+0000","last_change":"2023-09-23T20:56:35.543020+0000","last_active":"2023-09-23T20:59:44.589680+0000","last_peered":"2023-09-23T20:59:44.589680+0000","last_clean":"2023-09-23T20:59:44.589680+0000","last_became_active":"2023-09-23T20:56:35.542719+0000","last_became_peered":"2023-09-23T20:56:35.542719+0000","last_unstale":"2023-09-23T20:59:44.589680+0000","last_undegraded":"2023-09-23T20:59:44.589680+0000","last_fullsized":"2023-09-23T20:59:44.589680+0000","mapping_epoch":199,"log_start":"212'3600","ondisk_log_start":"212'3600","created":199,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_clean_scrub_stamp":"2023-09-23T20:56:34.529103+0000","objects_scrubbed":0,"log_size":6199,"log_dups_size":0,"ondisk_log_size":6199,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:52:21.088628+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":177799168,"num_objects":2713,"num_object_clones":0,"num_object_copies":8139,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2713,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9799,"num_write_kb":400384,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.4","version":"79'161","reported_seq":520,"reported_epoch":241,"state":"active+clean","last_fresh":"2023-09-23T20:59:36.568210+0000","last_change":"2023-09-23T20:57:20.012904+0000","last_active":"2023-09-23T20:59:36.568210+0000","last_peered":"2023-09-23T20:59:36.568210+0000","last_clean":"2023-09-23T20:59:36.568210+0000","last_became_active":"2023-09-23T20:57:08.237996+0000","last_became_peered":"2023-09-23T20:57:08.237996+0000","last_unstale":"2023-09-23T20:59:36.568210+0000","last_undegraded":"2023-09-23T20:59:36.568210+0000","last_fullsized":"2023-09-23T20:59:36.568210+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":3,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:20.012764+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:20.012764+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:51:28.796946+0000","scrub_duration":0.0045251919999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":191,"reported_epoch":241,"state":"active+clean","last_fresh":"2023-09-23T20:59:36.548192+0000","last_change":"2023-09-23T20:55:35.949541+0000","last_active":"2023-09-23T20:59:36.548192+0000","last_peered":"2023-09-23T20:59:36.548192+0000","last_clean":"2023-09-23T20:59:36.548192+0000","last_became_active":"2023-09-23T20:55:35.944635+0000","last_became_peered":"2023-09-23T20:55:35.944635+0000","last_unstale":"2023-09-23T20:59:36.548192+0000","last_undegraded":"2023-09-23T20:59:36.548192+0000","last_fullsized":"2023-09-23T20:59:36.548192+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:30:36.033585+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"70'1","reported_seq":230,"reported_epoch":241,"state":"active+clean","last_fresh":"2023-09-23T20:59:36.548620+0000","last_change":"2023-09-23T20:54:14.009723+0000","last_active":"2023-09-23T20:59:36.548620+0000","last_peered":"2023-09-23T20:59:36.548620+0000","last_clean":"2023-09-23T20:59:36.548620+0000","last_became_active":"2023-09-23T20:54:13.941771+0000","last_became_peered":"2023-09-23T20:54:13.941771+0000","last_unstale":"2023-09-23T20:59:36.548620+0000","last_undegraded":"2023-09-23T20:59:36.548620+0000","last_fullsized":"2023-09-23T20:59:36.548620+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:48:25.588116+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":146,"reported_epoch":241,"state":"active+clean","last_fresh":"2023-09-23T20:59:36.552389+0000","last_change":"2023-09-23T20:55:35.949780+0000","last_active":"2023-09-23T20:59:36.552389+0000","last_peered":"2023-09-23T20:59:36.552389+0000","last_clean":"2023-09-23T20:59:36.552389+0000","last_became_active":"2023-09-23T20:55:35.949497+0000","last_became_peered":"2023-09-23T20:55:35.949497+0000","last_unstale":"2023-09-23T20:59:36.552389+0000","last_undegraded":"2023-09-23T20:59:36.552389+0000","last_fullsized":"2023-09-23T20:59:36.552389+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:09:37.420882+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":205,"reported_epoch":241,"state":"active+clean","last_fresh":"2023-09-23T20:59:36.547525+0000","last_change":"2023-09-23T20:54:13.929196+0000","last_active":"2023-09-23T20:59:36.547525+0000","last_peered":"2023-09-23T20:59:36.547525+0000","last_clean":"2023-09-23T20:59:36.547525+0000","last_became_active":"2023-09-23T20:54:13.928831+0000","last_became_peered":"2023-09-23T20:54:13.928831+0000","last_unstale":"2023-09-23T20:59:36.547525+0000","last_undegraded":"2023-09-23T20:59:36.547525+0000","last_fullsized":"2023-09-23T20:59:36.547525+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:28:44.316851+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":210,"reported_epoch":241,"state":"active+clean","last_fresh":"2023-09-23T20:59:36.570939+0000","last_change":"2023-09-23T20:52:12.855430+0000","last_active":"2023-09-23T20:59:36.570939+0000","last_peered":"2023-09-23T20:59:36.570939+0000","last_clean":"2023-09-23T20:59:36.570939+0000","last_became_active":"2023-09-23T20:52:12.855115+0000","last_became_peered":"2023-09-23T20:52:12.855115+0000","last_unstale":"2023-09-23T20:59:36.570939+0000","last_undegraded":"2023-09-23T20:59:36.570939+0000","last_fullsized":"2023-09-23T20:59:36.570939+0000","mapping_epoch":100,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":101,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:33:23.802214+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":119,"reported_epoch":241,"state":"active+clean","last_fresh":"2023-09-23T20:59:36.556527+0000","last_change":"2023-09-23T20:54:31.902946+0000","last_active":"2023-09-23T20:59:36.556527+0000","last_peered":"2023-09-23T20:59:36.556527+0000","last_clean":"2023-09-23T20:59:36.556527+0000","last_became_active":"2023-09-23T20:54:31.889484+0000","last_became_peered":"2023-09-23T20:54:31.889484+0000","last_unstale":"2023-09-23T20:59:36.556527+0000","last_undegraded":"2023-09-23T20:59:36.556527+0000","last_fullsized":"2023-09-23T20:59:36.556527+0000","mapping_epoch":136,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:44:44.278398+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":119,"reported_epoch":241,"state":"active+clean","last_fresh":"2023-09-23T20:59:36.546855+0000","last_change":"2023-09-23T20:54:13.950351+0000","last_active":"2023-09-23T20:59:36.546855+0000","last_peered":"2023-09-23T20:59:36.546855+0000","last_clean":"2023-09-23T20:59:36.546855+0000","last_became_active":"2023-09-23T20:54:13.949987+0000","last_became_peered":"2023-09-23T20:54:13.949987+0000","last_unstale":"2023-09-23T20:59:36.546855+0000","last_undegraded":"2023-09-23T20:59:36.546855+0000","last_fullsized":"2023-09-23T20:59:36.546855+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:46:17.765134+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":150,"reported_epoch":241,"state":"active+clean","last_fresh":"2023-09-23T20:59:36.556893+0000","last_change":"2023-09-23T20:56:18.141728+0000","last_active":"2023-09-23T20:59:36.556893+0000","last_peered":"2023-09-23T20:59:36.556893+0000","last_clean":"2023-09-23T20:59:36.556893+0000","last_became_active":"2023-09-23T20:56:18.137979+0000","last_became_peered":"2023-09-23T20:56:18.137979+0000","last_unstale":"2023-09-23T20:59:36.556893+0000","last_undegraded":"2023-09-23T20:59:36.556893+0000","last_fullsized":"2023-09-23T20:59:36.556893+0000","mapping_epoch":192,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":193,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:58:05.658443+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":119,"reported_epoch":241,"state":"active+clean","last_fresh":"2023-09-23T20:59:36.546489+0000","last_change":"2023-09-23T20:54:13.935307+0000","last_active":"2023-09-23T20:59:36.546489+0000","last_peered":"2023-09-23T20:59:36.546489+0000","last_clean":"2023-09-23T20:59:36.546489+0000","last_became_active":"2023-09-23T20:54:13.935000+0000","last_became_peered":"2023-09-23T20:54:13.935000+0000","last_unstale":"2023-09-23T20:59:36.546489+0000","last_undegraded":"2023-09-23T20:59:36.546489+0000","last_fullsized":"2023-09-23T20:59:36.546489+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:47:40.974082+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":229,"reported_epoch":241,"state":"active+clean","last_fresh":"2023-09-23T20:59:36.570194+0000","last_change":"2023-09-23T20:55:35.954870+0000","last_active":"2023-09-23T20:59:36.570194+0000","last_peered":"2023-09-23T20:59:36.570194+0000","last_clean":"2023-09-23T20:59:36.570194+0000","last_became_active":"2023-09-23T20:55:35.954568+0000","last_became_peered":"2023-09-23T20:55:35.954568+0000","last_unstale":"2023-09-23T20:59:36.570194+0000","last_undegraded":"2023-09-23T20:59:36.570194+0000","last_fullsized":"2023-09-23T20:59:36.570194+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:06:22.423566+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":228,"reported_epoch":241,"state":"active+clean","last_fresh":"2023-09-23T20:59:36.567727+0000","last_change":"2023-09-23T20:51:10.838334+0000","last_active":"2023-09-23T20:59:36.567727+0000","last_peered":"2023-09-23T20:59:36.567727+0000","last_clean":"2023-09-23T20:59:36.567727+0000","last_became_active":"2023-09-23T20:51:10.838026+0000","last_became_peered":"2023-09-23T20:51:10.838026+0000","last_unstale":"2023-09-23T20:59:36.567727+0000","last_undegraded":"2023-09-23T20:59:36.567727+0000","last_fullsized":"2023-09-23T20:59:36.567727+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:04:37.009457+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":232,"reported_epoch":241,"state":"active+clean","last_fresh":"2023-09-23T20:59:36.547780+0000","last_change":"2023-09-23T20:54:31.903139+0000","last_active":"2023-09-23T20:59:36.547780+0000","last_peered":"2023-09-23T20:59:36.547780+0000","last_clean":"2023-09-23T20:59:36.547780+0000","last_became_active":"2023-09-23T20:54:31.884520+0000","last_became_peered":"2023-09-23T20:54:31.884520+0000","last_unstale":"2023-09-23T20:59:36.547780+0000","last_undegraded":"2023-09-23T20:59:36.547780+0000","last_fullsized":"2023-09-23T20:59:36.547780+0000","mapping_epoch":136,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:39:34.269705+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1108,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":136,"reported_epoch":241,"state":"active+clean","last_fresh":"2023-09-23T20:59:36.546166+0000","last_change":"2023-09-23T20:55:35.951572+0000","last_active":"2023-09-23T20:59:36.546166+0000","last_peered":"2023-09-23T20:59:36.546166+0000","last_clean":"2023-09-23T20:59:36.546166+0000","last_became_active":"2023-09-23T20:55:35.951294+0000","last_became_peered":"2023-09-23T20:55:35.951294+0000","last_unstale":"2023-09-23T20:59:36.546166+0000","last_undegraded":"2023-09-23T20:59:36.546166+0000","last_fullsized":"2023-09-23T20:59:36.546166+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:36:32.447210+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":7,"num_pg":16,"stat_sum":{"num_bytes":2901803032,"num_objects":44279,"num_object_clones":0,"num_object_copies":132837,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":11714,"num_objects_misplaced":1828,"num_objects_unfound":0,"num_objects_dirty":44279,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":159241,"num_write_kb":6512577,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2800,"num_bytes_recovered":183500800,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":9371332608,"data_stored":9371320392,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":119141,"ondisk_log_size":119141,"up":48,"acting":45,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":22,"num_bytes_recovered":5990,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":21,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3381,"ondisk_log_size":3381,"up":63,"acting":63,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201512,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1063068,"kb_used_data":879124,"kb_used_omap":0,"kb_used_meta":183936,"kb_avail":92682084,"statfs":{"total":95995035648,"available":94906454016,"internally_reserved":0,"allocated":900222976,"data_stored":899363282,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":188350464},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,2],"upper_bound":4},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:59:46 2023","interfaces":[{"interface":"back","average":{"1min":1.6559999999999999,"5min":1.802,"15min":1.0840000000000001},"min":{"1min":0.245,"5min":0.21099999999999999,"15min":0.21099999999999999},"max":{"1min":6.4420000000000002,"5min":43.994999999999997,"15min":43.994999999999997},"last":1.956},{"interface":"front","average":{"1min":1.6339999999999999,"5min":1.9159999999999999,"15min":1.1699999999999999},"min":{"1min":0.318,"5min":0.28199999999999997,"15min":0.218},"max":{"1min":6.4610000000000003,"5min":44.206000000000003,"15min":44.206000000000003},"last":1.75}]},{"osd":1,"last update":"Sat Sep 23 20:59:19 2023","interfaces":[{"interface":"back","average":{"1min":1.74,"5min":1.835,"15min":1.2230000000000001},"min":{"1min":0.26600000000000001,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":5.4359999999999999,"5min":10.622,"15min":22.591999999999999},"last":0.54900000000000004},{"interface":"front","average":{"1min":2.0600000000000001,"5min":1.639,"15min":1.0860000000000001},"min":{"1min":0.32100000000000001,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":6.7709999999999999,"5min":10.097,"15min":21.437000000000001},"last":7.9249999999999998}]},{"osd":2,"last update":"Sat Sep 23 20:58:54 2023","interfaces":[{"interface":"back","average":{"1min":3.7370000000000001,"5min":2.016,"15min":1.1499999999999999},"min":{"1min":0.27600000000000002,"5min":0.26500000000000001,"15min":0.24399999999999999},"max":{"1min":10.805999999999999,"5min":10.805999999999999,"15min":10.805999999999999},"last":2.6320000000000001},{"interface":"front","average":{"1min":2.3969999999999998,"5min":2.1520000000000001,"15min":1.1919999999999999},"min":{"1min":0.27800000000000002,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":9.4290000000000003,"5min":45.25,"15min":45.25},"last":7.4500000000000002}]},{"osd":3,"last update":"Sat Sep 23 20:59:30 2023","interfaces":[{"interface":"back","average":{"1min":2.6200000000000001,"5min":2.508,"15min":1.7250000000000001},"min":{"1min":0.33500000000000002,"5min":0.32100000000000001,"15min":0.314},"max":{"1min":8.4390000000000001,"5min":13.007,"15min":36.408000000000001},"last":2.9620000000000002},{"interface":"front","average":{"1min":2.7789999999999999,"5min":2.3079999999999998,"15min":1.552},"min":{"1min":0.41599999999999998,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":13.198,"5min":14.220000000000001,"15min":35.593000000000004},"last":7.1829999999999998}]},{"osd":4,"last update":"Sat Sep 23 20:59:19 2023","interfaces":[{"interface":"back","average":{"1min":0.41099999999999998,"5min":0.78500000000000003,"15min":0.60499999999999998},"min":{"1min":0.25700000000000001,"5min":0.23799999999999999,"15min":0.23699999999999999},"max":{"1min":1.1659999999999999,"5min":5.8920000000000003,"15min":5.8920000000000003},"last":2.379},{"interface":"front","average":{"1min":0.51900000000000002,"5min":0.77600000000000002,"15min":0.61299999999999999},"min":{"1min":0.28599999999999998,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":2.952,"5min":5.3200000000000003,"15min":5.3200000000000003},"last":2.173}]},{"osd":5,"last update":"Sat Sep 23 20:59:19 2023","interfaces":[{"interface":"back","average":{"1min":0.443,"5min":0.78100000000000003,"15min":0.628},"min":{"1min":0.316,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":0.64000000000000001,"5min":7.0389999999999997,"15min":7.0389999999999997},"last":0.57899999999999996},{"interface":"front","average":{"1min":0.55200000000000005,"5min":0.72299999999999998,"15min":0.61399999999999999},"min":{"1min":0.30199999999999999,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":1.909,"5min":4.5999999999999996,"15min":4.5999999999999996},"last":1.9299999999999999}]},{"osd":6,"last update":"Sat Sep 23 20:59:42 2023","interfaces":[{"interface":"back","average":{"1min":0.48299999999999998,"5min":0.68000000000000005,"15min":0.67100000000000004},"min":{"1min":0.28899999999999998,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":0.77300000000000002,"5min":3.5249999999999999,"15min":3.5249999999999999},"last":1.782},{"interface":"front","average":{"1min":0.48499999999999999,"5min":0.749,"15min":0.78600000000000003},"min":{"1min":0.34699999999999998,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":0.95499999999999996,"5min":3.5409999999999999,"15min":3.5409999999999999},"last":1.9099999999999999}]}]},{"osd":6,"up_from":164,"seq":704374636595,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1027012,"kb_used_data":922108,"kb_used_omap":10,"kb_used_meta":104885,"kb_avail":92718140,"statfs":{"total":95995035648,"available":94943375360,"internally_reserved":0,"allocated":944238592,"data_stored":943394812,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":10328,"internal_metadata":107403176},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,1,4,0,2],"upper_bound":128},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:58:43 2023","interfaces":[{"interface":"back","average":{"1min":2.7330000000000001,"5min":1.4410000000000001,"15min":1.0569999999999999},"min":{"1min":0.25800000000000001,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":7.5179999999999998,"5min":8.7319999999999993,"15min":8.7319999999999993},"last":0.55900000000000005},{"interface":"front","average":{"1min":3.1709999999999998,"5min":1.502,"15min":1.1559999999999999},"min":{"1min":0.25,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":6.7140000000000004,"5min":6.7140000000000004,"15min":6.7140000000000004},"last":2.5790000000000002}]},{"osd":1,"last update":"Sat Sep 23 20:58:43 2023","interfaces":[{"interface":"back","average":{"1min":1.5449999999999999,"5min":1.097,"15min":0.89100000000000001},"min":{"1min":0.36799999999999999,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":5.2350000000000003,"5min":6.6029999999999998,"15min":6.6029999999999998},"last":0.40400000000000003},{"interface":"front","average":{"1min":2.5779999999999998,"5min":1.244,"15min":0.99299999999999999},"min":{"1min":0.23000000000000001,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":10.324999999999999,"5min":10.324999999999999,"15min":10.324999999999999},"last":0.54200000000000004}]},{"osd":2,"last update":"Sat Sep 23 20:58:43 2023","interfaces":[{"interface":"back","average":{"1min":2.286,"5min":1.982,"15min":1.758},"min":{"1min":0.28999999999999998,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":7.0519999999999996,"5min":13.750999999999999,"15min":13.750999999999999},"last":2.681},{"interface":"front","average":{"1min":2.419,"5min":1.5289999999999999,"15min":1.2470000000000001},"min":{"1min":0.26200000000000001,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":8.0890000000000004,"5min":8.0890000000000004,"15min":8.0890000000000004},"last":2.3300000000000001}]},{"osd":3,"last update":"Sat Sep 23 20:58:43 2023","interfaces":[{"interface":"back","average":{"1min":1.8680000000000001,"5min":1.2849999999999999,"15min":1.173},"min":{"1min":0.32000000000000001,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":6.9000000000000004,"5min":6.9000000000000004,"15min":6.9000000000000004},"last":1.298},{"interface":"front","average":{"1min":2.1819999999999999,"5min":1.74,"15min":1.6479999999999999},"min":{"1min":0.28599999999999998,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":11.021000000000001,"5min":11.021000000000001,"15min":11.021000000000001},"last":4.3659999999999997}]},{"osd":4,"last update":"Sat Sep 23 20:58:43 2023","interfaces":[{"interface":"back","average":{"1min":0.49099999999999999,"5min":0.85899999999999999,"15min":0.98699999999999999},"min":{"1min":0.23000000000000001,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":2.7919999999999998,"5min":9.0079999999999991,"15min":9.0079999999999991},"last":0.57599999999999996},{"interface":"front","average":{"1min":0.49299999999999999,"5min":0.50800000000000001,"15min":0.52300000000000002},"min":{"1min":0.27200000000000002,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":2.0059999999999998,"5min":2.0059999999999998,"15min":2.0059999999999998},"last":0.52700000000000002}]},{"osd":5,"last update":"Sat Sep 23 20:58:43 2023","interfaces":[{"interface":"back","average":{"1min":0.53700000000000003,"5min":0.56899999999999995,"15min":0.55000000000000004},"min":{"1min":0.27300000000000002,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":2.8239999999999998,"5min":2.8239999999999998,"15min":2.8239999999999998},"last":0.60199999999999998},{"interface":"front","average":{"1min":0.81000000000000005,"5min":0.78500000000000003,"15min":0.80100000000000005},"min":{"1min":0.33400000000000002,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":3.1379999999999999,"5min":3.1379999999999999,"15min":3.1379999999999999},"last":0.58599999999999997}]},{"osd":7,"last update":"Sat Sep 23 20:58:43 2023","interfaces":[{"interface":"back","average":{"1min":0.52700000000000002,"5min":0.85399999999999998,"15min":0.90400000000000003},"min":{"1min":0.312,"5min":0.312,"15min":0.312},"max":{"1min":2.073,"5min":7.8570000000000002,"15min":7.8570000000000002},"last":0.61799999999999999},{"interface":"front","average":{"1min":0.61899999999999999,"5min":0.66200000000000003,"15min":0.57399999999999995},"min":{"1min":0.30299999999999999,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":2.6989999999999998,"5min":8.282,"15min":8.282},"last":0.437}]}]},{"osd":1,"up_from":88,"seq":377957122166,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1668168,"kb_used_data":1441024,"kb_used_omap":9,"kb_used_meta":226934,"kb_avail":92076984,"statfs":{"total":95995035648,"available":94286831616,"internally_reserved":0,"allocated":1475608576,"data_stored":1474414227,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9239,"internal_metadata":232381417},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:59:23 2023","interfaces":[{"interface":"back","average":{"1min":3.9180000000000001,"5min":3.1070000000000002,"15min":1.5389999999999999},"min":{"1min":0.20699999999999999,"5min":0.20699999999999999,"15min":0.20699999999999999},"max":{"1min":14.813000000000001,"5min":16.646999999999998,"15min":16.646999999999998},"last":7.5519999999999996},{"interface":"front","average":{"1min":3.77,"5min":3.335,"15min":1.6819999999999999},"min":{"1min":0.33300000000000002,"5min":0.246,"15min":0.22500000000000001},"max":{"1min":9.484,"5min":16.664999999999999,"15min":18.457999999999998},"last":7.415}]},{"osd":2,"last update":"Sat Sep 23 20:58:53 2023","interfaces":[{"interface":"back","average":{"1min":2.7130000000000001,"5min":2.4140000000000001,"15min":1.1799999999999999},"min":{"1min":0.24299999999999999,"5min":0.221,"15min":0.20300000000000001},"max":{"1min":8.6829999999999998,"5min":14.335000000000001,"15min":14.335000000000001},"last":0.496},{"interface":"front","average":{"1min":5.8019999999999996,"5min":3.27,"15min":1.758},"min":{"1min":0.39500000000000002,"5min":0.248,"15min":0.248},"max":{"1min":16.048999999999999,"5min":16.048999999999999,"15min":38.951000000000001},"last":0.29499999999999998}]},{"osd":3,"last update":"Sat Sep 23 20:59:23 2023","interfaces":[{"interface":"back","average":{"1min":3.1539999999999999,"5min":2.629,"15min":1.5589999999999999},"min":{"1min":0.42499999999999999,"5min":0.246,"15min":0.246},"max":{"1min":9.4450000000000003,"5min":16.055,"15min":25.314},"last":1.284},{"interface":"front","average":{"1min":3.0870000000000002,"5min":2.4860000000000002,"15min":1.411},"min":{"1min":0.29799999999999999,"5min":0.20599999999999999,"15min":0.20599999999999999},"max":{"1min":10.204000000000001,"5min":14.225,"15min":19.881},"last":1.1970000000000001}]},{"osd":4,"last update":"Sat Sep 23 20:59:23 2023","interfaces":[{"interface":"back","average":{"1min":1.871,"5min":1.6439999999999999,"15min":1.2130000000000001},"min":{"1min":0.28299999999999997,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":9.3480000000000008,"5min":10.598000000000001,"15min":12.006},"last":0.86199999999999999},{"interface":"front","average":{"1min":1.633,"5min":1.675,"15min":1.0800000000000001},"min":{"1min":0.38400000000000001,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":7.532,"5min":7.532,"15min":11.234999999999999},"last":0.58899999999999997}]},{"osd":5,"last update":"Sat Sep 23 20:59:23 2023","interfaces":[{"interface":"back","average":{"1min":2.3700000000000001,"5min":2.1469999999999998,"15min":1.292},"min":{"1min":0.55800000000000005,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":8.4410000000000007,"5min":15.041,"15min":15.041},"last":0.48199999999999998},{"interface":"front","average":{"1min":1.9370000000000001,"5min":1.756,"15min":1.2390000000000001},"min":{"1min":0.432,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":9.3889999999999993,"5min":10.647,"15min":12.08},"last":0.91300000000000003}]},{"osd":6,"last update":"Sat Sep 23 20:59:41 2023","interfaces":[{"interface":"back","average":{"1min":2.266,"5min":1.5609999999999999,"15min":1.397},"min":{"1min":0.46200000000000002,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":10.159000000000001,"5min":10.673999999999999,"15min":10.673999999999999},"last":0.95999999999999996},{"interface":"front","average":{"1min":1.8660000000000001,"5min":1.6299999999999999,"15min":1.3779999999999999},"min":{"1min":0.33700000000000002,"5min":0.309,"15min":0.309},"max":{"1min":7.6779999999999999,"5min":8.7530000000000001,"15min":8.7530000000000001},"last":0.56399999999999995}]},{"osd":7,"last update":"Sat Sep 23 20:59:23 2023","interfaces":[{"interface":"back","average":{"1min":2.089,"5min":1.871,"15min":1.163},"min":{"1min":0.433,"5min":0.36199999999999999,"15min":0.35699999999999998},"max":{"1min":8.7769999999999992,"5min":8.7769999999999992,"15min":11.186999999999999},"last":0.53200000000000003},{"interface":"front","average":{"1min":2.5430000000000001,"5min":2.202,"15min":1.325},"min":{"1min":0.497,"5min":0.40899999999999997,"15min":0.34599999999999997},"max":{"1min":8.4939999999999998,"5min":15.098000000000001,"15min":15.098000000000001},"last":0.51500000000000001}]}]},{"osd":0,"up_from":83,"seq":356482285693,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1595508,"kb_used_data":1401388,"kb_used_omap":28,"kb_used_meta":194019,"kb_avail":92149644,"statfs":{"total":95995035648,"available":94361235456,"internally_reserved":0,"allocated":1435021312,"data_stored":1433852573,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":28848,"internal_metadata":198676304},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,3,0,0,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 20:59:19 2023","interfaces":[{"interface":"back","average":{"1min":3.694,"5min":3.121,"15min":1.4850000000000001},"min":{"1min":0.22700000000000001,"5min":0.191,"15min":0.191},"max":{"1min":15.577999999999999,"5min":16.484999999999999,"15min":18.863},"last":9.5960000000000001},{"interface":"front","average":{"1min":4.4610000000000003,"5min":3.0339999999999998,"15min":1.5389999999999999},"min":{"1min":0.20999999999999999,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":15.771000000000001,"5min":15.771000000000001,"15min":17.704999999999998},"last":0.39500000000000002}]},{"osd":2,"last update":"Sat Sep 23 20:58:50 2023","interfaces":[{"interface":"back","average":{"1min":4.843,"5min":3.2549999999999999,"15min":1.6359999999999999},"min":{"1min":0.434,"5min":0.24299999999999999,"15min":0.23799999999999999},"max":{"1min":14.942,"5min":14.976000000000001,"15min":17.753},"last":0.52300000000000002},{"interface":"front","average":{"1min":7.774,"5min":3.8180000000000001,"15min":1.726},"min":{"1min":0.38500000000000001,"5min":0.25800000000000001,"15min":0.21099999999999999},"max":{"1min":21.747,"5min":21.747,"15min":21.747},"last":0.54600000000000004}]},{"osd":3,"last update":"Sat Sep 23 20:59:19 2023","interfaces":[{"interface":"back","average":{"1min":4.7839999999999998,"5min":3.0590000000000002,"15min":1.5900000000000001},"min":{"1min":0.38300000000000001,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":21.184000000000001,"5min":21.184000000000001,"15min":21.184000000000001},"last":8.6029999999999998},{"interface":"front","average":{"1min":6.0469999999999997,"5min":3.5510000000000002,"15min":1.794},"min":{"1min":0.30599999999999999,"5min":0.28000000000000003,"15min":0.23300000000000001},"max":{"1min":21.167000000000002,"5min":21.167000000000002,"15min":21.167000000000002},"last":0.67800000000000005}]},{"osd":4,"last update":"Sat Sep 23 20:58:50 2023","interfaces":[{"interface":"back","average":{"1min":6.6399999999999997,"5min":2.9279999999999999,"15min":1.4730000000000001},"min":{"1min":0.36599999999999999,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":21.594999999999999,"5min":21.594999999999999,"15min":21.594999999999999},"last":2.5699999999999998},{"interface":"front","average":{"1min":2.657,"5min":1.9790000000000001,"15min":1.131},"min":{"1min":0.35199999999999998,"5min":0.31,"15min":0.28100000000000003},"max":{"1min":10.393000000000001,"5min":18.724,"15min":18.724},"last":8.5470000000000006}]},{"osd":5,"last update":"Sat Sep 23 20:58:50 2023","interfaces":[{"interface":"back","average":{"1min":2.2719999999999998,"5min":1.8069999999999999,"15min":1.177},"min":{"1min":0.34999999999999998,"5min":0.318,"15min":0.318},"max":{"1min":8.8970000000000002,"5min":8.8970000000000002,"15min":8.8970000000000002},"last":0.433},{"interface":"front","average":{"1min":5.7110000000000003,"5min":2.8919999999999999,"15min":1.4950000000000001},"min":{"1min":0.40600000000000003,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":20.512,"5min":20.512,"15min":20.512},"last":2.0289999999999999}]},{"osd":6,"last update":"Sat Sep 23 20:59:46 2023","interfaces":[{"interface":"back","average":{"1min":2.9180000000000001,"5min":2.145,"15min":1.819},"min":{"1min":0.45900000000000002,"5min":0.37,"15min":0.37},"max":{"1min":13.134,"5min":13.134,"15min":13.134},"last":8.5150000000000006},{"interface":"front","average":{"1min":2.9140000000000001,"5min":2.198,"15min":1.8959999999999999},"min":{"1min":0.43099999999999999,"5min":0.377,"15min":0.377},"max":{"1min":13.254,"5min":13.254,"15min":13.254},"last":8.6259999999999994}]},{"osd":7,"last update":"Sat Sep 23 20:58:50 2023","interfaces":[{"interface":"back","average":{"1min":2.4729999999999999,"5min":1.875,"15min":1.22},"min":{"1min":0.38100000000000001,"5min":0.379,"15min":0.379},"max":{"1min":8.7729999999999997,"5min":8.7729999999999997,"15min":8.7729999999999997},"last":1.893},{"interface":"front","average":{"1min":2.7010000000000001,"5min":2.032,"15min":1.1619999999999999},"min":{"1min":0.377,"5min":0.377,"15min":0.377},"max":{"1min":10.359999999999999,"5min":18.673999999999999,"15min":18.673999999999999},"last":8.5790000000000006}]}]},{"osd":2,"up_from":93,"seq":399431958640,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1717028,"kb_used_data":1403484,"kb_used_omap":81,"kb_used_meta":313454,"kb_avail":92028124,"statfs":{"total":95995035648,"available":94236798976,"internally_reserved":0,"allocated":1437167616,"data_stored":1436011625,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":83750,"internal_metadata":320977114},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,4,2,2],"upper_bound":32},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:58:47 2023","interfaces":[{"interface":"back","average":{"1min":3.1429999999999998,"5min":2.0670000000000002,"15min":1.079},"min":{"1min":0.22700000000000001,"5min":0.17999999999999999,"15min":0.17999999999999999},"max":{"1min":14.773999999999999,"5min":14.773999999999999,"15min":14.773999999999999},"last":4.3899999999999997},{"interface":"front","average":{"1min":5.1459999999999999,"5min":3.133,"15min":1.4319999999999999},"min":{"1min":0.32400000000000001,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":17.045000000000002,"5min":17.045000000000002,"15min":17.045000000000002},"last":4.1589999999999998}]},{"osd":1,"last update":"Sat Sep 23 20:58:47 2023","interfaces":[{"interface":"back","average":{"1min":4.1470000000000002,"5min":2.718,"15min":1.48},"min":{"1min":0.29299999999999998,"5min":0.20200000000000001,"15min":0.20200000000000001},"max":{"1min":14.709,"5min":14.709,"15min":16.081},"last":4.3410000000000002},{"interface":"front","average":{"1min":5.3819999999999997,"5min":3.6389999999999998,"15min":1.77},"min":{"1min":0.41499999999999998,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":11.492000000000001,"5min":12.07,"15min":16.041},"last":1.9590000000000001}]},{"osd":3,"last update":"Sat Sep 23 20:59:29 2023","interfaces":[{"interface":"back","average":{"1min":5.0449999999999999,"5min":3.6579999999999999,"15min":1.893},"min":{"1min":0.30299999999999999,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":27.614999999999998,"5min":27.614999999999998,"15min":27.614999999999998},"last":1.98},{"interface":"front","average":{"1min":5.9119999999999999,"5min":4.234,"15min":2.1930000000000001},"min":{"1min":0.33000000000000002,"5min":0.22800000000000001,"15min":0.221},"max":{"1min":27.562000000000001,"5min":27.562000000000001,"15min":27.562000000000001},"last":2.8860000000000001}]},{"osd":4,"last update":"Sat Sep 23 20:58:47 2023","interfaces":[{"interface":"back","average":{"1min":2.2519999999999998,"5min":1.9910000000000001,"15min":1.1439999999999999},"min":{"1min":0.28699999999999998,"5min":0.28699999999999998,"15min":0.26400000000000001},"max":{"1min":8.4580000000000002,"5min":9.8119999999999994,"15min":9.8119999999999994},"last":1.8819999999999999},{"interface":"front","average":{"1min":2.137,"5min":2.1560000000000001,"15min":1.2270000000000001},"min":{"1min":0.32400000000000001,"5min":0.313,"15min":0.313},"max":{"1min":8.1649999999999991,"5min":10.609,"15min":10.609},"last":2.7999999999999998}]},{"osd":5,"last update":"Sat Sep 23 20:58:47 2023","interfaces":[{"interface":"back","average":{"1min":2.1779999999999999,"5min":2.238,"15min":1.2470000000000001},"min":{"1min":0.32300000000000001,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":8.5,"5min":9.7629999999999999,"15min":9.7629999999999999},"last":1.9370000000000001},{"interface":"front","average":{"1min":2.0670000000000002,"5min":2.1709999999999998,"15min":1.242},"min":{"1min":0.36399999999999999,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":8.1170000000000009,"5min":10.574,"15min":10.574},"last":2.8650000000000002}]},{"osd":6,"last update":"Sat Sep 23 20:59:45 2023","interfaces":[{"interface":"back","average":{"1min":2.9060000000000001,"5min":2.0640000000000001,"15min":1.8},"min":{"1min":0.35699999999999998,"5min":0.312,"15min":0.312},"max":{"1min":10.012,"5min":14.244999999999999,"15min":14.244999999999999},"last":2.839},{"interface":"front","average":{"1min":2.8740000000000001,"5min":1.8129999999999999,"15min":1.2909999999999999},"min":{"1min":0.41599999999999998,"5min":0.374,"15min":0.374},"max":{"1min":10.032999999999999,"5min":10.032999999999999,"15min":10.032999999999999},"last":2.7719999999999998}]},{"osd":7,"last update":"Sat Sep 23 20:58:47 2023","interfaces":[{"interface":"back","average":{"1min":1.4830000000000001,"5min":1.4339999999999999,"15min":0.995},"min":{"1min":0.315,"5min":0.315,"15min":0.315},"max":{"1min":5.8570000000000002,"5min":8.0820000000000007,"15min":8.0820000000000007},"last":4.415},{"interface":"front","average":{"1min":2.2090000000000001,"5min":2.2109999999999999,"15min":1.262},"min":{"1min":0.35099999999999998,"5min":0.35099999999999998,"15min":0.34499999999999997},"max":{"1min":8.5250000000000004,"5min":9.7859999999999996,"15min":9.7859999999999996},"last":1.9139999999999999}]}]},{"osd":3,"up_from":98,"seq":420906795113,"num_pgs":49,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1610760,"kb_used_data":1252864,"kb_used_omap":7,"kb_used_meta":357880,"kb_avail":92134392,"statfs":{"total":95995035648,"available":94345617408,"internally_reserved":0,"allocated":1282932736,"data_stored":1281801911,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7576,"internal_metadata":366469736},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,1,1,1],"upper_bound":128},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:59:28 2023","interfaces":[{"interface":"back","average":{"1min":4.056,"5min":2.399,"15min":1.7350000000000001},"min":{"1min":0.30199999999999999,"5min":0.20599999999999999,"15min":0.20599999999999999},"max":{"1min":13.566000000000001,"5min":13.566000000000001,"15min":23.123999999999999},"last":1.3540000000000001},{"interface":"front","average":{"1min":5.2229999999999999,"5min":3.1099999999999999,"15min":1.6259999999999999},"min":{"1min":0.23200000000000001,"5min":0.19800000000000001,"15min":0.19800000000000001},"max":{"1min":13.455,"5min":15.079000000000001,"15min":15.079000000000001},"last":2.3250000000000002}]},{"osd":1,"last update":"Sat Sep 23 20:59:28 2023","interfaces":[{"interface":"back","average":{"1min":2.9510000000000001,"5min":2.8220000000000001,"15min":1.6120000000000001},"min":{"1min":0.23899999999999999,"5min":0.23899999999999999,"15min":0.22900000000000001},"max":{"1min":11.343,"5min":16.318000000000001,"15min":21.385000000000002},"last":0.61099999999999999},{"interface":"front","average":{"1min":2.972,"5min":2.2290000000000001,"15min":1.6830000000000001},"min":{"1min":0.23400000000000001,"5min":0.218,"15min":0.218},"max":{"1min":11.782,"5min":11.782,"15min":25.353000000000002},"last":0.25}]},{"osd":2,"last update":"Sat Sep 23 20:59:28 2023","interfaces":[{"interface":"back","average":{"1min":4.7210000000000001,"5min":3.1459999999999999,"15min":1.7310000000000001},"min":{"1min":0.33000000000000002,"5min":0.33000000000000002,"15min":0.30099999999999999},"max":{"1min":13.616,"5min":15.476000000000001,"15min":24.154},"last":0.51400000000000001},{"interface":"front","average":{"1min":3.665,"5min":2.4790000000000001,"15min":1.792},"min":{"1min":0.27300000000000002,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":13.125999999999999,"5min":16.032,"15min":25.305},"last":0.66500000000000004}]},{"osd":4,"last update":"Sat Sep 23 20:59:28 2023","interfaces":[{"interface":"back","average":{"1min":3.0070000000000001,"5min":2.0299999999999998,"15min":1.3},"min":{"1min":0.379,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":11.089,"5min":11.089,"15min":11.861000000000001},"last":0.40899999999999997},{"interface":"front","average":{"1min":1.954,"5min":1.359,"15min":1.335},"min":{"1min":0.309,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":9.9779999999999998,"5min":9.9779999999999998,"15min":22.853000000000002},"last":0.317}]},{"osd":5,"last update":"Sat Sep 23 20:59:28 2023","interfaces":[{"interface":"back","average":{"1min":2.8540000000000001,"5min":2.052,"15min":1.2929999999999999},"min":{"1min":0.39000000000000001,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":11.048,"5min":11.048,"15min":11.891999999999999},"last":0.48399999999999999},{"interface":"front","average":{"1min":3.585,"5min":1.9510000000000001,"15min":1.272},"min":{"1min":0.53200000000000003,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":13.43,"5min":13.43,"15min":13.43},"last":2.274}]},{"osd":6,"last update":"Sat Sep 23 20:58:43 2023","interfaces":[{"interface":"back","average":{"1min":1.9450000000000001,"5min":1.1930000000000001,"15min":1.0309999999999999},"min":{"1min":0.33500000000000002,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":10.018000000000001,"5min":10.018000000000001,"15min":10.018000000000001},"last":0.35199999999999998},{"interface":"front","average":{"1min":1.4770000000000001,"5min":1.48,"15min":1.4339999999999999},"min":{"1min":0.38700000000000001,"5min":0.38700000000000001,"15min":0.38700000000000001},"max":{"1min":5.3239999999999998,"5min":7.234,"15min":7.234},"last":2.2400000000000002}]},{"osd":7,"last update":"Sat Sep 23 20:59:28 2023","interfaces":[{"interface":"back","average":{"1min":2.8700000000000001,"5min":2.3839999999999999,"15min":1.4670000000000001},"min":{"1min":0.42799999999999999,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":11.114000000000001,"5min":16.364999999999998,"15min":16.364999999999998},"last":0.45300000000000001},{"interface":"front","average":{"1min":3.6400000000000001,"5min":2.077,"15min":1.3440000000000001},"min":{"1min":0.58399999999999996,"5min":0.379,"15min":0.379},"max":{"1min":13.355,"5min":13.355,"15min":13.355},"last":2.2999999999999998}]}]},{"osd":4,"up_from":31,"seq":133143986421,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1019804,"kb_used_data":884372,"kb_used_omap":0,"kb_used_meta":135424,"kb_avail":92725348,"statfs":{"total":95995035648,"available":94950756352,"internally_reserved":0,"allocated":905596928,"data_stored":904737487,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":138674176},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,0,1,3],"upper_bound":128},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:58:53 2023","interfaces":[{"interface":"back","average":{"1min":1.917,"5min":1.7170000000000001,"15min":1.042},"min":{"1min":0.245,"5min":0.214,"15min":0.214},"max":{"1min":8.8439999999999994,"5min":11.784000000000001,"15min":11.784000000000001},"last":0.34499999999999997},{"interface":"front","average":{"1min":1.8149999999999999,"5min":1.7889999999999999,"15min":1.23},"min":{"1min":0.314,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":8.8300000000000001,"5min":11.766,"15min":11.766},"last":0.32800000000000001}]},{"osd":1,"last update":"Sat Sep 23 20:59:22 2023","interfaces":[{"interface":"back","average":{"1min":2.4119999999999999,"5min":1.6930000000000001,"15min":1.1200000000000001},"min":{"1min":0.35499999999999998,"5min":0.246,"15min":0.23599999999999999},"max":{"1min":7.3310000000000004,"5min":10.153,"15min":10.153},"last":0.25800000000000001},{"interface":"front","average":{"1min":3.2669999999999999,"5min":2.0099999999999998,"15min":1.1419999999999999},"min":{"1min":0.33000000000000002,"5min":0.26100000000000001,"15min":0.23200000000000001},"max":{"1min":9.7509999999999994,"5min":12.292999999999999,"15min":12.292999999999999},"last":0.40300000000000002}]},{"osd":2,"last update":"Sat Sep 23 20:58:53 2023","interfaces":[{"interface":"back","average":{"1min":1.8700000000000001,"5min":1.764,"15min":1.115},"min":{"1min":0.34300000000000003,"5min":0.311,"15min":0.29099999999999998},"max":{"1min":7.3150000000000004,"5min":15.288,"15min":15.288},"last":0.5},{"interface":"front","average":{"1min":1.633,"5min":1.974,"15min":1.2629999999999999},"min":{"1min":0.32600000000000001,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":8.7959999999999994,"5min":15.262,"15min":15.262},"last":0.91900000000000004}]},{"osd":3,"last update":"Sat Sep 23 20:59:22 2023","interfaces":[{"interface":"back","average":{"1min":2.21,"5min":2.1850000000000001,"15min":1.6180000000000001},"min":{"1min":0.30499999999999999,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":4.835,"5min":14.824999999999999,"15min":31.562999999999999},"last":0.39200000000000002},{"interface":"front","average":{"1min":2.2770000000000001,"5min":2.2989999999999999,"15min":1.6899999999999999},"min":{"1min":0.30299999999999999,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":4.8490000000000002,"5min":14.839,"15min":31.574000000000002},"last":0.53200000000000003}]},{"osd":5,"last update":"Sat Sep 23 20:59:36 2023","interfaces":[{"interface":"back","average":{"1min":0.41099999999999998,"5min":0.61099999999999999,"15min":0.52800000000000002},"min":{"1min":0.28899999999999998,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":0.90700000000000003,"5min":3.3279999999999998,"15min":3.3279999999999998},"last":0.42499999999999999},{"interface":"front","average":{"1min":1.002,"5min":0.77800000000000002,"15min":0.61499999999999999},"min":{"1min":0.29999999999999999,"5min":0.251,"15min":0.251},"max":{"1min":3.4830000000000001,"5min":7.7619999999999996,"15min":7.7619999999999996},"last":1.7669999999999999}]},{"osd":6,"last update":"Sat Sep 23 20:59:46 2023","interfaces":[{"interface":"back","average":{"1min":0.62,"5min":0.70399999999999996,"15min":0.55700000000000005},"min":{"1min":0.32100000000000001,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":3.4180000000000001,"5min":4.6909999999999998,"15min":4.6909999999999998},"last":0.308},{"interface":"front","average":{"1min":0.752,"5min":0.73699999999999999,"15min":0.67500000000000004},"min":{"1min":0.34499999999999997,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":3.2999999999999998,"5min":3.456,"15min":3.456},"last":0.58999999999999997}]},{"osd":7,"last update":"Sat Sep 23 20:59:22 2023","interfaces":[{"interface":"back","average":{"1min":0.83499999999999996,"5min":0.752,"15min":0.69399999999999995},"min":{"1min":0.32600000000000001,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":2.8889999999999998,"5min":6.2240000000000002,"15min":7.383},"last":0.56200000000000006},{"interface":"front","average":{"1min":0.73999999999999999,"5min":0.78400000000000003,"15min":0.67500000000000004},"min":{"1min":0.34200000000000003,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":2.8679999999999999,"5min":7.8310000000000004,"15min":7.8310000000000004},"last":0.442}]}]},{"osd":5,"up_from":39,"seq":167503724786,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1167248,"kb_used_data":1044744,"kb_used_omap":0,"kb_used_meta":122496,"kb_avail":92577904,"statfs":{"total":95995035648,"available":94799773696,"internally_reserved":0,"allocated":1069817856,"data_stored":1068966991,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":125435904},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,0,0,0,0,2],"upper_bound":64},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:59:38 2023","interfaces":[{"interface":"back","average":{"1min":4.0339999999999998,"5min":1.619,"15min":0.96299999999999997},"min":{"1min":0.27400000000000002,"5min":0.20399999999999999,"15min":0.20399999999999999},"max":{"1min":12.795999999999999,"5min":12.795999999999999,"15min":12.795999999999999},"last":0.35699999999999998},{"interface":"front","average":{"1min":2.7330000000000001,"5min":1.948,"15min":1.125},"min":{"1min":0.24099999999999999,"5min":0.223,"15min":0.223},"max":{"1min":16.219999999999999,"5min":16.219999999999999,"15min":16.219999999999999},"last":0.377}]},{"osd":1,"last update":"Sat Sep 23 20:59:17 2023","interfaces":[{"interface":"back","average":{"1min":2.7749999999999999,"5min":1.758,"15min":1.099},"min":{"1min":0.248,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":8.1829999999999998,"5min":8.3520000000000003,"15min":8.3520000000000003},"last":0.28299999999999997},{"interface":"front","average":{"1min":2.3690000000000002,"5min":1.825,"15min":1.099},"min":{"1min":0.27200000000000002,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":8.5399999999999991,"5min":10.285,"15min":10.285},"last":0.45500000000000002}]},{"osd":2,"last update":"Sat Sep 23 20:58:52 2023","interfaces":[{"interface":"back","average":{"1min":1.9299999999999999,"5min":1.6000000000000001,"15min":0.97799999999999998},"min":{"1min":0.311,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":6,"5min":12.651,"15min":12.651},"last":0.64800000000000002},{"interface":"front","average":{"1min":3.077,"5min":1.8440000000000001,"15min":1.085},"min":{"1min":0.27100000000000002,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":12.300000000000001,"5min":12.300000000000001,"15min":12.300000000000001},"last":4.3979999999999997}]},{"osd":3,"last update":"Sat Sep 23 20:59:17 2023","interfaces":[{"interface":"back","average":{"1min":3.5880000000000001,"5min":2.2090000000000001,"15min":1.351},"min":{"1min":0.46700000000000003,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":12.188000000000001,"5min":12.188000000000001,"15min":12.188000000000001},"last":0.53600000000000003},{"interface":"front","average":{"1min":2.0739999999999998,"5min":1.6060000000000001,"15min":1.071},"min":{"1min":0.32500000000000001,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":7.258,"5min":10.457000000000001,"15min":10.457000000000001},"last":0.39700000000000002}]},{"osd":4,"last update":"Sat Sep 23 20:59:34 2023","interfaces":[{"interface":"back","average":{"1min":0.81699999999999995,"5min":0.63300000000000001,"15min":0.55300000000000005},"min":{"1min":0.32100000000000001,"5min":0.252,"15min":0.252},"max":{"1min":2.9169999999999998,"5min":3.8479999999999999,"15min":3.8479999999999999},"last":0.42499999999999999},{"interface":"front","average":{"1min":0.83499999999999996,"5min":0.67500000000000004,"15min":0.57499999999999996},"min":{"1min":0.27400000000000002,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":3.3530000000000002,"5min":3.3530000000000002,"15min":3.3530000000000002},"last":0.51400000000000001}]},{"osd":6,"last update":"Sat Sep 23 20:59:38 2023","interfaces":[{"interface":"back","average":{"1min":0.86099999999999999,"5min":0.65100000000000002,"15min":0.59099999999999997},"min":{"1min":0.29199999999999998,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":4.1120000000000001,"5min":4.1120000000000001,"15min":4.1120000000000001},"last":4.359},{"interface":"front","average":{"1min":1.0269999999999999,"5min":0.755,"15min":0.747},"min":{"1min":0.36899999999999999,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":2.9620000000000002,"5min":3.6560000000000001,"15min":3.6560000000000001},"last":0.48499999999999999}]},{"osd":7,"last update":"Sat Sep 23 20:59:17 2023","interfaces":[{"interface":"back","average":{"1min":0.71099999999999997,"5min":0.80200000000000005,"15min":0.64200000000000002},"min":{"1min":0.33400000000000002,"5min":0.307,"15min":0.30199999999999999},"max":{"1min":2.516,"5min":8.5649999999999995,"15min":8.5649999999999995},"last":4.4240000000000004},{"interface":"front","average":{"1min":0.77200000000000002,"5min":0.75,"15min":0.65700000000000003},"min":{"1min":0.34399999999999997,"5min":0.29799999999999999,"15min":0.27700000000000002},"max":{"1min":3.0150000000000001,"5min":4.5430000000000001,"15min":4.5430000000000001},"last":0.77600000000000002}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1421807616,"data_stored":1421803544,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1462243328,"data_stored":1462239256,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1427111936,"data_stored":1427111936,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1276968960,"data_stored":1276968960,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":890961920,"data_stored":890961920,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1066926080,"data_stored":1066926080,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":936382464,"data_stored":936378392,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":888930304,"data_stored":888930304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T20:59:50.221 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:59:50.317 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:59:50.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:59:50 smithi026 ceph-mon[179688]: Health check update: Degraded data redundancy: 11714/132909 objects degraded (8.814%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T20:59:50.388 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:59:50 smithi026 ceph-mon[175151]: Health check update: Degraded data redundancy: 11714/132909 objects degraded (8.814%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T20:59:50.505 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:59:50.520 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:59:50.521 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:59:50.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:59:50 smithi070 ceph-mon[168282]: Health check update: Degraded data redundancy: 11714/132909 objects degraded (8.814%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T20:59:50.621 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:59:50.651 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:59:50.671 INFO:teuthology.orchestra.run.smithi026.stdout:ERROR: (22) Invalid argument 2023-09-23T20:59:50.671 INFO:teuthology.orchestra.run.smithi026.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:59:50.688 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:59:50.690 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:59:50.690 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:59:50.791 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:59:50.820 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:59:50.858 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:59:50.864 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:59:50.864 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:59:50.965 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:59:51.046 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:59:51.137 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:59:51.142 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:59:51.143 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:59:51.182 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_ops_in_flight 2023-09-23T20:59:51.244 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:59:51.390 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:59:51.501 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:59:51.514 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:59:51.514 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:59:51.615 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:59:51.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:59:51 smithi070 ceph-mon[168282]: pgmap v603: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 2.6 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 591 op/s; 11714/128871 objects degraded (9.090%); 1828/128871 objects misplaced (1.418%); 1.9 MiB/s, 29 objects/s recovering 2023-09-23T20:59:51.636 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:59:51 smithi026 ceph-mon[175151]: pgmap v603: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 2.6 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 591 op/s; 11714/128871 objects degraded (9.090%); 1828/128871 objects misplaced (1.418%); 1.9 MiB/s, 29 objects/s recovering 2023-09-23T20:59:51.636 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:59:51 smithi026 ceph-mon[179688]: pgmap v603: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 2.6 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 591 op/s; 11714/128871 objects degraded (9.090%); 1828/128871 objects misplaced (1.418%); 1.9 MiB/s, 29 objects/s recovering 2023-09-23T20:59:51.646 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:59:51.688 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:59:51.693 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:59:51.693 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:59:51.794 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:59:51.824 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:59:51.866 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:59:51.871 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:59:51.871 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:59:51.908 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T20:59:51.972 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:59:52.064 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:59:52.165 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:59:52.169 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:59:52.170 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:59:52.253 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:59:52.270 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:59:52.299 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:59:52.310 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:59:52 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/1492873981' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:59:52.310 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:59:52 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/1492873981' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:59:52.341 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:59:52.346 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:59:52.346 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:59:52.362 INFO:teuthology.orchestra.run.smithi026.stderr:noscrub is unset 2023-09-23T20:59:52.447 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:59:52.519 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:59:52.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:59:52 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/1492873981' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:59:53.181 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T20:59:53.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:59:53 smithi070 ceph-mon[168282]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:59:53.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:59:53 smithi070 ceph-mon[168282]: failed to encode map e243 with expected crc 2023-09-23T20:59:53.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:59:53 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/1492873981' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T20:59:53.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:59:53 smithi070 ceph-mon[168282]: osdmap e243: 8 total, 8 up, 8 in 2023-09-23T20:59:53.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:59:53 smithi070 ceph-mon[168282]: failed to encode map e243 with expected crc 2023-09-23T20:59:53.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:59:53 smithi070 ceph-mon[168282]: failed to encode map e243 with expected crc 2023-09-23T20:59:53.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:59:53 smithi070 ceph-mon[168282]: pgmap v605: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 2.5 GiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 697 op/s; 11749/123888 objects degraded (9.484%); 1654/123888 objects misplaced (1.335%); 2.2 MiB/s, 34 objects/s recovering 2023-09-23T20:59:53.631 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:59:53 smithi026 ceph-mon[179688]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:59:53.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:59:53 smithi026 ceph-mon[179688]: failed to encode map e243 with expected crc 2023-09-23T20:59:53.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:59:53 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/1492873981' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T20:59:53.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:59:53 smithi026 ceph-mon[179688]: osdmap e243: 8 total, 8 up, 8 in 2023-09-23T20:59:53.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:59:53 smithi026 ceph-mon[179688]: failed to encode map e243 with expected crc 2023-09-23T20:59:53.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:59:53 smithi026 ceph-mon[179688]: failed to encode map e243 with expected crc 2023-09-23T20:59:53.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:59:53 smithi026 ceph-mon[179688]: pgmap v605: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 2.5 GiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 697 op/s; 11749/123888 objects degraded (9.484%); 1654/123888 objects misplaced (1.335%); 2.2 MiB/s, 34 objects/s recovering 2023-09-23T20:59:53.633 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:59:53 smithi026 ceph-mon[175151]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:59:53.633 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:59:53 smithi026 ceph-mon[175151]: failed to encode map e243 with expected crc 2023-09-23T20:59:53.633 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:59:53 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/1492873981' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T20:59:53.633 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:59:53 smithi026 ceph-mon[175151]: osdmap e243: 8 total, 8 up, 8 in 2023-09-23T20:59:53.633 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:59:53 smithi026 ceph-mon[175151]: failed to encode map e243 with expected crc 2023-09-23T20:59:53.633 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:59:53 smithi026 ceph-mon[175151]: failed to encode map e243 with expected crc 2023-09-23T20:59:53.633 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:59:53 smithi026 ceph-mon[175151]: pgmap v605: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 2.5 GiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 697 op/s; 11749/123888 objects degraded (9.484%); 1654/123888 objects misplaced (1.335%); 2.2 MiB/s, 34 objects/s recovering 2023-09-23T20:59:54.330 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:59:54.340 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:59:54.340 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:59:54.441 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:59:54.531 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:59:54.721 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:59:54.737 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:59:54.737 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:59:54.838 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:59:54.867 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:59:54.905 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:59:54.909 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:59:54.910 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:59:55.010 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:59:55.110 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:59:55.225 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:59:55.226 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:59:55.227 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:59:55.327 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:59:55.392 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:59:55.428 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:59:55.506 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:59:55.513 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:59:55.518 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:59:55.519 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:59:55.620 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:59:55.651 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:59:55.693 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:59:55.697 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:59:55.697 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:59:55.708 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:59:55 smithi026 ceph-mon[179688]: pgmap v606: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 2.5 GiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 593 op/s; 11749/121569 objects degraded (9.664%); 1654/121569 objects misplaced (1.361%); 1.8 MiB/s, 29 objects/s recovering 2023-09-23T20:59:55.708 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:59:55 smithi026 ceph-mon[179688]: Health check update: Degraded data redundancy: 11749/121569 objects degraded (9.664%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T20:59:55.708 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:59:55 smithi026 ceph-mon[175151]: pgmap v606: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 2.5 GiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 593 op/s; 11749/121569 objects degraded (9.664%); 1654/121569 objects misplaced (1.361%); 1.8 MiB/s, 29 objects/s recovering 2023-09-23T20:59:55.708 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:59:55 smithi026 ceph-mon[175151]: Health check update: Degraded data redundancy: 11749/121569 objects degraded (9.664%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T20:59:55.798 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:59:55.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:59:55 smithi070 ceph-mon[168282]: pgmap v606: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 2.5 GiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 593 op/s; 11749/121569 objects degraded (9.664%); 1654/121569 objects misplaced (1.361%); 1.8 MiB/s, 29 objects/s recovering 2023-09-23T20:59:55.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:59:55 smithi070 ceph-mon[168282]: Health check update: Degraded data redundancy: 11749/121569 objects degraded (9.664%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T20:59:55.893 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:59:56.021 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:59:56.042 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:59:56.042 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:59:56.143 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:59:56.171 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:59:56.213 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:59:56.216 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:59:56.217 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:59:56.281 INFO:teuthology.orchestra.run.smithi026.stdout:ERROR: (22) Invalid argument 2023-09-23T20:59:56.281 INFO:teuthology.orchestra.run.smithi026.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:59:56.317 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:59:56.418 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:59:56.711 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:59:56.717 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:59:56.718 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:59:56.812 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T20:59:56.818 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:59:56.908 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:59:56.995 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:59:57.007 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:59:57.008 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:59:57.108 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:59:57.271 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:59:57.365 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:59:57.376 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:59:57.377 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:59:57.478 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:59:57.509 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_blocked_ops 2023-09-23T20:59:57.576 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:59:57 smithi026 ceph-mon[175151]: pgmap v607: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 2.4 GiB data, 9.7 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 598 op/s; 11749/119802 objects degraded (9.807%); 1654/119802 objects misplaced (1.381%); 1.8 MiB/s, 29 objects/s recovering 2023-09-23T20:59:57.576 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:59:57 smithi026 ceph-mon[179688]: pgmap v607: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 2.4 GiB data, 9.7 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 598 op/s; 11749/119802 objects degraded (9.807%); 1654/119802 objects misplaced (1.381%); 1.8 MiB/s, 29 objects/s recovering 2023-09-23T20:59:57.576 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:59:57.722 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:59:57.736 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:59:57.737 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:59:57.837 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:59:57.862 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:59:57 smithi070 ceph-mon[168282]: pgmap v607: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 2.4 GiB data, 9.7 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 598 op/s; 11749/119802 objects degraded (9.807%); 1654/119802 objects misplaced (1.381%); 1.8 MiB/s, 29 objects/s recovering 2023-09-23T20:59:57.865 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:59:57.900 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:59:57.902 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:59:57.903 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:59:58.004 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:59:58.153 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:59:58.182 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:59:58.228 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0: {} 2023-09-23T20:59:58.251 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:59:58.269 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:59:58.270 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:59:58.297 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1: {} 2023-09-23T20:59:58.343 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2: {} 2023-09-23T20:59:58.370 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:59:58.402 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3: {} 2023-09-23T20:59:58.407 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:59:58.443 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:59:58.445 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:59:58.446 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:59:58.463 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4: {} 2023-09-23T20:59:58.511 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T20:59:58.546 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:59:58.552 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6: {} 2023-09-23T20:59:58.586 INFO:teuthology.orchestra.run.smithi026.stdout: 2023-09-23T20:59:58.591 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T20:59:58.607 INFO:teuthology.orchestra.run.smithi026.stderr:dumped all 2023-09-23T20:59:58.632 INFO:teuthology.orchestra.run.smithi026.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T20:59:58.632 INFO:teuthology.orchestra.run.smithi026.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T20:59:58.632 INFO:teuthology.orchestra.run.smithi026.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T20:59:58.632 INFO:teuthology.orchestra.run.smithi026.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T20:59:58.632 INFO:teuthology.orchestra.run.smithi026.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T20:59:58.632 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T20:59:58.632 INFO:teuthology.orchestra.run.smithi026.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T20:59:58.633 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T20:59:58.661 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:59:58.839 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:59:58 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/2880657692' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:59:58.839 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:59:58 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/2880657692' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:59:58.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:59:58 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/2880657692' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:59:59.192 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:59:59.203 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:59:59.204 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:59:59.304 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:59:59.336 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:59:59.376 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:59:59.381 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:59:59.381 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:59:59.474 INFO:teuthology.orchestra.run.smithi026.stdout:{"pg_ready":true,"pg_map":{"version":608,"stamp":"2023-09-23T20:59:58.470446+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":2504071096,"num_objects":38225,"num_object_clones":0,"num_object_copies":114675,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":11798,"num_objects_misplaced":1452,"num_objects_unfound":0,"num_objects_dirty":38225,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":165380,"num_write_kb":6512606,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3114,"num_bytes_recovered":202643302,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":122355,"ondisk_log_size":122355,"up":327,"acting":324,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":331,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":9825516,"kb_used_data":8143340,"kb_used_omap":136,"kb_used_meta":1681591,"kb_avail":740135700,"statfs":{"total":767960285184,"available":757898956800,"internally_reserved":0,"allocated":8338780160,"data_stored":8330676346,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":139741,"internal_metadata":1721949731},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":9,"apply_latency_ms":9,"commit_latency_ns":9000000,"apply_latency_ns":9000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-398327808,"num_objects":-6078,"num_object_clones":0,"num_object_copies":-18234,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":84,"num_objects_misplaced":-376,"num_objects_unfound":0,"num_objects_dirty":-6078,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6078,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":292,"num_bytes_recovered":19136512,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.011814"},"pg_stats":[{"pgid":"3.1a","version":"68'1","reported_seq":120,"reported_epoch":242,"state":"active+clean","last_fresh":"2023-09-23T20:59:44.080976+0000","last_change":"2023-09-23T20:54:13.946695+0000","last_active":"2023-09-23T20:59:44.080976+0000","last_peered":"2023-09-23T20:59:44.080976+0000","last_clean":"2023-09-23T20:59:44.080976+0000","last_became_active":"2023-09-23T20:54:13.945399+0000","last_became_peered":"2023-09-23T20:54:13.945399+0000","last_unstale":"2023-09-23T20:59:44.080976+0000","last_undegraded":"2023-09-23T20:59:44.080976+0000","last_fullsized":"2023-09-23T20:59:44.080976+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:32:52.365464+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":235,"reported_epoch":242,"state":"active+clean","last_fresh":"2023-09-23T20:59:44.095213+0000","last_change":"2023-09-23T20:54:13.942019+0000","last_active":"2023-09-23T20:59:44.095213+0000","last_peered":"2023-09-23T20:59:44.095213+0000","last_clean":"2023-09-23T20:59:44.095213+0000","last_became_active":"2023-09-23T20:54:13.941647+0000","last_became_peered":"2023-09-23T20:54:13.941647+0000","last_unstale":"2023-09-23T20:59:44.095213+0000","last_undegraded":"2023-09-23T20:59:44.095213+0000","last_fullsized":"2023-09-23T20:59:44.095213+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:30:54.400944+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":213,"reported_epoch":242,"state":"active+clean","last_fresh":"2023-09-23T20:59:44.340387+0000","last_change":"2023-09-23T20:54:13.941394+0000","last_active":"2023-09-23T20:59:44.340387+0000","last_peered":"2023-09-23T20:59:44.340387+0000","last_clean":"2023-09-23T20:59:44.340387+0000","last_became_active":"2023-09-23T20:54:13.941042+0000","last_became_peered":"2023-09-23T20:54:13.941042+0000","last_unstale":"2023-09-23T20:59:44.340387+0000","last_undegraded":"2023-09-23T20:59:44.340387+0000","last_fullsized":"2023-09-23T20:59:44.340387+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:19:41.401184+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":228,"reported_epoch":242,"state":"active+clean","last_fresh":"2023-09-23T20:59:44.081551+0000","last_change":"2023-09-23T20:57:44.979811+0000","last_active":"2023-09-23T20:59:44.081551+0000","last_peered":"2023-09-23T20:59:44.081551+0000","last_clean":"2023-09-23T20:59:44.081551+0000","last_became_active":"2023-09-23T20:57:44.979515+0000","last_became_peered":"2023-09-23T20:57:44.979515+0000","last_unstale":"2023-09-23T20:59:44.081551+0000","last_undegraded":"2023-09-23T20:59:44.081551+0000","last_fullsized":"2023-09-23T20:59:44.081551+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:38:22.162140+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":226,"reported_epoch":242,"state":"active+clean","last_fresh":"2023-09-23T20:59:44.093566+0000","last_change":"2023-09-23T20:51:10.833613+0000","last_active":"2023-09-23T20:59:44.093566+0000","last_peered":"2023-09-23T20:59:44.093566+0000","last_clean":"2023-09-23T20:59:44.093566+0000","last_became_active":"2023-09-23T20:51:10.833208+0000","last_became_peered":"2023-09-23T20:51:10.833208+0000","last_unstale":"2023-09-23T20:59:44.093566+0000","last_undegraded":"2023-09-23T20:59:44.093566+0000","last_fullsized":"2023-09-23T20:59:44.093566+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:51:28.246014+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":208,"reported_epoch":242,"state":"active+clean","last_fresh":"2023-09-23T20:59:44.073083+0000","last_change":"2023-09-23T20:55:35.958384+0000","last_active":"2023-09-23T20:59:44.073083+0000","last_peered":"2023-09-23T20:59:44.073083+0000","last_clean":"2023-09-23T20:59:44.073083+0000","last_became_active":"2023-09-23T20:55:35.954563+0000","last_became_peered":"2023-09-23T20:55:35.954563+0000","last_unstale":"2023-09-23T20:59:44.073083+0000","last_undegraded":"2023-09-23T20:59:44.073083+0000","last_fullsized":"2023-09-23T20:59:44.073083+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:37:48.332259+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":218,"reported_epoch":242,"state":"active+clean","last_fresh":"2023-09-23T20:59:44.079907+0000","last_change":"2023-09-23T20:54:31.885514+0000","last_active":"2023-09-23T20:59:44.079907+0000","last_peered":"2023-09-23T20:59:44.079907+0000","last_clean":"2023-09-23T20:59:44.079907+0000","last_became_active":"2023-09-23T20:54:31.884986+0000","last_became_peered":"2023-09-23T20:54:31.884986+0000","last_unstale":"2023-09-23T20:59:44.079907+0000","last_undegraded":"2023-09-23T20:59:44.079907+0000","last_fullsized":"2023-09-23T20:59:44.079907+0000","mapping_epoch":136,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:26:08.100220+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":223,"reported_epoch":242,"state":"active+clean","last_fresh":"2023-09-23T20:59:44.093635+0000","last_change":"2023-09-23T20:55:35.951889+0000","last_active":"2023-09-23T20:59:44.093635+0000","last_peered":"2023-09-23T20:59:44.093635+0000","last_clean":"2023-09-23T20:59:44.093635+0000","last_became_active":"2023-09-23T20:55:35.951563+0000","last_became_peered":"2023-09-23T20:55:35.951563+0000","last_unstale":"2023-09-23T20:59:44.093635+0000","last_undegraded":"2023-09-23T20:59:44.093635+0000","last_fullsized":"2023-09-23T20:59:44.093635+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:41:53.098781+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"79'161","reported_seq":530,"reported_epoch":242,"state":"active+clean","last_fresh":"2023-09-23T20:59:44.086150+0000","last_change":"2023-09-23T20:57:25.096559+0000","last_active":"2023-09-23T20:59:44.086150+0000","last_peered":"2023-09-23T20:59:44.086150+0000","last_clean":"2023-09-23T20:59:44.086150+0000","last_became_active":"2023-09-23T20:57:15.975674+0000","last_became_peered":"2023-09-23T20:57:15.975674+0000","last_unstale":"2023-09-23T20:59:44.086150+0000","last_undegraded":"2023-09-23T20:59:44.086150+0000","last_fullsized":"2023-09-23T20:59:44.086150+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:25.096401+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:25.096401+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:05:05.448404+0000","scrub_duration":0.002859393,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":116,"reported_epoch":242,"state":"active+clean","last_fresh":"2023-09-23T20:59:44.070257+0000","last_change":"2023-09-23T20:54:31.890678+0000","last_active":"2023-09-23T20:59:44.070257+0000","last_peered":"2023-09-23T20:59:44.070257+0000","last_clean":"2023-09-23T20:59:44.070257+0000","last_became_active":"2023-09-23T20:54:31.890469+0000","last_became_peered":"2023-09-23T20:54:31.890469+0000","last_unstale":"2023-09-23T20:59:44.070257+0000","last_undegraded":"2023-09-23T20:59:44.070257+0000","last_fullsized":"2023-09-23T20:59:44.070257+0000","mapping_epoch":136,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:55:08.790146+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":184,"reported_epoch":242,"state":"active+clean","last_fresh":"2023-09-23T20:59:44.112158+0000","last_change":"2023-09-23T20:55:35.952661+0000","last_active":"2023-09-23T20:59:44.112158+0000","last_peered":"2023-09-23T20:59:44.112158+0000","last_clean":"2023-09-23T20:59:44.112158+0000","last_became_active":"2023-09-23T20:55:35.952349+0000","last_became_peered":"2023-09-23T20:55:35.952349+0000","last_unstale":"2023-09-23T20:59:44.112158+0000","last_undegraded":"2023-09-23T20:59:44.112158+0000","last_fullsized":"2023-09-23T20:59:44.112158+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:00:03.110126+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":61,"reported_epoch":242,"state":"active+clean","last_fresh":"2023-09-23T20:59:44.095308+0000","last_change":"2023-09-23T20:56:45.988607+0000","last_active":"2023-09-23T20:59:44.095308+0000","last_peered":"2023-09-23T20:59:44.095308+0000","last_clean":"2023-09-23T20:59:44.095308+0000","last_became_active":"2023-09-23T20:56:45.988206+0000","last_became_peered":"2023-09-23T20:56:45.988206+0000","last_unstale":"2023-09-23T20:59:44.095308+0000","last_undegraded":"2023-09-23T20:59:44.095308+0000","last_fullsized":"2023-09-23T20:59:44.095308+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":205,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:01:19.783688+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,4],"acting":[3,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":60,"reported_epoch":242,"state":"active+clean","last_fresh":"2023-09-23T20:59:44.070056+0000","last_change":"2023-09-23T20:56:18.139142+0000","last_active":"2023-09-23T20:59:44.070056+0000","last_peered":"2023-09-23T20:59:44.070056+0000","last_clean":"2023-09-23T20:59:44.070056+0000","last_became_active":"2023-09-23T20:56:18.138844+0000","last_became_peered":"2023-09-23T20:56:18.138844+0000","last_unstale":"2023-09-23T20:59:44.070056+0000","last_undegraded":"2023-09-23T20:59:44.070056+0000","last_fullsized":"2023-09-23T20:59:44.070056+0000","mapping_epoch":192,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":193,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:09:02.217683+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,2],"acting":[5,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.14","version":"79'22","reported_seq":147,"reported_epoch":242,"state":"active+clean","last_fresh":"2023-09-23T20:59:44.081191+0000","last_change":"2023-09-23T20:55:35.949418+0000","last_active":"2023-09-23T20:59:44.081191+0000","last_peered":"2023-09-23T20:59:44.081191+0000","last_clean":"2023-09-23T20:59:44.081191+0000","last_became_active":"2023-09-23T20:55:35.948566+0000","last_became_peered":"2023-09-23T20:55:35.948566+0000","last_unstale":"2023-09-23T20:59:44.081191+0000","last_undegraded":"2023-09-23T20:59:44.081191+0000","last_fullsized":"2023-09-23T20:59:44.081191+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:59:37.433128+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4734,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":137,"reported_epoch":242,"state":"active+clean","last_fresh":"2023-09-23T20:59:44.083779+0000","last_change":"2023-09-23T20:55:35.949178+0000","last_active":"2023-09-23T20:59:44.083779+0000","last_peered":"2023-09-23T20:59:44.083779+0000","last_clean":"2023-09-23T20:59:44.083779+0000","last_became_active":"2023-09-23T20:55:35.948795+0000","last_became_peered":"2023-09-23T20:55:35.948795+0000","last_unstale":"2023-09-23T20:59:44.083779+0000","last_undegraded":"2023-09-23T20:59:44.083779+0000","last_fullsized":"2023-09-23T20:59:44.083779+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:13:08.996932+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.11","version":"79'161","reported_seq":472,"reported_epoch":242,"state":"active+clean","last_fresh":"2023-09-23T20:59:44.078608+0000","last_change":"2023-09-23T20:57:28.756796+0000","last_active":"2023-09-23T20:59:44.078608+0000","last_peered":"2023-09-23T20:59:44.078608+0000","last_clean":"2023-09-23T20:59:44.078608+0000","last_became_active":"2023-09-23T20:57:15.980606+0000","last_became_peered":"2023-09-23T20:57:15.980606+0000","last_unstale":"2023-09-23T20:59:44.078608+0000","last_undegraded":"2023-09-23T20:59:44.078608+0000","last_fullsized":"2023-09-23T20:59:44.078608+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:28.756634+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:28.756634+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:09:19.150791+0000","scrub_duration":0.0064925119999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":163,"reported_epoch":242,"state":"active+clean","last_fresh":"2023-09-23T20:59:44.083702+0000","last_change":"2023-09-23T20:56:45.992577+0000","last_active":"2023-09-23T20:59:44.083702+0000","last_peered":"2023-09-23T20:59:44.083702+0000","last_clean":"2023-09-23T20:59:44.083702+0000","last_became_active":"2023-09-23T20:56:45.992297+0000","last_became_peered":"2023-09-23T20:56:45.992297+0000","last_unstale":"2023-09-23T20:59:44.083702+0000","last_undegraded":"2023-09-23T20:59:44.083702+0000","last_fullsized":"2023-09-23T20:59:44.083702+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":205,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:00:45.547202+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,2],"acting":[4,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":208,"reported_epoch":242,"state":"active+clean","last_fresh":"2023-09-23T20:59:44.095991+0000","last_change":"2023-09-23T20:51:10.831262+0000","last_active":"2023-09-23T20:59:44.095991+0000","last_peered":"2023-09-23T20:59:44.095991+0000","last_clean":"2023-09-23T20:59:44.095991+0000","last_became_active":"2023-09-23T20:51:10.830931+0000","last_became_peered":"2023-09-23T20:51:10.830931+0000","last_unstale":"2023-09-23T20:59:44.095991+0000","last_undegraded":"2023-09-23T20:59:44.095991+0000","last_fullsized":"2023-09-23T20:59:44.095991+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:51:54.982505+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.10","version":"79'161","reported_seq":480,"reported_epoch":242,"state":"active+clean","last_fresh":"2023-09-23T20:59:44.069307+0000","last_change":"2023-09-23T20:57:20.708003+0000","last_active":"2023-09-23T20:59:44.069307+0000","last_peered":"2023-09-23T20:59:44.069307+0000","last_clean":"2023-09-23T20:59:44.069307+0000","last_became_active":"2023-09-23T20:57:15.975302+0000","last_became_peered":"2023-09-23T20:57:15.975302+0000","last_unstale":"2023-09-23T20:59:44.069307+0000","last_undegraded":"2023-09-23T20:59:44.069307+0000","last_fullsized":"2023-09-23T20:59:44.069307+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:20.707939+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:20.707939+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:16:51.721994+0000","scrub_duration":0.0017733409999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":218,"reported_epoch":242,"state":"active+clean","last_fresh":"2023-09-23T20:59:44.079001+0000","last_change":"2023-09-23T20:57:44.979736+0000","last_active":"2023-09-23T20:59:44.079001+0000","last_peered":"2023-09-23T20:59:44.079001+0000","last_clean":"2023-09-23T20:59:44.079001+0000","last_became_active":"2023-09-23T20:57:44.979356+0000","last_became_peered":"2023-09-23T20:57:44.979356+0000","last_unstale":"2023-09-23T20:59:44.079001+0000","last_undegraded":"2023-09-23T20:59:44.079001+0000","last_fullsized":"2023-09-23T20:59:44.079001+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:14:43.848832+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":116,"reported_epoch":242,"state":"active+clean","last_fresh":"2023-09-23T20:59:44.069802+0000","last_change":"2023-09-23T20:54:31.887814+0000","last_active":"2023-09-23T20:59:44.069802+0000","last_peered":"2023-09-23T20:59:44.069802+0000","last_clean":"2023-09-23T20:59:44.069802+0000","last_became_active":"2023-09-23T20:54:31.887195+0000","last_became_peered":"2023-09-23T20:54:31.887195+0000","last_unstale":"2023-09-23T20:59:44.069802+0000","last_undegraded":"2023-09-23T20:59:44.069802+0000","last_fullsized":"2023-09-23T20:59:44.069802+0000","mapping_epoch":136,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:10:52.089553+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.13","version":"79'161","reported_seq":543,"reported_epoch":242,"state":"active+clean","last_fresh":"2023-09-23T20:59:44.087164+0000","last_change":"2023-09-23T20:57:48.262970+0000","last_active":"2023-09-23T20:59:44.087164+0000","last_peered":"2023-09-23T20:59:44.087164+0000","last_clean":"2023-09-23T20:59:44.087164+0000","last_became_active":"2023-09-23T20:57:48.262663+0000","last_became_peered":"2023-09-23T20:57:48.262663+0000","last_unstale":"2023-09-23T20:59:44.087164+0000","last_undegraded":"2023-09-23T20:59:44.087164+0000","last_fullsized":"2023-09-23T20:59:44.087164+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:36.420362+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:36.420362+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:28:28.613492+0000","scrub_duration":0.028838444000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":187,"reported_epoch":242,"state":"active+clean","last_fresh":"2023-09-23T20:59:44.084399+0000","last_change":"2023-09-23T20:56:45.991822+0000","last_active":"2023-09-23T20:59:44.084399+0000","last_peered":"2023-09-23T20:59:44.084399+0000","last_clean":"2023-09-23T20:59:44.084399+0000","last_became_active":"2023-09-23T20:56:45.991460+0000","last_became_peered":"2023-09-23T20:56:45.991460+0000","last_unstale":"2023-09-23T20:59:44.084399+0000","last_undegraded":"2023-09-23T20:59:44.084399+0000","last_fullsized":"2023-09-23T20:59:44.084399+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":205,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:20:37.503317+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":200,"reported_epoch":242,"state":"active+clean","last_fresh":"2023-09-23T20:59:44.081668+0000","last_change":"2023-09-23T20:51:10.840849+0000","last_active":"2023-09-23T20:59:44.081668+0000","last_peered":"2023-09-23T20:59:44.081668+0000","last_clean":"2023-09-23T20:59:44.081668+0000","last_became_active":"2023-09-23T20:51:10.840593+0000","last_became_peered":"2023-09-23T20:51:10.840593+0000","last_unstale":"2023-09-23T20:59:44.081668+0000","last_undegraded":"2023-09-23T20:59:44.081668+0000","last_fullsized":"2023-09-23T20:59:44.081668+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:30:52.297625+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.12","version":"79'161","reported_seq":480,"reported_epoch":242,"state":"active+clean","last_fresh":"2023-09-23T20:59:44.069802+0000","last_change":"2023-09-23T20:57:21.750765+0000","last_active":"2023-09-23T20:59:44.069802+0000","last_peered":"2023-09-23T20:59:44.069802+0000","last_clean":"2023-09-23T20:59:44.069802+0000","last_became_active":"2023-09-23T20:57:15.979195+0000","last_became_peered":"2023-09-23T20:57:15.979195+0000","last_unstale":"2023-09-23T20:59:44.069802+0000","last_undegraded":"2023-09-23T20:59:44.069802+0000","last_fullsized":"2023-09-23T20:59:44.069802+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:21.750680+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:21.750680+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:03:41.915962+0000","scrub_duration":0.0092391680000000007,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":235,"reported_epoch":242,"state":"active+clean","last_fresh":"2023-09-23T20:59:44.072593+0000","last_change":"2023-09-23T20:55:36.943969+0000","last_active":"2023-09-23T20:59:44.072593+0000","last_peered":"2023-09-23T20:59:44.072593+0000","last_clean":"2023-09-23T20:59:44.072593+0000","last_became_active":"2023-09-23T20:55:36.943580+0000","last_became_peered":"2023-09-23T20:55:36.943580+0000","last_unstale":"2023-09-23T20:59:44.072593+0000","last_undegraded":"2023-09-23T20:59:44.072593+0000","last_fullsized":"2023-09-23T20:59:44.072593+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:49:06.969396+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":31,"reported_epoch":242,"state":"active+clean","last_fresh":"2023-09-23T20:59:44.102752+0000","last_change":"2023-09-23T20:57:44.980498+0000","last_active":"2023-09-23T20:59:44.102752+0000","last_peered":"2023-09-23T20:59:44.102752+0000","last_clean":"2023-09-23T20:59:44.102752+0000","last_became_active":"2023-09-23T20:57:44.980182+0000","last_became_peered":"2023-09-23T20:57:44.980182+0000","last_unstale":"2023-09-23T20:59:44.102752+0000","last_undegraded":"2023-09-23T20:59:44.102752+0000","last_fullsized":"2023-09-23T20:59:44.102752+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:34:51.620544+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.d","version":"79'161","reported_seq":525,"reported_epoch":242,"state":"active+clean","last_fresh":"2023-09-23T20:59:44.081318+0000","last_change":"2023-09-23T20:57:31.075213+0000","last_active":"2023-09-23T20:59:44.081318+0000","last_peered":"2023-09-23T20:59:44.081318+0000","last_clean":"2023-09-23T20:59:44.081318+0000","last_became_active":"2023-09-23T20:57:15.988188+0000","last_became_peered":"2023-09-23T20:57:15.988188+0000","last_unstale":"2023-09-23T20:59:44.081318+0000","last_undegraded":"2023-09-23T20:59:44.081318+0000","last_fullsized":"2023-09-23T20:59:44.081318+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:31.075098+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:31.075098+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:41:52.142362+0000","scrub_duration":0.010304610000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.b","version":"243'10190","reported_seq":10243,"reported_epoch":243,"state":"active+clean","last_fresh":"2023-09-23T20:59:54.291299+0000","last_change":"2023-09-23T20:56:35.543994+0000","last_active":"2023-09-23T20:59:54.291299+0000","last_peered":"2023-09-23T20:59:54.291299+0000","last_clean":"2023-09-23T20:59:54.291299+0000","last_became_active":"2023-09-23T20:56:35.543758+0000","last_became_peered":"2023-09-23T20:56:35.543758+0000","last_unstale":"2023-09-23T20:59:54.291299+0000","last_undegraded":"2023-09-23T20:59:54.291299+0000","last_fullsized":"2023-09-23T20:59:54.291299+0000","mapping_epoch":199,"log_start":"210'3200","ondisk_log_start":"210'3200","created":199,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_clean_scrub_stamp":"2023-09-23T20:56:34.529103+0000","objects_scrubbed":0,"log_size":6990,"log_dups_size":0,"ondisk_log_size":6990,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:21:29.041484+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":163184640,"num_objects":2490,"num_object_clones":0,"num_object_copies":7470,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2490,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10190,"num_write_kb":405760,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":174,"reported_epoch":242,"state":"active+clean","last_fresh":"2023-09-23T20:59:44.083465+0000","last_change":"2023-09-23T20:56:45.991841+0000","last_active":"2023-09-23T20:59:44.083465+0000","last_peered":"2023-09-23T20:59:44.083465+0000","last_clean":"2023-09-23T20:59:44.083465+0000","last_became_active":"2023-09-23T20:56:45.991491+0000","last_became_peered":"2023-09-23T20:56:45.991491+0000","last_unstale":"2023-09-23T20:59:44.083465+0000","last_undegraded":"2023-09-23T20:59:44.083465+0000","last_fullsized":"2023-09-23T20:59:44.083465+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":205,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:30:37.405319+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":133,"reported_epoch":242,"state":"active+clean","last_fresh":"2023-09-23T20:59:44.069473+0000","last_change":"2023-09-23T20:55:35.957768+0000","last_active":"2023-09-23T20:59:44.069473+0000","last_peered":"2023-09-23T20:59:44.069473+0000","last_clean":"2023-09-23T20:59:44.069473+0000","last_became_active":"2023-09-23T20:55:35.957421+0000","last_became_peered":"2023-09-23T20:55:35.957421+0000","last_unstale":"2023-09-23T20:59:44.069473+0000","last_undegraded":"2023-09-23T20:59:44.069473+0000","last_fullsized":"2023-09-23T20:59:44.069473+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:49:41.585961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.c","version":"79'161","reported_seq":544,"reported_epoch":242,"state":"active+clean","last_fresh":"2023-09-23T20:59:44.095833+0000","last_change":"2023-09-23T20:57:22.041071+0000","last_active":"2023-09-23T20:59:44.095833+0000","last_peered":"2023-09-23T20:59:44.095833+0000","last_clean":"2023-09-23T20:59:44.095833+0000","last_became_active":"2023-09-23T20:57:15.973204+0000","last_became_peered":"2023-09-23T20:57:15.973204+0000","last_unstale":"2023-09-23T20:59:44.095833+0000","last_undegraded":"2023-09-23T20:59:44.095833+0000","last_fullsized":"2023-09-23T20:59:44.095833+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:22.040907+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:22.040907+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:47:02.019978+0000","scrub_duration":0.013021176000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.a","version":"243'10266","reported_seq":10320,"reported_epoch":243,"state":"active+clean","last_fresh":"2023-09-23T20:59:54.618251+0000","last_change":"2023-09-23T20:56:35.547120+0000","last_active":"2023-09-23T20:59:54.618251+0000","last_peered":"2023-09-23T20:59:54.618251+0000","last_clean":"2023-09-23T20:59:54.618251+0000","last_became_active":"2023-09-23T20:56:35.546838+0000","last_became_peered":"2023-09-23T20:56:35.546838+0000","last_unstale":"2023-09-23T20:59:54.618251+0000","last_undegraded":"2023-09-23T20:59:54.618251+0000","last_fullsized":"2023-09-23T20:59:54.618251+0000","mapping_epoch":199,"log_start":"217'4100","ondisk_log_start":"217'4100","created":199,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_clean_scrub_stamp":"2023-09-23T20:56:34.529103+0000","objects_scrubbed":0,"log_size":6166,"log_dups_size":0,"ondisk_log_size":6166,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:48:24.132768+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":161873920,"num_objects":2470,"num_object_clones":0,"num_object_copies":7410,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2470,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10266,"num_write_kb":407552,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,5],"acting":[3,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":161,"reported_epoch":242,"state":"active+clean","last_fresh":"2023-09-23T20:59:44.081553+0000","last_change":"2023-09-23T20:56:45.991821+0000","last_active":"2023-09-23T20:59:44.081553+0000","last_peered":"2023-09-23T20:59:44.081553+0000","last_clean":"2023-09-23T20:59:44.081553+0000","last_became_active":"2023-09-23T20:56:45.991447+0000","last_became_peered":"2023-09-23T20:56:45.991447+0000","last_unstale":"2023-09-23T20:59:44.081553+0000","last_undegraded":"2023-09-23T20:59:44.081553+0000","last_fullsized":"2023-09-23T20:59:44.081553+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":205,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:14:25.525222+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":164,"reported_epoch":242,"state":"active+clean","last_fresh":"2023-09-23T20:59:44.084690+0000","last_change":"2023-09-23T20:57:44.982059+0000","last_active":"2023-09-23T20:59:44.084690+0000","last_peered":"2023-09-23T20:59:44.084690+0000","last_clean":"2023-09-23T20:59:44.084690+0000","last_became_active":"2023-09-23T20:57:44.981742+0000","last_became_peered":"2023-09-23T20:57:44.981742+0000","last_unstale":"2023-09-23T20:59:44.084690+0000","last_undegraded":"2023-09-23T20:59:44.084690+0000","last_fullsized":"2023-09-23T20:59:44.084690+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:41:24.042916+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.f","version":"79'161","reported_seq":547,"reported_epoch":242,"state":"active+clean","last_fresh":"2023-09-23T20:59:44.072906+0000","last_change":"2023-09-23T20:57:27.880841+0000","last_active":"2023-09-23T20:59:44.072906+0000","last_peered":"2023-09-23T20:59:44.072906+0000","last_clean":"2023-09-23T20:59:44.072906+0000","last_became_active":"2023-09-23T20:57:15.970925+0000","last_became_peered":"2023-09-23T20:57:15.970925+0000","last_unstale":"2023-09-23T20:59:44.072906+0000","last_undegraded":"2023-09-23T20:59:44.072906+0000","last_fullsized":"2023-09-23T20:59:44.072906+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:27.880776+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:27.880776+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:15:13.438014+0000","scrub_duration":0.0014004899999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.9","version":"243'10363","reported_seq":10417,"reported_epoch":243,"state":"active+clean","last_fresh":"2023-09-23T20:59:56.688864+0000","last_change":"2023-09-23T20:56:35.549387+0000","last_active":"2023-09-23T20:59:56.688864+0000","last_peered":"2023-09-23T20:59:56.688864+0000","last_clean":"2023-09-23T20:59:56.688864+0000","last_became_active":"2023-09-23T20:56:35.549087+0000","last_became_peered":"2023-09-23T20:56:35.549087+0000","last_unstale":"2023-09-23T20:59:56.688864+0000","last_undegraded":"2023-09-23T20:59:56.688864+0000","last_fullsized":"2023-09-23T20:59:56.688864+0000","mapping_epoch":199,"log_start":"207'3000","ondisk_log_start":"207'3000","created":199,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_clean_scrub_stamp":"2023-09-23T20:56:34.529103+0000","objects_scrubbed":0,"log_size":7363,"log_dups_size":0,"ondisk_log_size":7363,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:04:51.547362+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":153550848,"num_objects":2343,"num_object_clones":0,"num_object_copies":7029,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2343,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10363,"num_write_kb":406592,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":147,"reported_epoch":242,"state":"active+clean","last_fresh":"2023-09-23T20:59:44.069425+0000","last_change":"2023-09-23T20:56:45.989153+0000","last_active":"2023-09-23T20:59:44.069425+0000","last_peered":"2023-09-23T20:59:44.069425+0000","last_clean":"2023-09-23T20:59:44.069425+0000","last_became_active":"2023-09-23T20:56:45.988800+0000","last_became_peered":"2023-09-23T20:56:45.988800+0000","last_unstale":"2023-09-23T20:59:44.069425+0000","last_undegraded":"2023-09-23T20:59:44.069425+0000","last_fullsized":"2023-09-23T20:59:44.069425+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":205,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:04:41.130194+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":182,"reported_epoch":242,"state":"active+clean","last_fresh":"2023-09-23T20:59:44.072752+0000","last_change":"2023-09-23T20:55:35.965427+0000","last_active":"2023-09-23T20:59:44.072752+0000","last_peered":"2023-09-23T20:59:44.072752+0000","last_clean":"2023-09-23T20:59:44.072752+0000","last_became_active":"2023-09-23T20:55:35.963969+0000","last_became_peered":"2023-09-23T20:55:35.963969+0000","last_unstale":"2023-09-23T20:59:44.072752+0000","last_undegraded":"2023-09-23T20:59:44.072752+0000","last_fullsized":"2023-09-23T20:59:44.072752+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:38:40.612083+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.e","version":"79'161","reported_seq":454,"reported_epoch":242,"state":"active+clean","last_fresh":"2023-09-23T20:59:44.081437+0000","last_change":"2023-09-23T20:57:34.132738+0000","last_active":"2023-09-23T20:59:44.081437+0000","last_peered":"2023-09-23T20:59:44.081437+0000","last_clean":"2023-09-23T20:59:44.081437+0000","last_became_active":"2023-09-23T20:57:15.974318+0000","last_became_peered":"2023-09-23T20:57:15.974318+0000","last_unstale":"2023-09-23T20:59:44.081437+0000","last_undegraded":"2023-09-23T20:59:44.081437+0000","last_fullsized":"2023-09-23T20:59:44.081437+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:34.132660+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:34.132660+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:07:07.739438+0000","scrub_duration":0.005229694,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.8","version":"243'10461","reported_seq":10515,"reported_epoch":243,"state":"active+clean","last_fresh":"2023-09-23T20:59:56.459656+0000","last_change":"2023-09-23T20:56:35.548119+0000","last_active":"2023-09-23T20:59:56.459656+0000","last_peered":"2023-09-23T20:59:56.459656+0000","last_clean":"2023-09-23T20:59:56.459656+0000","last_became_active":"2023-09-23T20:56:35.547914+0000","last_became_peered":"2023-09-23T20:56:35.547914+0000","last_unstale":"2023-09-23T20:59:56.459656+0000","last_undegraded":"2023-09-23T20:59:56.459656+0000","last_fullsized":"2023-09-23T20:59:56.459656+0000","mapping_epoch":199,"log_start":"205'1600","ondisk_log_start":"205'1600","created":199,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_clean_scrub_stamp":"2023-09-23T20:56:34.529103+0000","objects_scrubbed":0,"log_size":8861,"log_dups_size":0,"ondisk_log_size":8861,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:42:54.255023+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":156827648,"num_objects":2393,"num_object_clones":0,"num_object_copies":7179,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2393,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10461,"num_write_kb":411328,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":133,"reported_epoch":242,"state":"active+clean","last_fresh":"2023-09-23T20:59:44.069606+0000","last_change":"2023-09-23T20:55:35.962005+0000","last_active":"2023-09-23T20:59:44.069606+0000","last_peered":"2023-09-23T20:59:44.069606+0000","last_clean":"2023-09-23T20:59:44.069606+0000","last_became_active":"2023-09-23T20:55:35.961695+0000","last_became_peered":"2023-09-23T20:55:35.961695+0000","last_unstale":"2023-09-23T20:59:44.069606+0000","last_undegraded":"2023-09-23T20:59:44.069606+0000","last_fullsized":"2023-09-23T20:59:44.069606+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:28:22.876201+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":231,"reported_epoch":242,"state":"active+clean","last_fresh":"2023-09-23T20:59:44.079135+0000","last_change":"2023-09-23T20:54:13.928715+0000","last_active":"2023-09-23T20:59:44.079135+0000","last_peered":"2023-09-23T20:59:44.079135+0000","last_clean":"2023-09-23T20:59:44.079135+0000","last_became_active":"2023-09-23T20:54:13.928200+0000","last_became_peered":"2023-09-23T20:54:13.928200+0000","last_unstale":"2023-09-23T20:59:44.079135+0000","last_undegraded":"2023-09-23T20:59:44.079135+0000","last_fullsized":"2023-09-23T20:59:44.079135+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:41:16.927513+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.f","version":"243'10530","reported_seq":10584,"reported_epoch":243,"state":"active+clean","last_fresh":"2023-09-23T20:59:56.709524+0000","last_change":"2023-09-23T20:56:35.545128+0000","last_active":"2023-09-23T20:59:56.709524+0000","last_peered":"2023-09-23T20:59:56.709524+0000","last_clean":"2023-09-23T20:59:56.709524+0000","last_became_active":"2023-09-23T20:56:35.544838+0000","last_became_peered":"2023-09-23T20:56:35.544838+0000","last_unstale":"2023-09-23T20:59:56.709524+0000","last_undegraded":"2023-09-23T20:59:56.709524+0000","last_fullsized":"2023-09-23T20:59:56.709524+0000","mapping_epoch":199,"log_start":"208'3200","ondisk_log_start":"208'3200","created":199,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_clean_scrub_stamp":"2023-09-23T20:56:34.529103+0000","objects_scrubbed":0,"log_size":7330,"log_dups_size":0,"ondisk_log_size":7330,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:30:43.388717+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":153092096,"num_objects":2336,"num_object_clones":0,"num_object_copies":7008,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2336,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10530,"num_write_kb":411712,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.9","version":"79'161","reported_seq":616,"reported_epoch":242,"state":"active+clean","last_fresh":"2023-09-23T20:59:44.112076+0000","last_change":"2023-09-23T20:56:46.921939+0000","last_active":"2023-09-23T20:59:44.112076+0000","last_peered":"2023-09-23T20:59:44.112076+0000","last_clean":"2023-09-23T20:59:44.112076+0000","last_became_active":"2023-09-23T20:56:45.989908+0000","last_became_peered":"2023-09-23T20:56:45.989908+0000","last_unstale":"2023-09-23T20:59:44.112076+0000","last_undegraded":"2023-09-23T20:59:44.112076+0000","last_fullsized":"2023-09-23T20:59:44.112076+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":205,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:56:46.921739+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:56:46.921739+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:20:42.635110+0000","scrub_duration":0.0071954109999999996,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":232,"reported_epoch":242,"state":"active+clean","last_fresh":"2023-09-23T20:59:44.098621+0000","last_change":"2023-09-23T20:54:13.961788+0000","last_active":"2023-09-23T20:59:44.098621+0000","last_peered":"2023-09-23T20:59:44.098621+0000","last_clean":"2023-09-23T20:59:44.098621+0000","last_became_active":"2023-09-23T20:54:13.940937+0000","last_became_peered":"2023-09-23T20:54:13.940937+0000","last_unstale":"2023-09-23T20:59:44.098621+0000","last_undegraded":"2023-09-23T20:59:44.098621+0000","last_fullsized":"2023-09-23T20:59:44.098621+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:11:56.785299+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":137,"reported_epoch":242,"state":"active+clean","last_fresh":"2023-09-23T20:59:44.081361+0000","last_change":"2023-09-23T20:55:35.949670+0000","last_active":"2023-09-23T20:59:44.081361+0000","last_peered":"2023-09-23T20:59:44.081361+0000","last_clean":"2023-09-23T20:59:44.081361+0000","last_became_active":"2023-09-23T20:55:35.949140+0000","last_became_peered":"2023-09-23T20:55:35.949140+0000","last_unstale":"2023-09-23T20:59:44.081361+0000","last_undegraded":"2023-09-23T20:59:44.081361+0000","last_fullsized":"2023-09-23T20:59:44.081361+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:55:59.340303+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.e","version":"243'10424","reported_seq":10478,"reported_epoch":243,"state":"active+clean","last_fresh":"2023-09-23T20:59:56.454814+0000","last_change":"2023-09-23T20:56:35.547184+0000","last_active":"2023-09-23T20:59:56.454814+0000","last_peered":"2023-09-23T20:59:56.454814+0000","last_clean":"2023-09-23T20:59:56.454814+0000","last_became_active":"2023-09-23T20:56:35.546912+0000","last_became_peered":"2023-09-23T20:56:35.546912+0000","last_unstale":"2023-09-23T20:59:56.454814+0000","last_undegraded":"2023-09-23T20:59:56.454814+0000","last_fullsized":"2023-09-23T20:59:56.454814+0000","mapping_epoch":199,"log_start":"205'1600","ondisk_log_start":"205'1600","created":199,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_clean_scrub_stamp":"2023-09-23T20:56:34.529103+0000","objects_scrubbed":0,"log_size":8824,"log_dups_size":0,"ondisk_log_size":8824,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:57:07.422792+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":154402816,"num_objects":2356,"num_object_clones":0,"num_object_copies":7068,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2356,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10424,"num_write_kb":408960,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,0],"acting":[2,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.8","version":"79'161","reported_seq":486,"reported_epoch":242,"state":"active+clean","last_fresh":"2023-09-23T20:59:44.069400+0000","last_change":"2023-09-23T20:56:57.505737+0000","last_active":"2023-09-23T20:59:44.069400+0000","last_peered":"2023-09-23T20:59:44.069400+0000","last_clean":"2023-09-23T20:59:44.069400+0000","last_became_active":"2023-09-23T20:56:04.982345+0000","last_became_peered":"2023-09-23T20:56:04.982345+0000","last_unstale":"2023-09-23T20:59:44.069400+0000","last_undegraded":"2023-09-23T20:59:44.069400+0000","last_fullsized":"2023-09-23T20:59:44.069400+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:56:57.505663+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:56:57.505663+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:34:30.109724+0000","scrub_duration":0.026284108,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":205,"reported_epoch":242,"state":"active+clean","last_fresh":"2023-09-23T20:59:44.072863+0000","last_change":"2023-09-23T20:55:35.959942+0000","last_active":"2023-09-23T20:59:44.072863+0000","last_peered":"2023-09-23T20:59:44.072863+0000","last_clean":"2023-09-23T20:59:44.072863+0000","last_became_active":"2023-09-23T20:55:35.957071+0000","last_became_peered":"2023-09-23T20:55:35.957071+0000","last_unstale":"2023-09-23T20:59:44.072863+0000","last_undegraded":"2023-09-23T20:59:44.072863+0000","last_fullsized":"2023-09-23T20:59:44.072863+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:44:30.308036+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":120,"reported_epoch":242,"state":"active+clean","last_fresh":"2023-09-23T20:59:44.081659+0000","last_change":"2023-09-23T20:54:13.951459+0000","last_active":"2023-09-23T20:59:44.081659+0000","last_peered":"2023-09-23T20:59:44.081659+0000","last_clean":"2023-09-23T20:59:44.081659+0000","last_became_active":"2023-09-23T20:54:13.948409+0000","last_became_peered":"2023-09-23T20:54:13.948409+0000","last_unstale":"2023-09-23T20:59:44.081659+0000","last_undegraded":"2023-09-23T20:59:44.081659+0000","last_fullsized":"2023-09-23T20:59:44.081659+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:38:10.148061+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.b","version":"79'161","reported_seq":526,"reported_epoch":242,"state":"active+clean","last_fresh":"2023-09-23T20:59:44.087257+0000","last_change":"2023-09-23T20:57:27.320172+0000","last_active":"2023-09-23T20:59:44.087257+0000","last_peered":"2023-09-23T20:59:44.087257+0000","last_clean":"2023-09-23T20:59:44.087257+0000","last_became_active":"2023-09-23T20:57:16.009630+0000","last_became_peered":"2023-09-23T20:57:16.009630+0000","last_unstale":"2023-09-23T20:59:44.087257+0000","last_undegraded":"2023-09-23T20:59:44.087257+0000","last_fullsized":"2023-09-23T20:59:44.087257+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:27.320022+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:27.320022+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:31:35.221558+0000","scrub_duration":0.0043894679999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,5],"acting":[1,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.d","version":"243'10363","reported_seq":10416,"reported_epoch":243,"state":"active+clean","last_fresh":"2023-09-23T20:59:54.027505+0000","last_change":"2023-09-23T20:56:35.544132+0000","last_active":"2023-09-23T20:59:54.027505+0000","last_peered":"2023-09-23T20:59:54.027505+0000","last_clean":"2023-09-23T20:59:54.027505+0000","last_became_active":"2023-09-23T20:56:35.543836+0000","last_became_peered":"2023-09-23T20:56:35.543836+0000","last_unstale":"2023-09-23T20:59:54.027505+0000","last_undegraded":"2023-09-23T20:59:54.027505+0000","last_fullsized":"2023-09-23T20:59:54.027505+0000","mapping_epoch":199,"log_start":"207'3000","ondisk_log_start":"207'3000","created":199,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_clean_scrub_stamp":"2023-09-23T20:56:34.529103+0000","objects_scrubbed":0,"log_size":7363,"log_dups_size":0,"ondisk_log_size":7363,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:27:14.636991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":158990360,"num_objects":2427,"num_object_clones":0,"num_object_copies":7281,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2427,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":10363,"num_write_kb":409217,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":120,"reported_epoch":242,"state":"active+clean","last_fresh":"2023-09-23T20:59:44.084478+0000","last_change":"2023-09-23T20:54:13.950254+0000","last_active":"2023-09-23T20:59:44.084478+0000","last_peered":"2023-09-23T20:59:44.084478+0000","last_clean":"2023-09-23T20:59:44.084478+0000","last_became_active":"2023-09-23T20:54:13.949934+0000","last_became_peered":"2023-09-23T20:54:13.949934+0000","last_unstale":"2023-09-23T20:59:44.084478+0000","last_undegraded":"2023-09-23T20:59:44.084478+0000","last_fullsized":"2023-09-23T20:59:44.084478+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:30:32.847167+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":137,"reported_epoch":242,"state":"active+clean","last_fresh":"2023-09-23T20:59:44.081615+0000","last_change":"2023-09-23T20:55:35.956153+0000","last_active":"2023-09-23T20:59:44.081615+0000","last_peered":"2023-09-23T20:59:44.081615+0000","last_clean":"2023-09-23T20:59:44.081615+0000","last_became_active":"2023-09-23T20:55:35.955630+0000","last_became_peered":"2023-09-23T20:55:35.955630+0000","last_unstale":"2023-09-23T20:59:44.081615+0000","last_undegraded":"2023-09-23T20:59:44.081615+0000","last_fullsized":"2023-09-23T20:59:44.081615+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:45:20.282929+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.c","version":"243'10299","reported_seq":10359,"reported_epoch":243,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T20:59:54.067263+0000","last_change":"2023-09-23T20:57:46.463005+0000","last_active":"2023-09-23T20:59:54.067263+0000","last_peered":"2023-09-23T20:59:54.067263+0000","last_clean":"2023-09-23T20:57:44.015119+0000","last_became_active":"2023-09-23T20:57:46.448702+0000","last_became_peered":"2023-09-23T20:57:46.448702+0000","last_unstale":"2023-09-23T20:59:54.067263+0000","last_undegraded":"2023-09-23T20:57:46.010874+0000","last_fullsized":"2023-09-23T20:57:45.978434+0000","mapping_epoch":223,"log_start":"207'2900","ondisk_log_start":"207'2900","created":199,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_clean_scrub_stamp":"2023-09-23T20:56:34.529103+0000","objects_scrubbed":0,"log_size":7399,"log_dups_size":0,"ondisk_log_size":7399,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:08:34.180651+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":164429824,"num_objects":2509,"num_object_clones":0,"num_object_copies":7527,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5473,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2509,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10299,"num_write_kb":409856,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,7],"avail_no_missing":["6","2","7"],"object_location_counts":[{"shards":"2,6,7","objects":1578},{"shards":"6,7","objects":-2964}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.a","version":"79'161","reported_seq":486,"reported_epoch":242,"state":"active+clean","last_fresh":"2023-09-23T20:59:44.070176+0000","last_change":"2023-09-23T20:56:44.487778+0000","last_active":"2023-09-23T20:59:44.070176+0000","last_peered":"2023-09-23T20:59:44.070176+0000","last_clean":"2023-09-23T20:59:44.070176+0000","last_became_active":"2023-09-23T20:56:05.006740+0000","last_became_peered":"2023-09-23T20:56:05.006740+0000","last_unstale":"2023-09-23T20:59:44.070176+0000","last_undegraded":"2023-09-23T20:59:44.070176+0000","last_fullsized":"2023-09-23T20:59:44.070176+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:56:44.487678+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:56:44.487678+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:04:50.086749+0000","scrub_duration":0.0017408899999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":32,"reported_epoch":243,"state":"active+clean","last_fresh":"2023-09-23T20:59:52.337330+0000","last_change":"2023-09-23T20:57:44.982513+0000","last_active":"2023-09-23T20:59:52.337330+0000","last_peered":"2023-09-23T20:59:52.337330+0000","last_clean":"2023-09-23T20:59:52.337330+0000","last_became_active":"2023-09-23T20:57:44.982112+0000","last_became_peered":"2023-09-23T20:57:44.982112+0000","last_unstale":"2023-09-23T20:59:52.337330+0000","last_undegraded":"2023-09-23T20:59:52.337330+0000","last_fullsized":"2023-09-23T20:59:52.337330+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:09:05.630041+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":213,"reported_epoch":242,"state":"active+clean","last_fresh":"2023-09-23T20:59:44.093971+0000","last_change":"2023-09-23T20:54:13.940748+0000","last_active":"2023-09-23T20:59:44.093971+0000","last_peered":"2023-09-23T20:59:44.093971+0000","last_clean":"2023-09-23T20:59:44.093971+0000","last_became_active":"2023-09-23T20:54:13.940419+0000","last_became_peered":"2023-09-23T20:54:13.940419+0000","last_unstale":"2023-09-23T20:59:44.093971+0000","last_undegraded":"2023-09-23T20:59:44.093971+0000","last_fullsized":"2023-09-23T20:59:44.093971+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:15:01.923816+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.1","version":"243'10410","reported_seq":10475,"reported_epoch":243,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T20:59:57.194421+0000","last_change":"2023-09-23T20:57:46.483248+0000","last_active":"2023-09-23T20:59:57.194421+0000","last_peered":"2023-09-23T20:59:57.194421+0000","last_clean":"2023-09-23T20:57:44.047957+0000","last_became_active":"2023-09-23T20:57:46.475568+0000","last_became_peered":"2023-09-23T20:57:46.475568+0000","last_unstale":"2023-09-23T20:59:57.194421+0000","last_undegraded":"2023-09-23T20:57:46.083591+0000","last_fullsized":"2023-09-23T20:57:45.978436+0000","mapping_epoch":223,"log_start":"214'3700","ondisk_log_start":"214'3700","created":199,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_clean_scrub_stamp":"2023-09-23T20:56:34.529103+0000","objects_scrubbed":0,"log_size":6710,"log_dups_size":0,"ondisk_log_size":6710,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:22:45.067269+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":155713536,"num_objects":2376,"num_object_clones":0,"num_object_copies":7128,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5470,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2376,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10410,"num_write_kb":409152,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,6],"avail_no_missing":["4","1","6"],"object_location_counts":[{"shards":"1,4,6","objects":1453},{"shards":"4,6","objects":-3094}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.7","version":"79'161","reported_seq":548,"reported_epoch":242,"state":"active+clean","last_fresh":"2023-09-23T20:59:44.073129+0000","last_change":"2023-09-23T20:57:48.243572+0000","last_active":"2023-09-23T20:59:44.073129+0000","last_peered":"2023-09-23T20:59:44.073129+0000","last_clean":"2023-09-23T20:59:44.073129+0000","last_became_active":"2023-09-23T20:57:48.243229+0000","last_became_peered":"2023-09-23T20:57:48.243229+0000","last_unstale":"2023-09-23T20:59:44.073129+0000","last_undegraded":"2023-09-23T20:59:44.073129+0000","last_fullsized":"2023-09-23T20:59:44.073129+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":3,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:25.916867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:25.916867+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:23:19.385831+0000","scrub_duration":0.001665323,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":218,"reported_epoch":242,"state":"active+clean","last_fresh":"2023-09-23T20:59:44.088857+0000","last_change":"2023-09-23T20:50:05.867130+0000","last_active":"2023-09-23T20:59:44.088857+0000","last_peered":"2023-09-23T20:59:44.088857+0000","last_clean":"2023-09-23T20:59:44.088857+0000","last_became_active":"2023-09-23T20:50:05.866758+0000","last_became_peered":"2023-09-23T20:50:05.866758+0000","last_unstale":"2023-09-23T20:59:44.088857+0000","last_undegraded":"2023-09-23T20:59:44.088857+0000","last_fullsized":"2023-09-23T20:59:44.088857+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:55:07.548209+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":116,"reported_epoch":242,"state":"active+clean","last_fresh":"2023-09-23T20:59:44.069875+0000","last_change":"2023-09-23T20:54:31.887604+0000","last_active":"2023-09-23T20:59:44.069875+0000","last_peered":"2023-09-23T20:59:44.069875+0000","last_clean":"2023-09-23T20:59:44.069875+0000","last_became_active":"2023-09-23T20:54:31.886752+0000","last_became_peered":"2023-09-23T20:54:31.886752+0000","last_unstale":"2023-09-23T20:59:44.069875+0000","last_undegraded":"2023-09-23T20:59:44.069875+0000","last_fullsized":"2023-09-23T20:59:44.069875+0000","mapping_epoch":136,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:13:58.872176+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":216,"reported_epoch":242,"state":"active+clean","last_fresh":"2023-09-23T20:59:44.088107+0000","last_change":"2023-09-23T20:54:31.899082+0000","last_active":"2023-09-23T20:59:44.088107+0000","last_peered":"2023-09-23T20:59:44.088107+0000","last_clean":"2023-09-23T20:59:44.088107+0000","last_became_active":"2023-09-23T20:54:31.898521+0000","last_became_peered":"2023-09-23T20:54:31.898521+0000","last_unstale":"2023-09-23T20:59:44.088107+0000","last_undegraded":"2023-09-23T20:59:44.088107+0000","last_fullsized":"2023-09-23T20:59:44.088107+0000","mapping_epoch":136,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:25:36.059711+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.7","version":"243'10382","reported_seq":10434,"reported_epoch":243,"state":"active+clean","last_fresh":"2023-09-23T20:59:54.055146+0000","last_change":"2023-09-23T20:56:35.541695+0000","last_active":"2023-09-23T20:59:54.055146+0000","last_peered":"2023-09-23T20:59:54.055146+0000","last_clean":"2023-09-23T20:59:54.055146+0000","last_became_active":"2023-09-23T20:56:35.541392+0000","last_became_peered":"2023-09-23T20:56:35.541392+0000","last_unstale":"2023-09-23T20:59:54.055146+0000","last_undegraded":"2023-09-23T20:59:54.055146+0000","last_fullsized":"2023-09-23T20:59:54.055146+0000","mapping_epoch":199,"log_start":"207'3000","ondisk_log_start":"207'3000","created":199,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_clean_scrub_stamp":"2023-09-23T20:56:34.529103+0000","objects_scrubbed":0,"log_size":7382,"log_dups_size":0,"ondisk_log_size":7382,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:57:22.478253+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":159121408,"num_objects":2428,"num_object_clones":0,"num_object_copies":7284,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2428,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10382,"num_write_kb":409920,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.1","version":"79'161","reported_seq":506,"reported_epoch":242,"state":"active+clean","last_fresh":"2023-09-23T20:59:44.069418+0000","last_change":"2023-09-23T20:57:19.669346+0000","last_active":"2023-09-23T20:59:44.069418+0000","last_peered":"2023-09-23T20:59:44.069418+0000","last_clean":"2023-09-23T20:59:44.069418+0000","last_became_active":"2023-09-23T20:57:08.246046+0000","last_became_peered":"2023-09-23T20:57:08.246046+0000","last_unstale":"2023-09-23T20:59:44.069418+0000","last_undegraded":"2023-09-23T20:59:44.069418+0000","last_fullsized":"2023-09-23T20:59:44.069418+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:19.669259+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:19.669259+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:14:03.435531+0000","scrub_duration":0.0045759559999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":133,"reported_epoch":242,"state":"active+clean","last_fresh":"2023-09-23T20:59:44.069938+0000","last_change":"2023-09-23T20:55:35.959514+0000","last_active":"2023-09-23T20:59:44.069938+0000","last_peered":"2023-09-23T20:59:44.069938+0000","last_clean":"2023-09-23T20:59:44.069938+0000","last_became_active":"2023-09-23T20:55:35.959244+0000","last_became_peered":"2023-09-23T20:55:35.959244+0000","last_unstale":"2023-09-23T20:59:44.069938+0000","last_undegraded":"2023-09-23T20:59:44.069938+0000","last_fullsized":"2023-09-23T20:59:44.069938+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:47:32.665316+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":137,"reported_epoch":242,"state":"active+clean","last_fresh":"2023-09-23T20:59:44.083899+0000","last_change":"2023-09-23T20:55:35.949000+0000","last_active":"2023-09-23T20:59:44.083899+0000","last_peered":"2023-09-23T20:59:44.083899+0000","last_clean":"2023-09-23T20:59:44.083899+0000","last_became_active":"2023-09-23T20:55:35.948410+0000","last_became_peered":"2023-09-23T20:55:35.948410+0000","last_unstale":"2023-09-23T20:59:44.083899+0000","last_undegraded":"2023-09-23T20:59:44.083899+0000","last_fullsized":"2023-09-23T20:59:44.083899+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:26:50.477289+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":210,"reported_epoch":242,"state":"active+clean","last_fresh":"2023-09-23T20:59:44.088643+0000","last_change":"2023-09-23T20:51:10.829889+0000","last_active":"2023-09-23T20:59:44.088643+0000","last_peered":"2023-09-23T20:59:44.088643+0000","last_clean":"2023-09-23T20:59:44.088643+0000","last_became_active":"2023-09-23T20:51:10.829533+0000","last_became_peered":"2023-09-23T20:51:10.829533+0000","last_unstale":"2023-09-23T20:59:44.088643+0000","last_undegraded":"2023-09-23T20:59:44.088643+0000","last_fullsized":"2023-09-23T20:59:44.088643+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:43:04.603460+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.4","version":"243'10381","reported_seq":10435,"reported_epoch":243,"state":"active+clean","last_fresh":"2023-09-23T20:59:56.688572+0000","last_change":"2023-09-23T20:56:35.549909+0000","last_active":"2023-09-23T20:59:56.688572+0000","last_peered":"2023-09-23T20:59:56.688572+0000","last_clean":"2023-09-23T20:59:56.688572+0000","last_became_active":"2023-09-23T20:56:35.549618+0000","last_became_peered":"2023-09-23T20:56:35.549618+0000","last_unstale":"2023-09-23T20:59:56.688572+0000","last_undegraded":"2023-09-23T20:59:56.688572+0000","last_fullsized":"2023-09-23T20:59:56.688572+0000","mapping_epoch":199,"log_start":"207'3000","ondisk_log_start":"207'3000","created":199,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_clean_scrub_stamp":"2023-09-23T20:56:34.529103+0000","objects_scrubbed":0,"log_size":7381,"log_dups_size":0,"ondisk_log_size":7381,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:24:25.918475+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":153419776,"num_objects":2341,"num_object_clones":0,"num_object_copies":7023,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2341,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10381,"num_write_kb":407104,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.2","version":"79'161","reported_seq":473,"reported_epoch":242,"state":"active+clean","last_fresh":"2023-09-23T20:59:44.078918+0000","last_change":"2023-09-23T20:57:24.802586+0000","last_active":"2023-09-23T20:59:44.078918+0000","last_peered":"2023-09-23T20:59:44.078918+0000","last_clean":"2023-09-23T20:59:44.078918+0000","last_became_active":"2023-09-23T20:57:08.236114+0000","last_became_peered":"2023-09-23T20:57:08.236114+0000","last_unstale":"2023-09-23T20:59:44.078918+0000","last_undegraded":"2023-09-23T20:59:44.078918+0000","last_fullsized":"2023-09-23T20:59:44.078918+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:24.802451+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:24.802451+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:06:22.309980+0000","scrub_duration":0.0090861250000000005,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,4],"acting":[0,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":229,"reported_epoch":242,"state":"active+clean","last_fresh":"2023-09-23T20:59:44.084701+0000","last_change":"2023-09-23T20:51:10.838054+0000","last_active":"2023-09-23T20:59:44.084701+0000","last_peered":"2023-09-23T20:59:44.084701+0000","last_clean":"2023-09-23T20:59:44.084701+0000","last_became_active":"2023-09-23T20:51:10.837750+0000","last_became_peered":"2023-09-23T20:51:10.837750+0000","last_unstale":"2023-09-23T20:59:44.084701+0000","last_undegraded":"2023-09-23T20:59:44.084701+0000","last_fullsized":"2023-09-23T20:59:44.084701+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:15:51.648085+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":210,"reported_epoch":242,"state":"active+clean","last_fresh":"2023-09-23T20:59:44.087233+0000","last_change":"2023-09-23T20:55:35.951146+0000","last_active":"2023-09-23T20:59:44.087233+0000","last_peered":"2023-09-23T20:59:44.087233+0000","last_clean":"2023-09-23T20:59:44.087233+0000","last_became_active":"2023-09-23T20:55:35.950874+0000","last_became_peered":"2023-09-23T20:55:35.950874+0000","last_unstale":"2023-09-23T20:59:44.087233+0000","last_undegraded":"2023-09-23T20:59:44.087233+0000","last_fullsized":"2023-09-23T20:59:44.087233+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:58:13.957677+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":231,"reported_epoch":242,"state":"active+clean","last_fresh":"2023-09-23T20:59:44.093902+0000","last_change":"2023-09-23T20:54:31.898870+0000","last_active":"2023-09-23T20:59:44.093902+0000","last_peered":"2023-09-23T20:59:44.093902+0000","last_clean":"2023-09-23T20:59:44.093902+0000","last_became_active":"2023-09-23T20:54:31.898467+0000","last_became_peered":"2023-09-23T20:54:31.898467+0000","last_unstale":"2023-09-23T20:59:44.093902+0000","last_undegraded":"2023-09-23T20:59:44.093902+0000","last_fullsized":"2023-09-23T20:59:44.093902+0000","mapping_epoch":136,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:56:22.483408+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.6","version":"243'10239","reported_seq":16484,"reported_epoch":243,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-09-23T20:59:56.760965+0000","last_change":"2023-09-23T20:58:39.453197+0000","last_active":"2023-09-23T20:59:56.760965+0000","last_peered":"2023-09-23T20:59:56.760965+0000","last_clean":"2023-09-23T20:57:44.031597+0000","last_became_active":"2023-09-23T20:57:46.443927+0000","last_became_peered":"2023-09-23T20:57:46.443927+0000","last_unstale":"2023-09-23T20:59:56.760965+0000","last_undegraded":"2023-09-23T20:58:39.450464+0000","last_fullsized":"2023-09-23T20:57:45.977227+0000","mapping_epoch":223,"log_start":"207'2900","ondisk_log_start":"207'2900","created":199,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_clean_scrub_stamp":"2023-09-23T20:56:34.529103+0000","objects_scrubbed":0,"log_size":7339,"log_dups_size":0,"ondisk_log_size":7339,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:37:23.911131+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":153944064,"num_objects":2349,"num_object_clones":0,"num_object_copies":7047,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":855,"num_objects_misplaced":1452,"num_objects_unfound":0,"num_objects_dirty":2349,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10239,"num_write_kb":402816,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3091,"num_bytes_recovered":202571776,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7],"avail_no_missing":["0","5","7"],"object_location_counts":[{"shards":"0,5,7","objects":1452},{"shards":"0,7","objects":42}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":116,"reported_epoch":242,"state":"active+clean","last_fresh":"2023-09-23T20:59:44.074012+0000","last_change":"2023-09-23T20:54:31.883504+0000","last_active":"2023-09-23T20:59:44.074012+0000","last_peered":"2023-09-23T20:59:44.074012+0000","last_clean":"2023-09-23T20:59:44.074012+0000","last_became_active":"2023-09-23T20:54:31.883172+0000","last_became_peered":"2023-09-23T20:54:31.883172+0000","last_unstale":"2023-09-23T20:59:44.074012+0000","last_undegraded":"2023-09-23T20:59:44.074012+0000","last_fullsized":"2023-09-23T20:59:44.074012+0000","mapping_epoch":136,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:45:40.289120+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"79'161","reported_seq":473,"reported_epoch":242,"state":"active+clean","last_fresh":"2023-09-23T20:59:44.078582+0000","last_change":"2023-09-23T20:57:20.738417+0000","last_active":"2023-09-23T20:59:44.078582+0000","last_peered":"2023-09-23T20:59:44.078582+0000","last_clean":"2023-09-23T20:59:44.078582+0000","last_became_active":"2023-09-23T20:57:08.236842+0000","last_became_peered":"2023-09-23T20:57:08.236842+0000","last_unstale":"2023-09-23T20:59:44.078582+0000","last_undegraded":"2023-09-23T20:59:44.078582+0000","last_fullsized":"2023-09-23T20:59:44.078582+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":0,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:20.738273+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:20.738273+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:18:28.327990+0000","scrub_duration":0.0088336339999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":221,"reported_epoch":242,"state":"active+clean","last_fresh":"2023-09-23T20:59:44.093968+0000","last_change":"2023-09-23T20:55:35.950777+0000","last_active":"2023-09-23T20:59:44.093968+0000","last_peered":"2023-09-23T20:59:44.093968+0000","last_clean":"2023-09-23T20:59:44.093968+0000","last_became_active":"2023-09-23T20:55:35.950427+0000","last_became_peered":"2023-09-23T20:55:35.950427+0000","last_unstale":"2023-09-23T20:59:44.093968+0000","last_undegraded":"2023-09-23T20:59:44.093968+0000","last_fullsized":"2023-09-23T20:59:44.093968+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:20:10.371751+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":187,"reported_epoch":242,"state":"active+clean","last_fresh":"2023-09-23T20:59:44.073264+0000","last_change":"2023-09-23T20:55:35.978140+0000","last_active":"2023-09-23T20:59:44.073264+0000","last_peered":"2023-09-23T20:59:44.073264+0000","last_clean":"2023-09-23T20:59:44.073264+0000","last_became_active":"2023-09-23T20:55:35.977831+0000","last_became_peered":"2023-09-23T20:55:35.977831+0000","last_unstale":"2023-09-23T20:59:44.073264+0000","last_undegraded":"2023-09-23T20:59:44.073264+0000","last_fullsized":"2023-09-23T20:59:44.073264+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:25:47.678023+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.5","version":"243'10374","reported_seq":10435,"reported_epoch":243,"state":"active+clean","last_fresh":"2023-09-23T20:59:56.440072+0000","last_change":"2023-09-23T20:56:45.990110+0000","last_active":"2023-09-23T20:59:56.440072+0000","last_peered":"2023-09-23T20:59:56.440072+0000","last_clean":"2023-09-23T20:59:56.440072+0000","last_became_active":"2023-09-23T20:56:45.988394+0000","last_became_peered":"2023-09-23T20:56:45.988394+0000","last_unstale":"2023-09-23T20:59:56.440072+0000","last_undegraded":"2023-09-23T20:59:56.440072+0000","last_fullsized":"2023-09-23T20:59:56.440072+0000","mapping_epoch":204,"log_start":"205'1500","ondisk_log_start":"205'1500","created":199,"last_epoch_clean":205,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_clean_scrub_stamp":"2023-09-23T20:56:34.529103+0000","objects_scrubbed":0,"log_size":8874,"log_dups_size":0,"ondisk_log_size":8874,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:22:03.814210+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":150994944,"num_objects":2304,"num_object_clones":0,"num_object_copies":6912,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2304,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10374,"num_write_kb":405696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,0],"acting":[2,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"79'161","reported_seq":509,"reported_epoch":242,"state":"active+clean","last_fresh":"2023-09-23T20:59:44.087416+0000","last_change":"2023-09-23T20:57:20.378531+0000","last_active":"2023-09-23T20:59:44.087416+0000","last_peered":"2023-09-23T20:59:44.087416+0000","last_clean":"2023-09-23T20:59:44.087416+0000","last_became_active":"2023-09-23T20:57:08.238019+0000","last_became_peered":"2023-09-23T20:57:08.238019+0000","last_unstale":"2023-09-23T20:59:44.087416+0000","last_undegraded":"2023-09-23T20:59:44.087416+0000","last_fullsized":"2023-09-23T20:59:44.087416+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":3,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:20.377714+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:20.377714+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:00:08.305602+0000","scrub_duration":0.003015228,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":112,"reported_epoch":242,"state":"active+clean","last_fresh":"2023-09-23T20:59:44.087370+0000","last_change":"2023-09-23T20:56:45.991884+0000","last_active":"2023-09-23T20:59:44.087370+0000","last_peered":"2023-09-23T20:59:44.087370+0000","last_clean":"2023-09-23T20:59:44.087370+0000","last_became_active":"2023-09-23T20:56:45.991568+0000","last_became_peered":"2023-09-23T20:56:45.991568+0000","last_unstale":"2023-09-23T20:59:44.087370+0000","last_undegraded":"2023-09-23T20:59:44.087370+0000","last_fullsized":"2023-09-23T20:59:44.087370+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":205,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:21:20.775511+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,0],"acting":[1,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":237,"reported_epoch":243,"state":"active+clean","last_fresh":"2023-09-23T20:59:52.353201+0000","last_change":"2023-09-23T20:57:44.979950+0000","last_active":"2023-09-23T20:59:52.353201+0000","last_peered":"2023-09-23T20:59:52.353201+0000","last_clean":"2023-09-23T20:59:52.353201+0000","last_became_active":"2023-09-23T20:57:44.979346+0000","last_became_peered":"2023-09-23T20:57:44.979346+0000","last_unstale":"2023-09-23T20:59:52.353201+0000","last_undegraded":"2023-09-23T20:59:52.353201+0000","last_fullsized":"2023-09-23T20:59:52.353201+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:10:24.372800+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":215,"reported_epoch":242,"state":"active+clean","last_fresh":"2023-09-23T20:59:44.081326+0000","last_change":"2023-09-23T20:50:37.982235+0000","last_active":"2023-09-23T20:59:44.081326+0000","last_peered":"2023-09-23T20:59:44.081326+0000","last_clean":"2023-09-23T20:59:44.081326+0000","last_became_active":"2023-09-23T20:50:37.981913+0000","last_became_peered":"2023-09-23T20:50:37.981913+0000","last_unstale":"2023-09-23T20:59:44.081326+0000","last_undegraded":"2023-09-23T20:59:44.081326+0000","last_fullsized":"2023-09-23T20:59:44.081326+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:59:57.122584+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.0","version":"243'10335","reported_seq":10387,"reported_epoch":243,"state":"active+clean","last_fresh":"2023-09-23T20:59:57.118811+0000","last_change":"2023-09-23T20:56:35.550635+0000","last_active":"2023-09-23T20:59:57.118811+0000","last_peered":"2023-09-23T20:59:57.118811+0000","last_clean":"2023-09-23T20:59:57.118811+0000","last_became_active":"2023-09-23T20:56:35.550376+0000","last_became_peered":"2023-09-23T20:56:35.550376+0000","last_unstale":"2023-09-23T20:59:57.118811+0000","last_undegraded":"2023-09-23T20:59:57.118811+0000","last_fullsized":"2023-09-23T20:59:57.118811+0000","mapping_epoch":199,"log_start":"205'1700","ondisk_log_start":"205'1700","created":199,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_clean_scrub_stamp":"2023-09-23T20:56:34.529103+0000","objects_scrubbed":0,"log_size":8635,"log_dups_size":0,"ondisk_log_size":8635,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:40:55.020049+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":153812992,"num_objects":2347,"num_object_clones":0,"num_object_copies":7041,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2347,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10335,"num_write_kb":405824,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.6","version":"79'161","reported_seq":525,"reported_epoch":242,"state":"active+clean","last_fresh":"2023-09-23T20:59:44.073046+0000","last_change":"2023-09-23T20:57:26.907505+0000","last_active":"2023-09-23T20:59:44.073046+0000","last_peered":"2023-09-23T20:59:44.073046+0000","last_clean":"2023-09-23T20:59:44.073046+0000","last_became_active":"2023-09-23T20:57:08.240321+0000","last_became_peered":"2023-09-23T20:57:08.240321+0000","last_unstale":"2023-09-23T20:59:44.073046+0000","last_undegraded":"2023-09-23T20:59:44.073046+0000","last_fullsized":"2023-09-23T20:59:44.073046+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":3,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:26.907408+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:26.907408+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:34:17.392928+0000","scrub_duration":0.010337038,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":159,"reported_epoch":242,"state":"active+clean","last_fresh":"2023-09-23T20:59:44.079403+0000","last_change":"2023-09-23T20:56:45.988566+0000","last_active":"2023-09-23T20:59:44.079403+0000","last_peered":"2023-09-23T20:59:44.079403+0000","last_clean":"2023-09-23T20:59:44.079403+0000","last_became_active":"2023-09-23T20:56:45.988249+0000","last_became_peered":"2023-09-23T20:56:45.988249+0000","last_unstale":"2023-09-23T20:59:44.079403+0000","last_undegraded":"2023-09-23T20:59:44.079403+0000","last_fullsized":"2023-09-23T20:59:44.079403+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":205,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:51:36.644764+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,4],"acting":[0,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":213,"reported_epoch":242,"state":"active+clean","last_fresh":"2023-09-23T20:59:44.088779+0000","last_change":"2023-09-23T20:54:31.898965+0000","last_active":"2023-09-23T20:59:44.088779+0000","last_peered":"2023-09-23T20:59:44.088779+0000","last_clean":"2023-09-23T20:59:44.088779+0000","last_became_active":"2023-09-23T20:54:31.898663+0000","last_became_peered":"2023-09-23T20:54:31.898663+0000","last_unstale":"2023-09-23T20:59:44.088779+0000","last_undegraded":"2023-09-23T20:59:44.088779+0000","last_fullsized":"2023-09-23T20:59:44.088779+0000","mapping_epoch":136,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:53:05.058087+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":204,"reported_epoch":243,"state":"active+clean","last_fresh":"2023-09-23T20:59:52.353850+0000","last_change":"2023-09-23T20:54:31.884717+0000","last_active":"2023-09-23T20:59:52.353850+0000","last_peered":"2023-09-23T20:59:52.353850+0000","last_clean":"2023-09-23T20:59:52.353850+0000","last_became_active":"2023-09-23T20:54:31.884345+0000","last_became_peered":"2023-09-23T20:54:31.884345+0000","last_unstale":"2023-09-23T20:59:52.353850+0000","last_undegraded":"2023-09-23T20:59:52.353850+0000","last_fullsized":"2023-09-23T20:59:52.353850+0000","mapping_epoch":136,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:02:26.393868+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.3","version":"243'10151","reported_seq":10205,"reported_epoch":243,"state":"active+clean","last_fresh":"2023-09-23T20:59:54.642965+0000","last_change":"2023-09-23T20:56:35.549655+0000","last_active":"2023-09-23T20:59:54.642965+0000","last_peered":"2023-09-23T20:59:54.642965+0000","last_clean":"2023-09-23T20:59:54.642965+0000","last_became_active":"2023-09-23T20:56:35.549357+0000","last_became_peered":"2023-09-23T20:56:35.549357+0000","last_unstale":"2023-09-23T20:59:54.642965+0000","last_undegraded":"2023-09-23T20:59:54.642965+0000","last_fullsized":"2023-09-23T20:59:54.642965+0000","mapping_epoch":199,"log_start":"217'4000","ondisk_log_start":"217'4000","created":199,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_clean_scrub_stamp":"2023-09-23T20:56:34.529103+0000","objects_scrubbed":0,"log_size":6151,"log_dups_size":0,"ondisk_log_size":6151,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:37:58.752518+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":155385856,"num_objects":2371,"num_object_clones":0,"num_object_copies":7113,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2371,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10151,"num_write_kb":400704,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.5","version":"79'161","reported_seq":506,"reported_epoch":242,"state":"active+clean","last_fresh":"2023-09-23T20:59:44.081393+0000","last_change":"2023-09-23T20:57:30.029479+0000","last_active":"2023-09-23T20:59:44.081393+0000","last_peered":"2023-09-23T20:59:44.081393+0000","last_clean":"2023-09-23T20:59:44.081393+0000","last_became_active":"2023-09-23T20:57:08.249909+0000","last_became_peered":"2023-09-23T20:57:08.249909+0000","last_unstale":"2023-09-23T20:59:44.081393+0000","last_undegraded":"2023-09-23T20:59:44.081393+0000","last_fullsized":"2023-09-23T20:59:44.081393+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":3,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:30.029396+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:30.029396+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:16:46.083145+0000","scrub_duration":0.012914682,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":207,"reported_epoch":242,"state":"active+clean","last_fresh":"2023-09-23T20:59:49.636222+0000","last_change":"2023-09-23T20:51:10.836910+0000","last_active":"2023-09-23T20:59:49.636222+0000","last_peered":"2023-09-23T20:59:49.636222+0000","last_clean":"2023-09-23T20:59:49.636222+0000","last_became_active":"2023-09-23T20:51:10.836557+0000","last_became_peered":"2023-09-23T20:51:10.836557+0000","last_unstale":"2023-09-23T20:59:49.636222+0000","last_undegraded":"2023-09-23T20:59:49.636222+0000","last_fullsized":"2023-09-23T20:59:49.636222+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:51:54.019969+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":213,"reported_epoch":242,"state":"active+clean","last_fresh":"2023-09-23T20:59:44.087567+0000","last_change":"2023-09-23T20:55:35.953444+0000","last_active":"2023-09-23T20:59:44.087567+0000","last_peered":"2023-09-23T20:59:44.087567+0000","last_clean":"2023-09-23T20:59:44.087567+0000","last_became_active":"2023-09-23T20:55:35.953151+0000","last_became_peered":"2023-09-23T20:55:35.953151+0000","last_unstale":"2023-09-23T20:59:44.087567+0000","last_undegraded":"2023-09-23T20:59:44.087567+0000","last_fullsized":"2023-09-23T20:59:44.087567+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:39:12.404391+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":32,"reported_epoch":242,"state":"active+clean","last_fresh":"2023-09-23T20:59:44.633896+0000","last_change":"2023-09-23T20:57:44.982509+0000","last_active":"2023-09-23T20:59:44.633896+0000","last_peered":"2023-09-23T20:59:44.633896+0000","last_clean":"2023-09-23T20:59:44.633896+0000","last_became_active":"2023-09-23T20:57:44.982175+0000","last_became_peered":"2023-09-23T20:57:44.982175+0000","last_unstale":"2023-09-23T20:59:44.633896+0000","last_undegraded":"2023-09-23T20:59:44.633896+0000","last_fullsized":"2023-09-23T20:59:44.633896+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:37:49.215488+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.2","version":"243'10151","reported_seq":10205,"reported_epoch":243,"state":"active+clean","last_fresh":"2023-09-23T20:59:54.638980+0000","last_change":"2023-09-23T20:56:35.543020+0000","last_active":"2023-09-23T20:59:54.638980+0000","last_peered":"2023-09-23T20:59:54.638980+0000","last_clean":"2023-09-23T20:59:54.638980+0000","last_became_active":"2023-09-23T20:56:35.542719+0000","last_became_peered":"2023-09-23T20:56:35.542719+0000","last_unstale":"2023-09-23T20:59:54.638980+0000","last_undegraded":"2023-09-23T20:59:54.638980+0000","last_fullsized":"2023-09-23T20:59:54.638980+0000","mapping_epoch":199,"log_start":"217'4000","ondisk_log_start":"217'4000","created":199,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_clean_scrub_stamp":"2023-09-23T20:56:34.529103+0000","objects_scrubbed":0,"log_size":6151,"log_dups_size":0,"ondisk_log_size":6151,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:52:21.088628+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":154730496,"num_objects":2361,"num_object_clones":0,"num_object_copies":7083,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2361,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10151,"num_write_kb":400384,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.4","version":"79'161","reported_seq":521,"reported_epoch":242,"state":"active+clean","last_fresh":"2023-09-23T20:59:44.095916+0000","last_change":"2023-09-23T20:57:20.012904+0000","last_active":"2023-09-23T20:59:44.095916+0000","last_peered":"2023-09-23T20:59:44.095916+0000","last_clean":"2023-09-23T20:59:44.095916+0000","last_became_active":"2023-09-23T20:57:08.237996+0000","last_became_peered":"2023-09-23T20:57:08.237996+0000","last_unstale":"2023-09-23T20:59:44.095916+0000","last_undegraded":"2023-09-23T20:59:44.095916+0000","last_fullsized":"2023-09-23T20:59:44.095916+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":3,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:20.012764+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:20.012764+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:51:28.796946+0000","scrub_duration":0.0045251919999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":192,"reported_epoch":242,"state":"active+clean","last_fresh":"2023-09-23T20:59:44.073036+0000","last_change":"2023-09-23T20:55:35.949541+0000","last_active":"2023-09-23T20:59:44.073036+0000","last_peered":"2023-09-23T20:59:44.073036+0000","last_clean":"2023-09-23T20:59:44.073036+0000","last_became_active":"2023-09-23T20:55:35.944635+0000","last_became_peered":"2023-09-23T20:55:35.944635+0000","last_unstale":"2023-09-23T20:59:44.073036+0000","last_undegraded":"2023-09-23T20:59:44.073036+0000","last_fullsized":"2023-09-23T20:59:44.073036+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:30:36.033585+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"70'1","reported_seq":231,"reported_epoch":242,"state":"active+clean","last_fresh":"2023-09-23T20:59:44.081416+0000","last_change":"2023-09-23T20:54:14.009723+0000","last_active":"2023-09-23T20:59:44.081416+0000","last_peered":"2023-09-23T20:59:44.081416+0000","last_clean":"2023-09-23T20:59:44.081416+0000","last_became_active":"2023-09-23T20:54:13.941771+0000","last_became_peered":"2023-09-23T20:54:13.941771+0000","last_unstale":"2023-09-23T20:59:44.081416+0000","last_undegraded":"2023-09-23T20:59:44.081416+0000","last_fullsized":"2023-09-23T20:59:44.081416+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:48:25.588116+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":147,"reported_epoch":242,"state":"active+clean","last_fresh":"2023-09-23T20:59:44.084556+0000","last_change":"2023-09-23T20:55:35.949780+0000","last_active":"2023-09-23T20:59:44.084556+0000","last_peered":"2023-09-23T20:59:44.084556+0000","last_clean":"2023-09-23T20:59:44.084556+0000","last_became_active":"2023-09-23T20:55:35.949497+0000","last_became_peered":"2023-09-23T20:55:35.949497+0000","last_unstale":"2023-09-23T20:59:44.084556+0000","last_undegraded":"2023-09-23T20:59:44.084556+0000","last_fullsized":"2023-09-23T20:59:44.084556+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:09:37.420882+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":206,"reported_epoch":242,"state":"active+clean","last_fresh":"2023-09-23T20:59:44.079214+0000","last_change":"2023-09-23T20:54:13.929196+0000","last_active":"2023-09-23T20:59:44.079214+0000","last_peered":"2023-09-23T20:59:44.079214+0000","last_clean":"2023-09-23T20:59:44.079214+0000","last_became_active":"2023-09-23T20:54:13.928831+0000","last_became_peered":"2023-09-23T20:54:13.928831+0000","last_unstale":"2023-09-23T20:59:44.079214+0000","last_undegraded":"2023-09-23T20:59:44.079214+0000","last_fullsized":"2023-09-23T20:59:44.079214+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:28:44.316851+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":211,"reported_epoch":242,"state":"active+clean","last_fresh":"2023-09-23T20:59:44.086068+0000","last_change":"2023-09-23T20:52:12.855430+0000","last_active":"2023-09-23T20:59:44.086068+0000","last_peered":"2023-09-23T20:59:44.086068+0000","last_clean":"2023-09-23T20:59:44.086068+0000","last_became_active":"2023-09-23T20:52:12.855115+0000","last_became_peered":"2023-09-23T20:52:12.855115+0000","last_unstale":"2023-09-23T20:59:44.086068+0000","last_undegraded":"2023-09-23T20:59:44.086068+0000","last_fullsized":"2023-09-23T20:59:44.086068+0000","mapping_epoch":100,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":101,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:33:23.802214+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":120,"reported_epoch":242,"state":"active+clean","last_fresh":"2023-09-23T20:59:44.069676+0000","last_change":"2023-09-23T20:54:31.902946+0000","last_active":"2023-09-23T20:59:44.069676+0000","last_peered":"2023-09-23T20:59:44.069676+0000","last_clean":"2023-09-23T20:59:44.069676+0000","last_became_active":"2023-09-23T20:54:31.889484+0000","last_became_peered":"2023-09-23T20:54:31.889484+0000","last_unstale":"2023-09-23T20:59:44.069676+0000","last_undegraded":"2023-09-23T20:59:44.069676+0000","last_fullsized":"2023-09-23T20:59:44.069676+0000","mapping_epoch":136,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:44:44.278398+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":120,"reported_epoch":242,"state":"active+clean","last_fresh":"2023-09-23T20:59:44.083994+0000","last_change":"2023-09-23T20:54:13.950351+0000","last_active":"2023-09-23T20:59:44.083994+0000","last_peered":"2023-09-23T20:59:44.083994+0000","last_clean":"2023-09-23T20:59:44.083994+0000","last_became_active":"2023-09-23T20:54:13.949987+0000","last_became_peered":"2023-09-23T20:54:13.949987+0000","last_unstale":"2023-09-23T20:59:44.083994+0000","last_undegraded":"2023-09-23T20:59:44.083994+0000","last_fullsized":"2023-09-23T20:59:44.083994+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:46:17.765134+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":151,"reported_epoch":242,"state":"active+clean","last_fresh":"2023-09-23T20:59:44.069954+0000","last_change":"2023-09-23T20:56:18.141728+0000","last_active":"2023-09-23T20:59:44.069954+0000","last_peered":"2023-09-23T20:59:44.069954+0000","last_clean":"2023-09-23T20:59:44.069954+0000","last_became_active":"2023-09-23T20:56:18.137979+0000","last_became_peered":"2023-09-23T20:56:18.137979+0000","last_unstale":"2023-09-23T20:59:44.069954+0000","last_undegraded":"2023-09-23T20:59:44.069954+0000","last_fullsized":"2023-09-23T20:59:44.069954+0000","mapping_epoch":192,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":193,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:58:05.658443+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":120,"reported_epoch":242,"state":"active+clean","last_fresh":"2023-09-23T20:59:44.081088+0000","last_change":"2023-09-23T20:54:13.935307+0000","last_active":"2023-09-23T20:59:44.081088+0000","last_peered":"2023-09-23T20:59:44.081088+0000","last_clean":"2023-09-23T20:59:44.081088+0000","last_became_active":"2023-09-23T20:54:13.935000+0000","last_became_peered":"2023-09-23T20:54:13.935000+0000","last_unstale":"2023-09-23T20:59:44.081088+0000","last_undegraded":"2023-09-23T20:59:44.081088+0000","last_fullsized":"2023-09-23T20:59:44.081088+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:47:40.974082+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":230,"reported_epoch":242,"state":"active+clean","last_fresh":"2023-09-23T20:59:44.095664+0000","last_change":"2023-09-23T20:55:35.954870+0000","last_active":"2023-09-23T20:59:44.095664+0000","last_peered":"2023-09-23T20:59:44.095664+0000","last_clean":"2023-09-23T20:59:44.095664+0000","last_became_active":"2023-09-23T20:55:35.954568+0000","last_became_peered":"2023-09-23T20:55:35.954568+0000","last_unstale":"2023-09-23T20:59:44.095664+0000","last_undegraded":"2023-09-23T20:59:44.095664+0000","last_fullsized":"2023-09-23T20:59:44.095664+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:06:22.423566+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":229,"reported_epoch":242,"state":"active+clean","last_fresh":"2023-09-23T20:59:44.111977+0000","last_change":"2023-09-23T20:51:10.838334+0000","last_active":"2023-09-23T20:59:44.111977+0000","last_peered":"2023-09-23T20:59:44.111977+0000","last_clean":"2023-09-23T20:59:44.111977+0000","last_became_active":"2023-09-23T20:51:10.838026+0000","last_became_peered":"2023-09-23T20:51:10.838026+0000","last_unstale":"2023-09-23T20:59:44.111977+0000","last_undegraded":"2023-09-23T20:59:44.111977+0000","last_fullsized":"2023-09-23T20:59:44.111977+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:04:37.009457+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":234,"reported_epoch":243,"state":"active+clean","last_fresh":"2023-09-23T20:59:52.354613+0000","last_change":"2023-09-23T20:54:31.903139+0000","last_active":"2023-09-23T20:59:52.354613+0000","last_peered":"2023-09-23T20:59:52.354613+0000","last_clean":"2023-09-23T20:59:52.354613+0000","last_became_active":"2023-09-23T20:54:31.884520+0000","last_became_peered":"2023-09-23T20:54:31.884520+0000","last_unstale":"2023-09-23T20:59:52.354613+0000","last_undegraded":"2023-09-23T20:59:52.354613+0000","last_fullsized":"2023-09-23T20:59:52.354613+0000","mapping_epoch":136,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:39:34.269705+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1108,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":137,"reported_epoch":242,"state":"active+clean","last_fresh":"2023-09-23T20:59:44.080750+0000","last_change":"2023-09-23T20:55:35.951572+0000","last_active":"2023-09-23T20:59:44.080750+0000","last_peered":"2023-09-23T20:59:44.080750+0000","last_clean":"2023-09-23T20:59:44.080750+0000","last_became_active":"2023-09-23T20:55:35.951294+0000","last_became_peered":"2023-09-23T20:55:35.951294+0000","last_unstale":"2023-09-23T20:59:44.080750+0000","last_undegraded":"2023-09-23T20:59:44.080750+0000","last_fullsized":"2023-09-23T20:59:44.080750+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:36:32.447210+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":7,"num_pg":16,"stat_sum":{"num_bytes":2503475224,"num_objects":38201,"num_object_clones":0,"num_object_copies":114603,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":11798,"num_objects_misplaced":1452,"num_objects_unfound":0,"num_objects_dirty":38201,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":165319,"num_write_kb":6512577,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3092,"num_bytes_recovered":202637312,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8263118848,"data_stored":8263106632,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":118919,"ondisk_log_size":118919,"up":48,"acting":45,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":22,"num_bytes_recovered":5990,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":21,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3381,"ondisk_log_size":3381,"up":63,"acting":63,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201514,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":951400,"kb_used_data":757536,"kb_used_omap":0,"kb_used_meta":193792,"kb_avail":92793752,"statfs":{"total":95995035648,"available":95020802048,"internally_reserved":0,"allocated":775716864,"data_stored":774853157,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":198443008},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:59:46 2023","interfaces":[{"interface":"back","average":{"1min":1.6559999999999999,"5min":1.802,"15min":1.0840000000000001},"min":{"1min":0.245,"5min":0.21099999999999999,"15min":0.21099999999999999},"max":{"1min":6.4420000000000002,"5min":43.994999999999997,"15min":43.994999999999997},"last":4.8899999999999997},{"interface":"front","average":{"1min":1.6339999999999999,"5min":1.9159999999999999,"15min":1.1699999999999999},"min":{"1min":0.318,"5min":0.28199999999999997,"15min":0.218},"max":{"1min":6.4610000000000003,"5min":44.206000000000003,"15min":44.206000000000003},"last":4.9050000000000002}]},{"osd":1,"last update":"Sat Sep 23 20:59:19 2023","interfaces":[{"interface":"back","average":{"1min":1.74,"5min":1.835,"15min":1.2230000000000001},"min":{"1min":0.26600000000000001,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":5.4359999999999999,"5min":10.622,"15min":22.591999999999999},"last":0.35499999999999998},{"interface":"front","average":{"1min":2.0600000000000001,"5min":1.639,"15min":1.0860000000000001},"min":{"1min":0.32100000000000001,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":6.7709999999999999,"5min":10.097,"15min":21.437000000000001},"last":0.36499999999999999}]},{"osd":2,"last update":"Sat Sep 23 20:59:55 2023","interfaces":[{"interface":"back","average":{"1min":2.7149999999999999,"5min":2.2000000000000002,"15min":1.294},"min":{"1min":0.30399999999999999,"5min":0.26500000000000001,"15min":0.24399999999999999},"max":{"1min":8.5289999999999999,"5min":10.805999999999999,"15min":10.805999999999999},"last":0.30399999999999999},{"interface":"front","average":{"1min":2.339,"5min":2.2639999999999998,"15min":1.3089999999999999},"min":{"1min":0.26500000000000001,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":11.590999999999999,"5min":45.25,"15min":45.25},"last":0.39700000000000002}]},{"osd":3,"last update":"Sat Sep 23 20:59:30 2023","interfaces":[{"interface":"back","average":{"1min":2.6200000000000001,"5min":2.508,"15min":1.7250000000000001},"min":{"1min":0.33500000000000002,"5min":0.32100000000000001,"15min":0.314},"max":{"1min":8.4390000000000001,"5min":13.007,"15min":36.408000000000001},"last":0.46100000000000002},{"interface":"front","average":{"1min":2.7789999999999999,"5min":2.3079999999999998,"15min":1.552},"min":{"1min":0.41599999999999998,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":13.198,"5min":14.220000000000001,"15min":35.593000000000004},"last":1.147}]},{"osd":4,"last update":"Sat Sep 23 20:59:19 2023","interfaces":[{"interface":"back","average":{"1min":0.41099999999999998,"5min":0.78500000000000003,"15min":0.60499999999999998},"min":{"1min":0.25700000000000001,"5min":0.23799999999999999,"15min":0.23699999999999999},"max":{"1min":1.1659999999999999,"5min":5.8920000000000003,"15min":5.8920000000000003},"last":0.318},{"interface":"front","average":{"1min":0.51900000000000002,"5min":0.77600000000000002,"15min":0.61299999999999999},"min":{"1min":0.28599999999999998,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":2.952,"5min":5.3200000000000003,"15min":5.3200000000000003},"last":0.378}]},{"osd":5,"last update":"Sat Sep 23 20:59:19 2023","interfaces":[{"interface":"back","average":{"1min":0.443,"5min":0.78100000000000003,"15min":0.628},"min":{"1min":0.316,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":0.64000000000000001,"5min":7.0389999999999997,"15min":7.0389999999999997},"last":0.56399999999999995},{"interface":"front","average":{"1min":0.55200000000000005,"5min":0.72299999999999998,"15min":0.61399999999999999},"min":{"1min":0.30199999999999999,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":1.909,"5min":4.5999999999999996,"15min":4.5999999999999996},"last":0.44800000000000001}]},{"osd":6,"last update":"Sat Sep 23 20:59:42 2023","interfaces":[{"interface":"back","average":{"1min":0.48299999999999998,"5min":0.68000000000000005,"15min":0.67100000000000004},"min":{"1min":0.28899999999999998,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":0.77300000000000002,"5min":3.5249999999999999,"15min":3.5249999999999999},"last":0.40799999999999997},{"interface":"front","average":{"1min":0.48499999999999999,"5min":0.749,"15min":0.78600000000000003},"min":{"1min":0.34699999999999998,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":0.95499999999999996,"5min":3.5409999999999999,"15min":3.5409999999999999},"last":0.64300000000000002}]}]},{"osd":6,"up_from":164,"seq":704374636597,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":914652,"kb_used_data":799956,"kb_used_omap":10,"kb_used_meta":114677,"kb_avail":92830500,"statfs":{"total":95995035648,"available":95058432000,"internally_reserved":0,"allocated":819154944,"data_stored":818303138,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":10328,"internal_metadata":117430184},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:59:45 2023","interfaces":[{"interface":"back","average":{"1min":1.1599999999999999,"5min":1.5,"15min":1.0760000000000001},"min":{"1min":0.23599999999999999,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":4.0960000000000001,"5min":8.7319999999999993,"15min":8.7319999999999993},"last":0.28599999999999998},{"interface":"front","average":{"1min":1.5029999999999999,"5min":1.6060000000000001,"15min":1.1899999999999999},"min":{"1min":0.23999999999999999,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":4.2869999999999999,"5min":6.7140000000000004,"15min":6.7140000000000004},"last":3.3340000000000001}]},{"osd":1,"last update":"Sat Sep 23 20:59:45 2023","interfaces":[{"interface":"back","average":{"1min":1.554,"5min":1.25,"15min":0.94199999999999995},"min":{"1min":0.27200000000000002,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":8.4309999999999992,"5min":8.4309999999999992,"15min":8.4309999999999992},"last":3.839},{"interface":"front","average":{"1min":1.506,"5min":1.3720000000000001,"15min":1.036},"min":{"1min":0.26100000000000001,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":7.2069999999999999,"5min":10.324999999999999,"15min":10.324999999999999},"last":0.38600000000000001}]},{"osd":2,"last update":"Sat Sep 23 20:59:45 2023","interfaces":[{"interface":"back","average":{"1min":2.097,"5min":2.0720000000000001,"15min":1.788},"min":{"1min":0.312,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":6.0670000000000002,"5min":13.750999999999999,"15min":13.750999999999999},"last":2.944},{"interface":"front","average":{"1min":2.46,"5min":1.8,"15min":1.3380000000000001},"min":{"1min":0.32000000000000001,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":5.6130000000000004,"5min":8.0890000000000004,"15min":8.0890000000000004},"last":2.9100000000000001}]},{"osd":3,"last update":"Sat Sep 23 20:59:45 2023","interfaces":[{"interface":"back","average":{"1min":1.768,"5min":1.415,"15min":1.216},"min":{"1min":0.45000000000000001,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":4.7220000000000004,"5min":6.9000000000000004,"15min":6.9000000000000004},"last":3.0179999999999998},{"interface":"front","average":{"1min":1.6479999999999999,"5min":1.7490000000000001,"15min":1.651},"min":{"1min":0.34999999999999998,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":4.6289999999999996,"5min":11.021000000000001,"15min":11.021000000000001},"last":1.5920000000000001}]},{"osd":4,"last update":"Sat Sep 23 20:59:45 2023","interfaces":[{"interface":"back","average":{"1min":0.68700000000000006,"5min":0.78600000000000003,"15min":0.96299999999999997},"min":{"1min":0.27100000000000002,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":2.3460000000000001,"5min":9.0079999999999991,"15min":9.0079999999999991},"last":0.41399999999999998},{"interface":"front","average":{"1min":0.48599999999999999,"5min":0.499,"15min":0.52000000000000002},"min":{"1min":0.29899999999999999,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":1.115,"5min":2.0059999999999998,"15min":2.0059999999999998},"last":0.371}]},{"osd":5,"last update":"Sat Sep 23 20:59:45 2023","interfaces":[{"interface":"back","average":{"1min":0.78100000000000003,"5min":0.61699999999999999,"15min":0.56599999999999995},"min":{"1min":0.28100000000000003,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":2.3730000000000002,"5min":2.8239999999999998,"15min":2.8239999999999998},"last":0.34799999999999998},{"interface":"front","average":{"1min":0.58299999999999996,"5min":0.73999999999999999,"15min":0.78600000000000003},"min":{"1min":0.34000000000000002,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":1.2290000000000001,"5min":3.1379999999999999,"15min":3.1379999999999999},"last":0.438}]},{"osd":7,"last update":"Sat Sep 23 20:59:45 2023","interfaces":[{"interface":"back","average":{"1min":0.51700000000000002,"5min":0.77200000000000002,"15min":0.876},"min":{"1min":0.318,"5min":0.312,"15min":0.312},"max":{"1min":1.1819999999999999,"5min":7.8570000000000002,"15min":7.8570000000000002},"last":0.55900000000000005},{"interface":"front","average":{"1min":0.46500000000000002,"5min":0.64900000000000002,"15min":0.56899999999999995},"min":{"1min":0.33800000000000002,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":0.63200000000000001,"5min":8.282,"15min":8.282},"last":0.56499999999999995}]}]},{"osd":1,"up_from":88,"seq":377957122168,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1533280,"kb_used_data":1302232,"kb_used_omap":9,"kb_used_meta":231030,"kb_avail":92211872,"statfs":{"total":95995035648,"available":94424956928,"internally_reserved":0,"allocated":1333485568,"data_stored":1332283193,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9239,"internal_metadata":236575721},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:59:23 2023","interfaces":[{"interface":"back","average":{"1min":3.9180000000000001,"5min":3.1070000000000002,"15min":1.5389999999999999},"min":{"1min":0.20699999999999999,"5min":0.20699999999999999,"15min":0.20699999999999999},"max":{"1min":14.813000000000001,"5min":16.646999999999998,"15min":16.646999999999998},"last":0.44},{"interface":"front","average":{"1min":3.77,"5min":3.335,"15min":1.6819999999999999},"min":{"1min":0.33300000000000002,"5min":0.246,"15min":0.22500000000000001},"max":{"1min":9.484,"5min":16.664999999999999,"15min":18.457999999999998},"last":8.3219999999999992}]},{"osd":2,"last update":"Sat Sep 23 20:58:53 2023","interfaces":[{"interface":"back","average":{"1min":2.7130000000000001,"5min":2.4140000000000001,"15min":1.1799999999999999},"min":{"1min":0.24299999999999999,"5min":0.221,"15min":0.20300000000000001},"max":{"1min":8.6829999999999998,"5min":14.335000000000001,"15min":14.335000000000001},"last":6.7990000000000004},{"interface":"front","average":{"1min":5.8019999999999996,"5min":3.27,"15min":1.758},"min":{"1min":0.39500000000000002,"5min":0.248,"15min":0.248},"max":{"1min":16.048999999999999,"5min":16.048999999999999,"15min":38.951000000000001},"last":8.2919999999999998}]},{"osd":3,"last update":"Sat Sep 23 20:59:23 2023","interfaces":[{"interface":"back","average":{"1min":3.1539999999999999,"5min":2.629,"15min":1.5589999999999999},"min":{"1min":0.42499999999999999,"5min":0.246,"15min":0.246},"max":{"1min":9.4450000000000003,"5min":16.055,"15min":25.314},"last":22.686},{"interface":"front","average":{"1min":3.0870000000000002,"5min":2.4860000000000002,"15min":1.411},"min":{"1min":0.29799999999999999,"5min":0.20599999999999999,"15min":0.20599999999999999},"max":{"1min":10.204000000000001,"5min":14.225,"15min":19.881},"last":0.54900000000000004}]},{"osd":4,"last update":"Sat Sep 23 20:59:23 2023","interfaces":[{"interface":"back","average":{"1min":1.871,"5min":1.6439999999999999,"15min":1.2130000000000001},"min":{"1min":0.28299999999999997,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":9.3480000000000008,"5min":10.598000000000001,"15min":12.006},"last":18.231000000000002},{"interface":"front","average":{"1min":1.633,"5min":1.675,"15min":1.0800000000000001},"min":{"1min":0.38400000000000001,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":7.532,"5min":7.532,"15min":11.234999999999999},"last":0.35399999999999998}]},{"osd":5,"last update":"Sat Sep 23 20:59:23 2023","interfaces":[{"interface":"back","average":{"1min":2.3700000000000001,"5min":2.1469999999999998,"15min":1.292},"min":{"1min":0.55800000000000005,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":8.4410000000000007,"5min":15.041,"15min":15.041},"last":8.2050000000000001},{"interface":"front","average":{"1min":1.9370000000000001,"5min":1.756,"15min":1.2390000000000001},"min":{"1min":0.432,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":9.3889999999999993,"5min":10.647,"15min":12.08},"last":18.280999999999999}]},{"osd":6,"last update":"Sat Sep 23 20:59:41 2023","interfaces":[{"interface":"back","average":{"1min":2.266,"5min":1.5609999999999999,"15min":1.397},"min":{"1min":0.46200000000000002,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":10.159000000000001,"5min":10.673999999999999,"15min":10.673999999999999},"last":18.984999999999999},{"interface":"front","average":{"1min":1.8660000000000001,"5min":1.6299999999999999,"15min":1.3779999999999999},"min":{"1min":0.33700000000000002,"5min":0.309,"15min":0.309},"max":{"1min":7.6779999999999999,"5min":8.7530000000000001,"15min":8.7530000000000001},"last":0.38900000000000001}]},{"osd":7,"last update":"Sat Sep 23 20:59:23 2023","interfaces":[{"interface":"back","average":{"1min":2.089,"5min":1.871,"15min":1.163},"min":{"1min":0.433,"5min":0.36199999999999999,"15min":0.35699999999999998},"max":{"1min":8.7769999999999992,"5min":8.7769999999999992,"15min":11.186999999999999},"last":0.41699999999999998},{"interface":"front","average":{"1min":2.5430000000000001,"5min":2.202,"15min":1.325},"min":{"1min":0.497,"5min":0.40899999999999997,"15min":0.34599999999999997},"max":{"1min":8.4939999999999998,"5min":15.098000000000001,"15min":15.098000000000001},"last":8.2449999999999992}]}]},{"osd":0,"up_from":83,"seq":356482285695,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1406848,"kb_used_data":1204600,"kb_used_omap":28,"kb_used_meta":202211,"kb_avail":92338304,"statfs":{"total":95995035648,"available":94554423296,"internally_reserved":0,"allocated":1233510400,"data_stored":1232337648,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":28848,"internal_metadata":207064912},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 20:59:19 2023","interfaces":[{"interface":"back","average":{"1min":3.694,"5min":3.121,"15min":1.4850000000000001},"min":{"1min":0.22700000000000001,"5min":0.191,"15min":0.191},"max":{"1min":15.577999999999999,"5min":16.484999999999999,"15min":18.863},"last":4.6479999999999997},{"interface":"front","average":{"1min":4.4610000000000003,"5min":3.0339999999999998,"15min":1.5389999999999999},"min":{"1min":0.20999999999999999,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":15.771000000000001,"5min":15.771000000000001,"15min":17.704999999999998},"last":4.6609999999999996}]},{"osd":2,"last update":"Sat Sep 23 20:59:51 2023","interfaces":[{"interface":"back","average":{"1min":3.8879999999999999,"5min":3.4180000000000001,"15min":1.867},"min":{"1min":0.50900000000000001,"5min":0.28299999999999997,"15min":0.23799999999999999},"max":{"1min":21.053000000000001,"5min":21.053000000000001,"15min":21.053000000000001},"last":4.718},{"interface":"front","average":{"1min":3.6579999999999999,"5min":3.7450000000000001,"15min":1.944},"min":{"1min":0.26300000000000001,"5min":0.25800000000000001,"15min":0.21099999999999999},"max":{"1min":13.956,"5min":21.747,"15min":21.747},"last":3.9089999999999998}]},{"osd":3,"last update":"Sat Sep 23 20:59:19 2023","interfaces":[{"interface":"back","average":{"1min":4.7839999999999998,"5min":3.0590000000000002,"15min":1.5900000000000001},"min":{"1min":0.38300000000000001,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":21.184000000000001,"5min":21.184000000000001,"15min":21.184000000000001},"last":6.3109999999999999},{"interface":"front","average":{"1min":6.0469999999999997,"5min":3.5510000000000002,"15min":1.794},"min":{"1min":0.30599999999999999,"5min":0.28000000000000003,"15min":0.23300000000000001},"max":{"1min":21.167000000000002,"5min":21.167000000000002,"15min":21.167000000000002},"last":5.9059999999999997}]},{"osd":4,"last update":"Sat Sep 23 20:59:51 2023","interfaces":[{"interface":"back","average":{"1min":1.9359999999999999,"5min":2.823,"15min":1.5589999999999999},"min":{"1min":0.38500000000000001,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":7.931,"5min":21.594999999999999,"15min":21.594999999999999},"last":0.622},{"interface":"front","average":{"1min":2.8639999999999999,"5min":1.9830000000000001,"15min":1.278},"min":{"1min":0.41099999999999998,"5min":0.31,"15min":0.28100000000000003},"max":{"1min":13.173999999999999,"5min":13.173999999999999,"15min":18.724},"last":0.49399999999999999}]},{"osd":5,"last update":"Sat Sep 23 20:59:51 2023","interfaces":[{"interface":"back","average":{"1min":1.859,"5min":1.889,"15min":1.258},"min":{"1min":0.38700000000000001,"5min":0.318,"15min":0.318},"max":{"1min":6.9980000000000002,"5min":8.8970000000000002,"15min":8.8970000000000002},"last":4.7919999999999998},{"interface":"front","average":{"1min":1.748,"5min":2.7290000000000001,"15min":1.5649999999999999},"min":{"1min":0.41899999999999998,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":7.9820000000000002,"5min":20.512,"15min":20.512},"last":0.65500000000000003}]},{"osd":6,"last update":"Sat Sep 23 20:59:46 2023","interfaces":[{"interface":"back","average":{"1min":2.9180000000000001,"5min":2.145,"15min":1.819},"min":{"1min":0.45900000000000002,"5min":0.37,"15min":0.37},"max":{"1min":13.134,"5min":13.134,"15min":13.134},"last":0.55700000000000005},{"interface":"front","average":{"1min":2.9140000000000001,"5min":2.198,"15min":1.8959999999999999},"min":{"1min":0.43099999999999999,"5min":0.377,"15min":0.377},"max":{"1min":13.254,"5min":13.254,"15min":13.254},"last":0.53000000000000003}]},{"osd":7,"last update":"Sat Sep 23 20:59:51 2023","interfaces":[{"interface":"back","average":{"1min":2.1520000000000001,"5min":2.0139999999999998,"15min":1.3169999999999999},"min":{"1min":0.433,"5min":0.379,"15min":0.379},"max":{"1min":6.9699999999999998,"5min":8.7729999999999997,"15min":8.7729999999999997},"last":4.7640000000000002},{"interface":"front","average":{"1min":3.0459999999999998,"5min":2.105,"15min":1.3200000000000001},"min":{"1min":0.44600000000000001,"5min":0.377,"15min":0.377},"max":{"1min":13.218999999999999,"5min":13.218999999999999,"15min":18.673999999999999},"last":3.8639999999999999}]}]},{"osd":2,"up_from":93,"seq":399431958642,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1575984,"kb_used_data":1258024,"kb_used_omap":81,"kb_used_meta":317550,"kb_avail":92169168,"statfs":{"total":95995035648,"available":94381228032,"internally_reserved":0,"allocated":1288216576,"data_stored":1287056572,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":83750,"internal_metadata":325171418},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:59:50 2023","interfaces":[{"interface":"back","average":{"1min":4.8730000000000002,"5min":2.7000000000000002,"15min":1.3799999999999999},"min":{"1min":0.189,"5min":0.17999999999999999,"15min":0.17999999999999999},"max":{"1min":29.498000000000001,"5min":29.498000000000001,"15min":29.498000000000001},"last":5.8780000000000001},{"interface":"front","average":{"1min":4.032,"5min":3.3370000000000002,"15min":1.6739999999999999},"min":{"1min":0.20599999999999999,"5min":0.20599999999999999,"15min":0.20599999999999999},"max":{"1min":17.251000000000001,"5min":17.251000000000001,"15min":17.251000000000001},"last":6.7009999999999996}]},{"osd":1,"last update":"Sat Sep 23 20:59:50 2023","interfaces":[{"interface":"back","average":{"1min":2.6030000000000002,"5min":2.5619999999999998,"15min":1.625},"min":{"1min":0.25700000000000001,"5min":0.20200000000000001,"15min":0.20200000000000001},"max":{"1min":7.9800000000000004,"5min":14.709,"15min":16.081},"last":2.843},{"interface":"front","average":{"1min":3.427,"5min":3.2570000000000001,"15min":1.97},"min":{"1min":0.26600000000000001,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":9.9090000000000007,"5min":11.717000000000001,"15min":16.041},"last":6.7329999999999997}]},{"osd":3,"last update":"Sat Sep 23 20:59:29 2023","interfaces":[{"interface":"back","average":{"1min":5.0449999999999999,"5min":3.6579999999999999,"15min":1.893},"min":{"1min":0.30299999999999999,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":27.614999999999998,"5min":27.614999999999998,"15min":27.614999999999998},"last":10.409000000000001},{"interface":"front","average":{"1min":5.9119999999999999,"5min":4.234,"15min":2.1930000000000001},"min":{"1min":0.33000000000000002,"5min":0.22800000000000001,"15min":0.221},"max":{"1min":27.562000000000001,"5min":27.562000000000001,"15min":27.562000000000001},"last":11.673999999999999}]},{"osd":4,"last update":"Sat Sep 23 20:59:50 2023","interfaces":[{"interface":"back","average":{"1min":1.575,"5min":1.7749999999999999,"15min":1.2070000000000001},"min":{"1min":0.36499999999999999,"5min":0.28699999999999998,"15min":0.26400000000000001},"max":{"1min":9.8659999999999997,"5min":9.8659999999999997,"15min":9.8659999999999997},"last":5.5789999999999997},{"interface":"front","average":{"1min":2.8439999999999999,"5min":2.1880000000000002,"15min":1.3720000000000001},"min":{"1min":0.32400000000000001,"5min":0.313,"15min":0.313},"max":{"1min":9.9719999999999995,"5min":10.609,"15min":10.609},"last":5.6970000000000001}]},{"osd":5,"last update":"Sat Sep 23 20:59:50 2023","interfaces":[{"interface":"back","average":{"1min":1.4179999999999999,"5min":1.99,"15min":1.298},"min":{"1min":0.33000000000000002,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":6.3570000000000002,"5min":9.0329999999999995,"15min":9.7629999999999999},"last":5.6100000000000003},{"interface":"front","average":{"1min":2.5760000000000001,"5min":2.1419999999999999,"15min":1.3720000000000001},"min":{"1min":0.35599999999999998,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":9.9220000000000006,"5min":10.574,"15min":10.574},"last":5.7119999999999997}]},{"osd":6,"last update":"Sat Sep 23 20:59:45 2023","interfaces":[{"interface":"back","average":{"1min":2.9060000000000001,"5min":2.0640000000000001,"15min":1.8},"min":{"1min":0.35699999999999998,"5min":0.312,"15min":0.312},"max":{"1min":10.012,"5min":14.244999999999999,"15min":14.244999999999999},"last":5.681},{"interface":"front","average":{"1min":2.8740000000000001,"5min":1.8129999999999999,"15min":1.2909999999999999},"min":{"1min":0.41599999999999998,"5min":0.374,"15min":0.374},"max":{"1min":10.032999999999999,"5min":10.032999999999999,"15min":10.032999999999999},"last":6.8739999999999997}]},{"osd":7,"last update":"Sat Sep 23 20:59:50 2023","interfaces":[{"interface":"back","average":{"1min":1.3859999999999999,"5min":1.4830000000000001,"15min":1.0449999999999999},"min":{"1min":0.35899999999999999,"5min":0.315,"15min":0.315},"max":{"1min":5.9199999999999999,"5min":8.0820000000000007,"15min":8.0820000000000007},"last":2.8119999999999998},{"interface":"front","average":{"1min":1.121,"5min":1.873,"15min":1.292},"min":{"1min":0.443,"5min":0.35099999999999998,"15min":0.34499999999999997},"max":{"1min":6.3899999999999997,"5min":8.6219999999999999,"15min":9.7859999999999996},"last":5.6269999999999998}]}]},{"osd":3,"up_from":98,"seq":420906795115,"num_pgs":49,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1457428,"kb_used_data":1107724,"kb_used_omap":7,"kb_used_meta":349688,"kb_avail":92287724,"statfs":{"total":95995035648,"available":94502629376,"internally_reserved":0,"allocated":1134309376,"data_stored":1133174538,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7576,"internal_metadata":358081128},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:59:28 2023","interfaces":[{"interface":"back","average":{"1min":4.056,"5min":2.399,"15min":1.7350000000000001},"min":{"1min":0.30199999999999999,"5min":0.20599999999999999,"15min":0.20599999999999999},"max":{"1min":13.566000000000001,"5min":13.566000000000001,"15min":23.123999999999999},"last":14.779},{"interface":"front","average":{"1min":5.2229999999999999,"5min":3.1099999999999999,"15min":1.6259999999999999},"min":{"1min":0.23200000000000001,"5min":0.19800000000000001,"15min":0.19800000000000001},"max":{"1min":13.455,"5min":15.079000000000001,"15min":15.079000000000001},"last":0.47199999999999998}]},{"osd":1,"last update":"Sat Sep 23 20:59:28 2023","interfaces":[{"interface":"back","average":{"1min":2.9510000000000001,"5min":2.8220000000000001,"15min":1.6120000000000001},"min":{"1min":0.23899999999999999,"5min":0.23899999999999999,"15min":0.22900000000000001},"max":{"1min":11.343,"5min":16.318000000000001,"15min":21.385000000000002},"last":5.1920000000000002},{"interface":"front","average":{"1min":2.972,"5min":2.2290000000000001,"15min":1.6830000000000001},"min":{"1min":0.23400000000000001,"5min":0.218,"15min":0.218},"max":{"1min":11.782,"5min":11.782,"15min":25.353000000000002},"last":14.734999999999999}]},{"osd":2,"last update":"Sat Sep 23 20:59:28 2023","interfaces":[{"interface":"back","average":{"1min":4.7210000000000001,"5min":3.1459999999999999,"15min":1.7310000000000001},"min":{"1min":0.33000000000000002,"5min":0.33000000000000002,"15min":0.30099999999999999},"max":{"1min":13.616,"5min":15.476000000000001,"15min":24.154},"last":5.7030000000000003},{"interface":"front","average":{"1min":3.665,"5min":2.4790000000000001,"15min":1.792},"min":{"1min":0.27300000000000002,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":13.125999999999999,"5min":16.032,"15min":25.305},"last":14.858000000000001}]},{"osd":4,"last update":"Sat Sep 23 20:59:28 2023","interfaces":[{"interface":"back","average":{"1min":3.0070000000000001,"5min":2.0299999999999998,"15min":1.3},"min":{"1min":0.379,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":11.089,"5min":11.089,"15min":11.861000000000001},"last":1.952},{"interface":"front","average":{"1min":1.954,"5min":1.359,"15min":1.335},"min":{"1min":0.309,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":9.9779999999999998,"5min":9.9779999999999998,"15min":22.853000000000002},"last":14.808}]},{"osd":5,"last update":"Sat Sep 23 20:59:28 2023","interfaces":[{"interface":"back","average":{"1min":2.8540000000000001,"5min":2.052,"15min":1.2929999999999999},"min":{"1min":0.39000000000000001,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":11.048,"5min":11.048,"15min":11.891999999999999},"last":1.909},{"interface":"front","average":{"1min":3.585,"5min":1.9510000000000001,"15min":1.272},"min":{"1min":0.53200000000000003,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":13.43,"5min":13.43,"15min":13.43},"last":0.41899999999999998}]},{"osd":6,"last update":"Sat Sep 23 20:59:47 2023","interfaces":[{"interface":"back","average":{"1min":2.0579999999999998,"5min":1.415,"15min":1.105},"min":{"1min":0.33700000000000002,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":13.499000000000001,"5min":13.499000000000001,"15min":13.499000000000001},"last":14.833},{"interface":"front","average":{"1min":3.8410000000000002,"5min":1.966,"15min":1.5960000000000001},"min":{"1min":0.55300000000000005,"5min":0.38700000000000001,"15min":0.38700000000000001},"max":{"1min":13.398999999999999,"5min":13.398999999999999,"15min":13.398999999999999},"last":0.45000000000000001}]},{"osd":7,"last update":"Sat Sep 23 20:59:28 2023","interfaces":[{"interface":"back","average":{"1min":2.8700000000000001,"5min":2.3839999999999999,"15min":1.4670000000000001},"min":{"1min":0.42799999999999999,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":11.114000000000001,"5min":16.364999999999998,"15min":16.364999999999998},"last":1.9770000000000001},{"interface":"front","average":{"1min":3.6400000000000001,"5min":2.077,"15min":1.3440000000000001},"min":{"1min":0.58399999999999996,"5min":0.379,"15min":0.379},"max":{"1min":13.355,"5min":13.355,"15min":13.355},"last":0.51200000000000001}]}]},{"osd":4,"up_from":31,"seq":133143986423,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":905640,"kb_used_data":760544,"kb_used_omap":0,"kb_used_meta":145088,"kb_avail":92839512,"statfs":{"total":95995035648,"available":95067660288,"internally_reserved":0,"allocated":778797056,"data_stored":777933602,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":148570112},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:59:53 2023","interfaces":[{"interface":"back","average":{"1min":2.1469999999999998,"5min":1.915,"15min":1.143},"min":{"1min":0.255,"5min":0.214,"15min":0.214},"max":{"1min":9.0749999999999993,"5min":11.784000000000001,"15min":11.784000000000001},"last":0.25900000000000001},{"interface":"front","average":{"1min":2.4969999999999999,"5min":2.0329999999999999,"15min":1.335},"min":{"1min":0.32800000000000001,"5min":0.26700000000000002,"15min":0.25800000000000001},"max":{"1min":9.1029999999999998,"5min":11.766,"15min":11.766},"last":0.35799999999999998}]},{"osd":1,"last update":"Sat Sep 23 20:59:22 2023","interfaces":[{"interface":"back","average":{"1min":2.4119999999999999,"5min":1.6930000000000001,"15min":1.1200000000000001},"min":{"1min":0.35499999999999998,"5min":0.246,"15min":0.23599999999999999},"max":{"1min":7.3310000000000004,"5min":10.153,"15min":10.153},"last":0.28199999999999997},{"interface":"front","average":{"1min":3.2669999999999999,"5min":2.0099999999999998,"15min":1.1419999999999999},"min":{"1min":0.33000000000000002,"5min":0.26100000000000001,"15min":0.23200000000000001},"max":{"1min":9.7509999999999994,"5min":12.292999999999999,"15min":12.292999999999999},"last":0.34100000000000003}]},{"osd":2,"last update":"Sat Sep 23 20:59:53 2023","interfaces":[{"interface":"back","average":{"1min":2.573,"5min":1.7430000000000001,"15min":1.2410000000000001},"min":{"1min":0.33600000000000002,"5min":0.311,"15min":0.29099999999999998},"max":{"1min":8.1509999999999998,"5min":8.1509999999999998,"15min":15.288},"last":3.4609999999999999},{"interface":"front","average":{"1min":2.8330000000000002,"5min":1.831,"15min":1.3999999999999999},"min":{"1min":0.28599999999999998,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":12.855,"5min":12.855,"15min":15.262},"last":3.774}]},{"osd":3,"last update":"Sat Sep 23 20:59:22 2023","interfaces":[{"interface":"back","average":{"1min":2.21,"5min":2.1850000000000001,"15min":1.6180000000000001},"min":{"1min":0.30499999999999999,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":4.835,"5min":14.824999999999999,"15min":31.562999999999999},"last":2.5009999999999999},{"interface":"front","average":{"1min":2.2770000000000001,"5min":2.2989999999999999,"15min":1.6899999999999999},"min":{"1min":0.30299999999999999,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":4.8490000000000002,"5min":14.839,"15min":31.574000000000002},"last":2.4569999999999999}]},{"osd":5,"last update":"Sat Sep 23 20:59:36 2023","interfaces":[{"interface":"back","average":{"1min":0.41099999999999998,"5min":0.61099999999999999,"15min":0.52800000000000002},"min":{"1min":0.28899999999999998,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":0.90700000000000003,"5min":3.3279999999999998,"15min":3.3279999999999998},"last":0.314},{"interface":"front","average":{"1min":1.002,"5min":0.77800000000000002,"15min":0.61499999999999999},"min":{"1min":0.29999999999999999,"5min":0.251,"15min":0.251},"max":{"1min":3.4830000000000001,"5min":7.7619999999999996,"15min":7.7619999999999996},"last":2.6539999999999999}]},{"osd":6,"last update":"Sat Sep 23 20:59:46 2023","interfaces":[{"interface":"back","average":{"1min":0.62,"5min":0.70399999999999996,"15min":0.55700000000000005},"min":{"1min":0.32100000000000001,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":3.4180000000000001,"5min":4.6909999999999998,"15min":4.6909999999999998},"last":1.8009999999999999},{"interface":"front","average":{"1min":0.752,"5min":0.73699999999999999,"15min":0.67500000000000004},"min":{"1min":0.34499999999999997,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":3.2999999999999998,"5min":3.456,"15min":3.456},"last":1.8340000000000001}]},{"osd":7,"last update":"Sat Sep 23 20:59:22 2023","interfaces":[{"interface":"back","average":{"1min":0.83499999999999996,"5min":0.752,"15min":0.69399999999999995},"min":{"1min":0.32600000000000001,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":2.8889999999999998,"5min":6.2240000000000002,"15min":7.383},"last":2.302},{"interface":"front","average":{"1min":0.73999999999999999,"5min":0.78400000000000003,"15min":0.67500000000000004},"min":{"1min":0.34200000000000003,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":2.8679999999999999,"5min":7.8310000000000004,"15min":7.8310000000000004},"last":0.39000000000000001}]}]},{"osd":5,"up_from":39,"seq":167503724788,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1080284,"kb_used_data":952724,"kb_used_omap":0,"kb_used_meta":127552,"kb_avail":92664868,"statfs":{"total":95995035648,"available":94888824832,"internally_reserved":0,"allocated":975589376,"data_stored":974734498,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":130613248},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:59:38 2023","interfaces":[{"interface":"back","average":{"1min":4.0339999999999998,"5min":1.619,"15min":0.96299999999999997},"min":{"1min":0.27400000000000002,"5min":0.20399999999999999,"15min":0.20399999999999999},"max":{"1min":12.795999999999999,"5min":12.795999999999999,"15min":12.795999999999999},"last":0.83499999999999996},{"interface":"front","average":{"1min":2.7330000000000001,"5min":1.948,"15min":1.125},"min":{"1min":0.24099999999999999,"5min":0.223,"15min":0.223},"max":{"1min":16.219999999999999,"5min":16.219999999999999,"15min":16.219999999999999},"last":8.8450000000000006}]},{"osd":1,"last update":"Sat Sep 23 20:59:17 2023","interfaces":[{"interface":"back","average":{"1min":2.7749999999999999,"5min":1.758,"15min":1.099},"min":{"1min":0.248,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":8.1829999999999998,"5min":8.3520000000000003,"15min":8.3520000000000003},"last":0.60699999999999998},{"interface":"front","average":{"1min":2.3690000000000002,"5min":1.825,"15min":1.099},"min":{"1min":0.27200000000000002,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":8.5399999999999991,"5min":10.285,"15min":10.285},"last":0.95099999999999996}]},{"osd":2,"last update":"Sat Sep 23 20:59:54 2023","interfaces":[{"interface":"back","average":{"1min":2.6819999999999999,"5min":1.6699999999999999,"15min":1.121},"min":{"1min":0.27400000000000002,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":10.371,"5min":10.579000000000001,"15min":12.651},"last":3.9060000000000001},{"interface":"front","average":{"1min":3.423,"5min":2.258,"15min":1.274},"min":{"1min":0.28799999999999998,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":10.333,"5min":12.300000000000001,"15min":12.300000000000001},"last":2.8130000000000002}]},{"osd":3,"last update":"Sat Sep 23 20:59:17 2023","interfaces":[{"interface":"back","average":{"1min":3.5880000000000001,"5min":2.2090000000000001,"15min":1.351},"min":{"1min":0.46700000000000003,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":12.188000000000001,"5min":12.188000000000001,"15min":12.188000000000001},"last":3.411},{"interface":"front","average":{"1min":2.0739999999999998,"5min":1.6060000000000001,"15min":1.071},"min":{"1min":0.32500000000000001,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":7.258,"5min":10.457000000000001,"15min":10.457000000000001},"last":3.258}]},{"osd":4,"last update":"Sat Sep 23 20:59:34 2023","interfaces":[{"interface":"back","average":{"1min":0.81699999999999995,"5min":0.63300000000000001,"15min":0.55300000000000005},"min":{"1min":0.32100000000000001,"5min":0.252,"15min":0.252},"max":{"1min":2.9169999999999998,"5min":3.8479999999999999,"15min":3.8479999999999999},"last":0.45000000000000001},{"interface":"front","average":{"1min":0.83499999999999996,"5min":0.67500000000000004,"15min":0.57499999999999996},"min":{"1min":0.27400000000000002,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":3.3530000000000002,"5min":3.3530000000000002,"15min":3.3530000000000002},"last":0.30299999999999999}]},{"osd":6,"last update":"Sat Sep 23 20:59:38 2023","interfaces":[{"interface":"back","average":{"1min":0.86099999999999999,"5min":0.65100000000000002,"15min":0.59099999999999997},"min":{"1min":0.29199999999999998,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":4.1120000000000001,"5min":4.1120000000000001,"15min":4.1120000000000001},"last":1.3700000000000001},{"interface":"front","average":{"1min":1.0269999999999999,"5min":0.755,"15min":0.747},"min":{"1min":0.36899999999999999,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":2.9620000000000002,"5min":3.6560000000000001,"15min":3.6560000000000001},"last":1.3879999999999999}]},{"osd":7,"last update":"Sat Sep 23 20:59:17 2023","interfaces":[{"interface":"back","average":{"1min":0.71099999999999997,"5min":0.80200000000000005,"15min":0.64200000000000002},"min":{"1min":0.33400000000000002,"5min":0.307,"15min":0.30199999999999999},"max":{"1min":2.516,"5min":8.5649999999999995,"15min":8.5649999999999995},"last":1.1200000000000001},{"interface":"front","average":{"1min":0.77200000000000002,"5min":0.75,"15min":0.65700000000000003},"min":{"1min":0.34399999999999997,"5min":0.29799999999999999,"15min":0.27700000000000002},"max":{"1min":3.0150000000000001,"5min":4.5430000000000001,"15min":4.5430000000000001},"last":0.40500000000000003}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1224413184,"data_stored":1224409112,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1316032512,"data_stored":1316028440,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1279852544,"data_stored":1279852544,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1127809024,"data_stored":1127809024,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":765984768,"data_stored":765984768,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":972685312,"data_stored":972685312,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":810291200,"data_stored":810287128,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":766050304,"data_stored":766050304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T20:59:59.482 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph status --format=json 2023-09-23T20:59:59.482 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:59:59.509 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:59:59.549 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:59:59.553 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:59:59.553 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:59:59.654 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:59:59.775 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:59:59 smithi026 ceph-mon[179688]: pgmap v608: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 2.3 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 607 op/s; 11798/114675 objects degraded (10.288%); 1452/114675 objects misplaced (1.266%); 1.8 MiB/s, 29 objects/s recovering 2023-09-23T20:59:59.775 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 20:59:59 smithi026 ceph-mon[179688]: from='client.35340 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:59:59.776 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:59:59 smithi026 ceph-mon[175151]: pgmap v608: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 2.3 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 607 op/s; 11798/114675 objects degraded (10.288%); 1452/114675 objects misplaced (1.266%); 1.8 MiB/s, 29 objects/s recovering 2023-09-23T20:59:59.776 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 20:59:59 smithi026 ceph-mon[175151]: from='client.35340 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:59:59.777 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 20:59:59 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[164029]: ::ffff:172.21.15.70 - - [23/Sep/2023:20:59:59] "GET /metrics HTTP/1.1" 200 36323 "" "Prometheus/2.43.0" 2023-09-23T20:59:59.793 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:59:59.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:59:59 smithi070 ceph-mon[168282]: pgmap v608: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 2.3 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 607 op/s; 11798/114675 objects degraded (10.288%); 1452/114675 objects misplaced (1.266%); 1.8 MiB/s, 29 objects/s recovering 2023-09-23T20:59:59.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 20:59:59 smithi070 ceph-mon[168282]: from='client.35340 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:59:59.969 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:59:59.976 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:59:59.979 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:59:59.979 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:00:00.080 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:00:00.120 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:00:00.166 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:00:00.169 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:00:00.170 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:00:00.271 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:00:00.315 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:00:00.355 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:00:00.365 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:00:00.365 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:00:00.466 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:00:00.498 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:00:00.546 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:00:00.552 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:00:00.553 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:00:00.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:00:00 smithi070 ceph-mon[168282]: Health detail: HEALTH_WARN nodeep-scrub flag(s) set; Degraded data redundancy: 11798/114675 objects degraded (10.288%), 3 pgs degraded, 3 pgs undersized 2023-09-23T21:00:00.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:00:00 smithi070 ceph-mon[168282]: [WRN] OSDMAP_FLAGS: nodeep-scrub flag(s) set 2023-09-23T21:00:00.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:00:00 smithi070 ceph-mon[168282]: [WRN] PG_DEGRADED: Degraded data redundancy: 11798/114675 objects degraded (10.288%), 3 pgs degraded, 3 pgs undersized 2023-09-23T21:00:00.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:00:00 smithi070 ceph-mon[168282]: pg 7.1 is stuck undersized for 2m, current state active+recovery_wait+undersized+degraded+remapped, last acting [4,6] 2023-09-23T21:00:00.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:00:00 smithi070 ceph-mon[168282]: pg 7.6 is stuck undersized for 2m, current state active+recovering+undersized+degraded+remapped, last acting [0,7] 2023-09-23T21:00:00.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:00:00 smithi070 ceph-mon[168282]: pg 7.c is stuck undersized for 2m, current state active+recovery_wait+undersized+degraded+remapped, last acting [6,7] 2023-09-23T21:00:00.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:00:00 smithi070 ceph-mon[168282]: Health check update: Degraded data redundancy: 11798/114675 objects degraded (10.288%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T21:00:00.654 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:00:00.685 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:00:00.724 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:00:00.726 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:00:00.727 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:00:00.827 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:00:00.881 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:00:00 smithi026 ceph-mon[179688]: Health detail: HEALTH_WARN nodeep-scrub flag(s) set; Degraded data redundancy: 11798/114675 objects degraded (10.288%), 3 pgs degraded, 3 pgs undersized 2023-09-23T21:00:00.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:00:00 smithi026 ceph-mon[179688]: [WRN] OSDMAP_FLAGS: nodeep-scrub flag(s) set 2023-09-23T21:00:00.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:00:00 smithi026 ceph-mon[179688]: [WRN] PG_DEGRADED: Degraded data redundancy: 11798/114675 objects degraded (10.288%), 3 pgs degraded, 3 pgs undersized 2023-09-23T21:00:00.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:00:00 smithi026 ceph-mon[179688]: pg 7.1 is stuck undersized for 2m, current state active+recovery_wait+undersized+degraded+remapped, last acting [4,6] 2023-09-23T21:00:00.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:00:00 smithi026 ceph-mon[179688]: pg 7.6 is stuck undersized for 2m, current state active+recovering+undersized+degraded+remapped, last acting [0,7] 2023-09-23T21:00:00.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:00:00 smithi026 ceph-mon[179688]: pg 7.c is stuck undersized for 2m, current state active+recovery_wait+undersized+degraded+remapped, last acting [6,7] 2023-09-23T21:00:00.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:00:00 smithi026 ceph-mon[179688]: Health check update: Degraded data redundancy: 11798/114675 objects degraded (10.288%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T21:00:00.883 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:00:00 smithi026 ceph-mon[175151]: Health detail: HEALTH_WARN nodeep-scrub flag(s) set; Degraded data redundancy: 11798/114675 objects degraded (10.288%), 3 pgs degraded, 3 pgs undersized 2023-09-23T21:00:00.883 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:00:00 smithi026 ceph-mon[175151]: [WRN] OSDMAP_FLAGS: nodeep-scrub flag(s) set 2023-09-23T21:00:00.883 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:00:00 smithi026 ceph-mon[175151]: [WRN] PG_DEGRADED: Degraded data redundancy: 11798/114675 objects degraded (10.288%), 3 pgs degraded, 3 pgs undersized 2023-09-23T21:00:00.884 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:00:00 smithi026 ceph-mon[175151]: pg 7.1 is stuck undersized for 2m, current state active+recovery_wait+undersized+degraded+remapped, last acting [4,6] 2023-09-23T21:00:00.884 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:00:00 smithi026 ceph-mon[175151]: pg 7.6 is stuck undersized for 2m, current state active+recovering+undersized+degraded+remapped, last acting [0,7] 2023-09-23T21:00:00.884 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:00:00 smithi026 ceph-mon[175151]: pg 7.c is stuck undersized for 2m, current state active+recovery_wait+undersized+degraded+remapped, last acting [6,7] 2023-09-23T21:00:00.884 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:00:00 smithi026 ceph-mon[175151]: Health check update: Degraded data redundancy: 11798/114675 objects degraded (10.288%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T21:00:00.931 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:00:01.744 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:00:01.758 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:00:01.758 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:00:01.829 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T21:00:01.859 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:00:02.006 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:00:02.121 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:00:02.147 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:00:02.147 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:00:02.248 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:00:02.295 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:00:02 smithi026 ceph-mon[175151]: pgmap v609: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 2.3 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 598 op/s; 11798/110904 objects degraded (10.638%); 1452/110904 objects misplaced (1.309%); 1.8 MiB/s, 29 objects/s recovering 2023-09-23T21:00:02.295 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:00:02 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:00:02.295 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:00:02 smithi026 ceph-mon[179688]: pgmap v609: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 2.3 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 598 op/s; 11798/110904 objects degraded (10.638%); 1452/110904 objects misplaced (1.309%); 1.8 MiB/s, 29 objects/s recovering 2023-09-23T21:00:02.295 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:00:02 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:00:02.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:00:02 smithi070 ceph-mon[168282]: pgmap v609: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 2.3 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 598 op/s; 11798/110904 objects degraded (10.638%); 1452/110904 objects misplaced (1.309%); 1.8 MiB/s, 29 objects/s recovering 2023-09-23T21:00:02.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:00:02 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:00:02.390 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:00:02.744 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:00:02.752 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:00:02.752 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:00:02.814 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:00:02.853 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:00:03.016 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:00:03.030 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:00:03.149 INFO:teuthology.orchestra.run.smithi026.stderr:nodeep-scrub is unset 2023-09-23T21:00:03.224 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:00:03.235 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:00:03.235 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:00:03.277 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:00:03 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T21:00:03.277 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:00:03 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/1378743577' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:00:03.277 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:00:03 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T21:00:03.277 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:00:03 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/1378743577' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:00:03.337 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:00:03.380 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:00:03 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T21:00:03.380 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:00:03 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/1378743577' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:00:03.461 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:00:03.972 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:00:03.976 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:00:03.976 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:00:04.027 INFO:teuthology.orchestra.run.smithi026.stdout:{ 2023-09-23T21:00:04.027 INFO:teuthology.orchestra.run.smithi026.stdout: "ops": [], 2023-09-23T21:00:04.027 INFO:teuthology.orchestra.run.smithi026.stdout: "complaint_time": 30, 2023-09-23T21:00:04.027 INFO:teuthology.orchestra.run.smithi026.stdout: "num_blocked_ops": 0 2023-09-23T21:00:04.027 INFO:teuthology.orchestra.run.smithi026.stdout:} 2023-09-23T21:00:04.077 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:00:04.153 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:00:04.384 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:00:04 smithi026 ceph-mon[175151]: pgmap v610: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 2.2 GiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 581 op/s; 11801/106194 objects degraded (11.113%); 1282/106194 objects misplaced (1.207%); 2.0 MiB/s, 31 objects/s recovering 2023-09-23T21:00:04.384 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:00:04 smithi026 ceph-mon[175151]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T21:00:04.384 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:00:04 smithi026 ceph-mon[175151]: failed to encode map e244 with expected crc 2023-09-23T21:00:04.384 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:00:04 smithi026 ceph-mon[175151]: failed to encode map e244 with expected crc 2023-09-23T21:00:04.384 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:00:04 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/1378743577' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T21:00:04.384 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:00:04 smithi026 ceph-mon[175151]: osdmap e244: 8 total, 8 up, 8 in 2023-09-23T21:00:04.385 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:00:04 smithi026 ceph-mon[175151]: failed to encode map e244 with expected crc 2023-09-23T21:00:04.385 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:00:04 smithi026 ceph-mon[179688]: pgmap v610: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 2.2 GiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 581 op/s; 11801/106194 objects degraded (11.113%); 1282/106194 objects misplaced (1.207%); 2.0 MiB/s, 31 objects/s recovering 2023-09-23T21:00:04.385 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:00:04 smithi026 ceph-mon[179688]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T21:00:04.385 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:00:04 smithi026 ceph-mon[179688]: failed to encode map e244 with expected crc 2023-09-23T21:00:04.385 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:00:04 smithi026 ceph-mon[179688]: failed to encode map e244 with expected crc 2023-09-23T21:00:04.385 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:00:04 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/1378743577' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T21:00:04.385 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:00:04 smithi026 ceph-mon[179688]: osdmap e244: 8 total, 8 up, 8 in 2023-09-23T21:00:04.385 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:00:04 smithi026 ceph-mon[179688]: failed to encode map e244 with expected crc 2023-09-23T21:00:04.473 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:00:04.481 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:00:04.482 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:00:04.583 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:00:04.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:00:04 smithi070 ceph-mon[168282]: pgmap v610: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 2.2 GiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 581 op/s; 11801/106194 objects degraded (11.113%); 1282/106194 objects misplaced (1.207%); 2.0 MiB/s, 31 objects/s recovering 2023-09-23T21:00:04.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:00:04 smithi070 ceph-mon[168282]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T21:00:04.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:00:04 smithi070 ceph-mon[168282]: failed to encode map e244 with expected crc 2023-09-23T21:00:04.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:00:04 smithi070 ceph-mon[168282]: failed to encode map e244 with expected crc 2023-09-23T21:00:04.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:00:04 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/1378743577' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T21:00:04.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:00:04 smithi070 ceph-mon[168282]: osdmap e244: 8 total, 8 up, 8 in 2023-09-23T21:00:04.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:00:04 smithi070 ceph-mon[168282]: failed to encode map e244 with expected crc 2023-09-23T21:00:04.695 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:00:05.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:00:05 smithi026 ceph-mon[175151]: Health check update: Degraded data redundancy: 11801/106194 objects degraded (11.113%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T21:00:05.383 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:00:05 smithi026 ceph-mon[179688]: Health check update: Degraded data redundancy: 11801/106194 objects degraded (11.113%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T21:00:05.411 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_historic_ops 2023-09-23T21:00:05.479 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:00:05.483 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:00:05.483 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:00:05.584 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:00:05.615 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:00:05 smithi070 ceph-mon[168282]: Health check update: Degraded data redundancy: 11801/106194 objects degraded (11.113%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T21:00:05.619 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:00:05.662 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:00:05.665 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:00:05.665 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:00:05.766 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:00:05.793 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:00:05.819 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T21:00:05.832 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:00:05.835 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:00:05.835 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:00:05.936 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:00:06.131 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:00:06.235 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:00:06.248 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:00:06.248 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:00:06.349 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:00:06.385 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:00:06 smithi026 ceph-mon[175151]: pgmap v612: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 2.1 GiB data, 8.7 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 586 op/s; 11801/103962 objects degraded (11.351%); 1282/103962 objects misplaced (1.233%); 2.0 MiB/s, 31 objects/s recovering 2023-09-23T21:00:06.385 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:00:06 smithi026 ceph-mon[179688]: pgmap v612: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 2.1 GiB data, 8.7 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 586 op/s; 11801/103962 objects degraded (11.351%); 1282/103962 objects misplaced (1.233%); 2.0 MiB/s, 31 objects/s recovering 2023-09-23T21:00:06.506 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:00:06.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:00:06 smithi070 ceph-mon[168282]: pgmap v612: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 2.1 GiB data, 8.7 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 586 op/s; 11801/103962 objects degraded (11.351%); 1282/103962 objects misplaced (1.233%); 2.0 MiB/s, 31 objects/s recovering 2023-09-23T21:00:06.643 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0: {} 2023-09-23T21:00:06.652 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:00:06.657 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:00:06.657 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:00:06.705 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1: {} 2023-09-23T21:00:06.729 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:00:06.739 INFO:teuthology.orchestra.run.smithi026.stdout: 2023-09-23T21:00:06.739 INFO:teuthology.orchestra.run.smithi026.stdout:{"fsid":"b3f8e4ec-5a50-11ee-8db4-212e2dc638e7","health":{"status":"HEALTH_WARN","checks":{"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 11801/103962 objects degraded (11.351%), 3 pgs degraded, 3 pgs undersized","count":6},"muted":false}},"mutes":[]},"election_epoch":30,"quorum":[0,1,2],"quorum_names":["a","b","c"],"quorum_age":725,"monmap":{"epoch":4,"min_mon_release_name":"reef","num_mons":3},"osdmap":{"epoch":244,"num_osds":8,"num_up_osds":8,"osd_up_since":1695502534,"num_in_osds":8,"osd_in_since":1695502470,"num_remapped_pgs":3},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":106},{"state_name":"active+recovery_wait+undersized+degraded+remapped","count":2},{"state_name":"active+recovering+undersized+degraded+remapped","count":1}],"num_pgs":109,"num_pools":5,"num_objects":34654,"data_bytes":2270042040,"bytes_used":9349832704,"bytes_avail":758610452480,"bytes_total":767960285184,"degraded_objects":11801,"degraded_total":103962,"degraded_ratio":0.11351262961466689,"misplaced_objects":1282,"misplaced_total":103962,"misplaced_ratio":0.012331428791289125,"recovering_objects_per_sec":31,"recovering_bytes_per_sec":2096412,"recovering_keys_per_sec":0,"num_objects_recovered":320,"num_bytes_recovered":20971520,"num_keys_recovered":0,"write_bytes_sec":0,"read_op_per_sec":0,"write_op_per_sec":586},"fsmap":{"epoch":8,"id":1,"up":1,"in":1,"max":1,"by_rank":[{"filesystem_id":1,"rank":0,"name":"foo.smithi026.znfpnl","status":"up:active","gid":14658}],"up:standby":1},"mgrmap":{"available":true,"num_standbys":1,"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"services":{"dashboard":"https://172.21.15.26:8443/","prometheus":"http://172.21.15.26:9283/"}},"servicemap":{"epoch":21,"modified":"2023-09-23T20:56:34.314324+0000","services":{}},"progress_events":{"8b64cbfe-d367-468b-a773-0e71ee4ff645":{"message":"Global Recovery Event (0s)\n [............................] ","progress":0,"add_to_ceph_s":true},"fc746e9d-3f76-434d-b6d2-8fc9e8a72a5e":{"message":"Upgrade to 18.2.0-460-gbb0daeff (4m)\n [============................] (remaining: 5m)","progress":0.44999998807907104,"add_to_ceph_s":true}}} 2023-09-23T21:00:06.754 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2: {} 2023-09-23T21:00:06.758 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:00:06.816 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:00:06.834 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3: {} 2023-09-23T21:00:06.884 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4: {} 2023-09-23T21:00:06.889 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:00:06.951 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T21:00:06.999 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6: {} 2023-09-23T21:00:07.058 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T21:00:07.096 INFO:teuthology.orchestra.run.smithi026.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T21:00:07.096 INFO:teuthology.orchestra.run.smithi026.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T21:00:07.096 INFO:teuthology.orchestra.run.smithi026.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T21:00:07.096 INFO:teuthology.orchestra.run.smithi026.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T21:00:07.096 INFO:teuthology.orchestra.run.smithi026.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T21:00:07.096 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T21:00:07.096 INFO:teuthology.orchestra.run.smithi026.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T21:00:07.096 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T21:00:07.383 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:00:07 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/2770699868' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:00:07.383 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:00:07 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/2886632095' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-23T21:00:07.383 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:00:07 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/2770699868' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:00:07.383 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:00:07 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/2886632095' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-23T21:00:07.621 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:00:07 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/2770699868' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:00:07.621 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:00:07 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/2886632095' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-23T21:00:08.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:00:08 smithi070 ceph-mon[168282]: pgmap v613: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 2.1 GiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 581 op/s; 11801/102336 objects degraded (11.532%); 1282/102336 objects misplaced (1.253%); 2.0 MiB/s, 31 objects/s recovering 2023-09-23T21:00:08.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:00:08 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:00:08.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:00:08 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:00:08.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:00:08 smithi026 ceph-mon[179688]: pgmap v613: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 2.1 GiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 581 op/s; 11801/102336 objects degraded (11.532%); 1282/102336 objects misplaced (1.253%); 2.0 MiB/s, 31 objects/s recovering 2023-09-23T21:00:08.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:00:08 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:00:08.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:00:08 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:00:08.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:00:08 smithi026 ceph-mon[175151]: pgmap v613: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 2.1 GiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 581 op/s; 11801/102336 objects degraded (11.532%); 1282/102336 objects misplaced (1.253%); 2.0 MiB/s, 31 objects/s recovering 2023-09-23T21:00:08.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:00:08 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:00:08.633 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:00:08 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:00:09.403 INFO:tasks.cephadm.ceph_manager.ceph:making progress, resetting timeout 2023-09-23T21:00:09.403 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T21:00:09.462 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:00:09.470 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:00:09.470 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:00:09.571 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:00:09.685 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:00:09.880 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 21:00:09 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[164029]: ::ffff:172.21.15.70 - - [23/Sep/2023:21:00:09] "GET /metrics HTTP/1.1" 200 36313 "" "Prometheus/2.43.0" 2023-09-23T21:00:10.477 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:00:10 smithi026 ceph-mon[175151]: pgmap v614: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 2.0 GiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 565 op/s; 11827/97704 objects degraded (12.105%); 1115/97704 objects misplaced (1.141%); 1.9 MiB/s, 30 objects/s recovering 2023-09-23T21:00:10.477 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:00:10 smithi026 ceph-mon[175151]: Health check update: Degraded data redundancy: 11827/97704 objects degraded (12.105%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T21:00:10.479 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:00:10 smithi026 ceph-mon[179688]: pgmap v614: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 2.0 GiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 565 op/s; 11827/97704 objects degraded (12.105%); 1115/97704 objects misplaced (1.141%); 1.9 MiB/s, 30 objects/s recovering 2023-09-23T21:00:10.479 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:00:10 smithi026 ceph-mon[179688]: Health check update: Degraded data redundancy: 11827/97704 objects degraded (12.105%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T21:00:10.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:00:10 smithi070 ceph-mon[168282]: pgmap v614: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 2.0 GiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 565 op/s; 11827/97704 objects degraded (12.105%); 1115/97704 objects misplaced (1.141%); 1.9 MiB/s, 30 objects/s recovering 2023-09-23T21:00:10.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:00:10 smithi070 ceph-mon[168282]: Health check update: Degraded data redundancy: 11827/97704 objects degraded (12.105%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T21:00:10.848 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:00:10.854 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:00:10.855 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:00:10.955 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:00:11.136 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:00:11.454 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:00:11.461 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:00:11.462 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:00:11.499 INFO:teuthology.orchestra.run.smithi026.stdout:ERROR: (22) Invalid argument 2023-09-23T21:00:11.499 INFO:teuthology.orchestra.run.smithi026.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T21:00:11.563 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:00:11.594 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:00:11.631 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:00:11.634 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:00:11.634 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:00:11.735 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:00:11.767 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:00:11.805 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:00:11.812 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:00:11.812 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:00:11.913 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:00:11.941 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:00:11.980 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:00:11.982 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:00:11.983 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:00:12.083 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:00:12.192 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:00:12.364 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:00:12.373 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:00:12.374 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:00:12.399 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:00:12.409 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T21:00:12.431 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_ops_in_flight 2023-09-23T21:00:12.435 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:00:12 smithi026 ceph-mon[175151]: pgmap v615: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 1.9 GiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 567 op/s; 11827/93861 objects degraded (12.601%); 1115/93861 objects misplaced (1.188%); 1.9 MiB/s, 30 objects/s recovering 2023-09-23T21:00:12.435 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:00:12 smithi026 ceph-mon[179688]: pgmap v615: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 1.9 GiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 567 op/s; 11827/93861 objects degraded (12.601%); 1115/93861 objects misplaced (1.188%); 1.9 MiB/s, 30 objects/s recovering 2023-09-23T21:00:12.456 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:00:12 smithi070 ceph-mon[168282]: pgmap v615: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 1.9 GiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 567 op/s; 11827/93861 objects degraded (12.601%); 1115/93861 objects misplaced (1.188%); 1.9 MiB/s, 30 objects/s recovering 2023-09-23T21:00:12.475 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:00:12.505 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:00:12.554 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:00:12.557 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:00:12.558 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:00:12.658 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:00:12.696 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:00:12.735 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:00:12.739 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:00:12.739 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:00:12.840 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:00:12.959 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:00:13.001 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T21:00:13.294 INFO:teuthology.orchestra.run.smithi026.stderr:noscrub is set 2023-09-23T21:00:13.308 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:00:13.320 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:00:13.320 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:00:13.321 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:00:13 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/3603466069' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T21:00:13.321 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:00:13 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T21:00:13.321 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:00:13 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/3603466069' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T21:00:13.321 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:00:13 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T21:00:13.421 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:00:13.508 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:00:13.547 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:00:13 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/3603466069' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T21:00:13.547 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:00:13 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T21:00:13.597 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:00:13.610 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:00:13.611 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:00:13.711 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:00:13.744 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:00:13.796 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:00:13.803 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:00:13.803 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:00:13.904 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:00:14.031 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:00:14.151 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:00:14.159 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:00:14.160 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:00:14.169 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:00:14.220 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:00:14 smithi026 ceph-mon[175151]: pgmap v616: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 1.8 GiB data, 7.9 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 583 op/s; 11874/88695 objects degraded (13.387%); 900/88695 objects misplaced (1.015%); 1.9 MiB/s, 30 objects/s recovering 2023-09-23T21:00:14.220 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:00:14 smithi026 ceph-mon[175151]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:00:14.220 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:00:14 smithi026 ceph-mon[175151]: failed to encode map e245 with expected crc 2023-09-23T21:00:14.220 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:00:14 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T21:00:14.220 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:00:14 smithi026 ceph-mon[175151]: osdmap e245: 8 total, 8 up, 8 in 2023-09-23T21:00:14.220 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:00:14 smithi026 ceph-mon[175151]: failed to encode map e245 with expected crc 2023-09-23T21:00:14.261 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:00:14.295 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:00:14.319 INFO:teuthology.orchestra.run.smithi070.stdout:ERROR: (22) Invalid argument 2023-09-23T21:00:14.319 INFO:teuthology.orchestra.run.smithi070.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T21:00:14.341 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:00:14.341 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:00:14 smithi070 ceph-mon[168282]: pgmap v616: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 1.8 GiB data, 7.9 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 583 op/s; 11874/88695 objects degraded (13.387%); 900/88695 objects misplaced (1.015%); 1.9 MiB/s, 30 objects/s recovering 2023-09-23T21:00:14.342 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:00:14 smithi070 ceph-mon[168282]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:00:14.342 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:00:14 smithi070 ceph-mon[168282]: failed to encode map e245 with expected crc 2023-09-23T21:00:14.342 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:00:14 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T21:00:14.342 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:00:14 smithi070 ceph-mon[168282]: osdmap e245: 8 total, 8 up, 8 in 2023-09-23T21:00:14.342 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:00:14 smithi070 ceph-mon[168282]: failed to encode map e245 with expected crc 2023-09-23T21:00:14.342 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:00:14 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:00:14.342 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:00:14 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:00:14.345 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:00:14.346 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:00:14.447 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:00:14.525 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:00:14 smithi026 ceph-mon[179688]: pgmap v616: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 1.8 GiB data, 7.9 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 583 op/s; 11874/88695 objects degraded (13.387%); 900/88695 objects misplaced (1.015%); 1.9 MiB/s, 30 objects/s recovering 2023-09-23T21:00:14.525 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:00:14 smithi026 ceph-mon[179688]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:00:14.525 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:00:14 smithi026 ceph-mon[179688]: failed to encode map e245 with expected crc 2023-09-23T21:00:14.525 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:00:14 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T21:00:14.525 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:00:14 smithi026 ceph-mon[179688]: osdmap e245: 8 total, 8 up, 8 in 2023-09-23T21:00:14.525 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:00:14 smithi026 ceph-mon[179688]: failed to encode map e245 with expected crc 2023-09-23T21:00:14.525 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:00:14 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:00:14.525 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:00:14 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:00:14.526 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:00:14 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:00:14.526 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:00:14 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:00:14.553 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:00:14.634 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:00:14.639 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:00:14.639 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:00:14.687 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_blocked_ops 2023-09-23T21:00:14.740 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:00:14.842 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:00:15.080 INFO:teuthology.orchestra.run.smithi026.stdout: 2023-09-23T21:00:15.101 INFO:teuthology.orchestra.run.smithi026.stderr:dumped all 2023-09-23T21:00:15.359 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T21:00:15.571 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:00:15 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T21:00:15.571 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:00:15 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T21:00:15.571 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:00:15 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:00:15.571 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:00:15 smithi070 ceph-mon[168282]: Health check update: Degraded data redundancy: 11874/88695 objects degraded (13.387%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T21:00:15.636 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:00:15 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T21:00:15.636 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:00:15 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T21:00:15.636 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:00:15 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:00:15.636 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:00:15 smithi026 ceph-mon[175151]: Health check update: Degraded data redundancy: 11874/88695 objects degraded (13.387%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T21:00:15.636 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:00:15 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T21:00:15.637 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:00:15 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T21:00:15.637 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:00:15 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:00:15.637 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:00:15 smithi026 ceph-mon[179688]: Health check update: Degraded data redundancy: 11874/88695 objects degraded (13.387%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T21:00:15.965 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T21:00:16.042 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T21:00:16.047 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:00:16.047 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:00:16.140 INFO:teuthology.orchestra.run.smithi026.stdout:{"pg_ready":true,"pg_map":{"version":618,"stamp":"2023-09-23T21:00:14.478733+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":1890654136,"num_objects":28865,"num_object_clones":0,"num_object_copies":86595,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":11874,"num_objects_misplaced":900,"num_objects_unfound":0,"num_objects_dirty":28865,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":174740,"num_write_kb":6512606,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3590,"num_bytes_recovered":233838438,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":122515,"ondisk_log_size":122515,"up":327,"acting":324,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":331,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":8143404,"kb_used_data":6464876,"kb_used_omap":136,"kb_used_meta":1678135,"kb_avail":741817812,"statfs":{"total":767960285184,"available":759621439488,"internally_reserved":0,"allocated":6620033024,"data_stored":6611881666,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":139741,"internal_metadata":1718410787},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":9,"apply_latency_ms":9,"commit_latency_ns":9000000,"apply_latency_ns":9000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-379387904,"num_objects":-5789,"num_object_clones":0,"num_object_copies":-17367,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":73,"num_objects_misplaced":-382,"num_objects_unfound":0,"num_objects_dirty":-5789,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5789,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":309,"num_bytes_recovered":20250624,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.006010"},"pg_stats":[{"pgid":"3.1a","version":"68'1","reported_seq":121,"reported_epoch":243,"state":"active+clean","last_fresh":"2023-09-23T20:59:52.342798+0000","last_change":"2023-09-23T20:54:13.946695+0000","last_active":"2023-09-23T20:59:52.342798+0000","last_peered":"2023-09-23T20:59:52.342798+0000","last_clean":"2023-09-23T20:59:52.342798+0000","last_became_active":"2023-09-23T20:54:13.945399+0000","last_became_peered":"2023-09-23T20:54:13.945399+0000","last_unstale":"2023-09-23T20:59:52.342798+0000","last_undegraded":"2023-09-23T20:59:52.342798+0000","last_fullsized":"2023-09-23T20:59:52.342798+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:32:52.365464+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":236,"reported_epoch":243,"state":"active+clean","last_fresh":"2023-09-23T20:59:52.341155+0000","last_change":"2023-09-23T20:54:13.942019+0000","last_active":"2023-09-23T20:59:52.341155+0000","last_peered":"2023-09-23T20:59:52.341155+0000","last_clean":"2023-09-23T20:59:52.341155+0000","last_became_active":"2023-09-23T20:54:13.941647+0000","last_became_peered":"2023-09-23T20:54:13.941647+0000","last_unstale":"2023-09-23T20:59:52.341155+0000","last_undegraded":"2023-09-23T20:59:52.341155+0000","last_fullsized":"2023-09-23T20:59:52.341155+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:30:54.400944+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":215,"reported_epoch":244,"state":"active+clean","last_fresh":"2023-09-23T21:00:03.120302+0000","last_change":"2023-09-23T20:54:13.941394+0000","last_active":"2023-09-23T21:00:03.120302+0000","last_peered":"2023-09-23T21:00:03.120302+0000","last_clean":"2023-09-23T21:00:03.120302+0000","last_became_active":"2023-09-23T20:54:13.941042+0000","last_became_peered":"2023-09-23T20:54:13.941042+0000","last_unstale":"2023-09-23T21:00:03.120302+0000","last_undegraded":"2023-09-23T21:00:03.120302+0000","last_fullsized":"2023-09-23T21:00:03.120302+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:19:41.401184+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":229,"reported_epoch":243,"state":"active+clean","last_fresh":"2023-09-23T20:59:52.355523+0000","last_change":"2023-09-23T20:57:44.979811+0000","last_active":"2023-09-23T20:59:52.355523+0000","last_peered":"2023-09-23T20:59:52.355523+0000","last_clean":"2023-09-23T20:59:52.355523+0000","last_became_active":"2023-09-23T20:57:44.979515+0000","last_became_peered":"2023-09-23T20:57:44.979515+0000","last_unstale":"2023-09-23T20:59:52.355523+0000","last_undegraded":"2023-09-23T20:59:52.355523+0000","last_fullsized":"2023-09-23T20:59:52.355523+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:38:22.162140+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":227,"reported_epoch":243,"state":"active+clean","last_fresh":"2023-09-23T20:59:52.341217+0000","last_change":"2023-09-23T20:51:10.833613+0000","last_active":"2023-09-23T20:59:52.341217+0000","last_peered":"2023-09-23T20:59:52.341217+0000","last_clean":"2023-09-23T20:59:52.341217+0000","last_became_active":"2023-09-23T20:51:10.833208+0000","last_became_peered":"2023-09-23T20:51:10.833208+0000","last_unstale":"2023-09-23T20:59:52.341217+0000","last_undegraded":"2023-09-23T20:59:52.341217+0000","last_fullsized":"2023-09-23T20:59:52.341217+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:51:28.246014+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":210,"reported_epoch":244,"state":"active+clean","last_fresh":"2023-09-23T21:00:03.129542+0000","last_change":"2023-09-23T20:55:35.958384+0000","last_active":"2023-09-23T21:00:03.129542+0000","last_peered":"2023-09-23T21:00:03.129542+0000","last_clean":"2023-09-23T21:00:03.129542+0000","last_became_active":"2023-09-23T20:55:35.954563+0000","last_became_peered":"2023-09-23T20:55:35.954563+0000","last_unstale":"2023-09-23T21:00:03.129542+0000","last_undegraded":"2023-09-23T21:00:03.129542+0000","last_fullsized":"2023-09-23T21:00:03.129542+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:37:48.332259+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":219,"reported_epoch":243,"state":"active+clean","last_fresh":"2023-09-23T20:59:52.354533+0000","last_change":"2023-09-23T20:54:31.885514+0000","last_active":"2023-09-23T20:59:52.354533+0000","last_peered":"2023-09-23T20:59:52.354533+0000","last_clean":"2023-09-23T20:59:52.354533+0000","last_became_active":"2023-09-23T20:54:31.884986+0000","last_became_peered":"2023-09-23T20:54:31.884986+0000","last_unstale":"2023-09-23T20:59:52.354533+0000","last_undegraded":"2023-09-23T20:59:52.354533+0000","last_fullsized":"2023-09-23T20:59:52.354533+0000","mapping_epoch":136,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:26:08.100220+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":224,"reported_epoch":243,"state":"active+clean","last_fresh":"2023-09-23T20:59:52.341289+0000","last_change":"2023-09-23T20:55:35.951889+0000","last_active":"2023-09-23T20:59:52.341289+0000","last_peered":"2023-09-23T20:59:52.341289+0000","last_clean":"2023-09-23T20:59:52.341289+0000","last_became_active":"2023-09-23T20:55:35.951563+0000","last_became_peered":"2023-09-23T20:55:35.951563+0000","last_unstale":"2023-09-23T20:59:52.341289+0000","last_undegraded":"2023-09-23T20:59:52.341289+0000","last_fullsized":"2023-09-23T20:59:52.341289+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:41:53.098781+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"79'161","reported_seq":531,"reported_epoch":243,"state":"active+clean","last_fresh":"2023-09-23T20:59:52.350254+0000","last_change":"2023-09-23T20:57:25.096559+0000","last_active":"2023-09-23T20:59:52.350254+0000","last_peered":"2023-09-23T20:59:52.350254+0000","last_clean":"2023-09-23T20:59:52.350254+0000","last_became_active":"2023-09-23T20:57:15.975674+0000","last_became_peered":"2023-09-23T20:57:15.975674+0000","last_unstale":"2023-09-23T20:59:52.350254+0000","last_undegraded":"2023-09-23T20:59:52.350254+0000","last_fullsized":"2023-09-23T20:59:52.350254+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:25.096401+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:25.096401+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:05:05.448404+0000","scrub_duration":0.002859393,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":117,"reported_epoch":243,"state":"active+clean","last_fresh":"2023-09-23T20:59:52.331056+0000","last_change":"2023-09-23T20:54:31.890678+0000","last_active":"2023-09-23T20:59:52.331056+0000","last_peered":"2023-09-23T20:59:52.331056+0000","last_clean":"2023-09-23T20:59:52.331056+0000","last_became_active":"2023-09-23T20:54:31.890469+0000","last_became_peered":"2023-09-23T20:54:31.890469+0000","last_unstale":"2023-09-23T20:59:52.331056+0000","last_undegraded":"2023-09-23T20:59:52.331056+0000","last_fullsized":"2023-09-23T20:59:52.331056+0000","mapping_epoch":136,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:55:08.790146+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":185,"reported_epoch":243,"state":"active+clean","last_fresh":"2023-09-23T20:59:52.341313+0000","last_change":"2023-09-23T20:55:35.952661+0000","last_active":"2023-09-23T20:59:52.341313+0000","last_peered":"2023-09-23T20:59:52.341313+0000","last_clean":"2023-09-23T20:59:52.341313+0000","last_became_active":"2023-09-23T20:55:35.952349+0000","last_became_peered":"2023-09-23T20:55:35.952349+0000","last_unstale":"2023-09-23T20:59:52.341313+0000","last_undegraded":"2023-09-23T20:59:52.341313+0000","last_fullsized":"2023-09-23T20:59:52.341313+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:00:03.110126+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":62,"reported_epoch":243,"state":"active+clean","last_fresh":"2023-09-23T20:59:52.341404+0000","last_change":"2023-09-23T20:56:45.988607+0000","last_active":"2023-09-23T20:59:52.341404+0000","last_peered":"2023-09-23T20:59:52.341404+0000","last_clean":"2023-09-23T20:59:52.341404+0000","last_became_active":"2023-09-23T20:56:45.988206+0000","last_became_peered":"2023-09-23T20:56:45.988206+0000","last_unstale":"2023-09-23T20:59:52.341404+0000","last_undegraded":"2023-09-23T20:59:52.341404+0000","last_fullsized":"2023-09-23T20:59:52.341404+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":205,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:01:19.783688+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,4],"acting":[3,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":61,"reported_epoch":243,"state":"active+clean","last_fresh":"2023-09-23T20:59:52.333653+0000","last_change":"2023-09-23T20:56:18.139142+0000","last_active":"2023-09-23T20:59:52.333653+0000","last_peered":"2023-09-23T20:59:52.333653+0000","last_clean":"2023-09-23T20:59:52.333653+0000","last_became_active":"2023-09-23T20:56:18.138844+0000","last_became_peered":"2023-09-23T20:56:18.138844+0000","last_unstale":"2023-09-23T20:59:52.333653+0000","last_undegraded":"2023-09-23T20:59:52.333653+0000","last_fullsized":"2023-09-23T20:59:52.333653+0000","mapping_epoch":192,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":193,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:09:02.217683+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,2],"acting":[5,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.14","version":"79'22","reported_seq":148,"reported_epoch":243,"state":"active+clean","last_fresh":"2023-09-23T20:59:52.343252+0000","last_change":"2023-09-23T20:55:35.949418+0000","last_active":"2023-09-23T20:59:52.343252+0000","last_peered":"2023-09-23T20:59:52.343252+0000","last_clean":"2023-09-23T20:59:52.343252+0000","last_became_active":"2023-09-23T20:55:35.948566+0000","last_became_peered":"2023-09-23T20:55:35.948566+0000","last_unstale":"2023-09-23T20:59:52.343252+0000","last_undegraded":"2023-09-23T20:59:52.343252+0000","last_fullsized":"2023-09-23T20:59:52.343252+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:59:37.433128+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4734,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":138,"reported_epoch":243,"state":"active+clean","last_fresh":"2023-09-23T20:59:52.343267+0000","last_change":"2023-09-23T20:55:35.949178+0000","last_active":"2023-09-23T20:59:52.343267+0000","last_peered":"2023-09-23T20:59:52.343267+0000","last_clean":"2023-09-23T20:59:52.343267+0000","last_became_active":"2023-09-23T20:55:35.948795+0000","last_became_peered":"2023-09-23T20:55:35.948795+0000","last_unstale":"2023-09-23T20:59:52.343267+0000","last_undegraded":"2023-09-23T20:59:52.343267+0000","last_fullsized":"2023-09-23T20:59:52.343267+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:13:08.996932+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.11","version":"79'161","reported_seq":473,"reported_epoch":243,"state":"active+clean","last_fresh":"2023-09-23T20:59:52.352888+0000","last_change":"2023-09-23T20:57:28.756796+0000","last_active":"2023-09-23T20:59:52.352888+0000","last_peered":"2023-09-23T20:59:52.352888+0000","last_clean":"2023-09-23T20:59:52.352888+0000","last_became_active":"2023-09-23T20:57:15.980606+0000","last_became_peered":"2023-09-23T20:57:15.980606+0000","last_unstale":"2023-09-23T20:59:52.352888+0000","last_undegraded":"2023-09-23T20:59:52.352888+0000","last_fullsized":"2023-09-23T20:59:52.352888+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:28.756634+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:28.756634+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:09:19.150791+0000","scrub_duration":0.0064925119999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":164,"reported_epoch":243,"state":"active+clean","last_fresh":"2023-09-23T20:59:52.343131+0000","last_change":"2023-09-23T20:56:45.992577+0000","last_active":"2023-09-23T20:59:52.343131+0000","last_peered":"2023-09-23T20:59:52.343131+0000","last_clean":"2023-09-23T20:59:52.343131+0000","last_became_active":"2023-09-23T20:56:45.992297+0000","last_became_peered":"2023-09-23T20:56:45.992297+0000","last_unstale":"2023-09-23T20:59:52.343131+0000","last_undegraded":"2023-09-23T20:59:52.343131+0000","last_fullsized":"2023-09-23T20:59:52.343131+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":205,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:00:45.547202+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,2],"acting":[4,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":209,"reported_epoch":243,"state":"active+clean","last_fresh":"2023-09-23T20:59:52.354662+0000","last_change":"2023-09-23T20:51:10.831262+0000","last_active":"2023-09-23T20:59:52.354662+0000","last_peered":"2023-09-23T20:59:52.354662+0000","last_clean":"2023-09-23T20:59:52.354662+0000","last_became_active":"2023-09-23T20:51:10.830931+0000","last_became_peered":"2023-09-23T20:51:10.830931+0000","last_unstale":"2023-09-23T20:59:52.354662+0000","last_undegraded":"2023-09-23T20:59:52.354662+0000","last_fullsized":"2023-09-23T20:59:52.354662+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:51:54.982505+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.10","version":"79'161","reported_seq":481,"reported_epoch":243,"state":"active+clean","last_fresh":"2023-09-23T20:59:52.330637+0000","last_change":"2023-09-23T20:57:20.708003+0000","last_active":"2023-09-23T20:59:52.330637+0000","last_peered":"2023-09-23T20:59:52.330637+0000","last_clean":"2023-09-23T20:59:52.330637+0000","last_became_active":"2023-09-23T20:57:15.975302+0000","last_became_peered":"2023-09-23T20:57:15.975302+0000","last_unstale":"2023-09-23T20:59:52.330637+0000","last_undegraded":"2023-09-23T20:59:52.330637+0000","last_fullsized":"2023-09-23T20:59:52.330637+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:20.707939+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:20.707939+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:16:51.721994+0000","scrub_duration":0.0017733409999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":219,"reported_epoch":243,"state":"active+clean","last_fresh":"2023-09-23T20:59:52.353406+0000","last_change":"2023-09-23T20:57:44.979736+0000","last_active":"2023-09-23T20:59:52.353406+0000","last_peered":"2023-09-23T20:59:52.353406+0000","last_clean":"2023-09-23T20:59:52.353406+0000","last_became_active":"2023-09-23T20:57:44.979356+0000","last_became_peered":"2023-09-23T20:57:44.979356+0000","last_unstale":"2023-09-23T20:59:52.353406+0000","last_undegraded":"2023-09-23T20:59:52.353406+0000","last_fullsized":"2023-09-23T20:59:52.353406+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:14:43.848832+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":117,"reported_epoch":243,"state":"active+clean","last_fresh":"2023-09-23T20:59:52.331104+0000","last_change":"2023-09-23T20:54:31.887814+0000","last_active":"2023-09-23T20:59:52.331104+0000","last_peered":"2023-09-23T20:59:52.331104+0000","last_clean":"2023-09-23T20:59:52.331104+0000","last_became_active":"2023-09-23T20:54:31.887195+0000","last_became_peered":"2023-09-23T20:54:31.887195+0000","last_unstale":"2023-09-23T20:59:52.331104+0000","last_undegraded":"2023-09-23T20:59:52.331104+0000","last_fullsized":"2023-09-23T20:59:52.331104+0000","mapping_epoch":136,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:10:52.089553+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.13","version":"79'161","reported_seq":545,"reported_epoch":244,"state":"active+clean","last_fresh":"2023-09-23T21:00:03.120361+0000","last_change":"2023-09-23T20:57:48.262970+0000","last_active":"2023-09-23T21:00:03.120361+0000","last_peered":"2023-09-23T21:00:03.120361+0000","last_clean":"2023-09-23T21:00:03.120361+0000","last_became_active":"2023-09-23T20:57:48.262663+0000","last_became_peered":"2023-09-23T20:57:48.262663+0000","last_unstale":"2023-09-23T21:00:03.120361+0000","last_undegraded":"2023-09-23T21:00:03.120361+0000","last_fullsized":"2023-09-23T21:00:03.120361+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:36.420362+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:36.420362+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:28:28.613492+0000","scrub_duration":0.028838444000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":188,"reported_epoch":243,"state":"active+clean","last_fresh":"2023-09-23T20:59:52.344490+0000","last_change":"2023-09-23T20:56:45.991822+0000","last_active":"2023-09-23T20:59:52.344490+0000","last_peered":"2023-09-23T20:59:52.344490+0000","last_clean":"2023-09-23T20:59:52.344490+0000","last_became_active":"2023-09-23T20:56:45.991460+0000","last_became_peered":"2023-09-23T20:56:45.991460+0000","last_unstale":"2023-09-23T20:59:52.344490+0000","last_undegraded":"2023-09-23T20:59:52.344490+0000","last_fullsized":"2023-09-23T20:59:52.344490+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":205,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:20:37.503317+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":201,"reported_epoch":243,"state":"active+clean","last_fresh":"2023-09-23T20:59:52.355654+0000","last_change":"2023-09-23T20:51:10.840849+0000","last_active":"2023-09-23T20:59:52.355654+0000","last_peered":"2023-09-23T20:59:52.355654+0000","last_clean":"2023-09-23T20:59:52.355654+0000","last_became_active":"2023-09-23T20:51:10.840593+0000","last_became_peered":"2023-09-23T20:51:10.840593+0000","last_unstale":"2023-09-23T20:59:52.355654+0000","last_undegraded":"2023-09-23T20:59:52.355654+0000","last_fullsized":"2023-09-23T20:59:52.355654+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:30:52.297625+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.12","version":"79'161","reported_seq":481,"reported_epoch":243,"state":"active+clean","last_fresh":"2023-09-23T20:59:52.333403+0000","last_change":"2023-09-23T20:57:21.750765+0000","last_active":"2023-09-23T20:59:52.333403+0000","last_peered":"2023-09-23T20:59:52.333403+0000","last_clean":"2023-09-23T20:59:52.333403+0000","last_became_active":"2023-09-23T20:57:15.979195+0000","last_became_peered":"2023-09-23T20:57:15.979195+0000","last_unstale":"2023-09-23T20:59:52.333403+0000","last_undegraded":"2023-09-23T20:59:52.333403+0000","last_fullsized":"2023-09-23T20:59:52.333403+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:21.750680+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:21.750680+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:03:41.915962+0000","scrub_duration":0.0092391680000000007,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":237,"reported_epoch":244,"state":"active+clean","last_fresh":"2023-09-23T21:00:03.129042+0000","last_change":"2023-09-23T20:55:36.943969+0000","last_active":"2023-09-23T21:00:03.129042+0000","last_peered":"2023-09-23T21:00:03.129042+0000","last_clean":"2023-09-23T21:00:03.129042+0000","last_became_active":"2023-09-23T20:55:36.943580+0000","last_became_peered":"2023-09-23T20:55:36.943580+0000","last_unstale":"2023-09-23T21:00:03.129042+0000","last_undegraded":"2023-09-23T21:00:03.129042+0000","last_fullsized":"2023-09-23T21:00:03.129042+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:49:06.969396+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":33,"reported_epoch":244,"state":"active+clean","last_fresh":"2023-09-23T21:00:03.149759+0000","last_change":"2023-09-23T20:57:44.980498+0000","last_active":"2023-09-23T21:00:03.149759+0000","last_peered":"2023-09-23T21:00:03.149759+0000","last_clean":"2023-09-23T21:00:03.149759+0000","last_became_active":"2023-09-23T20:57:44.980182+0000","last_became_peered":"2023-09-23T20:57:44.980182+0000","last_unstale":"2023-09-23T21:00:03.149759+0000","last_undegraded":"2023-09-23T21:00:03.149759+0000","last_fullsized":"2023-09-23T21:00:03.149759+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:34:51.620544+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.d","version":"79'161","reported_seq":526,"reported_epoch":243,"state":"active+clean","last_fresh":"2023-09-23T20:59:52.343220+0000","last_change":"2023-09-23T20:57:31.075213+0000","last_active":"2023-09-23T20:59:52.343220+0000","last_peered":"2023-09-23T20:59:52.343220+0000","last_clean":"2023-09-23T20:59:52.343220+0000","last_became_active":"2023-09-23T20:57:15.988188+0000","last_became_peered":"2023-09-23T20:57:15.988188+0000","last_unstale":"2023-09-23T20:59:52.343220+0000","last_undegraded":"2023-09-23T20:59:52.343220+0000","last_fullsized":"2023-09-23T20:59:52.343220+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:31.075098+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:31.075098+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:41:52.142362+0000","scrub_duration":0.010304610000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.b","version":"245'10938","reported_seq":10994,"reported_epoch":245,"state":"active+clean","last_fresh":"2023-09-23T21:00:14.248598+0000","last_change":"2023-09-23T20:56:35.543994+0000","last_active":"2023-09-23T21:00:14.248598+0000","last_peered":"2023-09-23T21:00:14.248598+0000","last_clean":"2023-09-23T21:00:14.248598+0000","last_became_active":"2023-09-23T20:56:35.543758+0000","last_became_peered":"2023-09-23T20:56:35.543758+0000","last_unstale":"2023-09-23T21:00:14.248598+0000","last_undegraded":"2023-09-23T21:00:14.248598+0000","last_fullsized":"2023-09-23T21:00:14.248598+0000","mapping_epoch":199,"log_start":"216'3900","ondisk_log_start":"216'3900","created":199,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_clean_scrub_stamp":"2023-09-23T20:56:34.529103+0000","objects_scrubbed":0,"log_size":7038,"log_dups_size":0,"ondisk_log_size":7038,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:21:29.041484+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":114163712,"num_objects":1742,"num_object_clones":0,"num_object_copies":5226,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1742,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10938,"num_write_kb":405760,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":175,"reported_epoch":243,"state":"active+clean","last_fresh":"2023-09-23T20:59:52.343673+0000","last_change":"2023-09-23T20:56:45.991841+0000","last_active":"2023-09-23T20:59:52.343673+0000","last_peered":"2023-09-23T20:59:52.343673+0000","last_clean":"2023-09-23T20:59:52.343673+0000","last_became_active":"2023-09-23T20:56:45.991491+0000","last_became_peered":"2023-09-23T20:56:45.991491+0000","last_unstale":"2023-09-23T20:59:52.343673+0000","last_undegraded":"2023-09-23T20:59:52.343673+0000","last_fullsized":"2023-09-23T20:59:52.343673+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":205,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:30:37.405319+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":134,"reported_epoch":243,"state":"active+clean","last_fresh":"2023-09-23T20:59:52.330746+0000","last_change":"2023-09-23T20:55:35.957768+0000","last_active":"2023-09-23T20:59:52.330746+0000","last_peered":"2023-09-23T20:59:52.330746+0000","last_clean":"2023-09-23T20:59:52.330746+0000","last_became_active":"2023-09-23T20:55:35.957421+0000","last_became_peered":"2023-09-23T20:55:35.957421+0000","last_unstale":"2023-09-23T20:59:52.330746+0000","last_undegraded":"2023-09-23T20:59:52.330746+0000","last_fullsized":"2023-09-23T20:59:52.330746+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:49:41.585961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.c","version":"79'161","reported_seq":545,"reported_epoch":243,"state":"active+clean","last_fresh":"2023-09-23T20:59:52.355221+0000","last_change":"2023-09-23T20:57:22.041071+0000","last_active":"2023-09-23T20:59:52.355221+0000","last_peered":"2023-09-23T20:59:52.355221+0000","last_clean":"2023-09-23T20:59:52.355221+0000","last_became_active":"2023-09-23T20:57:15.973204+0000","last_became_peered":"2023-09-23T20:57:15.973204+0000","last_unstale":"2023-09-23T20:59:52.355221+0000","last_undegraded":"2023-09-23T20:59:52.355221+0000","last_fullsized":"2023-09-23T20:59:52.355221+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:22.040907+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:22.040907+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:47:02.019978+0000","scrub_duration":0.013021176000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.a","version":"244'10831","reported_seq":10885,"reported_epoch":244,"state":"active+clean","last_fresh":"2023-09-23T21:00:09.679017+0000","last_change":"2023-09-23T20:56:35.547120+0000","last_active":"2023-09-23T21:00:09.679017+0000","last_peered":"2023-09-23T21:00:09.679017+0000","last_clean":"2023-09-23T21:00:09.679017+0000","last_became_active":"2023-09-23T20:56:35.546838+0000","last_became_peered":"2023-09-23T20:56:35.546838+0000","last_unstale":"2023-09-23T21:00:09.679017+0000","last_undegraded":"2023-09-23T21:00:09.679017+0000","last_fullsized":"2023-09-23T21:00:09.679017+0000","mapping_epoch":199,"log_start":"219'4700","ondisk_log_start":"219'4700","created":199,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_clean_scrub_stamp":"2023-09-23T20:56:34.529103+0000","objects_scrubbed":0,"log_size":6131,"log_dups_size":0,"ondisk_log_size":6131,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:48:24.132768+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":124846080,"num_objects":1905,"num_object_clones":0,"num_object_copies":5715,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1905,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10831,"num_write_kb":407552,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,5],"acting":[3,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":162,"reported_epoch":243,"state":"active+clean","last_fresh":"2023-09-23T20:59:52.343840+0000","last_change":"2023-09-23T20:56:45.991821+0000","last_active":"2023-09-23T20:59:52.343840+0000","last_peered":"2023-09-23T20:59:52.343840+0000","last_clean":"2023-09-23T20:59:52.343840+0000","last_became_active":"2023-09-23T20:56:45.991447+0000","last_became_peered":"2023-09-23T20:56:45.991447+0000","last_unstale":"2023-09-23T20:59:52.343840+0000","last_undegraded":"2023-09-23T20:59:52.343840+0000","last_fullsized":"2023-09-23T20:59:52.343840+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":205,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:14:25.525222+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":165,"reported_epoch":243,"state":"active+clean","last_fresh":"2023-09-23T20:59:52.344717+0000","last_change":"2023-09-23T20:57:44.982059+0000","last_active":"2023-09-23T20:59:52.344717+0000","last_peered":"2023-09-23T20:59:52.344717+0000","last_clean":"2023-09-23T20:59:52.344717+0000","last_became_active":"2023-09-23T20:57:44.981742+0000","last_became_peered":"2023-09-23T20:57:44.981742+0000","last_unstale":"2023-09-23T20:59:52.344717+0000","last_undegraded":"2023-09-23T20:59:52.344717+0000","last_fullsized":"2023-09-23T20:59:52.344717+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:41:24.042916+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.f","version":"79'161","reported_seq":549,"reported_epoch":244,"state":"active+clean","last_fresh":"2023-09-23T21:00:03.129310+0000","last_change":"2023-09-23T20:57:27.880841+0000","last_active":"2023-09-23T21:00:03.129310+0000","last_peered":"2023-09-23T21:00:03.129310+0000","last_clean":"2023-09-23T21:00:03.129310+0000","last_became_active":"2023-09-23T20:57:15.970925+0000","last_became_peered":"2023-09-23T20:57:15.970925+0000","last_unstale":"2023-09-23T21:00:03.129310+0000","last_undegraded":"2023-09-23T21:00:03.129310+0000","last_fullsized":"2023-09-23T21:00:03.129310+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:27.880776+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:27.880776+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:15:13.438014+0000","scrub_duration":0.0014004899999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.9","version":"244'10899","reported_seq":10954,"reported_epoch":244,"state":"active+clean","last_fresh":"2023-09-23T21:00:11.768156+0000","last_change":"2023-09-23T20:56:35.549387+0000","last_active":"2023-09-23T21:00:11.768156+0000","last_peered":"2023-09-23T21:00:11.768156+0000","last_clean":"2023-09-23T21:00:11.768156+0000","last_became_active":"2023-09-23T20:56:35.549087+0000","last_became_peered":"2023-09-23T20:56:35.549087+0000","last_unstale":"2023-09-23T21:00:11.768156+0000","last_undegraded":"2023-09-23T21:00:11.768156+0000","last_fullsized":"2023-09-23T21:00:11.768156+0000","mapping_epoch":199,"log_start":"211'3500","ondisk_log_start":"211'3500","created":199,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_clean_scrub_stamp":"2023-09-23T20:56:34.529103+0000","objects_scrubbed":0,"log_size":7399,"log_dups_size":0,"ondisk_log_size":7399,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:04:51.547362+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":118423552,"num_objects":1807,"num_object_clones":0,"num_object_copies":5421,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1807,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10899,"num_write_kb":406592,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":148,"reported_epoch":243,"state":"active+clean","last_fresh":"2023-09-23T20:59:52.330844+0000","last_change":"2023-09-23T20:56:45.989153+0000","last_active":"2023-09-23T20:59:52.330844+0000","last_peered":"2023-09-23T20:59:52.330844+0000","last_clean":"2023-09-23T20:59:52.330844+0000","last_became_active":"2023-09-23T20:56:45.988800+0000","last_became_peered":"2023-09-23T20:56:45.988800+0000","last_unstale":"2023-09-23T20:59:52.330844+0000","last_undegraded":"2023-09-23T20:59:52.330844+0000","last_fullsized":"2023-09-23T20:59:52.330844+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":205,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:04:41.130194+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":184,"reported_epoch":244,"state":"active+clean","last_fresh":"2023-09-23T21:00:03.129150+0000","last_change":"2023-09-23T20:55:35.965427+0000","last_active":"2023-09-23T21:00:03.129150+0000","last_peered":"2023-09-23T21:00:03.129150+0000","last_clean":"2023-09-23T21:00:03.129150+0000","last_became_active":"2023-09-23T20:55:35.963969+0000","last_became_peered":"2023-09-23T20:55:35.963969+0000","last_unstale":"2023-09-23T21:00:03.129150+0000","last_undegraded":"2023-09-23T21:00:03.129150+0000","last_fullsized":"2023-09-23T21:00:03.129150+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:38:40.612083+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.e","version":"79'161","reported_seq":455,"reported_epoch":243,"state":"active+clean","last_fresh":"2023-09-23T20:59:52.343707+0000","last_change":"2023-09-23T20:57:34.132738+0000","last_active":"2023-09-23T20:59:52.343707+0000","last_peered":"2023-09-23T20:59:52.343707+0000","last_clean":"2023-09-23T20:59:52.343707+0000","last_became_active":"2023-09-23T20:57:15.974318+0000","last_became_peered":"2023-09-23T20:57:15.974318+0000","last_unstale":"2023-09-23T20:59:52.343707+0000","last_undegraded":"2023-09-23T20:59:52.343707+0000","last_fullsized":"2023-09-23T20:59:52.343707+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:34.132660+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:34.132660+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:07:07.739438+0000","scrub_duration":0.005229694,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.8","version":"244'11027","reported_seq":11082,"reported_epoch":244,"state":"active+clean","last_fresh":"2023-09-23T21:00:11.460702+0000","last_change":"2023-09-23T20:56:35.548119+0000","last_active":"2023-09-23T21:00:11.460702+0000","last_peered":"2023-09-23T21:00:11.460702+0000","last_clean":"2023-09-23T21:00:11.460702+0000","last_became_active":"2023-09-23T20:56:35.547914+0000","last_became_peered":"2023-09-23T20:56:35.547914+0000","last_unstale":"2023-09-23T21:00:11.460702+0000","last_undegraded":"2023-09-23T21:00:11.460702+0000","last_fullsized":"2023-09-23T21:00:11.460702+0000","mapping_epoch":199,"log_start":"206'2200","ondisk_log_start":"206'2200","created":199,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_clean_scrub_stamp":"2023-09-23T20:56:34.529103+0000","objects_scrubbed":0,"log_size":8827,"log_dups_size":0,"ondisk_log_size":8827,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:42:54.255023+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":119734272,"num_objects":1827,"num_object_clones":0,"num_object_copies":5481,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1827,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11027,"num_write_kb":411328,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":134,"reported_epoch":243,"state":"active+clean","last_fresh":"2023-09-23T20:59:52.330839+0000","last_change":"2023-09-23T20:55:35.962005+0000","last_active":"2023-09-23T20:59:52.330839+0000","last_peered":"2023-09-23T20:59:52.330839+0000","last_clean":"2023-09-23T20:59:52.330839+0000","last_became_active":"2023-09-23T20:55:35.961695+0000","last_became_peered":"2023-09-23T20:55:35.961695+0000","last_unstale":"2023-09-23T20:59:52.330839+0000","last_undegraded":"2023-09-23T20:59:52.330839+0000","last_fullsized":"2023-09-23T20:59:52.330839+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:28:22.876201+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":232,"reported_epoch":243,"state":"active+clean","last_fresh":"2023-09-23T20:59:52.353643+0000","last_change":"2023-09-23T20:54:13.928715+0000","last_active":"2023-09-23T20:59:52.353643+0000","last_peered":"2023-09-23T20:59:52.353643+0000","last_clean":"2023-09-23T20:59:52.353643+0000","last_became_active":"2023-09-23T20:54:13.928200+0000","last_became_peered":"2023-09-23T20:54:13.928200+0000","last_unstale":"2023-09-23T20:59:52.353643+0000","last_undegraded":"2023-09-23T20:59:52.353643+0000","last_fullsized":"2023-09-23T20:59:52.353643+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:41:16.927513+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.f","version":"244'11067","reported_seq":11120,"reported_epoch":244,"state":"active+clean","last_fresh":"2023-09-23T21:00:11.813068+0000","last_change":"2023-09-23T20:56:35.545128+0000","last_active":"2023-09-23T21:00:11.813068+0000","last_peered":"2023-09-23T21:00:11.813068+0000","last_clean":"2023-09-23T21:00:11.813068+0000","last_became_active":"2023-09-23T20:56:35.544838+0000","last_became_peered":"2023-09-23T20:56:35.544838+0000","last_unstale":"2023-09-23T21:00:11.813068+0000","last_undegraded":"2023-09-23T21:00:11.813068+0000","last_fullsized":"2023-09-23T21:00:11.813068+0000","mapping_epoch":199,"log_start":"212'3700","ondisk_log_start":"212'3700","created":199,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_clean_scrub_stamp":"2023-09-23T20:56:34.529103+0000","objects_scrubbed":0,"log_size":7367,"log_dups_size":0,"ondisk_log_size":7367,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:30:43.388717+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":117899264,"num_objects":1799,"num_object_clones":0,"num_object_copies":5397,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1799,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11067,"num_write_kb":411712,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.9","version":"79'161","reported_seq":617,"reported_epoch":243,"state":"active+clean","last_fresh":"2023-09-23T20:59:52.341242+0000","last_change":"2023-09-23T20:56:46.921939+0000","last_active":"2023-09-23T20:59:52.341242+0000","last_peered":"2023-09-23T20:59:52.341242+0000","last_clean":"2023-09-23T20:59:52.341242+0000","last_became_active":"2023-09-23T20:56:45.989908+0000","last_became_peered":"2023-09-23T20:56:45.989908+0000","last_unstale":"2023-09-23T20:59:52.341242+0000","last_undegraded":"2023-09-23T20:59:52.341242+0000","last_fullsized":"2023-09-23T20:59:52.341242+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":205,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:56:46.921739+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:56:46.921739+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:20:42.635110+0000","scrub_duration":0.0071954109999999996,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":234,"reported_epoch":244,"state":"active+clean","last_fresh":"2023-09-23T21:00:03.146745+0000","last_change":"2023-09-23T20:54:13.961788+0000","last_active":"2023-09-23T21:00:03.146745+0000","last_peered":"2023-09-23T21:00:03.146745+0000","last_clean":"2023-09-23T21:00:03.146745+0000","last_became_active":"2023-09-23T20:54:13.940937+0000","last_became_peered":"2023-09-23T20:54:13.940937+0000","last_unstale":"2023-09-23T21:00:03.146745+0000","last_undegraded":"2023-09-23T21:00:03.146745+0000","last_fullsized":"2023-09-23T21:00:03.146745+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:11:56.785299+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":138,"reported_epoch":243,"state":"active+clean","last_fresh":"2023-09-23T20:59:52.343424+0000","last_change":"2023-09-23T20:55:35.949670+0000","last_active":"2023-09-23T20:59:52.343424+0000","last_peered":"2023-09-23T20:59:52.343424+0000","last_clean":"2023-09-23T20:59:52.343424+0000","last_became_active":"2023-09-23T20:55:35.949140+0000","last_became_peered":"2023-09-23T20:55:35.949140+0000","last_unstale":"2023-09-23T20:59:52.343424+0000","last_undegraded":"2023-09-23T20:59:52.343424+0000","last_fullsized":"2023-09-23T20:59:52.343424+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:55:59.340303+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.e","version":"244'10962","reported_seq":11017,"reported_epoch":244,"state":"active+clean","last_fresh":"2023-09-23T21:00:11.503938+0000","last_change":"2023-09-23T20:56:35.547184+0000","last_active":"2023-09-23T21:00:11.503938+0000","last_peered":"2023-09-23T21:00:11.503938+0000","last_clean":"2023-09-23T21:00:11.503938+0000","last_became_active":"2023-09-23T20:56:35.546912+0000","last_became_peered":"2023-09-23T20:56:35.546912+0000","last_unstale":"2023-09-23T21:00:11.503938+0000","last_undegraded":"2023-09-23T21:00:11.503938+0000","last_fullsized":"2023-09-23T21:00:11.503938+0000","mapping_epoch":199,"log_start":"206'2100","ondisk_log_start":"206'2100","created":199,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_clean_scrub_stamp":"2023-09-23T20:56:34.529103+0000","objects_scrubbed":0,"log_size":8862,"log_dups_size":0,"ondisk_log_size":8862,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:57:07.422792+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":119144448,"num_objects":1818,"num_object_clones":0,"num_object_copies":5454,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1818,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10962,"num_write_kb":408960,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,0],"acting":[2,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.8","version":"79'161","reported_seq":487,"reported_epoch":243,"state":"active+clean","last_fresh":"2023-09-23T20:59:52.330746+0000","last_change":"2023-09-23T20:56:57.505737+0000","last_active":"2023-09-23T20:59:52.330746+0000","last_peered":"2023-09-23T20:59:52.330746+0000","last_clean":"2023-09-23T20:59:52.330746+0000","last_became_active":"2023-09-23T20:56:04.982345+0000","last_became_peered":"2023-09-23T20:56:04.982345+0000","last_unstale":"2023-09-23T20:59:52.330746+0000","last_undegraded":"2023-09-23T20:59:52.330746+0000","last_fullsized":"2023-09-23T20:59:52.330746+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:56:57.505663+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:56:57.505663+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:34:30.109724+0000","scrub_duration":0.026284108,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":207,"reported_epoch":244,"state":"active+clean","last_fresh":"2023-09-23T21:00:03.129351+0000","last_change":"2023-09-23T20:55:35.959942+0000","last_active":"2023-09-23T21:00:03.129351+0000","last_peered":"2023-09-23T21:00:03.129351+0000","last_clean":"2023-09-23T21:00:03.129351+0000","last_became_active":"2023-09-23T20:55:35.957071+0000","last_became_peered":"2023-09-23T20:55:35.957071+0000","last_unstale":"2023-09-23T21:00:03.129351+0000","last_undegraded":"2023-09-23T21:00:03.129351+0000","last_fullsized":"2023-09-23T21:00:03.129351+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:44:30.308036+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":121,"reported_epoch":243,"state":"active+clean","last_fresh":"2023-09-23T20:59:52.343614+0000","last_change":"2023-09-23T20:54:13.951459+0000","last_active":"2023-09-23T20:59:52.343614+0000","last_peered":"2023-09-23T20:59:52.343614+0000","last_clean":"2023-09-23T20:59:52.343614+0000","last_became_active":"2023-09-23T20:54:13.948409+0000","last_became_peered":"2023-09-23T20:54:13.948409+0000","last_unstale":"2023-09-23T20:59:52.343614+0000","last_undegraded":"2023-09-23T20:59:52.343614+0000","last_fullsized":"2023-09-23T20:59:52.343614+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:38:10.148061+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.b","version":"79'161","reported_seq":528,"reported_epoch":244,"state":"active+clean","last_fresh":"2023-09-23T21:00:03.120451+0000","last_change":"2023-09-23T20:57:27.320172+0000","last_active":"2023-09-23T21:00:03.120451+0000","last_peered":"2023-09-23T21:00:03.120451+0000","last_clean":"2023-09-23T21:00:03.120451+0000","last_became_active":"2023-09-23T20:57:16.009630+0000","last_became_peered":"2023-09-23T20:57:16.009630+0000","last_unstale":"2023-09-23T21:00:03.120451+0000","last_undegraded":"2023-09-23T21:00:03.120451+0000","last_fullsized":"2023-09-23T21:00:03.120451+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:27.320022+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:27.320022+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:31:35.221558+0000","scrub_duration":0.0043894679999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,5],"acting":[1,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.d","version":"245'11053","reported_seq":11108,"reported_epoch":245,"state":"active+clean","last_fresh":"2023-09-23T21:00:14.001525+0000","last_change":"2023-09-23T20:56:35.544132+0000","last_active":"2023-09-23T21:00:14.001525+0000","last_peered":"2023-09-23T21:00:14.001525+0000","last_clean":"2023-09-23T21:00:14.001525+0000","last_became_active":"2023-09-23T20:56:35.543836+0000","last_became_peered":"2023-09-23T20:56:35.543836+0000","last_unstale":"2023-09-23T21:00:14.001525+0000","last_undegraded":"2023-09-23T21:00:14.001525+0000","last_fullsized":"2023-09-23T21:00:14.001525+0000","mapping_epoch":199,"log_start":"213'3700","ondisk_log_start":"213'3700","created":199,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_clean_scrub_stamp":"2023-09-23T20:56:34.529103+0000","objects_scrubbed":0,"log_size":7353,"log_dups_size":0,"ondisk_log_size":7353,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:27:14.636991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":113770520,"num_objects":1737,"num_object_clones":0,"num_object_copies":5211,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1737,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":11053,"num_write_kb":409217,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":121,"reported_epoch":243,"state":"active+clean","last_fresh":"2023-09-23T20:59:52.344554+0000","last_change":"2023-09-23T20:54:13.950254+0000","last_active":"2023-09-23T20:59:52.344554+0000","last_peered":"2023-09-23T20:59:52.344554+0000","last_clean":"2023-09-23T20:59:52.344554+0000","last_became_active":"2023-09-23T20:54:13.949934+0000","last_became_peered":"2023-09-23T20:54:13.949934+0000","last_unstale":"2023-09-23T20:59:52.344554+0000","last_undegraded":"2023-09-23T20:59:52.344554+0000","last_fullsized":"2023-09-23T20:59:52.344554+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:30:32.847167+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":138,"reported_epoch":243,"state":"active+clean","last_fresh":"2023-09-23T20:59:52.343913+0000","last_change":"2023-09-23T20:55:35.956153+0000","last_active":"2023-09-23T20:59:52.343913+0000","last_peered":"2023-09-23T20:59:52.343913+0000","last_clean":"2023-09-23T20:59:52.343913+0000","last_became_active":"2023-09-23T20:55:35.955630+0000","last_became_peered":"2023-09-23T20:55:35.955630+0000","last_unstale":"2023-09-23T20:59:52.343913+0000","last_undegraded":"2023-09-23T20:59:52.343913+0000","last_fullsized":"2023-09-23T20:59:52.343913+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:45:20.282929+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.c","version":"245'11026","reported_seq":11090,"reported_epoch":245,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T21:00:14.092304+0000","last_change":"2023-09-23T20:57:46.463005+0000","last_active":"2023-09-23T21:00:14.092304+0000","last_peered":"2023-09-23T21:00:14.092304+0000","last_clean":"2023-09-23T20:57:44.015119+0000","last_became_active":"2023-09-23T20:57:46.448702+0000","last_became_peered":"2023-09-23T20:57:46.448702+0000","last_unstale":"2023-09-23T21:00:14.092304+0000","last_undegraded":"2023-09-23T20:57:46.010874+0000","last_fullsized":"2023-09-23T20:57:45.978434+0000","mapping_epoch":223,"log_start":"214'3700","ondisk_log_start":"214'3700","created":199,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_clean_scrub_stamp":"2023-09-23T20:56:34.529103+0000","objects_scrubbed":0,"log_size":7326,"log_dups_size":0,"ondisk_log_size":7326,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:08:34.180651+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":116785152,"num_objects":1782,"num_object_clones":0,"num_object_copies":5346,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5473,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1782,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11026,"num_write_kb":409856,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,7],"avail_no_missing":["6","2","7"],"object_location_counts":[{"shards":"2,6,7","objects":851},{"shards":"6,7","objects":-3691}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.a","version":"79'161","reported_seq":487,"reported_epoch":243,"state":"active+clean","last_fresh":"2023-09-23T20:59:52.333785+0000","last_change":"2023-09-23T20:56:44.487778+0000","last_active":"2023-09-23T20:59:52.333785+0000","last_peered":"2023-09-23T20:59:52.333785+0000","last_clean":"2023-09-23T20:59:52.333785+0000","last_became_active":"2023-09-23T20:56:05.006740+0000","last_became_peered":"2023-09-23T20:56:05.006740+0000","last_unstale":"2023-09-23T20:59:52.333785+0000","last_undegraded":"2023-09-23T20:59:52.333785+0000","last_fullsized":"2023-09-23T20:59:52.333785+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:56:44.487678+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:56:44.487678+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:04:50.086749+0000","scrub_duration":0.0017408899999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":33,"reported_epoch":243,"state":"active+clean","last_fresh":"2023-09-23T20:59:54.645966+0000","last_change":"2023-09-23T20:57:44.982513+0000","last_active":"2023-09-23T20:59:54.645966+0000","last_peered":"2023-09-23T20:59:54.645966+0000","last_clean":"2023-09-23T20:59:54.645966+0000","last_became_active":"2023-09-23T20:57:44.982112+0000","last_became_peered":"2023-09-23T20:57:44.982112+0000","last_unstale":"2023-09-23T20:59:54.645966+0000","last_undegraded":"2023-09-23T20:59:54.645966+0000","last_fullsized":"2023-09-23T20:59:54.645966+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:09:05.630041+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":215,"reported_epoch":244,"state":"active+clean","last_fresh":"2023-09-23T21:00:03.122549+0000","last_change":"2023-09-23T20:54:13.940748+0000","last_active":"2023-09-23T21:00:03.122549+0000","last_peered":"2023-09-23T21:00:03.122549+0000","last_clean":"2023-09-23T21:00:03.122549+0000","last_became_active":"2023-09-23T20:54:13.940419+0000","last_became_peered":"2023-09-23T20:54:13.940419+0000","last_unstale":"2023-09-23T21:00:03.122549+0000","last_undegraded":"2023-09-23T21:00:03.122549+0000","last_fullsized":"2023-09-23T21:00:03.122549+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:15:01.923816+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.1","version":"244'10936","reported_seq":11002,"reported_epoch":244,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T21:00:12.169782+0000","last_change":"2023-09-23T20:57:46.483248+0000","last_active":"2023-09-23T21:00:12.169782+0000","last_peered":"2023-09-23T21:00:12.169782+0000","last_clean":"2023-09-23T20:57:44.047957+0000","last_became_active":"2023-09-23T20:57:46.475568+0000","last_became_peered":"2023-09-23T20:57:46.475568+0000","last_unstale":"2023-09-23T21:00:12.169782+0000","last_undegraded":"2023-09-23T20:57:46.083591+0000","last_fullsized":"2023-09-23T20:57:45.978436+0000","mapping_epoch":223,"log_start":"217'4200","ondisk_log_start":"217'4200","created":199,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_clean_scrub_stamp":"2023-09-23T20:56:34.529103+0000","objects_scrubbed":0,"log_size":6736,"log_dups_size":0,"ondisk_log_size":6736,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:22:45.067269+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":121241600,"num_objects":1850,"num_object_clones":0,"num_object_copies":5550,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5470,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1850,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10936,"num_write_kb":409152,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,6],"avail_no_missing":["4","1","6"],"object_location_counts":[{"shards":"1,4,6","objects":927},{"shards":"4,6","objects":-3620}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.7","version":"79'161","reported_seq":550,"reported_epoch":244,"state":"active+clean","last_fresh":"2023-09-23T21:00:03.129621+0000","last_change":"2023-09-23T20:57:48.243572+0000","last_active":"2023-09-23T21:00:03.129621+0000","last_peered":"2023-09-23T21:00:03.129621+0000","last_clean":"2023-09-23T21:00:03.129621+0000","last_became_active":"2023-09-23T20:57:48.243229+0000","last_became_peered":"2023-09-23T20:57:48.243229+0000","last_unstale":"2023-09-23T21:00:03.129621+0000","last_undegraded":"2023-09-23T21:00:03.129621+0000","last_fullsized":"2023-09-23T21:00:03.129621+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":3,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:25.916867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:25.916867+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:23:19.385831+0000","scrub_duration":0.001665323,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":220,"reported_epoch":244,"state":"active+clean","last_fresh":"2023-09-23T21:00:03.120771+0000","last_change":"2023-09-23T20:50:05.867130+0000","last_active":"2023-09-23T21:00:03.120771+0000","last_peered":"2023-09-23T21:00:03.120771+0000","last_clean":"2023-09-23T21:00:03.120771+0000","last_became_active":"2023-09-23T20:50:05.866758+0000","last_became_peered":"2023-09-23T20:50:05.866758+0000","last_unstale":"2023-09-23T21:00:03.120771+0000","last_undegraded":"2023-09-23T21:00:03.120771+0000","last_fullsized":"2023-09-23T21:00:03.120771+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:55:07.548209+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":117,"reported_epoch":243,"state":"active+clean","last_fresh":"2023-09-23T20:59:52.330963+0000","last_change":"2023-09-23T20:54:31.887604+0000","last_active":"2023-09-23T20:59:52.330963+0000","last_peered":"2023-09-23T20:59:52.330963+0000","last_clean":"2023-09-23T20:59:52.330963+0000","last_became_active":"2023-09-23T20:54:31.886752+0000","last_became_peered":"2023-09-23T20:54:31.886752+0000","last_unstale":"2023-09-23T20:59:52.330963+0000","last_undegraded":"2023-09-23T20:59:52.330963+0000","last_fullsized":"2023-09-23T20:59:52.330963+0000","mapping_epoch":136,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:13:58.872176+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":218,"reported_epoch":244,"state":"active+clean","last_fresh":"2023-09-23T21:00:03.120444+0000","last_change":"2023-09-23T20:54:31.899082+0000","last_active":"2023-09-23T21:00:03.120444+0000","last_peered":"2023-09-23T21:00:03.120444+0000","last_clean":"2023-09-23T21:00:03.120444+0000","last_became_active":"2023-09-23T20:54:31.898521+0000","last_became_peered":"2023-09-23T20:54:31.898521+0000","last_unstale":"2023-09-23T21:00:03.120444+0000","last_undegraded":"2023-09-23T21:00:03.120444+0000","last_fullsized":"2023-09-23T21:00:03.120444+0000","mapping_epoch":136,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:25:36.059711+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.7","version":"245'11099","reported_seq":11153,"reported_epoch":245,"state":"active+clean","last_fresh":"2023-09-23T21:00:14.092499+0000","last_change":"2023-09-23T20:56:35.541695+0000","last_active":"2023-09-23T21:00:14.092499+0000","last_peered":"2023-09-23T21:00:14.092499+0000","last_clean":"2023-09-23T21:00:14.092499+0000","last_became_active":"2023-09-23T20:56:35.541392+0000","last_became_peered":"2023-09-23T20:56:35.541392+0000","last_unstale":"2023-09-23T21:00:14.092499+0000","last_undegraded":"2023-09-23T21:00:14.092499+0000","last_fullsized":"2023-09-23T21:00:14.092499+0000","mapping_epoch":199,"log_start":"213'3700","ondisk_log_start":"213'3700","created":199,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_clean_scrub_stamp":"2023-09-23T20:56:34.529103+0000","objects_scrubbed":0,"log_size":7399,"log_dups_size":0,"ondisk_log_size":7399,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:57:22.478253+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":112132096,"num_objects":1711,"num_object_clones":0,"num_object_copies":5133,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1711,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11099,"num_write_kb":409920,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.1","version":"79'161","reported_seq":507,"reported_epoch":243,"state":"active+clean","last_fresh":"2023-09-23T20:59:52.330699+0000","last_change":"2023-09-23T20:57:19.669346+0000","last_active":"2023-09-23T20:59:52.330699+0000","last_peered":"2023-09-23T20:59:52.330699+0000","last_clean":"2023-09-23T20:59:52.330699+0000","last_became_active":"2023-09-23T20:57:08.246046+0000","last_became_peered":"2023-09-23T20:57:08.246046+0000","last_unstale":"2023-09-23T20:59:52.330699+0000","last_undegraded":"2023-09-23T20:59:52.330699+0000","last_fullsized":"2023-09-23T20:59:52.330699+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:19.669259+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:19.669259+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:14:03.435531+0000","scrub_duration":0.0045759559999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":134,"reported_epoch":243,"state":"active+clean","last_fresh":"2023-09-23T20:59:52.333534+0000","last_change":"2023-09-23T20:55:35.959514+0000","last_active":"2023-09-23T20:59:52.333534+0000","last_peered":"2023-09-23T20:59:52.333534+0000","last_clean":"2023-09-23T20:59:52.333534+0000","last_became_active":"2023-09-23T20:55:35.959244+0000","last_became_peered":"2023-09-23T20:55:35.959244+0000","last_unstale":"2023-09-23T20:59:52.333534+0000","last_undegraded":"2023-09-23T20:59:52.333534+0000","last_fullsized":"2023-09-23T20:59:52.333534+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:47:32.665316+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":138,"reported_epoch":243,"state":"active+clean","last_fresh":"2023-09-23T20:59:52.343392+0000","last_change":"2023-09-23T20:55:35.949000+0000","last_active":"2023-09-23T20:59:52.343392+0000","last_peered":"2023-09-23T20:59:52.343392+0000","last_clean":"2023-09-23T20:59:52.343392+0000","last_became_active":"2023-09-23T20:55:35.948410+0000","last_became_peered":"2023-09-23T20:55:35.948410+0000","last_unstale":"2023-09-23T20:59:52.343392+0000","last_undegraded":"2023-09-23T20:59:52.343392+0000","last_fullsized":"2023-09-23T20:59:52.343392+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:26:50.477289+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":212,"reported_epoch":244,"state":"active+clean","last_fresh":"2023-09-23T21:00:03.120532+0000","last_change":"2023-09-23T20:51:10.829889+0000","last_active":"2023-09-23T21:00:03.120532+0000","last_peered":"2023-09-23T21:00:03.120532+0000","last_clean":"2023-09-23T21:00:03.120532+0000","last_became_active":"2023-09-23T20:51:10.829533+0000","last_became_peered":"2023-09-23T20:51:10.829533+0000","last_unstale":"2023-09-23T21:00:03.120532+0000","last_undegraded":"2023-09-23T21:00:03.120532+0000","last_fullsized":"2023-09-23T21:00:03.120532+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:43:04.603460+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.4","version":"244'10903","reported_seq":10958,"reported_epoch":244,"state":"active+clean","last_fresh":"2023-09-23T21:00:11.741284+0000","last_change":"2023-09-23T20:56:35.549909+0000","last_active":"2023-09-23T21:00:11.741284+0000","last_peered":"2023-09-23T21:00:11.741284+0000","last_clean":"2023-09-23T21:00:11.741284+0000","last_became_active":"2023-09-23T20:56:35.549618+0000","last_became_peered":"2023-09-23T20:56:35.549618+0000","last_unstale":"2023-09-23T21:00:11.741284+0000","last_undegraded":"2023-09-23T21:00:11.741284+0000","last_fullsized":"2023-09-23T21:00:11.741284+0000","mapping_epoch":199,"log_start":"211'3500","ondisk_log_start":"211'3500","created":199,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_clean_scrub_stamp":"2023-09-23T20:56:34.529103+0000","objects_scrubbed":0,"log_size":7403,"log_dups_size":0,"ondisk_log_size":7403,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:24:25.918475+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":119209984,"num_objects":1819,"num_object_clones":0,"num_object_copies":5457,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1819,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10903,"num_write_kb":407104,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.2","version":"79'161","reported_seq":474,"reported_epoch":243,"state":"active+clean","last_fresh":"2023-09-23T20:59:52.353334+0000","last_change":"2023-09-23T20:57:24.802586+0000","last_active":"2023-09-23T20:59:52.353334+0000","last_peered":"2023-09-23T20:59:52.353334+0000","last_clean":"2023-09-23T20:59:52.353334+0000","last_became_active":"2023-09-23T20:57:08.236114+0000","last_became_peered":"2023-09-23T20:57:08.236114+0000","last_unstale":"2023-09-23T20:59:52.353334+0000","last_undegraded":"2023-09-23T20:59:52.353334+0000","last_fullsized":"2023-09-23T20:59:52.353334+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:24.802451+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:24.802451+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:06:22.309980+0000","scrub_duration":0.0090861250000000005,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,4],"acting":[0,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":230,"reported_epoch":243,"state":"active+clean","last_fresh":"2023-09-23T20:59:52.349952+0000","last_change":"2023-09-23T20:51:10.838054+0000","last_active":"2023-09-23T20:59:52.349952+0000","last_peered":"2023-09-23T20:59:52.349952+0000","last_clean":"2023-09-23T20:59:52.349952+0000","last_became_active":"2023-09-23T20:51:10.837750+0000","last_became_peered":"2023-09-23T20:51:10.837750+0000","last_unstale":"2023-09-23T20:59:52.349952+0000","last_undegraded":"2023-09-23T20:59:52.349952+0000","last_fullsized":"2023-09-23T20:59:52.349952+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:15:51.648085+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":213,"reported_epoch":244,"state":"active+clean","last_fresh":"2023-09-23T21:00:04.345559+0000","last_change":"2023-09-23T20:55:35.951146+0000","last_active":"2023-09-23T21:00:04.345559+0000","last_peered":"2023-09-23T21:00:04.345559+0000","last_clean":"2023-09-23T21:00:04.345559+0000","last_became_active":"2023-09-23T20:55:35.950874+0000","last_became_peered":"2023-09-23T20:55:35.950874+0000","last_unstale":"2023-09-23T21:00:04.345559+0000","last_undegraded":"2023-09-23T21:00:04.345559+0000","last_fullsized":"2023-09-23T21:00:04.345559+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:58:13.957677+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":233,"reported_epoch":244,"state":"active+clean","last_fresh":"2023-09-23T21:00:03.122469+0000","last_change":"2023-09-23T20:54:31.898870+0000","last_active":"2023-09-23T21:00:03.122469+0000","last_peered":"2023-09-23T21:00:03.122469+0000","last_clean":"2023-09-23T21:00:03.122469+0000","last_became_active":"2023-09-23T20:54:31.898467+0000","last_became_peered":"2023-09-23T20:54:31.898467+0000","last_unstale":"2023-09-23T21:00:03.122469+0000","last_undegraded":"2023-09-23T21:00:03.122469+0000","last_fullsized":"2023-09-23T21:00:03.122469+0000","mapping_epoch":136,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:56:22.483408+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.6","version":"244'10789","reported_seq":17987,"reported_epoch":244,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-09-23T21:00:11.802189+0000","last_change":"2023-09-23T20:58:39.453197+0000","last_active":"2023-09-23T21:00:11.802189+0000","last_peered":"2023-09-23T21:00:11.802189+0000","last_clean":"2023-09-23T20:57:44.031597+0000","last_became_active":"2023-09-23T20:57:46.443927+0000","last_became_peered":"2023-09-23T20:57:46.443927+0000","last_unstale":"2023-09-23T21:00:11.802189+0000","last_undegraded":"2023-09-23T20:58:39.450464+0000","last_fullsized":"2023-09-23T20:57:45.977227+0000","mapping_epoch":223,"log_start":"211'3400","ondisk_log_start":"211'3400","created":199,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_clean_scrub_stamp":"2023-09-23T20:56:34.529103+0000","objects_scrubbed":0,"log_size":7389,"log_dups_size":0,"ondisk_log_size":7389,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:37:23.911131+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":117899264,"num_objects":1799,"num_object_clones":0,"num_object_copies":5397,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":931,"num_objects_misplaced":900,"num_objects_unfound":0,"num_objects_dirty":1799,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10789,"num_write_kb":402816,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3567,"num_bytes_recovered":233766912,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7],"avail_no_missing":["0","5","7"],"object_location_counts":[{"shards":"0,5,7","objects":900},{"shards":"0,7","objects":-32}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":117,"reported_epoch":243,"state":"active+clean","last_fresh":"2023-09-23T20:59:52.331019+0000","last_change":"2023-09-23T20:54:31.883504+0000","last_active":"2023-09-23T20:59:52.331019+0000","last_peered":"2023-09-23T20:59:52.331019+0000","last_clean":"2023-09-23T20:59:52.331019+0000","last_became_active":"2023-09-23T20:54:31.883172+0000","last_became_peered":"2023-09-23T20:54:31.883172+0000","last_unstale":"2023-09-23T20:59:52.331019+0000","last_undegraded":"2023-09-23T20:59:52.331019+0000","last_fullsized":"2023-09-23T20:59:52.331019+0000","mapping_epoch":136,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:45:40.289120+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"79'161","reported_seq":474,"reported_epoch":243,"state":"active+clean","last_fresh":"2023-09-23T20:59:52.354692+0000","last_change":"2023-09-23T20:57:20.738417+0000","last_active":"2023-09-23T20:59:52.354692+0000","last_peered":"2023-09-23T20:59:52.354692+0000","last_clean":"2023-09-23T20:59:52.354692+0000","last_became_active":"2023-09-23T20:57:08.236842+0000","last_became_peered":"2023-09-23T20:57:08.236842+0000","last_unstale":"2023-09-23T20:59:52.354692+0000","last_undegraded":"2023-09-23T20:59:52.354692+0000","last_fullsized":"2023-09-23T20:59:52.354692+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":0,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:20.738273+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:20.738273+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:18:28.327990+0000","scrub_duration":0.0088336339999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":222,"reported_epoch":243,"state":"active+clean","last_fresh":"2023-09-23T20:59:52.342178+0000","last_change":"2023-09-23T20:55:35.950777+0000","last_active":"2023-09-23T20:59:52.342178+0000","last_peered":"2023-09-23T20:59:52.342178+0000","last_clean":"2023-09-23T20:59:52.342178+0000","last_became_active":"2023-09-23T20:55:35.950427+0000","last_became_peered":"2023-09-23T20:55:35.950427+0000","last_unstale":"2023-09-23T20:59:52.342178+0000","last_undegraded":"2023-09-23T20:59:52.342178+0000","last_fullsized":"2023-09-23T20:59:52.342178+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:20:10.371751+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":189,"reported_epoch":244,"state":"active+clean","last_fresh":"2023-09-23T21:00:03.130527+0000","last_change":"2023-09-23T20:55:35.978140+0000","last_active":"2023-09-23T21:00:03.130527+0000","last_peered":"2023-09-23T21:00:03.130527+0000","last_clean":"2023-09-23T21:00:03.130527+0000","last_became_active":"2023-09-23T20:55:35.977831+0000","last_became_peered":"2023-09-23T20:55:35.977831+0000","last_unstale":"2023-09-23T21:00:03.130527+0000","last_undegraded":"2023-09-23T21:00:03.130527+0000","last_fullsized":"2023-09-23T21:00:03.130527+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:25:47.678023+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.5","version":"244'10896","reported_seq":10958,"reported_epoch":244,"state":"active+clean","last_fresh":"2023-09-23T21:00:11.520630+0000","last_change":"2023-09-23T20:56:45.990110+0000","last_active":"2023-09-23T21:00:11.520630+0000","last_peered":"2023-09-23T21:00:11.520630+0000","last_clean":"2023-09-23T21:00:11.520630+0000","last_became_active":"2023-09-23T20:56:45.988394+0000","last_became_peered":"2023-09-23T20:56:45.988394+0000","last_unstale":"2023-09-23T21:00:11.520630+0000","last_undegraded":"2023-09-23T21:00:11.520630+0000","last_fullsized":"2023-09-23T21:00:11.520630+0000","mapping_epoch":204,"log_start":"206'2000","ondisk_log_start":"206'2000","created":199,"last_epoch_clean":205,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_clean_scrub_stamp":"2023-09-23T20:56:34.529103+0000","objects_scrubbed":0,"log_size":8896,"log_dups_size":0,"ondisk_log_size":8896,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:22:03.814210+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":116785152,"num_objects":1782,"num_object_clones":0,"num_object_copies":5346,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1782,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10896,"num_write_kb":405696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,0],"acting":[2,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"79'161","reported_seq":512,"reported_epoch":245,"state":"active+clean","last_fresh":"2023-09-23T21:00:13.274980+0000","last_change":"2023-09-23T20:57:20.378531+0000","last_active":"2023-09-23T21:00:13.274980+0000","last_peered":"2023-09-23T21:00:13.274980+0000","last_clean":"2023-09-23T21:00:13.274980+0000","last_became_active":"2023-09-23T20:57:08.238019+0000","last_became_peered":"2023-09-23T20:57:08.238019+0000","last_unstale":"2023-09-23T21:00:13.274980+0000","last_undegraded":"2023-09-23T21:00:13.274980+0000","last_fullsized":"2023-09-23T21:00:13.274980+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":3,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:20.377714+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:20.377714+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:00:08.305602+0000","scrub_duration":0.003015228,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":114,"reported_epoch":244,"state":"active+clean","last_fresh":"2023-09-23T21:00:03.121013+0000","last_change":"2023-09-23T20:56:45.991884+0000","last_active":"2023-09-23T21:00:03.121013+0000","last_peered":"2023-09-23T21:00:03.121013+0000","last_clean":"2023-09-23T21:00:03.121013+0000","last_became_active":"2023-09-23T20:56:45.991568+0000","last_became_peered":"2023-09-23T20:56:45.991568+0000","last_unstale":"2023-09-23T21:00:03.121013+0000","last_undegraded":"2023-09-23T21:00:03.121013+0000","last_fullsized":"2023-09-23T21:00:03.121013+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":205,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:21:20.775511+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,0],"acting":[1,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":238,"reported_epoch":243,"state":"active+clean","last_fresh":"2023-09-23T20:59:56.760876+0000","last_change":"2023-09-23T20:57:44.979950+0000","last_active":"2023-09-23T20:59:56.760876+0000","last_peered":"2023-09-23T20:59:56.760876+0000","last_clean":"2023-09-23T20:59:56.760876+0000","last_became_active":"2023-09-23T20:57:44.979346+0000","last_became_peered":"2023-09-23T20:57:44.979346+0000","last_unstale":"2023-09-23T20:59:56.760876+0000","last_undegraded":"2023-09-23T20:59:56.760876+0000","last_fullsized":"2023-09-23T20:59:56.760876+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:10:24.372800+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":216,"reported_epoch":243,"state":"active+clean","last_fresh":"2023-09-23T20:59:52.355293+0000","last_change":"2023-09-23T20:50:37.982235+0000","last_active":"2023-09-23T20:59:52.355293+0000","last_peered":"2023-09-23T20:59:52.355293+0000","last_clean":"2023-09-23T20:59:52.355293+0000","last_became_active":"2023-09-23T20:50:37.981913+0000","last_became_peered":"2023-09-23T20:50:37.981913+0000","last_unstale":"2023-09-23T20:59:52.355293+0000","last_undegraded":"2023-09-23T20:59:52.355293+0000","last_fullsized":"2023-09-23T20:59:52.355293+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:59:57.122584+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.0","version":"244'10874","reported_seq":10927,"reported_epoch":244,"state":"active+clean","last_fresh":"2023-09-23T21:00:12.055643+0000","last_change":"2023-09-23T20:56:35.550635+0000","last_active":"2023-09-23T21:00:12.055643+0000","last_peered":"2023-09-23T21:00:12.055643+0000","last_clean":"2023-09-23T21:00:12.055643+0000","last_became_active":"2023-09-23T20:56:35.550376+0000","last_became_peered":"2023-09-23T20:56:35.550376+0000","last_unstale":"2023-09-23T21:00:12.055643+0000","last_undegraded":"2023-09-23T21:00:12.055643+0000","last_fullsized":"2023-09-23T21:00:12.055643+0000","mapping_epoch":199,"log_start":"206'2300","ondisk_log_start":"206'2300","created":199,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_clean_scrub_stamp":"2023-09-23T20:56:34.529103+0000","objects_scrubbed":0,"log_size":8574,"log_dups_size":0,"ondisk_log_size":8574,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:40:55.020049+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":118489088,"num_objects":1808,"num_object_clones":0,"num_object_copies":5424,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1808,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10874,"num_write_kb":405824,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.6","version":"79'161","reported_seq":527,"reported_epoch":244,"state":"active+clean","last_fresh":"2023-09-23T21:00:03.129742+0000","last_change":"2023-09-23T20:57:26.907505+0000","last_active":"2023-09-23T21:00:03.129742+0000","last_peered":"2023-09-23T21:00:03.129742+0000","last_clean":"2023-09-23T21:00:03.129742+0000","last_became_active":"2023-09-23T20:57:08.240321+0000","last_became_peered":"2023-09-23T20:57:08.240321+0000","last_unstale":"2023-09-23T21:00:03.129742+0000","last_undegraded":"2023-09-23T21:00:03.129742+0000","last_fullsized":"2023-09-23T21:00:03.129742+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":3,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:26.907408+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:26.907408+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:34:17.392928+0000","scrub_duration":0.010337038,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":160,"reported_epoch":243,"state":"active+clean","last_fresh":"2023-09-23T20:59:52.354277+0000","last_change":"2023-09-23T20:56:45.988566+0000","last_active":"2023-09-23T20:59:52.354277+0000","last_peered":"2023-09-23T20:59:52.354277+0000","last_clean":"2023-09-23T20:59:52.354277+0000","last_became_active":"2023-09-23T20:56:45.988249+0000","last_became_peered":"2023-09-23T20:56:45.988249+0000","last_unstale":"2023-09-23T20:59:52.354277+0000","last_undegraded":"2023-09-23T20:59:52.354277+0000","last_fullsized":"2023-09-23T20:59:52.354277+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":205,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:51:36.644764+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,4],"acting":[0,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":215,"reported_epoch":244,"state":"active+clean","last_fresh":"2023-09-23T21:00:03.120681+0000","last_change":"2023-09-23T20:54:31.898965+0000","last_active":"2023-09-23T21:00:03.120681+0000","last_peered":"2023-09-23T21:00:03.120681+0000","last_clean":"2023-09-23T21:00:03.120681+0000","last_became_active":"2023-09-23T20:54:31.898663+0000","last_became_peered":"2023-09-23T20:54:31.898663+0000","last_unstale":"2023-09-23T21:00:03.120681+0000","last_undegraded":"2023-09-23T21:00:03.120681+0000","last_fullsized":"2023-09-23T21:00:03.120681+0000","mapping_epoch":136,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:53:05.058087+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":205,"reported_epoch":243,"state":"active+clean","last_fresh":"2023-09-23T20:59:56.760931+0000","last_change":"2023-09-23T20:54:31.884717+0000","last_active":"2023-09-23T20:59:56.760931+0000","last_peered":"2023-09-23T20:59:56.760931+0000","last_clean":"2023-09-23T20:59:56.760931+0000","last_became_active":"2023-09-23T20:54:31.884345+0000","last_became_peered":"2023-09-23T20:54:31.884345+0000","last_unstale":"2023-09-23T20:59:56.760931+0000","last_undegraded":"2023-09-23T20:59:56.760931+0000","last_fullsized":"2023-09-23T20:59:56.760931+0000","mapping_epoch":136,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:02:26.393868+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.3","version":"244'10682","reported_seq":10736,"reported_epoch":244,"state":"active+clean","last_fresh":"2023-09-23T21:00:09.675382+0000","last_change":"2023-09-23T20:56:35.549655+0000","last_active":"2023-09-23T21:00:09.675382+0000","last_peered":"2023-09-23T21:00:09.675382+0000","last_clean":"2023-09-23T21:00:09.675382+0000","last_became_active":"2023-09-23T20:56:35.549357+0000","last_became_peered":"2023-09-23T20:56:35.549357+0000","last_unstale":"2023-09-23T21:00:09.675382+0000","last_undegraded":"2023-09-23T21:00:09.675382+0000","last_fullsized":"2023-09-23T21:00:09.675382+0000","mapping_epoch":199,"log_start":"217'4500","ondisk_log_start":"217'4500","created":199,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_clean_scrub_stamp":"2023-09-23T20:56:34.529103+0000","objects_scrubbed":0,"log_size":6182,"log_dups_size":0,"ondisk_log_size":6182,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:37:58.752518+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":120586240,"num_objects":1840,"num_object_clones":0,"num_object_copies":5520,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1840,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10682,"num_write_kb":400704,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.5","version":"79'161","reported_seq":507,"reported_epoch":243,"state":"active+clean","last_fresh":"2023-09-23T20:59:52.343324+0000","last_change":"2023-09-23T20:57:30.029479+0000","last_active":"2023-09-23T20:59:52.343324+0000","last_peered":"2023-09-23T20:59:52.343324+0000","last_clean":"2023-09-23T20:59:52.343324+0000","last_became_active":"2023-09-23T20:57:08.249909+0000","last_became_peered":"2023-09-23T20:57:08.249909+0000","last_unstale":"2023-09-23T20:59:52.343324+0000","last_undegraded":"2023-09-23T20:59:52.343324+0000","last_fullsized":"2023-09-23T20:59:52.343324+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":3,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:30.029396+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:30.029396+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:16:46.083145+0000","scrub_duration":0.012914682,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":208,"reported_epoch":243,"state":"active+clean","last_fresh":"2023-09-23T20:59:52.341855+0000","last_change":"2023-09-23T20:51:10.836910+0000","last_active":"2023-09-23T20:59:52.341855+0000","last_peered":"2023-09-23T20:59:52.341855+0000","last_clean":"2023-09-23T20:59:52.341855+0000","last_became_active":"2023-09-23T20:51:10.836557+0000","last_became_peered":"2023-09-23T20:51:10.836557+0000","last_unstale":"2023-09-23T20:59:52.341855+0000","last_undegraded":"2023-09-23T20:59:52.341855+0000","last_fullsized":"2023-09-23T20:59:52.341855+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:51:54.019969+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":215,"reported_epoch":244,"state":"active+clean","last_fresh":"2023-09-23T21:00:03.120551+0000","last_change":"2023-09-23T20:55:35.953444+0000","last_active":"2023-09-23T21:00:03.120551+0000","last_peered":"2023-09-23T21:00:03.120551+0000","last_clean":"2023-09-23T21:00:03.120551+0000","last_became_active":"2023-09-23T20:55:35.953151+0000","last_became_peered":"2023-09-23T20:55:35.953151+0000","last_unstale":"2023-09-23T21:00:03.120551+0000","last_undegraded":"2023-09-23T21:00:03.120551+0000","last_fullsized":"2023-09-23T21:00:03.120551+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:39:12.404391+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":33,"reported_epoch":243,"state":"active+clean","last_fresh":"2023-09-23T20:59:52.337413+0000","last_change":"2023-09-23T20:57:44.982509+0000","last_active":"2023-09-23T20:59:52.337413+0000","last_peered":"2023-09-23T20:59:52.337413+0000","last_clean":"2023-09-23T20:59:52.337413+0000","last_became_active":"2023-09-23T20:57:44.982175+0000","last_became_peered":"2023-09-23T20:57:44.982175+0000","last_unstale":"2023-09-23T20:59:52.337413+0000","last_undegraded":"2023-09-23T20:59:52.337413+0000","last_fullsized":"2023-09-23T20:59:52.337413+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:37:49.215488+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.2","version":"244'10697","reported_seq":10752,"reported_epoch":244,"state":"active+clean","last_fresh":"2023-09-23T21:00:09.591967+0000","last_change":"2023-09-23T20:56:35.543020+0000","last_active":"2023-09-23T21:00:09.591967+0000","last_peered":"2023-09-23T21:00:09.591967+0000","last_clean":"2023-09-23T21:00:09.591967+0000","last_became_active":"2023-09-23T20:56:35.542719+0000","last_became_peered":"2023-09-23T20:56:35.542719+0000","last_unstale":"2023-09-23T21:00:09.591967+0000","last_undegraded":"2023-09-23T21:00:09.591967+0000","last_fullsized":"2023-09-23T21:00:09.591967+0000","mapping_epoch":199,"log_start":"217'4500","ondisk_log_start":"217'4500","created":199,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_clean_scrub_stamp":"2023-09-23T20:56:34.529103+0000","objects_scrubbed":0,"log_size":6197,"log_dups_size":0,"ondisk_log_size":6197,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:52:21.088628+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":118947840,"num_objects":1815,"num_object_clones":0,"num_object_copies":5445,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1815,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10697,"num_write_kb":400384,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.4","version":"79'161","reported_seq":522,"reported_epoch":243,"state":"active+clean","last_fresh":"2023-09-23T20:59:52.354577+0000","last_change":"2023-09-23T20:57:20.012904+0000","last_active":"2023-09-23T20:59:52.354577+0000","last_peered":"2023-09-23T20:59:52.354577+0000","last_clean":"2023-09-23T20:59:52.354577+0000","last_became_active":"2023-09-23T20:57:08.237996+0000","last_became_peered":"2023-09-23T20:57:08.237996+0000","last_unstale":"2023-09-23T20:59:52.354577+0000","last_undegraded":"2023-09-23T20:59:52.354577+0000","last_fullsized":"2023-09-23T20:59:52.354577+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":3,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:20.012764+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:20.012764+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:51:28.796946+0000","scrub_duration":0.0045251919999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":194,"reported_epoch":244,"state":"active+clean","last_fresh":"2023-09-23T21:00:03.129471+0000","last_change":"2023-09-23T20:55:35.949541+0000","last_active":"2023-09-23T21:00:03.129471+0000","last_peered":"2023-09-23T21:00:03.129471+0000","last_clean":"2023-09-23T21:00:03.129471+0000","last_became_active":"2023-09-23T20:55:35.944635+0000","last_became_peered":"2023-09-23T20:55:35.944635+0000","last_unstale":"2023-09-23T21:00:03.129471+0000","last_undegraded":"2023-09-23T21:00:03.129471+0000","last_fullsized":"2023-09-23T21:00:03.129471+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:30:36.033585+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"70'1","reported_seq":232,"reported_epoch":243,"state":"active+clean","last_fresh":"2023-09-23T20:59:52.355386+0000","last_change":"2023-09-23T20:54:14.009723+0000","last_active":"2023-09-23T20:59:52.355386+0000","last_peered":"2023-09-23T20:59:52.355386+0000","last_clean":"2023-09-23T20:59:52.355386+0000","last_became_active":"2023-09-23T20:54:13.941771+0000","last_became_peered":"2023-09-23T20:54:13.941771+0000","last_unstale":"2023-09-23T20:59:52.355386+0000","last_undegraded":"2023-09-23T20:59:52.355386+0000","last_fullsized":"2023-09-23T20:59:52.355386+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:48:25.588116+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":148,"reported_epoch":243,"state":"active+clean","last_fresh":"2023-09-23T20:59:52.344614+0000","last_change":"2023-09-23T20:55:35.949780+0000","last_active":"2023-09-23T20:59:52.344614+0000","last_peered":"2023-09-23T20:59:52.344614+0000","last_clean":"2023-09-23T20:59:52.344614+0000","last_became_active":"2023-09-23T20:55:35.949497+0000","last_became_peered":"2023-09-23T20:55:35.949497+0000","last_unstale":"2023-09-23T20:59:52.344614+0000","last_undegraded":"2023-09-23T20:59:52.344614+0000","last_fullsized":"2023-09-23T20:59:52.344614+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:09:37.420882+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":207,"reported_epoch":243,"state":"active+clean","last_fresh":"2023-09-23T20:59:52.353781+0000","last_change":"2023-09-23T20:54:13.929196+0000","last_active":"2023-09-23T20:59:52.353781+0000","last_peered":"2023-09-23T20:59:52.353781+0000","last_clean":"2023-09-23T20:59:52.353781+0000","last_became_active":"2023-09-23T20:54:13.928831+0000","last_became_peered":"2023-09-23T20:54:13.928831+0000","last_unstale":"2023-09-23T20:59:52.353781+0000","last_undegraded":"2023-09-23T20:59:52.353781+0000","last_fullsized":"2023-09-23T20:59:52.353781+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:28:44.316851+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":212,"reported_epoch":243,"state":"active+clean","last_fresh":"2023-09-23T20:59:52.350167+0000","last_change":"2023-09-23T20:52:12.855430+0000","last_active":"2023-09-23T20:59:52.350167+0000","last_peered":"2023-09-23T20:59:52.350167+0000","last_clean":"2023-09-23T20:59:52.350167+0000","last_became_active":"2023-09-23T20:52:12.855115+0000","last_became_peered":"2023-09-23T20:52:12.855115+0000","last_unstale":"2023-09-23T20:59:52.350167+0000","last_undegraded":"2023-09-23T20:59:52.350167+0000","last_fullsized":"2023-09-23T20:59:52.350167+0000","mapping_epoch":100,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":101,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:33:23.802214+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":121,"reported_epoch":243,"state":"active+clean","last_fresh":"2023-09-23T20:59:52.330898+0000","last_change":"2023-09-23T20:54:31.902946+0000","last_active":"2023-09-23T20:59:52.330898+0000","last_peered":"2023-09-23T20:59:52.330898+0000","last_clean":"2023-09-23T20:59:52.330898+0000","last_became_active":"2023-09-23T20:54:31.889484+0000","last_became_peered":"2023-09-23T20:54:31.889484+0000","last_unstale":"2023-09-23T20:59:52.330898+0000","last_undegraded":"2023-09-23T20:59:52.330898+0000","last_fullsized":"2023-09-23T20:59:52.330898+0000","mapping_epoch":136,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:44:44.278398+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":121,"reported_epoch":243,"state":"active+clean","last_fresh":"2023-09-23T20:59:52.343495+0000","last_change":"2023-09-23T20:54:13.950351+0000","last_active":"2023-09-23T20:59:52.343495+0000","last_peered":"2023-09-23T20:59:52.343495+0000","last_clean":"2023-09-23T20:59:52.343495+0000","last_became_active":"2023-09-23T20:54:13.949987+0000","last_became_peered":"2023-09-23T20:54:13.949987+0000","last_unstale":"2023-09-23T20:59:52.343495+0000","last_undegraded":"2023-09-23T20:59:52.343495+0000","last_fullsized":"2023-09-23T20:59:52.343495+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:46:17.765134+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":152,"reported_epoch":243,"state":"active+clean","last_fresh":"2023-09-23T20:59:52.331053+0000","last_change":"2023-09-23T20:56:18.141728+0000","last_active":"2023-09-23T20:59:52.331053+0000","last_peered":"2023-09-23T20:59:52.331053+0000","last_clean":"2023-09-23T20:59:52.331053+0000","last_became_active":"2023-09-23T20:56:18.137979+0000","last_became_peered":"2023-09-23T20:56:18.137979+0000","last_unstale":"2023-09-23T20:59:52.331053+0000","last_undegraded":"2023-09-23T20:59:52.331053+0000","last_fullsized":"2023-09-23T20:59:52.331053+0000","mapping_epoch":192,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":193,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:58:05.658443+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":121,"reported_epoch":243,"state":"active+clean","last_fresh":"2023-09-23T20:59:52.342909+0000","last_change":"2023-09-23T20:54:13.935307+0000","last_active":"2023-09-23T20:59:52.342909+0000","last_peered":"2023-09-23T20:59:52.342909+0000","last_clean":"2023-09-23T20:59:52.342909+0000","last_became_active":"2023-09-23T20:54:13.935000+0000","last_became_peered":"2023-09-23T20:54:13.935000+0000","last_unstale":"2023-09-23T20:59:52.342909+0000","last_undegraded":"2023-09-23T20:59:52.342909+0000","last_fullsized":"2023-09-23T20:59:52.342909+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:47:40.974082+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":231,"reported_epoch":243,"state":"active+clean","last_fresh":"2023-09-23T20:59:52.344970+0000","last_change":"2023-09-23T20:55:35.954870+0000","last_active":"2023-09-23T20:59:52.344970+0000","last_peered":"2023-09-23T20:59:52.344970+0000","last_clean":"2023-09-23T20:59:52.344970+0000","last_became_active":"2023-09-23T20:55:35.954568+0000","last_became_peered":"2023-09-23T20:55:35.954568+0000","last_unstale":"2023-09-23T20:59:52.344970+0000","last_undegraded":"2023-09-23T20:59:52.344970+0000","last_fullsized":"2023-09-23T20:59:52.344970+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:06:22.423566+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":230,"reported_epoch":243,"state":"active+clean","last_fresh":"2023-09-23T20:59:52.341162+0000","last_change":"2023-09-23T20:51:10.838334+0000","last_active":"2023-09-23T20:59:52.341162+0000","last_peered":"2023-09-23T20:59:52.341162+0000","last_clean":"2023-09-23T20:59:52.341162+0000","last_became_active":"2023-09-23T20:51:10.838026+0000","last_became_peered":"2023-09-23T20:51:10.838026+0000","last_unstale":"2023-09-23T20:59:52.341162+0000","last_undegraded":"2023-09-23T20:59:52.341162+0000","last_fullsized":"2023-09-23T20:59:52.341162+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:04:37.009457+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":235,"reported_epoch":243,"state":"active+clean","last_fresh":"2023-09-23T20:59:56.763118+0000","last_change":"2023-09-23T20:54:31.903139+0000","last_active":"2023-09-23T20:59:56.763118+0000","last_peered":"2023-09-23T20:59:56.763118+0000","last_clean":"2023-09-23T20:59:56.763118+0000","last_became_active":"2023-09-23T20:54:31.884520+0000","last_became_peered":"2023-09-23T20:54:31.884520+0000","last_unstale":"2023-09-23T20:59:56.763118+0000","last_undegraded":"2023-09-23T20:59:56.763118+0000","last_fullsized":"2023-09-23T20:59:56.763118+0000","mapping_epoch":136,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:39:34.269705+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1108,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":138,"reported_epoch":243,"state":"active+clean","last_fresh":"2023-09-23T20:59:52.342636+0000","last_change":"2023-09-23T20:55:35.951572+0000","last_active":"2023-09-23T20:59:52.342636+0000","last_peered":"2023-09-23T20:59:52.342636+0000","last_clean":"2023-09-23T20:59:52.342636+0000","last_became_active":"2023-09-23T20:55:35.951294+0000","last_became_peered":"2023-09-23T20:55:35.951294+0000","last_unstale":"2023-09-23T20:59:52.342636+0000","last_undegraded":"2023-09-23T20:59:52.342636+0000","last_fullsized":"2023-09-23T20:59:52.342636+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:36:32.447210+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":7,"num_pg":16,"stat_sum":{"num_bytes":1890058264,"num_objects":28841,"num_object_clones":0,"num_object_copies":86523,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":11874,"num_objects_misplaced":900,"num_objects_unfound":0,"num_objects_dirty":28841,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":174679,"num_write_kb":6512577,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3568,"num_bytes_recovered":233832448,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":6541357056,"data_stored":6541344840,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":119079,"ondisk_log_size":119079,"up":48,"acting":45,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":22,"num_bytes_recovered":5990,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":21,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3381,"ondisk_log_size":3381,"up":63,"acting":63,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201517,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":771700,"kb_used_data":583724,"kb_used_omap":0,"kb_used_meta":187968,"kb_avail":92973452,"statfs":{"total":95995035648,"available":95204814848,"internally_reserved":0,"allocated":597733376,"data_stored":596865656,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":192479232},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:59:46 2023","interfaces":[{"interface":"back","average":{"1min":1.6559999999999999,"5min":1.802,"15min":1.0840000000000001},"min":{"1min":0.245,"5min":0.21099999999999999,"15min":0.21099999999999999},"max":{"1min":6.4420000000000002,"5min":43.994999999999997,"15min":43.994999999999997},"last":2.2149999999999999},{"interface":"front","average":{"1min":1.6339999999999999,"5min":1.9159999999999999,"15min":1.1699999999999999},"min":{"1min":0.318,"5min":0.28199999999999997,"15min":0.218},"max":{"1min":6.4610000000000003,"5min":44.206000000000003,"15min":44.206000000000003},"last":2.234}]},{"osd":1,"last update":"Sat Sep 23 20:59:19 2023","interfaces":[{"interface":"back","average":{"1min":1.74,"5min":1.835,"15min":1.2230000000000001},"min":{"1min":0.26600000000000001,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":5.4359999999999999,"5min":10.622,"15min":22.591999999999999},"last":0.75900000000000001},{"interface":"front","average":{"1min":2.0600000000000001,"5min":1.639,"15min":1.0860000000000001},"min":{"1min":0.32100000000000001,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":6.7709999999999999,"5min":10.097,"15min":21.437000000000001},"last":0.495}]},{"osd":2,"last update":"Sat Sep 23 20:59:55 2023","interfaces":[{"interface":"back","average":{"1min":2.7149999999999999,"5min":2.2000000000000002,"15min":1.294},"min":{"1min":0.30399999999999999,"5min":0.26500000000000001,"15min":0.24399999999999999},"max":{"1min":8.5289999999999999,"5min":10.805999999999999,"15min":10.805999999999999},"last":0.314},{"interface":"front","average":{"1min":2.339,"5min":2.2639999999999998,"15min":1.3089999999999999},"min":{"1min":0.26500000000000001,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":11.590999999999999,"5min":45.25,"15min":45.25},"last":9.6660000000000004}]},{"osd":3,"last update":"Sat Sep 23 20:59:30 2023","interfaces":[{"interface":"back","average":{"1min":2.6200000000000001,"5min":2.508,"15min":1.7250000000000001},"min":{"1min":0.33500000000000002,"5min":0.32100000000000001,"15min":0.314},"max":{"1min":8.4390000000000001,"5min":13.007,"15min":36.408000000000001},"last":0.39300000000000002},{"interface":"front","average":{"1min":2.7789999999999999,"5min":2.3079999999999998,"15min":1.552},"min":{"1min":0.41599999999999998,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":13.198,"5min":14.220000000000001,"15min":35.593000000000004},"last":0.48199999999999998}]},{"osd":4,"last update":"Sat Sep 23 20:59:19 2023","interfaces":[{"interface":"back","average":{"1min":0.41099999999999998,"5min":0.78500000000000003,"15min":0.60499999999999998},"min":{"1min":0.25700000000000001,"5min":0.23799999999999999,"15min":0.23699999999999999},"max":{"1min":1.1659999999999999,"5min":5.8920000000000003,"15min":5.8920000000000003},"last":0.46899999999999997},{"interface":"front","average":{"1min":0.51900000000000002,"5min":0.77600000000000002,"15min":0.61299999999999999},"min":{"1min":0.28599999999999998,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":2.952,"5min":5.3200000000000003,"15min":5.3200000000000003},"last":0.34599999999999997}]},{"osd":5,"last update":"Sat Sep 23 20:59:19 2023","interfaces":[{"interface":"back","average":{"1min":0.443,"5min":0.78100000000000003,"15min":0.628},"min":{"1min":0.316,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":0.64000000000000001,"5min":7.0389999999999997,"15min":7.0389999999999997},"last":0.35999999999999999},{"interface":"front","average":{"1min":0.55200000000000005,"5min":0.72299999999999998,"15min":0.61399999999999999},"min":{"1min":0.30199999999999999,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":1.909,"5min":4.5999999999999996,"15min":4.5999999999999996},"last":0.51000000000000001}]},{"osd":6,"last update":"Sat Sep 23 20:59:42 2023","interfaces":[{"interface":"back","average":{"1min":0.48299999999999998,"5min":0.68000000000000005,"15min":0.67100000000000004},"min":{"1min":0.28899999999999998,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":0.77300000000000002,"5min":3.5249999999999999,"15min":3.5249999999999999},"last":0.38200000000000001},{"interface":"front","average":{"1min":0.48499999999999999,"5min":0.749,"15min":0.78600000000000003},"min":{"1min":0.34699999999999998,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":0.95499999999999996,"5min":3.5409999999999999,"15min":3.5409999999999999},"last":0.40899999999999997}]}]},{"osd":6,"up_from":164,"seq":704374636601,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":685300,"kb_used_data":571628,"kb_used_omap":10,"kb_used_meta":113653,"kb_avail":93059852,"statfs":{"total":95995035648,"available":95293288448,"internally_reserved":0,"allocated":585347072,"data_stored":584487240,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":10328,"internal_metadata":116381608},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:59:45 2023","interfaces":[{"interface":"back","average":{"1min":1.1599999999999999,"5min":1.5,"15min":1.0760000000000001},"min":{"1min":0.23599999999999999,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":4.0960000000000001,"5min":8.7319999999999993,"15min":8.7319999999999993},"last":10.404},{"interface":"front","average":{"1min":1.5029999999999999,"5min":1.6060000000000001,"15min":1.1899999999999999},"min":{"1min":0.23999999999999999,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":4.2869999999999999,"5min":6.7140000000000004,"15min":6.7140000000000004},"last":0.47199999999999998}]},{"osd":1,"last update":"Sat Sep 23 20:59:45 2023","interfaces":[{"interface":"back","average":{"1min":1.554,"5min":1.25,"15min":0.94199999999999995},"min":{"1min":0.27200000000000002,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":8.4309999999999992,"5min":8.4309999999999992,"15min":8.4309999999999992},"last":1.3959999999999999},{"interface":"front","average":{"1min":1.506,"5min":1.3720000000000001,"15min":1.036},"min":{"1min":0.26100000000000001,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":7.2069999999999999,"5min":10.324999999999999,"15min":10.324999999999999},"last":10.518000000000001}]},{"osd":2,"last update":"Sat Sep 23 20:59:45 2023","interfaces":[{"interface":"back","average":{"1min":2.097,"5min":2.0720000000000001,"15min":1.788},"min":{"1min":0.312,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":6.0670000000000002,"5min":13.750999999999999,"15min":13.750999999999999},"last":2.194},{"interface":"front","average":{"1min":2.46,"5min":1.8,"15min":1.3380000000000001},"min":{"1min":0.32000000000000001,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":5.6130000000000004,"5min":8.0890000000000004,"15min":8.0890000000000004},"last":1.1559999999999999}]},{"osd":3,"last update":"Sat Sep 23 20:59:45 2023","interfaces":[{"interface":"back","average":{"1min":1.768,"5min":1.415,"15min":1.216},"min":{"1min":0.45000000000000001,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":4.7220000000000004,"5min":6.9000000000000004,"15min":6.9000000000000004},"last":14.144},{"interface":"front","average":{"1min":1.6479999999999999,"5min":1.7490000000000001,"15min":1.651},"min":{"1min":0.34999999999999998,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":4.6289999999999996,"5min":11.021000000000001,"15min":11.021000000000001},"last":1.639}]},{"osd":4,"last update":"Sat Sep 23 20:59:45 2023","interfaces":[{"interface":"back","average":{"1min":0.68700000000000006,"5min":0.78600000000000003,"15min":0.96299999999999997},"min":{"1min":0.27100000000000002,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":2.3460000000000001,"5min":9.0079999999999991,"15min":9.0079999999999991},"last":0.32700000000000001},{"interface":"front","average":{"1min":0.48599999999999999,"5min":0.499,"15min":0.52000000000000002},"min":{"1min":0.29899999999999999,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":1.115,"5min":2.0059999999999998,"15min":2.0059999999999998},"last":0.311}]},{"osd":5,"last update":"Sat Sep 23 20:59:45 2023","interfaces":[{"interface":"back","average":{"1min":0.78100000000000003,"5min":0.61699999999999999,"15min":0.56599999999999995},"min":{"1min":0.28100000000000003,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":2.3730000000000002,"5min":2.8239999999999998,"15min":2.8239999999999998},"last":0.36199999999999999},{"interface":"front","average":{"1min":0.58299999999999996,"5min":0.73999999999999999,"15min":0.78600000000000003},"min":{"1min":0.34000000000000002,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":1.2290000000000001,"5min":3.1379999999999999,"15min":3.1379999999999999},"last":0.45400000000000001}]},{"osd":7,"last update":"Sat Sep 23 20:59:45 2023","interfaces":[{"interface":"back","average":{"1min":0.51700000000000002,"5min":0.77200000000000002,"15min":0.876},"min":{"1min":0.318,"5min":0.312,"15min":0.312},"max":{"1min":1.1819999999999999,"5min":7.8570000000000002,"15min":7.8570000000000002},"last":0.48199999999999998},{"interface":"front","average":{"1min":0.46500000000000002,"5min":0.64900000000000002,"15min":0.56899999999999995},"min":{"1min":0.33800000000000002,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":0.63200000000000001,"5min":8.282,"15min":8.282},"last":0.40100000000000002}]}]},{"osd":1,"up_from":88,"seq":377957122172,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1254908,"kb_used_data":1023732,"kb_used_omap":9,"kb_used_meta":231030,"kb_avail":92490244,"statfs":{"total":95995035648,"available":94710009856,"internally_reserved":0,"allocated":1048301568,"data_stored":1047087460,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9239,"internal_metadata":236575721},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:59:23 2023","interfaces":[{"interface":"back","average":{"1min":3.9180000000000001,"5min":3.1070000000000002,"15min":1.5389999999999999},"min":{"1min":0.20699999999999999,"5min":0.20699999999999999,"15min":0.20699999999999999},"max":{"1min":14.813000000000001,"5min":16.646999999999998,"15min":16.646999999999998},"last":4.3899999999999997},{"interface":"front","average":{"1min":3.77,"5min":3.335,"15min":1.6819999999999999},"min":{"1min":0.33300000000000002,"5min":0.246,"15min":0.22500000000000001},"max":{"1min":9.484,"5min":16.664999999999999,"15min":18.457999999999998},"last":6.2169999999999996}]},{"osd":2,"last update":"Sat Sep 23 20:59:55 2023","interfaces":[{"interface":"back","average":{"1min":2.4809999999999999,"5min":2.3210000000000002,"15min":1.319},"min":{"1min":0.222,"5min":0.221,"15min":0.20300000000000001},"max":{"1min":13.752000000000001,"5min":14.335000000000001,"15min":14.335000000000001},"last":3.77},{"interface":"front","average":{"1min":3.4470000000000001,"5min":3.3519999999999999,"15min":1.948},"min":{"1min":0.27400000000000002,"5min":0.248,"15min":0.248},"max":{"1min":18.026,"5min":18.026,"15min":38.951000000000001},"last":6.2450000000000001}]},{"osd":3,"last update":"Sat Sep 23 20:59:23 2023","interfaces":[{"interface":"back","average":{"1min":3.1539999999999999,"5min":2.629,"15min":1.5589999999999999},"min":{"1min":0.42499999999999999,"5min":0.246,"15min":0.246},"max":{"1min":9.4450000000000003,"5min":16.055,"15min":25.314},"last":3.5750000000000002},{"interface":"front","average":{"1min":3.0870000000000002,"5min":2.4860000000000002,"15min":1.411},"min":{"1min":0.29799999999999999,"5min":0.20599999999999999,"15min":0.20599999999999999},"max":{"1min":10.204000000000001,"5min":14.225,"15min":19.881},"last":3.5990000000000002}]},{"osd":4,"last update":"Sat Sep 23 20:59:23 2023","interfaces":[{"interface":"back","average":{"1min":1.871,"5min":1.6439999999999999,"15min":1.2130000000000001},"min":{"1min":0.28299999999999997,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":9.3480000000000008,"5min":10.598000000000001,"15min":12.006},"last":3.6280000000000001},{"interface":"front","average":{"1min":1.633,"5min":1.675,"15min":1.0800000000000001},"min":{"1min":0.38400000000000001,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":7.532,"5min":7.532,"15min":11.234999999999999},"last":3.653}]},{"osd":5,"last update":"Sat Sep 23 20:59:23 2023","interfaces":[{"interface":"back","average":{"1min":2.3700000000000001,"5min":2.1469999999999998,"15min":1.292},"min":{"1min":0.55800000000000005,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":8.4410000000000007,"5min":15.041,"15min":15.041},"last":6.0999999999999996},{"interface":"front","average":{"1min":1.9370000000000001,"5min":1.756,"15min":1.2390000000000001},"min":{"1min":0.432,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":9.3889999999999993,"5min":10.647,"15min":12.08},"last":3.673}]},{"osd":6,"last update":"Sat Sep 23 20:59:41 2023","interfaces":[{"interface":"back","average":{"1min":2.266,"5min":1.5609999999999999,"15min":1.397},"min":{"1min":0.46200000000000002,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":10.159000000000001,"5min":10.673999999999999,"15min":10.673999999999999},"last":3.7050000000000001},{"interface":"front","average":{"1min":1.8660000000000001,"5min":1.6299999999999999,"15min":1.3779999999999999},"min":{"1min":0.33700000000000002,"5min":0.309,"15min":0.309},"max":{"1min":7.6779999999999999,"5min":8.7530000000000001,"15min":8.7530000000000001},"last":3.6869999999999998}]},{"osd":7,"last update":"Sat Sep 23 20:59:23 2023","interfaces":[{"interface":"back","average":{"1min":2.089,"5min":1.871,"15min":1.163},"min":{"1min":0.433,"5min":0.36199999999999999,"15min":0.35699999999999998},"max":{"1min":8.7769999999999992,"5min":8.7769999999999992,"15min":11.186999999999999},"last":3.7240000000000002},{"interface":"front","average":{"1min":2.5430000000000001,"5min":2.202,"15min":1.325},"min":{"1min":0.497,"5min":0.40899999999999997,"15min":0.34599999999999997},"max":{"1min":8.4939999999999998,"5min":15.098000000000001,"15min":15.098000000000001},"last":6.1520000000000001}]}]},{"osd":0,"up_from":83,"seq":356482285698,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1132432,"kb_used_data":930184,"kb_used_omap":28,"kb_used_meta":202211,"kb_avail":92612720,"statfs":{"total":95995035648,"available":94835425280,"internally_reserved":0,"allocated":952508416,"data_stored":951327944,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":28848,"internal_metadata":207064912},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 20:59:19 2023","interfaces":[{"interface":"back","average":{"1min":3.694,"5min":3.121,"15min":1.4850000000000001},"min":{"1min":0.22700000000000001,"5min":0.191,"15min":0.191},"max":{"1min":15.577999999999999,"5min":16.484999999999999,"15min":18.863},"last":19.146000000000001},{"interface":"front","average":{"1min":4.4610000000000003,"5min":3.0339999999999998,"15min":1.5389999999999999},"min":{"1min":0.20999999999999999,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":15.771000000000001,"5min":15.771000000000001,"15min":17.704999999999998},"last":4.101}]},{"osd":2,"last update":"Sat Sep 23 20:59:51 2023","interfaces":[{"interface":"back","average":{"1min":3.8879999999999999,"5min":3.4180000000000001,"15min":1.867},"min":{"1min":0.50900000000000001,"5min":0.28299999999999997,"15min":0.23799999999999999},"max":{"1min":21.053000000000001,"5min":21.053000000000001,"15min":21.053000000000001},"last":1.1439999999999999},{"interface":"front","average":{"1min":3.6579999999999999,"5min":3.7450000000000001,"15min":1.944},"min":{"1min":0.26300000000000001,"5min":0.25800000000000001,"15min":0.21099999999999999},"max":{"1min":13.956,"5min":21.747,"15min":21.747},"last":1.3480000000000001}]},{"osd":3,"last update":"Sat Sep 23 20:59:19 2023","interfaces":[{"interface":"back","average":{"1min":4.7839999999999998,"5min":3.0590000000000002,"15min":1.5900000000000001},"min":{"1min":0.38300000000000001,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":21.184000000000001,"5min":21.184000000000001,"15min":21.184000000000001},"last":15.082000000000001},{"interface":"front","average":{"1min":6.0469999999999997,"5min":3.5510000000000002,"15min":1.794},"min":{"1min":0.30599999999999999,"5min":0.28000000000000003,"15min":0.23300000000000001},"max":{"1min":21.167000000000002,"5min":21.167000000000002,"15min":21.167000000000002},"last":0.96199999999999997}]},{"osd":4,"last update":"Sat Sep 23 20:59:51 2023","interfaces":[{"interface":"back","average":{"1min":1.9359999999999999,"5min":2.823,"15min":1.5589999999999999},"min":{"1min":0.38500000000000001,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":7.931,"5min":21.594999999999999,"15min":21.594999999999999},"last":0.42799999999999999},{"interface":"front","average":{"1min":2.8639999999999999,"5min":1.9830000000000001,"15min":1.278},"min":{"1min":0.41099999999999998,"5min":0.31,"15min":0.28100000000000003},"max":{"1min":13.173999999999999,"5min":13.173999999999999,"15min":18.724},"last":8.4849999999999994}]},{"osd":5,"last update":"Sat Sep 23 20:59:51 2023","interfaces":[{"interface":"back","average":{"1min":1.859,"5min":1.889,"15min":1.258},"min":{"1min":0.38700000000000001,"5min":0.318,"15min":0.318},"max":{"1min":6.9980000000000002,"5min":8.8970000000000002,"15min":8.8970000000000002},"last":0.309},{"interface":"front","average":{"1min":1.748,"5min":2.7290000000000001,"15min":1.5649999999999999},"min":{"1min":0.41899999999999998,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":7.9820000000000002,"5min":20.512,"15min":20.512},"last":0.38700000000000001}]},{"osd":6,"last update":"Sat Sep 23 20:59:46 2023","interfaces":[{"interface":"back","average":{"1min":2.9180000000000001,"5min":2.145,"15min":1.819},"min":{"1min":0.45900000000000002,"5min":0.37,"15min":0.37},"max":{"1min":13.134,"5min":13.134,"15min":13.134},"last":8.5350000000000001},{"interface":"front","average":{"1min":2.9140000000000001,"5min":2.198,"15min":1.8959999999999999},"min":{"1min":0.43099999999999999,"5min":0.377,"15min":0.377},"max":{"1min":13.254,"5min":13.254,"15min":13.254},"last":8.5609999999999999}]},{"osd":7,"last update":"Sat Sep 23 20:59:51 2023","interfaces":[{"interface":"back","average":{"1min":2.1520000000000001,"5min":2.0139999999999998,"15min":1.3169999999999999},"min":{"1min":0.433,"5min":0.379,"15min":0.379},"max":{"1min":6.9699999999999998,"5min":8.7729999999999997,"15min":8.7729999999999997},"last":0.40300000000000002},{"interface":"front","average":{"1min":3.0459999999999998,"5min":2.105,"15min":1.3200000000000001},"min":{"1min":0.44600000000000001,"5min":0.377,"15min":0.377},"max":{"1min":13.218999999999999,"5min":13.218999999999999,"15min":18.673999999999999},"last":8.5879999999999992}]}]},{"osd":2,"up_from":93,"seq":399431958645,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1362236,"kb_used_data":1048692,"kb_used_omap":81,"kb_used_meta":313454,"kb_avail":92382916,"statfs":{"total":95995035648,"available":94600105984,"internally_reserved":0,"allocated":1073860608,"data_stored":1072696591,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":83750,"internal_metadata":320977114},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:59:50 2023","interfaces":[{"interface":"back","average":{"1min":4.8730000000000002,"5min":2.7000000000000002,"15min":1.3799999999999999},"min":{"1min":0.189,"5min":0.17999999999999999,"15min":0.17999999999999999},"max":{"1min":29.498000000000001,"5min":29.498000000000001,"15min":29.498000000000001},"last":10.946},{"interface":"front","average":{"1min":4.032,"5min":3.3370000000000002,"15min":1.6739999999999999},"min":{"1min":0.20599999999999999,"5min":0.20599999999999999,"15min":0.20599999999999999},"max":{"1min":17.251000000000001,"5min":17.251000000000001,"15min":17.251000000000001},"last":1.9930000000000001}]},{"osd":1,"last update":"Sat Sep 23 20:59:50 2023","interfaces":[{"interface":"back","average":{"1min":2.6030000000000002,"5min":2.5619999999999998,"15min":1.625},"min":{"1min":0.25700000000000001,"5min":0.20200000000000001,"15min":0.20200000000000001},"max":{"1min":7.9800000000000004,"5min":14.709,"15min":16.081},"last":0.52000000000000002},{"interface":"front","average":{"1min":3.427,"5min":3.2570000000000001,"15min":1.97},"min":{"1min":0.26600000000000001,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":9.9090000000000007,"5min":11.717000000000001,"15min":16.041},"last":0.36199999999999999}]},{"osd":3,"last update":"Sat Sep 23 20:59:29 2023","interfaces":[{"interface":"back","average":{"1min":5.0449999999999999,"5min":3.6579999999999999,"15min":1.893},"min":{"1min":0.30299999999999999,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":27.614999999999998,"5min":27.614999999999998,"15min":27.614999999999998},"last":1.7949999999999999},{"interface":"front","average":{"1min":5.9119999999999999,"5min":4.234,"15min":2.1930000000000001},"min":{"1min":0.33000000000000002,"5min":0.22800000000000001,"15min":0.221},"max":{"1min":27.562000000000001,"5min":27.562000000000001,"15min":27.562000000000001},"last":1.8620000000000001}]},{"osd":4,"last update":"Sat Sep 23 20:59:50 2023","interfaces":[{"interface":"back","average":{"1min":1.575,"5min":1.7749999999999999,"15min":1.2070000000000001},"min":{"1min":0.36499999999999999,"5min":0.28699999999999998,"15min":0.26400000000000001},"max":{"1min":9.8659999999999997,"5min":9.8659999999999997,"15min":9.8659999999999997},"last":0.42599999999999999},{"interface":"front","average":{"1min":2.8439999999999999,"5min":2.1880000000000002,"15min":1.3720000000000001},"min":{"1min":0.32400000000000001,"5min":0.313,"15min":0.313},"max":{"1min":9.9719999999999995,"5min":10.609,"15min":10.609},"last":1.9059999999999999}]},{"osd":5,"last update":"Sat Sep 23 20:59:50 2023","interfaces":[{"interface":"back","average":{"1min":1.4179999999999999,"5min":1.99,"15min":1.298},"min":{"1min":0.33000000000000002,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":6.3570000000000002,"5min":9.0329999999999995,"15min":9.7629999999999999},"last":0.44600000000000001},{"interface":"front","average":{"1min":2.5760000000000001,"5min":2.1419999999999999,"15min":1.3720000000000001},"min":{"1min":0.35599999999999998,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":9.9220000000000006,"5min":10.574,"15min":10.574},"last":1.9370000000000001}]},{"osd":6,"last update":"Sat Sep 23 20:59:45 2023","interfaces":[{"interface":"back","average":{"1min":2.9060000000000001,"5min":2.0640000000000001,"15min":1.8},"min":{"1min":0.35699999999999998,"5min":0.312,"15min":0.312},"max":{"1min":10.012,"5min":14.244999999999999,"15min":14.244999999999999},"last":2.008},{"interface":"front","average":{"1min":2.8740000000000001,"5min":1.8129999999999999,"15min":1.2909999999999999},"min":{"1min":0.41599999999999998,"5min":0.374,"15min":0.374},"max":{"1min":10.032999999999999,"5min":10.032999999999999,"15min":10.032999999999999},"last":1.972}]},{"osd":7,"last update":"Sat Sep 23 20:59:50 2023","interfaces":[{"interface":"back","average":{"1min":1.3859999999999999,"5min":1.4830000000000001,"15min":1.0449999999999999},"min":{"1min":0.35899999999999999,"5min":0.315,"15min":0.315},"max":{"1min":5.9199999999999999,"5min":8.0820000000000007,"15min":8.0820000000000007},"last":0.45900000000000002},{"interface":"front","average":{"1min":1.121,"5min":1.873,"15min":1.292},"min":{"1min":0.443,"5min":0.35099999999999998,"15min":0.34499999999999997},"max":{"1min":6.3899999999999997,"5min":8.6219999999999999,"15min":9.7859999999999996},"last":0.40200000000000002}]}]},{"osd":3,"up_from":98,"seq":420906795118,"num_pgs":49,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1255456,"kb_used_data":905688,"kb_used_omap":7,"kb_used_meta":349688,"kb_avail":92489696,"statfs":{"total":95995035648,"available":94709448704,"internally_reserved":0,"allocated":927424512,"data_stored":926285661,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7576,"internal_metadata":358081128},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:59:28 2023","interfaces":[{"interface":"back","average":{"1min":4.056,"5min":2.399,"15min":1.7350000000000001},"min":{"1min":0.30199999999999999,"5min":0.20599999999999999,"15min":0.20599999999999999},"max":{"1min":13.566000000000001,"5min":13.566000000000001,"15min":23.123999999999999},"last":7.6269999999999998},{"interface":"front","average":{"1min":5.2229999999999999,"5min":3.1099999999999999,"15min":1.6259999999999999},"min":{"1min":0.23200000000000001,"5min":0.19800000000000001,"15min":0.19800000000000001},"max":{"1min":13.455,"5min":15.079000000000001,"15min":15.079000000000001},"last":7.8360000000000003}]},{"osd":1,"last update":"Sat Sep 23 20:59:28 2023","interfaces":[{"interface":"back","average":{"1min":2.9510000000000001,"5min":2.8220000000000001,"15min":1.6120000000000001},"min":{"1min":0.23899999999999999,"5min":0.23899999999999999,"15min":0.22900000000000001},"max":{"1min":11.343,"5min":16.318000000000001,"15min":21.385000000000002},"last":7.226},{"interface":"front","average":{"1min":2.972,"5min":2.2290000000000001,"15min":1.6830000000000001},"min":{"1min":0.23400000000000001,"5min":0.218,"15min":0.218},"max":{"1min":11.782,"5min":11.782,"15min":25.353000000000002},"last":3.8010000000000002}]},{"osd":2,"last update":"Sat Sep 23 20:59:28 2023","interfaces":[{"interface":"back","average":{"1min":4.7210000000000001,"5min":3.1459999999999999,"15min":1.7310000000000001},"min":{"1min":0.33000000000000002,"5min":0.33000000000000002,"15min":0.30099999999999999},"max":{"1min":13.616,"5min":15.476000000000001,"15min":24.154},"last":3.1230000000000002},{"interface":"front","average":{"1min":3.665,"5min":2.4790000000000001,"15min":1.792},"min":{"1min":0.27300000000000002,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":13.125999999999999,"5min":16.032,"15min":25.305},"last":3.9660000000000002}]},{"osd":4,"last update":"Sat Sep 23 20:59:28 2023","interfaces":[{"interface":"back","average":{"1min":3.0070000000000001,"5min":2.0299999999999998,"15min":1.3},"min":{"1min":0.379,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":11.089,"5min":11.089,"15min":11.861000000000001},"last":0.36299999999999999},{"interface":"front","average":{"1min":1.954,"5min":1.359,"15min":1.335},"min":{"1min":0.309,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":9.9779999999999998,"5min":9.9779999999999998,"15min":22.853000000000002},"last":3.452}]},{"osd":5,"last update":"Sat Sep 23 20:59:28 2023","interfaces":[{"interface":"back","average":{"1min":2.8540000000000001,"5min":2.052,"15min":1.2929999999999999},"min":{"1min":0.39000000000000001,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":11.048,"5min":11.048,"15min":11.891999999999999},"last":0.33200000000000002},{"interface":"front","average":{"1min":3.585,"5min":1.9510000000000001,"15min":1.272},"min":{"1min":0.53200000000000003,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":13.43,"5min":13.43,"15min":13.43},"last":0.64400000000000002}]},{"osd":6,"last update":"Sat Sep 23 20:59:47 2023","interfaces":[{"interface":"back","average":{"1min":2.0579999999999998,"5min":1.415,"15min":1.105},"min":{"1min":0.33700000000000002,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":13.499000000000001,"5min":13.499000000000001,"15min":13.499000000000001},"last":3.391},{"interface":"front","average":{"1min":3.8410000000000002,"5min":1.966,"15min":1.5960000000000001},"min":{"1min":0.55300000000000005,"5min":0.38700000000000001,"15min":0.38700000000000001},"max":{"1min":13.398999999999999,"5min":13.398999999999999,"15min":13.398999999999999},"last":0.67900000000000005}]},{"osd":7,"last update":"Sat Sep 23 20:59:28 2023","interfaces":[{"interface":"back","average":{"1min":2.8700000000000001,"5min":2.3839999999999999,"15min":1.4670000000000001},"min":{"1min":0.42799999999999999,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":11.114000000000001,"5min":16.364999999999998,"15min":16.364999999999998},"last":0.39200000000000002},{"interface":"front","average":{"1min":3.6400000000000001,"5min":2.077,"15min":1.3440000000000001},"min":{"1min":0.58399999999999996,"5min":0.379,"15min":0.379},"max":{"1min":13.355,"5min":13.355,"15min":13.355},"last":0.70299999999999996}]}]},{"osd":4,"up_from":31,"seq":133143986426,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":727732,"kb_used_data":588844,"kb_used_omap":0,"kb_used_meta":138880,"kb_avail":93017420,"statfs":{"total":95995035648,"available":95249838080,"internally_reserved":0,"allocated":602976256,"data_stored":602108789,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":142213120},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:59:53 2023","interfaces":[{"interface":"back","average":{"1min":2.1469999999999998,"5min":1.915,"15min":1.143},"min":{"1min":0.255,"5min":0.214,"15min":0.214},"max":{"1min":9.0749999999999993,"5min":11.784000000000001,"15min":11.784000000000001},"last":2.5899999999999999},{"interface":"front","average":{"1min":2.4969999999999999,"5min":2.0329999999999999,"15min":1.335},"min":{"1min":0.32800000000000001,"5min":0.26700000000000002,"15min":0.25800000000000001},"max":{"1min":9.1029999999999998,"5min":11.766,"15min":11.766},"last":2.6030000000000002}]},{"osd":1,"last update":"Sat Sep 23 20:59:22 2023","interfaces":[{"interface":"back","average":{"1min":2.4119999999999999,"5min":1.6930000000000001,"15min":1.1200000000000001},"min":{"1min":0.35499999999999998,"5min":0.246,"15min":0.23599999999999999},"max":{"1min":7.3310000000000004,"5min":10.153,"15min":10.153},"last":0.35399999999999998},{"interface":"front","average":{"1min":3.2669999999999999,"5min":2.0099999999999998,"15min":1.1419999999999999},"min":{"1min":0.33000000000000002,"5min":0.26100000000000001,"15min":0.23200000000000001},"max":{"1min":9.7509999999999994,"5min":12.292999999999999,"15min":12.292999999999999},"last":1.5209999999999999}]},{"osd":2,"last update":"Sat Sep 23 20:59:53 2023","interfaces":[{"interface":"back","average":{"1min":2.573,"5min":1.7430000000000001,"15min":1.2410000000000001},"min":{"1min":0.33600000000000002,"5min":0.311,"15min":0.29099999999999998},"max":{"1min":8.1509999999999998,"5min":8.1509999999999998,"15min":15.288},"last":4.5549999999999997},{"interface":"front","average":{"1min":2.8330000000000002,"5min":1.831,"15min":1.3999999999999999},"min":{"1min":0.28599999999999998,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":12.855,"5min":12.855,"15min":15.262},"last":1.6220000000000001}]},{"osd":3,"last update":"Sat Sep 23 20:59:22 2023","interfaces":[{"interface":"back","average":{"1min":2.21,"5min":2.1850000000000001,"15min":1.6180000000000001},"min":{"1min":0.30499999999999999,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":4.835,"5min":14.824999999999999,"15min":31.562999999999999},"last":1.579},{"interface":"front","average":{"1min":2.2770000000000001,"5min":2.2989999999999999,"15min":1.6899999999999999},"min":{"1min":0.30299999999999999,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":4.8490000000000002,"5min":14.839,"15min":31.574000000000002},"last":1.589}]},{"osd":5,"last update":"Sat Sep 23 20:59:36 2023","interfaces":[{"interface":"back","average":{"1min":0.41099999999999998,"5min":0.61099999999999999,"15min":0.52800000000000002},"min":{"1min":0.28899999999999998,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":0.90700000000000003,"5min":3.3279999999999998,"15min":3.3279999999999998},"last":0.29599999999999999},{"interface":"front","average":{"1min":1.002,"5min":0.77800000000000002,"15min":0.61499999999999999},"min":{"1min":0.29999999999999999,"5min":0.251,"15min":0.251},"max":{"1min":3.4830000000000001,"5min":7.7619999999999996,"15min":7.7619999999999996},"last":0.434}]},{"osd":6,"last update":"Sat Sep 23 20:59:46 2023","interfaces":[{"interface":"back","average":{"1min":0.62,"5min":0.70399999999999996,"15min":0.55700000000000005},"min":{"1min":0.32100000000000001,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":3.4180000000000001,"5min":4.6909999999999998,"15min":4.6909999999999998},"last":0.38500000000000001},{"interface":"front","average":{"1min":0.752,"5min":0.73699999999999999,"15min":0.67500000000000004},"min":{"1min":0.34499999999999997,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":3.2999999999999998,"5min":3.456,"15min":3.456},"last":0.39500000000000002}]},{"osd":7,"last update":"Sat Sep 23 20:59:22 2023","interfaces":[{"interface":"back","average":{"1min":0.83499999999999996,"5min":0.752,"15min":0.69399999999999995},"min":{"1min":0.32600000000000001,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":2.8889999999999998,"5min":6.2240000000000002,"15min":7.383},"last":0.36799999999999999},{"interface":"front","average":{"1min":0.73999999999999999,"5min":0.78400000000000003,"15min":0.67500000000000004},"min":{"1min":0.34200000000000003,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":2.8679999999999999,"5min":7.8310000000000004,"15min":7.8310000000000004},"last":0.56399999999999995}]}]},{"osd":5,"up_from":39,"seq":167503724791,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":953640,"kb_used_data":812384,"kb_used_omap":0,"kb_used_meta":141248,"kb_avail":92791512,"statfs":{"total":95995035648,"available":95018508288,"internally_reserved":0,"allocated":831881216,"data_stored":831022325,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":144637952},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:59:38 2023","interfaces":[{"interface":"back","average":{"1min":4.0339999999999998,"5min":1.619,"15min":0.96299999999999997},"min":{"1min":0.27400000000000002,"5min":0.20399999999999999,"15min":0.20399999999999999},"max":{"1min":12.795999999999999,"5min":12.795999999999999,"15min":12.795999999999999},"last":2.4319999999999999},{"interface":"front","average":{"1min":2.7330000000000001,"5min":1.948,"15min":1.125},"min":{"1min":0.24099999999999999,"5min":0.223,"15min":0.223},"max":{"1min":16.219999999999999,"5min":16.219999999999999,"15min":16.219999999999999},"last":3.5470000000000002}]},{"osd":1,"last update":"Sat Sep 23 20:59:17 2023","interfaces":[{"interface":"back","average":{"1min":2.7749999999999999,"5min":1.758,"15min":1.099},"min":{"1min":0.248,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":8.1829999999999998,"5min":8.3520000000000003,"15min":8.3520000000000003},"last":3.1829999999999998},{"interface":"front","average":{"1min":2.3690000000000002,"5min":1.825,"15min":1.099},"min":{"1min":0.27200000000000002,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":8.5399999999999991,"5min":10.285,"15min":10.285},"last":2.9399999999999999}]},{"osd":2,"last update":"Sat Sep 23 20:59:54 2023","interfaces":[{"interface":"back","average":{"1min":2.6819999999999999,"5min":1.6699999999999999,"15min":1.121},"min":{"1min":0.27400000000000002,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":10.371,"5min":10.579000000000001,"15min":12.651},"last":1.0820000000000001},{"interface":"front","average":{"1min":3.423,"5min":2.258,"15min":1.274},"min":{"1min":0.28799999999999998,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":10.333,"5min":12.300000000000001,"15min":12.300000000000001},"last":2.5870000000000002}]},{"osd":3,"last update":"Sat Sep 23 20:59:17 2023","interfaces":[{"interface":"back","average":{"1min":3.5880000000000001,"5min":2.2090000000000001,"15min":1.351},"min":{"1min":0.46700000000000003,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":12.188000000000001,"5min":12.188000000000001,"15min":12.188000000000001},"last":5.0490000000000004},{"interface":"front","average":{"1min":2.0739999999999998,"5min":1.6060000000000001,"15min":1.071},"min":{"1min":0.32500000000000001,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":7.258,"5min":10.457000000000001,"15min":10.457000000000001},"last":1.992}]},{"osd":4,"last update":"Sat Sep 23 20:59:34 2023","interfaces":[{"interface":"back","average":{"1min":0.81699999999999995,"5min":0.63300000000000001,"15min":0.55300000000000005},"min":{"1min":0.32100000000000001,"5min":0.252,"15min":0.252},"max":{"1min":2.9169999999999998,"5min":3.8479999999999999,"15min":3.8479999999999999},"last":0.36699999999999999},{"interface":"front","average":{"1min":0.83499999999999996,"5min":0.67500000000000004,"15min":0.57499999999999996},"min":{"1min":0.27400000000000002,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":3.3530000000000002,"5min":3.3530000000000002,"15min":3.3530000000000002},"last":0.30399999999999999}]},{"osd":6,"last update":"Sat Sep 23 20:59:38 2023","interfaces":[{"interface":"back","average":{"1min":0.86099999999999999,"5min":0.65100000000000002,"15min":0.59099999999999997},"min":{"1min":0.29199999999999998,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":4.1120000000000001,"5min":4.1120000000000001,"15min":4.1120000000000001},"last":2.2050000000000001},{"interface":"front","average":{"1min":1.0269999999999999,"5min":0.755,"15min":0.747},"min":{"1min":0.36899999999999999,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":2.9620000000000002,"5min":3.6560000000000001,"15min":3.6560000000000001},"last":2.1949999999999998}]},{"osd":7,"last update":"Sat Sep 23 20:59:17 2023","interfaces":[{"interface":"back","average":{"1min":0.71099999999999997,"5min":0.80200000000000005,"15min":0.64200000000000002},"min":{"1min":0.33400000000000002,"5min":0.307,"15min":0.30199999999999999},"max":{"1min":2.516,"5min":8.5649999999999995,"15min":8.5649999999999995},"last":0.499},{"interface":"front","average":{"1min":0.77200000000000002,"5min":0.75,"15min":0.65700000000000003},"min":{"1min":0.34399999999999997,"5min":0.29799999999999999,"15min":0.27700000000000002},"max":{"1min":3.0150000000000001,"5min":4.5430000000000001,"15min":4.5430000000000001},"last":0.435}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":942804992,"data_stored":942800920,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1033703424,"data_stored":1033699352,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1066795008,"data_stored":1066795008,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":915865600,"data_stored":915865600,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":590413824,"data_stored":590413824,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":828178432,"data_stored":828178432,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":575016960,"data_stored":575012888,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":588578816,"data_stored":588578816,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T21:00:16.148 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:00:16.288 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:00:16.359 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:00:16 smithi026 ceph-mon[175151]: pgmap v618: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 1.8 GiB data, 7.8 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 578 op/s; 11874/86595 objects degraded (13.712%); 900/86595 objects misplaced (1.039%); 1.9 MiB/s, 30 objects/s recovering 2023-09-23T21:00:16.359 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:00:16 smithi026 ceph-mon[175151]: from='client.45374 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T21:00:16.359 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:00:16 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:00:16.359 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:00:16 smithi026 ceph-mon[179688]: pgmap v618: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 1.8 GiB data, 7.8 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 578 op/s; 11874/86595 objects degraded (13.712%); 900/86595 objects misplaced (1.039%); 1.9 MiB/s, 30 objects/s recovering 2023-09-23T21:00:16.360 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:00:16 smithi026 ceph-mon[179688]: from='client.45374 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T21:00:16.360 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:00:16 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:00:16.361 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:00:16.368 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:00:16.369 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:00:16.470 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:00:16.572 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:00:16.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:00:16 smithi070 ceph-mon[168282]: pgmap v618: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 1.8 GiB data, 7.8 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 578 op/s; 11874/86595 objects degraded (13.712%); 900/86595 objects misplaced (1.039%); 1.9 MiB/s, 30 objects/s recovering 2023-09-23T21:00:16.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:00:16 smithi070 ceph-mon[168282]: from='client.45374 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T21:00:16.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:00:16 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:00:16.635 INFO:teuthology.orchestra.run.smithi070.stdout:ERROR: (22) Invalid argument 2023-09-23T21:00:16.635 INFO:teuthology.orchestra.run.smithi070.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T21:00:16.683 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:00:16.698 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:00:16.699 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:00:16.800 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:00:16.904 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:00:16.992 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:00:16.999 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:00:17.000 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:00:17.044 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_historic_ops 2023-09-23T21:00:17.101 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:00:17.159 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:00:17.226 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:00:17.294 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0: {} 2023-09-23T21:00:17.329 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T21:00:17.330 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1: {} 2023-09-23T21:00:17.343 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:00:17.343 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:00:17.371 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2: {} 2023-09-23T21:00:17.409 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3: {} 2023-09-23T21:00:17.444 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:00:17.455 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4: {} 2023-09-23T21:00:17.495 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T21:00:17.536 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6: {} 2023-09-23T21:00:17.550 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:00:17 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/1228349858' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:00:17.550 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:00:17 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/1228349858' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:00:17.576 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T21:00:17.578 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:00:17.585 INFO:teuthology.orchestra.run.smithi026.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T21:00:17.585 INFO:teuthology.orchestra.run.smithi026.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T21:00:17.586 INFO:teuthology.orchestra.run.smithi026.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T21:00:17.586 INFO:teuthology.orchestra.run.smithi026.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T21:00:17.586 INFO:teuthology.orchestra.run.smithi026.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T21:00:17.586 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T21:00:17.586 INFO:teuthology.orchestra.run.smithi026.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T21:00:17.586 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T21:00:17.617 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T21:00:17.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:00:17 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/1228349858' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:00:18.069 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:00:18.076 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:00:18.077 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:00:18.178 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:00:18.218 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:00:18.384 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:00:18 smithi026 ceph-mon[175151]: pgmap v619: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 1.7 GiB data, 7.7 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 574 op/s; 11874/85095 objects degraded (13.954%); 900/85095 objects misplaced (1.058%); 1.9 MiB/s, 30 objects/s recovering 2023-09-23T21:00:18.384 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:00:18 smithi026 ceph-mon[179688]: pgmap v619: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 1.7 GiB data, 7.7 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 574 op/s; 11874/85095 objects degraded (13.954%); 900/85095 objects misplaced (1.058%); 1.9 MiB/s, 30 objects/s recovering 2023-09-23T21:00:18.396 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:00:18.399 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:00:18.399 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:00:18.500 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:00:18.529 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:00:18 smithi070 ceph-mon[168282]: pgmap v619: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 1.7 GiB data, 7.7 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 574 op/s; 11874/85095 objects degraded (13.954%); 900/85095 objects misplaced (1.058%); 1.9 MiB/s, 30 objects/s recovering 2023-09-23T21:00:18.535 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:00:18.648 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:00:18.656 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:00:18.656 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:00:18.757 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:00:18.840 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:00:18.986 INFO:teuthology.orchestra.run.smithi070.stdout:{ 2023-09-23T21:00:18.986 INFO:teuthology.orchestra.run.smithi070.stdout: "size": 20, 2023-09-23T21:00:18.986 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 600, 2023-09-23T21:00:18.986 INFO:teuthology.orchestra.run.smithi070.stdout: "ops": [ 2023-09-23T21:00:18.986 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:18.986 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.34797.0:114997 6.12 6.28c54f92 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e132)", 2023-09-23T21:00:18.986 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T20:54:12.985231+0000", 2023-09-23T21:00:18.987 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 365.99758850199999, 2023-09-23T21:00:18.987 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.939145275, 2023-09-23T21:00:18.987 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:00:18.987 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "reached pg", 2023-09-23T21:00:18.987 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:00:18.987 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.34797", 2023-09-23T21:00:18.987 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T21:00:18.987 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 114997 2023-09-23T21:00:18.987 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:18.987 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:00:18.987 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:18.987 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:00:18.988 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:54:12.985231+0000", 2023-09-23T21:00:18.988 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:00:18.988 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:18.988 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:18.988 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:00:18.988 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:54:12.985231+0000", 2023-09-23T21:00:18.988 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:00:18.988 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:18.988 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:18.988 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:00:18.988 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:54:12.985228+0000", 2023-09-23T21:00:18.988 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999976 2023-09-23T21:00:18.989 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:18.989 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:18.989 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:00:18.989 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:54:12.985282+0000", 2023-09-23T21:00:18.989 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.3285000000000002e-05 2023-09-23T21:00:18.989 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:18.989 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:18.989 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:00:18.989 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:54:12.985298+0000", 2023-09-23T21:00:18.989 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.6677e-05 2023-09-23T21:00:18.989 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:18.989 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:18.990 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:00:18.990 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:54:12.985306+0000", 2023-09-23T21:00:18.990 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.8590000000000005e-06 2023-09-23T21:00:18.990 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:18.990 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:18.990 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:00:18.990 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:54:12.985354+0000", 2023-09-23T21:00:18.990 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.8124000000000002e-05 2023-09-23T21:00:18.990 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:18.990 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:18.990 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:00:18.991 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:54:12.985364+0000", 2023-09-23T21:00:18.991 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.6500000000000008e-06 2023-09-23T21:00:18.991 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:18.991 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:18.991 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:00:18.991 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:54:13.924362+0000", 2023-09-23T21:00:18.992 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.93899794599999997 2023-09-23T21:00:18.992 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:18.992 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:18.992 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:00:18.992 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:54:13.924376+0000", 2023-09-23T21:00:18.992 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.4238e-05 2023-09-23T21:00:18.992 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:00:18.992 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:00:18.992 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:00:18.992 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:18.992 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:18.993 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.34797.0:156782 6.12 6:4ab9e603:::benchmark_data_smithi026_223868_object55656:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e150)", 2023-09-23T21:00:18.993 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T20:55:07.352049+0000", 2023-09-23T21:00:18.993 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 311.63077064100003, 2023-09-23T21:00:18.993 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.014989538, 2023-09-23T21:00:18.993 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:00:18.993 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:00:18.993 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:00:18.993 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.34797", 2023-09-23T21:00:18.993 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T21:00:18.993 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 156782 2023-09-23T21:00:18.993 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:18.993 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:00:18.994 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:18.994 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:00:18.994 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:07.352049+0000", 2023-09-23T21:00:18.994 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:00:18.994 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:18.994 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:18.994 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:00:18.994 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:07.352049+0000", 2023-09-23T21:00:18.994 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:00:18.994 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:18.994 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:18.995 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:00:18.995 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:07.352047+0000", 2023-09-23T21:00:18.995 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T21:00:18.995 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:18.995 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:18.995 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:00:18.995 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:07.352052+0000", 2023-09-23T21:00:18.995 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.9280000000000001e-06 2023-09-23T21:00:18.995 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:18.995 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:18.995 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:00:18.995 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:07.352064+0000", 2023-09-23T21:00:18.995 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.1902999999999999e-05 2023-09-23T21:00:18.996 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:18.996 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:18.996 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:00:18.996 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:07.352070+0000", 2023-09-23T21:00:18.996 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.8459999999999996e-06 2023-09-23T21:00:18.996 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:18.996 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:18.996 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:00:18.996 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:07.352358+0000", 2023-09-23T21:00:18.996 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00028820300000000001 2023-09-23T21:00:18.996 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:18.996 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:18.997 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:00:18.997 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:07.352372+0000", 2023-09-23T21:00:18.997 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.3669000000000001e-05 2023-09-23T21:00:18.997 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:18.997 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:18.997 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:00:18.997 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.361399+0000", 2023-09-23T21:00:18.997 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.0090268060000001 2023-09-23T21:00:18.997 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:18.997 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:18.997 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:00:18.998 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.361509+0000", 2023-09-23T21:00:18.998 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000110451 2023-09-23T21:00:18.998 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:18.998 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:18.998 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 2,3", 2023-09-23T21:00:18.998 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.361803+0000", 2023-09-23T21:00:18.998 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00029413900000000001 2023-09-23T21:00:18.999 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:18.999 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:18.999 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:00:18.999 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.363432+0000", 2023-09-23T21:00:18.999 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.001628188 2023-09-23T21:00:18.999 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:18.999 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:19.000 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:00:19.000 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.366824+0000", 2023-09-23T21:00:19.000 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0033921910000000001 2023-09-23T21:00:19.000 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:19.000 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:19.000 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:00:19.000 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.366953+0000", 2023-09-23T21:00:19.000 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00012964399999999999 2023-09-23T21:00:19.000 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:19.001 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:19.001 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:00:19.001 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.367000+0000", 2023-09-23T21:00:19.001 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.6319999999999997e-05 2023-09-23T21:00:19.001 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:19.001 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:19.001 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:00:19.002 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.367038+0000", 2023-09-23T21:00:19.002 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.8498000000000003e-05 2023-09-23T21:00:19.002 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:00:19.002 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:00:19.002 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:00:19.002 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:19.002 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:19.002 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.34797.0:156812 6.12 6:4c46565a:::benchmark_data_smithi026_223868_object55686:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e150)", 2023-09-23T21:00:19.002 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T20:55:07.386674+0000", 2023-09-23T21:00:19.002 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 311.59614548299999, 2023-09-23T21:00:19.002 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.98619251500000005, 2023-09-23T21:00:19.003 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:00:19.003 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:00:19.003 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:00:19.003 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.34797", 2023-09-23T21:00:19.003 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T21:00:19.003 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 156812 2023-09-23T21:00:19.003 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:19.003 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:00:19.003 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:19.003 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:00:19.003 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:07.386674+0000", 2023-09-23T21:00:19.003 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:00:19.004 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:19.004 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:19.004 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:00:19.004 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:07.386674+0000", 2023-09-23T21:00:19.004 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:00:19.004 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:19.004 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:19.004 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:00:19.004 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:07.386672+0000", 2023-09-23T21:00:19.004 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T21:00:19.004 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:19.005 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:19.005 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:00:19.005 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:07.386681+0000", 2023-09-23T21:00:19.005 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.2679999999999994e-06 2023-09-23T21:00:19.005 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:19.005 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:19.005 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:00:19.005 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:07.386702+0000", 2023-09-23T21:00:19.005 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.173e-05 2023-09-23T21:00:19.005 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:19.005 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:19.005 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:00:19.006 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:07.386710+0000", 2023-09-23T21:00:19.006 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.0069999999999997e-06 2023-09-23T21:00:19.006 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:19.006 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:19.006 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:00:19.006 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:07.386770+0000", 2023-09-23T21:00:19.006 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.9499000000000001e-05 2023-09-23T21:00:19.006 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:19.006 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:19.006 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:00:19.006 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:07.386780+0000", 2023-09-23T21:00:19.007 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.5619999999999993e-06 2023-09-23T21:00:19.007 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:19.007 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:19.007 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:00:19.007 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.363453+0000", 2023-09-23T21:00:19.007 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.97667318999999997 2023-09-23T21:00:19.007 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:19.007 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:19.007 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:00:19.007 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.363538+0000", 2023-09-23T21:00:19.007 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.5484999999999999e-05 2023-09-23T21:00:19.007 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:19.008 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:19.008 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 2,3", 2023-09-23T21:00:19.008 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.363717+0000", 2023-09-23T21:00:19.008 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00017910699999999999 2023-09-23T21:00:19.008 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:19.008 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:19.008 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:00:19.008 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.366914+0000", 2023-09-23T21:00:19.008 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0031969059999999998 2023-09-23T21:00:19.008 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:19.008 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:19.009 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:00:19.009 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.372737+0000", 2023-09-23T21:00:19.009 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00582243 2023-09-23T21:00:19.009 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:19.009 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:19.009 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:00:19.009 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.372790+0000", 2023-09-23T21:00:19.009 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.3454e-05 2023-09-23T21:00:19.009 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:19.009 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:19.009 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:00:19.009 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.372821+0000", 2023-09-23T21:00:19.010 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.0725999999999997e-05 2023-09-23T21:00:19.010 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:19.010 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:19.010 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:00:19.010 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.372866+0000", 2023-09-23T21:00:19.010 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.5433999999999999e-05 2023-09-23T21:00:19.010 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:00:19.010 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:00:19.010 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:00:19.010 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:19.010 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:19.011 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.34797.0:156863 6.12 6:48481dbe:::benchmark_data_smithi026_223868_object55737:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e150)", 2023-09-23T21:00:19.011 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T20:55:07.442400+0000", 2023-09-23T21:00:19.011 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 311.54041911299998, 2023-09-23T21:00:19.011 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.93060301300000003, 2023-09-23T21:00:19.011 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:00:19.011 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:00:19.011 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:00:19.011 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.34797", 2023-09-23T21:00:19.011 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T21:00:19.011 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 156863 2023-09-23T21:00:19.011 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:19.011 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:00:19.012 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:19.012 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:00:19.012 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:07.442400+0000", 2023-09-23T21:00:19.012 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:00:19.012 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:19.012 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:19.012 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:00:19.012 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:07.442400+0000", 2023-09-23T21:00:19.012 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:00:19.012 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:19.012 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:19.012 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:00:19.013 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:07.442399+0000", 2023-09-23T21:00:19.013 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T21:00:19.013 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:19.013 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:19.013 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:00:19.013 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:07.442405+0000", 2023-09-23T21:00:19.013 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.7490000000000001e-06 2023-09-23T21:00:19.013 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:19.013 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:19.013 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:00:19.013 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:07.442421+0000", 2023-09-23T21:00:19.014 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.6110999999999999e-05 2023-09-23T21:00:19.014 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:19.014 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:19.014 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:00:19.014 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:07.442428+0000", 2023-09-23T21:00:19.014 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.9349999999999997e-06 2023-09-23T21:00:19.014 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:19.014 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:19.014 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:00:19.014 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:07.442498+0000", 2023-09-23T21:00:19.014 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.9488999999999998e-05 2023-09-23T21:00:19.015 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:19.015 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:19.015 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:00:19.015 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:07.442508+0000", 2023-09-23T21:00:19.015 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.0329000000000001e-05 2023-09-23T21:00:19.015 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:19.015 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:19.015 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:00:19.015 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.364821+0000", 2023-09-23T21:00:19.015 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.92231251999999997 2023-09-23T21:00:19.015 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:19.015 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:19.016 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:00:19.016 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.364917+0000", 2023-09-23T21:00:19.016 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.5866000000000007e-05 2023-09-23T21:00:19.016 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:19.016 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:19.016 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 2,3", 2023-09-23T21:00:19.016 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.365161+0000", 2023-09-23T21:00:19.016 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00024452300000000002 2023-09-23T21:00:19.016 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:19.016 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:19.016 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:00:19.017 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.367047+0000", 2023-09-23T21:00:19.017 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.001885947 2023-09-23T21:00:19.017 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:19.017 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:19.017 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:00:19.017 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.372916+0000", 2023-09-23T21:00:19.017 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0058686110000000001 2023-09-23T21:00:19.017 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:19.017 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:19.017 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:00:19.017 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.372948+0000", 2023-09-23T21:00:19.017 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.2159000000000002e-05 2023-09-23T21:00:19.018 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:19.018 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:19.018 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:00:19.018 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.372972+0000", 2023-09-23T21:00:19.018 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.4244999999999999e-05 2023-09-23T21:00:19.018 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:19.018 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:19.018 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:00:19.019 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.373003+0000", 2023-09-23T21:00:19.019 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.112e-05 2023-09-23T21:00:19.019 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:00:19.019 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:00:19.019 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:00:19.019 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:19.019 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:19.019 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.34797.0:156879 6.12 6:4b4fed44:::benchmark_data_smithi026_223868_object55753:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e150)", 2023-09-23T21:00:19.019 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T20:55:07.459292+0000", 2023-09-23T21:00:19.019 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 311.523527624, 2023-09-23T21:00:19.020 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.91741719200000005, 2023-09-23T21:00:19.020 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:00:19.020 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:00:19.020 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:00:19.020 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.34797", 2023-09-23T21:00:19.020 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T21:00:19.020 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 156879 2023-09-23T21:00:19.020 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:19.020 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:00:19.020 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:19.020 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:00:19.020 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:07.459292+0000", 2023-09-23T21:00:19.021 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:00:19.021 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:19.021 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:19.021 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:00:19.021 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:07.459292+0000", 2023-09-23T21:00:19.021 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:00:19.021 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:19.021 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:19.021 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:00:19.021 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:07.459290+0000", 2023-09-23T21:00:19.021 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999981 2023-09-23T21:00:19.021 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:19.022 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:19.022 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:00:19.022 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:07.459299+0000", 2023-09-23T21:00:19.022 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.1679999999999993e-06 2023-09-23T21:00:19.022 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:19.022 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:19.022 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:00:19.022 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:07.459319+0000", 2023-09-23T21:00:19.022 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.9945999999999998e-05 2023-09-23T21:00:19.022 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:19.022 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:19.023 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:00:19.023 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:07.459328+0000", 2023-09-23T21:00:19.023 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.5439999999999999e-06 2023-09-23T21:00:19.023 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:19.023 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:19.023 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:00:19.023 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:07.459397+0000", 2023-09-23T21:00:19.023 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.9640000000000004e-05 2023-09-23T21:00:19.023 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:19.023 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:19.023 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:00:19.024 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:07.459408+0000", 2023-09-23T21:00:19.024 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.1228e-05 2023-09-23T21:00:19.024 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:19.024 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:19.024 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:00:19.024 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.367075+0000", 2023-09-23T21:00:19.024 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.90766636599999995 2023-09-23T21:00:19.024 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:19.024 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:19.024 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:00:19.024 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.367196+0000", 2023-09-23T21:00:19.024 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000121274 2023-09-23T21:00:19.025 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:19.025 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:19.025 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 2,3", 2023-09-23T21:00:19.025 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.367465+0000", 2023-09-23T21:00:19.025 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00026886999999999999 2023-09-23T21:00:19.025 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:19.025 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:19.025 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:00:19.025 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.372639+0000", 2023-09-23T21:00:19.025 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0051736430000000003 2023-09-23T21:00:19.025 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:19.026 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:19.026 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:00:19.026 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.376593+0000", 2023-09-23T21:00:19.026 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0039539919999999999 2023-09-23T21:00:19.026 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:19.026 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:19.026 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:00:19.026 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.376628+0000", 2023-09-23T21:00:19.026 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.5759000000000001e-05 2023-09-23T21:00:19.026 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:19.026 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:19.026 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:00:19.027 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.376671+0000", 2023-09-23T21:00:19.027 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.2969999999999997e-05 2023-09-23T21:00:19.027 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:19.027 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:19.027 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:00:19.027 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.376709+0000", 2023-09-23T21:00:19.027 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.7540000000000003e-05 2023-09-23T21:00:19.027 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:00:19.027 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:00:19.027 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:00:19.027 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:19.028 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:19.028 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.34797.0:156990 6.12 6:491f6f4b:::benchmark_data_smithi026_223868_object55864:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e150)", 2023-09-23T21:00:19.028 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T20:55:07.581755+0000", 2023-09-23T21:00:19.028 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 311.40106419599999, 2023-09-23T21:00:19.028 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.79863560899999997, 2023-09-23T21:00:19.028 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:00:19.028 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:00:19.028 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:00:19.028 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.34797", 2023-09-23T21:00:19.028 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T21:00:19.028 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 156990 2023-09-23T21:00:19.028 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:19.029 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:00:19.029 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:19.029 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:00:19.029 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:07.581755+0000", 2023-09-23T21:00:19.029 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:00:19.029 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:19.029 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:19.029 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:00:19.029 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:07.581755+0000", 2023-09-23T21:00:19.029 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:00:19.029 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:19.029 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:19.030 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:00:19.030 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:07.581754+0000", 2023-09-23T21:00:19.030 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T21:00:19.030 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:19.030 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:19.030 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:00:19.030 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:07.581760+0000", 2023-09-23T21:00:19.030 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.02e-06 2023-09-23T21:00:19.030 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:19.030 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:19.030 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:00:19.030 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:07.581773+0000", 2023-09-23T21:00:19.031 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.2921e-05 2023-09-23T21:00:19.031 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:19.031 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:19.031 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:00:19.031 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:07.581779+0000", 2023-09-23T21:00:19.031 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.7889999999999997e-06 2023-09-23T21:00:19.031 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:19.031 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:19.031 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:00:19.031 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:07.581838+0000", 2023-09-23T21:00:19.031 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.8850000000000001e-05 2023-09-23T21:00:19.031 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:19.032 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:19.032 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:00:19.032 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:07.581848+0000", 2023-09-23T21:00:19.032 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.9340000000000001e-06 2023-09-23T21:00:19.032 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:19.032 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:19.032 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:00:19.032 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.368883+0000", 2023-09-23T21:00:19.032 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.78703539700000003 2023-09-23T21:00:19.032 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:19.032 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:19.032 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:00:19.033 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.368984+0000", 2023-09-23T21:00:19.033 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000100717 2023-09-23T21:00:19.033 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:19.033 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:19.033 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 2,3", 2023-09-23T21:00:19.033 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.369202+0000", 2023-09-23T21:00:19.033 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000218304 2023-09-23T21:00:19.033 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:19.033 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:19.033 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:00:19.033 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.372876+0000", 2023-09-23T21:00:19.034 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.003673511 2023-09-23T21:00:19.034 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:19.034 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:19.034 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:00:19.034 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.380243+0000", 2023-09-23T21:00:19.034 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0073674049999999996 2023-09-23T21:00:19.034 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:19.034 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:19.034 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:00:19.034 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.380308+0000", 2023-09-23T21:00:19.034 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.4309999999999999e-05 2023-09-23T21:00:19.034 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:19.035 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:19.035 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:00:19.035 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.380357+0000", 2023-09-23T21:00:19.035 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.9095999999999999e-05 2023-09-23T21:00:19.035 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:19.035 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:19.035 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:00:19.035 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.380391+0000", 2023-09-23T21:00:19.035 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.3850999999999998e-05 2023-09-23T21:00:19.035 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:00:19.035 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:00:19.035 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:00:19.036 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:19.036 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:19.036 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.34797.0:157031 6.12 6:48e5af49:::benchmark_data_smithi026_223868_object55905:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e150)", 2023-09-23T21:00:19.036 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T20:55:07.628542+0000", 2023-09-23T21:00:19.036 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 311.354276884, 2023-09-23T21:00:19.036 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.75194783899999995, 2023-09-23T21:00:19.036 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:00:19.036 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:00:19.036 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:00:19.036 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.34797", 2023-09-23T21:00:19.036 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T21:00:19.036 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 157031 2023-09-23T21:00:19.037 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:19.037 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:00:19.037 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:19.037 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:00:19.037 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:07.628542+0000", 2023-09-23T21:00:19.037 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:00:19.037 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:19.037 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:19.037 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:00:19.037 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:07.628542+0000", 2023-09-23T21:00:19.037 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:00:19.037 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:19.038 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:19.038 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:00:19.038 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:07.628541+0000", 2023-09-23T21:00:19.038 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.999999 2023-09-23T21:00:19.038 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:19.038 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:19.038 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:00:19.038 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:07.628549+0000", 2023-09-23T21:00:19.038 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.9019999999999999e-06 2023-09-23T21:00:19.038 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:19.038 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:19.039 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:00:19.039 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:07.628562+0000", 2023-09-23T21:00:19.039 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.2986999999999999e-05 2023-09-23T21:00:19.039 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:19.039 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:19.039 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:00:19.039 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:07.628567+0000", 2023-09-23T21:00:19.039 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.322e-06 2023-09-23T21:00:19.039 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:19.039 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:19.039 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:00:19.039 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:07.628615+0000", 2023-09-23T21:00:19.040 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.7564000000000001e-05 2023-09-23T21:00:19.040 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:19.040 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:19.040 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:00:19.040 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:07.628622+0000", 2023-09-23T21:00:19.040 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.6440000000000003e-06 2023-09-23T21:00:19.040 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:19.040 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:19.040 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:00:19.040 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.370771+0000", 2023-09-23T21:00:19.040 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.74214981700000004 2023-09-23T21:00:19.041 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:19.041 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:19.041 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:00:19.041 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.370876+0000", 2023-09-23T21:00:19.041 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00010476000000000001 2023-09-23T21:00:19.041 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:19.041 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:19.041 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 2,3", 2023-09-23T21:00:19.041 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.371134+0000", 2023-09-23T21:00:19.041 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00025788100000000001 2023-09-23T21:00:19.041 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:19.041 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:19.042 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:00:19.042 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.373009+0000", 2023-09-23T21:00:19.042 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.001875334 2023-09-23T21:00:19.042 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:19.042 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:19.042 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:00:19.042 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.380281+0000", 2023-09-23T21:00:19.042 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0072716710000000004 2023-09-23T21:00:19.042 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:19.042 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:19.042 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:00:19.043 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.380422+0000", 2023-09-23T21:00:19.043 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00014123299999999999 2023-09-23T21:00:19.043 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:19.043 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:19.043 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:00:19.043 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.380460+0000", 2023-09-23T21:00:19.043 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.7372000000000001e-05 2023-09-23T21:00:19.043 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:19.043 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:19.043 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:00:19.044 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.380490+0000", 2023-09-23T21:00:19.044 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.0493000000000001e-05 2023-09-23T21:00:19.044 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:00:19.044 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:00:19.044 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:00:19.044 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:19.044 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:19.044 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.34797.0:157050 6.12 6:4be05e13:::benchmark_data_smithi026_223868_object55924:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e150)", 2023-09-23T21:00:19.044 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T20:55:07.649612+0000", 2023-09-23T21:00:19.044 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 311.33320768499999, 2023-09-23T21:00:19.045 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.73668807700000005, 2023-09-23T21:00:19.045 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:00:19.045 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:00:19.045 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:00:19.045 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.34797", 2023-09-23T21:00:19.045 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T21:00:19.045 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 157050 2023-09-23T21:00:19.045 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:19.045 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:00:19.045 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:19.045 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:00:19.046 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:07.649612+0000", 2023-09-23T21:00:19.046 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:00:19.046 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:19.046 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:19.046 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:00:19.046 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:07.649612+0000", 2023-09-23T21:00:19.046 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:00:19.046 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:19.046 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:19.047 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:00:19.047 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:07.649610+0000", 2023-09-23T21:00:19.047 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T21:00:19.047 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:19.047 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:19.047 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:00:19.048 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:07.649619+0000", 2023-09-23T21:00:19.048 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.7439999999999996e-06 2023-09-23T21:00:19.048 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:19.048 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:19.048 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:00:19.048 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:07.649633+0000", 2023-09-23T21:00:19.048 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.4056000000000001e-05 2023-09-23T21:00:19.048 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:19.048 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:19.049 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:00:19.049 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:07.649641+0000", 2023-09-23T21:00:19.049 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.5290000000000003e-06 2023-09-23T21:00:19.049 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:19.049 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:19.049 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:00:19.049 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:07.649701+0000", 2023-09-23T21:00:19.050 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.0715999999999999e-05 2023-09-23T21:00:19.050 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:19.050 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:19.050 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:00:19.050 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:07.649711+0000", 2023-09-23T21:00:19.051 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.5529999999999998e-06 2023-09-23T21:00:19.051 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:19.051 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:19.051 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:00:19.051 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.373038+0000", 2023-09-23T21:00:19.051 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.72332733999999999 2023-09-23T21:00:19.051 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:19.051 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:19.052 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:00:19.052 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.373159+0000", 2023-09-23T21:00:19.052 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000120704 2023-09-23T21:00:19.052 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:19.052 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:19.052 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 2,3", 2023-09-23T21:00:19.052 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.373420+0000", 2023-09-23T21:00:19.052 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00026133599999999998 2023-09-23T21:00:19.052 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:19.052 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:19.053 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:00:19.053 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.376599+0000", 2023-09-23T21:00:19.053 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0031784790000000001 2023-09-23T21:00:19.053 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:19.053 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:19.053 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:00:19.053 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.380519+0000", 2023-09-23T21:00:19.053 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0039201569999999996 2023-09-23T21:00:19.053 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:19.054 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:19.054 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:00:19.054 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.386243+0000", 2023-09-23T21:00:19.054 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0057240770000000002 2023-09-23T21:00:19.054 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:19.054 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:19.054 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:00:19.054 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.386271+0000", 2023-09-23T21:00:19.055 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.8518e-05 2023-09-23T21:00:19.055 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:19.055 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:19.055 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:00:19.055 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.386300+0000", 2023-09-23T21:00:19.055 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.8211000000000001e-05 2023-09-23T21:00:19.055 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:00:19.055 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:00:19.055 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:00:19.055 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:19.055 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:19.055 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.34797.0:157117 6.12 6:4d4c4893:::benchmark_data_smithi026_223868_object55991:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e150)", 2023-09-23T21:00:19.056 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T20:55:07.747668+0000", 2023-09-23T21:00:19.056 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 311.23515091799999, 2023-09-23T21:00:19.056 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.63872341099999996, 2023-09-23T21:00:19.056 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:00:19.056 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:00:19.056 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:00:19.056 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.34797", 2023-09-23T21:00:19.056 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T21:00:19.056 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 157117 2023-09-23T21:00:19.056 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:19.056 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:00:19.057 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:19.057 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:00:19.057 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:07.747668+0000", 2023-09-23T21:00:19.057 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:00:19.057 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:19.057 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:19.057 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:00:19.057 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:07.747668+0000", 2023-09-23T21:00:19.057 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:00:19.057 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:19.057 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:19.057 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:00:19.058 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:07.747667+0000", 2023-09-23T21:00:19.058 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T21:00:19.058 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:19.058 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:19.058 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:00:19.058 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:07.747673+0000", 2023-09-23T21:00:19.058 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.2580000000000001e-06 2023-09-23T21:00:19.058 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:19.058 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:19.058 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:00:19.058 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:07.747689+0000", 2023-09-23T21:00:19.059 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.5837000000000001e-05 2023-09-23T21:00:19.059 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:19.059 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:19.059 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:00:19.059 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:07.747698+0000", 2023-09-23T21:00:19.059 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.6039999999999993e-06 2023-09-23T21:00:19.059 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:19.059 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:19.059 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:00:19.059 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:07.747754+0000", 2023-09-23T21:00:19.059 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.6487000000000003e-05 2023-09-23T21:00:19.059 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:19.060 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:19.060 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:00:19.060 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:07.747764+0000", 2023-09-23T21:00:19.060 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.0108e-05 2023-09-23T21:00:19.060 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:19.060 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:19.060 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:00:19.060 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.374677+0000", 2023-09-23T21:00:19.060 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.62691302900000001 2023-09-23T21:00:19.060 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:19.060 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:19.060 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:00:19.060 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.374761+0000", 2023-09-23T21:00:19.061 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.3807999999999994e-05 2023-09-23T21:00:19.061 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:19.061 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:19.061 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 2,3", 2023-09-23T21:00:19.061 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.374943+0000", 2023-09-23T21:00:19.061 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00018220399999999999 2023-09-23T21:00:19.061 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:19.061 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:19.061 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:00:19.061 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.378533+0000", 2023-09-23T21:00:19.061 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0035901190000000001 2023-09-23T21:00:19.061 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:19.062 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:19.062 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:00:19.062 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.380547+0000", 2023-09-23T21:00:19.062 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.002013379 2023-09-23T21:00:19.062 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:19.062 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:19.062 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:00:19.062 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.386342+0000", 2023-09-23T21:00:19.062 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0057952480000000002 2023-09-23T21:00:19.062 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:19.062 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:19.063 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:00:19.063 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.386365+0000", 2023-09-23T21:00:19.063 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.3411e-05 2023-09-23T21:00:19.063 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:19.063 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:19.063 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:00:19.063 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.386392+0000", 2023-09-23T21:00:19.063 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.6301000000000001e-05 2023-09-23T21:00:19.063 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:00:19.063 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:00:19.063 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:00:19.063 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:19.064 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:19.064 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.34797.0:157161 6.12 6:4e589f92:::benchmark_data_smithi026_223868_object56035:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e150)", 2023-09-23T21:00:19.064 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T20:55:07.800631+0000", 2023-09-23T21:00:19.064 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 311.18218832299999, 2023-09-23T21:00:19.064 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.58585551000000002, 2023-09-23T21:00:19.064 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:00:19.064 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:00:19.064 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:00:19.064 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.34797", 2023-09-23T21:00:19.064 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T21:00:19.064 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 157161 2023-09-23T21:00:19.064 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:19.065 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:00:19.065 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:19.065 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:00:19.065 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:07.800631+0000", 2023-09-23T21:00:19.065 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:00:19.065 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:19.065 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:19.065 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:00:19.065 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:07.800631+0000", 2023-09-23T21:00:19.065 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:00:19.065 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:19.065 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:19.066 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:00:19.066 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:07.800630+0000", 2023-09-23T21:00:19.066 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T21:00:19.066 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:19.066 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:19.066 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:00:19.066 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:07.800636+0000", 2023-09-23T21:00:19.066 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.9100000000000002e-06 2023-09-23T21:00:19.066 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:19.066 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:19.066 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:00:19.067 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:07.800650+0000", 2023-09-23T21:00:19.067 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.4409999999999999e-05 2023-09-23T21:00:19.067 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:19.067 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:19.067 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:00:19.067 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:07.800657+0000", 2023-09-23T21:00:19.067 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.0829999999999998e-06 2023-09-23T21:00:19.067 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:19.067 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:19.067 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:00:19.067 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:07.800709+0000", 2023-09-23T21:00:19.067 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.1907999999999998e-05 2023-09-23T21:00:19.068 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:19.068 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:19.068 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:00:19.068 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:07.800721+0000", 2023-09-23T21:00:19.068 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.2529e-05 2023-09-23T21:00:19.068 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:19.068 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:19.068 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:00:19.068 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.376733+0000", 2023-09-23T21:00:19.068 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.57601174499999996 2023-09-23T21:00:19.068 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:19.068 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:19.069 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:00:19.069 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.376824+0000", 2023-09-23T21:00:19.069 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.1108999999999998e-05 2023-09-23T21:00:19.069 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:19.069 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:19.069 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 2,3", 2023-09-23T21:00:19.069 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.377089+0000", 2023-09-23T21:00:19.069 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000265128 2023-09-23T21:00:19.069 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:19.069 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:19.069 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:00:19.069 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.380251+0000", 2023-09-23T21:00:19.070 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0031611230000000001 2023-09-23T21:00:19.070 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:19.070 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:19.070 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:00:19.070 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.386220+0000", 2023-09-23T21:00:19.070 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0059696009999999997 2023-09-23T21:00:19.070 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:19.070 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:19.070 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:00:19.070 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.386427+0000", 2023-09-23T21:00:19.070 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000206951 2023-09-23T21:00:19.070 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:19.071 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:19.071 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:00:19.071 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.386466+0000", 2023-09-23T21:00:19.071 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.8874000000000001e-05 2023-09-23T21:00:19.071 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:19.071 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:19.071 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:00:19.071 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.386486+0000", 2023-09-23T21:00:19.071 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.0457000000000001e-05 2023-09-23T21:00:19.071 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:00:19.071 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:00:19.071 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:00:19.072 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:19.072 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:19.072 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.34797.0:157227 6.12 6:4d4ca5fd:::benchmark_data_smithi026_223868_object56101:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e150)", 2023-09-23T21:00:19.072 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T20:55:07.900387+0000", 2023-09-23T21:00:19.072 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 311.08243204799999, 2023-09-23T21:00:19.072 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.49162882600000002, 2023-09-23T21:00:19.072 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:00:19.072 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:00:19.072 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:00:19.072 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.34797", 2023-09-23T21:00:19.072 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T21:00:19.072 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 157227 2023-09-23T21:00:19.073 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:19.073 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:00:19.073 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:19.073 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:00:19.073 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:07.900387+0000", 2023-09-23T21:00:19.073 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:00:19.073 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:19.073 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:19.073 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:00:19.073 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:07.900387+0000", 2023-09-23T21:00:19.074 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:00:19.074 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:19.074 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:19.074 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:00:19.074 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:07.900386+0000", 2023-09-23T21:00:19.074 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T21:00:19.074 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:19.074 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:19.074 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:00:19.074 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:07.900392+0000", 2023-09-23T21:00:19.074 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.5899999999999996e-06 2023-09-23T21:00:19.075 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:19.075 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:19.075 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:00:19.075 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:07.900408+0000", 2023-09-23T21:00:19.075 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.5430999999999998e-05 2023-09-23T21:00:19.075 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:19.075 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:19.075 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:00:19.076 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:07.900415+0000", 2023-09-23T21:00:19.076 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.7739999999999999e-06 2023-09-23T21:00:19.076 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:19.076 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:19.076 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:00:19.076 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:07.900476+0000", 2023-09-23T21:00:19.076 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.1452999999999996e-05 2023-09-23T21:00:19.076 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:19.076 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:19.076 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:00:19.077 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:07.900485+0000", 2023-09-23T21:00:19.077 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.3260000000000002e-06 2023-09-23T21:00:19.077 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:19.077 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:19.077 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:00:19.077 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.378561+0000", 2023-09-23T21:00:19.077 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.47807563199999997 2023-09-23T21:00:19.077 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:19.077 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:19.077 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:00:19.077 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.378655+0000", 2023-09-23T21:00:19.078 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.3793000000000001e-05 2023-09-23T21:00:19.078 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:19.078 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:19.078 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 2,3", 2023-09-23T21:00:19.078 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.378841+0000", 2023-09-23T21:00:19.078 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00018573600000000001 2023-09-23T21:00:19.078 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:19.078 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:19.078 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:00:19.078 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.386326+0000", 2023-09-23T21:00:19.078 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0074849620000000004 2023-09-23T21:00:19.079 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:19.079 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:19.079 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:00:19.079 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.386517+0000", 2023-09-23T21:00:19.079 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00019191999999999999 2023-09-23T21:00:19.079 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:19.079 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:19.079 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:00:19.079 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.391931+0000", 2023-09-23T21:00:19.079 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0054132310000000001 2023-09-23T21:00:19.079 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:19.080 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:19.080 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:00:19.080 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.391976+0000", 2023-09-23T21:00:19.080 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.5697e-05 2023-09-23T21:00:19.080 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:19.080 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:19.080 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:00:19.080 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.392016+0000", 2023-09-23T21:00:19.080 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.9644000000000002e-05 2023-09-23T21:00:19.081 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:00:19.081 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:00:19.081 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:00:19.081 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:19.081 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:19.081 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.34797.0:157259 6.12 6:4e9f8804:::benchmark_data_smithi026_223868_object56133:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e150)", 2023-09-23T21:00:19.081 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T20:55:07.947142+0000", 2023-09-23T21:00:19.081 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 311.03567681599998, 2023-09-23T21:00:19.081 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.44774977900000001, 2023-09-23T21:00:19.081 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:00:19.081 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:00:19.082 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:00:19.082 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.34797", 2023-09-23T21:00:19.082 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T21:00:19.082 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 157259 2023-09-23T21:00:19.082 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:19.082 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:00:19.082 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:19.082 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:00:19.082 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:07.947142+0000", 2023-09-23T21:00:19.082 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:00:19.082 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:19.083 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:19.083 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:00:19.083 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:07.947142+0000", 2023-09-23T21:00:19.083 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:00:19.083 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:19.083 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:19.083 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:00:19.083 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:07.947142+0000", 2023-09-23T21:00:19.083 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999995 2023-09-23T21:00:19.083 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:19.083 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:19.084 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:00:19.084 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:07.947146+0000", 2023-09-23T21:00:19.084 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.0500000000000002e-06 2023-09-23T21:00:19.084 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:19.084 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:19.084 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:00:19.084 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:07.947160+0000", 2023-09-23T21:00:19.084 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.4662e-05 2023-09-23T21:00:19.084 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:19.084 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:19.084 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:00:19.084 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:07.947166+0000", 2023-09-23T21:00:19.085 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.784e-06 2023-09-23T21:00:19.085 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:19.085 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:19.085 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:00:19.085 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:07.947223+0000", 2023-09-23T21:00:19.085 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.6854000000000001e-05 2023-09-23T21:00:19.085 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:19.085 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:19.085 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:00:19.085 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:07.947238+0000", 2023-09-23T21:00:19.085 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.4952e-05 2023-09-23T21:00:19.086 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:19.086 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:19.086 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:00:19.086 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.380572+0000", 2023-09-23T21:00:19.086 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.43333436400000003 2023-09-23T21:00:19.086 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:19.086 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:19.086 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:00:19.086 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.380689+0000", 2023-09-23T21:00:19.086 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00011646600000000001 2023-09-23T21:00:19.086 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:19.087 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:19.087 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 2,3", 2023-09-23T21:00:19.087 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.380934+0000", 2023-09-23T21:00:19.087 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00024519199999999998 2023-09-23T21:00:19.087 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:19.087 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:19.087 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:00:19.087 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.390072+0000", 2023-09-23T21:00:19.087 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0091384410000000006 2023-09-23T21:00:19.087 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:19.087 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:19.087 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:00:19.088 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.392023+0000", 2023-09-23T21:00:19.088 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0019504170000000001 2023-09-23T21:00:19.088 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:19.088 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:19.088 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:00:19.088 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.394809+0000", 2023-09-23T21:00:19.088 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0027863760000000001 2023-09-23T21:00:19.088 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:19.088 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:19.088 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:00:19.088 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.394852+0000", 2023-09-23T21:00:19.088 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.3130999999999998e-05 2023-09-23T21:00:19.089 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:19.089 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:19.089 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:00:19.089 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.394892+0000", 2023-09-23T21:00:19.089 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.9801000000000001e-05 2023-09-23T21:00:19.089 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:00:19.089 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:00:19.089 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:00:19.089 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:19.089 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:19.089 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.34797.0:157328 6.12 6:4811798e:::benchmark_data_smithi026_223868_object56202:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e150)", 2023-09-23T21:00:19.089 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T20:55:08.060527+0000", 2023-09-23T21:00:19.090 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 310.92229208999998, 2023-09-23T21:00:19.090 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.33450326899999999, 2023-09-23T21:00:19.090 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:00:19.090 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:00:19.090 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:00:19.090 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.34797", 2023-09-23T21:00:19.090 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T21:00:19.090 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 157328 2023-09-23T21:00:19.090 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:19.090 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:00:19.091 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:19.091 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:00:19.091 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.060527+0000", 2023-09-23T21:00:19.091 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:00:19.091 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:19.091 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:19.091 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:00:19.091 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.060527+0000", 2023-09-23T21:00:19.091 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:00:19.091 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:19.091 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:19.091 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:00:19.092 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.060526+0000", 2023-09-23T21:00:19.092 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.999999 2023-09-23T21:00:19.092 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:19.092 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:19.092 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:00:19.092 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.060531+0000", 2023-09-23T21:00:19.092 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.25e-06 2023-09-23T21:00:19.092 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:19.092 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:19.092 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:00:19.092 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.060540+0000", 2023-09-23T21:00:19.093 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.0259999999999996e-06 2023-09-23T21:00:19.093 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:19.093 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:19.093 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:00:19.093 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.060544+0000", 2023-09-23T21:00:19.093 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.1370000000000004e-06 2023-09-23T21:00:19.093 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:19.093 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:19.093 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:00:19.093 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.060605+0000", 2023-09-23T21:00:19.093 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.0939e-05 2023-09-23T21:00:19.093 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:19.094 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:19.094 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:00:19.094 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.060616+0000", 2023-09-23T21:00:19.094 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.1824999999999999e-05 2023-09-23T21:00:19.094 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:19.094 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:19.094 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:00:19.094 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.382773+0000", 2023-09-23T21:00:19.094 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.32215636800000003 2023-09-23T21:00:19.094 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:19.094 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:19.095 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:00:19.095 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.382925+0000", 2023-09-23T21:00:19.095 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00015193499999999999 2023-09-23T21:00:19.095 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:19.095 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:19.095 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 2,3", 2023-09-23T21:00:19.095 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.383267+0000", 2023-09-23T21:00:19.095 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00034205500000000002 2023-09-23T21:00:19.095 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:19.095 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:19.095 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:00:19.095 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.390103+0000", 2023-09-23T21:00:19.096 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0068357510000000002 2023-09-23T21:00:19.096 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:19.096 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:19.096 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:00:19.096 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.392028+0000", 2023-09-23T21:00:19.096 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.001925775 2023-09-23T21:00:19.096 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:19.096 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:19.096 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:00:19.096 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.394957+0000", 2023-09-23T21:00:19.096 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0029286030000000001 2023-09-23T21:00:19.096 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:19.096 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:19.097 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:00:19.097 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.394996+0000", 2023-09-23T21:00:19.097 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.9410999999999999e-05 2023-09-23T21:00:19.097 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:19.097 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:19.097 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:00:19.097 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.395030+0000", 2023-09-23T21:00:19.097 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.4112000000000001e-05 2023-09-23T21:00:19.097 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:00:19.097 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:00:19.097 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:00:19.097 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:19.098 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:19.098 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.34797.0:157346 6.12 6:4c4003c4:::benchmark_data_smithi026_223868_object56220:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e150)", 2023-09-23T21:00:19.098 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T20:55:08.089574+0000", 2023-09-23T21:00:19.098 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 310.89324564600003, 2023-09-23T21:00:19.098 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.305631714, 2023-09-23T21:00:19.098 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:00:19.098 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:00:19.098 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:00:19.098 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.34797", 2023-09-23T21:00:19.098 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T21:00:19.098 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 157346 2023-09-23T21:00:19.099 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:19.099 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:00:19.099 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:19.099 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:00:19.099 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.089574+0000", 2023-09-23T21:00:19.099 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:00:19.099 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:19.099 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:19.099 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:00:19.099 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.089574+0000", 2023-09-23T21:00:19.099 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:00:19.099 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:19.100 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:19.100 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:00:19.100 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.089572+0000", 2023-09-23T21:00:19.100 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999981 2023-09-23T21:00:19.100 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:19.100 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:19.100 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:00:19.100 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.089579+0000", 2023-09-23T21:00:19.100 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.6290000000000001e-06 2023-09-23T21:00:19.100 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:19.100 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:19.100 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:00:19.100 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.089596+0000", 2023-09-23T21:00:19.101 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.6511000000000002e-05 2023-09-23T21:00:19.101 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:19.101 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:19.101 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:00:19.101 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.089605+0000", 2023-09-23T21:00:19.101 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.5390000000000007e-06 2023-09-23T21:00:19.101 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:19.101 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:19.101 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:00:19.101 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.089665+0000", 2023-09-23T21:00:19.101 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.0859000000000001e-05 2023-09-23T21:00:19.102 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:19.102 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:19.102 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:00:19.102 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.089678+0000", 2023-09-23T21:00:19.102 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.2133000000000001e-05 2023-09-23T21:00:19.102 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:19.102 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:19.102 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:00:19.102 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.384720+0000", 2023-09-23T21:00:19.102 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.29504224499999998 2023-09-23T21:00:19.102 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:19.102 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:19.103 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:00:19.103 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.384814+0000", 2023-09-23T21:00:19.103 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.4611999999999995e-05 2023-09-23T21:00:19.103 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:19.103 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:19.103 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 2,3", 2023-09-23T21:00:19.103 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.385022+0000", 2023-09-23T21:00:19.103 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00020736999999999999 2023-09-23T21:00:19.103 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:19.103 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:19.103 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:00:19.103 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.392033+0000", 2023-09-23T21:00:19.104 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0070114649999999997 2023-09-23T21:00:19.104 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:19.104 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:19.104 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:00:19.104 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.395074+0000", 2023-09-23T21:00:19.104 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.003040297 2023-09-23T21:00:19.104 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:19.104 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:19.104 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:00:19.104 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.395133+0000", 2023-09-23T21:00:19.104 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.9700000000000001e-05 2023-09-23T21:00:19.104 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:19.105 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:19.105 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:00:19.105 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.395173+0000", 2023-09-23T21:00:19.105 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.9956999999999997e-05 2023-09-23T21:00:19.105 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:19.105 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:19.105 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:00:19.105 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.395205+0000", 2023-09-23T21:00:19.105 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.2146e-05 2023-09-23T21:00:19.105 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:00:19.105 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:00:19.105 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:00:19.105 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:19.106 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:19.106 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.34797.0:157407 6.12 6:494044ed:::benchmark_data_smithi026_223868_object56281:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e150)", 2023-09-23T21:00:19.106 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T20:55:08.202324+0000", 2023-09-23T21:00:19.106 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 310.78049550600002, 2023-09-23T21:00:19.106 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.19300698499999999, 2023-09-23T21:00:19.106 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:00:19.106 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:00:19.106 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:00:19.106 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.34797", 2023-09-23T21:00:19.106 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T21:00:19.106 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 157407 2023-09-23T21:00:19.107 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:19.107 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:00:19.107 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:19.107 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:00:19.107 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.202324+0000", 2023-09-23T21:00:19.107 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:00:19.107 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:19.107 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:19.107 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:00:19.107 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.202324+0000", 2023-09-23T21:00:19.107 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:00:19.107 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:19.108 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:19.108 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:00:19.108 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.202322+0000", 2023-09-23T21:00:19.108 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T21:00:19.108 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:19.108 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:19.108 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:00:19.108 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.202330+0000", 2023-09-23T21:00:19.108 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.8730000000000007e-06 2023-09-23T21:00:19.108 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:19.108 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:19.108 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:00:19.108 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.202346+0000", 2023-09-23T21:00:19.109 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.5807e-05 2023-09-23T21:00:19.109 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:19.109 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:19.109 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:00:19.109 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.202352+0000", 2023-09-23T21:00:19.109 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.4609999999999996e-06 2023-09-23T21:00:19.109 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:19.109 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:19.109 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:00:19.109 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.202412+0000", 2023-09-23T21:00:19.109 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.9930999999999999e-05 2023-09-23T21:00:19.109 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:19.110 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:19.110 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:00:19.110 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.202425+0000", 2023-09-23T21:00:19.110 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.2272e-05 2023-09-23T21:00:19.110 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:19.110 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:19.110 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:00:19.110 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.386530+0000", 2023-09-23T21:00:19.110 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.18410531799999999 2023-09-23T21:00:19.110 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:19.110 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:19.111 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:00:19.111 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.386605+0000", 2023-09-23T21:00:19.111 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.4880000000000001e-05 2023-09-23T21:00:19.111 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:19.111 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:19.111 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 2,3", 2023-09-23T21:00:19.111 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.386802+0000", 2023-09-23T21:00:19.111 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00019716199999999999 2023-09-23T21:00:19.111 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:19.111 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:19.111 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:00:19.111 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.392038+0000", 2023-09-23T21:00:19.112 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.005235941 2023-09-23T21:00:19.112 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:19.112 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:19.112 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:00:19.112 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.395108+0000", 2023-09-23T21:00:19.112 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0030700570000000002 2023-09-23T21:00:19.112 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:19.112 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:19.112 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:00:19.112 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.395266+0000", 2023-09-23T21:00:19.112 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00015852499999999999 2023-09-23T21:00:19.113 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:19.113 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:19.113 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:00:19.113 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.395302+0000", 2023-09-23T21:00:19.113 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.5374999999999999e-05 2023-09-23T21:00:19.113 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:19.113 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:19.113 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:00:19.113 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.395331+0000", 2023-09-23T21:00:19.113 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.8892999999999999e-05 2023-09-23T21:00:19.113 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:00:19.114 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:00:19.114 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:00:19.114 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:19.114 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:19.114 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.34797.0:157436 6.12 6:4be31d9d:::benchmark_data_smithi026_223868_object56310:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e150)", 2023-09-23T21:00:19.114 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T20:55:08.273211+0000", 2023-09-23T21:00:19.114 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 310.70960811100002, 2023-09-23T21:00:19.114 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.12224776599999999, 2023-09-23T21:00:19.114 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:00:19.114 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:00:19.114 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:00:19.115 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.34797", 2023-09-23T21:00:19.115 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T21:00:19.115 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 157436 2023-09-23T21:00:19.115 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:19.115 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:00:19.115 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:19.115 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:00:19.115 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.273211+0000", 2023-09-23T21:00:19.115 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:00:19.115 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:19.115 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:19.115 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:00:19.116 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.273211+0000", 2023-09-23T21:00:19.116 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:00:19.116 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:19.116 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:19.116 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:00:19.116 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.273210+0000", 2023-09-23T21:00:19.116 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.999999 2023-09-23T21:00:19.116 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:19.116 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:19.116 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:00:19.116 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.273214+0000", 2023-09-23T21:00:19.117 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.1080000000000003e-06 2023-09-23T21:00:19.117 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:19.117 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:19.117 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:00:19.117 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.273226+0000", 2023-09-23T21:00:19.117 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.168e-05 2023-09-23T21:00:19.117 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:19.117 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:19.117 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:00:19.117 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.273231+0000", 2023-09-23T21:00:19.117 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.6670000000000004e-06 2023-09-23T21:00:19.117 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:19.118 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:19.118 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:00:19.118 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.273295+0000", 2023-09-23T21:00:19.118 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.4342000000000002e-05 2023-09-23T21:00:19.118 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:19.118 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:19.118 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:00:19.118 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.273305+0000", 2023-09-23T21:00:19.118 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.0458e-05 2023-09-23T21:00:19.118 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:19.118 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:19.119 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:00:19.119 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.388109+0000", 2023-09-23T21:00:19.119 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.114803402 2023-09-23T21:00:19.119 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:19.119 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:19.119 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:00:19.119 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.388202+0000", 2023-09-23T21:00:19.119 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.3173999999999999e-05 2023-09-23T21:00:19.119 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:19.119 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:19.119 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 2,3", 2023-09-23T21:00:19.120 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.388429+0000", 2023-09-23T21:00:19.120 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00022721899999999999 2023-09-23T21:00:19.120 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:19.120 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:19.120 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:00:19.120 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.392043+0000", 2023-09-23T21:00:19.120 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0036135849999999999 2023-09-23T21:00:19.120 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:19.120 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:19.120 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:00:19.120 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.395241+0000", 2023-09-23T21:00:19.120 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0031982820000000002 2023-09-23T21:00:19.120 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:19.121 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:19.121 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:00:19.121 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.395384+0000", 2023-09-23T21:00:19.121 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00014287100000000001 2023-09-23T21:00:19.121 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:19.121 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:19.121 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:00:19.121 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.395420+0000", 2023-09-23T21:00:19.121 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.6174000000000003e-05 2023-09-23T21:00:19.121 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:19.121 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:19.121 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:00:19.121 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.395459+0000", 2023-09-23T21:00:19.122 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.8887999999999998e-05 2023-09-23T21:00:19.122 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:00:19.122 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:00:19.122 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:00:19.122 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:19.122 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:19.122 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.35100.0:94301 7.1 7:8f95fbda:::benchmark_data_smithi026_260489_object94300:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e228)", 2023-09-23T21:00:19.122 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T20:57:57.540024+0000", 2023-09-23T21:00:19.122 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 141.44279481999999, 2023-09-23T21:00:19.122 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.14579631700000001, 2023-09-23T21:00:19.122 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:00:19.123 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:00:19.123 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:00:19.123 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.35100", 2023-09-23T21:00:19.123 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/4108972961", 2023-09-23T21:00:19.123 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 94301 2023-09-23T21:00:19.123 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:19.123 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:00:19.123 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:19.123 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:00:19.123 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:57:57.540024+0000", 2023-09-23T21:00:19.123 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:00:19.123 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:19.124 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:19.124 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:00:19.124 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:57:57.540024+0000", 2023-09-23T21:00:19.124 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:00:19.124 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:19.124 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:19.124 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:00:19.124 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:57:57.540024+0000", 2023-09-23T21:00:19.124 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999995 2023-09-23T21:00:19.124 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:19.124 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:19.124 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:00:19.125 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:57:57.540060+0000", 2023-09-23T21:00:19.125 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.6013000000000002e-05 2023-09-23T21:00:19.125 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:19.125 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:19.125 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:00:19.125 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:57:57.540067+0000", 2023-09-23T21:00:19.125 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.866e-06 2023-09-23T21:00:19.125 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:19.125 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:19.125 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:00:19.125 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:57:57.540075+0000", 2023-09-23T21:00:19.125 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.7239999999999992e-06 2023-09-23T21:00:19.126 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:19.126 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:19.126 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:00:19.126 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:57:57.549778+0000", 2023-09-23T21:00:19.126 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0097031600000000006 2023-09-23T21:00:19.126 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:19.126 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:19.126 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:00:19.126 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:57:57.549889+0000", 2023-09-23T21:00:19.126 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000110818 2023-09-23T21:00:19.126 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:19.126 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:19.127 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 3,6", 2023-09-23T21:00:19.127 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:57:57.550024+0000", 2023-09-23T21:00:19.127 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00013545899999999999 2023-09-23T21:00:19.127 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:19.127 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:19.127 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:00:19.127 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:57:57.551983+0000", 2023-09-23T21:00:19.127 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.001958871 2023-09-23T21:00:19.127 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:19.127 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:19.127 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:00:19.127 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:57:57.552271+0000", 2023-09-23T21:00:19.128 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00028848799999999999 2023-09-23T21:00:19.128 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:19.128 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:19.128 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:00:19.128 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:57:57.572759+0000", 2023-09-23T21:00:19.128 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.020487701000000001 2023-09-23T21:00:19.128 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:19.128 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:19.128 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:00:19.128 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:57:57.572816+0000", 2023-09-23T21:00:19.128 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.7247999999999998e-05 2023-09-23T21:00:19.129 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:19.129 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:19.129 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:00:19.129 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:57:57.685821+0000", 2023-09-23T21:00:19.129 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.1130044 2023-09-23T21:00:19.129 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:00:19.129 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:00:19.129 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:00:19.129 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:19.129 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:19.129 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_repop(client.35100.0:94315 7.8 e228/199 7:126e5865:::benchmark_data_smithi026_260489_object94314:head v 228'5947, mlcod=228'5944)", 2023-09-23T21:00:19.130 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T20:57:57.570899+0000", 2023-09-23T21:00:19.130 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 141.411920012, 2023-09-23T21:00:19.130 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.11494793, 2023-09-23T21:00:19.130 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:00:19.130 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:00:19.130 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:00:19.130 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:19.131 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:00:19.131 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:57:57.570899+0000", 2023-09-23T21:00:19.131 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:00:19.131 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:19.131 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:19.131 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:00:19.131 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:57:57.570899+0000", 2023-09-23T21:00:19.131 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:00:19.131 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:19.131 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:19.131 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:00:19.132 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:57:57.570899+0000", 2023-09-23T21:00:19.132 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999995 2023-09-23T21:00:19.132 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:19.132 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:19.132 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:00:19.132 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:57:57.571002+0000", 2023-09-23T21:00:19.132 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00010277600000000001 2023-09-23T21:00:19.132 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:19.132 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:19.132 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:00:19.132 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:57:57.571010+0000", 2023-09-23T21:00:19.132 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.9029999999999993e-06 2023-09-23T21:00:19.133 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:19.133 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:19.133 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:00:19.133 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:57:57.571015+0000", 2023-09-23T21:00:19.133 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.702e-06 2023-09-23T21:00:19.133 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:19.133 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:19.133 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:00:19.133 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:57:57.571058+0000", 2023-09-23T21:00:19.133 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.2369e-05 2023-09-23T21:00:19.133 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:19.133 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:19.134 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:00:19.134 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:57:57.571076+0000", 2023-09-23T21:00:19.134 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.8805e-05 2023-09-23T21:00:19.134 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:19.134 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:19.134 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:00:19.134 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:57:57.685810+0000", 2023-09-23T21:00:19.134 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.114733351 2023-09-23T21:00:19.134 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:19.134 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:19.134 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:00:19.135 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:57:57.685847+0000", 2023-09-23T21:00:19.135 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.7502000000000001e-05 2023-09-23T21:00:19.135 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:00:19.135 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:00:19.135 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:00:19.135 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:19.135 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:19.135 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.35100.0:94323 7.1 7:86b69fc0:::benchmark_data_smithi026_260489_object94322:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e228)", 2023-09-23T21:00:19.135 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T20:57:57.573982+0000", 2023-09-23T21:00:19.135 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 141.40883689500001, 2023-09-23T21:00:19.135 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.144034897, 2023-09-23T21:00:19.135 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:00:19.136 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:00:19.136 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:00:19.136 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.35100", 2023-09-23T21:00:19.136 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/4108972961", 2023-09-23T21:00:19.136 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 94323 2023-09-23T21:00:19.136 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:19.136 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:00:19.136 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:19.136 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:00:19.136 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:57:57.573982+0000", 2023-09-23T21:00:19.136 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:00:19.136 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:19.136 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:19.137 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:00:19.137 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:57:57.573982+0000", 2023-09-23T21:00:19.137 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:00:19.137 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:19.137 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:19.137 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:00:19.137 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:57:57.573981+0000", 2023-09-23T21:00:19.137 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T21:00:19.137 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:19.137 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:19.137 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:00:19.137 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:57:57.574033+0000", 2023-09-23T21:00:19.138 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.2389999999999998e-05 2023-09-23T21:00:19.138 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:19.138 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:19.138 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:00:19.138 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:57:57.574044+0000", 2023-09-23T21:00:19.138 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.1079e-05 2023-09-23T21:00:19.138 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:19.138 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:19.138 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:00:19.138 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:57:57.574051+0000", 2023-09-23T21:00:19.138 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.2689999999999996e-06 2023-09-23T21:00:19.138 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:19.139 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:19.139 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:00:19.139 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:57:57.685883+0000", 2023-09-23T21:00:19.139 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.11183256599999999 2023-09-23T21:00:19.139 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:19.139 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:19.139 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:00:19.139 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:57:57.686002+0000", 2023-09-23T21:00:19.139 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00011926600000000001 2023-09-23T21:00:19.139 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:19.139 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:19.140 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 3,6", 2023-09-23T21:00:19.140 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:57:57.686143+0000", 2023-09-23T21:00:19.140 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00014049200000000001 2023-09-23T21:00:19.140 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:19.140 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:19.140 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:00:19.140 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:57:57.692125+0000", 2023-09-23T21:00:19.140 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0059817849999999999 2023-09-23T21:00:19.140 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:19.140 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:19.140 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:00:19.140 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:57:57.697824+0000", 2023-09-23T21:00:19.141 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00569959 2023-09-23T21:00:19.141 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:19.141 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:19.141 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:00:19.141 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:57:57.714832+0000", 2023-09-23T21:00:19.141 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.017007967999999998 2023-09-23T21:00:19.141 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:19.141 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:19.141 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:00:19.141 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:57:57.714872+0000", 2023-09-23T21:00:19.141 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.9638000000000002e-05 2023-09-23T21:00:19.141 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:19.142 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:19.142 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:00:19.142 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:57:57.718017+0000", 2023-09-23T21:00:19.142 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0031452939999999999 2023-09-23T21:00:19.142 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:00:19.142 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:00:19.142 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:00:19.142 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:19.142 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:19.142 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_repop(client.35100.0:94326 7.9 e228/199 7:94da320a:::benchmark_data_smithi026_260489_object94325:head v 228'5898, mlcod=228'5896)", 2023-09-23T21:00:19.142 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T20:57:57.578378+0000", 2023-09-23T21:00:19.143 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 141.40444130700001, 2023-09-23T21:00:19.143 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.11680721299999999, 2023-09-23T21:00:19.143 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:00:19.143 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:00:19.143 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:00:19.143 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:19.143 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:00:19.143 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:57:57.578378+0000", 2023-09-23T21:00:19.143 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:00:19.143 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:19.143 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:19.143 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:00:19.144 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:57:57.578378+0000", 2023-09-23T21:00:19.144 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:00:19.144 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:19.144 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:19.144 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:00:19.144 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:57:57.578377+0000", 2023-09-23T21:00:19.144 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.999999 2023-09-23T21:00:19.144 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:19.144 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:19.144 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:00:19.144 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:57:57.578434+0000", 2023-09-23T21:00:19.144 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.6963999999999999e-05 2023-09-23T21:00:19.145 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:19.145 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:19.145 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:00:19.145 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:57:57.685806+0000", 2023-09-23T21:00:19.145 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.10737245400000001 2023-09-23T21:00:19.145 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:19.145 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:19.145 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:00:19.145 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:57:57.685817+0000", 2023-09-23T21:00:19.145 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.0964e-05 2023-09-23T21:00:19.145 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:19.145 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:19.146 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:00:19.146 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:57:57.685869+0000", 2023-09-23T21:00:19.146 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.1773e-05 2023-09-23T21:00:19.146 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:19.146 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:19.146 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:00:19.146 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:57:57.685890+0000", 2023-09-23T21:00:19.146 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.0489999999999999e-05 2023-09-23T21:00:19.146 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:19.146 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:19.146 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:00:19.147 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:57:57.695157+0000", 2023-09-23T21:00:19.147 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0092676800000000004 2023-09-23T21:00:19.147 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:19.147 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:19.147 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:00:19.147 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:57:57.695185+0000", 2023-09-23T21:00:19.147 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.7773999999999999e-05 2023-09-23T21:00:19.147 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:00:19.147 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:00:19.147 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:00:19.147 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:00:19.147 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:00:19.148 INFO:teuthology.orchestra.run.smithi070.stdout:} 2023-09-23T21:00:19.148 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T21:00:19.149 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:00:19.149 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:00:19.150 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:00:19.251 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:00:19.359 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:00:19.392 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_ops_in_flight 2023-09-23T21:00:19.512 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T21:00:19.525 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:00:19.526 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:00:19.526 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:00:19 smithi026 ceph-mon[179688]: pgmap v620: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 1.6 GiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 577 op/s; 11885/80349 objects degraded (14.792%); 717/80349 objects misplaced (0.892%); 2.1 MiB/s, 33 objects/s recovering 2023-09-23T21:00:19.529 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:00:19 smithi026 ceph-mon[175151]: pgmap v620: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 1.6 GiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 577 op/s; 11885/80349 objects degraded (14.792%); 717/80349 objects misplaced (0.892%); 2.1 MiB/s, 33 objects/s recovering 2023-09-23T21:00:19.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:00:19 smithi070 ceph-mon[168282]: pgmap v620: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 1.6 GiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 577 op/s; 11885/80349 objects degraded (14.792%); 717/80349 objects misplaced (0.892%); 2.1 MiB/s, 33 objects/s recovering 2023-09-23T21:00:19.627 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:00:19.720 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:00:19.793 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:00:19.803 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:00:19.803 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:00:19.806 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 21:00:19 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[164029]: ::ffff:172.21.15.70 - - [23/Sep/2023:21:00:19] "GET /metrics HTTP/1.1" 200 36315 "" "Prometheus/2.43.0" 2023-09-23T21:00:19.904 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:00:19.941 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:00:19.991 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:00:19.994 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:00:19.994 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:00:20.020 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:00:20.090 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T21:00:20.095 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:00:20.136 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T21:00:20.185 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:00:20.307 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:00:20.314 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:00:20.315 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:00:20.374 INFO:teuthology.orchestra.run.smithi026.stderr:nodeep-scrub is set 2023-09-23T21:00:20.416 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:00:20.421 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:00:20 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/4109401934' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:00:20.421 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:00:20 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:00:20.421 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:00:20 smithi026 ceph-mon[175151]: Health check update: Degraded data redundancy: 11885/80349 objects degraded (14.792%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T21:00:20.421 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:00:20 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/4109401934' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:00:20.421 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:00:20 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:00:20.421 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:00:20 smithi026 ceph-mon[179688]: Health check update: Degraded data redundancy: 11885/80349 objects degraded (14.792%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T21:00:20.535 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:00:20.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:00:20 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/4109401934' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:00:20.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:00:20 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:00:20.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:00:20 smithi070 ceph-mon[168282]: Health check update: Degraded data redundancy: 11885/80349 objects degraded (14.792%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T21:00:20.833 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:00:20.838 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:00:20.839 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:00:20.940 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:00:20.978 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:00:21.190 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:00:21.193 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:00:21.193 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:00:21.294 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:00:21.395 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:00:21.463 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:00:21.473 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:00:21.473 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:00:21.542 INFO:teuthology.orchestra.run.smithi070.stdout:{ 2023-09-23T21:00:21.542 INFO:teuthology.orchestra.run.smithi070.stdout: "ops": [ 2023-09-23T21:00:21.542 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:21.542 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.35100.0:180256 7.0 7:00171e72:::benchmark_data_smithi026_260489_object78494:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e246)", 2023-09-23T21:00:21.542 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:00:21.522567+0000", 2023-09-23T21:00:21.542 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 0.019092735999999999, 2023-09-23T21:00:21.542 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.019120231000000001, 2023-09-23T21:00:21.543 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:00:21.543 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "waiting for sub ops", 2023-09-23T21:00:21.543 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:00:21.543 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.35100", 2023-09-23T21:00:21.543 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/4108972961", 2023-09-23T21:00:21.543 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 180256 2023-09-23T21:00:21.543 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:21.543 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:00:21.543 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:21.543 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:00:21.543 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:00:21.522567+0000", 2023-09-23T21:00:21.543 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:00:21.544 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:21.544 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:21.544 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:00:21.544 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:00:21.522567+0000", 2023-09-23T21:00:21.544 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:00:21.544 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:21.544 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:21.544 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:00:21.544 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:00:21.522565+0000", 2023-09-23T21:00:21.544 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999981 2023-09-23T21:00:21.544 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:21.544 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:21.545 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:00:21.545 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:00:21.522575+0000", 2023-09-23T21:00:21.545 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.3430000000000002e-06 2023-09-23T21:00:21.545 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:21.545 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:21.545 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:00:21.545 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:00:21.522596+0000", 2023-09-23T21:00:21.545 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.0954999999999998e-05 2023-09-23T21:00:21.545 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:21.545 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:21.545 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:00:21.545 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:00:21.522605+0000", 2023-09-23T21:00:21.546 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.0010000000000007e-06 2023-09-23T21:00:21.546 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:21.546 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:21.546 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:00:21.546 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:00:21.522653+0000", 2023-09-23T21:00:21.546 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.8486999999999998e-05 2023-09-23T21:00:21.546 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:21.546 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:21.546 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:00:21.546 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:00:21.522753+0000", 2023-09-23T21:00:21.546 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00010045 2023-09-23T21:00:21.547 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:21.547 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:21.547 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 1,3", 2023-09-23T21:00:21.547 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:00:21.522834+0000", 2023-09-23T21:00:21.547 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.0716000000000004e-05 2023-09-23T21:00:21.547 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:21.547 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:21.547 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:00:21.547 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:00:21.529267+0000", 2023-09-23T21:00:21.547 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0064324129999999997 2023-09-23T21:00:21.547 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:21.547 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:21.548 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:00:21.548 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:00:21.533276+0000", 2023-09-23T21:00:21.548 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0040090799999999999 2023-09-23T21:00:21.548 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:00:21.548 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:00:21.548 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:00:21.548 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:21.548 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:21.548 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.35100.0:180257 7.0 7:04be17fd:::benchmark_data_smithi026_260489_object78495:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e246)", 2023-09-23T21:00:21.548 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:00:21.522973+0000", 2023-09-23T21:00:21.548 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 0.01868676, 2023-09-23T21:00:21.548 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.018807675999999999, 2023-09-23T21:00:21.549 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:00:21.549 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "waiting for sub ops", 2023-09-23T21:00:21.549 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:00:21.549 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.35100", 2023-09-23T21:00:21.549 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/4108972961", 2023-09-23T21:00:21.549 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 180257 2023-09-23T21:00:21.549 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:21.549 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:00:21.549 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:21.549 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:00:21.549 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:00:21.522973+0000", 2023-09-23T21:00:21.549 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:00:21.549 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:21.550 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:21.550 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:00:21.550 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:00:21.522973+0000", 2023-09-23T21:00:21.550 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:00:21.550 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:21.550 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:21.550 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:00:21.550 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:00:21.522972+0000", 2023-09-23T21:00:21.550 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.999999 2023-09-23T21:00:21.550 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:21.550 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:21.550 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:00:21.551 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:00:21.522977+0000", 2023-09-23T21:00:21.551 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.6689999999999997e-06 2023-09-23T21:00:21.551 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:21.551 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:21.551 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:00:21.551 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:00:21.522991+0000", 2023-09-23T21:00:21.551 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.3671e-05 2023-09-23T21:00:21.551 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:21.551 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:21.551 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:00:21.551 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:00:21.522997+0000", 2023-09-23T21:00:21.551 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.2190000000000001e-06 2023-09-23T21:00:21.552 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:21.552 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:21.552 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:00:21.552 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:00:21.526067+0000", 2023-09-23T21:00:21.552 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0030697900000000002 2023-09-23T21:00:21.552 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:21.552 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:21.552 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:00:21.552 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:00:21.526152+0000", 2023-09-23T21:00:21.552 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.5168999999999997e-05 2023-09-23T21:00:21.552 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:21.552 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:21.552 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 1,3", 2023-09-23T21:00:21.553 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:00:21.526250+0000", 2023-09-23T21:00:21.553 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.8437999999999995e-05 2023-09-23T21:00:21.553 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:21.553 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:21.553 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:00:21.553 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:00:21.529380+0000", 2023-09-23T21:00:21.553 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0031300019999999998 2023-09-23T21:00:21.553 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:21.553 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:21.553 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:00:21.554 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:00:21.533305+0000", 2023-09-23T21:00:21.554 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0039244040000000003 2023-09-23T21:00:21.554 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:00:21.554 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:00:21.554 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:00:21.554 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:21.554 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:21.555 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.35100.0:180261 7.0 7:07ab9b18:::benchmark_data_smithi026_260489_object78499:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e246)", 2023-09-23T21:00:21.555 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:00:21.534024+0000", 2023-09-23T21:00:21.555 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 0.0076352590000000001, 2023-09-23T21:00:21.555 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0078429120000000005, 2023-09-23T21:00:21.555 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:00:21.555 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "waiting for sub ops", 2023-09-23T21:00:21.556 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:00:21.556 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.35100", 2023-09-23T21:00:21.556 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/4108972961", 2023-09-23T21:00:21.556 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 180261 2023-09-23T21:00:21.556 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:21.556 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:00:21.556 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:21.557 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:00:21.557 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:00:21.534024+0000", 2023-09-23T21:00:21.557 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:00:21.557 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:21.557 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:21.558 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:00:21.558 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:00:21.534024+0000", 2023-09-23T21:00:21.558 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:00:21.558 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:21.558 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:21.558 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:00:21.559 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:00:21.534023+0000", 2023-09-23T21:00:21.559 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T21:00:21.559 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:21.559 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:21.559 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:00:21.559 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:00:21.534030+0000", 2023-09-23T21:00:21.559 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.973e-06 2023-09-23T21:00:21.560 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:21.560 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:21.560 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:00:21.560 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:00:21.534046+0000", 2023-09-23T21:00:21.560 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.5982999999999999e-05 2023-09-23T21:00:21.560 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:21.560 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:21.561 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:00:21.561 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:00:21.534052+0000", 2023-09-23T21:00:21.561 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.9880000000000001e-06 2023-09-23T21:00:21.561 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:21.561 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:21.561 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:00:21.562 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:00:21.534103+0000", 2023-09-23T21:00:21.562 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.0509e-05 2023-09-23T21:00:21.562 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:21.562 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:21.562 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:00:21.562 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:00:21.534224+0000", 2023-09-23T21:00:21.562 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000121166 2023-09-23T21:00:21.562 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:21.562 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:21.562 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 1,3", 2023-09-23T21:00:21.563 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:00:21.534355+0000", 2023-09-23T21:00:21.563 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000131335 2023-09-23T21:00:21.563 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:21.563 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:21.563 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:00:21.563 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:00:21.538491+0000", 2023-09-23T21:00:21.563 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0041355289999999998 2023-09-23T21:00:21.564 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:21.564 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:21.564 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:00:21.564 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:00:21.539319+0000", 2023-09-23T21:00:21.564 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000828085 2023-09-23T21:00:21.564 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:00:21.564 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:00:21.564 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:00:21.564 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:00:21.564 INFO:teuthology.orchestra.run.smithi070.stdout: ], 2023-09-23T21:00:21.564 INFO:teuthology.orchestra.run.smithi070.stdout: "num_ops": 3 2023-09-23T21:00:21.564 INFO:teuthology.orchestra.run.smithi070.stdout:} 2023-09-23T21:00:21.567 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:00:21 smithi070 ceph-mon[168282]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:00:21.567 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:00:21 smithi070 ceph-mon[168282]: failed to encode map e246 with expected crc 2023-09-23T21:00:21.568 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:00:21 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T21:00:21.568 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:00:21 smithi070 ceph-mon[168282]: osdmap e246: 8 total, 8 up, 8 in 2023-09-23T21:00:21.568 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:00:21 smithi070 ceph-mon[168282]: failed to encode map e246 with expected crc 2023-09-23T21:00:21.568 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:00:21 smithi070 ceph-mon[168282]: failed to encode map e246 with expected crc 2023-09-23T21:00:21.568 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:00:21 smithi070 ceph-mon[168282]: pgmap v622: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 1.6 GiB data, 7.2 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 508 op/s; 11885/76476 objects degraded (15.541%); 717/76476 objects misplaced (0.938%); 1.3 MiB/s, 21 objects/s recovering 2023-09-23T21:00:21.574 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:00:21.634 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:00:21 smithi026 ceph-mon[179688]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:00:21.634 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:00:21 smithi026 ceph-mon[179688]: failed to encode map e246 with expected crc 2023-09-23T21:00:21.634 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:00:21 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T21:00:21.634 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:00:21 smithi026 ceph-mon[179688]: osdmap e246: 8 total, 8 up, 8 in 2023-09-23T21:00:21.634 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:00:21 smithi026 ceph-mon[179688]: failed to encode map e246 with expected crc 2023-09-23T21:00:21.634 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:00:21 smithi026 ceph-mon[179688]: failed to encode map e246 with expected crc 2023-09-23T21:00:21.635 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:00:21 smithi026 ceph-mon[179688]: pgmap v622: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 1.6 GiB data, 7.2 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 508 op/s; 11885/76476 objects degraded (15.541%); 717/76476 objects misplaced (0.938%); 1.3 MiB/s, 21 objects/s recovering 2023-09-23T21:00:21.635 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:00:21 smithi026 ceph-mon[175151]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:00:21.635 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:00:21 smithi026 ceph-mon[175151]: failed to encode map e246 with expected crc 2023-09-23T21:00:21.635 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:00:21 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T21:00:21.635 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:00:21 smithi026 ceph-mon[175151]: osdmap e246: 8 total, 8 up, 8 in 2023-09-23T21:00:21.635 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:00:21 smithi026 ceph-mon[175151]: failed to encode map e246 with expected crc 2023-09-23T21:00:21.635 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:00:21 smithi026 ceph-mon[175151]: failed to encode map e246 with expected crc 2023-09-23T21:00:21.635 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:00:21 smithi026 ceph-mon[175151]: pgmap v622: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 1.6 GiB data, 7.2 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 508 op/s; 11885/76476 objects degraded (15.541%); 717/76476 objects misplaced (0.938%); 1.3 MiB/s, 21 objects/s recovering 2023-09-23T21:00:21.668 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:00:21.977 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_blocked_ops 2023-09-23T21:00:22.681 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T21:00:22.929 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T21:00:23.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:00:23 smithi070 ceph-mon[168282]: pgmap v623: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 1.4 GiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 628 op/s; 11892/71253 objects degraded (16.690%); 522/71253 objects misplaced (0.733%); 2.4 MiB/s, 38 objects/s recovering 2023-09-23T21:00:23.885 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:00:23 smithi026 ceph-mon[175151]: pgmap v623: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 1.4 GiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 628 op/s; 11892/71253 objects degraded (16.690%); 522/71253 objects misplaced (0.733%); 2.4 MiB/s, 38 objects/s recovering 2023-09-23T21:00:23.886 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:00:23 smithi026 ceph-mon[179688]: pgmap v623: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 1.4 GiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 628 op/s; 11892/71253 objects degraded (16.690%); 522/71253 objects misplaced (0.733%); 2.4 MiB/s, 38 objects/s recovering 2023-09-23T21:00:23.971 INFO:teuthology.orchestra.run.smithi070.stdout:{ 2023-09-23T21:00:23.971 INFO:teuthology.orchestra.run.smithi070.stdout: "ops": [], 2023-09-23T21:00:23.971 INFO:teuthology.orchestra.run.smithi070.stdout: "complaint_time": 30, 2023-09-23T21:00:23.971 INFO:teuthology.orchestra.run.smithi070.stdout: "num_blocked_ops": 0 2023-09-23T21:00:23.971 INFO:teuthology.orchestra.run.smithi070.stdout:} 2023-09-23T21:00:24.145 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:00:24.146 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T21:00:24.157 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:00:24.158 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:00:24.258 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:00:24.362 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:00:24.382 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_historic_ops 2023-09-23T21:00:25.093 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T21:00:25.165 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:00:25.173 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:00:25.173 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:00:25.274 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:00:25.352 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:00:25.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:00:25 smithi070 ceph-mon[168282]: pgmap v624: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 1.4 GiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 591 op/s; 11892/68826 objects degraded (17.278%); 522/68826 objects misplaced (0.758%); 2.2 MiB/s, 35 objects/s recovering 2023-09-23T21:00:25.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:00:25 smithi070 ceph-mon[168282]: Health check update: Degraded data redundancy: 11892/68826 objects degraded (17.278%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T21:00:25.714 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:00:25.724 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:00:25.724 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:00:25.825 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:00:25.883 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:00:25 smithi026 ceph-mon[179688]: pgmap v624: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 1.4 GiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 591 op/s; 11892/68826 objects degraded (17.278%); 522/68826 objects misplaced (0.758%); 2.2 MiB/s, 35 objects/s recovering 2023-09-23T21:00:25.883 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:00:25 smithi026 ceph-mon[179688]: Health check update: Degraded data redundancy: 11892/68826 objects degraded (17.278%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T21:00:25.883 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:00:25 smithi026 ceph-mon[175151]: pgmap v624: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 1.4 GiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 591 op/s; 11892/68826 objects degraded (17.278%); 522/68826 objects misplaced (0.758%); 2.2 MiB/s, 35 objects/s recovering 2023-09-23T21:00:25.883 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:00:25 smithi026 ceph-mon[175151]: Health check update: Degraded data redundancy: 11892/68826 objects degraded (17.278%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T21:00:25.903 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:00:26.297 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:00:27.186 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:00:27.193 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:00:27.193 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:00:27.294 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:00:27.402 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:00:27.664 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:00:27.670 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:00:27.670 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:00:27.771 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:00:27.776 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:00:27 smithi026 ceph-mon[179688]: pgmap v625: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 1.4 GiB data, 6.7 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 604 op/s; 11892/66957 objects degraded (17.761%); 522/66957 objects misplaced (0.780%); 2.2 MiB/s, 35 objects/s recovering 2023-09-23T21:00:27.781 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:00:27 smithi026 ceph-mon[175151]: pgmap v625: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 1.4 GiB data, 6.7 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 604 op/s; 11892/66957 objects degraded (17.761%); 522/66957 objects misplaced (0.780%); 2.2 MiB/s, 35 objects/s recovering 2023-09-23T21:00:27.845 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:00:27 smithi070 ceph-mon[168282]: pgmap v625: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 1.4 GiB data, 6.7 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 604 op/s; 11892/66957 objects degraded (17.761%); 522/66957 objects misplaced (0.780%); 2.2 MiB/s, 35 objects/s recovering 2023-09-23T21:00:27.869 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:00:28.018 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T21:00:28.030 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:00:28.031 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:00:28.131 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:00:28.217 INFO:teuthology.orchestra.run.smithi026.stdout: 2023-09-23T21:00:28.222 INFO:teuthology.orchestra.run.smithi070.stdout:{ 2023-09-23T21:00:28.222 INFO:teuthology.orchestra.run.smithi070.stdout: "size": 20, 2023-09-23T21:00:28.222 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 600, 2023-09-23T21:00:28.222 INFO:teuthology.orchestra.run.smithi070.stdout: "ops": [ 2023-09-23T21:00:28.222 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:28.222 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.34797.0:130474 6.d 6.9b7cf5d (undecoded) ondisk+retry+write+known_if_redirected+supports_pool_eio e136)", 2023-09-23T21:00:28.222 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T20:54:30.903072+0000", 2023-09-23T21:00:28.222 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 357.31322176200001, 2023-09-23T21:00:28.222 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.97617976500000003, 2023-09-23T21:00:28.223 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:00:28.223 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "reached pg", 2023-09-23T21:00:28.223 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:00:28.223 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.34797", 2023-09-23T21:00:28.223 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T21:00:28.223 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 130474 2023-09-23T21:00:28.223 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:28.223 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:00:28.223 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:28.223 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:00:28.223 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:54:30.903072+0000", 2023-09-23T21:00:28.223 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:00:28.224 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:28.224 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:28.224 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:00:28.224 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:54:30.903072+0000", 2023-09-23T21:00:28.224 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:00:28.224 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:28.224 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:28.224 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:00:28.224 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:54:30.903071+0000", 2023-09-23T21:00:28.224 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.999999 2023-09-23T21:00:28.224 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:28.225 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:28.225 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:00:28.225 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:54:30.903074+0000", 2023-09-23T21:00:28.225 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.7100000000000001e-06 2023-09-23T21:00:28.225 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:28.225 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:28.225 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:00:28.225 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:54:30.903083+0000", 2023-09-23T21:00:28.225 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.3240000000000003e-06 2023-09-23T21:00:28.225 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:28.225 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:28.225 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:00:28.225 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:54:30.903088+0000", 2023-09-23T21:00:28.226 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.4280000000000004e-06 2023-09-23T21:00:28.226 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:28.226 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:28.226 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:00:28.226 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:54:30.903139+0000", 2023-09-23T21:00:28.226 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.0769000000000001e-05 2023-09-23T21:00:28.226 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:28.226 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:28.226 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:00:28.226 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:54:30.903144+0000", 2023-09-23T21:00:28.226 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.2320000000000001e-06 2023-09-23T21:00:28.227 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:28.227 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:28.227 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:00:28.227 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:54:31.879231+0000", 2023-09-23T21:00:28.227 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.97608724099999999 2023-09-23T21:00:28.227 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:28.227 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:28.227 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:00:28.227 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:54:31.879251+0000", 2023-09-23T21:00:28.227 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.0156000000000001e-05 2023-09-23T21:00:28.227 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:00:28.228 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:00:28.228 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:00:28.228 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:28.228 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:28.228 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.34797.0:167244 6.a 6:5fa8594b:::benchmark_data_smithi026_223868_object66118:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e158)", 2023-09-23T21:00:28.228 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T20:55:20.930710+0000", 2023-09-23T21:00:28.228 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 307.28558337999999, 2023-09-23T21:00:28.228 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.0357931300000001, 2023-09-23T21:00:28.228 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:00:28.228 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:00:28.228 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:00:28.229 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.34797", 2023-09-23T21:00:28.229 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T21:00:28.229 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 167244 2023-09-23T21:00:28.229 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:28.229 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:00:28.229 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:28.229 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:00:28.229 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:20.930710+0000", 2023-09-23T21:00:28.229 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:00:28.229 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:28.229 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:28.229 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:00:28.230 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:20.930710+0000", 2023-09-23T21:00:28.230 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:00:28.230 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:28.230 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:28.230 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:00:28.230 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:20.930709+0000", 2023-09-23T21:00:28.230 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.999999 2023-09-23T21:00:28.230 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:28.230 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:28.230 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:00:28.230 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:20.930716+0000", 2023-09-23T21:00:28.231 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.2579999999999998e-06 2023-09-23T21:00:28.231 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:28.231 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:28.231 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:00:28.231 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:20.930726+0000", 2023-09-23T21:00:28.231 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.0087e-05 2023-09-23T21:00:28.231 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:28.231 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:28.231 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:00:28.231 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:20.930731+0000", 2023-09-23T21:00:28.231 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.7140000000000001e-06 2023-09-23T21:00:28.232 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:28.232 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:28.232 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:00:28.232 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:20.930770+0000", 2023-09-23T21:00:28.232 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.9199999999999997e-05 2023-09-23T21:00:28.232 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:28.232 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:28.232 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:00:28.232 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:20.930779+0000", 2023-09-23T21:00:28.232 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.3380000000000005e-06 2023-09-23T21:00:28.232 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:28.233 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:28.233 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:00:28.233 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:21.956800+0000", 2023-09-23T21:00:28.233 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.026021538 2023-09-23T21:00:28.233 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:28.233 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:28.233 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:00:28.233 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:21.956900+0000", 2023-09-23T21:00:28.233 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.9709000000000004e-05 2023-09-23T21:00:28.233 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:28.233 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:28.234 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 0,2", 2023-09-23T21:00:28.234 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:21.957010+0000", 2023-09-23T21:00:28.234 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000110504 2023-09-23T21:00:28.234 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:28.234 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:28.234 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:00:28.234 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:21.959547+0000", 2023-09-23T21:00:28.234 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0025366799999999999 2023-09-23T21:00:28.234 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:28.234 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:28.235 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:00:28.235 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:21.966355+0000", 2023-09-23T21:00:28.235 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.006807848 2023-09-23T21:00:28.235 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:28.235 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:28.235 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:00:28.235 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:21.966407+0000", 2023-09-23T21:00:28.235 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.1666999999999998e-05 2023-09-23T21:00:28.235 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:28.235 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:28.235 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:00:28.236 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:21.966442+0000", 2023-09-23T21:00:28.236 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.5602000000000002e-05 2023-09-23T21:00:28.236 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:28.236 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:28.236 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:00:28.236 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:21.966503+0000", 2023-09-23T21:00:28.236 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.1019e-05 2023-09-23T21:00:28.236 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:00:28.236 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:00:28.236 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:00:28.236 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:28.236 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:28.237 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.34797.0:167247 6.a 6:5b80d451:::benchmark_data_smithi026_223868_object66121:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e158)", 2023-09-23T21:00:28.237 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T20:55:20.930743+0000", 2023-09-23T21:00:28.237 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 307.28555033399999, 2023-09-23T21:00:28.237 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.0409182340000001, 2023-09-23T21:00:28.237 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:00:28.237 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:00:28.237 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:00:28.237 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.34797", 2023-09-23T21:00:28.237 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T21:00:28.237 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 167247 2023-09-23T21:00:28.237 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:28.237 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:00:28.238 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:28.238 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:00:28.238 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:20.930743+0000", 2023-09-23T21:00:28.238 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:00:28.238 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:28.238 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:28.238 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:00:28.238 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:20.930743+0000", 2023-09-23T21:00:28.238 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:00:28.238 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:28.238 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:28.239 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:00:28.239 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:20.930742+0000", 2023-09-23T21:00:28.239 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999995 2023-09-23T21:00:28.239 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:28.239 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:28.239 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:00:28.239 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:20.930746+0000", 2023-09-23T21:00:28.239 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.1879999999999998e-06 2023-09-23T21:00:28.239 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:28.239 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:28.239 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:00:28.240 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:20.930752+0000", 2023-09-23T21:00:28.240 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.0379999999999996e-06 2023-09-23T21:00:28.240 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:28.240 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:28.240 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:00:28.240 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:20.930756+0000", 2023-09-23T21:00:28.240 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.1030000000000004e-06 2023-09-23T21:00:28.240 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:28.240 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:28.240 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:00:28.240 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:20.930808+0000", 2023-09-23T21:00:28.240 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.2305999999999997e-05 2023-09-23T21:00:28.241 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:28.241 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:28.241 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:00:28.241 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:20.930813+0000", 2023-09-23T21:00:28.241 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.9910000000000002e-06 2023-09-23T21:00:28.241 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:28.241 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:28.241 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:00:28.241 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:21.959580+0000", 2023-09-23T21:00:28.241 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.0287672910000001 2023-09-23T21:00:28.241 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:28.241 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:28.241 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:00:28.241 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:21.959671+0000", 2023-09-23T21:00:28.242 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.0928999999999996e-05 2023-09-23T21:00:28.242 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:28.242 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:28.242 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 0,2", 2023-09-23T21:00:28.242 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:21.959739+0000", 2023-09-23T21:00:28.242 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.7663000000000006e-05 2023-09-23T21:00:28.242 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:28.242 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:28.242 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:00:28.242 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:21.969123+0000", 2023-09-23T21:00:28.242 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0093844800000000006 2023-09-23T21:00:28.242 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:28.243 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:28.243 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:00:28.243 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:21.971537+0000", 2023-09-23T21:00:28.243 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.002413157 2023-09-23T21:00:28.243 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:28.243 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:28.243 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:00:28.243 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:21.971574+0000", 2023-09-23T21:00:28.243 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.7681000000000001e-05 2023-09-23T21:00:28.243 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:28.243 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:28.243 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:00:28.243 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:21.971604+0000", 2023-09-23T21:00:28.243 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.0176999999999999e-05 2023-09-23T21:00:28.244 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:28.244 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:28.244 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:00:28.244 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:21.971661+0000", 2023-09-23T21:00:28.244 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.6857999999999997e-05 2023-09-23T21:00:28.244 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:00:28.244 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:00:28.244 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:00:28.244 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:28.244 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:28.244 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.34797.0:167248 6.a 6:59b345d1:::benchmark_data_smithi026_223868_object66122:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e158)", 2023-09-23T21:00:28.244 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T20:55:20.930761+0000", 2023-09-23T21:00:28.244 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 307.28553275399997, 2023-09-23T21:00:28.245 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.055093563, 2023-09-23T21:00:28.245 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:00:28.245 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:00:28.245 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:00:28.245 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.34797", 2023-09-23T21:00:28.245 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T21:00:28.245 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 167248 2023-09-23T21:00:28.245 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:28.245 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:00:28.245 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:28.245 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:00:28.245 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:20.930761+0000", 2023-09-23T21:00:28.246 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:00:28.246 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:28.246 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:28.246 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:00:28.246 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:20.930761+0000", 2023-09-23T21:00:28.246 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:00:28.246 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:28.246 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:28.246 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:00:28.246 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:20.930760+0000", 2023-09-23T21:00:28.246 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999995 2023-09-23T21:00:28.246 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:28.247 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:28.247 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:00:28.247 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:20.930762+0000", 2023-09-23T21:00:28.247 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.1119999999999999e-06 2023-09-23T21:00:28.247 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:28.247 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:28.247 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:00:28.247 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:20.930768+0000", 2023-09-23T21:00:28.247 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.4539999999999998e-06 2023-09-23T21:00:28.247 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:28.247 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:28.247 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:00:28.247 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:20.930771+0000", 2023-09-23T21:00:28.248 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.5899999999999999e-06 2023-09-23T21:00:28.248 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:28.248 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:28.248 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:00:28.248 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:20.930827+0000", 2023-09-23T21:00:28.248 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.5949000000000003e-05 2023-09-23T21:00:28.248 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:28.248 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:28.248 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:00:28.248 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:20.930831+0000", 2023-09-23T21:00:28.248 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.4199999999999999e-06 2023-09-23T21:00:28.248 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:28.248 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:28.248 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:00:28.249 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:21.961980+0000", 2023-09-23T21:00:28.249 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.0311488959999999 2023-09-23T21:00:28.249 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:28.249 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:28.249 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:00:28.249 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:21.962078+0000", 2023-09-23T21:00:28.249 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.8317000000000001e-05 2023-09-23T21:00:28.249 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:28.249 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:28.249 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 0,2", 2023-09-23T21:00:28.249 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:21.962194+0000", 2023-09-23T21:00:28.249 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000115528 2023-09-23T21:00:28.249 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:28.250 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:28.250 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:00:28.250 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:21.969134+0000", 2023-09-23T21:00:28.250 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0069402370000000001 2023-09-23T21:00:28.250 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:28.250 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:28.250 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:00:28.250 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:21.976008+0000", 2023-09-23T21:00:28.250 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.006874326 2023-09-23T21:00:28.250 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:28.250 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:28.250 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:00:28.250 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:21.985762+0000", 2023-09-23T21:00:28.251 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0097536199999999993 2023-09-23T21:00:28.251 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:28.251 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:28.251 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:00:28.251 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:21.985793+0000", 2023-09-23T21:00:28.251 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.1628999999999998e-05 2023-09-23T21:00:28.251 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:28.251 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:28.251 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:00:28.251 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:21.985854+0000", 2023-09-23T21:00:28.251 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.0838000000000003e-05 2023-09-23T21:00:28.251 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:00:28.251 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:00:28.252 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:00:28.252 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:28.252 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:28.252 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.34797.0:167255 6.a 6:574da207:::benchmark_data_smithi026_223868_object66129:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e158)", 2023-09-23T21:00:28.252 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T20:55:20.936722+0000", 2023-09-23T21:00:28.252 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 307.27957148600001, 2023-09-23T21:00:28.252 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.053378076, 2023-09-23T21:00:28.252 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:00:28.252 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:00:28.252 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:00:28.252 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.34797", 2023-09-23T21:00:28.252 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T21:00:28.252 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 167255 2023-09-23T21:00:28.252 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:28.253 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:00:28.253 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:28.253 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:00:28.253 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:20.936722+0000", 2023-09-23T21:00:28.253 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:00:28.253 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:28.253 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:28.253 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:00:28.253 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:20.936722+0000", 2023-09-23T21:00:28.253 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:00:28.253 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:28.253 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:28.253 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:00:28.254 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:20.936721+0000", 2023-09-23T21:00:28.254 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.999999 2023-09-23T21:00:28.254 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:28.254 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:28.254 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:00:28.254 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:20.936725+0000", 2023-09-23T21:00:28.254 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.3150000000000002e-06 2023-09-23T21:00:28.254 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:28.254 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:28.254 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:00:28.254 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:20.936734+0000", 2023-09-23T21:00:28.254 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.2199999999999998e-06 2023-09-23T21:00:28.254 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:28.255 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:28.255 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:00:28.255 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:20.936739+0000", 2023-09-23T21:00:28.255 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.4309999999999996e-06 2023-09-23T21:00:28.255 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:28.255 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:28.255 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:00:28.255 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:20.936768+0000", 2023-09-23T21:00:28.255 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.9567999999999999e-05 2023-09-23T21:00:28.255 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:28.255 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:28.255 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:00:28.255 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:20.936774+0000", 2023-09-23T21:00:28.256 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.485e-06 2023-09-23T21:00:28.256 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:28.256 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:28.256 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:00:28.256 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:21.964309+0000", 2023-09-23T21:00:28.256 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.027534704 2023-09-23T21:00:28.256 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:28.256 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:28.256 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:00:28.256 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:21.964430+0000", 2023-09-23T21:00:28.256 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00012104000000000001 2023-09-23T21:00:28.256 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:28.256 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:28.257 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 0,2", 2023-09-23T21:00:28.257 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:21.964551+0000", 2023-09-23T21:00:28.257 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00012153 2023-09-23T21:00:28.257 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:28.257 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:28.257 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:00:28.257 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:21.971489+0000", 2023-09-23T21:00:28.257 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0069378219999999997 2023-09-23T21:00:28.257 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:28.257 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:28.257 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:00:28.257 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:21.981600+0000", 2023-09-23T21:00:28.257 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.010111063 2023-09-23T21:00:28.257 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:28.258 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:28.258 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:00:28.258 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:21.989954+0000", 2023-09-23T21:00:28.258 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0083543060000000006 2023-09-23T21:00:28.258 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:28.258 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:28.258 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:00:28.258 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:21.990003+0000", 2023-09-23T21:00:28.258 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.8662999999999998e-05 2023-09-23T21:00:28.258 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:28.258 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:28.258 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:00:28.258 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:21.990100+0000", 2023-09-23T21:00:28.259 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.6912999999999999e-05 2023-09-23T21:00:28.259 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:00:28.259 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:00:28.259 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:00:28.259 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:28.259 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:28.259 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.34797.0:167281 6.a 6:5de3ccd5:::benchmark_data_smithi026_223868_object66155:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e158)", 2023-09-23T21:00:28.259 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T20:55:20.963939+0000", 2023-09-23T21:00:28.259 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 307.25235407100001, 2023-09-23T21:00:28.259 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.026322833, 2023-09-23T21:00:28.259 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:00:28.259 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:00:28.259 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:00:28.260 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.34797", 2023-09-23T21:00:28.260 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T21:00:28.260 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 167281 2023-09-23T21:00:28.260 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:28.260 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:00:28.260 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:28.260 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:00:28.260 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:20.963939+0000", 2023-09-23T21:00:28.260 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:00:28.260 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:28.260 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:28.260 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:00:28.260 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:20.963939+0000", 2023-09-23T21:00:28.260 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:00:28.261 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:28.261 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:28.261 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:00:28.261 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:20.963938+0000", 2023-09-23T21:00:28.261 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T21:00:28.261 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:28.261 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:28.261 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:00:28.261 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:20.963943+0000", 2023-09-23T21:00:28.261 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.2739999999999998e-06 2023-09-23T21:00:28.261 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:28.261 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:28.261 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:00:28.262 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:20.963958+0000", 2023-09-23T21:00:28.262 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.5096000000000001e-05 2023-09-23T21:00:28.262 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:28.262 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:28.262 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:00:28.262 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:20.963966+0000", 2023-09-23T21:00:28.262 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.5140000000000004e-06 2023-09-23T21:00:28.262 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:28.262 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:28.262 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:00:28.262 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:20.964006+0000", 2023-09-23T21:00:28.262 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.0504999999999999e-05 2023-09-23T21:00:28.262 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:28.263 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:28.263 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:00:28.263 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:20.964015+0000", 2023-09-23T21:00:28.263 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.8070000000000006e-06 2023-09-23T21:00:28.263 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:28.263 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:28.263 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:00:28.263 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:21.966539+0000", 2023-09-23T21:00:28.263 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.002524204 2023-09-23T21:00:28.263 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:28.263 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:28.263 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:00:28.264 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:21.966674+0000", 2023-09-23T21:00:28.264 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00013492699999999999 2023-09-23T21:00:28.264 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:28.264 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:28.264 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 0,2", 2023-09-23T21:00:28.264 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:21.967167+0000", 2023-09-23T21:00:28.264 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000492549 2023-09-23T21:00:28.264 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:28.264 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:28.264 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:00:28.264 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:21.971671+0000", 2023-09-23T21:00:28.264 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0045045529999999997 2023-09-23T21:00:28.264 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:28.264 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:28.265 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:00:28.265 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:21.981643+0000", 2023-09-23T21:00:28.265 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0099714829999999997 2023-09-23T21:00:28.265 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:28.265 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:28.265 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:00:28.265 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:21.990135+0000", 2023-09-23T21:00:28.265 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0084919009999999996 2023-09-23T21:00:28.265 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:28.265 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:28.265 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:00:28.265 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:21.990164+0000", 2023-09-23T21:00:28.265 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.9549000000000001e-05 2023-09-23T21:00:28.266 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:28.266 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:28.266 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:00:28.266 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:21.990262+0000", 2023-09-23T21:00:28.266 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.781e-05 2023-09-23T21:00:28.266 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:00:28.266 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:00:28.266 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:00:28.266 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:28.266 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:28.266 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.34797.0:167305 6.a 6:5183351c:::benchmark_data_smithi026_223868_object66179:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e158)", 2023-09-23T21:00:28.266 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T20:55:20.991170+0000", 2023-09-23T21:00:28.266 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 307.22512291499999, 2023-09-23T21:00:28.267 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.999303524, 2023-09-23T21:00:28.267 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:00:28.267 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:00:28.267 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:00:28.267 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.34797", 2023-09-23T21:00:28.267 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T21:00:28.267 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 167305 2023-09-23T21:00:28.267 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:28.267 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:00:28.267 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:28.267 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:00:28.267 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:20.991170+0000", 2023-09-23T21:00:28.267 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:00:28.267 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:28.268 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:28.268 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:00:28.268 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:20.991170+0000", 2023-09-23T21:00:28.268 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:00:28.268 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:28.268 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:28.268 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:00:28.268 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:20.991169+0000", 2023-09-23T21:00:28.268 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T21:00:28.268 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:28.268 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:28.268 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:00:28.268 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:20.991176+0000", 2023-09-23T21:00:28.269 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.2219999999999996e-06 2023-09-23T21:00:28.269 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:28.269 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:28.269 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:00:28.269 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:20.991192+0000", 2023-09-23T21:00:28.269 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.5798e-05 2023-09-23T21:00:28.269 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:28.269 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:28.269 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:00:28.269 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:20.991199+0000", 2023-09-23T21:00:28.269 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.516e-06 2023-09-23T21:00:28.269 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:28.269 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:28.270 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:00:28.270 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:20.991264+0000", 2023-09-23T21:00:28.270 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.5135000000000006e-05 2023-09-23T21:00:28.270 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:28.270 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:28.270 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:00:28.270 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:20.991272+0000", 2023-09-23T21:00:28.270 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.7250000000000003e-06 2023-09-23T21:00:28.270 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:28.270 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:28.270 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:00:28.270 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:21.969162+0000", 2023-09-23T21:00:28.271 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.97788947999999998 2023-09-23T21:00:28.271 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:28.271 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:28.271 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:00:28.271 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:21.969309+0000", 2023-09-23T21:00:28.271 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00014732499999999999 2023-09-23T21:00:28.271 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:28.271 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:28.271 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 0,2", 2023-09-23T21:00:28.271 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:21.969431+0000", 2023-09-23T21:00:28.271 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000121797 2023-09-23T21:00:28.271 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:28.271 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:28.272 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:00:28.272 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:21.976014+0000", 2023-09-23T21:00:28.272 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0065832219999999997 2023-09-23T21:00:28.272 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:28.272 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:28.272 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:00:28.272 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:21.985730+0000", 2023-09-23T21:00:28.272 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0097158490000000004 2023-09-23T21:00:28.272 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:28.272 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:28.272 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:00:28.272 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:21.990318+0000", 2023-09-23T21:00:28.273 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0045880269999999997 2023-09-23T21:00:28.273 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:28.273 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:28.273 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:00:28.273 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:21.990372+0000", 2023-09-23T21:00:28.273 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.3658e-05 2023-09-23T21:00:28.273 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:28.273 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:28.273 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:00:28.273 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:21.990474+0000", 2023-09-23T21:00:28.273 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00010228799999999999 2023-09-23T21:00:28.273 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:00:28.273 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:00:28.274 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:00:28.274 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:28.274 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:28.274 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.34797.0:167315 6.a 6:5232d899:::benchmark_data_smithi026_223868_object66189:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e158)", 2023-09-23T21:00:28.274 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T20:55:21.002335+0000", 2023-09-23T21:00:28.274 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 307.21395818600001, 2023-09-23T21:00:28.274 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.988365296, 2023-09-23T21:00:28.274 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:00:28.274 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:00:28.274 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:00:28.274 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.34797", 2023-09-23T21:00:28.275 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T21:00:28.275 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 167315 2023-09-23T21:00:28.275 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:28.275 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:00:28.275 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:28.275 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:00:28.275 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:21.002335+0000", 2023-09-23T21:00:28.275 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:00:28.275 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:28.275 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:28.275 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:00:28.275 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:21.002335+0000", 2023-09-23T21:00:28.276 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:00:28.276 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:28.276 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:28.276 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:00:28.276 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:21.002333+0000", 2023-09-23T21:00:28.276 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999981 2023-09-23T21:00:28.276 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:28.276 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:28.276 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:00:28.276 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:21.002342+0000", 2023-09-23T21:00:28.276 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.0149999999999992e-06 2023-09-23T21:00:28.276 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:28.277 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:28.277 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:00:28.277 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:21.002357+0000", 2023-09-23T21:00:28.277 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.4859e-05 2023-09-23T21:00:28.277 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:28.277 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:28.277 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:00:28.277 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:21.002366+0000", 2023-09-23T21:00:28.277 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.7099999999999996e-06 2023-09-23T21:00:28.277 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:28.277 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:28.277 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:00:28.278 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:21.002492+0000", 2023-09-23T21:00:28.278 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00012614200000000001 2023-09-23T21:00:28.278 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:28.278 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:28.278 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:00:28.278 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:21.002502+0000", 2023-09-23T21:00:28.278 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.0079e-05 2023-09-23T21:00:28.278 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:28.278 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:28.278 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:00:28.278 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:21.971710+0000", 2023-09-23T21:00:28.279 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.96920738699999998 2023-09-23T21:00:28.279 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:28.279 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:28.279 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:00:28.279 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:21.971831+0000", 2023-09-23T21:00:28.279 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00012119499999999999 2023-09-23T21:00:28.279 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:28.279 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:28.279 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 0,2", 2023-09-23T21:00:28.279 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:21.971946+0000", 2023-09-23T21:00:28.279 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000115135 2023-09-23T21:00:28.279 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:28.280 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:28.280 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:00:28.280 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:21.979629+0000", 2023-09-23T21:00:28.280 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0076829059999999998 2023-09-23T21:00:28.280 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:28.280 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:28.280 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:00:28.280 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:21.990521+0000", 2023-09-23T21:00:28.280 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.010892013000000001 2023-09-23T21:00:28.280 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:28.280 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:28.280 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:00:28.280 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:21.990565+0000", 2023-09-23T21:00:28.281 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.4579000000000001e-05 2023-09-23T21:00:28.281 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:28.281 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:28.281 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:00:28.281 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:21.990605+0000", 2023-09-23T21:00:28.281 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.9854000000000002e-05 2023-09-23T21:00:28.281 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:28.281 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:28.281 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:00:28.281 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:21.990701+0000", 2023-09-23T21:00:28.281 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.5227000000000002e-05 2023-09-23T21:00:28.281 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:00:28.281 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:00:28.282 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:00:28.282 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:28.282 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:28.282 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.34797.0:167338 6.a 6:571cce03:::benchmark_data_smithi026_223868_object66212:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e158)", 2023-09-23T21:00:28.282 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T20:55:21.027265+0000", 2023-09-23T21:00:28.282 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 307.18902842199998, 2023-09-23T21:00:28.282 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.976460098, 2023-09-23T21:00:28.282 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:00:28.282 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:00:28.282 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:00:28.282 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.34797", 2023-09-23T21:00:28.282 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T21:00:28.283 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 167338 2023-09-23T21:00:28.283 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:28.283 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:00:28.283 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:28.283 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:00:28.283 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:21.027265+0000", 2023-09-23T21:00:28.283 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:00:28.283 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:28.283 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:28.283 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:00:28.283 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:21.027265+0000", 2023-09-23T21:00:28.283 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:00:28.283 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:28.284 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:28.284 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:00:28.284 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:21.027263+0000", 2023-09-23T21:00:28.284 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999981 2023-09-23T21:00:28.284 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:28.284 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:28.284 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:00:28.284 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:21.027270+0000", 2023-09-23T21:00:28.284 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.5329999999999997e-06 2023-09-23T21:00:28.284 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:28.284 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:28.284 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:00:28.285 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:21.027283+0000", 2023-09-23T21:00:28.285 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.2516e-05 2023-09-23T21:00:28.285 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:28.285 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:28.285 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:00:28.285 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:21.027290+0000", 2023-09-23T21:00:28.285 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.4989999999999999e-06 2023-09-23T21:00:28.285 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:28.285 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:28.285 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:00:28.285 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:21.027342+0000", 2023-09-23T21:00:28.286 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.2244000000000003e-05 2023-09-23T21:00:28.286 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:28.286 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:28.286 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:00:28.286 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:21.027351+0000", 2023-09-23T21:00:28.286 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.8410000000000006e-06 2023-09-23T21:00:28.286 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:28.286 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:28.286 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:00:28.286 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:21.974196+0000", 2023-09-23T21:00:28.286 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.94684518299999998 2023-09-23T21:00:28.286 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:28.287 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:28.287 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:00:28.287 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:21.974318+0000", 2023-09-23T21:00:28.287 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00012264 2023-09-23T21:00:28.287 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:28.287 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:28.287 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 0,2", 2023-09-23T21:00:28.287 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:21.974384+0000", 2023-09-23T21:00:28.287 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.5337000000000001e-05 2023-09-23T21:00:28.287 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:28.287 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:28.288 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:00:28.288 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:21.981609+0000", 2023-09-23T21:00:28.288 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00722553 2023-09-23T21:00:28.288 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:28.288 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:28.288 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:00:28.288 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:21.995381+0000", 2023-09-23T21:00:28.288 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.013771781 2023-09-23T21:00:28.288 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:28.288 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:28.288 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:00:28.289 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:22.003516+0000", 2023-09-23T21:00:28.289 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0081353399999999996 2023-09-23T21:00:28.289 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:28.289 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:28.289 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:00:28.289 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:22.003568+0000", 2023-09-23T21:00:28.289 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.1354000000000003e-05 2023-09-23T21:00:28.289 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:28.289 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:28.289 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:00:28.289 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:22.003725+0000", 2023-09-23T21:00:28.289 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00015731400000000001 2023-09-23T21:00:28.289 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:00:28.290 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:00:28.290 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:00:28.290 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:28.290 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:28.290 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.34797.0:177896 6.a 6:540ceadc:::benchmark_data_smithi026_223868_object76770:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e163)", 2023-09-23T21:00:28.290 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T20:55:34.908670+0000", 2023-09-23T21:00:28.290 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 293.30762386700002, 2023-09-23T21:00:28.290 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.1466390440000001, 2023-09-23T21:00:28.290 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:00:28.290 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "reached pg", 2023-09-23T21:00:28.290 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:00:28.291 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.34797", 2023-09-23T21:00:28.291 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T21:00:28.291 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 177896 2023-09-23T21:00:28.291 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:28.291 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:00:28.291 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:28.291 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:00:28.291 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:34.908670+0000", 2023-09-23T21:00:28.291 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:00:28.291 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:28.291 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:28.291 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:00:28.292 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:34.908670+0000", 2023-09-23T21:00:28.292 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:00:28.292 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:28.292 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:28.292 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:00:28.292 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:34.908668+0000", 2023-09-23T21:00:28.292 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.999999 2023-09-23T21:00:28.292 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:28.292 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:28.292 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:00:28.292 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:34.908673+0000", 2023-09-23T21:00:28.292 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.4769999999999997e-06 2023-09-23T21:00:28.292 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:28.293 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:28.293 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:00:28.293 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:34.908683+0000", 2023-09-23T21:00:28.293 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.0254e-05 2023-09-23T21:00:28.293 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:28.293 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:28.293 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:00:28.293 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:34.908687+0000", 2023-09-23T21:00:28.293 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.1219999999999997e-06 2023-09-23T21:00:28.293 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:28.293 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:28.293 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:00:28.294 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:34.908728+0000", 2023-09-23T21:00:28.294 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.0781999999999997e-05 2023-09-23T21:00:28.294 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:28.294 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:28.294 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:00:28.294 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:34.908820+0000", 2023-09-23T21:00:28.294 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.2058999999999994e-05 2023-09-23T21:00:28.294 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:28.294 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:28.294 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 0,2", 2023-09-23T21:00:28.294 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:34.908888+0000", 2023-09-23T21:00:28.294 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.8287000000000005e-05 2023-09-23T21:00:28.295 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:28.295 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:28.295 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:00:28.295 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:34.911386+0000", 2023-09-23T21:00:28.295 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0024977459999999999 2023-09-23T21:00:28.295 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:28.295 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:28.295 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:00:28.295 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:34.923103+0000", 2023-09-23T21:00:28.295 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.011717293 2023-09-23T21:00:28.295 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:28.295 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:28.295 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:00:28.295 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:34.926697+0000", 2023-09-23T21:00:28.296 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0035937220000000001 2023-09-23T21:00:28.296 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:28.296 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:28.296 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:00:28.296 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:34.926708+0000", 2023-09-23T21:00:28.296 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.0339e-05 2023-09-23T21:00:28.296 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:28.296 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:28.296 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:00:28.296 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:35.946870+0000", 2023-09-23T21:00:28.296 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.0201623930000001 2023-09-23T21:00:28.296 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:28.296 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:28.297 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:00:28.297 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:35.946882+0000", 2023-09-23T21:00:28.297 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.1936999999999999e-05 2023-09-23T21:00:28.297 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:28.297 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:28.297 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:00:28.297 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:37.039221+0000", 2023-09-23T21:00:28.297 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.092338928 2023-09-23T21:00:28.297 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:28.297 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:28.297 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for missing object", 2023-09-23T21:00:28.297 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:37.039314+0000", 2023-09-23T21:00:28.297 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.3096000000000004e-05 2023-09-23T21:00:28.298 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:28.298 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:28.298 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:00:28.298 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:37.055200+0000", 2023-09-23T21:00:28.298 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.015886493000000002 2023-09-23T21:00:28.298 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:28.298 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:28.298 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:00:28.298 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:37.055309+0000", 2023-09-23T21:00:28.298 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00010818 2023-09-23T21:00:28.298 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:00:28.298 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:00:28.298 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:00:28.298 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:28.299 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:28.299 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.34797.0:177903 6.a 6:5c2bf67a:::benchmark_data_smithi026_223868_object76777:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e163)", 2023-09-23T21:00:28.299 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T20:55:34.915862+0000", 2023-09-23T21:00:28.299 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 293.300431285, 2023-09-23T21:00:28.299 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.1423018539999998, 2023-09-23T21:00:28.299 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:00:28.299 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "reached pg", 2023-09-23T21:00:28.299 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:00:28.299 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.34797", 2023-09-23T21:00:28.299 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T21:00:28.299 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 177903 2023-09-23T21:00:28.299 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:28.300 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:00:28.300 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:28.300 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:00:28.300 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:34.915862+0000", 2023-09-23T21:00:28.300 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:00:28.300 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:28.300 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:28.300 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:00:28.300 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:34.915862+0000", 2023-09-23T21:00:28.300 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:00:28.300 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:28.300 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:28.300 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:00:28.300 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:34.915861+0000", 2023-09-23T21:00:28.301 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.999999 2023-09-23T21:00:28.301 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:28.301 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:28.301 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:00:28.301 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:34.915866+0000", 2023-09-23T21:00:28.301 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.78e-06 2023-09-23T21:00:28.301 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:28.301 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:28.301 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:00:28.301 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:34.915881+0000", 2023-09-23T21:00:28.301 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.552e-05 2023-09-23T21:00:28.301 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:28.301 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:28.302 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:00:28.302 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:34.915889+0000", 2023-09-23T21:00:28.302 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.1960000000000002e-06 2023-09-23T21:00:28.302 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:28.302 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:28.302 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:00:28.302 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:34.915932+0000", 2023-09-23T21:00:28.302 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.3272000000000002e-05 2023-09-23T21:00:28.302 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:28.302 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:28.302 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:00:28.302 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:34.916061+0000", 2023-09-23T21:00:28.302 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00012904999999999999 2023-09-23T21:00:28.303 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:28.303 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:28.303 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 0,2", 2023-09-23T21:00:28.303 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:34.916135+0000", 2023-09-23T21:00:28.303 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.4144e-05 2023-09-23T21:00:28.303 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:28.303 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:28.303 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:00:28.303 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:34.918475+0000", 2023-09-23T21:00:28.303 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.002339921 2023-09-23T21:00:28.303 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:28.303 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:28.303 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:00:28.303 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:34.926898+0000", 2023-09-23T21:00:28.304 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0084229939999999996 2023-09-23T21:00:28.304 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:28.304 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:28.304 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:00:28.304 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:34.926907+0000", 2023-09-23T21:00:28.304 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.9590000000000001e-06 2023-09-23T21:00:28.304 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:28.304 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:28.304 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:00:28.304 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:35.947668+0000", 2023-09-23T21:00:28.304 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.0207606359999999 2023-09-23T21:00:28.304 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:28.305 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:28.305 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:00:28.305 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:35.947676+0000", 2023-09-23T21:00:28.305 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.7029999999999995e-06 2023-09-23T21:00:28.305 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:28.305 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:28.305 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:00:28.305 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:37.039586+0000", 2023-09-23T21:00:28.305 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.0919099109999999 2023-09-23T21:00:28.305 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:28.305 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:28.305 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for missing object", 2023-09-23T21:00:28.305 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:37.039684+0000", 2023-09-23T21:00:28.306 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.7732999999999994e-05 2023-09-23T21:00:28.306 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:28.306 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:28.306 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:00:28.306 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:37.058038+0000", 2023-09-23T21:00:28.306 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.018353669999999999 2023-09-23T21:00:28.306 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:28.306 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:28.306 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:00:28.306 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:37.058164+0000", 2023-09-23T21:00:28.306 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00012637400000000001 2023-09-23T21:00:28.306 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:00:28.306 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:00:28.306 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:00:28.307 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:28.307 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:28.307 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.34797.0:177914 6.a 6:5c0a39bf:::benchmark_data_smithi026_223868_object76788:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e164)", 2023-09-23T21:00:28.307 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T20:55:34.934127+0000", 2023-09-23T21:00:28.307 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 293.282166684, 2023-09-23T21:00:28.307 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.1279153630000001, 2023-09-23T21:00:28.307 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:00:28.307 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:00:28.307 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:00:28.307 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.34797", 2023-09-23T21:00:28.307 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T21:00:28.308 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 177914 2023-09-23T21:00:28.308 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:28.308 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:00:28.308 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:28.308 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:00:28.308 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:34.934127+0000", 2023-09-23T21:00:28.308 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:00:28.308 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:28.308 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:28.308 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:00:28.308 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:34.934127+0000", 2023-09-23T21:00:28.308 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:00:28.309 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:28.309 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:28.309 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:00:28.309 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:34.934126+0000", 2023-09-23T21:00:28.309 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999995 2023-09-23T21:00:28.309 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:28.309 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:28.309 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:00:28.309 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:34.934129+0000", 2023-09-23T21:00:28.309 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.1690000000000001e-06 2023-09-23T21:00:28.309 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:28.310 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:28.310 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:00:28.310 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:34.934136+0000", 2023-09-23T21:00:28.310 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.8120000000000002e-06 2023-09-23T21:00:28.310 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:28.310 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:28.310 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:00:28.310 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:34.934141+0000", 2023-09-23T21:00:28.310 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.7090000000000001e-06 2023-09-23T21:00:28.310 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:28.310 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:28.310 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:00:28.311 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:34.935261+0000", 2023-09-23T21:00:28.311 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0011200369999999999 2023-09-23T21:00:28.311 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:28.311 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:28.311 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:00:28.311 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:34.935272+0000", 2023-09-23T21:00:28.311 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.1202999999999999e-05 2023-09-23T21:00:28.311 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:28.311 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:28.311 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:00:28.311 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:35.947706+0000", 2023-09-23T21:00:28.311 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.012434187 2023-09-23T21:00:28.311 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:28.312 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:28.312 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:00:28.312 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:35.947714+0000", 2023-09-23T21:00:28.312 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.8990000000000001e-06 2023-09-23T21:00:28.312 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:28.312 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:28.312 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:00:28.312 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:37.039714+0000", 2023-09-23T21:00:28.312 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.091999768 2023-09-23T21:00:28.312 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:28.312 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:28.312 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:00:28.313 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:37.039838+0000", 2023-09-23T21:00:28.313 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00012372300000000001 2023-09-23T21:00:28.313 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:28.313 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:28.313 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 0,6", 2023-09-23T21:00:28.313 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:37.039921+0000", 2023-09-23T21:00:28.313 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.3489e-05 2023-09-23T21:00:28.313 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:28.313 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:28.313 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:00:28.314 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:37.053854+0000", 2023-09-23T21:00:28.314 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.013933164 2023-09-23T21:00:28.314 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:28.314 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:28.314 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:00:28.314 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:37.054159+0000", 2023-09-23T21:00:28.314 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00030508299999999999 2023-09-23T21:00:28.314 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:28.314 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:28.314 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:00:28.314 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:37.061887+0000", 2023-09-23T21:00:28.315 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0077278579999999998 2023-09-23T21:00:28.315 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:28.315 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:28.315 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:00:28.315 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:37.061930+0000", 2023-09-23T21:00:28.315 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.2454999999999999e-05 2023-09-23T21:00:28.315 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:28.315 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:28.315 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:00:28.315 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:37.062042+0000", 2023-09-23T21:00:28.315 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000112333 2023-09-23T21:00:28.315 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:00:28.316 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:00:28.316 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:00:28.316 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:28.316 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:28.316 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.34797.0:177920 6.a 6:5d69f263:::benchmark_data_smithi026_223868_object76794:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e164)", 2023-09-23T21:00:28.316 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T20:55:34.954622+0000", 2023-09-23T21:00:28.316 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 293.26167109400001, 2023-09-23T21:00:28.316 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.1076863440000002, 2023-09-23T21:00:28.316 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:00:28.316 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:00:28.316 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:00:28.316 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.34797", 2023-09-23T21:00:28.317 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T21:00:28.317 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 177920 2023-09-23T21:00:28.317 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:28.317 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:00:28.317 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:28.317 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:00:28.317 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:34.954622+0000", 2023-09-23T21:00:28.317 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:00:28.317 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:28.317 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:28.317 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:00:28.317 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:34.954622+0000", 2023-09-23T21:00:28.317 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:00:28.318 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:28.318 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:28.318 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:00:28.318 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:34.954621+0000", 2023-09-23T21:00:28.318 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T21:00:28.318 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:28.318 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:28.318 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:00:28.318 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:34.954627+0000", 2023-09-23T21:00:28.318 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.7440000000000004e-06 2023-09-23T21:00:28.318 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:28.318 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:28.318 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:00:28.318 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:34.954638+0000", 2023-09-23T21:00:28.319 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.1487e-05 2023-09-23T21:00:28.319 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:28.319 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:28.319 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:00:28.319 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:34.954644+0000", 2023-09-23T21:00:28.319 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.0519999999999998e-06 2023-09-23T21:00:28.319 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:28.319 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:28.319 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:00:28.319 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:34.954700+0000", 2023-09-23T21:00:28.319 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.5337999999999998e-05 2023-09-23T21:00:28.319 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:28.320 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:28.320 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:00:28.320 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:34.954713+0000", 2023-09-23T21:00:28.320 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.3003000000000001e-05 2023-09-23T21:00:28.320 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:28.320 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:28.320 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:00:28.320 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:35.947751+0000", 2023-09-23T21:00:28.320 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.99303834199999996 2023-09-23T21:00:28.320 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:28.320 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:28.320 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:00:28.320 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:35.947761+0000", 2023-09-23T21:00:28.321 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.4979999999999995e-06 2023-09-23T21:00:28.321 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:28.321 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:28.321 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:00:28.321 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:37.042465+0000", 2023-09-23T21:00:28.321 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.0947049710000001 2023-09-23T21:00:28.321 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:28.321 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:28.321 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:00:28.321 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:37.042599+0000", 2023-09-23T21:00:28.321 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00013388199999999999 2023-09-23T21:00:28.321 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:28.321 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:28.322 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 0,6", 2023-09-23T21:00:28.322 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:37.042722+0000", 2023-09-23T21:00:28.322 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000122893 2023-09-23T21:00:28.322 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:28.322 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:28.322 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:00:28.322 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:37.053862+0000", 2023-09-23T21:00:28.322 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.011139540999999999 2023-09-23T21:00:28.322 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:28.322 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:28.322 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:00:28.322 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:37.057624+0000", 2023-09-23T21:00:28.322 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.003762661 2023-09-23T21:00:28.322 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:28.323 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:28.323 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:00:28.323 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:37.062109+0000", 2023-09-23T21:00:28.323 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0044847259999999996 2023-09-23T21:00:28.323 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:28.323 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:28.323 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:00:28.323 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:37.062163+0000", 2023-09-23T21:00:28.323 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.346e-05 2023-09-23T21:00:28.323 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:28.323 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:28.323 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:00:28.323 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:37.062309+0000", 2023-09-23T21:00:28.324 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00014601300000000001 2023-09-23T21:00:28.324 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:00:28.324 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:00:28.324 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:00:28.324 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:28.324 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:28.324 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.34797.0:177923 6.a 6:53679877:::benchmark_data_smithi026_223868_object76797:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e164)", 2023-09-23T21:00:28.324 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T20:55:34.956263+0000", 2023-09-23T21:00:28.324 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 293.26003021899999, 2023-09-23T21:00:28.324 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.1062447940000002, 2023-09-23T21:00:28.324 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:00:28.324 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:00:28.324 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:00:28.325 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.34797", 2023-09-23T21:00:28.325 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T21:00:28.325 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 177923 2023-09-23T21:00:28.325 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:28.325 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:00:28.325 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:28.325 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:00:28.325 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:34.956263+0000", 2023-09-23T21:00:28.325 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:00:28.325 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:28.325 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:28.325 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:00:28.326 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:34.956263+0000", 2023-09-23T21:00:28.326 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:00:28.326 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:28.326 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:28.326 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:00:28.326 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:34.956262+0000", 2023-09-23T21:00:28.326 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T21:00:28.326 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:28.326 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:28.326 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:00:28.326 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:34.956268+0000", 2023-09-23T21:00:28.326 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.6189999999999996e-06 2023-09-23T21:00:28.326 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:28.327 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:28.327 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:00:28.327 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:34.956283+0000", 2023-09-23T21:00:28.327 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.4725999999999999e-05 2023-09-23T21:00:28.327 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:28.327 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:28.327 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:00:28.327 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:34.956289+0000", 2023-09-23T21:00:28.327 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.1380000000000004e-06 2023-09-23T21:00:28.327 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:28.327 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:28.328 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:00:28.328 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:34.956345+0000", 2023-09-23T21:00:28.328 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.5288000000000003e-05 2023-09-23T21:00:28.328 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:28.328 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:28.328 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:00:28.328 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:34.956354+0000", 2023-09-23T21:00:28.328 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.9150000000000002e-06 2023-09-23T21:00:28.328 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:28.328 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:28.328 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:00:28.328 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:35.947803+0000", 2023-09-23T21:00:28.328 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.99144918699999995 2023-09-23T21:00:28.329 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:28.329 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:28.329 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:00:28.329 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:35.947811+0000", 2023-09-23T21:00:28.329 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.5129999999999996e-06 2023-09-23T21:00:28.329 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:28.329 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:28.329 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:00:28.329 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:37.047285+0000", 2023-09-23T21:00:28.329 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.099473857 2023-09-23T21:00:28.329 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:28.329 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:28.329 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:00:28.330 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:37.047408+0000", 2023-09-23T21:00:28.330 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00012244 2023-09-23T21:00:28.330 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:28.330 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:28.330 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 0,6", 2023-09-23T21:00:28.330 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:37.047526+0000", 2023-09-23T21:00:28.330 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000118539 2023-09-23T21:00:28.330 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:28.330 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:28.330 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:00:28.330 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:37.053868+0000", 2023-09-23T21:00:28.330 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0063422560000000001 2023-09-23T21:00:28.331 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:28.331 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:28.331 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:00:28.331 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:37.058203+0000", 2023-09-23T21:00:28.331 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0043349929999999997 2023-09-23T21:00:28.331 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:28.331 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:28.331 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:00:28.331 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:37.062353+0000", 2023-09-23T21:00:28.331 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0041494890000000001 2023-09-23T21:00:28.331 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:28.331 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:28.331 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:00:28.332 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:37.062398+0000", 2023-09-23T21:00:28.332 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.5639999999999997e-05 2023-09-23T21:00:28.332 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:28.332 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:28.332 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:00:28.332 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:37.062508+0000", 2023-09-23T21:00:28.332 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00010949400000000001 2023-09-23T21:00:28.332 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:00:28.332 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:00:28.332 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:00:28.332 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:28.332 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:28.332 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.34797.0:177948 6.a 6:597c0f94:::benchmark_data_smithi026_223868_object76822:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e164)", 2023-09-23T21:00:28.333 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T20:55:34.996671+0000", 2023-09-23T21:00:28.333 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 293.21962237100001, 2023-09-23T21:00:28.333 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.06610118, 2023-09-23T21:00:28.333 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:00:28.333 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:00:28.333 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:00:28.333 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.34797", 2023-09-23T21:00:28.333 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T21:00:28.333 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 177948 2023-09-23T21:00:28.333 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:28.333 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:00:28.333 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:28.333 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:00:28.334 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:34.996671+0000", 2023-09-23T21:00:28.334 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:00:28.334 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:28.334 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:28.334 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:00:28.334 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:34.996671+0000", 2023-09-23T21:00:28.334 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:00:28.334 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:28.334 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:28.334 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:00:28.334 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:34.996670+0000", 2023-09-23T21:00:28.334 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.999999 2023-09-23T21:00:28.335 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:28.335 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:28.335 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:00:28.335 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:34.996675+0000", 2023-09-23T21:00:28.335 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.6249999999999998e-06 2023-09-23T21:00:28.335 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:28.335 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:28.335 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:00:28.335 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:34.996684+0000", 2023-09-23T21:00:28.335 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.2450000000000004e-06 2023-09-23T21:00:28.335 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:28.335 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:28.335 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:00:28.336 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:34.996687+0000", 2023-09-23T21:00:28.336 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.563e-06 2023-09-23T21:00:28.336 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:28.336 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:28.336 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:00:28.336 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:34.996739+0000", 2023-09-23T21:00:28.336 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.1579000000000002e-05 2023-09-23T21:00:28.336 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:28.336 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:28.336 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:00:28.336 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:34.996746+0000", 2023-09-23T21:00:28.336 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.2690000000000002e-06 2023-09-23T21:00:28.336 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:28.336 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:28.337 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:00:28.337 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:35.947834+0000", 2023-09-23T21:00:28.337 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.95108812399999998 2023-09-23T21:00:28.337 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:28.337 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:28.337 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:00:28.337 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:35.947842+0000", 2023-09-23T21:00:28.337 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.7370000000000006e-06 2023-09-23T21:00:28.337 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:28.337 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:28.337 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:00:28.337 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:37.050299+0000", 2023-09-23T21:00:28.337 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.1024569989999999 2023-09-23T21:00:28.338 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:28.338 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:28.338 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:00:28.338 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:37.050434+0000", 2023-09-23T21:00:28.338 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00013435199999999999 2023-09-23T21:00:28.338 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:28.338 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:28.338 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 0,6", 2023-09-23T21:00:28.338 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:37.050565+0000", 2023-09-23T21:00:28.338 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00013146300000000001 2023-09-23T21:00:28.338 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:28.338 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:28.338 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:00:28.339 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:37.053877+0000", 2023-09-23T21:00:28.339 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0033121589999999998 2023-09-23T21:00:28.339 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:28.339 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:28.339 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:00:28.339 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:37.058255+0000", 2023-09-23T21:00:28.339 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0043776930000000002 2023-09-23T21:00:28.339 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:28.339 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:28.339 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:00:28.339 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:37.062560+0000", 2023-09-23T21:00:28.339 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0043055719999999997 2023-09-23T21:00:28.339 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:28.340 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:28.340 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:00:28.340 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:37.062636+0000", 2023-09-23T21:00:28.340 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.5417e-05 2023-09-23T21:00:28.340 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:28.340 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:28.340 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:00:28.340 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:37.062772+0000", 2023-09-23T21:00:28.340 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00013637499999999999 2023-09-23T21:00:28.340 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:00:28.340 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:00:28.340 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:00:28.340 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:28.340 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:28.341 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.34797.0:184464 6.17 6:ec4eeb9e:::benchmark_data_smithi026_223868_object83338:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e170)", 2023-09-23T21:00:28.341 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T20:55:46.018382+0000", 2023-09-23T21:00:28.341 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 282.19791096199998, 2023-09-23T21:00:28.341 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.97416476299999999, 2023-09-23T21:00:28.341 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:00:28.341 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:00:28.341 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:00:28.341 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.34797", 2023-09-23T21:00:28.341 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T21:00:28.341 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 184464 2023-09-23T21:00:28.341 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:28.341 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:00:28.341 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:28.342 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:00:28.342 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:46.018382+0000", 2023-09-23T21:00:28.342 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:00:28.342 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:28.342 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:28.342 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:00:28.342 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:46.018382+0000", 2023-09-23T21:00:28.342 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:00:28.342 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:28.342 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:28.343 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:00:28.343 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:46.018382+0000", 2023-09-23T21:00:28.343 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999995 2023-09-23T21:00:28.343 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:28.343 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:28.343 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:00:28.343 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:46.018385+0000", 2023-09-23T21:00:28.343 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.571e-06 2023-09-23T21:00:28.343 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:28.343 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:28.344 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:00:28.344 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:46.018390+0000", 2023-09-23T21:00:28.344 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.6629999999999998e-06 2023-09-23T21:00:28.344 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:28.344 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:28.344 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:00:28.344 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:46.018394+0000", 2023-09-23T21:00:28.344 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.8659999999999999e-06 2023-09-23T21:00:28.344 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:28.344 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:28.345 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:00:28.345 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:46.018524+0000", 2023-09-23T21:00:28.345 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00012990000000000001 2023-09-23T21:00:28.345 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:28.345 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:28.345 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:00:28.345 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:46.018529+0000", 2023-09-23T21:00:28.345 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.1680000000000003e-06 2023-09-23T21:00:28.345 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:28.345 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:28.345 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:00:28.345 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:46.969211+0000", 2023-09-23T21:00:28.345 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.95068135499999995 2023-09-23T21:00:28.346 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:28.346 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:28.346 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:00:28.346 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:46.969338+0000", 2023-09-23T21:00:28.346 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000127699 2023-09-23T21:00:28.346 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:28.346 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:28.346 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 3,4", 2023-09-23T21:00:28.346 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:46.969463+0000", 2023-09-23T21:00:28.346 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000124833 2023-09-23T21:00:28.346 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:28.346 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:28.346 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:00:28.347 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:46.975159+0000", 2023-09-23T21:00:28.347 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0056955929999999997 2023-09-23T21:00:28.347 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:28.347 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:28.347 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:00:28.347 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:46.976924+0000", 2023-09-23T21:00:28.347 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.001765567 2023-09-23T21:00:28.347 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:28.347 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:28.347 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:00:28.347 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:46.992475+0000", 2023-09-23T21:00:28.347 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.015550402 2023-09-23T21:00:28.347 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:28.348 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:28.348 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:00:28.348 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:46.992514+0000", 2023-09-23T21:00:28.348 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.9075000000000001e-05 2023-09-23T21:00:28.348 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:28.348 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:28.348 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:00:28.348 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:46.992547+0000", 2023-09-23T21:00:28.348 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.3463999999999997e-05 2023-09-23T21:00:28.348 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:00:28.348 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:00:28.348 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:00:28.348 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:28.349 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:28.349 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.34797.0:185827 6.a 6:51461ffe:::benchmark_data_smithi026_223868_object84701:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e172)", 2023-09-23T21:00:28.349 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T20:55:48.221799+0000", 2023-09-23T21:00:28.349 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 279.99449425900002, 2023-09-23T21:00:28.349 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.038071078, 2023-09-23T21:00:28.349 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:00:28.349 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:00:28.349 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:00:28.349 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.34797", 2023-09-23T21:00:28.349 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T21:00:28.349 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 185827 2023-09-23T21:00:28.350 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:28.350 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:00:28.350 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:28.350 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:00:28.350 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:48.221799+0000", 2023-09-23T21:00:28.350 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:00:28.350 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:28.350 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:28.350 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:00:28.350 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:48.221799+0000", 2023-09-23T21:00:28.350 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:00:28.351 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:28.351 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:28.351 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:00:28.351 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:48.221799+0000", 2023-09-23T21:00:28.351 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999995 2023-09-23T21:00:28.351 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:28.351 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:28.351 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:00:28.351 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:48.221802+0000", 2023-09-23T21:00:28.351 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.2049999999999998e-06 2023-09-23T21:00:28.351 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:28.351 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:28.351 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:00:28.352 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:48.221808+0000", 2023-09-23T21:00:28.352 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.19e-06 2023-09-23T21:00:28.352 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:28.352 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:28.352 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:00:28.352 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:48.221811+0000", 2023-09-23T21:00:28.352 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.52e-06 2023-09-23T21:00:28.352 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:28.352 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:28.352 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:00:28.352 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:48.221859+0000", 2023-09-23T21:00:28.352 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.8857999999999999e-05 2023-09-23T21:00:28.352 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:28.353 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:28.353 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:00:28.353 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:48.221871+0000", 2023-09-23T21:00:28.353 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.1174e-05 2023-09-23T21:00:28.353 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:28.353 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:28.353 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:00:28.353 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:49.249764+0000", 2023-09-23T21:00:28.353 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.027893758 2023-09-23T21:00:28.353 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:28.353 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:28.353 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:00:28.353 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:49.249884+0000", 2023-09-23T21:00:28.354 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000119438 2023-09-23T21:00:28.354 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:28.354 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:28.354 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 0,6", 2023-09-23T21:00:28.354 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:49.249965+0000", 2023-09-23T21:00:28.354 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.1518000000000001e-05 2023-09-23T21:00:28.354 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:28.354 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:28.354 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:00:28.354 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:49.252936+0000", 2023-09-23T21:00:28.355 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0029711009999999999 2023-09-23T21:00:28.355 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:28.355 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:28.355 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:00:28.355 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:49.256217+0000", 2023-09-23T21:00:28.355 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0032802489999999998 2023-09-23T21:00:28.355 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:28.355 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:28.355 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:00:28.355 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:49.259769+0000", 2023-09-23T21:00:28.355 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0035519509999999998 2023-09-23T21:00:28.355 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:28.356 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:28.356 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:00:28.356 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:49.259825+0000", 2023-09-23T21:00:28.356 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.6854999999999997e-05 2023-09-23T21:00:28.356 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:28.356 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:28.356 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:00:28.356 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:49.259870+0000", 2023-09-23T21:00:28.356 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.4774999999999997e-05 2023-09-23T21:00:28.356 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:00:28.356 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:00:28.356 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:00:28.356 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:28.357 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:28.357 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.34797.0:185834 6.a 6:59f98fb0:::benchmark_data_smithi026_223868_object84708:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e172)", 2023-09-23T21:00:28.357 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T20:55:48.229707+0000", 2023-09-23T21:00:28.357 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 279.98658643099998, 2023-09-23T21:00:28.357 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.0303501639999999, 2023-09-23T21:00:28.357 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:00:28.357 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:00:28.357 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:00:28.357 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.34797", 2023-09-23T21:00:28.357 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T21:00:28.357 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 185834 2023-09-23T21:00:28.357 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:28.357 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:00:28.358 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:28.358 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:00:28.358 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:48.229707+0000", 2023-09-23T21:00:28.358 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:00:28.358 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:28.358 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:28.358 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:00:28.358 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:48.229707+0000", 2023-09-23T21:00:28.358 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:00:28.358 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:28.358 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:28.358 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:00:28.358 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:48.229706+0000", 2023-09-23T21:00:28.359 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.999999 2023-09-23T21:00:28.359 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:28.359 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:28.359 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:00:28.359 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:48.229712+0000", 2023-09-23T21:00:28.359 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.7400000000000001e-06 2023-09-23T21:00:28.359 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:28.359 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:28.359 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:00:28.359 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:48.229724+0000", 2023-09-23T21:00:28.359 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.2797e-05 2023-09-23T21:00:28.359 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:28.360 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:28.360 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:00:28.360 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:48.229731+0000", 2023-09-23T21:00:28.360 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.5080000000000002e-06 2023-09-23T21:00:28.360 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:28.360 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:28.360 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:00:28.360 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:48.229783+0000", 2023-09-23T21:00:28.360 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.1953000000000002e-05 2023-09-23T21:00:28.360 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:28.360 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:28.360 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:00:28.361 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:48.229794+0000", 2023-09-23T21:00:28.361 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.1061e-05 2023-09-23T21:00:28.361 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:28.361 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:28.361 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:00:28.361 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:49.253001+0000", 2023-09-23T21:00:28.361 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.023207209 2023-09-23T21:00:28.361 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:28.361 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:28.361 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:00:28.361 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:49.253151+0000", 2023-09-23T21:00:28.361 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00014949599999999999 2023-09-23T21:00:28.361 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:28.362 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:28.362 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 0,6", 2023-09-23T21:00:28.362 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:49.253282+0000", 2023-09-23T21:00:28.362 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000131922 2023-09-23T21:00:28.362 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:28.362 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:28.362 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:00:28.362 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:49.259649+0000", 2023-09-23T21:00:28.362 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0063662659999999998 2023-09-23T21:00:28.362 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:28.362 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:28.362 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:00:28.363 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:49.259934+0000", 2023-09-23T21:00:28.363 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00028488500000000002 2023-09-23T21:00:28.363 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:28.363 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:28.363 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:00:28.363 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:49.259975+0000", 2023-09-23T21:00:28.363 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.1811000000000003e-05 2023-09-23T21:00:28.363 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:28.363 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:28.363 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:00:28.363 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:49.260024+0000", 2023-09-23T21:00:28.363 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.8155000000000002e-05 2023-09-23T21:00:28.364 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:28.364 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:28.364 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:00:28.364 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:49.260057+0000", 2023-09-23T21:00:28.364 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.3522000000000002e-05 2023-09-23T21:00:28.364 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:00:28.364 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:00:28.364 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:00:28.364 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:28.364 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:28.364 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.34797.0:185844 6.a 6:5b27a6f2:::benchmark_data_smithi026_223868_object84718:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e172)", 2023-09-23T21:00:28.364 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T20:55:48.237898+0000", 2023-09-23T21:00:28.364 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 279.97839520600002, 2023-09-23T21:00:28.365 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.022291493, 2023-09-23T21:00:28.365 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:00:28.365 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:00:28.365 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:00:28.365 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.34797", 2023-09-23T21:00:28.365 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T21:00:28.365 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 185844 2023-09-23T21:00:28.365 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:28.365 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:00:28.365 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:28.365 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:00:28.365 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:48.237898+0000", 2023-09-23T21:00:28.365 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:00:28.366 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:28.366 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:28.366 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:00:28.366 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:48.237898+0000", 2023-09-23T21:00:28.366 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:00:28.366 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:28.366 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:28.366 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:00:28.366 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:48.237897+0000", 2023-09-23T21:00:28.366 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.999999 2023-09-23T21:00:28.366 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:28.366 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:28.367 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:00:28.367 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:48.237903+0000", 2023-09-23T21:00:28.367 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.2179999999999998e-06 2023-09-23T21:00:28.367 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:28.367 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:28.367 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:00:28.367 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:48.237916+0000", 2023-09-23T21:00:28.367 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.3753e-05 2023-09-23T21:00:28.367 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:28.367 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:28.367 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:00:28.367 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:48.237923+0000", 2023-09-23T21:00:28.367 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.6200000000000001e-06 2023-09-23T21:00:28.368 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:28.368 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:28.368 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:00:28.368 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:48.237984+0000", 2023-09-23T21:00:28.368 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.1314000000000003e-05 2023-09-23T21:00:28.368 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:28.368 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:28.368 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:00:28.368 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:48.237994+0000", 2023-09-23T21:00:28.368 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.8050000000000001e-06 2023-09-23T21:00:28.368 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:28.368 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:28.369 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:00:28.369 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:49.256263+0000", 2023-09-23T21:00:28.369 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.018269149 2023-09-23T21:00:28.369 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:28.369 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:28.369 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:00:28.369 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:49.256351+0000", 2023-09-23T21:00:28.369 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.8261000000000001e-05 2023-09-23T21:00:28.369 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:28.369 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:28.369 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 0,6", 2023-09-23T21:00:28.369 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:49.256430+0000", 2023-09-23T21:00:28.369 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.8651000000000003e-05 2023-09-23T21:00:28.369 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:28.370 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:28.370 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:00:28.370 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:49.259924+0000", 2023-09-23T21:00:28.370 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0034939390000000002 2023-09-23T21:00:28.370 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:28.370 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:28.370 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:00:28.370 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:49.259942+0000", 2023-09-23T21:00:28.370 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.8261999999999999e-05 2023-09-23T21:00:28.370 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:28.370 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:28.371 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:00:28.371 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:49.260103+0000", 2023-09-23T21:00:28.371 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000160679 2023-09-23T21:00:28.371 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:28.371 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:28.371 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:00:28.371 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:49.260148+0000", 2023-09-23T21:00:28.371 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.5330999999999997e-05 2023-09-23T21:00:28.371 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:28.371 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:28.371 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:00:28.371 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:49.260190+0000", 2023-09-23T21:00:28.371 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.1409999999999998e-05 2023-09-23T21:00:28.372 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:00:28.372 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:00:28.372 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:00:28.372 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:28.372 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:28.372 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.34797.0:185884 6.a 6:5db88532:::benchmark_data_smithi026_223868_object84758:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e172)", 2023-09-23T21:00:28.372 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T20:55:48.282714+0000", 2023-09-23T21:00:28.372 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 279.93357938299999, 2023-09-23T21:00:28.372 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.99305554799999995, 2023-09-23T21:00:28.372 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:00:28.372 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:00:28.372 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:00:28.372 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.34797", 2023-09-23T21:00:28.372 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T21:00:28.373 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 185884 2023-09-23T21:00:28.373 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:28.373 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:00:28.373 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:28.373 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:00:28.373 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:48.282714+0000", 2023-09-23T21:00:28.373 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:00:28.373 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:28.373 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:28.373 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:00:28.373 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:48.282714+0000", 2023-09-23T21:00:28.373 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:00:28.373 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:28.373 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:28.374 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:00:28.374 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:48.282714+0000", 2023-09-23T21:00:28.374 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999995 2023-09-23T21:00:28.374 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:28.374 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:28.374 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:00:28.374 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:48.282717+0000", 2023-09-23T21:00:28.374 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.1760000000000002e-06 2023-09-23T21:00:28.374 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:28.374 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:28.374 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:00:28.374 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:48.282727+0000", 2023-09-23T21:00:28.374 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.8079999999999999e-06 2023-09-23T21:00:28.375 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:28.375 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:28.375 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:00:28.375 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:48.282731+0000", 2023-09-23T21:00:28.375 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.4129999999999999e-06 2023-09-23T21:00:28.375 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:28.375 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:28.375 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:00:28.375 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:48.282781+0000", 2023-09-23T21:00:28.375 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.9870999999999997e-05 2023-09-23T21:00:28.375 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:28.375 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:28.376 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:00:28.376 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:48.282789+0000", 2023-09-23T21:00:28.376 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.1689999999999992e-06 2023-09-23T21:00:28.376 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:28.376 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:28.376 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:00:28.376 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:49.260248+0000", 2023-09-23T21:00:28.376 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.97745879800000002 2023-09-23T21:00:28.376 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:28.376 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:28.376 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:00:28.376 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:49.260330+0000", 2023-09-23T21:00:28.376 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.2013000000000002e-05 2023-09-23T21:00:28.376 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:28.377 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:28.377 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 0,6", 2023-09-23T21:00:28.377 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:49.260394+0000", 2023-09-23T21:00:28.377 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.4393000000000005e-05 2023-09-23T21:00:28.377 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:28.377 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:28.377 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:00:28.377 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:49.269250+0000", 2023-09-23T21:00:28.377 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0088559889999999999 2023-09-23T21:00:28.377 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:28.377 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:28.377 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:00:28.378 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:49.269282+0000", 2023-09-23T21:00:28.378 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.1733999999999998e-05 2023-09-23T21:00:28.378 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:28.378 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:28.378 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:00:28.378 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:49.275670+0000", 2023-09-23T21:00:28.378 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0063884220000000004 2023-09-23T21:00:28.378 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:28.378 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:28.379 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:00:28.379 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:49.275729+0000", 2023-09-23T21:00:28.379 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.8371000000000001e-05 2023-09-23T21:00:28.379 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:28.379 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:28.379 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:00:28.379 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:49.275770+0000", 2023-09-23T21:00:28.379 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.0884999999999999e-05 2023-09-23T21:00:28.379 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:00:28.380 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:00:28.380 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:00:28.380 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:00:28.380 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:00:28.380 INFO:teuthology.orchestra.run.smithi070.stdout:} 2023-09-23T21:00:28.381 INFO:teuthology.orchestra.run.smithi026.stderr:dumped all 2023-09-23T21:00:28.381 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:00:28.595 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_ops_in_flight 2023-09-23T21:00:28.844 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T21:00:28.851 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:00:28.852 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:00:28.858 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:00:28 smithi026 ceph-mon[175151]: from='client.55234 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T21:00:28.858 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:00:28 smithi026 ceph-mon[179688]: from='client.55234 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T21:00:28.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:00:28 smithi070 ceph-mon[168282]: from='client.55234 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T21:00:28.953 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:00:28.963 INFO:teuthology.orchestra.run.smithi026.stdout:{"pg_ready":true,"pg_map":{"version":625,"stamp":"2023-09-23T21:00:26.486458+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":1461655480,"num_objects":22319,"num_object_clones":0,"num_object_copies":66957,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":11892,"num_objects_misplaced":522,"num_objects_unfound":0,"num_objects_dirty":22319,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":181286,"num_write_kb":6512606,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3950,"num_bytes_recovered":257431398,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":122461,"ondisk_log_size":122461,"up":327,"acting":324,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":331,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":7019992,"kb_used_data":5328216,"kb_used_omap":136,"kb_used_meta":1691127,"kb_avail":742941224,"statfs":{"total":767960285184,"available":760771813376,"internally_reserved":0,"allocated":5456093184,"data_stored":5447881937,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":139741,"internal_metadata":1731714595},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[3,1,5,5,6,7,1],"upper_bound":128},"perf_stat":{"commit_latency_ms":8,"apply_latency_ms":8,"commit_latency_ns":8000000,"apply_latency_ns":8000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-396230656,"num_objects":-6046,"num_object_clones":0,"num_object_copies":-18138,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":18,"num_objects_misplaced":-378,"num_objects_unfound":0,"num_objects_dirty":-6046,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6046,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":360,"num_bytes_recovered":23592960,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.007104"},"pg_stats":[{"pgid":"3.1a","version":"68'1","reported_seq":123,"reported_epoch":245,"state":"active+clean","last_fresh":"2023-09-23T21:00:13.257335+0000","last_change":"2023-09-23T20:54:13.946695+0000","last_active":"2023-09-23T21:00:13.257335+0000","last_peered":"2023-09-23T21:00:13.257335+0000","last_clean":"2023-09-23T21:00:13.257335+0000","last_became_active":"2023-09-23T20:54:13.945399+0000","last_became_peered":"2023-09-23T20:54:13.945399+0000","last_unstale":"2023-09-23T21:00:13.257335+0000","last_undegraded":"2023-09-23T21:00:13.257335+0000","last_fullsized":"2023-09-23T21:00:13.257335+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:32:52.365464+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":238,"reported_epoch":245,"state":"active+clean","last_fresh":"2023-09-23T21:00:13.249681+0000","last_change":"2023-09-23T20:54:13.942019+0000","last_active":"2023-09-23T21:00:13.249681+0000","last_peered":"2023-09-23T21:00:13.249681+0000","last_clean":"2023-09-23T21:00:13.249681+0000","last_became_active":"2023-09-23T20:54:13.941647+0000","last_became_peered":"2023-09-23T20:54:13.941647+0000","last_unstale":"2023-09-23T21:00:13.249681+0000","last_undegraded":"2023-09-23T21:00:13.249681+0000","last_fullsized":"2023-09-23T21:00:13.249681+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:30:54.400944+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":216,"reported_epoch":245,"state":"active+clean","last_fresh":"2023-09-23T21:00:13.275343+0000","last_change":"2023-09-23T20:54:13.941394+0000","last_active":"2023-09-23T21:00:13.275343+0000","last_peered":"2023-09-23T21:00:13.275343+0000","last_clean":"2023-09-23T21:00:13.275343+0000","last_became_active":"2023-09-23T20:54:13.941042+0000","last_became_peered":"2023-09-23T20:54:13.941042+0000","last_unstale":"2023-09-23T21:00:13.275343+0000","last_undegraded":"2023-09-23T21:00:13.275343+0000","last_fullsized":"2023-09-23T21:00:13.275343+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:19:41.401184+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":231,"reported_epoch":245,"state":"active+clean","last_fresh":"2023-09-23T21:00:13.270553+0000","last_change":"2023-09-23T20:57:44.979811+0000","last_active":"2023-09-23T21:00:13.270553+0000","last_peered":"2023-09-23T21:00:13.270553+0000","last_clean":"2023-09-23T21:00:13.270553+0000","last_became_active":"2023-09-23T20:57:44.979515+0000","last_became_peered":"2023-09-23T20:57:44.979515+0000","last_unstale":"2023-09-23T21:00:13.270553+0000","last_undegraded":"2023-09-23T21:00:13.270553+0000","last_fullsized":"2023-09-23T21:00:13.270553+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:38:22.162140+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":229,"reported_epoch":245,"state":"active+clean","last_fresh":"2023-09-23T21:00:13.249677+0000","last_change":"2023-09-23T20:51:10.833613+0000","last_active":"2023-09-23T21:00:13.249677+0000","last_peered":"2023-09-23T21:00:13.249677+0000","last_clean":"2023-09-23T21:00:13.249677+0000","last_became_active":"2023-09-23T20:51:10.833208+0000","last_became_peered":"2023-09-23T20:51:10.833208+0000","last_unstale":"2023-09-23T21:00:13.249677+0000","last_undegraded":"2023-09-23T21:00:13.249677+0000","last_fullsized":"2023-09-23T21:00:13.249677+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:51:28.246014+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":211,"reported_epoch":245,"state":"active+clean","last_fresh":"2023-09-23T21:00:13.240996+0000","last_change":"2023-09-23T20:55:35.958384+0000","last_active":"2023-09-23T21:00:13.240996+0000","last_peered":"2023-09-23T21:00:13.240996+0000","last_clean":"2023-09-23T21:00:13.240996+0000","last_became_active":"2023-09-23T20:55:35.954563+0000","last_became_peered":"2023-09-23T20:55:35.954563+0000","last_unstale":"2023-09-23T21:00:13.240996+0000","last_undegraded":"2023-09-23T21:00:13.240996+0000","last_fullsized":"2023-09-23T21:00:13.240996+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:37:48.332259+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":221,"reported_epoch":245,"state":"active+clean","last_fresh":"2023-09-23T21:00:13.270906+0000","last_change":"2023-09-23T20:54:31.885514+0000","last_active":"2023-09-23T21:00:13.270906+0000","last_peered":"2023-09-23T21:00:13.270906+0000","last_clean":"2023-09-23T21:00:13.270906+0000","last_became_active":"2023-09-23T20:54:31.884986+0000","last_became_peered":"2023-09-23T20:54:31.884986+0000","last_unstale":"2023-09-23T21:00:13.270906+0000","last_undegraded":"2023-09-23T21:00:13.270906+0000","last_fullsized":"2023-09-23T21:00:13.270906+0000","mapping_epoch":136,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:26:08.100220+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":226,"reported_epoch":245,"state":"active+clean","last_fresh":"2023-09-23T21:00:13.249754+0000","last_change":"2023-09-23T20:55:35.951889+0000","last_active":"2023-09-23T21:00:13.249754+0000","last_peered":"2023-09-23T21:00:13.249754+0000","last_clean":"2023-09-23T21:00:13.249754+0000","last_became_active":"2023-09-23T20:55:35.951563+0000","last_became_peered":"2023-09-23T20:55:35.951563+0000","last_unstale":"2023-09-23T21:00:13.249754+0000","last_undegraded":"2023-09-23T21:00:13.249754+0000","last_fullsized":"2023-09-23T21:00:13.249754+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:41:53.098781+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"79'161","reported_seq":533,"reported_epoch":245,"state":"active+clean","last_fresh":"2023-09-23T21:00:13.262029+0000","last_change":"2023-09-23T20:57:25.096559+0000","last_active":"2023-09-23T21:00:13.262029+0000","last_peered":"2023-09-23T21:00:13.262029+0000","last_clean":"2023-09-23T21:00:13.262029+0000","last_became_active":"2023-09-23T20:57:15.975674+0000","last_became_peered":"2023-09-23T20:57:15.975674+0000","last_unstale":"2023-09-23T21:00:13.262029+0000","last_undegraded":"2023-09-23T21:00:13.262029+0000","last_fullsized":"2023-09-23T21:00:13.262029+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:25.096401+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:25.096401+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:05:05.448404+0000","scrub_duration":0.002859393,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":119,"reported_epoch":245,"state":"active+clean","last_fresh":"2023-09-23T21:00:13.251795+0000","last_change":"2023-09-23T20:54:31.890678+0000","last_active":"2023-09-23T21:00:13.251795+0000","last_peered":"2023-09-23T21:00:13.251795+0000","last_clean":"2023-09-23T21:00:13.251795+0000","last_became_active":"2023-09-23T20:54:31.890469+0000","last_became_peered":"2023-09-23T20:54:31.890469+0000","last_unstale":"2023-09-23T21:00:13.251795+0000","last_undegraded":"2023-09-23T21:00:13.251795+0000","last_fullsized":"2023-09-23T21:00:13.251795+0000","mapping_epoch":136,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:55:08.790146+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":187,"reported_epoch":245,"state":"active+clean","last_fresh":"2023-09-23T21:00:13.250058+0000","last_change":"2023-09-23T20:55:35.952661+0000","last_active":"2023-09-23T21:00:13.250058+0000","last_peered":"2023-09-23T21:00:13.250058+0000","last_clean":"2023-09-23T21:00:13.250058+0000","last_became_active":"2023-09-23T20:55:35.952349+0000","last_became_peered":"2023-09-23T20:55:35.952349+0000","last_unstale":"2023-09-23T21:00:13.250058+0000","last_undegraded":"2023-09-23T21:00:13.250058+0000","last_fullsized":"2023-09-23T21:00:13.250058+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:00:03.110126+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":64,"reported_epoch":245,"state":"active+clean","last_fresh":"2023-09-23T21:00:13.249780+0000","last_change":"2023-09-23T20:56:45.988607+0000","last_active":"2023-09-23T21:00:13.249780+0000","last_peered":"2023-09-23T21:00:13.249780+0000","last_clean":"2023-09-23T21:00:13.249780+0000","last_became_active":"2023-09-23T20:56:45.988206+0000","last_became_peered":"2023-09-23T20:56:45.988206+0000","last_unstale":"2023-09-23T21:00:13.249780+0000","last_undegraded":"2023-09-23T21:00:13.249780+0000","last_fullsized":"2023-09-23T21:00:13.249780+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":205,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:01:19.783688+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,4],"acting":[3,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":63,"reported_epoch":245,"state":"active+clean","last_fresh":"2023-09-23T21:00:13.253507+0000","last_change":"2023-09-23T20:56:18.139142+0000","last_active":"2023-09-23T21:00:13.253507+0000","last_peered":"2023-09-23T21:00:13.253507+0000","last_clean":"2023-09-23T21:00:13.253507+0000","last_became_active":"2023-09-23T20:56:18.138844+0000","last_became_peered":"2023-09-23T20:56:18.138844+0000","last_unstale":"2023-09-23T21:00:13.253507+0000","last_undegraded":"2023-09-23T21:00:13.253507+0000","last_fullsized":"2023-09-23T21:00:13.253507+0000","mapping_epoch":192,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":193,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:09:02.217683+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,2],"acting":[5,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.14","version":"79'22","reported_seq":150,"reported_epoch":245,"state":"active+clean","last_fresh":"2023-09-23T21:00:13.257738+0000","last_change":"2023-09-23T20:55:35.949418+0000","last_active":"2023-09-23T21:00:13.257738+0000","last_peered":"2023-09-23T21:00:13.257738+0000","last_clean":"2023-09-23T21:00:13.257738+0000","last_became_active":"2023-09-23T20:55:35.948566+0000","last_became_peered":"2023-09-23T20:55:35.948566+0000","last_unstale":"2023-09-23T21:00:13.257738+0000","last_undegraded":"2023-09-23T21:00:13.257738+0000","last_fullsized":"2023-09-23T21:00:13.257738+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:59:37.433128+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4734,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":140,"reported_epoch":245,"state":"active+clean","last_fresh":"2023-09-23T21:00:13.257873+0000","last_change":"2023-09-23T20:55:35.949178+0000","last_active":"2023-09-23T21:00:13.257873+0000","last_peered":"2023-09-23T21:00:13.257873+0000","last_clean":"2023-09-23T21:00:13.257873+0000","last_became_active":"2023-09-23T20:55:35.948795+0000","last_became_peered":"2023-09-23T20:55:35.948795+0000","last_unstale":"2023-09-23T21:00:13.257873+0000","last_undegraded":"2023-09-23T21:00:13.257873+0000","last_fullsized":"2023-09-23T21:00:13.257873+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:13:08.996932+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.11","version":"79'161","reported_seq":475,"reported_epoch":245,"state":"active+clean","last_fresh":"2023-09-23T21:00:13.269926+0000","last_change":"2023-09-23T20:57:28.756796+0000","last_active":"2023-09-23T21:00:13.269926+0000","last_peered":"2023-09-23T21:00:13.269926+0000","last_clean":"2023-09-23T21:00:13.269926+0000","last_became_active":"2023-09-23T20:57:15.980606+0000","last_became_peered":"2023-09-23T20:57:15.980606+0000","last_unstale":"2023-09-23T21:00:13.269926+0000","last_undegraded":"2023-09-23T21:00:13.269926+0000","last_fullsized":"2023-09-23T21:00:13.269926+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:28.756634+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:28.756634+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:09:19.150791+0000","scrub_duration":0.0064925119999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":166,"reported_epoch":245,"state":"active+clean","last_fresh":"2023-09-23T21:00:13.257783+0000","last_change":"2023-09-23T20:56:45.992577+0000","last_active":"2023-09-23T21:00:13.257783+0000","last_peered":"2023-09-23T21:00:13.257783+0000","last_clean":"2023-09-23T21:00:13.257783+0000","last_became_active":"2023-09-23T20:56:45.992297+0000","last_became_peered":"2023-09-23T20:56:45.992297+0000","last_unstale":"2023-09-23T21:00:13.257783+0000","last_undegraded":"2023-09-23T21:00:13.257783+0000","last_fullsized":"2023-09-23T21:00:13.257783+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":205,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:00:45.547202+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,2],"acting":[4,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":211,"reported_epoch":245,"state":"active+clean","last_fresh":"2023-09-23T21:00:13.251976+0000","last_change":"2023-09-23T20:51:10.831262+0000","last_active":"2023-09-23T21:00:13.251976+0000","last_peered":"2023-09-23T21:00:13.251976+0000","last_clean":"2023-09-23T21:00:13.251976+0000","last_became_active":"2023-09-23T20:51:10.830931+0000","last_became_peered":"2023-09-23T20:51:10.830931+0000","last_unstale":"2023-09-23T21:00:13.251976+0000","last_undegraded":"2023-09-23T21:00:13.251976+0000","last_fullsized":"2023-09-23T21:00:13.251976+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:51:54.982505+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.10","version":"79'161","reported_seq":483,"reported_epoch":245,"state":"active+clean","last_fresh":"2023-09-23T21:00:13.251825+0000","last_change":"2023-09-23T20:57:20.708003+0000","last_active":"2023-09-23T21:00:13.251825+0000","last_peered":"2023-09-23T21:00:13.251825+0000","last_clean":"2023-09-23T21:00:13.251825+0000","last_became_active":"2023-09-23T20:57:15.975302+0000","last_became_peered":"2023-09-23T20:57:15.975302+0000","last_unstale":"2023-09-23T21:00:13.251825+0000","last_undegraded":"2023-09-23T21:00:13.251825+0000","last_fullsized":"2023-09-23T21:00:13.251825+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:20.707939+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:20.707939+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:16:51.721994+0000","scrub_duration":0.0017733409999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":221,"reported_epoch":245,"state":"active+clean","last_fresh":"2023-09-23T21:00:13.273977+0000","last_change":"2023-09-23T20:57:44.979736+0000","last_active":"2023-09-23T21:00:13.273977+0000","last_peered":"2023-09-23T21:00:13.273977+0000","last_clean":"2023-09-23T21:00:13.273977+0000","last_became_active":"2023-09-23T20:57:44.979356+0000","last_became_peered":"2023-09-23T20:57:44.979356+0000","last_unstale":"2023-09-23T21:00:13.273977+0000","last_undegraded":"2023-09-23T21:00:13.273977+0000","last_fullsized":"2023-09-23T21:00:13.273977+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:14:43.848832+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":119,"reported_epoch":245,"state":"active+clean","last_fresh":"2023-09-23T21:00:13.251346+0000","last_change":"2023-09-23T20:54:31.887814+0000","last_active":"2023-09-23T21:00:13.251346+0000","last_peered":"2023-09-23T21:00:13.251346+0000","last_clean":"2023-09-23T21:00:13.251346+0000","last_became_active":"2023-09-23T20:54:31.887195+0000","last_became_peered":"2023-09-23T20:54:31.887195+0000","last_unstale":"2023-09-23T21:00:13.251346+0000","last_undegraded":"2023-09-23T21:00:13.251346+0000","last_fullsized":"2023-09-23T21:00:13.251346+0000","mapping_epoch":136,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:10:52.089553+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.13","version":"79'161","reported_seq":546,"reported_epoch":245,"state":"active+clean","last_fresh":"2023-09-23T21:00:13.274736+0000","last_change":"2023-09-23T20:57:48.262970+0000","last_active":"2023-09-23T21:00:13.274736+0000","last_peered":"2023-09-23T21:00:13.274736+0000","last_clean":"2023-09-23T21:00:13.274736+0000","last_became_active":"2023-09-23T20:57:48.262663+0000","last_became_peered":"2023-09-23T20:57:48.262663+0000","last_unstale":"2023-09-23T21:00:13.274736+0000","last_undegraded":"2023-09-23T21:00:13.274736+0000","last_fullsized":"2023-09-23T21:00:13.274736+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:36.420362+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:36.420362+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:28:28.613492+0000","scrub_duration":0.028838444000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":190,"reported_epoch":245,"state":"active+clean","last_fresh":"2023-09-23T21:00:13.257920+0000","last_change":"2023-09-23T20:56:45.991822+0000","last_active":"2023-09-23T21:00:13.257920+0000","last_peered":"2023-09-23T21:00:13.257920+0000","last_clean":"2023-09-23T21:00:13.257920+0000","last_became_active":"2023-09-23T20:56:45.991460+0000","last_became_peered":"2023-09-23T20:56:45.991460+0000","last_unstale":"2023-09-23T21:00:13.257920+0000","last_undegraded":"2023-09-23T21:00:13.257920+0000","last_fullsized":"2023-09-23T21:00:13.257920+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":205,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:20:37.503317+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":203,"reported_epoch":245,"state":"active+clean","last_fresh":"2023-09-23T21:00:13.270694+0000","last_change":"2023-09-23T20:51:10.840849+0000","last_active":"2023-09-23T21:00:13.270694+0000","last_peered":"2023-09-23T21:00:13.270694+0000","last_clean":"2023-09-23T21:00:13.270694+0000","last_became_active":"2023-09-23T20:51:10.840593+0000","last_became_peered":"2023-09-23T20:51:10.840593+0000","last_unstale":"2023-09-23T21:00:13.270694+0000","last_undegraded":"2023-09-23T21:00:13.270694+0000","last_fullsized":"2023-09-23T21:00:13.270694+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:30:52.297625+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.12","version":"79'161","reported_seq":483,"reported_epoch":245,"state":"active+clean","last_fresh":"2023-09-23T21:00:13.253220+0000","last_change":"2023-09-23T20:57:21.750765+0000","last_active":"2023-09-23T21:00:13.253220+0000","last_peered":"2023-09-23T21:00:13.253220+0000","last_clean":"2023-09-23T21:00:13.253220+0000","last_became_active":"2023-09-23T20:57:15.979195+0000","last_became_peered":"2023-09-23T20:57:15.979195+0000","last_unstale":"2023-09-23T21:00:13.253220+0000","last_undegraded":"2023-09-23T21:00:13.253220+0000","last_fullsized":"2023-09-23T21:00:13.253220+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:21.750680+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:21.750680+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:03:41.915962+0000","scrub_duration":0.0092391680000000007,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":238,"reported_epoch":245,"state":"active+clean","last_fresh":"2023-09-23T21:00:13.240376+0000","last_change":"2023-09-23T20:55:36.943969+0000","last_active":"2023-09-23T21:00:13.240376+0000","last_peered":"2023-09-23T21:00:13.240376+0000","last_clean":"2023-09-23T21:00:13.240376+0000","last_became_active":"2023-09-23T20:55:36.943580+0000","last_became_peered":"2023-09-23T20:55:36.943580+0000","last_unstale":"2023-09-23T21:00:13.240376+0000","last_undegraded":"2023-09-23T21:00:13.240376+0000","last_fullsized":"2023-09-23T21:00:13.240376+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:49:06.969396+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":35,"reported_epoch":245,"state":"active+clean","last_fresh":"2023-09-23T21:00:13.250058+0000","last_change":"2023-09-23T20:57:44.980498+0000","last_active":"2023-09-23T21:00:13.250058+0000","last_peered":"2023-09-23T21:00:13.250058+0000","last_clean":"2023-09-23T21:00:13.250058+0000","last_became_active":"2023-09-23T20:57:44.980182+0000","last_became_peered":"2023-09-23T20:57:44.980182+0000","last_unstale":"2023-09-23T21:00:13.250058+0000","last_undegraded":"2023-09-23T21:00:13.250058+0000","last_fullsized":"2023-09-23T21:00:13.250058+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:34:51.620544+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.d","version":"79'161","reported_seq":528,"reported_epoch":245,"state":"active+clean","last_fresh":"2023-09-23T21:00:13.257568+0000","last_change":"2023-09-23T20:57:31.075213+0000","last_active":"2023-09-23T21:00:13.257568+0000","last_peered":"2023-09-23T21:00:13.257568+0000","last_clean":"2023-09-23T21:00:13.257568+0000","last_became_active":"2023-09-23T20:57:15.988188+0000","last_became_peered":"2023-09-23T20:57:15.988188+0000","last_unstale":"2023-09-23T21:00:13.257568+0000","last_undegraded":"2023-09-23T21:00:13.257568+0000","last_fullsized":"2023-09-23T21:00:13.257568+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:31.075098+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:31.075098+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:41:52.142362+0000","scrub_duration":0.010304610000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.b","version":"246'11328","reported_seq":11385,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-09-23T21:00:24.356832+0000","last_change":"2023-09-23T20:56:35.543994+0000","last_active":"2023-09-23T21:00:24.356832+0000","last_peered":"2023-09-23T21:00:24.356832+0000","last_clean":"2023-09-23T21:00:24.356832+0000","last_became_active":"2023-09-23T20:56:35.543758+0000","last_became_peered":"2023-09-23T20:56:35.543758+0000","last_unstale":"2023-09-23T21:00:24.356832+0000","last_undegraded":"2023-09-23T21:00:24.356832+0000","last_fullsized":"2023-09-23T21:00:24.356832+0000","mapping_epoch":199,"log_start":"217'4300","ondisk_log_start":"217'4300","created":199,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_clean_scrub_stamp":"2023-09-23T20:56:34.529103+0000","objects_scrubbed":0,"log_size":7028,"log_dups_size":0,"ondisk_log_size":7028,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:21:29.041484+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":88604672,"num_objects":1352,"num_object_clones":0,"num_object_copies":4056,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1352,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11328,"num_write_kb":405760,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":177,"reported_epoch":245,"state":"active+clean","last_fresh":"2023-09-23T21:00:13.258647+0000","last_change":"2023-09-23T20:56:45.991841+0000","last_active":"2023-09-23T21:00:13.258647+0000","last_peered":"2023-09-23T21:00:13.258647+0000","last_clean":"2023-09-23T21:00:13.258647+0000","last_became_active":"2023-09-23T20:56:45.991491+0000","last_became_peered":"2023-09-23T20:56:45.991491+0000","last_unstale":"2023-09-23T21:00:13.258647+0000","last_undegraded":"2023-09-23T21:00:13.258647+0000","last_fullsized":"2023-09-23T21:00:13.258647+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":205,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:30:37.405319+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":136,"reported_epoch":245,"state":"active+clean","last_fresh":"2023-09-23T21:00:13.253788+0000","last_change":"2023-09-23T20:55:35.957768+0000","last_active":"2023-09-23T21:00:13.253788+0000","last_peered":"2023-09-23T21:00:13.253788+0000","last_clean":"2023-09-23T21:00:13.253788+0000","last_became_active":"2023-09-23T20:55:35.957421+0000","last_became_peered":"2023-09-23T20:55:35.957421+0000","last_unstale":"2023-09-23T21:00:13.253788+0000","last_undegraded":"2023-09-23T21:00:13.253788+0000","last_fullsized":"2023-09-23T21:00:13.253788+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:49:41.585961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.c","version":"79'161","reported_seq":547,"reported_epoch":245,"state":"active+clean","last_fresh":"2023-09-23T21:00:13.251795+0000","last_change":"2023-09-23T20:57:22.041071+0000","last_active":"2023-09-23T21:00:13.251795+0000","last_peered":"2023-09-23T21:00:13.251795+0000","last_clean":"2023-09-23T21:00:13.251795+0000","last_became_active":"2023-09-23T20:57:15.973204+0000","last_became_peered":"2023-09-23T20:57:15.973204+0000","last_unstale":"2023-09-23T21:00:13.251795+0000","last_undegraded":"2023-09-23T21:00:13.251795+0000","last_fullsized":"2023-09-23T21:00:13.251795+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:22.040907+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:22.040907+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:47:02.019978+0000","scrub_duration":0.013021176000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.a","version":"246'11386","reported_seq":11443,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-09-23T21:00:24.683607+0000","last_change":"2023-09-23T20:56:35.547120+0000","last_active":"2023-09-23T21:00:24.683607+0000","last_peered":"2023-09-23T21:00:24.683607+0000","last_clean":"2023-09-23T21:00:24.683607+0000","last_became_active":"2023-09-23T20:56:35.546838+0000","last_became_peered":"2023-09-23T20:56:35.546838+0000","last_unstale":"2023-09-23T21:00:24.683607+0000","last_undegraded":"2023-09-23T21:00:24.683607+0000","last_fullsized":"2023-09-23T21:00:24.683607+0000","mapping_epoch":199,"log_start":"220'5200","ondisk_log_start":"220'5200","created":199,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_clean_scrub_stamp":"2023-09-23T20:56:34.529103+0000","objects_scrubbed":0,"log_size":6186,"log_dups_size":0,"ondisk_log_size":6186,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:48:24.132768+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":88473600,"num_objects":1350,"num_object_clones":0,"num_object_copies":4050,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1350,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11386,"num_write_kb":407552,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,5],"acting":[3,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":164,"reported_epoch":245,"state":"active+clean","last_fresh":"2023-09-23T21:00:13.258197+0000","last_change":"2023-09-23T20:56:45.991821+0000","last_active":"2023-09-23T21:00:13.258197+0000","last_peered":"2023-09-23T21:00:13.258197+0000","last_clean":"2023-09-23T21:00:13.258197+0000","last_became_active":"2023-09-23T20:56:45.991447+0000","last_became_peered":"2023-09-23T20:56:45.991447+0000","last_unstale":"2023-09-23T21:00:13.258197+0000","last_undegraded":"2023-09-23T21:00:13.258197+0000","last_fullsized":"2023-09-23T21:00:13.258197+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":205,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:14:25.525222+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":167,"reported_epoch":245,"state":"active+clean","last_fresh":"2023-09-23T21:00:13.258215+0000","last_change":"2023-09-23T20:57:44.982059+0000","last_active":"2023-09-23T21:00:13.258215+0000","last_peered":"2023-09-23T21:00:13.258215+0000","last_clean":"2023-09-23T21:00:13.258215+0000","last_became_active":"2023-09-23T20:57:44.981742+0000","last_became_peered":"2023-09-23T20:57:44.981742+0000","last_unstale":"2023-09-23T21:00:13.258215+0000","last_undegraded":"2023-09-23T21:00:13.258215+0000","last_fullsized":"2023-09-23T21:00:13.258215+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:41:24.042916+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.f","version":"79'161","reported_seq":550,"reported_epoch":245,"state":"active+clean","last_fresh":"2023-09-23T21:00:13.240836+0000","last_change":"2023-09-23T20:57:27.880841+0000","last_active":"2023-09-23T21:00:13.240836+0000","last_peered":"2023-09-23T21:00:13.240836+0000","last_clean":"2023-09-23T21:00:13.240836+0000","last_became_active":"2023-09-23T20:57:15.970925+0000","last_became_peered":"2023-09-23T20:57:15.970925+0000","last_unstale":"2023-09-23T21:00:13.240836+0000","last_undegraded":"2023-09-23T21:00:13.240836+0000","last_fullsized":"2023-09-23T21:00:13.240836+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:27.880776+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:27.880776+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:15:13.438014+0000","scrub_duration":0.0014004899999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.9","version":"246'11243","reported_seq":11300,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-09-23T21:00:21.797281+0000","last_change":"2023-09-23T20:56:35.549387+0000","last_active":"2023-09-23T21:00:21.797281+0000","last_peered":"2023-09-23T21:00:21.797281+0000","last_clean":"2023-09-23T21:00:21.797281+0000","last_became_active":"2023-09-23T20:56:35.549087+0000","last_became_peered":"2023-09-23T20:56:35.549087+0000","last_unstale":"2023-09-23T21:00:21.797281+0000","last_undegraded":"2023-09-23T21:00:21.797281+0000","last_fullsized":"2023-09-23T21:00:21.797281+0000","mapping_epoch":199,"log_start":"216'3900","ondisk_log_start":"216'3900","created":199,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_clean_scrub_stamp":"2023-09-23T20:56:34.529103+0000","objects_scrubbed":0,"log_size":7343,"log_dups_size":0,"ondisk_log_size":7343,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:04:51.547362+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":95879168,"num_objects":1463,"num_object_clones":0,"num_object_copies":4389,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1463,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11243,"num_write_kb":406592,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":150,"reported_epoch":245,"state":"active+clean","last_fresh":"2023-09-23T21:00:13.251273+0000","last_change":"2023-09-23T20:56:45.989153+0000","last_active":"2023-09-23T21:00:13.251273+0000","last_peered":"2023-09-23T21:00:13.251273+0000","last_clean":"2023-09-23T21:00:13.251273+0000","last_became_active":"2023-09-23T20:56:45.988800+0000","last_became_peered":"2023-09-23T20:56:45.988800+0000","last_unstale":"2023-09-23T21:00:13.251273+0000","last_undegraded":"2023-09-23T21:00:13.251273+0000","last_fullsized":"2023-09-23T21:00:13.251273+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":205,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:04:41.130194+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":185,"reported_epoch":245,"state":"active+clean","last_fresh":"2023-09-23T21:00:13.240583+0000","last_change":"2023-09-23T20:55:35.965427+0000","last_active":"2023-09-23T21:00:13.240583+0000","last_peered":"2023-09-23T21:00:13.240583+0000","last_clean":"2023-09-23T21:00:13.240583+0000","last_became_active":"2023-09-23T20:55:35.963969+0000","last_became_peered":"2023-09-23T20:55:35.963969+0000","last_unstale":"2023-09-23T21:00:13.240583+0000","last_undegraded":"2023-09-23T21:00:13.240583+0000","last_fullsized":"2023-09-23T21:00:13.240583+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:38:40.612083+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.e","version":"79'161","reported_seq":457,"reported_epoch":245,"state":"active+clean","last_fresh":"2023-09-23T21:00:13.258056+0000","last_change":"2023-09-23T20:57:34.132738+0000","last_active":"2023-09-23T21:00:13.258056+0000","last_peered":"2023-09-23T21:00:13.258056+0000","last_clean":"2023-09-23T21:00:13.258056+0000","last_became_active":"2023-09-23T20:57:15.974318+0000","last_became_peered":"2023-09-23T20:57:15.974318+0000","last_unstale":"2023-09-23T21:00:13.258056+0000","last_undegraded":"2023-09-23T21:00:13.258056+0000","last_fullsized":"2023-09-23T21:00:13.258056+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:34.132660+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:34.132660+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:07:07.739438+0000","scrub_duration":0.005229694,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.8","version":"246'11403","reported_seq":11460,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-09-23T21:00:21.445464+0000","last_change":"2023-09-23T20:56:35.548119+0000","last_active":"2023-09-23T21:00:21.445464+0000","last_peered":"2023-09-23T21:00:21.445464+0000","last_clean":"2023-09-23T21:00:21.445464+0000","last_became_active":"2023-09-23T20:56:35.547914+0000","last_became_peered":"2023-09-23T20:56:35.547914+0000","last_unstale":"2023-09-23T21:00:21.445464+0000","last_undegraded":"2023-09-23T21:00:21.445464+0000","last_fullsized":"2023-09-23T21:00:21.445464+0000","mapping_epoch":199,"log_start":"206'2500","ondisk_log_start":"206'2500","created":199,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_clean_scrub_stamp":"2023-09-23T20:56:34.529103+0000","objects_scrubbed":0,"log_size":8903,"log_dups_size":0,"ondisk_log_size":8903,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:42:54.255023+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":95092736,"num_objects":1451,"num_object_clones":0,"num_object_copies":4353,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1451,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11403,"num_write_kb":411328,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":136,"reported_epoch":245,"state":"active+clean","last_fresh":"2023-09-23T21:00:13.251460+0000","last_change":"2023-09-23T20:55:35.962005+0000","last_active":"2023-09-23T21:00:13.251460+0000","last_peered":"2023-09-23T21:00:13.251460+0000","last_clean":"2023-09-23T21:00:13.251460+0000","last_became_active":"2023-09-23T20:55:35.961695+0000","last_became_peered":"2023-09-23T20:55:35.961695+0000","last_unstale":"2023-09-23T21:00:13.251460+0000","last_undegraded":"2023-09-23T21:00:13.251460+0000","last_fullsized":"2023-09-23T21:00:13.251460+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:28:22.876201+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":234,"reported_epoch":245,"state":"active+clean","last_fresh":"2023-09-23T21:00:13.270365+0000","last_change":"2023-09-23T20:54:13.928715+0000","last_active":"2023-09-23T21:00:13.270365+0000","last_peered":"2023-09-23T21:00:13.270365+0000","last_clean":"2023-09-23T21:00:13.270365+0000","last_became_active":"2023-09-23T20:54:13.928200+0000","last_became_peered":"2023-09-23T20:54:13.928200+0000","last_unstale":"2023-09-23T21:00:13.270365+0000","last_undegraded":"2023-09-23T21:00:13.270365+0000","last_fullsized":"2023-09-23T21:00:13.270365+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:41:16.927513+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.f","version":"246'11441","reported_seq":11498,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-09-23T21:00:21.792541+0000","last_change":"2023-09-23T20:56:35.545128+0000","last_active":"2023-09-23T21:00:21.792541+0000","last_peered":"2023-09-23T21:00:21.792541+0000","last_clean":"2023-09-23T21:00:21.792541+0000","last_became_active":"2023-09-23T20:56:35.544838+0000","last_became_peered":"2023-09-23T20:56:35.544838+0000","last_unstale":"2023-09-23T21:00:21.792541+0000","last_undegraded":"2023-09-23T21:00:21.792541+0000","last_fullsized":"2023-09-23T21:00:21.792541+0000","mapping_epoch":199,"log_start":"216'4100","ondisk_log_start":"216'4100","created":199,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_clean_scrub_stamp":"2023-09-23T20:56:34.529103+0000","objects_scrubbed":0,"log_size":7341,"log_dups_size":0,"ondisk_log_size":7341,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:30:43.388717+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":93388800,"num_objects":1425,"num_object_clones":0,"num_object_copies":4275,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1425,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11441,"num_write_kb":411712,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.9","version":"79'161","reported_seq":619,"reported_epoch":245,"state":"active+clean","last_fresh":"2023-09-23T21:00:13.249978+0000","last_change":"2023-09-23T20:56:46.921939+0000","last_active":"2023-09-23T21:00:13.249978+0000","last_peered":"2023-09-23T21:00:13.249978+0000","last_clean":"2023-09-23T21:00:13.249978+0000","last_became_active":"2023-09-23T20:56:45.989908+0000","last_became_peered":"2023-09-23T20:56:45.989908+0000","last_unstale":"2023-09-23T21:00:13.249978+0000","last_undegraded":"2023-09-23T21:00:13.249978+0000","last_fullsized":"2023-09-23T21:00:13.249978+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":205,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:56:46.921739+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:56:46.921739+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:20:42.635110+0000","scrub_duration":0.0071954109999999996,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":236,"reported_epoch":245,"state":"active+clean","last_fresh":"2023-09-23T21:00:13.250049+0000","last_change":"2023-09-23T20:54:13.961788+0000","last_active":"2023-09-23T21:00:13.250049+0000","last_peered":"2023-09-23T21:00:13.250049+0000","last_clean":"2023-09-23T21:00:13.250049+0000","last_became_active":"2023-09-23T20:54:13.940937+0000","last_became_peered":"2023-09-23T20:54:13.940937+0000","last_unstale":"2023-09-23T21:00:13.250049+0000","last_undegraded":"2023-09-23T21:00:13.250049+0000","last_fullsized":"2023-09-23T21:00:13.250049+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:11:56.785299+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":140,"reported_epoch":245,"state":"active+clean","last_fresh":"2023-09-23T21:00:13.258415+0000","last_change":"2023-09-23T20:55:35.949670+0000","last_active":"2023-09-23T21:00:13.258415+0000","last_peered":"2023-09-23T21:00:13.258415+0000","last_clean":"2023-09-23T21:00:13.258415+0000","last_became_active":"2023-09-23T20:55:35.949140+0000","last_became_peered":"2023-09-23T20:55:35.949140+0000","last_unstale":"2023-09-23T21:00:13.258415+0000","last_undegraded":"2023-09-23T21:00:13.258415+0000","last_fullsized":"2023-09-23T21:00:13.258415+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:55:59.340303+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.e","version":"246'11313","reported_seq":11370,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-09-23T21:00:21.513214+0000","last_change":"2023-09-23T20:56:35.547184+0000","last_active":"2023-09-23T21:00:21.513214+0000","last_peered":"2023-09-23T21:00:21.513214+0000","last_clean":"2023-09-23T21:00:21.513214+0000","last_became_active":"2023-09-23T20:56:35.546912+0000","last_became_peered":"2023-09-23T20:56:35.546912+0000","last_unstale":"2023-09-23T21:00:21.513214+0000","last_undegraded":"2023-09-23T21:00:21.513214+0000","last_fullsized":"2023-09-23T21:00:21.513214+0000","mapping_epoch":199,"log_start":"206'2400","ondisk_log_start":"206'2400","created":199,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_clean_scrub_stamp":"2023-09-23T20:56:34.529103+0000","objects_scrubbed":0,"log_size":8913,"log_dups_size":0,"ondisk_log_size":8913,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:57:07.422792+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":96141312,"num_objects":1467,"num_object_clones":0,"num_object_copies":4401,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1467,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11313,"num_write_kb":408960,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,0],"acting":[2,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.8","version":"79'161","reported_seq":489,"reported_epoch":245,"state":"active+clean","last_fresh":"2023-09-23T21:00:13.253720+0000","last_change":"2023-09-23T20:56:57.505737+0000","last_active":"2023-09-23T21:00:13.253720+0000","last_peered":"2023-09-23T21:00:13.253720+0000","last_clean":"2023-09-23T21:00:13.253720+0000","last_became_active":"2023-09-23T20:56:04.982345+0000","last_became_peered":"2023-09-23T20:56:04.982345+0000","last_unstale":"2023-09-23T21:00:13.253720+0000","last_undegraded":"2023-09-23T21:00:13.253720+0000","last_fullsized":"2023-09-23T21:00:13.253720+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:56:57.505663+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:56:57.505663+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:34:30.109724+0000","scrub_duration":0.026284108,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":208,"reported_epoch":245,"state":"active+clean","last_fresh":"2023-09-23T21:00:13.240795+0000","last_change":"2023-09-23T20:55:35.959942+0000","last_active":"2023-09-23T21:00:13.240795+0000","last_peered":"2023-09-23T21:00:13.240795+0000","last_clean":"2023-09-23T21:00:13.240795+0000","last_became_active":"2023-09-23T20:55:35.957071+0000","last_became_peered":"2023-09-23T20:55:35.957071+0000","last_unstale":"2023-09-23T21:00:13.240795+0000","last_undegraded":"2023-09-23T21:00:13.240795+0000","last_fullsized":"2023-09-23T21:00:13.240795+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:44:30.308036+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":123,"reported_epoch":245,"state":"active+clean","last_fresh":"2023-09-23T21:00:13.257957+0000","last_change":"2023-09-23T20:54:13.951459+0000","last_active":"2023-09-23T21:00:13.257957+0000","last_peered":"2023-09-23T21:00:13.257957+0000","last_clean":"2023-09-23T21:00:13.257957+0000","last_became_active":"2023-09-23T20:54:13.948409+0000","last_became_peered":"2023-09-23T20:54:13.948409+0000","last_unstale":"2023-09-23T21:00:13.257957+0000","last_undegraded":"2023-09-23T21:00:13.257957+0000","last_fullsized":"2023-09-23T21:00:13.257957+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:38:10.148061+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.b","version":"79'161","reported_seq":529,"reported_epoch":245,"state":"active+clean","last_fresh":"2023-09-23T21:00:13.274823+0000","last_change":"2023-09-23T20:57:27.320172+0000","last_active":"2023-09-23T21:00:13.274823+0000","last_peered":"2023-09-23T21:00:13.274823+0000","last_clean":"2023-09-23T21:00:13.274823+0000","last_became_active":"2023-09-23T20:57:16.009630+0000","last_became_peered":"2023-09-23T20:57:16.009630+0000","last_unstale":"2023-09-23T21:00:13.274823+0000","last_undegraded":"2023-09-23T21:00:13.274823+0000","last_fullsized":"2023-09-23T21:00:13.274823+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:27.320022+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:27.320022+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:31:35.221558+0000","scrub_duration":0.0043894679999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,5],"acting":[1,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.d","version":"246'11433","reported_seq":11489,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-09-23T21:00:24.065209+0000","last_change":"2023-09-23T20:56:35.544132+0000","last_active":"2023-09-23T21:00:24.065209+0000","last_peered":"2023-09-23T21:00:24.065209+0000","last_clean":"2023-09-23T21:00:24.065209+0000","last_became_active":"2023-09-23T20:56:35.543836+0000","last_became_peered":"2023-09-23T20:56:35.543836+0000","last_unstale":"2023-09-23T21:00:24.065209+0000","last_undegraded":"2023-09-23T21:00:24.065209+0000","last_fullsized":"2023-09-23T21:00:24.065209+0000","mapping_epoch":199,"log_start":"217'4100","ondisk_log_start":"217'4100","created":199,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_clean_scrub_stamp":"2023-09-23T20:56:34.529103+0000","objects_scrubbed":0,"log_size":7333,"log_dups_size":0,"ondisk_log_size":7333,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:27:14.636991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":88866840,"num_objects":1357,"num_object_clones":0,"num_object_copies":4071,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1357,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":11433,"num_write_kb":409217,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":123,"reported_epoch":245,"state":"active+clean","last_fresh":"2023-09-23T21:00:13.258001+0000","last_change":"2023-09-23T20:54:13.950254+0000","last_active":"2023-09-23T21:00:13.258001+0000","last_peered":"2023-09-23T21:00:13.258001+0000","last_clean":"2023-09-23T21:00:13.258001+0000","last_became_active":"2023-09-23T20:54:13.949934+0000","last_became_peered":"2023-09-23T20:54:13.949934+0000","last_unstale":"2023-09-23T21:00:13.258001+0000","last_undegraded":"2023-09-23T21:00:13.258001+0000","last_fullsized":"2023-09-23T21:00:13.258001+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:30:32.847167+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":140,"reported_epoch":245,"state":"active+clean","last_fresh":"2023-09-23T21:00:13.258279+0000","last_change":"2023-09-23T20:55:35.956153+0000","last_active":"2023-09-23T21:00:13.258279+0000","last_peered":"2023-09-23T21:00:13.258279+0000","last_clean":"2023-09-23T21:00:13.258279+0000","last_became_active":"2023-09-23T20:55:35.955630+0000","last_became_peered":"2023-09-23T20:55:35.955630+0000","last_unstale":"2023-09-23T21:00:13.258279+0000","last_undegraded":"2023-09-23T21:00:13.258279+0000","last_fullsized":"2023-09-23T21:00:13.258279+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:45:20.282929+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.c","version":"246'11418","reported_seq":11483,"reported_epoch":246,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T21:00:24.078548+0000","last_change":"2023-09-23T20:57:46.463005+0000","last_active":"2023-09-23T21:00:24.078548+0000","last_peered":"2023-09-23T21:00:24.078548+0000","last_clean":"2023-09-23T20:57:44.015119+0000","last_became_active":"2023-09-23T20:57:46.448702+0000","last_became_peered":"2023-09-23T20:57:46.448702+0000","last_unstale":"2023-09-23T21:00:24.078548+0000","last_undegraded":"2023-09-23T20:57:46.010874+0000","last_fullsized":"2023-09-23T20:57:45.978434+0000","mapping_epoch":223,"log_start":"217'4100","ondisk_log_start":"217'4100","created":199,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_clean_scrub_stamp":"2023-09-23T20:56:34.529103+0000","objects_scrubbed":0,"log_size":7318,"log_dups_size":0,"ondisk_log_size":7318,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:08:34.180651+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":91095040,"num_objects":1390,"num_object_clones":0,"num_object_copies":4170,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5473,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1390,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11418,"num_write_kb":409856,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,7],"avail_no_missing":["6","2","7"],"object_location_counts":[{"shards":"2,6,7","objects":459},{"shards":"6,7","objects":-4083}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.a","version":"79'161","reported_seq":489,"reported_epoch":245,"state":"active+clean","last_fresh":"2023-09-23T21:00:13.253625+0000","last_change":"2023-09-23T20:56:44.487778+0000","last_active":"2023-09-23T21:00:13.253625+0000","last_peered":"2023-09-23T21:00:13.253625+0000","last_clean":"2023-09-23T21:00:13.253625+0000","last_became_active":"2023-09-23T20:56:05.006740+0000","last_became_peered":"2023-09-23T20:56:05.006740+0000","last_unstale":"2023-09-23T21:00:13.253625+0000","last_undegraded":"2023-09-23T21:00:13.253625+0000","last_fullsized":"2023-09-23T21:00:13.253625+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:56:44.487678+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:56:44.487678+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:04:50.086749+0000","scrub_duration":0.0017408899999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":35,"reported_epoch":245,"state":"active+clean","last_fresh":"2023-09-23T21:00:13.249360+0000","last_change":"2023-09-23T20:57:44.982513+0000","last_active":"2023-09-23T21:00:13.249360+0000","last_peered":"2023-09-23T21:00:13.249360+0000","last_clean":"2023-09-23T21:00:13.249360+0000","last_became_active":"2023-09-23T20:57:44.982112+0000","last_became_peered":"2023-09-23T20:57:44.982112+0000","last_unstale":"2023-09-23T21:00:13.249360+0000","last_undegraded":"2023-09-23T21:00:13.249360+0000","last_fullsized":"2023-09-23T21:00:13.249360+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:09:05.630041+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":216,"reported_epoch":245,"state":"active+clean","last_fresh":"2023-09-23T21:00:13.273369+0000","last_change":"2023-09-23T20:54:13.940748+0000","last_active":"2023-09-23T21:00:13.273369+0000","last_peered":"2023-09-23T21:00:13.273369+0000","last_clean":"2023-09-23T21:00:13.273369+0000","last_became_active":"2023-09-23T20:54:13.940419+0000","last_became_peered":"2023-09-23T20:54:13.940419+0000","last_unstale":"2023-09-23T21:00:13.273369+0000","last_undegraded":"2023-09-23T21:00:13.273369+0000","last_fullsized":"2023-09-23T21:00:13.273369+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:15:01.923816+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.1","version":"246'11320","reported_seq":11388,"reported_epoch":246,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T21:00:22.218310+0000","last_change":"2023-09-23T20:57:46.483248+0000","last_active":"2023-09-23T21:00:22.218310+0000","last_peered":"2023-09-23T21:00:22.218310+0000","last_clean":"2023-09-23T20:57:44.047957+0000","last_became_active":"2023-09-23T20:57:46.475568+0000","last_became_peered":"2023-09-23T20:57:46.475568+0000","last_unstale":"2023-09-23T21:00:22.218310+0000","last_undegraded":"2023-09-23T20:57:46.083591+0000","last_fullsized":"2023-09-23T20:57:45.978436+0000","mapping_epoch":223,"log_start":"218'4600","ondisk_log_start":"218'4600","created":199,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_clean_scrub_stamp":"2023-09-23T20:56:34.529103+0000","objects_scrubbed":0,"log_size":6720,"log_dups_size":0,"ondisk_log_size":6720,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:22:45.067269+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":96075776,"num_objects":1466,"num_object_clones":0,"num_object_copies":4398,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5470,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1466,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11320,"num_write_kb":409152,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,6],"avail_no_missing":["4","1","6"],"object_location_counts":[{"shards":"1,4,6","objects":543},{"shards":"4,6","objects":-4004}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.7","version":"79'161","reported_seq":551,"reported_epoch":245,"state":"active+clean","last_fresh":"2023-09-23T21:00:13.241039+0000","last_change":"2023-09-23T20:57:48.243572+0000","last_active":"2023-09-23T21:00:13.241039+0000","last_peered":"2023-09-23T21:00:13.241039+0000","last_clean":"2023-09-23T21:00:13.241039+0000","last_became_active":"2023-09-23T20:57:48.243229+0000","last_became_peered":"2023-09-23T20:57:48.243229+0000","last_unstale":"2023-09-23T21:00:13.241039+0000","last_undegraded":"2023-09-23T21:00:13.241039+0000","last_fullsized":"2023-09-23T21:00:13.241039+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":3,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:25.916867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:25.916867+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:23:19.385831+0000","scrub_duration":0.001665323,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":221,"reported_epoch":245,"state":"active+clean","last_fresh":"2023-09-23T21:00:13.273384+0000","last_change":"2023-09-23T20:50:05.867130+0000","last_active":"2023-09-23T21:00:13.273384+0000","last_peered":"2023-09-23T21:00:13.273384+0000","last_clean":"2023-09-23T21:00:13.273384+0000","last_became_active":"2023-09-23T20:50:05.866758+0000","last_became_peered":"2023-09-23T20:50:05.866758+0000","last_unstale":"2023-09-23T21:00:13.273384+0000","last_undegraded":"2023-09-23T21:00:13.273384+0000","last_fullsized":"2023-09-23T21:00:13.273384+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:55:07.548209+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":119,"reported_epoch":245,"state":"active+clean","last_fresh":"2023-09-23T21:00:13.251420+0000","last_change":"2023-09-23T20:54:31.887604+0000","last_active":"2023-09-23T21:00:13.251420+0000","last_peered":"2023-09-23T21:00:13.251420+0000","last_clean":"2023-09-23T21:00:13.251420+0000","last_became_active":"2023-09-23T20:54:31.886752+0000","last_became_peered":"2023-09-23T20:54:31.886752+0000","last_unstale":"2023-09-23T21:00:13.251420+0000","last_undegraded":"2023-09-23T21:00:13.251420+0000","last_fullsized":"2023-09-23T21:00:13.251420+0000","mapping_epoch":136,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:13:58.872176+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":219,"reported_epoch":245,"state":"active+clean","last_fresh":"2023-09-23T21:00:13.275342+0000","last_change":"2023-09-23T20:54:31.899082+0000","last_active":"2023-09-23T21:00:13.275342+0000","last_peered":"2023-09-23T21:00:13.275342+0000","last_clean":"2023-09-23T21:00:13.275342+0000","last_became_active":"2023-09-23T20:54:31.898521+0000","last_became_peered":"2023-09-23T20:54:31.898521+0000","last_unstale":"2023-09-23T21:00:13.275342+0000","last_undegraded":"2023-09-23T21:00:13.275342+0000","last_fullsized":"2023-09-23T21:00:13.275342+0000","mapping_epoch":136,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:25:36.059711+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.7","version":"246'11489","reported_seq":11544,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-09-23T21:00:24.032934+0000","last_change":"2023-09-23T20:56:35.541695+0000","last_active":"2023-09-23T21:00:24.032934+0000","last_peered":"2023-09-23T21:00:24.032934+0000","last_clean":"2023-09-23T21:00:24.032934+0000","last_became_active":"2023-09-23T20:56:35.541392+0000","last_became_peered":"2023-09-23T20:56:35.541392+0000","last_unstale":"2023-09-23T21:00:24.032934+0000","last_undegraded":"2023-09-23T21:00:24.032934+0000","last_fullsized":"2023-09-23T21:00:24.032934+0000","mapping_epoch":199,"log_start":"217'4100","ondisk_log_start":"217'4100","created":199,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_clean_scrub_stamp":"2023-09-23T20:56:34.529103+0000","objects_scrubbed":0,"log_size":7389,"log_dups_size":0,"ondisk_log_size":7389,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:57:22.478253+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":86573056,"num_objects":1321,"num_object_clones":0,"num_object_copies":3963,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1321,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11489,"num_write_kb":409920,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.1","version":"79'161","reported_seq":509,"reported_epoch":245,"state":"active+clean","last_fresh":"2023-09-23T21:00:13.251171+0000","last_change":"2023-09-23T20:57:19.669346+0000","last_active":"2023-09-23T21:00:13.251171+0000","last_peered":"2023-09-23T21:00:13.251171+0000","last_clean":"2023-09-23T21:00:13.251171+0000","last_became_active":"2023-09-23T20:57:08.246046+0000","last_became_peered":"2023-09-23T20:57:08.246046+0000","last_unstale":"2023-09-23T21:00:13.251171+0000","last_undegraded":"2023-09-23T21:00:13.251171+0000","last_fullsized":"2023-09-23T21:00:13.251171+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:19.669259+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:19.669259+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:14:03.435531+0000","scrub_duration":0.0045759559999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":136,"reported_epoch":245,"state":"active+clean","last_fresh":"2023-09-23T21:00:13.253357+0000","last_change":"2023-09-23T20:55:35.959514+0000","last_active":"2023-09-23T21:00:13.253357+0000","last_peered":"2023-09-23T21:00:13.253357+0000","last_clean":"2023-09-23T21:00:13.253357+0000","last_became_active":"2023-09-23T20:55:35.959244+0000","last_became_peered":"2023-09-23T20:55:35.959244+0000","last_unstale":"2023-09-23T21:00:13.253357+0000","last_undegraded":"2023-09-23T21:00:13.253357+0000","last_fullsized":"2023-09-23T21:00:13.253357+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:47:32.665316+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":140,"reported_epoch":245,"state":"active+clean","last_fresh":"2023-09-23T21:00:13.257971+0000","last_change":"2023-09-23T20:55:35.949000+0000","last_active":"2023-09-23T21:00:13.257971+0000","last_peered":"2023-09-23T21:00:13.257971+0000","last_clean":"2023-09-23T21:00:13.257971+0000","last_became_active":"2023-09-23T20:55:35.948410+0000","last_became_peered":"2023-09-23T20:55:35.948410+0000","last_unstale":"2023-09-23T21:00:13.257971+0000","last_undegraded":"2023-09-23T21:00:13.257971+0000","last_fullsized":"2023-09-23T21:00:13.257971+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:26:50.477289+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":213,"reported_epoch":245,"state":"active+clean","last_fresh":"2023-09-23T21:00:13.273168+0000","last_change":"2023-09-23T20:51:10.829889+0000","last_active":"2023-09-23T21:00:13.273168+0000","last_peered":"2023-09-23T21:00:13.273168+0000","last_clean":"2023-09-23T21:00:13.273168+0000","last_became_active":"2023-09-23T20:51:10.829533+0000","last_became_peered":"2023-09-23T20:51:10.829533+0000","last_unstale":"2023-09-23T21:00:13.273168+0000","last_undegraded":"2023-09-23T21:00:13.273168+0000","last_fullsized":"2023-09-23T21:00:13.273168+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:43:04.603460+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.4","version":"246'11287","reported_seq":11344,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-09-23T21:00:21.797228+0000","last_change":"2023-09-23T20:56:35.549909+0000","last_active":"2023-09-23T21:00:21.797228+0000","last_peered":"2023-09-23T21:00:21.797228+0000","last_clean":"2023-09-23T21:00:21.797228+0000","last_became_active":"2023-09-23T20:56:35.549618+0000","last_became_peered":"2023-09-23T20:56:35.549618+0000","last_unstale":"2023-09-23T21:00:21.797228+0000","last_undegraded":"2023-09-23T21:00:21.797228+0000","last_fullsized":"2023-09-23T21:00:21.797228+0000","mapping_epoch":199,"log_start":"216'3900","ondisk_log_start":"216'3900","created":199,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_clean_scrub_stamp":"2023-09-23T20:56:34.529103+0000","objects_scrubbed":0,"log_size":7387,"log_dups_size":0,"ondisk_log_size":7387,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:24:25.918475+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":94044160,"num_objects":1435,"num_object_clones":0,"num_object_copies":4305,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1435,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11287,"num_write_kb":407104,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.2","version":"79'161","reported_seq":476,"reported_epoch":245,"state":"active+clean","last_fresh":"2023-09-23T21:00:13.273920+0000","last_change":"2023-09-23T20:57:24.802586+0000","last_active":"2023-09-23T21:00:13.273920+0000","last_peered":"2023-09-23T21:00:13.273920+0000","last_clean":"2023-09-23T21:00:13.273920+0000","last_became_active":"2023-09-23T20:57:08.236114+0000","last_became_peered":"2023-09-23T20:57:08.236114+0000","last_unstale":"2023-09-23T21:00:13.273920+0000","last_undegraded":"2023-09-23T21:00:13.273920+0000","last_fullsized":"2023-09-23T21:00:13.273920+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:24.802451+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:24.802451+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:06:22.309980+0000","scrub_duration":0.0090861250000000005,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,4],"acting":[0,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":232,"reported_epoch":245,"state":"active+clean","last_fresh":"2023-09-23T21:00:13.261752+0000","last_change":"2023-09-23T20:51:10.838054+0000","last_active":"2023-09-23T21:00:13.261752+0000","last_peered":"2023-09-23T21:00:13.261752+0000","last_clean":"2023-09-23T21:00:13.261752+0000","last_became_active":"2023-09-23T20:51:10.837750+0000","last_became_peered":"2023-09-23T20:51:10.837750+0000","last_unstale":"2023-09-23T21:00:13.261752+0000","last_undegraded":"2023-09-23T21:00:13.261752+0000","last_fullsized":"2023-09-23T21:00:13.261752+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:15:51.648085+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":214,"reported_epoch":245,"state":"active+clean","last_fresh":"2023-09-23T21:00:13.273262+0000","last_change":"2023-09-23T20:55:35.951146+0000","last_active":"2023-09-23T21:00:13.273262+0000","last_peered":"2023-09-23T21:00:13.273262+0000","last_clean":"2023-09-23T21:00:13.273262+0000","last_became_active":"2023-09-23T20:55:35.950874+0000","last_became_peered":"2023-09-23T20:55:35.950874+0000","last_unstale":"2023-09-23T21:00:13.273262+0000","last_undegraded":"2023-09-23T21:00:13.273262+0000","last_fullsized":"2023-09-23T21:00:13.273262+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:58:13.957677+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":234,"reported_epoch":245,"state":"active+clean","last_fresh":"2023-09-23T21:00:13.273278+0000","last_change":"2023-09-23T20:54:31.898870+0000","last_active":"2023-09-23T21:00:13.273278+0000","last_peered":"2023-09-23T21:00:13.273278+0000","last_clean":"2023-09-23T21:00:13.273278+0000","last_became_active":"2023-09-23T20:54:31.898467+0000","last_became_peered":"2023-09-23T20:54:31.898467+0000","last_unstale":"2023-09-23T21:00:13.273278+0000","last_undegraded":"2023-09-23T21:00:13.273278+0000","last_fullsized":"2023-09-23T21:00:13.273278+0000","mapping_epoch":136,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:56:22.483408+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.6","version":"246'11148","reported_seq":19068,"reported_epoch":246,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-09-23T21:00:21.824667+0000","last_change":"2023-09-23T20:58:39.453197+0000","last_active":"2023-09-23T21:00:21.824667+0000","last_peered":"2023-09-23T21:00:21.824667+0000","last_clean":"2023-09-23T20:57:44.031597+0000","last_became_active":"2023-09-23T20:57:46.443927+0000","last_became_peered":"2023-09-23T20:57:46.443927+0000","last_unstale":"2023-09-23T21:00:21.824667+0000","last_undegraded":"2023-09-23T20:58:39.450464+0000","last_fullsized":"2023-09-23T20:57:45.977227+0000","mapping_epoch":223,"log_start":"216'3800","ondisk_log_start":"216'3800","created":199,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_clean_scrub_stamp":"2023-09-23T20:56:34.529103+0000","objects_scrubbed":0,"log_size":7348,"log_dups_size":0,"ondisk_log_size":7348,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:37:23.911131+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":94371840,"num_objects":1440,"num_object_clones":0,"num_object_copies":4320,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":949,"num_objects_misplaced":522,"num_objects_unfound":0,"num_objects_dirty":1440,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11148,"num_write_kb":402816,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3927,"num_bytes_recovered":257359872,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7],"avail_no_missing":["0","5","7"],"object_location_counts":[{"shards":"0,5,7","objects":522},{"shards":"0,7","objects":-31}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":119,"reported_epoch":245,"state":"active+clean","last_fresh":"2023-09-23T21:00:13.251742+0000","last_change":"2023-09-23T20:54:31.883504+0000","last_active":"2023-09-23T21:00:13.251742+0000","last_peered":"2023-09-23T21:00:13.251742+0000","last_clean":"2023-09-23T21:00:13.251742+0000","last_became_active":"2023-09-23T20:54:31.883172+0000","last_became_peered":"2023-09-23T20:54:31.883172+0000","last_unstale":"2023-09-23T21:00:13.251742+0000","last_undegraded":"2023-09-23T21:00:13.251742+0000","last_fullsized":"2023-09-23T21:00:13.251742+0000","mapping_epoch":136,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:45:40.289120+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"79'161","reported_seq":476,"reported_epoch":245,"state":"active+clean","last_fresh":"2023-09-23T21:00:13.269928+0000","last_change":"2023-09-23T20:57:20.738417+0000","last_active":"2023-09-23T21:00:13.269928+0000","last_peered":"2023-09-23T21:00:13.269928+0000","last_clean":"2023-09-23T21:00:13.269928+0000","last_became_active":"2023-09-23T20:57:08.236842+0000","last_became_peered":"2023-09-23T20:57:08.236842+0000","last_unstale":"2023-09-23T21:00:13.269928+0000","last_undegraded":"2023-09-23T21:00:13.269928+0000","last_fullsized":"2023-09-23T21:00:13.269928+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":0,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:20.738273+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:20.738273+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:18:28.327990+0000","scrub_duration":0.0088336339999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":224,"reported_epoch":245,"state":"active+clean","last_fresh":"2023-09-23T21:00:13.252776+0000","last_change":"2023-09-23T20:55:35.950777+0000","last_active":"2023-09-23T21:00:13.252776+0000","last_peered":"2023-09-23T21:00:13.252776+0000","last_clean":"2023-09-23T21:00:13.252776+0000","last_became_active":"2023-09-23T20:55:35.950427+0000","last_became_peered":"2023-09-23T20:55:35.950427+0000","last_unstale":"2023-09-23T21:00:13.252776+0000","last_undegraded":"2023-09-23T21:00:13.252776+0000","last_fullsized":"2023-09-23T21:00:13.252776+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:20:10.371751+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":190,"reported_epoch":245,"state":"active+clean","last_fresh":"2023-09-23T21:00:13.240814+0000","last_change":"2023-09-23T20:55:35.978140+0000","last_active":"2023-09-23T21:00:13.240814+0000","last_peered":"2023-09-23T21:00:13.240814+0000","last_clean":"2023-09-23T21:00:13.240814+0000","last_became_active":"2023-09-23T20:55:35.977831+0000","last_became_peered":"2023-09-23T20:55:35.977831+0000","last_unstale":"2023-09-23T21:00:13.240814+0000","last_undegraded":"2023-09-23T21:00:13.240814+0000","last_fullsized":"2023-09-23T21:00:13.240814+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:25:47.678023+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.5","version":"246'11259","reported_seq":11323,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-09-23T21:00:21.506638+0000","last_change":"2023-09-23T20:56:45.990110+0000","last_active":"2023-09-23T21:00:21.506638+0000","last_peered":"2023-09-23T21:00:21.506638+0000","last_clean":"2023-09-23T21:00:21.506638+0000","last_became_active":"2023-09-23T20:56:45.988394+0000","last_became_peered":"2023-09-23T20:56:45.988394+0000","last_unstale":"2023-09-23T21:00:21.506638+0000","last_undegraded":"2023-09-23T21:00:21.506638+0000","last_fullsized":"2023-09-23T21:00:21.506638+0000","mapping_epoch":204,"log_start":"206'2400","ondisk_log_start":"206'2400","created":199,"last_epoch_clean":205,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_clean_scrub_stamp":"2023-09-23T20:56:34.529103+0000","objects_scrubbed":0,"log_size":8859,"log_dups_size":0,"ondisk_log_size":8859,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:22:03.814210+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":92995584,"num_objects":1419,"num_object_clones":0,"num_object_copies":4257,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1419,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11259,"num_write_kb":405696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,0],"acting":[2,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"79'161","reported_seq":513,"reported_epoch":245,"state":"active+clean","last_fresh":"2023-09-23T21:00:14.351960+0000","last_change":"2023-09-23T20:57:20.378531+0000","last_active":"2023-09-23T21:00:14.351960+0000","last_peered":"2023-09-23T21:00:14.351960+0000","last_clean":"2023-09-23T21:00:14.351960+0000","last_became_active":"2023-09-23T20:57:08.238019+0000","last_became_peered":"2023-09-23T20:57:08.238019+0000","last_unstale":"2023-09-23T21:00:14.351960+0000","last_undegraded":"2023-09-23T21:00:14.351960+0000","last_fullsized":"2023-09-23T21:00:14.351960+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":3,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:20.377714+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:20.377714+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:00:08.305602+0000","scrub_duration":0.003015228,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":115,"reported_epoch":245,"state":"active+clean","last_fresh":"2023-09-23T21:00:13.273405+0000","last_change":"2023-09-23T20:56:45.991884+0000","last_active":"2023-09-23T21:00:13.273405+0000","last_peered":"2023-09-23T21:00:13.273405+0000","last_clean":"2023-09-23T21:00:13.273405+0000","last_became_active":"2023-09-23T20:56:45.991568+0000","last_became_peered":"2023-09-23T20:56:45.991568+0000","last_unstale":"2023-09-23T21:00:13.273405+0000","last_undegraded":"2023-09-23T21:00:13.273405+0000","last_fullsized":"2023-09-23T21:00:13.273405+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":205,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:21:20.775511+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,0],"acting":[1,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":240,"reported_epoch":245,"state":"active+clean","last_fresh":"2023-09-23T21:00:13.270263+0000","last_change":"2023-09-23T20:57:44.979950+0000","last_active":"2023-09-23T21:00:13.270263+0000","last_peered":"2023-09-23T21:00:13.270263+0000","last_clean":"2023-09-23T21:00:13.270263+0000","last_became_active":"2023-09-23T20:57:44.979346+0000","last_became_peered":"2023-09-23T20:57:44.979346+0000","last_unstale":"2023-09-23T21:00:13.270263+0000","last_undegraded":"2023-09-23T21:00:13.270263+0000","last_fullsized":"2023-09-23T21:00:13.270263+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:10:24.372800+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":219,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-09-23T21:00:20.334405+0000","last_change":"2023-09-23T20:50:37.982235+0000","last_active":"2023-09-23T21:00:20.334405+0000","last_peered":"2023-09-23T21:00:20.334405+0000","last_clean":"2023-09-23T21:00:20.334405+0000","last_became_active":"2023-09-23T20:50:37.981913+0000","last_became_peered":"2023-09-23T20:50:37.981913+0000","last_unstale":"2023-09-23T21:00:20.334405+0000","last_undegraded":"2023-09-23T21:00:20.334405+0000","last_fullsized":"2023-09-23T21:00:20.334405+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:59:57.122584+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.0","version":"246'11262","reported_seq":11315,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-09-23T21:00:22.020368+0000","last_change":"2023-09-23T20:56:35.550635+0000","last_active":"2023-09-23T21:00:22.020368+0000","last_peered":"2023-09-23T21:00:22.020368+0000","last_clean":"2023-09-23T21:00:22.020368+0000","last_became_active":"2023-09-23T20:56:35.550376+0000","last_became_peered":"2023-09-23T20:56:35.550376+0000","last_unstale":"2023-09-23T21:00:22.020368+0000","last_undegraded":"2023-09-23T21:00:22.020368+0000","last_fullsized":"2023-09-23T21:00:22.020368+0000","mapping_epoch":199,"log_start":"206'2600","ondisk_log_start":"206'2600","created":199,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_clean_scrub_stamp":"2023-09-23T20:56:34.529103+0000","objects_scrubbed":0,"log_size":8662,"log_dups_size":0,"ondisk_log_size":8662,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:40:55.020049+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":93061120,"num_objects":1420,"num_object_clones":0,"num_object_copies":4260,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1420,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11262,"num_write_kb":405824,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.6","version":"79'161","reported_seq":528,"reported_epoch":245,"state":"active+clean","last_fresh":"2023-09-23T21:00:13.240825+0000","last_change":"2023-09-23T20:57:26.907505+0000","last_active":"2023-09-23T21:00:13.240825+0000","last_peered":"2023-09-23T21:00:13.240825+0000","last_clean":"2023-09-23T21:00:13.240825+0000","last_became_active":"2023-09-23T20:57:08.240321+0000","last_became_peered":"2023-09-23T20:57:08.240321+0000","last_unstale":"2023-09-23T21:00:13.240825+0000","last_undegraded":"2023-09-23T21:00:13.240825+0000","last_fullsized":"2023-09-23T21:00:13.240825+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":3,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:26.907408+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:26.907408+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:34:17.392928+0000","scrub_duration":0.010337038,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":162,"reported_epoch":245,"state":"active+clean","last_fresh":"2023-09-23T21:00:13.274868+0000","last_change":"2023-09-23T20:56:45.988566+0000","last_active":"2023-09-23T21:00:13.274868+0000","last_peered":"2023-09-23T21:00:13.274868+0000","last_clean":"2023-09-23T21:00:13.274868+0000","last_became_active":"2023-09-23T20:56:45.988249+0000","last_became_peered":"2023-09-23T20:56:45.988249+0000","last_unstale":"2023-09-23T21:00:13.274868+0000","last_undegraded":"2023-09-23T21:00:13.274868+0000","last_fullsized":"2023-09-23T21:00:13.274868+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":205,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:51:36.644764+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,4],"acting":[0,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":216,"reported_epoch":245,"state":"active+clean","last_fresh":"2023-09-23T21:00:13.273306+0000","last_change":"2023-09-23T20:54:31.898965+0000","last_active":"2023-09-23T21:00:13.273306+0000","last_peered":"2023-09-23T21:00:13.273306+0000","last_clean":"2023-09-23T21:00:13.273306+0000","last_became_active":"2023-09-23T20:54:31.898663+0000","last_became_peered":"2023-09-23T20:54:31.898663+0000","last_unstale":"2023-09-23T21:00:13.273306+0000","last_undegraded":"2023-09-23T21:00:13.273306+0000","last_fullsized":"2023-09-23T21:00:13.273306+0000","mapping_epoch":136,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:53:05.058087+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":207,"reported_epoch":245,"state":"active+clean","last_fresh":"2023-09-23T21:00:13.270594+0000","last_change":"2023-09-23T20:54:31.884717+0000","last_active":"2023-09-23T21:00:13.270594+0000","last_peered":"2023-09-23T21:00:13.270594+0000","last_clean":"2023-09-23T21:00:13.270594+0000","last_became_active":"2023-09-23T20:54:31.884345+0000","last_became_peered":"2023-09-23T20:54:31.884345+0000","last_unstale":"2023-09-23T21:00:13.270594+0000","last_undegraded":"2023-09-23T21:00:13.270594+0000","last_fullsized":"2023-09-23T21:00:13.270594+0000","mapping_epoch":136,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:02:26.393868+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.3","version":"246'11251","reported_seq":11308,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-09-23T21:00:24.733864+0000","last_change":"2023-09-23T20:56:35.549655+0000","last_active":"2023-09-23T21:00:24.733864+0000","last_peered":"2023-09-23T21:00:24.733864+0000","last_clean":"2023-09-23T21:00:24.733864+0000","last_became_active":"2023-09-23T20:56:35.549357+0000","last_became_peered":"2023-09-23T20:56:35.549357+0000","last_unstale":"2023-09-23T21:00:24.733864+0000","last_undegraded":"2023-09-23T21:00:24.733864+0000","last_fullsized":"2023-09-23T21:00:24.733864+0000","mapping_epoch":199,"log_start":"220'5100","ondisk_log_start":"220'5100","created":199,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_clean_scrub_stamp":"2023-09-23T20:56:34.529103+0000","objects_scrubbed":0,"log_size":6151,"log_dups_size":0,"ondisk_log_size":6151,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:37:58.752518+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":83296256,"num_objects":1271,"num_object_clones":0,"num_object_copies":3813,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1271,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11251,"num_write_kb":400704,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.5","version":"79'161","reported_seq":509,"reported_epoch":245,"state":"active+clean","last_fresh":"2023-09-23T21:00:13.257656+0000","last_change":"2023-09-23T20:57:30.029479+0000","last_active":"2023-09-23T21:00:13.257656+0000","last_peered":"2023-09-23T21:00:13.257656+0000","last_clean":"2023-09-23T21:00:13.257656+0000","last_became_active":"2023-09-23T20:57:08.249909+0000","last_became_peered":"2023-09-23T20:57:08.249909+0000","last_unstale":"2023-09-23T21:00:13.257656+0000","last_undegraded":"2023-09-23T21:00:13.257656+0000","last_fullsized":"2023-09-23T21:00:13.257656+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":3,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:30.029396+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:30.029396+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:16:46.083145+0000","scrub_duration":0.012914682,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":210,"reported_epoch":245,"state":"active+clean","last_fresh":"2023-09-23T21:00:13.250381+0000","last_change":"2023-09-23T20:51:10.836910+0000","last_active":"2023-09-23T21:00:13.250381+0000","last_peered":"2023-09-23T21:00:13.250381+0000","last_clean":"2023-09-23T21:00:13.250381+0000","last_became_active":"2023-09-23T20:51:10.836557+0000","last_became_peered":"2023-09-23T20:51:10.836557+0000","last_unstale":"2023-09-23T21:00:13.250381+0000","last_undegraded":"2023-09-23T21:00:13.250381+0000","last_fullsized":"2023-09-23T21:00:13.250381+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:51:54.019969+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":216,"reported_epoch":245,"state":"active+clean","last_fresh":"2023-09-23T21:00:13.274589+0000","last_change":"2023-09-23T20:55:35.953444+0000","last_active":"2023-09-23T21:00:13.274589+0000","last_peered":"2023-09-23T21:00:13.274589+0000","last_clean":"2023-09-23T21:00:13.274589+0000","last_became_active":"2023-09-23T20:55:35.953151+0000","last_became_peered":"2023-09-23T20:55:35.953151+0000","last_unstale":"2023-09-23T21:00:13.274589+0000","last_undegraded":"2023-09-23T21:00:13.274589+0000","last_fullsized":"2023-09-23T21:00:13.274589+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:39:12.404391+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":35,"reported_epoch":245,"state":"active+clean","last_fresh":"2023-09-23T21:00:13.249453+0000","last_change":"2023-09-23T20:57:44.982509+0000","last_active":"2023-09-23T21:00:13.249453+0000","last_peered":"2023-09-23T21:00:13.249453+0000","last_clean":"2023-09-23T21:00:13.249453+0000","last_became_active":"2023-09-23T20:57:44.982175+0000","last_became_peered":"2023-09-23T20:57:44.982175+0000","last_unstale":"2023-09-23T21:00:13.249453+0000","last_undegraded":"2023-09-23T21:00:13.249453+0000","last_fullsized":"2023-09-23T21:00:13.249453+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:37:49.215488+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.2","version":"246'11244","reported_seq":11301,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-09-23T21:00:24.708970+0000","last_change":"2023-09-23T20:56:35.543020+0000","last_active":"2023-09-23T21:00:24.708970+0000","last_peered":"2023-09-23T21:00:24.708970+0000","last_clean":"2023-09-23T21:00:24.708970+0000","last_became_active":"2023-09-23T20:56:35.542719+0000","last_became_peered":"2023-09-23T20:56:35.542719+0000","last_unstale":"2023-09-23T21:00:24.708970+0000","last_undegraded":"2023-09-23T21:00:24.708970+0000","last_fullsized":"2023-09-23T21:00:24.708970+0000","mapping_epoch":199,"log_start":"220'5100","ondisk_log_start":"220'5100","created":199,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_clean_scrub_stamp":"2023-09-23T20:56:34.529103+0000","objects_scrubbed":0,"log_size":6144,"log_dups_size":0,"ondisk_log_size":6144,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:52:21.088628+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":83099648,"num_objects":1268,"num_object_clones":0,"num_object_copies":3804,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1268,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11244,"num_write_kb":400384,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.4","version":"79'161","reported_seq":525,"reported_epoch":245,"state":"active+clean","last_fresh":"2023-09-23T21:00:14.730892+0000","last_change":"2023-09-23T20:57:20.012904+0000","last_active":"2023-09-23T21:00:14.730892+0000","last_peered":"2023-09-23T21:00:14.730892+0000","last_clean":"2023-09-23T21:00:14.730892+0000","last_became_active":"2023-09-23T20:57:08.237996+0000","last_became_peered":"2023-09-23T20:57:08.237996+0000","last_unstale":"2023-09-23T21:00:14.730892+0000","last_undegraded":"2023-09-23T21:00:14.730892+0000","last_fullsized":"2023-09-23T21:00:14.730892+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":3,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:20.012764+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:20.012764+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:51:28.796946+0000","scrub_duration":0.0045251919999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":195,"reported_epoch":245,"state":"active+clean","last_fresh":"2023-09-23T21:00:13.240956+0000","last_change":"2023-09-23T20:55:35.949541+0000","last_active":"2023-09-23T21:00:13.240956+0000","last_peered":"2023-09-23T21:00:13.240956+0000","last_clean":"2023-09-23T21:00:13.240956+0000","last_became_active":"2023-09-23T20:55:35.944635+0000","last_became_peered":"2023-09-23T20:55:35.944635+0000","last_unstale":"2023-09-23T21:00:13.240956+0000","last_undegraded":"2023-09-23T21:00:13.240956+0000","last_fullsized":"2023-09-23T21:00:13.240956+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:30:36.033585+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"70'1","reported_seq":235,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-09-23T21:00:20.334489+0000","last_change":"2023-09-23T20:54:14.009723+0000","last_active":"2023-09-23T21:00:20.334489+0000","last_peered":"2023-09-23T21:00:20.334489+0000","last_clean":"2023-09-23T21:00:20.334489+0000","last_became_active":"2023-09-23T20:54:13.941771+0000","last_became_peered":"2023-09-23T20:54:13.941771+0000","last_unstale":"2023-09-23T21:00:20.334489+0000","last_undegraded":"2023-09-23T21:00:20.334489+0000","last_fullsized":"2023-09-23T21:00:20.334489+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:48:25.588116+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":150,"reported_epoch":245,"state":"active+clean","last_fresh":"2023-09-23T21:00:13.258081+0000","last_change":"2023-09-23T20:55:35.949780+0000","last_active":"2023-09-23T21:00:13.258081+0000","last_peered":"2023-09-23T21:00:13.258081+0000","last_clean":"2023-09-23T21:00:13.258081+0000","last_became_active":"2023-09-23T20:55:35.949497+0000","last_became_peered":"2023-09-23T20:55:35.949497+0000","last_unstale":"2023-09-23T21:00:13.258081+0000","last_undegraded":"2023-09-23T21:00:13.258081+0000","last_fullsized":"2023-09-23T21:00:13.258081+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:09:37.420882+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":209,"reported_epoch":245,"state":"active+clean","last_fresh":"2023-09-23T21:00:13.270519+0000","last_change":"2023-09-23T20:54:13.929196+0000","last_active":"2023-09-23T21:00:13.270519+0000","last_peered":"2023-09-23T21:00:13.270519+0000","last_clean":"2023-09-23T21:00:13.270519+0000","last_became_active":"2023-09-23T20:54:13.928831+0000","last_became_peered":"2023-09-23T20:54:13.928831+0000","last_unstale":"2023-09-23T21:00:13.270519+0000","last_undegraded":"2023-09-23T21:00:13.270519+0000","last_fullsized":"2023-09-23T21:00:13.270519+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:28:44.316851+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":214,"reported_epoch":245,"state":"active+clean","last_fresh":"2023-09-23T21:00:13.261953+0000","last_change":"2023-09-23T20:52:12.855430+0000","last_active":"2023-09-23T21:00:13.261953+0000","last_peered":"2023-09-23T21:00:13.261953+0000","last_clean":"2023-09-23T21:00:13.261953+0000","last_became_active":"2023-09-23T20:52:12.855115+0000","last_became_peered":"2023-09-23T20:52:12.855115+0000","last_unstale":"2023-09-23T21:00:13.261953+0000","last_undegraded":"2023-09-23T21:00:13.261953+0000","last_fullsized":"2023-09-23T21:00:13.261953+0000","mapping_epoch":100,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":101,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:33:23.802214+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":123,"reported_epoch":245,"state":"active+clean","last_fresh":"2023-09-23T21:00:13.251542+0000","last_change":"2023-09-23T20:54:31.902946+0000","last_active":"2023-09-23T21:00:13.251542+0000","last_peered":"2023-09-23T21:00:13.251542+0000","last_clean":"2023-09-23T21:00:13.251542+0000","last_became_active":"2023-09-23T20:54:31.889484+0000","last_became_peered":"2023-09-23T20:54:31.889484+0000","last_unstale":"2023-09-23T21:00:13.251542+0000","last_undegraded":"2023-09-23T21:00:13.251542+0000","last_fullsized":"2023-09-23T21:00:13.251542+0000","mapping_epoch":136,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:44:44.278398+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":123,"reported_epoch":245,"state":"active+clean","last_fresh":"2023-09-23T21:00:13.258102+0000","last_change":"2023-09-23T20:54:13.950351+0000","last_active":"2023-09-23T21:00:13.258102+0000","last_peered":"2023-09-23T21:00:13.258102+0000","last_clean":"2023-09-23T21:00:13.258102+0000","last_became_active":"2023-09-23T20:54:13.949987+0000","last_became_peered":"2023-09-23T20:54:13.949987+0000","last_unstale":"2023-09-23T21:00:13.258102+0000","last_undegraded":"2023-09-23T21:00:13.258102+0000","last_fullsized":"2023-09-23T21:00:13.258102+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:46:17.765134+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":154,"reported_epoch":245,"state":"active+clean","last_fresh":"2023-09-23T21:00:13.251500+0000","last_change":"2023-09-23T20:56:18.141728+0000","last_active":"2023-09-23T21:00:13.251500+0000","last_peered":"2023-09-23T21:00:13.251500+0000","last_clean":"2023-09-23T21:00:13.251500+0000","last_became_active":"2023-09-23T20:56:18.137979+0000","last_became_peered":"2023-09-23T20:56:18.137979+0000","last_unstale":"2023-09-23T21:00:13.251500+0000","last_undegraded":"2023-09-23T21:00:13.251500+0000","last_fullsized":"2023-09-23T21:00:13.251500+0000","mapping_epoch":192,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":193,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:58:05.658443+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":123,"reported_epoch":245,"state":"active+clean","last_fresh":"2023-09-23T21:00:13.257475+0000","last_change":"2023-09-23T20:54:13.935307+0000","last_active":"2023-09-23T21:00:13.257475+0000","last_peered":"2023-09-23T21:00:13.257475+0000","last_clean":"2023-09-23T21:00:13.257475+0000","last_became_active":"2023-09-23T20:54:13.935000+0000","last_became_peered":"2023-09-23T20:54:13.935000+0000","last_unstale":"2023-09-23T21:00:13.257475+0000","last_undegraded":"2023-09-23T21:00:13.257475+0000","last_fullsized":"2023-09-23T21:00:13.257475+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:47:40.974082+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":233,"reported_epoch":245,"state":"active+clean","last_fresh":"2023-09-23T21:00:13.254177+0000","last_change":"2023-09-23T20:55:35.954870+0000","last_active":"2023-09-23T21:00:13.254177+0000","last_peered":"2023-09-23T21:00:13.254177+0000","last_clean":"2023-09-23T21:00:13.254177+0000","last_became_active":"2023-09-23T20:55:35.954568+0000","last_became_peered":"2023-09-23T20:55:35.954568+0000","last_unstale":"2023-09-23T21:00:13.254177+0000","last_undegraded":"2023-09-23T21:00:13.254177+0000","last_fullsized":"2023-09-23T21:00:13.254177+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:06:22.423566+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":232,"reported_epoch":245,"state":"active+clean","last_fresh":"2023-09-23T21:00:13.249887+0000","last_change":"2023-09-23T20:51:10.838334+0000","last_active":"2023-09-23T21:00:13.249887+0000","last_peered":"2023-09-23T21:00:13.249887+0000","last_clean":"2023-09-23T21:00:13.249887+0000","last_became_active":"2023-09-23T20:51:10.838026+0000","last_became_peered":"2023-09-23T20:51:10.838026+0000","last_unstale":"2023-09-23T21:00:13.249887+0000","last_undegraded":"2023-09-23T21:00:13.249887+0000","last_fullsized":"2023-09-23T21:00:13.249887+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:04:37.009457+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":237,"reported_epoch":245,"state":"active+clean","last_fresh":"2023-09-23T21:00:13.270983+0000","last_change":"2023-09-23T20:54:31.903139+0000","last_active":"2023-09-23T21:00:13.270983+0000","last_peered":"2023-09-23T21:00:13.270983+0000","last_clean":"2023-09-23T21:00:13.270983+0000","last_became_active":"2023-09-23T20:54:31.884520+0000","last_became_peered":"2023-09-23T20:54:31.884520+0000","last_unstale":"2023-09-23T21:00:13.270983+0000","last_undegraded":"2023-09-23T21:00:13.270983+0000","last_fullsized":"2023-09-23T21:00:13.270983+0000","mapping_epoch":136,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:39:34.269705+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1108,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":140,"reported_epoch":245,"state":"active+clean","last_fresh":"2023-09-23T21:00:13.257772+0000","last_change":"2023-09-23T20:55:35.951572+0000","last_active":"2023-09-23T21:00:13.257772+0000","last_peered":"2023-09-23T21:00:13.257772+0000","last_clean":"2023-09-23T21:00:13.257772+0000","last_became_active":"2023-09-23T20:55:35.951294+0000","last_became_peered":"2023-09-23T20:55:35.951294+0000","last_unstale":"2023-09-23T21:00:13.257772+0000","last_undegraded":"2023-09-23T21:00:13.257772+0000","last_fullsized":"2023-09-23T21:00:13.257772+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:36:32.447210+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":7,"num_pg":16,"stat_sum":{"num_bytes":1461059608,"num_objects":22295,"num_object_clones":0,"num_object_copies":66885,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":11892,"num_objects_misplaced":522,"num_objects_unfound":0,"num_objects_dirty":22295,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":181225,"num_write_kb":6512577,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3928,"num_bytes_recovered":257425408,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":5369180160,"data_stored":5369167944,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":119025,"ondisk_log_size":119025,"up":48,"acting":45,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":22,"num_bytes_recovered":5990,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":21,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3381,"ondisk_log_size":3381,"up":63,"acting":63,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201519,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":646412,"kb_used_data":464900,"kb_used_omap":0,"kb_used_meta":181440,"kb_avail":93098740,"statfs":{"total":95995035648,"available":95333109760,"internally_reserved":0,"allocated":476057600,"data_stored":475181854,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":185794560},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:59:46 2023","interfaces":[{"interface":"back","average":{"1min":1.6559999999999999,"5min":1.802,"15min":1.0840000000000001},"min":{"1min":0.245,"5min":0.21099999999999999,"15min":0.21099999999999999},"max":{"1min":6.4420000000000002,"5min":43.994999999999997,"15min":43.994999999999997},"last":1.7090000000000001},{"interface":"front","average":{"1min":1.6339999999999999,"5min":1.9159999999999999,"15min":1.1699999999999999},"min":{"1min":0.318,"5min":0.28199999999999997,"15min":0.218},"max":{"1min":6.4610000000000003,"5min":44.206000000000003,"15min":44.206000000000003},"last":1.694}]},{"osd":1,"last update":"Sat Sep 23 20:59:19 2023","interfaces":[{"interface":"back","average":{"1min":1.74,"5min":1.835,"15min":1.2230000000000001},"min":{"1min":0.26600000000000001,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":5.4359999999999999,"5min":10.622,"15min":22.591999999999999},"last":0.438},{"interface":"front","average":{"1min":2.0600000000000001,"5min":1.639,"15min":1.0860000000000001},"min":{"1min":0.32100000000000001,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":6.7709999999999999,"5min":10.097,"15min":21.437000000000001},"last":0.38900000000000001}]},{"osd":2,"last update":"Sat Sep 23 20:59:55 2023","interfaces":[{"interface":"back","average":{"1min":2.7149999999999999,"5min":2.2000000000000002,"15min":1.294},"min":{"1min":0.30399999999999999,"5min":0.26500000000000001,"15min":0.24399999999999999},"max":{"1min":8.5289999999999999,"5min":10.805999999999999,"15min":10.805999999999999},"last":1.802},{"interface":"front","average":{"1min":2.339,"5min":2.2639999999999998,"15min":1.3089999999999999},"min":{"1min":0.26500000000000001,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":11.590999999999999,"5min":45.25,"15min":45.25},"last":0.41799999999999998}]},{"osd":3,"last update":"Sat Sep 23 20:59:30 2023","interfaces":[{"interface":"back","average":{"1min":2.6200000000000001,"5min":2.508,"15min":1.7250000000000001},"min":{"1min":0.33500000000000002,"5min":0.32100000000000001,"15min":0.314},"max":{"1min":8.4390000000000001,"5min":13.007,"15min":36.408000000000001},"last":0.498},{"interface":"front","average":{"1min":2.7789999999999999,"5min":2.3079999999999998,"15min":1.552},"min":{"1min":0.41599999999999998,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":13.198,"5min":14.220000000000001,"15min":35.593000000000004},"last":9.7970000000000006}]},{"osd":4,"last update":"Sat Sep 23 20:59:19 2023","interfaces":[{"interface":"back","average":{"1min":0.41099999999999998,"5min":0.78500000000000003,"15min":0.60499999999999998},"min":{"1min":0.25700000000000001,"5min":0.23799999999999999,"15min":0.23699999999999999},"max":{"1min":1.1659999999999999,"5min":5.8920000000000003,"15min":5.8920000000000003},"last":0.45600000000000002},{"interface":"front","average":{"1min":0.51900000000000002,"5min":0.77600000000000002,"15min":0.61299999999999999},"min":{"1min":0.28599999999999998,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":2.952,"5min":5.3200000000000003,"15min":5.3200000000000003},"last":0.34000000000000002}]},{"osd":5,"last update":"Sat Sep 23 20:59:19 2023","interfaces":[{"interface":"back","average":{"1min":0.443,"5min":0.78100000000000003,"15min":0.628},"min":{"1min":0.316,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":0.64000000000000001,"5min":7.0389999999999997,"15min":7.0389999999999997},"last":0.40100000000000002},{"interface":"front","average":{"1min":0.55200000000000005,"5min":0.72299999999999998,"15min":0.61399999999999999},"min":{"1min":0.30199999999999999,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":1.909,"5min":4.5999999999999996,"15min":4.5999999999999996},"last":0.60599999999999998}]},{"osd":6,"last update":"Sat Sep 23 20:59:42 2023","interfaces":[{"interface":"back","average":{"1min":0.48299999999999998,"5min":0.68000000000000005,"15min":0.67100000000000004},"min":{"1min":0.28899999999999998,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":0.77300000000000002,"5min":3.5249999999999999,"15min":3.5249999999999999},"last":0.55900000000000005},{"interface":"front","average":{"1min":0.48499999999999999,"5min":0.749,"15min":0.78600000000000003},"min":{"1min":0.34699999999999998,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":0.95499999999999996,"5min":3.5409999999999999,"15min":3.5409999999999999},"last":1.605}]}]},{"osd":6,"up_from":164,"seq":704374636603,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":554688,"kb_used_data":447480,"kb_used_omap":10,"kb_used_meta":107125,"kb_avail":93190464,"statfs":{"total":95995035648,"available":95427035136,"internally_reserved":0,"allocated":458219520,"data_stored":457355675,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":10328,"internal_metadata":109696936},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,3],"upper_bound":32},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:59:45 2023","interfaces":[{"interface":"back","average":{"1min":1.1599999999999999,"5min":1.5,"15min":1.0760000000000001},"min":{"1min":0.23599999999999999,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":4.0960000000000001,"5min":8.7319999999999993,"15min":8.7319999999999993},"last":0.76500000000000001},{"interface":"front","average":{"1min":1.5029999999999999,"5min":1.6060000000000001,"15min":1.1899999999999999},"min":{"1min":0.23999999999999999,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":4.2869999999999999,"5min":6.7140000000000004,"15min":6.7140000000000004},"last":1.881}]},{"osd":1,"last update":"Sat Sep 23 20:59:45 2023","interfaces":[{"interface":"back","average":{"1min":1.554,"5min":1.25,"15min":0.94199999999999995},"min":{"1min":0.27200000000000002,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":8.4309999999999992,"5min":8.4309999999999992,"15min":8.4309999999999992},"last":0.38800000000000001},{"interface":"front","average":{"1min":1.506,"5min":1.3720000000000001,"15min":1.036},"min":{"1min":0.26100000000000001,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":7.2069999999999999,"5min":10.324999999999999,"15min":10.324999999999999},"last":6.1289999999999996}]},{"osd":2,"last update":"Sat Sep 23 20:59:45 2023","interfaces":[{"interface":"back","average":{"1min":2.097,"5min":2.0720000000000001,"15min":1.788},"min":{"1min":0.312,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":6.0670000000000002,"5min":13.750999999999999,"15min":13.750999999999999},"last":8.6769999999999996},{"interface":"front","average":{"1min":2.46,"5min":1.8,"15min":1.3380000000000001},"min":{"1min":0.32000000000000001,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":5.6130000000000004,"5min":8.0890000000000004,"15min":8.0890000000000004},"last":0.44}]},{"osd":3,"last update":"Sat Sep 23 20:59:45 2023","interfaces":[{"interface":"back","average":{"1min":1.768,"5min":1.415,"15min":1.216},"min":{"1min":0.45000000000000001,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":4.7220000000000004,"5min":6.9000000000000004,"15min":6.9000000000000004},"last":13.214},{"interface":"front","average":{"1min":1.6479999999999999,"5min":1.7490000000000001,"15min":1.651},"min":{"1min":0.34999999999999998,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":4.6289999999999996,"5min":11.021000000000001,"15min":11.021000000000001},"last":0.86199999999999999}]},{"osd":4,"last update":"Sat Sep 23 20:59:45 2023","interfaces":[{"interface":"back","average":{"1min":0.68700000000000006,"5min":0.78600000000000003,"15min":0.96299999999999997},"min":{"1min":0.27100000000000002,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":2.3460000000000001,"5min":9.0079999999999991,"15min":9.0079999999999991},"last":0.33700000000000002},{"interface":"front","average":{"1min":0.48599999999999999,"5min":0.499,"15min":0.52000000000000002},"min":{"1min":0.29899999999999999,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":1.115,"5min":2.0059999999999998,"15min":2.0059999999999998},"last":0.45400000000000001}]},{"osd":5,"last update":"Sat Sep 23 20:59:45 2023","interfaces":[{"interface":"back","average":{"1min":0.78100000000000003,"5min":0.61699999999999999,"15min":0.56599999999999995},"min":{"1min":0.28100000000000003,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":2.3730000000000002,"5min":2.8239999999999998,"15min":2.8239999999999998},"last":0.372},{"interface":"front","average":{"1min":0.58299999999999996,"5min":0.73999999999999999,"15min":0.78600000000000003},"min":{"1min":0.34000000000000002,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":1.2290000000000001,"5min":3.1379999999999999,"15min":3.1379999999999999},"last":0.51700000000000002}]},{"osd":7,"last update":"Sat Sep 23 20:59:45 2023","interfaces":[{"interface":"back","average":{"1min":0.51700000000000002,"5min":0.77200000000000002,"15min":0.876},"min":{"1min":0.318,"5min":0.312,"15min":0.312},"max":{"1min":1.1819999999999999,"5min":7.8570000000000002,"15min":7.8570000000000002},"last":0.48999999999999999},{"interface":"front","average":{"1min":0.46500000000000002,"5min":0.64900000000000002,"15min":0.56899999999999995},"min":{"1min":0.33800000000000002,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":0.63200000000000001,"5min":8.282,"15min":8.282},"last":0.41799999999999998}]}]},{"osd":1,"up_from":88,"seq":377957122174,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1117900,"kb_used_data":876932,"kb_used_omap":9,"kb_used_meta":240886,"kb_avail":92627252,"statfs":{"total":95995035648,"available":94850306048,"internally_reserved":0,"allocated":897978368,"data_stored":896756540,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9239,"internal_metadata":246668265},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,0,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:59:23 2023","interfaces":[{"interface":"back","average":{"1min":3.9180000000000001,"5min":3.1070000000000002,"15min":1.5389999999999999},"min":{"1min":0.20699999999999999,"5min":0.20699999999999999,"15min":0.20699999999999999},"max":{"1min":14.813000000000001,"5min":16.646999999999998,"15min":16.646999999999998},"last":0.49199999999999999},{"interface":"front","average":{"1min":3.77,"5min":3.335,"15min":1.6819999999999999},"min":{"1min":0.33300000000000002,"5min":0.246,"15min":0.22500000000000001},"max":{"1min":9.484,"5min":16.664999999999999,"15min":18.457999999999998},"last":0.41099999999999998}]},{"osd":2,"last update":"Sat Sep 23 20:59:55 2023","interfaces":[{"interface":"back","average":{"1min":2.4809999999999999,"5min":2.3210000000000002,"15min":1.319},"min":{"1min":0.222,"5min":0.221,"15min":0.20300000000000001},"max":{"1min":13.752000000000001,"5min":14.335000000000001,"15min":14.335000000000001},"last":0.70899999999999996},{"interface":"front","average":{"1min":3.4470000000000001,"5min":3.3519999999999999,"15min":1.948},"min":{"1min":0.27400000000000002,"5min":0.248,"15min":0.248},"max":{"1min":18.026,"5min":18.026,"15min":38.951000000000001},"last":0.82399999999999995}]},{"osd":3,"last update":"Sat Sep 23 20:59:23 2023","interfaces":[{"interface":"back","average":{"1min":3.1539999999999999,"5min":2.629,"15min":1.5589999999999999},"min":{"1min":0.42499999999999999,"5min":0.246,"15min":0.246},"max":{"1min":9.4450000000000003,"5min":16.055,"15min":25.314},"last":0.99199999999999999},{"interface":"front","average":{"1min":3.0870000000000002,"5min":2.4860000000000002,"15min":1.411},"min":{"1min":0.29799999999999999,"5min":0.20599999999999999,"15min":0.20599999999999999},"max":{"1min":10.204000000000001,"5min":14.225,"15min":19.881},"last":0.72899999999999998}]},{"osd":4,"last update":"Sat Sep 23 20:59:23 2023","interfaces":[{"interface":"back","average":{"1min":1.871,"5min":1.6439999999999999,"15min":1.2130000000000001},"min":{"1min":0.28299999999999997,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":9.3480000000000008,"5min":10.598000000000001,"15min":12.006},"last":0.34499999999999997},{"interface":"front","average":{"1min":1.633,"5min":1.675,"15min":1.0800000000000001},"min":{"1min":0.38400000000000001,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":7.532,"5min":7.532,"15min":11.234999999999999},"last":2.3039999999999998}]},{"osd":5,"last update":"Sat Sep 23 20:59:23 2023","interfaces":[{"interface":"back","average":{"1min":2.3700000000000001,"5min":2.1469999999999998,"15min":1.292},"min":{"1min":0.55800000000000005,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":8.4410000000000007,"5min":15.041,"15min":15.041},"last":0.52500000000000002},{"interface":"front","average":{"1min":1.9370000000000001,"5min":1.756,"15min":1.2390000000000001},"min":{"1min":0.432,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":9.3889999999999993,"5min":10.647,"15min":12.08},"last":0.40000000000000002}]},{"osd":6,"last update":"Sat Sep 23 20:59:41 2023","interfaces":[{"interface":"back","average":{"1min":2.266,"5min":1.5609999999999999,"15min":1.397},"min":{"1min":0.46200000000000002,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":10.159000000000001,"5min":10.673999999999999,"15min":10.673999999999999},"last":0.438},{"interface":"front","average":{"1min":1.8660000000000001,"5min":1.6299999999999999,"15min":1.3779999999999999},"min":{"1min":0.33700000000000002,"5min":0.309,"15min":0.309},"max":{"1min":7.6779999999999999,"5min":8.7530000000000001,"15min":8.7530000000000001},"last":0.63300000000000001}]},{"osd":7,"last update":"Sat Sep 23 20:59:23 2023","interfaces":[{"interface":"back","average":{"1min":2.089,"5min":1.871,"15min":1.163},"min":{"1min":0.433,"5min":0.36199999999999999,"15min":0.35699999999999998},"max":{"1min":8.7769999999999992,"5min":8.7769999999999992,"15min":11.186999999999999},"last":0.68000000000000005},{"interface":"front","average":{"1min":2.5430000000000001,"5min":2.202,"15min":1.325},"min":{"1min":0.497,"5min":0.40899999999999997,"15min":0.34599999999999997},"max":{"1min":8.4939999999999998,"5min":15.098000000000001,"15min":15.098000000000001},"last":0.58199999999999996}]}]},{"osd":0,"up_from":83,"seq":356482285700,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":937256,"kb_used_data":743008,"kb_used_omap":28,"kb_used_meta":194019,"kb_avail":92807896,"statfs":{"total":95995035648,"available":95035285504,"internally_reserved":0,"allocated":760840192,"data_stored":759651694,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":28848,"internal_metadata":198676304},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,0,2],"upper_bound":64},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 21:00:20 2023","interfaces":[{"interface":"back","average":{"1min":6.4480000000000004,"5min":3.7120000000000002,"15min":1.893},"min":{"1min":0.20000000000000001,"5min":0.191,"15min":0.191},"max":{"1min":19.146000000000001,"5min":19.146000000000001,"15min":19.146000000000001},"last":6.4509999999999996},{"interface":"front","average":{"1min":5.4379999999999997,"5min":3.6789999999999998,"15min":1.8759999999999999},"min":{"1min":0.39500000000000002,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":11.074,"5min":15.771000000000001,"15min":17.704999999999998},"last":3.2530000000000001}]},{"osd":2,"last update":"Sat Sep 23 20:59:51 2023","interfaces":[{"interface":"back","average":{"1min":3.8879999999999999,"5min":3.4180000000000001,"15min":1.867},"min":{"1min":0.50900000000000001,"5min":0.28299999999999997,"15min":0.23799999999999999},"max":{"1min":21.053000000000001,"5min":21.053000000000001,"15min":21.053000000000001},"last":0.34899999999999998},{"interface":"front","average":{"1min":3.6579999999999999,"5min":3.7450000000000001,"15min":1.944},"min":{"1min":0.26300000000000001,"5min":0.25800000000000001,"15min":0.21099999999999999},"max":{"1min":13.956,"5min":21.747,"15min":21.747},"last":3.133}]},{"osd":3,"last update":"Sat Sep 23 21:00:20 2023","interfaces":[{"interface":"back","average":{"1min":6.532,"5min":3.8690000000000002,"15min":1.9990000000000001},"min":{"1min":1.052,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":15.082000000000001,"5min":21.184000000000001,"15min":21.184000000000001},"last":2.839},{"interface":"front","average":{"1min":4.9050000000000002,"5min":4.008,"15min":2.0899999999999999},"min":{"1min":0.48499999999999999,"5min":0.28000000000000003,"15min":0.23300000000000001},"max":{"1min":9.7560000000000002,"5min":21.167000000000002,"15min":21.167000000000002},"last":6.1719999999999997}]},{"osd":4,"last update":"Sat Sep 23 20:59:51 2023","interfaces":[{"interface":"back","average":{"1min":1.9359999999999999,"5min":2.823,"15min":1.5589999999999999},"min":{"1min":0.38500000000000001,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":7.931,"5min":21.594999999999999,"15min":21.594999999999999},"last":3.2040000000000002},{"interface":"front","average":{"1min":2.8639999999999999,"5min":1.9830000000000001,"15min":1.278},"min":{"1min":0.41099999999999998,"5min":0.31,"15min":0.28100000000000003},"max":{"1min":13.173999999999999,"5min":13.173999999999999,"15min":18.724},"last":2.8999999999999999}]},{"osd":5,"last update":"Sat Sep 23 20:59:51 2023","interfaces":[{"interface":"back","average":{"1min":1.859,"5min":1.889,"15min":1.258},"min":{"1min":0.38700000000000001,"5min":0.318,"15min":0.318},"max":{"1min":6.9980000000000002,"5min":8.8970000000000002,"15min":8.8970000000000002},"last":0.39300000000000002},{"interface":"front","average":{"1min":1.748,"5min":2.7290000000000001,"15min":1.5649999999999999},"min":{"1min":0.41899999999999998,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":7.9820000000000002,"5min":20.512,"15min":20.512},"last":3.1749999999999998}]},{"osd":6,"last update":"Sat Sep 23 20:59:46 2023","interfaces":[{"interface":"back","average":{"1min":2.9180000000000001,"5min":2.145,"15min":1.819},"min":{"1min":0.45900000000000002,"5min":0.37,"15min":0.37},"max":{"1min":13.134,"5min":13.134,"15min":13.134},"last":2.9249999999999998},{"interface":"front","average":{"1min":2.9140000000000001,"5min":2.198,"15min":1.8959999999999999},"min":{"1min":0.43099999999999999,"5min":0.377,"15min":0.377},"max":{"1min":13.254,"5min":13.254,"15min":13.254},"last":2.9740000000000002}]},{"osd":7,"last update":"Sat Sep 23 20:59:51 2023","interfaces":[{"interface":"back","average":{"1min":2.1520000000000001,"5min":2.0139999999999998,"15min":1.3169999999999999},"min":{"1min":0.433,"5min":0.379,"15min":0.379},"max":{"1min":6.9699999999999998,"5min":8.7729999999999997,"15min":8.7729999999999997},"last":0.42799999999999999},{"interface":"front","average":{"1min":3.0459999999999998,"5min":2.105,"15min":1.3200000000000001},"min":{"1min":0.44600000000000001,"5min":0.377,"15min":0.377},"max":{"1min":13.218999999999999,"5min":13.218999999999999,"15min":18.673999999999999},"last":2.948}]}]},{"osd":2,"up_from":93,"seq":399431958647,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1229716,"kb_used_data":907980,"kb_used_omap":81,"kb_used_meta":321646,"kb_avail":92515436,"statfs":{"total":95995035648,"available":94735806464,"internally_reserved":0,"allocated":929771520,"data_stored":928599477,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":83750,"internal_metadata":329365722},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,2,4,1,2,1],"upper_bound":128},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:59:50 2023","interfaces":[{"interface":"back","average":{"1min":4.8730000000000002,"5min":2.7000000000000002,"15min":1.3799999999999999},"min":{"1min":0.189,"5min":0.17999999999999999,"15min":0.17999999999999999},"max":{"1min":29.498000000000001,"5min":29.498000000000001,"15min":29.498000000000001},"last":0.51700000000000002},{"interface":"front","average":{"1min":4.032,"5min":3.3370000000000002,"15min":1.6739999999999999},"min":{"1min":0.20599999999999999,"5min":0.20599999999999999,"15min":0.20599999999999999},"max":{"1min":17.251000000000001,"5min":17.251000000000001,"15min":17.251000000000001},"last":3.411}]},{"osd":1,"last update":"Sat Sep 23 20:59:50 2023","interfaces":[{"interface":"back","average":{"1min":2.6030000000000002,"5min":2.5619999999999998,"15min":1.625},"min":{"1min":0.25700000000000001,"5min":0.20200000000000001,"15min":0.20200000000000001},"max":{"1min":7.9800000000000004,"5min":14.709,"15min":16.081},"last":0.70499999999999996},{"interface":"front","average":{"1min":3.427,"5min":3.2570000000000001,"15min":1.97},"min":{"1min":0.26600000000000001,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":9.9090000000000007,"5min":11.717000000000001,"15min":16.041},"last":3.4550000000000001}]},{"osd":3,"last update":"Sat Sep 23 20:59:29 2023","interfaces":[{"interface":"back","average":{"1min":5.0449999999999999,"5min":3.6579999999999999,"15min":1.893},"min":{"1min":0.30299999999999999,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":27.614999999999998,"5min":27.614999999999998,"15min":27.614999999999998},"last":3.5870000000000002},{"interface":"front","average":{"1min":5.9119999999999999,"5min":4.234,"15min":2.1930000000000001},"min":{"1min":0.33000000000000002,"5min":0.22800000000000001,"15min":0.221},"max":{"1min":27.562000000000001,"5min":27.562000000000001,"15min":27.562000000000001},"last":0.64400000000000002}]},{"osd":4,"last update":"Sat Sep 23 20:59:50 2023","interfaces":[{"interface":"back","average":{"1min":1.575,"5min":1.7749999999999999,"15min":1.2070000000000001},"min":{"1min":0.36499999999999999,"5min":0.28699999999999998,"15min":0.26400000000000001},"max":{"1min":9.8659999999999997,"5min":9.8659999999999997,"15min":9.8659999999999997},"last":3.5569999999999999},{"interface":"front","average":{"1min":2.8439999999999999,"5min":2.1880000000000002,"15min":1.3720000000000001},"min":{"1min":0.32400000000000001,"5min":0.313,"15min":0.313},"max":{"1min":9.9719999999999995,"5min":10.609,"15min":10.609},"last":0.61499999999999999}]},{"osd":5,"last update":"Sat Sep 23 20:59:50 2023","interfaces":[{"interface":"back","average":{"1min":1.4179999999999999,"5min":1.99,"15min":1.298},"min":{"1min":0.33000000000000002,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":6.3570000000000002,"5min":9.0329999999999995,"15min":9.7629999999999999},"last":3.52},{"interface":"front","average":{"1min":2.5760000000000001,"5min":2.1419999999999999,"15min":1.3720000000000001},"min":{"1min":0.35599999999999998,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":9.9220000000000006,"5min":10.574,"15min":10.574},"last":0.98199999999999998}]},{"osd":6,"last update":"Sat Sep 23 20:59:45 2023","interfaces":[{"interface":"back","average":{"1min":2.9060000000000001,"5min":2.0640000000000001,"15min":1.8},"min":{"1min":0.35699999999999998,"5min":0.312,"15min":0.312},"max":{"1min":10.012,"5min":14.244999999999999,"15min":14.244999999999999},"last":2.4870000000000001},{"interface":"front","average":{"1min":2.8740000000000001,"5min":1.8129999999999999,"15min":1.2909999999999999},"min":{"1min":0.41599999999999998,"5min":0.374,"15min":0.374},"max":{"1min":10.032999999999999,"5min":10.032999999999999,"15min":10.032999999999999},"last":2.4489999999999998}]},{"osd":7,"last update":"Sat Sep 23 20:59:50 2023","interfaces":[{"interface":"back","average":{"1min":1.3859999999999999,"5min":1.4830000000000001,"15min":1.0449999999999999},"min":{"1min":0.35899999999999999,"5min":0.315,"15min":0.315},"max":{"1min":5.9199999999999999,"5min":8.0820000000000007,"15min":8.0820000000000007},"last":0.55000000000000004},{"interface":"front","average":{"1min":1.121,"5min":1.873,"15min":1.292},"min":{"1min":0.443,"5min":0.35099999999999998,"15min":0.34499999999999997},"max":{"1min":6.3899999999999997,"5min":8.6219999999999999,"15min":9.7859999999999996},"last":3.4910000000000001}]}]},{"osd":3,"up_from":98,"seq":420906795121,"num_pgs":49,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1055480,"kb_used_data":687664,"kb_used_omap":7,"kb_used_meta":367736,"kb_avail":92689672,"statfs":{"total":95995035648,"available":94914224128,"internally_reserved":0,"allocated":704167936,"data_stored":703021059,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7576,"internal_metadata":376562280},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,2,1,1,3],"upper_bound":64},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:59:28 2023","interfaces":[{"interface":"back","average":{"1min":4.056,"5min":2.399,"15min":1.7350000000000001},"min":{"1min":0.30199999999999999,"5min":0.20599999999999999,"15min":0.20599999999999999},"max":{"1min":13.566000000000001,"5min":13.566000000000001,"15min":23.123999999999999},"last":1.6599999999999999},{"interface":"front","average":{"1min":5.2229999999999999,"5min":3.1099999999999999,"15min":1.6259999999999999},"min":{"1min":0.23200000000000001,"5min":0.19800000000000001,"15min":0.19800000000000001},"max":{"1min":13.455,"5min":15.079000000000001,"15min":15.079000000000001},"last":0.99399999999999999}]},{"osd":1,"last update":"Sat Sep 23 20:59:28 2023","interfaces":[{"interface":"back","average":{"1min":2.9510000000000001,"5min":2.8220000000000001,"15min":1.6120000000000001},"min":{"1min":0.23899999999999999,"5min":0.23899999999999999,"15min":0.22900000000000001},"max":{"1min":11.343,"5min":16.318000000000001,"15min":21.385000000000002},"last":0.22600000000000001},{"interface":"front","average":{"1min":2.972,"5min":2.2290000000000001,"15min":1.6830000000000001},"min":{"1min":0.23400000000000001,"5min":0.218,"15min":0.218},"max":{"1min":11.782,"5min":11.782,"15min":25.353000000000002},"last":0.29399999999999998}]},{"osd":2,"last update":"Sat Sep 23 20:59:28 2023","interfaces":[{"interface":"back","average":{"1min":4.7210000000000001,"5min":3.1459999999999999,"15min":1.7310000000000001},"min":{"1min":0.33000000000000002,"5min":0.33000000000000002,"15min":0.30099999999999999},"max":{"1min":13.616,"5min":15.476000000000001,"15min":24.154},"last":1.105},{"interface":"front","average":{"1min":3.665,"5min":2.4790000000000001,"15min":1.792},"min":{"1min":0.27300000000000002,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":13.125999999999999,"5min":16.032,"15min":25.305},"last":1.1180000000000001}]},{"osd":4,"last update":"Sat Sep 23 20:59:28 2023","interfaces":[{"interface":"back","average":{"1min":3.0070000000000001,"5min":2.0299999999999998,"15min":1.3},"min":{"1min":0.379,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":11.089,"5min":11.089,"15min":11.861000000000001},"last":0.314},{"interface":"front","average":{"1min":1.954,"5min":1.359,"15min":1.335},"min":{"1min":0.309,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":9.9779999999999998,"5min":9.9779999999999998,"15min":22.853000000000002},"last":0.435}]},{"osd":5,"last update":"Sat Sep 23 20:59:28 2023","interfaces":[{"interface":"back","average":{"1min":2.8540000000000001,"5min":2.052,"15min":1.2929999999999999},"min":{"1min":0.39000000000000001,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":11.048,"5min":11.048,"15min":11.891999999999999},"last":0.47399999999999998},{"interface":"front","average":{"1min":3.585,"5min":1.9510000000000001,"15min":1.272},"min":{"1min":0.53200000000000003,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":13.43,"5min":13.43,"15min":13.43},"last":0.57299999999999995}]},{"osd":6,"last update":"Sat Sep 23 20:59:47 2023","interfaces":[{"interface":"back","average":{"1min":2.0579999999999998,"5min":1.415,"15min":1.105},"min":{"1min":0.33700000000000002,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":13.499000000000001,"5min":13.499000000000001,"15min":13.499000000000001},"last":0.40899999999999997},{"interface":"front","average":{"1min":3.8410000000000002,"5min":1.966,"15min":1.5960000000000001},"min":{"1min":0.55300000000000005,"5min":0.38700000000000001,"15min":0.38700000000000001},"max":{"1min":13.398999999999999,"5min":13.398999999999999,"15min":13.398999999999999},"last":0.54900000000000004}]},{"osd":7,"last update":"Sat Sep 23 20:59:28 2023","interfaces":[{"interface":"back","average":{"1min":2.8700000000000001,"5min":2.3839999999999999,"15min":1.4670000000000001},"min":{"1min":0.42799999999999999,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":11.114000000000001,"5min":16.364999999999998,"15min":16.364999999999998},"last":0.443},{"interface":"front","average":{"1min":3.6400000000000001,"5min":2.077,"15min":1.3440000000000001},"min":{"1min":0.58399999999999996,"5min":0.379,"15min":0.379},"max":{"1min":13.355,"5min":13.355,"15min":13.355},"last":0.51200000000000001}]}]},{"osd":4,"up_from":31,"seq":133143986428,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":619916,"kb_used_data":471428,"kb_used_omap":0,"kb_used_meta":148480,"kb_avail":93125236,"statfs":{"total":95995035648,"available":95360241664,"internally_reserved":0,"allocated":482742272,"data_stored":481866779,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":152043520},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:59:53 2023","interfaces":[{"interface":"back","average":{"1min":2.1469999999999998,"5min":1.915,"15min":1.143},"min":{"1min":0.255,"5min":0.214,"15min":0.214},"max":{"1min":9.0749999999999993,"5min":11.784000000000001,"15min":11.784000000000001},"last":0.63200000000000001},{"interface":"front","average":{"1min":2.4969999999999999,"5min":2.0329999999999999,"15min":1.335},"min":{"1min":0.32800000000000001,"5min":0.26700000000000002,"15min":0.25800000000000001},"max":{"1min":9.1029999999999998,"5min":11.766,"15min":11.766},"last":0.64100000000000001}]},{"osd":1,"last update":"Sat Sep 23 20:59:22 2023","interfaces":[{"interface":"back","average":{"1min":2.4119999999999999,"5min":1.6930000000000001,"15min":1.1200000000000001},"min":{"1min":0.35499999999999998,"5min":0.246,"15min":0.23599999999999999},"max":{"1min":7.3310000000000004,"5min":10.153,"15min":10.153},"last":0.35699999999999998},{"interface":"front","average":{"1min":3.2669999999999999,"5min":2.0099999999999998,"15min":1.1419999999999999},"min":{"1min":0.33000000000000002,"5min":0.26100000000000001,"15min":0.23200000000000001},"max":{"1min":9.7509999999999994,"5min":12.292999999999999,"15min":12.292999999999999},"last":5.359}]},{"osd":2,"last update":"Sat Sep 23 20:59:53 2023","interfaces":[{"interface":"back","average":{"1min":2.573,"5min":1.7430000000000001,"15min":1.2410000000000001},"min":{"1min":0.33600000000000002,"5min":0.311,"15min":0.29099999999999998},"max":{"1min":8.1509999999999998,"5min":8.1509999999999998,"15min":15.288},"last":0.32500000000000001},{"interface":"front","average":{"1min":2.8330000000000002,"5min":1.831,"15min":1.3999999999999999},"min":{"1min":0.28599999999999998,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":12.855,"5min":12.855,"15min":15.262},"last":0.54900000000000004}]},{"osd":3,"last update":"Sat Sep 23 20:59:22 2023","interfaces":[{"interface":"back","average":{"1min":2.21,"5min":2.1850000000000001,"15min":1.6180000000000001},"min":{"1min":0.30499999999999999,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":4.835,"5min":14.824999999999999,"15min":31.562999999999999},"last":8.5389999999999997},{"interface":"front","average":{"1min":2.2770000000000001,"5min":2.2989999999999999,"15min":1.6899999999999999},"min":{"1min":0.30299999999999999,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":4.8490000000000002,"5min":14.839,"15min":31.574000000000002},"last":8.5570000000000004}]},{"osd":5,"last update":"Sat Sep 23 20:59:36 2023","interfaces":[{"interface":"back","average":{"1min":0.41099999999999998,"5min":0.61099999999999999,"15min":0.52800000000000002},"min":{"1min":0.28899999999999998,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":0.90700000000000003,"5min":3.3279999999999998,"15min":3.3279999999999998},"last":0.28799999999999998},{"interface":"front","average":{"1min":1.002,"5min":0.77800000000000002,"15min":0.61499999999999999},"min":{"1min":0.29999999999999999,"5min":0.251,"15min":0.251},"max":{"1min":3.4830000000000001,"5min":7.7619999999999996,"15min":7.7619999999999996},"last":0.30499999999999999}]},{"osd":6,"last update":"Sat Sep 23 20:59:46 2023","interfaces":[{"interface":"back","average":{"1min":0.62,"5min":0.70399999999999996,"15min":0.55700000000000005},"min":{"1min":0.32100000000000001,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":3.4180000000000001,"5min":4.6909999999999998,"15min":4.6909999999999998},"last":0.38500000000000001},{"interface":"front","average":{"1min":0.752,"5min":0.73699999999999999,"15min":0.67500000000000004},"min":{"1min":0.34499999999999997,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":3.2999999999999998,"5min":3.456,"15min":3.456},"last":0.374}]},{"osd":7,"last update":"Sat Sep 23 20:59:22 2023","interfaces":[{"interface":"back","average":{"1min":0.83499999999999996,"5min":0.752,"15min":0.69399999999999995},"min":{"1min":0.32600000000000001,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":2.8889999999999998,"5min":6.2240000000000002,"15min":7.383},"last":0.34899999999999998},{"interface":"front","average":{"1min":0.73999999999999999,"5min":0.78400000000000003,"15min":0.67500000000000004},"min":{"1min":0.34200000000000003,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":2.8679999999999999,"5min":7.8310000000000004,"15min":7.8310000000000004},"last":0.501}]}]},{"osd":5,"up_from":39,"seq":167503724793,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":858624,"kb_used_data":728824,"kb_used_omap":0,"kb_used_meta":129792,"kb_avail":92886528,"statfs":{"total":95995035648,"available":95115804672,"internally_reserved":0,"allocated":746315776,"data_stored":745448859,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":132907008},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,0,0,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:59:38 2023","interfaces":[{"interface":"back","average":{"1min":4.0339999999999998,"5min":1.619,"15min":0.96299999999999997},"min":{"1min":0.27400000000000002,"5min":0.20399999999999999,"15min":0.20399999999999999},"max":{"1min":12.795999999999999,"5min":12.795999999999999,"15min":12.795999999999999},"last":5.7359999999999998},{"interface":"front","average":{"1min":2.7330000000000001,"5min":1.948,"15min":1.125},"min":{"1min":0.24099999999999999,"5min":0.223,"15min":0.223},"max":{"1min":16.219999999999999,"5min":16.219999999999999,"15min":16.219999999999999},"last":2.3410000000000002}]},{"osd":1,"last update":"Sat Sep 23 21:00:18 2023","interfaces":[{"interface":"back","average":{"1min":2.8759999999999999,"5min":1.8779999999999999,"15min":1.2529999999999999},"min":{"1min":0.26600000000000001,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":8.7720000000000002,"5min":8.7720000000000002,"15min":8.7720000000000002},"last":2.6680000000000001},{"interface":"front","average":{"1min":2.3639999999999999,"5min":1.9330000000000001,"15min":1.22},"min":{"1min":0.34200000000000003,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":8.5109999999999992,"5min":10.285,"15min":10.285},"last":3.6139999999999999}]},{"osd":2,"last update":"Sat Sep 23 20:59:54 2023","interfaces":[{"interface":"back","average":{"1min":2.6819999999999999,"5min":1.6699999999999999,"15min":1.121},"min":{"1min":0.27400000000000002,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":10.371,"5min":10.579000000000001,"15min":12.651},"last":6.3200000000000003},{"interface":"front","average":{"1min":3.423,"5min":2.258,"15min":1.274},"min":{"1min":0.28799999999999998,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":10.333,"5min":12.300000000000001,"15min":12.300000000000001},"last":0.28599999999999998}]},{"osd":3,"last update":"Sat Sep 23 21:00:18 2023","interfaces":[{"interface":"back","average":{"1min":4.931,"5min":2.859,"15min":1.6379999999999999},"min":{"1min":0.35999999999999999,"5min":0.35399999999999998,"15min":0.28599999999999998},"max":{"1min":20.788,"5min":20.788,"15min":20.788},"last":0.39900000000000002},{"interface":"front","average":{"1min":3.9870000000000001,"5min":2.177,"15min":1.2949999999999999},"min":{"1min":0.39100000000000001,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":20.539000000000001,"5min":20.539000000000001,"15min":20.539000000000001},"last":2.8130000000000002}]},{"osd":4,"last update":"Sat Sep 23 20:59:34 2023","interfaces":[{"interface":"back","average":{"1min":0.81699999999999995,"5min":0.63300000000000001,"15min":0.55300000000000005},"min":{"1min":0.32100000000000001,"5min":0.252,"15min":0.252},"max":{"1min":2.9169999999999998,"5min":3.8479999999999999,"15min":3.8479999999999999},"last":0.371},{"interface":"front","average":{"1min":0.83499999999999996,"5min":0.67500000000000004,"15min":0.57499999999999996},"min":{"1min":0.27400000000000002,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":3.3530000000000002,"5min":3.3530000000000002,"15min":3.3530000000000002},"last":0.58499999999999996}]},{"osd":6,"last update":"Sat Sep 23 20:59:38 2023","interfaces":[{"interface":"back","average":{"1min":0.86099999999999999,"5min":0.65100000000000002,"15min":0.59099999999999997},"min":{"1min":0.29199999999999998,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":4.1120000000000001,"5min":4.1120000000000001,"15min":4.1120000000000001},"last":0.33800000000000002},{"interface":"front","average":{"1min":1.0269999999999999,"5min":0.755,"15min":0.747},"min":{"1min":0.36899999999999999,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":2.9620000000000002,"5min":3.6560000000000001,"15min":3.6560000000000001},"last":0.443}]},{"osd":7,"last update":"Sat Sep 23 21:00:18 2023","interfaces":[{"interface":"back","average":{"1min":1.0189999999999999,"5min":0.81000000000000005,"15min":0.67700000000000005},"min":{"1min":0.35499999999999998,"5min":0.307,"15min":0.30199999999999999},"max":{"1min":4.4240000000000004,"5min":4.4240000000000004,"15min":8.5649999999999995},"last":0.47699999999999998},{"interface":"front","average":{"1min":0.752,"5min":0.71899999999999997,"15min":0.67400000000000004},"min":{"1min":0.373,"5min":0.29799999999999999,"15min":0.27700000000000002},"max":{"1min":2.1789999999999998,"5min":3.8149999999999999,"15min":4.5430000000000001},"last":0.52400000000000002}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":752947200,"data_stored":752943128,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":881594368,"data_stored":881590296,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":925302784,"data_stored":925302784,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":692649984,"data_stored":692649984,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":467927040,"data_stored":467927040,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":733282304,"data_stored":733282304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":447025152,"data_stored":447021080,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":468451328,"data_stored":468451328,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T21:00:28.970 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph status --format=json 2023-09-23T21:00:29.061 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:00:29.262 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T21:00:29.280 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:00:29.290 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:00:29.291 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:00:29.392 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:00:29.436 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:00:29.531 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:00:29.534 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:00:29.534 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:00:29.635 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:00:29.666 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:00:29.708 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:00:29.710 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:00:29 smithi070 ceph-mon[168282]: pgmap v626: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 1.2 GiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 634 op/s; 11892/61317 objects degraded (19.394%); 302/61317 objects misplaced (0.493%); 2.5 MiB/s, 40 objects/s recovering 2023-09-23T21:00:29.711 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:00:29.711 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:00:29.772 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0: {} 2023-09-23T21:00:29.804 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1: {} 2023-09-23T21:00:29.812 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:00:29.876 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2: {} 2023-09-23T21:00:29.895 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:00:29 smithi026 ceph-mon[175151]: pgmap v626: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 1.2 GiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 634 op/s; 11892/61317 objects degraded (19.394%); 302/61317 objects misplaced (0.493%); 2.5 MiB/s, 40 objects/s recovering 2023-09-23T21:00:29.896 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 21:00:29 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[164029]: ::ffff:172.21.15.70 - - [23/Sep/2023:21:00:29] "GET /metrics HTTP/1.1" 200 36315 "" "Prometheus/2.43.0" 2023-09-23T21:00:29.896 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:00:29 smithi026 ceph-mon[179688]: pgmap v626: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 1.2 GiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 634 op/s; 11892/61317 objects degraded (19.394%); 302/61317 objects misplaced (0.493%); 2.5 MiB/s, 40 objects/s recovering 2023-09-23T21:00:29.912 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3: {} 2023-09-23T21:00:29.926 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:00:29.952 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4: {} 2023-09-23T21:00:29.980 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:00:29.988 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T21:00:30.009 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:00:30.013 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:00:30.013 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:00:30.027 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6: {} 2023-09-23T21:00:30.066 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T21:00:30.071 INFO:teuthology.orchestra.run.smithi026.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T21:00:30.071 INFO:teuthology.orchestra.run.smithi026.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T21:00:30.071 INFO:teuthology.orchestra.run.smithi026.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T21:00:30.072 INFO:teuthology.orchestra.run.smithi026.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T21:00:30.072 INFO:teuthology.orchestra.run.smithi026.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T21:00:30.072 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T21:00:30.072 INFO:teuthology.orchestra.run.smithi026.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T21:00:30.072 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T21:00:30.114 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:00:30.149 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:00:30.196 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:00:30.201 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:00:30.201 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:00:30.302 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:00:30.396 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:00:30.545 INFO:teuthology.orchestra.run.smithi070.stdout:ERROR: (22) Invalid argument 2023-09-23T21:00:30.545 INFO:teuthology.orchestra.run.smithi070.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T21:00:30.578 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T21:00:30.587 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:00:30.587 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:00:30.688 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:00:30.695 INFO:teuthology.orchestra.run.smithi026.stderr:noscrub is unset 2023-09-23T21:00:30.803 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:00:30.863 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:00:30 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/742535280' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:00:30.863 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:00:30 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/3325848760' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T21:00:30.863 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:00:30 smithi026 ceph-mon[175151]: Health check update: Degraded data redundancy: 11892/61317 objects degraded (19.394%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T21:00:30.865 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:00:30 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/742535280' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:00:30.865 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:00:30 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/3325848760' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T21:00:30.865 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:00:30 smithi026 ceph-mon[179688]: Health check update: Degraded data redundancy: 11892/61317 objects degraded (19.394%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T21:00:30.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:00:30 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/742535280' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:00:30.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:00:30 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/3325848760' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T21:00:30.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:00:30 smithi070 ceph-mon[168282]: Health check update: Degraded data redundancy: 11892/61317 objects degraded (19.394%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T21:00:30.943 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_blocked_ops 2023-09-23T21:00:31.143 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:00:31.153 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:00:31.153 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:00:31.254 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:00:31.365 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:00:31.608 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:00:31.619 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T21:00:31.622 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:00:31.624 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:00:31.725 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:00:31.759 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:00:31.855 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:00:31.859 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:00:31.860 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:00:31.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:00:31 smithi026 ceph-mon[179688]: pgmap v627: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 1.2 GiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 632 op/s; 11892/57189 objects degraded (20.794%); 302/57189 objects misplaced (0.528%); 2.5 MiB/s, 40 objects/s recovering 2023-09-23T21:00:31.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:00:31 smithi026 ceph-mon[179688]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:00:31.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:00:31 smithi026 ceph-mon[179688]: failed to encode map e247 with expected crc 2023-09-23T21:00:31.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:00:31 smithi026 ceph-mon[179688]: failed to encode map e247 with expected crc 2023-09-23T21:00:31.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:00:31 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/3325848760' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T21:00:31.883 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:00:31 smithi026 ceph-mon[179688]: osdmap e247: 8 total, 8 up, 8 in 2023-09-23T21:00:31.883 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:00:31 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:00:31.883 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:00:31 smithi026 ceph-mon[175151]: pgmap v627: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 1.2 GiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 632 op/s; 11892/57189 objects degraded (20.794%); 302/57189 objects misplaced (0.528%); 2.5 MiB/s, 40 objects/s recovering 2023-09-23T21:00:31.883 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:00:31 smithi026 ceph-mon[175151]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:00:31.883 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:00:31 smithi026 ceph-mon[175151]: failed to encode map e247 with expected crc 2023-09-23T21:00:31.883 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:00:31 smithi026 ceph-mon[175151]: failed to encode map e247 with expected crc 2023-09-23T21:00:31.883 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:00:31 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/3325848760' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T21:00:31.883 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:00:31 smithi026 ceph-mon[175151]: osdmap e247: 8 total, 8 up, 8 in 2023-09-23T21:00:31.883 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:00:31 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:00:31.892 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:00:31 smithi070 ceph-mon[168282]: pgmap v627: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 1.2 GiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 632 op/s; 11892/57189 objects degraded (20.794%); 302/57189 objects misplaced (0.528%); 2.5 MiB/s, 40 objects/s recovering 2023-09-23T21:00:31.892 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:00:31 smithi070 ceph-mon[168282]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:00:31.892 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:00:31 smithi070 ceph-mon[168282]: failed to encode map e247 with expected crc 2023-09-23T21:00:31.892 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:00:31 smithi070 ceph-mon[168282]: failed to encode map e247 with expected crc 2023-09-23T21:00:31.892 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:00:31 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/3325848760' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T21:00:31.892 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:00:31 smithi070 ceph-mon[168282]: osdmap e247: 8 total, 8 up, 8 in 2023-09-23T21:00:31.892 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:00:31 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:00:31.961 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:00:32.040 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:00:32.206 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:00:32.211 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:00:32.211 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:00:32.312 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:00:32.351 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:00:32.400 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:00:32.403 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:00:32.404 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:00:32.505 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:00:32.610 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:00:32.695 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:00:32.696 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T21:00:32.704 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:00:32.705 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:00:32.806 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:00:32.850 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:00:32.895 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:00:32.899 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:00:32.900 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:00:32.913 INFO:teuthology.orchestra.run.smithi070.stdout:ERROR: (22) Invalid argument 2023-09-23T21:00:32.913 INFO:teuthology.orchestra.run.smithi070.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T21:00:32.935 INFO:teuthology.orchestra.run.smithi026.stdout: 2023-09-23T21:00:32.935 INFO:teuthology.orchestra.run.smithi026.stdout:{"fsid":"b3f8e4ec-5a50-11ee-8db4-212e2dc638e7","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"nodeep-scrub flag(s) set","count":12},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 11892/51978 objects degraded (22.879%), 3 pgs degraded, 3 pgs undersized","count":6},"muted":false}},"mutes":[]},"election_epoch":30,"quorum":[0,1,2],"quorum_names":["a","b","c"],"quorum_age":751,"monmap":{"epoch":4,"min_mon_release_name":"reef","num_mons":3},"osdmap":{"epoch":247,"num_osds":8,"num_up_osds":8,"osd_up_since":1695502534,"num_in_osds":8,"osd_in_since":1695502470,"num_remapped_pgs":3},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":106},{"state_name":"active+recovery_wait+undersized+degraded+remapped","count":2},{"state_name":"active+recovering+undersized+degraded+remapped","count":1}],"num_pgs":109,"num_pools":5,"num_objects":17326,"data_bytes":1134434232,"bytes_used":6244290560,"bytes_avail":761715994624,"bytes_total":767960285184,"degraded_objects":11892,"degraded_total":51978,"degraded_ratio":0.22878910308207318,"misplaced_objects":90,"misplaced_total":51978,"misplaced_ratio":0.0017315017892185155,"recovering_objects_per_sec":43,"recovering_bytes_per_sec":2830093,"recovering_keys_per_sec":0,"num_objects_recovered":432,"num_bytes_recovered":28311552,"num_keys_recovered":0,"write_bytes_sec":0,"read_op_per_sec":0,"write_op_per_sec":642},"fsmap":{"epoch":8,"id":1,"up":1,"in":1,"max":1,"by_rank":[{"filesystem_id":1,"rank":0,"name":"foo.smithi026.znfpnl","status":"up:active","gid":14658}],"up:standby":1},"mgrmap":{"available":true,"num_standbys":1,"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"services":{"dashboard":"https://172.21.15.26:8443/","prometheus":"http://172.21.15.26:9283/"}},"servicemap":{"epoch":21,"modified":"2023-09-23T20:56:34.314324+0000","services":{}},"progress_events":{"8b64cbfe-d367-468b-a773-0e71ee4ff645":{"message":"Global Recovery Event (0s)\n [............................] ","progress":0,"add_to_ceph_s":true},"fc746e9d-3f76-434d-b6d2-8fc9e8a72a5e":{"message":"Upgrade to 18.2.0-460-gbb0daeff (4m)\n [============................] (remaining: 5m)","progress":0.44999998807907104,"add_to_ceph_s":true}}} 2023-09-23T21:00:33.001 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:00:33.054 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:00:33.240 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T21:00:33.265 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:00:33.268 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:00:33.268 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:00:33.321 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_historic_ops 2023-09-23T21:00:33.366 INFO:tasks.cephadm.ceph_manager.ceph:making progress, resetting timeout 2023-09-23T21:00:33.366 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T21:00:33.369 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:00:33.473 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:00:33.600 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:00:33.608 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:00:33.608 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:00:33.710 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:00:33.858 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:00:33.864 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:00:33.886 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:00:33 smithi026 ceph-mon[179688]: pgmap v629: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 1.1 GiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 642 op/s; 11892/51978 objects degraded (22.879%); 90/51978 objects misplaced (0.173%); 2.7 MiB/s, 43 objects/s recovering 2023-09-23T21:00:33.886 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:00:33 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/3537347377' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-23T21:00:33.886 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:00:33 smithi026 ceph-mon[175151]: pgmap v629: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 1.1 GiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 642 op/s; 11892/51978 objects degraded (22.879%); 90/51978 objects misplaced (0.173%); 2.7 MiB/s, 43 objects/s recovering 2023-09-23T21:00:33.886 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:00:33 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/3537347377' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-23T21:00:33.964 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:00:33.974 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T21:00:33.974 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:00:33.975 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:00:34.076 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:00:34.117 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:00:33 smithi070 ceph-mon[168282]: pgmap v629: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 1.1 GiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 642 op/s; 11892/51978 objects degraded (22.879%); 90/51978 objects misplaced (0.173%); 2.7 MiB/s, 43 objects/s recovering 2023-09-23T21:00:34.117 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:00:33 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/3537347377' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-23T21:00:34.120 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:00:34.163 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:00:34.169 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:00:34.169 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:00:34.270 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:00:34.373 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:00:34.508 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:00:34.517 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:00:34.517 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:00:34.580 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:00:34.618 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:00:34.736 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:00:34.915 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:00:34.922 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:00:34.922 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:00:34.971 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:00:35.023 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:00:35.131 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:00:35.271 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:00:35.272 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:00:35.272 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:00:35.352 INFO:teuthology.orchestra.run.smithi070.stdout:ERROR: (22) Invalid argument 2023-09-23T21:00:35.352 INFO:teuthology.orchestra.run.smithi070.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T21:00:35.374 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:00:35.418 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:00:35.699 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:00:35.703 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:00:35.703 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:00:35.738 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:00:35 smithi026 ceph-mon[175151]: pgmap v630: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 1.0 GiB data, 5.7 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 636 op/s; 11892/49734 objects degraded (23.911%); 90/49734 objects misplaced (0.181%); 2.7 MiB/s, 43 objects/s recovering 2023-09-23T21:00:35.738 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:00:35 smithi026 ceph-mon[175151]: Health check update: Degraded data redundancy: 11892/49734 objects degraded (23.911%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T21:00:35.738 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:00:35 smithi026 ceph-mon[179688]: pgmap v630: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 1.0 GiB data, 5.7 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 636 op/s; 11892/49734 objects degraded (23.911%); 90/49734 objects misplaced (0.181%); 2.7 MiB/s, 43 objects/s recovering 2023-09-23T21:00:35.739 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:00:35 smithi026 ceph-mon[179688]: Health check update: Degraded data redundancy: 11892/49734 objects degraded (23.911%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T21:00:35.771 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_ops_in_flight 2023-09-23T21:00:35.804 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:00:35.883 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:00:36.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:00:35 smithi070 ceph-mon[168282]: pgmap v630: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 1.0 GiB data, 5.7 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 636 op/s; 11892/49734 objects degraded (23.911%); 90/49734 objects misplaced (0.181%); 2.7 MiB/s, 43 objects/s recovering 2023-09-23T21:00:36.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:00:35 smithi070 ceph-mon[168282]: Health check update: Degraded data redundancy: 11892/49734 objects degraded (23.911%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T21:00:36.232 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:00:36.238 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:00:36.238 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:00:36.339 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:00:36.465 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T21:00:36.471 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:00:36.539 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:00:36.543 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:00:36.544 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:00:36.631 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0: {} 2023-09-23T21:00:36.645 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:00:36.675 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:00:36.698 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1: {} 2023-09-23T21:00:36.715 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:00:36 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/1695143500' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:00:36.725 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:00:36.730 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:00:36.730 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:00:36.762 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2: {} 2023-09-23T21:00:36.810 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3: {} 2023-09-23T21:00:36.831 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:00:36.854 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4: {} 2023-09-23T21:00:36.891 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T21:00:36.926 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:00:36.959 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6: {} 2023-09-23T21:00:36.993 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:00:37.007 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:00:37.007 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:00:37.009 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:00:36 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/1695143500' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:00:37.021 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T21:00:37.046 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:00:36 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/1695143500' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:00:37.056 INFO:teuthology.orchestra.run.smithi026.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T21:00:37.056 INFO:teuthology.orchestra.run.smithi026.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T21:00:37.056 INFO:teuthology.orchestra.run.smithi026.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T21:00:37.056 INFO:teuthology.orchestra.run.smithi026.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T21:00:37.056 INFO:teuthology.orchestra.run.smithi026.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T21:00:37.056 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T21:00:37.056 INFO:teuthology.orchestra.run.smithi026.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T21:00:37.056 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T21:00:37.108 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:00:37.235 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:00:37.337 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T21:00:37.351 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:00:37.351 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:00:37.452 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:00:37.570 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:00:37.723 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:00:37.758 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:00:37.758 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:00:37.830 INFO:teuthology.orchestra.run.smithi070.stdout:{ 2023-09-23T21:00:37.831 INFO:teuthology.orchestra.run.smithi070.stdout: "ops": [ 2023-09-23T21:00:37.831 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:37.831 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.35100.0:190391 7.d 7:b7b05034:::benchmark_data_smithi026_260489_object88629:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e247)", 2023-09-23T21:00:37.831 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:00:37.828440+0000", 2023-09-23T21:00:37.831 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 0.001494521, 2023-09-23T21:00:37.831 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0015168099999999999, 2023-09-23T21:00:37.831 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:00:37.831 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "waiting for sub ops", 2023-09-23T21:00:37.831 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:00:37.831 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.35100", 2023-09-23T21:00:37.831 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/4108972961", 2023-09-23T21:00:37.832 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 190391 2023-09-23T21:00:37.832 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:37.832 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:00:37.832 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:37.832 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:00:37.832 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:00:37.828440+0000", 2023-09-23T21:00:37.832 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:00:37.832 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:37.832 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:37.832 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:00:37.832 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:00:37.828440+0000", 2023-09-23T21:00:37.832 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:00:37.833 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:37.833 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:37.833 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:00:37.833 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:00:37.828439+0000", 2023-09-23T21:00:37.833 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T21:00:37.833 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:37.833 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:37.833 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:00:37.833 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:00:37.828447+0000", 2023-09-23T21:00:37.833 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.8679999999999999e-06 2023-09-23T21:00:37.833 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:37.833 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:37.833 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:00:37.834 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:00:37.828464+0000", 2023-09-23T21:00:37.834 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.7212999999999999e-05 2023-09-23T21:00:37.834 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:37.834 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:37.834 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:00:37.834 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:00:37.828470+0000", 2023-09-23T21:00:37.834 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.3790000000000003e-06 2023-09-23T21:00:37.834 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:37.834 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:37.834 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:00:37.834 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:00:37.828521+0000", 2023-09-23T21:00:37.835 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.0788999999999998e-05 2023-09-23T21:00:37.835 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:37.835 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:37.835 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:00:37.835 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:00:37.828669+0000", 2023-09-23T21:00:37.835 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00014776500000000001 2023-09-23T21:00:37.835 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:37.835 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:37.835 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 0,1", 2023-09-23T21:00:37.835 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:00:37.828821+0000", 2023-09-23T21:00:37.835 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00015171299999999999 2023-09-23T21:00:37.835 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:00:37.836 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:00:37.836 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:00:37.836 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:00:37.836 INFO:teuthology.orchestra.run.smithi070.stdout: ], 2023-09-23T21:00:37.836 INFO:teuthology.orchestra.run.smithi070.stdout: "num_ops": 1 2023-09-23T21:00:37.836 INFO:teuthology.orchestra.run.smithi070.stdout:} 2023-09-23T21:00:37.853 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:00:37 smithi070 ceph-mon[168282]: pgmap v631: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 1000 MiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 629 op/s; 11892/48060 objects degraded (24.744%); 90/48060 objects misplaced (0.187%); 2.7 MiB/s, 43 objects/s recovering 2023-09-23T21:00:37.860 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:00:37.885 INFO:teuthology.orchestra.run.smithi026.stdout: 2023-09-23T21:00:37.887 INFO:teuthology.orchestra.run.smithi026.stderr:dumped all 2023-09-23T21:00:37.942 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:00:37.982 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:00:37.984 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:00:37 smithi026 ceph-mon[179688]: pgmap v631: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 1000 MiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 629 op/s; 11892/48060 objects degraded (24.744%); 90/48060 objects misplaced (0.187%); 2.7 MiB/s, 43 objects/s recovering 2023-09-23T21:00:37.984 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:00:37 smithi026 ceph-mon[175151]: pgmap v631: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 1000 MiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 629 op/s; 11892/48060 objects degraded (24.744%); 90/48060 objects misplaced (0.187%); 2.7 MiB/s, 43 objects/s recovering 2023-09-23T21:00:37.985 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:00:37.985 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:00:38.086 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:00:38.122 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:00:38.232 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:00:38.234 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:00:38.234 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:00:38.335 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_blocked_ops 2023-09-23T21:00:38.336 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:00:38.381 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:00:38.432 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:00:38.439 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:00:38.439 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:00:38.540 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:00:38.624 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:00:38.740 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:00:38 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/2914602822' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:00:38.740 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:00:38 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:00:38.740 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:00:38 smithi070 ceph-mon[168282]: from='client.55258 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T21:00:38.769 INFO:teuthology.orchestra.run.smithi026.stderr:nodeep-scrub is unset 2023-09-23T21:00:38.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:00:38 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/2914602822' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:00:38.883 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:00:38 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:00:38.883 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:00:38 smithi026 ceph-mon[179688]: from='client.55258 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T21:00:38.883 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:00:38 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/2914602822' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:00:38.883 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:00:38 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:00:38.883 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:00:38 smithi026 ceph-mon[175151]: from='client.55258 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T21:00:39.038 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T21:00:39.078 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:00:39.082 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:00:39.082 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:00:39.183 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:00:39.217 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:00:39.265 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:00:39.268 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:00:39.268 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:00:39.284 INFO:teuthology.orchestra.run.smithi026.stdout:{"pg_ready":true,"pg_map":{"version":631,"stamp":"2023-09-23T21:00:36.490184+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":1048844216,"num_objects":16020,"num_object_clones":0,"num_object_copies":48060,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":11892,"num_objects_misplaced":90,"num_objects_unfound":0,"num_objects_dirty":16020,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":187585,"num_write_kb":6512606,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4382,"num_bytes_recovered":285742950,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":122560,"ondisk_log_size":122560,"up":327,"acting":324,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":331,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":5911736,"kb_used_data":4179512,"kb_used_omap":136,"kb_used_meta":1731703,"kb_avail":744049480,"statfs":{"total":767960285184,"available":761906667520,"internally_reserved":0,"allocated":4279820288,"data_stored":4271576937,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":139741,"internal_metadata":1773264419},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":11,"apply_latency_ms":11,"commit_latency_ns":11000000,"apply_latency_ns":11000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-412811264,"num_objects":-6299,"num_object_clones":0,"num_object_copies":-18897,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":-432,"num_objects_unfound":0,"num_objects_dirty":-6299,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6299,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":432,"num_bytes_recovered":28311552,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.003726"},"pg_stats":[{"pgid":"3.1a","version":"68'1","reported_seq":124,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-09-23T21:00:20.336666+0000","last_change":"2023-09-23T20:54:13.946695+0000","last_active":"2023-09-23T21:00:20.336666+0000","last_peered":"2023-09-23T21:00:20.336666+0000","last_clean":"2023-09-23T21:00:20.336666+0000","last_became_active":"2023-09-23T20:54:13.945399+0000","last_became_peered":"2023-09-23T20:54:13.945399+0000","last_unstale":"2023-09-23T21:00:20.336666+0000","last_undegraded":"2023-09-23T21:00:20.336666+0000","last_fullsized":"2023-09-23T21:00:20.336666+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:32:52.365464+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":239,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-09-23T21:00:20.342954+0000","last_change":"2023-09-23T20:54:13.942019+0000","last_active":"2023-09-23T21:00:20.342954+0000","last_peered":"2023-09-23T21:00:20.342954+0000","last_clean":"2023-09-23T21:00:20.342954+0000","last_became_active":"2023-09-23T20:54:13.941647+0000","last_became_peered":"2023-09-23T20:54:13.941647+0000","last_unstale":"2023-09-23T21:00:20.342954+0000","last_undegraded":"2023-09-23T21:00:20.342954+0000","last_fullsized":"2023-09-23T21:00:20.342954+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:30:54.400944+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":217,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-09-23T21:00:20.340319+0000","last_change":"2023-09-23T20:54:13.941394+0000","last_active":"2023-09-23T21:00:20.340319+0000","last_peered":"2023-09-23T21:00:20.340319+0000","last_clean":"2023-09-23T21:00:20.340319+0000","last_became_active":"2023-09-23T20:54:13.941042+0000","last_became_peered":"2023-09-23T20:54:13.941042+0000","last_unstale":"2023-09-23T21:00:20.340319+0000","last_undegraded":"2023-09-23T21:00:20.340319+0000","last_fullsized":"2023-09-23T21:00:20.340319+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:19:41.401184+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":232,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-09-23T21:00:20.342667+0000","last_change":"2023-09-23T20:57:44.979811+0000","last_active":"2023-09-23T21:00:20.342667+0000","last_peered":"2023-09-23T21:00:20.342667+0000","last_clean":"2023-09-23T21:00:20.342667+0000","last_became_active":"2023-09-23T20:57:44.979515+0000","last_became_peered":"2023-09-23T20:57:44.979515+0000","last_unstale":"2023-09-23T21:00:20.342667+0000","last_undegraded":"2023-09-23T21:00:20.342667+0000","last_fullsized":"2023-09-23T21:00:20.342667+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:38:22.162140+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":230,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-09-23T21:00:20.342738+0000","last_change":"2023-09-23T20:51:10.833613+0000","last_active":"2023-09-23T21:00:20.342738+0000","last_peered":"2023-09-23T21:00:20.342738+0000","last_clean":"2023-09-23T21:00:20.342738+0000","last_became_active":"2023-09-23T20:51:10.833208+0000","last_became_peered":"2023-09-23T20:51:10.833208+0000","last_unstale":"2023-09-23T21:00:20.342738+0000","last_undegraded":"2023-09-23T21:00:20.342738+0000","last_fullsized":"2023-09-23T21:00:20.342738+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:51:28.246014+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":212,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-09-23T21:00:20.324668+0000","last_change":"2023-09-23T20:55:35.958384+0000","last_active":"2023-09-23T21:00:20.324668+0000","last_peered":"2023-09-23T21:00:20.324668+0000","last_clean":"2023-09-23T21:00:20.324668+0000","last_became_active":"2023-09-23T20:55:35.954563+0000","last_became_peered":"2023-09-23T20:55:35.954563+0000","last_unstale":"2023-09-23T21:00:20.324668+0000","last_undegraded":"2023-09-23T21:00:20.324668+0000","last_fullsized":"2023-09-23T21:00:20.324668+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:37:48.332259+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":222,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-09-23T21:00:20.333707+0000","last_change":"2023-09-23T20:54:31.885514+0000","last_active":"2023-09-23T21:00:20.333707+0000","last_peered":"2023-09-23T21:00:20.333707+0000","last_clean":"2023-09-23T21:00:20.333707+0000","last_became_active":"2023-09-23T20:54:31.884986+0000","last_became_peered":"2023-09-23T20:54:31.884986+0000","last_unstale":"2023-09-23T21:00:20.333707+0000","last_undegraded":"2023-09-23T21:00:20.333707+0000","last_fullsized":"2023-09-23T21:00:20.333707+0000","mapping_epoch":136,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:26:08.100220+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":228,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-09-23T21:00:29.762066+0000","last_change":"2023-09-23T20:55:35.951889+0000","last_active":"2023-09-23T21:00:29.762066+0000","last_peered":"2023-09-23T21:00:29.762066+0000","last_clean":"2023-09-23T21:00:29.762066+0000","last_became_active":"2023-09-23T20:55:35.951563+0000","last_became_peered":"2023-09-23T20:55:35.951563+0000","last_unstale":"2023-09-23T21:00:29.762066+0000","last_undegraded":"2023-09-23T21:00:29.762066+0000","last_fullsized":"2023-09-23T21:00:29.762066+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:41:53.098781+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"79'161","reported_seq":534,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-09-23T21:00:20.347313+0000","last_change":"2023-09-23T20:57:25.096559+0000","last_active":"2023-09-23T21:00:20.347313+0000","last_peered":"2023-09-23T21:00:20.347313+0000","last_clean":"2023-09-23T21:00:20.347313+0000","last_became_active":"2023-09-23T20:57:15.975674+0000","last_became_peered":"2023-09-23T20:57:15.975674+0000","last_unstale":"2023-09-23T21:00:20.347313+0000","last_undegraded":"2023-09-23T21:00:20.347313+0000","last_fullsized":"2023-09-23T21:00:20.347313+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:25.096401+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:25.096401+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:05:05.448404+0000","scrub_duration":0.002859393,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":120,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-09-23T21:00:20.348839+0000","last_change":"2023-09-23T20:54:31.890678+0000","last_active":"2023-09-23T21:00:20.348839+0000","last_peered":"2023-09-23T21:00:20.348839+0000","last_clean":"2023-09-23T21:00:20.348839+0000","last_became_active":"2023-09-23T20:54:31.890469+0000","last_became_peered":"2023-09-23T20:54:31.890469+0000","last_unstale":"2023-09-23T21:00:20.348839+0000","last_undegraded":"2023-09-23T21:00:20.348839+0000","last_fullsized":"2023-09-23T21:00:20.348839+0000","mapping_epoch":136,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:55:08.790146+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":189,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-09-23T21:00:29.762101+0000","last_change":"2023-09-23T20:55:35.952661+0000","last_active":"2023-09-23T21:00:29.762101+0000","last_peered":"2023-09-23T21:00:29.762101+0000","last_clean":"2023-09-23T21:00:29.762101+0000","last_became_active":"2023-09-23T20:55:35.952349+0000","last_became_peered":"2023-09-23T20:55:35.952349+0000","last_unstale":"2023-09-23T21:00:29.762101+0000","last_undegraded":"2023-09-23T21:00:29.762101+0000","last_fullsized":"2023-09-23T21:00:29.762101+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:00:03.110126+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":65,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-09-23T21:00:20.343082+0000","last_change":"2023-09-23T20:56:45.988607+0000","last_active":"2023-09-23T21:00:20.343082+0000","last_peered":"2023-09-23T21:00:20.343082+0000","last_clean":"2023-09-23T21:00:20.343082+0000","last_became_active":"2023-09-23T20:56:45.988206+0000","last_became_peered":"2023-09-23T20:56:45.988206+0000","last_unstale":"2023-09-23T21:00:20.343082+0000","last_undegraded":"2023-09-23T21:00:20.343082+0000","last_fullsized":"2023-09-23T21:00:20.343082+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":205,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:01:19.783688+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,4],"acting":[3,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":64,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-09-23T21:00:20.347871+0000","last_change":"2023-09-23T20:56:18.139142+0000","last_active":"2023-09-23T21:00:20.347871+0000","last_peered":"2023-09-23T21:00:20.347871+0000","last_clean":"2023-09-23T21:00:20.347871+0000","last_became_active":"2023-09-23T20:56:18.138844+0000","last_became_peered":"2023-09-23T20:56:18.138844+0000","last_unstale":"2023-09-23T21:00:20.347871+0000","last_undegraded":"2023-09-23T21:00:20.347871+0000","last_fullsized":"2023-09-23T21:00:20.347871+0000","mapping_epoch":192,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":193,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:09:02.217683+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,2],"acting":[5,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.14","version":"79'22","reported_seq":151,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-09-23T21:00:20.336736+0000","last_change":"2023-09-23T20:55:35.949418+0000","last_active":"2023-09-23T21:00:20.336736+0000","last_peered":"2023-09-23T21:00:20.336736+0000","last_clean":"2023-09-23T21:00:20.336736+0000","last_became_active":"2023-09-23T20:55:35.948566+0000","last_became_peered":"2023-09-23T20:55:35.948566+0000","last_unstale":"2023-09-23T21:00:20.336736+0000","last_undegraded":"2023-09-23T21:00:20.336736+0000","last_fullsized":"2023-09-23T21:00:20.336736+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:59:37.433128+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4734,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":141,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-09-23T21:00:20.336962+0000","last_change":"2023-09-23T20:55:35.949178+0000","last_active":"2023-09-23T21:00:20.336962+0000","last_peered":"2023-09-23T21:00:20.336962+0000","last_clean":"2023-09-23T21:00:20.336962+0000","last_became_active":"2023-09-23T20:55:35.948795+0000","last_became_peered":"2023-09-23T20:55:35.948795+0000","last_unstale":"2023-09-23T21:00:20.336962+0000","last_undegraded":"2023-09-23T21:00:20.336962+0000","last_fullsized":"2023-09-23T21:00:20.336962+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:13:08.996932+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.11","version":"79'161","reported_seq":476,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-09-23T21:00:20.334029+0000","last_change":"2023-09-23T20:57:28.756796+0000","last_active":"2023-09-23T21:00:20.334029+0000","last_peered":"2023-09-23T21:00:20.334029+0000","last_clean":"2023-09-23T21:00:20.334029+0000","last_became_active":"2023-09-23T20:57:15.980606+0000","last_became_peered":"2023-09-23T20:57:15.980606+0000","last_unstale":"2023-09-23T21:00:20.334029+0000","last_undegraded":"2023-09-23T21:00:20.334029+0000","last_fullsized":"2023-09-23T21:00:20.334029+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:28.756634+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:28.756634+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:09:19.150791+0000","scrub_duration":0.0064925119999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":167,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-09-23T21:00:20.336819+0000","last_change":"2023-09-23T20:56:45.992577+0000","last_active":"2023-09-23T21:00:20.336819+0000","last_peered":"2023-09-23T21:00:20.336819+0000","last_clean":"2023-09-23T21:00:20.336819+0000","last_became_active":"2023-09-23T20:56:45.992297+0000","last_became_peered":"2023-09-23T20:56:45.992297+0000","last_unstale":"2023-09-23T21:00:20.336819+0000","last_undegraded":"2023-09-23T21:00:20.336819+0000","last_fullsized":"2023-09-23T21:00:20.336819+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":205,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:00:45.547202+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,2],"acting":[4,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":213,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-09-23T21:00:29.762178+0000","last_change":"2023-09-23T20:51:10.831262+0000","last_active":"2023-09-23T21:00:29.762178+0000","last_peered":"2023-09-23T21:00:29.762178+0000","last_clean":"2023-09-23T21:00:29.762178+0000","last_became_active":"2023-09-23T20:51:10.830931+0000","last_became_peered":"2023-09-23T20:51:10.830931+0000","last_unstale":"2023-09-23T21:00:29.762178+0000","last_undegraded":"2023-09-23T21:00:29.762178+0000","last_fullsized":"2023-09-23T21:00:29.762178+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:51:54.982505+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.10","version":"79'161","reported_seq":484,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-09-23T21:00:20.347059+0000","last_change":"2023-09-23T20:57:20.708003+0000","last_active":"2023-09-23T21:00:20.347059+0000","last_peered":"2023-09-23T21:00:20.347059+0000","last_clean":"2023-09-23T21:00:20.347059+0000","last_became_active":"2023-09-23T20:57:15.975302+0000","last_became_peered":"2023-09-23T20:57:15.975302+0000","last_unstale":"2023-09-23T21:00:20.347059+0000","last_undegraded":"2023-09-23T21:00:20.347059+0000","last_fullsized":"2023-09-23T21:00:20.347059+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:20.707939+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:20.707939+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:16:51.721994+0000","scrub_duration":0.0017733409999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":222,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-09-23T21:00:20.341722+0000","last_change":"2023-09-23T20:57:44.979736+0000","last_active":"2023-09-23T21:00:20.341722+0000","last_peered":"2023-09-23T21:00:20.341722+0000","last_clean":"2023-09-23T21:00:20.341722+0000","last_became_active":"2023-09-23T20:57:44.979356+0000","last_became_peered":"2023-09-23T20:57:44.979356+0000","last_unstale":"2023-09-23T21:00:20.341722+0000","last_undegraded":"2023-09-23T21:00:20.341722+0000","last_fullsized":"2023-09-23T21:00:20.341722+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:14:43.848832+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":120,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-09-23T21:00:20.347611+0000","last_change":"2023-09-23T20:54:31.887814+0000","last_active":"2023-09-23T21:00:20.347611+0000","last_peered":"2023-09-23T21:00:20.347611+0000","last_clean":"2023-09-23T21:00:20.347611+0000","last_became_active":"2023-09-23T20:54:31.887195+0000","last_became_peered":"2023-09-23T20:54:31.887195+0000","last_unstale":"2023-09-23T21:00:20.347611+0000","last_undegraded":"2023-09-23T21:00:20.347611+0000","last_fullsized":"2023-09-23T21:00:20.347611+0000","mapping_epoch":136,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:10:52.089553+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.13","version":"79'161","reported_seq":547,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-09-23T21:00:20.346468+0000","last_change":"2023-09-23T20:57:48.262970+0000","last_active":"2023-09-23T21:00:20.346468+0000","last_peered":"2023-09-23T21:00:20.346468+0000","last_clean":"2023-09-23T21:00:20.346468+0000","last_became_active":"2023-09-23T20:57:48.262663+0000","last_became_peered":"2023-09-23T20:57:48.262663+0000","last_unstale":"2023-09-23T21:00:20.346468+0000","last_undegraded":"2023-09-23T21:00:20.346468+0000","last_fullsized":"2023-09-23T21:00:20.346468+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:36.420362+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:36.420362+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:28:28.613492+0000","scrub_duration":0.028838444000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":191,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-09-23T21:00:20.338557+0000","last_change":"2023-09-23T20:56:45.991822+0000","last_active":"2023-09-23T21:00:20.338557+0000","last_peered":"2023-09-23T21:00:20.338557+0000","last_clean":"2023-09-23T21:00:20.338557+0000","last_became_active":"2023-09-23T20:56:45.991460+0000","last_became_peered":"2023-09-23T20:56:45.991460+0000","last_unstale":"2023-09-23T21:00:20.338557+0000","last_undegraded":"2023-09-23T21:00:20.338557+0000","last_fullsized":"2023-09-23T21:00:20.338557+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":205,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:20:37.503317+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":204,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-09-23T21:00:20.341874+0000","last_change":"2023-09-23T20:51:10.840849+0000","last_active":"2023-09-23T21:00:20.341874+0000","last_peered":"2023-09-23T21:00:20.341874+0000","last_clean":"2023-09-23T21:00:20.341874+0000","last_became_active":"2023-09-23T20:51:10.840593+0000","last_became_peered":"2023-09-23T20:51:10.840593+0000","last_unstale":"2023-09-23T21:00:20.341874+0000","last_undegraded":"2023-09-23T21:00:20.341874+0000","last_fullsized":"2023-09-23T21:00:20.341874+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:30:52.297625+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.12","version":"79'161","reported_seq":484,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-09-23T21:00:20.347596+0000","last_change":"2023-09-23T20:57:21.750765+0000","last_active":"2023-09-23T21:00:20.347596+0000","last_peered":"2023-09-23T21:00:20.347596+0000","last_clean":"2023-09-23T21:00:20.347596+0000","last_became_active":"2023-09-23T20:57:15.979195+0000","last_became_peered":"2023-09-23T20:57:15.979195+0000","last_unstale":"2023-09-23T21:00:20.347596+0000","last_undegraded":"2023-09-23T21:00:20.347596+0000","last_fullsized":"2023-09-23T21:00:20.347596+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:21.750680+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:21.750680+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:03:41.915962+0000","scrub_duration":0.0092391680000000007,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":239,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-09-23T21:00:20.323835+0000","last_change":"2023-09-23T20:55:36.943969+0000","last_active":"2023-09-23T21:00:20.323835+0000","last_peered":"2023-09-23T21:00:20.323835+0000","last_clean":"2023-09-23T21:00:20.323835+0000","last_became_active":"2023-09-23T20:55:36.943580+0000","last_became_peered":"2023-09-23T20:55:36.943580+0000","last_unstale":"2023-09-23T21:00:20.323835+0000","last_undegraded":"2023-09-23T21:00:20.323835+0000","last_fullsized":"2023-09-23T21:00:20.323835+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:49:06.969396+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":36,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-09-23T21:00:20.343386+0000","last_change":"2023-09-23T20:57:44.980498+0000","last_active":"2023-09-23T21:00:20.343386+0000","last_peered":"2023-09-23T21:00:20.343386+0000","last_clean":"2023-09-23T21:00:20.343386+0000","last_became_active":"2023-09-23T20:57:44.980182+0000","last_became_peered":"2023-09-23T20:57:44.980182+0000","last_unstale":"2023-09-23T21:00:20.343386+0000","last_undegraded":"2023-09-23T21:00:20.343386+0000","last_fullsized":"2023-09-23T21:00:20.343386+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:34:51.620544+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.d","version":"79'161","reported_seq":529,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-09-23T21:00:20.336791+0000","last_change":"2023-09-23T20:57:31.075213+0000","last_active":"2023-09-23T21:00:20.336791+0000","last_peered":"2023-09-23T21:00:20.336791+0000","last_clean":"2023-09-23T21:00:20.336791+0000","last_became_active":"2023-09-23T20:57:15.988188+0000","last_became_peered":"2023-09-23T20:57:15.988188+0000","last_unstale":"2023-09-23T21:00:20.336791+0000","last_undegraded":"2023-09-23T21:00:20.336791+0000","last_fullsized":"2023-09-23T21:00:20.336791+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:31.075098+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:31.075098+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:41:52.142362+0000","scrub_duration":0.010304610000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.b","version":"247'11736","reported_seq":11794,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-23T21:00:34.358608+0000","last_change":"2023-09-23T20:56:35.543994+0000","last_active":"2023-09-23T21:00:34.358608+0000","last_peered":"2023-09-23T21:00:34.358608+0000","last_clean":"2023-09-23T21:00:34.358608+0000","last_became_active":"2023-09-23T20:56:35.543758+0000","last_became_peered":"2023-09-23T20:56:35.543758+0000","last_unstale":"2023-09-23T21:00:34.358608+0000","last_undegraded":"2023-09-23T21:00:34.358608+0000","last_fullsized":"2023-09-23T21:00:34.358608+0000","mapping_epoch":199,"log_start":"219'4700","ondisk_log_start":"219'4700","created":199,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_clean_scrub_stamp":"2023-09-23T20:56:34.529103+0000","objects_scrubbed":0,"log_size":7036,"log_dups_size":0,"ondisk_log_size":7036,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:21:29.041484+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":61865984,"num_objects":944,"num_object_clones":0,"num_object_copies":2832,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":944,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11736,"num_write_kb":405760,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":178,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-09-23T21:00:20.337224+0000","last_change":"2023-09-23T20:56:45.991841+0000","last_active":"2023-09-23T21:00:20.337224+0000","last_peered":"2023-09-23T21:00:20.337224+0000","last_clean":"2023-09-23T21:00:20.337224+0000","last_became_active":"2023-09-23T20:56:45.991491+0000","last_became_peered":"2023-09-23T20:56:45.991491+0000","last_unstale":"2023-09-23T21:00:20.337224+0000","last_undegraded":"2023-09-23T21:00:20.337224+0000","last_fullsized":"2023-09-23T21:00:20.337224+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":205,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:30:37.405319+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":137,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-09-23T21:00:20.347168+0000","last_change":"2023-09-23T20:55:35.957768+0000","last_active":"2023-09-23T21:00:20.347168+0000","last_peered":"2023-09-23T21:00:20.347168+0000","last_clean":"2023-09-23T21:00:20.347168+0000","last_became_active":"2023-09-23T20:55:35.957421+0000","last_became_peered":"2023-09-23T20:55:35.957421+0000","last_unstale":"2023-09-23T21:00:20.347168+0000","last_undegraded":"2023-09-23T21:00:20.347168+0000","last_fullsized":"2023-09-23T21:00:20.347168+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:49:41.585961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.c","version":"79'161","reported_seq":548,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-09-23T21:00:20.344494+0000","last_change":"2023-09-23T20:57:22.041071+0000","last_active":"2023-09-23T21:00:20.344494+0000","last_peered":"2023-09-23T21:00:20.344494+0000","last_clean":"2023-09-23T21:00:20.344494+0000","last_became_active":"2023-09-23T20:57:15.973204+0000","last_became_peered":"2023-09-23T20:57:15.973204+0000","last_unstale":"2023-09-23T21:00:20.344494+0000","last_undegraded":"2023-09-23T21:00:20.344494+0000","last_fullsized":"2023-09-23T21:00:20.344494+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:22.040907+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:22.040907+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:47:02.019978+0000","scrub_duration":0.013021176000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.a","version":"247'11766","reported_seq":11824,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-23T21:00:34.756286+0000","last_change":"2023-09-23T20:56:35.547120+0000","last_active":"2023-09-23T21:00:34.756286+0000","last_peered":"2023-09-23T21:00:34.756286+0000","last_clean":"2023-09-23T21:00:34.756286+0000","last_became_active":"2023-09-23T20:56:35.546838+0000","last_became_peered":"2023-09-23T20:56:35.546838+0000","last_unstale":"2023-09-23T21:00:34.756286+0000","last_undegraded":"2023-09-23T21:00:34.756286+0000","last_fullsized":"2023-09-23T21:00:34.756286+0000","mapping_epoch":199,"log_start":"227'5600","ondisk_log_start":"227'5600","created":199,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_clean_scrub_stamp":"2023-09-23T20:56:34.529103+0000","objects_scrubbed":0,"log_size":6166,"log_dups_size":0,"ondisk_log_size":6166,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:48:24.132768+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":63569920,"num_objects":970,"num_object_clones":0,"num_object_copies":2910,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":970,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11766,"num_write_kb":407552,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,5],"acting":[3,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":165,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-09-23T21:00:20.336970+0000","last_change":"2023-09-23T20:56:45.991821+0000","last_active":"2023-09-23T21:00:20.336970+0000","last_peered":"2023-09-23T21:00:20.336970+0000","last_clean":"2023-09-23T21:00:20.336970+0000","last_became_active":"2023-09-23T20:56:45.991447+0000","last_became_peered":"2023-09-23T20:56:45.991447+0000","last_unstale":"2023-09-23T21:00:20.336970+0000","last_undegraded":"2023-09-23T21:00:20.336970+0000","last_fullsized":"2023-09-23T21:00:20.336970+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":205,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:14:25.525222+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":168,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-09-23T21:00:20.338861+0000","last_change":"2023-09-23T20:57:44.982059+0000","last_active":"2023-09-23T21:00:20.338861+0000","last_peered":"2023-09-23T21:00:20.338861+0000","last_clean":"2023-09-23T21:00:20.338861+0000","last_became_active":"2023-09-23T20:57:44.981742+0000","last_became_peered":"2023-09-23T20:57:44.981742+0000","last_unstale":"2023-09-23T21:00:20.338861+0000","last_undegraded":"2023-09-23T21:00:20.338861+0000","last_fullsized":"2023-09-23T21:00:20.338861+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:41:24.042916+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.f","version":"79'161","reported_seq":551,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-09-23T21:00:20.324467+0000","last_change":"2023-09-23T20:57:27.880841+0000","last_active":"2023-09-23T21:00:20.324467+0000","last_peered":"2023-09-23T21:00:20.324467+0000","last_clean":"2023-09-23T21:00:20.324467+0000","last_became_active":"2023-09-23T20:57:15.970925+0000","last_became_peered":"2023-09-23T20:57:15.970925+0000","last_unstale":"2023-09-23T21:00:20.324467+0000","last_undegraded":"2023-09-23T21:00:20.324467+0000","last_fullsized":"2023-09-23T21:00:20.324467+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:27.880776+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:27.880776+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:15:13.438014+0000","scrub_duration":0.0014004899999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.9","version":"247'11651","reported_seq":11709,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-23T21:00:31.845041+0000","last_change":"2023-09-23T20:56:35.549387+0000","last_active":"2023-09-23T21:00:31.845041+0000","last_peered":"2023-09-23T21:00:31.845041+0000","last_clean":"2023-09-23T21:00:31.845041+0000","last_became_active":"2023-09-23T20:56:35.549087+0000","last_became_peered":"2023-09-23T20:56:35.549087+0000","last_unstale":"2023-09-23T21:00:31.845041+0000","last_undegraded":"2023-09-23T21:00:31.845041+0000","last_fullsized":"2023-09-23T21:00:31.845041+0000","mapping_epoch":199,"log_start":"217'4300","ondisk_log_start":"217'4300","created":199,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_clean_scrub_stamp":"2023-09-23T20:56:34.529103+0000","objects_scrubbed":0,"log_size":7351,"log_dups_size":0,"ondisk_log_size":7351,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:04:51.547362+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":69140480,"num_objects":1055,"num_object_clones":0,"num_object_copies":3165,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1055,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11651,"num_write_kb":406592,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":151,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-09-23T21:00:20.347213+0000","last_change":"2023-09-23T20:56:45.989153+0000","last_active":"2023-09-23T21:00:20.347213+0000","last_peered":"2023-09-23T21:00:20.347213+0000","last_clean":"2023-09-23T21:00:20.347213+0000","last_became_active":"2023-09-23T20:56:45.988800+0000","last_became_peered":"2023-09-23T20:56:45.988800+0000","last_unstale":"2023-09-23T21:00:20.347213+0000","last_undegraded":"2023-09-23T21:00:20.347213+0000","last_fullsized":"2023-09-23T21:00:20.347213+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":205,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:04:41.130194+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":186,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-09-23T21:00:20.323985+0000","last_change":"2023-09-23T20:55:35.965427+0000","last_active":"2023-09-23T21:00:20.323985+0000","last_peered":"2023-09-23T21:00:20.323985+0000","last_clean":"2023-09-23T21:00:20.323985+0000","last_became_active":"2023-09-23T20:55:35.963969+0000","last_became_peered":"2023-09-23T20:55:35.963969+0000","last_unstale":"2023-09-23T21:00:20.323985+0000","last_undegraded":"2023-09-23T21:00:20.323985+0000","last_fullsized":"2023-09-23T21:00:20.323985+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:38:40.612083+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.e","version":"79'161","reported_seq":458,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-09-23T21:00:20.336839+0000","last_change":"2023-09-23T20:57:34.132738+0000","last_active":"2023-09-23T21:00:20.336839+0000","last_peered":"2023-09-23T21:00:20.336839+0000","last_clean":"2023-09-23T21:00:20.336839+0000","last_became_active":"2023-09-23T20:57:15.974318+0000","last_became_peered":"2023-09-23T20:57:15.974318+0000","last_unstale":"2023-09-23T21:00:20.336839+0000","last_undegraded":"2023-09-23T21:00:20.336839+0000","last_fullsized":"2023-09-23T21:00:20.336839+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:34.132660+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:34.132660+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:07:07.739438+0000","scrub_duration":0.005229694,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.8","version":"247'11801","reported_seq":11859,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-23T21:00:31.519102+0000","last_change":"2023-09-23T20:56:35.548119+0000","last_active":"2023-09-23T21:00:31.519102+0000","last_peered":"2023-09-23T21:00:31.519102+0000","last_clean":"2023-09-23T21:00:31.519102+0000","last_became_active":"2023-09-23T20:56:35.547914+0000","last_became_peered":"2023-09-23T20:56:35.547914+0000","last_unstale":"2023-09-23T21:00:31.519102+0000","last_undegraded":"2023-09-23T21:00:31.519102+0000","last_fullsized":"2023-09-23T21:00:31.519102+0000","mapping_epoch":199,"log_start":"207'2900","ondisk_log_start":"207'2900","created":199,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_clean_scrub_stamp":"2023-09-23T20:56:34.529103+0000","objects_scrubbed":0,"log_size":8901,"log_dups_size":0,"ondisk_log_size":8901,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:42:54.255023+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":69009408,"num_objects":1053,"num_object_clones":0,"num_object_copies":3159,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1053,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11801,"num_write_kb":411328,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":137,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-09-23T21:00:20.347316+0000","last_change":"2023-09-23T20:55:35.962005+0000","last_active":"2023-09-23T21:00:20.347316+0000","last_peered":"2023-09-23T21:00:20.347316+0000","last_clean":"2023-09-23T21:00:20.347316+0000","last_became_active":"2023-09-23T20:55:35.961695+0000","last_became_peered":"2023-09-23T20:55:35.961695+0000","last_unstale":"2023-09-23T21:00:20.347316+0000","last_undegraded":"2023-09-23T21:00:20.347316+0000","last_fullsized":"2023-09-23T21:00:20.347316+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:28:22.876201+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":235,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-09-23T21:00:20.335063+0000","last_change":"2023-09-23T20:54:13.928715+0000","last_active":"2023-09-23T21:00:20.335063+0000","last_peered":"2023-09-23T21:00:20.335063+0000","last_clean":"2023-09-23T21:00:20.335063+0000","last_became_active":"2023-09-23T20:54:13.928200+0000","last_became_peered":"2023-09-23T20:54:13.928200+0000","last_unstale":"2023-09-23T21:00:20.335063+0000","last_undegraded":"2023-09-23T21:00:20.335063+0000","last_fullsized":"2023-09-23T21:00:20.335063+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:41:16.927513+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.f","version":"247'11862","reported_seq":11920,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-23T21:00:31.825721+0000","last_change":"2023-09-23T20:56:35.545128+0000","last_active":"2023-09-23T21:00:31.825721+0000","last_peered":"2023-09-23T21:00:31.825721+0000","last_clean":"2023-09-23T21:00:31.825721+0000","last_became_active":"2023-09-23T20:56:35.544838+0000","last_became_peered":"2023-09-23T20:56:35.544838+0000","last_unstale":"2023-09-23T21:00:31.825721+0000","last_undegraded":"2023-09-23T21:00:31.825721+0000","last_fullsized":"2023-09-23T21:00:31.825721+0000","mapping_epoch":199,"log_start":"217'4500","ondisk_log_start":"217'4500","created":199,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_clean_scrub_stamp":"2023-09-23T20:56:34.529103+0000","objects_scrubbed":0,"log_size":7362,"log_dups_size":0,"ondisk_log_size":7362,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:30:43.388717+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":65798144,"num_objects":1004,"num_object_clones":0,"num_object_copies":3012,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1004,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11862,"num_write_kb":411712,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.9","version":"79'161","reported_seq":620,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-09-23T21:00:20.362268+0000","last_change":"2023-09-23T20:56:46.921939+0000","last_active":"2023-09-23T21:00:20.362268+0000","last_peered":"2023-09-23T21:00:20.362268+0000","last_clean":"2023-09-23T21:00:20.362268+0000","last_became_active":"2023-09-23T20:56:45.989908+0000","last_became_peered":"2023-09-23T20:56:45.989908+0000","last_unstale":"2023-09-23T21:00:20.362268+0000","last_undegraded":"2023-09-23T21:00:20.362268+0000","last_fullsized":"2023-09-23T21:00:20.362268+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":205,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:56:46.921739+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:56:46.921739+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:20:42.635110+0000","scrub_duration":0.0071954109999999996,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":237,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-09-23T21:00:20.343681+0000","last_change":"2023-09-23T20:54:13.961788+0000","last_active":"2023-09-23T21:00:20.343681+0000","last_peered":"2023-09-23T21:00:20.343681+0000","last_clean":"2023-09-23T21:00:20.343681+0000","last_became_active":"2023-09-23T20:54:13.940937+0000","last_became_peered":"2023-09-23T20:54:13.940937+0000","last_unstale":"2023-09-23T21:00:20.343681+0000","last_undegraded":"2023-09-23T21:00:20.343681+0000","last_fullsized":"2023-09-23T21:00:20.343681+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:11:56.785299+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":141,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-09-23T21:00:20.336814+0000","last_change":"2023-09-23T20:55:35.949670+0000","last_active":"2023-09-23T21:00:20.336814+0000","last_peered":"2023-09-23T21:00:20.336814+0000","last_clean":"2023-09-23T21:00:20.336814+0000","last_became_active":"2023-09-23T20:55:35.949140+0000","last_became_peered":"2023-09-23T20:55:35.949140+0000","last_unstale":"2023-09-23T21:00:20.336814+0000","last_undegraded":"2023-09-23T21:00:20.336814+0000","last_fullsized":"2023-09-23T21:00:20.336814+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:55:59.340303+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.e","version":"247'11711","reported_seq":11768,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-23T21:00:31.550616+0000","last_change":"2023-09-23T20:56:35.547184+0000","last_active":"2023-09-23T21:00:31.550616+0000","last_peered":"2023-09-23T21:00:31.550616+0000","last_clean":"2023-09-23T21:00:31.550616+0000","last_became_active":"2023-09-23T20:56:35.546912+0000","last_became_peered":"2023-09-23T20:56:35.546912+0000","last_unstale":"2023-09-23T21:00:31.550616+0000","last_undegraded":"2023-09-23T21:00:31.550616+0000","last_fullsized":"2023-09-23T21:00:31.550616+0000","mapping_epoch":199,"log_start":"206'2800","ondisk_log_start":"206'2800","created":199,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_clean_scrub_stamp":"2023-09-23T20:56:34.529103+0000","objects_scrubbed":0,"log_size":8911,"log_dups_size":0,"ondisk_log_size":8911,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:57:07.422792+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70057984,"num_objects":1069,"num_object_clones":0,"num_object_copies":3207,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1069,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11711,"num_write_kb":408960,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,0],"acting":[2,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.8","version":"79'161","reported_seq":490,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-09-23T21:00:20.347156+0000","last_change":"2023-09-23T20:56:57.505737+0000","last_active":"2023-09-23T21:00:20.347156+0000","last_peered":"2023-09-23T21:00:20.347156+0000","last_clean":"2023-09-23T21:00:20.347156+0000","last_became_active":"2023-09-23T20:56:04.982345+0000","last_became_peered":"2023-09-23T20:56:04.982345+0000","last_unstale":"2023-09-23T21:00:20.347156+0000","last_undegraded":"2023-09-23T21:00:20.347156+0000","last_fullsized":"2023-09-23T21:00:20.347156+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:56:57.505663+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:56:57.505663+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:34:30.109724+0000","scrub_duration":0.026284108,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":209,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-09-23T21:00:20.324189+0000","last_change":"2023-09-23T20:55:35.959942+0000","last_active":"2023-09-23T21:00:20.324189+0000","last_peered":"2023-09-23T21:00:20.324189+0000","last_clean":"2023-09-23T21:00:20.324189+0000","last_became_active":"2023-09-23T20:55:35.957071+0000","last_became_peered":"2023-09-23T20:55:35.957071+0000","last_unstale":"2023-09-23T21:00:20.324189+0000","last_undegraded":"2023-09-23T21:00:20.324189+0000","last_fullsized":"2023-09-23T21:00:20.324189+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:44:30.308036+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":124,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-09-23T21:00:20.337181+0000","last_change":"2023-09-23T20:54:13.951459+0000","last_active":"2023-09-23T21:00:20.337181+0000","last_peered":"2023-09-23T21:00:20.337181+0000","last_clean":"2023-09-23T21:00:20.337181+0000","last_became_active":"2023-09-23T20:54:13.948409+0000","last_became_peered":"2023-09-23T20:54:13.948409+0000","last_unstale":"2023-09-23T21:00:20.337181+0000","last_undegraded":"2023-09-23T21:00:20.337181+0000","last_fullsized":"2023-09-23T21:00:20.337181+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:38:10.148061+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.b","version":"79'161","reported_seq":530,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-09-23T21:00:20.346559+0000","last_change":"2023-09-23T20:57:27.320172+0000","last_active":"2023-09-23T21:00:20.346559+0000","last_peered":"2023-09-23T21:00:20.346559+0000","last_clean":"2023-09-23T21:00:20.346559+0000","last_became_active":"2023-09-23T20:57:16.009630+0000","last_became_peered":"2023-09-23T20:57:16.009630+0000","last_unstale":"2023-09-23T21:00:20.346559+0000","last_undegraded":"2023-09-23T21:00:20.346559+0000","last_fullsized":"2023-09-23T21:00:20.346559+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:27.320022+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:27.320022+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:31:35.221558+0000","scrub_duration":0.0043894679999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,5],"acting":[1,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.d","version":"247'11808","reported_seq":11865,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-23T21:00:34.087393+0000","last_change":"2023-09-23T20:56:35.544132+0000","last_active":"2023-09-23T21:00:34.087393+0000","last_peered":"2023-09-23T21:00:34.087393+0000","last_clean":"2023-09-23T21:00:34.087393+0000","last_became_active":"2023-09-23T20:56:35.543836+0000","last_became_peered":"2023-09-23T20:56:35.543836+0000","last_unstale":"2023-09-23T21:00:34.087393+0000","last_undegraded":"2023-09-23T21:00:34.087393+0000","last_fullsized":"2023-09-23T21:00:34.087393+0000","mapping_epoch":199,"log_start":"217'4400","ondisk_log_start":"217'4400","created":199,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_clean_scrub_stamp":"2023-09-23T20:56:34.529103+0000","objects_scrubbed":0,"log_size":7408,"log_dups_size":0,"ondisk_log_size":7408,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:27:14.636991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":64290840,"num_objects":982,"num_object_clones":0,"num_object_copies":2946,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":982,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":11808,"num_write_kb":409217,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":124,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-09-23T21:00:20.338662+0000","last_change":"2023-09-23T20:54:13.950254+0000","last_active":"2023-09-23T21:00:20.338662+0000","last_peered":"2023-09-23T21:00:20.338662+0000","last_clean":"2023-09-23T21:00:20.338662+0000","last_became_active":"2023-09-23T20:54:13.949934+0000","last_became_peered":"2023-09-23T20:54:13.949934+0000","last_unstale":"2023-09-23T21:00:20.338662+0000","last_undegraded":"2023-09-23T21:00:20.338662+0000","last_fullsized":"2023-09-23T21:00:20.338662+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:30:32.847167+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":141,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-09-23T21:00:20.337251+0000","last_change":"2023-09-23T20:55:35.956153+0000","last_active":"2023-09-23T21:00:20.337251+0000","last_peered":"2023-09-23T21:00:20.337251+0000","last_clean":"2023-09-23T21:00:20.337251+0000","last_became_active":"2023-09-23T20:55:35.955630+0000","last_became_peered":"2023-09-23T20:55:35.955630+0000","last_unstale":"2023-09-23T21:00:20.337251+0000","last_undegraded":"2023-09-23T21:00:20.337251+0000","last_fullsized":"2023-09-23T21:00:20.337251+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:45:20.282929+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.c","version":"247'11824","reported_seq":11890,"reported_epoch":247,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T21:00:34.057473+0000","last_change":"2023-09-23T20:57:46.463005+0000","last_active":"2023-09-23T21:00:34.057473+0000","last_peered":"2023-09-23T21:00:34.057473+0000","last_clean":"2023-09-23T20:57:44.015119+0000","last_became_active":"2023-09-23T20:57:46.448702+0000","last_became_peered":"2023-09-23T20:57:46.448702+0000","last_unstale":"2023-09-23T21:00:34.057473+0000","last_undegraded":"2023-09-23T20:57:46.010874+0000","last_fullsized":"2023-09-23T20:57:45.978434+0000","mapping_epoch":223,"log_start":"217'4500","ondisk_log_start":"217'4500","created":199,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_clean_scrub_stamp":"2023-09-23T20:56:34.529103+0000","objects_scrubbed":0,"log_size":7324,"log_dups_size":0,"ondisk_log_size":7324,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:08:34.180651+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":64487424,"num_objects":984,"num_object_clones":0,"num_object_copies":2952,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5473,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":984,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11824,"num_write_kb":409856,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,7],"avail_no_missing":["6","2","7"],"object_location_counts":[{"shards":"2,6,7","objects":53},{"shards":"6,7","objects":-4489}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.a","version":"79'161","reported_seq":490,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-09-23T21:00:20.347998+0000","last_change":"2023-09-23T20:56:44.487778+0000","last_active":"2023-09-23T21:00:20.347998+0000","last_peered":"2023-09-23T21:00:20.347998+0000","last_clean":"2023-09-23T21:00:20.347998+0000","last_became_active":"2023-09-23T20:56:05.006740+0000","last_became_peered":"2023-09-23T20:56:05.006740+0000","last_unstale":"2023-09-23T21:00:20.347998+0000","last_undegraded":"2023-09-23T21:00:20.347998+0000","last_fullsized":"2023-09-23T21:00:20.347998+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:56:44.487678+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:56:44.487678+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:04:50.086749+0000","scrub_duration":0.0017408899999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":36,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-09-23T21:00:20.342467+0000","last_change":"2023-09-23T20:57:44.982513+0000","last_active":"2023-09-23T21:00:20.342467+0000","last_peered":"2023-09-23T21:00:20.342467+0000","last_clean":"2023-09-23T21:00:20.342467+0000","last_became_active":"2023-09-23T20:57:44.982112+0000","last_became_peered":"2023-09-23T20:57:44.982112+0000","last_unstale":"2023-09-23T21:00:20.342467+0000","last_undegraded":"2023-09-23T21:00:20.342467+0000","last_fullsized":"2023-09-23T21:00:20.342467+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:09:05.630041+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":217,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-09-23T21:00:20.338038+0000","last_change":"2023-09-23T20:54:13.940748+0000","last_active":"2023-09-23T21:00:20.338038+0000","last_peered":"2023-09-23T21:00:20.338038+0000","last_clean":"2023-09-23T21:00:20.338038+0000","last_became_active":"2023-09-23T20:54:13.940419+0000","last_became_peered":"2023-09-23T20:54:13.940419+0000","last_unstale":"2023-09-23T21:00:20.338038+0000","last_undegraded":"2023-09-23T21:00:20.338038+0000","last_fullsized":"2023-09-23T21:00:20.338038+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:15:01.923816+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.1","version":"247'11715","reported_seq":11784,"reported_epoch":247,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T21:00:32.188788+0000","last_change":"2023-09-23T20:57:46.483248+0000","last_active":"2023-09-23T21:00:32.188788+0000","last_peered":"2023-09-23T21:00:32.188788+0000","last_clean":"2023-09-23T20:57:44.047957+0000","last_became_active":"2023-09-23T20:57:46.475568+0000","last_became_peered":"2023-09-23T20:57:46.475568+0000","last_unstale":"2023-09-23T21:00:32.188788+0000","last_undegraded":"2023-09-23T20:57:46.083591+0000","last_fullsized":"2023-09-23T20:57:45.978436+0000","mapping_epoch":223,"log_start":"219'5000","ondisk_log_start":"219'5000","created":199,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_clean_scrub_stamp":"2023-09-23T20:56:34.529103+0000","objects_scrubbed":0,"log_size":6715,"log_dups_size":0,"ondisk_log_size":6715,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:22:45.067269+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70189056,"num_objects":1071,"num_object_clones":0,"num_object_copies":3213,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5470,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1071,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11715,"num_write_kb":409152,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,6],"avail_no_missing":["4","1","6"],"object_location_counts":[{"shards":"1,4,6","objects":148},{"shards":"4,6","objects":-4399}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.7","version":"79'161","reported_seq":552,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-09-23T21:00:20.324716+0000","last_change":"2023-09-23T20:57:48.243572+0000","last_active":"2023-09-23T21:00:20.324716+0000","last_peered":"2023-09-23T21:00:20.324716+0000","last_clean":"2023-09-23T21:00:20.324716+0000","last_became_active":"2023-09-23T20:57:48.243229+0000","last_became_peered":"2023-09-23T20:57:48.243229+0000","last_unstale":"2023-09-23T21:00:20.324716+0000","last_undegraded":"2023-09-23T21:00:20.324716+0000","last_fullsized":"2023-09-23T21:00:20.324716+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":3,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:25.916867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:25.916867+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:23:19.385831+0000","scrub_duration":0.001665323,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":223,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-09-23T21:00:29.373832+0000","last_change":"2023-09-23T20:50:05.867130+0000","last_active":"2023-09-23T21:00:29.373832+0000","last_peered":"2023-09-23T21:00:29.373832+0000","last_clean":"2023-09-23T21:00:29.373832+0000","last_became_active":"2023-09-23T20:50:05.866758+0000","last_became_peered":"2023-09-23T20:50:05.866758+0000","last_unstale":"2023-09-23T21:00:29.373832+0000","last_undegraded":"2023-09-23T21:00:29.373832+0000","last_fullsized":"2023-09-23T21:00:29.373832+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:55:07.548209+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":120,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-09-23T21:00:20.347627+0000","last_change":"2023-09-23T20:54:31.887604+0000","last_active":"2023-09-23T21:00:20.347627+0000","last_peered":"2023-09-23T21:00:20.347627+0000","last_clean":"2023-09-23T21:00:20.347627+0000","last_became_active":"2023-09-23T20:54:31.886752+0000","last_became_peered":"2023-09-23T20:54:31.886752+0000","last_unstale":"2023-09-23T21:00:20.347627+0000","last_undegraded":"2023-09-23T21:00:20.347627+0000","last_fullsized":"2023-09-23T21:00:20.347627+0000","mapping_epoch":136,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:13:58.872176+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":221,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-09-23T21:00:29.373739+0000","last_change":"2023-09-23T20:54:31.899082+0000","last_active":"2023-09-23T21:00:29.373739+0000","last_peered":"2023-09-23T21:00:29.373739+0000","last_clean":"2023-09-23T21:00:29.373739+0000","last_became_active":"2023-09-23T20:54:31.898521+0000","last_became_peered":"2023-09-23T20:54:31.898521+0000","last_unstale":"2023-09-23T21:00:29.373739+0000","last_undegraded":"2023-09-23T21:00:29.373739+0000","last_fullsized":"2023-09-23T21:00:29.373739+0000","mapping_epoch":136,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:25:36.059711+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.7","version":"247'11861","reported_seq":11917,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-23T21:00:34.106510+0000","last_change":"2023-09-23T20:56:35.541695+0000","last_active":"2023-09-23T21:00:34.106510+0000","last_peered":"2023-09-23T21:00:34.106510+0000","last_clean":"2023-09-23T21:00:34.106510+0000","last_became_active":"2023-09-23T20:56:35.541392+0000","last_became_peered":"2023-09-23T20:56:35.541392+0000","last_unstale":"2023-09-23T21:00:34.106510+0000","last_undegraded":"2023-09-23T21:00:34.106510+0000","last_fullsized":"2023-09-23T21:00:34.106510+0000","mapping_epoch":199,"log_start":"217'4500","ondisk_log_start":"217'4500","created":199,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_clean_scrub_stamp":"2023-09-23T20:56:34.529103+0000","objects_scrubbed":0,"log_size":7361,"log_dups_size":0,"ondisk_log_size":7361,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:57:22.478253+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":62193664,"num_objects":949,"num_object_clones":0,"num_object_copies":2847,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":949,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11861,"num_write_kb":409920,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.1","version":"79'161","reported_seq":510,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-09-23T21:00:20.347107+0000","last_change":"2023-09-23T20:57:19.669346+0000","last_active":"2023-09-23T21:00:20.347107+0000","last_peered":"2023-09-23T21:00:20.347107+0000","last_clean":"2023-09-23T21:00:20.347107+0000","last_became_active":"2023-09-23T20:57:08.246046+0000","last_became_peered":"2023-09-23T20:57:08.246046+0000","last_unstale":"2023-09-23T21:00:20.347107+0000","last_undegraded":"2023-09-23T21:00:20.347107+0000","last_fullsized":"2023-09-23T21:00:20.347107+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:19.669259+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:19.669259+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:14:03.435531+0000","scrub_duration":0.0045759559999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":137,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-09-23T21:00:20.347736+0000","last_change":"2023-09-23T20:55:35.959514+0000","last_active":"2023-09-23T21:00:20.347736+0000","last_peered":"2023-09-23T21:00:20.347736+0000","last_clean":"2023-09-23T21:00:20.347736+0000","last_became_active":"2023-09-23T20:55:35.959244+0000","last_became_peered":"2023-09-23T20:55:35.959244+0000","last_unstale":"2023-09-23T21:00:20.347736+0000","last_undegraded":"2023-09-23T21:00:20.347736+0000","last_fullsized":"2023-09-23T21:00:20.347736+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:47:32.665316+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":141,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-09-23T21:00:20.337039+0000","last_change":"2023-09-23T20:55:35.949000+0000","last_active":"2023-09-23T21:00:20.337039+0000","last_peered":"2023-09-23T21:00:20.337039+0000","last_clean":"2023-09-23T21:00:20.337039+0000","last_became_active":"2023-09-23T20:55:35.948410+0000","last_became_peered":"2023-09-23T20:55:35.948410+0000","last_unstale":"2023-09-23T21:00:20.337039+0000","last_undegraded":"2023-09-23T21:00:20.337039+0000","last_fullsized":"2023-09-23T21:00:20.337039+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:26:50.477289+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":215,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-09-23T21:00:29.373797+0000","last_change":"2023-09-23T20:51:10.829889+0000","last_active":"2023-09-23T21:00:29.373797+0000","last_peered":"2023-09-23T21:00:29.373797+0000","last_clean":"2023-09-23T21:00:29.373797+0000","last_became_active":"2023-09-23T20:51:10.829533+0000","last_became_peered":"2023-09-23T20:51:10.829533+0000","last_unstale":"2023-09-23T21:00:29.373797+0000","last_undegraded":"2023-09-23T21:00:29.373797+0000","last_fullsized":"2023-09-23T21:00:29.373797+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:43:04.603460+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.4","version":"247'11691","reported_seq":11749,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-23T21:00:31.868109+0000","last_change":"2023-09-23T20:56:35.549909+0000","last_active":"2023-09-23T21:00:31.868109+0000","last_peered":"2023-09-23T21:00:31.868109+0000","last_clean":"2023-09-23T21:00:31.868109+0000","last_became_active":"2023-09-23T20:56:35.549618+0000","last_became_peered":"2023-09-23T20:56:35.549618+0000","last_unstale":"2023-09-23T21:00:31.868109+0000","last_undegraded":"2023-09-23T21:00:31.868109+0000","last_fullsized":"2023-09-23T21:00:31.868109+0000","mapping_epoch":199,"log_start":"217'4300","ondisk_log_start":"217'4300","created":199,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_clean_scrub_stamp":"2023-09-23T20:56:34.529103+0000","objects_scrubbed":0,"log_size":7391,"log_dups_size":0,"ondisk_log_size":7391,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:24:25.918475+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":67567616,"num_objects":1031,"num_object_clones":0,"num_object_copies":3093,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1031,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11691,"num_write_kb":407104,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.2","version":"79'161","reported_seq":477,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-09-23T21:00:20.341638+0000","last_change":"2023-09-23T20:57:24.802586+0000","last_active":"2023-09-23T21:00:20.341638+0000","last_peered":"2023-09-23T21:00:20.341638+0000","last_clean":"2023-09-23T21:00:20.341638+0000","last_became_active":"2023-09-23T20:57:08.236114+0000","last_became_peered":"2023-09-23T20:57:08.236114+0000","last_unstale":"2023-09-23T21:00:20.341638+0000","last_undegraded":"2023-09-23T21:00:20.341638+0000","last_fullsized":"2023-09-23T21:00:20.341638+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:24.802451+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:24.802451+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:06:22.309980+0000","scrub_duration":0.0090861250000000005,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,4],"acting":[0,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":233,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-09-23T21:00:20.347799+0000","last_change":"2023-09-23T20:51:10.838054+0000","last_active":"2023-09-23T21:00:20.347799+0000","last_peered":"2023-09-23T21:00:20.347799+0000","last_clean":"2023-09-23T21:00:20.347799+0000","last_became_active":"2023-09-23T20:51:10.837750+0000","last_became_peered":"2023-09-23T20:51:10.837750+0000","last_unstale":"2023-09-23T21:00:20.347799+0000","last_undegraded":"2023-09-23T21:00:20.347799+0000","last_fullsized":"2023-09-23T21:00:20.347799+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:15:51.648085+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":215,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-09-23T21:00:20.337441+0000","last_change":"2023-09-23T20:55:35.951146+0000","last_active":"2023-09-23T21:00:20.337441+0000","last_peered":"2023-09-23T21:00:20.337441+0000","last_clean":"2023-09-23T21:00:20.337441+0000","last_became_active":"2023-09-23T20:55:35.950874+0000","last_became_peered":"2023-09-23T20:55:35.950874+0000","last_unstale":"2023-09-23T21:00:20.337441+0000","last_undegraded":"2023-09-23T21:00:20.337441+0000","last_fullsized":"2023-09-23T21:00:20.337441+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:58:13.957677+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":235,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-09-23T21:00:20.337947+0000","last_change":"2023-09-23T20:54:31.898870+0000","last_active":"2023-09-23T21:00:20.337947+0000","last_peered":"2023-09-23T21:00:20.337947+0000","last_clean":"2023-09-23T21:00:20.337947+0000","last_became_active":"2023-09-23T20:54:31.898467+0000","last_became_peered":"2023-09-23T20:54:31.898467+0000","last_unstale":"2023-09-23T21:00:20.337947+0000","last_undegraded":"2023-09-23T21:00:20.337947+0000","last_fullsized":"2023-09-23T21:00:20.337947+0000","mapping_epoch":136,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:56:22.483408+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.6","version":"247'11539","reported_seq":20323,"reported_epoch":247,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-09-23T21:00:31.887818+0000","last_change":"2023-09-23T20:58:39.453197+0000","last_active":"2023-09-23T21:00:31.887818+0000","last_peered":"2023-09-23T21:00:31.887818+0000","last_clean":"2023-09-23T20:57:44.031597+0000","last_became_active":"2023-09-23T20:57:46.443927+0000","last_became_peered":"2023-09-23T20:57:46.443927+0000","last_unstale":"2023-09-23T21:00:31.887818+0000","last_undegraded":"2023-09-23T20:58:39.450464+0000","last_fullsized":"2023-09-23T20:57:45.977227+0000","mapping_epoch":223,"log_start":"217'4200","ondisk_log_start":"217'4200","created":199,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_clean_scrub_stamp":"2023-09-23T20:56:34.529103+0000","objects_scrubbed":0,"log_size":7339,"log_dups_size":0,"ondisk_log_size":7339,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:37:23.911131+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":68747264,"num_objects":1049,"num_object_clones":0,"num_object_copies":3147,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":949,"num_objects_misplaced":90,"num_objects_unfound":0,"num_objects_dirty":1049,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11539,"num_write_kb":402816,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4359,"num_bytes_recovered":285671424,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7],"avail_no_missing":["0","5","7"],"object_location_counts":[{"shards":"0,5,7","objects":90},{"shards":"0,7","objects":10}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":120,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-09-23T21:00:20.348313+0000","last_change":"2023-09-23T20:54:31.883504+0000","last_active":"2023-09-23T21:00:20.348313+0000","last_peered":"2023-09-23T21:00:20.348313+0000","last_clean":"2023-09-23T21:00:20.348313+0000","last_became_active":"2023-09-23T20:54:31.883172+0000","last_became_peered":"2023-09-23T20:54:31.883172+0000","last_unstale":"2023-09-23T21:00:20.348313+0000","last_undegraded":"2023-09-23T21:00:20.348313+0000","last_fullsized":"2023-09-23T21:00:20.348313+0000","mapping_epoch":136,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:45:40.289120+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"79'161","reported_seq":477,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-09-23T21:00:20.333572+0000","last_change":"2023-09-23T20:57:20.738417+0000","last_active":"2023-09-23T21:00:20.333572+0000","last_peered":"2023-09-23T21:00:20.333572+0000","last_clean":"2023-09-23T21:00:20.333572+0000","last_became_active":"2023-09-23T20:57:08.236842+0000","last_became_peered":"2023-09-23T20:57:08.236842+0000","last_unstale":"2023-09-23T21:00:20.333572+0000","last_undegraded":"2023-09-23T21:00:20.333572+0000","last_fullsized":"2023-09-23T21:00:20.333572+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":0,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:20.738273+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:20.738273+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:18:28.327990+0000","scrub_duration":0.0088336339999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":225,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-09-23T21:00:20.344160+0000","last_change":"2023-09-23T20:55:35.950777+0000","last_active":"2023-09-23T21:00:20.344160+0000","last_peered":"2023-09-23T21:00:20.344160+0000","last_clean":"2023-09-23T21:00:20.344160+0000","last_became_active":"2023-09-23T20:55:35.950427+0000","last_became_peered":"2023-09-23T20:55:35.950427+0000","last_unstale":"2023-09-23T21:00:20.344160+0000","last_undegraded":"2023-09-23T21:00:20.344160+0000","last_fullsized":"2023-09-23T21:00:20.344160+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:20:10.371751+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":191,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-09-23T21:00:20.324164+0000","last_change":"2023-09-23T20:55:35.978140+0000","last_active":"2023-09-23T21:00:20.324164+0000","last_peered":"2023-09-23T21:00:20.324164+0000","last_clean":"2023-09-23T21:00:20.324164+0000","last_became_active":"2023-09-23T20:55:35.977831+0000","last_became_peered":"2023-09-23T20:55:35.977831+0000","last_unstale":"2023-09-23T21:00:20.324164+0000","last_undegraded":"2023-09-23T21:00:20.324164+0000","last_fullsized":"2023-09-23T21:00:20.324164+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:25:47.678023+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.5","version":"247'11660","reported_seq":11725,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-23T21:00:31.491264+0000","last_change":"2023-09-23T20:56:45.990110+0000","last_active":"2023-09-23T21:00:31.491264+0000","last_peered":"2023-09-23T21:00:31.491264+0000","last_clean":"2023-09-23T21:00:31.491264+0000","last_became_active":"2023-09-23T20:56:45.988394+0000","last_became_peered":"2023-09-23T20:56:45.988394+0000","last_unstale":"2023-09-23T21:00:31.491264+0000","last_undegraded":"2023-09-23T21:00:31.491264+0000","last_fullsized":"2023-09-23T21:00:31.491264+0000","mapping_epoch":204,"log_start":"206'2800","ondisk_log_start":"206'2800","created":199,"last_epoch_clean":205,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_clean_scrub_stamp":"2023-09-23T20:56:34.529103+0000","objects_scrubbed":0,"log_size":8860,"log_dups_size":0,"ondisk_log_size":8860,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:22:03.814210+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":66715648,"num_objects":1018,"num_object_clones":0,"num_object_copies":3054,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1018,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11660,"num_write_kb":405696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,0],"acting":[2,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"79'161","reported_seq":514,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-09-23T21:00:20.346714+0000","last_change":"2023-09-23T20:57:20.378531+0000","last_active":"2023-09-23T21:00:20.346714+0000","last_peered":"2023-09-23T21:00:20.346714+0000","last_clean":"2023-09-23T21:00:20.346714+0000","last_became_active":"2023-09-23T20:57:08.238019+0000","last_became_peered":"2023-09-23T20:57:08.238019+0000","last_unstale":"2023-09-23T21:00:20.346714+0000","last_undegraded":"2023-09-23T21:00:20.346714+0000","last_fullsized":"2023-09-23T21:00:20.346714+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":3,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:20.377714+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:20.377714+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:00:08.305602+0000","scrub_duration":0.003015228,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":116,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-09-23T21:00:20.337580+0000","last_change":"2023-09-23T20:56:45.991884+0000","last_active":"2023-09-23T21:00:20.337580+0000","last_peered":"2023-09-23T21:00:20.337580+0000","last_clean":"2023-09-23T21:00:20.337580+0000","last_became_active":"2023-09-23T20:56:45.991568+0000","last_became_peered":"2023-09-23T20:56:45.991568+0000","last_unstale":"2023-09-23T21:00:20.337580+0000","last_undegraded":"2023-09-23T21:00:20.337580+0000","last_fullsized":"2023-09-23T21:00:20.337580+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":205,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:21:20.775511+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,0],"acting":[1,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":241,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-09-23T21:00:20.334326+0000","last_change":"2023-09-23T20:57:44.979950+0000","last_active":"2023-09-23T21:00:20.334326+0000","last_peered":"2023-09-23T21:00:20.334326+0000","last_clean":"2023-09-23T21:00:20.334326+0000","last_became_active":"2023-09-23T20:57:44.979346+0000","last_became_peered":"2023-09-23T20:57:44.979346+0000","last_unstale":"2023-09-23T21:00:20.334326+0000","last_undegraded":"2023-09-23T21:00:20.334326+0000","last_fullsized":"2023-09-23T21:00:20.334326+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:10:24.372800+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":220,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-09-23T21:00:21.831766+0000","last_change":"2023-09-23T20:50:37.982235+0000","last_active":"2023-09-23T21:00:21.831766+0000","last_peered":"2023-09-23T21:00:21.831766+0000","last_clean":"2023-09-23T21:00:21.831766+0000","last_became_active":"2023-09-23T20:50:37.981913+0000","last_became_peered":"2023-09-23T20:50:37.981913+0000","last_unstale":"2023-09-23T21:00:21.831766+0000","last_undegraded":"2023-09-23T21:00:21.831766+0000","last_fullsized":"2023-09-23T21:00:21.831766+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:59:57.122584+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.0","version":"247'11663","reported_seq":11719,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-23T21:00:32.005970+0000","last_change":"2023-09-23T20:56:35.550635+0000","last_active":"2023-09-23T21:00:32.005970+0000","last_peered":"2023-09-23T21:00:32.005970+0000","last_clean":"2023-09-23T21:00:32.005970+0000","last_became_active":"2023-09-23T20:56:35.550376+0000","last_became_peered":"2023-09-23T20:56:35.550376+0000","last_unstale":"2023-09-23T21:00:32.005970+0000","last_undegraded":"2023-09-23T21:00:32.005970+0000","last_fullsized":"2023-09-23T21:00:32.005970+0000","mapping_epoch":199,"log_start":"207'3000","ondisk_log_start":"207'3000","created":199,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_clean_scrub_stamp":"2023-09-23T20:56:34.529103+0000","objects_scrubbed":0,"log_size":8663,"log_dups_size":0,"ondisk_log_size":8663,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:40:55.020049+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":66781184,"num_objects":1019,"num_object_clones":0,"num_object_copies":3057,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1019,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11663,"num_write_kb":405824,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.6","version":"79'161","reported_seq":529,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-09-23T21:00:20.324480+0000","last_change":"2023-09-23T20:57:26.907505+0000","last_active":"2023-09-23T21:00:20.324480+0000","last_peered":"2023-09-23T21:00:20.324480+0000","last_clean":"2023-09-23T21:00:20.324480+0000","last_became_active":"2023-09-23T20:57:08.240321+0000","last_became_peered":"2023-09-23T20:57:08.240321+0000","last_unstale":"2023-09-23T21:00:20.324480+0000","last_undegraded":"2023-09-23T21:00:20.324480+0000","last_fullsized":"2023-09-23T21:00:20.324480+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":3,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:26.907408+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:26.907408+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:34:17.392928+0000","scrub_duration":0.010337038,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":163,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-09-23T21:00:20.341986+0000","last_change":"2023-09-23T20:56:45.988566+0000","last_active":"2023-09-23T21:00:20.341986+0000","last_peered":"2023-09-23T21:00:20.341986+0000","last_clean":"2023-09-23T21:00:20.341986+0000","last_became_active":"2023-09-23T20:56:45.988249+0000","last_became_peered":"2023-09-23T20:56:45.988249+0000","last_unstale":"2023-09-23T21:00:20.341986+0000","last_undegraded":"2023-09-23T21:00:20.341986+0000","last_fullsized":"2023-09-23T21:00:20.341986+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":205,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:51:36.644764+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,4],"acting":[0,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":218,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-09-23T21:00:29.373816+0000","last_change":"2023-09-23T20:54:31.898965+0000","last_active":"2023-09-23T21:00:29.373816+0000","last_peered":"2023-09-23T21:00:29.373816+0000","last_clean":"2023-09-23T21:00:29.373816+0000","last_became_active":"2023-09-23T20:54:31.898663+0000","last_became_peered":"2023-09-23T20:54:31.898663+0000","last_unstale":"2023-09-23T21:00:29.373816+0000","last_undegraded":"2023-09-23T21:00:29.373816+0000","last_fullsized":"2023-09-23T21:00:29.373816+0000","mapping_epoch":136,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:53:05.058087+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":208,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-09-23T21:00:20.335295+0000","last_change":"2023-09-23T20:54:31.884717+0000","last_active":"2023-09-23T21:00:20.335295+0000","last_peered":"2023-09-23T21:00:20.335295+0000","last_clean":"2023-09-23T21:00:20.335295+0000","last_became_active":"2023-09-23T20:54:31.884345+0000","last_became_peered":"2023-09-23T20:54:31.884345+0000","last_unstale":"2023-09-23T21:00:20.335295+0000","last_undegraded":"2023-09-23T21:00:20.335295+0000","last_fullsized":"2023-09-23T21:00:20.335295+0000","mapping_epoch":136,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:02:26.393868+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.3","version":"247'11609","reported_seq":11667,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-23T21:00:34.713688+0000","last_change":"2023-09-23T20:56:35.549655+0000","last_active":"2023-09-23T21:00:34.713688+0000","last_peered":"2023-09-23T21:00:34.713688+0000","last_clean":"2023-09-23T21:00:34.713688+0000","last_became_active":"2023-09-23T20:56:35.549357+0000","last_became_peered":"2023-09-23T20:56:35.549357+0000","last_unstale":"2023-09-23T21:00:34.713688+0000","last_undegraded":"2023-09-23T21:00:34.713688+0000","last_fullsized":"2023-09-23T21:00:34.713688+0000","mapping_epoch":199,"log_start":"224'5400","ondisk_log_start":"224'5400","created":199,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_clean_scrub_stamp":"2023-09-23T20:56:34.529103+0000","objects_scrubbed":0,"log_size":6209,"log_dups_size":0,"ondisk_log_size":6209,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:37:58.752518+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":59834368,"num_objects":913,"num_object_clones":0,"num_object_copies":2739,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":913,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11609,"num_write_kb":400704,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.5","version":"79'161","reported_seq":510,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-09-23T21:00:20.336878+0000","last_change":"2023-09-23T20:57:30.029479+0000","last_active":"2023-09-23T21:00:20.336878+0000","last_peered":"2023-09-23T21:00:20.336878+0000","last_clean":"2023-09-23T21:00:20.336878+0000","last_became_active":"2023-09-23T20:57:08.249909+0000","last_became_peered":"2023-09-23T20:57:08.249909+0000","last_unstale":"2023-09-23T21:00:20.336878+0000","last_undegraded":"2023-09-23T21:00:20.336878+0000","last_fullsized":"2023-09-23T21:00:20.336878+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":3,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:30.029396+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:30.029396+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:16:46.083145+0000","scrub_duration":0.012914682,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":211,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-09-23T21:00:20.358172+0000","last_change":"2023-09-23T20:51:10.836910+0000","last_active":"2023-09-23T21:00:20.358172+0000","last_peered":"2023-09-23T21:00:20.358172+0000","last_clean":"2023-09-23T21:00:20.358172+0000","last_became_active":"2023-09-23T20:51:10.836557+0000","last_became_peered":"2023-09-23T20:51:10.836557+0000","last_unstale":"2023-09-23T21:00:20.358172+0000","last_undegraded":"2023-09-23T21:00:20.358172+0000","last_fullsized":"2023-09-23T21:00:20.358172+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:51:54.019969+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":217,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-09-23T21:00:20.337595+0000","last_change":"2023-09-23T20:55:35.953444+0000","last_active":"2023-09-23T21:00:20.337595+0000","last_peered":"2023-09-23T21:00:20.337595+0000","last_clean":"2023-09-23T21:00:20.337595+0000","last_became_active":"2023-09-23T20:55:35.953151+0000","last_became_peered":"2023-09-23T20:55:35.953151+0000","last_unstale":"2023-09-23T21:00:20.337595+0000","last_undegraded":"2023-09-23T21:00:20.337595+0000","last_fullsized":"2023-09-23T21:00:20.337595+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:39:12.404391+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":36,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-09-23T21:00:20.366365+0000","last_change":"2023-09-23T20:57:44.982509+0000","last_active":"2023-09-23T21:00:20.366365+0000","last_peered":"2023-09-23T21:00:20.366365+0000","last_clean":"2023-09-23T21:00:20.366365+0000","last_became_active":"2023-09-23T20:57:44.982175+0000","last_became_peered":"2023-09-23T20:57:44.982175+0000","last_unstale":"2023-09-23T21:00:20.366365+0000","last_undegraded":"2023-09-23T21:00:20.366365+0000","last_fullsized":"2023-09-23T21:00:20.366365+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:37:49.215488+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.2","version":"247'11627","reported_seq":11685,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-23T21:00:34.713747+0000","last_change":"2023-09-23T20:56:35.543020+0000","last_active":"2023-09-23T21:00:34.713747+0000","last_peered":"2023-09-23T21:00:34.713747+0000","last_clean":"2023-09-23T21:00:34.713747+0000","last_became_active":"2023-09-23T20:56:35.542719+0000","last_became_peered":"2023-09-23T20:56:35.542719+0000","last_unstale":"2023-09-23T21:00:34.713747+0000","last_undegraded":"2023-09-23T21:00:34.713747+0000","last_fullsized":"2023-09-23T21:00:34.713747+0000","mapping_epoch":199,"log_start":"226'5500","ondisk_log_start":"226'5500","created":199,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_clean_scrub_stamp":"2023-09-23T20:56:34.529103+0000","objects_scrubbed":0,"log_size":6127,"log_dups_size":0,"ondisk_log_size":6127,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:52:21.088628+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":57999360,"num_objects":885,"num_object_clones":0,"num_object_copies":2655,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":885,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11627,"num_write_kb":400384,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.4","version":"79'161","reported_seq":526,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-09-23T21:00:20.344581+0000","last_change":"2023-09-23T20:57:20.012904+0000","last_active":"2023-09-23T21:00:20.344581+0000","last_peered":"2023-09-23T21:00:20.344581+0000","last_clean":"2023-09-23T21:00:20.344581+0000","last_became_active":"2023-09-23T20:57:08.237996+0000","last_became_peered":"2023-09-23T20:57:08.237996+0000","last_unstale":"2023-09-23T21:00:20.344581+0000","last_undegraded":"2023-09-23T21:00:20.344581+0000","last_fullsized":"2023-09-23T21:00:20.344581+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":3,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:20.012764+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:20.012764+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:51:28.796946+0000","scrub_duration":0.0045251919999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":196,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-09-23T21:00:20.324604+0000","last_change":"2023-09-23T20:55:35.949541+0000","last_active":"2023-09-23T21:00:20.324604+0000","last_peered":"2023-09-23T21:00:20.324604+0000","last_clean":"2023-09-23T21:00:20.324604+0000","last_became_active":"2023-09-23T20:55:35.944635+0000","last_became_peered":"2023-09-23T20:55:35.944635+0000","last_unstale":"2023-09-23T21:00:20.324604+0000","last_undegraded":"2023-09-23T21:00:20.324604+0000","last_fullsized":"2023-09-23T21:00:20.324604+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:30:36.033585+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"70'1","reported_seq":236,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-09-23T21:00:21.831786+0000","last_change":"2023-09-23T20:54:14.009723+0000","last_active":"2023-09-23T21:00:21.831786+0000","last_peered":"2023-09-23T21:00:21.831786+0000","last_clean":"2023-09-23T21:00:21.831786+0000","last_became_active":"2023-09-23T20:54:13.941771+0000","last_became_peered":"2023-09-23T20:54:13.941771+0000","last_unstale":"2023-09-23T21:00:21.831786+0000","last_undegraded":"2023-09-23T21:00:21.831786+0000","last_fullsized":"2023-09-23T21:00:21.831786+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:48:25.588116+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":151,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-09-23T21:00:20.338736+0000","last_change":"2023-09-23T20:55:35.949780+0000","last_active":"2023-09-23T21:00:20.338736+0000","last_peered":"2023-09-23T21:00:20.338736+0000","last_clean":"2023-09-23T21:00:20.338736+0000","last_became_active":"2023-09-23T20:55:35.949497+0000","last_became_peered":"2023-09-23T20:55:35.949497+0000","last_unstale":"2023-09-23T21:00:20.338736+0000","last_undegraded":"2023-09-23T21:00:20.338736+0000","last_fullsized":"2023-09-23T21:00:20.338736+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:09:37.420882+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":210,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-09-23T21:00:20.335216+0000","last_change":"2023-09-23T20:54:13.929196+0000","last_active":"2023-09-23T21:00:20.335216+0000","last_peered":"2023-09-23T21:00:20.335216+0000","last_clean":"2023-09-23T21:00:20.335216+0000","last_became_active":"2023-09-23T20:54:13.928831+0000","last_became_peered":"2023-09-23T20:54:13.928831+0000","last_unstale":"2023-09-23T21:00:20.335216+0000","last_undegraded":"2023-09-23T21:00:20.335216+0000","last_fullsized":"2023-09-23T21:00:20.335216+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:28:44.316851+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":215,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-09-23T21:00:20.347236+0000","last_change":"2023-09-23T20:52:12.855430+0000","last_active":"2023-09-23T21:00:20.347236+0000","last_peered":"2023-09-23T21:00:20.347236+0000","last_clean":"2023-09-23T21:00:20.347236+0000","last_became_active":"2023-09-23T20:52:12.855115+0000","last_became_peered":"2023-09-23T20:52:12.855115+0000","last_unstale":"2023-09-23T21:00:20.347236+0000","last_undegraded":"2023-09-23T21:00:20.347236+0000","last_fullsized":"2023-09-23T21:00:20.347236+0000","mapping_epoch":100,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":101,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:33:23.802214+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":124,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-09-23T21:00:20.347400+0000","last_change":"2023-09-23T20:54:31.902946+0000","last_active":"2023-09-23T21:00:20.347400+0000","last_peered":"2023-09-23T21:00:20.347400+0000","last_clean":"2023-09-23T21:00:20.347400+0000","last_became_active":"2023-09-23T20:54:31.889484+0000","last_became_peered":"2023-09-23T20:54:31.889484+0000","last_unstale":"2023-09-23T21:00:20.347400+0000","last_undegraded":"2023-09-23T21:00:20.347400+0000","last_fullsized":"2023-09-23T21:00:20.347400+0000","mapping_epoch":136,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:44:44.278398+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":124,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-09-23T21:00:20.337086+0000","last_change":"2023-09-23T20:54:13.950351+0000","last_active":"2023-09-23T21:00:20.337086+0000","last_peered":"2023-09-23T21:00:20.337086+0000","last_clean":"2023-09-23T21:00:20.337086+0000","last_became_active":"2023-09-23T20:54:13.949987+0000","last_became_peered":"2023-09-23T20:54:13.949987+0000","last_unstale":"2023-09-23T21:00:20.337086+0000","last_undegraded":"2023-09-23T21:00:20.337086+0000","last_fullsized":"2023-09-23T21:00:20.337086+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:46:17.765134+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":155,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-09-23T21:00:20.347709+0000","last_change":"2023-09-23T20:56:18.141728+0000","last_active":"2023-09-23T21:00:20.347709+0000","last_peered":"2023-09-23T21:00:20.347709+0000","last_clean":"2023-09-23T21:00:20.347709+0000","last_became_active":"2023-09-23T20:56:18.137979+0000","last_became_peered":"2023-09-23T20:56:18.137979+0000","last_unstale":"2023-09-23T21:00:20.347709+0000","last_undegraded":"2023-09-23T21:00:20.347709+0000","last_fullsized":"2023-09-23T21:00:20.347709+0000","mapping_epoch":192,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":193,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:58:05.658443+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":124,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-09-23T21:00:20.336721+0000","last_change":"2023-09-23T20:54:13.935307+0000","last_active":"2023-09-23T21:00:20.336721+0000","last_peered":"2023-09-23T21:00:20.336721+0000","last_clean":"2023-09-23T21:00:20.336721+0000","last_became_active":"2023-09-23T20:54:13.935000+0000","last_became_peered":"2023-09-23T20:54:13.935000+0000","last_unstale":"2023-09-23T21:00:20.336721+0000","last_undegraded":"2023-09-23T21:00:20.336721+0000","last_fullsized":"2023-09-23T21:00:20.336721+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:47:40.974082+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":234,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-09-23T21:00:20.343224+0000","last_change":"2023-09-23T20:55:35.954870+0000","last_active":"2023-09-23T21:00:20.343224+0000","last_peered":"2023-09-23T21:00:20.343224+0000","last_clean":"2023-09-23T21:00:20.343224+0000","last_became_active":"2023-09-23T20:55:35.954568+0000","last_became_peered":"2023-09-23T20:55:35.954568+0000","last_unstale":"2023-09-23T21:00:20.343224+0000","last_undegraded":"2023-09-23T21:00:20.343224+0000","last_fullsized":"2023-09-23T21:00:20.343224+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:06:22.423566+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":233,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-09-23T21:00:20.362177+0000","last_change":"2023-09-23T20:51:10.838334+0000","last_active":"2023-09-23T21:00:20.362177+0000","last_peered":"2023-09-23T21:00:20.362177+0000","last_clean":"2023-09-23T21:00:20.362177+0000","last_became_active":"2023-09-23T20:51:10.838026+0000","last_became_peered":"2023-09-23T20:51:10.838026+0000","last_unstale":"2023-09-23T21:00:20.362177+0000","last_undegraded":"2023-09-23T21:00:20.362177+0000","last_fullsized":"2023-09-23T21:00:20.362177+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:04:37.009457+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":238,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-09-23T21:00:20.333791+0000","last_change":"2023-09-23T20:54:31.903139+0000","last_active":"2023-09-23T21:00:20.333791+0000","last_peered":"2023-09-23T21:00:20.333791+0000","last_clean":"2023-09-23T21:00:20.333791+0000","last_became_active":"2023-09-23T20:54:31.884520+0000","last_became_peered":"2023-09-23T20:54:31.884520+0000","last_unstale":"2023-09-23T21:00:20.333791+0000","last_undegraded":"2023-09-23T21:00:20.333791+0000","last_fullsized":"2023-09-23T21:00:20.333791+0000","mapping_epoch":136,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:39:34.269705+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1108,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":141,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-09-23T21:00:20.336576+0000","last_change":"2023-09-23T20:55:35.951572+0000","last_active":"2023-09-23T21:00:20.336576+0000","last_peered":"2023-09-23T21:00:20.336576+0000","last_clean":"2023-09-23T21:00:20.336576+0000","last_became_active":"2023-09-23T20:55:35.951294+0000","last_became_peered":"2023-09-23T20:55:35.951294+0000","last_unstale":"2023-09-23T21:00:20.336576+0000","last_undegraded":"2023-09-23T21:00:20.336576+0000","last_fullsized":"2023-09-23T21:00:20.336576+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:36:32.447210+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":7,"num_pg":16,"stat_sum":{"num_bytes":1048248344,"num_objects":15996,"num_object_clones":0,"num_object_copies":47988,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":11892,"num_objects_misplaced":90,"num_objects_unfound":0,"num_objects_dirty":15996,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":187524,"num_write_kb":6512577,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4360,"num_bytes_recovered":285736960,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":4203556864,"data_stored":4203544648,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":119124,"ondisk_log_size":119124,"up":48,"acting":45,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":22,"num_bytes_recovered":5990,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":21,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3381,"ondisk_log_size":3381,"up":63,"acting":63,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201521,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":527128,"kb_used_data":335760,"kb_used_omap":0,"kb_used_meta":191360,"kb_avail":93218024,"statfs":{"total":95995035648,"available":95455256576,"internally_reserved":0,"allocated":343818240,"data_stored":342938481,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":195952640},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:59:46 2023","interfaces":[{"interface":"back","average":{"1min":1.6559999999999999,"5min":1.802,"15min":1.0840000000000001},"min":{"1min":0.245,"5min":0.21099999999999999,"15min":0.21099999999999999},"max":{"1min":6.4420000000000002,"5min":43.994999999999997,"15min":43.994999999999997},"last":1.5900000000000001},{"interface":"front","average":{"1min":1.6339999999999999,"5min":1.9159999999999999,"15min":1.1699999999999999},"min":{"1min":0.318,"5min":0.28199999999999997,"15min":0.218},"max":{"1min":6.4610000000000003,"5min":44.206000000000003,"15min":44.206000000000003},"last":1.6040000000000001}]},{"osd":1,"last update":"Sat Sep 23 21:00:23 2023","interfaces":[{"interface":"back","average":{"1min":1.4419999999999999,"5min":1.6559999999999999,"15min":1.2829999999999999},"min":{"1min":0.28199999999999997,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":6.8330000000000002,"5min":10.622,"15min":22.591999999999999},"last":0.45500000000000002},{"interface":"front","average":{"1min":1.823,"5min":1.7110000000000001,"15min":1.171},"min":{"1min":0.26500000000000001,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":7.9249999999999998,"5min":10.097,"15min":21.437000000000001},"last":0.40200000000000002}]},{"osd":2,"last update":"Sat Sep 23 20:59:55 2023","interfaces":[{"interface":"back","average":{"1min":2.7149999999999999,"5min":2.2000000000000002,"15min":1.294},"min":{"1min":0.30399999999999999,"5min":0.26500000000000001,"15min":0.24399999999999999},"max":{"1min":8.5289999999999999,"5min":10.805999999999999,"15min":10.805999999999999},"last":1.5289999999999999},{"interface":"front","average":{"1min":2.339,"5min":2.2639999999999998,"15min":1.3089999999999999},"min":{"1min":0.26500000000000001,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":11.590999999999999,"5min":45.25,"15min":45.25},"last":0.56200000000000006}]},{"osd":3,"last update":"Sat Sep 23 20:59:30 2023","interfaces":[{"interface":"back","average":{"1min":2.6200000000000001,"5min":2.508,"15min":1.7250000000000001},"min":{"1min":0.33500000000000002,"5min":0.32100000000000001,"15min":0.314},"max":{"1min":8.4390000000000001,"5min":13.007,"15min":36.408000000000001},"last":2.5179999999999998},{"interface":"front","average":{"1min":2.7789999999999999,"5min":2.3079999999999998,"15min":1.552},"min":{"1min":0.41599999999999998,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":13.198,"5min":14.220000000000001,"15min":35.593000000000004},"last":2.5920000000000001}]},{"osd":4,"last update":"Sat Sep 23 21:00:23 2023","interfaces":[{"interface":"back","average":{"1min":0.76200000000000001,"5min":0.68100000000000005,"15min":0.627},"min":{"1min":0.27900000000000003,"5min":0.23799999999999999,"15min":0.23699999999999999},"max":{"1min":3.8929999999999998,"5min":4.5410000000000004,"15min":5.8920000000000003},"last":0.34300000000000003},{"interface":"front","average":{"1min":0.86799999999999999,"5min":0.72899999999999998,"15min":0.64300000000000002},"min":{"1min":0.34000000000000002,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":3.3999999999999999,"5min":4.9900000000000002,"15min":5.3200000000000003},"last":0.36399999999999999}]},{"osd":5,"last update":"Sat Sep 23 21:00:23 2023","interfaces":[{"interface":"back","average":{"1min":0.56999999999999995,"5min":0.69099999999999995,"15min":0.63500000000000001},"min":{"1min":0.28999999999999998,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":2.5249999999999999,"5min":7.0389999999999997,"15min":7.0389999999999997},"last":0.48699999999999999},{"interface":"front","average":{"1min":0.86899999999999999,"5min":0.752,"15min":0.63900000000000001},"min":{"1min":0.36899999999999999,"5min":0.29999999999999999,"15min":0.27800000000000002},"max":{"1min":3.8719999999999999,"5min":4.5999999999999996,"15min":4.5999999999999996},"last":0.52800000000000002}]},{"osd":6,"last update":"Sat Sep 23 20:59:42 2023","interfaces":[{"interface":"back","average":{"1min":0.48299999999999998,"5min":0.68000000000000005,"15min":0.67100000000000004},"min":{"1min":0.28899999999999998,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":0.77300000000000002,"5min":3.5249999999999999,"15min":3.5249999999999999},"last":0.624},{"interface":"front","average":{"1min":0.48499999999999999,"5min":0.749,"15min":0.78600000000000003},"min":{"1min":0.34699999999999998,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":0.95499999999999996,"5min":3.5409999999999999,"15min":3.5409999999999999},"last":0.84199999999999997}]}]},{"osd":6,"up_from":164,"seq":704374636605,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":439116,"kb_used_data":322116,"kb_used_omap":10,"kb_used_meta":116981,"kb_avail":93306036,"statfs":{"total":95995035648,"available":95545380864,"internally_reserved":0,"allocated":329846784,"data_stored":328978926,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":10328,"internal_metadata":119789480},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:59:45 2023","interfaces":[{"interface":"back","average":{"1min":1.1599999999999999,"5min":1.5,"15min":1.0760000000000001},"min":{"1min":0.23599999999999999,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":4.0960000000000001,"5min":8.7319999999999993,"15min":8.7319999999999993},"last":0.34599999999999997},{"interface":"front","average":{"1min":1.5029999999999999,"5min":1.6060000000000001,"15min":1.1899999999999999},"min":{"1min":0.23999999999999999,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":4.2869999999999999,"5min":6.7140000000000004,"15min":6.7140000000000004},"last":3.2759999999999998}]},{"osd":1,"last update":"Sat Sep 23 20:59:45 2023","interfaces":[{"interface":"back","average":{"1min":1.554,"5min":1.25,"15min":0.94199999999999995},"min":{"1min":0.27200000000000002,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":8.4309999999999992,"5min":8.4309999999999992,"15min":8.4309999999999992},"last":2.7349999999999999},{"interface":"front","average":{"1min":1.506,"5min":1.3720000000000001,"15min":1.036},"min":{"1min":0.26100000000000001,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":7.2069999999999999,"5min":10.324999999999999,"15min":10.324999999999999},"last":0.39500000000000002}]},{"osd":2,"last update":"Sat Sep 23 20:59:45 2023","interfaces":[{"interface":"back","average":{"1min":2.097,"5min":2.0720000000000001,"15min":1.788},"min":{"1min":0.312,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":6.0670000000000002,"5min":13.750999999999999,"15min":13.750999999999999},"last":2.7149999999999999},{"interface":"front","average":{"1min":2.46,"5min":1.8,"15min":1.3380000000000001},"min":{"1min":0.32000000000000001,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":5.6130000000000004,"5min":8.0890000000000004,"15min":8.0890000000000004},"last":4.6159999999999997}]},{"osd":3,"last update":"Sat Sep 23 20:59:45 2023","interfaces":[{"interface":"back","average":{"1min":1.768,"5min":1.415,"15min":1.216},"min":{"1min":0.45000000000000001,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":4.7220000000000004,"5min":6.9000000000000004,"15min":6.9000000000000004},"last":3.5619999999999998},{"interface":"front","average":{"1min":1.6479999999999999,"5min":1.7490000000000001,"15min":1.651},"min":{"1min":0.34999999999999998,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":4.6289999999999996,"5min":11.021000000000001,"15min":11.021000000000001},"last":3.2200000000000002}]},{"osd":4,"last update":"Sat Sep 23 20:59:45 2023","interfaces":[{"interface":"back","average":{"1min":0.68700000000000006,"5min":0.78600000000000003,"15min":0.96299999999999997},"min":{"1min":0.27100000000000002,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":2.3460000000000001,"5min":9.0079999999999991,"15min":9.0079999999999991},"last":0.36599999999999999},{"interface":"front","average":{"1min":0.48599999999999999,"5min":0.499,"15min":0.52000000000000002},"min":{"1min":0.29899999999999999,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":1.115,"5min":2.0059999999999998,"15min":2.0059999999999998},"last":0.504}]},{"osd":5,"last update":"Sat Sep 23 20:59:45 2023","interfaces":[{"interface":"back","average":{"1min":0.78100000000000003,"5min":0.61699999999999999,"15min":0.56599999999999995},"min":{"1min":0.28100000000000003,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":2.3730000000000002,"5min":2.8239999999999998,"15min":2.8239999999999998},"last":0.56200000000000006},{"interface":"front","average":{"1min":0.58299999999999996,"5min":0.73999999999999999,"15min":0.78600000000000003},"min":{"1min":0.34000000000000002,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":1.2290000000000001,"5min":3.1379999999999999,"15min":3.1379999999999999},"last":0.63}]},{"osd":7,"last update":"Sat Sep 23 20:59:45 2023","interfaces":[{"interface":"back","average":{"1min":0.51700000000000002,"5min":0.77200000000000002,"15min":0.876},"min":{"1min":0.318,"5min":0.312,"15min":0.312},"max":{"1min":1.1819999999999999,"5min":7.8570000000000002,"15min":7.8570000000000002},"last":0.54100000000000004},{"interface":"front","average":{"1min":0.46500000000000002,"5min":0.64900000000000002,"15min":0.56899999999999995},"min":{"1min":0.33800000000000002,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":0.63200000000000001,"5min":8.282,"15min":8.282},"last":0.54800000000000004}]}]},{"osd":1,"up_from":88,"seq":377957122176,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":991832,"kb_used_data":724688,"kb_used_omap":9,"kb_used_meta":267126,"kb_avail":92753320,"statfs":{"total":95995035648,"available":94979399680,"internally_reserved":0,"allocated":742080512,"data_stored":740854671,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9239,"internal_metadata":273538025},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:00:26 2023","interfaces":[{"interface":"back","average":{"1min":2.9870000000000001,"5min":3.048,"15min":1.7110000000000001},"min":{"1min":0.26000000000000001,"5min":0.20699999999999999,"15min":0.20699999999999999},"max":{"1min":8.8000000000000007,"5min":16.646999999999998,"15min":16.646999999999998},"last":0.218},{"interface":"front","average":{"1min":4.7480000000000002,"5min":3.577,"15min":1.9690000000000001},"min":{"1min":0.38200000000000001,"5min":0.246,"15min":0.22500000000000001},"max":{"1min":27.797999999999998,"5min":27.797999999999998,"15min":27.797999999999998},"last":0.44500000000000001}]},{"osd":2,"last update":"Sat Sep 23 20:59:55 2023","interfaces":[{"interface":"back","average":{"1min":2.4809999999999999,"5min":2.3210000000000002,"15min":1.319},"min":{"1min":0.222,"5min":0.221,"15min":0.20300000000000001},"max":{"1min":13.752000000000001,"5min":14.335000000000001,"15min":14.335000000000001},"last":0.251},{"interface":"front","average":{"1min":3.4470000000000001,"5min":3.3519999999999999,"15min":1.948},"min":{"1min":0.27400000000000002,"5min":0.248,"15min":0.248},"max":{"1min":18.026,"5min":18.026,"15min":38.951000000000001},"last":0.499}]},{"osd":3,"last update":"Sat Sep 23 21:00:26 2023","interfaces":[{"interface":"back","average":{"1min":3.8849999999999998,"5min":2.8220000000000001,"15min":1.794},"min":{"1min":0.38500000000000001,"5min":0.246,"15min":0.246},"max":{"1min":22.686,"5min":22.686,"15min":25.314},"last":0.26500000000000001},{"interface":"front","average":{"1min":3.4510000000000001,"5min":2.605,"15min":1.613},"min":{"1min":0.36699999999999999,"5min":0.20599999999999999,"15min":0.20599999999999999},"max":{"1min":14.473000000000001,"5min":14.473000000000001,"15min":19.881},"last":0.51500000000000001}]},{"osd":4,"last update":"Sat Sep 23 21:00:26 2023","interfaces":[{"interface":"back","average":{"1min":2.383,"5min":1.704,"15min":1.329},"min":{"1min":0.34499999999999997,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":18.231000000000002,"5min":18.231000000000002,"15min":18.231000000000002},"last":0.311},{"interface":"front","average":{"1min":1.6080000000000001,"5min":1.55,"15min":1.1419999999999999},"min":{"1min":0.35399999999999998,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":6.6360000000000001,"5min":7.532,"15min":11.234999999999999},"last":0.54500000000000004}]},{"osd":5,"last update":"Sat Sep 23 21:00:26 2023","interfaces":[{"interface":"back","average":{"1min":2.6080000000000001,"5min":2.2999999999999998,"15min":1.419},"min":{"1min":0.34999999999999998,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":12.699999999999999,"5min":15.041,"15min":15.041},"last":6.7750000000000004},{"interface":"front","average":{"1min":2.415,"5min":1.762,"15min":1.357},"min":{"1min":0.40000000000000002,"5min":0.32600000000000001,"15min":0.32200000000000001},"max":{"1min":18.280999999999999,"5min":18.280999999999999,"15min":18.280999999999999},"last":0.35999999999999999}]},{"osd":6,"last update":"Sat Sep 23 20:59:41 2023","interfaces":[{"interface":"back","average":{"1min":2.266,"5min":1.5609999999999999,"15min":1.397},"min":{"1min":0.46200000000000002,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":10.159000000000001,"5min":10.673999999999999,"15min":10.673999999999999},"last":0.38600000000000001},{"interface":"front","average":{"1min":1.8660000000000001,"5min":1.6299999999999999,"15min":1.3779999999999999},"min":{"1min":0.33700000000000002,"5min":0.309,"15min":0.309},"max":{"1min":7.6779999999999999,"5min":8.7530000000000001,"15min":8.7530000000000001},"last":0.56799999999999995}]},{"osd":7,"last update":"Sat Sep 23 21:00:26 2023","interfaces":[{"interface":"back","average":{"1min":1.8180000000000001,"5min":1.7829999999999999,"15min":1.24},"min":{"1min":0.41699999999999998,"5min":0.36199999999999999,"15min":0.35699999999999998},"max":{"1min":6.657,"5min":8.7769999999999992,"15min":11.186999999999999},"last":0.58999999999999997},{"interface":"front","average":{"1min":2.415,"5min":2.319,"15min":1.4399999999999999},"min":{"1min":0.41499999999999998,"5min":0.40899999999999997,"15min":0.34599999999999997},"max":{"1min":12.84,"5min":15.098000000000001,"15min":15.098000000000001},"last":1.536}]}]},{"osd":0,"up_from":83,"seq":356482285702,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":741556,"kb_used_data":539116,"kb_used_omap":28,"kb_used_meta":202211,"kb_avail":93003596,"statfs":{"total":95995035648,"available":95235682304,"internally_reserved":0,"allocated":552054784,"data_stored":550862273,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":28848,"internal_metadata":207064912},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 21:00:20 2023","interfaces":[{"interface":"back","average":{"1min":6.4480000000000004,"5min":3.7120000000000002,"15min":1.893},"min":{"1min":0.20000000000000001,"5min":0.191,"15min":0.191},"max":{"1min":19.146000000000001,"5min":19.146000000000001,"15min":19.146000000000001},"last":0.39900000000000002},{"interface":"front","average":{"1min":5.4379999999999997,"5min":3.6789999999999998,"15min":1.8759999999999999},"min":{"1min":0.39500000000000002,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":11.074,"5min":15.771000000000001,"15min":17.704999999999998},"last":1.069}]},{"osd":2,"last update":"Sat Sep 23 20:59:51 2023","interfaces":[{"interface":"back","average":{"1min":3.8879999999999999,"5min":3.4180000000000001,"15min":1.867},"min":{"1min":0.50900000000000001,"5min":0.28299999999999997,"15min":0.23799999999999999},"max":{"1min":21.053000000000001,"5min":21.053000000000001,"15min":21.053000000000001},"last":3.4430000000000001},{"interface":"front","average":{"1min":3.6579999999999999,"5min":3.7450000000000001,"15min":1.944},"min":{"1min":0.26300000000000001,"5min":0.25800000000000001,"15min":0.21099999999999999},"max":{"1min":13.956,"5min":21.747,"15min":21.747},"last":3.472}]},{"osd":3,"last update":"Sat Sep 23 21:00:20 2023","interfaces":[{"interface":"back","average":{"1min":6.532,"5min":3.8690000000000002,"15min":1.9990000000000001},"min":{"1min":1.052,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":15.082000000000001,"5min":21.184000000000001,"15min":21.184000000000001},"last":0.42999999999999999},{"interface":"front","average":{"1min":4.9050000000000002,"5min":4.008,"15min":2.0899999999999999},"min":{"1min":0.48499999999999999,"5min":0.28000000000000003,"15min":0.23300000000000001},"max":{"1min":9.7560000000000002,"5min":21.167000000000002,"15min":21.167000000000002},"last":1.8520000000000001}]},{"osd":4,"last update":"Sat Sep 23 20:59:51 2023","interfaces":[{"interface":"back","average":{"1min":1.9359999999999999,"5min":2.823,"15min":1.5589999999999999},"min":{"1min":0.38500000000000001,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":7.931,"5min":21.594999999999999,"15min":21.594999999999999},"last":1.8919999999999999},{"interface":"front","average":{"1min":2.8639999999999999,"5min":1.9830000000000001,"15min":1.278},"min":{"1min":0.41099999999999998,"5min":0.31,"15min":0.28100000000000003},"max":{"1min":13.173999999999999,"5min":13.173999999999999,"15min":18.724},"last":0.45600000000000002}]},{"osd":5,"last update":"Sat Sep 23 20:59:51 2023","interfaces":[{"interface":"back","average":{"1min":1.859,"5min":1.889,"15min":1.258},"min":{"1min":0.38700000000000001,"5min":0.318,"15min":0.318},"max":{"1min":6.9980000000000002,"5min":8.8970000000000002,"15min":8.8970000000000002},"last":1.1739999999999999},{"interface":"front","average":{"1min":1.748,"5min":2.7290000000000001,"15min":1.5649999999999999},"min":{"1min":0.41899999999999998,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":7.9820000000000002,"5min":20.512,"15min":20.512},"last":1.9179999999999999}]},{"osd":6,"last update":"Sat Sep 23 20:59:46 2023","interfaces":[{"interface":"back","average":{"1min":2.9180000000000001,"5min":2.145,"15min":1.819},"min":{"1min":0.45900000000000002,"5min":0.37,"15min":0.37},"max":{"1min":13.134,"5min":13.134,"15min":13.134},"last":0.48199999999999998},{"interface":"front","average":{"1min":2.9140000000000001,"5min":2.198,"15min":1.8959999999999999},"min":{"1min":0.43099999999999999,"5min":0.377,"15min":0.377},"max":{"1min":13.254,"5min":13.254,"15min":13.254},"last":0.503}]},{"osd":7,"last update":"Sat Sep 23 20:59:51 2023","interfaces":[{"interface":"back","average":{"1min":2.1520000000000001,"5min":2.0139999999999998,"15min":1.3169999999999999},"min":{"1min":0.433,"5min":0.379,"15min":0.379},"max":{"1min":6.9699999999999998,"5min":8.7729999999999997,"15min":8.7729999999999997},"last":1.2010000000000001},{"interface":"front","average":{"1min":3.0459999999999998,"5min":2.105,"15min":1.3200000000000001},"min":{"1min":0.44600000000000001,"5min":0.377,"15min":0.377},"max":{"1min":13.218999999999999,"5min":13.218999999999999,"15min":18.673999999999999},"last":0.999}]}]},{"osd":2,"up_from":93,"seq":399431958649,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1069600,"kb_used_data":755992,"kb_used_omap":81,"kb_used_meta":313454,"kb_avail":92675552,"statfs":{"total":95995035648,"available":94899765248,"internally_reserved":0,"allocated":774135808,"data_stored":772959752,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":83750,"internal_metadata":320977114},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:59:50 2023","interfaces":[{"interface":"back","average":{"1min":4.8730000000000002,"5min":2.7000000000000002,"15min":1.3799999999999999},"min":{"1min":0.189,"5min":0.17999999999999999,"15min":0.17999999999999999},"max":{"1min":29.498000000000001,"5min":29.498000000000001,"15min":29.498000000000001},"last":7.0199999999999996},{"interface":"front","average":{"1min":4.032,"5min":3.3370000000000002,"15min":1.6739999999999999},"min":{"1min":0.20599999999999999,"5min":0.20599999999999999,"15min":0.20599999999999999},"max":{"1min":17.251000000000001,"5min":17.251000000000001,"15min":17.251000000000001},"last":6.1799999999999997}]},{"osd":1,"last update":"Sat Sep 23 20:59:50 2023","interfaces":[{"interface":"back","average":{"1min":2.6030000000000002,"5min":2.5619999999999998,"15min":1.625},"min":{"1min":0.25700000000000001,"5min":0.20200000000000001,"15min":0.20200000000000001},"max":{"1min":7.9800000000000004,"5min":14.709,"15min":16.081},"last":4.1669999999999998},{"interface":"front","average":{"1min":3.427,"5min":3.2570000000000001,"15min":1.97},"min":{"1min":0.26600000000000001,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":9.9090000000000007,"5min":11.717000000000001,"15min":16.041},"last":6.1529999999999996}]},{"osd":3,"last update":"Sat Sep 23 21:00:31 2023","interfaces":[{"interface":"back","average":{"1min":4.71,"5min":3.8450000000000002,"15min":2.1800000000000002},"min":{"1min":0.28599999999999998,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":10.468,"5min":27.614999999999998,"15min":27.614999999999998},"last":6.1109999999999998},{"interface":"front","average":{"1min":4.2939999999999996,"5min":4.0309999999999997,"15min":2.4540000000000002},"min":{"1min":0.64400000000000002,"5min":0.23899999999999999,"15min":0.221},"max":{"1min":11.673999999999999,"5min":27.562000000000001,"15min":27.562000000000001},"last":5.0469999999999997}]},{"osd":4,"last update":"Sat Sep 23 20:59:50 2023","interfaces":[{"interface":"back","average":{"1min":1.575,"5min":1.7749999999999999,"15min":1.2070000000000001},"min":{"1min":0.36499999999999999,"5min":0.28699999999999998,"15min":0.26400000000000001},"max":{"1min":9.8659999999999997,"5min":9.8659999999999997,"15min":9.8659999999999997},"last":0.45000000000000001},{"interface":"front","average":{"1min":2.8439999999999999,"5min":2.1880000000000002,"15min":1.3720000000000001},"min":{"1min":0.32400000000000001,"5min":0.313,"15min":0.313},"max":{"1min":9.9719999999999995,"5min":10.609,"15min":10.609},"last":0.80200000000000005}]},{"osd":5,"last update":"Sat Sep 23 20:59:50 2023","interfaces":[{"interface":"back","average":{"1min":1.4179999999999999,"5min":1.99,"15min":1.298},"min":{"1min":0.33000000000000002,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":6.3570000000000002,"5min":9.0329999999999995,"15min":9.7629999999999999},"last":0.497},{"interface":"front","average":{"1min":2.5760000000000001,"5min":2.1419999999999999,"15min":1.3720000000000001},"min":{"1min":0.35599999999999998,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":9.9220000000000006,"5min":10.574,"15min":10.574},"last":0.74299999999999999}]},{"osd":6,"last update":"Sat Sep 23 20:59:45 2023","interfaces":[{"interface":"back","average":{"1min":2.9060000000000001,"5min":2.0640000000000001,"15min":1.8},"min":{"1min":0.35699999999999998,"5min":0.312,"15min":0.312},"max":{"1min":10.012,"5min":14.244999999999999,"15min":14.244999999999999},"last":0.77500000000000002},{"interface":"front","average":{"1min":2.8740000000000001,"5min":1.8129999999999999,"15min":1.2909999999999999},"min":{"1min":0.41599999999999998,"5min":0.374,"15min":0.374},"max":{"1min":10.032999999999999,"5min":10.032999999999999,"15min":10.032999999999999},"last":0.82699999999999996}]},{"osd":7,"last update":"Sat Sep 23 20:59:50 2023","interfaces":[{"interface":"back","average":{"1min":1.3859999999999999,"5min":1.4830000000000001,"15min":1.0449999999999999},"min":{"1min":0.35899999999999999,"5min":0.315,"15min":0.315},"max":{"1min":5.9199999999999999,"5min":8.0820000000000007,"15min":8.0820000000000007},"last":4.1269999999999998},{"interface":"front","average":{"1min":1.121,"5min":1.873,"15min":1.292},"min":{"1min":0.443,"5min":0.35099999999999998,"15min":0.34499999999999997},"max":{"1min":6.3899999999999997,"5min":8.6219999999999999,"15min":9.7859999999999996},"last":0.53000000000000003}]}]},{"osd":3,"up_from":98,"seq":420906795123,"num_pgs":49,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":907716,"kb_used_data":543996,"kb_used_omap":7,"kb_used_meta":363640,"kb_avail":92837436,"statfs":{"total":95995035648,"available":95065534464,"internally_reserved":0,"allocated":557051904,"data_stored":555901014,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7576,"internal_metadata":372367976},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:00:28 2023","interfaces":[{"interface":"back","average":{"1min":5.6130000000000004,"5min":3.0470000000000002,"15min":2.0840000000000001},"min":{"1min":0.20399999999999999,"5min":0.20399999999999999,"15min":0.20399999999999999},"max":{"1min":27.962,"5min":27.962,"15min":27.962},"last":11.398},{"interface":"front","average":{"1min":4.984,"5min":3.5169999999999999,"15min":1.9339999999999999},"min":{"1min":0.21099999999999999,"5min":0.19800000000000001,"15min":0.19800000000000001},"max":{"1min":28.058,"5min":28.058,"15min":28.058},"last":0.35399999999999998}]},{"osd":1,"last update":"Sat Sep 23 21:00:28 2023","interfaces":[{"interface":"back","average":{"1min":4.8730000000000002,"5min":3.1760000000000002,"15min":1.9119999999999999},"min":{"1min":0.22600000000000001,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":22.681999999999999,"5min":22.681999999999999,"15min":22.681999999999999},"last":5.0899999999999999},{"interface":"front","average":{"1min":4.0789999999999997,"5min":2.5169999999999999,"15min":1.929},"min":{"1min":0.25,"5min":0.218,"15min":0.218},"max":{"1min":19.329000000000001,"5min":19.329000000000001,"15min":25.353000000000002},"last":11.446}]},{"osd":2,"last update":"Sat Sep 23 21:00:28 2023","interfaces":[{"interface":"back","average":{"1min":6.1680000000000001,"5min":3.8039999999999998,"15min":2.113},"min":{"1min":0.36199999999999999,"5min":0.33000000000000002,"15min":0.30099999999999999},"max":{"1min":28.251999999999999,"5min":28.251999999999999,"15min":28.251999999999999},"last":3.6030000000000002},{"interface":"front","average":{"1min":4.6779999999999999,"5min":2.9300000000000002,"15min":2.0750000000000002},"min":{"1min":0.372,"5min":0.23999999999999999,"15min":0.22600000000000001},"max":{"1min":16.213999999999999,"5min":16.213999999999999,"15min":25.305},"last":11.534000000000001}]},{"osd":4,"last update":"Sat Sep 23 21:00:28 2023","interfaces":[{"interface":"back","average":{"1min":3.258,"5min":2.2869999999999999,"15min":1.4750000000000001},"min":{"1min":0.314,"5min":0.29899999999999999,"15min":0.29299999999999998},"max":{"1min":18.239000000000001,"5min":18.239000000000001,"15min":18.239000000000001},"last":5.1900000000000004},{"interface":"front","average":{"1min":2.7200000000000002,"5min":1.5860000000000001,"15min":1.474},"min":{"1min":0.317,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":14.808,"5min":14.808,"15min":22.853000000000002},"last":7.7140000000000004}]},{"osd":5,"last update":"Sat Sep 23 21:00:28 2023","interfaces":[{"interface":"back","average":{"1min":3.2930000000000001,"5min":2.3399999999999999,"15min":1.4690000000000001},"min":{"1min":0.33200000000000002,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":18.324999999999999,"5min":18.324999999999999,"15min":18.324999999999999},"last":3.6709999999999998},{"interface":"front","average":{"1min":3.0910000000000002,"5min":2.363,"15min":1.4359999999999999},"min":{"1min":0.33700000000000002,"5min":0.33300000000000002,"15min":0.33100000000000002},"max":{"1min":11.327,"5min":13.43,"15min":13.43},"last":0.38500000000000001}]},{"osd":6,"last update":"Sat Sep 23 20:59:47 2023","interfaces":[{"interface":"back","average":{"1min":2.0579999999999998,"5min":1.415,"15min":1.105},"min":{"1min":0.33700000000000002,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":13.499000000000001,"5min":13.499000000000001,"15min":13.499000000000001},"last":7.6669999999999998},{"interface":"front","average":{"1min":3.8410000000000002,"5min":1.966,"15min":1.5960000000000001},"min":{"1min":0.55300000000000005,"5min":0.38700000000000001,"15min":0.38700000000000001},"max":{"1min":13.398999999999999,"5min":13.398999999999999,"15min":13.398999999999999},"last":5.6769999999999996}]},{"osd":7,"last update":"Sat Sep 23 21:00:28 2023","interfaces":[{"interface":"back","average":{"1min":3.2879999999999998,"5min":2.5649999999999999,"15min":1.6399999999999999},"min":{"1min":0.39200000000000002,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":18.289999999999999,"5min":18.289999999999999,"15min":18.289999999999999},"last":3.6349999999999998},{"interface":"front","average":{"1min":3.1219999999999999,"5min":2.399,"15min":1.506},"min":{"1min":0.38900000000000001,"5min":0.38900000000000001,"15min":0.379},"max":{"1min":11.378,"5min":13.355,"15min":13.355},"last":1.9139999999999999}]}]},{"osd":4,"up_from":31,"seq":133143986430,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":475032,"kb_used_data":333072,"kb_used_omap":0,"kb_used_meta":141952,"kb_avail":93270120,"statfs":{"total":95995035648,"available":95508602880,"internally_reserved":0,"allocated":341065728,"data_stored":340186222,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":145358848},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:59:53 2023","interfaces":[{"interface":"back","average":{"1min":2.1469999999999998,"5min":1.915,"15min":1.143},"min":{"1min":0.255,"5min":0.214,"15min":0.214},"max":{"1min":9.0749999999999993,"5min":11.784000000000001,"15min":11.784000000000001},"last":3.536},{"interface":"front","average":{"1min":2.4969999999999999,"5min":2.0329999999999999,"15min":1.335},"min":{"1min":0.32800000000000001,"5min":0.26700000000000002,"15min":0.25800000000000001},"max":{"1min":9.1029999999999998,"5min":11.766,"15min":11.766},"last":3.5219999999999998}]},{"osd":1,"last update":"Sat Sep 23 21:00:22 2023","interfaces":[{"interface":"back","average":{"1min":1.4350000000000001,"5min":1.673,"15min":1.1779999999999999},"min":{"1min":0.25800000000000001,"5min":0.246,"15min":0.23599999999999999},"max":{"1min":5.5570000000000004,"5min":10.153,"15min":10.153},"last":3.2759999999999998},{"interface":"front","average":{"1min":2.6040000000000001,"5min":2.2749999999999999,"15min":1.2749999999999999},"min":{"1min":0.29199999999999998,"5min":0.26100000000000001,"15min":0.23200000000000001},"max":{"1min":10.882999999999999,"5min":12.292999999999999,"15min":12.292999999999999},"last":0.86799999999999999}]},{"osd":2,"last update":"Sat Sep 23 20:59:53 2023","interfaces":[{"interface":"back","average":{"1min":2.573,"5min":1.7430000000000001,"15min":1.2410000000000001},"min":{"1min":0.33600000000000002,"5min":0.311,"15min":0.29099999999999998},"max":{"1min":8.1509999999999998,"5min":8.1509999999999998,"15min":15.288},"last":0.39200000000000002},{"interface":"front","average":{"1min":2.8330000000000002,"5min":1.831,"15min":1.3999999999999999},"min":{"1min":0.28599999999999998,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":12.855,"5min":12.855,"15min":15.262},"last":0.33600000000000002}]},{"osd":3,"last update":"Sat Sep 23 21:00:22 2023","interfaces":[{"interface":"back","average":{"1min":3.5569999999999999,"5min":2.3479999999999999,"15min":1.8120000000000001},"min":{"1min":0.39200000000000002,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":8.5389999999999997,"5min":14.824999999999999,"15min":31.562999999999999},"last":1.1060000000000001},{"interface":"front","average":{"1min":3.5129999999999999,"5min":2.4550000000000001,"15min":1.8740000000000001},"min":{"1min":0.48399999999999999,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":8.5570000000000004,"5min":14.839,"15min":31.574000000000002},"last":1.0880000000000001}]},{"osd":5,"last update":"Sat Sep 23 20:59:36 2023","interfaces":[{"interface":"back","average":{"1min":0.41099999999999998,"5min":0.61099999999999999,"15min":0.52800000000000002},"min":{"1min":0.28899999999999998,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":0.90700000000000003,"5min":3.3279999999999998,"15min":3.3279999999999998},"last":0.308},{"interface":"front","average":{"1min":1.002,"5min":0.77800000000000002,"15min":0.61499999999999999},"min":{"1min":0.29999999999999999,"5min":0.251,"15min":0.251},"max":{"1min":3.4830000000000001,"5min":7.7619999999999996,"15min":7.7619999999999996},"last":0.29599999999999999}]},{"osd":6,"last update":"Sat Sep 23 20:59:46 2023","interfaces":[{"interface":"back","average":{"1min":0.62,"5min":0.70399999999999996,"15min":0.55700000000000005},"min":{"1min":0.32100000000000001,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":3.4180000000000001,"5min":4.6909999999999998,"15min":4.6909999999999998},"last":0.27300000000000002},{"interface":"front","average":{"1min":0.752,"5min":0.73699999999999999,"15min":0.67500000000000004},"min":{"1min":0.34499999999999997,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":3.2999999999999998,"5min":3.456,"15min":3.456},"last":0.373}]},{"osd":7,"last update":"Sat Sep 23 21:00:22 2023","interfaces":[{"interface":"back","average":{"1min":0.52100000000000002,"5min":0.754,"15min":0.69099999999999995},"min":{"1min":0.33400000000000002,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":2.302,"5min":6.2240000000000002,"15min":7.383},"last":0.35399999999999998},{"interface":"front","average":{"1min":0.56100000000000005,"5min":0.77500000000000002,"15min":0.67700000000000005},"min":{"1min":0.32700000000000001,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":2.4420000000000002,"5min":7.8310000000000004,"15min":7.8310000000000004},"last":0.36299999999999999}]}]},{"osd":5,"up_from":39,"seq":167503724795,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":759756,"kb_used_data":624772,"kb_used_omap":0,"kb_used_meta":134976,"kb_avail":92985396,"statfs":{"total":95995035648,"available":95217045504,"internally_reserved":0,"allocated":639766528,"data_stored":638895598,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":138215424},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:59:38 2023","interfaces":[{"interface":"back","average":{"1min":4.0339999999999998,"5min":1.619,"15min":0.96299999999999997},"min":{"1min":0.27400000000000002,"5min":0.20399999999999999,"15min":0.20399999999999999},"max":{"1min":12.795999999999999,"5min":12.795999999999999,"15min":12.795999999999999},"last":2.7269999999999999},{"interface":"front","average":{"1min":2.7330000000000001,"5min":1.948,"15min":1.125},"min":{"1min":0.24099999999999999,"5min":0.223,"15min":0.223},"max":{"1min":16.219999999999999,"5min":16.219999999999999,"15min":16.219999999999999},"last":0.27000000000000002}]},{"osd":1,"last update":"Sat Sep 23 21:00:18 2023","interfaces":[{"interface":"back","average":{"1min":2.8759999999999999,"5min":1.8779999999999999,"15min":1.2529999999999999},"min":{"1min":0.26600000000000001,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":8.7720000000000002,"5min":8.7720000000000002,"15min":8.7720000000000002},"last":1.1950000000000001},{"interface":"front","average":{"1min":2.3639999999999999,"5min":1.9330000000000001,"15min":1.22},"min":{"1min":0.34200000000000003,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":8.5109999999999992,"5min":10.285,"15min":10.285},"last":4.4089999999999998}]},{"osd":2,"last update":"Sat Sep 23 20:59:54 2023","interfaces":[{"interface":"back","average":{"1min":2.6819999999999999,"5min":1.6699999999999999,"15min":1.121},"min":{"1min":0.27400000000000002,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":10.371,"5min":10.579000000000001,"15min":12.651},"last":4.6420000000000003},{"interface":"front","average":{"1min":3.423,"5min":2.258,"15min":1.274},"min":{"1min":0.28799999999999998,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":10.333,"5min":12.300000000000001,"15min":12.300000000000001},"last":2.9620000000000002}]},{"osd":3,"last update":"Sat Sep 23 21:00:18 2023","interfaces":[{"interface":"back","average":{"1min":4.931,"5min":2.859,"15min":1.6379999999999999},"min":{"1min":0.35999999999999999,"5min":0.35399999999999998,"15min":0.28599999999999998},"max":{"1min":20.788,"5min":20.788,"15min":20.788},"last":1.6950000000000001},{"interface":"front","average":{"1min":3.9870000000000001,"5min":2.177,"15min":1.2949999999999999},"min":{"1min":0.39100000000000001,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":20.539000000000001,"5min":20.539000000000001,"15min":20.539000000000001},"last":0.35299999999999998}]},{"osd":4,"last update":"Sat Sep 23 20:59:34 2023","interfaces":[{"interface":"back","average":{"1min":0.81699999999999995,"5min":0.63300000000000001,"15min":0.55300000000000005},"min":{"1min":0.32100000000000001,"5min":0.252,"15min":0.252},"max":{"1min":2.9169999999999998,"5min":3.8479999999999999,"15min":3.8479999999999999},"last":0.29999999999999999},{"interface":"front","average":{"1min":0.83499999999999996,"5min":0.67500000000000004,"15min":0.57499999999999996},"min":{"1min":0.27400000000000002,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":3.3530000000000002,"5min":3.3530000000000002,"15min":3.3530000000000002},"last":0.28999999999999998}]},{"osd":6,"last update":"Sat Sep 23 20:59:38 2023","interfaces":[{"interface":"back","average":{"1min":0.86099999999999999,"5min":0.65100000000000002,"15min":0.59099999999999997},"min":{"1min":0.29199999999999998,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":4.1120000000000001,"5min":4.1120000000000001,"15min":4.1120000000000001},"last":0.314},{"interface":"front","average":{"1min":1.0269999999999999,"5min":0.755,"15min":0.747},"min":{"1min":0.36899999999999999,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":2.9620000000000002,"5min":3.6560000000000001,"15min":3.6560000000000001},"last":0.36899999999999999}]},{"osd":7,"last update":"Sat Sep 23 21:00:18 2023","interfaces":[{"interface":"back","average":{"1min":1.0189999999999999,"5min":0.81000000000000005,"15min":0.67700000000000005},"min":{"1min":0.35499999999999998,"5min":0.307,"15min":0.30199999999999999},"max":{"1min":4.4240000000000004,"5min":4.4240000000000004,"15min":8.5649999999999995},"last":0.40500000000000003},{"interface":"front","average":{"1min":0.752,"5min":0.71899999999999997,"15min":0.67400000000000004},"min":{"1min":0.373,"5min":0.29799999999999999,"15min":0.27700000000000002},"max":{"1min":2.1789999999999998,"5min":3.8149999999999999,"15min":4.5430000000000001},"last":0.34200000000000003}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":543232000,"data_stored":543227928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":727257088,"data_stored":727253016,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":768475136,"data_stored":768475136,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":544866304,"data_stored":544866304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":337772544,"data_stored":337772544,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":627900416,"data_stored":627900416,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":318574592,"data_stored":318570520,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":335478784,"data_stored":335478784,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T21:00:39.369 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:00:39.443 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:00:39.524 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:00:39.536 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:00:39.536 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:00:39.637 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:00:39.678 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:00:39.776 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T21:00:39.786 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:00:39.790 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:00:39.791 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:00:39.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:00:39 smithi070 ceph-mon[168282]: pgmap v632: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 895 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 610 op/s; 11784/43002 objects degraded (27.403%); 1.9 MiB/s, 40 objects/s recovering 2023-09-23T21:00:39.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:00:39 smithi070 ceph-mon[168282]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T21:00:39.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:00:39 smithi070 ceph-mon[168282]: failed to encode map e248 with expected crc 2023-09-23T21:00:39.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:00:39 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T21:00:39.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:00:39 smithi070 ceph-mon[168282]: osdmap e248: 8 total, 8 up, 8 in 2023-09-23T21:00:39.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:00:39 smithi070 ceph-mon[168282]: failed to encode map e248 with expected crc 2023-09-23T21:00:39.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:00:39 smithi070 ceph-mon[168282]: failed to encode map e248 with expected crc 2023-09-23T21:00:39.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:00:39 smithi070 ceph-mon[168282]: failed to encode map e248 with expected crc 2023-09-23T21:00:39.871 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 21:00:39 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[164029]: ::ffff:172.21.15.70 - - [23/Sep/2023:21:00:39] "GET /metrics HTTP/1.1" 200 36314 "" "Prometheus/2.43.0" 2023-09-23T21:00:39.874 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:00:39 smithi026 ceph-mon[179688]: pgmap v632: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 895 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 610 op/s; 11784/43002 objects degraded (27.403%); 1.9 MiB/s, 40 objects/s recovering 2023-09-23T21:00:39.874 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:00:39 smithi026 ceph-mon[179688]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T21:00:39.874 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:00:39 smithi026 ceph-mon[179688]: failed to encode map e248 with expected crc 2023-09-23T21:00:39.874 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:00:39 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T21:00:39.875 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:00:39 smithi026 ceph-mon[179688]: osdmap e248: 8 total, 8 up, 8 in 2023-09-23T21:00:39.875 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:00:39 smithi026 ceph-mon[179688]: failed to encode map e248 with expected crc 2023-09-23T21:00:39.875 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:00:39 smithi026 ceph-mon[179688]: failed to encode map e248 with expected crc 2023-09-23T21:00:39.875 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:00:39 smithi026 ceph-mon[179688]: failed to encode map e248 with expected crc 2023-09-23T21:00:39.875 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:00:39 smithi026 ceph-mon[175151]: pgmap v632: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 895 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 610 op/s; 11784/43002 objects degraded (27.403%); 1.9 MiB/s, 40 objects/s recovering 2023-09-23T21:00:39.875 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:00:39 smithi026 ceph-mon[175151]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T21:00:39.875 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:00:39 smithi026 ceph-mon[175151]: failed to encode map e248 with expected crc 2023-09-23T21:00:39.875 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:00:39 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T21:00:39.875 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:00:39 smithi026 ceph-mon[175151]: osdmap e248: 8 total, 8 up, 8 in 2023-09-23T21:00:39.876 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:00:39 smithi026 ceph-mon[175151]: failed to encode map e248 with expected crc 2023-09-23T21:00:39.876 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:00:39 smithi026 ceph-mon[175151]: failed to encode map e248 with expected crc 2023-09-23T21:00:39.876 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:00:39 smithi026 ceph-mon[175151]: failed to encode map e248 with expected crc 2023-09-23T21:00:39.892 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:00:40.032 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:00:40.104 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:00:40.110 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:00:40.111 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:00:40.211 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:00:40.309 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:00:40.358 INFO:teuthology.orchestra.run.smithi070.stdout:{ 2023-09-23T21:00:40.359 INFO:teuthology.orchestra.run.smithi070.stdout: "ops": [], 2023-09-23T21:00:40.359 INFO:teuthology.orchestra.run.smithi070.stdout: "complaint_time": 30, 2023-09-23T21:00:40.359 INFO:teuthology.orchestra.run.smithi070.stdout: "num_blocked_ops": 0 2023-09-23T21:00:40.359 INFO:teuthology.orchestra.run.smithi070.stdout:} 2023-09-23T21:00:40.394 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:00:40.398 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:00:40.399 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:00:40.499 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:00:40.566 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:00:40.632 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:00:40.639 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:00:40.641 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:00:40.642 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:00:40.726 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_historic_ops 2023-09-23T21:00:40.743 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:00:40.811 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:00:40 smithi026 ceph-mon[179688]: Health check update: Degraded data redundancy: 11784/43002 objects degraded (27.403%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T21:00:40.811 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:00:40 smithi026 ceph-mon[175151]: Health check update: Degraded data redundancy: 11784/43002 objects degraded (27.403%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T21:00:40.815 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:00:40.956 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:00:40.966 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:00:40.967 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:00:41.067 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:00:41.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:00:40 smithi070 ceph-mon[168282]: Health check update: Degraded data redundancy: 11784/43002 objects degraded (27.403%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T21:00:41.147 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:00:41.247 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:00:41.257 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:00:41.258 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:00:41.350 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T21:00:41.359 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:00:41.397 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T21:00:41.512 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:00:41.644 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T21:00:41.658 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:00:41.659 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:00:41.759 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:00:41.882 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:00:42.090 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T21:00:42.099 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:00:42.100 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:00:42.200 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:00:42.207 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:00:42 smithi070 ceph-mon[168282]: pgmap v634: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 812 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 614 op/s; 11784/39018 objects degraded (30.201%); 1.9 MiB/s, 41 objects/s recovering 2023-09-23T21:00:42.292 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T21:00:42.315 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:00:42.386 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:00:42 smithi026 ceph-mon[175151]: pgmap v634: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 812 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 614 op/s; 11784/39018 objects degraded (30.201%); 1.9 MiB/s, 41 objects/s recovering 2023-09-23T21:00:42.386 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:00:42 smithi026 ceph-mon[179688]: pgmap v634: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 812 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 614 op/s; 11784/39018 objects degraded (30.201%); 1.9 MiB/s, 41 objects/s recovering 2023-09-23T21:00:42.399 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:00:42.411 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:00:42.411 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:00:42.438 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:00:42.512 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:00:42.518 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0: {} 2023-09-23T21:00:42.551 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:00:42.587 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1: {} 2023-09-23T21:00:42.595 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:00:42.602 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:00:42.603 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:00:42.626 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2: {} 2023-09-23T21:00:42.666 INFO:teuthology.orchestra.run.smithi070.stdout:{ 2023-09-23T21:00:42.667 INFO:teuthology.orchestra.run.smithi070.stdout: "size": 20, 2023-09-23T21:00:42.667 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 600, 2023-09-23T21:00:42.667 INFO:teuthology.orchestra.run.smithi070.stdout: "ops": [ 2023-09-23T21:00:42.667 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:42.667 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.35100.0:95752 7.c 7:320aef38:::benchmark_data_smithi026_260489_object95751:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e229)", 2023-09-23T21:00:42.667 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T20:57:59.846564+0000", 2023-09-23T21:00:42.667 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 162.81707111200001, 2023-09-23T21:00:42.667 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.089005447000000001, 2023-09-23T21:00:42.667 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:00:42.667 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:00:42.667 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:00:42.668 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.35100", 2023-09-23T21:00:42.668 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/4108972961", 2023-09-23T21:00:42.668 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 95752 2023-09-23T21:00:42.668 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:42.668 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:00:42.668 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:42.668 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:00:42.668 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:57:59.846564+0000", 2023-09-23T21:00:42.668 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:00:42.668 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:42.668 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:42.668 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:00:42.669 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:57:59.846564+0000", 2023-09-23T21:00:42.669 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:00:42.669 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:42.669 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:42.669 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:00:42.669 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:57:59.846562+0000", 2023-09-23T21:00:42.669 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T21:00:42.669 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:42.669 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:42.669 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:00:42.669 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:57:59.846603+0000", 2023-09-23T21:00:42.669 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.0246000000000001e-05 2023-09-23T21:00:42.670 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:42.670 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:42.670 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:00:42.670 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:57:59.846617+0000", 2023-09-23T21:00:42.670 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.4535999999999999e-05 2023-09-23T21:00:42.670 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:42.670 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:42.670 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:00:42.670 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:57:59.846624+0000", 2023-09-23T21:00:42.670 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.9099999999999999e-06 2023-09-23T21:00:42.670 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:42.670 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:42.671 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:00:42.671 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:57:59.848264+0000", 2023-09-23T21:00:42.671 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0016401510000000001 2023-09-23T21:00:42.671 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:42.671 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:42.671 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:00:42.671 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:57:59.848461+0000", 2023-09-23T21:00:42.671 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00019658300000000001 2023-09-23T21:00:42.671 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:42.671 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:42.671 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 3,7", 2023-09-23T21:00:42.671 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:57:59.848655+0000", 2023-09-23T21:00:42.672 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000193857 2023-09-23T21:00:42.672 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:42.672 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:42.672 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:00:42.672 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:57:59.854332+0000", 2023-09-23T21:00:42.672 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0056769489999999997 2023-09-23T21:00:42.672 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:42.672 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:42.672 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:00:42.672 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:57:59.854365+0000", 2023-09-23T21:00:42.672 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.3426000000000002e-05 2023-09-23T21:00:42.672 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:42.673 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:42.673 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:00:42.673 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:57:59.931644+0000", 2023-09-23T21:00:42.673 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.077279129000000002 2023-09-23T21:00:42.673 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:42.673 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:42.673 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:00:42.673 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:57:59.931714+0000", 2023-09-23T21:00:42.673 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.9828999999999998e-05 2023-09-23T21:00:42.673 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:42.673 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:42.673 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:00:42.674 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:57:59.935569+0000", 2023-09-23T21:00:42.674 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0038553820000000001 2023-09-23T21:00:42.674 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:00:42.674 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:00:42.674 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:00:42.674 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:42.674 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:42.674 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.35100.0:95754 7.c 7:3d89f732:::benchmark_data_smithi026_260489_object95753:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e229)", 2023-09-23T21:00:42.674 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T20:57:59.846639+0000", 2023-09-23T21:00:42.674 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 162.81699604299999, 2023-09-23T21:00:42.674 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.092545945000000004, 2023-09-23T21:00:42.675 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:00:42.675 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:00:42.675 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:00:42.675 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.35100", 2023-09-23T21:00:42.675 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/4108972961", 2023-09-23T21:00:42.675 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 95754 2023-09-23T21:00:42.675 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:42.675 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:00:42.675 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:42.675 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:00:42.675 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:57:59.846639+0000", 2023-09-23T21:00:42.675 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:00:42.675 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:42.676 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:42.676 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:00:42.676 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:57:59.846639+0000", 2023-09-23T21:00:42.676 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:00:42.676 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:42.676 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:42.676 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:00:42.676 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:57:59.846639+0000", 2023-09-23T21:00:42.676 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999995 2023-09-23T21:00:42.676 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:42.676 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:42.677 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:00:42.677 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:57:59.846668+0000", 2023-09-23T21:00:42.677 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.9835000000000001e-05 2023-09-23T21:00:42.677 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:42.677 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:42.677 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:00:42.677 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:57:59.846680+0000", 2023-09-23T21:00:42.677 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.1327e-05 2023-09-23T21:00:42.677 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:42.677 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:42.677 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:00:42.677 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:57:59.846685+0000", 2023-09-23T21:00:42.677 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.9960000000000001e-06 2023-09-23T21:00:42.678 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:42.678 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:42.678 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:00:42.678 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:57:59.851222+0000", 2023-09-23T21:00:42.678 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0045377059999999999 2023-09-23T21:00:42.678 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:42.678 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:42.678 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:00:42.678 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:57:59.851417+0000", 2023-09-23T21:00:42.678 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000194129 2023-09-23T21:00:42.678 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:42.678 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:42.679 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 3,7", 2023-09-23T21:00:42.679 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:57:59.851613+0000", 2023-09-23T21:00:42.679 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00019676000000000001 2023-09-23T21:00:42.679 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:42.679 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:42.679 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:00:42.679 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:57:59.854360+0000", 2023-09-23T21:00:42.679 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0027465409999999999 2023-09-23T21:00:42.679 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:42.679 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:42.679 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:00:42.679 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:57:59.854410+0000", 2023-09-23T21:00:42.680 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.0345e-05 2023-09-23T21:00:42.680 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:42.680 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:42.680 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:00:42.680 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:57:59.935640+0000", 2023-09-23T21:00:42.680 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.081229928000000007 2023-09-23T21:00:42.680 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:42.680 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:42.680 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:00:42.680 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:57:59.935705+0000", 2023-09-23T21:00:42.680 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.4956000000000006e-05 2023-09-23T21:00:42.680 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:42.680 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:42.681 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:00:42.681 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:57:59.939185+0000", 2023-09-23T21:00:42.681 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.003479883 2023-09-23T21:00:42.681 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:00:42.681 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:00:42.681 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:00:42.681 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:42.681 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:42.681 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.35100.0:95758 7.c 7:34c0c949:::benchmark_data_smithi026_260489_object95757:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e229)", 2023-09-23T21:00:42.681 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T20:57:59.857820+0000", 2023-09-23T21:00:42.681 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 162.805814739, 2023-09-23T21:00:42.681 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.084066121999999993, 2023-09-23T21:00:42.682 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:00:42.682 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:00:42.682 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:00:42.682 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.35100", 2023-09-23T21:00:42.682 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/4108972961", 2023-09-23T21:00:42.682 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 95758 2023-09-23T21:00:42.682 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:42.682 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:00:42.682 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:42.682 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:00:42.682 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:57:59.857820+0000", 2023-09-23T21:00:42.683 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:00:42.683 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:42.683 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:42.683 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:00:42.683 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:57:59.857820+0000", 2023-09-23T21:00:42.683 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:00:42.683 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:42.683 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:42.683 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:00:42.683 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:57:59.857817+0000", 2023-09-23T21:00:42.683 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999967 2023-09-23T21:00:42.683 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:42.683 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:42.684 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:00:42.684 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:57:59.857886+0000", 2023-09-23T21:00:42.684 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.9015000000000002e-05 2023-09-23T21:00:42.684 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:42.684 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:42.684 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:00:42.684 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:57:59.857930+0000", 2023-09-23T21:00:42.684 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.3705000000000003e-05 2023-09-23T21:00:42.684 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:42.684 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:42.684 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:00:42.684 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:57:59.857948+0000", 2023-09-23T21:00:42.685 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.7859999999999998e-05 2023-09-23T21:00:42.685 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:42.685 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:42.685 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:00:42.685 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:57:59.858102+0000", 2023-09-23T21:00:42.685 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000154692 2023-09-23T21:00:42.685 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:42.685 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:42.685 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:00:42.685 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:57:59.858363+0000", 2023-09-23T21:00:42.685 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000260528 2023-09-23T21:00:42.685 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:42.685 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:42.686 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 3,7", 2023-09-23T21:00:42.686 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:57:59.858530+0000", 2023-09-23T21:00:42.686 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00016747299999999999 2023-09-23T21:00:42.686 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:42.686 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:42.686 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:00:42.686 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:57:59.864064+0000", 2023-09-23T21:00:42.686 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0055341380000000001 2023-09-23T21:00:42.686 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:42.686 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:42.686 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:00:42.686 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:57:59.867020+0000", 2023-09-23T21:00:42.687 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0029556460000000001 2023-09-23T21:00:42.687 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:42.687 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:42.687 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:00:42.687 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:57:59.939248+0000", 2023-09-23T21:00:42.687 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.072227623000000005 2023-09-23T21:00:42.687 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:42.687 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:42.687 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:00:42.687 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:57:59.939314+0000", 2023-09-23T21:00:42.687 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.6471999999999996e-05 2023-09-23T21:00:42.687 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:42.687 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:42.688 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:00:42.688 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:57:59.941886+0000", 2023-09-23T21:00:42.688 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.002572253 2023-09-23T21:00:42.688 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:00:42.688 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:00:42.688 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:00:42.688 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:42.688 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:42.688 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.35100.0:95763 7.c 7:31a4ed80:::benchmark_data_smithi026_260489_object95762:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e229)", 2023-09-23T21:00:42.688 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T20:57:59.858111+0000", 2023-09-23T21:00:42.688 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 162.80552415700001, 2023-09-23T21:00:42.688 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.086271913000000006, 2023-09-23T21:00:42.688 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:00:42.689 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:00:42.689 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:00:42.689 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.35100", 2023-09-23T21:00:42.689 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/4108972961", 2023-09-23T21:00:42.689 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 95763 2023-09-23T21:00:42.689 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:42.689 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:00:42.689 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:42.689 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:00:42.689 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:57:59.858111+0000", 2023-09-23T21:00:42.689 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:00:42.689 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:42.690 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:42.690 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:00:42.690 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:57:59.858111+0000", 2023-09-23T21:00:42.690 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:00:42.690 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:42.690 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:42.690 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:00:42.690 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:57:59.858110+0000", 2023-09-23T21:00:42.690 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.999999 2023-09-23T21:00:42.690 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:42.690 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:42.690 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:00:42.691 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:57:59.858168+0000", 2023-09-23T21:00:42.691 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.8628000000000002e-05 2023-09-23T21:00:42.691 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:42.691 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:42.691 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:00:42.691 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:57:59.858188+0000", 2023-09-23T21:00:42.691 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.9347e-05 2023-09-23T21:00:42.691 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:42.691 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:42.691 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:00:42.691 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:57:59.858200+0000", 2023-09-23T21:00:42.691 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.1793e-05 2023-09-23T21:00:42.691 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:42.692 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:42.692 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:00:42.692 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:57:59.861505+0000", 2023-09-23T21:00:42.692 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.003305835 2023-09-23T21:00:42.692 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:42.692 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:42.692 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:00:42.692 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:57:59.861684+0000", 2023-09-23T21:00:42.692 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00017906000000000001 2023-09-23T21:00:42.692 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:42.692 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:42.692 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 3,7", 2023-09-23T21:00:42.693 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:57:59.861824+0000", 2023-09-23T21:00:42.693 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00013954000000000001 2023-09-23T21:00:42.693 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:42.693 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:42.693 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:00:42.693 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:57:59.864266+0000", 2023-09-23T21:00:42.693 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.002441772 2023-09-23T21:00:42.693 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:42.693 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:42.693 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:00:42.693 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:57:59.867174+0000", 2023-09-23T21:00:42.693 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0029080899999999999 2023-09-23T21:00:42.693 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:42.694 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:42.694 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:00:42.694 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:57:59.941951+0000", 2023-09-23T21:00:42.694 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.074777307000000001 2023-09-23T21:00:42.694 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:42.694 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:42.694 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:00:42.694 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:57:59.941997+0000", 2023-09-23T21:00:42.694 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.5574000000000001e-05 2023-09-23T21:00:42.694 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:42.694 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:42.694 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:00:42.695 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:57:59.944383+0000", 2023-09-23T21:00:42.695 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0023860969999999998 2023-09-23T21:00:42.695 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:00:42.695 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:00:42.695 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:00:42.695 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:42.695 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:42.695 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.35100.0:116411 7.c 7:386f85a2:::benchmark_data_smithi026_260489_object14649:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e234)", 2023-09-23T21:00:42.695 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T20:58:32.707360+0000", 2023-09-23T21:00:42.695 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 129.956274627, 2023-09-23T21:00:42.695 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.092441081999999994, 2023-09-23T21:00:42.695 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:00:42.696 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:00:42.696 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:00:42.696 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.35100", 2023-09-23T21:00:42.696 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/4108972961", 2023-09-23T21:00:42.696 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 116411 2023-09-23T21:00:42.696 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:42.696 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:00:42.696 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:42.696 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:00:42.696 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:58:32.707360+0000", 2023-09-23T21:00:42.696 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:00:42.696 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:42.696 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:42.697 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:00:42.697 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:58:32.707360+0000", 2023-09-23T21:00:42.697 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:00:42.697 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:42.697 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:42.697 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:00:42.697 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:58:32.707360+0000", 2023-09-23T21:00:42.697 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999995 2023-09-23T21:00:42.697 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:42.697 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:42.697 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:00:42.697 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:58:32.707363+0000", 2023-09-23T21:00:42.697 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.8119999999999999e-06 2023-09-23T21:00:42.698 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:42.698 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:42.698 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:00:42.698 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:58:32.707371+0000", 2023-09-23T21:00:42.698 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.8809999999999996e-06 2023-09-23T21:00:42.698 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:42.698 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:42.698 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:00:42.698 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:58:32.707375+0000", 2023-09-23T21:00:42.698 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.9790000000000004e-06 2023-09-23T21:00:42.698 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:42.698 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:42.699 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:00:42.699 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:58:32.707425+0000", 2023-09-23T21:00:42.699 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.0643999999999998e-05 2023-09-23T21:00:42.699 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:42.699 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:42.699 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:00:42.699 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:58:32.707574+0000", 2023-09-23T21:00:42.699 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000148715 2023-09-23T21:00:42.699 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:42.699 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:42.699 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 3,7", 2023-09-23T21:00:42.699 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:58:32.707711+0000", 2023-09-23T21:00:42.700 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00013714099999999999 2023-09-23T21:00:42.700 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:42.700 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:42.700 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:00:42.700 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:58:32.714308+0000", 2023-09-23T21:00:42.700 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.006597182 2023-09-23T21:00:42.700 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:42.700 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:42.700 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:00:42.700 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:58:32.714392+0000", 2023-09-23T21:00:42.700 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.3598999999999997e-05 2023-09-23T21:00:42.700 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:42.700 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:42.701 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:00:42.701 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:58:32.797014+0000", 2023-09-23T21:00:42.701 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.082622152000000004 2023-09-23T21:00:42.701 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:42.701 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:42.701 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:00:42.701 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:58:32.797076+0000", 2023-09-23T21:00:42.701 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.1576999999999997e-05 2023-09-23T21:00:42.701 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:42.701 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:42.701 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:00:42.701 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:58:32.799801+0000", 2023-09-23T21:00:42.702 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0027257779999999999 2023-09-23T21:00:42.702 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:00:42.702 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:00:42.702 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:00:42.702 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:42.702 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:42.702 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.35100.0:116412 7.c 7:32401c65:::benchmark_data_smithi026_260489_object14650:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e234)", 2023-09-23T21:00:42.702 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T20:58:32.707388+0000", 2023-09-23T21:00:42.702 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 129.956247179, 2023-09-23T21:00:42.702 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.094168145999999994, 2023-09-23T21:00:42.702 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:00:42.702 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:00:42.703 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:00:42.703 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.35100", 2023-09-23T21:00:42.703 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/4108972961", 2023-09-23T21:00:42.703 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 116412 2023-09-23T21:00:42.703 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:42.703 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:00:42.703 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:42.703 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:00:42.703 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:58:32.707388+0000", 2023-09-23T21:00:42.703 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:00:42.703 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:42.703 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:42.704 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:00:42.704 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:58:32.707388+0000", 2023-09-23T21:00:42.704 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:00:42.704 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:42.704 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:42.704 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:00:42.704 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:58:32.707388+0000", 2023-09-23T21:00:42.704 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999995 2023-09-23T21:00:42.704 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:42.704 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:42.704 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:00:42.704 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:58:32.707390+0000", 2023-09-23T21:00:42.704 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.3970000000000001e-06 2023-09-23T21:00:42.705 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:42.705 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:42.705 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:00:42.705 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:58:32.707774+0000", 2023-09-23T21:00:42.705 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00038455999999999999 2023-09-23T21:00:42.705 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:42.705 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:42.705 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:00:42.705 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:58:32.707781+0000", 2023-09-23T21:00:42.705 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.392e-06 2023-09-23T21:00:42.705 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:42.705 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:42.705 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:00:42.706 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:58:32.711093+0000", 2023-09-23T21:00:42.706 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0033124309999999998 2023-09-23T21:00:42.706 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:42.706 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:42.706 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:00:42.706 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:58:32.711302+0000", 2023-09-23T21:00:42.706 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000208307 2023-09-23T21:00:42.706 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:42.706 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:42.706 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 3,7", 2023-09-23T21:00:42.706 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:58:32.711476+0000", 2023-09-23T21:00:42.706 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00017412100000000001 2023-09-23T21:00:42.707 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:42.707 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:42.707 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:00:42.707 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:58:32.714404+0000", 2023-09-23T21:00:42.707 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0029286080000000001 2023-09-23T21:00:42.707 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:42.707 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:42.707 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:00:42.707 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:58:32.718305+0000", 2023-09-23T21:00:42.707 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.003900447 2023-09-23T21:00:42.707 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:42.707 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:42.707 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:00:42.708 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:58:32.799851+0000", 2023-09-23T21:00:42.708 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.081545845000000006 2023-09-23T21:00:42.708 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:42.708 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:42.708 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:00:42.708 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:58:32.799895+0000", 2023-09-23T21:00:42.708 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.3883e-05 2023-09-23T21:00:42.708 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:42.708 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:42.708 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:00:42.708 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:58:32.801556+0000", 2023-09-23T21:00:42.708 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.001661479 2023-09-23T21:00:42.709 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:00:42.709 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:00:42.709 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:00:42.709 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:42.709 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:42.709 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.35100.0:116416 7.c 7:315889a0:::benchmark_data_smithi026_260489_object14654:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e234)", 2023-09-23T21:00:42.709 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T20:58:32.714704+0000", 2023-09-23T21:00:42.709 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 129.94893067500001, 2023-09-23T21:00:42.709 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.088565651999999995, 2023-09-23T21:00:42.709 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:00:42.709 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:00:42.709 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:00:42.709 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.35100", 2023-09-23T21:00:42.709 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/4108972961", 2023-09-23T21:00:42.710 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 116416 2023-09-23T21:00:42.710 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:42.710 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:00:42.710 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:42.710 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:00:42.710 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:58:32.714704+0000", 2023-09-23T21:00:42.710 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:00:42.710 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:42.710 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:42.710 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:00:42.710 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:58:32.714704+0000", 2023-09-23T21:00:42.710 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:00:42.711 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:42.711 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:42.711 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:00:42.711 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:58:32.714703+0000", 2023-09-23T21:00:42.711 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T21:00:42.711 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:42.711 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:42.711 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:00:42.711 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:58:32.714711+0000", 2023-09-23T21:00:42.711 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.6709999999999999e-06 2023-09-23T21:00:42.711 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:42.711 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:42.712 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:00:42.712 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:58:32.714724+0000", 2023-09-23T21:00:42.712 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.3249e-05 2023-09-23T21:00:42.712 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:42.712 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:42.712 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:00:42.712 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:58:32.714730+0000", 2023-09-23T21:00:42.712 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.9399999999999999e-06 2023-09-23T21:00:42.712 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:42.712 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:42.712 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:00:42.712 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:58:32.714770+0000", 2023-09-23T21:00:42.713 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.0296000000000002e-05 2023-09-23T21:00:42.713 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:42.713 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:42.713 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:00:42.713 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:58:32.714893+0000", 2023-09-23T21:00:42.713 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00012327599999999999 2023-09-23T21:00:42.713 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:42.713 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:42.713 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 3,7", 2023-09-23T21:00:42.713 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:58:32.715066+0000", 2023-09-23T21:00:42.713 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00017291999999999999 2023-09-23T21:00:42.713 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:42.714 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:42.714 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:00:42.714 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:58:32.718350+0000", 2023-09-23T21:00:42.714 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0032839549999999999 2023-09-23T21:00:42.714 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:42.714 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:42.714 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:00:42.714 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:58:32.719377+0000", 2023-09-23T21:00:42.714 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.001026721 2023-09-23T21:00:42.714 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:42.714 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:42.714 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:00:42.714 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:58:32.801595+0000", 2023-09-23T21:00:42.715 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.082218264999999999 2023-09-23T21:00:42.715 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:42.715 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:42.715 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:00:42.715 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:58:32.801636+0000", 2023-09-23T21:00:42.715 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.0797000000000003e-05 2023-09-23T21:00:42.715 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:42.715 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:42.715 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:00:42.715 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:58:32.803270+0000", 2023-09-23T21:00:42.715 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0016339690000000001 2023-09-23T21:00:42.715 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:00:42.716 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:00:42.716 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:00:42.716 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:42.716 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:42.716 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.35100.0:117031 7.c 7:36f10993:::benchmark_data_smithi026_260489_object15269:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e234)", 2023-09-23T21:00:42.716 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T20:58:33.829583+0000", 2023-09-23T21:00:42.716 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 128.834051974, 2023-09-23T21:00:42.716 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.082900809000000006, 2023-09-23T21:00:42.716 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:00:42.716 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:00:42.716 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:00:42.716 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.35100", 2023-09-23T21:00:42.717 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/4108972961", 2023-09-23T21:00:42.717 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 117031 2023-09-23T21:00:42.717 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:42.717 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:00:42.717 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:42.717 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:00:42.717 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:58:33.829583+0000", 2023-09-23T21:00:42.717 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:00:42.717 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:42.717 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:42.717 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:00:42.717 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:58:33.829583+0000", 2023-09-23T21:00:42.718 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:00:42.718 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:42.718 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:42.718 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:00:42.718 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:58:33.829583+0000", 2023-09-23T21:00:42.718 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999995 2023-09-23T21:00:42.718 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:42.718 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:42.718 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:00:42.718 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:58:33.829586+0000", 2023-09-23T21:00:42.718 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.0259999999999999e-06 2023-09-23T21:00:42.718 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:42.719 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:42.719 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:00:42.719 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:58:33.829592+0000", 2023-09-23T21:00:42.719 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.6980000000000001e-06 2023-09-23T21:00:42.719 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:42.719 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:42.719 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:00:42.719 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:58:33.829597+0000", 2023-09-23T21:00:42.719 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.6e-06 2023-09-23T21:00:42.719 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:42.719 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:42.719 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:00:42.719 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:58:33.837654+0000", 2023-09-23T21:00:42.720 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0080573050000000007 2023-09-23T21:00:42.720 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:42.720 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:42.720 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:00:42.720 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:58:33.837852+0000", 2023-09-23T21:00:42.720 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00019767 2023-09-23T21:00:42.720 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:42.720 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:42.720 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 3,7", 2023-09-23T21:00:42.720 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:58:33.838054+0000", 2023-09-23T21:00:42.720 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00020198199999999999 2023-09-23T21:00:42.720 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:42.721 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:42.721 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:00:42.721 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:58:33.841132+0000", 2023-09-23T21:00:42.721 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0030777809999999999 2023-09-23T21:00:42.721 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:42.721 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:42.721 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:00:42.721 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:58:33.842230+0000", 2023-09-23T21:00:42.721 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0010978680000000001 2023-09-23T21:00:42.721 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:42.721 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:42.721 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:00:42.721 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:58:33.909535+0000", 2023-09-23T21:00:42.722 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.067305746 2023-09-23T21:00:42.722 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:42.722 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:42.722 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:00:42.722 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:58:33.909601+0000", 2023-09-23T21:00:42.722 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.5905e-05 2023-09-23T21:00:42.722 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:42.722 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:42.722 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:00:42.722 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:58:33.912484+0000", 2023-09-23T21:00:42.722 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0028826989999999999 2023-09-23T21:00:42.723 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:00:42.723 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:00:42.723 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:00:42.723 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:42.723 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:42.723 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.35100.0:158324 7.c 7:3c192457:::benchmark_data_smithi026_260489_object56562:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e242)", 2023-09-23T21:00:42.723 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T20:59:44.165752+0000", 2023-09-23T21:00:42.723 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 58.497882998999998, 2023-09-23T21:00:42.723 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.080809798000000002, 2023-09-23T21:00:42.723 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:00:42.723 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:00:42.723 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:00:42.724 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.35100", 2023-09-23T21:00:42.724 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/4108972961", 2023-09-23T21:00:42.724 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 158324 2023-09-23T21:00:42.724 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:42.724 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:00:42.724 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:42.724 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:00:42.724 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:59:44.165752+0000", 2023-09-23T21:00:42.724 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:00:42.724 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:42.724 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:42.724 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:00:42.724 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:59:44.165752+0000", 2023-09-23T21:00:42.725 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:00:42.725 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:42.725 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:42.725 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:00:42.725 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:59:44.165751+0000", 2023-09-23T21:00:42.725 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999995 2023-09-23T21:00:42.725 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:42.725 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:42.725 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:00:42.725 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:59:44.165755+0000", 2023-09-23T21:00:42.725 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.298e-06 2023-09-23T21:00:42.725 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:42.725 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:42.726 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:00:42.726 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:59:44.165762+0000", 2023-09-23T21:00:42.726 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.9110000000000004e-06 2023-09-23T21:00:42.726 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:42.726 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:42.726 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:00:42.726 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:59:44.165766+0000", 2023-09-23T21:00:42.726 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.6949999999999999e-06 2023-09-23T21:00:42.726 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:42.726 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:42.726 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:00:42.726 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:59:44.172062+0000", 2023-09-23T21:00:42.727 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.006295541 2023-09-23T21:00:42.727 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:42.727 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:42.727 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:00:42.727 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:59:44.172245+0000", 2023-09-23T21:00:42.727 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00018293000000000001 2023-09-23T21:00:42.727 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:42.727 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:42.727 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 3,7", 2023-09-23T21:00:42.727 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:59:44.172426+0000", 2023-09-23T21:00:42.727 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00018088699999999999 2023-09-23T21:00:42.727 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:42.727 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:42.728 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:00:42.728 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:59:44.174944+0000", 2023-09-23T21:00:42.728 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0025177670000000002 2023-09-23T21:00:42.728 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:42.728 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:42.728 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:00:42.728 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:59:44.175365+0000", 2023-09-23T21:00:42.728 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00042120200000000002 2023-09-23T21:00:42.728 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:42.728 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:42.728 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:00:42.728 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:59:44.245350+0000", 2023-09-23T21:00:42.728 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.069984897000000004 2023-09-23T21:00:42.729 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:42.729 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:42.729 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:00:42.729 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:59:44.245391+0000", 2023-09-23T21:00:42.729 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.1149000000000002e-05 2023-09-23T21:00:42.729 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:42.729 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:42.729 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:00:42.729 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:59:44.246562+0000", 2023-09-23T21:00:42.729 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0011710659999999999 2023-09-23T21:00:42.729 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:00:42.729 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:00:42.730 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:00:42.730 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:42.730 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:42.730 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.35100.0:159312 7.c 7:35fe94fa:::benchmark_data_smithi026_260489_object57550:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e242)", 2023-09-23T21:00:42.730 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T20:59:45.714683+0000", 2023-09-23T21:00:42.730 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 56.948951694999998, 2023-09-23T21:00:42.730 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.098395191000000007, 2023-09-23T21:00:42.730 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:00:42.730 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:00:42.730 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:00:42.730 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.35100", 2023-09-23T21:00:42.731 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/4108972961", 2023-09-23T21:00:42.731 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 159312 2023-09-23T21:00:42.731 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:42.731 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:00:42.731 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:42.731 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:00:42.731 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:59:45.714683+0000", 2023-09-23T21:00:42.731 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:00:42.731 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:42.731 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:42.731 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:00:42.731 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:59:45.714683+0000", 2023-09-23T21:00:42.732 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:00:42.732 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:42.732 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:42.732 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:00:42.732 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:59:45.714683+0000", 2023-09-23T21:00:42.732 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999995 2023-09-23T21:00:42.732 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:42.732 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:42.732 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:00:42.732 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:59:45.714686+0000", 2023-09-23T21:00:42.732 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.1e-06 2023-09-23T21:00:42.732 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:42.732 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:42.733 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:00:42.733 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:59:45.714693+0000", 2023-09-23T21:00:42.733 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.3660000000000003e-06 2023-09-23T21:00:42.733 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:42.733 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:42.733 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:00:42.733 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:59:45.714698+0000", 2023-09-23T21:00:42.733 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.8330000000000001e-06 2023-09-23T21:00:42.733 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:42.733 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:42.733 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:00:42.733 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:59:45.714785+0000", 2023-09-23T21:00:42.733 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.6920000000000001e-05 2023-09-23T21:00:42.733 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:42.734 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:42.734 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:00:42.734 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:59:45.714945+0000", 2023-09-23T21:00:42.734 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00015988100000000001 2023-09-23T21:00:42.734 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:42.734 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:42.734 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 3,7", 2023-09-23T21:00:42.734 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:59:45.715100+0000", 2023-09-23T21:00:42.734 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000155119 2023-09-23T21:00:42.734 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:42.734 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:42.734 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:00:42.735 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:59:45.721013+0000", 2023-09-23T21:00:42.735 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0059126630000000003 2023-09-23T21:00:42.735 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:42.735 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:42.735 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:00:42.735 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:59:45.729983+0000", 2023-09-23T21:00:42.735 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0089698169999999997 2023-09-23T21:00:42.735 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:42.735 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:42.735 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:00:42.735 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:59:45.810696+0000", 2023-09-23T21:00:42.735 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.080713268000000005 2023-09-23T21:00:42.736 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:42.736 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:42.736 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:00:42.736 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:59:45.810748+0000", 2023-09-23T21:00:42.736 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.2618999999999999e-05 2023-09-23T21:00:42.736 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:42.736 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:42.736 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:00:42.736 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:59:45.813079+0000", 2023-09-23T21:00:42.736 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.002330024 2023-09-23T21:00:42.736 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:00:42.736 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:00:42.736 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:00:42.736 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:42.737 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:42.737 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.35100.0:159314 7.c 7:3dd9b3f5:::benchmark_data_smithi026_260489_object57552:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e242)", 2023-09-23T21:00:42.737 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T20:59:45.715055+0000", 2023-09-23T21:00:42.737 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 56.948579832999997, 2023-09-23T21:00:42.737 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.099656759999999997, 2023-09-23T21:00:42.737 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:00:42.737 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:00:42.737 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:00:42.737 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.35100", 2023-09-23T21:00:42.737 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/4108972961", 2023-09-23T21:00:42.737 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 159314 2023-09-23T21:00:42.738 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:42.738 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:00:42.738 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:42.738 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:00:42.738 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:59:45.715055+0000", 2023-09-23T21:00:42.738 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:00:42.738 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:42.738 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:42.738 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:00:42.738 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:59:45.715055+0000", 2023-09-23T21:00:42.738 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:00:42.738 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:42.739 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:42.739 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:00:42.739 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:59:45.715055+0000", 2023-09-23T21:00:42.739 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999995 2023-09-23T21:00:42.739 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:42.739 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:42.739 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:00:42.739 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:59:45.715059+0000", 2023-09-23T21:00:42.739 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.2660000000000003e-06 2023-09-23T21:00:42.739 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:42.739 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:42.739 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:00:42.739 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:59:45.715067+0000", 2023-09-23T21:00:42.739 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.9219999999999997e-06 2023-09-23T21:00:42.740 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:42.740 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:42.740 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:00:42.740 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:59:45.715072+0000", 2023-09-23T21:00:42.740 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.2689999999999999e-06 2023-09-23T21:00:42.740 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:42.740 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:42.740 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:00:42.740 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:59:45.717941+0000", 2023-09-23T21:00:42.740 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.002869267 2023-09-23T21:00:42.740 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:42.740 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:42.740 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:00:42.741 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:59:45.718091+0000", 2023-09-23T21:00:42.741 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00014939 2023-09-23T21:00:42.741 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:42.741 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:42.741 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 3,7", 2023-09-23T21:00:42.741 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:59:45.718272+0000", 2023-09-23T21:00:42.741 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000181249 2023-09-23T21:00:42.741 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:42.741 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:42.741 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:00:42.741 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:59:45.721107+0000", 2023-09-23T21:00:42.741 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.002835335 2023-09-23T21:00:42.741 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:42.742 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:42.742 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:00:42.742 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:59:45.730021+0000", 2023-09-23T21:00:42.742 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0089133249999999997 2023-09-23T21:00:42.742 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:42.742 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:42.742 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:00:42.742 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:59:45.813147+0000", 2023-09-23T21:00:42.742 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.083126633000000005 2023-09-23T21:00:42.742 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:42.742 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:42.743 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:00:42.743 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:59:45.813186+0000", 2023-09-23T21:00:42.743 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.9128999999999998e-05 2023-09-23T21:00:42.743 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:42.743 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:42.743 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:00:42.743 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:59:45.814712+0000", 2023-09-23T21:00:42.743 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0015256040000000001 2023-09-23T21:00:42.743 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:00:42.743 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:00:42.743 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:00:42.743 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:42.744 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:42.744 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.35100.0:159321 7.c 7:33459c82:::benchmark_data_smithi026_260489_object57559:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e242)", 2023-09-23T21:00:42.744 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T20:59:45.723923+0000", 2023-09-23T21:00:42.744 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 56.939712491000002, 2023-09-23T21:00:42.744 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.092489334000000006, 2023-09-23T21:00:42.744 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:00:42.744 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:00:42.744 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:00:42.744 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.35100", 2023-09-23T21:00:42.744 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/4108972961", 2023-09-23T21:00:42.744 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 159321 2023-09-23T21:00:42.745 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:42.745 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:00:42.745 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:42.745 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:00:42.745 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:59:45.723923+0000", 2023-09-23T21:00:42.745 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:00:42.745 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:42.745 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:42.745 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:00:42.745 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:59:45.723923+0000", 2023-09-23T21:00:42.745 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:00:42.745 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:42.745 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:42.746 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:00:42.746 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:59:45.723921+0000", 2023-09-23T21:00:42.746 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.999999 2023-09-23T21:00:42.746 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:42.746 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:42.746 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:00:42.746 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:59:45.723927+0000", 2023-09-23T21:00:42.746 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.541e-06 2023-09-23T21:00:42.746 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:42.746 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:42.746 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:00:42.746 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:59:45.723942+0000", 2023-09-23T21:00:42.747 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.49e-05 2023-09-23T21:00:42.747 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:42.747 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:42.747 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:00:42.747 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:59:45.723949+0000", 2023-09-23T21:00:42.747 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.6959999999999999e-06 2023-09-23T21:00:42.747 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:42.747 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:42.747 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:00:42.747 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:59:45.723997+0000", 2023-09-23T21:00:42.747 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.7963000000000001e-05 2023-09-23T21:00:42.747 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:42.747 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:42.748 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:00:42.748 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:59:45.724141+0000", 2023-09-23T21:00:42.748 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00014393399999999999 2023-09-23T21:00:42.748 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:42.748 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:42.748 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 3,7", 2023-09-23T21:00:42.748 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:59:45.724301+0000", 2023-09-23T21:00:42.748 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00016078300000000001 2023-09-23T21:00:42.748 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:42.748 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:42.748 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:00:42.748 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:59:45.727866+0000", 2023-09-23T21:00:42.748 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0035644069999999999 2023-09-23T21:00:42.749 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:42.749 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:42.749 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:00:42.749 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:59:45.730056+0000", 2023-09-23T21:00:42.749 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0021899060000000001 2023-09-23T21:00:42.749 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:42.749 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:42.749 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:00:42.749 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:59:45.814772+0000", 2023-09-23T21:00:42.749 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.084716738 2023-09-23T21:00:42.749 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:42.749 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:42.750 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:00:42.750 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:59:45.814811+0000", 2023-09-23T21:00:42.750 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.8770999999999999e-05 2023-09-23T21:00:42.750 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:42.750 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:42.750 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:00:42.750 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:59:45.816412+0000", 2023-09-23T21:00:42.750 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.001600757 2023-09-23T21:00:42.750 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:00:42.750 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:00:42.750 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:00:42.750 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:42.750 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:42.751 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.35100.0:159326 7.c 7:3145ce85:::benchmark_data_smithi026_260489_object57564:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e242)", 2023-09-23T21:00:42.751 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T20:59:45.731631+0000", 2023-09-23T21:00:42.751 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 56.932003922, 2023-09-23T21:00:42.751 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.086931219000000004, 2023-09-23T21:00:42.751 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:00:42.751 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:00:42.751 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:00:42.751 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.35100", 2023-09-23T21:00:42.751 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/4108972961", 2023-09-23T21:00:42.751 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 159326 2023-09-23T21:00:42.751 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:42.751 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:00:42.752 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:42.752 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:00:42.752 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:59:45.731631+0000", 2023-09-23T21:00:42.752 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:00:42.752 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:42.752 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:42.752 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:00:42.752 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:59:45.731631+0000", 2023-09-23T21:00:42.752 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:00:42.752 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:42.752 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:42.752 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:00:42.752 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:59:45.731630+0000", 2023-09-23T21:00:42.753 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.999999 2023-09-23T21:00:42.753 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:42.753 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:42.753 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:00:42.753 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:59:45.731636+0000", 2023-09-23T21:00:42.753 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.6350000000000003e-06 2023-09-23T21:00:42.753 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:42.753 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:42.753 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:00:42.753 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:59:45.731649+0000", 2023-09-23T21:00:42.753 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.3694e-05 2023-09-23T21:00:42.753 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:42.753 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:42.754 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:00:42.754 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:59:45.731656+0000", 2023-09-23T21:00:42.754 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.9870000000000002e-06 2023-09-23T21:00:42.754 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:42.754 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:42.754 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:00:42.754 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:59:45.731705+0000", 2023-09-23T21:00:42.754 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.8285999999999998e-05 2023-09-23T21:00:42.754 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:42.754 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:42.754 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:00:42.755 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:59:45.731864+0000", 2023-09-23T21:00:42.755 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000158999 2023-09-23T21:00:42.755 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:42.755 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:42.755 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 3,7", 2023-09-23T21:00:42.755 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:59:45.732063+0000", 2023-09-23T21:00:42.755 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00019966999999999999 2023-09-23T21:00:42.755 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:42.755 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:42.755 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:00:42.755 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:59:45.735028+0000", 2023-09-23T21:00:42.755 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0029652630000000001 2023-09-23T21:00:42.755 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:42.756 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:42.756 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:00:42.756 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:59:45.736064+0000", 2023-09-23T21:00:42.756 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.001035366 2023-09-23T21:00:42.756 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:42.756 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:42.756 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:00:42.756 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:59:45.816467+0000", 2023-09-23T21:00:42.756 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.080403536999999997 2023-09-23T21:00:42.756 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:42.756 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:42.756 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:00:42.757 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:59:45.816526+0000", 2023-09-23T21:00:42.757 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.8727000000000002e-05 2023-09-23T21:00:42.757 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:42.757 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:42.757 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:00:42.757 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:59:45.818562+0000", 2023-09-23T21:00:42.757 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0020362010000000001 2023-09-23T21:00:42.757 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:00:42.757 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:00:42.757 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:00:42.757 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:42.757 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:42.757 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.35100.0:168557 7.c 7:3a3d85f5:::benchmark_data_smithi026_260489_object66795:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e243)", 2023-09-23T21:00:42.758 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:00:01.247853+0000", 2023-09-23T21:00:42.758 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 41.415781533000001, 2023-09-23T21:00:42.758 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.091415162999999994, 2023-09-23T21:00:42.758 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:00:42.758 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:00:42.758 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:00:42.758 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.35100", 2023-09-23T21:00:42.758 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/4108972961", 2023-09-23T21:00:42.758 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 168557 2023-09-23T21:00:42.758 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:42.758 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:00:42.758 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:42.758 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:00:42.759 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:00:01.247853+0000", 2023-09-23T21:00:42.759 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:00:42.759 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:42.759 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:42.759 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:00:42.759 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:00:01.247853+0000", 2023-09-23T21:00:42.759 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:00:42.759 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:42.759 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:42.759 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:00:42.759 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:00:01.247853+0000", 2023-09-23T21:00:42.759 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999995 2023-09-23T21:00:42.760 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:42.760 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:42.760 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:00:42.760 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:00:01.247856+0000", 2023-09-23T21:00:42.760 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.7080000000000002e-06 2023-09-23T21:00:42.760 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:42.760 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:42.760 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:00:42.760 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:00:01.247861+0000", 2023-09-23T21:00:42.760 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.5419999999999996e-06 2023-09-23T21:00:42.760 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:42.760 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:42.761 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:00:42.761 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:00:01.247865+0000", 2023-09-23T21:00:42.761 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.9999999999999998e-06 2023-09-23T21:00:42.761 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:42.761 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:42.761 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:00:42.761 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:00:01.249686+0000", 2023-09-23T21:00:42.761 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0018203990000000001 2023-09-23T21:00:42.761 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:42.761 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:42.761 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:00:42.761 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:00:01.249824+0000", 2023-09-23T21:00:42.761 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00013816800000000001 2023-09-23T21:00:42.762 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:42.762 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:42.762 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 3,7", 2023-09-23T21:00:42.762 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:00:01.249943+0000", 2023-09-23T21:00:42.762 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00011900600000000001 2023-09-23T21:00:42.762 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:42.762 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:42.762 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:00:42.762 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:00:01.257468+0000", 2023-09-23T21:00:42.762 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.007524729 2023-09-23T21:00:42.762 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:42.762 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:42.763 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:00:42.763 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:00:01.257586+0000", 2023-09-23T21:00:42.763 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00011883 2023-09-23T21:00:42.763 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:42.763 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:42.763 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:00:42.763 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:00:01.337271+0000", 2023-09-23T21:00:42.763 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.079684028000000004 2023-09-23T21:00:42.763 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:42.763 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:42.763 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:00:42.763 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:00:01.337308+0000", 2023-09-23T21:00:42.764 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.7438000000000003e-05 2023-09-23T21:00:42.764 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:42.764 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:42.764 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:00:42.764 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:00:01.339269+0000", 2023-09-23T21:00:42.764 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0019606939999999998 2023-09-23T21:00:42.764 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:00:42.764 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:00:42.764 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:00:42.764 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:42.764 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:42.765 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.35100.0:168558 7.c 7:3c767518:::benchmark_data_smithi026_260489_object66796:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e243)", 2023-09-23T21:00:42.765 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:00:01.247879+0000", 2023-09-23T21:00:42.765 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 41.415756000999998, 2023-09-23T21:00:42.765 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.100175694, 2023-09-23T21:00:42.765 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:00:42.765 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:00:42.765 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:00:42.765 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.35100", 2023-09-23T21:00:42.765 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/4108972961", 2023-09-23T21:00:42.765 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 168558 2023-09-23T21:00:42.765 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:42.765 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:00:42.766 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:42.766 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:00:42.766 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:00:01.247879+0000", 2023-09-23T21:00:42.766 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:00:42.766 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:42.766 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:42.766 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:00:42.766 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:00:01.247879+0000", 2023-09-23T21:00:42.766 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:00:42.766 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:42.766 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:42.766 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:00:42.766 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:00:01.247879+0000", 2023-09-23T21:00:42.767 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999995 2023-09-23T21:00:42.767 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:42.767 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:42.767 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:00:42.767 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:00:01.247881+0000", 2023-09-23T21:00:42.767 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.362e-06 2023-09-23T21:00:42.767 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:42.767 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:42.767 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:00:42.767 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:00:01.247889+0000", 2023-09-23T21:00:42.767 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.2220000000000002e-06 2023-09-23T21:00:42.767 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:42.767 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:42.768 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:00:42.768 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:00:01.247893+0000", 2023-09-23T21:00:42.768 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.5729999999999999e-06 2023-09-23T21:00:42.768 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:42.768 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:42.768 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:00:42.768 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:00:01.253563+0000", 2023-09-23T21:00:42.768 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00567024 2023-09-23T21:00:42.768 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:42.768 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:42.768 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:00:42.768 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:00:01.253723+0000", 2023-09-23T21:00:42.768 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00016021500000000001 2023-09-23T21:00:42.769 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:42.769 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:42.769 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 3,7", 2023-09-23T21:00:42.769 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:00:01.253851+0000", 2023-09-23T21:00:42.769 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00012726600000000001 2023-09-23T21:00:42.769 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:42.769 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:42.769 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:00:42.769 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:00:01.257608+0000", 2023-09-23T21:00:42.769 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0037574180000000002 2023-09-23T21:00:42.769 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:42.769 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:42.770 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:00:42.770 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:00:01.257732+0000", 2023-09-23T21:00:42.770 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00012452000000000001 2023-09-23T21:00:42.770 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:42.770 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:42.770 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:00:42.770 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:00:01.345514+0000", 2023-09-23T21:00:42.770 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.087781146000000004 2023-09-23T21:00:42.770 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:42.770 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:42.770 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:00:42.770 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:00:01.345578+0000", 2023-09-23T21:00:42.771 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.4691999999999997e-05 2023-09-23T21:00:42.771 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:42.771 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:42.771 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:00:42.771 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:00:01.348055+0000", 2023-09-23T21:00:42.771 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0024764119999999999 2023-09-23T21:00:42.771 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:00:42.771 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:00:42.771 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:00:42.771 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:42.771 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:42.771 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.35100.0:171218 7.c 7:3a087124:::benchmark_data_smithi026_260489_object69456:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e244)", 2023-09-23T21:00:42.772 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:00:06.071740+0000", 2023-09-23T21:00:42.772 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 36.59189465, 2023-09-23T21:00:42.772 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.091567966000000001, 2023-09-23T21:00:42.772 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:00:42.772 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:00:42.772 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:00:42.772 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.35100", 2023-09-23T21:00:42.772 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/4108972961", 2023-09-23T21:00:42.772 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 171218 2023-09-23T21:00:42.772 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:42.772 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:00:42.772 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:42.773 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:00:42.773 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:00:06.071740+0000", 2023-09-23T21:00:42.773 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:00:42.773 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:42.773 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:42.773 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:00:42.773 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:00:06.071740+0000", 2023-09-23T21:00:42.773 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:00:42.773 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:42.773 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:42.773 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:00:42.773 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:00:06.071739+0000", 2023-09-23T21:00:42.773 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.999999 2023-09-23T21:00:42.774 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:42.774 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:42.774 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:00:42.774 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:00:06.071744+0000", 2023-09-23T21:00:42.774 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.0889999999999999e-06 2023-09-23T21:00:42.774 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:42.774 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:42.774 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:00:42.774 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:00:06.071756+0000", 2023-09-23T21:00:42.774 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.1184e-05 2023-09-23T21:00:42.774 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:42.775 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:42.775 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:00:42.775 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:00:06.071762+0000", 2023-09-23T21:00:42.775 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.8480000000000003e-06 2023-09-23T21:00:42.775 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:42.775 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:42.775 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:00:42.775 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:00:06.071810+0000", 2023-09-23T21:00:42.775 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.7756999999999997e-05 2023-09-23T21:00:42.775 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:42.775 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:42.775 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:00:42.776 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:00:06.071934+0000", 2023-09-23T21:00:42.776 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000123504 2023-09-23T21:00:42.776 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:42.776 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:42.776 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 3,7", 2023-09-23T21:00:42.776 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:00:06.072047+0000", 2023-09-23T21:00:42.776 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00011350599999999999 2023-09-23T21:00:42.776 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:42.776 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:42.776 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:00:42.776 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:00:06.078092+0000", 2023-09-23T21:00:42.776 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0060443750000000003 2023-09-23T21:00:42.777 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:42.777 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:42.777 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:00:42.777 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:00:06.078106+0000", 2023-09-23T21:00:42.777 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.4623e-05 2023-09-23T21:00:42.777 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:42.777 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:42.777 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:00:42.777 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:00:06.161295+0000", 2023-09-23T21:00:42.777 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.083188512000000006 2023-09-23T21:00:42.777 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:42.777 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:42.777 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:00:42.778 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:00:06.161363+0000", 2023-09-23T21:00:42.778 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.8614000000000004e-05 2023-09-23T21:00:42.778 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:42.778 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:42.778 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:00:42.778 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:00:06.163308+0000", 2023-09-23T21:00:42.778 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.001944985 2023-09-23T21:00:42.778 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:00:42.778 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:00:42.778 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:00:42.778 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:42.779 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:42.779 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.35100.0:171221 7.c 7:3c6c6944:::benchmark_data_smithi026_260489_object69459:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e244)", 2023-09-23T21:00:42.779 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:00:06.071777+0000", 2023-09-23T21:00:42.779 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 36.591858164999998, 2023-09-23T21:00:42.779 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.092901023999999999, 2023-09-23T21:00:42.779 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:00:42.779 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:00:42.779 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:00:42.779 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.35100", 2023-09-23T21:00:42.779 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/4108972961", 2023-09-23T21:00:42.779 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 171221 2023-09-23T21:00:42.779 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:42.780 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:00:42.780 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:42.780 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:00:42.780 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:00:06.071777+0000", 2023-09-23T21:00:42.780 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:00:42.780 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:42.780 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:42.780 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:00:42.780 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:00:06.071777+0000", 2023-09-23T21:00:42.780 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:00:42.780 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:42.780 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:42.781 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:00:42.781 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:00:06.071776+0000", 2023-09-23T21:00:42.781 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999995 2023-09-23T21:00:42.781 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:42.781 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:42.781 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:00:42.781 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:00:06.071779+0000", 2023-09-23T21:00:42.781 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.0460000000000001e-06 2023-09-23T21:00:42.781 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:42.781 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:42.781 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:00:42.781 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:00:06.071786+0000", 2023-09-23T21:00:42.782 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.7959999999999998e-06 2023-09-23T21:00:42.782 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:42.782 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:42.782 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:00:42.782 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:00:06.071791+0000", 2023-09-23T21:00:42.782 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.8180000000000003e-06 2023-09-23T21:00:42.782 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:42.782 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:42.782 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:00:42.782 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:00:06.075244+0000", 2023-09-23T21:00:42.782 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0034532849999999999 2023-09-23T21:00:42.782 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:42.783 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:42.783 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:00:42.783 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:00:06.075361+0000", 2023-09-23T21:00:42.783 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00011628 2023-09-23T21:00:42.783 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:42.783 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:42.783 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 3,7", 2023-09-23T21:00:42.783 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:00:06.075477+0000", 2023-09-23T21:00:42.783 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000116014 2023-09-23T21:00:42.783 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:42.783 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:42.783 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:00:42.784 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:00:06.078157+0000", 2023-09-23T21:00:42.784 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00268016 2023-09-23T21:00:42.784 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:42.784 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:42.784 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:00:42.784 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:00:06.078214+0000", 2023-09-23T21:00:42.784 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.7129000000000001e-05 2023-09-23T21:00:42.784 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:42.784 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:42.784 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:00:42.784 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:00:06.163374+0000", 2023-09-23T21:00:42.784 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.085160562999999995 2023-09-23T21:00:42.785 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:42.785 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:42.785 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:00:42.785 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:00:06.163417+0000", 2023-09-23T21:00:42.785 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.2329000000000001e-05 2023-09-23T21:00:42.785 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:42.785 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:42.785 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:00:42.785 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:00:06.164678+0000", 2023-09-23T21:00:42.785 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0012610550000000001 2023-09-23T21:00:42.785 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:00:42.785 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:00:42.785 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:00:42.786 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:42.786 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:42.786 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.35100.0:185573 7.c 7:301f47e9:::benchmark_data_smithi026_260489_object83811:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e246)", 2023-09-23T21:00:42.786 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:00:29.742836+0000", 2023-09-23T21:00:42.786 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 12.920798625, 2023-09-23T21:00:42.786 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.095769330999999999, 2023-09-23T21:00:42.786 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:00:42.786 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:00:42.786 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:00:42.786 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.35100", 2023-09-23T21:00:42.786 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/4108972961", 2023-09-23T21:00:42.787 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 185573 2023-09-23T21:00:42.787 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:42.787 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:00:42.787 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:42.787 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:00:42.787 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:00:29.742836+0000", 2023-09-23T21:00:42.787 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:00:42.787 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:42.787 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:42.787 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:00:42.787 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:00:29.742836+0000", 2023-09-23T21:00:42.787 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:00:42.787 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:42.788 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:42.788 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:00:42.788 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:00:29.742836+0000", 2023-09-23T21:00:42.788 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999995 2023-09-23T21:00:42.788 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:42.788 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:42.788 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:00:42.788 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:00:29.742839+0000", 2023-09-23T21:00:42.788 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.4539999999999999e-06 2023-09-23T21:00:42.788 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:42.788 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:42.788 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:00:42.789 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:00:29.742846+0000", 2023-09-23T21:00:42.789 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.9110000000000004e-06 2023-09-23T21:00:42.789 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:42.789 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:42.789 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:00:42.789 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:00:29.742852+0000", 2023-09-23T21:00:42.789 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.037e-06 2023-09-23T21:00:42.789 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:42.789 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:42.789 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:00:42.789 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:00:29.742901+0000", 2023-09-23T21:00:42.789 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.8470999999999997e-05 2023-09-23T21:00:42.789 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:42.789 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:42.790 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:00:42.790 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:00:29.743048+0000", 2023-09-23T21:00:42.790 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000147093 2023-09-23T21:00:42.790 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:42.790 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:42.790 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 3,7", 2023-09-23T21:00:42.790 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:00:29.743199+0000", 2023-09-23T21:00:42.790 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000150853 2023-09-23T21:00:42.790 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:42.790 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:42.790 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:00:42.791 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:00:29.749663+0000", 2023-09-23T21:00:42.791 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0064643019999999999 2023-09-23T21:00:42.791 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:42.791 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:42.791 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:00:42.791 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:00:29.749673+0000", 2023-09-23T21:00:42.791 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.6059999999999992e-06 2023-09-23T21:00:42.791 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:42.791 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:42.791 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:00:42.791 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:00:29.836854+0000", 2023-09-23T21:00:42.791 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.087180890999999996 2023-09-23T21:00:42.791 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:42.791 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:42.792 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:00:42.792 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:00:29.836913+0000", 2023-09-23T21:00:42.792 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.9373999999999998e-05 2023-09-23T21:00:42.792 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:42.792 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:42.792 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:00:42.792 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:00:29.838606+0000", 2023-09-23T21:00:42.792 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0016928119999999999 2023-09-23T21:00:42.792 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:00:42.792 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:00:42.792 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:00:42.792 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:42.792 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:42.793 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.35100.0:185577 7.c 7:341b734d:::benchmark_data_smithi026_260489_object83815:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e246)", 2023-09-23T21:00:42.793 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:00:29.751610+0000", 2023-09-23T21:00:42.793 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 12.912025025, 2023-09-23T21:00:42.793 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.088188907999999996, 2023-09-23T21:00:42.793 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:00:42.793 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:00:42.793 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:00:42.793 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.35100", 2023-09-23T21:00:42.793 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/4108972961", 2023-09-23T21:00:42.793 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 185577 2023-09-23T21:00:42.793 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:42.793 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:00:42.794 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:42.794 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:00:42.794 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:00:29.751610+0000", 2023-09-23T21:00:42.794 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:00:42.794 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:42.794 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:42.794 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:00:42.794 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:00:29.751610+0000", 2023-09-23T21:00:42.794 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:00:42.794 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:42.794 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:42.794 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:00:42.795 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:00:29.751608+0000", 2023-09-23T21:00:42.795 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999981 2023-09-23T21:00:42.795 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:42.795 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:42.795 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:00:42.795 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:00:29.751619+0000", 2023-09-23T21:00:42.795 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.0593e-05 2023-09-23T21:00:42.795 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:42.795 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:42.795 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:00:42.795 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:00:29.751636+0000", 2023-09-23T21:00:42.795 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.7456999999999999e-05 2023-09-23T21:00:42.795 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:42.796 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:42.796 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:00:42.796 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:00:29.751643+0000", 2023-09-23T21:00:42.796 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.888e-06 2023-09-23T21:00:42.796 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:42.796 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:42.796 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:00:42.796 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:00:29.751698+0000", 2023-09-23T21:00:42.796 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.4901999999999997e-05 2023-09-23T21:00:42.796 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:42.796 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:42.796 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:00:42.797 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:00:29.751895+0000", 2023-09-23T21:00:42.797 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00019697599999999999 2023-09-23T21:00:42.797 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:42.797 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:42.797 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 3,7", 2023-09-23T21:00:42.797 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:00:29.752092+0000", 2023-09-23T21:00:42.797 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00019664199999999999 2023-09-23T21:00:42.797 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:42.797 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:42.797 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:00:42.797 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:00:29.755131+0000", 2023-09-23T21:00:42.797 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0030398130000000002 2023-09-23T21:00:42.798 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:42.798 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:42.798 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:00:42.798 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:00:29.755633+0000", 2023-09-23T21:00:42.798 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00050131700000000004 2023-09-23T21:00:42.798 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:42.798 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:42.798 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:00:42.798 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:00:29.838674+0000", 2023-09-23T21:00:42.798 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.083041175999999994 2023-09-23T21:00:42.798 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:42.798 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:42.799 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:00:42.799 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:00:29.838738+0000", 2023-09-23T21:00:42.799 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.3665999999999997e-05 2023-09-23T21:00:42.799 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:42.799 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:42.799 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:00:42.799 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:00:29.839799+0000", 2023-09-23T21:00:42.799 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.001061331 2023-09-23T21:00:42.799 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:00:42.799 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:00:42.799 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:00:42.799 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:42.800 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:42.800 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.35100.0:191727 7.c 7:3d4724be:::benchmark_data_smithi026_260489_object89965:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e248)", 2023-09-23T21:00:42.800 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:00:39.842015+0000", 2023-09-23T21:00:42.800 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 2.8216197420000002, 2023-09-23T21:00:42.800 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.093273197000000002, 2023-09-23T21:00:42.800 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:00:42.800 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:00:42.800 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:00:42.800 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.35100", 2023-09-23T21:00:42.800 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/4108972961", 2023-09-23T21:00:42.800 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 191727 2023-09-23T21:00:42.800 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:42.801 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:00:42.801 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:42.801 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:00:42.801 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:00:39.842015+0000", 2023-09-23T21:00:42.801 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:00:42.801 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:42.801 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:42.801 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:00:42.801 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:00:39.842015+0000", 2023-09-23T21:00:42.801 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:00:42.801 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:42.801 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:42.802 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:00:42.802 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:00:39.842015+0000", 2023-09-23T21:00:42.802 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999995 2023-09-23T21:00:42.802 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:42.802 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:42.802 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:00:42.802 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:00:39.842017+0000", 2023-09-23T21:00:42.802 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.6290000000000002e-06 2023-09-23T21:00:42.802 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:42.802 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:42.802 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:00:42.802 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:00:39.842023+0000", 2023-09-23T21:00:42.803 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.9050000000000002e-06 2023-09-23T21:00:42.803 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:42.803 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:42.803 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:00:42.803 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:00:39.842027+0000", 2023-09-23T21:00:42.803 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.9779999999999999e-06 2023-09-23T21:00:42.803 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:42.803 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:42.803 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:00:42.803 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:00:39.849182+0000", 2023-09-23T21:00:42.803 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0071541549999999997 2023-09-23T21:00:42.803 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:42.804 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:42.804 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:00:42.804 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:00:39.849347+0000", 2023-09-23T21:00:42.804 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00016532999999999999 2023-09-23T21:00:42.804 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:42.804 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:42.804 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 3,7", 2023-09-23T21:00:42.804 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:00:39.849454+0000", 2023-09-23T21:00:42.804 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00010754100000000001 2023-09-23T21:00:42.804 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:42.804 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:42.804 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:00:42.805 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:00:39.853545+0000", 2023-09-23T21:00:42.805 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0040905259999999997 2023-09-23T21:00:42.805 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:42.805 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:42.805 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:00:42.805 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:00:39.856489+0000", 2023-09-23T21:00:42.805 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.002943701 2023-09-23T21:00:42.805 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:42.805 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:42.805 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:00:42.805 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:00:39.933771+0000", 2023-09-23T21:00:42.805 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.077281924000000002 2023-09-23T21:00:42.806 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:42.806 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:42.806 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:00:42.806 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:00:39.933807+0000", 2023-09-23T21:00:42.806 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.6025999999999997e-05 2023-09-23T21:00:42.806 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:00:42.806 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:00:42.806 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:00:42.806 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:00:39.935288+0000", 2023-09-23T21:00:42.806 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0014819119999999999 2023-09-23T21:00:42.806 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:00:42.806 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:00:42.807 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:00:42.807 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:00:42.807 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:00:42.807 INFO:teuthology.orchestra.run.smithi070.stdout:} 2023-09-23T21:00:42.807 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:00:42.808 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3: {} 2023-09-23T21:00:42.808 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4: {} 2023-09-23T21:00:42.808 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T21:00:42.820 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6: {} 2023-09-23T21:00:42.841 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:00:42.869 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T21:00:42.909 INFO:teuthology.orchestra.run.smithi026.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T21:00:42.910 INFO:teuthology.orchestra.run.smithi026.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T21:00:42.910 INFO:teuthology.orchestra.run.smithi026.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T21:00:42.910 INFO:teuthology.orchestra.run.smithi026.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T21:00:42.910 INFO:teuthology.orchestra.run.smithi026.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T21:00:42.910 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T21:00:42.910 INFO:teuthology.orchestra.run.smithi026.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T21:00:42.910 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T21:00:43.006 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:00:43.010 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:00:43.011 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:00:43.077 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_ops_in_flight 2023-09-23T21:00:43.111 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:00:43.115 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:00:43 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/648103089' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:00:43.211 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:00:43.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:00:43 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/648103089' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:00:43.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:00:43 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/648103089' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:00:43.675 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:00:43.682 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:00:43.682 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:00:43.782 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:00:43.784 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:00:43.978 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:00:44.119 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:00:44 smithi026 ceph-mon[179688]: pgmap v635: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 704 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 604 op/s; 11566/33816 objects degraded (34.203%); 576 KiB/s, 41 objects/s recovering 2023-09-23T21:00:44.121 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:00:44 smithi026 ceph-mon[175151]: pgmap v635: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 704 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 604 op/s; 11566/33816 objects degraded (34.203%); 576 KiB/s, 41 objects/s recovering 2023-09-23T21:00:44.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:00:44 smithi070 ceph-mon[168282]: pgmap v635: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 704 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 604 op/s; 11566/33816 objects degraded (34.203%); 576 KiB/s, 41 objects/s recovering 2023-09-23T21:00:44.751 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:00:44.755 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:00:44.755 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:00:44.857 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:00:44.887 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:00:44.934 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:00:44.939 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:00:44.940 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:00:45.041 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:00:45.129 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:00:45.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:00:45 smithi070 ceph-mon[168282]: Health check update: Degraded data redundancy: 11566/33816 objects degraded (34.203%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T21:00:45.383 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:00:45 smithi026 ceph-mon[179688]: Health check update: Degraded data redundancy: 11566/33816 objects degraded (34.203%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T21:00:45.386 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:00:45 smithi026 ceph-mon[175151]: Health check update: Degraded data redundancy: 11566/33816 objects degraded (34.203%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T21:00:45.658 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T21:00:45.854 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T21:00:45.859 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:00:45.860 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:00:45.961 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:00:46.056 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:00:46.284 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:00:46 smithi026 ceph-mon[175151]: pgmap v636: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 654 MiB data, 4.7 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 609 op/s; 11566/31422 objects degraded (36.809%); 576 KiB/s, 41 objects/s recovering 2023-09-23T21:00:46.288 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:00:46 smithi026 ceph-mon[179688]: pgmap v636: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 654 MiB data, 4.7 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 609 op/s; 11566/31422 objects degraded (36.809%); 576 KiB/s, 41 objects/s recovering 2023-09-23T21:00:46.308 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:00:46.317 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:00:46.317 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:00:46.418 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:00:46.444 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:00:46 smithi070 ceph-mon[168282]: pgmap v636: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 654 MiB data, 4.7 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 609 op/s; 11566/31422 objects degraded (36.809%); 576 KiB/s, 41 objects/s recovering 2023-09-23T21:00:46.446 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:00:46.484 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:00:46.491 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:00:46.494 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:00:46.494 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:00:46.595 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:00:46.676 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:00:47.042 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T21:00:47.054 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:00:47.055 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:00:47.156 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:00:47.184 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:00:47.229 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:00:47.231 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:00:47.232 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:00:47.242 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:00:47 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:00:47.242 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:00:47 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:00:47.332 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:00:47.361 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:00:47.400 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:00:47.402 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:00:47.403 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:00:47.504 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:00:47.517 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:00:47 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:00:47.517 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:00:47 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:00:47.525 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:00:47.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:00:47 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:00:47.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:00:47 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:00:47.797 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:00:48.188 INFO:teuthology.orchestra.run.smithi026.stdout: 2023-09-23T21:00:48.195 INFO:teuthology.orchestra.run.smithi026.stderr:dumped all 2023-09-23T21:00:48.274 INFO:teuthology.orchestra.run.smithi026.stderr:noscrub is set 2023-09-23T21:00:48.621 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:00:48 smithi070 ceph-mon[168282]: pgmap v637: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 618 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 611 op/s; 11566/29688 objects degraded (38.959%); 576 KiB/s, 41 objects/s recovering 2023-09-23T21:00:48.621 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:00:48 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/926386469' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T21:00:48.621 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:00:48 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T21:00:48.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:00:48 smithi026 ceph-mon[179688]: pgmap v637: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 618 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 611 op/s; 11566/29688 objects degraded (38.959%); 576 KiB/s, 41 objects/s recovering 2023-09-23T21:00:48.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:00:48 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/926386469' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T21:00:48.633 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:00:48 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T21:00:48.638 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:00:48 smithi026 ceph-mon[175151]: pgmap v637: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 618 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 611 op/s; 11566/29688 objects degraded (38.959%); 576 KiB/s, 41 objects/s recovering 2023-09-23T21:00:48.638 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:00:48 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/926386469' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T21:00:48.638 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:00:48 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T21:00:49.304 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T21:00:49.311 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:00:49.312 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:00:49.413 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:00:49.603 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:00:49 smithi026 ceph-mon[179688]: from='client.45419 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T21:00:49.603 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:00:49 smithi026 ceph-mon[179688]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:00:49.603 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:00:49 smithi026 ceph-mon[179688]: failed to encode map e249 with expected crc 2023-09-23T21:00:49.603 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:00:49 smithi026 ceph-mon[179688]: failed to encode map e249 with expected crc 2023-09-23T21:00:49.604 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:00:49 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T21:00:49.604 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:00:49 smithi026 ceph-mon[179688]: osdmap e249: 8 total, 8 up, 8 in 2023-09-23T21:00:49.604 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:00:49 smithi026 ceph-mon[179688]: failed to encode map e249 with expected crc 2023-09-23T21:00:49.604 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:00:49 smithi026 ceph-mon[179688]: failed to encode map e249 with expected crc 2023-09-23T21:00:49.606 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:00:49 smithi026 ceph-mon[175151]: from='client.45419 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T21:00:49.606 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:00:49 smithi026 ceph-mon[175151]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:00:49.606 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:00:49 smithi026 ceph-mon[175151]: failed to encode map e249 with expected crc 2023-09-23T21:00:49.606 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:00:49 smithi026 ceph-mon[175151]: failed to encode map e249 with expected crc 2023-09-23T21:00:49.607 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:00:49 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T21:00:49.607 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:00:49 smithi026 ceph-mon[175151]: osdmap e249: 8 total, 8 up, 8 in 2023-09-23T21:00:49.607 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:00:49 smithi026 ceph-mon[175151]: failed to encode map e249 with expected crc 2023-09-23T21:00:49.607 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:00:49 smithi026 ceph-mon[175151]: failed to encode map e249 with expected crc 2023-09-23T21:00:49.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:00:49 smithi070 ceph-mon[168282]: from='client.45419 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T21:00:49.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:00:49 smithi070 ceph-mon[168282]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:00:49.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:00:49 smithi070 ceph-mon[168282]: failed to encode map e249 with expected crc 2023-09-23T21:00:49.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:00:49 smithi070 ceph-mon[168282]: failed to encode map e249 with expected crc 2023-09-23T21:00:49.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:00:49 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T21:00:49.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:00:49 smithi070 ceph-mon[168282]: osdmap e249: 8 total, 8 up, 8 in 2023-09-23T21:00:49.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:00:49 smithi070 ceph-mon[168282]: failed to encode map e249 with expected crc 2023-09-23T21:00:49.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:00:49 smithi070 ceph-mon[168282]: failed to encode map e249 with expected crc 2023-09-23T21:00:49.799 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:00:49.882 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 21:00:49 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[164029]: ::ffff:172.21.15.70 - - [23/Sep/2023:21:00:49] "GET /metrics HTTP/1.1" 200 36309 "" "Prometheus/2.43.0" 2023-09-23T21:00:50.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:00:50 smithi070 ceph-mon[168282]: pgmap v639: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 510 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 630 op/s; 11338/24528 objects degraded (46.225%); 0 B/s, 45 objects/s recovering 2023-09-23T21:00:50.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:00:50 smithi070 ceph-mon[168282]: Health check update: Degraded data redundancy: 11338/24528 objects degraded (46.225%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T21:00:50.631 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:00:50 smithi026 ceph-mon[175151]: pgmap v639: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 510 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 630 op/s; 11338/24528 objects degraded (46.225%); 0 B/s, 45 objects/s recovering 2023-09-23T21:00:50.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:00:50 smithi026 ceph-mon[175151]: Health check update: Degraded data redundancy: 11338/24528 objects degraded (46.225%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T21:00:50.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:00:50 smithi026 ceph-mon[179688]: pgmap v639: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 510 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 630 op/s; 11338/24528 objects degraded (46.225%); 0 B/s, 45 objects/s recovering 2023-09-23T21:00:50.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:00:50 smithi026 ceph-mon[179688]: Health check update: Degraded data redundancy: 11338/24528 objects degraded (46.225%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T21:00:50.949 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T21:00:50.962 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:00:50.962 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:00:51.063 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:00:51.149 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:00:51.256 INFO:teuthology.orchestra.run.smithi026.stdout:{"pg_ready":true,"pg_map":{"version":637,"stamp":"2023-09-23T21:00:46.496976+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":647501752,"num_objects":9896,"num_object_clones":0,"num_object_copies":29688,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":11566,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":9896,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":193709,"num_write_kb":6512606,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4798,"num_bytes_recovered":291641190,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":122484,"ondisk_log_size":122484,"up":327,"acting":324,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":331,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":4789212,"kb_used_data":3020956,"kb_used_omap":136,"kb_used_meta":1767223,"kb_avail":745172004,"statfs":{"total":767960285184,"available":763056132096,"internally_reserved":0,"allocated":3093458944,"data_stored":3085179782,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":139741,"internal_metadata":1809636899},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,1,4,13,3,1],"upper_bound":128},"perf_stat":{"commit_latency_ms":10,"apply_latency_ms":10,"commit_latency_ns":10000000,"apply_latency_ns":10000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-401342464,"num_objects":-6124,"num_object_clones":0,"num_object_copies":-18372,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":-326,"num_objects_misplaced":-90,"num_objects_unfound":0,"num_objects_dirty":-6124,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6124,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":416,"num_bytes_recovered":5898240,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.006791"},"pg_stats":[{"pgid":"3.1a","version":"68'1","reported_seq":125,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-23T21:00:30.664065+0000","last_change":"2023-09-23T20:54:13.946695+0000","last_active":"2023-09-23T21:00:30.664065+0000","last_peered":"2023-09-23T21:00:30.664065+0000","last_clean":"2023-09-23T21:00:30.664065+0000","last_became_active":"2023-09-23T20:54:13.945399+0000","last_became_peered":"2023-09-23T20:54:13.945399+0000","last_unstale":"2023-09-23T21:00:30.664065+0000","last_undegraded":"2023-09-23T21:00:30.664065+0000","last_fullsized":"2023-09-23T21:00:30.664065+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:32:52.365464+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":240,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-23T21:00:30.655905+0000","last_change":"2023-09-23T20:54:13.942019+0000","last_active":"2023-09-23T21:00:30.655905+0000","last_peered":"2023-09-23T21:00:30.655905+0000","last_clean":"2023-09-23T21:00:30.655905+0000","last_became_active":"2023-09-23T20:54:13.941647+0000","last_became_peered":"2023-09-23T20:54:13.941647+0000","last_unstale":"2023-09-23T21:00:30.655905+0000","last_undegraded":"2023-09-23T21:00:30.655905+0000","last_fullsized":"2023-09-23T21:00:30.655905+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:30:54.400944+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":218,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-23T21:00:30.657340+0000","last_change":"2023-09-23T20:54:13.941394+0000","last_active":"2023-09-23T21:00:30.657340+0000","last_peered":"2023-09-23T21:00:30.657340+0000","last_clean":"2023-09-23T21:00:30.657340+0000","last_became_active":"2023-09-23T20:54:13.941042+0000","last_became_peered":"2023-09-23T20:54:13.941042+0000","last_unstale":"2023-09-23T21:00:30.657340+0000","last_undegraded":"2023-09-23T21:00:30.657340+0000","last_fullsized":"2023-09-23T21:00:30.657340+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:19:41.401184+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":233,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-23T21:00:30.668842+0000","last_change":"2023-09-23T20:57:44.979811+0000","last_active":"2023-09-23T21:00:30.668842+0000","last_peered":"2023-09-23T21:00:30.668842+0000","last_clean":"2023-09-23T21:00:30.668842+0000","last_became_active":"2023-09-23T20:57:44.979515+0000","last_became_peered":"2023-09-23T20:57:44.979515+0000","last_unstale":"2023-09-23T21:00:30.668842+0000","last_undegraded":"2023-09-23T21:00:30.668842+0000","last_fullsized":"2023-09-23T21:00:30.668842+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:38:22.162140+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":231,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-23T21:00:30.653330+0000","last_change":"2023-09-23T20:51:10.833613+0000","last_active":"2023-09-23T21:00:30.653330+0000","last_peered":"2023-09-23T21:00:30.653330+0000","last_clean":"2023-09-23T21:00:30.653330+0000","last_became_active":"2023-09-23T20:51:10.833208+0000","last_became_peered":"2023-09-23T20:51:10.833208+0000","last_unstale":"2023-09-23T21:00:30.653330+0000","last_undegraded":"2023-09-23T21:00:30.653330+0000","last_fullsized":"2023-09-23T21:00:30.653330+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:51:28.246014+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":213,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-23T21:00:30.647390+0000","last_change":"2023-09-23T20:55:35.958384+0000","last_active":"2023-09-23T21:00:30.647390+0000","last_peered":"2023-09-23T21:00:30.647390+0000","last_clean":"2023-09-23T21:00:30.647390+0000","last_became_active":"2023-09-23T20:55:35.954563+0000","last_became_peered":"2023-09-23T20:55:35.954563+0000","last_unstale":"2023-09-23T21:00:30.647390+0000","last_undegraded":"2023-09-23T21:00:30.647390+0000","last_fullsized":"2023-09-23T21:00:30.647390+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:37:48.332259+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":223,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-23T21:00:30.668703+0000","last_change":"2023-09-23T20:54:31.885514+0000","last_active":"2023-09-23T21:00:30.668703+0000","last_peered":"2023-09-23T21:00:30.668703+0000","last_clean":"2023-09-23T21:00:30.668703+0000","last_became_active":"2023-09-23T20:54:31.884986+0000","last_became_peered":"2023-09-23T20:54:31.884986+0000","last_unstale":"2023-09-23T21:00:30.668703+0000","last_undegraded":"2023-09-23T21:00:30.668703+0000","last_fullsized":"2023-09-23T21:00:30.668703+0000","mapping_epoch":136,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:26:08.100220+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":229,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-23T21:00:30.653399+0000","last_change":"2023-09-23T20:55:35.951889+0000","last_active":"2023-09-23T21:00:30.653399+0000","last_peered":"2023-09-23T21:00:30.653399+0000","last_clean":"2023-09-23T21:00:30.653399+0000","last_became_active":"2023-09-23T20:55:35.951563+0000","last_became_peered":"2023-09-23T20:55:35.951563+0000","last_unstale":"2023-09-23T21:00:30.653399+0000","last_undegraded":"2023-09-23T21:00:30.653399+0000","last_fullsized":"2023-09-23T21:00:30.653399+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:41:53.098781+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"79'161","reported_seq":535,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-23T21:00:30.668070+0000","last_change":"2023-09-23T20:57:25.096559+0000","last_active":"2023-09-23T21:00:30.668070+0000","last_peered":"2023-09-23T21:00:30.668070+0000","last_clean":"2023-09-23T21:00:30.668070+0000","last_became_active":"2023-09-23T20:57:15.975674+0000","last_became_peered":"2023-09-23T20:57:15.975674+0000","last_unstale":"2023-09-23T21:00:30.668070+0000","last_undegraded":"2023-09-23T21:00:30.668070+0000","last_fullsized":"2023-09-23T21:00:30.668070+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:25.096401+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:25.096401+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:05:05.448404+0000","scrub_duration":0.002859393,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":121,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-23T21:00:30.650549+0000","last_change":"2023-09-23T20:54:31.890678+0000","last_active":"2023-09-23T21:00:30.650549+0000","last_peered":"2023-09-23T21:00:30.650549+0000","last_clean":"2023-09-23T21:00:30.650549+0000","last_became_active":"2023-09-23T20:54:31.890469+0000","last_became_peered":"2023-09-23T20:54:31.890469+0000","last_unstale":"2023-09-23T21:00:30.650549+0000","last_undegraded":"2023-09-23T21:00:30.650549+0000","last_fullsized":"2023-09-23T21:00:30.650549+0000","mapping_epoch":136,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:55:08.790146+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":190,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-23T21:00:30.663602+0000","last_change":"2023-09-23T20:55:35.952661+0000","last_active":"2023-09-23T21:00:30.663602+0000","last_peered":"2023-09-23T21:00:30.663602+0000","last_clean":"2023-09-23T21:00:30.663602+0000","last_became_active":"2023-09-23T20:55:35.952349+0000","last_became_peered":"2023-09-23T20:55:35.952349+0000","last_unstale":"2023-09-23T21:00:30.663602+0000","last_undegraded":"2023-09-23T21:00:30.663602+0000","last_fullsized":"2023-09-23T21:00:30.663602+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:00:03.110126+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":66,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-23T21:00:30.655984+0000","last_change":"2023-09-23T20:56:45.988607+0000","last_active":"2023-09-23T21:00:30.655984+0000","last_peered":"2023-09-23T21:00:30.655984+0000","last_clean":"2023-09-23T21:00:30.655984+0000","last_became_active":"2023-09-23T20:56:45.988206+0000","last_became_peered":"2023-09-23T20:56:45.988206+0000","last_unstale":"2023-09-23T21:00:30.655984+0000","last_undegraded":"2023-09-23T21:00:30.655984+0000","last_fullsized":"2023-09-23T21:00:30.655984+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":205,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:01:19.783688+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,4],"acting":[3,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":65,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-23T21:00:30.650013+0000","last_change":"2023-09-23T20:56:18.139142+0000","last_active":"2023-09-23T21:00:30.650013+0000","last_peered":"2023-09-23T21:00:30.650013+0000","last_clean":"2023-09-23T21:00:30.650013+0000","last_became_active":"2023-09-23T20:56:18.138844+0000","last_became_peered":"2023-09-23T20:56:18.138844+0000","last_unstale":"2023-09-23T21:00:30.650013+0000","last_undegraded":"2023-09-23T21:00:30.650013+0000","last_fullsized":"2023-09-23T21:00:30.650013+0000","mapping_epoch":192,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":193,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:09:02.217683+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,2],"acting":[5,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.14","version":"79'22","reported_seq":152,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-23T21:00:30.664707+0000","last_change":"2023-09-23T20:55:35.949418+0000","last_active":"2023-09-23T21:00:30.664707+0000","last_peered":"2023-09-23T21:00:30.664707+0000","last_clean":"2023-09-23T21:00:30.664707+0000","last_became_active":"2023-09-23T20:55:35.948566+0000","last_became_peered":"2023-09-23T20:55:35.948566+0000","last_unstale":"2023-09-23T21:00:30.664707+0000","last_undegraded":"2023-09-23T21:00:30.664707+0000","last_fullsized":"2023-09-23T21:00:30.664707+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:59:37.433128+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4734,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":142,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-23T21:00:30.664354+0000","last_change":"2023-09-23T20:55:35.949178+0000","last_active":"2023-09-23T21:00:30.664354+0000","last_peered":"2023-09-23T21:00:30.664354+0000","last_clean":"2023-09-23T21:00:30.664354+0000","last_became_active":"2023-09-23T20:55:35.948795+0000","last_became_peered":"2023-09-23T20:55:35.948795+0000","last_unstale":"2023-09-23T21:00:30.664354+0000","last_undegraded":"2023-09-23T21:00:30.664354+0000","last_fullsized":"2023-09-23T21:00:30.664354+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:13:08.996932+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.11","version":"79'161","reported_seq":477,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-23T21:00:30.661705+0000","last_change":"2023-09-23T20:57:28.756796+0000","last_active":"2023-09-23T21:00:30.661705+0000","last_peered":"2023-09-23T21:00:30.661705+0000","last_clean":"2023-09-23T21:00:30.661705+0000","last_became_active":"2023-09-23T20:57:15.980606+0000","last_became_peered":"2023-09-23T20:57:15.980606+0000","last_unstale":"2023-09-23T21:00:30.661705+0000","last_undegraded":"2023-09-23T21:00:30.661705+0000","last_fullsized":"2023-09-23T21:00:30.661705+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:28.756634+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:28.756634+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:09:19.150791+0000","scrub_duration":0.0064925119999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":168,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-23T21:00:30.664237+0000","last_change":"2023-09-23T20:56:45.992577+0000","last_active":"2023-09-23T21:00:30.664237+0000","last_peered":"2023-09-23T21:00:30.664237+0000","last_clean":"2023-09-23T21:00:30.664237+0000","last_became_active":"2023-09-23T20:56:45.992297+0000","last_became_peered":"2023-09-23T20:56:45.992297+0000","last_unstale":"2023-09-23T21:00:30.664237+0000","last_undegraded":"2023-09-23T21:00:30.664237+0000","last_fullsized":"2023-09-23T21:00:30.664237+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":205,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:00:45.547202+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,2],"acting":[4,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":214,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-23T21:00:30.660622+0000","last_change":"2023-09-23T20:51:10.831262+0000","last_active":"2023-09-23T21:00:30.660622+0000","last_peered":"2023-09-23T21:00:30.660622+0000","last_clean":"2023-09-23T21:00:30.660622+0000","last_became_active":"2023-09-23T20:51:10.830931+0000","last_became_peered":"2023-09-23T20:51:10.830931+0000","last_unstale":"2023-09-23T21:00:30.660622+0000","last_undegraded":"2023-09-23T21:00:30.660622+0000","last_fullsized":"2023-09-23T21:00:30.660622+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:51:54.982505+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.10","version":"79'161","reported_seq":485,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-23T21:00:30.649612+0000","last_change":"2023-09-23T20:57:20.708003+0000","last_active":"2023-09-23T21:00:30.649612+0000","last_peered":"2023-09-23T21:00:30.649612+0000","last_clean":"2023-09-23T21:00:30.649612+0000","last_became_active":"2023-09-23T20:57:15.975302+0000","last_became_peered":"2023-09-23T20:57:15.975302+0000","last_unstale":"2023-09-23T21:00:30.649612+0000","last_undegraded":"2023-09-23T21:00:30.649612+0000","last_fullsized":"2023-09-23T21:00:30.649612+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:20.707939+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:20.707939+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:16:51.721994+0000","scrub_duration":0.0017733409999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":223,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-23T21:00:30.661538+0000","last_change":"2023-09-23T20:57:44.979736+0000","last_active":"2023-09-23T21:00:30.661538+0000","last_peered":"2023-09-23T21:00:30.661538+0000","last_clean":"2023-09-23T21:00:30.661538+0000","last_became_active":"2023-09-23T20:57:44.979356+0000","last_became_peered":"2023-09-23T20:57:44.979356+0000","last_unstale":"2023-09-23T21:00:30.661538+0000","last_undegraded":"2023-09-23T21:00:30.661538+0000","last_fullsized":"2023-09-23T21:00:30.661538+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:14:43.848832+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":121,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-23T21:00:30.650398+0000","last_change":"2023-09-23T20:54:31.887814+0000","last_active":"2023-09-23T21:00:30.650398+0000","last_peered":"2023-09-23T21:00:30.650398+0000","last_clean":"2023-09-23T21:00:30.650398+0000","last_became_active":"2023-09-23T20:54:31.887195+0000","last_became_peered":"2023-09-23T20:54:31.887195+0000","last_unstale":"2023-09-23T21:00:30.650398+0000","last_undegraded":"2023-09-23T21:00:30.650398+0000","last_fullsized":"2023-09-23T21:00:30.650398+0000","mapping_epoch":136,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:10:52.089553+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.13","version":"79'161","reported_seq":548,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-23T21:00:30.658168+0000","last_change":"2023-09-23T20:57:48.262970+0000","last_active":"2023-09-23T21:00:30.658168+0000","last_peered":"2023-09-23T21:00:30.658168+0000","last_clean":"2023-09-23T21:00:30.658168+0000","last_became_active":"2023-09-23T20:57:48.262663+0000","last_became_peered":"2023-09-23T20:57:48.262663+0000","last_unstale":"2023-09-23T21:00:30.658168+0000","last_undegraded":"2023-09-23T21:00:30.658168+0000","last_fullsized":"2023-09-23T21:00:30.658168+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:36.420362+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:36.420362+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:28:28.613492+0000","scrub_duration":0.028838444000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":192,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-23T21:00:30.664049+0000","last_change":"2023-09-23T20:56:45.991822+0000","last_active":"2023-09-23T21:00:30.664049+0000","last_peered":"2023-09-23T21:00:30.664049+0000","last_clean":"2023-09-23T21:00:30.664049+0000","last_became_active":"2023-09-23T20:56:45.991460+0000","last_became_peered":"2023-09-23T20:56:45.991460+0000","last_unstale":"2023-09-23T21:00:30.664049+0000","last_undegraded":"2023-09-23T21:00:30.664049+0000","last_fullsized":"2023-09-23T21:00:30.664049+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":205,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:20:37.503317+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":205,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-23T21:00:30.668979+0000","last_change":"2023-09-23T20:51:10.840849+0000","last_active":"2023-09-23T21:00:30.668979+0000","last_peered":"2023-09-23T21:00:30.668979+0000","last_clean":"2023-09-23T21:00:30.668979+0000","last_became_active":"2023-09-23T20:51:10.840593+0000","last_became_peered":"2023-09-23T20:51:10.840593+0000","last_unstale":"2023-09-23T21:00:30.668979+0000","last_undegraded":"2023-09-23T21:00:30.668979+0000","last_fullsized":"2023-09-23T21:00:30.668979+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:30:52.297625+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.12","version":"79'161","reported_seq":485,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-23T21:00:30.649694+0000","last_change":"2023-09-23T20:57:21.750765+0000","last_active":"2023-09-23T21:00:30.649694+0000","last_peered":"2023-09-23T21:00:30.649694+0000","last_clean":"2023-09-23T21:00:30.649694+0000","last_became_active":"2023-09-23T20:57:15.979195+0000","last_became_peered":"2023-09-23T20:57:15.979195+0000","last_unstale":"2023-09-23T21:00:30.649694+0000","last_undegraded":"2023-09-23T21:00:30.649694+0000","last_fullsized":"2023-09-23T21:00:30.649694+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:21.750680+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:21.750680+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:03:41.915962+0000","scrub_duration":0.0092391680000000007,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":240,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-23T21:00:30.646966+0000","last_change":"2023-09-23T20:55:36.943969+0000","last_active":"2023-09-23T21:00:30.646966+0000","last_peered":"2023-09-23T21:00:30.646966+0000","last_clean":"2023-09-23T21:00:30.646966+0000","last_became_active":"2023-09-23T20:55:36.943580+0000","last_became_peered":"2023-09-23T20:55:36.943580+0000","last_unstale":"2023-09-23T21:00:30.646966+0000","last_undegraded":"2023-09-23T21:00:30.646966+0000","last_fullsized":"2023-09-23T21:00:30.646966+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:49:06.969396+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":37,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-23T21:00:30.661329+0000","last_change":"2023-09-23T20:57:44.980498+0000","last_active":"2023-09-23T21:00:30.661329+0000","last_peered":"2023-09-23T21:00:30.661329+0000","last_clean":"2023-09-23T21:00:30.661329+0000","last_became_active":"2023-09-23T20:57:44.980182+0000","last_became_peered":"2023-09-23T20:57:44.980182+0000","last_unstale":"2023-09-23T21:00:30.661329+0000","last_undegraded":"2023-09-23T21:00:30.661329+0000","last_fullsized":"2023-09-23T21:00:30.661329+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:34:51.620544+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.d","version":"79'161","reported_seq":530,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-23T21:00:30.664219+0000","last_change":"2023-09-23T20:57:31.075213+0000","last_active":"2023-09-23T21:00:30.664219+0000","last_peered":"2023-09-23T21:00:30.664219+0000","last_clean":"2023-09-23T21:00:30.664219+0000","last_became_active":"2023-09-23T20:57:15.988188+0000","last_became_peered":"2023-09-23T20:57:15.988188+0000","last_unstale":"2023-09-23T21:00:30.664219+0000","last_undegraded":"2023-09-23T21:00:30.664219+0000","last_fullsized":"2023-09-23T21:00:30.664219+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:31.075098+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:31.075098+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:41:52.142362+0000","scrub_duration":0.010304610000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.b","version":"248'12118","reported_seq":12177,"reported_epoch":248,"state":"active+clean","last_fresh":"2023-09-23T21:00:44.358137+0000","last_change":"2023-09-23T20:56:35.543994+0000","last_active":"2023-09-23T21:00:44.358137+0000","last_peered":"2023-09-23T21:00:44.358137+0000","last_clean":"2023-09-23T21:00:44.358137+0000","last_became_active":"2023-09-23T20:56:35.543758+0000","last_became_peered":"2023-09-23T20:56:35.543758+0000","last_unstale":"2023-09-23T21:00:44.358137+0000","last_undegraded":"2023-09-23T21:00:44.358137+0000","last_fullsized":"2023-09-23T21:00:44.358137+0000","mapping_epoch":199,"log_start":"220'5100","ondisk_log_start":"220'5100","created":199,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_clean_scrub_stamp":"2023-09-23T20:56:34.529103+0000","objects_scrubbed":0,"log_size":7018,"log_dups_size":0,"ondisk_log_size":7018,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:21:29.041484+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":36831232,"num_objects":562,"num_object_clones":0,"num_object_copies":1686,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":562,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12118,"num_write_kb":405760,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":179,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-23T21:00:30.666636+0000","last_change":"2023-09-23T20:56:45.991841+0000","last_active":"2023-09-23T21:00:30.666636+0000","last_peered":"2023-09-23T21:00:30.666636+0000","last_clean":"2023-09-23T21:00:30.666636+0000","last_became_active":"2023-09-23T20:56:45.991491+0000","last_became_peered":"2023-09-23T20:56:45.991491+0000","last_unstale":"2023-09-23T21:00:30.666636+0000","last_undegraded":"2023-09-23T21:00:30.666636+0000","last_fullsized":"2023-09-23T21:00:30.666636+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":205,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:30:37.405319+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":138,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-23T21:00:30.649706+0000","last_change":"2023-09-23T20:55:35.957768+0000","last_active":"2023-09-23T21:00:30.649706+0000","last_peered":"2023-09-23T21:00:30.649706+0000","last_clean":"2023-09-23T21:00:30.649706+0000","last_became_active":"2023-09-23T20:55:35.957421+0000","last_became_peered":"2023-09-23T20:55:35.957421+0000","last_unstale":"2023-09-23T21:00:30.649706+0000","last_undegraded":"2023-09-23T21:00:30.649706+0000","last_fullsized":"2023-09-23T21:00:30.649706+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:49:41.585961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.c","version":"79'161","reported_seq":549,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-23T21:00:30.660448+0000","last_change":"2023-09-23T20:57:22.041071+0000","last_active":"2023-09-23T21:00:30.660448+0000","last_peered":"2023-09-23T21:00:30.660448+0000","last_clean":"2023-09-23T21:00:30.660448+0000","last_became_active":"2023-09-23T20:57:15.973204+0000","last_became_peered":"2023-09-23T20:57:15.973204+0000","last_unstale":"2023-09-23T21:00:30.660448+0000","last_undegraded":"2023-09-23T21:00:30.660448+0000","last_fullsized":"2023-09-23T21:00:30.660448+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:22.040907+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:22.040907+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:47:02.019978+0000","scrub_duration":0.013021176000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.a","version":"248'12155","reported_seq":12214,"reported_epoch":248,"state":"active+clean","last_fresh":"2023-09-23T21:00:44.767981+0000","last_change":"2023-09-23T20:56:35.547120+0000","last_active":"2023-09-23T21:00:44.767981+0000","last_peered":"2023-09-23T21:00:44.767981+0000","last_clean":"2023-09-23T21:00:44.767981+0000","last_became_active":"2023-09-23T20:56:35.546838+0000","last_became_peered":"2023-09-23T20:56:35.546838+0000","last_unstale":"2023-09-23T21:00:44.767981+0000","last_undegraded":"2023-09-23T21:00:44.767981+0000","last_fullsized":"2023-09-23T21:00:44.767981+0000","mapping_epoch":199,"log_start":"229'6000","ondisk_log_start":"229'6000","created":199,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_clean_scrub_stamp":"2023-09-23T20:56:34.529103+0000","objects_scrubbed":0,"log_size":6155,"log_dups_size":0,"ondisk_log_size":6155,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:48:24.132768+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":38076416,"num_objects":581,"num_object_clones":0,"num_object_copies":1743,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":581,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12155,"num_write_kb":407552,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,5],"acting":[3,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":166,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-23T21:00:30.666565+0000","last_change":"2023-09-23T20:56:45.991821+0000","last_active":"2023-09-23T21:00:30.666565+0000","last_peered":"2023-09-23T21:00:30.666565+0000","last_clean":"2023-09-23T21:00:30.666565+0000","last_became_active":"2023-09-23T20:56:45.991447+0000","last_became_peered":"2023-09-23T20:56:45.991447+0000","last_unstale":"2023-09-23T21:00:30.666565+0000","last_undegraded":"2023-09-23T21:00:30.666565+0000","last_fullsized":"2023-09-23T21:00:30.666565+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":205,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:14:25.525222+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":169,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-23T21:00:30.666224+0000","last_change":"2023-09-23T20:57:44.982059+0000","last_active":"2023-09-23T21:00:30.666224+0000","last_peered":"2023-09-23T21:00:30.666224+0000","last_clean":"2023-09-23T21:00:30.666224+0000","last_became_active":"2023-09-23T20:57:44.981742+0000","last_became_peered":"2023-09-23T20:57:44.981742+0000","last_unstale":"2023-09-23T21:00:30.666224+0000","last_undegraded":"2023-09-23T21:00:30.666224+0000","last_fullsized":"2023-09-23T21:00:30.666224+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:41:24.042916+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.f","version":"79'161","reported_seq":552,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-23T21:00:30.647034+0000","last_change":"2023-09-23T20:57:27.880841+0000","last_active":"2023-09-23T21:00:30.647034+0000","last_peered":"2023-09-23T21:00:30.647034+0000","last_clean":"2023-09-23T21:00:30.647034+0000","last_became_active":"2023-09-23T20:57:15.970925+0000","last_became_peered":"2023-09-23T20:57:15.970925+0000","last_unstale":"2023-09-23T21:00:30.647034+0000","last_undegraded":"2023-09-23T21:00:30.647034+0000","last_fullsized":"2023-09-23T21:00:30.647034+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:27.880776+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:27.880776+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:15:13.438014+0000","scrub_duration":0.0014004899999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.9","version":"248'12065","reported_seq":12124,"reported_epoch":248,"state":"active+clean","last_fresh":"2023-09-23T21:00:41.821868+0000","last_change":"2023-09-23T20:56:35.549387+0000","last_active":"2023-09-23T21:00:41.821868+0000","last_peered":"2023-09-23T21:00:41.821868+0000","last_clean":"2023-09-23T21:00:41.821868+0000","last_became_active":"2023-09-23T20:56:35.549087+0000","last_became_peered":"2023-09-23T20:56:35.549087+0000","last_unstale":"2023-09-23T21:00:41.821868+0000","last_undegraded":"2023-09-23T21:00:41.821868+0000","last_fullsized":"2023-09-23T21:00:41.821868+0000","mapping_epoch":199,"log_start":"219'4700","ondisk_log_start":"219'4700","created":199,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_clean_scrub_stamp":"2023-09-23T20:56:34.529103+0000","objects_scrubbed":0,"log_size":7365,"log_dups_size":0,"ondisk_log_size":7365,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:04:51.547362+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":42008576,"num_objects":641,"num_object_clones":0,"num_object_copies":1923,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":641,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12065,"num_write_kb":406592,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":152,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-23T21:00:30.650325+0000","last_change":"2023-09-23T20:56:45.989153+0000","last_active":"2023-09-23T21:00:30.650325+0000","last_peered":"2023-09-23T21:00:30.650325+0000","last_clean":"2023-09-23T21:00:30.650325+0000","last_became_active":"2023-09-23T20:56:45.988800+0000","last_became_peered":"2023-09-23T20:56:45.988800+0000","last_unstale":"2023-09-23T21:00:30.650325+0000","last_undegraded":"2023-09-23T21:00:30.650325+0000","last_fullsized":"2023-09-23T21:00:30.650325+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":205,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:04:41.130194+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":187,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-23T21:00:30.646727+0000","last_change":"2023-09-23T20:55:35.965427+0000","last_active":"2023-09-23T21:00:30.646727+0000","last_peered":"2023-09-23T21:00:30.646727+0000","last_clean":"2023-09-23T21:00:30.646727+0000","last_became_active":"2023-09-23T20:55:35.963969+0000","last_became_peered":"2023-09-23T20:55:35.963969+0000","last_unstale":"2023-09-23T21:00:30.646727+0000","last_undegraded":"2023-09-23T21:00:30.646727+0000","last_fullsized":"2023-09-23T21:00:30.646727+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:38:40.612083+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.e","version":"79'161","reported_seq":459,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-23T21:00:30.664207+0000","last_change":"2023-09-23T20:57:34.132738+0000","last_active":"2023-09-23T21:00:30.664207+0000","last_peered":"2023-09-23T21:00:30.664207+0000","last_clean":"2023-09-23T21:00:30.664207+0000","last_became_active":"2023-09-23T20:57:15.974318+0000","last_became_peered":"2023-09-23T20:57:15.974318+0000","last_unstale":"2023-09-23T21:00:30.664207+0000","last_undegraded":"2023-09-23T21:00:30.664207+0000","last_fullsized":"2023-09-23T21:00:30.664207+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:34.132660+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:34.132660+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:07:07.739438+0000","scrub_duration":0.005229694,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.8","version":"248'12160","reported_seq":12219,"reported_epoch":248,"state":"active+clean","last_fresh":"2023-09-23T21:00:41.458713+0000","last_change":"2023-09-23T20:56:35.548119+0000","last_active":"2023-09-23T21:00:41.458713+0000","last_peered":"2023-09-23T21:00:41.458713+0000","last_clean":"2023-09-23T21:00:41.458713+0000","last_became_active":"2023-09-23T20:56:35.547914+0000","last_became_peered":"2023-09-23T20:56:35.547914+0000","last_unstale":"2023-09-23T21:00:41.458713+0000","last_undegraded":"2023-09-23T21:00:41.458713+0000","last_fullsized":"2023-09-23T21:00:41.458713+0000","mapping_epoch":199,"log_start":"210'3300","ondisk_log_start":"210'3300","created":199,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_clean_scrub_stamp":"2023-09-23T20:56:34.529103+0000","objects_scrubbed":0,"log_size":8860,"log_dups_size":0,"ondisk_log_size":8860,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:42:54.255023+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":45481984,"num_objects":694,"num_object_clones":0,"num_object_copies":2082,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":694,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12160,"num_write_kb":411328,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":138,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-23T21:00:30.650078+0000","last_change":"2023-09-23T20:55:35.962005+0000","last_active":"2023-09-23T21:00:30.650078+0000","last_peered":"2023-09-23T21:00:30.650078+0000","last_clean":"2023-09-23T21:00:30.650078+0000","last_became_active":"2023-09-23T20:55:35.961695+0000","last_became_peered":"2023-09-23T20:55:35.961695+0000","last_unstale":"2023-09-23T21:00:30.650078+0000","last_undegraded":"2023-09-23T21:00:30.650078+0000","last_fullsized":"2023-09-23T21:00:30.650078+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:28:22.876201+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":236,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-23T21:00:30.661645+0000","last_change":"2023-09-23T20:54:13.928715+0000","last_active":"2023-09-23T21:00:30.661645+0000","last_peered":"2023-09-23T21:00:30.661645+0000","last_clean":"2023-09-23T21:00:30.661645+0000","last_became_active":"2023-09-23T20:54:13.928200+0000","last_became_peered":"2023-09-23T20:54:13.928200+0000","last_unstale":"2023-09-23T21:00:30.661645+0000","last_undegraded":"2023-09-23T21:00:30.661645+0000","last_fullsized":"2023-09-23T21:00:30.661645+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:41:16.927513+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.f","version":"248'12235","reported_seq":12294,"reported_epoch":248,"state":"active+clean","last_fresh":"2023-09-23T21:00:41.777374+0000","last_change":"2023-09-23T20:56:35.545128+0000","last_active":"2023-09-23T21:00:41.777374+0000","last_peered":"2023-09-23T21:00:41.777374+0000","last_clean":"2023-09-23T21:00:41.777374+0000","last_became_active":"2023-09-23T20:56:35.544838+0000","last_became_peered":"2023-09-23T20:56:35.544838+0000","last_unstale":"2023-09-23T21:00:41.777374+0000","last_undegraded":"2023-09-23T21:00:41.777374+0000","last_fullsized":"2023-09-23T21:00:41.777374+0000","mapping_epoch":199,"log_start":"219'4900","ondisk_log_start":"219'4900","created":199,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_clean_scrub_stamp":"2023-09-23T20:56:34.529103+0000","objects_scrubbed":0,"log_size":7335,"log_dups_size":0,"ondisk_log_size":7335,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:30:43.388717+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":41353216,"num_objects":631,"num_object_clones":0,"num_object_copies":1893,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":631,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12235,"num_write_kb":411712,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.9","version":"79'161","reported_seq":621,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-23T21:00:30.663517+0000","last_change":"2023-09-23T20:56:46.921939+0000","last_active":"2023-09-23T21:00:30.663517+0000","last_peered":"2023-09-23T21:00:30.663517+0000","last_clean":"2023-09-23T21:00:30.663517+0000","last_became_active":"2023-09-23T20:56:45.989908+0000","last_became_peered":"2023-09-23T20:56:45.989908+0000","last_unstale":"2023-09-23T21:00:30.663517+0000","last_undegraded":"2023-09-23T21:00:30.663517+0000","last_fullsized":"2023-09-23T21:00:30.663517+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":205,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:56:46.921739+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:56:46.921739+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:20:42.635110+0000","scrub_duration":0.0071954109999999996,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":238,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-23T21:00:30.655698+0000","last_change":"2023-09-23T20:54:13.961788+0000","last_active":"2023-09-23T21:00:30.655698+0000","last_peered":"2023-09-23T21:00:30.655698+0000","last_clean":"2023-09-23T21:00:30.655698+0000","last_became_active":"2023-09-23T20:54:13.940937+0000","last_became_peered":"2023-09-23T20:54:13.940937+0000","last_unstale":"2023-09-23T21:00:30.655698+0000","last_undegraded":"2023-09-23T21:00:30.655698+0000","last_fullsized":"2023-09-23T21:00:30.655698+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:11:56.785299+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":142,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-23T21:00:30.664844+0000","last_change":"2023-09-23T20:55:35.949670+0000","last_active":"2023-09-23T21:00:30.664844+0000","last_peered":"2023-09-23T21:00:30.664844+0000","last_clean":"2023-09-23T21:00:30.664844+0000","last_became_active":"2023-09-23T20:55:35.949140+0000","last_became_peered":"2023-09-23T20:55:35.949140+0000","last_unstale":"2023-09-23T21:00:30.664844+0000","last_undegraded":"2023-09-23T21:00:30.664844+0000","last_fullsized":"2023-09-23T21:00:30.664844+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:55:59.340303+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.e","version":"248'12094","reported_seq":12153,"reported_epoch":248,"state":"active+clean","last_fresh":"2023-09-23T21:00:41.548977+0000","last_change":"2023-09-23T20:56:35.547184+0000","last_active":"2023-09-23T21:00:41.548977+0000","last_peered":"2023-09-23T21:00:41.548977+0000","last_clean":"2023-09-23T21:00:41.548977+0000","last_became_active":"2023-09-23T20:56:35.546912+0000","last_became_peered":"2023-09-23T20:56:35.546912+0000","last_unstale":"2023-09-23T21:00:41.548977+0000","last_undegraded":"2023-09-23T21:00:41.548977+0000","last_fullsized":"2023-09-23T21:00:41.548977+0000","mapping_epoch":199,"log_start":"209'3200","ondisk_log_start":"209'3200","created":199,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_clean_scrub_stamp":"2023-09-23T20:56:34.529103+0000","objects_scrubbed":0,"log_size":8894,"log_dups_size":0,"ondisk_log_size":8894,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:57:07.422792+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":44957696,"num_objects":686,"num_object_clones":0,"num_object_copies":2058,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":686,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12094,"num_write_kb":408960,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,0],"acting":[2,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.8","version":"79'161","reported_seq":491,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-23T21:00:30.649710+0000","last_change":"2023-09-23T20:56:57.505737+0000","last_active":"2023-09-23T21:00:30.649710+0000","last_peered":"2023-09-23T21:00:30.649710+0000","last_clean":"2023-09-23T21:00:30.649710+0000","last_became_active":"2023-09-23T20:56:04.982345+0000","last_became_peered":"2023-09-23T20:56:04.982345+0000","last_unstale":"2023-09-23T21:00:30.649710+0000","last_undegraded":"2023-09-23T21:00:30.649710+0000","last_fullsized":"2023-09-23T21:00:30.649710+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:56:57.505663+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:56:57.505663+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:34:30.109724+0000","scrub_duration":0.026284108,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":210,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-23T21:00:30.646873+0000","last_change":"2023-09-23T20:55:35.959942+0000","last_active":"2023-09-23T21:00:30.646873+0000","last_peered":"2023-09-23T21:00:30.646873+0000","last_clean":"2023-09-23T21:00:30.646873+0000","last_became_active":"2023-09-23T20:55:35.957071+0000","last_became_peered":"2023-09-23T20:55:35.957071+0000","last_unstale":"2023-09-23T21:00:30.646873+0000","last_undegraded":"2023-09-23T21:00:30.646873+0000","last_fullsized":"2023-09-23T21:00:30.646873+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:44:30.308036+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":125,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-23T21:00:30.664556+0000","last_change":"2023-09-23T20:54:13.951459+0000","last_active":"2023-09-23T21:00:30.664556+0000","last_peered":"2023-09-23T21:00:30.664556+0000","last_clean":"2023-09-23T21:00:30.664556+0000","last_became_active":"2023-09-23T20:54:13.948409+0000","last_became_peered":"2023-09-23T20:54:13.948409+0000","last_unstale":"2023-09-23T21:00:30.664556+0000","last_undegraded":"2023-09-23T21:00:30.664556+0000","last_fullsized":"2023-09-23T21:00:30.664556+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:38:10.148061+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.b","version":"79'161","reported_seq":531,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-23T21:00:30.658245+0000","last_change":"2023-09-23T20:57:27.320172+0000","last_active":"2023-09-23T21:00:30.658245+0000","last_peered":"2023-09-23T21:00:30.658245+0000","last_clean":"2023-09-23T21:00:30.658245+0000","last_became_active":"2023-09-23T20:57:16.009630+0000","last_became_peered":"2023-09-23T20:57:16.009630+0000","last_unstale":"2023-09-23T21:00:30.658245+0000","last_undegraded":"2023-09-23T21:00:30.658245+0000","last_fullsized":"2023-09-23T21:00:30.658245+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:27.320022+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:27.320022+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:31:35.221558+0000","scrub_duration":0.0043894679999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,5],"acting":[1,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.d","version":"248'12196","reported_seq":12254,"reported_epoch":248,"state":"active+clean","last_fresh":"2023-09-23T21:00:44.064924+0000","last_change":"2023-09-23T20:56:35.544132+0000","last_active":"2023-09-23T21:00:44.064924+0000","last_peered":"2023-09-23T21:00:44.064924+0000","last_clean":"2023-09-23T21:00:44.064924+0000","last_became_active":"2023-09-23T20:56:35.543836+0000","last_became_peered":"2023-09-23T20:56:35.543836+0000","last_unstale":"2023-09-23T21:00:44.064924+0000","last_undegraded":"2023-09-23T21:00:44.064924+0000","last_fullsized":"2023-09-23T21:00:44.064924+0000","mapping_epoch":199,"log_start":"219'4800","ondisk_log_start":"219'4800","created":199,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_clean_scrub_stamp":"2023-09-23T20:56:34.529103+0000","objects_scrubbed":0,"log_size":7396,"log_dups_size":0,"ondisk_log_size":7396,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:27:14.636991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":38862872,"num_objects":594,"num_object_clones":0,"num_object_copies":1782,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":594,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":12196,"num_write_kb":409217,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":125,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-23T21:00:30.666891+0000","last_change":"2023-09-23T20:54:13.950254+0000","last_active":"2023-09-23T21:00:30.666891+0000","last_peered":"2023-09-23T21:00:30.666891+0000","last_clean":"2023-09-23T21:00:30.666891+0000","last_became_active":"2023-09-23T20:54:13.949934+0000","last_became_peered":"2023-09-23T20:54:13.949934+0000","last_unstale":"2023-09-23T21:00:30.666891+0000","last_undegraded":"2023-09-23T21:00:30.666891+0000","last_fullsized":"2023-09-23T21:00:30.666891+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:30:32.847167+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":142,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-23T21:00:30.666603+0000","last_change":"2023-09-23T20:55:35.956153+0000","last_active":"2023-09-23T21:00:30.666603+0000","last_peered":"2023-09-23T21:00:30.666603+0000","last_clean":"2023-09-23T21:00:30.666603+0000","last_became_active":"2023-09-23T20:55:35.955630+0000","last_became_peered":"2023-09-23T20:55:35.955630+0000","last_unstale":"2023-09-23T21:00:30.666603+0000","last_undegraded":"2023-09-23T21:00:30.666603+0000","last_fullsized":"2023-09-23T21:00:30.666603+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:45:20.282929+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.c","version":"248'12228","reported_seq":12295,"reported_epoch":248,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T21:00:44.017818+0000","last_change":"2023-09-23T20:57:46.463005+0000","last_active":"2023-09-23T21:00:44.017818+0000","last_peered":"2023-09-23T21:00:44.017818+0000","last_clean":"2023-09-23T20:57:44.015119+0000","last_became_active":"2023-09-23T20:57:46.448702+0000","last_became_peered":"2023-09-23T20:57:46.448702+0000","last_unstale":"2023-09-23T21:00:44.017818+0000","last_undegraded":"2023-09-23T20:57:46.010874+0000","last_fullsized":"2023-09-23T20:57:45.978434+0000","mapping_epoch":223,"log_start":"219'4900","ondisk_log_start":"219'4900","created":199,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_clean_scrub_stamp":"2023-09-23T20:56:34.529103+0000","objects_scrubbed":0,"log_size":7328,"log_dups_size":0,"ondisk_log_size":7328,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:08:34.180651+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":38010880,"num_objects":580,"num_object_clones":0,"num_object_copies":1740,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5473,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":580,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12228,"num_write_kb":409856,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,7],"avail_no_missing":["6","2","7"],"object_location_counts":[{"shards":"6,7","objects":-4893}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.a","version":"79'161","reported_seq":491,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-23T21:00:30.650131+0000","last_change":"2023-09-23T20:56:44.487778+0000","last_active":"2023-09-23T21:00:30.650131+0000","last_peered":"2023-09-23T21:00:30.650131+0000","last_clean":"2023-09-23T21:00:30.650131+0000","last_became_active":"2023-09-23T20:56:05.006740+0000","last_became_peered":"2023-09-23T20:56:05.006740+0000","last_unstale":"2023-09-23T21:00:30.650131+0000","last_undegraded":"2023-09-23T21:00:30.650131+0000","last_fullsized":"2023-09-23T21:00:30.650131+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:56:44.487678+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:56:44.487678+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:04:50.086749+0000","scrub_duration":0.0017408899999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":37,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-23T21:00:30.653010+0000","last_change":"2023-09-23T20:57:44.982513+0000","last_active":"2023-09-23T21:00:30.653010+0000","last_peered":"2023-09-23T21:00:30.653010+0000","last_clean":"2023-09-23T21:00:30.653010+0000","last_became_active":"2023-09-23T20:57:44.982112+0000","last_became_peered":"2023-09-23T20:57:44.982112+0000","last_unstale":"2023-09-23T21:00:30.653010+0000","last_undegraded":"2023-09-23T21:00:30.653010+0000","last_fullsized":"2023-09-23T21:00:30.653010+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:09:05.630041+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":218,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-23T21:00:30.657871+0000","last_change":"2023-09-23T20:54:13.940748+0000","last_active":"2023-09-23T21:00:30.657871+0000","last_peered":"2023-09-23T21:00:30.657871+0000","last_clean":"2023-09-23T21:00:30.657871+0000","last_became_active":"2023-09-23T20:54:13.940419+0000","last_became_peered":"2023-09-23T20:54:13.940419+0000","last_unstale":"2023-09-23T21:00:30.657871+0000","last_undegraded":"2023-09-23T21:00:30.657871+0000","last_fullsized":"2023-09-23T21:00:30.657871+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:15:01.923816+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.1","version":"248'12122","reported_seq":12192,"reported_epoch":248,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T21:00:42.166328+0000","last_change":"2023-09-23T20:57:46.483248+0000","last_active":"2023-09-23T21:00:42.166328+0000","last_peered":"2023-09-23T21:00:42.166328+0000","last_clean":"2023-09-23T20:57:44.047957+0000","last_became_active":"2023-09-23T20:57:46.475568+0000","last_became_peered":"2023-09-23T20:57:46.475568+0000","last_unstale":"2023-09-23T21:00:42.166328+0000","last_undegraded":"2023-09-23T20:57:46.083591+0000","last_fullsized":"2023-09-23T20:57:45.978436+0000","mapping_epoch":223,"log_start":"221'5400","ondisk_log_start":"221'5400","created":199,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_clean_scrub_stamp":"2023-09-23T20:56:34.529103+0000","objects_scrubbed":0,"log_size":6722,"log_dups_size":0,"ondisk_log_size":6722,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:22:45.067269+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":43515904,"num_objects":664,"num_object_clones":0,"num_object_copies":1992,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5470,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":664,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12122,"num_write_kb":409152,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,6],"avail_no_missing":["4","1","6"],"object_location_counts":[{"shards":"4,6","objects":-4806}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.7","version":"79'161","reported_seq":553,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-23T21:00:30.647476+0000","last_change":"2023-09-23T20:57:48.243572+0000","last_active":"2023-09-23T21:00:30.647476+0000","last_peered":"2023-09-23T21:00:30.647476+0000","last_clean":"2023-09-23T21:00:30.647476+0000","last_became_active":"2023-09-23T20:57:48.243229+0000","last_became_peered":"2023-09-23T20:57:48.243229+0000","last_unstale":"2023-09-23T21:00:30.647476+0000","last_undegraded":"2023-09-23T21:00:30.647476+0000","last_fullsized":"2023-09-23T21:00:30.647476+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":3,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:25.916867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:25.916867+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:23:19.385831+0000","scrub_duration":0.001665323,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":224,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-23T21:00:30.660033+0000","last_change":"2023-09-23T20:50:05.867130+0000","last_active":"2023-09-23T21:00:30.660033+0000","last_peered":"2023-09-23T21:00:30.660033+0000","last_clean":"2023-09-23T21:00:30.660033+0000","last_became_active":"2023-09-23T20:50:05.866758+0000","last_became_peered":"2023-09-23T20:50:05.866758+0000","last_unstale":"2023-09-23T21:00:30.660033+0000","last_undegraded":"2023-09-23T21:00:30.660033+0000","last_fullsized":"2023-09-23T21:00:30.660033+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:55:07.548209+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":121,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-23T21:00:30.650468+0000","last_change":"2023-09-23T20:54:31.887604+0000","last_active":"2023-09-23T21:00:30.650468+0000","last_peered":"2023-09-23T21:00:30.650468+0000","last_clean":"2023-09-23T21:00:30.650468+0000","last_became_active":"2023-09-23T20:54:31.886752+0000","last_became_peered":"2023-09-23T20:54:31.886752+0000","last_unstale":"2023-09-23T21:00:30.650468+0000","last_undegraded":"2023-09-23T21:00:30.650468+0000","last_fullsized":"2023-09-23T21:00:30.650468+0000","mapping_epoch":136,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:13:58.872176+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":222,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-23T21:00:30.656062+0000","last_change":"2023-09-23T20:54:31.899082+0000","last_active":"2023-09-23T21:00:30.656062+0000","last_peered":"2023-09-23T21:00:30.656062+0000","last_clean":"2023-09-23T21:00:30.656062+0000","last_became_active":"2023-09-23T20:54:31.898521+0000","last_became_peered":"2023-09-23T20:54:31.898521+0000","last_unstale":"2023-09-23T21:00:30.656062+0000","last_undegraded":"2023-09-23T21:00:30.656062+0000","last_fullsized":"2023-09-23T21:00:30.656062+0000","mapping_epoch":136,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:25:36.059711+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.7","version":"248'12244","reported_seq":12299,"reported_epoch":248,"state":"active+clean","last_fresh":"2023-09-23T21:00:44.097045+0000","last_change":"2023-09-23T20:56:35.541695+0000","last_active":"2023-09-23T21:00:44.097045+0000","last_peered":"2023-09-23T21:00:44.097045+0000","last_clean":"2023-09-23T21:00:44.097045+0000","last_became_active":"2023-09-23T20:56:35.541392+0000","last_became_peered":"2023-09-23T20:56:35.541392+0000","last_unstale":"2023-09-23T21:00:44.097045+0000","last_undegraded":"2023-09-23T21:00:44.097045+0000","last_fullsized":"2023-09-23T21:00:44.097045+0000","mapping_epoch":199,"log_start":"219'4900","ondisk_log_start":"219'4900","created":199,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_clean_scrub_stamp":"2023-09-23T20:56:34.529103+0000","objects_scrubbed":0,"log_size":7344,"log_dups_size":0,"ondisk_log_size":7344,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:57:22.478253+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":37093376,"num_objects":566,"num_object_clones":0,"num_object_copies":1698,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":566,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12244,"num_write_kb":409920,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.1","version":"79'161","reported_seq":511,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-23T21:00:30.649811+0000","last_change":"2023-09-23T20:57:19.669346+0000","last_active":"2023-09-23T21:00:30.649811+0000","last_peered":"2023-09-23T21:00:30.649811+0000","last_clean":"2023-09-23T21:00:30.649811+0000","last_became_active":"2023-09-23T20:57:08.246046+0000","last_became_peered":"2023-09-23T20:57:08.246046+0000","last_unstale":"2023-09-23T21:00:30.649811+0000","last_undegraded":"2023-09-23T21:00:30.649811+0000","last_fullsized":"2023-09-23T21:00:30.649811+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:19.669259+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:19.669259+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:14:03.435531+0000","scrub_duration":0.0045759559999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":138,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-23T21:00:30.649835+0000","last_change":"2023-09-23T20:55:35.959514+0000","last_active":"2023-09-23T21:00:30.649835+0000","last_peered":"2023-09-23T21:00:30.649835+0000","last_clean":"2023-09-23T21:00:30.649835+0000","last_became_active":"2023-09-23T20:55:35.959244+0000","last_became_peered":"2023-09-23T20:55:35.959244+0000","last_unstale":"2023-09-23T21:00:30.649835+0000","last_undegraded":"2023-09-23T21:00:30.649835+0000","last_fullsized":"2023-09-23T21:00:30.649835+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:47:32.665316+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":142,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-23T21:00:30.664522+0000","last_change":"2023-09-23T20:55:35.949000+0000","last_active":"2023-09-23T21:00:30.664522+0000","last_peered":"2023-09-23T21:00:30.664522+0000","last_clean":"2023-09-23T21:00:30.664522+0000","last_became_active":"2023-09-23T20:55:35.948410+0000","last_became_peered":"2023-09-23T20:55:35.948410+0000","last_unstale":"2023-09-23T21:00:30.664522+0000","last_undegraded":"2023-09-23T21:00:30.664522+0000","last_fullsized":"2023-09-23T21:00:30.664522+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:26:50.477289+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":216,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-23T21:00:30.659840+0000","last_change":"2023-09-23T20:51:10.829889+0000","last_active":"2023-09-23T21:00:30.659840+0000","last_peered":"2023-09-23T21:00:30.659840+0000","last_clean":"2023-09-23T21:00:30.659840+0000","last_became_active":"2023-09-23T20:51:10.829533+0000","last_became_peered":"2023-09-23T20:51:10.829533+0000","last_unstale":"2023-09-23T21:00:30.659840+0000","last_undegraded":"2023-09-23T21:00:30.659840+0000","last_fullsized":"2023-09-23T21:00:30.659840+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:43:04.603460+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.4","version":"248'12059","reported_seq":12118,"reported_epoch":248,"state":"active+clean","last_fresh":"2023-09-23T21:00:41.858337+0000","last_change":"2023-09-23T20:56:35.549909+0000","last_active":"2023-09-23T21:00:41.858337+0000","last_peered":"2023-09-23T21:00:41.858337+0000","last_clean":"2023-09-23T21:00:41.858337+0000","last_became_active":"2023-09-23T20:56:35.549618+0000","last_became_peered":"2023-09-23T20:56:35.549618+0000","last_unstale":"2023-09-23T21:00:41.858337+0000","last_undegraded":"2023-09-23T21:00:41.858337+0000","last_fullsized":"2023-09-23T21:00:41.858337+0000","mapping_epoch":199,"log_start":"219'4700","ondisk_log_start":"219'4700","created":199,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_clean_scrub_stamp":"2023-09-23T20:56:34.529103+0000","objects_scrubbed":0,"log_size":7359,"log_dups_size":0,"ondisk_log_size":7359,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:24:25.918475+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":43450368,"num_objects":663,"num_object_clones":0,"num_object_copies":1989,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":663,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12059,"num_write_kb":407104,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.2","version":"79'161","reported_seq":478,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-23T21:00:30.661510+0000","last_change":"2023-09-23T20:57:24.802586+0000","last_active":"2023-09-23T21:00:30.661510+0000","last_peered":"2023-09-23T21:00:30.661510+0000","last_clean":"2023-09-23T21:00:30.661510+0000","last_became_active":"2023-09-23T20:57:08.236114+0000","last_became_peered":"2023-09-23T20:57:08.236114+0000","last_unstale":"2023-09-23T21:00:30.661510+0000","last_undegraded":"2023-09-23T21:00:30.661510+0000","last_fullsized":"2023-09-23T21:00:30.661510+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:24.802451+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:24.802451+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:06:22.309980+0000","scrub_duration":0.0090861250000000005,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,4],"acting":[0,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":234,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-23T21:00:30.666707+0000","last_change":"2023-09-23T20:51:10.838054+0000","last_active":"2023-09-23T21:00:30.666707+0000","last_peered":"2023-09-23T21:00:30.666707+0000","last_clean":"2023-09-23T21:00:30.666707+0000","last_became_active":"2023-09-23T20:51:10.837750+0000","last_became_peered":"2023-09-23T20:51:10.837750+0000","last_unstale":"2023-09-23T21:00:30.666707+0000","last_undegraded":"2023-09-23T21:00:30.666707+0000","last_fullsized":"2023-09-23T21:00:30.666707+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:15:51.648085+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":216,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-23T21:00:30.657456+0000","last_change":"2023-09-23T20:55:35.951146+0000","last_active":"2023-09-23T21:00:30.657456+0000","last_peered":"2023-09-23T21:00:30.657456+0000","last_clean":"2023-09-23T21:00:30.657456+0000","last_became_active":"2023-09-23T20:55:35.950874+0000","last_became_peered":"2023-09-23T20:55:35.950874+0000","last_unstale":"2023-09-23T21:00:30.657456+0000","last_undegraded":"2023-09-23T21:00:30.657456+0000","last_fullsized":"2023-09-23T21:00:30.657456+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:58:13.957677+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":236,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-23T21:00:30.657801+0000","last_change":"2023-09-23T20:54:31.898870+0000","last_active":"2023-09-23T21:00:30.657801+0000","last_peered":"2023-09-23T21:00:30.657801+0000","last_clean":"2023-09-23T21:00:30.657801+0000","last_became_active":"2023-09-23T20:54:31.898467+0000","last_became_peered":"2023-09-23T20:54:31.898467+0000","last_unstale":"2023-09-23T21:00:30.657801+0000","last_undegraded":"2023-09-23T21:00:30.657801+0000","last_fullsized":"2023-09-23T21:00:30.657801+0000","mapping_epoch":136,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:56:22.483408+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.6","version":"248'11930","reported_seq":21548,"reported_epoch":248,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-09-23T21:00:41.886414+0000","last_change":"2023-09-23T20:58:39.453197+0000","last_active":"2023-09-23T21:00:41.886414+0000","last_peered":"2023-09-23T21:00:41.886414+0000","last_clean":"2023-09-23T20:57:44.031597+0000","last_became_active":"2023-09-23T20:57:46.443927+0000","last_became_peered":"2023-09-23T20:57:46.443927+0000","last_unstale":"2023-09-23T21:00:41.886414+0000","last_undegraded":"2023-09-23T20:58:39.450464+0000","last_fullsized":"2023-09-23T20:57:45.977227+0000","mapping_epoch":223,"log_start":"219'4600","ondisk_log_start":"219'4600","created":199,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_clean_scrub_stamp":"2023-09-23T20:56:34.529103+0000","objects_scrubbed":0,"log_size":7330,"log_dups_size":0,"ondisk_log_size":7330,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:37:23.911131+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":43122688,"num_objects":658,"num_object_clones":0,"num_object_copies":1974,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":623,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":658,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11930,"num_write_kb":402816,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4775,"num_bytes_recovered":291569664,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7],"avail_no_missing":["0","5","7"],"object_location_counts":[{"shards":"0,7","objects":35}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":121,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-23T21:00:30.650388+0000","last_change":"2023-09-23T20:54:31.883504+0000","last_active":"2023-09-23T21:00:30.650388+0000","last_peered":"2023-09-23T21:00:30.650388+0000","last_clean":"2023-09-23T21:00:30.650388+0000","last_became_active":"2023-09-23T20:54:31.883172+0000","last_became_peered":"2023-09-23T20:54:31.883172+0000","last_unstale":"2023-09-23T21:00:30.650388+0000","last_undegraded":"2023-09-23T21:00:30.650388+0000","last_fullsized":"2023-09-23T21:00:30.650388+0000","mapping_epoch":136,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:45:40.289120+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"79'161","reported_seq":478,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-23T21:00:30.661269+0000","last_change":"2023-09-23T20:57:20.738417+0000","last_active":"2023-09-23T21:00:30.661269+0000","last_peered":"2023-09-23T21:00:30.661269+0000","last_clean":"2023-09-23T21:00:30.661269+0000","last_became_active":"2023-09-23T20:57:08.236842+0000","last_became_peered":"2023-09-23T20:57:08.236842+0000","last_unstale":"2023-09-23T21:00:30.661269+0000","last_undegraded":"2023-09-23T21:00:30.661269+0000","last_fullsized":"2023-09-23T21:00:30.661269+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":0,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:20.738273+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:20.738273+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:18:28.327990+0000","scrub_duration":0.0088336339999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":226,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-23T21:00:30.656333+0000","last_change":"2023-09-23T20:55:35.950777+0000","last_active":"2023-09-23T21:00:30.656333+0000","last_peered":"2023-09-23T21:00:30.656333+0000","last_clean":"2023-09-23T21:00:30.656333+0000","last_became_active":"2023-09-23T20:55:35.950427+0000","last_became_peered":"2023-09-23T20:55:35.950427+0000","last_unstale":"2023-09-23T21:00:30.656333+0000","last_undegraded":"2023-09-23T21:00:30.656333+0000","last_fullsized":"2023-09-23T21:00:30.656333+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:20:10.371751+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":192,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-23T21:00:30.654717+0000","last_change":"2023-09-23T20:55:35.978140+0000","last_active":"2023-09-23T21:00:30.654717+0000","last_peered":"2023-09-23T21:00:30.654717+0000","last_clean":"2023-09-23T21:00:30.654717+0000","last_became_active":"2023-09-23T20:55:35.977831+0000","last_became_peered":"2023-09-23T20:55:35.977831+0000","last_unstale":"2023-09-23T21:00:30.654717+0000","last_undegraded":"2023-09-23T21:00:30.654717+0000","last_fullsized":"2023-09-23T21:00:30.654717+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:25:47.678023+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.5","version":"248'12022","reported_seq":12088,"reported_epoch":248,"state":"active+clean","last_fresh":"2023-09-23T21:00:41.521875+0000","last_change":"2023-09-23T20:56:45.990110+0000","last_active":"2023-09-23T21:00:41.521875+0000","last_peered":"2023-09-23T21:00:41.521875+0000","last_clean":"2023-09-23T21:00:41.521875+0000","last_became_active":"2023-09-23T20:56:45.988394+0000","last_became_peered":"2023-09-23T20:56:45.988394+0000","last_unstale":"2023-09-23T21:00:41.521875+0000","last_undegraded":"2023-09-23T21:00:41.521875+0000","last_fullsized":"2023-09-23T21:00:41.521875+0000","mapping_epoch":204,"log_start":"208'3100","ondisk_log_start":"208'3100","created":199,"last_epoch_clean":205,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_clean_scrub_stamp":"2023-09-23T20:56:34.529103+0000","objects_scrubbed":0,"log_size":8922,"log_dups_size":0,"ondisk_log_size":8922,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:22:03.814210+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":42991616,"num_objects":656,"num_object_clones":0,"num_object_copies":1968,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":656,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12022,"num_write_kb":405696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,0],"acting":[2,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"79'161","reported_seq":515,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-23T21:00:30.658396+0000","last_change":"2023-09-23T20:57:20.378531+0000","last_active":"2023-09-23T21:00:30.658396+0000","last_peered":"2023-09-23T21:00:30.658396+0000","last_clean":"2023-09-23T21:00:30.658396+0000","last_became_active":"2023-09-23T20:57:08.238019+0000","last_became_peered":"2023-09-23T20:57:08.238019+0000","last_unstale":"2023-09-23T21:00:30.658396+0000","last_undegraded":"2023-09-23T21:00:30.658396+0000","last_fullsized":"2023-09-23T21:00:30.658396+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":3,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:20.377714+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:20.377714+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:00:08.305602+0000","scrub_duration":0.003015228,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":117,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-23T21:00:30.657641+0000","last_change":"2023-09-23T20:56:45.991884+0000","last_active":"2023-09-23T21:00:30.657641+0000","last_peered":"2023-09-23T21:00:30.657641+0000","last_clean":"2023-09-23T21:00:30.657641+0000","last_became_active":"2023-09-23T20:56:45.991568+0000","last_became_peered":"2023-09-23T20:56:45.991568+0000","last_unstale":"2023-09-23T21:00:30.657641+0000","last_undegraded":"2023-09-23T21:00:30.657641+0000","last_fullsized":"2023-09-23T21:00:30.657641+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":205,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:21:20.775511+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,0],"acting":[1,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":242,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-23T21:00:30.661898+0000","last_change":"2023-09-23T20:57:44.979950+0000","last_active":"2023-09-23T21:00:30.661898+0000","last_peered":"2023-09-23T21:00:30.661898+0000","last_clean":"2023-09-23T21:00:30.661898+0000","last_became_active":"2023-09-23T20:57:44.979346+0000","last_became_peered":"2023-09-23T20:57:44.979346+0000","last_unstale":"2023-09-23T21:00:30.661898+0000","last_undegraded":"2023-09-23T21:00:30.661898+0000","last_fullsized":"2023-09-23T21:00:30.661898+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:10:24.372800+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":221,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-23T21:00:30.668459+0000","last_change":"2023-09-23T20:50:37.982235+0000","last_active":"2023-09-23T21:00:30.668459+0000","last_peered":"2023-09-23T21:00:30.668459+0000","last_clean":"2023-09-23T21:00:30.668459+0000","last_became_active":"2023-09-23T20:50:37.981913+0000","last_became_peered":"2023-09-23T20:50:37.981913+0000","last_unstale":"2023-09-23T21:00:30.668459+0000","last_undegraded":"2023-09-23T21:00:30.668459+0000","last_fullsized":"2023-09-23T21:00:30.668459+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:59:57.122584+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.0","version":"248'12026","reported_seq":12083,"reported_epoch":248,"state":"active+clean","last_fresh":"2023-09-23T21:00:41.982913+0000","last_change":"2023-09-23T20:56:35.550635+0000","last_active":"2023-09-23T21:00:41.982913+0000","last_peered":"2023-09-23T21:00:41.982913+0000","last_clean":"2023-09-23T21:00:41.982913+0000","last_became_active":"2023-09-23T20:56:35.550376+0000","last_became_peered":"2023-09-23T20:56:35.550376+0000","last_unstale":"2023-09-23T21:00:41.982913+0000","last_undegraded":"2023-09-23T21:00:41.982913+0000","last_fullsized":"2023-09-23T21:00:41.982913+0000","mapping_epoch":199,"log_start":"210'3400","ondisk_log_start":"210'3400","created":199,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_clean_scrub_stamp":"2023-09-23T20:56:34.529103+0000","objects_scrubbed":0,"log_size":8626,"log_dups_size":0,"ondisk_log_size":8626,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:40:55.020049+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":42991616,"num_objects":656,"num_object_clones":0,"num_object_copies":1968,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":656,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12026,"num_write_kb":405824,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.6","version":"79'161","reported_seq":530,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-23T21:00:30.647137+0000","last_change":"2023-09-23T20:57:26.907505+0000","last_active":"2023-09-23T21:00:30.647137+0000","last_peered":"2023-09-23T21:00:30.647137+0000","last_clean":"2023-09-23T21:00:30.647137+0000","last_became_active":"2023-09-23T20:57:08.240321+0000","last_became_peered":"2023-09-23T20:57:08.240321+0000","last_unstale":"2023-09-23T21:00:30.647137+0000","last_undegraded":"2023-09-23T21:00:30.647137+0000","last_fullsized":"2023-09-23T21:00:30.647137+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":3,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:26.907408+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:26.907408+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:34:17.392928+0000","scrub_duration":0.010337038,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":164,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-23T21:00:30.668464+0000","last_change":"2023-09-23T20:56:45.988566+0000","last_active":"2023-09-23T21:00:30.668464+0000","last_peered":"2023-09-23T21:00:30.668464+0000","last_clean":"2023-09-23T21:00:30.668464+0000","last_became_active":"2023-09-23T20:56:45.988249+0000","last_became_peered":"2023-09-23T20:56:45.988249+0000","last_unstale":"2023-09-23T21:00:30.668464+0000","last_undegraded":"2023-09-23T21:00:30.668464+0000","last_fullsized":"2023-09-23T21:00:30.668464+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":205,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:51:36.644764+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,4],"acting":[0,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":219,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-23T21:00:30.659967+0000","last_change":"2023-09-23T20:54:31.898965+0000","last_active":"2023-09-23T21:00:30.659967+0000","last_peered":"2023-09-23T21:00:30.659967+0000","last_clean":"2023-09-23T21:00:30.659967+0000","last_became_active":"2023-09-23T20:54:31.898663+0000","last_became_peered":"2023-09-23T20:54:31.898663+0000","last_unstale":"2023-09-23T21:00:30.659967+0000","last_undegraded":"2023-09-23T21:00:30.659967+0000","last_fullsized":"2023-09-23T21:00:30.659967+0000","mapping_epoch":136,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:53:05.058087+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":209,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-23T21:00:30.661783+0000","last_change":"2023-09-23T20:54:31.884717+0000","last_active":"2023-09-23T21:00:30.661783+0000","last_peered":"2023-09-23T21:00:30.661783+0000","last_clean":"2023-09-23T21:00:30.661783+0000","last_became_active":"2023-09-23T20:54:31.884345+0000","last_became_peered":"2023-09-23T20:54:31.884345+0000","last_unstale":"2023-09-23T21:00:30.661783+0000","last_undegraded":"2023-09-23T21:00:30.661783+0000","last_fullsized":"2023-09-23T21:00:30.661783+0000","mapping_epoch":136,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:02:26.393868+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.3","version":"248'11989","reported_seq":12048,"reported_epoch":248,"state":"active+clean","last_fresh":"2023-09-23T21:00:44.748132+0000","last_change":"2023-09-23T20:56:35.549655+0000","last_active":"2023-09-23T21:00:44.748132+0000","last_peered":"2023-09-23T21:00:44.748132+0000","last_clean":"2023-09-23T21:00:44.748132+0000","last_became_active":"2023-09-23T20:56:35.549357+0000","last_became_peered":"2023-09-23T20:56:35.549357+0000","last_unstale":"2023-09-23T21:00:44.748132+0000","last_undegraded":"2023-09-23T21:00:44.748132+0000","last_fullsized":"2023-09-23T21:00:44.748132+0000","mapping_epoch":199,"log_start":"228'5800","ondisk_log_start":"228'5800","created":199,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_clean_scrub_stamp":"2023-09-23T20:56:34.529103+0000","objects_scrubbed":0,"log_size":6189,"log_dups_size":0,"ondisk_log_size":6189,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:37:58.752518+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34930688,"num_objects":533,"num_object_clones":0,"num_object_copies":1599,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":533,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11989,"num_write_kb":400704,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.5","version":"79'161","reported_seq":511,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-23T21:00:30.664298+0000","last_change":"2023-09-23T20:57:30.029479+0000","last_active":"2023-09-23T21:00:30.664298+0000","last_peered":"2023-09-23T21:00:30.664298+0000","last_clean":"2023-09-23T21:00:30.664298+0000","last_became_active":"2023-09-23T20:57:08.249909+0000","last_became_peered":"2023-09-23T20:57:08.249909+0000","last_unstale":"2023-09-23T21:00:30.664298+0000","last_undegraded":"2023-09-23T21:00:30.664298+0000","last_fullsized":"2023-09-23T21:00:30.664298+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":3,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:30.029396+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:30.029396+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:16:46.083145+0000","scrub_duration":0.012914682,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":212,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-23T21:00:30.661624+0000","last_change":"2023-09-23T20:51:10.836910+0000","last_active":"2023-09-23T21:00:30.661624+0000","last_peered":"2023-09-23T21:00:30.661624+0000","last_clean":"2023-09-23T21:00:30.661624+0000","last_became_active":"2023-09-23T20:51:10.836557+0000","last_became_peered":"2023-09-23T20:51:10.836557+0000","last_unstale":"2023-09-23T21:00:30.661624+0000","last_undegraded":"2023-09-23T21:00:30.661624+0000","last_fullsized":"2023-09-23T21:00:30.661624+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:51:54.019969+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":218,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-23T21:00:30.654459+0000","last_change":"2023-09-23T20:55:35.953444+0000","last_active":"2023-09-23T21:00:30.654459+0000","last_peered":"2023-09-23T21:00:30.654459+0000","last_clean":"2023-09-23T21:00:30.654459+0000","last_became_active":"2023-09-23T20:55:35.953151+0000","last_became_peered":"2023-09-23T20:55:35.953151+0000","last_unstale":"2023-09-23T21:00:30.654459+0000","last_undegraded":"2023-09-23T21:00:30.654459+0000","last_fullsized":"2023-09-23T21:00:30.654459+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:39:12.404391+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":37,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-23T21:00:30.653089+0000","last_change":"2023-09-23T20:57:44.982509+0000","last_active":"2023-09-23T21:00:30.653089+0000","last_peered":"2023-09-23T21:00:30.653089+0000","last_clean":"2023-09-23T21:00:30.653089+0000","last_became_active":"2023-09-23T20:57:44.982175+0000","last_became_peered":"2023-09-23T20:57:44.982175+0000","last_unstale":"2023-09-23T21:00:30.653089+0000","last_undegraded":"2023-09-23T21:00:30.653089+0000","last_fullsized":"2023-09-23T21:00:30.653089+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:37:49.215488+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.2","version":"248'12005","reported_seq":12064,"reported_epoch":248,"state":"active+clean","last_fresh":"2023-09-23T21:00:44.782517+0000","last_change":"2023-09-23T20:56:35.543020+0000","last_active":"2023-09-23T21:00:44.782517+0000","last_peered":"2023-09-23T21:00:44.782517+0000","last_clean":"2023-09-23T21:00:44.782517+0000","last_became_active":"2023-09-23T20:56:35.542719+0000","last_became_peered":"2023-09-23T20:56:35.542719+0000","last_unstale":"2023-09-23T21:00:44.782517+0000","last_undegraded":"2023-09-23T21:00:44.782517+0000","last_fullsized":"2023-09-23T21:00:44.782517+0000","mapping_epoch":199,"log_start":"228'5800","ondisk_log_start":"228'5800","created":199,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_clean_scrub_stamp":"2023-09-23T20:56:34.529103+0000","objects_scrubbed":0,"log_size":6205,"log_dups_size":0,"ondisk_log_size":6205,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:52:21.088628+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":33226752,"num_objects":507,"num_object_clones":0,"num_object_copies":1521,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":507,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12005,"num_write_kb":400384,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.4","version":"79'161","reported_seq":527,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-23T21:00:30.660549+0000","last_change":"2023-09-23T20:57:20.012904+0000","last_active":"2023-09-23T21:00:30.660549+0000","last_peered":"2023-09-23T21:00:30.660549+0000","last_clean":"2023-09-23T21:00:30.660549+0000","last_became_active":"2023-09-23T20:57:08.237996+0000","last_became_peered":"2023-09-23T20:57:08.237996+0000","last_unstale":"2023-09-23T21:00:30.660549+0000","last_undegraded":"2023-09-23T21:00:30.660549+0000","last_fullsized":"2023-09-23T21:00:30.660549+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":3,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:20.012764+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:20.012764+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:51:28.796946+0000","scrub_duration":0.0045251919999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":197,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-23T21:00:30.647348+0000","last_change":"2023-09-23T20:55:35.949541+0000","last_active":"2023-09-23T21:00:30.647348+0000","last_peered":"2023-09-23T21:00:30.647348+0000","last_clean":"2023-09-23T21:00:30.647348+0000","last_became_active":"2023-09-23T20:55:35.944635+0000","last_became_peered":"2023-09-23T20:55:35.944635+0000","last_unstale":"2023-09-23T21:00:30.647348+0000","last_undegraded":"2023-09-23T21:00:30.647348+0000","last_fullsized":"2023-09-23T21:00:30.647348+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:30:36.033585+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"70'1","reported_seq":237,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-23T21:00:30.668703+0000","last_change":"2023-09-23T20:54:14.009723+0000","last_active":"2023-09-23T21:00:30.668703+0000","last_peered":"2023-09-23T21:00:30.668703+0000","last_clean":"2023-09-23T21:00:30.668703+0000","last_became_active":"2023-09-23T20:54:13.941771+0000","last_became_peered":"2023-09-23T20:54:13.941771+0000","last_unstale":"2023-09-23T21:00:30.668703+0000","last_undegraded":"2023-09-23T21:00:30.668703+0000","last_fullsized":"2023-09-23T21:00:30.668703+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:48:25.588116+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":152,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-23T21:00:30.666113+0000","last_change":"2023-09-23T20:55:35.949780+0000","last_active":"2023-09-23T21:00:30.666113+0000","last_peered":"2023-09-23T21:00:30.666113+0000","last_clean":"2023-09-23T21:00:30.666113+0000","last_became_active":"2023-09-23T20:55:35.949497+0000","last_became_peered":"2023-09-23T20:55:35.949497+0000","last_unstale":"2023-09-23T21:00:30.666113+0000","last_undegraded":"2023-09-23T21:00:30.666113+0000","last_fullsized":"2023-09-23T21:00:30.666113+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:09:37.420882+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":211,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-23T21:00:30.661733+0000","last_change":"2023-09-23T20:54:13.929196+0000","last_active":"2023-09-23T21:00:30.661733+0000","last_peered":"2023-09-23T21:00:30.661733+0000","last_clean":"2023-09-23T21:00:30.661733+0000","last_became_active":"2023-09-23T20:54:13.928831+0000","last_became_peered":"2023-09-23T20:54:13.928831+0000","last_unstale":"2023-09-23T21:00:30.661733+0000","last_undegraded":"2023-09-23T21:00:30.661733+0000","last_fullsized":"2023-09-23T21:00:30.661733+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:28:44.316851+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":217,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-23T21:00:36.560812+0000","last_change":"2023-09-23T20:52:12.855430+0000","last_active":"2023-09-23T21:00:36.560812+0000","last_peered":"2023-09-23T21:00:36.560812+0000","last_clean":"2023-09-23T21:00:36.560812+0000","last_became_active":"2023-09-23T20:52:12.855115+0000","last_became_peered":"2023-09-23T20:52:12.855115+0000","last_unstale":"2023-09-23T21:00:36.560812+0000","last_undegraded":"2023-09-23T21:00:36.560812+0000","last_fullsized":"2023-09-23T21:00:36.560812+0000","mapping_epoch":100,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":101,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:33:23.802214+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":125,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-23T21:00:30.650178+0000","last_change":"2023-09-23T20:54:31.902946+0000","last_active":"2023-09-23T21:00:30.650178+0000","last_peered":"2023-09-23T21:00:30.650178+0000","last_clean":"2023-09-23T21:00:30.650178+0000","last_became_active":"2023-09-23T20:54:31.889484+0000","last_became_peered":"2023-09-23T20:54:31.889484+0000","last_unstale":"2023-09-23T21:00:30.650178+0000","last_undegraded":"2023-09-23T21:00:30.650178+0000","last_fullsized":"2023-09-23T21:00:30.650178+0000","mapping_epoch":136,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:44:44.278398+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":125,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-23T21:00:30.664613+0000","last_change":"2023-09-23T20:54:13.950351+0000","last_active":"2023-09-23T21:00:30.664613+0000","last_peered":"2023-09-23T21:00:30.664613+0000","last_clean":"2023-09-23T21:00:30.664613+0000","last_became_active":"2023-09-23T20:54:13.949987+0000","last_became_peered":"2023-09-23T20:54:13.949987+0000","last_unstale":"2023-09-23T21:00:30.664613+0000","last_undegraded":"2023-09-23T21:00:30.664613+0000","last_fullsized":"2023-09-23T21:00:30.664613+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:46:17.765134+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":156,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-23T21:00:30.650544+0000","last_change":"2023-09-23T20:56:18.141728+0000","last_active":"2023-09-23T21:00:30.650544+0000","last_peered":"2023-09-23T21:00:30.650544+0000","last_clean":"2023-09-23T21:00:30.650544+0000","last_became_active":"2023-09-23T20:56:18.137979+0000","last_became_peered":"2023-09-23T20:56:18.137979+0000","last_unstale":"2023-09-23T21:00:30.650544+0000","last_undegraded":"2023-09-23T21:00:30.650544+0000","last_fullsized":"2023-09-23T21:00:30.650544+0000","mapping_epoch":192,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":193,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:58:05.658443+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":125,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-23T21:00:30.664144+0000","last_change":"2023-09-23T20:54:13.935307+0000","last_active":"2023-09-23T21:00:30.664144+0000","last_peered":"2023-09-23T21:00:30.664144+0000","last_clean":"2023-09-23T21:00:30.664144+0000","last_became_active":"2023-09-23T20:54:13.935000+0000","last_became_peered":"2023-09-23T20:54:13.935000+0000","last_unstale":"2023-09-23T21:00:30.664144+0000","last_undegraded":"2023-09-23T21:00:30.664144+0000","last_fullsized":"2023-09-23T21:00:30.664144+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:47:40.974082+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":235,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-23T21:00:30.659916+0000","last_change":"2023-09-23T20:55:35.954870+0000","last_active":"2023-09-23T21:00:30.659916+0000","last_peered":"2023-09-23T21:00:30.659916+0000","last_clean":"2023-09-23T21:00:30.659916+0000","last_became_active":"2023-09-23T20:55:35.954568+0000","last_became_peered":"2023-09-23T20:55:35.954568+0000","last_unstale":"2023-09-23T21:00:30.659916+0000","last_undegraded":"2023-09-23T21:00:30.659916+0000","last_fullsized":"2023-09-23T21:00:30.659916+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:06:22.423566+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":234,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-23T21:00:30.663417+0000","last_change":"2023-09-23T20:51:10.838334+0000","last_active":"2023-09-23T21:00:30.663417+0000","last_peered":"2023-09-23T21:00:30.663417+0000","last_clean":"2023-09-23T21:00:30.663417+0000","last_became_active":"2023-09-23T20:51:10.838026+0000","last_became_peered":"2023-09-23T20:51:10.838026+0000","last_unstale":"2023-09-23T21:00:30.663417+0000","last_undegraded":"2023-09-23T21:00:30.663417+0000","last_fullsized":"2023-09-23T21:00:30.663417+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:04:37.009457+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":239,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-23T21:00:30.668739+0000","last_change":"2023-09-23T20:54:31.903139+0000","last_active":"2023-09-23T21:00:30.668739+0000","last_peered":"2023-09-23T21:00:30.668739+0000","last_clean":"2023-09-23T21:00:30.668739+0000","last_became_active":"2023-09-23T20:54:31.884520+0000","last_became_peered":"2023-09-23T20:54:31.884520+0000","last_unstale":"2023-09-23T21:00:30.668739+0000","last_undegraded":"2023-09-23T21:00:30.668739+0000","last_fullsized":"2023-09-23T21:00:30.668739+0000","mapping_epoch":136,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:39:34.269705+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1108,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":142,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-23T21:00:30.663882+0000","last_change":"2023-09-23T20:55:35.951572+0000","last_active":"2023-09-23T21:00:30.663882+0000","last_peered":"2023-09-23T21:00:30.663882+0000","last_clean":"2023-09-23T21:00:30.663882+0000","last_became_active":"2023-09-23T20:55:35.951294+0000","last_became_peered":"2023-09-23T20:55:35.951294+0000","last_unstale":"2023-09-23T21:00:30.663882+0000","last_undegraded":"2023-09-23T21:00:30.663882+0000","last_fullsized":"2023-09-23T21:00:30.663882+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:36:32.447210+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":7,"num_pg":16,"stat_sum":{"num_bytes":646905880,"num_objects":9872,"num_object_clones":0,"num_object_copies":29616,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":11566,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":9872,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":193648,"num_write_kb":6512577,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4776,"num_bytes_recovered":291635200,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":3011653632,"data_stored":3011641416,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":119048,"ondisk_log_size":119048,"up":48,"acting":45,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":22,"num_bytes_recovered":5990,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":21,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3381,"ondisk_log_size":3381,"up":63,"acting":63,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201523,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":400036,"kb_used_data":215260,"kb_used_omap":0,"kb_used_meta":184768,"kb_avail":93345116,"statfs":{"total":95995035648,"available":95585398784,"internally_reserved":0,"allocated":220426240,"data_stored":219542468,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":189202432},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:59:46 2023","interfaces":[{"interface":"back","average":{"1min":1.6559999999999999,"5min":1.802,"15min":1.0840000000000001},"min":{"1min":0.245,"5min":0.21099999999999999,"15min":0.21099999999999999},"max":{"1min":6.4420000000000002,"5min":43.994999999999997,"15min":43.994999999999997},"last":1.141},{"interface":"front","average":{"1min":1.6339999999999999,"5min":1.9159999999999999,"15min":1.1699999999999999},"min":{"1min":0.318,"5min":0.28199999999999997,"15min":0.218},"max":{"1min":6.4610000000000003,"5min":44.206000000000003,"15min":44.206000000000003},"last":1.1279999999999999}]},{"osd":1,"last update":"Sat Sep 23 21:00:23 2023","interfaces":[{"interface":"back","average":{"1min":1.4419999999999999,"5min":1.6559999999999999,"15min":1.2829999999999999},"min":{"1min":0.28199999999999997,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":6.8330000000000002,"5min":10.622,"15min":22.591999999999999},"last":0.36899999999999999},{"interface":"front","average":{"1min":1.823,"5min":1.7110000000000001,"15min":1.171},"min":{"1min":0.26500000000000001,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":7.9249999999999998,"5min":10.097,"15min":21.437000000000001},"last":2.0680000000000001}]},{"osd":2,"last update":"Sat Sep 23 20:59:55 2023","interfaces":[{"interface":"back","average":{"1min":2.7149999999999999,"5min":2.2000000000000002,"15min":1.294},"min":{"1min":0.30399999999999999,"5min":0.26500000000000001,"15min":0.24399999999999999},"max":{"1min":8.5289999999999999,"5min":10.805999999999999,"15min":10.805999999999999},"last":0.313},{"interface":"front","average":{"1min":2.339,"5min":2.2639999999999998,"15min":1.3089999999999999},"min":{"1min":0.26500000000000001,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":11.590999999999999,"5min":45.25,"15min":45.25},"last":0.42099999999999999}]},{"osd":3,"last update":"Sat Sep 23 21:00:32 2023","interfaces":[{"interface":"back","average":{"1min":1.9099999999999999,"5min":2.157,"15min":1.8149999999999999},"min":{"1min":0.39300000000000002,"5min":0.32100000000000001,"15min":0.314},"max":{"1min":5.2549999999999999,"5min":9.9939999999999998,"15min":36.408000000000001},"last":4.9359999999999999},{"interface":"front","average":{"1min":3.5870000000000002,"5min":2.5219999999999998,"15min":1.7529999999999999},"min":{"1min":0.40699999999999997,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":13.83,"5min":14.220000000000001,"15min":35.593000000000004},"last":4.9820000000000002}]},{"osd":4,"last update":"Sat Sep 23 21:00:23 2023","interfaces":[{"interface":"back","average":{"1min":0.76200000000000001,"5min":0.68100000000000005,"15min":0.627},"min":{"1min":0.27900000000000003,"5min":0.23799999999999999,"15min":0.23699999999999999},"max":{"1min":3.8929999999999998,"5min":4.5410000000000004,"15min":5.8920000000000003},"last":0.34999999999999998},{"interface":"front","average":{"1min":0.86799999999999999,"5min":0.72899999999999998,"15min":0.64300000000000002},"min":{"1min":0.34000000000000002,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":3.3999999999999999,"5min":4.9900000000000002,"15min":5.3200000000000003},"last":0.45600000000000002}]},{"osd":5,"last update":"Sat Sep 23 21:00:23 2023","interfaces":[{"interface":"back","average":{"1min":0.56999999999999995,"5min":0.69099999999999995,"15min":0.63500000000000001},"min":{"1min":0.28999999999999998,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":2.5249999999999999,"5min":7.0389999999999997,"15min":7.0389999999999997},"last":0.437},{"interface":"front","average":{"1min":0.86899999999999999,"5min":0.752,"15min":0.63900000000000001},"min":{"1min":0.36899999999999999,"5min":0.29999999999999999,"15min":0.27800000000000002},"max":{"1min":3.8719999999999999,"5min":4.5999999999999996,"15min":4.5999999999999996},"last":0.38800000000000001}]},{"osd":6,"last update":"Sat Sep 23 20:59:42 2023","interfaces":[{"interface":"back","average":{"1min":0.48299999999999998,"5min":0.68000000000000005,"15min":0.67100000000000004},"min":{"1min":0.28899999999999998,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":0.77300000000000002,"5min":3.5249999999999999,"15min":3.5249999999999999},"last":0.40400000000000003},{"interface":"front","average":{"1min":0.48499999999999999,"5min":0.749,"15min":0.78600000000000003},"min":{"1min":0.34699999999999998,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":0.95499999999999996,"5min":3.5409999999999999,"15min":3.5409999999999999},"last":0.49299999999999999}]}]},{"osd":6,"up_from":164,"seq":704374636607,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":309592,"kb_used_data":199120,"kb_used_omap":10,"kb_used_meta":110453,"kb_avail":93435560,"statfs":{"total":95995035648,"available":95678013440,"internally_reserved":0,"allocated":203898880,"data_stored":203027009,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":10328,"internal_metadata":113104808},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1,1,4],"upper_bound":32},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:59:45 2023","interfaces":[{"interface":"back","average":{"1min":1.1599999999999999,"5min":1.5,"15min":1.0760000000000001},"min":{"1min":0.23599999999999999,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":4.0960000000000001,"5min":8.7319999999999993,"15min":8.7319999999999993},"last":0.26400000000000001},{"interface":"front","average":{"1min":1.5029999999999999,"5min":1.6060000000000001,"15min":1.1899999999999999},"min":{"1min":0.23999999999999999,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":4.2869999999999999,"5min":6.7140000000000004,"15min":6.7140000000000004},"last":0.311}]},{"osd":1,"last update":"Sat Sep 23 20:59:45 2023","interfaces":[{"interface":"back","average":{"1min":1.554,"5min":1.25,"15min":0.94199999999999995},"min":{"1min":0.27200000000000002,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":8.4309999999999992,"5min":8.4309999999999992,"15min":8.4309999999999992},"last":0.36699999999999999},{"interface":"front","average":{"1min":1.506,"5min":1.3720000000000001,"15min":1.036},"min":{"1min":0.26100000000000001,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":7.2069999999999999,"5min":10.324999999999999,"15min":10.324999999999999},"last":3.5009999999999999}]},{"osd":2,"last update":"Sat Sep 23 20:59:45 2023","interfaces":[{"interface":"back","average":{"1min":2.097,"5min":2.0720000000000001,"15min":1.788},"min":{"1min":0.312,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":6.0670000000000002,"5min":13.750999999999999,"15min":13.750999999999999},"last":2.5590000000000002},{"interface":"front","average":{"1min":2.46,"5min":1.8,"15min":1.3380000000000001},"min":{"1min":0.32000000000000001,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":5.6130000000000004,"5min":8.0890000000000004,"15min":8.0890000000000004},"last":3.4700000000000002}]},{"osd":3,"last update":"Sat Sep 23 20:59:45 2023","interfaces":[{"interface":"back","average":{"1min":1.768,"5min":1.415,"15min":1.216},"min":{"1min":0.45000000000000001,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":4.7220000000000004,"5min":6.9000000000000004,"15min":6.9000000000000004},"last":4.3799999999999999},{"interface":"front","average":{"1min":1.6479999999999999,"5min":1.7490000000000001,"15min":1.651},"min":{"1min":0.34999999999999998,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":4.6289999999999996,"5min":11.021000000000001,"15min":11.021000000000001},"last":4.6100000000000003}]},{"osd":4,"last update":"Sat Sep 23 20:59:45 2023","interfaces":[{"interface":"back","average":{"1min":0.68700000000000006,"5min":0.78600000000000003,"15min":0.96299999999999997},"min":{"1min":0.27100000000000002,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":2.3460000000000001,"5min":9.0079999999999991,"15min":9.0079999999999991},"last":3.4260000000000002},{"interface":"front","average":{"1min":0.48599999999999999,"5min":0.499,"15min":0.52000000000000002},"min":{"1min":0.29899999999999999,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":1.115,"5min":2.0059999999999998,"15min":2.0059999999999998},"last":1.3380000000000001}]},{"osd":5,"last update":"Sat Sep 23 20:59:45 2023","interfaces":[{"interface":"back","average":{"1min":0.78100000000000003,"5min":0.61699999999999999,"15min":0.56599999999999995},"min":{"1min":0.28100000000000003,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":2.3730000000000002,"5min":2.8239999999999998,"15min":2.8239999999999998},"last":5.4119999999999999},{"interface":"front","average":{"1min":0.58299999999999996,"5min":0.73999999999999999,"15min":0.78600000000000003},"min":{"1min":0.34000000000000002,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":1.2290000000000001,"5min":3.1379999999999999,"15min":3.1379999999999999},"last":0.50800000000000001}]},{"osd":7,"last update":"Sat Sep 23 20:59:45 2023","interfaces":[{"interface":"back","average":{"1min":0.51700000000000002,"5min":0.77200000000000002,"15min":0.876},"min":{"1min":0.318,"5min":0.312,"15min":0.312},"max":{"1min":1.1819999999999999,"5min":7.8570000000000002,"15min":7.8570000000000002},"last":3.645},{"interface":"front","average":{"1min":0.46500000000000002,"5min":0.64900000000000002,"15min":0.56899999999999995},"min":{"1min":0.33800000000000002,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":0.63200000000000001,"5min":8.282,"15min":8.282},"last":0.54300000000000004}]}]},{"osd":1,"up_from":88,"seq":377957122178,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":857636,"kb_used_data":580444,"kb_used_omap":9,"kb_used_meta":276982,"kb_avail":92887516,"statfs":{"total":95995035648,"available":95116816384,"internally_reserved":0,"allocated":594374656,"data_stored":593144802,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9239,"internal_metadata":283630569},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,2,4],"upper_bound":32},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:00:26 2023","interfaces":[{"interface":"back","average":{"1min":2.9870000000000001,"5min":3.048,"15min":1.7110000000000001},"min":{"1min":0.26000000000000001,"5min":0.20699999999999999,"15min":0.20699999999999999},"max":{"1min":8.8000000000000007,"5min":16.646999999999998,"15min":16.646999999999998},"last":0.91700000000000004},{"interface":"front","average":{"1min":4.7480000000000002,"5min":3.577,"15min":1.9690000000000001},"min":{"1min":0.38200000000000001,"5min":0.246,"15min":0.22500000000000001},"max":{"1min":27.797999999999998,"5min":27.797999999999998,"15min":27.797999999999998},"last":7.1609999999999996}]},{"osd":2,"last update":"Sat Sep 23 20:59:55 2023","interfaces":[{"interface":"back","average":{"1min":2.4809999999999999,"5min":2.3210000000000002,"15min":1.319},"min":{"1min":0.222,"5min":0.221,"15min":0.20300000000000001},"max":{"1min":13.752000000000001,"5min":14.335000000000001,"15min":14.335000000000001},"last":0.439},{"interface":"front","average":{"1min":3.4470000000000001,"5min":3.3519999999999999,"15min":1.948},"min":{"1min":0.27400000000000002,"5min":0.248,"15min":0.248},"max":{"1min":18.026,"5min":18.026,"15min":38.951000000000001},"last":0.60599999999999998}]},{"osd":3,"last update":"Sat Sep 23 21:00:26 2023","interfaces":[{"interface":"back","average":{"1min":3.8849999999999998,"5min":2.8220000000000001,"15min":1.794},"min":{"1min":0.38500000000000001,"5min":0.246,"15min":0.246},"max":{"1min":22.686,"5min":22.686,"15min":25.314},"last":4.234},{"interface":"front","average":{"1min":3.4510000000000001,"5min":2.605,"15min":1.613},"min":{"1min":0.36699999999999999,"5min":0.20599999999999999,"15min":0.20599999999999999},"max":{"1min":14.473000000000001,"5min":14.473000000000001,"15min":19.881},"last":0.56899999999999995}]},{"osd":4,"last update":"Sat Sep 23 21:00:26 2023","interfaces":[{"interface":"back","average":{"1min":2.383,"5min":1.704,"15min":1.329},"min":{"1min":0.34499999999999997,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":18.231000000000002,"5min":18.231000000000002,"15min":18.231000000000002},"last":2.7570000000000001},{"interface":"front","average":{"1min":1.6080000000000001,"5min":1.55,"15min":1.1419999999999999},"min":{"1min":0.35399999999999998,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":6.6360000000000001,"5min":7.532,"15min":11.234999999999999},"last":0.51100000000000001}]},{"osd":5,"last update":"Sat Sep 23 21:00:26 2023","interfaces":[{"interface":"back","average":{"1min":2.6080000000000001,"5min":2.2999999999999998,"15min":1.419},"min":{"1min":0.34999999999999998,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":12.699999999999999,"5min":15.041,"15min":15.041},"last":7.2640000000000002},{"interface":"front","average":{"1min":2.415,"5min":1.762,"15min":1.357},"min":{"1min":0.40000000000000002,"5min":0.32600000000000001,"15min":0.32200000000000001},"max":{"1min":18.280999999999999,"5min":18.280999999999999,"15min":18.280999999999999},"last":2.8490000000000002}]},{"osd":6,"last update":"Sat Sep 23 20:59:41 2023","interfaces":[{"interface":"back","average":{"1min":2.266,"5min":1.5609999999999999,"15min":1.397},"min":{"1min":0.46200000000000002,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":10.159000000000001,"5min":10.673999999999999,"15min":10.673999999999999},"last":2.8079999999999998},{"interface":"front","average":{"1min":1.8660000000000001,"5min":1.6299999999999999,"15min":1.3779999999999999},"min":{"1min":0.33700000000000002,"5min":0.309,"15min":0.309},"max":{"1min":7.6779999999999999,"5min":8.7530000000000001,"15min":8.7530000000000001},"last":0.47699999999999998}]},{"osd":7,"last update":"Sat Sep 23 21:00:26 2023","interfaces":[{"interface":"back","average":{"1min":1.8180000000000001,"5min":1.7829999999999999,"15min":1.24},"min":{"1min":0.41699999999999998,"5min":0.36199999999999999,"15min":0.35699999999999998},"max":{"1min":6.657,"5min":8.7769999999999992,"15min":11.186999999999999},"last":0.53800000000000003},{"interface":"front","average":{"1min":2.415,"5min":2.319,"15min":1.4399999999999999},"min":{"1min":0.41499999999999998,"5min":0.40899999999999997,"15min":0.34599999999999997},"max":{"1min":12.84,"5min":15.098000000000001,"15min":15.098000000000001},"last":7.2969999999999997}]}]},{"osd":0,"up_from":83,"seq":356482285704,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":578880,"kb_used_data":343928,"kb_used_omap":28,"kb_used_meta":234531,"kb_avail":93166272,"statfs":{"total":95995035648,"available":95402262528,"internally_reserved":0,"allocated":352182272,"data_stored":350985748,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":28848,"internal_metadata":240160592},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,1,2],"upper_bound":32},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 21:00:20 2023","interfaces":[{"interface":"back","average":{"1min":6.4480000000000004,"5min":3.7120000000000002,"15min":1.893},"min":{"1min":0.20000000000000001,"5min":0.191,"15min":0.191},"max":{"1min":19.146000000000001,"5min":19.146000000000001,"15min":19.146000000000001},"last":4.5039999999999996},{"interface":"front","average":{"1min":5.4379999999999997,"5min":3.6789999999999998,"15min":1.8759999999999999},"min":{"1min":0.39500000000000002,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":11.074,"5min":15.771000000000001,"15min":17.704999999999998},"last":1.2450000000000001}]},{"osd":2,"last update":"Sat Sep 23 20:59:51 2023","interfaces":[{"interface":"back","average":{"1min":3.8879999999999999,"5min":3.4180000000000001,"15min":1.867},"min":{"1min":0.50900000000000001,"5min":0.28299999999999997,"15min":0.23799999999999999},"max":{"1min":21.053000000000001,"5min":21.053000000000001,"15min":21.053000000000001},"last":1.2709999999999999},{"interface":"front","average":{"1min":3.6579999999999999,"5min":3.7450000000000001,"15min":1.944},"min":{"1min":0.26300000000000001,"5min":0.25800000000000001,"15min":0.21099999999999999},"max":{"1min":13.956,"5min":21.747,"15min":21.747},"last":5.5449999999999999}]},{"osd":3,"last update":"Sat Sep 23 21:00:20 2023","interfaces":[{"interface":"back","average":{"1min":6.532,"5min":3.8690000000000002,"15min":1.9990000000000001},"min":{"1min":1.052,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":15.082000000000001,"5min":21.184000000000001,"15min":21.184000000000001},"last":4.5629999999999997},{"interface":"front","average":{"1min":4.9050000000000002,"5min":4.008,"15min":2.0899999999999999},"min":{"1min":0.48499999999999999,"5min":0.28000000000000003,"15min":0.23300000000000001},"max":{"1min":9.7560000000000002,"5min":21.167000000000002,"15min":21.167000000000002},"last":1.2170000000000001}]},{"osd":4,"last update":"Sat Sep 23 20:59:51 2023","interfaces":[{"interface":"back","average":{"1min":1.9359999999999999,"5min":2.823,"15min":1.5589999999999999},"min":{"1min":0.38500000000000001,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":7.931,"5min":21.594999999999999,"15min":21.594999999999999},"last":5.4829999999999997},{"interface":"front","average":{"1min":2.8639999999999999,"5min":1.9830000000000001,"15min":1.278},"min":{"1min":0.41099999999999998,"5min":0.31,"15min":0.28100000000000003},"max":{"1min":13.173999999999999,"5min":13.173999999999999,"15min":18.724},"last":4.4409999999999998}]},{"osd":5,"last update":"Sat Sep 23 20:59:51 2023","interfaces":[{"interface":"back","average":{"1min":1.859,"5min":1.889,"15min":1.258},"min":{"1min":0.38700000000000001,"5min":0.318,"15min":0.318},"max":{"1min":6.9980000000000002,"5min":8.8970000000000002,"15min":8.8970000000000002},"last":1.1870000000000001},{"interface":"front","average":{"1min":1.748,"5min":2.7290000000000001,"15min":1.5649999999999999},"min":{"1min":0.41899999999999998,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":7.9820000000000002,"5min":20.512,"15min":20.512},"last":5.5179999999999998}]},{"osd":6,"last update":"Sat Sep 23 20:59:46 2023","interfaces":[{"interface":"back","average":{"1min":2.9180000000000001,"5min":2.145,"15min":1.819},"min":{"1min":0.45900000000000002,"5min":0.37,"15min":0.37},"max":{"1min":13.134,"5min":13.134,"15min":13.134},"last":4.4790000000000001},{"interface":"front","average":{"1min":2.9140000000000001,"5min":2.198,"15min":1.8959999999999999},"min":{"1min":0.43099999999999999,"5min":0.377,"15min":0.377},"max":{"1min":13.254,"5min":13.254,"15min":13.254},"last":4.5960000000000001}]},{"osd":7,"last update":"Sat Sep 23 20:59:51 2023","interfaces":[{"interface":"back","average":{"1min":2.1520000000000001,"5min":2.0139999999999998,"15min":1.3169999999999999},"min":{"1min":0.433,"5min":0.379,"15min":0.379},"max":{"1min":6.9699999999999998,"5min":8.7729999999999997,"15min":8.7729999999999997},"last":1.1479999999999999},{"interface":"front","average":{"1min":3.0459999999999998,"5min":2.105,"15min":1.3200000000000001},"min":{"1min":0.44600000000000001,"5min":0.377,"15min":0.377},"max":{"1min":13.218999999999999,"5min":13.218999999999999,"15min":18.673999999999999},"last":4.5380000000000003}]}]},{"osd":2,"up_from":93,"seq":399431958651,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":937392,"kb_used_data":615656,"kb_used_omap":81,"kb_used_meta":321646,"kb_avail":92807760,"statfs":{"total":95995035648,"available":95035146240,"internally_reserved":0,"allocated":630431744,"data_stored":629247968,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":83750,"internal_metadata":329365722},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1],"upper_bound":4},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:59:50 2023","interfaces":[{"interface":"back","average":{"1min":4.8730000000000002,"5min":2.7000000000000002,"15min":1.3799999999999999},"min":{"1min":0.189,"5min":0.17999999999999999,"15min":0.17999999999999999},"max":{"1min":29.498000000000001,"5min":29.498000000000001,"15min":29.498000000000001},"last":0.52600000000000002},{"interface":"front","average":{"1min":4.032,"5min":3.3370000000000002,"15min":1.6739999999999999},"min":{"1min":0.20599999999999999,"5min":0.20599999999999999,"15min":0.20599999999999999},"max":{"1min":17.251000000000001,"5min":17.251000000000001,"15min":17.251000000000001},"last":0.41099999999999998}]},{"osd":1,"last update":"Sat Sep 23 20:59:50 2023","interfaces":[{"interface":"back","average":{"1min":2.6030000000000002,"5min":2.5619999999999998,"15min":1.625},"min":{"1min":0.25700000000000001,"5min":0.20200000000000001,"15min":0.20200000000000001},"max":{"1min":7.9800000000000004,"5min":14.709,"15min":16.081},"last":0.60899999999999999},{"interface":"front","average":{"1min":3.427,"5min":3.2570000000000001,"15min":1.97},"min":{"1min":0.26600000000000001,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":9.9090000000000007,"5min":11.717000000000001,"15min":16.041},"last":0.59899999999999998}]},{"osd":3,"last update":"Sat Sep 23 21:00:31 2023","interfaces":[{"interface":"back","average":{"1min":4.71,"5min":3.8450000000000002,"15min":2.1800000000000002},"min":{"1min":0.28599999999999998,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":10.468,"5min":27.614999999999998,"15min":27.614999999999998},"last":1.03},{"interface":"front","average":{"1min":4.2939999999999996,"5min":4.0309999999999997,"15min":2.4540000000000002},"min":{"1min":0.64400000000000002,"5min":0.23899999999999999,"15min":0.221},"max":{"1min":11.673999999999999,"5min":27.562000000000001,"15min":27.562000000000001},"last":1.9099999999999999}]},{"osd":4,"last update":"Sat Sep 23 20:59:50 2023","interfaces":[{"interface":"back","average":{"1min":1.575,"5min":1.7749999999999999,"15min":1.2070000000000001},"min":{"1min":0.36499999999999999,"5min":0.28699999999999998,"15min":0.26400000000000001},"max":{"1min":9.8659999999999997,"5min":9.8659999999999997,"15min":9.8659999999999997},"last":0.54900000000000004},{"interface":"front","average":{"1min":2.8439999999999999,"5min":2.1880000000000002,"15min":1.3720000000000001},"min":{"1min":0.32400000000000001,"5min":0.313,"15min":0.313},"max":{"1min":9.9719999999999995,"5min":10.609,"15min":10.609},"last":1.8700000000000001}]},{"osd":5,"last update":"Sat Sep 23 20:59:50 2023","interfaces":[{"interface":"back","average":{"1min":1.4179999999999999,"5min":1.99,"15min":1.298},"min":{"1min":0.33000000000000002,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":6.3570000000000002,"5min":9.0329999999999995,"15min":9.7629999999999999},"last":0.48099999999999998},{"interface":"front","average":{"1min":2.5760000000000001,"5min":2.1419999999999999,"15min":1.3720000000000001},"min":{"1min":0.35599999999999998,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":9.9220000000000006,"5min":10.574,"15min":10.574},"last":1.8240000000000001}]},{"osd":6,"last update":"Sat Sep 23 20:59:45 2023","interfaces":[{"interface":"back","average":{"1min":2.9060000000000001,"5min":2.0640000000000001,"15min":1.8},"min":{"1min":0.35699999999999998,"5min":0.312,"15min":0.312},"max":{"1min":10.012,"5min":14.244999999999999,"15min":14.244999999999999},"last":1.895},{"interface":"front","average":{"1min":2.8740000000000001,"5min":1.8129999999999999,"15min":1.2909999999999999},"min":{"1min":0.41599999999999998,"5min":0.374,"15min":0.374},"max":{"1min":10.032999999999999,"5min":10.032999999999999,"15min":10.032999999999999},"last":1.931}]},{"osd":7,"last update":"Sat Sep 23 20:59:50 2023","interfaces":[{"interface":"back","average":{"1min":1.3859999999999999,"5min":1.4830000000000001,"15min":1.0449999999999999},"min":{"1min":0.35899999999999999,"5min":0.315,"15min":0.315},"max":{"1min":5.9199999999999999,"5min":8.0820000000000007,"15min":8.0820000000000007},"last":0.56000000000000005},{"interface":"front","average":{"1min":1.121,"5min":1.873,"15min":1.292},"min":{"1min":0.443,"5min":0.35099999999999998,"15min":0.34499999999999997},"max":{"1min":6.3899999999999997,"5min":8.6219999999999999,"15min":9.7859999999999996},"last":0.51800000000000002}]}]},{"osd":3,"up_from":98,"seq":420906795125,"num_pgs":49,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":687312,"kb_used_data":327560,"kb_used_omap":7,"kb_used_meta":359544,"kb_avail":93057840,"statfs":{"total":95995035648,"available":95291228160,"internally_reserved":0,"allocated":335421440,"data_stored":334266537,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7576,"internal_metadata":368173672},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,2,3,1],"upper_bound":128},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:00:28 2023","interfaces":[{"interface":"back","average":{"1min":5.6130000000000004,"5min":3.0470000000000002,"15min":2.0840000000000001},"min":{"1min":0.20399999999999999,"5min":0.20399999999999999,"15min":0.20399999999999999},"max":{"1min":27.962,"5min":27.962,"15min":27.962},"last":5.5890000000000004},{"interface":"front","average":{"1min":4.984,"5min":3.5169999999999999,"15min":1.9339999999999999},"min":{"1min":0.21099999999999999,"5min":0.19800000000000001,"15min":0.19800000000000001},"max":{"1min":28.058,"5min":28.058,"15min":28.058},"last":12.132999999999999}]},{"osd":1,"last update":"Sat Sep 23 21:00:28 2023","interfaces":[{"interface":"back","average":{"1min":4.8730000000000002,"5min":3.1760000000000002,"15min":1.9119999999999999},"min":{"1min":0.22600000000000001,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":22.681999999999999,"5min":22.681999999999999,"15min":22.681999999999999},"last":0.53100000000000003},{"interface":"front","average":{"1min":4.0789999999999997,"5min":2.5169999999999999,"15min":1.929},"min":{"1min":0.25,"5min":0.218,"15min":0.218},"max":{"1min":19.329000000000001,"5min":19.329000000000001,"15min":25.353000000000002},"last":5.6539999999999999}]},{"osd":2,"last update":"Sat Sep 23 21:00:28 2023","interfaces":[{"interface":"back","average":{"1min":6.1680000000000001,"5min":3.8039999999999998,"15min":2.113},"min":{"1min":0.36199999999999999,"5min":0.33000000000000002,"15min":0.30099999999999999},"max":{"1min":28.251999999999999,"5min":28.251999999999999,"15min":28.251999999999999},"last":3.1120000000000001},{"interface":"front","average":{"1min":4.6779999999999999,"5min":2.9300000000000002,"15min":2.0750000000000002},"min":{"1min":0.372,"5min":0.23999999999999999,"15min":0.22600000000000001},"max":{"1min":16.213999999999999,"5min":16.213999999999999,"15min":25.305},"last":5.6859999999999999}]},{"osd":4,"last update":"Sat Sep 23 21:00:28 2023","interfaces":[{"interface":"back","average":{"1min":3.258,"5min":2.2869999999999999,"15min":1.4750000000000001},"min":{"1min":0.314,"5min":0.29899999999999999,"15min":0.29299999999999998},"max":{"1min":18.239000000000001,"5min":18.239000000000001,"15min":18.239000000000001},"last":0.46800000000000003},{"interface":"front","average":{"1min":2.7200000000000002,"5min":1.5860000000000001,"15min":1.474},"min":{"1min":0.317,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":14.808,"5min":14.808,"15min":22.853000000000002},"last":5.5300000000000002}]},{"osd":5,"last update":"Sat Sep 23 21:00:28 2023","interfaces":[{"interface":"back","average":{"1min":3.2930000000000001,"5min":2.3399999999999999,"15min":1.4690000000000001},"min":{"1min":0.33200000000000002,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":18.324999999999999,"5min":18.324999999999999,"15min":18.324999999999999},"last":0.85399999999999998},{"interface":"front","average":{"1min":3.0910000000000002,"5min":2.363,"15min":1.4359999999999999},"min":{"1min":0.33700000000000002,"5min":0.33300000000000002,"15min":0.33100000000000002},"max":{"1min":11.327,"5min":13.43,"15min":13.43},"last":12.013999999999999}]},{"osd":6,"last update":"Sat Sep 23 20:59:47 2023","interfaces":[{"interface":"back","average":{"1min":2.0579999999999998,"5min":1.415,"15min":1.105},"min":{"1min":0.33700000000000002,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":13.499000000000001,"5min":13.499000000000001,"15min":13.499000000000001},"last":5.6239999999999997},{"interface":"front","average":{"1min":3.8410000000000002,"5min":1.966,"15min":1.5960000000000001},"min":{"1min":0.55300000000000005,"5min":0.38700000000000001,"15min":0.38700000000000001},"max":{"1min":13.398999999999999,"5min":13.398999999999999,"15min":13.398999999999999},"last":12.063000000000001}]},{"osd":7,"last update":"Sat Sep 23 21:00:28 2023","interfaces":[{"interface":"back","average":{"1min":3.2879999999999998,"5min":2.5649999999999999,"15min":1.6399999999999999},"min":{"1min":0.39200000000000002,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":18.289999999999999,"5min":18.289999999999999,"15min":18.289999999999999},"last":0.50600000000000001},{"interface":"front","average":{"1min":3.1219999999999999,"5min":2.399,"15min":1.506},"min":{"1min":0.38900000000000001,"5min":0.38900000000000001,"15min":0.379},"max":{"1min":11.378,"5min":13.355,"15min":13.355},"last":12.096}]}]},{"osd":4,"up_from":31,"seq":133143986432,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":362916,"kb_used_data":211292,"kb_used_omap":0,"kb_used_meta":151552,"kb_avail":93382236,"statfs":{"total":95995035648,"available":95623409664,"internally_reserved":0,"allocated":216363008,"data_stored":215479489,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":155189248},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:59:53 2023","interfaces":[{"interface":"back","average":{"1min":2.1469999999999998,"5min":1.915,"15min":1.143},"min":{"1min":0.255,"5min":0.214,"15min":0.214},"max":{"1min":9.0749999999999993,"5min":11.784000000000001,"15min":11.784000000000001},"last":1.5429999999999999},{"interface":"front","average":{"1min":2.4969999999999999,"5min":2.0329999999999999,"15min":1.335},"min":{"1min":0.32800000000000001,"5min":0.26700000000000002,"15min":0.25800000000000001},"max":{"1min":9.1029999999999998,"5min":11.766,"15min":11.766},"last":1.5649999999999999}]},{"osd":1,"last update":"Sat Sep 23 21:00:22 2023","interfaces":[{"interface":"back","average":{"1min":1.4350000000000001,"5min":1.673,"15min":1.1779999999999999},"min":{"1min":0.25800000000000001,"5min":0.246,"15min":0.23599999999999999},"max":{"1min":5.5570000000000004,"5min":10.153,"15min":10.153},"last":2.194},{"interface":"front","average":{"1min":2.6040000000000001,"5min":2.2749999999999999,"15min":1.2749999999999999},"min":{"1min":0.29199999999999998,"5min":0.26100000000000001,"15min":0.23200000000000001},"max":{"1min":10.882999999999999,"5min":12.292999999999999,"15min":12.292999999999999},"last":1.302}]},{"osd":2,"last update":"Sat Sep 23 20:59:53 2023","interfaces":[{"interface":"back","average":{"1min":2.573,"5min":1.7430000000000001,"15min":1.2410000000000001},"min":{"1min":0.33600000000000002,"5min":0.311,"15min":0.29099999999999998},"max":{"1min":8.1509999999999998,"5min":8.1509999999999998,"15min":15.288},"last":2.524},{"interface":"front","average":{"1min":2.8330000000000002,"5min":1.831,"15min":1.3999999999999999},"min":{"1min":0.28599999999999998,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":12.855,"5min":12.855,"15min":15.262},"last":3.5179999999999998}]},{"osd":3,"last update":"Sat Sep 23 21:00:22 2023","interfaces":[{"interface":"back","average":{"1min":3.5569999999999999,"5min":2.3479999999999999,"15min":1.8120000000000001},"min":{"1min":0.39200000000000002,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":8.5389999999999997,"5min":14.824999999999999,"15min":31.562999999999999},"last":4.3620000000000001},{"interface":"front","average":{"1min":3.5129999999999999,"5min":2.4550000000000001,"15min":1.8740000000000001},"min":{"1min":0.48399999999999999,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":8.5570000000000004,"5min":14.839,"15min":31.574000000000002},"last":4.3490000000000002}]},{"osd":5,"last update":"Sat Sep 23 21:00:39 2023","interfaces":[{"interface":"back","average":{"1min":0.33800000000000002,"5min":0.56999999999999995,"15min":0.51900000000000002},"min":{"1min":0.251,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":0.495,"5min":3.008,"15min":3.3279999999999998},"last":0.44900000000000001},{"interface":"front","average":{"1min":0.55500000000000005,"5min":0.71199999999999997,"15min":0.621},"min":{"1min":0.27600000000000002,"5min":0.25600000000000001,"15min":0.251},"max":{"1min":2.6539999999999999,"5min":3.4830000000000001,"15min":7.7619999999999996},"last":3.069}]},{"osd":6,"last update":"Sat Sep 23 20:59:46 2023","interfaces":[{"interface":"back","average":{"1min":0.62,"5min":0.70399999999999996,"15min":0.55700000000000005},"min":{"1min":0.32100000000000001,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":3.4180000000000001,"5min":4.6909999999999998,"15min":4.6909999999999998},"last":2.2170000000000001},{"interface":"front","average":{"1min":0.752,"5min":0.73699999999999999,"15min":0.67500000000000004},"min":{"1min":0.34499999999999997,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":3.2999999999999998,"5min":3.456,"15min":3.456},"last":0.54700000000000004}]},{"osd":7,"last update":"Sat Sep 23 21:00:22 2023","interfaces":[{"interface":"back","average":{"1min":0.52100000000000002,"5min":0.754,"15min":0.69099999999999995},"min":{"1min":0.33400000000000002,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":2.302,"5min":6.2240000000000002,"15min":7.383},"last":0.52700000000000002},{"interface":"front","average":{"1min":0.56100000000000005,"5min":0.77500000000000002,"15min":0.67700000000000005},"min":{"1min":0.32700000000000001,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":2.4420000000000002,"5min":7.8310000000000004,"15min":7.8310000000000004},"last":0.64300000000000002}]}]},{"osd":5,"up_from":39,"seq":167503724797,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":655448,"kb_used_data":527696,"kb_used_omap":0,"kb_used_meta":127744,"kb_avail":93089704,"statfs":{"total":95995035648,"available":95323856896,"internally_reserved":0,"allocated":540360704,"data_stored":539485761,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":130809856},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:00:41 2023","interfaces":[{"interface":"back","average":{"1min":3.0920000000000001,"5min":2.0419999999999998,"15min":1.1359999999999999},"min":{"1min":0.35199999999999998,"5min":0.20399999999999999,"15min":0.20399999999999999},"max":{"1min":6.665,"5min":12.795999999999999,"15min":12.795999999999999},"last":0.35199999999999998},{"interface":"front","average":{"1min":3.7250000000000001,"5min":2.2349999999999999,"15min":1.335},"min":{"1min":0.27000000000000002,"5min":0.24099999999999999,"15min":0.223},"max":{"1min":9.1099999999999994,"5min":16.219999999999999,"15min":16.219999999999999},"last":0.39200000000000002}]},{"osd":1,"last update":"Sat Sep 23 21:00:18 2023","interfaces":[{"interface":"back","average":{"1min":2.8759999999999999,"5min":1.8779999999999999,"15min":1.2529999999999999},"min":{"1min":0.26600000000000001,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":8.7720000000000002,"5min":8.7720000000000002,"15min":8.7720000000000002},"last":1.837},{"interface":"front","average":{"1min":2.3639999999999999,"5min":1.9330000000000001,"15min":1.22},"min":{"1min":0.34200000000000003,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":8.5109999999999992,"5min":10.285,"15min":10.285},"last":1.9079999999999999}]},{"osd":2,"last update":"Sat Sep 23 20:59:54 2023","interfaces":[{"interface":"back","average":{"1min":2.6819999999999999,"5min":1.6699999999999999,"15min":1.121},"min":{"1min":0.27400000000000002,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":10.371,"5min":10.579000000000001,"15min":12.651},"last":0.44600000000000001},{"interface":"front","average":{"1min":3.423,"5min":2.258,"15min":1.274},"min":{"1min":0.28799999999999998,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":10.333,"5min":12.300000000000001,"15min":12.300000000000001},"last":0.29399999999999998}]},{"osd":3,"last update":"Sat Sep 23 21:00:18 2023","interfaces":[{"interface":"back","average":{"1min":4.931,"5min":2.859,"15min":1.6379999999999999},"min":{"1min":0.35999999999999999,"5min":0.35399999999999998,"15min":0.28599999999999998},"max":{"1min":20.788,"5min":20.788,"15min":20.788},"last":0.55400000000000005},{"interface":"front","average":{"1min":3.9870000000000001,"5min":2.177,"15min":1.2949999999999999},"min":{"1min":0.39100000000000001,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":20.539000000000001,"5min":20.539000000000001,"15min":20.539000000000001},"last":0.41899999999999998}]},{"osd":4,"last update":"Sat Sep 23 21:00:35 2023","interfaces":[{"interface":"back","average":{"1min":0.69999999999999996,"5min":0.625,"15min":0.56899999999999995},"min":{"1min":0.23400000000000001,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":4.1520000000000001,"5min":4.1520000000000001,"15min":4.1520000000000001},"last":0.45800000000000002},{"interface":"front","average":{"1min":0.79600000000000004,"5min":0.69799999999999995,"15min":0.59599999999999997},"min":{"1min":0.28999999999999998,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":3.7469999999999999,"5min":3.7469999999999999,"15min":3.7469999999999999},"last":0.47699999999999998}]},{"osd":6,"last update":"Sat Sep 23 21:00:41 2023","interfaces":[{"interface":"back","average":{"1min":0.91600000000000004,"5min":0.72199999999999998,"15min":0.61399999999999999},"min":{"1min":0.314,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":4.359,"5min":4.359,"15min":4.359},"last":0.32700000000000001},{"interface":"front","average":{"1min":0.92200000000000004,"5min":0.79100000000000004,"15min":0.75900000000000001},"min":{"1min":0.36899999999999999,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":3.847,"5min":3.847,"15min":3.847},"last":0.48699999999999999}]},{"osd":7,"last update":"Sat Sep 23 21:00:18 2023","interfaces":[{"interface":"back","average":{"1min":1.0189999999999999,"5min":0.81000000000000005,"15min":0.67700000000000005},"min":{"1min":0.35499999999999998,"5min":0.307,"15min":0.30199999999999999},"max":{"1min":4.4240000000000004,"5min":4.4240000000000004,"15min":8.5649999999999995},"last":0.42699999999999999},{"interface":"front","average":{"1min":0.752,"5min":0.71899999999999997,"15min":0.67400000000000004},"min":{"1min":0.373,"5min":0.29799999999999999,"15min":0.27700000000000002},"max":{"1min":2.1789999999999998,"5min":3.8149999999999999,"15min":4.5430000000000001},"last":0.51700000000000002}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":343740416,"data_stored":343736344,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":579014656,"data_stored":579010584,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":623378432,"data_stored":623378432,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":319815680,"data_stored":319815680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":213057536,"data_stored":213057536,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":527826944,"data_stored":527826944,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":191238144,"data_stored":191234072,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":213581824,"data_stored":213581824,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T21:00:51.264 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph status --format=json 2023-09-23T21:00:51.328 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:00:51.334 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:00:51.334 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:00:51.435 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:00:51.467 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:00:51.505 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:00:51.507 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:00:51.508 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:00:51.608 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:00:51.637 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:00:51.642 INFO:teuthology.orchestra.run.smithi026.stdout:ERROR: (22) Invalid argument 2023-09-23T21:00:51.643 INFO:teuthology.orchestra.run.smithi026.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T21:00:51.674 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:00:51.678 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:00:51.679 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:00:51.780 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:00:51.812 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:00:51.853 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:00:51.858 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:00:51.859 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:00:51.960 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:00:52.039 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:00:52.290 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:00:52.294 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:00:52.295 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:00:52.395 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:00:52.397 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_blocked_ops 2023-09-23T21:00:52.539 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:00:52.561 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:00:52.568 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:00:52 smithi026 ceph-mon[179688]: pgmap v640: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 423 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 621 op/s; 11338/20355 objects degraded (55.701%); 0 B/s, 44 objects/s recovering 2023-09-23T21:00:52.573 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:00:52 smithi026 ceph-mon[175151]: pgmap v640: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 423 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 621 op/s; 11338/20355 objects degraded (55.701%); 0 B/s, 44 objects/s recovering 2023-09-23T21:00:52.616 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:00:52.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:00:52 smithi070 ceph-mon[168282]: pgmap v640: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 423 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 621 op/s; 11338/20355 objects degraded (55.701%); 0 B/s, 44 objects/s recovering 2023-09-23T21:00:52.633 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:00:52.633 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:00:52.734 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:00:52.763 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:00:52.802 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:00:52.805 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:00:52.805 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:00:52.906 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:00:52.986 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:00:53.039 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0: {} 2023-09-23T21:00:53.109 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1: {} 2023-09-23T21:00:53.125 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:00:53.142 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:00:53.143 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:00:53.167 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2: {} 2023-09-23T21:00:53.212 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3: {} 2023-09-23T21:00:53.244 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:00:53.245 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4: {} 2023-09-23T21:00:53.279 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:00:53.287 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T21:00:53.307 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T21:00:53.317 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:00:53.321 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:00:53.321 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:00:53.338 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6: {} 2023-09-23T21:00:53.374 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T21:00:53.394 INFO:teuthology.orchestra.run.smithi026.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T21:00:53.394 INFO:teuthology.orchestra.run.smithi026.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T21:00:53.394 INFO:teuthology.orchestra.run.smithi026.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T21:00:53.394 INFO:teuthology.orchestra.run.smithi026.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T21:00:53.394 INFO:teuthology.orchestra.run.smithi026.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T21:00:53.394 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T21:00:53.394 INFO:teuthology.orchestra.run.smithi026.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T21:00:53.394 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T21:00:53.422 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:00:53.450 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:00:53.486 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:00:53.488 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:00:53.488 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:00:53.589 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:00:53.616 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:00:53 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/4063800230' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:00:53.618 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:00:53.636 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:00:53 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/4063800230' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:00:53.636 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:00:53 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/4063800230' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:00:53.657 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:00:53.660 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:00:53.660 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:00:53.761 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:00:53.837 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:00:53.982 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:00:54.008 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:00:54.009 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:00:54.053 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:00:54.110 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:00:54.256 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:00:54.514 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T21:00:54.523 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:00:54.523 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:00:54.586 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:00:54 smithi026 ceph-mon[175151]: pgmap v641: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 314 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 623 op/s; 11080/15102 objects degraded (73.368%); 0 B/s, 48 objects/s recovering 2023-09-23T21:00:54.586 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:00:54 smithi026 ceph-mon[179688]: pgmap v641: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 314 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 623 op/s; 11080/15102 objects degraded (73.368%); 0 B/s, 48 objects/s recovering 2023-09-23T21:00:54.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:00:54 smithi070 ceph-mon[168282]: pgmap v641: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 314 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 623 op/s; 11080/15102 objects degraded (73.368%); 0 B/s, 48 objects/s recovering 2023-09-23T21:00:54.624 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:00:54.733 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:00:54.788 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:00:55.082 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:00:55.086 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:00:55.087 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:00:55.188 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:00:55.221 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:00:55.239 INFO:teuthology.orchestra.run.smithi026.stdout: 2023-09-23T21:00:55.239 INFO:teuthology.orchestra.run.smithi026.stdout:{"fsid":"b3f8e4ec-5a50-11ee-8db4-212e2dc638e7","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"noscrub flag(s) set","count":7},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 11080/15102 objects degraded (73.368%), 3 pgs degraded, 3 pgs undersized","count":6},"muted":false}},"mutes":[]},"election_epoch":30,"quorum":[0,1,2],"quorum_names":["a","b","c"],"quorum_age":773,"monmap":{"epoch":4,"min_mon_release_name":"reef","num_mons":3},"osdmap":{"epoch":250,"num_osds":8,"num_up_osds":8,"osd_up_since":1695502534,"num_in_osds":8,"osd_in_since":1695502470,"num_remapped_pgs":2},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":106},{"state_name":"active+recovery_wait+undersized+degraded+remapped","count":2},{"state_name":"active+recovering+undersized+degraded+remapped","count":1}],"num_pgs":109,"num_pools":5,"num_objects":5034,"data_bytes":328865720,"bytes_used":3939897344,"bytes_avail":764020387840,"bytes_total":767960285184,"degraded_objects":11080,"degraded_total":15102,"degraded_ratio":0.73367765858826639,"recovering_objects_per_sec":48,"recovering_bytes_per_sec":0,"recovering_keys_per_sec":0,"num_objects_recovered":486,"num_bytes_recovered":0,"num_keys_recovered":0,"write_bytes_sec":0,"read_op_per_sec":0,"write_op_per_sec":623},"fsmap":{"epoch":8,"id":1,"up":1,"in":1,"max":1,"by_rank":[{"filesystem_id":1,"rank":0,"name":"foo.smithi026.znfpnl","status":"up:active","gid":14658}],"up:standby":1},"mgrmap":{"available":true,"num_standbys":1,"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"services":{"dashboard":"https://172.21.15.26:8443/","prometheus":"http://172.21.15.26:9283/"}},"servicemap":{"epoch":21,"modified":"2023-09-23T20:56:34.314324+0000","services":{}},"progress_events":{"8b64cbfe-d367-468b-a773-0e71ee4ff645":{"message":"Global Recovery Event (0s)\n [............................] ","progress":0,"add_to_ceph_s":true},"fc746e9d-3f76-434d-b6d2-8fc9e8a72a5e":{"message":"Upgrade to 18.2.0-460-gbb0daeff (4m)\n [============................] (remaining: 5m)","progress":0.44999998807907104,"add_to_ceph_s":true}}} 2023-09-23T21:00:55.270 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:00:55.275 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:00:55.275 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:00:55.376 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:00:55.410 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:00:55.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:00:55 smithi070 ceph-mon[168282]: pgmap v642: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 267 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 618 op/s; 11080/12849 objects degraded (86.232%); 0 B/s, 48 objects/s recovering 2023-09-23T21:00:55.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:00:55 smithi070 ceph-mon[168282]: Health check update: Degraded data redundancy: 11080/15102 objects degraded (73.368%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T21:00:55.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:00:55 smithi070 ceph-mon[168282]: failed to encode map e250 with expected crc 2023-09-23T21:00:55.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:00:55 smithi070 ceph-mon[168282]: osdmap e250: 8 total, 8 up, 8 in 2023-09-23T21:00:55.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:00:55 smithi070 ceph-mon[168282]: failed to encode map e250 with expected crc 2023-09-23T21:00:55.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:00:55 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/1449104138' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-23T21:00:55.626 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:00:55 smithi026 ceph-mon[179688]: pgmap v642: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 267 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 618 op/s; 11080/12849 objects degraded (86.232%); 0 B/s, 48 objects/s recovering 2023-09-23T21:00:55.626 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:00:55 smithi026 ceph-mon[179688]: Health check update: Degraded data redundancy: 11080/15102 objects degraded (73.368%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T21:00:55.626 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:00:55 smithi026 ceph-mon[179688]: failed to encode map e250 with expected crc 2023-09-23T21:00:55.626 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:00:55 smithi026 ceph-mon[179688]: osdmap e250: 8 total, 8 up, 8 in 2023-09-23T21:00:55.626 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:00:55 smithi026 ceph-mon[179688]: failed to encode map e250 with expected crc 2023-09-23T21:00:55.627 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:00:55 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/1449104138' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-23T21:00:55.627 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:00:55 smithi026 ceph-mon[175151]: pgmap v642: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 267 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 618 op/s; 11080/12849 objects degraded (86.232%); 0 B/s, 48 objects/s recovering 2023-09-23T21:00:55.627 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:00:55 smithi026 ceph-mon[175151]: Health check update: Degraded data redundancy: 11080/15102 objects degraded (73.368%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T21:00:55.627 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:00:55 smithi026 ceph-mon[175151]: failed to encode map e250 with expected crc 2023-09-23T21:00:55.627 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:00:55 smithi026 ceph-mon[175151]: osdmap e250: 8 total, 8 up, 8 in 2023-09-23T21:00:55.627 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:00:55 smithi026 ceph-mon[175151]: failed to encode map e250 with expected crc 2023-09-23T21:00:55.627 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:00:55 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/1449104138' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-23T21:00:55.675 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:00:55.678 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:00:55.678 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:00:55.698 INFO:tasks.cephadm.ceph_manager.ceph:making progress, resetting timeout 2023-09-23T21:00:55.698 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T21:00:55.779 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:00:55.812 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:00:55.950 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T21:00:55.954 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:00:55.954 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:00:56.035 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T21:00:56.055 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:00:56.110 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:00:56.211 INFO:teuthology.orchestra.run.smithi026.stdout:{ 2023-09-23T21:00:56.211 INFO:teuthology.orchestra.run.smithi026.stdout: "ops": [], 2023-09-23T21:00:56.211 INFO:teuthology.orchestra.run.smithi026.stdout: "complaint_time": 30, 2023-09-23T21:00:56.211 INFO:teuthology.orchestra.run.smithi026.stdout: "num_blocked_ops": 0 2023-09-23T21:00:56.211 INFO:teuthology.orchestra.run.smithi026.stdout:} 2023-09-23T21:00:56.261 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:00:56.273 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:00:56.274 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:00:56.375 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:00:56.487 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:00:56.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:00:56 smithi070 ceph-mon[168282]: failed to encode map e250 with expected crc 2023-09-23T21:00:56.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:00:56 smithi070 ceph-mon[168282]: failed to encode map e250 with expected crc 2023-09-23T21:00:56.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:00:56 smithi070 ceph-mon[168282]: failed to encode map e250 with expected crc 2023-09-23T21:00:56.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:00:56 smithi070 ceph-mon[168282]: osdmap e251: 8 total, 8 up, 8 in 2023-09-23T21:00:56.635 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:00:56 smithi026 ceph-mon[175151]: failed to encode map e250 with expected crc 2023-09-23T21:00:56.635 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:00:56 smithi026 ceph-mon[175151]: failed to encode map e250 with expected crc 2023-09-23T21:00:56.635 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:00:56 smithi026 ceph-mon[175151]: failed to encode map e250 with expected crc 2023-09-23T21:00:56.635 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:00:56 smithi026 ceph-mon[175151]: osdmap e251: 8 total, 8 up, 8 in 2023-09-23T21:00:56.635 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:00:56 smithi026 ceph-mon[179688]: failed to encode map e250 with expected crc 2023-09-23T21:00:56.636 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:00:56 smithi026 ceph-mon[179688]: failed to encode map e250 with expected crc 2023-09-23T21:00:56.636 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:00:56 smithi026 ceph-mon[179688]: failed to encode map e250 with expected crc 2023-09-23T21:00:56.636 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:00:56 smithi026 ceph-mon[179688]: osdmap e251: 8 total, 8 up, 8 in 2023-09-23T21:00:56.659 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:00:56.668 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:00:56.669 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:00:56.691 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_historic_ops 2023-09-23T21:00:56.769 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:00:56.920 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:00:57.072 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:00:57.076 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:00:57.077 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:00:57.177 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:00:57.178 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:00:57.226 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:00:57.306 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:00:57.445 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:00:57.485 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:00:57.486 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:00:57.586 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:00:57.613 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:00:57 smithi070 ceph-mon[168282]: failed to encode map e251 with expected crc 2023-09-23T21:00:57.613 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:00:57 smithi070 ceph-mon[168282]: failed to encode map e251 with expected crc 2023-09-23T21:00:57.613 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:00:57 smithi070 ceph-mon[168282]: failed to encode map e251 with expected crc 2023-09-23T21:00:57.613 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:00:57 smithi070 ceph-mon[168282]: failed to encode map e251 with expected crc 2023-09-23T21:00:57.613 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:00:57 smithi070 ceph-mon[168282]: pgmap v645: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 233 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 554 op/s; 11080/11214 objects degraded (98.805%); 0 B/s, 32 objects/s recovering 2023-09-23T21:00:57.618 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:00:57.663 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:00:57.669 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:00:57.670 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:00:57.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:00:57 smithi026 ceph-mon[175151]: failed to encode map e251 with expected crc 2023-09-23T21:00:57.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:00:57 smithi026 ceph-mon[175151]: failed to encode map e251 with expected crc 2023-09-23T21:00:57.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:00:57 smithi026 ceph-mon[175151]: failed to encode map e251 with expected crc 2023-09-23T21:00:57.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:00:57 smithi026 ceph-mon[175151]: failed to encode map e251 with expected crc 2023-09-23T21:00:57.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:00:57 smithi026 ceph-mon[175151]: pgmap v645: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 233 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 554 op/s; 11080/11214 objects degraded (98.805%); 0 B/s, 32 objects/s recovering 2023-09-23T21:00:57.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:00:57 smithi026 ceph-mon[179688]: failed to encode map e251 with expected crc 2023-09-23T21:00:57.737 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:00:57 smithi026 ceph-mon[179688]: failed to encode map e251 with expected crc 2023-09-23T21:00:57.737 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:00:57 smithi026 ceph-mon[179688]: failed to encode map e251 with expected crc 2023-09-23T21:00:57.737 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:00:57 smithi026 ceph-mon[179688]: failed to encode map e251 with expected crc 2023-09-23T21:00:57.737 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:00:57 smithi026 ceph-mon[179688]: pgmap v645: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 233 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 554 op/s; 11080/11214 objects degraded (98.805%); 0 B/s, 32 objects/s recovering 2023-09-23T21:00:57.770 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:00:57.806 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:00:57.844 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:00:57.848 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:00:57.849 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:00:57.949 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:00:57.989 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:00:58.036 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:00:58.039 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:00:58.040 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:00:58.141 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:00:58.376 INFO:teuthology.orchestra.run.smithi026.stderr:nodeep-scrub is set 2023-09-23T21:00:58.581 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:00:58.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:00:58 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/3096811473' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:00:58.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:00:58 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:00:58.634 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:00:58 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/3096811473' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:00:58.634 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:00:58 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:00:58.634 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:00:58 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/3096811473' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:00:58.634 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:00:58 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:00:59.615 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:00:59 smithi026 ceph-mon[179688]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:00:59.615 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:00:59 smithi026 ceph-mon[179688]: failed to encode map e252 with expected crc 2023-09-23T21:00:59.615 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:00:59 smithi026 ceph-mon[179688]: failed to encode map e252 with expected crc 2023-09-23T21:00:59.615 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:00:59 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T21:00:59.615 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:00:59 smithi026 ceph-mon[179688]: osdmap e252: 8 total, 8 up, 8 in 2023-09-23T21:00:59.616 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:00:59 smithi026 ceph-mon[179688]: failed to encode map e252 with expected crc 2023-09-23T21:00:59.616 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:00:59 smithi026 ceph-mon[179688]: pgmap v647: 109 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 107 active+clean; 146 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 445 op/s; 10943/7077 objects degraded (154.628%); 0 B/s, 22 objects/s recovering 2023-09-23T21:00:59.616 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:00:59 smithi026 ceph-mon[175151]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:00:59.616 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:00:59 smithi026 ceph-mon[175151]: failed to encode map e252 with expected crc 2023-09-23T21:00:59.616 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:00:59 smithi026 ceph-mon[175151]: failed to encode map e252 with expected crc 2023-09-23T21:00:59.616 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:00:59 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T21:00:59.616 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:00:59 smithi026 ceph-mon[175151]: osdmap e252: 8 total, 8 up, 8 in 2023-09-23T21:00:59.616 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:00:59 smithi026 ceph-mon[175151]: failed to encode map e252 with expected crc 2023-09-23T21:00:59.616 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:00:59 smithi026 ceph-mon[175151]: pgmap v647: 109 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 107 active+clean; 146 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 445 op/s; 10943/7077 objects degraded (154.628%); 0 B/s, 22 objects/s recovering 2023-09-23T21:00:59.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:00:59 smithi070 ceph-mon[168282]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:00:59.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:00:59 smithi070 ceph-mon[168282]: failed to encode map e252 with expected crc 2023-09-23T21:00:59.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:00:59 smithi070 ceph-mon[168282]: failed to encode map e252 with expected crc 2023-09-23T21:00:59.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:00:59 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T21:00:59.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:00:59 smithi070 ceph-mon[168282]: osdmap e252: 8 total, 8 up, 8 in 2023-09-23T21:00:59.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:00:59 smithi070 ceph-mon[168282]: failed to encode map e252 with expected crc 2023-09-23T21:00:59.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:00:59 smithi070 ceph-mon[168282]: pgmap v647: 109 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 107 active+clean; 146 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 445 op/s; 10943/7077 objects degraded (154.628%); 0 B/s, 22 objects/s recovering 2023-09-23T21:00:59.855 INFO:tasks.radosbench.radosbench.0.smithi026.stdout:Removed 101759 objects 2023-09-23T21:00:59.856 INFO:tasks.radosbench.radosbench.0.smithi026.stdout:Clean up completed and total clean up time :170.864 2023-09-23T21:00:59.881 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 21:00:59 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[164029]: ::ffff:172.21.15.70 - - [23/Sep/2023:21:00:59] "GET /metrics HTTP/1.1" 200 36309 "" "Prometheus/2.43.0" 2023-09-23T21:01:00.123 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:01:00.127 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:01:00.127 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:01:00.207 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:01:00.229 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:01:00.254 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:01:00.287 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:01:00.289 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:01:00.290 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:01:00.391 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:01:00.417 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:01:00.449 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:01:00.451 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:01:00.452 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:01:00.553 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:01:00.603 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:01:00.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:01:00 smithi070 ceph-mon[168282]: Health check update: Degraded data redundancy: 10943/7077 objects degraded (154.628%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T21:01:00.633 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:01:00 smithi026 ceph-mon[175151]: Health check update: Degraded data redundancy: 10943/7077 objects degraded (154.628%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T21:01:00.633 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:01:00 smithi026 ceph-mon[179688]: Health check update: Degraded data redundancy: 10943/7077 objects degraded (154.628%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T21:01:00.716 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:01:00.719 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:01:00.719 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:01:00.820 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:01:00.852 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:01:00.988 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:01:00.991 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:01:00.991 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:01:01.094 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:01:01.122 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:01:01.196 INFO:tasks.cephadm.ceph_manager.ceph:removing pool_name unique_pool_1 2023-09-23T21:01:01.196 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph osd pool rm unique_pool_1 unique_pool_1 --yes-i-really-really-mean-it 2023-09-23T21:01:01.509 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0: {} 2023-09-23T21:01:01.510 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:01:01.515 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:01:01.515 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:01:01.533 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1: {} 2023-09-23T21:01:01.553 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2: {} 2023-09-23T21:01:01.576 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3: {} 2023-09-23T21:01:01.599 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4: {} 2023-09-23T21:01:01.616 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:01:01.619 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T21:01:01.643 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:01:01.652 INFO:teuthology.orchestra.run.smithi026.stdout: 2023-09-23T21:01:01.653 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6: {} 2023-09-23T21:01:01.657 INFO:teuthology.orchestra.run.smithi026.stderr:dumped all 2023-09-23T21:01:01.677 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:01:01.677 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T21:01:01.680 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:01:01.680 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:01:01.683 INFO:teuthology.orchestra.run.smithi026.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T21:01:01.683 INFO:teuthology.orchestra.run.smithi026.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T21:01:01.684 INFO:teuthology.orchestra.run.smithi026.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T21:01:01.684 INFO:teuthology.orchestra.run.smithi026.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T21:01:01.684 INFO:teuthology.orchestra.run.smithi026.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T21:01:01.684 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T21:01:01.684 INFO:teuthology.orchestra.run.smithi026.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T21:01:01.684 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T21:01:01.780 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:01:01.807 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:01:01.844 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:01:01.847 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:01:01.848 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:01:01.948 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:01:01.976 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:01:02.009 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:01:02.011 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:01:02.012 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:01:02.113 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:01:02.145 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:01:02.207 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T21:01:02.228 INFO:teuthology.orchestra.run.smithi026.stdout:ERROR: (22) Invalid argument 2023-09-23T21:01:02.228 INFO:teuthology.orchestra.run.smithi026.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T21:01:02.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:01:02 smithi070 ceph-mon[168282]: pgmap v648: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 107 active+clean; 77 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 506 op/s; 10696/3720 objects degraded (287.527%); 0 B/s, 63 objects/s recovering 2023-09-23T21:01:02.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:01:02 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:01:02.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:01:02 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:01:02.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:01:02 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/71176304' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:01:02.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:01:02 smithi026 ceph-mon[179688]: pgmap v648: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 107 active+clean; 77 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 506 op/s; 10696/3720 objects degraded (287.527%); 0 B/s, 63 objects/s recovering 2023-09-23T21:01:02.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:01:02 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:01:02.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:01:02 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:01:02.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:01:02 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/71176304' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:01:02.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:01:02 smithi026 ceph-mon[175151]: pgmap v648: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 107 active+clean; 77 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 506 op/s; 10696/3720 objects degraded (287.527%); 0 B/s, 63 objects/s recovering 2023-09-23T21:01:02.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:01:02 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:01:02.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:01:02 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:01:02.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:01:02 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/71176304' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:01:03.221 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:01:03.223 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:01:03.224 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:01:03.325 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:01:03.354 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:01:03.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:01:03 smithi070 ceph-mon[168282]: from='client.45440 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T21:01:03.631 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:01:03 smithi026 ceph-mon[175151]: from='client.45440 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T21:01:03.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:01:03 smithi026 ceph-mon[179688]: from='client.45440 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T21:01:03.778 INFO:teuthology.orchestra.run.smithi026.stdout:{"pg_ready":true,"pg_map":{"version":648,"stamp":"2023-09-23T21:01:00.503941+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":80222136,"num_objects":1240,"num_object_clones":0,"num_object_copies":3720,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":10696,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1240,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":202365,"num_write_kb":6512606,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5668,"num_bytes_recovered":291641190,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":122740,"ondisk_log_size":122740,"up":327,"acting":325,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":330,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":3160372,"kb_used_data":1351540,"kb_used_omap":136,"kb_used_meta":1808055,"kb_avail":746800844,"statfs":{"total":767960285184,"available":764724064256,"internally_reserved":0,"allocated":1383976960,"data_stored":1375563364,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":139741,"internal_metadata":1851448867},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,1,4,4,4,0,5,4,3],"upper_bound":512},"perf_stat":{"commit_latency_ms":9,"apply_latency_ms":9,"commit_latency_ns":9000000,"apply_latency_ns":9000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-199426048,"num_objects":-3043,"num_object_clones":0,"num_object_copies":-9129,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":-384,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":-3043,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3043,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":384,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"6.004085"},"pg_stats":[{"pgid":"3.1a","version":"68'1","reported_seq":128,"reported_epoch":250,"state":"active+clean","last_fresh":"2023-09-23T21:00:55.117685+0000","last_change":"2023-09-23T20:54:13.946695+0000","last_active":"2023-09-23T21:00:55.117685+0000","last_peered":"2023-09-23T21:00:55.117685+0000","last_clean":"2023-09-23T21:00:55.117685+0000","last_became_active":"2023-09-23T20:54:13.945399+0000","last_became_peered":"2023-09-23T20:54:13.945399+0000","last_unstale":"2023-09-23T21:00:55.117685+0000","last_undegraded":"2023-09-23T21:00:55.117685+0000","last_fullsized":"2023-09-23T21:00:55.117685+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:32:52.365464+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":244,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-09-23T21:00:56.121002+0000","last_change":"2023-09-23T20:54:13.942019+0000","last_active":"2023-09-23T21:00:56.121002+0000","last_peered":"2023-09-23T21:00:56.121002+0000","last_clean":"2023-09-23T21:00:56.121002+0000","last_became_active":"2023-09-23T20:54:13.941647+0000","last_became_peered":"2023-09-23T20:54:13.941647+0000","last_unstale":"2023-09-23T21:00:56.121002+0000","last_undegraded":"2023-09-23T21:00:56.121002+0000","last_fullsized":"2023-09-23T21:00:56.121002+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:30:54.400944+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":222,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-09-23T21:00:56.192916+0000","last_change":"2023-09-23T20:54:13.941394+0000","last_active":"2023-09-23T21:00:56.192916+0000","last_peered":"2023-09-23T21:00:56.192916+0000","last_clean":"2023-09-23T21:00:56.192916+0000","last_became_active":"2023-09-23T20:54:13.941042+0000","last_became_peered":"2023-09-23T20:54:13.941042+0000","last_unstale":"2023-09-23T21:00:56.192916+0000","last_undegraded":"2023-09-23T21:00:56.192916+0000","last_fullsized":"2023-09-23T21:00:56.192916+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:19:41.401184+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":236,"reported_epoch":250,"state":"active+clean","last_fresh":"2023-09-23T21:00:55.109734+0000","last_change":"2023-09-23T20:57:44.979811+0000","last_active":"2023-09-23T21:00:55.109734+0000","last_peered":"2023-09-23T21:00:55.109734+0000","last_clean":"2023-09-23T21:00:55.109734+0000","last_became_active":"2023-09-23T20:57:44.979515+0000","last_became_peered":"2023-09-23T20:57:44.979515+0000","last_unstale":"2023-09-23T21:00:55.109734+0000","last_undegraded":"2023-09-23T21:00:55.109734+0000","last_fullsized":"2023-09-23T21:00:55.109734+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:38:22.162140+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":236,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-09-23T21:00:56.120807+0000","last_change":"2023-09-23T20:51:10.833613+0000","last_active":"2023-09-23T21:00:56.120807+0000","last_peered":"2023-09-23T21:00:56.120807+0000","last_clean":"2023-09-23T21:00:56.120807+0000","last_became_active":"2023-09-23T20:51:10.833208+0000","last_became_peered":"2023-09-23T20:51:10.833208+0000","last_unstale":"2023-09-23T21:00:56.120807+0000","last_undegraded":"2023-09-23T21:00:56.120807+0000","last_fullsized":"2023-09-23T21:00:56.120807+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:51:28.246014+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":217,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-09-23T21:00:56.115335+0000","last_change":"2023-09-23T20:55:35.958384+0000","last_active":"2023-09-23T21:00:56.115335+0000","last_peered":"2023-09-23T21:00:56.115335+0000","last_clean":"2023-09-23T21:00:56.115335+0000","last_became_active":"2023-09-23T20:55:35.954563+0000","last_became_peered":"2023-09-23T20:55:35.954563+0000","last_unstale":"2023-09-23T21:00:56.115335+0000","last_undegraded":"2023-09-23T21:00:56.115335+0000","last_fullsized":"2023-09-23T21:00:56.115335+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:37:48.332259+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":226,"reported_epoch":250,"state":"active+clean","last_fresh":"2023-09-23T21:00:55.111937+0000","last_change":"2023-09-23T20:54:31.885514+0000","last_active":"2023-09-23T21:00:55.111937+0000","last_peered":"2023-09-23T21:00:55.111937+0000","last_clean":"2023-09-23T21:00:55.111937+0000","last_became_active":"2023-09-23T20:54:31.884986+0000","last_became_peered":"2023-09-23T20:54:31.884986+0000","last_unstale":"2023-09-23T21:00:55.111937+0000","last_undegraded":"2023-09-23T21:00:55.111937+0000","last_fullsized":"2023-09-23T21:00:55.111937+0000","mapping_epoch":136,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:26:08.100220+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":233,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-09-23T21:00:56.120723+0000","last_change":"2023-09-23T20:55:35.951889+0000","last_active":"2023-09-23T21:00:56.120723+0000","last_peered":"2023-09-23T21:00:56.120723+0000","last_clean":"2023-09-23T21:00:56.120723+0000","last_became_active":"2023-09-23T20:55:35.951563+0000","last_became_peered":"2023-09-23T20:55:35.951563+0000","last_unstale":"2023-09-23T21:00:56.120723+0000","last_undegraded":"2023-09-23T21:00:56.120723+0000","last_fullsized":"2023-09-23T21:00:56.120723+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:41:53.098781+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"79'161","reported_seq":538,"reported_epoch":250,"state":"active+clean","last_fresh":"2023-09-23T21:00:55.125852+0000","last_change":"2023-09-23T20:57:25.096559+0000","last_active":"2023-09-23T21:00:55.125852+0000","last_peered":"2023-09-23T21:00:55.125852+0000","last_clean":"2023-09-23T21:00:55.125852+0000","last_became_active":"2023-09-23T20:57:15.975674+0000","last_became_peered":"2023-09-23T20:57:15.975674+0000","last_unstale":"2023-09-23T21:00:55.125852+0000","last_undegraded":"2023-09-23T21:00:55.125852+0000","last_fullsized":"2023-09-23T21:00:55.125852+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:25.096401+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:25.096401+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:05:05.448404+0000","scrub_duration":0.002859393,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":124,"reported_epoch":250,"state":"active+clean","last_fresh":"2023-09-23T21:00:55.104151+0000","last_change":"2023-09-23T20:54:31.890678+0000","last_active":"2023-09-23T21:00:55.104151+0000","last_peered":"2023-09-23T21:00:55.104151+0000","last_clean":"2023-09-23T21:00:55.104151+0000","last_became_active":"2023-09-23T20:54:31.890469+0000","last_became_peered":"2023-09-23T20:54:31.890469+0000","last_unstale":"2023-09-23T21:00:55.104151+0000","last_undegraded":"2023-09-23T21:00:55.104151+0000","last_fullsized":"2023-09-23T21:00:55.104151+0000","mapping_epoch":136,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:55:08.790146+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":194,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-09-23T21:00:56.120833+0000","last_change":"2023-09-23T20:55:35.952661+0000","last_active":"2023-09-23T21:00:56.120833+0000","last_peered":"2023-09-23T21:00:56.120833+0000","last_clean":"2023-09-23T21:00:56.120833+0000","last_became_active":"2023-09-23T20:55:35.952349+0000","last_became_peered":"2023-09-23T20:55:35.952349+0000","last_unstale":"2023-09-23T21:00:56.120833+0000","last_undegraded":"2023-09-23T21:00:56.120833+0000","last_fullsized":"2023-09-23T21:00:56.120833+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:00:03.110126+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":70,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-09-23T21:00:56.121012+0000","last_change":"2023-09-23T20:56:45.988607+0000","last_active":"2023-09-23T21:00:56.121012+0000","last_peered":"2023-09-23T21:00:56.121012+0000","last_clean":"2023-09-23T21:00:56.121012+0000","last_became_active":"2023-09-23T20:56:45.988206+0000","last_became_peered":"2023-09-23T20:56:45.988206+0000","last_unstale":"2023-09-23T21:00:56.121012+0000","last_undegraded":"2023-09-23T21:00:56.121012+0000","last_fullsized":"2023-09-23T21:00:56.121012+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":205,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:01:19.783688+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,4],"acting":[3,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":68,"reported_epoch":250,"state":"active+clean","last_fresh":"2023-09-23T21:00:55.104377+0000","last_change":"2023-09-23T20:56:18.139142+0000","last_active":"2023-09-23T21:00:55.104377+0000","last_peered":"2023-09-23T21:00:55.104377+0000","last_clean":"2023-09-23T21:00:55.104377+0000","last_became_active":"2023-09-23T20:56:18.138844+0000","last_became_peered":"2023-09-23T20:56:18.138844+0000","last_unstale":"2023-09-23T21:00:55.104377+0000","last_undegraded":"2023-09-23T21:00:55.104377+0000","last_fullsized":"2023-09-23T21:00:55.104377+0000","mapping_epoch":192,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":193,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:09:02.217683+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,2],"acting":[5,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.14","version":"79'22","reported_seq":155,"reported_epoch":250,"state":"active+clean","last_fresh":"2023-09-23T21:00:55.117816+0000","last_change":"2023-09-23T20:55:35.949418+0000","last_active":"2023-09-23T21:00:55.117816+0000","last_peered":"2023-09-23T21:00:55.117816+0000","last_clean":"2023-09-23T21:00:55.117816+0000","last_became_active":"2023-09-23T20:55:35.948566+0000","last_became_peered":"2023-09-23T20:55:35.948566+0000","last_unstale":"2023-09-23T21:00:55.117816+0000","last_undegraded":"2023-09-23T21:00:55.117816+0000","last_fullsized":"2023-09-23T21:00:55.117816+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:59:37.433128+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4734,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":145,"reported_epoch":250,"state":"active+clean","last_fresh":"2023-09-23T21:00:55.117880+0000","last_change":"2023-09-23T20:55:35.949178+0000","last_active":"2023-09-23T21:00:55.117880+0000","last_peered":"2023-09-23T21:00:55.117880+0000","last_clean":"2023-09-23T21:00:55.117880+0000","last_became_active":"2023-09-23T20:55:35.948795+0000","last_became_peered":"2023-09-23T20:55:35.948795+0000","last_unstale":"2023-09-23T21:00:55.117880+0000","last_undegraded":"2023-09-23T21:00:55.117880+0000","last_fullsized":"2023-09-23T21:00:55.117880+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:13:08.996932+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.11","version":"79'161","reported_seq":480,"reported_epoch":250,"state":"active+clean","last_fresh":"2023-09-23T21:00:55.108608+0000","last_change":"2023-09-23T20:57:28.756796+0000","last_active":"2023-09-23T21:00:55.108608+0000","last_peered":"2023-09-23T21:00:55.108608+0000","last_clean":"2023-09-23T21:00:55.108608+0000","last_became_active":"2023-09-23T20:57:15.980606+0000","last_became_peered":"2023-09-23T20:57:15.980606+0000","last_unstale":"2023-09-23T21:00:55.108608+0000","last_undegraded":"2023-09-23T21:00:55.108608+0000","last_fullsized":"2023-09-23T21:00:55.108608+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:28.756634+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:28.756634+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:09:19.150791+0000","scrub_duration":0.0064925119999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":171,"reported_epoch":250,"state":"active+clean","last_fresh":"2023-09-23T21:00:55.117805+0000","last_change":"2023-09-23T20:56:45.992577+0000","last_active":"2023-09-23T21:00:55.117805+0000","last_peered":"2023-09-23T21:00:55.117805+0000","last_clean":"2023-09-23T21:00:55.117805+0000","last_became_active":"2023-09-23T20:56:45.992297+0000","last_became_peered":"2023-09-23T20:56:45.992297+0000","last_unstale":"2023-09-23T21:00:55.117805+0000","last_undegraded":"2023-09-23T21:00:55.117805+0000","last_fullsized":"2023-09-23T21:00:55.117805+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":205,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:00:45.547202+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,2],"acting":[4,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":218,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-09-23T21:00:56.120682+0000","last_change":"2023-09-23T20:51:10.831262+0000","last_active":"2023-09-23T21:00:56.120682+0000","last_peered":"2023-09-23T21:00:56.120682+0000","last_clean":"2023-09-23T21:00:56.120682+0000","last_became_active":"2023-09-23T20:51:10.830931+0000","last_became_peered":"2023-09-23T20:51:10.830931+0000","last_unstale":"2023-09-23T21:00:56.120682+0000","last_undegraded":"2023-09-23T21:00:56.120682+0000","last_fullsized":"2023-09-23T21:00:56.120682+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:51:54.982505+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.10","version":"79'161","reported_seq":488,"reported_epoch":250,"state":"active+clean","last_fresh":"2023-09-23T21:00:55.102873+0000","last_change":"2023-09-23T20:57:20.708003+0000","last_active":"2023-09-23T21:00:55.102873+0000","last_peered":"2023-09-23T21:00:55.102873+0000","last_clean":"2023-09-23T21:00:55.102873+0000","last_became_active":"2023-09-23T20:57:15.975302+0000","last_became_peered":"2023-09-23T20:57:15.975302+0000","last_unstale":"2023-09-23T21:00:55.102873+0000","last_undegraded":"2023-09-23T21:00:55.102873+0000","last_fullsized":"2023-09-23T21:00:55.102873+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:20.707939+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:20.707939+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:16:51.721994+0000","scrub_duration":0.0017733409999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":226,"reported_epoch":250,"state":"active+clean","last_fresh":"2023-09-23T21:00:55.109161+0000","last_change":"2023-09-23T20:57:44.979736+0000","last_active":"2023-09-23T21:00:55.109161+0000","last_peered":"2023-09-23T21:00:55.109161+0000","last_clean":"2023-09-23T21:00:55.109161+0000","last_became_active":"2023-09-23T20:57:44.979356+0000","last_became_peered":"2023-09-23T20:57:44.979356+0000","last_unstale":"2023-09-23T21:00:55.109161+0000","last_undegraded":"2023-09-23T21:00:55.109161+0000","last_fullsized":"2023-09-23T21:00:55.109161+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:14:43.848832+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":124,"reported_epoch":250,"state":"active+clean","last_fresh":"2023-09-23T21:00:55.103426+0000","last_change":"2023-09-23T20:54:31.887814+0000","last_active":"2023-09-23T21:00:55.103426+0000","last_peered":"2023-09-23T21:00:55.103426+0000","last_clean":"2023-09-23T21:00:55.103426+0000","last_became_active":"2023-09-23T20:54:31.887195+0000","last_became_peered":"2023-09-23T20:54:31.887195+0000","last_unstale":"2023-09-23T21:00:55.103426+0000","last_undegraded":"2023-09-23T21:00:55.103426+0000","last_fullsized":"2023-09-23T21:00:55.103426+0000","mapping_epoch":136,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:10:52.089553+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.13","version":"79'161","reported_seq":552,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-09-23T21:00:56.194045+0000","last_change":"2023-09-23T20:57:48.262970+0000","last_active":"2023-09-23T21:00:56.194045+0000","last_peered":"2023-09-23T21:00:56.194045+0000","last_clean":"2023-09-23T21:00:56.194045+0000","last_became_active":"2023-09-23T20:57:48.262663+0000","last_became_peered":"2023-09-23T20:57:48.262663+0000","last_unstale":"2023-09-23T21:00:56.194045+0000","last_undegraded":"2023-09-23T21:00:56.194045+0000","last_fullsized":"2023-09-23T21:00:56.194045+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:36.420362+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:36.420362+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:28:28.613492+0000","scrub_duration":0.028838444000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":195,"reported_epoch":250,"state":"active+clean","last_fresh":"2023-09-23T21:00:55.124628+0000","last_change":"2023-09-23T20:56:45.991822+0000","last_active":"2023-09-23T21:00:55.124628+0000","last_peered":"2023-09-23T21:00:55.124628+0000","last_clean":"2023-09-23T21:00:55.124628+0000","last_became_active":"2023-09-23T20:56:45.991460+0000","last_became_peered":"2023-09-23T20:56:45.991460+0000","last_unstale":"2023-09-23T21:00:55.124628+0000","last_undegraded":"2023-09-23T21:00:55.124628+0000","last_fullsized":"2023-09-23T21:00:55.124628+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":205,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:20:37.503317+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":208,"reported_epoch":250,"state":"active+clean","last_fresh":"2023-09-23T21:00:55.109854+0000","last_change":"2023-09-23T20:51:10.840849+0000","last_active":"2023-09-23T21:00:55.109854+0000","last_peered":"2023-09-23T21:00:55.109854+0000","last_clean":"2023-09-23T21:00:55.109854+0000","last_became_active":"2023-09-23T20:51:10.840593+0000","last_became_peered":"2023-09-23T20:51:10.840593+0000","last_unstale":"2023-09-23T21:00:55.109854+0000","last_undegraded":"2023-09-23T21:00:55.109854+0000","last_fullsized":"2023-09-23T21:00:55.109854+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:30:52.297625+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.12","version":"79'161","reported_seq":488,"reported_epoch":250,"state":"active+clean","last_fresh":"2023-09-23T21:00:55.104210+0000","last_change":"2023-09-23T20:57:21.750765+0000","last_active":"2023-09-23T21:00:55.104210+0000","last_peered":"2023-09-23T21:00:55.104210+0000","last_clean":"2023-09-23T21:00:55.104210+0000","last_became_active":"2023-09-23T20:57:15.979195+0000","last_became_peered":"2023-09-23T20:57:15.979195+0000","last_unstale":"2023-09-23T21:00:55.104210+0000","last_undegraded":"2023-09-23T21:00:55.104210+0000","last_fullsized":"2023-09-23T21:00:55.104210+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:21.750680+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:21.750680+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:03:41.915962+0000","scrub_duration":0.0092391680000000007,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":244,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-09-23T21:00:56.111932+0000","last_change":"2023-09-23T20:55:36.943969+0000","last_active":"2023-09-23T21:00:56.111932+0000","last_peered":"2023-09-23T21:00:56.111932+0000","last_clean":"2023-09-23T21:00:56.111932+0000","last_became_active":"2023-09-23T20:55:36.943580+0000","last_became_peered":"2023-09-23T20:55:36.943580+0000","last_unstale":"2023-09-23T21:00:56.111932+0000","last_undegraded":"2023-09-23T21:00:56.111932+0000","last_fullsized":"2023-09-23T21:00:56.111932+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:49:06.969396+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":41,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-09-23T21:00:56.121463+0000","last_change":"2023-09-23T20:57:44.980498+0000","last_active":"2023-09-23T21:00:56.121463+0000","last_peered":"2023-09-23T21:00:56.121463+0000","last_clean":"2023-09-23T21:00:56.121463+0000","last_became_active":"2023-09-23T20:57:44.980182+0000","last_became_peered":"2023-09-23T20:57:44.980182+0000","last_unstale":"2023-09-23T21:00:56.121463+0000","last_undegraded":"2023-09-23T21:00:56.121463+0000","last_fullsized":"2023-09-23T21:00:56.121463+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:34:51.620544+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.d","version":"79'161","reported_seq":533,"reported_epoch":250,"state":"active+clean","last_fresh":"2023-09-23T21:00:55.118136+0000","last_change":"2023-09-23T20:57:31.075213+0000","last_active":"2023-09-23T21:00:55.118136+0000","last_peered":"2023-09-23T21:00:55.118136+0000","last_clean":"2023-09-23T21:00:55.118136+0000","last_became_active":"2023-09-23T20:57:15.988188+0000","last_became_peered":"2023-09-23T20:57:15.988188+0000","last_unstale":"2023-09-23T21:00:55.118136+0000","last_undegraded":"2023-09-23T21:00:55.118136+0000","last_fullsized":"2023-09-23T21:00:55.118136+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:31.075098+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:31.075098+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:41:52.142362+0000","scrub_duration":0.010304610000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.b","version":"252'12665","reported_seq":12728,"reported_epoch":252,"state":"active+clean","last_fresh":"2023-09-23T21:00:59.367494+0000","last_change":"2023-09-23T20:56:35.543994+0000","last_active":"2023-09-23T21:00:59.367494+0000","last_peered":"2023-09-23T21:00:59.367494+0000","last_clean":"2023-09-23T21:00:59.367494+0000","last_became_active":"2023-09-23T20:56:35.543758+0000","last_became_peered":"2023-09-23T20:56:35.543758+0000","last_unstale":"2023-09-23T21:00:59.367494+0000","last_undegraded":"2023-09-23T21:00:59.367494+0000","last_fullsized":"2023-09-23T21:00:59.367494+0000","mapping_epoch":199,"log_start":"227'5600","ondisk_log_start":"227'5600","created":199,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_clean_scrub_stamp":"2023-09-23T20:56:34.529103+0000","objects_scrubbed":0,"log_size":7065,"log_dups_size":0,"ondisk_log_size":7065,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:21:29.041484+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":983040,"num_objects":15,"num_object_clones":0,"num_object_copies":45,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":15,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12665,"num_write_kb":405760,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":182,"reported_epoch":250,"state":"active+clean","last_fresh":"2023-09-23T21:00:55.118551+0000","last_change":"2023-09-23T20:56:45.991841+0000","last_active":"2023-09-23T21:00:55.118551+0000","last_peered":"2023-09-23T21:00:55.118551+0000","last_clean":"2023-09-23T21:00:55.118551+0000","last_became_active":"2023-09-23T20:56:45.991491+0000","last_became_peered":"2023-09-23T20:56:45.991491+0000","last_unstale":"2023-09-23T21:00:55.118551+0000","last_undegraded":"2023-09-23T21:00:55.118551+0000","last_fullsized":"2023-09-23T21:00:55.118551+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":205,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:30:37.405319+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":141,"reported_epoch":250,"state":"active+clean","last_fresh":"2023-09-23T21:00:55.103046+0000","last_change":"2023-09-23T20:55:35.957768+0000","last_active":"2023-09-23T21:00:55.103046+0000","last_peered":"2023-09-23T21:00:55.103046+0000","last_clean":"2023-09-23T21:00:55.103046+0000","last_became_active":"2023-09-23T20:55:35.957421+0000","last_became_peered":"2023-09-23T20:55:35.957421+0000","last_unstale":"2023-09-23T21:00:55.103046+0000","last_undegraded":"2023-09-23T21:00:55.103046+0000","last_fullsized":"2023-09-23T21:00:55.103046+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:49:41.585961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.c","version":"79'161","reported_seq":553,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-09-23T21:00:56.120484+0000","last_change":"2023-09-23T20:57:22.041071+0000","last_active":"2023-09-23T21:00:56.120484+0000","last_peered":"2023-09-23T21:00:56.120484+0000","last_clean":"2023-09-23T21:00:56.120484+0000","last_became_active":"2023-09-23T20:57:15.973204+0000","last_became_peered":"2023-09-23T20:57:15.973204+0000","last_unstale":"2023-09-23T21:00:56.120484+0000","last_undegraded":"2023-09-23T21:00:56.120484+0000","last_fullsized":"2023-09-23T21:00:56.120484+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:22.040907+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:22.040907+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:47:02.019978+0000","scrub_duration":0.013021176000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.a","version":"252'12735","reported_seq":12799,"reported_epoch":252,"state":"active+clean","last_fresh":"2023-09-23T21:00:59.785507+0000","last_change":"2023-09-23T20:56:35.547120+0000","last_active":"2023-09-23T21:00:59.785507+0000","last_peered":"2023-09-23T21:00:59.785507+0000","last_clean":"2023-09-23T21:00:59.785507+0000","last_became_active":"2023-09-23T20:56:35.546838+0000","last_became_peered":"2023-09-23T20:56:35.546838+0000","last_unstale":"2023-09-23T21:00:59.785507+0000","last_undegraded":"2023-09-23T21:00:59.785507+0000","last_fullsized":"2023-09-23T21:00:59.785507+0000","mapping_epoch":199,"log_start":"232'6600","ondisk_log_start":"232'6600","created":199,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_clean_scrub_stamp":"2023-09-23T20:56:34.529103+0000","objects_scrubbed":0,"log_size":6135,"log_dups_size":0,"ondisk_log_size":6135,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:48:24.132768+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":65536,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12735,"num_write_kb":407552,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,5],"acting":[3,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":169,"reported_epoch":250,"state":"active+clean","last_fresh":"2023-09-23T21:00:55.118338+0000","last_change":"2023-09-23T20:56:45.991821+0000","last_active":"2023-09-23T21:00:55.118338+0000","last_peered":"2023-09-23T21:00:55.118338+0000","last_clean":"2023-09-23T21:00:55.118338+0000","last_became_active":"2023-09-23T20:56:45.991447+0000","last_became_peered":"2023-09-23T20:56:45.991447+0000","last_unstale":"2023-09-23T21:00:55.118338+0000","last_undegraded":"2023-09-23T21:00:55.118338+0000","last_fullsized":"2023-09-23T21:00:55.118338+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":205,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:14:25.525222+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":172,"reported_epoch":250,"state":"active+clean","last_fresh":"2023-09-23T21:00:55.124912+0000","last_change":"2023-09-23T20:57:44.982059+0000","last_active":"2023-09-23T21:00:55.124912+0000","last_peered":"2023-09-23T21:00:55.124912+0000","last_clean":"2023-09-23T21:00:55.124912+0000","last_became_active":"2023-09-23T20:57:44.981742+0000","last_became_peered":"2023-09-23T20:57:44.981742+0000","last_unstale":"2023-09-23T21:00:55.124912+0000","last_undegraded":"2023-09-23T21:00:55.124912+0000","last_fullsized":"2023-09-23T21:00:55.124912+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:41:24.042916+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.f","version":"79'161","reported_seq":556,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-09-23T21:00:56.112843+0000","last_change":"2023-09-23T20:57:27.880841+0000","last_active":"2023-09-23T21:00:56.112843+0000","last_peered":"2023-09-23T21:00:56.112843+0000","last_clean":"2023-09-23T21:00:56.112843+0000","last_became_active":"2023-09-23T20:57:15.970925+0000","last_became_peered":"2023-09-23T20:57:15.970925+0000","last_unstale":"2023-09-23T21:00:56.112843+0000","last_undegraded":"2023-09-23T21:00:56.112843+0000","last_fullsized":"2023-09-23T21:00:56.112843+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:27.880776+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:27.880776+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:15:13.438014+0000","scrub_duration":0.0014004899999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.9","version":"251'12598","reported_seq":12660,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-09-23T21:00:56.895924+0000","last_change":"2023-09-23T20:56:35.549387+0000","last_active":"2023-09-23T21:00:56.895924+0000","last_peered":"2023-09-23T21:00:56.895924+0000","last_clean":"2023-09-23T21:00:56.895924+0000","last_became_active":"2023-09-23T20:56:35.549087+0000","last_became_peered":"2023-09-23T20:56:35.549087+0000","last_unstale":"2023-09-23T21:00:56.895924+0000","last_undegraded":"2023-09-23T21:00:56.895924+0000","last_fullsized":"2023-09-23T21:00:56.895924+0000","mapping_epoch":199,"log_start":"220'5200","ondisk_log_start":"220'5200","created":199,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_clean_scrub_stamp":"2023-09-23T20:56:34.529103+0000","objects_scrubbed":0,"log_size":7398,"log_dups_size":0,"ondisk_log_size":7398,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:04:51.547362+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":7077888,"num_objects":108,"num_object_clones":0,"num_object_copies":324,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":108,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12598,"num_write_kb":406592,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":155,"reported_epoch":250,"state":"active+clean","last_fresh":"2023-09-23T21:00:55.103386+0000","last_change":"2023-09-23T20:56:45.989153+0000","last_active":"2023-09-23T21:00:55.103386+0000","last_peered":"2023-09-23T21:00:55.103386+0000","last_clean":"2023-09-23T21:00:55.103386+0000","last_became_active":"2023-09-23T20:56:45.988800+0000","last_became_peered":"2023-09-23T20:56:45.988800+0000","last_unstale":"2023-09-23T21:00:55.103386+0000","last_undegraded":"2023-09-23T21:00:55.103386+0000","last_fullsized":"2023-09-23T21:00:55.103386+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":205,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:04:41.130194+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":191,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-09-23T21:00:56.112013+0000","last_change":"2023-09-23T20:55:35.965427+0000","last_active":"2023-09-23T21:00:56.112013+0000","last_peered":"2023-09-23T21:00:56.112013+0000","last_clean":"2023-09-23T21:00:56.112013+0000","last_became_active":"2023-09-23T20:55:35.963969+0000","last_became_peered":"2023-09-23T20:55:35.963969+0000","last_unstale":"2023-09-23T21:00:56.112013+0000","last_undegraded":"2023-09-23T21:00:56.112013+0000","last_fullsized":"2023-09-23T21:00:56.112013+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:38:40.612083+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.e","version":"79'161","reported_seq":462,"reported_epoch":250,"state":"active+clean","last_fresh":"2023-09-23T21:00:55.118212+0000","last_change":"2023-09-23T20:57:34.132738+0000","last_active":"2023-09-23T21:00:55.118212+0000","last_peered":"2023-09-23T21:00:55.118212+0000","last_clean":"2023-09-23T21:00:55.118212+0000","last_became_active":"2023-09-23T20:57:15.974318+0000","last_became_peered":"2023-09-23T20:57:15.974318+0000","last_unstale":"2023-09-23T21:00:55.118212+0000","last_undegraded":"2023-09-23T21:00:55.118212+0000","last_fullsized":"2023-09-23T21:00:55.118212+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:34.132660+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:34.132660+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:07:07.739438+0000","scrub_duration":0.005229694,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.8","version":"251'12700","reported_seq":12763,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-09-23T21:00:56.540015+0000","last_change":"2023-09-23T20:56:35.548119+0000","last_active":"2023-09-23T21:00:56.540015+0000","last_peered":"2023-09-23T21:00:56.540015+0000","last_clean":"2023-09-23T21:00:56.540015+0000","last_became_active":"2023-09-23T20:56:35.547914+0000","last_became_peered":"2023-09-23T20:56:35.547914+0000","last_unstale":"2023-09-23T21:00:56.540015+0000","last_undegraded":"2023-09-23T21:00:56.540015+0000","last_fullsized":"2023-09-23T21:00:56.540015+0000","mapping_epoch":199,"log_start":"215'3800","ondisk_log_start":"215'3800","created":199,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_clean_scrub_stamp":"2023-09-23T20:56:34.529103+0000","objects_scrubbed":0,"log_size":8900,"log_dups_size":0,"ondisk_log_size":8900,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:42:54.255023+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":10092544,"num_objects":154,"num_object_clones":0,"num_object_copies":462,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":154,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12700,"num_write_kb":411328,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":141,"reported_epoch":250,"state":"active+clean","last_fresh":"2023-09-23T21:00:55.103249+0000","last_change":"2023-09-23T20:55:35.962005+0000","last_active":"2023-09-23T21:00:55.103249+0000","last_peered":"2023-09-23T21:00:55.103249+0000","last_clean":"2023-09-23T21:00:55.103249+0000","last_became_active":"2023-09-23T20:55:35.961695+0000","last_became_peered":"2023-09-23T20:55:35.961695+0000","last_unstale":"2023-09-23T21:00:55.103249+0000","last_undegraded":"2023-09-23T21:00:55.103249+0000","last_fullsized":"2023-09-23T21:00:55.103249+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:28:22.876201+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":240,"reported_epoch":250,"state":"active+clean","last_fresh":"2023-09-23T21:00:55.108795+0000","last_change":"2023-09-23T20:54:13.928715+0000","last_active":"2023-09-23T21:00:55.108795+0000","last_peered":"2023-09-23T21:00:55.108795+0000","last_clean":"2023-09-23T21:00:55.108795+0000","last_became_active":"2023-09-23T20:54:13.928200+0000","last_became_peered":"2023-09-23T20:54:13.928200+0000","last_unstale":"2023-09-23T21:00:55.108795+0000","last_undegraded":"2023-09-23T21:00:55.108795+0000","last_fullsized":"2023-09-23T21:00:55.108795+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:41:16.927513+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.f","version":"251'12771","reported_seq":12833,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-09-23T21:00:56.906892+0000","last_change":"2023-09-23T20:56:35.545128+0000","last_active":"2023-09-23T21:00:56.906892+0000","last_peered":"2023-09-23T21:00:56.906892+0000","last_clean":"2023-09-23T21:00:56.906892+0000","last_became_active":"2023-09-23T20:56:35.544838+0000","last_became_peered":"2023-09-23T20:56:35.544838+0000","last_unstale":"2023-09-23T21:00:56.906892+0000","last_undegraded":"2023-09-23T21:00:56.906892+0000","last_fullsized":"2023-09-23T21:00:56.906892+0000","mapping_epoch":199,"log_start":"220'5400","ondisk_log_start":"220'5400","created":199,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_clean_scrub_stamp":"2023-09-23T20:56:34.529103+0000","objects_scrubbed":0,"log_size":7371,"log_dups_size":0,"ondisk_log_size":7371,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:30:43.388717+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":6225920,"num_objects":95,"num_object_clones":0,"num_object_copies":285,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":95,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12771,"num_write_kb":411712,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.9","version":"79'161","reported_seq":625,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-09-23T21:00:56.120732+0000","last_change":"2023-09-23T20:56:46.921939+0000","last_active":"2023-09-23T21:00:56.120732+0000","last_peered":"2023-09-23T21:00:56.120732+0000","last_clean":"2023-09-23T21:00:56.120732+0000","last_became_active":"2023-09-23T20:56:45.989908+0000","last_became_peered":"2023-09-23T20:56:45.989908+0000","last_unstale":"2023-09-23T21:00:56.120732+0000","last_undegraded":"2023-09-23T21:00:56.120732+0000","last_fullsized":"2023-09-23T21:00:56.120732+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":205,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:56:46.921739+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:56:46.921739+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:20:42.635110+0000","scrub_duration":0.0071954109999999996,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":242,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-09-23T21:00:56.120631+0000","last_change":"2023-09-23T20:54:13.961788+0000","last_active":"2023-09-23T21:00:56.120631+0000","last_peered":"2023-09-23T21:00:56.120631+0000","last_clean":"2023-09-23T21:00:56.120631+0000","last_became_active":"2023-09-23T20:54:13.940937+0000","last_became_peered":"2023-09-23T20:54:13.940937+0000","last_unstale":"2023-09-23T21:00:56.120631+0000","last_undegraded":"2023-09-23T21:00:56.120631+0000","last_fullsized":"2023-09-23T21:00:56.120631+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:11:56.785299+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":145,"reported_epoch":250,"state":"active+clean","last_fresh":"2023-09-23T21:00:55.118151+0000","last_change":"2023-09-23T20:55:35.949670+0000","last_active":"2023-09-23T21:00:55.118151+0000","last_peered":"2023-09-23T21:00:55.118151+0000","last_clean":"2023-09-23T21:00:55.118151+0000","last_became_active":"2023-09-23T20:55:35.949140+0000","last_became_peered":"2023-09-23T20:55:35.949140+0000","last_unstale":"2023-09-23T21:00:55.118151+0000","last_undegraded":"2023-09-23T21:00:55.118151+0000","last_fullsized":"2023-09-23T21:00:55.118151+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:55:59.340303+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.e","version":"251'12647","reported_seq":12709,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-09-23T21:00:56.564218+0000","last_change":"2023-09-23T20:56:35.547184+0000","last_active":"2023-09-23T21:00:56.564218+0000","last_peered":"2023-09-23T21:00:56.564218+0000","last_clean":"2023-09-23T21:00:56.564218+0000","last_became_active":"2023-09-23T20:56:35.546912+0000","last_became_peered":"2023-09-23T20:56:35.546912+0000","last_unstale":"2023-09-23T21:00:56.564218+0000","last_undegraded":"2023-09-23T21:00:56.564218+0000","last_fullsized":"2023-09-23T21:00:56.564218+0000","mapping_epoch":199,"log_start":"215'3800","ondisk_log_start":"215'3800","created":199,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_clean_scrub_stamp":"2023-09-23T20:56:34.529103+0000","objects_scrubbed":0,"log_size":8847,"log_dups_size":0,"ondisk_log_size":8847,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:57:07.422792+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":8716288,"num_objects":133,"num_object_clones":0,"num_object_copies":399,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":133,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12647,"num_write_kb":408960,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,0],"acting":[2,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.8","version":"79'161","reported_seq":494,"reported_epoch":250,"state":"active+clean","last_fresh":"2023-09-23T21:00:55.102969+0000","last_change":"2023-09-23T20:56:57.505737+0000","last_active":"2023-09-23T21:00:55.102969+0000","last_peered":"2023-09-23T21:00:55.102969+0000","last_clean":"2023-09-23T21:00:55.102969+0000","last_became_active":"2023-09-23T20:56:04.982345+0000","last_became_peered":"2023-09-23T20:56:04.982345+0000","last_unstale":"2023-09-23T21:00:55.102969+0000","last_undegraded":"2023-09-23T21:00:55.102969+0000","last_fullsized":"2023-09-23T21:00:55.102969+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:56:57.505663+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:56:57.505663+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:34:30.109724+0000","scrub_duration":0.026284108,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":214,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-09-23T21:00:56.112205+0000","last_change":"2023-09-23T20:55:35.959942+0000","last_active":"2023-09-23T21:00:56.112205+0000","last_peered":"2023-09-23T21:00:56.112205+0000","last_clean":"2023-09-23T21:00:56.112205+0000","last_became_active":"2023-09-23T20:55:35.957071+0000","last_became_peered":"2023-09-23T20:55:35.957071+0000","last_unstale":"2023-09-23T21:00:56.112205+0000","last_undegraded":"2023-09-23T21:00:56.112205+0000","last_fullsized":"2023-09-23T21:00:56.112205+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:44:30.308036+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":128,"reported_epoch":250,"state":"active+clean","last_fresh":"2023-09-23T21:00:55.118473+0000","last_change":"2023-09-23T20:54:13.951459+0000","last_active":"2023-09-23T21:00:55.118473+0000","last_peered":"2023-09-23T21:00:55.118473+0000","last_clean":"2023-09-23T21:00:55.118473+0000","last_became_active":"2023-09-23T20:54:13.948409+0000","last_became_peered":"2023-09-23T20:54:13.948409+0000","last_unstale":"2023-09-23T21:00:55.118473+0000","last_undegraded":"2023-09-23T21:00:55.118473+0000","last_fullsized":"2023-09-23T21:00:55.118473+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:38:10.148061+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.b","version":"79'161","reported_seq":535,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-09-23T21:00:56.194088+0000","last_change":"2023-09-23T20:57:27.320172+0000","last_active":"2023-09-23T21:00:56.194088+0000","last_peered":"2023-09-23T21:00:56.194088+0000","last_clean":"2023-09-23T21:00:56.194088+0000","last_became_active":"2023-09-23T20:57:16.009630+0000","last_became_peered":"2023-09-23T20:57:16.009630+0000","last_unstale":"2023-09-23T21:00:56.194088+0000","last_undegraded":"2023-09-23T21:00:56.194088+0000","last_fullsized":"2023-09-23T21:00:56.194088+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:27.320022+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:27.320022+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:31:35.221558+0000","scrub_duration":0.0043894679999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,5],"acting":[1,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.d","version":"252'12759","reported_seq":12821,"reported_epoch":252,"state":"active+clean","last_fresh":"2023-09-23T21:00:59.070143+0000","last_change":"2023-09-23T20:56:35.544132+0000","last_active":"2023-09-23T21:00:59.070143+0000","last_peered":"2023-09-23T21:00:59.070143+0000","last_clean":"2023-09-23T21:00:59.070143+0000","last_became_active":"2023-09-23T20:56:35.543836+0000","last_became_peered":"2023-09-23T20:56:35.543836+0000","last_unstale":"2023-09-23T21:00:59.070143+0000","last_undegraded":"2023-09-23T21:00:59.070143+0000","last_fullsized":"2023-09-23T21:00:59.070143+0000","mapping_epoch":199,"log_start":"221'5400","ondisk_log_start":"221'5400","created":199,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_clean_scrub_stamp":"2023-09-23T20:56:34.529103+0000","objects_scrubbed":0,"log_size":7359,"log_dups_size":0,"ondisk_log_size":7359,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:27:14.636991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":1966104,"num_objects":31,"num_object_clones":0,"num_object_copies":93,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":31,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":12759,"num_write_kb":409217,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":128,"reported_epoch":250,"state":"active+clean","last_fresh":"2023-09-23T21:00:55.124705+0000","last_change":"2023-09-23T20:54:13.950254+0000","last_active":"2023-09-23T21:00:55.124705+0000","last_peered":"2023-09-23T21:00:55.124705+0000","last_clean":"2023-09-23T21:00:55.124705+0000","last_became_active":"2023-09-23T20:54:13.949934+0000","last_became_peered":"2023-09-23T20:54:13.949934+0000","last_unstale":"2023-09-23T21:00:55.124705+0000","last_undegraded":"2023-09-23T21:00:55.124705+0000","last_fullsized":"2023-09-23T21:00:55.124705+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:30:32.847167+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":145,"reported_epoch":250,"state":"active+clean","last_fresh":"2023-09-23T21:00:55.118345+0000","last_change":"2023-09-23T20:55:35.956153+0000","last_active":"2023-09-23T21:00:55.118345+0000","last_peered":"2023-09-23T21:00:55.118345+0000","last_clean":"2023-09-23T21:00:55.118345+0000","last_became_active":"2023-09-23T20:55:35.955630+0000","last_became_peered":"2023-09-23T20:55:35.955630+0000","last_unstale":"2023-09-23T21:00:55.118345+0000","last_undegraded":"2023-09-23T21:00:55.118345+0000","last_fullsized":"2023-09-23T21:00:55.118345+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:45:20.282929+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.c","version":"252'12775","reported_seq":13341,"reported_epoch":252,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-09-23T21:00:59.098687+0000","last_change":"2023-09-23T21:00:54.702933+0000","last_active":"2023-09-23T21:00:59.098687+0000","last_peered":"2023-09-23T21:00:59.098687+0000","last_clean":"2023-09-23T20:57:44.015119+0000","last_became_active":"2023-09-23T20:57:46.448702+0000","last_became_peered":"2023-09-23T20:57:46.448702+0000","last_unstale":"2023-09-23T21:00:59.098687+0000","last_undegraded":"2023-09-23T20:57:46.010874+0000","last_fullsized":"2023-09-23T20:57:45.978434+0000","mapping_epoch":223,"log_start":"221'5400","ondisk_log_start":"221'5400","created":199,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_clean_scrub_stamp":"2023-09-23T20:56:34.529103+0000","objects_scrubbed":0,"log_size":7375,"log_dups_size":0,"ondisk_log_size":7375,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:08:34.180651+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":2162688,"num_objects":33,"num_object_clones":0,"num_object_copies":99,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5226,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":33,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12775,"num_write_kb":409856,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":247,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,7],"avail_no_missing":["6","2","7"],"object_location_counts":[{"shards":"6,7","objects":-5193}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.a","version":"79'161","reported_seq":494,"reported_epoch":250,"state":"active+clean","last_fresh":"2023-09-23T21:00:55.104446+0000","last_change":"2023-09-23T20:56:44.487778+0000","last_active":"2023-09-23T21:00:55.104446+0000","last_peered":"2023-09-23T21:00:55.104446+0000","last_clean":"2023-09-23T21:00:55.104446+0000","last_became_active":"2023-09-23T20:56:05.006740+0000","last_became_peered":"2023-09-23T20:56:05.006740+0000","last_unstale":"2023-09-23T21:00:55.104446+0000","last_undegraded":"2023-09-23T21:00:55.104446+0000","last_fullsized":"2023-09-23T21:00:55.104446+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:56:44.487678+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:56:44.487678+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:04:50.086749+0000","scrub_duration":0.0017408899999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":41,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-09-23T21:00:56.120176+0000","last_change":"2023-09-23T20:57:44.982513+0000","last_active":"2023-09-23T21:00:56.120176+0000","last_peered":"2023-09-23T21:00:56.120176+0000","last_clean":"2023-09-23T21:00:56.120176+0000","last_became_active":"2023-09-23T20:57:44.982112+0000","last_became_peered":"2023-09-23T20:57:44.982112+0000","last_unstale":"2023-09-23T21:00:56.120176+0000","last_undegraded":"2023-09-23T21:00:56.120176+0000","last_fullsized":"2023-09-23T21:00:56.120176+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:09:05.630041+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":222,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-09-23T21:00:56.193923+0000","last_change":"2023-09-23T20:54:13.940748+0000","last_active":"2023-09-23T21:00:56.193923+0000","last_peered":"2023-09-23T21:00:56.193923+0000","last_clean":"2023-09-23T21:00:56.193923+0000","last_became_active":"2023-09-23T20:54:13.940419+0000","last_became_peered":"2023-09-23T20:54:13.940419+0000","last_unstale":"2023-09-23T21:00:56.193923+0000","last_undegraded":"2023-09-23T21:00:56.193923+0000","last_fullsized":"2023-09-23T21:00:56.193923+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:15:01.923816+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.1","version":"251'12686","reported_seq":12759,"reported_epoch":251,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T21:00:57.198424+0000","last_change":"2023-09-23T20:57:46.483248+0000","last_active":"2023-09-23T21:00:57.198424+0000","last_peered":"2023-09-23T21:00:57.198424+0000","last_clean":"2023-09-23T20:57:44.047957+0000","last_became_active":"2023-09-23T20:57:46.475568+0000","last_became_peered":"2023-09-23T20:57:46.475568+0000","last_unstale":"2023-09-23T21:00:57.198424+0000","last_undegraded":"2023-09-23T20:57:46.083591+0000","last_fullsized":"2023-09-23T20:57:45.978436+0000","mapping_epoch":223,"log_start":"229'6000","ondisk_log_start":"229'6000","created":199,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_clean_scrub_stamp":"2023-09-23T20:56:34.529103+0000","objects_scrubbed":0,"log_size":6686,"log_dups_size":0,"ondisk_log_size":6686,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:22:45.067269+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":6553600,"num_objects":100,"num_object_clones":0,"num_object_copies":300,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5470,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12686,"num_write_kb":409152,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,6],"avail_no_missing":["4","1","6"],"object_location_counts":[{"shards":"4,6","objects":-5370}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.7","version":"79'161","reported_seq":557,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-09-23T21:00:56.115358+0000","last_change":"2023-09-23T20:57:48.243572+0000","last_active":"2023-09-23T21:00:56.115358+0000","last_peered":"2023-09-23T21:00:56.115358+0000","last_clean":"2023-09-23T21:00:56.115358+0000","last_became_active":"2023-09-23T20:57:48.243229+0000","last_became_peered":"2023-09-23T20:57:48.243229+0000","last_unstale":"2023-09-23T21:00:56.115358+0000","last_undegraded":"2023-09-23T21:00:56.115358+0000","last_fullsized":"2023-09-23T21:00:56.115358+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":3,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:25.916867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:25.916867+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:23:19.385831+0000","scrub_duration":0.001665323,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":228,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-09-23T21:00:56.193317+0000","last_change":"2023-09-23T20:50:05.867130+0000","last_active":"2023-09-23T21:00:56.193317+0000","last_peered":"2023-09-23T21:00:56.193317+0000","last_clean":"2023-09-23T21:00:56.193317+0000","last_became_active":"2023-09-23T20:50:05.866758+0000","last_became_peered":"2023-09-23T20:50:05.866758+0000","last_unstale":"2023-09-23T21:00:56.193317+0000","last_undegraded":"2023-09-23T21:00:56.193317+0000","last_fullsized":"2023-09-23T21:00:56.193317+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:55:07.548209+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":124,"reported_epoch":250,"state":"active+clean","last_fresh":"2023-09-23T21:00:55.103464+0000","last_change":"2023-09-23T20:54:31.887604+0000","last_active":"2023-09-23T21:00:55.103464+0000","last_peered":"2023-09-23T21:00:55.103464+0000","last_clean":"2023-09-23T21:00:55.103464+0000","last_became_active":"2023-09-23T20:54:31.886752+0000","last_became_peered":"2023-09-23T20:54:31.886752+0000","last_unstale":"2023-09-23T21:00:55.103464+0000","last_undegraded":"2023-09-23T21:00:55.103464+0000","last_fullsized":"2023-09-23T21:00:55.103464+0000","mapping_epoch":136,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:13:58.872176+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":226,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-09-23T21:00:56.193686+0000","last_change":"2023-09-23T20:54:31.899082+0000","last_active":"2023-09-23T21:00:56.193686+0000","last_peered":"2023-09-23T21:00:56.193686+0000","last_clean":"2023-09-23T21:00:56.193686+0000","last_became_active":"2023-09-23T20:54:31.898521+0000","last_became_peered":"2023-09-23T20:54:31.898521+0000","last_unstale":"2023-09-23T21:00:56.193686+0000","last_undegraded":"2023-09-23T21:00:56.193686+0000","last_fullsized":"2023-09-23T21:00:56.193686+0000","mapping_epoch":136,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:25:36.059711+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.7","version":"252'12773","reported_seq":12834,"reported_epoch":252,"state":"active+clean","last_fresh":"2023-09-23T21:00:59.091832+0000","last_change":"2023-09-23T20:56:35.541695+0000","last_active":"2023-09-23T21:00:59.091832+0000","last_peered":"2023-09-23T21:00:59.091832+0000","last_clean":"2023-09-23T21:00:59.091832+0000","last_became_active":"2023-09-23T20:56:35.541392+0000","last_became_peered":"2023-09-23T20:56:35.541392+0000","last_unstale":"2023-09-23T21:00:59.091832+0000","last_undegraded":"2023-09-23T21:00:59.091832+0000","last_fullsized":"2023-09-23T21:00:59.091832+0000","mapping_epoch":199,"log_start":"221'5400","ondisk_log_start":"221'5400","created":199,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_clean_scrub_stamp":"2023-09-23T20:56:34.529103+0000","objects_scrubbed":0,"log_size":7373,"log_dups_size":0,"ondisk_log_size":7373,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:57:22.478253+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":2424832,"num_objects":37,"num_object_clones":0,"num_object_copies":111,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":37,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12773,"num_write_kb":409920,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.1","version":"79'161","reported_seq":514,"reported_epoch":250,"state":"active+clean","last_fresh":"2023-09-23T21:00:55.103077+0000","last_change":"2023-09-23T20:57:19.669346+0000","last_active":"2023-09-23T21:00:55.103077+0000","last_peered":"2023-09-23T21:00:55.103077+0000","last_clean":"2023-09-23T21:00:55.103077+0000","last_became_active":"2023-09-23T20:57:08.246046+0000","last_became_peered":"2023-09-23T20:57:08.246046+0000","last_unstale":"2023-09-23T21:00:55.103077+0000","last_undegraded":"2023-09-23T21:00:55.103077+0000","last_fullsized":"2023-09-23T21:00:55.103077+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:19.669259+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:19.669259+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:14:03.435531+0000","scrub_duration":0.0045759559999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":141,"reported_epoch":250,"state":"active+clean","last_fresh":"2023-09-23T21:00:55.104307+0000","last_change":"2023-09-23T20:55:35.959514+0000","last_active":"2023-09-23T21:00:55.104307+0000","last_peered":"2023-09-23T21:00:55.104307+0000","last_clean":"2023-09-23T21:00:55.104307+0000","last_became_active":"2023-09-23T20:55:35.959244+0000","last_became_peered":"2023-09-23T20:55:35.959244+0000","last_unstale":"2023-09-23T21:00:55.104307+0000","last_undegraded":"2023-09-23T21:00:55.104307+0000","last_fullsized":"2023-09-23T21:00:55.104307+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:47:32.665316+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":145,"reported_epoch":250,"state":"active+clean","last_fresh":"2023-09-23T21:00:55.117960+0000","last_change":"2023-09-23T20:55:35.949000+0000","last_active":"2023-09-23T21:00:55.117960+0000","last_peered":"2023-09-23T21:00:55.117960+0000","last_clean":"2023-09-23T21:00:55.117960+0000","last_became_active":"2023-09-23T20:55:35.948410+0000","last_became_peered":"2023-09-23T20:55:35.948410+0000","last_unstale":"2023-09-23T21:00:55.117960+0000","last_undegraded":"2023-09-23T21:00:55.117960+0000","last_fullsized":"2023-09-23T21:00:55.117960+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:26:50.477289+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":220,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-09-23T21:00:56.193116+0000","last_change":"2023-09-23T20:51:10.829889+0000","last_active":"2023-09-23T21:00:56.193116+0000","last_peered":"2023-09-23T21:00:56.193116+0000","last_clean":"2023-09-23T21:00:56.193116+0000","last_became_active":"2023-09-23T20:51:10.829533+0000","last_became_peered":"2023-09-23T20:51:10.829533+0000","last_unstale":"2023-09-23T21:00:56.193116+0000","last_undegraded":"2023-09-23T21:00:56.193116+0000","last_fullsized":"2023-09-23T21:00:56.193116+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:43:04.603460+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.4","version":"251'12579","reported_seq":12642,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-09-23T21:00:56.839213+0000","last_change":"2023-09-23T20:56:35.549909+0000","last_active":"2023-09-23T21:00:56.839213+0000","last_peered":"2023-09-23T21:00:56.839213+0000","last_clean":"2023-09-23T21:00:56.839213+0000","last_became_active":"2023-09-23T20:56:35.549618+0000","last_became_peered":"2023-09-23T20:56:35.549618+0000","last_unstale":"2023-09-23T21:00:56.839213+0000","last_undegraded":"2023-09-23T21:00:56.839213+0000","last_fullsized":"2023-09-23T21:00:56.839213+0000","mapping_epoch":199,"log_start":"220'5200","ondisk_log_start":"220'5200","created":199,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_clean_scrub_stamp":"2023-09-23T20:56:34.529103+0000","objects_scrubbed":0,"log_size":7379,"log_dups_size":0,"ondisk_log_size":7379,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:24:25.918475+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":9371648,"num_objects":143,"num_object_clones":0,"num_object_copies":429,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":143,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12579,"num_write_kb":407104,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.2","version":"79'161","reported_seq":482,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-09-23T21:00:56.112914+0000","last_change":"2023-09-23T20:57:24.802586+0000","last_active":"2023-09-23T21:00:56.112914+0000","last_peered":"2023-09-23T21:00:56.112914+0000","last_clean":"2023-09-23T21:00:56.112914+0000","last_became_active":"2023-09-23T20:57:08.236114+0000","last_became_peered":"2023-09-23T20:57:08.236114+0000","last_unstale":"2023-09-23T21:00:56.112914+0000","last_undegraded":"2023-09-23T21:00:56.112914+0000","last_fullsized":"2023-09-23T21:00:56.112914+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:24.802451+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:24.802451+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:06:22.309980+0000","scrub_duration":0.0090861250000000005,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,4],"acting":[0,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":237,"reported_epoch":250,"state":"active+clean","last_fresh":"2023-09-23T21:00:55.125545+0000","last_change":"2023-09-23T20:51:10.838054+0000","last_active":"2023-09-23T21:00:55.125545+0000","last_peered":"2023-09-23T21:00:55.125545+0000","last_clean":"2023-09-23T21:00:55.125545+0000","last_became_active":"2023-09-23T20:51:10.837750+0000","last_became_peered":"2023-09-23T20:51:10.837750+0000","last_unstale":"2023-09-23T21:00:55.125545+0000","last_undegraded":"2023-09-23T21:00:55.125545+0000","last_fullsized":"2023-09-23T21:00:55.125545+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:15:51.648085+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":220,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-09-23T21:00:56.192905+0000","last_change":"2023-09-23T20:55:35.951146+0000","last_active":"2023-09-23T21:00:56.192905+0000","last_peered":"2023-09-23T21:00:56.192905+0000","last_clean":"2023-09-23T21:00:56.192905+0000","last_became_active":"2023-09-23T20:55:35.950874+0000","last_became_peered":"2023-09-23T20:55:35.950874+0000","last_unstale":"2023-09-23T21:00:56.192905+0000","last_undegraded":"2023-09-23T21:00:56.192905+0000","last_fullsized":"2023-09-23T21:00:56.192905+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:58:13.957677+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":241,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-09-23T21:00:56.193883+0000","last_change":"2023-09-23T20:54:31.898870+0000","last_active":"2023-09-23T21:00:56.193883+0000","last_peered":"2023-09-23T21:00:56.193883+0000","last_clean":"2023-09-23T21:00:56.193883+0000","last_became_active":"2023-09-23T20:54:31.898467+0000","last_became_peered":"2023-09-23T20:54:31.898467+0000","last_unstale":"2023-09-23T21:00:56.193883+0000","last_undegraded":"2023-09-23T21:00:56.193883+0000","last_fullsized":"2023-09-23T21:00:56.193883+0000","mapping_epoch":136,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:56:22.483408+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.6","version":"251'12478","reported_seq":23354,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-09-23T21:00:56.923857+0000","last_change":"2023-09-23T21:00:56.183303+0000","last_active":"2023-09-23T21:00:56.923857+0000","last_peered":"2023-09-23T21:00:56.923857+0000","last_clean":"2023-09-23T21:00:56.923857+0000","last_became_active":"2023-09-23T21:00:56.125755+0000","last_became_peered":"2023-09-23T21:00:56.125755+0000","last_unstale":"2023-09-23T21:00:56.923857+0000","last_undegraded":"2023-09-23T21:00:56.923857+0000","last_fullsized":"2023-09-23T21:00:56.923857+0000","mapping_epoch":250,"log_start":"219'5000","ondisk_log_start":"219'5000","created":199,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_clean_scrub_stamp":"2023-09-23T20:56:34.529103+0000","objects_scrubbed":0,"log_size":7478,"log_dups_size":0,"ondisk_log_size":7478,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:47:57.476464+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":7208960,"num_objects":110,"num_object_clones":0,"num_object_copies":330,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":110,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12478,"num_write_kb":402816,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5398,"num_bytes_recovered":291569664,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":124,"reported_epoch":250,"state":"active+clean","last_fresh":"2023-09-23T21:00:55.103284+0000","last_change":"2023-09-23T20:54:31.883504+0000","last_active":"2023-09-23T21:00:55.103284+0000","last_peered":"2023-09-23T21:00:55.103284+0000","last_clean":"2023-09-23T21:00:55.103284+0000","last_became_active":"2023-09-23T20:54:31.883172+0000","last_became_peered":"2023-09-23T20:54:31.883172+0000","last_unstale":"2023-09-23T21:00:55.103284+0000","last_undegraded":"2023-09-23T21:00:55.103284+0000","last_fullsized":"2023-09-23T21:00:55.103284+0000","mapping_epoch":136,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:45:40.289120+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"79'161","reported_seq":482,"reported_epoch":250,"state":"active+clean","last_fresh":"2023-09-23T21:00:55.108430+0000","last_change":"2023-09-23T20:57:20.738417+0000","last_active":"2023-09-23T21:00:55.108430+0000","last_peered":"2023-09-23T21:00:55.108430+0000","last_clean":"2023-09-23T21:00:55.108430+0000","last_became_active":"2023-09-23T20:57:08.236842+0000","last_became_peered":"2023-09-23T20:57:08.236842+0000","last_unstale":"2023-09-23T21:00:55.108430+0000","last_undegraded":"2023-09-23T21:00:55.108430+0000","last_fullsized":"2023-09-23T21:00:55.108430+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":0,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:20.738273+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:20.738273+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:18:28.327990+0000","scrub_duration":0.0088336339999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":230,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-09-23T21:00:56.120772+0000","last_change":"2023-09-23T20:55:35.950777+0000","last_active":"2023-09-23T21:00:56.120772+0000","last_peered":"2023-09-23T21:00:56.120772+0000","last_clean":"2023-09-23T21:00:56.120772+0000","last_became_active":"2023-09-23T20:55:35.950427+0000","last_became_peered":"2023-09-23T20:55:35.950427+0000","last_unstale":"2023-09-23T21:00:56.120772+0000","last_undegraded":"2023-09-23T21:00:56.120772+0000","last_fullsized":"2023-09-23T21:00:56.120772+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:20:10.371751+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":196,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-09-23T21:00:56.113095+0000","last_change":"2023-09-23T20:55:35.978140+0000","last_active":"2023-09-23T21:00:56.113095+0000","last_peered":"2023-09-23T21:00:56.113095+0000","last_clean":"2023-09-23T21:00:56.113095+0000","last_became_active":"2023-09-23T20:55:35.977831+0000","last_became_peered":"2023-09-23T20:55:35.977831+0000","last_unstale":"2023-09-23T21:00:56.113095+0000","last_undegraded":"2023-09-23T21:00:56.113095+0000","last_fullsized":"2023-09-23T21:00:56.113095+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:25:47.678023+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.5","version":"251'12538","reported_seq":12608,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-09-23T21:00:56.556839+0000","last_change":"2023-09-23T20:56:45.990110+0000","last_active":"2023-09-23T21:00:56.556839+0000","last_peered":"2023-09-23T21:00:56.556839+0000","last_clean":"2023-09-23T21:00:56.556839+0000","last_became_active":"2023-09-23T20:56:45.988394+0000","last_became_peered":"2023-09-23T20:56:45.988394+0000","last_unstale":"2023-09-23T21:00:56.556839+0000","last_undegraded":"2023-09-23T21:00:56.556839+0000","last_fullsized":"2023-09-23T21:00:56.556839+0000","mapping_epoch":204,"log_start":"213'3700","ondisk_log_start":"213'3700","created":199,"last_epoch_clean":205,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_clean_scrub_stamp":"2023-09-23T20:56:34.529103+0000","objects_scrubbed":0,"log_size":8838,"log_dups_size":0,"ondisk_log_size":8838,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:22:03.814210+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":9175040,"num_objects":140,"num_object_clones":0,"num_object_copies":420,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":140,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12538,"num_write_kb":405696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,0],"acting":[2,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"79'161","reported_seq":519,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-09-23T21:00:56.194175+0000","last_change":"2023-09-23T20:57:20.378531+0000","last_active":"2023-09-23T21:00:56.194175+0000","last_peered":"2023-09-23T21:00:56.194175+0000","last_clean":"2023-09-23T21:00:56.194175+0000","last_became_active":"2023-09-23T20:57:08.238019+0000","last_became_peered":"2023-09-23T20:57:08.238019+0000","last_unstale":"2023-09-23T21:00:56.194175+0000","last_undegraded":"2023-09-23T21:00:56.194175+0000","last_fullsized":"2023-09-23T21:00:56.194175+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":3,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:20.377714+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:20.377714+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:00:08.305602+0000","scrub_duration":0.003015228,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":122,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-09-23T21:00:56.193045+0000","last_change":"2023-09-23T20:56:45.991884+0000","last_active":"2023-09-23T21:00:56.193045+0000","last_peered":"2023-09-23T21:00:56.193045+0000","last_clean":"2023-09-23T21:00:56.193045+0000","last_became_active":"2023-09-23T20:56:45.991568+0000","last_became_peered":"2023-09-23T20:56:45.991568+0000","last_unstale":"2023-09-23T21:00:56.193045+0000","last_undegraded":"2023-09-23T21:00:56.193045+0000","last_fullsized":"2023-09-23T21:00:56.193045+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":205,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:21:20.775511+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,0],"acting":[1,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":245,"reported_epoch":250,"state":"active+clean","last_fresh":"2023-09-23T21:00:55.108877+0000","last_change":"2023-09-23T20:57:44.979950+0000","last_active":"2023-09-23T21:00:55.108877+0000","last_peered":"2023-09-23T21:00:55.108877+0000","last_clean":"2023-09-23T21:00:55.108877+0000","last_became_active":"2023-09-23T20:57:44.979346+0000","last_became_peered":"2023-09-23T20:57:44.979346+0000","last_unstale":"2023-09-23T21:00:55.108877+0000","last_undegraded":"2023-09-23T21:00:55.108877+0000","last_fullsized":"2023-09-23T21:00:55.108877+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:10:24.372800+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":224,"reported_epoch":250,"state":"active+clean","last_fresh":"2023-09-23T21:00:55.109521+0000","last_change":"2023-09-23T20:50:37.982235+0000","last_active":"2023-09-23T21:00:55.109521+0000","last_peered":"2023-09-23T21:00:55.109521+0000","last_clean":"2023-09-23T21:00:55.109521+0000","last_became_active":"2023-09-23T20:50:37.981913+0000","last_became_peered":"2023-09-23T20:50:37.981913+0000","last_unstale":"2023-09-23T21:00:55.109521+0000","last_undegraded":"2023-09-23T21:00:55.109521+0000","last_fullsized":"2023-09-23T21:00:55.109521+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:59:57.122584+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.0","version":"251'12566","reported_seq":12626,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-09-23T21:00:57.041935+0000","last_change":"2023-09-23T20:56:35.550635+0000","last_active":"2023-09-23T21:00:57.041935+0000","last_peered":"2023-09-23T21:00:57.041935+0000","last_clean":"2023-09-23T21:00:57.041935+0000","last_became_active":"2023-09-23T20:56:35.550376+0000","last_became_peered":"2023-09-23T20:56:35.550376+0000","last_unstale":"2023-09-23T21:00:57.041935+0000","last_undegraded":"2023-09-23T21:00:57.041935+0000","last_fullsized":"2023-09-23T21:00:57.041935+0000","mapping_epoch":199,"log_start":"216'3900","ondisk_log_start":"216'3900","created":199,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_clean_scrub_stamp":"2023-09-23T20:56:34.529103+0000","objects_scrubbed":0,"log_size":8666,"log_dups_size":0,"ondisk_log_size":8666,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:40:55.020049+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":7602176,"num_objects":116,"num_object_clones":0,"num_object_copies":348,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":116,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12566,"num_write_kb":405824,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.6","version":"79'161","reported_seq":534,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-09-23T21:00:56.112948+0000","last_change":"2023-09-23T20:57:26.907505+0000","last_active":"2023-09-23T21:00:56.112948+0000","last_peered":"2023-09-23T21:00:56.112948+0000","last_clean":"2023-09-23T21:00:56.112948+0000","last_became_active":"2023-09-23T20:57:08.240321+0000","last_became_peered":"2023-09-23T20:57:08.240321+0000","last_unstale":"2023-09-23T21:00:56.112948+0000","last_undegraded":"2023-09-23T21:00:56.112948+0000","last_fullsized":"2023-09-23T21:00:56.112948+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":3,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:26.907408+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:26.907408+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:34:17.392928+0000","scrub_duration":0.010337038,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":168,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-09-23T21:00:56.117779+0000","last_change":"2023-09-23T20:56:45.988566+0000","last_active":"2023-09-23T21:00:56.117779+0000","last_peered":"2023-09-23T21:00:56.117779+0000","last_clean":"2023-09-23T21:00:56.117779+0000","last_became_active":"2023-09-23T20:56:45.988249+0000","last_became_peered":"2023-09-23T20:56:45.988249+0000","last_unstale":"2023-09-23T21:00:56.117779+0000","last_undegraded":"2023-09-23T21:00:56.117779+0000","last_fullsized":"2023-09-23T21:00:56.117779+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":205,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:51:36.644764+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,4],"acting":[0,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":223,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-09-23T21:00:56.193246+0000","last_change":"2023-09-23T20:54:31.898965+0000","last_active":"2023-09-23T21:00:56.193246+0000","last_peered":"2023-09-23T21:00:56.193246+0000","last_clean":"2023-09-23T21:00:56.193246+0000","last_became_active":"2023-09-23T20:54:31.898663+0000","last_became_peered":"2023-09-23T20:54:31.898663+0000","last_unstale":"2023-09-23T21:00:56.193246+0000","last_undegraded":"2023-09-23T21:00:56.193246+0000","last_fullsized":"2023-09-23T21:00:56.193246+0000","mapping_epoch":136,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:53:05.058087+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":212,"reported_epoch":250,"state":"active+clean","last_fresh":"2023-09-23T21:00:55.108940+0000","last_change":"2023-09-23T20:54:31.884717+0000","last_active":"2023-09-23T21:00:55.108940+0000","last_peered":"2023-09-23T21:00:55.108940+0000","last_clean":"2023-09-23T21:00:55.108940+0000","last_became_active":"2023-09-23T20:54:31.884345+0000","last_became_peered":"2023-09-23T20:54:31.884345+0000","last_unstale":"2023-09-23T21:00:55.108940+0000","last_undegraded":"2023-09-23T21:00:55.108940+0000","last_fullsized":"2023-09-23T21:00:55.108940+0000","mapping_epoch":136,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:02:26.393868+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.3","version":"252'12522","reported_seq":12586,"reported_epoch":252,"state":"active+clean","last_fresh":"2023-09-23T21:00:59.799720+0000","last_change":"2023-09-23T20:56:35.549655+0000","last_active":"2023-09-23T21:00:59.799720+0000","last_peered":"2023-09-23T21:00:59.799720+0000","last_clean":"2023-09-23T21:00:59.799720+0000","last_became_active":"2023-09-23T20:56:35.549357+0000","last_became_peered":"2023-09-23T20:56:35.549357+0000","last_unstale":"2023-09-23T21:00:59.799720+0000","last_undegraded":"2023-09-23T21:00:59.799720+0000","last_fullsized":"2023-09-23T21:00:59.799720+0000","mapping_epoch":199,"log_start":"230'6300","ondisk_log_start":"230'6300","created":199,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_clean_scrub_stamp":"2023-09-23T20:56:34.529103+0000","objects_scrubbed":0,"log_size":6222,"log_dups_size":0,"ondisk_log_size":6222,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:37:58.752518+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12522,"num_write_kb":400704,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.5","version":"79'161","reported_seq":514,"reported_epoch":250,"state":"active+clean","last_fresh":"2023-09-23T21:00:55.118214+0000","last_change":"2023-09-23T20:57:30.029479+0000","last_active":"2023-09-23T21:00:55.118214+0000","last_peered":"2023-09-23T21:00:55.118214+0000","last_clean":"2023-09-23T21:00:55.118214+0000","last_became_active":"2023-09-23T20:57:08.249909+0000","last_became_peered":"2023-09-23T20:57:08.249909+0000","last_unstale":"2023-09-23T21:00:55.118214+0000","last_undegraded":"2023-09-23T21:00:55.118214+0000","last_fullsized":"2023-09-23T21:00:55.118214+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":3,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:30.029396+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:30.029396+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:16:46.083145+0000","scrub_duration":0.012914682,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":216,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-09-23T21:00:56.121651+0000","last_change":"2023-09-23T20:51:10.836910+0000","last_active":"2023-09-23T21:00:56.121651+0000","last_peered":"2023-09-23T21:00:56.121651+0000","last_clean":"2023-09-23T21:00:56.121651+0000","last_became_active":"2023-09-23T20:51:10.836557+0000","last_became_peered":"2023-09-23T20:51:10.836557+0000","last_unstale":"2023-09-23T21:00:56.121651+0000","last_undegraded":"2023-09-23T21:00:56.121651+0000","last_fullsized":"2023-09-23T21:00:56.121651+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:51:54.019969+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":223,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-09-23T21:00:56.193286+0000","last_change":"2023-09-23T20:55:35.953444+0000","last_active":"2023-09-23T21:00:56.193286+0000","last_peered":"2023-09-23T21:00:56.193286+0000","last_clean":"2023-09-23T21:00:56.193286+0000","last_became_active":"2023-09-23T20:55:35.953151+0000","last_became_peered":"2023-09-23T20:55:35.953151+0000","last_unstale":"2023-09-23T21:00:56.193286+0000","last_undegraded":"2023-09-23T21:00:56.193286+0000","last_fullsized":"2023-09-23T21:00:56.193286+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:39:12.404391+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":41,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-09-23T21:00:56.120212+0000","last_change":"2023-09-23T20:57:44.982509+0000","last_active":"2023-09-23T21:00:56.120212+0000","last_peered":"2023-09-23T21:00:56.120212+0000","last_clean":"2023-09-23T21:00:56.120212+0000","last_became_active":"2023-09-23T20:57:44.982175+0000","last_became_peered":"2023-09-23T20:57:44.982175+0000","last_unstale":"2023-09-23T21:00:56.120212+0000","last_undegraded":"2023-09-23T21:00:56.120212+0000","last_fullsized":"2023-09-23T21:00:56.120212+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:37:49.215488+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.2","version":"252'12512","reported_seq":12576,"reported_epoch":252,"state":"active+clean","last_fresh":"2023-09-23T21:00:59.779886+0000","last_change":"2023-09-23T20:56:35.543020+0000","last_active":"2023-09-23T21:00:59.779886+0000","last_peered":"2023-09-23T21:00:59.779886+0000","last_clean":"2023-09-23T21:00:59.779886+0000","last_became_active":"2023-09-23T20:56:35.542719+0000","last_became_peered":"2023-09-23T20:56:35.542719+0000","last_unstale":"2023-09-23T21:00:59.779886+0000","last_undegraded":"2023-09-23T21:00:59.779886+0000","last_fullsized":"2023-09-23T21:00:59.779886+0000","mapping_epoch":199,"log_start":"230'6300","ondisk_log_start":"230'6300","created":199,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:56:34.529103+0000","last_clean_scrub_stamp":"2023-09-23T20:56:34.529103+0000","objects_scrubbed":0,"log_size":6212,"log_dups_size":0,"ondisk_log_size":6212,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:52:21.088628+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12512,"num_write_kb":400384,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.4","version":"79'161","reported_seq":531,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-09-23T21:00:56.120579+0000","last_change":"2023-09-23T20:57:20.012904+0000","last_active":"2023-09-23T21:00:56.120579+0000","last_peered":"2023-09-23T21:00:56.120579+0000","last_clean":"2023-09-23T21:00:56.120579+0000","last_became_active":"2023-09-23T20:57:08.237996+0000","last_became_peered":"2023-09-23T20:57:08.237996+0000","last_unstale":"2023-09-23T21:00:56.120579+0000","last_undegraded":"2023-09-23T21:00:56.120579+0000","last_fullsized":"2023-09-23T21:00:56.120579+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":3,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:20.012764+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:20.012764+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:51:28.796946+0000","scrub_duration":0.0045251919999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":201,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-09-23T21:00:56.113014+0000","last_change":"2023-09-23T20:55:35.949541+0000","last_active":"2023-09-23T21:00:56.113014+0000","last_peered":"2023-09-23T21:00:56.113014+0000","last_clean":"2023-09-23T21:00:56.113014+0000","last_became_active":"2023-09-23T20:55:35.944635+0000","last_became_peered":"2023-09-23T20:55:35.944635+0000","last_unstale":"2023-09-23T21:00:56.113014+0000","last_undegraded":"2023-09-23T21:00:56.113014+0000","last_fullsized":"2023-09-23T21:00:56.113014+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:30:36.033585+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"70'1","reported_seq":240,"reported_epoch":250,"state":"active+clean","last_fresh":"2023-09-23T21:00:55.109608+0000","last_change":"2023-09-23T20:54:14.009723+0000","last_active":"2023-09-23T21:00:55.109608+0000","last_peered":"2023-09-23T21:00:55.109608+0000","last_clean":"2023-09-23T21:00:55.109608+0000","last_became_active":"2023-09-23T20:54:13.941771+0000","last_became_peered":"2023-09-23T20:54:13.941771+0000","last_unstale":"2023-09-23T21:00:55.109608+0000","last_undegraded":"2023-09-23T21:00:55.109608+0000","last_fullsized":"2023-09-23T21:00:55.109608+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:48:25.588116+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":155,"reported_epoch":250,"state":"active+clean","last_fresh":"2023-09-23T21:00:55.124781+0000","last_change":"2023-09-23T20:55:35.949780+0000","last_active":"2023-09-23T21:00:55.124781+0000","last_peered":"2023-09-23T21:00:55.124781+0000","last_clean":"2023-09-23T21:00:55.124781+0000","last_became_active":"2023-09-23T20:55:35.949497+0000","last_became_peered":"2023-09-23T20:55:35.949497+0000","last_unstale":"2023-09-23T21:00:55.124781+0000","last_undegraded":"2023-09-23T21:00:55.124781+0000","last_fullsized":"2023-09-23T21:00:55.124781+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:09:37.420882+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":214,"reported_epoch":250,"state":"active+clean","last_fresh":"2023-09-23T21:00:55.108899+0000","last_change":"2023-09-23T20:54:13.929196+0000","last_active":"2023-09-23T21:00:55.108899+0000","last_peered":"2023-09-23T21:00:55.108899+0000","last_clean":"2023-09-23T21:00:55.108899+0000","last_became_active":"2023-09-23T20:54:13.928831+0000","last_became_peered":"2023-09-23T20:54:13.928831+0000","last_unstale":"2023-09-23T21:00:55.108899+0000","last_undegraded":"2023-09-23T21:00:55.108899+0000","last_fullsized":"2023-09-23T21:00:55.108899+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:28:44.316851+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":220,"reported_epoch":250,"state":"active+clean","last_fresh":"2023-09-23T21:00:55.125768+0000","last_change":"2023-09-23T20:52:12.855430+0000","last_active":"2023-09-23T21:00:55.125768+0000","last_peered":"2023-09-23T21:00:55.125768+0000","last_clean":"2023-09-23T21:00:55.125768+0000","last_became_active":"2023-09-23T20:52:12.855115+0000","last_became_peered":"2023-09-23T20:52:12.855115+0000","last_unstale":"2023-09-23T21:00:55.125768+0000","last_undegraded":"2023-09-23T21:00:55.125768+0000","last_fullsized":"2023-09-23T21:00:55.125768+0000","mapping_epoch":100,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":101,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:33:23.802214+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":128,"reported_epoch":250,"state":"active+clean","last_fresh":"2023-09-23T21:00:55.103299+0000","last_change":"2023-09-23T20:54:31.902946+0000","last_active":"2023-09-23T21:00:55.103299+0000","last_peered":"2023-09-23T21:00:55.103299+0000","last_clean":"2023-09-23T21:00:55.103299+0000","last_became_active":"2023-09-23T20:54:31.889484+0000","last_became_peered":"2023-09-23T20:54:31.889484+0000","last_unstale":"2023-09-23T21:00:55.103299+0000","last_undegraded":"2023-09-23T21:00:55.103299+0000","last_fullsized":"2023-09-23T21:00:55.103299+0000","mapping_epoch":136,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:44:44.278398+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":128,"reported_epoch":250,"state":"active+clean","last_fresh":"2023-09-23T21:00:55.117998+0000","last_change":"2023-09-23T20:54:13.950351+0000","last_active":"2023-09-23T21:00:55.117998+0000","last_peered":"2023-09-23T21:00:55.117998+0000","last_clean":"2023-09-23T21:00:55.117998+0000","last_became_active":"2023-09-23T20:54:13.949987+0000","last_became_peered":"2023-09-23T20:54:13.949987+0000","last_unstale":"2023-09-23T21:00:55.117998+0000","last_undegraded":"2023-09-23T21:00:55.117998+0000","last_fullsized":"2023-09-23T21:00:55.117998+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:46:17.765134+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":159,"reported_epoch":250,"state":"active+clean","last_fresh":"2023-09-23T21:00:55.103508+0000","last_change":"2023-09-23T20:56:18.141728+0000","last_active":"2023-09-23T21:00:55.103508+0000","last_peered":"2023-09-23T21:00:55.103508+0000","last_clean":"2023-09-23T21:00:55.103508+0000","last_became_active":"2023-09-23T20:56:18.137979+0000","last_became_peered":"2023-09-23T20:56:18.137979+0000","last_unstale":"2023-09-23T21:00:55.103508+0000","last_undegraded":"2023-09-23T21:00:55.103508+0000","last_fullsized":"2023-09-23T21:00:55.103508+0000","mapping_epoch":192,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":193,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:58:05.658443+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":128,"reported_epoch":250,"state":"active+clean","last_fresh":"2023-09-23T21:00:55.117874+0000","last_change":"2023-09-23T20:54:13.935307+0000","last_active":"2023-09-23T21:00:55.117874+0000","last_peered":"2023-09-23T21:00:55.117874+0000","last_clean":"2023-09-23T21:00:55.117874+0000","last_became_active":"2023-09-23T20:54:13.935000+0000","last_became_peered":"2023-09-23T20:54:13.935000+0000","last_unstale":"2023-09-23T21:00:55.117874+0000","last_undegraded":"2023-09-23T21:00:55.117874+0000","last_fullsized":"2023-09-23T21:00:55.117874+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:47:40.974082+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":239,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-09-23T21:00:56.121331+0000","last_change":"2023-09-23T20:55:35.954870+0000","last_active":"2023-09-23T21:00:56.121331+0000","last_peered":"2023-09-23T21:00:56.121331+0000","last_clean":"2023-09-23T21:00:56.121331+0000","last_became_active":"2023-09-23T20:55:35.954568+0000","last_became_peered":"2023-09-23T20:55:35.954568+0000","last_unstale":"2023-09-23T21:00:56.121331+0000","last_undegraded":"2023-09-23T21:00:56.121331+0000","last_fullsized":"2023-09-23T21:00:56.121331+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:06:22.423566+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":239,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-09-23T21:00:56.120613+0000","last_change":"2023-09-23T20:51:10.838334+0000","last_active":"2023-09-23T21:00:56.120613+0000","last_peered":"2023-09-23T21:00:56.120613+0000","last_clean":"2023-09-23T21:00:56.120613+0000","last_became_active":"2023-09-23T20:51:10.838026+0000","last_became_peered":"2023-09-23T20:51:10.838026+0000","last_unstale":"2023-09-23T21:00:56.120613+0000","last_undegraded":"2023-09-23T21:00:56.120613+0000","last_fullsized":"2023-09-23T21:00:56.120613+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:04:37.009457+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":242,"reported_epoch":250,"state":"active+clean","last_fresh":"2023-09-23T21:00:55.112032+0000","last_change":"2023-09-23T20:54:31.903139+0000","last_active":"2023-09-23T21:00:55.112032+0000","last_peered":"2023-09-23T21:00:55.112032+0000","last_clean":"2023-09-23T21:00:55.112032+0000","last_became_active":"2023-09-23T20:54:31.884520+0000","last_became_peered":"2023-09-23T20:54:31.884520+0000","last_unstale":"2023-09-23T21:00:55.112032+0000","last_undegraded":"2023-09-23T21:00:55.112032+0000","last_fullsized":"2023-09-23T21:00:55.112032+0000","mapping_epoch":136,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:39:34.269705+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1108,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":145,"reported_epoch":250,"state":"active+clean","last_fresh":"2023-09-23T21:00:55.117557+0000","last_change":"2023-09-23T20:55:35.951572+0000","last_active":"2023-09-23T21:00:55.117557+0000","last_peered":"2023-09-23T21:00:55.117557+0000","last_clean":"2023-09-23T21:00:55.117557+0000","last_became_active":"2023-09-23T20:55:35.951294+0000","last_became_peered":"2023-09-23T20:55:35.951294+0000","last_unstale":"2023-09-23T21:00:55.117557+0000","last_undegraded":"2023-09-23T21:00:55.117557+0000","last_fullsized":"2023-09-23T21:00:55.117557+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:36:32.447210+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":7,"num_pg":16,"stat_sum":{"num_bytes":79626264,"num_objects":1216,"num_object_clones":0,"num_object_copies":3648,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":10696,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1216,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":202304,"num_write_kb":6512577,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5646,"num_bytes_recovered":291635200,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1307717632,"data_stored":1307705416,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":119304,"ondisk_log_size":119304,"up":48,"acting":46,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":22,"num_bytes_recovered":5990,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":21,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3381,"ondisk_log_size":3381,"up":63,"acting":63,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201526,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":238728,"kb_used_data":43456,"kb_used_omap":0,"kb_used_meta":195264,"kb_avail":93506424,"statfs":{"total":95995035648,"available":95750578176,"internally_reserved":0,"allocated":44498944,"data_stored":43603075,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":199950336},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,3],"upper_bound":8},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:00:49 2023","interfaces":[{"interface":"back","average":{"1min":3.766,"5min":2.3260000000000001,"15min":1.302},"min":{"1min":0.23200000000000001,"5min":0.216,"15min":0.21099999999999999},"max":{"1min":33.326000000000001,"5min":43.994999999999997,"15min":43.994999999999997},"last":0.28999999999999998},{"interface":"front","average":{"1min":3.8399999999999999,"5min":2.3530000000000002,"15min":1.389},"min":{"1min":0.33200000000000002,"5min":0.28199999999999997,"15min":0.218},"max":{"1min":33.354999999999997,"5min":44.206000000000003,"15min":44.206000000000003},"last":0.47499999999999998}]},{"osd":1,"last update":"Sat Sep 23 21:00:23 2023","interfaces":[{"interface":"back","average":{"1min":1.4419999999999999,"5min":1.6559999999999999,"15min":1.2829999999999999},"min":{"1min":0.28199999999999997,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":6.8330000000000002,"5min":10.622,"15min":22.591999999999999},"last":0.40699999999999997},{"interface":"front","average":{"1min":1.823,"5min":1.7110000000000001,"15min":1.171},"min":{"1min":0.26500000000000001,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":7.9249999999999998,"5min":10.097,"15min":21.437000000000001},"last":0.45800000000000002}]},{"osd":2,"last update":"Sat Sep 23 20:59:55 2023","interfaces":[{"interface":"back","average":{"1min":2.7149999999999999,"5min":2.2000000000000002,"15min":1.294},"min":{"1min":0.30399999999999999,"5min":0.26500000000000001,"15min":0.24399999999999999},"max":{"1min":8.5289999999999999,"5min":10.805999999999999,"15min":10.805999999999999},"last":0.50800000000000001},{"interface":"front","average":{"1min":2.339,"5min":2.2639999999999998,"15min":1.3089999999999999},"min":{"1min":0.26500000000000001,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":11.590999999999999,"5min":45.25,"15min":45.25},"last":0.35499999999999998}]},{"osd":3,"last update":"Sat Sep 23 21:00:32 2023","interfaces":[{"interface":"back","average":{"1min":1.9099999999999999,"5min":2.157,"15min":1.8149999999999999},"min":{"1min":0.39300000000000002,"5min":0.32100000000000001,"15min":0.314},"max":{"1min":5.2549999999999999,"5min":9.9939999999999998,"15min":36.408000000000001},"last":0.44700000000000001},{"interface":"front","average":{"1min":3.5870000000000002,"5min":2.5219999999999998,"15min":1.7529999999999999},"min":{"1min":0.40699999999999997,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":13.83,"5min":14.220000000000001,"15min":35.593000000000004},"last":0.51700000000000002}]},{"osd":4,"last update":"Sat Sep 23 21:00:23 2023","interfaces":[{"interface":"back","average":{"1min":0.76200000000000001,"5min":0.68100000000000005,"15min":0.627},"min":{"1min":0.27900000000000003,"5min":0.23799999999999999,"15min":0.23699999999999999},"max":{"1min":3.8929999999999998,"5min":4.5410000000000004,"15min":5.8920000000000003},"last":0.32500000000000001},{"interface":"front","average":{"1min":0.86799999999999999,"5min":0.72899999999999998,"15min":0.64300000000000002},"min":{"1min":0.34000000000000002,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":3.3999999999999999,"5min":4.9900000000000002,"15min":5.3200000000000003},"last":0.39200000000000002}]},{"osd":5,"last update":"Sat Sep 23 21:00:23 2023","interfaces":[{"interface":"back","average":{"1min":0.56999999999999995,"5min":0.69099999999999995,"15min":0.63500000000000001},"min":{"1min":0.28999999999999998,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":2.5249999999999999,"5min":7.0389999999999997,"15min":7.0389999999999997},"last":0.48599999999999999},{"interface":"front","average":{"1min":0.86899999999999999,"5min":0.752,"15min":0.63900000000000001},"min":{"1min":0.36899999999999999,"5min":0.29999999999999999,"15min":0.27800000000000002},"max":{"1min":3.8719999999999999,"5min":4.5999999999999996,"15min":4.5999999999999996},"last":0.38300000000000001}]},{"osd":6,"last update":"Sat Sep 23 21:00:46 2023","interfaces":[{"interface":"back","average":{"1min":0.85599999999999998,"5min":0.71799999999999997,"15min":0.68400000000000005},"min":{"1min":0.38200000000000001,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":3.4329999999999998,"5min":3.5249999999999999,"15min":3.5249999999999999},"last":0.41799999999999998},{"interface":"front","average":{"1min":1.0620000000000001,"5min":0.80100000000000005,"15min":0.80300000000000005},"min":{"1min":0.40899999999999997,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":3.5569999999999999,"5min":3.5569999999999999,"15min":3.5569999999999999},"last":0.497}]}]},{"osd":6,"up_from":164,"seq":704374636610,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":144904,"kb_used_data":19776,"kb_used_omap":10,"kb_used_meta":125109,"kb_avail":93600248,"statfs":{"total":95995035648,"available":95846653952,"internally_reserved":0,"allocated":20250624,"data_stored":19362614,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":10328,"internal_metadata":128112552},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1,0,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:00:46 2023","interfaces":[{"interface":"back","average":{"1min":2.1469999999999998,"5min":1.7569999999999999,"15min":1.1619999999999999},"min":{"1min":0.26400000000000001,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":10.404,"5min":10.404,"15min":10.404},"last":0.255},{"interface":"front","average":{"1min":2.895,"5min":1.9890000000000001,"15min":1.3180000000000001},"min":{"1min":0.28599999999999998,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":11.023,"5min":11.023,"15min":11.023},"last":3.9820000000000002}]},{"osd":1,"last update":"Sat Sep 23 21:00:46 2023","interfaces":[{"interface":"back","average":{"1min":1.601,"5min":1.4119999999999999,"15min":0.996},"min":{"1min":0.32900000000000001,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":3.839,"5min":8.4309999999999992,"15min":8.4309999999999992},"last":2.4950000000000001},{"interface":"front","average":{"1min":2.6560000000000001,"5min":1.7290000000000001,"15min":1.155},"min":{"1min":0.26200000000000001,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":10.518000000000001,"5min":10.518000000000001,"15min":10.518000000000001},"last":5.3070000000000004}]},{"osd":2,"last update":"Sat Sep 23 21:00:46 2023","interfaces":[{"interface":"back","average":{"1min":2.9249999999999998,"5min":2.3279999999999998,"15min":1.8740000000000001},"min":{"1min":0.371,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":10.109999999999999,"5min":13.750999999999999,"15min":13.750999999999999},"last":1.879},{"interface":"front","average":{"1min":3.1920000000000002,"5min":2.2170000000000001,"15min":1.4770000000000001},"min":{"1min":0.34599999999999997,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":20.195,"5min":20.195,"15min":20.195},"last":4.165}]},{"osd":3,"last update":"Sat Sep 23 21:00:46 2023","interfaces":[{"interface":"back","average":{"1min":4.1660000000000004,"5min":2.0249999999999999,"15min":1.419},"min":{"1min":0.48699999999999999,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":14.144,"5min":14.144,"15min":14.144},"last":2.927},{"interface":"front","average":{"1min":2.7919999999999998,"5min":1.986,"15min":1.73},"min":{"1min":0.38700000000000001,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":11.989000000000001,"5min":11.989000000000001,"15min":11.989000000000001},"last":0.39900000000000002}]},{"osd":4,"last update":"Sat Sep 23 21:00:46 2023","interfaces":[{"interface":"back","average":{"1min":0.81599999999999995,"5min":0.73899999999999999,"15min":0.94699999999999995},"min":{"1min":0.28100000000000003,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":3.4260000000000002,"5min":9.0079999999999991,"15min":9.0079999999999991},"last":4.1230000000000002},{"interface":"front","average":{"1min":0.626,"5min":0.51800000000000002,"15min":0.52600000000000002},"min":{"1min":0.29799999999999999,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":2.4860000000000002,"5min":2.4860000000000002,"15min":2.4860000000000002},"last":0.29699999999999999}]},{"osd":5,"last update":"Sat Sep 23 21:00:46 2023","interfaces":[{"interface":"back","average":{"1min":0.94499999999999995,"5min":0.69799999999999995,"15min":0.59299999999999997},"min":{"1min":0.32700000000000001,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":5.4119999999999999,"5min":5.4119999999999999,"15min":5.4119999999999999},"last":4.149},{"interface":"front","average":{"1min":0.90600000000000003,"5min":0.75900000000000001,"15min":0.79300000000000004},"min":{"1min":0.34499999999999997,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":3.536,"5min":3.536,"15min":3.536},"last":0.32500000000000001}]},{"osd":7,"last update":"Sat Sep 23 21:00:46 2023","interfaces":[{"interface":"back","average":{"1min":1.151,"5min":0.81599999999999995,"15min":0.89100000000000001},"min":{"1min":0.36299999999999999,"5min":0.312,"15min":0.312},"max":{"1min":3.8929999999999998,"5min":7.8570000000000002,"15min":7.8570000000000002},"last":0.38500000000000001},{"interface":"front","average":{"1min":1.0700000000000001,"5min":0.75700000000000001,"15min":0.60499999999999998},"min":{"1min":0.40100000000000002,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":3.3279999999999998,"5min":8.282,"15min":8.282},"last":0.35999999999999999}]}]},{"osd":1,"up_from":88,"seq":377957122181,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":660500,"kb_used_data":371148,"kb_used_omap":9,"kb_used_meta":289270,"kb_avail":93084652,"statfs":{"total":95995035648,"available":95318683648,"internally_reserved":0,"allocated":380055552,"data_stored":378809559,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9239,"internal_metadata":296213481},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1],"upper_bound":2},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:00:26 2023","interfaces":[{"interface":"back","average":{"1min":2.9870000000000001,"5min":3.048,"15min":1.7110000000000001},"min":{"1min":0.26000000000000001,"5min":0.20699999999999999,"15min":0.20699999999999999},"max":{"1min":8.8000000000000007,"5min":16.646999999999998,"15min":16.646999999999998},"last":0.77100000000000002},{"interface":"front","average":{"1min":4.7480000000000002,"5min":3.577,"15min":1.9690000000000001},"min":{"1min":0.38200000000000001,"5min":0.246,"15min":0.22500000000000001},"max":{"1min":27.797999999999998,"5min":27.797999999999998,"15min":27.797999999999998},"last":2.9940000000000002}]},{"osd":2,"last update":"Sat Sep 23 21:00:57 2023","interfaces":[{"interface":"back","average":{"1min":2.3559999999999999,"5min":2.198,"15min":1.4510000000000001},"min":{"1min":0.251,"5min":0.221,"15min":0.20300000000000001},"max":{"1min":7.1280000000000001,"5min":13.752000000000001,"15min":14.335000000000001},"last":0.22600000000000001},{"interface":"front","average":{"1min":5.29,"5min":3.9430000000000001,"15min":2.262},"min":{"1min":0.23699999999999999,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":17.405999999999999,"5min":18.026,"15min":38.951000000000001},"last":7.2649999999999997}]},{"osd":3,"last update":"Sat Sep 23 21:00:26 2023","interfaces":[{"interface":"back","average":{"1min":3.8849999999999998,"5min":2.8220000000000001,"15min":1.794},"min":{"1min":0.38500000000000001,"5min":0.246,"15min":0.246},"max":{"1min":22.686,"5min":22.686,"15min":25.314},"last":0.99399999999999999},{"interface":"front","average":{"1min":3.4510000000000001,"5min":2.605,"15min":1.613},"min":{"1min":0.36699999999999999,"5min":0.20599999999999999,"15min":0.20599999999999999},"max":{"1min":14.473000000000001,"5min":14.473000000000001,"15min":19.881},"last":0.747}]},{"osd":4,"last update":"Sat Sep 23 21:00:26 2023","interfaces":[{"interface":"back","average":{"1min":2.383,"5min":1.704,"15min":1.329},"min":{"1min":0.34499999999999997,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":18.231000000000002,"5min":18.231000000000002,"15min":18.231000000000002},"last":0.317},{"interface":"front","average":{"1min":1.6080000000000001,"5min":1.55,"15min":1.1419999999999999},"min":{"1min":0.35399999999999998,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":6.6360000000000001,"5min":7.532,"15min":11.234999999999999},"last":0.44500000000000001}]},{"osd":5,"last update":"Sat Sep 23 21:00:26 2023","interfaces":[{"interface":"back","average":{"1min":2.6080000000000001,"5min":2.2999999999999998,"15min":1.419},"min":{"1min":0.34999999999999998,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":12.699999999999999,"5min":15.041,"15min":15.041},"last":3.0710000000000002},{"interface":"front","average":{"1min":2.415,"5min":1.762,"15min":1.357},"min":{"1min":0.40000000000000002,"5min":0.32600000000000001,"15min":0.32200000000000001},"max":{"1min":18.280999999999999,"5min":18.280999999999999,"15min":18.280999999999999},"last":0.40000000000000002}]},{"osd":6,"last update":"Sat Sep 23 21:00:45 2023","interfaces":[{"interface":"back","average":{"1min":2.298,"5min":1.7569999999999999,"15min":1.462},"min":{"1min":0.35799999999999998,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":18.984999999999999,"5min":18.984999999999999,"15min":18.984999999999999},"last":0.95799999999999996},{"interface":"front","average":{"1min":1.5840000000000001,"5min":1.696,"15min":1.3999999999999999},"min":{"1min":0.38900000000000001,"5min":0.309,"15min":0.309},"max":{"1min":6.7779999999999996,"5min":8.7530000000000001,"15min":8.7530000000000001},"last":0.72099999999999997}]},{"osd":7,"last update":"Sat Sep 23 21:00:26 2023","interfaces":[{"interface":"back","average":{"1min":1.8180000000000001,"5min":1.7829999999999999,"15min":1.24},"min":{"1min":0.41699999999999998,"5min":0.36199999999999999,"15min":0.35699999999999998},"max":{"1min":6.657,"5min":8.7769999999999992,"15min":11.186999999999999},"last":0.47799999999999998},{"interface":"front","average":{"1min":2.415,"5min":2.319,"15min":1.4399999999999999},"min":{"1min":0.41499999999999998,"5min":0.40899999999999997,"15min":0.34599999999999997},"max":{"1min":12.84,"5min":15.098000000000001,"15min":15.098000000000001},"last":2.9540000000000002}]}]},{"osd":0,"up_from":83,"seq":356482285707,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297584,"kb_used_data":67112,"kb_used_omap":28,"kb_used_meta":230435,"kb_avail":93447568,"statfs":{"total":95995035648,"available":95690309632,"internally_reserved":0,"allocated":68722688,"data_stored":67502983,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":28848,"internal_metadata":235966288},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,0,3,0,0,3,4,3],"upper_bound":512},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 21:00:20 2023","interfaces":[{"interface":"back","average":{"1min":6.4480000000000004,"5min":3.7120000000000002,"15min":1.893},"min":{"1min":0.20000000000000001,"5min":0.191,"15min":0.191},"max":{"1min":19.146000000000001,"5min":19.146000000000001,"15min":19.146000000000001},"last":0.44},{"interface":"front","average":{"1min":5.4379999999999997,"5min":3.6789999999999998,"15min":1.8759999999999999},"min":{"1min":0.39500000000000002,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":11.074,"5min":15.771000000000001,"15min":17.704999999999998},"last":1.016}]},{"osd":2,"last update":"Sat Sep 23 21:00:55 2023","interfaces":[{"interface":"back","average":{"1min":3.9039999999999999,"5min":3.798,"15min":2.1000000000000001},"min":{"1min":0.32100000000000001,"5min":0.30599999999999999,"15min":0.23799999999999999},"max":{"1min":10.914,"5min":21.053000000000001,"15min":21.053000000000001},"last":0.40899999999999997},{"interface":"front","average":{"1min":4.1200000000000001,"5min":4.0369999999999999,"15min":2.1930000000000001},"min":{"1min":0.36099999999999999,"5min":0.25800000000000001,"15min":0.21099999999999999},"max":{"1min":9.548,"5min":21.747,"15min":21.747},"last":0.36099999999999999}]},{"osd":3,"last update":"Sat Sep 23 21:00:20 2023","interfaces":[{"interface":"back","average":{"1min":6.532,"5min":3.8690000000000002,"15min":1.9990000000000001},"min":{"1min":1.052,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":15.082000000000001,"5min":21.184000000000001,"15min":21.184000000000001},"last":0.47699999999999998},{"interface":"front","average":{"1min":4.9050000000000002,"5min":4.008,"15min":2.0899999999999999},"min":{"1min":0.48499999999999999,"5min":0.28000000000000003,"15min":0.23300000000000001},"max":{"1min":9.7560000000000002,"5min":21.167000000000002,"15min":21.167000000000002},"last":0.372}]},{"osd":4,"last update":"Sat Sep 23 21:00:55 2023","interfaces":[{"interface":"back","average":{"1min":2.52,"5min":2.875,"15min":1.6839999999999999},"min":{"1min":0.39200000000000002,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":9.6029999999999998,"5min":21.594999999999999,"15min":21.594999999999999},"last":0.39200000000000002},{"interface":"front","average":{"1min":3.786,"5min":2.4020000000000001,"15min":1.486},"min":{"1min":0.437,"5min":0.31,"15min":0.28100000000000003},"max":{"1min":13.904,"5min":13.904,"15min":18.724},"last":0.504}]},{"osd":5,"last update":"Sat Sep 23 21:00:55 2023","interfaces":[{"interface":"back","average":{"1min":2.794,"5min":2.1339999999999999,"15min":1.401},"min":{"1min":0.309,"5min":0.309,"15min":0.309},"max":{"1min":6.343,"5min":8.8970000000000002,"15min":8.8970000000000002},"last":0.45500000000000002},{"interface":"front","average":{"1min":2.5430000000000001,"5min":2.7109999999999999,"15min":1.6870000000000001},"min":{"1min":0.38700000000000001,"5min":0.38700000000000001,"15min":0.34999999999999998},"max":{"1min":9.6340000000000003,"5min":20.512,"15min":20.512},"last":0.42899999999999999}]},{"osd":6,"last update":"Sat Sep 23 21:00:50 2023","interfaces":[{"interface":"back","average":{"1min":3.4049999999999998,"5min":2.4950000000000001,"15min":1.9359999999999999},"min":{"1min":0.48199999999999998,"5min":0.37,"15min":0.37},"max":{"1min":13.933,"5min":13.933,"15min":13.933},"last":0.53000000000000003},{"interface":"front","average":{"1min":3.8260000000000001,"5min":2.6139999999999999,"15min":2.0350000000000001},"min":{"1min":0.46400000000000002,"5min":0.377,"15min":0.377},"max":{"1min":13.962,"5min":13.962,"15min":13.962},"last":0.57599999999999996}]},{"osd":7,"last update":"Sat Sep 23 21:00:55 2023","interfaces":[{"interface":"back","average":{"1min":2.7919999999999998,"5min":2.254,"15min":1.4570000000000001},"min":{"1min":0.40300000000000002,"5min":0.379,"15min":0.379},"max":{"1min":6.298,"5min":8.7729999999999997,"15min":8.7729999999999997},"last":0.64300000000000002},{"interface":"front","average":{"1min":3.605,"5min":2.4830000000000001,"15min":1.516},"min":{"1min":0.49099999999999999,"5min":0.377,"15min":0.377},"max":{"1min":13.856999999999999,"5min":13.856999999999999,"15min":18.673999999999999},"last":0.55000000000000004}]}]},{"osd":2,"up_from":93,"seq":399431958654,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":726816,"kb_used_data":408984,"kb_used_omap":81,"kb_used_meta":317550,"kb_avail":93018336,"statfs":{"total":95995035648,"available":95250776064,"internally_reserved":0,"allocated":418799616,"data_stored":417592659,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":83750,"internal_metadata":325171418},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:00:50 2023","interfaces":[{"interface":"back","average":{"1min":4.3099999999999996,"5min":3.1720000000000002,"15min":1.6439999999999999},"min":{"1min":0.27300000000000002,"5min":0.17999999999999999,"15min":0.17999999999999999},"max":{"1min":13.494,"5min":29.498000000000001,"15min":29.498000000000001},"last":0.33500000000000002},{"interface":"front","average":{"1min":4.2800000000000002,"5min":3.681,"15min":1.9319999999999999},"min":{"1min":0.29699999999999999,"5min":0.20599999999999999,"15min":0.20599999999999999},"max":{"1min":11.629,"5min":17.251000000000001,"15min":17.251000000000001},"last":0.51600000000000001}]},{"osd":1,"last update":"Sat Sep 23 21:00:50 2023","interfaces":[{"interface":"back","average":{"1min":2.7599999999999998,"5min":2.661,"15min":1.782},"min":{"1min":0.27800000000000002,"5min":0.21199999999999999,"15min":0.20200000000000001},"max":{"1min":13.523,"5min":14.709,"15min":16.081},"last":0.26100000000000001},{"interface":"front","average":{"1min":6.0609999999999999,"5min":3.952,"15min":2.3460000000000001},"min":{"1min":0.26300000000000001,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":26.552,"5min":26.552,"15min":26.552},"last":0.45700000000000002}]},{"osd":3,"last update":"Sat Sep 23 21:00:31 2023","interfaces":[{"interface":"back","average":{"1min":4.71,"5min":3.8450000000000002,"15min":2.1800000000000002},"min":{"1min":0.28599999999999998,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":10.468,"5min":27.614999999999998,"15min":27.614999999999998},"last":0.35299999999999998},{"interface":"front","average":{"1min":4.2939999999999996,"5min":4.0309999999999997,"15min":2.4540000000000002},"min":{"1min":0.64400000000000002,"5min":0.23899999999999999,"15min":0.221},"max":{"1min":11.673999999999999,"5min":27.562000000000001,"15min":27.562000000000001},"last":0.46800000000000003}]},{"osd":4,"last update":"Sat Sep 23 21:00:50 2023","interfaces":[{"interface":"back","average":{"1min":2.1099999999999999,"5min":1.8400000000000001,"15min":1.3069999999999999},"min":{"1min":0.29299999999999998,"5min":0.28699999999999998,"15min":0.26400000000000001},"max":{"1min":10.045,"5min":10.045,"15min":10.045},"last":0.64200000000000002},{"interface":"front","average":{"1min":3.0459999999999998,"5min":2.222,"15min":1.53},"min":{"1min":0.42699999999999999,"5min":0.313,"15min":0.313},"max":{"1min":14.039999999999999,"5min":14.039999999999999,"15min":14.039999999999999},"last":0.60099999999999998}]},{"osd":5,"last update":"Sat Sep 23 21:00:50 2023","interfaces":[{"interface":"back","average":{"1min":2.1150000000000002,"5min":1.857,"15min":1.3959999999999999},"min":{"1min":0.34200000000000003,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":10.069000000000001,"5min":10.069000000000001,"15min":10.069000000000001},"last":0.57499999999999996},{"interface":"front","average":{"1min":3.0329999999999999,"5min":2.1230000000000002,"15min":1.532},"min":{"1min":0.46600000000000003,"5min":0.32600000000000001,"15min":0.30299999999999999},"max":{"1min":13.98,"5min":13.98,"15min":13.98},"last":0.627}]},{"osd":6,"last update":"Sat Sep 23 21:00:47 2023","interfaces":[{"interface":"back","average":{"1min":3.2559999999999998,"5min":2.3820000000000001,"15min":1.9059999999999999},"min":{"1min":0.54500000000000004,"5min":0.312,"15min":0.312},"max":{"1min":14.016,"5min":14.244999999999999,"15min":14.244999999999999},"last":0.65900000000000003},{"interface":"front","average":{"1min":3.702,"5min":2.3479999999999999,"15min":1.4690000000000001},"min":{"1min":0.61399999999999999,"5min":0.374,"15min":0.374},"max":{"1min":14.068,"5min":14.068,"15min":14.068},"last":0.54600000000000004}]},{"osd":7,"last update":"Sat Sep 23 21:00:50 2023","interfaces":[{"interface":"back","average":{"1min":2.0670000000000002,"5min":1.496,"15min":1.1399999999999999},"min":{"1min":0.41899999999999998,"5min":0.315,"15min":0.315},"max":{"1min":13.427,"5min":13.427,"15min":13.427},"last":0.52900000000000003},{"interface":"front","average":{"1min":2.29,"5min":1.907,"15min":1.401},"min":{"1min":0.372,"5min":0.35099999999999998,"15min":0.34499999999999997},"max":{"1min":10.092000000000001,"5min":10.092000000000001,"15min":10.092000000000001},"last":0.61899999999999999}]}]},{"osd":3,"up_from":98,"seq":420906795128,"num_pgs":49,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":378184,"kb_used_data":14272,"kb_used_omap":7,"kb_used_meta":363640,"kb_avail":93366968,"statfs":{"total":95995035648,"available":95607775232,"internally_reserved":0,"allocated":14614528,"data_stored":13436080,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7576,"internal_metadata":372367976},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,1,3,0,2],"upper_bound":128},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:00:28 2023","interfaces":[{"interface":"back","average":{"1min":5.6130000000000004,"5min":3.0470000000000002,"15min":2.0840000000000001},"min":{"1min":0.20399999999999999,"5min":0.20399999999999999,"15min":0.20399999999999999},"max":{"1min":27.962,"5min":27.962,"15min":27.962},"last":14.797000000000001},{"interface":"front","average":{"1min":4.984,"5min":3.5169999999999999,"15min":1.9339999999999999},"min":{"1min":0.21099999999999999,"5min":0.19800000000000001,"15min":0.19800000000000001},"max":{"1min":28.058,"5min":28.058,"15min":28.058},"last":4.3109999999999999}]},{"osd":1,"last update":"Sat Sep 23 21:00:28 2023","interfaces":[{"interface":"back","average":{"1min":4.8730000000000002,"5min":3.1760000000000002,"15min":1.9119999999999999},"min":{"1min":0.22600000000000001,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":22.681999999999999,"5min":22.681999999999999,"15min":22.681999999999999},"last":8.1790000000000003},{"interface":"front","average":{"1min":4.0789999999999997,"5min":2.5169999999999999,"15min":1.929},"min":{"1min":0.25,"5min":0.218,"15min":0.218},"max":{"1min":19.329000000000001,"5min":19.329000000000001,"15min":25.353000000000002},"last":6.6689999999999996}]},{"osd":2,"last update":"Sat Sep 23 21:00:28 2023","interfaces":[{"interface":"back","average":{"1min":6.1680000000000001,"5min":3.8039999999999998,"15min":2.113},"min":{"1min":0.36199999999999999,"5min":0.33000000000000002,"15min":0.30099999999999999},"max":{"1min":28.251999999999999,"5min":28.251999999999999,"15min":28.251999999999999},"last":8.3040000000000003},{"interface":"front","average":{"1min":4.6779999999999999,"5min":2.9300000000000002,"15min":2.0750000000000002},"min":{"1min":0.372,"5min":0.23999999999999999,"15min":0.22600000000000001},"max":{"1min":16.213999999999999,"5min":16.213999999999999,"15min":25.305},"last":4.0960000000000001}]},{"osd":4,"last update":"Sat Sep 23 21:00:28 2023","interfaces":[{"interface":"back","average":{"1min":3.258,"5min":2.2869999999999999,"15min":1.4750000000000001},"min":{"1min":0.314,"5min":0.29899999999999999,"15min":0.29299999999999998},"max":{"1min":18.239000000000001,"5min":18.239000000000001,"15min":18.239000000000001},"last":8.2780000000000005},{"interface":"front","average":{"1min":2.7200000000000002,"5min":1.5860000000000001,"15min":1.474},"min":{"1min":0.317,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":14.808,"5min":14.808,"15min":22.853000000000002},"last":4.0629999999999997}]},{"osd":5,"last update":"Sat Sep 23 21:00:28 2023","interfaces":[{"interface":"back","average":{"1min":3.2930000000000001,"5min":2.3399999999999999,"15min":1.4690000000000001},"min":{"1min":0.33200000000000002,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":18.324999999999999,"5min":18.324999999999999,"15min":18.324999999999999},"last":8.2240000000000002},{"interface":"front","average":{"1min":3.0910000000000002,"5min":2.363,"15min":1.4359999999999999},"min":{"1min":0.33700000000000002,"5min":0.33300000000000002,"15min":0.33100000000000002},"max":{"1min":11.327,"5min":13.43,"15min":13.43},"last":0.67400000000000004}]},{"osd":6,"last update":"Sat Sep 23 21:00:51 2023","interfaces":[{"interface":"back","average":{"1min":3.601,"5min":1.9450000000000001,"15min":1.2809999999999999},"min":{"1min":0.373,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":14.833,"5min":14.833,"15min":14.833},"last":4.1230000000000002},{"interface":"front","average":{"1min":4.641,"5min":2.6110000000000002,"15min":1.8109999999999999},"min":{"1min":0.40899999999999997,"5min":0.38700000000000001,"15min":0.38700000000000001},"max":{"1min":13.576000000000001,"5min":13.576000000000001,"15min":13.576000000000001},"last":0.77100000000000002}]},{"osd":7,"last update":"Sat Sep 23 21:00:28 2023","interfaces":[{"interface":"back","average":{"1min":3.2879999999999998,"5min":2.5649999999999999,"15min":1.6399999999999999},"min":{"1min":0.39200000000000002,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":18.289999999999999,"5min":18.289999999999999,"15min":18.289999999999999},"last":8.2509999999999994},{"interface":"front","average":{"1min":3.1219999999999999,"5min":2.399,"15min":1.506},"min":{"1min":0.38900000000000001,"5min":0.38900000000000001,"15min":0.379},"max":{"1min":11.378,"5min":13.355,"15min":13.355},"last":0.63700000000000001}]}]},{"osd":4,"up_from":31,"seq":133143986435,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":191240,"kb_used_data":41728,"kb_used_omap":0,"kb_used_meta":149440,"kb_avail":93553912,"statfs":{"total":95995035648,"available":95799205888,"internally_reserved":0,"allocated":42729472,"data_stored":41833856,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":153026560},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:00:55 2023","interfaces":[{"interface":"back","average":{"1min":1.8089999999999999,"5min":1.893,"15min":1.2210000000000001},"min":{"1min":0.218,"5min":0.214,"15min":0.214},"max":{"1min":8.9429999999999996,"5min":11.784000000000001,"15min":11.784000000000001},"last":0.48299999999999998},{"interface":"front","average":{"1min":1.835,"5min":1.9259999999999999,"15min":1.397},"min":{"1min":0.27700000000000002,"5min":0.27700000000000002,"15min":0.25800000000000001},"max":{"1min":8.9049999999999994,"5min":11.766,"15min":11.766},"last":0.38900000000000001}]},{"osd":1,"last update":"Sat Sep 23 21:00:22 2023","interfaces":[{"interface":"back","average":{"1min":1.4350000000000001,"5min":1.673,"15min":1.1779999999999999},"min":{"1min":0.25800000000000001,"5min":0.246,"15min":0.23599999999999999},"max":{"1min":5.5570000000000004,"5min":10.153,"15min":10.153},"last":0.40600000000000003},{"interface":"front","average":{"1min":2.6040000000000001,"5min":2.2749999999999999,"15min":1.2749999999999999},"min":{"1min":0.29199999999999998,"5min":0.26100000000000001,"15min":0.23200000000000001},"max":{"1min":10.882999999999999,"5min":12.292999999999999,"15min":12.292999999999999},"last":0.59399999999999997}]},{"osd":2,"last update":"Sat Sep 23 21:00:55 2023","interfaces":[{"interface":"back","average":{"1min":2.9249999999999998,"5min":1.9490000000000001,"15min":1.3899999999999999},"min":{"1min":0.32500000000000001,"5min":0.313,"15min":0.29099999999999998},"max":{"1min":9.8469999999999995,"5min":9.8469999999999995,"15min":15.288},"last":0.42899999999999999},{"interface":"front","average":{"1min":2.2240000000000002,"5min":1.905,"15min":1.496},"min":{"1min":0.316,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":7.6829999999999998,"5min":12.855,"15min":15.262},"last":0.54600000000000004}]},{"osd":3,"last update":"Sat Sep 23 21:00:22 2023","interfaces":[{"interface":"back","average":{"1min":3.5569999999999999,"5min":2.3479999999999999,"15min":1.8120000000000001},"min":{"1min":0.39200000000000002,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":8.5389999999999997,"5min":14.824999999999999,"15min":31.562999999999999},"last":0.50600000000000001},{"interface":"front","average":{"1min":3.5129999999999999,"5min":2.4550000000000001,"15min":1.8740000000000001},"min":{"1min":0.48399999999999999,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":8.5570000000000004,"5min":14.839,"15min":31.574000000000002},"last":0.62}]},{"osd":5,"last update":"Sat Sep 23 21:00:39 2023","interfaces":[{"interface":"back","average":{"1min":0.33800000000000002,"5min":0.56999999999999995,"15min":0.51900000000000002},"min":{"1min":0.251,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":0.495,"5min":3.008,"15min":3.3279999999999998},"last":0.34499999999999997},{"interface":"front","average":{"1min":0.55500000000000005,"5min":0.71199999999999997,"15min":0.621},"min":{"1min":0.27600000000000002,"5min":0.25600000000000001,"15min":0.251},"max":{"1min":2.6539999999999999,"5min":3.4830000000000001,"15min":7.7619999999999996},"last":0.46899999999999997}]},{"osd":6,"last update":"Sat Sep 23 21:00:51 2023","interfaces":[{"interface":"back","average":{"1min":0.66400000000000003,"5min":0.73999999999999999,"15min":0.56899999999999995},"min":{"1min":0.27300000000000002,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":2.9239999999999999,"5min":4.6909999999999998,"15min":4.6909999999999998},"last":0.45800000000000002},{"interface":"front","average":{"1min":0.54700000000000004,"5min":0.71799999999999997,"15min":0.66800000000000004},"min":{"1min":0.33000000000000002,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":1.8340000000000001,"5min":3.456,"15min":3.456},"last":0.56999999999999995}]},{"osd":7,"last update":"Sat Sep 23 21:00:22 2023","interfaces":[{"interface":"back","average":{"1min":0.52100000000000002,"5min":0.754,"15min":0.69099999999999995},"min":{"1min":0.33400000000000002,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":2.302,"5min":6.2240000000000002,"15min":7.383},"last":0.51800000000000002},{"interface":"front","average":{"1min":0.56100000000000005,"5min":0.77500000000000002,"15min":0.67700000000000005},"min":{"1min":0.32700000000000001,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":2.4420000000000002,"5min":7.8310000000000004,"15min":7.8310000000000004},"last":0.497}]}]},{"osd":5,"up_from":39,"seq":167503724800,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":522416,"kb_used_data":385064,"kb_used_omap":0,"kb_used_meta":137344,"kb_avail":93222736,"statfs":{"total":95995035648,"available":95460081664,"internally_reserved":0,"allocated":394305536,"data_stored":393422538,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":140640256},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:00:41 2023","interfaces":[{"interface":"back","average":{"1min":3.0920000000000001,"5min":2.0419999999999998,"15min":1.1359999999999999},"min":{"1min":0.35199999999999998,"5min":0.20399999999999999,"15min":0.20399999999999999},"max":{"1min":6.665,"5min":12.795999999999999,"15min":12.795999999999999},"last":1.698},{"interface":"front","average":{"1min":3.7250000000000001,"5min":2.2349999999999999,"15min":1.335},"min":{"1min":0.27000000000000002,"5min":0.24099999999999999,"15min":0.223},"max":{"1min":9.1099999999999994,"5min":16.219999999999999,"15min":16.219999999999999},"last":1.577}]},{"osd":1,"last update":"Sat Sep 23 21:00:18 2023","interfaces":[{"interface":"back","average":{"1min":2.8759999999999999,"5min":1.8779999999999999,"15min":1.2529999999999999},"min":{"1min":0.26600000000000001,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":8.7720000000000002,"5min":8.7720000000000002,"15min":8.7720000000000002},"last":0.32900000000000001},{"interface":"front","average":{"1min":2.3639999999999999,"5min":1.9330000000000001,"15min":1.22},"min":{"1min":0.34200000000000003,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":8.5109999999999992,"5min":10.285,"15min":10.285},"last":11.196999999999999}]},{"osd":2,"last update":"Sat Sep 23 20:59:54 2023","interfaces":[{"interface":"back","average":{"1min":2.6819999999999999,"5min":1.6699999999999999,"15min":1.121},"min":{"1min":0.27400000000000002,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":10.371,"5min":10.579000000000001,"15min":12.651},"last":0.40899999999999997},{"interface":"front","average":{"1min":3.423,"5min":2.258,"15min":1.274},"min":{"1min":0.28799999999999998,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":10.333,"5min":12.300000000000001,"15min":12.300000000000001},"last":1.3400000000000001}]},{"osd":3,"last update":"Sat Sep 23 21:00:18 2023","interfaces":[{"interface":"back","average":{"1min":4.931,"5min":2.859,"15min":1.6379999999999999},"min":{"1min":0.35999999999999999,"5min":0.35399999999999998,"15min":0.28599999999999998},"max":{"1min":20.788,"5min":20.788,"15min":20.788},"last":4.71},{"interface":"front","average":{"1min":3.9870000000000001,"5min":2.177,"15min":1.2949999999999999},"min":{"1min":0.39100000000000001,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":20.539000000000001,"5min":20.539000000000001,"15min":20.539000000000001},"last":2.7040000000000002}]},{"osd":4,"last update":"Sat Sep 23 21:00:35 2023","interfaces":[{"interface":"back","average":{"1min":0.69999999999999996,"5min":0.625,"15min":0.56899999999999995},"min":{"1min":0.23400000000000001,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":4.1520000000000001,"5min":4.1520000000000001,"15min":4.1520000000000001},"last":0.26100000000000001},{"interface":"front","average":{"1min":0.79600000000000004,"5min":0.69799999999999995,"15min":0.59599999999999997},"min":{"1min":0.28999999999999998,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":3.7469999999999999,"5min":3.7469999999999999,"15min":3.7469999999999999},"last":0.311}]},{"osd":6,"last update":"Sat Sep 23 21:00:41 2023","interfaces":[{"interface":"back","average":{"1min":0.91600000000000004,"5min":0.72199999999999998,"15min":0.61399999999999999},"min":{"1min":0.314,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":4.359,"5min":4.359,"15min":4.359},"last":0.27900000000000003},{"interface":"front","average":{"1min":0.92200000000000004,"5min":0.79100000000000004,"15min":0.75900000000000001},"min":{"1min":0.36899999999999999,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":3.847,"5min":3.847,"15min":3.847},"last":0.37}]},{"osd":7,"last update":"Sat Sep 23 21:00:18 2023","interfaces":[{"interface":"back","average":{"1min":1.0189999999999999,"5min":0.81000000000000005,"15min":0.67700000000000005},"min":{"1min":0.35499999999999998,"5min":0.307,"15min":0.30199999999999999},"max":{"1min":4.4240000000000004,"5min":4.4240000000000004,"15min":8.5649999999999995},"last":0.35799999999999998},{"interface":"front","average":{"1min":0.752,"5min":0.71899999999999997,"15min":0.67400000000000004},"min":{"1min":0.373,"5min":0.29799999999999999,"15min":0.27700000000000002},"max":{"1min":2.1789999999999998,"5min":3.8149999999999999,"15min":4.5430000000000001},"last":0.33900000000000002}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":62394368,"data_stored":62390296,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":366022656,"data_stored":366018584,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":412352512,"data_stored":412352512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":524288,"data_stored":524288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":37027840,"data_stored":37027840,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":381943808,"data_stored":381943808,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":10031104,"data_stored":10027032,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":37421056,"data_stored":37421056,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T21:01:04.055 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:01:04.058 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:01:04.058 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:01:04.159 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:01:04.186 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:01:04.220 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:01:04.222 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:01:04.223 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:01:04.324 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:01:04.348 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:01:04 smithi026 ceph-mon[175151]: pgmap v649: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 107 active+clean; 7.8 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 485 op/s; 10696/423 objects degraded (2528.605%); 0 B/s, 51 objects/s recovering 2023-09-23T21:01:04.348 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:01:04 smithi026 ceph-mon[179688]: pgmap v649: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 107 active+clean; 7.8 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 485 op/s; 10696/423 objects degraded (2528.605%); 0 B/s, 51 objects/s recovering 2023-09-23T21:01:04.351 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:01:04.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:01:04 smithi070 ceph-mon[168282]: pgmap v649: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 107 active+clean; 7.8 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 485 op/s; 10696/423 objects degraded (2528.605%); 0 B/s, 51 objects/s recovering 2023-09-23T21:01:05.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:01:05 smithi070 ceph-mon[168282]: Health check update: Degraded data redundancy: 10696/423 objects degraded (2528.605%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T21:01:05.633 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:01:05 smithi026 ceph-mon[175151]: Health check update: Degraded data redundancy: 10696/423 objects degraded (2528.605%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T21:01:05.633 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:01:05 smithi026 ceph-mon[179688]: Health check update: Degraded data redundancy: 10696/423 objects degraded (2528.605%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T21:01:05.958 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:01:05.961 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:01:05.962 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:01:06.025 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_ops_in_flight 2023-09-23T21:01:06.063 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:01:06.104 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:01:06.140 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:01:06.143 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:01:06.143 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:01:06.151 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T21:01:06.199 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:01:06.199 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:01:06.213 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:01:06 smithi026 ceph-mon[175151]: pgmap v650: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 107 active+clean; 646 KiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 442 op/s; 10200/75 objects degraded (13600.000%); 0 B/s, 104 objects/s recovering 2023-09-23T21:01:06.213 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:01:06 smithi026 ceph-mon[179688]: pgmap v650: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 107 active+clean; 646 KiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 442 op/s; 10200/75 objects degraded (13600.000%); 0 B/s, 104 objects/s recovering 2023-09-23T21:01:06.244 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:01:06.272 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:01:06.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:01:06 smithi070 ceph-mon[168282]: pgmap v650: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 107 active+clean; 646 KiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 442 op/s; 10200/75 objects degraded (13600.000%); 0 B/s, 104 objects/s recovering 2023-09-23T21:01:06.791 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T21:01:07.191 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:01:07.194 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:01:07.195 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:01:07.295 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:01:07.324 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:01:07.356 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:01:07.359 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:01:07.359 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:01:07.460 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:01:07.492 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:01:08.282 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:01:08 smithi026 ceph-mon[175151]: pgmap v651: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 107 active+clean; 582 KiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 371 op/s; 10200/72 objects degraded (14166.667%); 0 B/s, 87 objects/s recovering 2023-09-23T21:01:08.282 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:01:08 smithi026 ceph-mon[179688]: pgmap v651: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 107 active+clean; 582 KiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 371 op/s; 10200/72 objects degraded (14166.667%); 0 B/s, 87 objects/s recovering 2023-09-23T21:01:08.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:01:08 smithi070 ceph-mon[168282]: pgmap v651: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 107 active+clean; 582 KiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 371 op/s; 10200/72 objects degraded (14166.667%); 0 B/s, 87 objects/s recovering 2023-09-23T21:01:09.901 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 21:01:09 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[164029]: ::ffff:172.21.15.70 - - [23/Sep/2023:21:01:09] "GET /metrics HTTP/1.1" 200 36316 "" "Prometheus/2.43.0" 2023-09-23T21:01:10.108 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:01:10.112 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:01:10.112 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:01:10.213 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:01:10.240 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:01:10.273 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:01:10.275 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:01:10.276 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:01:10.376 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:01:10.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:01:10 smithi026 ceph-mon[179688]: pgmap v652: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 107 active+clean; 582 KiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 230 op/s; 10200/72 objects degraded (14166.667%); 0 B/s, 73 objects/s recovering 2023-09-23T21:01:10.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:01:10 smithi026 ceph-mon[179688]: Health check update: Degraded data redundancy: 10200/72 objects degraded (14166.667%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T21:01:10.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:01:10 smithi026 ceph-mon[175151]: pgmap v652: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 107 active+clean; 582 KiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 230 op/s; 10200/72 objects degraded (14166.667%); 0 B/s, 73 objects/s recovering 2023-09-23T21:01:10.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:01:10 smithi026 ceph-mon[175151]: Health check update: Degraded data redundancy: 10200/72 objects degraded (14166.667%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T21:01:10.409 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:01:10.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:01:10 smithi070 ceph-mon[168282]: pgmap v652: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 107 active+clean; 582 KiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 230 op/s; 10200/72 objects degraded (14166.667%); 0 B/s, 73 objects/s recovering 2023-09-23T21:01:10.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:01:10 smithi070 ceph-mon[168282]: Health check update: Degraded data redundancy: 10200/72 objects degraded (14166.667%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T21:01:12.266 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:01:12.269 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:01:12 smithi026 ceph-mon[175151]: pgmap v653: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 107 active+clean; 582 KiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 194 op/s; 9540/72 objects degraded (13250.000%); 0 B/s, 116 objects/s recovering 2023-09-23T21:01:12.269 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:01:12 smithi026 ceph-mon[179688]: pgmap v653: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 107 active+clean; 582 KiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 194 op/s; 9540/72 objects degraded (13250.000%); 0 B/s, 116 objects/s recovering 2023-09-23T21:01:12.269 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:01:12.270 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:01:12.370 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:01:12.397 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:01:12.430 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:01:12.433 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:01:12.433 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:01:12.534 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:01:12.564 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:01:12.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:01:12 smithi070 ceph-mon[168282]: pgmap v653: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 107 active+clean; 582 KiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 194 op/s; 9540/72 objects degraded (13250.000%); 0 B/s, 116 objects/s recovering 2023-09-23T21:01:13.080 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:01:13.085 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:01:13.085 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:01:13.089 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:01:13.186 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:01:13.212 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:01:13.245 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:01:13.247 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:01:13.248 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:01:13.349 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:01:13.350 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:01:13.374 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:01:13.406 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:01:13.408 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:01:13.409 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:01:13.510 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:01:13.538 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:01:14.155 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:01:14.355 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T21:01:14.359 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:01:14.359 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:01:14.460 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:01:14.587 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:01:14 smithi026 ceph-mon[179688]: pgmap v654: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 107 active+clean; 582 KiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 101 op/s; 9540/72 objects degraded (13250.000%); 0 B/s, 96 objects/s recovering 2023-09-23T21:01:14.588 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:01:14 smithi026 ceph-mon[175151]: pgmap v654: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 107 active+clean; 582 KiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 101 op/s; 9540/72 objects degraded (13250.000%); 0 B/s, 96 objects/s recovering 2023-09-23T21:01:14.591 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:01:14.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:01:14 smithi070 ceph-mon[168282]: pgmap v654: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 107 active+clean; 582 KiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 101 op/s; 9540/72 objects degraded (13250.000%); 0 B/s, 96 objects/s recovering 2023-09-23T21:01:15.205 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:01:15.209 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:01:15.209 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:01:15.310 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:01:15.338 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:01:15.377 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:01:15.380 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:01:15.380 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:01:15.481 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:01:15 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T21:01:15.484 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:01:15 smithi026 ceph-mon[175151]: Health check update: Degraded data redundancy: 9540/72 objects degraded (13250.000%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T21:01:15.484 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:01:15 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T21:01:15.484 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:01:15 smithi026 ceph-mon[179688]: Health check update: Degraded data redundancy: 9540/72 objects degraded (13250.000%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T21:01:15.484 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:01:15.508 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:01:15 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T21:01:15.508 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:01:15 smithi070 ceph-mon[168282]: Health check update: Degraded data redundancy: 9540/72 objects degraded (13250.000%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T21:01:15.511 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:01:15.547 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:01:15.548 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:01:15.548 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:01:15.649 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:01:15.680 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:01:15.821 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T21:01:15.824 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:01:15.824 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:01:15.925 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:01:15.957 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:01:16.096 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T21:01:16.099 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:01:16.099 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:01:16.200 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:01:16.238 INFO:teuthology.orchestra.run.smithi026.stderr:pool 'unique_pool_1' removed 2023-09-23T21:01:16.244 INFO:teuthology.orchestra.run.smithi026.stderr:noscrub is unset 2023-09-23T21:01:16.248 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:01:16.305 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:01:16.307 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:01:16.307 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:01:16.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:01:16 smithi026 ceph-mon[179688]: pgmap v655: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 107 active+clean; 582 KiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 9 op/s; 8762/72 objects degraded (12169.444%); 0 B/s, 161 objects/s recovering 2023-09-23T21:01:16.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:01:16 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/2738899981' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-23T21:01:16.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:01:16 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-23T21:01:16.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:01:16 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/1594940434' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T21:01:16.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:01:16 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T21:01:16.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:01:16 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:01:16.383 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:01:16 smithi026 ceph-mon[175151]: pgmap v655: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 107 active+clean; 582 KiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 9 op/s; 8762/72 objects degraded (12169.444%); 0 B/s, 161 objects/s recovering 2023-09-23T21:01:16.383 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:01:16 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/2738899981' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-23T21:01:16.383 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:01:16 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-23T21:01:16.383 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:01:16 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/1594940434' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T21:01:16.383 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:01:16 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T21:01:16.383 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:01:16 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:01:16.409 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:01:16.435 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:01:16.572 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:01:16.581 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:01:16 smithi070 ceph-mon[168282]: pgmap v655: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 107 active+clean; 582 KiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 9 op/s; 8762/72 objects degraded (12169.444%); 0 B/s, 161 objects/s recovering 2023-09-23T21:01:16.582 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:01:16 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/2738899981' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-23T21:01:16.582 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:01:16 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-23T21:01:16.582 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:01:16 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/1594940434' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T21:01:16.582 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:01:16 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T21:01:16.582 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:01:16 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:01:16.583 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:01:16.583 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:01:16.684 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:01:16.712 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:01:16.794 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:01:16.797 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:01:16.798 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:01:16.898 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:01:16.926 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:01:16.957 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:01:16.961 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:01:16.961 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:01:17.062 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:01:17.090 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:01:17.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:01:17 smithi026 ceph-mon[175151]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:01:17.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:01:17 smithi026 ceph-mon[175151]: failed to encode map e253 with expected crc 2023-09-23T21:01:17.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:01:17 smithi026 ceph-mon[175151]: failed to encode map e253 with expected crc 2023-09-23T21:01:17.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:01:17 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true}]': finished 2023-09-23T21:01:17.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:01:17 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T21:01:17.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:01:17 smithi026 ceph-mon[175151]: osdmap e253: 8 total, 8 up, 8 in 2023-09-23T21:01:17.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:01:17 smithi026 ceph-mon[175151]: failed to encode map e253 with expected crc 2023-09-23T21:01:17.383 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:01:17 smithi026 ceph-mon[179688]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:01:17.383 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:01:17 smithi026 ceph-mon[179688]: failed to encode map e253 with expected crc 2023-09-23T21:01:17.383 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:01:17 smithi026 ceph-mon[179688]: failed to encode map e253 with expected crc 2023-09-23T21:01:17.383 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:01:17 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true}]': finished 2023-09-23T21:01:17.383 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:01:17 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T21:01:17.383 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:01:17 smithi026 ceph-mon[179688]: osdmap e253: 8 total, 8 up, 8 in 2023-09-23T21:01:17.383 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:01:17 smithi026 ceph-mon[179688]: failed to encode map e253 with expected crc 2023-09-23T21:01:17.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:01:17 smithi070 ceph-mon[168282]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:01:17.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:01:17 smithi070 ceph-mon[168282]: failed to encode map e253 with expected crc 2023-09-23T21:01:17.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:01:17 smithi070 ceph-mon[168282]: failed to encode map e253 with expected crc 2023-09-23T21:01:17.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:01:17 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true}]': finished 2023-09-23T21:01:17.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:01:17 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T21:01:17.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:01:17 smithi070 ceph-mon[168282]: osdmap e253: 8 total, 8 up, 8 in 2023-09-23T21:01:17.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:01:17 smithi070 ceph-mon[168282]: failed to encode map e253 with expected crc 2023-09-23T21:01:18.446 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:01:18.450 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:01:18.450 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:01:18.488 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:01:18 smithi026 ceph-mon[175151]: pgmap v657: 93 pgs: 93 active+clean; 582 KiB data, 2.8 GiB used, 712 GiB / 715 GiB avail 2023-09-23T21:01:18.489 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:01:18 smithi026 ceph-mon[175151]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 8762/72 objects degraded (12169.444%), 2 pgs degraded, 2 pgs undersized) 2023-09-23T21:01:18.489 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:01:18 smithi026 ceph-mon[179688]: pgmap v657: 93 pgs: 93 active+clean; 582 KiB data, 2.8 GiB used, 712 GiB / 715 GiB avail 2023-09-23T21:01:18.489 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:01:18 smithi026 ceph-mon[179688]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 8762/72 objects degraded (12169.444%), 2 pgs degraded, 2 pgs undersized) 2023-09-23T21:01:18.552 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:01:18.584 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:01:18.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:01:18 smithi070 ceph-mon[168282]: pgmap v657: 93 pgs: 93 active+clean; 582 KiB data, 2.8 GiB used, 712 GiB / 715 GiB avail 2023-09-23T21:01:18.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:01:18 smithi070 ceph-mon[168282]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 8762/72 objects degraded (12169.444%), 2 pgs degraded, 2 pgs undersized) 2023-09-23T21:01:19.298 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T21:01:19.301 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:01:19.302 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:01:19.403 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:01:19.432 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:01:19.465 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:01:19.468 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:01:19.468 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:01:19.532 INFO:teuthology.task.full_sequential:In full_sequential, running task radosbench... 2023-09-23T21:01:19.533 INFO:tasks.radosbench:Beginning radosbench... 2023-09-23T21:01:19.533 INFO:tasks.cephadm.ceph_manager.ceph:creating pool_name unique_pool_2 2023-09-23T21:01:19.533 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph osd pool create unique_pool_2 16 2023-09-23T21:01:19.569 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:01:19.597 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:01:19.679 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:01:19.682 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:01:19.682 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:01:19.783 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:01:19.809 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:01:19.820 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0: {} 2023-09-23T21:01:19.842 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:01:19.845 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:01:19.846 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:01:19.850 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1: {} 2023-09-23T21:01:19.869 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2: {} 2023-09-23T21:01:19.881 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 21:01:19 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[164029]: ::ffff:172.21.15.70 - - [23/Sep/2023:21:01:19] "GET /metrics HTTP/1.1" 200 36296 "" "Prometheus/2.43.0" 2023-09-23T21:01:19.888 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3: {} 2023-09-23T21:01:19.911 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4: {} 2023-09-23T21:01:19.933 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T21:01:19.946 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:01:19.957 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6: {} 2023-09-23T21:01:19.978 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T21:01:19.980 INFO:teuthology.orchestra.run.smithi026.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T21:01:19.980 INFO:teuthology.orchestra.run.smithi026.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T21:01:19.980 INFO:teuthology.orchestra.run.smithi026.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T21:01:19.980 INFO:teuthology.orchestra.run.smithi026.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T21:01:19.981 INFO:teuthology.orchestra.run.smithi026.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T21:01:19.981 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T21:01:19.981 INFO:teuthology.orchestra.run.smithi026.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T21:01:19.981 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T21:01:19.983 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:01:20.071 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T21:01:20.075 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:01:20.075 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:01:20.117 INFO:teuthology.orchestra.run.smithi026.stdout: 2023-09-23T21:01:20.119 INFO:teuthology.orchestra.run.smithi026.stderr:dumped all 2023-09-23T21:01:20.176 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:01:20.210 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:01:20.260 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:01:20.263 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:01:20 smithi026 ceph-mon[175151]: pgmap v658: 93 pgs: 93 active+clean; 582 KiB data, 2.8 GiB used, 712 GiB / 715 GiB avail 2023-09-23T21:01:20.264 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:01:20 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/434151883' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:01:20.264 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:01:20 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:01:20.264 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:01:20 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:01:20.264 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:01:20 smithi026 ceph-mon[179688]: pgmap v658: 93 pgs: 93 active+clean; 582 KiB data, 2.8 GiB used, 712 GiB / 715 GiB avail 2023-09-23T21:01:20.264 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:01:20 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/434151883' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:01:20.264 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:01:20 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:01:20.264 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:01:20 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:01:20.265 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:01:20.265 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:01:20.297 INFO:teuthology.orchestra.run.smithi026.stdout:{ 2023-09-23T21:01:20.298 INFO:teuthology.orchestra.run.smithi026.stdout: "ops": [], 2023-09-23T21:01:20.298 INFO:teuthology.orchestra.run.smithi026.stdout: "num_ops": 0 2023-09-23T21:01:20.298 INFO:teuthology.orchestra.run.smithi026.stdout:} 2023-09-23T21:01:20.366 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:01:20.398 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:01:20.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:01:20 smithi070 ceph-mon[168282]: pgmap v658: 93 pgs: 93 active+clean; 582 KiB data, 2.8 GiB used, 712 GiB / 715 GiB avail 2023-09-23T21:01:20.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:01:20 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/434151883' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:01:20.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:01:20 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:01:20.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:01:20 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:01:20.884 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:01:20.887 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:01:20.887 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:01:20.889 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_blocked_ops 2023-09-23T21:01:20.890 INFO:teuthology.orchestra.run.smithi026.stdout:{"pg_ready":true,"pg_map":{"version":658,"stamp":"2023-09-23T21:01:18.509066+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":595872,"num_objects":24,"num_object_clones":0,"num_object_copies":72,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":24,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":22,"num_bytes_recovered":5990,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3436,"ondisk_log_size":3436,"up":279,"acting":279,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":330,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":2899008,"kb_used_data":1031744,"kb_used_omap":136,"kb_used_meta":1867063,"kb_avail":747062208,"statfs":{"total":767960285184,"available":764991700992,"internally_reserved":0,"allocated":1056505856,"data_stored":1048072417,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":139741,"internal_metadata":1911873059},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":42,"apply_latency_ms":42,"commit_latency_ns":42000000,"apply_latency_ns":42000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":-10200,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":-1,"num_read_kb":-1,"num_write":-203520,"num_write_kb":-6512577,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":-6142,"num_bytes_recovered":-291635200,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.002613"},"pg_stats":[{"pgid":"3.1a","version":"68'1","reported_seq":130,"reported_epoch":252,"state":"active+clean","last_fresh":"2023-09-23T21:00:58.373600+0000","last_change":"2023-09-23T20:54:13.946695+0000","last_active":"2023-09-23T21:00:58.373600+0000","last_peered":"2023-09-23T21:00:58.373600+0000","last_clean":"2023-09-23T21:00:58.373600+0000","last_became_active":"2023-09-23T20:54:13.945399+0000","last_became_peered":"2023-09-23T20:54:13.945399+0000","last_unstale":"2023-09-23T21:00:58.373600+0000","last_undegraded":"2023-09-23T21:00:58.373600+0000","last_fullsized":"2023-09-23T21:00:58.373600+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:32:52.365464+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":245,"reported_epoch":252,"state":"active+clean","last_fresh":"2023-09-23T21:00:58.404664+0000","last_change":"2023-09-23T20:54:13.942019+0000","last_active":"2023-09-23T21:00:58.404664+0000","last_peered":"2023-09-23T21:00:58.404664+0000","last_clean":"2023-09-23T21:00:58.404664+0000","last_became_active":"2023-09-23T20:54:13.941647+0000","last_became_peered":"2023-09-23T20:54:13.941647+0000","last_unstale":"2023-09-23T21:00:58.404664+0000","last_undegraded":"2023-09-23T21:00:58.404664+0000","last_fullsized":"2023-09-23T21:00:58.404664+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:30:54.400944+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":222,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-09-23T21:00:56.192916+0000","last_change":"2023-09-23T20:54:13.941394+0000","last_active":"2023-09-23T21:00:56.192916+0000","last_peered":"2023-09-23T21:00:56.192916+0000","last_clean":"2023-09-23T21:00:56.192916+0000","last_became_active":"2023-09-23T20:54:13.941042+0000","last_became_peered":"2023-09-23T20:54:13.941042+0000","last_unstale":"2023-09-23T21:00:56.192916+0000","last_undegraded":"2023-09-23T21:00:56.192916+0000","last_fullsized":"2023-09-23T21:00:56.192916+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:19:41.401184+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":238,"reported_epoch":252,"state":"active+clean","last_fresh":"2023-09-23T21:00:58.410398+0000","last_change":"2023-09-23T20:57:44.979811+0000","last_active":"2023-09-23T21:00:58.410398+0000","last_peered":"2023-09-23T21:00:58.410398+0000","last_clean":"2023-09-23T21:00:58.410398+0000","last_became_active":"2023-09-23T20:57:44.979515+0000","last_became_peered":"2023-09-23T20:57:44.979515+0000","last_unstale":"2023-09-23T21:00:58.410398+0000","last_undegraded":"2023-09-23T21:00:58.410398+0000","last_fullsized":"2023-09-23T21:00:58.410398+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:38:22.162140+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":236,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-09-23T21:00:56.120807+0000","last_change":"2023-09-23T20:51:10.833613+0000","last_active":"2023-09-23T21:00:56.120807+0000","last_peered":"2023-09-23T21:00:56.120807+0000","last_clean":"2023-09-23T21:00:56.120807+0000","last_became_active":"2023-09-23T20:51:10.833208+0000","last_became_peered":"2023-09-23T20:51:10.833208+0000","last_unstale":"2023-09-23T21:00:56.120807+0000","last_undegraded":"2023-09-23T21:00:56.120807+0000","last_fullsized":"2023-09-23T21:00:56.120807+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:51:28.246014+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":217,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-09-23T21:00:56.115335+0000","last_change":"2023-09-23T20:55:35.958384+0000","last_active":"2023-09-23T21:00:56.115335+0000","last_peered":"2023-09-23T21:00:56.115335+0000","last_clean":"2023-09-23T21:00:56.115335+0000","last_became_active":"2023-09-23T20:55:35.954563+0000","last_became_peered":"2023-09-23T20:55:35.954563+0000","last_unstale":"2023-09-23T21:00:56.115335+0000","last_undegraded":"2023-09-23T21:00:56.115335+0000","last_fullsized":"2023-09-23T21:00:56.115335+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:37:48.332259+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":228,"reported_epoch":252,"state":"active+clean","last_fresh":"2023-09-23T21:00:58.409667+0000","last_change":"2023-09-23T20:54:31.885514+0000","last_active":"2023-09-23T21:00:58.409667+0000","last_peered":"2023-09-23T21:00:58.409667+0000","last_clean":"2023-09-23T21:00:58.409667+0000","last_became_active":"2023-09-23T20:54:31.884986+0000","last_became_peered":"2023-09-23T20:54:31.884986+0000","last_unstale":"2023-09-23T21:00:58.409667+0000","last_undegraded":"2023-09-23T21:00:58.409667+0000","last_fullsized":"2023-09-23T21:00:58.409667+0000","mapping_epoch":136,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:26:08.100220+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":233,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-09-23T21:00:56.120723+0000","last_change":"2023-09-23T20:55:35.951889+0000","last_active":"2023-09-23T21:00:56.120723+0000","last_peered":"2023-09-23T21:00:56.120723+0000","last_clean":"2023-09-23T21:00:56.120723+0000","last_became_active":"2023-09-23T20:55:35.951563+0000","last_became_peered":"2023-09-23T20:55:35.951563+0000","last_unstale":"2023-09-23T21:00:56.120723+0000","last_undegraded":"2023-09-23T21:00:56.120723+0000","last_fullsized":"2023-09-23T21:00:56.120723+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:41:53.098781+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"79'161","reported_seq":540,"reported_epoch":252,"state":"active+clean","last_fresh":"2023-09-23T21:00:58.412738+0000","last_change":"2023-09-23T20:57:25.096559+0000","last_active":"2023-09-23T21:00:58.412738+0000","last_peered":"2023-09-23T21:00:58.412738+0000","last_clean":"2023-09-23T21:00:58.412738+0000","last_became_active":"2023-09-23T20:57:15.975674+0000","last_became_peered":"2023-09-23T20:57:15.975674+0000","last_unstale":"2023-09-23T21:00:58.412738+0000","last_undegraded":"2023-09-23T21:00:58.412738+0000","last_fullsized":"2023-09-23T21:00:58.412738+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:25.096401+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:25.096401+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:05:05.448404+0000","scrub_duration":0.002859393,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":126,"reported_epoch":252,"state":"active+clean","last_fresh":"2023-09-23T21:00:58.423739+0000","last_change":"2023-09-23T20:54:31.890678+0000","last_active":"2023-09-23T21:00:58.423739+0000","last_peered":"2023-09-23T21:00:58.423739+0000","last_clean":"2023-09-23T21:00:58.423739+0000","last_became_active":"2023-09-23T20:54:31.890469+0000","last_became_peered":"2023-09-23T20:54:31.890469+0000","last_unstale":"2023-09-23T21:00:58.423739+0000","last_undegraded":"2023-09-23T21:00:58.423739+0000","last_fullsized":"2023-09-23T21:00:58.423739+0000","mapping_epoch":136,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:55:08.790146+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":194,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-09-23T21:00:56.120833+0000","last_change":"2023-09-23T20:55:35.952661+0000","last_active":"2023-09-23T21:00:56.120833+0000","last_peered":"2023-09-23T21:00:56.120833+0000","last_clean":"2023-09-23T21:00:56.120833+0000","last_became_active":"2023-09-23T20:55:35.952349+0000","last_became_peered":"2023-09-23T20:55:35.952349+0000","last_unstale":"2023-09-23T21:00:56.120833+0000","last_undegraded":"2023-09-23T21:00:56.120833+0000","last_fullsized":"2023-09-23T21:00:56.120833+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:00:03.110126+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":70,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-09-23T21:00:56.121012+0000","last_change":"2023-09-23T20:56:45.988607+0000","last_active":"2023-09-23T21:00:56.121012+0000","last_peered":"2023-09-23T21:00:56.121012+0000","last_clean":"2023-09-23T21:00:56.121012+0000","last_became_active":"2023-09-23T20:56:45.988206+0000","last_became_peered":"2023-09-23T20:56:45.988206+0000","last_unstale":"2023-09-23T21:00:56.121012+0000","last_undegraded":"2023-09-23T21:00:56.121012+0000","last_fullsized":"2023-09-23T21:00:56.121012+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":205,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:01:19.783688+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,4],"acting":[3,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":70,"reported_epoch":252,"state":"active+clean","last_fresh":"2023-09-23T21:00:58.423129+0000","last_change":"2023-09-23T20:56:18.139142+0000","last_active":"2023-09-23T21:00:58.423129+0000","last_peered":"2023-09-23T21:00:58.423129+0000","last_clean":"2023-09-23T21:00:58.423129+0000","last_became_active":"2023-09-23T20:56:18.138844+0000","last_became_peered":"2023-09-23T20:56:18.138844+0000","last_unstale":"2023-09-23T21:00:58.423129+0000","last_undegraded":"2023-09-23T21:00:58.423129+0000","last_fullsized":"2023-09-23T21:00:58.423129+0000","mapping_epoch":192,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":193,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:09:02.217683+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,2],"acting":[5,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.14","version":"79'22","reported_seq":157,"reported_epoch":252,"state":"active+clean","last_fresh":"2023-09-23T21:00:58.374724+0000","last_change":"2023-09-23T20:55:35.949418+0000","last_active":"2023-09-23T21:00:58.374724+0000","last_peered":"2023-09-23T21:00:58.374724+0000","last_clean":"2023-09-23T21:00:58.374724+0000","last_became_active":"2023-09-23T20:55:35.948566+0000","last_became_peered":"2023-09-23T20:55:35.948566+0000","last_unstale":"2023-09-23T21:00:58.374724+0000","last_undegraded":"2023-09-23T21:00:58.374724+0000","last_fullsized":"2023-09-23T21:00:58.374724+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:59:37.433128+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4734,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":147,"reported_epoch":252,"state":"active+clean","last_fresh":"2023-09-23T21:00:58.373792+0000","last_change":"2023-09-23T20:55:35.949178+0000","last_active":"2023-09-23T21:00:58.373792+0000","last_peered":"2023-09-23T21:00:58.373792+0000","last_clean":"2023-09-23T21:00:58.373792+0000","last_became_active":"2023-09-23T20:55:35.948795+0000","last_became_peered":"2023-09-23T20:55:35.948795+0000","last_unstale":"2023-09-23T21:00:58.373792+0000","last_undegraded":"2023-09-23T21:00:58.373792+0000","last_fullsized":"2023-09-23T21:00:58.373792+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:13:08.996932+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.11","version":"79'161","reported_seq":482,"reported_epoch":252,"state":"active+clean","last_fresh":"2023-09-23T21:00:58.407437+0000","last_change":"2023-09-23T20:57:28.756796+0000","last_active":"2023-09-23T21:00:58.407437+0000","last_peered":"2023-09-23T21:00:58.407437+0000","last_clean":"2023-09-23T21:00:58.407437+0000","last_became_active":"2023-09-23T20:57:15.980606+0000","last_became_peered":"2023-09-23T20:57:15.980606+0000","last_unstale":"2023-09-23T21:00:58.407437+0000","last_undegraded":"2023-09-23T21:00:58.407437+0000","last_fullsized":"2023-09-23T21:00:58.407437+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:28.756634+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:28.756634+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:09:19.150791+0000","scrub_duration":0.0064925119999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":173,"reported_epoch":252,"state":"active+clean","last_fresh":"2023-09-23T21:00:58.373674+0000","last_change":"2023-09-23T20:56:45.992577+0000","last_active":"2023-09-23T21:00:58.373674+0000","last_peered":"2023-09-23T21:00:58.373674+0000","last_clean":"2023-09-23T21:00:58.373674+0000","last_became_active":"2023-09-23T20:56:45.992297+0000","last_became_peered":"2023-09-23T20:56:45.992297+0000","last_unstale":"2023-09-23T21:00:58.373674+0000","last_undegraded":"2023-09-23T21:00:58.373674+0000","last_fullsized":"2023-09-23T21:00:58.373674+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":205,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:00:45.547202+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,2],"acting":[4,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":218,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-09-23T21:00:56.120682+0000","last_change":"2023-09-23T20:51:10.831262+0000","last_active":"2023-09-23T21:00:56.120682+0000","last_peered":"2023-09-23T21:00:56.120682+0000","last_clean":"2023-09-23T21:00:56.120682+0000","last_became_active":"2023-09-23T20:51:10.830931+0000","last_became_peered":"2023-09-23T20:51:10.830931+0000","last_unstale":"2023-09-23T21:00:56.120682+0000","last_undegraded":"2023-09-23T21:00:56.120682+0000","last_fullsized":"2023-09-23T21:00:56.120682+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:51:54.982505+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.10","version":"79'161","reported_seq":490,"reported_epoch":252,"state":"active+clean","last_fresh":"2023-09-23T21:00:58.422658+0000","last_change":"2023-09-23T20:57:20.708003+0000","last_active":"2023-09-23T21:00:58.422658+0000","last_peered":"2023-09-23T21:00:58.422658+0000","last_clean":"2023-09-23T21:00:58.422658+0000","last_became_active":"2023-09-23T20:57:15.975302+0000","last_became_peered":"2023-09-23T20:57:15.975302+0000","last_unstale":"2023-09-23T21:00:58.422658+0000","last_undegraded":"2023-09-23T21:00:58.422658+0000","last_fullsized":"2023-09-23T21:00:58.422658+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:20.707939+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:20.707939+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:16:51.721994+0000","scrub_duration":0.0017733409999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":229,"reported_epoch":252,"state":"active+clean","last_fresh":"2023-09-23T21:01:11.953331+0000","last_change":"2023-09-23T20:57:44.979736+0000","last_active":"2023-09-23T21:01:11.953331+0000","last_peered":"2023-09-23T21:01:11.953331+0000","last_clean":"2023-09-23T21:01:11.953331+0000","last_became_active":"2023-09-23T20:57:44.979356+0000","last_became_peered":"2023-09-23T20:57:44.979356+0000","last_unstale":"2023-09-23T21:01:11.953331+0000","last_undegraded":"2023-09-23T21:01:11.953331+0000","last_fullsized":"2023-09-23T21:01:11.953331+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:14:43.848832+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":126,"reported_epoch":252,"state":"active+clean","last_fresh":"2023-09-23T21:00:58.423124+0000","last_change":"2023-09-23T20:54:31.887814+0000","last_active":"2023-09-23T21:00:58.423124+0000","last_peered":"2023-09-23T21:00:58.423124+0000","last_clean":"2023-09-23T21:00:58.423124+0000","last_became_active":"2023-09-23T20:54:31.887195+0000","last_became_peered":"2023-09-23T20:54:31.887195+0000","last_unstale":"2023-09-23T21:00:58.423124+0000","last_undegraded":"2023-09-23T21:00:58.423124+0000","last_fullsized":"2023-09-23T21:00:58.423124+0000","mapping_epoch":136,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:10:52.089553+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.13","version":"79'161","reported_seq":552,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-09-23T21:00:56.194045+0000","last_change":"2023-09-23T20:57:48.262970+0000","last_active":"2023-09-23T21:00:56.194045+0000","last_peered":"2023-09-23T21:00:56.194045+0000","last_clean":"2023-09-23T21:00:56.194045+0000","last_became_active":"2023-09-23T20:57:48.262663+0000","last_became_peered":"2023-09-23T20:57:48.262663+0000","last_unstale":"2023-09-23T21:00:56.194045+0000","last_undegraded":"2023-09-23T21:00:56.194045+0000","last_fullsized":"2023-09-23T21:00:56.194045+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:36.420362+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:36.420362+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:28:28.613492+0000","scrub_duration":0.028838444000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":197,"reported_epoch":252,"state":"active+clean","last_fresh":"2023-09-23T21:00:58.373535+0000","last_change":"2023-09-23T20:56:45.991822+0000","last_active":"2023-09-23T21:00:58.373535+0000","last_peered":"2023-09-23T21:00:58.373535+0000","last_clean":"2023-09-23T21:00:58.373535+0000","last_became_active":"2023-09-23T20:56:45.991460+0000","last_became_peered":"2023-09-23T20:56:45.991460+0000","last_unstale":"2023-09-23T21:00:58.373535+0000","last_undegraded":"2023-09-23T21:00:58.373535+0000","last_fullsized":"2023-09-23T21:00:58.373535+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":205,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:20:37.503317+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":210,"reported_epoch":252,"state":"active+clean","last_fresh":"2023-09-23T21:00:58.409220+0000","last_change":"2023-09-23T20:51:10.840849+0000","last_active":"2023-09-23T21:00:58.409220+0000","last_peered":"2023-09-23T21:00:58.409220+0000","last_clean":"2023-09-23T21:00:58.409220+0000","last_became_active":"2023-09-23T20:51:10.840593+0000","last_became_peered":"2023-09-23T20:51:10.840593+0000","last_unstale":"2023-09-23T21:00:58.409220+0000","last_undegraded":"2023-09-23T21:00:58.409220+0000","last_fullsized":"2023-09-23T21:00:58.409220+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:30:52.297625+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.12","version":"79'161","reported_seq":490,"reported_epoch":252,"state":"active+clean","last_fresh":"2023-09-23T21:00:58.422873+0000","last_change":"2023-09-23T20:57:21.750765+0000","last_active":"2023-09-23T21:00:58.422873+0000","last_peered":"2023-09-23T21:00:58.422873+0000","last_clean":"2023-09-23T21:00:58.422873+0000","last_became_active":"2023-09-23T20:57:15.979195+0000","last_became_peered":"2023-09-23T20:57:15.979195+0000","last_unstale":"2023-09-23T21:00:58.422873+0000","last_undegraded":"2023-09-23T21:00:58.422873+0000","last_fullsized":"2023-09-23T21:00:58.422873+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:21.750680+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:21.750680+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:03:41.915962+0000","scrub_duration":0.0092391680000000007,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":244,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-09-23T21:00:56.111932+0000","last_change":"2023-09-23T20:55:36.943969+0000","last_active":"2023-09-23T21:00:56.111932+0000","last_peered":"2023-09-23T21:00:56.111932+0000","last_clean":"2023-09-23T21:00:56.111932+0000","last_became_active":"2023-09-23T20:55:36.943580+0000","last_became_peered":"2023-09-23T20:55:36.943580+0000","last_unstale":"2023-09-23T21:00:56.111932+0000","last_undegraded":"2023-09-23T21:00:56.111932+0000","last_fullsized":"2023-09-23T21:00:56.111932+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:49:06.969396+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":41,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-09-23T21:00:56.121463+0000","last_change":"2023-09-23T20:57:44.980498+0000","last_active":"2023-09-23T21:00:56.121463+0000","last_peered":"2023-09-23T21:00:56.121463+0000","last_clean":"2023-09-23T21:00:56.121463+0000","last_became_active":"2023-09-23T20:57:44.980182+0000","last_became_peered":"2023-09-23T20:57:44.980182+0000","last_unstale":"2023-09-23T21:00:56.121463+0000","last_undegraded":"2023-09-23T21:00:56.121463+0000","last_fullsized":"2023-09-23T21:00:56.121463+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:34:51.620544+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.d","version":"79'161","reported_seq":535,"reported_epoch":252,"state":"active+clean","last_fresh":"2023-09-23T21:00:58.374303+0000","last_change":"2023-09-23T20:57:31.075213+0000","last_active":"2023-09-23T21:00:58.374303+0000","last_peered":"2023-09-23T21:00:58.374303+0000","last_clean":"2023-09-23T21:00:58.374303+0000","last_became_active":"2023-09-23T20:57:15.988188+0000","last_became_peered":"2023-09-23T20:57:15.988188+0000","last_unstale":"2023-09-23T21:00:58.374303+0000","last_undegraded":"2023-09-23T21:00:58.374303+0000","last_fullsized":"2023-09-23T21:00:58.374303+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:31.075098+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:31.075098+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:41:52.142362+0000","scrub_duration":0.010304610000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":184,"reported_epoch":252,"state":"active+clean","last_fresh":"2023-09-23T21:00:58.374051+0000","last_change":"2023-09-23T20:56:45.991841+0000","last_active":"2023-09-23T21:00:58.374051+0000","last_peered":"2023-09-23T21:00:58.374051+0000","last_clean":"2023-09-23T21:00:58.374051+0000","last_became_active":"2023-09-23T20:56:45.991491+0000","last_became_peered":"2023-09-23T20:56:45.991491+0000","last_unstale":"2023-09-23T21:00:58.374051+0000","last_undegraded":"2023-09-23T21:00:58.374051+0000","last_fullsized":"2023-09-23T21:00:58.374051+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":205,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:30:37.405319+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":143,"reported_epoch":252,"state":"active+clean","last_fresh":"2023-09-23T21:00:58.422746+0000","last_change":"2023-09-23T20:55:35.957768+0000","last_active":"2023-09-23T21:00:58.422746+0000","last_peered":"2023-09-23T21:00:58.422746+0000","last_clean":"2023-09-23T21:00:58.422746+0000","last_became_active":"2023-09-23T20:55:35.957421+0000","last_became_peered":"2023-09-23T20:55:35.957421+0000","last_unstale":"2023-09-23T21:00:58.422746+0000","last_undegraded":"2023-09-23T21:00:58.422746+0000","last_fullsized":"2023-09-23T21:00:58.422746+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:49:41.585961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.c","version":"79'161","reported_seq":553,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-09-23T21:00:56.120484+0000","last_change":"2023-09-23T20:57:22.041071+0000","last_active":"2023-09-23T21:00:56.120484+0000","last_peered":"2023-09-23T21:00:56.120484+0000","last_clean":"2023-09-23T21:00:56.120484+0000","last_became_active":"2023-09-23T20:57:15.973204+0000","last_became_peered":"2023-09-23T20:57:15.973204+0000","last_unstale":"2023-09-23T21:00:56.120484+0000","last_undegraded":"2023-09-23T21:00:56.120484+0000","last_fullsized":"2023-09-23T21:00:56.120484+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:22.040907+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:22.040907+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:47:02.019978+0000","scrub_duration":0.013021176000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":171,"reported_epoch":252,"state":"active+clean","last_fresh":"2023-09-23T21:00:58.373839+0000","last_change":"2023-09-23T20:56:45.991821+0000","last_active":"2023-09-23T21:00:58.373839+0000","last_peered":"2023-09-23T21:00:58.373839+0000","last_clean":"2023-09-23T21:00:58.373839+0000","last_became_active":"2023-09-23T20:56:45.991447+0000","last_became_peered":"2023-09-23T20:56:45.991447+0000","last_unstale":"2023-09-23T21:00:58.373839+0000","last_undegraded":"2023-09-23T21:00:58.373839+0000","last_fullsized":"2023-09-23T21:00:58.373839+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":205,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:14:25.525222+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":174,"reported_epoch":252,"state":"active+clean","last_fresh":"2023-09-23T21:00:58.373803+0000","last_change":"2023-09-23T20:57:44.982059+0000","last_active":"2023-09-23T21:00:58.373803+0000","last_peered":"2023-09-23T21:00:58.373803+0000","last_clean":"2023-09-23T21:00:58.373803+0000","last_became_active":"2023-09-23T20:57:44.981742+0000","last_became_peered":"2023-09-23T20:57:44.981742+0000","last_unstale":"2023-09-23T21:00:58.373803+0000","last_undegraded":"2023-09-23T21:00:58.373803+0000","last_fullsized":"2023-09-23T21:00:58.373803+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:41:24.042916+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.f","version":"79'161","reported_seq":556,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-09-23T21:00:56.112843+0000","last_change":"2023-09-23T20:57:27.880841+0000","last_active":"2023-09-23T21:00:56.112843+0000","last_peered":"2023-09-23T21:00:56.112843+0000","last_clean":"2023-09-23T21:00:56.112843+0000","last_became_active":"2023-09-23T20:57:15.970925+0000","last_became_peered":"2023-09-23T20:57:15.970925+0000","last_unstale":"2023-09-23T21:00:56.112843+0000","last_undegraded":"2023-09-23T21:00:56.112843+0000","last_fullsized":"2023-09-23T21:00:56.112843+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:27.880776+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:27.880776+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:15:13.438014+0000","scrub_duration":0.0014004899999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":157,"reported_epoch":252,"state":"active+clean","last_fresh":"2023-09-23T21:00:58.423052+0000","last_change":"2023-09-23T20:56:45.989153+0000","last_active":"2023-09-23T21:00:58.423052+0000","last_peered":"2023-09-23T21:00:58.423052+0000","last_clean":"2023-09-23T21:00:58.423052+0000","last_became_active":"2023-09-23T20:56:45.988800+0000","last_became_peered":"2023-09-23T20:56:45.988800+0000","last_unstale":"2023-09-23T21:00:58.423052+0000","last_undegraded":"2023-09-23T21:00:58.423052+0000","last_fullsized":"2023-09-23T21:00:58.423052+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":205,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:04:41.130194+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":191,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-09-23T21:00:56.112013+0000","last_change":"2023-09-23T20:55:35.965427+0000","last_active":"2023-09-23T21:00:56.112013+0000","last_peered":"2023-09-23T21:00:56.112013+0000","last_clean":"2023-09-23T21:00:56.112013+0000","last_became_active":"2023-09-23T20:55:35.963969+0000","last_became_peered":"2023-09-23T20:55:35.963969+0000","last_unstale":"2023-09-23T21:00:56.112013+0000","last_undegraded":"2023-09-23T21:00:56.112013+0000","last_fullsized":"2023-09-23T21:00:56.112013+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:38:40.612083+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.e","version":"79'161","reported_seq":464,"reported_epoch":252,"state":"active+clean","last_fresh":"2023-09-23T21:00:58.373686+0000","last_change":"2023-09-23T20:57:34.132738+0000","last_active":"2023-09-23T21:00:58.373686+0000","last_peered":"2023-09-23T21:00:58.373686+0000","last_clean":"2023-09-23T21:00:58.373686+0000","last_became_active":"2023-09-23T20:57:15.974318+0000","last_became_peered":"2023-09-23T20:57:15.974318+0000","last_unstale":"2023-09-23T21:00:58.373686+0000","last_undegraded":"2023-09-23T21:00:58.373686+0000","last_fullsized":"2023-09-23T21:00:58.373686+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:34.132660+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:34.132660+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:07:07.739438+0000","scrub_duration":0.005229694,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":143,"reported_epoch":252,"state":"active+clean","last_fresh":"2023-09-23T21:00:58.422959+0000","last_change":"2023-09-23T20:55:35.962005+0000","last_active":"2023-09-23T21:00:58.422959+0000","last_peered":"2023-09-23T21:00:58.422959+0000","last_clean":"2023-09-23T21:00:58.422959+0000","last_became_active":"2023-09-23T20:55:35.961695+0000","last_became_peered":"2023-09-23T20:55:35.961695+0000","last_unstale":"2023-09-23T21:00:58.422959+0000","last_undegraded":"2023-09-23T21:00:58.422959+0000","last_fullsized":"2023-09-23T21:00:58.422959+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:28:22.876201+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":242,"reported_epoch":252,"state":"active+clean","last_fresh":"2023-09-23T21:00:58.408123+0000","last_change":"2023-09-23T20:54:13.928715+0000","last_active":"2023-09-23T21:00:58.408123+0000","last_peered":"2023-09-23T21:00:58.408123+0000","last_clean":"2023-09-23T21:00:58.408123+0000","last_became_active":"2023-09-23T20:54:13.928200+0000","last_became_peered":"2023-09-23T20:54:13.928200+0000","last_unstale":"2023-09-23T21:00:58.408123+0000","last_undegraded":"2023-09-23T21:00:58.408123+0000","last_fullsized":"2023-09-23T21:00:58.408123+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:41:16.927513+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.9","version":"79'161","reported_seq":626,"reported_epoch":252,"state":"active+clean","last_fresh":"2023-09-23T21:00:58.405758+0000","last_change":"2023-09-23T20:56:46.921939+0000","last_active":"2023-09-23T21:00:58.405758+0000","last_peered":"2023-09-23T21:00:58.405758+0000","last_clean":"2023-09-23T21:00:58.405758+0000","last_became_active":"2023-09-23T20:56:45.989908+0000","last_became_peered":"2023-09-23T20:56:45.989908+0000","last_unstale":"2023-09-23T21:00:58.405758+0000","last_undegraded":"2023-09-23T21:00:58.405758+0000","last_fullsized":"2023-09-23T21:00:58.405758+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":205,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:56:46.921739+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:56:46.921739+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:20:42.635110+0000","scrub_duration":0.0071954109999999996,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":242,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-09-23T21:00:56.120631+0000","last_change":"2023-09-23T20:54:13.961788+0000","last_active":"2023-09-23T21:00:56.120631+0000","last_peered":"2023-09-23T21:00:56.120631+0000","last_clean":"2023-09-23T21:00:56.120631+0000","last_became_active":"2023-09-23T20:54:13.940937+0000","last_became_peered":"2023-09-23T20:54:13.940937+0000","last_unstale":"2023-09-23T21:00:56.120631+0000","last_undegraded":"2023-09-23T21:00:56.120631+0000","last_fullsized":"2023-09-23T21:00:56.120631+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:11:56.785299+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":147,"reported_epoch":252,"state":"active+clean","last_fresh":"2023-09-23T21:00:58.374882+0000","last_change":"2023-09-23T20:55:35.949670+0000","last_active":"2023-09-23T21:00:58.374882+0000","last_peered":"2023-09-23T21:00:58.374882+0000","last_clean":"2023-09-23T21:00:58.374882+0000","last_became_active":"2023-09-23T20:55:35.949140+0000","last_became_peered":"2023-09-23T20:55:35.949140+0000","last_unstale":"2023-09-23T21:00:58.374882+0000","last_undegraded":"2023-09-23T21:00:58.374882+0000","last_fullsized":"2023-09-23T21:00:58.374882+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:55:59.340303+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.8","version":"79'161","reported_seq":496,"reported_epoch":252,"state":"active+clean","last_fresh":"2023-09-23T21:00:58.422762+0000","last_change":"2023-09-23T20:56:57.505737+0000","last_active":"2023-09-23T21:00:58.422762+0000","last_peered":"2023-09-23T21:00:58.422762+0000","last_clean":"2023-09-23T21:00:58.422762+0000","last_became_active":"2023-09-23T20:56:04.982345+0000","last_became_peered":"2023-09-23T20:56:04.982345+0000","last_unstale":"2023-09-23T21:00:58.422762+0000","last_undegraded":"2023-09-23T21:00:58.422762+0000","last_fullsized":"2023-09-23T21:00:58.422762+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:56:57.505663+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:56:57.505663+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:34:30.109724+0000","scrub_duration":0.026284108,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":214,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-09-23T21:00:56.112205+0000","last_change":"2023-09-23T20:55:35.959942+0000","last_active":"2023-09-23T21:00:56.112205+0000","last_peered":"2023-09-23T21:00:56.112205+0000","last_clean":"2023-09-23T21:00:56.112205+0000","last_became_active":"2023-09-23T20:55:35.957071+0000","last_became_peered":"2023-09-23T20:55:35.957071+0000","last_unstale":"2023-09-23T21:00:56.112205+0000","last_undegraded":"2023-09-23T21:00:56.112205+0000","last_fullsized":"2023-09-23T21:00:56.112205+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:44:30.308036+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":130,"reported_epoch":252,"state":"active+clean","last_fresh":"2023-09-23T21:00:58.374594+0000","last_change":"2023-09-23T20:54:13.951459+0000","last_active":"2023-09-23T21:00:58.374594+0000","last_peered":"2023-09-23T21:00:58.374594+0000","last_clean":"2023-09-23T21:00:58.374594+0000","last_became_active":"2023-09-23T20:54:13.948409+0000","last_became_peered":"2023-09-23T20:54:13.948409+0000","last_unstale":"2023-09-23T21:00:58.374594+0000","last_undegraded":"2023-09-23T21:00:58.374594+0000","last_fullsized":"2023-09-23T21:00:58.374594+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:38:10.148061+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.b","version":"79'161","reported_seq":535,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-09-23T21:00:56.194088+0000","last_change":"2023-09-23T20:57:27.320172+0000","last_active":"2023-09-23T21:00:56.194088+0000","last_peered":"2023-09-23T21:00:56.194088+0000","last_clean":"2023-09-23T21:00:56.194088+0000","last_became_active":"2023-09-23T20:57:16.009630+0000","last_became_peered":"2023-09-23T20:57:16.009630+0000","last_unstale":"2023-09-23T21:00:56.194088+0000","last_undegraded":"2023-09-23T21:00:56.194088+0000","last_fullsized":"2023-09-23T21:00:56.194088+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:27.320022+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:27.320022+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:31:35.221558+0000","scrub_duration":0.0043894679999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,5],"acting":[1,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":130,"reported_epoch":252,"state":"active+clean","last_fresh":"2023-09-23T21:00:58.373611+0000","last_change":"2023-09-23T20:54:13.950254+0000","last_active":"2023-09-23T21:00:58.373611+0000","last_peered":"2023-09-23T21:00:58.373611+0000","last_clean":"2023-09-23T21:00:58.373611+0000","last_became_active":"2023-09-23T20:54:13.949934+0000","last_became_peered":"2023-09-23T20:54:13.949934+0000","last_unstale":"2023-09-23T21:00:58.373611+0000","last_undegraded":"2023-09-23T21:00:58.373611+0000","last_fullsized":"2023-09-23T21:00:58.373611+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:30:32.847167+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":147,"reported_epoch":252,"state":"active+clean","last_fresh":"2023-09-23T21:00:58.373918+0000","last_change":"2023-09-23T20:55:35.956153+0000","last_active":"2023-09-23T21:00:58.373918+0000","last_peered":"2023-09-23T21:00:58.373918+0000","last_clean":"2023-09-23T21:00:58.373918+0000","last_became_active":"2023-09-23T20:55:35.955630+0000","last_became_peered":"2023-09-23T20:55:35.955630+0000","last_unstale":"2023-09-23T21:00:58.373918+0000","last_undegraded":"2023-09-23T21:00:58.373918+0000","last_fullsized":"2023-09-23T21:00:58.373918+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:45:20.282929+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.a","version":"79'161","reported_seq":496,"reported_epoch":252,"state":"active+clean","last_fresh":"2023-09-23T21:00:58.423207+0000","last_change":"2023-09-23T20:56:44.487778+0000","last_active":"2023-09-23T21:00:58.423207+0000","last_peered":"2023-09-23T21:00:58.423207+0000","last_clean":"2023-09-23T21:00:58.423207+0000","last_became_active":"2023-09-23T20:56:05.006740+0000","last_became_peered":"2023-09-23T20:56:05.006740+0000","last_unstale":"2023-09-23T21:00:58.423207+0000","last_undegraded":"2023-09-23T21:00:58.423207+0000","last_fullsized":"2023-09-23T21:00:58.423207+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:56:44.487678+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:56:44.487678+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:04:50.086749+0000","scrub_duration":0.0017408899999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":41,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-09-23T21:00:56.120176+0000","last_change":"2023-09-23T20:57:44.982513+0000","last_active":"2023-09-23T21:00:56.120176+0000","last_peered":"2023-09-23T21:00:56.120176+0000","last_clean":"2023-09-23T21:00:56.120176+0000","last_became_active":"2023-09-23T20:57:44.982112+0000","last_became_peered":"2023-09-23T20:57:44.982112+0000","last_unstale":"2023-09-23T21:00:56.120176+0000","last_undegraded":"2023-09-23T21:00:56.120176+0000","last_fullsized":"2023-09-23T21:00:56.120176+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:09:05.630041+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":222,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-09-23T21:00:56.193923+0000","last_change":"2023-09-23T20:54:13.940748+0000","last_active":"2023-09-23T21:00:56.193923+0000","last_peered":"2023-09-23T21:00:56.193923+0000","last_clean":"2023-09-23T21:00:56.193923+0000","last_became_active":"2023-09-23T20:54:13.940419+0000","last_became_peered":"2023-09-23T20:54:13.940419+0000","last_unstale":"2023-09-23T21:00:56.193923+0000","last_undegraded":"2023-09-23T21:00:56.193923+0000","last_fullsized":"2023-09-23T21:00:56.193923+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:15:01.923816+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.7","version":"79'161","reported_seq":557,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-09-23T21:00:56.115358+0000","last_change":"2023-09-23T20:57:48.243572+0000","last_active":"2023-09-23T21:00:56.115358+0000","last_peered":"2023-09-23T21:00:56.115358+0000","last_clean":"2023-09-23T21:00:56.115358+0000","last_became_active":"2023-09-23T20:57:48.243229+0000","last_became_peered":"2023-09-23T20:57:48.243229+0000","last_unstale":"2023-09-23T21:00:56.115358+0000","last_undegraded":"2023-09-23T21:00:56.115358+0000","last_fullsized":"2023-09-23T21:00:56.115358+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":3,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:25.916867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:25.916867+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:23:19.385831+0000","scrub_duration":0.001665323,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":228,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-09-23T21:00:56.193317+0000","last_change":"2023-09-23T20:50:05.867130+0000","last_active":"2023-09-23T21:00:56.193317+0000","last_peered":"2023-09-23T21:00:56.193317+0000","last_clean":"2023-09-23T21:00:56.193317+0000","last_became_active":"2023-09-23T20:50:05.866758+0000","last_became_peered":"2023-09-23T20:50:05.866758+0000","last_unstale":"2023-09-23T21:00:56.193317+0000","last_undegraded":"2023-09-23T21:00:56.193317+0000","last_fullsized":"2023-09-23T21:00:56.193317+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:55:07.548209+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":126,"reported_epoch":252,"state":"active+clean","last_fresh":"2023-09-23T21:00:58.423198+0000","last_change":"2023-09-23T20:54:31.887604+0000","last_active":"2023-09-23T21:00:58.423198+0000","last_peered":"2023-09-23T21:00:58.423198+0000","last_clean":"2023-09-23T21:00:58.423198+0000","last_became_active":"2023-09-23T20:54:31.886752+0000","last_became_peered":"2023-09-23T20:54:31.886752+0000","last_unstale":"2023-09-23T21:00:58.423198+0000","last_undegraded":"2023-09-23T21:00:58.423198+0000","last_fullsized":"2023-09-23T21:00:58.423198+0000","mapping_epoch":136,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:13:58.872176+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":226,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-09-23T21:00:56.193686+0000","last_change":"2023-09-23T20:54:31.899082+0000","last_active":"2023-09-23T21:00:56.193686+0000","last_peered":"2023-09-23T21:00:56.193686+0000","last_clean":"2023-09-23T21:00:56.193686+0000","last_became_active":"2023-09-23T20:54:31.898521+0000","last_became_peered":"2023-09-23T20:54:31.898521+0000","last_unstale":"2023-09-23T21:00:56.193686+0000","last_undegraded":"2023-09-23T21:00:56.193686+0000","last_fullsized":"2023-09-23T21:00:56.193686+0000","mapping_epoch":136,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:25:36.059711+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"79'161","reported_seq":516,"reported_epoch":252,"state":"active+clean","last_fresh":"2023-09-23T21:00:58.422701+0000","last_change":"2023-09-23T20:57:19.669346+0000","last_active":"2023-09-23T21:00:58.422701+0000","last_peered":"2023-09-23T21:00:58.422701+0000","last_clean":"2023-09-23T21:00:58.422701+0000","last_became_active":"2023-09-23T20:57:08.246046+0000","last_became_peered":"2023-09-23T20:57:08.246046+0000","last_unstale":"2023-09-23T21:00:58.422701+0000","last_undegraded":"2023-09-23T21:00:58.422701+0000","last_fullsized":"2023-09-23T21:00:58.422701+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:19.669259+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:19.669259+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:14:03.435531+0000","scrub_duration":0.0045759559999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":143,"reported_epoch":252,"state":"active+clean","last_fresh":"2023-09-23T21:00:58.423011+0000","last_change":"2023-09-23T20:55:35.959514+0000","last_active":"2023-09-23T21:00:58.423011+0000","last_peered":"2023-09-23T21:00:58.423011+0000","last_clean":"2023-09-23T21:00:58.423011+0000","last_became_active":"2023-09-23T20:55:35.959244+0000","last_became_peered":"2023-09-23T20:55:35.959244+0000","last_unstale":"2023-09-23T21:00:58.423011+0000","last_undegraded":"2023-09-23T21:00:58.423011+0000","last_fullsized":"2023-09-23T21:00:58.423011+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:47:32.665316+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":147,"reported_epoch":252,"state":"active+clean","last_fresh":"2023-09-23T21:00:58.373932+0000","last_change":"2023-09-23T20:55:35.949000+0000","last_active":"2023-09-23T21:00:58.373932+0000","last_peered":"2023-09-23T21:00:58.373932+0000","last_clean":"2023-09-23T21:00:58.373932+0000","last_became_active":"2023-09-23T20:55:35.948410+0000","last_became_peered":"2023-09-23T20:55:35.948410+0000","last_unstale":"2023-09-23T21:00:58.373932+0000","last_undegraded":"2023-09-23T21:00:58.373932+0000","last_fullsized":"2023-09-23T21:00:58.373932+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:26:50.477289+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":220,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-09-23T21:00:56.193116+0000","last_change":"2023-09-23T20:51:10.829889+0000","last_active":"2023-09-23T21:00:56.193116+0000","last_peered":"2023-09-23T21:00:56.193116+0000","last_clean":"2023-09-23T21:00:56.193116+0000","last_became_active":"2023-09-23T20:51:10.829533+0000","last_became_peered":"2023-09-23T20:51:10.829533+0000","last_unstale":"2023-09-23T21:00:56.193116+0000","last_undegraded":"2023-09-23T21:00:56.193116+0000","last_fullsized":"2023-09-23T21:00:56.193116+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:43:04.603460+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"79'161","reported_seq":484,"reported_epoch":252,"state":"active+clean","last_fresh":"2023-09-23T21:00:58.407939+0000","last_change":"2023-09-23T20:57:24.802586+0000","last_active":"2023-09-23T21:00:58.407939+0000","last_peered":"2023-09-23T21:00:58.407939+0000","last_clean":"2023-09-23T21:00:58.407939+0000","last_became_active":"2023-09-23T20:57:08.236114+0000","last_became_peered":"2023-09-23T20:57:08.236114+0000","last_unstale":"2023-09-23T21:00:58.407939+0000","last_undegraded":"2023-09-23T21:00:58.407939+0000","last_fullsized":"2023-09-23T21:00:58.407939+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:24.802451+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:24.802451+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:06:22.309980+0000","scrub_duration":0.0090861250000000005,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,4],"acting":[0,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":240,"reported_epoch":252,"state":"active+clean","last_fresh":"2023-09-23T21:01:01.571080+0000","last_change":"2023-09-23T20:51:10.838054+0000","last_active":"2023-09-23T21:01:01.571080+0000","last_peered":"2023-09-23T21:01:01.571080+0000","last_clean":"2023-09-23T21:01:01.571080+0000","last_became_active":"2023-09-23T20:51:10.837750+0000","last_became_peered":"2023-09-23T20:51:10.837750+0000","last_unstale":"2023-09-23T21:01:01.571080+0000","last_undegraded":"2023-09-23T21:01:01.571080+0000","last_fullsized":"2023-09-23T21:01:01.571080+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:15:51.648085+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":220,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-09-23T21:00:56.192905+0000","last_change":"2023-09-23T20:55:35.951146+0000","last_active":"2023-09-23T21:00:56.192905+0000","last_peered":"2023-09-23T21:00:56.192905+0000","last_clean":"2023-09-23T21:00:56.192905+0000","last_became_active":"2023-09-23T20:55:35.950874+0000","last_became_peered":"2023-09-23T20:55:35.950874+0000","last_unstale":"2023-09-23T21:00:56.192905+0000","last_undegraded":"2023-09-23T21:00:56.192905+0000","last_fullsized":"2023-09-23T21:00:56.192905+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:58:13.957677+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":241,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-09-23T21:00:56.193883+0000","last_change":"2023-09-23T20:54:31.898870+0000","last_active":"2023-09-23T21:00:56.193883+0000","last_peered":"2023-09-23T21:00:56.193883+0000","last_clean":"2023-09-23T21:00:56.193883+0000","last_became_active":"2023-09-23T20:54:31.898467+0000","last_became_peered":"2023-09-23T20:54:31.898467+0000","last_unstale":"2023-09-23T21:00:56.193883+0000","last_undegraded":"2023-09-23T21:00:56.193883+0000","last_fullsized":"2023-09-23T21:00:56.193883+0000","mapping_epoch":136,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:56:22.483408+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":126,"reported_epoch":252,"state":"active+clean","last_fresh":"2023-09-23T21:00:58.425349+0000","last_change":"2023-09-23T20:54:31.883504+0000","last_active":"2023-09-23T21:00:58.425349+0000","last_peered":"2023-09-23T21:00:58.425349+0000","last_clean":"2023-09-23T21:00:58.425349+0000","last_became_active":"2023-09-23T20:54:31.883172+0000","last_became_peered":"2023-09-23T20:54:31.883172+0000","last_unstale":"2023-09-23T21:00:58.425349+0000","last_undegraded":"2023-09-23T21:00:58.425349+0000","last_fullsized":"2023-09-23T21:00:58.425349+0000","mapping_epoch":136,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:45:40.289120+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"79'161","reported_seq":484,"reported_epoch":252,"state":"active+clean","last_fresh":"2023-09-23T21:00:58.407607+0000","last_change":"2023-09-23T20:57:20.738417+0000","last_active":"2023-09-23T21:00:58.407607+0000","last_peered":"2023-09-23T21:00:58.407607+0000","last_clean":"2023-09-23T21:00:58.407607+0000","last_became_active":"2023-09-23T20:57:08.236842+0000","last_became_peered":"2023-09-23T20:57:08.236842+0000","last_unstale":"2023-09-23T21:00:58.407607+0000","last_undegraded":"2023-09-23T21:00:58.407607+0000","last_fullsized":"2023-09-23T21:00:58.407607+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":0,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:20.738273+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:20.738273+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:18:28.327990+0000","scrub_duration":0.0088336339999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":230,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-09-23T21:00:56.120772+0000","last_change":"2023-09-23T20:55:35.950777+0000","last_active":"2023-09-23T21:00:56.120772+0000","last_peered":"2023-09-23T21:00:56.120772+0000","last_clean":"2023-09-23T21:00:56.120772+0000","last_became_active":"2023-09-23T20:55:35.950427+0000","last_became_peered":"2023-09-23T20:55:35.950427+0000","last_unstale":"2023-09-23T21:00:56.120772+0000","last_undegraded":"2023-09-23T21:00:56.120772+0000","last_fullsized":"2023-09-23T21:00:56.120772+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:20:10.371751+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":196,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-09-23T21:00:56.113095+0000","last_change":"2023-09-23T20:55:35.978140+0000","last_active":"2023-09-23T21:00:56.113095+0000","last_peered":"2023-09-23T21:00:56.113095+0000","last_clean":"2023-09-23T21:00:56.113095+0000","last_became_active":"2023-09-23T20:55:35.977831+0000","last_became_peered":"2023-09-23T20:55:35.977831+0000","last_unstale":"2023-09-23T21:00:56.113095+0000","last_undegraded":"2023-09-23T21:00:56.113095+0000","last_fullsized":"2023-09-23T21:00:56.113095+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:25:47.678023+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.3","version":"79'161","reported_seq":519,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-09-23T21:00:56.194175+0000","last_change":"2023-09-23T20:57:20.378531+0000","last_active":"2023-09-23T21:00:56.194175+0000","last_peered":"2023-09-23T21:00:56.194175+0000","last_clean":"2023-09-23T21:00:56.194175+0000","last_became_active":"2023-09-23T20:57:08.238019+0000","last_became_peered":"2023-09-23T20:57:08.238019+0000","last_unstale":"2023-09-23T21:00:56.194175+0000","last_undegraded":"2023-09-23T21:00:56.194175+0000","last_fullsized":"2023-09-23T21:00:56.194175+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":3,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:20.377714+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:20.377714+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:00:08.305602+0000","scrub_duration":0.003015228,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":122,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-09-23T21:00:56.193045+0000","last_change":"2023-09-23T20:56:45.991884+0000","last_active":"2023-09-23T21:00:56.193045+0000","last_peered":"2023-09-23T21:00:56.193045+0000","last_clean":"2023-09-23T21:00:56.193045+0000","last_became_active":"2023-09-23T20:56:45.991568+0000","last_became_peered":"2023-09-23T20:56:45.991568+0000","last_unstale":"2023-09-23T21:00:56.193045+0000","last_undegraded":"2023-09-23T21:00:56.193045+0000","last_fullsized":"2023-09-23T21:00:56.193045+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":205,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:21:20.775511+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,0],"acting":[1,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":247,"reported_epoch":252,"state":"active+clean","last_fresh":"2023-09-23T21:00:58.408652+0000","last_change":"2023-09-23T20:57:44.979950+0000","last_active":"2023-09-23T21:00:58.408652+0000","last_peered":"2023-09-23T21:00:58.408652+0000","last_clean":"2023-09-23T21:00:58.408652+0000","last_became_active":"2023-09-23T20:57:44.979346+0000","last_became_peered":"2023-09-23T20:57:44.979346+0000","last_unstale":"2023-09-23T21:00:58.408652+0000","last_undegraded":"2023-09-23T21:00:58.408652+0000","last_fullsized":"2023-09-23T21:00:58.408652+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:10:24.372800+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":226,"reported_epoch":252,"state":"active+clean","last_fresh":"2023-09-23T21:00:58.408150+0000","last_change":"2023-09-23T20:50:37.982235+0000","last_active":"2023-09-23T21:00:58.408150+0000","last_peered":"2023-09-23T21:00:58.408150+0000","last_clean":"2023-09-23T21:00:58.408150+0000","last_became_active":"2023-09-23T20:50:37.981913+0000","last_became_peered":"2023-09-23T20:50:37.981913+0000","last_unstale":"2023-09-23T21:00:58.408150+0000","last_undegraded":"2023-09-23T21:00:58.408150+0000","last_fullsized":"2023-09-23T21:00:58.408150+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:59:57.122584+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.6","version":"79'161","reported_seq":534,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-09-23T21:00:56.112948+0000","last_change":"2023-09-23T20:57:26.907505+0000","last_active":"2023-09-23T21:00:56.112948+0000","last_peered":"2023-09-23T21:00:56.112948+0000","last_clean":"2023-09-23T21:00:56.112948+0000","last_became_active":"2023-09-23T20:57:08.240321+0000","last_became_peered":"2023-09-23T20:57:08.240321+0000","last_unstale":"2023-09-23T21:00:56.112948+0000","last_undegraded":"2023-09-23T21:00:56.112948+0000","last_fullsized":"2023-09-23T21:00:56.112948+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":3,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:26.907408+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:26.907408+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:34:17.392928+0000","scrub_duration":0.010337038,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":170,"reported_epoch":252,"state":"active+clean","last_fresh":"2023-09-23T21:00:58.408531+0000","last_change":"2023-09-23T20:56:45.988566+0000","last_active":"2023-09-23T21:00:58.408531+0000","last_peered":"2023-09-23T21:00:58.408531+0000","last_clean":"2023-09-23T21:00:58.408531+0000","last_became_active":"2023-09-23T20:56:45.988249+0000","last_became_peered":"2023-09-23T20:56:45.988249+0000","last_unstale":"2023-09-23T21:00:58.408531+0000","last_undegraded":"2023-09-23T21:00:58.408531+0000","last_fullsized":"2023-09-23T21:00:58.408531+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":205,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:51:36.644764+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,4],"acting":[0,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":223,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-09-23T21:00:56.193246+0000","last_change":"2023-09-23T20:54:31.898965+0000","last_active":"2023-09-23T21:00:56.193246+0000","last_peered":"2023-09-23T21:00:56.193246+0000","last_clean":"2023-09-23T21:00:56.193246+0000","last_became_active":"2023-09-23T20:54:31.898663+0000","last_became_peered":"2023-09-23T20:54:31.898663+0000","last_unstale":"2023-09-23T21:00:56.193246+0000","last_undegraded":"2023-09-23T21:00:56.193246+0000","last_fullsized":"2023-09-23T21:00:56.193246+0000","mapping_epoch":136,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:53:05.058087+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":214,"reported_epoch":252,"state":"active+clean","last_fresh":"2023-09-23T21:00:58.408562+0000","last_change":"2023-09-23T20:54:31.884717+0000","last_active":"2023-09-23T21:00:58.408562+0000","last_peered":"2023-09-23T21:00:58.408562+0000","last_clean":"2023-09-23T21:00:58.408562+0000","last_became_active":"2023-09-23T20:54:31.884345+0000","last_became_peered":"2023-09-23T20:54:31.884345+0000","last_unstale":"2023-09-23T21:00:58.408562+0000","last_undegraded":"2023-09-23T21:00:58.408562+0000","last_fullsized":"2023-09-23T21:00:58.408562+0000","mapping_epoch":136,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:02:26.393868+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.5","version":"79'161","reported_seq":516,"reported_epoch":252,"state":"active+clean","last_fresh":"2023-09-23T21:00:58.374370+0000","last_change":"2023-09-23T20:57:30.029479+0000","last_active":"2023-09-23T21:00:58.374370+0000","last_peered":"2023-09-23T21:00:58.374370+0000","last_clean":"2023-09-23T21:00:58.374370+0000","last_became_active":"2023-09-23T20:57:08.249909+0000","last_became_peered":"2023-09-23T20:57:08.249909+0000","last_unstale":"2023-09-23T21:00:58.374370+0000","last_undegraded":"2023-09-23T21:00:58.374370+0000","last_fullsized":"2023-09-23T21:00:58.374370+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":3,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:30.029396+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:30.029396+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:16:46.083145+0000","scrub_duration":0.012914682,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":216,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-09-23T21:00:56.121651+0000","last_change":"2023-09-23T20:51:10.836910+0000","last_active":"2023-09-23T21:00:56.121651+0000","last_peered":"2023-09-23T21:00:56.121651+0000","last_clean":"2023-09-23T21:00:56.121651+0000","last_became_active":"2023-09-23T20:51:10.836557+0000","last_became_peered":"2023-09-23T20:51:10.836557+0000","last_unstale":"2023-09-23T21:00:56.121651+0000","last_undegraded":"2023-09-23T21:00:56.121651+0000","last_fullsized":"2023-09-23T21:00:56.121651+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:51:54.019969+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":223,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-09-23T21:00:56.193286+0000","last_change":"2023-09-23T20:55:35.953444+0000","last_active":"2023-09-23T21:00:56.193286+0000","last_peered":"2023-09-23T21:00:56.193286+0000","last_clean":"2023-09-23T21:00:56.193286+0000","last_became_active":"2023-09-23T20:55:35.953151+0000","last_became_peered":"2023-09-23T20:55:35.953151+0000","last_unstale":"2023-09-23T21:00:56.193286+0000","last_undegraded":"2023-09-23T21:00:56.193286+0000","last_fullsized":"2023-09-23T21:00:56.193286+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:39:12.404391+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":41,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-09-23T21:00:56.120212+0000","last_change":"2023-09-23T20:57:44.982509+0000","last_active":"2023-09-23T21:00:56.120212+0000","last_peered":"2023-09-23T21:00:56.120212+0000","last_clean":"2023-09-23T21:00:56.120212+0000","last_became_active":"2023-09-23T20:57:44.982175+0000","last_became_peered":"2023-09-23T20:57:44.982175+0000","last_unstale":"2023-09-23T21:00:56.120212+0000","last_undegraded":"2023-09-23T21:00:56.120212+0000","last_fullsized":"2023-09-23T21:00:56.120212+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:37:49.215488+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.4","version":"79'161","reported_seq":531,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-09-23T21:00:56.120579+0000","last_change":"2023-09-23T20:57:20.012904+0000","last_active":"2023-09-23T21:00:56.120579+0000","last_peered":"2023-09-23T21:00:56.120579+0000","last_clean":"2023-09-23T21:00:56.120579+0000","last_became_active":"2023-09-23T20:57:08.237996+0000","last_became_peered":"2023-09-23T20:57:08.237996+0000","last_unstale":"2023-09-23T21:00:56.120579+0000","last_undegraded":"2023-09-23T21:00:56.120579+0000","last_fullsized":"2023-09-23T21:00:56.120579+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":3,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:20.012764+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:20.012764+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:51:28.796946+0000","scrub_duration":0.0045251919999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":201,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-09-23T21:00:56.113014+0000","last_change":"2023-09-23T20:55:35.949541+0000","last_active":"2023-09-23T21:00:56.113014+0000","last_peered":"2023-09-23T21:00:56.113014+0000","last_clean":"2023-09-23T21:00:56.113014+0000","last_became_active":"2023-09-23T20:55:35.944635+0000","last_became_peered":"2023-09-23T20:55:35.944635+0000","last_unstale":"2023-09-23T21:00:56.113014+0000","last_undegraded":"2023-09-23T21:00:56.113014+0000","last_fullsized":"2023-09-23T21:00:56.113014+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:30:36.033585+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"70'1","reported_seq":242,"reported_epoch":252,"state":"active+clean","last_fresh":"2023-09-23T21:00:58.408347+0000","last_change":"2023-09-23T20:54:14.009723+0000","last_active":"2023-09-23T21:00:58.408347+0000","last_peered":"2023-09-23T21:00:58.408347+0000","last_clean":"2023-09-23T21:00:58.408347+0000","last_became_active":"2023-09-23T20:54:13.941771+0000","last_became_peered":"2023-09-23T20:54:13.941771+0000","last_unstale":"2023-09-23T21:00:58.408347+0000","last_undegraded":"2023-09-23T21:00:58.408347+0000","last_fullsized":"2023-09-23T21:00:58.408347+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:48:25.588116+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":157,"reported_epoch":252,"state":"active+clean","last_fresh":"2023-09-23T21:00:58.373682+0000","last_change":"2023-09-23T20:55:35.949780+0000","last_active":"2023-09-23T21:00:58.373682+0000","last_peered":"2023-09-23T21:00:58.373682+0000","last_clean":"2023-09-23T21:00:58.373682+0000","last_became_active":"2023-09-23T20:55:35.949497+0000","last_became_peered":"2023-09-23T20:55:35.949497+0000","last_unstale":"2023-09-23T21:00:58.373682+0000","last_undegraded":"2023-09-23T21:00:58.373682+0000","last_fullsized":"2023-09-23T21:00:58.373682+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:09:37.420882+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":216,"reported_epoch":252,"state":"active+clean","last_fresh":"2023-09-23T21:00:58.408232+0000","last_change":"2023-09-23T20:54:13.929196+0000","last_active":"2023-09-23T21:00:58.408232+0000","last_peered":"2023-09-23T21:00:58.408232+0000","last_clean":"2023-09-23T21:00:58.408232+0000","last_became_active":"2023-09-23T20:54:13.928831+0000","last_became_peered":"2023-09-23T20:54:13.928831+0000","last_unstale":"2023-09-23T21:00:58.408232+0000","last_undegraded":"2023-09-23T21:00:58.408232+0000","last_fullsized":"2023-09-23T21:00:58.408232+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:28:44.316851+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":222,"reported_epoch":252,"state":"active+clean","last_fresh":"2023-09-23T21:00:58.412688+0000","last_change":"2023-09-23T20:52:12.855430+0000","last_active":"2023-09-23T21:00:58.412688+0000","last_peered":"2023-09-23T21:00:58.412688+0000","last_clean":"2023-09-23T21:00:58.412688+0000","last_became_active":"2023-09-23T20:52:12.855115+0000","last_became_peered":"2023-09-23T20:52:12.855115+0000","last_unstale":"2023-09-23T21:00:58.412688+0000","last_undegraded":"2023-09-23T21:00:58.412688+0000","last_fullsized":"2023-09-23T21:00:58.412688+0000","mapping_epoch":100,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":101,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:33:23.802214+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":130,"reported_epoch":252,"state":"active+clean","last_fresh":"2023-09-23T21:00:58.423041+0000","last_change":"2023-09-23T20:54:31.902946+0000","last_active":"2023-09-23T21:00:58.423041+0000","last_peered":"2023-09-23T21:00:58.423041+0000","last_clean":"2023-09-23T21:00:58.423041+0000","last_became_active":"2023-09-23T20:54:31.889484+0000","last_became_peered":"2023-09-23T20:54:31.889484+0000","last_unstale":"2023-09-23T21:00:58.423041+0000","last_undegraded":"2023-09-23T21:00:58.423041+0000","last_fullsized":"2023-09-23T21:00:58.423041+0000","mapping_epoch":136,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:44:44.278398+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":130,"reported_epoch":252,"state":"active+clean","last_fresh":"2023-09-23T21:00:58.374040+0000","last_change":"2023-09-23T20:54:13.950351+0000","last_active":"2023-09-23T21:00:58.374040+0000","last_peered":"2023-09-23T21:00:58.374040+0000","last_clean":"2023-09-23T21:00:58.374040+0000","last_became_active":"2023-09-23T20:54:13.949987+0000","last_became_peered":"2023-09-23T20:54:13.949987+0000","last_unstale":"2023-09-23T21:00:58.374040+0000","last_undegraded":"2023-09-23T21:00:58.374040+0000","last_fullsized":"2023-09-23T21:00:58.374040+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:46:17.765134+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":161,"reported_epoch":252,"state":"active+clean","last_fresh":"2023-09-23T21:00:58.423279+0000","last_change":"2023-09-23T20:56:18.141728+0000","last_active":"2023-09-23T21:00:58.423279+0000","last_peered":"2023-09-23T21:00:58.423279+0000","last_clean":"2023-09-23T21:00:58.423279+0000","last_became_active":"2023-09-23T20:56:18.137979+0000","last_became_peered":"2023-09-23T20:56:18.137979+0000","last_unstale":"2023-09-23T21:00:58.423279+0000","last_undegraded":"2023-09-23T21:00:58.423279+0000","last_fullsized":"2023-09-23T21:00:58.423279+0000","mapping_epoch":192,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":193,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:58:05.658443+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":130,"reported_epoch":252,"state":"active+clean","last_fresh":"2023-09-23T21:00:58.373738+0000","last_change":"2023-09-23T20:54:13.935307+0000","last_active":"2023-09-23T21:00:58.373738+0000","last_peered":"2023-09-23T21:00:58.373738+0000","last_clean":"2023-09-23T21:00:58.373738+0000","last_became_active":"2023-09-23T20:54:13.935000+0000","last_became_peered":"2023-09-23T20:54:13.935000+0000","last_unstale":"2023-09-23T21:00:58.373738+0000","last_undegraded":"2023-09-23T21:00:58.373738+0000","last_fullsized":"2023-09-23T21:00:58.373738+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:47:40.974082+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":240,"reported_epoch":252,"state":"active+clean","last_fresh":"2023-09-23T21:00:58.408420+0000","last_change":"2023-09-23T20:55:35.954870+0000","last_active":"2023-09-23T21:00:58.408420+0000","last_peered":"2023-09-23T21:00:58.408420+0000","last_clean":"2023-09-23T21:00:58.408420+0000","last_became_active":"2023-09-23T20:55:35.954568+0000","last_became_peered":"2023-09-23T20:55:35.954568+0000","last_unstale":"2023-09-23T21:00:58.408420+0000","last_undegraded":"2023-09-23T21:00:58.408420+0000","last_fullsized":"2023-09-23T21:00:58.408420+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:06:22.423566+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":239,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-09-23T21:00:56.120613+0000","last_change":"2023-09-23T20:51:10.838334+0000","last_active":"2023-09-23T21:00:56.120613+0000","last_peered":"2023-09-23T21:00:56.120613+0000","last_clean":"2023-09-23T21:00:56.120613+0000","last_became_active":"2023-09-23T20:51:10.838026+0000","last_became_peered":"2023-09-23T20:51:10.838026+0000","last_unstale":"2023-09-23T21:00:56.120613+0000","last_undegraded":"2023-09-23T21:00:56.120613+0000","last_fullsized":"2023-09-23T21:00:56.120613+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:04:37.009457+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":244,"reported_epoch":252,"state":"active+clean","last_fresh":"2023-09-23T21:00:58.409826+0000","last_change":"2023-09-23T20:54:31.903139+0000","last_active":"2023-09-23T21:00:58.409826+0000","last_peered":"2023-09-23T21:00:58.409826+0000","last_clean":"2023-09-23T21:00:58.409826+0000","last_became_active":"2023-09-23T20:54:31.884520+0000","last_became_peered":"2023-09-23T20:54:31.884520+0000","last_unstale":"2023-09-23T21:00:58.409826+0000","last_undegraded":"2023-09-23T21:00:58.409826+0000","last_fullsized":"2023-09-23T21:00:58.409826+0000","mapping_epoch":136,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:39:34.269705+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1108,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":147,"reported_epoch":252,"state":"active+clean","last_fresh":"2023-09-23T21:00:58.373414+0000","last_change":"2023-09-23T20:55:35.951572+0000","last_active":"2023-09-23T21:00:58.373414+0000","last_peered":"2023-09-23T21:00:58.373414+0000","last_clean":"2023-09-23T21:00:58.373414+0000","last_became_active":"2023-09-23T20:55:35.951294+0000","last_became_peered":"2023-09-23T20:55:35.951294+0000","last_unstale":"2023-09-23T21:00:58.373414+0000","last_undegraded":"2023-09-23T21:00:58.373414+0000","last_fullsized":"2023-09-23T21:00:58.373414+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:36:32.447210+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":22,"num_bytes_recovered":5990,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":21,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3381,"ondisk_log_size":3381,"up":63,"acting":63,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201530,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":202516,"kb_used_data":2636,"kb_used_omap":0,"kb_used_meta":199872,"kb_avail":93542636,"statfs":{"total":95995035648,"available":95787659264,"internally_reserved":0,"allocated":2699264,"data_stored":1799353,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":204668928},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":8,"apply_latency_ms":8,"commit_latency_ns":8000000,"apply_latency_ns":8000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:00:49 2023","interfaces":[{"interface":"back","average":{"1min":3.766,"5min":2.3260000000000001,"15min":1.302},"min":{"1min":0.23200000000000001,"5min":0.216,"15min":0.21099999999999999},"max":{"1min":33.326000000000001,"5min":43.994999999999997,"15min":43.994999999999997},"last":0.49199999999999999},{"interface":"front","average":{"1min":3.8399999999999999,"5min":2.3530000000000002,"15min":1.389},"min":{"1min":0.33200000000000002,"5min":0.28199999999999997,"15min":0.218},"max":{"1min":33.354999999999997,"5min":44.206000000000003,"15min":44.206000000000003},"last":0.51400000000000001}]},{"osd":1,"last update":"Sat Sep 23 21:00:23 2023","interfaces":[{"interface":"back","average":{"1min":1.4419999999999999,"5min":1.6559999999999999,"15min":1.2829999999999999},"min":{"1min":0.28199999999999997,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":6.8330000000000002,"5min":10.622,"15min":22.591999999999999},"last":0.53000000000000003},{"interface":"front","average":{"1min":1.823,"5min":1.7110000000000001,"15min":1.171},"min":{"1min":0.26500000000000001,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":7.9249999999999998,"5min":10.097,"15min":21.437000000000001},"last":0.36899999999999999}]},{"osd":2,"last update":"Sat Sep 23 21:00:59 2023","interfaces":[{"interface":"back","average":{"1min":2.3799999999999999,"5min":2.1699999999999999,"15min":1.4159999999999999},"min":{"1min":0.313,"5min":0.26500000000000001,"15min":0.24399999999999999},"max":{"1min":7.2050000000000001,"5min":10.805999999999999,"15min":10.805999999999999},"last":0.38700000000000001},{"interface":"front","average":{"1min":2.976,"5min":2.399,"15min":1.4690000000000001},"min":{"1min":0.35499999999999998,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":9.6660000000000004,"5min":45.25,"15min":45.25},"last":0.54200000000000004}]},{"osd":3,"last update":"Sat Sep 23 21:00:32 2023","interfaces":[{"interface":"back","average":{"1min":1.9099999999999999,"5min":2.157,"15min":1.8149999999999999},"min":{"1min":0.39300000000000002,"5min":0.32100000000000001,"15min":0.314},"max":{"1min":5.2549999999999999,"5min":9.9939999999999998,"15min":36.408000000000001},"last":0.58699999999999997},{"interface":"front","average":{"1min":3.5870000000000002,"5min":2.5219999999999998,"15min":1.7529999999999999},"min":{"1min":0.40699999999999997,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":13.83,"5min":14.220000000000001,"15min":35.593000000000004},"last":0.48399999999999999}]},{"osd":4,"last update":"Sat Sep 23 21:00:23 2023","interfaces":[{"interface":"back","average":{"1min":0.76200000000000001,"5min":0.68100000000000005,"15min":0.627},"min":{"1min":0.27900000000000003,"5min":0.23799999999999999,"15min":0.23699999999999999},"max":{"1min":3.8929999999999998,"5min":4.5410000000000004,"15min":5.8920000000000003},"last":0.32000000000000001},{"interface":"front","average":{"1min":0.86799999999999999,"5min":0.72899999999999998,"15min":0.64300000000000002},"min":{"1min":0.34000000000000002,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":3.3999999999999999,"5min":4.9900000000000002,"15min":5.3200000000000003},"last":0.42799999999999999}]},{"osd":5,"last update":"Sat Sep 23 21:00:23 2023","interfaces":[{"interface":"back","average":{"1min":0.56999999999999995,"5min":0.69099999999999995,"15min":0.63500000000000001},"min":{"1min":0.28999999999999998,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":2.5249999999999999,"5min":7.0389999999999997,"15min":7.0389999999999997},"last":0.29899999999999999},{"interface":"front","average":{"1min":0.86899999999999999,"5min":0.752,"15min":0.63900000000000001},"min":{"1min":0.36899999999999999,"5min":0.29999999999999999,"15min":0.27800000000000002},"max":{"1min":3.8719999999999999,"5min":4.5999999999999996,"15min":4.5999999999999996},"last":0.443}]},{"osd":6,"last update":"Sat Sep 23 21:00:46 2023","interfaces":[{"interface":"back","average":{"1min":0.85599999999999998,"5min":0.71799999999999997,"15min":0.68400000000000005},"min":{"1min":0.38200000000000001,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":3.4329999999999998,"5min":3.5249999999999999,"15min":3.5249999999999999},"last":0.40000000000000002},{"interface":"front","average":{"1min":1.0620000000000001,"5min":0.80100000000000005,"15min":0.80300000000000005},"min":{"1min":0.40899999999999997,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":3.5569999999999999,"5min":3.5569999999999999,"15min":3.5569999999999999},"last":0.46400000000000002}]}]},{"osd":6,"up_from":164,"seq":704374636613,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":128324,"kb_used_data":2940,"kb_used_omap":10,"kb_used_meta":125365,"kb_avail":93616828,"statfs":{"total":95995035648,"available":95863631872,"internally_reserved":0,"allocated":3010560,"data_stored":2126622,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":10328,"internal_metadata":128374696},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:00:46 2023","interfaces":[{"interface":"back","average":{"1min":2.1469999999999998,"5min":1.7569999999999999,"15min":1.1619999999999999},"min":{"1min":0.26400000000000001,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":10.404,"5min":10.404,"15min":10.404},"last":0.57399999999999995},{"interface":"front","average":{"1min":2.895,"5min":1.9890000000000001,"15min":1.3180000000000001},"min":{"1min":0.28599999999999998,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":11.023,"5min":11.023,"15min":11.023},"last":0.38400000000000001}]},{"osd":1,"last update":"Sat Sep 23 21:00:46 2023","interfaces":[{"interface":"back","average":{"1min":1.601,"5min":1.4119999999999999,"15min":0.996},"min":{"1min":0.32900000000000001,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":3.839,"5min":8.4309999999999992,"15min":8.4309999999999992},"last":0.47299999999999998},{"interface":"front","average":{"1min":2.6560000000000001,"5min":1.7290000000000001,"15min":1.155},"min":{"1min":0.26200000000000001,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":10.518000000000001,"5min":10.518000000000001,"15min":10.518000000000001},"last":0.40200000000000002}]},{"osd":2,"last update":"Sat Sep 23 21:00:46 2023","interfaces":[{"interface":"back","average":{"1min":2.9249999999999998,"5min":2.3279999999999998,"15min":1.8740000000000001},"min":{"1min":0.371,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":10.109999999999999,"5min":13.750999999999999,"15min":13.750999999999999},"last":0.41599999999999998},{"interface":"front","average":{"1min":3.1920000000000002,"5min":2.2170000000000001,"15min":1.4770000000000001},"min":{"1min":0.34599999999999997,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":20.195,"5min":20.195,"15min":20.195},"last":0.503}]},{"osd":3,"last update":"Sat Sep 23 21:00:46 2023","interfaces":[{"interface":"back","average":{"1min":4.1660000000000004,"5min":2.0249999999999999,"15min":1.419},"min":{"1min":0.48699999999999999,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":14.144,"5min":14.144,"15min":14.144},"last":0.48399999999999999},{"interface":"front","average":{"1min":2.7919999999999998,"5min":1.986,"15min":1.73},"min":{"1min":0.38700000000000001,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":11.989000000000001,"5min":11.989000000000001,"15min":11.989000000000001},"last":0.45200000000000001}]},{"osd":4,"last update":"Sat Sep 23 21:00:46 2023","interfaces":[{"interface":"back","average":{"1min":0.81599999999999995,"5min":0.73899999999999999,"15min":0.94699999999999995},"min":{"1min":0.28100000000000003,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":3.4260000000000002,"5min":9.0079999999999991,"15min":9.0079999999999991},"last":0.439},{"interface":"front","average":{"1min":0.626,"5min":0.51800000000000002,"15min":0.52600000000000002},"min":{"1min":0.29799999999999999,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":2.4860000000000002,"5min":2.4860000000000002,"15min":2.4860000000000002},"last":0.36799999999999999}]},{"osd":5,"last update":"Sat Sep 23 21:00:46 2023","interfaces":[{"interface":"back","average":{"1min":0.94499999999999995,"5min":0.69799999999999995,"15min":0.59299999999999997},"min":{"1min":0.32700000000000001,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":5.4119999999999999,"5min":5.4119999999999999,"15min":5.4119999999999999},"last":0.46400000000000002},{"interface":"front","average":{"1min":0.90600000000000003,"5min":0.75900000000000001,"15min":0.79300000000000004},"min":{"1min":0.34499999999999997,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":3.536,"5min":3.536,"15min":3.536},"last":0.42599999999999999}]},{"osd":7,"last update":"Sat Sep 23 21:00:46 2023","interfaces":[{"interface":"back","average":{"1min":1.151,"5min":0.81599999999999995,"15min":0.89100000000000001},"min":{"1min":0.36299999999999999,"5min":0.312,"15min":0.312},"max":{"1min":3.8929999999999998,"5min":7.8570000000000002,"15min":7.8570000000000002},"last":0.45700000000000002},{"interface":"front","average":{"1min":1.0700000000000001,"5min":0.75700000000000001,"15min":0.60499999999999998},"min":{"1min":0.40100000000000002,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":3.3279999999999998,"5min":8.282,"15min":8.282},"last":0.495}]}]},{"osd":1,"up_from":88,"seq":377957122184,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":642448,"kb_used_data":353160,"kb_used_omap":9,"kb_used_meta":289270,"kb_avail":93102704,"statfs":{"total":95995035648,"available":95337168896,"internally_reserved":0,"allocated":361635840,"data_stored":360393919,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9239,"internal_metadata":296213481},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:00:26 2023","interfaces":[{"interface":"back","average":{"1min":2.9870000000000001,"5min":3.048,"15min":1.7110000000000001},"min":{"1min":0.26000000000000001,"5min":0.20699999999999999,"15min":0.20699999999999999},"max":{"1min":8.8000000000000007,"5min":16.646999999999998,"15min":16.646999999999998},"last":0.23100000000000001},{"interface":"front","average":{"1min":4.7480000000000002,"5min":3.577,"15min":1.9690000000000001},"min":{"1min":0.38200000000000001,"5min":0.246,"15min":0.22500000000000001},"max":{"1min":27.797999999999998,"5min":27.797999999999998,"15min":27.797999999999998},"last":0.32300000000000001}]},{"osd":2,"last update":"Sat Sep 23 21:00:57 2023","interfaces":[{"interface":"back","average":{"1min":2.3559999999999999,"5min":2.198,"15min":1.4510000000000001},"min":{"1min":0.251,"5min":0.221,"15min":0.20300000000000001},"max":{"1min":7.1280000000000001,"5min":13.752000000000001,"15min":14.335000000000001},"last":0.27000000000000002},{"interface":"front","average":{"1min":5.29,"5min":3.9430000000000001,"15min":2.262},"min":{"1min":0.23699999999999999,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":17.405999999999999,"5min":18.026,"15min":38.951000000000001},"last":0.249}]},{"osd":3,"last update":"Sat Sep 23 21:00:26 2023","interfaces":[{"interface":"back","average":{"1min":3.8849999999999998,"5min":2.8220000000000001,"15min":1.794},"min":{"1min":0.38500000000000001,"5min":0.246,"15min":0.246},"max":{"1min":22.686,"5min":22.686,"15min":25.314},"last":0.26000000000000001},{"interface":"front","average":{"1min":3.4510000000000001,"5min":2.605,"15min":1.613},"min":{"1min":0.36699999999999999,"5min":0.20599999999999999,"15min":0.20599999999999999},"max":{"1min":14.473000000000001,"5min":14.473000000000001,"15min":19.881},"last":0.36099999999999999}]},{"osd":4,"last update":"Sat Sep 23 21:00:26 2023","interfaces":[{"interface":"back","average":{"1min":2.383,"5min":1.704,"15min":1.329},"min":{"1min":0.34499999999999997,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":18.231000000000002,"5min":18.231000000000002,"15min":18.231000000000002},"last":0.34799999999999998},{"interface":"front","average":{"1min":1.6080000000000001,"5min":1.55,"15min":1.1419999999999999},"min":{"1min":0.35399999999999998,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":6.6360000000000001,"5min":7.532,"15min":11.234999999999999},"last":0.497}]},{"osd":5,"last update":"Sat Sep 23 21:00:26 2023","interfaces":[{"interface":"back","average":{"1min":2.6080000000000001,"5min":2.2999999999999998,"15min":1.419},"min":{"1min":0.34999999999999998,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":12.699999999999999,"5min":15.041,"15min":15.041},"last":0.47399999999999998},{"interface":"front","average":{"1min":2.415,"5min":1.762,"15min":1.357},"min":{"1min":0.40000000000000002,"5min":0.32600000000000001,"15min":0.32200000000000001},"max":{"1min":18.280999999999999,"5min":18.280999999999999,"15min":18.280999999999999},"last":0.51800000000000002}]},{"osd":6,"last update":"Sat Sep 23 21:00:45 2023","interfaces":[{"interface":"back","average":{"1min":2.298,"5min":1.7569999999999999,"15min":1.462},"min":{"1min":0.35799999999999998,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":18.984999999999999,"5min":18.984999999999999,"15min":18.984999999999999},"last":0.46200000000000002},{"interface":"front","average":{"1min":1.5840000000000001,"5min":1.696,"15min":1.3999999999999999},"min":{"1min":0.38900000000000001,"5min":0.309,"15min":0.309},"max":{"1min":6.7779999999999996,"5min":8.7530000000000001,"15min":8.7530000000000001},"last":0.54300000000000004}]},{"osd":7,"last update":"Sat Sep 23 21:00:26 2023","interfaces":[{"interface":"back","average":{"1min":1.8180000000000001,"5min":1.7829999999999999,"15min":1.24},"min":{"1min":0.41699999999999998,"5min":0.36199999999999999,"15min":0.35699999999999998},"max":{"1min":6.657,"5min":8.7769999999999992,"15min":11.186999999999999},"last":0.56999999999999995},{"interface":"front","average":{"1min":2.415,"5min":2.319,"15min":1.4399999999999999},"min":{"1min":0.41499999999999998,"5min":0.40899999999999997,"15min":0.34599999999999997},"max":{"1min":12.84,"5min":15.098000000000001,"15min":15.098000000000001},"last":0.53100000000000003}]}]},{"osd":0,"up_from":83,"seq":356482285711,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":237880,"kb_used_data":3312,"kb_used_omap":28,"kb_used_meta":234531,"kb_avail":93507272,"statfs":{"total":95995035648,"available":95751446528,"internally_reserved":0,"allocated":3391488,"data_stored":2171813,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":28848,"internal_metadata":240160592},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":12,"apply_latency_ms":12,"commit_latency_ns":12000000,"apply_latency_ns":12000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 21:00:20 2023","interfaces":[{"interface":"back","average":{"1min":6.4480000000000004,"5min":3.7120000000000002,"15min":1.893},"min":{"1min":0.20000000000000001,"5min":0.191,"15min":0.191},"max":{"1min":19.146000000000001,"5min":19.146000000000001,"15min":19.146000000000001},"last":0.24199999999999999},{"interface":"front","average":{"1min":5.4379999999999997,"5min":3.6789999999999998,"15min":1.8759999999999999},"min":{"1min":0.39500000000000002,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":11.074,"5min":15.771000000000001,"15min":17.704999999999998},"last":0.309}]},{"osd":2,"last update":"Sat Sep 23 21:00:55 2023","interfaces":[{"interface":"back","average":{"1min":3.9039999999999999,"5min":3.798,"15min":2.1000000000000001},"min":{"1min":0.32100000000000001,"5min":0.30599999999999999,"15min":0.23799999999999999},"max":{"1min":10.914,"5min":21.053000000000001,"15min":21.053000000000001},"last":0.39000000000000001},{"interface":"front","average":{"1min":4.1200000000000001,"5min":4.0369999999999999,"15min":2.1930000000000001},"min":{"1min":0.36099999999999999,"5min":0.25800000000000001,"15min":0.21099999999999999},"max":{"1min":9.548,"5min":21.747,"15min":21.747},"last":0.23100000000000001}]},{"osd":3,"last update":"Sat Sep 23 21:00:20 2023","interfaces":[{"interface":"back","average":{"1min":6.532,"5min":3.8690000000000002,"15min":1.9990000000000001},"min":{"1min":1.052,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":15.082000000000001,"5min":21.184000000000001,"15min":21.184000000000001},"last":0.34799999999999998},{"interface":"front","average":{"1min":4.9050000000000002,"5min":4.008,"15min":2.0899999999999999},"min":{"1min":0.48499999999999999,"5min":0.28000000000000003,"15min":0.23300000000000001},"max":{"1min":9.7560000000000002,"5min":21.167000000000002,"15min":21.167000000000002},"last":0.27400000000000002}]},{"osd":4,"last update":"Sat Sep 23 21:00:55 2023","interfaces":[{"interface":"back","average":{"1min":2.52,"5min":2.875,"15min":1.6839999999999999},"min":{"1min":0.39200000000000002,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":9.6029999999999998,"5min":21.594999999999999,"15min":21.594999999999999},"last":0.28599999999999998},{"interface":"front","average":{"1min":3.786,"5min":2.4020000000000001,"15min":1.486},"min":{"1min":0.437,"5min":0.31,"15min":0.28100000000000003},"max":{"1min":13.904,"5min":13.904,"15min":18.724},"last":0.40100000000000002}]},{"osd":5,"last update":"Sat Sep 23 21:00:55 2023","interfaces":[{"interface":"back","average":{"1min":2.794,"5min":2.1339999999999999,"15min":1.401},"min":{"1min":0.309,"5min":0.309,"15min":0.309},"max":{"1min":6.343,"5min":8.8970000000000002,"15min":8.8970000000000002},"last":0.439},{"interface":"front","average":{"1min":2.5430000000000001,"5min":2.7109999999999999,"15min":1.6870000000000001},"min":{"1min":0.38700000000000001,"5min":0.38700000000000001,"15min":0.34999999999999998},"max":{"1min":9.6340000000000003,"5min":20.512,"15min":20.512},"last":0.373}]},{"osd":6,"last update":"Sat Sep 23 21:00:50 2023","interfaces":[{"interface":"back","average":{"1min":3.4049999999999998,"5min":2.4950000000000001,"15min":1.9359999999999999},"min":{"1min":0.48199999999999998,"5min":0.37,"15min":0.37},"max":{"1min":13.933,"5min":13.933,"15min":13.933},"last":0.47599999999999998},{"interface":"front","average":{"1min":3.8260000000000001,"5min":2.6139999999999999,"15min":2.0350000000000001},"min":{"1min":0.46400000000000002,"5min":0.377,"15min":0.377},"max":{"1min":13.962,"5min":13.962,"15min":13.962},"last":0.45900000000000002}]},{"osd":7,"last update":"Sat Sep 23 21:00:55 2023","interfaces":[{"interface":"back","average":{"1min":2.7919999999999998,"5min":2.254,"15min":1.4570000000000001},"min":{"1min":0.40300000000000002,"5min":0.379,"15min":0.379},"max":{"1min":6.298,"5min":8.7729999999999997,"15min":8.7729999999999997},"last":0.46899999999999997},{"interface":"front","average":{"1min":3.605,"5min":2.4830000000000001,"15min":1.516},"min":{"1min":0.49099999999999999,"5min":0.377,"15min":0.377},"max":{"1min":13.856999999999999,"5min":13.856999999999999,"15min":18.673999999999999},"last":0.49199999999999999}]}]},{"osd":2,"up_from":93,"seq":399431958658,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":671152,"kb_used_data":353512,"kb_used_omap":81,"kb_used_meta":317550,"kb_avail":93074000,"statfs":{"total":95995035648,"available":95307776000,"internally_reserved":0,"allocated":361996288,"data_stored":360781582,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":83750,"internal_metadata":325171418},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":6,"apply_latency_ms":6,"commit_latency_ns":6000000,"apply_latency_ns":6000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:00:50 2023","interfaces":[{"interface":"back","average":{"1min":4.3099999999999996,"5min":3.1720000000000002,"15min":1.6439999999999999},"min":{"1min":0.27300000000000002,"5min":0.17999999999999999,"15min":0.17999999999999999},"max":{"1min":13.494,"5min":29.498000000000001,"15min":29.498000000000001},"last":0.32300000000000001},{"interface":"front","average":{"1min":4.2800000000000002,"5min":3.681,"15min":1.9319999999999999},"min":{"1min":0.29699999999999999,"5min":0.20599999999999999,"15min":0.20599999999999999},"max":{"1min":11.629,"5min":17.251000000000001,"15min":17.251000000000001},"last":0.40300000000000002}]},{"osd":1,"last update":"Sat Sep 23 21:00:50 2023","interfaces":[{"interface":"back","average":{"1min":2.7599999999999998,"5min":2.661,"15min":1.782},"min":{"1min":0.27800000000000002,"5min":0.21199999999999999,"15min":0.20200000000000001},"max":{"1min":13.523,"5min":14.709,"15min":16.081},"last":0.34799999999999998},{"interface":"front","average":{"1min":6.0609999999999999,"5min":3.952,"15min":2.3460000000000001},"min":{"1min":0.26300000000000001,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":26.552,"5min":26.552,"15min":26.552},"last":0.44}]},{"osd":3,"last update":"Sat Sep 23 21:00:31 2023","interfaces":[{"interface":"back","average":{"1min":4.71,"5min":3.8450000000000002,"15min":2.1800000000000002},"min":{"1min":0.28599999999999998,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":10.468,"5min":27.614999999999998,"15min":27.614999999999998},"last":0.35799999999999998},{"interface":"front","average":{"1min":4.2939999999999996,"5min":4.0309999999999997,"15min":2.4540000000000002},"min":{"1min":0.64400000000000002,"5min":0.23899999999999999,"15min":0.221},"max":{"1min":11.673999999999999,"5min":27.562000000000001,"15min":27.562000000000001},"last":0.33500000000000002}]},{"osd":4,"last update":"Sat Sep 23 21:00:50 2023","interfaces":[{"interface":"back","average":{"1min":2.1099999999999999,"5min":1.8400000000000001,"15min":1.3069999999999999},"min":{"1min":0.29299999999999998,"5min":0.28699999999999998,"15min":0.26400000000000001},"max":{"1min":10.045,"5min":10.045,"15min":10.045},"last":0.626},{"interface":"front","average":{"1min":3.0459999999999998,"5min":2.222,"15min":1.53},"min":{"1min":0.42699999999999999,"5min":0.313,"15min":0.313},"max":{"1min":14.039999999999999,"5min":14.039999999999999,"15min":14.039999999999999},"last":0.45300000000000001}]},{"osd":5,"last update":"Sat Sep 23 21:00:50 2023","interfaces":[{"interface":"back","average":{"1min":2.1150000000000002,"5min":1.857,"15min":1.3959999999999999},"min":{"1min":0.34200000000000003,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":10.069000000000001,"5min":10.069000000000001,"15min":10.069000000000001},"last":0.66000000000000003},{"interface":"front","average":{"1min":3.0329999999999999,"5min":2.1230000000000002,"15min":1.532},"min":{"1min":0.46600000000000003,"5min":0.32600000000000001,"15min":0.30299999999999999},"max":{"1min":13.98,"5min":13.98,"15min":13.98},"last":0.47499999999999998}]},{"osd":6,"last update":"Sat Sep 23 21:00:47 2023","interfaces":[{"interface":"back","average":{"1min":3.2559999999999998,"5min":2.3820000000000001,"15min":1.9059999999999999},"min":{"1min":0.54500000000000004,"5min":0.312,"15min":0.312},"max":{"1min":14.016,"5min":14.244999999999999,"15min":14.244999999999999},"last":0.48999999999999999},{"interface":"front","average":{"1min":3.702,"5min":2.3479999999999999,"15min":1.4690000000000001},"min":{"1min":0.61399999999999999,"5min":0.374,"15min":0.374},"max":{"1min":14.068,"5min":14.068,"15min":14.068},"last":0.41999999999999998}]},{"osd":7,"last update":"Sat Sep 23 21:00:50 2023","interfaces":[{"interface":"back","average":{"1min":2.0670000000000002,"5min":1.496,"15min":1.1399999999999999},"min":{"1min":0.41899999999999998,"5min":0.315,"15min":0.315},"max":{"1min":13.427,"5min":13.427,"15min":13.427},"last":0.51800000000000002},{"interface":"front","average":{"1min":2.29,"5min":1.907,"15min":1.401},"min":{"1min":0.372,"5min":0.35099999999999998,"15min":0.34499999999999997},"max":{"1min":10.092000000000001,"5min":10.092000000000001,"15min":10.092000000000001},"last":0.68899999999999995}]}]},{"osd":3,"up_from":98,"seq":420906795131,"num_pgs":49,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":410632,"kb_used_data":3520,"kb_used_omap":7,"kb_used_meta":407096,"kb_avail":93334520,"statfs":{"total":95995035648,"available":95574548480,"internally_reserved":0,"allocated":3604480,"data_stored":2426032,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7576,"internal_metadata":416866920},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:00:28 2023","interfaces":[{"interface":"back","average":{"1min":5.6130000000000004,"5min":3.0470000000000002,"15min":2.0840000000000001},"min":{"1min":0.20399999999999999,"5min":0.20399999999999999,"15min":0.20399999999999999},"max":{"1min":27.962,"5min":27.962,"15min":27.962},"last":0.32000000000000001},{"interface":"front","average":{"1min":4.984,"5min":3.5169999999999999,"15min":1.9339999999999999},"min":{"1min":0.21099999999999999,"5min":0.19800000000000001,"15min":0.19800000000000001},"max":{"1min":28.058,"5min":28.058,"15min":28.058},"last":0.34899999999999998}]},{"osd":1,"last update":"Sat Sep 23 21:00:28 2023","interfaces":[{"interface":"back","average":{"1min":4.8730000000000002,"5min":3.1760000000000002,"15min":1.9119999999999999},"min":{"1min":0.22600000000000001,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":22.681999999999999,"5min":22.681999999999999,"15min":22.681999999999999},"last":0.28699999999999998},{"interface":"front","average":{"1min":4.0789999999999997,"5min":2.5169999999999999,"15min":1.929},"min":{"1min":0.25,"5min":0.218,"15min":0.218},"max":{"1min":19.329000000000001,"5min":19.329000000000001,"15min":25.353000000000002},"last":0.38}]},{"osd":2,"last update":"Sat Sep 23 21:00:28 2023","interfaces":[{"interface":"back","average":{"1min":6.1680000000000001,"5min":3.8039999999999998,"15min":2.113},"min":{"1min":0.36199999999999999,"5min":0.33000000000000002,"15min":0.30099999999999999},"max":{"1min":28.251999999999999,"5min":28.251999999999999,"15min":28.251999999999999},"last":0.34100000000000003},{"interface":"front","average":{"1min":4.6779999999999999,"5min":2.9300000000000002,"15min":2.0750000000000002},"min":{"1min":0.372,"5min":0.23999999999999999,"15min":0.22600000000000001},"max":{"1min":16.213999999999999,"5min":16.213999999999999,"15min":25.305},"last":0.41699999999999998}]},{"osd":4,"last update":"Sat Sep 23 21:00:28 2023","interfaces":[{"interface":"back","average":{"1min":3.258,"5min":2.2869999999999999,"15min":1.4750000000000001},"min":{"1min":0.314,"5min":0.29899999999999999,"15min":0.29299999999999998},"max":{"1min":18.239000000000001,"5min":18.239000000000001,"15min":18.239000000000001},"last":0.39600000000000002},{"interface":"front","average":{"1min":2.7200000000000002,"5min":1.5860000000000001,"15min":1.474},"min":{"1min":0.317,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":14.808,"5min":14.808,"15min":22.853000000000002},"last":0.48699999999999999}]},{"osd":5,"last update":"Sat Sep 23 21:00:28 2023","interfaces":[{"interface":"back","average":{"1min":3.2930000000000001,"5min":2.3399999999999999,"15min":1.4690000000000001},"min":{"1min":0.33200000000000002,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":18.324999999999999,"5min":18.324999999999999,"15min":18.324999999999999},"last":0.442},{"interface":"front","average":{"1min":3.0910000000000002,"5min":2.363,"15min":1.4359999999999999},"min":{"1min":0.33700000000000002,"5min":0.33300000000000002,"15min":0.33100000000000002},"max":{"1min":11.327,"5min":13.43,"15min":13.43},"last":0.42699999999999999}]},{"osd":6,"last update":"Sat Sep 23 21:00:51 2023","interfaces":[{"interface":"back","average":{"1min":3.601,"5min":1.9450000000000001,"15min":1.2809999999999999},"min":{"1min":0.373,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":14.833,"5min":14.833,"15min":14.833},"last":0.46000000000000002},{"interface":"front","average":{"1min":4.641,"5min":2.6110000000000002,"15min":1.8109999999999999},"min":{"1min":0.40899999999999997,"5min":0.38700000000000001,"15min":0.38700000000000001},"max":{"1min":13.576000000000001,"5min":13.576000000000001,"15min":13.576000000000001},"last":0.51900000000000002}]},{"osd":7,"last update":"Sat Sep 23 21:00:28 2023","interfaces":[{"interface":"back","average":{"1min":3.2879999999999998,"5min":2.5649999999999999,"15min":1.6399999999999999},"min":{"1min":0.39200000000000002,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":18.289999999999999,"5min":18.289999999999999,"15min":18.289999999999999},"last":0.5},{"interface":"front","average":{"1min":3.1219999999999999,"5min":2.399,"15min":1.506},"min":{"1min":0.38900000000000001,"5min":0.38900000000000001,"15min":0.379},"max":{"1min":11.378,"5min":13.355,"15min":13.355},"last":0.56799999999999995}]}]},{"osd":4,"up_from":31,"seq":133143986439,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":154144,"kb_used_data":2648,"kb_used_omap":0,"kb_used_meta":151488,"kb_avail":93591008,"statfs":{"total":95995035648,"available":95837192192,"internally_reserved":0,"allocated":2711552,"data_stored":1807794,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":155123712},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":11,"apply_latency_ms":11,"commit_latency_ns":11000000,"apply_latency_ns":11000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:00:55 2023","interfaces":[{"interface":"back","average":{"1min":1.8089999999999999,"5min":1.893,"15min":1.2210000000000001},"min":{"1min":0.218,"5min":0.214,"15min":0.214},"max":{"1min":8.9429999999999996,"5min":11.784000000000001,"15min":11.784000000000001},"last":0.51100000000000001},{"interface":"front","average":{"1min":1.835,"5min":1.9259999999999999,"15min":1.397},"min":{"1min":0.27700000000000002,"5min":0.27700000000000002,"15min":0.25800000000000001},"max":{"1min":8.9049999999999994,"5min":11.766,"15min":11.766},"last":0.439}]},{"osd":1,"last update":"Sat Sep 23 21:00:22 2023","interfaces":[{"interface":"back","average":{"1min":1.4350000000000001,"5min":1.673,"15min":1.1779999999999999},"min":{"1min":0.25800000000000001,"5min":0.246,"15min":0.23599999999999999},"max":{"1min":5.5570000000000004,"5min":10.153,"15min":10.153},"last":0.27700000000000002},{"interface":"front","average":{"1min":2.6040000000000001,"5min":2.2749999999999999,"15min":1.2749999999999999},"min":{"1min":0.29199999999999998,"5min":0.26100000000000001,"15min":0.23200000000000001},"max":{"1min":10.882999999999999,"5min":12.292999999999999,"15min":12.292999999999999},"last":0.42299999999999999}]},{"osd":2,"last update":"Sat Sep 23 21:00:55 2023","interfaces":[{"interface":"back","average":{"1min":2.9249999999999998,"5min":1.9490000000000001,"15min":1.3899999999999999},"min":{"1min":0.32500000000000001,"5min":0.313,"15min":0.29099999999999998},"max":{"1min":9.8469999999999995,"5min":9.8469999999999995,"15min":15.288},"last":0.33400000000000002},{"interface":"front","average":{"1min":2.2240000000000002,"5min":1.905,"15min":1.496},"min":{"1min":0.316,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":7.6829999999999998,"5min":12.855,"15min":15.262},"last":0.44700000000000001}]},{"osd":3,"last update":"Sat Sep 23 21:00:22 2023","interfaces":[{"interface":"back","average":{"1min":3.5569999999999999,"5min":2.3479999999999999,"15min":1.8120000000000001},"min":{"1min":0.39200000000000002,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":8.5389999999999997,"5min":14.824999999999999,"15min":31.562999999999999},"last":0.318},{"interface":"front","average":{"1min":3.5129999999999999,"5min":2.4550000000000001,"15min":1.8740000000000001},"min":{"1min":0.48399999999999999,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":8.5570000000000004,"5min":14.839,"15min":31.574000000000002},"last":0.47899999999999998}]},{"osd":5,"last update":"Sat Sep 23 21:00:39 2023","interfaces":[{"interface":"back","average":{"1min":0.33800000000000002,"5min":0.56999999999999995,"15min":0.51900000000000002},"min":{"1min":0.251,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":0.495,"5min":3.008,"15min":3.3279999999999998},"last":0.38300000000000001},{"interface":"front","average":{"1min":0.55500000000000005,"5min":0.71199999999999997,"15min":0.621},"min":{"1min":0.27600000000000002,"5min":0.25600000000000001,"15min":0.251},"max":{"1min":2.6539999999999999,"5min":3.4830000000000001,"15min":7.7619999999999996},"last":0.30099999999999999}]},{"osd":6,"last update":"Sat Sep 23 21:00:51 2023","interfaces":[{"interface":"back","average":{"1min":0.66400000000000003,"5min":0.73999999999999999,"15min":0.56899999999999995},"min":{"1min":0.27300000000000002,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":2.9239999999999999,"5min":4.6909999999999998,"15min":4.6909999999999998},"last":0.34599999999999997},{"interface":"front","average":{"1min":0.54700000000000004,"5min":0.71799999999999997,"15min":0.66800000000000004},"min":{"1min":0.33000000000000002,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":1.8340000000000001,"5min":3.456,"15min":3.456},"last":0.46300000000000002}]},{"osd":7,"last update":"Sat Sep 23 21:00:22 2023","interfaces":[{"interface":"back","average":{"1min":0.52100000000000002,"5min":0.754,"15min":0.69099999999999995},"min":{"1min":0.33400000000000002,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":2.302,"5min":6.2240000000000002,"15min":7.383},"last":0.496},{"interface":"front","average":{"1min":0.56100000000000005,"5min":0.77500000000000002,"15min":0.67700000000000005},"min":{"1min":0.32700000000000001,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":2.4420000000000002,"5min":7.8310000000000004,"15min":7.8310000000000004},"last":0.36799999999999999}]}]},{"osd":5,"up_from":39,"seq":167503724804,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":451912,"kb_used_data":310016,"kb_used_omap":0,"kb_used_meta":141888,"kb_avail":93293240,"statfs":{"total":95995035648,"available":95532277760,"internally_reserved":0,"allocated":317456384,"data_stored":316565302,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":145293312},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":5,"apply_latency_ms":5,"commit_latency_ns":5000000,"apply_latency_ns":5000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:00:41 2023","interfaces":[{"interface":"back","average":{"1min":3.0920000000000001,"5min":2.0419999999999998,"15min":1.1359999999999999},"min":{"1min":0.35199999999999998,"5min":0.20399999999999999,"15min":0.20399999999999999},"max":{"1min":6.665,"5min":12.795999999999999,"15min":12.795999999999999},"last":0.45900000000000002},{"interface":"front","average":{"1min":3.7250000000000001,"5min":2.2349999999999999,"15min":1.335},"min":{"1min":0.27000000000000002,"5min":0.24099999999999999,"15min":0.223},"max":{"1min":9.1099999999999994,"5min":16.219999999999999,"15min":16.219999999999999},"last":0.52300000000000002}]},{"osd":1,"last update":"Sat Sep 23 21:00:18 2023","interfaces":[{"interface":"back","average":{"1min":2.8759999999999999,"5min":1.8779999999999999,"15min":1.2529999999999999},"min":{"1min":0.26600000000000001,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":8.7720000000000002,"5min":8.7720000000000002,"15min":8.7720000000000002},"last":0.33500000000000002},{"interface":"front","average":{"1min":2.3639999999999999,"5min":1.9330000000000001,"15min":1.22},"min":{"1min":0.34200000000000003,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":8.5109999999999992,"5min":10.285,"15min":10.285},"last":0.55800000000000005}]},{"osd":2,"last update":"Sat Sep 23 21:00:59 2023","interfaces":[{"interface":"back","average":{"1min":2.4279999999999999,"5min":1.837,"15min":1.246},"min":{"1min":0.26700000000000002,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":6.7039999999999997,"5min":10.579000000000001,"15min":12.651},"last":0.37},{"interface":"front","average":{"1min":2.3450000000000002,"5min":2.2280000000000002,"15min":1.391},"min":{"1min":0.28599999999999998,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":7.5759999999999996,"5min":12.300000000000001,"15min":12.300000000000001},"last":0.39100000000000001}]},{"osd":3,"last update":"Sat Sep 23 21:00:18 2023","interfaces":[{"interface":"back","average":{"1min":4.931,"5min":2.859,"15min":1.6379999999999999},"min":{"1min":0.35999999999999999,"5min":0.35399999999999998,"15min":0.28599999999999998},"max":{"1min":20.788,"5min":20.788,"15min":20.788},"last":0.499},{"interface":"front","average":{"1min":3.9870000000000001,"5min":2.177,"15min":1.2949999999999999},"min":{"1min":0.39100000000000001,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":20.539000000000001,"5min":20.539000000000001,"15min":20.539000000000001},"last":0.47999999999999998}]},{"osd":4,"last update":"Sat Sep 23 21:00:35 2023","interfaces":[{"interface":"back","average":{"1min":0.69999999999999996,"5min":0.625,"15min":0.56899999999999995},"min":{"1min":0.23400000000000001,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":4.1520000000000001,"5min":4.1520000000000001,"15min":4.1520000000000001},"last":0.54100000000000004},{"interface":"front","average":{"1min":0.79600000000000004,"5min":0.69799999999999995,"15min":0.59599999999999997},"min":{"1min":0.28999999999999998,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":3.7469999999999999,"5min":3.7469999999999999,"15min":3.7469999999999999},"last":0.42099999999999999}]},{"osd":6,"last update":"Sat Sep 23 21:00:41 2023","interfaces":[{"interface":"back","average":{"1min":0.91600000000000004,"5min":0.72199999999999998,"15min":0.61399999999999999},"min":{"1min":0.314,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":4.359,"5min":4.359,"15min":4.359},"last":0.44700000000000001},{"interface":"front","average":{"1min":0.92200000000000004,"5min":0.79100000000000004,"15min":0.75900000000000001},"min":{"1min":0.36899999999999999,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":3.847,"5min":3.847,"15min":3.847},"last":0.59299999999999997}]},{"osd":7,"last update":"Sat Sep 23 21:00:18 2023","interfaces":[{"interface":"back","average":{"1min":1.0189999999999999,"5min":0.81000000000000005,"15min":0.67700000000000005},"min":{"1min":0.35499999999999998,"5min":0.307,"15min":0.30199999999999999},"max":{"1min":4.4240000000000004,"5min":4.4240000000000004,"15min":8.5649999999999995},"last":0.48999999999999999},{"interface":"front","average":{"1min":0.752,"5min":0.71899999999999997,"15min":0.67400000000000004},"min":{"1min":0.373,"5min":0.29799999999999999,"15min":0.27700000000000002},"max":{"1min":2.1789999999999998,"5min":3.8149999999999999,"15min":4.5430000000000001},"last":0.57499999999999996}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T21:01:20.896 INFO:tasks.cephadm.ceph_manager.ceph:recovered! 2023-09-23T21:01:20.896 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T21:01:20.989 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:01:20.989 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T21:01:21.020 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:01:21.055 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:01:21.058 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:01:21.058 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:01:21.159 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:01:21.187 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:01:21.220 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:01:21.223 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:01:21.223 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:01:21.324 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:01:21.352 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:01:21.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:01:21 smithi070 ceph-mon[168282]: from='client.45470 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T21:01:21.391 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:01:21.394 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:01:21.394 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:01:21.470 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:01:21.495 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:01:21.520 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:01:21 smithi026 ceph-mon[179688]: from='client.45470 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T21:01:21.521 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:01:21 smithi026 ceph-mon[175151]: from='client.45470 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T21:01:21.523 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:01:21.602 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:01:21.605 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:01:21.606 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:01:21.606 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:01:21.707 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:01:21.736 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:01:21.771 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:01:21.774 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:01:21.774 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:01:21.875 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:01:21.902 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:01:21.933 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:01:21.934 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:01:21.937 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:01:21.937 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:01:22.038 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:01:22.064 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:01:22.097 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:01:22.100 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:01:22.100 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:01:22.201 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:01:22.229 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:01:22 smithi026 ceph-mon[175151]: pgmap v659: 93 pgs: 93 active+clean; 582 KiB data, 2.8 GiB used, 712 GiB / 715 GiB avail 2023-09-23T21:01:22.229 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:01:22 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:01:22.229 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:01:22 smithi026 ceph-mon[179688]: pgmap v659: 93 pgs: 93 active+clean; 582 KiB data, 2.8 GiB used, 712 GiB / 715 GiB avail 2023-09-23T21:01:22.229 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:01:22 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:01:22.231 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:01:22.368 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:01:22.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:01:22 smithi070 ceph-mon[168282]: pgmap v659: 93 pgs: 93 active+clean; 582 KiB data, 2.8 GiB used, 712 GiB / 715 GiB avail 2023-09-23T21:01:22.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:01:22 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:01:22.371 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:01:22.371 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:01:22.472 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:01:22.497 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:01:22.529 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:01:22.531 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:01:22.532 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:01:22.633 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:01:22.679 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:01:22.961 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:01:22.964 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:01:22.965 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:01:23.066 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:01:23.094 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:01:23.127 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:01:23.129 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:01:23.130 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:01:23.230 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:01:23.253 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T21:01:23.262 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:01:23.414 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:01:23.417 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:01:23.417 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:01:23.518 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:01:23.548 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:01:23.607 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:01:23.609 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:01:23.610 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:01:23.711 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:01:23.742 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:01:23.879 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:01:23.882 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:01:23.883 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:01:23.939 INFO:teuthology.orchestra.run.smithi026.stdout:{ 2023-09-23T21:01:23.939 INFO:teuthology.orchestra.run.smithi026.stdout: "ops": [], 2023-09-23T21:01:23.939 INFO:teuthology.orchestra.run.smithi026.stdout: "complaint_time": 30, 2023-09-23T21:01:23.939 INFO:teuthology.orchestra.run.smithi026.stdout: "num_blocked_ops": 0 2023-09-23T21:01:23.940 INFO:teuthology.orchestra.run.smithi026.stdout:} 2023-09-23T21:01:23.984 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:01:24.017 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:01:24.054 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:01:24.056 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:01:24.057 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:01:24.157 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:01:24.189 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:01:24.277 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:01:24 smithi026 ceph-mon[175151]: pgmap v660: 93 pgs: 93 active+clean; 582 KiB data, 2.8 GiB used, 712 GiB / 715 GiB avail 2023-09-23T21:01:24.339 INFO:teuthology.orchestra.run.smithi026.stderr:pool 'unique_pool_2' created 2023-09-23T21:01:24.356 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:01:24.360 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:01:24.360 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:01:24.394 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_historic_ops 2023-09-23T21:01:24.461 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:01:24.493 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:01:24.530 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:01:24.532 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:01:24 smithi070 ceph-mon[168282]: pgmap v660: 93 pgs: 93 active+clean; 582 KiB data, 2.8 GiB used, 712 GiB / 715 GiB avail 2023-09-23T21:01:24.533 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:01:24.533 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:01:24.631 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:01:24 smithi026 ceph-mon[179688]: pgmap v660: 93 pgs: 93 active+clean; 582 KiB data, 2.8 GiB used, 712 GiB / 715 GiB avail 2023-09-23T21:01:24.634 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:01:24.662 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:01:24.697 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:01:24.700 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:01:24.700 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:01:24.786 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph osd pool application enable unique_pool_2 rados --yes-i-really-mean-it || true 2023-09-23T21:01:24.801 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:01:24.842 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:01:24.868 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:01:25.234 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T21:01:25.237 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:01:25.238 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:01:25.338 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:01:25.341 INFO:teuthology.orchestra.run.smithi026.stderr:nodeep-scrub is unset 2023-09-23T21:01:25.369 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:01:25.403 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:01:25.406 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:01:25.407 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:01:25.508 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:01:25.517 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:01:25 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/1036596041' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16}]: dispatch 2023-09-23T21:01:25.517 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:01:25 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16}]: dispatch 2023-09-23T21:01:25.518 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:01:25 smithi026 ceph-mon[175151]: failed to encode map e254 with expected crc 2023-09-23T21:01:25.518 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:01:25 smithi026 ceph-mon[175151]: failed to encode map e254 with expected crc 2023-09-23T21:01:25.518 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:01:25 smithi026 ceph-mon[175151]: failed to encode map e254 with expected crc 2023-09-23T21:01:25.518 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:01:25 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16}]': finished 2023-09-23T21:01:25.518 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:01:25 smithi026 ceph-mon[175151]: osdmap e254: 8 total, 8 up, 8 in 2023-09-23T21:01:25.518 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:01:25 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/1861015998' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:01:25.518 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:01:25 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:01:25.519 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:01:25 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/1036596041' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16}]: dispatch 2023-09-23T21:01:25.519 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:01:25 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16}]: dispatch 2023-09-23T21:01:25.519 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:01:25 smithi026 ceph-mon[179688]: failed to encode map e254 with expected crc 2023-09-23T21:01:25.519 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:01:25 smithi026 ceph-mon[179688]: failed to encode map e254 with expected crc 2023-09-23T21:01:25.519 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:01:25 smithi026 ceph-mon[179688]: failed to encode map e254 with expected crc 2023-09-23T21:01:25.519 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:01:25 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16}]': finished 2023-09-23T21:01:25.519 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:01:25 smithi026 ceph-mon[179688]: osdmap e254: 8 total, 8 up, 8 in 2023-09-23T21:01:25.520 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:01:25 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/1861015998' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:01:25.520 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:01:25 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:01:25.534 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:01:25 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/1036596041' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16}]: dispatch 2023-09-23T21:01:25.534 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:01:25 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16}]: dispatch 2023-09-23T21:01:25.535 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:01:25 smithi070 ceph-mon[168282]: failed to encode map e254 with expected crc 2023-09-23T21:01:25.535 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:01:25 smithi070 ceph-mon[168282]: failed to encode map e254 with expected crc 2023-09-23T21:01:25.535 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:01:25 smithi070 ceph-mon[168282]: failed to encode map e254 with expected crc 2023-09-23T21:01:25.535 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:01:25 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16}]': finished 2023-09-23T21:01:25.535 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:01:25 smithi070 ceph-mon[168282]: osdmap e254: 8 total, 8 up, 8 in 2023-09-23T21:01:25.535 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:01:25 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/1861015998' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:01:25.535 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:01:25 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:01:25.537 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:01:25.573 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:01:25.576 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:01:25.576 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:01:25.677 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:01:25.709 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:01:25.887 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:01:25.890 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:01:25.891 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:01:25.992 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:01:26.029 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:01:26.074 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:01:26.078 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:01:26.079 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:01:26.179 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:01:26.209 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:01:26.331 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:01:26.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:01:26 smithi070 ceph-mon[168282]: pgmap v662: 109 pgs: 4 creating+peering, 12 unknown, 93 active+clean; 582 KiB data, 2.8 GiB used, 712 GiB / 715 GiB avail 2023-09-23T21:01:26.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:01:26 smithi070 ceph-mon[168282]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T21:01:26.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:01:26 smithi070 ceph-mon[168282]: Cluster is now healthy 2023-09-23T21:01:26.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:01:26 smithi070 ceph-mon[168282]: failed to encode map e255 with expected crc 2023-09-23T21:01:26.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:01:26 smithi070 ceph-mon[168282]: failed to encode map e255 with expected crc 2023-09-23T21:01:26.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:01:26 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T21:01:26.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:01:26 smithi070 ceph-mon[168282]: osdmap e255: 8 total, 8 up, 8 in 2023-09-23T21:01:26.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:01:26 smithi070 ceph-mon[168282]: failed to encode map e255 with expected crc 2023-09-23T21:01:26.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:01:26 smithi070 ceph-mon[168282]: failed to encode map e255 with expected crc 2023-09-23T21:01:26.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:01:26 smithi070 ceph-mon[168282]: failed to encode map e255 with expected crc 2023-09-23T21:01:26.631 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:01:26 smithi026 ceph-mon[175151]: pgmap v662: 109 pgs: 4 creating+peering, 12 unknown, 93 active+clean; 582 KiB data, 2.8 GiB used, 712 GiB / 715 GiB avail 2023-09-23T21:01:26.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:01:26 smithi026 ceph-mon[175151]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T21:01:26.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:01:26 smithi026 ceph-mon[175151]: Cluster is now healthy 2023-09-23T21:01:26.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:01:26 smithi026 ceph-mon[175151]: failed to encode map e255 with expected crc 2023-09-23T21:01:26.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:01:26 smithi026 ceph-mon[175151]: failed to encode map e255 with expected crc 2023-09-23T21:01:26.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:01:26 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T21:01:26.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:01:26 smithi026 ceph-mon[175151]: osdmap e255: 8 total, 8 up, 8 in 2023-09-23T21:01:26.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:01:26 smithi026 ceph-mon[175151]: failed to encode map e255 with expected crc 2023-09-23T21:01:26.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:01:26 smithi026 ceph-mon[175151]: failed to encode map e255 with expected crc 2023-09-23T21:01:26.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:01:26 smithi026 ceph-mon[175151]: failed to encode map e255 with expected crc 2023-09-23T21:01:26.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:01:26 smithi026 ceph-mon[179688]: pgmap v662: 109 pgs: 4 creating+peering, 12 unknown, 93 active+clean; 582 KiB data, 2.8 GiB used, 712 GiB / 715 GiB avail 2023-09-23T21:01:26.633 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:01:26 smithi026 ceph-mon[179688]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T21:01:26.633 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:01:26 smithi026 ceph-mon[179688]: Cluster is now healthy 2023-09-23T21:01:26.633 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:01:26 smithi026 ceph-mon[179688]: failed to encode map e255 with expected crc 2023-09-23T21:01:26.633 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:01:26 smithi026 ceph-mon[179688]: failed to encode map e255 with expected crc 2023-09-23T21:01:26.633 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:01:26 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T21:01:26.633 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:01:26 smithi026 ceph-mon[179688]: osdmap e255: 8 total, 8 up, 8 in 2023-09-23T21:01:26.633 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:01:26 smithi026 ceph-mon[179688]: failed to encode map e255 with expected crc 2023-09-23T21:01:26.633 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:01:26 smithi026 ceph-mon[179688]: failed to encode map e255 with expected crc 2023-09-23T21:01:26.633 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:01:26 smithi026 ceph-mon[179688]: failed to encode map e255 with expected crc 2023-09-23T21:01:26.710 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0: {} 2023-09-23T21:01:26.729 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1: {} 2023-09-23T21:01:26.745 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T21:01:26.748 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:01:26.748 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:01:26.750 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2: {} 2023-09-23T21:01:26.770 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3: {} 2023-09-23T21:01:26.789 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4: {} 2023-09-23T21:01:26.808 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T21:01:26.828 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6: {} 2023-09-23T21:01:26.848 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T21:01:26.849 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:01:26.850 INFO:teuthology.orchestra.run.smithi026.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T21:01:26.850 INFO:teuthology.orchestra.run.smithi026.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T21:01:26.850 INFO:teuthology.orchestra.run.smithi026.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T21:01:26.850 INFO:teuthology.orchestra.run.smithi026.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T21:01:26.850 INFO:teuthology.orchestra.run.smithi026.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T21:01:26.850 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T21:01:26.850 INFO:teuthology.orchestra.run.smithi026.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T21:01:26.850 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T21:01:26.882 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:01:26.940 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:01:27.256 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T21:01:27.258 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:01:27.259 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:01:27.360 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:01:27.391 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:01:27.428 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:01:27.430 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:01:27.431 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:01:27.532 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:01:27.543 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:01:27 smithi026 ceph-mon[175151]: failed to encode map e256 with expected crc 2023-09-23T21:01:27.543 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:01:27 smithi026 ceph-mon[175151]: failed to encode map e256 with expected crc 2023-09-23T21:01:27.543 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:01:27 smithi026 ceph-mon[175151]: osdmap e256: 8 total, 8 up, 8 in 2023-09-23T21:01:27.544 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:01:27 smithi026 ceph-mon[175151]: pgmap v665: 109 pgs: 7 creating+peering, 9 unknown, 93 active+clean; 582 KiB data, 2.8 GiB used, 712 GiB / 715 GiB avail 2023-09-23T21:01:27.544 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:01:27 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/749557103' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:01:27.544 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:01:27 smithi026 ceph-mon[175151]: failed to encode map e256 with expected crc 2023-09-23T21:01:27.544 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:01:27 smithi026 ceph-mon[175151]: failed to encode map e256 with expected crc 2023-09-23T21:01:27.544 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:01:27 smithi026 ceph-mon[179688]: failed to encode map e256 with expected crc 2023-09-23T21:01:27.544 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:01:27 smithi026 ceph-mon[179688]: failed to encode map e256 with expected crc 2023-09-23T21:01:27.544 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:01:27 smithi026 ceph-mon[179688]: osdmap e256: 8 total, 8 up, 8 in 2023-09-23T21:01:27.545 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:01:27 smithi026 ceph-mon[179688]: pgmap v665: 109 pgs: 7 creating+peering, 9 unknown, 93 active+clean; 582 KiB data, 2.8 GiB used, 712 GiB / 715 GiB avail 2023-09-23T21:01:27.545 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:01:27 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/749557103' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:01:27.545 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:01:27 smithi026 ceph-mon[179688]: failed to encode map e256 with expected crc 2023-09-23T21:01:27.545 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:01:27 smithi026 ceph-mon[179688]: failed to encode map e256 with expected crc 2023-09-23T21:01:27.561 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:01:27.594 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:01:27.596 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:01:27 smithi070 ceph-mon[168282]: failed to encode map e256 with expected crc 2023-09-23T21:01:27.597 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:01:27 smithi070 ceph-mon[168282]: failed to encode map e256 with expected crc 2023-09-23T21:01:27.597 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:01:27 smithi070 ceph-mon[168282]: osdmap e256: 8 total, 8 up, 8 in 2023-09-23T21:01:27.597 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:01:27 smithi070 ceph-mon[168282]: pgmap v665: 109 pgs: 7 creating+peering, 9 unknown, 93 active+clean; 582 KiB data, 2.8 GiB used, 712 GiB / 715 GiB avail 2023-09-23T21:01:27.597 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:01:27 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/749557103' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:01:27.597 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:01:27 smithi070 ceph-mon[168282]: failed to encode map e256 with expected crc 2023-09-23T21:01:27.597 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:01:27 smithi070 ceph-mon[168282]: failed to encode map e256 with expected crc 2023-09-23T21:01:27.598 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:01:27.598 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:01:27.699 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:01:27.727 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:01:27.762 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:01:27.764 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:01:27.765 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:01:27.866 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:01:27.893 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:01:27.927 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:01:27.930 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:01:27.930 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:01:28.031 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:01:28.062 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:01:28.361 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:01:28.364 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:01:28.364 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:01:28.465 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:01:28.494 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:01:28.548 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:01:28.552 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:01:28.552 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:01:28.653 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:01:28.684 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:01:28.778 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:01:28.782 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:01:28.782 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:01:28.883 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:01:28.912 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:01:28.950 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:01:28.953 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:01:28.953 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:01:28.955 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T21:01:29.055 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:01:29.084 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:01:29.117 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:01:29.120 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:01:29.121 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:01:29.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:01:28 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:01:29.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:01:28 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:01:29.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:01:28 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T21:01:29.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:01:28 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T21:01:29.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:01:28 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:01:29.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:01:28 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:01:29.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:01:28 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:01:29.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:01:28 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T21:01:29.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:01:28 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T21:01:29.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:01:28 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:01:29.139 INFO:teuthology.orchestra.run.smithi026.stdout:ERROR: (22) Invalid argument 2023-09-23T21:01:29.139 INFO:teuthology.orchestra.run.smithi026.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T21:01:29.222 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:01:29.246 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:01:28 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:01:29.246 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:01:28 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:01:29.246 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:01:28 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T21:01:29.246 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:01:28 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T21:01:29.247 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:01:28 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:01:29.249 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:01:29.281 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:01:29.284 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:01:29.285 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:01:29.385 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:01:29.396 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T21:01:29.412 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:01:29.443 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:01:29.446 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:01:29.446 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:01:29.547 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:01:29.573 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:01:29.577 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_ops_in_flight 2023-09-23T21:01:29.605 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:01:29.608 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:01:29.608 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:01:29.709 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:01:29.737 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 21:01:29 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[164029]: ::ffff:172.21.15.70 - - [23/Sep/2023:21:01:29] "GET /metrics HTTP/1.1" 200 36296 "" "Prometheus/2.43.0" 2023-09-23T21:01:29.741 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:01:29.771 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:01:29.903 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:01:29.906 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:01:29.906 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:01:29.945 INFO:teuthology.orchestra.run.smithi026.stderr:enabled application 'rados' on pool 'unique_pool_2' 2023-09-23T21:01:30.007 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:01:30.014 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:01:29 smithi026 ceph-mon[179688]: pgmap v666: 109 pgs: 7 creating+peering, 2 unknown, 100 active+clean; 582 KiB data, 2.7 GiB used, 712 GiB / 715 GiB avail 2023-09-23T21:01:30.014 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:01:29 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/1485332930' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T21:01:30.014 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:01:29 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T21:01:30.014 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:01:29 smithi026 ceph-mon[175151]: pgmap v666: 109 pgs: 7 creating+peering, 2 unknown, 100 active+clean; 582 KiB data, 2.7 GiB used, 712 GiB / 715 GiB avail 2023-09-23T21:01:30.015 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:01:29 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/1485332930' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T21:01:30.015 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:01:29 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T21:01:30.040 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:01:30.077 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:01:30.079 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:01:30.079 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:01:30.180 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:01:30.206 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:01:29 smithi070 ceph-mon[168282]: pgmap v666: 109 pgs: 7 creating+peering, 2 unknown, 100 active+clean; 582 KiB data, 2.7 GiB used, 712 GiB / 715 GiB avail 2023-09-23T21:01:30.206 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:01:29 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/1485332930' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T21:01:30.207 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:01:29 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T21:01:30.209 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:01:30.224 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:01:30.242 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:01:30.245 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:01:30.246 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:01:30.346 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:01:30.374 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:01:30.408 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:01:30.410 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:01:30.411 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:01:30.512 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:01:30.541 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:01:30.573 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:01:30.575 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:01:30.576 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:01:30.676 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:01:30.709 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:01:30.743 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:01:30.745 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:01:30.745 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:01:30.788 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:01:30.846 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:01:30.878 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:01:30.912 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:01:30.914 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:01:30.915 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:01:31.015 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:01:31.048 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:01:31.206 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:01:30 smithi026 ceph-mon[179688]: failed to encode map e257 with expected crc 2023-09-23T21:01:31.207 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:01:30 smithi026 ceph-mon[179688]: failed to encode map e257 with expected crc 2023-09-23T21:01:31.207 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:01:30 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-09-23T21:01:31.207 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:01:30 smithi026 ceph-mon[179688]: osdmap e257: 8 total, 8 up, 8 in 2023-09-23T21:01:31.207 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:01:30 smithi026 ceph-mon[179688]: failed to encode map e257 with expected crc 2023-09-23T21:01:31.207 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:01:30 smithi026 ceph-mon[175151]: failed to encode map e257 with expected crc 2023-09-23T21:01:31.207 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:01:30 smithi026 ceph-mon[175151]: failed to encode map e257 with expected crc 2023-09-23T21:01:31.207 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:01:30 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-09-23T21:01:31.208 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:01:30 smithi026 ceph-mon[175151]: osdmap e257: 8 total, 8 up, 8 in 2023-09-23T21:01:31.208 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:01:30 smithi026 ceph-mon[175151]: failed to encode map e257 with expected crc 2023-09-23T21:01:31.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:01:30 smithi070 ceph-mon[168282]: failed to encode map e257 with expected crc 2023-09-23T21:01:31.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:01:30 smithi070 ceph-mon[168282]: failed to encode map e257 with expected crc 2023-09-23T21:01:31.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:01:30 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-09-23T21:01:31.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:01:30 smithi070 ceph-mon[168282]: osdmap e257: 8 total, 8 up, 8 in 2023-09-23T21:01:31.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:01:30 smithi070 ceph-mon[168282]: failed to encode map e257 with expected crc 2023-09-23T21:01:31.524 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:01:31.529 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:01:31.529 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:01:31.618 DEBUG:teuthology.orchestra.run.smithi026:> /bin/sh -c 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados --no-log-to-stderr --name client.0 -b 65536 --object-size 65536 -p unique_pool_2 bench 90 write' 2023-09-23T21:01:31.618 INFO:tasks.thrashosds.thrasher:fixing pg num pool .mgr 2023-09-23T21:01:31.619 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T21:01:31.621 INFO:tasks.radosbench:joining radosbench (timing out after 3000s) 2023-09-23T21:01:31.621 INFO:teuthology.orchestra.run:waiting for 3000 2023-09-23T21:01:31.630 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:01:31.695 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:01:31.727 INFO:tasks.radosbench.radosbench.0.smithi026.stdout:hints = 1 2023-09-23T21:01:31.727 INFO:tasks.radosbench.radosbench.0.smithi026.stdout:Maintaining 16 concurrent writes of 65536 bytes to objects of size 65536 for up to 90 seconds or 0 objects 2023-09-23T21:01:31.727 INFO:tasks.radosbench.radosbench.0.smithi026.stdout:Object prefix: benchmark_data_smithi026_301269 2023-09-23T21:01:31.728 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T21:01:31.728 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 0 0 0 0 0 0 - 0 2023-09-23T21:01:31.749 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:01:31.758 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:01:31.759 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:01:31.860 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:01:31.968 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:01:32.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:01:31 smithi026 ceph-mon[175151]: failed to encode map e257 with expected crc 2023-09-23T21:01:32.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:01:31 smithi026 ceph-mon[175151]: pgmap v668: 109 pgs: 109 active+clean; 582 KiB data, 2.7 GiB used, 712 GiB / 715 GiB avail 2023-09-23T21:01:32.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:01:31 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:01:32.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:01:31 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:01:32.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:01:31 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/194618689' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T21:01:32.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:01:31 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T21:01:32.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:01:31 smithi026 ceph-mon[179688]: failed to encode map e257 with expected crc 2023-09-23T21:01:32.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:01:31 smithi026 ceph-mon[179688]: pgmap v668: 109 pgs: 109 active+clean; 582 KiB data, 2.7 GiB used, 712 GiB / 715 GiB avail 2023-09-23T21:01:32.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:01:31 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:01:32.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:01:31 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:01:32.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:01:31 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/194618689' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T21:01:32.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:01:31 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T21:01:32.205 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:01:32.216 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:01:32.217 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:01:32.228 INFO:teuthology.orchestra.run.smithi026.stderr:noscrub is set 2023-09-23T21:01:32.318 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:01:32.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:01:31 smithi070 ceph-mon[168282]: failed to encode map e257 with expected crc 2023-09-23T21:01:32.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:01:31 smithi070 ceph-mon[168282]: pgmap v668: 109 pgs: 109 active+clean; 582 KiB data, 2.7 GiB used, 712 GiB / 715 GiB avail 2023-09-23T21:01:32.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:01:31 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:01:32.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:01:31 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:01:32.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:01:31 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/194618689' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T21:01:32.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:01:31 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T21:01:32.390 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:01:32.728 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 1 16 2965 2949 184.302 184.312 0.00355271 0.00541213 2023-09-23T21:01:32.794 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:01:32.798 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:01:32.799 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:01:32.899 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:01:32.938 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:01:32.983 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:01:32.987 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:01:32.988 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:01:33.089 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:01:33.120 INFO:teuthology.orchestra.run.smithi026.stdout:ERROR: (22) Invalid argument 2023-09-23T21:01:33.120 INFO:teuthology.orchestra.run.smithi026.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T21:01:33.123 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:01:33.171 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:01:33.175 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:01:33.175 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:01:33.189 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:01:33.276 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:01:33.288 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:01:32 smithi026 ceph-mon[175151]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:01:33.288 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:01:32 smithi026 ceph-mon[175151]: failed to encode map e258 with expected crc 2023-09-23T21:01:33.288 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:01:32 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T21:01:33.288 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:01:32 smithi026 ceph-mon[175151]: osdmap e258: 8 total, 8 up, 8 in 2023-09-23T21:01:33.288 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:01:32 smithi026 ceph-mon[179688]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:01:33.288 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:01:32 smithi026 ceph-mon[179688]: failed to encode map e258 with expected crc 2023-09-23T21:01:33.288 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:01:32 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T21:01:33.289 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:01:32 smithi026 ceph-mon[179688]: osdmap e258: 8 total, 8 up, 8 in 2023-09-23T21:01:33.309 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:01:32 smithi070 ceph-mon[168282]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:01:33.309 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:01:32 smithi070 ceph-mon[168282]: failed to encode map e258 with expected crc 2023-09-23T21:01:33.309 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:01:32 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T21:01:33.309 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:01:32 smithi070 ceph-mon[168282]: osdmap e258: 8 total, 8 up, 8 in 2023-09-23T21:01:33.312 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:01:33.315 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0: {} 2023-09-23T21:01:33.358 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1: {} 2023-09-23T21:01:33.360 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:01:33.364 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:01:33.364 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:01:33.398 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2: {} 2023-09-23T21:01:33.439 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3: {} 2023-09-23T21:01:33.465 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:01:33.483 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4: {} 2023-09-23T21:01:33.499 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:01:33.508 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T21:01:33.548 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:01:33.550 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6: {} 2023-09-23T21:01:33.552 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:01:33.553 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:01:33.590 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T21:01:33.608 INFO:teuthology.orchestra.run.smithi026.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T21:01:33.608 INFO:teuthology.orchestra.run.smithi026.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T21:01:33.608 INFO:teuthology.orchestra.run.smithi026.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T21:01:33.608 INFO:teuthology.orchestra.run.smithi026.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T21:01:33.608 INFO:teuthology.orchestra.run.smithi026.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T21:01:33.608 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T21:01:33.608 INFO:teuthology.orchestra.run.smithi026.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T21:01:33.608 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T21:01:33.642 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_blocked_ops 2023-09-23T21:01:33.654 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:01:33.728 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 2 16 5755 5739 179.33 174.375 0.0121133 0.00555796 2023-09-23T21:01:33.746 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:01:34.051 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:01:34.058 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:01:34.059 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:01:34.160 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:01:34.197 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:01:34.213 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:01:33 smithi026 ceph-mon[175151]: failed to encode map e258 with expected crc 2023-09-23T21:01:34.213 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:01:33 smithi026 ceph-mon[175151]: pgmap v670: 109 pgs: 109 active+clean; 24 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 3.7 MiB/s wr, 59 op/s 2023-09-23T21:01:34.213 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:01:33 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/2833916346' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:01:34.213 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:01:33 smithi026 ceph-mon[179688]: failed to encode map e258 with expected crc 2023-09-23T21:01:34.213 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:01:33 smithi026 ceph-mon[179688]: pgmap v670: 109 pgs: 109 active+clean; 24 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 3.7 MiB/s wr, 59 op/s 2023-09-23T21:01:34.213 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:01:33 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/2833916346' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:01:34.252 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:01:34.255 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:01:33 smithi070 ceph-mon[168282]: failed to encode map e258 with expected crc 2023-09-23T21:01:34.256 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:01:33 smithi070 ceph-mon[168282]: pgmap v670: 109 pgs: 109 active+clean; 24 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 3.7 MiB/s wr, 59 op/s 2023-09-23T21:01:34.256 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:01:33 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/2833916346' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:01:34.256 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:01:34.257 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:01:34.357 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:01:34.412 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:01:34.483 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:01:34.487 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:01:34.487 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:01:34.588 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:01:34.674 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:01:34.728 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 3 16 8460 8444 175.899 169.062 0.00509471 0.00567761 2023-09-23T21:01:34.763 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T21:01:34.768 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:01:34.769 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:01:34.869 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:01:34.903 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:01:34.948 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:01:34.952 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:01:34.952 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:01:35.053 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:01:35.069 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T21:01:35.092 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:01:35.115 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:01:35.132 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:01:35.135 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:01:35.135 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:01:35.236 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:01:35.271 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:01:35.317 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:01:35.320 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:01:35.321 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:01:35.422 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:01:35.454 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:01:35.497 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:01:35.500 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:01:35.500 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:01:35.601 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:01:35.625 INFO:teuthology.orchestra.run.smithi026.stdout: 2023-09-23T21:01:35.633 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:01:35.637 INFO:teuthology.orchestra.run.smithi026.stderr:dumped all 2023-09-23T21:01:35.676 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:01:35.679 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:01:35.679 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:01:35.728 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 4 16 10812 10796 168.671 147 0.00942391 0.00591771 2023-09-23T21:01:35.779 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:01:35.818 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:01:35.872 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:01:35.876 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:01:35.876 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:01:35.977 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:01:36.013 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:01:36.069 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:01:36.072 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:01:36.072 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:01:36.109 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:01:36 smithi026 ceph-mon[175151]: pgmap v671: 109 pgs: 109 active+clean; 195 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 24 MiB/s wr, 388 op/s 2023-09-23T21:01:36.173 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:01:36.217 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:01:36.264 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:01:36.267 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:01:36 smithi070 ceph-mon[168282]: pgmap v671: 109 pgs: 109 active+clean; 195 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 24 MiB/s wr, 388 op/s 2023-09-23T21:01:36.268 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:01:36.268 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:01:36.369 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:01:36.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:01:36 smithi026 ceph-mon[179688]: pgmap v671: 109 pgs: 109 active+clean; 195 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 24 MiB/s wr, 388 op/s 2023-09-23T21:01:36.409 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:01:36.463 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:01:36.472 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:01:36.472 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:01:36.535 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T21:01:36.573 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:01:36.609 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:01:36.656 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:01:36.659 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:01:36.659 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:01:36.729 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 5 16 13537 13521 168.978 170.312 0.007006 0.0059107 2023-09-23T21:01:36.760 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:01:36.802 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:01:36.855 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:01:36.858 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:01:36.858 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:01:36.959 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:01:37.002 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:01:37.044 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:01:37.047 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:01:37.047 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:01:37.148 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:01:37.351 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:01:37.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:01:37 smithi070 ceph-mon[168282]: from='client.55366 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T21:01:37.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:01:37 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:01:37.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:01:37 smithi026 ceph-mon[175151]: from='client.55366 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T21:01:37.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:01:37 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:01:37.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:01:37 smithi026 ceph-mon[179688]: from='client.55366 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T21:01:37.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:01:37 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:01:37.729 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 6 16 16219 16203 168.744 167.625 0.00548522 0.00591977 2023-09-23T21:01:38.381 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:01:38 smithi026 ceph-mon[179688]: pgmap v672: 109 pgs: 109 active+clean; 296 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 37 MiB/s wr, 590 op/s 2023-09-23T21:01:38.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:01:38 smithi026 ceph-mon[175151]: pgmap v672: 109 pgs: 109 active+clean; 296 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 37 MiB/s wr, 590 op/s 2023-09-23T21:01:38.414 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:01:38.419 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:01:38.419 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:01:38.520 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:01:38.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:01:38 smithi070 ceph-mon[168282]: pgmap v672: 109 pgs: 109 active+clean; 296 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 37 MiB/s wr, 590 op/s 2023-09-23T21:01:38.729 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 7 16 18773 18757 167.439 159.625 0.0141949 0.00596484 2023-09-23T21:01:38.732 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:01:39.099 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:01:39.137 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:01:39.163 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:01:39.164 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:01:39.165 INFO:teuthology.orchestra.run.smithi026.stdout:{"pg_ready":true,"pg_map":{"version":671,"stamp":"2023-09-23T21:01:34.513563+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":204281760,"num_objects":3132,"num_object_clones":0,"num_object_copies":9396,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3132,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3169,"num_write_kb":198941,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":22,"num_bytes_recovered":5990,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6544,"ondisk_log_size":6544,"up":327,"acting":327,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":330,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":3115540,"kb_used_data":1221972,"kb_used_omap":136,"kb_used_meta":1893367,"kb_avail":746845676,"statfs":{"total":767960285184,"available":764769972224,"internally_reserved":0,"allocated":1251299328,"data_stored":1242787677,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":139741,"internal_metadata":1938808355},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":203685888,"num_objects":3108,"num_object_clones":0,"num_object_copies":9324,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3108,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3108,"num_write_kb":198912,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"8.002445"},"pg_stats":[{"pgid":"3.1a","version":"68'1","reported_seq":135,"reported_epoch":257,"state":"active+clean","last_fresh":"2023-09-23T21:01:30.015317+0000","last_change":"2023-09-23T20:54:13.946695+0000","last_active":"2023-09-23T21:01:30.015317+0000","last_peered":"2023-09-23T21:01:30.015317+0000","last_clean":"2023-09-23T21:01:30.015317+0000","last_became_active":"2023-09-23T20:54:13.945399+0000","last_became_peered":"2023-09-23T20:54:13.945399+0000","last_unstale":"2023-09-23T21:01:30.015317+0000","last_undegraded":"2023-09-23T21:01:30.015317+0000","last_fullsized":"2023-09-23T21:01:30.015317+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:32:52.365464+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":249,"reported_epoch":255,"state":"active+clean","last_fresh":"2023-09-23T21:01:25.332452+0000","last_change":"2023-09-23T20:54:13.942019+0000","last_active":"2023-09-23T21:01:25.332452+0000","last_peered":"2023-09-23T21:01:25.332452+0000","last_clean":"2023-09-23T21:01:25.332452+0000","last_became_active":"2023-09-23T20:54:13.941647+0000","last_became_peered":"2023-09-23T20:54:13.941647+0000","last_unstale":"2023-09-23T21:01:25.332452+0000","last_undegraded":"2023-09-23T21:01:25.332452+0000","last_fullsized":"2023-09-23T21:01:25.332452+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:30:54.400944+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":228,"reported_epoch":257,"state":"active+clean","last_fresh":"2023-09-23T21:01:30.015299+0000","last_change":"2023-09-23T20:54:13.941394+0000","last_active":"2023-09-23T21:01:30.015299+0000","last_peered":"2023-09-23T21:01:30.015299+0000","last_clean":"2023-09-23T21:01:30.015299+0000","last_became_active":"2023-09-23T20:54:13.941042+0000","last_became_peered":"2023-09-23T20:54:13.941042+0000","last_unstale":"2023-09-23T21:01:30.015299+0000","last_undegraded":"2023-09-23T21:01:30.015299+0000","last_fullsized":"2023-09-23T21:01:30.015299+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:19:41.401184+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":243,"reported_epoch":257,"state":"active+clean","last_fresh":"2023-09-23T21:01:29.941655+0000","last_change":"2023-09-23T20:57:44.979811+0000","last_active":"2023-09-23T21:01:29.941655+0000","last_peered":"2023-09-23T21:01:29.941655+0000","last_clean":"2023-09-23T21:01:29.941655+0000","last_became_active":"2023-09-23T20:57:44.979515+0000","last_became_peered":"2023-09-23T20:57:44.979515+0000","last_unstale":"2023-09-23T21:01:29.941655+0000","last_undegraded":"2023-09-23T21:01:29.941655+0000","last_fullsized":"2023-09-23T21:01:29.941655+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:38:22.162140+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":240,"reported_epoch":255,"state":"active+clean","last_fresh":"2023-09-23T21:01:25.332346+0000","last_change":"2023-09-23T20:51:10.833613+0000","last_active":"2023-09-23T21:01:25.332346+0000","last_peered":"2023-09-23T21:01:25.332346+0000","last_clean":"2023-09-23T21:01:25.332346+0000","last_became_active":"2023-09-23T20:51:10.833208+0000","last_became_peered":"2023-09-23T20:51:10.833208+0000","last_unstale":"2023-09-23T21:01:25.332346+0000","last_undegraded":"2023-09-23T21:01:25.332346+0000","last_fullsized":"2023-09-23T21:01:25.332346+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:51:28.246014+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":223,"reported_epoch":257,"state":"active+clean","last_fresh":"2023-09-23T21:01:29.942744+0000","last_change":"2023-09-23T20:55:35.958384+0000","last_active":"2023-09-23T21:01:29.942744+0000","last_peered":"2023-09-23T21:01:29.942744+0000","last_clean":"2023-09-23T21:01:29.942744+0000","last_became_active":"2023-09-23T20:55:35.954563+0000","last_became_peered":"2023-09-23T20:55:35.954563+0000","last_unstale":"2023-09-23T21:01:29.942744+0000","last_undegraded":"2023-09-23T21:01:29.942744+0000","last_fullsized":"2023-09-23T21:01:29.942744+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:37:48.332259+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":232,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-09-23T21:01:26.739378+0000","last_change":"2023-09-23T20:54:31.885514+0000","last_active":"2023-09-23T21:01:26.739378+0000","last_peered":"2023-09-23T21:01:26.739378+0000","last_clean":"2023-09-23T21:01:26.739378+0000","last_became_active":"2023-09-23T20:54:31.884986+0000","last_became_peered":"2023-09-23T20:54:31.884986+0000","last_unstale":"2023-09-23T21:01:26.739378+0000","last_undegraded":"2023-09-23T21:01:26.739378+0000","last_fullsized":"2023-09-23T21:01:26.739378+0000","mapping_epoch":136,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:26:08.100220+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":237,"reported_epoch":255,"state":"active+clean","last_fresh":"2023-09-23T21:01:25.332438+0000","last_change":"2023-09-23T20:55:35.951889+0000","last_active":"2023-09-23T21:01:25.332438+0000","last_peered":"2023-09-23T21:01:25.332438+0000","last_clean":"2023-09-23T21:01:25.332438+0000","last_became_active":"2023-09-23T20:55:35.951563+0000","last_became_peered":"2023-09-23T20:55:35.951563+0000","last_unstale":"2023-09-23T21:01:25.332438+0000","last_undegraded":"2023-09-23T21:01:25.332438+0000","last_fullsized":"2023-09-23T21:01:25.332438+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:41:53.098781+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"79'161","reported_seq":544,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-09-23T21:01:26.738572+0000","last_change":"2023-09-23T20:57:25.096559+0000","last_active":"2023-09-23T21:01:26.738572+0000","last_peered":"2023-09-23T21:01:26.738572+0000","last_clean":"2023-09-23T21:01:26.738572+0000","last_became_active":"2023-09-23T20:57:15.975674+0000","last_became_peered":"2023-09-23T20:57:15.975674+0000","last_unstale":"2023-09-23T21:01:26.738572+0000","last_undegraded":"2023-09-23T21:01:26.738572+0000","last_fullsized":"2023-09-23T21:01:26.738572+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:25.096401+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:25.096401+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:05:05.448404+0000","scrub_duration":0.002859393,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":130,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-09-23T21:01:26.742563+0000","last_change":"2023-09-23T20:54:31.890678+0000","last_active":"2023-09-23T21:01:26.742563+0000","last_peered":"2023-09-23T21:01:26.742563+0000","last_clean":"2023-09-23T21:01:26.742563+0000","last_became_active":"2023-09-23T20:54:31.890469+0000","last_became_peered":"2023-09-23T20:54:31.890469+0000","last_unstale":"2023-09-23T21:01:26.742563+0000","last_undegraded":"2023-09-23T21:01:26.742563+0000","last_fullsized":"2023-09-23T21:01:26.742563+0000","mapping_epoch":136,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:55:08.790146+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":198,"reported_epoch":255,"state":"active+clean","last_fresh":"2023-09-23T21:01:25.332273+0000","last_change":"2023-09-23T20:55:35.952661+0000","last_active":"2023-09-23T21:01:25.332273+0000","last_peered":"2023-09-23T21:01:25.332273+0000","last_clean":"2023-09-23T21:01:25.332273+0000","last_became_active":"2023-09-23T20:55:35.952349+0000","last_became_peered":"2023-09-23T20:55:35.952349+0000","last_unstale":"2023-09-23T21:01:25.332273+0000","last_undegraded":"2023-09-23T21:01:25.332273+0000","last_fullsized":"2023-09-23T21:01:25.332273+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:00:03.110126+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":74,"reported_epoch":255,"state":"active+clean","last_fresh":"2023-09-23T21:01:25.332545+0000","last_change":"2023-09-23T20:56:45.988607+0000","last_active":"2023-09-23T21:01:25.332545+0000","last_peered":"2023-09-23T21:01:25.332545+0000","last_clean":"2023-09-23T21:01:25.332545+0000","last_became_active":"2023-09-23T20:56:45.988206+0000","last_became_peered":"2023-09-23T20:56:45.988206+0000","last_unstale":"2023-09-23T21:01:25.332545+0000","last_undegraded":"2023-09-23T21:01:25.332545+0000","last_fullsized":"2023-09-23T21:01:25.332545+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":205,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:01:19.783688+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,4],"acting":[3,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":74,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-09-23T21:01:26.742397+0000","last_change":"2023-09-23T20:56:18.139142+0000","last_active":"2023-09-23T21:01:26.742397+0000","last_peered":"2023-09-23T21:01:26.742397+0000","last_clean":"2023-09-23T21:01:26.742397+0000","last_became_active":"2023-09-23T20:56:18.138844+0000","last_became_peered":"2023-09-23T20:56:18.138844+0000","last_unstale":"2023-09-23T21:01:26.742397+0000","last_undegraded":"2023-09-23T21:01:26.742397+0000","last_fullsized":"2023-09-23T21:01:26.742397+0000","mapping_epoch":192,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":193,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:09:02.217683+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,2],"acting":[5,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.14","version":"79'22","reported_seq":162,"reported_epoch":257,"state":"active+clean","last_fresh":"2023-09-23T21:01:30.014974+0000","last_change":"2023-09-23T20:55:35.949418+0000","last_active":"2023-09-23T21:01:30.014974+0000","last_peered":"2023-09-23T21:01:30.014974+0000","last_clean":"2023-09-23T21:01:30.014974+0000","last_became_active":"2023-09-23T20:55:35.948566+0000","last_became_peered":"2023-09-23T20:55:35.948566+0000","last_unstale":"2023-09-23T21:01:30.014974+0000","last_undegraded":"2023-09-23T21:01:30.014974+0000","last_fullsized":"2023-09-23T21:01:30.014974+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:59:37.433128+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4734,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":152,"reported_epoch":257,"state":"active+clean","last_fresh":"2023-09-23T21:01:30.015194+0000","last_change":"2023-09-23T20:55:35.949178+0000","last_active":"2023-09-23T21:01:30.015194+0000","last_peered":"2023-09-23T21:01:30.015194+0000","last_clean":"2023-09-23T21:01:30.015194+0000","last_became_active":"2023-09-23T20:55:35.948795+0000","last_became_peered":"2023-09-23T20:55:35.948795+0000","last_unstale":"2023-09-23T21:01:30.015194+0000","last_undegraded":"2023-09-23T21:01:30.015194+0000","last_fullsized":"2023-09-23T21:01:30.015194+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:13:08.996932+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.11","version":"79'161","reported_seq":486,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-09-23T21:01:26.739867+0000","last_change":"2023-09-23T20:57:28.756796+0000","last_active":"2023-09-23T21:01:26.739867+0000","last_peered":"2023-09-23T21:01:26.739867+0000","last_clean":"2023-09-23T21:01:26.739867+0000","last_became_active":"2023-09-23T20:57:15.980606+0000","last_became_peered":"2023-09-23T20:57:15.980606+0000","last_unstale":"2023-09-23T21:01:26.739867+0000","last_undegraded":"2023-09-23T21:01:26.739867+0000","last_fullsized":"2023-09-23T21:01:26.739867+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:28.756634+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:28.756634+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:09:19.150791+0000","scrub_duration":0.0064925119999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":178,"reported_epoch":257,"state":"active+clean","last_fresh":"2023-09-23T21:01:30.014982+0000","last_change":"2023-09-23T20:56:45.992577+0000","last_active":"2023-09-23T21:01:30.014982+0000","last_peered":"2023-09-23T21:01:30.014982+0000","last_clean":"2023-09-23T21:01:30.014982+0000","last_became_active":"2023-09-23T20:56:45.992297+0000","last_became_peered":"2023-09-23T20:56:45.992297+0000","last_unstale":"2023-09-23T21:01:30.014982+0000","last_undegraded":"2023-09-23T21:01:30.014982+0000","last_fullsized":"2023-09-23T21:01:30.014982+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":205,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:00:45.547202+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,2],"acting":[4,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":222,"reported_epoch":255,"state":"active+clean","last_fresh":"2023-09-23T21:01:25.332850+0000","last_change":"2023-09-23T20:51:10.831262+0000","last_active":"2023-09-23T21:01:25.332850+0000","last_peered":"2023-09-23T21:01:25.332850+0000","last_clean":"2023-09-23T21:01:25.332850+0000","last_became_active":"2023-09-23T20:51:10.830931+0000","last_became_peered":"2023-09-23T20:51:10.830931+0000","last_unstale":"2023-09-23T21:01:25.332850+0000","last_undegraded":"2023-09-23T21:01:25.332850+0000","last_fullsized":"2023-09-23T21:01:25.332850+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:51:54.982505+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.10","version":"79'161","reported_seq":494,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-09-23T21:01:26.742416+0000","last_change":"2023-09-23T20:57:20.708003+0000","last_active":"2023-09-23T21:01:26.742416+0000","last_peered":"2023-09-23T21:01:26.742416+0000","last_clean":"2023-09-23T21:01:26.742416+0000","last_became_active":"2023-09-23T20:57:15.975302+0000","last_became_peered":"2023-09-23T20:57:15.975302+0000","last_unstale":"2023-09-23T21:01:26.742416+0000","last_undegraded":"2023-09-23T21:01:26.742416+0000","last_fullsized":"2023-09-23T21:01:26.742416+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:20.707939+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:20.707939+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:16:51.721994+0000","scrub_duration":0.0017733409999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":233,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-09-23T21:01:26.738385+0000","last_change":"2023-09-23T20:57:44.979736+0000","last_active":"2023-09-23T21:01:26.738385+0000","last_peered":"2023-09-23T21:01:26.738385+0000","last_clean":"2023-09-23T21:01:26.738385+0000","last_became_active":"2023-09-23T20:57:44.979356+0000","last_became_peered":"2023-09-23T20:57:44.979356+0000","last_unstale":"2023-09-23T21:01:26.738385+0000","last_undegraded":"2023-09-23T21:01:26.738385+0000","last_fullsized":"2023-09-23T21:01:26.738385+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:14:43.848832+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":130,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-09-23T21:01:26.742294+0000","last_change":"2023-09-23T20:54:31.887814+0000","last_active":"2023-09-23T21:01:26.742294+0000","last_peered":"2023-09-23T21:01:26.742294+0000","last_clean":"2023-09-23T21:01:26.742294+0000","last_became_active":"2023-09-23T20:54:31.887195+0000","last_became_peered":"2023-09-23T20:54:31.887195+0000","last_unstale":"2023-09-23T21:01:26.742294+0000","last_undegraded":"2023-09-23T21:01:26.742294+0000","last_fullsized":"2023-09-23T21:01:26.742294+0000","mapping_epoch":136,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:10:52.089553+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.13","version":"79'161","reported_seq":558,"reported_epoch":257,"state":"active+clean","last_fresh":"2023-09-23T21:01:30.014825+0000","last_change":"2023-09-23T20:57:48.262970+0000","last_active":"2023-09-23T21:01:30.014825+0000","last_peered":"2023-09-23T21:01:30.014825+0000","last_clean":"2023-09-23T21:01:30.014825+0000","last_became_active":"2023-09-23T20:57:48.262663+0000","last_became_peered":"2023-09-23T20:57:48.262663+0000","last_unstale":"2023-09-23T21:01:30.014825+0000","last_undegraded":"2023-09-23T21:01:30.014825+0000","last_fullsized":"2023-09-23T21:01:30.014825+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:36.420362+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:36.420362+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:28:28.613492+0000","scrub_duration":0.028838444000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":202,"reported_epoch":257,"state":"active+clean","last_fresh":"2023-09-23T21:01:30.014584+0000","last_change":"2023-09-23T20:56:45.991822+0000","last_active":"2023-09-23T21:01:30.014584+0000","last_peered":"2023-09-23T21:01:30.014584+0000","last_clean":"2023-09-23T21:01:30.014584+0000","last_became_active":"2023-09-23T20:56:45.991460+0000","last_became_peered":"2023-09-23T20:56:45.991460+0000","last_unstale":"2023-09-23T21:01:30.014584+0000","last_undegraded":"2023-09-23T21:01:30.014584+0000","last_fullsized":"2023-09-23T21:01:30.014584+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":205,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:20:37.503317+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":214,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-09-23T21:01:26.739242+0000","last_change":"2023-09-23T20:51:10.840849+0000","last_active":"2023-09-23T21:01:26.739242+0000","last_peered":"2023-09-23T21:01:26.739242+0000","last_clean":"2023-09-23T21:01:26.739242+0000","last_became_active":"2023-09-23T20:51:10.840593+0000","last_became_peered":"2023-09-23T20:51:10.840593+0000","last_unstale":"2023-09-23T21:01:26.739242+0000","last_undegraded":"2023-09-23T21:01:26.739242+0000","last_fullsized":"2023-09-23T21:01:26.739242+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:30:52.297625+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.12","version":"79'161","reported_seq":494,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-09-23T21:01:26.742176+0000","last_change":"2023-09-23T20:57:21.750765+0000","last_active":"2023-09-23T21:01:26.742176+0000","last_peered":"2023-09-23T21:01:26.742176+0000","last_clean":"2023-09-23T21:01:26.742176+0000","last_became_active":"2023-09-23T20:57:15.979195+0000","last_became_peered":"2023-09-23T20:57:15.979195+0000","last_unstale":"2023-09-23T21:01:26.742176+0000","last_undegraded":"2023-09-23T21:01:26.742176+0000","last_fullsized":"2023-09-23T21:01:26.742176+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:21.750680+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:21.750680+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:03:41.915962+0000","scrub_duration":0.0092391680000000007,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":250,"reported_epoch":257,"state":"active+clean","last_fresh":"2023-09-23T21:01:29.942089+0000","last_change":"2023-09-23T20:55:36.943969+0000","last_active":"2023-09-23T21:01:29.942089+0000","last_peered":"2023-09-23T21:01:29.942089+0000","last_clean":"2023-09-23T21:01:29.942089+0000","last_became_active":"2023-09-23T20:55:36.943580+0000","last_became_peered":"2023-09-23T20:55:36.943580+0000","last_unstale":"2023-09-23T21:01:29.942089+0000","last_undegraded":"2023-09-23T21:01:29.942089+0000","last_fullsized":"2023-09-23T21:01:29.942089+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:49:06.969396+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":45,"reported_epoch":255,"state":"active+clean","last_fresh":"2023-09-23T21:01:25.333575+0000","last_change":"2023-09-23T20:57:44.980498+0000","last_active":"2023-09-23T21:01:25.333575+0000","last_peered":"2023-09-23T21:01:25.333575+0000","last_clean":"2023-09-23T21:01:25.333575+0000","last_became_active":"2023-09-23T20:57:44.980182+0000","last_became_peered":"2023-09-23T20:57:44.980182+0000","last_unstale":"2023-09-23T21:01:25.333575+0000","last_undegraded":"2023-09-23T21:01:25.333575+0000","last_fullsized":"2023-09-23T21:01:25.333575+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:34:51.620544+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.4","version":"257'50","reported_seq":61,"reported_epoch":257,"state":"active+clean","last_fresh":"2023-09-23T21:01:32.065787+0000","last_change":"2023-09-23T21:01:25.429865+0000","last_active":"2023-09-23T21:01:32.065787+0000","last_peered":"2023-09-23T21:01:32.065787+0000","last_clean":"2023-09-23T21:01:32.065787+0000","last_became_active":"2023-09-23T21:01:25.429555+0000","last_became_peered":"2023-09-23T21:01:25.429555+0000","last_unstale":"2023-09-23T21:01:32.065787+0000","last_undegraded":"2023-09-23T21:01:32.065787+0000","last_fullsized":"2023-09-23T21:01:32.065787+0000","mapping_epoch":254,"log_start":"0'0","ondisk_log_start":"0'0","created":254,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:01:24.325318+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:01:24.325318+0000","last_clean_scrub_stamp":"2023-09-23T21:01:24.325318+0000","objects_scrubbed":0,"log_size":50,"log_dups_size":0,"ondisk_log_size":50,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:28:38.557662+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":3276800,"num_objects":50,"num_object_clones":0,"num_object_copies":150,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":50,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":50,"num_write_kb":3200,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,2],"acting":[5,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.d","version":"79'161","reported_seq":540,"reported_epoch":257,"state":"active+clean","last_fresh":"2023-09-23T21:01:30.015038+0000","last_change":"2023-09-23T20:57:31.075213+0000","last_active":"2023-09-23T21:01:30.015038+0000","last_peered":"2023-09-23T21:01:30.015038+0000","last_clean":"2023-09-23T21:01:30.015038+0000","last_became_active":"2023-09-23T20:57:15.988188+0000","last_became_peered":"2023-09-23T20:57:15.988188+0000","last_unstale":"2023-09-23T21:01:30.015038+0000","last_undegraded":"2023-09-23T21:01:30.015038+0000","last_fullsized":"2023-09-23T21:01:30.015038+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:31.075098+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:31.075098+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:41:52.142362+0000","scrub_duration":0.010304610000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":189,"reported_epoch":257,"state":"active+clean","last_fresh":"2023-09-23T21:01:30.015337+0000","last_change":"2023-09-23T20:56:45.991841+0000","last_active":"2023-09-23T21:01:30.015337+0000","last_peered":"2023-09-23T21:01:30.015337+0000","last_clean":"2023-09-23T21:01:30.015337+0000","last_became_active":"2023-09-23T20:56:45.991491+0000","last_became_peered":"2023-09-23T20:56:45.991491+0000","last_unstale":"2023-09-23T21:01:30.015337+0000","last_undegraded":"2023-09-23T21:01:30.015337+0000","last_fullsized":"2023-09-23T21:01:30.015337+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":205,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:30:37.405319+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":147,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-09-23T21:01:26.742568+0000","last_change":"2023-09-23T20:55:35.957768+0000","last_active":"2023-09-23T21:01:26.742568+0000","last_peered":"2023-09-23T21:01:26.742568+0000","last_clean":"2023-09-23T21:01:26.742568+0000","last_became_active":"2023-09-23T20:55:35.957421+0000","last_became_peered":"2023-09-23T20:55:35.957421+0000","last_unstale":"2023-09-23T21:01:26.742568+0000","last_undegraded":"2023-09-23T21:01:26.742568+0000","last_fullsized":"2023-09-23T21:01:26.742568+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:49:41.585961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"8.5","version":"258'399","reported_seq":411,"reported_epoch":258,"state":"active+clean","last_fresh":"2023-09-23T21:01:34.102320+0000","last_change":"2023-09-23T21:01:25.560708+0000","last_active":"2023-09-23T21:01:34.102320+0000","last_peered":"2023-09-23T21:01:34.102320+0000","last_clean":"2023-09-23T21:01:34.102320+0000","last_became_active":"2023-09-23T21:01:25.560472+0000","last_became_peered":"2023-09-23T21:01:25.560472+0000","last_unstale":"2023-09-23T21:01:34.102320+0000","last_undegraded":"2023-09-23T21:01:34.102320+0000","last_fullsized":"2023-09-23T21:01:34.102320+0000","mapping_epoch":254,"log_start":"0'0","ondisk_log_start":"0'0","created":254,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:01:24.325318+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:01:24.325318+0000","last_clean_scrub_stamp":"2023-09-23T21:01:24.325318+0000","objects_scrubbed":0,"log_size":399,"log_dups_size":0,"ondisk_log_size":399,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:04:02.549068+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26148864,"num_objects":399,"num_object_clones":0,"num_object_copies":1197,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":399,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":399,"num_write_kb":25536,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.c","version":"79'161","reported_seq":557,"reported_epoch":255,"state":"active+clean","last_fresh":"2023-09-23T21:01:25.332700+0000","last_change":"2023-09-23T20:57:22.041071+0000","last_active":"2023-09-23T21:01:25.332700+0000","last_peered":"2023-09-23T21:01:25.332700+0000","last_clean":"2023-09-23T21:01:25.332700+0000","last_became_active":"2023-09-23T20:57:15.973204+0000","last_became_peered":"2023-09-23T20:57:15.973204+0000","last_unstale":"2023-09-23T21:01:25.332700+0000","last_undegraded":"2023-09-23T21:01:25.332700+0000","last_fullsized":"2023-09-23T21:01:25.332700+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:22.040907+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:22.040907+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:47:02.019978+0000","scrub_duration":0.013021176000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":176,"reported_epoch":257,"state":"active+clean","last_fresh":"2023-09-23T21:01:30.015524+0000","last_change":"2023-09-23T20:56:45.991821+0000","last_active":"2023-09-23T21:01:30.015524+0000","last_peered":"2023-09-23T21:01:30.015524+0000","last_clean":"2023-09-23T21:01:30.015524+0000","last_became_active":"2023-09-23T20:56:45.991447+0000","last_became_peered":"2023-09-23T20:56:45.991447+0000","last_unstale":"2023-09-23T21:01:30.015524+0000","last_undegraded":"2023-09-23T21:01:30.015524+0000","last_fullsized":"2023-09-23T21:01:30.015524+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":205,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:14:25.525222+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":179,"reported_epoch":257,"state":"active+clean","last_fresh":"2023-09-23T21:01:30.014857+0000","last_change":"2023-09-23T20:57:44.982059+0000","last_active":"2023-09-23T21:01:30.014857+0000","last_peered":"2023-09-23T21:01:30.014857+0000","last_clean":"2023-09-23T21:01:30.014857+0000","last_became_active":"2023-09-23T20:57:44.981742+0000","last_became_peered":"2023-09-23T20:57:44.981742+0000","last_unstale":"2023-09-23T21:01:30.014857+0000","last_undegraded":"2023-09-23T21:01:30.014857+0000","last_fullsized":"2023-09-23T21:01:30.014857+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:41:24.042916+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.6","version":"257'31","reported_seq":43,"reported_epoch":257,"state":"active+clean","last_fresh":"2023-09-23T21:01:31.941064+0000","last_change":"2023-09-23T21:01:25.560370+0000","last_active":"2023-09-23T21:01:31.941064+0000","last_peered":"2023-09-23T21:01:31.941064+0000","last_clean":"2023-09-23T21:01:31.941064+0000","last_became_active":"2023-09-23T21:01:25.560018+0000","last_became_peered":"2023-09-23T21:01:25.560018+0000","last_unstale":"2023-09-23T21:01:31.941064+0000","last_undegraded":"2023-09-23T21:01:31.941064+0000","last_fullsized":"2023-09-23T21:01:31.941064+0000","mapping_epoch":254,"log_start":"0'0","ondisk_log_start":"0'0","created":254,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:01:24.325318+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:01:24.325318+0000","last_clean_scrub_stamp":"2023-09-23T21:01:24.325318+0000","objects_scrubbed":0,"log_size":31,"log_dups_size":0,"ondisk_log_size":31,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:45:39.782872+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":2031616,"num_objects":31,"num_object_clones":0,"num_object_copies":93,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":31,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":31,"num_write_kb":1984,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,7],"acting":[0,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.f","version":"79'161","reported_seq":562,"reported_epoch":257,"state":"active+clean","last_fresh":"2023-09-23T21:01:29.942634+0000","last_change":"2023-09-23T20:57:27.880841+0000","last_active":"2023-09-23T21:01:29.942634+0000","last_peered":"2023-09-23T21:01:29.942634+0000","last_clean":"2023-09-23T21:01:29.942634+0000","last_became_active":"2023-09-23T20:57:15.970925+0000","last_became_peered":"2023-09-23T20:57:15.970925+0000","last_unstale":"2023-09-23T21:01:29.942634+0000","last_undegraded":"2023-09-23T21:01:29.942634+0000","last_fullsized":"2023-09-23T21:01:29.942634+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:27.880776+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:27.880776+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:15:13.438014+0000","scrub_duration":0.0014004899999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":161,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-09-23T21:01:26.742443+0000","last_change":"2023-09-23T20:56:45.989153+0000","last_active":"2023-09-23T21:01:26.742443+0000","last_peered":"2023-09-23T21:01:26.742443+0000","last_clean":"2023-09-23T21:01:26.742443+0000","last_became_active":"2023-09-23T20:56:45.988800+0000","last_became_peered":"2023-09-23T20:56:45.988800+0000","last_unstale":"2023-09-23T21:01:26.742443+0000","last_undegraded":"2023-09-23T21:01:26.742443+0000","last_fullsized":"2023-09-23T21:01:26.742443+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":205,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:04:41.130194+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":197,"reported_epoch":257,"state":"active+clean","last_fresh":"2023-09-23T21:01:29.942451+0000","last_change":"2023-09-23T20:55:35.965427+0000","last_active":"2023-09-23T21:01:29.942451+0000","last_peered":"2023-09-23T21:01:29.942451+0000","last_clean":"2023-09-23T21:01:29.942451+0000","last_became_active":"2023-09-23T20:55:35.963969+0000","last_became_peered":"2023-09-23T20:55:35.963969+0000","last_unstale":"2023-09-23T21:01:29.942451+0000","last_undegraded":"2023-09-23T21:01:29.942451+0000","last_fullsized":"2023-09-23T21:01:29.942451+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:38:40.612083+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.7","version":"258'493","reported_seq":506,"reported_epoch":258,"state":"active+clean","last_fresh":"2023-09-23T21:01:34.395936+0000","last_change":"2023-09-23T21:01:25.341425+0000","last_active":"2023-09-23T21:01:34.395936+0000","last_peered":"2023-09-23T21:01:34.395936+0000","last_clean":"2023-09-23T21:01:34.395936+0000","last_became_active":"2023-09-23T21:01:25.341114+0000","last_became_peered":"2023-09-23T21:01:25.341114+0000","last_unstale":"2023-09-23T21:01:34.395936+0000","last_undegraded":"2023-09-23T21:01:34.395936+0000","last_fullsized":"2023-09-23T21:01:34.395936+0000","mapping_epoch":254,"log_start":"0'0","ondisk_log_start":"0'0","created":254,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:01:24.325318+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:01:24.325318+0000","last_clean_scrub_stamp":"2023-09-23T21:01:24.325318+0000","objects_scrubbed":0,"log_size":493,"log_dups_size":0,"ondisk_log_size":493,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:15:08.338532+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":32309248,"num_objects":493,"num_object_clones":0,"num_object_copies":1479,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":493,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":493,"num_write_kb":31552,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.e","version":"79'161","reported_seq":469,"reported_epoch":257,"state":"active+clean","last_fresh":"2023-09-23T21:01:30.015269+0000","last_change":"2023-09-23T20:57:34.132738+0000","last_active":"2023-09-23T21:01:30.015269+0000","last_peered":"2023-09-23T21:01:30.015269+0000","last_clean":"2023-09-23T21:01:30.015269+0000","last_became_active":"2023-09-23T20:57:15.974318+0000","last_became_peered":"2023-09-23T20:57:15.974318+0000","last_unstale":"2023-09-23T21:01:30.015269+0000","last_undegraded":"2023-09-23T21:01:30.015269+0000","last_fullsized":"2023-09-23T21:01:30.015269+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:34.132660+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:34.132660+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:07:07.739438+0000","scrub_duration":0.005229694,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":147,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-09-23T21:01:26.741781+0000","last_change":"2023-09-23T20:55:35.962005+0000","last_active":"2023-09-23T21:01:26.741781+0000","last_peered":"2023-09-23T21:01:26.741781+0000","last_clean":"2023-09-23T21:01:26.741781+0000","last_became_active":"2023-09-23T20:55:35.961695+0000","last_became_peered":"2023-09-23T20:55:35.961695+0000","last_unstale":"2023-09-23T21:01:26.741781+0000","last_undegraded":"2023-09-23T21:01:26.741781+0000","last_fullsized":"2023-09-23T21:01:26.741781+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:28:22.876201+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":246,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-09-23T21:01:26.739336+0000","last_change":"2023-09-23T20:54:13.928715+0000","last_active":"2023-09-23T21:01:26.739336+0000","last_peered":"2023-09-23T21:01:26.739336+0000","last_clean":"2023-09-23T21:01:26.739336+0000","last_became_active":"2023-09-23T20:54:13.928200+0000","last_became_peered":"2023-09-23T20:54:13.928200+0000","last_unstale":"2023-09-23T21:01:26.739336+0000","last_undegraded":"2023-09-23T21:01:26.739336+0000","last_fullsized":"2023-09-23T21:01:26.739336+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:41:16.927513+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.0","version":"0'0","reported_seq":11,"reported_epoch":255,"state":"active+clean","last_fresh":"2023-09-23T21:01:25.343218+0000","last_change":"2023-09-23T21:01:25.342305+0000","last_active":"2023-09-23T21:01:25.343218+0000","last_peered":"2023-09-23T21:01:25.343218+0000","last_clean":"2023-09-23T21:01:25.343218+0000","last_became_active":"2023-09-23T21:01:25.342042+0000","last_became_peered":"2023-09-23T21:01:25.342042+0000","last_unstale":"2023-09-23T21:01:25.343218+0000","last_undegraded":"2023-09-23T21:01:25.343218+0000","last_fullsized":"2023-09-23T21:01:25.343218+0000","mapping_epoch":254,"log_start":"0'0","ondisk_log_start":"0'0","created":254,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:01:24.325318+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:01:24.325318+0000","last_clean_scrub_stamp":"2023-09-23T21:01:24.325318+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:06:29.755463+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,1],"acting":[3,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.9","version":"79'161","reported_seq":630,"reported_epoch":255,"state":"active+clean","last_fresh":"2023-09-23T21:01:25.332234+0000","last_change":"2023-09-23T20:56:46.921939+0000","last_active":"2023-09-23T21:01:25.332234+0000","last_peered":"2023-09-23T21:01:25.332234+0000","last_clean":"2023-09-23T21:01:25.332234+0000","last_became_active":"2023-09-23T20:56:45.989908+0000","last_became_peered":"2023-09-23T20:56:45.989908+0000","last_unstale":"2023-09-23T21:01:25.332234+0000","last_undegraded":"2023-09-23T21:01:25.332234+0000","last_fullsized":"2023-09-23T21:01:25.332234+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":205,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:56:46.921739+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:56:46.921739+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:20:42.635110+0000","scrub_duration":0.0071954109999999996,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":246,"reported_epoch":255,"state":"active+clean","last_fresh":"2023-09-23T21:01:25.334266+0000","last_change":"2023-09-23T20:54:13.961788+0000","last_active":"2023-09-23T21:01:25.334266+0000","last_peered":"2023-09-23T21:01:25.334266+0000","last_clean":"2023-09-23T21:01:25.334266+0000","last_became_active":"2023-09-23T20:54:13.940937+0000","last_became_peered":"2023-09-23T20:54:13.940937+0000","last_unstale":"2023-09-23T21:01:25.334266+0000","last_undegraded":"2023-09-23T21:01:25.334266+0000","last_fullsized":"2023-09-23T21:01:25.334266+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:11:56.785299+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":152,"reported_epoch":257,"state":"active+clean","last_fresh":"2023-09-23T21:01:30.015073+0000","last_change":"2023-09-23T20:55:35.949670+0000","last_active":"2023-09-23T21:01:30.015073+0000","last_peered":"2023-09-23T21:01:30.015073+0000","last_clean":"2023-09-23T21:01:30.015073+0000","last_became_active":"2023-09-23T20:55:35.949140+0000","last_became_peered":"2023-09-23T20:55:35.949140+0000","last_unstale":"2023-09-23T21:01:30.015073+0000","last_undegraded":"2023-09-23T21:01:30.015073+0000","last_fullsized":"2023-09-23T21:01:30.015073+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:55:59.340303+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.1","version":"258'497","reported_seq":510,"reported_epoch":258,"state":"active+clean","last_fresh":"2023-09-23T21:01:34.375590+0000","last_change":"2023-09-23T21:01:25.429922+0000","last_active":"2023-09-23T21:01:34.375590+0000","last_peered":"2023-09-23T21:01:34.375590+0000","last_clean":"2023-09-23T21:01:34.375590+0000","last_became_active":"2023-09-23T21:01:25.429650+0000","last_became_peered":"2023-09-23T21:01:25.429650+0000","last_unstale":"2023-09-23T21:01:34.375590+0000","last_undegraded":"2023-09-23T21:01:34.375590+0000","last_fullsized":"2023-09-23T21:01:34.375590+0000","mapping_epoch":254,"log_start":"0'0","ondisk_log_start":"0'0","created":254,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:01:24.325318+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:01:24.325318+0000","last_clean_scrub_stamp":"2023-09-23T21:01:24.325318+0000","objects_scrubbed":0,"log_size":497,"log_dups_size":0,"ondisk_log_size":497,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:18:57.243292+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":32571392,"num_objects":497,"num_object_clones":0,"num_object_copies":1491,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":497,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":497,"num_write_kb":31808,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,2],"acting":[1,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.8","version":"79'161","reported_seq":500,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-09-23T21:01:26.742527+0000","last_change":"2023-09-23T20:56:57.505737+0000","last_active":"2023-09-23T21:01:26.742527+0000","last_peered":"2023-09-23T21:01:26.742527+0000","last_clean":"2023-09-23T21:01:26.742527+0000","last_became_active":"2023-09-23T20:56:04.982345+0000","last_became_peered":"2023-09-23T20:56:04.982345+0000","last_unstale":"2023-09-23T21:01:26.742527+0000","last_undegraded":"2023-09-23T21:01:26.742527+0000","last_fullsized":"2023-09-23T21:01:26.742527+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:56:57.505663+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:56:57.505663+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:34:30.109724+0000","scrub_duration":0.026284108,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":220,"reported_epoch":257,"state":"active+clean","last_fresh":"2023-09-23T21:01:29.942336+0000","last_change":"2023-09-23T20:55:35.959942+0000","last_active":"2023-09-23T21:01:29.942336+0000","last_peered":"2023-09-23T21:01:29.942336+0000","last_clean":"2023-09-23T21:01:29.942336+0000","last_became_active":"2023-09-23T20:55:35.957071+0000","last_became_peered":"2023-09-23T20:55:35.957071+0000","last_unstale":"2023-09-23T21:01:29.942336+0000","last_undegraded":"2023-09-23T21:01:29.942336+0000","last_fullsized":"2023-09-23T21:01:29.942336+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:44:30.308036+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":135,"reported_epoch":257,"state":"active+clean","last_fresh":"2023-09-23T21:01:30.015429+0000","last_change":"2023-09-23T20:54:13.951459+0000","last_active":"2023-09-23T21:01:30.015429+0000","last_peered":"2023-09-23T21:01:30.015429+0000","last_clean":"2023-09-23T21:01:30.015429+0000","last_became_active":"2023-09-23T20:54:13.948409+0000","last_became_peered":"2023-09-23T20:54:13.948409+0000","last_unstale":"2023-09-23T21:01:30.015429+0000","last_undegraded":"2023-09-23T21:01:30.015429+0000","last_fullsized":"2023-09-23T21:01:30.015429+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:38:10.148061+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.2","version":"258'104","reported_seq":116,"reported_epoch":258,"state":"active+clean","last_fresh":"2023-09-23T21:01:32.223338+0000","last_change":"2023-09-23T21:01:25.343022+0000","last_active":"2023-09-23T21:01:32.223338+0000","last_peered":"2023-09-23T21:01:32.223338+0000","last_clean":"2023-09-23T21:01:32.223338+0000","last_became_active":"2023-09-23T21:01:25.342540+0000","last_became_peered":"2023-09-23T21:01:25.342540+0000","last_unstale":"2023-09-23T21:01:32.223338+0000","last_undegraded":"2023-09-23T21:01:32.223338+0000","last_fullsized":"2023-09-23T21:01:32.223338+0000","mapping_epoch":254,"log_start":"0'0","ondisk_log_start":"0'0","created":254,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:01:24.325318+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:01:24.325318+0000","last_clean_scrub_stamp":"2023-09-23T21:01:24.325318+0000","objects_scrubbed":0,"log_size":104,"log_dups_size":0,"ondisk_log_size":104,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:57:19.741924+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":6815744,"num_objects":104,"num_object_clones":0,"num_object_copies":312,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":104,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":104,"num_write_kb":6656,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,5],"acting":[4,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.b","version":"79'161","reported_seq":541,"reported_epoch":257,"state":"active+clean","last_fresh":"2023-09-23T21:01:30.014894+0000","last_change":"2023-09-23T20:57:27.320172+0000","last_active":"2023-09-23T21:01:30.014894+0000","last_peered":"2023-09-23T21:01:30.014894+0000","last_clean":"2023-09-23T21:01:30.014894+0000","last_became_active":"2023-09-23T20:57:16.009630+0000","last_became_peered":"2023-09-23T20:57:16.009630+0000","last_unstale":"2023-09-23T21:01:30.014894+0000","last_undegraded":"2023-09-23T21:01:30.014894+0000","last_fullsized":"2023-09-23T21:01:30.014894+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:27.320022+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:27.320022+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:31:35.221558+0000","scrub_duration":0.0043894679999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,5],"acting":[1,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":135,"reported_epoch":257,"state":"active+clean","last_fresh":"2023-09-23T21:01:30.014626+0000","last_change":"2023-09-23T20:54:13.950254+0000","last_active":"2023-09-23T21:01:30.014626+0000","last_peered":"2023-09-23T21:01:30.014626+0000","last_clean":"2023-09-23T21:01:30.014626+0000","last_became_active":"2023-09-23T20:54:13.949934+0000","last_became_peered":"2023-09-23T20:54:13.949934+0000","last_unstale":"2023-09-23T21:01:30.014626+0000","last_undegraded":"2023-09-23T21:01:30.014626+0000","last_fullsized":"2023-09-23T21:01:30.014626+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:30:32.847167+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":152,"reported_epoch":257,"state":"active+clean","last_fresh":"2023-09-23T21:01:30.015437+0000","last_change":"2023-09-23T20:55:35.956153+0000","last_active":"2023-09-23T21:01:30.015437+0000","last_peered":"2023-09-23T21:01:30.015437+0000","last_clean":"2023-09-23T21:01:30.015437+0000","last_became_active":"2023-09-23T20:55:35.955630+0000","last_became_peered":"2023-09-23T20:55:35.955630+0000","last_unstale":"2023-09-23T21:01:30.015437+0000","last_undegraded":"2023-09-23T21:01:30.015437+0000","last_fullsized":"2023-09-23T21:01:30.015437+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:45:20.282929+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.3","version":"258'457","reported_seq":470,"reported_epoch":258,"state":"active+clean","last_fresh":"2023-09-23T21:01:34.397968+0000","last_change":"2023-09-23T21:01:25.342380+0000","last_active":"2023-09-23T21:01:34.397968+0000","last_peered":"2023-09-23T21:01:34.397968+0000","last_clean":"2023-09-23T21:01:34.397968+0000","last_became_active":"2023-09-23T21:01:25.342099+0000","last_became_peered":"2023-09-23T21:01:25.342099+0000","last_unstale":"2023-09-23T21:01:34.397968+0000","last_undegraded":"2023-09-23T21:01:34.397968+0000","last_fullsized":"2023-09-23T21:01:34.397968+0000","mapping_epoch":254,"log_start":"0'0","ondisk_log_start":"0'0","created":254,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:01:24.325318+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:01:24.325318+0000","last_clean_scrub_stamp":"2023-09-23T21:01:24.325318+0000","objects_scrubbed":0,"log_size":457,"log_dups_size":0,"ondisk_log_size":457,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:09:21.965969+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":29949952,"num_objects":457,"num_object_clones":0,"num_object_copies":1371,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":457,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":457,"num_write_kb":29248,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.a","version":"79'161","reported_seq":500,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-09-23T21:01:26.742672+0000","last_change":"2023-09-23T20:56:44.487778+0000","last_active":"2023-09-23T21:01:26.742672+0000","last_peered":"2023-09-23T21:01:26.742672+0000","last_clean":"2023-09-23T21:01:26.742672+0000","last_became_active":"2023-09-23T20:56:05.006740+0000","last_became_peered":"2023-09-23T20:56:05.006740+0000","last_unstale":"2023-09-23T21:01:26.742672+0000","last_undegraded":"2023-09-23T21:01:26.742672+0000","last_fullsized":"2023-09-23T21:01:26.742672+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:56:44.487678+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:56:44.487678+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:04:50.086749+0000","scrub_duration":0.0017408899999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":45,"reported_epoch":255,"state":"active+clean","last_fresh":"2023-09-23T21:01:25.332082+0000","last_change":"2023-09-23T20:57:44.982513+0000","last_active":"2023-09-23T21:01:25.332082+0000","last_peered":"2023-09-23T21:01:25.332082+0000","last_clean":"2023-09-23T21:01:25.332082+0000","last_became_active":"2023-09-23T20:57:44.982112+0000","last_became_peered":"2023-09-23T20:57:44.982112+0000","last_unstale":"2023-09-23T21:01:25.332082+0000","last_undegraded":"2023-09-23T21:01:25.332082+0000","last_fullsized":"2023-09-23T21:01:25.332082+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:09:05.630041+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":229,"reported_epoch":257,"state":"active+clean","last_fresh":"2023-09-23T21:01:30.014470+0000","last_change":"2023-09-23T20:54:13.940748+0000","last_active":"2023-09-23T21:01:30.014470+0000","last_peered":"2023-09-23T21:01:30.014470+0000","last_clean":"2023-09-23T21:01:30.014470+0000","last_became_active":"2023-09-23T20:54:13.940419+0000","last_became_peered":"2023-09-23T20:54:13.940419+0000","last_unstale":"2023-09-23T21:01:30.014470+0000","last_undegraded":"2023-09-23T21:01:30.014470+0000","last_fullsized":"2023-09-23T21:01:30.014470+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:15:01.923816+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.e","version":"257'34","reported_seq":46,"reported_epoch":257,"state":"active+clean","last_fresh":"2023-09-23T21:01:31.917557+0000","last_change":"2023-09-23T21:01:25.430111+0000","last_active":"2023-09-23T21:01:31.917557+0000","last_peered":"2023-09-23T21:01:31.917557+0000","last_clean":"2023-09-23T21:01:31.917557+0000","last_became_active":"2023-09-23T21:01:25.429811+0000","last_became_peered":"2023-09-23T21:01:25.429811+0000","last_unstale":"2023-09-23T21:01:31.917557+0000","last_undegraded":"2023-09-23T21:01:31.917557+0000","last_fullsized":"2023-09-23T21:01:31.917557+0000","mapping_epoch":254,"log_start":"0'0","ondisk_log_start":"0'0","created":254,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:01:24.325318+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:01:24.325318+0000","last_clean_scrub_stamp":"2023-09-23T21:01:24.325318+0000","objects_scrubbed":0,"log_size":34,"log_dups_size":0,"ondisk_log_size":34,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:18:01.918758+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":2228224,"num_objects":34,"num_object_clones":0,"num_object_copies":102,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":34,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":34,"num_write_kb":2176,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.7","version":"79'161","reported_seq":563,"reported_epoch":257,"state":"active+clean","last_fresh":"2023-09-23T21:01:29.942797+0000","last_change":"2023-09-23T20:57:48.243572+0000","last_active":"2023-09-23T21:01:29.942797+0000","last_peered":"2023-09-23T21:01:29.942797+0000","last_clean":"2023-09-23T21:01:29.942797+0000","last_became_active":"2023-09-23T20:57:48.243229+0000","last_became_peered":"2023-09-23T20:57:48.243229+0000","last_unstale":"2023-09-23T21:01:29.942797+0000","last_undegraded":"2023-09-23T21:01:29.942797+0000","last_fullsized":"2023-09-23T21:01:29.942797+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":3,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:25.916867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:25.916867+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:23:19.385831+0000","scrub_duration":0.001665323,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":234,"reported_epoch":257,"state":"active+clean","last_fresh":"2023-09-23T21:01:30.014273+0000","last_change":"2023-09-23T20:50:05.867130+0000","last_active":"2023-09-23T21:01:30.014273+0000","last_peered":"2023-09-23T21:01:30.014273+0000","last_clean":"2023-09-23T21:01:30.014273+0000","last_became_active":"2023-09-23T20:50:05.866758+0000","last_became_peered":"2023-09-23T20:50:05.866758+0000","last_unstale":"2023-09-23T21:01:30.014273+0000","last_undegraded":"2023-09-23T21:01:30.014273+0000","last_fullsized":"2023-09-23T21:01:30.014273+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:55:07.548209+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":130,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-09-23T21:01:26.742416+0000","last_change":"2023-09-23T20:54:31.887604+0000","last_active":"2023-09-23T21:01:26.742416+0000","last_peered":"2023-09-23T21:01:26.742416+0000","last_clean":"2023-09-23T21:01:26.742416+0000","last_became_active":"2023-09-23T20:54:31.886752+0000","last_became_peered":"2023-09-23T20:54:31.886752+0000","last_unstale":"2023-09-23T21:01:26.742416+0000","last_undegraded":"2023-09-23T21:01:26.742416+0000","last_fullsized":"2023-09-23T21:01:26.742416+0000","mapping_epoch":136,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:13:58.872176+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":232,"reported_epoch":257,"state":"active+clean","last_fresh":"2023-09-23T21:01:30.013266+0000","last_change":"2023-09-23T20:54:31.899082+0000","last_active":"2023-09-23T21:01:30.013266+0000","last_peered":"2023-09-23T21:01:30.013266+0000","last_clean":"2023-09-23T21:01:30.013266+0000","last_became_active":"2023-09-23T20:54:31.898521+0000","last_became_peered":"2023-09-23T20:54:31.898521+0000","last_unstale":"2023-09-23T21:01:30.013266+0000","last_undegraded":"2023-09-23T21:01:30.013266+0000","last_fullsized":"2023-09-23T21:01:30.013266+0000","mapping_epoch":136,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:25:36.059711+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.8","version":"257'62","reported_seq":73,"reported_epoch":257,"state":"active+clean","last_fresh":"2023-09-23T21:01:32.064404+0000","last_change":"2023-09-23T21:01:25.560581+0000","last_active":"2023-09-23T21:01:32.064404+0000","last_peered":"2023-09-23T21:01:32.064404+0000","last_clean":"2023-09-23T21:01:32.064404+0000","last_became_active":"2023-09-23T21:01:25.560318+0000","last_became_peered":"2023-09-23T21:01:25.560318+0000","last_unstale":"2023-09-23T21:01:32.064404+0000","last_undegraded":"2023-09-23T21:01:32.064404+0000","last_fullsized":"2023-09-23T21:01:32.064404+0000","mapping_epoch":254,"log_start":"0'0","ondisk_log_start":"0'0","created":254,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:01:24.325318+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:01:24.325318+0000","last_clean_scrub_stamp":"2023-09-23T21:01:24.325318+0000","objects_scrubbed":0,"log_size":62,"log_dups_size":0,"ondisk_log_size":62,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:07:28.203012+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4063232,"num_objects":62,"num_object_clones":0,"num_object_copies":186,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":62,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":3968,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[5,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.1","version":"79'161","reported_seq":520,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-09-23T21:01:26.741574+0000","last_change":"2023-09-23T20:57:19.669346+0000","last_active":"2023-09-23T21:01:26.741574+0000","last_peered":"2023-09-23T21:01:26.741574+0000","last_clean":"2023-09-23T21:01:26.741574+0000","last_became_active":"2023-09-23T20:57:08.246046+0000","last_became_peered":"2023-09-23T20:57:08.246046+0000","last_unstale":"2023-09-23T21:01:26.741574+0000","last_undegraded":"2023-09-23T21:01:26.741574+0000","last_fullsized":"2023-09-23T21:01:26.741574+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:19.669259+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:19.669259+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:14:03.435531+0000","scrub_duration":0.0045759559999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":147,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-09-23T21:01:26.742275+0000","last_change":"2023-09-23T20:55:35.959514+0000","last_active":"2023-09-23T21:01:26.742275+0000","last_peered":"2023-09-23T21:01:26.742275+0000","last_clean":"2023-09-23T21:01:26.742275+0000","last_became_active":"2023-09-23T20:55:35.959244+0000","last_became_peered":"2023-09-23T20:55:35.959244+0000","last_unstale":"2023-09-23T21:01:26.742275+0000","last_undegraded":"2023-09-23T21:01:26.742275+0000","last_fullsized":"2023-09-23T21:01:26.742275+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:47:32.665316+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":152,"reported_epoch":257,"state":"active+clean","last_fresh":"2023-09-23T21:01:30.015199+0000","last_change":"2023-09-23T20:55:35.949000+0000","last_active":"2023-09-23T21:01:30.015199+0000","last_peered":"2023-09-23T21:01:30.015199+0000","last_clean":"2023-09-23T21:01:30.015199+0000","last_became_active":"2023-09-23T20:55:35.948410+0000","last_became_peered":"2023-09-23T20:55:35.948410+0000","last_unstale":"2023-09-23T21:01:30.015199+0000","last_undegraded":"2023-09-23T21:01:30.015199+0000","last_fullsized":"2023-09-23T21:01:30.015199+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:26:50.477289+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":226,"reported_epoch":257,"state":"active+clean","last_fresh":"2023-09-23T21:01:30.014162+0000","last_change":"2023-09-23T20:51:10.829889+0000","last_active":"2023-09-23T21:01:30.014162+0000","last_peered":"2023-09-23T21:01:30.014162+0000","last_clean":"2023-09-23T21:01:30.014162+0000","last_became_active":"2023-09-23T20:51:10.829533+0000","last_became_peered":"2023-09-23T20:51:10.829533+0000","last_unstale":"2023-09-23T21:01:30.014162+0000","last_undegraded":"2023-09-23T21:01:30.014162+0000","last_fullsized":"2023-09-23T21:01:30.014162+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:43:04.603460+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.b","version":"0'0","reported_seq":11,"reported_epoch":255,"state":"active+clean","last_fresh":"2023-09-23T21:01:25.341083+0000","last_change":"2023-09-23T21:01:25.340569+0000","last_active":"2023-09-23T21:01:25.341083+0000","last_peered":"2023-09-23T21:01:25.341083+0000","last_clean":"2023-09-23T21:01:25.341083+0000","last_became_active":"2023-09-23T21:01:25.340209+0000","last_became_peered":"2023-09-23T21:01:25.340209+0000","last_unstale":"2023-09-23T21:01:25.341083+0000","last_undegraded":"2023-09-23T21:01:25.341083+0000","last_fullsized":"2023-09-23T21:01:25.341083+0000","mapping_epoch":254,"log_start":"0'0","ondisk_log_start":"0'0","created":254,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:01:24.325318+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:01:24.325318+0000","last_clean_scrub_stamp":"2023-09-23T21:01:24.325318+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:53:59.553810+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.2","version":"79'161","reported_seq":488,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-09-23T21:01:26.738294+0000","last_change":"2023-09-23T20:57:24.802586+0000","last_active":"2023-09-23T21:01:26.738294+0000","last_peered":"2023-09-23T21:01:26.738294+0000","last_clean":"2023-09-23T21:01:26.738294+0000","last_became_active":"2023-09-23T20:57:08.236114+0000","last_became_peered":"2023-09-23T20:57:08.236114+0000","last_unstale":"2023-09-23T21:01:26.738294+0000","last_undegraded":"2023-09-23T21:01:26.738294+0000","last_fullsized":"2023-09-23T21:01:26.738294+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:24.802451+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:24.802451+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:06:22.309980+0000","scrub_duration":0.0090861250000000005,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,4],"acting":[0,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":244,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-09-23T21:01:26.738790+0000","last_change":"2023-09-23T20:51:10.838054+0000","last_active":"2023-09-23T21:01:26.738790+0000","last_peered":"2023-09-23T21:01:26.738790+0000","last_clean":"2023-09-23T21:01:26.738790+0000","last_became_active":"2023-09-23T20:51:10.837750+0000","last_became_peered":"2023-09-23T20:51:10.837750+0000","last_unstale":"2023-09-23T21:01:26.738790+0000","last_undegraded":"2023-09-23T21:01:26.738790+0000","last_fullsized":"2023-09-23T21:01:26.738790+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:15:51.648085+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":226,"reported_epoch":257,"state":"active+clean","last_fresh":"2023-09-23T21:01:30.013044+0000","last_change":"2023-09-23T20:55:35.951146+0000","last_active":"2023-09-23T21:01:30.013044+0000","last_peered":"2023-09-23T21:01:30.013044+0000","last_clean":"2023-09-23T21:01:30.013044+0000","last_became_active":"2023-09-23T20:55:35.950874+0000","last_became_peered":"2023-09-23T20:55:35.950874+0000","last_unstale":"2023-09-23T21:01:30.013044+0000","last_undegraded":"2023-09-23T21:01:30.013044+0000","last_fullsized":"2023-09-23T21:01:30.013044+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:58:13.957677+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":247,"reported_epoch":257,"state":"active+clean","last_fresh":"2023-09-23T21:01:30.014433+0000","last_change":"2023-09-23T20:54:31.898870+0000","last_active":"2023-09-23T21:01:30.014433+0000","last_peered":"2023-09-23T21:01:30.014433+0000","last_clean":"2023-09-23T21:01:30.014433+0000","last_became_active":"2023-09-23T20:54:31.898467+0000","last_became_peered":"2023-09-23T20:54:31.898467+0000","last_unstale":"2023-09-23T21:01:30.014433+0000","last_undegraded":"2023-09-23T21:01:30.014433+0000","last_fullsized":"2023-09-23T21:01:30.014433+0000","mapping_epoch":136,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:56:22.483408+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.9","version":"258'400","reported_seq":412,"reported_epoch":258,"state":"active+clean","last_fresh":"2023-09-23T21:01:34.120442+0000","last_change":"2023-09-23T21:01:25.342548+0000","last_active":"2023-09-23T21:01:34.120442+0000","last_peered":"2023-09-23T21:01:34.120442+0000","last_clean":"2023-09-23T21:01:34.120442+0000","last_became_active":"2023-09-23T21:01:25.342227+0000","last_became_peered":"2023-09-23T21:01:25.342227+0000","last_unstale":"2023-09-23T21:01:34.120442+0000","last_undegraded":"2023-09-23T21:01:34.120442+0000","last_fullsized":"2023-09-23T21:01:34.120442+0000","mapping_epoch":254,"log_start":"0'0","ondisk_log_start":"0'0","created":254,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:01:24.325318+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:01:24.325318+0000","last_clean_scrub_stamp":"2023-09-23T21:01:24.325318+0000","objects_scrubbed":0,"log_size":400,"log_dups_size":0,"ondisk_log_size":400,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:38:15.944067+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26214400,"num_objects":400,"num_object_clones":0,"num_object_copies":1200,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":400,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":400,"num_write_kb":25600,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,0],"acting":[6,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":130,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-09-23T21:01:26.741837+0000","last_change":"2023-09-23T20:54:31.883504+0000","last_active":"2023-09-23T21:01:26.741837+0000","last_peered":"2023-09-23T21:01:26.741837+0000","last_clean":"2023-09-23T21:01:26.741837+0000","last_became_active":"2023-09-23T20:54:31.883172+0000","last_became_peered":"2023-09-23T20:54:31.883172+0000","last_unstale":"2023-09-23T21:01:26.741837+0000","last_undegraded":"2023-09-23T21:01:26.741837+0000","last_fullsized":"2023-09-23T21:01:26.741837+0000","mapping_epoch":136,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:45:40.289120+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"79'161","reported_seq":488,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-09-23T21:01:26.739087+0000","last_change":"2023-09-23T20:57:20.738417+0000","last_active":"2023-09-23T21:01:26.739087+0000","last_peered":"2023-09-23T21:01:26.739087+0000","last_clean":"2023-09-23T21:01:26.739087+0000","last_became_active":"2023-09-23T20:57:08.236842+0000","last_became_peered":"2023-09-23T20:57:08.236842+0000","last_unstale":"2023-09-23T21:01:26.739087+0000","last_undegraded":"2023-09-23T21:01:26.739087+0000","last_fullsized":"2023-09-23T21:01:26.739087+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":0,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:20.738273+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:20.738273+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:18:28.327990+0000","scrub_duration":0.0088336339999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":234,"reported_epoch":255,"state":"active+clean","last_fresh":"2023-09-23T21:01:25.332392+0000","last_change":"2023-09-23T20:55:35.950777+0000","last_active":"2023-09-23T21:01:25.332392+0000","last_peered":"2023-09-23T21:01:25.332392+0000","last_clean":"2023-09-23T21:01:25.332392+0000","last_became_active":"2023-09-23T20:55:35.950427+0000","last_became_peered":"2023-09-23T20:55:35.950427+0000","last_unstale":"2023-09-23T21:01:25.332392+0000","last_undegraded":"2023-09-23T21:01:25.332392+0000","last_fullsized":"2023-09-23T21:01:25.332392+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:20:10.371751+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":202,"reported_epoch":257,"state":"active+clean","last_fresh":"2023-09-23T21:01:29.942799+0000","last_change":"2023-09-23T20:55:35.978140+0000","last_active":"2023-09-23T21:01:29.942799+0000","last_peered":"2023-09-23T21:01:29.942799+0000","last_clean":"2023-09-23T21:01:29.942799+0000","last_became_active":"2023-09-23T20:55:35.977831+0000","last_became_peered":"2023-09-23T20:55:35.977831+0000","last_unstale":"2023-09-23T21:01:29.942799+0000","last_undegraded":"2023-09-23T21:01:29.942799+0000","last_fullsized":"2023-09-23T21:01:29.942799+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:25:47.678023+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.a","version":"258'494","reported_seq":506,"reported_epoch":258,"state":"active+clean","last_fresh":"2023-09-23T21:01:34.395053+0000","last_change":"2023-09-23T21:01:25.429899+0000","last_active":"2023-09-23T21:01:34.395053+0000","last_peered":"2023-09-23T21:01:34.395053+0000","last_clean":"2023-09-23T21:01:34.395053+0000","last_became_active":"2023-09-23T21:01:25.429583+0000","last_became_peered":"2023-09-23T21:01:25.429583+0000","last_unstale":"2023-09-23T21:01:34.395053+0000","last_undegraded":"2023-09-23T21:01:34.395053+0000","last_fullsized":"2023-09-23T21:01:34.395053+0000","mapping_epoch":254,"log_start":"0'0","ondisk_log_start":"0'0","created":254,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:01:24.325318+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:01:24.325318+0000","last_clean_scrub_stamp":"2023-09-23T21:01:24.325318+0000","objects_scrubbed":0,"log_size":494,"log_dups_size":0,"ondisk_log_size":494,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:31:11.594129+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":32374784,"num_objects":494,"num_object_clones":0,"num_object_copies":1482,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":494,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":494,"num_write_kb":31616,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.3","version":"79'161","reported_seq":525,"reported_epoch":257,"state":"active+clean","last_fresh":"2023-09-23T21:01:30.014963+0000","last_change":"2023-09-23T20:57:20.378531+0000","last_active":"2023-09-23T21:01:30.014963+0000","last_peered":"2023-09-23T21:01:30.014963+0000","last_clean":"2023-09-23T21:01:30.014963+0000","last_became_active":"2023-09-23T20:57:08.238019+0000","last_became_peered":"2023-09-23T20:57:08.238019+0000","last_unstale":"2023-09-23T21:01:30.014963+0000","last_undegraded":"2023-09-23T21:01:30.014963+0000","last_fullsized":"2023-09-23T21:01:30.014963+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":3,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:20.377714+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:20.377714+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:00:08.305602+0000","scrub_duration":0.003015228,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":128,"reported_epoch":257,"state":"active+clean","last_fresh":"2023-09-23T21:01:30.013125+0000","last_change":"2023-09-23T20:56:45.991884+0000","last_active":"2023-09-23T21:01:30.013125+0000","last_peered":"2023-09-23T21:01:30.013125+0000","last_clean":"2023-09-23T21:01:30.013125+0000","last_became_active":"2023-09-23T20:56:45.991568+0000","last_became_peered":"2023-09-23T20:56:45.991568+0000","last_unstale":"2023-09-23T21:01:30.013125+0000","last_undegraded":"2023-09-23T21:01:30.013125+0000","last_fullsized":"2023-09-23T21:01:30.013125+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":205,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:21:20.775511+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,0],"acting":[1,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":251,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-09-23T21:01:26.740063+0000","last_change":"2023-09-23T20:57:44.979950+0000","last_active":"2023-09-23T21:01:26.740063+0000","last_peered":"2023-09-23T21:01:26.740063+0000","last_clean":"2023-09-23T21:01:26.740063+0000","last_became_active":"2023-09-23T20:57:44.979346+0000","last_became_peered":"2023-09-23T20:57:44.979346+0000","last_unstale":"2023-09-23T21:01:26.740063+0000","last_undegraded":"2023-09-23T21:01:26.740063+0000","last_fullsized":"2023-09-23T21:01:26.740063+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:10:24.372800+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":230,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-09-23T21:01:26.738969+0000","last_change":"2023-09-23T20:50:37.982235+0000","last_active":"2023-09-23T21:01:26.738969+0000","last_peered":"2023-09-23T21:01:26.738969+0000","last_clean":"2023-09-23T21:01:26.738969+0000","last_became_active":"2023-09-23T20:50:37.981913+0000","last_became_peered":"2023-09-23T20:50:37.981913+0000","last_unstale":"2023-09-23T21:01:26.738969+0000","last_undegraded":"2023-09-23T21:01:26.738969+0000","last_fullsized":"2023-09-23T21:01:26.738969+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:59:57.122584+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.f","version":"0'0","reported_seq":11,"reported_epoch":255,"state":"active+clean","last_fresh":"2023-09-23T21:01:25.432417+0000","last_change":"2023-09-23T21:01:25.432176+0000","last_active":"2023-09-23T21:01:25.432417+0000","last_peered":"2023-09-23T21:01:25.432417+0000","last_clean":"2023-09-23T21:01:25.432417+0000","last_became_active":"2023-09-23T21:01:25.431835+0000","last_became_peered":"2023-09-23T21:01:25.431835+0000","last_unstale":"2023-09-23T21:01:25.432417+0000","last_undegraded":"2023-09-23T21:01:25.432417+0000","last_fullsized":"2023-09-23T21:01:25.432417+0000","mapping_epoch":254,"log_start":"0'0","ondisk_log_start":"0'0","created":254,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:01:24.325318+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:01:24.325318+0000","last_clean_scrub_stamp":"2023-09-23T21:01:24.325318+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:47:26.553643+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,1],"acting":[3,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.6","version":"79'161","reported_seq":540,"reported_epoch":257,"state":"active+clean","last_fresh":"2023-09-23T21:01:29.942698+0000","last_change":"2023-09-23T20:57:26.907505+0000","last_active":"2023-09-23T21:01:29.942698+0000","last_peered":"2023-09-23T21:01:29.942698+0000","last_clean":"2023-09-23T21:01:29.942698+0000","last_became_active":"2023-09-23T20:57:08.240321+0000","last_became_peered":"2023-09-23T20:57:08.240321+0000","last_unstale":"2023-09-23T21:01:29.942698+0000","last_undegraded":"2023-09-23T21:01:29.942698+0000","last_fullsized":"2023-09-23T21:01:29.942698+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":3,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:26.907408+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:26.907408+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:34:17.392928+0000","scrub_duration":0.010337038,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":174,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-09-23T21:01:26.738494+0000","last_change":"2023-09-23T20:56:45.988566+0000","last_active":"2023-09-23T21:01:26.738494+0000","last_peered":"2023-09-23T21:01:26.738494+0000","last_clean":"2023-09-23T21:01:26.738494+0000","last_became_active":"2023-09-23T20:56:45.988249+0000","last_became_peered":"2023-09-23T20:56:45.988249+0000","last_unstale":"2023-09-23T21:01:26.738494+0000","last_undegraded":"2023-09-23T21:01:26.738494+0000","last_fullsized":"2023-09-23T21:01:26.738494+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":205,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:51:36.644764+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,4],"acting":[0,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":229,"reported_epoch":257,"state":"active+clean","last_fresh":"2023-09-23T21:01:30.014234+0000","last_change":"2023-09-23T20:54:31.898965+0000","last_active":"2023-09-23T21:01:30.014234+0000","last_peered":"2023-09-23T21:01:30.014234+0000","last_clean":"2023-09-23T21:01:30.014234+0000","last_became_active":"2023-09-23T20:54:31.898663+0000","last_became_peered":"2023-09-23T20:54:31.898663+0000","last_unstale":"2023-09-23T21:01:30.014234+0000","last_undegraded":"2023-09-23T21:01:30.014234+0000","last_fullsized":"2023-09-23T21:01:30.014234+0000","mapping_epoch":136,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:53:05.058087+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":218,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-09-23T21:01:26.739621+0000","last_change":"2023-09-23T20:54:31.884717+0000","last_active":"2023-09-23T21:01:26.739621+0000","last_peered":"2023-09-23T21:01:26.739621+0000","last_clean":"2023-09-23T21:01:26.739621+0000","last_became_active":"2023-09-23T20:54:31.884345+0000","last_became_peered":"2023-09-23T20:54:31.884345+0000","last_unstale":"2023-09-23T21:01:26.739621+0000","last_undegraded":"2023-09-23T21:01:26.739621+0000","last_fullsized":"2023-09-23T21:01:26.739621+0000","mapping_epoch":136,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:02:26.393868+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.c","version":"257'34","reported_seq":46,"reported_epoch":257,"state":"active+clean","last_fresh":"2023-09-23T21:01:31.926432+0000","last_change":"2023-09-23T21:01:25.560907+0000","last_active":"2023-09-23T21:01:31.926432+0000","last_peered":"2023-09-23T21:01:31.926432+0000","last_clean":"2023-09-23T21:01:31.926432+0000","last_became_active":"2023-09-23T21:01:25.560592+0000","last_became_peered":"2023-09-23T21:01:25.560592+0000","last_unstale":"2023-09-23T21:01:31.926432+0000","last_undegraded":"2023-09-23T21:01:31.926432+0000","last_fullsized":"2023-09-23T21:01:31.926432+0000","mapping_epoch":254,"log_start":"0'0","ondisk_log_start":"0'0","created":254,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:01:24.325318+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:01:24.325318+0000","last_clean_scrub_stamp":"2023-09-23T21:01:24.325318+0000","objects_scrubbed":0,"log_size":34,"log_dups_size":0,"ondisk_log_size":34,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:14:22.102068+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":2228224,"num_objects":34,"num_object_clones":0,"num_object_copies":102,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":34,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":34,"num_write_kb":2176,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.5","version":"79'161","reported_seq":521,"reported_epoch":257,"state":"active+clean","last_fresh":"2023-09-23T21:01:30.015103+0000","last_change":"2023-09-23T20:57:30.029479+0000","last_active":"2023-09-23T21:01:30.015103+0000","last_peered":"2023-09-23T21:01:30.015103+0000","last_clean":"2023-09-23T21:01:30.015103+0000","last_became_active":"2023-09-23T20:57:08.249909+0000","last_became_peered":"2023-09-23T20:57:08.249909+0000","last_unstale":"2023-09-23T21:01:30.015103+0000","last_undegraded":"2023-09-23T21:01:30.015103+0000","last_fullsized":"2023-09-23T21:01:30.015103+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":3,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:30.029396+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:30.029396+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:16:46.083145+0000","scrub_duration":0.012914682,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":220,"reported_epoch":255,"state":"active+clean","last_fresh":"2023-09-23T21:01:25.333776+0000","last_change":"2023-09-23T20:51:10.836910+0000","last_active":"2023-09-23T21:01:25.333776+0000","last_peered":"2023-09-23T21:01:25.333776+0000","last_clean":"2023-09-23T21:01:25.333776+0000","last_became_active":"2023-09-23T20:51:10.836557+0000","last_became_peered":"2023-09-23T20:51:10.836557+0000","last_unstale":"2023-09-23T21:01:25.333776+0000","last_undegraded":"2023-09-23T21:01:25.333776+0000","last_fullsized":"2023-09-23T21:01:25.333776+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:51:54.019969+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":229,"reported_epoch":257,"state":"active+clean","last_fresh":"2023-09-23T21:01:30.013987+0000","last_change":"2023-09-23T20:55:35.953444+0000","last_active":"2023-09-23T21:01:30.013987+0000","last_peered":"2023-09-23T21:01:30.013987+0000","last_clean":"2023-09-23T21:01:30.013987+0000","last_became_active":"2023-09-23T20:55:35.953151+0000","last_became_peered":"2023-09-23T20:55:35.953151+0000","last_unstale":"2023-09-23T21:01:30.013987+0000","last_undegraded":"2023-09-23T21:01:30.013987+0000","last_fullsized":"2023-09-23T21:01:30.013987+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:39:12.404391+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":45,"reported_epoch":255,"state":"active+clean","last_fresh":"2023-09-23T21:01:25.332175+0000","last_change":"2023-09-23T20:57:44.982509+0000","last_active":"2023-09-23T21:01:25.332175+0000","last_peered":"2023-09-23T21:01:25.332175+0000","last_clean":"2023-09-23T21:01:25.332175+0000","last_became_active":"2023-09-23T20:57:44.982175+0000","last_became_peered":"2023-09-23T20:57:44.982175+0000","last_unstale":"2023-09-23T21:01:25.332175+0000","last_undegraded":"2023-09-23T21:01:25.332175+0000","last_fullsized":"2023-09-23T21:01:25.332175+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:37:49.215488+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.d","version":"257'53","reported_seq":64,"reported_epoch":257,"state":"active+clean","last_fresh":"2023-09-23T21:01:32.060613+0000","last_change":"2023-09-23T21:01:25.560531+0000","last_active":"2023-09-23T21:01:32.060613+0000","last_peered":"2023-09-23T21:01:32.060613+0000","last_clean":"2023-09-23T21:01:32.060613+0000","last_became_active":"2023-09-23T21:01:25.560228+0000","last_became_peered":"2023-09-23T21:01:25.560228+0000","last_unstale":"2023-09-23T21:01:32.060613+0000","last_undegraded":"2023-09-23T21:01:32.060613+0000","last_fullsized":"2023-09-23T21:01:32.060613+0000","mapping_epoch":254,"log_start":"0'0","ondisk_log_start":"0'0","created":254,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:01:24.325318+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:01:24.325318+0000","last_clean_scrub_stamp":"2023-09-23T21:01:24.325318+0000","objects_scrubbed":0,"log_size":53,"log_dups_size":0,"ondisk_log_size":53,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:19:04.504855+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":3473408,"num_objects":53,"num_object_clones":0,"num_object_copies":159,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":53,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":53,"num_write_kb":3392,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,2],"acting":[5,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.4","version":"79'161","reported_seq":535,"reported_epoch":255,"state":"active+clean","last_fresh":"2023-09-23T21:01:25.332778+0000","last_change":"2023-09-23T20:57:20.012904+0000","last_active":"2023-09-23T21:01:25.332778+0000","last_peered":"2023-09-23T21:01:25.332778+0000","last_clean":"2023-09-23T21:01:25.332778+0000","last_became_active":"2023-09-23T20:57:08.237996+0000","last_became_peered":"2023-09-23T20:57:08.237996+0000","last_unstale":"2023-09-23T21:01:25.332778+0000","last_undegraded":"2023-09-23T21:01:25.332778+0000","last_fullsized":"2023-09-23T21:01:25.332778+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":3,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:20.012764+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:20.012764+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:51:28.796946+0000","scrub_duration":0.0045251919999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":207,"reported_epoch":257,"state":"active+clean","last_fresh":"2023-09-23T21:01:29.942688+0000","last_change":"2023-09-23T20:55:35.949541+0000","last_active":"2023-09-23T21:01:29.942688+0000","last_peered":"2023-09-23T21:01:29.942688+0000","last_clean":"2023-09-23T21:01:29.942688+0000","last_became_active":"2023-09-23T20:55:35.944635+0000","last_became_peered":"2023-09-23T20:55:35.944635+0000","last_unstale":"2023-09-23T21:01:29.942688+0000","last_undegraded":"2023-09-23T21:01:29.942688+0000","last_fullsized":"2023-09-23T21:01:29.942688+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:30:36.033585+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"70'1","reported_seq":246,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-09-23T21:01:26.739058+0000","last_change":"2023-09-23T20:54:14.009723+0000","last_active":"2023-09-23T21:01:26.739058+0000","last_peered":"2023-09-23T21:01:26.739058+0000","last_clean":"2023-09-23T21:01:26.739058+0000","last_became_active":"2023-09-23T20:54:13.941771+0000","last_became_peered":"2023-09-23T20:54:13.941771+0000","last_unstale":"2023-09-23T21:01:26.739058+0000","last_undegraded":"2023-09-23T21:01:26.739058+0000","last_fullsized":"2023-09-23T21:01:26.739058+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:48:25.588116+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":162,"reported_epoch":257,"state":"active+clean","last_fresh":"2023-09-23T21:01:30.014677+0000","last_change":"2023-09-23T20:55:35.949780+0000","last_active":"2023-09-23T21:01:30.014677+0000","last_peered":"2023-09-23T21:01:30.014677+0000","last_clean":"2023-09-23T21:01:30.014677+0000","last_became_active":"2023-09-23T20:55:35.949497+0000","last_became_peered":"2023-09-23T20:55:35.949497+0000","last_unstale":"2023-09-23T21:01:30.014677+0000","last_undegraded":"2023-09-23T21:01:30.014677+0000","last_fullsized":"2023-09-23T21:01:30.014677+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:09:37.420882+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":220,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-09-23T21:01:26.739544+0000","last_change":"2023-09-23T20:54:13.929196+0000","last_active":"2023-09-23T21:01:26.739544+0000","last_peered":"2023-09-23T21:01:26.739544+0000","last_clean":"2023-09-23T21:01:26.739544+0000","last_became_active":"2023-09-23T20:54:13.928831+0000","last_became_peered":"2023-09-23T20:54:13.928831+0000","last_unstale":"2023-09-23T21:01:26.739544+0000","last_undegraded":"2023-09-23T21:01:26.739544+0000","last_fullsized":"2023-09-23T21:01:26.739544+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:28:44.316851+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":226,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-09-23T21:01:26.738455+0000","last_change":"2023-09-23T20:52:12.855430+0000","last_active":"2023-09-23T21:01:26.738455+0000","last_peered":"2023-09-23T21:01:26.738455+0000","last_clean":"2023-09-23T21:01:26.738455+0000","last_became_active":"2023-09-23T20:52:12.855115+0000","last_became_peered":"2023-09-23T20:52:12.855115+0000","last_unstale":"2023-09-23T21:01:26.738455+0000","last_undegraded":"2023-09-23T21:01:26.738455+0000","last_fullsized":"2023-09-23T21:01:26.738455+0000","mapping_epoch":100,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":101,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:33:23.802214+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":134,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-09-23T21:01:26.741886+0000","last_change":"2023-09-23T20:54:31.902946+0000","last_active":"2023-09-23T21:01:26.741886+0000","last_peered":"2023-09-23T21:01:26.741886+0000","last_clean":"2023-09-23T21:01:26.741886+0000","last_became_active":"2023-09-23T20:54:31.889484+0000","last_became_peered":"2023-09-23T20:54:31.889484+0000","last_unstale":"2023-09-23T21:01:26.741886+0000","last_undegraded":"2023-09-23T21:01:26.741886+0000","last_fullsized":"2023-09-23T21:01:26.741886+0000","mapping_epoch":136,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:44:44.278398+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":135,"reported_epoch":257,"state":"active+clean","last_fresh":"2023-09-23T21:01:30.015443+0000","last_change":"2023-09-23T20:54:13.950351+0000","last_active":"2023-09-23T21:01:30.015443+0000","last_peered":"2023-09-23T21:01:30.015443+0000","last_clean":"2023-09-23T21:01:30.015443+0000","last_became_active":"2023-09-23T20:54:13.949987+0000","last_became_peered":"2023-09-23T20:54:13.949987+0000","last_unstale":"2023-09-23T21:01:30.015443+0000","last_undegraded":"2023-09-23T21:01:30.015443+0000","last_fullsized":"2023-09-23T21:01:30.015443+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:46:17.765134+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":165,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-09-23T21:01:26.742555+0000","last_change":"2023-09-23T20:56:18.141728+0000","last_active":"2023-09-23T21:01:26.742555+0000","last_peered":"2023-09-23T21:01:26.742555+0000","last_clean":"2023-09-23T21:01:26.742555+0000","last_became_active":"2023-09-23T20:56:18.137979+0000","last_became_peered":"2023-09-23T20:56:18.137979+0000","last_unstale":"2023-09-23T21:01:26.742555+0000","last_undegraded":"2023-09-23T21:01:26.742555+0000","last_fullsized":"2023-09-23T21:01:26.742555+0000","mapping_epoch":192,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":193,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:58:05.658443+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":135,"reported_epoch":257,"state":"active+clean","last_fresh":"2023-09-23T21:01:30.015156+0000","last_change":"2023-09-23T20:54:13.935307+0000","last_active":"2023-09-23T21:01:30.015156+0000","last_peered":"2023-09-23T21:01:30.015156+0000","last_clean":"2023-09-23T21:01:30.015156+0000","last_became_active":"2023-09-23T20:54:13.935000+0000","last_became_peered":"2023-09-23T20:54:13.935000+0000","last_unstale":"2023-09-23T21:01:30.015156+0000","last_undegraded":"2023-09-23T21:01:30.015156+0000","last_fullsized":"2023-09-23T21:01:30.015156+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:47:40.974082+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":244,"reported_epoch":255,"state":"active+clean","last_fresh":"2023-09-23T21:01:25.334086+0000","last_change":"2023-09-23T20:55:35.954870+0000","last_active":"2023-09-23T21:01:25.334086+0000","last_peered":"2023-09-23T21:01:25.334086+0000","last_clean":"2023-09-23T21:01:25.334086+0000","last_became_active":"2023-09-23T20:55:35.954568+0000","last_became_peered":"2023-09-23T20:55:35.954568+0000","last_unstale":"2023-09-23T21:01:25.334086+0000","last_undegraded":"2023-09-23T21:01:25.334086+0000","last_fullsized":"2023-09-23T21:01:25.334086+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:06:22.423566+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":243,"reported_epoch":255,"state":"active+clean","last_fresh":"2023-09-23T21:01:25.332189+0000","last_change":"2023-09-23T20:51:10.838334+0000","last_active":"2023-09-23T21:01:25.332189+0000","last_peered":"2023-09-23T21:01:25.332189+0000","last_clean":"2023-09-23T21:01:25.332189+0000","last_became_active":"2023-09-23T20:51:10.838026+0000","last_became_peered":"2023-09-23T20:51:10.838026+0000","last_unstale":"2023-09-23T21:01:25.332189+0000","last_undegraded":"2023-09-23T21:01:25.332189+0000","last_fullsized":"2023-09-23T21:01:25.332189+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:04:37.009457+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":248,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-09-23T21:01:26.739428+0000","last_change":"2023-09-23T20:54:31.903139+0000","last_active":"2023-09-23T21:01:26.739428+0000","last_peered":"2023-09-23T21:01:26.739428+0000","last_clean":"2023-09-23T21:01:26.739428+0000","last_became_active":"2023-09-23T20:54:31.884520+0000","last_became_peered":"2023-09-23T20:54:31.884520+0000","last_unstale":"2023-09-23T21:01:26.739428+0000","last_undegraded":"2023-09-23T21:01:26.739428+0000","last_fullsized":"2023-09-23T21:01:26.739428+0000","mapping_epoch":136,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:39:34.269705+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1108,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":152,"reported_epoch":257,"state":"active+clean","last_fresh":"2023-09-23T21:01:30.014543+0000","last_change":"2023-09-23T20:55:35.951572+0000","last_active":"2023-09-23T21:01:30.014543+0000","last_peered":"2023-09-23T21:01:30.014543+0000","last_clean":"2023-09-23T21:01:30.014543+0000","last_became_active":"2023-09-23T20:55:35.951294+0000","last_became_peered":"2023-09-23T20:55:35.951294+0000","last_unstale":"2023-09-23T21:01:30.014543+0000","last_undegraded":"2023-09-23T21:01:30.014543+0000","last_fullsized":"2023-09-23T21:01:30.014543+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:36:32.447210+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":22,"num_bytes_recovered":5990,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":8,"num_pg":16,"stat_sum":{"num_bytes":203685888,"num_objects":3108,"num_object_clones":0,"num_object_copies":9324,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3108,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3108,"num_write_kb":198912,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":450887680,"data_stored":450887680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3108,"ondisk_log_size":3108,"up":48,"acting":48,"num_store_stats":8},{"poolid":1,"num_pg":21,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3381,"ondisk_log_size":3381,"up":63,"acting":63,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201533,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":206848,"kb_used_data":5176,"kb_used_omap":0,"kb_used_meta":201664,"kb_avail":93538304,"statfs":{"total":95995035648,"available":95783223296,"internally_reserved":0,"allocated":5300224,"data_stored":4396178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":206503936},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:00:49 2023","interfaces":[{"interface":"back","average":{"1min":3.766,"5min":2.3260000000000001,"15min":1.302},"min":{"1min":0.23200000000000001,"5min":0.216,"15min":0.21099999999999999},"max":{"1min":33.326000000000001,"5min":43.994999999999997,"15min":43.994999999999997},"last":0.35999999999999999},{"interface":"front","average":{"1min":3.8399999999999999,"5min":2.3530000000000002,"15min":1.389},"min":{"1min":0.33200000000000002,"5min":0.28199999999999997,"15min":0.218},"max":{"1min":33.354999999999997,"5min":44.206000000000003,"15min":44.206000000000003},"last":0.45300000000000001}]},{"osd":1,"last update":"Sat Sep 23 21:01:25 2023","interfaces":[{"interface":"back","average":{"1min":0.83099999999999996,"5min":1.3660000000000001,"15min":1.302},"min":{"1min":0.253,"5min":0.253,"15min":0.23300000000000001},"max":{"1min":2.9340000000000002,"5min":10.622,"15min":22.591999999999999},"last":0.47999999999999998},{"interface":"front","average":{"1min":1.603,"5min":1.625,"15min":1.242},"min":{"1min":0.33400000000000002,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":13.981,"5min":13.981,"15min":21.437000000000001},"last":0.56699999999999995}]},{"osd":2,"last update":"Sat Sep 23 21:00:59 2023","interfaces":[{"interface":"back","average":{"1min":2.3799999999999999,"5min":2.1699999999999999,"15min":1.4159999999999999},"min":{"1min":0.313,"5min":0.26500000000000001,"15min":0.24399999999999999},"max":{"1min":7.2050000000000001,"5min":10.805999999999999,"15min":10.805999999999999},"last":0.41499999999999998},{"interface":"front","average":{"1min":2.976,"5min":2.399,"15min":1.4690000000000001},"min":{"1min":0.35499999999999998,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":9.6660000000000004,"5min":45.25,"15min":45.25},"last":0.64800000000000002}]},{"osd":3,"last update":"Sat Sep 23 21:00:32 2023","interfaces":[{"interface":"back","average":{"1min":1.9099999999999999,"5min":2.157,"15min":1.8149999999999999},"min":{"1min":0.39300000000000002,"5min":0.32100000000000001,"15min":0.314},"max":{"1min":5.2549999999999999,"5min":9.9939999999999998,"15min":36.408000000000001},"last":0.60599999999999998},{"interface":"front","average":{"1min":3.5870000000000002,"5min":2.5219999999999998,"15min":1.7529999999999999},"min":{"1min":0.40699999999999997,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":13.83,"5min":14.220000000000001,"15min":35.593000000000004},"last":0.66900000000000004}]},{"osd":4,"last update":"Sat Sep 23 21:01:25 2023","interfaces":[{"interface":"back","average":{"1min":0.36499999999999999,"5min":0.57099999999999995,"15min":0.622},"min":{"1min":0.28499999999999998,"5min":0.25700000000000001,"15min":0.23699999999999999},"max":{"1min":0.60699999999999998,"5min":4.5410000000000004,"15min":5.8920000000000003},"last":0.69499999999999995},{"interface":"front","average":{"1min":0.52400000000000002,"5min":0.627,"15min":0.64600000000000002},"min":{"1min":0.30099999999999999,"5min":0.27100000000000002,"15min":0.26300000000000001},"max":{"1min":1.6259999999999999,"5min":4.4080000000000004,"15min":5.3200000000000003},"last":0.59399999999999997}]},{"osd":5,"last update":"Sat Sep 23 21:01:25 2023","interfaces":[{"interface":"back","average":{"1min":0.44700000000000001,"5min":0.55000000000000004,"15min":0.63500000000000001},"min":{"1min":0.29899999999999999,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":0.77900000000000003,"5min":2.774,"15min":7.0389999999999997},"last":0.55000000000000004},{"interface":"front","average":{"1min":0.47499999999999998,"5min":0.64600000000000002,"15min":0.63900000000000001},"min":{"1min":0.32100000000000001,"5min":0.30199999999999999,"15min":0.27800000000000002},"max":{"1min":0.65200000000000002,"5min":4.5999999999999996,"15min":4.5999999999999996},"last":0.74399999999999999}]},{"osd":6,"last update":"Sat Sep 23 21:00:46 2023","interfaces":[{"interface":"back","average":{"1min":0.85599999999999998,"5min":0.71799999999999997,"15min":0.68400000000000005},"min":{"1min":0.38200000000000001,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":3.4329999999999998,"5min":3.5249999999999999,"15min":3.5249999999999999},"last":0.51000000000000001},{"interface":"front","average":{"1min":1.0620000000000001,"5min":0.80100000000000005,"15min":0.80300000000000005},"min":{"1min":0.40899999999999997,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":3.5569999999999999,"5min":3.5569999999999999,"15min":3.5569999999999999},"last":0.63100000000000001}]}]},{"osd":6,"up_from":164,"seq":704374636617,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":231928,"kb_used_data":115504,"kb_used_omap":10,"kb_used_meta":116405,"kb_avail":93513224,"statfs":{"total":95995035648,"available":95757541376,"internally_reserved":0,"allocated":118276096,"data_stored":117388019,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":10328,"internal_metadata":119199656},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:00:46 2023","interfaces":[{"interface":"back","average":{"1min":2.1469999999999998,"5min":1.7569999999999999,"15min":1.1619999999999999},"min":{"1min":0.26400000000000001,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":10.404,"5min":10.404,"15min":10.404},"last":0.41999999999999998},{"interface":"front","average":{"1min":2.895,"5min":1.9890000000000001,"15min":1.3180000000000001},"min":{"1min":0.28599999999999998,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":11.023,"5min":11.023,"15min":11.023},"last":1.1679999999999999}]},{"osd":1,"last update":"Sat Sep 23 21:00:46 2023","interfaces":[{"interface":"back","average":{"1min":1.601,"5min":1.4119999999999999,"15min":0.996},"min":{"1min":0.32900000000000001,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":3.839,"5min":8.4309999999999992,"15min":8.4309999999999992},"last":0.34300000000000003},{"interface":"front","average":{"1min":2.6560000000000001,"5min":1.7290000000000001,"15min":1.155},"min":{"1min":0.26200000000000001,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":10.518000000000001,"5min":10.518000000000001,"15min":10.518000000000001},"last":1.8759999999999999}]},{"osd":2,"last update":"Sat Sep 23 21:00:46 2023","interfaces":[{"interface":"back","average":{"1min":2.9249999999999998,"5min":2.3279999999999998,"15min":1.8740000000000001},"min":{"1min":0.371,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":10.109999999999999,"5min":13.750999999999999,"15min":13.750999999999999},"last":0.312},{"interface":"front","average":{"1min":3.1920000000000002,"5min":2.2170000000000001,"15min":1.4770000000000001},"min":{"1min":0.34599999999999997,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":20.195,"5min":20.195,"15min":20.195},"last":1.3620000000000001}]},{"osd":3,"last update":"Sat Sep 23 21:00:46 2023","interfaces":[{"interface":"back","average":{"1min":4.1660000000000004,"5min":2.0249999999999999,"15min":1.419},"min":{"1min":0.48699999999999999,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":14.144,"5min":14.144,"15min":14.144},"last":0.39400000000000002},{"interface":"front","average":{"1min":2.7919999999999998,"5min":1.986,"15min":1.73},"min":{"1min":0.38700000000000001,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":11.989000000000001,"5min":11.989000000000001,"15min":11.989000000000001},"last":3.262}]},{"osd":4,"last update":"Sat Sep 23 21:00:46 2023","interfaces":[{"interface":"back","average":{"1min":0.81599999999999995,"5min":0.73899999999999999,"15min":0.94699999999999995},"min":{"1min":0.28100000000000003,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":3.4260000000000002,"5min":9.0079999999999991,"15min":9.0079999999999991},"last":0.24299999999999999},{"interface":"front","average":{"1min":0.626,"5min":0.51800000000000002,"15min":0.52600000000000002},"min":{"1min":0.29799999999999999,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":2.4860000000000002,"5min":2.4860000000000002,"15min":2.4860000000000002},"last":0.35499999999999998}]},{"osd":5,"last update":"Sat Sep 23 21:00:46 2023","interfaces":[{"interface":"back","average":{"1min":0.94499999999999995,"5min":0.69799999999999995,"15min":0.59299999999999997},"min":{"1min":0.32700000000000001,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":5.4119999999999999,"5min":5.4119999999999999,"15min":5.4119999999999999},"last":0.27500000000000002},{"interface":"front","average":{"1min":0.90600000000000003,"5min":0.75900000000000001,"15min":0.79300000000000004},"min":{"1min":0.34499999999999997,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":3.536,"5min":3.536,"15min":3.536},"last":0.29199999999999998}]},{"osd":7,"last update":"Sat Sep 23 21:00:46 2023","interfaces":[{"interface":"back","average":{"1min":1.151,"5min":0.81599999999999995,"15min":0.89100000000000001},"min":{"1min":0.36299999999999999,"5min":0.312,"15min":0.312},"max":{"1min":3.8929999999999998,"5min":7.8570000000000002,"15min":7.8570000000000002},"last":0.33000000000000002},{"interface":"front","average":{"1min":1.0700000000000001,"5min":0.75700000000000001,"15min":0.60499999999999998},"min":{"1min":0.40100000000000002,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":3.3279999999999998,"5min":8.282,"15min":8.282},"last":0.28100000000000003}]}]},{"osd":1,"up_from":88,"seq":377957122188,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":778644,"kb_used_data":479500,"kb_used_omap":9,"kb_used_meta":299126,"kb_avail":92966508,"statfs":{"total":95995035648,"available":95197704192,"internally_reserved":0,"allocated":491008000,"data_stored":489747877,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9239,"internal_metadata":306306025},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:01:30 2023","interfaces":[{"interface":"back","average":{"1min":1.399,"5min":2.7629999999999999,"15min":1.7769999999999999},"min":{"1min":0.218,"5min":0.20699999999999999,"15min":0.20699999999999999},"max":{"1min":8.2230000000000008,"5min":16.646999999999998,"15min":16.646999999999998},"last":0.97399999999999998},{"interface":"front","average":{"1min":1.796,"5min":3.327,"15min":2.0600000000000001},"min":{"1min":0.27600000000000002,"5min":0.246,"15min":0.22500000000000001},"max":{"1min":10.638999999999999,"5min":27.797999999999998,"15min":27.797999999999998},"last":0.44400000000000001}]},{"osd":2,"last update":"Sat Sep 23 21:00:57 2023","interfaces":[{"interface":"back","average":{"1min":2.3559999999999999,"5min":2.198,"15min":1.4510000000000001},"min":{"1min":0.251,"5min":0.221,"15min":0.20300000000000001},"max":{"1min":7.1280000000000001,"5min":13.752000000000001,"15min":14.335000000000001},"last":0.95699999999999996},{"interface":"front","average":{"1min":5.29,"5min":3.9430000000000001,"15min":2.262},"min":{"1min":0.23699999999999999,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":17.405999999999999,"5min":18.026,"15min":38.951000000000001},"last":3.1389999999999998}]},{"osd":3,"last update":"Sat Sep 23 21:01:30 2023","interfaces":[{"interface":"back","average":{"1min":1.591,"5min":2.5339999999999998,"15min":1.8759999999999999},"min":{"1min":0.26000000000000001,"5min":0.246,"15min":0.246},"max":{"1min":7.9269999999999996,"5min":22.686,"15min":25.314},"last":1.403},{"interface":"front","average":{"1min":1.52,"5min":2.4540000000000002,"15min":1.6870000000000001},"min":{"1min":0.33000000000000002,"5min":0.29799999999999999,"15min":0.20599999999999999},"max":{"1min":6.8099999999999996,"5min":14.473000000000001,"15min":19.881},"last":1.0289999999999999}]},{"osd":4,"last update":"Sat Sep 23 21:01:30 2023","interfaces":[{"interface":"back","average":{"1min":0.77400000000000002,"5min":1.4850000000000001,"15min":1.337},"min":{"1min":0.311,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":2.7570000000000001,"5min":18.231000000000002,"15min":18.231000000000002},"last":1.5660000000000001},{"interface":"front","average":{"1min":1.252,"5min":1.5509999999999999,"15min":1.181},"min":{"1min":0.44,"5min":0.311,"15min":0.27600000000000002},"max":{"1min":6.7290000000000001,"5min":7.532,"15min":11.234999999999999},"last":0.91900000000000004}]},{"osd":5,"last update":"Sat Sep 23 21:01:30 2023","interfaces":[{"interface":"back","average":{"1min":1.9099999999999999,"5min":2.202,"15min":1.5},"min":{"1min":0.373,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":8.1929999999999996,"5min":15.041,"15min":15.041},"last":0.34699999999999998},{"interface":"front","average":{"1min":0.82899999999999996,"5min":1.5149999999999999,"15min":1.369},"min":{"1min":0.35999999999999999,"5min":0.33200000000000002,"15min":0.32200000000000001},"max":{"1min":2.8490000000000002,"5min":18.280999999999999,"15min":18.280999999999999},"last":1.53}]},{"osd":6,"last update":"Sat Sep 23 21:00:45 2023","interfaces":[{"interface":"back","average":{"1min":2.298,"5min":1.7569999999999999,"15min":1.462},"min":{"1min":0.35799999999999998,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":18.984999999999999,"5min":18.984999999999999,"15min":18.984999999999999},"last":1.476},{"interface":"front","average":{"1min":1.5840000000000001,"5min":1.696,"15min":1.3999999999999999},"min":{"1min":0.38900000000000001,"5min":0.309,"15min":0.309},"max":{"1min":6.7779999999999996,"5min":8.7530000000000001,"15min":8.7530000000000001},"last":1.012}]},{"osd":7,"last update":"Sat Sep 23 21:01:30 2023","interfaces":[{"interface":"back","average":{"1min":1.2450000000000001,"5min":1.6879999999999999,"15min":1.278},"min":{"1min":0.47799999999999998,"5min":0.39600000000000002,"15min":0.35699999999999998},"max":{"1min":6.8419999999999996,"5min":8.7769999999999992,"15min":11.186999999999999},"last":1.044},{"interface":"front","average":{"1min":1.645,"5min":2.2130000000000001,"15min":1.504},"min":{"1min":0.40300000000000002,"5min":0.40300000000000002,"15min":0.34599999999999997},"max":{"1min":8.2309999999999999,"5min":15.098000000000001,"15min":15.098000000000001},"last":0.5}]}]},{"osd":0,"up_from":83,"seq":356482285714,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":245876,"kb_used_data":11308,"kb_used_omap":28,"kb_used_meta":234531,"kb_avail":93499276,"statfs":{"total":95995035648,"available":95743258624,"internally_reserved":0,"allocated":11579392,"data_stored":10341519,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":28848,"internal_metadata":240160592},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 21:01:20 2023","interfaces":[{"interface":"back","average":{"1min":2.6949999999999998,"5min":3.7280000000000002,"15min":2.0499999999999998},"min":{"1min":0.24199999999999999,"5min":0.20000000000000001,"15min":0.191},"max":{"1min":14.231,"5min":19.146000000000001,"15min":19.146000000000001},"last":0.308},{"interface":"front","average":{"1min":2.5099999999999998,"5min":3.5379999999999998,"15min":2.0169999999999999},"min":{"1min":0.24099999999999999,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":12.708,"5min":15.771000000000001,"15min":17.704999999999998},"last":0.33400000000000002}]},{"osd":2,"last update":"Sat Sep 23 21:00:55 2023","interfaces":[{"interface":"back","average":{"1min":3.9039999999999999,"5min":3.798,"15min":2.1000000000000001},"min":{"1min":0.32100000000000001,"5min":0.30599999999999999,"15min":0.23799999999999999},"max":{"1min":10.914,"5min":21.053000000000001,"15min":21.053000000000001},"last":0.48999999999999999},{"interface":"front","average":{"1min":4.1200000000000001,"5min":4.0369999999999999,"15min":2.1930000000000001},"min":{"1min":0.36099999999999999,"5min":0.25800000000000001,"15min":0.21099999999999999},"max":{"1min":9.548,"5min":21.747,"15min":21.747},"last":0.35199999999999998}]},{"osd":3,"last update":"Sat Sep 23 21:01:20 2023","interfaces":[{"interface":"back","average":{"1min":1.4470000000000001,"5min":3.6360000000000001,"15min":2.069},"min":{"1min":0.27100000000000002,"5min":0.27100000000000002,"15min":0.25800000000000001},"max":{"1min":8.0679999999999996,"5min":21.184000000000001,"15min":21.184000000000001},"last":0.44500000000000001},{"interface":"front","average":{"1min":2.448,"5min":3.6789999999999998,"15min":2.2210000000000001},"min":{"1min":0.27400000000000002,"5min":0.27400000000000002,"15min":0.23300000000000001},"max":{"1min":12.167999999999999,"5min":21.167000000000002,"15min":21.167000000000002},"last":0.38100000000000001}]},{"osd":4,"last update":"Sat Sep 23 21:00:55 2023","interfaces":[{"interface":"back","average":{"1min":2.52,"5min":2.875,"15min":1.6839999999999999},"min":{"1min":0.39200000000000002,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":9.6029999999999998,"5min":21.594999999999999,"15min":21.594999999999999},"last":0.42999999999999999},{"interface":"front","average":{"1min":3.786,"5min":2.4020000000000001,"15min":1.486},"min":{"1min":0.437,"5min":0.31,"15min":0.28100000000000003},"max":{"1min":13.904,"5min":13.904,"15min":18.724},"last":0.51800000000000002}]},{"osd":5,"last update":"Sat Sep 23 21:00:55 2023","interfaces":[{"interface":"back","average":{"1min":2.794,"5min":2.1339999999999999,"15min":1.401},"min":{"1min":0.309,"5min":0.309,"15min":0.309},"max":{"1min":6.343,"5min":8.8970000000000002,"15min":8.8970000000000002},"last":0.56200000000000006},{"interface":"front","average":{"1min":2.5430000000000001,"5min":2.7109999999999999,"15min":1.6870000000000001},"min":{"1min":0.38700000000000001,"5min":0.38700000000000001,"15min":0.34999999999999998},"max":{"1min":9.6340000000000003,"5min":20.512,"15min":20.512},"last":0.47899999999999998}]},{"osd":6,"last update":"Sat Sep 23 21:00:50 2023","interfaces":[{"interface":"back","average":{"1min":3.4049999999999998,"5min":2.4950000000000001,"15min":1.9359999999999999},"min":{"1min":0.48199999999999998,"5min":0.37,"15min":0.37},"max":{"1min":13.933,"5min":13.933,"15min":13.933},"last":0.58499999999999996},{"interface":"front","average":{"1min":3.8260000000000001,"5min":2.6139999999999999,"15min":2.0350000000000001},"min":{"1min":0.46400000000000002,"5min":0.377,"15min":0.377},"max":{"1min":13.962,"5min":13.962,"15min":13.962},"last":0.65600000000000003}]},{"osd":7,"last update":"Sat Sep 23 21:00:55 2023","interfaces":[{"interface":"back","average":{"1min":2.7919999999999998,"5min":2.254,"15min":1.4570000000000001},"min":{"1min":0.40300000000000002,"5min":0.379,"15min":0.379},"max":{"1min":6.298,"5min":8.7729999999999997,"15min":8.7729999999999997},"last":0.63300000000000001},{"interface":"front","average":{"1min":3.605,"5min":2.4830000000000001,"15min":1.516},"min":{"1min":0.49099999999999999,"5min":0.377,"15min":0.377},"max":{"1min":13.856999999999999,"5min":13.856999999999999,"15min":18.673999999999999},"last":0.623}]}]},{"osd":2,"up_from":93,"seq":399431958661,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":658604,"kb_used_data":322916,"kb_used_omap":81,"kb_used_meta":335598,"kb_avail":93086548,"statfs":{"total":95995035648,"available":95320625152,"internally_reserved":0,"allocated":330665984,"data_stored":329433080,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":83750,"internal_metadata":343652570},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:00:50 2023","interfaces":[{"interface":"back","average":{"1min":4.3099999999999996,"5min":3.1720000000000002,"15min":1.6439999999999999},"min":{"1min":0.27300000000000002,"5min":0.17999999999999999,"15min":0.17999999999999999},"max":{"1min":13.494,"5min":29.498000000000001,"15min":29.498000000000001},"last":0.38400000000000001},{"interface":"front","average":{"1min":4.2800000000000002,"5min":3.681,"15min":1.9319999999999999},"min":{"1min":0.29699999999999999,"5min":0.20599999999999999,"15min":0.20599999999999999},"max":{"1min":11.629,"5min":17.251000000000001,"15min":17.251000000000001},"last":0.41099999999999998}]},{"osd":1,"last update":"Sat Sep 23 21:00:50 2023","interfaces":[{"interface":"back","average":{"1min":2.7599999999999998,"5min":2.661,"15min":1.782},"min":{"1min":0.27800000000000002,"5min":0.21199999999999999,"15min":0.20200000000000001},"max":{"1min":13.523,"5min":14.709,"15min":16.081},"last":0.33600000000000002},{"interface":"front","average":{"1min":6.0609999999999999,"5min":3.952,"15min":2.3460000000000001},"min":{"1min":0.26300000000000001,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":26.552,"5min":26.552,"15min":26.552},"last":0.35599999999999998}]},{"osd":3,"last update":"Sat Sep 23 21:01:31 2023","interfaces":[{"interface":"back","average":{"1min":3.8500000000000001,"5min":4.1210000000000004,"15min":2.4089999999999998},"min":{"1min":0.251,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":41.057000000000002,"5min":41.057000000000002,"15min":41.057000000000002},"last":0.29399999999999998},{"interface":"front","average":{"1min":4.4669999999999996,"5min":4.4139999999999997,"15min":2.726},"min":{"1min":0.26000000000000001,"5min":0.26000000000000001,"15min":0.221},"max":{"1min":41.012,"5min":41.012,"15min":41.012},"last":0.39300000000000002}]},{"osd":4,"last update":"Sat Sep 23 21:00:50 2023","interfaces":[{"interface":"back","average":{"1min":2.1099999999999999,"5min":1.8400000000000001,"15min":1.3069999999999999},"min":{"1min":0.29299999999999998,"5min":0.28699999999999998,"15min":0.26400000000000001},"max":{"1min":10.045,"5min":10.045,"15min":10.045},"last":0.60499999999999998},{"interface":"front","average":{"1min":3.0459999999999998,"5min":2.222,"15min":1.53},"min":{"1min":0.42699999999999999,"5min":0.313,"15min":0.313},"max":{"1min":14.039999999999999,"5min":14.039999999999999,"15min":14.039999999999999},"last":0.77900000000000003}]},{"osd":5,"last update":"Sat Sep 23 21:00:50 2023","interfaces":[{"interface":"back","average":{"1min":2.1150000000000002,"5min":1.857,"15min":1.3959999999999999},"min":{"1min":0.34200000000000003,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":10.069000000000001,"5min":10.069000000000001,"15min":10.069000000000001},"last":0.73199999999999998},{"interface":"front","average":{"1min":3.0329999999999999,"5min":2.1230000000000002,"15min":1.532},"min":{"1min":0.46600000000000003,"5min":0.32600000000000001,"15min":0.30299999999999999},"max":{"1min":13.98,"5min":13.98,"15min":13.98},"last":0.59699999999999998}]},{"osd":6,"last update":"Sat Sep 23 21:00:47 2023","interfaces":[{"interface":"back","average":{"1min":3.2559999999999998,"5min":2.3820000000000001,"15min":1.9059999999999999},"min":{"1min":0.54500000000000004,"5min":0.312,"15min":0.312},"max":{"1min":14.016,"5min":14.244999999999999,"15min":14.244999999999999},"last":0.71899999999999997},{"interface":"front","average":{"1min":3.702,"5min":2.3479999999999999,"15min":1.4690000000000001},"min":{"1min":0.61399999999999999,"5min":0.374,"15min":0.374},"max":{"1min":14.068,"5min":14.068,"15min":14.068},"last":0.63700000000000001}]},{"osd":7,"last update":"Sat Sep 23 21:00:50 2023","interfaces":[{"interface":"back","average":{"1min":2.0670000000000002,"5min":1.496,"15min":1.1399999999999999},"min":{"1min":0.41899999999999998,"5min":0.315,"15min":0.315},"max":{"1min":13.427,"5min":13.427,"15min":13.427},"last":0.76500000000000001},{"interface":"front","average":{"1min":2.29,"5min":1.907,"15min":1.401},"min":{"1min":0.372,"5min":0.35099999999999998,"15min":0.34499999999999997},"max":{"1min":10.092000000000001,"5min":10.092000000000001,"15min":10.092000000000001},"last":0.81499999999999995}]}]},{"osd":3,"up_from":98,"seq":420906795134,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":414772,"kb_used_data":3564,"kb_used_omap":7,"kb_used_meta":411192,"kb_avail":93330380,"statfs":{"total":95995035648,"available":95570309120,"internally_reserved":0,"allocated":3649536,"data_stored":2459918,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7576,"internal_metadata":421061224},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:00:28 2023","interfaces":[{"interface":"back","average":{"1min":5.6130000000000004,"5min":3.0470000000000002,"15min":2.0840000000000001},"min":{"1min":0.20399999999999999,"5min":0.20399999999999999,"15min":0.20399999999999999},"max":{"1min":27.962,"5min":27.962,"15min":27.962},"last":0.26900000000000002},{"interface":"front","average":{"1min":4.984,"5min":3.5169999999999999,"15min":1.9339999999999999},"min":{"1min":0.21099999999999999,"5min":0.19800000000000001,"15min":0.19800000000000001},"max":{"1min":28.058,"5min":28.058,"15min":28.058},"last":0.34399999999999997}]},{"osd":1,"last update":"Sat Sep 23 21:00:28 2023","interfaces":[{"interface":"back","average":{"1min":4.8730000000000002,"5min":3.1760000000000002,"15min":1.9119999999999999},"min":{"1min":0.22600000000000001,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":22.681999999999999,"5min":22.681999999999999,"15min":22.681999999999999},"last":0.371},{"interface":"front","average":{"1min":4.0789999999999997,"5min":2.5169999999999999,"15min":1.929},"min":{"1min":0.25,"5min":0.218,"15min":0.218},"max":{"1min":19.329000000000001,"5min":19.329000000000001,"15min":25.353000000000002},"last":0.39800000000000002}]},{"osd":2,"last update":"Sat Sep 23 21:00:28 2023","interfaces":[{"interface":"back","average":{"1min":6.1680000000000001,"5min":3.8039999999999998,"15min":2.113},"min":{"1min":0.36199999999999999,"5min":0.33000000000000002,"15min":0.30099999999999999},"max":{"1min":28.251999999999999,"5min":28.251999999999999,"15min":28.251999999999999},"last":0.41899999999999998},{"interface":"front","average":{"1min":4.6779999999999999,"5min":2.9300000000000002,"15min":2.0750000000000002},"min":{"1min":0.372,"5min":0.23999999999999999,"15min":0.22600000000000001},"max":{"1min":16.213999999999999,"5min":16.213999999999999,"15min":25.305},"last":0.44}]},{"osd":4,"last update":"Sat Sep 23 21:00:28 2023","interfaces":[{"interface":"back","average":{"1min":3.258,"5min":2.2869999999999999,"15min":1.4750000000000001},"min":{"1min":0.314,"5min":0.29899999999999999,"15min":0.29299999999999998},"max":{"1min":18.239000000000001,"5min":18.239000000000001,"15min":18.239000000000001},"last":0.90000000000000002},{"interface":"front","average":{"1min":2.7200000000000002,"5min":1.5860000000000001,"15min":1.474},"min":{"1min":0.317,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":14.808,"5min":14.808,"15min":22.853000000000002},"last":0.69199999999999995}]},{"osd":5,"last update":"Sat Sep 23 21:00:28 2023","interfaces":[{"interface":"back","average":{"1min":3.2930000000000001,"5min":2.3399999999999999,"15min":1.4690000000000001},"min":{"1min":0.33200000000000002,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":18.324999999999999,"5min":18.324999999999999,"15min":18.324999999999999},"last":0.84599999999999997},{"interface":"front","average":{"1min":3.0910000000000002,"5min":2.363,"15min":1.4359999999999999},"min":{"1min":0.33700000000000002,"5min":0.33300000000000002,"15min":0.33100000000000002},"max":{"1min":11.327,"5min":13.43,"15min":13.43},"last":0.81000000000000005}]},{"osd":6,"last update":"Sat Sep 23 21:00:51 2023","interfaces":[{"interface":"back","average":{"1min":3.601,"5min":1.9450000000000001,"15min":1.2809999999999999},"min":{"1min":0.373,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":14.833,"5min":14.833,"15min":14.833},"last":0.96299999999999997},{"interface":"front","average":{"1min":4.641,"5min":2.6110000000000002,"15min":1.8109999999999999},"min":{"1min":0.40899999999999997,"5min":0.38700000000000001,"15min":0.38700000000000001},"max":{"1min":13.576000000000001,"5min":13.576000000000001,"15min":13.576000000000001},"last":0.88400000000000001}]},{"osd":7,"last update":"Sat Sep 23 21:00:28 2023","interfaces":[{"interface":"back","average":{"1min":3.2879999999999998,"5min":2.5649999999999999,"15min":1.6399999999999999},"min":{"1min":0.39200000000000002,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":18.289999999999999,"5min":18.289999999999999,"15min":18.289999999999999},"last":0.93899999999999995},{"interface":"front","average":{"1min":3.1219999999999999,"5min":2.399,"15min":1.506},"min":{"1min":0.38900000000000001,"5min":0.38900000000000001,"15min":0.379},"max":{"1min":11.378,"5min":13.355,"15min":13.355},"last":0.73199999999999998}]}]},{"osd":4,"up_from":31,"seq":133143986442,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":154240,"kb_used_data":2680,"kb_used_omap":0,"kb_used_meta":151552,"kb_avail":93590912,"statfs":{"total":95995035648,"available":95837093888,"internally_reserved":0,"allocated":2744320,"data_stored":1840527,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":155189248},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:00:55 2023","interfaces":[{"interface":"back","average":{"1min":1.8089999999999999,"5min":1.893,"15min":1.2210000000000001},"min":{"1min":0.218,"5min":0.214,"15min":0.214},"max":{"1min":8.9429999999999996,"5min":11.784000000000001,"15min":11.784000000000001},"last":0.44},{"interface":"front","average":{"1min":1.835,"5min":1.9259999999999999,"15min":1.397},"min":{"1min":0.27700000000000002,"5min":0.27700000000000002,"15min":0.25800000000000001},"max":{"1min":8.9049999999999994,"5min":11.766,"15min":11.766},"last":0.42099999999999999}]},{"osd":1,"last update":"Sat Sep 23 21:01:23 2023","interfaces":[{"interface":"back","average":{"1min":1.95,"5min":1.7509999999999999,"15min":1.2709999999999999},"min":{"1min":0.27700000000000002,"5min":0.246,"15min":0.23599999999999999},"max":{"1min":8.5030000000000001,"5min":10.153,"15min":10.153},"last":0.53500000000000003},{"interface":"front","average":{"1min":1.5309999999999999,"5min":2.2010000000000001,"15min":1.337},"min":{"1min":0.376,"5min":0.26100000000000001,"15min":0.23200000000000001},"max":{"1min":8.9749999999999996,"5min":12.292999999999999,"15min":12.292999999999999},"last":0.61499999999999999}]},{"osd":2,"last update":"Sat Sep 23 21:00:55 2023","interfaces":[{"interface":"back","average":{"1min":2.9249999999999998,"5min":1.9490000000000001,"15min":1.3899999999999999},"min":{"1min":0.32500000000000001,"5min":0.313,"15min":0.29099999999999998},"max":{"1min":9.8469999999999995,"5min":9.8469999999999995,"15min":15.288},"last":0.68200000000000005},{"interface":"front","average":{"1min":2.2240000000000002,"5min":1.905,"15min":1.496},"min":{"1min":0.316,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":7.6829999999999998,"5min":12.855,"15min":15.262},"last":0.72399999999999998}]},{"osd":3,"last update":"Sat Sep 23 21:01:23 2023","interfaces":[{"interface":"back","average":{"1min":2.254,"5min":2.306,"15min":1.919},"min":{"1min":0.318,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":12.106999999999999,"5min":12.106999999999999,"15min":31.562999999999999},"last":0.58099999999999996},{"interface":"front","average":{"1min":2.2709999999999999,"5min":2.3660000000000001,"15min":1.974},"min":{"1min":0.47899999999999998,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":12.477,"5min":12.477,"15min":31.574000000000002},"last":0.76400000000000001}]},{"osd":5,"last update":"Sat Sep 23 21:00:39 2023","interfaces":[{"interface":"back","average":{"1min":0.33800000000000002,"5min":0.56999999999999995,"15min":0.51900000000000002},"min":{"1min":0.251,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":0.495,"5min":3.008,"15min":3.3279999999999998},"last":0.79400000000000004},{"interface":"front","average":{"1min":0.55500000000000005,"5min":0.71199999999999997,"15min":0.621},"min":{"1min":0.27600000000000002,"5min":0.25600000000000001,"15min":0.251},"max":{"1min":2.6539999999999999,"5min":3.4830000000000001,"15min":7.7619999999999996},"last":0.66100000000000003}]},{"osd":6,"last update":"Sat Sep 23 21:00:51 2023","interfaces":[{"interface":"back","average":{"1min":0.66400000000000003,"5min":0.73999999999999999,"15min":0.56899999999999995},"min":{"1min":0.27300000000000002,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":2.9239999999999999,"5min":4.6909999999999998,"15min":4.6909999999999998},"last":0.63700000000000001},{"interface":"front","average":{"1min":0.54700000000000004,"5min":0.71799999999999997,"15min":0.66800000000000004},"min":{"1min":0.33000000000000002,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":1.8340000000000001,"5min":3.456,"15min":3.456},"last":0.81799999999999995}]},{"osd":7,"last update":"Sat Sep 23 21:01:23 2023","interfaces":[{"interface":"back","average":{"1min":0.629,"5min":0.68600000000000005,"15min":0.69599999999999995},"min":{"1min":0.29299999999999998,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":3.4940000000000002,"5min":3.4940000000000002,"15min":7.383},"last":0.83999999999999997},{"interface":"front","average":{"1min":0.74399999999999999,"5min":0.71499999999999997,"15min":0.69099999999999995},"min":{"1min":0.36299999999999999,"5min":0.32700000000000001,"15min":0.28199999999999997},"max":{"1min":5.6980000000000004,"5min":5.6980000000000004,"15min":7.8310000000000004},"last":0.70599999999999996}]}]},{"osd":5,"up_from":39,"seq":167503724807,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":424628,"kb_used_data":281324,"kb_used_omap":0,"kb_used_meta":143296,"kb_avail":93320524,"statfs":{"total":95995035648,"available":95560216576,"internally_reserved":0,"allocated":288075776,"data_stored":287180559,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":146735104},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:00:41 2023","interfaces":[{"interface":"back","average":{"1min":3.0920000000000001,"5min":2.0419999999999998,"15min":1.1359999999999999},"min":{"1min":0.35199999999999998,"5min":0.20399999999999999,"15min":0.20399999999999999},"max":{"1min":6.665,"5min":12.795999999999999,"15min":12.795999999999999},"last":0.51100000000000001},{"interface":"front","average":{"1min":3.7250000000000001,"5min":2.2349999999999999,"15min":1.335},"min":{"1min":0.27000000000000002,"5min":0.24099999999999999,"15min":0.223},"max":{"1min":9.1099999999999994,"5min":16.219999999999999,"15min":16.219999999999999},"last":0.46400000000000002}]},{"osd":1,"last update":"Sat Sep 23 21:01:18 2023","interfaces":[{"interface":"back","average":{"1min":1.2310000000000001,"5min":1.8240000000000001,"15min":1.298},"min":{"1min":0.28199999999999997,"5min":0.23899999999999999,"15min":0.23300000000000001},"max":{"1min":3.3849999999999998,"5min":8.7720000000000002,"15min":8.7720000000000002},"last":0.48299999999999998},{"interface":"front","average":{"1min":2.222,"5min":1.984,"15min":1.331},"min":{"1min":0.27800000000000002,"5min":0.26200000000000001,"15min":0.22800000000000001},"max":{"1min":11.196999999999999,"5min":11.196999999999999,"15min":11.196999999999999},"last":0.42699999999999999}]},{"osd":2,"last update":"Sat Sep 23 21:00:59 2023","interfaces":[{"interface":"back","average":{"1min":2.4279999999999999,"5min":1.837,"15min":1.246},"min":{"1min":0.26700000000000002,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":6.7039999999999997,"5min":10.579000000000001,"15min":12.651},"last":0.435},{"interface":"front","average":{"1min":2.3450000000000002,"5min":2.2280000000000002,"15min":1.391},"min":{"1min":0.28599999999999998,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":7.5759999999999996,"5min":12.300000000000001,"15min":12.300000000000001},"last":0.40799999999999997}]},{"osd":3,"last update":"Sat Sep 23 21:01:18 2023","interfaces":[{"interface":"back","average":{"1min":1.5129999999999999,"5min":2.7280000000000002,"15min":1.698},"min":{"1min":0.33000000000000002,"5min":0.33000000000000002,"15min":0.28599999999999998},"max":{"1min":5.4210000000000003,"5min":20.788,"15min":20.788},"last":0.49299999999999999},{"interface":"front","average":{"1min":1.492,"5min":2.0019999999999998,"15min":1.3520000000000001},"min":{"1min":0.32400000000000001,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":6.6079999999999997,"5min":20.539000000000001,"15min":20.539000000000001},"last":0.47299999999999998}]},{"osd":4,"last update":"Sat Sep 23 21:00:35 2023","interfaces":[{"interface":"back","average":{"1min":0.69999999999999996,"5min":0.625,"15min":0.56899999999999995},"min":{"1min":0.23400000000000001,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":4.1520000000000001,"5min":4.1520000000000001,"15min":4.1520000000000001},"last":0.375},{"interface":"front","average":{"1min":0.79600000000000004,"5min":0.69799999999999995,"15min":0.59599999999999997},"min":{"1min":0.28999999999999998,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":3.7469999999999999,"5min":3.7469999999999999,"15min":3.7469999999999999},"last":0.39000000000000001}]},{"osd":6,"last update":"Sat Sep 23 21:00:41 2023","interfaces":[{"interface":"back","average":{"1min":0.91600000000000004,"5min":0.72199999999999998,"15min":0.61399999999999999},"min":{"1min":0.314,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":4.359,"5min":4.359,"15min":4.359},"last":0.45300000000000001},{"interface":"front","average":{"1min":0.92200000000000004,"5min":0.79100000000000004,"15min":0.75900000000000001},"min":{"1min":0.36899999999999999,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":3.847,"5min":3.847,"15min":3.847},"last":0.53000000000000003}]},{"osd":7,"last update":"Sat Sep 23 21:01:18 2023","interfaces":[{"interface":"back","average":{"1min":0.61699999999999999,"5min":0.753,"15min":0.68100000000000005},"min":{"1min":0.35799999999999998,"5min":0.307,"15min":0.30199999999999999},"max":{"1min":2.9990000000000001,"5min":4.4240000000000004,"15min":8.5649999999999995},"last":0.52300000000000002},{"interface":"front","average":{"1min":0.67900000000000005,"5min":0.70899999999999996,"15min":0.68200000000000005},"min":{"1min":0.33900000000000002,"5min":0.33900000000000002,"15min":0.27700000000000002},"max":{"1min":3.8860000000000001,"5min":3.8860000000000001,"15min":4.5430000000000001},"last":0.55600000000000005}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":14417920,"data_stored":14417920,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":219348992,"data_stored":219348992,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":31588352,"data_stored":31588352,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":30408704,"data_stored":30408704,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":136249344,"data_stored":136249344,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":18874368,"data_stored":18874368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T21:01:39.172 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph osd pool set .mgr pgp_num 21 2023-09-23T21:01:39.265 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:01:39.330 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:01:39.616 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 21:01:39 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[164029]: ::ffff:172.21.15.70 - - [23/Sep/2023:21:01:39] "GET /metrics HTTP/1.1" 200 36227 "" "Prometheus/2.43.0" 2023-09-23T21:01:39.660 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:01:39.667 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:01:39.668 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:01:39.729 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 8 16 20924 20908 163.313 134.438 0.00317515 0.00611655 2023-09-23T21:01:39.769 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:01:39.810 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:01:39.859 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:01:39.863 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:01:39.864 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:01:39.964 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:01:39.993 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:01:40.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:01:40 smithi026 ceph-mon[175151]: pgmap v673: 109 pgs: 109 active+clean; 672 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail; 78 MiB/s wr, 1.25k op/s 2023-09-23T21:01:40.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:01:40 smithi026 ceph-mon[179688]: pgmap v673: 109 pgs: 109 active+clean; 672 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail; 78 MiB/s wr, 1.25k op/s 2023-09-23T21:01:40.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:01:40 smithi070 ceph-mon[168282]: pgmap v673: 109 pgs: 109 active+clean; 672 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail; 78 MiB/s wr, 1.25k op/s 2023-09-23T21:01:40.730 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 9 16 23066 23050 160.025 133.875 0.00509151 0.00624368 2023-09-23T21:01:41.300 INFO:teuthology.orchestra.run.smithi026.stdout:{ 2023-09-23T21:01:41.301 INFO:teuthology.orchestra.run.smithi026.stdout: "ops": [], 2023-09-23T21:01:41.301 INFO:teuthology.orchestra.run.smithi026.stdout: "complaint_time": 30, 2023-09-23T21:01:41.301 INFO:teuthology.orchestra.run.smithi026.stdout: "num_blocked_ops": 0 2023-09-23T21:01:41.301 INFO:teuthology.orchestra.run.smithi026.stdout:} 2023-09-23T21:01:41.399 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T21:01:41.409 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:01:41.410 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:01:41.456 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:01:41.510 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:01:41.572 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:01:41.730 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 10 16 25174 25158 157.197 131.75 0.0111499 0.00635577 2023-09-23T21:01:42.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:01:42 smithi070 ceph-mon[168282]: pgmap v674: 109 pgs: 109 active+clean; 1.1 GiB data, 5.5 GiB used, 710 GiB / 715 GiB avail; 111 MiB/s wr, 1.78k op/s 2023-09-23T21:01:42.636 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:01:42 smithi026 ceph-mon[175151]: pgmap v674: 109 pgs: 109 active+clean; 1.1 GiB data, 5.5 GiB used, 710 GiB / 715 GiB avail; 111 MiB/s wr, 1.78k op/s 2023-09-23T21:01:42.636 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:01:42 smithi026 ceph-mon[179688]: pgmap v674: 109 pgs: 109 active+clean; 1.1 GiB data, 5.5 GiB used, 710 GiB / 715 GiB avail; 111 MiB/s wr, 1.78k op/s 2023-09-23T21:01:42.731 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 11 16 27360 27344 155.324 136.625 0.00914392 0.00643397 2023-09-23T21:01:42.902 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_historic_ops 2023-09-23T21:01:43.108 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T21:01:43.113 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:01:43.114 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:01:43.214 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:01:43.267 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:01:43.585 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:01:43.591 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:01:43.592 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:01:43.661 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:01:43.694 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:01:43.731 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 12 16 29043 29027 151.145 105.188 0.00286537 0.00660966 2023-09-23T21:01:43.782 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:01:44.295 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:01:44.308 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:01:44.309 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:01:44.387 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:01:44 smithi026 ceph-mon[175151]: pgmap v675: 109 pgs: 109 active+clean; 1.4 GiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 136 MiB/s wr, 2.18k op/s 2023-09-23T21:01:44.387 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:01:44 smithi026 ceph-mon[179688]: pgmap v675: 109 pgs: 109 active+clean; 1.4 GiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 136 MiB/s wr, 2.18k op/s 2023-09-23T21:01:44.410 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:01:44.448 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:01:44 smithi070 ceph-mon[168282]: pgmap v675: 109 pgs: 109 active+clean; 1.4 GiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 136 MiB/s wr, 2.18k op/s 2023-09-23T21:01:44.453 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:01:44.499 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:01:44.504 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:01:44.504 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:01:44.605 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:01:44.640 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:01:44.692 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:01:44.696 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:01:44.696 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:01:44.731 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 13 16 30731 30715 147.634 105.5 0.00995107 0.00676893 2023-09-23T21:01:44.797 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:01:44.834 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:01:44.886 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:01:44.889 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:01:44.889 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:01:44.990 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:01:45.066 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:01:45.290 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:01:45.293 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:01:45.293 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:01:45.394 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:01:45.449 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:01:45.507 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:01:45.510 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:01:45.510 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:01:45.611 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:01:45.640 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:01:45.731 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 14 16 32324 32308 144.2 99.5625 0.00801939 0.00691 2023-09-23T21:01:45.828 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:01:45.830 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:01:45.831 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:01:45.931 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:01:45.959 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:01:45.990 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:01:45.993 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:01:45.993 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:01:46.094 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:01:46.156 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:01:46.324 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T21:01:46.331 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:01:46.332 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:01:46.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:01:46 smithi070 ceph-mon[168282]: pgmap v676: 109 pgs: 109 active+clean; 1.6 GiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 136 MiB/s wr, 2.18k op/s 2023-09-23T21:01:46.432 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:01:46.474 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:01:46.518 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:01:46.522 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:01:46.523 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:01:46.552 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:01:46 smithi026 ceph-mon[179688]: pgmap v676: 109 pgs: 109 active+clean; 1.6 GiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 136 MiB/s wr, 2.18k op/s 2023-09-23T21:01:46.554 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:01:46 smithi026 ceph-mon[175151]: pgmap v676: 109 pgs: 109 active+clean; 1.6 GiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 136 MiB/s wr, 2.18k op/s 2023-09-23T21:01:46.623 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:01:46.661 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:01:46.710 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:01:46.716 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:01:46.716 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:01:46.731 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 15 16 33807 33791 140.766 92.6875 0.00947692 0.00709785 2023-09-23T21:01:46.817 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:01:46.856 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:01:46.900 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:01:46.906 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:01:46.907 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:01:47.007 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:01:47.024 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:01:47.062 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0: {} 2023-09-23T21:01:47.093 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:01:47.117 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1: {} 2023-09-23T21:01:47.154 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2: {} 2023-09-23T21:01:47.199 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3: {} 2023-09-23T21:01:47.212 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:01:47.221 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:01:47.222 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:01:47.244 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4: {} 2023-09-23T21:01:47.252 INFO:teuthology.orchestra.run.smithi026.stderr:nodeep-scrub is set 2023-09-23T21:01:47.275 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T21:01:47.320 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6: {} 2023-09-23T21:01:47.323 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:01:47.376 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T21:01:47.398 INFO:teuthology.orchestra.run.smithi026.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T21:01:47.398 INFO:teuthology.orchestra.run.smithi026.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T21:01:47.398 INFO:teuthology.orchestra.run.smithi026.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T21:01:47.398 INFO:teuthology.orchestra.run.smithi026.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T21:01:47.398 INFO:teuthology.orchestra.run.smithi026.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T21:01:47.398 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T21:01:47.398 INFO:teuthology.orchestra.run.smithi026.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T21:01:47.398 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T21:01:47.418 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:01:47.574 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:01:47 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:01:47.574 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:01:47 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:01:47.574 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:01:47 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/890498622' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:01:47.575 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:01:47 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/3320491410' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:01:47.575 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:01:47 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:01:47.575 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:01:47 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:01:47.575 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:01:47 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:01:47.575 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:01:47 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/890498622' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:01:47.575 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:01:47 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/3320491410' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:01:47.575 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:01:47 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:01:47.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:01:47 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:01:47.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:01:47 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:01:47.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:01:47 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/890498622' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:01:47.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:01:47 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/3320491410' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:01:47.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:01:47 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:01:47.731 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 16 16 35251 35235 137.608 90.25 0.00494311 0.00726092 2023-09-23T21:01:47.742 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T21:01:47.749 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:01:47.749 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:01:47.850 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:01:47.913 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:01:48.136 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:01:48.145 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:01:48.146 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:01:48.246 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:01:48.259 INFO:teuthology.orchestra.run.smithi026.stderr:set pool 1 pgp_num to 21 2023-09-23T21:01:48.289 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:01:48.357 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:01:48.361 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:01:48.362 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:01:48.462 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:01:48.496 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:01:48 smithi070 ceph-mon[168282]: pgmap v677: 109 pgs: 109 active+clean; 1.7 GiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 131 MiB/s wr, 2.10k op/s 2023-09-23T21:01:48.496 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:01:48 smithi070 ceph-mon[168282]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:01:48.496 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:01:48 smithi070 ceph-mon[168282]: failed to encode map e259 with expected crc 2023-09-23T21:01:48.496 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:01:48 smithi070 ceph-mon[168282]: failed to encode map e259 with expected crc 2023-09-23T21:01:48.497 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:01:48 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T21:01:48.497 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:01:48 smithi070 ceph-mon[168282]: osdmap e259: 8 total, 8 up, 8 in 2023-09-23T21:01:48.497 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:01:48 smithi070 ceph-mon[168282]: failed to encode map e259 with expected crc 2023-09-23T21:01:48.497 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:01:48 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/2866767125' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "21"}]: dispatch 2023-09-23T21:01:48.497 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:01:48 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "21"}]: dispatch 2023-09-23T21:01:48.503 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:01:48.552 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:01:48.557 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:01:48.558 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:01:48.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:01:48 smithi026 ceph-mon[179688]: pgmap v677: 109 pgs: 109 active+clean; 1.7 GiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 131 MiB/s wr, 2.10k op/s 2023-09-23T21:01:48.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:01:48 smithi026 ceph-mon[179688]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:01:48.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:01:48 smithi026 ceph-mon[179688]: failed to encode map e259 with expected crc 2023-09-23T21:01:48.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:01:48 smithi026 ceph-mon[179688]: failed to encode map e259 with expected crc 2023-09-23T21:01:48.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:01:48 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T21:01:48.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:01:48 smithi026 ceph-mon[179688]: osdmap e259: 8 total, 8 up, 8 in 2023-09-23T21:01:48.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:01:48 smithi026 ceph-mon[179688]: failed to encode map e259 with expected crc 2023-09-23T21:01:48.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:01:48 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/2866767125' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "21"}]: dispatch 2023-09-23T21:01:48.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:01:48 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "21"}]: dispatch 2023-09-23T21:01:48.633 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:01:48 smithi026 ceph-mon[175151]: pgmap v677: 109 pgs: 109 active+clean; 1.7 GiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 131 MiB/s wr, 2.10k op/s 2023-09-23T21:01:48.633 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:01:48 smithi026 ceph-mon[175151]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:01:48.633 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:01:48 smithi026 ceph-mon[175151]: failed to encode map e259 with expected crc 2023-09-23T21:01:48.633 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:01:48 smithi026 ceph-mon[175151]: failed to encode map e259 with expected crc 2023-09-23T21:01:48.633 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:01:48 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T21:01:48.633 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:01:48 smithi026 ceph-mon[175151]: osdmap e259: 8 total, 8 up, 8 in 2023-09-23T21:01:48.633 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:01:48 smithi026 ceph-mon[175151]: failed to encode map e259 with expected crc 2023-09-23T21:01:48.633 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:01:48 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/2866767125' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "21"}]: dispatch 2023-09-23T21:01:48.633 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:01:48 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "21"}]: dispatch 2023-09-23T21:01:48.659 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:01:48.700 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:01:48.731 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 17 16 36894 36878 135.554 102.688 0.00664347 0.00737237 2023-09-23T21:01:48.751 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:01:48.756 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:01:48.757 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:01:48.858 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:01:48.898 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:01:48.953 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:01:48.956 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:01:48.956 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:01:49.057 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:01:49.097 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:01:49.145 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:01:49.149 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:01:49.151 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:01:49.252 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:01:49.289 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:01:49.342 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:01:49.346 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:01:49 smithi070 ceph-mon[168282]: failed to encode map e260 with expected crc 2023-09-23T21:01:49.346 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:01:49 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "21"}]': finished 2023-09-23T21:01:49.346 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:01:49 smithi070 ceph-mon[168282]: osdmap e260: 8 total, 8 up, 8 in 2023-09-23T21:01:49.346 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:01:49 smithi070 ceph-mon[168282]: failed to encode map e260 with expected crc 2023-09-23T21:01:49.346 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:01:49 smithi070 ceph-mon[168282]: failed to encode map e260 with expected crc 2023-09-23T21:01:49.346 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:01:49 smithi070 ceph-mon[168282]: failed to encode map e260 with expected crc 2023-09-23T21:01:49.347 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:01:49.347 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:01:49.383 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:01:49 smithi026 ceph-mon[179688]: failed to encode map e260 with expected crc 2023-09-23T21:01:49.383 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:01:49 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "21"}]': finished 2023-09-23T21:01:49.383 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:01:49 smithi026 ceph-mon[179688]: osdmap e260: 8 total, 8 up, 8 in 2023-09-23T21:01:49.383 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:01:49 smithi026 ceph-mon[179688]: failed to encode map e260 with expected crc 2023-09-23T21:01:49.384 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:01:49 smithi026 ceph-mon[179688]: failed to encode map e260 with expected crc 2023-09-23T21:01:49.384 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:01:49 smithi026 ceph-mon[179688]: failed to encode map e260 with expected crc 2023-09-23T21:01:49.386 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:01:49 smithi026 ceph-mon[175151]: failed to encode map e260 with expected crc 2023-09-23T21:01:49.386 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:01:49 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "21"}]': finished 2023-09-23T21:01:49.386 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:01:49 smithi026 ceph-mon[175151]: osdmap e260: 8 total, 8 up, 8 in 2023-09-23T21:01:49.386 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:01:49 smithi026 ceph-mon[175151]: failed to encode map e260 with expected crc 2023-09-23T21:01:49.386 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:01:49 smithi026 ceph-mon[175151]: failed to encode map e260 with expected crc 2023-09-23T21:01:49.386 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:01:49 smithi026 ceph-mon[175151]: failed to encode map e260 with expected crc 2023-09-23T21:01:49.448 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:01:49.539 INFO:teuthology.orchestra.run.smithi026.stdout:ERROR: (22) Invalid argument 2023-09-23T21:01:49.539 INFO:teuthology.orchestra.run.smithi026.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T21:01:49.545 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:01:49.731 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 18 16 38553 38537 133.783 103.688 0.0108109 0.00746917 2023-09-23T21:01:49.882 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 21:01:49 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[164029]: ::ffff:172.21.15.70 - - [23/Sep/2023:21:01:49] "GET /metrics HTTP/1.1" 200 36285 "" "Prometheus/2.43.0" 2023-09-23T21:01:49.931 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:01:49.937 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:01:49.938 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:01:50.039 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:01:50.046 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_ops_in_flight 2023-09-23T21:01:50.073 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T21:01:50.076 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:01:50.122 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:01:50.125 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:01:50.126 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:01:50.227 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:01:50.266 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:01:50 smithi026 ceph-mon[175151]: pgmap v680: 109 pgs: 109 active+clean; 2.0 GiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 111 MiB/s wr, 1.78k op/s 2023-09-23T21:01:50.315 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:01:50.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:01:50 smithi070 ceph-mon[168282]: pgmap v680: 109 pgs: 109 active+clean; 2.0 GiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 111 MiB/s wr, 1.78k op/s 2023-09-23T21:01:50.404 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:01:50.418 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:01:50.419 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:01:50.519 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:01:50.582 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:01:50 smithi026 ceph-mon[179688]: pgmap v680: 109 pgs: 109 active+clean; 2.0 GiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 111 MiB/s wr, 1.78k op/s 2023-09-23T21:01:50.589 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:01:50.653 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T21:01:50.665 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:01:50.665 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:01:50.732 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 19 16 40068 40052 131.725 94.6875 0.0107926 0.0075853 2023-09-23T21:01:50.766 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:01:50.801 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:01:50.812 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T21:01:50.854 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:01:50.859 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:01:50.859 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:01:50.888 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:01:50.960 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:01:51.065 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:01:51.079 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:01:51.291 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T21:01:51.301 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:01:51.301 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:01:51.402 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:01:51.477 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:01:51.594 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:01:51.597 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:01:51.598 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:01:51.698 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:01:51.735 INFO:tasks.radosbench.radosbench.0.smithi026.stdout:2023-09-23T21:01:51.735503+0000 min lat: 0.00122595 max lat: 0.138259 avg lat: 0.00771514 2023-09-23T21:01:51.735 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T21:01:51.735 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 20 16 41472 41456 129.506 87.75 0.00924537 0.00771514 2023-09-23T21:01:51.761 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:01:51.884 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:01:51.892 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:01:51.893 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:01:51.993 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:01:52.019 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:01:52.050 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:01:52.054 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:01:52.054 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:01:52.155 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:01:52.189 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:01:52.234 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:01:52.238 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:01:52.239 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:01:52.339 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:01:52.368 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:01:52.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:01:52 smithi070 ceph-mon[168282]: pgmap v681: 109 pgs: 109 active+clean; 2.2 GiB data, 9.1 GiB used, 706 GiB / 715 GiB avail; 107 MiB/s wr, 1.71k op/s 2023-09-23T21:01:52.633 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:01:52 smithi026 ceph-mon[175151]: pgmap v681: 109 pgs: 109 active+clean; 2.2 GiB data, 9.1 GiB used, 706 GiB / 715 GiB avail; 107 MiB/s wr, 1.71k op/s 2023-09-23T21:01:52.633 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:01:52 smithi026 ceph-mon[179688]: pgmap v681: 109 pgs: 109 active+clean; 2.2 GiB data, 9.1 GiB used, 706 GiB / 715 GiB avail; 107 MiB/s wr, 1.71k op/s 2023-09-23T21:01:52.736 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 21 16 42761 42745 127.167 80.5625 0.00473812 0.00785811 2023-09-23T21:01:53.736 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 22 16 44522 44506 126.389 110.062 0.0045389 0.00790734 2023-09-23T21:01:54.135 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 6, 7, 4, 5] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 7, 6] 2023-09-23T21:01:54.136 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph osd reweight 0 1 2023-09-23T21:01:54.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:01:54 smithi026 ceph-mon[175151]: pgmap v682: 109 pgs: 109 active+clean; 2.4 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 104 MiB/s wr, 1.67k op/s 2023-09-23T21:01:54.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:01:54 smithi026 ceph-mon[179688]: pgmap v682: 109 pgs: 109 active+clean; 2.4 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 104 MiB/s wr, 1.67k op/s 2023-09-23T21:01:54.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:01:54 smithi070 ceph-mon[168282]: pgmap v682: 109 pgs: 109 active+clean; 2.4 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 104 MiB/s wr, 1.67k op/s 2023-09-23T21:01:54.737 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 23 16 45870 45854 124.557 84.25 0.00817166 0.00802383 2023-09-23T21:01:55.091 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:01:55.095 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:01:55.095 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:01:55.196 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:01:55.237 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:01:55.282 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:01:55.285 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:01:55.285 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:01:55.328 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:01:55.386 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:01:55.449 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:01:55.737 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 24 16 47208 47192 122.852 83.625 0.00710721 0.00813502 2023-09-23T21:01:55.787 INFO:teuthology.orchestra.run.smithi026.stdout:ERROR: (22) Invalid argument 2023-09-23T21:01:55.787 INFO:teuthology.orchestra.run.smithi026.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T21:01:55.921 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:01:55.927 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:01:55.928 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:01:56.029 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:01:56.070 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:01:56.122 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:01:56.127 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:01:56.128 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:01:56.229 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:01:56.304 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:01:56.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:01:56 smithi070 ceph-mon[168282]: pgmap v683: 109 pgs: 109 active+clean; 2.6 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 111 MiB/s wr, 1.78k op/s 2023-09-23T21:01:56.536 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:01:56.543 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:01:56.544 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:01:56.544 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:01:56 smithi026 ceph-mon[175151]: pgmap v683: 109 pgs: 109 active+clean; 2.6 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 111 MiB/s wr, 1.78k op/s 2023-09-23T21:01:56.544 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:01:56 smithi026 ceph-mon[179688]: pgmap v683: 109 pgs: 109 active+clean; 2.6 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 111 MiB/s wr, 1.78k op/s 2023-09-23T21:01:56.588 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:01:56.589 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_blocked_ops 2023-09-23T21:01:56.645 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:01:56.737 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 25 16 48497 48481 121.16 80.5625 0.0140102 0.00824727 2023-09-23T21:01:56.765 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:01:56.866 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T21:01:56.875 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:01:56.876 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:01:56.977 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:01:57.072 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:01:57.335 INFO:teuthology.orchestra.run.smithi026.stderr:noscrub is unset 2023-09-23T21:01:57.377 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T21:01:57.380 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:01:57.381 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:01:57.481 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:01:57.528 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:01:57.579 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:01:57.581 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:01:57.581 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:01:57.585 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:01:57 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/3749850382' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T21:01:57.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:01:57 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/3749850382' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T21:01:57.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:01:57 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/3749850382' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T21:01:57.682 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:01:57.722 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0: {} 2023-09-23T21:01:57.737 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 26 16 49734 49718 119.474 77.3125 0.0170324 0.00836602 2023-09-23T21:01:57.753 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:01:57.773 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1: {} 2023-09-23T21:01:57.820 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:01:57.826 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2: {} 2023-09-23T21:01:57.833 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:01:57.833 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:01:57.862 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3: {} 2023-09-23T21:01:57.905 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4: {} 2023-09-23T21:01:57.934 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:01:57.960 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T21:01:57.983 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:01:58.004 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6: {} 2023-09-23T21:01:58.030 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:01:58.035 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:01:58.036 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:01:58.047 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T21:01:58.054 INFO:teuthology.orchestra.run.smithi026.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T21:01:58.055 INFO:teuthology.orchestra.run.smithi026.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T21:01:58.055 INFO:teuthology.orchestra.run.smithi026.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T21:01:58.055 INFO:teuthology.orchestra.run.smithi026.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T21:01:58.055 INFO:teuthology.orchestra.run.smithi026.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T21:01:58.055 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T21:01:58.055 INFO:teuthology.orchestra.run.smithi026.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T21:01:58.055 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T21:01:58.137 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:01:58.188 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:01:58.232 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:01:58.235 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:01:58.235 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:01:58.321 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:01:58 smithi026 ceph-mon[175151]: pgmap v684: 109 pgs: 109 active+clean; 2.7 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 83 MiB/s wr, 1.33k op/s 2023-09-23T21:01:58.321 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:01:58 smithi026 ceph-mon[175151]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:01:58.321 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:01:58 smithi026 ceph-mon[175151]: failed to encode map e261 with expected crc 2023-09-23T21:01:58.321 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:01:58 smithi026 ceph-mon[175151]: failed to encode map e261 with expected crc 2023-09-23T21:01:58.321 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:01:58 smithi026 ceph-mon[175151]: failed to encode map e261 with expected crc 2023-09-23T21:01:58.321 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:01:58 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/3749850382' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T21:01:58.321 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:01:58 smithi026 ceph-mon[175151]: osdmap e261: 8 total, 8 up, 8 in 2023-09-23T21:01:58.321 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:01:58 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/3564966557' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:01:58.322 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:01:58 smithi026 ceph-mon[179688]: pgmap v684: 109 pgs: 109 active+clean; 2.7 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 83 MiB/s wr, 1.33k op/s 2023-09-23T21:01:58.322 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:01:58 smithi026 ceph-mon[179688]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:01:58.322 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:01:58 smithi026 ceph-mon[179688]: failed to encode map e261 with expected crc 2023-09-23T21:01:58.322 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:01:58 smithi026 ceph-mon[179688]: failed to encode map e261 with expected crc 2023-09-23T21:01:58.322 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:01:58 smithi026 ceph-mon[179688]: failed to encode map e261 with expected crc 2023-09-23T21:01:58.322 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:01:58 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/3749850382' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T21:01:58.322 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:01:58 smithi026 ceph-mon[179688]: osdmap e261: 8 total, 8 up, 8 in 2023-09-23T21:01:58.322 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:01:58 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/3564966557' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:01:58.336 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:01:58.371 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:01:58.421 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:01:58.425 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:01:58.426 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:01:58.526 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:01:58.570 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:01:58 smithi070 ceph-mon[168282]: pgmap v684: 109 pgs: 109 active+clean; 2.7 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 83 MiB/s wr, 1.33k op/s 2023-09-23T21:01:58.570 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:01:58 smithi070 ceph-mon[168282]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:01:58.570 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:01:58 smithi070 ceph-mon[168282]: failed to encode map e261 with expected crc 2023-09-23T21:01:58.570 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:01:58 smithi070 ceph-mon[168282]: failed to encode map e261 with expected crc 2023-09-23T21:01:58.570 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:01:58 smithi070 ceph-mon[168282]: failed to encode map e261 with expected crc 2023-09-23T21:01:58.570 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:01:58 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/3749850382' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T21:01:58.571 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:01:58 smithi070 ceph-mon[168282]: osdmap e261: 8 total, 8 up, 8 in 2023-09-23T21:01:58.571 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:01:58 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/3564966557' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:01:58.577 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:01:58.629 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:01:58.632 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:01:58.633 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:01:58.700 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:01:58.733 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:01:58.737 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 27 16 51144 51128 118.313 88.125 0.0131665 0.00844711 2023-09-23T21:01:58.827 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:01:59.019 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:01:59.026 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:01:59.026 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:01:59.127 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:01:59.197 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:01:59.683 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:01:59.692 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 21:01:59 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[164029]: ::ffff:172.21.15.70 - - [23/Sep/2023:21:01:59] "GET /metrics HTTP/1.1" 200 36285 "" "Prometheus/2.43.0" 2023-09-23T21:01:59.692 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:01:59 smithi026 ceph-mon[179688]: pgmap v686: 109 pgs: 109 active+clean; 2.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 94 MiB/s wr, 1.50k op/s 2023-09-23T21:01:59.693 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:01:59 smithi026 ceph-mon[175151]: pgmap v686: 109 pgs: 109 active+clean; 2.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 94 MiB/s wr, 1.50k op/s 2023-09-23T21:01:59.693 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:01:59.693 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:01:59.740 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 28 16 52402 52386 116.882 78.625 0.0100412 0.00854999 2023-09-23T21:01:59.794 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:01:59.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:01:59 smithi070 ceph-mon[168282]: pgmap v686: 109 pgs: 109 active+clean; 2.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 94 MiB/s wr, 1.50k op/s 2023-09-23T21:01:59.881 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:01:59.954 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:01:59.959 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:01:59.959 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:02:00.060 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:02:00.101 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:02:00.146 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:02:00.152 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:02:00.152 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:02:00.253 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:02:00.329 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:02:00.373 INFO:teuthology.orchestra.run.smithi026.stdout:{ 2023-09-23T21:02:00.373 INFO:teuthology.orchestra.run.smithi026.stdout: "ops": [], 2023-09-23T21:02:00.373 INFO:teuthology.orchestra.run.smithi026.stdout: "complaint_time": 30, 2023-09-23T21:02:00.373 INFO:teuthology.orchestra.run.smithi026.stdout: "num_blocked_ops": 0 2023-09-23T21:02:00.373 INFO:teuthology.orchestra.run.smithi026.stdout:} 2023-09-23T21:02:00.405 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:02:00.412 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:02:00.413 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:02:00.429 INFO:teuthology.orchestra.run.smithi026.stderr:reweighted osd.0 to 1 (10000) 2023-09-23T21:02:00.514 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:02:00.586 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:02:00 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/1882186742' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-09-23T21:02:00.587 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:02:00 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/1882186742' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-09-23T21:02:00.590 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:02:00.740 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 29 15 53616 53601 115.471 75.9375 0.00336525 0.00865569 2023-09-23T21:02:00.743 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T21:02:00.745 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T21:02:00.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:02:00 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/1882186742' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-09-23T21:02:00.881 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:02:00.888 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:02:00.888 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:02:00.922 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_historic_ops 2023-09-23T21:02:00.989 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:02:01.141 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:02:01.144 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph osd reweight 1 1 2023-09-23T21:02:01.242 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:02:01.250 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:02:01.250 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:02:01.351 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:02:01.384 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:02:01.431 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:02:01.434 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:02:01.434 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:02:01.437 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:02:01 smithi026 ceph-mon[179688]: failed to encode map e262 with expected crc 2023-09-23T21:02:01.437 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:02:01 smithi026 ceph-mon[179688]: failed to encode map e262 with expected crc 2023-09-23T21:02:01.437 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:02:01 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/1882186742' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-09-23T21:02:01.437 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:02:01 smithi026 ceph-mon[179688]: osdmap e262: 8 total, 8 up, 8 in 2023-09-23T21:02:01.437 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:02:01 smithi026 ceph-mon[179688]: failed to encode map e262 with expected crc 2023-09-23T21:02:01.437 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:02:01 smithi026 ceph-mon[179688]: pgmap v688: 109 pgs: 109 active+clean; 3.1 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 84 MiB/s wr, 1.35k op/s 2023-09-23T21:02:01.437 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:02:01 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:02:01.535 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:02:01.574 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:02:01.577 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:02:01.619 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:02:01.623 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:02:01.624 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:02:01.637 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:02:01.725 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:02:01.740 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 30 16 54571 54555 113.61 59.625 0.0155468 0.00879728 2023-09-23T21:02:01.763 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:02:01 smithi070 ceph-mon[168282]: failed to encode map e262 with expected crc 2023-09-23T21:02:01.763 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:02:01 smithi070 ceph-mon[168282]: failed to encode map e262 with expected crc 2023-09-23T21:02:01.763 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:02:01 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/1882186742' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-09-23T21:02:01.763 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:02:01 smithi070 ceph-mon[168282]: osdmap e262: 8 total, 8 up, 8 in 2023-09-23T21:02:01.763 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:02:01 smithi070 ceph-mon[168282]: failed to encode map e262 with expected crc 2023-09-23T21:02:01.763 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:02:01 smithi070 ceph-mon[168282]: pgmap v688: 109 pgs: 109 active+clean; 3.1 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 84 MiB/s wr, 1.35k op/s 2023-09-23T21:02:01.763 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:02:01 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:02:01.769 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:02:01.817 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:02:01.822 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:02:01.822 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:02:01.865 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:02:01.873 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:02:01 smithi026 ceph-mon[175151]: failed to encode map e262 with expected crc 2023-09-23T21:02:01.873 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:02:01 smithi026 ceph-mon[175151]: failed to encode map e262 with expected crc 2023-09-23T21:02:01.873 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:02:01 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/1882186742' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-09-23T21:02:01.873 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:02:01 smithi026 ceph-mon[175151]: osdmap e262: 8 total, 8 up, 8 in 2023-09-23T21:02:01.873 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:02:01 smithi026 ceph-mon[175151]: failed to encode map e262 with expected crc 2023-09-23T21:02:01.873 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:02:01 smithi026 ceph-mon[175151]: pgmap v688: 109 pgs: 109 active+clean; 3.1 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 84 MiB/s wr, 1.35k op/s 2023-09-23T21:02:01.873 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:02:01 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:02:01.923 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:02:02.006 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:02:02.275 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:02:02.282 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:02:02.283 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:02:02.383 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:02:02.426 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:02:02.469 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:02:02.471 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:02:02.471 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:02:02.572 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:02:02.635 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:02:02.694 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:02:02.704 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:02:02.712 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:02:02.712 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:02:02.740 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 31 16 55778 55762 112.378 75.4375 0.0172192 0.00889212 2023-09-23T21:02:02.813 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:02:02.887 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:02:03.338 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:02:03.342 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:02:03.343 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:02:03.443 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:02:03.519 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:02:03.592 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:02:03 smithi026 ceph-mon[175151]: pgmap v689: 109 pgs: 109 active+clean; 3.3 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 84 MiB/s wr, 1.34k op/s 2023-09-23T21:02:03.741 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 32 16 57077 57061 111.4 81.1875 0.0114807 0.00897154 2023-09-23T21:02:03.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:02:03 smithi070 ceph-mon[168282]: pgmap v689: 109 pgs: 109 active+clean; 3.3 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 84 MiB/s wr, 1.34k op/s 2023-09-23T21:02:03.883 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:02:03 smithi026 ceph-mon[179688]: pgmap v689: 109 pgs: 109 active+clean; 3.3 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 84 MiB/s wr, 1.34k op/s 2023-09-23T21:02:03.916 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:02:03.921 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:02:03.921 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:02:04.022 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:02:04.107 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:02:04.232 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:02:04.245 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:02:04.245 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:02:04.346 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:02:04.380 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:02:04.418 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:02:04.422 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:02:04.422 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:02:04.523 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:02:04.556 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:02:04.580 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0: {} 2023-09-23T21:02:04.608 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:02:04.614 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:02:04.614 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:02:04.622 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1: {} 2023-09-23T21:02:04.651 INFO:teuthology.orchestra.run.smithi026.stderr:nodeep-scrub is unset 2023-09-23T21:02:04.659 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2: {} 2023-09-23T21:02:04.695 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3: {} 2023-09-23T21:02:04.715 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:02:04.728 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4: {} 2023-09-23T21:02:04.741 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 33 16 58169 58153 110.093 68.25 0.0076358 0.00907681 2023-09-23T21:02:04.763 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:02:04 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/1505867243' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:02:04.763 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:02:04 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/1567688918' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:02:04.763 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:02:04 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:02:04.764 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:02:04.769 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T21:02:04.813 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:02:04.816 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6: {} 2023-09-23T21:02:04.816 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:02:04.817 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:02:04.853 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T21:02:04.857 INFO:teuthology.orchestra.run.smithi026.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T21:02:04.857 INFO:teuthology.orchestra.run.smithi026.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T21:02:04.857 INFO:teuthology.orchestra.run.smithi026.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T21:02:04.857 INFO:teuthology.orchestra.run.smithi026.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T21:02:04.857 INFO:teuthology.orchestra.run.smithi026.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T21:02:04.858 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T21:02:04.858 INFO:teuthology.orchestra.run.smithi026.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T21:02:04.858 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T21:02:04.867 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:02:04 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/1505867243' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:02:04.867 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:02:04 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/1567688918' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:02:04.867 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:02:04 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:02:04.869 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:02:04 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/1505867243' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:02:04.869 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:02:04 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/1567688918' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:02:04.869 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:02:04 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:02:04.917 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:02:04.968 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:02:05.017 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:02:05.025 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:02:05.025 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:02:05.126 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:02:05.166 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:02:05.220 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:02:05.225 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:02:05.225 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:02:05.326 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:02:05.368 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:02:05.414 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:02:05.416 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:02:05.416 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:02:05.501 INFO:teuthology.orchestra.run.smithi026.stdout:ERROR: (22) Invalid argument 2023-09-23T21:02:05.501 INFO:teuthology.orchestra.run.smithi026.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T21:02:05.517 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:02:05.552 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:02:05.598 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:02:05.601 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:02:05.601 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:02:05.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:02:05 smithi070 ceph-mon[168282]: pgmap v690: 109 pgs: 109 active+clean; 3.4 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 90 MiB/s wr, 1.45k op/s 2023-09-23T21:02:05.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:02:05 smithi070 ceph-mon[168282]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T21:02:05.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:02:05 smithi070 ceph-mon[168282]: Cluster is now healthy 2023-09-23T21:02:05.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:02:05 smithi070 ceph-mon[168282]: failed to encode map e263 with expected crc 2023-09-23T21:02:05.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:02:05 smithi070 ceph-mon[168282]: failed to encode map e263 with expected crc 2023-09-23T21:02:05.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:02:05 smithi070 ceph-mon[168282]: failed to encode map e263 with expected crc 2023-09-23T21:02:05.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:02:05 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T21:02:05.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:02:05 smithi070 ceph-mon[168282]: osdmap e263: 8 total, 8 up, 8 in 2023-09-23T21:02:05.702 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:02:05.741 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 34 16 59406 59390 109.129 77.3125 0.01194 0.00915856 2023-09-23T21:02:05.789 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:02:05 smithi026 ceph-mon[175151]: pgmap v690: 109 pgs: 109 active+clean; 3.4 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 90 MiB/s wr, 1.45k op/s 2023-09-23T21:02:05.789 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:02:05 smithi026 ceph-mon[175151]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T21:02:05.789 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:02:05 smithi026 ceph-mon[175151]: Cluster is now healthy 2023-09-23T21:02:05.789 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:02:05 smithi026 ceph-mon[175151]: failed to encode map e263 with expected crc 2023-09-23T21:02:05.790 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:02:05 smithi026 ceph-mon[175151]: failed to encode map e263 with expected crc 2023-09-23T21:02:05.790 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:02:05 smithi026 ceph-mon[175151]: failed to encode map e263 with expected crc 2023-09-23T21:02:05.790 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:02:05 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T21:02:05.790 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:02:05 smithi026 ceph-mon[175151]: osdmap e263: 8 total, 8 up, 8 in 2023-09-23T21:02:05.790 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:02:05 smithi026 ceph-mon[179688]: pgmap v690: 109 pgs: 109 active+clean; 3.4 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 90 MiB/s wr, 1.45k op/s 2023-09-23T21:02:05.790 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:02:05 smithi026 ceph-mon[179688]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T21:02:05.790 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:02:05 smithi026 ceph-mon[179688]: Cluster is now healthy 2023-09-23T21:02:05.790 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:02:05 smithi026 ceph-mon[179688]: failed to encode map e263 with expected crc 2023-09-23T21:02:05.790 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:02:05 smithi026 ceph-mon[179688]: failed to encode map e263 with expected crc 2023-09-23T21:02:05.790 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:02:05 smithi026 ceph-mon[179688]: failed to encode map e263 with expected crc 2023-09-23T21:02:05.791 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:02:05 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T21:02:05.791 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:02:05 smithi026 ceph-mon[179688]: osdmap e263: 8 total, 8 up, 8 in 2023-09-23T21:02:05.797 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:02:06.062 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:02:06.067 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:02:06.068 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:02:06.169 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:02:06.211 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:02:06.217 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_ops_in_flight 2023-09-23T21:02:06.274 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:02:06.279 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:02:06.279 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:02:06.380 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:02:06.442 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:02:06.504 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:02:06.511 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:02:06.511 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:02:06.612 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:02:06.695 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:02:06.742 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 35 16 60654 60638 108.24 78 0.019287 0.00923269 2023-09-23T21:02:06.758 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T21:02:06.768 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:02:06.769 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:02:06.870 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:02:06.943 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:02:06.983 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T21:02:07.010 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:02:07.016 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:02:07.016 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:02:07.117 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:02:07.151 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:02:07.260 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:02:07.266 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:02:07.266 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:02:07.367 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:02:07.441 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:02:07.488 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:02:07.496 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:02:07.496 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:02:07.543 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T21:02:07.597 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:02:07.645 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:02:07.671 INFO:teuthology.orchestra.run.smithi026.stderr:reweighted osd.1 to 1 (10000) 2023-09-23T21:02:07.700 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:02:07 smithi026 ceph-mon[175151]: pgmap v692: 109 pgs: 109 active+clean; 3.5 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 74 MiB/s wr, 1.19k op/s 2023-09-23T21:02:07.701 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:02:07 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/109795809' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-09-23T21:02:07.701 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:02:07 smithi026 ceph-mon[179688]: pgmap v692: 109 pgs: 109 active+clean; 3.5 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 74 MiB/s wr, 1.19k op/s 2023-09-23T21:02:07.701 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:02:07 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/109795809' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-09-23T21:02:07.743 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 36 16 61917 61901 107.421 78.9375 0.0244335 0.00930349 2023-09-23T21:02:07.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:02:07 smithi070 ceph-mon[168282]: pgmap v692: 109 pgs: 109 active+clean; 3.5 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 74 MiB/s wr, 1.19k op/s 2023-09-23T21:02:07.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:02:07 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/109795809' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-09-23T21:02:07.929 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:02:07.933 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:02:07.934 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:02:08.034 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:02:08.082 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T21:02:08.097 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph osd reweight 2 1 2023-09-23T21:02:08.121 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:02:08.218 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:02:08.222 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:02:08.223 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:02:08.324 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:02:08.507 INFO:teuthology.orchestra.run.smithi070.stdout:ERROR: (22) Invalid argument 2023-09-23T21:02:08.507 INFO:teuthology.orchestra.run.smithi070.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T21:02:08.507 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:02:08.544 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:02:08.652 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:02:08.657 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:02:08.658 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:02:08.744 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 37 16 62997 62981 106.342 67.5 0.00671452 0.00939914 2023-09-23T21:02:08.759 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:02:08.797 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:02:08 smithi070 ceph-mon[168282]: failed to encode map e264 with expected crc 2023-09-23T21:02:08.797 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:02:08 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/109795809' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-09-23T21:02:08.797 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:02:08 smithi070 ceph-mon[168282]: osdmap e264: 8 total, 8 up, 8 in 2023-09-23T21:02:08.797 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:02:08 smithi070 ceph-mon[168282]: failed to encode map e264 with expected crc 2023-09-23T21:02:08.797 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:02:08 smithi070 ceph-mon[168282]: failed to encode map e264 with expected crc 2023-09-23T21:02:08.797 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:02:08 smithi070 ceph-mon[168282]: failed to encode map e264 with expected crc 2023-09-23T21:02:08.801 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:02:08.804 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:02:08 smithi026 ceph-mon[175151]: failed to encode map e264 with expected crc 2023-09-23T21:02:08.804 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:02:08 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/109795809' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-09-23T21:02:08.804 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:02:08 smithi026 ceph-mon[175151]: osdmap e264: 8 total, 8 up, 8 in 2023-09-23T21:02:08.804 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:02:08 smithi026 ceph-mon[175151]: failed to encode map e264 with expected crc 2023-09-23T21:02:08.804 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:02:08 smithi026 ceph-mon[175151]: failed to encode map e264 with expected crc 2023-09-23T21:02:08.804 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:02:08 smithi026 ceph-mon[175151]: failed to encode map e264 with expected crc 2023-09-23T21:02:08.805 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:02:08 smithi026 ceph-mon[179688]: failed to encode map e264 with expected crc 2023-09-23T21:02:08.805 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:02:08 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/109795809' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-09-23T21:02:08.805 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:02:08 smithi026 ceph-mon[179688]: osdmap e264: 8 total, 8 up, 8 in 2023-09-23T21:02:08.805 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:02:08 smithi026 ceph-mon[179688]: failed to encode map e264 with expected crc 2023-09-23T21:02:08.805 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:02:08 smithi026 ceph-mon[179688]: failed to encode map e264 with expected crc 2023-09-23T21:02:08.805 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:02:08 smithi026 ceph-mon[179688]: failed to encode map e264 with expected crc 2023-09-23T21:02:08.858 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:02:08.861 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:02:08.862 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:02:08.908 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_blocked_ops 2023-09-23T21:02:08.963 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:02:09.021 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:02:09.079 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:02:09.085 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:02:09.086 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:02:09.186 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:02:09.259 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:02:09.322 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:02:09.327 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:02:09.327 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:02:09.381 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:02:09.428 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:02:09.516 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:02:09.535 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:02:09.567 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T21:02:09.645 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 21:02:09 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[164029]: ::ffff:172.21.15.70 - - [23/Sep/2023:21:02:09] "GET /metrics HTTP/1.1" 200 36290 "" "Prometheus/2.43.0" 2023-09-23T21:02:09.744 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 38 16 64121 64105 105.392 70.25 0.0190472 0.00948348 2023-09-23T21:02:09.799 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T21:02:09.810 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:02:09.811 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:02:09.912 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:02:09.919 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:02:09 smithi026 ceph-mon[179688]: pgmap v694: 109 pgs: 109 active+clean; 3.6 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 68 MiB/s wr, 1.08k op/s 2023-09-23T21:02:09.919 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:02:09 smithi026 ceph-mon[175151]: pgmap v694: 109 pgs: 109 active+clean; 3.6 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 68 MiB/s wr, 1.08k op/s 2023-09-23T21:02:09.952 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:02:09 smithi070 ceph-mon[168282]: pgmap v694: 109 pgs: 109 active+clean; 3.6 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 68 MiB/s wr, 1.08k op/s 2023-09-23T21:02:09.955 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:02:10.014 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:02:10.019 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:02:10.019 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:02:10.120 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:02:10.185 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:02:10.388 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:02:10.391 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:02:10.391 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:02:10.492 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:02:10.553 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:02:10.630 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:02:10.633 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:02:10.633 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:02:10.734 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:02:10.744 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 39 16 65204 65188 104.425 67.6875 0.0199804 0.00957134 2023-09-23T21:02:10.790 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:02:10.889 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:02:10.894 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:02:10.894 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:02:10.995 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:02:11.027 INFO:teuthology.orchestra.run.smithi070.stdout:ERROR: (22) Invalid argument 2023-09-23T21:02:11.027 INFO:teuthology.orchestra.run.smithi070.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T21:02:11.049 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:02:11.144 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:02:11.146 INFO:teuthology.orchestra.run.smithi026.stderr:noscrub is set 2023-09-23T21:02:11.146 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:02:11 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/2767045626' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T21:02:11.147 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:02:11.147 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:02:11.248 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:02:11.289 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:02:11.356 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:02:11 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/2767045626' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T21:02:11.356 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:02:11 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/2767045626' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T21:02:11.392 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:02:11.397 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:02:11.398 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:02:11.399 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_historic_ops 2023-09-23T21:02:11.498 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:02:11.537 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:02:11.589 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:02:11.591 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0: {} 2023-09-23T21:02:11.593 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:02:11.594 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:02:11.638 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1: {} 2023-09-23T21:02:11.671 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2: {} 2023-09-23T21:02:11.695 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:02:11.711 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3: {} 2023-09-23T21:02:11.746 INFO:tasks.radosbench.radosbench.0.smithi026.stdout:2023-09-23T21:02:11.746580+0000 min lat: 0.00122595 max lat: 0.190811 avg lat: 0.00965291 2023-09-23T21:02:11.746 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T21:02:11.746 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 40 16 66309 66293 103.537 69.0625 0.0247473 0.00965291 2023-09-23T21:02:11.774 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:02:11.780 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4: {} 2023-09-23T21:02:11.840 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T21:02:11.867 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T21:02:11.876 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:02:11.878 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:02:11.878 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6: {} 2023-09-23T21:02:11.916 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T21:02:11.928 INFO:teuthology.orchestra.run.smithi026.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T21:02:11.929 INFO:teuthology.orchestra.run.smithi026.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T21:02:11.929 INFO:teuthology.orchestra.run.smithi026.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T21:02:11.929 INFO:teuthology.orchestra.run.smithi026.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T21:02:11.929 INFO:teuthology.orchestra.run.smithi026.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T21:02:11.929 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T21:02:11.929 INFO:teuthology.orchestra.run.smithi026.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T21:02:11.929 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T21:02:11.978 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:02:12.009 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:02:12.064 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:02:12.068 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:02:12.068 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:02:12.169 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:02:12.178 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T21:02:12.220 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:02:12.326 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:02:12.329 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:02:12 smithi070 ceph-mon[168282]: pgmap v695: 109 pgs: 109 active+clean; 3.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 73 MiB/s wr, 1.17k op/s 2023-09-23T21:02:12.330 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:02:12 smithi070 ceph-mon[168282]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:02:12.330 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:02:12 smithi070 ceph-mon[168282]: failed to encode map e265 with expected crc 2023-09-23T21:02:12.330 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:02:12 smithi070 ceph-mon[168282]: failed to encode map e265 with expected crc 2023-09-23T21:02:12.330 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:02:12 smithi070 ceph-mon[168282]: failed to encode map e265 with expected crc 2023-09-23T21:02:12.330 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:02:12 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/2767045626' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T21:02:12.330 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:02:12 smithi070 ceph-mon[168282]: osdmap e265: 8 total, 8 up, 8 in 2023-09-23T21:02:12.330 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:02:12 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/236494329' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:02:12.331 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:02:12.331 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:02:12.366 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:02:12 smithi026 ceph-mon[179688]: pgmap v695: 109 pgs: 109 active+clean; 3.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 73 MiB/s wr, 1.17k op/s 2023-09-23T21:02:12.366 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:02:12 smithi026 ceph-mon[179688]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:02:12.366 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:02:12 smithi026 ceph-mon[179688]: failed to encode map e265 with expected crc 2023-09-23T21:02:12.366 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:02:12 smithi026 ceph-mon[179688]: failed to encode map e265 with expected crc 2023-09-23T21:02:12.366 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:02:12 smithi026 ceph-mon[179688]: failed to encode map e265 with expected crc 2023-09-23T21:02:12.366 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:02:12 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/2767045626' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T21:02:12.366 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:02:12 smithi026 ceph-mon[179688]: osdmap e265: 8 total, 8 up, 8 in 2023-09-23T21:02:12.366 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:02:12 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/236494329' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:02:12.367 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:02:12 smithi026 ceph-mon[175151]: pgmap v695: 109 pgs: 109 active+clean; 3.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 73 MiB/s wr, 1.17k op/s 2023-09-23T21:02:12.367 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:02:12 smithi026 ceph-mon[175151]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:02:12.367 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:02:12 smithi026 ceph-mon[175151]: failed to encode map e265 with expected crc 2023-09-23T21:02:12.367 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:02:12 smithi026 ceph-mon[175151]: failed to encode map e265 with expected crc 2023-09-23T21:02:12.367 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:02:12 smithi026 ceph-mon[175151]: failed to encode map e265 with expected crc 2023-09-23T21:02:12.367 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:02:12 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/2767045626' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T21:02:12.367 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:02:12 smithi026 ceph-mon[175151]: osdmap e265: 8 total, 8 up, 8 in 2023-09-23T21:02:12.367 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:02:12 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/236494329' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:02:12.431 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:02:12.489 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:02:12.548 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:02:12.551 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:02:12.552 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:02:12.652 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:02:12.722 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:02:12.746 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 41 16 67420 67404 102.705 69.4375 0.00624947 0.00973238 2023-09-23T21:02:12.801 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T21:02:12.808 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:02:12.808 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:02:12.909 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:02:12.962 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:02:13.018 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:02:13.029 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:02:13.029 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:02:13.130 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:02:13.169 INFO:teuthology.orchestra.run.smithi026.stderr:reweighted osd.2 to 1 (10000) 2023-09-23T21:02:13.202 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:02:13.280 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:02:13.283 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:02:13 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/2668179268' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T21:02:13.283 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:02:13 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T21:02:13.284 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:02:13.284 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:02:13.385 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:02:13.440 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:02:13 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/2668179268' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T21:02:13.440 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:02:13 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T21:02:13.444 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:02:13 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/2668179268' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T21:02:13.445 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:02:13 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T21:02:13.454 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:02:13.608 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:02:13.616 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:02:13.617 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:02:13.647 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph osd reweight 3 1 2023-09-23T21:02:13.692 INFO:teuthology.orchestra.run.smithi070.stdout:{ 2023-09-23T21:02:13.692 INFO:teuthology.orchestra.run.smithi070.stdout: "size": 20, 2023-09-23T21:02:13.692 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 600, 2023-09-23T21:02:13.692 INFO:teuthology.orchestra.run.smithi070.stdout: "ops": [ 2023-09-23T21:02:13.693 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:13.693 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.34797.0:114997 6.12 6.28c54f92 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e132)", 2023-09-23T21:02:13.693 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T20:54:12.985231+0000", 2023-09-23T21:02:13.693 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 480.70183818300001, 2023-09-23T21:02:13.693 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.939145275, 2023-09-23T21:02:13.693 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:02:13.693 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "reached pg", 2023-09-23T21:02:13.693 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:02:13.693 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.34797", 2023-09-23T21:02:13.693 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T21:02:13.693 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 114997 2023-09-23T21:02:13.693 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:13.694 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:02:13.694 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:13.694 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:02:13.694 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:54:12.985231+0000", 2023-09-23T21:02:13.694 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:02:13.694 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:13.694 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:13.694 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:02:13.694 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:54:12.985231+0000", 2023-09-23T21:02:13.694 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:02:13.694 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:13.694 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:13.695 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:02:13.695 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:54:12.985228+0000", 2023-09-23T21:02:13.695 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999976 2023-09-23T21:02:13.695 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:13.695 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:13.695 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:02:13.695 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:54:12.985282+0000", 2023-09-23T21:02:13.695 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.3285000000000002e-05 2023-09-23T21:02:13.695 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:13.695 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:13.695 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:02:13.695 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:54:12.985298+0000", 2023-09-23T21:02:13.696 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.6677e-05 2023-09-23T21:02:13.696 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:13.696 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:13.696 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:02:13.696 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:54:12.985306+0000", 2023-09-23T21:02:13.696 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.8590000000000005e-06 2023-09-23T21:02:13.696 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:13.696 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:13.696 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:02:13.696 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:54:12.985354+0000", 2023-09-23T21:02:13.696 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.8124000000000002e-05 2023-09-23T21:02:13.696 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:13.696 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:13.697 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:02:13.697 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:54:12.985364+0000", 2023-09-23T21:02:13.697 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.6500000000000008e-06 2023-09-23T21:02:13.697 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:13.697 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:13.697 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:02:13.697 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:54:13.924362+0000", 2023-09-23T21:02:13.697 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.93899794599999997 2023-09-23T21:02:13.697 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:13.697 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:13.697 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:02:13.697 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:54:13.924376+0000", 2023-09-23T21:02:13.697 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.4238e-05 2023-09-23T21:02:13.698 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:02:13.698 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:02:13.698 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:02:13.698 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:13.698 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:13.698 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.34797.0:156782 6.12 6:4ab9e603:::benchmark_data_smithi026_223868_object55656:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e150)", 2023-09-23T21:02:13.698 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T20:55:07.352049+0000", 2023-09-23T21:02:13.698 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 426.33502032199999, 2023-09-23T21:02:13.698 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.014989538, 2023-09-23T21:02:13.698 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:02:13.698 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:02:13.698 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:02:13.699 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.34797", 2023-09-23T21:02:13.699 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T21:02:13.699 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 156782 2023-09-23T21:02:13.699 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:13.699 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:02:13.699 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:13.699 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:02:13.699 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:07.352049+0000", 2023-09-23T21:02:13.699 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:02:13.699 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:13.699 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:13.699 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:02:13.699 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:07.352049+0000", 2023-09-23T21:02:13.700 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:02:13.700 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:13.700 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:13.700 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:02:13.700 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:07.352047+0000", 2023-09-23T21:02:13.700 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T21:02:13.700 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:13.700 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:13.700 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:02:13.700 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:07.352052+0000", 2023-09-23T21:02:13.700 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.9280000000000001e-06 2023-09-23T21:02:13.700 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:13.700 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:13.701 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:02:13.701 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:07.352064+0000", 2023-09-23T21:02:13.701 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.1902999999999999e-05 2023-09-23T21:02:13.701 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:13.701 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:13.701 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:02:13.701 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:07.352070+0000", 2023-09-23T21:02:13.701 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.8459999999999996e-06 2023-09-23T21:02:13.701 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:13.701 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:13.701 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:02:13.701 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:07.352358+0000", 2023-09-23T21:02:13.701 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00028820300000000001 2023-09-23T21:02:13.702 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:13.702 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:13.702 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:02:13.702 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:07.352372+0000", 2023-09-23T21:02:13.702 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.3669000000000001e-05 2023-09-23T21:02:13.702 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:13.702 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:13.702 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:02:13.702 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.361399+0000", 2023-09-23T21:02:13.702 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.0090268060000001 2023-09-23T21:02:13.702 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:13.702 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:13.703 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:02:13.703 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.361509+0000", 2023-09-23T21:02:13.703 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000110451 2023-09-23T21:02:13.703 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:13.703 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:13.703 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 2,3", 2023-09-23T21:02:13.703 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.361803+0000", 2023-09-23T21:02:13.703 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00029413900000000001 2023-09-23T21:02:13.703 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:13.703 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:13.703 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:02:13.703 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.363432+0000", 2023-09-23T21:02:13.703 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.001628188 2023-09-23T21:02:13.704 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:13.704 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:13.704 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:02:13.704 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.366824+0000", 2023-09-23T21:02:13.704 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0033921910000000001 2023-09-23T21:02:13.704 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:13.704 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:13.704 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:02:13.704 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.366953+0000", 2023-09-23T21:02:13.704 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00012964399999999999 2023-09-23T21:02:13.704 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:13.704 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:13.704 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:02:13.705 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.367000+0000", 2023-09-23T21:02:13.705 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.6319999999999997e-05 2023-09-23T21:02:13.705 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:13.705 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:13.705 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:02:13.705 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.367038+0000", 2023-09-23T21:02:13.705 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.8498000000000003e-05 2023-09-23T21:02:13.705 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:02:13.705 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:02:13.705 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:02:13.705 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:13.705 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:13.705 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.34797.0:156812 6.12 6:4c46565a:::benchmark_data_smithi026_223868_object55686:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e150)", 2023-09-23T21:02:13.706 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T20:55:07.386674+0000", 2023-09-23T21:02:13.706 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 426.30039516400001, 2023-09-23T21:02:13.706 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.98619251500000005, 2023-09-23T21:02:13.706 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:02:13.706 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:02:13.706 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:02:13.706 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.34797", 2023-09-23T21:02:13.706 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T21:02:13.706 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 156812 2023-09-23T21:02:13.706 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:13.706 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:02:13.706 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:13.707 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:02:13.707 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:07.386674+0000", 2023-09-23T21:02:13.707 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:02:13.707 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:13.707 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:13.707 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:02:13.707 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:07.386674+0000", 2023-09-23T21:02:13.707 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:02:13.707 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:13.707 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:13.707 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:02:13.707 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:07.386672+0000", 2023-09-23T21:02:13.707 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T21:02:13.708 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:13.708 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:13.708 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:02:13.708 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:07.386681+0000", 2023-09-23T21:02:13.708 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.2679999999999994e-06 2023-09-23T21:02:13.708 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:13.708 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:13.708 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:02:13.708 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:07.386702+0000", 2023-09-23T21:02:13.708 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.173e-05 2023-09-23T21:02:13.708 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:13.708 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:13.709 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:02:13.709 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:07.386710+0000", 2023-09-23T21:02:13.709 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.0069999999999997e-06 2023-09-23T21:02:13.709 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:13.709 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:13.709 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:02:13.709 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:07.386770+0000", 2023-09-23T21:02:13.709 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.9499000000000001e-05 2023-09-23T21:02:13.709 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:13.709 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:13.709 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:02:13.709 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:07.386780+0000", 2023-09-23T21:02:13.710 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.5619999999999993e-06 2023-09-23T21:02:13.710 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:13.710 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:13.710 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:02:13.710 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.363453+0000", 2023-09-23T21:02:13.710 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.97667318999999997 2023-09-23T21:02:13.710 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:13.710 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:13.710 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:02:13.710 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.363538+0000", 2023-09-23T21:02:13.710 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.5484999999999999e-05 2023-09-23T21:02:13.710 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:13.710 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:13.711 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 2,3", 2023-09-23T21:02:13.711 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.363717+0000", 2023-09-23T21:02:13.711 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00017910699999999999 2023-09-23T21:02:13.711 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:13.711 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:13.711 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:02:13.711 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.366914+0000", 2023-09-23T21:02:13.711 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0031969059999999998 2023-09-23T21:02:13.711 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:13.711 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:13.711 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:02:13.711 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.372737+0000", 2023-09-23T21:02:13.711 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00582243 2023-09-23T21:02:13.712 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:13.712 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:13.712 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:02:13.712 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.372790+0000", 2023-09-23T21:02:13.712 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.3454e-05 2023-09-23T21:02:13.712 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:13.712 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:13.712 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:02:13.712 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.372821+0000", 2023-09-23T21:02:13.712 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.0725999999999997e-05 2023-09-23T21:02:13.712 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:13.712 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:13.712 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:02:13.713 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.372866+0000", 2023-09-23T21:02:13.713 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.5433999999999999e-05 2023-09-23T21:02:13.713 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:02:13.713 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:02:13.713 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:02:13.713 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:13.713 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:13.713 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.34797.0:156863 6.12 6:48481dbe:::benchmark_data_smithi026_223868_object55737:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e150)", 2023-09-23T21:02:13.713 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T20:55:07.442400+0000", 2023-09-23T21:02:13.713 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 426.24466879400001, 2023-09-23T21:02:13.713 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.93060301300000003, 2023-09-23T21:02:13.713 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:02:13.714 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:02:13.714 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:02:13.714 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.34797", 2023-09-23T21:02:13.714 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T21:02:13.714 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 156863 2023-09-23T21:02:13.714 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:13.714 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:02:13.714 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:13.714 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:02:13.714 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:07.442400+0000", 2023-09-23T21:02:13.714 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:02:13.714 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:13.714 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:13.715 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:02:13.715 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:07.442400+0000", 2023-09-23T21:02:13.715 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:02:13.715 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:13.715 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:13.715 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:02:13.715 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:07.442399+0000", 2023-09-23T21:02:13.715 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T21:02:13.715 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:13.715 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:13.715 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:02:13.715 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:07.442405+0000", 2023-09-23T21:02:13.715 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.7490000000000001e-06 2023-09-23T21:02:13.716 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:13.716 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:13.716 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:02:13.716 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:07.442421+0000", 2023-09-23T21:02:13.716 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.6110999999999999e-05 2023-09-23T21:02:13.716 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:13.716 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:13.716 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:02:13.716 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:07.442428+0000", 2023-09-23T21:02:13.716 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.9349999999999997e-06 2023-09-23T21:02:13.716 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:13.716 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:13.716 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:02:13.717 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:07.442498+0000", 2023-09-23T21:02:13.717 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.9488999999999998e-05 2023-09-23T21:02:13.717 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:13.717 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:13.717 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:02:13.717 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:07.442508+0000", 2023-09-23T21:02:13.717 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.0329000000000001e-05 2023-09-23T21:02:13.717 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:13.717 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:13.717 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:02:13.717 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.364821+0000", 2023-09-23T21:02:13.717 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.92231251999999997 2023-09-23T21:02:13.717 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:13.718 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:13.718 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:02:13.718 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.364917+0000", 2023-09-23T21:02:13.718 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.5866000000000007e-05 2023-09-23T21:02:13.718 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:13.718 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:13.718 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 2,3", 2023-09-23T21:02:13.718 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.365161+0000", 2023-09-23T21:02:13.718 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00024452300000000002 2023-09-23T21:02:13.718 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:13.718 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:13.718 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:02:13.719 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.367047+0000", 2023-09-23T21:02:13.719 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.001885947 2023-09-23T21:02:13.719 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:13.719 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:13.719 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:02:13.719 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.372916+0000", 2023-09-23T21:02:13.719 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0058686110000000001 2023-09-23T21:02:13.719 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:13.719 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:13.719 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:02:13.719 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.372948+0000", 2023-09-23T21:02:13.719 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.2159000000000002e-05 2023-09-23T21:02:13.719 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:13.720 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:13.720 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:02:13.720 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.372972+0000", 2023-09-23T21:02:13.720 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.4244999999999999e-05 2023-09-23T21:02:13.720 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:13.720 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:13.720 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:02:13.720 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.373003+0000", 2023-09-23T21:02:13.720 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.112e-05 2023-09-23T21:02:13.720 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:02:13.720 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:02:13.720 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:02:13.720 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:13.721 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:13.721 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.34797.0:156879 6.12 6:4b4fed44:::benchmark_data_smithi026_223868_object55753:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e150)", 2023-09-23T21:02:13.721 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T20:55:07.459292+0000", 2023-09-23T21:02:13.721 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 426.22777730500002, 2023-09-23T21:02:13.721 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.91741719200000005, 2023-09-23T21:02:13.721 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:02:13.721 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:02:13.721 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:02:13.721 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.34797", 2023-09-23T21:02:13.721 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T21:02:13.721 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 156879 2023-09-23T21:02:13.721 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:13.721 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:02:13.722 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:13.722 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:02:13.722 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:07.459292+0000", 2023-09-23T21:02:13.722 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:02:13.722 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:13.722 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:13.722 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:02:13.722 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:07.459292+0000", 2023-09-23T21:02:13.722 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:02:13.722 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:13.722 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:13.722 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:02:13.723 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:07.459290+0000", 2023-09-23T21:02:13.723 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999981 2023-09-23T21:02:13.723 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:13.723 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:13.723 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:02:13.723 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:07.459299+0000", 2023-09-23T21:02:13.723 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.1679999999999993e-06 2023-09-23T21:02:13.723 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:13.723 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:13.723 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:02:13.723 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:07.459319+0000", 2023-09-23T21:02:13.723 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.9945999999999998e-05 2023-09-23T21:02:13.724 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:13.724 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:13.724 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:02:13.724 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:07.459328+0000", 2023-09-23T21:02:13.724 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.5439999999999999e-06 2023-09-23T21:02:13.724 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:13.724 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:13.724 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:02:13.724 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:07.459397+0000", 2023-09-23T21:02:13.724 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.9640000000000004e-05 2023-09-23T21:02:13.724 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:13.724 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:13.724 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:02:13.725 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:07.459408+0000", 2023-09-23T21:02:13.725 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.1228e-05 2023-09-23T21:02:13.725 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:13.725 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:13.725 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:02:13.725 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.367075+0000", 2023-09-23T21:02:13.725 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.90766636599999995 2023-09-23T21:02:13.725 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:13.725 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:13.725 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:02:13.725 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.367196+0000", 2023-09-23T21:02:13.725 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000121274 2023-09-23T21:02:13.726 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:13.726 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:13.726 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 2,3", 2023-09-23T21:02:13.726 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.367465+0000", 2023-09-23T21:02:13.726 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00026886999999999999 2023-09-23T21:02:13.726 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:13.726 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:13.726 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:02:13.726 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.372639+0000", 2023-09-23T21:02:13.726 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0051736430000000003 2023-09-23T21:02:13.726 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:13.726 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:13.727 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:02:13.727 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.376593+0000", 2023-09-23T21:02:13.727 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0039539919999999999 2023-09-23T21:02:13.727 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:13.727 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:13.727 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:02:13.727 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.376628+0000", 2023-09-23T21:02:13.727 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.5759000000000001e-05 2023-09-23T21:02:13.727 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:13.727 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:13.727 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:02:13.727 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.376671+0000", 2023-09-23T21:02:13.727 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.2969999999999997e-05 2023-09-23T21:02:13.728 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:13.728 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:13.728 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:02:13.728 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.376709+0000", 2023-09-23T21:02:13.728 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.7540000000000003e-05 2023-09-23T21:02:13.728 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:02:13.728 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:02:13.728 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:02:13.728 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:13.728 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:13.728 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.34797.0:156990 6.12 6:491f6f4b:::benchmark_data_smithi026_223868_object55864:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e150)", 2023-09-23T21:02:13.728 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T20:55:07.581755+0000", 2023-09-23T21:02:13.728 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 426.10531387700001, 2023-09-23T21:02:13.729 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.79863560899999997, 2023-09-23T21:02:13.729 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:02:13.729 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:02:13.729 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:02:13.729 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.34797", 2023-09-23T21:02:13.729 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T21:02:13.729 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 156990 2023-09-23T21:02:13.729 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:13.729 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:02:13.729 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:13.729 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:02:13.729 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:07.581755+0000", 2023-09-23T21:02:13.730 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:02:13.730 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:13.730 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:13.730 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:02:13.730 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:07.581755+0000", 2023-09-23T21:02:13.730 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:02:13.730 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:13.730 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:13.730 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:02:13.730 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:07.581754+0000", 2023-09-23T21:02:13.730 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T21:02:13.730 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:13.730 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:13.731 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:02:13.731 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:07.581760+0000", 2023-09-23T21:02:13.731 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.02e-06 2023-09-23T21:02:13.731 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:13.731 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:13.731 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:02:13.731 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:07.581773+0000", 2023-09-23T21:02:13.731 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.2921e-05 2023-09-23T21:02:13.731 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:13.731 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:13.731 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:02:13.731 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:07.581779+0000", 2023-09-23T21:02:13.732 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.7889999999999997e-06 2023-09-23T21:02:13.732 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:13.732 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:13.732 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:02:13.732 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:07.581838+0000", 2023-09-23T21:02:13.732 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.8850000000000001e-05 2023-09-23T21:02:13.732 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:13.732 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:13.732 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:02:13.732 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:07.581848+0000", 2023-09-23T21:02:13.732 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.9340000000000001e-06 2023-09-23T21:02:13.732 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:13.732 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:13.733 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:02:13.733 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.368883+0000", 2023-09-23T21:02:13.733 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.78703539700000003 2023-09-23T21:02:13.733 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:13.733 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:13.733 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:02:13.733 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.368984+0000", 2023-09-23T21:02:13.733 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000100717 2023-09-23T21:02:13.733 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:13.733 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:13.733 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 2,3", 2023-09-23T21:02:13.733 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.369202+0000", 2023-09-23T21:02:13.734 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000218304 2023-09-23T21:02:13.734 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:13.734 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:13.734 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:02:13.734 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.372876+0000", 2023-09-23T21:02:13.734 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.003673511 2023-09-23T21:02:13.734 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:13.734 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:13.734 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:02:13.734 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.380243+0000", 2023-09-23T21:02:13.734 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0073674049999999996 2023-09-23T21:02:13.734 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:13.734 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:13.735 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:02:13.735 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.380308+0000", 2023-09-23T21:02:13.735 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.4309999999999999e-05 2023-09-23T21:02:13.735 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:13.735 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:13.735 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:02:13.735 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.380357+0000", 2023-09-23T21:02:13.735 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.9095999999999999e-05 2023-09-23T21:02:13.735 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:13.735 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:13.735 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:02:13.735 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.380391+0000", 2023-09-23T21:02:13.736 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.3850999999999998e-05 2023-09-23T21:02:13.736 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:02:13.736 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:02:13.736 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:02:13.736 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:13.736 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:13.736 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.34797.0:157031 6.12 6:48e5af49:::benchmark_data_smithi026_223868_object55905:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e150)", 2023-09-23T21:02:13.736 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T20:55:07.628542+0000", 2023-09-23T21:02:13.736 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 426.05852656500002, 2023-09-23T21:02:13.736 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.75194783899999995, 2023-09-23T21:02:13.736 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:02:13.736 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:02:13.737 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:02:13.737 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.34797", 2023-09-23T21:02:13.737 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T21:02:13.737 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 157031 2023-09-23T21:02:13.737 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:13.737 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:02:13.737 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:13.737 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:02:13.737 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:07.628542+0000", 2023-09-23T21:02:13.737 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:02:13.737 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:13.737 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:13.738 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:02:13.738 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:07.628542+0000", 2023-09-23T21:02:13.738 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:02:13.738 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:13.738 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:13.738 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:02:13.738 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:07.628541+0000", 2023-09-23T21:02:13.738 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.999999 2023-09-23T21:02:13.738 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:13.738 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:13.738 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:02:13.739 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:07.628549+0000", 2023-09-23T21:02:13.739 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.9019999999999999e-06 2023-09-23T21:02:13.739 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:13.739 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:13.739 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:02:13.739 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:07.628562+0000", 2023-09-23T21:02:13.739 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.2986999999999999e-05 2023-09-23T21:02:13.739 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:13.739 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:13.739 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:02:13.739 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:07.628567+0000", 2023-09-23T21:02:13.739 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.322e-06 2023-09-23T21:02:13.739 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:13.740 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:13.740 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:02:13.740 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:07.628615+0000", 2023-09-23T21:02:13.740 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.7564000000000001e-05 2023-09-23T21:02:13.740 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:13.740 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:13.740 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:02:13.740 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:07.628622+0000", 2023-09-23T21:02:13.740 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.6440000000000003e-06 2023-09-23T21:02:13.740 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:13.740 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:13.740 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:02:13.741 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.370771+0000", 2023-09-23T21:02:13.741 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.74214981700000004 2023-09-23T21:02:13.741 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:13.741 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:13.741 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:02:13.741 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.370876+0000", 2023-09-23T21:02:13.741 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00010476000000000001 2023-09-23T21:02:13.741 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:13.741 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:13.741 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 2,3", 2023-09-23T21:02:13.741 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.371134+0000", 2023-09-23T21:02:13.741 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00025788100000000001 2023-09-23T21:02:13.742 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:13.742 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:13.742 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:02:13.742 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.373009+0000", 2023-09-23T21:02:13.742 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.001875334 2023-09-23T21:02:13.742 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:13.742 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:13.742 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:02:13.742 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.380281+0000", 2023-09-23T21:02:13.742 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0072716710000000004 2023-09-23T21:02:13.742 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:13.742 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:13.743 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:02:13.743 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.380422+0000", 2023-09-23T21:02:13.743 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00014123299999999999 2023-09-23T21:02:13.743 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:13.743 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:13.743 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:02:13.743 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.380460+0000", 2023-09-23T21:02:13.743 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.7372000000000001e-05 2023-09-23T21:02:13.743 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:13.743 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:13.743 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:02:13.743 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.380490+0000", 2023-09-23T21:02:13.743 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.0493000000000001e-05 2023-09-23T21:02:13.744 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:02:13.744 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:02:13.744 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:02:13.744 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:13.744 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:13.744 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.34797.0:157050 6.12 6:4be05e13:::benchmark_data_smithi026_223868_object55924:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e150)", 2023-09-23T21:02:13.744 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T20:55:07.649612+0000", 2023-09-23T21:02:13.744 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 426.03745736600001, 2023-09-23T21:02:13.744 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.73668807700000005, 2023-09-23T21:02:13.744 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:02:13.744 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:02:13.744 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:02:13.745 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.34797", 2023-09-23T21:02:13.745 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T21:02:13.745 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 157050 2023-09-23T21:02:13.745 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:13.745 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:02:13.745 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:13.745 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:02:13.745 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:07.649612+0000", 2023-09-23T21:02:13.745 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:02:13.745 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:13.745 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:13.745 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:02:13.745 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:07.649612+0000", 2023-09-23T21:02:13.746 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:02:13.746 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:13.746 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:13.746 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:02:13.746 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:07.649610+0000", 2023-09-23T21:02:13.746 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T21:02:13.746 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:13.746 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:13.746 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:02:13.746 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:07.649619+0000", 2023-09-23T21:02:13.746 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.7439999999999996e-06 2023-09-23T21:02:13.746 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:13.747 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:13.747 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:02:13.747 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:07.649633+0000", 2023-09-23T21:02:13.747 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.4056000000000001e-05 2023-09-23T21:02:13.747 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:13.747 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:13.747 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:02:13.747 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:07.649641+0000", 2023-09-23T21:02:13.747 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.5290000000000003e-06 2023-09-23T21:02:13.747 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:13.747 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:13.747 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:02:13.748 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:07.649701+0000", 2023-09-23T21:02:13.748 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.0715999999999999e-05 2023-09-23T21:02:13.748 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:13.748 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:13.748 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:02:13.748 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:07.649711+0000", 2023-09-23T21:02:13.748 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.5529999999999998e-06 2023-09-23T21:02:13.748 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:13.748 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:13.748 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:02:13.748 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.373038+0000", 2023-09-23T21:02:13.748 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.72332733999999999 2023-09-23T21:02:13.748 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:13.749 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:13.749 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:02:13.749 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.373159+0000", 2023-09-23T21:02:13.749 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000120704 2023-09-23T21:02:13.749 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:13.749 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:13.749 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 2,3", 2023-09-23T21:02:13.749 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.373420+0000", 2023-09-23T21:02:13.749 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00026133599999999998 2023-09-23T21:02:13.749 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:13.749 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:13.749 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:02:13.750 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.376599+0000", 2023-09-23T21:02:13.750 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0031784790000000001 2023-09-23T21:02:13.750 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:13.750 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:13.750 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:02:13.750 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.380519+0000", 2023-09-23T21:02:13.750 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0039201569999999996 2023-09-23T21:02:13.750 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:13.750 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:13.750 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:02:13.750 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.386243+0000", 2023-09-23T21:02:13.750 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0057240770000000002 2023-09-23T21:02:13.751 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:13.751 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:13.751 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:02:13.751 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.386271+0000", 2023-09-23T21:02:13.751 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.8518e-05 2023-09-23T21:02:13.751 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:13.751 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:13.751 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:02:13.751 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.386300+0000", 2023-09-23T21:02:13.751 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.8211000000000001e-05 2023-09-23T21:02:13.751 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:02:13.751 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:02:13.751 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:02:13.752 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:13.752 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:13.752 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.34797.0:157117 6.12 6:4d4c4893:::benchmark_data_smithi026_223868_object55991:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e150)", 2023-09-23T21:02:13.752 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T20:55:07.747668+0000", 2023-09-23T21:02:13.752 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 425.93940059900001, 2023-09-23T21:02:13.752 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.63872341099999996, 2023-09-23T21:02:13.752 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:02:13.752 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:02:13.752 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:02:13.752 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.34797", 2023-09-23T21:02:13.752 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T21:02:13.752 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 157117 2023-09-23T21:02:13.753 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:13.753 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:02:13.753 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:13.753 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:02:13.753 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:07.747668+0000", 2023-09-23T21:02:13.753 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:02:13.753 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:13.753 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:13.753 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:02:13.753 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:07.747668+0000", 2023-09-23T21:02:13.753 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:02:13.753 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:13.753 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:13.754 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:02:13.754 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:07.747667+0000", 2023-09-23T21:02:13.754 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T21:02:13.754 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:13.754 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:13.754 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:02:13.754 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:07.747673+0000", 2023-09-23T21:02:13.754 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.2580000000000001e-06 2023-09-23T21:02:13.754 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:13.754 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:13.754 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:02:13.754 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:07.747689+0000", 2023-09-23T21:02:13.755 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.5837000000000001e-05 2023-09-23T21:02:13.755 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:13.755 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:13.755 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:02:13.755 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:07.747698+0000", 2023-09-23T21:02:13.755 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.6039999999999993e-06 2023-09-23T21:02:13.755 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:13.755 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:13.755 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:02:13.755 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:07.747754+0000", 2023-09-23T21:02:13.755 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.6487000000000003e-05 2023-09-23T21:02:13.755 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:13.756 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:13.756 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:02:13.756 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:07.747764+0000", 2023-09-23T21:02:13.756 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.0108e-05 2023-09-23T21:02:13.756 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:13.756 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:13.756 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:02:13.756 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.374677+0000", 2023-09-23T21:02:13.756 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.62691302900000001 2023-09-23T21:02:13.756 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:13.756 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:13.756 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:02:13.757 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.374761+0000", 2023-09-23T21:02:13.757 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.3807999999999994e-05 2023-09-23T21:02:13.757 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:13.757 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:13.757 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 2,3", 2023-09-23T21:02:13.757 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.374943+0000", 2023-09-23T21:02:13.757 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00018220399999999999 2023-09-23T21:02:13.757 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:13.757 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:13.757 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:02:13.757 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.378533+0000", 2023-09-23T21:02:13.757 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0035901190000000001 2023-09-23T21:02:13.757 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:13.758 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:13.758 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:02:13.758 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.380547+0000", 2023-09-23T21:02:13.758 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.002013379 2023-09-23T21:02:13.758 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:13.758 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:13.758 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:02:13.758 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.386342+0000", 2023-09-23T21:02:13.758 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0057952480000000002 2023-09-23T21:02:13.758 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:13.758 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:13.758 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:02:13.758 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.386365+0000", 2023-09-23T21:02:13.759 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.3411e-05 2023-09-23T21:02:13.759 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:13.759 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:13.759 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:02:13.759 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.386392+0000", 2023-09-23T21:02:13.759 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.6301000000000001e-05 2023-09-23T21:02:13.759 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:02:13.759 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:02:13.759 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:02:13.759 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:13.759 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:13.759 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.34797.0:157161 6.12 6:4e589f92:::benchmark_data_smithi026_223868_object56035:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e150)", 2023-09-23T21:02:13.760 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T20:55:07.800631+0000", 2023-09-23T21:02:13.760 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 425.88643800400001, 2023-09-23T21:02:13.760 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.58585551000000002, 2023-09-23T21:02:13.760 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:02:13.760 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:02:13.760 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:02:13.760 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.34797", 2023-09-23T21:02:13.760 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T21:02:13.760 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 157161 2023-09-23T21:02:13.760 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:13.760 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:02:13.760 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:13.760 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:02:13.761 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:07.800631+0000", 2023-09-23T21:02:13.761 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:02:13.761 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:13.761 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:13.761 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:02:13.761 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:07.800631+0000", 2023-09-23T21:02:13.761 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:02:13.761 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:13.761 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:13.761 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:02:13.761 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:07.800630+0000", 2023-09-23T21:02:13.761 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T21:02:13.762 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:13.762 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:13.762 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:02:13.762 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:07.800636+0000", 2023-09-23T21:02:13.762 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.9100000000000002e-06 2023-09-23T21:02:13.762 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:13.762 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:13.762 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:02:13.762 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:07.800650+0000", 2023-09-23T21:02:13.762 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.4409999999999999e-05 2023-09-23T21:02:13.762 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:13.762 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:13.763 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:02:13.763 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:07.800657+0000", 2023-09-23T21:02:13.763 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.0829999999999998e-06 2023-09-23T21:02:13.763 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:13.763 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:13.763 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:02:13.763 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:07.800709+0000", 2023-09-23T21:02:13.763 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.1907999999999998e-05 2023-09-23T21:02:13.763 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:13.763 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:13.763 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:02:13.763 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:07.800721+0000", 2023-09-23T21:02:13.764 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.2529e-05 2023-09-23T21:02:13.764 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:13.764 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:13.764 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:02:13.764 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.376733+0000", 2023-09-23T21:02:13.764 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.57601174499999996 2023-09-23T21:02:13.764 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:13.764 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:13.764 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:02:13.764 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.376824+0000", 2023-09-23T21:02:13.764 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.1108999999999998e-05 2023-09-23T21:02:13.764 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:13.764 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:13.765 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 2,3", 2023-09-23T21:02:13.765 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.377089+0000", 2023-09-23T21:02:13.765 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000265128 2023-09-23T21:02:13.765 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:13.765 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:13.765 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:02:13.765 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.380251+0000", 2023-09-23T21:02:13.765 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0031611230000000001 2023-09-23T21:02:13.765 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:13.765 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:13.765 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:02:13.765 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.386220+0000", 2023-09-23T21:02:13.766 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0059696009999999997 2023-09-23T21:02:13.766 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:13.766 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:13.766 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:02:13.766 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.386427+0000", 2023-09-23T21:02:13.766 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000206951 2023-09-23T21:02:13.766 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:13.766 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:13.766 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:02:13.766 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.386466+0000", 2023-09-23T21:02:13.766 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.8874000000000001e-05 2023-09-23T21:02:13.766 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:13.766 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:13.767 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:02:13.767 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.386486+0000", 2023-09-23T21:02:13.767 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.0457000000000001e-05 2023-09-23T21:02:13.767 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:02:13.767 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:02:13.767 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:02:13.767 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:13.767 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:13.767 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.34797.0:157227 6.12 6:4d4ca5fd:::benchmark_data_smithi026_223868_object56101:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e150)", 2023-09-23T21:02:13.767 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T20:55:07.900387+0000", 2023-09-23T21:02:13.767 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 425.78668172900001, 2023-09-23T21:02:13.767 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.49162882600000002, 2023-09-23T21:02:13.768 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:02:13.768 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:02:13.768 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:02:13.768 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.34797", 2023-09-23T21:02:13.768 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T21:02:13.768 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 157227 2023-09-23T21:02:13.768 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:13.768 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:02:13.768 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:13.768 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:02:13.768 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:07.900387+0000", 2023-09-23T21:02:13.768 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:02:13.768 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:13.769 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:13.769 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:02:13.769 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:07.900387+0000", 2023-09-23T21:02:13.769 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:02:13.769 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:13.769 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:13.769 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:02:13.769 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:07.900386+0000", 2023-09-23T21:02:13.769 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T21:02:13.769 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:13.769 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:13.769 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:02:13.770 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:07.900392+0000", 2023-09-23T21:02:13.770 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.5899999999999996e-06 2023-09-23T21:02:13.770 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:13.770 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:13.770 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:02:13.770 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:07.900408+0000", 2023-09-23T21:02:13.770 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.5430999999999998e-05 2023-09-23T21:02:13.770 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:13.770 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:13.770 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:02:13.770 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:07.900415+0000", 2023-09-23T21:02:13.770 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.7739999999999999e-06 2023-09-23T21:02:13.771 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:13.771 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:13.771 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:02:13.771 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:07.900476+0000", 2023-09-23T21:02:13.771 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.1452999999999996e-05 2023-09-23T21:02:13.771 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:13.771 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:13.771 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:02:13.771 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:07.900485+0000", 2023-09-23T21:02:13.771 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.3260000000000002e-06 2023-09-23T21:02:13.771 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:13.771 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:13.772 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:02:13.772 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.378561+0000", 2023-09-23T21:02:13.772 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.47807563199999997 2023-09-23T21:02:13.772 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:13.772 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:13.772 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:02:13.772 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.378655+0000", 2023-09-23T21:02:13.772 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.3793000000000001e-05 2023-09-23T21:02:13.772 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:13.772 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:13.772 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 2,3", 2023-09-23T21:02:13.772 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.378841+0000", 2023-09-23T21:02:13.773 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00018573600000000001 2023-09-23T21:02:13.773 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:13.773 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:13.773 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:02:13.773 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.386326+0000", 2023-09-23T21:02:13.773 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0074849620000000004 2023-09-23T21:02:13.773 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:13.773 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:13.773 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:02:13.773 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.386517+0000", 2023-09-23T21:02:13.773 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00019191999999999999 2023-09-23T21:02:13.773 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:13.773 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:13.774 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:02:13.774 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.391931+0000", 2023-09-23T21:02:13.774 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0054132310000000001 2023-09-23T21:02:13.774 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:13.774 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:13.774 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:02:13.774 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.391976+0000", 2023-09-23T21:02:13.774 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.5697e-05 2023-09-23T21:02:13.774 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:13.774 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:13.774 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:02:13.774 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.392016+0000", 2023-09-23T21:02:13.775 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.9644000000000002e-05 2023-09-23T21:02:13.775 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:02:13.775 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:02:13.775 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:02:13.775 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:13.775 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:13.775 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.34797.0:157259 6.12 6:4e9f8804:::benchmark_data_smithi026_223868_object56133:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e150)", 2023-09-23T21:02:13.775 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T20:55:07.947142+0000", 2023-09-23T21:02:13.775 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 425.739926497, 2023-09-23T21:02:13.775 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.44774977900000001, 2023-09-23T21:02:13.775 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:02:13.775 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:02:13.776 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:02:13.776 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.34797", 2023-09-23T21:02:13.776 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T21:02:13.776 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 157259 2023-09-23T21:02:13.776 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:13.776 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:02:13.776 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:13.776 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:02:13.776 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:07.947142+0000", 2023-09-23T21:02:13.776 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:02:13.776 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:13.776 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:13.777 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:02:13.777 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:07.947142+0000", 2023-09-23T21:02:13.777 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:02:13.777 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:13.777 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:13.777 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:02:13.777 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:07.947142+0000", 2023-09-23T21:02:13.777 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999995 2023-09-23T21:02:13.777 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:13.777 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:13.777 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:02:13.777 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:07.947146+0000", 2023-09-23T21:02:13.777 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.0500000000000002e-06 2023-09-23T21:02:13.778 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:13.778 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:13.778 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:02:13.778 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:07.947160+0000", 2023-09-23T21:02:13.778 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.4662e-05 2023-09-23T21:02:13.778 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:13.778 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:13.778 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:02:13.778 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:07.947166+0000", 2023-09-23T21:02:13.778 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.784e-06 2023-09-23T21:02:13.778 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:13.778 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:13.779 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:02:13.779 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:07.947223+0000", 2023-09-23T21:02:13.779 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.6854000000000001e-05 2023-09-23T21:02:13.779 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:13.779 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:13.779 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:02:13.779 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:07.947238+0000", 2023-09-23T21:02:13.779 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.4952e-05 2023-09-23T21:02:13.779 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:13.779 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:13.779 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:02:13.779 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.380572+0000", 2023-09-23T21:02:13.780 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.43333436400000003 2023-09-23T21:02:13.780 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:13.780 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:13.780 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:02:13.780 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.380689+0000", 2023-09-23T21:02:13.780 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00011646600000000001 2023-09-23T21:02:13.780 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:13.780 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:13.780 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 2,3", 2023-09-23T21:02:13.780 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.380934+0000", 2023-09-23T21:02:13.780 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00024519199999999998 2023-09-23T21:02:13.780 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:13.780 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:13.781 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:02:13.781 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.390072+0000", 2023-09-23T21:02:13.781 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0091384410000000006 2023-09-23T21:02:13.781 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:13.781 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:13.781 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:02:13.781 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.392023+0000", 2023-09-23T21:02:13.781 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0019504170000000001 2023-09-23T21:02:13.781 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:13.781 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:13.781 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:02:13.781 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.394809+0000", 2023-09-23T21:02:13.782 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0027863760000000001 2023-09-23T21:02:13.782 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:13.782 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:13.782 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:02:13.782 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.394852+0000", 2023-09-23T21:02:13.782 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.3130999999999998e-05 2023-09-23T21:02:13.782 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:13.782 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:13.782 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:02:13.782 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.394892+0000", 2023-09-23T21:02:13.782 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.9801000000000001e-05 2023-09-23T21:02:13.782 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:02:13.783 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:02:13.783 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:02:13.783 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:13.783 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:13.783 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.34797.0:157328 6.12 6:4811798e:::benchmark_data_smithi026_223868_object56202:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e150)", 2023-09-23T21:02:13.783 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T20:55:08.060527+0000", 2023-09-23T21:02:13.783 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 425.62654177100001, 2023-09-23T21:02:13.783 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.33450326899999999, 2023-09-23T21:02:13.783 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:02:13.783 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:02:13.783 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:02:13.783 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.34797", 2023-09-23T21:02:13.783 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T21:02:13.784 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 157328 2023-09-23T21:02:13.784 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:13.784 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:02:13.784 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:13.784 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:02:13.784 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.060527+0000", 2023-09-23T21:02:13.784 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:02:13.784 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:13.784 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:13.784 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:02:13.784 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.060527+0000", 2023-09-23T21:02:13.784 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:02:13.785 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:13.785 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:13.785 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:02:13.785 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.060526+0000", 2023-09-23T21:02:13.785 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.999999 2023-09-23T21:02:13.785 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:13.785 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:13.785 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:02:13.785 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.060531+0000", 2023-09-23T21:02:13.785 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.25e-06 2023-09-23T21:02:13.785 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:13.785 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:13.785 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:02:13.786 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.060540+0000", 2023-09-23T21:02:13.786 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.0259999999999996e-06 2023-09-23T21:02:13.786 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:13.786 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:13.786 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:02:13.786 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.060544+0000", 2023-09-23T21:02:13.786 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.1370000000000004e-06 2023-09-23T21:02:13.786 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:13.786 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:13.786 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:02:13.786 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.060605+0000", 2023-09-23T21:02:13.786 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.0939e-05 2023-09-23T21:02:13.787 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:13.787 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:13.787 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:02:13.787 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.060616+0000", 2023-09-23T21:02:13.787 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.1824999999999999e-05 2023-09-23T21:02:13.787 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:13.787 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:13.787 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:02:13.787 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.382773+0000", 2023-09-23T21:02:13.787 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.32215636800000003 2023-09-23T21:02:13.787 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:13.787 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:13.788 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:02:13.788 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.382925+0000", 2023-09-23T21:02:13.788 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00015193499999999999 2023-09-23T21:02:13.788 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:13.788 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:13.788 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 2,3", 2023-09-23T21:02:13.788 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.383267+0000", 2023-09-23T21:02:13.788 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00034205500000000002 2023-09-23T21:02:13.788 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:13.788 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:13.788 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:02:13.788 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.390103+0000", 2023-09-23T21:02:13.789 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0068357510000000002 2023-09-23T21:02:13.789 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:13.789 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:13.789 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:02:13.789 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.392028+0000", 2023-09-23T21:02:13.789 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.001925775 2023-09-23T21:02:13.789 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:13.789 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:13.789 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:02:13.789 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.394957+0000", 2023-09-23T21:02:13.789 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0029286030000000001 2023-09-23T21:02:13.789 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:13.789 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:13.790 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:02:13.790 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.394996+0000", 2023-09-23T21:02:13.790 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.9410999999999999e-05 2023-09-23T21:02:13.790 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:13.790 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:13.790 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:02:13.790 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.395030+0000", 2023-09-23T21:02:13.790 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.4112000000000001e-05 2023-09-23T21:02:13.790 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:02:13.790 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:02:13.790 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:02:13.790 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:13.791 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:13.791 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.34797.0:157346 6.12 6:4c4003c4:::benchmark_data_smithi026_223868_object56220:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e150)", 2023-09-23T21:02:13.791 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T20:55:08.089574+0000", 2023-09-23T21:02:13.791 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 425.59749532699999, 2023-09-23T21:02:13.791 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.305631714, 2023-09-23T21:02:13.791 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:02:13.791 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:02:13.791 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:02:13.791 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.34797", 2023-09-23T21:02:13.791 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T21:02:13.791 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 157346 2023-09-23T21:02:13.791 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:13.791 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:02:13.792 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:13.792 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:02:13.792 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.089574+0000", 2023-09-23T21:02:13.792 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:02:13.792 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:13.792 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:13.792 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:02:13.792 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.089574+0000", 2023-09-23T21:02:13.792 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:02:13.792 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:13.792 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:13.792 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:02:13.793 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.089572+0000", 2023-09-23T21:02:13.793 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999981 2023-09-23T21:02:13.793 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:13.793 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:13.793 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:02:13.793 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.089579+0000", 2023-09-23T21:02:13.793 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.6290000000000001e-06 2023-09-23T21:02:13.793 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:13.793 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:13.793 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:02:13.793 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.089596+0000", 2023-09-23T21:02:13.793 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.6511000000000002e-05 2023-09-23T21:02:13.793 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:13.794 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:13.794 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:02:13.794 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.089605+0000", 2023-09-23T21:02:13.794 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.5390000000000007e-06 2023-09-23T21:02:13.794 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:13.794 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:13.794 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:02:13.794 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.089665+0000", 2023-09-23T21:02:13.794 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.0859000000000001e-05 2023-09-23T21:02:13.794 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:13.794 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:13.794 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:02:13.795 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.089678+0000", 2023-09-23T21:02:13.795 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.2133000000000001e-05 2023-09-23T21:02:13.795 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:13.795 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:13.795 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:02:13.795 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.384720+0000", 2023-09-23T21:02:13.795 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.29504224499999998 2023-09-23T21:02:13.795 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:13.795 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:13.795 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:02:13.795 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.384814+0000", 2023-09-23T21:02:13.795 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.4611999999999995e-05 2023-09-23T21:02:13.796 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:13.796 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:13.796 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 2,3", 2023-09-23T21:02:13.796 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.385022+0000", 2023-09-23T21:02:13.796 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00020736999999999999 2023-09-23T21:02:13.796 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:13.796 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:13.796 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:02:13.796 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.392033+0000", 2023-09-23T21:02:13.796 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0070114649999999997 2023-09-23T21:02:13.796 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:13.796 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:13.797 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:02:13.797 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.395074+0000", 2023-09-23T21:02:13.797 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.003040297 2023-09-23T21:02:13.797 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:13.797 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:13.797 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:02:13.797 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.395133+0000", 2023-09-23T21:02:13.797 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.9700000000000001e-05 2023-09-23T21:02:13.797 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:13.797 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:13.797 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:02:13.797 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.395173+0000", 2023-09-23T21:02:13.797 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.9956999999999997e-05 2023-09-23T21:02:13.798 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:13.798 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:13.798 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:02:13.798 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.395205+0000", 2023-09-23T21:02:13.798 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.2146e-05 2023-09-23T21:02:13.798 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:02:13.798 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:02:13.798 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:02:13.798 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:13.798 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:13.798 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.34797.0:157407 6.12 6:494044ed:::benchmark_data_smithi026_223868_object56281:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e150)", 2023-09-23T21:02:13.798 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T20:55:08.202324+0000", 2023-09-23T21:02:13.799 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 425.48474518699999, 2023-09-23T21:02:13.799 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.19300698499999999, 2023-09-23T21:02:13.799 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:02:13.799 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:02:13.799 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:02:13.799 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.34797", 2023-09-23T21:02:13.799 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T21:02:13.799 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 157407 2023-09-23T21:02:13.799 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:13.799 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:02:13.799 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:13.799 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:02:13.800 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.202324+0000", 2023-09-23T21:02:13.800 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:02:13.800 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:13.800 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:13.800 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:02:13.800 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.202324+0000", 2023-09-23T21:02:13.800 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:02:13.800 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:13.800 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:13.800 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:02:13.800 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.202322+0000", 2023-09-23T21:02:13.800 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T21:02:13.801 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:13.801 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:13.801 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:02:13.801 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.202330+0000", 2023-09-23T21:02:13.801 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.8730000000000007e-06 2023-09-23T21:02:13.801 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:13.801 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:13.801 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:02:13.801 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.202346+0000", 2023-09-23T21:02:13.801 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.5807e-05 2023-09-23T21:02:13.801 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:13.801 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:13.801 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:02:13.802 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.202352+0000", 2023-09-23T21:02:13.802 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.4609999999999996e-06 2023-09-23T21:02:13.802 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:13.802 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:13.802 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:02:13.802 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.202412+0000", 2023-09-23T21:02:13.802 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.9930999999999999e-05 2023-09-23T21:02:13.802 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:13.802 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:13.802 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:02:13.802 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.202425+0000", 2023-09-23T21:02:13.802 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.2272e-05 2023-09-23T21:02:13.803 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:13.803 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:13.803 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:02:13.803 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.386530+0000", 2023-09-23T21:02:13.803 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.18410531799999999 2023-09-23T21:02:13.803 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:13.803 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:13.803 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:02:13.803 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.386605+0000", 2023-09-23T21:02:13.803 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.4880000000000001e-05 2023-09-23T21:02:13.803 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:13.803 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:13.804 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 2,3", 2023-09-23T21:02:13.804 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.386802+0000", 2023-09-23T21:02:13.804 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00019716199999999999 2023-09-23T21:02:13.804 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:13.804 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:13.804 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:02:13.804 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.392038+0000", 2023-09-23T21:02:13.804 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.005235941 2023-09-23T21:02:13.804 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:13.804 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:13.804 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:02:13.804 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.395108+0000", 2023-09-23T21:02:13.805 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0030700570000000002 2023-09-23T21:02:13.805 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:13.805 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:13.805 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:02:13.805 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.395266+0000", 2023-09-23T21:02:13.805 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00015852499999999999 2023-09-23T21:02:13.805 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:13.805 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:13.805 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:02:13.805 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.395302+0000", 2023-09-23T21:02:13.805 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.5374999999999999e-05 2023-09-23T21:02:13.805 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:13.806 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:13.806 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:02:13.806 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.395331+0000", 2023-09-23T21:02:13.806 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.8892999999999999e-05 2023-09-23T21:02:13.806 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:02:13.806 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:02:13.806 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:02:13.806 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:13.806 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:13.806 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.34797.0:157436 6.12 6:4be31d9d:::benchmark_data_smithi026_223868_object56310:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e150)", 2023-09-23T21:02:13.806 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T20:55:08.273211+0000", 2023-09-23T21:02:13.806 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 425.41385779199999, 2023-09-23T21:02:13.807 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.12224776599999999, 2023-09-23T21:02:13.807 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:02:13.807 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:02:13.807 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:02:13.807 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.34797", 2023-09-23T21:02:13.807 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T21:02:13.807 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 157436 2023-09-23T21:02:13.807 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:13.807 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:02:13.807 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:13.807 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:02:13.807 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.273211+0000", 2023-09-23T21:02:13.808 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:02:13.808 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:13.808 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:13.808 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:02:13.808 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.273211+0000", 2023-09-23T21:02:13.808 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:02:13.808 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:13.808 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:13.808 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:02:13.808 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.273210+0000", 2023-09-23T21:02:13.808 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.999999 2023-09-23T21:02:13.808 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:13.809 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:13.809 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:02:13.809 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.273214+0000", 2023-09-23T21:02:13.809 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.1080000000000003e-06 2023-09-23T21:02:13.809 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:13.809 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:13.809 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:02:13.809 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.273226+0000", 2023-09-23T21:02:13.809 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.168e-05 2023-09-23T21:02:13.809 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:13.809 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:13.809 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:02:13.809 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.273231+0000", 2023-09-23T21:02:13.810 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.6670000000000004e-06 2023-09-23T21:02:13.810 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:13.810 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:13.810 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:02:13.810 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.273295+0000", 2023-09-23T21:02:13.810 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.4342000000000002e-05 2023-09-23T21:02:13.810 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:13.810 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:13.810 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:02:13.810 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.273305+0000", 2023-09-23T21:02:13.810 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.0458e-05 2023-09-23T21:02:13.810 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:13.811 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:13.811 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:02:13.811 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.388109+0000", 2023-09-23T21:02:13.811 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.114803402 2023-09-23T21:02:13.811 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:13.811 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:13.811 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:02:13.811 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.388202+0000", 2023-09-23T21:02:13.811 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.3173999999999999e-05 2023-09-23T21:02:13.811 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:13.811 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:13.811 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 2,3", 2023-09-23T21:02:13.812 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.388429+0000", 2023-09-23T21:02:13.812 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00022721899999999999 2023-09-23T21:02:13.812 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:13.812 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:13.812 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:02:13.812 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.392043+0000", 2023-09-23T21:02:13.812 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0036135849999999999 2023-09-23T21:02:13.812 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:13.812 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:13.812 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:02:13.812 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.395241+0000", 2023-09-23T21:02:13.812 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0031982820000000002 2023-09-23T21:02:13.813 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:13.813 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:13.813 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:02:13.813 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.395384+0000", 2023-09-23T21:02:13.813 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00014287100000000001 2023-09-23T21:02:13.813 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:13.813 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:13.813 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:02:13.813 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.395420+0000", 2023-09-23T21:02:13.813 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.6174000000000003e-05 2023-09-23T21:02:13.813 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:13.813 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:13.813 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:02:13.814 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:08.395459+0000", 2023-09-23T21:02:13.814 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.8887999999999998e-05 2023-09-23T21:02:13.814 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:02:13.814 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:02:13.814 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:02:13.814 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:13.814 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:13.814 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.35100.0:94301 7.1 7:8f95fbda:::benchmark_data_smithi026_260489_object94300:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e228)", 2023-09-23T21:02:13.814 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T20:57:57.540024+0000", 2023-09-23T21:02:13.814 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 256.14704450099998, 2023-09-23T21:02:13.814 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.14579631700000001, 2023-09-23T21:02:13.814 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:02:13.815 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:02:13.815 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:02:13.815 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.35100", 2023-09-23T21:02:13.815 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/4108972961", 2023-09-23T21:02:13.815 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 94301 2023-09-23T21:02:13.815 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:13.815 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:02:13.815 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:13.815 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:02:13.815 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:57:57.540024+0000", 2023-09-23T21:02:13.815 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:02:13.815 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:13.816 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:13.816 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:02:13.816 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:57:57.540024+0000", 2023-09-23T21:02:13.816 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:02:13.816 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:13.816 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:13.816 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:02:13.816 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:57:57.540024+0000", 2023-09-23T21:02:13.816 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999995 2023-09-23T21:02:13.816 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:13.816 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:13.816 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:02:13.816 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:57:57.540060+0000", 2023-09-23T21:02:13.817 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.6013000000000002e-05 2023-09-23T21:02:13.817 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:13.817 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:13.817 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:02:13.817 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:57:57.540067+0000", 2023-09-23T21:02:13.817 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.866e-06 2023-09-23T21:02:13.817 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:13.817 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:13.817 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:02:13.817 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:57:57.540075+0000", 2023-09-23T21:02:13.817 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.7239999999999992e-06 2023-09-23T21:02:13.817 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:13.817 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:13.818 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:02:13.818 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:57:57.549778+0000", 2023-09-23T21:02:13.818 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0097031600000000006 2023-09-23T21:02:13.818 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:13.818 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:13.818 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:02:13.818 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:57:57.549889+0000", 2023-09-23T21:02:13.818 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000110818 2023-09-23T21:02:13.818 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:13.818 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:13.818 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 3,6", 2023-09-23T21:02:13.818 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:57:57.550024+0000", 2023-09-23T21:02:13.819 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00013545899999999999 2023-09-23T21:02:13.819 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:13.819 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:13.819 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:02:13.819 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:57:57.551983+0000", 2023-09-23T21:02:13.819 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.001958871 2023-09-23T21:02:13.819 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:13.819 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:13.819 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:02:13.819 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:57:57.552271+0000", 2023-09-23T21:02:13.819 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00028848799999999999 2023-09-23T21:02:13.819 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:13.819 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:13.820 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:02:13.820 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:57:57.572759+0000", 2023-09-23T21:02:13.820 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.020487701000000001 2023-09-23T21:02:13.820 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:13.820 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:13.820 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:02:13.820 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:57:57.572816+0000", 2023-09-23T21:02:13.820 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.7247999999999998e-05 2023-09-23T21:02:13.820 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:13.820 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:13.820 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:02:13.820 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:57:57.685821+0000", 2023-09-23T21:02:13.820 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.1130044 2023-09-23T21:02:13.821 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:02:13.821 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:02:13.821 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:02:13.821 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:13.821 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:13.821 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.35100.0:94323 7.1 7:86b69fc0:::benchmark_data_smithi026_260489_object94322:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e228)", 2023-09-23T21:02:13.821 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T20:57:57.573982+0000", 2023-09-23T21:02:13.821 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 256.113086576, 2023-09-23T21:02:13.821 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.144034897, 2023-09-23T21:02:13.821 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:02:13.821 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:02:13.821 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:02:13.822 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.35100", 2023-09-23T21:02:13.822 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/4108972961", 2023-09-23T21:02:13.822 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 94323 2023-09-23T21:02:13.822 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:13.822 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:02:13.822 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:13.822 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:02:13.822 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:57:57.573982+0000", 2023-09-23T21:02:13.822 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:02:13.822 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:13.822 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:13.822 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:02:13.822 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:57:57.573982+0000", 2023-09-23T21:02:13.823 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:02:13.823 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:13.823 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:13.823 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:02:13.823 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:57:57.573981+0000", 2023-09-23T21:02:13.823 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T21:02:13.823 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:13.823 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:13.823 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:02:13.823 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:57:57.574033+0000", 2023-09-23T21:02:13.823 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.2389999999999998e-05 2023-09-23T21:02:13.823 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:13.824 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:13.824 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:02:13.824 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:57:57.574044+0000", 2023-09-23T21:02:13.824 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.1079e-05 2023-09-23T21:02:13.824 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:13.824 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:13.824 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:02:13.824 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:57:57.574051+0000", 2023-09-23T21:02:13.824 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.2689999999999996e-06 2023-09-23T21:02:13.824 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:13.824 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:13.824 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:02:13.824 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:57:57.685883+0000", 2023-09-23T21:02:13.825 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.11183256599999999 2023-09-23T21:02:13.825 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:13.825 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:13.825 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:02:13.825 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:57:57.686002+0000", 2023-09-23T21:02:13.825 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00011926600000000001 2023-09-23T21:02:13.825 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:13.825 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:13.825 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 3,6", 2023-09-23T21:02:13.825 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:57:57.686143+0000", 2023-09-23T21:02:13.825 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00014049200000000001 2023-09-23T21:02:13.825 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:13.826 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:13.826 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:02:13.826 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:57:57.692125+0000", 2023-09-23T21:02:13.826 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0059817849999999999 2023-09-23T21:02:13.826 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:13.826 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:13.826 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:02:13.826 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:57:57.697824+0000", 2023-09-23T21:02:13.826 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00569959 2023-09-23T21:02:13.826 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:13.826 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:13.826 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:02:13.826 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:57:57.714832+0000", 2023-09-23T21:02:13.827 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.017007967999999998 2023-09-23T21:02:13.827 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:13.827 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:13.827 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:02:13.827 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:57:57.714872+0000", 2023-09-23T21:02:13.827 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.9638000000000002e-05 2023-09-23T21:02:13.827 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:13.827 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:13.827 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:02:13.827 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:57:57.718017+0000", 2023-09-23T21:02:13.827 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0031452939999999999 2023-09-23T21:02:13.827 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:02:13.827 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:02:13.828 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:02:13.828 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:13.828 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:13.828 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.35469.0:32275 8.2 8:447e03ba:::benchmark_data_smithi026_301269_object32274:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e258)", 2023-09-23T21:02:13.828 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:01:45.594018+0000", 2023-09-23T21:02:13.828 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 28.093051435, 2023-09-23T21:02:13.828 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.12938208000000001, 2023-09-23T21:02:13.828 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:02:13.828 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:02:13.828 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:02:13.828 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.35469", 2023-09-23T21:02:13.828 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/4041424623", 2023-09-23T21:02:13.829 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 32275 2023-09-23T21:02:13.829 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:13.829 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:02:13.829 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:13.829 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:02:13.829 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:01:45.594018+0000", 2023-09-23T21:02:13.829 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:02:13.829 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:13.829 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:13.829 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:02:13.829 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:01:45.594018+0000", 2023-09-23T21:02:13.829 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:02:13.829 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:13.830 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:13.830 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:02:13.830 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:01:45.594012+0000", 2023-09-23T21:02:13.830 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999943 2023-09-23T21:02:13.830 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:13.830 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:13.830 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:02:13.830 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:01:45.594146+0000", 2023-09-23T21:02:13.830 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00013374799999999999 2023-09-23T21:02:13.830 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:13.830 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:13.830 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:02:13.830 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:01:45.594161+0000", 2023-09-23T21:02:13.831 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.5688999999999998e-05 2023-09-23T21:02:13.831 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:13.831 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:13.831 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:02:13.831 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:01:45.594169+0000", 2023-09-23T21:02:13.831 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.418e-06 2023-09-23T21:02:13.831 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:13.831 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:13.831 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:02:13.831 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:01:45.594220+0000", 2023-09-23T21:02:13.831 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.1205000000000002e-05 2023-09-23T21:02:13.831 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:13.832 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:13.832 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:02:13.832 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:01:45.594366+0000", 2023-09-23T21:02:13.832 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00014647799999999999 2023-09-23T21:02:13.832 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:13.832 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:13.832 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 0,5", 2023-09-23T21:02:13.832 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:01:45.594592+0000", 2023-09-23T21:02:13.832 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00022560200000000001 2023-09-23T21:02:13.832 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:13.832 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:13.832 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:02:13.832 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:01:45.597415+0000", 2023-09-23T21:02:13.833 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.002822728 2023-09-23T21:02:13.833 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:13.833 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:13.833 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:02:13.833 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:01:45.599382+0000", 2023-09-23T21:02:13.833 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0019670120000000002 2023-09-23T21:02:13.833 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:13.833 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:13.833 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:02:13.833 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:01:45.723302+0000", 2023-09-23T21:02:13.833 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.1239204 2023-09-23T21:02:13.833 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:13.834 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:13.834 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:02:13.834 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:01:45.723357+0000", 2023-09-23T21:02:13.834 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.5377999999999997e-05 2023-09-23T21:02:13.834 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:13.834 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:13.834 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:02:13.834 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:01:45.723400+0000", 2023-09-23T21:02:13.834 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.2103e-05 2023-09-23T21:02:13.834 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:02:13.834 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:02:13.834 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:02:13.835 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:13.835 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:13.835 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.35469.0:32285 8.2 8:4b8471a3:::benchmark_data_smithi026_301269_object32284:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e258)", 2023-09-23T21:02:13.835 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:01:45.597927+0000", 2023-09-23T21:02:13.835 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 28.089142381999999, 2023-09-23T21:02:13.835 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.12566041999999999, 2023-09-23T21:02:13.835 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:02:13.835 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:02:13.835 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:02:13.835 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.35469", 2023-09-23T21:02:13.835 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/4041424623", 2023-09-23T21:02:13.835 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 32285 2023-09-23T21:02:13.836 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:13.836 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:02:13.836 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:13.836 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:02:13.836 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:01:45.597927+0000", 2023-09-23T21:02:13.836 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:02:13.836 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:13.836 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:13.836 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:02:13.836 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:01:45.597927+0000", 2023-09-23T21:02:13.836 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:02:13.836 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:13.837 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:13.837 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:02:13.837 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:01:45.597925+0000", 2023-09-23T21:02:13.837 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.999999 2023-09-23T21:02:13.837 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:13.837 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:13.837 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:02:13.837 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:01:45.597980+0000", 2023-09-23T21:02:13.837 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.4230000000000001e-05 2023-09-23T21:02:13.837 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:13.837 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:13.837 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:02:13.837 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:01:45.597990+0000", 2023-09-23T21:02:13.838 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.0125999999999999e-05 2023-09-23T21:02:13.838 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:13.838 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:13.838 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:02:13.838 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:01:45.597995+0000", 2023-09-23T21:02:13.838 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.7620000000000003e-06 2023-09-23T21:02:13.838 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:13.838 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:13.838 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:02:13.838 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:01:45.598027+0000", 2023-09-23T21:02:13.838 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.222e-05 2023-09-23T21:02:13.838 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:13.838 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:13.839 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:02:13.839 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:01:45.598108+0000", 2023-09-23T21:02:13.839 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.1519000000000003e-05 2023-09-23T21:02:13.839 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:13.839 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:13.839 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 0,5", 2023-09-23T21:02:13.839 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:01:45.598193+0000", 2023-09-23T21:02:13.839 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.4546e-05 2023-09-23T21:02:13.841 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:13.841 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:13.841 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:02:13.841 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:01:45.599711+0000", 2023-09-23T21:02:13.841 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.001517722 2023-09-23T21:02:13.841 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:13.841 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:13.841 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:02:13.841 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:01:45.600134+0000", 2023-09-23T21:02:13.842 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00042341500000000001 2023-09-23T21:02:13.842 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:13.842 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:13.842 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:02:13.842 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:01:45.723478+0000", 2023-09-23T21:02:13.842 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.123343779 2023-09-23T21:02:13.842 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:13.842 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:13.842 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:02:13.842 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:01:45.723548+0000", 2023-09-23T21:02:13.842 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.0638999999999999e-05 2023-09-23T21:02:13.842 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:13.843 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:13.843 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:02:13.843 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:01:45.723587+0000", 2023-09-23T21:02:13.843 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.8634000000000003e-05 2023-09-23T21:02:13.843 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:02:13.843 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:02:13.843 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:02:13.843 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:02:13.843 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:02:13.843 INFO:teuthology.orchestra.run.smithi070.stdout:} 2023-09-23T21:02:13.844 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:02:13.845 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 42 16 68630 68614 102.06 75.625 0.00753776 0.00979302 2023-09-23T21:02:13.919 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T21:02:13.929 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:02:14.007 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:02:14.023 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:02:14.023 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:02:14.122 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_ops_in_flight 2023-09-23T21:02:14.124 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:02:14.212 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:02:14.266 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:02:14.282 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:02:14.283 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:02:14.384 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:02:14.439 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:02:14 smithi026 ceph-mon[175151]: pgmap v697: 109 pgs: 109 active+clean; 4.0 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 75 MiB/s wr, 1.20k op/s 2023-09-23T21:02:14.439 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:02:14 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-09-23T21:02:14.439 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:02:14 smithi026 ceph-mon[175151]: osdmap e266: 8 total, 8 up, 8 in 2023-09-23T21:02:14.439 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:02:14 smithi026 ceph-mon[175151]: failed to encode map e266 with expected crc 2023-09-23T21:02:14.440 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:02:14 smithi026 ceph-mon[175151]: failed to encode map e266 with expected crc 2023-09-23T21:02:14.440 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:02:14 smithi026 ceph-mon[175151]: failed to encode map e266 with expected crc 2023-09-23T21:02:14.440 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:02:14 smithi026 ceph-mon[175151]: failed to encode map e266 with expected crc 2023-09-23T21:02:14.440 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:02:14 smithi026 ceph-mon[175151]: failed to encode map e266 with expected crc 2023-09-23T21:02:14.440 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:02:14 smithi026 ceph-mon[179688]: pgmap v697: 109 pgs: 109 active+clean; 4.0 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 75 MiB/s wr, 1.20k op/s 2023-09-23T21:02:14.440 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:02:14 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-09-23T21:02:14.440 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:02:14 smithi026 ceph-mon[179688]: osdmap e266: 8 total, 8 up, 8 in 2023-09-23T21:02:14.441 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:02:14 smithi026 ceph-mon[179688]: failed to encode map e266 with expected crc 2023-09-23T21:02:14.441 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:02:14 smithi026 ceph-mon[179688]: failed to encode map e266 with expected crc 2023-09-23T21:02:14.441 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:02:14 smithi026 ceph-mon[179688]: failed to encode map e266 with expected crc 2023-09-23T21:02:14.441 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:02:14 smithi026 ceph-mon[179688]: failed to encode map e266 with expected crc 2023-09-23T21:02:14.441 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:02:14 smithi026 ceph-mon[179688]: failed to encode map e266 with expected crc 2023-09-23T21:02:14.441 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:02:14.454 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T21:02:14.496 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:02:14.501 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:02:14.502 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:02:14.602 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:02:14.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:02:14 smithi070 ceph-mon[168282]: pgmap v697: 109 pgs: 109 active+clean; 4.0 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 75 MiB/s wr, 1.20k op/s 2023-09-23T21:02:14.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:02:14 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-09-23T21:02:14.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:02:14 smithi070 ceph-mon[168282]: osdmap e266: 8 total, 8 up, 8 in 2023-09-23T21:02:14.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:02:14 smithi070 ceph-mon[168282]: failed to encode map e266 with expected crc 2023-09-23T21:02:14.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:02:14 smithi070 ceph-mon[168282]: failed to encode map e266 with expected crc 2023-09-23T21:02:14.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:02:14 smithi070 ceph-mon[168282]: failed to encode map e266 with expected crc 2023-09-23T21:02:14.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:02:14 smithi070 ceph-mon[168282]: failed to encode map e266 with expected crc 2023-09-23T21:02:14.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:02:14 smithi070 ceph-mon[168282]: failed to encode map e266 with expected crc 2023-09-23T21:02:14.646 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:02:14.698 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:02:14.746 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 43 16 69598 69582 101.094 60.5 0.0135479 0.00988559 2023-09-23T21:02:14.841 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T21:02:14.938 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:02:14.986 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:02:14.991 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:02:14.991 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:02:15.093 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:02:15.144 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:02:15.213 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:02:15.218 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:02:15.219 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:02:15.320 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:02:15.396 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:02:15.746 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 44 16 70692 70676 100.35 68.375 0.0121493 0.0099605 2023-09-23T21:02:15.958 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T21:02:15.963 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:02:15.963 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:02:16.064 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:02:16.146 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:02:16.261 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:02:16.299 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:02:16.308 INFO:teuthology.orchestra.run.smithi070.stdout:{ 2023-09-23T21:02:16.308 INFO:teuthology.orchestra.run.smithi070.stdout: "ops": [ 2023-09-23T21:02:16.308 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:16.308 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.35469.0:71287 8.8 8:16686a73:::benchmark_data_smithi026_301269_object71286:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e266)", 2023-09-23T21:02:16.308 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:02:16.291070+0000", 2023-09-23T21:02:16.308 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 0.001876859, 2023-09-23T21:02:16.309 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.001904144, 2023-09-23T21:02:16.309 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:02:16.309 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "waiting for sub ops", 2023-09-23T21:02:16.309 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:02:16.309 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.35469", 2023-09-23T21:02:16.309 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/4041424623", 2023-09-23T21:02:16.309 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 71287 2023-09-23T21:02:16.309 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:16.309 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:02:16.309 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:16.309 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:02:16.310 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:02:16.291070+0000", 2023-09-23T21:02:16.310 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:02:16.310 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:16.310 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:16.310 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:02:16.310 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:02:16.291070+0000", 2023-09-23T21:02:16.310 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:02:16.310 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:16.310 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:16.310 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:02:16.310 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:02:16.291069+0000", 2023-09-23T21:02:16.310 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999981 2023-09-23T21:02:16.311 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:16.311 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:16.311 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:02:16.311 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:02:16.291114+0000", 2023-09-23T21:02:16.311 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.4926000000000003e-05 2023-09-23T21:02:16.311 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:16.311 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:16.311 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:02:16.311 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:02:16.291130+0000", 2023-09-23T21:02:16.311 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.6807999999999999e-05 2023-09-23T21:02:16.311 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:16.311 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:16.312 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:02:16.312 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:02:16.291137+0000", 2023-09-23T21:02:16.312 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.5989999999999998e-06 2023-09-23T21:02:16.312 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:16.312 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:16.312 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:02:16.312 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:02:16.291186+0000", 2023-09-23T21:02:16.312 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.8893000000000001e-05 2023-09-23T21:02:16.312 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:16.312 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:16.312 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:02:16.312 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:02:16.291327+0000", 2023-09-23T21:02:16.313 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000141211 2023-09-23T21:02:16.313 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:16.313 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:16.313 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 3,7", 2023-09-23T21:02:16.313 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:02:16.291477+0000", 2023-09-23T21:02:16.313 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000149727 2023-09-23T21:02:16.313 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:02:16.313 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:02:16.313 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:02:16.313 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:16.313 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:16.313 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_repop(client.35469.0:71289 8.9 e266/254 8:9a0a66de:::benchmark_data_smithi026_301269_object71288:head v 266'4412, mlcod=266'4408)", 2023-09-23T21:02:16.314 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:02:16.291614+0000", 2023-09-23T21:02:16.314 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 0.0013329080000000001, 2023-09-23T21:02:16.314 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.001409979, 2023-09-23T21:02:16.314 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:02:16.314 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "started", 2023-09-23T21:02:16.314 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:02:16.314 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:16.314 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:02:16.314 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:02:16.291614+0000", 2023-09-23T21:02:16.314 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:02:16.314 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:16.314 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:16.315 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:02:16.315 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:02:16.291614+0000", 2023-09-23T21:02:16.315 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:02:16.315 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:16.315 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:16.315 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:02:16.315 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:02:16.291614+0000", 2023-09-23T21:02:16.315 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.999999 2023-09-23T21:02:16.315 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:16.315 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:16.315 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:02:16.315 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:02:16.291636+0000", 2023-09-23T21:02:16.315 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.2212999999999999e-05 2023-09-23T21:02:16.316 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:16.316 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:16.316 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:02:16.316 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:02:16.291646+0000", 2023-09-23T21:02:16.316 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.9739999999999997e-06 2023-09-23T21:02:16.316 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:16.316 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:16.316 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:02:16.316 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:02:16.291651+0000", 2023-09-23T21:02:16.316 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.4790000000000004e-06 2023-09-23T21:02:16.316 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:16.316 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:16.317 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:02:16.317 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:02:16.291705+0000", 2023-09-23T21:02:16.317 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.3217000000000002e-05 2023-09-23T21:02:16.317 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:16.317 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:16.317 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:02:16.317 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:02:16.291722+0000", 2023-09-23T21:02:16.317 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.7402e-05 2023-09-23T21:02:16.317 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:02:16.317 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:02:16.317 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:02:16.317 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:02:16.318 INFO:teuthology.orchestra.run.smithi070.stdout: ], 2023-09-23T21:02:16.318 INFO:teuthology.orchestra.run.smithi070.stdout: "num_ops": 2 2023-09-23T21:02:16.318 INFO:teuthology.orchestra.run.smithi070.stdout:} 2023-09-23T21:02:16.318 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:02:16.319 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:02:16.321 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:02:16 smithi070 ceph-mon[168282]: pgmap v699: 109 pgs: 109 active+clean; 4.1 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 71 MiB/s wr, 1.13k op/s 2023-09-23T21:02:16.395 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:02:16 smithi026 ceph-mon[175151]: pgmap v699: 109 pgs: 109 active+clean; 4.1 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 71 MiB/s wr, 1.13k op/s 2023-09-23T21:02:16.395 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:02:16 smithi026 ceph-mon[179688]: pgmap v699: 109 pgs: 109 active+clean; 4.1 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 71 MiB/s wr, 1.13k op/s 2023-09-23T21:02:16.420 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:02:16.493 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:02:16.706 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_blocked_ops 2023-09-23T21:02:16.754 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 45 16 71794 71778 99.6435 68.875 0.0154689 0.010031 2023-09-23T21:02:16.834 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T21:02:16.848 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:02:16.848 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:02:16.949 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:02:17.040 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:02:17.335 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:02:17 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:02:17.335 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:02:17 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:02:17.342 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:02:17.348 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:02:17.348 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:02:17.419 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T21:02:17.449 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:02:17.518 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:02:17 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:02:17.522 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:02:17.594 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:02:17.599 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:02:17.600 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:02:17.701 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:02:17.750 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 46 16 72813 72797 98.8615 63.6875 0.0147294 0.0101096 2023-09-23T21:02:17.788 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:02:17.852 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:02:17.872 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:02:17.872 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:02:17.973 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:02:18.034 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:02:18.148 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:02:18.157 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:02:18.158 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:02:18.217 INFO:teuthology.orchestra.run.smithi026.stderr:reweighted osd.3 to 1 (10000) 2023-09-23T21:02:18.258 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:02:18.334 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:02:18 smithi026 ceph-mon[179688]: pgmap v700: 109 pgs: 109 active+clean; 4.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 69 MiB/s wr, 1.11k op/s 2023-09-23T21:02:18.334 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:02:18 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/1724457367' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-09-23T21:02:18.335 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:02:18 smithi026 ceph-mon[175151]: pgmap v700: 109 pgs: 109 active+clean; 4.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 69 MiB/s wr, 1.11k op/s 2023-09-23T21:02:18.336 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:02:18 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/1724457367' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-09-23T21:02:18.339 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:02:18.468 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T21:02:18.475 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:02:18.475 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:02:18.481 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:02:18 smithi070 ceph-mon[168282]: pgmap v700: 109 pgs: 109 active+clean; 4.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 69 MiB/s wr, 1.11k op/s 2023-09-23T21:02:18.481 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:02:18 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/1724457367' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-09-23T21:02:18.544 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0: {} 2023-09-23T21:02:18.576 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1: {} 2023-09-23T21:02:18.576 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:02:18.609 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2: {} 2023-09-23T21:02:18.647 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3: {} 2023-09-23T21:02:18.649 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:02:18.701 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4: {} 2023-09-23T21:02:18.727 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph osd reweight 6 1 2023-09-23T21:02:18.735 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T21:02:18.743 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:02:18.750 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 47 16 73739 73723 97.9896 57.875 0.00906626 0.0102001 2023-09-23T21:02:18.752 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:02:18.752 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:02:18.775 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6: {} 2023-09-23T21:02:18.789 INFO:teuthology.orchestra.run.smithi070.stdout:ERROR: (22) Invalid argument 2023-09-23T21:02:18.789 INFO:teuthology.orchestra.run.smithi070.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T21:02:18.820 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T21:02:18.827 INFO:teuthology.orchestra.run.smithi026.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T21:02:18.827 INFO:teuthology.orchestra.run.smithi026.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T21:02:18.827 INFO:teuthology.orchestra.run.smithi026.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T21:02:18.827 INFO:teuthology.orchestra.run.smithi026.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T21:02:18.827 INFO:teuthology.orchestra.run.smithi026.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T21:02:18.827 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T21:02:18.827 INFO:teuthology.orchestra.run.smithi026.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T21:02:18.827 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T21:02:18.853 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:02:18.918 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:02:19.136 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:02:19.140 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:02:19.140 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:02:19.232 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_historic_ops 2023-09-23T21:02:19.235 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:02:19 smithi026 ceph-mon[175151]: failed to encode map e267 with expected crc 2023-09-23T21:02:19.236 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:02:19 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/1724457367' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-09-23T21:02:19.236 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:02:19 smithi026 ceph-mon[175151]: osdmap e267: 8 total, 8 up, 8 in 2023-09-23T21:02:19.236 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:02:19 smithi026 ceph-mon[175151]: failed to encode map e267 with expected crc 2023-09-23T21:02:19.236 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:02:19 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/2450453525' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:02:19.236 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:02:19 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:02:19.236 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:02:19 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/2718878546' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:02:19.236 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:02:19 smithi026 ceph-mon[179688]: failed to encode map e267 with expected crc 2023-09-23T21:02:19.236 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:02:19 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/1724457367' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-09-23T21:02:19.236 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:02:19 smithi026 ceph-mon[179688]: osdmap e267: 8 total, 8 up, 8 in 2023-09-23T21:02:19.237 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:02:19 smithi026 ceph-mon[179688]: failed to encode map e267 with expected crc 2023-09-23T21:02:19.237 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:02:19 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/2450453525' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:02:19.237 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:02:19 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:02:19.237 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:02:19 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/2718878546' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:02:19.241 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:02:19.269 INFO:teuthology.orchestra.run.smithi026.stderr:nodeep-scrub is set 2023-09-23T21:02:19.340 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:02:19.401 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:02:19.404 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:02:19 smithi070 ceph-mon[168282]: failed to encode map e267 with expected crc 2023-09-23T21:02:19.404 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:02:19 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/1724457367' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-09-23T21:02:19.404 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:02:19 smithi070 ceph-mon[168282]: osdmap e267: 8 total, 8 up, 8 in 2023-09-23T21:02:19.404 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:02:19 smithi070 ceph-mon[168282]: failed to encode map e267 with expected crc 2023-09-23T21:02:19.405 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:02:19 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/2450453525' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:02:19.405 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:02:19 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:02:19.405 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:02:19 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/2718878546' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:02:19.406 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:02:19.406 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:02:19.507 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:02:19.588 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:02:19.654 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:02:19.676 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:02:19.676 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:02:19.679 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:02:19.750 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 48 16 74759 74743 97.2765 63.75 0.00437949 0.0102747 2023-09-23T21:02:19.778 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:02:19.832 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 21:02:19 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[164029]: ::ffff:172.21.15.70 - - [23/Sep/2023:21:02:19] "GET /metrics HTTP/1.1" 200 36293 "" "Prometheus/2.43.0" 2023-09-23T21:02:19.834 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:02:19.883 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T21:02:19.898 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:02:19.903 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:02:19.903 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:02:20.004 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:02:20.057 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:02:20.164 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:02:20.168 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:02:20.169 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:02:20.269 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:02:20.356 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:02:20 smithi070 ceph-mon[168282]: pgmap v702: 109 pgs: 109 active+clean; 4.3 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 67 MiB/s wr, 1.07k op/s 2023-09-23T21:02:20.356 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:02:20 smithi070 ceph-mon[168282]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:02:20.356 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:02:20 smithi070 ceph-mon[168282]: failed to encode map e268 with expected crc 2023-09-23T21:02:20.356 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:02:20 smithi070 ceph-mon[168282]: failed to encode map e268 with expected crc 2023-09-23T21:02:20.356 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:02:20 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T21:02:20.356 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:02:20 smithi070 ceph-mon[168282]: osdmap e268: 8 total, 8 up, 8 in 2023-09-23T21:02:20.356 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:02:20 smithi070 ceph-mon[168282]: failed to encode map e268 with expected crc 2023-09-23T21:02:20.361 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:02:20.530 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:02:20.539 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:02:20.540 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:02:20 smithi026 ceph-mon[175151]: pgmap v702: 109 pgs: 109 active+clean; 4.3 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 67 MiB/s wr, 1.07k op/s 2023-09-23T21:02:20.540 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:02:20 smithi026 ceph-mon[175151]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:02:20.540 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:02:20 smithi026 ceph-mon[175151]: failed to encode map e268 with expected crc 2023-09-23T21:02:20.540 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:02:20 smithi026 ceph-mon[175151]: failed to encode map e268 with expected crc 2023-09-23T21:02:20.541 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:02:20 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T21:02:20.541 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:02:20 smithi026 ceph-mon[175151]: osdmap e268: 8 total, 8 up, 8 in 2023-09-23T21:02:20.541 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:02:20 smithi026 ceph-mon[175151]: failed to encode map e268 with expected crc 2023-09-23T21:02:20.541 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:02:20 smithi026 ceph-mon[179688]: pgmap v702: 109 pgs: 109 active+clean; 4.3 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 67 MiB/s wr, 1.07k op/s 2023-09-23T21:02:20.541 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:02:20 smithi026 ceph-mon[179688]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:02:20.542 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:02:20 smithi026 ceph-mon[179688]: failed to encode map e268 with expected crc 2023-09-23T21:02:20.542 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:02:20 smithi026 ceph-mon[179688]: failed to encode map e268 with expected crc 2023-09-23T21:02:20.542 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:02:20 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T21:02:20.542 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:02:20 smithi026 ceph-mon[179688]: osdmap e268: 8 total, 8 up, 8 in 2023-09-23T21:02:20.542 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:02:20 smithi026 ceph-mon[179688]: failed to encode map e268 with expected crc 2023-09-23T21:02:20.543 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:02:20.644 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:02:20.731 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:02:20.751 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 49 16 75840 75824 96.6691 67.5625 0.0263051 0.010338 2023-09-23T21:02:20.823 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T21:02:20.828 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:02:20.828 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:02:20.929 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:02:21.029 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:02:21.199 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:02:21.205 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:02:21.206 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:02:21.297 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T21:02:21.306 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:02:21.309 INFO:teuthology.orchestra.run.smithi070.stdout:ERROR: (22) Invalid argument 2023-09-23T21:02:21.310 INFO:teuthology.orchestra.run.smithi070.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T21:02:21.417 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:02:21.519 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:02:21.524 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:02:21.525 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:02:21.626 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:02:21.658 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:02:21.689 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T21:02:21.705 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_ops_in_flight 2023-09-23T21:02:21.714 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:02:21.716 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:02:21.717 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:02:21.751 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 50 16 76817 76801 95.9571 61.0625 0.0180969 0.010417 2023-09-23T21:02:21.817 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:02:21.949 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:02:22.031 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:02:22.042 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:02:22.042 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:02:22.115 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:02:22.144 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:02:22.238 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:02:22.284 INFO:teuthology.orchestra.run.smithi026.stderr:reweighted osd.6 to 1 (10000) 2023-09-23T21:02:22.418 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T21:02:22.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:02:22 smithi070 ceph-mon[168282]: pgmap v704: 109 pgs: 109 active+clean; 4.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 71 MiB/s wr, 1.14k op/s 2023-09-23T21:02:22.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:02:22 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/3465274317' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-09-23T21:02:22.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:02:22 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-09-23T21:02:22.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:02:22 smithi026 ceph-mon[179688]: pgmap v704: 109 pgs: 109 active+clean; 4.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 71 MiB/s wr, 1.14k op/s 2023-09-23T21:02:22.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:02:22 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/3465274317' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-09-23T21:02:22.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:02:22 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-09-23T21:02:22.634 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:02:22 smithi026 ceph-mon[175151]: pgmap v704: 109 pgs: 109 active+clean; 4.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 71 MiB/s wr, 1.14k op/s 2023-09-23T21:02:22.635 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:02:22 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/3465274317' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-09-23T21:02:22.635 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:02:22 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-09-23T21:02:22.751 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 51 16 77830 77814 95.3171 63.3125 0.0179514 0.0104865 2023-09-23T21:02:23.364 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:02:23 smithi070 ceph-mon[168282]: failed to encode map e269 with expected crc 2023-09-23T21:02:23.364 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:02:23 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-09-23T21:02:23.364 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:02:23 smithi070 ceph-mon[168282]: osdmap e269: 8 total, 8 up, 8 in 2023-09-23T21:02:23.364 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:02:23 smithi070 ceph-mon[168282]: failed to encode map e269 with expected crc 2023-09-23T21:02:23.631 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:02:23 smithi026 ceph-mon[179688]: failed to encode map e269 with expected crc 2023-09-23T21:02:23.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:02:23 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-09-23T21:02:23.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:02:23 smithi026 ceph-mon[179688]: osdmap e269: 8 total, 8 up, 8 in 2023-09-23T21:02:23.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:02:23 smithi026 ceph-mon[179688]: failed to encode map e269 with expected crc 2023-09-23T21:02:23.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:02:23 smithi026 ceph-mon[175151]: failed to encode map e269 with expected crc 2023-09-23T21:02:23.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:02:23 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-09-23T21:02:23.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:02:23 smithi026 ceph-mon[175151]: osdmap e269: 8 total, 8 up, 8 in 2023-09-23T21:02:23.633 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:02:23 smithi026 ceph-mon[175151]: failed to encode map e269 with expected crc 2023-09-23T21:02:23.751 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 52 16 78828 78812 94.6838 62.375 0.0163958 0.0105566 2023-09-23T21:02:24.012 INFO:teuthology.orchestra.run.smithi070.stdout:ERROR: (22) Invalid argument 2023-09-23T21:02:24.012 INFO:teuthology.orchestra.run.smithi070.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T21:02:24.346 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:02:24.353 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:02:24.353 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:02:24.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:02:24 smithi070 ceph-mon[168282]: pgmap v706: 109 pgs: 109 active+clean; 4.6 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 78 MiB/s wr, 1.24k op/s 2023-09-23T21:02:24.454 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:02:24.509 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:02:24.550 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:02:24 smithi026 ceph-mon[175151]: pgmap v706: 109 pgs: 109 active+clean; 4.6 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 78 MiB/s wr, 1.24k op/s 2023-09-23T21:02:24.550 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:02:24 smithi026 ceph-mon[179688]: pgmap v706: 109 pgs: 109 active+clean; 4.6 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 78 MiB/s wr, 1.24k op/s 2023-09-23T21:02:24.751 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 53 16 79932 79916 94.1992 69 0.0132595 0.0106119 2023-09-23T21:02:24.886 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:02:24.891 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:02:24.892 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:02:24.983 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_blocked_ops 2023-09-23T21:02:24.993 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:02:25.108 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:02:25.159 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:02:25.173 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:02:25.174 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:02:25.275 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:02:25.316 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph osd reweight 7 1 2023-09-23T21:02:25.321 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:02:25.380 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:02:25.383 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:02:25.384 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:02:25.484 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:02:25.500 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:02:25.573 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:02:25.663 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T21:02:25.695 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:02:25.700 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:02:25.701 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:02:25.753 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 54 16 80891 80875 93.5624 59.9375 0.0209285 0.0106821 2023-09-23T21:02:25.801 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:02:25.890 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:02:26.014 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:02:26.021 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:02:26.021 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:02:26.122 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:02:26.169 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:02:26.244 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0: {} 2023-09-23T21:02:26.288 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1: {} 2023-09-23T21:02:26.326 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2: {} 2023-09-23T21:02:26.366 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3: {} 2023-09-23T21:02:26.381 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:02:26.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:02:26 smithi026 ceph-mon[179688]: pgmap v707: 109 pgs: 109 active+clean; 4.7 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 69 MiB/s wr, 1.10k op/s 2023-09-23T21:02:26.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:02:26 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/3509105852' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:02:26.384 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:02:26.384 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:02:26.386 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:02:26 smithi026 ceph-mon[175151]: pgmap v707: 109 pgs: 109 active+clean; 4.7 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 69 MiB/s wr, 1.10k op/s 2023-09-23T21:02:26.387 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:02:26 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/3509105852' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:02:26.415 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4: {} 2023-09-23T21:02:26.453 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T21:02:26.484 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:02:26.492 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6: {} 2023-09-23T21:02:26.531 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T21:02:26.533 INFO:teuthology.orchestra.run.smithi026.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T21:02:26.533 INFO:teuthology.orchestra.run.smithi026.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T21:02:26.533 INFO:teuthology.orchestra.run.smithi026.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T21:02:26.534 INFO:teuthology.orchestra.run.smithi026.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T21:02:26.534 INFO:teuthology.orchestra.run.smithi026.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T21:02:26.534 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T21:02:26.534 INFO:teuthology.orchestra.run.smithi026.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T21:02:26.534 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T21:02:26.567 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:02:26.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:02:26 smithi070 ceph-mon[168282]: pgmap v707: 109 pgs: 109 active+clean; 4.7 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 69 MiB/s wr, 1.10k op/s 2023-09-23T21:02:26.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:02:26 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/3509105852' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:02:26.732 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:02:26.745 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:02:26.753 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 55 16 81838 81822 92.9375 59.1875 0.0142479 0.0107549 2023-09-23T21:02:26.754 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:02:26.755 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:02:26.856 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:02:26.943 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:02:27.015 INFO:teuthology.orchestra.run.smithi070.stdout:{ 2023-09-23T21:02:27.015 INFO:teuthology.orchestra.run.smithi070.stdout: "ops": [], 2023-09-23T21:02:27.015 INFO:teuthology.orchestra.run.smithi070.stdout: "complaint_time": 30, 2023-09-23T21:02:27.016 INFO:teuthology.orchestra.run.smithi070.stdout: "num_blocked_ops": 0 2023-09-23T21:02:27.016 INFO:teuthology.orchestra.run.smithi070.stdout:} 2023-09-23T21:02:27.019 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:02:27.032 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:02:27.032 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:02:27.133 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:02:27.179 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:02:27.379 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_historic_ops 2023-09-23T21:02:27.386 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:02:27.389 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:02:27.389 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:02:27.490 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:02:27.587 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:02:27.637 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:02:27.645 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:02:27.645 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:02:27.746 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:02:27.753 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 56 16 82823 82807 92.3773 61.5625 0.0151292 0.0108199 2023-09-23T21:02:27.814 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:02:27.855 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:02:27.863 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:02:27.863 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:02:27.964 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:02:27.998 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:02:28.047 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T21:02:28.060 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:02:28.064 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:02:28.065 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:02:28.165 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:02:28.233 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:02:28.324 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:02:28.333 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:02:28.334 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:02:28.345 INFO:teuthology.orchestra.run.smithi026.stderr:noscrub is unset 2023-09-23T21:02:28.435 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:02:28.507 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:02:28 smithi026 ceph-mon[179688]: pgmap v708: 109 pgs: 109 active+clean; 4.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 62 MiB/s wr, 988 op/s 2023-09-23T21:02:28.507 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:02:28 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/3471393929' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T21:02:28.507 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:02:28 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T21:02:28.509 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:02:28 smithi026 ceph-mon[175151]: pgmap v708: 109 pgs: 109 active+clean; 4.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 62 MiB/s wr, 988 op/s 2023-09-23T21:02:28.509 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:02:28 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/3471393929' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T21:02:28.509 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:02:28 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T21:02:28.517 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:02:28.594 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:02:28 smithi070 ceph-mon[168282]: pgmap v708: 109 pgs: 109 active+clean; 4.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 62 MiB/s wr, 988 op/s 2023-09-23T21:02:28.595 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:02:28 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/3471393929' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T21:02:28.595 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:02:28 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T21:02:28.678 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:02:28.685 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:02:28.686 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:02:28.759 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 57 16 83776 83760 91.7923 59.5625 0.0129524 0.0108889 2023-09-23T21:02:28.787 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:02:28.870 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:02:28.991 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:02:28.995 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:02:28.996 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:02:29.097 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:02:29.160 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T21:02:29.179 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:02:29.231 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:02:29.237 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:02:29.237 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:02:29.338 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:02:29.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:02:29 smithi070 ceph-mon[168282]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:02:29.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:02:29 smithi070 ceph-mon[168282]: failed to encode map e270 with expected crc 2023-09-23T21:02:29.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:02:29 smithi070 ceph-mon[168282]: failed to encode map e270 with expected crc 2023-09-23T21:02:29.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:02:29 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T21:02:29.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:02:29 smithi070 ceph-mon[168282]: osdmap e270: 8 total, 8 up, 8 in 2023-09-23T21:02:29.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:02:29 smithi070 ceph-mon[168282]: failed to encode map e270 with expected crc 2023-09-23T21:02:29.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:02:29 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T21:02:29.374 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:02:29.397 INFO:teuthology.orchestra.run.smithi070.stdout:{ 2023-09-23T21:02:29.398 INFO:teuthology.orchestra.run.smithi070.stdout: "size": 20, 2023-09-23T21:02:29.398 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 600, 2023-09-23T21:02:29.398 INFO:teuthology.orchestra.run.smithi070.stdout: "ops": [ 2023-09-23T21:02:29.398 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:29.398 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.34797.0:165112 6.15 6:ac7f8ed1:::benchmark_data_smithi026_223868_object63986:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e154)", 2023-09-23T21:02:29.398 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T20:55:17.460747+0000", 2023-09-23T21:02:29.398 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 431.92690923700002, 2023-09-23T21:02:29.398 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.034762846, 2023-09-23T21:02:29.398 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:02:29.398 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "reached pg", 2023-09-23T21:02:29.398 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:02:29.399 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.34797", 2023-09-23T21:02:29.399 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T21:02:29.399 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 165112 2023-09-23T21:02:29.399 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:29.399 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:02:29.399 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:29.399 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:02:29.399 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:17.460747+0000", 2023-09-23T21:02:29.399 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:02:29.399 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:29.399 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:29.399 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:02:29.400 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:17.460747+0000", 2023-09-23T21:02:29.400 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:02:29.400 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:29.400 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:29.400 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:02:29.400 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:17.460745+0000", 2023-09-23T21:02:29.400 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999981 2023-09-23T21:02:29.400 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:29.400 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:29.400 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:02:29.400 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:17.460754+0000", 2023-09-23T21:02:29.400 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.422e-06 2023-09-23T21:02:29.401 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:29.401 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:29.401 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:02:29.401 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:17.460773+0000", 2023-09-23T21:02:29.401 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.8783999999999998e-05 2023-09-23T21:02:29.401 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:29.401 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:29.401 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:02:29.401 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:17.460780+0000", 2023-09-23T21:02:29.401 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.5120000000000002e-06 2023-09-23T21:02:29.401 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:29.401 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:29.402 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:02:29.402 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:17.460867+0000", 2023-09-23T21:02:29.402 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.7041999999999998e-05 2023-09-23T21:02:29.402 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:29.402 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:29.402 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:02:29.402 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:17.460988+0000", 2023-09-23T21:02:29.402 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00012111400000000001 2023-09-23T21:02:29.402 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:29.402 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:29.402 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 3,6", 2023-09-23T21:02:29.402 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:17.461422+0000", 2023-09-23T21:02:29.403 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00043413399999999997 2023-09-23T21:02:29.403 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:29.403 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:29.403 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:02:29.403 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:17.471562+0000", 2023-09-23T21:02:29.403 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.010139886000000001 2023-09-23T21:02:29.403 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:29.403 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:29.403 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:02:29.403 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:17.471570+0000", 2023-09-23T21:02:29.403 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.6930000000000007e-06 2023-09-23T21:02:29.403 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:29.404 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:29.404 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:02:29.404 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:18.495442+0000", 2023-09-23T21:02:29.404 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.0238723940000001 2023-09-23T21:02:29.404 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:29.404 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:29.404 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:02:29.405 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:18.495510+0000", 2023-09-23T21:02:29.405 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.7722e-05 2023-09-23T21:02:29.405 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:02:29.405 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:02:29.405 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:02:29.405 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:29.405 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:29.405 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.34797.0:165123 6.3 6:c4ee4362:::benchmark_data_smithi026_223868_object63997:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e155)", 2023-09-23T21:02:29.406 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T20:55:17.470560+0000", 2023-09-23T21:02:29.406 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 431.917096378, 2023-09-23T21:02:29.406 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.0360116610000001, 2023-09-23T21:02:29.406 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:02:29.406 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:02:29.406 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:02:29.406 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.34797", 2023-09-23T21:02:29.406 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T21:02:29.406 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 165123 2023-09-23T21:02:29.406 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:29.406 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:02:29.406 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:29.407 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:02:29.407 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:17.470560+0000", 2023-09-23T21:02:29.407 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:02:29.407 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:29.407 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:29.407 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:02:29.407 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:17.470560+0000", 2023-09-23T21:02:29.407 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:02:29.407 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:29.407 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:29.407 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:02:29.407 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:17.470559+0000", 2023-09-23T21:02:29.408 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T21:02:29.408 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:29.408 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:29.408 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:02:29.408 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:17.470564+0000", 2023-09-23T21:02:29.408 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.8590000000000004e-06 2023-09-23T21:02:29.408 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:29.408 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:29.408 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:02:29.408 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:17.470573+0000", 2023-09-23T21:02:29.408 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.1519999999999998e-06 2023-09-23T21:02:29.408 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:29.408 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:29.409 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:02:29.409 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:17.470580+0000", 2023-09-23T21:02:29.409 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.7360000000000004e-06 2023-09-23T21:02:29.409 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:29.409 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:29.409 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:02:29.409 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:17.487447+0000", 2023-09-23T21:02:29.409 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.016867607999999999 2023-09-23T21:02:29.409 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:29.409 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:29.409 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:02:29.409 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:17.487458+0000", 2023-09-23T21:02:29.409 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.0387999999999999e-05 2023-09-23T21:02:29.410 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:29.410 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:29.410 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:02:29.410 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:18.496800+0000", 2023-09-23T21:02:29.410 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.009342958 2023-09-23T21:02:29.410 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:29.410 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:29.410 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:02:29.410 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:18.496887+0000", 2023-09-23T21:02:29.410 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.6244999999999998e-05 2023-09-23T21:02:29.410 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:29.410 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:29.411 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 2", 2023-09-23T21:02:29.411 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:18.497281+0000", 2023-09-23T21:02:29.411 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00039466799999999998 2023-09-23T21:02:29.411 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:29.411 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:29.411 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:02:29.411 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:18.500593+0000", 2023-09-23T21:02:29.411 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0033114680000000001 2023-09-23T21:02:29.411 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:29.411 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:29.411 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:02:29.411 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:18.506426+0000", 2023-09-23T21:02:29.412 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0058334160000000001 2023-09-23T21:02:29.412 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:29.412 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:29.412 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:02:29.412 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:18.506484+0000", 2023-09-23T21:02:29.412 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.7330999999999997e-05 2023-09-23T21:02:29.412 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:29.412 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:29.412 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:02:29.412 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:18.506572+0000", 2023-09-23T21:02:29.412 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.8185999999999999e-05 2023-09-23T21:02:29.412 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:02:29.412 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:02:29.413 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:02:29.413 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:29.413 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:29.413 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.34797.0:165126 6.13 6:ce6e7106:::benchmark_data_smithi026_223868_object64000:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e155)", 2023-09-23T21:02:29.413 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T20:55:17.481274+0000", 2023-09-23T21:02:29.413 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 431.90638290599998, 2023-09-23T21:02:29.413 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.033641289, 2023-09-23T21:02:29.413 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:02:29.413 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:02:29.413 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:02:29.413 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.34797", 2023-09-23T21:02:29.414 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T21:02:29.414 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 165126 2023-09-23T21:02:29.414 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:29.414 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:02:29.414 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:29.414 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:02:29.414 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:17.481274+0000", 2023-09-23T21:02:29.414 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:02:29.414 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:29.414 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:29.414 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:02:29.414 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:17.481274+0000", 2023-09-23T21:02:29.415 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:02:29.415 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:29.415 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:29.415 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:02:29.415 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:17.481272+0000", 2023-09-23T21:02:29.415 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999981 2023-09-23T21:02:29.415 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:29.415 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:29.415 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:02:29.415 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:17.481280+0000", 2023-09-23T21:02:29.415 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.1300000000000001e-06 2023-09-23T21:02:29.415 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:29.415 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:29.416 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:02:29.416 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:17.481293+0000", 2023-09-23T21:02:29.416 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.2808e-05 2023-09-23T21:02:29.416 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:29.416 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:29.416 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:02:29.416 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:17.481299+0000", 2023-09-23T21:02:29.416 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.6449999999999999e-06 2023-09-23T21:02:29.416 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:29.416 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:29.416 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:02:29.416 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:17.488352+0000", 2023-09-23T21:02:29.416 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0070529449999999997 2023-09-23T21:02:29.417 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:29.417 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:29.417 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:02:29.417 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:17.488360+0000", 2023-09-23T21:02:29.417 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.1659999999999994e-06 2023-09-23T21:02:29.417 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:29.417 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:29.417 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:02:29.417 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:18.502688+0000", 2023-09-23T21:02:29.417 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.0143277129999999 2023-09-23T21:02:29.417 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:29.417 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:29.417 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:02:29.418 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:18.502752+0000", 2023-09-23T21:02:29.418 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.4089000000000003e-05 2023-09-23T21:02:29.418 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:29.418 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:29.418 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 2", 2023-09-23T21:02:29.418 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:18.503096+0000", 2023-09-23T21:02:29.418 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00034352900000000002 2023-09-23T21:02:29.418 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:29.418 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:29.418 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:02:29.418 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:18.506370+0000", 2023-09-23T21:02:29.418 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0032744889999999998 2023-09-23T21:02:29.419 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:29.419 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:29.419 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:02:29.419 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:18.514787+0000", 2023-09-23T21:02:29.419 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0084169259999999999 2023-09-23T21:02:29.419 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:29.419 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:29.419 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:02:29.419 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:18.514829+0000", 2023-09-23T21:02:29.419 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.1576999999999999e-05 2023-09-23T21:02:29.419 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:29.419 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:29.419 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:02:29.420 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:18.514915+0000", 2023-09-23T21:02:29.420 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.6071999999999999e-05 2023-09-23T21:02:29.420 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:02:29.420 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:02:29.420 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:02:29.420 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:29.420 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:29.420 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.34797.0:165127 6.5 6:a3ee0f5c:::benchmark_data_smithi026_223868_object64001:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e155)", 2023-09-23T21:02:29.420 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T20:55:17.488312+0000", 2023-09-23T21:02:29.420 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 431.89934434200001, 2023-09-23T21:02:29.420 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.0197466340000001, 2023-09-23T21:02:29.420 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:02:29.420 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:02:29.420 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:02:29.421 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.34797", 2023-09-23T21:02:29.421 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T21:02:29.421 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 165127 2023-09-23T21:02:29.421 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:29.421 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:02:29.421 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:29.421 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:02:29.421 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:17.488312+0000", 2023-09-23T21:02:29.421 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:02:29.421 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:29.421 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:29.421 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:02:29.422 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:17.488312+0000", 2023-09-23T21:02:29.422 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:02:29.422 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:29.422 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:29.422 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:02:29.422 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:17.488311+0000", 2023-09-23T21:02:29.422 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T21:02:29.422 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:29.422 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:29.422 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:02:29.422 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:17.488317+0000", 2023-09-23T21:02:29.422 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.2519999999999996e-06 2023-09-23T21:02:29.422 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:29.423 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:29.423 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:02:29.423 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:17.488331+0000", 2023-09-23T21:02:29.423 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.3621000000000001e-05 2023-09-23T21:02:29.423 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:29.423 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:29.423 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:02:29.423 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:17.488338+0000", 2023-09-23T21:02:29.423 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.3289999999999996e-06 2023-09-23T21:02:29.423 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:29.423 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:29.423 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:02:29.424 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:17.489326+0000", 2023-09-23T21:02:29.424 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00098795400000000009 2023-09-23T21:02:29.424 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:29.424 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:29.424 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:02:29.424 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:17.489333+0000", 2023-09-23T21:02:29.424 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.1810000000000004e-06 2023-09-23T21:02:29.424 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:29.424 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:29.424 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:02:29.424 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:18.497619+0000", 2023-09-23T21:02:29.424 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.0082864410000001 2023-09-23T21:02:29.425 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:29.425 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:29.425 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:02:29.425 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:18.497762+0000", 2023-09-23T21:02:29.425 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000142733 2023-09-23T21:02:29.425 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:29.425 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:29.425 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 3", 2023-09-23T21:02:29.425 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:18.498195+0000", 2023-09-23T21:02:29.425 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00043258599999999999 2023-09-23T21:02:29.425 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:29.426 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:29.426 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:02:29.426 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:18.502126+0000", 2023-09-23T21:02:29.426 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0039310480000000004 2023-09-23T21:02:29.426 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:29.426 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:29.426 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:02:29.426 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:18.507912+0000", 2023-09-23T21:02:29.426 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0057865850000000003 2023-09-23T21:02:29.426 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:29.426 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:29.426 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:02:29.427 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:18.507966+0000", 2023-09-23T21:02:29.427 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.3939e-05 2023-09-23T21:02:29.427 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:29.427 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:29.427 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:02:29.427 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:18.508059+0000", 2023-09-23T21:02:29.427 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.2459000000000004e-05 2023-09-23T21:02:29.427 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:02:29.427 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:02:29.427 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:02:29.427 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:29.427 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:29.427 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.34797.0:165131 6.5 6:a6e346e0:::benchmark_data_smithi026_223868_object64005:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e155)", 2023-09-23T21:02:29.428 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T20:55:17.490453+0000", 2023-09-23T21:02:29.428 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 431.89720309299997, 2023-09-23T21:02:29.428 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.0178180109999999, 2023-09-23T21:02:29.428 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:02:29.428 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:02:29.428 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:02:29.428 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.34797", 2023-09-23T21:02:29.428 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T21:02:29.428 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 165131 2023-09-23T21:02:29.428 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:29.428 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:02:29.428 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:29.428 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:02:29.429 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:17.490453+0000", 2023-09-23T21:02:29.429 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:02:29.429 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:29.429 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:29.429 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:02:29.429 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:17.490453+0000", 2023-09-23T21:02:29.429 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:02:29.429 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:29.429 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:29.429 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:02:29.429 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:17.490453+0000", 2023-09-23T21:02:29.429 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999995 2023-09-23T21:02:29.430 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:29.430 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:29.430 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:02:29.430 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:17.490457+0000", 2023-09-23T21:02:29.430 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.8759999999999998e-06 2023-09-23T21:02:29.430 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:29.430 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:29.430 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:02:29.430 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:17.490464+0000", 2023-09-23T21:02:29.430 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.4120000000000004e-06 2023-09-23T21:02:29.430 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:29.430 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:29.431 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:02:29.431 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:17.490468+0000", 2023-09-23T21:02:29.431 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.1690000000000002e-06 2023-09-23T21:02:29.431 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:29.431 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:29.431 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:02:29.431 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:17.490616+0000", 2023-09-23T21:02:29.431 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00014773 2023-09-23T21:02:29.431 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:29.431 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:29.431 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:02:29.431 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:17.490623+0000", 2023-09-23T21:02:29.432 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.9569999999999996e-06 2023-09-23T21:02:29.432 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:29.432 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:29.432 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:02:29.432 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:18.500289+0000", 2023-09-23T21:02:29.432 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.0096661769999999 2023-09-23T21:02:29.432 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:29.432 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:29.432 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:02:29.432 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:18.500369+0000", 2023-09-23T21:02:29.432 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.9552e-05 2023-09-23T21:02:29.432 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:29.433 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:29.433 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 3", 2023-09-23T21:02:29.433 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:18.500618+0000", 2023-09-23T21:02:29.433 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00024934399999999998 2023-09-23T21:02:29.433 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:29.433 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:29.433 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:02:29.433 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:18.502288+0000", 2023-09-23T21:02:29.433 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0016698819999999999 2023-09-23T21:02:29.433 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:29.433 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:29.433 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:02:29.434 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:18.508125+0000", 2023-09-23T21:02:29.434 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0058374070000000002 2023-09-23T21:02:29.434 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:29.434 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:29.434 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:02:29.434 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:18.508179+0000", 2023-09-23T21:02:29.434 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.3885999999999999e-05 2023-09-23T21:02:29.434 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:29.434 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:29.434 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:02:29.434 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:18.508271+0000", 2023-09-23T21:02:29.434 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.2120000000000006e-05 2023-09-23T21:02:29.435 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:02:29.435 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:02:29.435 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:02:29.435 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:29.435 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:29.435 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.34797.0:165142 6.3 6:c41d7515:::benchmark_data_smithi026_223868_object64016:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e155)", 2023-09-23T21:02:29.435 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T20:55:17.510429+0000", 2023-09-23T21:02:29.435 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 431.87722701199999, 2023-09-23T21:02:29.435 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.004486011, 2023-09-23T21:02:29.435 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:02:29.435 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:02:29.435 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:02:29.435 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.34797", 2023-09-23T21:02:29.436 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T21:02:29.436 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 165142 2023-09-23T21:02:29.436 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:29.436 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:02:29.436 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:29.436 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:02:29.436 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:17.510429+0000", 2023-09-23T21:02:29.436 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:02:29.436 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:29.436 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:29.436 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:02:29.436 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:17.510429+0000", 2023-09-23T21:02:29.436 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:02:29.437 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:29.437 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:29.437 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:02:29.437 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:17.510428+0000", 2023-09-23T21:02:29.437 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T21:02:29.437 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:29.437 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:29.437 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:02:29.437 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:17.510436+0000", 2023-09-23T21:02:29.437 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.174e-06 2023-09-23T21:02:29.437 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:29.437 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:29.438 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:02:29.438 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:17.510453+0000", 2023-09-23T21:02:29.438 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.7091e-05 2023-09-23T21:02:29.438 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:29.438 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:29.438 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:02:29.438 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:17.510461+0000", 2023-09-23T21:02:29.438 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.6170000000000001e-06 2023-09-23T21:02:29.438 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:29.438 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:29.438 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:02:29.438 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:17.510513+0000", 2023-09-23T21:02:29.439 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.1829999999999997e-05 2023-09-23T21:02:29.439 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:29.439 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:29.439 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:02:29.439 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:17.510522+0000", 2023-09-23T21:02:29.439 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.8810000000000002e-06 2023-09-23T21:02:29.439 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:29.439 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:29.439 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:02:29.439 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:18.498926+0000", 2023-09-23T21:02:29.439 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.98840431699999998 2023-09-23T21:02:29.439 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:29.439 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:29.440 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:02:29.440 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:18.498986+0000", 2023-09-23T21:02:29.440 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.9976999999999999e-05 2023-09-23T21:02:29.440 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:29.440 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:29.440 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 2", 2023-09-23T21:02:29.440 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:18.499178+0000", 2023-09-23T21:02:29.440 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00019188899999999999 2023-09-23T21:02:29.440 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:29.440 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:29.440 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:02:29.440 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:18.500630+0000", 2023-09-23T21:02:29.441 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.001452271 2023-09-23T21:02:29.441 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:29.441 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:29.441 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:02:29.441 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:18.514775+0000", 2023-09-23T21:02:29.441 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.014144647999999999 2023-09-23T21:02:29.441 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:29.441 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:29.441 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:02:29.441 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:18.514826+0000", 2023-09-23T21:02:29.441 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.1479e-05 2023-09-23T21:02:29.441 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:29.442 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:29.442 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:02:29.442 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:18.514915+0000", 2023-09-23T21:02:29.442 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.9297999999999998e-05 2023-09-23T21:02:29.442 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:02:29.442 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:02:29.442 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:02:29.442 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:29.442 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:29.442 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.34797.0:165143 6.13 6:c91cb188:::benchmark_data_smithi026_223868_object64017:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e155)", 2023-09-23T21:02:29.442 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T20:55:17.510722+0000", 2023-09-23T21:02:29.442 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 431.87693474999998, 2023-09-23T21:02:29.443 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.004352135, 2023-09-23T21:02:29.443 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:02:29.443 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:02:29.443 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:02:29.443 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.34797", 2023-09-23T21:02:29.443 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T21:02:29.443 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 165143 2023-09-23T21:02:29.443 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:29.443 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:02:29.443 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:29.443 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:02:29.443 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:17.510722+0000", 2023-09-23T21:02:29.443 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:02:29.444 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:29.444 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:29.444 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:02:29.444 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:17.510722+0000", 2023-09-23T21:02:29.444 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:02:29.444 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:29.444 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:29.444 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:02:29.444 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:17.510721+0000", 2023-09-23T21:02:29.444 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999995 2023-09-23T21:02:29.444 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:29.444 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:29.445 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:02:29.445 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:17.510730+0000", 2023-09-23T21:02:29.445 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.551e-06 2023-09-23T21:02:29.445 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:29.445 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:29.445 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:02:29.445 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:17.510742+0000", 2023-09-23T21:02:29.445 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.2227e-05 2023-09-23T21:02:29.445 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:29.445 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:29.445 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:02:29.446 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:17.510748+0000", 2023-09-23T21:02:29.446 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.1029999999999998e-06 2023-09-23T21:02:29.446 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:29.446 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:29.446 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:02:29.446 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:17.510789+0000", 2023-09-23T21:02:29.446 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.1362999999999997e-05 2023-09-23T21:02:29.446 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:29.446 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:29.446 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:02:29.446 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:17.510794+0000", 2023-09-23T21:02:29.446 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.668e-06 2023-09-23T21:02:29.447 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:29.447 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:29.447 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:02:29.447 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:18.504669+0000", 2023-09-23T21:02:29.447 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.99387451400000004 2023-09-23T21:02:29.447 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:29.447 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:29.447 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:02:29.447 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:18.504731+0000", 2023-09-23T21:02:29.447 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.2508999999999994e-05 2023-09-23T21:02:29.447 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:29.447 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:29.447 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 2", 2023-09-23T21:02:29.448 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:18.504915+0000", 2023-09-23T21:02:29.448 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000184258 2023-09-23T21:02:29.448 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:29.448 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:29.448 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:02:29.448 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:18.508092+0000", 2023-09-23T21:02:29.448 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0031764779999999999 2023-09-23T21:02:29.448 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:29.448 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:29.448 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:02:29.448 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:18.514951+0000", 2023-09-23T21:02:29.448 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0068590439999999999 2023-09-23T21:02:29.448 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:29.449 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:29.449 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:02:29.449 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:18.514990+0000", 2023-09-23T21:02:29.449 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.8831000000000001e-05 2023-09-23T21:02:29.449 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:29.449 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:29.449 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:02:29.449 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:18.515074+0000", 2023-09-23T21:02:29.449 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.4239000000000004e-05 2023-09-23T21:02:29.449 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:02:29.449 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:02:29.449 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:02:29.450 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:29.450 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:29.450 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.34797.0:177909 6.13 6:c9b40212:::benchmark_data_smithi026_223868_object76783:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e164)", 2023-09-23T21:02:29.450 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T20:55:34.933784+0000", 2023-09-23T21:02:29.450 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 414.453872698, 2023-09-23T21:02:29.450 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.1089723249999999, 2023-09-23T21:02:29.450 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:02:29.450 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:02:29.450 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:02:29.450 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.34797", 2023-09-23T21:02:29.450 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T21:02:29.451 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 177909 2023-09-23T21:02:29.451 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:29.451 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:02:29.451 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:29.451 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:02:29.451 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:34.933784+0000", 2023-09-23T21:02:29.451 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:02:29.451 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:29.451 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:29.451 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:02:29.451 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:34.933784+0000", 2023-09-23T21:02:29.451 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:02:29.451 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:29.452 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:29.452 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:02:29.452 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:34.933782+0000", 2023-09-23T21:02:29.452 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T21:02:29.452 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:29.452 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:29.452 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:02:29.452 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:34.933791+0000", 2023-09-23T21:02:29.452 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.3259999999999996e-06 2023-09-23T21:02:29.452 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:29.452 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:29.452 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:02:29.453 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:34.933804+0000", 2023-09-23T21:02:29.453 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.3155e-05 2023-09-23T21:02:29.453 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:29.453 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:29.453 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:02:29.453 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:34.933811+0000", 2023-09-23T21:02:29.453 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.2919999999999998e-06 2023-09-23T21:02:29.453 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:29.453 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:29.453 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:02:29.453 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:34.940148+0000", 2023-09-23T21:02:29.453 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0063370350000000004 2023-09-23T21:02:29.454 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:29.454 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:29.454 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:02:29.454 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:34.940159+0000", 2023-09-23T21:02:29.454 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.0611999999999999e-05 2023-09-23T21:02:29.454 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:29.454 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:29.454 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:02:29.454 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:35.938118+0000", 2023-09-23T21:02:29.454 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.99795963499999996 2023-09-23T21:02:29.454 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:29.455 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:29.455 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:02:29.455 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:35.938130+0000", 2023-09-23T21:02:29.455 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.13e-05 2023-09-23T21:02:29.455 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:29.455 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:29.455 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:02:29.455 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:37.032096+0000", 2023-09-23T21:02:29.455 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.093965928 2023-09-23T21:02:29.455 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:29.455 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:29.455 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:02:29.455 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:37.032231+0000", 2023-09-23T21:02:29.456 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00013501999999999999 2023-09-23T21:02:29.456 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:29.456 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:29.456 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 2,6", 2023-09-23T21:02:29.456 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:37.032750+0000", 2023-09-23T21:02:29.456 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00051932900000000001 2023-09-23T21:02:29.456 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:29.456 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:29.456 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:02:29.456 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:37.040969+0000", 2023-09-23T21:02:29.456 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0082194039999999996 2023-09-23T21:02:29.456 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:29.457 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:29.457 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:02:29.457 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:37.041526+0000", 2023-09-23T21:02:29.457 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000556347 2023-09-23T21:02:29.457 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:29.457 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:29.457 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:02:29.457 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:37.042559+0000", 2023-09-23T21:02:29.457 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.001033405 2023-09-23T21:02:29.457 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:29.457 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:29.457 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:02:29.457 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:37.042611+0000", 2023-09-23T21:02:29.458 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.2101999999999997e-05 2023-09-23T21:02:29.458 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:29.458 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:29.458 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:02:29.458 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:37.042756+0000", 2023-09-23T21:02:29.458 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00014501499999999999 2023-09-23T21:02:29.458 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:02:29.458 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:02:29.458 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:02:29.458 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:29.458 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:29.458 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.34797.0:177918 6.13 6:cb5d9774:::benchmark_data_smithi026_223868_object76792:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e164)", 2023-09-23T21:02:29.459 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T20:55:34.943354+0000", 2023-09-23T21:02:29.459 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 414.44430202500001, 2023-09-23T21:02:29.459 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.1104338490000001, 2023-09-23T21:02:29.459 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:02:29.459 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:02:29.459 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:02:29.459 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.34797", 2023-09-23T21:02:29.459 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T21:02:29.459 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 177918 2023-09-23T21:02:29.459 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:29.459 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:02:29.459 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:29.459 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:02:29.460 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:34.943354+0000", 2023-09-23T21:02:29.460 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:02:29.460 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:29.460 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:29.460 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:02:29.460 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:34.943354+0000", 2023-09-23T21:02:29.460 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:02:29.460 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:29.460 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:29.460 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:02:29.460 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:34.943353+0000", 2023-09-23T21:02:29.460 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999981 2023-09-23T21:02:29.460 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:29.461 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:29.461 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:02:29.461 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:34.943362+0000", 2023-09-23T21:02:29.461 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.5310000000000007e-06 2023-09-23T21:02:29.461 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:29.461 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:29.461 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:02:29.461 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:34.943381+0000", 2023-09-23T21:02:29.461 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.8284999999999999e-05 2023-09-23T21:02:29.461 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:29.461 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:29.461 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:02:29.462 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:34.943390+0000", 2023-09-23T21:02:29.462 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.2010000000000005e-06 2023-09-23T21:02:29.462 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:29.462 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:29.462 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:02:29.462 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:34.943462+0000", 2023-09-23T21:02:29.462 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.2002000000000006e-05 2023-09-23T21:02:29.462 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:29.462 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:29.462 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:02:29.462 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:34.943472+0000", 2023-09-23T21:02:29.463 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.0356e-05 2023-09-23T21:02:29.463 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:29.463 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:29.463 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:02:29.463 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:35.938155+0000", 2023-09-23T21:02:29.463 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.99468292700000005 2023-09-23T21:02:29.463 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:29.463 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:29.463 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:02:29.463 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:35.938162+0000", 2023-09-23T21:02:29.463 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.0589999999999997e-06 2023-09-23T21:02:29.463 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:29.464 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:29.464 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:02:29.464 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:37.034935+0000", 2023-09-23T21:02:29.464 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.0967725079999999 2023-09-23T21:02:29.464 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:29.464 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:29.464 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:02:29.464 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:37.035045+0000", 2023-09-23T21:02:29.464 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000110213 2023-09-23T21:02:29.464 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:29.464 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:29.464 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 2,6", 2023-09-23T21:02:29.465 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:37.035460+0000", 2023-09-23T21:02:29.465 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00041482800000000001 2023-09-23T21:02:29.465 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:29.465 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:29.465 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:02:29.465 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:37.041023+0000", 2023-09-23T21:02:29.465 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0055634040000000001 2023-09-23T21:02:29.465 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:29.465 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:29.465 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:02:29.465 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:37.049549+0000", 2023-09-23T21:02:29.465 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0085260170000000003 2023-09-23T21:02:29.466 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:29.466 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:29.466 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:02:29.466 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:37.053587+0000", 2023-09-23T21:02:29.466 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0040375999999999997 2023-09-23T21:02:29.466 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:29.466 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:29.466 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:02:29.466 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:37.053643+0000", 2023-09-23T21:02:29.466 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.6524999999999998e-05 2023-09-23T21:02:29.466 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:29.466 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:29.467 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:02:29.467 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:37.053788+0000", 2023-09-23T21:02:29.467 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00014508699999999999 2023-09-23T21:02:29.467 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:02:29.467 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:02:29.467 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:02:29.467 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:29.467 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:29.467 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.34797.0:177940 6.13 6:c8fa8a85:::benchmark_data_smithi026_223868_object76814:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e164)", 2023-09-23T21:02:29.467 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T20:55:34.983208+0000", 2023-09-23T21:02:29.467 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 414.40444833599997, 2023-09-23T21:02:29.467 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.0707293249999998, 2023-09-23T21:02:29.468 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:02:29.468 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:02:29.468 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:02:29.468 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.34797", 2023-09-23T21:02:29.468 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T21:02:29.468 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 177940 2023-09-23T21:02:29.468 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:29.468 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:02:29.468 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:29.468 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:02:29.468 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:34.983208+0000", 2023-09-23T21:02:29.468 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:02:29.469 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:29.469 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:29.469 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:02:29.469 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:34.983208+0000", 2023-09-23T21:02:29.469 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:02:29.469 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:29.469 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:29.469 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:02:29.469 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:34.983207+0000", 2023-09-23T21:02:29.469 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.999999 2023-09-23T21:02:29.469 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:29.469 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:29.469 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:02:29.470 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:34.983212+0000", 2023-09-23T21:02:29.470 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.3419999999999998e-06 2023-09-23T21:02:29.470 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:29.470 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:29.470 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:02:29.470 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:34.983220+0000", 2023-09-23T21:02:29.470 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.4610000000000002e-06 2023-09-23T21:02:29.470 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:29.470 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:29.470 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:02:29.470 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:34.983224+0000", 2023-09-23T21:02:29.470 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.4610000000000001e-06 2023-09-23T21:02:29.471 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:29.471 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:29.471 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:02:29.471 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:34.983289+0000", 2023-09-23T21:02:29.471 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.5041000000000003e-05 2023-09-23T21:02:29.471 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:29.471 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:29.471 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:02:29.471 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:34.983297+0000", 2023-09-23T21:02:29.471 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.046e-06 2023-09-23T21:02:29.471 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:29.471 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:29.472 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:02:29.472 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:35.938873+0000", 2023-09-23T21:02:29.472 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.95557668900000003 2023-09-23T21:02:29.472 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:29.472 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:29.472 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:02:29.472 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:35.938882+0000", 2023-09-23T21:02:29.472 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.5059999999999995e-06 2023-09-23T21:02:29.472 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:29.472 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:29.472 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:02:29.472 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:37.037926+0000", 2023-09-23T21:02:29.473 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.0990439059999999 2023-09-23T21:02:29.473 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:29.473 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:29.473 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:02:29.473 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:37.038068+0000", 2023-09-23T21:02:29.473 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00014263599999999999 2023-09-23T21:02:29.473 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:29.473 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:29.473 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 2,6", 2023-09-23T21:02:29.473 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:37.038541+0000", 2023-09-23T21:02:29.473 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00047294699999999998 2023-09-23T21:02:29.473 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:29.474 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:29.474 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:02:29.474 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:37.041561+0000", 2023-09-23T21:02:29.474 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0030193889999999999 2023-09-23T21:02:29.474 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:29.474 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:29.474 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:02:29.474 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:37.049637+0000", 2023-09-23T21:02:29.474 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0080764089999999997 2023-09-23T21:02:29.474 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:29.474 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:29.474 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:02:29.474 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:37.053824+0000", 2023-09-23T21:02:29.475 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.004186871 2023-09-23T21:02:29.475 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:29.475 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:29.475 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:02:29.475 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:37.053870+0000", 2023-09-23T21:02:29.475 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.5782000000000003e-05 2023-09-23T21:02:29.475 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:29.475 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:29.475 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:02:29.475 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:37.053937+0000", 2023-09-23T21:02:29.475 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.7842999999999994e-05 2023-09-23T21:02:29.475 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:02:29.476 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:02:29.476 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:02:29.476 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:29.476 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:29.476 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.34797.0:177941 6.15 6:ac4602e1:::benchmark_data_smithi026_223868_object76815:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e164)", 2023-09-23T21:02:29.476 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T20:55:34.983250+0000", 2023-09-23T21:02:29.476 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 414.40440671599998, 2023-09-23T21:02:29.476 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.0517934200000001, 2023-09-23T21:02:29.476 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:02:29.476 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:02:29.476 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:02:29.476 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.34797", 2023-09-23T21:02:29.477 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T21:02:29.477 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 177941 2023-09-23T21:02:29.477 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:29.477 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:02:29.477 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:29.477 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:02:29.477 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:34.983250+0000", 2023-09-23T21:02:29.477 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:02:29.477 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:29.477 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:29.477 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:02:29.477 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:34.983250+0000", 2023-09-23T21:02:29.477 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:02:29.478 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:29.478 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:29.478 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:02:29.479 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:34.983249+0000", 2023-09-23T21:02:29.479 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999995 2023-09-23T21:02:29.479 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:29.479 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:29.479 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:02:29.479 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:34.983252+0000", 2023-09-23T21:02:29.479 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.0149999999999999e-06 2023-09-23T21:02:29.479 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:29.479 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:29.479 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:02:29.480 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:34.983259+0000", 2023-09-23T21:02:29.480 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.5540000000000003e-06 2023-09-23T21:02:29.480 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:29.480 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:29.480 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:02:29.480 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:34.983263+0000", 2023-09-23T21:02:29.480 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.1010000000000002e-06 2023-09-23T21:02:29.480 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:29.480 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:29.480 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:02:29.480 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:34.983303+0000", 2023-09-23T21:02:29.480 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.9650999999999997e-05 2023-09-23T21:02:29.480 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:29.481 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:29.481 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:02:29.481 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:34.983309+0000", 2023-09-23T21:02:29.481 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.9320000000000001e-06 2023-09-23T21:02:29.481 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:29.481 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:29.481 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:02:29.481 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:35.939378+0000", 2023-09-23T21:02:29.481 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.95606941999999995 2023-09-23T21:02:29.481 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:29.481 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:29.481 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:02:29.481 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:35.939387+0000", 2023-09-23T21:02:29.482 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.9740000000000008e-06 2023-09-23T21:02:29.482 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:29.482 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:29.482 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:02:29.482 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:37.026628+0000", 2023-09-23T21:02:29.482 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.0872411820000001 2023-09-23T21:02:29.482 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:29.482 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:29.482 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:02:29.482 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:37.026766+0000", 2023-09-23T21:02:29.482 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00013818599999999999 2023-09-23T21:02:29.483 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:29.483 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:29.483 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 3,6", 2023-09-23T21:02:29.483 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:37.027155+0000", 2023-09-23T21:02:29.483 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00038901000000000002 2023-09-23T21:02:29.483 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:29.483 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:29.483 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:02:29.483 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:37.034577+0000", 2023-09-23T21:02:29.483 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0074220909999999996 2023-09-23T21:02:29.483 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:29.483 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:29.483 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:02:29.484 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:37.034818+0000", 2023-09-23T21:02:29.484 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00024096600000000001 2023-09-23T21:02:29.484 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:29.484 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:29.484 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:02:29.484 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:37.034889+0000", 2023-09-23T21:02:29.484 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.0186999999999997e-05 2023-09-23T21:02:29.484 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:29.484 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:29.484 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:02:29.484 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:37.034942+0000", 2023-09-23T21:02:29.484 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.3251999999999998e-05 2023-09-23T21:02:29.485 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:29.485 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:29.485 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:02:29.485 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:37.035043+0000", 2023-09-23T21:02:29.485 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000101322 2023-09-23T21:02:29.485 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:02:29.485 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:02:29.485 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:02:29.485 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:29.485 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:29.485 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.34797.0:177943 6.15 6:ae2b0147:::benchmark_data_smithi026_223868_object76817:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e164)", 2023-09-23T21:02:29.485 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T20:55:34.984662+0000", 2023-09-23T21:02:29.486 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 414.40299493100002, 2023-09-23T21:02:29.486 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.0517963250000002, 2023-09-23T21:02:29.486 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:02:29.486 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:02:29.486 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:02:29.486 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.34797", 2023-09-23T21:02:29.486 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T21:02:29.486 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 177943 2023-09-23T21:02:29.486 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:29.486 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:02:29.486 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:29.486 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:02:29.486 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:34.984662+0000", 2023-09-23T21:02:29.487 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:02:29.487 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:29.487 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:29.487 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:02:29.487 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:34.984662+0000", 2023-09-23T21:02:29.487 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:02:29.487 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:29.487 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:29.487 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:02:29.487 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:34.984661+0000", 2023-09-23T21:02:29.487 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999995 2023-09-23T21:02:29.487 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:29.487 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:29.488 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:02:29.488 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:34.984665+0000", 2023-09-23T21:02:29.488 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.0400000000000003e-06 2023-09-23T21:02:29.488 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:29.488 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:29.488 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:02:29.488 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:34.984673+0000", 2023-09-23T21:02:29.488 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.6590000000000007e-06 2023-09-23T21:02:29.488 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:29.488 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:29.488 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:02:29.488 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:34.984677+0000", 2023-09-23T21:02:29.488 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.9450000000000003e-06 2023-09-23T21:02:29.488 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:29.489 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:29.489 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:02:29.489 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:34.984720+0000", 2023-09-23T21:02:29.489 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.3679e-05 2023-09-23T21:02:29.489 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:29.489 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:29.489 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:02:29.489 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:34.984726+0000", 2023-09-23T21:02:29.489 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.0650000000000004e-06 2023-09-23T21:02:29.489 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:29.489 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:29.489 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:02:29.489 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:35.939411+0000", 2023-09-23T21:02:29.489 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.95468514100000001 2023-09-23T21:02:29.490 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:29.490 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:29.490 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:02:29.490 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:35.939419+0000", 2023-09-23T21:02:29.490 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.1169999999999998e-06 2023-09-23T21:02:29.490 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:29.490 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:29.490 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:02:29.490 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:37.029284+0000", 2023-09-23T21:02:29.490 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.0898655939999999 2023-09-23T21:02:29.490 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:29.490 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:29.491 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:02:29.491 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:37.029432+0000", 2023-09-23T21:02:29.491 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000147978 2023-09-23T21:02:29.491 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:29.491 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:29.491 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 3,6", 2023-09-23T21:02:29.491 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:37.029829+0000", 2023-09-23T21:02:29.491 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000397272 2023-09-23T21:02:29.491 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:29.491 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:29.491 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:02:29.491 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:37.034635+0000", 2023-09-23T21:02:29.492 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0048057719999999998 2023-09-23T21:02:29.492 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:29.492 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:29.492 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:02:29.492 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:37.034857+0000", 2023-09-23T21:02:29.492 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00022135300000000001 2023-09-23T21:02:29.492 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:29.492 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:29.492 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:02:29.492 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:37.036289+0000", 2023-09-23T21:02:29.492 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.001431978 2023-09-23T21:02:29.492 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:29.493 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:29.493 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:02:29.493 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:37.036342+0000", 2023-09-23T21:02:29.493 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.3029999999999999e-05 2023-09-23T21:02:29.493 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:29.493 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:29.493 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:02:29.493 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:37.036458+0000", 2023-09-23T21:02:29.493 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00011630500000000001 2023-09-23T21:02:29.493 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:02:29.493 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:02:29.493 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:02:29.493 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:29.494 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:29.494 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.34797.0:177955 6.15 6:aed95e65:::benchmark_data_smithi026_223868_object76829:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e164)", 2023-09-23T21:02:29.494 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T20:55:35.010017+0000", 2023-09-23T21:02:29.494 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 414.37763944300002, 2023-09-23T21:02:29.494 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.026625954, 2023-09-23T21:02:29.494 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:02:29.494 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:02:29.494 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:02:29.494 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.34797", 2023-09-23T21:02:29.494 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T21:02:29.494 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 177955 2023-09-23T21:02:29.494 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:29.495 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:02:29.495 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:29.495 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:02:29.495 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:35.010017+0000", 2023-09-23T21:02:29.495 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:02:29.495 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:29.495 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:29.495 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:02:29.495 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:35.010017+0000", 2023-09-23T21:02:29.495 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:02:29.495 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:29.495 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:29.495 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:02:29.496 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:35.010016+0000", 2023-09-23T21:02:29.496 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999995 2023-09-23T21:02:29.496 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:29.496 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:29.496 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:02:29.496 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:35.010020+0000", 2023-09-23T21:02:29.496 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.416e-06 2023-09-23T21:02:29.496 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:29.496 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:29.496 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:02:29.496 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:35.010028+0000", 2023-09-23T21:02:29.496 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.4640000000000006e-06 2023-09-23T21:02:29.496 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:29.496 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:29.497 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:02:29.497 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:35.010032+0000", 2023-09-23T21:02:29.497 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.985e-06 2023-09-23T21:02:29.497 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:29.497 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:29.497 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:02:29.497 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:35.010088+0000", 2023-09-23T21:02:29.497 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.5614e-05 2023-09-23T21:02:29.497 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:29.497 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:29.497 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:02:29.497 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:35.010098+0000", 2023-09-23T21:02:29.497 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.0553e-05 2023-09-23T21:02:29.498 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:29.498 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:29.498 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:02:29.498 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:35.939934+0000", 2023-09-23T21:02:29.498 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.92983511600000002 2023-09-23T21:02:29.498 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:29.498 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:29.498 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:02:29.498 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:35.939942+0000", 2023-09-23T21:02:29.498 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.3049999999999993e-06 2023-09-23T21:02:29.498 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:29.498 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:29.498 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:02:29.499 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:37.032144+0000", 2023-09-23T21:02:29.499 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.09220184 2023-09-23T21:02:29.499 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:29.499 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:29.499 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:02:29.499 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:37.032273+0000", 2023-09-23T21:02:29.499 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00012959399999999999 2023-09-23T21:02:29.499 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:29.499 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:29.499 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 3,6", 2023-09-23T21:02:29.499 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:37.032579+0000", 2023-09-23T21:02:29.499 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000306075 2023-09-23T21:02:29.499 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:29.500 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:29.500 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:02:29.500 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:37.034826+0000", 2023-09-23T21:02:29.500 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0022461669999999999 2023-09-23T21:02:29.500 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:29.500 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:29.500 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:02:29.500 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:37.036494+0000", 2023-09-23T21:02:29.500 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0016682559999999999 2023-09-23T21:02:29.500 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:29.500 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:29.500 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:02:29.500 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:37.036520+0000", 2023-09-23T21:02:29.501 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.5797999999999999e-05 2023-09-23T21:02:29.501 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:29.501 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:29.501 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:02:29.501 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:37.036557+0000", 2023-09-23T21:02:29.501 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.6993000000000003e-05 2023-09-23T21:02:29.501 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:29.501 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:29.501 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:02:29.501 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:37.036643+0000", 2023-09-23T21:02:29.501 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.6441999999999997e-05 2023-09-23T21:02:29.501 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:02:29.501 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:02:29.502 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:02:29.502 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:29.502 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:29.502 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.34797.0:183991 6.3 6:c69266c2:::benchmark_data_smithi026_223868_object82865:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e169)", 2023-09-23T21:02:29.502 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T20:55:44.983566+0000", 2023-09-23T21:02:29.502 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 404.404090225, 2023-09-23T21:02:29.502 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.009211377, 2023-09-23T21:02:29.502 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:02:29.502 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:02:29.502 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:02:29.502 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.34797", 2023-09-23T21:02:29.502 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T21:02:29.503 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 183991 2023-09-23T21:02:29.503 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:29.503 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:02:29.503 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:29.503 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:02:29.503 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:44.983566+0000", 2023-09-23T21:02:29.503 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:02:29.503 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:29.503 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:29.503 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:02:29.503 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:44.983566+0000", 2023-09-23T21:02:29.503 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:02:29.503 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:29.504 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:29.504 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:02:29.504 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:44.983566+0000", 2023-09-23T21:02:29.504 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999995 2023-09-23T21:02:29.504 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:29.504 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:29.504 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:02:29.504 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:44.983570+0000", 2023-09-23T21:02:29.504 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.8160000000000004e-06 2023-09-23T21:02:29.504 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:29.504 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:29.504 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:02:29.504 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:44.983580+0000", 2023-09-23T21:02:29.505 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.0499999999999999e-05 2023-09-23T21:02:29.505 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:29.505 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:29.505 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:02:29.505 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:44.983585+0000", 2023-09-23T21:02:29.505 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.933e-06 2023-09-23T21:02:29.505 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:29.505 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:29.505 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:02:29.505 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:44.983689+0000", 2023-09-23T21:02:29.505 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000104318 2023-09-23T21:02:29.505 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:29.505 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:29.505 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:02:29.506 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:44.983701+0000", 2023-09-23T21:02:29.506 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.1369000000000001e-05 2023-09-23T21:02:29.506 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:29.506 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:29.506 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:02:29.506 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:45.952012+0000", 2023-09-23T21:02:29.506 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.968311335 2023-09-23T21:02:29.506 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:29.506 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:29.506 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:02:29.506 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:45.952141+0000", 2023-09-23T21:02:29.506 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000129121 2023-09-23T21:02:29.507 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:29.507 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:29.507 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 2,6", 2023-09-23T21:02:29.507 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:45.952615+0000", 2023-09-23T21:02:29.507 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00047399399999999999 2023-09-23T21:02:29.507 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:29.507 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:29.507 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:02:29.507 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:45.965254+0000", 2023-09-23T21:02:29.507 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.012638666 2023-09-23T21:02:29.507 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:29.507 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:29.507 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:02:29.508 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:45.992657+0000", 2023-09-23T21:02:29.508 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.027402724999999999 2023-09-23T21:02:29.508 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:29.508 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:29.508 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:02:29.508 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:45.992700+0000", 2023-09-23T21:02:29.508 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.3248999999999999e-05 2023-09-23T21:02:29.508 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:29.508 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:29.508 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:02:29.508 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:45.992736+0000", 2023-09-23T21:02:29.508 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.5775000000000002e-05 2023-09-23T21:02:29.508 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:29.509 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:29.509 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:02:29.509 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:45.992778+0000", 2023-09-23T21:02:29.509 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.2018999999999999e-05 2023-09-23T21:02:29.509 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:02:29.509 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:02:29.509 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:02:29.509 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:29.509 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:29.509 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.34797.0:184039 6.3 6:c6d77d60:::benchmark_data_smithi026_223868_object82913:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e169)", 2023-09-23T21:02:29.509 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T20:55:45.045261+0000", 2023-09-23T21:02:29.509 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 404.34239578299997, 2023-09-23T21:02:29.509 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.94769778800000004, 2023-09-23T21:02:29.509 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:02:29.510 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:02:29.510 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:02:29.510 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.34797", 2023-09-23T21:02:29.510 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T21:02:29.510 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 184039 2023-09-23T21:02:29.510 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:29.510 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:02:29.510 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:29.510 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:02:29.510 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:45.045261+0000", 2023-09-23T21:02:29.510 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:02:29.510 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:29.510 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:29.511 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:02:29.511 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:45.045261+0000", 2023-09-23T21:02:29.511 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:02:29.511 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:29.511 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:29.511 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:02:29.511 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:45.045260+0000", 2023-09-23T21:02:29.511 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999995 2023-09-23T21:02:29.511 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:29.511 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:29.511 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:02:29.511 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:45.045265+0000", 2023-09-23T21:02:29.511 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.9169999999999997e-06 2023-09-23T21:02:29.511 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:29.512 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:29.512 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:02:29.512 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:45.045275+0000", 2023-09-23T21:02:29.512 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.8220000000000002e-06 2023-09-23T21:02:29.512 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:29.512 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:29.512 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:02:29.512 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:45.045280+0000", 2023-09-23T21:02:29.512 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.694e-06 2023-09-23T21:02:29.512 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:29.512 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:29.512 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:02:29.512 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:45.046604+0000", 2023-09-23T21:02:29.513 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0013231950000000001 2023-09-23T21:02:29.513 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:29.513 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:29.513 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:02:29.513 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:45.046612+0000", 2023-09-23T21:02:29.513 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.7099999999999996e-06 2023-09-23T21:02:29.513 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:29.513 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:29.513 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:02:29.513 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:45.954669+0000", 2023-09-23T21:02:29.513 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.90805630100000001 2023-09-23T21:02:29.513 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:29.513 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:29.514 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:02:29.514 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:45.954788+0000", 2023-09-23T21:02:29.514 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000118996 2023-09-23T21:02:29.514 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:29.514 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:29.514 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 2,6", 2023-09-23T21:02:29.514 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:45.955114+0000", 2023-09-23T21:02:29.514 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00032657500000000001 2023-09-23T21:02:29.514 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:29.514 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:29.514 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:02:29.514 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:45.967963+0000", 2023-09-23T21:02:29.515 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.012848607999999999 2023-09-23T21:02:29.515 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:29.515 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:29.515 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:02:29.515 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:45.992828+0000", 2023-09-23T21:02:29.515 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.024865343000000002 2023-09-23T21:02:29.515 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:29.515 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:29.515 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:02:29.515 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:45.992875+0000", 2023-09-23T21:02:29.515 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.6767999999999997e-05 2023-09-23T21:02:29.515 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:29.515 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:29.516 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:02:29.516 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:45.992921+0000", 2023-09-23T21:02:29.516 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.5787000000000001e-05 2023-09-23T21:02:29.516 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:29.516 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:29.516 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:02:29.516 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:45.992959+0000", 2023-09-23T21:02:29.516 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.7725000000000002e-05 2023-09-23T21:02:29.516 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:02:29.516 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:02:29.516 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:02:29.516 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:29.516 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:29.516 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.34797.0:184064 6.3 6:c1be5d02:::benchmark_data_smithi026_223868_object82938:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e169)", 2023-09-23T21:02:29.517 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T20:55:45.083151+0000", 2023-09-23T21:02:29.517 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 404.30450588600002, 2023-09-23T21:02:29.517 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.90993612000000001, 2023-09-23T21:02:29.517 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:02:29.517 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:02:29.517 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:02:29.517 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.34797", 2023-09-23T21:02:29.517 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T21:02:29.517 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 184064 2023-09-23T21:02:29.517 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:29.517 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:02:29.517 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:29.517 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:02:29.518 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:45.083151+0000", 2023-09-23T21:02:29.518 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:02:29.518 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:29.518 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:29.518 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:02:29.518 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:45.083151+0000", 2023-09-23T21:02:29.518 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:02:29.518 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:29.518 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:29.518 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:02:29.518 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:45.083149+0000", 2023-09-23T21:02:29.518 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T21:02:29.518 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:29.519 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:29.519 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:02:29.519 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:45.083157+0000", 2023-09-23T21:02:29.519 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.8669999999999994e-06 2023-09-23T21:02:29.519 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:29.519 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:29.519 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:02:29.519 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:45.083171+0000", 2023-09-23T21:02:29.519 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.3759e-05 2023-09-23T21:02:29.519 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:29.519 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:29.519 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:02:29.519 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:45.083178+0000", 2023-09-23T21:02:29.519 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.1729999999999998e-06 2023-09-23T21:02:29.520 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:29.520 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:29.520 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:02:29.520 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:45.083258+0000", 2023-09-23T21:02:29.520 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.0216000000000006e-05 2023-09-23T21:02:29.520 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:29.520 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:29.520 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:02:29.520 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:45.083268+0000", 2023-09-23T21:02:29.520 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.8670000000000006e-06 2023-09-23T21:02:29.520 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:29.520 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:29.520 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:02:29.520 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:45.957501+0000", 2023-09-23T21:02:29.521 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.87423300400000004 2023-09-23T21:02:29.521 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:29.521 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:29.521 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:02:29.521 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:45.957638+0000", 2023-09-23T21:02:29.521 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000136828 2023-09-23T21:02:29.521 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:29.521 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:29.521 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 2,6", 2023-09-23T21:02:29.521 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:45.958030+0000", 2023-09-23T21:02:29.521 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00039222699999999998 2023-09-23T21:02:29.521 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:29.521 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:29.521 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:02:29.522 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:45.967973+0000", 2023-09-23T21:02:29.522 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0099433109999999998 2023-09-23T21:02:29.522 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:29.522 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:29.522 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:02:29.522 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:45.993006+0000", 2023-09-23T21:02:29.522 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.025032180000000001 2023-09-23T21:02:29.522 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:29.522 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:29.522 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:02:29.522 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:45.993029+0000", 2023-09-23T21:02:29.522 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.3254000000000001e-05 2023-09-23T21:02:29.522 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:29.523 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:29.523 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:02:29.523 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:45.993055+0000", 2023-09-23T21:02:29.523 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.5859000000000001e-05 2023-09-23T21:02:29.523 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:29.523 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:29.523 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:02:29.523 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:45.993087+0000", 2023-09-23T21:02:29.523 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.2115e-05 2023-09-23T21:02:29.523 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:02:29.523 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:02:29.524 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:02:29.524 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:29.524 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:29.524 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.34797.0:184090 6.3 6:c64af2e3:::benchmark_data_smithi026_223868_object82964:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e169)", 2023-09-23T21:02:29.524 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T20:55:45.111638+0000", 2023-09-23T21:02:29.524 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 404.27601830499998, 2023-09-23T21:02:29.524 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.881604795, 2023-09-23T21:02:29.524 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:02:29.524 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:02:29.524 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:02:29.524 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.34797", 2023-09-23T21:02:29.524 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T21:02:29.524 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 184090 2023-09-23T21:02:29.525 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:29.525 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:02:29.525 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:29.525 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:02:29.525 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:45.111638+0000", 2023-09-23T21:02:29.525 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:02:29.525 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:29.525 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:29.525 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:02:29.525 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:45.111638+0000", 2023-09-23T21:02:29.525 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:02:29.525 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:29.525 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:29.525 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:02:29.526 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:45.111637+0000", 2023-09-23T21:02:29.526 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.999999 2023-09-23T21:02:29.526 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:29.526 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:29.526 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:02:29.526 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:45.111642+0000", 2023-09-23T21:02:29.526 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.3210000000000004e-06 2023-09-23T21:02:29.526 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:29.526 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:29.526 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:02:29.526 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:45.111654+0000", 2023-09-23T21:02:29.526 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.149e-05 2023-09-23T21:02:29.527 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:29.527 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:29.527 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:02:29.527 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:45.111659+0000", 2023-09-23T21:02:29.527 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.7280000000000003e-06 2023-09-23T21:02:29.527 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:29.527 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:29.527 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:02:29.527 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:45.111728+0000", 2023-09-23T21:02:29.527 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.9608000000000002e-05 2023-09-23T21:02:29.527 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:29.527 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:29.527 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:02:29.528 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:45.111741+0000", 2023-09-23T21:02:29.528 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.2393e-05 2023-09-23T21:02:29.528 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:29.528 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:29.528 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:02:29.528 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:45.960178+0000", 2023-09-23T21:02:29.528 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.84843696700000004 2023-09-23T21:02:29.528 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:29.528 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:29.528 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:02:29.528 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:45.960306+0000", 2023-09-23T21:02:29.528 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000128641 2023-09-23T21:02:29.528 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:29.528 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:29.529 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 2,6", 2023-09-23T21:02:29.529 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:45.960651+0000", 2023-09-23T21:02:29.529 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00034484499999999998 2023-09-23T21:02:29.529 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:29.529 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:29.529 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:02:29.529 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:45.967982+0000", 2023-09-23T21:02:29.529 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0073310069999999996 2023-09-23T21:02:29.529 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:29.529 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:29.529 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:02:29.529 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:45.993150+0000", 2023-09-23T21:02:29.530 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.025167622000000001 2023-09-23T21:02:29.530 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:29.530 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:29.530 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:02:29.530 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:45.993181+0000", 2023-09-23T21:02:29.530 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.1446999999999999e-05 2023-09-23T21:02:29.530 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:29.530 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:29.530 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:02:29.530 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:45.993206+0000", 2023-09-23T21:02:29.530 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.4564000000000001e-05 2023-09-23T21:02:29.530 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:29.530 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:29.531 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:02:29.531 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:45.993243+0000", 2023-09-23T21:02:29.531 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.7181999999999997e-05 2023-09-23T21:02:29.531 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:02:29.531 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:02:29.531 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:02:29.531 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:29.531 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:29.531 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.34797.0:184092 6.3 6:c0ffe052:::benchmark_data_smithi026_223868_object82966:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e169)", 2023-09-23T21:02:29.531 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T20:55:45.112343+0000", 2023-09-23T21:02:29.531 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 404.27531329099997, 2023-09-23T21:02:29.531 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.88104986100000005, 2023-09-23T21:02:29.531 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:02:29.532 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:02:29.532 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:02:29.532 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.34797", 2023-09-23T21:02:29.532 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T21:02:29.532 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 184092 2023-09-23T21:02:29.532 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:29.532 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:02:29.532 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:29.532 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:02:29.532 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:45.112343+0000", 2023-09-23T21:02:29.532 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:02:29.532 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:29.533 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:29.533 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:02:29.533 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:45.112343+0000", 2023-09-23T21:02:29.533 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:02:29.533 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:29.533 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:29.533 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:02:29.533 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:45.112342+0000", 2023-09-23T21:02:29.533 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.999999 2023-09-23T21:02:29.533 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:29.533 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:29.533 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:02:29.533 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:45.112348+0000", 2023-09-23T21:02:29.534 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.3229999999999997e-06 2023-09-23T21:02:29.534 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:29.534 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:29.534 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:02:29.534 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:45.112364+0000", 2023-09-23T21:02:29.534 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.6351999999999999e-05 2023-09-23T21:02:29.534 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:29.534 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:29.534 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:02:29.534 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:45.112371+0000", 2023-09-23T21:02:29.534 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.8650000000000003e-06 2023-09-23T21:02:29.535 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:29.535 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:29.535 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:02:29.535 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:45.112424+0000", 2023-09-23T21:02:29.535 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.3529000000000002e-05 2023-09-23T21:02:29.535 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:29.535 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:29.535 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:02:29.535 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:45.112433+0000", 2023-09-23T21:02:29.535 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.4100000000000008e-06 2023-09-23T21:02:29.535 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:29.535 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:29.535 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:02:29.536 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:45.962520+0000", 2023-09-23T21:02:29.536 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.85008725299999999 2023-09-23T21:02:29.536 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:29.536 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:29.536 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:02:29.536 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:45.962624+0000", 2023-09-23T21:02:29.536 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000104347 2023-09-23T21:02:29.536 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:29.536 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:29.536 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 2,6", 2023-09-23T21:02:29.536 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:45.962955+0000", 2023-09-23T21:02:29.536 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00033040100000000002 2023-09-23T21:02:29.537 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:29.537 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:29.537 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:02:29.537 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:45.973201+0000", 2023-09-23T21:02:29.537 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.010246151 2023-09-23T21:02:29.537 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:29.537 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:29.537 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:02:29.537 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:45.993284+0000", 2023-09-23T21:02:29.537 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.020082724999999999 2023-09-23T21:02:29.537 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:29.537 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:29.538 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:02:29.538 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:45.993318+0000", 2023-09-23T21:02:29.538 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.3816000000000003e-05 2023-09-23T21:02:29.538 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:29.538 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:29.538 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:02:29.538 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:45.993360+0000", 2023-09-23T21:02:29.538 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.2348000000000002e-05 2023-09-23T21:02:29.538 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:29.538 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:29.538 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:02:29.538 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:45.993393+0000", 2023-09-23T21:02:29.538 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.3204999999999998e-05 2023-09-23T21:02:29.539 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:02:29.539 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:02:29.539 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:02:29.539 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:29.539 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:29.539 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.34797.0:184168 6.3 6:c7a50969:::benchmark_data_smithi026_223868_object83042:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e169)", 2023-09-23T21:02:29.539 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T20:55:45.196004+0000", 2023-09-23T21:02:29.539 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 404.19165247699999, 2023-09-23T21:02:29.539 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.79750679000000002, 2023-09-23T21:02:29.539 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:02:29.539 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:02:29.539 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:02:29.540 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.34797", 2023-09-23T21:02:29.540 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T21:02:29.540 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 184168 2023-09-23T21:02:29.540 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:29.540 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:02:29.540 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:29.540 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:02:29.540 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:45.196004+0000", 2023-09-23T21:02:29.540 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:02:29.540 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:29.540 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:29.540 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:02:29.540 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:45.196004+0000", 2023-09-23T21:02:29.541 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:02:29.541 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:29.541 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:29.541 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:02:29.541 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:45.196002+0000", 2023-09-23T21:02:29.541 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T21:02:29.541 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:29.541 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:29.541 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:02:29.541 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:45.196010+0000", 2023-09-23T21:02:29.541 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.289e-06 2023-09-23T21:02:29.541 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:29.541 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:29.541 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:02:29.542 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:45.196025+0000", 2023-09-23T21:02:29.542 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.5281000000000002e-05 2023-09-23T21:02:29.542 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:29.542 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:29.542 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:02:29.542 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:45.196033+0000", 2023-09-23T21:02:29.542 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.447e-06 2023-09-23T21:02:29.542 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:29.542 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:29.542 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:02:29.542 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:45.196083+0000", 2023-09-23T21:02:29.542 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.0841999999999999e-05 2023-09-23T21:02:29.542 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:29.543 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:29.543 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:02:29.543 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:45.196091+0000", 2023-09-23T21:02:29.543 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.8010000000000004e-06 2023-09-23T21:02:29.543 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:29.543 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:29.543 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:02:29.543 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:45.965291+0000", 2023-09-23T21:02:29.543 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.76920029499999998 2023-09-23T21:02:29.543 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:29.543 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:29.543 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:02:29.544 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:45.965400+0000", 2023-09-23T21:02:29.544 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00010818400000000001 2023-09-23T21:02:29.544 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:29.544 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:29.544 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 2,6", 2023-09-23T21:02:29.544 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:45.965675+0000", 2023-09-23T21:02:29.544 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00027548699999999999 2023-09-23T21:02:29.544 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:29.544 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:29.544 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:02:29.544 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:45.977533+0000", 2023-09-23T21:02:29.544 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.011858172 2023-09-23T21:02:29.544 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:29.545 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:29.545 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:02:29.545 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:45.993435+0000", 2023-09-23T21:02:29.545 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.015902143000000001 2023-09-23T21:02:29.545 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:29.545 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:29.545 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:02:29.545 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:45.993455+0000", 2023-09-23T21:02:29.545 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.9777000000000001e-05 2023-09-23T21:02:29.545 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:29.545 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:29.545 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:02:29.545 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:45.993481+0000", 2023-09-23T21:02:29.546 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.6021e-05 2023-09-23T21:02:29.546 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:29.546 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:29.546 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:02:29.546 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:45.993511+0000", 2023-09-23T21:02:29.546 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.9578e-05 2023-09-23T21:02:29.546 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:02:29.546 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:02:29.546 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:02:29.546 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:29.546 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:29.546 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.34797.0:184197 6.3 6:c34c1e61:::benchmark_data_smithi026_223868_object83071:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e169)", 2023-09-23T21:02:29.546 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T20:55:45.233107+0000", 2023-09-23T21:02:29.547 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 404.15454925099999, 2023-09-23T21:02:29.547 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.760519681, 2023-09-23T21:02:29.547 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:02:29.547 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:02:29.547 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:02:29.547 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.34797", 2023-09-23T21:02:29.547 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T21:02:29.547 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 184197 2023-09-23T21:02:29.547 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:29.547 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:02:29.547 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:29.547 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:02:29.547 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:45.233107+0000", 2023-09-23T21:02:29.547 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:02:29.548 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:29.548 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:29.548 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:02:29.548 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:45.233107+0000", 2023-09-23T21:02:29.548 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:02:29.548 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:29.548 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:29.548 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:02:29.548 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:45.233105+0000", 2023-09-23T21:02:29.548 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999981 2023-09-23T21:02:29.548 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:29.548 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:29.548 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:02:29.549 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:45.233113+0000", 2023-09-23T21:02:29.549 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.4240000000000004e-06 2023-09-23T21:02:29.549 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:29.549 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:29.549 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:02:29.549 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:45.233130+0000", 2023-09-23T21:02:29.549 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.6966999999999999e-05 2023-09-23T21:02:29.549 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:29.549 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:29.549 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:02:29.549 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:45.233137+0000", 2023-09-23T21:02:29.549 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.8730000000000001e-06 2023-09-23T21:02:29.549 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:29.549 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:29.550 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:02:29.550 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:45.233204+0000", 2023-09-23T21:02:29.550 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.7105000000000002e-05 2023-09-23T21:02:29.550 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:29.550 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:29.550 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:02:29.550 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:45.233215+0000", 2023-09-23T21:02:29.550 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.1371e-05 2023-09-23T21:02:29.550 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:29.550 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:29.550 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:02:29.550 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:45.968007+0000", 2023-09-23T21:02:29.550 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.73479188900000003 2023-09-23T21:02:29.550 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:29.551 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:29.551 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:02:29.551 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:45.968089+0000", 2023-09-23T21:02:29.551 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.1660000000000001e-05 2023-09-23T21:02:29.551 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:29.551 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:29.551 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 2,6", 2023-09-23T21:02:29.551 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:45.968445+0000", 2023-09-23T21:02:29.551 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00035676499999999999 2023-09-23T21:02:29.551 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:29.551 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:29.551 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:02:29.551 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:45.977542+0000", 2023-09-23T21:02:29.552 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0090968160000000006 2023-09-23T21:02:29.552 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:29.552 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:29.552 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:02:29.552 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:45.993553+0000", 2023-09-23T21:02:29.552 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.016010277 2023-09-23T21:02:29.552 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:29.552 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:29.552 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:02:29.552 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:45.993573+0000", 2023-09-23T21:02:29.552 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.0268e-05 2023-09-23T21:02:29.552 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:29.552 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:29.552 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:02:29.553 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:45.993597+0000", 2023-09-23T21:02:29.553 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.3978e-05 2023-09-23T21:02:29.553 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:29.553 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:29.553 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:02:29.553 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:45.993627+0000", 2023-09-23T21:02:29.553 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.0122999999999999e-05 2023-09-23T21:02:29.553 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:02:29.553 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:02:29.553 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:02:29.553 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:02:29.553 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:02:29.553 INFO:teuthology.orchestra.run.smithi070.stdout:} 2023-09-23T21:02:29.554 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:02:29.555 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:02:29.555 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:02:29.613 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:02:29 smithi026 ceph-mon[179688]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:02:29.613 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:02:29 smithi026 ceph-mon[179688]: failed to encode map e270 with expected crc 2023-09-23T21:02:29.614 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:02:29 smithi026 ceph-mon[179688]: failed to encode map e270 with expected crc 2023-09-23T21:02:29.614 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:02:29 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T21:02:29.614 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:02:29 smithi026 ceph-mon[179688]: osdmap e270: 8 total, 8 up, 8 in 2023-09-23T21:02:29.614 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:02:29 smithi026 ceph-mon[179688]: failed to encode map e270 with expected crc 2023-09-23T21:02:29.614 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:02:29 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T21:02:29.615 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 21:02:29 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[164029]: ::ffff:172.21.15.70 - - [23/Sep/2023:21:02:29] "GET /metrics HTTP/1.1" 200 36293 "" "Prometheus/2.43.0" 2023-09-23T21:02:29.615 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:02:29 smithi026 ceph-mon[175151]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:02:29.615 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:02:29 smithi026 ceph-mon[175151]: failed to encode map e270 with expected crc 2023-09-23T21:02:29.615 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:02:29 smithi026 ceph-mon[175151]: failed to encode map e270 with expected crc 2023-09-23T21:02:29.615 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:02:29 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T21:02:29.615 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:02:29 smithi026 ceph-mon[175151]: osdmap e270: 8 total, 8 up, 8 in 2023-09-23T21:02:29.615 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:02:29 smithi026 ceph-mon[175151]: failed to encode map e270 with expected crc 2023-09-23T21:02:29.615 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:02:29 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T21:02:29.656 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:02:29.759 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 58 16 84600 84584 91.0979 51.5 0.0246582 0.0109719 2023-09-23T21:02:29.766 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:02:29.845 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_ops_in_flight 2023-09-23T21:02:29.850 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:02:29.856 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:02:29.857 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:02:29.957 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:02:30.026 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:02:30.037 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:02:30.191 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T21:02:30.195 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:02:30.196 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:02:30.296 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:02:30.371 INFO:teuthology.orchestra.run.smithi026.stderr:reweighted osd.7 to 1 (10000) 2023-09-23T21:02:30.388 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:02:30.584 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:02:30.588 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:02:30.589 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:02:30.589 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:02:30 smithi026 ceph-mon[179688]: pgmap v710: 109 pgs: 109 active+clean; 4.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 57 MiB/s wr, 913 op/s 2023-09-23T21:02:30.589 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:02:30 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/2721777720' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-09-23T21:02:30.589 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:02:30 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-09-23T21:02:30.589 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:02:30 smithi026 ceph-mon[175151]: pgmap v710: 109 pgs: 109 active+clean; 4.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 57 MiB/s wr, 913 op/s 2023-09-23T21:02:30.589 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:02:30 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/2721777720' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-09-23T21:02:30.590 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:02:30 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-09-23T21:02:30.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:02:30 smithi070 ceph-mon[168282]: pgmap v710: 109 pgs: 109 active+clean; 4.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 57 MiB/s wr, 913 op/s 2023-09-23T21:02:30.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:02:30 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/2721777720' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-09-23T21:02:30.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:02:30 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-09-23T21:02:30.645 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T21:02:30.689 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:02:30.759 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 59 16 85501 85485 90.5082 56.3125 0.0119642 0.0110433 2023-09-23T21:02:30.769 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:02:31.086 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T21:02:31.189 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph osd reweight 4 1 2023-09-23T21:02:31.286 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:02:31.299 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:02:31.300 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:02:31.400 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:02:31.475 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:02:31 smithi026 ceph-mon[179688]: failed to encode map e271 with expected crc 2023-09-23T21:02:31.475 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:02:31 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-09-23T21:02:31.475 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:02:31 smithi026 ceph-mon[179688]: osdmap e271: 8 total, 8 up, 8 in 2023-09-23T21:02:31.475 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:02:31 smithi026 ceph-mon[179688]: failed to encode map e271 with expected crc 2023-09-23T21:02:31.475 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:02:31 smithi026 ceph-mon[179688]: failed to encode map e271 with expected crc 2023-09-23T21:02:31.475 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:02:31 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:02:31.480 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:02:31 smithi026 ceph-mon[175151]: failed to encode map e271 with expected crc 2023-09-23T21:02:31.480 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:02:31 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-09-23T21:02:31.480 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:02:31 smithi026 ceph-mon[175151]: osdmap e271: 8 total, 8 up, 8 in 2023-09-23T21:02:31.480 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:02:31 smithi026 ceph-mon[175151]: failed to encode map e271 with expected crc 2023-09-23T21:02:31.480 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:02:31 smithi026 ceph-mon[175151]: failed to encode map e271 with expected crc 2023-09-23T21:02:31.480 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:02:31 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:02:31.486 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:02:31.567 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:02:31.576 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:02:31.577 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:02:31.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:02:31 smithi070 ceph-mon[168282]: failed to encode map e271 with expected crc 2023-09-23T21:02:31.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:02:31 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-09-23T21:02:31.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:02:31 smithi070 ceph-mon[168282]: osdmap e271: 8 total, 8 up, 8 in 2023-09-23T21:02:31.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:02:31 smithi070 ceph-mon[168282]: failed to encode map e271 with expected crc 2023-09-23T21:02:31.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:02:31 smithi070 ceph-mon[168282]: failed to encode map e271 with expected crc 2023-09-23T21:02:31.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:02:31 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:02:31.677 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:02:31.760 INFO:tasks.radosbench.radosbench.0.smithi026.stdout:2023-09-23T21:02:31.760422+0000 min lat: 0.00122595 max lat: 0.190811 avg lat: 0.0111241 2023-09-23T21:02:31.760 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T21:02:31.760 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 60 16 86322 86306 89.8547 51.3125 0.0124014 0.0111241 2023-09-23T21:02:31.780 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:02:31.945 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:02:31.949 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:02:31.950 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:02:32.050 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:02:32.069 INFO:teuthology.orchestra.run.smithi070.stdout:{ 2023-09-23T21:02:32.069 INFO:teuthology.orchestra.run.smithi070.stdout: "ops": [ 2023-09-23T21:02:32.070 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:32.070 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.35469.0:86573 8.9 8:96d28ad7:::benchmark_data_smithi026_301269_object86572:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e271)", 2023-09-23T21:02:32.070 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:02:32.058863+0000", 2023-09-23T21:02:32.070 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 0.010756442999999999, 2023-09-23T21:02:32.070 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.010780971, 2023-09-23T21:02:32.070 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:02:32.070 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "waiting for sub ops", 2023-09-23T21:02:32.070 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:02:32.070 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.35469", 2023-09-23T21:02:32.070 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/4041424623", 2023-09-23T21:02:32.070 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 86573 2023-09-23T21:02:32.071 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:32.071 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:02:32.071 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:32.071 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:02:32.071 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:02:32.058863+0000", 2023-09-23T21:02:32.071 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:02:32.071 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:32.071 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:32.071 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:02:32.071 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:02:32.058863+0000", 2023-09-23T21:02:32.071 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:02:32.071 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:32.072 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:32.072 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:02:32.072 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:02:32.058862+0000", 2023-09-23T21:02:32.072 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T21:02:32.072 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:32.072 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:32.072 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:02:32.072 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:02:32.058914+0000", 2023-09-23T21:02:32.072 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.2379000000000001e-05 2023-09-23T21:02:32.072 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:32.072 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:32.072 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:02:32.073 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:02:32.058928+0000", 2023-09-23T21:02:32.073 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.4158999999999999e-05 2023-09-23T21:02:32.073 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:32.073 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:32.073 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:02:32.073 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:02:32.058935+0000", 2023-09-23T21:02:32.073 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.7730000000000002e-06 2023-09-23T21:02:32.073 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:32.073 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:32.073 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:02:32.073 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:02:32.058972+0000", 2023-09-23T21:02:32.073 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.6677999999999997e-05 2023-09-23T21:02:32.073 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:32.074 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:32.074 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:02:32.074 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:02:32.059080+0000", 2023-09-23T21:02:32.074 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00010826400000000001 2023-09-23T21:02:32.074 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:32.074 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:32.074 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 0,5", 2023-09-23T21:02:32.074 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:02:32.059179+0000", 2023-09-23T21:02:32.074 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.8777999999999995e-05 2023-09-23T21:02:32.074 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:32.074 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:32.074 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:02:32.075 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:02:32.061652+0000", 2023-09-23T21:02:32.075 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0024735460000000001 2023-09-23T21:02:32.075 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:32.075 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:32.075 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:02:32.075 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:02:32.062716+0000", 2023-09-23T21:02:32.075 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.001063858 2023-09-23T21:02:32.075 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:02:32.075 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:02:32.075 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:02:32.075 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:32.075 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:32.076 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_repop(client.35469.0:86571 8.3 e271/254 8:cb0295c6:::benchmark_data_smithi026_301269_object86570:head v 271'5258, mlcod=271'5256)", 2023-09-23T21:02:32.076 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:02:32.066598+0000", 2023-09-23T21:02:32.076 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 0.003021391, 2023-09-23T21:02:32.076 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0031457239999999999, 2023-09-23T21:02:32.076 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:02:32.076 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:02:32.076 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:02:32.076 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:32.076 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:02:32.076 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:02:32.066598+0000", 2023-09-23T21:02:32.076 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:02:32.076 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:32.076 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:32.077 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:02:32.077 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:02:32.066598+0000", 2023-09-23T21:02:32.077 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:02:32.077 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:32.077 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:32.077 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:02:32.077 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:02:32.066597+0000", 2023-09-23T21:02:32.077 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.999999 2023-09-23T21:02:32.077 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:32.077 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:32.077 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:02:32.077 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:02:32.066646+0000", 2023-09-23T21:02:32.078 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.8415999999999999e-05 2023-09-23T21:02:32.078 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:32.078 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:32.078 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:02:32.078 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:02:32.066662+0000", 2023-09-23T21:02:32.078 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.6528000000000002e-05 2023-09-23T21:02:32.078 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:32.078 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:32.078 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:02:32.078 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:02:32.066669+0000", 2023-09-23T21:02:32.078 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.4189999999999999e-06 2023-09-23T21:02:32.078 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:32.079 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:32.079 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:02:32.079 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:02:32.066717+0000", 2023-09-23T21:02:32.079 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.8569999999999997e-05 2023-09-23T21:02:32.079 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:32.079 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:32.079 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:02:32.079 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:02:32.066744+0000", 2023-09-23T21:02:32.079 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.6843e-05 2023-09-23T21:02:32.079 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:32.079 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:32.079 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:02:32.079 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:02:32.069703+0000", 2023-09-23T21:02:32.080 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0029584580000000002 2023-09-23T21:02:32.080 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:32.080 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:32.080 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:02:32.080 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:02:32.069744+0000", 2023-09-23T21:02:32.080 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.1356000000000002e-05 2023-09-23T21:02:32.080 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:02:32.080 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:02:32.080 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:02:32.080 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:02:32.080 INFO:teuthology.orchestra.run.smithi070.stdout: ], 2023-09-23T21:02:32.080 INFO:teuthology.orchestra.run.smithi070.stdout: "num_ops": 2 2023-09-23T21:02:32.080 INFO:teuthology.orchestra.run.smithi070.stdout:} 2023-09-23T21:02:32.145 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:02:32.321 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:02:32.323 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:02:32.336 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:02:32.339 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:02:32.339 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:02:32.431 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0: {} 2023-09-23T21:02:32.440 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:02:32.461 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_blocked_ops 2023-09-23T21:02:32.483 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1: {} 2023-09-23T21:02:32.494 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:02:32.529 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2: {} 2023-09-23T21:02:32.563 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3: {} 2023-09-23T21:02:32.605 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:02:32.610 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4: {} 2023-09-23T21:02:32.612 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:02:32.613 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:02:32.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:02:32 smithi026 ceph-mon[175151]: pgmap v712: 109 pgs: 109 active+clean; 5.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 61 MiB/s wr, 975 op/s 2023-09-23T21:02:32.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:02:32 smithi026 ceph-mon[179688]: pgmap v712: 109 pgs: 109 active+clean; 5.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 61 MiB/s wr, 975 op/s 2023-09-23T21:02:32.660 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T21:02:32.702 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6: {} 2023-09-23T21:02:32.713 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:02:32.741 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T21:02:32.760 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 61 16 87136 87120 89.2159 50.875 0.0128126 0.0112033 2023-09-23T21:02:32.762 INFO:teuthology.orchestra.run.smithi026.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T21:02:32.762 INFO:teuthology.orchestra.run.smithi026.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T21:02:32.762 INFO:teuthology.orchestra.run.smithi026.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T21:02:32.762 INFO:teuthology.orchestra.run.smithi026.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T21:02:32.762 INFO:teuthology.orchestra.run.smithi026.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T21:02:32.762 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T21:02:32.762 INFO:teuthology.orchestra.run.smithi026.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T21:02:32.762 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T21:02:32.763 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:02:32 smithi070 ceph-mon[168282]: pgmap v712: 109 pgs: 109 active+clean; 5.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 61 MiB/s wr, 975 op/s 2023-09-23T21:02:32.768 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:02:32.850 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:02:32.854 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:02:32.854 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:02:32.955 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:02:33.019 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:02:33.083 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:02:33.086 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:02:33.087 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:02:33.188 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:02:33.300 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T21:02:33.303 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:02:33.570 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:02:33 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/1574950903' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:02:33.570 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:02:33 smithi026 ceph-mon[175151]: pgmap v713: 109 pgs: 109 active+clean; 5.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 61 MiB/s wr, 971 op/s 2023-09-23T21:02:33.570 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:02:33 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/1574950903' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:02:33.570 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:02:33 smithi026 ceph-mon[179688]: pgmap v713: 109 pgs: 109 active+clean; 5.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 61 MiB/s wr, 971 op/s 2023-09-23T21:02:33.599 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:02:33.603 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:02:33.603 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:02:33.705 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:02:33.761 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 62 16 88043 88027 88.69 56.6875 0.00479344 0.0112689 2023-09-23T21:02:33.765 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:02:33.780 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:02:33 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/1574950903' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:02:33.780 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:02:33 smithi070 ceph-mon[168282]: pgmap v713: 109 pgs: 109 active+clean; 5.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 61 MiB/s wr, 971 op/s 2023-09-23T21:02:34.373 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:02:34.377 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:02:34.377 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:02:34.478 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:02:34.557 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:02:34.659 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:02:34.665 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:02:34.666 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:02:34.761 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 63 16 88982 88966 88.2139 58.6875 0.0116104 0.011331 2023-09-23T21:02:34.766 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:02:34.859 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:02:35.039 INFO:teuthology.orchestra.run.smithi070.stdout:ERROR: (22) Invalid argument 2023-09-23T21:02:35.039 INFO:teuthology.orchestra.run.smithi070.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T21:02:35.046 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:02:35.052 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:02:35.053 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:02:35.153 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:02:35.195 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:02:35.699 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T21:02:35.763 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 64 16 89800 89784 87.6319 51.125 0.00491744 0.011406 2023-09-23T21:02:36.005 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:02:36.010 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:02:36.011 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:02:36.111 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:02:36.153 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:02:36.170 INFO:teuthology.orchestra.run.smithi026.stderr:nodeep-scrub is unset 2023-09-23T21:02:36.170 INFO:teuthology.orchestra.run.smithi026.stderr:reweighted osd.4 to 1 (10000) 2023-09-23T21:02:36.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:02:36 smithi070 ceph-mon[168282]: pgmap v714: 109 pgs: 109 active+clean; 5.3 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 67 MiB/s wr, 1.06k op/s 2023-09-23T21:02:36.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:02:36 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/3868241569' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-09-23T21:02:36.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:02:36 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/3594779558' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:02:36.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:02:36 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:02:36.479 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:02:36 smithi026 ceph-mon[175151]: pgmap v714: 109 pgs: 109 active+clean; 5.3 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 67 MiB/s wr, 1.06k op/s 2023-09-23T21:02:36.479 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:02:36 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/3868241569' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-09-23T21:02:36.480 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:02:36 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/3594779558' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:02:36.480 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:02:36 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:02:36.480 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:02:36 smithi026 ceph-mon[179688]: pgmap v714: 109 pgs: 109 active+clean; 5.3 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 67 MiB/s wr, 1.06k op/s 2023-09-23T21:02:36.480 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:02:36 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/3868241569' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-09-23T21:02:36.480 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:02:36 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/3594779558' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:02:36.480 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:02:36 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:02:36.764 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 65 16 90680 90664 87.1283 55 0.0258453 0.0114725 2023-09-23T21:02:36.900 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph osd reweight 5 1 2023-09-23T21:02:37.195 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:02:37 smithi026 ceph-mon[175151]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T21:02:37.195 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:02:37 smithi026 ceph-mon[175151]: Cluster is now healthy 2023-09-23T21:02:37.195 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:02:37 smithi026 ceph-mon[175151]: failed to encode map e272 with expected crc 2023-09-23T21:02:37.195 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:02:37 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/3868241569' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-09-23T21:02:37.195 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:02:37 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T21:02:37.195 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:02:37 smithi026 ceph-mon[175151]: osdmap e272: 8 total, 8 up, 8 in 2023-09-23T21:02:37.196 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:02:37 smithi026 ceph-mon[175151]: failed to encode map e272 with expected crc 2023-09-23T21:02:37.196 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:02:37 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:02:37.196 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:02:37 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:02:37.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:02:37 smithi026 ceph-mon[179688]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T21:02:37.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:02:37 smithi026 ceph-mon[179688]: Cluster is now healthy 2023-09-23T21:02:37.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:02:37 smithi026 ceph-mon[179688]: failed to encode map e272 with expected crc 2023-09-23T21:02:37.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:02:37 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/3868241569' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-09-23T21:02:37.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:02:37 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T21:02:37.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:02:37 smithi026 ceph-mon[179688]: osdmap e272: 8 total, 8 up, 8 in 2023-09-23T21:02:37.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:02:37 smithi026 ceph-mon[179688]: failed to encode map e272 with expected crc 2023-09-23T21:02:37.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:02:37 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:02:37.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:02:37 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:02:37.316 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:02:37.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:02:37 smithi070 ceph-mon[168282]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T21:02:37.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:02:37 smithi070 ceph-mon[168282]: Cluster is now healthy 2023-09-23T21:02:37.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:02:37 smithi070 ceph-mon[168282]: failed to encode map e272 with expected crc 2023-09-23T21:02:37.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:02:37 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/3868241569' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-09-23T21:02:37.621 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:02:37 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T21:02:37.621 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:02:37 smithi070 ceph-mon[168282]: osdmap e272: 8 total, 8 up, 8 in 2023-09-23T21:02:37.621 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:02:37 smithi070 ceph-mon[168282]: failed to encode map e272 with expected crc 2023-09-23T21:02:37.621 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:02:37 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:02:37.621 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:02:37 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:02:37.768 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 66 15 91503 91488 86.5835 51.5 0.0123751 0.011545 2023-09-23T21:02:37.822 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:02:38.021 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:02:38.024 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:02:38.024 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:02:38.125 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:02:38.129 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_historic_ops 2023-09-23T21:02:38.183 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:02:38.261 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:02:38.265 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:02:38 smithi026 ceph-mon[175151]: pgmap v716: 109 pgs: 109 active+clean; 5.4 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 55 MiB/s wr, 885 op/s 2023-09-23T21:02:38.266 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:02:38 smithi026 ceph-mon[179688]: pgmap v716: 109 pgs: 109 active+clean; 5.4 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 55 MiB/s wr, 885 op/s 2023-09-23T21:02:38.266 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:02:38.267 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:02:38.367 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:02:38.375 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:02:38 smithi070 ceph-mon[168282]: pgmap v716: 109 pgs: 109 active+clean; 5.4 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 55 MiB/s wr, 885 op/s 2023-09-23T21:02:38.405 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:02:38.566 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:02:38.572 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:02:38.573 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:02:38.673 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:02:38.764 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:02:38.768 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 67 16 92423 92407 86.1487 57.4375 0.0413552 0.011603 2023-09-23T21:02:38.832 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:02:38.835 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:02:38.836 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:02:38.936 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:02:39.004 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0: {} 2023-09-23T21:02:39.011 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:02:39.020 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T21:02:39.036 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1: {} 2023-09-23T21:02:39.074 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2: {} 2023-09-23T21:02:39.110 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3: {} 2023-09-23T21:02:39.147 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4: {} 2023-09-23T21:02:39.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:02:39 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/2568563228' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:02:39.198 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:02:39 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/2568563228' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:02:39.198 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T21:02:39.227 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:02:39.229 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T21:02:39.235 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:02:39.235 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:02:39.243 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6: {} 2023-09-23T21:02:39.289 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T21:02:39.299 INFO:teuthology.orchestra.run.smithi026.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T21:02:39.299 INFO:teuthology.orchestra.run.smithi026.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T21:02:39.299 INFO:teuthology.orchestra.run.smithi026.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T21:02:39.299 INFO:teuthology.orchestra.run.smithi026.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T21:02:39.300 INFO:teuthology.orchestra.run.smithi026.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T21:02:39.300 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T21:02:39.300 INFO:teuthology.orchestra.run.smithi026.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T21:02:39.300 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T21:02:39.336 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:02:39.357 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:02:39 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/2568563228' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:02:39.457 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:02:39.557 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:02:39.563 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:02:39.564 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:02:39.664 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:02:39.742 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:02:39.775 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 68 16 93250 93234 85.6367 51.6875 0.0135896 0.0116722 2023-09-23T21:02:39.884 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 21:02:39 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[164029]: ::ffff:172.21.15.70 - - [23/Sep/2023:21:02:39] "GET /metrics HTTP/1.1" 200 36292 "" "Prometheus/2.43.0" 2023-09-23T21:02:39.972 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:02:39.980 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:02:39.981 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:02:40.081 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:02:40.127 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:02:40.182 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:02:40.190 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:02:40.191 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:02:40.250 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:02:40.259 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:02:40 smithi070 ceph-mon[168282]: pgmap v717: 109 pgs: 109 active+clean; 5.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 48 MiB/s wr, 765 op/s 2023-09-23T21:02:40.260 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:02:40 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:02:40.260 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:02:40 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:02:40.292 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:02:40.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:02:40 smithi026 ceph-mon[175151]: pgmap v717: 109 pgs: 109 active+clean; 5.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 48 MiB/s wr, 765 op/s 2023-09-23T21:02:40.383 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:02:40 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:02:40.383 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:02:40 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:02:40.383 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:02:40 smithi026 ceph-mon[179688]: pgmap v717: 109 pgs: 109 active+clean; 5.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 48 MiB/s wr, 765 op/s 2023-09-23T21:02:40.383 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:02:40 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:02:40.383 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:02:40 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:02:40.398 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:02:40.518 INFO:teuthology.orchestra.run.smithi070.stdout:{ 2023-09-23T21:02:40.518 INFO:teuthology.orchestra.run.smithi070.stdout: "size": 20, 2023-09-23T21:02:40.518 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 600, 2023-09-23T21:02:40.518 INFO:teuthology.orchestra.run.smithi070.stdout: "ops": [ 2023-09-23T21:02:40.519 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:40.519 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.35100.0:95752 7.c 7:320aef38:::benchmark_data_smithi026_260489_object95751:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e229)", 2023-09-23T21:02:40.519 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T20:57:59.846564+0000", 2023-09-23T21:02:40.519 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 280.66892843699998, 2023-09-23T21:02:40.519 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.089005447000000001, 2023-09-23T21:02:40.519 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:02:40.519 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:02:40.519 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:02:40.519 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.35100", 2023-09-23T21:02:40.519 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/4108972961", 2023-09-23T21:02:40.519 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 95752 2023-09-23T21:02:40.520 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:40.520 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:02:40.520 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:40.520 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:02:40.520 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:57:59.846564+0000", 2023-09-23T21:02:40.520 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:02:40.520 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:40.520 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:40.520 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:02:40.520 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:57:59.846564+0000", 2023-09-23T21:02:40.521 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:02:40.521 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:40.521 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:40.521 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:02:40.521 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:57:59.846562+0000", 2023-09-23T21:02:40.521 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T21:02:40.521 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:40.521 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:40.521 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:02:40.521 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:57:59.846603+0000", 2023-09-23T21:02:40.521 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.0246000000000001e-05 2023-09-23T21:02:40.521 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:40.522 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:40.522 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:02:40.522 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:57:59.846617+0000", 2023-09-23T21:02:40.522 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.4535999999999999e-05 2023-09-23T21:02:40.522 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:40.522 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:40.522 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:02:40.522 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:57:59.846624+0000", 2023-09-23T21:02:40.522 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.9099999999999999e-06 2023-09-23T21:02:40.522 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:40.522 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:40.523 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:02:40.523 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:57:59.848264+0000", 2023-09-23T21:02:40.523 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0016401510000000001 2023-09-23T21:02:40.523 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:40.523 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:40.523 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:02:40.523 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:57:59.848461+0000", 2023-09-23T21:02:40.523 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00019658300000000001 2023-09-23T21:02:40.523 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:40.523 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:40.523 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 3,7", 2023-09-23T21:02:40.523 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:57:59.848655+0000", 2023-09-23T21:02:40.524 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000193857 2023-09-23T21:02:40.524 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:40.524 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:40.524 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:02:40.524 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:57:59.854332+0000", 2023-09-23T21:02:40.524 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0056769489999999997 2023-09-23T21:02:40.524 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:40.524 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:40.524 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:02:40.524 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:57:59.854365+0000", 2023-09-23T21:02:40.524 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.3426000000000002e-05 2023-09-23T21:02:40.525 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:40.525 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:40.525 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:02:40.525 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:57:59.931644+0000", 2023-09-23T21:02:40.525 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.077279129000000002 2023-09-23T21:02:40.525 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:40.525 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:40.525 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:02:40.525 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:57:59.931714+0000", 2023-09-23T21:02:40.525 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.9828999999999998e-05 2023-09-23T21:02:40.525 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:40.525 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:40.526 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:02:40.526 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:57:59.935569+0000", 2023-09-23T21:02:40.526 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0038553820000000001 2023-09-23T21:02:40.526 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:02:40.526 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:02:40.526 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:02:40.526 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:40.526 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:40.526 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.35100.0:95754 7.c 7:3d89f732:::benchmark_data_smithi026_260489_object95753:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e229)", 2023-09-23T21:02:40.527 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T20:57:59.846639+0000", 2023-09-23T21:02:40.527 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 280.66885336799999, 2023-09-23T21:02:40.527 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.092545945000000004, 2023-09-23T21:02:40.527 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:02:40.527 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:02:40.527 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:02:40.527 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.35100", 2023-09-23T21:02:40.527 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/4108972961", 2023-09-23T21:02:40.527 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 95754 2023-09-23T21:02:40.527 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:40.527 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:02:40.528 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:40.528 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:02:40.528 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:57:59.846639+0000", 2023-09-23T21:02:40.528 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:02:40.528 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:40.528 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:40.528 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:02:40.528 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:57:59.846639+0000", 2023-09-23T21:02:40.528 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:02:40.528 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:40.528 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:40.528 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:02:40.529 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:57:59.846639+0000", 2023-09-23T21:02:40.529 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999995 2023-09-23T21:02:40.529 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:40.529 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:40.529 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:02:40.529 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:57:59.846668+0000", 2023-09-23T21:02:40.529 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.9835000000000001e-05 2023-09-23T21:02:40.529 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:40.529 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:40.529 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:02:40.529 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:57:59.846680+0000", 2023-09-23T21:02:40.529 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.1327e-05 2023-09-23T21:02:40.530 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:40.530 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:40.530 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:02:40.530 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:57:59.846685+0000", 2023-09-23T21:02:40.530 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.9960000000000001e-06 2023-09-23T21:02:40.530 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:40.530 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:40.530 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:02:40.530 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:57:59.851222+0000", 2023-09-23T21:02:40.530 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0045377059999999999 2023-09-23T21:02:40.530 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:40.530 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:40.531 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:02:40.531 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:57:59.851417+0000", 2023-09-23T21:02:40.531 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000194129 2023-09-23T21:02:40.531 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:40.531 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:40.531 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 3,7", 2023-09-23T21:02:40.531 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:57:59.851613+0000", 2023-09-23T21:02:40.531 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00019676000000000001 2023-09-23T21:02:40.531 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:40.531 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:40.531 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:02:40.531 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:57:59.854360+0000", 2023-09-23T21:02:40.532 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0027465409999999999 2023-09-23T21:02:40.532 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:40.532 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:40.532 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:02:40.532 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:57:59.854410+0000", 2023-09-23T21:02:40.532 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.0345e-05 2023-09-23T21:02:40.532 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:40.532 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:40.532 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:02:40.532 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:57:59.935640+0000", 2023-09-23T21:02:40.532 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.081229928000000007 2023-09-23T21:02:40.532 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:40.533 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:40.533 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:02:40.533 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:57:59.935705+0000", 2023-09-23T21:02:40.533 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.4956000000000006e-05 2023-09-23T21:02:40.533 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:40.533 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:40.533 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:02:40.533 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:57:59.939185+0000", 2023-09-23T21:02:40.533 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.003479883 2023-09-23T21:02:40.533 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:02:40.533 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:02:40.533 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:02:40.534 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:40.534 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:40.534 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.35100.0:116411 7.c 7:386f85a2:::benchmark_data_smithi026_260489_object14649:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e234)", 2023-09-23T21:02:40.534 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T20:58:32.707360+0000", 2023-09-23T21:02:40.534 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 247.808131952, 2023-09-23T21:02:40.534 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.092441081999999994, 2023-09-23T21:02:40.534 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:02:40.534 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:02:40.534 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:02:40.534 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.35100", 2023-09-23T21:02:40.534 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/4108972961", 2023-09-23T21:02:40.535 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 116411 2023-09-23T21:02:40.535 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:40.535 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:02:40.535 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:40.535 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:02:40.535 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:58:32.707360+0000", 2023-09-23T21:02:40.535 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:02:40.535 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:40.535 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:40.535 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:02:40.535 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:58:32.707360+0000", 2023-09-23T21:02:40.536 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:02:40.536 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:40.536 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:40.536 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:02:40.536 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:58:32.707360+0000", 2023-09-23T21:02:40.536 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999995 2023-09-23T21:02:40.536 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:40.536 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:40.536 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:02:40.536 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:58:32.707363+0000", 2023-09-23T21:02:40.536 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.8119999999999999e-06 2023-09-23T21:02:40.536 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:40.537 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:40.537 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:02:40.537 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:58:32.707371+0000", 2023-09-23T21:02:40.537 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.8809999999999996e-06 2023-09-23T21:02:40.537 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:40.537 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:40.537 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:02:40.537 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:58:32.707375+0000", 2023-09-23T21:02:40.537 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.9790000000000004e-06 2023-09-23T21:02:40.537 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:40.537 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:40.537 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:02:40.537 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:58:32.707425+0000", 2023-09-23T21:02:40.538 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.0643999999999998e-05 2023-09-23T21:02:40.538 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:40.538 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:40.538 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:02:40.538 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:58:32.707574+0000", 2023-09-23T21:02:40.538 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000148715 2023-09-23T21:02:40.538 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:40.538 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:40.538 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 3,7", 2023-09-23T21:02:40.538 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:58:32.707711+0000", 2023-09-23T21:02:40.538 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00013714099999999999 2023-09-23T21:02:40.539 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:40.539 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:40.539 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:02:40.539 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:58:32.714308+0000", 2023-09-23T21:02:40.539 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.006597182 2023-09-23T21:02:40.539 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:40.539 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:40.539 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:02:40.539 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:58:32.714392+0000", 2023-09-23T21:02:40.539 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.3598999999999997e-05 2023-09-23T21:02:40.539 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:40.539 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:40.539 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:02:40.540 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:58:32.797014+0000", 2023-09-23T21:02:40.540 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.082622152000000004 2023-09-23T21:02:40.540 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:40.540 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:40.540 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:02:40.540 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:58:32.797076+0000", 2023-09-23T21:02:40.540 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.1576999999999997e-05 2023-09-23T21:02:40.540 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:40.540 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:40.540 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:02:40.540 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:58:32.799801+0000", 2023-09-23T21:02:40.541 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0027257779999999999 2023-09-23T21:02:40.541 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:02:40.541 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:02:40.541 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:02:40.541 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:40.541 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:40.541 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.35100.0:116412 7.c 7:32401c65:::benchmark_data_smithi026_260489_object14650:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e234)", 2023-09-23T21:02:40.541 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T20:58:32.707388+0000", 2023-09-23T21:02:40.541 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 247.808104504, 2023-09-23T21:02:40.541 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.094168145999999994, 2023-09-23T21:02:40.541 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:02:40.541 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:02:40.542 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:02:40.542 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.35100", 2023-09-23T21:02:40.542 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/4108972961", 2023-09-23T21:02:40.542 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 116412 2023-09-23T21:02:40.542 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:40.542 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:02:40.542 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:40.542 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:02:40.542 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:58:32.707388+0000", 2023-09-23T21:02:40.542 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:02:40.542 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:40.542 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:40.543 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:02:40.543 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:58:32.707388+0000", 2023-09-23T21:02:40.543 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:02:40.543 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:40.543 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:40.543 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:02:40.543 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:58:32.707388+0000", 2023-09-23T21:02:40.543 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999995 2023-09-23T21:02:40.543 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:40.543 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:40.543 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:02:40.543 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:58:32.707390+0000", 2023-09-23T21:02:40.544 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.3970000000000001e-06 2023-09-23T21:02:40.544 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:40.544 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:40.544 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:02:40.544 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:58:32.707774+0000", 2023-09-23T21:02:40.544 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00038455999999999999 2023-09-23T21:02:40.544 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:40.544 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:40.544 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:02:40.544 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:58:32.707781+0000", 2023-09-23T21:02:40.544 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.392e-06 2023-09-23T21:02:40.544 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:40.545 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:40.545 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:02:40.545 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:58:32.711093+0000", 2023-09-23T21:02:40.545 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0033124309999999998 2023-09-23T21:02:40.545 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:40.545 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:40.545 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:02:40.545 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:58:32.711302+0000", 2023-09-23T21:02:40.545 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000208307 2023-09-23T21:02:40.545 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:40.545 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:40.546 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 3,7", 2023-09-23T21:02:40.546 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:58:32.711476+0000", 2023-09-23T21:02:40.546 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00017412100000000001 2023-09-23T21:02:40.546 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:40.546 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:40.546 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:02:40.546 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:58:32.714404+0000", 2023-09-23T21:02:40.546 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0029286080000000001 2023-09-23T21:02:40.546 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:40.546 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:40.546 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:02:40.547 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:58:32.718305+0000", 2023-09-23T21:02:40.547 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.003900447 2023-09-23T21:02:40.547 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:40.547 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:40.547 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:02:40.547 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:58:32.799851+0000", 2023-09-23T21:02:40.547 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.081545845000000006 2023-09-23T21:02:40.547 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:40.547 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:40.547 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:02:40.547 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:58:32.799895+0000", 2023-09-23T21:02:40.547 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.3883e-05 2023-09-23T21:02:40.548 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:40.548 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:40.548 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:02:40.548 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:58:32.801556+0000", 2023-09-23T21:02:40.548 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.001661479 2023-09-23T21:02:40.548 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:02:40.548 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:02:40.548 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:02:40.548 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:40.548 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:40.548 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.35100.0:116416 7.c 7:315889a0:::benchmark_data_smithi026_260489_object14654:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e234)", 2023-09-23T21:02:40.548 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T20:58:32.714704+0000", 2023-09-23T21:02:40.549 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 247.80078800000001, 2023-09-23T21:02:40.549 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.088565651999999995, 2023-09-23T21:02:40.549 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:02:40.549 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:02:40.549 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:02:40.549 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.35100", 2023-09-23T21:02:40.549 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/4108972961", 2023-09-23T21:02:40.549 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 116416 2023-09-23T21:02:40.549 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:40.549 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:02:40.549 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:40.549 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:02:40.550 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:58:32.714704+0000", 2023-09-23T21:02:40.550 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:02:40.550 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:40.550 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:40.550 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:02:40.550 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:58:32.714704+0000", 2023-09-23T21:02:40.550 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:02:40.550 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:40.550 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:40.550 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:02:40.550 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:58:32.714703+0000", 2023-09-23T21:02:40.551 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T21:02:40.551 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:40.551 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:40.551 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:02:40.551 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:58:32.714711+0000", 2023-09-23T21:02:40.551 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.6709999999999999e-06 2023-09-23T21:02:40.551 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:40.551 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:40.551 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:02:40.551 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:58:32.714724+0000", 2023-09-23T21:02:40.551 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.3249e-05 2023-09-23T21:02:40.551 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:40.552 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:40.552 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:02:40.552 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:58:32.714730+0000", 2023-09-23T21:02:40.552 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.9399999999999999e-06 2023-09-23T21:02:40.552 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:40.552 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:40.552 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:02:40.552 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:58:32.714770+0000", 2023-09-23T21:02:40.552 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.0296000000000002e-05 2023-09-23T21:02:40.552 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:40.552 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:40.552 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:02:40.553 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:58:32.714893+0000", 2023-09-23T21:02:40.553 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00012327599999999999 2023-09-23T21:02:40.553 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:40.553 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:40.553 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 3,7", 2023-09-23T21:02:40.553 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:58:32.715066+0000", 2023-09-23T21:02:40.553 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00017291999999999999 2023-09-23T21:02:40.553 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:40.553 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:40.553 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:02:40.553 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:58:32.718350+0000", 2023-09-23T21:02:40.553 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0032839549999999999 2023-09-23T21:02:40.554 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:40.554 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:40.554 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:02:40.554 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:58:32.719377+0000", 2023-09-23T21:02:40.554 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.001026721 2023-09-23T21:02:40.554 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:40.554 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:40.554 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:02:40.554 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:58:32.801595+0000", 2023-09-23T21:02:40.554 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.082218264999999999 2023-09-23T21:02:40.554 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:40.555 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:40.555 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:02:40.555 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:58:32.801636+0000", 2023-09-23T21:02:40.555 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.0797000000000003e-05 2023-09-23T21:02:40.555 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:40.555 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:40.555 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:02:40.555 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:58:32.803270+0000", 2023-09-23T21:02:40.555 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0016339690000000001 2023-09-23T21:02:40.555 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:02:40.555 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:02:40.555 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:02:40.556 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:40.556 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:40.556 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.35100.0:159312 7.c 7:35fe94fa:::benchmark_data_smithi026_260489_object57550:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e242)", 2023-09-23T21:02:40.556 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T20:59:45.714683+0000", 2023-09-23T21:02:40.556 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 174.80080902, 2023-09-23T21:02:40.556 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.098395191000000007, 2023-09-23T21:02:40.556 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:02:40.556 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:02:40.556 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:02:40.556 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.35100", 2023-09-23T21:02:40.556 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/4108972961", 2023-09-23T21:02:40.556 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 159312 2023-09-23T21:02:40.556 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:40.557 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:02:40.557 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:40.557 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:02:40.557 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:59:45.714683+0000", 2023-09-23T21:02:40.557 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:02:40.557 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:40.557 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:40.557 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:02:40.557 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:59:45.714683+0000", 2023-09-23T21:02:40.557 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:02:40.557 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:40.557 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:40.558 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:02:40.558 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:59:45.714683+0000", 2023-09-23T21:02:40.558 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999995 2023-09-23T21:02:40.558 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:40.558 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:40.558 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:02:40.558 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:59:45.714686+0000", 2023-09-23T21:02:40.558 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.1e-06 2023-09-23T21:02:40.558 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:40.558 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:40.558 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:02:40.558 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:59:45.714693+0000", 2023-09-23T21:02:40.559 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.3660000000000003e-06 2023-09-23T21:02:40.559 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:40.559 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:40.559 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:02:40.559 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:59:45.714698+0000", 2023-09-23T21:02:40.559 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.8330000000000001e-06 2023-09-23T21:02:40.559 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:40.559 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:40.559 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:02:40.559 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:59:45.714785+0000", 2023-09-23T21:02:40.559 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.6920000000000001e-05 2023-09-23T21:02:40.559 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:40.560 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:40.560 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:02:40.560 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:59:45.714945+0000", 2023-09-23T21:02:40.560 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00015988100000000001 2023-09-23T21:02:40.560 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:40.560 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:40.560 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 3,7", 2023-09-23T21:02:40.560 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:59:45.715100+0000", 2023-09-23T21:02:40.560 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000155119 2023-09-23T21:02:40.560 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:40.560 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:40.560 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:02:40.561 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:59:45.721013+0000", 2023-09-23T21:02:40.561 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0059126630000000003 2023-09-23T21:02:40.561 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:40.561 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:40.561 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:02:40.561 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:59:45.729983+0000", 2023-09-23T21:02:40.561 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0089698169999999997 2023-09-23T21:02:40.561 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:40.561 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:40.561 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:02:40.561 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:59:45.810696+0000", 2023-09-23T21:02:40.561 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.080713268000000005 2023-09-23T21:02:40.562 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:40.562 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:40.562 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:02:40.562 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:59:45.810748+0000", 2023-09-23T21:02:40.562 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.2618999999999999e-05 2023-09-23T21:02:40.562 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:40.562 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:40.562 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:02:40.562 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:59:45.813079+0000", 2023-09-23T21:02:40.562 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.002330024 2023-09-23T21:02:40.562 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:02:40.562 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:02:40.563 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:02:40.563 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:40.563 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:40.563 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.35100.0:159314 7.c 7:3dd9b3f5:::benchmark_data_smithi026_260489_object57552:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e242)", 2023-09-23T21:02:40.563 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T20:59:45.715055+0000", 2023-09-23T21:02:40.563 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 174.80043715799999, 2023-09-23T21:02:40.563 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.099656759999999997, 2023-09-23T21:02:40.563 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:02:40.563 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:02:40.563 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:02:40.563 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.35100", 2023-09-23T21:02:40.563 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/4108972961", 2023-09-23T21:02:40.564 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 159314 2023-09-23T21:02:40.564 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:40.564 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:02:40.564 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:40.564 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:02:40.564 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:59:45.715055+0000", 2023-09-23T21:02:40.564 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:02:40.564 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:40.564 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:40.564 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:02:40.564 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:59:45.715055+0000", 2023-09-23T21:02:40.564 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:02:40.565 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:40.565 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:40.565 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:02:40.565 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:59:45.715055+0000", 2023-09-23T21:02:40.565 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999995 2023-09-23T21:02:40.565 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:40.565 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:40.565 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:02:40.565 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:59:45.715059+0000", 2023-09-23T21:02:40.565 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.2660000000000003e-06 2023-09-23T21:02:40.565 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:40.565 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:40.566 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:02:40.566 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:59:45.715067+0000", 2023-09-23T21:02:40.566 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.9219999999999997e-06 2023-09-23T21:02:40.566 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:40.566 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:40.566 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:02:40.566 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:59:45.715072+0000", 2023-09-23T21:02:40.566 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.2689999999999999e-06 2023-09-23T21:02:40.566 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:40.566 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:40.566 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:02:40.566 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:59:45.717941+0000", 2023-09-23T21:02:40.567 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.002869267 2023-09-23T21:02:40.567 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:40.567 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:40.567 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:02:40.567 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:59:45.718091+0000", 2023-09-23T21:02:40.567 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00014939 2023-09-23T21:02:40.567 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:40.567 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:40.567 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 3,7", 2023-09-23T21:02:40.567 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:59:45.718272+0000", 2023-09-23T21:02:40.567 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000181249 2023-09-23T21:02:40.567 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:40.568 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:40.568 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:02:40.568 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:59:45.721107+0000", 2023-09-23T21:02:40.568 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.002835335 2023-09-23T21:02:40.568 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:40.568 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:40.568 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:02:40.568 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:59:45.730021+0000", 2023-09-23T21:02:40.568 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0089133249999999997 2023-09-23T21:02:40.568 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:40.568 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:40.568 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:02:40.569 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:59:45.813147+0000", 2023-09-23T21:02:40.569 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.083126633000000005 2023-09-23T21:02:40.569 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:40.569 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:40.569 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:02:40.569 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:59:45.813186+0000", 2023-09-23T21:02:40.569 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.9128999999999998e-05 2023-09-23T21:02:40.569 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:40.569 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:40.569 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:02:40.569 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:59:45.814712+0000", 2023-09-23T21:02:40.569 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0015256040000000001 2023-09-23T21:02:40.569 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:02:40.570 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:02:40.570 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:02:40.570 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:40.570 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:40.570 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.35100.0:159321 7.c 7:33459c82:::benchmark_data_smithi026_260489_object57559:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e242)", 2023-09-23T21:02:40.570 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T20:59:45.723923+0000", 2023-09-23T21:02:40.570 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 174.79156981599999, 2023-09-23T21:02:40.570 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.092489334000000006, 2023-09-23T21:02:40.570 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:02:40.570 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:02:40.570 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:02:40.570 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.35100", 2023-09-23T21:02:40.571 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/4108972961", 2023-09-23T21:02:40.571 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 159321 2023-09-23T21:02:40.571 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:40.571 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:02:40.571 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:40.571 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:02:40.571 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:59:45.723923+0000", 2023-09-23T21:02:40.571 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:02:40.571 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:40.571 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:40.571 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:02:40.571 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:59:45.723923+0000", 2023-09-23T21:02:40.572 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:02:40.572 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:40.572 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:40.572 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:02:40.572 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:59:45.723921+0000", 2023-09-23T21:02:40.572 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.999999 2023-09-23T21:02:40.572 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:40.572 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:40.572 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:02:40.572 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:59:45.723927+0000", 2023-09-23T21:02:40.572 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.541e-06 2023-09-23T21:02:40.572 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:40.573 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:40.573 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:02:40.573 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:59:45.723942+0000", 2023-09-23T21:02:40.573 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.49e-05 2023-09-23T21:02:40.573 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:40.573 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:40.573 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:02:40.573 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:59:45.723949+0000", 2023-09-23T21:02:40.573 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.6959999999999999e-06 2023-09-23T21:02:40.573 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:40.573 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:40.573 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:02:40.574 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:59:45.723997+0000", 2023-09-23T21:02:40.574 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.7963000000000001e-05 2023-09-23T21:02:40.574 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:40.574 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:40.574 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:02:40.574 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:59:45.724141+0000", 2023-09-23T21:02:40.574 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00014393399999999999 2023-09-23T21:02:40.574 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:40.574 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:40.574 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 3,7", 2023-09-23T21:02:40.574 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:59:45.724301+0000", 2023-09-23T21:02:40.574 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00016078300000000001 2023-09-23T21:02:40.575 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:40.575 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:40.575 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:02:40.575 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:59:45.727866+0000", 2023-09-23T21:02:40.575 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0035644069999999999 2023-09-23T21:02:40.575 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:40.575 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:40.575 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:02:40.575 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:59:45.730056+0000", 2023-09-23T21:02:40.575 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0021899060000000001 2023-09-23T21:02:40.575 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:40.575 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:40.576 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:02:40.576 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:59:45.814772+0000", 2023-09-23T21:02:40.576 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.084716738 2023-09-23T21:02:40.576 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:40.576 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:40.576 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:02:40.576 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:59:45.814811+0000", 2023-09-23T21:02:40.576 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.8770999999999999e-05 2023-09-23T21:02:40.576 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:40.576 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:40.576 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:02:40.576 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:59:45.816412+0000", 2023-09-23T21:02:40.577 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.001600757 2023-09-23T21:02:40.577 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:02:40.577 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:02:40.577 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:02:40.577 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:40.577 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:40.577 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.35100.0:168557 7.c 7:3a3d85f5:::benchmark_data_smithi026_260489_object66795:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e243)", 2023-09-23T21:02:40.577 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:00:01.247853+0000", 2023-09-23T21:02:40.577 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 159.267638858, 2023-09-23T21:02:40.577 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.091415162999999994, 2023-09-23T21:02:40.577 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:02:40.577 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:02:40.578 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:02:40.578 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.35100", 2023-09-23T21:02:40.578 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/4108972961", 2023-09-23T21:02:40.578 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 168557 2023-09-23T21:02:40.578 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:40.578 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:02:40.578 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:40.578 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:02:40.578 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:00:01.247853+0000", 2023-09-23T21:02:40.578 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:02:40.578 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:40.578 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:40.579 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:02:40.579 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:00:01.247853+0000", 2023-09-23T21:02:40.579 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:02:40.579 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:40.579 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:40.579 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:02:40.579 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:00:01.247853+0000", 2023-09-23T21:02:40.579 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999995 2023-09-23T21:02:40.579 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:40.579 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:40.579 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:02:40.579 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:00:01.247856+0000", 2023-09-23T21:02:40.580 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.7080000000000002e-06 2023-09-23T21:02:40.580 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:40.580 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:40.580 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:02:40.580 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:00:01.247861+0000", 2023-09-23T21:02:40.580 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.5419999999999996e-06 2023-09-23T21:02:40.580 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:40.580 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:40.580 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:02:40.580 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:00:01.247865+0000", 2023-09-23T21:02:40.580 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.9999999999999998e-06 2023-09-23T21:02:40.580 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:40.580 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:40.581 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:02:40.581 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:00:01.249686+0000", 2023-09-23T21:02:40.581 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0018203990000000001 2023-09-23T21:02:40.581 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:40.581 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:40.581 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:02:40.581 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:00:01.249824+0000", 2023-09-23T21:02:40.581 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00013816800000000001 2023-09-23T21:02:40.581 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:40.581 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:40.581 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 3,7", 2023-09-23T21:02:40.581 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:00:01.249943+0000", 2023-09-23T21:02:40.582 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00011900600000000001 2023-09-23T21:02:40.582 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:40.582 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:40.582 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:02:40.582 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:00:01.257468+0000", 2023-09-23T21:02:40.582 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.007524729 2023-09-23T21:02:40.582 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:40.582 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:40.582 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:02:40.582 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:00:01.257586+0000", 2023-09-23T21:02:40.582 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00011883 2023-09-23T21:02:40.582 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:40.583 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:40.583 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:02:40.583 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:00:01.337271+0000", 2023-09-23T21:02:40.583 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.079684028000000004 2023-09-23T21:02:40.583 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:40.583 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:40.583 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:02:40.583 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:00:01.337308+0000", 2023-09-23T21:02:40.583 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.7438000000000003e-05 2023-09-23T21:02:40.583 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:40.583 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:40.583 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:02:40.584 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:00:01.339269+0000", 2023-09-23T21:02:40.584 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0019606939999999998 2023-09-23T21:02:40.584 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:02:40.584 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:02:40.584 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:02:40.584 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:40.584 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:40.584 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.35100.0:168558 7.c 7:3c767518:::benchmark_data_smithi026_260489_object66796:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e243)", 2023-09-23T21:02:40.584 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:00:01.247879+0000", 2023-09-23T21:02:40.584 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 159.267613326, 2023-09-23T21:02:40.584 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.100175694, 2023-09-23T21:02:40.584 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:02:40.585 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:02:40.585 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:02:40.585 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.35100", 2023-09-23T21:02:40.585 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/4108972961", 2023-09-23T21:02:40.585 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 168558 2023-09-23T21:02:40.585 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:40.585 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:02:40.585 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:40.585 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:02:40.585 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:00:01.247879+0000", 2023-09-23T21:02:40.585 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:02:40.585 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:40.586 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:40.586 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:02:40.586 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:00:01.247879+0000", 2023-09-23T21:02:40.586 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:02:40.586 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:40.586 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:40.586 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:02:40.586 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:00:01.247879+0000", 2023-09-23T21:02:40.586 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999995 2023-09-23T21:02:40.586 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:40.586 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:40.586 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:02:40.587 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:00:01.247881+0000", 2023-09-23T21:02:40.587 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.362e-06 2023-09-23T21:02:40.587 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:40.587 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:40.587 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:02:40.587 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:00:01.247889+0000", 2023-09-23T21:02:40.587 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.2220000000000002e-06 2023-09-23T21:02:40.587 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:40.587 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:40.587 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:02:40.587 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:00:01.247893+0000", 2023-09-23T21:02:40.587 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.5729999999999999e-06 2023-09-23T21:02:40.588 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:40.588 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:40.588 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:02:40.588 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:00:01.253563+0000", 2023-09-23T21:02:40.588 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00567024 2023-09-23T21:02:40.588 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:40.588 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:40.588 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:02:40.588 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:00:01.253723+0000", 2023-09-23T21:02:40.588 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00016021500000000001 2023-09-23T21:02:40.588 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:40.588 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:40.589 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 3,7", 2023-09-23T21:02:40.589 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:00:01.253851+0000", 2023-09-23T21:02:40.589 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00012726600000000001 2023-09-23T21:02:40.589 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:40.589 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:40.589 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:02:40.589 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:00:01.257608+0000", 2023-09-23T21:02:40.589 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0037574180000000002 2023-09-23T21:02:40.589 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:40.589 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:40.589 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:02:40.589 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:00:01.257732+0000", 2023-09-23T21:02:40.590 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00012452000000000001 2023-09-23T21:02:40.590 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:40.590 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:40.590 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:02:40.590 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:00:01.345514+0000", 2023-09-23T21:02:40.590 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.087781146000000004 2023-09-23T21:02:40.590 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:40.590 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:40.590 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:02:40.590 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:00:01.345578+0000", 2023-09-23T21:02:40.590 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.4691999999999997e-05 2023-09-23T21:02:40.591 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:40.591 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:40.591 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:02:40.591 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:00:01.348055+0000", 2023-09-23T21:02:40.591 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0024764119999999999 2023-09-23T21:02:40.591 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:02:40.591 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:02:40.591 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:02:40.591 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:40.591 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:40.591 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.35100.0:171218 7.c 7:3a087124:::benchmark_data_smithi026_260489_object69456:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e244)", 2023-09-23T21:02:40.591 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:00:06.071740+0000", 2023-09-23T21:02:40.592 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 154.443751975, 2023-09-23T21:02:40.592 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.091567966000000001, 2023-09-23T21:02:40.592 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:02:40.592 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:02:40.592 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:02:40.592 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.35100", 2023-09-23T21:02:40.592 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/4108972961", 2023-09-23T21:02:40.592 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 171218 2023-09-23T21:02:40.592 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:40.592 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:02:40.592 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:40.592 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:02:40.593 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:00:06.071740+0000", 2023-09-23T21:02:40.593 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:02:40.593 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:40.593 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:40.593 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:02:40.593 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:00:06.071740+0000", 2023-09-23T21:02:40.593 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:02:40.593 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:40.593 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:40.593 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:02:40.593 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:00:06.071739+0000", 2023-09-23T21:02:40.593 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.999999 2023-09-23T21:02:40.594 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:40.594 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:40.594 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:02:40.594 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:00:06.071744+0000", 2023-09-23T21:02:40.594 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.0889999999999999e-06 2023-09-23T21:02:40.594 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:40.594 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:40.594 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:02:40.594 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:00:06.071756+0000", 2023-09-23T21:02:40.594 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.1184e-05 2023-09-23T21:02:40.594 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:40.594 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:40.595 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:02:40.595 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:00:06.071762+0000", 2023-09-23T21:02:40.595 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.8480000000000003e-06 2023-09-23T21:02:40.595 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:40.595 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:40.595 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:02:40.595 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:00:06.071810+0000", 2023-09-23T21:02:40.595 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.7756999999999997e-05 2023-09-23T21:02:40.595 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:40.595 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:40.595 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:02:40.595 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:00:06.071934+0000", 2023-09-23T21:02:40.596 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000123504 2023-09-23T21:02:40.596 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:40.596 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:40.596 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 3,7", 2023-09-23T21:02:40.596 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:00:06.072047+0000", 2023-09-23T21:02:40.596 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00011350599999999999 2023-09-23T21:02:40.596 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:40.596 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:40.596 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:02:40.596 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:00:06.078092+0000", 2023-09-23T21:02:40.596 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0060443750000000003 2023-09-23T21:02:40.596 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:40.596 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:40.597 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:02:40.597 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:00:06.078106+0000", 2023-09-23T21:02:40.597 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.4623e-05 2023-09-23T21:02:40.597 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:40.597 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:40.597 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:02:40.597 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:00:06.161295+0000", 2023-09-23T21:02:40.597 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.083188512000000006 2023-09-23T21:02:40.597 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:40.597 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:40.597 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:02:40.597 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:00:06.161363+0000", 2023-09-23T21:02:40.598 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.8614000000000004e-05 2023-09-23T21:02:40.598 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:40.598 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:40.598 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:02:40.598 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:00:06.163308+0000", 2023-09-23T21:02:40.598 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.001944985 2023-09-23T21:02:40.598 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:02:40.598 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:02:40.598 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:02:40.598 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:40.598 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:40.598 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.35100.0:171221 7.c 7:3c6c6944:::benchmark_data_smithi026_260489_object69459:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e244)", 2023-09-23T21:02:40.599 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:00:06.071777+0000", 2023-09-23T21:02:40.599 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 154.44371548999999, 2023-09-23T21:02:40.599 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.092901023999999999, 2023-09-23T21:02:40.599 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:02:40.599 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:02:40.599 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:02:40.599 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.35100", 2023-09-23T21:02:40.599 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/4108972961", 2023-09-23T21:02:40.599 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 171221 2023-09-23T21:02:40.599 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:40.599 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:02:40.599 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:40.600 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:02:40.600 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:00:06.071777+0000", 2023-09-23T21:02:40.600 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:02:40.600 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:40.600 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:40.600 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:02:40.600 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:00:06.071777+0000", 2023-09-23T21:02:40.600 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:02:40.600 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:40.600 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:40.600 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:02:40.600 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:00:06.071776+0000", 2023-09-23T21:02:40.600 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999995 2023-09-23T21:02:40.601 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:40.601 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:40.601 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:02:40.601 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:00:06.071779+0000", 2023-09-23T21:02:40.601 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.0460000000000001e-06 2023-09-23T21:02:40.601 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:40.601 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:40.601 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:02:40.601 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:00:06.071786+0000", 2023-09-23T21:02:40.601 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.7959999999999998e-06 2023-09-23T21:02:40.601 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:40.601 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:40.602 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:02:40.602 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:00:06.071791+0000", 2023-09-23T21:02:40.602 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.8180000000000003e-06 2023-09-23T21:02:40.602 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:40.602 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:40.602 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:02:40.602 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:00:06.075244+0000", 2023-09-23T21:02:40.602 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0034532849999999999 2023-09-23T21:02:40.602 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:40.602 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:40.602 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:02:40.603 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:00:06.075361+0000", 2023-09-23T21:02:40.603 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00011628 2023-09-23T21:02:40.603 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:40.603 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:40.603 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 3,7", 2023-09-23T21:02:40.603 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:00:06.075477+0000", 2023-09-23T21:02:40.603 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000116014 2023-09-23T21:02:40.603 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:40.603 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:40.603 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:02:40.603 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:00:06.078157+0000", 2023-09-23T21:02:40.603 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00268016 2023-09-23T21:02:40.603 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:40.604 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:40.604 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:02:40.604 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:00:06.078214+0000", 2023-09-23T21:02:40.604 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.7129000000000001e-05 2023-09-23T21:02:40.604 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:40.604 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:40.604 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:02:40.604 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:00:06.163374+0000", 2023-09-23T21:02:40.604 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.085160562999999995 2023-09-23T21:02:40.604 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:40.604 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:40.604 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:02:40.605 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:00:06.163417+0000", 2023-09-23T21:02:40.605 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.2329000000000001e-05 2023-09-23T21:02:40.605 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:40.605 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:40.605 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:02:40.605 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:00:06.164678+0000", 2023-09-23T21:02:40.605 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0012610550000000001 2023-09-23T21:02:40.605 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:02:40.605 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:02:40.605 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:02:40.605 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:40.605 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:40.605 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.35100.0:185573 7.c 7:301f47e9:::benchmark_data_smithi026_260489_object83811:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e246)", 2023-09-23T21:02:40.606 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:00:29.742836+0000", 2023-09-23T21:02:40.606 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 130.77265595, 2023-09-23T21:02:40.606 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.095769330999999999, 2023-09-23T21:02:40.606 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:02:40.606 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:02:40.606 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:02:40.606 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.35100", 2023-09-23T21:02:40.606 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/4108972961", 2023-09-23T21:02:40.606 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 185573 2023-09-23T21:02:40.606 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:40.606 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:02:40.606 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:40.607 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:02:40.607 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:00:29.742836+0000", 2023-09-23T21:02:40.607 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:02:40.607 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:40.607 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:40.607 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:02:40.607 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:00:29.742836+0000", 2023-09-23T21:02:40.607 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:02:40.607 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:40.607 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:40.607 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:02:40.607 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:00:29.742836+0000", 2023-09-23T21:02:40.607 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999995 2023-09-23T21:02:40.608 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:40.608 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:40.608 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:02:40.608 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:00:29.742839+0000", 2023-09-23T21:02:40.608 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.4539999999999999e-06 2023-09-23T21:02:40.608 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:40.608 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:40.608 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:02:40.608 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:00:29.742846+0000", 2023-09-23T21:02:40.608 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.9110000000000004e-06 2023-09-23T21:02:40.608 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:40.608 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:40.609 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:02:40.609 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:00:29.742852+0000", 2023-09-23T21:02:40.609 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.037e-06 2023-09-23T21:02:40.609 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:40.609 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:40.609 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:02:40.609 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:00:29.742901+0000", 2023-09-23T21:02:40.609 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.8470999999999997e-05 2023-09-23T21:02:40.609 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:40.609 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:40.609 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:02:40.609 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:00:29.743048+0000", 2023-09-23T21:02:40.610 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000147093 2023-09-23T21:02:40.610 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:40.610 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:40.610 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 3,7", 2023-09-23T21:02:40.610 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:00:29.743199+0000", 2023-09-23T21:02:40.610 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000150853 2023-09-23T21:02:40.610 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:40.610 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:40.610 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:02:40.610 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:00:29.749663+0000", 2023-09-23T21:02:40.610 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0064643019999999999 2023-09-23T21:02:40.611 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:40.611 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:40.611 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:02:40.611 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:00:29.749673+0000", 2023-09-23T21:02:40.611 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.6059999999999992e-06 2023-09-23T21:02:40.611 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:40.611 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:40.611 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:02:40.611 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:00:29.836854+0000", 2023-09-23T21:02:40.611 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.087180890999999996 2023-09-23T21:02:40.611 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:40.611 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:40.612 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:02:40.612 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:00:29.836913+0000", 2023-09-23T21:02:40.612 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.9373999999999998e-05 2023-09-23T21:02:40.612 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:40.612 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:40.612 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:02:40.612 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:00:29.838606+0000", 2023-09-23T21:02:40.612 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0016928119999999999 2023-09-23T21:02:40.612 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:02:40.612 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:02:40.612 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:02:40.612 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:40.613 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:40.613 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.35100.0:191727 7.c 7:3d4724be:::benchmark_data_smithi026_260489_object89965:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e248)", 2023-09-23T21:02:40.613 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:00:39.842015+0000", 2023-09-23T21:02:40.613 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 120.67347706699999, 2023-09-23T21:02:40.613 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.093273197000000002, 2023-09-23T21:02:40.613 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:02:40.613 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:02:40.613 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:02:40.613 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.35100", 2023-09-23T21:02:40.613 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/4108972961", 2023-09-23T21:02:40.613 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 191727 2023-09-23T21:02:40.613 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:40.614 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:02:40.614 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:40.614 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:02:40.614 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:00:39.842015+0000", 2023-09-23T21:02:40.614 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:02:40.614 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:40.614 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:40.614 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:02:40.614 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:00:39.842015+0000", 2023-09-23T21:02:40.614 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:02:40.614 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:40.615 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:40.615 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:02:40.615 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:00:39.842015+0000", 2023-09-23T21:02:40.615 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999995 2023-09-23T21:02:40.615 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:40.615 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:40.615 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:02:40.615 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:00:39.842017+0000", 2023-09-23T21:02:40.615 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.6290000000000002e-06 2023-09-23T21:02:40.615 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:40.615 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:40.616 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:02:40.616 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:00:39.842023+0000", 2023-09-23T21:02:40.616 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.9050000000000002e-06 2023-09-23T21:02:40.616 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:40.616 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:40.616 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:02:40.616 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:00:39.842027+0000", 2023-09-23T21:02:40.616 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.9779999999999999e-06 2023-09-23T21:02:40.616 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:40.616 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:40.616 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:02:40.616 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:00:39.849182+0000", 2023-09-23T21:02:40.616 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0071541549999999997 2023-09-23T21:02:40.617 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:40.617 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:40.617 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:02:40.617 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:00:39.849347+0000", 2023-09-23T21:02:40.617 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00016532999999999999 2023-09-23T21:02:40.617 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:40.617 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:40.617 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 3,7", 2023-09-23T21:02:40.617 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:00:39.849454+0000", 2023-09-23T21:02:40.617 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00010754100000000001 2023-09-23T21:02:40.617 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:40.618 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:40.618 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:02:40.618 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:00:39.853545+0000", 2023-09-23T21:02:40.618 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0040905259999999997 2023-09-23T21:02:40.618 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:40.618 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:40.618 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:02:40.618 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:00:39.856489+0000", 2023-09-23T21:02:40.618 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.002943701 2023-09-23T21:02:40.618 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:40.618 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:40.619 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:02:40.619 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:00:39.933771+0000", 2023-09-23T21:02:40.619 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.077281924000000002 2023-09-23T21:02:40.619 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:40.619 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:40.619 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:02:40.619 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:00:39.933807+0000", 2023-09-23T21:02:40.619 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.6025999999999997e-05 2023-09-23T21:02:40.619 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:40.619 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:40.619 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:02:40.619 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:00:39.935288+0000", 2023-09-23T21:02:40.619 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0014819119999999999 2023-09-23T21:02:40.620 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:02:40.620 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:02:40.620 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:02:40.620 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:40.620 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:40.620 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.35469.0:29524 8.9 8:902f043e:::benchmark_data_smithi026_301269_object29523:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e258)", 2023-09-23T21:02:40.620 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:01:43.969600+0000", 2023-09-23T21:02:40.620 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 56.545892127999998, 2023-09-23T21:02:40.620 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.089767586999999996, 2023-09-23T21:02:40.620 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:02:40.620 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:02:40.621 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:02:40.621 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.35469", 2023-09-23T21:02:40.621 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/4041424623", 2023-09-23T21:02:40.621 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 29524 2023-09-23T21:02:40.621 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:40.621 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:02:40.621 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:40.621 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:02:40.621 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:01:43.969600+0000", 2023-09-23T21:02:40.621 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:02:40.621 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:40.621 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:40.622 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:02:40.622 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:01:43.969600+0000", 2023-09-23T21:02:40.622 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:02:40.622 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:40.622 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:40.622 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:02:40.622 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:01:43.969599+0000", 2023-09-23T21:02:40.622 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.999999 2023-09-23T21:02:40.622 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:40.622 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:40.622 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:02:40.622 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:01:43.969655+0000", 2023-09-23T21:02:40.623 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.5977999999999998e-05 2023-09-23T21:02:40.623 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:40.623 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:40.623 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:02:40.623 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:01:43.969665+0000", 2023-09-23T21:02:40.623 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.0025e-05 2023-09-23T21:02:40.623 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:40.623 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:40.623 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:02:40.623 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:01:43.969671+0000", 2023-09-23T21:02:40.623 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.1030000000000001e-06 2023-09-23T21:02:40.623 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:40.624 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:40.624 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:02:40.624 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:01:43.969751+0000", 2023-09-23T21:02:40.624 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.0123999999999994e-05 2023-09-23T21:02:40.624 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:40.624 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:40.624 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:02:40.624 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:01:43.969863+0000", 2023-09-23T21:02:40.624 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0001126 2023-09-23T21:02:40.624 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:40.624 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:40.624 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 0,5", 2023-09-23T21:02:40.625 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:01:43.969986+0000", 2023-09-23T21:02:40.625 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00012316000000000001 2023-09-23T21:02:40.625 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:40.625 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:40.625 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:02:40.625 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:01:43.973907+0000", 2023-09-23T21:02:40.625 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0039203609999999998 2023-09-23T21:02:40.625 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:40.625 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:40.625 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:02:40.625 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:01:43.974452+0000", 2023-09-23T21:02:40.625 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00054483000000000005 2023-09-23T21:02:40.626 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:40.626 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:40.626 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:02:40.626 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:01:44.059304+0000", 2023-09-23T21:02:40.626 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.084852346999999995 2023-09-23T21:02:40.626 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:40.626 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:40.626 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:02:40.626 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:01:44.059338+0000", 2023-09-23T21:02:40.626 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.4477000000000002e-05 2023-09-23T21:02:40.626 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:40.626 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:40.627 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:02:40.627 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:01:44.059368+0000", 2023-09-23T21:02:40.627 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.94e-05 2023-09-23T21:02:40.627 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:02:40.627 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:02:40.627 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:02:40.627 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:40.627 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:40.627 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.35469.0:29527 8.9 8:9da44672:::benchmark_data_smithi026_301269_object29526:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e258)", 2023-09-23T21:02:40.627 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:01:43.971063+0000", 2023-09-23T21:02:40.627 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 56.544429547999997, 2023-09-23T21:02:40.627 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.088417546, 2023-09-23T21:02:40.628 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:02:40.628 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:02:40.628 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:02:40.628 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.35469", 2023-09-23T21:02:40.628 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/4041424623", 2023-09-23T21:02:40.628 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 29527 2023-09-23T21:02:40.628 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:40.628 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:02:40.628 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:40.628 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:02:40.628 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:01:43.971063+0000", 2023-09-23T21:02:40.628 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:02:40.629 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:40.629 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:40.629 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:02:40.629 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:01:43.971063+0000", 2023-09-23T21:02:40.629 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:02:40.629 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:40.629 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:40.629 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:02:40.629 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:01:43.971062+0000", 2023-09-23T21:02:40.629 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.999999 2023-09-23T21:02:40.629 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:40.629 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:40.629 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:02:40.630 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:01:43.971124+0000", 2023-09-23T21:02:40.630 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.2626999999999995e-05 2023-09-23T21:02:40.630 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:40.630 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:40.630 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:02:40.630 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:01:43.971135+0000", 2023-09-23T21:02:40.630 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.1049999999999999e-05 2023-09-23T21:02:40.630 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:40.630 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:40.630 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:02:40.630 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:01:43.971142+0000", 2023-09-23T21:02:40.631 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.781e-06 2023-09-23T21:02:40.631 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:40.631 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:40.631 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:02:40.631 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:01:43.971239+0000", 2023-09-23T21:02:40.631 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.6928000000000004e-05 2023-09-23T21:02:40.631 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:40.631 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:40.631 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:02:40.631 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:01:43.971349+0000", 2023-09-23T21:02:40.631 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000110036 2023-09-23T21:02:40.631 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:40.631 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:40.632 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 0,5", 2023-09-23T21:02:40.632 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:01:43.971479+0000", 2023-09-23T21:02:40.632 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00013015300000000001 2023-09-23T21:02:40.632 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:40.632 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:40.632 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:02:40.632 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:01:43.974440+0000", 2023-09-23T21:02:40.632 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0029605040000000001 2023-09-23T21:02:40.632 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:40.632 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:40.632 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:02:40.632 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:01:43.974457+0000", 2023-09-23T21:02:40.632 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.7107000000000001e-05 2023-09-23T21:02:40.633 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:40.633 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:40.633 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:02:40.633 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:01:44.059407+0000", 2023-09-23T21:02:40.633 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.084950065000000005 2023-09-23T21:02:40.633 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:40.633 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:40.633 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:02:40.633 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:01:44.059455+0000", 2023-09-23T21:02:40.633 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.7923000000000002e-05 2023-09-23T21:02:40.633 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:40.633 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:40.634 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:02:40.634 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:01:44.059480+0000", 2023-09-23T21:02:40.634 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.5559e-05 2023-09-23T21:02:40.634 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:02:40.634 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:02:40.634 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:02:40.634 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:40.634 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:40.634 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.35469.0:29528 8.9 8:90ac8ef6:::benchmark_data_smithi026_301269_object29527:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e258)", 2023-09-23T21:02:40.634 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:01:43.971157+0000", 2023-09-23T21:02:40.634 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 56.544335468, 2023-09-23T21:02:40.634 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.088400083000000004, 2023-09-23T21:02:40.635 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:02:40.635 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:02:40.635 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:02:40.635 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.35469", 2023-09-23T21:02:40.635 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/4041424623", 2023-09-23T21:02:40.635 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 29528 2023-09-23T21:02:40.635 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:40.637 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:02:40.637 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:40.637 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:02:40.637 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:01:43.971157+0000", 2023-09-23T21:02:40.637 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:02:40.637 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:40.637 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:40.637 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:02:40.637 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:01:43.971157+0000", 2023-09-23T21:02:40.637 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:02:40.637 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:40.638 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:40.638 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:02:40.638 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:01:43.971156+0000", 2023-09-23T21:02:40.638 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999995 2023-09-23T21:02:40.638 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:40.638 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:40.638 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:02:40.638 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:01:43.971202+0000", 2023-09-23T21:02:40.638 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.5729999999999998e-05 2023-09-23T21:02:40.638 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:40.638 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:40.638 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:02:40.639 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:01:43.971209+0000", 2023-09-23T21:02:40.639 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.097e-06 2023-09-23T21:02:40.639 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:40.639 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:40.639 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:02:40.639 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:01:43.971215+0000", 2023-09-23T21:02:40.639 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.3990000000000003e-06 2023-09-23T21:02:40.639 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:40.639 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:40.639 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:02:40.639 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:01:43.972499+0000", 2023-09-23T21:02:40.640 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0012845700000000001 2023-09-23T21:02:40.640 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:40.640 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:40.640 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:02:40.640 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:01:43.972606+0000", 2023-09-23T21:02:40.640 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00010654699999999999 2023-09-23T21:02:40.640 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:40.640 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:40.640 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 0,5", 2023-09-23T21:02:40.640 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:01:43.972724+0000", 2023-09-23T21:02:40.640 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000118504 2023-09-23T21:02:40.640 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:40.641 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:40.641 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:02:40.641 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:01:43.974461+0000", 2023-09-23T21:02:40.641 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0017371019999999999 2023-09-23T21:02:40.641 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:40.641 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:40.641 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:02:40.641 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:01:43.975705+0000", 2023-09-23T21:02:40.641 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0012435899999999999 2023-09-23T21:02:40.641 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:40.641 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:40.641 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:02:40.642 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:01:44.059507+0000", 2023-09-23T21:02:40.642 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.083802046000000005 2023-09-23T21:02:40.642 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:40.642 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:40.642 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:02:40.642 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:01:44.059533+0000", 2023-09-23T21:02:40.642 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.5792999999999999e-05 2023-09-23T21:02:40.642 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:40.642 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:40.642 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:02:40.642 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:01:44.059557+0000", 2023-09-23T21:02:40.642 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.4216e-05 2023-09-23T21:02:40.643 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:02:40.643 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:02:40.643 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:02:40.643 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:40.643 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:40.643 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.35469.0:32271 8.9 8:9f8b0668:::benchmark_data_smithi026_301269_object32270:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e258)", 2023-09-23T21:02:40.643 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:01:45.593890+0000", 2023-09-23T21:02:40.643 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 54.921602356999998, 2023-09-23T21:02:40.643 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.12794244900000001, 2023-09-23T21:02:40.643 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:02:40.643 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:02:40.644 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:02:40.644 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.35469", 2023-09-23T21:02:40.644 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/4041424623", 2023-09-23T21:02:40.644 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 32271 2023-09-23T21:02:40.644 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:40.644 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:02:40.644 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:40.644 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:02:40.644 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:01:45.593890+0000", 2023-09-23T21:02:40.644 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:02:40.644 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:40.644 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:40.645 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:02:40.645 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:01:45.593890+0000", 2023-09-23T21:02:40.645 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:02:40.645 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:40.645 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:40.645 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:02:40.645 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:01:45.593888+0000", 2023-09-23T21:02:40.645 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T21:02:40.645 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:40.645 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:40.645 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:02:40.645 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:01:45.593913+0000", 2023-09-23T21:02:40.646 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.4788999999999999e-05 2023-09-23T21:02:40.646 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:40.646 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:40.646 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:02:40.646 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:01:45.593926+0000", 2023-09-23T21:02:40.646 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.2853999999999999e-05 2023-09-23T21:02:40.646 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:40.646 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:40.646 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:02:40.646 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:01:45.593931+0000", 2023-09-23T21:02:40.646 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.9060000000000001e-06 2023-09-23T21:02:40.646 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:40.647 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:40.647 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:02:40.647 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:01:45.593968+0000", 2023-09-23T21:02:40.647 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.7348000000000002e-05 2023-09-23T21:02:40.647 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:40.647 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:40.647 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:02:40.647 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:01:45.594118+0000", 2023-09-23T21:02:40.647 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000149505 2023-09-23T21:02:40.647 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:40.647 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:40.647 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 0,5", 2023-09-23T21:02:40.648 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:01:45.594261+0000", 2023-09-23T21:02:40.648 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000142898 2023-09-23T21:02:40.648 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:40.648 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:40.648 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:02:40.648 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:01:45.595709+0000", 2023-09-23T21:02:40.648 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.001448401 2023-09-23T21:02:40.648 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:40.648 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:40.648 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:02:40.648 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:01:45.598407+0000", 2023-09-23T21:02:40.648 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0026978649999999998 2023-09-23T21:02:40.649 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:40.649 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:40.649 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:02:40.649 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:01:45.721746+0000", 2023-09-23T21:02:40.649 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.123338667 2023-09-23T21:02:40.649 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:40.649 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:40.649 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:02:40.649 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:01:45.721788+0000", 2023-09-23T21:02:40.649 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.2216999999999999e-05 2023-09-23T21:02:40.649 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:40.649 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:40.649 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:02:40.650 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:01:45.721832+0000", 2023-09-23T21:02:40.650 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.4508999999999997e-05 2023-09-23T21:02:40.650 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:02:40.650 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:02:40.650 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:02:40.650 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:40.650 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:40.650 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.35469.0:32287 8.9 8:9972558c:::benchmark_data_smithi026_301269_object32286:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e258)", 2023-09-23T21:02:40.650 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:01:45.599821+0000", 2023-09-23T21:02:40.650 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 54.915671191000001, 2023-09-23T21:02:40.650 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.12320073400000001, 2023-09-23T21:02:40.650 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:02:40.651 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:02:40.651 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:02:40.651 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.35469", 2023-09-23T21:02:40.651 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/4041424623", 2023-09-23T21:02:40.651 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 32287 2023-09-23T21:02:40.651 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:40.651 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:02:40.651 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:40.651 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:02:40.651 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:01:45.599821+0000", 2023-09-23T21:02:40.651 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:02:40.651 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:40.652 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:40.652 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:02:40.652 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:01:45.599821+0000", 2023-09-23T21:02:40.652 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:02:40.652 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:40.652 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:40.652 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:02:40.652 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:01:45.599820+0000", 2023-09-23T21:02:40.652 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T21:02:40.652 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:40.652 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:40.652 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:02:40.652 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:01:45.600479+0000", 2023-09-23T21:02:40.653 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00065974299999999996 2023-09-23T21:02:40.653 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:40.653 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:40.653 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:02:40.653 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:01:45.600494+0000", 2023-09-23T21:02:40.653 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.4545999999999999e-05 2023-09-23T21:02:40.653 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:40.653 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:40.653 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:02:40.653 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:01:45.600499+0000", 2023-09-23T21:02:40.653 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.2120000000000002e-06 2023-09-23T21:02:40.653 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:40.654 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:40.654 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:02:40.654 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:01:45.600551+0000", 2023-09-23T21:02:40.654 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.2065999999999999e-05 2023-09-23T21:02:40.654 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:40.654 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:40.654 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:02:40.654 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:01:45.600718+0000", 2023-09-23T21:02:40.654 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00016687599999999999 2023-09-23T21:02:40.654 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:40.654 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:40.654 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 0,5", 2023-09-23T21:02:40.655 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:01:45.600838+0000", 2023-09-23T21:02:40.655 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000119998 2023-09-23T21:02:40.655 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:40.655 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:40.655 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:02:40.655 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:01:45.603516+0000", 2023-09-23T21:02:40.655 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0026779400000000002 2023-09-23T21:02:40.655 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:40.655 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:40.655 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:02:40.655 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:01:45.603529+0000", 2023-09-23T21:02:40.655 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.2928999999999999e-05 2023-09-23T21:02:40.655 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:40.656 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:40.656 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:02:40.656 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:01:45.722921+0000", 2023-09-23T21:02:40.656 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.119392413 2023-09-23T21:02:40.656 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:40.656 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:40.656 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:02:40.656 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:01:45.722978+0000", 2023-09-23T21:02:40.656 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.7216999999999998e-05 2023-09-23T21:02:40.656 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:40.656 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:40.656 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:02:40.657 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:01:45.723022+0000", 2023-09-23T21:02:40.657 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.3423000000000001e-05 2023-09-23T21:02:40.657 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:02:40.657 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:02:40.657 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:02:40.657 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:40.657 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:40.657 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.35469.0:32296 8.9 8:9c0e9cd4:::benchmark_data_smithi026_301269_object32295:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e258)", 2023-09-23T21:02:40.657 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:01:45.601673+0000", 2023-09-23T21:02:40.657 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 54.913819156000002, 2023-09-23T21:02:40.657 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.12187394, 2023-09-23T21:02:40.657 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:02:40.658 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:02:40.658 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:02:40.658 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.35469", 2023-09-23T21:02:40.658 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/4041424623", 2023-09-23T21:02:40.658 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 32296 2023-09-23T21:02:40.658 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:40.658 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:02:40.658 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:40.658 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:02:40.658 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:01:45.601673+0000", 2023-09-23T21:02:40.658 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:02:40.658 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:40.659 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:40.659 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:02:40.659 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:01:45.601673+0000", 2023-09-23T21:02:40.659 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:02:40.659 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:40.659 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:40.659 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:02:40.659 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:01:45.601673+0000", 2023-09-23T21:02:40.659 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999995 2023-09-23T21:02:40.659 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:40.659 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:40.659 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:02:40.660 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:01:45.601740+0000", 2023-09-23T21:02:40.660 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.7317999999999995e-05 2023-09-23T21:02:40.660 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:40.660 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:40.660 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:02:40.660 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:01:45.601745+0000", 2023-09-23T21:02:40.660 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.1569999999999999e-06 2023-09-23T21:02:40.660 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:40.660 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:40.660 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:02:40.660 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:01:45.601748+0000", 2023-09-23T21:02:40.660 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.2119999999999999e-06 2023-09-23T21:02:40.661 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:40.661 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:40.661 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:02:40.661 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:01:45.602167+0000", 2023-09-23T21:02:40.661 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00041881500000000001 2023-09-23T21:02:40.661 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:40.661 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:40.661 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:02:40.661 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:01:45.602314+0000", 2023-09-23T21:02:40.661 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00014687700000000001 2023-09-23T21:02:40.661 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:40.661 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:40.661 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 0,5", 2023-09-23T21:02:40.662 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:01:45.602459+0000", 2023-09-23T21:02:40.662 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000144685 2023-09-23T21:02:40.662 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:40.662 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:40.662 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:02:40.662 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:01:45.603679+0000", 2023-09-23T21:02:40.662 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0012202980000000001 2023-09-23T21:02:40.662 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:40.662 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:40.662 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:02:40.662 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:01:45.603823+0000", 2023-09-23T21:02:40.662 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000143938 2023-09-23T21:02:40.663 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:40.663 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:40.663 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:02:40.663 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:01:45.723450+0000", 2023-09-23T21:02:40.663 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.11962711500000001 2023-09-23T21:02:40.663 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:40.663 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:40.663 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:02:40.663 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:01:45.723503+0000", 2023-09-23T21:02:40.663 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.2828999999999999e-05 2023-09-23T21:02:40.663 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:02:40.663 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:02:40.664 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:02:40.664 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:01:45.723547+0000", 2023-09-23T21:02:40.664 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.4249999999999998e-05 2023-09-23T21:02:40.664 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:02:40.664 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:02:40.664 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:02:40.664 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:02:40.664 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:02:40.664 INFO:teuthology.orchestra.run.smithi070.stdout:} 2023-09-23T21:02:40.665 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:02:40.665 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T21:02:40.666 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:02:40.766 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:02:40.773 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 69 16 94090 94074 85.1566 52.5 0.0180594 0.0117383 2023-09-23T21:02:40.827 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:02:40.876 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:02:40.882 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:02:40.882 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:02:40.919 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_ops_in_flight 2023-09-23T21:02:40.984 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:02:41.015 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:02:41.065 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:02:41.068 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:02:41.069 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:02:41.170 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:02:41.232 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:02:41 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T21:02:41.232 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:02:41 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/2643906003' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-09-23T21:02:41.232 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:02:41 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T21:02:41.232 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:02:41 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:02:41.239 INFO:teuthology.orchestra.run.smithi026.stderr:reweighted osd.5 to 1 (10000) 2023-09-23T21:02:41.267 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:02:41.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:02:41 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T21:02:41.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:02:41 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/2643906003' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-09-23T21:02:41.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:02:41 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T21:02:41.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:02:41 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:02:41.639 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:02:41 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T21:02:41.639 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:02:41 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/2643906003' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-09-23T21:02:41.639 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:02:41 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T21:02:41.640 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:02:41 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:02:41.662 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:02:41.670 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:02:41.670 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:02:41.771 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:02:41.775 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 70 16 94947 94931 84.7021 53.5625 0.0130161 0.0118016 2023-09-23T21:02:41.782 INFO:tasks.cephadm.ceph_manager.ceph:waiting for recovery to complete 2023-09-23T21:02:41.783 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T21:02:41.813 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:02:41.863 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:02:41.867 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:02:41.867 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:02:41.968 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:02:42.020 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:02:42.075 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:02:42.078 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:02:42.079 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:02:42.107 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T21:02:42.180 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:02:42.209 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:02:42.240 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:02:42.249 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:02:42.252 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:02:42.252 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:02:42.353 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:02:42.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:02:42 smithi070 ceph-mon[168282]: pgmap v718: 109 pgs: 109 active+clean; 5.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 54 MiB/s wr, 865 op/s 2023-09-23T21:02:42.371 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:02:42 smithi070 ceph-mon[168282]: failed to encode map e273 with expected crc 2023-09-23T21:02:42.371 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:02:42 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/2643906003' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-09-23T21:02:42.371 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:02:42 smithi070 ceph-mon[168282]: osdmap e273: 8 total, 8 up, 8 in 2023-09-23T21:02:42.371 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:02:42 smithi070 ceph-mon[168282]: failed to encode map e273 with expected crc 2023-09-23T21:02:42.371 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:02:42 smithi070 ceph-mon[168282]: failed to encode map e273 with expected crc 2023-09-23T21:02:42.387 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:02:42.428 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:02:42.430 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:02:42.430 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:02:42.531 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:02:42.569 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:02:42 smithi026 ceph-mon[175151]: pgmap v718: 109 pgs: 109 active+clean; 5.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 54 MiB/s wr, 865 op/s 2023-09-23T21:02:42.570 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:02:42 smithi026 ceph-mon[175151]: failed to encode map e273 with expected crc 2023-09-23T21:02:42.570 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:02:42 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/2643906003' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-09-23T21:02:42.570 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:02:42 smithi026 ceph-mon[175151]: osdmap e273: 8 total, 8 up, 8 in 2023-09-23T21:02:42.570 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:02:42 smithi026 ceph-mon[175151]: failed to encode map e273 with expected crc 2023-09-23T21:02:42.570 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:02:42 smithi026 ceph-mon[175151]: failed to encode map e273 with expected crc 2023-09-23T21:02:42.570 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:02:42 smithi026 ceph-mon[179688]: pgmap v718: 109 pgs: 109 active+clean; 5.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 54 MiB/s wr, 865 op/s 2023-09-23T21:02:42.570 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:02:42 smithi026 ceph-mon[179688]: failed to encode map e273 with expected crc 2023-09-23T21:02:42.570 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:02:42 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/2643906003' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-09-23T21:02:42.570 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:02:42 smithi026 ceph-mon[179688]: osdmap e273: 8 total, 8 up, 8 in 2023-09-23T21:02:42.571 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:02:42 smithi026 ceph-mon[179688]: failed to encode map e273 with expected crc 2023-09-23T21:02:42.571 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:02:42 smithi026 ceph-mon[179688]: failed to encode map e273 with expected crc 2023-09-23T21:02:42.627 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:02:42.713 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:02:42.717 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:02:42.718 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:02:42.776 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 71 16 95674 95658 84.1493 45.4375 0.0156903 0.0118784 2023-09-23T21:02:42.819 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:02:42.902 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:02:42.988 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:02:43.040 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:02:43.049 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:02:43.050 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:02:43.150 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:02:43.207 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:02:43.258 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:02:43 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/540347353' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T21:02:43.258 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:02:43 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/540347353' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T21:02:43.261 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:02:43.266 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:02:43.267 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:02:43.274 INFO:teuthology.orchestra.run.smithi026.stderr:noscrub is set 2023-09-23T21:02:43.368 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:02:43.407 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:02:43.416 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:02:43.458 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:02:43.460 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:02:43 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/540347353' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T21:02:43.461 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:02:43.461 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:02:43.562 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:02:43.651 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:02:43.776 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 72 16 96549 96533 83.7403 54.6875 0.0258335 0.0119364 2023-09-23T21:02:43.828 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:02:43.839 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:02:43.839 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:02:43.940 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:02:44.037 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:02:44.110 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:02:44.120 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:02:44.121 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:02:44.221 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:02:44.328 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:02:44.387 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:02:44 smithi026 ceph-mon[175151]: pgmap v720: 109 pgs: 109 active+clean; 5.7 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 55 MiB/s wr, 874 op/s 2023-09-23T21:02:44.388 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:02:44 smithi026 ceph-mon[175151]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:02:44.388 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:02:44 smithi026 ceph-mon[175151]: failed to encode map e274 with expected crc 2023-09-23T21:02:44.388 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:02:44 smithi026 ceph-mon[175151]: failed to encode map e274 with expected crc 2023-09-23T21:02:44.388 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:02:44 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/540347353' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T21:02:44.388 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:02:44 smithi026 ceph-mon[175151]: osdmap e274: 8 total, 8 up, 8 in 2023-09-23T21:02:44.388 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:02:44 smithi026 ceph-mon[175151]: failed to encode map e274 with expected crc 2023-09-23T21:02:44.389 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:02:44 smithi026 ceph-mon[179688]: pgmap v720: 109 pgs: 109 active+clean; 5.7 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 55 MiB/s wr, 874 op/s 2023-09-23T21:02:44.389 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:02:44 smithi026 ceph-mon[179688]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:02:44.389 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:02:44 smithi026 ceph-mon[179688]: failed to encode map e274 with expected crc 2023-09-23T21:02:44.389 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:02:44 smithi026 ceph-mon[179688]: failed to encode map e274 with expected crc 2023-09-23T21:02:44.389 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:02:44 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/540347353' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T21:02:44.389 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:02:44 smithi026 ceph-mon[179688]: osdmap e274: 8 total, 8 up, 8 in 2023-09-23T21:02:44.389 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:02:44 smithi026 ceph-mon[179688]: failed to encode map e274 with expected crc 2023-09-23T21:02:44.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:02:44 smithi070 ceph-mon[168282]: pgmap v720: 109 pgs: 109 active+clean; 5.7 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 55 MiB/s wr, 874 op/s 2023-09-23T21:02:44.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:02:44 smithi070 ceph-mon[168282]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:02:44.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:02:44 smithi070 ceph-mon[168282]: failed to encode map e274 with expected crc 2023-09-23T21:02:44.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:02:44 smithi070 ceph-mon[168282]: failed to encode map e274 with expected crc 2023-09-23T21:02:44.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:02:44 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/540347353' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T21:02:44.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:02:44 smithi070 ceph-mon[168282]: osdmap e274: 8 total, 8 up, 8 in 2023-09-23T21:02:44.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:02:44 smithi070 ceph-mon[168282]: failed to encode map e274 with expected crc 2023-09-23T21:02:44.654 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:02:44.676 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:02:44.677 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:02:44.715 INFO:teuthology.orchestra.run.smithi026.stdout:{ 2023-09-23T21:02:44.715 INFO:teuthology.orchestra.run.smithi026.stdout: "ops": [ 2023-09-23T21:02:44.715 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:44.716 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35469.0:97274 8.e 8:79b4bbc3:::benchmark_data_smithi026_301269_object97273:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e274)", 2023-09-23T21:02:44.716 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T21:02:44.621834+0000", 2023-09-23T21:02:44.716 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 0.01137209, 2023-09-23T21:02:44.716 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.011410709, 2023-09-23T21:02:44.716 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:02:44.716 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "waiting for sub ops", 2023-09-23T21:02:44.716 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T21:02:44.716 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35469", 2023-09-23T21:02:44.716 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/4041424623", 2023-09-23T21:02:44.716 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 97274 2023-09-23T21:02:44.716 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:44.716 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:02:44.717 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:44.717 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:02:44.717 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:02:44.621834+0000", 2023-09-23T21:02:44.717 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:02:44.717 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:44.717 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:44.717 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:02:44.717 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:02:44.621834+0000", 2023-09-23T21:02:44.717 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:02:44.717 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:44.717 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:44.717 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:02:44.718 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:02:44.621832+0000", 2023-09-23T21:02:44.718 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T21:02:44.718 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:44.718 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:44.718 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:02:44.718 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:02:44.621865+0000", 2023-09-23T21:02:44.718 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.3077000000000002e-05 2023-09-23T21:02:44.718 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:44.718 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:44.719 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:02:44.719 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:02:44.621881+0000", 2023-09-23T21:02:44.719 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.5951e-05 2023-09-23T21:02:44.719 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:44.719 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:44.719 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:02:44.719 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:02:44.621890+0000", 2023-09-23T21:02:44.719 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.1309999999999995e-06 2023-09-23T21:02:44.719 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:44.719 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:44.719 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:02:44.719 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:02:44.621963+0000", 2023-09-23T21:02:44.719 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.3015000000000005e-05 2023-09-23T21:02:44.720 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:44.720 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:44.720 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:02:44.720 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:02:44.622101+0000", 2023-09-23T21:02:44.720 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000137661 2023-09-23T21:02:44.720 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:44.720 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:44.720 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 2,4", 2023-09-23T21:02:44.720 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:02:44.622238+0000", 2023-09-23T21:02:44.720 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00013733199999999999 2023-09-23T21:02:44.720 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:44.720 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:44.720 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:02:44.721 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:02:44.631637+0000", 2023-09-23T21:02:44.721 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0093991309999999998 2023-09-23T21:02:44.721 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:44.721 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:44.721 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T21:02:44.721 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:02:44.631645+0000", 2023-09-23T21:02:44.721 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.8509999999999999e-06 2023-09-23T21:02:44.721 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:02:44.721 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:02:44.721 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:02:44.721 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:44.721 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:44.721 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35469.0:97281 8.e 8:73bf6688:::benchmark_data_smithi026_301269_object97280:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e274)", 2023-09-23T21:02:44.722 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T21:02:44.626354+0000", 2023-09-23T21:02:44.722 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 0.006851551, 2023-09-23T21:02:44.722 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0070002880000000003, 2023-09-23T21:02:44.722 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:02:44.722 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "waiting for sub ops", 2023-09-23T21:02:44.722 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T21:02:44.722 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35469", 2023-09-23T21:02:44.722 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/4041424623", 2023-09-23T21:02:44.722 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 97281 2023-09-23T21:02:44.723 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:44.723 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:02:44.723 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:44.723 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:02:44.723 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:02:44.626354+0000", 2023-09-23T21:02:44.723 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:02:44.723 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:44.723 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:44.723 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:02:44.723 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:02:44.626354+0000", 2023-09-23T21:02:44.723 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:02:44.724 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:44.724 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:44.724 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:02:44.724 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:02:44.626353+0000", 2023-09-23T21:02:44.724 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999981 2023-09-23T21:02:44.724 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:44.724 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:44.724 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:02:44.724 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:02:44.626378+0000", 2023-09-23T21:02:44.724 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.5247000000000001e-05 2023-09-23T21:02:44.724 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:44.725 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:44.725 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:02:44.725 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:02:44.626396+0000", 2023-09-23T21:02:44.725 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.8457999999999998e-05 2023-09-23T21:02:44.725 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:44.725 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:44.725 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:02:44.725 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:02:44.626405+0000", 2023-09-23T21:02:44.725 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.0389999999999993e-06 2023-09-23T21:02:44.725 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:44.725 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:44.726 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:02:44.726 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:02:44.631675+0000", 2023-09-23T21:02:44.726 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0052701010000000001 2023-09-23T21:02:44.726 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:44.726 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:44.726 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:02:44.726 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:02:44.631840+0000", 2023-09-23T21:02:44.726 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00016432999999999999 2023-09-23T21:02:44.726 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:44.726 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:44.727 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 2,4", 2023-09-23T21:02:44.727 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:02:44.632002+0000", 2023-09-23T21:02:44.727 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00016242599999999999 2023-09-23T21:02:44.727 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:02:44.727 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:02:44.727 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:02:44.727 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:02:44.727 INFO:teuthology.orchestra.run.smithi026.stdout: ], 2023-09-23T21:02:44.727 INFO:teuthology.orchestra.run.smithi026.stdout: "num_ops": 2 2023-09-23T21:02:44.727 INFO:teuthology.orchestra.run.smithi026.stdout:} 2023-09-23T21:02:44.776 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 73 16 97382 97366 83.3065 52.0625 0.0117265 0.0119993 2023-09-23T21:02:44.778 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:02:44.824 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:02:44.878 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:02:44.881 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:02:44.881 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:02:44.982 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:02:45.054 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:02:45.544 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:02:45.553 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:02:45.554 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:02:45.654 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:02:45.686 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:02:45.730 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:02:45.736 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:02:45.737 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:02:45.776 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 74 16 98273 98257 82.9334 55.6875 0.0196993 0.0120529 2023-09-23T21:02:45.794 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_blocked_ops 2023-09-23T21:02:45.838 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:02:45.886 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T21:02:45.965 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:02:46.169 INFO:teuthology.orchestra.run.smithi026.stdout: 2023-09-23T21:02:46.180 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T21:02:46.190 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:02:46.190 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:02:46.210 INFO:teuthology.orchestra.run.smithi026.stderr:dumped all 2023-09-23T21:02:46.291 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:02:46.374 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0: {} 2023-09-23T21:02:46.390 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:02:46 smithi026 ceph-mon[179688]: pgmap v722: 109 pgs: 109 active+clean; 5.8 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 60 MiB/s wr, 965 op/s 2023-09-23T21:02:46.390 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:02:46 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:02:46.390 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:02:46 smithi026 ceph-mon[175151]: pgmap v722: 109 pgs: 109 active+clean; 5.8 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 60 MiB/s wr, 965 op/s 2023-09-23T21:02:46.390 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:02:46 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:02:46.392 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:02:46.421 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1: {} 2023-09-23T21:02:46.461 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2: {} 2023-09-23T21:02:46.511 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3: {} 2023-09-23T21:02:46.551 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4: {} 2023-09-23T21:02:46.589 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T21:02:46.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:02:46 smithi070 ceph-mon[168282]: pgmap v722: 109 pgs: 109 active+clean; 5.8 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 60 MiB/s wr, 965 op/s 2023-09-23T21:02:46.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:02:46 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:02:46.629 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:02:46.633 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6: {} 2023-09-23T21:02:46.635 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:02:46.636 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:02:46.664 INFO:teuthology.orchestra.run.smithi026.stdout:{"pg_ready":true,"pg_map":{"version":722,"stamp":"2023-09-23T21:02:44.550058+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":6262888352,"num_objects":95579,"num_object_clones":0,"num_object_copies":286737,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":95579,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":95616,"num_write_kb":6115549,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":22,"num_bytes_recovered":5990,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":98991,"ondisk_log_size":98991,"up":327,"acting":327,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":330,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":20991312,"kb_used_data":18767760,"kb_used_omap":136,"kb_used_meta":2223351,"kb_avail":728969904,"statfs":{"total":767960285184,"available":746465181696,"internally_reserved":0,"allocated":19218186240,"data_stored":19209553722,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":139741,"internal_metadata":2276711971},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,12,12,2,3],"upper_bound":64},"perf_stat":{"commit_latency_ms":12,"apply_latency_ms":12,"commit_latency_ns":12000000,"apply_latency_ns":12000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":506265600,"num_objects":7725,"num_object_clones":0,"num_object_copies":23175,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":7725,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7725,"num_write_kb":494400,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"8.002349"},"pg_stats":[{"pgid":"3.1a","version":"68'1","reported_seq":150,"reported_epoch":272,"state":"active+clean","last_fresh":"2023-09-23T21:02:36.168908+0000","last_change":"2023-09-23T20:54:13.946695+0000","last_active":"2023-09-23T21:02:36.168908+0000","last_peered":"2023-09-23T21:02:36.168908+0000","last_clean":"2023-09-23T21:02:36.168908+0000","last_became_active":"2023-09-23T20:54:13.945399+0000","last_became_peered":"2023-09-23T20:54:13.945399+0000","last_unstale":"2023-09-23T21:02:36.168908+0000","last_undegraded":"2023-09-23T21:02:36.168908+0000","last_fullsized":"2023-09-23T21:02:36.168908+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:32:52.365464+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":265,"reported_epoch":271,"state":"active+clean","last_fresh":"2023-09-23T21:02:30.345033+0000","last_change":"2023-09-23T20:54:13.942019+0000","last_active":"2023-09-23T21:02:30.345033+0000","last_peered":"2023-09-23T21:02:30.345033+0000","last_clean":"2023-09-23T21:02:30.345033+0000","last_became_active":"2023-09-23T20:54:13.941647+0000","last_became_peered":"2023-09-23T20:54:13.941647+0000","last_unstale":"2023-09-23T21:02:30.345033+0000","last_undegraded":"2023-09-23T21:02:30.345033+0000","last_fullsized":"2023-09-23T21:02:30.345033+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:30:54.400944+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":245,"reported_epoch":273,"state":"active+clean","last_fresh":"2023-09-23T21:02:41.218814+0000","last_change":"2023-09-23T20:54:13.941394+0000","last_active":"2023-09-23T21:02:41.218814+0000","last_peered":"2023-09-23T21:02:41.218814+0000","last_clean":"2023-09-23T21:02:41.218814+0000","last_became_active":"2023-09-23T20:54:13.941042+0000","last_became_peered":"2023-09-23T20:54:13.941042+0000","last_unstale":"2023-09-23T21:02:41.218814+0000","last_undegraded":"2023-09-23T21:02:41.218814+0000","last_fullsized":"2023-09-23T21:02:41.218814+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:19:41.401184+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":259,"reported_epoch":272,"state":"active+clean","last_fresh":"2023-09-23T21:02:36.165958+0000","last_change":"2023-09-23T20:57:44.979811+0000","last_active":"2023-09-23T21:02:36.165958+0000","last_peered":"2023-09-23T21:02:36.165958+0000","last_clean":"2023-09-23T21:02:36.165958+0000","last_became_active":"2023-09-23T20:57:44.979515+0000","last_became_peered":"2023-09-23T20:57:44.979515+0000","last_unstale":"2023-09-23T21:02:36.165958+0000","last_undegraded":"2023-09-23T21:02:36.165958+0000","last_fullsized":"2023-09-23T21:02:36.165958+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:38:22.162140+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":256,"reported_epoch":271,"state":"active+clean","last_fresh":"2023-09-23T21:02:30.344637+0000","last_change":"2023-09-23T20:51:10.833613+0000","last_active":"2023-09-23T21:02:30.344637+0000","last_peered":"2023-09-23T21:02:30.344637+0000","last_clean":"2023-09-23T21:02:30.344637+0000","last_became_active":"2023-09-23T20:51:10.833208+0000","last_became_peered":"2023-09-23T20:51:10.833208+0000","last_unstale":"2023-09-23T21:02:30.344637+0000","last_undegraded":"2023-09-23T21:02:30.344637+0000","last_fullsized":"2023-09-23T21:02:30.344637+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:51:28.246014+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":239,"reported_epoch":273,"state":"active+clean","last_fresh":"2023-09-23T21:02:41.218631+0000","last_change":"2023-09-23T20:55:35.958384+0000","last_active":"2023-09-23T21:02:41.218631+0000","last_peered":"2023-09-23T21:02:41.218631+0000","last_clean":"2023-09-23T21:02:41.218631+0000","last_became_active":"2023-09-23T20:55:35.954563+0000","last_became_peered":"2023-09-23T20:55:35.954563+0000","last_unstale":"2023-09-23T21:02:41.218631+0000","last_undegraded":"2023-09-23T21:02:41.218631+0000","last_fullsized":"2023-09-23T21:02:41.218631+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:37:48.332259+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":249,"reported_epoch":272,"state":"active+clean","last_fresh":"2023-09-23T21:02:36.163986+0000","last_change":"2023-09-23T20:54:31.885514+0000","last_active":"2023-09-23T21:02:36.163986+0000","last_peered":"2023-09-23T21:02:36.163986+0000","last_clean":"2023-09-23T21:02:36.163986+0000","last_became_active":"2023-09-23T20:54:31.884986+0000","last_became_peered":"2023-09-23T20:54:31.884986+0000","last_unstale":"2023-09-23T21:02:36.163986+0000","last_undegraded":"2023-09-23T21:02:36.163986+0000","last_fullsized":"2023-09-23T21:02:36.163986+0000","mapping_epoch":136,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:26:08.100220+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":253,"reported_epoch":271,"state":"active+clean","last_fresh":"2023-09-23T21:02:30.344735+0000","last_change":"2023-09-23T20:55:35.951889+0000","last_active":"2023-09-23T21:02:30.344735+0000","last_peered":"2023-09-23T21:02:30.344735+0000","last_clean":"2023-09-23T21:02:30.344735+0000","last_became_active":"2023-09-23T20:55:35.951563+0000","last_became_peered":"2023-09-23T20:55:35.951563+0000","last_unstale":"2023-09-23T21:02:30.344735+0000","last_undegraded":"2023-09-23T21:02:30.344735+0000","last_fullsized":"2023-09-23T21:02:30.344735+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:41:53.098781+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"79'161","reported_seq":561,"reported_epoch":272,"state":"active+clean","last_fresh":"2023-09-23T21:02:36.170556+0000","last_change":"2023-09-23T20:57:25.096559+0000","last_active":"2023-09-23T21:02:36.170556+0000","last_peered":"2023-09-23T21:02:36.170556+0000","last_clean":"2023-09-23T21:02:36.170556+0000","last_became_active":"2023-09-23T20:57:15.975674+0000","last_became_peered":"2023-09-23T20:57:15.975674+0000","last_unstale":"2023-09-23T21:02:36.170556+0000","last_undegraded":"2023-09-23T21:02:36.170556+0000","last_fullsized":"2023-09-23T21:02:36.170556+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:25.096401+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:25.096401+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:05:05.448404+0000","scrub_duration":0.002859393,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":146,"reported_epoch":272,"state":"active+clean","last_fresh":"2023-09-23T21:02:36.166965+0000","last_change":"2023-09-23T20:54:31.890678+0000","last_active":"2023-09-23T21:02:36.166965+0000","last_peered":"2023-09-23T21:02:36.166965+0000","last_clean":"2023-09-23T21:02:36.166965+0000","last_became_active":"2023-09-23T20:54:31.890469+0000","last_became_peered":"2023-09-23T20:54:31.890469+0000","last_unstale":"2023-09-23T21:02:36.166965+0000","last_undegraded":"2023-09-23T21:02:36.166965+0000","last_fullsized":"2023-09-23T21:02:36.166965+0000","mapping_epoch":136,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:55:08.790146+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":214,"reported_epoch":271,"state":"active+clean","last_fresh":"2023-09-23T21:02:30.344714+0000","last_change":"2023-09-23T20:55:35.952661+0000","last_active":"2023-09-23T21:02:30.344714+0000","last_peered":"2023-09-23T21:02:30.344714+0000","last_clean":"2023-09-23T21:02:30.344714+0000","last_became_active":"2023-09-23T20:55:35.952349+0000","last_became_peered":"2023-09-23T20:55:35.952349+0000","last_unstale":"2023-09-23T21:02:30.344714+0000","last_undegraded":"2023-09-23T21:02:30.344714+0000","last_fullsized":"2023-09-23T21:02:30.344714+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:00:03.110126+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":91,"reported_epoch":271,"state":"active+clean","last_fresh":"2023-09-23T21:02:30.345062+0000","last_change":"2023-09-23T20:56:45.988607+0000","last_active":"2023-09-23T21:02:30.345062+0000","last_peered":"2023-09-23T21:02:30.345062+0000","last_clean":"2023-09-23T21:02:30.345062+0000","last_became_active":"2023-09-23T20:56:45.988206+0000","last_became_peered":"2023-09-23T20:56:45.988206+0000","last_unstale":"2023-09-23T21:02:30.345062+0000","last_undegraded":"2023-09-23T21:02:30.345062+0000","last_fullsized":"2023-09-23T21:02:30.345062+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":205,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:01:19.783688+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,4],"acting":[3,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":90,"reported_epoch":272,"state":"active+clean","last_fresh":"2023-09-23T21:02:36.161735+0000","last_change":"2023-09-23T20:56:18.139142+0000","last_active":"2023-09-23T21:02:36.161735+0000","last_peered":"2023-09-23T21:02:36.161735+0000","last_clean":"2023-09-23T21:02:36.161735+0000","last_became_active":"2023-09-23T20:56:18.138844+0000","last_became_peered":"2023-09-23T20:56:18.138844+0000","last_unstale":"2023-09-23T21:02:36.161735+0000","last_undegraded":"2023-09-23T21:02:36.161735+0000","last_fullsized":"2023-09-23T21:02:36.161735+0000","mapping_epoch":192,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":193,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:09:02.217683+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,2],"acting":[5,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.14","version":"79'22","reported_seq":177,"reported_epoch":272,"state":"active+clean","last_fresh":"2023-09-23T21:02:36.169219+0000","last_change":"2023-09-23T20:55:35.949418+0000","last_active":"2023-09-23T21:02:36.169219+0000","last_peered":"2023-09-23T21:02:36.169219+0000","last_clean":"2023-09-23T21:02:36.169219+0000","last_became_active":"2023-09-23T20:55:35.948566+0000","last_became_peered":"2023-09-23T20:55:35.948566+0000","last_unstale":"2023-09-23T21:02:36.169219+0000","last_undegraded":"2023-09-23T21:02:36.169219+0000","last_fullsized":"2023-09-23T21:02:36.169219+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:59:37.433128+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4734,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":167,"reported_epoch":272,"state":"active+clean","last_fresh":"2023-09-23T21:02:36.169224+0000","last_change":"2023-09-23T20:55:35.949178+0000","last_active":"2023-09-23T21:02:36.169224+0000","last_peered":"2023-09-23T21:02:36.169224+0000","last_clean":"2023-09-23T21:02:36.169224+0000","last_became_active":"2023-09-23T20:55:35.948795+0000","last_became_peered":"2023-09-23T20:55:35.948795+0000","last_unstale":"2023-09-23T21:02:36.169224+0000","last_undegraded":"2023-09-23T21:02:36.169224+0000","last_fullsized":"2023-09-23T21:02:36.169224+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:13:08.996932+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.11","version":"79'161","reported_seq":503,"reported_epoch":272,"state":"active+clean","last_fresh":"2023-09-23T21:02:36.166789+0000","last_change":"2023-09-23T20:57:28.756796+0000","last_active":"2023-09-23T21:02:36.166789+0000","last_peered":"2023-09-23T21:02:36.166789+0000","last_clean":"2023-09-23T21:02:36.166789+0000","last_became_active":"2023-09-23T20:57:15.980606+0000","last_became_peered":"2023-09-23T20:57:15.980606+0000","last_unstale":"2023-09-23T21:02:36.166789+0000","last_undegraded":"2023-09-23T21:02:36.166789+0000","last_fullsized":"2023-09-23T21:02:36.166789+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:28.756634+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:28.756634+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:09:19.150791+0000","scrub_duration":0.0064925119999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":193,"reported_epoch":272,"state":"active+clean","last_fresh":"2023-09-23T21:02:36.169133+0000","last_change":"2023-09-23T20:56:45.992577+0000","last_active":"2023-09-23T21:02:36.169133+0000","last_peered":"2023-09-23T21:02:36.169133+0000","last_clean":"2023-09-23T21:02:36.169133+0000","last_became_active":"2023-09-23T20:56:45.992297+0000","last_became_peered":"2023-09-23T20:56:45.992297+0000","last_unstale":"2023-09-23T21:02:36.169133+0000","last_undegraded":"2023-09-23T21:02:36.169133+0000","last_fullsized":"2023-09-23T21:02:36.169133+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":205,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:00:45.547202+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,2],"acting":[4,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":238,"reported_epoch":271,"state":"active+clean","last_fresh":"2023-09-23T21:02:30.344961+0000","last_change":"2023-09-23T20:51:10.831262+0000","last_active":"2023-09-23T21:02:30.344961+0000","last_peered":"2023-09-23T21:02:30.344961+0000","last_clean":"2023-09-23T21:02:30.344961+0000","last_became_active":"2023-09-23T20:51:10.830931+0000","last_became_peered":"2023-09-23T20:51:10.830931+0000","last_unstale":"2023-09-23T21:02:30.344961+0000","last_undegraded":"2023-09-23T21:02:30.344961+0000","last_fullsized":"2023-09-23T21:02:30.344961+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:51:54.982505+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.10","version":"79'161","reported_seq":510,"reported_epoch":272,"state":"active+clean","last_fresh":"2023-09-23T21:02:36.160283+0000","last_change":"2023-09-23T20:57:20.708003+0000","last_active":"2023-09-23T21:02:36.160283+0000","last_peered":"2023-09-23T21:02:36.160283+0000","last_clean":"2023-09-23T21:02:36.160283+0000","last_became_active":"2023-09-23T20:57:15.975302+0000","last_became_peered":"2023-09-23T20:57:15.975302+0000","last_unstale":"2023-09-23T21:02:36.160283+0000","last_undegraded":"2023-09-23T21:02:36.160283+0000","last_fullsized":"2023-09-23T21:02:36.160283+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:20.707939+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:20.707939+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:16:51.721994+0000","scrub_duration":0.0017733409999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":249,"reported_epoch":272,"state":"active+clean","last_fresh":"2023-09-23T21:02:36.163351+0000","last_change":"2023-09-23T20:57:44.979736+0000","last_active":"2023-09-23T21:02:36.163351+0000","last_peered":"2023-09-23T21:02:36.163351+0000","last_clean":"2023-09-23T21:02:36.163351+0000","last_became_active":"2023-09-23T20:57:44.979356+0000","last_became_peered":"2023-09-23T20:57:44.979356+0000","last_unstale":"2023-09-23T21:02:36.163351+0000","last_undegraded":"2023-09-23T21:02:36.163351+0000","last_fullsized":"2023-09-23T21:02:36.163351+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:14:43.848832+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":146,"reported_epoch":272,"state":"active+clean","last_fresh":"2023-09-23T21:02:36.167047+0000","last_change":"2023-09-23T20:54:31.887814+0000","last_active":"2023-09-23T21:02:36.167047+0000","last_peered":"2023-09-23T21:02:36.167047+0000","last_clean":"2023-09-23T21:02:36.167047+0000","last_became_active":"2023-09-23T20:54:31.887195+0000","last_became_peered":"2023-09-23T20:54:31.887195+0000","last_unstale":"2023-09-23T21:02:36.167047+0000","last_undegraded":"2023-09-23T21:02:36.167047+0000","last_fullsized":"2023-09-23T21:02:36.167047+0000","mapping_epoch":136,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:10:52.089553+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.13","version":"79'161","reported_seq":575,"reported_epoch":273,"state":"active+clean","last_fresh":"2023-09-23T21:02:41.218777+0000","last_change":"2023-09-23T20:57:48.262970+0000","last_active":"2023-09-23T21:02:41.218777+0000","last_peered":"2023-09-23T21:02:41.218777+0000","last_clean":"2023-09-23T21:02:41.218777+0000","last_became_active":"2023-09-23T20:57:48.262663+0000","last_became_peered":"2023-09-23T20:57:48.262663+0000","last_unstale":"2023-09-23T21:02:41.218777+0000","last_undegraded":"2023-09-23T21:02:41.218777+0000","last_fullsized":"2023-09-23T21:02:41.218777+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:36.420362+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:36.420362+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:28:28.613492+0000","scrub_duration":0.028838444000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":217,"reported_epoch":272,"state":"active+clean","last_fresh":"2023-09-23T21:02:36.168496+0000","last_change":"2023-09-23T20:56:45.991822+0000","last_active":"2023-09-23T21:02:36.168496+0000","last_peered":"2023-09-23T21:02:36.168496+0000","last_clean":"2023-09-23T21:02:36.168496+0000","last_became_active":"2023-09-23T20:56:45.991460+0000","last_became_peered":"2023-09-23T20:56:45.991460+0000","last_unstale":"2023-09-23T21:02:36.168496+0000","last_undegraded":"2023-09-23T21:02:36.168496+0000","last_fullsized":"2023-09-23T21:02:36.168496+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":205,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:20:37.503317+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":231,"reported_epoch":272,"state":"active+clean","last_fresh":"2023-09-23T21:02:36.166083+0000","last_change":"2023-09-23T20:51:10.840849+0000","last_active":"2023-09-23T21:02:36.166083+0000","last_peered":"2023-09-23T21:02:36.166083+0000","last_clean":"2023-09-23T21:02:36.166083+0000","last_became_active":"2023-09-23T20:51:10.840593+0000","last_became_peered":"2023-09-23T20:51:10.840593+0000","last_unstale":"2023-09-23T21:02:36.166083+0000","last_undegraded":"2023-09-23T21:02:36.166083+0000","last_fullsized":"2023-09-23T21:02:36.166083+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:30:52.297625+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.12","version":"79'161","reported_seq":510,"reported_epoch":272,"state":"active+clean","last_fresh":"2023-09-23T21:02:36.161518+0000","last_change":"2023-09-23T20:57:21.750765+0000","last_active":"2023-09-23T21:02:36.161518+0000","last_peered":"2023-09-23T21:02:36.161518+0000","last_clean":"2023-09-23T21:02:36.161518+0000","last_became_active":"2023-09-23T20:57:15.979195+0000","last_became_peered":"2023-09-23T20:57:15.979195+0000","last_unstale":"2023-09-23T21:02:36.161518+0000","last_undegraded":"2023-09-23T21:02:36.161518+0000","last_fullsized":"2023-09-23T21:02:36.161518+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:21.750680+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:21.750680+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:03:41.915962+0000","scrub_duration":0.0092391680000000007,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":266,"reported_epoch":273,"state":"active+clean","last_fresh":"2023-09-23T21:02:41.217456+0000","last_change":"2023-09-23T20:55:36.943969+0000","last_active":"2023-09-23T21:02:41.217456+0000","last_peered":"2023-09-23T21:02:41.217456+0000","last_clean":"2023-09-23T21:02:41.217456+0000","last_became_active":"2023-09-23T20:55:36.943580+0000","last_became_peered":"2023-09-23T20:55:36.943580+0000","last_unstale":"2023-09-23T21:02:41.217456+0000","last_undegraded":"2023-09-23T21:02:41.217456+0000","last_fullsized":"2023-09-23T21:02:41.217456+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:49:06.969396+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":62,"reported_epoch":271,"state":"active+clean","last_fresh":"2023-09-23T21:02:34.858716+0000","last_change":"2023-09-23T20:57:44.980498+0000","last_active":"2023-09-23T21:02:34.858716+0000","last_peered":"2023-09-23T21:02:34.858716+0000","last_clean":"2023-09-23T21:02:34.858716+0000","last_became_active":"2023-09-23T20:57:44.980182+0000","last_became_peered":"2023-09-23T20:57:44.980182+0000","last_unstale":"2023-09-23T21:02:34.858716+0000","last_undegraded":"2023-09-23T21:02:34.858716+0000","last_fullsized":"2023-09-23T21:02:34.858716+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:34:51.620544+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.4","version":"273'5961","reported_seq":5988,"reported_epoch":273,"state":"active+clean","last_fresh":"2023-09-23T21:02:42.056868+0000","last_change":"2023-09-23T21:01:25.429865+0000","last_active":"2023-09-23T21:02:42.056868+0000","last_peered":"2023-09-23T21:02:42.056868+0000","last_clean":"2023-09-23T21:02:42.056868+0000","last_became_active":"2023-09-23T21:01:25.429555+0000","last_became_peered":"2023-09-23T21:01:25.429555+0000","last_unstale":"2023-09-23T21:02:42.056868+0000","last_undegraded":"2023-09-23T21:02:42.056868+0000","last_fullsized":"2023-09-23T21:02:42.056868+0000","mapping_epoch":254,"log_start":"0'0","ondisk_log_start":"0'0","created":254,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:01:24.325318+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:01:24.325318+0000","last_clean_scrub_stamp":"2023-09-23T21:01:24.325318+0000","objects_scrubbed":0,"log_size":5961,"log_dups_size":0,"ondisk_log_size":5961,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:28:38.557662+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":390660096,"num_objects":5961,"num_object_clones":0,"num_object_copies":17883,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5961,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5961,"num_write_kb":381504,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,2],"acting":[5,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.d","version":"79'161","reported_seq":555,"reported_epoch":272,"state":"active+clean","last_fresh":"2023-09-23T21:02:36.169302+0000","last_change":"2023-09-23T20:57:31.075213+0000","last_active":"2023-09-23T21:02:36.169302+0000","last_peered":"2023-09-23T21:02:36.169302+0000","last_clean":"2023-09-23T21:02:36.169302+0000","last_became_active":"2023-09-23T20:57:15.988188+0000","last_became_peered":"2023-09-23T20:57:15.988188+0000","last_unstale":"2023-09-23T21:02:36.169302+0000","last_undegraded":"2023-09-23T21:02:36.169302+0000","last_fullsized":"2023-09-23T21:02:36.169302+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:31.075098+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:31.075098+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:41:52.142362+0000","scrub_duration":0.010304610000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":204,"reported_epoch":272,"state":"active+clean","last_fresh":"2023-09-23T21:02:36.170205+0000","last_change":"2023-09-23T20:56:45.991841+0000","last_active":"2023-09-23T21:02:36.170205+0000","last_peered":"2023-09-23T21:02:36.170205+0000","last_clean":"2023-09-23T21:02:36.170205+0000","last_became_active":"2023-09-23T20:56:45.991491+0000","last_became_peered":"2023-09-23T20:56:45.991491+0000","last_unstale":"2023-09-23T21:02:36.170205+0000","last_undegraded":"2023-09-23T21:02:36.170205+0000","last_fullsized":"2023-09-23T21:02:36.170205+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":205,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:30:37.405319+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":163,"reported_epoch":272,"state":"active+clean","last_fresh":"2023-09-23T21:02:36.160494+0000","last_change":"2023-09-23T20:55:35.957768+0000","last_active":"2023-09-23T21:02:36.160494+0000","last_peered":"2023-09-23T21:02:36.160494+0000","last_clean":"2023-09-23T21:02:36.160494+0000","last_became_active":"2023-09-23T20:55:35.957421+0000","last_became_peered":"2023-09-23T20:55:35.957421+0000","last_unstale":"2023-09-23T21:02:36.160494+0000","last_undegraded":"2023-09-23T21:02:36.160494+0000","last_fullsized":"2023-09-23T21:02:36.160494+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:49:41.585961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"8.5","version":"274'6123","reported_seq":6151,"reported_epoch":274,"state":"active+clean","last_fresh":"2023-09-23T21:02:44.087543+0000","last_change":"2023-09-23T21:01:25.560708+0000","last_active":"2023-09-23T21:02:44.087543+0000","last_peered":"2023-09-23T21:02:44.087543+0000","last_clean":"2023-09-23T21:02:44.087543+0000","last_became_active":"2023-09-23T21:01:25.560472+0000","last_became_peered":"2023-09-23T21:01:25.560472+0000","last_unstale":"2023-09-23T21:02:44.087543+0000","last_undegraded":"2023-09-23T21:02:44.087543+0000","last_fullsized":"2023-09-23T21:02:44.087543+0000","mapping_epoch":254,"log_start":"0'0","ondisk_log_start":"0'0","created":254,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:01:24.325318+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:01:24.325318+0000","last_clean_scrub_stamp":"2023-09-23T21:01:24.325318+0000","objects_scrubbed":0,"log_size":6123,"log_dups_size":0,"ondisk_log_size":6123,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:04:02.549068+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":401276928,"num_objects":6123,"num_object_clones":0,"num_object_copies":18369,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6123,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6123,"num_write_kb":391872,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.c","version":"79'161","reported_seq":574,"reported_epoch":271,"state":"active+clean","last_fresh":"2023-09-23T21:02:30.344863+0000","last_change":"2023-09-23T20:57:22.041071+0000","last_active":"2023-09-23T21:02:30.344863+0000","last_peered":"2023-09-23T21:02:30.344863+0000","last_clean":"2023-09-23T21:02:30.344863+0000","last_became_active":"2023-09-23T20:57:15.973204+0000","last_became_peered":"2023-09-23T20:57:15.973204+0000","last_unstale":"2023-09-23T21:02:30.344863+0000","last_undegraded":"2023-09-23T21:02:30.344863+0000","last_fullsized":"2023-09-23T21:02:30.344863+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:22.040907+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:22.040907+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:47:02.019978+0000","scrub_duration":0.013021176000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":191,"reported_epoch":272,"state":"active+clean","last_fresh":"2023-09-23T21:02:36.170267+0000","last_change":"2023-09-23T20:56:45.991821+0000","last_active":"2023-09-23T21:02:36.170267+0000","last_peered":"2023-09-23T21:02:36.170267+0000","last_clean":"2023-09-23T21:02:36.170267+0000","last_became_active":"2023-09-23T20:56:45.991447+0000","last_became_peered":"2023-09-23T20:56:45.991447+0000","last_unstale":"2023-09-23T21:02:36.170267+0000","last_undegraded":"2023-09-23T21:02:36.170267+0000","last_fullsized":"2023-09-23T21:02:36.170267+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":205,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:14:25.525222+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":194,"reported_epoch":272,"state":"active+clean","last_fresh":"2023-09-23T21:02:36.168755+0000","last_change":"2023-09-23T20:57:44.982059+0000","last_active":"2023-09-23T21:02:36.168755+0000","last_peered":"2023-09-23T21:02:36.168755+0000","last_clean":"2023-09-23T21:02:36.168755+0000","last_became_active":"2023-09-23T20:57:44.981742+0000","last_became_peered":"2023-09-23T20:57:44.981742+0000","last_unstale":"2023-09-23T21:02:36.168755+0000","last_undegraded":"2023-09-23T21:02:36.168755+0000","last_fullsized":"2023-09-23T21:02:36.168755+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:41:24.042916+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.6","version":"273'6110","reported_seq":6138,"reported_epoch":273,"state":"active+clean","last_fresh":"2023-09-23T21:02:42.069655+0000","last_change":"2023-09-23T21:01:25.560370+0000","last_active":"2023-09-23T21:02:42.069655+0000","last_peered":"2023-09-23T21:02:42.069655+0000","last_clean":"2023-09-23T21:02:42.069655+0000","last_became_active":"2023-09-23T21:01:25.560018+0000","last_became_peered":"2023-09-23T21:01:25.560018+0000","last_unstale":"2023-09-23T21:02:42.069655+0000","last_undegraded":"2023-09-23T21:02:42.069655+0000","last_fullsized":"2023-09-23T21:02:42.069655+0000","mapping_epoch":254,"log_start":"0'0","ondisk_log_start":"0'0","created":254,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:01:24.325318+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:01:24.325318+0000","last_clean_scrub_stamp":"2023-09-23T21:01:24.325318+0000","objects_scrubbed":0,"log_size":6110,"log_dups_size":0,"ondisk_log_size":6110,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:45:39.782872+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":400424960,"num_objects":6110,"num_object_clones":0,"num_object_copies":18330,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6110,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6110,"num_write_kb":391040,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,7],"acting":[0,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.f","version":"79'161","reported_seq":578,"reported_epoch":273,"state":"active+clean","last_fresh":"2023-09-23T21:02:41.218470+0000","last_change":"2023-09-23T20:57:27.880841+0000","last_active":"2023-09-23T21:02:41.218470+0000","last_peered":"2023-09-23T21:02:41.218470+0000","last_clean":"2023-09-23T21:02:41.218470+0000","last_became_active":"2023-09-23T20:57:15.970925+0000","last_became_peered":"2023-09-23T20:57:15.970925+0000","last_unstale":"2023-09-23T21:02:41.218470+0000","last_undegraded":"2023-09-23T21:02:41.218470+0000","last_fullsized":"2023-09-23T21:02:41.218470+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:27.880776+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:27.880776+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:15:13.438014+0000","scrub_duration":0.0014004899999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":177,"reported_epoch":272,"state":"active+clean","last_fresh":"2023-09-23T21:02:36.166967+0000","last_change":"2023-09-23T20:56:45.989153+0000","last_active":"2023-09-23T21:02:36.166967+0000","last_peered":"2023-09-23T21:02:36.166967+0000","last_clean":"2023-09-23T21:02:36.166967+0000","last_became_active":"2023-09-23T20:56:45.988800+0000","last_became_peered":"2023-09-23T20:56:45.988800+0000","last_unstale":"2023-09-23T21:02:36.166967+0000","last_undegraded":"2023-09-23T21:02:36.166967+0000","last_fullsized":"2023-09-23T21:02:36.166967+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":205,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:04:41.130194+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":213,"reported_epoch":273,"state":"active+clean","last_fresh":"2023-09-23T21:02:41.217555+0000","last_change":"2023-09-23T20:55:35.965427+0000","last_active":"2023-09-23T21:02:41.217555+0000","last_peered":"2023-09-23T21:02:41.217555+0000","last_clean":"2023-09-23T21:02:41.217555+0000","last_became_active":"2023-09-23T20:55:35.963969+0000","last_became_peered":"2023-09-23T20:55:35.963969+0000","last_unstale":"2023-09-23T21:02:41.217555+0000","last_undegraded":"2023-09-23T21:02:41.217555+0000","last_fullsized":"2023-09-23T21:02:41.217555+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:38:40.612083+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.7","version":"274'6036","reported_seq":6065,"reported_epoch":274,"state":"active+clean","last_fresh":"2023-09-23T21:02:44.406930+0000","last_change":"2023-09-23T21:01:25.341425+0000","last_active":"2023-09-23T21:02:44.406930+0000","last_peered":"2023-09-23T21:02:44.406930+0000","last_clean":"2023-09-23T21:02:44.406930+0000","last_became_active":"2023-09-23T21:01:25.341114+0000","last_became_peered":"2023-09-23T21:01:25.341114+0000","last_unstale":"2023-09-23T21:02:44.406930+0000","last_undegraded":"2023-09-23T21:02:44.406930+0000","last_fullsized":"2023-09-23T21:02:44.406930+0000","mapping_epoch":254,"log_start":"0'0","ondisk_log_start":"0'0","created":254,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:01:24.325318+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:01:24.325318+0000","last_clean_scrub_stamp":"2023-09-23T21:01:24.325318+0000","objects_scrubbed":0,"log_size":6036,"log_dups_size":0,"ondisk_log_size":6036,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:15:08.338532+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":395575296,"num_objects":6036,"num_object_clones":0,"num_object_copies":18108,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6036,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6036,"num_write_kb":386304,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.e","version":"79'161","reported_seq":484,"reported_epoch":272,"state":"active+clean","last_fresh":"2023-09-23T21:02:36.170106+0000","last_change":"2023-09-23T20:57:34.132738+0000","last_active":"2023-09-23T21:02:36.170106+0000","last_peered":"2023-09-23T21:02:36.170106+0000","last_clean":"2023-09-23T21:02:36.170106+0000","last_became_active":"2023-09-23T20:57:15.974318+0000","last_became_peered":"2023-09-23T20:57:15.974318+0000","last_unstale":"2023-09-23T21:02:36.170106+0000","last_undegraded":"2023-09-23T21:02:36.170106+0000","last_fullsized":"2023-09-23T21:02:36.170106+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:34.132660+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:34.132660+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:07:07.739438+0000","scrub_duration":0.005229694,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":163,"reported_epoch":272,"state":"active+clean","last_fresh":"2023-09-23T21:02:36.168531+0000","last_change":"2023-09-23T20:55:35.962005+0000","last_active":"2023-09-23T21:02:36.168531+0000","last_peered":"2023-09-23T21:02:36.168531+0000","last_clean":"2023-09-23T21:02:36.168531+0000","last_became_active":"2023-09-23T20:55:35.961695+0000","last_became_peered":"2023-09-23T20:55:35.961695+0000","last_unstale":"2023-09-23T21:02:36.168531+0000","last_undegraded":"2023-09-23T21:02:36.168531+0000","last_fullsized":"2023-09-23T21:02:36.168531+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:28:22.876201+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":262,"reported_epoch":272,"state":"active+clean","last_fresh":"2023-09-23T21:02:36.162308+0000","last_change":"2023-09-23T20:54:13.928715+0000","last_active":"2023-09-23T21:02:36.162308+0000","last_peered":"2023-09-23T21:02:36.162308+0000","last_clean":"2023-09-23T21:02:36.162308+0000","last_became_active":"2023-09-23T20:54:13.928200+0000","last_became_peered":"2023-09-23T20:54:13.928200+0000","last_unstale":"2023-09-23T21:02:36.162308+0000","last_undegraded":"2023-09-23T21:02:36.162308+0000","last_fullsized":"2023-09-23T21:02:36.162308+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:41:16.927513+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.0","version":"272'5830","reported_seq":5857,"reported_epoch":272,"state":"active+clean","last_fresh":"2023-09-23T21:02:39.812653+0000","last_change":"2023-09-23T21:01:25.342305+0000","last_active":"2023-09-23T21:02:39.812653+0000","last_peered":"2023-09-23T21:02:39.812653+0000","last_clean":"2023-09-23T21:02:39.812653+0000","last_became_active":"2023-09-23T21:01:25.342042+0000","last_became_peered":"2023-09-23T21:01:25.342042+0000","last_unstale":"2023-09-23T21:02:39.812653+0000","last_undegraded":"2023-09-23T21:02:39.812653+0000","last_fullsized":"2023-09-23T21:02:39.812653+0000","mapping_epoch":254,"log_start":"0'0","ondisk_log_start":"0'0","created":254,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:01:24.325318+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:01:24.325318+0000","last_clean_scrub_stamp":"2023-09-23T21:01:24.325318+0000","objects_scrubbed":0,"log_size":5830,"log_dups_size":0,"ondisk_log_size":5830,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:06:29.755463+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":382074880,"num_objects":5830,"num_object_clones":0,"num_object_copies":17490,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5830,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5830,"num_write_kb":373120,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,1],"acting":[3,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.9","version":"79'161","reported_seq":646,"reported_epoch":271,"state":"active+clean","last_fresh":"2023-09-23T21:02:30.344625+0000","last_change":"2023-09-23T20:56:46.921939+0000","last_active":"2023-09-23T21:02:30.344625+0000","last_peered":"2023-09-23T21:02:30.344625+0000","last_clean":"2023-09-23T21:02:30.344625+0000","last_became_active":"2023-09-23T20:56:45.989908+0000","last_became_peered":"2023-09-23T20:56:45.989908+0000","last_unstale":"2023-09-23T21:02:30.344625+0000","last_undegraded":"2023-09-23T21:02:30.344625+0000","last_fullsized":"2023-09-23T21:02:30.344625+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":205,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:56:46.921739+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:56:46.921739+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:20:42.635110+0000","scrub_duration":0.0071954109999999996,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":263,"reported_epoch":271,"state":"active+clean","last_fresh":"2023-09-23T21:02:30.344615+0000","last_change":"2023-09-23T20:54:13.961788+0000","last_active":"2023-09-23T21:02:30.344615+0000","last_peered":"2023-09-23T21:02:30.344615+0000","last_clean":"2023-09-23T21:02:30.344615+0000","last_became_active":"2023-09-23T20:54:13.940937+0000","last_became_peered":"2023-09-23T20:54:13.940937+0000","last_unstale":"2023-09-23T21:02:30.344615+0000","last_undegraded":"2023-09-23T21:02:30.344615+0000","last_fullsized":"2023-09-23T21:02:30.344615+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:11:56.785299+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":167,"reported_epoch":272,"state":"active+clean","last_fresh":"2023-09-23T21:02:36.169312+0000","last_change":"2023-09-23T20:55:35.949670+0000","last_active":"2023-09-23T21:02:36.169312+0000","last_peered":"2023-09-23T21:02:36.169312+0000","last_clean":"2023-09-23T21:02:36.169312+0000","last_became_active":"2023-09-23T20:55:35.949140+0000","last_became_peered":"2023-09-23T20:55:35.949140+0000","last_unstale":"2023-09-23T21:02:36.169312+0000","last_undegraded":"2023-09-23T21:02:36.169312+0000","last_fullsized":"2023-09-23T21:02:36.169312+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:55:59.340303+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.1","version":"274'6198","reported_seq":6227,"reported_epoch":274,"state":"active+clean","last_fresh":"2023-09-23T21:02:44.416905+0000","last_change":"2023-09-23T21:01:25.429922+0000","last_active":"2023-09-23T21:02:44.416905+0000","last_peered":"2023-09-23T21:02:44.416905+0000","last_clean":"2023-09-23T21:02:44.416905+0000","last_became_active":"2023-09-23T21:01:25.429650+0000","last_became_peered":"2023-09-23T21:01:25.429650+0000","last_unstale":"2023-09-23T21:02:44.416905+0000","last_undegraded":"2023-09-23T21:02:44.416905+0000","last_fullsized":"2023-09-23T21:02:44.416905+0000","mapping_epoch":254,"log_start":"0'0","ondisk_log_start":"0'0","created":254,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:01:24.325318+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:01:24.325318+0000","last_clean_scrub_stamp":"2023-09-23T21:01:24.325318+0000","objects_scrubbed":0,"log_size":6198,"log_dups_size":0,"ondisk_log_size":6198,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:18:57.243292+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":406192128,"num_objects":6198,"num_object_clones":0,"num_object_copies":18594,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6198,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6198,"num_write_kb":396672,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,2],"acting":[1,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.8","version":"79'161","reported_seq":516,"reported_epoch":272,"state":"active+clean","last_fresh":"2023-09-23T21:02:36.160430+0000","last_change":"2023-09-23T20:56:57.505737+0000","last_active":"2023-09-23T21:02:36.160430+0000","last_peered":"2023-09-23T21:02:36.160430+0000","last_clean":"2023-09-23T21:02:36.160430+0000","last_became_active":"2023-09-23T20:56:04.982345+0000","last_became_peered":"2023-09-23T20:56:04.982345+0000","last_unstale":"2023-09-23T21:02:36.160430+0000","last_undegraded":"2023-09-23T21:02:36.160430+0000","last_fullsized":"2023-09-23T21:02:36.160430+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:56:57.505663+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:56:57.505663+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:34:30.109724+0000","scrub_duration":0.026284108,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":236,"reported_epoch":273,"state":"active+clean","last_fresh":"2023-09-23T21:02:41.217715+0000","last_change":"2023-09-23T20:55:35.959942+0000","last_active":"2023-09-23T21:02:41.217715+0000","last_peered":"2023-09-23T21:02:41.217715+0000","last_clean":"2023-09-23T21:02:41.217715+0000","last_became_active":"2023-09-23T20:55:35.957071+0000","last_became_peered":"2023-09-23T20:55:35.957071+0000","last_unstale":"2023-09-23T21:02:41.217715+0000","last_undegraded":"2023-09-23T21:02:41.217715+0000","last_fullsized":"2023-09-23T21:02:41.217715+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:44:30.308036+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":150,"reported_epoch":272,"state":"active+clean","last_fresh":"2023-09-23T21:02:36.169627+0000","last_change":"2023-09-23T20:54:13.951459+0000","last_active":"2023-09-23T21:02:36.169627+0000","last_peered":"2023-09-23T21:02:36.169627+0000","last_clean":"2023-09-23T21:02:36.169627+0000","last_became_active":"2023-09-23T20:54:13.948409+0000","last_became_peered":"2023-09-23T20:54:13.948409+0000","last_unstale":"2023-09-23T21:02:36.169627+0000","last_undegraded":"2023-09-23T21:02:36.169627+0000","last_fullsized":"2023-09-23T21:02:36.169627+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:38:10.148061+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.2","version":"273'6046","reported_seq":6073,"reported_epoch":273,"state":"active+clean","last_fresh":"2023-09-23T21:02:42.248636+0000","last_change":"2023-09-23T21:01:25.343022+0000","last_active":"2023-09-23T21:02:42.248636+0000","last_peered":"2023-09-23T21:02:42.248636+0000","last_clean":"2023-09-23T21:02:42.248636+0000","last_became_active":"2023-09-23T21:01:25.342540+0000","last_became_peered":"2023-09-23T21:01:25.342540+0000","last_unstale":"2023-09-23T21:02:42.248636+0000","last_undegraded":"2023-09-23T21:02:42.248636+0000","last_fullsized":"2023-09-23T21:02:42.248636+0000","mapping_epoch":254,"log_start":"0'0","ondisk_log_start":"0'0","created":254,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:01:24.325318+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:01:24.325318+0000","last_clean_scrub_stamp":"2023-09-23T21:01:24.325318+0000","objects_scrubbed":0,"log_size":6046,"log_dups_size":0,"ondisk_log_size":6046,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:57:19.741924+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":396230656,"num_objects":6046,"num_object_clones":0,"num_object_copies":18138,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6046,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6046,"num_write_kb":386944,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,5],"acting":[4,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.b","version":"79'161","reported_seq":558,"reported_epoch":273,"state":"active+clean","last_fresh":"2023-09-23T21:02:41.218869+0000","last_change":"2023-09-23T20:57:27.320172+0000","last_active":"2023-09-23T21:02:41.218869+0000","last_peered":"2023-09-23T21:02:41.218869+0000","last_clean":"2023-09-23T21:02:41.218869+0000","last_became_active":"2023-09-23T20:57:16.009630+0000","last_became_peered":"2023-09-23T20:57:16.009630+0000","last_unstale":"2023-09-23T21:02:41.218869+0000","last_undegraded":"2023-09-23T21:02:41.218869+0000","last_fullsized":"2023-09-23T21:02:41.218869+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:27.320022+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:27.320022+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:31:35.221558+0000","scrub_duration":0.0043894679999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,5],"acting":[1,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":150,"reported_epoch":272,"state":"active+clean","last_fresh":"2023-09-23T21:02:36.168576+0000","last_change":"2023-09-23T20:54:13.950254+0000","last_active":"2023-09-23T21:02:36.168576+0000","last_peered":"2023-09-23T21:02:36.168576+0000","last_clean":"2023-09-23T21:02:36.168576+0000","last_became_active":"2023-09-23T20:54:13.949934+0000","last_became_peered":"2023-09-23T20:54:13.949934+0000","last_unstale":"2023-09-23T21:02:36.168576+0000","last_undegraded":"2023-09-23T21:02:36.168576+0000","last_fullsized":"2023-09-23T21:02:36.168576+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:30:32.847167+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":167,"reported_epoch":272,"state":"active+clean","last_fresh":"2023-09-23T21:02:36.170297+0000","last_change":"2023-09-23T20:55:35.956153+0000","last_active":"2023-09-23T21:02:36.170297+0000","last_peered":"2023-09-23T21:02:36.170297+0000","last_clean":"2023-09-23T21:02:36.170297+0000","last_became_active":"2023-09-23T20:55:35.955630+0000","last_became_peered":"2023-09-23T20:55:35.955630+0000","last_unstale":"2023-09-23T21:02:36.170297+0000","last_undegraded":"2023-09-23T21:02:36.170297+0000","last_fullsized":"2023-09-23T21:02:36.170297+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:45:20.282929+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.3","version":"274'5855","reported_seq":5884,"reported_epoch":274,"state":"active+clean","last_fresh":"2023-09-23T21:02:44.422815+0000","last_change":"2023-09-23T21:01:25.342380+0000","last_active":"2023-09-23T21:02:44.422815+0000","last_peered":"2023-09-23T21:02:44.422815+0000","last_clean":"2023-09-23T21:02:44.422815+0000","last_became_active":"2023-09-23T21:01:25.342099+0000","last_became_peered":"2023-09-23T21:01:25.342099+0000","last_unstale":"2023-09-23T21:02:44.422815+0000","last_undegraded":"2023-09-23T21:02:44.422815+0000","last_fullsized":"2023-09-23T21:02:44.422815+0000","mapping_epoch":254,"log_start":"0'0","ondisk_log_start":"0'0","created":254,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:01:24.325318+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:01:24.325318+0000","last_clean_scrub_stamp":"2023-09-23T21:01:24.325318+0000","objects_scrubbed":0,"log_size":5855,"log_dups_size":0,"ondisk_log_size":5855,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:09:21.965969+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":383713280,"num_objects":5855,"num_object_clones":0,"num_object_copies":17565,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5855,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5855,"num_write_kb":374720,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.a","version":"79'161","reported_seq":516,"reported_epoch":272,"state":"active+clean","last_fresh":"2023-09-23T21:02:36.161830+0000","last_change":"2023-09-23T20:56:44.487778+0000","last_active":"2023-09-23T21:02:36.161830+0000","last_peered":"2023-09-23T21:02:36.161830+0000","last_clean":"2023-09-23T21:02:36.161830+0000","last_became_active":"2023-09-23T20:56:05.006740+0000","last_became_peered":"2023-09-23T20:56:05.006740+0000","last_unstale":"2023-09-23T21:02:36.161830+0000","last_undegraded":"2023-09-23T21:02:36.161830+0000","last_fullsized":"2023-09-23T21:02:36.161830+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:56:44.487678+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:56:44.487678+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:04:50.086749+0000","scrub_duration":0.0017408899999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":62,"reported_epoch":271,"state":"active+clean","last_fresh":"2023-09-23T21:02:30.344380+0000","last_change":"2023-09-23T20:57:44.982513+0000","last_active":"2023-09-23T21:02:30.344380+0000","last_peered":"2023-09-23T21:02:30.344380+0000","last_clean":"2023-09-23T21:02:30.344380+0000","last_became_active":"2023-09-23T20:57:44.982112+0000","last_became_peered":"2023-09-23T20:57:44.982112+0000","last_unstale":"2023-09-23T21:02:30.344380+0000","last_undegraded":"2023-09-23T21:02:30.344380+0000","last_fullsized":"2023-09-23T21:02:30.344380+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:09:05.630041+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":245,"reported_epoch":273,"state":"active+clean","last_fresh":"2023-09-23T21:02:41.223514+0000","last_change":"2023-09-23T20:54:13.940748+0000","last_active":"2023-09-23T21:02:41.223514+0000","last_peered":"2023-09-23T21:02:41.223514+0000","last_clean":"2023-09-23T21:02:41.223514+0000","last_became_active":"2023-09-23T20:54:13.940419+0000","last_became_peered":"2023-09-23T20:54:13.940419+0000","last_unstale":"2023-09-23T21:02:41.223514+0000","last_undegraded":"2023-09-23T21:02:41.223514+0000","last_fullsized":"2023-09-23T21:02:41.223514+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:15:01.923816+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.e","version":"273'6021","reported_seq":6049,"reported_epoch":273,"state":"active+clean","last_fresh":"2023-09-23T21:02:42.085076+0000","last_change":"2023-09-23T21:01:25.430111+0000","last_active":"2023-09-23T21:02:42.085076+0000","last_peered":"2023-09-23T21:02:42.085076+0000","last_clean":"2023-09-23T21:02:42.085076+0000","last_became_active":"2023-09-23T21:01:25.429811+0000","last_became_peered":"2023-09-23T21:01:25.429811+0000","last_unstale":"2023-09-23T21:02:42.085076+0000","last_undegraded":"2023-09-23T21:02:42.085076+0000","last_fullsized":"2023-09-23T21:02:42.085076+0000","mapping_epoch":254,"log_start":"0'0","ondisk_log_start":"0'0","created":254,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:01:24.325318+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:01:24.325318+0000","last_clean_scrub_stamp":"2023-09-23T21:01:24.325318+0000","objects_scrubbed":0,"log_size":6021,"log_dups_size":0,"ondisk_log_size":6021,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:18:01.918758+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":394592256,"num_objects":6021,"num_object_clones":0,"num_object_copies":18063,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6021,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6021,"num_write_kb":385344,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.7","version":"79'161","reported_seq":579,"reported_epoch":273,"state":"active+clean","last_fresh":"2023-09-23T21:02:41.218614+0000","last_change":"2023-09-23T20:57:48.243572+0000","last_active":"2023-09-23T21:02:41.218614+0000","last_peered":"2023-09-23T21:02:41.218614+0000","last_clean":"2023-09-23T21:02:41.218614+0000","last_became_active":"2023-09-23T20:57:48.243229+0000","last_became_peered":"2023-09-23T20:57:48.243229+0000","last_unstale":"2023-09-23T21:02:41.218614+0000","last_undegraded":"2023-09-23T21:02:41.218614+0000","last_fullsized":"2023-09-23T21:02:41.218614+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":3,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:25.916867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:25.916867+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:23:19.385831+0000","scrub_duration":0.001665323,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":251,"reported_epoch":273,"state":"active+clean","last_fresh":"2023-09-23T21:02:41.219195+0000","last_change":"2023-09-23T20:50:05.867130+0000","last_active":"2023-09-23T21:02:41.219195+0000","last_peered":"2023-09-23T21:02:41.219195+0000","last_clean":"2023-09-23T21:02:41.219195+0000","last_became_active":"2023-09-23T20:50:05.866758+0000","last_became_peered":"2023-09-23T20:50:05.866758+0000","last_unstale":"2023-09-23T21:02:41.219195+0000","last_undegraded":"2023-09-23T21:02:41.219195+0000","last_fullsized":"2023-09-23T21:02:41.219195+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:55:07.548209+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":146,"reported_epoch":272,"state":"active+clean","last_fresh":"2023-09-23T21:02:36.167121+0000","last_change":"2023-09-23T20:54:31.887604+0000","last_active":"2023-09-23T21:02:36.167121+0000","last_peered":"2023-09-23T21:02:36.167121+0000","last_clean":"2023-09-23T21:02:36.167121+0000","last_became_active":"2023-09-23T20:54:31.886752+0000","last_became_peered":"2023-09-23T20:54:31.886752+0000","last_unstale":"2023-09-23T21:02:36.167121+0000","last_undegraded":"2023-09-23T21:02:36.167121+0000","last_fullsized":"2023-09-23T21:02:36.167121+0000","mapping_epoch":136,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:13:58.872176+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":249,"reported_epoch":273,"state":"active+clean","last_fresh":"2023-09-23T21:02:41.218604+0000","last_change":"2023-09-23T20:54:31.899082+0000","last_active":"2023-09-23T21:02:41.218604+0000","last_peered":"2023-09-23T21:02:41.218604+0000","last_clean":"2023-09-23T21:02:41.218604+0000","last_became_active":"2023-09-23T20:54:31.898521+0000","last_became_peered":"2023-09-23T20:54:31.898521+0000","last_unstale":"2023-09-23T21:02:41.218604+0000","last_undegraded":"2023-09-23T21:02:41.218604+0000","last_fullsized":"2023-09-23T21:02:41.218604+0000","mapping_epoch":136,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:25:36.059711+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.8","version":"273'5947","reported_seq":5974,"reported_epoch":273,"state":"active+clean","last_fresh":"2023-09-23T21:02:42.077188+0000","last_change":"2023-09-23T21:01:25.560581+0000","last_active":"2023-09-23T21:02:42.077188+0000","last_peered":"2023-09-23T21:02:42.077188+0000","last_clean":"2023-09-23T21:02:42.077188+0000","last_became_active":"2023-09-23T21:01:25.560318+0000","last_became_peered":"2023-09-23T21:01:25.560318+0000","last_unstale":"2023-09-23T21:02:42.077188+0000","last_undegraded":"2023-09-23T21:02:42.077188+0000","last_fullsized":"2023-09-23T21:02:42.077188+0000","mapping_epoch":254,"log_start":"0'0","ondisk_log_start":"0'0","created":254,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:01:24.325318+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:01:24.325318+0000","last_clean_scrub_stamp":"2023-09-23T21:01:24.325318+0000","objects_scrubbed":0,"log_size":5947,"log_dups_size":0,"ondisk_log_size":5947,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:07:28.203012+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":389742592,"num_objects":5947,"num_object_clones":0,"num_object_copies":17841,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5947,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5947,"num_write_kb":380608,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[5,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.1","version":"79'161","reported_seq":536,"reported_epoch":272,"state":"active+clean","last_fresh":"2023-09-23T21:02:36.161863+0000","last_change":"2023-09-23T20:57:19.669346+0000","last_active":"2023-09-23T21:02:36.161863+0000","last_peered":"2023-09-23T21:02:36.161863+0000","last_clean":"2023-09-23T21:02:36.161863+0000","last_became_active":"2023-09-23T20:57:08.246046+0000","last_became_peered":"2023-09-23T20:57:08.246046+0000","last_unstale":"2023-09-23T21:02:36.161863+0000","last_undegraded":"2023-09-23T21:02:36.161863+0000","last_fullsized":"2023-09-23T21:02:36.161863+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:19.669259+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:19.669259+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:14:03.435531+0000","scrub_duration":0.0045759559999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":163,"reported_epoch":272,"state":"active+clean","last_fresh":"2023-09-23T21:02:36.161594+0000","last_change":"2023-09-23T20:55:35.959514+0000","last_active":"2023-09-23T21:02:36.161594+0000","last_peered":"2023-09-23T21:02:36.161594+0000","last_clean":"2023-09-23T21:02:36.161594+0000","last_became_active":"2023-09-23T20:55:35.959244+0000","last_became_peered":"2023-09-23T20:55:35.959244+0000","last_unstale":"2023-09-23T21:02:36.161594+0000","last_undegraded":"2023-09-23T21:02:36.161594+0000","last_fullsized":"2023-09-23T21:02:36.161594+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:47:32.665316+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":167,"reported_epoch":272,"state":"active+clean","last_fresh":"2023-09-23T21:02:36.169381+0000","last_change":"2023-09-23T20:55:35.949000+0000","last_active":"2023-09-23T21:02:36.169381+0000","last_peered":"2023-09-23T21:02:36.169381+0000","last_clean":"2023-09-23T21:02:36.169381+0000","last_became_active":"2023-09-23T20:55:35.948410+0000","last_became_peered":"2023-09-23T20:55:35.948410+0000","last_unstale":"2023-09-23T21:02:36.169381+0000","last_undegraded":"2023-09-23T21:02:36.169381+0000","last_fullsized":"2023-09-23T21:02:36.169381+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:26:50.477289+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":243,"reported_epoch":273,"state":"active+clean","last_fresh":"2023-09-23T21:02:41.218814+0000","last_change":"2023-09-23T20:51:10.829889+0000","last_active":"2023-09-23T21:02:41.218814+0000","last_peered":"2023-09-23T21:02:41.218814+0000","last_clean":"2023-09-23T21:02:41.218814+0000","last_became_active":"2023-09-23T20:51:10.829533+0000","last_became_peered":"2023-09-23T20:51:10.829533+0000","last_unstale":"2023-09-23T21:02:41.218814+0000","last_undegraded":"2023-09-23T21:02:41.218814+0000","last_fullsized":"2023-09-23T21:02:41.218814+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:43:04.603460+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.b","version":"272'5855","reported_seq":5882,"reported_epoch":272,"state":"active+clean","last_fresh":"2023-09-23T21:02:39.812535+0000","last_change":"2023-09-23T21:01:25.340569+0000","last_active":"2023-09-23T21:02:39.812535+0000","last_peered":"2023-09-23T21:02:39.812535+0000","last_clean":"2023-09-23T21:02:39.812535+0000","last_became_active":"2023-09-23T21:01:25.340209+0000","last_became_peered":"2023-09-23T21:01:25.340209+0000","last_unstale":"2023-09-23T21:02:39.812535+0000","last_undegraded":"2023-09-23T21:02:39.812535+0000","last_fullsized":"2023-09-23T21:02:39.812535+0000","mapping_epoch":254,"log_start":"0'0","ondisk_log_start":"0'0","created":254,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:01:24.325318+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:01:24.325318+0000","last_clean_scrub_stamp":"2023-09-23T21:01:24.325318+0000","objects_scrubbed":0,"log_size":5855,"log_dups_size":0,"ondisk_log_size":5855,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:53:59.553810+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":383713280,"num_objects":5855,"num_object_clones":0,"num_object_copies":17565,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5855,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5855,"num_write_kb":374720,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.2","version":"79'161","reported_seq":504,"reported_epoch":272,"state":"active+clean","last_fresh":"2023-09-23T21:02:36.163304+0000","last_change":"2023-09-23T20:57:24.802586+0000","last_active":"2023-09-23T21:02:36.163304+0000","last_peered":"2023-09-23T21:02:36.163304+0000","last_clean":"2023-09-23T21:02:36.163304+0000","last_became_active":"2023-09-23T20:57:08.236114+0000","last_became_peered":"2023-09-23T20:57:08.236114+0000","last_unstale":"2023-09-23T21:02:36.163304+0000","last_undegraded":"2023-09-23T21:02:36.163304+0000","last_fullsized":"2023-09-23T21:02:36.163304+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:24.802451+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:24.802451+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:06:22.309980+0000","scrub_duration":0.0090861250000000005,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,4],"acting":[0,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":260,"reported_epoch":272,"state":"active+clean","last_fresh":"2023-09-23T21:02:36.169953+0000","last_change":"2023-09-23T20:51:10.838054+0000","last_active":"2023-09-23T21:02:36.169953+0000","last_peered":"2023-09-23T21:02:36.169953+0000","last_clean":"2023-09-23T21:02:36.169953+0000","last_became_active":"2023-09-23T20:51:10.837750+0000","last_became_peered":"2023-09-23T20:51:10.837750+0000","last_unstale":"2023-09-23T21:02:36.169953+0000","last_undegraded":"2023-09-23T21:02:36.169953+0000","last_fullsized":"2023-09-23T21:02:36.169953+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:15:51.648085+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":243,"reported_epoch":273,"state":"active+clean","last_fresh":"2023-09-23T21:02:41.218294+0000","last_change":"2023-09-23T20:55:35.951146+0000","last_active":"2023-09-23T21:02:41.218294+0000","last_peered":"2023-09-23T21:02:41.218294+0000","last_clean":"2023-09-23T21:02:41.218294+0000","last_became_active":"2023-09-23T20:55:35.950874+0000","last_became_peered":"2023-09-23T20:55:35.950874+0000","last_unstale":"2023-09-23T21:02:41.218294+0000","last_undegraded":"2023-09-23T21:02:41.218294+0000","last_fullsized":"2023-09-23T21:02:41.218294+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:58:13.957677+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":263,"reported_epoch":273,"state":"active+clean","last_fresh":"2023-09-23T21:02:41.223405+0000","last_change":"2023-09-23T20:54:31.898870+0000","last_active":"2023-09-23T21:02:41.223405+0000","last_peered":"2023-09-23T21:02:41.223405+0000","last_clean":"2023-09-23T21:02:41.223405+0000","last_became_active":"2023-09-23T20:54:31.898467+0000","last_became_peered":"2023-09-23T20:54:31.898467+0000","last_unstale":"2023-09-23T21:02:41.223405+0000","last_undegraded":"2023-09-23T21:02:41.223405+0000","last_fullsized":"2023-09-23T21:02:41.223405+0000","mapping_epoch":136,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:56:22.483408+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.9","version":"274'5920","reported_seq":5948,"reported_epoch":274,"state":"active+clean","last_fresh":"2023-09-23T21:02:44.084651+0000","last_change":"2023-09-23T21:01:25.342548+0000","last_active":"2023-09-23T21:02:44.084651+0000","last_peered":"2023-09-23T21:02:44.084651+0000","last_clean":"2023-09-23T21:02:44.084651+0000","last_became_active":"2023-09-23T21:01:25.342227+0000","last_became_peered":"2023-09-23T21:01:25.342227+0000","last_unstale":"2023-09-23T21:02:44.084651+0000","last_undegraded":"2023-09-23T21:02:44.084651+0000","last_fullsized":"2023-09-23T21:02:44.084651+0000","mapping_epoch":254,"log_start":"0'0","ondisk_log_start":"0'0","created":254,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:01:24.325318+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:01:24.325318+0000","last_clean_scrub_stamp":"2023-09-23T21:01:24.325318+0000","objects_scrubbed":0,"log_size":5920,"log_dups_size":0,"ondisk_log_size":5920,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:38:15.944067+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":387973120,"num_objects":5920,"num_object_clones":0,"num_object_copies":17760,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5920,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5920,"num_write_kb":378880,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,0],"acting":[6,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":146,"reported_epoch":272,"state":"active+clean","last_fresh":"2023-09-23T21:02:36.160142+0000","last_change":"2023-09-23T20:54:31.883504+0000","last_active":"2023-09-23T21:02:36.160142+0000","last_peered":"2023-09-23T21:02:36.160142+0000","last_clean":"2023-09-23T21:02:36.160142+0000","last_became_active":"2023-09-23T20:54:31.883172+0000","last_became_peered":"2023-09-23T20:54:31.883172+0000","last_unstale":"2023-09-23T21:02:36.160142+0000","last_undegraded":"2023-09-23T21:02:36.160142+0000","last_fullsized":"2023-09-23T21:02:36.160142+0000","mapping_epoch":136,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:45:40.289120+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"79'161","reported_seq":504,"reported_epoch":272,"state":"active+clean","last_fresh":"2023-09-23T21:02:36.161951+0000","last_change":"2023-09-23T20:57:20.738417+0000","last_active":"2023-09-23T21:02:36.161951+0000","last_peered":"2023-09-23T21:02:36.161951+0000","last_clean":"2023-09-23T21:02:36.161951+0000","last_became_active":"2023-09-23T20:57:08.236842+0000","last_became_peered":"2023-09-23T20:57:08.236842+0000","last_unstale":"2023-09-23T21:02:36.161951+0000","last_undegraded":"2023-09-23T21:02:36.161951+0000","last_fullsized":"2023-09-23T21:02:36.161951+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":0,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:20.738273+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:20.738273+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:18:28.327990+0000","scrub_duration":0.0088336339999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":251,"reported_epoch":271,"state":"active+clean","last_fresh":"2023-09-23T21:02:30.344409+0000","last_change":"2023-09-23T20:55:35.950777+0000","last_active":"2023-09-23T21:02:30.344409+0000","last_peered":"2023-09-23T21:02:30.344409+0000","last_clean":"2023-09-23T21:02:30.344409+0000","last_became_active":"2023-09-23T20:55:35.950427+0000","last_became_peered":"2023-09-23T20:55:35.950427+0000","last_unstale":"2023-09-23T21:02:30.344409+0000","last_undegraded":"2023-09-23T21:02:30.344409+0000","last_fullsized":"2023-09-23T21:02:30.344409+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:20:10.371751+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":218,"reported_epoch":273,"state":"active+clean","last_fresh":"2023-09-23T21:02:41.218368+0000","last_change":"2023-09-23T20:55:35.978140+0000","last_active":"2023-09-23T21:02:41.218368+0000","last_peered":"2023-09-23T21:02:41.218368+0000","last_clean":"2023-09-23T21:02:41.218368+0000","last_became_active":"2023-09-23T20:55:35.977831+0000","last_became_peered":"2023-09-23T20:55:35.977831+0000","last_unstale":"2023-09-23T21:02:41.218368+0000","last_undegraded":"2023-09-23T21:02:41.218368+0000","last_fullsized":"2023-09-23T21:02:41.218368+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:25:47.678023+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.a","version":"274'6006","reported_seq":6034,"reported_epoch":274,"state":"active+clean","last_fresh":"2023-09-23T21:02:44.415557+0000","last_change":"2023-09-23T21:01:25.429899+0000","last_active":"2023-09-23T21:02:44.415557+0000","last_peered":"2023-09-23T21:02:44.415557+0000","last_clean":"2023-09-23T21:02:44.415557+0000","last_became_active":"2023-09-23T21:01:25.429583+0000","last_became_peered":"2023-09-23T21:01:25.429583+0000","last_unstale":"2023-09-23T21:02:44.415557+0000","last_undegraded":"2023-09-23T21:02:44.415557+0000","last_fullsized":"2023-09-23T21:02:44.415557+0000","mapping_epoch":254,"log_start":"0'0","ondisk_log_start":"0'0","created":254,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:01:24.325318+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:01:24.325318+0000","last_clean_scrub_stamp":"2023-09-23T21:01:24.325318+0000","objects_scrubbed":0,"log_size":6006,"log_dups_size":0,"ondisk_log_size":6006,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:31:11.594129+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":393609216,"num_objects":6006,"num_object_clones":0,"num_object_copies":18018,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6006,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6006,"num_write_kb":384384,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.3","version":"79'161","reported_seq":542,"reported_epoch":273,"state":"active+clean","last_fresh":"2023-09-23T21:02:41.218958+0000","last_change":"2023-09-23T20:57:20.378531+0000","last_active":"2023-09-23T21:02:41.218958+0000","last_peered":"2023-09-23T21:02:41.218958+0000","last_clean":"2023-09-23T21:02:41.218958+0000","last_became_active":"2023-09-23T20:57:08.238019+0000","last_became_peered":"2023-09-23T20:57:08.238019+0000","last_unstale":"2023-09-23T21:02:41.218958+0000","last_undegraded":"2023-09-23T21:02:41.218958+0000","last_fullsized":"2023-09-23T21:02:41.218958+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":3,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:20.377714+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:20.377714+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:00:08.305602+0000","scrub_duration":0.003015228,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":144,"reported_epoch":273,"state":"active+clean","last_fresh":"2023-09-23T21:02:41.218441+0000","last_change":"2023-09-23T20:56:45.991884+0000","last_active":"2023-09-23T21:02:41.218441+0000","last_peered":"2023-09-23T21:02:41.218441+0000","last_clean":"2023-09-23T21:02:41.218441+0000","last_became_active":"2023-09-23T20:56:45.991568+0000","last_became_peered":"2023-09-23T20:56:45.991568+0000","last_unstale":"2023-09-23T21:02:41.218441+0000","last_undegraded":"2023-09-23T21:02:41.218441+0000","last_fullsized":"2023-09-23T21:02:41.218441+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":205,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:21:20.775511+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,0],"acting":[1,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":268,"reported_epoch":272,"state":"active+clean","last_fresh":"2023-09-23T21:02:36.167044+0000","last_change":"2023-09-23T20:57:44.979950+0000","last_active":"2023-09-23T21:02:36.167044+0000","last_peered":"2023-09-23T21:02:36.167044+0000","last_clean":"2023-09-23T21:02:36.167044+0000","last_became_active":"2023-09-23T20:57:44.979346+0000","last_became_peered":"2023-09-23T20:57:44.979346+0000","last_unstale":"2023-09-23T21:02:36.167044+0000","last_undegraded":"2023-09-23T21:02:36.167044+0000","last_fullsized":"2023-09-23T21:02:36.167044+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:10:24.372800+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":247,"reported_epoch":272,"state":"active+clean","last_fresh":"2023-09-23T21:02:36.165761+0000","last_change":"2023-09-23T20:50:37.982235+0000","last_active":"2023-09-23T21:02:36.165761+0000","last_peered":"2023-09-23T21:02:36.165761+0000","last_clean":"2023-09-23T21:02:36.165761+0000","last_became_active":"2023-09-23T20:50:37.981913+0000","last_became_peered":"2023-09-23T20:50:37.981913+0000","last_unstale":"2023-09-23T21:02:36.165761+0000","last_undegraded":"2023-09-23T21:02:36.165761+0000","last_fullsized":"2023-09-23T21:02:36.165761+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:59:57.122584+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.f","version":"272'5751","reported_seq":5775,"reported_epoch":272,"state":"active+clean","last_fresh":"2023-09-23T21:02:39.830690+0000","last_change":"2023-09-23T21:01:25.432176+0000","last_active":"2023-09-23T21:02:39.830690+0000","last_peered":"2023-09-23T21:02:39.830690+0000","last_clean":"2023-09-23T21:02:39.830690+0000","last_became_active":"2023-09-23T21:01:25.431835+0000","last_became_peered":"2023-09-23T21:01:25.431835+0000","last_unstale":"2023-09-23T21:02:39.830690+0000","last_undegraded":"2023-09-23T21:02:39.830690+0000","last_fullsized":"2023-09-23T21:02:39.830690+0000","mapping_epoch":254,"log_start":"0'0","ondisk_log_start":"0'0","created":254,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:01:24.325318+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:01:24.325318+0000","last_clean_scrub_stamp":"2023-09-23T21:01:24.325318+0000","objects_scrubbed":0,"log_size":5751,"log_dups_size":0,"ondisk_log_size":5751,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:47:26.553643+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":376897536,"num_objects":5751,"num_object_clones":0,"num_object_copies":17253,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5751,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5751,"num_write_kb":368064,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,1],"acting":[3,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.6","version":"79'161","reported_seq":556,"reported_epoch":273,"state":"active+clean","last_fresh":"2023-09-23T21:02:41.218494+0000","last_change":"2023-09-23T20:57:26.907505+0000","last_active":"2023-09-23T21:02:41.218494+0000","last_peered":"2023-09-23T21:02:41.218494+0000","last_clean":"2023-09-23T21:02:41.218494+0000","last_became_active":"2023-09-23T20:57:08.240321+0000","last_became_peered":"2023-09-23T20:57:08.240321+0000","last_unstale":"2023-09-23T21:02:41.218494+0000","last_undegraded":"2023-09-23T21:02:41.218494+0000","last_fullsized":"2023-09-23T21:02:41.218494+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":3,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:26.907408+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:26.907408+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:34:17.392928+0000","scrub_duration":0.010337038,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":190,"reported_epoch":272,"state":"active+clean","last_fresh":"2023-09-23T21:02:36.164649+0000","last_change":"2023-09-23T20:56:45.988566+0000","last_active":"2023-09-23T21:02:36.164649+0000","last_peered":"2023-09-23T21:02:36.164649+0000","last_clean":"2023-09-23T21:02:36.164649+0000","last_became_active":"2023-09-23T20:56:45.988249+0000","last_became_peered":"2023-09-23T20:56:45.988249+0000","last_unstale":"2023-09-23T21:02:36.164649+0000","last_undegraded":"2023-09-23T21:02:36.164649+0000","last_fullsized":"2023-09-23T21:02:36.164649+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":205,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:51:36.644764+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,4],"acting":[0,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":246,"reported_epoch":273,"state":"active+clean","last_fresh":"2023-09-23T21:02:41.219468+0000","last_change":"2023-09-23T20:54:31.898965+0000","last_active":"2023-09-23T21:02:41.219468+0000","last_peered":"2023-09-23T21:02:41.219468+0000","last_clean":"2023-09-23T21:02:41.219468+0000","last_became_active":"2023-09-23T20:54:31.898663+0000","last_became_peered":"2023-09-23T20:54:31.898663+0000","last_unstale":"2023-09-23T21:02:41.219468+0000","last_undegraded":"2023-09-23T21:02:41.219468+0000","last_fullsized":"2023-09-23T21:02:41.219468+0000","mapping_epoch":136,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:53:05.058087+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":235,"reported_epoch":272,"state":"active+clean","last_fresh":"2023-09-23T21:02:36.162574+0000","last_change":"2023-09-23T20:54:31.884717+0000","last_active":"2023-09-23T21:02:36.162574+0000","last_peered":"2023-09-23T21:02:36.162574+0000","last_clean":"2023-09-23T21:02:36.162574+0000","last_became_active":"2023-09-23T20:54:31.884345+0000","last_became_peered":"2023-09-23T20:54:31.884345+0000","last_unstale":"2023-09-23T21:02:36.162574+0000","last_undegraded":"2023-09-23T21:02:36.162574+0000","last_fullsized":"2023-09-23T21:02:36.162574+0000","mapping_epoch":136,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:02:26.393868+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.c","version":"273'6024","reported_seq":6052,"reported_epoch":273,"state":"active+clean","last_fresh":"2023-09-23T21:02:42.053992+0000","last_change":"2023-09-23T21:01:25.560907+0000","last_active":"2023-09-23T21:02:42.053992+0000","last_peered":"2023-09-23T21:02:42.053992+0000","last_clean":"2023-09-23T21:02:42.053992+0000","last_became_active":"2023-09-23T21:01:25.560592+0000","last_became_peered":"2023-09-23T21:01:25.560592+0000","last_unstale":"2023-09-23T21:02:42.053992+0000","last_undegraded":"2023-09-23T21:02:42.053992+0000","last_fullsized":"2023-09-23T21:02:42.053992+0000","mapping_epoch":254,"log_start":"0'0","ondisk_log_start":"0'0","created":254,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:01:24.325318+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:01:24.325318+0000","last_clean_scrub_stamp":"2023-09-23T21:01:24.325318+0000","objects_scrubbed":0,"log_size":6024,"log_dups_size":0,"ondisk_log_size":6024,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:14:22.102068+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":394788864,"num_objects":6024,"num_object_clones":0,"num_object_copies":18072,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6024,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6024,"num_write_kb":385536,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.5","version":"79'161","reported_seq":536,"reported_epoch":272,"state":"active+clean","last_fresh":"2023-09-23T21:02:36.169388+0000","last_change":"2023-09-23T20:57:30.029479+0000","last_active":"2023-09-23T21:02:36.169388+0000","last_peered":"2023-09-23T21:02:36.169388+0000","last_clean":"2023-09-23T21:02:36.169388+0000","last_became_active":"2023-09-23T20:57:08.249909+0000","last_became_peered":"2023-09-23T20:57:08.249909+0000","last_unstale":"2023-09-23T21:02:36.169388+0000","last_undegraded":"2023-09-23T21:02:36.169388+0000","last_fullsized":"2023-09-23T21:02:36.169388+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":3,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:30.029396+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:30.029396+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:16:46.083145+0000","scrub_duration":0.012914682,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":237,"reported_epoch":271,"state":"active+clean","last_fresh":"2023-09-23T21:02:30.345403+0000","last_change":"2023-09-23T20:51:10.836910+0000","last_active":"2023-09-23T21:02:30.345403+0000","last_peered":"2023-09-23T21:02:30.345403+0000","last_clean":"2023-09-23T21:02:30.345403+0000","last_became_active":"2023-09-23T20:51:10.836557+0000","last_became_peered":"2023-09-23T20:51:10.836557+0000","last_unstale":"2023-09-23T21:02:30.345403+0000","last_undegraded":"2023-09-23T21:02:30.345403+0000","last_fullsized":"2023-09-23T21:02:30.345403+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:51:54.019969+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":245,"reported_epoch":273,"state":"active+clean","last_fresh":"2023-09-23T21:02:41.219335+0000","last_change":"2023-09-23T20:55:35.953444+0000","last_active":"2023-09-23T21:02:41.219335+0000","last_peered":"2023-09-23T21:02:41.219335+0000","last_clean":"2023-09-23T21:02:41.219335+0000","last_became_active":"2023-09-23T20:55:35.953151+0000","last_became_peered":"2023-09-23T20:55:35.953151+0000","last_unstale":"2023-09-23T21:02:41.219335+0000","last_undegraded":"2023-09-23T21:02:41.219335+0000","last_fullsized":"2023-09-23T21:02:41.219335+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:39:12.404391+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":62,"reported_epoch":271,"state":"active+clean","last_fresh":"2023-09-23T21:02:30.344462+0000","last_change":"2023-09-23T20:57:44.982509+0000","last_active":"2023-09-23T21:02:30.344462+0000","last_peered":"2023-09-23T21:02:30.344462+0000","last_clean":"2023-09-23T21:02:30.344462+0000","last_became_active":"2023-09-23T20:57:44.982175+0000","last_became_peered":"2023-09-23T20:57:44.982175+0000","last_unstale":"2023-09-23T21:02:30.344462+0000","last_undegraded":"2023-09-23T21:02:30.344462+0000","last_fullsized":"2023-09-23T21:02:30.344462+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:37:49.215488+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.d","version":"273'5872","reported_seq":5899,"reported_epoch":273,"state":"active+clean","last_fresh":"2023-09-23T21:02:42.068273+0000","last_change":"2023-09-23T21:01:25.560531+0000","last_active":"2023-09-23T21:02:42.068273+0000","last_peered":"2023-09-23T21:02:42.068273+0000","last_clean":"2023-09-23T21:02:42.068273+0000","last_became_active":"2023-09-23T21:01:25.560228+0000","last_became_peered":"2023-09-23T21:01:25.560228+0000","last_unstale":"2023-09-23T21:02:42.068273+0000","last_undegraded":"2023-09-23T21:02:42.068273+0000","last_fullsized":"2023-09-23T21:02:42.068273+0000","mapping_epoch":254,"log_start":"0'0","ondisk_log_start":"0'0","created":254,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:01:24.325318+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:01:24.325318+0000","last_clean_scrub_stamp":"2023-09-23T21:01:24.325318+0000","objects_scrubbed":0,"log_size":5872,"log_dups_size":0,"ondisk_log_size":5872,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:19:04.504855+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":384827392,"num_objects":5872,"num_object_clones":0,"num_object_copies":17616,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5872,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5872,"num_write_kb":375808,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,2],"acting":[5,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.4","version":"79'161","reported_seq":552,"reported_epoch":271,"state":"active+clean","last_fresh":"2023-09-23T21:02:30.344911+0000","last_change":"2023-09-23T20:57:20.012904+0000","last_active":"2023-09-23T21:02:30.344911+0000","last_peered":"2023-09-23T21:02:30.344911+0000","last_clean":"2023-09-23T21:02:30.344911+0000","last_became_active":"2023-09-23T20:57:08.237996+0000","last_became_peered":"2023-09-23T20:57:08.237996+0000","last_unstale":"2023-09-23T21:02:30.344911+0000","last_undegraded":"2023-09-23T21:02:30.344911+0000","last_fullsized":"2023-09-23T21:02:30.344911+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":3,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:20.012764+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:20.012764+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:51:28.796946+0000","scrub_duration":0.0045251919999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":223,"reported_epoch":273,"state":"active+clean","last_fresh":"2023-09-23T21:02:41.218540+0000","last_change":"2023-09-23T20:55:35.949541+0000","last_active":"2023-09-23T21:02:41.218540+0000","last_peered":"2023-09-23T21:02:41.218540+0000","last_clean":"2023-09-23T21:02:41.218540+0000","last_became_active":"2023-09-23T20:55:35.944635+0000","last_became_peered":"2023-09-23T20:55:35.944635+0000","last_unstale":"2023-09-23T21:02:41.218540+0000","last_undegraded":"2023-09-23T21:02:41.218540+0000","last_fullsized":"2023-09-23T21:02:41.218540+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:30:36.033585+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"70'1","reported_seq":263,"reported_epoch":272,"state":"active+clean","last_fresh":"2023-09-23T21:02:36.165883+0000","last_change":"2023-09-23T20:54:14.009723+0000","last_active":"2023-09-23T21:02:36.165883+0000","last_peered":"2023-09-23T21:02:36.165883+0000","last_clean":"2023-09-23T21:02:36.165883+0000","last_became_active":"2023-09-23T20:54:13.941771+0000","last_became_peered":"2023-09-23T20:54:13.941771+0000","last_unstale":"2023-09-23T21:02:36.165883+0000","last_undegraded":"2023-09-23T21:02:36.165883+0000","last_fullsized":"2023-09-23T21:02:36.165883+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:48:25.588116+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":177,"reported_epoch":272,"state":"active+clean","last_fresh":"2023-09-23T21:02:36.168628+0000","last_change":"2023-09-23T20:55:35.949780+0000","last_active":"2023-09-23T21:02:36.168628+0000","last_peered":"2023-09-23T21:02:36.168628+0000","last_clean":"2023-09-23T21:02:36.168628+0000","last_became_active":"2023-09-23T20:55:35.949497+0000","last_became_peered":"2023-09-23T20:55:35.949497+0000","last_unstale":"2023-09-23T21:02:36.168628+0000","last_undegraded":"2023-09-23T21:02:36.168628+0000","last_fullsized":"2023-09-23T21:02:36.168628+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:09:37.420882+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":237,"reported_epoch":272,"state":"active+clean","last_fresh":"2023-09-23T21:02:36.162496+0000","last_change":"2023-09-23T20:54:13.929196+0000","last_active":"2023-09-23T21:02:36.162496+0000","last_peered":"2023-09-23T21:02:36.162496+0000","last_clean":"2023-09-23T21:02:36.162496+0000","last_became_active":"2023-09-23T20:54:13.928831+0000","last_became_peered":"2023-09-23T20:54:13.928831+0000","last_unstale":"2023-09-23T21:02:36.162496+0000","last_undegraded":"2023-09-23T21:02:36.162496+0000","last_fullsized":"2023-09-23T21:02:36.162496+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:28:44.316851+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":242,"reported_epoch":272,"state":"active+clean","last_fresh":"2023-09-23T21:02:36.170455+0000","last_change":"2023-09-23T20:52:12.855430+0000","last_active":"2023-09-23T21:02:36.170455+0000","last_peered":"2023-09-23T21:02:36.170455+0000","last_clean":"2023-09-23T21:02:36.170455+0000","last_became_active":"2023-09-23T20:52:12.855115+0000","last_became_peered":"2023-09-23T20:52:12.855115+0000","last_unstale":"2023-09-23T21:02:36.170455+0000","last_undegraded":"2023-09-23T21:02:36.170455+0000","last_fullsized":"2023-09-23T21:02:36.170455+0000","mapping_epoch":100,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":101,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:33:23.802214+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":150,"reported_epoch":272,"state":"active+clean","last_fresh":"2023-09-23T21:02:36.168498+0000","last_change":"2023-09-23T20:54:31.902946+0000","last_active":"2023-09-23T21:02:36.168498+0000","last_peered":"2023-09-23T21:02:36.168498+0000","last_clean":"2023-09-23T21:02:36.168498+0000","last_became_active":"2023-09-23T20:54:31.889484+0000","last_became_peered":"2023-09-23T20:54:31.889484+0000","last_unstale":"2023-09-23T21:02:36.168498+0000","last_undegraded":"2023-09-23T21:02:36.168498+0000","last_fullsized":"2023-09-23T21:02:36.168498+0000","mapping_epoch":136,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:44:44.278398+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":150,"reported_epoch":272,"state":"active+clean","last_fresh":"2023-09-23T21:02:36.169508+0000","last_change":"2023-09-23T20:54:13.950351+0000","last_active":"2023-09-23T21:02:36.169508+0000","last_peered":"2023-09-23T21:02:36.169508+0000","last_clean":"2023-09-23T21:02:36.169508+0000","last_became_active":"2023-09-23T20:54:13.949987+0000","last_became_peered":"2023-09-23T20:54:13.949987+0000","last_unstale":"2023-09-23T21:02:36.169508+0000","last_undegraded":"2023-09-23T21:02:36.169508+0000","last_fullsized":"2023-09-23T21:02:36.169508+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:46:17.765134+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":181,"reported_epoch":272,"state":"active+clean","last_fresh":"2023-09-23T21:02:36.167201+0000","last_change":"2023-09-23T20:56:18.141728+0000","last_active":"2023-09-23T21:02:36.167201+0000","last_peered":"2023-09-23T21:02:36.167201+0000","last_clean":"2023-09-23T21:02:36.167201+0000","last_became_active":"2023-09-23T20:56:18.137979+0000","last_became_peered":"2023-09-23T20:56:18.137979+0000","last_unstale":"2023-09-23T21:02:36.167201+0000","last_undegraded":"2023-09-23T21:02:36.167201+0000","last_fullsized":"2023-09-23T21:02:36.167201+0000","mapping_epoch":192,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":193,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:58:05.658443+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":150,"reported_epoch":272,"state":"active+clean","last_fresh":"2023-09-23T21:02:36.168958+0000","last_change":"2023-09-23T20:54:13.935307+0000","last_active":"2023-09-23T21:02:36.168958+0000","last_peered":"2023-09-23T21:02:36.168958+0000","last_clean":"2023-09-23T21:02:36.168958+0000","last_became_active":"2023-09-23T20:54:13.935000+0000","last_became_peered":"2023-09-23T20:54:13.935000+0000","last_unstale":"2023-09-23T21:02:36.168958+0000","last_undegraded":"2023-09-23T21:02:36.168958+0000","last_fullsized":"2023-09-23T21:02:36.168958+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:47:40.974082+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":260,"reported_epoch":271,"state":"active+clean","last_fresh":"2023-09-23T21:02:30.345406+0000","last_change":"2023-09-23T20:55:35.954870+0000","last_active":"2023-09-23T21:02:30.345406+0000","last_peered":"2023-09-23T21:02:30.345406+0000","last_clean":"2023-09-23T21:02:30.345406+0000","last_became_active":"2023-09-23T20:55:35.954568+0000","last_became_peered":"2023-09-23T20:55:35.954568+0000","last_unstale":"2023-09-23T21:02:30.345406+0000","last_undegraded":"2023-09-23T21:02:30.345406+0000","last_fullsized":"2023-09-23T21:02:30.345406+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:06:22.423566+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":259,"reported_epoch":271,"state":"active+clean","last_fresh":"2023-09-23T21:02:30.344530+0000","last_change":"2023-09-23T20:51:10.838334+0000","last_active":"2023-09-23T21:02:30.344530+0000","last_peered":"2023-09-23T21:02:30.344530+0000","last_clean":"2023-09-23T21:02:30.344530+0000","last_became_active":"2023-09-23T20:51:10.838026+0000","last_became_peered":"2023-09-23T20:51:10.838026+0000","last_unstale":"2023-09-23T21:02:30.344530+0000","last_undegraded":"2023-09-23T21:02:30.344530+0000","last_fullsized":"2023-09-23T21:02:30.344530+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:04:37.009457+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":265,"reported_epoch":272,"state":"active+clean","last_fresh":"2023-09-23T21:02:36.164013+0000","last_change":"2023-09-23T20:54:31.903139+0000","last_active":"2023-09-23T21:02:36.164013+0000","last_peered":"2023-09-23T21:02:36.164013+0000","last_clean":"2023-09-23T21:02:36.164013+0000","last_became_active":"2023-09-23T20:54:31.884520+0000","last_became_peered":"2023-09-23T20:54:31.884520+0000","last_unstale":"2023-09-23T21:02:36.164013+0000","last_undegraded":"2023-09-23T21:02:36.164013+0000","last_fullsized":"2023-09-23T21:02:36.164013+0000","mapping_epoch":136,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:39:34.269705+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1108,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":167,"reported_epoch":272,"state":"active+clean","last_fresh":"2023-09-23T21:02:36.168330+0000","last_change":"2023-09-23T20:55:35.951572+0000","last_active":"2023-09-23T21:02:36.168330+0000","last_peered":"2023-09-23T21:02:36.168330+0000","last_clean":"2023-09-23T21:02:36.168330+0000","last_became_active":"2023-09-23T20:55:35.951294+0000","last_became_peered":"2023-09-23T20:55:35.951294+0000","last_unstale":"2023-09-23T21:02:36.168330+0000","last_undegraded":"2023-09-23T21:02:36.168330+0000","last_fullsized":"2023-09-23T21:02:36.168330+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:36:32.447210+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":22,"num_bytes_recovered":5990,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":8,"num_pg":16,"stat_sum":{"num_bytes":6262292480,"num_objects":95555,"num_object_clones":0,"num_object_copies":286665,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":95555,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":95555,"num_write_kb":6115520,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":18752077824,"data_stored":18752077824,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":95555,"ondisk_log_size":95555,"up":48,"acting":48,"num_store_stats":8},{"poolid":1,"num_pg":21,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3381,"ondisk_log_size":3381,"up":63,"acting":63,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201547,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2145984,"kb_used_data":1918776,"kb_used_omap":0,"kb_used_meta":227200,"kb_avail":91599168,"statfs":{"total":95995035648,"available":93797548032,"internally_reserved":0,"allocated":1964826624,"data_stored":1963922514,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":232652800},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:01:49 2023","interfaces":[{"interface":"back","average":{"1min":0.63700000000000001,"5min":2.2869999999999999,"15min":1.3120000000000001},"min":{"1min":0.22800000000000001,"5min":0.22800000000000001,"15min":0.21099999999999999},"max":{"1min":1.798,"5min":43.994999999999997,"15min":43.994999999999997},"last":3.3340000000000001},{"interface":"front","average":{"1min":1.002,"5min":2.3940000000000001,"15min":1.4179999999999999},"min":{"1min":0.42899999999999999,"5min":0.318,"15min":0.218},"max":{"1min":3.633,"5min":44.206000000000003,"15min":44.206000000000003},"last":3.347}]},{"osd":1,"last update":"Sat Sep 23 21:02:30 2023","interfaces":[{"interface":"back","average":{"1min":1.034,"5min":1.3879999999999999,"15min":1.3340000000000001},"min":{"1min":0.26300000000000001,"5min":0.253,"15min":0.23300000000000001},"max":{"1min":5.1630000000000003,"5min":10.622,"15min":22.591999999999999},"last":2.073},{"interface":"front","average":{"1min":0.91900000000000004,"5min":1.647,"15min":1.2669999999999999},"min":{"1min":0.27100000000000002,"5min":0.25800000000000001,"15min":0.23699999999999999},"max":{"1min":3.2749999999999999,"5min":13.981,"15min":21.437000000000001},"last":3.444}]},{"osd":2,"last update":"Sat Sep 23 21:02:00 2023","interfaces":[{"interface":"back","average":{"1min":0.73899999999999999,"5min":2.1920000000000002,"15min":1.429},"min":{"1min":0.313,"5min":0.26800000000000002,"15min":0.24399999999999999},"max":{"1min":2.0899999999999999,"5min":10.805999999999999,"15min":10.805999999999999},"last":10.279999999999999},{"interface":"front","average":{"1min":0.78300000000000003,"5min":2.367,"15min":1.482},"min":{"1min":0.27900000000000003,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":2.2240000000000002,"5min":45.25,"15min":45.25},"last":2.831}]},{"osd":3,"last update":"Sat Sep 23 21:02:38 2023","interfaces":[{"interface":"back","average":{"1min":2.0640000000000001,"5min":2.6459999999999999,"15min":2.097},"min":{"1min":0.34000000000000002,"5min":0.315,"15min":0.314},"max":{"1min":11.625999999999999,"5min":31.786000000000001,"15min":36.408000000000001},"last":3.2400000000000002},{"interface":"front","average":{"1min":1.3069999999999999,"5min":2.6539999999999999,"15min":1.9650000000000001},"min":{"1min":0.309,"5min":0.309,"15min":0.26000000000000001},"max":{"1min":6.181,"5min":32.429000000000002,"15min":35.593000000000004},"last":2.9980000000000002}]},{"osd":4,"last update":"Sat Sep 23 21:02:30 2023","interfaces":[{"interface":"back","average":{"1min":0.71499999999999997,"5min":0.58999999999999997,"15min":0.63500000000000001},"min":{"1min":0.29399999999999998,"5min":0.25700000000000001,"15min":0.23699999999999999},"max":{"1min":2.2469999999999999,"5min":4.5410000000000004,"15min":5.8920000000000003},"last":2.7970000000000002},{"interface":"front","average":{"1min":1.0169999999999999,"5min":0.72599999999999998,"15min":0.67800000000000005},"min":{"1min":0.28000000000000003,"5min":0.27100000000000002,"15min":0.26300000000000001},"max":{"1min":6.3979999999999997,"5min":6.3979999999999997,"15min":6.3979999999999997},"last":3.0600000000000001}]},{"osd":5,"last update":"Sat Sep 23 21:02:30 2023","interfaces":[{"interface":"back","average":{"1min":1.4179999999999999,"5min":0.71399999999999997,"15min":0.69099999999999995},"min":{"1min":0.34300000000000003,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":5.2089999999999996,"5min":5.2089999999999996,"15min":7.0389999999999997},"last":2.0409999999999999},{"interface":"front","average":{"1min":0.76200000000000001,"5min":0.67600000000000005,"15min":0.65100000000000002},"min":{"1min":0.33300000000000002,"5min":0.30199999999999999,"15min":0.27800000000000002},"max":{"1min":2.2530000000000001,"5min":4.5999999999999996,"15min":4.5999999999999996},"last":3.1070000000000002}]},{"osd":6,"last update":"Sat Sep 23 21:01:46 2023","interfaces":[{"interface":"back","average":{"1min":0.77800000000000002,"5min":0.73999999999999999,"15min":0.69099999999999995},"min":{"1min":0.34899999999999998,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":3.1779999999999999,"5min":3.5249999999999999,"15min":3.5249999999999999},"last":3.0339999999999998},{"interface":"front","average":{"1min":0.77800000000000002,"5min":0.79500000000000004,"15min":0.80100000000000005},"min":{"1min":0.40500000000000003,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":2.9500000000000002,"5min":3.5569999999999999,"15min":3.5569999999999999},"last":3.0960000000000001}]}]},{"osd":6,"up_from":164,"seq":704374636631,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2071032,"kb_used_data":1922544,"kb_used_omap":10,"kb_used_meta":148469,"kb_avail":91674120,"statfs":{"total":95995035648,"available":93874298880,"internally_reserved":0,"allocated":1968685056,"data_stored":1967796915,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":10328,"internal_metadata":152033192},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,3,0,0,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:01:47 2023","interfaces":[{"interface":"back","average":{"1min":1.337,"5min":1.851,"15min":1.1930000000000001},"min":{"1min":0.255,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":8.5269999999999992,"5min":10.404,"15min":10.404},"last":5.7149999999999999},{"interface":"front","average":{"1min":1.327,"5min":2.0569999999999999,"15min":1.341},"min":{"1min":0.20999999999999999,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":7.3029999999999999,"5min":11.023,"15min":11.023},"last":3.589}]},{"osd":1,"last update":"Sat Sep 23 21:01:47 2023","interfaces":[{"interface":"back","average":{"1min":1.0920000000000001,"5min":1.4730000000000001,"15min":1.0169999999999999},"min":{"1min":0.28499999999999998,"5min":0.27200000000000002,"15min":0.25600000000000001},"max":{"1min":3.4590000000000001,"5min":8.4309999999999992,"15min":8.4309999999999992},"last":0.77900000000000003},{"interface":"front","average":{"1min":0.95699999999999996,"5min":1.7470000000000001,"15min":1.161},"min":{"1min":0.312,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":5.3070000000000004,"5min":10.518000000000001,"15min":10.518000000000001},"last":5.1799999999999997}]},{"osd":2,"last update":"Sat Sep 23 21:01:47 2023","interfaces":[{"interface":"back","average":{"1min":1.2310000000000001,"5min":2.2440000000000002,"15min":1.8460000000000001},"min":{"1min":0.312,"5min":0.28999999999999998,"15min":0.27700000000000002},"max":{"1min":4.5599999999999996,"5min":13.750999999999999,"15min":13.750999999999999},"last":4.5890000000000004},{"interface":"front","average":{"1min":1.028,"5min":2.2010000000000001,"15min":1.4710000000000001},"min":{"1min":0.247,"5min":0.247,"15min":0.247},"max":{"1min":4.165,"5min":20.195,"15min":20.195},"last":8.3670000000000009}]},{"osd":3,"last update":"Sat Sep 23 21:01:47 2023","interfaces":[{"interface":"back","average":{"1min":0.82399999999999995,"5min":1.966,"15min":1.3999999999999999},"min":{"1min":0.37,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":2.927,"5min":14.144,"15min":14.144},"last":0.97199999999999998},{"interface":"front","average":{"1min":1.125,"5min":1.891,"15min":1.6990000000000001},"min":{"1min":0.315,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":4.5830000000000002,"5min":11.989000000000001,"15min":11.989000000000001},"last":1.1639999999999999}]},{"osd":4,"last update":"Sat Sep 23 21:01:47 2023","interfaces":[{"interface":"back","average":{"1min":0.59399999999999997,"5min":0.64700000000000002,"15min":0.91700000000000004},"min":{"1min":0.24299999999999999,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":4.1230000000000002,"5min":4.1230000000000002,"15min":9.0079999999999991},"last":0.41899999999999998},{"interface":"front","average":{"1min":0.499,"5min":0.51200000000000001,"15min":0.52400000000000002},"min":{"1min":0.29699999999999999,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":0.94499999999999995,"5min":2.4860000000000002,"15min":2.4860000000000002},"last":0.70699999999999996}]},{"osd":5,"last update":"Sat Sep 23 21:01:47 2023","interfaces":[{"interface":"back","average":{"1min":0.75800000000000001,"5min":0.74099999999999999,"15min":0.60699999999999998},"min":{"1min":0.27500000000000002,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":4.149,"5min":5.4119999999999999,"15min":5.4119999999999999},"last":3.5169999999999999},{"interface":"front","average":{"1min":0.63600000000000001,"5min":0.72399999999999998,"15min":0.78100000000000003},"min":{"1min":0.29199999999999998,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":2.6280000000000001,"5min":3.536,"15min":3.536},"last":0.34100000000000003}]},{"osd":7,"last update":"Sat Sep 23 21:01:47 2023","interfaces":[{"interface":"back","average":{"1min":0.505,"5min":0.73099999999999998,"15min":0.86299999999999999},"min":{"1min":0.32400000000000001,"5min":0.312,"15min":0.312},"max":{"1min":0.97099999999999997,"5min":7.8570000000000002,"15min":7.8570000000000002},"last":0.54400000000000004},{"interface":"front","average":{"1min":0.59099999999999997,"5min":0.76900000000000002,"15min":0.60899999999999999},"min":{"1min":0.28100000000000003,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":1.829,"5min":8.282,"15min":8.282},"last":0.80800000000000005}]}]},{"osd":1,"up_from":88,"seq":377957122202,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3232380,"kb_used_data":2879092,"kb_used_omap":9,"kb_used_meta":353270,"kb_avail":90512772,"statfs":{"total":95995035648,"available":92685078528,"internally_reserved":0,"allocated":2948190208,"data_stored":2946893361,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9239,"internal_metadata":361749481},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,3,0,1,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:02:31 2023","interfaces":[{"interface":"back","average":{"1min":3.7730000000000001,"5min":3.081,"15min":2.0009999999999999},"min":{"1min":0.25,"5min":0.20699999999999999,"15min":0.20699999999999999},"max":{"1min":45.542000000000002,"5min":45.542000000000002,"15min":45.542000000000002},"last":0.26900000000000002},{"interface":"front","average":{"1min":4.1929999999999996,"5min":3.7440000000000002,"15min":2.3100000000000001},"min":{"1min":0.28499999999999998,"5min":0.247,"15min":0.22500000000000001},"max":{"1min":46.353000000000002,"5min":46.353000000000002,"15min":46.353000000000002},"last":0.55200000000000005}]},{"osd":2,"last update":"Sat Sep 23 21:02:00 2023","interfaces":[{"interface":"back","average":{"1min":0.79700000000000004,"5min":2.1840000000000002,"15min":1.478},"min":{"1min":0.20399999999999999,"5min":0.20399999999999999,"15min":0.20300000000000001},"max":{"1min":2.7759999999999998,"5min":13.752000000000001,"15min":14.335000000000001},"last":1.4039999999999999},{"interface":"front","average":{"1min":1.502,"5min":3.863,"15min":2.323},"min":{"1min":0.249,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":7.2649999999999997,"5min":18.026,"15min":38.951000000000001},"last":0.59599999999999997}]},{"osd":3,"last update":"Sat Sep 23 21:02:31 2023","interfaces":[{"interface":"back","average":{"1min":2.3900000000000001,"5min":2.8410000000000002,"15min":2.0110000000000001},"min":{"1min":0.26500000000000001,"5min":0.246,"15min":0.246},"max":{"1min":8.7390000000000008,"5min":22.686,"15min":25.314},"last":0.66900000000000004},{"interface":"front","average":{"1min":1.5509999999999999,"5min":2.4900000000000002,"15min":1.7629999999999999},"min":{"1min":0.25600000000000001,"5min":0.25600000000000001,"15min":0.20599999999999999},"max":{"1min":10.157999999999999,"5min":14.473000000000001,"15min":19.881},"last":0.35699999999999998}]},{"osd":4,"last update":"Sat Sep 23 21:02:31 2023","interfaces":[{"interface":"back","average":{"1min":3.5030000000000001,"5min":2.0470000000000002,"15min":1.528},"min":{"1min":0.28399999999999997,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":41.823,"5min":41.823,"15min":41.823},"last":0.70699999999999996},{"interface":"front","average":{"1min":3.097,"5min":1.9359999999999999,"15min":1.3420000000000001},"min":{"1min":0.33300000000000002,"5min":0.31900000000000001,"15min":0.27600000000000002},"max":{"1min":41.423000000000002,"5min":41.423000000000002,"15min":41.423000000000002},"last":0.48499999999999999}]},{"osd":5,"last update":"Sat Sep 23 21:02:31 2023","interfaces":[{"interface":"back","average":{"1min":1.446,"5min":2.2789999999999999,"15min":1.55},"min":{"1min":0.32000000000000001,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":7.5570000000000004,"5min":15.041,"15min":15.041},"last":0.628},{"interface":"front","average":{"1min":1.5649999999999999,"5min":1.6830000000000001,"15min":1.431},"min":{"1min":0.36099999999999999,"5min":0.33200000000000002,"15min":0.32200000000000001},"max":{"1min":5.4930000000000003,"5min":18.280999999999999,"15min":18.280999999999999},"last":0.77600000000000002}]},{"osd":6,"last update":"Sat Sep 23 21:01:48 2023","interfaces":[{"interface":"back","average":{"1min":0.76700000000000002,"5min":1.6479999999999999,"15min":1.4259999999999999},"min":{"1min":0.38600000000000001,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":2.2240000000000002,"5min":18.984999999999999,"15min":18.984999999999999},"last":0.74399999999999999},{"interface":"front","average":{"1min":1.073,"5min":1.6599999999999999,"15min":1.3879999999999999},"min":{"1min":0.37,"5min":0.33700000000000002,"15min":0.309},"max":{"1min":4.1589999999999998,"5min":8.7530000000000001,"15min":8.7530000000000001},"last":0.52100000000000002}]},{"osd":7,"last update":"Sat Sep 23 21:02:31 2023","interfaces":[{"interface":"back","average":{"1min":1.238,"5min":1.6910000000000001,"15min":1.3149999999999999},"min":{"1min":0.39600000000000002,"5min":0.39600000000000002,"15min":0.35699999999999998},"max":{"1min":3.9900000000000002,"5min":8.7769999999999992,"15min":11.186999999999999},"last":0.56899999999999995},{"interface":"front","average":{"1min":2.0329999999999999,"5min":2.3570000000000002,"15min":1.5940000000000001},"min":{"1min":0.38100000000000001,"5min":0.38100000000000001,"15min":0.34599999999999997},"max":{"1min":7.6100000000000003,"5min":15.098000000000001,"15min":15.098000000000001},"last":0.65500000000000003}]}]},{"osd":0,"up_from":83,"seq":356482285728,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2625488,"kb_used_data":2304776,"kb_used_omap":28,"kb_used_meta":320675,"kb_avail":91119664,"statfs":{"total":95995035648,"available":93306535936,"internally_reserved":0,"allocated":2360090624,"data_stored":2358827148,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":28848,"internal_metadata":328372048},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,6],"upper_bound":16},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 21:02:21 2023","interfaces":[{"interface":"back","average":{"1min":2.2930000000000001,"5min":3.782,"15min":2.181},"min":{"1min":0.19500000000000001,"5min":0.19500000000000001,"15min":0.191},"max":{"1min":22.411000000000001,"5min":22.411000000000001,"15min":22.411000000000001},"last":0.65500000000000003},{"interface":"front","average":{"1min":3.0870000000000002,"5min":3.8799999999999999,"15min":2.1970000000000001},"min":{"1min":0.29299999999999998,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":22.474,"5min":22.474,"15min":22.474},"last":0.56999999999999995}]},{"osd":2,"last update":"Sat Sep 23 21:01:55 2023","interfaces":[{"interface":"back","average":{"1min":1.052,"5min":3.2599999999999998,"15min":2.1419999999999999},"min":{"1min":0.23499999999999999,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":8.2479999999999993,"5min":21.053000000000001,"15min":21.053000000000001},"last":0.53800000000000003},{"interface":"front","average":{"1min":1.3,"5min":3.9390000000000001,"15min":2.2530000000000001},"min":{"1min":0.23100000000000001,"5min":0.23100000000000001,"15min":0.21099999999999999},"max":{"1min":6.6189999999999998,"5min":21.747,"15min":21.747},"last":5.9409999999999998}]},{"osd":3,"last update":"Sat Sep 23 21:02:21 2023","interfaces":[{"interface":"back","average":{"1min":1.6990000000000001,"5min":3.6360000000000001,"15min":2.1549999999999998},"min":{"1min":0.29599999999999999,"5min":0.27100000000000002,"15min":0.25800000000000001},"max":{"1min":11.617000000000001,"5min":21.184000000000001,"15min":21.184000000000001},"last":0.55100000000000005},{"interface":"front","average":{"1min":2.2280000000000002,"5min":3.8839999999999999,"15min":2.3380000000000001},"min":{"1min":0.32900000000000001,"5min":0.27400000000000002,"15min":0.23300000000000001},"max":{"1min":8.4510000000000005,"5min":21.167000000000002,"15min":21.167000000000002},"last":0.59399999999999997}]},{"osd":4,"last update":"Sat Sep 23 21:01:55 2023","interfaces":[{"interface":"back","average":{"1min":1.2390000000000001,"5min":2.8599999999999999,"15min":1.7230000000000001},"min":{"1min":0.28599999999999998,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":6.5620000000000003,"5min":21.594999999999999,"15min":21.594999999999999},"last":7.4169999999999998},{"interface":"front","average":{"1min":0.80400000000000005,"5min":2.2549999999999999,"15min":1.496},"min":{"1min":0.32900000000000001,"5min":0.311,"15min":0.28100000000000003},"max":{"1min":3.4289999999999998,"5min":13.904,"15min":18.724},"last":0.58099999999999996}]},{"osd":5,"last update":"Sat Sep 23 21:01:55 2023","interfaces":[{"interface":"back","average":{"1min":1.1459999999999999,"5min":1.9670000000000001,"15min":1.4339999999999999},"min":{"1min":0.34999999999999998,"5min":0.309,"15min":0.309},"max":{"1min":8.1899999999999995,"5min":8.8970000000000002,"15min":8.8970000000000002},"last":0.83799999999999997},{"interface":"front","average":{"1min":1.256,"5min":2.7000000000000002,"15min":1.724},"min":{"1min":0.373,"5min":0.373,"15min":0.34999999999999998},"max":{"1min":6.601,"5min":20.512,"15min":20.512},"last":6.3639999999999999}]},{"osd":6,"last update":"Sat Sep 23 21:01:52 2023","interfaces":[{"interface":"back","average":{"1min":0.78000000000000003,"5min":2.3199999999999998,"15min":1.8779999999999999},"min":{"1min":0.38,"5min":0.37,"15min":0.37},"max":{"1min":3.472,"5min":13.933,"15min":13.933},"last":4.2000000000000002},{"interface":"front","average":{"1min":0.76500000000000001,"5min":2.4180000000000001,"15min":1.97},"min":{"1min":0.379,"5min":0.377,"15min":0.377},"max":{"1min":3.5169999999999999,"5min":13.962,"15min":13.962},"last":0.61099999999999999}]},{"osd":7,"last update":"Sat Sep 23 21:01:55 2023","interfaces":[{"interface":"back","average":{"1min":1.151,"5min":2.0699999999999998,"15min":1.4870000000000001},"min":{"1min":0.40600000000000003,"5min":0.38100000000000001,"15min":0.379},"max":{"1min":8.2210000000000001,"5min":8.7729999999999997,"15min":8.7729999999999997},"last":0.80800000000000005},{"interface":"front","average":{"1min":0.86799999999999999,"5min":2.3220000000000001,"15min":1.5289999999999999},"min":{"1min":0.45400000000000001,"5min":0.377,"15min":0.377},"max":{"1min":3.4500000000000002,"5min":13.856999999999999,"15min":18.673999999999999},"last":0.63500000000000001}]}]},{"osd":2,"up_from":93,"seq":399431958675,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2816704,"kb_used_data":2444920,"kb_used_omap":81,"kb_used_meta":371694,"kb_avail":90928448,"statfs":{"total":95995035648,"available":93110730752,"internally_reserved":0,"allocated":2503598080,"data_stored":2502339569,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":83750,"internal_metadata":380614874},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1,4],"upper_bound":16},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:01:51 2023","interfaces":[{"interface":"back","average":{"1min":0.76900000000000002,"5min":3.0600000000000001,"15min":1.671},"min":{"1min":0.22900000000000001,"5min":0.189,"15min":0.17999999999999999},"max":{"1min":2.923,"5min":29.498000000000001,"15min":29.498000000000001},"last":0.49199999999999999},{"interface":"front","average":{"1min":0.624,"5min":3.4249999999999998,"15min":1.946},"min":{"1min":0.193,"5min":0.193,"15min":0.193},"max":{"1min":1.9059999999999999,"5min":17.251000000000001,"15min":17.251000000000001},"last":0.35799999999999998}]},{"osd":1,"last update":"Sat Sep 23 21:01:51 2023","interfaces":[{"interface":"back","average":{"1min":1.163,"5min":2.5800000000000001,"15min":1.831},"min":{"1min":0.246,"5min":0.21199999999999999,"15min":0.20200000000000001},"max":{"1min":6.077,"5min":14.709,"15min":16.081},"last":4.1219999999999999},{"interface":"front","average":{"1min":1.107,"5min":3.7879999999999998,"15min":2.3919999999999999},"min":{"1min":0.33500000000000002,"5min":0.26300000000000001,"15min":0.23599999999999999},"max":{"1min":6.125,"5min":26.552,"15min":26.552},"last":3.0019999999999998}]},{"osd":3,"last update":"Sat Sep 23 21:02:34 2023","interfaces":[{"interface":"back","average":{"1min":2.137,"5min":4.1699999999999999,"15min":2.524},"min":{"1min":0.45500000000000002,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":5.8310000000000004,"5min":41.057000000000002,"15min":41.057000000000002},"last":2.6320000000000001},{"interface":"front","average":{"1min":1.9319999999999999,"5min":4.4340000000000002,"15min":2.8300000000000001},"min":{"1min":0.25,"5min":0.25,"15min":0.221},"max":{"1min":9.4870000000000001,"5min":41.012,"15min":41.012},"last":7.5190000000000001}]},{"osd":4,"last update":"Sat Sep 23 21:01:51 2023","interfaces":[{"interface":"back","average":{"1min":0.72799999999999998,"5min":1.706,"15min":1.3140000000000001},"min":{"1min":0.28399999999999997,"5min":0.28399999999999997,"15min":0.26400000000000001},"max":{"1min":2.8290000000000002,"5min":10.045,"15min":10.045},"last":0.40000000000000002},{"interface":"front","average":{"1min":0.78000000000000003,"5min":2.1560000000000001,"15min":1.538},"min":{"1min":0.36399999999999999,"5min":0.32400000000000001,"15min":0.313},"max":{"1min":3.0550000000000002,"5min":14.039999999999999,"15min":14.039999999999999},"last":7.1529999999999996}]},{"osd":5,"last update":"Sat Sep 23 21:01:51 2023","interfaces":[{"interface":"back","average":{"1min":0.65300000000000002,"5min":1.637,"15min":1.3959999999999999},"min":{"1min":0.30399999999999999,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":1.9119999999999999,"5min":10.069000000000001,"15min":10.069000000000001},"last":1.3839999999999999},{"interface":"front","average":{"1min":0.66500000000000004,"5min":2.073,"15min":1.534},"min":{"1min":0.31900000000000001,"5min":0.31900000000000001,"15min":0.30299999999999999},"max":{"1min":2.0880000000000001,"5min":13.98,"15min":13.98},"last":7.1109999999999998}]},{"osd":6,"last update":"Sat Sep 23 21:01:50 2023","interfaces":[{"interface":"back","average":{"1min":0.94599999999999995,"5min":2.2370000000000001,"15min":1.857},"min":{"1min":0.36599999999999999,"5min":0.312,"15min":0.312},"max":{"1min":5.3490000000000002,"5min":14.016,"15min":14.244999999999999},"last":7.0670000000000002},{"interface":"front","average":{"1min":0.90700000000000003,"5min":2.323,"15min":1.4610000000000001},"min":{"1min":0.38,"5min":0.374,"15min":0.374},"max":{"1min":5.4500000000000002,"5min":14.068,"15min":14.068},"last":7.1879999999999997}]},{"osd":7,"last update":"Sat Sep 23 21:01:51 2023","interfaces":[{"interface":"back","average":{"1min":0.84799999999999998,"5min":1.4359999999999999,"15min":1.155},"min":{"1min":0.45600000000000002,"5min":0.315,"15min":0.315},"max":{"1min":2.98,"5min":13.427,"15min":13.427},"last":0.52700000000000002},{"interface":"front","average":{"1min":0.79100000000000004,"5min":1.643,"15min":1.4099999999999999},"min":{"1min":0.36699999999999999,"5min":0.35099999999999998,"15min":0.34499999999999997},"max":{"1min":2.883,"5min":10.092000000000001,"15min":10.092000000000001},"last":0.43099999999999999}]}]},{"osd":3,"up_from":98,"seq":420906795148,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2665624,"kb_used_data":2226512,"kb_used_omap":7,"kb_used_meta":439096,"kb_avail":91079528,"statfs":{"total":95995035648,"available":93265436672,"internally_reserved":0,"allocated":2279948288,"data_stored":2278725993,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7576,"internal_metadata":449634920},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,3,0,0,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:02:32 2023","interfaces":[{"interface":"back","average":{"1min":1.288,"5min":3.6619999999999999,"15min":2.4039999999999999},"min":{"1min":0.188,"5min":0.188,"15min":0.188},"max":{"1min":4.7949999999999999,"5min":27.962,"15min":27.962},"last":0.45800000000000002},{"interface":"front","average":{"1min":2.8519999999999999,"5min":3.794,"15min":2.2709999999999999},"min":{"1min":0.20499999999999999,"5min":0.19800000000000001,"15min":0.19800000000000001},"max":{"1min":10.497,"5min":28.058,"15min":28.058},"last":1.633}]},{"osd":1,"last update":"Sat Sep 23 21:02:32 2023","interfaces":[{"interface":"back","average":{"1min":5.2779999999999996,"5min":3.8439999999999999,"15min":2.371},"min":{"1min":0.23999999999999999,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":74.093000000000004,"5min":74.093000000000004,"15min":74.093000000000004},"last":2.1680000000000001},{"interface":"front","average":{"1min":5.1210000000000004,"5min":3.887,"15min":2.4969999999999999},"min":{"1min":0.27700000000000002,"5min":0.23400000000000001,"15min":0.218},"max":{"1min":76.527000000000001,"5min":76.527000000000001,"15min":76.527000000000001},"last":4.9880000000000004}]},{"osd":2,"last update":"Sat Sep 23 21:02:32 2023","interfaces":[{"interface":"back","average":{"1min":2.101,"5min":4.1749999999999998,"15min":2.399},"min":{"1min":0.23599999999999999,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":6.5549999999999997,"5min":28.251999999999999,"15min":28.251999999999999},"last":1.234},{"interface":"front","average":{"1min":2.097,"5min":3.3199999999999998,"15min":2.3759999999999999},"min":{"1min":0.22,"5min":0.22,"15min":0.22},"max":{"1min":12.68,"5min":16.213999999999999,"15min":25.305},"last":0.36699999999999999}]},{"osd":4,"last update":"Sat Sep 23 21:02:32 2023","interfaces":[{"interface":"back","average":{"1min":1.75,"5min":2.5680000000000001,"15min":1.6519999999999999},"min":{"1min":0.29399999999999998,"5min":0.29399999999999998,"15min":0.29299999999999998},"max":{"1min":7.6820000000000004,"5min":18.239000000000001,"15min":18.239000000000001},"last":4.3849999999999998},{"interface":"front","average":{"1min":1.5469999999999999,"5min":2.0030000000000001,"15min":1.6559999999999999},"min":{"1min":0.29499999999999998,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":6.0060000000000002,"5min":14.808,"15min":22.853000000000002},"last":2.536}]},{"osd":5,"last update":"Sat Sep 23 21:02:32 2023","interfaces":[{"interface":"back","average":{"1min":1.7410000000000001,"5min":2.6739999999999999,"15min":1.6679999999999999},"min":{"1min":0.32900000000000001,"5min":0.32900000000000001,"15min":0.32500000000000001},"max":{"1min":8.0459999999999994,"5min":18.324999999999999,"15min":18.324999999999999},"last":2.2080000000000002},{"interface":"front","average":{"1min":2.4860000000000002,"5min":2.9279999999999999,"15min":1.746},"min":{"1min":0.40899999999999997,"5min":0.33300000000000002,"15min":0.33100000000000002},"max":{"1min":10.558,"5min":13.529999999999999,"15min":13.529999999999999},"last":5.3449999999999998}]},{"osd":6,"last update":"Sat Sep 23 21:01:52 2023","interfaces":[{"interface":"back","average":{"1min":0.97199999999999998,"5min":1.9490000000000001,"15min":1.2829999999999999},"min":{"1min":0.40200000000000002,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":4.1230000000000002,"5min":14.833,"15min":14.833},"last":0.40100000000000002},{"interface":"front","average":{"1min":0.99099999999999999,"5min":2.5270000000000001,"15min":1.7829999999999999},"min":{"1min":0.377,"5min":0.377,"15min":0.377},"max":{"1min":2.242,"5min":13.576000000000001,"15min":13.576000000000001},"last":5.3079999999999998}]},{"osd":7,"last update":"Sat Sep 23 21:02:32 2023","interfaces":[{"interface":"back","average":{"1min":2.3420000000000001,"5min":2.7240000000000002,"15min":1.8440000000000001},"min":{"1min":0.40600000000000003,"5min":0.39200000000000002,"15min":0.36099999999999999},"max":{"1min":10.715999999999999,"5min":18.289999999999999,"15min":18.289999999999999},"last":1.278},{"interface":"front","average":{"1min":2.8959999999999999,"5min":3.0880000000000001,"15min":1.8440000000000001},"min":{"1min":0.46500000000000002,"5min":0.38900000000000001,"15min":0.379},"max":{"1min":11.798999999999999,"5min":13.603999999999999,"15min":13.603999999999999},"last":1.597}]}]},{"osd":4,"up_from":31,"seq":133143986456,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2097600,"kb_used_data":1920440,"kb_used_omap":0,"kb_used_meta":177152,"kb_avail":91647552,"statfs":{"total":95995035648,"available":93847093248,"internally_reserved":0,"allocated":1966530560,"data_stored":1965626703,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":181403648},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:01:55 2023","interfaces":[{"interface":"back","average":{"1min":0.55900000000000005,"5min":1.7729999999999999,"15min":1.216},"min":{"1min":0.254,"5min":0.214,"15min":0.214},"max":{"1min":1.712,"5min":11.784000000000001,"15min":11.784000000000001},"last":0.254},{"interface":"front","average":{"1min":0.66000000000000003,"5min":1.8200000000000001,"15min":1.379},"min":{"1min":0.31900000000000001,"5min":0.27700000000000002,"15min":0.25800000000000001},"max":{"1min":2.1360000000000001,"5min":11.766,"15min":11.766},"last":0.33700000000000002}]},{"osd":1,"last update":"Sat Sep 23 21:02:24 2023","interfaces":[{"interface":"back","average":{"1min":0.94899999999999995,"5min":1.696,"15min":1.2969999999999999},"min":{"1min":0.255,"5min":0.246,"15min":0.23599999999999999},"max":{"1min":5.0229999999999997,"5min":10.153,"15min":10.153},"last":0.85499999999999998},{"interface":"front","average":{"1min":1.0940000000000001,"5min":2.2280000000000002,"15min":1.3700000000000001},"min":{"1min":0.309,"5min":0.26100000000000001,"15min":0.23200000000000001},"max":{"1min":7.3209999999999997,"5min":12.292999999999999,"15min":12.292999999999999},"last":0.47799999999999998}]},{"osd":2,"last update":"Sat Sep 23 21:01:55 2023","interfaces":[{"interface":"back","average":{"1min":0.68300000000000005,"5min":1.927,"15min":1.3899999999999999},"min":{"1min":0.33400000000000002,"5min":0.32500000000000001,"15min":0.29099999999999998},"max":{"1min":1.883,"5min":9.8469999999999995,"15min":15.288},"last":7.6619999999999999},{"interface":"front","average":{"1min":0.75700000000000001,"5min":1.8680000000000001,"15min":1.4950000000000001},"min":{"1min":0.32700000000000001,"5min":0.28599999999999998,"15min":0.28299999999999997},"max":{"1min":2.1789999999999998,"5min":12.855,"15min":15.262},"last":7.609}]},{"osd":3,"last update":"Sat Sep 23 21:02:24 2023","interfaces":[{"interface":"back","average":{"1min":1.514,"5min":2.3109999999999999,"15min":1.976},"min":{"1min":0.42699999999999999,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":7.1609999999999996,"5min":12.106999999999999,"15min":31.562999999999999},"last":0.72099999999999997},{"interface":"front","average":{"1min":1.3069999999999999,"5min":2.3149999999999999,"15min":2.0110000000000001},"min":{"1min":0.499,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":7.1429999999999998,"5min":12.477,"15min":31.574000000000002},"last":0.71099999999999997}]},{"osd":5,"last update":"Sat Sep 23 21:01:40 2023","interfaces":[{"interface":"back","average":{"1min":0.46800000000000003,"5min":0.55400000000000005,"15min":0.52000000000000002},"min":{"1min":0.27900000000000003,"5min":0.248,"15min":0.22700000000000001},"max":{"1min":1.1830000000000001,"5min":3.008,"15min":3.3279999999999998},"last":0.42499999999999999},{"interface":"front","average":{"1min":0.84999999999999998,"5min":0.76900000000000002,"15min":0.64500000000000002},"min":{"1min":0.29699999999999999,"5min":0.27300000000000002,"15min":0.251},"max":{"1min":4.6219999999999999,"5min":4.6219999999999999,"15min":7.7619999999999996},"last":0.37}]},{"osd":6,"last update":"Sat Sep 23 21:01:51 2023","interfaces":[{"interface":"back","average":{"1min":0.45400000000000001,"5min":0.73399999999999999,"15min":0.56699999999999995},"min":{"1min":0.29899999999999999,"5min":0.26900000000000002,"15min":0.26500000000000001},"max":{"1min":0.65700000000000003,"5min":4.6909999999999998,"15min":4.6909999999999998},"last":0.32500000000000001},{"interface":"front","average":{"1min":0.54200000000000004,"5min":0.69699999999999995,"15min":0.66100000000000003},"min":{"1min":0.38,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":0.81799999999999995,"5min":3.456,"15min":3.456},"last":0.45400000000000001}]},{"osd":7,"last update":"Sat Sep 23 21:02:24 2023","interfaces":[{"interface":"back","average":{"1min":0.61399999999999999,"5min":0.68799999999999994,"15min":0.69899999999999995},"min":{"1min":0.33700000000000002,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":1.248,"5min":3.4940000000000002,"15min":7.383},"last":0.73599999999999999},{"interface":"front","average":{"1min":0.72699999999999998,"5min":0.74199999999999999,"15min":0.70099999999999996},"min":{"1min":0.377,"5min":0.32700000000000001,"15min":0.28199999999999997},"max":{"1min":2.1699999999999999,"5min":5.6980000000000004,"15min":7.8310000000000004},"last":0.39600000000000002}]}]},{"osd":5,"up_from":39,"seq":167503724821,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3336500,"kb_used_data":3150700,"kb_used_omap":0,"kb_used_meta":185792,"kb_avail":90408652,"statfs":{"total":95995035648,"available":92578459648,"internally_reserved":0,"allocated":3226316800,"data_stored":3225421519,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":190251008},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,2,2,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:01:45 2023","interfaces":[{"interface":"back","average":{"1min":0.83299999999999996,"5min":1.978,"15min":1.1579999999999999},"min":{"1min":0.23599999999999999,"5min":0.20399999999999999,"15min":0.20399999999999999},"max":{"1min":2.3279999999999998,"5min":12.795999999999999,"15min":12.795999999999999},"last":3.9700000000000002},{"interface":"front","average":{"1min":0.67700000000000005,"5min":2.117,"15min":1.341},"min":{"1min":0.311,"5min":0.24099999999999999,"15min":0.223},"max":{"1min":1.8320000000000001,"5min":16.219999999999999,"15min":16.219999999999999},"last":0.27300000000000002}]},{"osd":1,"last update":"Sat Sep 23 21:02:20 2023","interfaces":[{"interface":"back","average":{"1min":4.8710000000000004,"5min":2.661,"15min":1.585},"min":{"1min":0.26800000000000002,"5min":0.248,"15min":0.23300000000000001},"max":{"1min":62.320999999999998,"5min":62.320999999999998,"15min":62.320999999999998},"last":0.71299999999999997},{"interface":"front","average":{"1min":4.6079999999999997,"5min":2.7040000000000002,"15min":1.6020000000000001},"min":{"1min":0.30499999999999999,"5min":0.27200000000000002,"15min":0.22800000000000001},"max":{"1min":61.722999999999999,"5min":61.722999999999999,"15min":61.722999999999999},"last":1.0309999999999999}]},{"osd":2,"last update":"Sat Sep 23 21:01:59 2023","interfaces":[{"interface":"back","average":{"1min":0.82699999999999996,"5min":1.821,"15min":1.2649999999999999},"min":{"1min":0.23499999999999999,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":2.3090000000000002,"5min":10.579000000000001,"15min":12.651},"last":4.5270000000000001},{"interface":"front","average":{"1min":0.92300000000000004,"5min":2.1669999999999998,"15min":1.4139999999999999},"min":{"1min":0.34000000000000002,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":2.8879999999999999,"5min":12.300000000000001,"15min":12.300000000000001},"last":2.3330000000000002}]},{"osd":3,"last update":"Sat Sep 23 21:02:20 2023","interfaces":[{"interface":"back","average":{"1min":6.7009999999999996,"5min":3.8119999999999998,"15min":2.1030000000000002},"min":{"1min":0.33500000000000002,"5min":0.33000000000000002,"15min":0.28599999999999998},"max":{"1min":61.667999999999999,"5min":61.667999999999999,"15min":61.667999999999999},"last":0.375},{"interface":"front","average":{"1min":7.0460000000000003,"5min":3.2389999999999999,"15min":1.78},"min":{"1min":0.377,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":62.276000000000003,"5min":62.276000000000003,"15min":62.276000000000003},"last":4.3879999999999999}]},{"osd":4,"last update":"Sat Sep 23 21:02:37 2023","interfaces":[{"interface":"back","average":{"1min":0.96399999999999997,"5min":0.67400000000000004,"15min":0.59699999999999998},"min":{"1min":0.25700000000000001,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":3.9279999999999999,"5min":4.1520000000000001,"15min":4.1520000000000001},"last":0.34300000000000003},{"interface":"front","average":{"1min":1.288,"5min":0.76000000000000001,"15min":0.64600000000000002},"min":{"1min":0.28000000000000003,"5min":0.27400000000000002,"15min":0.23599999999999999},"max":{"1min":8.4350000000000005,"5min":8.4350000000000005,"15min":8.4350000000000005},"last":0.98899999999999999}]},{"osd":6,"last update":"Sat Sep 23 21:01:45 2023","interfaces":[{"interface":"back","average":{"1min":0.59999999999999998,"5min":0.72999999999999998,"15min":0.61699999999999999},"min":{"1min":0.27900000000000003,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":1.5820000000000001,"5min":4.359,"15min":4.359},"last":0.67700000000000005},{"interface":"front","average":{"1min":0.53800000000000003,"5min":0.75,"15min":0.746},"min":{"1min":0.37,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":0.89900000000000002,"5min":3.847,"15min":3.847},"last":0.76100000000000001}]},{"osd":7,"last update":"Sat Sep 23 21:02:20 2023","interfaces":[{"interface":"back","average":{"1min":1.1819999999999999,"5min":0.83799999999999997,"15min":0.71999999999999997},"min":{"1min":0.36099999999999999,"5min":0.316,"15min":0.30199999999999999},"max":{"1min":4.1310000000000002,"5min":4.4240000000000004,"15min":8.5649999999999995},"last":3.1280000000000001},{"interface":"front","average":{"1min":0.90100000000000002,"5min":0.74099999999999999,"15min":0.70199999999999996},"min":{"1min":0.35299999999999998,"5min":0.33900000000000002,"15min":0.27700000000000002},"max":{"1min":4.6479999999999997,"5min":4.6479999999999997,"15min":4.6479999999999997},"last":0.79500000000000004}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":2364407808,"data_stored":2364407808,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":2767192064,"data_stored":2767192064,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":2328756224,"data_stored":2328756224,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":2291597312,"data_stored":2291597312,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1967718400,"data_stored":1967718400,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":3093561344,"data_stored":3093561344,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1974665216,"data_stored":1974665216,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1964179456,"data_stored":1964179456,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T21:02:46.672 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T21:02:46.673 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T21:02:46.676 INFO:teuthology.orchestra.run.smithi026.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T21:02:46.677 INFO:teuthology.orchestra.run.smithi026.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T21:02:46.677 INFO:teuthology.orchestra.run.smithi026.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T21:02:46.677 INFO:teuthology.orchestra.run.smithi026.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T21:02:46.677 INFO:teuthology.orchestra.run.smithi026.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T21:02:46.677 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T21:02:46.677 INFO:teuthology.orchestra.run.smithi026.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T21:02:46.677 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T21:02:46.737 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:02:46.769 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:02:46.776 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 75 16 98936 98920 82.3804 41.4375 0.0139825 0.0121322 2023-09-23T21:02:46.811 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:02:46.817 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:02:46.817 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:02:46.875 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:02:46.918 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:02:46.945 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:02:46.988 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:02:46.995 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:02:46.995 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:02:47.096 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:02:47.178 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:02:47.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:02:47 smithi026 ceph-mon[179688]: from='client.45605 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T21:02:47.383 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:02:47 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/1075617171' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:02:47.383 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:02:47 smithi026 ceph-mon[175151]: from='client.45605 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T21:02:47.383 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:02:47 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/1075617171' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:02:47.466 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T21:02:47.473 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:02:47.474 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:02:47.574 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:02:47.606 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:02:47 smithi070 ceph-mon[168282]: from='client.45605 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T21:02:47.606 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:02:47 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/1075617171' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:02:47.613 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:02:47.659 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:02:47.662 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:02:47.662 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:02:47.763 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:02:47.765 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:02:47.786 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 76 16 99787 99771 81.9894 53.1875 0.0191722 0.012191 2023-09-23T21:02:47.870 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:02:48.087 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T21:02:48.111 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:02:48.112 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:02:48.213 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:02:48.304 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:02:48.567 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:02:48.568 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:02:48 smithi026 ceph-mon[175151]: pgmap v723: 109 pgs: 109 active+clean; 5.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 51 MiB/s wr, 821 op/s 2023-09-23T21:02:48.568 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:02:48 smithi026 ceph-mon[179688]: pgmap v723: 109 pgs: 109 active+clean; 5.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 51 MiB/s wr, 821 op/s 2023-09-23T21:02:48.576 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:02:48.576 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:02:48.609 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:02:48.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:02:48 smithi070 ceph-mon[168282]: pgmap v723: 109 pgs: 109 active+clean; 5.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 51 MiB/s wr, 821 op/s 2023-09-23T21:02:48.678 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:02:48.784 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 77 16 100644 100628 81.6195 53.5625 0.0183673 0.012246 2023-09-23T21:02:48.832 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:02:48.901 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:02:48.909 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:02:48.910 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:02:49.011 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:02:49.087 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:02:49.166 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:02:49.185 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:02:49.185 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:02:49.286 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:02:49.370 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:02:49.448 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T21:02:49.507 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:02:49.512 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:02:49.513 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:02:49.539 INFO:teuthology.orchestra.run.smithi026.stdout:ERROR: (22) Invalid argument 2023-09-23T21:02:49.539 INFO:teuthology.orchestra.run.smithi026.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T21:02:49.613 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:02:49.723 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:02:49.784 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 78 16 101418 101402 81.1935 48.375 0.0133344 0.0123111 2023-09-23T21:02:49.882 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 21:02:49 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[164029]: ::ffff:172.21.15.70 - - [23/Sep/2023:21:02:49] "GET /metrics HTTP/1.1" 200 36291 "" "Prometheus/2.43.0" 2023-09-23T21:02:49.919 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T21:02:49.923 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:02:49.923 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:02:50.024 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:02:50.097 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:02:50.233 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T21:02:50.237 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:02:50.238 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:02:50.338 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:02:50.434 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:02:50.636 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:02:50 smithi026 ceph-mon[175151]: pgmap v724: 109 pgs: 109 active+clean; 6.0 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 46 MiB/s wr, 743 op/s 2023-09-23T21:02:50.636 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:02:50 smithi026 ceph-mon[179688]: pgmap v724: 109 pgs: 109 active+clean; 6.0 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 46 MiB/s wr, 743 op/s 2023-09-23T21:02:50.788 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 79 16 102262 102246 80.8328 52.75 0.0120809 0.0123666 2023-09-23T21:02:50.789 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:02:50.795 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:02:50.796 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:02:50.835 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_historic_ops 2023-09-23T21:02:50.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:02:50 smithi070 ceph-mon[168282]: pgmap v724: 109 pgs: 109 active+clean; 6.0 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 46 MiB/s wr, 743 op/s 2023-09-23T21:02:50.896 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:02:50.936 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:02:50.980 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:02:50.983 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:02:50.984 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:02:51.085 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:02:51.175 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:02:51.200 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:02:51.264 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:02:51.278 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:02:51.279 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:02:51.379 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:02:51.421 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:02:51 smithi026 ceph-mon[175151]: pgmap v725: 109 pgs: 109 active+clean; 6.1 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 55 MiB/s wr, 880 op/s 2023-09-23T21:02:51.421 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:02:51 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/2409179114' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:02:51.421 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:02:51 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:02:51.421 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:02:51 smithi026 ceph-mon[179688]: pgmap v725: 109 pgs: 109 active+clean; 6.1 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 55 MiB/s wr, 880 op/s 2023-09-23T21:02:51.421 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:02:51 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/2409179114' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:02:51.421 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:02:51 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:02:51.426 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:02:51.449 INFO:teuthology.orchestra.run.smithi026.stderr:nodeep-scrub is set 2023-09-23T21:02:51.466 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:02:51.469 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:02:51.469 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:02:51.570 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:02:51.601 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:02:51.658 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:02:51.659 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:02:51 smithi070 ceph-mon[168282]: pgmap v725: 109 pgs: 109 active+clean; 6.1 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 55 MiB/s wr, 880 op/s 2023-09-23T21:02:51.659 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:02:51 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/2409179114' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:02:51.659 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:02:51 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:02:51.660 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:02:51.660 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:02:51.696 INFO:teuthology.orchestra.run.smithi026.stdout: 2023-09-23T21:02:51.711 INFO:teuthology.orchestra.run.smithi026.stderr:dumped all 2023-09-23T21:02:51.761 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:02:51.785 INFO:tasks.radosbench.radosbench.0.smithi026.stdout:2023-09-23T21:02:51.785606+0000 min lat: 0.00122595 max lat: 0.190811 avg lat: 0.0124356 2023-09-23T21:02:51.785 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T21:02:51.785 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 80 16 102977 102961 80.3812 44.6875 0.022209 0.0124356 2023-09-23T21:02:51.834 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:02:52.039 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T21:02:52.043 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:02:52.043 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:02:52.144 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:02:52.221 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:02:52.645 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:02:52.652 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:02:52 smithi026 ceph-mon[175151]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:02:52.652 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:02:52 smithi026 ceph-mon[175151]: failed to encode map e275 with expected crc 2023-09-23T21:02:52.652 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:02:52 smithi026 ceph-mon[175151]: failed to encode map e275 with expected crc 2023-09-23T21:02:52.652 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:02:52 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T21:02:52.652 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:02:52 smithi026 ceph-mon[175151]: osdmap e275: 8 total, 8 up, 8 in 2023-09-23T21:02:52.652 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:02:52 smithi026 ceph-mon[175151]: from='client.55486 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T21:02:52.652 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:02:52 smithi026 ceph-mon[179688]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:02:52.653 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:02:52 smithi026 ceph-mon[179688]: failed to encode map e275 with expected crc 2023-09-23T21:02:52.653 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:02:52 smithi026 ceph-mon[179688]: failed to encode map e275 with expected crc 2023-09-23T21:02:52.653 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:02:52 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T21:02:52.653 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:02:52 smithi026 ceph-mon[179688]: osdmap e275: 8 total, 8 up, 8 in 2023-09-23T21:02:52.653 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:02:52 smithi026 ceph-mon[179688]: from='client.55486 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T21:02:52.654 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:02:52.655 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:02:52.700 INFO:teuthology.orchestra.run.smithi026.stdout:{"pg_ready":true,"pg_map":{"version":726,"stamp":"2023-09-23T21:02:51.397249+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":6574839712,"num_objects":100339,"num_object_clones":0,"num_object_copies":301017,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100339,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":100376,"num_write_kb":6420189,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":22,"num_bytes_recovered":5990,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":103751,"ondisk_log_size":103751,"up":327,"acting":327,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":330,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":21897376,"kb_used_data":19613280,"kb_used_omap":136,"kb_used_meta":2283895,"kb_avail":728063840,"statfs":{"total":767960285184,"available":745537372160,"internally_reserved":0,"allocated":20083998720,"data_stored":20075351358,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":139741,"internal_metadata":2338709027},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,5,3,6,3],"upper_bound":64},"perf_stat":{"commit_latency_ms":8,"apply_latency_ms":8,"commit_latency_ns":8000000,"apply_latency_ns":8000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":414777344,"num_objects":6329,"num_object_clones":0,"num_object_copies":18987,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6329,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6329,"num_write_kb":405056,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"8.847901"},"pg_stats":[{"pgid":"3.1a","version":"68'1","reported_seq":151,"reported_epoch":273,"state":"active+clean","last_fresh":"2023-09-23T21:02:41.219333+0000","last_change":"2023-09-23T20:54:13.946695+0000","last_active":"2023-09-23T21:02:41.219333+0000","last_peered":"2023-09-23T21:02:41.219333+0000","last_clean":"2023-09-23T21:02:41.219333+0000","last_became_active":"2023-09-23T20:54:13.945399+0000","last_became_peered":"2023-09-23T20:54:13.945399+0000","last_unstale":"2023-09-23T21:02:41.219333+0000","last_undegraded":"2023-09-23T21:02:41.219333+0000","last_fullsized":"2023-09-23T21:02:41.219333+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:32:52.365464+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":267,"reported_epoch":273,"state":"active+clean","last_fresh":"2023-09-23T21:02:41.248163+0000","last_change":"2023-09-23T20:54:13.942019+0000","last_active":"2023-09-23T21:02:41.248163+0000","last_peered":"2023-09-23T21:02:41.248163+0000","last_clean":"2023-09-23T21:02:41.248163+0000","last_became_active":"2023-09-23T20:54:13.941647+0000","last_became_peered":"2023-09-23T20:54:13.941647+0000","last_unstale":"2023-09-23T21:02:41.248163+0000","last_undegraded":"2023-09-23T21:02:41.248163+0000","last_fullsized":"2023-09-23T21:02:41.248163+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:30:54.400944+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":245,"reported_epoch":273,"state":"active+clean","last_fresh":"2023-09-23T21:02:41.218814+0000","last_change":"2023-09-23T20:54:13.941394+0000","last_active":"2023-09-23T21:02:41.218814+0000","last_peered":"2023-09-23T21:02:41.218814+0000","last_clean":"2023-09-23T21:02:41.218814+0000","last_became_active":"2023-09-23T20:54:13.941042+0000","last_became_peered":"2023-09-23T20:54:13.941042+0000","last_unstale":"2023-09-23T21:02:41.218814+0000","last_undegraded":"2023-09-23T21:02:41.218814+0000","last_fullsized":"2023-09-23T21:02:41.218814+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:19:41.401184+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":260,"reported_epoch":273,"state":"active+clean","last_fresh":"2023-09-23T21:02:41.226514+0000","last_change":"2023-09-23T20:57:44.979811+0000","last_active":"2023-09-23T21:02:41.226514+0000","last_peered":"2023-09-23T21:02:41.226514+0000","last_clean":"2023-09-23T21:02:41.226514+0000","last_became_active":"2023-09-23T20:57:44.979515+0000","last_became_peered":"2023-09-23T20:57:44.979515+0000","last_unstale":"2023-09-23T21:02:41.226514+0000","last_undegraded":"2023-09-23T21:02:41.226514+0000","last_fullsized":"2023-09-23T21:02:41.226514+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:38:22.162140+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":258,"reported_epoch":273,"state":"active+clean","last_fresh":"2023-09-23T21:02:41.248963+0000","last_change":"2023-09-23T20:51:10.833613+0000","last_active":"2023-09-23T21:02:41.248963+0000","last_peered":"2023-09-23T21:02:41.248963+0000","last_clean":"2023-09-23T21:02:41.248963+0000","last_became_active":"2023-09-23T20:51:10.833208+0000","last_became_peered":"2023-09-23T20:51:10.833208+0000","last_unstale":"2023-09-23T21:02:41.248963+0000","last_undegraded":"2023-09-23T21:02:41.248963+0000","last_fullsized":"2023-09-23T21:02:41.248963+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:51:28.246014+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":239,"reported_epoch":273,"state":"active+clean","last_fresh":"2023-09-23T21:02:41.218631+0000","last_change":"2023-09-23T20:55:35.958384+0000","last_active":"2023-09-23T21:02:41.218631+0000","last_peered":"2023-09-23T21:02:41.218631+0000","last_clean":"2023-09-23T21:02:41.218631+0000","last_became_active":"2023-09-23T20:55:35.954563+0000","last_became_peered":"2023-09-23T20:55:35.954563+0000","last_unstale":"2023-09-23T21:02:41.218631+0000","last_undegraded":"2023-09-23T21:02:41.218631+0000","last_fullsized":"2023-09-23T21:02:41.218631+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:37:48.332259+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":250,"reported_epoch":273,"state":"active+clean","last_fresh":"2023-09-23T21:02:41.226526+0000","last_change":"2023-09-23T20:54:31.885514+0000","last_active":"2023-09-23T21:02:41.226526+0000","last_peered":"2023-09-23T21:02:41.226526+0000","last_clean":"2023-09-23T21:02:41.226526+0000","last_became_active":"2023-09-23T20:54:31.884986+0000","last_became_peered":"2023-09-23T20:54:31.884986+0000","last_unstale":"2023-09-23T21:02:41.226526+0000","last_undegraded":"2023-09-23T21:02:41.226526+0000","last_fullsized":"2023-09-23T21:02:41.226526+0000","mapping_epoch":136,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:26:08.100220+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":256,"reported_epoch":274,"state":"active+clean","last_fresh":"2023-09-23T21:02:43.287259+0000","last_change":"2023-09-23T20:55:35.951889+0000","last_active":"2023-09-23T21:02:43.287259+0000","last_peered":"2023-09-23T21:02:43.287259+0000","last_clean":"2023-09-23T21:02:43.287259+0000","last_became_active":"2023-09-23T20:55:35.951563+0000","last_became_peered":"2023-09-23T20:55:35.951563+0000","last_unstale":"2023-09-23T21:02:43.287259+0000","last_undegraded":"2023-09-23T21:02:43.287259+0000","last_fullsized":"2023-09-23T21:02:43.287259+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:41:53.098781+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"79'161","reported_seq":562,"reported_epoch":273,"state":"active+clean","last_fresh":"2023-09-23T21:02:41.243217+0000","last_change":"2023-09-23T20:57:25.096559+0000","last_active":"2023-09-23T21:02:41.243217+0000","last_peered":"2023-09-23T21:02:41.243217+0000","last_clean":"2023-09-23T21:02:41.243217+0000","last_became_active":"2023-09-23T20:57:15.975674+0000","last_became_peered":"2023-09-23T20:57:15.975674+0000","last_unstale":"2023-09-23T21:02:41.243217+0000","last_undegraded":"2023-09-23T21:02:41.243217+0000","last_fullsized":"2023-09-23T21:02:41.243217+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:25.096401+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:25.096401+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:05:05.448404+0000","scrub_duration":0.002859393,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":147,"reported_epoch":273,"state":"active+clean","last_fresh":"2023-09-23T21:02:41.225726+0000","last_change":"2023-09-23T20:54:31.890678+0000","last_active":"2023-09-23T21:02:41.225726+0000","last_peered":"2023-09-23T21:02:41.225726+0000","last_clean":"2023-09-23T21:02:41.225726+0000","last_became_active":"2023-09-23T20:54:31.890469+0000","last_became_peered":"2023-09-23T20:54:31.890469+0000","last_unstale":"2023-09-23T21:02:41.225726+0000","last_undegraded":"2023-09-23T21:02:41.225726+0000","last_fullsized":"2023-09-23T21:02:41.225726+0000","mapping_epoch":136,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:55:08.790146+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":217,"reported_epoch":274,"state":"active+clean","last_fresh":"2023-09-23T21:02:43.285270+0000","last_change":"2023-09-23T20:55:35.952661+0000","last_active":"2023-09-23T21:02:43.285270+0000","last_peered":"2023-09-23T21:02:43.285270+0000","last_clean":"2023-09-23T21:02:43.285270+0000","last_became_active":"2023-09-23T20:55:35.952349+0000","last_became_peered":"2023-09-23T20:55:35.952349+0000","last_unstale":"2023-09-23T21:02:43.285270+0000","last_undegraded":"2023-09-23T21:02:43.285270+0000","last_fullsized":"2023-09-23T21:02:43.285270+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:00:03.110126+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":93,"reported_epoch":273,"state":"active+clean","last_fresh":"2023-09-23T21:02:41.248241+0000","last_change":"2023-09-23T20:56:45.988607+0000","last_active":"2023-09-23T21:02:41.248241+0000","last_peered":"2023-09-23T21:02:41.248241+0000","last_clean":"2023-09-23T21:02:41.248241+0000","last_became_active":"2023-09-23T20:56:45.988206+0000","last_became_peered":"2023-09-23T20:56:45.988206+0000","last_unstale":"2023-09-23T21:02:41.248241+0000","last_undegraded":"2023-09-23T21:02:41.248241+0000","last_fullsized":"2023-09-23T21:02:41.248241+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":205,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:01:19.783688+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,4],"acting":[3,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":91,"reported_epoch":273,"state":"active+clean","last_fresh":"2023-09-23T21:02:41.225892+0000","last_change":"2023-09-23T20:56:18.139142+0000","last_active":"2023-09-23T21:02:41.225892+0000","last_peered":"2023-09-23T21:02:41.225892+0000","last_clean":"2023-09-23T21:02:41.225892+0000","last_became_active":"2023-09-23T20:56:18.138844+0000","last_became_peered":"2023-09-23T20:56:18.138844+0000","last_unstale":"2023-09-23T21:02:41.225892+0000","last_undegraded":"2023-09-23T21:02:41.225892+0000","last_fullsized":"2023-09-23T21:02:41.225892+0000","mapping_epoch":192,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":193,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:09:02.217683+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,2],"acting":[5,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.14","version":"79'22","reported_seq":178,"reported_epoch":273,"state":"active+clean","last_fresh":"2023-09-23T21:02:41.219475+0000","last_change":"2023-09-23T20:55:35.949418+0000","last_active":"2023-09-23T21:02:41.219475+0000","last_peered":"2023-09-23T21:02:41.219475+0000","last_clean":"2023-09-23T21:02:41.219475+0000","last_became_active":"2023-09-23T20:55:35.948566+0000","last_became_peered":"2023-09-23T20:55:35.948566+0000","last_unstale":"2023-09-23T21:02:41.219475+0000","last_undegraded":"2023-09-23T21:02:41.219475+0000","last_fullsized":"2023-09-23T21:02:41.219475+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:59:37.433128+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4734,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":168,"reported_epoch":273,"state":"active+clean","last_fresh":"2023-09-23T21:02:41.219743+0000","last_change":"2023-09-23T20:55:35.949178+0000","last_active":"2023-09-23T21:02:41.219743+0000","last_peered":"2023-09-23T21:02:41.219743+0000","last_clean":"2023-09-23T21:02:41.219743+0000","last_became_active":"2023-09-23T20:55:35.948795+0000","last_became_peered":"2023-09-23T20:55:35.948795+0000","last_unstale":"2023-09-23T21:02:41.219743+0000","last_undegraded":"2023-09-23T21:02:41.219743+0000","last_fullsized":"2023-09-23T21:02:41.219743+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:13:08.996932+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.11","version":"79'161","reported_seq":504,"reported_epoch":273,"state":"active+clean","last_fresh":"2023-09-23T21:02:41.226451+0000","last_change":"2023-09-23T20:57:28.756796+0000","last_active":"2023-09-23T21:02:41.226451+0000","last_peered":"2023-09-23T21:02:41.226451+0000","last_clean":"2023-09-23T21:02:41.226451+0000","last_became_active":"2023-09-23T20:57:15.980606+0000","last_became_peered":"2023-09-23T20:57:15.980606+0000","last_unstale":"2023-09-23T21:02:41.226451+0000","last_undegraded":"2023-09-23T21:02:41.226451+0000","last_fullsized":"2023-09-23T21:02:41.226451+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:28.756634+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:28.756634+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:09:19.150791+0000","scrub_duration":0.0064925119999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":194,"reported_epoch":273,"state":"active+clean","last_fresh":"2023-09-23T21:02:41.219605+0000","last_change":"2023-09-23T20:56:45.992577+0000","last_active":"2023-09-23T21:02:41.219605+0000","last_peered":"2023-09-23T21:02:41.219605+0000","last_clean":"2023-09-23T21:02:41.219605+0000","last_became_active":"2023-09-23T20:56:45.992297+0000","last_became_peered":"2023-09-23T20:56:45.992297+0000","last_unstale":"2023-09-23T21:02:41.219605+0000","last_undegraded":"2023-09-23T21:02:41.219605+0000","last_fullsized":"2023-09-23T21:02:41.219605+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":205,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:00:45.547202+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,2],"acting":[4,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":241,"reported_epoch":274,"state":"active+clean","last_fresh":"2023-09-23T21:02:43.285838+0000","last_change":"2023-09-23T20:51:10.831262+0000","last_active":"2023-09-23T21:02:43.285838+0000","last_peered":"2023-09-23T21:02:43.285838+0000","last_clean":"2023-09-23T21:02:43.285838+0000","last_became_active":"2023-09-23T20:51:10.830931+0000","last_became_peered":"2023-09-23T20:51:10.830931+0000","last_unstale":"2023-09-23T21:02:43.285838+0000","last_undegraded":"2023-09-23T21:02:43.285838+0000","last_fullsized":"2023-09-23T21:02:43.285838+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:51:54.982505+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.10","version":"79'161","reported_seq":511,"reported_epoch":273,"state":"active+clean","last_fresh":"2023-09-23T21:02:41.225198+0000","last_change":"2023-09-23T20:57:20.708003+0000","last_active":"2023-09-23T21:02:41.225198+0000","last_peered":"2023-09-23T21:02:41.225198+0000","last_clean":"2023-09-23T21:02:41.225198+0000","last_became_active":"2023-09-23T20:57:15.975302+0000","last_became_peered":"2023-09-23T20:57:15.975302+0000","last_unstale":"2023-09-23T21:02:41.225198+0000","last_undegraded":"2023-09-23T21:02:41.225198+0000","last_fullsized":"2023-09-23T21:02:41.225198+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:20.707939+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:20.707939+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:16:51.721994+0000","scrub_duration":0.0017733409999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":250,"reported_epoch":273,"state":"active+clean","last_fresh":"2023-09-23T21:02:41.226988+0000","last_change":"2023-09-23T20:57:44.979736+0000","last_active":"2023-09-23T21:02:41.226988+0000","last_peered":"2023-09-23T21:02:41.226988+0000","last_clean":"2023-09-23T21:02:41.226988+0000","last_became_active":"2023-09-23T20:57:44.979356+0000","last_became_peered":"2023-09-23T20:57:44.979356+0000","last_unstale":"2023-09-23T21:02:41.226988+0000","last_undegraded":"2023-09-23T21:02:41.226988+0000","last_fullsized":"2023-09-23T21:02:41.226988+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:14:43.848832+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":147,"reported_epoch":273,"state":"active+clean","last_fresh":"2023-09-23T21:02:41.225731+0000","last_change":"2023-09-23T20:54:31.887814+0000","last_active":"2023-09-23T21:02:41.225731+0000","last_peered":"2023-09-23T21:02:41.225731+0000","last_clean":"2023-09-23T21:02:41.225731+0000","last_became_active":"2023-09-23T20:54:31.887195+0000","last_became_peered":"2023-09-23T20:54:31.887195+0000","last_unstale":"2023-09-23T21:02:41.225731+0000","last_undegraded":"2023-09-23T21:02:41.225731+0000","last_fullsized":"2023-09-23T21:02:41.225731+0000","mapping_epoch":136,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:10:52.089553+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.13","version":"79'161","reported_seq":575,"reported_epoch":273,"state":"active+clean","last_fresh":"2023-09-23T21:02:41.218777+0000","last_change":"2023-09-23T20:57:48.262970+0000","last_active":"2023-09-23T21:02:41.218777+0000","last_peered":"2023-09-23T21:02:41.218777+0000","last_clean":"2023-09-23T21:02:41.218777+0000","last_became_active":"2023-09-23T20:57:48.262663+0000","last_became_peered":"2023-09-23T20:57:48.262663+0000","last_unstale":"2023-09-23T21:02:41.218777+0000","last_undegraded":"2023-09-23T21:02:41.218777+0000","last_fullsized":"2023-09-23T21:02:41.218777+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:36.420362+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:36.420362+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:28:28.613492+0000","scrub_duration":0.028838444000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":218,"reported_epoch":273,"state":"active+clean","last_fresh":"2023-09-23T21:02:41.219145+0000","last_change":"2023-09-23T20:56:45.991822+0000","last_active":"2023-09-23T21:02:41.219145+0000","last_peered":"2023-09-23T21:02:41.219145+0000","last_clean":"2023-09-23T21:02:41.219145+0000","last_became_active":"2023-09-23T20:56:45.991460+0000","last_became_peered":"2023-09-23T20:56:45.991460+0000","last_unstale":"2023-09-23T21:02:41.219145+0000","last_undegraded":"2023-09-23T21:02:41.219145+0000","last_fullsized":"2023-09-23T21:02:41.219145+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":205,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:20:37.503317+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":232,"reported_epoch":273,"state":"active+clean","last_fresh":"2023-09-23T21:02:41.226663+0000","last_change":"2023-09-23T20:51:10.840849+0000","last_active":"2023-09-23T21:02:41.226663+0000","last_peered":"2023-09-23T21:02:41.226663+0000","last_clean":"2023-09-23T21:02:41.226663+0000","last_became_active":"2023-09-23T20:51:10.840593+0000","last_became_peered":"2023-09-23T20:51:10.840593+0000","last_unstale":"2023-09-23T21:02:41.226663+0000","last_undegraded":"2023-09-23T21:02:41.226663+0000","last_fullsized":"2023-09-23T21:02:41.226663+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:30:52.297625+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.12","version":"79'161","reported_seq":511,"reported_epoch":273,"state":"active+clean","last_fresh":"2023-09-23T21:02:41.225774+0000","last_change":"2023-09-23T20:57:21.750765+0000","last_active":"2023-09-23T21:02:41.225774+0000","last_peered":"2023-09-23T21:02:41.225774+0000","last_clean":"2023-09-23T21:02:41.225774+0000","last_became_active":"2023-09-23T20:57:15.979195+0000","last_became_peered":"2023-09-23T20:57:15.979195+0000","last_unstale":"2023-09-23T21:02:41.225774+0000","last_undegraded":"2023-09-23T21:02:41.225774+0000","last_fullsized":"2023-09-23T21:02:41.225774+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:21.750680+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:21.750680+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:03:41.915962+0000","scrub_duration":0.0092391680000000007,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":266,"reported_epoch":273,"state":"active+clean","last_fresh":"2023-09-23T21:02:41.217456+0000","last_change":"2023-09-23T20:55:36.943969+0000","last_active":"2023-09-23T21:02:41.217456+0000","last_peered":"2023-09-23T21:02:41.217456+0000","last_clean":"2023-09-23T21:02:41.217456+0000","last_became_active":"2023-09-23T20:55:36.943580+0000","last_became_peered":"2023-09-23T20:55:36.943580+0000","last_unstale":"2023-09-23T21:02:41.217456+0000","last_undegraded":"2023-09-23T21:02:41.217456+0000","last_fullsized":"2023-09-23T21:02:41.217456+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:49:06.969396+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":64,"reported_epoch":273,"state":"active+clean","last_fresh":"2023-09-23T21:02:41.257798+0000","last_change":"2023-09-23T20:57:44.980498+0000","last_active":"2023-09-23T21:02:41.257798+0000","last_peered":"2023-09-23T21:02:41.257798+0000","last_clean":"2023-09-23T21:02:41.257798+0000","last_became_active":"2023-09-23T20:57:44.980182+0000","last_became_peered":"2023-09-23T20:57:44.980182+0000","last_unstale":"2023-09-23T21:02:41.257798+0000","last_undegraded":"2023-09-23T21:02:41.257798+0000","last_fullsized":"2023-09-23T21:02:41.257798+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:34:51.620544+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.4","version":"274'6213","reported_seq":6241,"reported_epoch":274,"state":"active+clean","last_fresh":"2023-09-23T21:02:47.080377+0000","last_change":"2023-09-23T21:01:25.429865+0000","last_active":"2023-09-23T21:02:47.080377+0000","last_peered":"2023-09-23T21:02:47.080377+0000","last_clean":"2023-09-23T21:02:47.080377+0000","last_became_active":"2023-09-23T21:01:25.429555+0000","last_became_peered":"2023-09-23T21:01:25.429555+0000","last_unstale":"2023-09-23T21:02:47.080377+0000","last_undegraded":"2023-09-23T21:02:47.080377+0000","last_fullsized":"2023-09-23T21:02:47.080377+0000","mapping_epoch":254,"log_start":"0'0","ondisk_log_start":"0'0","created":254,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:01:24.325318+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:01:24.325318+0000","last_clean_scrub_stamp":"2023-09-23T21:01:24.325318+0000","objects_scrubbed":0,"log_size":6213,"log_dups_size":0,"ondisk_log_size":6213,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:28:38.557662+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":407175168,"num_objects":6213,"num_object_clones":0,"num_object_copies":18639,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6213,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6213,"num_write_kb":397632,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,2],"acting":[5,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.d","version":"79'161","reported_seq":556,"reported_epoch":273,"state":"active+clean","last_fresh":"2023-09-23T21:02:41.219457+0000","last_change":"2023-09-23T20:57:31.075213+0000","last_active":"2023-09-23T21:02:41.219457+0000","last_peered":"2023-09-23T21:02:41.219457+0000","last_clean":"2023-09-23T21:02:41.219457+0000","last_became_active":"2023-09-23T20:57:15.988188+0000","last_became_peered":"2023-09-23T20:57:15.988188+0000","last_unstale":"2023-09-23T21:02:41.219457+0000","last_undegraded":"2023-09-23T21:02:41.219457+0000","last_fullsized":"2023-09-23T21:02:41.219457+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:31.075098+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:31.075098+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:41:52.142362+0000","scrub_duration":0.010304610000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":205,"reported_epoch":273,"state":"active+clean","last_fresh":"2023-09-23T21:02:41.219702+0000","last_change":"2023-09-23T20:56:45.991841+0000","last_active":"2023-09-23T21:02:41.219702+0000","last_peered":"2023-09-23T21:02:41.219702+0000","last_clean":"2023-09-23T21:02:41.219702+0000","last_became_active":"2023-09-23T20:56:45.991491+0000","last_became_peered":"2023-09-23T20:56:45.991491+0000","last_unstale":"2023-09-23T21:02:41.219702+0000","last_undegraded":"2023-09-23T21:02:41.219702+0000","last_fullsized":"2023-09-23T21:02:41.219702+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":205,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:30:37.405319+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":164,"reported_epoch":273,"state":"active+clean","last_fresh":"2023-09-23T21:02:41.225319+0000","last_change":"2023-09-23T20:55:35.957768+0000","last_active":"2023-09-23T21:02:41.225319+0000","last_peered":"2023-09-23T21:02:41.225319+0000","last_clean":"2023-09-23T21:02:41.225319+0000","last_became_active":"2023-09-23T20:55:35.957421+0000","last_became_peered":"2023-09-23T20:55:35.957421+0000","last_unstale":"2023-09-23T21:02:41.225319+0000","last_undegraded":"2023-09-23T21:02:41.225319+0000","last_fullsized":"2023-09-23T21:02:41.225319+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:49:41.585961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"8.5","version":"274'6373","reported_seq":6401,"reported_epoch":274,"state":"active+clean","last_fresh":"2023-09-23T21:02:49.100117+0000","last_change":"2023-09-23T21:01:25.560708+0000","last_active":"2023-09-23T21:02:49.100117+0000","last_peered":"2023-09-23T21:02:49.100117+0000","last_clean":"2023-09-23T21:02:49.100117+0000","last_became_active":"2023-09-23T21:01:25.560472+0000","last_became_peered":"2023-09-23T21:01:25.560472+0000","last_unstale":"2023-09-23T21:02:49.100117+0000","last_undegraded":"2023-09-23T21:02:49.100117+0000","last_fullsized":"2023-09-23T21:02:49.100117+0000","mapping_epoch":254,"log_start":"0'0","ondisk_log_start":"0'0","created":254,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:01:24.325318+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:01:24.325318+0000","last_clean_scrub_stamp":"2023-09-23T21:01:24.325318+0000","objects_scrubbed":0,"log_size":6373,"log_dups_size":0,"ondisk_log_size":6373,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:04:02.549068+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":417660928,"num_objects":6373,"num_object_clones":0,"num_object_copies":19119,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6373,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6373,"num_write_kb":407872,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.c","version":"79'161","reported_seq":576,"reported_epoch":273,"state":"active+clean","last_fresh":"2023-09-23T21:02:41.252996+0000","last_change":"2023-09-23T20:57:22.041071+0000","last_active":"2023-09-23T21:02:41.252996+0000","last_peered":"2023-09-23T21:02:41.252996+0000","last_clean":"2023-09-23T21:02:41.252996+0000","last_became_active":"2023-09-23T20:57:15.973204+0000","last_became_peered":"2023-09-23T20:57:15.973204+0000","last_unstale":"2023-09-23T21:02:41.252996+0000","last_undegraded":"2023-09-23T21:02:41.252996+0000","last_fullsized":"2023-09-23T21:02:41.252996+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:22.040907+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:22.040907+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:47:02.019978+0000","scrub_duration":0.013021176000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":192,"reported_epoch":273,"state":"active+clean","last_fresh":"2023-09-23T21:02:41.219595+0000","last_change":"2023-09-23T20:56:45.991821+0000","last_active":"2023-09-23T21:02:41.219595+0000","last_peered":"2023-09-23T21:02:41.219595+0000","last_clean":"2023-09-23T21:02:41.219595+0000","last_became_active":"2023-09-23T20:56:45.991447+0000","last_became_peered":"2023-09-23T20:56:45.991447+0000","last_unstale":"2023-09-23T21:02:41.219595+0000","last_undegraded":"2023-09-23T21:02:41.219595+0000","last_fullsized":"2023-09-23T21:02:41.219595+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":205,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:14:25.525222+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":195,"reported_epoch":273,"state":"active+clean","last_fresh":"2023-09-23T21:02:41.219314+0000","last_change":"2023-09-23T20:57:44.982059+0000","last_active":"2023-09-23T21:02:41.219314+0000","last_peered":"2023-09-23T21:02:41.219314+0000","last_clean":"2023-09-23T21:02:41.219314+0000","last_became_active":"2023-09-23T20:57:44.981742+0000","last_became_peered":"2023-09-23T20:57:44.981742+0000","last_unstale":"2023-09-23T21:02:41.219314+0000","last_undegraded":"2023-09-23T21:02:41.219314+0000","last_fullsized":"2023-09-23T21:02:41.219314+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:41:24.042916+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.6","version":"274'6362","reported_seq":6391,"reported_epoch":274,"state":"active+clean","last_fresh":"2023-09-23T21:02:47.034424+0000","last_change":"2023-09-23T21:01:25.560370+0000","last_active":"2023-09-23T21:02:47.034424+0000","last_peered":"2023-09-23T21:02:47.034424+0000","last_clean":"2023-09-23T21:02:47.034424+0000","last_became_active":"2023-09-23T21:01:25.560018+0000","last_became_peered":"2023-09-23T21:01:25.560018+0000","last_unstale":"2023-09-23T21:02:47.034424+0000","last_undegraded":"2023-09-23T21:02:47.034424+0000","last_fullsized":"2023-09-23T21:02:47.034424+0000","mapping_epoch":254,"log_start":"0'0","ondisk_log_start":"0'0","created":254,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:01:24.325318+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:01:24.325318+0000","last_clean_scrub_stamp":"2023-09-23T21:01:24.325318+0000","objects_scrubbed":0,"log_size":6362,"log_dups_size":0,"ondisk_log_size":6362,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:45:39.782872+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":416940032,"num_objects":6362,"num_object_clones":0,"num_object_copies":19086,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6362,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6362,"num_write_kb":407168,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,7],"acting":[0,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.f","version":"79'161","reported_seq":578,"reported_epoch":273,"state":"active+clean","last_fresh":"2023-09-23T21:02:41.218470+0000","last_change":"2023-09-23T20:57:27.880841+0000","last_active":"2023-09-23T21:02:41.218470+0000","last_peered":"2023-09-23T21:02:41.218470+0000","last_clean":"2023-09-23T21:02:41.218470+0000","last_became_active":"2023-09-23T20:57:15.970925+0000","last_became_peered":"2023-09-23T20:57:15.970925+0000","last_unstale":"2023-09-23T21:02:41.218470+0000","last_undegraded":"2023-09-23T21:02:41.218470+0000","last_fullsized":"2023-09-23T21:02:41.218470+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:27.880776+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:27.880776+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:15:13.438014+0000","scrub_duration":0.0014004899999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":178,"reported_epoch":273,"state":"active+clean","last_fresh":"2023-09-23T21:02:41.225650+0000","last_change":"2023-09-23T20:56:45.989153+0000","last_active":"2023-09-23T21:02:41.225650+0000","last_peered":"2023-09-23T21:02:41.225650+0000","last_clean":"2023-09-23T21:02:41.225650+0000","last_became_active":"2023-09-23T20:56:45.988800+0000","last_became_peered":"2023-09-23T20:56:45.988800+0000","last_unstale":"2023-09-23T21:02:41.225650+0000","last_undegraded":"2023-09-23T21:02:41.225650+0000","last_fullsized":"2023-09-23T21:02:41.225650+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":205,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:04:41.130194+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":213,"reported_epoch":273,"state":"active+clean","last_fresh":"2023-09-23T21:02:41.217555+0000","last_change":"2023-09-23T20:55:35.965427+0000","last_active":"2023-09-23T21:02:41.217555+0000","last_peered":"2023-09-23T21:02:41.217555+0000","last_clean":"2023-09-23T21:02:41.217555+0000","last_became_active":"2023-09-23T20:55:35.963969+0000","last_became_peered":"2023-09-23T20:55:35.963969+0000","last_unstale":"2023-09-23T21:02:41.217555+0000","last_undegraded":"2023-09-23T21:02:41.217555+0000","last_fullsized":"2023-09-23T21:02:41.217555+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:38:40.612083+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.7","version":"274'6283","reported_seq":6312,"reported_epoch":274,"state":"active+clean","last_fresh":"2023-09-23T21:02:49.441410+0000","last_change":"2023-09-23T21:01:25.341425+0000","last_active":"2023-09-23T21:02:49.441410+0000","last_peered":"2023-09-23T21:02:49.441410+0000","last_clean":"2023-09-23T21:02:49.441410+0000","last_became_active":"2023-09-23T21:01:25.341114+0000","last_became_peered":"2023-09-23T21:01:25.341114+0000","last_unstale":"2023-09-23T21:02:49.441410+0000","last_undegraded":"2023-09-23T21:02:49.441410+0000","last_fullsized":"2023-09-23T21:02:49.441410+0000","mapping_epoch":254,"log_start":"0'0","ondisk_log_start":"0'0","created":254,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:01:24.325318+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:01:24.325318+0000","last_clean_scrub_stamp":"2023-09-23T21:01:24.325318+0000","objects_scrubbed":0,"log_size":6283,"log_dups_size":0,"ondisk_log_size":6283,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:15:08.338532+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":411762688,"num_objects":6283,"num_object_clones":0,"num_object_copies":18849,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6283,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6283,"num_write_kb":402112,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.e","version":"79'161","reported_seq":485,"reported_epoch":273,"state":"active+clean","last_fresh":"2023-09-23T21:02:41.219536+0000","last_change":"2023-09-23T20:57:34.132738+0000","last_active":"2023-09-23T21:02:41.219536+0000","last_peered":"2023-09-23T21:02:41.219536+0000","last_clean":"2023-09-23T21:02:41.219536+0000","last_became_active":"2023-09-23T20:57:15.974318+0000","last_became_peered":"2023-09-23T20:57:15.974318+0000","last_unstale":"2023-09-23T21:02:41.219536+0000","last_undegraded":"2023-09-23T21:02:41.219536+0000","last_fullsized":"2023-09-23T21:02:41.219536+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:34.132660+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:34.132660+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:07:07.739438+0000","scrub_duration":0.005229694,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":164,"reported_epoch":273,"state":"active+clean","last_fresh":"2023-09-23T21:02:41.225544+0000","last_change":"2023-09-23T20:55:35.962005+0000","last_active":"2023-09-23T21:02:41.225544+0000","last_peered":"2023-09-23T21:02:41.225544+0000","last_clean":"2023-09-23T21:02:41.225544+0000","last_became_active":"2023-09-23T20:55:35.961695+0000","last_became_peered":"2023-09-23T20:55:35.961695+0000","last_unstale":"2023-09-23T21:02:41.225544+0000","last_undegraded":"2023-09-23T21:02:41.225544+0000","last_fullsized":"2023-09-23T21:02:41.225544+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:28:22.876201+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":263,"reported_epoch":273,"state":"active+clean","last_fresh":"2023-09-23T21:02:41.226740+0000","last_change":"2023-09-23T20:54:13.928715+0000","last_active":"2023-09-23T21:02:41.226740+0000","last_peered":"2023-09-23T21:02:41.226740+0000","last_clean":"2023-09-23T21:02:41.226740+0000","last_became_active":"2023-09-23T20:54:13.928200+0000","last_became_peered":"2023-09-23T20:54:13.928200+0000","last_unstale":"2023-09-23T21:02:41.226740+0000","last_undegraded":"2023-09-23T21:02:41.226740+0000","last_fullsized":"2023-09-23T21:02:41.226740+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:41:16.927513+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.0","version":"274'6332","reported_seq":6361,"reported_epoch":274,"state":"active+clean","last_fresh":"2023-09-23T21:02:49.810728+0000","last_change":"2023-09-23T21:01:25.342305+0000","last_active":"2023-09-23T21:02:49.810728+0000","last_peered":"2023-09-23T21:02:49.810728+0000","last_clean":"2023-09-23T21:02:49.810728+0000","last_became_active":"2023-09-23T21:01:25.342042+0000","last_became_peered":"2023-09-23T21:01:25.342042+0000","last_unstale":"2023-09-23T21:02:49.810728+0000","last_undegraded":"2023-09-23T21:02:49.810728+0000","last_fullsized":"2023-09-23T21:02:49.810728+0000","mapping_epoch":254,"log_start":"0'0","ondisk_log_start":"0'0","created":254,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:01:24.325318+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:01:24.325318+0000","last_clean_scrub_stamp":"2023-09-23T21:01:24.325318+0000","objects_scrubbed":0,"log_size":6332,"log_dups_size":0,"ondisk_log_size":6332,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:06:29.755463+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":414973952,"num_objects":6332,"num_object_clones":0,"num_object_copies":18996,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6332,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6332,"num_write_kb":405248,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,1],"acting":[3,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.9","version":"79'161","reported_seq":648,"reported_epoch":273,"state":"active+clean","last_fresh":"2023-09-23T21:02:41.247561+0000","last_change":"2023-09-23T20:56:46.921939+0000","last_active":"2023-09-23T21:02:41.247561+0000","last_peered":"2023-09-23T21:02:41.247561+0000","last_clean":"2023-09-23T21:02:41.247561+0000","last_became_active":"2023-09-23T20:56:45.989908+0000","last_became_peered":"2023-09-23T20:56:45.989908+0000","last_unstale":"2023-09-23T21:02:41.247561+0000","last_undegraded":"2023-09-23T21:02:41.247561+0000","last_fullsized":"2023-09-23T21:02:41.247561+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":205,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:56:46.921739+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:56:46.921739+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:20:42.635110+0000","scrub_duration":0.0071954109999999996,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":265,"reported_epoch":273,"state":"active+clean","last_fresh":"2023-09-23T21:02:41.247570+0000","last_change":"2023-09-23T20:54:13.961788+0000","last_active":"2023-09-23T21:02:41.247570+0000","last_peered":"2023-09-23T21:02:41.247570+0000","last_clean":"2023-09-23T21:02:41.247570+0000","last_became_active":"2023-09-23T20:54:13.940937+0000","last_became_peered":"2023-09-23T20:54:13.940937+0000","last_unstale":"2023-09-23T21:02:41.247570+0000","last_undegraded":"2023-09-23T21:02:41.247570+0000","last_fullsized":"2023-09-23T21:02:41.247570+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:11:56.785299+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":168,"reported_epoch":273,"state":"active+clean","last_fresh":"2023-09-23T21:02:41.219622+0000","last_change":"2023-09-23T20:55:35.949670+0000","last_active":"2023-09-23T21:02:41.219622+0000","last_peered":"2023-09-23T21:02:41.219622+0000","last_clean":"2023-09-23T21:02:41.219622+0000","last_became_active":"2023-09-23T20:55:35.949140+0000","last_became_peered":"2023-09-23T20:55:35.949140+0000","last_unstale":"2023-09-23T21:02:41.219622+0000","last_undegraded":"2023-09-23T21:02:41.219622+0000","last_fullsized":"2023-09-23T21:02:41.219622+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:55:59.340303+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.1","version":"274'6475","reported_seq":6504,"reported_epoch":274,"state":"active+clean","last_fresh":"2023-09-23T21:02:49.396426+0000","last_change":"2023-09-23T21:01:25.429922+0000","last_active":"2023-09-23T21:02:49.396426+0000","last_peered":"2023-09-23T21:02:49.396426+0000","last_clean":"2023-09-23T21:02:49.396426+0000","last_became_active":"2023-09-23T21:01:25.429650+0000","last_became_peered":"2023-09-23T21:01:25.429650+0000","last_unstale":"2023-09-23T21:02:49.396426+0000","last_undegraded":"2023-09-23T21:02:49.396426+0000","last_fullsized":"2023-09-23T21:02:49.396426+0000","mapping_epoch":254,"log_start":"0'0","ondisk_log_start":"0'0","created":254,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:01:24.325318+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:01:24.325318+0000","last_clean_scrub_stamp":"2023-09-23T21:01:24.325318+0000","objects_scrubbed":0,"log_size":6475,"log_dups_size":0,"ondisk_log_size":6475,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:18:57.243292+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":424345600,"num_objects":6475,"num_object_clones":0,"num_object_copies":19425,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6475,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6475,"num_write_kb":414400,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,2],"acting":[1,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.8","version":"79'161","reported_seq":517,"reported_epoch":273,"state":"active+clean","last_fresh":"2023-09-23T21:02:41.225291+0000","last_change":"2023-09-23T20:56:57.505737+0000","last_active":"2023-09-23T21:02:41.225291+0000","last_peered":"2023-09-23T21:02:41.225291+0000","last_clean":"2023-09-23T21:02:41.225291+0000","last_became_active":"2023-09-23T20:56:04.982345+0000","last_became_peered":"2023-09-23T20:56:04.982345+0000","last_unstale":"2023-09-23T21:02:41.225291+0000","last_undegraded":"2023-09-23T21:02:41.225291+0000","last_fullsized":"2023-09-23T21:02:41.225291+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:56:57.505663+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:56:57.505663+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:34:30.109724+0000","scrub_duration":0.026284108,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":236,"reported_epoch":273,"state":"active+clean","last_fresh":"2023-09-23T21:02:41.217715+0000","last_change":"2023-09-23T20:55:35.959942+0000","last_active":"2023-09-23T21:02:41.217715+0000","last_peered":"2023-09-23T21:02:41.217715+0000","last_clean":"2023-09-23T21:02:41.217715+0000","last_became_active":"2023-09-23T20:55:35.957071+0000","last_became_peered":"2023-09-23T20:55:35.957071+0000","last_unstale":"2023-09-23T21:02:41.217715+0000","last_undegraded":"2023-09-23T21:02:41.217715+0000","last_fullsized":"2023-09-23T21:02:41.217715+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:44:30.308036+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":151,"reported_epoch":273,"state":"active+clean","last_fresh":"2023-09-23T21:02:41.219833+0000","last_change":"2023-09-23T20:54:13.951459+0000","last_active":"2023-09-23T21:02:41.219833+0000","last_peered":"2023-09-23T21:02:41.219833+0000","last_clean":"2023-09-23T21:02:41.219833+0000","last_became_active":"2023-09-23T20:54:13.948409+0000","last_became_peered":"2023-09-23T20:54:13.948409+0000","last_unstale":"2023-09-23T21:02:41.219833+0000","last_undegraded":"2023-09-23T21:02:41.219833+0000","last_fullsized":"2023-09-23T21:02:41.219833+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:38:10.148061+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.2","version":"274'6320","reported_seq":6348,"reported_epoch":274,"state":"active+clean","last_fresh":"2023-09-23T21:02:47.257337+0000","last_change":"2023-09-23T21:01:25.343022+0000","last_active":"2023-09-23T21:02:47.257337+0000","last_peered":"2023-09-23T21:02:47.257337+0000","last_clean":"2023-09-23T21:02:47.257337+0000","last_became_active":"2023-09-23T21:01:25.342540+0000","last_became_peered":"2023-09-23T21:01:25.342540+0000","last_unstale":"2023-09-23T21:02:47.257337+0000","last_undegraded":"2023-09-23T21:02:47.257337+0000","last_fullsized":"2023-09-23T21:02:47.257337+0000","mapping_epoch":254,"log_start":"0'0","ondisk_log_start":"0'0","created":254,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:01:24.325318+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:01:24.325318+0000","last_clean_scrub_stamp":"2023-09-23T21:01:24.325318+0000","objects_scrubbed":0,"log_size":6320,"log_dups_size":0,"ondisk_log_size":6320,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:57:19.741924+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":414187520,"num_objects":6320,"num_object_clones":0,"num_object_copies":18960,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6320,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6320,"num_write_kb":404480,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,5],"acting":[4,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.b","version":"79'161","reported_seq":558,"reported_epoch":273,"state":"active+clean","last_fresh":"2023-09-23T21:02:41.218869+0000","last_change":"2023-09-23T20:57:27.320172+0000","last_active":"2023-09-23T21:02:41.218869+0000","last_peered":"2023-09-23T21:02:41.218869+0000","last_clean":"2023-09-23T21:02:41.218869+0000","last_became_active":"2023-09-23T20:57:16.009630+0000","last_became_peered":"2023-09-23T20:57:16.009630+0000","last_unstale":"2023-09-23T21:02:41.218869+0000","last_undegraded":"2023-09-23T21:02:41.218869+0000","last_fullsized":"2023-09-23T21:02:41.218869+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:27.320022+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:27.320022+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:31:35.221558+0000","scrub_duration":0.0043894679999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,5],"acting":[1,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":151,"reported_epoch":273,"state":"active+clean","last_fresh":"2023-09-23T21:02:41.219180+0000","last_change":"2023-09-23T20:54:13.950254+0000","last_active":"2023-09-23T21:02:41.219180+0000","last_peered":"2023-09-23T21:02:41.219180+0000","last_clean":"2023-09-23T21:02:41.219180+0000","last_became_active":"2023-09-23T20:54:13.949934+0000","last_became_peered":"2023-09-23T20:54:13.949934+0000","last_unstale":"2023-09-23T21:02:41.219180+0000","last_undegraded":"2023-09-23T21:02:41.219180+0000","last_fullsized":"2023-09-23T21:02:41.219180+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:30:32.847167+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":168,"reported_epoch":273,"state":"active+clean","last_fresh":"2023-09-23T21:02:41.219629+0000","last_change":"2023-09-23T20:55:35.956153+0000","last_active":"2023-09-23T21:02:41.219629+0000","last_peered":"2023-09-23T21:02:41.219629+0000","last_clean":"2023-09-23T21:02:41.219629+0000","last_became_active":"2023-09-23T20:55:35.955630+0000","last_became_peered":"2023-09-23T20:55:35.955630+0000","last_unstale":"2023-09-23T21:02:41.219629+0000","last_undegraded":"2023-09-23T21:02:41.219629+0000","last_fullsized":"2023-09-23T21:02:41.219629+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:45:20.282929+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.3","version":"274'6104","reported_seq":6133,"reported_epoch":274,"state":"active+clean","last_fresh":"2023-09-23T21:02:49.442419+0000","last_change":"2023-09-23T21:01:25.342380+0000","last_active":"2023-09-23T21:02:49.442419+0000","last_peered":"2023-09-23T21:02:49.442419+0000","last_clean":"2023-09-23T21:02:49.442419+0000","last_became_active":"2023-09-23T21:01:25.342099+0000","last_became_peered":"2023-09-23T21:01:25.342099+0000","last_unstale":"2023-09-23T21:02:49.442419+0000","last_undegraded":"2023-09-23T21:02:49.442419+0000","last_fullsized":"2023-09-23T21:02:49.442419+0000","mapping_epoch":254,"log_start":"0'0","ondisk_log_start":"0'0","created":254,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:01:24.325318+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:01:24.325318+0000","last_clean_scrub_stamp":"2023-09-23T21:01:24.325318+0000","objects_scrubbed":0,"log_size":6104,"log_dups_size":0,"ondisk_log_size":6104,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:09:21.965969+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":400031744,"num_objects":6104,"num_object_clones":0,"num_object_copies":18312,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6104,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6104,"num_write_kb":390656,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.a","version":"79'161","reported_seq":517,"reported_epoch":273,"state":"active+clean","last_fresh":"2023-09-23T21:02:41.226008+0000","last_change":"2023-09-23T20:56:44.487778+0000","last_active":"2023-09-23T21:02:41.226008+0000","last_peered":"2023-09-23T21:02:41.226008+0000","last_clean":"2023-09-23T21:02:41.226008+0000","last_became_active":"2023-09-23T20:56:05.006740+0000","last_became_peered":"2023-09-23T20:56:05.006740+0000","last_unstale":"2023-09-23T21:02:41.226008+0000","last_undegraded":"2023-09-23T21:02:41.226008+0000","last_fullsized":"2023-09-23T21:02:41.226008+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:56:44.487678+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:56:44.487678+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:04:50.086749+0000","scrub_duration":0.0017408899999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":64,"reported_epoch":273,"state":"active+clean","last_fresh":"2023-09-23T21:02:41.248778+0000","last_change":"2023-09-23T20:57:44.982513+0000","last_active":"2023-09-23T21:02:41.248778+0000","last_peered":"2023-09-23T21:02:41.248778+0000","last_clean":"2023-09-23T21:02:41.248778+0000","last_became_active":"2023-09-23T20:57:44.982112+0000","last_became_peered":"2023-09-23T20:57:44.982112+0000","last_unstale":"2023-09-23T21:02:41.248778+0000","last_undegraded":"2023-09-23T21:02:41.248778+0000","last_fullsized":"2023-09-23T21:02:41.248778+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:09:05.630041+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":245,"reported_epoch":273,"state":"active+clean","last_fresh":"2023-09-23T21:02:41.223514+0000","last_change":"2023-09-23T20:54:13.940748+0000","last_active":"2023-09-23T21:02:41.223514+0000","last_peered":"2023-09-23T21:02:41.223514+0000","last_clean":"2023-09-23T21:02:41.223514+0000","last_became_active":"2023-09-23T20:54:13.940419+0000","last_became_peered":"2023-09-23T20:54:13.940419+0000","last_unstale":"2023-09-23T21:02:41.223514+0000","last_undegraded":"2023-09-23T21:02:41.223514+0000","last_fullsized":"2023-09-23T21:02:41.223514+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:15:01.923816+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.e","version":"274'6255","reported_seq":6284,"reported_epoch":274,"state":"active+clean","last_fresh":"2023-09-23T21:02:47.075305+0000","last_change":"2023-09-23T21:01:25.430111+0000","last_active":"2023-09-23T21:02:47.075305+0000","last_peered":"2023-09-23T21:02:47.075305+0000","last_clean":"2023-09-23T21:02:47.075305+0000","last_became_active":"2023-09-23T21:01:25.429811+0000","last_became_peered":"2023-09-23T21:01:25.429811+0000","last_unstale":"2023-09-23T21:02:47.075305+0000","last_undegraded":"2023-09-23T21:02:47.075305+0000","last_fullsized":"2023-09-23T21:02:47.075305+0000","mapping_epoch":254,"log_start":"0'0","ondisk_log_start":"0'0","created":254,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:01:24.325318+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:01:24.325318+0000","last_clean_scrub_stamp":"2023-09-23T21:01:24.325318+0000","objects_scrubbed":0,"log_size":6255,"log_dups_size":0,"ondisk_log_size":6255,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:18:01.918758+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":409927680,"num_objects":6255,"num_object_clones":0,"num_object_copies":18765,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6255,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6255,"num_write_kb":400320,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.7","version":"79'161","reported_seq":579,"reported_epoch":273,"state":"active+clean","last_fresh":"2023-09-23T21:02:41.218614+0000","last_change":"2023-09-23T20:57:48.243572+0000","last_active":"2023-09-23T21:02:41.218614+0000","last_peered":"2023-09-23T21:02:41.218614+0000","last_clean":"2023-09-23T21:02:41.218614+0000","last_became_active":"2023-09-23T20:57:48.243229+0000","last_became_peered":"2023-09-23T20:57:48.243229+0000","last_unstale":"2023-09-23T21:02:41.218614+0000","last_undegraded":"2023-09-23T21:02:41.218614+0000","last_fullsized":"2023-09-23T21:02:41.218614+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":3,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:25.916867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:25.916867+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:23:19.385831+0000","scrub_duration":0.001665323,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":251,"reported_epoch":273,"state":"active+clean","last_fresh":"2023-09-23T21:02:41.219195+0000","last_change":"2023-09-23T20:50:05.867130+0000","last_active":"2023-09-23T21:02:41.219195+0000","last_peered":"2023-09-23T21:02:41.219195+0000","last_clean":"2023-09-23T21:02:41.219195+0000","last_became_active":"2023-09-23T20:50:05.866758+0000","last_became_peered":"2023-09-23T20:50:05.866758+0000","last_unstale":"2023-09-23T21:02:41.219195+0000","last_undegraded":"2023-09-23T21:02:41.219195+0000","last_fullsized":"2023-09-23T21:02:41.219195+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:55:07.548209+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":147,"reported_epoch":273,"state":"active+clean","last_fresh":"2023-09-23T21:02:41.225810+0000","last_change":"2023-09-23T20:54:31.887604+0000","last_active":"2023-09-23T21:02:41.225810+0000","last_peered":"2023-09-23T21:02:41.225810+0000","last_clean":"2023-09-23T21:02:41.225810+0000","last_became_active":"2023-09-23T20:54:31.886752+0000","last_became_peered":"2023-09-23T20:54:31.886752+0000","last_unstale":"2023-09-23T21:02:41.225810+0000","last_undegraded":"2023-09-23T21:02:41.225810+0000","last_fullsized":"2023-09-23T21:02:41.225810+0000","mapping_epoch":136,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:13:58.872176+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":249,"reported_epoch":273,"state":"active+clean","last_fresh":"2023-09-23T21:02:41.218604+0000","last_change":"2023-09-23T20:54:31.899082+0000","last_active":"2023-09-23T21:02:41.218604+0000","last_peered":"2023-09-23T21:02:41.218604+0000","last_clean":"2023-09-23T21:02:41.218604+0000","last_became_active":"2023-09-23T20:54:31.898521+0000","last_became_peered":"2023-09-23T20:54:31.898521+0000","last_unstale":"2023-09-23T21:02:41.218604+0000","last_undegraded":"2023-09-23T21:02:41.218604+0000","last_fullsized":"2023-09-23T21:02:41.218604+0000","mapping_epoch":136,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:25:36.059711+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.8","version":"274'6184","reported_seq":6212,"reported_epoch":274,"state":"active+clean","last_fresh":"2023-09-23T21:02:47.039109+0000","last_change":"2023-09-23T21:01:25.560581+0000","last_active":"2023-09-23T21:02:47.039109+0000","last_peered":"2023-09-23T21:02:47.039109+0000","last_clean":"2023-09-23T21:02:47.039109+0000","last_became_active":"2023-09-23T21:01:25.560318+0000","last_became_peered":"2023-09-23T21:01:25.560318+0000","last_unstale":"2023-09-23T21:02:47.039109+0000","last_undegraded":"2023-09-23T21:02:47.039109+0000","last_fullsized":"2023-09-23T21:02:47.039109+0000","mapping_epoch":254,"log_start":"0'0","ondisk_log_start":"0'0","created":254,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:01:24.325318+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:01:24.325318+0000","last_clean_scrub_stamp":"2023-09-23T21:01:24.325318+0000","objects_scrubbed":0,"log_size":6184,"log_dups_size":0,"ondisk_log_size":6184,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:07:28.203012+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":405274624,"num_objects":6184,"num_object_clones":0,"num_object_copies":18552,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6184,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6184,"num_write_kb":395776,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[5,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.1","version":"79'161","reported_seq":537,"reported_epoch":273,"state":"active+clean","last_fresh":"2023-09-23T21:02:41.225347+0000","last_change":"2023-09-23T20:57:19.669346+0000","last_active":"2023-09-23T21:02:41.225347+0000","last_peered":"2023-09-23T21:02:41.225347+0000","last_clean":"2023-09-23T21:02:41.225347+0000","last_became_active":"2023-09-23T20:57:08.246046+0000","last_became_peered":"2023-09-23T20:57:08.246046+0000","last_unstale":"2023-09-23T21:02:41.225347+0000","last_undegraded":"2023-09-23T21:02:41.225347+0000","last_fullsized":"2023-09-23T21:02:41.225347+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:19.669259+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:19.669259+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:14:03.435531+0000","scrub_duration":0.0045759559999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":164,"reported_epoch":273,"state":"active+clean","last_fresh":"2023-09-23T21:02:41.225819+0000","last_change":"2023-09-23T20:55:35.959514+0000","last_active":"2023-09-23T21:02:41.225819+0000","last_peered":"2023-09-23T21:02:41.225819+0000","last_clean":"2023-09-23T21:02:41.225819+0000","last_became_active":"2023-09-23T20:55:35.959244+0000","last_became_peered":"2023-09-23T20:55:35.959244+0000","last_unstale":"2023-09-23T21:02:41.225819+0000","last_undegraded":"2023-09-23T21:02:41.225819+0000","last_fullsized":"2023-09-23T21:02:41.225819+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:47:32.665316+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":168,"reported_epoch":273,"state":"active+clean","last_fresh":"2023-09-23T21:02:41.219875+0000","last_change":"2023-09-23T20:55:35.949000+0000","last_active":"2023-09-23T21:02:41.219875+0000","last_peered":"2023-09-23T21:02:41.219875+0000","last_clean":"2023-09-23T21:02:41.219875+0000","last_became_active":"2023-09-23T20:55:35.948410+0000","last_became_peered":"2023-09-23T20:55:35.948410+0000","last_unstale":"2023-09-23T21:02:41.219875+0000","last_undegraded":"2023-09-23T21:02:41.219875+0000","last_fullsized":"2023-09-23T21:02:41.219875+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:26:50.477289+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":243,"reported_epoch":273,"state":"active+clean","last_fresh":"2023-09-23T21:02:41.218814+0000","last_change":"2023-09-23T20:51:10.829889+0000","last_active":"2023-09-23T21:02:41.218814+0000","last_peered":"2023-09-23T21:02:41.218814+0000","last_clean":"2023-09-23T21:02:41.218814+0000","last_became_active":"2023-09-23T20:51:10.829533+0000","last_became_peered":"2023-09-23T20:51:10.829533+0000","last_unstale":"2023-09-23T21:02:41.218814+0000","last_undegraded":"2023-09-23T21:02:41.218814+0000","last_fullsized":"2023-09-23T21:02:41.218814+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:43:04.603460+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.b","version":"274'6353","reported_seq":6382,"reported_epoch":274,"state":"active+clean","last_fresh":"2023-09-23T21:02:49.850506+0000","last_change":"2023-09-23T21:01:25.340569+0000","last_active":"2023-09-23T21:02:49.850506+0000","last_peered":"2023-09-23T21:02:49.850506+0000","last_clean":"2023-09-23T21:02:49.850506+0000","last_became_active":"2023-09-23T21:01:25.340209+0000","last_became_peered":"2023-09-23T21:01:25.340209+0000","last_unstale":"2023-09-23T21:02:49.850506+0000","last_undegraded":"2023-09-23T21:02:49.850506+0000","last_fullsized":"2023-09-23T21:02:49.850506+0000","mapping_epoch":254,"log_start":"0'0","ondisk_log_start":"0'0","created":254,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:01:24.325318+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:01:24.325318+0000","last_clean_scrub_stamp":"2023-09-23T21:01:24.325318+0000","objects_scrubbed":0,"log_size":6353,"log_dups_size":0,"ondisk_log_size":6353,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:53:59.553810+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":416350208,"num_objects":6353,"num_object_clones":0,"num_object_copies":19059,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6353,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6353,"num_write_kb":406592,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.2","version":"79'161","reported_seq":505,"reported_epoch":273,"state":"active+clean","last_fresh":"2023-09-23T21:02:41.226291+0000","last_change":"2023-09-23T20:57:24.802586+0000","last_active":"2023-09-23T21:02:41.226291+0000","last_peered":"2023-09-23T21:02:41.226291+0000","last_clean":"2023-09-23T21:02:41.226291+0000","last_became_active":"2023-09-23T20:57:08.236114+0000","last_became_peered":"2023-09-23T20:57:08.236114+0000","last_unstale":"2023-09-23T21:02:41.226291+0000","last_undegraded":"2023-09-23T21:02:41.226291+0000","last_fullsized":"2023-09-23T21:02:41.226291+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:24.802451+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:24.802451+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:06:22.309980+0000","scrub_duration":0.0090861250000000005,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,4],"acting":[0,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":261,"reported_epoch":273,"state":"active+clean","last_fresh":"2023-09-23T21:02:41.241284+0000","last_change":"2023-09-23T20:51:10.838054+0000","last_active":"2023-09-23T21:02:41.241284+0000","last_peered":"2023-09-23T21:02:41.241284+0000","last_clean":"2023-09-23T21:02:41.241284+0000","last_became_active":"2023-09-23T20:51:10.837750+0000","last_became_peered":"2023-09-23T20:51:10.837750+0000","last_unstale":"2023-09-23T21:02:41.241284+0000","last_undegraded":"2023-09-23T21:02:41.241284+0000","last_fullsized":"2023-09-23T21:02:41.241284+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:15:51.648085+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":243,"reported_epoch":273,"state":"active+clean","last_fresh":"2023-09-23T21:02:41.218294+0000","last_change":"2023-09-23T20:55:35.951146+0000","last_active":"2023-09-23T21:02:41.218294+0000","last_peered":"2023-09-23T21:02:41.218294+0000","last_clean":"2023-09-23T21:02:41.218294+0000","last_became_active":"2023-09-23T20:55:35.950874+0000","last_became_peered":"2023-09-23T20:55:35.950874+0000","last_unstale":"2023-09-23T21:02:41.218294+0000","last_undegraded":"2023-09-23T21:02:41.218294+0000","last_fullsized":"2023-09-23T21:02:41.218294+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:58:13.957677+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":263,"reported_epoch":273,"state":"active+clean","last_fresh":"2023-09-23T21:02:41.223405+0000","last_change":"2023-09-23T20:54:31.898870+0000","last_active":"2023-09-23T21:02:41.223405+0000","last_peered":"2023-09-23T21:02:41.223405+0000","last_clean":"2023-09-23T21:02:41.223405+0000","last_became_active":"2023-09-23T20:54:31.898467+0000","last_became_peered":"2023-09-23T20:54:31.898467+0000","last_unstale":"2023-09-23T21:02:41.223405+0000","last_undegraded":"2023-09-23T21:02:41.223405+0000","last_fullsized":"2023-09-23T21:02:41.223405+0000","mapping_epoch":136,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:56:22.483408+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.9","version":"274'6159","reported_seq":6187,"reported_epoch":274,"state":"active+clean","last_fresh":"2023-09-23T21:02:49.049766+0000","last_change":"2023-09-23T21:01:25.342548+0000","last_active":"2023-09-23T21:02:49.049766+0000","last_peered":"2023-09-23T21:02:49.049766+0000","last_clean":"2023-09-23T21:02:49.049766+0000","last_became_active":"2023-09-23T21:01:25.342227+0000","last_became_peered":"2023-09-23T21:01:25.342227+0000","last_unstale":"2023-09-23T21:02:49.049766+0000","last_undegraded":"2023-09-23T21:02:49.049766+0000","last_fullsized":"2023-09-23T21:02:49.049766+0000","mapping_epoch":254,"log_start":"0'0","ondisk_log_start":"0'0","created":254,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:01:24.325318+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:01:24.325318+0000","last_clean_scrub_stamp":"2023-09-23T21:01:24.325318+0000","objects_scrubbed":0,"log_size":6159,"log_dups_size":0,"ondisk_log_size":6159,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:38:15.944067+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":403636224,"num_objects":6159,"num_object_clones":0,"num_object_copies":18477,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6159,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6159,"num_write_kb":394176,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,0],"acting":[6,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":147,"reported_epoch":273,"state":"active+clean","last_fresh":"2023-09-23T21:02:41.225573+0000","last_change":"2023-09-23T20:54:31.883504+0000","last_active":"2023-09-23T21:02:41.225573+0000","last_peered":"2023-09-23T21:02:41.225573+0000","last_clean":"2023-09-23T21:02:41.225573+0000","last_became_active":"2023-09-23T20:54:31.883172+0000","last_became_peered":"2023-09-23T20:54:31.883172+0000","last_unstale":"2023-09-23T21:02:41.225573+0000","last_undegraded":"2023-09-23T21:02:41.225573+0000","last_fullsized":"2023-09-23T21:02:41.225573+0000","mapping_epoch":136,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:45:40.289120+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"79'161","reported_seq":506,"reported_epoch":274,"state":"active+clean","last_fresh":"2023-09-23T21:02:43.275585+0000","last_change":"2023-09-23T20:57:20.738417+0000","last_active":"2023-09-23T21:02:43.275585+0000","last_peered":"2023-09-23T21:02:43.275585+0000","last_clean":"2023-09-23T21:02:43.275585+0000","last_became_active":"2023-09-23T20:57:08.236842+0000","last_became_peered":"2023-09-23T20:57:08.236842+0000","last_unstale":"2023-09-23T21:02:43.275585+0000","last_undegraded":"2023-09-23T21:02:43.275585+0000","last_fullsized":"2023-09-23T21:02:43.275585+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":0,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:20.738273+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:20.738273+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:18:28.327990+0000","scrub_duration":0.0088336339999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":253,"reported_epoch":273,"state":"active+clean","last_fresh":"2023-09-23T21:02:41.247960+0000","last_change":"2023-09-23T20:55:35.950777+0000","last_active":"2023-09-23T21:02:41.247960+0000","last_peered":"2023-09-23T21:02:41.247960+0000","last_clean":"2023-09-23T21:02:41.247960+0000","last_became_active":"2023-09-23T20:55:35.950427+0000","last_became_peered":"2023-09-23T20:55:35.950427+0000","last_unstale":"2023-09-23T21:02:41.247960+0000","last_undegraded":"2023-09-23T21:02:41.247960+0000","last_fullsized":"2023-09-23T21:02:41.247960+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:20:10.371751+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":218,"reported_epoch":273,"state":"active+clean","last_fresh":"2023-09-23T21:02:41.218368+0000","last_change":"2023-09-23T20:55:35.978140+0000","last_active":"2023-09-23T21:02:41.218368+0000","last_peered":"2023-09-23T21:02:41.218368+0000","last_clean":"2023-09-23T21:02:41.218368+0000","last_became_active":"2023-09-23T20:55:35.977831+0000","last_became_peered":"2023-09-23T20:55:35.977831+0000","last_unstale":"2023-09-23T21:02:41.218368+0000","last_undegraded":"2023-09-23T21:02:41.218368+0000","last_fullsized":"2023-09-23T21:02:41.218368+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:25:47.678023+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.a","version":"274'6241","reported_seq":6270,"reported_epoch":274,"state":"active+clean","last_fresh":"2023-09-23T21:02:49.401926+0000","last_change":"2023-09-23T21:01:25.429899+0000","last_active":"2023-09-23T21:02:49.401926+0000","last_peered":"2023-09-23T21:02:49.401926+0000","last_clean":"2023-09-23T21:02:49.401926+0000","last_became_active":"2023-09-23T21:01:25.429583+0000","last_became_peered":"2023-09-23T21:01:25.429583+0000","last_unstale":"2023-09-23T21:02:49.401926+0000","last_undegraded":"2023-09-23T21:02:49.401926+0000","last_fullsized":"2023-09-23T21:02:49.401926+0000","mapping_epoch":254,"log_start":"0'0","ondisk_log_start":"0'0","created":254,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:01:24.325318+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:01:24.325318+0000","last_clean_scrub_stamp":"2023-09-23T21:01:24.325318+0000","objects_scrubbed":0,"log_size":6241,"log_dups_size":0,"ondisk_log_size":6241,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:31:11.594129+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":409010176,"num_objects":6241,"num_object_clones":0,"num_object_copies":18723,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6241,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6241,"num_write_kb":399424,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.3","version":"79'161","reported_seq":542,"reported_epoch":273,"state":"active+clean","last_fresh":"2023-09-23T21:02:41.218958+0000","last_change":"2023-09-23T20:57:20.378531+0000","last_active":"2023-09-23T21:02:41.218958+0000","last_peered":"2023-09-23T21:02:41.218958+0000","last_clean":"2023-09-23T21:02:41.218958+0000","last_became_active":"2023-09-23T20:57:08.238019+0000","last_became_peered":"2023-09-23T20:57:08.238019+0000","last_unstale":"2023-09-23T21:02:41.218958+0000","last_undegraded":"2023-09-23T21:02:41.218958+0000","last_fullsized":"2023-09-23T21:02:41.218958+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":3,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:20.377714+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:20.377714+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:00:08.305602+0000","scrub_duration":0.003015228,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":145,"reported_epoch":274,"state":"active+clean","last_fresh":"2023-09-23T21:02:43.276580+0000","last_change":"2023-09-23T20:56:45.991884+0000","last_active":"2023-09-23T21:02:43.276580+0000","last_peered":"2023-09-23T21:02:43.276580+0000","last_clean":"2023-09-23T21:02:43.276580+0000","last_became_active":"2023-09-23T20:56:45.991568+0000","last_became_peered":"2023-09-23T20:56:45.991568+0000","last_unstale":"2023-09-23T21:02:43.276580+0000","last_undegraded":"2023-09-23T21:02:43.276580+0000","last_fullsized":"2023-09-23T21:02:43.276580+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":205,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:21:20.775511+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,0],"acting":[1,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":269,"reported_epoch":273,"state":"active+clean","last_fresh":"2023-09-23T21:02:41.226712+0000","last_change":"2023-09-23T20:57:44.979950+0000","last_active":"2023-09-23T21:02:41.226712+0000","last_peered":"2023-09-23T21:02:41.226712+0000","last_clean":"2023-09-23T21:02:41.226712+0000","last_became_active":"2023-09-23T20:57:44.979346+0000","last_became_peered":"2023-09-23T20:57:44.979346+0000","last_unstale":"2023-09-23T21:02:41.226712+0000","last_undegraded":"2023-09-23T21:02:41.226712+0000","last_fullsized":"2023-09-23T21:02:41.226712+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:10:24.372800+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":248,"reported_epoch":273,"state":"active+clean","last_fresh":"2023-09-23T21:02:41.226336+0000","last_change":"2023-09-23T20:50:37.982235+0000","last_active":"2023-09-23T21:02:41.226336+0000","last_peered":"2023-09-23T21:02:41.226336+0000","last_clean":"2023-09-23T21:02:41.226336+0000","last_became_active":"2023-09-23T20:50:37.981913+0000","last_became_peered":"2023-09-23T20:50:37.981913+0000","last_unstale":"2023-09-23T21:02:41.226336+0000","last_undegraded":"2023-09-23T21:02:41.226336+0000","last_fullsized":"2023-09-23T21:02:41.226336+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:59:57.122584+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.f","version":"274'6285","reported_seq":6314,"reported_epoch":274,"state":"active+clean","last_fresh":"2023-09-23T21:02:49.825212+0000","last_change":"2023-09-23T21:01:25.432176+0000","last_active":"2023-09-23T21:02:49.825212+0000","last_peered":"2023-09-23T21:02:49.825212+0000","last_clean":"2023-09-23T21:02:49.825212+0000","last_became_active":"2023-09-23T21:01:25.431835+0000","last_became_peered":"2023-09-23T21:01:25.431835+0000","last_unstale":"2023-09-23T21:02:49.825212+0000","last_undegraded":"2023-09-23T21:02:49.825212+0000","last_fullsized":"2023-09-23T21:02:49.825212+0000","mapping_epoch":254,"log_start":"0'0","ondisk_log_start":"0'0","created":254,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:01:24.325318+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:01:24.325318+0000","last_clean_scrub_stamp":"2023-09-23T21:01:24.325318+0000","objects_scrubbed":0,"log_size":6285,"log_dups_size":0,"ondisk_log_size":6285,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:47:26.553643+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":411893760,"num_objects":6285,"num_object_clones":0,"num_object_copies":18855,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6285,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6285,"num_write_kb":402240,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,1],"acting":[3,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.6","version":"79'161","reported_seq":556,"reported_epoch":273,"state":"active+clean","last_fresh":"2023-09-23T21:02:41.218494+0000","last_change":"2023-09-23T20:57:26.907505+0000","last_active":"2023-09-23T21:02:41.218494+0000","last_peered":"2023-09-23T21:02:41.218494+0000","last_clean":"2023-09-23T21:02:41.218494+0000","last_became_active":"2023-09-23T20:57:08.240321+0000","last_became_peered":"2023-09-23T20:57:08.240321+0000","last_unstale":"2023-09-23T21:02:41.218494+0000","last_undegraded":"2023-09-23T21:02:41.218494+0000","last_fullsized":"2023-09-23T21:02:41.218494+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":3,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:26.907408+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:26.907408+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:34:17.392928+0000","scrub_duration":0.010337038,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":191,"reported_epoch":273,"state":"active+clean","last_fresh":"2023-09-23T21:02:41.226787+0000","last_change":"2023-09-23T20:56:45.988566+0000","last_active":"2023-09-23T21:02:41.226787+0000","last_peered":"2023-09-23T21:02:41.226787+0000","last_clean":"2023-09-23T21:02:41.226787+0000","last_became_active":"2023-09-23T20:56:45.988249+0000","last_became_peered":"2023-09-23T20:56:45.988249+0000","last_unstale":"2023-09-23T21:02:41.226787+0000","last_undegraded":"2023-09-23T21:02:41.226787+0000","last_fullsized":"2023-09-23T21:02:41.226787+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":205,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:51:36.644764+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,4],"acting":[0,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":246,"reported_epoch":273,"state":"active+clean","last_fresh":"2023-09-23T21:02:41.219468+0000","last_change":"2023-09-23T20:54:31.898965+0000","last_active":"2023-09-23T21:02:41.219468+0000","last_peered":"2023-09-23T21:02:41.219468+0000","last_clean":"2023-09-23T21:02:41.219468+0000","last_became_active":"2023-09-23T20:54:31.898663+0000","last_became_peered":"2023-09-23T20:54:31.898663+0000","last_unstale":"2023-09-23T21:02:41.219468+0000","last_undegraded":"2023-09-23T21:02:41.219468+0000","last_fullsized":"2023-09-23T21:02:41.219468+0000","mapping_epoch":136,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:53:05.058087+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":236,"reported_epoch":273,"state":"active+clean","last_fresh":"2023-09-23T21:02:41.226899+0000","last_change":"2023-09-23T20:54:31.884717+0000","last_active":"2023-09-23T21:02:41.226899+0000","last_peered":"2023-09-23T21:02:41.226899+0000","last_clean":"2023-09-23T21:02:41.226899+0000","last_became_active":"2023-09-23T20:54:31.884345+0000","last_became_peered":"2023-09-23T20:54:31.884345+0000","last_unstale":"2023-09-23T21:02:41.226899+0000","last_undegraded":"2023-09-23T21:02:41.226899+0000","last_fullsized":"2023-09-23T21:02:41.226899+0000","mapping_epoch":136,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:02:26.393868+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.c","version":"274'6242","reported_seq":6271,"reported_epoch":274,"state":"active+clean","last_fresh":"2023-09-23T21:02:47.062696+0000","last_change":"2023-09-23T21:01:25.560907+0000","last_active":"2023-09-23T21:02:47.062696+0000","last_peered":"2023-09-23T21:02:47.062696+0000","last_clean":"2023-09-23T21:02:47.062696+0000","last_became_active":"2023-09-23T21:01:25.560592+0000","last_became_peered":"2023-09-23T21:01:25.560592+0000","last_unstale":"2023-09-23T21:02:47.062696+0000","last_undegraded":"2023-09-23T21:02:47.062696+0000","last_fullsized":"2023-09-23T21:02:47.062696+0000","mapping_epoch":254,"log_start":"0'0","ondisk_log_start":"0'0","created":254,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:01:24.325318+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:01:24.325318+0000","last_clean_scrub_stamp":"2023-09-23T21:01:24.325318+0000","objects_scrubbed":0,"log_size":6242,"log_dups_size":0,"ondisk_log_size":6242,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:14:22.102068+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":409075712,"num_objects":6242,"num_object_clones":0,"num_object_copies":18726,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6242,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6242,"num_write_kb":399488,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.5","version":"79'161","reported_seq":537,"reported_epoch":273,"state":"active+clean","last_fresh":"2023-09-23T21:02:41.219569+0000","last_change":"2023-09-23T20:57:30.029479+0000","last_active":"2023-09-23T21:02:41.219569+0000","last_peered":"2023-09-23T21:02:41.219569+0000","last_clean":"2023-09-23T21:02:41.219569+0000","last_became_active":"2023-09-23T20:57:08.249909+0000","last_became_peered":"2023-09-23T20:57:08.249909+0000","last_unstale":"2023-09-23T21:02:41.219569+0000","last_undegraded":"2023-09-23T21:02:41.219569+0000","last_fullsized":"2023-09-23T21:02:41.219569+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":3,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:30.029396+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:30.029396+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:16:46.083145+0000","scrub_duration":0.012914682,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":239,"reported_epoch":273,"state":"active+clean","last_fresh":"2023-09-23T21:02:41.257981+0000","last_change":"2023-09-23T20:51:10.836910+0000","last_active":"2023-09-23T21:02:41.257981+0000","last_peered":"2023-09-23T21:02:41.257981+0000","last_clean":"2023-09-23T21:02:41.257981+0000","last_became_active":"2023-09-23T20:51:10.836557+0000","last_became_peered":"2023-09-23T20:51:10.836557+0000","last_unstale":"2023-09-23T21:02:41.257981+0000","last_undegraded":"2023-09-23T21:02:41.257981+0000","last_fullsized":"2023-09-23T21:02:41.257981+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:51:54.019969+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":245,"reported_epoch":273,"state":"active+clean","last_fresh":"2023-09-23T21:02:41.219335+0000","last_change":"2023-09-23T20:55:35.953444+0000","last_active":"2023-09-23T21:02:41.219335+0000","last_peered":"2023-09-23T21:02:41.219335+0000","last_clean":"2023-09-23T21:02:41.219335+0000","last_became_active":"2023-09-23T20:55:35.953151+0000","last_became_peered":"2023-09-23T20:55:35.953151+0000","last_unstale":"2023-09-23T21:02:41.219335+0000","last_undegraded":"2023-09-23T21:02:41.219335+0000","last_fullsized":"2023-09-23T21:02:41.219335+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:39:12.404391+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":64,"reported_epoch":273,"state":"active+clean","last_fresh":"2023-09-23T21:02:41.248843+0000","last_change":"2023-09-23T20:57:44.982509+0000","last_active":"2023-09-23T21:02:41.248843+0000","last_peered":"2023-09-23T21:02:41.248843+0000","last_clean":"2023-09-23T21:02:41.248843+0000","last_became_active":"2023-09-23T20:57:44.982175+0000","last_became_peered":"2023-09-23T20:57:44.982175+0000","last_unstale":"2023-09-23T21:02:41.248843+0000","last_undegraded":"2023-09-23T21:02:41.248843+0000","last_fullsized":"2023-09-23T21:02:41.248843+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:37:49.215488+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.d","version":"274'6134","reported_seq":6162,"reported_epoch":274,"state":"active+clean","last_fresh":"2023-09-23T21:02:47.054774+0000","last_change":"2023-09-23T21:01:25.560531+0000","last_active":"2023-09-23T21:02:47.054774+0000","last_peered":"2023-09-23T21:02:47.054774+0000","last_clean":"2023-09-23T21:02:47.054774+0000","last_became_active":"2023-09-23T21:01:25.560228+0000","last_became_peered":"2023-09-23T21:01:25.560228+0000","last_unstale":"2023-09-23T21:02:47.054774+0000","last_undegraded":"2023-09-23T21:02:47.054774+0000","last_fullsized":"2023-09-23T21:02:47.054774+0000","mapping_epoch":254,"log_start":"0'0","ondisk_log_start":"0'0","created":254,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:01:24.325318+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:01:24.325318+0000","last_clean_scrub_stamp":"2023-09-23T21:01:24.325318+0000","objects_scrubbed":0,"log_size":6134,"log_dups_size":0,"ondisk_log_size":6134,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:19:04.504855+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":401997824,"num_objects":6134,"num_object_clones":0,"num_object_copies":18402,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6134,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6134,"num_write_kb":392576,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,2],"acting":[5,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.4","version":"79'161","reported_seq":554,"reported_epoch":273,"state":"active+clean","last_fresh":"2023-09-23T21:02:41.253087+0000","last_change":"2023-09-23T20:57:20.012904+0000","last_active":"2023-09-23T21:02:41.253087+0000","last_peered":"2023-09-23T21:02:41.253087+0000","last_clean":"2023-09-23T21:02:41.253087+0000","last_became_active":"2023-09-23T20:57:08.237996+0000","last_became_peered":"2023-09-23T20:57:08.237996+0000","last_unstale":"2023-09-23T21:02:41.253087+0000","last_undegraded":"2023-09-23T21:02:41.253087+0000","last_fullsized":"2023-09-23T21:02:41.253087+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":3,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:20.012764+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:20.012764+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:51:28.796946+0000","scrub_duration":0.0045251919999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":223,"reported_epoch":273,"state":"active+clean","last_fresh":"2023-09-23T21:02:41.218540+0000","last_change":"2023-09-23T20:55:35.949541+0000","last_active":"2023-09-23T21:02:41.218540+0000","last_peered":"2023-09-23T21:02:41.218540+0000","last_clean":"2023-09-23T21:02:41.218540+0000","last_became_active":"2023-09-23T20:55:35.944635+0000","last_became_peered":"2023-09-23T20:55:35.944635+0000","last_unstale":"2023-09-23T21:02:41.218540+0000","last_undegraded":"2023-09-23T21:02:41.218540+0000","last_fullsized":"2023-09-23T21:02:41.218540+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:30:36.033585+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"70'1","reported_seq":264,"reported_epoch":273,"state":"active+clean","last_fresh":"2023-09-23T21:02:41.226436+0000","last_change":"2023-09-23T20:54:14.009723+0000","last_active":"2023-09-23T21:02:41.226436+0000","last_peered":"2023-09-23T21:02:41.226436+0000","last_clean":"2023-09-23T21:02:41.226436+0000","last_became_active":"2023-09-23T20:54:13.941771+0000","last_became_peered":"2023-09-23T20:54:13.941771+0000","last_unstale":"2023-09-23T21:02:41.226436+0000","last_undegraded":"2023-09-23T21:02:41.226436+0000","last_fullsized":"2023-09-23T21:02:41.226436+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:48:25.588116+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":178,"reported_epoch":273,"state":"active+clean","last_fresh":"2023-09-23T21:02:41.219219+0000","last_change":"2023-09-23T20:55:35.949780+0000","last_active":"2023-09-23T21:02:41.219219+0000","last_peered":"2023-09-23T21:02:41.219219+0000","last_clean":"2023-09-23T21:02:41.219219+0000","last_became_active":"2023-09-23T20:55:35.949497+0000","last_became_peered":"2023-09-23T20:55:35.949497+0000","last_unstale":"2023-09-23T21:02:41.219219+0000","last_undegraded":"2023-09-23T21:02:41.219219+0000","last_fullsized":"2023-09-23T21:02:41.219219+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:09:37.420882+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":238,"reported_epoch":273,"state":"active+clean","last_fresh":"2023-09-23T21:02:41.226827+0000","last_change":"2023-09-23T20:54:13.929196+0000","last_active":"2023-09-23T21:02:41.226827+0000","last_peered":"2023-09-23T21:02:41.226827+0000","last_clean":"2023-09-23T21:02:41.226827+0000","last_became_active":"2023-09-23T20:54:13.928831+0000","last_became_peered":"2023-09-23T20:54:13.928831+0000","last_unstale":"2023-09-23T21:02:41.226827+0000","last_undegraded":"2023-09-23T21:02:41.226827+0000","last_fullsized":"2023-09-23T21:02:41.226827+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:28:44.316851+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":243,"reported_epoch":273,"state":"active+clean","last_fresh":"2023-09-23T21:02:41.243130+0000","last_change":"2023-09-23T20:52:12.855430+0000","last_active":"2023-09-23T21:02:41.243130+0000","last_peered":"2023-09-23T21:02:41.243130+0000","last_clean":"2023-09-23T21:02:41.243130+0000","last_became_active":"2023-09-23T20:52:12.855115+0000","last_became_peered":"2023-09-23T20:52:12.855115+0000","last_unstale":"2023-09-23T21:02:41.243130+0000","last_undegraded":"2023-09-23T21:02:41.243130+0000","last_fullsized":"2023-09-23T21:02:41.243130+0000","mapping_epoch":100,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":101,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:33:23.802214+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":151,"reported_epoch":273,"state":"active+clean","last_fresh":"2023-09-23T21:02:41.225621+0000","last_change":"2023-09-23T20:54:31.902946+0000","last_active":"2023-09-23T21:02:41.225621+0000","last_peered":"2023-09-23T21:02:41.225621+0000","last_clean":"2023-09-23T21:02:41.225621+0000","last_became_active":"2023-09-23T20:54:31.889484+0000","last_became_peered":"2023-09-23T20:54:31.889484+0000","last_unstale":"2023-09-23T21:02:41.225621+0000","last_undegraded":"2023-09-23T21:02:41.225621+0000","last_fullsized":"2023-09-23T21:02:41.225621+0000","mapping_epoch":136,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:44:44.278398+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":151,"reported_epoch":273,"state":"active+clean","last_fresh":"2023-09-23T21:02:41.219950+0000","last_change":"2023-09-23T20:54:13.950351+0000","last_active":"2023-09-23T21:02:41.219950+0000","last_peered":"2023-09-23T21:02:41.219950+0000","last_clean":"2023-09-23T21:02:41.219950+0000","last_became_active":"2023-09-23T20:54:13.949987+0000","last_became_peered":"2023-09-23T20:54:13.949987+0000","last_unstale":"2023-09-23T21:02:41.219950+0000","last_undegraded":"2023-09-23T21:02:41.219950+0000","last_fullsized":"2023-09-23T21:02:41.219950+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:46:17.765134+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":182,"reported_epoch":273,"state":"active+clean","last_fresh":"2023-09-23T21:02:41.225900+0000","last_change":"2023-09-23T20:56:18.141728+0000","last_active":"2023-09-23T21:02:41.225900+0000","last_peered":"2023-09-23T21:02:41.225900+0000","last_clean":"2023-09-23T21:02:41.225900+0000","last_became_active":"2023-09-23T20:56:18.137979+0000","last_became_peered":"2023-09-23T20:56:18.137979+0000","last_unstale":"2023-09-23T21:02:41.225900+0000","last_undegraded":"2023-09-23T21:02:41.225900+0000","last_fullsized":"2023-09-23T21:02:41.225900+0000","mapping_epoch":192,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":193,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:58:05.658443+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":151,"reported_epoch":273,"state":"active+clean","last_fresh":"2023-09-23T21:02:41.219406+0000","last_change":"2023-09-23T20:54:13.935307+0000","last_active":"2023-09-23T21:02:41.219406+0000","last_peered":"2023-09-23T21:02:41.219406+0000","last_clean":"2023-09-23T21:02:41.219406+0000","last_became_active":"2023-09-23T20:54:13.935000+0000","last_became_peered":"2023-09-23T20:54:13.935000+0000","last_unstale":"2023-09-23T21:02:41.219406+0000","last_undegraded":"2023-09-23T21:02:41.219406+0000","last_fullsized":"2023-09-23T21:02:41.219406+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:47:40.974082+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":262,"reported_epoch":273,"state":"active+clean","last_fresh":"2023-09-23T21:02:41.248585+0000","last_change":"2023-09-23T20:55:35.954870+0000","last_active":"2023-09-23T21:02:41.248585+0000","last_peered":"2023-09-23T21:02:41.248585+0000","last_clean":"2023-09-23T21:02:41.248585+0000","last_became_active":"2023-09-23T20:55:35.954568+0000","last_became_peered":"2023-09-23T20:55:35.954568+0000","last_unstale":"2023-09-23T21:02:41.248585+0000","last_undegraded":"2023-09-23T21:02:41.248585+0000","last_fullsized":"2023-09-23T21:02:41.248585+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:06:22.423566+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":261,"reported_epoch":273,"state":"active+clean","last_fresh":"2023-09-23T21:02:41.247429+0000","last_change":"2023-09-23T20:51:10.838334+0000","last_active":"2023-09-23T21:02:41.247429+0000","last_peered":"2023-09-23T21:02:41.247429+0000","last_clean":"2023-09-23T21:02:41.247429+0000","last_became_active":"2023-09-23T20:51:10.838026+0000","last_became_peered":"2023-09-23T20:51:10.838026+0000","last_unstale":"2023-09-23T21:02:41.247429+0000","last_undegraded":"2023-09-23T21:02:41.247429+0000","last_fullsized":"2023-09-23T21:02:41.247429+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:04:37.009457+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":266,"reported_epoch":273,"state":"active+clean","last_fresh":"2023-09-23T21:02:41.226621+0000","last_change":"2023-09-23T20:54:31.903139+0000","last_active":"2023-09-23T21:02:41.226621+0000","last_peered":"2023-09-23T21:02:41.226621+0000","last_clean":"2023-09-23T21:02:41.226621+0000","last_became_active":"2023-09-23T20:54:31.884520+0000","last_became_peered":"2023-09-23T20:54:31.884520+0000","last_unstale":"2023-09-23T21:02:41.226621+0000","last_undegraded":"2023-09-23T21:02:41.226621+0000","last_fullsized":"2023-09-23T21:02:41.226621+0000","mapping_epoch":136,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:39:34.269705+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1108,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":168,"reported_epoch":273,"state":"active+clean","last_fresh":"2023-09-23T21:02:41.219116+0000","last_change":"2023-09-23T20:55:35.951572+0000","last_active":"2023-09-23T21:02:41.219116+0000","last_peered":"2023-09-23T21:02:41.219116+0000","last_clean":"2023-09-23T21:02:41.219116+0000","last_became_active":"2023-09-23T20:55:35.951294+0000","last_became_peered":"2023-09-23T20:55:35.951294+0000","last_unstale":"2023-09-23T21:02:41.219116+0000","last_undegraded":"2023-09-23T21:02:41.219116+0000","last_fullsized":"2023-09-23T21:02:41.219116+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:36:32.447210+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":22,"num_bytes_recovered":5990,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":8,"num_pg":16,"stat_sum":{"num_bytes":6574243840,"num_objects":100315,"num_object_clones":0,"num_object_copies":300945,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100315,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":100315,"num_write_kb":6420160,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":19641270272,"data_stored":19641270272,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":100315,"ondisk_log_size":100315,"up":48,"acting":48,"num_store_stats":8},{"poolid":1,"num_pg":21,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3381,"ondisk_log_size":3381,"up":63,"acting":63,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201548,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2229768,"kb_used_data":1997248,"kb_used_omap":0,"kb_used_meta":232512,"kb_avail":91515384,"statfs":{"total":95995035648,"available":93711753216,"internally_reserved":0,"allocated":2045181952,"data_stored":2044277838,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":238092288},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:01:49 2023","interfaces":[{"interface":"back","average":{"1min":0.63700000000000001,"5min":2.2869999999999999,"15min":1.3120000000000001},"min":{"1min":0.22800000000000001,"5min":0.22800000000000001,"15min":0.21099999999999999},"max":{"1min":1.798,"5min":43.994999999999997,"15min":43.994999999999997},"last":5.0919999999999996},{"interface":"front","average":{"1min":1.002,"5min":2.3940000000000001,"15min":1.4179999999999999},"min":{"1min":0.42899999999999999,"5min":0.318,"15min":0.218},"max":{"1min":3.633,"5min":44.206000000000003,"15min":44.206000000000003},"last":5.0289999999999999}]},{"osd":1,"last update":"Sat Sep 23 21:02:30 2023","interfaces":[{"interface":"back","average":{"1min":1.034,"5min":1.3879999999999999,"15min":1.3340000000000001},"min":{"1min":0.26300000000000001,"5min":0.253,"15min":0.23300000000000001},"max":{"1min":5.1630000000000003,"5min":10.622,"15min":22.591999999999999},"last":2.8570000000000002},{"interface":"front","average":{"1min":0.91900000000000004,"5min":1.647,"15min":1.2669999999999999},"min":{"1min":0.27100000000000002,"5min":0.25800000000000001,"15min":0.23699999999999999},"max":{"1min":3.2749999999999999,"5min":13.981,"15min":21.437000000000001},"last":2.9550000000000001}]},{"osd":2,"last update":"Sat Sep 23 21:02:00 2023","interfaces":[{"interface":"back","average":{"1min":0.73899999999999999,"5min":2.1920000000000002,"15min":1.429},"min":{"1min":0.313,"5min":0.26800000000000002,"15min":0.24399999999999999},"max":{"1min":2.0899999999999999,"5min":10.805999999999999,"15min":10.805999999999999},"last":3.7909999999999999},{"interface":"front","average":{"1min":0.78300000000000003,"5min":2.367,"15min":1.482},"min":{"1min":0.27900000000000003,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":2.2240000000000002,"5min":45.25,"15min":45.25},"last":3.621}]},{"osd":3,"last update":"Sat Sep 23 21:02:38 2023","interfaces":[{"interface":"back","average":{"1min":2.0640000000000001,"5min":2.6459999999999999,"15min":2.097},"min":{"1min":0.34000000000000002,"5min":0.315,"15min":0.314},"max":{"1min":11.625999999999999,"5min":31.786000000000001,"15min":36.408000000000001},"last":1.458},{"interface":"front","average":{"1min":1.3069999999999999,"5min":2.6539999999999999,"15min":1.9650000000000001},"min":{"1min":0.309,"5min":0.309,"15min":0.26000000000000001},"max":{"1min":6.181,"5min":32.429000000000002,"15min":35.593000000000004},"last":4.2610000000000001}]},{"osd":4,"last update":"Sat Sep 23 21:02:30 2023","interfaces":[{"interface":"back","average":{"1min":0.71499999999999997,"5min":0.58999999999999997,"15min":0.63500000000000001},"min":{"1min":0.29399999999999998,"5min":0.25700000000000001,"15min":0.23699999999999999},"max":{"1min":2.2469999999999999,"5min":4.5410000000000004,"15min":5.8920000000000003},"last":0.38900000000000001},{"interface":"front","average":{"1min":1.0169999999999999,"5min":0.72599999999999998,"15min":0.67800000000000005},"min":{"1min":0.28000000000000003,"5min":0.27100000000000002,"15min":0.26300000000000001},"max":{"1min":6.3979999999999997,"5min":6.3979999999999997,"15min":6.3979999999999997},"last":0.40100000000000002}]},{"osd":5,"last update":"Sat Sep 23 21:02:30 2023","interfaces":[{"interface":"back","average":{"1min":1.4179999999999999,"5min":0.71399999999999997,"15min":0.69099999999999995},"min":{"1min":0.34300000000000003,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":5.2089999999999996,"5min":5.2089999999999996,"15min":7.0389999999999997},"last":0.33400000000000002},{"interface":"front","average":{"1min":0.76200000000000001,"5min":0.67600000000000005,"15min":0.65100000000000002},"min":{"1min":0.33300000000000002,"5min":0.30199999999999999,"15min":0.27800000000000002},"max":{"1min":2.2530000000000001,"5min":4.5999999999999996,"15min":4.5999999999999996},"last":0.439}]},{"osd":6,"last update":"Sat Sep 23 21:02:46 2023","interfaces":[{"interface":"back","average":{"1min":1.1539999999999999,"5min":0.82699999999999996,"15min":0.72399999999999998},"min":{"1min":0.35199999999999998,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":6.1219999999999999,"5min":6.1219999999999999,"15min":6.1219999999999999},"last":0.54900000000000004},{"interface":"front","average":{"1min":1.081,"5min":0.86099999999999999,"15min":0.81999999999999995},"min":{"1min":0.32300000000000001,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":6.0919999999999996,"5min":6.0919999999999996,"15min":6.0919999999999996},"last":0.58099999999999996}]}]},{"osd":6,"up_from":164,"seq":704374636632,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2156088,"kb_used_data":2002288,"kb_used_omap":10,"kb_used_meta":153781,"kb_avail":91589064,"statfs":{"total":95995035648,"available":93787201536,"internally_reserved":0,"allocated":2050342912,"data_stored":2049454771,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":10328,"internal_metadata":157472680},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1],"upper_bound":4},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:01:47 2023","interfaces":[{"interface":"back","average":{"1min":1.337,"5min":1.851,"15min":1.1930000000000001},"min":{"1min":0.255,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":8.5269999999999992,"5min":10.404,"15min":10.404},"last":0.25900000000000001},{"interface":"front","average":{"1min":1.327,"5min":2.0569999999999999,"15min":1.341},"min":{"1min":0.20999999999999999,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":7.3029999999999999,"5min":11.023,"15min":11.023},"last":10.962}]},{"osd":1,"last update":"Sat Sep 23 21:01:47 2023","interfaces":[{"interface":"back","average":{"1min":1.0920000000000001,"5min":1.4730000000000001,"15min":1.0169999999999999},"min":{"1min":0.28499999999999998,"5min":0.27200000000000002,"15min":0.25600000000000001},"max":{"1min":3.4590000000000001,"5min":8.4309999999999992,"15min":8.4309999999999992},"last":6.024},{"interface":"front","average":{"1min":0.95699999999999996,"5min":1.7470000000000001,"15min":1.161},"min":{"1min":0.312,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":5.3070000000000004,"5min":10.518000000000001,"15min":10.518000000000001},"last":0.48399999999999999}]},{"osd":2,"last update":"Sat Sep 23 21:01:47 2023","interfaces":[{"interface":"back","average":{"1min":1.2310000000000001,"5min":2.2440000000000002,"15min":1.8460000000000001},"min":{"1min":0.312,"5min":0.28999999999999998,"15min":0.27700000000000002},"max":{"1min":4.5599999999999996,"5min":13.750999999999999,"15min":13.750999999999999},"last":3.681},{"interface":"front","average":{"1min":1.028,"5min":2.2010000000000001,"15min":1.4710000000000001},"min":{"1min":0.247,"5min":0.247,"15min":0.247},"max":{"1min":4.165,"5min":20.195,"15min":20.195},"last":0.621}]},{"osd":3,"last update":"Sat Sep 23 21:01:47 2023","interfaces":[{"interface":"back","average":{"1min":0.82399999999999995,"5min":1.966,"15min":1.3999999999999999},"min":{"1min":0.37,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":2.927,"5min":14.144,"15min":14.144},"last":6.5789999999999997},{"interface":"front","average":{"1min":1.125,"5min":1.891,"15min":1.6990000000000001},"min":{"1min":0.315,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":4.5830000000000002,"5min":11.989000000000001,"15min":11.989000000000001},"last":1.907}]},{"osd":4,"last update":"Sat Sep 23 21:01:47 2023","interfaces":[{"interface":"back","average":{"1min":0.59399999999999997,"5min":0.64700000000000002,"15min":0.91700000000000004},"min":{"1min":0.24299999999999999,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":4.1230000000000002,"5min":4.1230000000000002,"15min":9.0079999999999991},"last":1.9750000000000001},{"interface":"front","average":{"1min":0.499,"5min":0.51200000000000001,"15min":0.52400000000000002},"min":{"1min":0.29699999999999999,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":0.94499999999999995,"5min":2.4860000000000002,"15min":2.4860000000000002},"last":2.3220000000000001}]},{"osd":5,"last update":"Sat Sep 23 21:01:47 2023","interfaces":[{"interface":"back","average":{"1min":0.75800000000000001,"5min":0.74099999999999999,"15min":0.60699999999999998},"min":{"1min":0.27500000000000002,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":4.149,"5min":5.4119999999999999,"15min":5.4119999999999999},"last":1.9450000000000001},{"interface":"front","average":{"1min":0.63600000000000001,"5min":0.72399999999999998,"15min":0.78100000000000003},"min":{"1min":0.29199999999999998,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":2.6280000000000001,"5min":3.536,"15min":3.536},"last":2.3479999999999999}]},{"osd":7,"last update":"Sat Sep 23 21:01:47 2023","interfaces":[{"interface":"back","average":{"1min":0.505,"5min":0.73099999999999998,"15min":0.86299999999999999},"min":{"1min":0.32400000000000001,"5min":0.312,"15min":0.312},"max":{"1min":0.97099999999999997,"5min":7.8570000000000002,"15min":7.8570000000000002},"last":2.286},{"interface":"front","average":{"1min":0.59099999999999997,"5min":0.76900000000000002,"15min":0.60899999999999999},"min":{"1min":0.28100000000000003,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":1.829,"5min":8.282,"15min":8.282},"last":1.871}]}]},{"osd":1,"up_from":88,"seq":377957122203,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3346684,"kb_used_data":2985204,"kb_used_omap":9,"kb_used_meta":361462,"kb_avail":90398468,"statfs":{"total":95995035648,"available":92568031232,"internally_reserved":0,"allocated":3056848896,"data_stored":3055552049,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9239,"internal_metadata":370138089},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,3,0,5,2],"upper_bound":64},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:02:31 2023","interfaces":[{"interface":"back","average":{"1min":3.7730000000000001,"5min":3.081,"15min":2.0009999999999999},"min":{"1min":0.25,"5min":0.20699999999999999,"15min":0.20699999999999999},"max":{"1min":45.542000000000002,"5min":45.542000000000002,"15min":45.542000000000002},"last":4.899},{"interface":"front","average":{"1min":4.1929999999999996,"5min":3.7440000000000002,"15min":2.3100000000000001},"min":{"1min":0.28499999999999998,"5min":0.247,"15min":0.22500000000000001},"max":{"1min":46.353000000000002,"5min":46.353000000000002,"15min":46.353000000000002},"last":0.874}]},{"osd":2,"last update":"Sat Sep 23 21:02:00 2023","interfaces":[{"interface":"back","average":{"1min":0.79700000000000004,"5min":2.1840000000000002,"15min":1.478},"min":{"1min":0.20399999999999999,"5min":0.20399999999999999,"15min":0.20300000000000001},"max":{"1min":2.7759999999999998,"5min":13.752000000000001,"15min":14.335000000000001},"last":4.9279999999999999},{"interface":"front","average":{"1min":1.502,"5min":3.863,"15min":2.323},"min":{"1min":0.249,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":7.2649999999999997,"5min":18.026,"15min":38.951000000000001},"last":12.188000000000001}]},{"osd":3,"last update":"Sat Sep 23 21:02:31 2023","interfaces":[{"interface":"back","average":{"1min":2.3900000000000001,"5min":2.8410000000000002,"15min":2.0110000000000001},"min":{"1min":0.26500000000000001,"5min":0.246,"15min":0.246},"max":{"1min":8.7390000000000008,"5min":22.686,"15min":25.314},"last":5.149},{"interface":"front","average":{"1min":1.5509999999999999,"5min":2.4900000000000002,"15min":1.7629999999999999},"min":{"1min":0.25600000000000001,"5min":0.25600000000000001,"15min":0.20599999999999999},"max":{"1min":10.157999999999999,"5min":14.473000000000001,"15min":19.881},"last":4.9530000000000003}]},{"osd":4,"last update":"Sat Sep 23 21:02:31 2023","interfaces":[{"interface":"back","average":{"1min":3.5030000000000001,"5min":2.0470000000000002,"15min":1.528},"min":{"1min":0.28399999999999997,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":41.823,"5min":41.823,"15min":41.823},"last":0.32800000000000001},{"interface":"front","average":{"1min":3.097,"5min":1.9359999999999999,"15min":1.3420000000000001},"min":{"1min":0.33300000000000002,"5min":0.31900000000000001,"15min":0.27600000000000002},"max":{"1min":41.423000000000002,"5min":41.423000000000002,"15min":41.423000000000002},"last":4.8700000000000001}]},{"osd":5,"last update":"Sat Sep 23 21:02:31 2023","interfaces":[{"interface":"back","average":{"1min":1.446,"5min":2.2789999999999999,"15min":1.55},"min":{"1min":0.32000000000000001,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":7.5570000000000004,"5min":15.041,"15min":15.041},"last":0.83299999999999996},{"interface":"front","average":{"1min":1.5649999999999999,"5min":1.6830000000000001,"15min":1.431},"min":{"1min":0.36099999999999999,"5min":0.33200000000000002,"15min":0.32200000000000001},"max":{"1min":5.4930000000000003,"5min":18.280999999999999,"15min":18.280999999999999},"last":5.0529999999999999}]},{"osd":6,"last update":"Sat Sep 23 21:01:48 2023","interfaces":[{"interface":"back","average":{"1min":0.76700000000000002,"5min":1.6479999999999999,"15min":1.4259999999999999},"min":{"1min":0.38600000000000001,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":2.2240000000000002,"5min":18.984999999999999,"15min":18.984999999999999},"last":5.085},{"interface":"front","average":{"1min":1.073,"5min":1.6599999999999999,"15min":1.3879999999999999},"min":{"1min":0.37,"5min":0.33700000000000002,"15min":0.309},"max":{"1min":4.1589999999999998,"5min":8.7530000000000001,"15min":8.7530000000000001},"last":4.7839999999999998}]},{"osd":7,"last update":"Sat Sep 23 21:02:31 2023","interfaces":[{"interface":"back","average":{"1min":1.238,"5min":1.6910000000000001,"15min":1.3149999999999999},"min":{"1min":0.39600000000000002,"5min":0.39600000000000002,"15min":0.35699999999999998},"max":{"1min":3.9900000000000002,"5min":8.7769999999999992,"15min":11.186999999999999},"last":4.8280000000000003},{"interface":"front","average":{"1min":2.0329999999999999,"5min":2.3570000000000002,"15min":1.5940000000000001},"min":{"1min":0.38100000000000001,"5min":0.38100000000000001,"15min":0.34599999999999997},"max":{"1min":7.6100000000000003,"5min":15.098000000000001,"15min":15.098000000000001},"last":0.89900000000000002}]}]},{"osd":0,"up_from":83,"seq":356482285729,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2724124,"kb_used_data":2399316,"kb_used_omap":28,"kb_used_meta":324771,"kb_avail":91021028,"statfs":{"total":95995035648,"available":93205532672,"internally_reserved":0,"allocated":2456899584,"data_stored":2455632397,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":28848,"internal_metadata":332566352},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 21:02:21 2023","interfaces":[{"interface":"back","average":{"1min":2.2930000000000001,"5min":3.782,"15min":2.181},"min":{"1min":0.19500000000000001,"5min":0.19500000000000001,"15min":0.191},"max":{"1min":22.411000000000001,"5min":22.411000000000001,"15min":22.411000000000001},"last":1.3879999999999999},{"interface":"front","average":{"1min":3.0870000000000002,"5min":3.8799999999999999,"15min":2.1970000000000001},"min":{"1min":0.29299999999999998,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":22.474,"5min":22.474,"15min":22.474},"last":2.5670000000000002}]},{"osd":2,"last update":"Sat Sep 23 21:01:55 2023","interfaces":[{"interface":"back","average":{"1min":1.052,"5min":3.2599999999999998,"15min":2.1419999999999999},"min":{"1min":0.23499999999999999,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":8.2479999999999993,"5min":21.053000000000001,"15min":21.053000000000001},"last":1.7629999999999999},{"interface":"front","average":{"1min":1.3,"5min":3.9390000000000001,"15min":2.2530000000000001},"min":{"1min":0.23100000000000001,"5min":0.23100000000000001,"15min":0.21099999999999999},"max":{"1min":6.6189999999999998,"5min":21.747,"15min":21.747},"last":1.909}]},{"osd":3,"last update":"Sat Sep 23 21:02:21 2023","interfaces":[{"interface":"back","average":{"1min":1.6990000000000001,"5min":3.6360000000000001,"15min":2.1549999999999998},"min":{"1min":0.29599999999999999,"5min":0.27100000000000002,"15min":0.25800000000000001},"max":{"1min":11.617000000000001,"5min":21.184000000000001,"15min":21.184000000000001},"last":1.355},{"interface":"front","average":{"1min":2.2280000000000002,"5min":3.8839999999999999,"15min":2.3380000000000001},"min":{"1min":0.32900000000000001,"5min":0.27400000000000002,"15min":0.23300000000000001},"max":{"1min":8.4510000000000005,"5min":21.167000000000002,"15min":21.167000000000002},"last":3.6339999999999999}]},{"osd":4,"last update":"Sat Sep 23 21:01:55 2023","interfaces":[{"interface":"back","average":{"1min":1.2390000000000001,"5min":2.8599999999999999,"15min":1.7230000000000001},"min":{"1min":0.28599999999999998,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":6.5620000000000003,"5min":21.594999999999999,"15min":21.594999999999999},"last":1.581},{"interface":"front","average":{"1min":0.80400000000000005,"5min":2.2549999999999999,"15min":1.496},"min":{"1min":0.32900000000000001,"5min":0.311,"15min":0.28100000000000003},"max":{"1min":3.4289999999999998,"5min":13.904,"15min":18.724},"last":1.4330000000000001}]},{"osd":5,"last update":"Sat Sep 23 21:01:55 2023","interfaces":[{"interface":"back","average":{"1min":1.1459999999999999,"5min":1.9670000000000001,"15min":1.4339999999999999},"min":{"1min":0.34999999999999998,"5min":0.309,"15min":0.309},"max":{"1min":8.1899999999999995,"5min":8.8970000000000002,"15min":8.8970000000000002},"last":0.35899999999999999},{"interface":"front","average":{"1min":1.256,"5min":2.7000000000000002,"15min":1.724},"min":{"1min":0.373,"5min":0.373,"15min":0.34999999999999998},"max":{"1min":6.601,"5min":20.512,"15min":20.512},"last":1.6140000000000001}]},{"osd":6,"last update":"Sat Sep 23 21:01:52 2023","interfaces":[{"interface":"back","average":{"1min":0.78000000000000003,"5min":2.3199999999999998,"15min":1.8779999999999999},"min":{"1min":0.38,"5min":0.37,"15min":0.37},"max":{"1min":3.472,"5min":13.933,"15min":13.933},"last":1.411},{"interface":"front","average":{"1min":0.76500000000000001,"5min":2.4180000000000001,"15min":1.97},"min":{"1min":0.379,"5min":0.377,"15min":0.377},"max":{"1min":3.5169999999999999,"5min":13.962,"15min":13.962},"last":1.4970000000000001}]},{"osd":7,"last update":"Sat Sep 23 21:01:55 2023","interfaces":[{"interface":"back","average":{"1min":1.151,"5min":2.0699999999999998,"15min":1.4870000000000001},"min":{"1min":0.40600000000000003,"5min":0.38100000000000001,"15min":0.379},"max":{"1min":8.2210000000000001,"5min":8.7729999999999997,"15min":8.7729999999999997},"last":0.39600000000000002},{"interface":"front","average":{"1min":0.86799999999999999,"5min":2.3220000000000001,"15min":1.5289999999999999},"min":{"1min":0.45400000000000001,"5min":0.377,"15min":0.377},"max":{"1min":3.4500000000000002,"5min":13.856999999999999,"15min":18.673999999999999},"last":1.456}]}]},{"osd":2,"up_from":93,"seq":399431958676,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2915032,"kb_used_data":2539152,"kb_used_omap":81,"kb_used_meta":375790,"kb_avail":90830120,"statfs":{"total":95995035648,"available":93010042880,"internally_reserved":0,"allocated":2600091648,"data_stored":2598825723,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":83750,"internal_metadata":384809178},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1,2],"upper_bound":16},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:01:51 2023","interfaces":[{"interface":"back","average":{"1min":0.76900000000000002,"5min":3.0600000000000001,"15min":1.671},"min":{"1min":0.22900000000000001,"5min":0.189,"15min":0.17999999999999999},"max":{"1min":2.923,"5min":29.498000000000001,"15min":29.498000000000001},"last":4.2409999999999997},{"interface":"front","average":{"1min":0.624,"5min":3.4249999999999998,"15min":1.946},"min":{"1min":0.193,"5min":0.193,"15min":0.193},"max":{"1min":1.9059999999999999,"5min":17.251000000000001,"15min":17.251000000000001},"last":4.2279999999999998}]},{"osd":1,"last update":"Sat Sep 23 21:01:51 2023","interfaces":[{"interface":"back","average":{"1min":1.163,"5min":2.5800000000000001,"15min":1.831},"min":{"1min":0.246,"5min":0.21199999999999999,"15min":0.20200000000000001},"max":{"1min":6.077,"5min":14.709,"15min":16.081},"last":1.3220000000000001},{"interface":"front","average":{"1min":1.107,"5min":3.7879999999999998,"15min":2.3919999999999999},"min":{"1min":0.33500000000000002,"5min":0.26300000000000001,"15min":0.23599999999999999},"max":{"1min":6.125,"5min":26.552,"15min":26.552},"last":1.339}]},{"osd":3,"last update":"Sat Sep 23 21:02:34 2023","interfaces":[{"interface":"back","average":{"1min":2.137,"5min":4.1699999999999999,"15min":2.524},"min":{"1min":0.45500000000000002,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":5.8310000000000004,"5min":41.057000000000002,"15min":41.057000000000002},"last":6.1360000000000001},{"interface":"front","average":{"1min":1.9319999999999999,"5min":4.4340000000000002,"15min":2.8300000000000001},"min":{"1min":0.25,"5min":0.25,"15min":0.221},"max":{"1min":9.4870000000000001,"5min":41.012,"15min":41.012},"last":6.2539999999999996}]},{"osd":4,"last update":"Sat Sep 23 21:01:51 2023","interfaces":[{"interface":"back","average":{"1min":0.72799999999999998,"5min":1.706,"15min":1.3140000000000001},"min":{"1min":0.28399999999999997,"5min":0.28399999999999997,"15min":0.26400000000000001},"max":{"1min":2.8290000000000002,"5min":10.045,"15min":10.045},"last":0.48999999999999999},{"interface":"front","average":{"1min":0.78000000000000003,"5min":2.1560000000000001,"15min":1.538},"min":{"1min":0.36399999999999999,"5min":0.32400000000000001,"15min":0.313},"max":{"1min":3.0550000000000002,"5min":14.039999999999999,"15min":14.039999999999999},"last":0.35799999999999998}]},{"osd":5,"last update":"Sat Sep 23 21:01:51 2023","interfaces":[{"interface":"back","average":{"1min":0.65300000000000002,"5min":1.637,"15min":1.3959999999999999},"min":{"1min":0.30399999999999999,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":1.9119999999999999,"5min":10.069000000000001,"15min":10.069000000000001},"last":0.46100000000000002},{"interface":"front","average":{"1min":0.66500000000000004,"5min":2.073,"15min":1.534},"min":{"1min":0.31900000000000001,"5min":0.31900000000000001,"15min":0.30299999999999999},"max":{"1min":2.0880000000000001,"5min":13.98,"15min":13.98},"last":0.38200000000000001}]},{"osd":6,"last update":"Sat Sep 23 21:01:50 2023","interfaces":[{"interface":"back","average":{"1min":0.94599999999999995,"5min":2.2370000000000001,"15min":1.857},"min":{"1min":0.36599999999999999,"5min":0.312,"15min":0.312},"max":{"1min":5.3490000000000002,"5min":14.016,"15min":14.244999999999999},"last":0.40100000000000002},{"interface":"front","average":{"1min":0.90700000000000003,"5min":2.323,"15min":1.4610000000000001},"min":{"1min":0.38,"5min":0.374,"15min":0.374},"max":{"1min":5.4500000000000002,"5min":14.068,"15min":14.068},"last":0.55500000000000005}]},{"osd":7,"last update":"Sat Sep 23 21:01:51 2023","interfaces":[{"interface":"back","average":{"1min":0.84799999999999998,"5min":1.4359999999999999,"15min":1.155},"min":{"1min":0.45600000000000002,"5min":0.315,"15min":0.315},"max":{"1min":2.98,"5min":13.427,"15min":13.427},"last":0.44700000000000001},{"interface":"front","average":{"1min":0.79100000000000004,"5min":1.643,"15min":1.4099999999999999},"min":{"1min":0.36699999999999999,"5min":0.35099999999999998,"15min":0.34499999999999997},"max":{"1min":2.883,"5min":10.092000000000001,"15min":10.092000000000001},"last":0.505}]}]},{"osd":3,"up_from":98,"seq":420906795150,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2888620,"kb_used_data":2419172,"kb_used_omap":7,"kb_used_meta":469432,"kb_avail":90856532,"statfs":{"total":95995035648,"available":93037088768,"internally_reserved":0,"allocated":2477232128,"data_stored":2476006126,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7576,"internal_metadata":480698984},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:02:32 2023","interfaces":[{"interface":"back","average":{"1min":1.288,"5min":3.6619999999999999,"15min":2.4039999999999999},"min":{"1min":0.188,"5min":0.188,"15min":0.188},"max":{"1min":4.7949999999999999,"5min":27.962,"15min":27.962},"last":1.8460000000000001},{"interface":"front","average":{"1min":2.8519999999999999,"5min":3.794,"15min":2.2709999999999999},"min":{"1min":0.20499999999999999,"5min":0.19800000000000001,"15min":0.19800000000000001},"max":{"1min":10.497,"5min":28.058,"15min":28.058},"last":7.2329999999999997}]},{"osd":1,"last update":"Sat Sep 23 21:02:32 2023","interfaces":[{"interface":"back","average":{"1min":5.2779999999999996,"5min":3.8439999999999999,"15min":2.371},"min":{"1min":0.23999999999999999,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":74.093000000000004,"5min":74.093000000000004,"15min":74.093000000000004},"last":0.38400000000000001},{"interface":"front","average":{"1min":5.1210000000000004,"5min":3.887,"15min":2.4969999999999999},"min":{"1min":0.27700000000000002,"5min":0.23400000000000001,"15min":0.218},"max":{"1min":76.527000000000001,"5min":76.527000000000001,"15min":76.527000000000001},"last":1.905}]},{"osd":2,"last update":"Sat Sep 23 21:02:32 2023","interfaces":[{"interface":"back","average":{"1min":2.101,"5min":4.1749999999999998,"15min":2.399},"min":{"1min":0.23599999999999999,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":6.5549999999999997,"5min":28.251999999999999,"15min":28.251999999999999},"last":0.56399999999999995},{"interface":"front","average":{"1min":2.097,"5min":3.3199999999999998,"15min":2.3759999999999999},"min":{"1min":0.22,"5min":0.22,"15min":0.22},"max":{"1min":12.68,"5min":16.213999999999999,"15min":25.305},"last":2.044}]},{"osd":4,"last update":"Sat Sep 23 21:02:32 2023","interfaces":[{"interface":"back","average":{"1min":1.75,"5min":2.5680000000000001,"15min":1.6519999999999999},"min":{"1min":0.29399999999999998,"5min":0.29399999999999998,"15min":0.29299999999999998},"max":{"1min":7.6820000000000004,"5min":18.239000000000001,"15min":18.239000000000001},"last":0.41899999999999998},{"interface":"front","average":{"1min":1.5469999999999999,"5min":2.0030000000000001,"15min":1.6559999999999999},"min":{"1min":0.29499999999999998,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":6.0060000000000002,"5min":14.808,"15min":22.853000000000002},"last":0.72799999999999998}]},{"osd":5,"last update":"Sat Sep 23 21:02:32 2023","interfaces":[{"interface":"back","average":{"1min":1.7410000000000001,"5min":2.6739999999999999,"15min":1.6679999999999999},"min":{"1min":0.32900000000000001,"5min":0.32900000000000001,"15min":0.32500000000000001},"max":{"1min":8.0459999999999994,"5min":18.324999999999999,"15min":18.324999999999999},"last":1.79},{"interface":"front","average":{"1min":2.4860000000000002,"5min":2.9279999999999999,"15min":1.746},"min":{"1min":0.40899999999999997,"5min":0.33300000000000002,"15min":0.33100000000000002},"max":{"1min":10.558,"5min":13.529999999999999,"15min":13.529999999999999},"last":6.9660000000000002}]},{"osd":6,"last update":"Sat Sep 23 21:01:52 2023","interfaces":[{"interface":"back","average":{"1min":0.97199999999999998,"5min":1.9490000000000001,"15min":1.2829999999999999},"min":{"1min":0.40200000000000002,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":4.1230000000000002,"5min":14.833,"15min":14.833},"last":0.77300000000000002},{"interface":"front","average":{"1min":0.99099999999999999,"5min":2.5270000000000001,"15min":1.7829999999999999},"min":{"1min":0.377,"5min":0.377,"15min":0.377},"max":{"1min":2.242,"5min":13.576000000000001,"15min":13.576000000000001},"last":6.9249999999999998}]},{"osd":7,"last update":"Sat Sep 23 21:02:32 2023","interfaces":[{"interface":"back","average":{"1min":2.3420000000000001,"5min":2.7240000000000002,"15min":1.8440000000000001},"min":{"1min":0.40600000000000003,"5min":0.39200000000000002,"15min":0.36099999999999999},"max":{"1min":10.715999999999999,"5min":18.289999999999999,"15min":18.289999999999999},"last":1.7609999999999999},{"interface":"front","average":{"1min":2.8959999999999999,"5min":3.0880000000000001,"15min":1.8440000000000001},"min":{"1min":0.46500000000000002,"5min":0.38900000000000001,"15min":0.379},"max":{"1min":11.798999999999999,"5min":13.603999999999999,"15min":13.603999999999999},"last":6.9950000000000001}]}]},{"osd":4,"up_from":31,"seq":133143986457,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2186184,"kb_used_data":1999616,"kb_used_omap":0,"kb_used_meta":186560,"kb_avail":91558968,"statfs":{"total":95995035648,"available":93756383232,"internally_reserved":0,"allocated":2047606784,"data_stored":2046702923,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":191037440},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:01:55 2023","interfaces":[{"interface":"back","average":{"1min":0.55900000000000005,"5min":1.7729999999999999,"15min":1.216},"min":{"1min":0.254,"5min":0.214,"15min":0.214},"max":{"1min":1.712,"5min":11.784000000000001,"15min":11.784000000000001},"last":0.28399999999999997},{"interface":"front","average":{"1min":0.66000000000000003,"5min":1.8200000000000001,"15min":1.379},"min":{"1min":0.31900000000000001,"5min":0.27700000000000002,"15min":0.25800000000000001},"max":{"1min":2.1360000000000001,"5min":11.766,"15min":11.766},"last":0.35299999999999998}]},{"osd":1,"last update":"Sat Sep 23 21:02:24 2023","interfaces":[{"interface":"back","average":{"1min":0.94899999999999995,"5min":1.696,"15min":1.2969999999999999},"min":{"1min":0.255,"5min":0.246,"15min":0.23599999999999999},"max":{"1min":5.0229999999999997,"5min":10.153,"15min":10.153},"last":0.36199999999999999},{"interface":"front","average":{"1min":1.0940000000000001,"5min":2.2280000000000002,"15min":1.3700000000000001},"min":{"1min":0.309,"5min":0.26100000000000001,"15min":0.23200000000000001},"max":{"1min":7.3209999999999997,"5min":12.292999999999999,"15min":12.292999999999999},"last":0.42199999999999999}]},{"osd":2,"last update":"Sat Sep 23 21:01:55 2023","interfaces":[{"interface":"back","average":{"1min":0.68300000000000005,"5min":1.927,"15min":1.3899999999999999},"min":{"1min":0.33400000000000002,"5min":0.32500000000000001,"15min":0.29099999999999998},"max":{"1min":1.883,"5min":9.8469999999999995,"15min":15.288},"last":1.0640000000000001},{"interface":"front","average":{"1min":0.75700000000000001,"5min":1.8680000000000001,"15min":1.4950000000000001},"min":{"1min":0.32700000000000001,"5min":0.28599999999999998,"15min":0.28299999999999997},"max":{"1min":2.1789999999999998,"5min":12.855,"15min":15.262},"last":0.317}]},{"osd":3,"last update":"Sat Sep 23 21:02:24 2023","interfaces":[{"interface":"back","average":{"1min":1.514,"5min":2.3109999999999999,"15min":1.976},"min":{"1min":0.42699999999999999,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":7.1609999999999996,"5min":12.106999999999999,"15min":31.562999999999999},"last":0.443},{"interface":"front","average":{"1min":1.3069999999999999,"5min":2.3149999999999999,"15min":2.0110000000000001},"min":{"1min":0.499,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":7.1429999999999998,"5min":12.477,"15min":31.574000000000002},"last":0.46500000000000002}]},{"osd":5,"last update":"Sat Sep 23 21:02:43 2023","interfaces":[{"interface":"back","average":{"1min":0.94399999999999995,"5min":0.59499999999999997,"15min":0.55100000000000005},"min":{"1min":0.27200000000000002,"5min":0.251,"15min":0.22700000000000001},"max":{"1min":6.6470000000000002,"5min":6.6470000000000002,"15min":6.6470000000000002},"last":0.253},{"interface":"front","average":{"1min":1.03,"5min":0.84299999999999997,"15min":0.68100000000000005},"min":{"1min":0.32400000000000001,"5min":0.27600000000000002,"15min":0.251},"max":{"1min":5.3120000000000003,"5min":5.3120000000000003,"15min":7.7619999999999996},"last":0.33200000000000002}]},{"osd":6,"last update":"Sat Sep 23 21:01:51 2023","interfaces":[{"interface":"back","average":{"1min":0.45400000000000001,"5min":0.73399999999999999,"15min":0.56699999999999995},"min":{"1min":0.29899999999999999,"5min":0.26900000000000002,"15min":0.26500000000000001},"max":{"1min":0.65700000000000003,"5min":4.6909999999999998,"15min":4.6909999999999998},"last":0.30399999999999999},{"interface":"front","average":{"1min":0.54200000000000004,"5min":0.69699999999999995,"15min":0.66100000000000003},"min":{"1min":0.38,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":0.81799999999999995,"5min":3.456,"15min":3.456},"last":0.379}]},{"osd":7,"last update":"Sat Sep 23 21:02:24 2023","interfaces":[{"interface":"back","average":{"1min":0.61399999999999999,"5min":0.68799999999999994,"15min":0.69899999999999995},"min":{"1min":0.33700000000000002,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":1.248,"5min":3.4940000000000002,"15min":7.383},"last":0.34200000000000003},{"interface":"front","average":{"1min":0.72699999999999998,"5min":0.74199999999999999,"15min":0.70099999999999996},"min":{"1min":0.377,"5min":0.32700000000000001,"15min":0.28199999999999997},"max":{"1min":2.1699999999999999,"5min":5.6980000000000004,"15min":7.8310000000000004},"last":0.371}]}]},{"osd":5,"up_from":39,"seq":167503724822,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3450876,"kb_used_data":3271284,"kb_used_omap":0,"kb_used_meta":179584,"kb_avail":90294276,"statfs":{"total":95995035648,"available":92461338624,"internally_reserved":0,"allocated":3349794816,"data_stored":3348899531,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":183894016},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,1,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:01:45 2023","interfaces":[{"interface":"back","average":{"1min":0.83299999999999996,"5min":1.978,"15min":1.1579999999999999},"min":{"1min":0.23599999999999999,"5min":0.20399999999999999,"15min":0.20399999999999999},"max":{"1min":2.3279999999999998,"5min":12.795999999999999,"15min":12.795999999999999},"last":0.29699999999999999},{"interface":"front","average":{"1min":0.67700000000000005,"5min":2.117,"15min":1.341},"min":{"1min":0.311,"5min":0.24099999999999999,"15min":0.223},"max":{"1min":1.8320000000000001,"5min":16.219999999999999,"15min":16.219999999999999},"last":4.7569999999999997}]},{"osd":1,"last update":"Sat Sep 23 21:02:20 2023","interfaces":[{"interface":"back","average":{"1min":4.8710000000000004,"5min":2.661,"15min":1.585},"min":{"1min":0.26800000000000002,"5min":0.248,"15min":0.23300000000000001},"max":{"1min":62.320999999999998,"5min":62.320999999999998,"15min":62.320999999999998},"last":1.0069999999999999},{"interface":"front","average":{"1min":4.6079999999999997,"5min":2.7040000000000002,"15min":1.6020000000000001},"min":{"1min":0.30499999999999999,"5min":0.27200000000000002,"15min":0.22800000000000001},"max":{"1min":61.722999999999999,"5min":61.722999999999999,"15min":61.722999999999999},"last":0.49299999999999999}]},{"osd":2,"last update":"Sat Sep 23 21:01:59 2023","interfaces":[{"interface":"back","average":{"1min":0.82699999999999996,"5min":1.821,"15min":1.2649999999999999},"min":{"1min":0.23499999999999999,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":2.3090000000000002,"5min":10.579000000000001,"15min":12.651},"last":4.8879999999999999},{"interface":"front","average":{"1min":0.92300000000000004,"5min":2.1669999999999998,"15min":1.4139999999999999},"min":{"1min":0.34000000000000002,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":2.8879999999999999,"5min":12.300000000000001,"15min":12.300000000000001},"last":0.443}]},{"osd":3,"last update":"Sat Sep 23 21:02:20 2023","interfaces":[{"interface":"back","average":{"1min":6.7009999999999996,"5min":3.8119999999999998,"15min":2.1030000000000002},"min":{"1min":0.33500000000000002,"5min":0.33000000000000002,"15min":0.28599999999999998},"max":{"1min":61.667999999999999,"5min":61.667999999999999,"15min":61.667999999999999},"last":0.313},{"interface":"front","average":{"1min":7.0460000000000003,"5min":3.2389999999999999,"15min":1.78},"min":{"1min":0.377,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":62.276000000000003,"5min":62.276000000000003,"15min":62.276000000000003},"last":4.5990000000000002}]},{"osd":4,"last update":"Sat Sep 23 21:02:37 2023","interfaces":[{"interface":"back","average":{"1min":0.96399999999999997,"5min":0.67400000000000004,"15min":0.59699999999999998},"min":{"1min":0.25700000000000001,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":3.9279999999999999,"5min":4.1520000000000001,"15min":4.1520000000000001},"last":0.33900000000000002},{"interface":"front","average":{"1min":1.288,"5min":0.76000000000000001,"15min":0.64600000000000002},"min":{"1min":0.28000000000000003,"5min":0.27400000000000002,"15min":0.23599999999999999},"max":{"1min":8.4350000000000005,"5min":8.4350000000000005,"15min":8.4350000000000005},"last":0.36199999999999999}]},{"osd":6,"last update":"Sat Sep 23 21:01:45 2023","interfaces":[{"interface":"back","average":{"1min":0.59999999999999998,"5min":0.72999999999999998,"15min":0.61699999999999999},"min":{"1min":0.27900000000000003,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":1.5820000000000001,"5min":4.359,"15min":4.359},"last":0.40100000000000002},{"interface":"front","average":{"1min":0.53800000000000003,"5min":0.75,"15min":0.746},"min":{"1min":0.37,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":0.89900000000000002,"5min":3.847,"15min":3.847},"last":0.39400000000000002}]},{"osd":7,"last update":"Sat Sep 23 21:02:20 2023","interfaces":[{"interface":"back","average":{"1min":1.1819999999999999,"5min":0.83799999999999997,"15min":0.71999999999999997},"min":{"1min":0.36099999999999999,"5min":0.316,"15min":0.30199999999999999},"max":{"1min":4.1310000000000002,"5min":4.4240000000000004,"15min":8.5649999999999995},"last":0.42499999999999999},{"interface":"front","average":{"1min":0.90100000000000002,"5min":0.74099999999999999,"15min":0.70199999999999996},"min":{"1min":0.35299999999999998,"5min":0.33900000000000002,"15min":0.27700000000000002},"max":{"1min":4.6479999999999997,"5min":4.6479999999999997,"15min":4.6479999999999997},"last":0.379}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":2460745728,"data_stored":2460745728,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":2885025792,"data_stored":2885025792,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":2429353984,"data_stored":2429353984,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":2488664064,"data_stored":2488664064,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":2051145728,"data_stored":2051145728,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":3225354240,"data_stored":3225354240,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":2056388608,"data_stored":2056388608,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":2044592128,"data_stored":2044592128,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T21:02:52.709 INFO:tasks.cephadm.ceph_manager.ceph:recovered! 2023-09-23T21:02:52.709 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T21:02:52.709 INFO:tasks.thrashosds.thrasher:fixing pg num pool unique_pool_2 2023-09-23T21:02:52.710 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T21:02:52.755 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:02:52.785 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 81 16 103824 103808 80.0425 52.9375 0.0250234 0.0124882 2023-09-23T21:02:52.787 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:02:52.868 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:02:52.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:02:52 smithi070 ceph-mon[168282]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:02:52.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:02:52 smithi070 ceph-mon[168282]: failed to encode map e275 with expected crc 2023-09-23T21:02:52.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:02:52 smithi070 ceph-mon[168282]: failed to encode map e275 with expected crc 2023-09-23T21:02:52.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:02:52 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T21:02:52.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:02:52 smithi070 ceph-mon[168282]: osdmap e275: 8 total, 8 up, 8 in 2023-09-23T21:02:52.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:02:52 smithi070 ceph-mon[168282]: from='client.55486 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T21:02:52.993 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:02:53.004 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:02:53.005 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:02:53.105 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:02:53.138 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:02:53.200 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:02:53.204 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:02:53.204 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:02:53.305 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:02:53.335 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:02:53.381 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:02:53.390 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:02:53.390 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:02:53.473 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0: {} 2023-09-23T21:02:53.491 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:02:53.513 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1: {} 2023-09-23T21:02:53.556 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2: {} 2023-09-23T21:02:53.567 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:02:53.595 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3: {} 2023-09-23T21:02:53.642 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:02:53.643 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4: {} 2023-09-23T21:02:53.649 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:02:53 smithi026 ceph-mon[179688]: pgmap v727: 109 pgs: 109 active+clean; 6.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 55 MiB/s wr, 873 op/s 2023-09-23T21:02:53.649 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:02:53 smithi026 ceph-mon[175151]: pgmap v727: 109 pgs: 109 active+clean; 6.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 55 MiB/s wr, 873 op/s 2023-09-23T21:02:53.653 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:02:53.653 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:02:53.688 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T21:02:53.741 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6: {} 2023-09-23T21:02:53.754 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:02:53.782 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T21:02:53.785 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 82 16 104572 104556 79.6367 46.75 0.0419629 0.0125523 2023-09-23T21:02:53.798 INFO:teuthology.orchestra.run.smithi026.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T21:02:53.799 INFO:teuthology.orchestra.run.smithi026.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T21:02:53.799 INFO:teuthology.orchestra.run.smithi026.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T21:02:53.799 INFO:teuthology.orchestra.run.smithi026.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T21:02:53.799 INFO:teuthology.orchestra.run.smithi026.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T21:02:53.799 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T21:02:53.799 INFO:teuthology.orchestra.run.smithi026.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T21:02:53.799 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T21:02:53.851 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:02:53.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:02:53 smithi070 ceph-mon[168282]: pgmap v727: 109 pgs: 109 active+clean; 6.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 55 MiB/s wr, 873 op/s 2023-09-23T21:02:54.117 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T21:02:54.193 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:02:54.197 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:02:54.197 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:02:54.298 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:02:54.335 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:02:54.378 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:02:54.384 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:02:54.384 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:02:54.393 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:02:54.485 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:02:54.521 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:02:54.575 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:02:54.587 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:02:54.588 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:02:54.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:02:54 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/1365793622' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:02:54.642 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:02:54 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/1365793622' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:02:54.642 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:02:54 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/1365793622' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:02:54.689 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:02:54.737 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:02:54.782 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:02:54.785 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 83 16 105380 105364 79.2858 50.5 0.0163761 0.0126076 2023-09-23T21:02:54.789 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:02:54.789 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:02:54.890 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:02:54.955 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:02:55.338 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:02:55.350 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:02:55.350 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:02:55.408 INFO:teuthology.orchestra.run.smithi026.stdout:{ 2023-09-23T21:02:55.408 INFO:teuthology.orchestra.run.smithi026.stdout: "size": 20, 2023-09-23T21:02:55.408 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 600, 2023-09-23T21:02:55.409 INFO:teuthology.orchestra.run.smithi026.stdout: "ops": [ 2023-09-23T21:02:55.409 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.409 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34797.0:130474 6.d 6:baf3ed90:::benchmark_data_smithi026_223868_object29348:head [delete] snapc 0=[] RETRY=2 ondisk+retry+write+known_if_redirected+supports_pool_eio e137)", 2023-09-23T21:02:55.409 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T20:54:31.883928+0000", 2023-09-23T21:02:55.409 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 503.44427290900001, 2023-09-23T21:02:55.409 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.143112095, 2023-09-23T21:02:55.409 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:02:55.409 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "reached pg", 2023-09-23T21:02:55.409 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T21:02:55.409 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34797", 2023-09-23T21:02:55.409 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T21:02:55.410 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 130474 2023-09-23T21:02:55.410 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.410 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:02:55.410 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.410 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:02:55.410 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:31.883928+0000", 2023-09-23T21:02:55.410 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:02:55.410 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.411 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.411 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:02:55.411 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:31.883928+0000", 2023-09-23T21:02:55.411 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:02:55.411 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.411 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.411 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:02:55.411 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:31.883927+0000", 2023-09-23T21:02:55.411 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999981 2023-09-23T21:02:55.411 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.411 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.411 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:02:55.411 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:31.883932+0000", 2023-09-23T21:02:55.412 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.2610000000000001e-06 2023-09-23T21:02:55.412 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.412 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.412 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:02:55.412 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:31.883943+0000", 2023-09-23T21:02:55.412 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0861e-05 2023-09-23T21:02:55.412 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.412 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.412 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:02:55.412 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:31.883954+0000", 2023-09-23T21:02:55.412 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0686999999999999e-05 2023-09-23T21:02:55.412 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.412 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.413 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:02:55.413 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:31.884281+0000", 2023-09-23T21:02:55.413 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00032779599999999999 2023-09-23T21:02:55.413 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.413 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.413 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T21:02:55.413 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:31.884291+0000", 2023-09-23T21:02:55.413 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.3000000000000007e-06 2023-09-23T21:02:55.413 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.414 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.414 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:02:55.414 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:33.025141+0000", 2023-09-23T21:02:55.414 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1408508580000001 2023-09-23T21:02:55.414 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.414 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.414 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T21:02:55.414 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:33.027041+0000", 2023-09-23T21:02:55.414 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0018991030000000001 2023-09-23T21:02:55.415 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:02:55.415 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:02:55.415 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:02:55.415 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.415 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.415 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34797.0:130489 6.d 6:bb89226e:::benchmark_data_smithi026_223868_object29363:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e137)", 2023-09-23T21:02:55.415 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T20:54:31.883974+0000", 2023-09-23T21:02:55.415 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 503.444227259, 2023-09-23T21:02:55.415 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.161869933, 2023-09-23T21:02:55.415 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:02:55.415 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:02:55.415 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T21:02:55.416 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34797", 2023-09-23T21:02:55.416 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T21:02:55.416 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 130489 2023-09-23T21:02:55.416 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.416 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:02:55.416 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.416 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:02:55.416 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:31.883974+0000", 2023-09-23T21:02:55.416 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:02:55.416 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.416 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.416 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:02:55.416 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:31.883974+0000", 2023-09-23T21:02:55.417 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:02:55.417 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.417 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.417 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:02:55.417 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:31.883974+0000", 2023-09-23T21:02:55.417 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T21:02:55.417 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.417 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.417 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:02:55.417 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:31.883977+0000", 2023-09-23T21:02:55.417 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.348e-06 2023-09-23T21:02:55.417 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.417 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.418 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:02:55.418 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:31.883985+0000", 2023-09-23T21:02:55.418 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.191e-06 2023-09-23T21:02:55.418 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.418 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.418 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:02:55.418 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:31.883992+0000", 2023-09-23T21:02:55.418 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.7610000000000002e-06 2023-09-23T21:02:55.418 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.418 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.418 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:02:55.418 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:31.884354+0000", 2023-09-23T21:02:55.419 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00036240399999999998 2023-09-23T21:02:55.419 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.419 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.419 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T21:02:55.419 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:31.884363+0000", 2023-09-23T21:02:55.419 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.9989999999999997e-06 2023-09-23T21:02:55.419 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.419 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.420 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:02:55.420 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:33.027448+0000", 2023-09-23T21:02:55.420 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.143084325 2023-09-23T21:02:55.420 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.420 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.420 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:02:55.421 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:33.028112+0000", 2023-09-23T21:02:55.421 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00066470099999999996 2023-09-23T21:02:55.421 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.421 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.421 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 1,2,5", 2023-09-23T21:02:55.421 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:33.028273+0000", 2023-09-23T21:02:55.422 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00016043599999999999 2023-09-23T21:02:55.422 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.422 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.422 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T21:02:55.422 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:33.035710+0000", 2023-09-23T21:02:55.422 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0074372700000000002 2023-09-23T21:02:55.423 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.423 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.423 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:02:55.423 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:33.038722+0000", 2023-09-23T21:02:55.423 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0030117439999999998 2023-09-23T21:02:55.423 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.424 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.424 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:02:55.424 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:33.040863+0000", 2023-09-23T21:02:55.424 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0021409070000000001 2023-09-23T21:02:55.424 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.424 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.424 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:02:55.425 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:33.045669+0000", 2023-09-23T21:02:55.425 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0048059039999999997 2023-09-23T21:02:55.425 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.425 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.425 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T21:02:55.425 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:33.045728+0000", 2023-09-23T21:02:55.425 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.9129999999999998e-05 2023-09-23T21:02:55.425 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.425 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.425 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T21:02:55.425 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:33.045844+0000", 2023-09-23T21:02:55.425 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000116372 2023-09-23T21:02:55.425 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:02:55.426 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:02:55.426 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:02:55.426 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.426 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.426 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34797.0:130501 6.a 6:577795ba:::benchmark_data_smithi026_223868_object29375:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e137)", 2023-09-23T21:02:55.426 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T20:54:31.883997+0000", 2023-09-23T21:02:55.426 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 503.44420433200003, 2023-09-23T21:02:55.426 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1379962809999999, 2023-09-23T21:02:55.426 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:02:55.426 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:02:55.426 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T21:02:55.426 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34797", 2023-09-23T21:02:55.427 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T21:02:55.427 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 130501 2023-09-23T21:02:55.427 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.427 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:02:55.427 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.427 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:02:55.427 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:31.883997+0000", 2023-09-23T21:02:55.427 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:02:55.427 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.427 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.428 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:02:55.428 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:31.883997+0000", 2023-09-23T21:02:55.428 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:02:55.428 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.428 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.428 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:02:55.428 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:31.883997+0000", 2023-09-23T21:02:55.428 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T21:02:55.428 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.428 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.428 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:02:55.428 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:31.883999+0000", 2023-09-23T21:02:55.429 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.8660000000000002e-06 2023-09-23T21:02:55.429 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.429 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.429 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:02:55.429 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:31.884006+0000", 2023-09-23T21:02:55.429 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.5950000000000004e-06 2023-09-23T21:02:55.429 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.429 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.429 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:02:55.429 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:31.884013+0000", 2023-09-23T21:02:55.429 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.9179999999999997e-06 2023-09-23T21:02:55.429 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.430 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.430 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:02:55.430 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:31.884335+0000", 2023-09-23T21:02:55.430 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00032235800000000002 2023-09-23T21:02:55.430 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.430 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.430 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T21:02:55.430 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:31.884344+0000", 2023-09-23T21:02:55.430 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.7930000000000003e-06 2023-09-23T21:02:55.430 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.430 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.430 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:02:55.430 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:33.012175+0000", 2023-09-23T21:02:55.431 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.127831298 2023-09-23T21:02:55.431 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.431 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.431 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:02:55.431 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:33.014065+0000", 2023-09-23T21:02:55.431 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.001889305 2023-09-23T21:02:55.431 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.431 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.431 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 2,5,6", 2023-09-23T21:02:55.431 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:33.014209+0000", 2023-09-23T21:02:55.431 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00014406099999999999 2023-09-23T21:02:55.431 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.431 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.432 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T21:02:55.432 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:33.021710+0000", 2023-09-23T21:02:55.432 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0075012999999999998 2023-09-23T21:02:55.432 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.432 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.432 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:02:55.432 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:33.021748+0000", 2023-09-23T21:02:55.432 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.782e-05 2023-09-23T21:02:55.432 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.432 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.432 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:02:55.432 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:33.021781+0000", 2023-09-23T21:02:55.433 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.2969999999999998e-05 2023-09-23T21:02:55.433 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.433 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.433 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:02:55.433 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:33.021834+0000", 2023-09-23T21:02:55.433 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.3510000000000001e-05 2023-09-23T21:02:55.433 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.433 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.433 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T21:02:55.433 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:33.021888+0000", 2023-09-23T21:02:55.433 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.3115000000000002e-05 2023-09-23T21:02:55.433 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.433 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.433 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T21:02:55.434 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:33.021993+0000", 2023-09-23T21:02:55.434 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000105827 2023-09-23T21:02:55.434 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:02:55.434 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:02:55.434 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:02:55.434 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.434 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.434 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34797.0:130513 6.a 6:5baf817b:::benchmark_data_smithi026_223868_object29387:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e137)", 2023-09-23T21:02:55.434 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T20:54:31.884022+0000", 2023-09-23T21:02:55.434 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 503.44417974599997, 2023-09-23T21:02:55.434 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1549781690000001, 2023-09-23T21:02:55.434 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:02:55.435 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:02:55.435 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T21:02:55.435 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34797", 2023-09-23T21:02:55.435 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T21:02:55.435 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 130513 2023-09-23T21:02:55.435 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.435 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:02:55.435 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.435 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:02:55.435 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:31.884022+0000", 2023-09-23T21:02:55.435 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:02:55.435 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.435 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.436 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:02:55.436 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:31.884022+0000", 2023-09-23T21:02:55.436 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:02:55.436 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.436 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.436 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:02:55.436 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:31.884021+0000", 2023-09-23T21:02:55.436 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T21:02:55.436 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.436 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.436 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:02:55.436 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:31.884024+0000", 2023-09-23T21:02:55.436 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.8720000000000002e-06 2023-09-23T21:02:55.437 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.437 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.437 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:02:55.437 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:31.884031+0000", 2023-09-23T21:02:55.437 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.1670000000000002e-06 2023-09-23T21:02:55.437 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.437 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.437 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:02:55.437 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:31.884038+0000", 2023-09-23T21:02:55.437 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.2600000000000002e-06 2023-09-23T21:02:55.437 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.437 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.437 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:02:55.438 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:31.884370+0000", 2023-09-23T21:02:55.438 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00033257499999999999 2023-09-23T21:02:55.438 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.438 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.438 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T21:02:55.438 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:31.884378+0000", 2023-09-23T21:02:55.438 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.1899999999999995e-06 2023-09-23T21:02:55.438 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.438 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.438 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:02:55.438 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:33.018253+0000", 2023-09-23T21:02:55.438 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.133874724 2023-09-23T21:02:55.438 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.439 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.439 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:02:55.439 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:33.018408+0000", 2023-09-23T21:02:55.439 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00015456199999999999 2023-09-23T21:02:55.439 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.439 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.439 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 2,5,6", 2023-09-23T21:02:55.439 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:33.018566+0000", 2023-09-23T21:02:55.439 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00015848300000000001 2023-09-23T21:02:55.439 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.439 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.439 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T21:02:55.439 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:33.022005+0000", 2023-09-23T21:02:55.440 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0034394249999999999 2023-09-23T21:02:55.440 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.440 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.440 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:02:55.440 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:33.025311+0000", 2023-09-23T21:02:55.440 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0033057469999999999 2023-09-23T21:02:55.440 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.440 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.440 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:02:55.440 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:33.025347+0000", 2023-09-23T21:02:55.440 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.5887999999999999e-05 2023-09-23T21:02:55.440 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.440 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.441 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:02:55.441 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:33.038801+0000", 2023-09-23T21:02:55.441 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.013454173 2023-09-23T21:02:55.441 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.441 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.441 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T21:02:55.441 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:33.038860+0000", 2023-09-23T21:02:55.441 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.8795999999999997e-05 2023-09-23T21:02:55.441 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.441 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.441 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T21:02:55.441 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:33.039000+0000", 2023-09-23T21:02:55.441 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00013974899999999999 2023-09-23T21:02:55.442 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:02:55.442 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:02:55.442 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:02:55.442 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.442 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.442 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34797.0:194304 6.10 6:0f79980e:::benchmark_data_smithi026_223868_object93178:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e179)", 2023-09-23T21:02:55.442 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T20:56:00.936645+0000", 2023-09-23T21:02:55.442 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 414.39155620600002, 2023-09-23T21:02:55.442 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.3260833879999998, 2023-09-23T21:02:55.442 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:02:55.442 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "reached pg", 2023-09-23T21:02:55.442 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T21:02:55.443 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34797", 2023-09-23T21:02:55.443 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T21:02:55.443 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 194304 2023-09-23T21:02:55.443 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.443 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:02:55.443 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.443 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:02:55.443 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:00.936645+0000", 2023-09-23T21:02:55.443 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:02:55.443 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.443 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.443 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:02:55.443 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:00.936645+0000", 2023-09-23T21:02:55.444 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:02:55.444 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.444 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.444 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:02:55.444 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:00.936643+0000", 2023-09-23T21:02:55.444 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999976 2023-09-23T21:02:55.444 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.444 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.444 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:02:55.444 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:00.936653+0000", 2023-09-23T21:02:55.444 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.5619999999999993e-06 2023-09-23T21:02:55.444 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.444 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.445 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:02:55.445 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:00.936672+0000", 2023-09-23T21:02:55.445 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.9179000000000001e-05 2023-09-23T21:02:55.445 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.445 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.445 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:02:55.445 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:00.936680+0000", 2023-09-23T21:02:55.445 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.2009999999999999e-06 2023-09-23T21:02:55.445 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.445 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.445 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:02:55.445 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:00.936746+0000", 2023-09-23T21:02:55.446 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.6188999999999999e-05 2023-09-23T21:02:55.446 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.446 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.446 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:02:55.446 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:00.936882+0000", 2023-09-23T21:02:55.446 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000135795 2023-09-23T21:02:55.446 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.446 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.446 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 2,3", 2023-09-23T21:02:55.446 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:00.937429+0000", 2023-09-23T21:02:55.446 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00054749499999999995 2023-09-23T21:02:55.447 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.447 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.447 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T21:02:55.447 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:00.940535+0000", 2023-09-23T21:02:55.447 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0031056090000000001 2023-09-23T21:02:55.447 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.447 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.447 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:02:55.447 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:00.964290+0000", 2023-09-23T21:02:55.447 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.023754941000000002 2023-09-23T21:02:55.447 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.447 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.447 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T21:02:55.448 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:00.964301+0000", 2023-09-23T21:02:55.448 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1242e-05 2023-09-23T21:02:55.448 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.448 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.448 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:02:55.448 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.950552+0000", 2023-09-23T21:02:55.448 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.98625030400000002 2023-09-23T21:02:55.448 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.448 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.448 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T21:02:55.448 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.950566+0000", 2023-09-23T21:02:55.448 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.488e-05 2023-09-23T21:02:55.448 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.449 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.449 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:02:55.449 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.254843+0000", 2023-09-23T21:02:55.449 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.3042760799999999 2023-09-23T21:02:55.449 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.449 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.449 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for missing object", 2023-09-23T21:02:55.449 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.254977+0000", 2023-09-23T21:02:55.449 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00013490299999999999 2023-09-23T21:02:55.449 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.449 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.449 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:02:55.450 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.262666+0000", 2023-09-23T21:02:55.450 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0076884479999999996 2023-09-23T21:02:55.450 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.450 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.450 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T21:02:55.450 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.262729+0000", 2023-09-23T21:02:55.450 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.2722e-05 2023-09-23T21:02:55.451 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:02:55.451 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:02:55.451 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:02:55.451 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.451 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.451 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34797.0:194354 6.10 6:0825a30e:::benchmark_data_smithi026_223868_object93228:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e180)", 2023-09-23T21:02:55.451 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T20:56:01.013619+0000", 2023-09-23T21:02:55.451 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 414.31458200899999, 2023-09-23T21:02:55.452 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.3123553139999999, 2023-09-23T21:02:55.452 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:02:55.452 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:02:55.452 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T21:02:55.452 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34797", 2023-09-23T21:02:55.452 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T21:02:55.452 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 194354 2023-09-23T21:02:55.452 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.453 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:02:55.453 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.453 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:02:55.453 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.013619+0000", 2023-09-23T21:02:55.453 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:02:55.453 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.453 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.453 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:02:55.454 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.013619+0000", 2023-09-23T21:02:55.454 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:02:55.454 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.454 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.454 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:02:55.454 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.013618+0000", 2023-09-23T21:02:55.455 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T21:02:55.455 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.455 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.455 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:02:55.455 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.013625+0000", 2023-09-23T21:02:55.455 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.6410000000000002e-06 2023-09-23T21:02:55.455 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.456 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.456 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:02:55.456 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.013649+0000", 2023-09-23T21:02:55.456 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.3589000000000001e-05 2023-09-23T21:02:55.456 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.456 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.456 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:02:55.456 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.013661+0000", 2023-09-23T21:02:55.457 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1861e-05 2023-09-23T21:02:55.457 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.457 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.457 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:02:55.457 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.013731+0000", 2023-09-23T21:02:55.457 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.9720000000000003e-05 2023-09-23T21:02:55.457 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.457 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.458 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T21:02:55.458 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.013745+0000", 2023-09-23T21:02:55.458 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.4389000000000001e-05 2023-09-23T21:02:55.458 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.458 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.458 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:02:55.458 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.950782+0000", 2023-09-23T21:02:55.458 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.93703733700000003 2023-09-23T21:02:55.458 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.458 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.458 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T21:02:55.458 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.950792+0000", 2023-09-23T21:02:55.459 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.8649999999999996e-06 2023-09-23T21:02:55.459 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.459 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.459 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:02:55.459 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.255412+0000", 2023-09-23T21:02:55.459 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.3046194689999999 2023-09-23T21:02:55.459 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.459 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.459 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for degraded object", 2023-09-23T21:02:55.459 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.255792+0000", 2023-09-23T21:02:55.459 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00038020699999999997 2023-09-23T21:02:55.459 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.459 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.460 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:02:55.460 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.263497+0000", 2023-09-23T21:02:55.460 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0077052049999999997 2023-09-23T21:02:55.460 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.460 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.460 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:02:55.460 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.263609+0000", 2023-09-23T21:02:55.460 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000111499 2023-09-23T21:02:55.460 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.460 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.460 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 1,5", 2023-09-23T21:02:55.460 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.264170+0000", 2023-09-23T21:02:55.460 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00056130799999999995 2023-09-23T21:02:55.460 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.461 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.461 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T21:02:55.461 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.270530+0000", 2023-09-23T21:02:55.461 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0063601439999999999 2023-09-23T21:02:55.461 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.461 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.461 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:02:55.461 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.321515+0000", 2023-09-23T21:02:55.461 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.050984578000000003 2023-09-23T21:02:55.461 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.461 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.461 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:02:55.461 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.325525+0000", 2023-09-23T21:02:55.462 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0040102269999999999 2023-09-23T21:02:55.462 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.462 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.462 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T21:02:55.462 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.325562+0000", 2023-09-23T21:02:55.462 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.7540999999999999e-05 2023-09-23T21:02:55.462 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.462 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.462 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T21:02:55.462 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.325975+0000", 2023-09-23T21:02:55.462 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00041231800000000001 2023-09-23T21:02:55.463 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:02:55.463 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:02:55.463 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:02:55.463 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.463 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.463 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34797.0:194386 6.10 6:0e6bbf83:::benchmark_data_smithi026_223868_object93260:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e180)", 2023-09-23T21:02:55.463 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T20:56:01.046640+0000", 2023-09-23T21:02:55.463 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 414.28156122899998, 2023-09-23T21:02:55.463 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.2797770470000001, 2023-09-23T21:02:55.463 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:02:55.463 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:02:55.463 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T21:02:55.463 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34797", 2023-09-23T21:02:55.464 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T21:02:55.464 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 194386 2023-09-23T21:02:55.464 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.464 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:02:55.464 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.464 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:02:55.464 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.046640+0000", 2023-09-23T21:02:55.464 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:02:55.464 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.464 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.464 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:02:55.464 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.046640+0000", 2023-09-23T21:02:55.465 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:02:55.465 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.465 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.465 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:02:55.465 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.046639+0000", 2023-09-23T21:02:55.465 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T21:02:55.465 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.465 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.465 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:02:55.465 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.046647+0000", 2023-09-23T21:02:55.465 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.9270000000000005e-06 2023-09-23T21:02:55.465 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.465 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.466 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:02:55.466 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.046667+0000", 2023-09-23T21:02:55.466 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.0699e-05 2023-09-23T21:02:55.466 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.466 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.466 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:02:55.466 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.046678+0000", 2023-09-23T21:02:55.466 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0336000000000001e-05 2023-09-23T21:02:55.466 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.466 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.466 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:02:55.466 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.046740+0000", 2023-09-23T21:02:55.467 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.2643000000000003e-05 2023-09-23T21:02:55.467 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.467 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.467 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T21:02:55.467 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.046757+0000", 2023-09-23T21:02:55.467 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.6673000000000001e-05 2023-09-23T21:02:55.467 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.467 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.467 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:02:55.467 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.950820+0000", 2023-09-23T21:02:55.467 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.90406270899999996 2023-09-23T21:02:55.467 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.468 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.468 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T21:02:55.468 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.950827+0000", 2023-09-23T21:02:55.468 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.3799999999999996e-06 2023-09-23T21:02:55.468 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.468 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.468 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:02:55.468 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.255844+0000", 2023-09-23T21:02:55.468 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.305017112 2023-09-23T21:02:55.468 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.468 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.468 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for degraded object", 2023-09-23T21:02:55.468 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.256176+0000", 2023-09-23T21:02:55.468 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00033179999999999999 2023-09-23T21:02:55.469 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.469 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.469 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:02:55.469 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.267675+0000", 2023-09-23T21:02:55.469 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.011498759000000001 2023-09-23T21:02:55.469 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.469 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.469 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:02:55.469 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.267748+0000", 2023-09-23T21:02:55.469 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.3232000000000006e-05 2023-09-23T21:02:55.469 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.469 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.469 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 1,5", 2023-09-23T21:02:55.470 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.268150+0000", 2023-09-23T21:02:55.470 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000402113 2023-09-23T21:02:55.470 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.470 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.470 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T21:02:55.470 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.273531+0000", 2023-09-23T21:02:55.470 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.005380625 2023-09-23T21:02:55.470 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.470 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.470 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:02:55.470 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.321574+0000", 2023-09-23T21:02:55.470 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.048043172000000002 2023-09-23T21:02:55.471 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.471 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.471 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:02:55.471 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.326045+0000", 2023-09-23T21:02:55.471 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0044713649999999997 2023-09-23T21:02:55.471 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.471 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.471 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T21:02:55.471 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.326080+0000", 2023-09-23T21:02:55.471 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.4918e-05 2023-09-23T21:02:55.471 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.471 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.471 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T21:02:55.472 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.326417+0000", 2023-09-23T21:02:55.472 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00033714999999999998 2023-09-23T21:02:55.472 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:02:55.472 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:02:55.472 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:02:55.472 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.472 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.472 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34797.0:194397 6.10 6:0cf77e90:::benchmark_data_smithi026_223868_object93271:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e180)", 2023-09-23T21:02:55.472 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T20:56:01.071586+0000", 2023-09-23T21:02:55.472 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 414.25661571099999, 2023-09-23T21:02:55.472 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.255280086, 2023-09-23T21:02:55.472 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:02:55.472 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:02:55.473 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T21:02:55.473 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34797", 2023-09-23T21:02:55.473 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T21:02:55.473 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 194397 2023-09-23T21:02:55.473 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.473 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:02:55.473 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.473 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:02:55.473 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.071586+0000", 2023-09-23T21:02:55.473 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:02:55.473 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.473 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.473 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:02:55.474 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.071586+0000", 2023-09-23T21:02:55.474 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:02:55.474 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.474 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.474 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:02:55.474 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.071585+0000", 2023-09-23T21:02:55.474 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T21:02:55.474 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.474 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.474 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:02:55.474 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.071588+0000", 2023-09-23T21:02:55.474 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.1950000000000002e-06 2023-09-23T21:02:55.474 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.475 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.475 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:02:55.475 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.071593+0000", 2023-09-23T21:02:55.475 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.7830000000000004e-06 2023-09-23T21:02:55.475 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.475 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.475 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:02:55.475 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.071599+0000", 2023-09-23T21:02:55.475 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.8900000000000004e-06 2023-09-23T21:02:55.475 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.475 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.475 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:02:55.475 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.071767+0000", 2023-09-23T21:02:55.476 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00016748500000000001 2023-09-23T21:02:55.476 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.476 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.476 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T21:02:55.476 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.071779+0000", 2023-09-23T21:02:55.476 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.2464e-05 2023-09-23T21:02:55.476 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.476 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.476 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:02:55.476 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.950851+0000", 2023-09-23T21:02:55.476 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.87907165899999995 2023-09-23T21:02:55.476 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.476 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.477 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T21:02:55.477 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.950859+0000", 2023-09-23T21:02:55.477 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.2940000000000006e-06 2023-09-23T21:02:55.477 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.477 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.477 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:02:55.477 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.256209+0000", 2023-09-23T21:02:55.477 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.3053493780000001 2023-09-23T21:02:55.477 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.477 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.477 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for degraded object", 2023-09-23T21:02:55.477 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.256530+0000", 2023-09-23T21:02:55.477 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00032157699999999997 2023-09-23T21:02:55.478 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.478 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.478 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:02:55.478 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.270992+0000", 2023-09-23T21:02:55.478 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.014462299 2023-09-23T21:02:55.478 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.478 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.478 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:02:55.478 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.271067+0000", 2023-09-23T21:02:55.478 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.4938000000000007e-05 2023-09-23T21:02:55.478 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.478 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.479 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 1,5", 2023-09-23T21:02:55.479 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.271446+0000", 2023-09-23T21:02:55.479 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00037825400000000001 2023-09-23T21:02:55.479 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.479 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.479 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T21:02:55.480 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.277790+0000", 2023-09-23T21:02:55.480 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0063448840000000003 2023-09-23T21:02:55.480 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.480 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.480 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:02:55.480 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.321615+0000", 2023-09-23T21:02:55.481 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.043824241999999999 2023-09-23T21:02:55.481 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.481 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.481 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:02:55.481 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.326572+0000", 2023-09-23T21:02:55.481 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0049568980000000004 2023-09-23T21:02:55.482 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.482 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.482 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T21:02:55.482 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.326624+0000", 2023-09-23T21:02:55.482 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.2061000000000002e-05 2023-09-23T21:02:55.482 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.482 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.482 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T21:02:55.482 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.326866+0000", 2023-09-23T21:02:55.482 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00024209700000000001 2023-09-23T21:02:55.482 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:02:55.482 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:02:55.483 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:02:55.483 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.483 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.483 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34797.0:194462 6.10 6:0e070208:::benchmark_data_smithi026_223868_object93336:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e180)", 2023-09-23T21:02:55.483 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T20:56:01.136635+0000", 2023-09-23T21:02:55.483 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 414.191566378, 2023-09-23T21:02:55.483 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.1905668330000001, 2023-09-23T21:02:55.483 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:02:55.483 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:02:55.483 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T21:02:55.483 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34797", 2023-09-23T21:02:55.483 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T21:02:55.483 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 194462 2023-09-23T21:02:55.484 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.484 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:02:55.484 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.484 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:02:55.484 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.136635+0000", 2023-09-23T21:02:55.484 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:02:55.484 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.484 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.484 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:02:55.484 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.136635+0000", 2023-09-23T21:02:55.484 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:02:55.484 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.484 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.485 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:02:55.485 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.136633+0000", 2023-09-23T21:02:55.485 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999981 2023-09-23T21:02:55.485 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.485 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.485 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:02:55.485 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.136640+0000", 2023-09-23T21:02:55.485 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.5989999999999998e-06 2023-09-23T21:02:55.485 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.485 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.485 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:02:55.485 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.136656+0000", 2023-09-23T21:02:55.485 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.6595e-05 2023-09-23T21:02:55.486 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.486 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.486 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:02:55.486 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.136665+0000", 2023-09-23T21:02:55.486 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.1540000000000002e-06 2023-09-23T21:02:55.486 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.486 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.486 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:02:55.486 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.136719+0000", 2023-09-23T21:02:55.486 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.4160000000000003e-05 2023-09-23T21:02:55.486 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.486 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.487 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T21:02:55.487 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.136726+0000", 2023-09-23T21:02:55.487 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.4429999999999997e-06 2023-09-23T21:02:55.487 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.487 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.487 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:02:55.487 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.950961+0000", 2023-09-23T21:02:55.487 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.81423474299999998 2023-09-23T21:02:55.487 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.487 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.487 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T21:02:55.487 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.950970+0000", 2023-09-23T21:02:55.487 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.2690000000000005e-06 2023-09-23T21:02:55.487 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.488 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.488 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:02:55.488 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.256671+0000", 2023-09-23T21:02:55.488 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.305701193 2023-09-23T21:02:55.488 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.488 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.488 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for degraded object", 2023-09-23T21:02:55.488 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.257010+0000", 2023-09-23T21:02:55.488 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00033869800000000002 2023-09-23T21:02:55.488 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.488 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.488 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:02:55.488 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.274991+0000", 2023-09-23T21:02:55.489 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.017981115999999998 2023-09-23T21:02:55.489 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.489 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.489 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:02:55.489 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.275118+0000", 2023-09-23T21:02:55.489 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00012721400000000001 2023-09-23T21:02:55.489 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.489 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.489 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 1,5", 2023-09-23T21:02:55.489 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.275667+0000", 2023-09-23T21:02:55.489 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00054888699999999996 2023-09-23T21:02:55.489 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.489 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.490 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T21:02:55.490 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.283025+0000", 2023-09-23T21:02:55.490 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0073580700000000004 2023-09-23T21:02:55.490 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.490 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.490 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:02:55.490 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.321667+0000", 2023-09-23T21:02:55.490 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.038641406000000003 2023-09-23T21:02:55.490 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.490 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.491 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:02:55.491 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.326925+0000", 2023-09-23T21:02:55.491 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0052579829999999999 2023-09-23T21:02:55.491 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.491 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.491 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T21:02:55.491 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.326974+0000", 2023-09-23T21:02:55.491 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.9079999999999998e-05 2023-09-23T21:02:55.491 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.491 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.491 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T21:02:55.491 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.327202+0000", 2023-09-23T21:02:55.491 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000228063 2023-09-23T21:02:55.492 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:02:55.492 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:02:55.492 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:02:55.492 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.492 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.492 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34797.0:194568 6.10 6:0af43279:::benchmark_data_smithi026_223868_object93442:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e180)", 2023-09-23T21:02:55.492 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T20:56:01.267334+0000", 2023-09-23T21:02:55.492 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 414.06086771899999, 2023-09-23T21:02:55.492 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.0602084349999998, 2023-09-23T21:02:55.492 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:02:55.492 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:02:55.492 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T21:02:55.492 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34797", 2023-09-23T21:02:55.493 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T21:02:55.493 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 194568 2023-09-23T21:02:55.493 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.493 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:02:55.493 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.493 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:02:55.493 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.267334+0000", 2023-09-23T21:02:55.493 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:02:55.493 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.493 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.494 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:02:55.494 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.267334+0000", 2023-09-23T21:02:55.494 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:02:55.494 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.494 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.494 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:02:55.494 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.267332+0000", 2023-09-23T21:02:55.494 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T21:02:55.494 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.494 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.494 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:02:55.494 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.267340+0000", 2023-09-23T21:02:55.495 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.6410000000000002e-06 2023-09-23T21:02:55.495 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.495 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.495 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:02:55.495 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.267356+0000", 2023-09-23T21:02:55.495 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.6409000000000002e-05 2023-09-23T21:02:55.495 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.495 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.495 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:02:55.495 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.267364+0000", 2023-09-23T21:02:55.495 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.6580000000000002e-06 2023-09-23T21:02:55.495 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.495 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.496 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:02:55.496 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.267423+0000", 2023-09-23T21:02:55.496 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.9392000000000003e-05 2023-09-23T21:02:55.496 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.496 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.496 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T21:02:55.496 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.267435+0000", 2023-09-23T21:02:55.496 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1653e-05 2023-09-23T21:02:55.496 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.496 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.496 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:02:55.496 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.950996+0000", 2023-09-23T21:02:55.496 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.68356135900000004 2023-09-23T21:02:55.497 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.497 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.497 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T21:02:55.497 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.951005+0000", 2023-09-23T21:02:55.497 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.5660000000000007e-06 2023-09-23T21:02:55.497 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.497 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.497 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:02:55.497 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.257054+0000", 2023-09-23T21:02:55.497 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.306049228 2023-09-23T21:02:55.497 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.497 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.497 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for degraded object", 2023-09-23T21:02:55.498 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.257353+0000", 2023-09-23T21:02:55.498 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00029845000000000002 2023-09-23T21:02:55.498 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.498 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.498 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:02:55.498 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.278511+0000", 2023-09-23T21:02:55.498 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.021158713999999999 2023-09-23T21:02:55.498 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.498 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.499 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:02:55.499 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.278633+0000", 2023-09-23T21:02:55.499 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00012163999999999999 2023-09-23T21:02:55.499 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.499 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.499 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 1,5", 2023-09-23T21:02:55.499 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.279079+0000", 2023-09-23T21:02:55.499 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000445807 2023-09-23T21:02:55.499 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.499 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.499 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T21:02:55.499 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.286413+0000", 2023-09-23T21:02:55.500 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0073338509999999997 2023-09-23T21:02:55.500 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.500 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.500 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:02:55.500 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.321694+0000", 2023-09-23T21:02:55.500 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.035281333999999998 2023-09-23T21:02:55.500 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.500 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.500 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:02:55.500 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.327259+0000", 2023-09-23T21:02:55.500 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0055648479999999998 2023-09-23T21:02:55.500 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.501 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.501 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T21:02:55.501 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.327308+0000", 2023-09-23T21:02:55.501 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.9527999999999997e-05 2023-09-23T21:02:55.501 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.501 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.501 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T21:02:55.501 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.327542+0000", 2023-09-23T21:02:55.501 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00023375499999999999 2023-09-23T21:02:55.501 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:02:55.501 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:02:55.501 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:02:55.501 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.502 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.502 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34797.0:194611 6.10 6:0de287e4:::benchmark_data_smithi026_223868_object93485:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e180)", 2023-09-23T21:02:55.502 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T20:56:01.323196+0000", 2023-09-23T21:02:55.502 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 414.005005106, 2023-09-23T21:02:55.502 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.0047136079999999, 2023-09-23T21:02:55.502 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:02:55.502 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:02:55.502 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T21:02:55.502 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34797", 2023-09-23T21:02:55.502 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T21:02:55.502 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 194611 2023-09-23T21:02:55.502 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.502 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:02:55.503 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.503 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:02:55.503 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.323196+0000", 2023-09-23T21:02:55.503 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:02:55.503 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.503 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.503 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:02:55.503 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.323196+0000", 2023-09-23T21:02:55.503 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:02:55.503 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.503 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.503 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:02:55.503 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.323196+0000", 2023-09-23T21:02:55.504 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T21:02:55.504 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.504 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.504 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:02:55.504 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.323199+0000", 2023-09-23T21:02:55.504 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.4470000000000002e-06 2023-09-23T21:02:55.504 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.504 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.504 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:02:55.504 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.323209+0000", 2023-09-23T21:02:55.504 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.4700000000000008e-06 2023-09-23T21:02:55.504 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.504 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.505 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:02:55.505 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.323216+0000", 2023-09-23T21:02:55.505 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.3900000000000004e-06 2023-09-23T21:02:55.505 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.505 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.505 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:02:55.505 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.323262+0000", 2023-09-23T21:02:55.505 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.5785000000000003e-05 2023-09-23T21:02:55.505 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.505 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.505 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T21:02:55.505 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.323273+0000", 2023-09-23T21:02:55.505 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0888e-05 2023-09-23T21:02:55.506 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.506 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.506 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:02:55.506 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.951032+0000", 2023-09-23T21:02:55.506 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.62775908800000002 2023-09-23T21:02:55.506 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.506 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.506 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T21:02:55.506 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.951039+0000", 2023-09-23T21:02:55.506 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.3100000000000003e-06 2023-09-23T21:02:55.506 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.506 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.507 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:02:55.507 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.257402+0000", 2023-09-23T21:02:55.507 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.306362724 2023-09-23T21:02:55.507 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.507 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.507 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for degraded object", 2023-09-23T21:02:55.507 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.257769+0000", 2023-09-23T21:02:55.507 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00036691000000000002 2023-09-23T21:02:55.507 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.507 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.507 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:02:55.507 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.283087+0000", 2023-09-23T21:02:55.507 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.025317925000000002 2023-09-23T21:02:55.508 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.508 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.508 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:02:55.508 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.283213+0000", 2023-09-23T21:02:55.508 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000126773 2023-09-23T21:02:55.508 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.508 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.508 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 1,5", 2023-09-23T21:02:55.508 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.283761+0000", 2023-09-23T21:02:55.508 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00054742500000000004 2023-09-23T21:02:55.508 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.508 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.508 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T21:02:55.509 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.290057+0000", 2023-09-23T21:02:55.509 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0062964869999999999 2023-09-23T21:02:55.509 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.509 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.509 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:02:55.509 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.321745+0000", 2023-09-23T21:02:55.509 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.031687651999999997 2023-09-23T21:02:55.509 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.509 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.509 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:02:55.510 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.327648+0000", 2023-09-23T21:02:55.510 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0059028600000000002 2023-09-23T21:02:55.510 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.510 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.510 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T21:02:55.510 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.327694+0000", 2023-09-23T21:02:55.510 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.6081000000000001e-05 2023-09-23T21:02:55.510 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.510 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.510 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T21:02:55.510 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.327910+0000", 2023-09-23T21:02:55.510 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00021589499999999999 2023-09-23T21:02:55.511 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:02:55.511 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:02:55.511 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:02:55.511 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.511 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.511 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34797.0:194684 6.10 6:0c8ebf5c:::benchmark_data_smithi026_223868_object93558:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e180)", 2023-09-23T21:02:55.511 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T20:56:01.410674+0000", 2023-09-23T21:02:55.511 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 413.91752691400001, 2023-09-23T21:02:55.511 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.9175628630000001, 2023-09-23T21:02:55.511 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:02:55.511 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:02:55.511 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T21:02:55.511 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34797", 2023-09-23T21:02:55.512 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T21:02:55.512 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 194684 2023-09-23T21:02:55.512 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.512 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:02:55.512 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.512 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:02:55.512 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.410674+0000", 2023-09-23T21:02:55.512 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:02:55.512 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.512 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.512 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:02:55.512 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.410674+0000", 2023-09-23T21:02:55.512 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:02:55.513 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.513 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.513 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:02:55.513 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.410674+0000", 2023-09-23T21:02:55.513 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T21:02:55.513 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.513 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.513 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:02:55.513 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.410677+0000", 2023-09-23T21:02:55.513 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.9579999999999999e-06 2023-09-23T21:02:55.513 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.513 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.513 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:02:55.514 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.410692+0000", 2023-09-23T21:02:55.514 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.539e-05 2023-09-23T21:02:55.514 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.514 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.514 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:02:55.514 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.410698+0000", 2023-09-23T21:02:55.514 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.2830000000000001e-06 2023-09-23T21:02:55.514 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.514 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.514 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:02:55.515 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.410771+0000", 2023-09-23T21:02:55.515 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.2899999999999997e-05 2023-09-23T21:02:55.515 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.515 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.515 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T21:02:55.515 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.410778+0000", 2023-09-23T21:02:55.515 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.4200000000000001e-06 2023-09-23T21:02:55.515 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.515 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.515 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:02:55.515 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.951065+0000", 2023-09-23T21:02:55.516 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.54028672700000002 2023-09-23T21:02:55.516 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.516 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.516 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T21:02:55.516 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.951073+0000", 2023-09-23T21:02:55.516 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.551e-06 2023-09-23T21:02:55.516 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.516 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.516 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:02:55.516 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.257812+0000", 2023-09-23T21:02:55.516 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.3067387959999999 2023-09-23T21:02:55.516 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.516 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.517 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for degraded object", 2023-09-23T21:02:55.517 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.258109+0000", 2023-09-23T21:02:55.517 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00029674299999999997 2023-09-23T21:02:55.517 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.517 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.517 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:02:55.517 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.286926+0000", 2023-09-23T21:02:55.517 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.028817470000000001 2023-09-23T21:02:55.517 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.517 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.517 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:02:55.517 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.286999+0000", 2023-09-23T21:02:55.518 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.3141999999999999e-05 2023-09-23T21:02:55.518 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.518 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.518 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 1,5", 2023-09-23T21:02:55.518 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.287434+0000", 2023-09-23T21:02:55.518 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00043483299999999998 2023-09-23T21:02:55.518 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.518 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.518 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T21:02:55.518 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.294642+0000", 2023-09-23T21:02:55.518 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0072078960000000001 2023-09-23T21:02:55.518 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.518 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.519 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:02:55.519 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.321776+0000", 2023-09-23T21:02:55.519 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.027134098999999998 2023-09-23T21:02:55.519 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.519 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.519 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:02:55.519 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.327964+0000", 2023-09-23T21:02:55.519 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0061875960000000001 2023-09-23T21:02:55.519 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.519 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.520 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T21:02:55.520 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.328014+0000", 2023-09-23T21:02:55.520 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.007e-05 2023-09-23T21:02:55.520 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.520 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.520 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T21:02:55.520 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.328237+0000", 2023-09-23T21:02:55.520 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000223391 2023-09-23T21:02:55.520 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:02:55.521 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:02:55.521 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:02:55.521 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.521 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.521 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34797.0:194713 6.10 6:0a64bb83:::benchmark_data_smithi026_223868_object93587:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e180)", 2023-09-23T21:02:55.521 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T20:56:01.445241+0000", 2023-09-23T21:02:55.521 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 413.88296082099998, 2023-09-23T21:02:55.521 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.8833663199999999, 2023-09-23T21:02:55.521 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:02:55.521 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:02:55.521 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T21:02:55.521 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34797", 2023-09-23T21:02:55.521 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T21:02:55.522 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 194713 2023-09-23T21:02:55.522 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.522 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:02:55.522 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.522 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:02:55.522 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.445241+0000", 2023-09-23T21:02:55.522 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:02:55.522 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.522 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.522 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:02:55.522 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.445241+0000", 2023-09-23T21:02:55.522 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:02:55.523 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.523 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.523 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:02:55.523 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.445239+0000", 2023-09-23T21:02:55.523 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T21:02:55.523 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.523 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.523 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:02:55.523 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.445246+0000", 2023-09-23T21:02:55.523 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.6950000000000003e-06 2023-09-23T21:02:55.523 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.523 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.524 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:02:55.524 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.445262+0000", 2023-09-23T21:02:55.524 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.5654999999999998e-05 2023-09-23T21:02:55.524 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.524 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.524 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:02:55.524 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.445271+0000", 2023-09-23T21:02:55.524 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.679e-06 2023-09-23T21:02:55.524 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.524 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.524 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:02:55.524 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.445322+0000", 2023-09-23T21:02:55.525 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.0516000000000002e-05 2023-09-23T21:02:55.525 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.525 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.525 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T21:02:55.525 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.445332+0000", 2023-09-23T21:02:55.525 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.9699999999999994e-06 2023-09-23T21:02:55.525 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.525 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.525 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:02:55.525 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.951102+0000", 2023-09-23T21:02:55.525 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.50577048800000002 2023-09-23T21:02:55.525 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.525 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.526 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T21:02:55.526 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.951110+0000", 2023-09-23T21:02:55.526 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.208e-06 2023-09-23T21:02:55.526 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.526 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.526 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:02:55.526 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.258157+0000", 2023-09-23T21:02:55.526 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.3070464900000001 2023-09-23T21:02:55.526 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.526 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.526 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for degraded object", 2023-09-23T21:02:55.526 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.258517+0000", 2023-09-23T21:02:55.527 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00035973599999999999 2023-09-23T21:02:55.527 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.527 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.527 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:02:55.527 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.291101+0000", 2023-09-23T21:02:55.527 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.032584136999999999 2023-09-23T21:02:55.527 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.527 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.527 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:02:55.527 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.291239+0000", 2023-09-23T21:02:55.527 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000137761 2023-09-23T21:02:55.527 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.527 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.528 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 1,5", 2023-09-23T21:02:55.528 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.291789+0000", 2023-09-23T21:02:55.528 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00055038199999999998 2023-09-23T21:02:55.528 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.528 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.528 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T21:02:55.528 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.295524+0000", 2023-09-23T21:02:55.528 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0037345519999999999 2023-09-23T21:02:55.528 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.528 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.528 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:02:55.528 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.321831+0000", 2023-09-23T21:02:55.528 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.026307456 2023-09-23T21:02:55.529 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.529 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.529 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:02:55.529 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.328336+0000", 2023-09-23T21:02:55.529 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0065046009999999996 2023-09-23T21:02:55.529 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.529 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.529 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T21:02:55.529 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.328382+0000", 2023-09-23T21:02:55.529 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.617e-05 2023-09-23T21:02:55.529 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.529 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.529 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T21:02:55.530 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.328607+0000", 2023-09-23T21:02:55.530 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00022515100000000001 2023-09-23T21:02:55.530 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:02:55.530 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:02:55.530 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:02:55.530 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.530 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.530 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34797.0:194727 6.10 6:0b60e6f5:::benchmark_data_smithi026_223868_object93601:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e180)", 2023-09-23T21:02:55.530 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T20:56:01.462928+0000", 2023-09-23T21:02:55.530 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 413.86527309100001, 2023-09-23T21:02:55.531 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.866058437, 2023-09-23T21:02:55.531 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:02:55.531 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:02:55.531 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T21:02:55.531 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34797", 2023-09-23T21:02:55.531 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T21:02:55.531 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 194727 2023-09-23T21:02:55.531 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.531 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:02:55.531 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.531 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:02:55.531 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.462928+0000", 2023-09-23T21:02:55.531 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:02:55.532 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.532 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.532 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:02:55.532 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.462928+0000", 2023-09-23T21:02:55.532 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:02:55.532 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.532 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.532 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:02:55.532 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.462927+0000", 2023-09-23T21:02:55.532 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.999999 2023-09-23T21:02:55.532 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.532 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.532 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:02:55.533 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.462933+0000", 2023-09-23T21:02:55.533 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.3730000000000001e-06 2023-09-23T21:02:55.533 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.533 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.533 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:02:55.533 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.462946+0000", 2023-09-23T21:02:55.533 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.3747000000000001e-05 2023-09-23T21:02:55.533 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.533 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.533 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:02:55.533 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.462955+0000", 2023-09-23T21:02:55.533 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.0000000000000002e-06 2023-09-23T21:02:55.533 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.534 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.534 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:02:55.534 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.463029+0000", 2023-09-23T21:02:55.534 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.3559000000000005e-05 2023-09-23T21:02:55.534 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.534 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.534 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T21:02:55.534 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.463042+0000", 2023-09-23T21:02:55.534 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.2661e-05 2023-09-23T21:02:55.534 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.534 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.534 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:02:55.535 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.951135+0000", 2023-09-23T21:02:55.535 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.48809338699999999 2023-09-23T21:02:55.535 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.535 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.535 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T21:02:55.535 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.951143+0000", 2023-09-23T21:02:55.535 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.7319999999999998e-06 2023-09-23T21:02:55.535 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.535 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.535 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:02:55.535 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.258558+0000", 2023-09-23T21:02:55.535 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.307415357 2023-09-23T21:02:55.535 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.536 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.536 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for degraded object", 2023-09-23T21:02:55.536 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.258901+0000", 2023-09-23T21:02:55.536 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000342515 2023-09-23T21:02:55.536 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.536 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.536 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:02:55.536 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.297191+0000", 2023-09-23T21:02:55.536 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.038290826 2023-09-23T21:02:55.537 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.537 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.537 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:02:55.537 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.297316+0000", 2023-09-23T21:02:55.537 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00012508899999999999 2023-09-23T21:02:55.537 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.537 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.537 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 1,5", 2023-09-23T21:02:55.537 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.297920+0000", 2023-09-23T21:02:55.537 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00060379499999999996 2023-09-23T21:02:55.537 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.537 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.537 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T21:02:55.538 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.304770+0000", 2023-09-23T21:02:55.538 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0068497499999999999 2023-09-23T21:02:55.538 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.538 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.538 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:02:55.538 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.325484+0000", 2023-09-23T21:02:55.538 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.020714521999999999 2023-09-23T21:02:55.538 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.538 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.538 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:02:55.538 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.328690+0000", 2023-09-23T21:02:55.538 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.003205875 2023-09-23T21:02:55.539 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.539 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.539 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T21:02:55.539 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.328740+0000", 2023-09-23T21:02:55.539 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:02:55.540 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.9492e-05 2023-09-23T21:02:55.540 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.540 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.540 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T21:02:55.541 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.328987+0000", 2023-09-23T21:02:55.541 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00024687500000000002 2023-09-23T21:02:55.541 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:02:55.541 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:02:55.541 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:02:55.541 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.541 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.541 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34797.0:194750 6.10 6:091e1945:::benchmark_data_smithi026_223868_object93624:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e180)", 2023-09-23T21:02:55.541 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T20:56:01.496055+0000", 2023-09-23T21:02:55.541 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 413.83214636100001, 2023-09-23T21:02:55.541 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.8333552829999999, 2023-09-23T21:02:55.541 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:02:55.542 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:02:55.542 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T21:02:55.542 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34797", 2023-09-23T21:02:55.542 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T21:02:55.542 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 194750 2023-09-23T21:02:55.542 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.542 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:02:55.542 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.542 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:02:55.542 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.496055+0000", 2023-09-23T21:02:55.542 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:02:55.542 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.543 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.543 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:02:55.543 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.496055+0000", 2023-09-23T21:02:55.543 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:02:55.543 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.543 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.543 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:02:55.543 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.496054+0000", 2023-09-23T21:02:55.543 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T21:02:55.543 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.543 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.543 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:02:55.543 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.496061+0000", 2023-09-23T21:02:55.544 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.5009999999999998e-06 2023-09-23T21:02:55.544 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.544 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.544 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:02:55.544 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.496078+0000", 2023-09-23T21:02:55.544 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.6917000000000001e-05 2023-09-23T21:02:55.544 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.544 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.544 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:02:55.544 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.496085+0000", 2023-09-23T21:02:55.544 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.1729999999999998e-06 2023-09-23T21:02:55.544 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.544 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.545 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:02:55.545 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.496150+0000", 2023-09-23T21:02:55.545 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.5225e-05 2023-09-23T21:02:55.545 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.545 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.545 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T21:02:55.545 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.496163+0000", 2023-09-23T21:02:55.545 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.2187e-05 2023-09-23T21:02:55.545 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.545 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.545 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:02:55.545 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.951168+0000", 2023-09-23T21:02:55.545 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.455005037 2023-09-23T21:02:55.546 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.546 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.546 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T21:02:55.546 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.951176+0000", 2023-09-23T21:02:55.546 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.9750000000000007e-06 2023-09-23T21:02:55.546 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.546 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.546 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:02:55.546 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.258954+0000", 2023-09-23T21:02:55.546 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.3077780589999999 2023-09-23T21:02:55.547 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.547 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.547 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for degraded object", 2023-09-23T21:02:55.547 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.259370+0000", 2023-09-23T21:02:55.547 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000416632 2023-09-23T21:02:55.547 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.547 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.547 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:02:55.547 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.301427+0000", 2023-09-23T21:02:55.547 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.042056291000000003 2023-09-23T21:02:55.547 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.547 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.547 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:02:55.548 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.301568+0000", 2023-09-23T21:02:55.548 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00014118900000000001 2023-09-23T21:02:55.548 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.548 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.548 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 1,5", 2023-09-23T21:02:55.548 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.302113+0000", 2023-09-23T21:02:55.548 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00054519200000000001 2023-09-23T21:02:55.548 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.548 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.548 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T21:02:55.548 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.309443+0000", 2023-09-23T21:02:55.548 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0073298130000000001 2023-09-23T21:02:55.549 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.549 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.549 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:02:55.549 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.326494+0000", 2023-09-23T21:02:55.549 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.017051667999999999 2023-09-23T21:02:55.549 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.549 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.549 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:02:55.549 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.329098+0000", 2023-09-23T21:02:55.549 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0026040550000000001 2023-09-23T21:02:55.549 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.549 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.549 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T21:02:55.550 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.329155+0000", 2023-09-23T21:02:55.550 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.6276000000000001e-05 2023-09-23T21:02:55.550 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.550 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.550 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T21:02:55.550 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.329410+0000", 2023-09-23T21:02:55.550 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00025554400000000002 2023-09-23T21:02:55.550 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:02:55.550 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:02:55.550 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:02:55.550 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.551 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.551 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34797.0:194898 6.10 6:0ab6bac4:::benchmark_data_smithi026_223868_object93772:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e180)", 2023-09-23T21:02:55.551 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T20:56:01.737608+0000", 2023-09-23T21:02:55.551 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 413.59059364799998, 2023-09-23T21:02:55.551 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.605071608, 2023-09-23T21:02:55.551 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:02:55.551 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:02:55.551 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T21:02:55.551 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34797", 2023-09-23T21:02:55.551 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T21:02:55.551 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 194898 2023-09-23T21:02:55.552 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.552 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:02:55.552 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.552 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:02:55.552 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.737608+0000", 2023-09-23T21:02:55.552 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:02:55.552 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.552 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.552 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:02:55.552 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.737608+0000", 2023-09-23T21:02:55.552 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:02:55.552 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.552 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.553 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:02:55.553 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.737606+0000", 2023-09-23T21:02:55.553 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999981 2023-09-23T21:02:55.553 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.553 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.553 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:02:55.553 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.737613+0000", 2023-09-23T21:02:55.553 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.5390000000000002e-06 2023-09-23T21:02:55.553 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.553 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.553 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:02:55.553 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.737626+0000", 2023-09-23T21:02:55.553 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.2232000000000001e-05 2023-09-23T21:02:55.554 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.554 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.554 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:02:55.554 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.737634+0000", 2023-09-23T21:02:55.554 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.0960000000000001e-06 2023-09-23T21:02:55.554 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.554 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.554 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:02:55.554 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.737715+0000", 2023-09-23T21:02:55.554 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.1599000000000003e-05 2023-09-23T21:02:55.554 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.554 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.555 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T21:02:55.555 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.737730+0000", 2023-09-23T21:02:55.555 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.4363e-05 2023-09-23T21:02:55.555 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.555 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.555 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:02:55.555 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.951200+0000", 2023-09-23T21:02:55.555 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.21347070400000001 2023-09-23T21:02:55.555 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.555 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.555 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T21:02:55.555 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.951208+0000", 2023-09-23T21:02:55.555 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.5599999999999996e-06 2023-09-23T21:02:55.556 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.556 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.556 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:02:55.556 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.259413+0000", 2023-09-23T21:02:55.556 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.3082054949999999 2023-09-23T21:02:55.556 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.556 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.556 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for degraded object", 2023-09-23T21:02:55.556 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.259853+0000", 2023-09-23T21:02:55.556 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00043965399999999999 2023-09-23T21:02:55.556 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.556 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.556 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:02:55.557 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.305598+0000", 2023-09-23T21:02:55.557 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.045744765999999999 2023-09-23T21:02:55.557 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.557 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.557 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:02:55.557 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.305723+0000", 2023-09-23T21:02:55.557 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000124807 2023-09-23T21:02:55.557 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.557 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.557 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 1,5", 2023-09-23T21:02:55.557 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.306293+0000", 2023-09-23T21:02:55.557 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00057089399999999996 2023-09-23T21:02:55.558 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.558 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.558 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T21:02:55.558 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.317574+0000", 2023-09-23T21:02:55.558 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.011280434000000001 2023-09-23T21:02:55.558 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.558 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.558 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:02:55.558 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.327600+0000", 2023-09-23T21:02:55.558 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.010026037999999999 2023-09-23T21:02:55.558 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.558 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.558 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:02:55.559 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.342119+0000", 2023-09-23T21:02:55.559 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.014519139 2023-09-23T21:02:55.559 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.559 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.559 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T21:02:55.559 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.342171+0000", 2023-09-23T21:02:55.559 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.1925000000000002e-05 2023-09-23T21:02:55.559 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.559 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.559 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T21:02:55.559 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.342679+0000", 2023-09-23T21:02:55.559 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00050835799999999999 2023-09-23T21:02:55.560 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:02:55.560 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:02:55.560 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:02:55.560 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.560 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.560 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34797.0:194908 6.10 6:0f877dc5:::benchmark_data_smithi026_223868_object93782:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e180)", 2023-09-23T21:02:55.560 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T20:56:01.753586+0000", 2023-09-23T21:02:55.560 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 413.57461544199998, 2023-09-23T21:02:55.560 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.5894383169999999, 2023-09-23T21:02:55.560 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:02:55.560 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:02:55.560 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T21:02:55.560 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34797", 2023-09-23T21:02:55.561 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T21:02:55.561 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 194908 2023-09-23T21:02:55.561 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.561 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:02:55.561 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.561 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:02:55.561 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.753586+0000", 2023-09-23T21:02:55.561 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:02:55.561 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.561 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.562 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:02:55.562 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.753586+0000", 2023-09-23T21:02:55.562 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:02:55.562 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.562 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.562 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:02:55.562 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.753584+0000", 2023-09-23T21:02:55.562 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T21:02:55.562 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.562 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.562 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:02:55.562 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.753592+0000", 2023-09-23T21:02:55.563 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.5349999999999999e-06 2023-09-23T21:02:55.563 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.563 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.563 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:02:55.563 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.753609+0000", 2023-09-23T21:02:55.563 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.696e-05 2023-09-23T21:02:55.563 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.563 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.563 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:02:55.563 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.753617+0000", 2023-09-23T21:02:55.563 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.2009999999999999e-06 2023-09-23T21:02:55.563 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.563 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.564 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:02:55.564 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.753675+0000", 2023-09-23T21:02:55.564 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.7856999999999998e-05 2023-09-23T21:02:55.564 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.564 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.564 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T21:02:55.564 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.753690+0000", 2023-09-23T21:02:55.564 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.5528000000000001e-05 2023-09-23T21:02:55.564 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.564 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.564 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:02:55.564 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.951234+0000", 2023-09-23T21:02:55.564 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.19754403500000001 2023-09-23T21:02:55.565 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.565 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.565 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T21:02:55.565 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.951242+0000", 2023-09-23T21:02:55.565 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.9330000000000001e-06 2023-09-23T21:02:55.565 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.565 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.565 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:02:55.565 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.259898+0000", 2023-09-23T21:02:55.565 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.308655608 2023-09-23T21:02:55.565 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.565 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.566 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for degraded object", 2023-09-23T21:02:55.566 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.260281+0000", 2023-09-23T21:02:55.566 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00038326300000000001 2023-09-23T21:02:55.566 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.566 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.566 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:02:55.566 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.309484+0000", 2023-09-23T21:02:55.566 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.049202391999999998 2023-09-23T21:02:55.566 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.566 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.566 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:02:55.566 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.309599+0000", 2023-09-23T21:02:55.567 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000115642 2023-09-23T21:02:55.567 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.567 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.567 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 1,5", 2023-09-23T21:02:55.567 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.310127+0000", 2023-09-23T21:02:55.567 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00052754099999999997 2023-09-23T21:02:55.567 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.567 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.567 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T21:02:55.567 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.320913+0000", 2023-09-23T21:02:55.567 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.010786568999999999 2023-09-23T21:02:55.567 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.567 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.568 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:02:55.568 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.328297+0000", 2023-09-23T21:02:55.568 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0073830889999999998 2023-09-23T21:02:55.568 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.568 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.568 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:02:55.568 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.342727+0000", 2023-09-23T21:02:55.568 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.014430493000000001 2023-09-23T21:02:55.568 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.568 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.568 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T21:02:55.568 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.342775+0000", 2023-09-23T21:02:55.568 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.7593000000000003e-05 2023-09-23T21:02:55.569 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.569 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.569 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T21:02:55.569 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.343024+0000", 2023-09-23T21:02:55.569 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000249652 2023-09-23T21:02:55.569 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:02:55.569 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:02:55.569 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:02:55.569 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.569 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.569 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34797.0:194925 6.10 6:0af4fd48:::benchmark_data_smithi026_223868_object93799:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e180)", 2023-09-23T21:02:55.569 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T20:56:01.778450+0000", 2023-09-23T21:02:55.569 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 413.54975175599998, 2023-09-23T21:02:55.570 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.5649251790000001, 2023-09-23T21:02:55.570 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:02:55.570 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:02:55.570 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T21:02:55.570 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34797", 2023-09-23T21:02:55.570 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T21:02:55.570 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 194925 2023-09-23T21:02:55.570 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.570 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:02:55.570 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.570 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:02:55.570 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.778450+0000", 2023-09-23T21:02:55.571 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:02:55.571 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.571 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.571 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:02:55.571 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.778450+0000", 2023-09-23T21:02:55.571 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:02:55.571 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.571 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.571 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:02:55.571 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.778448+0000", 2023-09-23T21:02:55.572 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T21:02:55.572 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.572 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.572 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:02:55.572 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.778456+0000", 2023-09-23T21:02:55.572 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.5270000000000001e-06 2023-09-23T21:02:55.572 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.572 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.572 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:02:55.572 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.778484+0000", 2023-09-23T21:02:55.572 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.8629999999999999e-05 2023-09-23T21:02:55.573 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.573 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.573 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:02:55.573 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.778496+0000", 2023-09-23T21:02:55.573 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.203e-05 2023-09-23T21:02:55.573 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.573 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.573 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:02:55.573 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.778564+0000", 2023-09-23T21:02:55.573 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.8099e-05 2023-09-23T21:02:55.573 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.573 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.573 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T21:02:55.574 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.778579+0000", 2023-09-23T21:02:55.574 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.4397e-05 2023-09-23T21:02:55.574 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.574 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.574 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:02:55.574 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.951272+0000", 2023-09-23T21:02:55.574 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.172692926 2023-09-23T21:02:55.574 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.574 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.574 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T21:02:55.575 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.951281+0000", 2023-09-23T21:02:55.575 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.0180000000000007e-06 2023-09-23T21:02:55.575 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.575 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.575 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:02:55.575 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.260332+0000", 2023-09-23T21:02:55.575 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.3090507920000001 2023-09-23T21:02:55.575 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.575 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.575 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for degraded object", 2023-09-23T21:02:55.575 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.260659+0000", 2023-09-23T21:02:55.575 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000327466 2023-09-23T21:02:55.575 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.576 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.576 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:02:55.576 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.313663+0000", 2023-09-23T21:02:55.576 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.053004308 2023-09-23T21:02:55.576 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.576 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.576 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:02:55.576 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.313780+0000", 2023-09-23T21:02:55.576 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00011642999999999999 2023-09-23T21:02:55.576 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.576 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.576 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 1,5", 2023-09-23T21:02:55.576 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.314321+0000", 2023-09-23T21:02:55.577 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00054174099999999999 2023-09-23T21:02:55.577 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.577 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.577 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T21:02:55.577 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.320923+0000", 2023-09-23T21:02:55.577 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0066012759999999997 2023-09-23T21:02:55.577 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.577 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.577 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:02:55.577 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.328654+0000", 2023-09-23T21:02:55.577 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0077311619999999998 2023-09-23T21:02:55.577 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.577 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.578 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:02:55.578 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.343069+0000", 2023-09-23T21:02:55.578 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.014414797 2023-09-23T21:02:55.578 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.578 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.578 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T21:02:55.578 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.343121+0000", 2023-09-23T21:02:55.578 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.2475999999999997e-05 2023-09-23T21:02:55.578 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.578 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.578 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T21:02:55.578 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.343375+0000", 2023-09-23T21:02:55.578 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00025362900000000001 2023-09-23T21:02:55.579 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:02:55.579 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:02:55.579 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:02:55.579 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.579 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.579 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34797.0:194948 6.10 6:0b718fd8:::benchmark_data_smithi026_223868_object93822:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e180)", 2023-09-23T21:02:55.579 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T20:56:01.821311+0000", 2023-09-23T21:02:55.579 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 413.50688992200003, 2023-09-23T21:02:55.579 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.5226795950000001, 2023-09-23T21:02:55.579 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:02:55.579 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:02:55.581 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T21:02:55.581 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34797", 2023-09-23T21:02:55.581 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T21:02:55.581 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 194948 2023-09-23T21:02:55.581 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.581 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:02:55.581 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.582 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:02:55.582 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.821311+0000", 2023-09-23T21:02:55.582 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:02:55.582 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.582 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.582 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:02:55.582 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.821311+0000", 2023-09-23T21:02:55.582 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:02:55.582 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.583 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.583 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:02:55.583 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.821310+0000", 2023-09-23T21:02:55.583 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T21:02:55.583 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.583 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.583 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:02:55.583 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.821318+0000", 2023-09-23T21:02:55.583 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.7640000000000005e-06 2023-09-23T21:02:55.583 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.583 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.583 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:02:55.583 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.821335+0000", 2023-09-23T21:02:55.584 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.7572999999999999e-05 2023-09-23T21:02:55.584 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.584 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.584 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:02:55.584 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.821344+0000", 2023-09-23T21:02:55.584 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.4009999999999997e-06 2023-09-23T21:02:55.584 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.584 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.584 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:02:55.584 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.821399+0000", 2023-09-23T21:02:55.584 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.5770000000000003e-05 2023-09-23T21:02:55.584 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.584 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.585 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T21:02:55.585 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.821410+0000", 2023-09-23T21:02:55.585 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0789e-05 2023-09-23T21:02:55.585 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.585 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.585 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:02:55.585 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.951308+0000", 2023-09-23T21:02:55.585 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.12989732600000001 2023-09-23T21:02:55.585 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.585 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.585 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T21:02:55.585 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.951315+0000", 2023-09-23T21:02:55.585 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.9449999999999993e-06 2023-09-23T21:02:55.586 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.586 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.586 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:02:55.586 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.260707+0000", 2023-09-23T21:02:55.586 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.309391529 2023-09-23T21:02:55.586 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.586 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.586 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for degraded object", 2023-09-23T21:02:55.586 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.261067+0000", 2023-09-23T21:02:55.586 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00035980499999999999 2023-09-23T21:02:55.586 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.586 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.587 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:02:55.587 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.317608+0000", 2023-09-23T21:02:55.587 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.056540900999999998 2023-09-23T21:02:55.587 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.587 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.587 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:02:55.587 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.317709+0000", 2023-09-23T21:02:55.587 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000101371 2023-09-23T21:02:55.587 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.587 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.587 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 1,5", 2023-09-23T21:02:55.587 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.318249+0000", 2023-09-23T21:02:55.587 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00054033200000000003 2023-09-23T21:02:55.588 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.588 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.588 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T21:02:55.588 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.321587+0000", 2023-09-23T21:02:55.588 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.003337904 2023-09-23T21:02:55.588 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.588 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.588 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:02:55.588 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.329047+0000", 2023-09-23T21:02:55.588 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0074594570000000001 2023-09-23T21:02:55.588 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.588 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.588 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:02:55.589 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.343443+0000", 2023-09-23T21:02:55.589 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.014395984000000001 2023-09-23T21:02:55.589 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.589 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.589 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T21:02:55.589 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.343740+0000", 2023-09-23T21:02:55.589 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000296886 2023-09-23T21:02:55.589 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.589 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.589 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T21:02:55.589 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.343991+0000", 2023-09-23T21:02:55.589 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00025143799999999999 2023-09-23T21:02:55.590 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:02:55.590 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:02:55.590 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:02:55.590 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.590 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.590 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34797.0:195032 6.10 6:08dbcd4e:::benchmark_data_smithi026_223868_object93906:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e181)", 2023-09-23T21:02:55.590 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T20:56:02.127089+0000", 2023-09-23T21:02:55.590 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 413.20111236499997, 2023-09-23T21:02:55.590 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.224242249, 2023-09-23T21:02:55.590 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:02:55.590 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:02:55.591 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T21:02:55.591 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34797", 2023-09-23T21:02:55.591 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T21:02:55.591 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 195032 2023-09-23T21:02:55.591 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.591 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:02:55.591 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.591 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:02:55.591 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:02.127089+0000", 2023-09-23T21:02:55.591 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:02:55.591 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.591 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.592 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:02:55.592 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:02.127089+0000", 2023-09-23T21:02:55.592 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:02:55.592 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.592 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.592 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:02:55.592 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:02.127088+0000", 2023-09-23T21:02:55.592 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T21:02:55.592 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.592 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.593 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:02:55.593 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:02.127094+0000", 2023-09-23T21:02:55.593 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.2330000000000003e-06 2023-09-23T21:02:55.593 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.593 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.593 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:02:55.593 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:02.127104+0000", 2023-09-23T21:02:55.593 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0019e-05 2023-09-23T21:02:55.593 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.593 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.593 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:02:55.593 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:02.127109+0000", 2023-09-23T21:02:55.594 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.4960000000000004e-06 2023-09-23T21:02:55.594 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.594 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.594 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:02:55.594 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:02.127158+0000", 2023-09-23T21:02:55.594 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.8550000000000001e-05 2023-09-23T21:02:55.594 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.594 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.594 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T21:02:55.594 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:02.127168+0000", 2023-09-23T21:02:55.594 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.8009999999999998e-06 2023-09-23T21:02:55.594 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.594 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.595 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:02:55.595 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.261102+0000", 2023-09-23T21:02:55.595 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1339344730000001 2023-09-23T21:02:55.595 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.595 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.595 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for degraded object", 2023-09-23T21:02:55.595 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.261501+0000", 2023-09-23T21:02:55.595 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00039857100000000002 2023-09-23T21:02:55.595 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.595 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.595 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:02:55.595 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.322489+0000", 2023-09-23T21:02:55.595 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.060988570999999998 2023-09-23T21:02:55.596 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.596 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.596 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:02:55.596 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.322601+0000", 2023-09-23T21:02:55.596 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000111908 2023-09-23T21:02:55.596 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.596 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.596 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 1,5", 2023-09-23T21:02:55.596 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.323185+0000", 2023-09-23T21:02:55.596 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00058331099999999999 2023-09-23T21:02:55.596 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.596 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.597 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T21:02:55.597 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.326510+0000", 2023-09-23T21:02:55.597 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.003325578 2023-09-23T21:02:55.597 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.597 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.597 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:02:55.597 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.329495+0000", 2023-09-23T21:02:55.597 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0029846249999999999 2023-09-23T21:02:55.597 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.597 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.597 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:02:55.597 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.350780+0000", 2023-09-23T21:02:55.597 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.021285182999999999 2023-09-23T21:02:55.598 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.598 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.598 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T21:02:55.598 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.350846+0000", 2023-09-23T21:02:55.598 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.5630999999999996e-05 2023-09-23T21:02:55.598 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:55.598 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:55.598 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T21:02:55.598 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.351331+0000", 2023-09-23T21:02:55.598 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00048559200000000002 2023-09-23T21:02:55.598 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:02:55.598 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:02:55.598 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:02:55.599 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:02:55.599 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:02:55.599 INFO:teuthology.orchestra.run.smithi026.stdout:} 2023-09-23T21:02:55.599 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:02:55.647 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:02:55.692 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:02:55.697 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:02:55.697 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:02:55.785 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 84 16 106198 106182 78.9507 51.125 0.0147672 0.0126615 2023-09-23T21:02:55.799 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:02:55.837 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:02:55.901 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:02:55.907 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:02:55.907 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:02:56.008 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:02:56.060 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_ops_in_flight 2023-09-23T21:02:56.099 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:02:56.322 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:02:56 smithi026 ceph-mon[175151]: pgmap v728: 109 pgs: 109 active+clean; 6.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 50 MiB/s wr, 796 op/s 2023-09-23T21:02:56.322 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:02:56 smithi026 ceph-mon[179688]: pgmap v728: 109 pgs: 109 active+clean; 6.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 50 MiB/s wr, 796 op/s 2023-09-23T21:02:56.348 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T21:02:56.395 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:02:56.406 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:02:56.407 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:02:56.507 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:02:56.538 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:02:56 smithi070 ceph-mon[168282]: pgmap v728: 109 pgs: 109 active+clean; 6.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 50 MiB/s wr, 796 op/s 2023-09-23T21:02:56.541 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:02:56.587 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:02:56.595 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:02:56.595 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:02:56.696 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:02:56.735 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:02:56.786 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 85 16 106965 106949 78.586 47.9375 0.0311694 0.01272 2023-09-23T21:02:56.794 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:02:56.802 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:02:56.803 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:02:56.904 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:02:56.947 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:02:56.989 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:02:56.998 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:02:56.998 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:02:57.099 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:02:57.214 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:02:57.325 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:02:57.333 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:02:57.334 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:02:57.434 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:02:57.447 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:02:57.466 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:02:57.503 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:02:57.506 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:02:57.506 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:02:57.607 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:02:57.687 INFO:teuthology.orchestra.run.smithi026.stdout: 2023-09-23T21:02:57.701 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:02:57.728 INFO:teuthology.orchestra.run.smithi026.stderr:dumped all 2023-09-23T21:02:57.746 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:02:57.785 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 86 16 107668 107652 78.1833 43.9375 0.0144092 0.0127843 2023-09-23T21:02:57.832 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T21:02:57.845 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:02:57.845 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:02:57.946 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:02:57.979 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:02:58.024 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:02:58.027 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:02:58.027 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:02:58.128 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:02:58.169 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:02:58.205 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:02:58.208 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:02:58.208 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:02:58.309 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:02:58.338 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:02:58 smithi070 ceph-mon[168282]: pgmap v729: 109 pgs: 109 active+clean; 6.4 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 50 MiB/s wr, 796 op/s 2023-09-23T21:02:58.339 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:02:58.381 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:02:58 smithi026 ceph-mon[175151]: pgmap v729: 109 pgs: 109 active+clean; 6.4 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 50 MiB/s wr, 796 op/s 2023-09-23T21:02:58.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:02:58 smithi026 ceph-mon[179688]: pgmap v729: 109 pgs: 109 active+clean; 6.4 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 50 MiB/s wr, 796 op/s 2023-09-23T21:02:58.396 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:02:58.399 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:02:58.400 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:02:58.501 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:02:58.542 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:02:58.599 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:02:58.612 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:02:58.612 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:02:58.694 INFO:teuthology.orchestra.run.smithi026.stdout:{"pg_ready":true,"pg_map":{"version":729,"stamp":"2023-09-23T21:02:56.553631+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":6834624416,"num_objects":104303,"num_object_clones":0,"num_object_copies":312909,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":104303,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":104340,"num_write_kb":6673885,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":22,"num_bytes_recovered":5990,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":107615,"ondisk_log_size":107615,"up":327,"acting":327,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":330,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":22618392,"kb_used_data":20365464,"kb_used_omap":136,"kb_used_meta":2252727,"kb_avail":727342824,"statfs":{"total":767960285184,"available":744799051776,"internally_reserved":0,"allocated":20854235136,"data_stored":20845587746,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":139741,"internal_metadata":2306792995},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":9,"apply_latency_ms":9,"commit_latency_ns":9000000,"apply_latency_ns":9000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":522059776,"num_objects":7966,"num_object_clones":0,"num_object_copies":23898,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":7966,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7966,"num_write_kb":509824,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.003023"},"pg_stats":[{"pgid":"3.1a","version":"68'1","reported_seq":152,"reported_epoch":274,"state":"active+clean","last_fresh":"2023-09-23T21:02:43.270265+0000","last_change":"2023-09-23T20:54:13.946695+0000","last_active":"2023-09-23T21:02:43.270265+0000","last_peered":"2023-09-23T21:02:43.270265+0000","last_clean":"2023-09-23T21:02:43.270265+0000","last_became_active":"2023-09-23T20:54:13.945399+0000","last_became_peered":"2023-09-23T20:54:13.945399+0000","last_unstale":"2023-09-23T21:02:43.270265+0000","last_undegraded":"2023-09-23T21:02:43.270265+0000","last_fullsized":"2023-09-23T21:02:43.270265+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:32:52.365464+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":268,"reported_epoch":274,"state":"active+clean","last_fresh":"2023-09-23T21:02:43.285471+0000","last_change":"2023-09-23T20:54:13.942019+0000","last_active":"2023-09-23T21:02:43.285471+0000","last_peered":"2023-09-23T21:02:43.285471+0000","last_clean":"2023-09-23T21:02:43.285471+0000","last_became_active":"2023-09-23T20:54:13.941647+0000","last_became_peered":"2023-09-23T20:54:13.941647+0000","last_unstale":"2023-09-23T21:02:43.285471+0000","last_undegraded":"2023-09-23T21:02:43.285471+0000","last_fullsized":"2023-09-23T21:02:43.285471+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:30:54.400944+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":246,"reported_epoch":274,"state":"active+clean","last_fresh":"2023-09-23T21:02:43.283506+0000","last_change":"2023-09-23T20:54:13.941394+0000","last_active":"2023-09-23T21:02:43.283506+0000","last_peered":"2023-09-23T21:02:43.283506+0000","last_clean":"2023-09-23T21:02:43.283506+0000","last_became_active":"2023-09-23T20:54:13.941042+0000","last_became_peered":"2023-09-23T20:54:13.941042+0000","last_unstale":"2023-09-23T21:02:43.283506+0000","last_undegraded":"2023-09-23T21:02:43.283506+0000","last_fullsized":"2023-09-23T21:02:43.283506+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:19:41.401184+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":261,"reported_epoch":274,"state":"active+clean","last_fresh":"2023-09-23T21:02:43.276348+0000","last_change":"2023-09-23T20:57:44.979811+0000","last_active":"2023-09-23T21:02:43.276348+0000","last_peered":"2023-09-23T21:02:43.276348+0000","last_clean":"2023-09-23T21:02:43.276348+0000","last_became_active":"2023-09-23T20:57:44.979515+0000","last_became_peered":"2023-09-23T20:57:44.979515+0000","last_unstale":"2023-09-23T21:02:43.276348+0000","last_undegraded":"2023-09-23T21:02:43.276348+0000","last_fullsized":"2023-09-23T21:02:43.276348+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:38:22.162140+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":259,"reported_epoch":274,"state":"active+clean","last_fresh":"2023-09-23T21:02:43.287258+0000","last_change":"2023-09-23T20:51:10.833613+0000","last_active":"2023-09-23T21:02:43.287258+0000","last_peered":"2023-09-23T21:02:43.287258+0000","last_clean":"2023-09-23T21:02:43.287258+0000","last_became_active":"2023-09-23T20:51:10.833208+0000","last_became_peered":"2023-09-23T20:51:10.833208+0000","last_unstale":"2023-09-23T21:02:43.287258+0000","last_undegraded":"2023-09-23T21:02:43.287258+0000","last_fullsized":"2023-09-23T21:02:43.287258+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:51:28.246014+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":240,"reported_epoch":274,"state":"active+clean","last_fresh":"2023-09-23T21:02:43.273838+0000","last_change":"2023-09-23T20:55:35.958384+0000","last_active":"2023-09-23T21:02:43.273838+0000","last_peered":"2023-09-23T21:02:43.273838+0000","last_clean":"2023-09-23T21:02:43.273838+0000","last_became_active":"2023-09-23T20:55:35.954563+0000","last_became_peered":"2023-09-23T20:55:35.954563+0000","last_unstale":"2023-09-23T21:02:43.273838+0000","last_undegraded":"2023-09-23T21:02:43.273838+0000","last_fullsized":"2023-09-23T21:02:43.273838+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:37:48.332259+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":251,"reported_epoch":274,"state":"active+clean","last_fresh":"2023-09-23T21:02:43.276458+0000","last_change":"2023-09-23T20:54:31.885514+0000","last_active":"2023-09-23T21:02:43.276458+0000","last_peered":"2023-09-23T21:02:43.276458+0000","last_clean":"2023-09-23T21:02:43.276458+0000","last_became_active":"2023-09-23T20:54:31.884986+0000","last_became_peered":"2023-09-23T20:54:31.884986+0000","last_unstale":"2023-09-23T21:02:43.276458+0000","last_undegraded":"2023-09-23T21:02:43.276458+0000","last_fullsized":"2023-09-23T21:02:43.276458+0000","mapping_epoch":136,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:26:08.100220+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":257,"reported_epoch":274,"state":"active+clean","last_fresh":"2023-09-23T21:02:49.864770+0000","last_change":"2023-09-23T20:55:35.951889+0000","last_active":"2023-09-23T21:02:49.864770+0000","last_peered":"2023-09-23T21:02:49.864770+0000","last_clean":"2023-09-23T21:02:49.864770+0000","last_became_active":"2023-09-23T20:55:35.951563+0000","last_became_peered":"2023-09-23T20:55:35.951563+0000","last_unstale":"2023-09-23T21:02:49.864770+0000","last_undegraded":"2023-09-23T21:02:49.864770+0000","last_fullsized":"2023-09-23T21:02:49.864770+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:41:53.098781+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"79'161","reported_seq":563,"reported_epoch":274,"state":"active+clean","last_fresh":"2023-09-23T21:02:43.263368+0000","last_change":"2023-09-23T20:57:25.096559+0000","last_active":"2023-09-23T21:02:43.263368+0000","last_peered":"2023-09-23T21:02:43.263368+0000","last_clean":"2023-09-23T21:02:43.263368+0000","last_became_active":"2023-09-23T20:57:15.975674+0000","last_became_peered":"2023-09-23T20:57:15.975674+0000","last_unstale":"2023-09-23T21:02:43.263368+0000","last_undegraded":"2023-09-23T21:02:43.263368+0000","last_fullsized":"2023-09-23T21:02:43.263368+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:25.096401+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:25.096401+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:05:05.448404+0000","scrub_duration":0.002859393,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":148,"reported_epoch":274,"state":"active+clean","last_fresh":"2023-09-23T21:02:43.275732+0000","last_change":"2023-09-23T20:54:31.890678+0000","last_active":"2023-09-23T21:02:43.275732+0000","last_peered":"2023-09-23T21:02:43.275732+0000","last_clean":"2023-09-23T21:02:43.275732+0000","last_became_active":"2023-09-23T20:54:31.890469+0000","last_became_peered":"2023-09-23T20:54:31.890469+0000","last_unstale":"2023-09-23T21:02:43.275732+0000","last_undegraded":"2023-09-23T21:02:43.275732+0000","last_fullsized":"2023-09-23T21:02:43.275732+0000","mapping_epoch":136,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:55:08.790146+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":218,"reported_epoch":274,"state":"active+clean","last_fresh":"2023-09-23T21:02:49.864819+0000","last_change":"2023-09-23T20:55:35.952661+0000","last_active":"2023-09-23T21:02:49.864819+0000","last_peered":"2023-09-23T21:02:49.864819+0000","last_clean":"2023-09-23T21:02:49.864819+0000","last_became_active":"2023-09-23T20:55:35.952349+0000","last_became_peered":"2023-09-23T20:55:35.952349+0000","last_unstale":"2023-09-23T21:02:49.864819+0000","last_undegraded":"2023-09-23T21:02:49.864819+0000","last_fullsized":"2023-09-23T21:02:49.864819+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:00:03.110126+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":94,"reported_epoch":274,"state":"active+clean","last_fresh":"2023-09-23T21:02:43.285608+0000","last_change":"2023-09-23T20:56:45.988607+0000","last_active":"2023-09-23T21:02:43.285608+0000","last_peered":"2023-09-23T21:02:43.285608+0000","last_clean":"2023-09-23T21:02:43.285608+0000","last_became_active":"2023-09-23T20:56:45.988206+0000","last_became_peered":"2023-09-23T20:56:45.988206+0000","last_unstale":"2023-09-23T21:02:43.285608+0000","last_undegraded":"2023-09-23T21:02:43.285608+0000","last_fullsized":"2023-09-23T21:02:43.285608+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":205,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:01:19.783688+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,4],"acting":[3,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":92,"reported_epoch":274,"state":"active+clean","last_fresh":"2023-09-23T21:02:43.275960+0000","last_change":"2023-09-23T20:56:18.139142+0000","last_active":"2023-09-23T21:02:43.275960+0000","last_peered":"2023-09-23T21:02:43.275960+0000","last_clean":"2023-09-23T21:02:43.275960+0000","last_became_active":"2023-09-23T20:56:18.138844+0000","last_became_peered":"2023-09-23T20:56:18.138844+0000","last_unstale":"2023-09-23T21:02:43.275960+0000","last_undegraded":"2023-09-23T21:02:43.275960+0000","last_fullsized":"2023-09-23T21:02:43.275960+0000","mapping_epoch":192,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":193,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:09:02.217683+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,2],"acting":[5,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.14","version":"79'22","reported_seq":179,"reported_epoch":274,"state":"active+clean","last_fresh":"2023-09-23T21:02:43.273152+0000","last_change":"2023-09-23T20:55:35.949418+0000","last_active":"2023-09-23T21:02:43.273152+0000","last_peered":"2023-09-23T21:02:43.273152+0000","last_clean":"2023-09-23T21:02:43.273152+0000","last_became_active":"2023-09-23T20:55:35.948566+0000","last_became_peered":"2023-09-23T20:55:35.948566+0000","last_unstale":"2023-09-23T21:02:43.273152+0000","last_undegraded":"2023-09-23T21:02:43.273152+0000","last_fullsized":"2023-09-23T21:02:43.273152+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:59:37.433128+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4734,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":169,"reported_epoch":274,"state":"active+clean","last_fresh":"2023-09-23T21:02:43.270038+0000","last_change":"2023-09-23T20:55:35.949178+0000","last_active":"2023-09-23T21:02:43.270038+0000","last_peered":"2023-09-23T21:02:43.270038+0000","last_clean":"2023-09-23T21:02:43.270038+0000","last_became_active":"2023-09-23T20:55:35.948795+0000","last_became_peered":"2023-09-23T20:55:35.948795+0000","last_unstale":"2023-09-23T21:02:43.270038+0000","last_undegraded":"2023-09-23T21:02:43.270038+0000","last_fullsized":"2023-09-23T21:02:43.270038+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:13:08.996932+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.11","version":"79'161","reported_seq":505,"reported_epoch":274,"state":"active+clean","last_fresh":"2023-09-23T21:02:43.275756+0000","last_change":"2023-09-23T20:57:28.756796+0000","last_active":"2023-09-23T21:02:43.275756+0000","last_peered":"2023-09-23T21:02:43.275756+0000","last_clean":"2023-09-23T21:02:43.275756+0000","last_became_active":"2023-09-23T20:57:15.980606+0000","last_became_peered":"2023-09-23T20:57:15.980606+0000","last_unstale":"2023-09-23T21:02:43.275756+0000","last_undegraded":"2023-09-23T21:02:43.275756+0000","last_fullsized":"2023-09-23T21:02:43.275756+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:28.756634+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:28.756634+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:09:19.150791+0000","scrub_duration":0.0064925119999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":195,"reported_epoch":274,"state":"active+clean","last_fresh":"2023-09-23T21:02:43.269912+0000","last_change":"2023-09-23T20:56:45.992577+0000","last_active":"2023-09-23T21:02:43.269912+0000","last_peered":"2023-09-23T21:02:43.269912+0000","last_clean":"2023-09-23T21:02:43.269912+0000","last_became_active":"2023-09-23T20:56:45.992297+0000","last_became_peered":"2023-09-23T20:56:45.992297+0000","last_unstale":"2023-09-23T21:02:43.269912+0000","last_undegraded":"2023-09-23T21:02:43.269912+0000","last_fullsized":"2023-09-23T21:02:43.269912+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":205,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:00:45.547202+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,2],"acting":[4,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":242,"reported_epoch":274,"state":"active+clean","last_fresh":"2023-09-23T21:02:49.864894+0000","last_change":"2023-09-23T20:51:10.831262+0000","last_active":"2023-09-23T21:02:49.864894+0000","last_peered":"2023-09-23T21:02:49.864894+0000","last_clean":"2023-09-23T21:02:49.864894+0000","last_became_active":"2023-09-23T20:51:10.830931+0000","last_became_peered":"2023-09-23T20:51:10.830931+0000","last_unstale":"2023-09-23T21:02:49.864894+0000","last_undegraded":"2023-09-23T21:02:49.864894+0000","last_fullsized":"2023-09-23T21:02:49.864894+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:51:54.982505+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.10","version":"79'161","reported_seq":512,"reported_epoch":274,"state":"active+clean","last_fresh":"2023-09-23T21:02:43.275070+0000","last_change":"2023-09-23T20:57:20.708003+0000","last_active":"2023-09-23T21:02:43.275070+0000","last_peered":"2023-09-23T21:02:43.275070+0000","last_clean":"2023-09-23T21:02:43.275070+0000","last_became_active":"2023-09-23T20:57:15.975302+0000","last_became_peered":"2023-09-23T20:57:15.975302+0000","last_unstale":"2023-09-23T21:02:43.275070+0000","last_undegraded":"2023-09-23T21:02:43.275070+0000","last_fullsized":"2023-09-23T21:02:43.275070+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:20.707939+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:20.707939+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:16:51.721994+0000","scrub_duration":0.0017733409999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":251,"reported_epoch":274,"state":"active+clean","last_fresh":"2023-09-23T21:02:43.276375+0000","last_change":"2023-09-23T20:57:44.979736+0000","last_active":"2023-09-23T21:02:43.276375+0000","last_peered":"2023-09-23T21:02:43.276375+0000","last_clean":"2023-09-23T21:02:43.276375+0000","last_became_active":"2023-09-23T20:57:44.979356+0000","last_became_peered":"2023-09-23T20:57:44.979356+0000","last_unstale":"2023-09-23T21:02:43.276375+0000","last_undegraded":"2023-09-23T21:02:43.276375+0000","last_fullsized":"2023-09-23T21:02:43.276375+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:14:43.848832+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":148,"reported_epoch":274,"state":"active+clean","last_fresh":"2023-09-23T21:02:43.276862+0000","last_change":"2023-09-23T20:54:31.887814+0000","last_active":"2023-09-23T21:02:43.276862+0000","last_peered":"2023-09-23T21:02:43.276862+0000","last_clean":"2023-09-23T21:02:43.276862+0000","last_became_active":"2023-09-23T20:54:31.887195+0000","last_became_peered":"2023-09-23T20:54:31.887195+0000","last_unstale":"2023-09-23T21:02:43.276862+0000","last_undegraded":"2023-09-23T21:02:43.276862+0000","last_fullsized":"2023-09-23T21:02:43.276862+0000","mapping_epoch":136,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:10:52.089553+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.13","version":"79'161","reported_seq":576,"reported_epoch":274,"state":"active+clean","last_fresh":"2023-09-23T21:02:43.281753+0000","last_change":"2023-09-23T20:57:48.262970+0000","last_active":"2023-09-23T21:02:43.281753+0000","last_peered":"2023-09-23T21:02:43.281753+0000","last_clean":"2023-09-23T21:02:43.281753+0000","last_became_active":"2023-09-23T20:57:48.262663+0000","last_became_peered":"2023-09-23T20:57:48.262663+0000","last_unstale":"2023-09-23T21:02:43.281753+0000","last_undegraded":"2023-09-23T21:02:43.281753+0000","last_fullsized":"2023-09-23T21:02:43.281753+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:36.420362+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:36.420362+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:28:28.613492+0000","scrub_duration":0.028838444000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":219,"reported_epoch":274,"state":"active+clean","last_fresh":"2023-09-23T21:02:43.269716+0000","last_change":"2023-09-23T20:56:45.991822+0000","last_active":"2023-09-23T21:02:43.269716+0000","last_peered":"2023-09-23T21:02:43.269716+0000","last_clean":"2023-09-23T21:02:43.269716+0000","last_became_active":"2023-09-23T20:56:45.991460+0000","last_became_peered":"2023-09-23T20:56:45.991460+0000","last_unstale":"2023-09-23T21:02:43.269716+0000","last_undegraded":"2023-09-23T21:02:43.269716+0000","last_fullsized":"2023-09-23T21:02:43.269716+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":205,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:20:37.503317+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":233,"reported_epoch":274,"state":"active+clean","last_fresh":"2023-09-23T21:02:43.276497+0000","last_change":"2023-09-23T20:51:10.840849+0000","last_active":"2023-09-23T21:02:43.276497+0000","last_peered":"2023-09-23T21:02:43.276497+0000","last_clean":"2023-09-23T21:02:43.276497+0000","last_became_active":"2023-09-23T20:51:10.840593+0000","last_became_peered":"2023-09-23T20:51:10.840593+0000","last_unstale":"2023-09-23T21:02:43.276497+0000","last_undegraded":"2023-09-23T21:02:43.276497+0000","last_fullsized":"2023-09-23T21:02:43.276497+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:30:52.297625+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.12","version":"79'161","reported_seq":512,"reported_epoch":274,"state":"active+clean","last_fresh":"2023-09-23T21:02:43.275842+0000","last_change":"2023-09-23T20:57:21.750765+0000","last_active":"2023-09-23T21:02:43.275842+0000","last_peered":"2023-09-23T21:02:43.275842+0000","last_clean":"2023-09-23T21:02:43.275842+0000","last_became_active":"2023-09-23T20:57:15.979195+0000","last_became_peered":"2023-09-23T20:57:15.979195+0000","last_unstale":"2023-09-23T21:02:43.275842+0000","last_undegraded":"2023-09-23T21:02:43.275842+0000","last_fullsized":"2023-09-23T21:02:43.275842+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:21.750680+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:21.750680+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:03:41.915962+0000","scrub_duration":0.0092391680000000007,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":267,"reported_epoch":274,"state":"active+clean","last_fresh":"2023-09-23T21:02:43.273017+0000","last_change":"2023-09-23T20:55:36.943969+0000","last_active":"2023-09-23T21:02:43.273017+0000","last_peered":"2023-09-23T21:02:43.273017+0000","last_clean":"2023-09-23T21:02:43.273017+0000","last_became_active":"2023-09-23T20:55:36.943580+0000","last_became_peered":"2023-09-23T20:55:36.943580+0000","last_unstale":"2023-09-23T21:02:43.273017+0000","last_undegraded":"2023-09-23T21:02:43.273017+0000","last_fullsized":"2023-09-23T21:02:43.273017+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:49:06.969396+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":65,"reported_epoch":274,"state":"active+clean","last_fresh":"2023-09-23T21:02:43.287985+0000","last_change":"2023-09-23T20:57:44.980498+0000","last_active":"2023-09-23T21:02:43.287985+0000","last_peered":"2023-09-23T21:02:43.287985+0000","last_clean":"2023-09-23T21:02:43.287985+0000","last_became_active":"2023-09-23T20:57:44.980182+0000","last_became_peered":"2023-09-23T20:57:44.980182+0000","last_unstale":"2023-09-23T21:02:43.287985+0000","last_undegraded":"2023-09-23T21:02:43.287985+0000","last_fullsized":"2023-09-23T21:02:43.287985+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:34:51.620544+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.4","version":"275'6473","reported_seq":6502,"reported_epoch":275,"state":"active+clean","last_fresh":"2023-09-23T21:02:52.016521+0000","last_change":"2023-09-23T21:01:25.429865+0000","last_active":"2023-09-23T21:02:52.016521+0000","last_peered":"2023-09-23T21:02:52.016521+0000","last_clean":"2023-09-23T21:02:52.016521+0000","last_became_active":"2023-09-23T21:01:25.429555+0000","last_became_peered":"2023-09-23T21:01:25.429555+0000","last_unstale":"2023-09-23T21:02:52.016521+0000","last_undegraded":"2023-09-23T21:02:52.016521+0000","last_fullsized":"2023-09-23T21:02:52.016521+0000","mapping_epoch":254,"log_start":"0'0","ondisk_log_start":"0'0","created":254,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:01:24.325318+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:01:24.325318+0000","last_clean_scrub_stamp":"2023-09-23T21:01:24.325318+0000","objects_scrubbed":0,"log_size":6473,"log_dups_size":0,"ondisk_log_size":6473,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:28:38.557662+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":424214528,"num_objects":6473,"num_object_clones":0,"num_object_copies":19419,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6473,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6473,"num_write_kb":414272,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,2],"acting":[5,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.d","version":"79'161","reported_seq":557,"reported_epoch":274,"state":"active+clean","last_fresh":"2023-09-23T21:02:43.273315+0000","last_change":"2023-09-23T20:57:31.075213+0000","last_active":"2023-09-23T21:02:43.273315+0000","last_peered":"2023-09-23T21:02:43.273315+0000","last_clean":"2023-09-23T21:02:43.273315+0000","last_became_active":"2023-09-23T20:57:15.988188+0000","last_became_peered":"2023-09-23T20:57:15.988188+0000","last_unstale":"2023-09-23T21:02:43.273315+0000","last_undegraded":"2023-09-23T21:02:43.273315+0000","last_fullsized":"2023-09-23T21:02:43.273315+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:31.075098+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:31.075098+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:41:52.142362+0000","scrub_duration":0.010304610000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":206,"reported_epoch":274,"state":"active+clean","last_fresh":"2023-09-23T21:02:43.273599+0000","last_change":"2023-09-23T20:56:45.991841+0000","last_active":"2023-09-23T21:02:43.273599+0000","last_peered":"2023-09-23T21:02:43.273599+0000","last_clean":"2023-09-23T21:02:43.273599+0000","last_became_active":"2023-09-23T20:56:45.991491+0000","last_became_peered":"2023-09-23T20:56:45.991491+0000","last_unstale":"2023-09-23T21:02:43.273599+0000","last_undegraded":"2023-09-23T21:02:43.273599+0000","last_fullsized":"2023-09-23T21:02:43.273599+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":205,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:30:37.405319+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":165,"reported_epoch":274,"state":"active+clean","last_fresh":"2023-09-23T21:02:43.275170+0000","last_change":"2023-09-23T20:55:35.957768+0000","last_active":"2023-09-23T21:02:43.275170+0000","last_peered":"2023-09-23T21:02:43.275170+0000","last_clean":"2023-09-23T21:02:43.275170+0000","last_became_active":"2023-09-23T20:55:35.957421+0000","last_became_peered":"2023-09-23T20:55:35.957421+0000","last_unstale":"2023-09-23T21:02:43.275170+0000","last_undegraded":"2023-09-23T21:02:43.275170+0000","last_fullsized":"2023-09-23T21:02:43.275170+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:49:41.585961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"8.5","version":"275'6609","reported_seq":6638,"reported_epoch":275,"state":"active+clean","last_fresh":"2023-09-23T21:02:54.082334+0000","last_change":"2023-09-23T21:01:25.560708+0000","last_active":"2023-09-23T21:02:54.082334+0000","last_peered":"2023-09-23T21:02:54.082334+0000","last_clean":"2023-09-23T21:02:54.082334+0000","last_became_active":"2023-09-23T21:01:25.560472+0000","last_became_peered":"2023-09-23T21:01:25.560472+0000","last_unstale":"2023-09-23T21:02:54.082334+0000","last_undegraded":"2023-09-23T21:02:54.082334+0000","last_fullsized":"2023-09-23T21:02:54.082334+0000","mapping_epoch":254,"log_start":"0'0","ondisk_log_start":"0'0","created":254,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:01:24.325318+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:01:24.325318+0000","last_clean_scrub_stamp":"2023-09-23T21:01:24.325318+0000","objects_scrubbed":0,"log_size":6609,"log_dups_size":0,"ondisk_log_size":6609,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:04:02.549068+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":433127424,"num_objects":6609,"num_object_clones":0,"num_object_copies":19827,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6609,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6609,"num_write_kb":422976,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.c","version":"79'161","reported_seq":577,"reported_epoch":274,"state":"active+clean","last_fresh":"2023-09-23T21:02:43.285691+0000","last_change":"2023-09-23T20:57:22.041071+0000","last_active":"2023-09-23T21:02:43.285691+0000","last_peered":"2023-09-23T21:02:43.285691+0000","last_clean":"2023-09-23T21:02:43.285691+0000","last_became_active":"2023-09-23T20:57:15.973204+0000","last_became_peered":"2023-09-23T20:57:15.973204+0000","last_unstale":"2023-09-23T21:02:43.285691+0000","last_undegraded":"2023-09-23T21:02:43.285691+0000","last_fullsized":"2023-09-23T21:02:43.285691+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:22.040907+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:22.040907+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:47:02.019978+0000","scrub_duration":0.013021176000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":193,"reported_epoch":274,"state":"active+clean","last_fresh":"2023-09-23T21:02:43.273887+0000","last_change":"2023-09-23T20:56:45.991821+0000","last_active":"2023-09-23T21:02:43.273887+0000","last_peered":"2023-09-23T21:02:43.273887+0000","last_clean":"2023-09-23T21:02:43.273887+0000","last_became_active":"2023-09-23T20:56:45.991447+0000","last_became_peered":"2023-09-23T20:56:45.991447+0000","last_unstale":"2023-09-23T21:02:43.273887+0000","last_undegraded":"2023-09-23T21:02:43.273887+0000","last_fullsized":"2023-09-23T21:02:43.273887+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":205,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:14:25.525222+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":196,"reported_epoch":274,"state":"active+clean","last_fresh":"2023-09-23T21:02:43.269992+0000","last_change":"2023-09-23T20:57:44.982059+0000","last_active":"2023-09-23T21:02:43.269992+0000","last_peered":"2023-09-23T21:02:43.269992+0000","last_clean":"2023-09-23T21:02:43.269992+0000","last_became_active":"2023-09-23T20:57:44.981742+0000","last_became_peered":"2023-09-23T20:57:44.981742+0000","last_unstale":"2023-09-23T21:02:43.269992+0000","last_undegraded":"2023-09-23T21:02:43.269992+0000","last_fullsized":"2023-09-23T21:02:43.269992+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:41:24.042916+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.6","version":"275'6607","reported_seq":6637,"reported_epoch":275,"state":"active+clean","last_fresh":"2023-09-23T21:02:52.023643+0000","last_change":"2023-09-23T21:01:25.560370+0000","last_active":"2023-09-23T21:02:52.023643+0000","last_peered":"2023-09-23T21:02:52.023643+0000","last_clean":"2023-09-23T21:02:52.023643+0000","last_became_active":"2023-09-23T21:01:25.560018+0000","last_became_peered":"2023-09-23T21:01:25.560018+0000","last_unstale":"2023-09-23T21:02:52.023643+0000","last_undegraded":"2023-09-23T21:02:52.023643+0000","last_fullsized":"2023-09-23T21:02:52.023643+0000","mapping_epoch":254,"log_start":"0'0","ondisk_log_start":"0'0","created":254,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:01:24.325318+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:01:24.325318+0000","last_clean_scrub_stamp":"2023-09-23T21:01:24.325318+0000","objects_scrubbed":0,"log_size":6607,"log_dups_size":0,"ondisk_log_size":6607,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:45:39.782872+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":432996352,"num_objects":6607,"num_object_clones":0,"num_object_copies":19821,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6607,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6607,"num_write_kb":422848,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,7],"acting":[0,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.f","version":"79'161","reported_seq":579,"reported_epoch":274,"state":"active+clean","last_fresh":"2023-09-23T21:02:43.273791+0000","last_change":"2023-09-23T20:57:27.880841+0000","last_active":"2023-09-23T21:02:43.273791+0000","last_peered":"2023-09-23T21:02:43.273791+0000","last_clean":"2023-09-23T21:02:43.273791+0000","last_became_active":"2023-09-23T20:57:15.970925+0000","last_became_peered":"2023-09-23T20:57:15.970925+0000","last_unstale":"2023-09-23T21:02:43.273791+0000","last_undegraded":"2023-09-23T21:02:43.273791+0000","last_fullsized":"2023-09-23T21:02:43.273791+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:27.880776+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:27.880776+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:15:13.438014+0000","scrub_duration":0.0014004899999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":179,"reported_epoch":274,"state":"active+clean","last_fresh":"2023-09-23T21:02:43.276781+0000","last_change":"2023-09-23T20:56:45.989153+0000","last_active":"2023-09-23T21:02:43.276781+0000","last_peered":"2023-09-23T21:02:43.276781+0000","last_clean":"2023-09-23T21:02:43.276781+0000","last_became_active":"2023-09-23T20:56:45.988800+0000","last_became_peered":"2023-09-23T20:56:45.988800+0000","last_unstale":"2023-09-23T21:02:43.276781+0000","last_undegraded":"2023-09-23T21:02:43.276781+0000","last_fullsized":"2023-09-23T21:02:43.276781+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":205,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:04:41.130194+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":214,"reported_epoch":274,"state":"active+clean","last_fresh":"2023-09-23T21:02:43.273173+0000","last_change":"2023-09-23T20:55:35.965427+0000","last_active":"2023-09-23T21:02:43.273173+0000","last_peered":"2023-09-23T21:02:43.273173+0000","last_clean":"2023-09-23T21:02:43.273173+0000","last_became_active":"2023-09-23T20:55:35.963969+0000","last_became_peered":"2023-09-23T20:55:35.963969+0000","last_unstale":"2023-09-23T21:02:43.273173+0000","last_undegraded":"2023-09-23T21:02:43.273173+0000","last_fullsized":"2023-09-23T21:02:43.273173+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:38:40.612083+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.7","version":"275'6512","reported_seq":6542,"reported_epoch":275,"state":"active+clean","last_fresh":"2023-09-23T21:02:54.436775+0000","last_change":"2023-09-23T21:01:25.341425+0000","last_active":"2023-09-23T21:02:54.436775+0000","last_peered":"2023-09-23T21:02:54.436775+0000","last_clean":"2023-09-23T21:02:54.436775+0000","last_became_active":"2023-09-23T21:01:25.341114+0000","last_became_peered":"2023-09-23T21:01:25.341114+0000","last_unstale":"2023-09-23T21:02:54.436775+0000","last_undegraded":"2023-09-23T21:02:54.436775+0000","last_fullsized":"2023-09-23T21:02:54.436775+0000","mapping_epoch":254,"log_start":"0'0","ondisk_log_start":"0'0","created":254,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:01:24.325318+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:01:24.325318+0000","last_clean_scrub_stamp":"2023-09-23T21:01:24.325318+0000","objects_scrubbed":0,"log_size":6512,"log_dups_size":0,"ondisk_log_size":6512,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:15:08.338532+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":426770432,"num_objects":6512,"num_object_clones":0,"num_object_copies":19536,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6512,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6512,"num_write_kb":416768,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.e","version":"79'161","reported_seq":486,"reported_epoch":274,"state":"active+clean","last_fresh":"2023-09-23T21:02:43.273759+0000","last_change":"2023-09-23T20:57:34.132738+0000","last_active":"2023-09-23T21:02:43.273759+0000","last_peered":"2023-09-23T21:02:43.273759+0000","last_clean":"2023-09-23T21:02:43.273759+0000","last_became_active":"2023-09-23T20:57:15.974318+0000","last_became_peered":"2023-09-23T20:57:15.974318+0000","last_unstale":"2023-09-23T21:02:43.273759+0000","last_undegraded":"2023-09-23T21:02:43.273759+0000","last_fullsized":"2023-09-23T21:02:43.273759+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:34.132660+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:34.132660+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:07:07.739438+0000","scrub_duration":0.005229694,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":165,"reported_epoch":274,"state":"active+clean","last_fresh":"2023-09-23T21:02:43.275412+0000","last_change":"2023-09-23T20:55:35.962005+0000","last_active":"2023-09-23T21:02:43.275412+0000","last_peered":"2023-09-23T21:02:43.275412+0000","last_clean":"2023-09-23T21:02:43.275412+0000","last_became_active":"2023-09-23T20:55:35.961695+0000","last_became_peered":"2023-09-23T20:55:35.961695+0000","last_unstale":"2023-09-23T21:02:43.275412+0000","last_undegraded":"2023-09-23T21:02:43.275412+0000","last_fullsized":"2023-09-23T21:02:43.275412+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:28:22.876201+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":264,"reported_epoch":274,"state":"active+clean","last_fresh":"2023-09-23T21:02:43.275895+0000","last_change":"2023-09-23T20:54:13.928715+0000","last_active":"2023-09-23T21:02:43.275895+0000","last_peered":"2023-09-23T21:02:43.275895+0000","last_clean":"2023-09-23T21:02:43.275895+0000","last_became_active":"2023-09-23T20:54:13.928200+0000","last_became_peered":"2023-09-23T20:54:13.928200+0000","last_unstale":"2023-09-23T21:02:43.275895+0000","last_undegraded":"2023-09-23T21:02:43.275895+0000","last_fullsized":"2023-09-23T21:02:43.275895+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:41:16.927513+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.0","version":"275'6562","reported_seq":6591,"reported_epoch":275,"state":"active+clean","last_fresh":"2023-09-23T21:02:54.863553+0000","last_change":"2023-09-23T21:01:25.342305+0000","last_active":"2023-09-23T21:02:54.863553+0000","last_peered":"2023-09-23T21:02:54.863553+0000","last_clean":"2023-09-23T21:02:54.863553+0000","last_became_active":"2023-09-23T21:01:25.342042+0000","last_became_peered":"2023-09-23T21:01:25.342042+0000","last_unstale":"2023-09-23T21:02:54.863553+0000","last_undegraded":"2023-09-23T21:02:54.863553+0000","last_fullsized":"2023-09-23T21:02:54.863553+0000","mapping_epoch":254,"log_start":"0'0","ondisk_log_start":"0'0","created":254,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:01:24.325318+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:01:24.325318+0000","last_clean_scrub_stamp":"2023-09-23T21:01:24.325318+0000","objects_scrubbed":0,"log_size":6562,"log_dups_size":0,"ondisk_log_size":6562,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:06:29.755463+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":430047232,"num_objects":6562,"num_object_clones":0,"num_object_copies":19686,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6562,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6562,"num_write_kb":419968,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,1],"acting":[3,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.9","version":"79'161","reported_seq":649,"reported_epoch":274,"state":"active+clean","last_fresh":"2023-09-23T21:02:43.285230+0000","last_change":"2023-09-23T20:56:46.921939+0000","last_active":"2023-09-23T21:02:43.285230+0000","last_peered":"2023-09-23T21:02:43.285230+0000","last_clean":"2023-09-23T21:02:43.285230+0000","last_became_active":"2023-09-23T20:56:45.989908+0000","last_became_peered":"2023-09-23T20:56:45.989908+0000","last_unstale":"2023-09-23T21:02:43.285230+0000","last_undegraded":"2023-09-23T21:02:43.285230+0000","last_fullsized":"2023-09-23T21:02:43.285230+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":205,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:56:46.921739+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:56:46.921739+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:20:42.635110+0000","scrub_duration":0.0071954109999999996,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":266,"reported_epoch":274,"state":"active+clean","last_fresh":"2023-09-23T21:02:43.285632+0000","last_change":"2023-09-23T20:54:13.961788+0000","last_active":"2023-09-23T21:02:43.285632+0000","last_peered":"2023-09-23T21:02:43.285632+0000","last_clean":"2023-09-23T21:02:43.285632+0000","last_became_active":"2023-09-23T20:54:13.940937+0000","last_became_peered":"2023-09-23T20:54:13.940937+0000","last_unstale":"2023-09-23T21:02:43.285632+0000","last_undegraded":"2023-09-23T21:02:43.285632+0000","last_fullsized":"2023-09-23T21:02:43.285632+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:11:56.785299+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":169,"reported_epoch":274,"state":"active+clean","last_fresh":"2023-09-23T21:02:43.273208+0000","last_change":"2023-09-23T20:55:35.949670+0000","last_active":"2023-09-23T21:02:43.273208+0000","last_peered":"2023-09-23T21:02:43.273208+0000","last_clean":"2023-09-23T21:02:43.273208+0000","last_became_active":"2023-09-23T20:55:35.949140+0000","last_became_peered":"2023-09-23T20:55:35.949140+0000","last_unstale":"2023-09-23T21:02:43.273208+0000","last_undegraded":"2023-09-23T21:02:43.273208+0000","last_fullsized":"2023-09-23T21:02:43.273208+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:55:59.340303+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.1","version":"275'6700","reported_seq":6730,"reported_epoch":275,"state":"active+clean","last_fresh":"2023-09-23T21:02:54.400385+0000","last_change":"2023-09-23T21:01:25.429922+0000","last_active":"2023-09-23T21:02:54.400385+0000","last_peered":"2023-09-23T21:02:54.400385+0000","last_clean":"2023-09-23T21:02:54.400385+0000","last_became_active":"2023-09-23T21:01:25.429650+0000","last_became_peered":"2023-09-23T21:01:25.429650+0000","last_unstale":"2023-09-23T21:02:54.400385+0000","last_undegraded":"2023-09-23T21:02:54.400385+0000","last_fullsized":"2023-09-23T21:02:54.400385+0000","mapping_epoch":254,"log_start":"0'0","ondisk_log_start":"0'0","created":254,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:01:24.325318+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:01:24.325318+0000","last_clean_scrub_stamp":"2023-09-23T21:01:24.325318+0000","objects_scrubbed":0,"log_size":6700,"log_dups_size":0,"ondisk_log_size":6700,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:18:57.243292+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":439091200,"num_objects":6700,"num_object_clones":0,"num_object_copies":20100,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6700,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6700,"num_write_kb":428800,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,2],"acting":[1,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.8","version":"79'161","reported_seq":518,"reported_epoch":274,"state":"active+clean","last_fresh":"2023-09-23T21:02:43.275166+0000","last_change":"2023-09-23T20:56:57.505737+0000","last_active":"2023-09-23T21:02:43.275166+0000","last_peered":"2023-09-23T21:02:43.275166+0000","last_clean":"2023-09-23T21:02:43.275166+0000","last_became_active":"2023-09-23T20:56:04.982345+0000","last_became_peered":"2023-09-23T20:56:04.982345+0000","last_unstale":"2023-09-23T21:02:43.275166+0000","last_undegraded":"2023-09-23T21:02:43.275166+0000","last_fullsized":"2023-09-23T21:02:43.275166+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:56:57.505663+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:56:57.505663+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:34:30.109724+0000","scrub_duration":0.026284108,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":237,"reported_epoch":274,"state":"active+clean","last_fresh":"2023-09-23T21:02:43.273307+0000","last_change":"2023-09-23T20:55:35.959942+0000","last_active":"2023-09-23T21:02:43.273307+0000","last_peered":"2023-09-23T21:02:43.273307+0000","last_clean":"2023-09-23T21:02:43.273307+0000","last_became_active":"2023-09-23T20:55:35.957071+0000","last_became_peered":"2023-09-23T20:55:35.957071+0000","last_unstale":"2023-09-23T21:02:43.273307+0000","last_undegraded":"2023-09-23T21:02:43.273307+0000","last_fullsized":"2023-09-23T21:02:43.273307+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:44:30.308036+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":152,"reported_epoch":274,"state":"active+clean","last_fresh":"2023-09-23T21:02:43.273593+0000","last_change":"2023-09-23T20:54:13.951459+0000","last_active":"2023-09-23T21:02:43.273593+0000","last_peered":"2023-09-23T21:02:43.273593+0000","last_clean":"2023-09-23T21:02:43.273593+0000","last_became_active":"2023-09-23T20:54:13.948409+0000","last_became_peered":"2023-09-23T20:54:13.948409+0000","last_unstale":"2023-09-23T21:02:43.273593+0000","last_undegraded":"2023-09-23T21:02:43.273593+0000","last_fullsized":"2023-09-23T21:02:43.273593+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:38:10.148061+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.2","version":"275'6578","reported_seq":6607,"reported_epoch":275,"state":"active+clean","last_fresh":"2023-09-23T21:02:52.247347+0000","last_change":"2023-09-23T21:01:25.343022+0000","last_active":"2023-09-23T21:02:52.247347+0000","last_peered":"2023-09-23T21:02:52.247347+0000","last_clean":"2023-09-23T21:02:52.247347+0000","last_became_active":"2023-09-23T21:01:25.342540+0000","last_became_peered":"2023-09-23T21:01:25.342540+0000","last_unstale":"2023-09-23T21:02:52.247347+0000","last_undegraded":"2023-09-23T21:02:52.247347+0000","last_fullsized":"2023-09-23T21:02:52.247347+0000","mapping_epoch":254,"log_start":"0'0","ondisk_log_start":"0'0","created":254,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:01:24.325318+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:01:24.325318+0000","last_clean_scrub_stamp":"2023-09-23T21:01:24.325318+0000","objects_scrubbed":0,"log_size":6578,"log_dups_size":0,"ondisk_log_size":6578,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:57:19.741924+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":431095808,"num_objects":6578,"num_object_clones":0,"num_object_copies":19734,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6578,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6578,"num_write_kb":420992,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,5],"acting":[4,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.b","version":"79'161","reported_seq":559,"reported_epoch":274,"state":"active+clean","last_fresh":"2023-09-23T21:02:43.281992+0000","last_change":"2023-09-23T20:57:27.320172+0000","last_active":"2023-09-23T21:02:43.281992+0000","last_peered":"2023-09-23T21:02:43.281992+0000","last_clean":"2023-09-23T21:02:43.281992+0000","last_became_active":"2023-09-23T20:57:16.009630+0000","last_became_peered":"2023-09-23T20:57:16.009630+0000","last_unstale":"2023-09-23T21:02:43.281992+0000","last_undegraded":"2023-09-23T21:02:43.281992+0000","last_fullsized":"2023-09-23T21:02:43.281992+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:27.320022+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:27.320022+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:31:35.221558+0000","scrub_duration":0.0043894679999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,5],"acting":[1,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":152,"reported_epoch":274,"state":"active+clean","last_fresh":"2023-09-23T21:02:43.269787+0000","last_change":"2023-09-23T20:54:13.950254+0000","last_active":"2023-09-23T21:02:43.269787+0000","last_peered":"2023-09-23T21:02:43.269787+0000","last_clean":"2023-09-23T21:02:43.269787+0000","last_became_active":"2023-09-23T20:54:13.949934+0000","last_became_peered":"2023-09-23T20:54:13.949934+0000","last_unstale":"2023-09-23T21:02:43.269787+0000","last_undegraded":"2023-09-23T21:02:43.269787+0000","last_fullsized":"2023-09-23T21:02:43.269787+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:30:32.847167+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":169,"reported_epoch":274,"state":"active+clean","last_fresh":"2023-09-23T21:02:43.273960+0000","last_change":"2023-09-23T20:55:35.956153+0000","last_active":"2023-09-23T21:02:43.273960+0000","last_peered":"2023-09-23T21:02:43.273960+0000","last_clean":"2023-09-23T21:02:43.273960+0000","last_became_active":"2023-09-23T20:55:35.955630+0000","last_became_peered":"2023-09-23T20:55:35.955630+0000","last_unstale":"2023-09-23T21:02:43.273960+0000","last_undegraded":"2023-09-23T21:02:43.273960+0000","last_fullsized":"2023-09-23T21:02:43.273960+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:45:20.282929+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.3","version":"275'6331","reported_seq":6361,"reported_epoch":275,"state":"active+clean","last_fresh":"2023-09-23T21:02:54.416900+0000","last_change":"2023-09-23T21:01:25.342380+0000","last_active":"2023-09-23T21:02:54.416900+0000","last_peered":"2023-09-23T21:02:54.416900+0000","last_clean":"2023-09-23T21:02:54.416900+0000","last_became_active":"2023-09-23T21:01:25.342099+0000","last_became_peered":"2023-09-23T21:01:25.342099+0000","last_unstale":"2023-09-23T21:02:54.416900+0000","last_undegraded":"2023-09-23T21:02:54.416900+0000","last_fullsized":"2023-09-23T21:02:54.416900+0000","mapping_epoch":254,"log_start":"0'0","ondisk_log_start":"0'0","created":254,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:01:24.325318+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:01:24.325318+0000","last_clean_scrub_stamp":"2023-09-23T21:01:24.325318+0000","objects_scrubbed":0,"log_size":6331,"log_dups_size":0,"ondisk_log_size":6331,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:09:21.965969+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":414908416,"num_objects":6331,"num_object_clones":0,"num_object_copies":18993,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6331,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6331,"num_write_kb":405184,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.a","version":"79'161","reported_seq":518,"reported_epoch":274,"state":"active+clean","last_fresh":"2023-09-23T21:02:43.276028+0000","last_change":"2023-09-23T20:56:44.487778+0000","last_active":"2023-09-23T21:02:43.276028+0000","last_peered":"2023-09-23T21:02:43.276028+0000","last_clean":"2023-09-23T21:02:43.276028+0000","last_became_active":"2023-09-23T20:56:05.006740+0000","last_became_peered":"2023-09-23T20:56:05.006740+0000","last_unstale":"2023-09-23T21:02:43.276028+0000","last_undegraded":"2023-09-23T21:02:43.276028+0000","last_fullsized":"2023-09-23T21:02:43.276028+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:56:44.487678+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:56:44.487678+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:04:50.086749+0000","scrub_duration":0.0017408899999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":65,"reported_epoch":274,"state":"active+clean","last_fresh":"2023-09-23T21:02:43.287149+0000","last_change":"2023-09-23T20:57:44.982513+0000","last_active":"2023-09-23T21:02:43.287149+0000","last_peered":"2023-09-23T21:02:43.287149+0000","last_clean":"2023-09-23T21:02:43.287149+0000","last_became_active":"2023-09-23T20:57:44.982112+0000","last_became_peered":"2023-09-23T20:57:44.982112+0000","last_unstale":"2023-09-23T21:02:43.287149+0000","last_undegraded":"2023-09-23T21:02:43.287149+0000","last_fullsized":"2023-09-23T21:02:43.287149+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:09:05.630041+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":246,"reported_epoch":274,"state":"active+clean","last_fresh":"2023-09-23T21:02:43.283639+0000","last_change":"2023-09-23T20:54:13.940748+0000","last_active":"2023-09-23T21:02:43.283639+0000","last_peered":"2023-09-23T21:02:43.283639+0000","last_clean":"2023-09-23T21:02:43.283639+0000","last_became_active":"2023-09-23T20:54:13.940419+0000","last_became_peered":"2023-09-23T20:54:13.940419+0000","last_unstale":"2023-09-23T21:02:43.283639+0000","last_undegraded":"2023-09-23T21:02:43.283639+0000","last_fullsized":"2023-09-23T21:02:43.283639+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:15:01.923816+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.e","version":"275'6521","reported_seq":6551,"reported_epoch":275,"state":"active+clean","last_fresh":"2023-09-23T21:02:52.030133+0000","last_change":"2023-09-23T21:01:25.430111+0000","last_active":"2023-09-23T21:02:52.030133+0000","last_peered":"2023-09-23T21:02:52.030133+0000","last_clean":"2023-09-23T21:02:52.030133+0000","last_became_active":"2023-09-23T21:01:25.429811+0000","last_became_peered":"2023-09-23T21:01:25.429811+0000","last_unstale":"2023-09-23T21:02:52.030133+0000","last_undegraded":"2023-09-23T21:02:52.030133+0000","last_fullsized":"2023-09-23T21:02:52.030133+0000","mapping_epoch":254,"log_start":"0'0","ondisk_log_start":"0'0","created":254,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:01:24.325318+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:01:24.325318+0000","last_clean_scrub_stamp":"2023-09-23T21:01:24.325318+0000","objects_scrubbed":0,"log_size":6521,"log_dups_size":0,"ondisk_log_size":6521,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:18:01.918758+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":427360256,"num_objects":6521,"num_object_clones":0,"num_object_copies":19563,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6521,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6521,"num_write_kb":417344,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.7","version":"79'161","reported_seq":580,"reported_epoch":274,"state":"active+clean","last_fresh":"2023-09-23T21:02:43.273895+0000","last_change":"2023-09-23T20:57:48.243572+0000","last_active":"2023-09-23T21:02:43.273895+0000","last_peered":"2023-09-23T21:02:43.273895+0000","last_clean":"2023-09-23T21:02:43.273895+0000","last_became_active":"2023-09-23T20:57:48.243229+0000","last_became_peered":"2023-09-23T20:57:48.243229+0000","last_unstale":"2023-09-23T21:02:43.273895+0000","last_undegraded":"2023-09-23T21:02:43.273895+0000","last_fullsized":"2023-09-23T21:02:43.273895+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":3,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:25.916867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:25.916867+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:23:19.385831+0000","scrub_duration":0.001665323,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":252,"reported_epoch":274,"state":"active+clean","last_fresh":"2023-09-23T21:02:43.276830+0000","last_change":"2023-09-23T20:50:05.867130+0000","last_active":"2023-09-23T21:02:43.276830+0000","last_peered":"2023-09-23T21:02:43.276830+0000","last_clean":"2023-09-23T21:02:43.276830+0000","last_became_active":"2023-09-23T20:50:05.866758+0000","last_became_peered":"2023-09-23T20:50:05.866758+0000","last_unstale":"2023-09-23T21:02:43.276830+0000","last_undegraded":"2023-09-23T21:02:43.276830+0000","last_fullsized":"2023-09-23T21:02:43.276830+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:55:07.548209+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":148,"reported_epoch":274,"state":"active+clean","last_fresh":"2023-09-23T21:02:43.276875+0000","last_change":"2023-09-23T20:54:31.887604+0000","last_active":"2023-09-23T21:02:43.276875+0000","last_peered":"2023-09-23T21:02:43.276875+0000","last_clean":"2023-09-23T21:02:43.276875+0000","last_became_active":"2023-09-23T20:54:31.886752+0000","last_became_peered":"2023-09-23T20:54:31.886752+0000","last_unstale":"2023-09-23T21:02:43.276875+0000","last_undegraded":"2023-09-23T21:02:43.276875+0000","last_fullsized":"2023-09-23T21:02:43.276875+0000","mapping_epoch":136,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:13:58.872176+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":250,"reported_epoch":274,"state":"active+clean","last_fresh":"2023-09-23T21:02:43.275912+0000","last_change":"2023-09-23T20:54:31.899082+0000","last_active":"2023-09-23T21:02:43.275912+0000","last_peered":"2023-09-23T21:02:43.275912+0000","last_clean":"2023-09-23T21:02:43.275912+0000","last_became_active":"2023-09-23T20:54:31.898521+0000","last_became_peered":"2023-09-23T20:54:31.898521+0000","last_unstale":"2023-09-23T21:02:43.275912+0000","last_undegraded":"2023-09-23T21:02:43.275912+0000","last_fullsized":"2023-09-23T21:02:43.275912+0000","mapping_epoch":136,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:25:36.059711+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.8","version":"275'6427","reported_seq":6456,"reported_epoch":275,"state":"active+clean","last_fresh":"2023-09-23T21:02:52.028981+0000","last_change":"2023-09-23T21:01:25.560581+0000","last_active":"2023-09-23T21:02:52.028981+0000","last_peered":"2023-09-23T21:02:52.028981+0000","last_clean":"2023-09-23T21:02:52.028981+0000","last_became_active":"2023-09-23T21:01:25.560318+0000","last_became_peered":"2023-09-23T21:01:25.560318+0000","last_unstale":"2023-09-23T21:02:52.028981+0000","last_undegraded":"2023-09-23T21:02:52.028981+0000","last_fullsized":"2023-09-23T21:02:52.028981+0000","mapping_epoch":254,"log_start":"0'0","ondisk_log_start":"0'0","created":254,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:01:24.325318+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:01:24.325318+0000","last_clean_scrub_stamp":"2023-09-23T21:01:24.325318+0000","objects_scrubbed":0,"log_size":6427,"log_dups_size":0,"ondisk_log_size":6427,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:07:28.203012+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":421199872,"num_objects":6427,"num_object_clones":0,"num_object_copies":19281,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6427,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6427,"num_write_kb":411328,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[5,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.1","version":"79'161","reported_seq":538,"reported_epoch":274,"state":"active+clean","last_fresh":"2023-09-23T21:02:43.275141+0000","last_change":"2023-09-23T20:57:19.669346+0000","last_active":"2023-09-23T21:02:43.275141+0000","last_peered":"2023-09-23T21:02:43.275141+0000","last_clean":"2023-09-23T21:02:43.275141+0000","last_became_active":"2023-09-23T20:57:08.246046+0000","last_became_peered":"2023-09-23T20:57:08.246046+0000","last_unstale":"2023-09-23T21:02:43.275141+0000","last_undegraded":"2023-09-23T21:02:43.275141+0000","last_fullsized":"2023-09-23T21:02:43.275141+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:19.669259+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:19.669259+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:14:03.435531+0000","scrub_duration":0.0045759559999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":165,"reported_epoch":274,"state":"active+clean","last_fresh":"2023-09-23T21:02:43.275891+0000","last_change":"2023-09-23T20:55:35.959514+0000","last_active":"2023-09-23T21:02:43.275891+0000","last_peered":"2023-09-23T21:02:43.275891+0000","last_clean":"2023-09-23T21:02:43.275891+0000","last_became_active":"2023-09-23T20:55:35.959244+0000","last_became_peered":"2023-09-23T20:55:35.959244+0000","last_unstale":"2023-09-23T21:02:43.275891+0000","last_undegraded":"2023-09-23T21:02:43.275891+0000","last_fullsized":"2023-09-23T21:02:43.275891+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:47:32.665316+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":169,"reported_epoch":274,"state":"active+clean","last_fresh":"2023-09-23T21:02:43.270163+0000","last_change":"2023-09-23T20:55:35.949000+0000","last_active":"2023-09-23T21:02:43.270163+0000","last_peered":"2023-09-23T21:02:43.270163+0000","last_clean":"2023-09-23T21:02:43.270163+0000","last_became_active":"2023-09-23T20:55:35.948410+0000","last_became_peered":"2023-09-23T20:55:35.948410+0000","last_unstale":"2023-09-23T21:02:43.270163+0000","last_undegraded":"2023-09-23T21:02:43.270163+0000","last_fullsized":"2023-09-23T21:02:43.270163+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:26:50.477289+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":244,"reported_epoch":274,"state":"active+clean","last_fresh":"2023-09-23T21:02:43.276603+0000","last_change":"2023-09-23T20:51:10.829889+0000","last_active":"2023-09-23T21:02:43.276603+0000","last_peered":"2023-09-23T21:02:43.276603+0000","last_clean":"2023-09-23T21:02:43.276603+0000","last_became_active":"2023-09-23T20:51:10.829533+0000","last_became_peered":"2023-09-23T20:51:10.829533+0000","last_unstale":"2023-09-23T21:02:43.276603+0000","last_undegraded":"2023-09-23T21:02:43.276603+0000","last_fullsized":"2023-09-23T21:02:43.276603+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:43:04.603460+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.b","version":"275'6631","reported_seq":6660,"reported_epoch":275,"state":"active+clean","last_fresh":"2023-09-23T21:02:54.839775+0000","last_change":"2023-09-23T21:01:25.340569+0000","last_active":"2023-09-23T21:02:54.839775+0000","last_peered":"2023-09-23T21:02:54.839775+0000","last_clean":"2023-09-23T21:02:54.839775+0000","last_became_active":"2023-09-23T21:01:25.340209+0000","last_became_peered":"2023-09-23T21:01:25.340209+0000","last_unstale":"2023-09-23T21:02:54.839775+0000","last_undegraded":"2023-09-23T21:02:54.839775+0000","last_fullsized":"2023-09-23T21:02:54.839775+0000","mapping_epoch":254,"log_start":"258'100","ondisk_log_start":"258'100","created":254,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:01:24.325318+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:01:24.325318+0000","last_clean_scrub_stamp":"2023-09-23T21:01:24.325318+0000","objects_scrubbed":0,"log_size":6531,"log_dups_size":0,"ondisk_log_size":6531,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:53:59.553810+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":434569216,"num_objects":6631,"num_object_clones":0,"num_object_copies":19893,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6631,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6631,"num_write_kb":424384,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.2","version":"79'161","reported_seq":506,"reported_epoch":274,"state":"active+clean","last_fresh":"2023-09-23T21:02:43.276280+0000","last_change":"2023-09-23T20:57:24.802586+0000","last_active":"2023-09-23T21:02:43.276280+0000","last_peered":"2023-09-23T21:02:43.276280+0000","last_clean":"2023-09-23T21:02:43.276280+0000","last_became_active":"2023-09-23T20:57:08.236114+0000","last_became_peered":"2023-09-23T20:57:08.236114+0000","last_unstale":"2023-09-23T21:02:43.276280+0000","last_undegraded":"2023-09-23T21:02:43.276280+0000","last_fullsized":"2023-09-23T21:02:43.276280+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:24.802451+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:24.802451+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:06:22.309980+0000","scrub_duration":0.0090861250000000005,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,4],"acting":[0,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":262,"reported_epoch":274,"state":"active+clean","last_fresh":"2023-09-23T21:02:43.269385+0000","last_change":"2023-09-23T20:51:10.838054+0000","last_active":"2023-09-23T21:02:43.269385+0000","last_peered":"2023-09-23T21:02:43.269385+0000","last_clean":"2023-09-23T21:02:43.269385+0000","last_became_active":"2023-09-23T20:51:10.837750+0000","last_became_peered":"2023-09-23T20:51:10.837750+0000","last_unstale":"2023-09-23T21:02:43.269385+0000","last_undegraded":"2023-09-23T21:02:43.269385+0000","last_fullsized":"2023-09-23T21:02:43.269385+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:15:51.648085+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":244,"reported_epoch":274,"state":"active+clean","last_fresh":"2023-09-23T21:02:43.276428+0000","last_change":"2023-09-23T20:55:35.951146+0000","last_active":"2023-09-23T21:02:43.276428+0000","last_peered":"2023-09-23T21:02:43.276428+0000","last_clean":"2023-09-23T21:02:43.276428+0000","last_became_active":"2023-09-23T20:55:35.950874+0000","last_became_peered":"2023-09-23T20:55:35.950874+0000","last_unstale":"2023-09-23T21:02:43.276428+0000","last_undegraded":"2023-09-23T21:02:43.276428+0000","last_fullsized":"2023-09-23T21:02:43.276428+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:58:13.957677+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":264,"reported_epoch":274,"state":"active+clean","last_fresh":"2023-09-23T21:02:43.276737+0000","last_change":"2023-09-23T20:54:31.898870+0000","last_active":"2023-09-23T21:02:43.276737+0000","last_peered":"2023-09-23T21:02:43.276737+0000","last_clean":"2023-09-23T21:02:43.276737+0000","last_became_active":"2023-09-23T20:54:31.898467+0000","last_became_peered":"2023-09-23T20:54:31.898467+0000","last_unstale":"2023-09-23T21:02:43.276737+0000","last_undegraded":"2023-09-23T21:02:43.276737+0000","last_fullsized":"2023-09-23T21:02:43.276737+0000","mapping_epoch":136,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:56:22.483408+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.9","version":"275'6403","reported_seq":6432,"reported_epoch":275,"state":"active+clean","last_fresh":"2023-09-23T21:02:54.121005+0000","last_change":"2023-09-23T21:01:25.342548+0000","last_active":"2023-09-23T21:02:54.121005+0000","last_peered":"2023-09-23T21:02:54.121005+0000","last_clean":"2023-09-23T21:02:54.121005+0000","last_became_active":"2023-09-23T21:01:25.342227+0000","last_became_peered":"2023-09-23T21:01:25.342227+0000","last_unstale":"2023-09-23T21:02:54.121005+0000","last_undegraded":"2023-09-23T21:02:54.121005+0000","last_fullsized":"2023-09-23T21:02:54.121005+0000","mapping_epoch":254,"log_start":"0'0","ondisk_log_start":"0'0","created":254,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:01:24.325318+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:01:24.325318+0000","last_clean_scrub_stamp":"2023-09-23T21:01:24.325318+0000","objects_scrubbed":0,"log_size":6403,"log_dups_size":0,"ondisk_log_size":6403,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:38:15.944067+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":419627008,"num_objects":6403,"num_object_clones":0,"num_object_copies":19209,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6403,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6403,"num_write_kb":409792,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,0],"acting":[6,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":148,"reported_epoch":274,"state":"active+clean","last_fresh":"2023-09-23T21:02:43.275374+0000","last_change":"2023-09-23T20:54:31.883504+0000","last_active":"2023-09-23T21:02:43.275374+0000","last_peered":"2023-09-23T21:02:43.275374+0000","last_clean":"2023-09-23T21:02:43.275374+0000","last_became_active":"2023-09-23T20:54:31.883172+0000","last_became_peered":"2023-09-23T20:54:31.883172+0000","last_unstale":"2023-09-23T21:02:43.275374+0000","last_undegraded":"2023-09-23T21:02:43.275374+0000","last_fullsized":"2023-09-23T21:02:43.275374+0000","mapping_epoch":136,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:45:40.289120+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"79'161","reported_seq":507,"reported_epoch":274,"state":"active+clean","last_fresh":"2023-09-23T21:02:47.093530+0000","last_change":"2023-09-23T20:57:20.738417+0000","last_active":"2023-09-23T21:02:47.093530+0000","last_peered":"2023-09-23T21:02:47.093530+0000","last_clean":"2023-09-23T21:02:47.093530+0000","last_became_active":"2023-09-23T20:57:08.236842+0000","last_became_peered":"2023-09-23T20:57:08.236842+0000","last_unstale":"2023-09-23T21:02:47.093530+0000","last_undegraded":"2023-09-23T21:02:47.093530+0000","last_fullsized":"2023-09-23T21:02:47.093530+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":0,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:20.738273+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:20.738273+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:18:28.327990+0000","scrub_duration":0.0088336339999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":254,"reported_epoch":274,"state":"active+clean","last_fresh":"2023-09-23T21:02:43.285272+0000","last_change":"2023-09-23T20:55:35.950777+0000","last_active":"2023-09-23T21:02:43.285272+0000","last_peered":"2023-09-23T21:02:43.285272+0000","last_clean":"2023-09-23T21:02:43.285272+0000","last_became_active":"2023-09-23T20:55:35.950427+0000","last_became_peered":"2023-09-23T20:55:35.950427+0000","last_unstale":"2023-09-23T21:02:43.285272+0000","last_undegraded":"2023-09-23T21:02:43.285272+0000","last_fullsized":"2023-09-23T21:02:43.285272+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:20:10.371751+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":219,"reported_epoch":274,"state":"active+clean","last_fresh":"2023-09-23T21:02:43.273618+0000","last_change":"2023-09-23T20:55:35.978140+0000","last_active":"2023-09-23T21:02:43.273618+0000","last_peered":"2023-09-23T21:02:43.273618+0000","last_clean":"2023-09-23T21:02:43.273618+0000","last_became_active":"2023-09-23T20:55:35.977831+0000","last_became_peered":"2023-09-23T20:55:35.977831+0000","last_unstale":"2023-09-23T21:02:43.273618+0000","last_undegraded":"2023-09-23T21:02:43.273618+0000","last_fullsized":"2023-09-23T21:02:43.273618+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:25:47.678023+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.a","version":"275'6502","reported_seq":6532,"reported_epoch":275,"state":"active+clean","last_fresh":"2023-09-23T21:02:54.371443+0000","last_change":"2023-09-23T21:01:25.429899+0000","last_active":"2023-09-23T21:02:54.371443+0000","last_peered":"2023-09-23T21:02:54.371443+0000","last_clean":"2023-09-23T21:02:54.371443+0000","last_became_active":"2023-09-23T21:01:25.429583+0000","last_became_peered":"2023-09-23T21:01:25.429583+0000","last_unstale":"2023-09-23T21:02:54.371443+0000","last_undegraded":"2023-09-23T21:02:54.371443+0000","last_fullsized":"2023-09-23T21:02:54.371443+0000","mapping_epoch":254,"log_start":"0'0","ondisk_log_start":"0'0","created":254,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:01:24.325318+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:01:24.325318+0000","last_clean_scrub_stamp":"2023-09-23T21:01:24.325318+0000","objects_scrubbed":0,"log_size":6502,"log_dups_size":0,"ondisk_log_size":6502,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:31:11.594129+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":426115072,"num_objects":6502,"num_object_clones":0,"num_object_copies":19506,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6502,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6502,"num_write_kb":416128,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.3","version":"79'161","reported_seq":543,"reported_epoch":274,"state":"active+clean","last_fresh":"2023-09-23T21:02:43.282073+0000","last_change":"2023-09-23T20:57:20.378531+0000","last_active":"2023-09-23T21:02:43.282073+0000","last_peered":"2023-09-23T21:02:43.282073+0000","last_clean":"2023-09-23T21:02:43.282073+0000","last_became_active":"2023-09-23T20:57:08.238019+0000","last_became_peered":"2023-09-23T20:57:08.238019+0000","last_unstale":"2023-09-23T21:02:43.282073+0000","last_undegraded":"2023-09-23T21:02:43.282073+0000","last_fullsized":"2023-09-23T21:02:43.282073+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":3,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:20.377714+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:20.377714+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:00:08.305602+0000","scrub_duration":0.003015228,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":146,"reported_epoch":274,"state":"active+clean","last_fresh":"2023-09-23T21:02:49.455791+0000","last_change":"2023-09-23T20:56:45.991884+0000","last_active":"2023-09-23T21:02:49.455791+0000","last_peered":"2023-09-23T21:02:49.455791+0000","last_clean":"2023-09-23T21:02:49.455791+0000","last_became_active":"2023-09-23T20:56:45.991568+0000","last_became_peered":"2023-09-23T20:56:45.991568+0000","last_unstale":"2023-09-23T21:02:49.455791+0000","last_undegraded":"2023-09-23T21:02:49.455791+0000","last_fullsized":"2023-09-23T21:02:49.455791+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":205,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:21:20.775511+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,0],"acting":[1,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":270,"reported_epoch":274,"state":"active+clean","last_fresh":"2023-09-23T21:02:43.276003+0000","last_change":"2023-09-23T20:57:44.979950+0000","last_active":"2023-09-23T21:02:43.276003+0000","last_peered":"2023-09-23T21:02:43.276003+0000","last_clean":"2023-09-23T21:02:43.276003+0000","last_became_active":"2023-09-23T20:57:44.979346+0000","last_became_peered":"2023-09-23T20:57:44.979346+0000","last_unstale":"2023-09-23T21:02:43.276003+0000","last_undegraded":"2023-09-23T21:02:43.276003+0000","last_fullsized":"2023-09-23T21:02:43.276003+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:10:24.372800+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":249,"reported_epoch":274,"state":"active+clean","last_fresh":"2023-09-23T21:02:43.276121+0000","last_change":"2023-09-23T20:50:37.982235+0000","last_active":"2023-09-23T21:02:43.276121+0000","last_peered":"2023-09-23T21:02:43.276121+0000","last_clean":"2023-09-23T21:02:43.276121+0000","last_became_active":"2023-09-23T20:50:37.981913+0000","last_became_peered":"2023-09-23T20:50:37.981913+0000","last_unstale":"2023-09-23T21:02:43.276121+0000","last_undegraded":"2023-09-23T21:02:43.276121+0000","last_fullsized":"2023-09-23T21:02:43.276121+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:59:57.122584+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.f","version":"275'6532","reported_seq":6562,"reported_epoch":275,"state":"active+clean","last_fresh":"2023-09-23T21:02:54.838407+0000","last_change":"2023-09-23T21:01:25.432176+0000","last_active":"2023-09-23T21:02:54.838407+0000","last_peered":"2023-09-23T21:02:54.838407+0000","last_clean":"2023-09-23T21:02:54.838407+0000","last_became_active":"2023-09-23T21:01:25.431835+0000","last_became_peered":"2023-09-23T21:01:25.431835+0000","last_unstale":"2023-09-23T21:02:54.838407+0000","last_undegraded":"2023-09-23T21:02:54.838407+0000","last_fullsized":"2023-09-23T21:02:54.838407+0000","mapping_epoch":254,"log_start":"0'0","ondisk_log_start":"0'0","created":254,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:01:24.325318+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:01:24.325318+0000","last_clean_scrub_stamp":"2023-09-23T21:01:24.325318+0000","objects_scrubbed":0,"log_size":6532,"log_dups_size":0,"ondisk_log_size":6532,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:47:26.553643+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":428081152,"num_objects":6532,"num_object_clones":0,"num_object_copies":19596,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6532,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6532,"num_write_kb":418048,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,1],"acting":[3,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.6","version":"79'161","reported_seq":557,"reported_epoch":274,"state":"active+clean","last_fresh":"2023-09-23T21:02:43.273653+0000","last_change":"2023-09-23T20:57:26.907505+0000","last_active":"2023-09-23T21:02:43.273653+0000","last_peered":"2023-09-23T21:02:43.273653+0000","last_clean":"2023-09-23T21:02:43.273653+0000","last_became_active":"2023-09-23T20:57:08.240321+0000","last_became_peered":"2023-09-23T20:57:08.240321+0000","last_unstale":"2023-09-23T21:02:43.273653+0000","last_undegraded":"2023-09-23T21:02:43.273653+0000","last_fullsized":"2023-09-23T21:02:43.273653+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":3,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:26.907408+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:26.907408+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:34:17.392928+0000","scrub_duration":0.010337038,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":192,"reported_epoch":274,"state":"active+clean","last_fresh":"2023-09-23T21:02:43.276097+0000","last_change":"2023-09-23T20:56:45.988566+0000","last_active":"2023-09-23T21:02:43.276097+0000","last_peered":"2023-09-23T21:02:43.276097+0000","last_clean":"2023-09-23T21:02:43.276097+0000","last_became_active":"2023-09-23T20:56:45.988249+0000","last_became_peered":"2023-09-23T20:56:45.988249+0000","last_unstale":"2023-09-23T21:02:43.276097+0000","last_undegraded":"2023-09-23T21:02:43.276097+0000","last_fullsized":"2023-09-23T21:02:43.276097+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":205,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:51:36.644764+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,4],"acting":[0,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":247,"reported_epoch":274,"state":"active+clean","last_fresh":"2023-09-23T21:02:43.276751+0000","last_change":"2023-09-23T20:54:31.898965+0000","last_active":"2023-09-23T21:02:43.276751+0000","last_peered":"2023-09-23T21:02:43.276751+0000","last_clean":"2023-09-23T21:02:43.276751+0000","last_became_active":"2023-09-23T20:54:31.898663+0000","last_became_peered":"2023-09-23T20:54:31.898663+0000","last_unstale":"2023-09-23T21:02:43.276751+0000","last_undegraded":"2023-09-23T21:02:43.276751+0000","last_fullsized":"2023-09-23T21:02:43.276751+0000","mapping_epoch":136,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:53:05.058087+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":237,"reported_epoch":274,"state":"active+clean","last_fresh":"2023-09-23T21:02:43.276149+0000","last_change":"2023-09-23T20:54:31.884717+0000","last_active":"2023-09-23T21:02:43.276149+0000","last_peered":"2023-09-23T21:02:43.276149+0000","last_clean":"2023-09-23T21:02:43.276149+0000","last_became_active":"2023-09-23T20:54:31.884345+0000","last_became_peered":"2023-09-23T20:54:31.884345+0000","last_unstale":"2023-09-23T21:02:43.276149+0000","last_undegraded":"2023-09-23T21:02:43.276149+0000","last_fullsized":"2023-09-23T21:02:43.276149+0000","mapping_epoch":136,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:02:26.393868+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.c","version":"275'6499","reported_seq":6529,"reported_epoch":275,"state":"active+clean","last_fresh":"2023-09-23T21:02:52.090017+0000","last_change":"2023-09-23T21:01:25.560907+0000","last_active":"2023-09-23T21:02:52.090017+0000","last_peered":"2023-09-23T21:02:52.090017+0000","last_clean":"2023-09-23T21:02:52.090017+0000","last_became_active":"2023-09-23T21:01:25.560592+0000","last_became_peered":"2023-09-23T21:01:25.560592+0000","last_unstale":"2023-09-23T21:02:52.090017+0000","last_undegraded":"2023-09-23T21:02:52.090017+0000","last_fullsized":"2023-09-23T21:02:52.090017+0000","mapping_epoch":254,"log_start":"0'0","ondisk_log_start":"0'0","created":254,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:01:24.325318+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:01:24.325318+0000","last_clean_scrub_stamp":"2023-09-23T21:01:24.325318+0000","objects_scrubbed":0,"log_size":6499,"log_dups_size":0,"ondisk_log_size":6499,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:14:22.102068+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":425918464,"num_objects":6499,"num_object_clones":0,"num_object_copies":19497,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6499,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6499,"num_write_kb":415936,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.5","version":"79'161","reported_seq":538,"reported_epoch":274,"state":"active+clean","last_fresh":"2023-09-23T21:02:43.273321+0000","last_change":"2023-09-23T20:57:30.029479+0000","last_active":"2023-09-23T21:02:43.273321+0000","last_peered":"2023-09-23T21:02:43.273321+0000","last_clean":"2023-09-23T21:02:43.273321+0000","last_became_active":"2023-09-23T20:57:08.249909+0000","last_became_peered":"2023-09-23T20:57:08.249909+0000","last_unstale":"2023-09-23T21:02:43.273321+0000","last_undegraded":"2023-09-23T21:02:43.273321+0000","last_fullsized":"2023-09-23T21:02:43.273321+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":3,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:30.029396+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:30.029396+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:16:46.083145+0000","scrub_duration":0.012914682,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":240,"reported_epoch":274,"state":"active+clean","last_fresh":"2023-09-23T21:02:43.288096+0000","last_change":"2023-09-23T20:51:10.836910+0000","last_active":"2023-09-23T21:02:43.288096+0000","last_peered":"2023-09-23T21:02:43.288096+0000","last_clean":"2023-09-23T21:02:43.288096+0000","last_became_active":"2023-09-23T20:51:10.836557+0000","last_became_peered":"2023-09-23T20:51:10.836557+0000","last_unstale":"2023-09-23T21:02:43.288096+0000","last_undegraded":"2023-09-23T21:02:43.288096+0000","last_fullsized":"2023-09-23T21:02:43.288096+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:51:54.019969+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":246,"reported_epoch":274,"state":"active+clean","last_fresh":"2023-09-23T21:02:43.281958+0000","last_change":"2023-09-23T20:55:35.953444+0000","last_active":"2023-09-23T21:02:43.281958+0000","last_peered":"2023-09-23T21:02:43.281958+0000","last_clean":"2023-09-23T21:02:43.281958+0000","last_became_active":"2023-09-23T20:55:35.953151+0000","last_became_peered":"2023-09-23T20:55:35.953151+0000","last_unstale":"2023-09-23T21:02:43.281958+0000","last_undegraded":"2023-09-23T21:02:43.281958+0000","last_fullsized":"2023-09-23T21:02:43.281958+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:39:12.404391+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":65,"reported_epoch":274,"state":"active+clean","last_fresh":"2023-09-23T21:02:43.287170+0000","last_change":"2023-09-23T20:57:44.982509+0000","last_active":"2023-09-23T21:02:43.287170+0000","last_peered":"2023-09-23T21:02:43.287170+0000","last_clean":"2023-09-23T21:02:43.287170+0000","last_became_active":"2023-09-23T20:57:44.982175+0000","last_became_peered":"2023-09-23T20:57:44.982175+0000","last_unstale":"2023-09-23T21:02:43.287170+0000","last_undegraded":"2023-09-23T21:02:43.287170+0000","last_fullsized":"2023-09-23T21:02:43.287170+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:37:49.215488+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.d","version":"275'6392","reported_seq":6421,"reported_epoch":275,"state":"active+clean","last_fresh":"2023-09-23T21:02:52.071131+0000","last_change":"2023-09-23T21:01:25.560531+0000","last_active":"2023-09-23T21:02:52.071131+0000","last_peered":"2023-09-23T21:02:52.071131+0000","last_clean":"2023-09-23T21:02:52.071131+0000","last_became_active":"2023-09-23T21:01:25.560228+0000","last_became_peered":"2023-09-23T21:01:25.560228+0000","last_unstale":"2023-09-23T21:02:52.071131+0000","last_undegraded":"2023-09-23T21:02:52.071131+0000","last_fullsized":"2023-09-23T21:02:52.071131+0000","mapping_epoch":254,"log_start":"0'0","ondisk_log_start":"0'0","created":254,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:01:24.325318+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:01:24.325318+0000","last_clean_scrub_stamp":"2023-09-23T21:01:24.325318+0000","objects_scrubbed":0,"log_size":6392,"log_dups_size":0,"ondisk_log_size":6392,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:19:04.504855+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":418906112,"num_objects":6392,"num_object_clones":0,"num_object_copies":19176,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6392,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6392,"num_write_kb":409088,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,2],"acting":[5,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.4","version":"79'161","reported_seq":555,"reported_epoch":274,"state":"active+clean","last_fresh":"2023-09-23T21:02:43.285769+0000","last_change":"2023-09-23T20:57:20.012904+0000","last_active":"2023-09-23T21:02:43.285769+0000","last_peered":"2023-09-23T21:02:43.285769+0000","last_clean":"2023-09-23T21:02:43.285769+0000","last_became_active":"2023-09-23T20:57:08.237996+0000","last_became_peered":"2023-09-23T20:57:08.237996+0000","last_unstale":"2023-09-23T21:02:43.285769+0000","last_undegraded":"2023-09-23T21:02:43.285769+0000","last_fullsized":"2023-09-23T21:02:43.285769+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":3,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:20.012764+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:20.012764+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:51:28.796946+0000","scrub_duration":0.0045251919999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":224,"reported_epoch":274,"state":"active+clean","last_fresh":"2023-09-23T21:02:43.273812+0000","last_change":"2023-09-23T20:55:35.949541+0000","last_active":"2023-09-23T21:02:43.273812+0000","last_peered":"2023-09-23T21:02:43.273812+0000","last_clean":"2023-09-23T21:02:43.273812+0000","last_became_active":"2023-09-23T20:55:35.944635+0000","last_became_peered":"2023-09-23T20:55:35.944635+0000","last_unstale":"2023-09-23T21:02:43.273812+0000","last_undegraded":"2023-09-23T21:02:43.273812+0000","last_fullsized":"2023-09-23T21:02:43.273812+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:30:36.033585+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"70'1","reported_seq":265,"reported_epoch":274,"state":"active+clean","last_fresh":"2023-09-23T21:02:43.276241+0000","last_change":"2023-09-23T20:54:14.009723+0000","last_active":"2023-09-23T21:02:43.276241+0000","last_peered":"2023-09-23T21:02:43.276241+0000","last_clean":"2023-09-23T21:02:43.276241+0000","last_became_active":"2023-09-23T20:54:13.941771+0000","last_became_peered":"2023-09-23T20:54:13.941771+0000","last_unstale":"2023-09-23T21:02:43.276241+0000","last_undegraded":"2023-09-23T21:02:43.276241+0000","last_fullsized":"2023-09-23T21:02:43.276241+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:48:25.588116+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":179,"reported_epoch":274,"state":"active+clean","last_fresh":"2023-09-23T21:02:43.269866+0000","last_change":"2023-09-23T20:55:35.949780+0000","last_active":"2023-09-23T21:02:43.269866+0000","last_peered":"2023-09-23T21:02:43.269866+0000","last_clean":"2023-09-23T21:02:43.269866+0000","last_became_active":"2023-09-23T20:55:35.949497+0000","last_became_peered":"2023-09-23T20:55:35.949497+0000","last_unstale":"2023-09-23T21:02:43.269866+0000","last_undegraded":"2023-09-23T21:02:43.269866+0000","last_fullsized":"2023-09-23T21:02:43.269866+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:09:37.420882+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":239,"reported_epoch":274,"state":"active+clean","last_fresh":"2023-09-23T21:02:43.276055+0000","last_change":"2023-09-23T20:54:13.929196+0000","last_active":"2023-09-23T21:02:43.276055+0000","last_peered":"2023-09-23T21:02:43.276055+0000","last_clean":"2023-09-23T21:02:43.276055+0000","last_became_active":"2023-09-23T20:54:13.928831+0000","last_became_peered":"2023-09-23T20:54:13.928831+0000","last_unstale":"2023-09-23T21:02:43.276055+0000","last_undegraded":"2023-09-23T21:02:43.276055+0000","last_fullsized":"2023-09-23T21:02:43.276055+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:28:44.316851+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":244,"reported_epoch":274,"state":"active+clean","last_fresh":"2023-09-23T21:02:43.263284+0000","last_change":"2023-09-23T20:52:12.855430+0000","last_active":"2023-09-23T21:02:43.263284+0000","last_peered":"2023-09-23T21:02:43.263284+0000","last_clean":"2023-09-23T21:02:43.263284+0000","last_became_active":"2023-09-23T20:52:12.855115+0000","last_became_peered":"2023-09-23T20:52:12.855115+0000","last_unstale":"2023-09-23T21:02:43.263284+0000","last_undegraded":"2023-09-23T21:02:43.263284+0000","last_fullsized":"2023-09-23T21:02:43.263284+0000","mapping_epoch":100,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":101,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:33:23.802214+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":152,"reported_epoch":274,"state":"active+clean","last_fresh":"2023-09-23T21:02:43.275495+0000","last_change":"2023-09-23T20:54:31.902946+0000","last_active":"2023-09-23T21:02:43.275495+0000","last_peered":"2023-09-23T21:02:43.275495+0000","last_clean":"2023-09-23T21:02:43.275495+0000","last_became_active":"2023-09-23T20:54:31.889484+0000","last_became_peered":"2023-09-23T20:54:31.889484+0000","last_unstale":"2023-09-23T21:02:43.275495+0000","last_undegraded":"2023-09-23T21:02:43.275495+0000","last_fullsized":"2023-09-23T21:02:43.275495+0000","mapping_epoch":136,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:44:44.278398+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":152,"reported_epoch":274,"state":"active+clean","last_fresh":"2023-09-23T21:02:43.270287+0000","last_change":"2023-09-23T20:54:13.950351+0000","last_active":"2023-09-23T21:02:43.270287+0000","last_peered":"2023-09-23T21:02:43.270287+0000","last_clean":"2023-09-23T21:02:43.270287+0000","last_became_active":"2023-09-23T20:54:13.949987+0000","last_became_peered":"2023-09-23T20:54:13.949987+0000","last_unstale":"2023-09-23T21:02:43.270287+0000","last_undegraded":"2023-09-23T21:02:43.270287+0000","last_fullsized":"2023-09-23T21:02:43.270287+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:46:17.765134+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":183,"reported_epoch":274,"state":"active+clean","last_fresh":"2023-09-23T21:02:43.276958+0000","last_change":"2023-09-23T20:56:18.141728+0000","last_active":"2023-09-23T21:02:43.276958+0000","last_peered":"2023-09-23T21:02:43.276958+0000","last_clean":"2023-09-23T21:02:43.276958+0000","last_became_active":"2023-09-23T20:56:18.137979+0000","last_became_peered":"2023-09-23T20:56:18.137979+0000","last_unstale":"2023-09-23T21:02:43.276958+0000","last_undegraded":"2023-09-23T21:02:43.276958+0000","last_fullsized":"2023-09-23T21:02:43.276958+0000","mapping_epoch":192,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":193,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:58:05.658443+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":152,"reported_epoch":274,"state":"active+clean","last_fresh":"2023-09-23T21:02:43.270363+0000","last_change":"2023-09-23T20:54:13.935307+0000","last_active":"2023-09-23T21:02:43.270363+0000","last_peered":"2023-09-23T21:02:43.270363+0000","last_clean":"2023-09-23T21:02:43.270363+0000","last_became_active":"2023-09-23T20:54:13.935000+0000","last_became_peered":"2023-09-23T20:54:13.935000+0000","last_unstale":"2023-09-23T21:02:43.270363+0000","last_undegraded":"2023-09-23T21:02:43.270363+0000","last_fullsized":"2023-09-23T21:02:43.270363+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:47:40.974082+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":263,"reported_epoch":274,"state":"active+clean","last_fresh":"2023-09-23T21:02:43.285995+0000","last_change":"2023-09-23T20:55:35.954870+0000","last_active":"2023-09-23T21:02:43.285995+0000","last_peered":"2023-09-23T21:02:43.285995+0000","last_clean":"2023-09-23T21:02:43.285995+0000","last_became_active":"2023-09-23T20:55:35.954568+0000","last_became_peered":"2023-09-23T20:55:35.954568+0000","last_unstale":"2023-09-23T21:02:43.285995+0000","last_undegraded":"2023-09-23T21:02:43.285995+0000","last_fullsized":"2023-09-23T21:02:43.285995+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:06:22.423566+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":262,"reported_epoch":274,"state":"active+clean","last_fresh":"2023-09-23T21:02:43.285193+0000","last_change":"2023-09-23T20:51:10.838334+0000","last_active":"2023-09-23T21:02:43.285193+0000","last_peered":"2023-09-23T21:02:43.285193+0000","last_clean":"2023-09-23T21:02:43.285193+0000","last_became_active":"2023-09-23T20:51:10.838026+0000","last_became_peered":"2023-09-23T20:51:10.838026+0000","last_unstale":"2023-09-23T21:02:43.285193+0000","last_undegraded":"2023-09-23T21:02:43.285193+0000","last_fullsized":"2023-09-23T21:02:43.285193+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:04:37.009457+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":267,"reported_epoch":274,"state":"active+clean","last_fresh":"2023-09-23T21:02:43.276559+0000","last_change":"2023-09-23T20:54:31.903139+0000","last_active":"2023-09-23T21:02:43.276559+0000","last_peered":"2023-09-23T21:02:43.276559+0000","last_clean":"2023-09-23T21:02:43.276559+0000","last_became_active":"2023-09-23T20:54:31.884520+0000","last_became_peered":"2023-09-23T20:54:31.884520+0000","last_unstale":"2023-09-23T21:02:43.276559+0000","last_undegraded":"2023-09-23T21:02:43.276559+0000","last_fullsized":"2023-09-23T21:02:43.276559+0000","mapping_epoch":136,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:39:34.269705+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1108,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":169,"reported_epoch":274,"state":"active+clean","last_fresh":"2023-09-23T21:02:43.269586+0000","last_change":"2023-09-23T20:55:35.951572+0000","last_active":"2023-09-23T21:02:43.269586+0000","last_peered":"2023-09-23T21:02:43.269586+0000","last_clean":"2023-09-23T21:02:43.269586+0000","last_became_active":"2023-09-23T20:55:35.951294+0000","last_became_peered":"2023-09-23T20:55:35.951294+0000","last_unstale":"2023-09-23T21:02:43.269586+0000","last_undegraded":"2023-09-23T21:02:43.269586+0000","last_fullsized":"2023-09-23T21:02:43.269586+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:36:32.447210+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":22,"num_bytes_recovered":5990,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":8,"num_pg":16,"stat_sum":{"num_bytes":6834028544,"num_objects":104279,"num_object_clones":0,"num_object_copies":312837,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":104279,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":104279,"num_write_kb":6673856,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":20429078528,"data_stored":20429078528,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":104179,"ondisk_log_size":104179,"up":48,"acting":48,"num_store_stats":8},{"poolid":1,"num_pg":21,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3381,"ondisk_log_size":3381,"up":63,"acting":63,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201549,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2297552,"kb_used_data":2076104,"kb_used_omap":0,"kb_used_meta":221440,"kb_avail":91447600,"statfs":{"total":95995035648,"available":93642342400,"internally_reserved":0,"allocated":2125930496,"data_stored":2125026378,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":226754560},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:01:49 2023","interfaces":[{"interface":"back","average":{"1min":0.63700000000000001,"5min":2.2869999999999999,"15min":1.3120000000000001},"min":{"1min":0.22800000000000001,"5min":0.22800000000000001,"15min":0.21099999999999999},"max":{"1min":1.798,"5min":43.994999999999997,"15min":43.994999999999997},"last":5.0919999999999996},{"interface":"front","average":{"1min":1.002,"5min":2.3940000000000001,"15min":1.4179999999999999},"min":{"1min":0.42899999999999999,"5min":0.318,"15min":0.218},"max":{"1min":3.633,"5min":44.206000000000003,"15min":44.206000000000003},"last":5.0289999999999999}]},{"osd":1,"last update":"Sat Sep 23 21:02:30 2023","interfaces":[{"interface":"back","average":{"1min":1.034,"5min":1.3879999999999999,"15min":1.3340000000000001},"min":{"1min":0.26300000000000001,"5min":0.253,"15min":0.23300000000000001},"max":{"1min":5.1630000000000003,"5min":10.622,"15min":22.591999999999999},"last":2.8570000000000002},{"interface":"front","average":{"1min":0.91900000000000004,"5min":1.647,"15min":1.2669999999999999},"min":{"1min":0.27100000000000002,"5min":0.25800000000000001,"15min":0.23699999999999999},"max":{"1min":3.2749999999999999,"5min":13.981,"15min":21.437000000000001},"last":2.9550000000000001}]},{"osd":2,"last update":"Sat Sep 23 21:02:00 2023","interfaces":[{"interface":"back","average":{"1min":0.73899999999999999,"5min":2.1920000000000002,"15min":1.429},"min":{"1min":0.313,"5min":0.26800000000000002,"15min":0.24399999999999999},"max":{"1min":2.0899999999999999,"5min":10.805999999999999,"15min":10.805999999999999},"last":3.7909999999999999},{"interface":"front","average":{"1min":0.78300000000000003,"5min":2.367,"15min":1.482},"min":{"1min":0.27900000000000003,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":2.2240000000000002,"5min":45.25,"15min":45.25},"last":3.621}]},{"osd":3,"last update":"Sat Sep 23 21:02:38 2023","interfaces":[{"interface":"back","average":{"1min":2.0640000000000001,"5min":2.6459999999999999,"15min":2.097},"min":{"1min":0.34000000000000002,"5min":0.315,"15min":0.314},"max":{"1min":11.625999999999999,"5min":31.786000000000001,"15min":36.408000000000001},"last":1.458},{"interface":"front","average":{"1min":1.3069999999999999,"5min":2.6539999999999999,"15min":1.9650000000000001},"min":{"1min":0.309,"5min":0.309,"15min":0.26000000000000001},"max":{"1min":6.181,"5min":32.429000000000002,"15min":35.593000000000004},"last":4.2610000000000001}]},{"osd":4,"last update":"Sat Sep 23 21:02:30 2023","interfaces":[{"interface":"back","average":{"1min":0.71499999999999997,"5min":0.58999999999999997,"15min":0.63500000000000001},"min":{"1min":0.29399999999999998,"5min":0.25700000000000001,"15min":0.23699999999999999},"max":{"1min":2.2469999999999999,"5min":4.5410000000000004,"15min":5.8920000000000003},"last":0.38900000000000001},{"interface":"front","average":{"1min":1.0169999999999999,"5min":0.72599999999999998,"15min":0.67800000000000005},"min":{"1min":0.28000000000000003,"5min":0.27100000000000002,"15min":0.26300000000000001},"max":{"1min":6.3979999999999997,"5min":6.3979999999999997,"15min":6.3979999999999997},"last":0.40100000000000002}]},{"osd":5,"last update":"Sat Sep 23 21:02:30 2023","interfaces":[{"interface":"back","average":{"1min":1.4179999999999999,"5min":0.71399999999999997,"15min":0.69099999999999995},"min":{"1min":0.34300000000000003,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":5.2089999999999996,"5min":5.2089999999999996,"15min":7.0389999999999997},"last":0.33400000000000002},{"interface":"front","average":{"1min":0.76200000000000001,"5min":0.67600000000000005,"15min":0.65100000000000002},"min":{"1min":0.33300000000000002,"5min":0.30199999999999999,"15min":0.27800000000000002},"max":{"1min":2.2530000000000001,"5min":4.5999999999999996,"15min":4.5999999999999996},"last":0.439}]},{"osd":6,"last update":"Sat Sep 23 21:02:46 2023","interfaces":[{"interface":"back","average":{"1min":1.1539999999999999,"5min":0.82699999999999996,"15min":0.72399999999999998},"min":{"1min":0.35199999999999998,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":6.1219999999999999,"5min":6.1219999999999999,"15min":6.1219999999999999},"last":0.54900000000000004},{"interface":"front","average":{"1min":1.081,"5min":0.86099999999999999,"15min":0.81999999999999995},"min":{"1min":0.32300000000000001,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":6.0919999999999996,"5min":6.0919999999999996,"15min":6.0919999999999996},"last":0.58099999999999996}]}]},{"osd":6,"up_from":164,"seq":704374636633,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2239872,"kb_used_data":2076728,"kb_used_omap":10,"kb_used_meta":163125,"kb_avail":91505280,"statfs":{"total":95995035648,"available":93701406720,"internally_reserved":0,"allocated":2126569472,"data_stored":2125681327,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":10328,"internal_metadata":167040936},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:02:49 2023","interfaces":[{"interface":"back","average":{"1min":2.7090000000000001,"5min":2.0169999999999999,"15min":1.3160000000000001},"min":{"1min":0.23499999999999999,"5min":0.23499999999999999,"15min":0.20999999999999999},"max":{"1min":14.151999999999999,"5min":14.151999999999999,"15min":14.151999999999999},"last":4.5490000000000004},{"interface":"front","average":{"1min":2.6709999999999998,"5min":2.3130000000000002,"15min":1.4530000000000001},"min":{"1min":0.29899999999999999,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":10.962,"5min":11.023,"15min":11.023},"last":5.0179999999999998}]},{"osd":1,"last update":"Sat Sep 23 21:02:49 2023","interfaces":[{"interface":"back","average":{"1min":2.1259999999999999,"5min":1.583,"15min":1.1060000000000001},"min":{"1min":0.26400000000000001,"5min":0.26400000000000001,"15min":0.25600000000000001},"max":{"1min":6.024,"5min":8.4309999999999992,"15min":8.4309999999999992},"last":0.86399999999999999},{"interface":"front","average":{"1min":2.3420000000000001,"5min":2.0070000000000001,"15min":1.2589999999999999},"min":{"1min":0.317,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":5.431,"5min":10.518000000000001,"15min":10.518000000000001},"last":2.0710000000000002}]},{"osd":2,"last update":"Sat Sep 23 21:02:49 2023","interfaces":[{"interface":"back","average":{"1min":3.9449999999999998,"5min":2.496,"15min":1.9990000000000001},"min":{"1min":0.28499999999999998,"5min":0.28499999999999998,"15min":0.27700000000000002},"max":{"1min":14.029999999999999,"5min":14.029999999999999,"15min":14.029999999999999},"last":1.712},{"interface":"front","average":{"1min":2.8500000000000001,"5min":2.3889999999999998,"15min":1.5880000000000001},"min":{"1min":0.503,"5min":0.247,"15min":0.247},"max":{"1min":8.3670000000000009,"5min":20.195,"15min":20.195},"last":1.7769999999999999}]},{"osd":3,"last update":"Sat Sep 23 21:02:49 2023","interfaces":[{"interface":"back","average":{"1min":3.04,"5min":2.3330000000000002,"15min":1.528},"min":{"1min":0.28799999999999998,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":9.6349999999999998,"5min":14.144,"15min":14.144},"last":2.5670000000000002},{"interface":"front","average":{"1min":2.448,"5min":2.0390000000000001,"15min":1.7549999999999999},"min":{"1min":0.33100000000000002,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":16.847999999999999,"5min":16.847999999999999,"15min":16.847999999999999},"last":2.7010000000000001}]},{"osd":4,"last update":"Sat Sep 23 21:02:49 2023","interfaces":[{"interface":"back","average":{"1min":1.4259999999999999,"5min":0.80200000000000005,"15min":0.94199999999999995},"min":{"1min":0.251,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":5.3920000000000003,"5min":5.3920000000000003,"15min":9.0079999999999991},"last":0.28199999999999997},{"interface":"front","average":{"1min":1.718,"5min":0.76400000000000001,"15min":0.60299999999999998},"min":{"1min":0.26600000000000001,"5min":0.26600000000000001,"15min":0.25800000000000001},"max":{"1min":4.8760000000000003,"5min":4.8760000000000003,"15min":4.8760000000000003},"last":0.29299999999999998}]},{"osd":5,"last update":"Sat Sep 23 21:02:49 2023","interfaces":[{"interface":"back","average":{"1min":1.8440000000000001,"5min":0.97299999999999998,"15min":0.69399999999999995},"min":{"1min":0.31,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":5.9000000000000004,"5min":5.9000000000000004,"15min":5.9000000000000004},"last":0.311},{"interface":"front","average":{"1min":1.5609999999999999,"5min":0.89900000000000002,"15min":0.83099999999999996},"min":{"1min":0.30599999999999999,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":5.3220000000000001,"5min":5.3220000000000001,"15min":5.3220000000000001},"last":0.40300000000000002}]},{"osd":7,"last update":"Sat Sep 23 21:02:49 2023","interfaces":[{"interface":"back","average":{"1min":1.9530000000000001,"5min":0.93000000000000005,"15min":0.93100000000000005},"min":{"1min":0.33400000000000002,"5min":0.312,"15min":0.312},"max":{"1min":5.0490000000000004,"5min":5.0490000000000004,"15min":7.8570000000000002},"last":0.375},{"interface":"front","average":{"1min":1.1499999999999999,"5min":0.77900000000000003,"15min":0.65100000000000002},"min":{"1min":0.35399999999999998,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":3.754,"5min":3.754,"15min":8.282},"last":0.81899999999999995}]}]},{"osd":1,"up_from":88,"seq":377957122204,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3432132,"kb_used_data":3082940,"kb_used_omap":9,"kb_used_meta":349174,"kb_avail":90313020,"statfs":{"total":95995035648,"available":92480532480,"internally_reserved":0,"allocated":3156930560,"data_stored":3155633709,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9239,"internal_metadata":357555177},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:02:31 2023","interfaces":[{"interface":"back","average":{"1min":3.7730000000000001,"5min":3.081,"15min":2.0009999999999999},"min":{"1min":0.25,"5min":0.20699999999999999,"15min":0.20699999999999999},"max":{"1min":45.542000000000002,"5min":45.542000000000002,"15min":45.542000000000002},"last":0.33000000000000002},{"interface":"front","average":{"1min":4.1929999999999996,"5min":3.7440000000000002,"15min":2.3100000000000001},"min":{"1min":0.28499999999999998,"5min":0.247,"15min":0.22500000000000001},"max":{"1min":46.353000000000002,"5min":46.353000000000002,"15min":46.353000000000002},"last":3.157}]},{"osd":2,"last update":"Sat Sep 23 21:02:00 2023","interfaces":[{"interface":"back","average":{"1min":0.79700000000000004,"5min":2.1840000000000002,"15min":1.478},"min":{"1min":0.20399999999999999,"5min":0.20399999999999999,"15min":0.20300000000000001},"max":{"1min":2.7759999999999998,"5min":13.752000000000001,"15min":14.335000000000001},"last":0.28599999999999998},{"interface":"front","average":{"1min":1.502,"5min":3.863,"15min":2.323},"min":{"1min":0.249,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":7.2649999999999997,"5min":18.026,"15min":38.951000000000001},"last":3.036}]},{"osd":3,"last update":"Sat Sep 23 21:02:31 2023","interfaces":[{"interface":"back","average":{"1min":2.3900000000000001,"5min":2.8410000000000002,"15min":2.0110000000000001},"min":{"1min":0.26500000000000001,"5min":0.246,"15min":0.246},"max":{"1min":8.7390000000000008,"5min":22.686,"15min":25.314},"last":10.587999999999999},{"interface":"front","average":{"1min":1.5509999999999999,"5min":2.4900000000000002,"15min":1.7629999999999999},"min":{"1min":0.25600000000000001,"5min":0.25600000000000001,"15min":0.20599999999999999},"max":{"1min":10.157999999999999,"5min":14.473000000000001,"15min":19.881},"last":7.1609999999999996}]},{"osd":4,"last update":"Sat Sep 23 21:02:31 2023","interfaces":[{"interface":"back","average":{"1min":3.5030000000000001,"5min":2.0470000000000002,"15min":1.528},"min":{"1min":0.28399999999999997,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":41.823,"5min":41.823,"15min":41.823},"last":2.04},{"interface":"front","average":{"1min":3.097,"5min":1.9359999999999999,"15min":1.3420000000000001},"min":{"1min":0.33300000000000002,"5min":0.31900000000000001,"15min":0.27600000000000002},"max":{"1min":41.423000000000002,"5min":41.423000000000002,"15min":41.423000000000002},"last":0.374}]},{"osd":5,"last update":"Sat Sep 23 21:02:31 2023","interfaces":[{"interface":"back","average":{"1min":1.446,"5min":2.2789999999999999,"15min":1.55},"min":{"1min":0.32000000000000001,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":7.5570000000000004,"5min":15.041,"15min":15.041},"last":3.1190000000000002},{"interface":"front","average":{"1min":1.5649999999999999,"5min":1.6830000000000001,"15min":1.431},"min":{"1min":0.36099999999999999,"5min":0.33200000000000002,"15min":0.32200000000000001},"max":{"1min":5.4930000000000003,"5min":18.280999999999999,"15min":18.280999999999999},"last":2.0790000000000002}]},{"osd":6,"last update":"Sat Sep 23 21:02:51 2023","interfaces":[{"interface":"back","average":{"1min":2.4129999999999998,"5min":1.8720000000000001,"15min":1.4990000000000001},"min":{"1min":0.36199999999999999,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":5.4690000000000003,"5min":18.984999999999999,"15min":18.984999999999999},"last":2.109},{"interface":"front","average":{"1min":1.6519999999999999,"5min":1.6000000000000001,"15min":1.415},"min":{"1min":0.33400000000000002,"5min":0.33400000000000002,"15min":0.309},"max":{"1min":4.7839999999999998,"5min":8.7530000000000001,"15min":8.7530000000000001},"last":0.40799999999999997}]},{"osd":7,"last update":"Sat Sep 23 21:02:31 2023","interfaces":[{"interface":"back","average":{"1min":1.238,"5min":1.6910000000000001,"15min":1.3149999999999999},"min":{"1min":0.39600000000000002,"5min":0.39600000000000002,"15min":0.35699999999999998},"max":{"1min":3.9900000000000002,"5min":8.7769999999999992,"15min":11.186999999999999},"last":0.441},{"interface":"front","average":{"1min":2.0329999999999999,"5min":2.3570000000000002,"15min":1.5940000000000001},"min":{"1min":0.38100000000000001,"5min":0.38100000000000001,"15min":0.34599999999999997},"max":{"1min":7.6100000000000003,"5min":15.098000000000001,"15min":15.098000000000001},"last":3.1850000000000001}]}]},{"osd":0,"up_from":83,"seq":356482285730,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2828452,"kb_used_data":2495452,"kb_used_omap":28,"kb_used_meta":332963,"kb_avail":90916700,"statfs":{"total":95995035648,"available":93098700800,"internally_reserved":0,"allocated":2555342848,"data_stored":2554075657,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":28848,"internal_metadata":340954960},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 21:02:21 2023","interfaces":[{"interface":"back","average":{"1min":2.2930000000000001,"5min":3.782,"15min":2.181},"min":{"1min":0.19500000000000001,"5min":0.19500000000000001,"15min":0.191},"max":{"1min":22.411000000000001,"5min":22.411000000000001,"15min":22.411000000000001},"last":0.98699999999999999},{"interface":"front","average":{"1min":3.0870000000000002,"5min":3.8799999999999999,"15min":2.1970000000000001},"min":{"1min":0.29299999999999998,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":22.474,"5min":22.474,"15min":22.474},"last":7.3070000000000004}]},{"osd":2,"last update":"Sat Sep 23 21:01:55 2023","interfaces":[{"interface":"back","average":{"1min":1.052,"5min":3.2599999999999998,"15min":2.1419999999999999},"min":{"1min":0.23499999999999999,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":8.2479999999999993,"5min":21.053000000000001,"15min":21.053000000000001},"last":7.202},{"interface":"front","average":{"1min":1.3,"5min":3.9390000000000001,"15min":2.2530000000000001},"min":{"1min":0.23100000000000001,"5min":0.23100000000000001,"15min":0.21099999999999999},"max":{"1min":6.6189999999999998,"5min":21.747,"15min":21.747},"last":4.3819999999999997}]},{"osd":3,"last update":"Sat Sep 23 21:02:21 2023","interfaces":[{"interface":"back","average":{"1min":1.6990000000000001,"5min":3.6360000000000001,"15min":2.1549999999999998},"min":{"1min":0.29599999999999999,"5min":0.27100000000000002,"15min":0.25800000000000001},"max":{"1min":11.617000000000001,"5min":21.184000000000001,"15min":21.184000000000001},"last":1.0349999999999999},{"interface":"front","average":{"1min":2.2280000000000002,"5min":3.8839999999999999,"15min":2.3380000000000001},"min":{"1min":0.32900000000000001,"5min":0.27400000000000002,"15min":0.23300000000000001},"max":{"1min":8.4510000000000005,"5min":21.167000000000002,"15min":21.167000000000002},"last":7.3369999999999997}]},{"osd":4,"last update":"Sat Sep 23 21:01:55 2023","interfaces":[{"interface":"back","average":{"1min":1.2390000000000001,"5min":2.8599999999999999,"15min":1.7230000000000001},"min":{"1min":0.28599999999999998,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":6.5620000000000003,"5min":21.594999999999999,"15min":21.594999999999999},"last":4.1879999999999997},{"interface":"front","average":{"1min":0.80400000000000005,"5min":2.2549999999999999,"15min":1.496},"min":{"1min":0.32900000000000001,"5min":0.311,"15min":0.28100000000000003},"max":{"1min":3.4289999999999998,"5min":13.904,"15min":18.724},"last":0.61399999999999999}]},{"osd":5,"last update":"Sat Sep 23 21:01:55 2023","interfaces":[{"interface":"back","average":{"1min":1.1459999999999999,"5min":1.9670000000000001,"15min":1.4339999999999999},"min":{"1min":0.34999999999999998,"5min":0.309,"15min":0.309},"max":{"1min":8.1899999999999995,"5min":8.8970000000000002,"15min":8.8970000000000002},"last":0.47099999999999997},{"interface":"front","average":{"1min":1.256,"5min":2.7000000000000002,"15min":1.724},"min":{"1min":0.373,"5min":0.373,"15min":0.34999999999999998},"max":{"1min":6.601,"5min":20.512,"15min":20.512},"last":4.2329999999999997}]},{"osd":6,"last update":"Sat Sep 23 21:01:52 2023","interfaces":[{"interface":"back","average":{"1min":0.78000000000000003,"5min":2.3199999999999998,"15min":1.8779999999999999},"min":{"1min":0.38,"5min":0.37,"15min":0.37},"max":{"1min":3.472,"5min":13.933,"15min":13.933},"last":0.64600000000000002},{"interface":"front","average":{"1min":0.76500000000000001,"5min":2.4180000000000001,"15min":1.97},"min":{"1min":0.379,"5min":0.377,"15min":0.377},"max":{"1min":3.5169999999999999,"5min":13.962,"15min":13.962},"last":0.58699999999999997}]},{"osd":7,"last update":"Sat Sep 23 21:01:55 2023","interfaces":[{"interface":"back","average":{"1min":1.151,"5min":2.0699999999999998,"15min":1.4870000000000001},"min":{"1min":0.40600000000000003,"5min":0.38100000000000001,"15min":0.379},"max":{"1min":8.2210000000000001,"5min":8.7729999999999997,"15min":8.7729999999999997},"last":0.50900000000000001},{"interface":"front","average":{"1min":0.86799999999999999,"5min":2.3220000000000001,"15min":1.5289999999999999},"min":{"1min":0.45400000000000001,"5min":0.377,"15min":0.377},"max":{"1min":3.4500000000000002,"5min":13.856999999999999,"15min":18.673999999999999},"last":0.55100000000000005}]}]},{"osd":2,"up_from":93,"seq":399431958677,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2992344,"kb_used_data":2628752,"kb_used_omap":81,"kb_used_meta":363502,"kb_avail":90752808,"statfs":{"total":95995035648,"available":92930875392,"internally_reserved":0,"allocated":2691842048,"data_stored":2690576123,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":83750,"internal_metadata":372226266},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:01:51 2023","interfaces":[{"interface":"back","average":{"1min":0.76900000000000002,"5min":3.0600000000000001,"15min":1.671},"min":{"1min":0.22900000000000001,"5min":0.189,"15min":0.17999999999999999},"max":{"1min":2.923,"5min":29.498000000000001,"15min":29.498000000000001},"last":0.56499999999999995},{"interface":"front","average":{"1min":0.624,"5min":3.4249999999999998,"15min":1.946},"min":{"1min":0.193,"5min":0.193,"15min":0.193},"max":{"1min":1.9059999999999999,"5min":17.251000000000001,"15min":17.251000000000001},"last":1.988}]},{"osd":1,"last update":"Sat Sep 23 21:01:51 2023","interfaces":[{"interface":"back","average":{"1min":1.163,"5min":2.5800000000000001,"15min":1.831},"min":{"1min":0.246,"5min":0.21199999999999999,"15min":0.20200000000000001},"max":{"1min":6.077,"5min":14.709,"15min":16.081},"last":2.1600000000000001},{"interface":"front","average":{"1min":1.107,"5min":3.7879999999999998,"15min":2.3919999999999999},"min":{"1min":0.33500000000000002,"5min":0.26300000000000001,"15min":0.23599999999999999},"max":{"1min":6.125,"5min":26.552,"15min":26.552},"last":2.0459999999999998}]},{"osd":3,"last update":"Sat Sep 23 21:02:34 2023","interfaces":[{"interface":"back","average":{"1min":2.137,"5min":4.1699999999999999,"15min":2.524},"min":{"1min":0.45500000000000002,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":5.8310000000000004,"5min":41.057000000000002,"15min":41.057000000000002},"last":2.0169999999999999},{"interface":"front","average":{"1min":1.9319999999999999,"5min":4.4340000000000002,"15min":2.8300000000000001},"min":{"1min":0.25,"5min":0.25,"15min":0.221},"max":{"1min":9.4870000000000001,"5min":41.012,"15min":41.012},"last":10.550000000000001}]},{"osd":4,"last update":"Sat Sep 23 21:01:51 2023","interfaces":[{"interface":"back","average":{"1min":0.72799999999999998,"5min":1.706,"15min":1.3140000000000001},"min":{"1min":0.28399999999999997,"5min":0.28399999999999997,"15min":0.26400000000000001},"max":{"1min":2.8290000000000002,"5min":10.045,"15min":10.045},"last":1.857},{"interface":"front","average":{"1min":0.78000000000000003,"5min":2.1560000000000001,"15min":1.538},"min":{"1min":0.36399999999999999,"5min":0.32400000000000001,"15min":0.313},"max":{"1min":3.0550000000000002,"5min":14.039999999999999,"15min":14.039999999999999},"last":2.0990000000000002}]},{"osd":5,"last update":"Sat Sep 23 21:01:51 2023","interfaces":[{"interface":"back","average":{"1min":0.65300000000000002,"5min":1.637,"15min":1.3959999999999999},"min":{"1min":0.30399999999999999,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":1.9119999999999999,"5min":10.069000000000001,"15min":10.069000000000001},"last":1.9079999999999999},{"interface":"front","average":{"1min":0.66500000000000004,"5min":2.073,"15min":1.534},"min":{"1min":0.31900000000000001,"5min":0.31900000000000001,"15min":0.30299999999999999},"max":{"1min":2.0880000000000001,"5min":13.98,"15min":13.98},"last":2.0760000000000001}]},{"osd":6,"last update":"Sat Sep 23 21:02:50 2023","interfaces":[{"interface":"back","average":{"1min":2.4369999999999998,"5min":2.3359999999999999,"15min":1.909},"min":{"1min":0.40100000000000002,"5min":0.35699999999999998,"15min":0.312},"max":{"1min":7.0670000000000002,"5min":14.016,"15min":14.244999999999999},"last":2.1459999999999999},{"interface":"front","average":{"1min":2.4790000000000001,"5min":2.431,"15min":1.5569999999999999},"min":{"1min":0.41499999999999998,"5min":0.38,"15min":0.374},"max":{"1min":7.4210000000000003,"5min":14.068,"15min":14.068},"last":2.1230000000000002}]},{"osd":7,"last update":"Sat Sep 23 21:01:51 2023","interfaces":[{"interface":"back","average":{"1min":0.84799999999999998,"5min":1.4359999999999999,"15min":1.155},"min":{"1min":0.45600000000000002,"5min":0.315,"15min":0.315},"max":{"1min":2.98,"5min":13.427,"15min":13.427},"last":0.52700000000000002},{"interface":"front","average":{"1min":0.79100000000000004,"5min":1.643,"15min":1.4099999999999999},"min":{"1min":0.36699999999999999,"5min":0.35099999999999998,"15min":0.34499999999999997},"max":{"1min":2.883,"5min":10.092000000000001,"15min":10.092000000000001},"last":1.9330000000000001}]}]},{"osd":3,"up_from":98,"seq":420906795151,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2988148,"kb_used_data":2530988,"kb_used_omap":7,"kb_used_meta":457144,"kb_avail":90757004,"statfs":{"total":95995035648,"available":92935172096,"internally_reserved":0,"allocated":2591731712,"data_stored":2590505706,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7576,"internal_metadata":468116072},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:02:32 2023","interfaces":[{"interface":"back","average":{"1min":1.288,"5min":3.6619999999999999,"15min":2.4039999999999999},"min":{"1min":0.188,"5min":0.188,"15min":0.188},"max":{"1min":4.7949999999999999,"5min":27.962,"15min":27.962},"last":11.717000000000001},{"interface":"front","average":{"1min":2.8519999999999999,"5min":3.794,"15min":2.2709999999999999},"min":{"1min":0.20499999999999999,"5min":0.19800000000000001,"15min":0.19800000000000001},"max":{"1min":10.497,"5min":28.058,"15min":28.058},"last":1.645}]},{"osd":1,"last update":"Sat Sep 23 21:02:32 2023","interfaces":[{"interface":"back","average":{"1min":5.2779999999999996,"5min":3.8439999999999999,"15min":2.371},"min":{"1min":0.23999999999999999,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":74.093000000000004,"5min":74.093000000000004,"15min":74.093000000000004},"last":0.56999999999999995},{"interface":"front","average":{"1min":5.1210000000000004,"5min":3.887,"15min":2.4969999999999999},"min":{"1min":0.27700000000000002,"5min":0.23400000000000001,"15min":0.218},"max":{"1min":76.527000000000001,"5min":76.527000000000001,"15min":76.527000000000001},"last":7.9640000000000004}]},{"osd":2,"last update":"Sat Sep 23 21:02:32 2023","interfaces":[{"interface":"back","average":{"1min":2.101,"5min":4.1749999999999998,"15min":2.399},"min":{"1min":0.23599999999999999,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":6.5549999999999997,"5min":28.251999999999999,"15min":28.251999999999999},"last":0.96099999999999997},{"interface":"front","average":{"1min":2.097,"5min":3.3199999999999998,"15min":2.3759999999999999},"min":{"1min":0.22,"5min":0.22,"15min":0.22},"max":{"1min":12.68,"5min":16.213999999999999,"15min":25.305},"last":18.321000000000002}]},{"osd":4,"last update":"Sat Sep 23 21:02:32 2023","interfaces":[{"interface":"back","average":{"1min":1.75,"5min":2.5680000000000001,"15min":1.6519999999999999},"min":{"1min":0.29399999999999998,"5min":0.29399999999999998,"15min":0.29299999999999998},"max":{"1min":7.6820000000000004,"5min":18.239000000000001,"15min":18.239000000000001},"last":0.623},{"interface":"front","average":{"1min":1.5469999999999999,"5min":2.0030000000000001,"15min":1.6559999999999999},"min":{"1min":0.29499999999999998,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":6.0060000000000002,"5min":14.808,"15min":22.853000000000002},"last":8.0579999999999998}]},{"osd":5,"last update":"Sat Sep 23 21:02:32 2023","interfaces":[{"interface":"back","average":{"1min":1.7410000000000001,"5min":2.6739999999999999,"15min":1.6679999999999999},"min":{"1min":0.32900000000000001,"5min":0.32900000000000001,"15min":0.32500000000000001},"max":{"1min":8.0459999999999994,"5min":18.324999999999999,"15min":18.324999999999999},"last":2.423},{"interface":"front","average":{"1min":2.4860000000000002,"5min":2.9279999999999999,"15min":1.746},"min":{"1min":0.40899999999999997,"5min":0.33300000000000002,"15min":0.33100000000000002},"max":{"1min":10.558,"5min":13.529999999999999,"15min":13.529999999999999},"last":0.66800000000000004}]},{"osd":6,"last update":"Sat Sep 23 21:01:52 2023","interfaces":[{"interface":"back","average":{"1min":0.97199999999999998,"5min":1.9490000000000001,"15min":1.2829999999999999},"min":{"1min":0.40200000000000002,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":4.1230000000000002,"5min":14.833,"15min":14.833},"last":8.2100000000000009},{"interface":"front","average":{"1min":0.99099999999999999,"5min":2.5270000000000001,"15min":1.7829999999999999},"min":{"1min":0.377,"5min":0.377,"15min":0.377},"max":{"1min":2.242,"5min":13.576000000000001,"15min":13.576000000000001},"last":2.403}]},{"osd":7,"last update":"Sat Sep 23 21:02:32 2023","interfaces":[{"interface":"back","average":{"1min":2.3420000000000001,"5min":2.7240000000000002,"15min":1.8440000000000001},"min":{"1min":0.40600000000000003,"5min":0.39200000000000002,"15min":0.36099999999999999},"max":{"1min":10.715999999999999,"5min":18.289999999999999,"15min":18.289999999999999},"last":0.59999999999999998},{"interface":"front","average":{"1min":2.8959999999999999,"5min":3.0880000000000001,"15min":1.8440000000000001},"min":{"1min":0.46500000000000002,"5min":0.38900000000000001,"15min":0.379},"max":{"1min":11.798999999999999,"5min":13.603999999999999,"15min":13.603999999999999},"last":0.68999999999999995}]}]},{"osd":4,"up_from":31,"seq":133143986458,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2260944,"kb_used_data":2085320,"kb_used_omap":0,"kb_used_meta":175616,"kb_avail":91484208,"statfs":{"total":95995035648,"available":93679828992,"internally_reserved":0,"allocated":2135367680,"data_stored":2134463815,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":179830784},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:01:55 2023","interfaces":[{"interface":"back","average":{"1min":0.55900000000000005,"5min":1.7729999999999999,"15min":1.216},"min":{"1min":0.254,"5min":0.214,"15min":0.214},"max":{"1min":1.712,"5min":11.784000000000001,"15min":11.784000000000001},"last":2.3610000000000002},{"interface":"front","average":{"1min":0.66000000000000003,"5min":1.8200000000000001,"15min":1.379},"min":{"1min":0.31900000000000001,"5min":0.27700000000000002,"15min":0.25800000000000001},"max":{"1min":2.1360000000000001,"5min":11.766,"15min":11.766},"last":2.371}]},{"osd":1,"last update":"Sat Sep 23 21:02:24 2023","interfaces":[{"interface":"back","average":{"1min":0.94899999999999995,"5min":1.696,"15min":1.2969999999999999},"min":{"1min":0.255,"5min":0.246,"15min":0.23599999999999999},"max":{"1min":5.0229999999999997,"5min":10.153,"15min":10.153},"last":1.069},{"interface":"front","average":{"1min":1.0940000000000001,"5min":2.2280000000000002,"15min":1.3700000000000001},"min":{"1min":0.309,"5min":0.26100000000000001,"15min":0.23200000000000001},"max":{"1min":7.3209999999999997,"5min":12.292999999999999,"15min":12.292999999999999},"last":1.4690000000000001}]},{"osd":2,"last update":"Sat Sep 23 21:01:55 2023","interfaces":[{"interface":"back","average":{"1min":0.68300000000000005,"5min":1.927,"15min":1.3899999999999999},"min":{"1min":0.33400000000000002,"5min":0.32500000000000001,"15min":0.29099999999999998},"max":{"1min":1.883,"5min":9.8469999999999995,"15min":15.288},"last":1.4950000000000001},{"interface":"front","average":{"1min":0.75700000000000001,"5min":1.8680000000000001,"15min":1.4950000000000001},"min":{"1min":0.32700000000000001,"5min":0.28599999999999998,"15min":0.28299999999999997},"max":{"1min":2.1789999999999998,"5min":12.855,"15min":15.262},"last":2.734}]},{"osd":3,"last update":"Sat Sep 23 21:02:24 2023","interfaces":[{"interface":"back","average":{"1min":1.514,"5min":2.3109999999999999,"15min":1.976},"min":{"1min":0.42699999999999999,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":7.1609999999999996,"5min":12.106999999999999,"15min":31.562999999999999},"last":2.8879999999999999},{"interface":"front","average":{"1min":1.3069999999999999,"5min":2.3149999999999999,"15min":2.0110000000000001},"min":{"1min":0.499,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":7.1429999999999998,"5min":12.477,"15min":31.574000000000002},"last":2.8769999999999998}]},{"osd":5,"last update":"Sat Sep 23 21:02:43 2023","interfaces":[{"interface":"back","average":{"1min":0.94399999999999995,"5min":0.59499999999999997,"15min":0.55100000000000005},"min":{"1min":0.27200000000000002,"5min":0.251,"15min":0.22700000000000001},"max":{"1min":6.6470000000000002,"5min":6.6470000000000002,"15min":6.6470000000000002},"last":1.409},{"interface":"front","average":{"1min":1.03,"5min":0.84299999999999997,"15min":0.68100000000000005},"min":{"1min":0.32400000000000001,"5min":0.27600000000000002,"15min":0.251},"max":{"1min":5.3120000000000003,"5min":5.3120000000000003,"15min":7.7619999999999996},"last":1.5489999999999999}]},{"osd":6,"last update":"Sat Sep 23 21:01:51 2023","interfaces":[{"interface":"back","average":{"1min":0.45400000000000001,"5min":0.73399999999999999,"15min":0.56699999999999995},"min":{"1min":0.29899999999999999,"5min":0.26900000000000002,"15min":0.26500000000000001},"max":{"1min":0.65700000000000003,"5min":4.6909999999999998,"15min":4.6909999999999998},"last":0.30299999999999999},{"interface":"front","average":{"1min":0.54200000000000004,"5min":0.69699999999999995,"15min":0.66100000000000003},"min":{"1min":0.38,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":0.81799999999999995,"5min":3.456,"15min":3.456},"last":1.4390000000000001}]},{"osd":7,"last update":"Sat Sep 23 21:02:24 2023","interfaces":[{"interface":"back","average":{"1min":0.61399999999999999,"5min":0.68799999999999994,"15min":0.69899999999999995},"min":{"1min":0.33700000000000002,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":1.248,"5min":3.4940000000000002,"15min":7.383},"last":1.639},{"interface":"front","average":{"1min":0.72699999999999998,"5min":0.74199999999999999,"15min":0.70099999999999996},"min":{"1min":0.377,"5min":0.32700000000000001,"15min":0.28199999999999997},"max":{"1min":2.1699999999999999,"5min":5.6980000000000004,"15min":7.8310000000000004},"last":1.764}]}]},{"osd":5,"up_from":39,"seq":167503724823,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3578948,"kb_used_data":3389180,"kb_used_omap":0,"kb_used_meta":189760,"kb_avail":90166204,"statfs":{"total":95995035648,"available":92330192896,"internally_reserved":0,"allocated":3470520320,"data_stored":3469625031,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":194314240},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:02:47 2023","interfaces":[{"interface":"back","average":{"1min":2.1619999999999999,"5min":2.181,"15min":1.27},"min":{"1min":0.22500000000000001,"5min":0.22500000000000001,"15min":0.20399999999999999},"max":{"1min":8.3249999999999993,"5min":12.795999999999999,"15min":12.795999999999999},"last":3.0529999999999999},{"interface":"front","average":{"1min":1.7589999999999999,"5min":2.1389999999999998,"15min":1.4199999999999999},"min":{"1min":0.23400000000000001,"5min":0.23400000000000001,"15min":0.223},"max":{"1min":6.6230000000000002,"5min":16.219999999999999,"15min":16.219999999999999},"last":3.4359999999999999}]},{"osd":1,"last update":"Sat Sep 23 21:02:20 2023","interfaces":[{"interface":"back","average":{"1min":4.8710000000000004,"5min":2.661,"15min":1.585},"min":{"1min":0.26800000000000002,"5min":0.248,"15min":0.23300000000000001},"max":{"1min":62.320999999999998,"5min":62.320999999999998,"15min":62.320999999999998},"last":3.581},{"interface":"front","average":{"1min":4.6079999999999997,"5min":2.7040000000000002,"15min":1.6020000000000001},"min":{"1min":0.30499999999999999,"5min":0.27200000000000002,"15min":0.22800000000000001},"max":{"1min":61.722999999999999,"5min":61.722999999999999,"15min":61.722999999999999},"last":3.0369999999999999}]},{"osd":2,"last update":"Sat Sep 23 21:01:59 2023","interfaces":[{"interface":"back","average":{"1min":0.82699999999999996,"5min":1.821,"15min":1.2649999999999999},"min":{"1min":0.23499999999999999,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":2.3090000000000002,"5min":10.579000000000001,"15min":12.651},"last":3.327},{"interface":"front","average":{"1min":0.92300000000000004,"5min":2.1669999999999998,"15min":1.4139999999999999},"min":{"1min":0.34000000000000002,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":2.8879999999999999,"5min":12.300000000000001,"15min":12.300000000000001},"last":1.1100000000000001}]},{"osd":3,"last update":"Sat Sep 23 21:02:20 2023","interfaces":[{"interface":"back","average":{"1min":6.7009999999999996,"5min":3.8119999999999998,"15min":2.1030000000000002},"min":{"1min":0.33500000000000002,"5min":0.33000000000000002,"15min":0.28599999999999998},"max":{"1min":61.667999999999999,"5min":61.667999999999999,"15min":61.667999999999999},"last":1.7050000000000001},{"interface":"front","average":{"1min":7.0460000000000003,"5min":3.2389999999999999,"15min":1.78},"min":{"1min":0.377,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":62.276000000000003,"5min":62.276000000000003,"15min":62.276000000000003},"last":0.48899999999999999}]},{"osd":4,"last update":"Sat Sep 23 21:02:37 2023","interfaces":[{"interface":"back","average":{"1min":0.96399999999999997,"5min":0.67400000000000004,"15min":0.59699999999999998},"min":{"1min":0.25700000000000001,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":3.9279999999999999,"5min":4.1520000000000001,"15min":4.1520000000000001},"last":0.378},{"interface":"front","average":{"1min":1.288,"5min":0.76000000000000001,"15min":0.64600000000000002},"min":{"1min":0.28000000000000003,"5min":0.27400000000000002,"15min":0.23599999999999999},"max":{"1min":8.4350000000000005,"5min":8.4350000000000005,"15min":8.4350000000000005},"last":0.436}]},{"osd":6,"last update":"Sat Sep 23 21:02:47 2023","interfaces":[{"interface":"back","average":{"1min":1.494,"5min":0.86699999999999999,"15min":0.67900000000000005},"min":{"1min":0.32200000000000001,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":11.128,"5min":11.128,"15min":11.128},"last":0.57599999999999996},{"interface":"front","average":{"1min":1.619,"5min":0.92100000000000004,"15min":0.80400000000000005},"min":{"1min":0.36799999999999999,"5min":0.36299999999999999,"15min":0.33200000000000002},"max":{"1min":11.112,"5min":11.112,"15min":11.112},"last":0.45300000000000001}]},{"osd":7,"last update":"Sat Sep 23 21:02:20 2023","interfaces":[{"interface":"back","average":{"1min":1.1819999999999999,"5min":0.83799999999999997,"15min":0.71999999999999997},"min":{"1min":0.36099999999999999,"5min":0.316,"15min":0.30199999999999999},"max":{"1min":4.1310000000000002,"5min":4.4240000000000004,"15min":8.5649999999999995},"last":0.54300000000000004},{"interface":"front","average":{"1min":0.90100000000000002,"5min":0.74099999999999999,"15min":0.70199999999999996},"min":{"1min":0.35299999999999998,"5min":0.33900000000000002,"15min":0.27700000000000002},"max":{"1min":4.6479999999999997,"5min":4.6479999999999997,"15min":4.6479999999999997},"last":0.40899999999999997}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":2561933312,"data_stored":2561933312,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":2995650560,"data_stored":2995650560,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":2529820672,"data_stored":2529820672,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":2588934144,"data_stored":2588934144,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":2137260032,"data_stored":2137260032,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":3356229632,"data_stored":3356229632,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":2132738048,"data_stored":2132738048,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":2126512128,"data_stored":2126512128,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T21:02:58.701 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph osd pool set unique_pool_2 pgp_num 16 2023-09-23T21:02:58.714 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:02:58.767 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:02:58.785 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 87 16 108457 108441 77.8515 49.3125 0.0255645 0.01284 2023-09-23T21:02:58.811 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:02:58.817 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:02:58.817 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:02:58.918 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:02:58.951 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:02:59.011 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:02:59.015 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:02:59.016 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:02:59.116 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:02:59.149 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:02:59.218 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:02:59.224 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:02:59 smithi070 ceph-mon[168282]: from='client.55489 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T21:02:59.225 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:02:59 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/6161458' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T21:02:59.225 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:02:59 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T21:02:59.225 INFO:teuthology.orchestra.run.smithi026.stderr:noscrub is unset 2023-09-23T21:02:59.225 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:02:59.226 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:02:59.269 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:02:59 smithi026 ceph-mon[175151]: from='client.55489 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T21:02:59.269 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:02:59 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/6161458' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T21:02:59.269 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:02:59 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T21:02:59.269 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:02:59 smithi026 ceph-mon[179688]: from='client.55489 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T21:02:59.270 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:02:59 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/6161458' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T21:02:59.270 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:02:59 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T21:02:59.326 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:02:59.458 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:02:59.620 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 21:02:59 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[164029]: ::ffff:172.21.15.70 - - [23/Sep/2023:21:02:59] "GET /metrics HTTP/1.1" 200 36291 "" "Prometheus/2.43.0" 2023-09-23T21:02:59.723 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:02:59.741 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:02:59.742 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:02:59.786 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 88 16 109227 109211 77.5139 48.125 0.0112121 0.0128955 2023-09-23T21:02:59.843 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:02:59.879 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:02:59.883 INFO:teuthology.orchestra.run.smithi026.stdout:{ 2023-09-23T21:02:59.883 INFO:teuthology.orchestra.run.smithi026.stdout: "ops": [ 2023-09-23T21:02:59.883 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:59.883 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35469.0:109286 8.a 8:5c8e9187:::benchmark_data_smithi026_301269_object109285:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e276)", 2023-09-23T21:02:59.883 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T21:02:59.861958+0000", 2023-09-23T21:02:59.883 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 0.018902710999999999, 2023-09-23T21:02:59.883 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.018938176000000001, 2023-09-23T21:02:59.883 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:02:59.884 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "waiting for sub ops", 2023-09-23T21:02:59.884 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T21:02:59.884 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35469", 2023-09-23T21:02:59.884 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/4041424623", 2023-09-23T21:02:59.884 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 109286 2023-09-23T21:02:59.884 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:59.884 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:02:59.884 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:59.884 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:02:59.884 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:02:59.861958+0000", 2023-09-23T21:02:59.884 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:02:59.885 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:59.885 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:59.885 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:02:59.885 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:02:59.861958+0000", 2023-09-23T21:02:59.885 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:02:59.885 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:59.885 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:59.885 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:02:59.885 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:02:59.861958+0000", 2023-09-23T21:02:59.885 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T21:02:59.886 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:59.886 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:59.886 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:02:59.886 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:02:59.861978+0000", 2023-09-23T21:02:59.886 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.0613000000000001e-05 2023-09-23T21:02:59.886 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:59.886 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:59.886 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:02:59.886 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:02:59.861987+0000", 2023-09-23T21:02:59.886 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.3669999999999997e-06 2023-09-23T21:02:59.886 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:59.886 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:59.887 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:02:59.887 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:02:59.861993+0000", 2023-09-23T21:02:59.887 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.9340000000000003e-06 2023-09-23T21:02:59.887 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:59.887 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:59.887 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:02:59.887 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:02:59.864908+0000", 2023-09-23T21:02:59.887 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.002915581 2023-09-23T21:02:59.887 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:59.887 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:59.887 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:02:59.887 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:02:59.865063+0000", 2023-09-23T21:02:59.888 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00015449300000000001 2023-09-23T21:02:59.888 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:59.888 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:59.888 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 2,5", 2023-09-23T21:02:59.888 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:02:59.865216+0000", 2023-09-23T21:02:59.888 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000152817 2023-09-23T21:02:59.888 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:59.888 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:59.888 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T21:02:59.888 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:02:59.874124+0000", 2023-09-23T21:02:59.888 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0089079959999999996 2023-09-23T21:02:59.888 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:59.889 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:59.889 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:02:59.889 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:02:59.874270+0000", 2023-09-23T21:02:59.889 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00014632100000000001 2023-09-23T21:02:59.889 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:02:59.889 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:02:59.889 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:02:59.889 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:59.889 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:59.889 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35469.0:109290 8.1 8:8ede7b0b:::benchmark_data_smithi026_301269_object109289:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e276)", 2023-09-23T21:02:59.889 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T21:02:59.862008+0000", 2023-09-23T21:02:59.889 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 0.018852705000000001, 2023-09-23T21:02:59.890 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.019020142, 2023-09-23T21:02:59.890 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:02:59.890 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "waiting for sub ops", 2023-09-23T21:02:59.890 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T21:02:59.890 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35469", 2023-09-23T21:02:59.890 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/4041424623", 2023-09-23T21:02:59.890 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 109290 2023-09-23T21:02:59.890 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:59.890 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:02:59.890 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:59.890 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:02:59.890 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:02:59.862008+0000", 2023-09-23T21:02:59.891 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:02:59.891 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:59.891 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:59.891 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:02:59.891 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:02:59.862008+0000", 2023-09-23T21:02:59.891 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:02:59.891 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:59.891 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:59.891 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:02:59.891 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:02:59.862008+0000", 2023-09-23T21:02:59.891 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T21:02:59.891 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:59.892 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:59.892 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:02:59.892 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:02:59.862028+0000", 2023-09-23T21:02:59.892 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.0506999999999999e-05 2023-09-23T21:02:59.892 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:59.892 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:59.892 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:02:59.892 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:02:59.862037+0000", 2023-09-23T21:02:59.892 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.4999999999999999e-06 2023-09-23T21:02:59.892 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:59.892 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:59.893 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:02:59.893 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:02:59.862043+0000", 2023-09-23T21:02:59.893 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.9789999999999998e-06 2023-09-23T21:02:59.893 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:59.893 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:59.893 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:02:59.893 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:02:59.862176+0000", 2023-09-23T21:02:59.893 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00013285 2023-09-23T21:02:59.893 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:59.893 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:59.893 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:02:59.894 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:02:59.862317+0000", 2023-09-23T21:02:59.894 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000141216 2023-09-23T21:02:59.894 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:59.894 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:59.894 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 0,2", 2023-09-23T21:02:59.894 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:02:59.862451+0000", 2023-09-23T21:02:59.894 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000134363 2023-09-23T21:02:59.894 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:59.894 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:59.894 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T21:02:59.895 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:02:59.870062+0000", 2023-09-23T21:02:59.895 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0076108219999999997 2023-09-23T21:02:59.895 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:59.895 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:59.895 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:02:59.895 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:02:59.880481+0000", 2023-09-23T21:02:59.895 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.010418736 2023-09-23T21:02:59.895 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:02:59.895 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:02:59.895 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:02:59.895 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:59.895 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:59.896 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35469.0:109291 8.a 8:55851947:::benchmark_data_smithi026_301269_object109290:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e276)", 2023-09-23T21:02:59.896 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T21:02:59.862057+0000", 2023-09-23T21:02:59.896 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 0.018804050999999999, 2023-09-23T21:02:59.896 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.019076352000000001, 2023-09-23T21:02:59.896 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:02:59.896 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "waiting for sub ops", 2023-09-23T21:02:59.896 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T21:02:59.896 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35469", 2023-09-23T21:02:59.896 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/4041424623", 2023-09-23T21:02:59.896 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 109291 2023-09-23T21:02:59.896 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:59.896 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:02:59.896 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:59.897 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:02:59.897 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:02:59.862057+0000", 2023-09-23T21:02:59.897 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:02:59.897 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:59.897 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:59.897 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:02:59.897 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:02:59.862057+0000", 2023-09-23T21:02:59.897 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:02:59.897 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:59.897 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:59.897 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:02:59.897 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:02:59.862057+0000", 2023-09-23T21:02:59.898 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T21:02:59.898 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:59.898 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:59.898 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:02:59.898 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:02:59.862079+0000", 2023-09-23T21:02:59.898 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.2498999999999999e-05 2023-09-23T21:02:59.898 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:59.898 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:59.898 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:02:59.898 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:02:59.862087+0000", 2023-09-23T21:02:59.898 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.1370000000000002e-06 2023-09-23T21:02:59.898 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:59.899 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:59.899 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:02:59.899 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:02:59.862093+0000", 2023-09-23T21:02:59.899 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.2999999999999998e-06 2023-09-23T21:02:59.899 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:59.899 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:59.899 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:02:59.899 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:02:59.869176+0000", 2023-09-23T21:02:59.899 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0070821039999999997 2023-09-23T21:02:59.899 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:59.899 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:59.899 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:02:59.900 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:02:59.869356+0000", 2023-09-23T21:02:59.900 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000180841 2023-09-23T21:02:59.900 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:59.900 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:59.900 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 2,5", 2023-09-23T21:02:59.900 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:02:59.869535+0000", 2023-09-23T21:02:59.900 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000178457 2023-09-23T21:02:59.900 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:59.900 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:59.900 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T21:02:59.900 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:02:59.874130+0000", 2023-09-23T21:02:59.900 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0045950110000000004 2023-09-23T21:02:59.901 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:59.901 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:59.901 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:02:59.901 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:02:59.874659+0000", 2023-09-23T21:02:59.901 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00052871299999999999 2023-09-23T21:02:59.901 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:02:59.901 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:02:59.901 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:02:59.901 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:59.901 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:59.901 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35469.0:109292 8.1 8:8331ea77:::benchmark_data_smithi026_301269_object109291:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e276)", 2023-09-23T21:02:59.901 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T21:02:59.862100+0000", 2023-09-23T21:02:59.902 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 0.018760796999999999, 2023-09-23T21:02:59.902 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.019137113000000001, 2023-09-23T21:02:59.902 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:02:59.902 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "waiting for sub ops", 2023-09-23T21:02:59.902 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T21:02:59.902 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35469", 2023-09-23T21:02:59.902 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/4041424623", 2023-09-23T21:02:59.902 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 109292 2023-09-23T21:02:59.902 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:59.902 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:02:59.902 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:59.902 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:02:59.903 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:02:59.862100+0000", 2023-09-23T21:02:59.903 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:02:59.903 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:59.903 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:59.903 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:02:59.903 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:02:59.862100+0000", 2023-09-23T21:02:59.903 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:02:59.903 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:59.903 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:59.903 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:02:59.903 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:02:59.862100+0000", 2023-09-23T21:02:59.903 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T21:02:59.904 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:59.904 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:59.904 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:02:59.904 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:02:59.862122+0000", 2023-09-23T21:02:59.904 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.1997999999999998e-05 2023-09-23T21:02:59.904 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:59.904 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:59.904 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:02:59.904 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:02:59.862130+0000", 2023-09-23T21:02:59.904 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.0660000000000004e-06 2023-09-23T21:02:59.904 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:59.904 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:59.905 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:02:59.905 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:02:59.862136+0000", 2023-09-23T21:02:59.905 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.1750000000000002e-06 2023-09-23T21:02:59.905 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:59.905 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:59.905 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:02:59.905 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:02:59.866018+0000", 2023-09-23T21:02:59.905 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0038822079999999998 2023-09-23T21:02:59.905 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:59.905 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:59.905 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:02:59.905 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:02:59.866103+0000", 2023-09-23T21:02:59.906 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.4460000000000001e-05 2023-09-23T21:02:59.906 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:59.906 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:59.906 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 0,2", 2023-09-23T21:02:59.906 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:02:59.866188+0000", 2023-09-23T21:02:59.906 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.4963999999999995e-05 2023-09-23T21:02:59.906 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:59.906 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:59.906 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T21:02:59.906 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:02:59.873830+0000", 2023-09-23T21:02:59.906 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0076418129999999999 2023-09-23T21:02:59.907 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:02:59.907 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:02:59.907 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:02:59.907 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:59.907 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:59.907 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_repop(client.35469.0:109281 8.0 e276/254 8:0fa0f016:::benchmark_data_smithi026_301269_object109280:head v 276'6799, mlcod=276'6797)", 2023-09-23T21:02:59.907 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T21:02:59.866812+0000", 2023-09-23T21:02:59.907 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 0.014049178000000001, 2023-09-23T21:02:59.907 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.014530866999999999, 2023-09-23T21:02:59.907 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:02:59.907 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "started", 2023-09-23T21:02:59.907 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:02:59.908 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:59.908 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:02:59.908 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:02:59.866812+0000", 2023-09-23T21:02:59.908 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:02:59.908 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:59.908 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:59.908 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:02:59.908 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:02:59.866812+0000", 2023-09-23T21:02:59.908 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:02:59.908 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:59.908 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:59.908 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:02:59.909 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:02:59.866811+0000", 2023-09-23T21:02:59.909 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T21:02:59.909 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:59.909 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:59.909 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:02:59.909 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:02:59.866843+0000", 2023-09-23T21:02:59.909 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.1773000000000002e-05 2023-09-23T21:02:59.909 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:59.909 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:59.909 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:02:59.909 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:02:59.866853+0000", 2023-09-23T21:02:59.909 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.4390000000000006e-06 2023-09-23T21:02:59.909 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:59.910 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:59.910 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:02:59.910 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:02:59.866859+0000", 2023-09-23T21:02:59.910 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.2650000000000002e-06 2023-09-23T21:02:59.910 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:59.910 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:59.910 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:02:59.910 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:02:59.870542+0000", 2023-09-23T21:02:59.910 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0036828609999999999 2023-09-23T21:02:59.910 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:59.910 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:59.911 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:02:59.911 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:02:59.870562+0000", 2023-09-23T21:02:59.911 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.0496000000000002e-05 2023-09-23T21:02:59.911 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:02:59.911 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:02:59.911 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:02:59.911 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:59.911 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:59.911 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_repop(client.35469.0:109293 8.0 e276/254)", 2023-09-23T21:02:59.911 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T21:02:59.866973+0000", 2023-09-23T21:02:59.911 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 0.013888281000000001, 2023-09-23T21:02:59.911 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.01443238, 2023-09-23T21:02:59.911 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:02:59.912 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "queued for pg", 2023-09-23T21:02:59.912 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:02:59.912 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:59.912 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:02:59.912 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:02:59.866973+0000", 2023-09-23T21:02:59.912 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:02:59.912 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:59.912 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:59.912 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:02:59.912 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:02:59.866973+0000", 2023-09-23T21:02:59.912 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:02:59.912 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:59.913 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:59.913 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:02:59.913 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:02:59.866973+0000", 2023-09-23T21:02:59.913 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967296 2023-09-23T21:02:59.913 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:59.913 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:59.913 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:02:59.913 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:02:59.866995+0000", 2023-09-23T21:02:59.913 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.2314e-05 2023-09-23T21:02:59.913 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:59.913 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:59.913 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:02:59.914 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:02:59.867001+0000", 2023-09-23T21:02:59.914 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.8760000000000002e-06 2023-09-23T21:02:59.914 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:59.914 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:59.914 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:02:59.914 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:02:59.867005+0000", 2023-09-23T21:02:59.914 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.267e-06 2023-09-23T21:02:59.914 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:02:59.914 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:02:59.914 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:02:59.914 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:59.914 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:59.914 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35469.0:109294 8.3 8:cb12aab7:::benchmark_data_smithi026_301269_object109293:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e276)", 2023-09-23T21:02:59.915 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T21:02:59.875734+0000", 2023-09-23T21:02:59.915 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 0.0051266819999999996, 2023-09-23T21:02:59.915 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0057335449999999996, 2023-09-23T21:02:59.915 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:02:59.915 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "waiting for sub ops", 2023-09-23T21:02:59.915 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T21:02:59.915 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35469", 2023-09-23T21:02:59.915 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/4041424623", 2023-09-23T21:02:59.915 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 109294 2023-09-23T21:02:59.915 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:59.915 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:02:59.915 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:59.916 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:02:59.916 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:02:59.875734+0000", 2023-09-23T21:02:59.916 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:02:59.916 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:59.916 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:59.916 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:02:59.916 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:02:59.875734+0000", 2023-09-23T21:02:59.916 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:02:59.916 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:59.916 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:59.916 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:02:59.916 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:02:59.875733+0000", 2023-09-23T21:02:59.917 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T21:02:59.917 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:59.917 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:59.917 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:02:59.917 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:02:59.875760+0000", 2023-09-23T21:02:59.917 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.7033999999999999e-05 2023-09-23T21:02:59.917 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:59.917 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:59.917 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:02:59.917 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:02:59.875778+0000", 2023-09-23T21:02:59.917 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.7655e-05 2023-09-23T21:02:59.917 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:59.917 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:59.918 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:02:59.918 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:02:59.875787+0000", 2023-09-23T21:02:59.918 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.9250000000000001e-06 2023-09-23T21:02:59.918 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:59.918 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:59.918 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:02:59.918 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:02:59.875839+0000", 2023-09-23T21:02:59.918 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.2200999999999997e-05 2023-09-23T21:02:59.918 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:59.918 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:59.918 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:02:59.918 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:02:59.875992+0000", 2023-09-23T21:02:59.919 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000153191 2023-09-23T21:02:59.919 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:59.919 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:59.919 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 5,6", 2023-09-23T21:02:59.919 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:02:59.876146+0000", 2023-09-23T21:02:59.919 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00015450800000000001 2023-09-23T21:02:59.919 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:02:59.919 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:02:59.919 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:02:59.919 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:59.919 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:59.919 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35469.0:109296 8.3 8:c8fda3d6:::benchmark_data_smithi026_301269_object109295:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e276)", 2023-09-23T21:02:59.919 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T21:02:59.875804+0000", 2023-09-23T21:02:59.920 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 0.0050570909999999997, 2023-09-23T21:02:59.920 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0057585500000000003, 2023-09-23T21:02:59.920 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:02:59.920 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "waiting for sub ops", 2023-09-23T21:02:59.920 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T21:02:59.920 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35469", 2023-09-23T21:02:59.920 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/4041424623", 2023-09-23T21:02:59.920 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 109296 2023-09-23T21:02:59.920 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:59.920 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:02:59.920 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:59.920 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:02:59.920 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:02:59.875804+0000", 2023-09-23T21:02:59.921 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:02:59.921 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:59.921 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:59.921 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:02:59.921 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:02:59.875804+0000", 2023-09-23T21:02:59.921 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:02:59.921 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:59.921 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:59.921 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:02:59.921 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:02:59.875803+0000", 2023-09-23T21:02:59.921 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T21:02:59.921 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:59.921 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:59.922 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:02:59.922 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:02:59.875826+0000", 2023-09-23T21:02:59.922 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.2540000000000001e-05 2023-09-23T21:02:59.922 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:59.922 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:59.922 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:02:59.922 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:02:59.875835+0000", 2023-09-23T21:02:59.922 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.8070000000000006e-06 2023-09-23T21:02:59.922 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:59.922 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:59.922 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:02:59.922 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:02:59.875842+0000", 2023-09-23T21:02:59.923 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.7610000000000002e-06 2023-09-23T21:02:59.923 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:59.923 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:59.923 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:02:59.923 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:02:59.880239+0000", 2023-09-23T21:02:59.923 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0043972129999999996 2023-09-23T21:02:59.923 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:59.923 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:59.923 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:02:59.923 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:02:59.880429+0000", 2023-09-23T21:02:59.923 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000190127 2023-09-23T21:02:59.923 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:59.923 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:59.924 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 5,6", 2023-09-23T21:02:59.924 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:02:59.880595+0000", 2023-09-23T21:02:59.924 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00016568900000000001 2023-09-23T21:02:59.924 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:02:59.924 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:02:59.924 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:02:59.924 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:59.924 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:59.924 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35469.0:109299 8.7 8:e05ed9a1:::benchmark_data_smithi026_301269_object109298:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e276)", 2023-09-23T21:02:59.924 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T21:02:59.875848+0000", 2023-09-23T21:02:59.924 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 0.0050129290000000002, 2023-09-23T21:02:59.924 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0058043790000000001, 2023-09-23T21:02:59.924 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:02:59.925 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "waiting for sub ops", 2023-09-23T21:02:59.925 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T21:02:59.925 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35469", 2023-09-23T21:02:59.925 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/4041424623", 2023-09-23T21:02:59.925 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 109299 2023-09-23T21:02:59.925 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:59.925 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:02:59.925 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:59.925 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:02:59.925 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:02:59.875848+0000", 2023-09-23T21:02:59.925 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:02:59.925 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:59.925 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:59.926 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:02:59.926 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:02:59.875848+0000", 2023-09-23T21:02:59.926 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:02:59.926 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:59.926 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:59.926 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:02:59.926 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:02:59.875848+0000", 2023-09-23T21:02:59.926 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T21:02:59.926 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:59.926 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:59.926 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:02:59.926 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:02:59.875873+0000", 2023-09-23T21:02:59.927 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.548e-05 2023-09-23T21:02:59.927 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:59.927 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:59.927 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:02:59.927 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:02:59.875882+0000", 2023-09-23T21:02:59.927 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.4470000000000006e-06 2023-09-23T21:02:59.927 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:59.927 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:59.927 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:02:59.927 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:02:59.875888+0000", 2023-09-23T21:02:59.927 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.4400000000000002e-06 2023-09-23T21:02:59.927 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:59.928 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:59.928 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:02:59.928 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:02:59.876077+0000", 2023-09-23T21:02:59.928 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00018867499999999999 2023-09-23T21:02:59.928 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:59.928 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:59.928 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:02:59.928 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:02:59.876197+0000", 2023-09-23T21:02:59.928 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000120655 2023-09-23T21:02:59.928 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:59.928 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:59.928 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 3,6", 2023-09-23T21:02:59.929 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:02:59.876323+0000", 2023-09-23T21:02:59.929 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00012559499999999999 2023-09-23T21:02:59.929 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:59.929 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:59.929 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:02:59.929 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:02:59.880483+0000", 2023-09-23T21:02:59.929 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0041599740000000003 2023-09-23T21:02:59.929 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:59.929 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:59.929 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T21:02:59.929 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:02:59.880730+0000", 2023-09-23T21:02:59.929 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00024686800000000002 2023-09-23T21:02:59.930 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:02:59.930 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:02:59.930 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:02:59.930 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:59.930 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:59.930 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_repop_reply(client.35469.0:109294 8.3 e276/254)", 2023-09-23T21:02:59.930 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T21:02:59.880125+0000", 2023-09-23T21:02:59.930 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 0.00073583299999999997, 2023-09-23T21:02:59.930 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0015923230000000001, 2023-09-23T21:02:59.930 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:02:59.930 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "queued for pg", 2023-09-23T21:02:59.931 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:02:59.931 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:59.931 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:02:59.931 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:02:59.880125+0000", 2023-09-23T21:02:59.931 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:02:59.931 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:59.931 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:59.931 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:02:59.931 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:02:59.880125+0000", 2023-09-23T21:02:59.931 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:02:59.931 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:59.931 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:59.931 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:02:59.932 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:02:59.880125+0000", 2023-09-23T21:02:59.932 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T21:02:59.932 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:59.932 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:59.932 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:02:59.932 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:02:59.880129+0000", 2023-09-23T21:02:59.932 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.8680000000000001e-06 2023-09-23T21:02:59.932 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:59.932 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:59.932 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:02:59.932 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:02:59.880140+0000", 2023-09-23T21:02:59.932 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1659e-05 2023-09-23T21:02:59.932 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:02:59.933 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:02:59.933 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:02:59.933 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:02:59.880147+0000", 2023-09-23T21:02:59.933 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.0859999999999996e-06 2023-09-23T21:02:59.933 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:02:59.933 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:02:59.933 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:02:59.933 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:02:59.933 INFO:teuthology.orchestra.run.smithi026.stdout: ], 2023-09-23T21:02:59.933 INFO:teuthology.orchestra.run.smithi026.stdout: "num_ops": 10 2023-09-23T21:02:59.933 INFO:teuthology.orchestra.run.smithi026.stdout:} 2023-09-23T21:02:59.934 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:02:59.935 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:02:59.935 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:02:59.935 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:03:00.036 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:03:00.074 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:03:00.129 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:03:00.133 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:03:00.134 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:03:00.234 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:03:00.283 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:03:00.328 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:03:00.331 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:03:00.331 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:03:00.432 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:03:00.437 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:03:00 smithi026 ceph-mon[179688]: pgmap v730: 109 pgs: 109 active+clean; 6.5 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 50 MiB/s wr, 801 op/s 2023-09-23T21:03:00.437 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:03:00 smithi026 ceph-mon[179688]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:03:00.437 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:03:00 smithi026 ceph-mon[179688]: failed to encode map e276 with expected crc 2023-09-23T21:03:00.437 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:03:00 smithi026 ceph-mon[179688]: failed to encode map e276 with expected crc 2023-09-23T21:03:00.437 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:03:00 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T21:03:00.437 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:03:00 smithi026 ceph-mon[179688]: osdmap e276: 8 total, 8 up, 8 in 2023-09-23T21:03:00.437 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:03:00 smithi026 ceph-mon[179688]: failed to encode map e276 with expected crc 2023-09-23T21:03:00.438 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:03:00 smithi026 ceph-mon[175151]: pgmap v730: 109 pgs: 109 active+clean; 6.5 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 50 MiB/s wr, 801 op/s 2023-09-23T21:03:00.438 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:03:00 smithi026 ceph-mon[175151]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:03:00.438 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:03:00 smithi026 ceph-mon[175151]: failed to encode map e276 with expected crc 2023-09-23T21:03:00.438 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:03:00 smithi026 ceph-mon[175151]: failed to encode map e276 with expected crc 2023-09-23T21:03:00.438 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:03:00 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T21:03:00.438 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:03:00 smithi026 ceph-mon[175151]: osdmap e276: 8 total, 8 up, 8 in 2023-09-23T21:03:00.438 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:03:00 smithi026 ceph-mon[175151]: failed to encode map e276 with expected crc 2023-09-23T21:03:00.443 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0: {} 2023-09-23T21:03:00.477 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1: {} 2023-09-23T21:03:00.510 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2: {} 2023-09-23T21:03:00.538 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:03:00.553 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3: {} 2023-09-23T21:03:00.592 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4: {} 2023-09-23T21:03:00.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:03:00 smithi070 ceph-mon[168282]: pgmap v730: 109 pgs: 109 active+clean; 6.5 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 50 MiB/s wr, 801 op/s 2023-09-23T21:03:00.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:03:00 smithi070 ceph-mon[168282]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:03:00.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:03:00 smithi070 ceph-mon[168282]: failed to encode map e276 with expected crc 2023-09-23T21:03:00.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:03:00 smithi070 ceph-mon[168282]: failed to encode map e276 with expected crc 2023-09-23T21:03:00.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:03:00 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T21:03:00.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:03:00 smithi070 ceph-mon[168282]: osdmap e276: 8 total, 8 up, 8 in 2023-09-23T21:03:00.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:03:00 smithi070 ceph-mon[168282]: failed to encode map e276 with expected crc 2023-09-23T21:03:00.645 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T21:03:00.681 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6: {} 2023-09-23T21:03:00.712 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T21:03:00.716 INFO:teuthology.orchestra.run.smithi026.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T21:03:00.716 INFO:teuthology.orchestra.run.smithi026.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T21:03:00.716 INFO:teuthology.orchestra.run.smithi026.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T21:03:00.716 INFO:teuthology.orchestra.run.smithi026.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T21:03:00.717 INFO:teuthology.orchestra.run.smithi026.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T21:03:00.717 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T21:03:00.717 INFO:teuthology.orchestra.run.smithi026.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T21:03:00.717 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T21:03:00.786 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 89 16 109990 109974 77.1789 47.6875 0.0183712 0.0129506 2023-09-23T21:03:00.877 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_blocked_ops 2023-09-23T21:03:01.367 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:03:01.373 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:03:01.373 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:03:01.375 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:03:01 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/3269169664' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:03:01.375 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:03:01 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/3269169664' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:03:01.474 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:03:01.564 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:03:01.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:03:01 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/3269169664' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:03:01.662 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:03:01.666 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:03:01.667 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:03:01.764 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T21:03:01.767 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:03:01.786 INFO:tasks.radosbench.radosbench.0.smithi026.stdout:Total time run: 90.0136 2023-09-23T21:03:01.786 INFO:tasks.radosbench.radosbench.0.smithi026.stdout:Total writes made: 110754 2023-09-23T21:03:01.786 INFO:tasks.radosbench.radosbench.0.smithi026.stdout:Write size: 65536 2023-09-23T21:03:01.786 INFO:tasks.radosbench.radosbench.0.smithi026.stdout:Object size: 65536 2023-09-23T21:03:01.786 INFO:tasks.radosbench.radosbench.0.smithi026.stdout:Bandwidth (MB/sec): 76.9009 2023-09-23T21:03:01.786 INFO:tasks.radosbench.radosbench.0.smithi026.stdout:Stddev Bandwidth: 34.1341 2023-09-23T21:03:01.787 INFO:tasks.radosbench.radosbench.0.smithi026.stdout:Max bandwidth (MB/sec): 184.312 2023-09-23T21:03:01.787 INFO:tasks.radosbench.radosbench.0.smithi026.stdout:Min bandwidth (MB/sec): 41.4375 2023-09-23T21:03:01.787 INFO:tasks.radosbench.radosbench.0.smithi026.stdout:Average IOPS: 1230 2023-09-23T21:03:01.787 INFO:tasks.radosbench.radosbench.0.smithi026.stdout:Stddev IOPS: 546.146 2023-09-23T21:03:01.787 INFO:tasks.radosbench.radosbench.0.smithi026.stdout:Max IOPS: 2949 2023-09-23T21:03:01.787 INFO:tasks.radosbench.radosbench.0.smithi026.stdout:Min IOPS: 663 2023-09-23T21:03:01.787 INFO:tasks.radosbench.radosbench.0.smithi026.stdout:Average Latency(s): 0.0130004 2023-09-23T21:03:01.787 INFO:tasks.radosbench.radosbench.0.smithi026.stdout:Stddev Latency(s): 0.00882625 2023-09-23T21:03:01.788 INFO:tasks.radosbench.radosbench.0.smithi026.stdout:Max latency(s): 0.190811 2023-09-23T21:03:01.788 INFO:tasks.radosbench.radosbench.0.smithi026.stdout:Min latency(s): 0.00122595 2023-09-23T21:03:01.790 INFO:tasks.radosbench.radosbench.0.smithi026.stdout:Cleaning up (deleting benchmark objects) 2023-09-23T21:03:01.894 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:03:02.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:03:02 smithi070 ceph-mon[168282]: pgmap v732: 109 pgs: 109 active+clean; 6.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 54 MiB/s wr, 860 op/s 2023-09-23T21:03:02.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:03:02 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:03:02.634 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:03:02 smithi026 ceph-mon[179688]: pgmap v732: 109 pgs: 109 active+clean; 6.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 54 MiB/s wr, 860 op/s 2023-09-23T21:03:02.634 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:03:02 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:03:02.634 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:03:02 smithi026 ceph-mon[175151]: pgmap v732: 109 pgs: 109 active+clean; 6.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 54 MiB/s wr, 860 op/s 2023-09-23T21:03:02.634 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:03:02 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:03:02.799 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:03:02.805 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:03:02.805 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:03:02.906 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:03:02.958 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:03:03.019 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:03:03.027 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:03:03.028 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:03:03.128 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:03:03.195 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:03:03.218 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:03:03.454 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T21:03:03.796 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:03:03.804 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:03:03.804 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:03:03.905 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:03:03.961 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:03:04.016 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:03:04.019 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:03:04.019 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:03:04.120 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:03:04.148 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:03:04.185 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:03:04.190 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:03:04.190 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:03:04.291 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:03:04.405 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:03:04.414 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:03:04 smithi026 ceph-mon[175151]: pgmap v733: 109 pgs: 109 active+clean; 6.7 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 102 B/s rd, 48 MiB/s wr, 775 op/s 2023-09-23T21:03:04.414 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:03:04 smithi026 ceph-mon[179688]: pgmap v733: 109 pgs: 109 active+clean; 6.7 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 102 B/s rd, 48 MiB/s wr, 775 op/s 2023-09-23T21:03:04.428 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:03:04.483 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:03:04.487 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:03:04.488 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:03:04.589 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:03:04.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:03:04 smithi070 ceph-mon[168282]: pgmap v733: 109 pgs: 109 active+clean; 6.7 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 102 B/s rd, 48 MiB/s wr, 775 op/s 2023-09-23T21:03:04.640 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:03:04.684 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:03:04.687 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:03:04.687 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:03:04.788 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:03:04.865 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:03:04.868 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:03:04.948 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:03:04.953 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:03:04.953 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:03:05.054 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:03:05.100 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:03:05.160 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:03:05.165 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:03:05.166 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:03:05.251 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:03:05 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/1340461002' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pgp_num", "val": "16"}]: dispatch 2023-09-23T21:03:05.251 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:03:05 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pgp_num", "val": "16"}]: dispatch 2023-09-23T21:03:05.267 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:03:05.270 INFO:teuthology.orchestra.run.smithi026.stderr:set pool 8 pgp_num to 16 2023-09-23T21:03:05.302 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:03:05.348 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:03:05.354 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:03:05.355 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:03:05.360 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:03:05 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/1340461002' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pgp_num", "val": "16"}]: dispatch 2023-09-23T21:03:05.360 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:03:05 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pgp_num", "val": "16"}]: dispatch 2023-09-23T21:03:05.456 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:03:05.492 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:03:05.502 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:03:05 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/1340461002' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pgp_num", "val": "16"}]: dispatch 2023-09-23T21:03:05.502 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:03:05 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pgp_num", "val": "16"}]: dispatch 2023-09-23T21:03:05.537 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:03:05.541 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:03:05.541 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:03:05.642 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:03:05.728 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:03:05.813 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T21:03:05.818 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:03:05.819 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:03:05.860 INFO:teuthology.orchestra.run.smithi026.stdout:ERROR: (22) Invalid argument 2023-09-23T21:03:05.860 INFO:teuthology.orchestra.run.smithi026.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T21:03:05.920 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:03:05.989 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:03:06.042 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:03:06.047 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:03:06.048 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:03:06.148 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:03:06.183 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:03:06.236 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:03:06.240 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:03:06.241 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:03:06.341 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:03:06.371 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:03:06.383 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:03:06 smithi026 ceph-mon[175151]: pgmap v734: 109 pgs: 109 active+clean; 6.7 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 102 B/s rd, 43 MiB/s wr, 787 op/s 2023-09-23T21:03:06.383 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:03:06 smithi026 ceph-mon[175151]: failed to encode map e277 with expected crc 2023-09-23T21:03:06.383 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:03:06 smithi026 ceph-mon[175151]: failed to encode map e277 with expected crc 2023-09-23T21:03:06.384 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:03:06 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pgp_num", "val": "16"}]': finished 2023-09-23T21:03:06.384 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:03:06 smithi026 ceph-mon[175151]: osdmap e277: 8 total, 8 up, 8 in 2023-09-23T21:03:06.384 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:03:06 smithi026 ceph-mon[179688]: pgmap v734: 109 pgs: 109 active+clean; 6.7 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 102 B/s rd, 43 MiB/s wr, 787 op/s 2023-09-23T21:03:06.384 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:03:06 smithi026 ceph-mon[179688]: failed to encode map e277 with expected crc 2023-09-23T21:03:06.384 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:03:06 smithi026 ceph-mon[179688]: failed to encode map e277 with expected crc 2023-09-23T21:03:06.384 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:03:06 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pgp_num", "val": "16"}]': finished 2023-09-23T21:03:06.384 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:03:06 smithi026 ceph-mon[179688]: osdmap e277: 8 total, 8 up, 8 in 2023-09-23T21:03:06.417 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:03:06.422 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:03:06.423 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:03:06.523 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:03:06.598 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:03:06.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:03:06 smithi070 ceph-mon[168282]: pgmap v734: 109 pgs: 109 active+clean; 6.7 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 102 B/s rd, 43 MiB/s wr, 787 op/s 2023-09-23T21:03:06.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:03:06 smithi070 ceph-mon[168282]: failed to encode map e277 with expected crc 2023-09-23T21:03:06.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:03:06 smithi070 ceph-mon[168282]: failed to encode map e277 with expected crc 2023-09-23T21:03:06.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:03:06 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pgp_num", "val": "16"}]': finished 2023-09-23T21:03:06.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:03:06 smithi070 ceph-mon[168282]: osdmap e277: 8 total, 8 up, 8 in 2023-09-23T21:03:06.921 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_historic_ops 2023-09-23T21:03:06.989 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:03:06.997 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:03:06.998 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:03:07.099 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:03:07.191 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:03:07.345 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:03:07.360 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0: {} 2023-09-23T21:03:07.366 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:03:07.367 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:03:07.405 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1: {} 2023-09-23T21:03:07.450 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2: {} 2023-09-23T21:03:07.468 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:03:07.495 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:03:07.501 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3: {} 2023-09-23T21:03:07.548 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:03:07.553 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:03:07.553 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:03:07.558 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4: {} 2023-09-23T21:03:07.610 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T21:03:07.654 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:03:07.657 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6: {} 2023-09-23T21:03:07.696 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T21:03:07.716 INFO:teuthology.orchestra.run.smithi026.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T21:03:07.716 INFO:teuthology.orchestra.run.smithi026.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T21:03:07.716 INFO:teuthology.orchestra.run.smithi026.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T21:03:07.717 INFO:teuthology.orchestra.run.smithi026.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T21:03:07.717 INFO:teuthology.orchestra.run.smithi026.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T21:03:07.717 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T21:03:07.717 INFO:teuthology.orchestra.run.smithi026.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T21:03:07.717 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T21:03:07.759 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:03:07.903 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:03:07.911 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:03:07.912 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:03:08.013 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:03:08.106 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:03:08.242 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T21:03:08.246 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:03:08.246 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:03:08.290 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:03:08.347 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:03:08.381 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:03:08 smithi026 ceph-mon[175151]: pgmap v736: 109 pgs: 109 active+clean; 6.7 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 127 B/s rd, 37 MiB/s wr, 765 op/s 2023-09-23T21:03:08.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:03:08 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/3656866016' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:03:08.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:03:08 smithi026 ceph-mon[179688]: pgmap v736: 109 pgs: 109 active+clean; 6.7 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 127 B/s rd, 37 MiB/s wr, 765 op/s 2023-09-23T21:03:08.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:03:08 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/3656866016' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:03:08.414 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:03:08.491 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:03:08.497 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:03:08.497 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:03:08.598 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:03:08.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:03:08 smithi070 ceph-mon[168282]: pgmap v736: 109 pgs: 109 active+clean; 6.7 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 127 B/s rd, 37 MiB/s wr, 765 op/s 2023-09-23T21:03:08.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:03:08 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/3656866016' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:03:08.645 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:03:08.691 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:03:08.697 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:03:08.698 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:03:08.799 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:03:08.851 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:03:08.897 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:03:08.899 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:03:08.900 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:03:09.000 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:03:09.042 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:03:09.098 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:03:09.103 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:03:09.104 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:03:09.204 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:03:09.269 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:03:09 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/3430811949' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:03:09.273 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:03:09 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/3430811949' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:03:09.282 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:03:09.297 INFO:teuthology.orchestra.run.smithi026.stderr:nodeep-scrub is unset 2023-09-23T21:03:09.360 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:03:09.373 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:03:09.373 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:03:09.474 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:03:09.526 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:03:09 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/3430811949' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:03:09.534 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:03:09.668 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T21:03:09.675 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:03:09.675 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:03:09.776 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:03:09.847 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 21:03:09 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[164029]: ::ffff:172.21.15.70 - - [23/Sep/2023:21:03:09] "GET /metrics HTTP/1.1" 200 36294 "" "Prometheus/2.43.0" 2023-09-23T21:03:09.858 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:03:09.986 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:03:09.997 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:03:09.998 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:03:10.057 INFO:teuthology.orchestra.run.smithi026.stdout:{ 2023-09-23T21:03:10.057 INFO:teuthology.orchestra.run.smithi026.stdout: "size": 20, 2023-09-23T21:03:10.058 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 600, 2023-09-23T21:03:10.058 INFO:teuthology.orchestra.run.smithi026.stdout: "ops": [ 2023-09-23T21:03:10.058 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:10.058 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34797.0:114984 6.e 6:77dbcdd0:::benchmark_data_smithi026_223868_object13858:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e132)", 2023-09-23T21:03:10.058 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T20:54:12.945909+0000", 2023-09-23T21:03:10.058 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 537.10750807399995, 2023-09-23T21:03:10.058 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.081195696, 2023-09-23T21:03:10.058 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:03:10.058 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:03:10.059 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T21:03:10.059 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34797", 2023-09-23T21:03:10.059 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T21:03:10.059 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 114984 2023-09-23T21:03:10.059 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:10.059 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:03:10.059 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:10.059 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:03:10.059 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:12.945909+0000", 2023-09-23T21:03:10.059 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:03:10.060 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:10.060 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:10.060 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:03:10.060 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:12.945909+0000", 2023-09-23T21:03:10.060 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:03:10.060 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:10.060 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:10.060 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:03:10.060 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:12.945908+0000", 2023-09-23T21:03:10.060 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T21:03:10.061 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:10.061 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:10.061 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:03:10.061 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:12.945913+0000", 2023-09-23T21:03:10.061 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.305e-06 2023-09-23T21:03:10.061 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:10.061 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:10.061 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:03:10.061 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:12.945924+0000", 2023-09-23T21:03:10.061 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.115e-05 2023-09-23T21:03:10.061 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:10.062 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:10.062 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:03:10.062 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:12.945933+0000", 2023-09-23T21:03:10.062 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.2969999999999992e-06 2023-09-23T21:03:10.062 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:10.062 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:10.062 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:03:10.062 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:12.950695+0000", 2023-09-23T21:03:10.062 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0047620850000000001 2023-09-23T21:03:10.063 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:10.063 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:10.063 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T21:03:10.063 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:12.950706+0000", 2023-09-23T21:03:10.063 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0858e-05 2023-09-23T21:03:10.063 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:10.063 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:10.063 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:03:10.063 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:14.019393+0000", 2023-09-23T21:03:10.063 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.068686851 2023-09-23T21:03:10.063 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:10.063 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:10.063 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:03:10.064 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:14.019595+0000", 2023-09-23T21:03:10.064 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000201988 2023-09-23T21:03:10.064 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:10.064 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:10.064 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 2,4", 2023-09-23T21:03:10.064 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:14.019783+0000", 2023-09-23T21:03:10.064 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000188104 2023-09-23T21:03:10.064 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:10.064 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:10.064 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:03:10.064 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:14.023500+0000", 2023-09-23T21:03:10.064 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00371645 2023-09-23T21:03:10.065 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:10.065 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:10.065 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T21:03:10.065 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:14.026242+0000", 2023-09-23T21:03:10.065 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.002742474 2023-09-23T21:03:10.065 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:10.065 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:10.065 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:03:10.065 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:14.026508+0000", 2023-09-23T21:03:10.065 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00026582400000000001 2023-09-23T21:03:10.065 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:10.065 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:10.066 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T21:03:10.066 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:14.026547+0000", 2023-09-23T21:03:10.066 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.9169999999999999e-05 2023-09-23T21:03:10.066 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:10.066 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:10.066 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T21:03:10.066 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:14.027105+0000", 2023-09-23T21:03:10.066 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00055742400000000005 2023-09-23T21:03:10.066 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:03:10.066 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:03:10.066 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:03:10.066 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:10.067 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:10.067 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34797.0:114994 6.e 6:7ff0892b:::benchmark_data_smithi026_223868_object13868:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e132)", 2023-09-23T21:03:10.067 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T20:54:12.968874+0000", 2023-09-23T21:03:10.067 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 537.08454332899998, 2023-09-23T21:03:10.067 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0589051819999999, 2023-09-23T21:03:10.067 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:03:10.067 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:03:10.067 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T21:03:10.067 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34797", 2023-09-23T21:03:10.067 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T21:03:10.067 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 114994 2023-09-23T21:03:10.067 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:10.067 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:03:10.068 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:10.068 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:03:10.068 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:12.968874+0000", 2023-09-23T21:03:10.068 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:03:10.068 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:10.068 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:10.068 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:03:10.068 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:12.968874+0000", 2023-09-23T21:03:10.068 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:03:10.068 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:10.068 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:10.068 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:03:10.069 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:12.968873+0000", 2023-09-23T21:03:10.069 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T21:03:10.069 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:10.069 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:10.069 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:03:10.069 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:12.968877+0000", 2023-09-23T21:03:10.069 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.1339999999999997e-06 2023-09-23T21:03:10.069 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:10.069 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:10.069 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:03:10.069 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:12.968886+0000", 2023-09-23T21:03:10.069 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.5050000000000007e-06 2023-09-23T21:03:10.070 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:10.070 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:10.070 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:03:10.070 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:12.968892+0000", 2023-09-23T21:03:10.070 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.8510000000000004e-06 2023-09-23T21:03:10.070 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:10.070 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:10.070 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:03:10.070 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:12.968950+0000", 2023-09-23T21:03:10.070 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.8746999999999998e-05 2023-09-23T21:03:10.070 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:10.070 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:10.071 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T21:03:10.071 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:12.968961+0000", 2023-09-23T21:03:10.071 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.115e-05 2023-09-23T21:03:10.071 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:10.071 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:10.071 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:03:10.071 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:14.023568+0000", 2023-09-23T21:03:10.071 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0546062759999999 2023-09-23T21:03:10.071 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:10.071 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:10.071 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:03:10.071 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:14.023704+0000", 2023-09-23T21:03:10.072 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000136188 2023-09-23T21:03:10.072 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:10.072 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:10.072 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 2,4", 2023-09-23T21:03:10.072 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:14.023794+0000", 2023-09-23T21:03:10.072 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.9986000000000002e-05 2023-09-23T21:03:10.072 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:10.072 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:10.072 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T21:03:10.072 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:14.027128+0000", 2023-09-23T21:03:10.072 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0033338510000000001 2023-09-23T21:03:10.072 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:10.072 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:10.073 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:03:10.073 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:14.027235+0000", 2023-09-23T21:03:10.073 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000106932 2023-09-23T21:03:10.073 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:10.073 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:10.073 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:03:10.073 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:14.027379+0000", 2023-09-23T21:03:10.073 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00014385 2023-09-23T21:03:10.073 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:10.073 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:10.073 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T21:03:10.073 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:14.027432+0000", 2023-09-23T21:03:10.074 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.3171999999999998e-05 2023-09-23T21:03:10.074 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:10.074 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:10.074 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T21:03:10.074 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:14.027779+0000", 2023-09-23T21:03:10.074 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00034717400000000001 2023-09-23T21:03:10.074 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:03:10.074 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:03:10.074 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:03:10.074 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:10.074 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:10.074 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34797.0:130503 6.4 6:236b9417:::benchmark_data_smithi026_223868_object29377:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e136)", 2023-09-23T21:03:10.075 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T20:54:30.930235+0000", 2023-09-23T21:03:10.075 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 519.12318234600002, 2023-09-23T21:03:10.075 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.1274586960000001, 2023-09-23T21:03:10.075 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:03:10.075 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:03:10.075 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T21:03:10.075 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34797", 2023-09-23T21:03:10.075 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T21:03:10.075 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 130503 2023-09-23T21:03:10.075 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:10.075 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:03:10.075 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:10.075 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:03:10.076 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:30.930235+0000", 2023-09-23T21:03:10.076 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:03:10.076 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:10.076 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:10.076 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:03:10.076 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:30.930235+0000", 2023-09-23T21:03:10.076 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:03:10.076 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:10.076 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:10.076 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:03:10.076 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:30.930233+0000", 2023-09-23T21:03:10.076 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T21:03:10.077 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:10.077 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:10.077 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:03:10.077 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:30.930242+0000", 2023-09-23T21:03:10.077 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.8769999999999999e-06 2023-09-23T21:03:10.077 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:10.077 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:10.077 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:03:10.077 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:30.930263+0000", 2023-09-23T21:03:10.077 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.1492999999999999e-05 2023-09-23T21:03:10.077 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:10.077 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:10.078 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:03:10.078 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:30.930273+0000", 2023-09-23T21:03:10.078 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.9350000000000006e-06 2023-09-23T21:03:10.078 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:10.078 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:10.078 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:03:10.078 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:30.930336+0000", 2023-09-23T21:03:10.078 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.2428999999999995e-05 2023-09-23T21:03:10.078 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:10.078 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:10.078 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T21:03:10.078 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:30.930348+0000", 2023-09-23T21:03:10.079 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.2551000000000001e-05 2023-09-23T21:03:10.079 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:10.079 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:10.079 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:03:10.079 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:31.883549+0000", 2023-09-23T21:03:10.079 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.95320037800000001 2023-09-23T21:03:10.079 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:10.079 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:10.079 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T21:03:10.079 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:31.883563+0000", 2023-09-23T21:03:10.079 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.384e-05 2023-09-23T21:03:10.079 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:10.079 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:10.080 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:03:10.080 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:33.031575+0000", 2023-09-23T21:03:10.080 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1480122770000001 2023-09-23T21:03:10.080 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:10.080 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:10.080 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:03:10.080 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:33.031710+0000", 2023-09-23T21:03:10.080 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00013495900000000001 2023-09-23T21:03:10.080 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:10.080 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:10.080 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 3,5,6", 2023-09-23T21:03:10.080 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:33.031848+0000", 2023-09-23T21:03:10.081 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000138135 2023-09-23T21:03:10.081 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:10.081 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:10.081 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T21:03:10.081 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:33.040431+0000", 2023-09-23T21:03:10.081 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0085826510000000002 2023-09-23T21:03:10.081 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:10.081 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:10.081 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:03:10.081 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:33.040561+0000", 2023-09-23T21:03:10.081 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00013077100000000001 2023-09-23T21:03:10.081 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:10.081 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:10.082 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:03:10.082 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:33.045365+0000", 2023-09-23T21:03:10.082 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0048039889999999998 2023-09-23T21:03:10.082 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:10.082 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:10.082 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:03:10.082 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:33.057544+0000", 2023-09-23T21:03:10.082 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.012178227 2023-09-23T21:03:10.082 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:10.082 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:10.082 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T21:03:10.082 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:33.057593+0000", 2023-09-23T21:03:10.083 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.9086999999999999e-05 2023-09-23T21:03:10.083 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:10.083 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:10.083 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T21:03:10.083 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:33.057693+0000", 2023-09-23T21:03:10.083 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000100697 2023-09-23T21:03:10.083 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:03:10.083 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:03:10.083 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:03:10.083 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:10.083 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:10.083 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34797.0:130517 6.14 6:2e6d4a50:::benchmark_data_smithi026_223868_object29391:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e136)", 2023-09-23T21:03:10.084 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T20:54:30.938770+0000", 2023-09-23T21:03:10.084 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 519.11464670400005, 2023-09-23T21:03:10.084 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.1191109109999999, 2023-09-23T21:03:10.084 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:03:10.084 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:03:10.084 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T21:03:10.084 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34797", 2023-09-23T21:03:10.084 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T21:03:10.084 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 130517 2023-09-23T21:03:10.084 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:10.084 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:03:10.084 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:10.085 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:03:10.085 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:30.938770+0000", 2023-09-23T21:03:10.085 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:03:10.085 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:10.085 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:10.085 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:03:10.085 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:30.938770+0000", 2023-09-23T21:03:10.085 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:03:10.085 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:10.085 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:10.085 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:03:10.086 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:30.938769+0000", 2023-09-23T21:03:10.086 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.999999 2023-09-23T21:03:10.086 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:10.086 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:10.086 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:03:10.086 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:30.938775+0000", 2023-09-23T21:03:10.086 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.0100000000000001e-06 2023-09-23T21:03:10.086 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:10.086 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:10.086 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:03:10.086 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:30.938790+0000", 2023-09-23T21:03:10.086 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.4391e-05 2023-09-23T21:03:10.087 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:10.087 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:10.087 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:03:10.087 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:30.938799+0000", 2023-09-23T21:03:10.087 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.2e-06 2023-09-23T21:03:10.087 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:10.087 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:10.087 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:03:10.087 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:30.938857+0000", 2023-09-23T21:03:10.087 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.8446999999999998e-05 2023-09-23T21:03:10.087 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:10.087 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:10.088 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T21:03:10.088 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:30.938871+0000", 2023-09-23T21:03:10.088 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.4042e-05 2023-09-23T21:03:10.088 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:10.088 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:10.088 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:03:10.088 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:31.884077+0000", 2023-09-23T21:03:10.088 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.94520575100000004 2023-09-23T21:03:10.088 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:10.088 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:10.088 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T21:03:10.089 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:31.884087+0000", 2023-09-23T21:03:10.089 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0135999999999999e-05 2023-09-23T21:03:10.089 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:10.089 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:10.089 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:03:10.089 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:33.034200+0000", 2023-09-23T21:03:10.089 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1501126020000001 2023-09-23T21:03:10.089 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:10.089 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:10.089 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:03:10.089 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:33.034313+0000", 2023-09-23T21:03:10.089 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000113507 2023-09-23T21:03:10.090 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:10.090 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:10.090 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 3,5,6", 2023-09-23T21:03:10.090 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:33.034465+0000", 2023-09-23T21:03:10.090 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00015155799999999999 2023-09-23T21:03:10.090 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:10.090 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:10.090 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:03:10.090 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:33.040587+0000", 2023-09-23T21:03:10.090 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0061222459999999996 2023-09-23T21:03:10.090 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:10.091 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:10.091 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T21:03:10.091 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:33.042787+0000", 2023-09-23T21:03:10.091 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0022003410000000002 2023-09-23T21:03:10.091 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:10.091 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:10.091 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:03:10.091 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:33.045400+0000", 2023-09-23T21:03:10.091 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0026128879999999998 2023-09-23T21:03:10.091 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:10.091 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:10.091 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:03:10.092 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:33.057734+0000", 2023-09-23T21:03:10.092 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.012333823000000001 2023-09-23T21:03:10.092 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:10.092 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:10.092 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T21:03:10.092 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:33.057774+0000", 2023-09-23T21:03:10.092 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.9455000000000001e-05 2023-09-23T21:03:10.092 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:10.092 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:10.092 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T21:03:10.092 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:33.057881+0000", 2023-09-23T21:03:10.092 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000107697 2023-09-23T21:03:10.093 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:03:10.093 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:03:10.093 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:03:10.093 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:10.093 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:10.093 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34797.0:130519 6.14 6:2fbfebb3:::benchmark_data_smithi026_223868_object29393:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e136)", 2023-09-23T21:03:10.093 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T20:54:30.939911+0000", 2023-09-23T21:03:10.093 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 519.11350560599999, 2023-09-23T21:03:10.093 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.1181728999999998, 2023-09-23T21:03:10.093 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:03:10.093 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:03:10.093 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T21:03:10.094 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34797", 2023-09-23T21:03:10.094 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T21:03:10.094 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 130519 2023-09-23T21:03:10.094 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:10.094 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:03:10.094 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:10.094 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:03:10.094 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:30.939911+0000", 2023-09-23T21:03:10.094 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:03:10.094 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:10.094 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:10.095 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:03:10.095 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:30.939911+0000", 2023-09-23T21:03:10.095 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:03:10.095 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:10.095 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:10.095 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:03:10.095 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:30.939911+0000", 2023-09-23T21:03:10.095 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.999999 2023-09-23T21:03:10.095 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:10.095 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:10.095 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:03:10.095 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:30.939916+0000", 2023-09-23T21:03:10.096 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.0220000000000004e-06 2023-09-23T21:03:10.096 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:10.096 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:10.096 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:03:10.096 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:30.939930+0000", 2023-09-23T21:03:10.096 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.4129e-05 2023-09-23T21:03:10.096 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:10.096 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:10.096 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:03:10.096 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:30.939938+0000", 2023-09-23T21:03:10.096 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.0909999999999993e-06 2023-09-23T21:03:10.096 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:10.097 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:10.097 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:03:10.097 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:30.939996+0000", 2023-09-23T21:03:10.097 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.8115999999999997e-05 2023-09-23T21:03:10.097 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:10.097 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:10.097 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T21:03:10.097 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:30.940008+0000", 2023-09-23T21:03:10.097 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1711e-05 2023-09-23T21:03:10.097 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:10.097 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:10.097 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:03:10.098 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:31.884115+0000", 2023-09-23T21:03:10.098 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.94410687100000001 2023-09-23T21:03:10.098 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:10.098 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:10.098 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T21:03:10.098 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:31.884123+0000", 2023-09-23T21:03:10.098 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.3990000000000004e-06 2023-09-23T21:03:10.098 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:10.098 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:10.098 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:03:10.098 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:33.036642+0000", 2023-09-23T21:03:10.098 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1525194350000001 2023-09-23T21:03:10.099 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:10.099 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:10.099 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:03:10.099 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:33.036778+0000", 2023-09-23T21:03:10.099 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000135609 2023-09-23T21:03:10.099 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:10.099 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:10.099 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 3,5,6", 2023-09-23T21:03:10.099 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:33.036924+0000", 2023-09-23T21:03:10.099 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000146198 2023-09-23T21:03:10.099 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:10.099 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:10.100 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T21:03:10.100 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:33.042794+0000", 2023-09-23T21:03:10.100 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0058697810000000001 2023-09-23T21:03:10.100 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:10.100 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:10.100 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:03:10.100 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:33.045426+0000", 2023-09-23T21:03:10.100 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0026324569999999999 2023-09-23T21:03:10.100 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:10.100 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:10.100 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:03:10.100 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:33.048941+0000", 2023-09-23T21:03:10.101 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.003514667 2023-09-23T21:03:10.101 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:10.101 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:10.101 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:03:10.101 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:33.057922+0000", 2023-09-23T21:03:10.101 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0089806850000000004 2023-09-23T21:03:10.101 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:10.101 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:10.101 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T21:03:10.101 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:33.057967+0000", 2023-09-23T21:03:10.101 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.5430999999999999e-05 2023-09-23T21:03:10.101 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:10.102 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:10.102 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T21:03:10.102 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:33.058084+0000", 2023-09-23T21:03:10.102 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000117081 2023-09-23T21:03:10.102 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:03:10.102 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:03:10.102 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:03:10.102 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:10.102 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:10.102 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34797.0:130536 6.14 6:2a329320:::benchmark_data_smithi026_223868_object29410:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e136)", 2023-09-23T21:03:10.102 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T20:54:30.980524+0000", 2023-09-23T21:03:10.102 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 519.07289322700001, 2023-09-23T21:03:10.103 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.0776815489999998, 2023-09-23T21:03:10.103 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:03:10.103 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:03:10.103 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T21:03:10.103 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34797", 2023-09-23T21:03:10.103 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T21:03:10.103 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 130536 2023-09-23T21:03:10.103 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:10.103 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:03:10.103 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:10.103 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:03:10.104 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:30.980524+0000", 2023-09-23T21:03:10.104 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:03:10.104 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:10.104 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:10.104 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:03:10.104 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:30.980524+0000", 2023-09-23T21:03:10.104 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:03:10.104 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:10.104 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:10.104 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:03:10.104 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:30.980522+0000", 2023-09-23T21:03:10.105 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999981 2023-09-23T21:03:10.105 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:10.105 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:10.105 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:03:10.105 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:30.980533+0000", 2023-09-23T21:03:10.105 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0721e-05 2023-09-23T21:03:10.105 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:10.105 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:10.105 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:03:10.105 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:30.980555+0000", 2023-09-23T21:03:10.105 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.1789000000000001e-05 2023-09-23T21:03:10.105 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:10.106 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:10.106 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:03:10.106 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:30.980565+0000", 2023-09-23T21:03:10.106 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0492999999999999e-05 2023-09-23T21:03:10.106 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:10.106 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:10.106 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:03:10.106 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:30.980633+0000", 2023-09-23T21:03:10.106 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.8361999999999994e-05 2023-09-23T21:03:10.106 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:10.106 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:10.107 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T21:03:10.107 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:30.980647+0000", 2023-09-23T21:03:10.107 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.3818999999999999e-05 2023-09-23T21:03:10.107 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:10.107 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:10.107 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:03:10.107 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:31.884138+0000", 2023-09-23T21:03:10.107 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.90349046700000002 2023-09-23T21:03:10.107 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:10.107 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:10.107 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T21:03:10.107 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:31.884142+0000", 2023-09-23T21:03:10.107 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.2660000000000003e-06 2023-09-23T21:03:10.108 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:10.108 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:10.108 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:03:10.108 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:33.038763+0000", 2023-09-23T21:03:10.108 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1546210180000001 2023-09-23T21:03:10.108 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:10.108 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:10.108 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:03:10.108 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:33.038886+0000", 2023-09-23T21:03:10.108 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00012294000000000001 2023-09-23T21:03:10.108 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:10.108 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:10.109 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 3,5,6", 2023-09-23T21:03:10.109 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:33.039020+0000", 2023-09-23T21:03:10.109 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000133621 2023-09-23T21:03:10.109 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:10.109 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:10.109 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T21:03:10.109 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:33.042797+0000", 2023-09-23T21:03:10.109 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0037779649999999999 2023-09-23T21:03:10.109 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:10.109 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:10.109 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:03:10.110 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:33.045454+0000", 2023-09-23T21:03:10.110 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.002656215 2023-09-23T21:03:10.110 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:10.110 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:10.110 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:03:10.110 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:33.048976+0000", 2023-09-23T21:03:10.110 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0035227209999999999 2023-09-23T21:03:10.110 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:10.110 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:10.110 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:03:10.110 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:33.058125+0000", 2023-09-23T21:03:10.110 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0091488279999999995 2023-09-23T21:03:10.111 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:10.111 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:10.111 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T21:03:10.111 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:33.058151+0000", 2023-09-23T21:03:10.111 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.5992000000000001e-05 2023-09-23T21:03:10.111 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:10.111 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:10.111 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T21:03:10.111 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:33.058205+0000", 2023-09-23T21:03:10.111 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.4104999999999998e-05 2023-09-23T21:03:10.111 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:03:10.112 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:03:10.112 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:03:10.112 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:10.112 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:10.112 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34797.0:165115 6.11 6:8a9c510c:::benchmark_data_smithi026_223868_object63989:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e154)", 2023-09-23T21:03:10.112 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T20:55:17.458662+0000", 2023-09-23T21:03:10.112 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 472.59475495300001, 2023-09-23T21:03:10.112 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0640936540000001, 2023-09-23T21:03:10.112 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:03:10.112 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "reached pg", 2023-09-23T21:03:10.112 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T21:03:10.113 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34797", 2023-09-23T21:03:10.113 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T21:03:10.114 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 165115 2023-09-23T21:03:10.114 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:10.114 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:03:10.114 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:10.114 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:03:10.114 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:17.458662+0000", 2023-09-23T21:03:10.114 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:03:10.114 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:10.114 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:10.114 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:03:10.115 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:17.458662+0000", 2023-09-23T21:03:10.115 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:03:10.115 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:10.115 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:10.115 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:03:10.115 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:17.458661+0000", 2023-09-23T21:03:10.115 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T21:03:10.115 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:10.115 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:10.115 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:03:10.115 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:17.458669+0000", 2023-09-23T21:03:10.116 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.2549999999999997e-06 2023-09-23T21:03:10.116 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:10.116 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:10.116 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:03:10.116 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:17.458686+0000", 2023-09-23T21:03:10.116 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.7163000000000002e-05 2023-09-23T21:03:10.116 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:10.116 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:10.116 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:03:10.116 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:17.458696+0000", 2023-09-23T21:03:10.116 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.6299999999999993e-06 2023-09-23T21:03:10.116 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:10.117 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:10.117 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:03:10.117 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:17.458754+0000", 2023-09-23T21:03:10.117 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.8409999999999998e-05 2023-09-23T21:03:10.117 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:10.117 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:10.117 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:03:10.117 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:17.458896+0000", 2023-09-23T21:03:10.117 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00014230699999999999 2023-09-23T21:03:10.117 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:10.117 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:10.117 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 2,6", 2023-09-23T21:03:10.117 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:17.459343+0000", 2023-09-23T21:03:10.118 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00044672499999999998 2023-09-23T21:03:10.118 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:10.118 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:10.118 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T21:03:10.118 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:17.465297+0000", 2023-09-23T21:03:10.118 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0059539279999999998 2023-09-23T21:03:10.118 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:10.118 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:10.118 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:03:10.118 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:17.473171+0000", 2023-09-23T21:03:10.118 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0078734059999999995 2023-09-23T21:03:10.119 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:10.119 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:10.119 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:03:10.119 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:17.487954+0000", 2023-09-23T21:03:10.119 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.014783889 2023-09-23T21:03:10.119 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:10.119 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:10.119 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T21:03:10.119 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:17.487966+0000", 2023-09-23T21:03:10.119 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1089e-05 2023-09-23T21:03:10.119 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:10.120 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:10.120 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:03:10.120 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:18.522668+0000", 2023-09-23T21:03:10.120 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0347028009999999 2023-09-23T21:03:10.120 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:10.120 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:10.120 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T21:03:10.120 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:18.522756+0000", 2023-09-23T21:03:10.120 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.7397000000000004e-05 2023-09-23T21:03:10.120 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:03:10.120 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:03:10.120 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:03:10.121 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:10.121 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:10.121 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34797.0:165119 6.11 6:888a8162:::benchmark_data_smithi026_223868_object63993:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e154)", 2023-09-23T21:03:10.121 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T20:55:17.470762+0000", 2023-09-23T21:03:10.121 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 472.58265514599998, 2023-09-23T21:03:10.121 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.071012407, 2023-09-23T21:03:10.121 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:03:10.121 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "reached pg", 2023-09-23T21:03:10.121 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T21:03:10.121 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34797", 2023-09-23T21:03:10.121 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T21:03:10.121 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 165119 2023-09-23T21:03:10.122 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:10.122 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:03:10.122 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:10.122 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:03:10.122 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:17.470762+0000", 2023-09-23T21:03:10.122 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:03:10.122 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:10.122 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:10.122 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:03:10.122 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:17.470762+0000", 2023-09-23T21:03:10.122 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:03:10.123 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:10.123 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:10.123 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:03:10.123 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:17.470760+0000", 2023-09-23T21:03:10.123 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999981 2023-09-23T21:03:10.123 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:10.123 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:10.123 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:03:10.123 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:17.470768+0000", 2023-09-23T21:03:10.123 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.3520000000000007e-06 2023-09-23T21:03:10.123 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:10.124 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:10.124 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:03:10.124 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:17.470789+0000", 2023-09-23T21:03:10.124 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.0230000000000001e-05 2023-09-23T21:03:10.124 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:10.124 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:10.124 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:03:10.124 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:17.470799+0000", 2023-09-23T21:03:10.124 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.083e-05 2023-09-23T21:03:10.124 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:10.124 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:10.124 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:03:10.124 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:17.470949+0000", 2023-09-23T21:03:10.125 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00014940900000000001 2023-09-23T21:03:10.125 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:10.125 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:10.125 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:03:10.125 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:17.471101+0000", 2023-09-23T21:03:10.125 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000152552 2023-09-23T21:03:10.125 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:10.125 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:10.125 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 2,6", 2023-09-23T21:03:10.125 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:17.471536+0000", 2023-09-23T21:03:10.125 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00043481999999999999 2023-09-23T21:03:10.126 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:10.126 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:10.126 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T21:03:10.126 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:17.479740+0000", 2023-09-23T21:03:10.126 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0082035990000000007 2023-09-23T21:03:10.126 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:10.126 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:10.126 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:03:10.126 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:17.488026+0000", 2023-09-23T21:03:10.126 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0082866199999999997 2023-09-23T21:03:10.126 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:10.126 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:10.127 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T21:03:10.127 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:17.488036+0000", 2023-09-23T21:03:10.127 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.3840000000000003e-06 2023-09-23T21:03:10.127 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:10.127 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:10.127 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:03:10.127 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:18.523188+0000", 2023-09-23T21:03:10.127 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.035152157 2023-09-23T21:03:10.127 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:10.127 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:10.127 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for missing object", 2023-09-23T21:03:10.127 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:18.523282+0000", 2023-09-23T21:03:10.127 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.4153000000000005e-05 2023-09-23T21:03:10.128 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:10.128 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:10.128 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:03:10.128 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:18.541721+0000", 2023-09-23T21:03:10.128 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.018438985000000001 2023-09-23T21:03:10.128 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:10.128 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:10.128 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T21:03:10.128 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:18.541774+0000", 2023-09-23T21:03:10.128 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.3115000000000002e-05 2023-09-23T21:03:10.128 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:03:10.128 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:03:10.129 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:03:10.129 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:10.129 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:10.129 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34797.0:165120 6.4 6:2711d346:::benchmark_data_smithi026_223868_object63994:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e154)", 2023-09-23T21:03:10.129 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T20:55:17.470823+0000", 2023-09-23T21:03:10.129 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 472.58259428500003, 2023-09-23T21:03:10.129 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.034766882, 2023-09-23T21:03:10.129 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:03:10.129 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "reached pg", 2023-09-23T21:03:10.129 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T21:03:10.129 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34797", 2023-09-23T21:03:10.129 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T21:03:10.130 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 165120 2023-09-23T21:03:10.130 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:10.130 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:03:10.130 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:10.130 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:03:10.130 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:17.470823+0000", 2023-09-23T21:03:10.130 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:03:10.130 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:10.130 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:10.130 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:03:10.130 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:17.470823+0000", 2023-09-23T21:03:10.130 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:03:10.131 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:10.131 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:10.131 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:03:10.131 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:17.470822+0000", 2023-09-23T21:03:10.131 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T21:03:10.131 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:10.131 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:10.131 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:03:10.131 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:17.470826+0000", 2023-09-23T21:03:10.131 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.7270000000000001e-06 2023-09-23T21:03:10.131 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:10.131 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:10.131 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:03:10.132 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:17.470834+0000", 2023-09-23T21:03:10.132 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.8679999999999999e-06 2023-09-23T21:03:10.132 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:10.132 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:10.132 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:03:10.132 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:17.470840+0000", 2023-09-23T21:03:10.132 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.2750000000000001e-06 2023-09-23T21:03:10.132 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:10.132 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:10.132 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:03:10.132 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:17.473228+0000", 2023-09-23T21:03:10.133 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0023883070000000001 2023-09-23T21:03:10.133 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:10.133 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:10.133 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:03:10.133 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:17.473373+0000", 2023-09-23T21:03:10.133 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00014514199999999999 2023-09-23T21:03:10.133 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:10.133 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:10.133 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 3,5,6", 2023-09-23T21:03:10.133 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:17.473855+0000", 2023-09-23T21:03:10.133 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00048128899999999998 2023-09-23T21:03:10.134 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:10.134 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:10.134 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:03:10.134 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:17.479648+0000", 2023-09-23T21:03:10.134 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0057931959999999996 2023-09-23T21:03:10.134 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:10.134 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:10.134 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T21:03:10.134 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:17.479658+0000", 2023-09-23T21:03:10.134 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0404999999999999e-05 2023-09-23T21:03:10.134 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:10.135 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:10.135 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:03:10.135 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:17.487974+0000", 2023-09-23T21:03:10.135 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.008316024 2023-09-23T21:03:10.135 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:10.135 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:10.135 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T21:03:10.135 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:17.487985+0000", 2023-09-23T21:03:10.135 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0193e-05 2023-09-23T21:03:10.135 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:10.135 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:10.135 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:03:10.136 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:18.505499+0000", 2023-09-23T21:03:10.136 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0175144899999999 2023-09-23T21:03:10.136 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:10.136 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:10.136 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T21:03:10.136 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:18.505590+0000", 2023-09-23T21:03:10.136 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.0523e-05 2023-09-23T21:03:10.136 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:03:10.136 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:03:10.136 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:03:10.136 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:10.136 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:10.137 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34797.0:165133 6.4 6:2151236b:::benchmark_data_smithi026_223868_object64007:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e155)", 2023-09-23T21:03:10.137 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T20:55:17.506170+0000", 2023-09-23T21:03:10.137 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 472.54724668300003, 2023-09-23T21:03:10.137 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.062355591, 2023-09-23T21:03:10.137 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:03:10.137 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:03:10.137 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T21:03:10.137 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34797", 2023-09-23T21:03:10.137 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T21:03:10.137 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 165133 2023-09-23T21:03:10.137 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:10.137 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:03:10.137 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:10.138 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:03:10.138 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:17.506170+0000", 2023-09-23T21:03:10.138 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:03:10.138 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:10.138 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:10.138 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:03:10.138 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:17.506170+0000", 2023-09-23T21:03:10.138 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:03:10.138 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:10.138 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:10.138 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:03:10.139 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:17.506169+0000", 2023-09-23T21:03:10.139 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999981 2023-09-23T21:03:10.139 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:10.139 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:10.139 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:03:10.139 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:17.506178+0000", 2023-09-23T21:03:10.139 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.3230000000000004e-06 2023-09-23T21:03:10.139 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:10.139 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:10.139 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:03:10.139 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:17.506202+0000", 2023-09-23T21:03:10.140 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.3856999999999999e-05 2023-09-23T21:03:10.140 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:10.140 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:10.140 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:03:10.140 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:17.506213+0000", 2023-09-23T21:03:10.140 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1454000000000001e-05 2023-09-23T21:03:10.140 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:10.140 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:10.140 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:03:10.140 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:17.506304+0000", 2023-09-23T21:03:10.140 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.1155000000000004e-05 2023-09-23T21:03:10.140 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:10.140 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:10.141 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T21:03:10.141 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:17.506314+0000", 2023-09-23T21:03:10.141 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.6509999999999996e-06 2023-09-23T21:03:10.141 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:10.141 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:10.141 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:03:10.141 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:18.505917+0000", 2023-09-23T21:03:10.141 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.99960311199999996 2023-09-23T21:03:10.141 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:10.141 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:10.141 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:03:10.142 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:18.506065+0000", 2023-09-23T21:03:10.142 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00014810299999999999 2023-09-23T21:03:10.142 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:10.142 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:10.142 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 3,5", 2023-09-23T21:03:10.142 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:18.506566+0000", 2023-09-23T21:03:10.142 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00050105199999999999 2023-09-23T21:03:10.142 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:10.142 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:10.142 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T21:03:10.142 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:18.508896+0000", 2023-09-23T21:03:10.142 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0023299129999999999 2023-09-23T21:03:10.143 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:10.143 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:10.143 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:03:10.143 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:18.509051+0000", 2023-09-23T21:03:10.143 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00015425899999999999 2023-09-23T21:03:10.143 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:10.143 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:10.143 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:03:10.143 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:18.568363+0000", 2023-09-23T21:03:10.143 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.059312294000000002 2023-09-23T21:03:10.143 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:10.143 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:10.143 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T21:03:10.144 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:18.568418+0000", 2023-09-23T21:03:10.144 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.5701000000000001e-05 2023-09-23T21:03:10.144 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:10.144 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:10.144 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T21:03:10.144 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:18.568526+0000", 2023-09-23T21:03:10.144 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000107416 2023-09-23T21:03:10.144 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:03:10.144 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:03:10.144 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:03:10.144 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:10.145 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:10.145 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34797.0:165136 6.1 6:8407df14:::benchmark_data_smithi026_223868_object64010:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e155)", 2023-09-23T21:03:10.145 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T20:55:17.506228+0000", 2023-09-23T21:03:10.145 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 472.547188823, 2023-09-23T21:03:10.145 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0166797729999999, 2023-09-23T21:03:10.145 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:03:10.145 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:03:10.145 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T21:03:10.145 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34797", 2023-09-23T21:03:10.145 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T21:03:10.145 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 165136 2023-09-23T21:03:10.145 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:10.145 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:03:10.146 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:10.146 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:03:10.146 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:17.506228+0000", 2023-09-23T21:03:10.146 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:03:10.146 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:10.146 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:10.146 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:03:10.146 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:17.506228+0000", 2023-09-23T21:03:10.146 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:03:10.146 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:10.146 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:10.147 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:03:10.147 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:17.506228+0000", 2023-09-23T21:03:10.147 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T21:03:10.147 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:10.147 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:10.147 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:03:10.147 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:17.506231+0000", 2023-09-23T21:03:10.147 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.0460000000000001e-06 2023-09-23T21:03:10.147 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:10.147 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:10.147 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:03:10.147 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:17.506237+0000", 2023-09-23T21:03:10.147 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.9440000000000002e-06 2023-09-23T21:03:10.148 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:10.148 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:10.148 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:03:10.148 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:17.506242+0000", 2023-09-23T21:03:10.148 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.1109999999999999e-06 2023-09-23T21:03:10.148 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:10.148 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:10.148 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:03:10.148 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:17.506295+0000", 2023-09-23T21:03:10.148 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.2828999999999999e-05 2023-09-23T21:03:10.148 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:10.148 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:10.149 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T21:03:10.149 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:17.506305+0000", 2023-09-23T21:03:10.149 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0373999999999999e-05 2023-09-23T21:03:10.149 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:10.149 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:10.149 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:03:10.149 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:18.494023+0000", 2023-09-23T21:03:10.149 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.98771804900000004 2023-09-23T21:03:10.149 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:10.149 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:10.149 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:03:10.149 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:18.494131+0000", 2023-09-23T21:03:10.150 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000107593 2023-09-23T21:03:10.150 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:10.150 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:10.150 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 2", 2023-09-23T21:03:10.150 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:18.494586+0000", 2023-09-23T21:03:10.150 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00045516799999999999 2023-09-23T21:03:10.150 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:10.150 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:10.150 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T21:03:10.150 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:18.522748+0000", 2023-09-23T21:03:10.150 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.028161951000000001 2023-09-23T21:03:10.150 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:10.151 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:10.151 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:03:10.151 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:18.522790+0000", 2023-09-23T21:03:10.151 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.1777999999999999e-05 2023-09-23T21:03:10.151 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:10.151 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:10.151 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T21:03:10.151 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:18.522828+0000", 2023-09-23T21:03:10.151 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.8386999999999997e-05 2023-09-23T21:03:10.151 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:10.151 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:10.151 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T21:03:10.151 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:18.522908+0000", 2023-09-23T21:03:10.152 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.9939999999999997e-05 2023-09-23T21:03:10.152 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:03:10.152 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:03:10.152 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:03:10.152 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:10.152 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:10.152 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34797.0:165140 6.1 6:804ea438:::benchmark_data_smithi026_223868_object64014:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e155)", 2023-09-23T21:03:10.152 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T20:55:17.506959+0000", 2023-09-23T21:03:10.152 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 472.546458223, 2023-09-23T21:03:10.152 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0315099489999999, 2023-09-23T21:03:10.152 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:03:10.152 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:03:10.153 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T21:03:10.153 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34797", 2023-09-23T21:03:10.153 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T21:03:10.153 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 165140 2023-09-23T21:03:10.153 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:10.153 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:03:10.153 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:10.153 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:03:10.153 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:17.506959+0000", 2023-09-23T21:03:10.153 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:03:10.153 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:10.153 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:10.153 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:03:10.154 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:17.506959+0000", 2023-09-23T21:03:10.154 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:03:10.154 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:10.154 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:10.154 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:03:10.154 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:17.506958+0000", 2023-09-23T21:03:10.154 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.999999 2023-09-23T21:03:10.154 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:10.154 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:10.154 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:03:10.154 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:17.506964+0000", 2023-09-23T21:03:10.154 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.9399999999999999e-06 2023-09-23T21:03:10.155 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:10.155 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:10.155 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:03:10.155 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:17.506979+0000", 2023-09-23T21:03:10.155 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.5164000000000001e-05 2023-09-23T21:03:10.155 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:10.155 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:10.155 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:03:10.155 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:17.506987+0000", 2023-09-23T21:03:10.155 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.0369999999999995e-06 2023-09-23T21:03:10.155 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:10.155 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:10.155 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:03:10.156 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:17.507038+0000", 2023-09-23T21:03:10.156 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.0719e-05 2023-09-23T21:03:10.156 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:10.156 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:10.156 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T21:03:10.156 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:17.507047+0000", 2023-09-23T21:03:10.156 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.5880000000000004e-06 2023-09-23T21:03:10.156 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:10.156 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:10.156 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:03:10.156 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:18.523352+0000", 2023-09-23T21:03:10.157 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0163044240000001 2023-09-23T21:03:10.157 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:10.157 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:10.157 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:03:10.157 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:18.523449+0000", 2023-09-23T21:03:10.157 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.7126999999999994e-05 2023-09-23T21:03:10.157 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:10.157 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:10.157 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 2", 2023-09-23T21:03:10.157 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:18.523867+0000", 2023-09-23T21:03:10.157 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00041800399999999998 2023-09-23T21:03:10.157 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:10.157 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:10.158 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T21:03:10.158 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:18.528419+0000", 2023-09-23T21:03:10.158 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0045518729999999997 2023-09-23T21:03:10.158 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:10.158 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:10.158 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:03:10.158 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:18.538351+0000", 2023-09-23T21:03:10.158 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0099326940000000006 2023-09-23T21:03:10.158 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:10.158 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:10.158 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T21:03:10.158 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:18.538403+0000", 2023-09-23T21:03:10.159 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.1396000000000001e-05 2023-09-23T21:03:10.159 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:10.159 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:10.159 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T21:03:10.159 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:18.538469+0000", 2023-09-23T21:03:10.159 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.5953999999999999e-05 2023-09-23T21:03:10.159 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:03:10.159 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:03:10.159 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:03:10.159 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:10.159 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:10.159 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34797.0:165141 6.11 6:8bd9c32d:::benchmark_data_smithi026_223868_object64015:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e155)", 2023-09-23T21:03:10.160 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T20:55:17.507141+0000", 2023-09-23T21:03:10.160 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 472.54627627299999, 2023-09-23T21:03:10.160 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.031517094, 2023-09-23T21:03:10.160 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:03:10.160 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:03:10.160 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T21:03:10.160 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34797", 2023-09-23T21:03:10.160 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T21:03:10.160 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 165141 2023-09-23T21:03:10.160 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:10.160 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:03:10.160 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:10.161 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:03:10.161 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:17.507141+0000", 2023-09-23T21:03:10.161 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:03:10.161 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:10.161 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:10.161 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:03:10.161 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:17.507141+0000", 2023-09-23T21:03:10.161 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:03:10.161 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:10.161 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:10.161 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:03:10.161 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:17.507140+0000", 2023-09-23T21:03:10.161 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.999999 2023-09-23T21:03:10.162 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:10.162 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:10.162 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:03:10.162 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:17.507146+0000", 2023-09-23T21:03:10.162 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.0249999999999999e-06 2023-09-23T21:03:10.162 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:10.162 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:10.162 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:03:10.162 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:17.507161+0000", 2023-09-23T21:03:10.162 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.541e-05 2023-09-23T21:03:10.162 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:10.162 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:10.163 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:03:10.163 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:17.507169+0000", 2023-09-23T21:03:10.163 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.9670000000000001e-06 2023-09-23T21:03:10.163 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:10.163 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:10.163 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:03:10.163 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:17.507225+0000", 2023-09-23T21:03:10.163 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.6239000000000001e-05 2023-09-23T21:03:10.163 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:10.163 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:10.163 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T21:03:10.163 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:17.507234+0000", 2023-09-23T21:03:10.163 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.8829999999999995e-06 2023-09-23T21:03:10.164 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:10.164 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:10.164 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:03:10.164 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:18.526050+0000", 2023-09-23T21:03:10.164 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0188153660000001 2023-09-23T21:03:10.164 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:10.164 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:10.164 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:03:10.164 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:18.526116+0000", 2023-09-23T21:03:10.164 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.6645999999999999e-05 2023-09-23T21:03:10.164 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:10.164 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:10.165 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 2", 2023-09-23T21:03:10.165 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:18.526440+0000", 2023-09-23T21:03:10.165 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00032320800000000001 2023-09-23T21:03:10.165 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:10.165 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:10.165 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T21:03:10.165 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:18.532621+0000", 2023-09-23T21:03:10.165 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0061811289999999996 2023-09-23T21:03:10.165 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:10.165 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:10.165 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:03:10.165 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:18.538524+0000", 2023-09-23T21:03:10.165 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0059037450000000002 2023-09-23T21:03:10.166 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:10.166 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:10.166 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T21:03:10.166 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:18.538569+0000", 2023-09-23T21:03:10.166 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.4977e-05 2023-09-23T21:03:10.166 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:10.166 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:10.166 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T21:03:10.166 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:18.538658+0000", 2023-09-23T21:03:10.166 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.8459000000000001e-05 2023-09-23T21:03:10.166 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:03:10.166 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:03:10.167 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:03:10.167 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:10.167 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:10.167 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34797.0:177915 6.11 6:8818350e:::benchmark_data_smithi026_223868_object76789:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e164)", 2023-09-23T21:03:10.167 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T20:55:34.933656+0000", 2023-09-23T21:03:10.167 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 455.11976116099999, 2023-09-23T21:03:10.167 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.0860430320000001, 2023-09-23T21:03:10.167 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:03:10.167 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:03:10.167 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T21:03:10.167 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34797", 2023-09-23T21:03:10.167 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T21:03:10.167 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 177915 2023-09-23T21:03:10.168 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:10.168 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:03:10.168 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:10.168 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:03:10.168 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:34.933656+0000", 2023-09-23T21:03:10.168 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:03:10.168 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:10.168 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:10.168 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:03:10.168 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:34.933656+0000", 2023-09-23T21:03:10.168 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:03:10.168 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:10.169 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:10.169 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:03:10.169 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:34.933654+0000", 2023-09-23T21:03:10.169 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999981 2023-09-23T21:03:10.169 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:10.169 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:10.169 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:03:10.169 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:34.933665+0000", 2023-09-23T21:03:10.169 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0963e-05 2023-09-23T21:03:10.169 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:10.169 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:10.169 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:03:10.170 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:34.933690+0000", 2023-09-23T21:03:10.170 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.5219999999999999e-05 2023-09-23T21:03:10.170 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:10.170 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:10.170 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:03:10.170 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:34.933701+0000", 2023-09-23T21:03:10.170 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0944000000000001e-05 2023-09-23T21:03:10.170 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:10.170 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:10.170 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:03:10.170 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:34.933803+0000", 2023-09-23T21:03:10.170 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000101654 2023-09-23T21:03:10.171 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:10.171 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:10.171 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op must wait for map", 2023-09-23T21:03:10.171 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:34.933811+0000", 2023-09-23T21:03:10.171 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.2190000000000004e-06 2023-09-23T21:03:10.171 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:10.171 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:10.171 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:03:10.171 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:34.946283+0000", 2023-09-23T21:03:10.171 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.012472298 2023-09-23T21:03:10.171 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:10.171 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:10.171 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T21:03:10.172 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:34.946294+0000", 2023-09-23T21:03:10.172 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0832999999999999e-05 2023-09-23T21:03:10.172 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:10.172 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:10.172 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:03:10.172 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:35.931861+0000", 2023-09-23T21:03:10.172 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.98556678499999995 2023-09-23T21:03:10.172 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:10.172 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:10.172 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T21:03:10.172 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:35.931870+0000", 2023-09-23T21:03:10.173 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.5799999999999992e-06 2023-09-23T21:03:10.173 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:10.173 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:10.173 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:03:10.173 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:37.013529+0000", 2023-09-23T21:03:10.173 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0816588920000001 2023-09-23T21:03:10.173 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:10.173 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:10.173 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:03:10.173 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:37.013686+0000", 2023-09-23T21:03:10.173 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000157825 2023-09-23T21:03:10.173 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:10.173 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:10.174 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 2,6", 2023-09-23T21:03:10.174 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:37.014148+0000", 2023-09-23T21:03:10.174 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00046177499999999999 2023-09-23T21:03:10.174 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:10.174 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:10.174 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T21:03:10.174 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:37.019388+0000", 2023-09-23T21:03:10.174 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0052396250000000004 2023-09-23T21:03:10.174 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:10.174 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:10.174 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:03:10.174 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:37.019453+0000", 2023-09-23T21:03:10.175 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.5685000000000006e-05 2023-09-23T21:03:10.175 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:10.175 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:10.175 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:03:10.175 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:37.019503+0000", 2023-09-23T21:03:10.175 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.9861999999999998e-05 2023-09-23T21:03:10.175 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:10.175 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:10.175 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T21:03:10.175 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:37.019559+0000", 2023-09-23T21:03:10.175 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.5605e-05 2023-09-23T21:03:10.175 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:10.176 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:10.176 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T21:03:10.176 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:37.019699+0000", 2023-09-23T21:03:10.176 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00013997099999999999 2023-09-23T21:03:10.176 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:03:10.176 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:03:10.176 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:03:10.176 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:10.176 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:10.176 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34797.0:177930 6.1 6:872a742b:::benchmark_data_smithi026_223868_object76804:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e164)", 2023-09-23T21:03:10.176 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T20:55:34.965696+0000", 2023-09-23T21:03:10.176 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 455.08772098200001, 2023-09-23T21:03:10.177 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.0561283210000001, 2023-09-23T21:03:10.177 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:03:10.177 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:03:10.177 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T21:03:10.177 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34797", 2023-09-23T21:03:10.177 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T21:03:10.177 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 177930 2023-09-23T21:03:10.177 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:10.177 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:03:10.177 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:10.177 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:03:10.177 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:34.965696+0000", 2023-09-23T21:03:10.178 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:03:10.178 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:10.178 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:10.178 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:03:10.178 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:34.965696+0000", 2023-09-23T21:03:10.178 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:03:10.178 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:10.178 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:10.178 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:03:10.178 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:34.965694+0000", 2023-09-23T21:03:10.178 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999981 2023-09-23T21:03:10.178 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:10.178 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:10.179 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:03:10.179 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:34.965702+0000", 2023-09-23T21:03:10.179 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.1000000000000004e-06 2023-09-23T21:03:10.179 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:10.179 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:10.179 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:03:10.179 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:34.965716+0000", 2023-09-23T21:03:10.179 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.3190000000000001e-05 2023-09-23T21:03:10.179 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:10.179 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:10.179 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:03:10.180 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:34.965724+0000", 2023-09-23T21:03:10.180 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.5699999999999993e-06 2023-09-23T21:03:10.180 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:10.180 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:10.180 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:03:10.180 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:34.965780+0000", 2023-09-23T21:03:10.180 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.5807999999999998e-05 2023-09-23T21:03:10.180 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:10.180 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:10.180 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T21:03:10.180 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:34.965791+0000", 2023-09-23T21:03:10.180 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0597e-05 2023-09-23T21:03:10.180 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:10.181 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:10.181 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:03:10.181 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:35.931819+0000", 2023-09-23T21:03:10.181 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.96602831899999997 2023-09-23T21:03:10.181 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:10.181 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:10.181 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T21:03:10.181 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:35.931833+0000", 2023-09-23T21:03:10.181 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.4409999999999999e-05 2023-09-23T21:03:10.181 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:10.181 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:10.181 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:03:10.182 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:37.016222+0000", 2023-09-23T21:03:10.182 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.084388186 2023-09-23T21:03:10.182 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:10.182 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:10.182 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:03:10.182 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:37.016369+0000", 2023-09-23T21:03:10.182 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00014728699999999999 2023-09-23T21:03:10.182 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:10.182 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:10.182 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 2,6", 2023-09-23T21:03:10.182 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:37.016878+0000", 2023-09-23T21:03:10.183 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000509068 2023-09-23T21:03:10.183 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:10.183 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:10.183 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T21:03:10.183 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:37.020205+0000", 2023-09-23T21:03:10.183 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0033272660000000002 2023-09-23T21:03:10.183 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:10.183 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:10.183 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:03:10.183 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:37.020577+0000", 2023-09-23T21:03:10.183 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00037149500000000001 2023-09-23T21:03:10.183 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:10.183 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:10.184 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:03:10.184 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:37.021628+0000", 2023-09-23T21:03:10.184 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0010512239999999999 2023-09-23T21:03:10.184 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:10.184 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:10.184 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T21:03:10.184 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:37.021685+0000", 2023-09-23T21:03:10.184 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.6715999999999997e-05 2023-09-23T21:03:10.184 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:10.184 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:10.184 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T21:03:10.185 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:37.021824+0000", 2023-09-23T21:03:10.185 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00013976799999999999 2023-09-23T21:03:10.185 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:03:10.185 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:03:10.185 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:03:10.185 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:10.185 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:10.185 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34797.0:186112 6.1 6:811b2b92:::benchmark_data_smithi026_223868_object84986:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e173)", 2023-09-23T21:03:10.185 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T20:55:49.293452+0000", 2023-09-23T21:03:10.185 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 440.759964868, 2023-09-23T21:03:10.185 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.94933223799999999, 2023-09-23T21:03:10.185 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:03:10.186 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:03:10.186 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T21:03:10.186 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34797", 2023-09-23T21:03:10.186 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T21:03:10.186 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 186112 2023-09-23T21:03:10.186 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:10.186 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:03:10.186 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:10.186 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:03:10.186 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:49.293452+0000", 2023-09-23T21:03:10.186 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:03:10.187 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:10.187 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:10.187 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:03:10.187 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:49.293452+0000", 2023-09-23T21:03:10.187 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:03:10.187 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:10.187 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:10.187 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:03:10.187 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:49.293450+0000", 2023-09-23T21:03:10.187 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T21:03:10.187 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:10.187 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:10.187 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:03:10.188 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:49.293459+0000", 2023-09-23T21:03:10.188 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.1119999999999996e-06 2023-09-23T21:03:10.188 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:10.188 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:10.188 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:03:10.188 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:49.293471+0000", 2023-09-23T21:03:10.188 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.2733e-05 2023-09-23T21:03:10.188 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:10.188 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:10.188 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:03:10.188 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:49.293496+0000", 2023-09-23T21:03:10.188 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.4491e-05 2023-09-23T21:03:10.189 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:10.189 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:10.189 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:03:10.189 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:49.293553+0000", 2023-09-23T21:03:10.189 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.7287999999999998e-05 2023-09-23T21:03:10.189 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:10.189 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:10.189 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T21:03:10.189 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:49.293563+0000", 2023-09-23T21:03:10.189 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.7280000000000007e-06 2023-09-23T21:03:10.189 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:10.189 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:10.190 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:03:10.190 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:50.239133+0000", 2023-09-23T21:03:10.190 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.94557034299999998 2023-09-23T21:03:10.190 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:10.190 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:10.190 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:03:10.190 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:50.239271+0000", 2023-09-23T21:03:10.190 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00013830900000000001 2023-09-23T21:03:10.190 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:10.190 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:10.190 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 2,6", 2023-09-23T21:03:10.191 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:50.239761+0000", 2023-09-23T21:03:10.191 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00048966600000000004 2023-09-23T21:03:10.191 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:10.191 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:10.191 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:03:10.191 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:50.242609+0000", 2023-09-23T21:03:10.191 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.002847788 2023-09-23T21:03:10.191 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:10.191 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:10.191 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:03:10.191 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:50.242679+0000", 2023-09-23T21:03:10.191 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.9982999999999997e-05 2023-09-23T21:03:10.192 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:10.192 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:10.192 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T21:03:10.192 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:50.242692+0000", 2023-09-23T21:03:10.192 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.3074999999999999e-05 2023-09-23T21:03:10.192 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:10.192 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:10.192 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T21:03:10.192 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:50.242742+0000", 2023-09-23T21:03:10.192 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.9588e-05 2023-09-23T21:03:10.192 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:10.192 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:10.193 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T21:03:10.193 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:50.242784+0000", 2023-09-23T21:03:10.193 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.2784999999999998e-05 2023-09-23T21:03:10.193 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:03:10.193 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:03:10.193 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:03:10.193 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:10.193 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:10.193 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34797.0:186150 6.1 6:8379824f:::benchmark_data_smithi026_223868_object85024:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e173)", 2023-09-23T21:03:10.193 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T20:55:49.342794+0000", 2023-09-23T21:03:10.193 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 440.71062292699997, 2023-09-23T21:03:10.193 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.91199801400000002, 2023-09-23T21:03:10.194 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:03:10.194 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:03:10.194 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T21:03:10.194 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34797", 2023-09-23T21:03:10.194 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T21:03:10.194 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 186150 2023-09-23T21:03:10.194 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:10.194 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:03:10.194 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:10.194 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:03:10.194 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:49.342794+0000", 2023-09-23T21:03:10.195 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:03:10.195 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:10.195 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:10.195 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:03:10.195 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:49.342794+0000", 2023-09-23T21:03:10.195 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:03:10.195 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:10.195 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:10.195 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:03:10.195 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:49.342794+0000", 2023-09-23T21:03:10.195 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T21:03:10.195 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:10.196 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:10.196 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:03:10.196 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:49.342797+0000", 2023-09-23T21:03:10.196 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.8550000000000004e-06 2023-09-23T21:03:10.196 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:10.196 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:10.196 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:03:10.196 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:49.342805+0000", 2023-09-23T21:03:10.196 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.5240000000000003e-06 2023-09-23T21:03:10.196 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:10.196 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:10.197 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:03:10.197 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:49.342811+0000", 2023-09-23T21:03:10.197 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.0100000000000001e-06 2023-09-23T21:03:10.197 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:10.197 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:10.197 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:03:10.197 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:49.345601+0000", 2023-09-23T21:03:10.197 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0027898060000000001 2023-09-23T21:03:10.197 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:10.197 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:10.197 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T21:03:10.197 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:49.345613+0000", 2023-09-23T21:03:10.197 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.2496e-05 2023-09-23T21:03:10.198 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:10.198 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:10.198 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:03:10.198 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:50.242813+0000", 2023-09-23T21:03:10.198 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.897200212 2023-09-23T21:03:10.198 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:10.198 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:10.198 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:03:10.198 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:50.242959+0000", 2023-09-23T21:03:10.198 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00014535700000000001 2023-09-23T21:03:10.198 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:10.199 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:10.199 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 2,6", 2023-09-23T21:03:10.199 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:50.243322+0000", 2023-09-23T21:03:10.199 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00036353299999999998 2023-09-23T21:03:10.199 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:10.199 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:10.199 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:03:10.199 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:50.248896+0000", 2023-09-23T21:03:10.199 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0055737640000000001 2023-09-23T21:03:10.199 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:10.199 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:10.200 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:03:10.200 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:50.248919+0000", 2023-09-23T21:03:10.200 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.2874000000000002e-05 2023-09-23T21:03:10.200 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:10.200 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:10.200 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T21:03:10.200 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:50.254682+0000", 2023-09-23T21:03:10.200 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.005763191 2023-09-23T21:03:10.200 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:10.200 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:10.200 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T21:03:10.200 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:50.254743+0000", 2023-09-23T21:03:10.201 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.0504000000000002e-05 2023-09-23T21:03:10.201 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:10.201 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:10.201 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T21:03:10.201 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:50.254792+0000", 2023-09-23T21:03:10.201 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.9422999999999998e-05 2023-09-23T21:03:10.201 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:03:10.201 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:03:10.201 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:03:10.201 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:10.201 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:10.201 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34797.0:186170 6.1 6:87d9c797:::benchmark_data_smithi026_223868_object85044:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e173)", 2023-09-23T21:03:10.202 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T20:55:49.366035+0000", 2023-09-23T21:03:10.202 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 440.68738208500002, 2023-09-23T21:03:10.202 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.88903586499999998, 2023-09-23T21:03:10.202 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:03:10.202 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:03:10.202 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T21:03:10.202 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:03:10.203 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34797", 2023-09-23T21:03:10.203 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T21:03:10.203 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 186170 2023-09-23T21:03:10.203 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:10.204 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:03:10.204 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:10.204 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:03:10.204 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:49.366035+0000", 2023-09-23T21:03:10.204 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:03:10.204 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:10.204 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:10.204 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:03:10.204 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:49.366035+0000", 2023-09-23T21:03:10.204 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:03:10.204 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:10.204 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:10.205 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:03:10.205 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:49.366033+0000", 2023-09-23T21:03:10.205 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T21:03:10.205 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:10.205 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:10.205 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:03:10.205 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:49.366042+0000", 2023-09-23T21:03:10.205 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.6959999999999994e-06 2023-09-23T21:03:10.205 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:10.205 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:10.205 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:03:10.206 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:49.366062+0000", 2023-09-23T21:03:10.206 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.0193000000000001e-05 2023-09-23T21:03:10.206 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:10.206 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:10.206 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:03:10.206 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:49.366072+0000", 2023-09-23T21:03:10.206 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.3090000000000001e-06 2023-09-23T21:03:10.206 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:10.206 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:10.206 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:03:10.206 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:49.366149+0000", 2023-09-23T21:03:10.206 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.7767000000000003e-05 2023-09-23T21:03:10.207 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:10.207 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:10.207 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T21:03:10.207 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:49.366162+0000", 2023-09-23T21:03:10.207 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.3036e-05 2023-09-23T21:03:10.207 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:10.207 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:10.207 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:03:10.207 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:50.245889+0000", 2023-09-23T21:03:10.207 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.87972683699999998 2023-09-23T21:03:10.207 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:10.208 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:10.208 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:03:10.208 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:50.246006+0000", 2023-09-23T21:03:10.208 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000116896 2023-09-23T21:03:10.208 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:10.208 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:10.208 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 2,6", 2023-09-23T21:03:10.208 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:50.246405+0000", 2023-09-23T21:03:10.208 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00039916399999999997 2023-09-23T21:03:10.208 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:10.208 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:10.208 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:03:10.208 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:50.254865+0000", 2023-09-23T21:03:10.209 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0084600379999999996 2023-09-23T21:03:10.209 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:10.209 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:10.209 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T21:03:10.209 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:50.254922+0000", 2023-09-23T21:03:10.209 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.6372000000000002e-05 2023-09-23T21:03:10.209 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:10.209 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:10.209 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:03:10.209 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:50.254974+0000", 2023-09-23T21:03:10.209 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.2856999999999999e-05 2023-09-23T21:03:10.209 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:10.209 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:10.210 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T21:03:10.210 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:50.255030+0000", 2023-09-23T21:03:10.210 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.5229000000000003e-05 2023-09-23T21:03:10.210 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:10.210 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:10.210 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T21:03:10.210 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:50.255071+0000", 2023-09-23T21:03:10.210 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.1087999999999997e-05 2023-09-23T21:03:10.210 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:03:10.210 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:03:10.210 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:03:10.211 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:10.211 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:10.211 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34797.0:186189 6.1 6:843a1f60:::benchmark_data_smithi026_223868_object85063:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e173)", 2023-09-23T21:03:10.211 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T20:55:49.388644+0000", 2023-09-23T21:03:10.211 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 440.66477287700002, 2023-09-23T21:03:10.211 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.86656724399999996, 2023-09-23T21:03:10.211 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:03:10.211 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:03:10.211 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T21:03:10.211 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34797", 2023-09-23T21:03:10.211 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T21:03:10.211 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 186189 2023-09-23T21:03:10.211 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:10.212 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:03:10.212 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:10.212 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:03:10.212 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:49.388644+0000", 2023-09-23T21:03:10.212 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:03:10.212 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:10.212 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:10.212 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:03:10.212 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:49.388644+0000", 2023-09-23T21:03:10.212 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:03:10.212 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:10.213 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:10.213 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:03:10.213 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:49.388642+0000", 2023-09-23T21:03:10.213 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999976 2023-09-23T21:03:10.213 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:10.213 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:10.213 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:03:10.213 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:49.388651+0000", 2023-09-23T21:03:10.213 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.6590000000000002e-06 2023-09-23T21:03:10.213 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:10.213 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:10.213 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:03:10.213 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:49.388672+0000", 2023-09-23T21:03:10.214 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.0905000000000001e-05 2023-09-23T21:03:10.214 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:10.214 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:10.214 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:03:10.214 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:49.388683+0000", 2023-09-23T21:03:10.214 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0185e-05 2023-09-23T21:03:10.214 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:10.214 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:10.214 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:03:10.214 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:49.388811+0000", 2023-09-23T21:03:10.214 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000128283 2023-09-23T21:03:10.215 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:10.215 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:10.215 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T21:03:10.215 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:49.388823+0000", 2023-09-23T21:03:10.215 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.2542e-05 2023-09-23T21:03:10.215 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:10.215 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:10.215 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:03:10.215 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:50.248973+0000", 2023-09-23T21:03:10.215 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.86014973400000005 2023-09-23T21:03:10.215 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:10.215 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:10.215 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:03:10.216 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:50.249073+0000", 2023-09-23T21:03:10.216 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00010032800000000001 2023-09-23T21:03:10.216 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:10.216 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:10.216 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 2,6", 2023-09-23T21:03:10.216 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:50.249412+0000", 2023-09-23T21:03:10.216 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00033831300000000003 2023-09-23T21:03:10.216 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:10.216 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:10.216 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:03:10.216 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:50.254915+0000", 2023-09-23T21:03:10.216 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0055030490000000003 2023-09-23T21:03:10.217 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:10.217 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:10.217 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T21:03:10.217 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:50.254926+0000", 2023-09-23T21:03:10.217 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0869e-05 2023-09-23T21:03:10.217 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:10.217 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:10.217 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:03:10.217 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:50.255121+0000", 2023-09-23T21:03:10.217 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000195453 2023-09-23T21:03:10.217 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:10.217 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:10.217 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T21:03:10.218 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:50.255169+0000", 2023-09-23T21:03:10.218 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.8062000000000002e-05 2023-09-23T21:03:10.218 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:10.218 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:10.218 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T21:03:10.218 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:50.255211+0000", 2023-09-23T21:03:10.218 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.2154999999999999e-05 2023-09-23T21:03:10.218 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:03:10.218 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:03:10.218 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:03:10.218 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:10.218 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:10.218 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34797.0:186247 6.1 6:87e4ccd2:::benchmark_data_smithi026_223868_object85121:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e173)", 2023-09-23T21:03:10.219 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T20:55:49.450543+0000", 2023-09-23T21:03:10.219 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 440.60287373400001, 2023-09-23T21:03:10.219 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.81119991499999999, 2023-09-23T21:03:10.219 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:03:10.219 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:03:10.219 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T21:03:10.219 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34797", 2023-09-23T21:03:10.219 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T21:03:10.219 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 186247 2023-09-23T21:03:10.219 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:10.219 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:03:10.220 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:10.220 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:03:10.220 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:49.450543+0000", 2023-09-23T21:03:10.220 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:03:10.220 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:10.220 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:10.220 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:03:10.220 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:49.450543+0000", 2023-09-23T21:03:10.220 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:03:10.220 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:10.220 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:10.220 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:03:10.221 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:49.450542+0000", 2023-09-23T21:03:10.221 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999981 2023-09-23T21:03:10.221 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:10.221 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:10.221 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:03:10.221 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:49.450548+0000", 2023-09-23T21:03:10.221 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.9020000000000004e-06 2023-09-23T21:03:10.221 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:10.221 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:10.221 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:03:10.221 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:49.450558+0000", 2023-09-23T21:03:10.221 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0064999999999999e-05 2023-09-23T21:03:10.221 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:10.222 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:10.222 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:03:10.222 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:49.450569+0000", 2023-09-23T21:03:10.222 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1717e-05 2023-09-23T21:03:10.222 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:10.222 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:10.222 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:03:10.222 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:49.450630+0000", 2023-09-23T21:03:10.222 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.0591999999999999e-05 2023-09-23T21:03:10.222 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:10.222 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:10.222 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T21:03:10.223 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:49.450642+0000", 2023-09-23T21:03:10.223 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.2224e-05 2023-09-23T21:03:10.223 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:10.223 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:10.223 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:03:10.223 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:50.251937+0000", 2023-09-23T21:03:10.223 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.80129495900000003 2023-09-23T21:03:10.223 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:10.223 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:10.223 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:03:10.223 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:50.252087+0000", 2023-09-23T21:03:10.223 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00015013799999999999 2023-09-23T21:03:10.223 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:10.224 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:10.224 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 2,6", 2023-09-23T21:03:10.224 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:50.252515+0000", 2023-09-23T21:03:10.224 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00042781500000000001 2023-09-23T21:03:10.224 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:10.224 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:10.224 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T21:03:10.224 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:50.254929+0000", 2023-09-23T21:03:10.224 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0024143659999999998 2023-09-23T21:03:10.224 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:10.224 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:10.224 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:03:10.225 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:50.255262+0000", 2023-09-23T21:03:10.225 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00033292800000000002 2023-09-23T21:03:10.225 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:10.225 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:10.225 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:03:10.225 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:50.261655+0000", 2023-09-23T21:03:10.225 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0063927589999999996 2023-09-23T21:03:10.225 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:10.225 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:10.225 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T21:03:10.225 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:50.261703+0000", 2023-09-23T21:03:10.225 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.8362000000000002e-05 2023-09-23T21:03:10.226 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:10.226 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:10.226 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T21:03:10.226 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:50.261743+0000", 2023-09-23T21:03:10.226 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.9758000000000001e-05 2023-09-23T21:03:10.226 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:03:10.226 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:03:10.226 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:03:10.226 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:03:10.226 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:03:10.226 INFO:teuthology.orchestra.run.smithi026.stdout:} 2023-09-23T21:03:10.245 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T21:03:10.292 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:03:10.523 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:03:10.534 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:03:10.535 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:03:10.539 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:03:10 smithi026 ceph-mon[175151]: pgmap v737: 109 pgs: 109 active+clean; 6.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 109 B/s rd, 28 MiB/s wr, 804 op/s 2023-09-23T21:03:10.539 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:03:10 smithi026 ceph-mon[175151]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T21:03:10.539 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:03:10 smithi026 ceph-mon[175151]: Cluster is now healthy 2023-09-23T21:03:10.539 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:03:10 smithi026 ceph-mon[175151]: failed to encode map e278 with expected crc 2023-09-23T21:03:10.539 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:03:10 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/3430811949' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T21:03:10.539 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:03:10 smithi026 ceph-mon[175151]: failed to encode map e278 with expected crc 2023-09-23T21:03:10.540 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:03:10 smithi026 ceph-mon[175151]: osdmap e278: 8 total, 8 up, 8 in 2023-09-23T21:03:10.540 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:03:10 smithi026 ceph-mon[179688]: pgmap v737: 109 pgs: 109 active+clean; 6.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 109 B/s rd, 28 MiB/s wr, 804 op/s 2023-09-23T21:03:10.540 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:03:10 smithi026 ceph-mon[179688]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T21:03:10.540 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:03:10 smithi026 ceph-mon[179688]: Cluster is now healthy 2023-09-23T21:03:10.540 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:03:10 smithi026 ceph-mon[179688]: failed to encode map e278 with expected crc 2023-09-23T21:03:10.540 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:03:10 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/3430811949' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T21:03:10.540 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:03:10 smithi026 ceph-mon[179688]: failed to encode map e278 with expected crc 2023-09-23T21:03:10.540 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:03:10 smithi026 ceph-mon[179688]: osdmap e278: 8 total, 8 up, 8 in 2023-09-23T21:03:10.618 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_ops_in_flight 2023-09-23T21:03:10.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:03:10 smithi070 ceph-mon[168282]: pgmap v737: 109 pgs: 109 active+clean; 6.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 109 B/s rd, 28 MiB/s wr, 804 op/s 2023-09-23T21:03:10.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:03:10 smithi070 ceph-mon[168282]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T21:03:10.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:03:10 smithi070 ceph-mon[168282]: Cluster is now healthy 2023-09-23T21:03:10.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:03:10 smithi070 ceph-mon[168282]: failed to encode map e278 with expected crc 2023-09-23T21:03:10.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:03:10 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/3430811949' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T21:03:10.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:03:10 smithi070 ceph-mon[168282]: failed to encode map e278 with expected crc 2023-09-23T21:03:10.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:03:10 smithi070 ceph-mon[168282]: osdmap e278: 8 total, 8 up, 8 in 2023-09-23T21:03:10.636 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:03:10.746 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:03:10.842 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:03:10.848 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:03:10.849 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:03:10.950 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:03:11.038 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:03:11.104 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:03:11.118 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:03:11.119 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:03:11.203 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:03:11.219 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:03:11.260 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:03:11.318 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:03:11.323 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:03:11.324 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:03:11.425 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:03:11.504 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:03:11.591 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:03:11.600 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:03:11.601 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:03:11.701 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:03:11.798 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:03:11.799 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:03:11.857 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:03:11.863 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:03:11.864 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:03:11.942 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 6, 7, 4, 5] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 7, 6] 2023-09-23T21:03:11.942 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T21:03:11.943 INFO:tasks.thrashosds.thrasher:Growing pool unique_pool_2 2023-09-23T21:03:11.943 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T21:03:11.965 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:03:11.969 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T21:03:12.011 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:03:12.070 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:03:12.081 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:03:12.081 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:03:12.182 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:03:12.287 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:03:12 smithi026 ceph-mon[175151]: pgmap v739: 109 pgs: 109 active+clean; 6.4 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 7.9 MiB/s wr, 803 op/s 2023-09-23T21:03:12.324 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:03:12.418 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:03:12.426 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:03:12.426 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:03:12.527 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:03:12.559 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:03:12 smithi070 ceph-mon[168282]: pgmap v739: 109 pgs: 109 active+clean; 6.4 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 7.9 MiB/s wr, 803 op/s 2023-09-23T21:03:12.561 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:03:12.605 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:03:12.610 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:03:12.610 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:03:12.634 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:03:12 smithi026 ceph-mon[179688]: pgmap v739: 109 pgs: 109 active+clean; 6.4 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 7.9 MiB/s wr, 803 op/s 2023-09-23T21:03:12.710 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:03:12.741 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:03:12.791 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:03:12.798 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:03:12.798 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:03:12.899 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:03:12.943 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:03:12.987 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:03:12.989 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:03:12.990 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:03:13.091 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:03:13.131 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:03:13.184 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:03:13.188 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:03:13.188 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:03:13.223 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:03:13.289 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:03:13.361 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0: {} 2023-09-23T21:03:13.401 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:03:13.404 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1: {} 2023-09-23T21:03:13.415 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:03:13.442 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2: {} 2023-09-23T21:03:13.494 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3: {} 2023-09-23T21:03:13.535 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4: {} 2023-09-23T21:03:13.573 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T21:03:13.606 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6: {} 2023-09-23T21:03:13.644 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T21:03:13.647 INFO:teuthology.orchestra.run.smithi026.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T21:03:13.647 INFO:teuthology.orchestra.run.smithi026.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T21:03:13.647 INFO:teuthology.orchestra.run.smithi026.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T21:03:13.647 INFO:teuthology.orchestra.run.smithi026.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T21:03:13.647 INFO:teuthology.orchestra.run.smithi026.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T21:03:13.647 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T21:03:13.648 INFO:teuthology.orchestra.run.smithi026.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T21:03:13.648 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T21:03:13.758 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:03:13.763 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:03:13.764 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:03:13.864 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:03:13.939 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:03:14.038 INFO:teuthology.orchestra.run.smithi026.stdout:ERROR: (22) Invalid argument 2023-09-23T21:03:14.038 INFO:teuthology.orchestra.run.smithi026.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T21:03:14.182 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:03:14.193 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:03:14.194 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:03:14.295 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:03:14.372 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:03:14.544 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:03:14 smithi026 ceph-mon[179688]: pgmap v740: 109 pgs: 109 active+clean; 6.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 2.2 MiB/s wr, 828 op/s 2023-09-23T21:03:14.545 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:03:14 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/3999559690' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:03:14.545 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:03:14 smithi026 ceph-mon[175151]: pgmap v740: 109 pgs: 109 active+clean; 6.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 2.2 MiB/s wr, 828 op/s 2023-09-23T21:03:14.545 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:03:14 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/3999559690' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:03:14.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:03:14 smithi070 ceph-mon[168282]: pgmap v740: 109 pgs: 109 active+clean; 6.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 2.2 MiB/s wr, 828 op/s 2023-09-23T21:03:14.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:03:14 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/3999559690' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:03:14.858 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T21:03:14.865 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:03:14.865 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:03:14.966 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:03:15.014 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:03:15.059 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:03:15.061 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:03:15.062 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:03:15.162 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:03:15.231 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:03:15.364 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_blocked_ops 2023-09-23T21:03:15.519 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T21:03:15.529 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:03:15.530 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:03:15.631 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:03:15.663 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:03:15.712 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:03:15.718 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:03:15.718 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:03:15.819 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:03:15.866 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:03:15.941 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:03:15.958 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:03:15.958 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:03:16.059 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:03:16.147 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:03:16.371 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:03:16.388 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:03:16.389 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:03:16.427 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T21:03:16.490 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:03:16.617 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:03:16 smithi026 ceph-mon[179688]: pgmap v741: 109 pgs: 109 active+clean; 6.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 784 op/s 2023-09-23T21:03:16.617 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:03:16 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:03:16.617 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:03:16 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:03:16.617 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:03:16 smithi026 ceph-mon[175151]: pgmap v741: 109 pgs: 109 active+clean; 6.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 784 op/s 2023-09-23T21:03:16.617 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:03:16 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:03:16.618 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:03:16 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:03:16.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:03:16 smithi070 ceph-mon[168282]: pgmap v741: 109 pgs: 109 active+clean; 6.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 784 op/s 2023-09-23T21:03:16.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:03:16 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:03:16.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:03:16 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:03:16.627 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:03:16.629 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:03:16.737 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:03:16.740 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:03:16.740 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:03:16.841 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:03:16.877 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:03:16.922 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:03:16.930 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:03:16.931 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:03:17.031 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:03:17.059 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:03:17.104 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:03:17.107 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:03:17.107 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:03:17.210 INFO:teuthology.orchestra.run.smithi026.stdout: 2023-09-23T21:03:17.212 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:03:17.242 INFO:teuthology.orchestra.run.smithi026.stderr:dumped all 2023-09-23T21:03:17.257 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:03:17.301 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:03:17.308 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:03:17.309 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:03:17.348 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:03:17 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/3225375976' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T21:03:17.348 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:03:17 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T21:03:17.375 INFO:teuthology.orchestra.run.smithi026.stderr:noscrub is set 2023-09-23T21:03:17.410 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:03:17.446 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:03:17.489 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:03:17.493 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:03:17.494 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:03:17.594 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:03:17.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:03:17 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/3225375976' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T21:03:17.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:03:17 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T21:03:17.639 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:03:17 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/3225375976' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T21:03:17.639 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:03:17 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T21:03:17.667 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:03:17.834 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T21:03:17.841 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:03:17.842 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:03:17.943 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:03:17.947 INFO:teuthology.orchestra.run.smithi026.stdout:{"pg_ready":true,"pg_map":{"version":742,"stamp":"2023-09-23T21:03:16.733810+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":6638213048,"num_objects":101307,"num_object_clones":0,"num_object_copies":303921,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":101307,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":120288,"num_write_kb":7088286,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":22,"num_bytes_recovered":5990,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":117163,"ondisk_log_size":117163,"up":327,"acting":327,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":330,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":22239556,"kb_used_data":19948228,"kb_used_omap":136,"kb_used_meta":2290679,"kb_avail":727721660,"statfs":{"total":767960285184,"available":745186979840,"internally_reserved":0,"allocated":20426985472,"data_stored":20418304544,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":139741,"internal_metadata":2345655843},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[5,3,1,3,7,2],"upper_bound":64},"perf_stat":{"commit_latency_ms":6,"apply_latency_ms":6,"commit_latency_ns":6000000,"apply_latency_ns":6000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-527171584,"num_objects":-8044,"num_object_clones":0,"num_object_copies":-24132,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":-8044,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8044,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.174923"},"pg_stats":[{"pgid":"3.1a","version":"68'1","reported_seq":155,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-09-23T21:03:05.261004+0000","last_change":"2023-09-23T20:54:13.946695+0000","last_active":"2023-09-23T21:03:05.261004+0000","last_peered":"2023-09-23T21:03:05.261004+0000","last_clean":"2023-09-23T21:03:05.261004+0000","last_became_active":"2023-09-23T20:54:13.945399+0000","last_became_peered":"2023-09-23T20:54:13.945399+0000","last_unstale":"2023-09-23T21:03:05.261004+0000","last_undegraded":"2023-09-23T21:03:05.261004+0000","last_fullsized":"2023-09-23T21:03:05.261004+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:32:52.365464+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":271,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-09-23T21:03:05.257926+0000","last_change":"2023-09-23T20:54:13.942019+0000","last_active":"2023-09-23T21:03:05.257926+0000","last_peered":"2023-09-23T21:03:05.257926+0000","last_clean":"2023-09-23T21:03:05.257926+0000","last_became_active":"2023-09-23T20:54:13.941647+0000","last_became_peered":"2023-09-23T20:54:13.941647+0000","last_unstale":"2023-09-23T21:03:05.257926+0000","last_undegraded":"2023-09-23T21:03:05.257926+0000","last_fullsized":"2023-09-23T21:03:05.257926+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:30:54.400944+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":249,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-09-23T21:03:05.293391+0000","last_change":"2023-09-23T20:54:13.941394+0000","last_active":"2023-09-23T21:03:05.293391+0000","last_peered":"2023-09-23T21:03:05.293391+0000","last_clean":"2023-09-23T21:03:05.293391+0000","last_became_active":"2023-09-23T20:54:13.941042+0000","last_became_peered":"2023-09-23T20:54:13.941042+0000","last_unstale":"2023-09-23T21:03:05.293391+0000","last_undegraded":"2023-09-23T21:03:05.293391+0000","last_fullsized":"2023-09-23T21:03:05.293391+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:19:41.401184+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":264,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-09-23T21:03:05.266296+0000","last_change":"2023-09-23T20:57:44.979811+0000","last_active":"2023-09-23T21:03:05.266296+0000","last_peered":"2023-09-23T21:03:05.266296+0000","last_clean":"2023-09-23T21:03:05.266296+0000","last_became_active":"2023-09-23T20:57:44.979515+0000","last_became_peered":"2023-09-23T20:57:44.979515+0000","last_unstale":"2023-09-23T21:03:05.266296+0000","last_undegraded":"2023-09-23T21:03:05.266296+0000","last_fullsized":"2023-09-23T21:03:05.266296+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:38:22.162140+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":263,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-09-23T21:03:09.298426+0000","last_change":"2023-09-23T20:51:10.833613+0000","last_active":"2023-09-23T21:03:09.298426+0000","last_peered":"2023-09-23T21:03:09.298426+0000","last_clean":"2023-09-23T21:03:09.298426+0000","last_became_active":"2023-09-23T20:51:10.833208+0000","last_became_peered":"2023-09-23T20:51:10.833208+0000","last_unstale":"2023-09-23T21:03:09.298426+0000","last_undegraded":"2023-09-23T21:03:09.298426+0000","last_fullsized":"2023-09-23T21:03:09.298426+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:51:28.246014+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":243,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-09-23T21:03:05.260369+0000","last_change":"2023-09-23T20:55:35.958384+0000","last_active":"2023-09-23T21:03:05.260369+0000","last_peered":"2023-09-23T21:03:05.260369+0000","last_clean":"2023-09-23T21:03:05.260369+0000","last_became_active":"2023-09-23T20:55:35.954563+0000","last_became_peered":"2023-09-23T20:55:35.954563+0000","last_unstale":"2023-09-23T21:03:05.260369+0000","last_undegraded":"2023-09-23T21:03:05.260369+0000","last_fullsized":"2023-09-23T21:03:05.260369+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:37:48.332259+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":254,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-09-23T21:03:05.266130+0000","last_change":"2023-09-23T20:54:31.885514+0000","last_active":"2023-09-23T21:03:05.266130+0000","last_peered":"2023-09-23T21:03:05.266130+0000","last_clean":"2023-09-23T21:03:05.266130+0000","last_became_active":"2023-09-23T20:54:31.884986+0000","last_became_peered":"2023-09-23T20:54:31.884986+0000","last_unstale":"2023-09-23T21:03:05.266130+0000","last_undegraded":"2023-09-23T21:03:05.266130+0000","last_fullsized":"2023-09-23T21:03:05.266130+0000","mapping_epoch":136,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:26:08.100220+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":260,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-09-23T21:03:05.257278+0000","last_change":"2023-09-23T20:55:35.951889+0000","last_active":"2023-09-23T21:03:05.257278+0000","last_peered":"2023-09-23T21:03:05.257278+0000","last_clean":"2023-09-23T21:03:05.257278+0000","last_became_active":"2023-09-23T20:55:35.951563+0000","last_became_peered":"2023-09-23T20:55:35.951563+0000","last_unstale":"2023-09-23T21:03:05.257278+0000","last_undegraded":"2023-09-23T21:03:05.257278+0000","last_fullsized":"2023-09-23T21:03:05.257278+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:41:53.098781+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"79'161","reported_seq":566,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-09-23T21:03:05.264585+0000","last_change":"2023-09-23T20:57:25.096559+0000","last_active":"2023-09-23T21:03:05.264585+0000","last_peered":"2023-09-23T21:03:05.264585+0000","last_clean":"2023-09-23T21:03:05.264585+0000","last_became_active":"2023-09-23T20:57:15.975674+0000","last_became_peered":"2023-09-23T20:57:15.975674+0000","last_unstale":"2023-09-23T21:03:05.264585+0000","last_undegraded":"2023-09-23T21:03:05.264585+0000","last_fullsized":"2023-09-23T21:03:05.264585+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:25.096401+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:25.096401+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:05:05.448404+0000","scrub_duration":0.002859393,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":151,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-09-23T21:03:05.276274+0000","last_change":"2023-09-23T20:54:31.890678+0000","last_active":"2023-09-23T21:03:05.276274+0000","last_peered":"2023-09-23T21:03:05.276274+0000","last_clean":"2023-09-23T21:03:05.276274+0000","last_became_active":"2023-09-23T20:54:31.890469+0000","last_became_peered":"2023-09-23T20:54:31.890469+0000","last_unstale":"2023-09-23T21:03:05.276274+0000","last_undegraded":"2023-09-23T21:03:05.276274+0000","last_fullsized":"2023-09-23T21:03:05.276274+0000","mapping_epoch":136,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:55:08.790146+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":221,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-09-23T21:03:05.257449+0000","last_change":"2023-09-23T20:55:35.952661+0000","last_active":"2023-09-23T21:03:05.257449+0000","last_peered":"2023-09-23T21:03:05.257449+0000","last_clean":"2023-09-23T21:03:05.257449+0000","last_became_active":"2023-09-23T20:55:35.952349+0000","last_became_peered":"2023-09-23T20:55:35.952349+0000","last_unstale":"2023-09-23T21:03:05.257449+0000","last_undegraded":"2023-09-23T21:03:05.257449+0000","last_fullsized":"2023-09-23T21:03:05.257449+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:00:03.110126+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":97,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-09-23T21:03:05.258003+0000","last_change":"2023-09-23T20:56:45.988607+0000","last_active":"2023-09-23T21:03:05.258003+0000","last_peered":"2023-09-23T21:03:05.258003+0000","last_clean":"2023-09-23T21:03:05.258003+0000","last_became_active":"2023-09-23T20:56:45.988206+0000","last_became_peered":"2023-09-23T20:56:45.988206+0000","last_unstale":"2023-09-23T21:03:05.258003+0000","last_undegraded":"2023-09-23T21:03:05.258003+0000","last_fullsized":"2023-09-23T21:03:05.258003+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":205,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:01:19.783688+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,4],"acting":[3,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":95,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-09-23T21:03:05.276644+0000","last_change":"2023-09-23T20:56:18.139142+0000","last_active":"2023-09-23T21:03:05.276644+0000","last_peered":"2023-09-23T21:03:05.276644+0000","last_clean":"2023-09-23T21:03:05.276644+0000","last_became_active":"2023-09-23T20:56:18.138844+0000","last_became_peered":"2023-09-23T20:56:18.138844+0000","last_unstale":"2023-09-23T21:03:05.276644+0000","last_undegraded":"2023-09-23T21:03:05.276644+0000","last_fullsized":"2023-09-23T21:03:05.276644+0000","mapping_epoch":192,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":193,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:09:02.217683+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,2],"acting":[5,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.14","version":"79'22","reported_seq":182,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-09-23T21:03:05.260943+0000","last_change":"2023-09-23T20:55:35.949418+0000","last_active":"2023-09-23T21:03:05.260943+0000","last_peered":"2023-09-23T21:03:05.260943+0000","last_clean":"2023-09-23T21:03:05.260943+0000","last_became_active":"2023-09-23T20:55:35.948566+0000","last_became_peered":"2023-09-23T20:55:35.948566+0000","last_unstale":"2023-09-23T21:03:05.260943+0000","last_undegraded":"2023-09-23T21:03:05.260943+0000","last_fullsized":"2023-09-23T21:03:05.260943+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:59:37.433128+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4734,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":172,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-09-23T21:03:05.261481+0000","last_change":"2023-09-23T20:55:35.949178+0000","last_active":"2023-09-23T21:03:05.261481+0000","last_peered":"2023-09-23T21:03:05.261481+0000","last_clean":"2023-09-23T21:03:05.261481+0000","last_became_active":"2023-09-23T20:55:35.948795+0000","last_became_peered":"2023-09-23T20:55:35.948795+0000","last_unstale":"2023-09-23T21:03:05.261481+0000","last_undegraded":"2023-09-23T21:03:05.261481+0000","last_fullsized":"2023-09-23T21:03:05.261481+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:13:08.996932+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.11","version":"79'161","reported_seq":508,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-09-23T21:03:05.265563+0000","last_change":"2023-09-23T20:57:28.756796+0000","last_active":"2023-09-23T21:03:05.265563+0000","last_peered":"2023-09-23T21:03:05.265563+0000","last_clean":"2023-09-23T21:03:05.265563+0000","last_became_active":"2023-09-23T20:57:15.980606+0000","last_became_peered":"2023-09-23T20:57:15.980606+0000","last_unstale":"2023-09-23T21:03:05.265563+0000","last_undegraded":"2023-09-23T21:03:05.265563+0000","last_fullsized":"2023-09-23T21:03:05.265563+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:28.756634+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:28.756634+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:09:19.150791+0000","scrub_duration":0.0064925119999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":198,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-09-23T21:03:05.261351+0000","last_change":"2023-09-23T20:56:45.992577+0000","last_active":"2023-09-23T21:03:05.261351+0000","last_peered":"2023-09-23T21:03:05.261351+0000","last_clean":"2023-09-23T21:03:05.261351+0000","last_became_active":"2023-09-23T20:56:45.992297+0000","last_became_peered":"2023-09-23T20:56:45.992297+0000","last_unstale":"2023-09-23T21:03:05.261351+0000","last_undegraded":"2023-09-23T21:03:05.261351+0000","last_fullsized":"2023-09-23T21:03:05.261351+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":205,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:00:45.547202+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,2],"acting":[4,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":245,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-09-23T21:03:05.259157+0000","last_change":"2023-09-23T20:51:10.831262+0000","last_active":"2023-09-23T21:03:05.259157+0000","last_peered":"2023-09-23T21:03:05.259157+0000","last_clean":"2023-09-23T21:03:05.259157+0000","last_became_active":"2023-09-23T20:51:10.830931+0000","last_became_peered":"2023-09-23T20:51:10.830931+0000","last_unstale":"2023-09-23T21:03:05.259157+0000","last_undegraded":"2023-09-23T21:03:05.259157+0000","last_fullsized":"2023-09-23T21:03:05.259157+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:51:54.982505+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.10","version":"79'161","reported_seq":515,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-09-23T21:03:05.275606+0000","last_change":"2023-09-23T20:57:20.708003+0000","last_active":"2023-09-23T21:03:05.275606+0000","last_peered":"2023-09-23T21:03:05.275606+0000","last_clean":"2023-09-23T21:03:05.275606+0000","last_became_active":"2023-09-23T20:57:15.975302+0000","last_became_peered":"2023-09-23T20:57:15.975302+0000","last_unstale":"2023-09-23T21:03:05.275606+0000","last_undegraded":"2023-09-23T21:03:05.275606+0000","last_fullsized":"2023-09-23T21:03:05.275606+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:20.707939+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:20.707939+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:16:51.721994+0000","scrub_duration":0.0017733409999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":254,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-09-23T21:03:05.266042+0000","last_change":"2023-09-23T20:57:44.979736+0000","last_active":"2023-09-23T21:03:05.266042+0000","last_peered":"2023-09-23T21:03:05.266042+0000","last_clean":"2023-09-23T21:03:05.266042+0000","last_became_active":"2023-09-23T20:57:44.979356+0000","last_became_peered":"2023-09-23T20:57:44.979356+0000","last_unstale":"2023-09-23T21:03:05.266042+0000","last_undegraded":"2023-09-23T21:03:05.266042+0000","last_fullsized":"2023-09-23T21:03:05.266042+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:14:43.848832+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":151,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-09-23T21:03:05.276120+0000","last_change":"2023-09-23T20:54:31.887814+0000","last_active":"2023-09-23T21:03:05.276120+0000","last_peered":"2023-09-23T21:03:05.276120+0000","last_clean":"2023-09-23T21:03:05.276120+0000","last_became_active":"2023-09-23T20:54:31.887195+0000","last_became_peered":"2023-09-23T20:54:31.887195+0000","last_unstale":"2023-09-23T21:03:05.276120+0000","last_undegraded":"2023-09-23T21:03:05.276120+0000","last_fullsized":"2023-09-23T21:03:05.276120+0000","mapping_epoch":136,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:10:52.089553+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.13","version":"79'161","reported_seq":579,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-09-23T21:03:05.297388+0000","last_change":"2023-09-23T20:57:48.262970+0000","last_active":"2023-09-23T21:03:05.297388+0000","last_peered":"2023-09-23T21:03:05.297388+0000","last_clean":"2023-09-23T21:03:05.297388+0000","last_became_active":"2023-09-23T20:57:48.262663+0000","last_became_peered":"2023-09-23T20:57:48.262663+0000","last_unstale":"2023-09-23T21:03:05.297388+0000","last_undegraded":"2023-09-23T21:03:05.297388+0000","last_fullsized":"2023-09-23T21:03:05.297388+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:36.420362+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:36.420362+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:28:28.613492+0000","scrub_duration":0.028838444000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":222,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-09-23T21:03:05.261822+0000","last_change":"2023-09-23T20:56:45.991822+0000","last_active":"2023-09-23T21:03:05.261822+0000","last_peered":"2023-09-23T21:03:05.261822+0000","last_clean":"2023-09-23T21:03:05.261822+0000","last_became_active":"2023-09-23T20:56:45.991460+0000","last_became_peered":"2023-09-23T20:56:45.991460+0000","last_unstale":"2023-09-23T21:03:05.261822+0000","last_undegraded":"2023-09-23T21:03:05.261822+0000","last_fullsized":"2023-09-23T21:03:05.261822+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":205,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:20:37.503317+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":236,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-09-23T21:03:05.266439+0000","last_change":"2023-09-23T20:51:10.840849+0000","last_active":"2023-09-23T21:03:05.266439+0000","last_peered":"2023-09-23T21:03:05.266439+0000","last_clean":"2023-09-23T21:03:05.266439+0000","last_became_active":"2023-09-23T20:51:10.840593+0000","last_became_peered":"2023-09-23T20:51:10.840593+0000","last_unstale":"2023-09-23T21:03:05.266439+0000","last_undegraded":"2023-09-23T21:03:05.266439+0000","last_fullsized":"2023-09-23T21:03:05.266439+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:30:52.297625+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.12","version":"79'161","reported_seq":515,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-09-23T21:03:05.276438+0000","last_change":"2023-09-23T20:57:21.750765+0000","last_active":"2023-09-23T21:03:05.276438+0000","last_peered":"2023-09-23T21:03:05.276438+0000","last_clean":"2023-09-23T21:03:05.276438+0000","last_became_active":"2023-09-23T20:57:15.979195+0000","last_became_peered":"2023-09-23T20:57:15.979195+0000","last_unstale":"2023-09-23T21:03:05.276438+0000","last_undegraded":"2023-09-23T21:03:05.276438+0000","last_fullsized":"2023-09-23T21:03:05.276438+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:21.750680+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:21.750680+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:03:41.915962+0000","scrub_duration":0.0092391680000000007,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":270,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-09-23T21:03:05.259690+0000","last_change":"2023-09-23T20:55:36.943969+0000","last_active":"2023-09-23T21:03:05.259690+0000","last_peered":"2023-09-23T21:03:05.259690+0000","last_clean":"2023-09-23T21:03:05.259690+0000","last_became_active":"2023-09-23T20:55:36.943580+0000","last_became_peered":"2023-09-23T20:55:36.943580+0000","last_unstale":"2023-09-23T21:03:05.259690+0000","last_undegraded":"2023-09-23T21:03:05.259690+0000","last_fullsized":"2023-09-23T21:03:05.259690+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:49:06.969396+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":68,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-09-23T21:03:05.258667+0000","last_change":"2023-09-23T20:57:44.980498+0000","last_active":"2023-09-23T21:03:05.258667+0000","last_peered":"2023-09-23T21:03:05.258667+0000","last_clean":"2023-09-23T21:03:05.258667+0000","last_became_active":"2023-09-23T20:57:44.980182+0000","last_became_peered":"2023-09-23T20:57:44.980182+0000","last_unstale":"2023-09-23T21:03:05.258667+0000","last_undegraded":"2023-09-23T21:03:05.258667+0000","last_fullsized":"2023-09-23T21:03:05.258667+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:34:51.620544+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.4","version":"278'7432","reported_seq":7464,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-09-23T21:03:12.085070+0000","last_change":"2023-09-23T21:01:25.429865+0000","last_active":"2023-09-23T21:03:12.085070+0000","last_peered":"2023-09-23T21:03:12.085070+0000","last_clean":"2023-09-23T21:03:12.085070+0000","last_became_active":"2023-09-23T21:01:25.429555+0000","last_became_peered":"2023-09-23T21:01:25.429555+0000","last_unstale":"2023-09-23T21:03:12.085070+0000","last_undegraded":"2023-09-23T21:03:12.085070+0000","last_fullsized":"2023-09-23T21:03:12.085070+0000","mapping_epoch":254,"log_start":"0'0","ondisk_log_start":"0'0","created":254,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:01:24.325318+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:01:24.325318+0000","last_clean_scrub_stamp":"2023-09-23T21:01:24.325318+0000","objects_scrubbed":0,"log_size":7432,"log_dups_size":0,"ondisk_log_size":7432,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:28:38.557662+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":420347904,"num_objects":6414,"num_object_clones":0,"num_object_copies":19242,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6414,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7432,"num_write_kb":443072,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,2],"acting":[5,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.d","version":"79'161","reported_seq":560,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-09-23T21:03:05.261190+0000","last_change":"2023-09-23T20:57:31.075213+0000","last_active":"2023-09-23T21:03:05.261190+0000","last_peered":"2023-09-23T21:03:05.261190+0000","last_clean":"2023-09-23T21:03:05.261190+0000","last_became_active":"2023-09-23T20:57:15.988188+0000","last_became_peered":"2023-09-23T20:57:15.988188+0000","last_unstale":"2023-09-23T21:03:05.261190+0000","last_undegraded":"2023-09-23T21:03:05.261190+0000","last_fullsized":"2023-09-23T21:03:05.261190+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:31.075098+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:31.075098+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:41:52.142362+0000","scrub_duration":0.010304610000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":209,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-09-23T21:03:05.262255+0000","last_change":"2023-09-23T20:56:45.991841+0000","last_active":"2023-09-23T21:03:05.262255+0000","last_peered":"2023-09-23T21:03:05.262255+0000","last_clean":"2023-09-23T21:03:05.262255+0000","last_became_active":"2023-09-23T20:56:45.991491+0000","last_became_peered":"2023-09-23T20:56:45.991491+0000","last_unstale":"2023-09-23T21:03:05.262255+0000","last_undegraded":"2023-09-23T21:03:05.262255+0000","last_fullsized":"2023-09-23T21:03:05.262255+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":205,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:30:37.405319+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":168,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-09-23T21:03:05.275860+0000","last_change":"2023-09-23T20:55:35.957768+0000","last_active":"2023-09-23T21:03:05.275860+0000","last_peered":"2023-09-23T21:03:05.275860+0000","last_clean":"2023-09-23T21:03:05.275860+0000","last_became_active":"2023-09-23T20:55:35.957421+0000","last_became_peered":"2023-09-23T20:55:35.957421+0000","last_unstale":"2023-09-23T21:03:05.275860+0000","last_undegraded":"2023-09-23T21:03:05.275860+0000","last_fullsized":"2023-09-23T21:03:05.275860+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:49:41.585961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"8.5","version":"278'7581","reported_seq":7613,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-09-23T21:03:14.124446+0000","last_change":"2023-09-23T21:01:25.560708+0000","last_active":"2023-09-23T21:03:14.124446+0000","last_peered":"2023-09-23T21:03:14.124446+0000","last_clean":"2023-09-23T21:03:14.124446+0000","last_became_active":"2023-09-23T21:01:25.560472+0000","last_became_peered":"2023-09-23T21:01:25.560472+0000","last_unstale":"2023-09-23T21:03:14.124446+0000","last_undegraded":"2023-09-23T21:03:14.124446+0000","last_fullsized":"2023-09-23T21:03:14.124446+0000","mapping_epoch":254,"log_start":"258'200","ondisk_log_start":"258'200","created":254,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:01:24.325318+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:01:24.325318+0000","last_clean_scrub_stamp":"2023-09-23T21:01:24.325318+0000","objects_scrubbed":0,"log_size":7381,"log_dups_size":0,"ondisk_log_size":7381,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:04:02.549068+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":418185216,"num_objects":6381,"num_object_clones":0,"num_object_copies":19143,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6381,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7581,"num_write_kb":446784,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.c","version":"79'161","reported_seq":580,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-09-23T21:03:05.259013+0000","last_change":"2023-09-23T20:57:22.041071+0000","last_active":"2023-09-23T21:03:05.259013+0000","last_peered":"2023-09-23T21:03:05.259013+0000","last_clean":"2023-09-23T21:03:05.259013+0000","last_became_active":"2023-09-23T20:57:15.973204+0000","last_became_peered":"2023-09-23T20:57:15.973204+0000","last_unstale":"2023-09-23T21:03:05.259013+0000","last_undegraded":"2023-09-23T21:03:05.259013+0000","last_fullsized":"2023-09-23T21:03:05.259013+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:22.040907+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:22.040907+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:47:02.019978+0000","scrub_duration":0.013021176000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":196,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-09-23T21:03:05.261556+0000","last_change":"2023-09-23T20:56:45.991821+0000","last_active":"2023-09-23T21:03:05.261556+0000","last_peered":"2023-09-23T21:03:05.261556+0000","last_clean":"2023-09-23T21:03:05.261556+0000","last_became_active":"2023-09-23T20:56:45.991447+0000","last_became_peered":"2023-09-23T20:56:45.991447+0000","last_unstale":"2023-09-23T21:03:05.261556+0000","last_undegraded":"2023-09-23T21:03:05.261556+0000","last_fullsized":"2023-09-23T21:03:05.261556+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":205,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:14:25.525222+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":199,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-09-23T21:03:05.262072+0000","last_change":"2023-09-23T20:57:44.982059+0000","last_active":"2023-09-23T21:03:05.262072+0000","last_peered":"2023-09-23T21:03:05.262072+0000","last_clean":"2023-09-23T21:03:05.262072+0000","last_became_active":"2023-09-23T20:57:44.981742+0000","last_became_peered":"2023-09-23T20:57:44.981742+0000","last_unstale":"2023-09-23T21:03:05.262072+0000","last_undegraded":"2023-09-23T21:03:05.262072+0000","last_fullsized":"2023-09-23T21:03:05.262072+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:41:24.042916+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.6","version":"278'7654","reported_seq":7687,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-09-23T21:03:12.040213+0000","last_change":"2023-09-23T21:01:25.560370+0000","last_active":"2023-09-23T21:03:12.040213+0000","last_peered":"2023-09-23T21:03:12.040213+0000","last_clean":"2023-09-23T21:03:12.040213+0000","last_became_active":"2023-09-23T21:01:25.560018+0000","last_became_peered":"2023-09-23T21:01:25.560018+0000","last_unstale":"2023-09-23T21:03:12.040213+0000","last_undegraded":"2023-09-23T21:03:12.040213+0000","last_fullsized":"2023-09-23T21:03:12.040213+0000","mapping_epoch":254,"log_start":"0'0","ondisk_log_start":"0'0","created":254,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:01:24.325318+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:01:24.325318+0000","last_clean_scrub_stamp":"2023-09-23T21:01:24.325318+0000","objects_scrubbed":0,"log_size":7654,"log_dups_size":0,"ondisk_log_size":7654,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:45:39.782872+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":429654016,"num_objects":6556,"num_object_clones":0,"num_object_copies":19668,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6556,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7654,"num_write_kb":454720,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,7],"acting":[0,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.f","version":"79'161","reported_seq":582,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-09-23T21:03:05.260291+0000","last_change":"2023-09-23T20:57:27.880841+0000","last_active":"2023-09-23T21:03:05.260291+0000","last_peered":"2023-09-23T21:03:05.260291+0000","last_clean":"2023-09-23T21:03:05.260291+0000","last_became_active":"2023-09-23T20:57:15.970925+0000","last_became_peered":"2023-09-23T20:57:15.970925+0000","last_unstale":"2023-09-23T21:03:05.260291+0000","last_undegraded":"2023-09-23T21:03:05.260291+0000","last_fullsized":"2023-09-23T21:03:05.260291+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:27.880776+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:27.880776+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:15:13.438014+0000","scrub_duration":0.0014004899999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":182,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-09-23T21:03:05.276137+0000","last_change":"2023-09-23T20:56:45.989153+0000","last_active":"2023-09-23T21:03:05.276137+0000","last_peered":"2023-09-23T21:03:05.276137+0000","last_clean":"2023-09-23T21:03:05.276137+0000","last_became_active":"2023-09-23T20:56:45.988800+0000","last_became_peered":"2023-09-23T20:56:45.988800+0000","last_unstale":"2023-09-23T21:03:05.276137+0000","last_undegraded":"2023-09-23T21:03:05.276137+0000","last_fullsized":"2023-09-23T21:03:05.276137+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":205,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:04:41.130194+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":217,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-09-23T21:03:05.259751+0000","last_change":"2023-09-23T20:55:35.965427+0000","last_active":"2023-09-23T21:03:05.259751+0000","last_peered":"2023-09-23T21:03:05.259751+0000","last_clean":"2023-09-23T21:03:05.259751+0000","last_became_active":"2023-09-23T20:55:35.963969+0000","last_became_peered":"2023-09-23T20:55:35.963969+0000","last_unstale":"2023-09-23T21:03:05.259751+0000","last_undegraded":"2023-09-23T21:03:05.259751+0000","last_fullsized":"2023-09-23T21:03:05.259751+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:38:40.612083+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.7","version":"278'7490","reported_seq":7521,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-09-23T21:03:14.465955+0000","last_change":"2023-09-23T21:01:25.341425+0000","last_active":"2023-09-23T21:03:14.465955+0000","last_peered":"2023-09-23T21:03:14.465955+0000","last_clean":"2023-09-23T21:03:14.465955+0000","last_became_active":"2023-09-23T21:01:25.341114+0000","last_became_peered":"2023-09-23T21:01:25.341114+0000","last_unstale":"2023-09-23T21:03:14.465955+0000","last_undegraded":"2023-09-23T21:03:14.465955+0000","last_fullsized":"2023-09-23T21:03:14.465955+0000","mapping_epoch":254,"log_start":"258'600","ondisk_log_start":"258'600","created":254,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:01:24.325318+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:01:24.325318+0000","last_clean_scrub_stamp":"2023-09-23T21:01:24.325318+0000","objects_scrubbed":0,"log_size":6890,"log_dups_size":0,"ondisk_log_size":6890,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:15:08.338532+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":404488192,"num_objects":6172,"num_object_clones":0,"num_object_copies":18516,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6172,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7490,"num_write_kb":437184,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.e","version":"79'161","reported_seq":489,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-09-23T21:03:05.261454+0000","last_change":"2023-09-23T20:57:34.132738+0000","last_active":"2023-09-23T21:03:05.261454+0000","last_peered":"2023-09-23T21:03:05.261454+0000","last_clean":"2023-09-23T21:03:05.261454+0000","last_became_active":"2023-09-23T20:57:15.974318+0000","last_became_peered":"2023-09-23T20:57:15.974318+0000","last_unstale":"2023-09-23T21:03:05.261454+0000","last_undegraded":"2023-09-23T21:03:05.261454+0000","last_fullsized":"2023-09-23T21:03:05.261454+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:34.132660+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:34.132660+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:07:07.739438+0000","scrub_duration":0.005229694,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":168,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-09-23T21:03:05.276719+0000","last_change":"2023-09-23T20:55:35.962005+0000","last_active":"2023-09-23T21:03:05.276719+0000","last_peered":"2023-09-23T21:03:05.276719+0000","last_clean":"2023-09-23T21:03:05.276719+0000","last_became_active":"2023-09-23T20:55:35.961695+0000","last_became_peered":"2023-09-23T20:55:35.961695+0000","last_unstale":"2023-09-23T21:03:05.276719+0000","last_undegraded":"2023-09-23T21:03:05.276719+0000","last_fullsized":"2023-09-23T21:03:05.276719+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:28:22.876201+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":268,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-09-23T21:03:05.265599+0000","last_change":"2023-09-23T20:54:13.928715+0000","last_active":"2023-09-23T21:03:05.265599+0000","last_peered":"2023-09-23T21:03:05.265599+0000","last_clean":"2023-09-23T21:03:05.265599+0000","last_became_active":"2023-09-23T20:54:13.928200+0000","last_became_peered":"2023-09-23T20:54:13.928200+0000","last_unstale":"2023-09-23T21:03:05.265599+0000","last_undegraded":"2023-09-23T21:03:05.265599+0000","last_fullsized":"2023-09-23T21:03:05.265599+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:41:16.927513+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.0","version":"278'7551","reported_seq":7584,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-09-23T21:03:14.876527+0000","last_change":"2023-09-23T21:01:25.342305+0000","last_active":"2023-09-23T21:03:14.876527+0000","last_peered":"2023-09-23T21:03:14.876527+0000","last_clean":"2023-09-23T21:03:14.876527+0000","last_became_active":"2023-09-23T21:01:25.342042+0000","last_became_peered":"2023-09-23T21:01:25.342042+0000","last_unstale":"2023-09-23T21:03:14.876527+0000","last_undegraded":"2023-09-23T21:03:14.876527+0000","last_fullsized":"2023-09-23T21:03:14.876527+0000","mapping_epoch":254,"log_start":"258'1000","ondisk_log_start":"258'1000","created":254,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:01:24.325318+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:01:24.325318+0000","last_clean_scrub_stamp":"2023-09-23T21:01:24.325318+0000","objects_scrubbed":0,"log_size":6551,"log_dups_size":0,"ondisk_log_size":6551,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:06:29.755463+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":407437312,"num_objects":6217,"num_object_clones":0,"num_object_copies":18651,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6217,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7551,"num_write_kb":440576,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,1],"acting":[3,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.9","version":"79'161","reported_seq":652,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-09-23T21:03:05.257379+0000","last_change":"2023-09-23T20:56:46.921939+0000","last_active":"2023-09-23T21:03:05.257379+0000","last_peered":"2023-09-23T21:03:05.257379+0000","last_clean":"2023-09-23T21:03:05.257379+0000","last_became_active":"2023-09-23T20:56:45.989908+0000","last_became_peered":"2023-09-23T20:56:45.989908+0000","last_unstale":"2023-09-23T21:03:05.257379+0000","last_undegraded":"2023-09-23T21:03:05.257379+0000","last_fullsized":"2023-09-23T21:03:05.257379+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":205,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:56:46.921739+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:56:46.921739+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:20:42.635110+0000","scrub_duration":0.0071954109999999996,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":269,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-09-23T21:03:05.257623+0000","last_change":"2023-09-23T20:54:13.961788+0000","last_active":"2023-09-23T21:03:05.257623+0000","last_peered":"2023-09-23T21:03:05.257623+0000","last_clean":"2023-09-23T21:03:05.257623+0000","last_became_active":"2023-09-23T20:54:13.940937+0000","last_became_peered":"2023-09-23T20:54:13.940937+0000","last_unstale":"2023-09-23T21:03:05.257623+0000","last_undegraded":"2023-09-23T21:03:05.257623+0000","last_fullsized":"2023-09-23T21:03:05.257623+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:11:56.785299+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":172,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-09-23T21:03:05.261065+0000","last_change":"2023-09-23T20:55:35.949670+0000","last_active":"2023-09-23T21:03:05.261065+0000","last_peered":"2023-09-23T21:03:05.261065+0000","last_clean":"2023-09-23T21:03:05.261065+0000","last_became_active":"2023-09-23T20:55:35.949140+0000","last_became_peered":"2023-09-23T20:55:35.949140+0000","last_unstale":"2023-09-23T21:03:05.261065+0000","last_undegraded":"2023-09-23T21:03:05.261065+0000","last_fullsized":"2023-09-23T21:03:05.261065+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:55:59.340303+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.1","version":"278'7704","reported_seq":7737,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-09-23T21:03:14.469872+0000","last_change":"2023-09-23T21:01:25.429922+0000","last_active":"2023-09-23T21:03:14.469872+0000","last_peered":"2023-09-23T21:03:14.469872+0000","last_clean":"2023-09-23T21:03:14.469872+0000","last_became_active":"2023-09-23T21:01:25.429650+0000","last_became_peered":"2023-09-23T21:01:25.429650+0000","last_unstale":"2023-09-23T21:03:14.469872+0000","last_undegraded":"2023-09-23T21:03:14.469872+0000","last_fullsized":"2023-09-23T21:03:14.469872+0000","mapping_epoch":254,"log_start":"258'800","ondisk_log_start":"258'800","created":254,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:01:24.325318+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:01:24.325318+0000","last_clean_scrub_stamp":"2023-09-23T21:01:24.325318+0000","objects_scrubbed":0,"log_size":6904,"log_dups_size":0,"ondisk_log_size":6904,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:18:57.243292+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":418643968,"num_objects":6388,"num_object_clones":0,"num_object_copies":19164,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6388,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7704,"num_write_kb":450944,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,2],"acting":[1,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.8","version":"79'161","reported_seq":521,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-09-23T21:03:05.275787+0000","last_change":"2023-09-23T20:56:57.505737+0000","last_active":"2023-09-23T21:03:05.275787+0000","last_peered":"2023-09-23T21:03:05.275787+0000","last_clean":"2023-09-23T21:03:05.275787+0000","last_became_active":"2023-09-23T20:56:04.982345+0000","last_became_peered":"2023-09-23T20:56:04.982345+0000","last_unstale":"2023-09-23T21:03:05.275787+0000","last_undegraded":"2023-09-23T21:03:05.275787+0000","last_fullsized":"2023-09-23T21:03:05.275787+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:56:57.505663+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:56:57.505663+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:34:30.109724+0000","scrub_duration":0.026284108,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":240,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-09-23T21:03:05.259928+0000","last_change":"2023-09-23T20:55:35.959942+0000","last_active":"2023-09-23T21:03:05.259928+0000","last_peered":"2023-09-23T21:03:05.259928+0000","last_clean":"2023-09-23T21:03:05.259928+0000","last_became_active":"2023-09-23T20:55:35.957071+0000","last_became_peered":"2023-09-23T20:55:35.957071+0000","last_unstale":"2023-09-23T21:03:05.259928+0000","last_undegraded":"2023-09-23T21:03:05.259928+0000","last_fullsized":"2023-09-23T21:03:05.259928+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:44:30.308036+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":155,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-09-23T21:03:05.261453+0000","last_change":"2023-09-23T20:54:13.951459+0000","last_active":"2023-09-23T21:03:05.261453+0000","last_peered":"2023-09-23T21:03:05.261453+0000","last_clean":"2023-09-23T21:03:05.261453+0000","last_became_active":"2023-09-23T20:54:13.948409+0000","last_became_peered":"2023-09-23T20:54:13.948409+0000","last_unstale":"2023-09-23T21:03:05.261453+0000","last_undegraded":"2023-09-23T21:03:05.261453+0000","last_fullsized":"2023-09-23T21:03:05.261453+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:38:10.148061+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.2","version":"278'7581","reported_seq":7613,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-09-23T21:03:12.192689+0000","last_change":"2023-09-23T21:01:25.343022+0000","last_active":"2023-09-23T21:03:12.192689+0000","last_peered":"2023-09-23T21:03:12.192689+0000","last_clean":"2023-09-23T21:03:12.192689+0000","last_became_active":"2023-09-23T21:01:25.342540+0000","last_became_peered":"2023-09-23T21:01:25.342540+0000","last_unstale":"2023-09-23T21:03:12.192689+0000","last_undegraded":"2023-09-23T21:03:12.192689+0000","last_fullsized":"2023-09-23T21:03:12.192689+0000","mapping_epoch":254,"log_start":"258'900","ondisk_log_start":"258'900","created":254,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:01:24.325318+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:01:24.325318+0000","last_clean_scrub_stamp":"2023-09-23T21:01:24.325318+0000","objects_scrubbed":0,"log_size":6681,"log_dups_size":0,"ondisk_log_size":6681,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:57:19.741924+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":424345600,"num_objects":6475,"num_object_clones":0,"num_object_copies":19425,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6475,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7581,"num_write_kb":449792,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,5],"acting":[4,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.b","version":"79'161","reported_seq":562,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-09-23T21:03:05.297539+0000","last_change":"2023-09-23T20:57:27.320172+0000","last_active":"2023-09-23T21:03:05.297539+0000","last_peered":"2023-09-23T21:03:05.297539+0000","last_clean":"2023-09-23T21:03:05.297539+0000","last_became_active":"2023-09-23T20:57:16.009630+0000","last_became_peered":"2023-09-23T20:57:16.009630+0000","last_unstale":"2023-09-23T21:03:05.297539+0000","last_undegraded":"2023-09-23T21:03:05.297539+0000","last_fullsized":"2023-09-23T21:03:05.297539+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:27.320022+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:27.320022+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:31:35.221558+0000","scrub_duration":0.0043894679999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,5],"acting":[1,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":155,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-09-23T21:03:05.261893+0000","last_change":"2023-09-23T20:54:13.950254+0000","last_active":"2023-09-23T21:03:05.261893+0000","last_peered":"2023-09-23T21:03:05.261893+0000","last_clean":"2023-09-23T21:03:05.261893+0000","last_became_active":"2023-09-23T20:54:13.949934+0000","last_became_peered":"2023-09-23T20:54:13.949934+0000","last_unstale":"2023-09-23T21:03:05.261893+0000","last_undegraded":"2023-09-23T21:03:05.261893+0000","last_fullsized":"2023-09-23T21:03:05.261893+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:30:32.847167+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":172,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-09-23T21:03:05.261614+0000","last_change":"2023-09-23T20:55:35.956153+0000","last_active":"2023-09-23T21:03:05.261614+0000","last_peered":"2023-09-23T21:03:05.261614+0000","last_clean":"2023-09-23T21:03:05.261614+0000","last_became_active":"2023-09-23T20:55:35.955630+0000","last_became_peered":"2023-09-23T20:55:35.955630+0000","last_unstale":"2023-09-23T21:03:05.261614+0000","last_undegraded":"2023-09-23T21:03:05.261614+0000","last_fullsized":"2023-09-23T21:03:05.261614+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:45:20.282929+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.3","version":"278'7307","reported_seq":7340,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-09-23T21:03:14.425671+0000","last_change":"2023-09-23T21:01:25.342380+0000","last_active":"2023-09-23T21:03:14.425671+0000","last_peered":"2023-09-23T21:03:14.425671+0000","last_clean":"2023-09-23T21:03:14.425671+0000","last_became_active":"2023-09-23T21:01:25.342099+0000","last_became_peered":"2023-09-23T21:01:25.342099+0000","last_unstale":"2023-09-23T21:03:14.425671+0000","last_undegraded":"2023-09-23T21:03:14.425671+0000","last_fullsized":"2023-09-23T21:03:14.425671+0000","mapping_epoch":254,"log_start":"258'400","ondisk_log_start":"258'400","created":254,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:01:24.325318+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:01:24.325318+0000","last_clean_scrub_stamp":"2023-09-23T21:01:24.325318+0000","objects_scrubbed":0,"log_size":6907,"log_dups_size":0,"ondisk_log_size":6907,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:09:21.965969+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":400097280,"num_objects":6105,"num_object_clones":0,"num_object_copies":18315,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6105,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7307,"num_write_kb":429184,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.a","version":"79'161","reported_seq":521,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-09-23T21:03:05.276785+0000","last_change":"2023-09-23T20:56:44.487778+0000","last_active":"2023-09-23T21:03:05.276785+0000","last_peered":"2023-09-23T21:03:05.276785+0000","last_clean":"2023-09-23T21:03:05.276785+0000","last_became_active":"2023-09-23T20:56:05.006740+0000","last_became_peered":"2023-09-23T20:56:05.006740+0000","last_unstale":"2023-09-23T21:03:05.276785+0000","last_undegraded":"2023-09-23T21:03:05.276785+0000","last_fullsized":"2023-09-23T21:03:05.276785+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:56:44.487678+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:56:44.487678+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:04:50.086749+0000","scrub_duration":0.0017408899999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":68,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-09-23T21:03:05.256989+0000","last_change":"2023-09-23T20:57:44.982513+0000","last_active":"2023-09-23T21:03:05.256989+0000","last_peered":"2023-09-23T21:03:05.256989+0000","last_clean":"2023-09-23T21:03:05.256989+0000","last_became_active":"2023-09-23T20:57:44.982112+0000","last_became_peered":"2023-09-23T20:57:44.982112+0000","last_unstale":"2023-09-23T21:03:05.256989+0000","last_undegraded":"2023-09-23T21:03:05.256989+0000","last_fullsized":"2023-09-23T21:03:05.256989+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:09:05.630041+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":249,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-09-23T21:03:05.301074+0000","last_change":"2023-09-23T20:54:13.940748+0000","last_active":"2023-09-23T21:03:05.301074+0000","last_peered":"2023-09-23T21:03:05.301074+0000","last_clean":"2023-09-23T21:03:05.301074+0000","last_became_active":"2023-09-23T20:54:13.940419+0000","last_became_peered":"2023-09-23T20:54:13.940419+0000","last_unstale":"2023-09-23T21:03:05.301074+0000","last_undegraded":"2023-09-23T21:03:05.301074+0000","last_fullsized":"2023-09-23T21:03:05.301074+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:15:01.923816+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.e","version":"278'7595","reported_seq":7628,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-09-23T21:03:12.090439+0000","last_change":"2023-09-23T21:01:25.430111+0000","last_active":"2023-09-23T21:03:12.090439+0000","last_peered":"2023-09-23T21:03:12.090439+0000","last_clean":"2023-09-23T21:03:12.090439+0000","last_became_active":"2023-09-23T21:01:25.429811+0000","last_became_peered":"2023-09-23T21:01:25.429811+0000","last_unstale":"2023-09-23T21:03:12.090439+0000","last_undegraded":"2023-09-23T21:03:12.090439+0000","last_fullsized":"2023-09-23T21:03:12.090439+0000","mapping_epoch":254,"log_start":"0'0","ondisk_log_start":"0'0","created":254,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:01:24.325318+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:01:24.325318+0000","last_clean_scrub_stamp":"2023-09-23T21:01:24.325318+0000","objects_scrubbed":0,"log_size":7595,"log_dups_size":0,"ondisk_log_size":7595,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:18:01.918758+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":422510592,"num_objects":6447,"num_object_clones":0,"num_object_copies":19341,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6447,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7595,"num_write_kb":449344,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.7","version":"79'161","reported_seq":583,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-09-23T21:03:05.260382+0000","last_change":"2023-09-23T20:57:48.243572+0000","last_active":"2023-09-23T21:03:05.260382+0000","last_peered":"2023-09-23T21:03:05.260382+0000","last_clean":"2023-09-23T21:03:05.260382+0000","last_became_active":"2023-09-23T20:57:48.243229+0000","last_became_peered":"2023-09-23T20:57:48.243229+0000","last_unstale":"2023-09-23T21:03:05.260382+0000","last_undegraded":"2023-09-23T21:03:05.260382+0000","last_fullsized":"2023-09-23T21:03:05.260382+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":3,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:25.916867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:25.916867+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:23:19.385831+0000","scrub_duration":0.001665323,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":255,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-09-23T21:03:05.297712+0000","last_change":"2023-09-23T20:50:05.867130+0000","last_active":"2023-09-23T21:03:05.297712+0000","last_peered":"2023-09-23T21:03:05.297712+0000","last_clean":"2023-09-23T21:03:05.297712+0000","last_became_active":"2023-09-23T20:50:05.866758+0000","last_became_peered":"2023-09-23T20:50:05.866758+0000","last_unstale":"2023-09-23T21:03:05.297712+0000","last_undegraded":"2023-09-23T21:03:05.297712+0000","last_fullsized":"2023-09-23T21:03:05.297712+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:55:07.548209+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":151,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-09-23T21:03:05.276208+0000","last_change":"2023-09-23T20:54:31.887604+0000","last_active":"2023-09-23T21:03:05.276208+0000","last_peered":"2023-09-23T21:03:05.276208+0000","last_clean":"2023-09-23T21:03:05.276208+0000","last_became_active":"2023-09-23T20:54:31.886752+0000","last_became_peered":"2023-09-23T20:54:31.886752+0000","last_unstale":"2023-09-23T21:03:05.276208+0000","last_undegraded":"2023-09-23T21:03:05.276208+0000","last_fullsized":"2023-09-23T21:03:05.276208+0000","mapping_epoch":136,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:13:58.872176+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":253,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-09-23T21:03:05.292881+0000","last_change":"2023-09-23T20:54:31.899082+0000","last_active":"2023-09-23T21:03:05.292881+0000","last_peered":"2023-09-23T21:03:05.292881+0000","last_clean":"2023-09-23T21:03:05.292881+0000","last_became_active":"2023-09-23T20:54:31.898521+0000","last_became_peered":"2023-09-23T20:54:31.898521+0000","last_unstale":"2023-09-23T21:03:05.292881+0000","last_undegraded":"2023-09-23T21:03:05.292881+0000","last_fullsized":"2023-09-23T21:03:05.292881+0000","mapping_epoch":136,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:25:36.059711+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.8","version":"278'7432","reported_seq":7464,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-09-23T21:03:12.010791+0000","last_change":"2023-09-23T21:01:25.560581+0000","last_active":"2023-09-23T21:03:12.010791+0000","last_peered":"2023-09-23T21:03:12.010791+0000","last_clean":"2023-09-23T21:03:12.010791+0000","last_became_active":"2023-09-23T21:01:25.560318+0000","last_became_peered":"2023-09-23T21:01:25.560318+0000","last_unstale":"2023-09-23T21:03:12.010791+0000","last_undegraded":"2023-09-23T21:03:12.010791+0000","last_fullsized":"2023-09-23T21:03:12.010791+0000","mapping_epoch":254,"log_start":"0'0","ondisk_log_start":"0'0","created":254,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:01:24.325318+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:01:24.325318+0000","last_clean_scrub_stamp":"2023-09-23T21:01:24.325318+0000","objects_scrubbed":0,"log_size":7432,"log_dups_size":0,"ondisk_log_size":7432,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:07:28.203012+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":421134336,"num_objects":6426,"num_object_clones":0,"num_object_copies":19278,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6426,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7432,"num_write_kb":443456,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[5,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.1","version":"79'161","reported_seq":541,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-09-23T21:03:05.275839+0000","last_change":"2023-09-23T20:57:19.669346+0000","last_active":"2023-09-23T21:03:05.275839+0000","last_peered":"2023-09-23T21:03:05.275839+0000","last_clean":"2023-09-23T21:03:05.275839+0000","last_became_active":"2023-09-23T20:57:08.246046+0000","last_became_peered":"2023-09-23T20:57:08.246046+0000","last_unstale":"2023-09-23T21:03:05.275839+0000","last_undegraded":"2023-09-23T21:03:05.275839+0000","last_fullsized":"2023-09-23T21:03:05.275839+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:19.669259+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:19.669259+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:14:03.435531+0000","scrub_duration":0.0045759559999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":168,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-09-23T21:03:05.276513+0000","last_change":"2023-09-23T20:55:35.959514+0000","last_active":"2023-09-23T21:03:05.276513+0000","last_peered":"2023-09-23T21:03:05.276513+0000","last_clean":"2023-09-23T21:03:05.276513+0000","last_became_active":"2023-09-23T20:55:35.959244+0000","last_became_peered":"2023-09-23T20:55:35.959244+0000","last_unstale":"2023-09-23T21:03:05.276513+0000","last_undegraded":"2023-09-23T21:03:05.276513+0000","last_fullsized":"2023-09-23T21:03:05.276513+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:47:32.665316+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":172,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-09-23T21:03:05.261573+0000","last_change":"2023-09-23T20:55:35.949000+0000","last_active":"2023-09-23T21:03:05.261573+0000","last_peered":"2023-09-23T21:03:05.261573+0000","last_clean":"2023-09-23T21:03:05.261573+0000","last_became_active":"2023-09-23T20:55:35.948410+0000","last_became_peered":"2023-09-23T20:55:35.948410+0000","last_unstale":"2023-09-23T21:03:05.261573+0000","last_undegraded":"2023-09-23T21:03:05.261573+0000","last_fullsized":"2023-09-23T21:03:05.261573+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:26:50.477289+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":247,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-09-23T21:03:05.297552+0000","last_change":"2023-09-23T20:51:10.829889+0000","last_active":"2023-09-23T21:03:05.297552+0000","last_peered":"2023-09-23T21:03:05.297552+0000","last_clean":"2023-09-23T21:03:05.297552+0000","last_became_active":"2023-09-23T20:51:10.829533+0000","last_became_peered":"2023-09-23T20:51:10.829533+0000","last_unstale":"2023-09-23T21:03:05.297552+0000","last_undegraded":"2023-09-23T21:03:05.297552+0000","last_fullsized":"2023-09-23T21:03:05.297552+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:43:04.603460+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.b","version":"278'7684","reported_seq":7717,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-09-23T21:03:14.863753+0000","last_change":"2023-09-23T21:01:25.340569+0000","last_active":"2023-09-23T21:03:14.863753+0000","last_peered":"2023-09-23T21:03:14.863753+0000","last_clean":"2023-09-23T21:03:14.863753+0000","last_became_active":"2023-09-23T21:01:25.340209+0000","last_became_peered":"2023-09-23T21:01:25.340209+0000","last_unstale":"2023-09-23T21:03:14.863753+0000","last_undegraded":"2023-09-23T21:03:14.863753+0000","last_fullsized":"2023-09-23T21:03:14.863753+0000","mapping_epoch":254,"log_start":"258'1100","ondisk_log_start":"258'1100","created":254,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:01:24.325318+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:01:24.325318+0000","last_clean_scrub_stamp":"2023-09-23T21:01:24.325318+0000","objects_scrubbed":0,"log_size":6584,"log_dups_size":0,"ondisk_log_size":6584,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:53:59.553810+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":416808960,"num_objects":6360,"num_object_clones":0,"num_object_copies":19080,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6360,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7684,"num_write_kb":449408,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.2","version":"79'161","reported_seq":510,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-09-23T21:03:05.265978+0000","last_change":"2023-09-23T20:57:24.802586+0000","last_active":"2023-09-23T21:03:05.265978+0000","last_peered":"2023-09-23T21:03:05.265978+0000","last_clean":"2023-09-23T21:03:05.265978+0000","last_became_active":"2023-09-23T20:57:08.236114+0000","last_became_peered":"2023-09-23T20:57:08.236114+0000","last_unstale":"2023-09-23T21:03:05.265978+0000","last_undegraded":"2023-09-23T21:03:05.265978+0000","last_fullsized":"2023-09-23T21:03:05.265978+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:24.802451+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:24.802451+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:06:22.309980+0000","scrub_duration":0.0090861250000000005,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,4],"acting":[0,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":266,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-09-23T21:03:06.642015+0000","last_change":"2023-09-23T20:51:10.838054+0000","last_active":"2023-09-23T21:03:06.642015+0000","last_peered":"2023-09-23T21:03:06.642015+0000","last_clean":"2023-09-23T21:03:06.642015+0000","last_became_active":"2023-09-23T20:51:10.837750+0000","last_became_peered":"2023-09-23T20:51:10.837750+0000","last_unstale":"2023-09-23T21:03:06.642015+0000","last_undegraded":"2023-09-23T21:03:06.642015+0000","last_fullsized":"2023-09-23T21:03:06.642015+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:15:51.648085+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":247,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-09-23T21:03:05.281578+0000","last_change":"2023-09-23T20:55:35.951146+0000","last_active":"2023-09-23T21:03:05.281578+0000","last_peered":"2023-09-23T21:03:05.281578+0000","last_clean":"2023-09-23T21:03:05.281578+0000","last_became_active":"2023-09-23T20:55:35.950874+0000","last_became_peered":"2023-09-23T20:55:35.950874+0000","last_unstale":"2023-09-23T21:03:05.281578+0000","last_undegraded":"2023-09-23T21:03:05.281578+0000","last_fullsized":"2023-09-23T21:03:05.281578+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:58:13.957677+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":268,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-09-23T21:03:05.301000+0000","last_change":"2023-09-23T20:54:31.898870+0000","last_active":"2023-09-23T21:03:05.301000+0000","last_peered":"2023-09-23T21:03:05.301000+0000","last_clean":"2023-09-23T21:03:05.301000+0000","last_became_active":"2023-09-23T20:54:31.898467+0000","last_became_peered":"2023-09-23T20:54:31.898467+0000","last_unstale":"2023-09-23T21:03:05.301000+0000","last_undegraded":"2023-09-23T21:03:05.301000+0000","last_fullsized":"2023-09-23T21:03:05.301000+0000","mapping_epoch":136,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:56:22.483408+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.9","version":"278'7374","reported_seq":7406,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-09-23T21:03:14.130535+0000","last_change":"2023-09-23T21:01:25.342548+0000","last_active":"2023-09-23T21:03:14.130535+0000","last_peered":"2023-09-23T21:03:14.130535+0000","last_clean":"2023-09-23T21:03:14.130535+0000","last_became_active":"2023-09-23T21:01:25.342227+0000","last_became_peered":"2023-09-23T21:01:25.342227+0000","last_unstale":"2023-09-23T21:03:14.130535+0000","last_undegraded":"2023-09-23T21:03:14.130535+0000","last_fullsized":"2023-09-23T21:03:14.130535+0000","mapping_epoch":254,"log_start":"0'0","ondisk_log_start":"0'0","created":254,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:01:24.325318+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:01:24.325318+0000","last_clean_scrub_stamp":"2023-09-23T21:01:24.325318+0000","objects_scrubbed":0,"log_size":7374,"log_dups_size":0,"ondisk_log_size":7374,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:38:15.944067+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":404619264,"num_objects":6174,"num_object_clones":0,"num_object_copies":18522,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6174,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7374,"num_write_kb":433536,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,0],"acting":[6,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":151,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-09-23T21:03:05.276010+0000","last_change":"2023-09-23T20:54:31.883504+0000","last_active":"2023-09-23T21:03:05.276010+0000","last_peered":"2023-09-23T21:03:05.276010+0000","last_clean":"2023-09-23T21:03:05.276010+0000","last_became_active":"2023-09-23T20:54:31.883172+0000","last_became_peered":"2023-09-23T20:54:31.883172+0000","last_unstale":"2023-09-23T21:03:05.276010+0000","last_undegraded":"2023-09-23T21:03:05.276010+0000","last_fullsized":"2023-09-23T21:03:05.276010+0000","mapping_epoch":136,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:45:40.289120+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"79'161","reported_seq":510,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-09-23T21:03:05.265322+0000","last_change":"2023-09-23T20:57:20.738417+0000","last_active":"2023-09-23T21:03:05.265322+0000","last_peered":"2023-09-23T21:03:05.265322+0000","last_clean":"2023-09-23T21:03:05.265322+0000","last_became_active":"2023-09-23T20:57:08.236842+0000","last_became_peered":"2023-09-23T20:57:08.236842+0000","last_unstale":"2023-09-23T21:03:05.265322+0000","last_undegraded":"2023-09-23T21:03:05.265322+0000","last_fullsized":"2023-09-23T21:03:05.265322+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":0,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:20.738273+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:20.738273+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:18:28.327990+0000","scrub_duration":0.0088336339999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":257,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-09-23T21:03:05.257750+0000","last_change":"2023-09-23T20:55:35.950777+0000","last_active":"2023-09-23T21:03:05.257750+0000","last_peered":"2023-09-23T21:03:05.257750+0000","last_clean":"2023-09-23T21:03:05.257750+0000","last_became_active":"2023-09-23T20:55:35.950427+0000","last_became_peered":"2023-09-23T20:55:35.950427+0000","last_unstale":"2023-09-23T21:03:05.257750+0000","last_undegraded":"2023-09-23T21:03:05.257750+0000","last_fullsized":"2023-09-23T21:03:05.257750+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:20:10.371751+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":222,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-09-23T21:03:05.260346+0000","last_change":"2023-09-23T20:55:35.978140+0000","last_active":"2023-09-23T21:03:05.260346+0000","last_peered":"2023-09-23T21:03:05.260346+0000","last_clean":"2023-09-23T21:03:05.260346+0000","last_became_active":"2023-09-23T20:55:35.977831+0000","last_became_peered":"2023-09-23T20:55:35.977831+0000","last_unstale":"2023-09-23T21:03:05.260346+0000","last_undegraded":"2023-09-23T21:03:05.260346+0000","last_fullsized":"2023-09-23T21:03:05.260346+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:25:47.678023+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.a","version":"278'7501","reported_seq":7534,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-09-23T21:03:14.447601+0000","last_change":"2023-09-23T21:01:25.429899+0000","last_active":"2023-09-23T21:03:14.447601+0000","last_peered":"2023-09-23T21:03:14.447601+0000","last_clean":"2023-09-23T21:03:14.447601+0000","last_became_active":"2023-09-23T21:01:25.429583+0000","last_became_peered":"2023-09-23T21:01:25.429583+0000","last_unstale":"2023-09-23T21:03:14.447601+0000","last_undegraded":"2023-09-23T21:03:14.447601+0000","last_fullsized":"2023-09-23T21:03:14.447601+0000","mapping_epoch":254,"log_start":"258'600","ondisk_log_start":"258'600","created":254,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:01:24.325318+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:01:24.325318+0000","last_clean_scrub_stamp":"2023-09-23T21:01:24.325318+0000","objects_scrubbed":0,"log_size":6901,"log_dups_size":0,"ondisk_log_size":6901,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:31:11.594129+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":407568384,"num_objects":6219,"num_object_clones":0,"num_object_copies":18657,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6219,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7501,"num_write_kb":439040,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.3","version":"79'161","reported_seq":546,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-09-23T21:03:05.297536+0000","last_change":"2023-09-23T20:57:20.378531+0000","last_active":"2023-09-23T21:03:05.297536+0000","last_peered":"2023-09-23T21:03:05.297536+0000","last_clean":"2023-09-23T21:03:05.297536+0000","last_became_active":"2023-09-23T20:57:08.238019+0000","last_became_peered":"2023-09-23T20:57:08.238019+0000","last_unstale":"2023-09-23T21:03:05.297536+0000","last_undegraded":"2023-09-23T21:03:05.297536+0000","last_fullsized":"2023-09-23T21:03:05.297536+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":3,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:20.377714+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:20.377714+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:00:08.305602+0000","scrub_duration":0.003015228,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":149,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-09-23T21:03:05.281700+0000","last_change":"2023-09-23T20:56:45.991884+0000","last_active":"2023-09-23T21:03:05.281700+0000","last_peered":"2023-09-23T21:03:05.281700+0000","last_clean":"2023-09-23T21:03:05.281700+0000","last_became_active":"2023-09-23T20:56:45.991568+0000","last_became_peered":"2023-09-23T20:56:45.991568+0000","last_unstale":"2023-09-23T21:03:05.281700+0000","last_undegraded":"2023-09-23T21:03:05.281700+0000","last_fullsized":"2023-09-23T21:03:05.281700+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":205,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:21:20.775511+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,0],"acting":[1,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":273,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-09-23T21:03:05.265705+0000","last_change":"2023-09-23T20:57:44.979950+0000","last_active":"2023-09-23T21:03:05.265705+0000","last_peered":"2023-09-23T21:03:05.265705+0000","last_clean":"2023-09-23T21:03:05.265705+0000","last_became_active":"2023-09-23T20:57:44.979346+0000","last_became_peered":"2023-09-23T20:57:44.979346+0000","last_unstale":"2023-09-23T21:03:05.265705+0000","last_undegraded":"2023-09-23T21:03:05.265705+0000","last_fullsized":"2023-09-23T21:03:05.265705+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:10:24.372800+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":252,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-09-23T21:03:05.266118+0000","last_change":"2023-09-23T20:50:37.982235+0000","last_active":"2023-09-23T21:03:05.266118+0000","last_peered":"2023-09-23T21:03:05.266118+0000","last_clean":"2023-09-23T21:03:05.266118+0000","last_became_active":"2023-09-23T20:50:37.981913+0000","last_became_peered":"2023-09-23T20:50:37.981913+0000","last_unstale":"2023-09-23T21:03:05.266118+0000","last_undegraded":"2023-09-23T21:03:05.266118+0000","last_fullsized":"2023-09-23T21:03:05.266118+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:59:57.122584+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.f","version":"278'7461","reported_seq":7494,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-09-23T21:03:14.852490+0000","last_change":"2023-09-23T21:01:25.432176+0000","last_active":"2023-09-23T21:03:14.852490+0000","last_peered":"2023-09-23T21:03:14.852490+0000","last_clean":"2023-09-23T21:03:14.852490+0000","last_became_active":"2023-09-23T21:01:25.431835+0000","last_became_peered":"2023-09-23T21:01:25.431835+0000","last_unstale":"2023-09-23T21:03:14.852490+0000","last_undegraded":"2023-09-23T21:03:14.852490+0000","last_fullsized":"2023-09-23T21:03:14.852490+0000","mapping_epoch":254,"log_start":"258'900","ondisk_log_start":"258'900","created":254,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:01:24.325318+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:01:24.325318+0000","last_clean_scrub_stamp":"2023-09-23T21:01:24.325318+0000","objects_scrubbed":0,"log_size":6561,"log_dups_size":0,"ondisk_log_size":6561,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:47:26.553643+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":405340160,"num_objects":6185,"num_object_clones":0,"num_object_copies":18555,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6185,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7461,"num_write_kb":436672,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,1],"acting":[3,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.6","version":"79'161","reported_seq":560,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-09-23T21:03:05.260182+0000","last_change":"2023-09-23T20:57:26.907505+0000","last_active":"2023-09-23T21:03:05.260182+0000","last_peered":"2023-09-23T21:03:05.260182+0000","last_clean":"2023-09-23T21:03:05.260182+0000","last_became_active":"2023-09-23T20:57:08.240321+0000","last_became_peered":"2023-09-23T20:57:08.240321+0000","last_unstale":"2023-09-23T21:03:05.260182+0000","last_undegraded":"2023-09-23T21:03:05.260182+0000","last_fullsized":"2023-09-23T21:03:05.260182+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":3,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:26.907408+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:26.907408+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:34:17.392928+0000","scrub_duration":0.010337038,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":196,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-09-23T21:03:05.265877+0000","last_change":"2023-09-23T20:56:45.988566+0000","last_active":"2023-09-23T21:03:05.265877+0000","last_peered":"2023-09-23T21:03:05.265877+0000","last_clean":"2023-09-23T21:03:05.265877+0000","last_became_active":"2023-09-23T20:56:45.988249+0000","last_became_peered":"2023-09-23T20:56:45.988249+0000","last_unstale":"2023-09-23T21:03:05.265877+0000","last_undegraded":"2023-09-23T21:03:05.265877+0000","last_fullsized":"2023-09-23T21:03:05.265877+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":205,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:51:36.644764+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,4],"acting":[0,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":250,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-09-23T21:03:05.297668+0000","last_change":"2023-09-23T20:54:31.898965+0000","last_active":"2023-09-23T21:03:05.297668+0000","last_peered":"2023-09-23T21:03:05.297668+0000","last_clean":"2023-09-23T21:03:05.297668+0000","last_became_active":"2023-09-23T20:54:31.898663+0000","last_became_peered":"2023-09-23T20:54:31.898663+0000","last_unstale":"2023-09-23T21:03:05.297668+0000","last_undegraded":"2023-09-23T21:03:05.297668+0000","last_fullsized":"2023-09-23T21:03:05.297668+0000","mapping_epoch":136,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:53:05.058087+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":240,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-09-23T21:03:05.266200+0000","last_change":"2023-09-23T20:54:31.884717+0000","last_active":"2023-09-23T21:03:05.266200+0000","last_peered":"2023-09-23T21:03:05.266200+0000","last_clean":"2023-09-23T21:03:05.266200+0000","last_became_active":"2023-09-23T20:54:31.884345+0000","last_became_peered":"2023-09-23T20:54:31.884345+0000","last_unstale":"2023-09-23T21:03:05.266200+0000","last_undegraded":"2023-09-23T21:03:05.266200+0000","last_fullsized":"2023-09-23T21:03:05.266200+0000","mapping_epoch":136,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:02:26.393868+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.c","version":"278'7485","reported_seq":7518,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-09-23T21:03:12.051284+0000","last_change":"2023-09-23T21:01:25.560907+0000","last_active":"2023-09-23T21:03:12.051284+0000","last_peered":"2023-09-23T21:03:12.051284+0000","last_clean":"2023-09-23T21:03:12.051284+0000","last_became_active":"2023-09-23T21:01:25.560592+0000","last_became_peered":"2023-09-23T21:01:25.560592+0000","last_unstale":"2023-09-23T21:03:12.051284+0000","last_undegraded":"2023-09-23T21:03:12.051284+0000","last_fullsized":"2023-09-23T21:03:12.051284+0000","mapping_epoch":254,"log_start":"0'0","ondisk_log_start":"0'0","created":254,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:01:24.325318+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:01:24.325318+0000","last_clean_scrub_stamp":"2023-09-23T21:01:24.325318+0000","objects_scrubbed":0,"log_size":7485,"log_dups_size":0,"ondisk_log_size":7485,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:14:22.102068+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":424083456,"num_objects":6471,"num_object_clones":0,"num_object_copies":19413,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6471,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7485,"num_write_kb":446592,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.5","version":"79'161","reported_seq":541,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-09-23T21:03:05.261285+0000","last_change":"2023-09-23T20:57:30.029479+0000","last_active":"2023-09-23T21:03:05.261285+0000","last_peered":"2023-09-23T21:03:05.261285+0000","last_clean":"2023-09-23T21:03:05.261285+0000","last_became_active":"2023-09-23T20:57:08.249909+0000","last_became_peered":"2023-09-23T20:57:08.249909+0000","last_unstale":"2023-09-23T21:03:05.261285+0000","last_undegraded":"2023-09-23T21:03:05.261285+0000","last_fullsized":"2023-09-23T21:03:05.261285+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":3,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:30.029396+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:30.029396+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:16:46.083145+0000","scrub_duration":0.012914682,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":243,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-09-23T21:03:05.258843+0000","last_change":"2023-09-23T20:51:10.836910+0000","last_active":"2023-09-23T21:03:05.258843+0000","last_peered":"2023-09-23T21:03:05.258843+0000","last_clean":"2023-09-23T21:03:05.258843+0000","last_became_active":"2023-09-23T20:51:10.836557+0000","last_became_peered":"2023-09-23T20:51:10.836557+0000","last_unstale":"2023-09-23T21:03:05.258843+0000","last_undegraded":"2023-09-23T21:03:05.258843+0000","last_fullsized":"2023-09-23T21:03:05.258843+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:51:54.019969+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":250,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-09-23T21:03:05.300702+0000","last_change":"2023-09-23T20:55:35.953444+0000","last_active":"2023-09-23T21:03:05.300702+0000","last_peered":"2023-09-23T21:03:05.300702+0000","last_clean":"2023-09-23T21:03:05.300702+0000","last_became_active":"2023-09-23T20:55:35.953151+0000","last_became_peered":"2023-09-23T20:55:35.953151+0000","last_unstale":"2023-09-23T21:03:05.300702+0000","last_undegraded":"2023-09-23T21:03:05.300702+0000","last_fullsized":"2023-09-23T21:03:05.300702+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:39:12.404391+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":68,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-09-23T21:03:05.257067+0000","last_change":"2023-09-23T20:57:44.982509+0000","last_active":"2023-09-23T21:03:05.257067+0000","last_peered":"2023-09-23T21:03:05.257067+0000","last_clean":"2023-09-23T21:03:05.257067+0000","last_became_active":"2023-09-23T20:57:44.982175+0000","last_became_peered":"2023-09-23T20:57:44.982175+0000","last_unstale":"2023-09-23T21:03:05.257067+0000","last_undegraded":"2023-09-23T21:03:05.257067+0000","last_fullsized":"2023-09-23T21:03:05.257067+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:37:49.215488+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.d","version":"278'7395","reported_seq":7428,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-09-23T21:03:12.047698+0000","last_change":"2023-09-23T21:01:25.560531+0000","last_active":"2023-09-23T21:03:12.047698+0000","last_peered":"2023-09-23T21:03:12.047698+0000","last_clean":"2023-09-23T21:03:12.047698+0000","last_became_active":"2023-09-23T21:01:25.560228+0000","last_became_peered":"2023-09-23T21:01:25.560228+0000","last_unstale":"2023-09-23T21:03:12.047698+0000","last_undegraded":"2023-09-23T21:03:12.047698+0000","last_fullsized":"2023-09-23T21:03:12.047698+0000","mapping_epoch":254,"log_start":"0'0","ondisk_log_start":"0'0","created":254,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:01:24.325318+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:01:24.325318+0000","last_clean_scrub_stamp":"2023-09-23T21:01:24.325318+0000","objects_scrubbed":0,"log_size":7395,"log_dups_size":0,"ondisk_log_size":7395,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:19:04.504855+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":412352536,"num_objects":6293,"num_object_clones":0,"num_object_copies":18879,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6293,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":7395,"num_write_kb":437953,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,2],"acting":[5,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.4","version":"79'161","reported_seq":558,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-09-23T21:03:05.259089+0000","last_change":"2023-09-23T20:57:20.012904+0000","last_active":"2023-09-23T21:03:05.259089+0000","last_peered":"2023-09-23T21:03:05.259089+0000","last_clean":"2023-09-23T21:03:05.259089+0000","last_became_active":"2023-09-23T20:57:08.237996+0000","last_became_peered":"2023-09-23T20:57:08.237996+0000","last_unstale":"2023-09-23T21:03:05.259089+0000","last_undegraded":"2023-09-23T21:03:05.259089+0000","last_fullsized":"2023-09-23T21:03:05.259089+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":3,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:20.012764+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:20.012764+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:51:28.796946+0000","scrub_duration":0.0045251919999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":227,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-09-23T21:03:05.260290+0000","last_change":"2023-09-23T20:55:35.949541+0000","last_active":"2023-09-23T21:03:05.260290+0000","last_peered":"2023-09-23T21:03:05.260290+0000","last_clean":"2023-09-23T21:03:05.260290+0000","last_became_active":"2023-09-23T20:55:35.944635+0000","last_became_peered":"2023-09-23T20:55:35.944635+0000","last_unstale":"2023-09-23T21:03:05.260290+0000","last_undegraded":"2023-09-23T21:03:05.260290+0000","last_fullsized":"2023-09-23T21:03:05.260290+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:30:36.033585+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"70'1","reported_seq":268,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-09-23T21:03:05.266214+0000","last_change":"2023-09-23T20:54:14.009723+0000","last_active":"2023-09-23T21:03:05.266214+0000","last_peered":"2023-09-23T21:03:05.266214+0000","last_clean":"2023-09-23T21:03:05.266214+0000","last_became_active":"2023-09-23T20:54:13.941771+0000","last_became_peered":"2023-09-23T20:54:13.941771+0000","last_unstale":"2023-09-23T21:03:05.266214+0000","last_undegraded":"2023-09-23T21:03:05.266214+0000","last_fullsized":"2023-09-23T21:03:05.266214+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:48:25.588116+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":182,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-09-23T21:03:05.261957+0000","last_change":"2023-09-23T20:55:35.949780+0000","last_active":"2023-09-23T21:03:05.261957+0000","last_peered":"2023-09-23T21:03:05.261957+0000","last_clean":"2023-09-23T21:03:05.261957+0000","last_became_active":"2023-09-23T20:55:35.949497+0000","last_became_peered":"2023-09-23T20:55:35.949497+0000","last_unstale":"2023-09-23T21:03:05.261957+0000","last_undegraded":"2023-09-23T21:03:05.261957+0000","last_fullsized":"2023-09-23T21:03:05.261957+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:09:37.420882+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":242,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-09-23T21:03:05.265902+0000","last_change":"2023-09-23T20:54:13.929196+0000","last_active":"2023-09-23T21:03:05.265902+0000","last_peered":"2023-09-23T21:03:05.265902+0000","last_clean":"2023-09-23T21:03:05.265902+0000","last_became_active":"2023-09-23T20:54:13.928831+0000","last_became_peered":"2023-09-23T20:54:13.928831+0000","last_unstale":"2023-09-23T21:03:05.265902+0000","last_undegraded":"2023-09-23T21:03:05.265902+0000","last_fullsized":"2023-09-23T21:03:05.265902+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:28:44.316851+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":248,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-09-23T21:03:06.642041+0000","last_change":"2023-09-23T20:52:12.855430+0000","last_active":"2023-09-23T21:03:06.642041+0000","last_peered":"2023-09-23T21:03:06.642041+0000","last_clean":"2023-09-23T21:03:06.642041+0000","last_became_active":"2023-09-23T20:52:12.855115+0000","last_became_peered":"2023-09-23T20:52:12.855115+0000","last_unstale":"2023-09-23T21:03:06.642041+0000","last_undegraded":"2023-09-23T21:03:06.642041+0000","last_fullsized":"2023-09-23T21:03:06.642041+0000","mapping_epoch":100,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":101,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:33:23.802214+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":155,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-09-23T21:03:05.276775+0000","last_change":"2023-09-23T20:54:31.902946+0000","last_active":"2023-09-23T21:03:05.276775+0000","last_peered":"2023-09-23T21:03:05.276775+0000","last_clean":"2023-09-23T21:03:05.276775+0000","last_became_active":"2023-09-23T20:54:31.889484+0000","last_became_peered":"2023-09-23T20:54:31.889484+0000","last_unstale":"2023-09-23T21:03:05.276775+0000","last_undegraded":"2023-09-23T21:03:05.276775+0000","last_fullsized":"2023-09-23T21:03:05.276775+0000","mapping_epoch":136,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:44:44.278398+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":155,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-09-23T21:03:05.261642+0000","last_change":"2023-09-23T20:54:13.950351+0000","last_active":"2023-09-23T21:03:05.261642+0000","last_peered":"2023-09-23T21:03:05.261642+0000","last_clean":"2023-09-23T21:03:05.261642+0000","last_became_active":"2023-09-23T20:54:13.949987+0000","last_became_peered":"2023-09-23T20:54:13.949987+0000","last_unstale":"2023-09-23T21:03:05.261642+0000","last_undegraded":"2023-09-23T21:03:05.261642+0000","last_fullsized":"2023-09-23T21:03:05.261642+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:46:17.765134+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":186,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-09-23T21:03:05.276222+0000","last_change":"2023-09-23T20:56:18.141728+0000","last_active":"2023-09-23T21:03:05.276222+0000","last_peered":"2023-09-23T21:03:05.276222+0000","last_clean":"2023-09-23T21:03:05.276222+0000","last_became_active":"2023-09-23T20:56:18.137979+0000","last_became_peered":"2023-09-23T20:56:18.137979+0000","last_unstale":"2023-09-23T21:03:05.276222+0000","last_undegraded":"2023-09-23T21:03:05.276222+0000","last_fullsized":"2023-09-23T21:03:05.276222+0000","mapping_epoch":192,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":193,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:58:05.658443+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":155,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-09-23T21:03:05.261166+0000","last_change":"2023-09-23T20:54:13.935307+0000","last_active":"2023-09-23T21:03:05.261166+0000","last_peered":"2023-09-23T21:03:05.261166+0000","last_clean":"2023-09-23T21:03:05.261166+0000","last_became_active":"2023-09-23T20:54:13.935000+0000","last_became_peered":"2023-09-23T20:54:13.935000+0000","last_unstale":"2023-09-23T21:03:05.261166+0000","last_undegraded":"2023-09-23T21:03:05.261166+0000","last_fullsized":"2023-09-23T21:03:05.261166+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:47:40.974082+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":267,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-09-23T21:03:09.306809+0000","last_change":"2023-09-23T20:55:35.954870+0000","last_active":"2023-09-23T21:03:09.306809+0000","last_peered":"2023-09-23T21:03:09.306809+0000","last_clean":"2023-09-23T21:03:09.306809+0000","last_became_active":"2023-09-23T20:55:35.954568+0000","last_became_peered":"2023-09-23T20:55:35.954568+0000","last_unstale":"2023-09-23T21:03:09.306809+0000","last_undegraded":"2023-09-23T21:03:09.306809+0000","last_fullsized":"2023-09-23T21:03:09.306809+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:06:22.423566+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":266,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-09-23T21:03:09.298545+0000","last_change":"2023-09-23T20:51:10.838334+0000","last_active":"2023-09-23T21:03:09.298545+0000","last_peered":"2023-09-23T21:03:09.298545+0000","last_clean":"2023-09-23T21:03:09.298545+0000","last_became_active":"2023-09-23T20:51:10.838026+0000","last_became_peered":"2023-09-23T20:51:10.838026+0000","last_unstale":"2023-09-23T21:03:09.298545+0000","last_undegraded":"2023-09-23T21:03:09.298545+0000","last_fullsized":"2023-09-23T21:03:09.298545+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:04:37.009457+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":270,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-09-23T21:03:05.266128+0000","last_change":"2023-09-23T20:54:31.903139+0000","last_active":"2023-09-23T21:03:05.266128+0000","last_peered":"2023-09-23T21:03:05.266128+0000","last_clean":"2023-09-23T21:03:05.266128+0000","last_became_active":"2023-09-23T20:54:31.884520+0000","last_became_peered":"2023-09-23T20:54:31.884520+0000","last_unstale":"2023-09-23T21:03:05.266128+0000","last_undegraded":"2023-09-23T21:03:05.266128+0000","last_fullsized":"2023-09-23T21:03:05.266128+0000","mapping_epoch":136,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:39:34.269705+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1108,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":172,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-09-23T21:03:05.260676+0000","last_change":"2023-09-23T20:55:35.951572+0000","last_active":"2023-09-23T21:03:05.260676+0000","last_peered":"2023-09-23T21:03:05.260676+0000","last_clean":"2023-09-23T21:03:05.260676+0000","last_became_active":"2023-09-23T20:55:35.951294+0000","last_became_peered":"2023-09-23T20:55:35.951294+0000","last_unstale":"2023-09-23T21:03:05.260676+0000","last_undegraded":"2023-09-23T21:03:05.260676+0000","last_fullsized":"2023-09-23T21:03:05.260676+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:36:32.447210+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":22,"num_bytes_recovered":5990,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":8,"num_pg":16,"stat_sum":{"num_bytes":6637617176,"num_objects":101283,"num_object_clones":0,"num_object_copies":303849,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":101283,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":120227,"num_write_kb":7088257,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":19986526208,"data_stored":19986513992,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":113727,"ondisk_log_size":113727,"up":48,"acting":48,"num_store_stats":8},{"poolid":1,"num_pg":21,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3381,"ondisk_log_size":3381,"up":63,"acting":63,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201553,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2299820,"kb_used_data":2069988,"kb_used_omap":0,"kb_used_meta":229504,"kb_avail":91445332,"statfs":{"total":95995035648,"available":93640019968,"internally_reserved":0,"allocated":2119667712,"data_stored":2118759510,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":235012096},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,0,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:02:52 2023","interfaces":[{"interface":"back","average":{"1min":2.1549999999999998,"5min":2.0529999999999999,"15min":1.423},"min":{"1min":0.23999999999999999,"5min":0.22800000000000001,"15min":0.21099999999999999},"max":{"1min":7.9299999999999997,"5min":33.326000000000001,"15min":43.994999999999997},"last":0.25800000000000001},{"interface":"front","average":{"1min":2.335,"5min":2.1720000000000002,"15min":1.536},"min":{"1min":0.28599999999999998,"5min":0.28599999999999998,"15min":0.218},"max":{"1min":7.9480000000000004,"5min":33.354999999999997,"15min":44.206000000000003},"last":5.0880000000000001}]},{"osd":1,"last update":"Sat Sep 23 21:02:30 2023","interfaces":[{"interface":"back","average":{"1min":1.034,"5min":1.3879999999999999,"15min":1.3340000000000001},"min":{"1min":0.26300000000000001,"5min":0.253,"15min":0.23300000000000001},"max":{"1min":5.1630000000000003,"5min":10.622,"15min":22.591999999999999},"last":0.29699999999999999},{"interface":"front","average":{"1min":0.91900000000000004,"5min":1.647,"15min":1.2669999999999999},"min":{"1min":0.27100000000000002,"5min":0.25800000000000001,"15min":0.23699999999999999},"max":{"1min":3.2749999999999999,"5min":13.981,"15min":21.437000000000001},"last":2.3610000000000002}]},{"osd":2,"last update":"Sat Sep 23 21:03:02 2023","interfaces":[{"interface":"back","average":{"1min":2.3780000000000001,"5min":2.3889999999999998,"15min":1.55},"min":{"1min":0.30099999999999999,"5min":0.27600000000000002,"15min":0.24399999999999999},"max":{"1min":10.279999999999999,"5min":10.805999999999999,"15min":10.805999999999999},"last":0.373},{"interface":"front","average":{"1min":1.7889999999999999,"5min":2.056,"15min":1.5620000000000001},"min":{"1min":0.309,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":4.8499999999999996,"5min":11.590999999999999,"15min":45.25},"last":0.34499999999999997}]},{"osd":3,"last update":"Sat Sep 23 21:02:38 2023","interfaces":[{"interface":"back","average":{"1min":2.0640000000000001,"5min":2.6459999999999999,"15min":2.097},"min":{"1min":0.34000000000000002,"5min":0.315,"15min":0.314},"max":{"1min":11.625999999999999,"5min":31.786000000000001,"15min":36.408000000000001},"last":0.73999999999999999},{"interface":"front","average":{"1min":1.3069999999999999,"5min":2.6539999999999999,"15min":1.9650000000000001},"min":{"1min":0.309,"5min":0.309,"15min":0.26000000000000001},"max":{"1min":6.181,"5min":32.429000000000002,"15min":35.593000000000004},"last":1.915}]},{"osd":4,"last update":"Sat Sep 23 21:02:30 2023","interfaces":[{"interface":"back","average":{"1min":0.71499999999999997,"5min":0.58999999999999997,"15min":0.63500000000000001},"min":{"1min":0.29399999999999998,"5min":0.25700000000000001,"15min":0.23699999999999999},"max":{"1min":2.2469999999999999,"5min":4.5410000000000004,"15min":5.8920000000000003},"last":1.3799999999999999},{"interface":"front","average":{"1min":1.0169999999999999,"5min":0.72599999999999998,"15min":0.67800000000000005},"min":{"1min":0.28000000000000003,"5min":0.27100000000000002,"15min":0.26300000000000001},"max":{"1min":6.3979999999999997,"5min":6.3979999999999997,"15min":6.3979999999999997},"last":1.9510000000000001}]},{"osd":5,"last update":"Sat Sep 23 21:02:30 2023","interfaces":[{"interface":"back","average":{"1min":1.4179999999999999,"5min":0.71399999999999997,"15min":0.69099999999999995},"min":{"1min":0.34300000000000003,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":5.2089999999999996,"5min":5.2089999999999996,"15min":7.0389999999999997},"last":0.59699999999999998},{"interface":"front","average":{"1min":0.76200000000000001,"5min":0.67600000000000005,"15min":0.65100000000000002},"min":{"1min":0.33300000000000002,"5min":0.30199999999999999,"15min":0.27800000000000002},"max":{"1min":2.2530000000000001,"5min":4.5999999999999996,"15min":4.5999999999999996},"last":4.0369999999999999}]},{"osd":6,"last update":"Sat Sep 23 21:02:46 2023","interfaces":[{"interface":"back","average":{"1min":1.1539999999999999,"5min":0.82699999999999996,"15min":0.72399999999999998},"min":{"1min":0.35199999999999998,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":6.1219999999999999,"5min":6.1219999999999999,"15min":6.1219999999999999},"last":1.9690000000000001},{"interface":"front","average":{"1min":1.081,"5min":0.86099999999999999,"15min":0.81999999999999995},"min":{"1min":0.32300000000000001,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":6.0919999999999996,"5min":6.0919999999999996,"15min":6.0919999999999996},"last":5.3929999999999998}]}]},{"osd":6,"up_from":164,"seq":704374636637,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2167448,"kb_used_data":2016912,"kb_used_omap":10,"kb_used_meta":150517,"kb_avail":91577704,"statfs":{"total":95995035648,"available":93775568896,"internally_reserved":0,"allocated":2065317888,"data_stored":2064429731,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":10328,"internal_metadata":154130344},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:02:49 2023","interfaces":[{"interface":"back","average":{"1min":2.7090000000000001,"5min":2.0169999999999999,"15min":1.3160000000000001},"min":{"1min":0.23499999999999999,"5min":0.23499999999999999,"15min":0.20999999999999999},"max":{"1min":14.151999999999999,"5min":14.151999999999999,"15min":14.151999999999999},"last":0.28100000000000003},{"interface":"front","average":{"1min":2.6709999999999998,"5min":2.3130000000000002,"15min":1.4530000000000001},"min":{"1min":0.29899999999999999,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":10.962,"5min":11.023,"15min":11.023},"last":0.34000000000000002}]},{"osd":1,"last update":"Sat Sep 23 21:02:49 2023","interfaces":[{"interface":"back","average":{"1min":2.1259999999999999,"5min":1.583,"15min":1.1060000000000001},"min":{"1min":0.26400000000000001,"5min":0.26400000000000001,"15min":0.25600000000000001},"max":{"1min":6.024,"5min":8.4309999999999992,"15min":8.4309999999999992},"last":2.3700000000000001},{"interface":"front","average":{"1min":2.3420000000000001,"5min":2.0070000000000001,"15min":1.2589999999999999},"min":{"1min":0.317,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":5.431,"5min":10.518000000000001,"15min":10.518000000000001},"last":2.6749999999999998}]},{"osd":2,"last update":"Sat Sep 23 21:02:49 2023","interfaces":[{"interface":"back","average":{"1min":3.9449999999999998,"5min":2.496,"15min":1.9990000000000001},"min":{"1min":0.28499999999999998,"5min":0.28499999999999998,"15min":0.27700000000000002},"max":{"1min":14.029999999999999,"5min":14.029999999999999,"15min":14.029999999999999},"last":0.40300000000000002},{"interface":"front","average":{"1min":2.8500000000000001,"5min":2.3889999999999998,"15min":1.5880000000000001},"min":{"1min":0.503,"5min":0.247,"15min":0.247},"max":{"1min":8.3670000000000009,"5min":20.195,"15min":20.195},"last":0.47699999999999998}]},{"osd":3,"last update":"Sat Sep 23 21:02:49 2023","interfaces":[{"interface":"back","average":{"1min":3.04,"5min":2.3330000000000002,"15min":1.528},"min":{"1min":0.28799999999999998,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":9.6349999999999998,"5min":14.144,"15min":14.144},"last":8.7240000000000002},{"interface":"front","average":{"1min":2.448,"5min":2.0390000000000001,"15min":1.7549999999999999},"min":{"1min":0.33100000000000002,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":16.847999999999999,"5min":16.847999999999999,"15min":16.847999999999999},"last":0.74099999999999999}]},{"osd":4,"last update":"Sat Sep 23 21:02:49 2023","interfaces":[{"interface":"back","average":{"1min":1.4259999999999999,"5min":0.80200000000000005,"15min":0.94199999999999995},"min":{"1min":0.251,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":5.3920000000000003,"5min":5.3920000000000003,"15min":9.0079999999999991},"last":0.44600000000000001},{"interface":"front","average":{"1min":1.718,"5min":0.76400000000000001,"15min":0.60299999999999998},"min":{"1min":0.26600000000000001,"5min":0.26600000000000001,"15min":0.25800000000000001},"max":{"1min":4.8760000000000003,"5min":4.8760000000000003,"15min":4.8760000000000003},"last":0.35899999999999999}]},{"osd":5,"last update":"Sat Sep 23 21:02:49 2023","interfaces":[{"interface":"back","average":{"1min":1.8440000000000001,"5min":0.97299999999999998,"15min":0.69399999999999995},"min":{"1min":0.31,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":5.9000000000000004,"5min":5.9000000000000004,"15min":5.9000000000000004},"last":0.505},{"interface":"front","average":{"1min":1.5609999999999999,"5min":0.89900000000000002,"15min":0.83099999999999996},"min":{"1min":0.30599999999999999,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":5.3220000000000001,"5min":5.3220000000000001,"15min":5.3220000000000001},"last":2.1819999999999999}]},{"osd":7,"last update":"Sat Sep 23 21:02:49 2023","interfaces":[{"interface":"back","average":{"1min":1.9530000000000001,"5min":0.93000000000000005,"15min":0.93100000000000005},"min":{"1min":0.33400000000000002,"5min":0.312,"15min":0.312},"max":{"1min":5.0490000000000004,"5min":5.0490000000000004,"15min":7.8570000000000002},"last":0.46700000000000003},{"interface":"front","average":{"1min":1.1499999999999999,"5min":0.77900000000000003,"15min":0.65100000000000002},"min":{"1min":0.35399999999999998,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":3.754,"5min":3.754,"15min":8.282},"last":0.48899999999999999}]}]},{"osd":1,"up_from":88,"seq":377957122208,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3300644,"kb_used_data":2929308,"kb_used_omap":9,"kb_used_meta":371318,"kb_avail":90444508,"statfs":{"total":95995035648,"available":92615176192,"internally_reserved":0,"allocated":2999611392,"data_stored":2998307455,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9239,"internal_metadata":380230633},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,0,1,0,2],"upper_bound":64},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:02:31 2023","interfaces":[{"interface":"back","average":{"1min":3.7730000000000001,"5min":3.081,"15min":2.0009999999999999},"min":{"1min":0.25,"5min":0.20699999999999999,"15min":0.20699999999999999},"max":{"1min":45.542000000000002,"5min":45.542000000000002,"15min":45.542000000000002},"last":1.639},{"interface":"front","average":{"1min":4.1929999999999996,"5min":3.7440000000000002,"15min":2.3100000000000001},"min":{"1min":0.28499999999999998,"5min":0.247,"15min":0.22500000000000001},"max":{"1min":46.353000000000002,"5min":46.353000000000002,"15min":46.353000000000002},"last":1.621}]},{"osd":2,"last update":"Sat Sep 23 21:03:00 2023","interfaces":[{"interface":"back","average":{"1min":2.6040000000000001,"5min":2.1899999999999999,"15min":1.6259999999999999},"min":{"1min":0.22900000000000001,"5min":0.20399999999999999,"15min":0.20300000000000001},"max":{"1min":9.5020000000000007,"5min":13.752000000000001,"15min":14.335000000000001},"last":0.49199999999999999},{"interface":"front","average":{"1min":4.4000000000000004,"5min":4.0880000000000001,"15min":2.5779999999999998},"min":{"1min":0.252,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":13.212,"5min":18.026,"15min":38.951000000000001},"last":0.64300000000000002}]},{"osd":3,"last update":"Sat Sep 23 21:02:31 2023","interfaces":[{"interface":"back","average":{"1min":2.3900000000000001,"5min":2.8410000000000002,"15min":2.0110000000000001},"min":{"1min":0.26500000000000001,"5min":0.246,"15min":0.246},"max":{"1min":8.7390000000000008,"5min":22.686,"15min":25.314},"last":3.125},{"interface":"front","average":{"1min":1.5509999999999999,"5min":2.4900000000000002,"15min":1.7629999999999999},"min":{"1min":0.25600000000000001,"5min":0.25600000000000001,"15min":0.20599999999999999},"max":{"1min":10.157999999999999,"5min":14.473000000000001,"15min":19.881},"last":0.46100000000000002}]},{"osd":4,"last update":"Sat Sep 23 21:02:31 2023","interfaces":[{"interface":"back","average":{"1min":3.5030000000000001,"5min":2.0470000000000002,"15min":1.528},"min":{"1min":0.28399999999999997,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":41.823,"5min":41.823,"15min":41.823},"last":3.1899999999999999},{"interface":"front","average":{"1min":3.097,"5min":1.9359999999999999,"15min":1.3420000000000001},"min":{"1min":0.33300000000000002,"5min":0.31900000000000001,"15min":0.27600000000000002},"max":{"1min":41.423000000000002,"5min":41.423000000000002,"15min":41.423000000000002},"last":0.52000000000000002}]},{"osd":5,"last update":"Sat Sep 23 21:02:31 2023","interfaces":[{"interface":"back","average":{"1min":1.446,"5min":2.2789999999999999,"15min":1.55},"min":{"1min":0.32000000000000001,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":7.5570000000000004,"5min":15.041,"15min":15.041},"last":0.38500000000000001},{"interface":"front","average":{"1min":1.5649999999999999,"5min":1.6830000000000001,"15min":1.431},"min":{"1min":0.36099999999999999,"5min":0.33200000000000002,"15min":0.32200000000000001},"max":{"1min":5.4930000000000003,"5min":18.280999999999999,"15min":18.280999999999999},"last":3.1600000000000001}]},{"osd":6,"last update":"Sat Sep 23 21:02:51 2023","interfaces":[{"interface":"back","average":{"1min":2.4129999999999998,"5min":1.8720000000000001,"15min":1.4990000000000001},"min":{"1min":0.36199999999999999,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":5.4690000000000003,"5min":18.984999999999999,"15min":18.984999999999999},"last":3.214},{"interface":"front","average":{"1min":1.6519999999999999,"5min":1.6000000000000001,"15min":1.415},"min":{"1min":0.33400000000000002,"5min":0.33400000000000002,"15min":0.309},"max":{"1min":4.7839999999999998,"5min":8.7530000000000001,"15min":8.7530000000000001},"last":0.70899999999999996}]},{"osd":7,"last update":"Sat Sep 23 21:02:31 2023","interfaces":[{"interface":"back","average":{"1min":1.238,"5min":1.6910000000000001,"15min":1.3149999999999999},"min":{"1min":0.39600000000000002,"5min":0.39600000000000002,"15min":0.35699999999999998},"max":{"1min":3.9900000000000002,"5min":8.7769999999999992,"15min":11.186999999999999},"last":0.74399999999999999},{"interface":"front","average":{"1min":2.0329999999999999,"5min":2.3570000000000002,"15min":1.5940000000000001},"min":{"1min":0.38100000000000001,"5min":0.38100000000000001,"15min":0.34599999999999997},"max":{"1min":7.6100000000000003,"5min":15.098000000000001,"15min":15.098000000000001},"last":0.58199999999999996}]}]},{"osd":0,"up_from":83,"seq":356482285734,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2823868,"kb_used_data":2489140,"kb_used_omap":28,"kb_used_meta":334627,"kb_avail":90921284,"statfs":{"total":95995035648,"available":93103394816,"internally_reserved":0,"allocated":2548879360,"data_stored":2547612157,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":28848,"internal_metadata":342658896},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[3,1,0,0,6],"upper_bound":32},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 21:02:21 2023","interfaces":[{"interface":"back","average":{"1min":2.2930000000000001,"5min":3.782,"15min":2.181},"min":{"1min":0.19500000000000001,"5min":0.19500000000000001,"15min":0.191},"max":{"1min":22.411000000000001,"5min":22.411000000000001,"15min":22.411000000000001},"last":0.45800000000000002},{"interface":"front","average":{"1min":3.0870000000000002,"5min":3.8799999999999999,"15min":2.1970000000000001},"min":{"1min":0.29299999999999998,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":22.474,"5min":22.474,"15min":22.474},"last":1.002}]},{"osd":2,"last update":"Sat Sep 23 21:03:00 2023","interfaces":[{"interface":"back","average":{"1min":3.2480000000000002,"5min":3.387,"15min":2.331},"min":{"1min":0.45800000000000002,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":11.739000000000001,"5min":21.053000000000001,"15min":21.053000000000001},"last":1.135},{"interface":"front","average":{"1min":3.3690000000000002,"5min":4.0439999999999996,"15min":2.452},"min":{"1min":0.25900000000000001,"5min":0.23100000000000001,"15min":0.21099999999999999},"max":{"1min":11.202999999999999,"5min":21.747,"15min":21.747},"last":1.1220000000000001}]},{"osd":3,"last update":"Sat Sep 23 21:02:21 2023","interfaces":[{"interface":"back","average":{"1min":1.6990000000000001,"5min":3.6360000000000001,"15min":2.1549999999999998},"min":{"1min":0.29599999999999999,"5min":0.27100000000000002,"15min":0.25800000000000001},"max":{"1min":11.617000000000001,"5min":21.184000000000001,"15min":21.184000000000001},"last":0.55200000000000005},{"interface":"front","average":{"1min":2.2280000000000002,"5min":3.8839999999999999,"15min":2.3380000000000001},"min":{"1min":0.32900000000000001,"5min":0.27400000000000002,"15min":0.23300000000000001},"max":{"1min":8.4510000000000005,"5min":21.167000000000002,"15min":21.167000000000002},"last":0.96299999999999997}]},{"osd":4,"last update":"Sat Sep 23 21:03:00 2023","interfaces":[{"interface":"back","average":{"1min":2.4790000000000001,"5min":2.9620000000000002,"15min":1.845},"min":{"1min":0.29599999999999999,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":7.9139999999999997,"5min":21.594999999999999,"15min":21.594999999999999},"last":0.436},{"interface":"front","average":{"1min":1.143,"5min":2.25,"15min":1.528},"min":{"1min":0.35299999999999998,"5min":0.32900000000000001,"15min":0.28100000000000003},"max":{"1min":4.6559999999999997,"5min":13.904,"15min":18.724},"last":0.52300000000000002}]},{"osd":5,"last update":"Sat Sep 23 21:03:00 2023","interfaces":[{"interface":"back","average":{"1min":2.6309999999999998,"5min":2.1400000000000001,"15min":1.5660000000000001},"min":{"1min":0.35899999999999999,"5min":0.309,"15min":0.309},"max":{"1min":11.707000000000001,"5min":11.707000000000001,"15min":11.707000000000001},"last":2.7959999999999998},{"interface":"front","average":{"1min":2.6019999999999999,"5min":2.7719999999999998,"15min":1.8500000000000001},"min":{"1min":0.373,"5min":0.373,"15min":0.34999999999999998},"max":{"1min":7.8579999999999997,"5min":20.512,"15min":20.512},"last":2.5550000000000002}]},{"osd":6,"last update":"Sat Sep 23 21:02:55 2023","interfaces":[{"interface":"back","average":{"1min":1.54,"5min":2.2639999999999998,"15min":1.8700000000000001},"min":{"1min":0.35899999999999999,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":4.6319999999999997,"5min":13.933,"15min":13.933},"last":0.66100000000000003},{"interface":"front","average":{"1min":1.262,"5min":2.2949999999999999,"15min":1.9370000000000001},"min":{"1min":0.42999999999999999,"5min":0.379,"15min":0.377},"max":{"1min":4.5659999999999998,"5min":13.962,"15min":13.962},"last":2.609}]},{"osd":7,"last update":"Sat Sep 23 21:03:00 2023","interfaces":[{"interface":"back","average":{"1min":2.673,"5min":2.2480000000000002,"15min":1.6180000000000001},"min":{"1min":0.38100000000000001,"5min":0.38100000000000001,"15min":0.379},"max":{"1min":11.657,"5min":11.657,"15min":11.657},"last":2.6259999999999999},{"interface":"front","average":{"1min":1.288,"5min":2.3010000000000002,"15min":1.5700000000000001},"min":{"1min":0.41499999999999998,"5min":0.377,"15min":0.377},"max":{"1min":4.5990000000000002,"5min":13.856999999999999,"15min":18.673999999999999},"last":2.5720000000000001}]}]},{"osd":2,"up_from":93,"seq":399431958681,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2979524,"kb_used_data":2611772,"kb_used_omap":81,"kb_used_meta":367598,"kb_avail":90765628,"statfs":{"total":95995035648,"available":92944003072,"internally_reserved":0,"allocated":2674454528,"data_stored":2673177441,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":83750,"internal_metadata":376420570},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,1],"upper_bound":4},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:02:51 2023","interfaces":[{"interface":"back","average":{"1min":1.724,"5min":2.9630000000000001,"15min":1.762},"min":{"1min":0.23200000000000001,"5min":0.189,"15min":0.17999999999999999},"max":{"1min":5.1870000000000003,"5min":29.498000000000001,"15min":29.498000000000001},"last":5.5549999999999997},{"interface":"front","average":{"1min":1.9970000000000001,"5min":3.2149999999999999,"15min":2.052},"min":{"1min":0.27700000000000002,"5min":0.193,"15min":0.193},"max":{"1min":5.5430000000000001,"5min":17.251000000000001,"15min":17.251000000000001},"last":5.1749999999999998}]},{"osd":1,"last update":"Sat Sep 23 21:02:51 2023","interfaces":[{"interface":"back","average":{"1min":2.9279999999999999,"5min":2.7200000000000002,"15min":1.9990000000000001},"min":{"1min":0.29899999999999999,"5min":0.246,"15min":0.20200000000000001},"max":{"1min":12.965999999999999,"5min":14.709,"15min":16.081},"last":5.5220000000000002},{"interface":"front","average":{"1min":3.121,"5min":3.819,"15min":2.5720000000000001},"min":{"1min":0.39500000000000002,"5min":0.26300000000000001,"15min":0.23599999999999999},"max":{"1min":13.098000000000001,"5min":26.552,"15min":26.552},"last":4.8659999999999997}]},{"osd":3,"last update":"Sat Sep 23 21:02:34 2023","interfaces":[{"interface":"back","average":{"1min":2.137,"5min":4.1699999999999999,"15min":2.524},"min":{"1min":0.45500000000000002,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":5.8310000000000004,"5min":41.057000000000002,"15min":41.057000000000002},"last":2.7839999999999998},{"interface":"front","average":{"1min":1.9319999999999999,"5min":4.4340000000000002,"15min":2.8300000000000001},"min":{"1min":0.25,"5min":0.25,"15min":0.221},"max":{"1min":9.4870000000000001,"5min":41.012,"15min":41.012},"last":4.79}]},{"osd":4,"last update":"Sat Sep 23 21:02:51 2023","interfaces":[{"interface":"back","average":{"1min":1.3240000000000001,"5min":1.597,"15min":1.361},"min":{"1min":0.35299999999999998,"5min":0.28399999999999997,"15min":0.26400000000000001},"max":{"1min":5.4500000000000002,"5min":10.045,"15min":10.045},"last":1.4199999999999999},{"interface":"front","average":{"1min":2.6230000000000002,"5min":2.286,"15min":1.6679999999999999},"min":{"1min":0.32900000000000001,"5min":0.32400000000000001,"15min":0.313},"max":{"1min":7.1529999999999996,"5min":14.039999999999999,"15min":14.039999999999999},"last":4.6230000000000002}]},{"osd":5,"last update":"Sat Sep 23 21:02:51 2023","interfaces":[{"interface":"back","average":{"1min":1.9399999999999999,"5min":1.6599999999999999,"15min":1.482},"min":{"1min":0.32300000000000001,"5min":0.30399999999999999,"15min":0.29299999999999998},"max":{"1min":8.0909999999999993,"5min":10.069000000000001,"15min":10.069000000000001},"last":0.48799999999999999},{"interface":"front","average":{"1min":2.3279999999999998,"5min":2.133,"15min":1.6459999999999999},"min":{"1min":0.36199999999999999,"5min":0.31900000000000001,"15min":0.30299999999999999},"max":{"1min":7.1109999999999998,"5min":13.98,"15min":13.98},"last":4.6909999999999998}]},{"osd":6,"last update":"Sat Sep 23 21:02:50 2023","interfaces":[{"interface":"back","average":{"1min":2.4369999999999998,"5min":2.3359999999999999,"15min":1.909},"min":{"1min":0.40100000000000002,"5min":0.35699999999999998,"15min":0.312},"max":{"1min":7.0670000000000002,"5min":14.016,"15min":14.244999999999999},"last":4.7169999999999996},{"interface":"front","average":{"1min":2.4790000000000001,"5min":2.431,"15min":1.5569999999999999},"min":{"1min":0.41499999999999998,"5min":0.38,"15min":0.374},"max":{"1min":7.4210000000000003,"5min":14.068,"15min":14.068},"last":4.665}]},{"osd":7,"last update":"Sat Sep 23 21:02:51 2023","interfaces":[{"interface":"back","average":{"1min":1.046,"5min":1.3660000000000001,"15min":1.1819999999999999},"min":{"1min":0.441,"5min":0.315,"15min":0.315},"max":{"1min":4.8040000000000003,"5min":13.427,"15min":13.427},"last":3.3919999999999999},{"interface":"front","average":{"1min":1.911,"5min":1.6639999999999999,"15min":1.4930000000000001},"min":{"1min":0.42999999999999999,"5min":0.35099999999999998,"15min":0.34499999999999997},"max":{"1min":7.6959999999999997,"5min":10.092000000000001,"15min":10.092000000000001},"last":1.379}]}]},{"osd":3,"up_from":98,"seq":420906795155,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2867092,"kb_used_data":2405836,"kb_used_omap":7,"kb_used_meta":461240,"kb_avail":90878060,"statfs":{"total":95995035648,"available":93059133440,"internally_reserved":0,"allocated":2463576064,"data_stored":2462342972,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7576,"internal_metadata":472310376},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:02:32 2023","interfaces":[{"interface":"back","average":{"1min":1.288,"5min":3.6619999999999999,"15min":2.4039999999999999},"min":{"1min":0.188,"5min":0.188,"15min":0.188},"max":{"1min":4.7949999999999999,"5min":27.962,"15min":27.962},"last":0.56299999999999994},{"interface":"front","average":{"1min":2.8519999999999999,"5min":3.794,"15min":2.2709999999999999},"min":{"1min":0.20499999999999999,"5min":0.19800000000000001,"15min":0.19800000000000001},"max":{"1min":10.497,"5min":28.058,"15min":28.058},"last":0.89200000000000002}]},{"osd":1,"last update":"Sat Sep 23 21:02:32 2023","interfaces":[{"interface":"back","average":{"1min":5.2779999999999996,"5min":3.8439999999999999,"15min":2.371},"min":{"1min":0.23999999999999999,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":74.093000000000004,"5min":74.093000000000004,"15min":74.093000000000004},"last":3.6459999999999999},{"interface":"front","average":{"1min":5.1210000000000004,"5min":3.887,"15min":2.4969999999999999},"min":{"1min":0.27700000000000002,"5min":0.23400000000000001,"15min":0.218},"max":{"1min":76.527000000000001,"5min":76.527000000000001,"15min":76.527000000000001},"last":0.875}]},{"osd":2,"last update":"Sat Sep 23 21:02:32 2023","interfaces":[{"interface":"back","average":{"1min":2.101,"5min":4.1749999999999998,"15min":2.399},"min":{"1min":0.23599999999999999,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":6.5549999999999997,"5min":28.251999999999999,"15min":28.251999999999999},"last":2.3759999999999999},{"interface":"front","average":{"1min":2.097,"5min":3.3199999999999998,"15min":2.3759999999999999},"min":{"1min":0.22,"5min":0.22,"15min":0.22},"max":{"1min":12.68,"5min":16.213999999999999,"15min":25.305},"last":2.3889999999999998}]},{"osd":4,"last update":"Sat Sep 23 21:02:32 2023","interfaces":[{"interface":"back","average":{"1min":1.75,"5min":2.5680000000000001,"15min":1.6519999999999999},"min":{"1min":0.29399999999999998,"5min":0.29399999999999998,"15min":0.29299999999999998},"max":{"1min":7.6820000000000004,"5min":18.239000000000001,"15min":18.239000000000001},"last":0.51600000000000001},{"interface":"front","average":{"1min":1.5469999999999999,"5min":2.0030000000000001,"15min":1.6559999999999999},"min":{"1min":0.29499999999999998,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":6.0060000000000002,"5min":14.808,"15min":22.853000000000002},"last":0.59499999999999997}]},{"osd":5,"last update":"Sat Sep 23 21:02:32 2023","interfaces":[{"interface":"back","average":{"1min":1.7410000000000001,"5min":2.6739999999999999,"15min":1.6679999999999999},"min":{"1min":0.32900000000000001,"5min":0.32900000000000001,"15min":0.32500000000000001},"max":{"1min":8.0459999999999994,"5min":18.324999999999999,"15min":18.324999999999999},"last":0.48299999999999998},{"interface":"front","average":{"1min":2.4860000000000002,"5min":2.9279999999999999,"15min":1.746},"min":{"1min":0.40899999999999997,"5min":0.33300000000000002,"15min":0.33100000000000002},"max":{"1min":10.558,"5min":13.529999999999999,"15min":13.529999999999999},"last":0.93500000000000005}]},{"osd":6,"last update":"Sat Sep 23 21:02:56 2023","interfaces":[{"interface":"back","average":{"1min":2.6099999999999999,"5min":2.2370000000000001,"15min":1.393},"min":{"1min":0.40100000000000002,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":10.994999999999999,"5min":14.833,"15min":14.833},"last":0.629},{"interface":"front","average":{"1min":2.7709999999999999,"5min":2.7440000000000002,"15min":1.8740000000000001},"min":{"1min":0.373,"5min":0.373,"15min":0.373},"max":{"1min":10.664,"5min":13.576000000000001,"15min":13.576000000000001},"last":0.96199999999999997}]},{"osd":7,"last update":"Sat Sep 23 21:02:32 2023","interfaces":[{"interface":"back","average":{"1min":2.3420000000000001,"5min":2.7240000000000002,"15min":1.8440000000000001},"min":{"1min":0.40600000000000003,"5min":0.39200000000000002,"15min":0.36099999999999999},"max":{"1min":10.715999999999999,"5min":18.289999999999999,"15min":18.289999999999999},"last":0.53800000000000003},{"interface":"front","average":{"1min":2.8959999999999999,"5min":3.0880000000000001,"15min":1.8440000000000001},"min":{"1min":0.46500000000000002,"5min":0.38900000000000001,"15min":0.379},"max":{"1min":11.798999999999999,"5min":13.603999999999999,"15min":13.603999999999999},"last":0.98899999999999999}]}]},{"osd":4,"up_from":31,"seq":133143986462,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2255720,"kb_used_data":2071968,"kb_used_omap":0,"kb_used_meta":183744,"kb_avail":91489432,"statfs":{"total":95995035648,"available":93685178368,"internally_reserved":0,"allocated":2121695232,"data_stored":2120791355,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":188153856},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:03:00 2023","interfaces":[{"interface":"back","average":{"1min":1.181,"5min":1.522,"15min":1.2529999999999999},"min":{"1min":0.251,"5min":0.218,"15min":0.214},"max":{"1min":6.4139999999999997,"5min":9.0749999999999993,"15min":11.784000000000001},"last":0.69099999999999995},{"interface":"front","average":{"1min":1.306,"5min":1.6220000000000001,"15min":1.405},"min":{"1min":0.30599999999999999,"5min":0.27700000000000002,"15min":0.25800000000000001},"max":{"1min":6.4249999999999998,"5min":9.1029999999999998,"15min":11.766},"last":0.84799999999999998}]},{"osd":1,"last update":"Sat Sep 23 21:02:24 2023","interfaces":[{"interface":"back","average":{"1min":0.94899999999999995,"5min":1.696,"15min":1.2969999999999999},"min":{"1min":0.255,"5min":0.246,"15min":0.23599999999999999},"max":{"1min":5.0229999999999997,"5min":10.153,"15min":10.153},"last":11.403},{"interface":"front","average":{"1min":1.0940000000000001,"5min":2.2280000000000002,"15min":1.3700000000000001},"min":{"1min":0.309,"5min":0.26100000000000001,"15min":0.23200000000000001},"max":{"1min":7.3209999999999997,"5min":12.292999999999999,"15min":12.292999999999999},"last":0.72799999999999998}]},{"osd":2,"last update":"Sat Sep 23 21:03:00 2023","interfaces":[{"interface":"back","average":{"1min":2.1560000000000001,"5min":2.0409999999999999,"15min":1.488},"min":{"1min":0.309,"5min":0.309,"15min":0.29099999999999998},"max":{"1min":10.635,"5min":10.635,"15min":15.288},"last":10.906000000000001},{"interface":"front","average":{"1min":1.5129999999999999,"5min":1.792,"15min":1.5449999999999999},"min":{"1min":0.317,"5min":0.28599999999999998,"15min":0.28299999999999997},"max":{"1min":7.609,"5min":12.855,"15min":15.262},"last":8.9329999999999998}]},{"osd":3,"last update":"Sat Sep 23 21:02:24 2023","interfaces":[{"interface":"back","average":{"1min":1.514,"5min":2.3109999999999999,"15min":1.976},"min":{"1min":0.42699999999999999,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":7.1609999999999996,"5min":12.106999999999999,"15min":31.562999999999999},"last":3.306},{"interface":"front","average":{"1min":1.3069999999999999,"5min":2.3149999999999999,"15min":2.0110000000000001},"min":{"1min":0.499,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":7.1429999999999998,"5min":12.477,"15min":31.574000000000002},"last":3.29}]},{"osd":5,"last update":"Sat Sep 23 21:02:43 2023","interfaces":[{"interface":"back","average":{"1min":0.94399999999999995,"5min":0.59499999999999997,"15min":0.55100000000000005},"min":{"1min":0.27200000000000002,"5min":0.251,"15min":0.22700000000000001},"max":{"1min":6.6470000000000002,"5min":6.6470000000000002,"15min":6.6470000000000002},"last":0.81599999999999995},{"interface":"front","average":{"1min":1.03,"5min":0.84299999999999997,"15min":0.68100000000000005},"min":{"1min":0.32400000000000001,"5min":0.27600000000000002,"15min":0.251},"max":{"1min":5.3120000000000003,"5min":5.3120000000000003,"15min":7.7619999999999996},"last":1.0069999999999999}]},{"osd":6,"last update":"Sat Sep 23 21:02:52 2023","interfaces":[{"interface":"back","average":{"1min":1.712,"5min":0.87,"15min":0.64900000000000002},"min":{"1min":0.29399999999999998,"5min":0.27300000000000002,"15min":0.26500000000000001},"max":{"1min":6.6109999999999998,"5min":6.6109999999999998,"15min":6.6109999999999998},"last":0.52800000000000002},{"interface":"front","average":{"1min":1.0289999999999999,"5min":0.73799999999999999,"15min":0.68700000000000006},"min":{"1min":0.36599999999999999,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":4.1870000000000003,"5min":4.1870000000000003,"15min":4.1870000000000003},"last":0.78300000000000003}]},{"osd":7,"last update":"Sat Sep 23 21:02:24 2023","interfaces":[{"interface":"back","average":{"1min":0.61399999999999999,"5min":0.68799999999999994,"15min":0.69899999999999995},"min":{"1min":0.33700000000000002,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":1.248,"5min":3.4940000000000002,"15min":7.383},"last":0.76100000000000001},{"interface":"front","average":{"1min":0.72699999999999998,"5min":0.74199999999999999,"15min":0.70099999999999996},"min":{"1min":0.377,"5min":0.32700000000000001,"15min":0.28199999999999997},"max":{"1min":2.1699999999999999,"5min":5.6980000000000004,"15min":7.8310000000000004},"last":0.97099999999999997}]}]},{"osd":5,"up_from":39,"seq":167503724827,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3545440,"kb_used_data":3353304,"kb_used_omap":0,"kb_used_meta":192128,"kb_avail":90199712,"statfs":{"total":95995035648,"available":92364505088,"internally_reserved":0,"allocated":3433783296,"data_stored":3432883923,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":196739072},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:02:47 2023","interfaces":[{"interface":"back","average":{"1min":2.1619999999999999,"5min":2.181,"15min":1.27},"min":{"1min":0.22500000000000001,"5min":0.22500000000000001,"15min":0.20399999999999999},"max":{"1min":8.3249999999999993,"5min":12.795999999999999,"15min":12.795999999999999},"last":0.22600000000000001},{"interface":"front","average":{"1min":1.7589999999999999,"5min":2.1389999999999998,"15min":1.4199999999999999},"min":{"1min":0.23400000000000001,"5min":0.23400000000000001,"15min":0.223},"max":{"1min":6.6230000000000002,"5min":16.219999999999999,"15min":16.219999999999999},"last":0.441}]},{"osd":1,"last update":"Sat Sep 23 21:02:20 2023","interfaces":[{"interface":"back","average":{"1min":4.8710000000000004,"5min":2.661,"15min":1.585},"min":{"1min":0.26800000000000002,"5min":0.248,"15min":0.23300000000000001},"max":{"1min":62.320999999999998,"5min":62.320999999999998,"15min":62.320999999999998},"last":0.92200000000000004},{"interface":"front","average":{"1min":4.6079999999999997,"5min":2.7040000000000002,"15min":1.6020000000000001},"min":{"1min":0.30499999999999999,"5min":0.27200000000000002,"15min":0.22800000000000001},"max":{"1min":61.722999999999999,"5min":61.722999999999999,"15min":61.722999999999999},"last":1.9470000000000001}]},{"osd":2,"last update":"Sat Sep 23 21:03:01 2023","interfaces":[{"interface":"back","average":{"1min":1.9299999999999999,"5min":1.9590000000000001,"15min":1.3580000000000001},"min":{"1min":0.31900000000000001,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":8.4000000000000004,"5min":10.371,"15min":12.651},"last":1.3049999999999999},{"interface":"front","average":{"1min":2.008,"5min":2.355,"15min":1.5089999999999999},"min":{"1min":0.32700000000000001,"5min":0.27100000000000002,"15min":0.26100000000000001},"max":{"1min":9.7650000000000006,"5min":12.300000000000001,"15min":12.300000000000001},"last":0.59399999999999997}]},{"osd":3,"last update":"Sat Sep 23 21:02:20 2023","interfaces":[{"interface":"back","average":{"1min":6.7009999999999996,"5min":3.8119999999999998,"15min":2.1030000000000002},"min":{"1min":0.33500000000000002,"5min":0.33000000000000002,"15min":0.28599999999999998},"max":{"1min":61.667999999999999,"5min":61.667999999999999,"15min":61.667999999999999},"last":3.782},{"interface":"front","average":{"1min":7.0460000000000003,"5min":3.2389999999999999,"15min":1.78},"min":{"1min":0.377,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":62.276000000000003,"5min":62.276000000000003,"15min":62.276000000000003},"last":6.8810000000000002}]},{"osd":4,"last update":"Sat Sep 23 21:02:37 2023","interfaces":[{"interface":"back","average":{"1min":0.96399999999999997,"5min":0.67400000000000004,"15min":0.59699999999999998},"min":{"1min":0.25700000000000001,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":3.9279999999999999,"5min":4.1520000000000001,"15min":4.1520000000000001},"last":0.41699999999999998},{"interface":"front","average":{"1min":1.288,"5min":0.76000000000000001,"15min":0.64600000000000002},"min":{"1min":0.28000000000000003,"5min":0.27400000000000002,"15min":0.23599999999999999},"max":{"1min":8.4350000000000005,"5min":8.4350000000000005,"15min":8.4350000000000005},"last":0.33000000000000002}]},{"osd":6,"last update":"Sat Sep 23 21:02:47 2023","interfaces":[{"interface":"back","average":{"1min":1.494,"5min":0.86699999999999999,"15min":0.67900000000000005},"min":{"1min":0.32200000000000001,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":11.128,"5min":11.128,"15min":11.128},"last":0.622},{"interface":"front","average":{"1min":1.619,"5min":0.92100000000000004,"15min":0.80400000000000005},"min":{"1min":0.36799999999999999,"5min":0.36299999999999999,"15min":0.33200000000000002},"max":{"1min":11.112,"5min":11.112,"15min":11.112},"last":0.47299999999999998}]},{"osd":7,"last update":"Sat Sep 23 21:02:20 2023","interfaces":[{"interface":"back","average":{"1min":1.1819999999999999,"5min":0.83799999999999997,"15min":0.71999999999999997},"min":{"1min":0.36099999999999999,"5min":0.316,"15min":0.30199999999999999},"max":{"1min":4.1310000000000002,"5min":4.4240000000000004,"15min":8.5649999999999995},"last":0.64700000000000002},{"interface":"front","average":{"1min":0.90100000000000002,"5min":0.74099999999999999,"15min":0.70199999999999996},"min":{"1min":0.35299999999999998,"5min":0.33900000000000002,"15min":0.27700000000000002},"max":{"1min":4.6479999999999997,"5min":4.6479999999999997,"15min":4.6479999999999997},"last":0.45900000000000002}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":2537881600,"data_stored":2537881600,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":2865168384,"data_stored":2865168384,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":2519928832,"data_stored":2519924760,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":2458058752,"data_stored":2458058752,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":2114125824,"data_stored":2114125824,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":3327594496,"data_stored":3327590424,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":2052915200,"data_stored":2052915200,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":2110853120,"data_stored":2110849048,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T21:03:17.954 INFO:tasks.cephadm.ceph_manager.ceph:increase pool size by 10 2023-09-23T21:03:17.954 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph osd pool set unique_pool_2 pg_num 26 2023-09-23T21:03:17.982 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:03:18.040 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:03:18.044 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:03:18.044 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:03:18.145 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:03:18.178 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:03:18.220 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:03:18.223 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:03:18.224 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:03:18.324 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:03:18.366 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:03:18.448 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:03:18.451 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:03:18.451 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:03:18.483 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:03:18 smithi026 ceph-mon[179688]: pgmap v742: 109 pgs: 109 active+clean; 6.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 790 op/s 2023-09-23T21:03:18.483 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:03:18 smithi026 ceph-mon[179688]: from='client.35607 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T21:03:18.483 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:03:18 smithi026 ceph-mon[179688]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:03:18.483 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:03:18 smithi026 ceph-mon[179688]: failed to encode map e279 with expected crc 2023-09-23T21:03:18.483 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:03:18 smithi026 ceph-mon[179688]: failed to encode map e279 with expected crc 2023-09-23T21:03:18.483 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:03:18 smithi026 ceph-mon[179688]: failed to encode map e279 with expected crc 2023-09-23T21:03:18.483 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:03:18 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T21:03:18.483 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:03:18 smithi026 ceph-mon[179688]: osdmap e279: 8 total, 8 up, 8 in 2023-09-23T21:03:18.483 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:03:18 smithi026 ceph-mon[179688]: failed to encode map e279 with expected crc 2023-09-23T21:03:18.490 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:03:18 smithi026 ceph-mon[175151]: pgmap v742: 109 pgs: 109 active+clean; 6.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 790 op/s 2023-09-23T21:03:18.490 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:03:18 smithi026 ceph-mon[175151]: from='client.35607 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T21:03:18.490 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:03:18 smithi026 ceph-mon[175151]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:03:18.490 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:03:18 smithi026 ceph-mon[175151]: failed to encode map e279 with expected crc 2023-09-23T21:03:18.490 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:03:18 smithi026 ceph-mon[175151]: failed to encode map e279 with expected crc 2023-09-23T21:03:18.490 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:03:18 smithi026 ceph-mon[175151]: failed to encode map e279 with expected crc 2023-09-23T21:03:18.490 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:03:18 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T21:03:18.490 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:03:18 smithi026 ceph-mon[175151]: osdmap e279: 8 total, 8 up, 8 in 2023-09-23T21:03:18.490 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:03:18 smithi026 ceph-mon[175151]: failed to encode map e279 with expected crc 2023-09-23T21:03:18.552 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:03:18.583 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:03:18.601 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:03:18.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:03:18 smithi070 ceph-mon[168282]: pgmap v742: 109 pgs: 109 active+clean; 6.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 790 op/s 2023-09-23T21:03:18.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:03:18 smithi070 ceph-mon[168282]: from='client.35607 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T21:03:18.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:03:18 smithi070 ceph-mon[168282]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:03:18.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:03:18 smithi070 ceph-mon[168282]: failed to encode map e279 with expected crc 2023-09-23T21:03:18.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:03:18 smithi070 ceph-mon[168282]: failed to encode map e279 with expected crc 2023-09-23T21:03:18.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:03:18 smithi070 ceph-mon[168282]: failed to encode map e279 with expected crc 2023-09-23T21:03:18.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:03:18 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T21:03:18.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:03:18 smithi070 ceph-mon[168282]: osdmap e279: 8 total, 8 up, 8 in 2023-09-23T21:03:18.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:03:18 smithi070 ceph-mon[168282]: failed to encode map e279 with expected crc 2023-09-23T21:03:18.655 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:03:18.663 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:03:18.663 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:03:18.764 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:03:18.868 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:03:18.928 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:03:18.944 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:03:18.945 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:03:19.046 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:03:19.107 INFO:teuthology.orchestra.run.smithi026.stdout:ERROR: (22) Invalid argument 2023-09-23T21:03:19.107 INFO:teuthology.orchestra.run.smithi026.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T21:03:19.126 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:03:19.130 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:03:19.206 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:03:19.218 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:03:19.219 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:03:19.320 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:03:19.355 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:03:19.402 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:03:19.410 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:03:19.411 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:03:19.512 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:03:19.586 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:03:19.661 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:03:19.671 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:03:19.671 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:03:19.703 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 21:03:19 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[164029]: ::ffff:172.21.15.70 - - [23/Sep/2023:21:03:19] "GET /metrics HTTP/1.1" 200 36294 "" "Prometheus/2.43.0" 2023-09-23T21:03:19.772 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:03:19.848 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:03:20.020 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:03:20.030 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:03:20.030 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:03:20.104 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_historic_ops 2023-09-23T21:03:20.131 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:03:20.238 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:03:20.531 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T21:03:20.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:03:20 smithi070 ceph-mon[168282]: pgmap v744: 109 pgs: 109 active+clean; 6.1 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 779 op/s 2023-09-23T21:03:20.625 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:03:20 smithi026 ceph-mon[175151]: pgmap v744: 109 pgs: 109 active+clean; 6.1 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 779 op/s 2023-09-23T21:03:20.627 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:03:20 smithi026 ceph-mon[179688]: pgmap v744: 109 pgs: 109 active+clean; 6.1 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 779 op/s 2023-09-23T21:03:22.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:03:22 smithi026 ceph-mon[175151]: pgmap v745: 109 pgs: 109 active+clean; 5.9 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 762 op/s 2023-09-23T21:03:22.633 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:03:22 smithi026 ceph-mon[179688]: pgmap v745: 109 pgs: 109 active+clean; 5.9 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 762 op/s 2023-09-23T21:03:22.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:03:22 smithi070 ceph-mon[168282]: pgmap v745: 109 pgs: 109 active+clean; 5.9 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 762 op/s 2023-09-23T21:03:23.328 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:03:23.333 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:03:23.333 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:03:23.434 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:03:23.552 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:03:23.824 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:03:23.832 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:03:23.833 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:03:23.934 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:03:23.984 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:03:24.041 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:03:24.048 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:03:24.048 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:03:24.149 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:03:24.191 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:03:24.242 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:03:24.256 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:03:24.256 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:03:24.357 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:03:24.407 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:03:24.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:03:24 smithi070 ceph-mon[168282]: pgmap v746: 109 pgs: 109 active+clean; 5.8 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 763 op/s 2023-09-23T21:03:24.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:03:24 smithi026 ceph-mon[179688]: pgmap v746: 109 pgs: 109 active+clean; 5.8 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 763 op/s 2023-09-23T21:03:24.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:03:24 smithi026 ceph-mon[175151]: pgmap v746: 109 pgs: 109 active+clean; 5.8 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 763 op/s 2023-09-23T21:03:24.968 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:03:25.624 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T21:03:25.630 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:03:25.630 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:03:25.731 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:03:25.773 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:03:25.826 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:03:25.837 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:03:25.837 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:03:25.938 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:03:26.002 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:03:26.067 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:03:26.076 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:03:26.077 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:03:26.178 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:03:26.214 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:03:26.266 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0: {} 2023-09-23T21:03:26.274 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:03:26.277 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:03:26.278 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:03:26.310 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:03:26.310 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1: {} 2023-09-23T21:03:26.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:03:26 smithi070 ceph-mon[168282]: pgmap v747: 109 pgs: 109 active+clean; 5.7 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 776 op/s 2023-09-23T21:03:26.378 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:03:26.406 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:03:26.413 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2: {} 2023-09-23T21:03:26.438 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:03:26 smithi026 ceph-mon[175151]: pgmap v747: 109 pgs: 109 active+clean; 5.7 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 776 op/s 2023-09-23T21:03:26.439 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:03:26 smithi026 ceph-mon[179688]: pgmap v747: 109 pgs: 109 active+clean; 5.7 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 776 op/s 2023-09-23T21:03:26.455 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:03:26.462 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:03:26.462 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:03:26.468 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3: {} 2023-09-23T21:03:26.514 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4: {} 2023-09-23T21:03:26.556 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T21:03:26.563 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:03:26.605 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6: {} 2023-09-23T21:03:26.605 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:03:26.654 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T21:03:26.660 INFO:teuthology.orchestra.run.smithi026.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T21:03:26.660 INFO:teuthology.orchestra.run.smithi026.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T21:03:26.660 INFO:teuthology.orchestra.run.smithi026.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T21:03:26.660 INFO:teuthology.orchestra.run.smithi026.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T21:03:26.660 INFO:teuthology.orchestra.run.smithi026.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T21:03:26.660 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T21:03:26.660 INFO:teuthology.orchestra.run.smithi026.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T21:03:26.660 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T21:03:26.661 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:03:26.665 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:03:26.665 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:03:26.766 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:03:26.867 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:03:27.202 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:03:27 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/1608972529' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:03:27.202 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:03:27 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/2199153893' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pg_num", "val": "26"}]: dispatch 2023-09-23T21:03:27.202 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:03:27 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pg_num", "val": "26"}]: dispatch 2023-09-23T21:03:27.225 INFO:teuthology.orchestra.run.smithi026.stderr:set pool 8 pg_num to 26 2023-09-23T21:03:27.243 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:03:27.259 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:03:27.260 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:03:27.360 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:03:27.399 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:03:27.449 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:03:27.454 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:03:27 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/1608972529' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:03:27.454 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:03:27 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/2199153893' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pg_num", "val": "26"}]: dispatch 2023-09-23T21:03:27.454 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:03:27 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pg_num", "val": "26"}]: dispatch 2023-09-23T21:03:27.455 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:03:27.455 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:03:27.497 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:03:27 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/1608972529' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:03:27.497 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:03:27 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/2199153893' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pg_num", "val": "26"}]: dispatch 2023-09-23T21:03:27.497 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:03:27 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pg_num", "val": "26"}]: dispatch 2023-09-23T21:03:27.556 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:03:27.635 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:03:28.030 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:03:28.034 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:03:28.035 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:03:28.136 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:03:28.233 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:03:28.288 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:03:28 smithi026 ceph-mon[175151]: pgmap v748: 109 pgs: 109 active+clean; 5.7 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 802 op/s 2023-09-23T21:03:28.289 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:03:28 smithi026 ceph-mon[175151]: failed to encode map e280 with expected crc 2023-09-23T21:03:28.289 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:03:28 smithi026 ceph-mon[175151]: failed to encode map e280 with expected crc 2023-09-23T21:03:28.289 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:03:28 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pg_num", "val": "26"}]': finished 2023-09-23T21:03:28.289 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:03:28 smithi026 ceph-mon[175151]: osdmap e280: 8 total, 8 up, 8 in 2023-09-23T21:03:28.289 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:03:28 smithi026 ceph-mon[175151]: failed to encode map e280 with expected crc 2023-09-23T21:03:28.289 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:03:28 smithi026 ceph-mon[179688]: pgmap v748: 109 pgs: 109 active+clean; 5.7 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 802 op/s 2023-09-23T21:03:28.289 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:03:28 smithi026 ceph-mon[179688]: failed to encode map e280 with expected crc 2023-09-23T21:03:28.289 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:03:28 smithi026 ceph-mon[179688]: failed to encode map e280 with expected crc 2023-09-23T21:03:28.290 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:03:28 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pg_num", "val": "26"}]': finished 2023-09-23T21:03:28.290 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:03:28 smithi026 ceph-mon[179688]: osdmap e280: 8 total, 8 up, 8 in 2023-09-23T21:03:28.290 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:03:28 smithi026 ceph-mon[179688]: failed to encode map e280 with expected crc 2023-09-23T21:03:28.572 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:03:28.579 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:03:28.579 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:03:28.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:03:28 smithi070 ceph-mon[168282]: pgmap v748: 109 pgs: 109 active+clean; 5.7 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 802 op/s 2023-09-23T21:03:28.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:03:28 smithi070 ceph-mon[168282]: failed to encode map e280 with expected crc 2023-09-23T21:03:28.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:03:28 smithi070 ceph-mon[168282]: failed to encode map e280 with expected crc 2023-09-23T21:03:28.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:03:28 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pg_num", "val": "26"}]': finished 2023-09-23T21:03:28.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:03:28 smithi070 ceph-mon[168282]: osdmap e280: 8 total, 8 up, 8 in 2023-09-23T21:03:28.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:03:28 smithi070 ceph-mon[168282]: failed to encode map e280 with expected crc 2023-09-23T21:03:28.680 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:03:28.757 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:03:28.850 INFO:teuthology.orchestra.run.smithi026.stdout:{ 2023-09-23T21:03:28.850 INFO:teuthology.orchestra.run.smithi026.stdout: "size": 20, 2023-09-23T21:03:28.850 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 600, 2023-09-23T21:03:28.850 INFO:teuthology.orchestra.run.smithi026.stdout: "ops": [ 2023-09-23T21:03:28.850 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:28.850 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34797.0:114968 6.f 6:fe20b540:::benchmark_data_smithi026_223868_object13842:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e131)", 2023-09-23T21:03:28.850 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T20:54:12.903762+0000", 2023-09-23T21:03:28.851 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 555.93911677999995, 2023-09-23T21:03:28.851 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1643464619999999, 2023-09-23T21:03:28.851 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:03:28.851 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "reached pg", 2023-09-23T21:03:28.851 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T21:03:28.851 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34797", 2023-09-23T21:03:28.851 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T21:03:28.851 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 114968 2023-09-23T21:03:28.851 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:28.851 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:03:28.852 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:28.852 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:03:28.852 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:12.903762+0000", 2023-09-23T21:03:28.852 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:03:28.852 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:28.852 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:28.852 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:03:28.852 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:12.903762+0000", 2023-09-23T21:03:28.852 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:03:28.852 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:28.852 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:28.853 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:03:28.853 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:12.903760+0000", 2023-09-23T21:03:28.853 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999981 2023-09-23T21:03:28.853 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:28.853 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:28.853 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:03:28.853 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:12.903769+0000", 2023-09-23T21:03:28.853 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.772e-06 2023-09-23T21:03:28.853 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:28.853 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:28.853 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:03:28.854 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:12.903784+0000", 2023-09-23T21:03:28.854 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.5063e-05 2023-09-23T21:03:28.854 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:28.854 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:28.854 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:03:28.854 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:12.903794+0000", 2023-09-23T21:03:28.854 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0274e-05 2023-09-23T21:03:28.854 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:28.854 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:28.854 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:03:28.855 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:12.903882+0000", 2023-09-23T21:03:28.855 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.7639999999999994e-05 2023-09-23T21:03:28.855 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:28.855 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:28.855 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:03:28.855 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:12.904022+0000", 2023-09-23T21:03:28.855 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000140724 2023-09-23T21:03:28.855 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:28.855 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:28.855 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 3,6", 2023-09-23T21:03:28.856 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:12.904139+0000", 2023-09-23T21:03:28.856 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000116762 2023-09-23T21:03:28.856 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:28.856 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:28.856 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T21:03:28.856 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:12.907761+0000", 2023-09-23T21:03:28.856 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0036214770000000001 2023-09-23T21:03:28.856 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:28.856 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:28.856 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:03:28.857 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:12.921765+0000", 2023-09-23T21:03:28.857 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.014004312 2023-09-23T21:03:28.857 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:28.857 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:28.857 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:03:28.857 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:12.950178+0000", 2023-09-23T21:03:28.857 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.028413560000000001 2023-09-23T21:03:28.857 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:28.857 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:28.857 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T21:03:28.857 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:12.950192+0000", 2023-09-23T21:03:28.857 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.3509000000000001e-05 2023-09-23T21:03:28.858 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:28.858 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:28.858 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:03:28.858 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:14.018502+0000", 2023-09-23T21:03:28.858 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0683096089999999 2023-09-23T21:03:28.858 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:28.858 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:28.858 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for missing object", 2023-09-23T21:03:28.858 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:14.018606+0000", 2023-09-23T21:03:28.858 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000104041 2023-09-23T21:03:28.858 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:28.858 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:28.859 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:03:28.859 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:14.068012+0000", 2023-09-23T21:03:28.859 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.049406377000000001 2023-09-23T21:03:28.859 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:28.859 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:28.859 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T21:03:28.859 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:14.068108+0000", 2023-09-23T21:03:28.859 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.6364000000000001e-05 2023-09-23T21:03:28.859 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:03:28.859 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:03:28.859 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:03:28.859 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:28.859 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:28.860 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34797.0:114973 6.f 6:f527467a:::benchmark_data_smithi026_223868_object13847:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e131)", 2023-09-23T21:03:28.860 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T20:54:12.913308+0000", 2023-09-23T21:03:28.860 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 555.92957103399999, 2023-09-23T21:03:28.860 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1662446850000001, 2023-09-23T21:03:28.860 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:03:28.860 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "reached pg", 2023-09-23T21:03:28.860 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T21:03:28.860 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34797", 2023-09-23T21:03:28.860 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T21:03:28.860 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 114973 2023-09-23T21:03:28.860 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:28.861 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:03:28.861 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:28.861 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:03:28.861 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:12.913308+0000", 2023-09-23T21:03:28.861 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:03:28.861 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:28.861 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:28.861 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:03:28.861 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:12.913308+0000", 2023-09-23T21:03:28.861 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:03:28.861 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:28.861 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:28.862 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:03:28.862 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:12.913306+0000", 2023-09-23T21:03:28.862 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T21:03:28.862 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:28.862 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:28.862 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:03:28.862 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:12.913314+0000", 2023-09-23T21:03:28.862 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.9820000000000008e-06 2023-09-23T21:03:28.862 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:28.862 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:28.862 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:03:28.862 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:12.913326+0000", 2023-09-23T21:03:28.863 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1749999999999999e-05 2023-09-23T21:03:28.863 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:28.863 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:28.863 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:03:28.863 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:12.913334+0000", 2023-09-23T21:03:28.863 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.7640000000000005e-06 2023-09-23T21:03:28.864 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:28.864 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:28.864 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:03:28.864 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:12.913407+0000", 2023-09-23T21:03:28.864 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.3134999999999997e-05 2023-09-23T21:03:28.864 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:28.864 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:28.864 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:03:28.864 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:12.913576+0000", 2023-09-23T21:03:28.864 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00016879199999999999 2023-09-23T21:03:28.865 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:28.865 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:28.865 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 3,6", 2023-09-23T21:03:28.865 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:12.913707+0000", 2023-09-23T21:03:28.865 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000131168 2023-09-23T21:03:28.865 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:28.865 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:28.865 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T21:03:28.865 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:12.918380+0000", 2023-09-23T21:03:28.865 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0046727549999999998 2023-09-23T21:03:28.865 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:28.865 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:28.865 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:03:28.866 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:12.927220+0000", 2023-09-23T21:03:28.866 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0088402120000000001 2023-09-23T21:03:28.866 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:28.866 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:28.866 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:03:28.866 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:12.950224+0000", 2023-09-23T21:03:28.866 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.023004 2023-09-23T21:03:28.866 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:28.866 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:28.866 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T21:03:28.866 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:12.950233+0000", 2023-09-23T21:03:28.866 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.4199999999999996e-06 2023-09-23T21:03:28.867 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:28.867 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:28.867 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:03:28.867 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:14.068138+0000", 2023-09-23T21:03:28.867 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.117904504 2023-09-23T21:03:28.867 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:28.867 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:28.867 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for missing object", 2023-09-23T21:03:28.867 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:14.068256+0000", 2023-09-23T21:03:28.867 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000117847 2023-09-23T21:03:28.867 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:28.868 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:28.868 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:03:28.868 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:14.079477+0000", 2023-09-23T21:03:28.868 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.011221287 2023-09-23T21:03:28.868 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:28.868 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:28.868 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T21:03:28.868 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:14.079552+0000", 2023-09-23T21:03:28.868 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.5480000000000002e-05 2023-09-23T21:03:28.868 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:03:28.868 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:03:28.868 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:03:28.869 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:28.869 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:28.869 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34797.0:114979 6.f 6:f96f8492:::benchmark_data_smithi026_223868_object13853:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e131)", 2023-09-23T21:03:28.869 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T20:54:12.932449+0000", 2023-09-23T21:03:28.869 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 555.91042976699998, 2023-09-23T21:03:28.869 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.152965391, 2023-09-23T21:03:28.869 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:03:28.869 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "reached pg", 2023-09-23T21:03:28.869 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T21:03:28.869 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34797", 2023-09-23T21:03:28.869 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T21:03:28.869 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 114979 2023-09-23T21:03:28.869 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:28.870 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:03:28.870 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:28.870 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:03:28.870 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:12.932449+0000", 2023-09-23T21:03:28.870 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:03:28.870 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:28.870 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:28.870 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:03:28.870 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:12.932449+0000", 2023-09-23T21:03:28.870 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:03:28.870 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:28.870 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:28.871 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:03:28.871 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:12.932447+0000", 2023-09-23T21:03:28.871 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999981 2023-09-23T21:03:28.871 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:28.871 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:28.871 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:03:28.871 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:12.932457+0000", 2023-09-23T21:03:28.871 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0331e-05 2023-09-23T21:03:28.871 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:28.871 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:28.871 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:03:28.871 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:12.932489+0000", 2023-09-23T21:03:28.871 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.2135999999999999e-05 2023-09-23T21:03:28.872 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:28.872 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:28.872 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:03:28.872 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:12.932501+0000", 2023-09-23T21:03:28.872 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1969e-05 2023-09-23T21:03:28.872 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:28.872 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:28.872 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:03:28.872 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:12.932581+0000", 2023-09-23T21:03:28.872 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.9510000000000003e-05 2023-09-23T21:03:28.872 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:28.872 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:28.872 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:03:28.872 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:12.932696+0000", 2023-09-23T21:03:28.873 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00011554599999999999 2023-09-23T21:03:28.873 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:28.873 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:28.873 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 3,6", 2023-09-23T21:03:28.873 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:12.932783+0000", 2023-09-23T21:03:28.873 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.6712000000000006e-05 2023-09-23T21:03:28.873 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:28.873 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:28.873 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:03:28.873 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:12.950261+0000", 2023-09-23T21:03:28.873 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.017477724 2023-09-23T21:03:28.873 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:28.873 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:28.874 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T21:03:28.874 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:12.950270+0000", 2023-09-23T21:03:28.874 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.2900000000000008e-06 2023-09-23T21:03:28.874 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:28.874 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:28.874 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:03:28.874 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:14.068290+0000", 2023-09-23T21:03:28.874 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1180193949999999 2023-09-23T21:03:28.874 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:28.874 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:28.874 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for missing object", 2023-09-23T21:03:28.874 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:14.068456+0000", 2023-09-23T21:03:28.875 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000166627 2023-09-23T21:03:28.875 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:28.875 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:28.875 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:03:28.875 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:14.085291+0000", 2023-09-23T21:03:28.875 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.016835249 2023-09-23T21:03:28.875 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:28.875 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:28.875 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T21:03:28.875 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:14.085414+0000", 2023-09-23T21:03:28.875 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000122853 2023-09-23T21:03:28.875 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:03:28.876 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:03:28.876 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:03:28.876 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:28.876 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:28.876 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34797.0:114981 6.f 6:f98e4163:::benchmark_data_smithi026_223868_object13855:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e131)", 2023-09-23T21:03:28.876 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T20:54:12.932519+0000", 2023-09-23T21:03:28.876 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 555.91035997400002, 2023-09-23T21:03:28.876 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.154672455, 2023-09-23T21:03:28.876 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:03:28.876 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "reached pg", 2023-09-23T21:03:28.876 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T21:03:28.876 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34797", 2023-09-23T21:03:28.876 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T21:03:28.877 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 114981 2023-09-23T21:03:28.877 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:28.877 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:03:28.877 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:28.877 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:03:28.877 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:12.932519+0000", 2023-09-23T21:03:28.877 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:03:28.877 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:28.877 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:28.877 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:03:28.877 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:12.932519+0000", 2023-09-23T21:03:28.877 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:03:28.878 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:28.878 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:28.878 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:03:28.878 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:12.932518+0000", 2023-09-23T21:03:28.878 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T21:03:28.878 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:28.878 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:28.878 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:03:28.878 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:12.932522+0000", 2023-09-23T21:03:28.878 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.5219999999999999e-06 2023-09-23T21:03:28.878 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:28.878 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:28.878 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:03:28.879 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:12.932529+0000", 2023-09-23T21:03:28.879 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.8940000000000004e-06 2023-09-23T21:03:28.879 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:28.879 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:28.879 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:03:28.879 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:12.932534+0000", 2023-09-23T21:03:28.879 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.2190000000000003e-06 2023-09-23T21:03:28.879 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:28.879 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:28.879 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:03:28.879 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:12.936410+0000", 2023-09-23T21:03:28.879 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0038765879999999998 2023-09-23T21:03:28.879 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:28.880 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:28.880 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:03:28.880 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:12.936579+0000", 2023-09-23T21:03:28.880 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000168844 2023-09-23T21:03:28.880 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:28.880 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:28.880 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 3,6", 2023-09-23T21:03:28.880 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:12.936731+0000", 2023-09-23T21:03:28.880 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00015202799999999999 2023-09-23T21:03:28.880 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:28.880 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:28.880 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:03:28.881 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:12.950298+0000", 2023-09-23T21:03:28.881 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.013566686 2023-09-23T21:03:28.881 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:28.881 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:28.881 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T21:03:28.881 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:12.950307+0000", 2023-09-23T21:03:28.881 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.8610000000000004e-06 2023-09-23T21:03:28.881 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:28.881 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:28.881 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:03:28.881 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:14.068540+0000", 2023-09-23T21:03:28.881 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1182336260000001 2023-09-23T21:03:28.881 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:28.882 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:28.882 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for missing object", 2023-09-23T21:03:28.882 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:14.068707+0000", 2023-09-23T21:03:28.882 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00016695499999999999 2023-09-23T21:03:28.882 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:28.882 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:28.882 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:03:28.882 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:14.087135+0000", 2023-09-23T21:03:28.882 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.018427507999999999 2023-09-23T21:03:28.882 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:28.882 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:28.882 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T21:03:28.883 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:14.087191+0000", 2023-09-23T21:03:28.883 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.6329000000000002e-05 2023-09-23T21:03:28.883 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:03:28.883 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:03:28.883 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:03:28.883 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:28.883 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:28.883 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34797.0:115000 6.f 6:feaa8694:::benchmark_data_smithi026_223868_object13874:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e132)", 2023-09-23T21:03:28.883 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T20:54:12.985062+0000", 2023-09-23T21:03:28.883 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 555.85781624200001, 2023-09-23T21:03:28.883 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.100759915, 2023-09-23T21:03:28.883 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:03:28.883 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:03:28.884 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T21:03:28.884 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34797", 2023-09-23T21:03:28.884 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T21:03:28.884 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 115000 2023-09-23T21:03:28.884 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:28.884 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:03:28.884 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:28.884 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:03:28.884 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:12.985062+0000", 2023-09-23T21:03:28.884 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:03:28.884 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:28.884 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:28.885 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:03:28.885 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:12.985062+0000", 2023-09-23T21:03:28.885 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:03:28.885 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:28.885 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:28.885 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:03:28.885 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:12.985060+0000", 2023-09-23T21:03:28.885 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999976 2023-09-23T21:03:28.885 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:28.885 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:28.885 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:03:28.885 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:12.985071+0000", 2023-09-23T21:03:28.886 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0922e-05 2023-09-23T21:03:28.886 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:28.886 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:28.886 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:03:28.886 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:12.985091+0000", 2023-09-23T21:03:28.886 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.0228e-05 2023-09-23T21:03:28.886 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:28.886 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:28.886 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:03:28.886 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:12.985103+0000", 2023-09-23T21:03:28.886 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1773000000000001e-05 2023-09-23T21:03:28.887 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:28.887 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:28.887 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:03:28.887 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:12.986211+0000", 2023-09-23T21:03:28.887 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0011084770000000001 2023-09-23T21:03:28.887 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:28.887 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:28.887 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T21:03:28.887 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:12.986223+0000", 2023-09-23T21:03:28.887 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1134999999999999e-05 2023-09-23T21:03:28.887 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:28.888 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:28.888 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:03:28.888 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:14.068739+0000", 2023-09-23T21:03:28.888 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0825159499999999 2023-09-23T21:03:28.888 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:28.888 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:28.888 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:03:28.888 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:14.068895+0000", 2023-09-23T21:03:28.888 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00015601699999999999 2023-09-23T21:03:28.888 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:28.888 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:28.888 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 3,4", 2023-09-23T21:03:28.889 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:14.069072+0000", 2023-09-23T21:03:28.889 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00017741799999999999 2023-09-23T21:03:28.889 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:28.889 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:28.889 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T21:03:28.889 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:14.079431+0000", 2023-09-23T21:03:28.889 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.010358539999999999 2023-09-23T21:03:28.889 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:28.889 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:28.889 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:03:28.889 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:14.084053+0000", 2023-09-23T21:03:28.890 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0046225789999999999 2023-09-23T21:03:28.890 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:28.890 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:28.890 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:03:28.890 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:14.085446+0000", 2023-09-23T21:03:28.890 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.001392403 2023-09-23T21:03:28.890 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:28.890 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:28.890 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T21:03:28.890 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:14.085514+0000", 2023-09-23T21:03:28.890 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.8056e-05 2023-09-23T21:03:28.891 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:28.891 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:28.891 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T21:03:28.891 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:14.085822+0000", 2023-09-23T21:03:28.891 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000308795 2023-09-23T21:03:28.891 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:03:28.891 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:03:28.891 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:03:28.891 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:28.891 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:28.891 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34797.0:115007 6.f 6:f1e2704f:::benchmark_data_smithi026_223868_object13881:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e132)", 2023-09-23T21:03:28.891 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T20:54:13.004902+0000", 2023-09-23T21:03:28.891 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 555.83797651899999, 2023-09-23T21:03:28.891 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0814330700000001, 2023-09-23T21:03:28.892 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:03:28.892 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:03:28.892 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T21:03:28.892 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34797", 2023-09-23T21:03:28.892 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T21:03:28.892 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 115007 2023-09-23T21:03:28.892 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:28.892 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:03:28.892 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:28.892 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:03:28.892 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:13.004902+0000", 2023-09-23T21:03:28.893 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:03:28.893 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:28.893 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:28.893 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:03:28.893 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:13.004902+0000", 2023-09-23T21:03:28.893 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:03:28.893 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:28.893 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:28.893 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:03:28.893 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:13.004901+0000", 2023-09-23T21:03:28.893 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T21:03:28.893 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:28.893 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:28.894 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:03:28.894 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:13.004908+0000", 2023-09-23T21:03:28.894 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.8599999999999993e-06 2023-09-23T21:03:28.894 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:28.894 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:28.894 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:03:28.894 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:13.004922+0000", 2023-09-23T21:03:28.894 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.3678e-05 2023-09-23T21:03:28.894 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:28.894 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:28.895 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:03:28.895 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:13.004929+0000", 2023-09-23T21:03:28.895 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.8770000000000004e-06 2023-09-23T21:03:28.895 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:28.895 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:28.895 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:03:28.895 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:13.004992+0000", 2023-09-23T21:03:28.895 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.2823000000000005e-05 2023-09-23T21:03:28.895 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:28.895 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:28.895 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T21:03:28.895 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:13.005003+0000", 2023-09-23T21:03:28.895 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1345999999999999e-05 2023-09-23T21:03:28.896 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:28.896 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:28.896 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:03:28.896 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:14.072238+0000", 2023-09-23T21:03:28.896 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.067234351 2023-09-23T21:03:28.896 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:28.896 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:28.896 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:03:28.896 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:14.072395+0000", 2023-09-23T21:03:28.896 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00015737999999999999 2023-09-23T21:03:28.896 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:28.896 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:28.896 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 3,4", 2023-09-23T21:03:28.897 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:14.072533+0000", 2023-09-23T21:03:28.897 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000137758 2023-09-23T21:03:28.897 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:28.897 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:28.897 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T21:03:28.897 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:14.079442+0000", 2023-09-23T21:03:28.897 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0069094589999999997 2023-09-23T21:03:28.897 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:28.897 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:28.897 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:03:28.897 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:14.084446+0000", 2023-09-23T21:03:28.897 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0050034479999999997 2023-09-23T21:03:28.897 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:28.898 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:28.898 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:03:28.898 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:14.085889+0000", 2023-09-23T21:03:28.898 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.001443819 2023-09-23T21:03:28.898 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:28.898 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:28.898 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T21:03:28.898 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:14.085956+0000", 2023-09-23T21:03:28.898 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.6622e-05 2023-09-23T21:03:28.898 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:28.898 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:28.898 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T21:03:28.899 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:14.086335+0000", 2023-09-23T21:03:28.899 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00037926700000000003 2023-09-23T21:03:28.899 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:03:28.899 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:03:28.899 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:03:28.899 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:28.899 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:28.899 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34797.0:115008 6.f 6:f2635b96:::benchmark_data_smithi026_223868_object13882:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e132)", 2023-09-23T21:03:28.899 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T20:54:13.004945+0000", 2023-09-23T21:03:28.899 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 555.83793360799996, 2023-09-23T21:03:28.899 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0825950639999999, 2023-09-23T21:03:28.899 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:03:28.899 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:03:28.900 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T21:03:28.900 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34797", 2023-09-23T21:03:28.900 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T21:03:28.900 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 115008 2023-09-23T21:03:28.900 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:28.900 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:03:28.900 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:28.900 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:03:28.900 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:13.004945+0000", 2023-09-23T21:03:28.900 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:03:28.900 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:28.900 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:28.900 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:03:28.901 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:13.004945+0000", 2023-09-23T21:03:28.901 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:03:28.901 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:28.901 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:28.901 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:03:28.901 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:13.004945+0000", 2023-09-23T21:03:28.901 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T21:03:28.901 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:28.901 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:28.901 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:03:28.901 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:13.004947+0000", 2023-09-23T21:03:28.901 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.7949999999999999e-06 2023-09-23T21:03:28.902 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:28.902 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:28.902 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:03:28.902 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:13.004954+0000", 2023-09-23T21:03:28.902 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.8190000000000003e-06 2023-09-23T21:03:28.902 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:28.902 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:28.902 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:03:28.902 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:13.004960+0000", 2023-09-23T21:03:28.902 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.5609999999999998e-06 2023-09-23T21:03:28.902 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:28.902 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:28.903 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:03:28.903 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:13.005034+0000", 2023-09-23T21:03:28.903 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.3818000000000004e-05 2023-09-23T21:03:28.903 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:28.903 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:28.903 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T21:03:28.903 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:13.005043+0000", 2023-09-23T21:03:28.903 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.9930000000000001e-06 2023-09-23T21:03:28.903 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:28.903 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:28.903 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:03:28.903 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:14.075534+0000", 2023-09-23T21:03:28.904 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0704913330000001 2023-09-23T21:03:28.904 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:28.904 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:28.904 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:03:28.904 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:14.075709+0000", 2023-09-23T21:03:28.904 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00017482899999999999 2023-09-23T21:03:28.904 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:28.904 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:28.904 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 3,4", 2023-09-23T21:03:28.904 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:14.075860+0000", 2023-09-23T21:03:28.904 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00015149599999999999 2023-09-23T21:03:28.904 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:28.904 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:28.905 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T21:03:28.905 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:14.079734+0000", 2023-09-23T21:03:28.905 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.003873816 2023-09-23T21:03:28.905 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:28.905 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:28.905 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:03:28.905 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:14.084488+0000", 2023-09-23T21:03:28.905 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0047534150000000004 2023-09-23T21:03:28.905 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:28.905 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:28.905 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:03:28.906 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:14.087241+0000", 2023-09-23T21:03:28.906 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.002753288 2023-09-23T21:03:28.906 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:28.906 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:28.906 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T21:03:28.906 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:14.087280+0000", 2023-09-23T21:03:28.906 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.9647000000000002e-05 2023-09-23T21:03:28.906 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:28.906 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:28.906 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T21:03:28.906 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:14.087540+0000", 2023-09-23T21:03:28.906 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00025969900000000001 2023-09-23T21:03:28.906 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:03:28.907 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:03:28.907 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:03:28.907 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:28.907 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:28.907 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34797.0:114997 6.12 6:49f2a314:::benchmark_data_smithi026_223868_object13871:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e133)", 2023-09-23T21:03:28.907 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T20:54:13.930512+0000", 2023-09-23T21:03:28.907 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 554.91236638500004, 2023-09-23T21:03:28.907 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0903023789999999, 2023-09-23T21:03:28.907 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:03:28.907 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:03:28.907 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T21:03:28.907 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34797", 2023-09-23T21:03:28.907 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T21:03:28.908 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 114997 2023-09-23T21:03:28.908 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:28.908 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:03:28.908 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:28.908 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:03:28.908 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:13.930512+0000", 2023-09-23T21:03:28.908 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:03:28.908 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:28.908 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:28.908 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:03:28.908 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:13.930512+0000", 2023-09-23T21:03:28.908 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:03:28.908 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:28.909 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:28.909 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:03:28.909 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:13.930510+0000", 2023-09-23T21:03:28.909 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999981 2023-09-23T21:03:28.909 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:28.909 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:28.909 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:03:28.909 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:13.930520+0000", 2023-09-23T21:03:28.909 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.6630000000000005e-06 2023-09-23T21:03:28.909 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:28.909 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:28.909 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:03:28.910 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:13.930540+0000", 2023-09-23T21:03:28.910 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.0469e-05 2023-09-23T21:03:28.910 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:28.910 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:28.910 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:03:28.910 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:13.930556+0000", 2023-09-23T21:03:28.910 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.5702999999999999e-05 2023-09-23T21:03:28.910 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:28.910 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:28.910 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:03:28.910 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:13.936376+0000", 2023-09-23T21:03:28.910 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0058194709999999997 2023-09-23T21:03:28.911 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:28.911 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:28.911 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T21:03:28.911 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:13.936391+0000", 2023-09-23T21:03:28.911 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.5276000000000001e-05 2023-09-23T21:03:28.911 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:28.911 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:28.911 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:03:28.911 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:14.999262+0000", 2023-09-23T21:03:28.911 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.062871597 2023-09-23T21:03:28.911 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:28.911 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:28.912 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:03:28.912 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:14.999394+0000", 2023-09-23T21:03:28.912 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000132008 2023-09-23T21:03:28.912 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:28.912 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:28.912 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 3,4", 2023-09-23T21:03:28.912 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:14.999530+0000", 2023-09-23T21:03:28.912 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000135718 2023-09-23T21:03:28.912 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:28.912 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:28.912 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:03:28.912 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:15.010731+0000", 2023-09-23T21:03:28.913 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.01120052 2023-09-23T21:03:28.913 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:28.913 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:28.913 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:03:28.913 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:15.012838+0000", 2023-09-23T21:03:28.913 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0021071369999999998 2023-09-23T21:03:28.913 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:28.913 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:28.913 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T21:03:28.913 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:15.020280+0000", 2023-09-23T21:03:28.913 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0074424410000000002 2023-09-23T21:03:28.913 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:28.913 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:28.914 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T21:03:28.914 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:15.020331+0000", 2023-09-23T21:03:28.914 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.0407e-05 2023-09-23T21:03:28.914 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:28.914 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:28.914 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T21:03:28.914 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:15.020815+0000", 2023-09-23T21:03:28.914 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00048407799999999998 2023-09-23T21:03:28.914 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:03:28.914 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:03:28.915 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:03:28.915 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:28.915 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:28.915 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35100.0:15749 7.5 7:a81081a1:::benchmark_data_smithi026_260489_object15748:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e204)", 2023-09-23T21:03:28.915 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T20:56:44.987630+0000", 2023-09-23T21:03:28.915 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 403.85524838100002, 2023-09-23T21:03:28.915 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0022299779999999, 2023-09-23T21:03:28.915 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:03:28.915 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "reached pg", 2023-09-23T21:03:28.915 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T21:03:28.915 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35100", 2023-09-23T21:03:28.915 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/4108972961", 2023-09-23T21:03:28.916 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 15749 2023-09-23T21:03:28.916 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:28.916 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:03:28.916 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:28.916 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:03:28.916 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:44.987630+0000", 2023-09-23T21:03:28.916 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:03:28.916 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:28.916 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:28.916 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:03:28.916 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:44.987630+0000", 2023-09-23T21:03:28.916 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:03:28.917 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:28.917 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:28.917 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:03:28.917 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:44.987629+0000", 2023-09-23T21:03:28.917 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T21:03:28.917 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:28.917 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:28.917 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:03:28.917 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:44.987658+0000", 2023-09-23T21:03:28.917 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.8971000000000001e-05 2023-09-23T21:03:28.917 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:28.917 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:28.917 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:03:28.918 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:44.987674+0000", 2023-09-23T21:03:28.918 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.6407999999999999e-05 2023-09-23T21:03:28.918 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:28.918 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:28.918 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:03:28.918 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:44.987683+0000", 2023-09-23T21:03:28.918 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.5189999999999992e-06 2023-09-23T21:03:28.918 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:28.918 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:28.918 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:03:28.918 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:44.987832+0000", 2023-09-23T21:03:28.919 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00014898099999999999 2023-09-23T21:03:28.919 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:28.919 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:28.919 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T21:03:28.919 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:44.987842+0000", 2023-09-23T21:03:28.919 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0008e-05 2023-09-23T21:03:28.919 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:28.919 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:28.919 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:03:28.919 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:45.989577+0000", 2023-09-23T21:03:28.919 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0017352289999999 2023-09-23T21:03:28.919 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:28.919 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:28.920 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T21:03:28.920 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:45.989860+0000", 2023-09-23T21:03:28.920 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00028352999999999999 2023-09-23T21:03:28.920 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:03:28.920 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:03:28.920 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:03:28.920 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:28.920 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:28.920 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35100.0:15759 7.5 7:ac55e34a:::benchmark_data_smithi026_260489_object15758:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e204)", 2023-09-23T21:03:28.920 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T20:56:44.988827+0000", 2023-09-23T21:03:28.920 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 403.854051443, 2023-09-23T21:03:28.920 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.006960361, 2023-09-23T21:03:28.920 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:03:28.921 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:03:28.921 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T21:03:28.921 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35100", 2023-09-23T21:03:28.921 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/4108972961", 2023-09-23T21:03:28.921 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 15759 2023-09-23T21:03:28.921 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:28.921 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:03:28.921 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:28.921 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:03:28.921 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:44.988827+0000", 2023-09-23T21:03:28.921 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:03:28.921 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:28.921 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:28.922 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:03:28.922 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:44.988827+0000", 2023-09-23T21:03:28.922 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:03:28.922 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:28.922 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:28.922 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:03:28.922 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:44.988827+0000", 2023-09-23T21:03:28.922 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T21:03:28.922 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:28.922 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:28.922 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:03:28.922 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:44.988858+0000", 2023-09-23T21:03:28.923 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.1634999999999998e-05 2023-09-23T21:03:28.923 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:28.923 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:28.923 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:03:28.923 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:44.988870+0000", 2023-09-23T21:03:28.923 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1303e-05 2023-09-23T21:03:28.923 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:28.923 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:28.923 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:03:28.923 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:44.988877+0000", 2023-09-23T21:03:28.923 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.8509999999999999e-06 2023-09-23T21:03:28.923 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:28.923 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:28.924 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:03:28.924 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:44.988952+0000", 2023-09-23T21:03:28.924 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.5056999999999996e-05 2023-09-23T21:03:28.924 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:28.924 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:28.924 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T21:03:28.924 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:44.988962+0000", 2023-09-23T21:03:28.924 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.9739999999999997e-06 2023-09-23T21:03:28.924 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:28.924 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:28.924 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:03:28.924 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:45.990191+0000", 2023-09-23T21:03:28.925 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0012284229999999 2023-09-23T21:03:28.925 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:28.925 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:28.925 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:03:28.925 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:45.990358+0000", 2023-09-23T21:03:28.925 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00016682700000000001 2023-09-23T21:03:28.925 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:28.925 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:28.925 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 0,7", 2023-09-23T21:03:28.925 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:45.990527+0000", 2023-09-23T21:03:28.925 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00016886900000000001 2023-09-23T21:03:28.925 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:28.926 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:28.926 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:03:28.926 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:45.993951+0000", 2023-09-23T21:03:28.926 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0034246889999999999 2023-09-23T21:03:28.926 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:28.926 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:28.926 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T21:03:28.926 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:45.994015+0000", 2023-09-23T21:03:28.926 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.3351999999999999e-05 2023-09-23T21:03:28.926 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:28.926 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:28.926 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:03:28.927 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:45.995715+0000", 2023-09-23T21:03:28.927 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0017001399999999999 2023-09-23T21:03:28.927 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:28.927 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:28.927 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T21:03:28.927 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:45.995748+0000", 2023-09-23T21:03:28.927 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.3105000000000002e-05 2023-09-23T21:03:28.927 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:28.928 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:28.928 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T21:03:28.928 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:45.995788+0000", 2023-09-23T21:03:28.928 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.9808000000000002e-05 2023-09-23T21:03:28.928 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:03:28.928 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:03:28.928 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:03:28.928 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:28.928 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:28.928 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35100.0:15775 7.5 7:acafc072:::benchmark_data_smithi026_260489_object15774:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e204)", 2023-09-23T21:03:28.928 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T20:56:45.005301+0000", 2023-09-23T21:03:28.929 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 403.83757795399998, 2023-09-23T21:03:28.929 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.99060783500000005, 2023-09-23T21:03:28.929 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:03:28.929 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:03:28.929 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T21:03:28.929 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35100", 2023-09-23T21:03:28.929 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/4108972961", 2023-09-23T21:03:28.929 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 15775 2023-09-23T21:03:28.929 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:28.929 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:03:28.929 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:28.929 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:03:28.930 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:45.005301+0000", 2023-09-23T21:03:28.930 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:03:28.930 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:28.930 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:28.930 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:03:28.930 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:45.005301+0000", 2023-09-23T21:03:28.930 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:03:28.930 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:28.930 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:28.930 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:03:28.930 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:45.005299+0000", 2023-09-23T21:03:28.931 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T21:03:28.931 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:28.931 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:28.931 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:03:28.931 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:45.005326+0000", 2023-09-23T21:03:28.931 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.6072e-05 2023-09-23T21:03:28.931 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:28.931 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:28.931 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:03:28.931 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:45.005337+0000", 2023-09-23T21:03:28.932 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1895e-05 2023-09-23T21:03:28.932 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:28.932 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:28.932 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:03:28.932 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:45.005345+0000", 2023-09-23T21:03:28.932 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.5660000000000001e-06 2023-09-23T21:03:28.932 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:28.932 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:28.932 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:03:28.932 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:45.005551+0000", 2023-09-23T21:03:28.932 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00020561700000000001 2023-09-23T21:03:28.932 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:28.932 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:28.933 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T21:03:28.933 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:45.005561+0000", 2023-09-23T21:03:28.933 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0617e-05 2023-09-23T21:03:28.933 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:28.933 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:28.933 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:03:28.933 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:45.991374+0000", 2023-09-23T21:03:28.933 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.98581316600000002 2023-09-23T21:03:28.933 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:28.933 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:28.933 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:03:28.933 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:45.991512+0000", 2023-09-23T21:03:28.934 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000137319 2023-09-23T21:03:28.934 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:28.934 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:28.934 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 0,7", 2023-09-23T21:03:28.934 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:45.991645+0000", 2023-09-23T21:03:28.934 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00013373200000000001 2023-09-23T21:03:28.934 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:28.934 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:28.934 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:03:28.934 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:45.994002+0000", 2023-09-23T21:03:28.934 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0023568299999999999 2023-09-23T21:03:28.934 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:28.935 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:28.935 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T21:03:28.935 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:45.994022+0000", 2023-09-23T21:03:28.935 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.9833000000000001e-05 2023-09-23T21:03:28.935 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:28.935 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:28.935 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:03:28.935 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:45.995852+0000", 2023-09-23T21:03:28.935 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.001829502 2023-09-23T21:03:28.935 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:28.935 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:28.935 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T21:03:28.935 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:45.995886+0000", 2023-09-23T21:03:28.936 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.4493000000000003e-05 2023-09-23T21:03:28.936 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:28.936 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:28.936 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T21:03:28.936 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:45.995909+0000", 2023-09-23T21:03:28.936 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.247e-05 2023-09-23T21:03:28.936 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:03:28.936 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:03:28.936 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:03:28.936 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:28.936 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:28.937 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35100.0:15776 7.5 7:a8ac59f4:::benchmark_data_smithi026_260489_object15775:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e204)", 2023-09-23T21:03:28.937 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T20:56:45.005362+0000", 2023-09-23T21:03:28.937 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 403.83751680699999, 2023-09-23T21:03:28.937 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.99064014700000003, 2023-09-23T21:03:28.937 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:03:28.937 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:03:28.937 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T21:03:28.937 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35100", 2023-09-23T21:03:28.937 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/4108972961", 2023-09-23T21:03:28.937 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 15776 2023-09-23T21:03:28.937 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:28.937 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:03:28.938 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:28.938 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:03:28.938 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:45.005362+0000", 2023-09-23T21:03:28.938 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:03:28.938 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:28.938 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:28.938 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:03:28.938 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:45.005362+0000", 2023-09-23T21:03:28.938 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:03:28.938 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:28.938 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:28.938 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:03:28.939 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:45.005361+0000", 2023-09-23T21:03:28.939 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T21:03:28.939 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:28.939 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:28.939 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:03:28.939 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:45.005389+0000", 2023-09-23T21:03:28.939 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.7640999999999998e-05 2023-09-23T21:03:28.939 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:28.939 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:28.939 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:03:28.939 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:45.005399+0000", 2023-09-23T21:03:28.939 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.9150000000000008e-06 2023-09-23T21:03:28.939 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:28.940 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:28.940 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:03:28.940 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:45.005405+0000", 2023-09-23T21:03:28.940 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.8629999999999996e-06 2023-09-23T21:03:28.940 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:28.940 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:28.940 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:03:28.940 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:45.005584+0000", 2023-09-23T21:03:28.940 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00017923700000000001 2023-09-23T21:03:28.940 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:28.940 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:28.940 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T21:03:28.941 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:45.005605+0000", 2023-09-23T21:03:28.941 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.0664000000000001e-05 2023-09-23T21:03:28.941 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:28.941 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:28.941 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:03:28.941 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:45.992242+0000", 2023-09-23T21:03:28.941 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.98663720899999996 2023-09-23T21:03:28.941 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:28.941 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:28.941 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:03:28.941 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:45.992353+0000", 2023-09-23T21:03:28.942 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00011114499999999999 2023-09-23T21:03:28.942 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:28.942 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:28.942 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 0,7", 2023-09-23T21:03:28.942 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:45.992490+0000", 2023-09-23T21:03:28.942 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00013666399999999999 2023-09-23T21:03:28.942 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:28.942 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:28.942 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T21:03:28.942 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:45.994029+0000", 2023-09-23T21:03:28.942 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0015394289999999999 2023-09-23T21:03:28.942 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:28.943 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:28.943 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:03:28.943 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:45.994059+0000", 2023-09-23T21:03:28.943 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.9774999999999999e-05 2023-09-23T21:03:28.943 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:28.943 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:28.943 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:03:28.943 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:45.995941+0000", 2023-09-23T21:03:28.943 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.001881916 2023-09-23T21:03:28.943 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:28.943 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:28.943 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T21:03:28.944 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:45.995969+0000", 2023-09-23T21:03:28.944 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.796e-05 2023-09-23T21:03:28.944 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:28.944 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:28.944 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T21:03:28.944 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:45.996002+0000", 2023-09-23T21:03:28.944 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.3192999999999999e-05 2023-09-23T21:03:28.944 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:03:28.944 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:03:28.944 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:03:28.944 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:28.944 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:28.945 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35100.0:15777 7.5 7:a793f4e0:::benchmark_data_smithi026_260489_object15776:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e204)", 2023-09-23T21:03:28.945 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T20:56:45.005411+0000", 2023-09-23T21:03:28.945 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 403.83746816199999, 2023-09-23T21:03:28.945 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.990746927, 2023-09-23T21:03:28.945 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:03:28.945 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:03:28.945 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T21:03:28.945 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35100", 2023-09-23T21:03:28.945 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/4108972961", 2023-09-23T21:03:28.945 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 15777 2023-09-23T21:03:28.945 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:28.946 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:03:28.946 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:28.946 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:03:28.946 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:45.005411+0000", 2023-09-23T21:03:28.946 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:03:28.946 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:28.946 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:28.946 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:03:28.946 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:45.005411+0000", 2023-09-23T21:03:28.946 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:03:28.946 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:28.946 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:28.947 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:03:28.947 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:45.005410+0000", 2023-09-23T21:03:28.947 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T21:03:28.947 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:28.947 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:28.947 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:03:28.947 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:45.005434+0000", 2023-09-23T21:03:28.947 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.4374000000000001e-05 2023-09-23T21:03:28.947 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:28.947 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:28.947 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:03:28.947 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:45.005444+0000", 2023-09-23T21:03:28.948 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.6639999999999993e-06 2023-09-23T21:03:28.948 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:28.948 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:28.948 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:03:28.948 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:45.005450+0000", 2023-09-23T21:03:28.948 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.4539999999999998e-06 2023-09-23T21:03:28.948 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:28.948 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:28.948 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:03:28.948 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:45.005627+0000", 2023-09-23T21:03:28.948 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00017721600000000001 2023-09-23T21:03:28.948 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:28.949 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:28.949 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T21:03:28.949 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:45.005638+0000", 2023-09-23T21:03:28.949 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1304999999999999e-05 2023-09-23T21:03:28.949 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:28.949 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:28.949 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:03:28.949 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:45.993087+0000", 2023-09-23T21:03:28.949 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.98744900199999996 2023-09-23T21:03:28.949 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:28.949 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:28.949 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:03:28.950 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:45.993208+0000", 2023-09-23T21:03:28.950 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000120631 2023-09-23T21:03:28.950 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:28.950 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:28.950 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 0,7", 2023-09-23T21:03:28.950 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:45.993346+0000", 2023-09-23T21:03:28.950 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00013816999999999999 2023-09-23T21:03:28.950 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:28.950 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:28.950 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:03:28.950 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:45.995683+0000", 2023-09-23T21:03:28.951 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00233716 2023-09-23T21:03:28.951 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:28.951 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:28.951 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T21:03:28.951 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:45.995691+0000", 2023-09-23T21:03:28.951 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.4760000000000001e-06 2023-09-23T21:03:28.951 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:28.951 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:28.951 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:03:28.951 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:45.996067+0000", 2023-09-23T21:03:28.951 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00037641399999999999 2023-09-23T21:03:28.951 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:28.951 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:28.952 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T21:03:28.952 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:45.996122+0000", 2023-09-23T21:03:28.952 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.4873000000000001e-05 2023-09-23T21:03:28.952 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:28.952 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:28.952 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T21:03:28.952 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:45.996157+0000", 2023-09-23T21:03:28.952 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.5627000000000003e-05 2023-09-23T21:03:28.952 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:03:28.952 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:03:28.952 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:03:28.952 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:28.952 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:28.953 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35100.0:15789 7.5 7:a3f6fe4b:::benchmark_data_smithi026_260489_object15788:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e204)", 2023-09-23T21:03:28.953 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T20:56:45.009801+0000", 2023-09-23T21:03:28.953 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 403.83307777800002, 2023-09-23T21:03:28.953 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.98652716399999996, 2023-09-23T21:03:28.953 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:03:28.953 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:03:28.953 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T21:03:28.953 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35100", 2023-09-23T21:03:28.953 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/4108972961", 2023-09-23T21:03:28.953 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 15789 2023-09-23T21:03:28.953 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:28.953 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:03:28.953 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:28.954 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:03:28.954 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:45.009801+0000", 2023-09-23T21:03:28.954 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:03:28.954 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:28.954 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:28.954 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:03:28.954 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:45.009801+0000", 2023-09-23T21:03:28.954 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:03:28.954 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:28.954 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:28.954 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:03:28.954 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:45.009799+0000", 2023-09-23T21:03:28.955 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999981 2023-09-23T21:03:28.955 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:28.955 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:28.955 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:03:28.955 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:45.009832+0000", 2023-09-23T21:03:28.955 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.2969000000000002e-05 2023-09-23T21:03:28.955 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:28.955 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:28.955 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:03:28.955 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:45.009855+0000", 2023-09-23T21:03:28.955 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.245e-05 2023-09-23T21:03:28.956 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:28.956 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:28.956 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:03:28.956 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:45.009867+0000", 2023-09-23T21:03:28.956 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.2541000000000001e-05 2023-09-23T21:03:28.956 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:28.956 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:28.956 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:03:28.956 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:45.009957+0000", 2023-09-23T21:03:28.956 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.0140000000000001e-05 2023-09-23T21:03:28.956 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:28.956 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:28.957 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T21:03:28.957 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:45.009980+0000", 2023-09-23T21:03:28.957 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.3153999999999999e-05 2023-09-23T21:03:28.957 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:28.957 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:28.957 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:03:28.957 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:45.994083+0000", 2023-09-23T21:03:28.957 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.984102747 2023-09-23T21:03:28.957 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:28.957 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:28.957 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:03:28.957 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:45.994232+0000", 2023-09-23T21:03:28.957 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00014901899999999999 2023-09-23T21:03:28.958 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:28.958 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:28.958 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 0,7", 2023-09-23T21:03:28.958 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:45.994401+0000", 2023-09-23T21:03:28.958 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00016840599999999999 2023-09-23T21:03:28.958 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:28.958 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:28.958 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T21:03:28.958 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:45.996021+0000", 2023-09-23T21:03:28.958 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.001620699 2023-09-23T21:03:28.958 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:28.958 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:28.959 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:03:28.959 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:45.996208+0000", 2023-09-23T21:03:28.959 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00018653699999999999 2023-09-23T21:03:28.959 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:28.959 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:28.959 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:03:28.959 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:45.996243+0000", 2023-09-23T21:03:28.959 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.5651000000000001e-05 2023-09-23T21:03:28.959 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:28.959 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:28.959 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T21:03:28.960 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:45.996290+0000", 2023-09-23T21:03:28.960 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.6078000000000002e-05 2023-09-23T21:03:28.960 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:28.960 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:28.960 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T21:03:28.960 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:45.996328+0000", 2023-09-23T21:03:28.960 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.8572000000000003e-05 2023-09-23T21:03:28.960 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:03:28.960 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:03:28.960 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:03:28.960 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:28.960 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:28.961 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35100.0:15790 7.5 7:a440413d:::benchmark_data_smithi026_260489_object15789:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e204)", 2023-09-23T21:03:28.961 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T20:56:45.009895+0000", 2023-09-23T21:03:28.961 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 403.83298408799999, 2023-09-23T21:03:28.961 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.994743763, 2023-09-23T21:03:28.961 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:03:28.961 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:03:28.961 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T21:03:28.961 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35100", 2023-09-23T21:03:28.961 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/4108972961", 2023-09-23T21:03:28.961 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 15790 2023-09-23T21:03:28.961 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:28.961 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:03:28.961 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:28.962 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:03:28.962 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:45.009895+0000", 2023-09-23T21:03:28.962 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:03:28.962 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:28.962 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:28.962 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:03:28.962 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:45.009895+0000", 2023-09-23T21:03:28.962 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:03:28.962 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:28.962 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:28.962 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:03:28.963 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:45.009894+0000", 2023-09-23T21:03:28.963 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.999999 2023-09-23T21:03:28.963 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:28.963 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:28.963 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:03:28.963 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:45.009919+0000", 2023-09-23T21:03:28.963 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.4708000000000001e-05 2023-09-23T21:03:28.963 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:28.963 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:28.963 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:03:28.963 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:45.009927+0000", 2023-09-23T21:03:28.963 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.1699999999999997e-06 2023-09-23T21:03:28.964 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:28.964 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:28.964 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:03:28.964 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:45.009933+0000", 2023-09-23T21:03:28.964 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.9449999999999998e-06 2023-09-23T21:03:28.964 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:28.964 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:28.964 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:03:28.965 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:45.010022+0000", 2023-09-23T21:03:28.966 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.9723999999999997e-05 2023-09-23T21:03:28.966 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:28.966 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:28.966 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T21:03:28.966 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:45.010034+0000", 2023-09-23T21:03:28.966 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1839e-05 2023-09-23T21:03:28.966 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:28.966 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:28.966 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:03:28.966 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:45.995073+0000", 2023-09-23T21:03:28.966 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.98503915200000003 2023-09-23T21:03:28.966 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:28.967 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:28.967 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:03:28.967 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:45.995200+0000", 2023-09-23T21:03:28.967 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00012642400000000001 2023-09-23T21:03:28.967 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:28.967 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:28.967 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 0,7", 2023-09-23T21:03:28.967 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:45.995287+0000", 2023-09-23T21:03:28.967 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.7268000000000005e-05 2023-09-23T21:03:28.967 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:28.967 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:28.967 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:03:28.968 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:45.996369+0000", 2023-09-23T21:03:28.968 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0010815449999999999 2023-09-23T21:03:28.968 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:28.968 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:28.968 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:03:28.968 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:45.996407+0000", 2023-09-23T21:03:28.968 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.8569999999999998e-05 2023-09-23T21:03:28.968 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:28.968 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:28.968 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T21:03:28.968 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:46.004549+0000", 2023-09-23T21:03:28.968 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0081419200000000004 2023-09-23T21:03:28.969 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:28.969 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:28.969 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T21:03:28.969 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:46.004599+0000", 2023-09-23T21:03:28.969 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.0314e-05 2023-09-23T21:03:28.969 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:28.969 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:28.969 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T21:03:28.969 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:46.004638+0000", 2023-09-23T21:03:28.969 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.8926e-05 2023-09-23T21:03:28.969 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:03:28.969 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:03:28.969 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:03:28.970 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:28.970 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:28.970 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35100.0:15804 7.5 7:a64bccef:::benchmark_data_smithi026_260489_object15803:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e204)", 2023-09-23T21:03:28.970 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T20:56:45.018381+0000", 2023-09-23T21:03:28.970 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 403.82449729199999, 2023-09-23T21:03:28.970 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.98634781400000004, 2023-09-23T21:03:28.970 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:03:28.970 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:03:28.970 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T21:03:28.970 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35100", 2023-09-23T21:03:28.970 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/4108972961", 2023-09-23T21:03:28.970 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 15804 2023-09-23T21:03:28.970 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:28.971 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:03:28.971 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:28.971 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:03:28.971 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:45.018381+0000", 2023-09-23T21:03:28.971 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:03:28.971 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:28.971 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:28.971 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:03:28.971 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:45.018381+0000", 2023-09-23T21:03:28.971 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:03:28.971 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:28.971 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:28.971 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:03:28.971 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:45.018381+0000", 2023-09-23T21:03:28.972 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T21:03:28.972 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:28.972 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:28.972 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:03:28.972 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:45.018405+0000", 2023-09-23T21:03:28.972 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.4698e-05 2023-09-23T21:03:28.972 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:28.972 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:28.972 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:03:28.972 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:45.018419+0000", 2023-09-23T21:03:28.972 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.3122000000000001e-05 2023-09-23T21:03:28.972 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:28.973 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:28.973 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:03:28.973 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:45.018426+0000", 2023-09-23T21:03:28.973 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.588e-06 2023-09-23T21:03:28.973 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:28.973 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:28.973 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:03:28.973 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:45.018501+0000", 2023-09-23T21:03:28.973 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.4888000000000005e-05 2023-09-23T21:03:28.973 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:28.973 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:28.973 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T21:03:28.974 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:45.018513+0000", 2023-09-23T21:03:28.974 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.239e-05 2023-09-23T21:03:28.974 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:28.974 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:28.974 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:03:28.974 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:45.996433+0000", 2023-09-23T21:03:28.974 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.97791965000000003 2023-09-23T21:03:28.974 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:28.974 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:28.974 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:03:28.974 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:45.996561+0000", 2023-09-23T21:03:28.975 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00012791199999999999 2023-09-23T21:03:28.975 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:28.975 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:28.975 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 0,7", 2023-09-23T21:03:28.975 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:45.996693+0000", 2023-09-23T21:03:28.975 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000132201 2023-09-23T21:03:28.975 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:28.975 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:28.975 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:03:28.975 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:45.998756+0000", 2023-09-23T21:03:28.975 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0020630599999999998 2023-09-23T21:03:28.975 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:28.976 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:28.976 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:03:28.976 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:45.998794+0000", 2023-09-23T21:03:28.976 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.7515999999999998e-05 2023-09-23T21:03:28.976 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:28.976 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:28.976 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T21:03:28.976 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:46.004652+0000", 2023-09-23T21:03:28.976 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0058583849999999998 2023-09-23T21:03:28.976 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:28.976 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:28.976 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T21:03:28.976 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:46.004693+0000", 2023-09-23T21:03:28.976 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.1081999999999997e-05 2023-09-23T21:03:28.977 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:28.977 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:28.977 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T21:03:28.977 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:46.004729+0000", 2023-09-23T21:03:28.977 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.5970999999999998e-05 2023-09-23T21:03:28.977 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:03:28.977 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:03:28.977 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:03:28.977 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:28.977 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:28.977 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35100.0:15841 7.5 7:a8f7bc4a:::benchmark_data_smithi026_260489_object15840:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e204)", 2023-09-23T21:03:28.977 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T20:56:45.039634+0000", 2023-09-23T21:03:28.978 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 403.803245151, 2023-09-23T21:03:28.978 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.96518164900000003, 2023-09-23T21:03:28.978 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:03:28.978 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:03:28.978 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T21:03:28.978 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35100", 2023-09-23T21:03:28.978 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/4108972961", 2023-09-23T21:03:28.978 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 15841 2023-09-23T21:03:28.978 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:28.978 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:03:28.978 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:28.979 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:03:28.979 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:45.039634+0000", 2023-09-23T21:03:28.979 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:03:28.979 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:28.979 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:28.979 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:03:28.979 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:45.039634+0000", 2023-09-23T21:03:28.979 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:03:28.979 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:28.979 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:28.979 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:03:28.979 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:45.039632+0000", 2023-09-23T21:03:28.979 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T21:03:28.980 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:28.980 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:28.980 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:03:28.980 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:45.039662+0000", 2023-09-23T21:03:28.980 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.0179e-05 2023-09-23T21:03:28.980 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:28.980 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:28.980 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:03:28.980 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:45.039680+0000", 2023-09-23T21:03:28.980 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.732e-05 2023-09-23T21:03:28.980 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:28.980 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:28.981 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:03:28.981 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:45.039688+0000", 2023-09-23T21:03:28.981 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.1589999999999993e-06 2023-09-23T21:03:28.981 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:28.981 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:28.981 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:03:28.981 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:45.039760+0000", 2023-09-23T21:03:28.981 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.2626999999999994e-05 2023-09-23T21:03:28.981 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:28.981 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:28.982 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T21:03:28.982 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:45.039774+0000", 2023-09-23T21:03:28.982 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.3258000000000001e-05 2023-09-23T21:03:28.982 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:28.982 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:28.982 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:03:28.982 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:45.997127+0000", 2023-09-23T21:03:28.982 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.95735300499999998 2023-09-23T21:03:28.982 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:28.982 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:28.982 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:03:28.982 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:45.997211+0000", 2023-09-23T21:03:28.983 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.3839999999999997e-05 2023-09-23T21:03:28.983 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:28.983 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:28.983 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 0,7", 2023-09-23T21:03:28.983 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:45.997408+0000", 2023-09-23T21:03:28.983 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00019728500000000001 2023-09-23T21:03:28.983 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:28.983 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:28.983 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:03:28.983 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:46.000390+0000", 2023-09-23T21:03:28.983 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.002981811 2023-09-23T21:03:28.983 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:28.984 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:28.984 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:03:28.984 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:46.000427+0000", 2023-09-23T21:03:28.984 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.7577000000000003e-05 2023-09-23T21:03:28.984 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:28.984 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:28.984 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T21:03:28.984 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:46.004740+0000", 2023-09-23T21:03:28.984 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0043126980000000002 2023-09-23T21:03:28.984 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:28.984 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:28.984 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T21:03:28.985 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:46.004782+0000", 2023-09-23T21:03:28.985 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.2274000000000002e-05 2023-09-23T21:03:28.985 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:28.985 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:28.985 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T21:03:28.985 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:46.004815+0000", 2023-09-23T21:03:28.985 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.3061000000000001e-05 2023-09-23T21:03:28.985 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:03:28.985 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:03:28.985 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:03:28.985 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:28.985 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:28.986 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35100.0:15864 7.5 7:a807304b:::benchmark_data_smithi026_260489_object15863:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e204)", 2023-09-23T21:03:28.986 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T20:56:45.054615+0000", 2023-09-23T21:03:28.986 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 403.78826351599997, 2023-09-23T21:03:28.986 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.95028003800000005, 2023-09-23T21:03:28.986 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:03:28.986 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:03:28.986 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T21:03:28.986 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35100", 2023-09-23T21:03:28.986 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/4108972961", 2023-09-23T21:03:28.986 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 15864 2023-09-23T21:03:28.986 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:28.987 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:03:28.987 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:28.987 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:03:28.987 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:45.054615+0000", 2023-09-23T21:03:28.987 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:03:28.987 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:28.987 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:28.987 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:03:28.987 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:45.054615+0000", 2023-09-23T21:03:28.987 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:03:28.987 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:28.987 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:28.988 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:03:28.988 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:45.054614+0000", 2023-09-23T21:03:28.988 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T21:03:28.988 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:28.988 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:28.988 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:03:28.988 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:45.054641+0000", 2023-09-23T21:03:28.988 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.6922000000000001e-05 2023-09-23T21:03:28.988 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:28.988 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:28.988 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:03:28.988 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:45.054653+0000", 2023-09-23T21:03:28.988 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.2768e-05 2023-09-23T21:03:28.989 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:28.989 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:28.989 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:03:28.989 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:45.054661+0000", 2023-09-23T21:03:28.989 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.6599999999999995e-06 2023-09-23T21:03:28.989 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:28.989 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:28.989 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:03:28.989 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:45.054726+0000", 2023-09-23T21:03:28.989 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.4469999999999998e-05 2023-09-23T21:03:28.989 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:28.989 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:28.989 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T21:03:28.990 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:45.054736+0000", 2023-09-23T21:03:28.990 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0377000000000001e-05 2023-09-23T21:03:28.990 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:28.990 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:28.990 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:03:28.990 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:45.997962+0000", 2023-09-23T21:03:28.990 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.94322652600000001 2023-09-23T21:03:28.990 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:28.990 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:28.990 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:03:28.990 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:45.998081+0000", 2023-09-23T21:03:28.991 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000118573 2023-09-23T21:03:28.991 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:28.991 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:28.991 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 0,7", 2023-09-23T21:03:28.991 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:45.998226+0000", 2023-09-23T21:03:28.991 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000144754 2023-09-23T21:03:28.991 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:28.991 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:28.991 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:03:28.991 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:46.001987+0000", 2023-09-23T21:03:28.991 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0037616559999999999 2023-09-23T21:03:28.992 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:28.992 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:28.992 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:03:28.992 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:46.002069+0000", 2023-09-23T21:03:28.992 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.1927000000000004e-05 2023-09-23T21:03:28.992 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:28.992 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:28.992 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T21:03:28.992 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:46.004826+0000", 2023-09-23T21:03:28.992 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0027565150000000002 2023-09-23T21:03:28.993 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:28.993 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:28.993 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T21:03:28.993 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:46.004863+0000", 2023-09-23T21:03:28.993 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.7113000000000002e-05 2023-09-23T21:03:28.993 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:28.993 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:28.993 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T21:03:28.993 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:46.004895+0000", 2023-09-23T21:03:28.993 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.2211e-05 2023-09-23T21:03:28.994 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:03:28.994 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:03:28.994 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:03:28.994 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:28.994 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:28.994 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35100.0:15898 7.5 7:a0c3cfcb:::benchmark_data_smithi026_260489_object15897:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e204)", 2023-09-23T21:03:28.994 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T20:56:45.079214+0000", 2023-09-23T21:03:28.994 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 403.76366505099998, 2023-09-23T21:03:28.994 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.92576644500000005, 2023-09-23T21:03:28.994 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:03:28.994 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:03:28.994 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T21:03:28.995 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35100", 2023-09-23T21:03:28.995 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/4108972961", 2023-09-23T21:03:28.995 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 15898 2023-09-23T21:03:28.995 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:28.995 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:03:28.995 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:28.995 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:03:28.995 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:45.079214+0000", 2023-09-23T21:03:28.995 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:03:28.995 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:28.995 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:28.996 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:03:28.996 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:45.079214+0000", 2023-09-23T21:03:28.996 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:03:28.996 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:28.996 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:28.996 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:03:28.996 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:45.079212+0000", 2023-09-23T21:03:28.996 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T21:03:28.996 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:28.996 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:28.996 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:03:28.996 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:45.079234+0000", 2023-09-23T21:03:28.997 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.2158999999999999e-05 2023-09-23T21:03:28.997 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:28.997 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:28.997 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:03:28.997 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:45.079246+0000", 2023-09-23T21:03:28.997 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1851999999999999e-05 2023-09-23T21:03:28.997 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:28.997 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:28.997 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:03:28.997 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:45.079254+0000", 2023-09-23T21:03:28.997 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.7980000000000006e-06 2023-09-23T21:03:28.998 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:28.998 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:28.998 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:03:28.998 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:45.079327+0000", 2023-09-23T21:03:28.998 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.3536999999999997e-05 2023-09-23T21:03:28.998 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:28.998 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:28.998 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T21:03:28.998 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:45.079339+0000", 2023-09-23T21:03:28.998 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1175000000000001e-05 2023-09-23T21:03:28.998 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:28.999 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:28.999 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:03:28.999 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:45.998823+0000", 2023-09-23T21:03:28.999 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.91948407200000004 2023-09-23T21:03:28.999 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:28.999 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:28.999 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:03:28.999 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:45.998950+0000", 2023-09-23T21:03:28.999 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00012778999999999999 2023-09-23T21:03:28.999 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:28.999 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:28.999 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 0,7", 2023-09-23T21:03:29.000 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:45.999134+0000", 2023-09-23T21:03:29.000 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000183525 2023-09-23T21:03:29.000 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:29.000 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:29.000 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:03:29.000 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:46.002016+0000", 2023-09-23T21:03:29.000 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0028819000000000002 2023-09-23T21:03:29.000 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:29.000 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:29.000 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:03:29.000 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:46.002096+0000", 2023-09-23T21:03:29.001 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.9865999999999997e-05 2023-09-23T21:03:29.001 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:29.001 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:29.001 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T21:03:29.001 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:46.004902+0000", 2023-09-23T21:03:29.001 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0028057820000000002 2023-09-23T21:03:29.001 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:29.001 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:29.001 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T21:03:29.001 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:46.004945+0000", 2023-09-23T21:03:29.001 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.3829999999999999e-05 2023-09-23T21:03:29.001 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:29.002 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:29.002 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T21:03:29.002 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:46.004980+0000", 2023-09-23T21:03:29.002 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.4709000000000003e-05 2023-09-23T21:03:29.002 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:03:29.002 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:03:29.002 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:03:29.002 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:29.002 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:29.002 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35100.0:15919 7.5 7:a70cf643:::benchmark_data_smithi026_260489_object15918:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e204)", 2023-09-23T21:03:29.002 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T20:56:45.095028+0000", 2023-09-23T21:03:29.003 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 403.74785103300002, 2023-09-23T21:03:29.003 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.91002998800000001, 2023-09-23T21:03:29.003 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:03:29.003 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:03:29.003 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T21:03:29.003 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35100", 2023-09-23T21:03:29.003 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/4108972961", 2023-09-23T21:03:29.003 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 15919 2023-09-23T21:03:29.003 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:29.003 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:03:29.004 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:29.004 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:03:29.004 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:45.095028+0000", 2023-09-23T21:03:29.004 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:03:29.004 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:29.004 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:29.004 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:03:29.004 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:45.095028+0000", 2023-09-23T21:03:29.004 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:03:29.004 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:29.004 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:29.005 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:03:29.005 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:45.095026+0000", 2023-09-23T21:03:29.005 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T21:03:29.005 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:29.005 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:29.005 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:03:29.005 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:45.095045+0000", 2023-09-23T21:03:29.005 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.8839999999999999e-05 2023-09-23T21:03:29.005 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:29.005 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:29.005 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:03:29.006 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:45.095062+0000", 2023-09-23T21:03:29.006 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.6512e-05 2023-09-23T21:03:29.006 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:29.006 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:29.006 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:03:29.006 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:45.095071+0000", 2023-09-23T21:03:29.006 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.0350000000000007e-06 2023-09-23T21:03:29.006 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:29.006 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:29.006 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:03:29.006 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:45.095147+0000", 2023-09-23T21:03:29.007 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.6063999999999995e-05 2023-09-23T21:03:29.007 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:29.007 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:29.007 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T21:03:29.007 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:45.095158+0000", 2023-09-23T21:03:29.007 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1464999999999999e-05 2023-09-23T21:03:29.007 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:29.007 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:29.007 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:03:29.007 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:45.999658+0000", 2023-09-23T21:03:29.007 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.90450018399999998 2023-09-23T21:03:29.007 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:29.008 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:29.008 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:03:29.008 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:45.999736+0000", 2023-09-23T21:03:29.008 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.7651000000000006e-05 2023-09-23T21:03:29.008 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:29.008 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:29.008 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 0,7", 2023-09-23T21:03:29.008 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:45.999874+0000", 2023-09-23T21:03:29.008 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00013834399999999999 2023-09-23T21:03:29.008 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:29.008 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:29.008 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:03:29.009 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:46.002043+0000", 2023-09-23T21:03:29.009 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0021684009999999999 2023-09-23T21:03:29.009 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:29.009 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:29.009 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:03:29.009 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:46.002125+0000", 2023-09-23T21:03:29.009 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.1880999999999998e-05 2023-09-23T21:03:29.009 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:29.009 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:29.009 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T21:03:29.009 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:46.004987+0000", 2023-09-23T21:03:29.009 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0028622539999999998 2023-09-23T21:03:29.010 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:29.010 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:29.010 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T21:03:29.010 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:46.005026+0000", 2023-09-23T21:03:29.010 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.8600000000000003e-05 2023-09-23T21:03:29.010 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:29.010 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:29.010 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T21:03:29.010 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:46.005058+0000", 2023-09-23T21:03:29.010 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.2156000000000002e-05 2023-09-23T21:03:29.010 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:03:29.010 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:03:29.010 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:03:29.011 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:03:29.011 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:03:29.011 INFO:teuthology.orchestra.run.smithi026.stdout:} 2023-09-23T21:03:29.011 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:03:29.012 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T21:03:29.012 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:03:29.113 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:03:29.169 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:03:29.232 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:03:29.236 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:03:29.237 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:03:29.249 INFO:teuthology.orchestra.run.smithi026.stderr:nodeep-scrub is set 2023-09-23T21:03:29.298 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_ops_in_flight 2023-09-23T21:03:29.337 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:03:29.350 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T21:03:29.444 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:03:29 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pg_num_actual", "val": "26"}]: dispatch 2023-09-23T21:03:29.444 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:03:29 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/3329797301' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:03:29.475 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:03:29 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pg_num_actual", "val": "26"}]: dispatch 2023-09-23T21:03:29.475 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:03:29 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/3329797301' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:03:29.475 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:03:29 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pg_num_actual", "val": "26"}]: dispatch 2023-09-23T21:03:29.476 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:03:29 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/3329797301' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:03:29.482 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:03:29.571 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:03:29.574 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:03:29.577 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:03:29.677 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:03:29.706 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:03:29.742 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T21:03:29.745 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 21:03:29 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[164029]: ::ffff:172.21.15.70 - - [23/Sep/2023:21:03:29] "GET /metrics HTTP/1.1" 200 36294 "" "Prometheus/2.43.0" 2023-09-23T21:03:29.746 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:03:29.746 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:03:29.847 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:03:29.874 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:03:29.906 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:03:29.909 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:03:29.909 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:03:30.010 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:03:30.038 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:03:30.075 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T21:03:30.078 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:03:30.078 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:03:30.179 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:03:30.184 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:03:30.208 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:03:30.230 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:03:30.336 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:03:30.350 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:03:30.351 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:03:30.351 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:03:30 smithi026 ceph-mon[179688]: pgmap v750: 109 pgs: 109 active+clean; 5.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 820 op/s 2023-09-23T21:03:30.351 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:03:30 smithi026 ceph-mon[179688]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:03:30.351 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:03:30 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pg_num_actual", "val": "26"}]': finished 2023-09-23T21:03:30.351 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:03:30 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/3329797301' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T21:03:30.351 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:03:30 smithi026 ceph-mon[179688]: osdmap e281: 8 total, 8 up, 8 in 2023-09-23T21:03:30.351 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:03:30 smithi026 ceph-mon[179688]: failed to encode map e281 with expected crc 2023-09-23T21:03:30.351 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:03:30 smithi026 ceph-mon[179688]: failed to encode map e281 with expected crc 2023-09-23T21:03:30.354 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:03:30 smithi026 ceph-mon[175151]: pgmap v750: 109 pgs: 109 active+clean; 5.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 820 op/s 2023-09-23T21:03:30.354 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:03:30 smithi026 ceph-mon[175151]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:03:30.354 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:03:30 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pg_num_actual", "val": "26"}]': finished 2023-09-23T21:03:30.354 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:03:30 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/3329797301' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T21:03:30.354 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:03:30 smithi026 ceph-mon[175151]: osdmap e281: 8 total, 8 up, 8 in 2023-09-23T21:03:30.354 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:03:30 smithi026 ceph-mon[175151]: failed to encode map e281 with expected crc 2023-09-23T21:03:30.354 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:03:30 smithi026 ceph-mon[175151]: failed to encode map e281 with expected crc 2023-09-23T21:03:30.452 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:03:30.493 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:03:30 smithi070 ceph-mon[168282]: pgmap v750: 109 pgs: 109 active+clean; 5.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 820 op/s 2023-09-23T21:03:30.493 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:03:30 smithi070 ceph-mon[168282]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:03:30.493 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:03:30 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pg_num_actual", "val": "26"}]': finished 2023-09-23T21:03:30.493 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:03:30 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/3329797301' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T21:03:30.494 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:03:30 smithi070 ceph-mon[168282]: osdmap e281: 8 total, 8 up, 8 in 2023-09-23T21:03:30.494 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:03:30 smithi070 ceph-mon[168282]: failed to encode map e281 with expected crc 2023-09-23T21:03:30.494 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:03:30 smithi070 ceph-mon[168282]: failed to encode map e281 with expected crc 2023-09-23T21:03:30.496 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:03:30.548 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:03:30.553 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:03:30.553 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:03:30.654 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:03:30.731 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:03:30.799 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:03:30.811 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:03:30.812 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:03:30.912 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:03:30.982 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:03:31.060 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:03:31.065 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:03:31.065 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:03:31.166 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:03:31.246 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:03:31.301 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T21:03:31.305 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:03:31.305 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:03:31.406 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:03:31.476 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:03:31 smithi026 ceph-mon[175151]: failed to encode map e281 with expected crc 2023-09-23T21:03:31.477 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:03:31 smithi026 ceph-mon[175151]: failed to encode map e282 with expected crc 2023-09-23T21:03:31.477 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:03:31 smithi026 ceph-mon[175151]: osdmap e282: 8 total, 8 up, 8 in 2023-09-23T21:03:31.477 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:03:31 smithi026 ceph-mon[175151]: failed to encode map e282 with expected crc 2023-09-23T21:03:31.477 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:03:31 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:03:31.477 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:03:31 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:03:31.477 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:03:31 smithi026 ceph-mon[179688]: failed to encode map e281 with expected crc 2023-09-23T21:03:31.477 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:03:31 smithi026 ceph-mon[179688]: failed to encode map e282 with expected crc 2023-09-23T21:03:31.477 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:03:31 smithi026 ceph-mon[179688]: osdmap e282: 8 total, 8 up, 8 in 2023-09-23T21:03:31.478 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:03:31 smithi026 ceph-mon[179688]: failed to encode map e282 with expected crc 2023-09-23T21:03:31.478 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:03:31 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:03:31.478 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:03:31 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:03:31.483 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:03:31.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:03:31 smithi070 ceph-mon[168282]: failed to encode map e281 with expected crc 2023-09-23T21:03:31.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:03:31 smithi070 ceph-mon[168282]: failed to encode map e282 with expected crc 2023-09-23T21:03:31.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:03:31 smithi070 ceph-mon[168282]: osdmap e282: 8 total, 8 up, 8 in 2023-09-23T21:03:31.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:03:31 smithi070 ceph-mon[168282]: failed to encode map e282 with expected crc 2023-09-23T21:03:31.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:03:31 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:03:31.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:03:31 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:03:31.662 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T21:03:31.667 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:03:31.667 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:03:31.768 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:03:31.804 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T21:03:31.809 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:03:31.860 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:03:31.865 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:03:31.865 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:03:31.966 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:03:32.049 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:03:32.294 INFO:teuthology.orchestra.run.smithi026.stdout:{ 2023-09-23T21:03:32.294 INFO:teuthology.orchestra.run.smithi026.stdout: "ops": [ 2023-09-23T21:03:32.294 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:32.294 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35469.0:135101 8.f 8:f1aa911b:::benchmark_data_smithi026_301269_object24344:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e282)", 2023-09-23T21:03:32.294 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T21:03:32.279774+0000", 2023-09-23T21:03:32.294 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 0.013224736000000001, 2023-09-23T21:03:32.294 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.013258569, 2023-09-23T21:03:32.295 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:03:32.295 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "waiting for sub ops", 2023-09-23T21:03:32.295 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T21:03:32.295 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35469", 2023-09-23T21:03:32.295 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/4041424623", 2023-09-23T21:03:32.295 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 135101 2023-09-23T21:03:32.295 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:32.295 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:03:32.295 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:32.295 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:03:32.295 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:03:32.279774+0000", 2023-09-23T21:03:32.295 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:03:32.296 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:32.296 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:32.296 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:03:32.296 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:03:32.279774+0000", 2023-09-23T21:03:32.296 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:03:32.296 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:32.296 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:32.296 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:03:32.296 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:03:32.279773+0000", 2023-09-23T21:03:32.296 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T21:03:32.296 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:32.296 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:32.296 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:03:32.297 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:03:32.279778+0000", 2023-09-23T21:03:32.297 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.215e-06 2023-09-23T21:03:32.297 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:32.297 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:32.297 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:03:32.297 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:03:32.279788+0000", 2023-09-23T21:03:32.297 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0108999999999999e-05 2023-09-23T21:03:32.297 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:32.297 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:32.297 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:03:32.297 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:03:32.279795+0000", 2023-09-23T21:03:32.297 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.9820000000000002e-06 2023-09-23T21:03:32.298 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:32.298 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:32.298 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:03:32.298 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:03:32.280363+0000", 2023-09-23T21:03:32.298 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00056768300000000003 2023-09-23T21:03:32.298 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:32.298 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:32.298 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:03:32.298 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:03:32.280498+0000", 2023-09-23T21:03:32.298 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00013480600000000001 2023-09-23T21:03:32.298 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:32.298 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:32.299 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 1,2", 2023-09-23T21:03:32.299 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:03:32.280614+0000", 2023-09-23T21:03:32.299 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000116191 2023-09-23T21:03:32.299 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:32.299 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:32.299 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T21:03:32.299 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:03:32.286002+0000", 2023-09-23T21:03:32.299 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.005387785 2023-09-23T21:03:32.299 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:32.299 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:32.299 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:03:32.299 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:03:32.290274+0000", 2023-09-23T21:03:32.299 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.004272054 2023-09-23T21:03:32.300 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:03:32.300 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:03:32.300 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:03:32.300 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:32.300 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:32.300 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35469.0:135117 8.0 8:01c26dbe:::benchmark_data_smithi026_301269_object24360:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e282)", 2023-09-23T21:03:32.300 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T21:03:32.291197+0000", 2023-09-23T21:03:32.300 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 0.0018025000000000001, 2023-09-23T21:03:32.300 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.001941578, 2023-09-23T21:03:32.300 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:03:32.300 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "waiting for sub ops", 2023-09-23T21:03:32.300 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T21:03:32.301 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35469", 2023-09-23T21:03:32.301 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/4041424623", 2023-09-23T21:03:32.301 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 135117 2023-09-23T21:03:32.301 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:32.301 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:03:32.301 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:32.301 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:03:32.301 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:03:32.291197+0000", 2023-09-23T21:03:32.301 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:03:32.301 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:32.301 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:32.301 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:03:32.301 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:03:32.291197+0000", 2023-09-23T21:03:32.302 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:03:32.302 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:32.302 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:32.302 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:03:32.302 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:03:32.291195+0000", 2023-09-23T21:03:32.302 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T21:03:32.302 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:32.302 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:32.302 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:03:32.302 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:03:32.291203+0000", 2023-09-23T21:03:32.302 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.4529999999999996e-06 2023-09-23T21:03:32.302 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:32.302 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:32.303 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:03:32.303 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:03:32.291220+0000", 2023-09-23T21:03:32.303 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.7159999999999998e-05 2023-09-23T21:03:32.303 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:32.303 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:32.303 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:03:32.303 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:03:32.291232+0000", 2023-09-23T21:03:32.303 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.2177999999999999e-05 2023-09-23T21:03:32.303 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:32.303 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:32.303 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:03:32.303 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:03:32.291277+0000", 2023-09-23T21:03:32.303 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.5278000000000003e-05 2023-09-23T21:03:32.304 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:32.304 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:32.304 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:03:32.304 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:03:32.291445+0000", 2023-09-23T21:03:32.304 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00016825400000000001 2023-09-23T21:03:32.304 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:32.304 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:32.304 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 1,5", 2023-09-23T21:03:32.304 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:03:32.291892+0000", 2023-09-23T21:03:32.304 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00044693700000000001 2023-09-23T21:03:32.304 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:03:32.304 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:03:32.305 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:03:32.305 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:32.305 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:32.305 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35469.0:135119 8.b 8:dd5014c9:::benchmark_data_smithi026_301269_object24362:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e282)", 2023-09-23T21:03:32.305 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T21:03:32.291245+0000", 2023-09-23T21:03:32.305 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 0.001754013, 2023-09-23T21:03:32.305 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0019745359999999998, 2023-09-23T21:03:32.305 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:03:32.305 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "waiting for sub ops", 2023-09-23T21:03:32.305 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T21:03:32.305 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35469", 2023-09-23T21:03:32.305 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/4041424623", 2023-09-23T21:03:32.305 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 135119 2023-09-23T21:03:32.306 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:32.306 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:03:32.306 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:32.306 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:03:32.306 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:03:32.291245+0000", 2023-09-23T21:03:32.306 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:03:32.306 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:32.306 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:32.306 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:03:32.306 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:03:32.291245+0000", 2023-09-23T21:03:32.306 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:03:32.306 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:32.307 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:32.307 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:03:32.307 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:03:32.291245+0000", 2023-09-23T21:03:32.307 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T21:03:32.307 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:32.307 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:32.307 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:03:32.307 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:03:32.291248+0000", 2023-09-23T21:03:32.307 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.954e-06 2023-09-23T21:03:32.307 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:32.307 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:32.307 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:03:32.307 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:03:32.291254+0000", 2023-09-23T21:03:32.308 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.4649999999999999e-06 2023-09-23T21:03:32.308 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:32.308 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:32.308 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:03:32.308 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:03:32.291259+0000", 2023-09-23T21:03:32.308 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.2680000000000002e-06 2023-09-23T21:03:32.308 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:32.308 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:32.308 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:03:32.308 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:03:32.291827+0000", 2023-09-23T21:03:32.308 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00056791000000000003 2023-09-23T21:03:32.308 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:32.308 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:32.309 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:03:32.309 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:03:32.291945+0000", 2023-09-23T21:03:32.309 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000118058 2023-09-23T21:03:32.309 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:32.309 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:32.309 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 1,4", 2023-09-23T21:03:32.309 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:03:32.292057+0000", 2023-09-23T21:03:32.309 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000111321 2023-09-23T21:03:32.309 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:03:32.309 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:03:32.309 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:03:32.310 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:32.310 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:32.310 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35469.0:135120 8.b 8.bab412ab (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e282)", 2023-09-23T21:03:32.310 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T21:03:32.291269+0000", 2023-09-23T21:03:32.310 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 0.0017305280000000001, 2023-09-23T21:03:32.310 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0020284919999999998, 2023-09-23T21:03:32.310 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:03:32.310 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "queued for pg", 2023-09-23T21:03:32.310 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T21:03:32.310 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35469", 2023-09-23T21:03:32.310 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/4041424623", 2023-09-23T21:03:32.310 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 135120 2023-09-23T21:03:32.311 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:32.311 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:03:32.311 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:32.311 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:03:32.311 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:03:32.291269+0000", 2023-09-23T21:03:32.311 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:03:32.311 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:32.311 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:32.311 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:03:32.311 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:03:32.291269+0000", 2023-09-23T21:03:32.311 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:03:32.311 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:32.311 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:32.312 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:03:32.312 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:03:32.291268+0000", 2023-09-23T21:03:32.312 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T21:03:32.312 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:32.312 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:32.312 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:03:32.312 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:03:32.291271+0000", 2023-09-23T21:03:32.312 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.3039999999999999e-06 2023-09-23T21:03:32.312 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:32.312 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:32.312 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:03:32.312 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:03:32.291276+0000", 2023-09-23T21:03:32.312 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.3569999999999997e-06 2023-09-23T21:03:32.313 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:32.313 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:32.313 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:03:32.313 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:03:32.291281+0000", 2023-09-23T21:03:32.313 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.6419999999999998e-06 2023-09-23T21:03:32.313 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:03:32.313 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:03:32.313 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:03:32.313 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:03:32.313 INFO:teuthology.orchestra.run.smithi026.stdout: ], 2023-09-23T21:03:32.313 INFO:teuthology.orchestra.run.smithi026.stdout: "num_ops": 4 2023-09-23T21:03:32.313 INFO:teuthology.orchestra.run.smithi026.stdout:} 2023-09-23T21:03:32.314 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:03:32.314 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:03:32.315 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:03:32.415 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:03:32.451 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:03:32.498 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:03:32.500 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:03:32 smithi070 ceph-mon[168282]: failed to encode map e282 with expected crc 2023-09-23T21:03:32.500 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:03:32 smithi070 ceph-mon[168282]: pgmap v753: 119 pgs: 4 peering, 10 unknown, 105 active+clean; 5.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 815 op/s 2023-09-23T21:03:32.501 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:03:32.501 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:03:32.602 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:03:32.609 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:03:32 smithi026 ceph-mon[179688]: failed to encode map e282 with expected crc 2023-09-23T21:03:32.610 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:03:32 smithi026 ceph-mon[179688]: pgmap v753: 119 pgs: 4 peering, 10 unknown, 105 active+clean; 5.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 815 op/s 2023-09-23T21:03:32.610 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:03:32 smithi026 ceph-mon[175151]: failed to encode map e282 with expected crc 2023-09-23T21:03:32.610 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:03:32 smithi026 ceph-mon[175151]: pgmap v753: 119 pgs: 4 peering, 10 unknown, 105 active+clean; 5.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 815 op/s 2023-09-23T21:03:32.663 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:03:32.708 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0: {} 2023-09-23T21:03:32.742 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1: {} 2023-09-23T21:03:32.786 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2: {} 2023-09-23T21:03:32.818 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3: {} 2023-09-23T21:03:32.842 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T21:03:32.848 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:03:32.849 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:03:32.856 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4: {} 2023-09-23T21:03:32.880 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_blocked_ops 2023-09-23T21:03:32.889 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T21:03:32.932 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6: {} 2023-09-23T21:03:32.949 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:03:32.976 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T21:03:33.003 INFO:teuthology.orchestra.run.smithi026.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T21:03:33.003 INFO:teuthology.orchestra.run.smithi026.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T21:03:33.004 INFO:teuthology.orchestra.run.smithi026.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T21:03:33.004 INFO:teuthology.orchestra.run.smithi026.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T21:03:33.004 INFO:teuthology.orchestra.run.smithi026.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T21:03:33.004 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T21:03:33.004 INFO:teuthology.orchestra.run.smithi026.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T21:03:33.004 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T21:03:33.033 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:03:33.074 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 6, 7, 4, 5] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 7, 6] 2023-09-23T21:03:33.074 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T21:03:33.074 INFO:tasks.thrashosds.thrasher:primary_affinity 2023-09-23T21:03:33.074 INFO:tasks.thrashosds.thrasher:Setting osd 6 primary_affinity to 1.000000 2023-09-23T21:03:33.075 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph osd primary-affinity 6 1 2023-09-23T21:03:33.097 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:03:33.123 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:03:33.144 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:03:33.144 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:03:33.245 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:03:33.276 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:03:33.315 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:03:33.318 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:03:33.319 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:03:33.420 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:03:33.462 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:03:33.503 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:03:33.509 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:03:33.509 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:03:33.510 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:03:33 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/4188628811' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:03:33.575 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:03:33 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/4188628811' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:03:33.576 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:03:33 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/4188628811' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:03:33.610 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:03:33.647 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:03:33.689 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:03:33.694 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:03:33.694 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:03:33.795 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:03:33.833 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:03:33.875 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:03:33.882 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:03:33.882 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:03:33.983 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:03:34.024 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:03:34.071 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:03:34.076 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:03:34.077 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:03:34.178 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:03:34.239 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:03:34 smithi026 ceph-mon[175151]: pgmap v754: 119 pgs: 4 peering, 6 unknown, 109 active+clean; 5.4 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 932 op/s; 0 B/s, 0 objects/s recovering 2023-09-23T21:03:34.254 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:03:34.333 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:03:34.349 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T21:03:34.355 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:03:34.356 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:03:34.358 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:03:34.457 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:03:34.495 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:03:34 smithi070 ceph-mon[168282]: pgmap v754: 119 pgs: 4 peering, 6 unknown, 109 active+clean; 5.4 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 932 op/s; 0 B/s, 0 objects/s recovering 2023-09-23T21:03:34.497 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:03:34.540 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:03:34.544 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:03:34.545 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:03:34.589 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:03:34 smithi026 ceph-mon[179688]: pgmap v754: 119 pgs: 4 peering, 6 unknown, 109 active+clean; 5.4 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 932 op/s; 0 B/s, 0 objects/s recovering 2023-09-23T21:03:34.645 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:03:34.681 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:03:34.726 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:03:34.729 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:03:34.729 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:03:34.830 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:03:34.891 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:03:35.021 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:03:35.026 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:03:35.027 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:03:35.127 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:03:35.166 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:03:35.204 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:03:35.207 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:03:35.208 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:03:35.308 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:03:35.354 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:03:35.394 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:03:35.399 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:03:35.399 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:03:35.500 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:03:35.589 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:03:35.630 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T21:03:35.880 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T21:03:35.886 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:03:35.887 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:03:35.988 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:03:36.082 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:03:36.281 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T21:03:36.282 INFO:teuthology.orchestra.run.smithi026.stderr:noscrub is unset 2023-09-23T21:03:36.284 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:03:36.284 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:03:36.385 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:03:36.480 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:03:36.533 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:03:36 smithi026 ceph-mon[175151]: pgmap v755: 119 pgs: 1 peering, 1 unknown, 117 active+clean; 5.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 747 op/s; 0 B/s, 0 objects/s recovering 2023-09-23T21:03:36.533 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:03:36 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/2609585886' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T21:03:36.533 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:03:36 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T21:03:36.534 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:03:36 smithi026 ceph-mon[179688]: pgmap v755: 119 pgs: 1 peering, 1 unknown, 117 active+clean; 5.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 747 op/s; 0 B/s, 0 objects/s recovering 2023-09-23T21:03:36.534 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:03:36 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/2609585886' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T21:03:36.534 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:03:36 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T21:03:36.605 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:03:36.612 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:03:36.613 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:03:36.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:03:36 smithi070 ceph-mon[168282]: pgmap v755: 119 pgs: 1 peering, 1 unknown, 117 active+clean; 5.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 747 op/s; 0 B/s, 0 objects/s recovering 2023-09-23T21:03:36.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:03:36 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/2609585886' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T21:03:36.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:03:36 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T21:03:36.710 INFO:teuthology.orchestra.run.smithi026.stdout:ERROR: (22) Invalid argument 2023-09-23T21:03:36.710 INFO:teuthology.orchestra.run.smithi026.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T21:03:36.713 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:03:36.785 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:03:37.277 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:03:37 smithi026 ceph-mon[179688]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:03:37.278 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:03:37 smithi026 ceph-mon[179688]: failed to encode map e283 with expected crc 2023-09-23T21:03:37.278 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:03:37 smithi026 ceph-mon[179688]: failed to encode map e283 with expected crc 2023-09-23T21:03:37.278 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:03:37 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T21:03:37.278 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:03:37 smithi026 ceph-mon[179688]: osdmap e283: 8 total, 8 up, 8 in 2023-09-23T21:03:37.278 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:03:37 smithi026 ceph-mon[179688]: failed to encode map e283 with expected crc 2023-09-23T21:03:37.278 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:03:37 smithi026 ceph-mon[179688]: 8.9 scrub starts 2023-09-23T21:03:37.278 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:03:37 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pgp_num_actual", "val": "17"}]: dispatch 2023-09-23T21:03:37.278 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:03:37 smithi026 ceph-mon[179688]: 8.0 scrub starts 2023-09-23T21:03:37.278 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:03:37 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/1968592859' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 6, "weight": 1.0}]: dispatch 2023-09-23T21:03:37.278 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:03:37 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 6, "weight": 1.0}]: dispatch 2023-09-23T21:03:37.303 INFO:teuthology.orchestra.run.smithi026.stderr:set osd.6 primary-affinity to 1 (8655362) 2023-09-23T21:03:37.488 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_historic_ops 2023-09-23T21:03:37.542 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:03:37 smithi026 ceph-mon[175151]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:03:37.542 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:03:37 smithi026 ceph-mon[175151]: failed to encode map e283 with expected crc 2023-09-23T21:03:37.542 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:03:37 smithi026 ceph-mon[175151]: failed to encode map e283 with expected crc 2023-09-23T21:03:37.542 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:03:37 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T21:03:37.542 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:03:37 smithi026 ceph-mon[175151]: osdmap e283: 8 total, 8 up, 8 in 2023-09-23T21:03:37.542 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:03:37 smithi026 ceph-mon[175151]: failed to encode map e283 with expected crc 2023-09-23T21:03:37.542 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:03:37 smithi026 ceph-mon[175151]: 8.9 scrub starts 2023-09-23T21:03:37.542 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:03:37 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pgp_num_actual", "val": "17"}]: dispatch 2023-09-23T21:03:37.543 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:03:37 smithi026 ceph-mon[175151]: 8.0 scrub starts 2023-09-23T21:03:37.543 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:03:37 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/1968592859' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 6, "weight": 1.0}]: dispatch 2023-09-23T21:03:37.543 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:03:37 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 6, "weight": 1.0}]: dispatch 2023-09-23T21:03:37.544 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:03:37.552 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:03:37.552 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:03:37.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:03:37 smithi070 ceph-mon[168282]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:03:37.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:03:37 smithi070 ceph-mon[168282]: failed to encode map e283 with expected crc 2023-09-23T21:03:37.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:03:37 smithi070 ceph-mon[168282]: failed to encode map e283 with expected crc 2023-09-23T21:03:37.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:03:37 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T21:03:37.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:03:37 smithi070 ceph-mon[168282]: osdmap e283: 8 total, 8 up, 8 in 2023-09-23T21:03:37.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:03:37 smithi070 ceph-mon[168282]: failed to encode map e283 with expected crc 2023-09-23T21:03:37.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:03:37 smithi070 ceph-mon[168282]: 8.9 scrub starts 2023-09-23T21:03:37.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:03:37 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pgp_num_actual", "val": "17"}]: dispatch 2023-09-23T21:03:37.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:03:37 smithi070 ceph-mon[168282]: 8.0 scrub starts 2023-09-23T21:03:37.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:03:37 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/1968592859' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 6, "weight": 1.0}]: dispatch 2023-09-23T21:03:37.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:03:37 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 6, "weight": 1.0}]: dispatch 2023-09-23T21:03:37.653 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:03:37.703 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:03:37.769 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T21:03:37.773 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:03:37.774 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:03:37.875 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:03:37.902 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:03:37.908 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:03:37.936 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:03:37.939 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:03:37.939 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:03:38.040 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:03:38.068 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:03:38.153 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:03:38.160 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:03:38.160 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:03:38.261 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:03:38.303 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:03:38.338 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:03:38 smithi026 ceph-mon[175151]: pgmap v757: 119 pgs: 119 active+clean; 5.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 760 op/s; 0 B/s, 0 objects/s recovering 2023-09-23T21:03:38.338 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:03:38 smithi026 ceph-mon[175151]: 8.9 scrub ok 2023-09-23T21:03:38.338 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:03:38 smithi026 ceph-mon[175151]: failed to encode map e284 with expected crc 2023-09-23T21:03:38.338 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:03:38 smithi026 ceph-mon[175151]: failed to encode map e284 with expected crc 2023-09-23T21:03:38.338 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:03:38 smithi026 ceph-mon[175151]: failed to encode map e284 with expected crc 2023-09-23T21:03:38.338 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:03:38 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pgp_num_actual", "val": "17"}]': finished 2023-09-23T21:03:38.338 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:03:38 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 6, "weight": 1.0}]': finished 2023-09-23T21:03:38.339 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:03:38 smithi026 ceph-mon[175151]: osdmap e284: 8 total, 8 up, 8 in 2023-09-23T21:03:38.339 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:03:38 smithi026 ceph-mon[175151]: 8.5 scrub starts 2023-09-23T21:03:38.339 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:03:38 smithi026 ceph-mon[175151]: 8.4 scrub starts 2023-09-23T21:03:38.339 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:03:38 smithi026 ceph-mon[179688]: pgmap v757: 119 pgs: 119 active+clean; 5.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 760 op/s; 0 B/s, 0 objects/s recovering 2023-09-23T21:03:38.339 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:03:38 smithi026 ceph-mon[179688]: 8.9 scrub ok 2023-09-23T21:03:38.340 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:03:38 smithi026 ceph-mon[179688]: failed to encode map e284 with expected crc 2023-09-23T21:03:38.340 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:03:38 smithi026 ceph-mon[179688]: failed to encode map e284 with expected crc 2023-09-23T21:03:38.340 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:03:38 smithi026 ceph-mon[179688]: failed to encode map e284 with expected crc 2023-09-23T21:03:38.340 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:03:38 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pgp_num_actual", "val": "17"}]': finished 2023-09-23T21:03:38.340 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:03:38 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 6, "weight": 1.0}]': finished 2023-09-23T21:03:38.340 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:03:38 smithi026 ceph-mon[179688]: osdmap e284: 8 total, 8 up, 8 in 2023-09-23T21:03:38.340 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:03:38 smithi026 ceph-mon[179688]: 8.5 scrub starts 2023-09-23T21:03:38.340 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:03:38 smithi026 ceph-mon[179688]: 8.4 scrub starts 2023-09-23T21:03:38.369 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:03:38.395 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:03:38.399 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:03:38.399 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:03:38.500 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:03:38.529 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:03:38.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:03:38 smithi070 ceph-mon[168282]: pgmap v757: 119 pgs: 119 active+clean; 5.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 760 op/s; 0 B/s, 0 objects/s recovering 2023-09-23T21:03:38.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:03:38 smithi070 ceph-mon[168282]: 8.9 scrub ok 2023-09-23T21:03:38.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:03:38 smithi070 ceph-mon[168282]: failed to encode map e284 with expected crc 2023-09-23T21:03:38.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:03:38 smithi070 ceph-mon[168282]: failed to encode map e284 with expected crc 2023-09-23T21:03:38.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:03:38 smithi070 ceph-mon[168282]: failed to encode map e284 with expected crc 2023-09-23T21:03:38.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:03:38 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pgp_num_actual", "val": "17"}]': finished 2023-09-23T21:03:38.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:03:38 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 6, "weight": 1.0}]': finished 2023-09-23T21:03:38.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:03:38 smithi070 ceph-mon[168282]: osdmap e284: 8 total, 8 up, 8 in 2023-09-23T21:03:38.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:03:38 smithi070 ceph-mon[168282]: 8.5 scrub starts 2023-09-23T21:03:38.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:03:38 smithi070 ceph-mon[168282]: 8.4 scrub starts 2023-09-23T21:03:38.674 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:03:38.677 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:03:38.677 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:03:38.778 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:03:38.804 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:03:38.836 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:03:38.838 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:03:38.839 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:03:38.939 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:03:38.969 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:03:39.011 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:03:39.014 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:03:39.015 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:03:39.115 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:03:39.141 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:03:39.172 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:03:39.175 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:03:39.175 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:03:39.276 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:03:39.314 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:03:39.548 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0: {} 2023-09-23T21:03:39.575 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1: {} 2023-09-23T21:03:39.603 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:03:39.606 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2: {} 2023-09-23T21:03:39.608 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:03:39.609 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:03:39.609 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:03:39 smithi026 ceph-mon[175151]: 8.5 scrub ok 2023-09-23T21:03:39.609 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:03:39 smithi026 ceph-mon[175151]: 8.1 scrub starts 2023-09-23T21:03:39.610 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:03:39 smithi026 ceph-mon[175151]: failed to encode map e285 with expected crc 2023-09-23T21:03:39.610 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:03:39 smithi026 ceph-mon[175151]: failed to encode map e285 with expected crc 2023-09-23T21:03:39.610 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:03:39 smithi026 ceph-mon[175151]: osdmap e285: 8 total, 8 up, 8 in 2023-09-23T21:03:39.610 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:03:39 smithi026 ceph-mon[175151]: failed to encode map e285 with expected crc 2023-09-23T21:03:39.610 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:03:39 smithi026 ceph-mon[175151]: 8.1 scrub ok 2023-09-23T21:03:39.610 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:03:39 smithi026 ceph-mon[175151]: 8.6 scrub starts 2023-09-23T21:03:39.610 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:03:39 smithi026 ceph-mon[175151]: 8.8 scrub starts 2023-09-23T21:03:39.610 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:03:39 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pgp_num_actual", "val": "18"}]: dispatch 2023-09-23T21:03:39.610 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:03:39 smithi026 ceph-mon[175151]: 8.0 scrub starts 2023-09-23T21:03:39.610 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:03:39 smithi026 ceph-mon[175151]: 8.2 scrub starts 2023-09-23T21:03:39.611 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:03:39 smithi026 ceph-mon[179688]: 8.5 scrub ok 2023-09-23T21:03:39.611 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:03:39 smithi026 ceph-mon[179688]: 8.1 scrub starts 2023-09-23T21:03:39.611 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:03:39 smithi026 ceph-mon[179688]: failed to encode map e285 with expected crc 2023-09-23T21:03:39.611 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:03:39 smithi026 ceph-mon[179688]: failed to encode map e285 with expected crc 2023-09-23T21:03:39.611 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:03:39 smithi026 ceph-mon[179688]: osdmap e285: 8 total, 8 up, 8 in 2023-09-23T21:03:39.611 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:03:39 smithi026 ceph-mon[179688]: failed to encode map e285 with expected crc 2023-09-23T21:03:39.611 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:03:39 smithi026 ceph-mon[179688]: 8.1 scrub ok 2023-09-23T21:03:39.611 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:03:39 smithi026 ceph-mon[179688]: 8.6 scrub starts 2023-09-23T21:03:39.612 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:03:39 smithi026 ceph-mon[179688]: 8.8 scrub starts 2023-09-23T21:03:39.612 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:03:39 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pgp_num_actual", "val": "18"}]: dispatch 2023-09-23T21:03:39.612 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:03:39 smithi026 ceph-mon[179688]: 8.0 scrub starts 2023-09-23T21:03:39.612 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:03:39 smithi026 ceph-mon[179688]: 8.2 scrub starts 2023-09-23T21:03:39.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:03:39 smithi070 ceph-mon[168282]: 8.5 scrub ok 2023-09-23T21:03:39.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:03:39 smithi070 ceph-mon[168282]: 8.1 scrub starts 2023-09-23T21:03:39.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:03:39 smithi070 ceph-mon[168282]: failed to encode map e285 with expected crc 2023-09-23T21:03:39.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:03:39 smithi070 ceph-mon[168282]: failed to encode map e285 with expected crc 2023-09-23T21:03:39.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:03:39 smithi070 ceph-mon[168282]: osdmap e285: 8 total, 8 up, 8 in 2023-09-23T21:03:39.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:03:39 smithi070 ceph-mon[168282]: failed to encode map e285 with expected crc 2023-09-23T21:03:39.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:03:39 smithi070 ceph-mon[168282]: 8.1 scrub ok 2023-09-23T21:03:39.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:03:39 smithi070 ceph-mon[168282]: 8.6 scrub starts 2023-09-23T21:03:39.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:03:39 smithi070 ceph-mon[168282]: 8.8 scrub starts 2023-09-23T21:03:39.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:03:39 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pgp_num_actual", "val": "18"}]: dispatch 2023-09-23T21:03:39.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:03:39 smithi070 ceph-mon[168282]: 8.0 scrub starts 2023-09-23T21:03:39.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:03:39 smithi070 ceph-mon[168282]: 8.2 scrub starts 2023-09-23T21:03:39.640 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3: {} 2023-09-23T21:03:39.665 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4: {} 2023-09-23T21:03:39.668 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T21:03:39.694 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T21:03:39.710 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:03:39.722 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6: {} 2023-09-23T21:03:39.746 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:03:39.747 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T21:03:39.750 INFO:teuthology.orchestra.run.smithi026.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T21:03:39.750 INFO:teuthology.orchestra.run.smithi026.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T21:03:39.750 INFO:teuthology.orchestra.run.smithi026.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T21:03:39.750 INFO:teuthology.orchestra.run.smithi026.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T21:03:39.750 INFO:teuthology.orchestra.run.smithi026.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T21:03:39.750 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T21:03:39.750 INFO:teuthology.orchestra.run.smithi026.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T21:03:39.750 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T21:03:39.789 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:03:39.792 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:03:39.792 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:03:39.858 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 21:03:39 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[164029]: ::ffff:172.21.15.70 - - [23/Sep/2023:21:03:39] "GET /metrics HTTP/1.1" 200 36296 "" "Prometheus/2.43.0" 2023-09-23T21:03:39.893 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:03:39.925 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:03:40.100 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T21:03:40.104 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:03:40.105 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:03:40.205 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:03:40.235 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:03:40.320 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:03:40.323 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:03:40.323 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:03:40.424 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:03:40.454 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:03:40.494 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:03:40.496 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:03:40.497 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:03:40.505 INFO:teuthology.orchestra.run.smithi026.stdout:{ 2023-09-23T21:03:40.505 INFO:teuthology.orchestra.run.smithi026.stdout: "size": 20, 2023-09-23T21:03:40.505 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 600, 2023-09-23T21:03:40.505 INFO:teuthology.orchestra.run.smithi026.stdout: "ops": [ 2023-09-23T21:03:40.505 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.505 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34797.0:114960 6.17 6:edae14e8:::benchmark_data_smithi026_223868_object13834:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e131)", 2023-09-23T21:03:40.506 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T20:54:12.901858+0000", 2023-09-23T21:03:40.506 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 567.60031870499995, 2023-09-23T21:03:40.506 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.1321539249999999, 2023-09-23T21:03:40.506 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:03:40.506 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "reached pg", 2023-09-23T21:03:40.506 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T21:03:40.506 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34797", 2023-09-23T21:03:40.506 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T21:03:40.506 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 114960 2023-09-23T21:03:40.506 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.506 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:03:40.506 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.507 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:03:40.507 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:12.901858+0000", 2023-09-23T21:03:40.507 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:03:40.507 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.507 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.507 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:03:40.507 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:12.901858+0000", 2023-09-23T21:03:40.507 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:03:40.507 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.507 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.507 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:03:40.507 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:12.901857+0000", 2023-09-23T21:03:40.508 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T21:03:40.508 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.508 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.508 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:03:40.508 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:12.901863+0000", 2023-09-23T21:03:40.508 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.3720000000000001e-06 2023-09-23T21:03:40.508 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.508 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.508 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:03:40.508 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:12.901875+0000", 2023-09-23T21:03:40.508 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1924e-05 2023-09-23T21:03:40.508 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.508 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.509 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:03:40.509 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:12.901883+0000", 2023-09-23T21:03:40.509 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.038e-06 2023-09-23T21:03:40.509 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.509 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.509 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:03:40.509 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:12.904218+0000", 2023-09-23T21:03:40.509 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0023356050000000001 2023-09-23T21:03:40.509 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.509 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.509 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for degraded object", 2023-09-23T21:03:40.509 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:12.909234+0000", 2023-09-23T21:03:40.510 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0050151680000000004 2023-09-23T21:03:40.510 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.510 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.510 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:03:40.510 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:12.927966+0000", 2023-09-23T21:03:40.510 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.018731945999999999 2023-09-23T21:03:40.510 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.510 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.510 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:03:40.510 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:12.928078+0000", 2023-09-23T21:03:40.510 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000112485 2023-09-23T21:03:40.510 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.510 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.511 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 1,7", 2023-09-23T21:03:40.511 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:12.928391+0000", 2023-09-23T21:03:40.511 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00031247000000000002 2023-09-23T21:03:40.511 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.511 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.511 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:03:40.511 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:12.951689+0000", 2023-09-23T21:03:40.511 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.023298762000000001 2023-09-23T21:03:40.511 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.511 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.511 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T21:03:40.511 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:12.951697+0000", 2023-09-23T21:03:40.511 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.8189999999999992e-06 2023-09-23T21:03:40.512 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.512 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.512 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:03:40.512 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:13.925895+0000", 2023-09-23T21:03:40.512 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.97419815399999998 2023-09-23T21:03:40.512 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.512 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.512 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T21:03:40.512 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:13.925906+0000", 2023-09-23T21:03:40.512 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0319000000000001e-05 2023-09-23T21:03:40.512 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.512 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.513 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:03:40.513 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:15.017793+0000", 2023-09-23T21:03:40.513 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0918876260000001 2023-09-23T21:03:40.513 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.513 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.513 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for missing object", 2023-09-23T21:03:40.513 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:15.017905+0000", 2023-09-23T21:03:40.513 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000112256 2023-09-23T21:03:40.513 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.513 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.513 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:03:40.513 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:15.033919+0000", 2023-09-23T21:03:40.513 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.016013263999999999 2023-09-23T21:03:40.514 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.514 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.514 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T21:03:40.514 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:15.034012+0000", 2023-09-23T21:03:40.514 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.3101000000000002e-05 2023-09-23T21:03:40.514 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:03:40.514 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:03:40.514 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:03:40.514 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.514 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.514 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34797.0:114965 6.17 6:eed136d7:::benchmark_data_smithi026_223868_object13839:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e131)", 2023-09-23T21:03:40.514 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T20:54:12.907032+0000", 2023-09-23T21:03:40.514 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 567.595144796, 2023-09-23T21:03:40.515 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.1278061450000001, 2023-09-23T21:03:40.515 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:03:40.515 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:03:40.515 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T21:03:40.515 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34797", 2023-09-23T21:03:40.515 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T21:03:40.515 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 114965 2023-09-23T21:03:40.515 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.515 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:03:40.515 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.515 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:03:40.515 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:12.907032+0000", 2023-09-23T21:03:40.516 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:03:40.516 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.516 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.516 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:03:40.516 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:12.907032+0000", 2023-09-23T21:03:40.516 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:03:40.516 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.516 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.516 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:03:40.516 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:12.907030+0000", 2023-09-23T21:03:40.516 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T21:03:40.516 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.516 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.517 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:03:40.517 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:12.907038+0000", 2023-09-23T21:03:40.517 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.3350000000000001e-06 2023-09-23T21:03:40.517 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.517 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.517 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:03:40.517 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:12.907050+0000", 2023-09-23T21:03:40.517 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.2398999999999999e-05 2023-09-23T21:03:40.517 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.517 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.517 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:03:40.517 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:12.907058+0000", 2023-09-23T21:03:40.517 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.1820000000000006e-06 2023-09-23T21:03:40.518 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.518 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.518 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:03:40.518 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:12.909340+0000", 2023-09-23T21:03:40.518 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.002281731 2023-09-23T21:03:40.518 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.518 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.518 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for degraded object", 2023-09-23T21:03:40.518 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:12.918086+0000", 2023-09-23T21:03:40.518 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.008745971 2023-09-23T21:03:40.518 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.518 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.519 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:03:40.519 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:12.952195+0000", 2023-09-23T21:03:40.519 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.034108861999999997 2023-09-23T21:03:40.519 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.519 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.519 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T21:03:40.519 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:12.952202+0000", 2023-09-23T21:03:40.519 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.686e-06 2023-09-23T21:03:40.519 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.519 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.519 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:03:40.519 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:13.925935+0000", 2023-09-23T21:03:40.519 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.97373368500000002 2023-09-23T21:03:40.520 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.520 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.520 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T21:03:40.520 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:13.925944+0000", 2023-09-23T21:03:40.520 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.9530000000000005e-06 2023-09-23T21:03:40.520 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.520 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.520 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:03:40.520 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:15.017933+0000", 2023-09-23T21:03:40.520 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0919890029999999 2023-09-23T21:03:40.520 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.520 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.521 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:03:40.521 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:15.018041+0000", 2023-09-23T21:03:40.521 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000107551 2023-09-23T21:03:40.521 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.521 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.521 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 1,4", 2023-09-23T21:03:40.521 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:15.018366+0000", 2023-09-23T21:03:40.521 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00032571500000000002 2023-09-23T21:03:40.521 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.521 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.521 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:03:40.521 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:15.031011+0000", 2023-09-23T21:03:40.521 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.012644479 2023-09-23T21:03:40.522 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.522 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.522 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:03:40.522 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:15.031048+0000", 2023-09-23T21:03:40.522 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.6822000000000001e-05 2023-09-23T21:03:40.522 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.522 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.522 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T21:03:40.522 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:15.034520+0000", 2023-09-23T21:03:40.522 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0034718459999999998 2023-09-23T21:03:40.522 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.522 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.522 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T21:03:40.523 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:15.034571+0000", 2023-09-23T21:03:40.523 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.1898999999999999e-05 2023-09-23T21:03:40.523 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.523 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.523 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T21:03:40.523 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:15.034838+0000", 2023-09-23T21:03:40.523 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00026648299999999998 2023-09-23T21:03:40.523 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:03:40.523 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:03:40.523 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:03:40.523 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.523 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.524 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34797.0:114977 6.16 6:6abc4ce0:::benchmark_data_smithi026_223868_object13851:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e131)", 2023-09-23T21:03:40.524 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T20:54:12.931076+0000", 2023-09-23T21:03:40.524 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 567.571101043, 2023-09-23T21:03:40.524 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.0790598340000002, 2023-09-23T21:03:40.524 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:03:40.524 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:03:40.524 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T21:03:40.524 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34797", 2023-09-23T21:03:40.524 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T21:03:40.524 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 114977 2023-09-23T21:03:40.524 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.524 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:03:40.525 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.525 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:03:40.525 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:12.931076+0000", 2023-09-23T21:03:40.525 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:03:40.525 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.525 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.525 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:03:40.525 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:12.931076+0000", 2023-09-23T21:03:40.525 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:03:40.525 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.525 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.525 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:03:40.525 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:12.931075+0000", 2023-09-23T21:03:40.526 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T21:03:40.526 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.526 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.526 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:03:40.526 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:12.931079+0000", 2023-09-23T21:03:40.526 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.5200000000000002e-06 2023-09-23T21:03:40.526 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.526 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.526 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:03:40.526 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:12.931086+0000", 2023-09-23T21:03:40.526 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.0299999999999996e-06 2023-09-23T21:03:40.526 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.526 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.527 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:03:40.527 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:12.931108+0000", 2023-09-23T21:03:40.527 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.2226999999999999e-05 2023-09-23T21:03:40.527 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.527 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.527 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:03:40.527 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:12.931257+0000", 2023-09-23T21:03:40.527 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00014939900000000001 2023-09-23T21:03:40.527 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.527 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.527 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T21:03:40.527 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:12.931281+0000", 2023-09-23T21:03:40.527 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.4057e-05 2023-09-23T21:03:40.528 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.528 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.528 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:03:40.528 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:13.925102+0000", 2023-09-23T21:03:40.528 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.993820284 2023-09-23T21:03:40.528 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.528 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.528 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T21:03:40.528 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:13.925116+0000", 2023-09-23T21:03:40.528 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.3886000000000001e-05 2023-09-23T21:03:40.528 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.528 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.529 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:03:40.529 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:15.000774+0000", 2023-09-23T21:03:40.529 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0756585869999999 2023-09-23T21:03:40.529 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.529 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.529 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:03:40.529 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:15.000859+0000", 2023-09-23T21:03:40.529 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.5370000000000004e-05 2023-09-23T21:03:40.529 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.529 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.529 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 4,7", 2023-09-23T21:03:40.529 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:15.001678+0000", 2023-09-23T21:03:40.529 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000818913 2023-09-23T21:03:40.530 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.530 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.530 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:03:40.530 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:15.008665+0000", 2023-09-23T21:03:40.530 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0069865600000000002 2023-09-23T21:03:40.530 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.530 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.530 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:03:40.530 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:15.008706+0000", 2023-09-23T21:03:40.530 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.1285999999999997e-05 2023-09-23T21:03:40.530 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.530 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.530 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T21:03:40.531 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:15.009602+0000", 2023-09-23T21:03:40.531 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00089599299999999998 2023-09-23T21:03:40.531 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.531 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.531 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T21:03:40.531 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:15.009659+0000", 2023-09-23T21:03:40.531 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.6289999999999998e-05 2023-09-23T21:03:40.531 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.531 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.531 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T21:03:40.531 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:15.010135+0000", 2023-09-23T21:03:40.531 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00047690699999999999 2023-09-23T21:03:40.531 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:03:40.532 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:03:40.532 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:03:40.532 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.532 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.532 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34797.0:114989 6.6 6:60ed2ddf:::benchmark_data_smithi026_223868_object13863:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e132)", 2023-09-23T21:03:40.532 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T20:54:12.964779+0000", 2023-09-23T21:03:40.532 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 567.53739734400006, 2023-09-23T21:03:40.532 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.0443830630000002, 2023-09-23T21:03:40.532 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:03:40.532 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:03:40.532 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T21:03:40.532 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34797", 2023-09-23T21:03:40.533 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T21:03:40.533 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 114989 2023-09-23T21:03:40.533 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.533 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:03:40.533 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.533 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:03:40.533 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:12.964779+0000", 2023-09-23T21:03:40.533 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:03:40.533 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.533 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.533 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:03:40.533 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:12.964779+0000", 2023-09-23T21:03:40.533 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:03:40.534 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.534 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.534 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:03:40.534 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:12.964778+0000", 2023-09-23T21:03:40.534 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T21:03:40.534 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.534 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.534 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:03:40.534 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:12.964784+0000", 2023-09-23T21:03:40.534 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.8989999999999998e-06 2023-09-23T21:03:40.534 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.534 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.534 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:03:40.535 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:12.964795+0000", 2023-09-23T21:03:40.535 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1214e-05 2023-09-23T21:03:40.535 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.535 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.535 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:03:40.535 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:12.964805+0000", 2023-09-23T21:03:40.535 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0346999999999999e-05 2023-09-23T21:03:40.535 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.535 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.535 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:03:40.535 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:12.964859+0000", 2023-09-23T21:03:40.535 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.3591000000000002e-05 2023-09-23T21:03:40.535 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.536 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.536 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T21:03:40.536 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:12.964871+0000", 2023-09-23T21:03:40.536 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.2109999999999999e-05 2023-09-23T21:03:40.536 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.536 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.536 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:03:40.536 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:13.925234+0000", 2023-09-23T21:03:40.536 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.96036344399999996 2023-09-23T21:03:40.536 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.536 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.536 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T21:03:40.537 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:13.925245+0000", 2023-09-23T21:03:40.537 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0054999999999999e-05 2023-09-23T21:03:40.537 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.537 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.537 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:03:40.537 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:14.996272+0000", 2023-09-23T21:03:40.537 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0710279009999999 2023-09-23T21:03:40.537 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.537 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.537 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:03:40.537 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:14.996437+0000", 2023-09-23T21:03:40.537 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00016484299999999999 2023-09-23T21:03:40.537 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.538 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.538 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 4,7", 2023-09-23T21:03:40.538 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:14.996837+0000", 2023-09-23T21:03:40.538 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00039951599999999999 2023-09-23T21:03:40.538 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.538 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.538 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:03:40.538 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:14.998787+0000", 2023-09-23T21:03:40.538 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0019506770000000001 2023-09-23T21:03:40.538 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.538 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.538 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:03:40.538 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:15.001358+0000", 2023-09-23T21:03:40.539 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0025709750000000001 2023-09-23T21:03:40.539 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.539 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.539 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T21:03:40.539 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:15.008625+0000", 2023-09-23T21:03:40.539 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0072665230000000004 2023-09-23T21:03:40.539 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.539 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.539 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T21:03:40.539 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:15.008685+0000", 2023-09-23T21:03:40.539 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.0244000000000001e-05 2023-09-23T21:03:40.539 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.539 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.540 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T21:03:40.540 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:15.009162+0000", 2023-09-23T21:03:40.540 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00047711299999999998 2023-09-23T21:03:40.540 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:03:40.540 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:03:40.540 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:03:40.540 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.540 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.540 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34797.0:114991 6.17 6:ed158f03:::benchmark_data_smithi026_223868_object13865:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e132)", 2023-09-23T21:03:40.540 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T20:54:12.968317+0000", 2023-09-23T21:03:40.540 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 567.53385913199998, 2023-09-23T21:03:40.540 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.0674853450000001, 2023-09-23T21:03:40.540 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:03:40.541 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:03:40.541 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T21:03:40.541 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34797", 2023-09-23T21:03:40.541 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T21:03:40.541 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 114991 2023-09-23T21:03:40.541 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.541 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:03:40.541 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.541 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:03:40.541 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:12.968317+0000", 2023-09-23T21:03:40.541 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:03:40.541 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.541 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.542 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:03:40.542 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:12.968317+0000", 2023-09-23T21:03:40.542 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:03:40.542 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.542 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.542 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:03:40.542 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:12.968316+0000", 2023-09-23T21:03:40.542 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T21:03:40.542 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.542 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.542 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:03:40.542 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:12.968321+0000", 2023-09-23T21:03:40.543 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.2669999999999997e-06 2023-09-23T21:03:40.543 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.543 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.543 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:03:40.543 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:12.968334+0000", 2023-09-23T21:03:40.543 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.3101999999999999e-05 2023-09-23T21:03:40.543 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.543 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.543 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:03:40.543 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:12.968342+0000", 2023-09-23T21:03:40.543 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.5089999999999996e-06 2023-09-23T21:03:40.543 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.543 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.544 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:03:40.544 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:12.968406+0000", 2023-09-23T21:03:40.544 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.4132000000000002e-05 2023-09-23T21:03:40.544 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.544 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.544 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T21:03:40.544 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:12.968418+0000", 2023-09-23T21:03:40.544 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.2459999999999999e-05 2023-09-23T21:03:40.544 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.544 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.544 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:03:40.544 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:13.925969+0000", 2023-09-23T21:03:40.544 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.95755082800000002 2023-09-23T21:03:40.545 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.545 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.545 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T21:03:40.545 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:13.925978+0000", 2023-09-23T21:03:40.545 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.0149999999999992e-06 2023-09-23T21:03:40.545 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.545 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.545 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:03:40.545 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:15.019699+0000", 2023-09-23T21:03:40.545 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0937209290000001 2023-09-23T21:03:40.545 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.545 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.546 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:03:40.546 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:15.019848+0000", 2023-09-23T21:03:40.546 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00014907899999999999 2023-09-23T21:03:40.546 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.546 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.546 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 1,4", 2023-09-23T21:03:40.546 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:15.020197+0000", 2023-09-23T21:03:40.546 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000348964 2023-09-23T21:03:40.546 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.546 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.546 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:03:40.546 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:15.033098+0000", 2023-09-23T21:03:40.546 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.012900327 2023-09-23T21:03:40.547 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.547 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.547 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T21:03:40.547 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:15.034847+0000", 2023-09-23T21:03:40.547 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0017495 2023-09-23T21:03:40.547 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.547 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.547 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:03:40.547 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:15.035482+0000", 2023-09-23T21:03:40.547 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00063472799999999998 2023-09-23T21:03:40.547 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.547 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.547 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T21:03:40.548 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:15.035528+0000", 2023-09-23T21:03:40.548 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.5874000000000002e-05 2023-09-23T21:03:40.548 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.548 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.548 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T21:03:40.548 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:15.035803+0000", 2023-09-23T21:03:40.548 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00027521799999999998 2023-09-23T21:03:40.548 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:03:40.548 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:03:40.548 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:03:40.548 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.548 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.549 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34797.0:114998 6.7 6:e21330c6:::benchmark_data_smithi026_223868_object13872:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e132)", 2023-09-23T21:03:40.549 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T20:54:12.985057+0000", 2023-09-23T21:03:40.549 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 567.51711942999998, 2023-09-23T21:03:40.549 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.0503685319999998, 2023-09-23T21:03:40.549 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:03:40.549 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:03:40.549 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T21:03:40.549 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34797", 2023-09-23T21:03:40.549 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T21:03:40.549 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 114998 2023-09-23T21:03:40.549 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.549 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:03:40.549 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.550 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:03:40.550 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:12.985057+0000", 2023-09-23T21:03:40.550 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:03:40.550 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.550 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.550 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:03:40.550 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:12.985057+0000", 2023-09-23T21:03:40.550 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:03:40.550 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.550 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.550 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:03:40.550 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:12.985054+0000", 2023-09-23T21:03:40.551 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999971 2023-09-23T21:03:40.551 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.551 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.551 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:03:40.551 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:12.985066+0000", 2023-09-23T21:03:40.551 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.2031000000000001e-05 2023-09-23T21:03:40.551 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.551 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.551 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:03:40.551 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:12.985089+0000", 2023-09-23T21:03:40.551 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.2501e-05 2023-09-23T21:03:40.551 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.551 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.552 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:03:40.552 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:12.985103+0000", 2023-09-23T21:03:40.552 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.4392e-05 2023-09-23T21:03:40.552 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.552 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.552 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:03:40.552 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:12.988980+0000", 2023-09-23T21:03:40.552 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0038764509999999999 2023-09-23T21:03:40.552 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.552 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.552 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T21:03:40.552 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:12.988996+0000", 2023-09-23T21:03:40.553 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.5953000000000001e-05 2023-09-23T21:03:40.553 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.553 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.553 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:03:40.553 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:13.925849+0000", 2023-09-23T21:03:40.553 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.93685326800000002 2023-09-23T21:03:40.553 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.553 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.553 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T21:03:40.553 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:13.925860+0000", 2023-09-23T21:03:40.553 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1090000000000001e-05 2023-09-23T21:03:40.554 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.554 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.554 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:03:40.554 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:15.019676+0000", 2023-09-23T21:03:40.554 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0938154149999999 2023-09-23T21:03:40.554 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.554 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.554 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:03:40.554 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:15.019809+0000", 2023-09-23T21:03:40.554 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00013316599999999999 2023-09-23T21:03:40.554 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.555 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.555 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 1,4", 2023-09-23T21:03:40.555 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:15.020197+0000", 2023-09-23T21:03:40.555 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00038824999999999998 2023-09-23T21:03:40.555 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.555 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.555 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:03:40.555 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:15.033893+0000", 2023-09-23T21:03:40.555 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.013696442 2023-09-23T21:03:40.555 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.555 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.555 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T21:03:40.556 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:15.034854+0000", 2023-09-23T21:03:40.556 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00096095700000000002 2023-09-23T21:03:40.556 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.556 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.556 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:03:40.556 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:15.034911+0000", 2023-09-23T21:03:40.556 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.6915e-05 2023-09-23T21:03:40.556 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.556 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.556 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T21:03:40.556 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:15.034967+0000", 2023-09-23T21:03:40.556 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.6249000000000003e-05 2023-09-23T21:03:40.557 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.557 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.557 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T21:03:40.557 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:15.035426+0000", 2023-09-23T21:03:40.557 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00045822899999999999 2023-09-23T21:03:40.557 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:03:40.557 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:03:40.557 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:03:40.557 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.557 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.557 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34797.0:115009 6.7 6:e365786c:::benchmark_data_smithi026_223868_object13883:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e133)", 2023-09-23T21:03:40.557 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T20:54:14.026745+0000", 2023-09-23T21:03:40.557 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 566.47543145899999, 2023-09-23T21:03:40.558 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0094194009999999, 2023-09-23T21:03:40.558 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:03:40.558 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:03:40.558 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T21:03:40.558 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34797", 2023-09-23T21:03:40.558 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T21:03:40.558 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 115009 2023-09-23T21:03:40.558 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.558 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:03:40.558 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.558 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:03:40.558 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:14.026745+0000", 2023-09-23T21:03:40.559 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:03:40.559 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.559 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.559 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:03:40.559 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:14.026745+0000", 2023-09-23T21:03:40.559 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:03:40.559 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.559 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.559 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:03:40.559 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:14.026743+0000", 2023-09-23T21:03:40.559 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999981 2023-09-23T21:03:40.559 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.559 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.560 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:03:40.560 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:14.026752+0000", 2023-09-23T21:03:40.560 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.1670000000000005e-06 2023-09-23T21:03:40.560 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.560 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.560 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:03:40.560 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:14.026774+0000", 2023-09-23T21:03:40.560 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.1203e-05 2023-09-23T21:03:40.560 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.560 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.560 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:03:40.560 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:14.026782+0000", 2023-09-23T21:03:40.561 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.4370000000000007e-06 2023-09-23T21:03:40.561 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.561 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.561 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:03:40.561 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:14.026855+0000", 2023-09-23T21:03:40.561 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.2911999999999996e-05 2023-09-23T21:03:40.561 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.561 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.561 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T21:03:40.561 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:14.026870+0000", 2023-09-23T21:03:40.561 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.4718e-05 2023-09-23T21:03:40.561 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.561 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.562 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:03:40.562 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:15.023554+0000", 2023-09-23T21:03:40.562 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.99668419500000005 2023-09-23T21:03:40.562 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.562 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.562 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:03:40.562 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:15.023665+0000", 2023-09-23T21:03:40.562 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000110841 2023-09-23T21:03:40.562 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.562 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.562 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 1,4", 2023-09-23T21:03:40.562 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:15.024005+0000", 2023-09-23T21:03:40.562 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00034030399999999999 2023-09-23T21:03:40.563 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.563 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.563 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:03:40.563 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:15.034505+0000", 2023-09-23T21:03:40.563 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.010499812000000001 2023-09-23T21:03:40.563 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.563 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.563 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T21:03:40.563 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:15.034861+0000", 2023-09-23T21:03:40.563 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000355991 2023-09-23T21:03:40.563 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.563 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.564 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:03:40.564 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:15.035843+0000", 2023-09-23T21:03:40.564 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00098196299999999998 2023-09-23T21:03:40.564 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.564 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.564 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T21:03:40.564 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:15.035888+0000", 2023-09-23T21:03:40.564 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.4678000000000002e-05 2023-09-23T21:03:40.564 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.564 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.564 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T21:03:40.564 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:15.036165+0000", 2023-09-23T21:03:40.565 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00027704800000000001 2023-09-23T21:03:40.565 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:03:40.565 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:03:40.565 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:03:40.565 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.565 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.565 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34797.0:130469 6.c 6:38e631ce:::benchmark_data_smithi026_223868_object29343:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e135)", 2023-09-23T21:03:40.565 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T20:54:30.880090+0000", 2023-09-23T21:03:40.565 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 549.62208616600003, 2023-09-23T21:03:40.565 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.1674490940000002, 2023-09-23T21:03:40.565 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:03:40.565 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "reached pg", 2023-09-23T21:03:40.566 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T21:03:40.566 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34797", 2023-09-23T21:03:40.566 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T21:03:40.566 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 130469 2023-09-23T21:03:40.566 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.566 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:03:40.566 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.566 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:03:40.566 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:30.880090+0000", 2023-09-23T21:03:40.566 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:03:40.566 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.566 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.567 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:03:40.567 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:30.880090+0000", 2023-09-23T21:03:40.567 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:03:40.567 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.567 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.567 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:03:40.567 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:30.880088+0000", 2023-09-23T21:03:40.567 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999981 2023-09-23T21:03:40.567 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.567 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.567 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:03:40.567 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:30.880097+0000", 2023-09-23T21:03:40.568 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.4610000000000008e-06 2023-09-23T21:03:40.568 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.568 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.568 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:03:40.568 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:30.880115+0000", 2023-09-23T21:03:40.568 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.7901e-05 2023-09-23T21:03:40.568 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.568 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.568 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:03:40.568 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:30.880124+0000", 2023-09-23T21:03:40.568 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.3379999999999994e-06 2023-09-23T21:03:40.568 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.568 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.569 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:03:40.569 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:30.880185+0000", 2023-09-23T21:03:40.569 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.1262999999999999e-05 2023-09-23T21:03:40.569 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.569 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.569 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:03:40.569 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:30.880348+0000", 2023-09-23T21:03:40.569 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00016296799999999999 2023-09-23T21:03:40.569 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.569 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.569 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 6,7", 2023-09-23T21:03:40.569 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:30.880434+0000", 2023-09-23T21:03:40.569 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.5793999999999999e-05 2023-09-23T21:03:40.570 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.570 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.570 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T21:03:40.570 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:30.888682+0000", 2023-09-23T21:03:40.570 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0082474290000000006 2023-09-23T21:03:40.570 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.570 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.570 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:03:40.570 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:30.915595+0000", 2023-09-23T21:03:40.570 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.026913060999999999 2023-09-23T21:03:40.570 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.570 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.571 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T21:03:40.571 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:30.915608+0000", 2023-09-23T21:03:40.571 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.3742e-05 2023-09-23T21:03:40.571 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.571 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.571 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:03:40.571 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:31.887950+0000", 2023-09-23T21:03:40.571 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.97234175 2023-09-23T21:03:40.571 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.571 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.571 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T21:03:40.571 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:31.887964+0000", 2023-09-23T21:03:40.571 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.3711e-05 2023-09-23T21:03:40.572 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.572 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.572 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:03:40.572 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:33.036111+0000", 2023-09-23T21:03:40.572 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1481468340000001 2023-09-23T21:03:40.572 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.572 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.572 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for missing object", 2023-09-23T21:03:40.572 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:33.036258+0000", 2023-09-23T21:03:40.572 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00014725900000000001 2023-09-23T21:03:40.572 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.572 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.572 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:03:40.573 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:33.047449+0000", 2023-09-23T21:03:40.573 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.011190752 2023-09-23T21:03:40.573 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.573 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.573 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T21:03:40.573 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:33.047540+0000", 2023-09-23T21:03:40.573 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.0873999999999998e-05 2023-09-23T21:03:40.573 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:03:40.573 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:03:40.573 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:03:40.573 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.573 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.574 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34797.0:130480 6.c 6:3c6c8e98:::benchmark_data_smithi026_223868_object29354:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e135)", 2023-09-23T21:03:40.574 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T20:54:30.892126+0000", 2023-09-23T21:03:40.574 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 549.61005031299999, 2023-09-23T21:03:40.574 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.1590397989999999, 2023-09-23T21:03:40.574 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:03:40.574 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "reached pg", 2023-09-23T21:03:40.574 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T21:03:40.574 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34797", 2023-09-23T21:03:40.574 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T21:03:40.574 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 130480 2023-09-23T21:03:40.574 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.575 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:03:40.575 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.575 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:03:40.575 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:30.892126+0000", 2023-09-23T21:03:40.575 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:03:40.575 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.575 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.575 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:03:40.575 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:30.892126+0000", 2023-09-23T21:03:40.575 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:03:40.575 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.575 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.575 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:03:40.576 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:30.892125+0000", 2023-09-23T21:03:40.576 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T21:03:40.576 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.576 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.576 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:03:40.576 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:30.892131+0000", 2023-09-23T21:03:40.576 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.7209999999999997e-06 2023-09-23T21:03:40.576 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.576 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.576 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:03:40.576 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:30.892144+0000", 2023-09-23T21:03:40.576 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.2488e-05 2023-09-23T21:03:40.576 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.577 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.577 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:03:40.577 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:30.892152+0000", 2023-09-23T21:03:40.577 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.7330000000000003e-06 2023-09-23T21:03:40.577 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.577 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.577 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:03:40.577 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:30.892201+0000", 2023-09-23T21:03:40.577 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.9678000000000001e-05 2023-09-23T21:03:40.577 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.577 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.577 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:03:40.578 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:30.892324+0000", 2023-09-23T21:03:40.578 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00012292299999999999 2023-09-23T21:03:40.578 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.578 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.578 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 6,7", 2023-09-23T21:03:40.578 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:30.892438+0000", 2023-09-23T21:03:40.578 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000113915 2023-09-23T21:03:40.578 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.578 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.578 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T21:03:40.578 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:30.907285+0000", 2023-09-23T21:03:40.578 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.014847057 2023-09-23T21:03:40.579 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.579 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.579 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:03:40.579 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:30.915644+0000", 2023-09-23T21:03:40.579 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0083591480000000003 2023-09-23T21:03:40.579 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.579 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.579 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T21:03:40.579 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:30.915653+0000", 2023-09-23T21:03:40.579 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.4300000000000006e-06 2023-09-23T21:03:40.579 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.579 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.579 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:03:40.580 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:31.888064+0000", 2023-09-23T21:03:40.580 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.97241095300000002 2023-09-23T21:03:40.580 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.580 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.580 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T21:03:40.580 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:31.888069+0000", 2023-09-23T21:03:40.580 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.6450000000000002e-06 2023-09-23T21:03:40.580 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.580 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.580 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:03:40.580 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:33.037086+0000", 2023-09-23T21:03:40.580 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1490164059999999 2023-09-23T21:03:40.580 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.581 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.581 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for missing object", 2023-09-23T21:03:40.581 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:33.037184+0000", 2023-09-23T21:03:40.581 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.8024999999999997e-05 2023-09-23T21:03:40.581 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.581 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.581 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:03:40.581 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:33.051071+0000", 2023-09-23T21:03:40.581 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.013886895999999999 2023-09-23T21:03:40.581 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.581 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.581 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T21:03:40.582 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:33.051166+0000", 2023-09-23T21:03:40.582 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.5426000000000004e-05 2023-09-23T21:03:40.582 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:03:40.582 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:03:40.582 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:03:40.582 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.582 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.582 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34797.0:130484 6.17 6:ebb95d79:::benchmark_data_smithi026_223868_object29358:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e136)", 2023-09-23T21:03:40.582 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T20:54:30.909236+0000", 2023-09-23T21:03:40.582 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 549.59294066099994, 2023-09-23T21:03:40.582 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.1373689279999999, 2023-09-23T21:03:40.582 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:03:40.583 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:03:40.583 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T21:03:40.583 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34797", 2023-09-23T21:03:40.583 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T21:03:40.583 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 130484 2023-09-23T21:03:40.583 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.583 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:03:40.583 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.583 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:03:40.583 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:30.909236+0000", 2023-09-23T21:03:40.583 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:03:40.583 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.584 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.584 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:03:40.584 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:30.909236+0000", 2023-09-23T21:03:40.584 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:03:40.584 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.584 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.584 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:03:40.584 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:30.909234+0000", 2023-09-23T21:03:40.584 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999981 2023-09-23T21:03:40.584 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.584 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.584 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:03:40.584 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:30.909241+0000", 2023-09-23T21:03:40.585 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.9630000000000001e-06 2023-09-23T21:03:40.585 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.585 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.585 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:03:40.585 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:30.909258+0000", 2023-09-23T21:03:40.585 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.7105999999999999e-05 2023-09-23T21:03:40.585 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.585 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.585 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:03:40.585 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:30.909267+0000", 2023-09-23T21:03:40.585 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.0440000000000001e-06 2023-09-23T21:03:40.585 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.585 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.586 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:03:40.586 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:30.914297+0000", 2023-09-23T21:03:40.586 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0050297720000000001 2023-09-23T21:03:40.586 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.586 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.586 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T21:03:40.586 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:30.914307+0000", 2023-09-23T21:03:40.586 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0166000000000001e-05 2023-09-23T21:03:40.586 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.586 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.586 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:03:40.586 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:31.880008+0000", 2023-09-23T21:03:40.587 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.96570099399999998 2023-09-23T21:03:40.587 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.587 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.587 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T21:03:40.587 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:31.880021+0000", 2023-09-23T21:03:40.587 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.2700999999999999e-05 2023-09-23T21:03:40.587 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.587 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.587 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:03:40.587 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:33.017285+0000", 2023-09-23T21:03:40.587 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1372641450000001 2023-09-23T21:03:40.587 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.587 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.588 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:03:40.588 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:33.017380+0000", 2023-09-23T21:03:40.588 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.5118000000000006e-05 2023-09-23T21:03:40.588 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.588 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.588 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 1,4,5", 2023-09-23T21:03:40.588 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:33.017728+0000", 2023-09-23T21:03:40.588 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000348 2023-09-23T21:03:40.588 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.588 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.588 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:03:40.588 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:33.021064+0000", 2023-09-23T21:03:40.589 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0033353430000000002 2023-09-23T21:03:40.589 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.589 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.589 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:03:40.589 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:33.042239+0000", 2023-09-23T21:03:40.589 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.021175225999999998 2023-09-23T21:03:40.589 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.589 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.589 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:03:40.589 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:33.042839+0000", 2023-09-23T21:03:40.589 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00060004999999999998 2023-09-23T21:03:40.589 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.589 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.590 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T21:03:40.590 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:33.046070+0000", 2023-09-23T21:03:40.590 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0032314000000000002 2023-09-23T21:03:40.590 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.590 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.590 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T21:03:40.590 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:33.046122+0000", 2023-09-23T21:03:40.590 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.1583999999999999e-05 2023-09-23T21:03:40.590 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.590 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.590 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T21:03:40.590 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:33.046605+0000", 2023-09-23T21:03:40.590 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000483005 2023-09-23T21:03:40.591 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:03:40.591 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:03:40.591 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:03:40.591 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.591 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.591 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34797.0:130487 6.c 6:3110d152:::benchmark_data_smithi026_223868_object29361:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e136)", 2023-09-23T21:03:40.591 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T20:54:30.913625+0000", 2023-09-23T21:03:40.591 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 549.58855194700004, 2023-09-23T21:03:40.591 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.1412422530000002, 2023-09-23T21:03:40.591 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:03:40.591 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:03:40.591 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T21:03:40.591 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34797", 2023-09-23T21:03:40.592 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T21:03:40.592 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 130487 2023-09-23T21:03:40.592 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.592 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:03:40.592 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.592 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:03:40.592 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:30.913625+0000", 2023-09-23T21:03:40.592 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:03:40.592 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.592 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.592 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:03:40.592 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:30.913625+0000", 2023-09-23T21:03:40.592 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:03:40.593 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.593 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.593 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:03:40.593 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:30.913623+0000", 2023-09-23T21:03:40.593 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T21:03:40.593 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.593 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.593 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:03:40.593 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:30.913631+0000", 2023-09-23T21:03:40.593 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.5979999999999999e-06 2023-09-23T21:03:40.593 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.593 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.593 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:03:40.593 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:30.913643+0000", 2023-09-23T21:03:40.594 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.223e-05 2023-09-23T21:03:40.594 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.594 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.594 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:03:40.594 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:30.913665+0000", 2023-09-23T21:03:40.594 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.1792000000000001e-05 2023-09-23T21:03:40.594 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.594 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.594 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:03:40.594 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:30.915686+0000", 2023-09-23T21:03:40.594 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0020213150000000001 2023-09-23T21:03:40.594 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.595 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.595 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T21:03:40.595 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:30.915699+0000", 2023-09-23T21:03:40.595 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.2496e-05 2023-09-23T21:03:40.595 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.595 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.595 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:03:40.595 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:31.888085+0000", 2023-09-23T21:03:40.595 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.97238664500000005 2023-09-23T21:03:40.595 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.595 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.595 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T21:03:40.595 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:31.888090+0000", 2023-09-23T21:03:40.596 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.4839999999999998e-06 2023-09-23T21:03:40.596 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.596 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.596 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:03:40.596 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:33.037214+0000", 2023-09-23T21:03:40.596 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1491243719999999 2023-09-23T21:03:40.596 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.596 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.596 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:03:40.596 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:33.037330+0000", 2023-09-23T21:03:40.596 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000115977 2023-09-23T21:03:40.596 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.597 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.597 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 5,6,7", 2023-09-23T21:03:40.597 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:33.037423+0000", 2023-09-23T21:03:40.597 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.2723e-05 2023-09-23T21:03:40.597 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.597 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.597 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:03:40.597 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:33.046955+0000", 2023-09-23T21:03:40.597 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0095323809999999995 2023-09-23T21:03:40.597 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.597 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.597 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:03:40.597 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:33.047055+0000", 2023-09-23T21:03:40.598 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.9870000000000004e-05 2023-09-23T21:03:40.598 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.598 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.598 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:03:40.598 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:33.051207+0000", 2023-09-23T21:03:40.598 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0041521850000000001 2023-09-23T21:03:40.598 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.598 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.598 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T21:03:40.598 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:33.054703+0000", 2023-09-23T21:03:40.598 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0034956599999999998 2023-09-23T21:03:40.598 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.599 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.599 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T21:03:40.599 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:33.054753+0000", 2023-09-23T21:03:40.599 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.0430999999999999e-05 2023-09-23T21:03:40.599 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.599 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.599 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T21:03:40.599 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:33.054867+0000", 2023-09-23T21:03:40.599 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000113626 2023-09-23T21:03:40.599 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:03:40.599 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:03:40.599 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:03:40.599 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.599 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.600 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34797.0:130508 6.c 6:33de9cb1:::benchmark_data_smithi026_223868_object29382:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e136)", 2023-09-23T21:03:40.600 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T20:54:30.927850+0000", 2023-09-23T21:03:40.600 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 549.57432613200001, 2023-09-23T21:03:40.600 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.1274186720000001, 2023-09-23T21:03:40.600 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:03:40.600 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:03:40.600 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T21:03:40.600 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34797", 2023-09-23T21:03:40.600 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T21:03:40.600 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 130508 2023-09-23T21:03:40.600 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.601 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:03:40.601 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.601 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:03:40.601 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:30.927850+0000", 2023-09-23T21:03:40.601 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:03:40.601 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.601 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.601 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:03:40.601 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:30.927850+0000", 2023-09-23T21:03:40.601 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:03:40.601 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.601 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.601 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:03:40.602 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:30.927850+0000", 2023-09-23T21:03:40.602 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T21:03:40.602 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.602 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.602 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:03:40.602 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:30.927854+0000", 2023-09-23T21:03:40.602 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.0949999999999998e-06 2023-09-23T21:03:40.602 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.602 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.602 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:03:40.602 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:30.927866+0000", 2023-09-23T21:03:40.603 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1843e-05 2023-09-23T21:03:40.603 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.603 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.603 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:03:40.603 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:30.927871+0000", 2023-09-23T21:03:40.603 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.9490000000000004e-06 2023-09-23T21:03:40.603 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.603 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.603 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:03:40.603 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:30.927913+0000", 2023-09-23T21:03:40.603 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.2367999999999998e-05 2023-09-23T21:03:40.603 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.603 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.604 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T21:03:40.604 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:30.927921+0000", 2023-09-23T21:03:40.604 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.2339999999999994e-06 2023-09-23T21:03:40.604 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.604 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.604 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:03:40.604 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:31.888102+0000", 2023-09-23T21:03:40.604 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.960181116 2023-09-23T21:03:40.604 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.604 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.604 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T21:03:40.604 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:31.888106+0000", 2023-09-23T21:03:40.605 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.0969999999999999e-06 2023-09-23T21:03:40.605 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.605 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.605 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:03:40.605 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:33.041530+0000", 2023-09-23T21:03:40.605 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1534237329999999 2023-09-23T21:03:40.605 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.605 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.605 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:03:40.605 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:33.041653+0000", 2023-09-23T21:03:40.605 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00012306099999999999 2023-09-23T21:03:40.605 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.605 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.606 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 5,6,7", 2023-09-23T21:03:40.606 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:33.041780+0000", 2023-09-23T21:03:40.606 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000126978 2023-09-23T21:03:40.606 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.606 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.606 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:03:40.606 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:33.047098+0000", 2023-09-23T21:03:40.606 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0053179400000000002 2023-09-23T21:03:40.606 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.606 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.606 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:03:40.606 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:33.047424+0000", 2023-09-23T21:03:40.607 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00032540299999999999 2023-09-23T21:03:40.607 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.607 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.607 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:03:40.607 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:33.051248+0000", 2023-09-23T21:03:40.607 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.003824858 2023-09-23T21:03:40.607 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.607 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.607 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T21:03:40.607 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:33.055098+0000", 2023-09-23T21:03:40.607 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0038491419999999998 2023-09-23T21:03:40.608 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.608 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.608 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T21:03:40.608 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:33.055154+0000", 2023-09-23T21:03:40.608 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.6585000000000001e-05 2023-09-23T21:03:40.608 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.608 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.608 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T21:03:40.608 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:33.055269+0000", 2023-09-23T21:03:40.608 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000114967 2023-09-23T21:03:40.608 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:03:40.608 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:03:40.609 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:03:40.609 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.609 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.609 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34797.0:130522 6.c 6:3f85e3c7:::benchmark_data_smithi026_223868_object29396:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e136)", 2023-09-23T21:03:40.609 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T20:54:30.945451+0000", 2023-09-23T21:03:40.609 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 549.55672608600003, 2023-09-23T21:03:40.609 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.1100408050000001, 2023-09-23T21:03:40.609 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:03:40.609 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:03:40.609 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T21:03:40.609 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34797", 2023-09-23T21:03:40.609 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T21:03:40.610 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 130522 2023-09-23T21:03:40.610 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.610 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:03:40.610 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.610 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:03:40.610 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:30.945451+0000", 2023-09-23T21:03:40.610 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:03:40.610 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.610 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.610 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:03:40.610 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:30.945451+0000", 2023-09-23T21:03:40.610 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:03:40.610 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.611 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.611 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:03:40.611 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:30.945449+0000", 2023-09-23T21:03:40.611 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T21:03:40.611 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.611 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.611 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:03:40.611 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:30.945456+0000", 2023-09-23T21:03:40.611 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.9360000000000002e-06 2023-09-23T21:03:40.611 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.611 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.611 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:03:40.612 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:30.945469+0000", 2023-09-23T21:03:40.612 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.3597e-05 2023-09-23T21:03:40.612 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.612 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.612 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:03:40.612 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:30.945490+0000", 2023-09-23T21:03:40.612 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.0840999999999999e-05 2023-09-23T21:03:40.612 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.612 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.612 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:03:40.612 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:30.945602+0000", 2023-09-23T21:03:40.613 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000112007 2023-09-23T21:03:40.613 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.613 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.613 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T21:03:40.613 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:30.945610+0000", 2023-09-23T21:03:40.613 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.9149999999999996e-06 2023-09-23T21:03:40.613 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.613 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.613 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:03:40.613 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:31.888125+0000", 2023-09-23T21:03:40.613 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.94251427099999996 2023-09-23T21:03:40.613 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.614 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.614 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T21:03:40.614 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:31.888132+0000", 2023-09-23T21:03:40.614 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.4599999999999997e-06 2023-09-23T21:03:40.614 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.614 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.614 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:03:40.614 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:33.047604+0000", 2023-09-23T21:03:40.614 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.159472442 2023-09-23T21:03:40.614 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.614 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.614 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:03:40.615 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:33.047716+0000", 2023-09-23T21:03:40.615 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00011191700000000001 2023-09-23T21:03:40.615 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.615 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.615 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 5,6,7", 2023-09-23T21:03:40.615 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:33.047852+0000", 2023-09-23T21:03:40.615 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000136124 2023-09-23T21:03:40.615 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.615 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.615 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:03:40.615 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:33.054959+0000", 2023-09-23T21:03:40.615 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0071068900000000003 2023-09-23T21:03:40.616 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.616 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.616 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:03:40.616 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:33.055005+0000", 2023-09-23T21:03:40.616 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.5657e-05 2023-09-23T21:03:40.616 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.616 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.616 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:03:40.616 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:33.055041+0000", 2023-09-23T21:03:40.616 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.5620000000000001e-05 2023-09-23T21:03:40.616 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.616 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.616 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T21:03:40.617 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:33.055333+0000", 2023-09-23T21:03:40.617 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00029220899999999998 2023-09-23T21:03:40.617 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.617 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.617 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T21:03:40.617 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:33.055376+0000", 2023-09-23T21:03:40.617 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.2719999999999998e-05 2023-09-23T21:03:40.617 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.617 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.617 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T21:03:40.617 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:33.055491+0000", 2023-09-23T21:03:40.617 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000115767 2023-09-23T21:03:40.617 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:03:40.618 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:03:40.618 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:03:40.618 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.618 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.618 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34797.0:130523 6.7 6:e5f3342f:::benchmark_data_smithi026_223868_object29397:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e136)", 2023-09-23T21:03:40.618 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T20:54:30.945619+0000", 2023-09-23T21:03:40.618 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 549.55655756900001, 2023-09-23T21:03:40.618 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.103502862, 2023-09-23T21:03:40.618 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:03:40.618 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:03:40.618 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T21:03:40.619 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34797", 2023-09-23T21:03:40.619 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T21:03:40.619 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 130523 2023-09-23T21:03:40.619 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.619 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:03:40.619 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.619 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:03:40.619 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:30.945619+0000", 2023-09-23T21:03:40.619 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:03:40.619 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.619 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.619 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:03:40.619 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:30.945619+0000", 2023-09-23T21:03:40.620 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:03:40.620 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.620 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.620 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:03:40.620 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:30.945618+0000", 2023-09-23T21:03:40.620 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T21:03:40.620 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.620 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.620 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:03:40.620 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:30.945623+0000", 2023-09-23T21:03:40.620 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.8640000000000004e-06 2023-09-23T21:03:40.620 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.620 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.621 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:03:40.621 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:30.945633+0000", 2023-09-23T21:03:40.621 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0064999999999999e-05 2023-09-23T21:03:40.621 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.621 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.621 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:03:40.621 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:30.945641+0000", 2023-09-23T21:03:40.621 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.5599999999999996e-06 2023-09-23T21:03:40.621 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.621 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.621 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:03:40.621 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:30.945689+0000", 2023-09-23T21:03:40.621 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.7846999999999998e-05 2023-09-23T21:03:40.621 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.622 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.622 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T21:03:40.622 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:30.945699+0000", 2023-09-23T21:03:40.622 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0176999999999999e-05 2023-09-23T21:03:40.622 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.622 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.622 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:03:40.622 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:31.884383+0000", 2023-09-23T21:03:40.622 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.93868386199999998 2023-09-23T21:03:40.622 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.622 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.623 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T21:03:40.623 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:31.884394+0000", 2023-09-23T21:03:40.623 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1119e-05 2023-09-23T21:03:40.623 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.623 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.623 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:03:40.623 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:33.034107+0000", 2023-09-23T21:03:40.623 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1497131439999999 2023-09-23T21:03:40.623 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.623 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.623 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:03:40.623 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:33.034256+0000", 2023-09-23T21:03:40.624 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00014860700000000001 2023-09-23T21:03:40.624 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.624 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.624 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 1,4,5", 2023-09-23T21:03:40.624 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:33.036344+0000", 2023-09-23T21:03:40.624 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.002088679 2023-09-23T21:03:40.624 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.624 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.624 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:03:40.624 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:33.042876+0000", 2023-09-23T21:03:40.624 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0065314140000000001 2023-09-23T21:03:40.624 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.625 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.625 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:03:40.625 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:33.043075+0000", 2023-09-23T21:03:40.625 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00019934600000000001 2023-09-23T21:03:40.625 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.625 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.625 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:03:40.625 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:33.046043+0000", 2023-09-23T21:03:40.625 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0029680100000000001 2023-09-23T21:03:40.625 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.625 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.625 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T21:03:40.625 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:33.048588+0000", 2023-09-23T21:03:40.626 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0025450569999999999 2023-09-23T21:03:40.626 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.626 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.626 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T21:03:40.626 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:33.048637+0000", 2023-09-23T21:03:40.626 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.8881000000000002e-05 2023-09-23T21:03:40.626 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.626 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.626 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T21:03:40.626 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:33.049122+0000", 2023-09-23T21:03:40.626 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00048494100000000002 2023-09-23T21:03:40.626 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:03:40.627 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:03:40.627 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:03:40.627 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.627 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.627 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34797.0:130528 6.c 6:3991ff63:::benchmark_data_smithi026_223868_object29402:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e136)", 2023-09-23T21:03:40.627 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T20:54:30.953130+0000", 2023-09-23T21:03:40.627 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 549.54904703700004, 2023-09-23T21:03:40.627 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.1058828260000002, 2023-09-23T21:03:40.627 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:03:40.627 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:03:40.627 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T21:03:40.627 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34797", 2023-09-23T21:03:40.629 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T21:03:40.629 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 130528 2023-09-23T21:03:40.629 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.629 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:03:40.629 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.629 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:03:40.629 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:30.953130+0000", 2023-09-23T21:03:40.629 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:03:40.630 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.630 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.630 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:03:40.630 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:30.953130+0000", 2023-09-23T21:03:40.630 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:03:40.630 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.630 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.630 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:03:40.630 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:30.953129+0000", 2023-09-23T21:03:40.630 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.999999 2023-09-23T21:03:40.630 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.631 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.631 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:03:40.631 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:30.953133+0000", 2023-09-23T21:03:40.631 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.5739999999999997e-06 2023-09-23T21:03:40.631 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.631 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.631 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:03:40.631 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:30.953142+0000", 2023-09-23T21:03:40.631 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.9309999999999997e-06 2023-09-23T21:03:40.631 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.631 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.631 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:03:40.632 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:30.953147+0000", 2023-09-23T21:03:40.632 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.1329999999999998e-06 2023-09-23T21:03:40.632 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.632 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.632 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:03:40.632 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:30.953195+0000", 2023-09-23T21:03:40.632 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.8015e-05 2023-09-23T21:03:40.632 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.632 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.632 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T21:03:40.632 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:30.953202+0000", 2023-09-23T21:03:40.632 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.9480000000000002e-06 2023-09-23T21:03:40.633 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.633 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.633 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:03:40.633 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:31.888161+0000", 2023-09-23T21:03:40.633 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.934958497 2023-09-23T21:03:40.633 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.633 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.633 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T21:03:40.633 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:31.888169+0000", 2023-09-23T21:03:40.633 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.4349999999999997e-06 2023-09-23T21:03:40.633 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.633 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.633 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:03:40.634 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:33.051353+0000", 2023-09-23T21:03:40.634 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.163184022 2023-09-23T21:03:40.634 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.634 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.634 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:03:40.634 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:33.051488+0000", 2023-09-23T21:03:40.634 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00013468200000000001 2023-09-23T21:03:40.634 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.634 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.634 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 5,6,7", 2023-09-23T21:03:40.634 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:33.051620+0000", 2023-09-23T21:03:40.635 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000131524 2023-09-23T21:03:40.635 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.635 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.635 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T21:03:40.635 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:33.056424+0000", 2023-09-23T21:03:40.635 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0048044180000000004 2023-09-23T21:03:40.635 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.635 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.635 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:03:40.635 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:33.058674+0000", 2023-09-23T21:03:40.635 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.002250326 2023-09-23T21:03:40.635 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.635 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.636 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:03:40.636 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:33.058712+0000", 2023-09-23T21:03:40.636 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.7469999999999999e-05 2023-09-23T21:03:40.636 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.636 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.636 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:03:40.636 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:33.058880+0000", 2023-09-23T21:03:40.636 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00016785699999999999 2023-09-23T21:03:40.636 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.636 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.636 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T21:03:40.636 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:33.058932+0000", 2023-09-23T21:03:40.637 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.2051e-05 2023-09-23T21:03:40.637 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.637 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.637 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T21:03:40.637 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:54:33.059012+0000", 2023-09-23T21:03:40.637 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.0708e-05 2023-09-23T21:03:40.637 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:03:40.637 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:03:40.637 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:03:40.637 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.637 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.637 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34797.0:151487 6.6 6:657607da:::benchmark_data_smithi026_223868_object50361:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e147)", 2023-09-23T21:03:40.637 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T20:55:01.023782+0000", 2023-09-23T21:03:40.638 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 519.47839494699997, 2023-09-23T21:03:40.638 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0492650269999999, 2023-09-23T21:03:40.638 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:03:40.638 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "reached pg", 2023-09-23T21:03:40.638 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T21:03:40.638 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34797", 2023-09-23T21:03:40.638 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T21:03:40.638 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 151487 2023-09-23T21:03:40.638 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.638 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:03:40.638 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.638 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:03:40.639 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:01.023782+0000", 2023-09-23T21:03:40.639 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:03:40.639 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.639 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.639 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:03:40.639 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:01.023782+0000", 2023-09-23T21:03:40.639 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:03:40.639 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.639 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.639 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:03:40.639 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:01.023781+0000", 2023-09-23T21:03:40.639 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T21:03:40.639 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.640 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.640 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:03:40.640 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:01.023784+0000", 2023-09-23T21:03:40.640 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.96e-06 2023-09-23T21:03:40.640 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.640 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.640 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:03:40.640 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:01.023790+0000", 2023-09-23T21:03:40.640 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.8259999999999998e-06 2023-09-23T21:03:40.640 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.640 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.640 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:03:40.640 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:01.023795+0000", 2023-09-23T21:03:40.640 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.7269999999999998e-06 2023-09-23T21:03:40.641 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.641 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.641 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:03:40.641 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:01.024359+0000", 2023-09-23T21:03:40.641 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00056414799999999999 2023-09-23T21:03:40.641 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.641 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.641 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:03:40.641 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:01.024503+0000", 2023-09-23T21:03:40.641 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00014404000000000001 2023-09-23T21:03:40.641 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.641 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.642 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 4,7", 2023-09-23T21:03:40.642 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:01.024955+0000", 2023-09-23T21:03:40.642 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000451736 2023-09-23T21:03:40.642 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.642 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.642 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:03:40.642 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:01.029855+0000", 2023-09-23T21:03:40.642 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0049002569999999999 2023-09-23T21:03:40.642 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.642 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.642 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T21:03:40.642 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:01.035683+0000", 2023-09-23T21:03:40.642 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0058282439999999998 2023-09-23T21:03:40.643 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.643 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.643 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:03:40.643 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:01.044471+0000", 2023-09-23T21:03:40.643 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0087875460000000002 2023-09-23T21:03:40.643 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.643 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.643 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T21:03:40.643 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:01.044492+0000", 2023-09-23T21:03:40.643 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.1117000000000001e-05 2023-09-23T21:03:40.643 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.643 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.643 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:03:40.644 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:02.054661+0000", 2023-09-23T21:03:40.644 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.010168873 2023-09-23T21:03:40.644 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.644 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.644 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for missing object", 2023-09-23T21:03:40.644 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:02.054793+0000", 2023-09-23T21:03:40.644 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00013191400000000001 2023-09-23T21:03:40.644 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.644 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.644 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:03:40.644 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:02.072945+0000", 2023-09-23T21:03:40.644 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.01815288 2023-09-23T21:03:40.644 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.645 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.645 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T21:03:40.645 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:02.073047+0000", 2023-09-23T21:03:40.645 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000101245 2023-09-23T21:03:40.645 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:03:40.645 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:03:40.645 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:03:40.645 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.645 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.645 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34797.0:151498 6.6 6:62751f79:::benchmark_data_smithi026_223868_object50372:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e148)", 2023-09-23T21:03:40.645 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T20:55:01.044337+0000", 2023-09-23T21:03:40.646 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 519.45783959400001, 2023-09-23T21:03:40.646 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0348560959999999, 2023-09-23T21:03:40.646 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:03:40.646 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:03:40.646 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T21:03:40.646 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34797", 2023-09-23T21:03:40.646 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T21:03:40.646 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 151498 2023-09-23T21:03:40.646 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.646 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:03:40.646 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.646 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:03:40.646 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:01.044337+0000", 2023-09-23T21:03:40.647 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:03:40.647 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.647 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.647 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:03:40.647 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:01.044337+0000", 2023-09-23T21:03:40.647 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:03:40.647 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.647 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.647 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:03:40.647 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:01.044335+0000", 2023-09-23T21:03:40.647 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999981 2023-09-23T21:03:40.647 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.647 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.648 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:03:40.648 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:01.044343+0000", 2023-09-23T21:03:40.648 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.6490000000000008e-06 2023-09-23T21:03:40.648 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.648 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.648 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:03:40.648 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:01.045000+0000", 2023-09-23T21:03:40.648 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00065786399999999997 2023-09-23T21:03:40.648 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.648 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.648 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:03:40.648 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:01.045010+0000", 2023-09-23T21:03:40.648 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0009e-05 2023-09-23T21:03:40.649 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.649 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.649 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:03:40.649 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:01.046191+0000", 2023-09-23T21:03:40.649 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.001180843 2023-09-23T21:03:40.649 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.649 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.649 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T21:03:40.649 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:01.046209+0000", 2023-09-23T21:03:40.649 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.7980000000000001e-05 2023-09-23T21:03:40.649 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.649 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.650 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:03:40.650 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:02.073061+0000", 2023-09-23T21:03:40.650 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0268516350000001 2023-09-23T21:03:40.650 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.650 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.650 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:03:40.650 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:02.073135+0000", 2023-09-23T21:03:40.650 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.4067999999999996e-05 2023-09-23T21:03:40.650 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.650 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.650 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 4,7", 2023-09-23T21:03:40.650 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:02.073539+0000", 2023-09-23T21:03:40.650 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00040385099999999999 2023-09-23T21:03:40.651 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.651 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.651 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:03:40.651 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:02.079023+0000", 2023-09-23T21:03:40.651 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00548461 2023-09-23T21:03:40.651 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.651 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.651 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T21:03:40.651 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:02.079040+0000", 2023-09-23T21:03:40.651 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.6812000000000001e-05 2023-09-23T21:03:40.651 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.651 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.651 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:03:40.652 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:02.079070+0000", 2023-09-23T21:03:40.652 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.9459e-05 2023-09-23T21:03:40.652 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.652 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.652 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T21:03:40.652 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:02.079108+0000", 2023-09-23T21:03:40.652 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.8037999999999997e-05 2023-09-23T21:03:40.652 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.652 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.652 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T21:03:40.652 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:02.079193+0000", 2023-09-23T21:03:40.652 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.5344000000000001e-05 2023-09-23T21:03:40.652 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:03:40.653 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:03:40.653 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:03:40.653 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.653 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.653 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34797.0:165117 6.c 6:3504fcb2:::benchmark_data_smithi026_223868_object63991:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e154)", 2023-09-23T21:03:40.653 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T20:55:17.462057+0000", 2023-09-23T21:03:40.653 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 503.04011933800001, 2023-09-23T21:03:40.653 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.091172386, 2023-09-23T21:03:40.653 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:03:40.653 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "reached pg", 2023-09-23T21:03:40.653 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T21:03:40.653 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34797", 2023-09-23T21:03:40.654 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T21:03:40.654 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 165117 2023-09-23T21:03:40.654 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.654 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:03:40.654 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.654 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:03:40.654 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:17.462057+0000", 2023-09-23T21:03:40.654 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:03:40.654 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.654 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.654 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:03:40.654 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:17.462057+0000", 2023-09-23T21:03:40.654 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:03:40.655 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.655 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.655 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:03:40.655 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:17.462056+0000", 2023-09-23T21:03:40.655 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999981 2023-09-23T21:03:40.655 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.655 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.655 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:03:40.655 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:17.462063+0000", 2023-09-23T21:03:40.655 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.7610000000000007e-06 2023-09-23T21:03:40.655 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.655 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.655 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:03:40.656 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:17.462077+0000", 2023-09-23T21:03:40.656 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.3570000000000001e-05 2023-09-23T21:03:40.656 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.656 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.656 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:03:40.656 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:17.462085+0000", 2023-09-23T21:03:40.656 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.2470000000000008e-06 2023-09-23T21:03:40.656 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.656 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.656 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:03:40.656 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:17.462146+0000", 2023-09-23T21:03:40.656 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.1288e-05 2023-09-23T21:03:40.656 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.657 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.657 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:03:40.657 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:17.462301+0000", 2023-09-23T21:03:40.657 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000154971 2023-09-23T21:03:40.657 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.657 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.657 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 5,6,7", 2023-09-23T21:03:40.657 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:17.462508+0000", 2023-09-23T21:03:40.657 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000206691 2023-09-23T21:03:40.657 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.657 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.657 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T21:03:40.657 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:17.467494+0000", 2023-09-23T21:03:40.658 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0049860490000000002 2023-09-23T21:03:40.658 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.658 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.658 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:03:40.658 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:17.473605+0000", 2023-09-23T21:03:40.658 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0061112429999999997 2023-09-23T21:03:40.658 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.658 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.658 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:03:40.658 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:17.486250+0000", 2023-09-23T21:03:40.658 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.012644931999999999 2023-09-23T21:03:40.659 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.659 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.659 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T21:03:40.659 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:17.486261+0000", 2023-09-23T21:03:40.659 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0638999999999999e-05 2023-09-23T21:03:40.659 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.659 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.659 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:03:40.659 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:18.548375+0000", 2023-09-23T21:03:40.659 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0621140440000001 2023-09-23T21:03:40.659 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.659 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.660 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for missing object", 2023-09-23T21:03:40.660 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:18.548503+0000", 2023-09-23T21:03:40.660 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000127668 2023-09-23T21:03:40.660 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.660 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.660 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:03:40.660 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:18.553178+0000", 2023-09-23T21:03:40.660 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.004675289 2023-09-23T21:03:40.660 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.660 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.660 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T21:03:40.660 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:18.553230+0000", 2023-09-23T21:03:40.661 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.1743999999999998e-05 2023-09-23T21:03:40.661 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:03:40.661 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:03:40.661 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:03:40.661 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.661 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.661 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34797.0:177908 6.c 6:3bd0ad3d:::benchmark_data_smithi026_223868_object76782:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e164)", 2023-09-23T21:03:40.661 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T20:55:34.934464+0000", 2023-09-23T21:03:40.661 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 485.567713045, 2023-09-23T21:03:40.661 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.0970397529999998, 2023-09-23T21:03:40.661 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:03:40.661 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:03:40.662 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T21:03:40.662 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34797", 2023-09-23T21:03:40.662 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T21:03:40.662 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 177908 2023-09-23T21:03:40.662 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.662 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:03:40.662 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.662 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:03:40.662 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:34.934464+0000", 2023-09-23T21:03:40.662 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:03:40.662 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.663 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.663 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:03:40.663 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:34.934464+0000", 2023-09-23T21:03:40.663 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:03:40.663 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.663 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.663 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:03:40.663 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:34.934463+0000", 2023-09-23T21:03:40.663 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T21:03:40.663 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.663 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.664 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:03:40.664 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:34.934467+0000", 2023-09-23T21:03:40.664 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.4979999999999998e-06 2023-09-23T21:03:40.664 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.664 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.664 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:03:40.664 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:34.934487+0000", 2023-09-23T21:03:40.664 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.0602e-05 2023-09-23T21:03:40.664 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.664 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.664 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:03:40.664 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:34.934494+0000", 2023-09-23T21:03:40.665 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.4010000000000002e-06 2023-09-23T21:03:40.665 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.665 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.665 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:03:40.665 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:34.950588+0000", 2023-09-23T21:03:40.665 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.016094285 2023-09-23T21:03:40.665 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.665 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.665 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T21:03:40.665 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:34.950600+0000", 2023-09-23T21:03:40.665 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.2224e-05 2023-09-23T21:03:40.665 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.666 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.666 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:03:40.666 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:35.937538+0000", 2023-09-23T21:03:40.666 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.98693769200000003 2023-09-23T21:03:40.666 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.666 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.666 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T21:03:40.666 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:35.937552+0000", 2023-09-23T21:03:40.666 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.3811e-05 2023-09-23T21:03:40.666 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.666 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.667 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:03:40.667 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:37.017091+0000", 2023-09-23T21:03:40.667 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.079539322 2023-09-23T21:03:40.667 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.667 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.667 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:03:40.667 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:37.017258+0000", 2023-09-23T21:03:40.667 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00016676500000000001 2023-09-23T21:03:40.667 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.667 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.667 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 5,6", 2023-09-23T21:03:40.667 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:37.017367+0000", 2023-09-23T21:03:40.668 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000108909 2023-09-23T21:03:40.668 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.668 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.668 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T21:03:40.668 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:37.027299+0000", 2023-09-23T21:03:40.668 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0099326350000000004 2023-09-23T21:03:40.668 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.668 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.668 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:03:40.668 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:37.028919+0000", 2023-09-23T21:03:40.668 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.001619756 2023-09-23T21:03:40.668 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.668 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.669 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:03:40.669 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:37.031159+0000", 2023-09-23T21:03:40.669 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0022400330000000002 2023-09-23T21:03:40.669 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.669 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.669 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T21:03:40.669 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:37.031241+0000", 2023-09-23T21:03:40.669 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.2249000000000005e-05 2023-09-23T21:03:40.669 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.669 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.669 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T21:03:40.670 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:37.031503+0000", 2023-09-23T21:03:40.670 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00026212400000000003 2023-09-23T21:03:40.670 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:03:40.670 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:03:40.670 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:03:40.670 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.670 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.670 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34797.0:177953 6.c 6:3867b4fa:::benchmark_data_smithi026_223868_object76827:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e164)", 2023-09-23T21:03:40.670 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T20:55:35.005631+0000", 2023-09-23T21:03:40.671 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 485.49654595499999, 2023-09-23T21:03:40.671 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.0261709309999998, 2023-09-23T21:03:40.671 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:03:40.671 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:03:40.671 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T21:03:40.671 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34797", 2023-09-23T21:03:40.671 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T21:03:40.671 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 177953 2023-09-23T21:03:40.671 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.671 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:03:40.671 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.671 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:03:40.671 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:35.005631+0000", 2023-09-23T21:03:40.672 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:03:40.672 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.672 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.672 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:03:40.672 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:35.005631+0000", 2023-09-23T21:03:40.672 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:03:40.672 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.672 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.672 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:03:40.672 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:35.005629+0000", 2023-09-23T21:03:40.672 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T21:03:40.672 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.672 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.673 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:03:40.673 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:35.005635+0000", 2023-09-23T21:03:40.673 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.0809999999999999e-06 2023-09-23T21:03:40.673 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.673 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.673 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:03:40.673 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:35.005651+0000", 2023-09-23T21:03:40.673 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.5191000000000001e-05 2023-09-23T21:03:40.673 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.673 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.673 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:03:40.673 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:35.005659+0000", 2023-09-23T21:03:40.674 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.6999999999999997e-06 2023-09-23T21:03:40.674 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.674 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.674 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:03:40.674 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:35.005721+0000", 2023-09-23T21:03:40.674 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.1610000000000001e-05 2023-09-23T21:03:40.674 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.674 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.674 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T21:03:40.674 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:35.005734+0000", 2023-09-23T21:03:40.674 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.3137e-05 2023-09-23T21:03:40.674 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.675 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.675 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:03:40.675 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:35.937576+0000", 2023-09-23T21:03:40.675 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.931841582 2023-09-23T21:03:40.675 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.675 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.675 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T21:03:40.675 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:35.937584+0000", 2023-09-23T21:03:40.675 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.123e-06 2023-09-23T21:03:40.675 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.675 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.675 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:03:40.676 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:37.020962+0000", 2023-09-23T21:03:40.676 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.083378282 2023-09-23T21:03:40.676 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.676 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.676 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:03:40.676 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:37.021075+0000", 2023-09-23T21:03:40.676 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000112896 2023-09-23T21:03:40.676 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.676 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.676 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 5,6", 2023-09-23T21:03:40.676 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:37.021209+0000", 2023-09-23T21:03:40.676 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00013392100000000001 2023-09-23T21:03:40.677 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.677 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.677 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T21:03:40.677 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:37.027309+0000", 2023-09-23T21:03:40.677 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.006100482 2023-09-23T21:03:40.677 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.677 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.677 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:03:40.677 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:37.028967+0000", 2023-09-23T21:03:40.677 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0016573969999999999 2023-09-23T21:03:40.677 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.677 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.677 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:03:40.678 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:37.031581+0000", 2023-09-23T21:03:40.678 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0026142130000000002 2023-09-23T21:03:40.678 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.678 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.678 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T21:03:40.678 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:37.031645+0000", 2023-09-23T21:03:40.678 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.3997000000000004e-05 2023-09-23T21:03:40.678 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:03:40.678 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:03:40.678 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T21:03:40.678 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:37.031802+0000", 2023-09-23T21:03:40.678 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00015662 2023-09-23T21:03:40.679 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:03:40.679 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:03:40.679 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:03:40.679 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:03:40.679 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:03:40.679 INFO:teuthology.orchestra.run.smithi026.stdout:} 2023-09-23T21:03:40.679 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:03:40.680 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:03:40 smithi070 ceph-mon[168282]: pgmap v760: 119 pgs: 119 active+clean; 5.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 787 op/s; 0 B/s, 0 objects/s recovering 2023-09-23T21:03:40.680 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:03:40 smithi070 ceph-mon[168282]: 8.6 scrub ok 2023-09-23T21:03:40.680 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:03:40 smithi070 ceph-mon[168282]: 8.0 scrub ok 2023-09-23T21:03:40.680 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:03:40 smithi070 ceph-mon[168282]: failed to encode map e286 with expected crc 2023-09-23T21:03:40.681 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:03:40 smithi070 ceph-mon[168282]: failed to encode map e286 with expected crc 2023-09-23T21:03:40.681 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:03:40 smithi070 ceph-mon[168282]: failed to encode map e286 with expected crc 2023-09-23T21:03:40.681 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:03:40 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pgp_num_actual", "val": "18"}]': finished 2023-09-23T21:03:40.681 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:03:40 smithi070 ceph-mon[168282]: osdmap e286: 8 total, 8 up, 8 in 2023-09-23T21:03:40.681 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:03:40 smithi070 ceph-mon[168282]: failed to encode map e286 with expected crc 2023-09-23T21:03:40.681 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:03:40 smithi070 ceph-mon[168282]: 8.10 starting backfill to osd.4 from (0'0,0'0] MAX to 283'8511 2023-09-23T21:03:40.681 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:03:40 smithi070 ceph-mon[168282]: 8.10 starting backfill to osd.7 from (0'0,0'0] MAX to 283'8511 2023-09-23T21:03:40.681 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:03:40 smithi070 ceph-mon[168282]: 8.16 scrub starts 2023-09-23T21:03:40.681 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:03:40 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/940051275' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:03:40.681 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:03:40 smithi070 ceph-mon[168282]: osdmap e287: 8 total, 8 up, 8 in 2023-09-23T21:03:40.681 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:03:40 smithi070 ceph-mon[168282]: failed to encode map e287 with expected crc 2023-09-23T21:03:40.681 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:03:40 smithi070 ceph-mon[168282]: failed to encode map e287 with expected crc 2023-09-23T21:03:40.682 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:03:40 smithi026 ceph-mon[175151]: pgmap v760: 119 pgs: 119 active+clean; 5.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 787 op/s; 0 B/s, 0 objects/s recovering 2023-09-23T21:03:40.682 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:03:40 smithi026 ceph-mon[175151]: 8.6 scrub ok 2023-09-23T21:03:40.682 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:03:40 smithi026 ceph-mon[175151]: 8.0 scrub ok 2023-09-23T21:03:40.682 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:03:40 smithi026 ceph-mon[175151]: failed to encode map e286 with expected crc 2023-09-23T21:03:40.682 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:03:40 smithi026 ceph-mon[175151]: failed to encode map e286 with expected crc 2023-09-23T21:03:40.682 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:03:40 smithi026 ceph-mon[175151]: failed to encode map e286 with expected crc 2023-09-23T21:03:40.682 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:03:40 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pgp_num_actual", "val": "18"}]': finished 2023-09-23T21:03:40.682 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:03:40 smithi026 ceph-mon[175151]: osdmap e286: 8 total, 8 up, 8 in 2023-09-23T21:03:40.682 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:03:40 smithi026 ceph-mon[175151]: failed to encode map e286 with expected crc 2023-09-23T21:03:40.682 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:03:40 smithi026 ceph-mon[175151]: 8.10 starting backfill to osd.4 from (0'0,0'0] MAX to 283'8511 2023-09-23T21:03:40.683 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:03:40 smithi026 ceph-mon[175151]: 8.10 starting backfill to osd.7 from (0'0,0'0] MAX to 283'8511 2023-09-23T21:03:40.683 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:03:40 smithi026 ceph-mon[175151]: 8.16 scrub starts 2023-09-23T21:03:40.683 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:03:40 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/940051275' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:03:40.683 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:03:40 smithi026 ceph-mon[175151]: osdmap e287: 8 total, 8 up, 8 in 2023-09-23T21:03:40.683 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:03:40 smithi026 ceph-mon[175151]: failed to encode map e287 with expected crc 2023-09-23T21:03:40.683 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:03:40 smithi026 ceph-mon[175151]: failed to encode map e287 with expected crc 2023-09-23T21:03:40.683 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:03:40 smithi026 ceph-mon[179688]: pgmap v760: 119 pgs: 119 active+clean; 5.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 787 op/s; 0 B/s, 0 objects/s recovering 2023-09-23T21:03:40.683 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:03:40 smithi026 ceph-mon[179688]: 8.6 scrub ok 2023-09-23T21:03:40.683 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:03:40 smithi026 ceph-mon[179688]: 8.0 scrub ok 2023-09-23T21:03:40.683 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:03:40 smithi026 ceph-mon[179688]: failed to encode map e286 with expected crc 2023-09-23T21:03:40.684 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:03:40 smithi026 ceph-mon[179688]: failed to encode map e286 with expected crc 2023-09-23T21:03:40.684 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:03:40 smithi026 ceph-mon[179688]: failed to encode map e286 with expected crc 2023-09-23T21:03:40.684 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:03:40 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pgp_num_actual", "val": "18"}]': finished 2023-09-23T21:03:40.684 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:03:40 smithi026 ceph-mon[179688]: osdmap e286: 8 total, 8 up, 8 in 2023-09-23T21:03:40.684 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:03:40 smithi026 ceph-mon[179688]: failed to encode map e286 with expected crc 2023-09-23T21:03:40.684 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:03:40 smithi026 ceph-mon[179688]: 8.10 starting backfill to osd.4 from (0'0,0'0] MAX to 283'8511 2023-09-23T21:03:40.684 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:03:40 smithi026 ceph-mon[179688]: 8.10 starting backfill to osd.7 from (0'0,0'0] MAX to 283'8511 2023-09-23T21:03:40.684 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:03:40 smithi026 ceph-mon[179688]: 8.16 scrub starts 2023-09-23T21:03:40.684 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:03:40 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/940051275' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:03:40.684 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:03:40 smithi026 ceph-mon[179688]: osdmap e287: 8 total, 8 up, 8 in 2023-09-23T21:03:40.684 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:03:40 smithi026 ceph-mon[179688]: failed to encode map e287 with expected crc 2023-09-23T21:03:40.684 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:03:40 smithi026 ceph-mon[179688]: failed to encode map e287 with expected crc 2023-09-23T21:03:40.685 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:03:40.712 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:03:40.754 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:03:40.757 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:03:40.757 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:03:40.858 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:03:40.888 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:03:40.955 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:03:40.958 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:03:40.959 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:03:40.986 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_ops_in_flight 2023-09-23T21:03:41.059 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:03:41.087 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:03:41.121 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:03:41.123 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:03:41.124 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:03:41.225 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:03:41.271 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:03:41.317 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:03:41.322 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:03:41.322 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:03:41.387 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:03:41 smithi026 ceph-mon[175151]: failed to encode map e287 with expected crc 2023-09-23T21:03:41.388 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:03:41 smithi026 ceph-mon[175151]: failed to encode map e287 with expected crc 2023-09-23T21:03:41.388 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:03:41 smithi026 ceph-mon[175151]: 8.16 scrub starts 2023-09-23T21:03:41.388 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:03:41 smithi026 ceph-mon[175151]: 8.19 scrub starts 2023-09-23T21:03:41.388 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:03:41 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T21:03:41.388 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:03:41 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pgp_num_actual", "val": "19"}]: dispatch 2023-09-23T21:03:41.388 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:03:41 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pgp_num_actual", "val": "19"}]': finished 2023-09-23T21:03:41.388 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:03:41 smithi026 ceph-mon[175151]: osdmap e288: 8 total, 8 up, 8 in 2023-09-23T21:03:41.388 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:03:41 smithi026 ceph-mon[179688]: failed to encode map e287 with expected crc 2023-09-23T21:03:41.388 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:03:41 smithi026 ceph-mon[179688]: failed to encode map e287 with expected crc 2023-09-23T21:03:41.389 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:03:41 smithi026 ceph-mon[179688]: 8.16 scrub starts 2023-09-23T21:03:41.389 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:03:41 smithi026 ceph-mon[179688]: 8.19 scrub starts 2023-09-23T21:03:41.389 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:03:41 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T21:03:41.389 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:03:41 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pgp_num_actual", "val": "19"}]: dispatch 2023-09-23T21:03:41.389 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:03:41 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pgp_num_actual", "val": "19"}]': finished 2023-09-23T21:03:41.389 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:03:41 smithi026 ceph-mon[179688]: osdmap e288: 8 total, 8 up, 8 in 2023-09-23T21:03:41.423 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:03:41.475 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:03:41.530 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:03:41.535 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:03:41.535 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:03:41 smithi070 ceph-mon[168282]: failed to encode map e287 with expected crc 2023-09-23T21:03:41.535 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:03:41 smithi070 ceph-mon[168282]: failed to encode map e287 with expected crc 2023-09-23T21:03:41.535 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:03:41 smithi070 ceph-mon[168282]: 8.16 scrub starts 2023-09-23T21:03:41.535 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:03:41 smithi070 ceph-mon[168282]: 8.19 scrub starts 2023-09-23T21:03:41.535 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:03:41 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T21:03:41.536 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:03:41 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pgp_num_actual", "val": "19"}]: dispatch 2023-09-23T21:03:41.536 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:03:41 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pgp_num_actual", "val": "19"}]': finished 2023-09-23T21:03:41.536 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:03:41 smithi070 ceph-mon[168282]: osdmap e288: 8 total, 8 up, 8 in 2023-09-23T21:03:41.536 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:03:41.637 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:03:41.704 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T21:03:41.749 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:03:42.019 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:03:42.022 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:03:42.023 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:03:42.106 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T21:03:42.124 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:03:42.167 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:03:42.216 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:03:42.219 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:03:42.219 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:03:42.295 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:03:42 smithi026 ceph-mon[175151]: pgmap v763: 119 pgs: 119 active+clean; 5.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail 2023-09-23T21:03:42.320 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:03:42.358 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:03:42.409 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:03:42.413 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:03:42.414 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:03:42.515 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:03:42.545 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:03:42 smithi070 ceph-mon[168282]: pgmap v763: 119 pgs: 119 active+clean; 5.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail 2023-09-23T21:03:42.545 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:03:42 smithi070 ceph-mon[168282]: 8.7 scrub starts 2023-09-23T21:03:42.545 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:03:42 smithi070 ceph-mon[168282]: failed to encode map e288 with expected crc 2023-09-23T21:03:42.545 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:03:42 smithi070 ceph-mon[168282]: failed to encode map e288 with expected crc 2023-09-23T21:03:42.545 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:03:42 smithi070 ceph-mon[168282]: failed to encode map e288 with expected crc 2023-09-23T21:03:42.545 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:03:42 smithi070 ceph-mon[168282]: 8.11 starting backfill to osd.5 from (0'0,0'0] MAX to 284'8701 2023-09-23T21:03:42.546 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:03:42 smithi070 ceph-mon[168282]: 8.11 starting backfill to osd.6 from (0'0,0'0] MAX to 284'8701 2023-09-23T21:03:42.546 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:03:42 smithi070 ceph-mon[168282]: 8.11 starting backfill to osd.7 from (0'0,0'0] MAX to 284'8701 2023-09-23T21:03:42.546 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:03:42 smithi070 ceph-mon[168282]: 8.7 scrub ok 2023-09-23T21:03:42.546 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:03:42 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:03:42.546 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:03:42 smithi070 ceph-mon[168282]: osdmap e289: 8 total, 8 up, 8 in 2023-09-23T21:03:42.549 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:03:42.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:03:42 smithi026 ceph-mon[175151]: 8.7 scrub starts 2023-09-23T21:03:42.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:03:42 smithi026 ceph-mon[175151]: failed to encode map e288 with expected crc 2023-09-23T21:03:42.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:03:42 smithi026 ceph-mon[175151]: failed to encode map e288 with expected crc 2023-09-23T21:03:42.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:03:42 smithi026 ceph-mon[175151]: failed to encode map e288 with expected crc 2023-09-23T21:03:42.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:03:42 smithi026 ceph-mon[175151]: 8.11 starting backfill to osd.5 from (0'0,0'0] MAX to 284'8701 2023-09-23T21:03:42.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:03:42 smithi026 ceph-mon[175151]: 8.11 starting backfill to osd.6 from (0'0,0'0] MAX to 284'8701 2023-09-23T21:03:42.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:03:42 smithi026 ceph-mon[175151]: 8.11 starting backfill to osd.7 from (0'0,0'0] MAX to 284'8701 2023-09-23T21:03:42.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:03:42 smithi026 ceph-mon[175151]: 8.7 scrub ok 2023-09-23T21:03:42.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:03:42 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:03:42.633 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:03:42 smithi026 ceph-mon[175151]: osdmap e289: 8 total, 8 up, 8 in 2023-09-23T21:03:42.633 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:03:42 smithi026 ceph-mon[179688]: pgmap v763: 119 pgs: 119 active+clean; 5.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail 2023-09-23T21:03:42.633 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:03:42 smithi026 ceph-mon[179688]: 8.7 scrub starts 2023-09-23T21:03:42.633 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:03:42 smithi026 ceph-mon[179688]: failed to encode map e288 with expected crc 2023-09-23T21:03:42.633 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:03:42 smithi026 ceph-mon[179688]: failed to encode map e288 with expected crc 2023-09-23T21:03:42.633 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:03:42 smithi026 ceph-mon[179688]: failed to encode map e288 with expected crc 2023-09-23T21:03:42.633 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:03:42 smithi026 ceph-mon[179688]: 8.11 starting backfill to osd.5 from (0'0,0'0] MAX to 284'8701 2023-09-23T21:03:42.633 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:03:42 smithi026 ceph-mon[179688]: 8.11 starting backfill to osd.6 from (0'0,0'0] MAX to 284'8701 2023-09-23T21:03:42.633 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:03:42 smithi026 ceph-mon[179688]: 8.11 starting backfill to osd.7 from (0'0,0'0] MAX to 284'8701 2023-09-23T21:03:42.633 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:03:42 smithi026 ceph-mon[179688]: 8.7 scrub ok 2023-09-23T21:03:42.633 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:03:42 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:03:42.634 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:03:42 smithi026 ceph-mon[179688]: osdmap e289: 8 total, 8 up, 8 in 2023-09-23T21:03:42.785 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:03:42.791 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:03:42.792 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:03:42.794 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 6, 7, 4, 5] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 7, 6] 2023-09-23T21:03:42.794 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph osd reweight 0 1 2023-09-23T21:03:42.801 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:03:42.892 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:03:42.932 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:03:42.971 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:03:42.974 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:03:42.975 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:03:43.075 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:03:43.106 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:03:43.199 INFO:teuthology.orchestra.run.smithi026.stderr:nodeep-scrub is unset 2023-09-23T21:03:43.246 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:03:43.250 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:03:43.250 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:03:43.351 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:03:43.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:03:43 smithi070 ceph-mon[168282]: failed to encode map e289 with expected crc 2023-09-23T21:03:43.371 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:03:43 smithi070 ceph-mon[168282]: failed to encode map e289 with expected crc 2023-09-23T21:03:43.371 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:03:43 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/3910325397' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:03:43.371 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:03:43 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:03:43.371 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:03:43 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pgp_num_actual", "val": "20"}]: dispatch 2023-09-23T21:03:43.371 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:03:43 smithi070 ceph-mon[168282]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T21:03:43.371 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:03:43 smithi070 ceph-mon[168282]: Cluster is now healthy 2023-09-23T21:03:43.371 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:03:43 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T21:03:43.371 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:03:43 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pgp_num_actual", "val": "20"}]': finished 2023-09-23T21:03:43.371 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:03:43 smithi070 ceph-mon[168282]: osdmap e290: 8 total, 8 up, 8 in 2023-09-23T21:03:43.411 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:03:43.524 INFO:teuthology.orchestra.run.smithi070.stdout:{ 2023-09-23T21:03:43.524 INFO:teuthology.orchestra.run.smithi070.stdout: "ops": [ 2023-09-23T21:03:43.524 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:43.524 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_repop(client.35469.0:141964 8.e e290/254 8:7cb960a7:::benchmark_data_smithi026_301269_object31207:head v 290'8980, mlcod=290'8978)", 2023-09-23T21:03:43.524 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:03:43.520565+0000", 2023-09-23T21:03:43.524 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 0.0030643129999999999, 2023-09-23T21:03:43.524 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00309253, 2023-09-23T21:03:43.524 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:03:43.525 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "started", 2023-09-23T21:03:43.525 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:03:43.525 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:43.525 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:03:43.525 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:43.520565+0000", 2023-09-23T21:03:43.525 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:03:43.525 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:03:43.525 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:43.525 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:03:43.525 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:43.520565+0000", 2023-09-23T21:03:43.526 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:03:43.526 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:03:43.526 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:43.526 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:03:43.526 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:43.520565+0000", 2023-09-23T21:03:43.526 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999995 2023-09-23T21:03:43.526 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:03:43.526 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:43.526 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:03:43.526 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:43.520570+0000", 2023-09-23T21:03:43.526 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.9390000000000002e-06 2023-09-23T21:03:43.527 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:03:43.527 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:43.527 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:03:43.527 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:43.520581+0000", 2023-09-23T21:03:43.527 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.0746e-05 2023-09-23T21:03:43.527 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:03:43.527 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:43.527 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:03:43.527 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:43.520587+0000", 2023-09-23T21:03:43.527 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.4970000000000001e-06 2023-09-23T21:03:43.527 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:03:43.528 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:43.528 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:03:43.528 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:43.520626+0000", 2023-09-23T21:03:43.528 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.8856000000000002e-05 2023-09-23T21:03:43.528 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:03:43.528 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:43.528 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:03:43.528 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:43.520646+0000", 2023-09-23T21:03:43.528 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.0811000000000001e-05 2023-09-23T21:03:43.528 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:03:43.528 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:03:43.529 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:03:43.529 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:03:43.529 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:43.529 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_repop(client.35469.0:141969 8.e e290/254 8:766fd627:::benchmark_data_smithi026_301269_object31212:head v 290'8981, mlcod=290'8978)", 2023-09-23T21:03:43.529 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:03:43.522315+0000", 2023-09-23T21:03:43.529 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 0.001314436, 2023-09-23T21:03:43.529 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0014148030000000001, 2023-09-23T21:03:43.529 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:03:43.529 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "started", 2023-09-23T21:03:43.529 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:03:43.529 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:43.530 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:03:43.530 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:43.522315+0000", 2023-09-23T21:03:43.530 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:03:43.530 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:03:43.530 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:43.530 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:03:43.530 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:43.522315+0000", 2023-09-23T21:03:43.530 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:03:43.531 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:03:43.531 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:43.531 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:03:43.531 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:43.522314+0000", 2023-09-23T21:03:43.531 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999995 2023-09-23T21:03:43.531 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:03:43.531 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:43.531 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:03:43.531 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:43.522319+0000", 2023-09-23T21:03:43.531 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.6700000000000002e-06 2023-09-23T21:03:43.531 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:03:43.531 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:43.532 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:03:43.532 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:43.522333+0000", 2023-09-23T21:03:43.532 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.4403000000000001e-05 2023-09-23T21:03:43.532 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:03:43.532 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:43.532 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:03:43.532 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:43.522340+0000", 2023-09-23T21:03:43.532 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.0379999999999996e-06 2023-09-23T21:03:43.532 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:03:43.532 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:43.532 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:03:43.532 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:43.523590+0000", 2023-09-23T21:03:43.533 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.001250183 2023-09-23T21:03:43.533 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:03:43.533 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:43.533 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:03:43.533 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:43.523607+0000", 2023-09-23T21:03:43.533 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.7501999999999999e-05 2023-09-23T21:03:43.533 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:03:43.533 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:03:43.533 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:03:43.533 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:03:43.533 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:43.533 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "pg_backfill(progress 8.10 e 290/290 lb 8:0e473680:::benchmark_data_smithi026_301269_object100833:head)", 2023-09-23T21:03:43.533 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:03:43.523548+0000", 2023-09-23T21:03:43.534 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 8.1933000000000003e-05, 2023-09-23T21:03:43.534 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00024085999999999999, 2023-09-23T21:03:43.534 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:03:43.534 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "reached pg", 2023-09-23T21:03:43.534 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:03:43.534 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:43.534 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:03:43.534 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:43.523548+0000", 2023-09-23T21:03:43.534 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:03:43.534 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:03:43.534 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:43.535 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:03:43.535 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:43.523548+0000", 2023-09-23T21:03:43.535 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:03:43.535 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:03:43.535 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:43.535 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:03:43.535 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:43.523547+0000", 2023-09-23T21:03:43.535 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999995 2023-09-23T21:03:43.535 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:03:43.535 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:43.535 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:03:43.535 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:43.523554+0000", 2023-09-23T21:03:43.536 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.6000000000000003e-06 2023-09-23T21:03:43.536 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:03:43.536 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:43.536 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:03:43.536 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:43.523561+0000", 2023-09-23T21:03:43.536 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.5900000000000002e-06 2023-09-23T21:03:43.536 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:03:43.536 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:43.536 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:03:43.536 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:43.523566+0000", 2023-09-23T21:03:43.536 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.3669999999999999e-06 2023-09-23T21:03:43.537 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:03:43.537 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:43.537 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:03:43.537 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:43.523787+0000", 2023-09-23T21:03:43.537 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00022121099999999999 2023-09-23T21:03:43.537 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:03:43.537 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:03:43.537 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:03:43.537 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:03:43.537 INFO:teuthology.orchestra.run.smithi070.stdout: ], 2023-09-23T21:03:43.537 INFO:teuthology.orchestra.run.smithi070.stdout: "num_ops": 3 2023-09-23T21:03:43.537 INFO:teuthology.orchestra.run.smithi070.stdout:} 2023-09-23T21:03:43.546 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:03:43 smithi026 ceph-mon[179688]: failed to encode map e289 with expected crc 2023-09-23T21:03:43.546 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:03:43 smithi026 ceph-mon[179688]: failed to encode map e289 with expected crc 2023-09-23T21:03:43.546 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:03:43 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/3910325397' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:03:43.546 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:03:43 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:03:43.546 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:03:43 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pgp_num_actual", "val": "20"}]: dispatch 2023-09-23T21:03:43.546 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:03:43 smithi026 ceph-mon[179688]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T21:03:43.546 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:03:43 smithi026 ceph-mon[179688]: Cluster is now healthy 2023-09-23T21:03:43.547 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:03:43 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T21:03:43.547 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:03:43 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pgp_num_actual", "val": "20"}]': finished 2023-09-23T21:03:43.547 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:03:43 smithi026 ceph-mon[179688]: osdmap e290: 8 total, 8 up, 8 in 2023-09-23T21:03:43.547 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:03:43 smithi026 ceph-mon[175151]: failed to encode map e289 with expected crc 2023-09-23T21:03:43.547 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:03:43 smithi026 ceph-mon[175151]: failed to encode map e289 with expected crc 2023-09-23T21:03:43.547 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:03:43 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/3910325397' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:03:43.547 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:03:43 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:03:43.547 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:03:43 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pgp_num_actual", "val": "20"}]: dispatch 2023-09-23T21:03:43.548 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:03:43 smithi026 ceph-mon[175151]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T21:03:43.548 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:03:43 smithi026 ceph-mon[175151]: Cluster is now healthy 2023-09-23T21:03:43.548 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:03:43 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T21:03:43.548 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:03:43 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pgp_num_actual", "val": "20"}]': finished 2023-09-23T21:03:43.548 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:03:43 smithi026 ceph-mon[175151]: osdmap e290: 8 total, 8 up, 8 in 2023-09-23T21:03:43.895 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T21:03:43.899 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:03:43.899 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:03:44.000 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:03:44.032 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:03:44.055 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_blocked_ops 2023-09-23T21:03:44.120 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:03:44.124 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:03:44.125 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:03:44.132 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:03:44.226 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:03:44.262 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:03:44.339 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0: {} 2023-09-23T21:03:44.353 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:03:44.357 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1: {} 2023-09-23T21:03:44.358 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:03:44 smithi070 ceph-mon[168282]: pgmap v766: 119 pgs: 1 active+remapped+backfill_wait, 1 active+remapped+backfilling, 117 active+clean; 4.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 10089/240549 objects misplaced (4.194%); 19 MiB/s, 309 objects/s recovering 2023-09-23T21:03:44.358 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:03:44 smithi070 ceph-mon[168282]: failed to encode map e290 with expected crc 2023-09-23T21:03:44.358 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:03:44 smithi070 ceph-mon[168282]: failed to encode map e290 with expected crc 2023-09-23T21:03:44.358 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:03:44 smithi070 ceph-mon[168282]: 8.12 starting backfill to osd.2 from (0'0,0'0] MAX to 286'8671 2023-09-23T21:03:44.358 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:03:44 smithi070 ceph-mon[168282]: 8.3 scrub starts 2023-09-23T21:03:44.359 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:03:44 smithi070 ceph-mon[168282]: 8.12 starting backfill to osd.6 from (0'0,0'0] MAX to 286'8671 2023-09-23T21:03:44.359 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:03:44 smithi070 ceph-mon[168282]: 8.12 starting backfill to osd.7 from (0'0,0'0] MAX to 286'8671 2023-09-23T21:03:44.359 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:03:44 smithi070 ceph-mon[168282]: osdmap e291: 8 total, 8 up, 8 in 2023-09-23T21:03:44.359 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:03:44 smithi070 ceph-mon[168282]: failed to encode map e291 with expected crc 2023-09-23T21:03:44.359 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:03:44.360 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:03:44.376 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2: {} 2023-09-23T21:03:44.381 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:03:44 smithi026 ceph-mon[175151]: pgmap v766: 119 pgs: 1 active+remapped+backfill_wait, 1 active+remapped+backfilling, 117 active+clean; 4.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 10089/240549 objects misplaced (4.194%); 19 MiB/s, 309 objects/s recovering 2023-09-23T21:03:44.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:03:44 smithi026 ceph-mon[175151]: failed to encode map e290 with expected crc 2023-09-23T21:03:44.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:03:44 smithi026 ceph-mon[175151]: failed to encode map e290 with expected crc 2023-09-23T21:03:44.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:03:44 smithi026 ceph-mon[175151]: 8.12 starting backfill to osd.2 from (0'0,0'0] MAX to 286'8671 2023-09-23T21:03:44.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:03:44 smithi026 ceph-mon[175151]: 8.3 scrub starts 2023-09-23T21:03:44.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:03:44 smithi026 ceph-mon[175151]: 8.12 starting backfill to osd.6 from (0'0,0'0] MAX to 286'8671 2023-09-23T21:03:44.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:03:44 smithi026 ceph-mon[175151]: 8.12 starting backfill to osd.7 from (0'0,0'0] MAX to 286'8671 2023-09-23T21:03:44.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:03:44 smithi026 ceph-mon[175151]: osdmap e291: 8 total, 8 up, 8 in 2023-09-23T21:03:44.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:03:44 smithi026 ceph-mon[175151]: failed to encode map e291 with expected crc 2023-09-23T21:03:44.383 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:03:44 smithi026 ceph-mon[179688]: pgmap v766: 119 pgs: 1 active+remapped+backfill_wait, 1 active+remapped+backfilling, 117 active+clean; 4.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 10089/240549 objects misplaced (4.194%); 19 MiB/s, 309 objects/s recovering 2023-09-23T21:03:44.383 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:03:44 smithi026 ceph-mon[179688]: failed to encode map e290 with expected crc 2023-09-23T21:03:44.383 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:03:44 smithi026 ceph-mon[179688]: failed to encode map e290 with expected crc 2023-09-23T21:03:44.383 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:03:44 smithi026 ceph-mon[179688]: 8.12 starting backfill to osd.2 from (0'0,0'0] MAX to 286'8671 2023-09-23T21:03:44.383 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:03:44 smithi026 ceph-mon[179688]: 8.3 scrub starts 2023-09-23T21:03:44.383 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:03:44 smithi026 ceph-mon[179688]: 8.12 starting backfill to osd.6 from (0'0,0'0] MAX to 286'8671 2023-09-23T21:03:44.383 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:03:44 smithi026 ceph-mon[179688]: 8.12 starting backfill to osd.7 from (0'0,0'0] MAX to 286'8671 2023-09-23T21:03:44.383 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:03:44 smithi026 ceph-mon[179688]: osdmap e291: 8 total, 8 up, 8 in 2023-09-23T21:03:44.383 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:03:44 smithi026 ceph-mon[179688]: failed to encode map e291 with expected crc 2023-09-23T21:03:44.395 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3: {} 2023-09-23T21:03:44.413 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4: {} 2023-09-23T21:03:44.433 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T21:03:44.456 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6: {} 2023-09-23T21:03:44.461 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:03:44.480 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T21:03:44.484 INFO:teuthology.orchestra.run.smithi026.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T21:03:44.484 INFO:teuthology.orchestra.run.smithi026.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T21:03:44.484 INFO:teuthology.orchestra.run.smithi026.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T21:03:44.484 INFO:teuthology.orchestra.run.smithi026.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T21:03:44.484 INFO:teuthology.orchestra.run.smithi026.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T21:03:44.484 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T21:03:44.484 INFO:teuthology.orchestra.run.smithi026.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T21:03:44.484 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T21:03:44.498 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:03:44.642 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:03:44.644 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:03:44.645 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:03:44.745 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:03:44.776 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:03:44.812 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:03:44.815 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:03:44.815 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:03:44.859 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T21:03:44.916 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:03:44.944 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:03:44.994 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:03:44.999 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:03:45.000 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:03:45.100 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:03:45.151 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:03:45.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:03:45 smithi026 ceph-mon[179688]: failed to encode map e291 with expected crc 2023-09-23T21:03:45.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:03:45 smithi026 ceph-mon[179688]: failed to encode map e291 with expected crc 2023-09-23T21:03:45.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:03:45 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/2906375794' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:03:45.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:03:45 smithi026 ceph-mon[179688]: 8.2 scrub starts 2023-09-23T21:03:45.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:03:45 smithi026 ceph-mon[179688]: failed to encode map e292 with expected crc 2023-09-23T21:03:45.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:03:45 smithi026 ceph-mon[179688]: osdmap e292: 8 total, 8 up, 8 in 2023-09-23T21:03:45.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:03:45 smithi026 ceph-mon[179688]: failed to encode map e292 with expected crc 2023-09-23T21:03:45.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:03:45 smithi026 ceph-mon[175151]: failed to encode map e291 with expected crc 2023-09-23T21:03:45.383 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:03:45 smithi026 ceph-mon[175151]: failed to encode map e291 with expected crc 2023-09-23T21:03:45.383 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:03:45 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/2906375794' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:03:45.383 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:03:45 smithi026 ceph-mon[175151]: 8.2 scrub starts 2023-09-23T21:03:45.383 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:03:45 smithi026 ceph-mon[175151]: failed to encode map e292 with expected crc 2023-09-23T21:03:45.383 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:03:45 smithi026 ceph-mon[175151]: osdmap e292: 8 total, 8 up, 8 in 2023-09-23T21:03:45.383 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:03:45 smithi026 ceph-mon[175151]: failed to encode map e292 with expected crc 2023-09-23T21:03:45.455 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:03:45.460 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:03:45.460 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:03:45.561 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:03:45.603 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:03:45 smithi070 ceph-mon[168282]: failed to encode map e291 with expected crc 2023-09-23T21:03:45.603 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:03:45 smithi070 ceph-mon[168282]: failed to encode map e291 with expected crc 2023-09-23T21:03:45.603 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:03:45 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/2906375794' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:03:45.603 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:03:45 smithi070 ceph-mon[168282]: 8.2 scrub starts 2023-09-23T21:03:45.603 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:03:45 smithi070 ceph-mon[168282]: failed to encode map e292 with expected crc 2023-09-23T21:03:45.603 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:03:45 smithi070 ceph-mon[168282]: osdmap e292: 8 total, 8 up, 8 in 2023-09-23T21:03:45.603 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:03:45 smithi070 ceph-mon[168282]: failed to encode map e292 with expected crc 2023-09-23T21:03:45.606 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:03:45.661 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:03:45.666 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:03:45.666 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:03:45.729 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T21:03:45.766 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:03:45.888 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:03:45.956 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:03:45.973 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:03:45.973 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:03:46.074 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:03:46.111 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:03:46.198 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:03:46.203 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:03:46.204 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:03:46.305 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:03:46.387 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:03:46.478 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:03:46.485 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:03:46.485 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:03:46.586 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:03:46.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:03:46 smithi070 ceph-mon[168282]: pgmap v769: 119 pgs: 1 remapped+peering, 1 active+remapped+backfill_wait, 1 active+remapped+backfilling, 116 active+clean; 4.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 10089/239217 objects misplaced (4.218%); 19 MiB/s, 309 objects/s recovering 2023-09-23T21:03:46.621 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:03:46 smithi070 ceph-mon[168282]: failed to encode map e292 with expected crc 2023-09-23T21:03:46.621 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:03:46 smithi070 ceph-mon[168282]: failed to encode map e292 with expected crc 2023-09-23T21:03:46.621 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:03:46 smithi070 ceph-mon[168282]: 8.13 starting backfill to osd.2 from (0'0,0'0] MAX to 289'8317 2023-09-23T21:03:46.621 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:03:46 smithi070 ceph-mon[168282]: 8.13 starting backfill to osd.3 from (0'0,0'0] MAX to 289'8317 2023-09-23T21:03:46.621 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:03:46 smithi070 ceph-mon[168282]: failed to encode map e293 with expected crc 2023-09-23T21:03:46.621 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:03:46 smithi070 ceph-mon[168282]: osdmap e293: 8 total, 8 up, 8 in 2023-09-23T21:03:46.621 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:03:46 smithi070 ceph-mon[168282]: failed to encode map e293 with expected crc 2023-09-23T21:03:46.621 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:03:46 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:03:46.621 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:03:46 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:03:46.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:03:46 smithi026 ceph-mon[179688]: pgmap v769: 119 pgs: 1 remapped+peering, 1 active+remapped+backfill_wait, 1 active+remapped+backfilling, 116 active+clean; 4.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 10089/239217 objects misplaced (4.218%); 19 MiB/s, 309 objects/s recovering 2023-09-23T21:03:46.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:03:46 smithi026 ceph-mon[179688]: failed to encode map e292 with expected crc 2023-09-23T21:03:46.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:03:46 smithi026 ceph-mon[179688]: failed to encode map e292 with expected crc 2023-09-23T21:03:46.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:03:46 smithi026 ceph-mon[179688]: 8.13 starting backfill to osd.2 from (0'0,0'0] MAX to 289'8317 2023-09-23T21:03:46.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:03:46 smithi026 ceph-mon[179688]: 8.13 starting backfill to osd.3 from (0'0,0'0] MAX to 289'8317 2023-09-23T21:03:46.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:03:46 smithi026 ceph-mon[179688]: failed to encode map e293 with expected crc 2023-09-23T21:03:46.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:03:46 smithi026 ceph-mon[179688]: osdmap e293: 8 total, 8 up, 8 in 2023-09-23T21:03:46.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:03:46 smithi026 ceph-mon[179688]: failed to encode map e293 with expected crc 2023-09-23T21:03:46.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:03:46 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:03:46.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:03:46 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:03:46.633 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:03:46 smithi026 ceph-mon[175151]: pgmap v769: 119 pgs: 1 remapped+peering, 1 active+remapped+backfill_wait, 1 active+remapped+backfilling, 116 active+clean; 4.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 10089/239217 objects misplaced (4.218%); 19 MiB/s, 309 objects/s recovering 2023-09-23T21:03:46.633 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:03:46 smithi026 ceph-mon[175151]: failed to encode map e292 with expected crc 2023-09-23T21:03:46.633 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:03:46 smithi026 ceph-mon[175151]: failed to encode map e292 with expected crc 2023-09-23T21:03:46.633 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:03:46 smithi026 ceph-mon[175151]: 8.13 starting backfill to osd.2 from (0'0,0'0] MAX to 289'8317 2023-09-23T21:03:46.633 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:03:46 smithi026 ceph-mon[175151]: 8.13 starting backfill to osd.3 from (0'0,0'0] MAX to 289'8317 2023-09-23T21:03:46.633 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:03:46 smithi026 ceph-mon[175151]: failed to encode map e293 with expected crc 2023-09-23T21:03:46.633 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:03:46 smithi026 ceph-mon[175151]: osdmap e293: 8 total, 8 up, 8 in 2023-09-23T21:03:46.633 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:03:46 smithi026 ceph-mon[175151]: failed to encode map e293 with expected crc 2023-09-23T21:03:46.633 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:03:46 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:03:46.633 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:03:46 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:03:46.640 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:03:46.691 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:03:47.077 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T21:03:47.378 INFO:teuthology.orchestra.run.smithi026.stderr:reweighted osd.0 to 1 (10000) 2023-09-23T21:03:47.407 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:03:47 smithi026 ceph-mon[175151]: failed to encode map e293 with expected crc 2023-09-23T21:03:47.407 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:03:47 smithi026 ceph-mon[175151]: failed to encode map e293 with expected crc 2023-09-23T21:03:47.407 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:03:47 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/869336337' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-09-23T21:03:47.408 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:03:47 smithi026 ceph-mon[179688]: failed to encode map e293 with expected crc 2023-09-23T21:03:47.408 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:03:47 smithi026 ceph-mon[179688]: failed to encode map e293 with expected crc 2023-09-23T21:03:47.408 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:03:47 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/869336337' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-09-23T21:03:47.447 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:03:47.450 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:03:47.451 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:03:47.451 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:03:47 smithi070 ceph-mon[168282]: failed to encode map e293 with expected crc 2023-09-23T21:03:47.452 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:03:47 smithi070 ceph-mon[168282]: failed to encode map e293 with expected crc 2023-09-23T21:03:47.452 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:03:47 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/869336337' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-09-23T21:03:47.552 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:03:47.633 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:03:47.674 INFO:teuthology.orchestra.run.smithi070.stdout:ERROR: (22) Invalid argument 2023-09-23T21:03:47.674 INFO:teuthology.orchestra.run.smithi070.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T21:03:47.835 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:03:47.842 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:03:47.842 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:03:47.943 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:03:48.004 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:03:48.149 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_historic_ops 2023-09-23T21:03:48.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:03:48 smithi070 ceph-mon[168282]: pgmap v772: 119 pgs: 1 remapped+peering, 1 active+remapped+backfill_wait, 1 active+remapped+backfilling, 116 active+clean; 4.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 10089/238713 objects misplaced (4.226%) 2023-09-23T21:03:48.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:03:48 smithi070 ceph-mon[168282]: failed to encode map e294 with expected crc 2023-09-23T21:03:48.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:03:48 smithi070 ceph-mon[168282]: failed to encode map e294 with expected crc 2023-09-23T21:03:48.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:03:48 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/869336337' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-09-23T21:03:48.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:03:48 smithi070 ceph-mon[168282]: osdmap e294: 8 total, 8 up, 8 in 2023-09-23T21:03:48.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:03:48 smithi070 ceph-mon[168282]: failed to encode map e294 with expected crc 2023-09-23T21:03:48.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:03:48 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:03:48.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:03:48 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:03:48.631 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:03:48 smithi026 ceph-mon[179688]: pgmap v772: 119 pgs: 1 remapped+peering, 1 active+remapped+backfill_wait, 1 active+remapped+backfilling, 116 active+clean; 4.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 10089/238713 objects misplaced (4.226%) 2023-09-23T21:03:48.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:03:48 smithi026 ceph-mon[179688]: failed to encode map e294 with expected crc 2023-09-23T21:03:48.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:03:48 smithi026 ceph-mon[179688]: failed to encode map e294 with expected crc 2023-09-23T21:03:48.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:03:48 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/869336337' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-09-23T21:03:48.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:03:48 smithi026 ceph-mon[179688]: osdmap e294: 8 total, 8 up, 8 in 2023-09-23T21:03:48.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:03:48 smithi026 ceph-mon[179688]: failed to encode map e294 with expected crc 2023-09-23T21:03:48.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:03:48 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:03:48.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:03:48 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:03:48.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:03:48 smithi026 ceph-mon[175151]: pgmap v772: 119 pgs: 1 remapped+peering, 1 active+remapped+backfill_wait, 1 active+remapped+backfilling, 116 active+clean; 4.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 10089/238713 objects misplaced (4.226%) 2023-09-23T21:03:48.633 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:03:48 smithi026 ceph-mon[175151]: failed to encode map e294 with expected crc 2023-09-23T21:03:48.633 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:03:48 smithi026 ceph-mon[175151]: failed to encode map e294 with expected crc 2023-09-23T21:03:48.633 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:03:48 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/869336337' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-09-23T21:03:48.633 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:03:48 smithi026 ceph-mon[175151]: osdmap e294: 8 total, 8 up, 8 in 2023-09-23T21:03:48.633 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:03:48 smithi026 ceph-mon[175151]: failed to encode map e294 with expected crc 2023-09-23T21:03:48.633 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:03:48 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:03:48.633 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:03:48 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:03:48.783 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T21:03:49.620 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:03:49 smithi026 ceph-mon[175151]: 8.2 scrub starts 2023-09-23T21:03:49.620 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:03:49 smithi026 ceph-mon[179688]: 8.2 scrub starts 2023-09-23T21:03:49.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:03:49 smithi070 ceph-mon[168282]: 8.2 scrub starts 2023-09-23T21:03:49.885 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 21:03:49 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[164029]: ::ffff:172.21.15.70 - - [23/Sep/2023:21:03:49] "GET /metrics HTTP/1.1" 200 36302 "" "Prometheus/2.43.0" 2023-09-23T21:03:50.291 INFO:teuthology.orchestra.run.smithi070.stdout:ERROR: (22) Invalid argument 2023-09-23T21:03:50.292 INFO:teuthology.orchestra.run.smithi070.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T21:03:50.341 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:03:50.348 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:03:50.349 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:03:50.431 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph osd reweight 1 1 2023-09-23T21:03:50.449 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:03:50.552 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:03:50.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:03:50 smithi070 ceph-mon[168282]: pgmap v774: 119 pgs: 1 active+remapped+backfill_wait, 1 active+remapped+backfilling, 1 remapped+peering, 116 active+clean; 4.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 13182/235830 objects misplaced (5.590%); 25 MiB/s, 400 objects/s recovering 2023-09-23T21:03:50.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:03:50 smithi070 ceph-mon[168282]: 8.2 scrub starts 2023-09-23T21:03:50.640 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:03:50 smithi026 ceph-mon[175151]: pgmap v774: 119 pgs: 1 active+remapped+backfill_wait, 1 active+remapped+backfilling, 1 remapped+peering, 116 active+clean; 4.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 13182/235830 objects misplaced (5.590%); 25 MiB/s, 400 objects/s recovering 2023-09-23T21:03:50.640 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:03:50 smithi026 ceph-mon[175151]: 8.2 scrub starts 2023-09-23T21:03:50.640 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:03:50 smithi026 ceph-mon[179688]: pgmap v774: 119 pgs: 1 active+remapped+backfill_wait, 1 active+remapped+backfilling, 1 remapped+peering, 116 active+clean; 4.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 13182/235830 objects misplaced (5.590%); 25 MiB/s, 400 objects/s recovering 2023-09-23T21:03:50.640 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:03:50 smithi026 ceph-mon[179688]: 8.2 scrub starts 2023-09-23T21:03:50.722 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_ops_in_flight 2023-09-23T21:03:50.749 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:03:50.757 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:03:50.758 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:03:50.858 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:03:50.947 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:03:51.170 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:03:51.177 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:03:51.177 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:03:51.278 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:03:51.317 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:03:51.357 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:03:51.360 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:03:51.360 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:03:51.442 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T21:03:51.462 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:03:51.533 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:03:51.763 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:03:51.772 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:03:51.773 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:03:51.873 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:03:51.908 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:03:51.919 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:03:52.022 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:03:52.224 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:03:52.231 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:03:52.231 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:03:52.332 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:03:52.383 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:03:52.428 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:03:52.431 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:03:52.431 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:03:52.532 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:03:52.642 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:03:52.686 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:03:52 smithi026 ceph-mon[175151]: pgmap v775: 119 pgs: 1 active+remapped+backfill_wait, 2 active+remapped+backfilling, 116 active+clean; 4.7 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 479 op/s; 16932/230592 objects misplaced (7.343%); 24 MiB/s, 378 objects/s recovering 2023-09-23T21:03:52.686 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:03:52 smithi026 ceph-mon[175151]: 8.2 scrub starts 2023-09-23T21:03:52.686 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:03:52 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:03:52.687 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:03:52 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:03:52.687 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:03:52 smithi026 ceph-mon[179688]: pgmap v775: 119 pgs: 1 active+remapped+backfill_wait, 2 active+remapped+backfilling, 116 active+clean; 4.7 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 479 op/s; 16932/230592 objects misplaced (7.343%); 24 MiB/s, 378 objects/s recovering 2023-09-23T21:03:52.687 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:03:52 smithi026 ceph-mon[179688]: 8.2 scrub starts 2023-09-23T21:03:52.687 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:03:52 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:03:52.688 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:03:52 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:03:52.780 INFO:teuthology.orchestra.run.smithi070.stdout:ERROR: (22) Invalid argument 2023-09-23T21:03:52.780 INFO:teuthology.orchestra.run.smithi070.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T21:03:52.806 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:03:52 smithi070 ceph-mon[168282]: pgmap v775: 119 pgs: 1 active+remapped+backfill_wait, 2 active+remapped+backfilling, 116 active+clean; 4.7 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 479 op/s; 16932/230592 objects misplaced (7.343%); 24 MiB/s, 378 objects/s recovering 2023-09-23T21:03:52.806 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:03:52 smithi070 ceph-mon[168282]: 8.2 scrub starts 2023-09-23T21:03:52.807 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:03:52 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:03:52.807 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:03:52 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:03:52.827 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:03:52.837 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:03:52.838 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:03:52.939 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:03:53.025 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:03:53.262 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_blocked_ops 2023-09-23T21:03:53.359 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:03:53.365 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:03:53.365 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:03:53.466 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:03:53.507 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:03:53 smithi070 ceph-mon[168282]: 8.2 scrub starts 2023-09-23T21:03:53.508 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:03:53 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T21:03:53.508 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:03:53 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T21:03:53.508 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:03:53 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:03:53.508 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:03:53 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/451186199' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T21:03:53.508 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:03:53 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T21:03:53.509 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:03:53.567 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:03:53.571 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:03:53.571 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:03:53.635 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:03:53 smithi026 ceph-mon[179688]: 8.2 scrub starts 2023-09-23T21:03:53.635 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:03:53 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T21:03:53.635 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:03:53 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T21:03:53.635 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:03:53 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:03:53.635 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:03:53 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/451186199' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T21:03:53.636 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:03:53 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T21:03:53.636 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:03:53 smithi026 ceph-mon[175151]: 8.2 scrub starts 2023-09-23T21:03:53.636 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:03:53 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T21:03:53.636 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:03:53 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T21:03:53.636 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:03:53 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:03:53.636 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:03:53 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/451186199' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T21:03:53.637 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:03:53 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T21:03:53.672 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:03:53.748 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:03:53.774 INFO:teuthology.orchestra.run.smithi026.stderr:noscrub is set 2023-09-23T21:03:53.807 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:03:53.811 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:03:53.811 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:03:53.912 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:03:54.009 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:03:54.027 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T21:03:54.094 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:03:54.101 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:03:54.102 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:03:54.202 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:03:54.299 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:03:54.468 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:03:54.483 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:03:54.484 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:03:54.584 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:03:54.585 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0: {} 2023-09-23T21:03:54.598 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:03:54 smithi070 ceph-mon[168282]: pgmap v776: 119 pgs: 1 active+remapped+backfill_wait, 2 active+remapped+backfilling, 116 active+clean; 4.5 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 669 op/s; 14490/223479 objects misplaced (6.484%); 24 MiB/s, 377 objects/s recovering 2023-09-23T21:03:54.598 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:03:54 smithi070 ceph-mon[168282]: 8.2 scrub starts 2023-09-23T21:03:54.598 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:03:54 smithi070 ceph-mon[168282]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:03:54.598 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:03:54 smithi070 ceph-mon[168282]: failed to encode map e295 with expected crc 2023-09-23T21:03:54.598 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:03:54 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T21:03:54.598 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:03:54 smithi070 ceph-mon[168282]: osdmap e295: 8 total, 8 up, 8 in 2023-09-23T21:03:54.598 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:03:54 smithi070 ceph-mon[168282]: failed to encode map e295 with expected crc 2023-09-23T21:03:54.599 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:03:54 smithi070 ceph-mon[168282]: failed to encode map e295 with expected crc 2023-09-23T21:03:54.627 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1: {} 2023-09-23T21:03:54.633 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:03:54 smithi026 ceph-mon[175151]: pgmap v776: 119 pgs: 1 active+remapped+backfill_wait, 2 active+remapped+backfilling, 116 active+clean; 4.5 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 669 op/s; 14490/223479 objects misplaced (6.484%); 24 MiB/s, 377 objects/s recovering 2023-09-23T21:03:54.633 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:03:54 smithi026 ceph-mon[175151]: 8.2 scrub starts 2023-09-23T21:03:54.633 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:03:54 smithi026 ceph-mon[175151]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:03:54.633 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:03:54 smithi026 ceph-mon[175151]: failed to encode map e295 with expected crc 2023-09-23T21:03:54.634 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:03:54 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T21:03:54.634 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:03:54 smithi026 ceph-mon[175151]: osdmap e295: 8 total, 8 up, 8 in 2023-09-23T21:03:54.634 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:03:54 smithi026 ceph-mon[175151]: failed to encode map e295 with expected crc 2023-09-23T21:03:54.634 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:03:54 smithi026 ceph-mon[175151]: failed to encode map e295 with expected crc 2023-09-23T21:03:54.635 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:03:54 smithi026 ceph-mon[179688]: pgmap v776: 119 pgs: 1 active+remapped+backfill_wait, 2 active+remapped+backfilling, 116 active+clean; 4.5 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 669 op/s; 14490/223479 objects misplaced (6.484%); 24 MiB/s, 377 objects/s recovering 2023-09-23T21:03:54.635 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:03:54 smithi026 ceph-mon[179688]: 8.2 scrub starts 2023-09-23T21:03:54.635 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:03:54 smithi026 ceph-mon[179688]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:03:54.636 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:03:54 smithi026 ceph-mon[179688]: failed to encode map e295 with expected crc 2023-09-23T21:03:54.636 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:03:54 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T21:03:54.636 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:03:54 smithi026 ceph-mon[179688]: osdmap e295: 8 total, 8 up, 8 in 2023-09-23T21:03:54.636 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:03:54 smithi026 ceph-mon[179688]: failed to encode map e295 with expected crc 2023-09-23T21:03:54.636 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:03:54 smithi026 ceph-mon[179688]: failed to encode map e295 with expected crc 2023-09-23T21:03:54.669 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:03:54.684 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2: {} 2023-09-23T21:03:54.723 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3: {} 2023-09-23T21:03:54.742 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:03:54.750 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:03:54.751 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:03:54.761 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4: {} 2023-09-23T21:03:54.795 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T21:03:54.842 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6: {} 2023-09-23T21:03:54.852 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:03:54.886 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T21:03:54.889 INFO:teuthology.orchestra.run.smithi026.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T21:03:54.889 INFO:teuthology.orchestra.run.smithi026.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T21:03:54.889 INFO:teuthology.orchestra.run.smithi026.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T21:03:54.890 INFO:teuthology.orchestra.run.smithi026.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T21:03:54.890 INFO:teuthology.orchestra.run.smithi026.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T21:03:54.890 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T21:03:54.890 INFO:teuthology.orchestra.run.smithi026.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T21:03:54.890 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T21:03:54.925 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:03:55.221 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T21:03:55.233 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:03:55.234 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:03:55.335 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:03:55.409 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:03:55.427 INFO:teuthology.orchestra.run.smithi026.stderr:reweighted osd.1 to 1 (10000) 2023-09-23T21:03:55.473 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:03:55.478 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:03:55 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/3631197862' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-09-23T21:03:55.478 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:03:55 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-09-23T21:03:55.479 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:03:55 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/517555631' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:03:55.479 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:03:55.480 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:03:55.487 INFO:teuthology.orchestra.run.smithi070.stdout:{ 2023-09-23T21:03:55.487 INFO:teuthology.orchestra.run.smithi070.stdout: "ops": [], 2023-09-23T21:03:55.487 INFO:teuthology.orchestra.run.smithi070.stdout: "complaint_time": 30, 2023-09-23T21:03:55.487 INFO:teuthology.orchestra.run.smithi070.stdout: "num_blocked_ops": 0 2023-09-23T21:03:55.487 INFO:teuthology.orchestra.run.smithi070.stdout:} 2023-09-23T21:03:55.580 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:03:55.596 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:03:55 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/3631197862' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-09-23T21:03:55.596 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:03:55 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-09-23T21:03:55.596 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:03:55 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/517555631' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:03:55.597 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:03:55 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/3631197862' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-09-23T21:03:55.597 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:03:55 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-09-23T21:03:55.597 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:03:55 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/517555631' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:03:55.633 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:03:55.887 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:03:55.889 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:03:55.890 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:03:55.909 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph osd reweight 2 1 2023-09-23T21:03:55.931 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_historic_ops 2023-09-23T21:03:55.991 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:03:56.077 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:03:56.162 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:03:56.168 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:03:56.169 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:03:56.269 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:03:56.341 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:03:56.401 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:03:56.408 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:03:56.408 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:03:56.508 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T21:03:56.509 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:03:56.549 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:03:56.562 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T21:03:56.606 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:03:56.613 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:03:56.614 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:03:56.714 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:03:56.752 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:03:56 smithi070 ceph-mon[168282]: pgmap v778: 119 pgs: 1 active+remapped+backfill_wait, 2 active+remapped+backfilling, 116 active+clean; 4.5 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 826 op/s; 13536/218841 objects misplaced (6.185%); 25 MiB/s, 404 objects/s recovering 2023-09-23T21:03:56.752 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:03:56 smithi070 ceph-mon[168282]: failed to encode map e296 with expected crc 2023-09-23T21:03:56.752 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:03:56 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-09-23T21:03:56.752 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:03:56 smithi070 ceph-mon[168282]: osdmap e296: 8 total, 8 up, 8 in 2023-09-23T21:03:56.753 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:03:56 smithi070 ceph-mon[168282]: failed to encode map e296 with expected crc 2023-09-23T21:03:56.753 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:03:56 smithi070 ceph-mon[168282]: failed to encode map e296 with expected crc 2023-09-23T21:03:56.754 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:03:56.847 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:03:56.851 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:03:56.852 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:03:56.857 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:03:56.885 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:03:56 smithi026 ceph-mon[175151]: pgmap v778: 119 pgs: 1 active+remapped+backfill_wait, 2 active+remapped+backfilling, 116 active+clean; 4.5 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 826 op/s; 13536/218841 objects misplaced (6.185%); 25 MiB/s, 404 objects/s recovering 2023-09-23T21:03:56.886 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:03:56 smithi026 ceph-mon[175151]: failed to encode map e296 with expected crc 2023-09-23T21:03:56.886 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:03:56 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-09-23T21:03:56.886 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:03:56 smithi026 ceph-mon[175151]: osdmap e296: 8 total, 8 up, 8 in 2023-09-23T21:03:56.886 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:03:56 smithi026 ceph-mon[175151]: failed to encode map e296 with expected crc 2023-09-23T21:03:56.886 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:03:56 smithi026 ceph-mon[175151]: failed to encode map e296 with expected crc 2023-09-23T21:03:56.886 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:03:56 smithi026 ceph-mon[179688]: pgmap v778: 119 pgs: 1 active+remapped+backfill_wait, 2 active+remapped+backfilling, 116 active+clean; 4.5 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 826 op/s; 13536/218841 objects misplaced (6.185%); 25 MiB/s, 404 objects/s recovering 2023-09-23T21:03:56.887 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:03:56 smithi026 ceph-mon[179688]: failed to encode map e296 with expected crc 2023-09-23T21:03:56.887 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:03:56 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-09-23T21:03:56.887 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:03:56 smithi026 ceph-mon[179688]: osdmap e296: 8 total, 8 up, 8 in 2023-09-23T21:03:56.887 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:03:56 smithi026 ceph-mon[179688]: failed to encode map e296 with expected crc 2023-09-23T21:03:56.887 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:03:56 smithi026 ceph-mon[179688]: failed to encode map e296 with expected crc 2023-09-23T21:03:56.953 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:03:57.019 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:03:57.171 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:03:57.171 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:03:57.172 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:03:57.272 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:03:57.304 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T21:03:57.343 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:03:57.414 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:03:57.418 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:03:57.418 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:03:57.519 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:03:57.561 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:03:57.567 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:03:57 smithi026 ceph-mon[175151]: pgmap v780: 119 pgs: 1 active+remapped+backfill_wait, 2 active+remapped+backfilling, 116 active+clean; 4.4 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 790 op/s; 13536/216831 objects misplaced (6.243%); 11 MiB/s, 176 objects/s recovering 2023-09-23T21:03:57.568 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:03:57 smithi026 ceph-mon[179688]: pgmap v780: 119 pgs: 1 active+remapped+backfill_wait, 2 active+remapped+backfilling, 116 active+clean; 4.4 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 790 op/s; 13536/216831 objects misplaced (6.243%); 11 MiB/s, 176 objects/s recovering 2023-09-23T21:03:57.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:03:57 smithi070 ceph-mon[168282]: pgmap v780: 119 pgs: 1 active+remapped+backfill_wait, 2 active+remapped+backfilling, 116 active+clean; 4.4 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 790 op/s; 13536/216831 objects misplaced (6.243%); 11 MiB/s, 176 objects/s recovering 2023-09-23T21:03:57.898 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:03:58.054 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:03:58.059 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:03:58.060 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:03:58.161 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:03:58.217 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:03:58.361 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:03:58.365 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:03:58.366 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:03:58.460 INFO:teuthology.orchestra.run.smithi070.stdout:{ 2023-09-23T21:03:58.460 INFO:teuthology.orchestra.run.smithi070.stdout: "size": 20, 2023-09-23T21:03:58.460 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 600, 2023-09-23T21:03:58.460 INFO:teuthology.orchestra.run.smithi070.stdout: "ops": [ 2023-09-23T21:03:58.460 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:58.460 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.34797.0:167244 6.a 6:5fa8594b:::benchmark_data_smithi026_223868_object66118:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e158)", 2023-09-23T21:03:58.460 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T20:55:20.930710+0000", 2023-09-23T21:03:58.460 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 517.52343493299998, 2023-09-23T21:03:58.460 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.0357931300000001, 2023-09-23T21:03:58.461 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:03:58.461 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:03:58.461 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:03:58.461 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.34797", 2023-09-23T21:03:58.461 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T21:03:58.461 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 167244 2023-09-23T21:03:58.461 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:03:58.461 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:03:58.461 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:58.461 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:03:58.461 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:20.930710+0000", 2023-09-23T21:03:58.461 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:03:58.462 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:03:58.462 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:58.462 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:03:58.462 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:20.930710+0000", 2023-09-23T21:03:58.462 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:03:58.462 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:03:58.462 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:58.462 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:03:58.462 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:20.930709+0000", 2023-09-23T21:03:58.462 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.999999 2023-09-23T21:03:58.462 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:03:58.463 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:58.463 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:03:58.463 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:20.930716+0000", 2023-09-23T21:03:58.463 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.2579999999999998e-06 2023-09-23T21:03:58.463 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:03:58.463 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:58.463 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:03:58.463 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:20.930726+0000", 2023-09-23T21:03:58.463 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.0087e-05 2023-09-23T21:03:58.463 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:03:58.463 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:58.463 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:03:58.463 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:20.930731+0000", 2023-09-23T21:03:58.464 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.7140000000000001e-06 2023-09-23T21:03:58.464 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:03:58.464 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:58.464 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:03:58.464 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:20.930770+0000", 2023-09-23T21:03:58.464 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.9199999999999997e-05 2023-09-23T21:03:58.464 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:03:58.464 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:58.464 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:03:58.464 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:20.930779+0000", 2023-09-23T21:03:58.464 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.3380000000000005e-06 2023-09-23T21:03:58.464 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:03:58.465 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:58.465 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:03:58.465 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:21.956800+0000", 2023-09-23T21:03:58.465 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.026021538 2023-09-23T21:03:58.465 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:03:58.465 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:58.465 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:03:58.465 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:21.956900+0000", 2023-09-23T21:03:58.465 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.9709000000000004e-05 2023-09-23T21:03:58.465 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:03:58.465 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:58.465 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 0,2", 2023-09-23T21:03:58.466 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:21.957010+0000", 2023-09-23T21:03:58.466 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000110504 2023-09-23T21:03:58.466 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:03:58.466 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:58.466 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:03:58.466 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:21.959547+0000", 2023-09-23T21:03:58.466 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0025366799999999999 2023-09-23T21:03:58.466 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:03:58.466 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:58.466 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:03:58.466 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:21.966355+0000", 2023-09-23T21:03:58.466 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.006807848 2023-09-23T21:03:58.467 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:03:58.467 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:58.467 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:03:58.467 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:21.966407+0000", 2023-09-23T21:03:58.467 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.1666999999999998e-05 2023-09-23T21:03:58.467 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:03:58.467 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:58.467 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:03:58.467 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:21.966442+0000", 2023-09-23T21:03:58.467 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.5602000000000002e-05 2023-09-23T21:03:58.467 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:03:58.467 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:58.468 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:03:58.468 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:21.966503+0000", 2023-09-23T21:03:58.468 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.1019e-05 2023-09-23T21:03:58.468 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:03:58.468 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:03:58.468 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:03:58.468 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:03:58.468 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:58.468 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.34797.0:167247 6.a 6:5b80d451:::benchmark_data_smithi026_223868_object66121:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e158)", 2023-09-23T21:03:58.468 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T20:55:20.930743+0000", 2023-09-23T21:03:58.468 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 517.52340188699998, 2023-09-23T21:03:58.468 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.0409182340000001, 2023-09-23T21:03:58.468 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:03:58.469 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:03:58.469 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:03:58.469 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.34797", 2023-09-23T21:03:58.469 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T21:03:58.469 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 167247 2023-09-23T21:03:58.469 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:03:58.469 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:03:58.469 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:58.469 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:03:58.469 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:20.930743+0000", 2023-09-23T21:03:58.469 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:03:58.469 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:03:58.469 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:58.470 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:03:58.470 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:20.930743+0000", 2023-09-23T21:03:58.470 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:03:58.470 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:03:58.470 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:58.470 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:03:58.470 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:20.930742+0000", 2023-09-23T21:03:58.470 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999995 2023-09-23T21:03:58.470 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:03:58.470 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:58.470 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:03:58.471 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:20.930746+0000", 2023-09-23T21:03:58.471 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.1879999999999998e-06 2023-09-23T21:03:58.471 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:03:58.471 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:58.471 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:03:58.471 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:20.930752+0000", 2023-09-23T21:03:58.471 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.0379999999999996e-06 2023-09-23T21:03:58.471 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:03:58.471 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:58.471 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:03:58.471 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:20.930756+0000", 2023-09-23T21:03:58.471 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.1030000000000004e-06 2023-09-23T21:03:58.471 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:03:58.472 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:58.472 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:03:58.472 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:20.930808+0000", 2023-09-23T21:03:58.472 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.2305999999999997e-05 2023-09-23T21:03:58.472 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:03:58.472 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:58.472 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:03:58.472 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:20.930813+0000", 2023-09-23T21:03:58.472 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.9910000000000002e-06 2023-09-23T21:03:58.472 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:03:58.472 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:58.472 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:03:58.473 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:21.959580+0000", 2023-09-23T21:03:58.473 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.0287672910000001 2023-09-23T21:03:58.473 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:03:58.473 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:58.473 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:03:58.473 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:21.959671+0000", 2023-09-23T21:03:58.473 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.0928999999999996e-05 2023-09-23T21:03:58.473 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:03:58.473 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:58.473 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 0,2", 2023-09-23T21:03:58.473 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:21.959739+0000", 2023-09-23T21:03:58.473 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.7663000000000006e-05 2023-09-23T21:03:58.474 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:03:58.474 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:58.474 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:03:58.474 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:21.969123+0000", 2023-09-23T21:03:58.474 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0093844800000000006 2023-09-23T21:03:58.474 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:03:58.474 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:58.474 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:03:58.474 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:21.971537+0000", 2023-09-23T21:03:58.474 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.002413157 2023-09-23T21:03:58.474 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:03:58.475 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:58.475 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:03:58.475 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:21.971574+0000", 2023-09-23T21:03:58.475 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.7681000000000001e-05 2023-09-23T21:03:58.475 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:03:58.475 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:58.475 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:03:58.475 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:21.971604+0000", 2023-09-23T21:03:58.475 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.0176999999999999e-05 2023-09-23T21:03:58.475 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:03:58.475 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:58.475 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:03:58.476 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:21.971661+0000", 2023-09-23T21:03:58.476 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.6857999999999997e-05 2023-09-23T21:03:58.476 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:03:58.476 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:03:58.476 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:03:58.476 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:03:58.476 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:58.476 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.34797.0:167248 6.a 6:59b345d1:::benchmark_data_smithi026_223868_object66122:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e158)", 2023-09-23T21:03:58.476 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T20:55:20.930761+0000", 2023-09-23T21:03:58.476 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 517.52338430700001, 2023-09-23T21:03:58.476 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.055093563, 2023-09-23T21:03:58.476 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:03:58.477 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:03:58.477 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:03:58.477 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.34797", 2023-09-23T21:03:58.477 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T21:03:58.477 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 167248 2023-09-23T21:03:58.477 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:03:58.477 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:03:58.477 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:58.477 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:03:58.477 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:20.930761+0000", 2023-09-23T21:03:58.477 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:03:58.477 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:03:58.478 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:58.478 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:03:58.478 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:20.930761+0000", 2023-09-23T21:03:58.478 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:03:58.478 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:03:58.478 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:58.478 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:03:58.478 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:20.930760+0000", 2023-09-23T21:03:58.478 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999995 2023-09-23T21:03:58.478 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:03:58.478 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:58.479 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:03:58.479 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:20.930762+0000", 2023-09-23T21:03:58.479 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.1119999999999999e-06 2023-09-23T21:03:58.479 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:03:58.479 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:58.479 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:03:58.479 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:20.930768+0000", 2023-09-23T21:03:58.479 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.4539999999999998e-06 2023-09-23T21:03:58.479 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:03:58.480 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:58.480 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:03:58.480 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:20.930771+0000", 2023-09-23T21:03:58.480 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.5899999999999999e-06 2023-09-23T21:03:58.480 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:03:58.480 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:58.481 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:03:58.481 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:20.930827+0000", 2023-09-23T21:03:58.481 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.5949000000000003e-05 2023-09-23T21:03:58.481 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:03:58.481 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:58.481 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:03:58.482 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:20.930831+0000", 2023-09-23T21:03:58.482 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.4199999999999999e-06 2023-09-23T21:03:58.482 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:03:58.482 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:58.482 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:03:58.483 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:21.961980+0000", 2023-09-23T21:03:58.483 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.0311488959999999 2023-09-23T21:03:58.483 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:03:58.483 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:58.483 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:03:58.483 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:21.962078+0000", 2023-09-23T21:03:58.483 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.8317000000000001e-05 2023-09-23T21:03:58.484 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:03:58.484 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:58.484 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 0,2", 2023-09-23T21:03:58.484 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:21.962194+0000", 2023-09-23T21:03:58.484 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000115528 2023-09-23T21:03:58.484 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:03:58.484 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:58.485 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:03:58.485 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:21.969134+0000", 2023-09-23T21:03:58.485 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0069402370000000001 2023-09-23T21:03:58.485 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:03:58.485 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:58.485 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:03:58.485 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:21.976008+0000", 2023-09-23T21:03:58.485 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.006874326 2023-09-23T21:03:58.485 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:03:58.486 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:58.486 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:03:58.486 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:21.985762+0000", 2023-09-23T21:03:58.486 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0097536199999999993 2023-09-23T21:03:58.486 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:03:58.486 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:58.486 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:03:58.486 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:21.985793+0000", 2023-09-23T21:03:58.486 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.1628999999999998e-05 2023-09-23T21:03:58.486 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:03:58.486 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:58.487 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:03:58.487 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:21.985854+0000", 2023-09-23T21:03:58.487 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.0838000000000003e-05 2023-09-23T21:03:58.487 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:03:58.487 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:03:58.487 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:03:58.487 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:03:58.487 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:58.487 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.34797.0:167255 6.a 6:574da207:::benchmark_data_smithi026_223868_object66129:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e158)", 2023-09-23T21:03:58.487 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T20:55:20.936722+0000", 2023-09-23T21:03:58.487 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 517.51742303900005, 2023-09-23T21:03:58.487 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.053378076, 2023-09-23T21:03:58.488 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:03:58.488 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:03:58.488 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:03:58.488 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.34797", 2023-09-23T21:03:58.488 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T21:03:58.488 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 167255 2023-09-23T21:03:58.488 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:03:58.488 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:03:58.488 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:58.488 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:03:58.488 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:20.936722+0000", 2023-09-23T21:03:58.488 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:03:58.489 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:03:58.489 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:58.489 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:03:58.489 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:20.936722+0000", 2023-09-23T21:03:58.489 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:03:58.489 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:03:58.489 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:58.489 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:03:58.489 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:20.936721+0000", 2023-09-23T21:03:58.489 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.999999 2023-09-23T21:03:58.489 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:03:58.489 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:58.490 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:03:58.490 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:20.936725+0000", 2023-09-23T21:03:58.490 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.3150000000000002e-06 2023-09-23T21:03:58.490 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:03:58.490 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:58.490 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:03:58.490 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:20.936734+0000", 2023-09-23T21:03:58.490 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.2199999999999998e-06 2023-09-23T21:03:58.490 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:03:58.490 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:58.491 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:03:58.491 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:20.936739+0000", 2023-09-23T21:03:58.491 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.4309999999999996e-06 2023-09-23T21:03:58.491 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:03:58.491 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:58.491 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:03:58.491 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:20.936768+0000", 2023-09-23T21:03:58.491 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.9567999999999999e-05 2023-09-23T21:03:58.491 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:03:58.491 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:58.491 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:03:58.491 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:20.936774+0000", 2023-09-23T21:03:58.492 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.485e-06 2023-09-23T21:03:58.492 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:03:58.492 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:58.492 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:03:58.492 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:21.964309+0000", 2023-09-23T21:03:58.492 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.027534704 2023-09-23T21:03:58.492 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:03:58.492 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:58.492 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:03:58.492 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:21.964430+0000", 2023-09-23T21:03:58.492 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00012104000000000001 2023-09-23T21:03:58.493 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:03:58.493 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:58.493 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 0,2", 2023-09-23T21:03:58.493 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:21.964551+0000", 2023-09-23T21:03:58.493 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00012153 2023-09-23T21:03:58.493 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:03:58.493 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:58.493 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:03:58.493 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:21.971489+0000", 2023-09-23T21:03:58.493 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0069378219999999997 2023-09-23T21:03:58.493 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:03:58.493 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:58.494 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:03:58.494 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:21.981600+0000", 2023-09-23T21:03:58.494 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.010111063 2023-09-23T21:03:58.494 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:03:58.494 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:58.494 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:03:58.494 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:21.989954+0000", 2023-09-23T21:03:58.494 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0083543060000000006 2023-09-23T21:03:58.494 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:03:58.494 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:58.494 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:03:58.494 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:21.990003+0000", 2023-09-23T21:03:58.495 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.8662999999999998e-05 2023-09-23T21:03:58.495 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:03:58.495 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:58.495 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:03:58.495 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:21.990100+0000", 2023-09-23T21:03:58.495 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.6912999999999999e-05 2023-09-23T21:03:58.495 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:03:58.495 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:03:58.495 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:03:58.495 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:03:58.495 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:58.496 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.34797.0:167281 6.a 6:5de3ccd5:::benchmark_data_smithi026_223868_object66155:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e158)", 2023-09-23T21:03:58.496 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T20:55:20.963939+0000", 2023-09-23T21:03:58.496 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 517.49020562400005, 2023-09-23T21:03:58.496 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.026322833, 2023-09-23T21:03:58.496 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:03:58.496 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:03:58.496 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:03:58.496 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.34797", 2023-09-23T21:03:58.496 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T21:03:58.496 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 167281 2023-09-23T21:03:58.496 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:03:58.496 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:03:58.497 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:58.497 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:03:58.497 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:20.963939+0000", 2023-09-23T21:03:58.497 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:03:58.497 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:03:58.497 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:58.497 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:03:58.497 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:20.963939+0000", 2023-09-23T21:03:58.497 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:03:58.497 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:03:58.497 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:58.497 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:03:58.498 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:20.963938+0000", 2023-09-23T21:03:58.498 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T21:03:58.498 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:03:58.498 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:58.498 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:03:58.498 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:20.963943+0000", 2023-09-23T21:03:58.498 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.2739999999999998e-06 2023-09-23T21:03:58.498 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:03:58.498 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:58.498 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:03:58.498 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:20.963958+0000", 2023-09-23T21:03:58.498 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.5096000000000001e-05 2023-09-23T21:03:58.499 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:03:58.499 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:58.499 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:03:58.499 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:20.963966+0000", 2023-09-23T21:03:58.499 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.5140000000000004e-06 2023-09-23T21:03:58.499 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:03:58.499 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:58.499 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:03:58.499 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:20.964006+0000", 2023-09-23T21:03:58.499 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.0504999999999999e-05 2023-09-23T21:03:58.499 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:03:58.499 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:58.499 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:03:58.500 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:20.964015+0000", 2023-09-23T21:03:58.500 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.8070000000000006e-06 2023-09-23T21:03:58.500 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:03:58.500 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:58.500 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:03:58.500 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:21.966539+0000", 2023-09-23T21:03:58.500 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.002524204 2023-09-23T21:03:58.500 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:03:58.500 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:58.500 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:03:58.500 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:21.966674+0000", 2023-09-23T21:03:58.500 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00013492699999999999 2023-09-23T21:03:58.501 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:03:58.501 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:58.501 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 0,2", 2023-09-23T21:03:58.501 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:21.967167+0000", 2023-09-23T21:03:58.501 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000492549 2023-09-23T21:03:58.501 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:03:58.501 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:58.501 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:03:58.501 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:21.971671+0000", 2023-09-23T21:03:58.501 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0045045529999999997 2023-09-23T21:03:58.501 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:03:58.501 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:58.501 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:03:58.502 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:21.981643+0000", 2023-09-23T21:03:58.502 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0099714829999999997 2023-09-23T21:03:58.502 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:03:58.502 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:58.502 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:03:58.502 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:21.990135+0000", 2023-09-23T21:03:58.502 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0084919009999999996 2023-09-23T21:03:58.502 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:03:58.502 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:58.502 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:03:58.502 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:21.990164+0000", 2023-09-23T21:03:58.503 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.9549000000000001e-05 2023-09-23T21:03:58.503 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:03:58.503 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:58.503 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:03:58.503 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:21.990262+0000", 2023-09-23T21:03:58.503 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.781e-05 2023-09-23T21:03:58.503 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:03:58.503 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:03:58.503 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:03:58.503 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:03:58.503 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:58.503 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.34797.0:167305 6.a 6:5183351c:::benchmark_data_smithi026_223868_object66179:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e158)", 2023-09-23T21:03:58.504 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T20:55:20.991170+0000", 2023-09-23T21:03:58.504 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 517.46297446799997, 2023-09-23T21:03:58.504 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.999303524, 2023-09-23T21:03:58.504 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:03:58.504 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:03:58.504 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:03:58.504 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.34797", 2023-09-23T21:03:58.504 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T21:03:58.504 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 167305 2023-09-23T21:03:58.504 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:03:58.504 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:03:58.504 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:58.504 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:03:58.504 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:20.991170+0000", 2023-09-23T21:03:58.505 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:03:58.505 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:03:58.505 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:58.505 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:03:58.505 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:20.991170+0000", 2023-09-23T21:03:58.505 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:03:58.505 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:03:58.505 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:58.505 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:03:58.505 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:20.991169+0000", 2023-09-23T21:03:58.505 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T21:03:58.505 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:03:58.506 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:58.506 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:03:58.506 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:20.991176+0000", 2023-09-23T21:03:58.506 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.2219999999999996e-06 2023-09-23T21:03:58.506 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:03:58.506 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:58.506 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:03:58.506 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:20.991192+0000", 2023-09-23T21:03:58.506 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.5798e-05 2023-09-23T21:03:58.506 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:03:58.506 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:58.506 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:03:58.507 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:20.991199+0000", 2023-09-23T21:03:58.507 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.516e-06 2023-09-23T21:03:58.507 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:03:58.507 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:58.507 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:03:58.507 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:20.991264+0000", 2023-09-23T21:03:58.507 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.5135000000000006e-05 2023-09-23T21:03:58.507 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:03:58.507 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:58.507 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:03:58.507 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:20.991272+0000", 2023-09-23T21:03:58.508 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.7250000000000003e-06 2023-09-23T21:03:58.508 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:03:58.508 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:58.508 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:03:58.508 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:21.969162+0000", 2023-09-23T21:03:58.508 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.97788947999999998 2023-09-23T21:03:58.508 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:03:58.509 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:58.509 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:03:58.509 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:21.969309+0000", 2023-09-23T21:03:58.509 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00014732499999999999 2023-09-23T21:03:58.509 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:03:58.509 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:58.509 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 0,2", 2023-09-23T21:03:58.509 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:21.969431+0000", 2023-09-23T21:03:58.510 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000121797 2023-09-23T21:03:58.510 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:03:58.510 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:58.510 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:03:58.510 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:21.976014+0000", 2023-09-23T21:03:58.510 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0065832219999999997 2023-09-23T21:03:58.510 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:03:58.511 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:58.511 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:03:58.511 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:21.985730+0000", 2023-09-23T21:03:58.511 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0097158490000000004 2023-09-23T21:03:58.511 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:03:58.511 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:58.511 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:03:58.511 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:21.990318+0000", 2023-09-23T21:03:58.512 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0045880269999999997 2023-09-23T21:03:58.512 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:03:58.512 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:58.512 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:03:58.512 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:21.990372+0000", 2023-09-23T21:03:58.512 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.3658e-05 2023-09-23T21:03:58.512 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:03:58.512 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:58.512 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:03:58.512 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:21.990474+0000", 2023-09-23T21:03:58.513 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00010228799999999999 2023-09-23T21:03:58.513 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:03:58.513 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:03:58.513 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:03:58.513 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:03:58.513 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:58.513 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.34797.0:167315 6.a 6:5232d899:::benchmark_data_smithi026_223868_object66189:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e158)", 2023-09-23T21:03:58.513 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T20:55:21.002335+0000", 2023-09-23T21:03:58.513 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 517.45180973900005, 2023-09-23T21:03:58.513 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.988365296, 2023-09-23T21:03:58.513 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:03:58.513 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:03:58.514 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:03:58.514 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.34797", 2023-09-23T21:03:58.514 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T21:03:58.514 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 167315 2023-09-23T21:03:58.514 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:03:58.514 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:03:58.514 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:58.514 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:03:58.514 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:21.002335+0000", 2023-09-23T21:03:58.514 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:03:58.514 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:03:58.515 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:58.515 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:03:58.515 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:21.002335+0000", 2023-09-23T21:03:58.515 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:03:58.515 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:03:58.515 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:58.515 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:03:58.515 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:21.002333+0000", 2023-09-23T21:03:58.515 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999981 2023-09-23T21:03:58.515 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:03:58.515 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:58.515 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:03:58.516 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:21.002342+0000", 2023-09-23T21:03:58.516 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.0149999999999992e-06 2023-09-23T21:03:58.516 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:03:58.516 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:58.516 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:03:58.516 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:21.002357+0000", 2023-09-23T21:03:58.516 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.4859e-05 2023-09-23T21:03:58.516 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:03:58.516 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:58.516 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:03:58.516 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:21.002366+0000", 2023-09-23T21:03:58.516 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.7099999999999996e-06 2023-09-23T21:03:58.517 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:03:58.517 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:58.517 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:03:58.517 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:21.002492+0000", 2023-09-23T21:03:58.517 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00012614200000000001 2023-09-23T21:03:58.517 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:03:58.517 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:58.517 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:03:58.517 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:21.002502+0000", 2023-09-23T21:03:58.517 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.0079e-05 2023-09-23T21:03:58.517 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:03:58.517 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:58.518 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:03:58.518 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:21.971710+0000", 2023-09-23T21:03:58.518 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.96920738699999998 2023-09-23T21:03:58.518 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:03:58.518 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:58.518 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:03:58.518 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:21.971831+0000", 2023-09-23T21:03:58.518 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00012119499999999999 2023-09-23T21:03:58.518 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:03:58.518 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:58.518 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 0,2", 2023-09-23T21:03:58.519 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:21.971946+0000", 2023-09-23T21:03:58.519 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000115135 2023-09-23T21:03:58.519 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:03:58.519 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:58.519 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:03:58.519 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:21.979629+0000", 2023-09-23T21:03:58.519 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0076829059999999998 2023-09-23T21:03:58.519 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:03:58.519 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:58.519 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:03:58.519 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:21.990521+0000", 2023-09-23T21:03:58.520 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.010892013000000001 2023-09-23T21:03:58.520 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:03:58.520 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:58.520 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:03:58.520 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:21.990565+0000", 2023-09-23T21:03:58.520 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.4579000000000001e-05 2023-09-23T21:03:58.520 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:03:58.520 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:58.520 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:03:58.520 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:21.990605+0000", 2023-09-23T21:03:58.520 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.9854000000000002e-05 2023-09-23T21:03:58.521 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:03:58.521 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:58.521 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:03:58.521 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:21.990701+0000", 2023-09-23T21:03:58.521 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.5227000000000002e-05 2023-09-23T21:03:58.521 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:03:58.521 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:03:58.521 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:03:58.521 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:03:58.521 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:58.521 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.34797.0:167338 6.a 6:571cce03:::benchmark_data_smithi026_223868_object66212:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e158)", 2023-09-23T21:03:58.521 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T20:55:21.027265+0000", 2023-09-23T21:03:58.521 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 517.42687997500002, 2023-09-23T21:03:58.522 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.976460098, 2023-09-23T21:03:58.522 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:03:58.522 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:03:58.522 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:03:58.522 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.34797", 2023-09-23T21:03:58.522 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T21:03:58.522 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 167338 2023-09-23T21:03:58.522 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:03:58.522 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:03:58.522 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:58.522 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:03:58.523 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:21.027265+0000", 2023-09-23T21:03:58.523 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:03:58.523 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:03:58.523 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:58.523 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:03:58.523 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:21.027265+0000", 2023-09-23T21:03:58.523 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:03:58.523 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:03:58.523 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:58.523 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:03:58.523 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:21.027263+0000", 2023-09-23T21:03:58.524 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999981 2023-09-23T21:03:58.524 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:03:58.524 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:58.524 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:03:58.524 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:21.027270+0000", 2023-09-23T21:03:58.524 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.5329999999999997e-06 2023-09-23T21:03:58.524 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:03:58.524 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:58.524 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:03:58.524 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:21.027283+0000", 2023-09-23T21:03:58.524 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.2516e-05 2023-09-23T21:03:58.524 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:03:58.525 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:58.525 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:03:58.525 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:21.027290+0000", 2023-09-23T21:03:58.525 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.4989999999999999e-06 2023-09-23T21:03:58.525 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:03:58.525 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:58.525 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:03:58.525 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:21.027342+0000", 2023-09-23T21:03:58.525 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.2244000000000003e-05 2023-09-23T21:03:58.525 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:03:58.525 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:58.525 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:03:58.526 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:21.027351+0000", 2023-09-23T21:03:58.526 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.8410000000000006e-06 2023-09-23T21:03:58.526 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:03:58.526 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:58.526 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:03:58.526 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:21.974196+0000", 2023-09-23T21:03:58.526 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.94684518299999998 2023-09-23T21:03:58.526 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:03:58.526 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:58.526 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:03:58.527 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:21.974318+0000", 2023-09-23T21:03:58.527 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00012264 2023-09-23T21:03:58.527 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:03:58.527 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:58.527 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 0,2", 2023-09-23T21:03:58.527 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:21.974384+0000", 2023-09-23T21:03:58.527 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.5337000000000001e-05 2023-09-23T21:03:58.527 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:03:58.527 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:58.527 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:03:58.527 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:21.981609+0000", 2023-09-23T21:03:58.527 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00722553 2023-09-23T21:03:58.527 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:03:58.528 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:58.528 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:03:58.528 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:21.995381+0000", 2023-09-23T21:03:58.528 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.013771781 2023-09-23T21:03:58.528 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:03:58.528 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:58.528 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:03:58.528 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:22.003516+0000", 2023-09-23T21:03:58.528 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0081353399999999996 2023-09-23T21:03:58.528 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:03:58.528 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:58.528 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:03:58.529 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:22.003568+0000", 2023-09-23T21:03:58.529 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.1354000000000003e-05 2023-09-23T21:03:58.529 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:03:58.529 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:58.529 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:03:58.529 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:22.003725+0000", 2023-09-23T21:03:58.529 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00015731400000000001 2023-09-23T21:03:58.529 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:03:58.529 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:03:58.529 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:03:58.529 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:03:58.529 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:58.530 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.34797.0:177896 6.a 6:540ceadc:::benchmark_data_smithi026_223868_object76770:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e163)", 2023-09-23T21:03:58.530 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T20:55:34.908670+0000", 2023-09-23T21:03:58.530 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 503.54547542, 2023-09-23T21:03:58.530 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.1466390440000001, 2023-09-23T21:03:58.530 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:03:58.530 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "reached pg", 2023-09-23T21:03:58.530 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:03:58.530 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.34797", 2023-09-23T21:03:58.530 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T21:03:58.530 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 177896 2023-09-23T21:03:58.530 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:03:58.530 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:03:58.531 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:58.531 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:03:58.531 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:34.908670+0000", 2023-09-23T21:03:58.531 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:03:58.531 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:03:58.531 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:58.531 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:03:58.531 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:34.908670+0000", 2023-09-23T21:03:58.531 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:03:58.531 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:03:58.531 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:58.531 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:03:58.531 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:34.908668+0000", 2023-09-23T21:03:58.532 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.999999 2023-09-23T21:03:58.532 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:03:58.532 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:58.532 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:03:58.532 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:34.908673+0000", 2023-09-23T21:03:58.532 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.4769999999999997e-06 2023-09-23T21:03:58.532 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:03:58.532 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:58.532 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:03:58.532 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:34.908683+0000", 2023-09-23T21:03:58.532 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.0254e-05 2023-09-23T21:03:58.532 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:03:58.532 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:58.533 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:03:58.533 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:34.908687+0000", 2023-09-23T21:03:58.533 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.1219999999999997e-06 2023-09-23T21:03:58.533 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:03:58.533 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:58.533 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:03:58.533 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:34.908728+0000", 2023-09-23T21:03:58.533 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.0781999999999997e-05 2023-09-23T21:03:58.533 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:03:58.533 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:58.533 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:03:58.533 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:34.908820+0000", 2023-09-23T21:03:58.533 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.2058999999999994e-05 2023-09-23T21:03:58.534 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:03:58.534 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:58.534 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 0,2", 2023-09-23T21:03:58.534 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:34.908888+0000", 2023-09-23T21:03:58.534 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.8287000000000005e-05 2023-09-23T21:03:58.534 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:03:58.534 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:58.534 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:03:58.534 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:34.911386+0000", 2023-09-23T21:03:58.534 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0024977459999999999 2023-09-23T21:03:58.534 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:03:58.535 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:58.535 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:03:58.535 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:34.923103+0000", 2023-09-23T21:03:58.535 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.011717293 2023-09-23T21:03:58.535 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:03:58.535 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:58.535 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:03:58.535 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:34.926697+0000", 2023-09-23T21:03:58.535 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0035937220000000001 2023-09-23T21:03:58.535 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:03:58.535 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:58.535 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:03:58.536 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:34.926708+0000", 2023-09-23T21:03:58.536 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.0339e-05 2023-09-23T21:03:58.536 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:03:58.536 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:58.536 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:03:58.536 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:35.946870+0000", 2023-09-23T21:03:58.536 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.0201623930000001 2023-09-23T21:03:58.536 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:03:58.536 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:58.536 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:03:58.536 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:35.946882+0000", 2023-09-23T21:03:58.536 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.1936999999999999e-05 2023-09-23T21:03:58.537 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:03:58.537 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:58.537 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:03:58.537 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:37.039221+0000", 2023-09-23T21:03:58.537 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.092338928 2023-09-23T21:03:58.537 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:03:58.537 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:58.537 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for missing object", 2023-09-23T21:03:58.537 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:37.039314+0000", 2023-09-23T21:03:58.537 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.3096000000000004e-05 2023-09-23T21:03:58.537 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:03:58.538 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:58.538 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:03:58.538 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:37.055200+0000", 2023-09-23T21:03:58.538 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.015886493000000002 2023-09-23T21:03:58.538 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:03:58.538 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:58.538 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:03:58.538 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:37.055309+0000", 2023-09-23T21:03:58.538 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00010818 2023-09-23T21:03:58.538 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:03:58.538 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:03:58.539 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:03:58.539 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:03:58.539 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:58.539 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.34797.0:177903 6.a 6:5c2bf67a:::benchmark_data_smithi026_223868_object76777:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e163)", 2023-09-23T21:03:58.539 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T20:55:34.915862+0000", 2023-09-23T21:03:58.539 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 503.53828283799999, 2023-09-23T21:03:58.539 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.1423018539999998, 2023-09-23T21:03:58.539 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:03:58.539 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "reached pg", 2023-09-23T21:03:58.539 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:03:58.539 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.34797", 2023-09-23T21:03:58.539 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T21:03:58.540 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 177903 2023-09-23T21:03:58.540 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:03:58.540 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:03:58.540 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:58.540 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:03:58.540 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:34.915862+0000", 2023-09-23T21:03:58.540 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:03:58.540 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:03:58.540 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:58.540 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:03:58.540 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:34.915862+0000", 2023-09-23T21:03:58.540 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:03:58.540 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:03:58.541 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:58.541 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:03:58.541 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:34.915861+0000", 2023-09-23T21:03:58.541 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.999999 2023-09-23T21:03:58.541 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:03:58.541 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:58.541 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:03:58.541 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:34.915866+0000", 2023-09-23T21:03:58.541 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.78e-06 2023-09-23T21:03:58.541 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:03:58.541 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:58.541 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:03:58.542 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:34.915881+0000", 2023-09-23T21:03:58.542 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.552e-05 2023-09-23T21:03:58.542 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:03:58.542 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:58.542 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:03:58.542 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:34.915889+0000", 2023-09-23T21:03:58.542 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.1960000000000002e-06 2023-09-23T21:03:58.542 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:03:58.542 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:58.542 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:03:58.542 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:34.915932+0000", 2023-09-23T21:03:58.542 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.3272000000000002e-05 2023-09-23T21:03:58.543 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:03:58.543 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:58.543 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:03:58.543 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:34.916061+0000", 2023-09-23T21:03:58.543 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00012904999999999999 2023-09-23T21:03:58.543 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:03:58.543 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:58.543 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 0,2", 2023-09-23T21:03:58.543 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:34.916135+0000", 2023-09-23T21:03:58.543 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.4144e-05 2023-09-23T21:03:58.543 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:03:58.543 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:58.544 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:03:58.544 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:34.918475+0000", 2023-09-23T21:03:58.544 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.002339921 2023-09-23T21:03:58.544 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:03:58.544 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:58.544 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:03:58.544 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:34.926898+0000", 2023-09-23T21:03:58.544 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0084229939999999996 2023-09-23T21:03:58.544 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:03:58.544 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:58.544 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:03:58.544 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:34.926907+0000", 2023-09-23T21:03:58.544 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.9590000000000001e-06 2023-09-23T21:03:58.545 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:03:58.545 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:58.545 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:03:58.545 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:35.947668+0000", 2023-09-23T21:03:58.545 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.0207606359999999 2023-09-23T21:03:58.545 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:03:58.545 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:58.545 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:03:58.545 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:35.947676+0000", 2023-09-23T21:03:58.545 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.7029999999999995e-06 2023-09-23T21:03:58.545 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:03:58.545 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:58.546 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:03:58.546 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:37.039586+0000", 2023-09-23T21:03:58.546 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.0919099109999999 2023-09-23T21:03:58.546 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:03:58.546 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:58.546 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for missing object", 2023-09-23T21:03:58.546 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:37.039684+0000", 2023-09-23T21:03:58.546 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.7732999999999994e-05 2023-09-23T21:03:58.546 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:03:58.546 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:58.546 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:03:58.547 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:37.058038+0000", 2023-09-23T21:03:58.547 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.018353669999999999 2023-09-23T21:03:58.547 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:03:58.547 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:58.547 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:03:58.547 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:37.058164+0000", 2023-09-23T21:03:58.547 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00012637400000000001 2023-09-23T21:03:58.547 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:03:58.547 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:03:58.547 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:03:58.547 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:03:58.547 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:58.547 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.34797.0:177914 6.a 6:5c0a39bf:::benchmark_data_smithi026_223868_object76788:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e164)", 2023-09-23T21:03:58.548 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T20:55:34.934127+0000", 2023-09-23T21:03:58.548 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 503.52001823699999, 2023-09-23T21:03:58.548 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.1279153630000001, 2023-09-23T21:03:58.548 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:03:58.548 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:03:58.548 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:03:58.548 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.34797", 2023-09-23T21:03:58.548 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T21:03:58.548 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 177914 2023-09-23T21:03:58.548 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:03:58.548 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:03:58.549 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:58.549 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:03:58.549 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:34.934127+0000", 2023-09-23T21:03:58.549 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:03:58.549 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:03:58.549 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:58.549 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:03:58.549 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:34.934127+0000", 2023-09-23T21:03:58.549 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:03:58.549 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:03:58.549 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:58.549 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:03:58.549 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:34.934126+0000", 2023-09-23T21:03:58.550 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999995 2023-09-23T21:03:58.550 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:03:58.550 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:58.550 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:03:58.550 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:34.934129+0000", 2023-09-23T21:03:58.550 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.1690000000000001e-06 2023-09-23T21:03:58.550 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:03:58.550 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:58.550 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:03:58.550 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:34.934136+0000", 2023-09-23T21:03:58.550 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.8120000000000002e-06 2023-09-23T21:03:58.551 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:03:58.551 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:58.551 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:03:58.551 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:34.934141+0000", 2023-09-23T21:03:58.551 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.7090000000000001e-06 2023-09-23T21:03:58.551 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:03:58.551 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:58.551 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:03:58.551 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:34.935261+0000", 2023-09-23T21:03:58.551 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0011200369999999999 2023-09-23T21:03:58.551 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:03:58.552 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:58.552 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:03:58.552 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:34.935272+0000", 2023-09-23T21:03:58.552 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.1202999999999999e-05 2023-09-23T21:03:58.552 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:03:58.552 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:58.552 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:03:58.552 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:35.947706+0000", 2023-09-23T21:03:58.552 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.012434187 2023-09-23T21:03:58.552 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:03:58.552 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:58.553 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:03:58.553 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:35.947714+0000", 2023-09-23T21:03:58.553 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.8990000000000001e-06 2023-09-23T21:03:58.553 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:03:58.553 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:58.553 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:03:58.553 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:37.039714+0000", 2023-09-23T21:03:58.553 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.091999768 2023-09-23T21:03:58.553 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:03:58.553 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:58.553 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:03:58.553 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:37.039838+0000", 2023-09-23T21:03:58.553 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00012372300000000001 2023-09-23T21:03:58.554 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:03:58.554 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:58.554 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 0,6", 2023-09-23T21:03:58.554 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:37.039921+0000", 2023-09-23T21:03:58.554 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.3489e-05 2023-09-23T21:03:58.554 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:03:58.554 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:58.554 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:03:58.554 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:37.053854+0000", 2023-09-23T21:03:58.554 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.013933164 2023-09-23T21:03:58.555 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:03:58.555 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:58.555 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:03:58.555 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:37.054159+0000", 2023-09-23T21:03:58.555 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00030508299999999999 2023-09-23T21:03:58.555 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:03:58.555 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:58.555 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:03:58.555 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:37.061887+0000", 2023-09-23T21:03:58.555 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0077278579999999998 2023-09-23T21:03:58.555 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:03:58.555 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:58.556 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:03:58.556 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:37.061930+0000", 2023-09-23T21:03:58.556 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.2454999999999999e-05 2023-09-23T21:03:58.556 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:03:58.556 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:58.556 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:03:58.556 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:37.062042+0000", 2023-09-23T21:03:58.556 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000112333 2023-09-23T21:03:58.556 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:03:58.556 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:03:58.556 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:03:58.557 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:03:58.557 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:58.557 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.34797.0:177920 6.a 6:5d69f263:::benchmark_data_smithi026_223868_object76794:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e164)", 2023-09-23T21:03:58.557 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T20:55:34.954622+0000", 2023-09-23T21:03:58.557 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 503.49952264699999, 2023-09-23T21:03:58.557 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.1076863440000002, 2023-09-23T21:03:58.557 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:03:58.557 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:03:58.557 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:03:58.557 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.34797", 2023-09-23T21:03:58.557 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T21:03:58.557 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 177920 2023-09-23T21:03:58.557 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:03:58.558 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:03:58.558 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:58.558 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:03:58.558 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:34.954622+0000", 2023-09-23T21:03:58.558 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:03:58.558 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:03:58.558 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:58.558 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:03:58.558 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:34.954622+0000", 2023-09-23T21:03:58.558 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:03:58.558 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:03:58.558 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:58.559 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:03:58.559 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:34.954621+0000", 2023-09-23T21:03:58.559 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T21:03:58.559 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:03:58.559 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:58.559 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:03:58.559 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:34.954627+0000", 2023-09-23T21:03:58.559 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.7440000000000004e-06 2023-09-23T21:03:58.559 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:03:58.559 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:58.559 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:03:58.559 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:34.954638+0000", 2023-09-23T21:03:58.559 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.1487e-05 2023-09-23T21:03:58.560 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:03:58.560 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:58.560 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:03:58.560 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:34.954644+0000", 2023-09-23T21:03:58.560 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.0519999999999998e-06 2023-09-23T21:03:58.560 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:03:58.560 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:58.560 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:03:58.560 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:34.954700+0000", 2023-09-23T21:03:58.560 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.5337999999999998e-05 2023-09-23T21:03:58.560 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:03:58.560 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:58.561 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:03:58.561 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:34.954713+0000", 2023-09-23T21:03:58.561 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.3003000000000001e-05 2023-09-23T21:03:58.561 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:03:58.561 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:58.561 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:03:58.561 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:35.947751+0000", 2023-09-23T21:03:58.561 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.99303834199999996 2023-09-23T21:03:58.561 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:03:58.561 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:58.561 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:03:58.562 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:35.947761+0000", 2023-09-23T21:03:58.562 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.4979999999999995e-06 2023-09-23T21:03:58.562 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:03:58.562 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:58.562 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:03:58.562 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:37.042465+0000", 2023-09-23T21:03:58.562 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.0947049710000001 2023-09-23T21:03:58.562 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:03:58.562 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:58.562 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:03:58.562 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:37.042599+0000", 2023-09-23T21:03:58.562 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00013388199999999999 2023-09-23T21:03:58.563 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:03:58.563 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:58.563 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 0,6", 2023-09-23T21:03:58.563 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:37.042722+0000", 2023-09-23T21:03:58.563 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000122893 2023-09-23T21:03:58.563 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:03:58.563 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:58.563 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:03:58.563 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:37.053862+0000", 2023-09-23T21:03:58.563 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.011139540999999999 2023-09-23T21:03:58.563 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:03:58.563 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:58.563 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:03:58.564 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:37.057624+0000", 2023-09-23T21:03:58.564 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.003762661 2023-09-23T21:03:58.564 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:03:58.564 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:58.564 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:03:58.564 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:37.062109+0000", 2023-09-23T21:03:58.564 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0044847259999999996 2023-09-23T21:03:58.564 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:03:58.564 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:58.564 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:03:58.564 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:37.062163+0000", 2023-09-23T21:03:58.564 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.346e-05 2023-09-23T21:03:58.564 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:03:58.565 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:58.565 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:03:58.565 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:37.062309+0000", 2023-09-23T21:03:58.565 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00014601300000000001 2023-09-23T21:03:58.565 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:03:58.565 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:03:58.565 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:03:58.565 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:03:58.565 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:58.565 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.34797.0:177923 6.a 6:53679877:::benchmark_data_smithi026_223868_object76797:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e164)", 2023-09-23T21:03:58.565 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T20:55:34.956263+0000", 2023-09-23T21:03:58.565 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 503.49788177200003, 2023-09-23T21:03:58.565 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.1062447940000002, 2023-09-23T21:03:58.566 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:03:58.566 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:03:58.566 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:03:58.566 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.34797", 2023-09-23T21:03:58.566 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T21:03:58.566 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 177923 2023-09-23T21:03:58.566 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:03:58.566 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:03:58.566 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:58.566 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:03:58.566 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:34.956263+0000", 2023-09-23T21:03:58.566 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:03:58.567 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:03:58.567 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:58.567 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:03:58.567 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:34.956263+0000", 2023-09-23T21:03:58.567 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:03:58.567 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:03:58.567 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:58.567 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:03:58.567 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:34.956262+0000", 2023-09-23T21:03:58.567 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T21:03:58.567 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:03:58.567 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:58.567 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:03:58.568 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:34.956268+0000", 2023-09-23T21:03:58.568 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.6189999999999996e-06 2023-09-23T21:03:58.568 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:03:58.568 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:58.568 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:03:58.568 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:34.956283+0000", 2023-09-23T21:03:58.568 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.4725999999999999e-05 2023-09-23T21:03:58.568 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:03:58.568 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:58.568 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:03:58.568 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:34.956289+0000", 2023-09-23T21:03:58.568 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.1380000000000004e-06 2023-09-23T21:03:58.568 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:03:58.569 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:58.569 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:03:58.569 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:34.956345+0000", 2023-09-23T21:03:58.569 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.5288000000000003e-05 2023-09-23T21:03:58.569 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:03:58.569 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:58.569 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:03:58.569 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:34.956354+0000", 2023-09-23T21:03:58.569 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.9150000000000002e-06 2023-09-23T21:03:58.569 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:03:58.569 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:58.569 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:03:58.569 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:35.947803+0000", 2023-09-23T21:03:58.570 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.99144918699999995 2023-09-23T21:03:58.570 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:03:58.570 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:58.570 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:03:58.570 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:35.947811+0000", 2023-09-23T21:03:58.570 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.5129999999999996e-06 2023-09-23T21:03:58.570 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:03:58.570 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:58.570 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:03:58.570 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:37.047285+0000", 2023-09-23T21:03:58.570 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.099473857 2023-09-23T21:03:58.570 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:03:58.571 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:58.571 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:03:58.571 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:37.047408+0000", 2023-09-23T21:03:58.571 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00012244 2023-09-23T21:03:58.571 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:03:58.571 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:58.571 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 0,6", 2023-09-23T21:03:58.571 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:37.047526+0000", 2023-09-23T21:03:58.571 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000118539 2023-09-23T21:03:58.571 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:03:58.571 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:58.571 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:03:58.572 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:37.053868+0000", 2023-09-23T21:03:58.572 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0063422560000000001 2023-09-23T21:03:58.572 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:03:58.572 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:58.572 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:03:58.572 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:37.058203+0000", 2023-09-23T21:03:58.572 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0043349929999999997 2023-09-23T21:03:58.572 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:03:58.572 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:58.572 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:03:58.572 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:37.062353+0000", 2023-09-23T21:03:58.572 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0041494890000000001 2023-09-23T21:03:58.572 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:03:58.572 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:58.573 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:03:58.573 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:37.062398+0000", 2023-09-23T21:03:58.573 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.5639999999999997e-05 2023-09-23T21:03:58.573 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:03:58.573 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:58.573 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:03:58.573 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:37.062508+0000", 2023-09-23T21:03:58.573 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00010949400000000001 2023-09-23T21:03:58.573 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:03:58.573 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:03:58.573 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:03:58.573 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:03:58.574 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:58.574 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.34797.0:177948 6.a 6:597c0f94:::benchmark_data_smithi026_223868_object76822:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e164)", 2023-09-23T21:03:58.574 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T20:55:34.996671+0000", 2023-09-23T21:03:58.574 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 503.457473924, 2023-09-23T21:03:58.574 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.06610118, 2023-09-23T21:03:58.574 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:03:58.574 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:03:58.574 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:03:58.574 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.34797", 2023-09-23T21:03:58.574 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T21:03:58.574 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 177948 2023-09-23T21:03:58.574 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:03:58.574 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:03:58.575 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:58.575 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:03:58.575 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:34.996671+0000", 2023-09-23T21:03:58.575 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:03:58.575 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:03:58.575 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:58.575 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:03:58.575 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:34.996671+0000", 2023-09-23T21:03:58.575 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:03:58.575 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:03:58.575 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:58.575 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:03:58.575 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:34.996670+0000", 2023-09-23T21:03:58.576 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.999999 2023-09-23T21:03:58.576 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:03:58.576 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:58.576 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:03:58.576 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:34.996675+0000", 2023-09-23T21:03:58.576 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.6249999999999998e-06 2023-09-23T21:03:58.576 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:03:58.576 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:58.576 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:03:58.576 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:34.996684+0000", 2023-09-23T21:03:58.576 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.2450000000000004e-06 2023-09-23T21:03:58.576 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:03:58.576 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:58.577 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:03:58.577 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:34.996687+0000", 2023-09-23T21:03:58.577 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.563e-06 2023-09-23T21:03:58.577 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:03:58.577 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:58.577 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:03:58.577 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:34.996739+0000", 2023-09-23T21:03:58.577 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.1579000000000002e-05 2023-09-23T21:03:58.577 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:03:58.577 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:58.577 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:03:58.577 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:34.996746+0000", 2023-09-23T21:03:58.577 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.2690000000000002e-06 2023-09-23T21:03:58.578 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:03:58.578 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:58.578 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:03:58.578 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:35.947834+0000", 2023-09-23T21:03:58.578 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.95108812399999998 2023-09-23T21:03:58.578 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:03:58.578 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:58.578 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:03:58.578 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:35.947842+0000", 2023-09-23T21:03:58.578 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.7370000000000006e-06 2023-09-23T21:03:58.578 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:03:58.578 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:58.579 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:03:58.579 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:37.050299+0000", 2023-09-23T21:03:58.579 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.1024569989999999 2023-09-23T21:03:58.579 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:03:58.579 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:58.579 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:03:58.579 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:37.050434+0000", 2023-09-23T21:03:58.579 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00013435199999999999 2023-09-23T21:03:58.579 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:03:58.579 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:58.579 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 0,6", 2023-09-23T21:03:58.579 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:37.050565+0000", 2023-09-23T21:03:58.579 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00013146300000000001 2023-09-23T21:03:58.580 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:03:58.580 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:58.580 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:03:58.580 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:37.053877+0000", 2023-09-23T21:03:58.580 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0033121589999999998 2023-09-23T21:03:58.580 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:03:58.580 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:58.580 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:03:58.580 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:37.058255+0000", 2023-09-23T21:03:58.580 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0043776930000000002 2023-09-23T21:03:58.580 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:03:58.580 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:58.580 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:03:58.581 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:37.062560+0000", 2023-09-23T21:03:58.581 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0043055719999999997 2023-09-23T21:03:58.581 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:03:58.581 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:58.581 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:03:58.581 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:37.062636+0000", 2023-09-23T21:03:58.581 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.5417e-05 2023-09-23T21:03:58.581 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:03:58.581 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:58.581 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:03:58.581 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:37.062772+0000", 2023-09-23T21:03:58.581 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00013637499999999999 2023-09-23T21:03:58.581 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:03:58.582 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:03:58.582 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:03:58.582 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:03:58.582 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:58.582 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.34797.0:185827 6.a 6:51461ffe:::benchmark_data_smithi026_223868_object84701:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e172)", 2023-09-23T21:03:58.582 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T20:55:48.221799+0000", 2023-09-23T21:03:58.582 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 490.23234581200001, 2023-09-23T21:03:58.582 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.038071078, 2023-09-23T21:03:58.582 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:03:58.582 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:03:58.582 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:03:58.582 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.34797", 2023-09-23T21:03:58.583 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T21:03:58.583 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 185827 2023-09-23T21:03:58.583 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:03:58.583 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:03:58.583 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:58.583 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:03:58.583 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:48.221799+0000", 2023-09-23T21:03:58.583 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:03:58.583 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:03:58.583 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:58.583 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:03:58.583 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:48.221799+0000", 2023-09-23T21:03:58.583 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:03:58.584 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:03:58.584 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:58.584 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:03:58.584 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:48.221799+0000", 2023-09-23T21:03:58.584 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999995 2023-09-23T21:03:58.584 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:03:58.584 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:58.584 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:03:58.584 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:48.221802+0000", 2023-09-23T21:03:58.584 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.2049999999999998e-06 2023-09-23T21:03:58.584 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:03:58.584 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:58.584 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:03:58.585 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:48.221808+0000", 2023-09-23T21:03:58.585 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.19e-06 2023-09-23T21:03:58.585 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:03:58.585 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:58.585 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:03:58.585 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:48.221811+0000", 2023-09-23T21:03:58.585 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.52e-06 2023-09-23T21:03:58.585 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:03:58.585 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:58.585 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:03:58.585 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:48.221859+0000", 2023-09-23T21:03:58.585 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.8857999999999999e-05 2023-09-23T21:03:58.585 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:03:58.586 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:58.586 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:03:58.586 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:48.221871+0000", 2023-09-23T21:03:58.586 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.1174e-05 2023-09-23T21:03:58.586 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:03:58.586 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:58.586 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:03:58.586 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:49.249764+0000", 2023-09-23T21:03:58.586 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.027893758 2023-09-23T21:03:58.586 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:03:58.586 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:58.586 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:03:58.587 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:49.249884+0000", 2023-09-23T21:03:58.587 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000119438 2023-09-23T21:03:58.587 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:03:58.587 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:58.587 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 0,6", 2023-09-23T21:03:58.587 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:49.249965+0000", 2023-09-23T21:03:58.587 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.1518000000000001e-05 2023-09-23T21:03:58.587 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:03:58.587 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:58.587 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:03:58.587 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:49.252936+0000", 2023-09-23T21:03:58.588 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0029711009999999999 2023-09-23T21:03:58.588 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:03:58.588 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:58.588 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:03:58.588 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:49.256217+0000", 2023-09-23T21:03:58.588 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0032802489999999998 2023-09-23T21:03:58.588 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:03:58.588 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:58.588 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:03:58.588 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:49.259769+0000", 2023-09-23T21:03:58.588 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0035519509999999998 2023-09-23T21:03:58.588 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:03:58.588 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:58.589 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:03:58.589 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:49.259825+0000", 2023-09-23T21:03:58.589 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.6854999999999997e-05 2023-09-23T21:03:58.589 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:03:58.589 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:58.589 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:03:58.589 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:49.259870+0000", 2023-09-23T21:03:58.589 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.4774999999999997e-05 2023-09-23T21:03:58.589 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:03:58.589 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:03:58.589 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:03:58.589 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:03:58.589 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:58.590 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.34797.0:185834 6.a 6:59f98fb0:::benchmark_data_smithi026_223868_object84708:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e172)", 2023-09-23T21:03:58.590 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T20:55:48.229707+0000", 2023-09-23T21:03:58.590 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 490.22443798400002, 2023-09-23T21:03:58.590 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.0303501639999999, 2023-09-23T21:03:58.590 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:03:58.590 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:03:58.590 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:03:58.590 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.34797", 2023-09-23T21:03:58.590 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T21:03:58.590 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 185834 2023-09-23T21:03:58.590 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:03:58.590 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:03:58.591 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:58.591 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:03:58.591 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:48.229707+0000", 2023-09-23T21:03:58.591 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:03:58.591 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:03:58.591 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:58.591 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:03:58.591 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:48.229707+0000", 2023-09-23T21:03:58.591 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:03:58.591 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:03:58.591 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:58.591 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:03:58.591 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:48.229706+0000", 2023-09-23T21:03:58.592 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.999999 2023-09-23T21:03:58.592 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:03:58.592 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:58.592 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:03:58.592 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:48.229712+0000", 2023-09-23T21:03:58.592 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.7400000000000001e-06 2023-09-23T21:03:58.592 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:03:58.592 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:58.592 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:03:58.592 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:48.229724+0000", 2023-09-23T21:03:58.592 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.2797e-05 2023-09-23T21:03:58.592 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:03:58.592 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:58.593 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:03:58.593 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:48.229731+0000", 2023-09-23T21:03:58.593 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.5080000000000002e-06 2023-09-23T21:03:58.593 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:03:58.593 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:58.593 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:03:58.593 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:48.229783+0000", 2023-09-23T21:03:58.593 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.1953000000000002e-05 2023-09-23T21:03:58.593 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:03:58.593 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:58.593 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:03:58.593 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:48.229794+0000", 2023-09-23T21:03:58.593 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.1061e-05 2023-09-23T21:03:58.594 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:03:58.594 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:58.594 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:03:58.594 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:49.253001+0000", 2023-09-23T21:03:58.594 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.023207209 2023-09-23T21:03:58.594 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:03:58.594 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:58.594 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:03:58.594 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:49.253151+0000", 2023-09-23T21:03:58.594 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00014949599999999999 2023-09-23T21:03:58.594 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:03:58.594 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:58.595 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 0,6", 2023-09-23T21:03:58.595 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:49.253282+0000", 2023-09-23T21:03:58.595 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000131922 2023-09-23T21:03:58.595 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:03:58.595 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:58.595 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:03:58.595 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:49.259649+0000", 2023-09-23T21:03:58.595 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0063662659999999998 2023-09-23T21:03:58.595 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:03:58.595 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:58.595 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:03:58.595 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:49.259934+0000", 2023-09-23T21:03:58.596 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00028488500000000002 2023-09-23T21:03:58.596 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:03:58.596 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:58.596 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:03:58.596 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:49.259975+0000", 2023-09-23T21:03:58.596 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.1811000000000003e-05 2023-09-23T21:03:58.596 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:03:58.596 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:58.596 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:03:58.596 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:49.260024+0000", 2023-09-23T21:03:58.596 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.8155000000000002e-05 2023-09-23T21:03:58.596 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:03:58.596 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:58.596 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:03:58.597 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:49.260057+0000", 2023-09-23T21:03:58.597 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.3522000000000002e-05 2023-09-23T21:03:58.597 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:03:58.597 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:03:58.597 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:03:58.597 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:03:58.597 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:58.597 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.34797.0:185844 6.a 6:5b27a6f2:::benchmark_data_smithi026_223868_object84718:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e172)", 2023-09-23T21:03:58.597 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T20:55:48.237898+0000", 2023-09-23T21:03:58.597 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 490.216246759, 2023-09-23T21:03:58.597 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.022291493, 2023-09-23T21:03:58.597 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:03:58.598 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:03:58.598 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:03:58.598 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.34797", 2023-09-23T21:03:58.598 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T21:03:58.598 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 185844 2023-09-23T21:03:58.598 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:03:58.598 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:03:58.598 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:58.598 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:03:58.598 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:48.237898+0000", 2023-09-23T21:03:58.598 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:03:58.598 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:03:58.599 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:58.599 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:03:58.599 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:48.237898+0000", 2023-09-23T21:03:58.599 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:03:58.599 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:03:58.599 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:58.599 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:03:58.599 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:48.237897+0000", 2023-09-23T21:03:58.599 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.999999 2023-09-23T21:03:58.599 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:03:58.599 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:58.599 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:03:58.599 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:48.237903+0000", 2023-09-23T21:03:58.600 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.2179999999999998e-06 2023-09-23T21:03:58.600 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:03:58.600 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:58.600 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:03:58.600 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:48.237916+0000", 2023-09-23T21:03:58.600 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.3753e-05 2023-09-23T21:03:58.600 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:03:58.600 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:58.600 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:03:58.600 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:48.237923+0000", 2023-09-23T21:03:58.600 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.6200000000000001e-06 2023-09-23T21:03:58.600 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:03:58.600 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:58.601 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:03:58.601 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:48.237984+0000", 2023-09-23T21:03:58.601 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.1314000000000003e-05 2023-09-23T21:03:58.601 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:03:58.601 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:58.601 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:03:58.601 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:48.237994+0000", 2023-09-23T21:03:58.601 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.8050000000000001e-06 2023-09-23T21:03:58.601 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:03:58.601 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:58.601 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:03:58.601 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:49.256263+0000", 2023-09-23T21:03:58.601 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.018269149 2023-09-23T21:03:58.602 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:03:58.602 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:58.602 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:03:58.602 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:49.256351+0000", 2023-09-23T21:03:58.602 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.8261000000000001e-05 2023-09-23T21:03:58.602 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:03:58.602 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:58.602 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 0,6", 2023-09-23T21:03:58.602 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:49.256430+0000", 2023-09-23T21:03:58.602 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.8651000000000003e-05 2023-09-23T21:03:58.602 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:03:58.602 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:58.602 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:03:58.603 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:49.259924+0000", 2023-09-23T21:03:58.603 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0034939390000000002 2023-09-23T21:03:58.603 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:03:58.603 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:58.603 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:03:58.603 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:49.259942+0000", 2023-09-23T21:03:58.603 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.8261999999999999e-05 2023-09-23T21:03:58.603 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:03:58.603 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:58.603 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:03:58.603 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:49.260103+0000", 2023-09-23T21:03:58.603 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000160679 2023-09-23T21:03:58.603 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:03:58.604 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:58.604 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:03:58.604 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:49.260148+0000", 2023-09-23T21:03:58.604 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.5330999999999997e-05 2023-09-23T21:03:58.604 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:03:58.604 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:58.604 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:03:58.604 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:49.260190+0000", 2023-09-23T21:03:58.604 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.1409999999999998e-05 2023-09-23T21:03:58.604 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:03:58.604 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:03:58.604 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:03:58.604 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:03:58.605 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:58.605 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.34797.0:185884 6.a 6:5db88532:::benchmark_data_smithi026_223868_object84758:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e172)", 2023-09-23T21:03:58.605 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T20:55:48.282714+0000", 2023-09-23T21:03:58.605 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 490.17143093599998, 2023-09-23T21:03:58.605 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.99305554799999995, 2023-09-23T21:03:58.605 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:03:58.605 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:03:58.605 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:03:58.605 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.34797", 2023-09-23T21:03:58.605 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:03:58.606 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:03:58.607 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T21:03:58.607 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 185884 2023-09-23T21:03:58.607 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:03:58.607 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:03:58.607 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:58.607 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:03:58.607 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:48.282714+0000", 2023-09-23T21:03:58.607 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:03:58.607 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:03:58.607 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:58.607 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:03:58.607 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:48.282714+0000", 2023-09-23T21:03:58.608 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:03:58.608 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:03:58.608 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:58.608 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:03:58.608 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:48.282714+0000", 2023-09-23T21:03:58.608 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999995 2023-09-23T21:03:58.608 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:03:58.608 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:58.608 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:03:58.608 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:48.282717+0000", 2023-09-23T21:03:58.608 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.1760000000000002e-06 2023-09-23T21:03:58.608 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:03:58.608 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:58.609 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:03:58.609 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:48.282727+0000", 2023-09-23T21:03:58.609 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.8079999999999999e-06 2023-09-23T21:03:58.609 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:03:58.609 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:58.609 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:03:58.609 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:48.282731+0000", 2023-09-23T21:03:58.609 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.4129999999999999e-06 2023-09-23T21:03:58.609 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:03:58.609 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:58.609 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:03:58.609 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:48.282781+0000", 2023-09-23T21:03:58.609 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.9870999999999997e-05 2023-09-23T21:03:58.610 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:03:58.610 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:58.610 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:03:58.610 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:48.282789+0000", 2023-09-23T21:03:58.610 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.1689999999999992e-06 2023-09-23T21:03:58.610 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:03:58.610 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:58.610 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:03:58.610 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:49.260248+0000", 2023-09-23T21:03:58.610 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.97745879800000002 2023-09-23T21:03:58.610 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:03:58.610 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:58.610 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:03:58.611 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:49.260330+0000", 2023-09-23T21:03:58.611 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.2013000000000002e-05 2023-09-23T21:03:58.611 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:03:58.611 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:58.611 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 0,6", 2023-09-23T21:03:58.611 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:49.260394+0000", 2023-09-23T21:03:58.611 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.4393000000000005e-05 2023-09-23T21:03:58.611 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:03:58.611 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:58.611 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:03:58.613 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:49.269250+0000", 2023-09-23T21:03:58.613 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0088559889999999999 2023-09-23T21:03:58.613 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:03:58.613 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:58.613 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:03:58.613 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:49.269282+0000", 2023-09-23T21:03:58.613 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.1733999999999998e-05 2023-09-23T21:03:58.613 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:03:58.613 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:58.613 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:03:58.614 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:49.275670+0000", 2023-09-23T21:03:58.614 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0063884220000000004 2023-09-23T21:03:58.614 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:03:58.614 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:58.614 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:03:58.614 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:49.275729+0000", 2023-09-23T21:03:58.614 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.8371000000000001e-05 2023-09-23T21:03:58.614 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:03:58.614 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:58.614 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:03:58.614 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:49.275770+0000", 2023-09-23T21:03:58.614 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.0884999999999999e-05 2023-09-23T21:03:58.615 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:03:58.615 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:03:58.615 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:03:58.615 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:03:58.615 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:58.615 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.35469.0:132994 8.14 8:2cb501c6:::benchmark_data_smithi026_301269_object22237:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e281)", 2023-09-23T21:03:58.615 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:03:29.314226+0000", 2023-09-23T21:03:58.615 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 29.139918652999999, 2023-09-23T21:03:58.615 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.98319095000000001, 2023-09-23T21:03:58.615 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:03:58.615 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "reached pg", 2023-09-23T21:03:58.615 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:03:58.616 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.35469", 2023-09-23T21:03:58.616 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/4041424623", 2023-09-23T21:03:58.616 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 132994 2023-09-23T21:03:58.616 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:03:58.616 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:03:58.616 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:58.616 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:03:58.616 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:29.314226+0000", 2023-09-23T21:03:58.616 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:03:58.616 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:03:58.616 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:58.616 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:03:58.616 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:29.314226+0000", 2023-09-23T21:03:58.617 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:03:58.617 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:03:58.617 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:58.617 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:03:58.617 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:29.314226+0000", 2023-09-23T21:03:58.617 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999995 2023-09-23T21:03:58.617 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:03:58.617 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:58.617 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:03:58.617 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:29.314229+0000", 2023-09-23T21:03:58.617 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.185e-06 2023-09-23T21:03:58.617 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:03:58.618 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:58.618 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:03:58.618 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:29.314241+0000", 2023-09-23T21:03:58.618 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.1833e-05 2023-09-23T21:03:58.618 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:03:58.618 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:58.618 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:03:58.618 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:29.314245+0000", 2023-09-23T21:03:58.618 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.1400000000000002e-06 2023-09-23T21:03:58.618 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:03:58.618 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:58.619 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:03:58.619 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:29.556982+0000", 2023-09-23T21:03:58.619 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.24273707 2023-09-23T21:03:58.619 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:03:58.619 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:58.619 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:03:58.619 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:29.556988+0000", 2023-09-23T21:03:58.619 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.4979999999999997e-06 2023-09-23T21:03:58.619 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:03:58.619 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:58.619 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:03:58.619 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:30.297327+0000", 2023-09-23T21:03:58.619 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.74033974999999996 2023-09-23T21:03:58.620 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:03:58.620 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:58.620 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:03:58.620 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:30.297417+0000", 2023-09-23T21:03:58.620 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.9876999999999993e-05 2023-09-23T21:03:58.620 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:03:58.620 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:03:58.620 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:03:58.620 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:03:58.620 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:58.620 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.35469.0:133006 8.14 8:2b3eeac4:::benchmark_data_smithi026_301269_object22249:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e281)", 2023-09-23T21:03:58.620 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:03:29.314264+0000", 2023-09-23T21:03:58.621 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 29.139881365000001, 2023-09-23T21:03:58.621 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.98846920999999999, 2023-09-23T21:03:58.621 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:03:58.621 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:03:58.621 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:03:58.621 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.35469", 2023-09-23T21:03:58.621 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/4041424623", 2023-09-23T21:03:58.621 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 133006 2023-09-23T21:03:58.621 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:03:58.621 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:03:58.621 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:58.621 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:03:58.621 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:29.314264+0000", 2023-09-23T21:03:58.622 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:03:58.622 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:03:58.622 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:58.622 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:03:58.622 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:29.314264+0000", 2023-09-23T21:03:58.622 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:03:58.622 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:03:58.622 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:58.622 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:03:58.622 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:29.314263+0000", 2023-09-23T21:03:58.622 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999995 2023-09-23T21:03:58.622 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:03:58.623 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:58.623 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:03:58.623 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:29.314266+0000", 2023-09-23T21:03:58.623 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.2840000000000001e-06 2023-09-23T21:03:58.623 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:03:58.623 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:58.623 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:03:58.623 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:29.314270+0000", 2023-09-23T21:03:58.623 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.4719999999999997e-06 2023-09-23T21:03:58.623 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:03:58.623 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:58.623 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:03:58.624 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:29.314273+0000", 2023-09-23T21:03:58.624 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.0189999999999998e-06 2023-09-23T21:03:58.624 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:03:58.624 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:58.624 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:03:58.624 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:29.557029+0000", 2023-09-23T21:03:58.624 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.242756307 2023-09-23T21:03:58.624 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:03:58.624 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:58.624 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:03:58.624 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:29.557034+0000", 2023-09-23T21:03:58.624 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.3939999999999998e-06 2023-09-23T21:03:58.625 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:03:58.625 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:58.625 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:03:58.625 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:30.297740+0000", 2023-09-23T21:03:58.625 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.74070645300000004 2023-09-23T21:03:58.625 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:03:58.625 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:58.625 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:03:58.625 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:30.297857+0000", 2023-09-23T21:03:58.625 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00011629900000000001 2023-09-23T21:03:58.625 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:03:58.625 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:58.626 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 2,4", 2023-09-23T21:03:58.626 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:30.298139+0000", 2023-09-23T21:03:58.626 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00028202499999999998 2023-09-23T21:03:58.626 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:03:58.626 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:58.626 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:03:58.626 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:30.299627+0000", 2023-09-23T21:03:58.626 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.001488238 2023-09-23T21:03:58.626 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:03:58.626 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:58.626 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:03:58.626 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:30.301505+0000", 2023-09-23T21:03:58.627 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.001877868 2023-09-23T21:03:58.627 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:03:58.627 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:58.627 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:03:58.627 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:30.302646+0000", 2023-09-23T21:03:58.627 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0011411889999999999 2023-09-23T21:03:58.627 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:03:58.627 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:58.627 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:03:58.627 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:30.302694+0000", 2023-09-23T21:03:58.627 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.7948000000000002e-05 2023-09-23T21:03:58.628 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:03:58.628 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:03:58.628 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:03:58.628 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:30.302733+0000", 2023-09-23T21:03:58.628 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.8961999999999997e-05 2023-09-23T21:03:58.628 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:03:58.628 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:03:58.628 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:03:58.628 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:03:58.628 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:03:58.628 INFO:teuthology.orchestra.run.smithi070.stdout:} 2023-09-23T21:03:58.712 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:03:58.948 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:03:58.952 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:03:58.953 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:03:58.978 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_ops_in_flight 2023-09-23T21:03:59.054 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:03:59.104 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:03:59.156 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:03:59.161 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:03:59.161 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:03:59.262 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:03:59.324 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:03:59.440 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:03:59.443 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:03:59.444 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:03:59.544 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:03:59.588 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:03:59.653 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T21:03:59.657 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:03:59.666 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:03:59.667 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:03:59.767 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:03:59.804 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:03:59.832 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 21:03:59 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[164029]: ::ffff:172.21.15.70 - - [23/Sep/2023:21:03:59] "GET /metrics HTTP/1.1" 200 36302 "" "Prometheus/2.43.0" 2023-09-23T21:03:59.837 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:03:59 smithi070 ceph-mon[168282]: pgmap v781: 119 pgs: 1 active+remapped+backfill_wait, 2 active+remapped+backfilling, 116 active+clean; 4.3 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 854 op/s; 11508/210030 objects misplaced (5.479%); 11 MiB/s, 182 objects/s recovering 2023-09-23T21:03:59.837 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:03:59 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/2692186645' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T21:03:59.838 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:03:59 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T21:03:59.876 INFO:teuthology.orchestra.run.smithi026.stderr:reweighted osd.2 to 1 (10000) 2023-09-23T21:03:59.902 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:03:59.906 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:03:59.906 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:04:00.007 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:04:00.077 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:04:00.139 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:03:59 smithi026 ceph-mon[175151]: pgmap v781: 119 pgs: 1 active+remapped+backfill_wait, 2 active+remapped+backfilling, 116 active+clean; 4.3 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 854 op/s; 11508/210030 objects misplaced (5.479%); 11 MiB/s, 182 objects/s recovering 2023-09-23T21:04:00.140 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:03:59 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/2692186645' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T21:04:00.140 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:03:59 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T21:04:00.140 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:03:59 smithi026 ceph-mon[179688]: pgmap v781: 119 pgs: 1 active+remapped+backfill_wait, 2 active+remapped+backfilling, 116 active+clean; 4.3 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 854 op/s; 11508/210030 objects misplaced (5.479%); 11 MiB/s, 182 objects/s recovering 2023-09-23T21:04:00.140 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:03:59 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/2692186645' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T21:04:00.140 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:03:59 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T21:04:00.443 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:04:00.449 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:04:00.450 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:04:00.545 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph osd reweight 3 1 2023-09-23T21:04:00.551 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:04:00.595 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:04:00.649 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:04:00.649 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:04:00.650 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:04:00.750 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:04:00.860 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:04:00.942 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T21:04:00.953 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:04:00.953 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:04:00.969 INFO:teuthology.orchestra.run.smithi070.stdout:{ 2023-09-23T21:04:00.970 INFO:teuthology.orchestra.run.smithi070.stdout: "ops": [ 2023-09-23T21:04:00.970 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:04:00.970 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "pg_backfill(progress 8.11 e 297/297 lb 8:8f106621:::benchmark_data_smithi026_301269_object46240:head)", 2023-09-23T21:04:00.970 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:04:00.968630+0000", 2023-09-23T21:04:00.970 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 0.000722648, 2023-09-23T21:04:00.970 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00075113399999999998, 2023-09-23T21:04:00.970 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:04:00.970 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "queued for pg", 2023-09-23T21:04:00.970 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:04:00.970 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:04:00.971 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:04:00.971 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:00.968630+0000", 2023-09-23T21:04:00.971 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:04:00.971 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:04:00.971 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:04:00.971 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:04:00.971 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:00.968630+0000", 2023-09-23T21:04:00.971 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:04:00.971 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:04:00.971 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:04:00.971 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:04:00.971 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:00.968629+0000", 2023-09-23T21:04:00.971 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.999999 2023-09-23T21:04:00.972 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:04:00.972 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:04:00.972 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:04:00.972 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:00.968641+0000", 2023-09-23T21:04:00.972 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.2047e-05 2023-09-23T21:04:00.972 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:04:00.972 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:04:00.972 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:04:00.972 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:00.968662+0000", 2023-09-23T21:04:00.972 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.1251e-05 2023-09-23T21:04:00.972 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:04:00.972 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:04:00.973 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:04:00.973 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:00.968671+0000", 2023-09-23T21:04:00.973 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.6500000000000002e-06 2023-09-23T21:04:00.973 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:04:00.973 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:04:00.973 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:04:00.973 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:04:00.973 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:04:00.973 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_repop(client.35469.0:156007 8.12 e297/289 8:4d8475a4:::benchmark_data_smithi026_301269_object45250:head v 297'9121, mlcod=297'9118)", 2023-09-23T21:04:00.973 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:04:00.968690+0000", 2023-09-23T21:04:00.973 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 0.00066239399999999996, 2023-09-23T21:04:00.973 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00074646800000000002, 2023-09-23T21:04:00.974 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:04:00.974 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "started", 2023-09-23T21:04:00.974 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:04:00.974 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:04:00.974 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:04:00.974 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:00.968690+0000", 2023-09-23T21:04:00.974 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:04:00.974 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:04:00.974 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:04:00.974 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:04:00.974 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:00.968690+0000", 2023-09-23T21:04:00.974 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:04:00.975 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:04:00.975 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:04:00.975 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:04:00.975 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:00.968690+0000", 2023-09-23T21:04:00.975 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999995 2023-09-23T21:04:00.975 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:04:00.975 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:04:00.975 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:04:00.975 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:00.968696+0000", 2023-09-23T21:04:00.975 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.8799999999999996e-06 2023-09-23T21:04:00.975 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:04:00.975 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:04:00.976 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:04:00.976 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:00.968703+0000", 2023-09-23T21:04:00.976 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.0110000000000003e-06 2023-09-23T21:04:00.976 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:04:00.976 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:04:00.976 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:04:00.976 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:00.968707+0000", 2023-09-23T21:04:00.976 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.3390000000000003e-06 2023-09-23T21:04:00.976 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:04:00.976 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:04:00.976 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:04:00.976 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:00.968764+0000", 2023-09-23T21:04:00.977 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.6601000000000002e-05 2023-09-23T21:04:00.977 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:04:00.977 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:04:00.977 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:04:00.977 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:00.968789+0000", 2023-09-23T21:04:00.977 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.4972000000000001e-05 2023-09-23T21:04:00.977 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:04:00.977 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:04:00.977 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:04:00.977 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:04:00.977 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:04:00.977 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_repop(client.35469.0:156005 8.11 e297/287 8:887e0490:::benchmark_data_smithi026_301269_object45248:head v 297'9162, mlcod=297'9160)", 2023-09-23T21:04:00.978 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:04:00.968720+0000", 2023-09-23T21:04:00.978 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 0.00063225399999999995, 2023-09-23T21:04:00.978 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00077687100000000005, 2023-09-23T21:04:00.978 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:04:00.978 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "started", 2023-09-23T21:04:00.978 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:04:00.978 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:04:00.978 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:04:00.978 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:00.968720+0000", 2023-09-23T21:04:00.978 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:04:00.978 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:04:00.978 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:04:00.978 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:04:00.979 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:00.968720+0000", 2023-09-23T21:04:00.979 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:04:00.979 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:04:00.979 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:04:00.979 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:04:00.979 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:00.968720+0000", 2023-09-23T21:04:00.979 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999995 2023-09-23T21:04:00.979 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:04:00.979 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:04:00.979 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:04:00.979 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:00.968731+0000", 2023-09-23T21:04:00.979 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.0747999999999999e-05 2023-09-23T21:04:00.980 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:04:00.980 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:04:00.980 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:04:00.980 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:00.968737+0000", 2023-09-23T21:04:00.980 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.6250000000000001e-06 2023-09-23T21:04:00.980 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:04:00.980 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:04:00.980 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:04:00.980 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:00.968742+0000", 2023-09-23T21:04:00.980 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.3440000000000003e-06 2023-09-23T21:04:00.980 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:04:00.980 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:04:00.980 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:04:00.981 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:00.968905+0000", 2023-09-23T21:04:00.981 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000163311 2023-09-23T21:04:00.981 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:04:00.981 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:04:00.981 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:04:00.981 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:00.968919+0000", 2023-09-23T21:04:00.981 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.3825000000000001e-05 2023-09-23T21:04:00.981 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:04:00.981 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:04:00.981 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:04:00.981 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:04:00.981 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:04:00.982 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "MOSDPGPush(8.11 297/287 [PushOp(8:8f116bbb:::benchmark_data_smithi026_301269_object88231:head, version: 271'5615, data_included: [0~65536], data_size: 65536, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(8:8f116bbb:::benchmark_data_smithi026_301269_object88231:head@271'5615, size: 65536, copy_subset: [0~65536], clone_subset: {}, snapset: 0=[]:{}, object_exist: 0), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:65536, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false, error:false))])", 2023-09-23T21:04:00.982 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:04:00.968746+0000", 2023-09-23T21:04:00.982 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 0.00060621600000000002, 2023-09-23T21:04:00.982 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00083005500000000001, 2023-09-23T21:04:00.982 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:04:00.982 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "queued for pg", 2023-09-23T21:04:00.982 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:04:00.982 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:04:00.982 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:04:00.982 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:00.968746+0000", 2023-09-23T21:04:00.982 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:04:00.982 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:04:00.983 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:04:00.983 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:04:00.983 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:00.968746+0000", 2023-09-23T21:04:00.983 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:04:00.983 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:04:00.983 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:04:00.983 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:04:00.983 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:00.968746+0000", 2023-09-23T21:04:00.983 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999995 2023-09-23T21:04:00.983 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:04:00.983 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:04:00.983 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:04:00.983 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:00.968786+0000", 2023-09-23T21:04:00.984 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.0000999999999999e-05 2023-09-23T21:04:00.984 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:04:00.984 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:04:00.984 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:04:00.984 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:00.968800+0000", 2023-09-23T21:04:00.984 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.3558e-05 2023-09-23T21:04:00.984 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:04:00.984 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:04:00.984 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:04:00.984 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:00.968809+0000", 2023-09-23T21:04:00.984 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.4800000000000007e-06 2023-09-23T21:04:00.985 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:04:00.985 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:04:00.985 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:04:00.985 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:04:00.985 INFO:teuthology.orchestra.run.smithi070.stdout: ], 2023-09-23T21:04:00.985 INFO:teuthology.orchestra.run.smithi070.stdout: "num_ops": 4 2023-09-23T21:04:00.985 INFO:teuthology.orchestra.run.smithi070.stdout:} 2023-09-23T21:04:01.000 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:04:00 smithi070 ceph-mon[168282]: failed to encode map e297 with expected crc 2023-09-23T21:04:01.000 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:04:00 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-09-23T21:04:01.000 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:04:00 smithi070 ceph-mon[168282]: osdmap e297: 8 total, 8 up, 8 in 2023-09-23T21:04:01.000 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:04:00 smithi070 ceph-mon[168282]: failed to encode map e297 with expected crc 2023-09-23T21:04:01.000 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:04:00 smithi070 ceph-mon[168282]: failed to encode map e297 with expected crc 2023-09-23T21:04:01.000 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:04:00 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/1047681033' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:04:01.000 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:04:00 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:04:01.054 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:04:01.113 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:04:01.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:04:00 smithi026 ceph-mon[175151]: failed to encode map e297 with expected crc 2023-09-23T21:04:01.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:04:00 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-09-23T21:04:01.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:04:00 smithi026 ceph-mon[175151]: osdmap e297: 8 total, 8 up, 8 in 2023-09-23T21:04:01.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:04:00 smithi026 ceph-mon[175151]: failed to encode map e297 with expected crc 2023-09-23T21:04:01.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:04:00 smithi026 ceph-mon[175151]: failed to encode map e297 with expected crc 2023-09-23T21:04:01.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:04:00 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/1047681033' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:04:01.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:04:00 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:04:01.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:04:00 smithi026 ceph-mon[179688]: failed to encode map e297 with expected crc 2023-09-23T21:04:01.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:04:00 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-09-23T21:04:01.134 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:04:00 smithi026 ceph-mon[179688]: osdmap e297: 8 total, 8 up, 8 in 2023-09-23T21:04:01.134 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:04:00 smithi026 ceph-mon[179688]: failed to encode map e297 with expected crc 2023-09-23T21:04:01.134 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:04:00 smithi026 ceph-mon[179688]: failed to encode map e297 with expected crc 2023-09-23T21:04:01.134 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:04:00 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/1047681033' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:04:01.134 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:04:00 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:04:01.239 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0: {} 2023-09-23T21:04:01.252 INFO:teuthology.orchestra.run.smithi026.stderr:nodeep-scrub is set 2023-09-23T21:04:01.274 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1: {} 2023-09-23T21:04:01.320 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2: {} 2023-09-23T21:04:01.380 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3: {} 2023-09-23T21:04:01.389 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:04:01.392 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:04:01.392 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:04:01.433 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4: {} 2023-09-23T21:04:01.475 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_blocked_ops 2023-09-23T21:04:01.482 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T21:04:01.493 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:04:01.542 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6: {} 2023-09-23T21:04:01.572 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T21:04:01.575 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:04:01.608 INFO:teuthology.orchestra.run.smithi026.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T21:04:01.608 INFO:teuthology.orchestra.run.smithi026.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T21:04:01.608 INFO:teuthology.orchestra.run.smithi026.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T21:04:01.608 INFO:teuthology.orchestra.run.smithi026.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T21:04:01.608 INFO:teuthology.orchestra.run.smithi026.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T21:04:01.608 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T21:04:01.608 INFO:teuthology.orchestra.run.smithi026.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T21:04:01.608 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T21:04:01.854 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:04:01 smithi026 ceph-mon[179688]: pgmap v783: 119 pgs: 1 active+remapped+backfill_wait, 2 active+remapped+backfilling, 116 active+clean; 4.1 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 956 op/s; 10564/203367 objects misplaced (5.195%); 11 MiB/s, 182 objects/s recovering 2023-09-23T21:04:01.854 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:04:01 smithi026 ceph-mon[179688]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:04:01.854 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:04:01 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/2360087376' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:04:01.854 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:04:01 smithi026 ceph-mon[179688]: failed to encode map e298 with expected crc 2023-09-23T21:04:01.854 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:04:01 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T21:04:01.854 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:04:01 smithi026 ceph-mon[175151]: pgmap v783: 119 pgs: 1 active+remapped+backfill_wait, 2 active+remapped+backfilling, 116 active+clean; 4.1 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 956 op/s; 10564/203367 objects misplaced (5.195%); 11 MiB/s, 182 objects/s recovering 2023-09-23T21:04:01.854 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:04:01 smithi026 ceph-mon[175151]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:04:01.855 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:04:01 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/2360087376' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:04:01.855 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:04:01 smithi026 ceph-mon[175151]: failed to encode map e298 with expected crc 2023-09-23T21:04:01.855 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:04:01 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T21:04:01.855 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:04:01 smithi026 ceph-mon[175151]: osdmap e298: 8 total, 8 up, 8 in 2023-09-23T21:04:01.855 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:04:01 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:04:01.855 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:04:01 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:04:01.855 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:04:01 smithi070 ceph-mon[168282]: pgmap v783: 119 pgs: 1 active+remapped+backfill_wait, 2 active+remapped+backfilling, 116 active+clean; 4.1 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 956 op/s; 10564/203367 objects misplaced (5.195%); 11 MiB/s, 182 objects/s recovering 2023-09-23T21:04:01.855 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:04:01 smithi070 ceph-mon[168282]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:04:01.855 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:04:01 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/2360087376' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:04:01.856 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:04:01 smithi070 ceph-mon[168282]: failed to encode map e298 with expected crc 2023-09-23T21:04:01.856 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:04:01 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T21:04:01.856 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:04:01 smithi070 ceph-mon[168282]: osdmap e298: 8 total, 8 up, 8 in 2023-09-23T21:04:01.856 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:04:01 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:04:01.856 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:04:01 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:04:02.134 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:04:01 smithi026 ceph-mon[179688]: osdmap e298: 8 total, 8 up, 8 in 2023-09-23T21:04:02.134 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:04:01 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:04:02.134 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:04:01 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:04:02.230 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T21:04:02.534 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:04:02.546 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:04:02.547 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:04:02.647 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:04:02.683 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:04:02.696 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:04:03.016 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:04:02 smithi026 ceph-mon[179688]: failed to encode map e298 with expected crc 2023-09-23T21:04:03.016 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:04:02 smithi026 ceph-mon[179688]: failed to encode map e298 with expected crc 2023-09-23T21:04:03.016 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:04:02 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pgp_num_actual", "val": "21"}]: dispatch 2023-09-23T21:04:03.017 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:04:02 smithi026 ceph-mon[175151]: failed to encode map e298 with expected crc 2023-09-23T21:04:03.017 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:04:02 smithi026 ceph-mon[175151]: failed to encode map e298 with expected crc 2023-09-23T21:04:03.017 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:04:02 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pgp_num_actual", "val": "21"}]: dispatch 2023-09-23T21:04:03.067 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:04:03.072 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:04:03.072 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:04:03.102 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:04:02 smithi070 ceph-mon[168282]: failed to encode map e298 with expected crc 2023-09-23T21:04:03.102 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:04:02 smithi070 ceph-mon[168282]: failed to encode map e298 with expected crc 2023-09-23T21:04:03.102 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:04:02 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pgp_num_actual", "val": "21"}]: dispatch 2023-09-23T21:04:03.173 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:04:03.230 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:04:03.480 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:04:03.484 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:04:03.485 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:04:03.586 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:04:03.614 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:04:04.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:04:03 smithi026 ceph-mon[179688]: pgmap v785: 119 pgs: 1 active+remapped+backfill_wait, 2 active+remapped+backfilling, 116 active+clean; 4.0 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 922 op/s; 8536/196458 objects misplaced (4.345%); 12 MiB/s, 188 objects/s recovering 2023-09-23T21:04:04.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:04:03 smithi026 ceph-mon[179688]: failed to encode map e299 with expected crc 2023-09-23T21:04:04.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:04:03 smithi026 ceph-mon[179688]: failed to encode map e299 with expected crc 2023-09-23T21:04:04.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:04:03 smithi026 ceph-mon[179688]: failed to encode map e299 with expected crc 2023-09-23T21:04:04.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:04:03 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pgp_num_actual", "val": "21"}]': finished 2023-09-23T21:04:04.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:04:03 smithi026 ceph-mon[179688]: osdmap e299: 8 total, 8 up, 8 in 2023-09-23T21:04:04.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:04:03 smithi026 ceph-mon[175151]: pgmap v785: 119 pgs: 1 active+remapped+backfill_wait, 2 active+remapped+backfilling, 116 active+clean; 4.0 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 922 op/s; 8536/196458 objects misplaced (4.345%); 12 MiB/s, 188 objects/s recovering 2023-09-23T21:04:04.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:04:03 smithi026 ceph-mon[175151]: failed to encode map e299 with expected crc 2023-09-23T21:04:04.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:04:03 smithi026 ceph-mon[175151]: failed to encode map e299 with expected crc 2023-09-23T21:04:04.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:04:03 smithi026 ceph-mon[175151]: failed to encode map e299 with expected crc 2023-09-23T21:04:04.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:04:03 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pgp_num_actual", "val": "21"}]': finished 2023-09-23T21:04:04.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:04:03 smithi026 ceph-mon[175151]: osdmap e299: 8 total, 8 up, 8 in 2023-09-23T21:04:04.186 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:04:04.187 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:04:03 smithi070 ceph-mon[168282]: pgmap v785: 119 pgs: 1 active+remapped+backfill_wait, 2 active+remapped+backfilling, 116 active+clean; 4.0 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 922 op/s; 8536/196458 objects misplaced (4.345%); 12 MiB/s, 188 objects/s recovering 2023-09-23T21:04:04.187 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:04:03 smithi070 ceph-mon[168282]: failed to encode map e299 with expected crc 2023-09-23T21:04:04.188 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:04:03 smithi070 ceph-mon[168282]: failed to encode map e299 with expected crc 2023-09-23T21:04:04.188 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:04:03 smithi070 ceph-mon[168282]: failed to encode map e299 with expected crc 2023-09-23T21:04:04.188 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:04:03 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pgp_num_actual", "val": "21"}]': finished 2023-09-23T21:04:04.188 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:04:03 smithi070 ceph-mon[168282]: osdmap e299: 8 total, 8 up, 8 in 2023-09-23T21:04:04.188 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:04:04.189 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:04:04.290 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:04:04.326 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:04:04.367 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T21:04:04.372 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:04:04.372 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:04:04.473 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:04:04.501 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:04:04.714 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T21:04:04.762 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T21:04:04.895 INFO:teuthology.orchestra.run.smithi026.stderr:reweighted osd.3 to 1 (10000) 2023-09-23T21:04:04.939 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:04:04.941 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:04:04.942 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:04:05.043 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:04:05.054 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:04:04 smithi026 ceph-mon[175151]: failed to encode map e300 with expected crc 2023-09-23T21:04:05.055 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:04:04 smithi026 ceph-mon[175151]: failed to encode map e300 with expected crc 2023-09-23T21:04:05.055 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:04:04 smithi026 ceph-mon[175151]: osdmap e300: 8 total, 8 up, 8 in 2023-09-23T21:04:05.055 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:04:04 smithi026 ceph-mon[175151]: failed to encode map e300 with expected crc 2023-09-23T21:04:05.055 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:04:04 smithi026 ceph-mon[175151]: failed to encode map e300 with expected crc 2023-09-23T21:04:05.055 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:04:04 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/3114252639' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-09-23T21:04:05.056 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:04:04 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pgp_num_actual", "val": "22"}]: dispatch 2023-09-23T21:04:05.056 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:04:04 smithi026 ceph-mon[179688]: failed to encode map e300 with expected crc 2023-09-23T21:04:05.056 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:04:04 smithi026 ceph-mon[179688]: failed to encode map e300 with expected crc 2023-09-23T21:04:05.056 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:04:04 smithi026 ceph-mon[179688]: osdmap e300: 8 total, 8 up, 8 in 2023-09-23T21:04:05.057 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:04:04 smithi026 ceph-mon[179688]: failed to encode map e300 with expected crc 2023-09-23T21:04:05.057 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:04:04 smithi026 ceph-mon[179688]: failed to encode map e300 with expected crc 2023-09-23T21:04:05.057 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:04:04 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/3114252639' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-09-23T21:04:05.057 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:04:04 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pgp_num_actual", "val": "22"}]: dispatch 2023-09-23T21:04:05.095 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:04:05.154 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:04:05.162 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:04:05.163 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:04:05.163 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:04:04 smithi070 ceph-mon[168282]: failed to encode map e300 with expected crc 2023-09-23T21:04:05.163 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:04:04 smithi070 ceph-mon[168282]: failed to encode map e300 with expected crc 2023-09-23T21:04:05.164 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:04:04 smithi070 ceph-mon[168282]: osdmap e300: 8 total, 8 up, 8 in 2023-09-23T21:04:05.164 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:04:04 smithi070 ceph-mon[168282]: failed to encode map e300 with expected crc 2023-09-23T21:04:05.164 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:04:04 smithi070 ceph-mon[168282]: failed to encode map e300 with expected crc 2023-09-23T21:04:05.164 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:04:04 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/3114252639' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-09-23T21:04:05.164 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:04:04 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pgp_num_actual", "val": "22"}]: dispatch 2023-09-23T21:04:05.264 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:04:05.267 INFO:teuthology.orchestra.run.smithi070.stdout:ERROR: (22) Invalid argument 2023-09-23T21:04:05.267 INFO:teuthology.orchestra.run.smithi070.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T21:04:05.310 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:04:05.350 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:04:05.352 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:04:05.353 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:04:05.365 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph osd reweight 6 1 2023-09-23T21:04:05.454 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:04:05.522 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:04:05.588 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:04:05.592 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:04:05.592 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:04:05.664 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:04:05.693 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:04:05.714 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:04:05.740 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:04:05.750 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_historic_ops 2023-09-23T21:04:05.811 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:04:05.815 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:04:05.815 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:04:05.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:04:05 smithi026 ceph-mon[175151]: pgmap v788: 119 pgs: 1 active+clean+remapped, 1 active+remapped+backfill_wait, 1 active+remapped+backfilling, 116 active+clean; 3.9 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 858 op/s; 6684/192837 objects misplaced (3.466%); 18 MiB/s, 285 objects/s recovering 2023-09-23T21:04:05.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:04:05 smithi026 ceph-mon[175151]: failed to encode map e301 with expected crc 2023-09-23T21:04:05.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:04:05 smithi026 ceph-mon[175151]: failed to encode map e301 with expected crc 2023-09-23T21:04:05.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:04:05 smithi026 ceph-mon[175151]: failed to encode map e301 with expected crc 2023-09-23T21:04:05.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:04:05 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/3114252639' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-09-23T21:04:05.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:04:05 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pgp_num_actual", "val": "22"}]': finished 2023-09-23T21:04:05.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:04:05 smithi026 ceph-mon[175151]: osdmap e301: 8 total, 8 up, 8 in 2023-09-23T21:04:05.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:04:05 smithi026 ceph-mon[175151]: 8.14 starting backfill to osd.0 from (0'0,0'0] MAX to 298'9088 2023-09-23T21:04:05.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:04:05 smithi026 ceph-mon[175151]: failed to encode map e302 with expected crc 2023-09-23T21:04:05.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:04:05 smithi026 ceph-mon[175151]: failed to encode map e302 with expected crc 2023-09-23T21:04:05.883 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:04:05 smithi026 ceph-mon[175151]: osdmap e302: 8 total, 8 up, 8 in 2023-09-23T21:04:05.916 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:04:05.946 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:04:05.989 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:04:05.993 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:04:05.994 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:04:06.095 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:04:06.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:04:05 smithi070 ceph-mon[168282]: pgmap v788: 119 pgs: 1 active+clean+remapped, 1 active+remapped+backfill_wait, 1 active+remapped+backfilling, 116 active+clean; 3.9 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 858 op/s; 6684/192837 objects misplaced (3.466%); 18 MiB/s, 285 objects/s recovering 2023-09-23T21:04:06.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:04:05 smithi070 ceph-mon[168282]: failed to encode map e301 with expected crc 2023-09-23T21:04:06.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:04:05 smithi070 ceph-mon[168282]: failed to encode map e301 with expected crc 2023-09-23T21:04:06.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:04:05 smithi070 ceph-mon[168282]: failed to encode map e301 with expected crc 2023-09-23T21:04:06.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:04:05 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/3114252639' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-09-23T21:04:06.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:04:05 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pgp_num_actual", "val": "22"}]': finished 2023-09-23T21:04:06.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:04:05 smithi070 ceph-mon[168282]: osdmap e301: 8 total, 8 up, 8 in 2023-09-23T21:04:06.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:04:05 smithi070 ceph-mon[168282]: 8.14 starting backfill to osd.0 from (0'0,0'0] MAX to 298'9088 2023-09-23T21:04:06.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:04:05 smithi070 ceph-mon[168282]: failed to encode map e302 with expected crc 2023-09-23T21:04:06.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:04:05 smithi070 ceph-mon[168282]: failed to encode map e302 with expected crc 2023-09-23T21:04:06.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:04:05 smithi070 ceph-mon[168282]: osdmap e302: 8 total, 8 up, 8 in 2023-09-23T21:04:06.131 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:04:06.131 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:04:05 smithi026 ceph-mon[179688]: pgmap v788: 119 pgs: 1 active+clean+remapped, 1 active+remapped+backfill_wait, 1 active+remapped+backfilling, 116 active+clean; 3.9 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 858 op/s; 6684/192837 objects misplaced (3.466%); 18 MiB/s, 285 objects/s recovering 2023-09-23T21:04:06.131 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:04:05 smithi026 ceph-mon[179688]: failed to encode map e301 with expected crc 2023-09-23T21:04:06.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:04:05 smithi026 ceph-mon[179688]: failed to encode map e301 with expected crc 2023-09-23T21:04:06.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:04:05 smithi026 ceph-mon[179688]: failed to encode map e301 with expected crc 2023-09-23T21:04:06.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:04:05 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/3114252639' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-09-23T21:04:06.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:04:05 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pgp_num_actual", "val": "22"}]': finished 2023-09-23T21:04:06.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:04:05 smithi026 ceph-mon[179688]: osdmap e301: 8 total, 8 up, 8 in 2023-09-23T21:04:06.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:04:05 smithi026 ceph-mon[179688]: 8.14 starting backfill to osd.0 from (0'0,0'0] MAX to 298'9088 2023-09-23T21:04:06.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:04:05 smithi026 ceph-mon[179688]: failed to encode map e302 with expected crc 2023-09-23T21:04:06.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:04:05 smithi026 ceph-mon[179688]: failed to encode map e302 with expected crc 2023-09-23T21:04:06.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:04:05 smithi026 ceph-mon[179688]: osdmap e302: 8 total, 8 up, 8 in 2023-09-23T21:04:06.180 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:04:06.184 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:04:06.184 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:04:06.285 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:04:06.344 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:04:06.402 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:04:06.409 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:04:06.410 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:04:06.410 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T21:04:06.511 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:04:06.563 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:04:06.672 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:04:06.677 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:04:06.678 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:04:06.778 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:04:06.836 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:04:06.894 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:04:06.907 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:04:06.908 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:04:06 smithi070 ceph-mon[168282]: failed to encode map e302 with expected crc 2023-09-23T21:04:06.908 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:04:06 smithi070 ceph-mon[168282]: failed to encode map e303 with expected crc 2023-09-23T21:04:06.908 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:04:06 smithi070 ceph-mon[168282]: failed to encode map e303 with expected crc 2023-09-23T21:04:06.908 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:04:06 smithi070 ceph-mon[168282]: failed to encode map e303 with expected crc 2023-09-23T21:04:06.908 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:04:06 smithi070 ceph-mon[168282]: osdmap e303: 8 total, 8 up, 8 in 2023-09-23T21:04:06.909 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:04:07.009 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:04:07.087 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:04:07.130 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:04:07.139 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:04:06 smithi026 ceph-mon[179688]: failed to encode map e302 with expected crc 2023-09-23T21:04:07.139 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:04:06 smithi026 ceph-mon[179688]: failed to encode map e303 with expected crc 2023-09-23T21:04:07.139 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:04:06 smithi026 ceph-mon[179688]: failed to encode map e303 with expected crc 2023-09-23T21:04:07.139 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:04:06 smithi026 ceph-mon[179688]: failed to encode map e303 with expected crc 2023-09-23T21:04:07.139 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:04:06 smithi026 ceph-mon[179688]: osdmap e303: 8 total, 8 up, 8 in 2023-09-23T21:04:07.140 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:04:06 smithi026 ceph-mon[175151]: failed to encode map e302 with expected crc 2023-09-23T21:04:07.140 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:04:06 smithi026 ceph-mon[175151]: failed to encode map e303 with expected crc 2023-09-23T21:04:07.140 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:04:06 smithi026 ceph-mon[175151]: failed to encode map e303 with expected crc 2023-09-23T21:04:07.140 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:04:06 smithi026 ceph-mon[175151]: failed to encode map e303 with expected crc 2023-09-23T21:04:07.140 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:04:06 smithi026 ceph-mon[175151]: osdmap e303: 8 total, 8 up, 8 in 2023-09-23T21:04:07.203 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:04:07.203 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:04:07.204 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:04:07.304 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:04:07.374 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:04:07.495 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:04:07.500 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:04:07.500 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:04:07.601 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:04:07.651 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:04:07.698 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:04:07.703 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:04:07.704 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:04:07.804 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:04:07.863 INFO:teuthology.orchestra.run.smithi070.stdout:ERROR: (22) Invalid argument 2023-09-23T21:04:07.863 INFO:teuthology.orchestra.run.smithi070.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T21:04:07.898 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:04:08.199 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:04:07 smithi070 ceph-mon[168282]: failed to encode map e303 with expected crc 2023-09-23T21:04:08.199 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:04:07 smithi070 ceph-mon[168282]: failed to encode map e303 with expected crc 2023-09-23T21:04:08.199 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:04:07 smithi070 ceph-mon[168282]: failed to encode map e303 with expected crc 2023-09-23T21:04:08.199 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:04:07 smithi070 ceph-mon[168282]: 8.15 starting backfill to osd.1 from (0'0,0'0] MAX to 299'9198 2023-09-23T21:04:08.199 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:04:07 smithi070 ceph-mon[168282]: pgmap v792: 119 pgs: 1 peering, 1 active+remapped+backfill_wait, 1 active+remapped+backfilling, 116 active+clean; 3.9 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 433 op/s; 6684/191388 objects misplaced (3.492%); 15 MiB/s, 234 objects/s recovering 2023-09-23T21:04:08.242 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_ops_in_flight 2023-09-23T21:04:08.325 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0: {} 2023-09-23T21:04:08.364 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1: {} 2023-09-23T21:04:08.383 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:04:07 smithi026 ceph-mon[175151]: failed to encode map e303 with expected crc 2023-09-23T21:04:08.383 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:04:07 smithi026 ceph-mon[175151]: failed to encode map e303 with expected crc 2023-09-23T21:04:08.383 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:04:07 smithi026 ceph-mon[175151]: failed to encode map e303 with expected crc 2023-09-23T21:04:08.384 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:04:07 smithi026 ceph-mon[175151]: 8.15 starting backfill to osd.1 from (0'0,0'0] MAX to 299'9198 2023-09-23T21:04:08.384 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:04:07 smithi026 ceph-mon[175151]: pgmap v792: 119 pgs: 1 peering, 1 active+remapped+backfill_wait, 1 active+remapped+backfilling, 116 active+clean; 3.9 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 433 op/s; 6684/191388 objects misplaced (3.492%); 15 MiB/s, 234 objects/s recovering 2023-09-23T21:04:08.385 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:04:07 smithi026 ceph-mon[179688]: failed to encode map e303 with expected crc 2023-09-23T21:04:08.385 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:04:07 smithi026 ceph-mon[179688]: failed to encode map e303 with expected crc 2023-09-23T21:04:08.385 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:04:07 smithi026 ceph-mon[179688]: failed to encode map e303 with expected crc 2023-09-23T21:04:08.386 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:04:07 smithi026 ceph-mon[179688]: 8.15 starting backfill to osd.1 from (0'0,0'0] MAX to 299'9198 2023-09-23T21:04:08.386 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:04:07 smithi026 ceph-mon[179688]: pgmap v792: 119 pgs: 1 peering, 1 active+remapped+backfill_wait, 1 active+remapped+backfilling, 116 active+clean; 3.9 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 433 op/s; 6684/191388 objects misplaced (3.492%); 15 MiB/s, 234 objects/s recovering 2023-09-23T21:04:08.402 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2: {} 2023-09-23T21:04:08.450 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3: {} 2023-09-23T21:04:08.492 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4: {} 2023-09-23T21:04:08.526 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T21:04:08.569 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6: {} 2023-09-23T21:04:08.605 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T21:04:08.607 INFO:teuthology.orchestra.run.smithi026.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T21:04:08.607 INFO:teuthology.orchestra.run.smithi026.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T21:04:08.607 INFO:teuthology.orchestra.run.smithi026.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T21:04:08.607 INFO:teuthology.orchestra.run.smithi026.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T21:04:08.607 INFO:teuthology.orchestra.run.smithi026.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T21:04:08.607 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T21:04:08.607 INFO:teuthology.orchestra.run.smithi026.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T21:04:08.607 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T21:04:08.904 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:04:08 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/836133464' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T21:04:08.904 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:04:08 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T21:04:08.904 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:04:08 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/60696987' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:04:08.904 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:04:08 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/836133464' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T21:04:08.904 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:04:08 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T21:04:08.904 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:04:08 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/60696987' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:04:08.908 INFO:teuthology.orchestra.run.smithi026.stderr:noscrub is unset 2023-09-23T21:04:08.909 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T21:04:09.063 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:04:09.069 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:04:09.069 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:04:09.164 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:04:08 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/836133464' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T21:04:09.164 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:04:08 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T21:04:09.165 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:04:08 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/60696987' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:04:09.170 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:04:09.238 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:04:09.719 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T21:04:09.725 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:04:09.725 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:04:09.826 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:04:09.884 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 21:04:09 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[164029]: ::ffff:172.21.15.70 - - [23/Sep/2023:21:04:09] "GET /metrics HTTP/1.1" 200 36319 "" "Prometheus/2.43.0" 2023-09-23T21:04:09.902 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:04:09.967 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:04:09.976 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:04:09.977 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:04:10.016 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:04:09 smithi070 ceph-mon[168282]: pgmap v793: 119 pgs: 1 active+remapped+backfill_wait, 2 active+remapped+backfilling, 1 peering, 115 active+clean; 3.8 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 599 op/s; 6870/187650 objects misplaced (3.661%); 32 MiB/s, 506 objects/s recovering 2023-09-23T21:04:10.017 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:04:09 smithi070 ceph-mon[168282]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:04:10.017 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:04:09 smithi070 ceph-mon[168282]: failed to encode map e304 with expected crc 2023-09-23T21:04:10.017 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:04:09 smithi070 ceph-mon[168282]: failed to encode map e304 with expected crc 2023-09-23T21:04:10.017 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:04:09 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T21:04:10.017 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:04:09 smithi070 ceph-mon[168282]: osdmap e304: 8 total, 8 up, 8 in 2023-09-23T21:04:10.017 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:04:09 smithi070 ceph-mon[168282]: failed to encode map e304 with expected crc 2023-09-23T21:04:10.077 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:04:10.134 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:04:10.213 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:04:09 smithi026 ceph-mon[179688]: pgmap v793: 119 pgs: 1 active+remapped+backfill_wait, 2 active+remapped+backfilling, 1 peering, 115 active+clean; 3.8 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 599 op/s; 6870/187650 objects misplaced (3.661%); 32 MiB/s, 506 objects/s recovering 2023-09-23T21:04:10.213 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:04:09 smithi026 ceph-mon[179688]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:04:10.213 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:04:09 smithi026 ceph-mon[179688]: failed to encode map e304 with expected crc 2023-09-23T21:04:10.213 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:04:09 smithi026 ceph-mon[179688]: failed to encode map e304 with expected crc 2023-09-23T21:04:10.214 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:04:09 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T21:04:10.214 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:04:09 smithi026 ceph-mon[179688]: osdmap e304: 8 total, 8 up, 8 in 2023-09-23T21:04:10.214 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:04:09 smithi026 ceph-mon[179688]: failed to encode map e304 with expected crc 2023-09-23T21:04:10.214 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:04:09 smithi026 ceph-mon[175151]: pgmap v793: 119 pgs: 1 active+remapped+backfill_wait, 2 active+remapped+backfilling, 1 peering, 115 active+clean; 3.8 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 599 op/s; 6870/187650 objects misplaced (3.661%); 32 MiB/s, 506 objects/s recovering 2023-09-23T21:04:10.214 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:04:09 smithi026 ceph-mon[175151]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:04:10.214 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:04:09 smithi026 ceph-mon[175151]: failed to encode map e304 with expected crc 2023-09-23T21:04:10.214 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:04:09 smithi026 ceph-mon[175151]: failed to encode map e304 with expected crc 2023-09-23T21:04:10.214 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:04:09 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T21:04:10.214 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:04:09 smithi026 ceph-mon[175151]: osdmap e304: 8 total, 8 up, 8 in 2023-09-23T21:04:10.215 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:04:09 smithi026 ceph-mon[175151]: failed to encode map e304 with expected crc 2023-09-23T21:04:10.220 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:04:10.234 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:04:10.234 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:04:10.285 INFO:teuthology.orchestra.run.smithi070.stdout:{ 2023-09-23T21:04:10.285 INFO:teuthology.orchestra.run.smithi070.stdout: "ops": [ 2023-09-23T21:04:10.285 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:04:10.285 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.35469.0:162500 8.9 8:903f4119:::benchmark_data_smithi026_301269_object51743:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e304)", 2023-09-23T21:04:10.285 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:04:10.267908+0000", 2023-09-23T21:04:10.285 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 0.016051757, 2023-09-23T21:04:10.285 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.016084114, 2023-09-23T21:04:10.285 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:04:10.285 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "waiting for sub ops", 2023-09-23T21:04:10.285 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:04:10.286 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.35469", 2023-09-23T21:04:10.286 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/4041424623", 2023-09-23T21:04:10.286 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 162500 2023-09-23T21:04:10.286 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:04:10.286 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:04:10.286 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:04:10.286 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:04:10.286 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:10.267908+0000", 2023-09-23T21:04:10.286 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:04:10.286 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:04:10.286 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:04:10.286 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:04:10.287 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:10.267908+0000", 2023-09-23T21:04:10.287 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:04:10.287 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:04:10.287 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:04:10.287 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:04:10.287 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:10.267906+0000", 2023-09-23T21:04:10.287 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.999999 2023-09-23T21:04:10.287 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:04:10.287 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:04:10.287 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:04:10.287 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:10.267913+0000", 2023-09-23T21:04:10.287 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.5250000000000002e-06 2023-09-23T21:04:10.288 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:04:10.288 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:04:10.288 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:04:10.288 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:10.267925+0000", 2023-09-23T21:04:10.288 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.2366e-05 2023-09-23T21:04:10.288 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:04:10.288 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:04:10.288 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:04:10.288 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:10.267930+0000", 2023-09-23T21:04:10.288 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.0969999999999996e-06 2023-09-23T21:04:10.288 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:04:10.288 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:04:10.288 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:04:10.289 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:10.267979+0000", 2023-09-23T21:04:10.289 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.8244000000000001e-05 2023-09-23T21:04:10.289 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:04:10.289 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:04:10.289 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:04:10.289 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:10.268119+0000", 2023-09-23T21:04:10.289 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00014028599999999999 2023-09-23T21:04:10.289 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:04:10.289 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:04:10.289 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 0,5", 2023-09-23T21:04:10.289 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:10.268513+0000", 2023-09-23T21:04:10.289 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00039373099999999997 2023-09-23T21:04:10.289 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:04:10.290 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:04:10.290 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:04:10.290 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:10.270511+0000", 2023-09-23T21:04:10.290 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0019978259999999999 2023-09-23T21:04:10.290 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:04:10.290 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:04:10.290 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:04:10.290 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:10.270873+0000", 2023-09-23T21:04:10.290 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00036266599999999999 2023-09-23T21:04:10.290 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:04:10.290 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:04:10.290 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:04:10.291 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:04:10.291 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:04:10.291 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_repop(client.35469.0:162512 8.3 e304/281 8:c1fad129:::benchmark_data_smithi026_301269_object51755:head v 304'8928, mlcod=304'8926)", 2023-09-23T21:04:10.291 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:04:10.282194+0000", 2023-09-23T21:04:10.291 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 0.001765815, 2023-09-23T21:04:10.291 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00189269, 2023-09-23T21:04:10.291 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:04:10.291 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "started", 2023-09-23T21:04:10.291 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:04:10.291 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:04:10.291 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:04:10.291 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:10.282194+0000", 2023-09-23T21:04:10.291 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:04:10.292 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:04:10.292 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:04:10.292 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:04:10.292 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:10.282194+0000", 2023-09-23T21:04:10.292 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:04:10.292 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:04:10.292 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:04:10.292 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:04:10.292 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:10.282193+0000", 2023-09-23T21:04:10.292 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.999999 2023-09-23T21:04:10.292 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:04:10.292 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:04:10.292 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:04:10.293 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:10.282201+0000", 2023-09-23T21:04:10.293 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.5520000000000005e-06 2023-09-23T21:04:10.293 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:04:10.293 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:04:10.293 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:04:10.293 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:10.282216+0000", 2023-09-23T21:04:10.293 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.4178999999999999e-05 2023-09-23T21:04:10.293 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:04:10.293 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:04:10.293 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:04:10.293 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:10.282222+0000", 2023-09-23T21:04:10.293 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.2079999999999997e-06 2023-09-23T21:04:10.293 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:04:10.294 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:04:10.294 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:04:10.294 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:10.282276+0000", 2023-09-23T21:04:10.294 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.4174e-05 2023-09-23T21:04:10.294 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:04:10.294 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:04:10.294 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:04:10.294 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:10.282296+0000", 2023-09-23T21:04:10.294 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.9717000000000001e-05 2023-09-23T21:04:10.294 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:04:10.294 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:04:10.295 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:04:10.295 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:04:10.295 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:04:10.295 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_repop(client.35469.0:162513 8.17 e304/281 8:eeb0650b:::benchmark_data_smithi026_301269_object51756:head v 304'9112, mlcod=304'9110)", 2023-09-23T21:04:10.295 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:04:10.282237+0000", 2023-09-23T21:04:10.295 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 0.0017219920000000001, 2023-09-23T21:04:10.295 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0019105260000000001, 2023-09-23T21:04:10.295 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:04:10.295 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "started", 2023-09-23T21:04:10.295 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:04:10.295 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:04:10.295 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:04:10.295 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:10.282237+0000", 2023-09-23T21:04:10.295 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:04:10.296 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:04:10.296 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:04:10.296 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:04:10.296 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:10.282237+0000", 2023-09-23T21:04:10.296 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:04:10.296 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:04:10.296 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:04:10.296 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:04:10.296 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:10.282237+0000", 2023-09-23T21:04:10.296 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999995 2023-09-23T21:04:10.296 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:04:10.296 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:04:10.296 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:04:10.297 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:10.282241+0000", 2023-09-23T21:04:10.297 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.3420000000000001e-06 2023-09-23T21:04:10.297 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:04:10.297 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:04:10.297 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:04:10.297 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:10.282248+0000", 2023-09-23T21:04:10.297 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.4439999999999996e-06 2023-09-23T21:04:10.297 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:04:10.297 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:04:10.297 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:04:10.297 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:10.282252+0000", 2023-09-23T21:04:10.297 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.7390000000000001e-06 2023-09-23T21:04:10.297 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:04:10.298 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:04:10.298 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:04:10.298 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:10.282590+0000", 2023-09-23T21:04:10.298 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00033879899999999998 2023-09-23T21:04:10.298 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:04:10.298 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:04:10.298 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:04:10.298 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:10.282616+0000", 2023-09-23T21:04:10.298 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.5712999999999999e-05 2023-09-23T21:04:10.298 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:04:10.298 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:04:10.298 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:04:10.298 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:04:10.299 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:04:10.299 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_repop(client.35469.0:162514 8.7 e304/281)", 2023-09-23T21:04:10.299 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:04:10.282263+0000", 2023-09-23T21:04:10.299 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 0.0016964040000000001, 2023-09-23T21:04:10.299 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0019402449999999999, 2023-09-23T21:04:10.299 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:04:10.299 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "queued for pg", 2023-09-23T21:04:10.299 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:04:10.299 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:04:10.299 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:04:10.299 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:10.282263+0000", 2023-09-23T21:04:10.299 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:04:10.299 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:04:10.300 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:04:10.300 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:04:10.300 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:10.282263+0000", 2023-09-23T21:04:10.300 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:04:10.300 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:04:10.300 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:04:10.300 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:04:10.300 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:10.282263+0000", 2023-09-23T21:04:10.300 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967296 2023-09-23T21:04:10.300 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:04:10.300 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:04:10.300 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:04:10.300 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:10.282266+0000", 2023-09-23T21:04:10.301 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.641e-06 2023-09-23T21:04:10.301 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:04:10.301 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:04:10.301 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:04:10.301 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:10.282272+0000", 2023-09-23T21:04:10.301 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.3009999999999997e-06 2023-09-23T21:04:10.301 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:04:10.301 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:04:10.301 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:04:10.301 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:10.282275+0000", 2023-09-23T21:04:10.301 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.2160000000000002e-06 2023-09-23T21:04:10.302 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:04:10.302 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:04:10.302 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:04:10.302 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:04:10.302 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:04:10.302 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_repop(client.35469.0:162515 8.15 e304/302 8:acdf8968:::benchmark_data_smithi026_301269_object51758:head v 304'9337, mlcod=304'9335)", 2023-09-23T21:04:10.302 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:04:10.282514+0000", 2023-09-23T21:04:10.302 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 0.0014450769999999999, 2023-09-23T21:04:10.302 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.001751519, 2023-09-23T21:04:10.302 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:04:10.302 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "started", 2023-09-23T21:04:10.302 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:04:10.303 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:04:10.303 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:04:10.303 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:10.282514+0000", 2023-09-23T21:04:10.303 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:04:10.303 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:04:10.303 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:04:10.303 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:04:10.303 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:10.282514+0000", 2023-09-23T21:04:10.303 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:04:10.303 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:04:10.303 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:04:10.303 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:04:10.303 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:10.282514+0000", 2023-09-23T21:04:10.304 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.999999 2023-09-23T21:04:10.304 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:04:10.304 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:04:10.304 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:04:10.304 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:10.282522+0000", 2023-09-23T21:04:10.304 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.0399999999999993e-06 2023-09-23T21:04:10.304 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:04:10.304 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:04:10.304 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:04:10.304 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:10.282537+0000", 2023-09-23T21:04:10.304 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.5248999999999999e-05 2023-09-23T21:04:10.304 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:04:10.304 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:04:10.304 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:04:10.305 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:10.282543+0000", 2023-09-23T21:04:10.305 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.9710000000000001e-06 2023-09-23T21:04:10.305 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:04:10.305 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:04:10.305 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:04:10.305 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:10.282610+0000", 2023-09-23T21:04:10.305 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.7119999999999994e-05 2023-09-23T21:04:10.305 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:04:10.305 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:04:10.305 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:04:10.305 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:10.282632+0000", 2023-09-23T21:04:10.305 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.2056e-05 2023-09-23T21:04:10.306 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:04:10.306 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:04:10.306 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:04:10.306 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:04:10.306 INFO:teuthology.orchestra.run.smithi070.stdout: ], 2023-09-23T21:04:10.306 INFO:teuthology.orchestra.run.smithi070.stdout: "num_ops": 5 2023-09-23T21:04:10.306 INFO:teuthology.orchestra.run.smithi070.stdout:} 2023-09-23T21:04:10.335 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:04:10.402 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:04:10.715 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:04:10.721 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:04:10.721 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:04:10.739 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_blocked_ops 2023-09-23T21:04:10.823 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:04:10.882 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:04:11.563 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T21:04:11.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:04:11 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pgp_num_actual", "val": "23"}]: dispatch 2023-09-23T21:04:11.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:04:11 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pgp_num_actual", "val": "23"}]: dispatch 2023-09-23T21:04:11.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:04:11 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pgp_num_actual", "val": "23"}]: dispatch 2023-09-23T21:04:12.262 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:04:12.264 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:04:12 smithi026 ceph-mon[175151]: pgmap v795: 119 pgs: 1 active+remapped+backfill_wait, 2 active+remapped+backfilling, 116 active+clean; 3.7 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 571 op/s; 6870/182724 objects misplaced (3.760%); 17 MiB/s, 266 objects/s recovering 2023-09-23T21:04:12.265 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:04:12 smithi026 ceph-mon[175151]: failed to encode map e305 with expected crc 2023-09-23T21:04:12.265 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:04:12 smithi026 ceph-mon[175151]: failed to encode map e305 with expected crc 2023-09-23T21:04:12.265 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:04:12 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pgp_num_actual", "val": "23"}]': finished 2023-09-23T21:04:12.265 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:04:12 smithi026 ceph-mon[175151]: osdmap e305: 8 total, 8 up, 8 in 2023-09-23T21:04:12.265 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:04:12 smithi026 ceph-mon[179688]: pgmap v795: 119 pgs: 1 active+remapped+backfill_wait, 2 active+remapped+backfilling, 116 active+clean; 3.7 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 571 op/s; 6870/182724 objects misplaced (3.760%); 17 MiB/s, 266 objects/s recovering 2023-09-23T21:04:12.265 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:04:12 smithi026 ceph-mon[179688]: failed to encode map e305 with expected crc 2023-09-23T21:04:12.265 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:04:12 smithi026 ceph-mon[179688]: failed to encode map e305 with expected crc 2023-09-23T21:04:12.265 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:04:12 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pgp_num_actual", "val": "23"}]': finished 2023-09-23T21:04:12.265 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:04:12 smithi026 ceph-mon[179688]: osdmap e305: 8 total, 8 up, 8 in 2023-09-23T21:04:12.266 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:04:12.266 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:04:12.367 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:04:12.402 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:04:12.445 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:04:12.449 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:04:12 smithi070 ceph-mon[168282]: pgmap v795: 119 pgs: 1 active+remapped+backfill_wait, 2 active+remapped+backfilling, 116 active+clean; 3.7 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 571 op/s; 6870/182724 objects misplaced (3.760%); 17 MiB/s, 266 objects/s recovering 2023-09-23T21:04:12.450 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:04:12 smithi070 ceph-mon[168282]: failed to encode map e305 with expected crc 2023-09-23T21:04:12.450 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:04:12 smithi070 ceph-mon[168282]: failed to encode map e305 with expected crc 2023-09-23T21:04:12.450 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:04:12 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pgp_num_actual", "val": "23"}]': finished 2023-09-23T21:04:12.450 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:04:12 smithi070 ceph-mon[168282]: osdmap e305: 8 total, 8 up, 8 in 2023-09-23T21:04:12.451 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:04:12.451 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:04:12.552 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:04:12.588 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:04:12.943 INFO:teuthology.orchestra.run.smithi070.stdout:{ 2023-09-23T21:04:12.943 INFO:teuthology.orchestra.run.smithi070.stdout: "ops": [], 2023-09-23T21:04:12.943 INFO:teuthology.orchestra.run.smithi070.stdout: "complaint_time": 30, 2023-09-23T21:04:12.943 INFO:teuthology.orchestra.run.smithi070.stdout: "num_blocked_ops": 0 2023-09-23T21:04:12.944 INFO:teuthology.orchestra.run.smithi070.stdout:} 2023-09-23T21:04:13.283 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:04:13 smithi070 ceph-mon[168282]: failed to encode map e306 with expected crc 2023-09-23T21:04:13.283 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:04:13 smithi070 ceph-mon[168282]: osdmap e306: 8 total, 8 up, 8 in 2023-09-23T21:04:13.283 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:04:13 smithi070 ceph-mon[168282]: failed to encode map e306 with expected crc 2023-09-23T21:04:13.284 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:04:13 smithi070 ceph-mon[168282]: failed to encode map e306 with expected crc 2023-09-23T21:04:13.284 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:04:13 smithi070 ceph-mon[168282]: failed to encode map e306 with expected crc 2023-09-23T21:04:13.284 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:04:13 smithi070 ceph-mon[168282]: failed to encode map e306 with expected crc 2023-09-23T21:04:13.284 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:04:13 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pgp_num_actual", "val": "24"}]: dispatch 2023-09-23T21:04:13.364 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_historic_ops 2023-09-23T21:04:13.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:04:13 smithi026 ceph-mon[179688]: failed to encode map e306 with expected crc 2023-09-23T21:04:13.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:04:13 smithi026 ceph-mon[179688]: osdmap e306: 8 total, 8 up, 8 in 2023-09-23T21:04:13.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:04:13 smithi026 ceph-mon[179688]: failed to encode map e306 with expected crc 2023-09-23T21:04:13.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:04:13 smithi026 ceph-mon[179688]: failed to encode map e306 with expected crc 2023-09-23T21:04:13.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:04:13 smithi026 ceph-mon[179688]: failed to encode map e306 with expected crc 2023-09-23T21:04:13.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:04:13 smithi026 ceph-mon[179688]: failed to encode map e306 with expected crc 2023-09-23T21:04:13.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:04:13 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pgp_num_actual", "val": "24"}]: dispatch 2023-09-23T21:04:13.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:04:13 smithi026 ceph-mon[175151]: failed to encode map e306 with expected crc 2023-09-23T21:04:13.633 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:04:13 smithi026 ceph-mon[175151]: osdmap e306: 8 total, 8 up, 8 in 2023-09-23T21:04:13.633 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:04:13 smithi026 ceph-mon[175151]: failed to encode map e306 with expected crc 2023-09-23T21:04:13.633 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:04:13 smithi026 ceph-mon[175151]: failed to encode map e306 with expected crc 2023-09-23T21:04:13.633 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:04:13 smithi026 ceph-mon[175151]: failed to encode map e306 with expected crc 2023-09-23T21:04:13.633 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:04:13 smithi026 ceph-mon[175151]: failed to encode map e306 with expected crc 2023-09-23T21:04:13.633 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:04:13 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pgp_num_actual", "val": "24"}]: dispatch 2023-09-23T21:04:14.028 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T21:04:14.370 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T21:04:14.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:04:14 smithi070 ceph-mon[168282]: pgmap v798: 119 pgs: 1 active+remapped+backfill_wait, 2 active+remapped+backfilling, 116 active+clean; 3.6 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 875 op/s; 3769/175626 objects misplaced (2.146%); 32 MiB/s, 511 objects/s recovering 2023-09-23T21:04:14.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:04:14 smithi070 ceph-mon[168282]: 8.4 scrub starts 2023-09-23T21:04:14.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:04:14 smithi070 ceph-mon[168282]: failed to encode map e307 with expected crc 2023-09-23T21:04:14.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:04:14 smithi070 ceph-mon[168282]: failed to encode map e307 with expected crc 2023-09-23T21:04:14.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:04:14 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pgp_num_actual", "val": "24"}]': finished 2023-09-23T21:04:14.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:04:14 smithi070 ceph-mon[168282]: osdmap e307: 8 total, 8 up, 8 in 2023-09-23T21:04:14.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:04:14 smithi070 ceph-mon[168282]: failed to encode map e307 with expected crc 2023-09-23T21:04:14.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:04:14 smithi070 ceph-mon[168282]: 8.16 starting backfill to osd.2 from (0'0,0'0] MAX to 304'9534 2023-09-23T21:04:14.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:04:14 smithi070 ceph-mon[168282]: failed to encode map e307 with expected crc 2023-09-23T21:04:14.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:04:14 smithi070 ceph-mon[168282]: 8.16 starting backfill to osd.3 from (0'0,0'0] MAX to 304'9534 2023-09-23T21:04:14.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:04:14 smithi070 ceph-mon[168282]: 8.16 starting backfill to osd.5 from (0'0,0'0] MAX to 304'9534 2023-09-23T21:04:14.634 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:04:14 smithi026 ceph-mon[175151]: pgmap v798: 119 pgs: 1 active+remapped+backfill_wait, 2 active+remapped+backfilling, 116 active+clean; 3.6 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 875 op/s; 3769/175626 objects misplaced (2.146%); 32 MiB/s, 511 objects/s recovering 2023-09-23T21:04:14.634 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:04:14 smithi026 ceph-mon[175151]: 8.4 scrub starts 2023-09-23T21:04:14.634 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:04:14 smithi026 ceph-mon[175151]: failed to encode map e307 with expected crc 2023-09-23T21:04:14.634 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:04:14 smithi026 ceph-mon[175151]: failed to encode map e307 with expected crc 2023-09-23T21:04:14.634 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:04:14 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pgp_num_actual", "val": "24"}]': finished 2023-09-23T21:04:14.634 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:04:14 smithi026 ceph-mon[175151]: osdmap e307: 8 total, 8 up, 8 in 2023-09-23T21:04:14.635 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:04:14 smithi026 ceph-mon[175151]: failed to encode map e307 with expected crc 2023-09-23T21:04:14.635 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:04:14 smithi026 ceph-mon[175151]: 8.16 starting backfill to osd.2 from (0'0,0'0] MAX to 304'9534 2023-09-23T21:04:14.635 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:04:14 smithi026 ceph-mon[175151]: failed to encode map e307 with expected crc 2023-09-23T21:04:14.635 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:04:14 smithi026 ceph-mon[175151]: 8.16 starting backfill to osd.3 from (0'0,0'0] MAX to 304'9534 2023-09-23T21:04:14.635 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:04:14 smithi026 ceph-mon[175151]: 8.16 starting backfill to osd.5 from (0'0,0'0] MAX to 304'9534 2023-09-23T21:04:14.635 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:04:14 smithi026 ceph-mon[179688]: pgmap v798: 119 pgs: 1 active+remapped+backfill_wait, 2 active+remapped+backfilling, 116 active+clean; 3.6 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 875 op/s; 3769/175626 objects misplaced (2.146%); 32 MiB/s, 511 objects/s recovering 2023-09-23T21:04:14.635 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:04:14 smithi026 ceph-mon[179688]: 8.4 scrub starts 2023-09-23T21:04:14.635 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:04:14 smithi026 ceph-mon[179688]: failed to encode map e307 with expected crc 2023-09-23T21:04:14.635 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:04:14 smithi026 ceph-mon[179688]: failed to encode map e307 with expected crc 2023-09-23T21:04:14.635 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:04:14 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pgp_num_actual", "val": "24"}]': finished 2023-09-23T21:04:14.636 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:04:14 smithi026 ceph-mon[179688]: osdmap e307: 8 total, 8 up, 8 in 2023-09-23T21:04:14.636 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:04:14 smithi026 ceph-mon[179688]: failed to encode map e307 with expected crc 2023-09-23T21:04:14.636 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:04:14 smithi026 ceph-mon[179688]: 8.16 starting backfill to osd.2 from (0'0,0'0] MAX to 304'9534 2023-09-23T21:04:14.636 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:04:14 smithi026 ceph-mon[179688]: failed to encode map e307 with expected crc 2023-09-23T21:04:14.636 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:04:14 smithi026 ceph-mon[179688]: 8.16 starting backfill to osd.3 from (0'0,0'0] MAX to 304'9534 2023-09-23T21:04:14.636 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:04:14 smithi026 ceph-mon[179688]: 8.16 starting backfill to osd.5 from (0'0,0'0] MAX to 304'9534 2023-09-23T21:04:15.099 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T21:04:15.102 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:04:15.103 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:04:15.203 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:04:15.238 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:04:15.283 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:04:15.286 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:04:15.286 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:04:15.369 INFO:teuthology.orchestra.run.smithi070.stdout:{ 2023-09-23T21:04:15.369 INFO:teuthology.orchestra.run.smithi070.stdout: "size": 20, 2023-09-23T21:04:15.369 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 600, 2023-09-23T21:04:15.369 INFO:teuthology.orchestra.run.smithi070.stdout: "ops": [ 2023-09-23T21:04:15.369 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:04:15.369 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.35469.0:132991 8.19 8:9cbd96dc:::benchmark_data_smithi026_301269_object22234:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e281)", 2023-09-23T21:04:15.369 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:03:29.311270+0000", 2023-09-23T21:04:15.369 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 46.054936052000002, 2023-09-23T21:04:15.370 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.004743988, 2023-09-23T21:04:15.370 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:04:15.370 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:04:15.370 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:04:15.370 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.35469", 2023-09-23T21:04:15.370 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/4041424623", 2023-09-23T21:04:15.370 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 132991 2023-09-23T21:04:15.370 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:04:15.370 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:04:15.370 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:04:15.370 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:04:15.370 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:29.311270+0000", 2023-09-23T21:04:15.371 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:04:15.371 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:04:15.371 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:04:15.371 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:04:15.371 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:29.311270+0000", 2023-09-23T21:04:15.371 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:04:15.371 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:04:15.371 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:04:15.371 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:04:15.371 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:29.311267+0000", 2023-09-23T21:04:15.371 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999971 2023-09-23T21:04:15.372 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:04:15.372 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:04:15.372 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:04:15.372 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:29.311282+0000", 2023-09-23T21:04:15.372 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.4018e-05 2023-09-23T21:04:15.372 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:04:15.372 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:04:15.372 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:04:15.372 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:29.311313+0000", 2023-09-23T21:04:15.372 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.1189000000000002e-05 2023-09-23T21:04:15.372 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:04:15.372 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:04:15.373 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:04:15.373 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:29.311325+0000", 2023-09-23T21:04:15.373 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.1951e-05 2023-09-23T21:04:15.373 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:04:15.373 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:04:15.373 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:04:15.373 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:29.453380+0000", 2023-09-23T21:04:15.373 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.14205537900000001 2023-09-23T21:04:15.373 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:04:15.373 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:04:15.373 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:04:15.374 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:29.453388+0000", 2023-09-23T21:04:15.374 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.5630000000000003e-06 2023-09-23T21:04:15.374 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:04:15.374 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:04:15.374 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:04:15.374 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:30.311125+0000", 2023-09-23T21:04:15.374 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.85773704500000003 2023-09-23T21:04:15.374 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:04:15.374 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:04:15.374 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:04:15.374 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:30.311242+0000", 2023-09-23T21:04:15.374 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000117271 2023-09-23T21:04:15.375 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:04:15.375 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:04:15.375 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 0,5", 2023-09-23T21:04:15.375 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:30.311447+0000", 2023-09-23T21:04:15.375 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00020477200000000001 2023-09-23T21:04:15.375 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:04:15.375 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:04:15.375 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:04:15.375 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:30.315503+0000", 2023-09-23T21:04:15.375 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0040560320000000002 2023-09-23T21:04:15.375 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:04:15.375 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:04:15.376 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:04:15.376 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:30.315868+0000", 2023-09-23T21:04:15.376 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00036542900000000001 2023-09-23T21:04:15.376 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:04:15.376 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:04:15.376 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:04:15.376 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:30.315916+0000", 2023-09-23T21:04:15.376 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.7395000000000003e-05 2023-09-23T21:04:15.376 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:04:15.376 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:04:15.376 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:04:15.377 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:30.315975+0000", 2023-09-23T21:04:15.377 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.9741000000000003e-05 2023-09-23T21:04:15.377 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:04:15.377 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:04:15.377 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:04:15.377 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:30.316014+0000", 2023-09-23T21:04:15.377 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.9153000000000003e-05 2023-09-23T21:04:15.377 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:04:15.377 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:04:15.377 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:04:15.377 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:04:15.377 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:04:15.378 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.35469.0:133002 8.19 8:9f35daf1:::benchmark_data_smithi026_301269_object22245:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e281)", 2023-09-23T21:04:15.378 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:03:29.311341+0000", 2023-09-23T21:04:15.378 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 46.054865675999999, 2023-09-23T21:04:15.378 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.0085733610000001, 2023-09-23T21:04:15.378 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:04:15.378 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:04:15.378 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:04:15.378 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.35469", 2023-09-23T21:04:15.378 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/4041424623", 2023-09-23T21:04:15.378 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 133002 2023-09-23T21:04:15.378 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:04:15.379 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:04:15.379 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:04:15.379 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:04:15.379 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:29.311341+0000", 2023-09-23T21:04:15.379 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:04:15.379 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:04:15.379 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:04:15.379 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:04:15.379 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:29.311341+0000", 2023-09-23T21:04:15.379 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:04:15.380 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:04:15.380 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:04:15.380 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:04:15.380 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:29.311340+0000", 2023-09-23T21:04:15.380 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999995 2023-09-23T21:04:15.380 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:04:15.380 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:04:15.380 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:04:15.380 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:29.311343+0000", 2023-09-23T21:04:15.380 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.8940000000000002e-06 2023-09-23T21:04:15.380 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:04:15.380 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:04:15.381 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:04:15.381 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:29.311349+0000", 2023-09-23T21:04:15.381 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.1199999999999999e-06 2023-09-23T21:04:15.381 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:04:15.381 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:04:15.381 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:04:15.381 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:29.311353+0000", 2023-09-23T21:04:15.381 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.805e-06 2023-09-23T21:04:15.381 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:04:15.381 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:04:15.381 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:04:15.382 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:29.453406+0000", 2023-09-23T21:04:15.382 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.14205331299999999 2023-09-23T21:04:15.382 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:04:15.382 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:04:15.382 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:04:15.382 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:29.453411+0000", 2023-09-23T21:04:15.382 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.9509999999999997e-06 2023-09-23T21:04:15.382 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:04:15.382 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:04:15.382 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:04:15.382 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:30.312451+0000", 2023-09-23T21:04:15.383 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.85903911499999996 2023-09-23T21:04:15.383 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:04:15.383 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:04:15.383 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:04:15.383 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:30.312530+0000", 2023-09-23T21:04:15.383 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.9346999999999998e-05 2023-09-23T21:04:15.383 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:04:15.383 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:04:15.383 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 0,5", 2023-09-23T21:04:15.383 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:30.312649+0000", 2023-09-23T21:04:15.383 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000118867 2023-09-23T21:04:15.383 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:04:15.384 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:04:15.384 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:04:15.384 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:30.315510+0000", 2023-09-23T21:04:15.384 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0028615839999999999 2023-09-23T21:04:15.384 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:04:15.384 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:04:15.384 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:04:15.384 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:30.316097+0000", 2023-09-23T21:04:15.384 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00058688200000000005 2023-09-23T21:04:15.384 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:04:15.384 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:04:15.384 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:04:15.385 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:30.319819+0000", 2023-09-23T21:04:15.385 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0037219219999999999 2023-09-23T21:04:15.385 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:04:15.385 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:04:15.385 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:04:15.385 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:30.319871+0000", 2023-09-23T21:04:15.385 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.1753999999999999e-05 2023-09-23T21:04:15.385 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:04:15.385 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:04:15.385 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:04:15.385 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:30.319914+0000", 2023-09-23T21:04:15.385 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.3278000000000001e-05 2023-09-23T21:04:15.386 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:04:15.386 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:04:15.386 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:04:15.386 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:04:15.386 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:04:15.386 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.35469.0:133004 8.19 8:98907399:::benchmark_data_smithi026_301269_object22247:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e281)", 2023-09-23T21:04:15.386 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:03:29.311357+0000", 2023-09-23T21:04:15.386 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 46.054849023000003, 2023-09-23T21:04:15.386 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.0087018400000001, 2023-09-23T21:04:15.386 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:04:15.386 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:04:15.387 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:04:15.387 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.35469", 2023-09-23T21:04:15.387 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/4041424623", 2023-09-23T21:04:15.387 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 133004 2023-09-23T21:04:15.387 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:04:15.387 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:04:15.387 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:04:15.387 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:04:15.387 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:29.311357+0000", 2023-09-23T21:04:15.387 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:04:15.387 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:04:15.387 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:04:15.389 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:04:15.389 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:29.311357+0000", 2023-09-23T21:04:15.389 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:04:15.389 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:04:15.389 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:04:15.389 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:04:15.389 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:29.311357+0000", 2023-09-23T21:04:15.390 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999995 2023-09-23T21:04:15.390 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:04:15.390 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:04:15.390 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:04:15.390 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:29.311359+0000", 2023-09-23T21:04:15.390 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.1509999999999998e-06 2023-09-23T21:04:15.390 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:04:15.390 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:04:15.390 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:04:15.390 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:29.311364+0000", 2023-09-23T21:04:15.390 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.9169999999999997e-06 2023-09-23T21:04:15.390 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:04:15.391 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:04:15.391 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:04:15.391 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:29.311367+0000", 2023-09-23T21:04:15.391 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.1870000000000001e-06 2023-09-23T21:04:15.391 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:04:15.391 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:04:15.391 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:04:15.391 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:29.453430+0000", 2023-09-23T21:04:15.391 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.142062514 2023-09-23T21:04:15.391 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:04:15.391 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:04:15.391 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:04:15.392 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:29.453435+0000", 2023-09-23T21:04:15.392 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.8729999999999998e-06 2023-09-23T21:04:15.392 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:04:15.392 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:04:15.392 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:04:15.392 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:30.313361+0000", 2023-09-23T21:04:15.392 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.85992620799999997 2023-09-23T21:04:15.392 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:04:15.392 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:04:15.392 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:04:15.392 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:30.313428+0000", 2023-09-23T21:04:15.392 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.7355999999999997e-05 2023-09-23T21:04:15.393 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:04:15.393 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:04:15.393 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 0,5", 2023-09-23T21:04:15.393 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:30.313551+0000", 2023-09-23T21:04:15.393 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00012231000000000001 2023-09-23T21:04:15.393 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:04:15.393 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:04:15.393 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:04:15.393 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:30.315819+0000", 2023-09-23T21:04:15.393 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.002268267 2023-09-23T21:04:15.393 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:04:15.393 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:04:15.394 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:04:15.394 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:30.316740+0000", 2023-09-23T21:04:15.394 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00092082700000000004 2023-09-23T21:04:15.394 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:04:15.394 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:04:15.394 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:04:15.394 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:30.319974+0000", 2023-09-23T21:04:15.394 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0032337220000000001 2023-09-23T21:04:15.394 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:04:15.394 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:04:15.394 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:04:15.394 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:30.320024+0000", 2023-09-23T21:04:15.395 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.0667999999999997e-05 2023-09-23T21:04:15.395 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:04:15.395 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:04:15.395 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:04:15.395 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:30.320059+0000", 2023-09-23T21:04:15.395 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.5120999999999998e-05 2023-09-23T21:04:15.395 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:04:15.395 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:04:15.395 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:04:15.395 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:04:15.395 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:04:15.396 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.35469.0:133011 8.5 8:a1043694:::benchmark_data_smithi026_301269_object22254:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e281)", 2023-09-23T21:04:15.396 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:03:29.556262+0000", 2023-09-23T21:04:15.396 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 45.809944135999999, 2023-09-23T21:04:15.396 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.74415403300000005, 2023-09-23T21:04:15.396 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:04:15.396 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:04:15.396 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:04:15.396 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.35469", 2023-09-23T21:04:15.396 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/4041424623", 2023-09-23T21:04:15.396 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 133011 2023-09-23T21:04:15.396 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:04:15.396 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:04:15.397 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:04:15.397 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:04:15.397 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:29.556262+0000", 2023-09-23T21:04:15.397 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:04:15.397 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:04:15.397 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:04:15.397 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:04:15.397 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:29.556262+0000", 2023-09-23T21:04:15.397 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:04:15.397 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:04:15.397 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:04:15.397 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:04:15.397 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:29.556260+0000", 2023-09-23T21:04:15.398 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999976 2023-09-23T21:04:15.398 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:04:15.398 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:04:15.398 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:04:15.398 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:29.556273+0000", 2023-09-23T21:04:15.398 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.2782000000000001e-05 2023-09-23T21:04:15.398 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:04:15.398 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:04:15.398 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:04:15.398 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:29.556299+0000", 2023-09-23T21:04:15.398 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.6855999999999998e-05 2023-09-23T21:04:15.398 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:04:15.399 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:04:15.399 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:04:15.399 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:29.556313+0000", 2023-09-23T21:04:15.399 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.3550000000000001e-05 2023-09-23T21:04:15.399 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:04:15.399 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:04:15.399 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:04:15.399 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:29.556374+0000", 2023-09-23T21:04:15.399 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.0738e-05 2023-09-23T21:04:15.399 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:04:15.399 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:04:15.400 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:04:15.400 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:29.556386+0000", 2023-09-23T21:04:15.400 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.1806e-05 2023-09-23T21:04:15.400 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:04:15.400 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:04:15.400 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:04:15.400 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:30.279558+0000", 2023-09-23T21:04:15.400 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.72317220800000004 2023-09-23T21:04:15.400 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:04:15.400 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:04:15.400 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:04:15.400 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:30.279690+0000", 2023-09-23T21:04:15.401 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00013265400000000001 2023-09-23T21:04:15.401 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:04:15.401 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:04:15.401 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 4,7", 2023-09-23T21:04:15.401 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:30.280070+0000", 2023-09-23T21:04:15.401 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00037984200000000003 2023-09-23T21:04:15.401 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:04:15.401 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:04:15.401 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:04:15.401 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:30.285413+0000", 2023-09-23T21:04:15.401 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0053427259999999999 2023-09-23T21:04:15.401 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:04:15.402 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:04:15.402 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:04:15.402 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:30.287185+0000", 2023-09-23T21:04:15.402 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.001771658 2023-09-23T21:04:15.402 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:04:15.402 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:04:15.402 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:04:15.402 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:30.300329+0000", 2023-09-23T21:04:15.402 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.013144718 2023-09-23T21:04:15.402 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:04:15.402 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:04:15.403 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:04:15.403 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:30.300379+0000", 2023-09-23T21:04:15.403 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.9703999999999997e-05 2023-09-23T21:04:15.403 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:04:15.403 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:04:15.403 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:04:15.403 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:30.300416+0000", 2023-09-23T21:04:15.403 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.7321999999999999e-05 2023-09-23T21:04:15.403 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:04:15.403 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:04:15.403 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:04:15.403 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:04:15.404 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:04:15.404 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.35469.0:133012 8.9 8:95bb4905:::benchmark_data_smithi026_301269_object22255:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e281)", 2023-09-23T21:04:15.404 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:03:29.606840+0000", 2023-09-23T21:04:15.404 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 45.759366888999999, 2023-09-23T21:04:15.404 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.68457533800000003, 2023-09-23T21:04:15.404 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:04:15.404 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:04:15.404 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:04:15.404 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.35469", 2023-09-23T21:04:15.404 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/4041424623", 2023-09-23T21:04:15.404 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 133012 2023-09-23T21:04:15.404 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:04:15.405 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:04:15.405 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:04:15.405 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:04:15.405 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:29.606840+0000", 2023-09-23T21:04:15.405 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:04:15.405 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:04:15.405 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:04:15.405 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:04:15.405 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:29.606840+0000", 2023-09-23T21:04:15.405 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:04:15.405 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:04:15.405 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:04:15.406 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:04:15.406 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:29.606838+0000", 2023-09-23T21:04:15.406 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999981 2023-09-23T21:04:15.406 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:04:15.406 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:04:15.406 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:04:15.406 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:29.606846+0000", 2023-09-23T21:04:15.406 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.2109999999999998e-06 2023-09-23T21:04:15.406 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:04:15.406 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:04:15.406 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:04:15.406 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:29.606880+0000", 2023-09-23T21:04:15.407 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.4286999999999999e-05 2023-09-23T21:04:15.407 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:04:15.407 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:04:15.407 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:04:15.407 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:29.606889+0000", 2023-09-23T21:04:15.407 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.6659999999999997e-06 2023-09-23T21:04:15.407 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:04:15.407 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:04:15.407 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:04:15.407 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:29.606945+0000", 2023-09-23T21:04:15.407 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.6005000000000003e-05 2023-09-23T21:04:15.407 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:04:15.407 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:04:15.408 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:04:15.408 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:29.606956+0000", 2023-09-23T21:04:15.408 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.1122e-05 2023-09-23T21:04:15.408 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:04:15.408 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:04:15.408 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:04:15.408 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:30.247854+0000", 2023-09-23T21:04:15.408 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.64089811100000005 2023-09-23T21:04:15.408 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:04:15.408 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:04:15.408 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:04:15.408 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:30.248009+0000", 2023-09-23T21:04:15.409 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00015453 2023-09-23T21:04:15.409 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:04:15.409 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:04:15.409 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 0,5", 2023-09-23T21:04:15.409 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:30.248490+0000", 2023-09-23T21:04:15.409 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00048119599999999998 2023-09-23T21:04:15.409 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:04:15.409 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:04:15.409 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:04:15.409 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:30.251137+0000", 2023-09-23T21:04:15.409 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0026472409999999998 2023-09-23T21:04:15.409 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:04:15.409 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:04:15.410 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:04:15.410 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:30.286568+0000", 2023-09-23T21:04:15.410 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.035431129999999998 2023-09-23T21:04:15.410 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:04:15.410 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:04:15.410 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:04:15.410 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:30.291324+0000", 2023-09-23T21:04:15.410 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.004755881 2023-09-23T21:04:15.410 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:04:15.410 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:04:15.410 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:04:15.410 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:30.291374+0000", 2023-09-23T21:04:15.411 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.9885000000000001e-05 2023-09-23T21:04:15.411 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:04:15.411 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:04:15.411 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:04:15.411 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:30.291415+0000", 2023-09-23T21:04:15.411 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.0858000000000001e-05 2023-09-23T21:04:15.411 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:04:15.411 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:04:15.411 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:04:15.411 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:04:15.411 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:04:15.411 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.35469.0:141176 8.12 8.59eb0f52 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e288)", 2023-09-23T21:04:15.412 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:03:41.352623+0000", 2023-09-23T21:04:15.412 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 34.013583056000002, 2023-09-23T21:04:15.412 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.829076496, 2023-09-23T21:04:15.412 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:04:15.412 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "reached pg", 2023-09-23T21:04:15.412 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:04:15.412 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.35469", 2023-09-23T21:04:15.412 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/4041424623", 2023-09-23T21:04:15.412 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 141176 2023-09-23T21:04:15.412 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:04:15.412 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:04:15.412 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:04:15.412 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:04:15.413 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:41.352623+0000", 2023-09-23T21:04:15.413 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:04:15.413 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:04:15.413 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:04:15.413 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:04:15.413 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:41.352623+0000", 2023-09-23T21:04:15.413 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:04:15.413 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:04:15.413 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:04:15.413 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:04:15.413 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:41.352623+0000", 2023-09-23T21:04:15.413 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999995 2023-09-23T21:04:15.414 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:04:15.414 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:04:15.414 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:04:15.414 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:41.352626+0000", 2023-09-23T21:04:15.414 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.7980000000000001e-06 2023-09-23T21:04:15.414 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:04:15.414 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:04:15.414 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:04:15.414 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:41.352636+0000", 2023-09-23T21:04:15.414 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.8630000000000003e-06 2023-09-23T21:04:15.414 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:04:15.414 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:04:15.415 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:04:15.415 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:41.352639+0000", 2023-09-23T21:04:15.415 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.8109999999999999e-06 2023-09-23T21:04:15.415 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:04:15.415 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:04:15.415 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:04:15.415 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:41.352683+0000", 2023-09-23T21:04:15.415 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.4365e-05 2023-09-23T21:04:15.415 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:04:15.415 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:04:15.415 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:04:15.415 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:41.352688+0000", 2023-09-23T21:04:15.415 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.164e-06 2023-09-23T21:04:15.416 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:04:15.416 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:04:15.416 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:04:15.416 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:42.181691+0000", 2023-09-23T21:04:15.416 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.82900300699999996 2023-09-23T21:04:15.416 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:04:15.416 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:04:15.416 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:04:15.416 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:42.181700+0000", 2023-09-23T21:04:15.416 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.8890000000000008e-06 2023-09-23T21:04:15.416 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:04:15.416 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:04:15.417 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:04:15.417 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:04:15.417 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:04:15.417 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.35469.0:141263 8.12 8.15bb0532 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e288)", 2023-09-23T21:04:15.417 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:03:41.434595+0000", 2023-09-23T21:04:15.417 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 33.931611414000002, 2023-09-23T21:04:15.417 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.74712263400000001, 2023-09-23T21:04:15.417 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:04:15.417 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "reached pg", 2023-09-23T21:04:15.417 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:04:15.417 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.35469", 2023-09-23T21:04:15.417 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/4041424623", 2023-09-23T21:04:15.417 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 141263 2023-09-23T21:04:15.418 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:04:15.418 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:04:15.418 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:04:15.418 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:04:15.418 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:41.434595+0000", 2023-09-23T21:04:15.418 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:04:15.418 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:04:15.418 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:04:15.418 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:04:15.418 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:41.434595+0000", 2023-09-23T21:04:15.418 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:04:15.418 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:04:15.418 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:04:15.419 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:04:15.419 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:41.434594+0000", 2023-09-23T21:04:15.419 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T21:04:15.419 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:04:15.419 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:04:15.419 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:04:15.419 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:41.434601+0000", 2023-09-23T21:04:15.419 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.8859999999999998e-06 2023-09-23T21:04:15.419 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:04:15.419 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:04:15.419 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:04:15.419 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:41.434613+0000", 2023-09-23T21:04:15.420 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.1922999999999999e-05 2023-09-23T21:04:15.420 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:04:15.420 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:04:15.420 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:04:15.420 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:41.434619+0000", 2023-09-23T21:04:15.420 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.6039999999999998e-06 2023-09-23T21:04:15.420 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:04:15.420 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:04:15.420 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:04:15.420 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:41.434729+0000", 2023-09-23T21:04:15.420 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000109573 2023-09-23T21:04:15.420 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:04:15.420 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:04:15.421 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:04:15.421 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:41.434736+0000", 2023-09-23T21:04:15.421 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.2250000000000003e-06 2023-09-23T21:04:15.421 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:04:15.421 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:04:15.421 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:04:15.421 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:42.181714+0000", 2023-09-23T21:04:15.421 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.74697798999999998 2023-09-23T21:04:15.421 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:04:15.421 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:04:15.421 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:04:15.421 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:42.181718+0000", 2023-09-23T21:04:15.422 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.749e-06 2023-09-23T21:04:15.422 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:04:15.422 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:04:15.422 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:04:15.422 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:04:15.422 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:04:15.422 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.35469.0:141302 8.12 8.247f1cf2 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e288)", 2023-09-23T21:04:15.422 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:03:41.472759+0000", 2023-09-23T21:04:15.422 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 33.893447862000002, 2023-09-23T21:04:15.422 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.70897215400000002, 2023-09-23T21:04:15.422 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:04:15.422 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "reached pg", 2023-09-23T21:04:15.422 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:04:15.423 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.35469", 2023-09-23T21:04:15.423 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/4041424623", 2023-09-23T21:04:15.423 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 141302 2023-09-23T21:04:15.423 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:04:15.423 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:04:15.423 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:04:15.423 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:04:15.423 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:41.472759+0000", 2023-09-23T21:04:15.423 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:04:15.423 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:04:15.423 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:04:15.423 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:04:15.423 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:41.472759+0000", 2023-09-23T21:04:15.424 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:04:15.424 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:04:15.424 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:04:15.424 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:04:15.424 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:41.472757+0000", 2023-09-23T21:04:15.424 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T21:04:15.424 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:04:15.424 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:04:15.424 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:04:15.424 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:41.472765+0000", 2023-09-23T21:04:15.424 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.3889999999999999e-06 2023-09-23T21:04:15.424 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:04:15.425 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:04:15.425 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:04:15.425 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:41.472784+0000", 2023-09-23T21:04:15.425 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.8918000000000001e-05 2023-09-23T21:04:15.425 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:04:15.425 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:04:15.425 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:04:15.425 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:41.472791+0000", 2023-09-23T21:04:15.425 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.5009999999999998e-06 2023-09-23T21:04:15.425 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:04:15.425 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:04:15.425 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:04:15.426 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:41.478507+0000", 2023-09-23T21:04:15.426 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0057159840000000003 2023-09-23T21:04:15.426 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:04:15.426 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:04:15.426 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:04:15.426 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:41.478517+0000", 2023-09-23T21:04:15.426 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.4900000000000006e-06 2023-09-23T21:04:15.426 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:04:15.426 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:04:15.426 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:04:15.426 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:42.181727+0000", 2023-09-23T21:04:15.426 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.70321072900000003 2023-09-23T21:04:15.426 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:04:15.427 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:04:15.427 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:04:15.427 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:42.181731+0000", 2023-09-23T21:04:15.427 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.4300000000000002e-06 2023-09-23T21:04:15.427 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:04:15.427 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:04:15.427 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:04:15.427 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:04:15.427 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:04:15.427 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.35469.0:141327 8.12 8.e4f6e4d2 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e288)", 2023-09-23T21:04:15.427 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:03:41.504514+0000", 2023-09-23T21:04:15.427 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 33.861692261000002, 2023-09-23T21:04:15.428 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.67722902600000001, 2023-09-23T21:04:15.428 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:04:15.428 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "reached pg", 2023-09-23T21:04:15.428 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:04:15.428 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.35469", 2023-09-23T21:04:15.428 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/4041424623", 2023-09-23T21:04:15.428 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 141327 2023-09-23T21:04:15.428 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:04:15.428 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:04:15.428 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:04:15.428 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:04:15.428 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:41.504514+0000", 2023-09-23T21:04:15.429 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:04:15.429 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:04:15.429 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:04:15.429 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:04:15.429 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:41.504514+0000", 2023-09-23T21:04:15.429 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:04:15.429 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:04:15.429 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:04:15.429 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:04:15.429 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:41.504513+0000", 2023-09-23T21:04:15.429 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T21:04:15.429 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:04:15.429 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:04:15.430 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:04:15.430 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:41.504519+0000", 2023-09-23T21:04:15.430 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.3690000000000003e-06 2023-09-23T21:04:15.430 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:04:15.430 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:04:15.430 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:04:15.430 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:41.504532+0000", 2023-09-23T21:04:15.430 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.2979999999999999e-05 2023-09-23T21:04:15.430 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:04:15.430 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:04:15.430 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:04:15.430 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:41.504539+0000", 2023-09-23T21:04:15.431 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.494e-06 2023-09-23T21:04:15.431 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:04:15.431 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:04:15.431 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:04:15.431 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:41.504615+0000", 2023-09-23T21:04:15.431 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.5727000000000002e-05 2023-09-23T21:04:15.431 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:04:15.431 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:04:15.431 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:04:15.431 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:41.504622+0000", 2023-09-23T21:04:15.431 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.8569999999999995e-06 2023-09-23T21:04:15.431 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:04:15.431 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:04:15.432 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:04:15.432 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:42.181740+0000", 2023-09-23T21:04:15.432 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.67711736499999997 2023-09-23T21:04:15.432 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:04:15.432 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:04:15.432 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:04:15.432 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:42.181743+0000", 2023-09-23T21:04:15.432 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.4350000000000001e-06 2023-09-23T21:04:15.432 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:04:15.432 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:04:15.432 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:04:15.432 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:04:15.432 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:04:15.433 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.35469.0:141331 8.12 8.f3501032 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e288)", 2023-09-23T21:04:15.433 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:03:41.504554+0000", 2023-09-23T21:04:15.433 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 33.861652206999999, 2023-09-23T21:04:15.433 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.67720773599999995, 2023-09-23T21:04:15.433 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:04:15.433 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "reached pg", 2023-09-23T21:04:15.433 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:04:15.433 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.35469", 2023-09-23T21:04:15.433 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/4041424623", 2023-09-23T21:04:15.433 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 141331 2023-09-23T21:04:15.433 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:04:15.433 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:04:15.434 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:04:15.434 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:04:15.434 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:41.504554+0000", 2023-09-23T21:04:15.434 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:04:15.434 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:04:15.434 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:04:15.434 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:04:15.434 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:41.504554+0000", 2023-09-23T21:04:15.434 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:04:15.434 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:04:15.434 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:04:15.435 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:04:15.435 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:41.504554+0000", 2023-09-23T21:04:15.435 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999995 2023-09-23T21:04:15.435 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:04:15.435 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:04:15.435 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:04:15.435 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:41.504557+0000", 2023-09-23T21:04:15.435 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.399e-06 2023-09-23T21:04:15.435 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:04:15.435 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:04:15.435 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:04:15.435 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:41.504564+0000", 2023-09-23T21:04:15.436 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.2189999999999998e-06 2023-09-23T21:04:15.436 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:04:15.436 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:04:15.436 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:04:15.436 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:41.504569+0000", 2023-09-23T21:04:15.436 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.2370000000000003e-06 2023-09-23T21:04:15.436 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:04:15.436 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:04:15.436 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:04:15.436 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:41.506362+0000", 2023-09-23T21:04:15.436 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0017935379999999999 2023-09-23T21:04:15.436 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:04:15.437 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:04:15.437 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:04:15.437 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:41.506369+0000", 2023-09-23T21:04:15.437 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.0299999999999996e-06 2023-09-23T21:04:15.437 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:04:15.437 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:04:15.437 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:04:15.437 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:42.181756+0000", 2023-09-23T21:04:15.437 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.67538684500000001 2023-09-23T21:04:15.437 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:04:15.437 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:04:15.437 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:04:15.438 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:42.181762+0000", 2023-09-23T21:04:15.438 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.8860000000000001e-06 2023-09-23T21:04:15.438 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:04:15.438 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:04:15.438 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:04:15.438 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:04:15.438 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:04:15.438 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.35469.0:141336 8.12 8.541a9572 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e288)", 2023-09-23T21:04:15.438 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:03:41.506356+0000", 2023-09-23T21:04:15.438 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 33.859850053999999, 2023-09-23T21:04:15.438 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.67542309499999997, 2023-09-23T21:04:15.439 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:04:15.439 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "reached pg", 2023-09-23T21:04:15.439 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:04:15.439 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.35469", 2023-09-23T21:04:15.439 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/4041424623", 2023-09-23T21:04:15.439 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 141336 2023-09-23T21:04:15.439 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:04:15.439 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:04:15.439 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:04:15.439 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:04:15.439 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:41.506356+0000", 2023-09-23T21:04:15.439 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:04:15.439 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:04:15.440 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:04:15.440 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:04:15.440 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:41.506356+0000", 2023-09-23T21:04:15.440 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:04:15.440 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:04:15.440 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:04:15.440 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:04:15.440 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:41.506355+0000", 2023-09-23T21:04:15.440 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.999999 2023-09-23T21:04:15.440 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:04:15.440 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:04:15.441 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:04:15.441 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:41.506360+0000", 2023-09-23T21:04:15.441 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.2089999999999996e-06 2023-09-23T21:04:15.441 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:04:15.441 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:04:15.441 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:04:15.441 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:41.506372+0000", 2023-09-23T21:04:15.441 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.1656e-05 2023-09-23T21:04:15.441 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:04:15.441 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:04:15.441 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:04:15.441 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:41.506378+0000", 2023-09-23T21:04:15.442 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.3180000000000003e-06 2023-09-23T21:04:15.442 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:04:15.442 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:04:15.442 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:04:15.442 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:41.506451+0000", 2023-09-23T21:04:15.442 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.2465999999999994e-05 2023-09-23T21:04:15.442 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:04:15.442 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:04:15.442 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:04:15.442 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:41.506456+0000", 2023-09-23T21:04:15.442 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.9989999999999999e-06 2023-09-23T21:04:15.442 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:04:15.443 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:04:15.443 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:04:15.443 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:42.181776+0000", 2023-09-23T21:04:15.443 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.675320013 2023-09-23T21:04:15.443 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:04:15.443 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:04:15.443 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:04:15.443 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:42.181780+0000", 2023-09-23T21:04:15.443 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.614e-06 2023-09-23T21:04:15.443 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:04:15.443 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:04:15.443 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:04:15.444 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:04:15.444 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:04:15.444 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.35469.0:141369 8.12 8.5d876312 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e288)", 2023-09-23T21:04:15.444 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:03:41.546526+0000", 2023-09-23T21:04:15.444 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 33.819680900000002, 2023-09-23T21:04:15.444 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.63526674999999999, 2023-09-23T21:04:15.444 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:04:15.444 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "reached pg", 2023-09-23T21:04:15.444 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:04:15.444 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.35469", 2023-09-23T21:04:15.444 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/4041424623", 2023-09-23T21:04:15.444 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 141369 2023-09-23T21:04:15.445 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:04:15.445 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:04:15.445 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:04:15.445 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:04:15.445 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:41.546526+0000", 2023-09-23T21:04:15.445 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:04:15.445 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:04:15.445 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:04:15.445 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:04:15.445 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:41.546526+0000", 2023-09-23T21:04:15.445 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:04:15.445 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:04:15.446 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:04:15.446 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:04:15.446 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:41.546525+0000", 2023-09-23T21:04:15.446 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999995 2023-09-23T21:04:15.446 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:04:15.446 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:04:15.446 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:04:15.446 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:41.546528+0000", 2023-09-23T21:04:15.446 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.6730000000000001e-06 2023-09-23T21:04:15.446 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:04:15.446 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:04:15.446 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:04:15.447 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:41.548135+0000", 2023-09-23T21:04:15.447 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0016073050000000001 2023-09-23T21:04:15.447 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:04:15.447 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:04:15.447 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:04:15.447 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:41.548143+0000", 2023-09-23T21:04:15.447 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.3210000000000005e-06 2023-09-23T21:04:15.447 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:04:15.447 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:04:15.447 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:04:15.447 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:41.548192+0000", 2023-09-23T21:04:15.447 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.8665999999999998e-05 2023-09-23T21:04:15.448 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:04:15.448 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:04:15.448 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:04:15.448 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:41.548200+0000", 2023-09-23T21:04:15.448 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.0539999999999995e-06 2023-09-23T21:04:15.448 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:04:15.448 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:04:15.448 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:04:15.448 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:42.181789+0000", 2023-09-23T21:04:15.448 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.63358875199999998 2023-09-23T21:04:15.448 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:04:15.448 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:04:15.449 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:04:15.449 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:42.181792+0000", 2023-09-23T21:04:15.449 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.4029999999999999e-06 2023-09-23T21:04:15.449 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:04:15.449 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:04:15.449 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:04:15.449 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:04:15.449 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:04:15.449 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.35469.0:141420 8.12 8.1a2d0b12 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e288)", 2023-09-23T21:04:15.449 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:03:41.604591+0000", 2023-09-23T21:04:15.449 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 33.761615040000002, 2023-09-23T21:04:15.449 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.57721354700000005, 2023-09-23T21:04:15.449 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:04:15.450 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "reached pg", 2023-09-23T21:04:15.450 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:04:15.450 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.35469", 2023-09-23T21:04:15.450 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/4041424623", 2023-09-23T21:04:15.450 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 141420 2023-09-23T21:04:15.450 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:04:15.450 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:04:15.450 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:04:15.450 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:04:15.450 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:41.604591+0000", 2023-09-23T21:04:15.450 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:04:15.451 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:04:15.451 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:04:15.451 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:04:15.451 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:41.604591+0000", 2023-09-23T21:04:15.451 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:04:15.451 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:04:15.451 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:04:15.451 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:04:15.451 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:41.604590+0000", 2023-09-23T21:04:15.451 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T21:04:15.451 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:04:15.451 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:04:15.452 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:04:15.452 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:41.604597+0000", 2023-09-23T21:04:15.452 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.4740000000000002e-06 2023-09-23T21:04:15.452 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:04:15.452 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:04:15.452 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:04:15.452 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:41.604612+0000", 2023-09-23T21:04:15.452 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.5313999999999999e-05 2023-09-23T21:04:15.452 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:04:15.452 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:04:15.452 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:04:15.452 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:41.604619+0000", 2023-09-23T21:04:15.453 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.6980000000000001e-06 2023-09-23T21:04:15.453 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:04:15.453 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:04:15.453 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:04:15.453 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:41.604676+0000", 2023-09-23T21:04:15.453 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.7535999999999999e-05 2023-09-23T21:04:15.453 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:04:15.453 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:04:15.453 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:04:15.453 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:41.604684+0000", 2023-09-23T21:04:15.453 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.7079999999999997e-06 2023-09-23T21:04:15.453 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:04:15.453 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:04:15.454 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:04:15.454 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:42.181802+0000", 2023-09-23T21:04:15.454 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.57711783100000003 2023-09-23T21:04:15.454 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:04:15.454 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:04:15.454 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:04:15.454 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:42.181805+0000", 2023-09-23T21:04:15.454 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.3869999999999999e-06 2023-09-23T21:04:15.454 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:04:15.454 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:04:15.454 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:04:15.454 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:04:15.455 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:04:15.455 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.35469.0:141482 8.12 8.7bcd7e92 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e288)", 2023-09-23T21:04:15.455 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:03:41.685265+0000", 2023-09-23T21:04:15.455 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 33.680941505, 2023-09-23T21:04:15.455 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.496552573, 2023-09-23T21:04:15.455 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:04:15.455 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "reached pg", 2023-09-23T21:04:15.455 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:04:15.455 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.35469", 2023-09-23T21:04:15.455 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/4041424623", 2023-09-23T21:04:15.455 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 141482 2023-09-23T21:04:15.455 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:04:15.456 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:04:15.456 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:04:15.456 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:04:15.456 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:41.685265+0000", 2023-09-23T21:04:15.456 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:04:15.456 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:04:15.456 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:04:15.456 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:04:15.456 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:41.685265+0000", 2023-09-23T21:04:15.456 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:04:15.456 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:04:15.456 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:04:15.456 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:04:15.457 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:41.685264+0000", 2023-09-23T21:04:15.457 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.999999 2023-09-23T21:04:15.457 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:04:15.457 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:04:15.457 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:04:15.457 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:41.685270+0000", 2023-09-23T21:04:15.457 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.2559999999999999e-06 2023-09-23T21:04:15.457 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:04:15.457 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:04:15.457 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:04:15.457 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:41.685285+0000", 2023-09-23T21:04:15.457 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.4681999999999999e-05 2023-09-23T21:04:15.457 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:04:15.458 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:04:15.458 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:04:15.458 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:41.685291+0000", 2023-09-23T21:04:15.458 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.4439999999999996e-06 2023-09-23T21:04:15.458 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:04:15.458 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:04:15.458 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:04:15.458 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:41.685501+0000", 2023-09-23T21:04:15.458 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00021014100000000001 2023-09-23T21:04:15.458 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:04:15.458 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:04:15.458 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:04:15.459 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:41.685508+0000", 2023-09-23T21:04:15.459 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.4690000000000002e-06 2023-09-23T21:04:15.459 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:04:15.459 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:04:15.459 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:04:15.459 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:42.181814+0000", 2023-09-23T21:04:15.459 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.496306303 2023-09-23T21:04:15.459 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:04:15.459 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:04:15.459 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:04:15.459 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:42.181818+0000", 2023-09-23T21:04:15.459 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.3850000000000002e-06 2023-09-23T21:04:15.459 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:04:15.460 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:04:15.460 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:04:15.460 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:04:15.460 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:04:15.460 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.35469.0:141511 8.12 8.d50f2f12 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e288)", 2023-09-23T21:04:15.460 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:03:41.727617+0000", 2023-09-23T21:04:15.460 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 33.638589793000001, 2023-09-23T21:04:15.460 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.45421352199999998, 2023-09-23T21:04:15.460 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:04:15.460 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "reached pg", 2023-09-23T21:04:15.460 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:04:15.460 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.35469", 2023-09-23T21:04:15.461 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/4041424623", 2023-09-23T21:04:15.461 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 141511 2023-09-23T21:04:15.461 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:04:15.461 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:04:15.461 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:04:15.461 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:04:15.461 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:41.727617+0000", 2023-09-23T21:04:15.461 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:04:15.461 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:04:15.461 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:04:15.461 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:04:15.461 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:41.727617+0000", 2023-09-23T21:04:15.462 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:04:15.462 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:04:15.462 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:04:15.462 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:04:15.462 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:41.727615+0000", 2023-09-23T21:04:15.462 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999981 2023-09-23T21:04:15.462 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:04:15.462 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:04:15.462 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:04:15.462 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:41.727625+0000", 2023-09-23T21:04:15.462 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.0067e-05 2023-09-23T21:04:15.462 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:04:15.463 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:04:15.463 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:04:15.463 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:41.727647+0000", 2023-09-23T21:04:15.463 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.245e-05 2023-09-23T21:04:15.463 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:04:15.463 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:04:15.463 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:04:15.463 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:41.727655+0000", 2023-09-23T21:04:15.463 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.3129999999999999e-06 2023-09-23T21:04:15.463 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:04:15.463 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:04:15.463 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:04:15.464 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:41.727719+0000", 2023-09-23T21:04:15.464 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.3626000000000004e-05 2023-09-23T21:04:15.464 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:04:15.464 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:04:15.464 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:04:15.464 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:41.727729+0000", 2023-09-23T21:04:15.464 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.7869999999999996e-06 2023-09-23T21:04:15.464 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:04:15.464 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:04:15.464 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:04:15.464 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:42.181827+0000", 2023-09-23T21:04:15.464 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.45409791999999999 2023-09-23T21:04:15.465 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:04:15.465 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:04:15.465 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:04:15.465 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:42.181830+0000", 2023-09-23T21:04:15.465 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.4180000000000001e-06 2023-09-23T21:04:15.465 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:04:15.465 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:04:15.465 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:04:15.465 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:04:15.465 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:04:15.465 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.35469.0:164178 8.12 8:4fdafa29:::benchmark_data_smithi026_301269_object53421:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e307)", 2023-09-23T21:04:15.465 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:04:13.339306+0000", 2023-09-23T21:04:15.466 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 2.0269004210000001, 2023-09-23T21:04:15.466 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.90960294200000003, 2023-09-23T21:04:15.466 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:04:15.466 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:04:15.466 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:04:15.466 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.35469", 2023-09-23T21:04:15.466 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/4041424623", 2023-09-23T21:04:15.466 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 164178 2023-09-23T21:04:15.466 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:04:15.466 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:04:15.466 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:04:15.466 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:04:15.467 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:13.339306+0000", 2023-09-23T21:04:15.467 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:04:15.467 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:04:15.467 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:04:15.467 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:04:15.467 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:13.339306+0000", 2023-09-23T21:04:15.467 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:04:15.467 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:04:15.467 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:04:15.467 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:04:15.467 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:13.339305+0000", 2023-09-23T21:04:15.467 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T21:04:15.468 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:04:15.468 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:04:15.468 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:04:15.468 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:13.339312+0000", 2023-09-23T21:04:15.468 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.0620000000000003e-06 2023-09-23T21:04:15.468 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:04:15.468 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:04:15.468 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:04:15.468 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:13.339327+0000", 2023-09-23T21:04:15.468 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.5359999999999999e-05 2023-09-23T21:04:15.468 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:04:15.468 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:04:15.468 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:04:15.469 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:13.339333+0000", 2023-09-23T21:04:15.469 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.913e-06 2023-09-23T21:04:15.469 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:04:15.469 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:04:15.469 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:04:15.469 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:13.339388+0000", 2023-09-23T21:04:15.469 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.4833000000000002e-05 2023-09-23T21:04:15.469 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:04:15.469 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:04:15.469 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:04:15.469 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:13.339397+0000", 2023-09-23T21:04:15.469 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.1509999999999993e-06 2023-09-23T21:04:15.470 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:04:15.470 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:04:15.470 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:04:15.470 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:14.241184+0000", 2023-09-23T21:04:15.470 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.90178749899999999 2023-09-23T21:04:15.470 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:04:15.470 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:04:15.470 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:04:15.470 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:14.241335+0000", 2023-09-23T21:04:15.470 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00015069700000000001 2023-09-23T21:04:15.470 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:04:15.471 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:04:15.471 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 2,7", 2023-09-23T21:04:15.471 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:14.241640+0000", 2023-09-23T21:04:15.471 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00030523300000000002 2023-09-23T21:04:15.471 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:04:15.471 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:04:15.471 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:04:15.471 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:14.248356+0000", 2023-09-23T21:04:15.471 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0067158019999999999 2023-09-23T21:04:15.471 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:04:15.471 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:04:15.471 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:04:15.471 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:14.248758+0000", 2023-09-23T21:04:15.472 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00040140800000000002 2023-09-23T21:04:15.472 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:04:15.472 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:04:15.472 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:04:15.472 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:14.248825+0000", 2023-09-23T21:04:15.472 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.6898999999999998e-05 2023-09-23T21:04:15.472 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:04:15.472 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:04:15.472 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:04:15.472 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:14.248872+0000", 2023-09-23T21:04:15.472 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.7113000000000001e-05 2023-09-23T21:04:15.472 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:04:15.473 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:04:15.473 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:04:15.473 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:14.248909+0000", 2023-09-23T21:04:15.473 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.7397000000000001e-05 2023-09-23T21:04:15.473 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:04:15.473 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:04:15.473 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:04:15.473 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:04:15.473 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:04:15.473 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.35469.0:164261 8.12 8:4e1b18c7:::benchmark_data_smithi026_301269_object53504:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e307)", 2023-09-23T21:04:15.473 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:04:13.426681+0000", 2023-09-23T21:04:15.473 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 1.9395259199999999, 2023-09-23T21:04:15.474 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.82230143600000005, 2023-09-23T21:04:15.474 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:04:15.474 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:04:15.474 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:04:15.474 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.35469", 2023-09-23T21:04:15.474 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/4041424623", 2023-09-23T21:04:15.474 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 164261 2023-09-23T21:04:15.474 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:04:15.474 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:04:15.474 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:04:15.474 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:04:15.474 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:13.426681+0000", 2023-09-23T21:04:15.475 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:04:15.475 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:04:15.475 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:04:15.475 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:04:15.475 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:13.426681+0000", 2023-09-23T21:04:15.475 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:04:15.475 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:04:15.475 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:04:15.475 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:04:15.476 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:13.426680+0000", 2023-09-23T21:04:15.476 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999995 2023-09-23T21:04:15.476 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:04:15.476 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:04:15.476 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:04:15.476 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:13.426684+0000", 2023-09-23T21:04:15.476 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.693e-06 2023-09-23T21:04:15.476 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:04:15.476 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:04:15.476 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:04:15.476 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:13.426691+0000", 2023-09-23T21:04:15.476 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.576e-06 2023-09-23T21:04:15.477 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:04:15.477 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:04:15.477 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:04:15.477 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:13.426696+0000", 2023-09-23T21:04:15.477 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.8799999999999999e-06 2023-09-23T21:04:15.477 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:04:15.477 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:04:15.477 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:04:15.477 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:13.428620+0000", 2023-09-23T21:04:15.477 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.001923545 2023-09-23T21:04:15.477 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:04:15.477 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:04:15.478 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:04:15.478 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:13.428628+0000", 2023-09-23T21:04:15.478 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.106e-06 2023-09-23T21:04:15.478 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:04:15.478 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:04:15.478 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:04:15.478 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:14.242955+0000", 2023-09-23T21:04:15.478 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.81432692500000003 2023-09-23T21:04:15.478 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:04:15.478 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:04:15.478 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:04:15.478 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:14.243064+0000", 2023-09-23T21:04:15.479 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000109474 2023-09-23T21:04:15.479 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:04:15.479 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:04:15.479 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 2,7", 2023-09-23T21:04:15.479 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:14.243268+0000", 2023-09-23T21:04:15.479 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000203947 2023-09-23T21:04:15.479 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:04:15.479 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:04:15.479 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:04:15.479 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:14.248363+0000", 2023-09-23T21:04:15.479 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0050952489999999996 2023-09-23T21:04:15.479 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:04:15.479 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:04:15.480 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:04:15.480 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:14.248792+0000", 2023-09-23T21:04:15.480 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000428837 2023-09-23T21:04:15.480 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:04:15.480 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:04:15.480 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:04:15.480 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:14.248936+0000", 2023-09-23T21:04:15.480 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00014367899999999999 2023-09-23T21:04:15.480 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:04:15.480 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:04:15.480 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:04:15.480 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:14.248962+0000", 2023-09-23T21:04:15.481 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.6135999999999998e-05 2023-09-23T21:04:15.481 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:04:15.481 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:04:15.481 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:04:15.481 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:14.248982+0000", 2023-09-23T21:04:15.481 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.9938000000000001e-05 2023-09-23T21:04:15.481 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:04:15.481 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:04:15.481 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:04:15.481 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:04:15.481 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:04:15.481 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.35469.0:164284 8.12 8:4e7b7a22:::benchmark_data_smithi026_301269_object53527:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e307)", 2023-09-23T21:04:15.481 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:04:13.453160+0000", 2023-09-23T21:04:15.482 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 1.9130467499999999, 2023-09-23T21:04:15.482 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.79592301499999996, 2023-09-23T21:04:15.482 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:04:15.482 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:04:15.482 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:04:15.482 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.35469", 2023-09-23T21:04:15.482 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/4041424623", 2023-09-23T21:04:15.482 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 164284 2023-09-23T21:04:15.482 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:04:15.482 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:04:15.482 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:04:15.482 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:04:15.483 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:13.453160+0000", 2023-09-23T21:04:15.483 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:04:15.483 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:04:15.483 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:04:15.483 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:04:15.483 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:13.453160+0000", 2023-09-23T21:04:15.483 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:04:15.483 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:04:15.483 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:04:15.483 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:04:15.483 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:13.453159+0000", 2023-09-23T21:04:15.483 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.999999 2023-09-23T21:04:15.484 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:04:15.484 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:04:15.484 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:04:15.484 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:13.453164+0000", 2023-09-23T21:04:15.484 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.2959999999999998e-06 2023-09-23T21:04:15.484 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:04:15.484 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:04:15.484 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:04:15.484 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:13.453176+0000", 2023-09-23T21:04:15.484 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.2211999999999999e-05 2023-09-23T21:04:15.484 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:04:15.484 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:04:15.485 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:04:15.485 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:13.453183+0000", 2023-09-23T21:04:15.485 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.686e-06 2023-09-23T21:04:15.485 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:04:15.485 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:04:15.485 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:04:15.485 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:13.453256+0000", 2023-09-23T21:04:15.485 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.2648e-05 2023-09-23T21:04:15.485 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:04:15.485 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:04:15.485 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:04:15.486 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:13.453263+0000", 2023-09-23T21:04:15.486 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.6680000000000001e-06 2023-09-23T21:04:15.486 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:04:15.486 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:04:15.486 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:04:15.486 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:14.244530+0000", 2023-09-23T21:04:15.486 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.79126670600000004 2023-09-23T21:04:15.486 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:04:15.486 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:04:15.486 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:04:15.486 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:14.244618+0000", 2023-09-23T21:04:15.486 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.8208000000000007e-05 2023-09-23T21:04:15.486 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:04:15.487 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:04:15.487 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 2,7", 2023-09-23T21:04:15.487 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:14.244775+0000", 2023-09-23T21:04:15.487 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000157114 2023-09-23T21:04:15.487 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:04:15.487 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:04:15.487 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:04:15.487 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:14.248447+0000", 2023-09-23T21:04:15.487 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.003671091 2023-09-23T21:04:15.487 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:04:15.487 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:04:15.487 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:04:15.488 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:14.249016+0000", 2023-09-23T21:04:15.488 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00056920099999999997 2023-09-23T21:04:15.488 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:04:15.488 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:04:15.488 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:04:15.488 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:14.249037+0000", 2023-09-23T21:04:15.488 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.0795e-05 2023-09-23T21:04:15.488 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:04:15.488 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:04:15.488 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:04:15.488 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:14.249063+0000", 2023-09-23T21:04:15.488 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.6128000000000001e-05 2023-09-23T21:04:15.489 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:04:15.489 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:04:15.489 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:04:15.489 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:14.249083+0000", 2023-09-23T21:04:15.489 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.0047999999999998e-05 2023-09-23T21:04:15.489 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:04:15.489 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:04:15.489 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:04:15.489 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:04:15.489 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:04:15.489 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.35469.0:164316 8.12 8:4f1a1a72:::benchmark_data_smithi026_301269_object53559:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e307)", 2023-09-23T21:04:15.489 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:04:13.490488+0000", 2023-09-23T21:04:15.489 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 1.875718835, 2023-09-23T21:04:15.490 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.75886759699999995, 2023-09-23T21:04:15.490 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:04:15.490 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:04:15.490 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:04:15.490 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.35469", 2023-09-23T21:04:15.490 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/4041424623", 2023-09-23T21:04:15.490 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 164316 2023-09-23T21:04:15.490 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:04:15.490 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:04:15.490 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:04:15.490 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:04:15.490 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:13.490488+0000", 2023-09-23T21:04:15.491 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:04:15.491 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:04:15.491 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:04:15.491 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:04:15.491 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:13.490488+0000", 2023-09-23T21:04:15.491 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:04:15.491 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:04:15.491 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:04:15.491 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:04:15.491 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:13.490486+0000", 2023-09-23T21:04:15.491 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.999999 2023-09-23T21:04:15.491 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:04:15.492 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:04:15.492 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:04:15.492 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:13.490492+0000", 2023-09-23T21:04:15.492 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.2900000000000002e-06 2023-09-23T21:04:15.492 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:04:15.492 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:04:15.492 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:04:15.492 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:13.490503+0000", 2023-09-23T21:04:15.492 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.1691e-05 2023-09-23T21:04:15.492 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:04:15.492 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:04:15.492 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:04:15.492 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:13.490508+0000", 2023-09-23T21:04:15.493 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.9849999999999997e-06 2023-09-23T21:04:15.493 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:04:15.493 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:04:15.493 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:04:15.493 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:13.490565+0000", 2023-09-23T21:04:15.493 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.6393e-05 2023-09-23T21:04:15.493 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:04:15.493 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:04:15.493 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:04:15.493 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:13.490575+0000", 2023-09-23T21:04:15.493 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.0541e-05 2023-09-23T21:04:15.493 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:04:15.494 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:04:15.494 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:04:15.494 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:14.245708+0000", 2023-09-23T21:04:15.494 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.75513238800000004 2023-09-23T21:04:15.494 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:04:15.494 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:04:15.494 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:04:15.494 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:14.245798+0000", 2023-09-23T21:04:15.494 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.0285999999999996e-05 2023-09-23T21:04:15.494 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:04:15.494 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:04:15.494 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 2,7", 2023-09-23T21:04:15.495 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:14.245936+0000", 2023-09-23T21:04:15.495 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00013843699999999999 2023-09-23T21:04:15.495 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:04:15.495 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:04:15.495 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:04:15.495 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:14.248455+0000", 2023-09-23T21:04:15.495 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0025187590000000002 2023-09-23T21:04:15.495 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:04:15.495 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:04:15.495 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:04:15.495 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:14.249200+0000", 2023-09-23T21:04:15.495 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00074436399999999996 2023-09-23T21:04:15.495 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:04:15.496 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:04:15.496 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:04:15.496 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:14.249278+0000", 2023-09-23T21:04:15.496 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.8651000000000003e-05 2023-09-23T21:04:15.496 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:04:15.496 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:04:15.496 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:04:15.496 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:14.249321+0000", 2023-09-23T21:04:15.496 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.3012000000000001e-05 2023-09-23T21:04:15.496 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:04:15.496 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:04:15.496 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:04:15.497 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:14.249355+0000", 2023-09-23T21:04:15.497 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.3973000000000002e-05 2023-09-23T21:04:15.497 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:04:15.497 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:04:15.497 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:04:15.497 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:04:15.497 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:04:15.497 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.35469.0:164325 8.12 8:49f268c0:::benchmark_data_smithi026_301269_object53568:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e307)", 2023-09-23T21:04:15.497 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:04:13.512173+0000", 2023-09-23T21:04:15.497 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 1.854033684, 2023-09-23T21:04:15.497 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.73732217, 2023-09-23T21:04:15.497 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:04:15.497 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:04:15.498 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:04:15.498 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.35469", 2023-09-23T21:04:15.498 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/4041424623", 2023-09-23T21:04:15.498 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 164325 2023-09-23T21:04:15.498 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:04:15.498 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:04:15.498 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:04:15.498 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:04:15.498 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:13.512173+0000", 2023-09-23T21:04:15.498 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:04:15.498 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:04:15.499 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:04:15.499 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:04:15.499 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:13.512173+0000", 2023-09-23T21:04:15.499 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:04:15.499 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:04:15.499 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:04:15.499 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:04:15.499 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:13.512171+0000", 2023-09-23T21:04:15.499 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T21:04:15.499 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:04:15.499 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:04:15.499 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:04:15.499 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:13.512181+0000", 2023-09-23T21:04:15.500 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.9669999999999996e-06 2023-09-23T21:04:15.500 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:04:15.500 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:04:15.500 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:04:15.500 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:13.512200+0000", 2023-09-23T21:04:15.500 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.8493000000000001e-05 2023-09-23T21:04:15.500 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:04:15.500 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:04:15.500 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:04:15.500 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:13.512207+0000", 2023-09-23T21:04:15.500 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.798e-06 2023-09-23T21:04:15.500 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:04:15.501 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:04:15.501 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:04:15.501 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:13.512264+0000", 2023-09-23T21:04:15.501 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.7809000000000002e-05 2023-09-23T21:04:15.501 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:04:15.501 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:04:15.501 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:04:15.501 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:13.512273+0000", 2023-09-23T21:04:15.501 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.6090000000000001e-06 2023-09-23T21:04:15.501 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:04:15.501 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:04:15.501 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:04:15.502 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:14.247094+0000", 2023-09-23T21:04:15.502 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.73482105499999995 2023-09-23T21:04:15.502 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:04:15.502 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:04:15.502 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:04:15.502 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:14.247205+0000", 2023-09-23T21:04:15.502 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000110849 2023-09-23T21:04:15.502 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:04:15.502 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:04:15.502 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 2,7", 2023-09-23T21:04:15.502 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:14.247437+0000", 2023-09-23T21:04:15.502 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00023193199999999999 2023-09-23T21:04:15.503 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:04:15.503 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:04:15.503 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:04:15.503 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:14.248483+0000", 2023-09-23T21:04:15.503 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0010465509999999999 2023-09-23T21:04:15.503 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:04:15.503 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:04:15.503 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:04:15.503 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:14.249249+0000", 2023-09-23T21:04:15.503 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00076557199999999995 2023-09-23T21:04:15.503 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:04:15.503 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:04:15.503 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:04:15.504 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:14.249416+0000", 2023-09-23T21:04:15.504 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00016682200000000001 2023-09-23T21:04:15.504 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:04:15.504 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:04:15.504 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:04:15.504 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:14.249458+0000", 2023-09-23T21:04:15.504 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.2305999999999998e-05 2023-09-23T21:04:15.504 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:04:15.504 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:04:15.504 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:04:15.504 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:14.249495+0000", 2023-09-23T21:04:15.504 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.6837e-05 2023-09-23T21:04:15.505 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:04:15.505 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:04:15.505 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:04:15.505 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:04:15.505 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:04:15.505 INFO:teuthology.orchestra.run.smithi070.stdout:} 2023-09-23T21:04:15.505 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:04:15.506 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:04:15 smithi026 ceph-mon[175151]: failed to encode map e308 with expected crc 2023-09-23T21:04:15.506 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:04:15 smithi026 ceph-mon[175151]: failed to encode map e308 with expected crc 2023-09-23T21:04:15.507 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:04:15 smithi026 ceph-mon[175151]: failed to encode map e308 with expected crc 2023-09-23T21:04:15.507 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:04:15 smithi026 ceph-mon[175151]: osdmap e308: 8 total, 8 up, 8 in 2023-09-23T21:04:15.507 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:04:15 smithi026 ceph-mon[175151]: failed to encode map e308 with expected crc 2023-09-23T21:04:15.507 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:04:15 smithi026 ceph-mon[175151]: failed to encode map e308 with expected crc 2023-09-23T21:04:15.507 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:04:15 smithi026 ceph-mon[175151]: failed to encode map e308 with expected crc 2023-09-23T21:04:15.507 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:04:15 smithi026 ceph-mon[175151]: failed to encode map e308 with expected crc 2023-09-23T21:04:15.507 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:04:15 smithi026 ceph-mon[175151]: 8.12 scrub starts 2023-09-23T21:04:15.507 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:04:15 smithi070 ceph-mon[168282]: failed to encode map e308 with expected crc 2023-09-23T21:04:15.507 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:04:15 smithi070 ceph-mon[168282]: failed to encode map e308 with expected crc 2023-09-23T21:04:15.508 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:04:15 smithi070 ceph-mon[168282]: failed to encode map e308 with expected crc 2023-09-23T21:04:15.508 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:04:15 smithi070 ceph-mon[168282]: osdmap e308: 8 total, 8 up, 8 in 2023-09-23T21:04:15.508 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:04:15 smithi070 ceph-mon[168282]: failed to encode map e308 with expected crc 2023-09-23T21:04:15.508 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:04:15 smithi070 ceph-mon[168282]: failed to encode map e308 with expected crc 2023-09-23T21:04:15.508 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:04:15 smithi070 ceph-mon[168282]: failed to encode map e308 with expected crc 2023-09-23T21:04:15.508 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:04:15 smithi070 ceph-mon[168282]: failed to encode map e308 with expected crc 2023-09-23T21:04:15.508 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:04:15 smithi070 ceph-mon[168282]: 8.12 scrub starts 2023-09-23T21:04:15.508 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:04:15 smithi026 ceph-mon[179688]: failed to encode map e308 with expected crc 2023-09-23T21:04:15.508 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:04:15 smithi026 ceph-mon[179688]: failed to encode map e308 with expected crc 2023-09-23T21:04:15.508 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:04:15 smithi026 ceph-mon[179688]: failed to encode map e308 with expected crc 2023-09-23T21:04:15.508 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:04:15 smithi026 ceph-mon[179688]: osdmap e308: 8 total, 8 up, 8 in 2023-09-23T21:04:15.509 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:04:15 smithi026 ceph-mon[179688]: failed to encode map e308 with expected crc 2023-09-23T21:04:15.509 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:04:15 smithi026 ceph-mon[179688]: failed to encode map e308 with expected crc 2023-09-23T21:04:15.509 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:04:15 smithi026 ceph-mon[179688]: failed to encode map e308 with expected crc 2023-09-23T21:04:15.509 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:04:15 smithi026 ceph-mon[179688]: failed to encode map e308 with expected crc 2023-09-23T21:04:15.509 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:04:15 smithi026 ceph-mon[179688]: 8.12 scrub starts 2023-09-23T21:04:15.546 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:04:15.913 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:04:15.920 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:04:15.921 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:04:16.022 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:04:16.099 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:04:16.197 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_ops_in_flight 2023-09-23T21:04:16.202 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:04:16.218 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:04:16.218 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:04:16.319 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:04:16.437 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:04:16.594 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:04:16.601 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:04:16 smithi026 ceph-mon[179688]: pgmap v801: 119 pgs: 1 remapped+peering, 1 peering, 1 active+remapped+backfill_wait, 1 active+remapped+backfilling, 115 active+clean; 3.5 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 783 op/s; 2068/173322 objects misplaced (1.193%); 31 MiB/s, 503 objects/s recovering 2023-09-23T21:04:16.601 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:04:16 smithi026 ceph-mon[179688]: failed to encode map e309 with expected crc 2023-09-23T21:04:16.601 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:04:16 smithi026 ceph-mon[179688]: failed to encode map e309 with expected crc 2023-09-23T21:04:16.602 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:04:16 smithi026 ceph-mon[179688]: osdmap e309: 8 total, 8 up, 8 in 2023-09-23T21:04:16.602 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:04:16 smithi026 ceph-mon[179688]: failed to encode map e309 with expected crc 2023-09-23T21:04:16.602 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:04:16 smithi026 ceph-mon[179688]: 8.17 starting backfill to osd.0 from (0'0,0'0] MAX to 305'9169 2023-09-23T21:04:16.602 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:04:16 smithi026 ceph-mon[179688]: 8.17 starting backfill to osd.7 from (0'0,0'0] MAX to 305'9169 2023-09-23T21:04:16.602 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:04:16 smithi026 ceph-mon[179688]: failed to encode map e309 with expected crc 2023-09-23T21:04:16.602 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:04:16 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:04:16.603 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:04:16 smithi026 ceph-mon[175151]: pgmap v801: 119 pgs: 1 remapped+peering, 1 peering, 1 active+remapped+backfill_wait, 1 active+remapped+backfilling, 115 active+clean; 3.5 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 783 op/s; 2068/173322 objects misplaced (1.193%); 31 MiB/s, 503 objects/s recovering 2023-09-23T21:04:16.603 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:04:16 smithi026 ceph-mon[175151]: failed to encode map e309 with expected crc 2023-09-23T21:04:16.603 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:04:16 smithi026 ceph-mon[175151]: failed to encode map e309 with expected crc 2023-09-23T21:04:16.603 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:04:16 smithi026 ceph-mon[175151]: osdmap e309: 8 total, 8 up, 8 in 2023-09-23T21:04:16.603 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:04:16 smithi026 ceph-mon[175151]: failed to encode map e309 with expected crc 2023-09-23T21:04:16.603 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:04:16 smithi026 ceph-mon[175151]: 8.17 starting backfill to osd.0 from (0'0,0'0] MAX to 305'9169 2023-09-23T21:04:16.603 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:04:16 smithi026 ceph-mon[175151]: 8.17 starting backfill to osd.7 from (0'0,0'0] MAX to 305'9169 2023-09-23T21:04:16.603 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:04:16 smithi026 ceph-mon[175151]: failed to encode map e309 with expected crc 2023-09-23T21:04:16.603 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:04:16 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:04:16.603 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:04:16.604 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:04:16.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:04:16 smithi070 ceph-mon[168282]: pgmap v801: 119 pgs: 1 remapped+peering, 1 peering, 1 active+remapped+backfill_wait, 1 active+remapped+backfilling, 115 active+clean; 3.5 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 783 op/s; 2068/173322 objects misplaced (1.193%); 31 MiB/s, 503 objects/s recovering 2023-09-23T21:04:16.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:04:16 smithi070 ceph-mon[168282]: failed to encode map e309 with expected crc 2023-09-23T21:04:16.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:04:16 smithi070 ceph-mon[168282]: failed to encode map e309 with expected crc 2023-09-23T21:04:16.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:04:16 smithi070 ceph-mon[168282]: osdmap e309: 8 total, 8 up, 8 in 2023-09-23T21:04:16.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:04:16 smithi070 ceph-mon[168282]: failed to encode map e309 with expected crc 2023-09-23T21:04:16.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:04:16 smithi070 ceph-mon[168282]: 8.17 starting backfill to osd.0 from (0'0,0'0] MAX to 305'9169 2023-09-23T21:04:16.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:04:16 smithi070 ceph-mon[168282]: 8.17 starting backfill to osd.7 from (0'0,0'0] MAX to 305'9169 2023-09-23T21:04:16.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:04:16 smithi070 ceph-mon[168282]: failed to encode map e309 with expected crc 2023-09-23T21:04:16.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:04:16 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:04:16.705 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:04:16.796 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:04:16.874 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:04:16.882 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:04:16.883 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:04:16.983 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:04:17.078 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:04:17.122 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:04:17.183 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:04:17.200 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:04:17.200 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:04:17.301 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:04:17.334 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:04:17.346 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:04:17.373 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:04:17.377 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:04:17.377 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:04:17.478 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:04:17.559 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:04:17.745 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:04:17.751 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:04:17.752 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:04:17.852 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:04:17.895 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:04:17.958 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T21:04:18.096 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:04:18.100 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:04:18.101 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:04:18.202 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:04:18.229 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:04:18.264 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:04:18.267 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:04:18.267 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:04:18.279 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:04:18 smithi026 ceph-mon[175151]: pgmap v803: 119 pgs: 1 remapped+peering, 1 peering, 1 active+remapped+backfill_wait, 1 active+remapped+backfilling, 115 active+clean; 3.5 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 365 op/s; 2068/171891 objects misplaced (1.203%); 19 MiB/s, 310 objects/s recovering 2023-09-23T21:04:18.279 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:04:18 smithi026 ceph-mon[175151]: failed to encode map e310 with expected crc 2023-09-23T21:04:18.279 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:04:18 smithi026 ceph-mon[175151]: osdmap e310: 8 total, 8 up, 8 in 2023-09-23T21:04:18.279 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:04:18 smithi026 ceph-mon[175151]: failed to encode map e310 with expected crc 2023-09-23T21:04:18.280 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:04:18 smithi026 ceph-mon[175151]: failed to encode map e310 with expected crc 2023-09-23T21:04:18.280 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:04:18 smithi026 ceph-mon[175151]: 8.19 scrub starts 2023-09-23T21:04:18.280 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:04:18 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/2345109466' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-09-23T21:04:18.280 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:04:18 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-09-23T21:04:18.280 INFO:teuthology.orchestra.run.smithi026.stderr:reweighted osd.6 to 1 (10000) 2023-09-23T21:04:18.368 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:04:18.461 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:04:18.565 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:04:18.582 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:04:18.583 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:04:18.583 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:04:18 smithi026 ceph-mon[179688]: pgmap v803: 119 pgs: 1 remapped+peering, 1 peering, 1 active+remapped+backfill_wait, 1 active+remapped+backfilling, 115 active+clean; 3.5 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 365 op/s; 2068/171891 objects misplaced (1.203%); 19 MiB/s, 310 objects/s recovering 2023-09-23T21:04:18.584 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:04:18 smithi026 ceph-mon[179688]: failed to encode map e310 with expected crc 2023-09-23T21:04:18.584 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:04:18 smithi026 ceph-mon[179688]: osdmap e310: 8 total, 8 up, 8 in 2023-09-23T21:04:18.584 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:04:18 smithi026 ceph-mon[179688]: failed to encode map e310 with expected crc 2023-09-23T21:04:18.584 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:04:18 smithi026 ceph-mon[179688]: failed to encode map e310 with expected crc 2023-09-23T21:04:18.584 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:04:18 smithi026 ceph-mon[179688]: 8.19 scrub starts 2023-09-23T21:04:18.584 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:04:18 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/2345109466' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-09-23T21:04:18.584 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:04:18 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-09-23T21:04:18.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:04:18 smithi070 ceph-mon[168282]: pgmap v803: 119 pgs: 1 remapped+peering, 1 peering, 1 active+remapped+backfill_wait, 1 active+remapped+backfilling, 115 active+clean; 3.5 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 365 op/s; 2068/171891 objects misplaced (1.203%); 19 MiB/s, 310 objects/s recovering 2023-09-23T21:04:18.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:04:18 smithi070 ceph-mon[168282]: failed to encode map e310 with expected crc 2023-09-23T21:04:18.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:04:18 smithi070 ceph-mon[168282]: osdmap e310: 8 total, 8 up, 8 in 2023-09-23T21:04:18.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:04:18 smithi070 ceph-mon[168282]: failed to encode map e310 with expected crc 2023-09-23T21:04:18.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:04:18 smithi070 ceph-mon[168282]: failed to encode map e310 with expected crc 2023-09-23T21:04:18.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:04:18 smithi070 ceph-mon[168282]: 8.19 scrub starts 2023-09-23T21:04:18.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:04:18 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/2345109466' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-09-23T21:04:18.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:04:18 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-09-23T21:04:18.684 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:04:18.757 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:04:18.918 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T21:04:18.933 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:04:18.933 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:04:18.959 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph osd reweight 7 1 2023-09-23T21:04:19.034 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:04:19.075 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:04:19.124 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:04:19.127 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:04:19.128 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:04:19.228 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:04:19.282 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:04:19.293 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:04:19 smithi026 ceph-mon[175151]: failed to encode map e311 with expected crc 2023-09-23T21:04:19.293 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:04:19 smithi026 ceph-mon[175151]: failed to encode map e311 with expected crc 2023-09-23T21:04:19.294 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:04:19 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-09-23T21:04:19.294 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:04:19 smithi026 ceph-mon[175151]: osdmap e311: 8 total, 8 up, 8 in 2023-09-23T21:04:19.294 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:04:19 smithi026 ceph-mon[175151]: failed to encode map e311 with expected crc 2023-09-23T21:04:19.294 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:04:19 smithi026 ceph-mon[175151]: failed to encode map e311 with expected crc 2023-09-23T21:04:19.294 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:04:19 smithi026 ceph-mon[175151]: failed to encode map e311 with expected crc 2023-09-23T21:04:19.324 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:04:19.325 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:04:19 smithi070 ceph-mon[168282]: failed to encode map e311 with expected crc 2023-09-23T21:04:19.326 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:04:19 smithi070 ceph-mon[168282]: failed to encode map e311 with expected crc 2023-09-23T21:04:19.326 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:04:19 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-09-23T21:04:19.326 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:04:19 smithi070 ceph-mon[168282]: osdmap e311: 8 total, 8 up, 8 in 2023-09-23T21:04:19.326 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:04:19 smithi070 ceph-mon[168282]: failed to encode map e311 with expected crc 2023-09-23T21:04:19.326 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:04:19 smithi070 ceph-mon[168282]: failed to encode map e311 with expected crc 2023-09-23T21:04:19.326 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:04:19 smithi070 ceph-mon[168282]: failed to encode map e311 with expected crc 2023-09-23T21:04:19.327 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:04:19.327 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:04:19.429 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:04:19.508 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:04:19.594 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:04:19.608 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:04:19.608 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:04:19.608 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:04:19 smithi026 ceph-mon[179688]: failed to encode map e311 with expected crc 2023-09-23T21:04:19.609 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:04:19 smithi026 ceph-mon[179688]: failed to encode map e311 with expected crc 2023-09-23T21:04:19.609 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:04:19 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-09-23T21:04:19.609 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:04:19 smithi026 ceph-mon[179688]: osdmap e311: 8 total, 8 up, 8 in 2023-09-23T21:04:19.609 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:04:19 smithi026 ceph-mon[179688]: failed to encode map e311 with expected crc 2023-09-23T21:04:19.609 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:04:19 smithi026 ceph-mon[179688]: failed to encode map e311 with expected crc 2023-09-23T21:04:19.609 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:04:19 smithi026 ceph-mon[179688]: failed to encode map e311 with expected crc 2023-09-23T21:04:19.696 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:04:19.706 INFO:teuthology.orchestra.run.smithi026.stdout:{ 2023-09-23T21:04:19.706 INFO:teuthology.orchestra.run.smithi026.stdout: "ops": [ 2023-09-23T21:04:19.706 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:19.707 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35469.0:167393 8.16 8:68fe2b8e:::benchmark_data_smithi026_301269_object56636:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e311)", 2023-09-23T21:04:19.707 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T21:04:19.690767+0000", 2023-09-23T21:04:19.707 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 0.0082507980000000002, 2023-09-23T21:04:19.707 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0082753340000000005, 2023-09-23T21:04:19.707 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:04:19.707 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "waiting for sub ops", 2023-09-23T21:04:19.707 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T21:04:19.707 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35469", 2023-09-23T21:04:19.707 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/4041424623", 2023-09-23T21:04:19.707 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 167393 2023-09-23T21:04:19.707 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:19.707 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:04:19.708 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:19.708 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:04:19.708 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:19.690767+0000", 2023-09-23T21:04:19.708 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:04:19.708 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:19.708 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:19.708 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:04:19.708 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:19.690767+0000", 2023-09-23T21:04:19.708 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:04:19.708 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:19.708 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:19.708 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:04:19.709 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:19.690766+0000", 2023-09-23T21:04:19.709 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.999999 2023-09-23T21:04:19.709 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:19.709 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:19.709 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:04:19.709 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:19.690771+0000", 2023-09-23T21:04:19.709 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.8709999999999996e-06 2023-09-23T21:04:19.709 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:19.709 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:19.709 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:04:19.709 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:19.690780+0000", 2023-09-23T21:04:19.709 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.0690000000000008e-06 2023-09-23T21:04:19.710 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:19.710 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:19.710 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:04:19.710 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:19.690787+0000", 2023-09-23T21:04:19.710 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.9099999999999999e-06 2023-09-23T21:04:19.710 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:19.710 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:19.710 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:04:19.710 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:19.691023+0000", 2023-09-23T21:04:19.710 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00023564500000000001 2023-09-23T21:04:19.710 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:19.710 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:19.711 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:04:19.711 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:19.691227+0000", 2023-09-23T21:04:19.711 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00020383399999999999 2023-09-23T21:04:19.711 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:19.711 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:19.711 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 2,3,5,6,7", 2023-09-23T21:04:19.711 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:19.691717+0000", 2023-09-23T21:04:19.711 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00049044699999999997 2023-09-23T21:04:19.711 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:04:19.711 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:04:19.712 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:04:19.712 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:19.712 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:19.712 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35469.0:167398 8.c 8:3bf0cd67:::benchmark_data_smithi026_301269_object56641:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e311)", 2023-09-23T21:04:19.712 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T21:04:19.691267+0000", 2023-09-23T21:04:19.712 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 0.0077507720000000004, 2023-09-23T21:04:19.712 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0078281070000000008, 2023-09-23T21:04:19.712 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:04:19.712 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "waiting for sub ops", 2023-09-23T21:04:19.712 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T21:04:19.712 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35469", 2023-09-23T21:04:19.712 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/4041424623", 2023-09-23T21:04:19.713 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 167398 2023-09-23T21:04:19.713 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:19.713 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:04:19.713 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:19.713 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:04:19.713 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:19.691267+0000", 2023-09-23T21:04:19.713 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:04:19.713 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:19.713 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:19.713 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:04:19.713 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:19.691267+0000", 2023-09-23T21:04:19.713 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:04:19.714 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:19.714 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:19.714 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:04:19.714 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:19.691267+0000", 2023-09-23T21:04:19.714 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.999999 2023-09-23T21:04:19.714 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:19.714 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:19.714 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:04:19.714 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:19.691272+0000", 2023-09-23T21:04:19.714 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.8250000000000001e-06 2023-09-23T21:04:19.714 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:19.715 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:19.715 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:04:19.715 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:19.691284+0000", 2023-09-23T21:04:19.715 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1466e-05 2023-09-23T21:04:19.715 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:19.715 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:19.715 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:04:19.715 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:19.691292+0000", 2023-09-23T21:04:19.715 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.2279999999999998e-06 2023-09-23T21:04:19.715 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:19.715 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:19.715 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:04:19.716 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:19.691343+0000", 2023-09-23T21:04:19.716 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.0501999999999999e-05 2023-09-23T21:04:19.716 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:19.716 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:19.716 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:04:19.716 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:19.691457+0000", 2023-09-23T21:04:19.716 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000114807 2023-09-23T21:04:19.716 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:19.716 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:19.716 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 3,7", 2023-09-23T21:04:19.716 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:19.691598+0000", 2023-09-23T21:04:19.716 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00014100399999999999 2023-09-23T21:04:19.717 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:19.717 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:19.717 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T21:04:19.717 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:19.697798+0000", 2023-09-23T21:04:19.717 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0061995059999999996 2023-09-23T21:04:19.717 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:19.717 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:19.717 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:04:19.717 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:19.698258+0000", 2023-09-23T21:04:19.717 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00046016399999999999 2023-09-23T21:04:19.717 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:04:19.717 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:04:19.717 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:04:19.718 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:19.718 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:19.718 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "MOSDPGPushReply(8.16 311/306 [PushReplyOp(8:6d887fb0:::benchmark_data_smithi026_301269_object66815:head)])", 2023-09-23T21:04:19.718 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T21:04:19.691859+0000", 2023-09-23T21:04:19.718 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 0.0071594500000000004, 2023-09-23T21:04:19.718 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0073101340000000002, 2023-09-23T21:04:19.718 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:04:19.718 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "queued for pg", 2023-09-23T21:04:19.718 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:04:19.718 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:19.718 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:04:19.719 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:19.691859+0000", 2023-09-23T21:04:19.719 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:04:19.719 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:19.719 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:19.719 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:04:19.719 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:19.691859+0000", 2023-09-23T21:04:19.719 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:04:19.719 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:19.719 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:19.719 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:04:19.719 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:19.691858+0000", 2023-09-23T21:04:19.719 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T21:04:19.720 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:19.720 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:19.720 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:04:19.720 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:19.691863+0000", 2023-09-23T21:04:19.720 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.464e-06 2023-09-23T21:04:19.720 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:19.720 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:19.720 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:04:19.720 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:19.691871+0000", 2023-09-23T21:04:19.720 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.9330000000000001e-06 2023-09-23T21:04:19.720 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:19.720 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:19.720 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:04:19.721 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:19.691878+0000", 2023-09-23T21:04:19.721 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.2819999999999999e-06 2023-09-23T21:04:19.721 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:04:19.721 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:04:19.721 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:04:19.721 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:19.721 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:19.721 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_repop_reply(client.35469.0:167393 8.16 e311/306)", 2023-09-23T21:04:19.721 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T21:04:19.696524+0000", 2023-09-23T21:04:19.721 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 0.0024942509999999998, 2023-09-23T21:04:19.721 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0026958590000000001, 2023-09-23T21:04:19.721 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:04:19.722 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "queued for pg", 2023-09-23T21:04:19.722 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:04:19.722 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:19.722 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:04:19.722 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:19.696524+0000", 2023-09-23T21:04:19.722 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:04:19.722 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:19.722 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:19.722 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:04:19.722 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:19.696524+0000", 2023-09-23T21:04:19.722 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:04:19.722 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:19.723 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:19.723 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:04:19.723 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:19.696523+0000", 2023-09-23T21:04:19.723 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.999999 2023-09-23T21:04:19.723 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:19.723 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:19.723 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:04:19.723 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:19.696530+0000", 2023-09-23T21:04:19.723 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.6619999999999999e-06 2023-09-23T21:04:19.723 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:19.723 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:19.723 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:04:19.724 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:19.696542+0000", 2023-09-23T21:04:19.724 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.224e-05 2023-09-23T21:04:19.724 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:19.724 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:19.724 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:04:19.724 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:19.696549+0000", 2023-09-23T21:04:19.724 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.2119999999999997e-06 2023-09-23T21:04:19.724 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:04:19.724 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:04:19.724 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:04:19.724 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:19.724 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:19.725 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_repop_reply(client.35469.0:167393 8.16 e311/306)", 2023-09-23T21:04:19.725 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T21:04:19.697577+0000", 2023-09-23T21:04:19.725 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 0.0014409550000000001, 2023-09-23T21:04:19.725 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.001696316, 2023-09-23T21:04:19.725 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:04:19.725 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "queued for pg", 2023-09-23T21:04:19.725 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:04:19.725 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:19.725 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:04:19.725 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:19.697577+0000", 2023-09-23T21:04:19.725 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:04:19.725 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:19.725 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:19.726 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:04:19.726 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:19.697577+0000", 2023-09-23T21:04:19.726 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:04:19.726 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:19.726 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:19.726 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:04:19.726 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:19.697577+0000", 2023-09-23T21:04:19.726 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T21:04:19.726 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:19.726 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:19.726 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:04:19.726 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:19.697581+0000", 2023-09-23T21:04:19.727 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.5789999999999997e-06 2023-09-23T21:04:19.727 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:19.727 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:19.727 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:04:19.727 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:19.697593+0000", 2023-09-23T21:04:19.727 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1848000000000001e-05 2023-09-23T21:04:19.727 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:19.727 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:19.727 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:04:19.727 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:19.697600+0000", 2023-09-23T21:04:19.727 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.3180000000000001e-06 2023-09-23T21:04:19.727 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:04:19.728 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:04:19.728 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:04:19.728 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:19.728 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:19.728 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "MOSDPGPushReply(8.16 311/306 [PushReplyOp(8:6d887fb0:::benchmark_data_smithi026_301269_object66815:head)])", 2023-09-23T21:04:19.728 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T21:04:19.697642+0000", 2023-09-23T21:04:19.728 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 0.001375813, 2023-09-23T21:04:19.728 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.001682879, 2023-09-23T21:04:19.728 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:04:19.728 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "queued for pg", 2023-09-23T21:04:19.728 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:04:19.728 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:19.728 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:04:19.729 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:19.697642+0000", 2023-09-23T21:04:19.729 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:04:19.729 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:19.729 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:19.729 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:04:19.729 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:19.697642+0000", 2023-09-23T21:04:19.729 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:04:19.729 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:19.729 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:19.729 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:04:19.729 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:19.697642+0000", 2023-09-23T21:04:19.729 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967296 2023-09-23T21:04:19.730 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:19.730 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:19.730 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:04:19.730 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:19.697646+0000", 2023-09-23T21:04:19.730 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.2389999999999998e-06 2023-09-23T21:04:19.730 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:19.730 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:19.730 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:04:19.730 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:19.697652+0000", 2023-09-23T21:04:19.730 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.4450000000000001e-06 2023-09-23T21:04:19.730 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:19.730 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:19.731 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:04:19.731 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:19.697657+0000", 2023-09-23T21:04:19.731 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.4929999999999998e-06 2023-09-23T21:04:19.731 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:04:19.731 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:04:19.731 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:04:19.731 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:19.731 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:19.731 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_repop_reply(client.35469.0:167393 8.16 e311/306)", 2023-09-23T21:04:19.731 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T21:04:19.697676+0000", 2023-09-23T21:04:19.731 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 0.001342092, 2023-09-23T21:04:19.731 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0016982760000000001, 2023-09-23T21:04:19.731 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:04:19.732 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "queued for pg", 2023-09-23T21:04:19.732 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:04:19.732 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:19.732 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:04:19.732 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:19.697676+0000", 2023-09-23T21:04:19.732 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:04:19.732 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:19.732 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:19.732 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:04:19.732 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:19.697676+0000", 2023-09-23T21:04:19.732 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:04:19.732 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:19.732 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:19.733 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:04:19.733 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:19.697676+0000", 2023-09-23T21:04:19.733 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T21:04:19.733 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:19.733 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:19.733 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:04:19.733 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:19.697679+0000", 2023-09-23T21:04:19.733 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.0299999999999998e-06 2023-09-23T21:04:19.733 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:19.733 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:19.733 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:04:19.733 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:19.697685+0000", 2023-09-23T21:04:19.734 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.5950000000000004e-06 2023-09-23T21:04:19.734 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:19.734 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:19.734 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:04:19.734 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:19.697690+0000", 2023-09-23T21:04:19.734 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.4599999999999996e-06 2023-09-23T21:04:19.734 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:04:19.734 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:04:19.734 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:04:19.734 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:19.734 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:19.734 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_repop_reply(client.35469.0:167393 8.16 e311/306)", 2023-09-23T21:04:19.734 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T21:04:19.697974+0000", 2023-09-23T21:04:19.735 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 0.0010440250000000001, 2023-09-23T21:04:19.735 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.001431821, 2023-09-23T21:04:19.735 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:04:19.735 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "queued for pg", 2023-09-23T21:04:19.735 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:04:19.735 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:19.735 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:04:19.735 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:19.697974+0000", 2023-09-23T21:04:19.735 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:04:19.735 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:19.735 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:19.735 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:04:19.736 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:19.697974+0000", 2023-09-23T21:04:19.736 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:04:19.736 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:19.736 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:19.736 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:04:19.736 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:19.697974+0000", 2023-09-23T21:04:19.736 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T21:04:19.736 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:19.736 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:19.736 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:04:19.736 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:19.697978+0000", 2023-09-23T21:04:19.736 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.3170000000000003e-06 2023-09-23T21:04:19.737 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:19.737 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:19.737 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:04:19.737 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:19.697989+0000", 2023-09-23T21:04:19.737 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0811e-05 2023-09-23T21:04:19.737 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:19.737 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:19.737 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:04:19.737 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:19.697996+0000", 2023-09-23T21:04:19.737 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.4229999999999999e-06 2023-09-23T21:04:19.737 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:04:19.737 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:04:19.738 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:04:19.738 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:19.738 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:19.738 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35469.0:167405 8.6 8.5634e086 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e311)", 2023-09-23T21:04:19.738 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T21:04:19.698232+0000", 2023-09-23T21:04:19.738 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 0.00078651999999999997, 2023-09-23T21:04:19.738 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0012021499999999999, 2023-09-23T21:04:19.738 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:04:19.738 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "queued for pg", 2023-09-23T21:04:19.738 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T21:04:19.738 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35469", 2023-09-23T21:04:19.738 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/4041424623", 2023-09-23T21:04:19.738 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 167405 2023-09-23T21:04:19.739 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:19.739 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:04:19.739 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:19.739 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:04:19.739 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:19.698232+0000", 2023-09-23T21:04:19.739 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:04:19.739 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:19.739 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:19.739 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:04:19.739 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:19.698232+0000", 2023-09-23T21:04:19.739 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:04:19.740 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:19.740 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:19.740 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:04:19.740 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:19.698230+0000", 2023-09-23T21:04:19.740 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T21:04:19.740 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:19.740 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:19.740 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:04:19.740 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:19.698235+0000", 2023-09-23T21:04:19.740 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.344e-06 2023-09-23T21:04:19.740 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:19.740 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:19.740 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:04:19.741 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:19.698245+0000", 2023-09-23T21:04:19.741 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.2380000000000003e-06 2023-09-23T21:04:19.741 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:19.741 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:19.741 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:04:19.741 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:19.698250+0000", 2023-09-23T21:04:19.741 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.7440000000000004e-06 2023-09-23T21:04:19.741 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:04:19.741 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:04:19.741 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:04:19.741 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:19.741 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:19.742 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35469.0:167406 8.e 8.c9b4c49e (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e311)", 2023-09-23T21:04:19.742 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T21:04:19.698294+0000", 2023-09-23T21:04:19.742 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 0.00072456500000000004, 2023-09-23T21:04:19.742 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0011741900000000001, 2023-09-23T21:04:19.742 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:04:19.742 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "queued for pg", 2023-09-23T21:04:19.742 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T21:04:19.742 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35469", 2023-09-23T21:04:19.742 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/4041424623", 2023-09-23T21:04:19.742 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 167406 2023-09-23T21:04:19.742 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:19.742 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:04:19.742 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:19.743 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:04:19.743 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:19.698294+0000", 2023-09-23T21:04:19.743 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:04:19.743 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:19.743 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:19.743 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:04:19.743 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:19.698294+0000", 2023-09-23T21:04:19.743 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:04:19.743 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:19.743 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:19.743 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:04:19.743 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:19.698293+0000", 2023-09-23T21:04:19.744 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T21:04:19.744 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:19.744 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:19.744 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:04:19.744 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:19.698297+0000", 2023-09-23T21:04:19.744 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.4180000000000001e-06 2023-09-23T21:04:19.744 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:19.744 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:19.744 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:04:19.744 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:19.698305+0000", 2023-09-23T21:04:19.744 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.0399999999999993e-06 2023-09-23T21:04:19.744 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:19.745 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:19.745 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:04:19.745 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:19.698310+0000", 2023-09-23T21:04:19.745 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.626e-06 2023-09-23T21:04:19.745 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:04:19.745 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:04:19.745 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:04:19.745 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:04:19.745 INFO:teuthology.orchestra.run.smithi026.stdout: ], 2023-09-23T21:04:19.745 INFO:teuthology.orchestra.run.smithi026.stdout: "num_ops": 10 2023-09-23T21:04:19.745 INFO:teuthology.orchestra.run.smithi026.stdout:} 2023-09-23T21:04:19.746 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:04:19.766 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0: {} 2023-09-23T21:04:19.826 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1: {} 2023-09-23T21:04:19.859 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:04:19.876 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2: {} 2023-09-23T21:04:19.884 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 21:04:19 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[164029]: ::ffff:172.21.15.70 - - [23/Sep/2023:21:04:19] "GET /metrics HTTP/1.1" 200 36316 "" "Prometheus/2.43.0" 2023-09-23T21:04:19.932 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3: {} 2023-09-23T21:04:19.970 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4: {} 2023-09-23T21:04:20.022 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T21:04:20.043 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6: {} 2023-09-23T21:04:20.093 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T21:04:20.103 INFO:teuthology.orchestra.run.smithi026.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T21:04:20.103 INFO:teuthology.orchestra.run.smithi026.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T21:04:20.103 INFO:teuthology.orchestra.run.smithi026.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T21:04:20.103 INFO:teuthology.orchestra.run.smithi026.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T21:04:20.103 INFO:teuthology.orchestra.run.smithi026.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T21:04:20.103 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T21:04:20.103 INFO:teuthology.orchestra.run.smithi026.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T21:04:20.103 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T21:04:20.346 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_blocked_ops 2023-09-23T21:04:20.402 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:04:20.406 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:04:20.406 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:04:20.507 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:04:20.608 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:04:20 smithi026 ceph-mon[175151]: pgmap v806: 119 pgs: 1 remapped+peering, 1 peering, 1 active+remapped+backfill_wait, 1 active+remapped+backfilling, 115 active+clean; 3.4 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 408 op/s; 4302/169212 objects misplaced (2.542%); 17 MiB/s, 277 objects/s recovering 2023-09-23T21:04:20.608 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:04:20 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/3244339176' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:04:20.610 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:04:20 smithi026 ceph-mon[179688]: pgmap v806: 119 pgs: 1 remapped+peering, 1 peering, 1 active+remapped+backfill_wait, 1 active+remapped+backfilling, 115 active+clean; 3.4 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 408 op/s; 4302/169212 objects misplaced (2.542%); 17 MiB/s, 277 objects/s recovering 2023-09-23T21:04:20.610 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:04:20 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/3244339176' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:04:20.619 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 21:04:20 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[153130]: logger=cleanup t=2023-09-23T21:04:20.213278058Z level=info msg="Completed cleanup jobs" duration=12.923197ms 2023-09-23T21:04:20.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:04:20 smithi070 ceph-mon[168282]: pgmap v806: 119 pgs: 1 remapped+peering, 1 peering, 1 active+remapped+backfill_wait, 1 active+remapped+backfilling, 115 active+clean; 3.4 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 408 op/s; 4302/169212 objects misplaced (2.542%); 17 MiB/s, 277 objects/s recovering 2023-09-23T21:04:20.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:04:20 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/3244339176' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:04:20.624 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:04:20.703 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:04:20.709 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:04:20.710 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:04:20.747 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:04:20.811 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:04:20.925 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:04:21.059 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T21:04:21.076 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:04:21.076 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:04:21.177 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:04:21.258 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:04:21.383 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:04:21 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pgp_num_actual", "val": "25"}]: dispatch 2023-09-23T21:04:21.384 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:04:21 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pgp_num_actual", "val": "25"}]: dispatch 2023-09-23T21:04:21.450 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:04:21.456 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:04:21.456 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:04:21.557 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:04:21.591 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:04:21 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pgp_num_actual", "val": "25"}]: dispatch 2023-09-23T21:04:21.595 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:04:21.639 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:04:21.643 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:04:21.643 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:04:21.744 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:04:21.773 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:04:21.811 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:04:21.814 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:04:21.814 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:04:21.915 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:04:21.977 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:04:22.127 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:04:22.127 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:04:22.131 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:04:22.132 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:04:22.232 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:04:22.280 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:04:22.368 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:04:22.372 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:04:22.373 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:04:22.473 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:04:22.499 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:04:22.533 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:04:22.535 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:04:22.536 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:04:22.540 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:04:22 smithi026 ceph-mon[175151]: pgmap v807: 119 pgs: 2 active+remapped+backfilling, 117 active+clean; 3.3 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 479 op/s; 4899/164694 objects misplaced (2.975%); 37 MiB/s, 595 objects/s recovering 2023-09-23T21:04:22.540 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:04:22 smithi026 ceph-mon[175151]: 8.2 scrub starts 2023-09-23T21:04:22.540 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:04:22 smithi026 ceph-mon[175151]: failed to encode map e312 with expected crc 2023-09-23T21:04:22.540 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:04:22 smithi026 ceph-mon[175151]: failed to encode map e312 with expected crc 2023-09-23T21:04:22.540 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:04:22 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pgp_num_actual", "val": "25"}]': finished 2023-09-23T21:04:22.540 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:04:22 smithi026 ceph-mon[175151]: osdmap e312: 8 total, 8 up, 8 in 2023-09-23T21:04:22.541 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:04:22 smithi026 ceph-mon[175151]: failed to encode map e312 with expected crc 2023-09-23T21:04:22.541 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:04:22 smithi026 ceph-mon[179688]: pgmap v807: 119 pgs: 2 active+remapped+backfilling, 117 active+clean; 3.3 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 479 op/s; 4899/164694 objects misplaced (2.975%); 37 MiB/s, 595 objects/s recovering 2023-09-23T21:04:22.541 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:04:22 smithi026 ceph-mon[179688]: 8.2 scrub starts 2023-09-23T21:04:22.541 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:04:22 smithi026 ceph-mon[179688]: failed to encode map e312 with expected crc 2023-09-23T21:04:22.541 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:04:22 smithi026 ceph-mon[179688]: failed to encode map e312 with expected crc 2023-09-23T21:04:22.541 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:04:22 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pgp_num_actual", "val": "25"}]': finished 2023-09-23T21:04:22.541 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:04:22 smithi026 ceph-mon[179688]: osdmap e312: 8 total, 8 up, 8 in 2023-09-23T21:04:22.541 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:04:22 smithi026 ceph-mon[179688]: failed to encode map e312 with expected crc 2023-09-23T21:04:22.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:04:22 smithi070 ceph-mon[168282]: pgmap v807: 119 pgs: 2 active+remapped+backfilling, 117 active+clean; 3.3 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 479 op/s; 4899/164694 objects misplaced (2.975%); 37 MiB/s, 595 objects/s recovering 2023-09-23T21:04:22.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:04:22 smithi070 ceph-mon[168282]: 8.2 scrub starts 2023-09-23T21:04:22.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:04:22 smithi070 ceph-mon[168282]: failed to encode map e312 with expected crc 2023-09-23T21:04:22.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:04:22 smithi070 ceph-mon[168282]: failed to encode map e312 with expected crc 2023-09-23T21:04:22.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:04:22 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pgp_num_actual", "val": "25"}]': finished 2023-09-23T21:04:22.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:04:22 smithi070 ceph-mon[168282]: osdmap e312: 8 total, 8 up, 8 in 2023-09-23T21:04:22.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:04:22 smithi070 ceph-mon[168282]: failed to encode map e312 with expected crc 2023-09-23T21:04:22.637 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:04:22.663 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:04:22.697 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:04:22.699 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:04:22.700 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:04:22.800 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:04:22.848 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:04:22.856 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T21:04:22.957 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T21:04:22.960 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:04:22.960 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:04:23.061 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:04:23.086 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:04:23.120 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:04:23.123 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:04:23.123 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:04:23.224 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:04:23.250 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:04:23.285 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:04:23.288 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:04:23.288 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:04:23.334 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:04:23 smithi070 ceph-mon[168282]: failed to encode map e313 with expected crc 2023-09-23T21:04:23.334 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:04:23 smithi070 ceph-mon[168282]: failed to encode map e313 with expected crc 2023-09-23T21:04:23.334 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:04:23 smithi070 ceph-mon[168282]: osdmap e313: 8 total, 8 up, 8 in 2023-09-23T21:04:23.334 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:04:23 smithi070 ceph-mon[168282]: failed to encode map e313 with expected crc 2023-09-23T21:04:23.334 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:04:23 smithi070 ceph-mon[168282]: failed to encode map e313 with expected crc 2023-09-23T21:04:23.335 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:04:23 smithi070 ceph-mon[168282]: failed to encode map e313 with expected crc 2023-09-23T21:04:23.335 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:04:23 smithi070 ceph-mon[168282]: failed to encode map e313 with expected crc 2023-09-23T21:04:23.335 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:04:23 smithi070 ceph-mon[168282]: failed to encode map e313 with expected crc 2023-09-23T21:04:23.335 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:04:23 smithi070 ceph-mon[168282]: failed to encode map e313 with expected crc 2023-09-23T21:04:23.335 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:04:23 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/1738761300' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:04:23.335 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:04:23 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:04:23.335 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:04:23 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pgp_num_actual", "val": "26"}]: dispatch 2023-09-23T21:04:23.335 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:04:23 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/2242939925' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-09-23T21:04:23.335 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:04:23 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-09-23T21:04:23.358 INFO:teuthology.orchestra.run.smithi026.stderr:reweighted osd.7 to 1 (10000) 2023-09-23T21:04:23.363 INFO:teuthology.orchestra.run.smithi026.stderr:nodeep-scrub is unset 2023-09-23T21:04:23.389 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:04:23.425 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:04:23.580 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:04:23 smithi026 ceph-mon[175151]: failed to encode map e313 with expected crc 2023-09-23T21:04:23.581 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:04:23 smithi026 ceph-mon[175151]: failed to encode map e313 with expected crc 2023-09-23T21:04:23.581 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:04:23 smithi026 ceph-mon[175151]: osdmap e313: 8 total, 8 up, 8 in 2023-09-23T21:04:23.581 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:04:23 smithi026 ceph-mon[175151]: failed to encode map e313 with expected crc 2023-09-23T21:04:23.581 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:04:23 smithi026 ceph-mon[175151]: failed to encode map e313 with expected crc 2023-09-23T21:04:23.581 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:04:23 smithi026 ceph-mon[175151]: failed to encode map e313 with expected crc 2023-09-23T21:04:23.581 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:04:23 smithi026 ceph-mon[175151]: failed to encode map e313 with expected crc 2023-09-23T21:04:23.581 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:04:23 smithi026 ceph-mon[175151]: failed to encode map e313 with expected crc 2023-09-23T21:04:23.581 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:04:23 smithi026 ceph-mon[175151]: failed to encode map e313 with expected crc 2023-09-23T21:04:23.581 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:04:23 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/1738761300' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:04:23.581 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:04:23 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:04:23.582 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:04:23 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pgp_num_actual", "val": "26"}]: dispatch 2023-09-23T21:04:23.582 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:04:23 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/2242939925' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-09-23T21:04:23.582 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:04:23 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-09-23T21:04:23.582 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:04:23 smithi026 ceph-mon[179688]: failed to encode map e313 with expected crc 2023-09-23T21:04:23.582 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:04:23 smithi026 ceph-mon[179688]: failed to encode map e313 with expected crc 2023-09-23T21:04:23.582 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:04:23 smithi026 ceph-mon[179688]: osdmap e313: 8 total, 8 up, 8 in 2023-09-23T21:04:23.582 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:04:23 smithi026 ceph-mon[179688]: failed to encode map e313 with expected crc 2023-09-23T21:04:23.582 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:04:23 smithi026 ceph-mon[179688]: failed to encode map e313 with expected crc 2023-09-23T21:04:23.582 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:04:23 smithi026 ceph-mon[179688]: failed to encode map e313 with expected crc 2023-09-23T21:04:23.582 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:04:23 smithi026 ceph-mon[179688]: failed to encode map e313 with expected crc 2023-09-23T21:04:23.583 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:04:23 smithi026 ceph-mon[179688]: failed to encode map e313 with expected crc 2023-09-23T21:04:23.583 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:04:23 smithi026 ceph-mon[179688]: failed to encode map e313 with expected crc 2023-09-23T21:04:23.583 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:04:23 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/1738761300' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:04:23.583 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:04:23 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:04:23.583 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:04:23 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pgp_num_actual", "val": "26"}]: dispatch 2023-09-23T21:04:23.583 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:04:23 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/2242939925' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-09-23T21:04:23.583 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:04:23 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-09-23T21:04:24.179 INFO:teuthology.orchestra.run.smithi026.stdout:ERROR: (22) Invalid argument 2023-09-23T21:04:24.179 INFO:teuthology.orchestra.run.smithi026.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T21:04:24.214 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph osd reweight 4 1 2023-09-23T21:04:24.242 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:04:24.245 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:04:24.245 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:04:24.346 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:04:24.373 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:04:24.413 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:04:24.416 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:04:24.417 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:04:24.460 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:04:24 smithi026 ceph-mon[179688]: pgmap v810: 119 pgs: 2 active+remapped+backfilling, 117 active+clean; 3.3 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 562 op/s; 3051/160233 objects misplaced (1.904%); 36 MiB/s, 579 objects/s recovering 2023-09-23T21:04:24.460 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:04:24 smithi026 ceph-mon[179688]: 8.14 scrub starts 2023-09-23T21:04:24.460 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:04:24 smithi026 ceph-mon[179688]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T21:04:24.460 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:04:24 smithi026 ceph-mon[179688]: Cluster is now healthy 2023-09-23T21:04:24.461 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:04:24 smithi026 ceph-mon[179688]: failed to encode map e314 with expected crc 2023-09-23T21:04:24.461 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:04:24 smithi026 ceph-mon[179688]: failed to encode map e314 with expected crc 2023-09-23T21:04:24.461 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:04:24 smithi026 ceph-mon[179688]: failed to encode map e314 with expected crc 2023-09-23T21:04:24.461 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:04:24 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T21:04:24.461 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:04:24 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pgp_num_actual", "val": "26"}]': finished 2023-09-23T21:04:24.461 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:04:24 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-09-23T21:04:24.461 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:04:24 smithi026 ceph-mon[179688]: osdmap e314: 8 total, 8 up, 8 in 2023-09-23T21:04:24.461 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:04:24 smithi026 ceph-mon[179688]: 8.18 starting backfill to osd.4 from (0'0,0'0] MAX to 311'9471 2023-09-23T21:04:24.461 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:04:24 smithi026 ceph-mon[179688]: failed to encode map e314 with expected crc 2023-09-23T21:04:24.462 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:04:24 smithi026 ceph-mon[175151]: pgmap v810: 119 pgs: 2 active+remapped+backfilling, 117 active+clean; 3.3 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 562 op/s; 3051/160233 objects misplaced (1.904%); 36 MiB/s, 579 objects/s recovering 2023-09-23T21:04:24.462 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:04:24 smithi026 ceph-mon[175151]: 8.14 scrub starts 2023-09-23T21:04:24.462 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:04:24 smithi026 ceph-mon[175151]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T21:04:24.462 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:04:24 smithi026 ceph-mon[175151]: Cluster is now healthy 2023-09-23T21:04:24.462 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:04:24 smithi026 ceph-mon[175151]: failed to encode map e314 with expected crc 2023-09-23T21:04:24.462 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:04:24 smithi026 ceph-mon[175151]: failed to encode map e314 with expected crc 2023-09-23T21:04:24.462 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:04:24 smithi026 ceph-mon[175151]: failed to encode map e314 with expected crc 2023-09-23T21:04:24.462 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:04:24 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T21:04:24.462 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:04:24 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pgp_num_actual", "val": "26"}]': finished 2023-09-23T21:04:24.463 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:04:24 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-09-23T21:04:24.463 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:04:24 smithi026 ceph-mon[175151]: osdmap e314: 8 total, 8 up, 8 in 2023-09-23T21:04:24.463 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:04:24 smithi026 ceph-mon[175151]: 8.18 starting backfill to osd.4 from (0'0,0'0] MAX to 311'9471 2023-09-23T21:04:24.463 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:04:24 smithi026 ceph-mon[175151]: failed to encode map e314 with expected crc 2023-09-23T21:04:24.517 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:04:24.543 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:04:24.576 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:04:24.578 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:04:24.579 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:04:24.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:04:24 smithi070 ceph-mon[168282]: pgmap v810: 119 pgs: 2 active+remapped+backfilling, 117 active+clean; 3.3 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 562 op/s; 3051/160233 objects misplaced (1.904%); 36 MiB/s, 579 objects/s recovering 2023-09-23T21:04:24.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:04:24 smithi070 ceph-mon[168282]: 8.14 scrub starts 2023-09-23T21:04:24.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:04:24 smithi070 ceph-mon[168282]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T21:04:24.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:04:24 smithi070 ceph-mon[168282]: Cluster is now healthy 2023-09-23T21:04:24.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:04:24 smithi070 ceph-mon[168282]: failed to encode map e314 with expected crc 2023-09-23T21:04:24.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:04:24 smithi070 ceph-mon[168282]: failed to encode map e314 with expected crc 2023-09-23T21:04:24.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:04:24 smithi070 ceph-mon[168282]: failed to encode map e314 with expected crc 2023-09-23T21:04:24.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:04:24 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T21:04:24.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:04:24 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pgp_num_actual", "val": "26"}]': finished 2023-09-23T21:04:24.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:04:24 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-09-23T21:04:24.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:04:24 smithi070 ceph-mon[168282]: osdmap e314: 8 total, 8 up, 8 in 2023-09-23T21:04:24.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:04:24 smithi070 ceph-mon[168282]: 8.18 starting backfill to osd.4 from (0'0,0'0] MAX to 311'9471 2023-09-23T21:04:24.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:04:24 smithi070 ceph-mon[168282]: failed to encode map e314 with expected crc 2023-09-23T21:04:24.680 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:04:24.706 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:04:24.712 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:04:24.740 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:04:24.742 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:04:24.743 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:04:24.756 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_historic_ops 2023-09-23T21:04:24.843 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:04:24.875 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:04:24.970 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T21:04:24.976 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:04:24.977 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:04:25.078 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:04:25.104 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:04:25.138 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:04:25.141 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:04:25.141 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:04:25.168 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:04:25.242 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:04:25.277 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:04:25.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:04:25 smithi070 ceph-mon[168282]: failed to encode map e315 with expected crc 2023-09-23T21:04:25.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:04:25 smithi070 ceph-mon[168282]: failed to encode map e315 with expected crc 2023-09-23T21:04:25.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:04:25 smithi070 ceph-mon[168282]: osdmap e315: 8 total, 8 up, 8 in 2023-09-23T21:04:25.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:04:25 smithi070 ceph-mon[168282]: failed to encode map e315 with expected crc 2023-09-23T21:04:25.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:04:25 smithi070 ceph-mon[168282]: failed to encode map e315 with expected crc 2023-09-23T21:04:25.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:04:25 smithi070 ceph-mon[168282]: failed to encode map e315 with expected crc 2023-09-23T21:04:25.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:04:25 smithi070 ceph-mon[168282]: failed to encode map e315 with expected crc 2023-09-23T21:04:25.375 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T21:04:25.382 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:04:25.383 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:04:25.426 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:04:25 smithi026 ceph-mon[179688]: failed to encode map e315 with expected crc 2023-09-23T21:04:25.426 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:04:25 smithi026 ceph-mon[179688]: failed to encode map e315 with expected crc 2023-09-23T21:04:25.426 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:04:25 smithi026 ceph-mon[179688]: osdmap e315: 8 total, 8 up, 8 in 2023-09-23T21:04:25.426 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:04:25 smithi026 ceph-mon[179688]: failed to encode map e315 with expected crc 2023-09-23T21:04:25.426 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:04:25 smithi026 ceph-mon[179688]: failed to encode map e315 with expected crc 2023-09-23T21:04:25.426 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:04:25 smithi026 ceph-mon[179688]: failed to encode map e315 with expected crc 2023-09-23T21:04:25.426 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:04:25 smithi026 ceph-mon[179688]: failed to encode map e315 with expected crc 2023-09-23T21:04:25.426 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:04:25 smithi026 ceph-mon[175151]: failed to encode map e315 with expected crc 2023-09-23T21:04:25.426 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:04:25 smithi026 ceph-mon[175151]: failed to encode map e315 with expected crc 2023-09-23T21:04:25.427 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:04:25 smithi026 ceph-mon[175151]: osdmap e315: 8 total, 8 up, 8 in 2023-09-23T21:04:25.427 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:04:25 smithi026 ceph-mon[175151]: failed to encode map e315 with expected crc 2023-09-23T21:04:25.427 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:04:25 smithi026 ceph-mon[175151]: failed to encode map e315 with expected crc 2023-09-23T21:04:25.427 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:04:25 smithi026 ceph-mon[175151]: failed to encode map e315 with expected crc 2023-09-23T21:04:25.427 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:04:25 smithi026 ceph-mon[175151]: failed to encode map e315 with expected crc 2023-09-23T21:04:25.483 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:04:25.561 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:04:25.996 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T21:04:26.003 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:04:26.004 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:04:26.104 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:04:26.144 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:04:26.194 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:04:26.199 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:04:26.199 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:04:26.300 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:04:26.316 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T21:04:26.334 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:04:26.355 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:04:26.385 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:04:26.387 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:04:26.387 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:04:26.488 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:04:26.520 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:04:26 smithi070 ceph-mon[168282]: pgmap v813: 119 pgs: 2 active+remapped+backfilling, 117 active+clean; 3.2 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 532 op/s; 3051/158307 objects misplaced (1.927%); 9.1 MiB/s, 145 objects/s recovering 2023-09-23T21:04:26.520 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:04:26 smithi070 ceph-mon[168282]: 8.3 scrub starts 2023-09-23T21:04:26.520 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:04:26 smithi070 ceph-mon[168282]: failed to encode map e316 with expected crc 2023-09-23T21:04:26.521 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:04:26 smithi070 ceph-mon[168282]: failed to encode map e316 with expected crc 2023-09-23T21:04:26.521 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:04:26 smithi070 ceph-mon[168282]: osdmap e316: 8 total, 8 up, 8 in 2023-09-23T21:04:26.521 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:04:26 smithi070 ceph-mon[168282]: failed to encode map e316 with expected crc 2023-09-23T21:04:26.521 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:04:26 smithi070 ceph-mon[168282]: 8.19 starting backfill to osd.4 from (0'0,0'0] MAX to 312'9290 2023-09-23T21:04:26.521 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:04:26 smithi070 ceph-mon[168282]: failed to encode map e316 with expected crc 2023-09-23T21:04:26.521 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:04:26 smithi070 ceph-mon[168282]: 8.3 scrub ok 2023-09-23T21:04:26.523 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:04:26.562 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:04:26.569 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:04:26.570 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:04:26.625 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:04:26 smithi026 ceph-mon[175151]: pgmap v813: 119 pgs: 2 active+remapped+backfilling, 117 active+clean; 3.2 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 532 op/s; 3051/158307 objects misplaced (1.927%); 9.1 MiB/s, 145 objects/s recovering 2023-09-23T21:04:26.625 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:04:26 smithi026 ceph-mon[175151]: 8.3 scrub starts 2023-09-23T21:04:26.626 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:04:26 smithi026 ceph-mon[175151]: failed to encode map e316 with expected crc 2023-09-23T21:04:26.626 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:04:26 smithi026 ceph-mon[175151]: failed to encode map e316 with expected crc 2023-09-23T21:04:26.626 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:04:26 smithi026 ceph-mon[175151]: osdmap e316: 8 total, 8 up, 8 in 2023-09-23T21:04:26.626 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:04:26 smithi026 ceph-mon[175151]: failed to encode map e316 with expected crc 2023-09-23T21:04:26.626 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:04:26 smithi026 ceph-mon[175151]: 8.19 starting backfill to osd.4 from (0'0,0'0] MAX to 312'9290 2023-09-23T21:04:26.626 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:04:26 smithi026 ceph-mon[175151]: failed to encode map e316 with expected crc 2023-09-23T21:04:26.626 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:04:26 smithi026 ceph-mon[175151]: 8.3 scrub ok 2023-09-23T21:04:26.626 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:04:26 smithi026 ceph-mon[179688]: pgmap v813: 119 pgs: 2 active+remapped+backfilling, 117 active+clean; 3.2 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 532 op/s; 3051/158307 objects misplaced (1.927%); 9.1 MiB/s, 145 objects/s recovering 2023-09-23T21:04:26.626 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:04:26 smithi026 ceph-mon[179688]: 8.3 scrub starts 2023-09-23T21:04:26.627 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:04:26 smithi026 ceph-mon[179688]: failed to encode map e316 with expected crc 2023-09-23T21:04:26.627 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:04:26 smithi026 ceph-mon[179688]: failed to encode map e316 with expected crc 2023-09-23T21:04:26.627 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:04:26 smithi026 ceph-mon[179688]: osdmap e316: 8 total, 8 up, 8 in 2023-09-23T21:04:26.627 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:04:26 smithi026 ceph-mon[179688]: failed to encode map e316 with expected crc 2023-09-23T21:04:26.627 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:04:26 smithi026 ceph-mon[179688]: 8.19 starting backfill to osd.4 from (0'0,0'0] MAX to 312'9290 2023-09-23T21:04:26.627 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:04:26 smithi026 ceph-mon[179688]: failed to encode map e316 with expected crc 2023-09-23T21:04:26.627 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:04:26 smithi026 ceph-mon[179688]: 8.3 scrub ok 2023-09-23T21:04:26.671 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:04:26.744 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:04:26.986 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:04:26.996 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:04:26.996 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:04:27.016 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0: {} 2023-09-23T21:04:27.058 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1: {} 2023-09-23T21:04:27.097 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:04:27.107 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2: {} 2023-09-23T21:04:27.154 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3: {} 2023-09-23T21:04:27.197 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4: {} 2023-09-23T21:04:27.226 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:04:27.234 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T21:04:27.274 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6: {} 2023-09-23T21:04:27.314 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T21:04:27.326 INFO:teuthology.orchestra.run.smithi026.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T21:04:27.326 INFO:teuthology.orchestra.run.smithi026.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T21:04:27.326 INFO:teuthology.orchestra.run.smithi026.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T21:04:27.326 INFO:teuthology.orchestra.run.smithi026.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T21:04:27.326 INFO:teuthology.orchestra.run.smithi026.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T21:04:27.326 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T21:04:27.326 INFO:teuthology.orchestra.run.smithi026.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T21:04:27.326 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T21:04:27.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:04:27 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/2931975148' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:04:27.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:04:27 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/2931975148' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:04:27.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:04:27 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/2931975148' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:04:27.673 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T21:04:27.678 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:04:27.678 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:04:27.779 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:04:27.828 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:04:27.873 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:04:27.881 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:04:27.881 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:04:27.982 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:04:28.024 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:04:28.082 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:04:28.086 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:04:28.086 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:04:28.187 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:04:28.254 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:04:28.332 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:04:28.339 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:04:28.340 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:04:28.391 INFO:teuthology.orchestra.run.smithi026.stderr:reweighted osd.4 to 1 (10000) 2023-09-23T21:04:28.441 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:04:28.450 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:04:28.480 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:04:28 smithi026 ceph-mon[175151]: pgmap v815: 119 pgs: 2 active+remapped+backfilling, 117 active+clean; 3.2 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 252 op/s; 3093/156867 objects misplaced (1.972%); 16 MiB/s, 250 objects/s recovering 2023-09-23T21:04:28.480 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:04:28 smithi026 ceph-mon[175151]: 8.12 scrub starts 2023-09-23T21:04:28.480 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:04:28 smithi026 ceph-mon[175151]: 8.13 scrub starts 2023-09-23T21:04:28.480 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:04:28 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/1668490738' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-09-23T21:04:28.480 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:04:28 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-09-23T21:04:28.480 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:04:28 smithi026 ceph-mon[179688]: pgmap v815: 119 pgs: 2 active+remapped+backfilling, 117 active+clean; 3.2 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 252 op/s; 3093/156867 objects misplaced (1.972%); 16 MiB/s, 250 objects/s recovering 2023-09-23T21:04:28.480 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:04:28 smithi026 ceph-mon[179688]: 8.12 scrub starts 2023-09-23T21:04:28.481 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:04:28 smithi026 ceph-mon[179688]: 8.13 scrub starts 2023-09-23T21:04:28.481 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:04:28 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/1668490738' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-09-23T21:04:28.481 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:04:28 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-09-23T21:04:28.512 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:04:28.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:04:28 smithi070 ceph-mon[168282]: pgmap v815: 119 pgs: 2 active+remapped+backfilling, 117 active+clean; 3.2 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 252 op/s; 3093/156867 objects misplaced (1.972%); 16 MiB/s, 250 objects/s recovering 2023-09-23T21:04:28.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:04:28 smithi070 ceph-mon[168282]: 8.12 scrub starts 2023-09-23T21:04:28.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:04:28 smithi070 ceph-mon[168282]: 8.13 scrub starts 2023-09-23T21:04:28.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:04:28 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/1668490738' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-09-23T21:04:28.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:04:28 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-09-23T21:04:28.755 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:04:28.759 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:04:28.760 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:04:28.860 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:04:28.892 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:04:29.160 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph osd reweight 5 1 2023-09-23T21:04:29.228 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:04:29.231 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:04:29.231 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:04:29.333 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:04:29.361 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:04:29.377 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:04:29 smithi026 ceph-mon[179688]: 8.17 scrub starts 2023-09-23T21:04:29.377 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:04:29 smithi026 ceph-mon[179688]: 8.12 scrub ok 2023-09-23T21:04:29.377 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:04:29 smithi026 ceph-mon[179688]: failed to encode map e317 with expected crc 2023-09-23T21:04:29.377 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:04:29 smithi026 ceph-mon[179688]: failed to encode map e317 with expected crc 2023-09-23T21:04:29.377 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:04:29 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-09-23T21:04:29.377 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:04:29 smithi026 ceph-mon[179688]: osdmap e317: 8 total, 8 up, 8 in 2023-09-23T21:04:29.377 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:04:29 smithi026 ceph-mon[179688]: failed to encode map e317 with expected crc 2023-09-23T21:04:29.407 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:04:29.411 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:04:29.411 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:04:29.512 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:04:29.557 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:04:29.617 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:04:29.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:04:29 smithi070 ceph-mon[168282]: 8.17 scrub starts 2023-09-23T21:04:29.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:04:29 smithi070 ceph-mon[168282]: 8.12 scrub ok 2023-09-23T21:04:29.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:04:29 smithi070 ceph-mon[168282]: failed to encode map e317 with expected crc 2023-09-23T21:04:29.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:04:29 smithi070 ceph-mon[168282]: failed to encode map e317 with expected crc 2023-09-23T21:04:29.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:04:29 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-09-23T21:04:29.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:04:29 smithi070 ceph-mon[168282]: osdmap e317: 8 total, 8 up, 8 in 2023-09-23T21:04:29.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:04:29 smithi070 ceph-mon[168282]: failed to encode map e317 with expected crc 2023-09-23T21:04:29.623 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:04:29.624 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:04:29.633 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:04:29 smithi026 ceph-mon[175151]: 8.17 scrub starts 2023-09-23T21:04:29.633 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:04:29 smithi026 ceph-mon[175151]: 8.12 scrub ok 2023-09-23T21:04:29.633 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:04:29 smithi026 ceph-mon[175151]: failed to encode map e317 with expected crc 2023-09-23T21:04:29.633 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:04:29 smithi026 ceph-mon[175151]: failed to encode map e317 with expected crc 2023-09-23T21:04:29.633 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:04:29 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-09-23T21:04:29.633 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:04:29 smithi026 ceph-mon[175151]: osdmap e317: 8 total, 8 up, 8 in 2023-09-23T21:04:29.633 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:04:29 smithi026 ceph-mon[175151]: failed to encode map e317 with expected crc 2023-09-23T21:04:29.633 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 21:04:29 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[164029]: ::ffff:172.21.15.70 - - [23/Sep/2023:21:04:29] "GET /metrics HTTP/1.1" 200 36316 "" "Prometheus/2.43.0" 2023-09-23T21:04:29.712 INFO:teuthology.orchestra.run.smithi026.stdout:{ 2023-09-23T21:04:29.713 INFO:teuthology.orchestra.run.smithi026.stdout: "size": 20, 2023-09-23T21:04:29.713 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 600, 2023-09-23T21:04:29.713 INFO:teuthology.orchestra.run.smithi026.stdout: "ops": [ 2023-09-23T21:04:29.713 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.713 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34797.0:194304 6.10 6:0f79980e:::benchmark_data_smithi026_223868_object93178:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e179)", 2023-09-23T21:04:29.713 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T20:56:00.936645+0000", 2023-09-23T21:04:29.713 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 508.76820984699998, 2023-09-23T21:04:29.713 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.3260833879999998, 2023-09-23T21:04:29.713 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:04:29.713 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "reached pg", 2023-09-23T21:04:29.713 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T21:04:29.713 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34797", 2023-09-23T21:04:29.713 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T21:04:29.714 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 194304 2023-09-23T21:04:29.714 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.714 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:04:29.714 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.714 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:04:29.714 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:00.936645+0000", 2023-09-23T21:04:29.714 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:04:29.714 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.714 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.714 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:04:29.714 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:00.936645+0000", 2023-09-23T21:04:29.715 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:04:29.715 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.715 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.715 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:04:29.715 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:00.936643+0000", 2023-09-23T21:04:29.715 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999976 2023-09-23T21:04:29.715 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.715 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.715 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:04:29.715 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:00.936653+0000", 2023-09-23T21:04:29.715 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.5619999999999993e-06 2023-09-23T21:04:29.715 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.715 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.716 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:04:29.716 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:00.936672+0000", 2023-09-23T21:04:29.716 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.9179000000000001e-05 2023-09-23T21:04:29.716 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.716 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.716 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:04:29.716 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:00.936680+0000", 2023-09-23T21:04:29.716 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.2009999999999999e-06 2023-09-23T21:04:29.716 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.716 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.716 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:04:29.716 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:00.936746+0000", 2023-09-23T21:04:29.717 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.6188999999999999e-05 2023-09-23T21:04:29.717 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.717 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.717 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:04:29.717 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:00.936882+0000", 2023-09-23T21:04:29.717 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000135795 2023-09-23T21:04:29.717 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.717 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.717 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 2,3", 2023-09-23T21:04:29.717 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:00.937429+0000", 2023-09-23T21:04:29.717 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00054749499999999995 2023-09-23T21:04:29.717 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.717 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.717 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T21:04:29.717 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:00.940535+0000", 2023-09-23T21:04:29.718 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0031056090000000001 2023-09-23T21:04:29.718 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.718 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.718 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:04:29.718 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:00.964290+0000", 2023-09-23T21:04:29.718 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.023754941000000002 2023-09-23T21:04:29.718 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.718 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.718 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T21:04:29.718 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:00.964301+0000", 2023-09-23T21:04:29.718 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1242e-05 2023-09-23T21:04:29.718 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.719 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.719 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:04:29.719 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.950552+0000", 2023-09-23T21:04:29.719 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.98625030400000002 2023-09-23T21:04:29.719 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.719 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.719 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T21:04:29.719 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.950566+0000", 2023-09-23T21:04:29.719 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.488e-05 2023-09-23T21:04:29.719 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.719 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.719 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:04:29.719 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.254843+0000", 2023-09-23T21:04:29.719 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.3042760799999999 2023-09-23T21:04:29.720 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.720 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.720 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for missing object", 2023-09-23T21:04:29.720 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.254977+0000", 2023-09-23T21:04:29.720 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00013490299999999999 2023-09-23T21:04:29.720 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.720 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.720 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:04:29.720 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.262666+0000", 2023-09-23T21:04:29.720 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0076884479999999996 2023-09-23T21:04:29.720 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.720 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.720 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T21:04:29.720 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.262729+0000", 2023-09-23T21:04:29.721 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.2722e-05 2023-09-23T21:04:29.721 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:04:29.721 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:04:29.721 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:04:29.721 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.721 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.721 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34797.0:194354 6.10 6:0825a30e:::benchmark_data_smithi026_223868_object93228:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e180)", 2023-09-23T21:04:29.721 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T20:56:01.013619+0000", 2023-09-23T21:04:29.721 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 508.69123565000001, 2023-09-23T21:04:29.721 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.3123553139999999, 2023-09-23T21:04:29.721 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:04:29.721 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:04:29.721 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T21:04:29.721 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34797", 2023-09-23T21:04:29.722 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T21:04:29.722 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 194354 2023-09-23T21:04:29.722 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.722 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:04:29.722 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.722 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:04:29.722 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.013619+0000", 2023-09-23T21:04:29.722 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:04:29.722 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.722 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.722 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:04:29.722 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.013619+0000", 2023-09-23T21:04:29.722 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:04:29.723 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.723 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.723 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:04:29.723 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.013618+0000", 2023-09-23T21:04:29.723 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T21:04:29.723 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.723 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.723 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:04:29.723 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.013625+0000", 2023-09-23T21:04:29.723 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.6410000000000002e-06 2023-09-23T21:04:29.723 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.723 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.723 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:04:29.724 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.013649+0000", 2023-09-23T21:04:29.724 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.3589000000000001e-05 2023-09-23T21:04:29.724 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.724 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.724 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:04:29.724 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.013661+0000", 2023-09-23T21:04:29.724 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1861e-05 2023-09-23T21:04:29.724 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.724 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.724 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:04:29.724 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.013731+0000", 2023-09-23T21:04:29.724 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.9720000000000003e-05 2023-09-23T21:04:29.724 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.725 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.725 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T21:04:29.725 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.013745+0000", 2023-09-23T21:04:29.725 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.4389000000000001e-05 2023-09-23T21:04:29.725 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.725 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.725 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:04:29.725 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.950782+0000", 2023-09-23T21:04:29.725 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.93703733700000003 2023-09-23T21:04:29.725 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.725 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.725 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T21:04:29.725 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.950792+0000", 2023-09-23T21:04:29.726 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.8649999999999996e-06 2023-09-23T21:04:29.726 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.726 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.726 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:04:29.726 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.255412+0000", 2023-09-23T21:04:29.726 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.3046194689999999 2023-09-23T21:04:29.726 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.726 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.726 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for degraded object", 2023-09-23T21:04:29.726 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.255792+0000", 2023-09-23T21:04:29.726 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00038020699999999997 2023-09-23T21:04:29.726 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.726 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.726 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:04:29.727 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.263497+0000", 2023-09-23T21:04:29.727 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0077052049999999997 2023-09-23T21:04:29.727 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.727 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.727 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:04:29.727 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.263609+0000", 2023-09-23T21:04:29.727 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000111499 2023-09-23T21:04:29.727 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.727 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.727 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 1,5", 2023-09-23T21:04:29.727 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.264170+0000", 2023-09-23T21:04:29.727 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00056130799999999995 2023-09-23T21:04:29.727 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.728 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.728 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T21:04:29.728 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.270530+0000", 2023-09-23T21:04:29.728 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0063601439999999999 2023-09-23T21:04:29.728 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.728 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.728 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:04:29.728 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.321515+0000", 2023-09-23T21:04:29.728 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.050984578000000003 2023-09-23T21:04:29.728 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.728 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.728 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:04:29.728 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.325525+0000", 2023-09-23T21:04:29.728 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0040102269999999999 2023-09-23T21:04:29.729 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.729 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.729 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T21:04:29.729 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.325562+0000", 2023-09-23T21:04:29.729 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.7540999999999999e-05 2023-09-23T21:04:29.729 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.729 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.729 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T21:04:29.729 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.325975+0000", 2023-09-23T21:04:29.729 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00041231800000000001 2023-09-23T21:04:29.729 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:04:29.729 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:04:29.729 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:04:29.730 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.730 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.730 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34797.0:194386 6.10 6:0e6bbf83:::benchmark_data_smithi026_223868_object93260:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e180)", 2023-09-23T21:04:29.730 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T20:56:01.046640+0000", 2023-09-23T21:04:29.730 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 508.65821486999999, 2023-09-23T21:04:29.730 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.2797770470000001, 2023-09-23T21:04:29.730 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:04:29.730 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:04:29.730 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T21:04:29.730 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34797", 2023-09-23T21:04:29.730 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T21:04:29.730 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 194386 2023-09-23T21:04:29.730 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.731 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:04:29.731 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.731 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:04:29.731 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.046640+0000", 2023-09-23T21:04:29.731 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:04:29.731 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.731 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.731 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:04:29.731 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.046640+0000", 2023-09-23T21:04:29.731 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:04:29.731 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.731 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.731 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:04:29.731 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.046639+0000", 2023-09-23T21:04:29.732 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T21:04:29.732 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.732 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.732 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:04:29.732 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.046647+0000", 2023-09-23T21:04:29.732 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.9270000000000005e-06 2023-09-23T21:04:29.732 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.732 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.732 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:04:29.732 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.046667+0000", 2023-09-23T21:04:29.732 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.0699e-05 2023-09-23T21:04:29.732 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.732 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.733 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:04:29.733 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.046678+0000", 2023-09-23T21:04:29.733 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0336000000000001e-05 2023-09-23T21:04:29.733 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.733 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.733 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:04:29.733 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.046740+0000", 2023-09-23T21:04:29.733 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.2643000000000003e-05 2023-09-23T21:04:29.733 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.733 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.733 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T21:04:29.733 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.046757+0000", 2023-09-23T21:04:29.733 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.6673000000000001e-05 2023-09-23T21:04:29.733 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.734 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.734 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:04:29.734 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.950820+0000", 2023-09-23T21:04:29.734 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.90406270899999996 2023-09-23T21:04:29.734 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.734 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.734 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T21:04:29.734 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.950827+0000", 2023-09-23T21:04:29.734 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.3799999999999996e-06 2023-09-23T21:04:29.734 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.734 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.734 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:04:29.734 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.255844+0000", 2023-09-23T21:04:29.735 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.305017112 2023-09-23T21:04:29.735 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.735 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.735 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for degraded object", 2023-09-23T21:04:29.735 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.256176+0000", 2023-09-23T21:04:29.735 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00033179999999999999 2023-09-23T21:04:29.735 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.735 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.735 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:04:29.735 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.267675+0000", 2023-09-23T21:04:29.735 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.011498759000000001 2023-09-23T21:04:29.735 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.735 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.736 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:04:29.736 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.267748+0000", 2023-09-23T21:04:29.736 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.3232000000000006e-05 2023-09-23T21:04:29.736 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.736 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.736 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 1,5", 2023-09-23T21:04:29.736 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.268150+0000", 2023-09-23T21:04:29.736 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000402113 2023-09-23T21:04:29.736 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.736 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.736 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T21:04:29.736 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.273531+0000", 2023-09-23T21:04:29.736 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.005380625 2023-09-23T21:04:29.736 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.737 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.737 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:04:29.737 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.321574+0000", 2023-09-23T21:04:29.737 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.048043172000000002 2023-09-23T21:04:29.737 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.737 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.737 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:04:29.737 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.326045+0000", 2023-09-23T21:04:29.737 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0044713649999999997 2023-09-23T21:04:29.737 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.737 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.737 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T21:04:29.737 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.326080+0000", 2023-09-23T21:04:29.738 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.4918e-05 2023-09-23T21:04:29.738 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.738 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.738 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T21:04:29.738 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.326417+0000", 2023-09-23T21:04:29.738 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00033714999999999998 2023-09-23T21:04:29.738 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:04:29.738 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:04:29.738 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:04:29.738 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.738 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.738 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34797.0:194397 6.10 6:0cf77e90:::benchmark_data_smithi026_223868_object93271:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e180)", 2023-09-23T21:04:29.738 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T20:56:01.071586+0000", 2023-09-23T21:04:29.738 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 508.63326935200001, 2023-09-23T21:04:29.739 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.255280086, 2023-09-23T21:04:29.739 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:04:29.739 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:04:29.739 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T21:04:29.739 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34797", 2023-09-23T21:04:29.739 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T21:04:29.739 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 194397 2023-09-23T21:04:29.739 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.739 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:04:29.739 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.739 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:04:29.739 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.071586+0000", 2023-09-23T21:04:29.739 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:04:29.739 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.740 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.740 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:04:29.740 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.071586+0000", 2023-09-23T21:04:29.740 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:04:29.740 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.740 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.740 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:04:29.740 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.071585+0000", 2023-09-23T21:04:29.740 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T21:04:29.740 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.740 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.740 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:04:29.741 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.071588+0000", 2023-09-23T21:04:29.741 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.1950000000000002e-06 2023-09-23T21:04:29.741 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.741 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.741 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:04:29.741 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.071593+0000", 2023-09-23T21:04:29.741 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.7830000000000004e-06 2023-09-23T21:04:29.741 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.741 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.741 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:04:29.741 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.071599+0000", 2023-09-23T21:04:29.741 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.8900000000000004e-06 2023-09-23T21:04:29.742 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.742 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.742 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:04:29.742 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.071767+0000", 2023-09-23T21:04:29.742 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00016748500000000001 2023-09-23T21:04:29.742 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.742 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.742 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T21:04:29.742 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.071779+0000", 2023-09-23T21:04:29.742 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.2464e-05 2023-09-23T21:04:29.742 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.742 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.742 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:04:29.743 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.950851+0000", 2023-09-23T21:04:29.743 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.87907165899999995 2023-09-23T21:04:29.743 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.743 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.743 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T21:04:29.743 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.950859+0000", 2023-09-23T21:04:29.743 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.2940000000000006e-06 2023-09-23T21:04:29.743 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.743 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.743 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:04:29.743 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.256209+0000", 2023-09-23T21:04:29.745 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.3053493780000001 2023-09-23T21:04:29.745 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.745 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.745 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for degraded object", 2023-09-23T21:04:29.745 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.256530+0000", 2023-09-23T21:04:29.745 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00032157699999999997 2023-09-23T21:04:29.745 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.745 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.745 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:04:29.745 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.270992+0000", 2023-09-23T21:04:29.746 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.014462299 2023-09-23T21:04:29.746 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.746 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.746 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:04:29.746 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.271067+0000", 2023-09-23T21:04:29.746 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.4938000000000007e-05 2023-09-23T21:04:29.746 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.746 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.746 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 1,5", 2023-09-23T21:04:29.746 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.271446+0000", 2023-09-23T21:04:29.746 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00037825400000000001 2023-09-23T21:04:29.746 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.747 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.747 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T21:04:29.747 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.277790+0000", 2023-09-23T21:04:29.747 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0063448840000000003 2023-09-23T21:04:29.747 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.747 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.747 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:04:29.747 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.321615+0000", 2023-09-23T21:04:29.747 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.043824241999999999 2023-09-23T21:04:29.747 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.747 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.747 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:04:29.747 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.326572+0000", 2023-09-23T21:04:29.748 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0049568980000000004 2023-09-23T21:04:29.748 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.748 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.748 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T21:04:29.748 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.326624+0000", 2023-09-23T21:04:29.748 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.2061000000000002e-05 2023-09-23T21:04:29.748 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.748 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.748 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T21:04:29.748 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.326866+0000", 2023-09-23T21:04:29.748 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00024209700000000001 2023-09-23T21:04:29.748 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:04:29.748 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:04:29.749 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:04:29.749 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.749 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.749 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34797.0:194462 6.10 6:0e070208:::benchmark_data_smithi026_223868_object93336:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e180)", 2023-09-23T21:04:29.749 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T20:56:01.136635+0000", 2023-09-23T21:04:29.749 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 508.56822001900002, 2023-09-23T21:04:29.749 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.1905668330000001, 2023-09-23T21:04:29.749 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:04:29.749 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:04:29.749 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T21:04:29.749 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34797", 2023-09-23T21:04:29.749 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T21:04:29.749 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 194462 2023-09-23T21:04:29.749 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.750 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:04:29.750 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.750 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:04:29.750 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.136635+0000", 2023-09-23T21:04:29.750 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:04:29.750 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.750 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.750 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:04:29.750 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.136635+0000", 2023-09-23T21:04:29.750 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:04:29.750 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.750 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.750 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:04:29.751 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.136633+0000", 2023-09-23T21:04:29.751 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999981 2023-09-23T21:04:29.751 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.751 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.751 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:04:29.751 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.136640+0000", 2023-09-23T21:04:29.751 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.5989999999999998e-06 2023-09-23T21:04:29.751 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.751 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.751 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:04:29.751 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.136656+0000", 2023-09-23T21:04:29.751 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.6595e-05 2023-09-23T21:04:29.751 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.752 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.752 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:04:29.752 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.136665+0000", 2023-09-23T21:04:29.752 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.1540000000000002e-06 2023-09-23T21:04:29.752 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.752 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.752 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:04:29.752 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.136719+0000", 2023-09-23T21:04:29.752 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.4160000000000003e-05 2023-09-23T21:04:29.752 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.752 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.752 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T21:04:29.752 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.136726+0000", 2023-09-23T21:04:29.753 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.4429999999999997e-06 2023-09-23T21:04:29.753 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.753 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.753 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:04:29.753 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.950961+0000", 2023-09-23T21:04:29.753 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.81423474299999998 2023-09-23T21:04:29.753 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.753 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.753 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T21:04:29.753 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.950970+0000", 2023-09-23T21:04:29.753 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.2690000000000005e-06 2023-09-23T21:04:29.753 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.753 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.754 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:04:29.754 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.256671+0000", 2023-09-23T21:04:29.754 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.305701193 2023-09-23T21:04:29.754 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.754 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.754 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for degraded object", 2023-09-23T21:04:29.754 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.257010+0000", 2023-09-23T21:04:29.754 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00033869800000000002 2023-09-23T21:04:29.754 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.754 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.754 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:04:29.754 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.274991+0000", 2023-09-23T21:04:29.755 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.017981115999999998 2023-09-23T21:04:29.755 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.755 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.755 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:04:29.755 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.275118+0000", 2023-09-23T21:04:29.755 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00012721400000000001 2023-09-23T21:04:29.755 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.755 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.755 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 1,5", 2023-09-23T21:04:29.755 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.275667+0000", 2023-09-23T21:04:29.755 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00054888699999999996 2023-09-23T21:04:29.755 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.755 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.755 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T21:04:29.756 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.283025+0000", 2023-09-23T21:04:29.756 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0073580700000000004 2023-09-23T21:04:29.756 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.756 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.756 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:04:29.756 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.321667+0000", 2023-09-23T21:04:29.756 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.038641406000000003 2023-09-23T21:04:29.756 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.756 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.756 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:04:29.756 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.326925+0000", 2023-09-23T21:04:29.756 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0052579829999999999 2023-09-23T21:04:29.757 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.757 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.757 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T21:04:29.757 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.326974+0000", 2023-09-23T21:04:29.757 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.9079999999999998e-05 2023-09-23T21:04:29.757 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.757 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.757 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T21:04:29.757 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.327202+0000", 2023-09-23T21:04:29.757 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000228063 2023-09-23T21:04:29.757 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:04:29.757 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:04:29.758 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:04:29.758 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.758 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.758 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34797.0:194568 6.10 6:0af43279:::benchmark_data_smithi026_223868_object93442:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e180)", 2023-09-23T21:04:29.758 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T20:56:01.267334+0000", 2023-09-23T21:04:29.758 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 508.43752136000001, 2023-09-23T21:04:29.758 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.0602084349999998, 2023-09-23T21:04:29.758 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:04:29.758 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:04:29.758 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T21:04:29.758 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34797", 2023-09-23T21:04:29.758 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T21:04:29.758 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 194568 2023-09-23T21:04:29.759 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.759 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:04:29.759 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.759 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:04:29.759 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.267334+0000", 2023-09-23T21:04:29.759 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:04:29.759 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.759 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.759 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:04:29.759 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.267334+0000", 2023-09-23T21:04:29.759 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:04:29.759 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.759 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.760 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:04:29.760 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.267332+0000", 2023-09-23T21:04:29.760 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T21:04:29.760 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.760 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.760 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:04:29.760 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.267340+0000", 2023-09-23T21:04:29.760 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.6410000000000002e-06 2023-09-23T21:04:29.760 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.760 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.760 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:04:29.760 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.267356+0000", 2023-09-23T21:04:29.760 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.6409000000000002e-05 2023-09-23T21:04:29.761 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.761 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.761 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:04:29.761 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.267364+0000", 2023-09-23T21:04:29.761 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.6580000000000002e-06 2023-09-23T21:04:29.761 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.761 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.761 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:04:29.761 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.267423+0000", 2023-09-23T21:04:29.761 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.9392000000000003e-05 2023-09-23T21:04:29.761 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.761 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.761 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T21:04:29.762 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.267435+0000", 2023-09-23T21:04:29.762 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1653e-05 2023-09-23T21:04:29.762 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.762 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.762 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:04:29.762 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.950996+0000", 2023-09-23T21:04:29.762 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.68356135900000004 2023-09-23T21:04:29.762 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.762 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.762 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T21:04:29.762 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.951005+0000", 2023-09-23T21:04:29.762 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.5660000000000007e-06 2023-09-23T21:04:29.763 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.763 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.763 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:04:29.763 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.257054+0000", 2023-09-23T21:04:29.763 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.306049228 2023-09-23T21:04:29.763 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.763 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.763 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for degraded object", 2023-09-23T21:04:29.763 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.257353+0000", 2023-09-23T21:04:29.763 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00029845000000000002 2023-09-23T21:04:29.763 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.763 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.763 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:04:29.764 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.278511+0000", 2023-09-23T21:04:29.764 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.021158713999999999 2023-09-23T21:04:29.764 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.764 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.764 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:04:29.764 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.278633+0000", 2023-09-23T21:04:29.764 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00012163999999999999 2023-09-23T21:04:29.764 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.764 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.764 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 1,5", 2023-09-23T21:04:29.764 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.279079+0000", 2023-09-23T21:04:29.764 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000445807 2023-09-23T21:04:29.764 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.765 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.765 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T21:04:29.765 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.286413+0000", 2023-09-23T21:04:29.765 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0073338509999999997 2023-09-23T21:04:29.765 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.765 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.765 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:04:29.765 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.321694+0000", 2023-09-23T21:04:29.765 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.035281333999999998 2023-09-23T21:04:29.765 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.765 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.765 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:04:29.765 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.327259+0000", 2023-09-23T21:04:29.765 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0055648479999999998 2023-09-23T21:04:29.766 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.766 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.766 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T21:04:29.766 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.327308+0000", 2023-09-23T21:04:29.766 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.9527999999999997e-05 2023-09-23T21:04:29.766 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.766 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.766 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T21:04:29.766 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.327542+0000", 2023-09-23T21:04:29.766 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00023375499999999999 2023-09-23T21:04:29.766 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:04:29.766 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:04:29.767 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:04:29.767 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.767 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.767 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34797.0:194611 6.10 6:0de287e4:::benchmark_data_smithi026_223868_object93485:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e180)", 2023-09-23T21:04:29.767 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T20:56:01.323196+0000", 2023-09-23T21:04:29.767 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 508.38165874700002, 2023-09-23T21:04:29.767 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.0047136079999999, 2023-09-23T21:04:29.767 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:04:29.767 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:04:29.767 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T21:04:29.767 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34797", 2023-09-23T21:04:29.767 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T21:04:29.767 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 194611 2023-09-23T21:04:29.767 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.768 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:04:29.768 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.768 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:04:29.768 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.323196+0000", 2023-09-23T21:04:29.768 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:04:29.768 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.768 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.768 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:04:29.768 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.323196+0000", 2023-09-23T21:04:29.768 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:04:29.768 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.768 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.769 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:04:29.769 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.323196+0000", 2023-09-23T21:04:29.769 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T21:04:29.769 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.769 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.769 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:04:29.769 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.323199+0000", 2023-09-23T21:04:29.769 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.4470000000000002e-06 2023-09-23T21:04:29.769 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.769 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.769 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:04:29.769 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.323209+0000", 2023-09-23T21:04:29.769 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.4700000000000008e-06 2023-09-23T21:04:29.769 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.770 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.770 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:04:29.770 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.323216+0000", 2023-09-23T21:04:29.770 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.3900000000000004e-06 2023-09-23T21:04:29.770 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.770 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.770 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:04:29.770 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.323262+0000", 2023-09-23T21:04:29.770 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.5785000000000003e-05 2023-09-23T21:04:29.770 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.770 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.770 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T21:04:29.771 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.323273+0000", 2023-09-23T21:04:29.771 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0888e-05 2023-09-23T21:04:29.771 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.771 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.771 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:04:29.771 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.951032+0000", 2023-09-23T21:04:29.771 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.62775908800000002 2023-09-23T21:04:29.771 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.771 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.771 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T21:04:29.771 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.951039+0000", 2023-09-23T21:04:29.771 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.3100000000000003e-06 2023-09-23T21:04:29.771 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.771 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.772 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:04:29.772 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.257402+0000", 2023-09-23T21:04:29.772 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.306362724 2023-09-23T21:04:29.772 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.772 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.772 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for degraded object", 2023-09-23T21:04:29.772 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.257769+0000", 2023-09-23T21:04:29.772 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00036691000000000002 2023-09-23T21:04:29.772 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.772 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.772 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:04:29.772 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.283087+0000", 2023-09-23T21:04:29.773 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.025317925000000002 2023-09-23T21:04:29.773 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.773 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.773 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:04:29.773 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.283213+0000", 2023-09-23T21:04:29.773 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000126773 2023-09-23T21:04:29.773 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.773 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.773 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 1,5", 2023-09-23T21:04:29.773 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.283761+0000", 2023-09-23T21:04:29.773 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00054742500000000004 2023-09-23T21:04:29.773 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.774 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.774 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T21:04:29.774 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.290057+0000", 2023-09-23T21:04:29.774 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0062964869999999999 2023-09-23T21:04:29.774 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.774 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.774 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:04:29.774 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.321745+0000", 2023-09-23T21:04:29.774 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.031687651999999997 2023-09-23T21:04:29.774 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.774 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.774 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:04:29.775 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.327648+0000", 2023-09-23T21:04:29.775 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0059028600000000002 2023-09-23T21:04:29.775 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.775 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.775 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T21:04:29.775 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.327694+0000", 2023-09-23T21:04:29.775 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.6081000000000001e-05 2023-09-23T21:04:29.775 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.775 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.775 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T21:04:29.775 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.327910+0000", 2023-09-23T21:04:29.775 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00021589499999999999 2023-09-23T21:04:29.775 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:04:29.776 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:04:29.776 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:04:29.776 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.776 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.776 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34797.0:194684 6.10 6:0c8ebf5c:::benchmark_data_smithi026_223868_object93558:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e180)", 2023-09-23T21:04:29.776 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T20:56:01.410674+0000", 2023-09-23T21:04:29.776 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 508.29418055500003, 2023-09-23T21:04:29.776 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.9175628630000001, 2023-09-23T21:04:29.776 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:04:29.776 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:04:29.776 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T21:04:29.776 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34797", 2023-09-23T21:04:29.776 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T21:04:29.777 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 194684 2023-09-23T21:04:29.777 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.777 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:04:29.777 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.777 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:04:29.777 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.410674+0000", 2023-09-23T21:04:29.777 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:04:29.777 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.777 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.777 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:04:29.777 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.410674+0000", 2023-09-23T21:04:29.777 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:04:29.777 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.778 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.778 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:04:29.778 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.410674+0000", 2023-09-23T21:04:29.778 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T21:04:29.778 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.778 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.778 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:04:29.778 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.410677+0000", 2023-09-23T21:04:29.778 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.9579999999999999e-06 2023-09-23T21:04:29.778 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.778 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.778 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:04:29.779 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.410692+0000", 2023-09-23T21:04:29.779 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.539e-05 2023-09-23T21:04:29.779 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.779 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.779 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:04:29.779 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.410698+0000", 2023-09-23T21:04:29.779 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.2830000000000001e-06 2023-09-23T21:04:29.779 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.779 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.779 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:04:29.779 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.410771+0000", 2023-09-23T21:04:29.779 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.2899999999999997e-05 2023-09-23T21:04:29.779 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.780 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.780 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T21:04:29.780 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.410778+0000", 2023-09-23T21:04:29.780 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.4200000000000001e-06 2023-09-23T21:04:29.780 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.780 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.780 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:04:29.780 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.951065+0000", 2023-09-23T21:04:29.780 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.54028672700000002 2023-09-23T21:04:29.780 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.780 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.780 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T21:04:29.780 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.951073+0000", 2023-09-23T21:04:29.781 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.551e-06 2023-09-23T21:04:29.781 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.781 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.781 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:04:29.781 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.257812+0000", 2023-09-23T21:04:29.781 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.3067387959999999 2023-09-23T21:04:29.781 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.781 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.781 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for degraded object", 2023-09-23T21:04:29.781 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.258109+0000", 2023-09-23T21:04:29.781 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00029674299999999997 2023-09-23T21:04:29.781 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.781 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.782 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:04:29.782 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.286926+0000", 2023-09-23T21:04:29.782 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.028817470000000001 2023-09-23T21:04:29.782 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.782 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.782 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:04:29.782 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.286999+0000", 2023-09-23T21:04:29.782 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.3141999999999999e-05 2023-09-23T21:04:29.782 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.782 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.782 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 1,5", 2023-09-23T21:04:29.782 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.287434+0000", 2023-09-23T21:04:29.783 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00043483299999999998 2023-09-23T21:04:29.783 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.783 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.783 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T21:04:29.783 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.294642+0000", 2023-09-23T21:04:29.783 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0072078960000000001 2023-09-23T21:04:29.783 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.783 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.783 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:04:29.783 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.321776+0000", 2023-09-23T21:04:29.783 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.027134098999999998 2023-09-23T21:04:29.783 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.783 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.784 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:04:29.784 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.327964+0000", 2023-09-23T21:04:29.784 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0061875960000000001 2023-09-23T21:04:29.784 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.784 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.784 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T21:04:29.784 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.328014+0000", 2023-09-23T21:04:29.784 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.007e-05 2023-09-23T21:04:29.784 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.784 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.784 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T21:04:29.784 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.328237+0000", 2023-09-23T21:04:29.784 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000223391 2023-09-23T21:04:29.785 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:04:29.785 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:04:29.785 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:04:29.785 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.785 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.785 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34797.0:194713 6.10 6:0a64bb83:::benchmark_data_smithi026_223868_object93587:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e180)", 2023-09-23T21:04:29.785 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T20:56:01.445241+0000", 2023-09-23T21:04:29.785 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 508.259614462, 2023-09-23T21:04:29.785 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.8833663199999999, 2023-09-23T21:04:29.785 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:04:29.785 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:04:29.785 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T21:04:29.786 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34797", 2023-09-23T21:04:29.786 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T21:04:29.786 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 194713 2023-09-23T21:04:29.786 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.786 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:04:29.786 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.786 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:04:29.786 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.445241+0000", 2023-09-23T21:04:29.786 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:04:29.786 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.786 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.786 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:04:29.786 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.445241+0000", 2023-09-23T21:04:29.787 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:04:29.787 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.787 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.787 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:04:29.787 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.445239+0000", 2023-09-23T21:04:29.787 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T21:04:29.787 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.787 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.787 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:04:29.787 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.445246+0000", 2023-09-23T21:04:29.787 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.6950000000000003e-06 2023-09-23T21:04:29.787 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.787 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.788 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:04:29.788 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.445262+0000", 2023-09-23T21:04:29.788 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.5654999999999998e-05 2023-09-23T21:04:29.788 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.788 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.788 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:04:29.788 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.445271+0000", 2023-09-23T21:04:29.788 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.679e-06 2023-09-23T21:04:29.788 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.788 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.788 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:04:29.788 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.445322+0000", 2023-09-23T21:04:29.788 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.0516000000000002e-05 2023-09-23T21:04:29.789 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.789 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.789 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T21:04:29.789 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.445332+0000", 2023-09-23T21:04:29.789 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.9699999999999994e-06 2023-09-23T21:04:29.789 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.789 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.789 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:04:29.789 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.951102+0000", 2023-09-23T21:04:29.789 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.50577048800000002 2023-09-23T21:04:29.789 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.789 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.789 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T21:04:29.790 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.951110+0000", 2023-09-23T21:04:29.790 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.208e-06 2023-09-23T21:04:29.790 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.790 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.790 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:04:29.790 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.258157+0000", 2023-09-23T21:04:29.790 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.3070464900000001 2023-09-23T21:04:29.790 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.790 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.790 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for degraded object", 2023-09-23T21:04:29.790 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.258517+0000", 2023-09-23T21:04:29.790 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00035973599999999999 2023-09-23T21:04:29.790 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.791 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.791 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:04:29.791 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.291101+0000", 2023-09-23T21:04:29.791 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.032584136999999999 2023-09-23T21:04:29.791 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.791 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.791 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:04:29.791 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.291239+0000", 2023-09-23T21:04:29.791 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000137761 2023-09-23T21:04:29.791 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.791 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.791 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 1,5", 2023-09-23T21:04:29.791 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.291789+0000", 2023-09-23T21:04:29.792 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00055038199999999998 2023-09-23T21:04:29.792 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.792 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.792 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T21:04:29.792 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.295524+0000", 2023-09-23T21:04:29.792 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0037345519999999999 2023-09-23T21:04:29.792 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.792 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.792 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:04:29.792 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.321831+0000", 2023-09-23T21:04:29.792 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.026307456 2023-09-23T21:04:29.792 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.792 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.792 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:04:29.793 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.328336+0000", 2023-09-23T21:04:29.793 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0065046009999999996 2023-09-23T21:04:29.793 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.793 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.793 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T21:04:29.793 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.328382+0000", 2023-09-23T21:04:29.793 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.617e-05 2023-09-23T21:04:29.793 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.793 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.793 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T21:04:29.793 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.328607+0000", 2023-09-23T21:04:29.793 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00022515100000000001 2023-09-23T21:04:29.793 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:04:29.794 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:04:29.794 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:04:29.794 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.794 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.794 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34797.0:194727 6.10 6:0b60e6f5:::benchmark_data_smithi026_223868_object93601:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e180)", 2023-09-23T21:04:29.794 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T20:56:01.462928+0000", 2023-09-23T21:04:29.794 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 508.24192673200002, 2023-09-23T21:04:29.794 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.866058437, 2023-09-23T21:04:29.794 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:04:29.794 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:04:29.794 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T21:04:29.795 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34797", 2023-09-23T21:04:29.795 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T21:04:29.795 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 194727 2023-09-23T21:04:29.795 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.795 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:04:29.795 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.795 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:04:29.795 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.462928+0000", 2023-09-23T21:04:29.795 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:04:29.795 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.795 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.795 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:04:29.795 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.462928+0000", 2023-09-23T21:04:29.796 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:04:29.796 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.796 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.796 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:04:29.796 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.462927+0000", 2023-09-23T21:04:29.796 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.999999 2023-09-23T21:04:29.796 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.796 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.796 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:04:29.796 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.462933+0000", 2023-09-23T21:04:29.796 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.3730000000000001e-06 2023-09-23T21:04:29.796 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.796 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.796 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:04:29.797 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.462946+0000", 2023-09-23T21:04:29.797 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.3747000000000001e-05 2023-09-23T21:04:29.797 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.797 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.797 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:04:29.797 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.462955+0000", 2023-09-23T21:04:29.797 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.0000000000000002e-06 2023-09-23T21:04:29.797 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.797 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.797 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:04:29.797 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.463029+0000", 2023-09-23T21:04:29.797 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.3559000000000005e-05 2023-09-23T21:04:29.798 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.798 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.798 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T21:04:29.798 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.463042+0000", 2023-09-23T21:04:29.798 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.2661e-05 2023-09-23T21:04:29.798 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.798 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.798 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:04:29.798 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.951135+0000", 2023-09-23T21:04:29.798 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.48809338699999999 2023-09-23T21:04:29.798 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.798 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.798 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T21:04:29.799 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.951143+0000", 2023-09-23T21:04:29.799 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.7319999999999998e-06 2023-09-23T21:04:29.799 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.799 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.799 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:04:29.799 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.258558+0000", 2023-09-23T21:04:29.799 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.307415357 2023-09-23T21:04:29.799 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.799 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.799 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for degraded object", 2023-09-23T21:04:29.799 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.258901+0000", 2023-09-23T21:04:29.799 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000342515 2023-09-23T21:04:29.799 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.800 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.800 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:04:29.800 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.297191+0000", 2023-09-23T21:04:29.800 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.038290826 2023-09-23T21:04:29.800 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.800 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.800 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:04:29.800 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.297316+0000", 2023-09-23T21:04:29.800 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00012508899999999999 2023-09-23T21:04:29.800 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.800 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.800 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 1,5", 2023-09-23T21:04:29.800 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.297920+0000", 2023-09-23T21:04:29.800 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00060379499999999996 2023-09-23T21:04:29.801 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.801 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.801 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T21:04:29.801 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.304770+0000", 2023-09-23T21:04:29.801 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0068497499999999999 2023-09-23T21:04:29.801 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.801 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.801 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:04:29.801 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.325484+0000", 2023-09-23T21:04:29.801 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.020714521999999999 2023-09-23T21:04:29.801 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.801 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.802 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:04:29.802 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.328690+0000", 2023-09-23T21:04:29.802 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.003205875 2023-09-23T21:04:29.802 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.802 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.802 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T21:04:29.802 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.328740+0000", 2023-09-23T21:04:29.802 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.9492e-05 2023-09-23T21:04:29.802 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.802 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.802 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T21:04:29.802 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.328987+0000", 2023-09-23T21:04:29.802 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00024687500000000002 2023-09-23T21:04:29.803 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:04:29.803 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:04:29.803 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:04:29.803 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.803 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.803 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34797.0:194750 6.10 6:091e1945:::benchmark_data_smithi026_223868_object93624:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e180)", 2023-09-23T21:04:29.803 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T20:56:01.496055+0000", 2023-09-23T21:04:29.803 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 508.20880000199998, 2023-09-23T21:04:29.803 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.8333552829999999, 2023-09-23T21:04:29.803 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:04:29.803 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:04:29.803 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T21:04:29.804 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34797", 2023-09-23T21:04:29.804 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T21:04:29.804 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 194750 2023-09-23T21:04:29.804 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.804 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:04:29.804 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.804 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:04:29.804 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.496055+0000", 2023-09-23T21:04:29.804 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:04:29.804 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.804 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.804 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:04:29.804 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.496055+0000", 2023-09-23T21:04:29.805 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:04:29.805 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.805 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.805 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:04:29.805 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.496054+0000", 2023-09-23T21:04:29.805 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T21:04:29.805 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.805 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.805 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:04:29.805 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.496061+0000", 2023-09-23T21:04:29.805 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.5009999999999998e-06 2023-09-23T21:04:29.805 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.805 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.806 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:04:29.806 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.496078+0000", 2023-09-23T21:04:29.806 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.6917000000000001e-05 2023-09-23T21:04:29.806 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.806 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.806 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:04:29.806 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.496085+0000", 2023-09-23T21:04:29.806 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.1729999999999998e-06 2023-09-23T21:04:29.806 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.806 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.806 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:04:29.806 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.496150+0000", 2023-09-23T21:04:29.807 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.5225e-05 2023-09-23T21:04:29.807 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.807 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.807 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T21:04:29.807 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.496163+0000", 2023-09-23T21:04:29.807 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.2187e-05 2023-09-23T21:04:29.807 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.807 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.807 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:04:29.807 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.951168+0000", 2023-09-23T21:04:29.807 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.455005037 2023-09-23T21:04:29.807 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.807 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.808 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T21:04:29.808 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.951176+0000", 2023-09-23T21:04:29.808 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.9750000000000007e-06 2023-09-23T21:04:29.808 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.808 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.808 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:04:29.808 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.258954+0000", 2023-09-23T21:04:29.808 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.3077780589999999 2023-09-23T21:04:29.808 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.808 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.808 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for degraded object", 2023-09-23T21:04:29.808 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.259370+0000", 2023-09-23T21:04:29.808 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000416632 2023-09-23T21:04:29.809 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.809 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.809 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:04:29.809 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.301427+0000", 2023-09-23T21:04:29.809 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.042056291000000003 2023-09-23T21:04:29.809 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.809 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.809 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:04:29.809 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.301568+0000", 2023-09-23T21:04:29.809 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00014118900000000001 2023-09-23T21:04:29.809 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.809 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.809 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 1,5", 2023-09-23T21:04:29.810 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.302113+0000", 2023-09-23T21:04:29.810 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00054519200000000001 2023-09-23T21:04:29.810 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.810 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.810 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T21:04:29.810 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.309443+0000", 2023-09-23T21:04:29.810 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0073298130000000001 2023-09-23T21:04:29.810 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.810 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.810 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:04:29.810 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.326494+0000", 2023-09-23T21:04:29.810 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.017051667999999999 2023-09-23T21:04:29.811 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.811 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.811 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:04:29.811 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.329098+0000", 2023-09-23T21:04:29.811 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0026040550000000001 2023-09-23T21:04:29.811 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.811 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.811 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T21:04:29.811 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.329155+0000", 2023-09-23T21:04:29.811 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.6276000000000001e-05 2023-09-23T21:04:29.811 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.811 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.811 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T21:04:29.811 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.329410+0000", 2023-09-23T21:04:29.812 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00025554400000000002 2023-09-23T21:04:29.812 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:04:29.812 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:04:29.812 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:04:29.812 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.812 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.812 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34797.0:194898 6.10 6:0ab6bac4:::benchmark_data_smithi026_223868_object93772:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e180)", 2023-09-23T21:04:29.812 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T20:56:01.737608+0000", 2023-09-23T21:04:29.812 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 507.967247289, 2023-09-23T21:04:29.812 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.605071608, 2023-09-23T21:04:29.812 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:04:29.812 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:04:29.813 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T21:04:29.813 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34797", 2023-09-23T21:04:29.813 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T21:04:29.813 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 194898 2023-09-23T21:04:29.813 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.813 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:04:29.813 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.813 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:04:29.813 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.737608+0000", 2023-09-23T21:04:29.813 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:04:29.813 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.813 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.813 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:04:29.814 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.737608+0000", 2023-09-23T21:04:29.814 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:04:29.814 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.814 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.814 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:04:29.814 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.737606+0000", 2023-09-23T21:04:29.814 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999981 2023-09-23T21:04:29.814 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.814 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.814 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:04:29.814 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.737613+0000", 2023-09-23T21:04:29.814 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.5390000000000002e-06 2023-09-23T21:04:29.815 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.815 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.815 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:04:29.815 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.737626+0000", 2023-09-23T21:04:29.815 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.2232000000000001e-05 2023-09-23T21:04:29.815 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.815 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.815 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:04:29.815 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.737634+0000", 2023-09-23T21:04:29.815 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.0960000000000001e-06 2023-09-23T21:04:29.815 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.815 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.815 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:04:29.816 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.737715+0000", 2023-09-23T21:04:29.816 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.1599000000000003e-05 2023-09-23T21:04:29.816 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.816 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.816 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T21:04:29.816 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.737730+0000", 2023-09-23T21:04:29.816 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.4363e-05 2023-09-23T21:04:29.816 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.816 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.816 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:04:29.816 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.951200+0000", 2023-09-23T21:04:29.816 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.21347070400000001 2023-09-23T21:04:29.816 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.817 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.817 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T21:04:29.817 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.951208+0000", 2023-09-23T21:04:29.817 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.5599999999999996e-06 2023-09-23T21:04:29.817 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.817 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.817 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:04:29.817 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.259413+0000", 2023-09-23T21:04:29.817 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.3082054949999999 2023-09-23T21:04:29.817 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.817 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.817 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for degraded object", 2023-09-23T21:04:29.817 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.259853+0000", 2023-09-23T21:04:29.818 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00043965399999999999 2023-09-23T21:04:29.818 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.818 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.818 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:04:29.818 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.305598+0000", 2023-09-23T21:04:29.818 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.045744765999999999 2023-09-23T21:04:29.818 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.818 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.818 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:04:29.818 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.305723+0000", 2023-09-23T21:04:29.818 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000124807 2023-09-23T21:04:29.818 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.818 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.819 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 1,5", 2023-09-23T21:04:29.819 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.306293+0000", 2023-09-23T21:04:29.819 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00057089399999999996 2023-09-23T21:04:29.819 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.819 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.819 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T21:04:29.819 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.317574+0000", 2023-09-23T21:04:29.819 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.011280434000000001 2023-09-23T21:04:29.819 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.819 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.819 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:04:29.819 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.327600+0000", 2023-09-23T21:04:29.819 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.010026037999999999 2023-09-23T21:04:29.820 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.820 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.820 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:04:29.820 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.342119+0000", 2023-09-23T21:04:29.820 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.014519139 2023-09-23T21:04:29.820 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.820 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.820 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T21:04:29.820 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.342171+0000", 2023-09-23T21:04:29.820 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.1925000000000002e-05 2023-09-23T21:04:29.820 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.820 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.820 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T21:04:29.821 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.342679+0000", 2023-09-23T21:04:29.821 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00050835799999999999 2023-09-23T21:04:29.821 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:04:29.821 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:04:29.821 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:04:29.821 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.821 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.821 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34797.0:194908 6.10 6:0f877dc5:::benchmark_data_smithi026_223868_object93782:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e180)", 2023-09-23T21:04:29.821 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T20:56:01.753586+0000", 2023-09-23T21:04:29.821 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 507.951269083, 2023-09-23T21:04:29.821 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.5894383169999999, 2023-09-23T21:04:29.821 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:04:29.821 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:04:29.822 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T21:04:29.822 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34797", 2023-09-23T21:04:29.822 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T21:04:29.822 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 194908 2023-09-23T21:04:29.822 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.822 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:04:29.822 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.822 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:04:29.822 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.753586+0000", 2023-09-23T21:04:29.822 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:04:29.822 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.822 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.823 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:04:29.823 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.753586+0000", 2023-09-23T21:04:29.823 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:04:29.823 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.823 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.823 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:04:29.823 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.753584+0000", 2023-09-23T21:04:29.823 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T21:04:29.823 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.823 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.823 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:04:29.823 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.753592+0000", 2023-09-23T21:04:29.823 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.5349999999999999e-06 2023-09-23T21:04:29.824 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.824 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.824 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:04:29.824 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.753609+0000", 2023-09-23T21:04:29.824 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.696e-05 2023-09-23T21:04:29.824 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.824 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.824 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:04:29.824 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.753617+0000", 2023-09-23T21:04:29.824 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.2009999999999999e-06 2023-09-23T21:04:29.824 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.824 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.824 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:04:29.825 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.753675+0000", 2023-09-23T21:04:29.825 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.7856999999999998e-05 2023-09-23T21:04:29.825 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.825 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.825 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T21:04:29.825 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.753690+0000", 2023-09-23T21:04:29.825 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.5528000000000001e-05 2023-09-23T21:04:29.825 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.825 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.825 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:04:29.825 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.951234+0000", 2023-09-23T21:04:29.825 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.19754403500000001 2023-09-23T21:04:29.825 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.826 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.826 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T21:04:29.826 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.951242+0000", 2023-09-23T21:04:29.826 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.9330000000000001e-06 2023-09-23T21:04:29.826 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.826 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.826 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:04:29.826 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.259898+0000", 2023-09-23T21:04:29.826 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.308655608 2023-09-23T21:04:29.826 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.826 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.826 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for degraded object", 2023-09-23T21:04:29.827 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.260281+0000", 2023-09-23T21:04:29.827 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00038326300000000001 2023-09-23T21:04:29.827 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.827 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.827 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:04:29.827 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.309484+0000", 2023-09-23T21:04:29.827 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.049202391999999998 2023-09-23T21:04:29.827 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.827 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.827 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:04:29.827 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.309599+0000", 2023-09-23T21:04:29.827 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000115642 2023-09-23T21:04:29.827 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.827 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.828 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 1,5", 2023-09-23T21:04:29.828 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.310127+0000", 2023-09-23T21:04:29.828 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00052754099999999997 2023-09-23T21:04:29.828 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.828 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.828 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T21:04:29.828 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.320913+0000", 2023-09-23T21:04:29.828 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.010786568999999999 2023-09-23T21:04:29.828 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.828 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.828 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:04:29.828 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.328297+0000", 2023-09-23T21:04:29.828 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0073830889999999998 2023-09-23T21:04:29.829 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.829 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.829 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:04:29.829 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.342727+0000", 2023-09-23T21:04:29.829 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.014430493000000001 2023-09-23T21:04:29.829 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.829 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.829 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T21:04:29.829 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.342775+0000", 2023-09-23T21:04:29.829 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.7593000000000003e-05 2023-09-23T21:04:29.829 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.829 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.830 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T21:04:29.830 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.343024+0000", 2023-09-23T21:04:29.830 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000249652 2023-09-23T21:04:29.830 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:04:29.830 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:04:29.830 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:04:29.830 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.830 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.830 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34797.0:194925 6.10 6:0af4fd48:::benchmark_data_smithi026_223868_object93799:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e180)", 2023-09-23T21:04:29.830 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T20:56:01.778450+0000", 2023-09-23T21:04:29.830 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 507.926405397, 2023-09-23T21:04:29.830 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.5649251790000001, 2023-09-23T21:04:29.830 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:04:29.831 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:04:29.831 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T21:04:29.831 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34797", 2023-09-23T21:04:29.831 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T21:04:29.831 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 194925 2023-09-23T21:04:29.831 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.831 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:04:29.831 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.831 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:04:29.831 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.778450+0000", 2023-09-23T21:04:29.831 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:04:29.831 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.832 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.832 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:04:29.832 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.778450+0000", 2023-09-23T21:04:29.832 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:04:29.832 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.832 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.832 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:04:29.832 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.778448+0000", 2023-09-23T21:04:29.832 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T21:04:29.832 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.832 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.832 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:04:29.832 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.778456+0000", 2023-09-23T21:04:29.833 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.5270000000000001e-06 2023-09-23T21:04:29.833 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.833 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.833 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:04:29.833 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.778484+0000", 2023-09-23T21:04:29.833 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.8629999999999999e-05 2023-09-23T21:04:29.833 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.833 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.833 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:04:29.833 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.778496+0000", 2023-09-23T21:04:29.833 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.203e-05 2023-09-23T21:04:29.833 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.833 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.834 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:04:29.834 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.778564+0000", 2023-09-23T21:04:29.834 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.8099e-05 2023-09-23T21:04:29.834 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.834 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.834 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T21:04:29.834 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.778579+0000", 2023-09-23T21:04:29.834 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.4397e-05 2023-09-23T21:04:29.834 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.834 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.834 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:04:29.834 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.951272+0000", 2023-09-23T21:04:29.835 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.172692926 2023-09-23T21:04:29.835 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.835 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.835 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T21:04:29.835 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.951281+0000", 2023-09-23T21:04:29.835 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.0180000000000007e-06 2023-09-23T21:04:29.835 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.835 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.835 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:04:29.835 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.260332+0000", 2023-09-23T21:04:29.835 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.3090507920000001 2023-09-23T21:04:29.835 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.836 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.836 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for degraded object", 2023-09-23T21:04:29.836 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.260659+0000", 2023-09-23T21:04:29.836 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000327466 2023-09-23T21:04:29.836 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.836 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.836 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:04:29.836 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.313663+0000", 2023-09-23T21:04:29.836 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.053004308 2023-09-23T21:04:29.836 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.836 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.836 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:04:29.836 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.313780+0000", 2023-09-23T21:04:29.837 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00011642999999999999 2023-09-23T21:04:29.837 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.837 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.837 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 1,5", 2023-09-23T21:04:29.837 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.314321+0000", 2023-09-23T21:04:29.837 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00054174099999999999 2023-09-23T21:04:29.837 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.837 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.837 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T21:04:29.837 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.320923+0000", 2023-09-23T21:04:29.837 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0066012759999999997 2023-09-23T21:04:29.837 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.837 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.838 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:04:29.838 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.328654+0000", 2023-09-23T21:04:29.838 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0077311619999999998 2023-09-23T21:04:29.838 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.838 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.838 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:04:29.838 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.343069+0000", 2023-09-23T21:04:29.838 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.014414797 2023-09-23T21:04:29.838 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.838 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.838 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T21:04:29.838 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.343121+0000", 2023-09-23T21:04:29.838 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.2475999999999997e-05 2023-09-23T21:04:29.839 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.839 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.839 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T21:04:29.839 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.343375+0000", 2023-09-23T21:04:29.839 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00025362900000000001 2023-09-23T21:04:29.839 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:04:29.839 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:04:29.839 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:04:29.839 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.839 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.839 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34797.0:194948 6.10 6:0b718fd8:::benchmark_data_smithi026_223868_object93822:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e180)", 2023-09-23T21:04:29.839 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T20:56:01.821311+0000", 2023-09-23T21:04:29.839 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 507.88354356299999, 2023-09-23T21:04:29.840 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.5226795950000001, 2023-09-23T21:04:29.840 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:04:29.840 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:04:29.840 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T21:04:29.840 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34797", 2023-09-23T21:04:29.840 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T21:04:29.840 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 194948 2023-09-23T21:04:29.840 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.840 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:04:29.840 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.840 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:04:29.840 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.821311+0000", 2023-09-23T21:04:29.840 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:04:29.841 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.841 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.841 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:04:29.841 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.821311+0000", 2023-09-23T21:04:29.841 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:04:29.841 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.841 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.841 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:04:29.841 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.821310+0000", 2023-09-23T21:04:29.841 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T21:04:29.841 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.841 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.841 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:04:29.842 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.821318+0000", 2023-09-23T21:04:29.842 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.7640000000000005e-06 2023-09-23T21:04:29.842 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.842 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.842 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:04:29.842 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.821335+0000", 2023-09-23T21:04:29.842 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.7572999999999999e-05 2023-09-23T21:04:29.842 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.842 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.842 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:04:29.842 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.821344+0000", 2023-09-23T21:04:29.842 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.4009999999999997e-06 2023-09-23T21:04:29.842 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.843 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.843 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:04:29.843 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.821399+0000", 2023-09-23T21:04:29.843 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.5770000000000003e-05 2023-09-23T21:04:29.843 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.843 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.843 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T21:04:29.843 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.821410+0000", 2023-09-23T21:04:29.843 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0789e-05 2023-09-23T21:04:29.843 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.843 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.843 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:04:29.844 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.951308+0000", 2023-09-23T21:04:29.844 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.12989732600000001 2023-09-23T21:04:29.844 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.844 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.844 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T21:04:29.844 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:01.951315+0000", 2023-09-23T21:04:29.844 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.9449999999999993e-06 2023-09-23T21:04:29.844 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.844 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.844 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:04:29.844 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.260707+0000", 2023-09-23T21:04:29.844 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.309391529 2023-09-23T21:04:29.844 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.845 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.845 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for degraded object", 2023-09-23T21:04:29.845 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.261067+0000", 2023-09-23T21:04:29.845 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00035980499999999999 2023-09-23T21:04:29.845 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.845 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.845 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:04:29.845 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.317608+0000", 2023-09-23T21:04:29.845 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.056540900999999998 2023-09-23T21:04:29.845 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.845 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.845 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:04:29.845 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.317709+0000", 2023-09-23T21:04:29.846 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000101371 2023-09-23T21:04:29.846 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.846 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.846 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 1,5", 2023-09-23T21:04:29.846 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.318249+0000", 2023-09-23T21:04:29.846 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00054033200000000003 2023-09-23T21:04:29.846 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.846 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.846 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T21:04:29.846 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.321587+0000", 2023-09-23T21:04:29.846 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.003337904 2023-09-23T21:04:29.846 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.846 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.847 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:04:29.847 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.329047+0000", 2023-09-23T21:04:29.847 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0074594570000000001 2023-09-23T21:04:29.847 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.847 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.847 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:04:29.847 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.343443+0000", 2023-09-23T21:04:29.847 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.014395984000000001 2023-09-23T21:04:29.847 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.847 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.847 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T21:04:29.847 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.343740+0000", 2023-09-23T21:04:29.847 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000296886 2023-09-23T21:04:29.848 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.848 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.848 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T21:04:29.848 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.343991+0000", 2023-09-23T21:04:29.848 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00025143799999999999 2023-09-23T21:04:29.848 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:04:29.848 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:04:29.848 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:04:29.848 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.848 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.848 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34797.0:195032 6.10 6:08dbcd4e:::benchmark_data_smithi026_223868_object93906:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e181)", 2023-09-23T21:04:29.848 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T20:56:02.127089+0000", 2023-09-23T21:04:29.848 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 507.57776600599999, 2023-09-23T21:04:29.849 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.224242249, 2023-09-23T21:04:29.849 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:04:29.849 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:04:29.849 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T21:04:29.849 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34797", 2023-09-23T21:04:29.849 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T21:04:29.849 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 195032 2023-09-23T21:04:29.849 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.849 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:04:29.849 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.849 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:04:29.849 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:02.127089+0000", 2023-09-23T21:04:29.849 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:04:29.850 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.850 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.850 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:04:29.850 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:02.127089+0000", 2023-09-23T21:04:29.850 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:04:29.850 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.850 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.850 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:04:29.850 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:02.127088+0000", 2023-09-23T21:04:29.850 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T21:04:29.850 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.850 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.850 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:04:29.851 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:02.127094+0000", 2023-09-23T21:04:29.851 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.2330000000000003e-06 2023-09-23T21:04:29.851 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.851 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.851 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:04:29.851 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:02.127104+0000", 2023-09-23T21:04:29.851 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0019e-05 2023-09-23T21:04:29.851 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.851 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.851 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:04:29.851 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:02.127109+0000", 2023-09-23T21:04:29.851 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.4960000000000004e-06 2023-09-23T21:04:29.851 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.851 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.852 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:04:29.852 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:02.127158+0000", 2023-09-23T21:04:29.852 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.8550000000000001e-05 2023-09-23T21:04:29.852 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.852 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.852 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T21:04:29.852 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:02.127168+0000", 2023-09-23T21:04:29.852 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.8009999999999998e-06 2023-09-23T21:04:29.852 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.852 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.852 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:04:29.852 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.261102+0000", 2023-09-23T21:04:29.852 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1339344730000001 2023-09-23T21:04:29.853 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.853 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.853 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for degraded object", 2023-09-23T21:04:29.853 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.261501+0000", 2023-09-23T21:04:29.853 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00039857100000000002 2023-09-23T21:04:29.853 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.853 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.853 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:04:29.853 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.322489+0000", 2023-09-23T21:04:29.853 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.060988570999999998 2023-09-23T21:04:29.853 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.853 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.853 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:04:29.854 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.322601+0000", 2023-09-23T21:04:29.854 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000111908 2023-09-23T21:04:29.854 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.854 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.854 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 1,5", 2023-09-23T21:04:29.854 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.323185+0000", 2023-09-23T21:04:29.854 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00058331099999999999 2023-09-23T21:04:29.854 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.854 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.854 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T21:04:29.854 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.326510+0000", 2023-09-23T21:04:29.854 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.003325578 2023-09-23T21:04:29.855 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.855 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.855 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:04:29.855 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.329495+0000", 2023-09-23T21:04:29.855 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0029846249999999999 2023-09-23T21:04:29.855 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.855 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.855 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:04:29.855 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.350780+0000", 2023-09-23T21:04:29.855 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.021285182999999999 2023-09-23T21:04:29.855 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.855 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.856 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T21:04:29.856 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.350846+0000", 2023-09-23T21:04:29.856 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.5630999999999996e-05 2023-09-23T21:04:29.856 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.856 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.856 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T21:04:29.856 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:56:03.351331+0000", 2023-09-23T21:04:29.856 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00048559200000000002 2023-09-23T21:04:29.856 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:04:29.856 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:04:29.856 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:04:29.856 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.856 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.857 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35469.0:141052 8.11 8:89489a5f:::benchmark_data_smithi026_301269_object30295:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e287)", 2023-09-23T21:04:29.857 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T21:03:40.177394+0000", 2023-09-23T21:04:29.857 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 49.527460904000002, 2023-09-23T21:04:29.857 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1791958380000001, 2023-09-23T21:04:29.857 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:04:29.857 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:04:29.857 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T21:04:29.857 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35469", 2023-09-23T21:04:29.857 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/4041424623", 2023-09-23T21:04:29.857 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 141052 2023-09-23T21:04:29.857 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.857 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:04:29.857 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.858 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:04:29.858 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:03:40.177394+0000", 2023-09-23T21:04:29.858 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:04:29.858 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.858 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.858 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:04:29.858 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:03:40.177394+0000", 2023-09-23T21:04:29.858 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:04:29.858 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.858 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.858 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:04:29.858 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:03:40.177394+0000", 2023-09-23T21:04:29.858 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T21:04:29.859 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.859 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.859 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:04:29.859 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:03:40.177396+0000", 2023-09-23T21:04:29.859 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.5110000000000002e-06 2023-09-23T21:04:29.859 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.859 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.859 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:04:29.859 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:03:40.177403+0000", 2023-09-23T21:04:29.859 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.3829999999999997e-06 2023-09-23T21:04:29.859 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.859 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.860 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:04:29.860 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:03:40.177412+0000", 2023-09-23T21:04:29.860 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.3030000000000005e-06 2023-09-23T21:04:29.860 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.860 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.860 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:04:29.860 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:03:40.177607+0000", 2023-09-23T21:04:29.860 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00019547299999999999 2023-09-23T21:04:29.860 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.860 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.860 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T21:04:29.860 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:03:40.177612+0000", 2023-09-23T21:04:29.861 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.1919999999999998e-06 2023-09-23T21:04:29.861 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.861 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.861 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:04:29.861 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:03:41.339975+0000", 2023-09-23T21:04:29.861 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1623630839999999 2023-09-23T21:04:29.861 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.861 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.861 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:04:29.861 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:03:41.340055+0000", 2023-09-23T21:04:29.861 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.0049000000000006e-05 2023-09-23T21:04:29.861 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.862 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.862 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 1,2,5,6,7", 2023-09-23T21:04:29.862 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:03:41.340123+0000", 2023-09-23T21:04:29.862 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.8595000000000003e-05 2023-09-23T21:04:29.862 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.862 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.862 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:04:29.862 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:03:41.344452+0000", 2023-09-23T21:04:29.862 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0043291099999999997 2023-09-23T21:04:29.862 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.862 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.862 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T21:04:29.863 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:03:41.344463+0000", 2023-09-23T21:04:29.863 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1041e-05 2023-09-23T21:04:29.863 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.863 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.863 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:04:29.863 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:03:41.348591+0000", 2023-09-23T21:04:29.863 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0041270120000000002 2023-09-23T21:04:29.863 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.863 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.863 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:04:29.863 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:03:41.351266+0000", 2023-09-23T21:04:29.863 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0026757199999999999 2023-09-23T21:04:29.863 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.864 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.864 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:04:29.864 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:03:41.352296+0000", 2023-09-23T21:04:29.864 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.001029664 2023-09-23T21:04:29.864 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.864 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.864 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:04:29.864 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:03:41.356420+0000", 2023-09-23T21:04:29.864 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0041241799999999999 2023-09-23T21:04:29.864 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.864 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.865 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T21:04:29.865 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:03:41.356451+0000", 2023-09-23T21:04:29.865 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.1276999999999999e-05 2023-09-23T21:04:29.865 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.865 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.865 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T21:04:29.865 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:03:41.356590+0000", 2023-09-23T21:04:29.865 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000138622 2023-09-23T21:04:29.865 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:04:29.865 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:04:29.865 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:04:29.865 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.865 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.866 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35469.0:141053 8.11 8:8ac65d6e:::benchmark_data_smithi026_301269_object30296:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e287)", 2023-09-23T21:04:29.866 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T21:03:40.177417+0000", 2023-09-23T21:04:29.866 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 49.527437601000003, 2023-09-23T21:04:29.866 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1794388659999999, 2023-09-23T21:04:29.866 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:04:29.866 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:04:29.866 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T21:04:29.866 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35469", 2023-09-23T21:04:29.866 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/4041424623", 2023-09-23T21:04:29.866 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 141053 2023-09-23T21:04:29.866 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.866 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:04:29.866 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.867 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:04:29.867 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:03:40.177417+0000", 2023-09-23T21:04:29.867 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:04:29.867 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.867 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.867 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:04:29.867 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:03:40.177417+0000", 2023-09-23T21:04:29.867 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:04:29.867 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.867 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.867 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:04:29.867 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:03:40.177417+0000", 2023-09-23T21:04:29.868 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T21:04:29.868 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.868 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.868 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:04:29.868 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:03:40.177420+0000", 2023-09-23T21:04:29.868 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.5110000000000002e-06 2023-09-23T21:04:29.868 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.868 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.868 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:04:29.868 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:03:40.177432+0000", 2023-09-23T21:04:29.868 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.2452e-05 2023-09-23T21:04:29.868 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.869 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.869 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:04:29.869 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:03:40.177438+0000", 2023-09-23T21:04:29.869 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.9989999999999997e-06 2023-09-23T21:04:29.869 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.869 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.869 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:04:29.869 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:03:40.177634+0000", 2023-09-23T21:04:29.869 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000195997 2023-09-23T21:04:29.869 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.869 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.869 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T21:04:29.869 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:03:40.177638+0000", 2023-09-23T21:04:29.870 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.8139999999999998e-06 2023-09-23T21:04:29.870 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.870 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.870 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:04:29.870 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:03:41.341453+0000", 2023-09-23T21:04:29.870 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.163815233 2023-09-23T21:04:29.870 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.870 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.870 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:04:29.870 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:03:41.341574+0000", 2023-09-23T21:04:29.870 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000121192 2023-09-23T21:04:29.870 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.870 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.871 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 1,2,5,6,7", 2023-09-23T21:04:29.871 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:03:41.341694+0000", 2023-09-23T21:04:29.871 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00011974800000000001 2023-09-23T21:04:29.871 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.871 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.871 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T21:04:29.871 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:03:41.344484+0000", 2023-09-23T21:04:29.871 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.002790276 2023-09-23T21:04:29.871 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.871 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.871 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:04:29.871 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:03:41.344511+0000", 2023-09-23T21:04:29.872 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.6458e-05 2023-09-23T21:04:29.872 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.872 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.872 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:04:29.872 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:03:41.348631+0000", 2023-09-23T21:04:29.872 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0041203580000000002 2023-09-23T21:04:29.872 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.872 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.872 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:04:29.872 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:03:41.352333+0000", 2023-09-23T21:04:29.872 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0037016140000000002 2023-09-23T21:04:29.872 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.872 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.873 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:04:29.873 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:03:41.353086+0000", 2023-09-23T21:04:29.873 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00075337099999999997 2023-09-23T21:04:29.873 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.873 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.873 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:04:29.873 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:03:41.356654+0000", 2023-09-23T21:04:29.873 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0035683809999999998 2023-09-23T21:04:29.873 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.873 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.873 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T21:04:29.873 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:03:41.356715+0000", 2023-09-23T21:04:29.874 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.0259e-05 2023-09-23T21:04:29.874 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.874 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.874 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T21:04:29.874 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:03:41.356856+0000", 2023-09-23T21:04:29.874 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000141636 2023-09-23T21:04:29.874 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:04:29.874 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:04:29.874 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:04:29.874 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.874 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.874 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35469.0:141120 8.11 8:8b730a96:::benchmark_data_smithi026_301269_object30363:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e287)", 2023-09-23T21:04:29.874 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T21:03:40.177443+0000", 2023-09-23T21:04:29.875 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 49.527412378000001, 2023-09-23T21:04:29.875 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1796452230000001, 2023-09-23T21:04:29.875 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:04:29.875 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:04:29.875 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T21:04:29.875 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35469", 2023-09-23T21:04:29.875 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/4041424623", 2023-09-23T21:04:29.875 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 141120 2023-09-23T21:04:29.875 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.875 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:04:29.875 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.875 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:04:29.876 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:03:40.177443+0000", 2023-09-23T21:04:29.876 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:04:29.876 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.876 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.876 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:04:29.876 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:03:40.177443+0000", 2023-09-23T21:04:29.876 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:04:29.876 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.876 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.876 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:04:29.876 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:03:40.177442+0000", 2023-09-23T21:04:29.876 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T21:04:29.876 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.877 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.877 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:04:29.877 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:03:40.177444+0000", 2023-09-23T21:04:29.877 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.6449999999999999e-06 2023-09-23T21:04:29.877 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.877 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.877 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:04:29.877 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:03:40.177448+0000", 2023-09-23T21:04:29.877 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.602e-06 2023-09-23T21:04:29.877 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.877 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.877 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:04:29.877 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:03:40.177451+0000", 2023-09-23T21:04:29.878 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.9610000000000001e-06 2023-09-23T21:04:29.878 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.878 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.878 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:04:29.878 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:03:40.177661+0000", 2023-09-23T21:04:29.878 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000209984 2023-09-23T21:04:29.878 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.878 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.878 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T21:04:29.878 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:03:40.177665+0000", 2023-09-23T21:04:29.878 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.2189999999999998e-06 2023-09-23T21:04:29.878 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.879 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.879 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:04:29.879 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:03:41.344534+0000", 2023-09-23T21:04:29.879 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.16686954 2023-09-23T21:04:29.879 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.879 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.879 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:04:29.879 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:03:41.344663+0000", 2023-09-23T21:04:29.879 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000128441 2023-09-23T21:04:29.879 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.879 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.879 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 1,2,5,6,7", 2023-09-23T21:04:29.879 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:03:41.344796+0000", 2023-09-23T21:04:29.880 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00013324499999999999 2023-09-23T21:04:29.880 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.880 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.880 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T21:04:29.880 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:03:41.348599+0000", 2023-09-23T21:04:29.880 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0038027040000000001 2023-09-23T21:04:29.880 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.880 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.880 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:04:29.880 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:03:41.348747+0000", 2023-09-23T21:04:29.880 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00014788299999999999 2023-09-23T21:04:29.880 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.880 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.880 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:04:29.881 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:03:41.352359+0000", 2023-09-23T21:04:29.881 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0036121980000000001 2023-09-23T21:04:29.881 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.881 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.881 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:04:29.881 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:03:41.353175+0000", 2023-09-23T21:04:29.881 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00081627299999999996 2023-09-23T21:04:29.881 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.881 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.881 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:04:29.881 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:03:41.354019+0000", 2023-09-23T21:04:29.881 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00084438700000000005 2023-09-23T21:04:29.881 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.882 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.882 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:04:29.882 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:03:41.356905+0000", 2023-09-23T21:04:29.882 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0028856699999999999 2023-09-23T21:04:29.882 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.882 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.882 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T21:04:29.882 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:03:41.356952+0000", 2023-09-23T21:04:29.882 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.6471999999999998e-05 2023-09-23T21:04:29.882 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.882 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.882 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T21:04:29.882 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:03:41.357088+0000", 2023-09-23T21:04:29.883 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00013626 2023-09-23T21:04:29.883 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:04:29.883 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:04:29.883 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:04:29.883 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.883 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.883 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35469.0:141157 8.11 8:88baa66d:::benchmark_data_smithi026_301269_object30400:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e287)", 2023-09-23T21:04:29.883 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T21:03:40.177463+0000", 2023-09-23T21:04:29.883 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 49.527391713, 2023-09-23T21:04:29.883 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.181319032, 2023-09-23T21:04:29.883 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:04:29.883 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:04:29.883 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T21:04:29.884 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35469", 2023-09-23T21:04:29.884 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/4041424623", 2023-09-23T21:04:29.884 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 141157 2023-09-23T21:04:29.884 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.884 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:04:29.884 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.884 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:04:29.884 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:03:40.177463+0000", 2023-09-23T21:04:29.884 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:04:29.884 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.884 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.884 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:04:29.884 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:03:40.177463+0000", 2023-09-23T21:04:29.885 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:04:29.885 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.885 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.885 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:04:29.885 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:03:40.177463+0000", 2023-09-23T21:04:29.885 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T21:04:29.885 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.885 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.885 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:04:29.885 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:03:40.177465+0000", 2023-09-23T21:04:29.885 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.3539999999999998e-06 2023-09-23T21:04:29.885 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.885 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.886 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:04:29.886 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:03:40.177471+0000", 2023-09-23T21:04:29.886 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.1149999999999999e-06 2023-09-23T21:04:29.886 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.886 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.886 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:04:29.886 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:03:40.177482+0000", 2023-09-23T21:04:29.886 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0020999999999999e-05 2023-09-23T21:04:29.886 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.886 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.886 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:04:29.886 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:03:40.177677+0000", 2023-09-23T21:04:29.886 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00019567099999999999 2023-09-23T21:04:29.887 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.887 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.887 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T21:04:29.887 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:03:40.177681+0000", 2023-09-23T21:04:29.887 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.507e-06 2023-09-23T21:04:29.887 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.887 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.887 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:04:29.887 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:03:41.346625+0000", 2023-09-23T21:04:29.887 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.168943949 2023-09-23T21:04:29.887 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.887 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.887 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:04:29.887 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:03:41.346740+0000", 2023-09-23T21:04:29.887 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000114907 2023-09-23T21:04:29.888 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.888 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.888 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 1,2,5,6,7", 2023-09-23T21:04:29.888 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:03:41.346855+0000", 2023-09-23T21:04:29.888 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000115623 2023-09-23T21:04:29.888 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.888 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.888 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:04:29.888 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:03:41.351219+0000", 2023-09-23T21:04:29.888 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0043639450000000002 2023-09-23T21:04:29.888 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.888 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.888 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T21:04:29.889 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:03:41.351277+0000", 2023-09-23T21:04:29.889 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.8063000000000003e-05 2023-09-23T21:04:29.889 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.889 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.889 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:04:29.889 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:03:41.352384+0000", 2023-09-23T21:04:29.889 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.001107308 2023-09-23T21:04:29.889 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.889 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.889 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:04:29.889 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:03:41.353975+0000", 2023-09-23T21:04:29.889 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.001590649 2023-09-23T21:04:29.889 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.890 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.890 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:04:29.890 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:03:41.354058+0000", 2023-09-23T21:04:29.890 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.3090000000000006e-05 2023-09-23T21:04:29.890 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.890 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.890 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:04:29.890 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:03:41.358570+0000", 2023-09-23T21:04:29.890 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0045119549999999998 2023-09-23T21:04:29.890 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.890 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.890 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T21:04:29.890 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:03:41.358608+0000", 2023-09-23T21:04:29.891 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.8098e-05 2023-09-23T21:04:29.891 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:29.891 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:29.891 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T21:04:29.891 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:03:41.358782+0000", 2023-09-23T21:04:29.891 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00017408300000000001 2023-09-23T21:04:29.891 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:04:29.891 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:04:29.891 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:04:29.891 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:04:29.891 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:04:29.891 INFO:teuthology.orchestra.run.smithi026.stdout:} 2023-09-23T21:04:29.892 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:04:29.967 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:04:30.188 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:04:30.194 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:04:30.195 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:04:30.196 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:04:30.243 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_ops_in_flight 2023-09-23T21:04:30.256 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T21:04:30.295 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:04:30.332 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:04:30.373 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:04:30.376 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:04:30.377 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:04:30.477 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:04:30.584 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:04:30 smithi026 ceph-mon[175151]: pgmap v817: 119 pgs: 1 active+remapped+backfill_wait, 1 active+remapped+backfilling, 117 active+clean; 3.1 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 397 op/s; 2207/153741 objects misplaced (1.436%); 26 MiB/s, 412 objects/s recovering 2023-09-23T21:04:30.584 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:04:30 smithi026 ceph-mon[175151]: failed to encode map e318 with expected crc 2023-09-23T21:04:30.584 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:04:30 smithi026 ceph-mon[175151]: failed to encode map e318 with expected crc 2023-09-23T21:04:30.584 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:04:30 smithi026 ceph-mon[175151]: failed to encode map e318 with expected crc 2023-09-23T21:04:30.584 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:04:30 smithi026 ceph-mon[175151]: osdmap e318: 8 total, 8 up, 8 in 2023-09-23T21:04:30.584 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:04:30 smithi026 ceph-mon[175151]: 8.16 scrub starts 2023-09-23T21:04:30.584 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:04:30 smithi026 ceph-mon[179688]: pgmap v817: 119 pgs: 1 active+remapped+backfill_wait, 1 active+remapped+backfilling, 117 active+clean; 3.1 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 397 op/s; 2207/153741 objects misplaced (1.436%); 26 MiB/s, 412 objects/s recovering 2023-09-23T21:04:30.585 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:04:30 smithi026 ceph-mon[179688]: failed to encode map e318 with expected crc 2023-09-23T21:04:30.585 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:04:30 smithi026 ceph-mon[179688]: failed to encode map e318 with expected crc 2023-09-23T21:04:30.585 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:04:30 smithi026 ceph-mon[179688]: failed to encode map e318 with expected crc 2023-09-23T21:04:30.585 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:04:30 smithi026 ceph-mon[179688]: osdmap e318: 8 total, 8 up, 8 in 2023-09-23T21:04:30.585 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:04:30 smithi026 ceph-mon[179688]: 8.16 scrub starts 2023-09-23T21:04:30.585 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:04:30.621 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:04:30 smithi070 ceph-mon[168282]: pgmap v817: 119 pgs: 1 active+remapped+backfill_wait, 1 active+remapped+backfilling, 117 active+clean; 3.1 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 397 op/s; 2207/153741 objects misplaced (1.436%); 26 MiB/s, 412 objects/s recovering 2023-09-23T21:04:30.621 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:04:30 smithi070 ceph-mon[168282]: failed to encode map e318 with expected crc 2023-09-23T21:04:30.621 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:04:30 smithi070 ceph-mon[168282]: failed to encode map e318 with expected crc 2023-09-23T21:04:30.621 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:04:30 smithi070 ceph-mon[168282]: failed to encode map e318 with expected crc 2023-09-23T21:04:30.621 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:04:30 smithi070 ceph-mon[168282]: osdmap e318: 8 total, 8 up, 8 in 2023-09-23T21:04:30.621 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:04:30 smithi070 ceph-mon[168282]: 8.16 scrub starts 2023-09-23T21:04:30.683 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T21:04:30.699 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:04:30.700 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:04:30.800 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:04:30.832 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:04:30.882 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:04:30.888 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:04:30.888 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:04:30.989 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:04:31.099 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:04:31.176 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T21:04:31.185 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:04:31.186 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:04:31.286 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:04:31.336 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:04:31.412 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:04:31.427 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:04:31.427 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:04:31.436 INFO:teuthology.orchestra.run.smithi026.stderr:noscrub is set 2023-09-23T21:04:31.473 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:04:31 smithi026 ceph-mon[179688]: 8.15 scrub starts 2023-09-23T21:04:31.473 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:04:31 smithi026 ceph-mon[179688]: 8.15 scrub ok 2023-09-23T21:04:31.473 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:04:31 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/2338561277' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T21:04:31.473 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:04:31 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T21:04:31.473 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:04:31 smithi026 ceph-mon[179688]: 8.8 scrub starts 2023-09-23T21:04:31.473 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:04:31 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:04:31.473 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:04:31 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:04:31.474 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:04:31 smithi026 ceph-mon[175151]: 8.15 scrub starts 2023-09-23T21:04:31.474 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:04:31 smithi026 ceph-mon[175151]: 8.15 scrub ok 2023-09-23T21:04:31.474 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:04:31 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/2338561277' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T21:04:31.474 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:04:31 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T21:04:31.474 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:04:31 smithi026 ceph-mon[175151]: 8.8 scrub starts 2023-09-23T21:04:31.474 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:04:31 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:04:31.474 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:04:31 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:04:31.528 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:04:31.597 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:04:31.610 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:04:31.720 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:04:31.725 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:04:31.726 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:04:31.827 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:04:31.860 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:04:31.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:04:31 smithi070 ceph-mon[168282]: 8.15 scrub starts 2023-09-23T21:04:31.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:04:31 smithi070 ceph-mon[168282]: 8.15 scrub ok 2023-09-23T21:04:31.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:04:31 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/2338561277' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T21:04:31.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:04:31 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T21:04:31.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:04:31 smithi070 ceph-mon[168282]: 8.8 scrub starts 2023-09-23T21:04:31.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:04:31 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:04:31.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:04:31 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:04:31.910 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:04:32.198 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:04:32.201 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:04:32.202 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:04:32.303 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:04:32.336 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:04:32.634 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:04:32 smithi026 ceph-mon[179688]: 8.17 scrub starts 2023-09-23T21:04:32.634 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:04:32 smithi026 ceph-mon[179688]: pgmap v819: 119 pgs: 1 active+remapped+backfill_wait, 1 active+remapped+backfilling, 117 active+clean; 3.0 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 2207/148230 objects misplaced (1.489%); 3.8 MiB/s, 59 objects/s recovering 2023-09-23T21:04:32.634 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:04:32 smithi026 ceph-mon[179688]: 8.18 deep-scrub starts 2023-09-23T21:04:32.634 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:04:32 smithi026 ceph-mon[179688]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:04:32.634 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:04:32 smithi026 ceph-mon[179688]: failed to encode map e319 with expected crc 2023-09-23T21:04:32.634 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:04:32 smithi026 ceph-mon[179688]: failed to encode map e319 with expected crc 2023-09-23T21:04:32.634 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:04:32 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T21:04:32.635 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:04:32 smithi026 ceph-mon[179688]: osdmap e319: 8 total, 8 up, 8 in 2023-09-23T21:04:32.635 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:04:32 smithi026 ceph-mon[179688]: failed to encode map e319 with expected crc 2023-09-23T21:04:32.635 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:04:32 smithi026 ceph-mon[175151]: 8.17 scrub starts 2023-09-23T21:04:32.635 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:04:32 smithi026 ceph-mon[175151]: pgmap v819: 119 pgs: 1 active+remapped+backfill_wait, 1 active+remapped+backfilling, 117 active+clean; 3.0 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 2207/148230 objects misplaced (1.489%); 3.8 MiB/s, 59 objects/s recovering 2023-09-23T21:04:32.635 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:04:32 smithi026 ceph-mon[175151]: 8.18 deep-scrub starts 2023-09-23T21:04:32.635 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:04:32 smithi026 ceph-mon[175151]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:04:32.635 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:04:32 smithi026 ceph-mon[175151]: failed to encode map e319 with expected crc 2023-09-23T21:04:32.635 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:04:32 smithi026 ceph-mon[175151]: failed to encode map e319 with expected crc 2023-09-23T21:04:32.636 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:04:32 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T21:04:32.636 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:04:32 smithi026 ceph-mon[175151]: osdmap e319: 8 total, 8 up, 8 in 2023-09-23T21:04:32.636 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:04:32 smithi026 ceph-mon[175151]: failed to encode map e319 with expected crc 2023-09-23T21:04:32.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:04:32 smithi070 ceph-mon[168282]: 8.17 scrub starts 2023-09-23T21:04:32.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:04:32 smithi070 ceph-mon[168282]: pgmap v819: 119 pgs: 1 active+remapped+backfill_wait, 1 active+remapped+backfilling, 117 active+clean; 3.0 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 2207/148230 objects misplaced (1.489%); 3.8 MiB/s, 59 objects/s recovering 2023-09-23T21:04:32.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:04:32 smithi070 ceph-mon[168282]: 8.18 deep-scrub starts 2023-09-23T21:04:32.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:04:32 smithi070 ceph-mon[168282]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:04:32.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:04:32 smithi070 ceph-mon[168282]: failed to encode map e319 with expected crc 2023-09-23T21:04:32.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:04:32 smithi070 ceph-mon[168282]: failed to encode map e319 with expected crc 2023-09-23T21:04:32.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:04:32 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T21:04:32.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:04:32 smithi070 ceph-mon[168282]: osdmap e319: 8 total, 8 up, 8 in 2023-09-23T21:04:32.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:04:32 smithi070 ceph-mon[168282]: failed to encode map e319 with expected crc 2023-09-23T21:04:33.103 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:04:33.112 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:04:33.112 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:04:33.213 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:04:33.318 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:04:33.441 INFO:teuthology.orchestra.run.smithi026.stderr:reweighted osd.5 to 1 (10000) 2023-09-23T21:04:33.747 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T21:04:33.751 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:04:33.752 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:04:33.756 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:04:33 smithi026 ceph-mon[175151]: failed to encode map e320 with expected crc 2023-09-23T21:04:33.756 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:04:33 smithi026 ceph-mon[175151]: failed to encode map e320 with expected crc 2023-09-23T21:04:33.756 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:04:33 smithi026 ceph-mon[175151]: failed to encode map e320 with expected crc 2023-09-23T21:04:33.756 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:04:33 smithi026 ceph-mon[175151]: osdmap e320: 8 total, 8 up, 8 in 2023-09-23T21:04:33.756 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:04:33 smithi026 ceph-mon[175151]: failed to encode map e320 with expected crc 2023-09-23T21:04:33.756 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:04:33 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/2401291022' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-09-23T21:04:33.756 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:04:33 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-09-23T21:04:33.756 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:04:33 smithi026 ceph-mon[179688]: failed to encode map e320 with expected crc 2023-09-23T21:04:33.757 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:04:33 smithi026 ceph-mon[179688]: failed to encode map e320 with expected crc 2023-09-23T21:04:33.757 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:04:33 smithi026 ceph-mon[179688]: failed to encode map e320 with expected crc 2023-09-23T21:04:33.757 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:04:33 smithi026 ceph-mon[179688]: osdmap e320: 8 total, 8 up, 8 in 2023-09-23T21:04:33.757 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:04:33 smithi026 ceph-mon[179688]: failed to encode map e320 with expected crc 2023-09-23T21:04:33.757 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:04:33 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/2401291022' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-09-23T21:04:33.757 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:04:33 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-09-23T21:04:33.810 INFO:teuthology.orchestra.run.smithi026.stdout:{ 2023-09-23T21:04:33.810 INFO:teuthology.orchestra.run.smithi026.stdout: "ops": [ 2023-09-23T21:04:33.810 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:33.810 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_repop(client.35469.0:176126 8.b e321/254)", 2023-09-23T21:04:33.810 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T21:04:33.807762+0000", 2023-09-23T21:04:33.810 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 0.0020606270000000002, 2023-09-23T21:04:33.810 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.002090457, 2023-09-23T21:04:33.811 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:04:33.811 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "queued for pg", 2023-09-23T21:04:33.811 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:04:33.811 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:33.811 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:04:33.811 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:33.807762+0000", 2023-09-23T21:04:33.811 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:04:33.811 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:33.811 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:33.811 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:04:33.811 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:33.807762+0000", 2023-09-23T21:04:33.811 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:04:33.811 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:33.812 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:33.812 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:04:33.812 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:33.807761+0000", 2023-09-23T21:04:33.812 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.999999 2023-09-23T21:04:33.812 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:33.812 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:33.812 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:04:33.812 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:33.807769+0000", 2023-09-23T21:04:33.812 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.028e-06 2023-09-23T21:04:33.812 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:33.812 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:33.812 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:04:33.813 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:33.807782+0000", 2023-09-23T21:04:33.813 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.2927e-05 2023-09-23T21:04:33.813 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:33.813 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:33.813 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:04:33.813 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:33.807791+0000", 2023-09-23T21:04:33.813 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.6209999999999993e-06 2023-09-23T21:04:33.813 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:04:33.813 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:04:33.813 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:04:33.813 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:04:33.813 INFO:teuthology.orchestra.run.smithi026.stdout: ], 2023-09-23T21:04:33.813 INFO:teuthology.orchestra.run.smithi026.stdout: "num_ops": 1 2023-09-23T21:04:33.814 INFO:teuthology.orchestra.run.smithi026.stdout:} 2023-09-23T21:04:33.853 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:04:33.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:04:33 smithi070 ceph-mon[168282]: failed to encode map e320 with expected crc 2023-09-23T21:04:33.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:04:33 smithi070 ceph-mon[168282]: failed to encode map e320 with expected crc 2023-09-23T21:04:33.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:04:33 smithi070 ceph-mon[168282]: failed to encode map e320 with expected crc 2023-09-23T21:04:33.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:04:33 smithi070 ceph-mon[168282]: osdmap e320: 8 total, 8 up, 8 in 2023-09-23T21:04:33.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:04:33 smithi070 ceph-mon[168282]: failed to encode map e320 with expected crc 2023-09-23T21:04:33.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:04:33 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/2401291022' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-09-23T21:04:33.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:04:33 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-09-23T21:04:33.942 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:04:34.174 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:04:34.181 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:04:34.181 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:04:34.266 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T21:04:34.282 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:04:34.327 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:04:34.380 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:04:34.383 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:04:34.384 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:04:34.412 INFO:tasks.cephadm.ceph_manager.ceph:waiting for recovery to complete 2023-09-23T21:04:34.412 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T21:04:34.485 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:04:34.519 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:04:34.562 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:04:34.567 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:04:34.567 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:04:34.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:04:34 smithi070 ceph-mon[168282]: pgmap v822: 119 pgs: 1 peering, 118 active+clean; 2.9 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 4.0 MiB/s, 63 objects/s recovering 2023-09-23T21:04:34.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:04:34 smithi070 ceph-mon[168282]: failed to encode map e321 with expected crc 2023-09-23T21:04:34.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:04:34 smithi070 ceph-mon[168282]: failed to encode map e321 with expected crc 2023-09-23T21:04:34.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:04:34 smithi070 ceph-mon[168282]: failed to encode map e321 with expected crc 2023-09-23T21:04:34.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:04:34 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-09-23T21:04:34.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:04:34 smithi070 ceph-mon[168282]: osdmap e321: 8 total, 8 up, 8 in 2023-09-23T21:04:34.621 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:04:34 smithi026 ceph-mon[175151]: pgmap v822: 119 pgs: 1 peering, 118 active+clean; 2.9 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 4.0 MiB/s, 63 objects/s recovering 2023-09-23T21:04:34.621 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:04:34 smithi026 ceph-mon[175151]: failed to encode map e321 with expected crc 2023-09-23T21:04:34.621 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:04:34 smithi026 ceph-mon[175151]: failed to encode map e321 with expected crc 2023-09-23T21:04:34.621 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:04:34 smithi026 ceph-mon[175151]: failed to encode map e321 with expected crc 2023-09-23T21:04:34.621 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:04:34 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-09-23T21:04:34.621 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:04:34 smithi026 ceph-mon[175151]: osdmap e321: 8 total, 8 up, 8 in 2023-09-23T21:04:34.622 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:04:34 smithi026 ceph-mon[179688]: pgmap v822: 119 pgs: 1 peering, 118 active+clean; 2.9 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 4.0 MiB/s, 63 objects/s recovering 2023-09-23T21:04:34.622 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:04:34 smithi026 ceph-mon[179688]: failed to encode map e321 with expected crc 2023-09-23T21:04:34.622 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:04:34 smithi026 ceph-mon[179688]: failed to encode map e321 with expected crc 2023-09-23T21:04:34.622 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:04:34 smithi026 ceph-mon[179688]: failed to encode map e321 with expected crc 2023-09-23T21:04:34.622 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:04:34 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-09-23T21:04:34.622 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:04:34 smithi026 ceph-mon[179688]: osdmap e321: 8 total, 8 up, 8 in 2023-09-23T21:04:34.659 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_blocked_ops 2023-09-23T21:04:34.668 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:04:34.701 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:04:34.745 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:04:34.750 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:04:34.750 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:04:34.853 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:04:35.024 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:04:35.152 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:04:35.154 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:04:35.155 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:04:35.255 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:04:35.302 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0: {} 2023-09-23T21:04:35.321 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:04:35.353 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:04:35.356 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1: {} 2023-09-23T21:04:35.413 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2: {} 2023-09-23T21:04:35.425 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T21:04:35.429 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:04:35.429 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:04:35.459 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3: {} 2023-09-23T21:04:35.506 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4: {} 2023-09-23T21:04:35.525 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:04:35.530 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:04:35.538 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:04:35.559 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T21:04:35.563 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:04:35.597 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6: {} 2023-09-23T21:04:35.607 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:04:35.611 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:04:35.611 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:04:35.634 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T21:04:35.660 INFO:teuthology.orchestra.run.smithi026.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T21:04:35.660 INFO:teuthology.orchestra.run.smithi026.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T21:04:35.661 INFO:teuthology.orchestra.run.smithi026.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T21:04:35.661 INFO:teuthology.orchestra.run.smithi026.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T21:04:35.661 INFO:teuthology.orchestra.run.smithi026.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T21:04:35.661 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T21:04:35.661 INFO:teuthology.orchestra.run.smithi026.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T21:04:35.661 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T21:04:35.712 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:04:35.748 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:04:35.810 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:04:35.818 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:04:35.818 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:04:35.919 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:04:35.959 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:04:36.009 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:04:36.012 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:04:36.012 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:04:36.113 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:04:36.163 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:04:36.220 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:04:36.229 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:04:36.229 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:04:36.330 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:04:36.373 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:04:36 smithi026 ceph-mon[179688]: pgmap v824: 119 pgs: 1 peering, 118 active+clean; 2.8 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 4.2 MiB/s, 66 objects/s recovering 2023-09-23T21:04:36.373 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:04:36 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/1222552151' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:04:36.374 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:04:36 smithi026 ceph-mon[175151]: pgmap v824: 119 pgs: 1 peering, 118 active+clean; 2.8 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 4.2 MiB/s, 66 objects/s recovering 2023-09-23T21:04:36.374 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:04:36 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/1222552151' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:04:36.424 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:04:36.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:04:36 smithi070 ceph-mon[168282]: pgmap v824: 119 pgs: 1 peering, 118 active+clean; 2.8 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 4.2 MiB/s, 66 objects/s recovering 2023-09-23T21:04:36.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:04:36 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/1222552151' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:04:37.082 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:04:37.088 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:04:37.088 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:04:37.189 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:04:37.228 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:04:37.278 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:04:37.284 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:04:37.284 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:04:37.385 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:04:37.423 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:04:37.464 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:04:37.473 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:04:37.474 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:04:37.575 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:04:37.652 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:04:38.002 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:04:38.016 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:04:38.016 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:04:38.104 INFO:teuthology.orchestra.run.smithi026.stdout:ERROR: (22) Invalid argument 2023-09-23T21:04:38.104 INFO:teuthology.orchestra.run.smithi026.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T21:04:38.118 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:04:38.198 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:04:38.283 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:04:38 smithi026 ceph-mon[175151]: pgmap v825: 119 pgs: 1 peering, 118 active+clean; 2.8 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 3.8 MiB/s, 60 objects/s recovering 2023-09-23T21:04:38.287 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:04:38 smithi026 ceph-mon[179688]: pgmap v825: 119 pgs: 1 peering, 118 active+clean; 2.8 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 3.8 MiB/s, 60 objects/s recovering 2023-09-23T21:04:38.345 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:04:38.364 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:04:38.365 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:04:38.465 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:04:38.505 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T21:04:38.555 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:04:38.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:04:38 smithi070 ceph-mon[168282]: pgmap v825: 119 pgs: 1 peering, 118 active+clean; 2.8 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 3.8 MiB/s, 60 objects/s recovering 2023-09-23T21:04:38.926 INFO:teuthology.orchestra.run.smithi026.stdout: 2023-09-23T21:04:38.945 INFO:teuthology.orchestra.run.smithi026.stderr:dumped all 2023-09-23T21:04:39.685 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_historic_ops 2023-09-23T21:04:39.711 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:04:39.724 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:04:39.724 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:04:39.825 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:04:39.881 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 21:04:39 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[164029]: ::ffff:172.21.15.70 - - [23/Sep/2023:21:04:39] "GET /metrics HTTP/1.1" 200 36310 "" "Prometheus/2.43.0" 2023-09-23T21:04:39.895 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:04:40.095 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T21:04:40.107 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:04:40.107 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:04:40.114 INFO:teuthology.orchestra.run.smithi026.stdout:{"pg_ready":true,"pg_map":{"version":826,"stamp":"2023-09-23T21:04:38.781549+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":2836436920,"num_objects":43296,"num_object_clones":0,"num_object_copies":129888,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":43296,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":135703,"num_write_kb":5093534,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":15274,"num_bytes_recovered":999298918,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":214331,"ondisk_log_size":214331,"up":357,"acting":357,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":376,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":12536220,"kb_used_data":9852636,"kb_used_omap":136,"kb_used_meta":2683191,"kb_avail":737424996,"statfs":{"total":767960285184,"available":755123195904,"internally_reserved":0,"allocated":10089099264,"data_stored":10079015880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":139741,"internal_metadata":2747588131},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,1,2,5],"upper_bound":32},"perf_stat":{"commit_latency_ms":4,"apply_latency_ms":4,"commit_latency_ns":4000000,"apply_latency_ns":4000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-400687104,"num_objects":-6114,"num_object_clones":0,"num_object_copies":-18342,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":-2207,"num_objects_unfound":0,"num_objects_dirty":-6114,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":649,"num_write_kb":-223392,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":364,"num_bytes_recovered":23920640,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"7.383438"},"pg_stats":[{"pgid":"8.11","version":"321'9911","reported_seq":19232,"reported_epoch":321,"state":"active+clean","last_fresh":"2023-09-23T21:04:37.064665+0000","last_change":"2023-09-23T21:04:04.904208+0000","last_active":"2023-09-23T21:04:37.064665+0000","last_peered":"2023-09-23T21:04:37.064665+0000","last_clean":"2023-09-23T21:04:37.064665+0000","last_became_active":"2023-09-23T21:04:04.892614+0000","last_became_peered":"2023-09-23T21:04:04.892614+0000","last_unstale":"2023-09-23T21:04:37.064665+0000","last_undegraded":"2023-09-23T21:04:37.064665+0000","last_fullsized":"2023-09-23T21:04:37.064665+0000","mapping_epoch":300,"log_start":"258'1600","ondisk_log_start":"258'1600","created":281,"last_epoch_clean":301,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:01:24.325318+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:01:24.325318+0000","last_clean_scrub_stamp":"2023-09-23T21:01:24.325318+0000","objects_scrubbed":0,"log_size":8311,"log_dups_size":0,"ondisk_log_size":8311,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90046464,"num_objects":1374,"num_object_clones":0,"num_object_copies":4122,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1374,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5672,"num_write_kb":225472,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2304,"num_bytes_recovered":150929408,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1a","version":"68'1","reported_seq":198,"reported_epoch":320,"state":"active+clean","last_fresh":"2023-09-23T21:04:32.400778+0000","last_change":"2023-09-23T20:54:13.946695+0000","last_active":"2023-09-23T21:04:32.400778+0000","last_peered":"2023-09-23T21:04:32.400778+0000","last_clean":"2023-09-23T21:04:32.400778+0000","last_became_active":"2023-09-23T20:54:13.945399+0000","last_became_peered":"2023-09-23T20:54:13.945399+0000","last_unstale":"2023-09-23T21:04:32.400778+0000","last_undegraded":"2023-09-23T21:04:32.400778+0000","last_fullsized":"2023-09-23T21:04:32.400778+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:32:52.365464+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":315,"reported_epoch":320,"state":"active+clean","last_fresh":"2023-09-23T21:04:32.423656+0000","last_change":"2023-09-23T20:54:13.942019+0000","last_active":"2023-09-23T21:04:32.423656+0000","last_peered":"2023-09-23T21:04:32.423656+0000","last_clean":"2023-09-23T21:04:32.423656+0000","last_became_active":"2023-09-23T20:54:13.941647+0000","last_became_peered":"2023-09-23T20:54:13.941647+0000","last_unstale":"2023-09-23T21:04:32.423656+0000","last_undegraded":"2023-09-23T21:04:32.423656+0000","last_fullsized":"2023-09-23T21:04:32.423656+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:30:54.400944+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.12","version":"321'9777","reported_seq":17500,"reported_epoch":321,"state":"active+clean","last_fresh":"2023-09-23T21:04:34.119754+0000","last_change":"2023-09-23T21:04:27.883034+0000","last_active":"2023-09-23T21:04:34.119754+0000","last_peered":"2023-09-23T21:04:34.119754+0000","last_clean":"2023-09-23T21:04:34.119754+0000","last_became_active":"2023-09-23T21:04:14.222393+0000","last_became_peered":"2023-09-23T21:04:14.222393+0000","last_unstale":"2023-09-23T21:04:34.119754+0000","last_undegraded":"2023-09-23T21:04:34.119754+0000","last_fullsized":"2023-09-23T21:04:34.119754+0000","mapping_epoch":307,"log_start":"258'1700","ondisk_log_start":"258'1700","created":281,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":5,"last_scrub":"316'9655","last_scrub_stamp":"2023-09-23T21:04:27.882966+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:01:24.325318+0000","last_clean_scrub_stamp":"2023-09-23T21:04:27.882966+0000","objects_scrubbed":1537,"log_size":8077,"log_dups_size":0,"ondisk_log_size":8077,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:14:00.916321+0000","scrub_duration":0.44158506600000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":92209152,"num_objects":1407,"num_object_clones":0,"num_object_copies":4221,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1407,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":130,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":293,"reported_epoch":321,"state":"active+clean","last_fresh":"2023-09-23T21:04:33.412006+0000","last_change":"2023-09-23T20:54:13.941394+0000","last_active":"2023-09-23T21:04:33.412006+0000","last_peered":"2023-09-23T21:04:33.412006+0000","last_clean":"2023-09-23T21:04:33.412006+0000","last_became_active":"2023-09-23T20:54:13.941042+0000","last_became_peered":"2023-09-23T20:54:13.941042+0000","last_unstale":"2023-09-23T21:04:33.412006+0000","last_undegraded":"2023-09-23T21:04:33.412006+0000","last_fullsized":"2023-09-23T21:04:33.412006+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:19:41.401184+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":308,"reported_epoch":320,"state":"active+clean","last_fresh":"2023-09-23T21:04:32.402383+0000","last_change":"2023-09-23T20:57:44.979811+0000","last_active":"2023-09-23T21:04:32.402383+0000","last_peered":"2023-09-23T21:04:32.402383+0000","last_clean":"2023-09-23T21:04:32.402383+0000","last_became_active":"2023-09-23T20:57:44.979515+0000","last_became_peered":"2023-09-23T20:57:44.979515+0000","last_unstale":"2023-09-23T21:04:32.402383+0000","last_undegraded":"2023-09-23T21:04:32.402383+0000","last_fullsized":"2023-09-23T21:04:32.402383+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:38:22.162140+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.13","version":"321'9339","reported_seq":15849,"reported_epoch":321,"state":"active+clean","last_fresh":"2023-09-23T21:04:34.827020+0000","last_change":"2023-09-23T21:04:27.787733+0000","last_active":"2023-09-23T21:04:34.827020+0000","last_peered":"2023-09-23T21:04:34.827020+0000","last_clean":"2023-09-23T21:04:34.827020+0000","last_became_active":"2023-09-23T21:04:05.210583+0000","last_became_peered":"2023-09-23T21:04:05.210583+0000","last_unstale":"2023-09-23T21:04:34.827020+0000","last_undegraded":"2023-09-23T21:04:34.827020+0000","last_fullsized":"2023-09-23T21:04:34.827020+0000","mapping_epoch":301,"log_start":"258'1200","ondisk_log_start":"258'1200","created":281,"last_epoch_clean":302,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:01:24.325318+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:01:24.325318+0000","last_clean_scrub_stamp":"2023-09-23T21:01:24.325318+0000","objects_scrubbed":0,"log_size":8139,"log_dups_size":0,"ondisk_log_size":8139,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":91029504,"num_objects":1389,"num_object_clones":0,"num_object_copies":4167,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1389,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5317,"num_write_kb":214592,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2135,"num_bytes_recovered":139788288,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,1],"acting":[3,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":306,"reported_epoch":320,"state":"active+clean","last_fresh":"2023-09-23T21:04:32.423915+0000","last_change":"2023-09-23T20:51:10.833613+0000","last_active":"2023-09-23T21:04:32.423915+0000","last_peered":"2023-09-23T21:04:32.423915+0000","last_clean":"2023-09-23T21:04:32.423915+0000","last_became_active":"2023-09-23T20:51:10.833208+0000","last_became_peered":"2023-09-23T20:51:10.833208+0000","last_unstale":"2023-09-23T21:04:32.423915+0000","last_undegraded":"2023-09-23T21:04:32.423915+0000","last_fullsized":"2023-09-23T21:04:32.423915+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:51:28.246014+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":286,"reported_epoch":320,"state":"active+clean","last_fresh":"2023-09-23T21:04:32.399989+0000","last_change":"2023-09-23T20:55:35.958384+0000","last_active":"2023-09-23T21:04:32.399989+0000","last_peered":"2023-09-23T21:04:32.399989+0000","last_clean":"2023-09-23T21:04:32.399989+0000","last_became_active":"2023-09-23T20:55:35.954563+0000","last_became_peered":"2023-09-23T20:55:35.954563+0000","last_unstale":"2023-09-23T21:04:32.399989+0000","last_undegraded":"2023-09-23T21:04:32.399989+0000","last_fullsized":"2023-09-23T21:04:32.399989+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:37:48.332259+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":298,"reported_epoch":320,"state":"active+clean","last_fresh":"2023-09-23T21:04:32.402058+0000","last_change":"2023-09-23T20:54:31.885514+0000","last_active":"2023-09-23T21:04:32.402058+0000","last_peered":"2023-09-23T21:04:32.402058+0000","last_clean":"2023-09-23T21:04:32.402058+0000","last_became_active":"2023-09-23T20:54:31.884986+0000","last_became_peered":"2023-09-23T20:54:31.884986+0000","last_unstale":"2023-09-23T21:04:32.402058+0000","last_undegraded":"2023-09-23T21:04:32.402058+0000","last_fullsized":"2023-09-23T21:04:32.402058+0000","mapping_epoch":136,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:26:08.100220+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":303,"reported_epoch":320,"state":"active+clean","last_fresh":"2023-09-23T21:04:32.423955+0000","last_change":"2023-09-23T20:55:35.951889+0000","last_active":"2023-09-23T21:04:32.423955+0000","last_peered":"2023-09-23T21:04:32.423955+0000","last_clean":"2023-09-23T21:04:32.423955+0000","last_became_active":"2023-09-23T20:55:35.951563+0000","last_became_peered":"2023-09-23T20:55:35.951563+0000","last_unstale":"2023-09-23T21:04:32.423955+0000","last_undegraded":"2023-09-23T21:04:32.423955+0000","last_fullsized":"2023-09-23T21:04:32.423955+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:41:53.098781+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"79'161","reported_seq":610,"reported_epoch":320,"state":"active+clean","last_fresh":"2023-09-23T21:04:32.424683+0000","last_change":"2023-09-23T20:57:25.096559+0000","last_active":"2023-09-23T21:04:32.424683+0000","last_peered":"2023-09-23T21:04:32.424683+0000","last_clean":"2023-09-23T21:04:32.424683+0000","last_became_active":"2023-09-23T20:57:15.975674+0000","last_became_peered":"2023-09-23T20:57:15.975674+0000","last_unstale":"2023-09-23T21:04:32.424683+0000","last_undegraded":"2023-09-23T21:04:32.424683+0000","last_fullsized":"2023-09-23T21:04:32.424683+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:25.096401+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:25.096401+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:05:05.448404+0000","scrub_duration":0.002859393,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":194,"reported_epoch":320,"state":"active+clean","last_fresh":"2023-09-23T21:04:32.399837+0000","last_change":"2023-09-23T20:54:31.890678+0000","last_active":"2023-09-23T21:04:32.399837+0000","last_peered":"2023-09-23T21:04:32.399837+0000","last_clean":"2023-09-23T21:04:32.399837+0000","last_became_active":"2023-09-23T20:54:31.890469+0000","last_became_peered":"2023-09-23T20:54:31.890469+0000","last_unstale":"2023-09-23T21:04:32.399837+0000","last_undegraded":"2023-09-23T21:04:32.399837+0000","last_fullsized":"2023-09-23T21:04:32.399837+0000","mapping_epoch":136,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:55:08.790146+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":264,"reported_epoch":320,"state":"active+clean","last_fresh":"2023-09-23T21:04:32.423315+0000","last_change":"2023-09-23T20:55:35.952661+0000","last_active":"2023-09-23T21:04:32.423315+0000","last_peered":"2023-09-23T21:04:32.423315+0000","last_clean":"2023-09-23T21:04:32.423315+0000","last_became_active":"2023-09-23T20:55:35.952349+0000","last_became_peered":"2023-09-23T20:55:35.952349+0000","last_unstale":"2023-09-23T21:04:32.423315+0000","last_undegraded":"2023-09-23T21:04:32.423315+0000","last_fullsized":"2023-09-23T21:04:32.423315+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:00:03.110126+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":141,"reported_epoch":320,"state":"active+clean","last_fresh":"2023-09-23T21:04:32.423750+0000","last_change":"2023-09-23T20:56:45.988607+0000","last_active":"2023-09-23T21:04:32.423750+0000","last_peered":"2023-09-23T21:04:32.423750+0000","last_clean":"2023-09-23T21:04:32.423750+0000","last_became_active":"2023-09-23T20:56:45.988206+0000","last_became_peered":"2023-09-23T20:56:45.988206+0000","last_unstale":"2023-09-23T21:04:32.423750+0000","last_undegraded":"2023-09-23T21:04:32.423750+0000","last_fullsized":"2023-09-23T21:04:32.423750+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":205,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:01:19.783688+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,4],"acting":[3,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":138,"reported_epoch":320,"state":"active+clean","last_fresh":"2023-09-23T21:04:32.401556+0000","last_change":"2023-09-23T20:56:18.139142+0000","last_active":"2023-09-23T21:04:32.401556+0000","last_peered":"2023-09-23T21:04:32.401556+0000","last_clean":"2023-09-23T21:04:32.401556+0000","last_became_active":"2023-09-23T20:56:18.138844+0000","last_became_peered":"2023-09-23T20:56:18.138844+0000","last_unstale":"2023-09-23T21:04:32.401556+0000","last_undegraded":"2023-09-23T21:04:32.401556+0000","last_fullsized":"2023-09-23T21:04:32.401556+0000","mapping_epoch":192,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":193,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:09:02.217683+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,2],"acting":[5,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.14","version":"79'22","reported_seq":225,"reported_epoch":320,"state":"active+clean","last_fresh":"2023-09-23T21:04:32.401186+0000","last_change":"2023-09-23T20:55:35.949418+0000","last_active":"2023-09-23T21:04:32.401186+0000","last_peered":"2023-09-23T21:04:32.401186+0000","last_clean":"2023-09-23T21:04:32.401186+0000","last_became_active":"2023-09-23T20:55:35.948566+0000","last_became_peered":"2023-09-23T20:55:35.948566+0000","last_unstale":"2023-09-23T21:04:32.401186+0000","last_undegraded":"2023-09-23T21:04:32.401186+0000","last_fullsized":"2023-09-23T21:04:32.401186+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:59:37.433128+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4734,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":215,"reported_epoch":320,"state":"active+clean","last_fresh":"2023-09-23T21:04:32.399530+0000","last_change":"2023-09-23T20:55:35.949178+0000","last_active":"2023-09-23T21:04:32.399530+0000","last_peered":"2023-09-23T21:04:32.399530+0000","last_clean":"2023-09-23T21:04:32.399530+0000","last_became_active":"2023-09-23T20:55:35.948795+0000","last_became_peered":"2023-09-23T20:55:35.948795+0000","last_unstale":"2023-09-23T21:04:32.399530+0000","last_undegraded":"2023-09-23T21:04:32.399530+0000","last_fullsized":"2023-09-23T21:04:32.399530+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:13:08.996932+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.18","version":"321'9788","reported_seq":13021,"reported_epoch":321,"state":"active+clean","last_fresh":"2023-09-23T21:04:37.261312+0000","last_change":"2023-09-23T21:04:31.325373+0000","last_active":"2023-09-23T21:04:37.261312+0000","last_peered":"2023-09-23T21:04:37.261312+0000","last_clean":"2023-09-23T21:04:37.261312+0000","last_became_active":"2023-09-23T21:04:29.386634+0000","last_became_peered":"2023-09-23T21:04:29.386634+0000","last_unstale":"2023-09-23T21:04:37.261312+0000","last_undegraded":"2023-09-23T21:04:37.261312+0000","last_fullsized":"2023-09-23T21:04:37.261312+0000","mapping_epoch":317,"log_start":"258'600","ondisk_log_start":"258'600","created":281,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:01:24.325318+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:01:24.325318+0000","last_clean_scrub_stamp":"2023-09-23T21:01:24.325318+0000","objects_scrubbed":0,"log_size":9188,"log_dups_size":0,"ondisk_log_size":9188,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":84377600,"num_objects":1288,"num_object_clones":0,"num_object_copies":3864,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1288,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5642,"num_write_kb":221728,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1561,"num_bytes_recovered":102301696,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,5],"acting":[4,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.11","version":"79'161","reported_seq":552,"reported_epoch":320,"state":"active+clean","last_fresh":"2023-09-23T21:04:32.401708+0000","last_change":"2023-09-23T20:57:28.756796+0000","last_active":"2023-09-23T21:04:32.401708+0000","last_peered":"2023-09-23T21:04:32.401708+0000","last_clean":"2023-09-23T21:04:32.401708+0000","last_became_active":"2023-09-23T20:57:15.980606+0000","last_became_peered":"2023-09-23T20:57:15.980606+0000","last_unstale":"2023-09-23T21:04:32.401708+0000","last_undegraded":"2023-09-23T21:04:32.401708+0000","last_fullsized":"2023-09-23T21:04:32.401708+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:28.756634+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:28.756634+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:09:19.150791+0000","scrub_duration":0.0064925119999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":241,"reported_epoch":320,"state":"active+clean","last_fresh":"2023-09-23T21:04:32.399385+0000","last_change":"2023-09-23T20:56:45.992577+0000","last_active":"2023-09-23T21:04:32.399385+0000","last_peered":"2023-09-23T21:04:32.399385+0000","last_clean":"2023-09-23T21:04:32.399385+0000","last_became_active":"2023-09-23T20:56:45.992297+0000","last_became_peered":"2023-09-23T20:56:45.992297+0000","last_unstale":"2023-09-23T21:04:32.399385+0000","last_undegraded":"2023-09-23T21:04:32.399385+0000","last_fullsized":"2023-09-23T21:04:32.399385+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":205,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:00:45.547202+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,2],"acting":[4,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":288,"reported_epoch":320,"state":"active+clean","last_fresh":"2023-09-23T21:04:32.424623+0000","last_change":"2023-09-23T20:51:10.831262+0000","last_active":"2023-09-23T21:04:32.424623+0000","last_peered":"2023-09-23T21:04:32.424623+0000","last_clean":"2023-09-23T21:04:32.424623+0000","last_became_active":"2023-09-23T20:51:10.830931+0000","last_became_peered":"2023-09-23T20:51:10.830931+0000","last_unstale":"2023-09-23T21:04:32.424623+0000","last_undegraded":"2023-09-23T21:04:32.424623+0000","last_fullsized":"2023-09-23T21:04:32.424623+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:51:54.982505+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.19","version":"321'9587","reported_seq":12644,"reported_epoch":321,"state":"active+clean","last_fresh":"2023-09-23T21:04:37.005037+0000","last_change":"2023-09-23T21:04:32.417540+0000","last_active":"2023-09-23T21:04:37.005037+0000","last_peered":"2023-09-23T21:04:37.005037+0000","last_clean":"2023-09-23T21:04:37.005037+0000","last_became_active":"2023-09-23T21:04:32.402388+0000","last_became_peered":"2023-09-23T21:04:32.402388+0000","last_unstale":"2023-09-23T21:04:37.005037+0000","last_undegraded":"2023-09-23T21:04:37.005037+0000","last_fullsized":"2023-09-23T21:04:37.005037+0000","mapping_epoch":319,"log_start":"258'700","ondisk_log_start":"258'700","created":281,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:01:24.325318+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:01:24.325318+0000","last_clean_scrub_stamp":"2023-09-23T21:01:24.325318+0000","objects_scrubbed":0,"log_size":8887,"log_dups_size":0,"ondisk_log_size":8887,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":81264640,"num_objects":1240,"num_object_clones":0,"num_object_copies":3720,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1240,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5534,"num_write_kb":216768,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1471,"num_bytes_recovered":96403456,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.10","version":"79'161","reported_seq":558,"reported_epoch":320,"state":"active+clean","last_fresh":"2023-09-23T21:04:32.401251+0000","last_change":"2023-09-23T20:57:20.708003+0000","last_active":"2023-09-23T21:04:32.401251+0000","last_peered":"2023-09-23T21:04:32.401251+0000","last_clean":"2023-09-23T21:04:32.401251+0000","last_became_active":"2023-09-23T20:57:15.975302+0000","last_became_peered":"2023-09-23T20:57:15.975302+0000","last_unstale":"2023-09-23T21:04:32.401251+0000","last_undegraded":"2023-09-23T21:04:32.401251+0000","last_fullsized":"2023-09-23T21:04:32.401251+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:20.707939+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:20.707939+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:16:51.721994+0000","scrub_duration":0.0017733409999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":298,"reported_epoch":320,"state":"active+clean","last_fresh":"2023-09-23T21:04:32.401782+0000","last_change":"2023-09-23T20:57:44.979736+0000","last_active":"2023-09-23T21:04:32.401782+0000","last_peered":"2023-09-23T21:04:32.401782+0000","last_clean":"2023-09-23T21:04:32.401782+0000","last_became_active":"2023-09-23T20:57:44.979356+0000","last_became_peered":"2023-09-23T20:57:44.979356+0000","last_unstale":"2023-09-23T21:04:32.401782+0000","last_undegraded":"2023-09-23T21:04:32.401782+0000","last_fullsized":"2023-09-23T21:04:32.401782+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:14:43.848832+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":194,"reported_epoch":320,"state":"active+clean","last_fresh":"2023-09-23T21:04:32.400467+0000","last_change":"2023-09-23T20:54:31.887814+0000","last_active":"2023-09-23T21:04:32.400467+0000","last_peered":"2023-09-23T21:04:32.400467+0000","last_clean":"2023-09-23T21:04:32.400467+0000","last_became_active":"2023-09-23T20:54:31.887195+0000","last_became_peered":"2023-09-23T20:54:31.887195+0000","last_unstale":"2023-09-23T21:04:32.400467+0000","last_undegraded":"2023-09-23T21:04:32.400467+0000","last_fullsized":"2023-09-23T21:04:32.400467+0000","mapping_epoch":136,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:10:52.089553+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.13","version":"79'161","reported_seq":623,"reported_epoch":320,"state":"active+clean","last_fresh":"2023-09-23T21:04:32.428192+0000","last_change":"2023-09-23T20:57:48.262970+0000","last_active":"2023-09-23T21:04:32.428192+0000","last_peered":"2023-09-23T21:04:32.428192+0000","last_clean":"2023-09-23T21:04:32.428192+0000","last_became_active":"2023-09-23T20:57:48.262663+0000","last_became_peered":"2023-09-23T20:57:48.262663+0000","last_unstale":"2023-09-23T21:04:32.428192+0000","last_undegraded":"2023-09-23T21:04:32.428192+0000","last_fullsized":"2023-09-23T21:04:32.428192+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:36.420362+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:36.420362+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:28:28.613492+0000","scrub_duration":0.028838444000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":265,"reported_epoch":320,"state":"active+clean","last_fresh":"2023-09-23T21:04:32.399910+0000","last_change":"2023-09-23T20:56:45.991822+0000","last_active":"2023-09-23T21:04:32.399910+0000","last_peered":"2023-09-23T21:04:32.399910+0000","last_clean":"2023-09-23T21:04:32.399910+0000","last_became_active":"2023-09-23T20:56:45.991460+0000","last_became_peered":"2023-09-23T20:56:45.991460+0000","last_unstale":"2023-09-23T21:04:32.399910+0000","last_undegraded":"2023-09-23T21:04:32.399910+0000","last_fullsized":"2023-09-23T21:04:32.399910+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":205,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:20:37.503317+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":280,"reported_epoch":320,"state":"active+clean","last_fresh":"2023-09-23T21:04:32.402485+0000","last_change":"2023-09-23T20:51:10.840849+0000","last_active":"2023-09-23T21:04:32.402485+0000","last_peered":"2023-09-23T21:04:32.402485+0000","last_clean":"2023-09-23T21:04:32.402485+0000","last_became_active":"2023-09-23T20:51:10.840593+0000","last_became_peered":"2023-09-23T20:51:10.840593+0000","last_unstale":"2023-09-23T21:04:32.402485+0000","last_undegraded":"2023-09-23T21:04:32.402485+0000","last_fullsized":"2023-09-23T21:04:32.402485+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:30:52.297625+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.12","version":"79'161","reported_seq":558,"reported_epoch":320,"state":"active+clean","last_fresh":"2023-09-23T21:04:32.401415+0000","last_change":"2023-09-23T20:57:21.750765+0000","last_active":"2023-09-23T21:04:32.401415+0000","last_peered":"2023-09-23T21:04:32.401415+0000","last_clean":"2023-09-23T21:04:32.401415+0000","last_became_active":"2023-09-23T20:57:15.979195+0000","last_became_peered":"2023-09-23T20:57:15.979195+0000","last_unstale":"2023-09-23T21:04:32.401415+0000","last_undegraded":"2023-09-23T21:04:32.401415+0000","last_fullsized":"2023-09-23T21:04:32.401415+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:21.750680+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:21.750680+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:03:41.915962+0000","scrub_duration":0.0092391680000000007,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":313,"reported_epoch":320,"state":"active+clean","last_fresh":"2023-09-23T21:04:32.399014+0000","last_change":"2023-09-23T20:55:36.943969+0000","last_active":"2023-09-23T21:04:32.399014+0000","last_peered":"2023-09-23T21:04:32.399014+0000","last_clean":"2023-09-23T21:04:32.399014+0000","last_became_active":"2023-09-23T20:55:36.943580+0000","last_became_peered":"2023-09-23T20:55:36.943580+0000","last_unstale":"2023-09-23T21:04:32.399014+0000","last_undegraded":"2023-09-23T21:04:32.399014+0000","last_fullsized":"2023-09-23T21:04:32.399014+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:49:06.969396+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":111,"reported_epoch":320,"state":"active+clean","last_fresh":"2023-09-23T21:04:32.423500+0000","last_change":"2023-09-23T20:57:44.980498+0000","last_active":"2023-09-23T21:04:32.423500+0000","last_peered":"2023-09-23T21:04:32.423500+0000","last_clean":"2023-09-23T21:04:32.423500+0000","last_became_active":"2023-09-23T20:57:44.980182+0000","last_became_peered":"2023-09-23T20:57:44.980182+0000","last_unstale":"2023-09-23T21:04:32.423500+0000","last_undegraded":"2023-09-23T21:04:32.423500+0000","last_fullsized":"2023-09-23T21:04:32.423500+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:34:51.620544+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.4","version":"321'9736","reported_seq":9830,"reported_epoch":321,"state":"active+clean","last_fresh":"2023-09-23T21:04:37.077909+0000","last_change":"2023-09-23T21:04:12.976123+0000","last_active":"2023-09-23T21:04:37.077909+0000","last_peered":"2023-09-23T21:04:37.077909+0000","last_clean":"2023-09-23T21:04:37.077909+0000","last_became_active":"2023-09-23T21:03:30.256563+0000","last_became_peered":"2023-09-23T21:03:30.256563+0000","last_unstale":"2023-09-23T21:04:37.077909+0000","last_undegraded":"2023-09-23T21:04:37.077909+0000","last_fullsized":"2023-09-23T21:04:37.077909+0000","mapping_epoch":254,"log_start":"258'500","ondisk_log_start":"258'500","created":254,"last_epoch_clean":282,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:01:24.325318+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:01:24.325318+0000","last_clean_scrub_stamp":"2023-09-23T21:01:24.325318+0000","objects_scrubbed":0,"log_size":9236,"log_dups_size":0,"ondisk_log_size":9236,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":87261184,"num_objects":1331,"num_object_clones":0,"num_object_copies":3993,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1331,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5591,"num_write_kb":221536,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,2],"acting":[5,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.d","version":"79'161","reported_seq":603,"reported_epoch":320,"state":"active+clean","last_fresh":"2023-09-23T21:04:32.401590+0000","last_change":"2023-09-23T20:57:31.075213+0000","last_active":"2023-09-23T21:04:32.401590+0000","last_peered":"2023-09-23T21:04:32.401590+0000","last_clean":"2023-09-23T21:04:32.401590+0000","last_became_active":"2023-09-23T20:57:15.988188+0000","last_became_peered":"2023-09-23T20:57:15.988188+0000","last_unstale":"2023-09-23T21:04:32.401590+0000","last_undegraded":"2023-09-23T21:04:32.401590+0000","last_fullsized":"2023-09-23T21:04:32.401590+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:31.075098+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:31.075098+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:41:52.142362+0000","scrub_duration":0.010304610000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":252,"reported_epoch":320,"state":"active+clean","last_fresh":"2023-09-23T21:04:32.401452+0000","last_change":"2023-09-23T20:56:45.991841+0000","last_active":"2023-09-23T21:04:32.401452+0000","last_peered":"2023-09-23T21:04:32.401452+0000","last_clean":"2023-09-23T21:04:32.401452+0000","last_became_active":"2023-09-23T20:56:45.991491+0000","last_became_peered":"2023-09-23T20:56:45.991491+0000","last_unstale":"2023-09-23T21:04:32.401452+0000","last_undegraded":"2023-09-23T21:04:32.401452+0000","last_fullsized":"2023-09-23T21:04:32.401452+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":205,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:30:37.405319+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":211,"reported_epoch":320,"state":"active+clean","last_fresh":"2023-09-23T21:04:32.401058+0000","last_change":"2023-09-23T20:55:35.957768+0000","last_active":"2023-09-23T21:04:32.401058+0000","last_peered":"2023-09-23T21:04:32.401058+0000","last_clean":"2023-09-23T21:04:32.401058+0000","last_became_active":"2023-09-23T20:55:35.957421+0000","last_became_peered":"2023-09-23T20:55:35.957421+0000","last_unstale":"2023-09-23T21:04:32.401058+0000","last_undegraded":"2023-09-23T21:04:32.401058+0000","last_fullsized":"2023-09-23T21:04:32.401058+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:49:41.585961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"8.5","version":"321'9706","reported_seq":9902,"reported_epoch":321,"state":"active+clean","last_fresh":"2023-09-23T21:04:34.141410+0000","last_change":"2023-09-23T21:03:37.840743+0000","last_active":"2023-09-23T21:04:34.141410+0000","last_peered":"2023-09-23T21:04:34.141410+0000","last_clean":"2023-09-23T21:04:34.141410+0000","last_became_active":"2023-09-23T21:03:30.251381+0000","last_became_peered":"2023-09-23T21:03:30.251381+0000","last_unstale":"2023-09-23T21:04:34.141410+0000","last_undegraded":"2023-09-23T21:04:34.141410+0000","last_fullsized":"2023-09-23T21:04:34.141410+0000","mapping_epoch":254,"log_start":"258'1000","ondisk_log_start":"258'1000","created":254,"last_epoch_clean":282,"parent":"0.0","parent_split_bits":0,"last_scrub":"284'8610","last_scrub_stamp":"2023-09-23T21:03:37.840682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:01:24.325318+0000","last_clean_scrub_stamp":"2023-09-23T21:03:37.840682+0000","objects_scrubbed":2517,"log_size":8706,"log_dups_size":0,"ondisk_log_size":8706,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:00:41.745968+0000","scrub_duration":0.31954776800000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":92798976,"num_objects":1416,"num_object_clones":0,"num_object_copies":4248,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1416,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1101,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.c","version":"79'161","reported_seq":624,"reported_epoch":320,"state":"active+clean","last_fresh":"2023-09-23T21:04:32.424449+0000","last_change":"2023-09-23T20:57:22.041071+0000","last_active":"2023-09-23T21:04:32.424449+0000","last_peered":"2023-09-23T21:04:32.424449+0000","last_clean":"2023-09-23T21:04:32.424449+0000","last_became_active":"2023-09-23T20:57:15.973204+0000","last_became_peered":"2023-09-23T20:57:15.973204+0000","last_unstale":"2023-09-23T21:04:32.424449+0000","last_undegraded":"2023-09-23T21:04:32.424449+0000","last_fullsized":"2023-09-23T21:04:32.424449+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:22.040907+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:22.040907+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:47:02.019978+0000","scrub_duration":0.013021176000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":239,"reported_epoch":320,"state":"active+clean","last_fresh":"2023-09-23T21:04:32.401977+0000","last_change":"2023-09-23T20:56:45.991821+0000","last_active":"2023-09-23T21:04:32.401977+0000","last_peered":"2023-09-23T21:04:32.401977+0000","last_clean":"2023-09-23T21:04:32.401977+0000","last_became_active":"2023-09-23T20:56:45.991447+0000","last_became_peered":"2023-09-23T20:56:45.991447+0000","last_unstale":"2023-09-23T21:04:32.401977+0000","last_undegraded":"2023-09-23T21:04:32.401977+0000","last_fullsized":"2023-09-23T21:04:32.401977+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":205,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:14:25.525222+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":242,"reported_epoch":320,"state":"active+clean","last_fresh":"2023-09-23T21:04:32.400196+0000","last_change":"2023-09-23T20:57:44.982059+0000","last_active":"2023-09-23T21:04:32.400196+0000","last_peered":"2023-09-23T21:04:32.400196+0000","last_clean":"2023-09-23T21:04:32.400196+0000","last_became_active":"2023-09-23T20:57:44.981742+0000","last_became_peered":"2023-09-23T20:57:44.981742+0000","last_unstale":"2023-09-23T21:04:32.400196+0000","last_undegraded":"2023-09-23T21:04:32.400196+0000","last_fullsized":"2023-09-23T21:04:32.400196+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:41:24.042916+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.6","version":"321'10000","reported_seq":10149,"reported_epoch":321,"state":"active+clean","last_fresh":"2023-09-23T21:04:37.148018+0000","last_change":"2023-09-23T21:03:38.793592+0000","last_active":"2023-09-23T21:04:37.148018+0000","last_peered":"2023-09-23T21:04:37.148018+0000","last_clean":"2023-09-23T21:04:37.148018+0000","last_became_active":"2023-09-23T21:03:30.271638+0000","last_became_peered":"2023-09-23T21:03:30.271638+0000","last_unstale":"2023-09-23T21:04:37.148018+0000","last_undegraded":"2023-09-23T21:04:37.148018+0000","last_fullsized":"2023-09-23T21:04:37.148018+0000","mapping_epoch":254,"log_start":"258'600","ondisk_log_start":"258'600","created":254,"last_epoch_clean":282,"parent":"0.0","parent_split_bits":0,"last_scrub":"284'8804","last_scrub_stamp":"2023-09-23T21:03:38.793436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:01:24.325318+0000","last_clean_scrub_stamp":"2023-09-23T21:03:38.793436+0000","objects_scrubbed":2552,"log_size":9400,"log_dups_size":0,"ondisk_log_size":9400,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:56:02.183614+0000","scrub_duration":0.31348443599999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":88866816,"num_objects":1356,"num_object_clones":0,"num_object_copies":4068,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1356,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1196,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,7],"acting":[0,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.f","version":"79'161","reported_seq":625,"reported_epoch":320,"state":"active+clean","last_fresh":"2023-09-23T21:04:32.399831+0000","last_change":"2023-09-23T20:57:27.880841+0000","last_active":"2023-09-23T21:04:32.399831+0000","last_peered":"2023-09-23T21:04:32.399831+0000","last_clean":"2023-09-23T21:04:32.399831+0000","last_became_active":"2023-09-23T20:57:15.970925+0000","last_became_peered":"2023-09-23T20:57:15.970925+0000","last_unstale":"2023-09-23T21:04:32.399831+0000","last_undegraded":"2023-09-23T21:04:32.399831+0000","last_fullsized":"2023-09-23T21:04:32.399831+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:27.880776+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:27.880776+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:15:13.438014+0000","scrub_duration":0.0014004899999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":225,"reported_epoch":320,"state":"active+clean","last_fresh":"2023-09-23T21:04:32.400736+0000","last_change":"2023-09-23T20:56:45.989153+0000","last_active":"2023-09-23T21:04:32.400736+0000","last_peered":"2023-09-23T21:04:32.400736+0000","last_clean":"2023-09-23T21:04:32.400736+0000","last_became_active":"2023-09-23T20:56:45.988800+0000","last_became_peered":"2023-09-23T20:56:45.988800+0000","last_unstale":"2023-09-23T21:04:32.400736+0000","last_undegraded":"2023-09-23T21:04:32.400736+0000","last_fullsized":"2023-09-23T21:04:32.400736+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":205,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:04:41.130194+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":260,"reported_epoch":320,"state":"active+clean","last_fresh":"2023-09-23T21:04:32.400854+0000","last_change":"2023-09-23T20:55:35.965427+0000","last_active":"2023-09-23T21:04:32.400854+0000","last_peered":"2023-09-23T21:04:32.400854+0000","last_clean":"2023-09-23T21:04:32.400854+0000","last_became_active":"2023-09-23T20:55:35.963969+0000","last_became_peered":"2023-09-23T20:55:35.963969+0000","last_unstale":"2023-09-23T21:04:32.400854+0000","last_undegraded":"2023-09-23T21:04:32.400854+0000","last_fullsized":"2023-09-23T21:04:32.400854+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:38:40.612083+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.7","version":"321'9550","reported_seq":9699,"reported_epoch":321,"state":"active+clean","last_fresh":"2023-09-23T21:04:34.600646+0000","last_change":"2023-09-23T21:03:41.677499+0000","last_active":"2023-09-23T21:04:34.600646+0000","last_peered":"2023-09-23T21:04:34.600646+0000","last_clean":"2023-09-23T21:04:34.600646+0000","last_became_active":"2023-09-23T21:03:30.267051+0000","last_became_peered":"2023-09-23T21:03:30.267051+0000","last_unstale":"2023-09-23T21:04:34.600646+0000","last_undegraded":"2023-09-23T21:04:34.600646+0000","last_fullsized":"2023-09-23T21:04:34.600646+0000","mapping_epoch":254,"log_start":"258'1300","ondisk_log_start":"258'1300","created":254,"last_epoch_clean":282,"parent":"0.0","parent_split_bits":0,"last_scrub":"288'8477","last_scrub_stamp":"2023-09-23T21:03:41.677318+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:01:24.325318+0000","last_clean_scrub_stamp":"2023-09-23T21:03:41.677318+0000","objects_scrubbed":2477,"log_size":8250,"log_dups_size":0,"ondisk_log_size":8250,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:23:04.253957+0000","scrub_duration":0.50562591800000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":91684864,"num_objects":1399,"num_object_clones":0,"num_object_copies":4197,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1399,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1078,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.e","version":"79'161","reported_seq":532,"reported_epoch":320,"state":"active+clean","last_fresh":"2023-09-23T21:04:32.401900+0000","last_change":"2023-09-23T20:57:34.132738+0000","last_active":"2023-09-23T21:04:32.401900+0000","last_peered":"2023-09-23T21:04:32.401900+0000","last_clean":"2023-09-23T21:04:32.401900+0000","last_became_active":"2023-09-23T20:57:15.974318+0000","last_became_peered":"2023-09-23T20:57:15.974318+0000","last_unstale":"2023-09-23T21:04:32.401900+0000","last_undegraded":"2023-09-23T21:04:32.401900+0000","last_fullsized":"2023-09-23T21:04:32.401900+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:34.132660+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:34.132660+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:07:07.739438+0000","scrub_duration":0.005229694,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":211,"reported_epoch":320,"state":"active+clean","last_fresh":"2023-09-23T21:04:32.400174+0000","last_change":"2023-09-23T20:55:35.962005+0000","last_active":"2023-09-23T21:04:32.400174+0000","last_peered":"2023-09-23T21:04:32.400174+0000","last_clean":"2023-09-23T21:04:32.400174+0000","last_became_active":"2023-09-23T20:55:35.961695+0000","last_became_peered":"2023-09-23T20:55:35.961695+0000","last_unstale":"2023-09-23T21:04:32.400174+0000","last_undegraded":"2023-09-23T21:04:32.400174+0000","last_fullsized":"2023-09-23T21:04:32.400174+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:28:22.876201+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":311,"reported_epoch":320,"state":"active+clean","last_fresh":"2023-09-23T21:04:32.401723+0000","last_change":"2023-09-23T20:54:13.928715+0000","last_active":"2023-09-23T21:04:32.401723+0000","last_peered":"2023-09-23T21:04:32.401723+0000","last_clean":"2023-09-23T21:04:32.401723+0000","last_became_active":"2023-09-23T20:54:13.928200+0000","last_became_peered":"2023-09-23T20:54:13.928200+0000","last_unstale":"2023-09-23T21:04:32.401723+0000","last_undegraded":"2023-09-23T21:04:32.401723+0000","last_fullsized":"2023-09-23T21:04:32.401723+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:41:16.927513+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.0","version":"321'9644","reported_seq":9794,"reported_epoch":321,"state":"active+clean","last_fresh":"2023-09-23T21:04:34.804255+0000","last_change":"2023-09-23T21:03:39.283021+0000","last_active":"2023-09-23T21:04:34.804255+0000","last_peered":"2023-09-23T21:04:34.804255+0000","last_clean":"2023-09-23T21:04:34.804255+0000","last_became_active":"2023-09-23T21:03:30.271894+0000","last_became_peered":"2023-09-23T21:03:30.271894+0000","last_unstale":"2023-09-23T21:04:34.804255+0000","last_undegraded":"2023-09-23T21:04:34.804255+0000","last_fullsized":"2023-09-23T21:04:34.804255+0000","mapping_epoch":254,"log_start":"258'1700","ondisk_log_start":"258'1700","created":254,"last_epoch_clean":282,"parent":"0.0","parent_split_bits":0,"last_scrub":"284'8499","last_scrub_stamp":"2023-09-23T21:03:39.282869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:01:24.325318+0000","last_clean_scrub_stamp":"2023-09-23T21:03:39.282869+0000","objects_scrubbed":2515,"log_size":7944,"log_dups_size":0,"ondisk_log_size":7944,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:51:43.305878+0000","scrub_duration":0.31450518900000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":89784320,"num_objects":1370,"num_object_clones":0,"num_object_copies":4110,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1370,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1145,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,1],"acting":[3,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.9","version":"79'161","reported_seq":696,"reported_epoch":320,"state":"active+clean","last_fresh":"2023-09-23T21:04:32.423231+0000","last_change":"2023-09-23T20:56:46.921939+0000","last_active":"2023-09-23T21:04:32.423231+0000","last_peered":"2023-09-23T21:04:32.423231+0000","last_clean":"2023-09-23T21:04:32.423231+0000","last_became_active":"2023-09-23T20:56:45.989908+0000","last_became_peered":"2023-09-23T20:56:45.989908+0000","last_unstale":"2023-09-23T21:04:32.423231+0000","last_undegraded":"2023-09-23T21:04:32.423231+0000","last_fullsized":"2023-09-23T21:04:32.423231+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":205,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:56:46.921739+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:56:46.921739+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:20:42.635110+0000","scrub_duration":0.0071954109999999996,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":312,"reported_epoch":320,"state":"active+clean","last_fresh":"2023-09-23T21:04:32.424446+0000","last_change":"2023-09-23T20:54:13.961788+0000","last_active":"2023-09-23T21:04:32.424446+0000","last_peered":"2023-09-23T21:04:32.424446+0000","last_clean":"2023-09-23T21:04:32.424446+0000","last_became_active":"2023-09-23T20:54:13.940937+0000","last_became_peered":"2023-09-23T20:54:13.940937+0000","last_unstale":"2023-09-23T21:04:32.424446+0000","last_undegraded":"2023-09-23T21:04:32.424446+0000","last_fullsized":"2023-09-23T21:04:32.424446+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:11:56.785299+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":215,"reported_epoch":320,"state":"active+clean","last_fresh":"2023-09-23T21:04:32.401348+0000","last_change":"2023-09-23T20:55:35.949670+0000","last_active":"2023-09-23T21:04:32.401348+0000","last_peered":"2023-09-23T21:04:32.401348+0000","last_clean":"2023-09-23T21:04:32.401348+0000","last_became_active":"2023-09-23T20:55:35.949140+0000","last_became_peered":"2023-09-23T20:55:35.949140+0000","last_unstale":"2023-09-23T21:04:32.401348+0000","last_undegraded":"2023-09-23T21:04:32.401348+0000","last_fullsized":"2023-09-23T21:04:32.401348+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:55:59.340303+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.1","version":"321'9846","reported_seq":9993,"reported_epoch":321,"state":"active+clean","last_fresh":"2023-09-23T21:04:34.490120+0000","last_change":"2023-09-23T21:03:38.452701+0000","last_active":"2023-09-23T21:04:34.490120+0000","last_peered":"2023-09-23T21:04:34.490120+0000","last_clean":"2023-09-23T21:04:34.490120+0000","last_became_active":"2023-09-23T21:03:30.268996+0000","last_became_peered":"2023-09-23T21:03:30.268996+0000","last_unstale":"2023-09-23T21:04:34.490120+0000","last_undegraded":"2023-09-23T21:04:34.490120+0000","last_fullsized":"2023-09-23T21:04:34.490120+0000","mapping_epoch":254,"log_start":"258'1600","ondisk_log_start":"258'1600","created":254,"last_epoch_clean":282,"parent":"0.0","parent_split_bits":0,"last_scrub":"284'8731","last_scrub_stamp":"2023-09-23T21:03:38.452464+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:01:24.325318+0000","last_clean_scrub_stamp":"2023-09-23T21:03:38.452464+0000","objects_scrubbed":2585,"log_size":8246,"log_dups_size":0,"ondisk_log_size":8246,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:01:42.038382+0000","scrub_duration":0.313423428,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":96337920,"num_objects":1470,"num_object_clones":0,"num_object_copies":4410,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1470,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1115,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,2],"acting":[1,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.8","version":"79'161","reported_seq":564,"reported_epoch":320,"state":"active+clean","last_fresh":"2023-09-23T21:04:32.401143+0000","last_change":"2023-09-23T20:56:57.505737+0000","last_active":"2023-09-23T21:04:32.401143+0000","last_peered":"2023-09-23T21:04:32.401143+0000","last_clean":"2023-09-23T21:04:32.401143+0000","last_became_active":"2023-09-23T20:56:04.982345+0000","last_became_peered":"2023-09-23T20:56:04.982345+0000","last_unstale":"2023-09-23T21:04:32.401143+0000","last_undegraded":"2023-09-23T21:04:32.401143+0000","last_fullsized":"2023-09-23T21:04:32.401143+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:56:57.505663+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:56:57.505663+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:34:30.109724+0000","scrub_duration":0.026284108,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":283,"reported_epoch":320,"state":"active+clean","last_fresh":"2023-09-23T21:04:32.401139+0000","last_change":"2023-09-23T20:55:35.959942+0000","last_active":"2023-09-23T21:04:32.401139+0000","last_peered":"2023-09-23T21:04:32.401139+0000","last_clean":"2023-09-23T21:04:32.401139+0000","last_became_active":"2023-09-23T20:55:35.957071+0000","last_became_peered":"2023-09-23T20:55:35.957071+0000","last_unstale":"2023-09-23T21:04:32.401139+0000","last_undegraded":"2023-09-23T21:04:32.401139+0000","last_fullsized":"2023-09-23T21:04:32.401139+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:44:30.308036+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":198,"reported_epoch":320,"state":"active+clean","last_fresh":"2023-09-23T21:04:32.401811+0000","last_change":"2023-09-23T20:54:13.951459+0000","last_active":"2023-09-23T21:04:32.401811+0000","last_peered":"2023-09-23T21:04:32.401811+0000","last_clean":"2023-09-23T21:04:32.401811+0000","last_became_active":"2023-09-23T20:54:13.948409+0000","last_became_peered":"2023-09-23T20:54:13.948409+0000","last_unstale":"2023-09-23T21:04:32.401811+0000","last_undegraded":"2023-09-23T21:04:32.401811+0000","last_fullsized":"2023-09-23T21:04:32.401811+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:38:10.148061+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.2","version":"321'9917","reported_seq":10027,"reported_epoch":321,"state":"active+clean","last_fresh":"2023-09-23T21:04:37.261410+0000","last_change":"2023-09-23T21:04:21.253437+0000","last_active":"2023-09-23T21:04:37.261410+0000","last_peered":"2023-09-23T21:04:37.261410+0000","last_clean":"2023-09-23T21:04:37.261410+0000","last_became_active":"2023-09-23T21:03:30.254079+0000","last_became_peered":"2023-09-23T21:03:30.254079+0000","last_unstale":"2023-09-23T21:04:37.261410+0000","last_undegraded":"2023-09-23T21:04:37.261410+0000","last_fullsized":"2023-09-23T21:04:37.261410+0000","mapping_epoch":254,"log_start":"258'1700","ondisk_log_start":"258'1700","created":254,"last_epoch_clean":282,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:01:24.325318+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:01:24.325318+0000","last_clean_scrub_stamp":"2023-09-23T21:01:24.325318+0000","objects_scrubbed":0,"log_size":8217,"log_dups_size":0,"ondisk_log_size":8217,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":86540288,"num_objects":1320,"num_object_clones":0,"num_object_copies":3960,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1320,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5707,"num_write_kb":224896,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,5],"acting":[4,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.b","version":"79'161","reported_seq":606,"reported_epoch":320,"state":"active+clean","last_fresh":"2023-09-23T21:04:32.428278+0000","last_change":"2023-09-23T20:57:27.320172+0000","last_active":"2023-09-23T21:04:32.428278+0000","last_peered":"2023-09-23T21:04:32.428278+0000","last_clean":"2023-09-23T21:04:32.428278+0000","last_became_active":"2023-09-23T20:57:16.009630+0000","last_became_peered":"2023-09-23T20:57:16.009630+0000","last_unstale":"2023-09-23T21:04:32.428278+0000","last_undegraded":"2023-09-23T21:04:32.428278+0000","last_fullsized":"2023-09-23T21:04:32.428278+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:27.320022+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:27.320022+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:31:35.221558+0000","scrub_duration":0.0043894679999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,5],"acting":[1,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":198,"reported_epoch":320,"state":"active+clean","last_fresh":"2023-09-23T21:04:32.399992+0000","last_change":"2023-09-23T20:54:13.950254+0000","last_active":"2023-09-23T21:04:32.399992+0000","last_peered":"2023-09-23T21:04:32.399992+0000","last_clean":"2023-09-23T21:04:32.399992+0000","last_became_active":"2023-09-23T20:54:13.949934+0000","last_became_peered":"2023-09-23T20:54:13.949934+0000","last_unstale":"2023-09-23T21:04:32.399992+0000","last_undegraded":"2023-09-23T21:04:32.399992+0000","last_fullsized":"2023-09-23T21:04:32.399992+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:30:32.847167+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":215,"reported_epoch":320,"state":"active+clean","last_fresh":"2023-09-23T21:04:32.402018+0000","last_change":"2023-09-23T20:55:35.956153+0000","last_active":"2023-09-23T21:04:32.402018+0000","last_peered":"2023-09-23T21:04:32.402018+0000","last_clean":"2023-09-23T21:04:32.402018+0000","last_became_active":"2023-09-23T20:55:35.955630+0000","last_became_peered":"2023-09-23T20:55:35.955630+0000","last_unstale":"2023-09-23T21:04:32.402018+0000","last_undegraded":"2023-09-23T21:04:32.402018+0000","last_fullsized":"2023-09-23T21:04:32.402018+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:45:20.282929+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.3","version":"321'9370","reported_seq":9502,"reported_epoch":321,"state":"active+clean","last_fresh":"2023-09-23T21:04:34.555728+0000","last_change":"2023-09-23T21:04:25.370748+0000","last_active":"2023-09-23T21:04:34.555728+0000","last_peered":"2023-09-23T21:04:34.555728+0000","last_clean":"2023-09-23T21:04:34.555728+0000","last_became_active":"2023-09-23T21:03:30.287720+0000","last_became_peered":"2023-09-23T21:03:30.287720+0000","last_unstale":"2023-09-23T21:04:34.555728+0000","last_undegraded":"2023-09-23T21:04:34.555728+0000","last_fullsized":"2023-09-23T21:04:34.555728+0000","mapping_epoch":254,"log_start":"258'1200","ondisk_log_start":"258'1200","created":254,"last_epoch_clean":282,"parent":"0.0","parent_split_bits":0,"last_scrub":"315'9153","last_scrub_stamp":"2023-09-23T21:04:25.370609+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:01:24.325318+0000","last_clean_scrub_stamp":"2023-09-23T21:04:25.370609+0000","objects_scrubbed":1550,"log_size":8170,"log_dups_size":0,"ondisk_log_size":8170,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:46:54.982966+0000","scrub_duration":0.20200885099999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":87359488,"num_objects":1333,"num_object_clones":0,"num_object_copies":3999,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1333,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":217,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.a","version":"79'161","reported_seq":564,"reported_epoch":320,"state":"active+clean","last_fresh":"2023-09-23T21:04:32.401632+0000","last_change":"2023-09-23T20:56:44.487778+0000","last_active":"2023-09-23T21:04:32.401632+0000","last_peered":"2023-09-23T21:04:32.401632+0000","last_clean":"2023-09-23T21:04:32.401632+0000","last_became_active":"2023-09-23T20:56:05.006740+0000","last_became_peered":"2023-09-23T20:56:05.006740+0000","last_unstale":"2023-09-23T21:04:32.401632+0000","last_undegraded":"2023-09-23T21:04:32.401632+0000","last_fullsized":"2023-09-23T21:04:32.401632+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:56:44.487678+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:56:44.487678+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:04:50.086749+0000","scrub_duration":0.0017408899999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":112,"reported_epoch":320,"state":"active+clean","last_fresh":"2023-09-23T21:04:32.423794+0000","last_change":"2023-09-23T20:57:44.982513+0000","last_active":"2023-09-23T21:04:32.423794+0000","last_peered":"2023-09-23T21:04:32.423794+0000","last_clean":"2023-09-23T21:04:32.423794+0000","last_became_active":"2023-09-23T20:57:44.982112+0000","last_became_peered":"2023-09-23T20:57:44.982112+0000","last_unstale":"2023-09-23T21:04:32.423794+0000","last_undegraded":"2023-09-23T21:04:32.423794+0000","last_fullsized":"2023-09-23T21:04:32.423794+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:09:05.630041+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":293,"reported_epoch":320,"state":"active+clean","last_fresh":"2023-09-23T21:04:32.428064+0000","last_change":"2023-09-23T20:54:13.940748+0000","last_active":"2023-09-23T21:04:32.428064+0000","last_peered":"2023-09-23T21:04:32.428064+0000","last_clean":"2023-09-23T21:04:32.428064+0000","last_became_active":"2023-09-23T20:54:13.940419+0000","last_became_peered":"2023-09-23T20:54:13.940419+0000","last_unstale":"2023-09-23T21:04:32.428064+0000","last_undegraded":"2023-09-23T21:04:32.428064+0000","last_fullsized":"2023-09-23T21:04:32.428064+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:15:01.923816+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.e","version":"321'11415","reported_seq":11492,"reported_epoch":321,"state":"active+clean","last_fresh":"2023-09-23T21:04:37.106714+0000","last_change":"2023-09-23T21:01:25.430111+0000","last_active":"2023-09-23T21:04:37.106714+0000","last_peered":"2023-09-23T21:04:37.106714+0000","last_clean":"2023-09-23T21:04:37.106714+0000","last_became_active":"2023-09-23T21:01:25.429811+0000","last_became_peered":"2023-09-23T21:01:25.429811+0000","last_unstale":"2023-09-23T21:04:37.106714+0000","last_undegraded":"2023-09-23T21:04:37.106714+0000","last_fullsized":"2023-09-23T21:04:37.106714+0000","mapping_epoch":254,"log_start":"266'4588","ondisk_log_start":"266'4588","created":254,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:01:24.325318+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:01:24.325318+0000","last_clean_scrub_stamp":"2023-09-23T21:01:24.325318+0000","objects_scrubbed":0,"log_size":6827,"log_dups_size":0,"ondisk_log_size":6827,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:18:01.918758+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":172163072,"num_objects":2627,"num_object_clones":0,"num_object_copies":7881,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2627,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11415,"num_write_kb":449344,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.7","version":"79'161","reported_seq":626,"reported_epoch":320,"state":"active+clean","last_fresh":"2023-09-23T21:04:32.400070+0000","last_change":"2023-09-23T20:57:48.243572+0000","last_active":"2023-09-23T21:04:32.400070+0000","last_peered":"2023-09-23T21:04:32.400070+0000","last_clean":"2023-09-23T21:04:32.400070+0000","last_became_active":"2023-09-23T20:57:48.243229+0000","last_became_peered":"2023-09-23T20:57:48.243229+0000","last_unstale":"2023-09-23T21:04:32.400070+0000","last_undegraded":"2023-09-23T21:04:32.400070+0000","last_fullsized":"2023-09-23T21:04:32.400070+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":3,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:25.916867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:25.916867+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:23:19.385831+0000","scrub_duration":0.001665323,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":298,"reported_epoch":320,"state":"active+clean","last_fresh":"2023-09-23T21:04:32.426822+0000","last_change":"2023-09-23T20:50:05.867130+0000","last_active":"2023-09-23T21:04:32.426822+0000","last_peered":"2023-09-23T21:04:32.426822+0000","last_clean":"2023-09-23T21:04:32.426822+0000","last_became_active":"2023-09-23T20:50:05.866758+0000","last_became_peered":"2023-09-23T20:50:05.866758+0000","last_unstale":"2023-09-23T21:04:32.426822+0000","last_undegraded":"2023-09-23T21:04:32.426822+0000","last_fullsized":"2023-09-23T21:04:32.426822+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:55:07.548209+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":194,"reported_epoch":320,"state":"active+clean","last_fresh":"2023-09-23T21:04:32.400549+0000","last_change":"2023-09-23T20:54:31.887604+0000","last_active":"2023-09-23T21:04:32.400549+0000","last_peered":"2023-09-23T21:04:32.400549+0000","last_clean":"2023-09-23T21:04:32.400549+0000","last_became_active":"2023-09-23T20:54:31.886752+0000","last_became_peered":"2023-09-23T20:54:31.886752+0000","last_unstale":"2023-09-23T21:04:32.400549+0000","last_undegraded":"2023-09-23T21:04:32.400549+0000","last_fullsized":"2023-09-23T21:04:32.400549+0000","mapping_epoch":136,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:13:58.872176+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":296,"reported_epoch":320,"state":"active+clean","last_fresh":"2023-09-23T21:04:32.426544+0000","last_change":"2023-09-23T20:54:31.899082+0000","last_active":"2023-09-23T21:04:32.426544+0000","last_peered":"2023-09-23T21:04:32.426544+0000","last_clean":"2023-09-23T21:04:32.426544+0000","last_became_active":"2023-09-23T20:54:31.898521+0000","last_became_peered":"2023-09-23T20:54:31.898521+0000","last_unstale":"2023-09-23T21:04:32.426544+0000","last_undegraded":"2023-09-23T21:04:32.426544+0000","last_fullsized":"2023-09-23T21:04:32.426544+0000","mapping_epoch":136,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:25:36.059711+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.8","version":"321'9784","reported_seq":9905,"reported_epoch":321,"state":"active+clean","last_fresh":"2023-09-23T21:04:37.078053+0000","last_change":"2023-09-23T21:04:31.416444+0000","last_active":"2023-09-23T21:04:37.078053+0000","last_peered":"2023-09-23T21:04:37.078053+0000","last_clean":"2023-09-23T21:04:37.078053+0000","last_became_active":"2023-09-23T21:03:30.259253+0000","last_became_peered":"2023-09-23T21:03:30.259253+0000","last_unstale":"2023-09-23T21:04:37.078053+0000","last_undegraded":"2023-09-23T21:04:37.078053+0000","last_fullsized":"2023-09-23T21:04:37.078053+0000","mapping_epoch":254,"log_start":"258'600","ondisk_log_start":"258'600","created":254,"last_epoch_clean":282,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:01:24.325318+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:01:24.325318+0000","last_clean_scrub_stamp":"2023-09-23T21:01:24.325318+0000","objects_scrubbed":649,"log_size":9184,"log_dups_size":0,"ondisk_log_size":9184,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":84639744,"num_objects":1291,"num_object_clones":0,"num_object_copies":3873,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1291,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5637,"num_write_kb":221728,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[5,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.1","version":"79'161","reported_seq":584,"reported_epoch":320,"state":"active+clean","last_fresh":"2023-09-23T21:04:32.400494+0000","last_change":"2023-09-23T20:57:19.669346+0000","last_active":"2023-09-23T21:04:32.400494+0000","last_peered":"2023-09-23T21:04:32.400494+0000","last_clean":"2023-09-23T21:04:32.400494+0000","last_became_active":"2023-09-23T20:57:08.246046+0000","last_became_peered":"2023-09-23T20:57:08.246046+0000","last_unstale":"2023-09-23T21:04:32.400494+0000","last_undegraded":"2023-09-23T21:04:32.400494+0000","last_fullsized":"2023-09-23T21:04:32.400494+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:19.669259+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:19.669259+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:14:03.435531+0000","scrub_duration":0.0045759559999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":211,"reported_epoch":320,"state":"active+clean","last_fresh":"2023-09-23T21:04:32.401473+0000","last_change":"2023-09-23T20:55:35.959514+0000","last_active":"2023-09-23T21:04:32.401473+0000","last_peered":"2023-09-23T21:04:32.401473+0000","last_clean":"2023-09-23T21:04:32.401473+0000","last_became_active":"2023-09-23T20:55:35.959244+0000","last_became_peered":"2023-09-23T20:55:35.959244+0000","last_unstale":"2023-09-23T21:04:32.401473+0000","last_undegraded":"2023-09-23T21:04:32.401473+0000","last_fullsized":"2023-09-23T21:04:32.401473+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:47:32.665316+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":215,"reported_epoch":320,"state":"active+clean","last_fresh":"2023-09-23T21:04:32.399676+0000","last_change":"2023-09-23T20:55:35.949000+0000","last_active":"2023-09-23T21:04:32.399676+0000","last_peered":"2023-09-23T21:04:32.399676+0000","last_clean":"2023-09-23T21:04:32.399676+0000","last_became_active":"2023-09-23T20:55:35.948410+0000","last_became_peered":"2023-09-23T20:55:35.948410+0000","last_unstale":"2023-09-23T21:04:32.399676+0000","last_undegraded":"2023-09-23T21:04:32.399676+0000","last_fullsized":"2023-09-23T21:04:32.399676+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:26:50.477289+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":290,"reported_epoch":320,"state":"active+clean","last_fresh":"2023-09-23T21:04:32.426602+0000","last_change":"2023-09-23T20:51:10.829889+0000","last_active":"2023-09-23T21:04:32.426602+0000","last_peered":"2023-09-23T21:04:32.426602+0000","last_clean":"2023-09-23T21:04:32.426602+0000","last_became_active":"2023-09-23T20:51:10.829533+0000","last_became_peered":"2023-09-23T20:51:10.829533+0000","last_unstale":"2023-09-23T21:04:32.426602+0000","last_undegraded":"2023-09-23T21:04:32.426602+0000","last_fullsized":"2023-09-23T21:04:32.426602+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:43:04.603460+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.b","version":"321'11242","reported_seq":11318,"reported_epoch":321,"state":"active+clean","last_fresh":"2023-09-23T21:04:34.897877+0000","last_change":"2023-09-23T21:01:25.340569+0000","last_active":"2023-09-23T21:04:34.897877+0000","last_peered":"2023-09-23T21:04:34.897877+0000","last_clean":"2023-09-23T21:04:34.897877+0000","last_became_active":"2023-09-23T21:01:25.340209+0000","last_became_peered":"2023-09-23T21:01:25.340209+0000","last_unstale":"2023-09-23T21:04:34.897877+0000","last_undegraded":"2023-09-23T21:04:34.897877+0000","last_fullsized":"2023-09-23T21:04:34.897877+0000","mapping_epoch":254,"log_start":"270'5300","ondisk_log_start":"270'5300","created":254,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:01:24.325318+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:01:24.325318+0000","last_clean_scrub_stamp":"2023-09-23T21:01:24.325318+0000","objects_scrubbed":0,"log_size":5942,"log_dups_size":0,"ondisk_log_size":5942,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:53:59.553810+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":183631872,"num_objects":2802,"num_object_clones":0,"num_object_copies":8406,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2802,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11242,"num_write_kb":449408,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.2","version":"79'161","reported_seq":553,"reported_epoch":320,"state":"active+clean","last_fresh":"2023-09-23T21:04:32.401762+0000","last_change":"2023-09-23T20:57:24.802586+0000","last_active":"2023-09-23T21:04:32.401762+0000","last_peered":"2023-09-23T21:04:32.401762+0000","last_clean":"2023-09-23T21:04:32.401762+0000","last_became_active":"2023-09-23T20:57:08.236114+0000","last_became_peered":"2023-09-23T20:57:08.236114+0000","last_unstale":"2023-09-23T21:04:32.401762+0000","last_undegraded":"2023-09-23T21:04:32.401762+0000","last_fullsized":"2023-09-23T21:04:32.401762+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:24.802451+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:24.802451+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:06:22.309980+0000","scrub_duration":0.0090861250000000005,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,4],"acting":[0,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":309,"reported_epoch":320,"state":"active+clean","last_fresh":"2023-09-23T21:04:32.423178+0000","last_change":"2023-09-23T20:51:10.838054+0000","last_active":"2023-09-23T21:04:32.423178+0000","last_peered":"2023-09-23T21:04:32.423178+0000","last_clean":"2023-09-23T21:04:32.423178+0000","last_became_active":"2023-09-23T20:51:10.837750+0000","last_became_peered":"2023-09-23T20:51:10.837750+0000","last_unstale":"2023-09-23T21:04:32.423178+0000","last_undegraded":"2023-09-23T21:04:32.423178+0000","last_fullsized":"2023-09-23T21:04:32.423178+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:15:51.648085+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":291,"reported_epoch":320,"state":"active+clean","last_fresh":"2023-09-23T21:04:32.426372+0000","last_change":"2023-09-23T20:55:35.951146+0000","last_active":"2023-09-23T21:04:32.426372+0000","last_peered":"2023-09-23T21:04:32.426372+0000","last_clean":"2023-09-23T21:04:32.426372+0000","last_became_active":"2023-09-23T20:55:35.950874+0000","last_became_peered":"2023-09-23T20:55:35.950874+0000","last_unstale":"2023-09-23T21:04:32.426372+0000","last_undegraded":"2023-09-23T21:04:32.426372+0000","last_fullsized":"2023-09-23T21:04:32.426372+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:58:13.957677+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":311,"reported_epoch":320,"state":"active+clean","last_fresh":"2023-09-23T21:04:32.427995+0000","last_change":"2023-09-23T20:54:31.898870+0000","last_active":"2023-09-23T21:04:32.427995+0000","last_peered":"2023-09-23T21:04:32.427995+0000","last_clean":"2023-09-23T21:04:32.427995+0000","last_became_active":"2023-09-23T20:54:31.898467+0000","last_became_peered":"2023-09-23T20:54:31.898467+0000","last_unstale":"2023-09-23T21:04:32.427995+0000","last_undegraded":"2023-09-23T21:04:32.427995+0000","last_fullsized":"2023-09-23T21:04:32.427995+0000","mapping_epoch":136,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:56:22.483408+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.9","version":"321'9451","reported_seq":9646,"reported_epoch":321,"state":"active+clean","last_fresh":"2023-09-23T21:04:34.151174+0000","last_change":"2023-09-23T21:03:37.216201+0000","last_active":"2023-09-23T21:04:34.151174+0000","last_peered":"2023-09-23T21:04:34.151174+0000","last_clean":"2023-09-23T21:04:34.151174+0000","last_became_active":"2023-09-23T21:03:30.239058+0000","last_became_peered":"2023-09-23T21:03:30.239058+0000","last_unstale":"2023-09-23T21:04:34.151174+0000","last_undegraded":"2023-09-23T21:04:34.151174+0000","last_fullsized":"2023-09-23T21:04:34.151174+0000","mapping_epoch":254,"log_start":"258'700","ondisk_log_start":"258'700","created":254,"last_epoch_clean":282,"parent":"0.0","parent_split_bits":0,"last_scrub":"283'8319","last_scrub_stamp":"2023-09-23T21:03:37.216105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:01:24.325318+0000","last_clean_scrub_stamp":"2023-09-23T21:03:37.216105+0000","objects_scrubbed":2492,"log_size":8751,"log_dups_size":0,"ondisk_log_size":8751,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:58:52.183586+0000","scrub_duration":0.68308542000000005,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":88670208,"num_objects":1353,"num_object_clones":0,"num_object_copies":4059,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1353,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1139,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,0],"acting":[6,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":194,"reported_epoch":320,"state":"active+clean","last_fresh":"2023-09-23T21:04:32.399568+0000","last_change":"2023-09-23T20:54:31.883504+0000","last_active":"2023-09-23T21:04:32.399568+0000","last_peered":"2023-09-23T21:04:32.399568+0000","last_clean":"2023-09-23T21:04:32.399568+0000","last_became_active":"2023-09-23T20:54:31.883172+0000","last_became_peered":"2023-09-23T20:54:31.883172+0000","last_unstale":"2023-09-23T21:04:32.399568+0000","last_undegraded":"2023-09-23T21:04:32.399568+0000","last_fullsized":"2023-09-23T21:04:32.399568+0000","mapping_epoch":136,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:45:40.289120+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"79'161","reported_seq":553,"reported_epoch":320,"state":"active+clean","last_fresh":"2023-09-23T21:04:32.401446+0000","last_change":"2023-09-23T20:57:20.738417+0000","last_active":"2023-09-23T21:04:32.401446+0000","last_peered":"2023-09-23T21:04:32.401446+0000","last_clean":"2023-09-23T21:04:32.401446+0000","last_became_active":"2023-09-23T20:57:08.236842+0000","last_became_peered":"2023-09-23T20:57:08.236842+0000","last_unstale":"2023-09-23T21:04:32.401446+0000","last_undegraded":"2023-09-23T21:04:32.401446+0000","last_fullsized":"2023-09-23T21:04:32.401446+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":0,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:20.738273+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:20.738273+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:18:28.327990+0000","scrub_duration":0.0088336339999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":301,"reported_epoch":320,"state":"active+clean","last_fresh":"2023-09-23T21:04:32.423972+0000","last_change":"2023-09-23T20:55:35.950777+0000","last_active":"2023-09-23T21:04:32.423972+0000","last_peered":"2023-09-23T21:04:32.423972+0000","last_clean":"2023-09-23T21:04:32.423972+0000","last_became_active":"2023-09-23T20:55:35.950427+0000","last_became_peered":"2023-09-23T20:55:35.950427+0000","last_unstale":"2023-09-23T21:04:32.423972+0000","last_undegraded":"2023-09-23T21:04:32.423972+0000","last_fullsized":"2023-09-23T21:04:32.423972+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:20:10.371751+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":265,"reported_epoch":320,"state":"active+clean","last_fresh":"2023-09-23T21:04:32.399611+0000","last_change":"2023-09-23T20:55:35.978140+0000","last_active":"2023-09-23T21:04:32.399611+0000","last_peered":"2023-09-23T21:04:32.399611+0000","last_clean":"2023-09-23T21:04:32.399611+0000","last_became_active":"2023-09-23T20:55:35.977831+0000","last_became_peered":"2023-09-23T20:55:35.977831+0000","last_unstale":"2023-09-23T21:04:32.399611+0000","last_undegraded":"2023-09-23T21:04:32.399611+0000","last_fullsized":"2023-09-23T21:04:32.399611+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:25:47.678023+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.a","version":"321'10977","reported_seq":11054,"reported_epoch":321,"state":"active+clean","last_fresh":"2023-09-23T21:04:34.568632+0000","last_change":"2023-09-23T21:01:25.429899+0000","last_active":"2023-09-23T21:04:34.568632+0000","last_peered":"2023-09-23T21:04:34.568632+0000","last_clean":"2023-09-23T21:04:34.568632+0000","last_became_active":"2023-09-23T21:01:25.429583+0000","last_became_peered":"2023-09-23T21:01:25.429583+0000","last_unstale":"2023-09-23T21:04:34.568632+0000","last_undegraded":"2023-09-23T21:04:34.568632+0000","last_fullsized":"2023-09-23T21:04:34.568632+0000","mapping_epoch":254,"log_start":"268'4669","ondisk_log_start":"268'4669","created":254,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:01:24.325318+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:01:24.325318+0000","last_clean_scrub_stamp":"2023-09-23T21:01:24.325318+0000","objects_scrubbed":0,"log_size":6308,"log_dups_size":0,"ondisk_log_size":6308,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:31:11.594129+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":179765248,"num_objects":2743,"num_object_clones":0,"num_object_copies":8229,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2743,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10977,"num_write_kb":439040,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.3","version":"79'161","reported_seq":590,"reported_epoch":320,"state":"active+clean","last_fresh":"2023-09-23T21:04:32.428324+0000","last_change":"2023-09-23T20:57:20.378531+0000","last_active":"2023-09-23T21:04:32.428324+0000","last_peered":"2023-09-23T21:04:32.428324+0000","last_clean":"2023-09-23T21:04:32.428324+0000","last_became_active":"2023-09-23T20:57:08.238019+0000","last_became_peered":"2023-09-23T20:57:08.238019+0000","last_unstale":"2023-09-23T21:04:32.428324+0000","last_undegraded":"2023-09-23T21:04:32.428324+0000","last_fullsized":"2023-09-23T21:04:32.428324+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":3,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:20.377714+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:20.377714+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:00:08.305602+0000","scrub_duration":0.003015228,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":192,"reported_epoch":320,"state":"active+clean","last_fresh":"2023-09-23T21:04:32.426517+0000","last_change":"2023-09-23T20:56:45.991884+0000","last_active":"2023-09-23T21:04:32.426517+0000","last_peered":"2023-09-23T21:04:32.426517+0000","last_clean":"2023-09-23T21:04:32.426517+0000","last_became_active":"2023-09-23T20:56:45.991568+0000","last_became_peered":"2023-09-23T20:56:45.991568+0000","last_unstale":"2023-09-23T21:04:32.426517+0000","last_undegraded":"2023-09-23T21:04:32.426517+0000","last_fullsized":"2023-09-23T21:04:32.426517+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":205,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:21:20.775511+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,0],"acting":[1,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":317,"reported_epoch":320,"state":"active+clean","last_fresh":"2023-09-23T21:04:32.402012+0000","last_change":"2023-09-23T20:57:44.979950+0000","last_active":"2023-09-23T21:04:32.402012+0000","last_peered":"2023-09-23T21:04:32.402012+0000","last_clean":"2023-09-23T21:04:32.402012+0000","last_became_active":"2023-09-23T20:57:44.979346+0000","last_became_peered":"2023-09-23T20:57:44.979346+0000","last_unstale":"2023-09-23T21:04:32.402012+0000","last_undegraded":"2023-09-23T21:04:32.402012+0000","last_fullsized":"2023-09-23T21:04:32.402012+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:10:24.372800+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":295,"reported_epoch":320,"state":"active+clean","last_fresh":"2023-09-23T21:04:32.402287+0000","last_change":"2023-09-23T20:50:37.982235+0000","last_active":"2023-09-23T21:04:32.402287+0000","last_peered":"2023-09-23T21:04:32.402287+0000","last_clean":"2023-09-23T21:04:32.402287+0000","last_became_active":"2023-09-23T20:50:37.981913+0000","last_became_peered":"2023-09-23T20:50:37.981913+0000","last_unstale":"2023-09-23T21:04:32.402287+0000","last_undegraded":"2023-09-23T21:04:32.402287+0000","last_fullsized":"2023-09-23T21:04:32.402287+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:59:57.122584+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.f","version":"321'10883","reported_seq":10960,"reported_epoch":321,"state":"active+clean","last_fresh":"2023-09-23T21:04:34.924960+0000","last_change":"2023-09-23T21:01:25.432176+0000","last_active":"2023-09-23T21:04:34.924960+0000","last_peered":"2023-09-23T21:04:34.924960+0000","last_clean":"2023-09-23T21:04:34.924960+0000","last_became_active":"2023-09-23T21:01:25.431835+0000","last_became_peered":"2023-09-23T21:01:25.431835+0000","last_unstale":"2023-09-23T21:04:34.924960+0000","last_undegraded":"2023-09-23T21:04:34.924960+0000","last_fullsized":"2023-09-23T21:04:34.924960+0000","mapping_epoch":254,"log_start":"269'4944","ondisk_log_start":"269'4944","created":254,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:01:24.325318+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:01:24.325318+0000","last_clean_scrub_stamp":"2023-09-23T21:01:24.325318+0000","objects_scrubbed":0,"log_size":5939,"log_dups_size":0,"ondisk_log_size":5939,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:47:26.553643+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":181075968,"num_objects":2763,"num_object_clones":0,"num_object_copies":8289,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2763,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10883,"num_write_kb":436672,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,1],"acting":[3,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.6","version":"79'161","reported_seq":603,"reported_epoch":320,"state":"active+clean","last_fresh":"2023-09-23T21:04:32.399738+0000","last_change":"2023-09-23T20:57:26.907505+0000","last_active":"2023-09-23T21:04:32.399738+0000","last_peered":"2023-09-23T21:04:32.399738+0000","last_clean":"2023-09-23T21:04:32.399738+0000","last_became_active":"2023-09-23T20:57:08.240321+0000","last_became_peered":"2023-09-23T20:57:08.240321+0000","last_unstale":"2023-09-23T21:04:32.399738+0000","last_undegraded":"2023-09-23T21:04:32.399738+0000","last_fullsized":"2023-09-23T21:04:32.399738+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":3,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:26.907408+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:26.907408+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:34:17.392928+0000","scrub_duration":0.010337038,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":239,"reported_epoch":320,"state":"active+clean","last_fresh":"2023-09-23T21:04:32.401885+0000","last_change":"2023-09-23T20:56:45.988566+0000","last_active":"2023-09-23T21:04:32.401885+0000","last_peered":"2023-09-23T21:04:32.401885+0000","last_clean":"2023-09-23T21:04:32.401885+0000","last_became_active":"2023-09-23T20:56:45.988249+0000","last_became_peered":"2023-09-23T20:56:45.988249+0000","last_unstale":"2023-09-23T21:04:32.401885+0000","last_undegraded":"2023-09-23T21:04:32.401885+0000","last_fullsized":"2023-09-23T21:04:32.401885+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":205,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:51:36.644764+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,4],"acting":[0,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":293,"reported_epoch":320,"state":"active+clean","last_fresh":"2023-09-23T21:04:32.426749+0000","last_change":"2023-09-23T20:54:31.898965+0000","last_active":"2023-09-23T21:04:32.426749+0000","last_peered":"2023-09-23T21:04:32.426749+0000","last_clean":"2023-09-23T21:04:32.426749+0000","last_became_active":"2023-09-23T20:54:31.898663+0000","last_became_peered":"2023-09-23T20:54:31.898663+0000","last_unstale":"2023-09-23T21:04:32.426749+0000","last_undegraded":"2023-09-23T21:04:32.426749+0000","last_fullsized":"2023-09-23T21:04:32.426749+0000","mapping_epoch":136,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:53:05.058087+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":284,"reported_epoch":320,"state":"active+clean","last_fresh":"2023-09-23T21:04:32.401836+0000","last_change":"2023-09-23T20:54:31.884717+0000","last_active":"2023-09-23T21:04:32.401836+0000","last_peered":"2023-09-23T21:04:32.401836+0000","last_clean":"2023-09-23T21:04:32.401836+0000","last_became_active":"2023-09-23T20:54:31.884345+0000","last_became_peered":"2023-09-23T20:54:31.884345+0000","last_unstale":"2023-09-23T21:04:32.401836+0000","last_undegraded":"2023-09-23T21:04:32.401836+0000","last_fullsized":"2023-09-23T21:04:32.401836+0000","mapping_epoch":136,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:02:26.393868+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.c","version":"321'11349","reported_seq":11426,"reported_epoch":321,"state":"active+clean","last_fresh":"2023-09-23T21:04:37.190756+0000","last_change":"2023-09-23T21:01:25.560907+0000","last_active":"2023-09-23T21:04:37.190756+0000","last_peered":"2023-09-23T21:04:37.190756+0000","last_clean":"2023-09-23T21:04:37.190756+0000","last_became_active":"2023-09-23T21:01:25.560592+0000","last_became_peered":"2023-09-23T21:01:25.560592+0000","last_unstale":"2023-09-23T21:04:37.190756+0000","last_undegraded":"2023-09-23T21:04:37.190756+0000","last_fullsized":"2023-09-23T21:04:37.190756+0000","mapping_epoch":254,"log_start":"266'4459","ondisk_log_start":"266'4459","created":254,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:01:24.325318+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:01:24.325318+0000","last_clean_scrub_stamp":"2023-09-23T21:01:24.325318+0000","objects_scrubbed":0,"log_size":6890,"log_dups_size":0,"ondisk_log_size":6890,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:14:22.102068+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":170852352,"num_objects":2607,"num_object_clones":0,"num_object_copies":7821,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2607,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11349,"num_write_kb":446592,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.5","version":"79'161","reported_seq":584,"reported_epoch":320,"state":"active+clean","last_fresh":"2023-09-23T21:04:32.401645+0000","last_change":"2023-09-23T20:57:30.029479+0000","last_active":"2023-09-23T21:04:32.401645+0000","last_peered":"2023-09-23T21:04:32.401645+0000","last_clean":"2023-09-23T21:04:32.401645+0000","last_became_active":"2023-09-23T20:57:08.249909+0000","last_became_peered":"2023-09-23T20:57:08.249909+0000","last_unstale":"2023-09-23T21:04:32.401645+0000","last_undegraded":"2023-09-23T21:04:32.401645+0000","last_fullsized":"2023-09-23T21:04:32.401645+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":3,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:30.029396+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:30.029396+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:16:46.083145+0000","scrub_duration":0.012914682,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":287,"reported_epoch":320,"state":"active+clean","last_fresh":"2023-09-23T21:04:32.423689+0000","last_change":"2023-09-23T20:51:10.836910+0000","last_active":"2023-09-23T21:04:32.423689+0000","last_peered":"2023-09-23T21:04:32.423689+0000","last_clean":"2023-09-23T21:04:32.423689+0000","last_became_active":"2023-09-23T20:51:10.836557+0000","last_became_peered":"2023-09-23T20:51:10.836557+0000","last_unstale":"2023-09-23T21:04:32.423689+0000","last_undegraded":"2023-09-23T21:04:32.423689+0000","last_fullsized":"2023-09-23T21:04:32.423689+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:51:54.019969+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":293,"reported_epoch":320,"state":"active+clean","last_fresh":"2023-09-23T21:04:32.427608+0000","last_change":"2023-09-23T20:55:35.953444+0000","last_active":"2023-09-23T21:04:32.427608+0000","last_peered":"2023-09-23T21:04:32.427608+0000","last_clean":"2023-09-23T21:04:32.427608+0000","last_became_active":"2023-09-23T20:55:35.953151+0000","last_became_peered":"2023-09-23T20:55:35.953151+0000","last_unstale":"2023-09-23T21:04:32.427608+0000","last_undegraded":"2023-09-23T21:04:32.427608+0000","last_fullsized":"2023-09-23T21:04:32.427608+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:39:12.404391+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":112,"reported_epoch":320,"state":"active+clean","last_fresh":"2023-09-23T21:04:32.423836+0000","last_change":"2023-09-23T20:57:44.982509+0000","last_active":"2023-09-23T21:04:32.423836+0000","last_peered":"2023-09-23T21:04:32.423836+0000","last_clean":"2023-09-23T21:04:32.423836+0000","last_became_active":"2023-09-23T20:57:44.982175+0000","last_became_peered":"2023-09-23T20:57:44.982175+0000","last_unstale":"2023-09-23T21:04:32.423836+0000","last_undegraded":"2023-09-23T21:04:32.423836+0000","last_fullsized":"2023-09-23T21:04:32.423836+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:37:49.215488+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.d","version":"321'11048","reported_seq":11124,"reported_epoch":321,"state":"active+clean","last_fresh":"2023-09-23T21:04:37.089677+0000","last_change":"2023-09-23T21:01:25.560531+0000","last_active":"2023-09-23T21:04:37.089677+0000","last_peered":"2023-09-23T21:04:37.089677+0000","last_clean":"2023-09-23T21:04:37.089677+0000","last_became_active":"2023-09-23T21:01:25.560228+0000","last_became_peered":"2023-09-23T21:01:25.560228+0000","last_unstale":"2023-09-23T21:04:37.089677+0000","last_undegraded":"2023-09-23T21:04:37.089677+0000","last_fullsized":"2023-09-23T21:04:37.089677+0000","mapping_epoch":254,"log_start":"267'4519","ondisk_log_start":"267'4519","created":254,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:01:24.325318+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:01:24.325318+0000","last_clean_scrub_stamp":"2023-09-23T21:01:24.325318+0000","objects_scrubbed":0,"log_size":6529,"log_dups_size":0,"ondisk_log_size":6529,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:19:04.504855+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":172949528,"num_objects":2640,"num_object_clones":0,"num_object_copies":7920,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2640,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":11048,"num_write_kb":437953,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,2],"acting":[5,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.4","version":"79'161","reported_seq":602,"reported_epoch":320,"state":"active+clean","last_fresh":"2023-09-23T21:04:32.424542+0000","last_change":"2023-09-23T20:57:20.012904+0000","last_active":"2023-09-23T21:04:32.424542+0000","last_peered":"2023-09-23T21:04:32.424542+0000","last_clean":"2023-09-23T21:04:32.424542+0000","last_became_active":"2023-09-23T20:57:08.237996+0000","last_became_peered":"2023-09-23T20:57:08.237996+0000","last_unstale":"2023-09-23T21:04:32.424542+0000","last_undegraded":"2023-09-23T21:04:32.424542+0000","last_fullsized":"2023-09-23T21:04:32.424542+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":3,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:20.012764+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:20.012764+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:51:28.796946+0000","scrub_duration":0.0045251919999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":270,"reported_epoch":320,"state":"active+clean","last_fresh":"2023-09-23T21:04:32.399915+0000","last_change":"2023-09-23T20:55:35.949541+0000","last_active":"2023-09-23T21:04:32.399915+0000","last_peered":"2023-09-23T21:04:32.399915+0000","last_clean":"2023-09-23T21:04:32.399915+0000","last_became_active":"2023-09-23T20:55:35.944635+0000","last_became_peered":"2023-09-23T20:55:35.944635+0000","last_unstale":"2023-09-23T21:04:32.399915+0000","last_undegraded":"2023-09-23T21:04:32.399915+0000","last_fullsized":"2023-09-23T21:04:32.399915+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:30:36.033585+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"70'1","reported_seq":311,"reported_epoch":320,"state":"active+clean","last_fresh":"2023-09-23T21:04:32.402332+0000","last_change":"2023-09-23T20:54:14.009723+0000","last_active":"2023-09-23T21:04:32.402332+0000","last_peered":"2023-09-23T21:04:32.402332+0000","last_clean":"2023-09-23T21:04:32.402332+0000","last_became_active":"2023-09-23T20:54:13.941771+0000","last_became_peered":"2023-09-23T20:54:13.941771+0000","last_unstale":"2023-09-23T21:04:32.402332+0000","last_undegraded":"2023-09-23T21:04:32.402332+0000","last_fullsized":"2023-09-23T21:04:32.402332+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:48:25.588116+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":225,"reported_epoch":320,"state":"active+clean","last_fresh":"2023-09-23T21:04:32.400067+0000","last_change":"2023-09-23T20:55:35.949780+0000","last_active":"2023-09-23T21:04:32.400067+0000","last_peered":"2023-09-23T21:04:32.400067+0000","last_clean":"2023-09-23T21:04:32.400067+0000","last_became_active":"2023-09-23T20:55:35.949497+0000","last_became_peered":"2023-09-23T20:55:35.949497+0000","last_unstale":"2023-09-23T21:04:32.400067+0000","last_undegraded":"2023-09-23T21:04:32.400067+0000","last_fullsized":"2023-09-23T21:04:32.400067+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:09:37.420882+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.10","version":"321'9759","reported_seq":17270,"reported_epoch":321,"state":"active+clean","last_fresh":"2023-09-23T21:04:37.005065+0000","last_change":"2023-09-23T21:03:46.218495+0000","last_active":"2023-09-23T21:04:37.005065+0000","last_peered":"2023-09-23T21:04:37.005065+0000","last_clean":"2023-09-23T21:04:37.005065+0000","last_became_active":"2023-09-23T21:03:46.185478+0000","last_became_peered":"2023-09-23T21:03:46.185478+0000","last_unstale":"2023-09-23T21:04:37.005065+0000","last_undegraded":"2023-09-23T21:04:37.005065+0000","last_fullsized":"2023-09-23T21:04:37.005065+0000","mapping_epoch":292,"log_start":"258'1700","ondisk_log_start":"258'1700","created":281,"last_epoch_clean":293,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:01:24.325318+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:01:24.325318+0000","last_clean_scrub_stamp":"2023-09-23T21:01:24.325318+0000","objects_scrubbed":0,"log_size":8059,"log_dups_size":0,"ondisk_log_size":8059,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":82903040,"num_objects":1265,"num_object_clones":0,"num_object_copies":3795,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1265,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5619,"num_write_kb":220288,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2466,"num_bytes_recovered":161611776,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,4],"acting":[5,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":286,"reported_epoch":320,"state":"active+clean","last_fresh":"2023-09-23T21:04:32.401844+0000","last_change":"2023-09-23T20:54:13.929196+0000","last_active":"2023-09-23T21:04:32.401844+0000","last_peered":"2023-09-23T21:04:32.401844+0000","last_clean":"2023-09-23T21:04:32.401844+0000","last_became_active":"2023-09-23T20:54:13.928831+0000","last_became_peered":"2023-09-23T20:54:13.928831+0000","last_unstale":"2023-09-23T21:04:32.401844+0000","last_undegraded":"2023-09-23T21:04:32.401844+0000","last_fullsized":"2023-09-23T21:04:32.401844+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:28:44.316851+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":291,"reported_epoch":320,"state":"active+clean","last_fresh":"2023-09-23T21:04:32.424825+0000","last_change":"2023-09-23T20:52:12.855430+0000","last_active":"2023-09-23T21:04:32.424825+0000","last_peered":"2023-09-23T21:04:32.424825+0000","last_clean":"2023-09-23T21:04:32.424825+0000","last_became_active":"2023-09-23T20:52:12.855115+0000","last_became_peered":"2023-09-23T20:52:12.855115+0000","last_unstale":"2023-09-23T21:04:32.424825+0000","last_undegraded":"2023-09-23T21:04:32.424825+0000","last_fullsized":"2023-09-23T21:04:32.424825+0000","mapping_epoch":100,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":101,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:33:23.802214+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"8.17","version":"321'9597","reported_seq":14615,"reported_epoch":321,"state":"active+clean","last_fresh":"2023-09-23T21:04:37.167414+0000","last_change":"2023-09-23T21:04:30.691764+0000","last_active":"2023-09-23T21:04:37.167414+0000","last_peered":"2023-09-23T21:04:37.167414+0000","last_clean":"2023-09-23T21:04:37.167414+0000","last_became_active":"2023-09-23T21:04:25.357991+0000","last_became_peered":"2023-09-23T21:04:25.357991+0000","last_unstale":"2023-09-23T21:04:37.167414+0000","last_undegraded":"2023-09-23T21:04:37.167414+0000","last_fullsized":"2023-09-23T21:04:37.167414+0000","mapping_epoch":315,"log_start":"258'1300","ondisk_log_start":"258'1300","created":281,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:01:24.325318+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:01:24.325318+0000","last_clean_scrub_stamp":"2023-09-23T21:01:24.325318+0000","objects_scrubbed":0,"log_size":8297,"log_dups_size":0,"ondisk_log_size":8297,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":87752704,"num_objects":1339,"num_object_clones":0,"num_object_copies":4017,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1339,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5492,"num_write_kb":218592,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1636,"num_bytes_recovered":107216896,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":198,"reported_epoch":320,"state":"active+clean","last_fresh":"2023-09-23T21:04:32.400332+0000","last_change":"2023-09-23T20:54:31.902946+0000","last_active":"2023-09-23T21:04:32.400332+0000","last_peered":"2023-09-23T21:04:32.400332+0000","last_clean":"2023-09-23T21:04:32.400332+0000","last_became_active":"2023-09-23T20:54:31.889484+0000","last_became_peered":"2023-09-23T20:54:31.889484+0000","last_unstale":"2023-09-23T21:04:32.400332+0000","last_undegraded":"2023-09-23T21:04:32.400332+0000","last_fullsized":"2023-09-23T21:04:32.400332+0000","mapping_epoch":136,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:44:44.278398+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":198,"reported_epoch":320,"state":"active+clean","last_fresh":"2023-09-23T21:04:32.400414+0000","last_change":"2023-09-23T20:54:13.950351+0000","last_active":"2023-09-23T21:04:32.400414+0000","last_peered":"2023-09-23T21:04:32.400414+0000","last_clean":"2023-09-23T21:04:32.400414+0000","last_became_active":"2023-09-23T20:54:13.949987+0000","last_became_peered":"2023-09-23T20:54:13.949987+0000","last_unstale":"2023-09-23T21:04:32.400414+0000","last_undegraded":"2023-09-23T21:04:32.400414+0000","last_fullsized":"2023-09-23T21:04:32.400414+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:46:17.765134+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.16","version":"321'9966","reported_seq":17282,"reported_epoch":321,"state":"active+clean","last_fresh":"2023-09-23T21:04:34.856045+0000","last_change":"2023-09-23T21:04:29.781752+0000","last_active":"2023-09-23T21:04:34.856045+0000","last_peered":"2023-09-23T21:04:34.856045+0000","last_clean":"2023-09-23T21:04:34.856045+0000","last_became_active":"2023-09-23T21:04:24.343644+0000","last_became_peered":"2023-09-23T21:04:24.343644+0000","last_unstale":"2023-09-23T21:04:34.856045+0000","last_undegraded":"2023-09-23T21:04:34.856045+0000","last_fullsized":"2023-09-23T21:04:34.856045+0000","mapping_epoch":314,"log_start":"258'600","ondisk_log_start":"258'600","created":281,"last_epoch_clean":315,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:01:24.325318+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:01:24.325318+0000","last_clean_scrub_stamp":"2023-09-23T21:01:24.325318+0000","objects_scrubbed":0,"log_size":9366,"log_dups_size":0,"ondisk_log_size":9366,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":92864512,"num_objects":1417,"num_object_clones":0,"num_object_copies":4251,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1417,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5688,"num_write_kb":227360,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1801,"num_bytes_recovered":118030336,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,5],"acting":[3,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":229,"reported_epoch":320,"state":"active+clean","last_fresh":"2023-09-23T21:04:32.400637+0000","last_change":"2023-09-23T20:56:18.141728+0000","last_active":"2023-09-23T21:04:32.400637+0000","last_peered":"2023-09-23T21:04:32.400637+0000","last_clean":"2023-09-23T21:04:32.400637+0000","last_became_active":"2023-09-23T20:56:18.137979+0000","last_became_peered":"2023-09-23T20:56:18.137979+0000","last_unstale":"2023-09-23T21:04:32.400637+0000","last_undegraded":"2023-09-23T21:04:32.400637+0000","last_fullsized":"2023-09-23T21:04:32.400637+0000","mapping_epoch":192,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":193,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:58:05.658443+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":198,"reported_epoch":320,"state":"active+clean","last_fresh":"2023-09-23T21:04:32.400920+0000","last_change":"2023-09-23T20:54:13.935307+0000","last_active":"2023-09-23T21:04:32.400920+0000","last_peered":"2023-09-23T21:04:32.400920+0000","last_clean":"2023-09-23T21:04:32.400920+0000","last_became_active":"2023-09-23T20:54:13.935000+0000","last_became_peered":"2023-09-23T20:54:13.935000+0000","last_unstale":"2023-09-23T21:04:32.400920+0000","last_undegraded":"2023-09-23T21:04:32.400920+0000","last_fullsized":"2023-09-23T21:04:32.400920+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:47:40.974082+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.15","version":"321'9781","reported_seq":13544,"reported_epoch":321,"state":"active+clean","last_fresh":"2023-09-23T21:04:34.471601+0000","last_change":"2023-09-23T21:04:30.663986+0000","last_active":"2023-09-23T21:04:34.471601+0000","last_peered":"2023-09-23T21:04:34.471601+0000","last_clean":"2023-09-23T21:04:34.471601+0000","last_became_active":"2023-09-23T21:04:18.282767+0000","last_became_peered":"2023-09-23T21:04:18.282767+0000","last_unstale":"2023-09-23T21:04:34.471601+0000","last_undegraded":"2023-09-23T21:04:34.471601+0000","last_fullsized":"2023-09-23T21:04:34.471601+0000","mapping_epoch":310,"log_start":"258'1000","ondisk_log_start":"258'1000","created":281,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":5,"last_scrub":"318'9682","last_scrub_stamp":"2023-09-23T21:04:30.663825+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:01:24.325318+0000","last_clean_scrub_stamp":"2023-09-23T21:04:30.663825+0000","objects_scrubbed":1516,"log_size":8781,"log_dups_size":0,"ondisk_log_size":8781,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:01:18.334239+0000","scrub_duration":0.41809476499999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":92733440,"num_objects":1415,"num_object_clones":0,"num_object_copies":4245,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1415,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,6],"acting":[1,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":310,"reported_epoch":320,"state":"active+clean","last_fresh":"2023-09-23T21:04:32.424305+0000","last_change":"2023-09-23T20:55:35.954870+0000","last_active":"2023-09-23T21:04:32.424305+0000","last_peered":"2023-09-23T21:04:32.424305+0000","last_clean":"2023-09-23T21:04:32.424305+0000","last_became_active":"2023-09-23T20:55:35.954568+0000","last_became_peered":"2023-09-23T20:55:35.954568+0000","last_unstale":"2023-09-23T21:04:32.424305+0000","last_undegraded":"2023-09-23T21:04:32.424305+0000","last_fullsized":"2023-09-23T21:04:32.424305+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:06:22.423566+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":309,"reported_epoch":320,"state":"active+clean","last_fresh":"2023-09-23T21:04:32.423130+0000","last_change":"2023-09-23T20:51:10.838334+0000","last_active":"2023-09-23T21:04:32.423130+0000","last_peered":"2023-09-23T21:04:32.423130+0000","last_clean":"2023-09-23T21:04:32.423130+0000","last_became_active":"2023-09-23T20:51:10.838026+0000","last_became_peered":"2023-09-23T20:51:10.838026+0000","last_unstale":"2023-09-23T21:04:32.423130+0000","last_undegraded":"2023-09-23T21:04:32.423130+0000","last_fullsized":"2023-09-23T21:04:32.423130+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:04:37.009457+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.14","version":"321'9751","reported_seq":13616,"reported_epoch":321,"state":"active+clean","last_fresh":"2023-09-23T21:04:37.267814+0000","last_change":"2023-09-23T21:04:23.270430+0000","last_active":"2023-09-23T21:04:37.267814+0000","last_peered":"2023-09-23T21:04:37.267814+0000","last_clean":"2023-09-23T21:04:37.267814+0000","last_became_active":"2023-09-23T21:04:14.227313+0000","last_became_peered":"2023-09-23T21:04:14.227313+0000","last_unstale":"2023-09-23T21:04:37.267814+0000","last_undegraded":"2023-09-23T21:04:37.267814+0000","last_fullsized":"2023-09-23T21:04:37.267814+0000","mapping_epoch":307,"log_start":"258'500","ondisk_log_start":"258'500","created":281,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:01:24.325318+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:01:24.325318+0000","last_clean_scrub_stamp":"2023-09-23T21:01:24.325318+0000","objects_scrubbed":0,"log_size":9251,"log_dups_size":0,"ondisk_log_size":9251,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":86278144,"num_objects":1317,"num_object_clones":0,"num_object_copies":3951,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1317,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5607,"num_write_kb":221536,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1877,"num_bytes_recovered":123011072,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,2],"acting":[4,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":314,"reported_epoch":320,"state":"active+clean","last_fresh":"2023-09-23T21:04:32.402107+0000","last_change":"2023-09-23T20:54:31.903139+0000","last_active":"2023-09-23T21:04:32.402107+0000","last_peered":"2023-09-23T21:04:32.402107+0000","last_clean":"2023-09-23T21:04:32.402107+0000","last_became_active":"2023-09-23T20:54:31.884520+0000","last_became_peered":"2023-09-23T20:54:31.884520+0000","last_unstale":"2023-09-23T21:04:32.402107+0000","last_undegraded":"2023-09-23T21:04:32.402107+0000","last_fullsized":"2023-09-23T21:04:32.402107+0000","mapping_epoch":136,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:39:34.269705+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1108,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":215,"reported_epoch":320,"state":"active+clean","last_fresh":"2023-09-23T21:04:32.400641+0000","last_change":"2023-09-23T20:55:35.951572+0000","last_active":"2023-09-23T21:04:32.400641+0000","last_peered":"2023-09-23T21:04:32.400641+0000","last_clean":"2023-09-23T21:04:32.400641+0000","last_became_active":"2023-09-23T20:55:35.951294+0000","last_became_peered":"2023-09-23T20:55:35.951294+0000","last_unstale":"2023-09-23T21:04:32.400641+0000","last_undegraded":"2023-09-23T21:04:32.400641+0000","last_fullsized":"2023-09-23T21:04:32.400641+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:36:32.447210+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":22,"num_bytes_recovered":5990,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":8,"num_pg":26,"stat_sum":{"num_bytes":2835841048,"num_objects":43272,"num_object_clones":0,"num_object_copies":129816,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":43272,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":135642,"num_write_kb":5093505,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":15252,"num_bytes_recovered":999292928,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":9911545856,"data_stored":9911533640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":210895,"ondisk_log_size":210895,"up":78,"acting":78,"num_store_stats":8},{"poolid":1,"num_pg":21,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3381,"ondisk_log_size":3381,"up":63,"acting":63,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201570,"num_pgs":50,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1466060,"kb_used_data":1192324,"kb_used_omap":0,"kb_used_meta":273728,"kb_avail":92279092,"statfs":{"total":95995035648,"available":94493790208,"internally_reserved":0,"allocated":1220939776,"data_stored":1219894984,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":280297472},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1],"upper_bound":2},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:03:53 2023","interfaces":[{"interface":"back","average":{"1min":1.653,"5min":1.9730000000000001,"15min":1.5},"min":{"1min":0.22700000000000001,"5min":0.22700000000000001,"15min":0.21099999999999999},"max":{"1min":8.4990000000000006,"5min":33.326000000000001,"15min":43.994999999999997},"last":0.253},{"interface":"front","average":{"1min":2.0990000000000002,"5min":2.1819999999999999,"15min":1.639},"min":{"1min":0.314,"5min":0.28599999999999998,"15min":0.218},"max":{"1min":8.5150000000000006,"5min":33.354999999999997,"15min":44.206000000000003},"last":0.42699999999999999}]},{"osd":1,"last update":"Sat Sep 23 21:04:34 2023","interfaces":[{"interface":"back","average":{"1min":1.2250000000000001,"5min":1.381,"15min":1.5009999999999999},"min":{"1min":0.245,"5min":0.245,"15min":0.23300000000000001},"max":{"1min":5.7270000000000003,"5min":12.061,"15min":22.591999999999999},"last":0.27300000000000002},{"interface":"front","average":{"1min":1.28,"5min":1.603,"15min":1.4390000000000001},"min":{"1min":0.27900000000000003,"5min":0.25900000000000001,"15min":0.23699999999999999},"max":{"1min":4.8170000000000002,"5min":13.981,"15min":21.437000000000001},"last":0.79600000000000004}]},{"osd":2,"last update":"Sat Sep 23 21:04:06 2023","interfaces":[{"interface":"back","average":{"1min":1.554,"5min":1.9530000000000001,"15min":1.617},"min":{"1min":0.34200000000000003,"5min":0.30099999999999999,"15min":0.24399999999999999},"max":{"1min":5.5890000000000004,"5min":10.279999999999999,"15min":10.805999999999999},"last":0.44400000000000001},{"interface":"front","average":{"1min":1.5069999999999999,"5min":1.8779999999999999,"15min":1.6240000000000001},"min":{"1min":0.29999999999999999,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":7.1589999999999998,"5min":11.590999999999999,"15min":45.25},"last":0.60099999999999998}]},{"osd":3,"last update":"Sat Sep 23 21:03:42 2023","interfaces":[{"interface":"back","average":{"1min":2.2189999999999999,"5min":2.419,"15min":2.2069999999999999},"min":{"1min":0.29099999999999998,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":7.8689999999999998,"5min":31.786000000000001,"15min":36.408000000000001},"last":0.5},{"interface":"front","average":{"1min":2.1120000000000001,"5min":2.5630000000000002,"15min":2.0680000000000001},"min":{"1min":0.34399999999999997,"5min":0.309,"15min":0.26000000000000001},"max":{"1min":6.6680000000000001,"5min":32.429000000000002,"15min":35.593000000000004},"last":0.55200000000000005}]},{"osd":4,"last update":"Sat Sep 23 21:04:34 2023","interfaces":[{"interface":"back","average":{"1min":0.55600000000000005,"5min":0.75600000000000001,"15min":0.69899999999999995},"min":{"1min":0.248,"5min":0.248,"15min":0.23699999999999999},"max":{"1min":2.4399999999999999,"5min":5.2640000000000002,"15min":5.8920000000000003},"last":0.34399999999999997},{"interface":"front","average":{"1min":0.66800000000000004,"5min":0.91000000000000003,"15min":0.75},"min":{"1min":0.28399999999999997,"5min":0.28000000000000003,"15min":0.26300000000000001},"max":{"1min":2.113,"5min":6.3979999999999997,"15min":6.3979999999999997},"last":0.47899999999999998}]},{"osd":5,"last update":"Sat Sep 23 21:04:34 2023","interfaces":[{"interface":"back","average":{"1min":0.52100000000000002,"5min":0.78100000000000003,"15min":0.71999999999999997},"min":{"1min":0.31900000000000001,"5min":0.28999999999999998,"15min":0.28499999999999998},"max":{"1min":1.2250000000000001,"5min":5.2089999999999996,"15min":7.0389999999999997},"last":0.31900000000000001},{"interface":"front","average":{"1min":0.63900000000000001,"5min":0.84699999999999998,"15min":0.71899999999999997},"min":{"1min":0.33900000000000002,"5min":0.32100000000000001,"15min":0.27800000000000002},"max":{"1min":1.5169999999999999,"5min":5.8529999999999998,"15min":5.8529999999999998},"last":0.46200000000000002}]},{"osd":6,"last update":"Sat Sep 23 21:03:49 2023","interfaces":[{"interface":"back","average":{"1min":1.2490000000000001,"5min":0.90400000000000003,"15min":0.76200000000000001},"min":{"1min":0.41899999999999998,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":5.4699999999999998,"5min":6.1219999999999999,"15min":6.1219999999999999},"last":0.57899999999999996},{"interface":"front","average":{"1min":1.3400000000000001,"5min":0.94899999999999995,"15min":0.85599999999999998},"min":{"1min":0.38500000000000001,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":6.0880000000000001,"5min":6.0919999999999996,"15min":6.0919999999999996},"last":0.52600000000000002}]}]},{"osd":6,"up_from":164,"seq":704374636653,"num_pgs":48,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1258872,"kb_used_data":1070704,"kb_used_omap":10,"kb_used_meta":188149,"kb_avail":92486280,"statfs":{"total":95995035648,"available":94705950720,"internally_reserved":0,"allocated":1096400896,"data_stored":1095376149,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":10328,"internal_metadata":192665512},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:03:51 2023","interfaces":[{"interface":"back","average":{"1min":1.621,"5min":1.794,"15min":1.367},"min":{"1min":0.246,"5min":0.23499999999999999,"15min":0.20999999999999999},"max":{"1min":4.6420000000000003,"5min":14.151999999999999,"15min":14.151999999999999},"last":0.41099999999999998},{"interface":"front","average":{"1min":1.2490000000000001,"5min":1.929,"15min":1.4710000000000001},"min":{"1min":0.25800000000000001,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":4.4450000000000003,"5min":11.023,"15min":11.023},"last":0.28399999999999997}]},{"osd":1,"last update":"Sat Sep 23 21:03:51 2023","interfaces":[{"interface":"back","average":{"1min":2.1070000000000002,"5min":1.696,"15min":1.194},"min":{"1min":0.35199999999999998,"5min":0.26400000000000001,"15min":0.25600000000000001},"max":{"1min":10.026,"5min":10.026,"15min":10.026},"last":1.331},{"interface":"front","average":{"1min":2.4009999999999998,"5min":1.972,"15min":1.361},"min":{"1min":0.26200000000000001,"5min":0.26100000000000001,"15min":0.23000000000000001},"max":{"1min":10.218,"5min":10.518000000000001,"15min":10.518000000000001},"last":0.26700000000000002}]},{"osd":2,"last update":"Sat Sep 23 21:03:51 2023","interfaces":[{"interface":"back","average":{"1min":1.696,"5min":2.3780000000000001,"15min":2.0019999999999998},"min":{"1min":0.36299999999999999,"5min":0.28499999999999998,"15min":0.27700000000000002},"max":{"1min":4.54,"5min":14.029999999999999,"15min":14.029999999999999},"last":1.3540000000000001},{"interface":"front","average":{"1min":3.4460000000000002,"5min":2.5950000000000002,"15min":1.7430000000000001},"min":{"1min":0.26500000000000001,"5min":0.247,"15min":0.247},"max":{"1min":11.672000000000001,"5min":20.195,"15min":20.195},"last":2.3580000000000001}]},{"osd":3,"last update":"Sat Sep 23 21:03:51 2023","interfaces":[{"interface":"back","average":{"1min":4.1580000000000004,"5min":2.7909999999999999,"15min":1.7310000000000001},"min":{"1min":0.28699999999999998,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":43.085999999999999,"5min":43.085999999999999,"15min":43.085999999999999},"last":3.9279999999999999},{"interface":"front","average":{"1min":4.0279999999999996,"5min":2.4079999999999999,"15min":1.917},"min":{"1min":0.313,"5min":0.313,"15min":0.28599999999999998},"max":{"1min":42.405000000000001,"5min":42.405000000000001,"15min":42.405000000000001},"last":5.5869999999999997}]},{"osd":4,"last update":"Sat Sep 23 21:03:51 2023","interfaces":[{"interface":"back","average":{"1min":1.1619999999999999,"5min":0.93700000000000006,"15min":0.94899999999999995},"min":{"1min":0.28799999999999998,"5min":0.24299999999999999,"15min":0.23000000000000001},"max":{"1min":2.242,"5min":5.3920000000000003,"15min":9.0079999999999991},"last":0.34599999999999997},{"interface":"front","average":{"1min":0.61699999999999999,"5min":0.78900000000000003,"15min":0.60899999999999999},"min":{"1min":0.30099999999999999,"5min":0.26600000000000001,"15min":0.25800000000000001},"max":{"1min":1.782,"5min":4.8760000000000003,"15min":4.8760000000000003},"last":0.45800000000000002}]},{"osd":5,"last update":"Sat Sep 23 21:03:51 2023","interfaces":[{"interface":"back","average":{"1min":1.341,"5min":1.133,"15min":0.748},"min":{"1min":0.34999999999999998,"5min":0.27500000000000002,"15min":0.27300000000000002},"max":{"1min":5.5170000000000003,"5min":5.9000000000000004,"15min":5.9000000000000004},"last":0.375},{"interface":"front","average":{"1min":1.403,"5min":1.0169999999999999,"15min":0.871},"min":{"1min":0.33200000000000002,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":8.9610000000000003,"5min":8.9610000000000003,"15min":8.9610000000000003},"last":0.47499999999999998}]},{"osd":7,"last update":"Sat Sep 23 21:03:51 2023","interfaces":[{"interface":"back","average":{"1min":1.3340000000000001,"5min":1.0920000000000001,"15min":0.95799999999999996},"min":{"1min":0.30499999999999999,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":8.9260000000000002,"5min":8.9260000000000002,"15min":8.9260000000000002},"last":0.44},{"interface":"front","average":{"1min":1.1719999999999999,"5min":0.88900000000000001,"15min":0.69399999999999995},"min":{"1min":0.379,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":4.6589999999999998,"5min":4.6589999999999998,"15min":8.282},"last":1.9870000000000001}]}]},{"osd":1,"up_from":88,"seq":377957122224,"num_pgs":48,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1701208,"kb_used_data":1303568,"kb_used_omap":9,"kb_used_meta":397558,"kb_avail":92043944,"statfs":{"total":95995035648,"available":94252998656,"internally_reserved":0,"allocated":1334853632,"data_stored":1333337662,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9239,"internal_metadata":407100393},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:04:32 2023","interfaces":[{"interface":"back","average":{"1min":2.3250000000000002,"5min":2.613,"15min":2.274},"min":{"1min":0.215,"5min":0.215,"15min":0.20699999999999999},"max":{"1min":9.7430000000000003,"5min":45.542000000000002,"15min":45.542000000000002},"last":9.7430000000000003},{"interface":"front","average":{"1min":2.8679999999999999,"5min":3.2570000000000001,"15min":2.6219999999999999},"min":{"1min":0.32200000000000001,"5min":0.248,"15min":0.22500000000000001},"max":{"1min":11.715,"5min":46.353000000000002,"15min":46.353000000000002},"last":9.8450000000000006}]},{"osd":2,"last update":"Sat Sep 23 21:04:04 2023","interfaces":[{"interface":"back","average":{"1min":1.9410000000000001,"5min":2.0350000000000001,"15min":1.7290000000000001},"min":{"1min":0.32200000000000001,"5min":0.20399999999999999,"15min":0.20300000000000001},"max":{"1min":6.9029999999999996,"5min":13.752000000000001,"15min":14.335000000000001},"last":1.6579999999999999},{"interface":"front","average":{"1min":4.444,"5min":3.8159999999999998,"15min":2.835},"min":{"1min":0.315,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":13.589,"5min":18.026,"15min":38.951000000000001},"last":0.38800000000000001}]},{"osd":3,"last update":"Sat Sep 23 21:04:32 2023","interfaces":[{"interface":"back","average":{"1min":1.528,"5min":2.7690000000000001,"15min":2.3620000000000001},"min":{"1min":0.23999999999999999,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":6.1689999999999996,"5min":22.686,"15min":25.314},"last":3.895},{"interface":"front","average":{"1min":1.472,"5min":2.1680000000000001,"15min":1.9970000000000001},"min":{"1min":0.33100000000000002,"5min":0.25600000000000001,"15min":0.20599999999999999},"max":{"1min":5.5679999999999996,"5min":14.473000000000001,"15min":19.881},"last":1.7549999999999999}]},{"osd":4,"last update":"Sat Sep 23 21:04:32 2023","interfaces":[{"interface":"back","average":{"1min":1.581,"5min":2.2709999999999999,"15min":1.7549999999999999},"min":{"1min":0.34399999999999997,"5min":0.28399999999999997,"15min":0.28299999999999997},"max":{"1min":9.7599999999999998,"5min":41.823,"15min":41.823},"last":3.343},{"interface":"front","average":{"1min":1.2410000000000001,"5min":1.8340000000000001,"15min":1.4670000000000001},"min":{"1min":0.30199999999999999,"5min":0.30199999999999999,"15min":0.27600000000000002},"max":{"1min":4.7160000000000002,"5min":41.423000000000002,"15min":41.423000000000002},"last":1.6120000000000001}]},{"osd":5,"last update":"Sat Sep 23 21:04:32 2023","interfaces":[{"interface":"back","average":{"1min":1.6279999999999999,"5min":2.0059999999999998,"15min":1.728},"min":{"1min":0.39100000000000001,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":6.1920000000000002,"5min":12.699999999999999,"15min":15.041},"last":0.42299999999999999},{"interface":"front","average":{"1min":1.2330000000000001,"5min":1.804,"15min":1.625},"min":{"1min":0.41099999999999998,"5min":0.35999999999999999,"15min":0.32200000000000001},"max":{"1min":4.242,"5min":18.280999999999999,"15min":18.280999999999999},"last":3.2759999999999998}]},{"osd":6,"last update":"Sat Sep 23 21:03:54 2023","interfaces":[{"interface":"back","average":{"1min":2.5110000000000001,"5min":2.0510000000000002,"15min":1.5780000000000001},"min":{"1min":0.40799999999999997,"5min":0.35799999999999998,"15min":0.30299999999999999},"max":{"1min":8.0980000000000008,"5min":18.984999999999999,"15min":18.984999999999999},"last":3.379},{"interface":"front","average":{"1min":1.857,"5min":1.6060000000000001,"15min":1.4550000000000001},"min":{"1min":0.40799999999999997,"5min":0.33400000000000002,"15min":0.309},"max":{"1min":6.7430000000000003,"5min":7.6779999999999999,"15min":8.7530000000000001},"last":1.6970000000000001}]},{"osd":7,"last update":"Sat Sep 23 21:04:32 2023","interfaces":[{"interface":"back","average":{"1min":1.339,"5min":1.6040000000000001,"15min":1.4730000000000001},"min":{"1min":0.41699999999999998,"5min":0.39600000000000002,"15min":0.35699999999999998},"max":{"1min":4.6760000000000002,"5min":9.5429999999999993,"15min":11.186999999999999},"last":1.7250000000000001},{"interface":"front","average":{"1min":1.6579999999999999,"5min":2.0219999999999998,"15min":1.77},"min":{"1min":0.44800000000000001,"5min":0.36499999999999999,"15min":0.34599999999999997},"max":{"1min":6.234,"5min":12.84,"15min":15.098000000000001},"last":0.44800000000000001}]}]},{"osd":0,"up_from":83,"seq":356482285751,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1561708,"kb_used_data":1168804,"kb_used_omap":28,"kb_used_meta":392867,"kb_avail":92183444,"statfs":{"total":95995035648,"available":94395846656,"internally_reserved":0,"allocated":1196855296,"data_stored":1195379705,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":28848,"internal_metadata":402296656},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 21:04:23 2023","interfaces":[{"interface":"back","average":{"1min":2.0190000000000001,"5min":3.4220000000000002,"15min":2.5150000000000001},"min":{"1min":0.216,"5min":0.19500000000000001,"15min":0.191},"max":{"1min":10.778,"5min":40.924999999999997,"15min":40.924999999999997},"last":0.307},{"interface":"front","average":{"1min":2.1659999999999999,"5min":3.2290000000000001,"15min":2.4870000000000001},"min":{"1min":0.32000000000000001,"5min":0.24099999999999999,"15min":0.20999999999999999},"max":{"1min":7.5590000000000002,"5min":22.474,"15min":22.474},"last":0.499}]},{"osd":2,"last update":"Sat Sep 23 21:04:01 2023","interfaces":[{"interface":"back","average":{"1min":2.3319999999999999,"5min":2.8839999999999999,"15min":2.4590000000000001},"min":{"1min":0.25600000000000001,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":5.673,"5min":21.053000000000001,"15min":21.053000000000001},"last":0.32900000000000001},{"interface":"front","average":{"1min":2.6659999999999999,"5min":3.0219999999999998,"15min":2.6040000000000001},"min":{"1min":0.23499999999999999,"5min":0.23100000000000001,"15min":0.21099999999999999},"max":{"1min":11.064,"5min":13.956,"15min":21.747},"last":0.34000000000000002}]},{"osd":3,"last update":"Sat Sep 23 21:04:23 2023","interfaces":[{"interface":"back","average":{"1min":2.133,"5min":3.1829999999999998,"15min":2.5179999999999998},"min":{"1min":0.28999999999999998,"5min":0.27100000000000002,"15min":0.25800000000000001},"max":{"1min":10.866,"5min":41.079000000000001,"15min":41.079000000000001},"last":0.35599999999999998},{"interface":"front","average":{"1min":2.0190000000000001,"5min":2.9009999999999998,"15min":2.6019999999999999},"min":{"1min":0.30099999999999999,"5min":0.27400000000000002,"15min":0.23300000000000001},"max":{"1min":5.6390000000000002,"5min":12.167999999999999,"15min":21.167000000000002},"last":0.38200000000000001}]},{"osd":4,"last update":"Sat Sep 23 21:04:01 2023","interfaces":[{"interface":"back","average":{"1min":1.643,"5min":1.9630000000000001,"15min":1.9119999999999999},"min":{"1min":0.28699999999999998,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":11.025,"5min":11.025,"15min":21.594999999999999},"last":0.54500000000000004},{"interface":"front","average":{"1min":1.9319999999999999,"5min":2.105,"15min":1.613},"min":{"1min":0.32600000000000001,"5min":0.32600000000000001,"15min":0.28100000000000003},"max":{"1min":10.907999999999999,"5min":13.904,"15min":18.724},"last":0.57299999999999995}]},{"osd":5,"last update":"Sat Sep 23 21:04:01 2023","interfaces":[{"interface":"back","average":{"1min":1.909,"5min":2.0670000000000002,"15min":1.6499999999999999},"min":{"1min":0.32300000000000001,"5min":0.309,"15min":0.309},"max":{"1min":5.5640000000000001,"5min":11.707000000000001,"15min":11.707000000000001},"last":0.46000000000000002},{"interface":"front","average":{"1min":2.452,"5min":2.1200000000000001,"15min":1.966},"min":{"1min":0.32700000000000001,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":10.742000000000001,"5min":10.742000000000001,"15min":20.512},"last":0.51200000000000001}]},{"osd":6,"last update":"Sat Sep 23 21:03:56 2023","interfaces":[{"interface":"back","average":{"1min":2.1589999999999998,"5min":2.1600000000000001,"15min":1.903},"min":{"1min":0.36099999999999999,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":11,"5min":13.933,"15min":13.933},"last":0.59499999999999997},{"interface":"front","average":{"1min":2.0129999999999999,"5min":2.1560000000000001,"15min":1.9550000000000001},"min":{"1min":0.41299999999999998,"5min":0.379,"15min":0.377},"max":{"1min":11.066000000000001,"5min":13.962,"15min":13.962},"last":0.55100000000000005}]},{"osd":7,"last update":"Sat Sep 23 21:04:01 2023","interfaces":[{"interface":"back","average":{"1min":1.9910000000000001,"5min":2.1509999999999998,"15min":1.704},"min":{"1min":0.36199999999999999,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":5.5990000000000002,"5min":11.657,"15min":11.657},"last":0.437},{"interface":"front","average":{"1min":4.1859999999999999,"5min":2.5979999999999999,"15min":1.804},"min":{"1min":0.47299999999999998,"5min":0.41499999999999998,"15min":0.377},"max":{"1min":40.984999999999999,"5min":40.984999999999999,"15min":40.984999999999999},"last":0.61199999999999999}]}]},{"osd":2,"up_from":93,"seq":399431958698,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1772980,"kb_used_data":1305132,"kb_used_omap":81,"kb_used_meta":467694,"kb_avail":91972172,"statfs":{"total":95995035648,"available":94179504128,"internally_reserved":0,"allocated":1336455168,"data_stored":1334969136,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":83750,"internal_metadata":478918874},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1,0,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:03:52 2023","interfaces":[{"interface":"back","average":{"1min":1.873,"5min":2.7090000000000001,"15min":1.863},"min":{"1min":0.22600000000000001,"5min":0.189,"15min":0.17999999999999999},"max":{"1min":7.4649999999999999,"5min":29.498000000000001,"15min":29.498000000000001},"last":2.9079999999999999},{"interface":"front","average":{"1min":2.1459999999999999,"5min":2.6150000000000002,"15min":2.1680000000000001},"min":{"1min":0.27000000000000002,"5min":0.193,"15min":0.193},"max":{"1min":7.5839999999999996,"5min":17.251000000000001,"15min":17.251000000000001},"last":0.34000000000000002}]},{"osd":1,"last update":"Sat Sep 23 21:03:52 2023","interfaces":[{"interface":"back","average":{"1min":2.29,"5min":2.3479999999999999,"15min":2.1240000000000001},"min":{"1min":0.27900000000000003,"5min":0.246,"15min":0.20200000000000001},"max":{"1min":10.698,"5min":13.523,"15min":16.081},"last":2.98},{"interface":"front","average":{"1min":2.1589999999999998,"5min":3.1749999999999998,"15min":2.6880000000000002},"min":{"1min":0.32200000000000001,"5min":0.26300000000000001,"15min":0.23599999999999999},"max":{"1min":8.8170000000000002,"5min":26.552,"15min":26.552},"last":0.45200000000000001}]},{"osd":3,"last update":"Sat Sep 23 21:03:34 2023","interfaces":[{"interface":"back","average":{"1min":3.6499999999999999,"5min":3.8780000000000001,"15min":2.7389999999999999},"min":{"1min":0.35199999999999998,"5min":0.251,"15min":0.23300000000000001},"max":{"1min":9.8989999999999991,"5min":41.057000000000002,"15min":41.057000000000002},"last":0.624},{"interface":"front","average":{"1min":4.8369999999999997,"5min":4.2880000000000003,"15min":3.1269999999999998},"min":{"1min":0.48599999999999999,"5min":0.25,"15min":0.221},"max":{"1min":10.550000000000001,"5min":41.012,"15min":41.012},"last":3.8799999999999999}]},{"osd":4,"last update":"Sat Sep 23 21:03:52 2023","interfaces":[{"interface":"back","average":{"1min":1.405,"5min":1.4279999999999999,"15min":1.413},"min":{"1min":0.38300000000000001,"5min":0.28399999999999997,"15min":0.26400000000000001},"max":{"1min":4.6470000000000002,"5min":10.045,"15min":10.045},"last":0.38300000000000001},{"interface":"front","average":{"1min":2.3130000000000002,"5min":2.3210000000000002,"15min":1.7769999999999999},"min":{"1min":0.36099999999999999,"5min":0.32400000000000001,"15min":0.313},"max":{"1min":9.6359999999999992,"5min":14.039999999999999,"15min":14.039999999999999},"last":3.851}]},{"osd":5,"last update":"Sat Sep 23 21:03:52 2023","interfaces":[{"interface":"back","average":{"1min":1.3149999999999999,"5min":1.488,"15min":1.526},"min":{"1min":0.34499999999999997,"5min":0.30399999999999999,"15min":0.29299999999999998},"max":{"1min":4.6820000000000004,"5min":10.069000000000001,"15min":10.069000000000001},"last":0.43099999999999999},{"interface":"front","average":{"1min":2.4260000000000002,"5min":2.2050000000000001,"15min":1.766},"min":{"1min":0.38700000000000001,"5min":0.31900000000000001,"15min":0.30299999999999999},"max":{"1min":9.7050000000000001,"5min":13.98,"15min":13.98},"last":3.794}]},{"osd":6,"last update":"Sat Sep 23 21:03:50 2023","interfaces":[{"interface":"back","average":{"1min":2.4430000000000001,"5min":2.3969999999999998,"15min":1.96},"min":{"1min":0.44800000000000001,"5min":0.35699999999999998,"15min":0.312},"max":{"1min":9.6820000000000004,"5min":14.016,"15min":14.244999999999999},"last":3.8290000000000002},{"interface":"front","average":{"1min":2.4209999999999998,"5min":2.476,"15min":1.6499999999999999},"min":{"1min":0.41299999999999998,"5min":0.38,"15min":0.374},"max":{"1min":9.734,"5min":14.068,"15min":14.068},"last":4.5350000000000001}]},{"osd":7,"last update":"Sat Sep 23 21:03:52 2023","interfaces":[{"interface":"back","average":{"1min":1.3300000000000001,"5min":1.335,"15min":1.228},"min":{"1min":0.39200000000000002,"5min":0.35899999999999999,"15min":0.315},"max":{"1min":3.5779999999999998,"5min":13.427,"15min":13.427},"last":2.944},{"interface":"front","average":{"1min":1.4510000000000001,"5min":1.512,"15min":1.5449999999999999},"min":{"1min":0.41199999999999998,"5min":0.36699999999999999,"15min":0.34499999999999997},"max":{"1min":4.71,"5min":10.092000000000001,"15min":10.092000000000001},"last":0.40799999999999997}]}]},{"osd":3,"up_from":98,"seq":420906795171,"num_pgs":51,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1717336,"kb_used_data":1219984,"kb_used_omap":7,"kb_used_meta":497336,"kb_avail":92027816,"statfs":{"total":95995035648,"available":94236483584,"internally_reserved":0,"allocated":1249263616,"data_stored":1247803794,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7576,"internal_metadata":509272680},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,2],"upper_bound":32},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:03:38 2023","interfaces":[{"interface":"back","average":{"1min":2.5150000000000001,"5min":3.5449999999999999,"15min":2.5470000000000002},"min":{"1min":0.189,"5min":0.188,"15min":0.188},"max":{"1min":11.717000000000001,"5min":27.962,"15min":27.962},"last":3.5499999999999998},{"interface":"front","average":{"1min":2.6880000000000002,"5min":3.7330000000000001,"15min":2.4260000000000002},"min":{"1min":0.27200000000000002,"5min":0.20499999999999999,"15min":0.19800000000000001},"max":{"1min":11.124000000000001,"5min":28.058,"15min":28.058},"last":0.442}]},{"osd":1,"last update":"Sat Sep 23 21:03:38 2023","interfaces":[{"interface":"back","average":{"1min":2.504,"5min":3.5880000000000001,"15min":2.5129999999999999},"min":{"1min":0.252,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":11.92,"5min":74.093000000000004,"15min":74.093000000000004},"last":1.7350000000000001},{"interface":"front","average":{"1min":2.8340000000000001,"5min":3.8359999999999999,"15min":2.6600000000000001},"min":{"1min":0.313,"5min":0.23400000000000001,"15min":0.218},"max":{"1min":11.097,"5min":76.527000000000001,"15min":76.527000000000001},"last":6.1420000000000003}]},{"osd":2,"last update":"Sat Sep 23 21:03:38 2023","interfaces":[{"interface":"back","average":{"1min":2.5169999999999999,"5min":3.7080000000000002,"15min":2.5390000000000001},"min":{"1min":0.29699999999999999,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":11.593999999999999,"5min":28.251999999999999,"15min":28.251999999999999},"last":3.71},{"interface":"front","average":{"1min":3.2949999999999999,"5min":3.4079999999999999,"15min":2.5670000000000002},"min":{"1min":0.36699999999999999,"5min":0.22,"15min":0.22},"max":{"1min":18.321000000000002,"5min":18.321000000000002,"15min":25.305},"last":3.9300000000000002}]},{"osd":4,"last update":"Sat Sep 23 21:03:38 2023","interfaces":[{"interface":"back","average":{"1min":2.2530000000000001,"5min":2.488,"15min":1.76},"min":{"1min":0.28399999999999997,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":11.505000000000001,"5min":18.239000000000001,"15min":18.239000000000001},"last":3.5289999999999999},{"interface":"front","average":{"1min":2.5659999999999998,"5min":2.2440000000000002,"15min":1.7849999999999999},"min":{"1min":0.30299999999999999,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":10.914,"5min":14.808,"15min":22.853000000000002},"last":3.6110000000000002}]},{"osd":5,"last update":"Sat Sep 23 21:03:38 2023","interfaces":[{"interface":"back","average":{"1min":2.2570000000000001,"5min":2.5350000000000001,"15min":1.7749999999999999},"min":{"1min":0.318,"5min":0.318,"15min":0.318},"max":{"1min":11.532999999999999,"5min":18.324999999999999,"15min":18.324999999999999},"last":3.4769999999999999},{"interface":"front","average":{"1min":2.7229999999999999,"5min":3.0640000000000001,"15min":1.885},"min":{"1min":0.34499999999999997,"5min":0.33700000000000002,"15min":0.33100000000000002},"max":{"1min":8.6370000000000005,"5min":13.529999999999999,"15min":13.529999999999999},"last":0.47599999999999998}]},{"osd":6,"last update":"Sat Sep 23 21:03:56 2023","interfaces":[{"interface":"back","average":{"1min":2.2879999999999998,"5min":2.3050000000000002,"15min":1.4830000000000001},"min":{"1min":0.33900000000000002,"5min":0.33700000000000002,"15min":0.33500000000000002},"max":{"1min":10.016,"5min":14.833,"15min":14.833},"last":3.6560000000000001},{"interface":"front","average":{"1min":2.1989999999999998,"5min":2.8879999999999999,"15min":1.9259999999999999},"min":{"1min":0.38400000000000001,"5min":0.373,"15min":0.373},"max":{"1min":8.5570000000000004,"5min":13.576000000000001,"15min":13.576000000000001},"last":0.51300000000000001}]},{"osd":7,"last update":"Sat Sep 23 21:03:38 2023","interfaces":[{"interface":"back","average":{"1min":2.5049999999999999,"5min":2.6230000000000002,"15min":1.9650000000000001},"min":{"1min":0.441,"5min":0.39200000000000002,"15min":0.36099999999999999},"max":{"1min":11.568,"5min":18.289999999999999,"15min":18.289999999999999},"last":3.6760000000000002},{"interface":"front","average":{"1min":2.423,"5min":3.129,"15min":1.9590000000000001},"min":{"1min":0.45400000000000001,"5min":0.38900000000000001,"15min":0.379},"max":{"1min":8.6039999999999992,"5min":13.603999999999999,"15min":13.603999999999999},"last":0.53900000000000003}]}]},{"osd":4,"up_from":31,"seq":133143986479,"num_pgs":51,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1316232,"kb_used_data":1093888,"kb_used_omap":0,"kb_used_meta":222336,"kb_avail":92428920,"statfs":{"total":95995035648,"available":94647214080,"internally_reserved":0,"allocated":1120141312,"data_stored":1119100845,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":227672064},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,1,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:04:00 2023","interfaces":[{"interface":"back","average":{"1min":1.9550000000000001,"5min":1.53,"15min":1.341},"min":{"1min":0.22800000000000001,"5min":0.218,"15min":0.214},"max":{"1min":10.757,"5min":10.757,"15min":11.784000000000001},"last":0.28499999999999998},{"interface":"front","average":{"1min":2.4009999999999998,"5min":1.7390000000000001,"15min":1.504},"min":{"1min":0.27200000000000002,"5min":0.27200000000000002,"15min":0.25800000000000001},"max":{"1min":10.744,"5min":10.744,"15min":11.766},"last":0.33700000000000002}]},{"osd":1,"last update":"Sat Sep 23 21:04:30 2023","interfaces":[{"interface":"back","average":{"1min":2.0070000000000001,"5min":1.8700000000000001,"15min":1.5569999999999999},"min":{"1min":0.309,"5min":0.255,"15min":0.23599999999999999},"max":{"1min":16.210000000000001,"5min":16.210000000000001,"15min":16.210000000000001},"last":0.312},{"interface":"front","average":{"1min":1.2330000000000001,"5min":1.784,"15min":1.5369999999999999},"min":{"1min":0.30099999999999999,"5min":0.29199999999999998,"15min":0.23200000000000001},"max":{"1min":3.2000000000000002,"5min":12.441000000000001,"15min":12.441000000000001},"last":0.374}]},{"osd":2,"last update":"Sat Sep 23 21:04:00 2023","interfaces":[{"interface":"back","average":{"1min":2.532,"5min":2.173,"15min":1.611},"min":{"1min":0.377,"5min":0.309,"15min":0.29099999999999998},"max":{"1min":10.906000000000001,"5min":10.906000000000001,"15min":15.288},"last":0.57299999999999995},{"interface":"front","average":{"1min":2.6179999999999999,"5min":1.9890000000000001,"15min":1.6679999999999999},"min":{"1min":0.307,"5min":0.28599999999999998,"15min":0.28299999999999997},"max":{"1min":8.9329999999999998,"5min":12.855,"15min":15.262},"last":0.42899999999999999}]},{"osd":3,"last update":"Sat Sep 23 21:04:30 2023","interfaces":[{"interface":"back","average":{"1min":1.8460000000000001,"5min":2.5489999999999999,"15min":2.2509999999999999},"min":{"1min":0.30499999999999999,"5min":0.30499999999999999,"15min":0.28899999999999998},"max":{"1min":4.9329999999999998,"5min":12.172000000000001,"15min":31.562999999999999},"last":1.3460000000000001},{"interface":"front","average":{"1min":1.4239999999999999,"5min":2.2789999999999999,"15min":2.1960000000000002},"min":{"1min":0.30199999999999999,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":5.0289999999999999,"5min":12.477,"15min":31.574000000000002},"last":1.359}]},{"osd":5,"last update":"Sat Sep 23 21:03:44 2023","interfaces":[{"interface":"back","average":{"1min":1.1950000000000001,"5min":0.67100000000000004,"15min":0.59899999999999998},"min":{"1min":0.253,"5min":0.251,"15min":0.22700000000000001},"max":{"1min":4.327,"5min":6.6470000000000002,"15min":6.6470000000000002},"last":0.40600000000000003},{"interface":"front","average":{"1min":1.153,"5min":0.91800000000000004,"15min":0.72499999999999998},"min":{"1min":0.28999999999999998,"5min":0.27600000000000002,"15min":0.251},"max":{"1min":6.4029999999999996,"5min":6.4029999999999996,"15min":7.7619999999999996},"last":0.55000000000000004}]},{"osd":6,"last update":"Sat Sep 23 21:03:54 2023","interfaces":[{"interface":"back","average":{"1min":1.9099999999999999,"5min":1.0720000000000001,"15min":0.74399999999999999},"min":{"1min":0.33400000000000002,"5min":0.27300000000000002,"15min":0.26500000000000001},"max":{"1min":7.4139999999999997,"5min":7.4139999999999997,"15min":7.4139999999999997},"last":0.495},{"interface":"front","average":{"1min":0.89400000000000002,"5min":0.752,"15min":0.70399999999999996},"min":{"1min":0.33900000000000002,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":7.1740000000000004,"5min":7.1740000000000004,"15min":7.1740000000000004},"last":0.53100000000000003}]},{"osd":7,"last update":"Sat Sep 23 21:04:30 2023","interfaces":[{"interface":"back","average":{"1min":1.2889999999999999,"5min":0.82999999999999996,"15min":0.77900000000000003},"min":{"1min":0.316,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":14.460000000000001,"5min":14.460000000000001,"15min":14.460000000000001},"last":0.45700000000000002},{"interface":"front","average":{"1min":1.373,"5min":0.95499999999999996,"15min":0.80700000000000005},"min":{"1min":0.32900000000000001,"5min":0.32700000000000001,"15min":0.28199999999999997},"max":{"1min":4.9880000000000004,"5min":6.4459999999999997,"15min":7.8310000000000004},"last":0.46999999999999997}]}]},{"osd":5,"up_from":39,"seq":167503724844,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1741824,"kb_used_data":1498232,"kb_used_omap":0,"kb_used_meta":243520,"kb_avail":92003328,"statfs":{"total":95995035648,"available":94211407872,"internally_reserved":0,"allocated":1534189568,"data_stored":1533153605,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":249364480},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:03:47 2023","interfaces":[{"interface":"back","average":{"1min":1.95,"5min":2.4140000000000001,"15min":1.3660000000000001},"min":{"1min":0.22600000000000001,"5min":0.22500000000000001,"15min":0.20399999999999999},"max":{"1min":10.278,"5min":12.795999999999999,"15min":12.795999999999999},"last":0.253},{"interface":"front","average":{"1min":2.8260000000000001,"5min":2.3439999999999999,"15min":1.5700000000000001},"min":{"1min":0.26700000000000002,"5min":0.23400000000000001,"15min":0.223},"max":{"1min":9.7479999999999993,"5min":16.219999999999999,"15min":16.219999999999999},"last":0.30099999999999999}]},{"osd":1,"last update":"Sat Sep 23 21:04:25 2023","interfaces":[{"interface":"back","average":{"1min":1.5509999999999999,"5min":2.508,"15min":1.7470000000000001},"min":{"1min":0.25800000000000001,"5min":0.25800000000000001,"15min":0.23300000000000001},"max":{"1min":6.4790000000000001,"5min":62.320999999999998,"15min":62.320999999999998},"last":1.6799999999999999},{"interface":"front","average":{"1min":2.1579999999999999,"5min":2.806,"15min":1.851},"min":{"1min":0.26700000000000002,"5min":0.249,"15min":0.22800000000000001},"max":{"1min":7.7510000000000003,"5min":61.722999999999999,"15min":61.722999999999999},"last":0.37}]},{"osd":2,"last update":"Sat Sep 23 21:04:04 2023","interfaces":[{"interface":"back","average":{"1min":1.7270000000000001,"5min":1.9179999999999999,"15min":1.4370000000000001},"min":{"1min":0.28899999999999998,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":6.8079999999999998,"5min":10.371,"15min":12.651},"last":0.96699999999999997},{"interface":"front","average":{"1min":2.4169999999999998,"5min":2.2229999999999999,"15min":1.631},"min":{"1min":0.28899999999999998,"5min":0.28599999999999998,"15min":0.26100000000000001},"max":{"1min":6.9800000000000004,"5min":10.333,"15min":12.300000000000001},"last":0.38800000000000001}]},{"osd":3,"last update":"Sat Sep 23 21:04:25 2023","interfaces":[{"interface":"back","average":{"1min":1.8959999999999999,"5min":3.488,"15min":2.306},"min":{"1min":0.38700000000000001,"5min":0.313,"15min":0.28599999999999998},"max":{"1min":8.3989999999999991,"5min":61.667999999999999,"15min":61.667999999999999},"last":0.89500000000000002},{"interface":"front","average":{"1min":1.109,"5min":3.2050000000000001,"15min":1.9279999999999999},"min":{"1min":0.35699999999999998,"5min":0.32400000000000001,"15min":0.28899999999999998},"max":{"1min":2.7370000000000001,"5min":62.276000000000003,"15min":62.276000000000003},"last":0.57799999999999996}]},{"osd":4,"last update":"Sat Sep 23 21:03:37 2023","interfaces":[{"interface":"back","average":{"1min":1.117,"5min":0.79900000000000004,"15min":0.63900000000000001},"min":{"1min":0.254,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":4.2009999999999996,"5min":4.2009999999999996,"15min":4.2009999999999996},"last":0.33600000000000002},{"interface":"front","average":{"1min":0.73099999999999998,"5min":0.82299999999999995,"15min":0.66100000000000003},"min":{"1min":0.28599999999999998,"5min":0.27400000000000002,"15min":0.23599999999999999},"max":{"1min":4.524,"5min":8.4350000000000005,"15min":8.4350000000000005},"last":0.32600000000000001}]},{"osd":6,"last update":"Sat Sep 23 21:03:47 2023","interfaces":[{"interface":"back","average":{"1min":1.1319999999999999,"5min":1,"15min":0.71699999999999997},"min":{"1min":0.27500000000000002,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":4.1289999999999996,"5min":11.128,"15min":11.128},"last":0.44500000000000001},{"interface":"front","average":{"1min":1.2210000000000001,"5min":1.0649999999999999,"15min":0.83599999999999997},"min":{"1min":0.38100000000000001,"5min":0.36799999999999999,"15min":0.33200000000000002},"max":{"1min":4.8099999999999996,"5min":11.112,"15min":11.112},"last":0.45900000000000002}]},{"osd":7,"last update":"Sat Sep 23 21:04:25 2023","interfaces":[{"interface":"back","average":{"1min":1.123,"5min":0.97399999999999998,"15min":0.78000000000000003},"min":{"1min":0.35899999999999999,"5min":0.309,"15min":0.30199999999999999},"max":{"1min":4.1950000000000003,"5min":4.4240000000000004,"15min":8.5649999999999995},"last":0.35099999999999998},{"interface":"front","average":{"1min":0.84699999999999998,"5min":0.88300000000000001,"15min":0.75900000000000001},"min":{"1min":0.32500000000000001,"5min":0.32500000000000001,"15min":0.27700000000000002},"max":{"1min":2.9180000000000001,"5min":4.6479999999999997,"15min":4.6479999999999997},"last":0.40799999999999997}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1177878528,"data_stored":1177878528,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1318977536,"data_stored":1318977536,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1301155840,"data_stored":1301151768,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1240334336,"data_stored":1240334336,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1091043328,"data_stored":1091043328,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1495207936,"data_stored":1495203864,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1081409536,"data_stored":1081409536,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1205538816,"data_stored":1205534744,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T21:04:40.124 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T21:04:40.209 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:04:40.249 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:04:40.250 INFO:teuthology.orchestra.run.smithi026.stderr:nodeep-scrub is set 2023-09-23T21:04:40.302 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:04:40.308 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:04:40.308 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:04:40.340 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:04:40.409 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:04:40.447 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:04:40.507 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:04:40.509 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:04:40.510 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:04:40.510 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:04:40 smithi070 ceph-mon[168282]: pgmap v826: 119 pgs: 119 active+clean; 2.6 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 87 op/s; 3.1 MiB/s, 49 objects/s recovering 2023-09-23T21:04:40.510 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:04:40 smithi070 ceph-mon[168282]: from='client.45788 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T21:04:40.510 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:04:40 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/1042687476' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:04:40.611 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:04:40.636 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:04:40 smithi026 ceph-mon[175151]: pgmap v826: 119 pgs: 119 active+clean; 2.6 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 87 op/s; 3.1 MiB/s, 49 objects/s recovering 2023-09-23T21:04:40.637 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:04:40 smithi026 ceph-mon[175151]: from='client.45788 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T21:04:40.637 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:04:40 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/1042687476' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:04:40.637 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:04:40 smithi026 ceph-mon[179688]: pgmap v826: 119 pgs: 119 active+clean; 2.6 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 87 op/s; 3.1 MiB/s, 49 objects/s recovering 2023-09-23T21:04:40.637 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:04:40 smithi026 ceph-mon[179688]: from='client.45788 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T21:04:40.637 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:04:40 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/1042687476' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:04:40.684 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:04:40.823 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:04:40.830 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:04:40.830 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:04:40.931 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:04:41.016 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:04:41.042 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:04:41.079 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:04:41.089 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:04:41.089 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:04:41.190 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:04:41.222 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:04:41 smithi026 ceph-mon[175151]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:04:41.222 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:04:41 smithi026 ceph-mon[175151]: failed to encode map e322 with expected crc 2023-09-23T21:04:41.222 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:04:41 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/1042687476' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T21:04:41.222 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:04:41 smithi026 ceph-mon[175151]: osdmap e322: 8 total, 8 up, 8 in 2023-09-23T21:04:41.222 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:04:41 smithi026 ceph-mon[175151]: failed to encode map e322 with expected crc 2023-09-23T21:04:41.223 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:04:41 smithi026 ceph-mon[175151]: failed to encode map e322 with expected crc 2023-09-23T21:04:41.238 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:04:41.306 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:04:41.313 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:04:41.313 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:04:41.414 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:04:41.434 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:04:41.474 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:04:41 smithi026 ceph-mon[179688]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:04:41.475 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:04:41 smithi026 ceph-mon[179688]: failed to encode map e322 with expected crc 2023-09-23T21:04:41.475 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:04:41 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/1042687476' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T21:04:41.475 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:04:41 smithi026 ceph-mon[179688]: osdmap e322: 8 total, 8 up, 8 in 2023-09-23T21:04:41.475 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:04:41 smithi026 ceph-mon[179688]: failed to encode map e322 with expected crc 2023-09-23T21:04:41.475 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:04:41 smithi026 ceph-mon[179688]: failed to encode map e322 with expected crc 2023-09-23T21:04:41.507 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:04:41.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:04:41 smithi070 ceph-mon[168282]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:04:41.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:04:41 smithi070 ceph-mon[168282]: failed to encode map e322 with expected crc 2023-09-23T21:04:41.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:04:41 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/1042687476' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T21:04:41.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:04:41 smithi070 ceph-mon[168282]: osdmap e322: 8 total, 8 up, 8 in 2023-09-23T21:04:41.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:04:41 smithi070 ceph-mon[168282]: failed to encode map e322 with expected crc 2023-09-23T21:04:41.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:04:41 smithi070 ceph-mon[168282]: failed to encode map e322 with expected crc 2023-09-23T21:04:41.832 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:04:41.837 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:04:41.837 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:04:41.938 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:04:42.028 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:04:42.132 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T21:04:42.139 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:04:42.140 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:04:42.240 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:04:42.302 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:04:42.340 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:04:42 smithi026 ceph-mon[175151]: pgmap v828: 119 pgs: 119 active+clean; 2.5 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 152 op/s 2023-09-23T21:04:42.342 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:04:42 smithi026 ceph-mon[179688]: pgmap v828: 119 pgs: 119 active+clean; 2.5 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 152 op/s 2023-09-23T21:04:42.355 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:04:42.359 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:04:42.359 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:04:42.460 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:04:42.553 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:04:42.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:04:42 smithi070 ceph-mon[168282]: pgmap v828: 119 pgs: 119 active+clean; 2.5 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 152 op/s 2023-09-23T21:04:42.621 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:04:42.629 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:04:42.629 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:04:42.707 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T21:04:42.730 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:04:42.815 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:04:43.071 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:04:43.074 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:04:43.075 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:04:43.101 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0: {} 2023-09-23T21:04:43.138 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1: {} 2023-09-23T21:04:43.176 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:04:43.182 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2: {} 2023-09-23T21:04:43.222 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3: {} 2023-09-23T21:04:43.259 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4: {} 2023-09-23T21:04:43.278 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:04:43.289 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T21:04:43.331 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6: {} 2023-09-23T21:04:43.404 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T21:04:43.405 INFO:teuthology.orchestra.run.smithi026.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T21:04:43.405 INFO:teuthology.orchestra.run.smithi026.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T21:04:43.405 INFO:teuthology.orchestra.run.smithi026.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T21:04:43.405 INFO:teuthology.orchestra.run.smithi026.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T21:04:43.405 INFO:teuthology.orchestra.run.smithi026.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T21:04:43.405 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T21:04:43.405 INFO:teuthology.orchestra.run.smithi026.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T21:04:43.405 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T21:04:43.617 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:04:43.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:04:43 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/863680334' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:04:43.625 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:04:43.626 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:04:43.627 INFO:teuthology.orchestra.run.smithi026.stdout:{ 2023-09-23T21:04:43.627 INFO:teuthology.orchestra.run.smithi026.stdout: "size": 20, 2023-09-23T21:04:43.627 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 600, 2023-09-23T21:04:43.627 INFO:teuthology.orchestra.run.smithi026.stdout: "ops": [ 2023-09-23T21:04:43.628 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.628 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34797.0:177915 6.11 6:8818350e:::benchmark_data_smithi026_223868_object76789:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e164)", 2023-09-23T21:04:43.628 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T20:55:34.933656+0000", 2023-09-23T21:04:43.628 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 548.68161045199997, 2023-09-23T21:04:43.628 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.0860430320000001, 2023-09-23T21:04:43.628 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:04:43.628 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:04:43.628 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T21:04:43.628 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34797", 2023-09-23T21:04:43.628 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T21:04:43.628 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 177915 2023-09-23T21:04:43.628 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.629 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:04:43.629 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.629 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:04:43.629 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:34.933656+0000", 2023-09-23T21:04:43.629 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:04:43.629 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.629 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.629 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:04:43.629 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:34.933656+0000", 2023-09-23T21:04:43.629 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:04:43.629 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.629 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.629 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:04:43.630 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:34.933654+0000", 2023-09-23T21:04:43.630 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999981 2023-09-23T21:04:43.630 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.630 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.630 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:04:43.630 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:34.933665+0000", 2023-09-23T21:04:43.630 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0963e-05 2023-09-23T21:04:43.630 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.630 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.630 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:04:43.630 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:34.933690+0000", 2023-09-23T21:04:43.630 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.5219999999999999e-05 2023-09-23T21:04:43.631 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.631 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.631 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:04:43.631 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:34.933701+0000", 2023-09-23T21:04:43.631 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0944000000000001e-05 2023-09-23T21:04:43.631 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.631 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.631 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:04:43.631 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:34.933803+0000", 2023-09-23T21:04:43.631 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000101654 2023-09-23T21:04:43.631 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.631 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.632 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op must wait for map", 2023-09-23T21:04:43.632 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:34.933811+0000", 2023-09-23T21:04:43.632 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.2190000000000004e-06 2023-09-23T21:04:43.632 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.632 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.632 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:04:43.632 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:34.946283+0000", 2023-09-23T21:04:43.632 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.012472298 2023-09-23T21:04:43.632 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.632 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.632 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T21:04:43.632 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:34.946294+0000", 2023-09-23T21:04:43.633 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0832999999999999e-05 2023-09-23T21:04:43.633 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.633 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.633 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:04:43.633 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:35.931861+0000", 2023-09-23T21:04:43.633 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.98556678499999995 2023-09-23T21:04:43.633 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.633 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.633 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T21:04:43.633 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:35.931870+0000", 2023-09-23T21:04:43.633 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.5799999999999992e-06 2023-09-23T21:04:43.633 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.634 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.634 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:04:43.634 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:37.013529+0000", 2023-09-23T21:04:43.634 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0816588920000001 2023-09-23T21:04:43.634 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.634 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.634 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:04:43.634 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:37.013686+0000", 2023-09-23T21:04:43.634 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000157825 2023-09-23T21:04:43.634 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.634 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.634 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 2,6", 2023-09-23T21:04:43.634 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:37.014148+0000", 2023-09-23T21:04:43.635 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00046177499999999999 2023-09-23T21:04:43.635 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.635 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.635 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T21:04:43.635 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:37.019388+0000", 2023-09-23T21:04:43.635 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0052396250000000004 2023-09-23T21:04:43.635 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.635 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.635 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:04:43.635 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:37.019453+0000", 2023-09-23T21:04:43.635 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.5685000000000006e-05 2023-09-23T21:04:43.635 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.636 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.636 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:04:43.636 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:37.019503+0000", 2023-09-23T21:04:43.636 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.9861999999999998e-05 2023-09-23T21:04:43.636 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.636 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.636 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T21:04:43.636 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:37.019559+0000", 2023-09-23T21:04:43.636 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.5605e-05 2023-09-23T21:04:43.636 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.636 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.636 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T21:04:43.636 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:37.019699+0000", 2023-09-23T21:04:43.637 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00013997099999999999 2023-09-23T21:04:43.637 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:04:43.637 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:04:43.637 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:04:43.637 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.637 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.637 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34797.0:177930 6.1 6:872a742b:::benchmark_data_smithi026_223868_object76804:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e164)", 2023-09-23T21:04:43.637 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T20:55:34.965696+0000", 2023-09-23T21:04:43.637 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 548.64957027299999, 2023-09-23T21:04:43.637 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.0561283210000001, 2023-09-23T21:04:43.637 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:04:43.637 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:04:43.638 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T21:04:43.638 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34797", 2023-09-23T21:04:43.638 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T21:04:43.638 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 177930 2023-09-23T21:04:43.638 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.638 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:04:43.638 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.638 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:04:43.638 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:34.965696+0000", 2023-09-23T21:04:43.638 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:04:43.638 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.638 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.638 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:04:43.639 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:34.965696+0000", 2023-09-23T21:04:43.639 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:04:43.639 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.639 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.639 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:04:43.639 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:34.965694+0000", 2023-09-23T21:04:43.639 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999981 2023-09-23T21:04:43.639 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.639 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.639 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:04:43.639 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:34.965702+0000", 2023-09-23T21:04:43.639 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.1000000000000004e-06 2023-09-23T21:04:43.639 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.640 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.640 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:04:43.640 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:34.965716+0000", 2023-09-23T21:04:43.640 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.3190000000000001e-05 2023-09-23T21:04:43.640 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.640 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.640 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:04:43.640 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:34.965724+0000", 2023-09-23T21:04:43.640 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.5699999999999993e-06 2023-09-23T21:04:43.640 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.640 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.640 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:04:43.640 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:34.965780+0000", 2023-09-23T21:04:43.641 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.5807999999999998e-05 2023-09-23T21:04:43.641 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.641 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.641 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T21:04:43.641 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:34.965791+0000", 2023-09-23T21:04:43.641 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0597e-05 2023-09-23T21:04:43.641 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.641 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.641 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:04:43.641 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:35.931819+0000", 2023-09-23T21:04:43.641 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.96602831899999997 2023-09-23T21:04:43.641 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.641 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.642 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T21:04:43.642 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:35.931833+0000", 2023-09-23T21:04:43.642 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.4409999999999999e-05 2023-09-23T21:04:43.642 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.642 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.642 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:04:43.642 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:37.016222+0000", 2023-09-23T21:04:43.642 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.084388186 2023-09-23T21:04:43.642 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.642 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.642 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:04:43.642 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:37.016369+0000", 2023-09-23T21:04:43.643 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00014728699999999999 2023-09-23T21:04:43.643 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.643 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.643 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 2,6", 2023-09-23T21:04:43.643 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:37.016878+0000", 2023-09-23T21:04:43.643 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000509068 2023-09-23T21:04:43.643 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.643 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.643 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T21:04:43.643 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:37.020205+0000", 2023-09-23T21:04:43.643 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0033272660000000002 2023-09-23T21:04:43.643 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.643 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.643 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:04:43.644 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:37.020577+0000", 2023-09-23T21:04:43.644 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00037149500000000001 2023-09-23T21:04:43.644 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.644 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.644 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:04:43.644 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:37.021628+0000", 2023-09-23T21:04:43.644 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0010512239999999999 2023-09-23T21:04:43.644 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.644 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.644 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T21:04:43.644 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:37.021685+0000", 2023-09-23T21:04:43.644 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.6715999999999997e-05 2023-09-23T21:04:43.644 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.645 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.645 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T21:04:43.645 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T20:55:37.021824+0000", 2023-09-23T21:04:43.645 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00013976799999999999 2023-09-23T21:04:43.645 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:04:43.645 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:04:43.645 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:04:43.645 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.645 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.645 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35469.0:164163 8.17 8:efa7e807:::benchmark_data_smithi026_301269_object53406:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e308)", 2023-09-23T21:04:43.645 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T21:04:14.221577+0000", 2023-09-23T21:04:43.645 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 29.393688872999999, 2023-09-23T21:04:43.645 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.5740775810000001, 2023-09-23T21:04:43.646 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:04:43.646 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:04:43.646 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T21:04:43.646 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35469", 2023-09-23T21:04:43.646 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/4041424623", 2023-09-23T21:04:43.646 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 164163 2023-09-23T21:04:43.646 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.646 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:04:43.646 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.646 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:04:43.646 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:14.221577+0000", 2023-09-23T21:04:43.646 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:04:43.646 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.647 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.647 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:04:43.647 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:14.221577+0000", 2023-09-23T21:04:43.647 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:04:43.647 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.647 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.647 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:04:43.647 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:14.221575+0000", 2023-09-23T21:04:43.647 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999976 2023-09-23T21:04:43.647 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.647 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.647 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:04:43.647 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:14.221583+0000", 2023-09-23T21:04:43.647 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.5780000000000001e-06 2023-09-23T21:04:43.648 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.648 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.648 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:04:43.648 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:14.221595+0000", 2023-09-23T21:04:43.648 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.2865e-05 2023-09-23T21:04:43.648 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.648 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.648 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:04:43.648 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:14.221604+0000", 2023-09-23T21:04:43.648 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.439e-06 2023-09-23T21:04:43.648 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.648 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.648 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:04:43.649 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:14.221765+0000", 2023-09-23T21:04:43.649 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000160679 2023-09-23T21:04:43.649 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.649 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.649 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op must wait for map", 2023-09-23T21:04:43.649 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:14.221772+0000", 2023-09-23T21:04:43.649 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.7500000000000003e-06 2023-09-23T21:04:43.649 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.649 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.649 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:04:43.649 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:14.263184+0000", 2023-09-23T21:04:43.649 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.041411824999999999 2023-09-23T21:04:43.649 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.650 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.650 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T21:04:43.650 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:14.263194+0000", 2023-09-23T21:04:43.650 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0403e-05 2023-09-23T21:04:43.650 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.650 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.650 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:04:43.650 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:15.760683+0000", 2023-09-23T21:04:43.650 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.497488309 2023-09-23T21:04:43.650 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.650 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.650 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:04:43.650 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:15.760808+0000", 2023-09-23T21:04:43.651 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000124719 2023-09-23T21:04:43.651 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.651 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.651 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 0,3,6,7", 2023-09-23T21:04:43.651 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:15.761152+0000", 2023-09-23T21:04:43.651 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00034497800000000002 2023-09-23T21:04:43.651 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.651 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.651 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T21:04:43.651 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:15.765520+0000", 2023-09-23T21:04:43.651 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0043679050000000001 2023-09-23T21:04:43.651 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.651 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.651 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:04:43.652 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:15.765564+0000", 2023-09-23T21:04:43.652 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.3409999999999999e-05 2023-09-23T21:04:43.652 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.652 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.652 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:04:43.652 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:15.765737+0000", 2023-09-23T21:04:43.652 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00017347799999999999 2023-09-23T21:04:43.652 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.652 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.652 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:04:43.652 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:15.765798+0000", 2023-09-23T21:04:43.652 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.0900000000000003e-05 2023-09-23T21:04:43.652 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.653 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.653 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:04:43.653 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:15.795432+0000", 2023-09-23T21:04:43.653 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.029633342 2023-09-23T21:04:43.653 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.653 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.653 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T21:04:43.653 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:15.795508+0000", 2023-09-23T21:04:43.653 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.6039999999999997e-05 2023-09-23T21:04:43.653 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.653 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.653 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T21:04:43.653 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:15.795655+0000", 2023-09-23T21:04:43.654 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00014744399999999999 2023-09-23T21:04:43.654 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:04:43.654 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:04:43.654 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:04:43.654 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.654 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.654 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35469.0:164217 8.17 8:e88160f7:::benchmark_data_smithi026_301269_object53460:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e308)", 2023-09-23T21:04:43.654 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T21:04:14.221619+0000", 2023-09-23T21:04:43.654 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 29.393647729000001, 2023-09-23T21:04:43.654 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.575365753, 2023-09-23T21:04:43.654 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:04:43.654 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:04:43.654 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T21:04:43.655 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35469", 2023-09-23T21:04:43.655 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/4041424623", 2023-09-23T21:04:43.655 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 164217 2023-09-23T21:04:43.655 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.655 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:04:43.655 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.655 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:04:43.655 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:14.221619+0000", 2023-09-23T21:04:43.655 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:04:43.655 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.655 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.655 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:04:43.655 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:14.221619+0000", 2023-09-23T21:04:43.655 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:04:43.656 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.656 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.656 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:04:43.656 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:14.221618+0000", 2023-09-23T21:04:43.656 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T21:04:43.656 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.656 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.656 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:04:43.656 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:14.221621+0000", 2023-09-23T21:04:43.656 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.5019999999999999e-06 2023-09-23T21:04:43.656 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.656 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.656 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:04:43.657 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:14.221626+0000", 2023-09-23T21:04:43.657 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.101e-06 2023-09-23T21:04:43.657 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.657 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.657 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:04:43.657 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:14.221629+0000", 2023-09-23T21:04:43.657 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.6610000000000002e-06 2023-09-23T21:04:43.657 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.657 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.657 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:04:43.657 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:14.221839+0000", 2023-09-23T21:04:43.657 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00020961100000000001 2023-09-23T21:04:43.657 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.658 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.658 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting_for_map not empty", 2023-09-23T21:04:43.658 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:14.221845+0000", 2023-09-23T21:04:43.658 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.054e-06 2023-09-23T21:04:43.658 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.658 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.658 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:04:43.658 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:14.263361+0000", 2023-09-23T21:04:43.658 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.041515735999999998 2023-09-23T21:04:43.658 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.658 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.658 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T21:04:43.659 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:14.263370+0000", 2023-09-23T21:04:43.659 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.8089999999999999e-06 2023-09-23T21:04:43.659 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.659 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.659 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:04:43.659 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:15.762660+0000", 2023-09-23T21:04:43.659 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.4992907529999999 2023-09-23T21:04:43.659 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.659 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.659 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:04:43.659 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:15.762745+0000", 2023-09-23T21:04:43.659 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.4602000000000001e-05 2023-09-23T21:04:43.659 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.659 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.660 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 0,3,6,7", 2023-09-23T21:04:43.660 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:15.762900+0000", 2023-09-23T21:04:43.660 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00015529899999999999 2023-09-23T21:04:43.660 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.660 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.660 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T21:04:43.660 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:15.765701+0000", 2023-09-23T21:04:43.660 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.002801118 2023-09-23T21:04:43.660 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.660 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.660 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:04:43.660 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:15.765767+0000", 2023-09-23T21:04:43.660 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.5473000000000001e-05 2023-09-23T21:04:43.661 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.661 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.661 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:04:43.661 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:15.765824+0000", 2023-09-23T21:04:43.661 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.6805000000000002e-05 2023-09-23T21:04:43.661 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.661 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.661 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:04:43.661 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:15.768812+0000", 2023-09-23T21:04:43.661 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0029878529999999999 2023-09-23T21:04:43.661 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.661 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.661 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:04:43.662 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:15.796800+0000", 2023-09-23T21:04:43.662 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.027988783999999999 2023-09-23T21:04:43.662 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.662 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.662 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T21:04:43.662 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:15.796855+0000", 2023-09-23T21:04:43.662 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.4697000000000002e-05 2023-09-23T21:04:43.662 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.662 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.662 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T21:04:43.662 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:15.796984+0000", 2023-09-23T21:04:43.662 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000129293 2023-09-23T21:04:43.662 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:04:43.663 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:04:43.663 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:04:43.663 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.663 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.663 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35469.0:164269 8.17 8:e942d974:::benchmark_data_smithi026_301269_object53512:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e308)", 2023-09-23T21:04:43.663 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T21:04:14.221634+0000", 2023-09-23T21:04:43.663 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 29.393632616000001, 2023-09-23T21:04:43.663 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.575577794, 2023-09-23T21:04:43.663 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:04:43.663 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:04:43.663 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T21:04:43.663 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35469", 2023-09-23T21:04:43.663 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/4041424623", 2023-09-23T21:04:43.664 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 164269 2023-09-23T21:04:43.664 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.664 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:04:43.664 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.664 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:04:43.664 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:14.221634+0000", 2023-09-23T21:04:43.664 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:04:43.664 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.664 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.664 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:04:43.664 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:14.221634+0000", 2023-09-23T21:04:43.664 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:04:43.665 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.665 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.665 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:04:43.665 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:14.221633+0000", 2023-09-23T21:04:43.665 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T21:04:43.665 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.665 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.665 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:04:43.665 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:14.221636+0000", 2023-09-23T21:04:43.665 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.1600000000000001e-06 2023-09-23T21:04:43.665 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.665 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.666 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:04:43.666 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:14.221641+0000", 2023-09-23T21:04:43.666 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.3290000000000001e-06 2023-09-23T21:04:43.666 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.666 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.666 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:04:43.666 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:14.221645+0000", 2023-09-23T21:04:43.666 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.5469999999999998e-06 2023-09-23T21:04:43.666 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.666 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.666 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:04:43.666 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:14.221869+0000", 2023-09-23T21:04:43.667 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000223694 2023-09-23T21:04:43.667 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.667 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.667 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting_for_map not empty", 2023-09-23T21:04:43.667 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:14.221887+0000", 2023-09-23T21:04:43.667 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.7884999999999999e-05 2023-09-23T21:04:43.667 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.667 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.667 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:04:43.667 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:14.263397+0000", 2023-09-23T21:04:43.667 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.041510175000000003 2023-09-23T21:04:43.667 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.668 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.668 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T21:04:43.668 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:14.263404+0000", 2023-09-23T21:04:43.668 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.8029999999999999e-06 2023-09-23T21:04:43.668 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.668 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.668 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:04:43.668 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:15.763962+0000", 2023-09-23T21:04:43.668 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.5005579390000001 2023-09-23T21:04:43.668 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.668 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.668 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:04:43.668 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:15.764062+0000", 2023-09-23T21:04:43.669 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.9908000000000006e-05 2023-09-23T21:04:43.669 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.669 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.669 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 0,3,6,7", 2023-09-23T21:04:43.669 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:15.764261+0000", 2023-09-23T21:04:43.669 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00019896900000000001 2023-09-23T21:04:43.669 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.669 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.669 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T21:04:43.669 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:15.765773+0000", 2023-09-23T21:04:43.669 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.001512546 2023-09-23T21:04:43.669 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.670 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.670 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:04:43.670 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:15.765850+0000", 2023-09-23T21:04:43.670 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.6221000000000001e-05 2023-09-23T21:04:43.670 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.670 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.670 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:04:43.670 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:15.765889+0000", 2023-09-23T21:04:43.670 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.9903000000000001e-05 2023-09-23T21:04:43.670 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.670 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.670 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:04:43.670 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:15.768852+0000", 2023-09-23T21:04:43.671 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0029622120000000001 2023-09-23T21:04:43.671 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.671 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.671 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:04:43.671 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:15.797039+0000", 2023-09-23T21:04:43.671 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.028187303 2023-09-23T21:04:43.671 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.671 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.671 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T21:04:43.671 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:15.797090+0000", 2023-09-23T21:04:43.671 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.0711000000000003e-05 2023-09-23T21:04:43.671 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.671 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.671 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T21:04:43.672 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:15.797211+0000", 2023-09-23T21:04:43.672 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000121789 2023-09-23T21:04:43.672 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:04:43.672 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:04:43.672 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:04:43.672 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.672 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.672 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35469.0:164297 8.17 8:e83f2c7f:::benchmark_data_smithi026_301269_object53540:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e308)", 2023-09-23T21:04:43.672 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T21:04:14.221649+0000", 2023-09-23T21:04:43.672 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 29.393617160000002, 2023-09-23T21:04:43.672 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.5760916090000001, 2023-09-23T21:04:43.672 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:04:43.673 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:04:43.673 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T21:04:43.673 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35469", 2023-09-23T21:04:43.673 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/4041424623", 2023-09-23T21:04:43.673 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 164297 2023-09-23T21:04:43.673 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.673 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:04:43.673 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.673 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:04:43.673 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:14.221649+0000", 2023-09-23T21:04:43.673 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:04:43.673 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.673 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.674 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:04:43.674 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:14.221649+0000", 2023-09-23T21:04:43.674 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:04:43.674 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.674 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.674 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:04:43.674 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:14.221649+0000", 2023-09-23T21:04:43.674 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T21:04:43.674 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.674 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.674 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:04:43.675 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:14.221651+0000", 2023-09-23T21:04:43.675 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.1600000000000001e-06 2023-09-23T21:04:43.675 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.675 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.675 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:04:43.675 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:14.221656+0000", 2023-09-23T21:04:43.675 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.4029999999999998e-06 2023-09-23T21:04:43.675 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.675 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.675 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:04:43.675 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:14.221660+0000", 2023-09-23T21:04:43.675 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.7840000000000001e-06 2023-09-23T21:04:43.675 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.676 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.676 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:04:43.676 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:14.221912+0000", 2023-09-23T21:04:43.676 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00025214299999999997 2023-09-23T21:04:43.676 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.676 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.676 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting_for_map not empty", 2023-09-23T21:04:43.676 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:14.221918+0000", 2023-09-23T21:04:43.676 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.1959999999999999e-06 2023-09-23T21:04:43.676 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.676 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.676 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:04:43.676 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:14.263429+0000", 2023-09-23T21:04:43.677 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.041511215999999997 2023-09-23T21:04:43.677 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.677 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.677 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T21:04:43.677 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:14.263437+0000", 2023-09-23T21:04:43.677 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.0730000000000005e-06 2023-09-23T21:04:43.677 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.677 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.677 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:04:43.677 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:15.765939+0000", 2023-09-23T21:04:43.677 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.5025021779999999 2023-09-23T21:04:43.677 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.678 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.678 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:04:43.678 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:15.766073+0000", 2023-09-23T21:04:43.678 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00013392699999999999 2023-09-23T21:04:43.678 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.678 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.678 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 0,3,6,7", 2023-09-23T21:04:43.678 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:15.766277+0000", 2023-09-23T21:04:43.678 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00020453600000000001 2023-09-23T21:04:43.678 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.678 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.678 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T21:04:43.678 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:15.768861+0000", 2023-09-23T21:04:43.679 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0025837970000000001 2023-09-23T21:04:43.679 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.679 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.679 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:04:43.679 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:15.768913+0000", 2023-09-23T21:04:43.679 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.1884e-05 2023-09-23T21:04:43.679 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.679 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.679 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:04:43.679 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:15.768944+0000", 2023-09-23T21:04:43.679 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.1185e-05 2023-09-23T21:04:43.679 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.679 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.679 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:04:43.680 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:15.769013+0000", 2023-09-23T21:04:43.680 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.9102000000000003e-05 2023-09-23T21:04:43.680 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.680 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.680 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:04:43.680 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:15.797548+0000", 2023-09-23T21:04:43.680 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.02853466 2023-09-23T21:04:43.680 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.680 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.680 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T21:04:43.680 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:15.797615+0000", 2023-09-23T21:04:43.680 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.7045999999999995e-05 2023-09-23T21:04:43.681 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.681 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.681 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T21:04:43.681 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:15.797741+0000", 2023-09-23T21:04:43.681 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00012561199999999999 2023-09-23T21:04:43.681 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:04:43.681 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:04:43.681 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:04:43.681 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.681 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.681 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35469.0:164301 8.17 8:ef1e8c10:::benchmark_data_smithi026_301269_object53544:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e308)", 2023-09-23T21:04:43.681 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T21:04:14.221664+0000", 2023-09-23T21:04:43.681 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 29.393602139999999, 2023-09-23T21:04:43.682 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.5783125409999998, 2023-09-23T21:04:43.682 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:04:43.682 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:04:43.682 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T21:04:43.682 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35469", 2023-09-23T21:04:43.682 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/4041424623", 2023-09-23T21:04:43.682 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 164301 2023-09-23T21:04:43.682 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.682 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:04:43.682 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.682 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:04:43.682 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:14.221664+0000", 2023-09-23T21:04:43.682 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:04:43.683 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.683 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.683 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:04:43.683 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:14.221664+0000", 2023-09-23T21:04:43.683 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:04:43.683 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.683 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.683 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:04:43.683 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:14.221664+0000", 2023-09-23T21:04:43.683 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T21:04:43.683 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.683 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.683 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:04:43.683 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:14.221666+0000", 2023-09-23T21:04:43.684 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.165e-06 2023-09-23T21:04:43.684 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.684 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.684 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:04:43.684 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:14.221672+0000", 2023-09-23T21:04:43.684 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.536e-06 2023-09-23T21:04:43.684 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.684 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.684 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:04:43.684 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:14.221677+0000", 2023-09-23T21:04:43.684 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.0980000000000001e-06 2023-09-23T21:04:43.684 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.684 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.685 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:04:43.685 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:14.221939+0000", 2023-09-23T21:04:43.685 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00026276700000000001 2023-09-23T21:04:43.685 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.685 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.685 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting_for_map not empty", 2023-09-23T21:04:43.685 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:14.221944+0000", 2023-09-23T21:04:43.685 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.583e-06 2023-09-23T21:04:43.685 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.685 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.685 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:04:43.685 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:14.263465+0000", 2023-09-23T21:04:43.685 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.041520872 2023-09-23T21:04:43.686 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.686 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.686 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T21:04:43.686 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:14.263491+0000", 2023-09-23T21:04:43.686 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.6435999999999999e-05 2023-09-23T21:04:43.686 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.686 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.686 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:04:43.686 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:15.767232+0000", 2023-09-23T21:04:43.686 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.50374085 2023-09-23T21:04:43.686 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.686 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.686 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:04:43.687 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:15.767389+0000", 2023-09-23T21:04:43.687 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00015733000000000001 2023-09-23T21:04:43.687 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.687 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.687 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 0,3,6,7", 2023-09-23T21:04:43.687 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:15.767649+0000", 2023-09-23T21:04:43.687 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00025981000000000002 2023-09-23T21:04:43.687 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.687 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.687 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T21:04:43.687 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:15.768952+0000", 2023-09-23T21:04:43.687 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.001302933 2023-09-23T21:04:43.687 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.688 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.688 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:04:43.688 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:15.770484+0000", 2023-09-23T21:04:43.688 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0015313849999999999 2023-09-23T21:04:43.688 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.688 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.688 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:04:43.688 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:15.770530+0000", 2023-09-23T21:04:43.688 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.6069999999999998e-05 2023-09-23T21:04:43.688 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.688 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.688 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:04:43.688 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:15.770569+0000", 2023-09-23T21:04:43.688 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.9425999999999998e-05 2023-09-23T21:04:43.689 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.689 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.689 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:04:43.689 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:15.799794+0000", 2023-09-23T21:04:43.689 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.029224792999999999 2023-09-23T21:04:43.689 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.689 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.689 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T21:04:43.689 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:15.799849+0000", 2023-09-23T21:04:43.689 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.5349000000000001e-05 2023-09-23T21:04:43.689 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.689 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.689 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T21:04:43.690 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:15.799977+0000", 2023-09-23T21:04:43.690 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00012746599999999999 2023-09-23T21:04:43.690 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:04:43.690 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:04:43.690 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:04:43.690 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.690 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.690 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35469.0:164305 8.17 8:ee90ce4e:::benchmark_data_smithi026_301269_object53548:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e308)", 2023-09-23T21:04:43.690 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T21:04:14.221681+0000", 2023-09-23T21:04:43.690 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 29.393585569999999, 2023-09-23T21:04:43.690 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.581583009, 2023-09-23T21:04:43.690 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:04:43.691 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:04:43.691 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T21:04:43.691 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35469", 2023-09-23T21:04:43.691 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/4041424623", 2023-09-23T21:04:43.691 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 164305 2023-09-23T21:04:43.691 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.691 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:04:43.691 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.691 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:04:43.691 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:14.221681+0000", 2023-09-23T21:04:43.691 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:04:43.691 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.691 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.692 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:04:43.692 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:14.221681+0000", 2023-09-23T21:04:43.692 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:04:43.692 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.692 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.692 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:04:43.692 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:14.221680+0000", 2023-09-23T21:04:43.692 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T21:04:43.692 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.692 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.692 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:04:43.692 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:14.221683+0000", 2023-09-23T21:04:43.693 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.74e-06 2023-09-23T21:04:43.693 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.693 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.693 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:04:43.693 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:14.221689+0000", 2023-09-23T21:04:43.693 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.2129999999999996e-06 2023-09-23T21:04:43.693 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.693 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.693 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:04:43.693 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:14.221694+0000", 2023-09-23T21:04:43.693 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.9509999999999997e-06 2023-09-23T21:04:43.693 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.693 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.694 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:04:43.694 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:14.221959+0000", 2023-09-23T21:04:43.694 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000264749 2023-09-23T21:04:43.694 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.694 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.694 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting_for_map not empty", 2023-09-23T21:04:43.694 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:14.221962+0000", 2023-09-23T21:04:43.694 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.5770000000000001e-06 2023-09-23T21:04:43.694 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.694 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.694 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:04:43.694 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:14.263523+0000", 2023-09-23T21:04:43.695 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.041561206000000003 2023-09-23T21:04:43.695 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.695 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.695 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T21:04:43.695 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:14.263530+0000", 2023-09-23T21:04:43.695 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.4739999999999999e-06 2023-09-23T21:04:43.695 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.695 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.695 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:04:43.695 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:15.769054+0000", 2023-09-23T21:04:43.695 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.505523975 2023-09-23T21:04:43.695 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.695 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.696 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:04:43.696 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:15.769141+0000", 2023-09-23T21:04:43.696 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.6434999999999995e-05 2023-09-23T21:04:43.696 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.696 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.696 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 0,3,6,7", 2023-09-23T21:04:43.696 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:15.769368+0000", 2023-09-23T21:04:43.696 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00022779699999999999 2023-09-23T21:04:43.696 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.696 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.696 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T21:04:43.696 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:15.772030+0000", 2023-09-23T21:04:43.697 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.002661189 2023-09-23T21:04:43.697 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.697 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.697 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:04:43.697 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:15.773529+0000", 2023-09-23T21:04:43.697 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0014994209999999999 2023-09-23T21:04:43.697 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.697 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.697 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:04:43.697 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:15.773560+0000", 2023-09-23T21:04:43.697 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.0886999999999998e-05 2023-09-23T21:04:43.697 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.697 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.698 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:04:43.698 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:15.773609+0000", 2023-09-23T21:04:43.698 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.9472000000000003e-05 2023-09-23T21:04:43.698 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.698 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.698 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:04:43.698 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:15.803015+0000", 2023-09-23T21:04:43.698 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.029405524999999998 2023-09-23T21:04:43.698 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.698 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.698 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T21:04:43.699 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:15.803080+0000", 2023-09-23T21:04:43.699 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.5246000000000005e-05 2023-09-23T21:04:43.699 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.699 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.699 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T21:04:43.699 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:15.803264+0000", 2023-09-23T21:04:43.699 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000183516 2023-09-23T21:04:43.699 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:04:43.699 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:04:43.699 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:04:43.699 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.699 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.699 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35469.0:164306 8.17 8:e99d09cb:::benchmark_data_smithi026_301269_object53549:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e308)", 2023-09-23T21:04:43.699 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T21:04:14.221699+0000", 2023-09-23T21:04:43.700 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 29.393567610000002, 2023-09-23T21:04:43.700 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.581815132, 2023-09-23T21:04:43.700 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:04:43.700 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:04:43.700 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T21:04:43.700 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35469", 2023-09-23T21:04:43.700 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/4041424623", 2023-09-23T21:04:43.700 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 164306 2023-09-23T21:04:43.700 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.700 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:04:43.700 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.700 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:04:43.701 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:14.221699+0000", 2023-09-23T21:04:43.701 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:04:43.701 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.701 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.701 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:04:43.701 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:14.221699+0000", 2023-09-23T21:04:43.701 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:04:43.701 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.701 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.701 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:04:43.701 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:14.221698+0000", 2023-09-23T21:04:43.701 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T21:04:43.702 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.702 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.702 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:04:43.702 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:14.221705+0000", 2023-09-23T21:04:43.702 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.7390000000000002e-06 2023-09-23T21:04:43.702 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.702 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.702 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:04:43.702 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:14.221711+0000", 2023-09-23T21:04:43.702 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.392e-06 2023-09-23T21:04:43.702 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.702 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.703 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:04:43.703 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:14.221717+0000", 2023-09-23T21:04:43.703 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.5149999999999997e-06 2023-09-23T21:04:43.703 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.703 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.703 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:04:43.703 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:14.221974+0000", 2023-09-23T21:04:43.703 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00025682100000000001 2023-09-23T21:04:43.703 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.703 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.703 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting_for_map not empty", 2023-09-23T21:04:43.703 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:14.221976+0000", 2023-09-23T21:04:43.703 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.4660000000000002e-06 2023-09-23T21:04:43.703 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.704 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.704 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:04:43.704 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:14.263558+0000", 2023-09-23T21:04:43.704 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.041581348999999997 2023-09-23T21:04:43.704 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.704 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.704 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T21:04:43.704 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:14.263565+0000", 2023-09-23T21:04:43.704 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.3579999999999997e-06 2023-09-23T21:04:43.704 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.704 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.704 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:04:43.704 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:15.770607+0000", 2023-09-23T21:04:43.705 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.507042191 2023-09-23T21:04:43.705 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.705 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.705 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:04:43.705 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:15.770740+0000", 2023-09-23T21:04:43.705 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00013265099999999999 2023-09-23T21:04:43.705 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.705 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.705 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 0,3,6,7", 2023-09-23T21:04:43.705 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:15.770992+0000", 2023-09-23T21:04:43.705 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00025236099999999999 2023-09-23T21:04:43.705 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.705 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.706 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T21:04:43.706 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:15.773623+0000", 2023-09-23T21:04:43.706 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0026309430000000002 2023-09-23T21:04:43.706 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.706 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.706 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:04:43.706 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:15.773655+0000", 2023-09-23T21:04:43.706 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.2033000000000003e-05 2023-09-23T21:04:43.706 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.706 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.706 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:04:43.706 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:15.773699+0000", 2023-09-23T21:04:43.706 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.3545999999999999e-05 2023-09-23T21:04:43.707 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.707 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.707 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:04:43.707 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:15.773829+0000", 2023-09-23T21:04:43.707 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000130524 2023-09-23T21:04:43.707 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.707 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.707 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:04:43.707 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:15.803314+0000", 2023-09-23T21:04:43.707 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.029484903999999999 2023-09-23T21:04:43.707 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.707 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.707 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T21:04:43.708 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:15.803380+0000", 2023-09-23T21:04:43.708 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.5976000000000007e-05 2023-09-23T21:04:43.708 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.708 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.708 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T21:04:43.708 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:15.803514+0000", 2023-09-23T21:04:43.708 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00013375799999999999 2023-09-23T21:04:43.708 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:04:43.708 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:04:43.708 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:04:43.708 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.708 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.708 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35469.0:164328 8.17 8:ec2bb224:::benchmark_data_smithi026_301269_object53571:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e308)", 2023-09-23T21:04:43.708 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T21:04:14.249164+0000", 2023-09-23T21:04:43.709 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 29.366102548000001, 2023-09-23T21:04:43.709 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.55822415, 2023-09-23T21:04:43.709 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:04:43.709 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:04:43.709 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T21:04:43.709 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35469", 2023-09-23T21:04:43.709 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/4041424623", 2023-09-23T21:04:43.709 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 164328 2023-09-23T21:04:43.709 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.709 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:04:43.709 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.709 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:04:43.709 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:14.249164+0000", 2023-09-23T21:04:43.710 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:04:43.710 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.710 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.710 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:04:43.710 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:14.249164+0000", 2023-09-23T21:04:43.710 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:04:43.710 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.710 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.710 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:04:43.710 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:14.249162+0000", 2023-09-23T21:04:43.710 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T21:04:43.710 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.710 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.711 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:04:43.711 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:14.249169+0000", 2023-09-23T21:04:43.711 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.9879999999999998e-06 2023-09-23T21:04:43.711 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.711 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.711 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:04:43.711 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:14.263868+0000", 2023-09-23T21:04:43.711 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.014698645999999999 2023-09-23T21:04:43.711 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.711 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.711 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:04:43.711 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:14.263881+0000", 2023-09-23T21:04:43.711 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.2553e-05 2023-09-23T21:04:43.712 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.712 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.712 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:04:43.712 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:14.264283+0000", 2023-09-23T21:04:43.712 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00040207099999999999 2023-09-23T21:04:43.712 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.712 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.712 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T21:04:43.712 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:14.264292+0000", 2023-09-23T21:04:43.712 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.9290000000000004e-06 2023-09-23T21:04:43.712 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.712 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.712 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:04:43.712 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:15.772071+0000", 2023-09-23T21:04:43.713 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.5077798410000001 2023-09-23T21:04:43.713 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.713 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.713 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:04:43.713 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:15.772203+0000", 2023-09-23T21:04:43.713 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00013204300000000001 2023-09-23T21:04:43.713 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.713 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.713 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 0,3,6,7", 2023-09-23T21:04:43.713 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:15.772377+0000", 2023-09-23T21:04:43.713 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00017358400000000001 2023-09-23T21:04:43.713 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.713 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.714 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T21:04:43.714 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:15.773634+0000", 2023-09-23T21:04:43.714 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.001257008 2023-09-23T21:04:43.714 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.714 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.714 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:04:43.714 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:15.773742+0000", 2023-09-23T21:04:43.714 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000107759 2023-09-23T21:04:43.714 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.714 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.714 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:04:43.714 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:15.773888+0000", 2023-09-23T21:04:43.715 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00014640899999999999 2023-09-23T21:04:43.715 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.715 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.715 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:04:43.715 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:15.773936+0000", 2023-09-23T21:04:43.715 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.7555000000000001e-05 2023-09-23T21:04:43.715 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.715 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.715 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:04:43.715 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:15.807209+0000", 2023-09-23T21:04:43.715 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.033272823 2023-09-23T21:04:43.715 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.716 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.716 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T21:04:43.716 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:15.807255+0000", 2023-09-23T21:04:43.716 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.6594999999999997e-05 2023-09-23T21:04:43.716 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.716 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.716 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T21:04:43.716 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:15.807388+0000", 2023-09-23T21:04:43.716 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00013274399999999999 2023-09-23T21:04:43.716 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:04:43.716 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:04:43.716 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:04:43.717 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.717 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.717 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35469.0:164364 8.17 8:e87a7c31:::benchmark_data_smithi026_301269_object53607:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e308)", 2023-09-23T21:04:43.717 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T21:04:14.308428+0000", 2023-09-23T21:04:43.717 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 29.306838584000001, 2023-09-23T21:04:43.717 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.5015923259999999, 2023-09-23T21:04:43.717 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:04:43.717 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:04:43.717 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T21:04:43.717 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35469", 2023-09-23T21:04:43.717 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/4041424623", 2023-09-23T21:04:43.717 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 164364 2023-09-23T21:04:43.717 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.718 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:04:43.718 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.718 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:04:43.718 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:14.308428+0000", 2023-09-23T21:04:43.718 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:04:43.718 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.718 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.718 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:04:43.718 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:14.308428+0000", 2023-09-23T21:04:43.718 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:04:43.718 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.718 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.719 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:04:43.719 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:14.308426+0000", 2023-09-23T21:04:43.719 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T21:04:43.719 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.719 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.719 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:04:43.719 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:14.308433+0000", 2023-09-23T21:04:43.719 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.3500000000000002e-06 2023-09-23T21:04:43.719 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.719 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.719 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:04:43.719 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:14.308448+0000", 2023-09-23T21:04:43.719 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.5443000000000001e-05 2023-09-23T21:04:43.720 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.720 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.720 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:04:43.720 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:14.308456+0000", 2023-09-23T21:04:43.720 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.1780000000000003e-06 2023-09-23T21:04:43.720 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.720 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.720 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:04:43.720 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:14.308516+0000", 2023-09-23T21:04:43.720 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.9799999999999997e-05 2023-09-23T21:04:43.720 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.720 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.720 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T21:04:43.721 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:14.308529+0000", 2023-09-23T21:04:43.721 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.2376e-05 2023-09-23T21:04:43.721 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.721 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.721 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:04:43.721 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:15.773954+0000", 2023-09-23T21:04:43.721 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.4654255780000001 2023-09-23T21:04:43.721 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.721 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.721 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:04:43.721 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:15.774030+0000", 2023-09-23T21:04:43.721 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.5551999999999998e-05 2023-09-23T21:04:43.721 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.722 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.722 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 0,3,6,7", 2023-09-23T21:04:43.722 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:15.774293+0000", 2023-09-23T21:04:43.722 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00026363899999999997 2023-09-23T21:04:43.722 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.722 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.722 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:04:43.722 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:15.778064+0000", 2023-09-23T21:04:43.722 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0037702920000000002 2023-09-23T21:04:43.722 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.722 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.722 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T21:04:43.723 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:15.778071+0000", 2023-09-23T21:04:43.723 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.413e-06 2023-09-23T21:04:43.723 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.723 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.723 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:04:43.723 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:15.778098+0000", 2023-09-23T21:04:43.723 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.6514000000000001e-05 2023-09-23T21:04:43.723 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.723 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.723 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:04:43.723 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:15.778156+0000", 2023-09-23T21:04:43.723 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.8581e-05 2023-09-23T21:04:43.723 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.724 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.724 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:04:43.724 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:15.809852+0000", 2023-09-23T21:04:43.724 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.031695657000000002 2023-09-23T21:04:43.724 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.724 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.724 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T21:04:43.724 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:15.809898+0000", 2023-09-23T21:04:43.724 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.5938e-05 2023-09-23T21:04:43.724 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.724 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.724 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T21:04:43.724 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:15.810020+0000", 2023-09-23T21:04:43.725 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00012233599999999999 2023-09-23T21:04:43.725 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:04:43.725 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:04:43.725 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:04:43.725 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.725 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.725 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35469.0:164461 8.17 8:efd86e4e:::benchmark_data_smithi026_301269_object53704:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e308)", 2023-09-23T21:04:43.725 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T21:04:14.431459+0000", 2023-09-23T21:04:43.725 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 29.183806915000002, 2023-09-23T21:04:43.725 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.3818049509999999, 2023-09-23T21:04:43.725 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:04:43.725 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:04:43.726 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T21:04:43.726 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35469", 2023-09-23T21:04:43.726 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/4041424623", 2023-09-23T21:04:43.726 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 164461 2023-09-23T21:04:43.726 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.726 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:04:43.726 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.726 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:04:43.726 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:14.431459+0000", 2023-09-23T21:04:43.726 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:04:43.726 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.726 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.727 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:04:43.727 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:14.431459+0000", 2023-09-23T21:04:43.727 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:04:43.727 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.727 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.727 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:04:43.727 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:14.431459+0000", 2023-09-23T21:04:43.727 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T21:04:43.727 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.727 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.727 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:04:43.727 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:14.431469+0000", 2023-09-23T21:04:43.727 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.8320000000000001e-06 2023-09-23T21:04:43.727 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.728 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.728 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:04:43.728 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:14.431491+0000", 2023-09-23T21:04:43.728 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.2648999999999999e-05 2023-09-23T21:04:43.728 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.728 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.728 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:04:43.728 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:14.431501+0000", 2023-09-23T21:04:43.728 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.8179999999999999e-06 2023-09-23T21:04:43.728 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.728 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.728 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:04:43.729 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:14.431555+0000", 2023-09-23T21:04:43.729 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.3579000000000003e-05 2023-09-23T21:04:43.729 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.729 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.729 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T21:04:43.729 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:14.431569+0000", 2023-09-23T21:04:43.729 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.4112e-05 2023-09-23T21:04:43.729 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.729 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.729 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:04:43.729 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:15.775335+0000", 2023-09-23T21:04:43.729 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.343765785 2023-09-23T21:04:43.729 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.730 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.730 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:04:43.730 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:15.775416+0000", 2023-09-23T21:04:43.730 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.1890000000000004e-05 2023-09-23T21:04:43.730 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.730 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.730 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 0,3,6,7", 2023-09-23T21:04:43.730 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:15.775636+0000", 2023-09-23T21:04:43.730 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00021961999999999999 2023-09-23T21:04:43.730 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.730 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.730 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T21:04:43.731 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:15.778170+0000", 2023-09-23T21:04:43.731 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.002534068 2023-09-23T21:04:43.731 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.731 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.731 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:04:43.731 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:15.778194+0000", 2023-09-23T21:04:43.731 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.3368000000000001e-05 2023-09-23T21:04:43.731 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.731 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.731 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:04:43.731 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:15.778248+0000", 2023-09-23T21:04:43.731 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.4153000000000001e-05 2023-09-23T21:04:43.731 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.732 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.732 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:04:43.732 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:15.779693+0000", 2023-09-23T21:04:43.732 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0014451469999999999 2023-09-23T21:04:43.732 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.732 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.732 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:04:43.732 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:15.813137+0000", 2023-09-23T21:04:43.732 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.033444164999999998 2023-09-23T21:04:43.732 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.732 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.732 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T21:04:43.732 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:15.813172+0000", 2023-09-23T21:04:43.733 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.5302000000000002e-05 2023-09-23T21:04:43.733 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.733 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.733 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T21:04:43.733 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:15.813264+0000", 2023-09-23T21:04:43.733 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.2016999999999997e-05 2023-09-23T21:04:43.733 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:04:43.733 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:04:43.733 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:04:43.733 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.733 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.733 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35469.0:164476 8.17 8:ec5aa70c:::benchmark_data_smithi026_301269_object53719:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e308)", 2023-09-23T21:04:43.733 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T21:04:14.445940+0000", 2023-09-23T21:04:43.734 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 29.169326013999999, 2023-09-23T21:04:43.734 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.379996805, 2023-09-23T21:04:43.734 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:04:43.734 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:04:43.734 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T21:04:43.734 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35469", 2023-09-23T21:04:43.734 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/4041424623", 2023-09-23T21:04:43.734 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 164476 2023-09-23T21:04:43.734 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.734 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:04:43.734 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.734 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:04:43.735 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:14.445940+0000", 2023-09-23T21:04:43.735 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:04:43.735 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.735 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.735 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:04:43.735 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:14.445940+0000", 2023-09-23T21:04:43.735 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:04:43.735 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.735 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.735 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:04:43.735 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:14.445939+0000", 2023-09-23T21:04:43.735 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.999999 2023-09-23T21:04:43.735 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.736 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.736 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:04:43.736 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:14.445946+0000", 2023-09-23T21:04:43.736 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.4019999999999999e-06 2023-09-23T21:04:43.736 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.736 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.736 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:04:43.736 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:14.445962+0000", 2023-09-23T21:04:43.736 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.6708e-05 2023-09-23T21:04:43.736 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.736 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.736 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:04:43.737 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:14.445970+0000", 2023-09-23T21:04:43.737 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.7659999999999998e-06 2023-09-23T21:04:43.737 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.737 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.737 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:04:43.737 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:14.447576+0000", 2023-09-23T21:04:43.737 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0016063819999999999 2023-09-23T21:04:43.737 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.737 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.737 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T21:04:43.737 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:14.447590+0000", 2023-09-23T21:04:43.737 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.3918e-05 2023-09-23T21:04:43.737 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.738 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.738 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:04:43.738 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:15.776700+0000", 2023-09-23T21:04:43.738 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.3291099630000001 2023-09-23T21:04:43.738 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.738 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.738 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:04:43.738 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:15.776805+0000", 2023-09-23T21:04:43.738 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000104743 2023-09-23T21:04:43.738 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.738 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.738 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 0,3,6,7", 2023-09-23T21:04:43.739 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:15.776987+0000", 2023-09-23T21:04:43.739 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00018181499999999999 2023-09-23T21:04:43.739 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.739 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.739 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T21:04:43.739 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:15.778262+0000", 2023-09-23T21:04:43.739 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.001275488 2023-09-23T21:04:43.739 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.739 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.739 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:04:43.739 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:15.779605+0000", 2023-09-23T21:04:43.739 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.001342719 2023-09-23T21:04:43.739 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.740 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.740 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:04:43.740 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:15.779637+0000", 2023-09-23T21:04:43.740 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.1792000000000003e-05 2023-09-23T21:04:43.740 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.740 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.740 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:04:43.740 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:15.779717+0000", 2023-09-23T21:04:43.740 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.0205999999999997e-05 2023-09-23T21:04:43.740 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.740 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.740 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:04:43.740 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:15.825740+0000", 2023-09-23T21:04:43.741 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.046023361999999998 2023-09-23T21:04:43.741 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.741 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.741 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T21:04:43.741 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:15.825798+0000", 2023-09-23T21:04:43.741 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.7941e-05 2023-09-23T21:04:43.741 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.741 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.741 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T21:04:43.741 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:15.825937+0000", 2023-09-23T21:04:43.741 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00013873600000000001 2023-09-23T21:04:43.741 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:04:43.741 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:04:43.742 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:04:43.742 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.742 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.742 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35469.0:164505 8.17 8:eac0c871:::benchmark_data_smithi026_301269_object53748:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e308)", 2023-09-23T21:04:43.742 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T21:04:14.495640+0000", 2023-09-23T21:04:43.742 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 29.119626180000001, 2023-09-23T21:04:43.742 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.3307733690000001, 2023-09-23T21:04:43.742 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:04:43.742 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:04:43.742 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T21:04:43.742 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35469", 2023-09-23T21:04:43.742 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/4041424623", 2023-09-23T21:04:43.743 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 164505 2023-09-23T21:04:43.743 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.743 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:04:43.743 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.743 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:04:43.743 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:14.495640+0000", 2023-09-23T21:04:43.743 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:04:43.743 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.743 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.743 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:04:43.743 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:14.495640+0000", 2023-09-23T21:04:43.743 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:04:43.743 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.744 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.744 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:04:43.744 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:14.495638+0000", 2023-09-23T21:04:43.744 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T21:04:43.744 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.744 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.744 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:04:43.744 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:14.495645+0000", 2023-09-23T21:04:43.744 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.9630000000000001e-06 2023-09-23T21:04:43.744 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.744 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.744 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:04:43.744 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:14.495656+0000", 2023-09-23T21:04:43.745 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0839000000000001e-05 2023-09-23T21:04:43.745 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.745 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.745 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:04:43.745 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:14.495664+0000", 2023-09-23T21:04:43.745 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.9400000000000002e-06 2023-09-23T21:04:43.745 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.745 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.745 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:04:43.745 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:14.495714+0000", 2023-09-23T21:04:43.745 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.9845000000000001e-05 2023-09-23T21:04:43.745 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.745 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.746 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T21:04:43.746 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:14.495725+0000", 2023-09-23T21:04:43.746 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1311999999999999e-05 2023-09-23T21:04:43.746 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.746 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.746 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:04:43.746 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:15.778301+0000", 2023-09-23T21:04:43.746 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.282575625 2023-09-23T21:04:43.746 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.746 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.746 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:04:43.746 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:15.778418+0000", 2023-09-23T21:04:43.747 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00011702 2023-09-23T21:04:43.747 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.747 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.747 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 0,3,6,7", 2023-09-23T21:04:43.747 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:15.778605+0000", 2023-09-23T21:04:43.747 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000187207 2023-09-23T21:04:43.747 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.747 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.747 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T21:04:43.747 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:15.781154+0000", 2023-09-23T21:04:43.747 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0025491490000000001 2023-09-23T21:04:43.747 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.747 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.748 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:04:43.748 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:15.782458+0000", 2023-09-23T21:04:43.748 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0013039779999999999 2023-09-23T21:04:43.748 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.748 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.748 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:04:43.748 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:15.782541+0000", 2023-09-23T21:04:43.748 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.2898999999999994e-05 2023-09-23T21:04:43.748 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.748 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.748 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:04:43.748 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:15.782595+0000", 2023-09-23T21:04:43.748 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.4079000000000002e-05 2023-09-23T21:04:43.749 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.749 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.749 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:04:43.749 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:15.826161+0000", 2023-09-23T21:04:43.749 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.043565319999999998 2023-09-23T21:04:43.749 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.749 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.749 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T21:04:43.749 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:15.826232+0000", 2023-09-23T21:04:43.749 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.1502999999999996e-05 2023-09-23T21:04:43.749 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.749 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.750 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T21:04:43.750 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:15.826413+0000", 2023-09-23T21:04:43.750 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00018132700000000001 2023-09-23T21:04:43.750 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:04:43.750 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:04:43.750 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:04:43.750 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.750 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.750 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35469.0:164563 8.17 8:eb9c7dfd:::benchmark_data_smithi026_301269_object53806:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e308)", 2023-09-23T21:04:43.750 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T21:04:14.588695+0000", 2023-09-23T21:04:43.750 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 29.026570936999999, 2023-09-23T21:04:43.750 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.238080692, 2023-09-23T21:04:43.750 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:04:43.751 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:04:43.751 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T21:04:43.751 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35469", 2023-09-23T21:04:43.751 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/4041424623", 2023-09-23T21:04:43.751 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 164563 2023-09-23T21:04:43.751 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.751 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:04:43.751 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.751 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:04:43.751 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:14.588695+0000", 2023-09-23T21:04:43.751 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:04:43.751 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.752 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.752 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:04:43.752 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:14.588695+0000", 2023-09-23T21:04:43.752 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:04:43.752 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.752 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.752 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:04:43.752 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:14.588694+0000", 2023-09-23T21:04:43.752 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999981 2023-09-23T21:04:43.752 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.752 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.752 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:04:43.752 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:14.588700+0000", 2023-09-23T21:04:43.752 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.4479999999999999e-06 2023-09-23T21:04:43.753 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.753 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.753 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:04:43.753 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:14.588717+0000", 2023-09-23T21:04:43.753 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.6793999999999999e-05 2023-09-23T21:04:43.753 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.753 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.753 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:04:43.753 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:14.588726+0000", 2023-09-23T21:04:43.753 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.1269999999999992e-06 2023-09-23T21:04:43.753 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.753 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.753 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:04:43.754 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:14.588808+0000", 2023-09-23T21:04:43.754 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.2060999999999999e-05 2023-09-23T21:04:43.754 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.754 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.754 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T21:04:43.754 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:14.588820+0000", 2023-09-23T21:04:43.754 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1698e-05 2023-09-23T21:04:43.754 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.754 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.754 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:04:43.754 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:15.779754+0000", 2023-09-23T21:04:43.754 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1909339560000001 2023-09-23T21:04:43.755 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.755 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.755 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:04:43.755 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:15.779892+0000", 2023-09-23T21:04:43.755 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00013873100000000001 2023-09-23T21:04:43.755 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.755 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.755 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 0,3,6,7", 2023-09-23T21:04:43.755 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:15.780132+0000", 2023-09-23T21:04:43.755 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00023909300000000001 2023-09-23T21:04:43.755 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.755 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.755 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T21:04:43.756 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:15.782607+0000", 2023-09-23T21:04:43.756 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0024757899999999998 2023-09-23T21:04:43.756 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.756 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.756 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:04:43.756 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:15.782668+0000", 2023-09-23T21:04:43.756 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.1153000000000002e-05 2023-09-23T21:04:43.756 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.756 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.756 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:04:43.756 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:15.782728+0000", 2023-09-23T21:04:43.756 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.9540999999999998e-05 2023-09-23T21:04:43.756 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.756 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.757 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:04:43.757 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:15.785719+0000", 2023-09-23T21:04:43.757 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.002990889 2023-09-23T21:04:43.757 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.757 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.757 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:04:43.757 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:15.826610+0000", 2023-09-23T21:04:43.757 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.040891248999999998 2023-09-23T21:04:43.757 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.757 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.757 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T21:04:43.757 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:15.826657+0000", 2023-09-23T21:04:43.757 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.7024999999999998e-05 2023-09-23T21:04:43.758 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.758 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.758 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T21:04:43.758 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:15.826776+0000", 2023-09-23T21:04:43.758 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000118854 2023-09-23T21:04:43.758 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:04:43.758 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:04:43.758 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:04:43.758 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.758 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.758 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35469.0:164631 8.17 8:ec596468:::benchmark_data_smithi026_301269_object53874:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e308)", 2023-09-23T21:04:43.758 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T21:04:14.720019+0000", 2023-09-23T21:04:43.758 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 28.895247429000001, 2023-09-23T21:04:43.759 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.107045319, 2023-09-23T21:04:43.759 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:04:43.759 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:04:43.759 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T21:04:43.759 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35469", 2023-09-23T21:04:43.759 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/4041424623", 2023-09-23T21:04:43.759 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 164631 2023-09-23T21:04:43.759 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.759 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:04:43.759 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.759 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:04:43.759 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:14.720019+0000", 2023-09-23T21:04:43.760 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:04:43.760 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.760 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.760 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:04:43.760 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:14.720019+0000", 2023-09-23T21:04:43.760 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:04:43.760 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.760 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.760 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:04:43.760 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:14.720017+0000", 2023-09-23T21:04:43.760 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T21:04:43.760 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.760 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.761 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:04:43.761 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:14.720025+0000", 2023-09-23T21:04:43.761 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.9859999999999994e-06 2023-09-23T21:04:43.761 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.761 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.761 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:04:43.761 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:14.720038+0000", 2023-09-23T21:04:43.761 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.2768e-05 2023-09-23T21:04:43.761 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.761 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.761 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:04:43.761 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:14.720047+0000", 2023-09-23T21:04:43.761 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.2949999999999994e-06 2023-09-23T21:04:43.762 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.762 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.762 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:04:43.762 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:14.720146+0000", 2023-09-23T21:04:43.762 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.9555999999999993e-05 2023-09-23T21:04:43.762 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.762 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.762 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T21:04:43.762 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:14.720158+0000", 2023-09-23T21:04:43.762 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1817000000000001e-05 2023-09-23T21:04:43.762 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.762 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.763 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:04:43.763 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:15.781201+0000", 2023-09-23T21:04:43.763 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.061043323 2023-09-23T21:04:43.763 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.763 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.763 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:04:43.763 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:15.781334+0000", 2023-09-23T21:04:43.763 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000132309 2023-09-23T21:04:43.763 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.763 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.763 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 0,3,6,7", 2023-09-23T21:04:43.763 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:15.781539+0000", 2023-09-23T21:04:43.763 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00020538899999999999 2023-09-23T21:04:43.764 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.764 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.764 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T21:04:43.764 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:15.782742+0000", 2023-09-23T21:04:43.764 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0012028729999999999 2023-09-23T21:04:43.764 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.764 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.764 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:04:43.764 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:15.785628+0000", 2023-09-23T21:04:43.764 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.002885903 2023-09-23T21:04:43.764 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.764 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.764 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:04:43.765 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:15.785682+0000", 2023-09-23T21:04:43.765 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.3974000000000002e-05 2023-09-23T21:04:43.765 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.765 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.765 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:04:43.765 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:15.785850+0000", 2023-09-23T21:04:43.765 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00016841299999999999 2023-09-23T21:04:43.765 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.765 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.765 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:04:43.765 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:15.826814+0000", 2023-09-23T21:04:43.765 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.040964226999999999 2023-09-23T21:04:43.765 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.765 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.766 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T21:04:43.766 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:15.826944+0000", 2023-09-23T21:04:43.766 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00012946100000000001 2023-09-23T21:04:43.766 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.766 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.766 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T21:04:43.766 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:15.827064+0000", 2023-09-23T21:04:43.766 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000120431 2023-09-23T21:04:43.766 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:04:43.766 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:04:43.766 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:04:43.766 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.766 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.767 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35469.0:177337 8.7 8:e6198cba:::benchmark_data_smithi026_301269_object66580:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e321)", 2023-09-23T21:04:43.767 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T21:04:35.139698+0000", 2023-09-23T21:04:43.767 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 8.4755684819999999, 2023-09-23T21:04:43.767 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.037658848000000002, 2023-09-23T21:04:43.767 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:04:43.767 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:04:43.767 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T21:04:43.767 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35469", 2023-09-23T21:04:43.767 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/4041424623", 2023-09-23T21:04:43.767 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 177337 2023-09-23T21:04:43.767 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.767 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:04:43.767 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.768 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:04:43.768 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:35.139698+0000", 2023-09-23T21:04:43.768 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:04:43.768 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.768 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.768 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:04:43.768 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:35.139698+0000", 2023-09-23T21:04:43.768 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:04:43.768 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.768 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.768 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:04:43.768 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:35.139696+0000", 2023-09-23T21:04:43.768 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T21:04:43.769 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.769 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.769 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:04:43.769 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:35.139703+0000", 2023-09-23T21:04:43.769 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.0690000000000004e-06 2023-09-23T21:04:43.769 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.769 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.769 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:04:43.769 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:35.139717+0000", 2023-09-23T21:04:43.769 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.4027e-05 2023-09-23T21:04:43.769 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.769 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.769 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:04:43.770 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:35.139726+0000", 2023-09-23T21:04:43.770 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.7909999999999993e-06 2023-09-23T21:04:43.770 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.770 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.770 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:04:43.770 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:35.140493+0000", 2023-09-23T21:04:43.770 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000766422 2023-09-23T21:04:43.770 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.770 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.770 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:04:43.770 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:35.140655+0000", 2023-09-23T21:04:43.770 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00016250600000000001 2023-09-23T21:04:43.770 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.771 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.771 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 3,6", 2023-09-23T21:04:43.771 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:35.141241+0000", 2023-09-23T21:04:43.771 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00058569500000000001 2023-09-23T21:04:43.771 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.771 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.771 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:04:43.771 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:35.158715+0000", 2023-09-23T21:04:43.771 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.017474168000000002 2023-09-23T21:04:43.771 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.771 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.771 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T21:04:43.771 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:35.158725+0000", 2023-09-23T21:04:43.772 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.9070000000000002e-06 2023-09-23T21:04:43.772 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.772 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.772 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:04:43.772 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:35.177273+0000", 2023-09-23T21:04:43.772 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.018547557999999999 2023-09-23T21:04:43.772 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.772 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.772 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T21:04:43.772 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:35.177318+0000", 2023-09-23T21:04:43.772 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.5877000000000001e-05 2023-09-23T21:04:43.772 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.772 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.772 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T21:04:43.773 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:35.177357+0000", 2023-09-23T21:04:43.773 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.8260999999999998e-05 2023-09-23T21:04:43.773 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:04:43.773 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:04:43.773 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:04:43.773 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.773 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.773 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35469.0:177460 8.7 8:e71dbf8c:::benchmark_data_smithi026_301269_object66703:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e321)", 2023-09-23T21:04:43.773 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T21:04:35.308721+0000", 2023-09-23T21:04:43.773 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 8.3065457639999991, 2023-09-23T21:04:43.773 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.038987831000000001, 2023-09-23T21:04:43.773 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:04:43.773 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:04:43.774 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T21:04:43.774 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35469", 2023-09-23T21:04:43.774 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/4041424623", 2023-09-23T21:04:43.774 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 177460 2023-09-23T21:04:43.774 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.774 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:04:43.774 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.774 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:04:43.774 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:35.308721+0000", 2023-09-23T21:04:43.774 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:04:43.774 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.774 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.774 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:04:43.775 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:35.308721+0000", 2023-09-23T21:04:43.775 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:04:43.775 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.775 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.775 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:04:43.775 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:35.308719+0000", 2023-09-23T21:04:43.775 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T21:04:43.775 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.775 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.775 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:04:43.775 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:35.308727+0000", 2023-09-23T21:04:43.775 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.5730000000000002e-06 2023-09-23T21:04:43.776 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.776 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.776 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:04:43.776 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:35.308740+0000", 2023-09-23T21:04:43.776 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.3264999999999999e-05 2023-09-23T21:04:43.776 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.776 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.776 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:04:43.776 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:35.308749+0000", 2023-09-23T21:04:43.776 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.2709999999999998e-06 2023-09-23T21:04:43.776 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.776 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.776 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:04:43.776 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:35.308849+0000", 2023-09-23T21:04:43.777 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00010021100000000001 2023-09-23T21:04:43.777 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.777 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.777 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:04:43.777 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:35.308999+0000", 2023-09-23T21:04:43.777 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000149359 2023-09-23T21:04:43.777 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.777 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.777 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 3,6", 2023-09-23T21:04:43.777 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:35.309490+0000", 2023-09-23T21:04:43.777 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00049118100000000004 2023-09-23T21:04:43.777 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.777 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.777 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:04:43.778 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:35.314858+0000", 2023-09-23T21:04:43.778 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0053677949999999999 2023-09-23T21:04:43.778 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.778 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.778 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T21:04:43.778 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:35.317752+0000", 2023-09-23T21:04:43.778 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0028947740000000001 2023-09-23T21:04:43.778 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.778 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.778 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:04:43.778 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:35.347625+0000", 2023-09-23T21:04:43.778 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.029873077000000001 2023-09-23T21:04:43.779 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.779 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.779 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T21:04:43.779 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:35.347673+0000", 2023-09-23T21:04:43.779 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.7054e-05 2023-09-23T21:04:43.779 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.779 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.779 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T21:04:43.779 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:35.347708+0000", 2023-09-23T21:04:43.779 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.5862000000000003e-05 2023-09-23T21:04:43.779 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:04:43.779 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:04:43.779 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:04:43.780 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.780 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.780 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35469.0:185086 8.1 8:83940022:::benchmark_data_smithi026_301269_object74329:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e322)", 2023-09-23T21:04:43.780 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T21:04:43.262772+0000", 2023-09-23T21:04:43.780 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 0.35249465000000002, 2023-09-23T21:04:43.780 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.041815819999999997, 2023-09-23T21:04:43.780 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:04:43.780 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:04:43.780 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T21:04:43.780 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35469", 2023-09-23T21:04:43.780 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/4041424623", 2023-09-23T21:04:43.780 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 185086 2023-09-23T21:04:43.780 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.781 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:04:43.781 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.781 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:04:43.781 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:43.262772+0000", 2023-09-23T21:04:43.781 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:04:43.781 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.781 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.781 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:04:43.781 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:43.262772+0000", 2023-09-23T21:04:43.781 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:04:43.781 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.781 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.781 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:04:43.782 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:43.262770+0000", 2023-09-23T21:04:43.782 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T21:04:43.782 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.782 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.782 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:04:43.782 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:43.262777+0000", 2023-09-23T21:04:43.782 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.426e-06 2023-09-23T21:04:43.782 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.782 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.782 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:04:43.782 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:43.262793+0000", 2023-09-23T21:04:43.782 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.6852999999999999e-05 2023-09-23T21:04:43.782 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.783 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.783 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:04:43.783 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:43.262809+0000", 2023-09-23T21:04:43.783 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.5766000000000001e-05 2023-09-23T21:04:43.783 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.783 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.783 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:04:43.783 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:43.262979+0000", 2023-09-23T21:04:43.783 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000169579 2023-09-23T21:04:43.783 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.783 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.783 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:04:43.783 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:43.263084+0000", 2023-09-23T21:04:43.784 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00010565699999999999 2023-09-23T21:04:43.784 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.784 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.784 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 0,2", 2023-09-23T21:04:43.784 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:43.263684+0000", 2023-09-23T21:04:43.784 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00059993199999999996 2023-09-23T21:04:43.784 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.784 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.784 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T21:04:43.784 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:43.268523+0000", 2023-09-23T21:04:43.784 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0048387079999999997 2023-09-23T21:04:43.784 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.784 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.784 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:04:43.785 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:43.277121+0000", 2023-09-23T21:04:43.785 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0085978019999999999 2023-09-23T21:04:43.785 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.785 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.785 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:04:43.785 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:43.304508+0000", 2023-09-23T21:04:43.785 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.027387563 2023-09-23T21:04:43.785 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.785 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.785 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T21:04:43.785 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:43.304550+0000", 2023-09-23T21:04:43.785 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.1436000000000001e-05 2023-09-23T21:04:43.785 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.786 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.786 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T21:04:43.786 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:43.304587+0000", 2023-09-23T21:04:43.786 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.7594e-05 2023-09-23T21:04:43.786 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:04:43.786 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:04:43.786 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:04:43.786 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.786 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.786 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35469.0:185104 8.3 8:c405dfec:::benchmark_data_smithi026_301269_object74347:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e322)", 2023-09-23T21:04:43.786 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T21:04:43.281796+0000", 2023-09-23T21:04:43.786 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 0.33347038899999998, 2023-09-23T21:04:43.787 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.037895676000000003, 2023-09-23T21:04:43.787 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:04:43.787 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:04:43.787 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T21:04:43.787 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35469", 2023-09-23T21:04:43.787 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/4041424623", 2023-09-23T21:04:43.787 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 185104 2023-09-23T21:04:43.787 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.787 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:04:43.787 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.787 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:04:43.787 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:43.281796+0000", 2023-09-23T21:04:43.787 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:04:43.787 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.788 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.788 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:04:43.788 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:43.281796+0000", 2023-09-23T21:04:43.788 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:04:43.788 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.788 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.788 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:04:43.788 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:43.281796+0000", 2023-09-23T21:04:43.788 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T21:04:43.788 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.788 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.788 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:04:43.788 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:43.281798+0000", 2023-09-23T21:04:43.789 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.0820000000000001e-06 2023-09-23T21:04:43.789 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.789 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.789 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:04:43.789 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:43.281806+0000", 2023-09-23T21:04:43.789 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.028e-06 2023-09-23T21:04:43.789 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.789 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.789 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:04:43.789 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:43.281811+0000", 2023-09-23T21:04:43.789 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.3070000000000002e-06 2023-09-23T21:04:43.789 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.789 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.790 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:04:43.790 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:43.305387+0000", 2023-09-23T21:04:43.790 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.02357573 2023-09-23T21:04:43.790 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.790 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.790 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:04:43.790 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:43.305527+0000", 2023-09-23T21:04:43.790 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000139779 2023-09-23T21:04:43.790 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.790 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.790 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 5,6", 2023-09-23T21:04:43.790 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:43.306044+0000", 2023-09-23T21:04:43.790 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00051787000000000003 2023-09-23T21:04:43.791 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.791 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.791 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:04:43.791 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:43.308631+0000", 2023-09-23T21:04:43.791 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0025861780000000002 2023-09-23T21:04:43.791 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.791 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.791 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:04:43.791 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:43.319565+0000", 2023-09-23T21:04:43.791 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.010934074 2023-09-23T21:04:43.791 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.791 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.791 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T21:04:43.792 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:43.319601+0000", 2023-09-23T21:04:43.792 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.6363999999999999e-05 2023-09-23T21:04:43.792 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.792 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.792 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T21:04:43.792 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:43.319649+0000", 2023-09-23T21:04:43.792 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.8013000000000003e-05 2023-09-23T21:04:43.792 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:43.792 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:43.792 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T21:04:43.792 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:43.319692+0000", 2023-09-23T21:04:43.792 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.2561999999999997e-05 2023-09-23T21:04:43.792 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:04:43.793 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:04:43.793 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:04:43.793 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:04:43.793 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:04:43.793 INFO:teuthology.orchestra.run.smithi026.stdout:} 2023-09-23T21:04:43.793 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:04:43 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/863680334' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:04:43.793 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:04:43 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/863680334' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:04:43.794 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:04:43.861 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:04:44.279 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:04:44 smithi026 ceph-mon[175151]: pgmap v829: 119 pgs: 119 active+clean; 2.3 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 385 op/s 2023-09-23T21:04:44.279 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:04:44 smithi026 ceph-mon[179688]: pgmap v829: 119 pgs: 119 active+clean; 2.3 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 385 op/s 2023-09-23T21:04:44.307 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:04:44.315 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:04:44.316 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:04:44.380 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_ops_in_flight 2023-09-23T21:04:44.380 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:04:44.417 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:04:44.516 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:04:44.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:04:44 smithi070 ceph-mon[168282]: pgmap v829: 119 pgs: 119 active+clean; 2.3 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 385 op/s 2023-09-23T21:04:44.622 INFO:teuthology.orchestra.run.smithi026.stdout: 2023-09-23T21:04:44.625 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:04:44.630 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:04:44.630 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:04:44.665 INFO:teuthology.orchestra.run.smithi026.stderr:dumped all 2023-09-23T21:04:44.731 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:04:44.788 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:04:44.865 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T21:04:44.872 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:04:44.872 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:04:44.973 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:04:45.021 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:04:45.068 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:04:45.073 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:04:45.074 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:04:45.174 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:04:45.219 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:04:45.277 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:04:45.282 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:04:45.283 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:04:45.384 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:04:45.417 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:04:45.482 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:04:45.485 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:04:45.485 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:04:45.586 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:04:45.621 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:04:45.686 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:04:45.689 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:04:45.689 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:04:45.790 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:04:45.823 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:04:45.873 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:04:45.878 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:04:45.879 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:04:45.979 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:04:46.010 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:04:46.055 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:04:46.060 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:04:46.060 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:04:46.080 INFO:teuthology.orchestra.run.smithi026.stdout:{"pg_ready":true,"pg_map":{"version":829,"stamp":"2023-09-23T21:04:42.782869+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":2521143224,"num_objects":38485,"num_object_clones":0,"num_object_copies":115455,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":38485,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":140514,"num_write_kb":5093534,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":15274,"num_bytes_recovered":999298918,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":217342,"ondisk_log_size":217342,"up":357,"acting":357,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":376,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":11402572,"kb_used_data":8710348,"kb_used_omap":136,"kb_used_meta":2691703,"kb_avail":738558644,"statfs":{"total":767960285184,"available":756284051456,"internally_reserved":0,"allocated":8919396352,"data_stored":8909305534,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":139741,"internal_metadata":2756304419},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":4,"apply_latency_ms":4,"commit_latency_ns":4000000,"apply_latency_ns":4000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-595132416,"num_objects":-9081,"num_object_clones":0,"num_object_copies":-27243,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":-9081,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3616,"num_write_kb":-223392,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":-1805,"num_bytes_recovered":-118226944,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"9.381092"},"pg_stats":[{"pgid":"8.11","version":"322'10069","reported_seq":19391,"reported_epoch":322,"state":"active+clean","last_fresh":"2023-09-23T21:04:42.050621+0000","last_change":"2023-09-23T21:04:04.904208+0000","last_active":"2023-09-23T21:04:42.050621+0000","last_peered":"2023-09-23T21:04:42.050621+0000","last_clean":"2023-09-23T21:04:42.050621+0000","last_became_active":"2023-09-23T21:04:04.892614+0000","last_became_peered":"2023-09-23T21:04:04.892614+0000","last_unstale":"2023-09-23T21:04:42.050621+0000","last_undegraded":"2023-09-23T21:04:42.050621+0000","last_fullsized":"2023-09-23T21:04:42.050621+0000","mapping_epoch":300,"log_start":"258'1600","ondisk_log_start":"258'1600","created":281,"last_epoch_clean":301,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:01:24.325318+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:01:24.325318+0000","last_clean_scrub_stamp":"2023-09-23T21:01:24.325318+0000","objects_scrubbed":0,"log_size":8469,"log_dups_size":0,"ondisk_log_size":8469,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":79691776,"num_objects":1216,"num_object_clones":0,"num_object_copies":3648,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1216,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5830,"num_write_kb":225472,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2304,"num_bytes_recovered":150929408,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1a","version":"68'1","reported_seq":199,"reported_epoch":321,"state":"active+clean","last_fresh":"2023-09-23T21:04:33.411599+0000","last_change":"2023-09-23T20:54:13.946695+0000","last_active":"2023-09-23T21:04:33.411599+0000","last_peered":"2023-09-23T21:04:33.411599+0000","last_clean":"2023-09-23T21:04:33.411599+0000","last_became_active":"2023-09-23T20:54:13.945399+0000","last_became_peered":"2023-09-23T20:54:13.945399+0000","last_unstale":"2023-09-23T21:04:33.411599+0000","last_undegraded":"2023-09-23T21:04:33.411599+0000","last_fullsized":"2023-09-23T21:04:33.411599+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:32:52.365464+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":315,"reported_epoch":320,"state":"active+clean","last_fresh":"2023-09-23T21:04:32.423656+0000","last_change":"2023-09-23T20:54:13.942019+0000","last_active":"2023-09-23T21:04:32.423656+0000","last_peered":"2023-09-23T21:04:32.423656+0000","last_clean":"2023-09-23T21:04:32.423656+0000","last_became_active":"2023-09-23T20:54:13.941647+0000","last_became_peered":"2023-09-23T20:54:13.941647+0000","last_unstale":"2023-09-23T21:04:32.423656+0000","last_undegraded":"2023-09-23T21:04:32.423656+0000","last_fullsized":"2023-09-23T21:04:32.423656+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:30:54.400944+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.12","version":"321'9919","reported_seq":17642,"reported_epoch":321,"state":"active+clean","last_fresh":"2023-09-23T21:04:39.082898+0000","last_change":"2023-09-23T21:04:27.883034+0000","last_active":"2023-09-23T21:04:39.082898+0000","last_peered":"2023-09-23T21:04:39.082898+0000","last_clean":"2023-09-23T21:04:39.082898+0000","last_became_active":"2023-09-23T21:04:14.222393+0000","last_became_peered":"2023-09-23T21:04:14.222393+0000","last_unstale":"2023-09-23T21:04:39.082898+0000","last_undegraded":"2023-09-23T21:04:39.082898+0000","last_fullsized":"2023-09-23T21:04:39.082898+0000","mapping_epoch":307,"log_start":"258'1700","ondisk_log_start":"258'1700","created":281,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":5,"last_scrub":"316'9655","last_scrub_stamp":"2023-09-23T21:04:27.882966+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:01:24.325318+0000","last_clean_scrub_stamp":"2023-09-23T21:04:27.882966+0000","objects_scrubbed":1537,"log_size":8219,"log_dups_size":0,"ondisk_log_size":8219,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:14:00.916321+0000","scrub_duration":0.44158506600000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":82903040,"num_objects":1265,"num_object_clones":0,"num_object_copies":3795,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1265,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":272,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":293,"reported_epoch":321,"state":"active+clean","last_fresh":"2023-09-23T21:04:33.412006+0000","last_change":"2023-09-23T20:54:13.941394+0000","last_active":"2023-09-23T21:04:33.412006+0000","last_peered":"2023-09-23T21:04:33.412006+0000","last_clean":"2023-09-23T21:04:33.412006+0000","last_became_active":"2023-09-23T20:54:13.941042+0000","last_became_peered":"2023-09-23T20:54:13.941042+0000","last_unstale":"2023-09-23T21:04:33.412006+0000","last_undegraded":"2023-09-23T21:04:33.412006+0000","last_fullsized":"2023-09-23T21:04:33.412006+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:19:41.401184+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":309,"reported_epoch":321,"state":"active+clean","last_fresh":"2023-09-23T21:04:33.429807+0000","last_change":"2023-09-23T20:57:44.979811+0000","last_active":"2023-09-23T21:04:33.429807+0000","last_peered":"2023-09-23T21:04:33.429807+0000","last_clean":"2023-09-23T21:04:33.429807+0000","last_became_active":"2023-09-23T20:57:44.979515+0000","last_became_peered":"2023-09-23T20:57:44.979515+0000","last_unstale":"2023-09-23T21:04:33.429807+0000","last_undegraded":"2023-09-23T21:04:33.429807+0000","last_fullsized":"2023-09-23T21:04:33.429807+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:38:22.162140+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.13","version":"321'9478","reported_seq":15988,"reported_epoch":321,"state":"active+clean","last_fresh":"2023-09-23T21:04:39.778249+0000","last_change":"2023-09-23T21:04:27.787733+0000","last_active":"2023-09-23T21:04:39.778249+0000","last_peered":"2023-09-23T21:04:39.778249+0000","last_clean":"2023-09-23T21:04:39.778249+0000","last_became_active":"2023-09-23T21:04:05.210583+0000","last_became_peered":"2023-09-23T21:04:05.210583+0000","last_unstale":"2023-09-23T21:04:39.778249+0000","last_undegraded":"2023-09-23T21:04:39.778249+0000","last_fullsized":"2023-09-23T21:04:39.778249+0000","mapping_epoch":301,"log_start":"258'1200","ondisk_log_start":"258'1200","created":281,"last_epoch_clean":302,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:01:24.325318+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:01:24.325318+0000","last_clean_scrub_stamp":"2023-09-23T21:01:24.325318+0000","objects_scrubbed":0,"log_size":8278,"log_dups_size":0,"ondisk_log_size":8278,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":81920000,"num_objects":1250,"num_object_clones":0,"num_object_copies":3750,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1250,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5456,"num_write_kb":214592,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2135,"num_bytes_recovered":139788288,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,1],"acting":[3,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":306,"reported_epoch":320,"state":"active+clean","last_fresh":"2023-09-23T21:04:32.423915+0000","last_change":"2023-09-23T20:51:10.833613+0000","last_active":"2023-09-23T21:04:32.423915+0000","last_peered":"2023-09-23T21:04:32.423915+0000","last_clean":"2023-09-23T21:04:32.423915+0000","last_became_active":"2023-09-23T20:51:10.833208+0000","last_became_peered":"2023-09-23T20:51:10.833208+0000","last_unstale":"2023-09-23T21:04:32.423915+0000","last_undegraded":"2023-09-23T21:04:32.423915+0000","last_fullsized":"2023-09-23T21:04:32.423915+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:51:28.246014+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":286,"reported_epoch":320,"state":"active+clean","last_fresh":"2023-09-23T21:04:32.399989+0000","last_change":"2023-09-23T20:55:35.958384+0000","last_active":"2023-09-23T21:04:32.399989+0000","last_peered":"2023-09-23T21:04:32.399989+0000","last_clean":"2023-09-23T21:04:32.399989+0000","last_became_active":"2023-09-23T20:55:35.954563+0000","last_became_peered":"2023-09-23T20:55:35.954563+0000","last_unstale":"2023-09-23T21:04:32.399989+0000","last_undegraded":"2023-09-23T21:04:32.399989+0000","last_fullsized":"2023-09-23T21:04:32.399989+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:37:48.332259+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":299,"reported_epoch":321,"state":"active+clean","last_fresh":"2023-09-23T21:04:33.430424+0000","last_change":"2023-09-23T20:54:31.885514+0000","last_active":"2023-09-23T21:04:33.430424+0000","last_peered":"2023-09-23T21:04:33.430424+0000","last_clean":"2023-09-23T21:04:33.430424+0000","last_became_active":"2023-09-23T20:54:31.884986+0000","last_became_peered":"2023-09-23T20:54:31.884986+0000","last_unstale":"2023-09-23T21:04:33.430424+0000","last_undegraded":"2023-09-23T21:04:33.430424+0000","last_fullsized":"2023-09-23T21:04:33.430424+0000","mapping_epoch":136,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:26:08.100220+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":303,"reported_epoch":320,"state":"active+clean","last_fresh":"2023-09-23T21:04:32.423955+0000","last_change":"2023-09-23T20:55:35.951889+0000","last_active":"2023-09-23T21:04:32.423955+0000","last_peered":"2023-09-23T21:04:32.423955+0000","last_clean":"2023-09-23T21:04:32.423955+0000","last_became_active":"2023-09-23T20:55:35.951563+0000","last_became_peered":"2023-09-23T20:55:35.951563+0000","last_unstale":"2023-09-23T21:04:32.423955+0000","last_undegraded":"2023-09-23T21:04:32.423955+0000","last_fullsized":"2023-09-23T21:04:32.423955+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:41:53.098781+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"79'161","reported_seq":611,"reported_epoch":321,"state":"active+clean","last_fresh":"2023-09-23T21:04:33.414855+0000","last_change":"2023-09-23T20:57:25.096559+0000","last_active":"2023-09-23T21:04:33.414855+0000","last_peered":"2023-09-23T21:04:33.414855+0000","last_clean":"2023-09-23T21:04:33.414855+0000","last_became_active":"2023-09-23T20:57:15.975674+0000","last_became_peered":"2023-09-23T20:57:15.975674+0000","last_unstale":"2023-09-23T21:04:33.414855+0000","last_undegraded":"2023-09-23T21:04:33.414855+0000","last_fullsized":"2023-09-23T21:04:33.414855+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:25.096401+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:25.096401+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:05:05.448404+0000","scrub_duration":0.002859393,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":195,"reported_epoch":321,"state":"active+clean","last_fresh":"2023-09-23T21:04:33.410728+0000","last_change":"2023-09-23T20:54:31.890678+0000","last_active":"2023-09-23T21:04:33.410728+0000","last_peered":"2023-09-23T21:04:33.410728+0000","last_clean":"2023-09-23T21:04:33.410728+0000","last_became_active":"2023-09-23T20:54:31.890469+0000","last_became_peered":"2023-09-23T20:54:31.890469+0000","last_unstale":"2023-09-23T21:04:33.410728+0000","last_undegraded":"2023-09-23T21:04:33.410728+0000","last_fullsized":"2023-09-23T21:04:33.410728+0000","mapping_epoch":136,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:55:08.790146+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":264,"reported_epoch":320,"state":"active+clean","last_fresh":"2023-09-23T21:04:32.423315+0000","last_change":"2023-09-23T20:55:35.952661+0000","last_active":"2023-09-23T21:04:32.423315+0000","last_peered":"2023-09-23T21:04:32.423315+0000","last_clean":"2023-09-23T21:04:32.423315+0000","last_became_active":"2023-09-23T20:55:35.952349+0000","last_became_peered":"2023-09-23T20:55:35.952349+0000","last_unstale":"2023-09-23T21:04:32.423315+0000","last_undegraded":"2023-09-23T21:04:32.423315+0000","last_fullsized":"2023-09-23T21:04:32.423315+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:00:03.110126+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":141,"reported_epoch":320,"state":"active+clean","last_fresh":"2023-09-23T21:04:32.423750+0000","last_change":"2023-09-23T20:56:45.988607+0000","last_active":"2023-09-23T21:04:32.423750+0000","last_peered":"2023-09-23T21:04:32.423750+0000","last_clean":"2023-09-23T21:04:32.423750+0000","last_became_active":"2023-09-23T20:56:45.988206+0000","last_became_peered":"2023-09-23T20:56:45.988206+0000","last_unstale":"2023-09-23T21:04:32.423750+0000","last_undegraded":"2023-09-23T21:04:32.423750+0000","last_fullsized":"2023-09-23T21:04:32.423750+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":205,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:01:19.783688+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,4],"acting":[3,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":139,"reported_epoch":321,"state":"active+clean","last_fresh":"2023-09-23T21:04:33.410510+0000","last_change":"2023-09-23T20:56:18.139142+0000","last_active":"2023-09-23T21:04:33.410510+0000","last_peered":"2023-09-23T21:04:33.410510+0000","last_clean":"2023-09-23T21:04:33.410510+0000","last_became_active":"2023-09-23T20:56:18.138844+0000","last_became_peered":"2023-09-23T20:56:18.138844+0000","last_unstale":"2023-09-23T21:04:33.410510+0000","last_undegraded":"2023-09-23T21:04:33.410510+0000","last_fullsized":"2023-09-23T21:04:33.410510+0000","mapping_epoch":192,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":193,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:09:02.217683+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,2],"acting":[5,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.14","version":"79'22","reported_seq":226,"reported_epoch":321,"state":"active+clean","last_fresh":"2023-09-23T21:04:33.411908+0000","last_change":"2023-09-23T20:55:35.949418+0000","last_active":"2023-09-23T21:04:33.411908+0000","last_peered":"2023-09-23T21:04:33.411908+0000","last_clean":"2023-09-23T21:04:33.411908+0000","last_became_active":"2023-09-23T20:55:35.948566+0000","last_became_peered":"2023-09-23T20:55:35.948566+0000","last_unstale":"2023-09-23T21:04:33.411908+0000","last_undegraded":"2023-09-23T21:04:33.411908+0000","last_fullsized":"2023-09-23T21:04:33.411908+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:59:37.433128+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4734,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":216,"reported_epoch":321,"state":"active+clean","last_fresh":"2023-09-23T21:04:33.411696+0000","last_change":"2023-09-23T20:55:35.949178+0000","last_active":"2023-09-23T21:04:33.411696+0000","last_peered":"2023-09-23T21:04:33.411696+0000","last_clean":"2023-09-23T21:04:33.411696+0000","last_became_active":"2023-09-23T20:55:35.948795+0000","last_became_peered":"2023-09-23T20:55:35.948795+0000","last_unstale":"2023-09-23T21:04:33.411696+0000","last_undegraded":"2023-09-23T21:04:33.411696+0000","last_fullsized":"2023-09-23T21:04:33.411696+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:13:08.996932+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.18","version":"322'9915","reported_seq":13149,"reported_epoch":322,"state":"active+clean","last_fresh":"2023-09-23T21:04:42.244111+0000","last_change":"2023-09-23T21:04:31.325373+0000","last_active":"2023-09-23T21:04:42.244111+0000","last_peered":"2023-09-23T21:04:42.244111+0000","last_clean":"2023-09-23T21:04:42.244111+0000","last_became_active":"2023-09-23T21:04:29.386634+0000","last_became_peered":"2023-09-23T21:04:29.386634+0000","last_unstale":"2023-09-23T21:04:42.244111+0000","last_undegraded":"2023-09-23T21:04:42.244111+0000","last_fullsized":"2023-09-23T21:04:42.244111+0000","mapping_epoch":317,"log_start":"258'600","ondisk_log_start":"258'600","created":281,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:01:24.325318+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:01:24.325318+0000","last_clean_scrub_stamp":"2023-09-23T21:01:24.325318+0000","objects_scrubbed":0,"log_size":9315,"log_dups_size":0,"ondisk_log_size":9315,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":76054528,"num_objects":1161,"num_object_clones":0,"num_object_copies":3483,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1161,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5769,"num_write_kb":221728,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1561,"num_bytes_recovered":102301696,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,5],"acting":[4,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.11","version":"79'161","reported_seq":553,"reported_epoch":321,"state":"active+clean","last_fresh":"2023-09-23T21:04:33.429209+0000","last_change":"2023-09-23T20:57:28.756796+0000","last_active":"2023-09-23T21:04:33.429209+0000","last_peered":"2023-09-23T21:04:33.429209+0000","last_clean":"2023-09-23T21:04:33.429209+0000","last_became_active":"2023-09-23T20:57:15.980606+0000","last_became_peered":"2023-09-23T20:57:15.980606+0000","last_unstale":"2023-09-23T21:04:33.429209+0000","last_undegraded":"2023-09-23T21:04:33.429209+0000","last_fullsized":"2023-09-23T21:04:33.429209+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:28.756634+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:28.756634+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:09:19.150791+0000","scrub_duration":0.0064925119999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":242,"reported_epoch":321,"state":"active+clean","last_fresh":"2023-09-23T21:04:33.411547+0000","last_change":"2023-09-23T20:56:45.992577+0000","last_active":"2023-09-23T21:04:33.411547+0000","last_peered":"2023-09-23T21:04:33.411547+0000","last_clean":"2023-09-23T21:04:33.411547+0000","last_became_active":"2023-09-23T20:56:45.992297+0000","last_became_peered":"2023-09-23T20:56:45.992297+0000","last_unstale":"2023-09-23T21:04:33.411547+0000","last_undegraded":"2023-09-23T21:04:33.411547+0000","last_fullsized":"2023-09-23T21:04:33.411547+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":205,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:00:45.547202+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,2],"acting":[4,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":288,"reported_epoch":320,"state":"active+clean","last_fresh":"2023-09-23T21:04:32.424623+0000","last_change":"2023-09-23T20:51:10.831262+0000","last_active":"2023-09-23T21:04:32.424623+0000","last_peered":"2023-09-23T21:04:32.424623+0000","last_clean":"2023-09-23T21:04:32.424623+0000","last_became_active":"2023-09-23T20:51:10.830931+0000","last_became_peered":"2023-09-23T20:51:10.830931+0000","last_unstale":"2023-09-23T21:04:32.424623+0000","last_undegraded":"2023-09-23T21:04:32.424623+0000","last_fullsized":"2023-09-23T21:04:32.424623+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:51:54.982505+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.19","version":"322'9742","reported_seq":12800,"reported_epoch":322,"state":"active+clean","last_fresh":"2023-09-23T21:04:42.061884+0000","last_change":"2023-09-23T21:04:32.417540+0000","last_active":"2023-09-23T21:04:42.061884+0000","last_peered":"2023-09-23T21:04:42.061884+0000","last_clean":"2023-09-23T21:04:42.061884+0000","last_became_active":"2023-09-23T21:04:32.402388+0000","last_became_peered":"2023-09-23T21:04:32.402388+0000","last_unstale":"2023-09-23T21:04:42.061884+0000","last_undegraded":"2023-09-23T21:04:42.061884+0000","last_fullsized":"2023-09-23T21:04:42.061884+0000","mapping_epoch":319,"log_start":"258'700","ondisk_log_start":"258'700","created":281,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:01:24.325318+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:01:24.325318+0000","last_clean_scrub_stamp":"2023-09-23T21:01:24.325318+0000","objects_scrubbed":0,"log_size":9042,"log_dups_size":0,"ondisk_log_size":9042,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":71106560,"num_objects":1085,"num_object_clones":0,"num_object_copies":3255,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1085,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5689,"num_write_kb":216768,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1471,"num_bytes_recovered":96403456,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.10","version":"79'161","reported_seq":559,"reported_epoch":321,"state":"active+clean","last_fresh":"2023-09-23T21:04:33.410327+0000","last_change":"2023-09-23T20:57:20.708003+0000","last_active":"2023-09-23T21:04:33.410327+0000","last_peered":"2023-09-23T21:04:33.410327+0000","last_clean":"2023-09-23T21:04:33.410327+0000","last_became_active":"2023-09-23T20:57:15.975302+0000","last_became_peered":"2023-09-23T20:57:15.975302+0000","last_unstale":"2023-09-23T21:04:33.410327+0000","last_undegraded":"2023-09-23T21:04:33.410327+0000","last_fullsized":"2023-09-23T21:04:33.410327+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:20.707939+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:20.707939+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:16:51.721994+0000","scrub_duration":0.0017733409999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":299,"reported_epoch":321,"state":"active+clean","last_fresh":"2023-09-23T21:04:33.429315+0000","last_change":"2023-09-23T20:57:44.979736+0000","last_active":"2023-09-23T21:04:33.429315+0000","last_peered":"2023-09-23T21:04:33.429315+0000","last_clean":"2023-09-23T21:04:33.429315+0000","last_became_active":"2023-09-23T20:57:44.979356+0000","last_became_peered":"2023-09-23T20:57:44.979356+0000","last_unstale":"2023-09-23T21:04:33.429315+0000","last_undegraded":"2023-09-23T21:04:33.429315+0000","last_fullsized":"2023-09-23T21:04:33.429315+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:14:43.848832+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":195,"reported_epoch":321,"state":"active+clean","last_fresh":"2023-09-23T21:04:33.410912+0000","last_change":"2023-09-23T20:54:31.887814+0000","last_active":"2023-09-23T21:04:33.410912+0000","last_peered":"2023-09-23T21:04:33.410912+0000","last_clean":"2023-09-23T21:04:33.410912+0000","last_became_active":"2023-09-23T20:54:31.887195+0000","last_became_peered":"2023-09-23T20:54:31.887195+0000","last_unstale":"2023-09-23T21:04:33.410912+0000","last_undegraded":"2023-09-23T21:04:33.410912+0000","last_fullsized":"2023-09-23T21:04:33.410912+0000","mapping_epoch":136,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:10:52.089553+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.13","version":"79'161","reported_seq":623,"reported_epoch":320,"state":"active+clean","last_fresh":"2023-09-23T21:04:32.428192+0000","last_change":"2023-09-23T20:57:48.262970+0000","last_active":"2023-09-23T21:04:32.428192+0000","last_peered":"2023-09-23T21:04:32.428192+0000","last_clean":"2023-09-23T21:04:32.428192+0000","last_became_active":"2023-09-23T20:57:48.262663+0000","last_became_peered":"2023-09-23T20:57:48.262663+0000","last_unstale":"2023-09-23T21:04:32.428192+0000","last_undegraded":"2023-09-23T21:04:32.428192+0000","last_fullsized":"2023-09-23T21:04:32.428192+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:36.420362+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:36.420362+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:28:28.613492+0000","scrub_duration":0.028838444000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":266,"reported_epoch":321,"state":"active+clean","last_fresh":"2023-09-23T21:04:33.411388+0000","last_change":"2023-09-23T20:56:45.991822+0000","last_active":"2023-09-23T21:04:33.411388+0000","last_peered":"2023-09-23T21:04:33.411388+0000","last_clean":"2023-09-23T21:04:33.411388+0000","last_became_active":"2023-09-23T20:56:45.991460+0000","last_became_peered":"2023-09-23T20:56:45.991460+0000","last_unstale":"2023-09-23T21:04:33.411388+0000","last_undegraded":"2023-09-23T21:04:33.411388+0000","last_fullsized":"2023-09-23T21:04:33.411388+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":205,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:20:37.503317+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":281,"reported_epoch":321,"state":"active+clean","last_fresh":"2023-09-23T21:04:33.429900+0000","last_change":"2023-09-23T20:51:10.840849+0000","last_active":"2023-09-23T21:04:33.429900+0000","last_peered":"2023-09-23T21:04:33.429900+0000","last_clean":"2023-09-23T21:04:33.429900+0000","last_became_active":"2023-09-23T20:51:10.840593+0000","last_became_peered":"2023-09-23T20:51:10.840593+0000","last_unstale":"2023-09-23T21:04:33.429900+0000","last_undegraded":"2023-09-23T21:04:33.429900+0000","last_fullsized":"2023-09-23T21:04:33.429900+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:30:52.297625+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.12","version":"79'161","reported_seq":559,"reported_epoch":321,"state":"active+clean","last_fresh":"2023-09-23T21:04:33.410292+0000","last_change":"2023-09-23T20:57:21.750765+0000","last_active":"2023-09-23T21:04:33.410292+0000","last_peered":"2023-09-23T21:04:33.410292+0000","last_clean":"2023-09-23T21:04:33.410292+0000","last_became_active":"2023-09-23T20:57:15.979195+0000","last_became_peered":"2023-09-23T20:57:15.979195+0000","last_unstale":"2023-09-23T21:04:33.410292+0000","last_undegraded":"2023-09-23T21:04:33.410292+0000","last_fullsized":"2023-09-23T21:04:33.410292+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:21.750680+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:21.750680+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:03:41.915962+0000","scrub_duration":0.0092391680000000007,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":313,"reported_epoch":320,"state":"active+clean","last_fresh":"2023-09-23T21:04:32.399014+0000","last_change":"2023-09-23T20:55:36.943969+0000","last_active":"2023-09-23T21:04:32.399014+0000","last_peered":"2023-09-23T21:04:32.399014+0000","last_clean":"2023-09-23T21:04:32.399014+0000","last_became_active":"2023-09-23T20:55:36.943580+0000","last_became_peered":"2023-09-23T20:55:36.943580+0000","last_unstale":"2023-09-23T21:04:32.399014+0000","last_undegraded":"2023-09-23T21:04:32.399014+0000","last_fullsized":"2023-09-23T21:04:32.399014+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:49:06.969396+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":111,"reported_epoch":320,"state":"active+clean","last_fresh":"2023-09-23T21:04:32.423500+0000","last_change":"2023-09-23T20:57:44.980498+0000","last_active":"2023-09-23T21:04:32.423500+0000","last_peered":"2023-09-23T21:04:32.423500+0000","last_clean":"2023-09-23T21:04:32.423500+0000","last_became_active":"2023-09-23T20:57:44.980182+0000","last_became_peered":"2023-09-23T20:57:44.980182+0000","last_unstale":"2023-09-23T21:04:32.423500+0000","last_undegraded":"2023-09-23T21:04:32.423500+0000","last_fullsized":"2023-09-23T21:04:32.423500+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:34:51.620544+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.4","version":"322'9893","reported_seq":9988,"reported_epoch":322,"state":"active+clean","last_fresh":"2023-09-23T21:04:42.060316+0000","last_change":"2023-09-23T21:04:12.976123+0000","last_active":"2023-09-23T21:04:42.060316+0000","last_peered":"2023-09-23T21:04:42.060316+0000","last_clean":"2023-09-23T21:04:42.060316+0000","last_became_active":"2023-09-23T21:03:30.256563+0000","last_became_peered":"2023-09-23T21:03:30.256563+0000","last_unstale":"2023-09-23T21:04:42.060316+0000","last_undegraded":"2023-09-23T21:04:42.060316+0000","last_fullsized":"2023-09-23T21:04:42.060316+0000","mapping_epoch":254,"log_start":"258'500","ondisk_log_start":"258'500","created":254,"last_epoch_clean":282,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:01:24.325318+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:01:24.325318+0000","last_clean_scrub_stamp":"2023-09-23T21:01:24.325318+0000","objects_scrubbed":0,"log_size":9393,"log_dups_size":0,"ondisk_log_size":9393,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":76972032,"num_objects":1174,"num_object_clones":0,"num_object_copies":3522,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1174,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5748,"num_write_kb":221536,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,2],"acting":[5,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.d","version":"79'161","reported_seq":604,"reported_epoch":321,"state":"active+clean","last_fresh":"2023-09-23T21:04:33.411869+0000","last_change":"2023-09-23T20:57:31.075213+0000","last_active":"2023-09-23T21:04:33.411869+0000","last_peered":"2023-09-23T21:04:33.411869+0000","last_clean":"2023-09-23T21:04:33.411869+0000","last_became_active":"2023-09-23T20:57:15.988188+0000","last_became_peered":"2023-09-23T20:57:15.988188+0000","last_unstale":"2023-09-23T21:04:33.411869+0000","last_undegraded":"2023-09-23T21:04:33.411869+0000","last_fullsized":"2023-09-23T21:04:33.411869+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:31.075098+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:31.075098+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:41:52.142362+0000","scrub_duration":0.010304610000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":253,"reported_epoch":321,"state":"active+clean","last_fresh":"2023-09-23T21:04:33.411661+0000","last_change":"2023-09-23T20:56:45.991841+0000","last_active":"2023-09-23T21:04:33.411661+0000","last_peered":"2023-09-23T21:04:33.411661+0000","last_clean":"2023-09-23T21:04:33.411661+0000","last_became_active":"2023-09-23T20:56:45.991491+0000","last_became_peered":"2023-09-23T20:56:45.991491+0000","last_unstale":"2023-09-23T21:04:33.411661+0000","last_undegraded":"2023-09-23T21:04:33.411661+0000","last_fullsized":"2023-09-23T21:04:33.411661+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":205,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:30:37.405319+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":212,"reported_epoch":321,"state":"active+clean","last_fresh":"2023-09-23T21:04:33.410131+0000","last_change":"2023-09-23T20:55:35.957768+0000","last_active":"2023-09-23T21:04:33.410131+0000","last_peered":"2023-09-23T21:04:33.410131+0000","last_clean":"2023-09-23T21:04:33.410131+0000","last_became_active":"2023-09-23T20:55:35.957421+0000","last_became_peered":"2023-09-23T20:55:35.957421+0000","last_unstale":"2023-09-23T21:04:33.410131+0000","last_undegraded":"2023-09-23T21:04:33.410131+0000","last_fullsized":"2023-09-23T21:04:33.410131+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:49:41.585961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"8.5","version":"321'9864","reported_seq":10060,"reported_epoch":321,"state":"active+clean","last_fresh":"2023-09-23T21:04:39.168836+0000","last_change":"2023-09-23T21:03:37.840743+0000","last_active":"2023-09-23T21:04:39.168836+0000","last_peered":"2023-09-23T21:04:39.168836+0000","last_clean":"2023-09-23T21:04:39.168836+0000","last_became_active":"2023-09-23T21:03:30.251381+0000","last_became_peered":"2023-09-23T21:03:30.251381+0000","last_unstale":"2023-09-23T21:04:39.168836+0000","last_undegraded":"2023-09-23T21:04:39.168836+0000","last_fullsized":"2023-09-23T21:04:39.168836+0000","mapping_epoch":254,"log_start":"258'1000","ondisk_log_start":"258'1000","created":254,"last_epoch_clean":282,"parent":"0.0","parent_split_bits":0,"last_scrub":"284'8610","last_scrub_stamp":"2023-09-23T21:03:37.840682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:01:24.325318+0000","last_clean_scrub_stamp":"2023-09-23T21:03:37.840682+0000","objects_scrubbed":2517,"log_size":8864,"log_dups_size":0,"ondisk_log_size":8864,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:00:41.745968+0000","scrub_duration":0.31954776800000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":82444288,"num_objects":1258,"num_object_clones":0,"num_object_copies":3774,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1258,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1259,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.c","version":"79'161","reported_seq":624,"reported_epoch":320,"state":"active+clean","last_fresh":"2023-09-23T21:04:32.424449+0000","last_change":"2023-09-23T20:57:22.041071+0000","last_active":"2023-09-23T21:04:32.424449+0000","last_peered":"2023-09-23T21:04:32.424449+0000","last_clean":"2023-09-23T21:04:32.424449+0000","last_became_active":"2023-09-23T20:57:15.973204+0000","last_became_peered":"2023-09-23T20:57:15.973204+0000","last_unstale":"2023-09-23T21:04:32.424449+0000","last_undegraded":"2023-09-23T21:04:32.424449+0000","last_fullsized":"2023-09-23T21:04:32.424449+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:22.040907+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:22.040907+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:47:02.019978+0000","scrub_duration":0.013021176000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":240,"reported_epoch":321,"state":"active+clean","last_fresh":"2023-09-23T21:04:33.412202+0000","last_change":"2023-09-23T20:56:45.991821+0000","last_active":"2023-09-23T21:04:33.412202+0000","last_peered":"2023-09-23T21:04:33.412202+0000","last_clean":"2023-09-23T21:04:33.412202+0000","last_became_active":"2023-09-23T20:56:45.991447+0000","last_became_peered":"2023-09-23T20:56:45.991447+0000","last_unstale":"2023-09-23T21:04:33.412202+0000","last_undegraded":"2023-09-23T21:04:33.412202+0000","last_fullsized":"2023-09-23T21:04:33.412202+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":205,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:14:25.525222+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":243,"reported_epoch":321,"state":"active+clean","last_fresh":"2023-09-23T21:04:33.411709+0000","last_change":"2023-09-23T20:57:44.982059+0000","last_active":"2023-09-23T21:04:33.411709+0000","last_peered":"2023-09-23T21:04:33.411709+0000","last_clean":"2023-09-23T21:04:33.411709+0000","last_became_active":"2023-09-23T20:57:44.981742+0000","last_became_peered":"2023-09-23T20:57:44.981742+0000","last_unstale":"2023-09-23T21:04:33.411709+0000","last_undegraded":"2023-09-23T21:04:33.411709+0000","last_fullsized":"2023-09-23T21:04:33.411709+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:41:24.042916+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.6","version":"322'10182","reported_seq":10332,"reported_epoch":322,"state":"active+clean","last_fresh":"2023-09-23T21:04:42.161230+0000","last_change":"2023-09-23T21:03:38.793592+0000","last_active":"2023-09-23T21:04:42.161230+0000","last_peered":"2023-09-23T21:04:42.161230+0000","last_clean":"2023-09-23T21:04:42.161230+0000","last_became_active":"2023-09-23T21:03:30.271638+0000","last_became_peered":"2023-09-23T21:03:30.271638+0000","last_unstale":"2023-09-23T21:04:42.161230+0000","last_undegraded":"2023-09-23T21:04:42.161230+0000","last_fullsized":"2023-09-23T21:04:42.161230+0000","mapping_epoch":254,"log_start":"258'600","ondisk_log_start":"258'600","created":254,"last_epoch_clean":282,"parent":"0.0","parent_split_bits":0,"last_scrub":"284'8804","last_scrub_stamp":"2023-09-23T21:03:38.793436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:01:24.325318+0000","last_clean_scrub_stamp":"2023-09-23T21:03:38.793436+0000","objects_scrubbed":2552,"log_size":9582,"log_dups_size":0,"ondisk_log_size":9582,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:56:02.183614+0000","scrub_duration":0.31348443599999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":76939264,"num_objects":1174,"num_object_clones":0,"num_object_copies":3522,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1174,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1378,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,7],"acting":[0,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.f","version":"79'161","reported_seq":625,"reported_epoch":320,"state":"active+clean","last_fresh":"2023-09-23T21:04:32.399831+0000","last_change":"2023-09-23T20:57:27.880841+0000","last_active":"2023-09-23T21:04:32.399831+0000","last_peered":"2023-09-23T21:04:32.399831+0000","last_clean":"2023-09-23T21:04:32.399831+0000","last_became_active":"2023-09-23T20:57:15.970925+0000","last_became_peered":"2023-09-23T20:57:15.970925+0000","last_unstale":"2023-09-23T21:04:32.399831+0000","last_undegraded":"2023-09-23T21:04:32.399831+0000","last_fullsized":"2023-09-23T21:04:32.399831+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:27.880776+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:27.880776+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:15:13.438014+0000","scrub_duration":0.0014004899999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":226,"reported_epoch":321,"state":"active+clean","last_fresh":"2023-09-23T21:04:33.410830+0000","last_change":"2023-09-23T20:56:45.989153+0000","last_active":"2023-09-23T21:04:33.410830+0000","last_peered":"2023-09-23T21:04:33.410830+0000","last_clean":"2023-09-23T21:04:33.410830+0000","last_became_active":"2023-09-23T20:56:45.988800+0000","last_became_peered":"2023-09-23T20:56:45.988800+0000","last_unstale":"2023-09-23T21:04:33.410830+0000","last_undegraded":"2023-09-23T21:04:33.410830+0000","last_fullsized":"2023-09-23T21:04:33.410830+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":205,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:04:41.130194+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":260,"reported_epoch":320,"state":"active+clean","last_fresh":"2023-09-23T21:04:32.400854+0000","last_change":"2023-09-23T20:55:35.965427+0000","last_active":"2023-09-23T21:04:32.400854+0000","last_peered":"2023-09-23T21:04:32.400854+0000","last_clean":"2023-09-23T21:04:32.400854+0000","last_became_active":"2023-09-23T20:55:35.963969+0000","last_became_peered":"2023-09-23T20:55:35.963969+0000","last_unstale":"2023-09-23T21:04:32.400854+0000","last_undegraded":"2023-09-23T21:04:32.400854+0000","last_fullsized":"2023-09-23T21:04:32.400854+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:38:40.612083+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.7","version":"321'9708","reported_seq":9857,"reported_epoch":321,"state":"active+clean","last_fresh":"2023-09-23T21:04:39.591235+0000","last_change":"2023-09-23T21:03:41.677499+0000","last_active":"2023-09-23T21:04:39.591235+0000","last_peered":"2023-09-23T21:04:39.591235+0000","last_clean":"2023-09-23T21:04:39.591235+0000","last_became_active":"2023-09-23T21:03:30.267051+0000","last_became_peered":"2023-09-23T21:03:30.267051+0000","last_unstale":"2023-09-23T21:04:39.591235+0000","last_undegraded":"2023-09-23T21:04:39.591235+0000","last_fullsized":"2023-09-23T21:04:39.591235+0000","mapping_epoch":254,"log_start":"258'1300","ondisk_log_start":"258'1300","created":254,"last_epoch_clean":282,"parent":"0.0","parent_split_bits":0,"last_scrub":"288'8477","last_scrub_stamp":"2023-09-23T21:03:41.677318+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:01:24.325318+0000","last_clean_scrub_stamp":"2023-09-23T21:03:41.677318+0000","objects_scrubbed":2477,"log_size":8408,"log_dups_size":0,"ondisk_log_size":8408,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:23:04.253957+0000","scrub_duration":0.50562591800000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":81330176,"num_objects":1241,"num_object_clones":0,"num_object_copies":3723,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1241,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1236,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.e","version":"79'161","reported_seq":533,"reported_epoch":321,"state":"active+clean","last_fresh":"2023-09-23T21:04:33.412136+0000","last_change":"2023-09-23T20:57:34.132738+0000","last_active":"2023-09-23T21:04:33.412136+0000","last_peered":"2023-09-23T21:04:33.412136+0000","last_clean":"2023-09-23T21:04:33.412136+0000","last_became_active":"2023-09-23T20:57:15.974318+0000","last_became_peered":"2023-09-23T20:57:15.974318+0000","last_unstale":"2023-09-23T21:04:33.412136+0000","last_undegraded":"2023-09-23T21:04:33.412136+0000","last_fullsized":"2023-09-23T21:04:33.412136+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:34.132660+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:34.132660+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:07:07.739438+0000","scrub_duration":0.005229694,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":212,"reported_epoch":321,"state":"active+clean","last_fresh":"2023-09-23T21:04:33.410879+0000","last_change":"2023-09-23T20:55:35.962005+0000","last_active":"2023-09-23T21:04:33.410879+0000","last_peered":"2023-09-23T21:04:33.410879+0000","last_clean":"2023-09-23T21:04:33.410879+0000","last_became_active":"2023-09-23T20:55:35.961695+0000","last_became_peered":"2023-09-23T20:55:35.961695+0000","last_unstale":"2023-09-23T21:04:33.410879+0000","last_undegraded":"2023-09-23T21:04:33.410879+0000","last_fullsized":"2023-09-23T21:04:33.410879+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:28:22.876201+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":312,"reported_epoch":321,"state":"active+clean","last_fresh":"2023-09-23T21:04:33.428675+0000","last_change":"2023-09-23T20:54:13.928715+0000","last_active":"2023-09-23T21:04:33.428675+0000","last_peered":"2023-09-23T21:04:33.428675+0000","last_clean":"2023-09-23T21:04:33.428675+0000","last_became_active":"2023-09-23T20:54:13.928200+0000","last_became_peered":"2023-09-23T20:54:13.928200+0000","last_unstale":"2023-09-23T21:04:33.428675+0000","last_undegraded":"2023-09-23T21:04:33.428675+0000","last_fullsized":"2023-09-23T21:04:33.428675+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:41:16.927513+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.0","version":"321'9796","reported_seq":9946,"reported_epoch":321,"state":"active+clean","last_fresh":"2023-09-23T21:04:39.819519+0000","last_change":"2023-09-23T21:03:39.283021+0000","last_active":"2023-09-23T21:04:39.819519+0000","last_peered":"2023-09-23T21:04:39.819519+0000","last_clean":"2023-09-23T21:04:39.819519+0000","last_became_active":"2023-09-23T21:03:30.271894+0000","last_became_peered":"2023-09-23T21:03:30.271894+0000","last_unstale":"2023-09-23T21:04:39.819519+0000","last_undegraded":"2023-09-23T21:04:39.819519+0000","last_fullsized":"2023-09-23T21:04:39.819519+0000","mapping_epoch":254,"log_start":"258'1700","ondisk_log_start":"258'1700","created":254,"last_epoch_clean":282,"parent":"0.0","parent_split_bits":0,"last_scrub":"284'8499","last_scrub_stamp":"2023-09-23T21:03:39.282869+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:01:24.325318+0000","last_clean_scrub_stamp":"2023-09-23T21:03:39.282869+0000","objects_scrubbed":2515,"log_size":8096,"log_dups_size":0,"ondisk_log_size":8096,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:51:43.305878+0000","scrub_duration":0.31450518900000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":79822848,"num_objects":1218,"num_object_clones":0,"num_object_copies":3654,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1218,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1297,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,1],"acting":[3,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.9","version":"79'161","reported_seq":696,"reported_epoch":320,"state":"active+clean","last_fresh":"2023-09-23T21:04:32.423231+0000","last_change":"2023-09-23T20:56:46.921939+0000","last_active":"2023-09-23T21:04:32.423231+0000","last_peered":"2023-09-23T21:04:32.423231+0000","last_clean":"2023-09-23T21:04:32.423231+0000","last_became_active":"2023-09-23T20:56:45.989908+0000","last_became_peered":"2023-09-23T20:56:45.989908+0000","last_unstale":"2023-09-23T21:04:32.423231+0000","last_undegraded":"2023-09-23T21:04:32.423231+0000","last_fullsized":"2023-09-23T21:04:32.423231+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":205,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:56:46.921739+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:56:46.921739+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:20:42.635110+0000","scrub_duration":0.0071954109999999996,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":313,"reported_epoch":321,"state":"active+clean","last_fresh":"2023-09-23T21:04:33.418896+0000","last_change":"2023-09-23T20:54:13.961788+0000","last_active":"2023-09-23T21:04:33.418896+0000","last_peered":"2023-09-23T21:04:33.418896+0000","last_clean":"2023-09-23T21:04:33.418896+0000","last_became_active":"2023-09-23T20:54:13.940937+0000","last_became_peered":"2023-09-23T20:54:13.940937+0000","last_unstale":"2023-09-23T21:04:33.418896+0000","last_undegraded":"2023-09-23T21:04:33.418896+0000","last_fullsized":"2023-09-23T21:04:33.418896+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:11:56.785299+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":216,"reported_epoch":321,"state":"active+clean","last_fresh":"2023-09-23T21:04:33.411979+0000","last_change":"2023-09-23T20:55:35.949670+0000","last_active":"2023-09-23T21:04:33.411979+0000","last_peered":"2023-09-23T21:04:33.411979+0000","last_clean":"2023-09-23T21:04:33.411979+0000","last_became_active":"2023-09-23T20:55:35.949140+0000","last_became_peered":"2023-09-23T20:55:35.949140+0000","last_unstale":"2023-09-23T21:04:33.411979+0000","last_undegraded":"2023-09-23T21:04:33.411979+0000","last_fullsized":"2023-09-23T21:04:33.411979+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:55:59.340303+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.1","version":"321'10003","reported_seq":10150,"reported_epoch":321,"state":"active+clean","last_fresh":"2023-09-23T21:04:39.593725+0000","last_change":"2023-09-23T21:03:38.452701+0000","last_active":"2023-09-23T21:04:39.593725+0000","last_peered":"2023-09-23T21:04:39.593725+0000","last_clean":"2023-09-23T21:04:39.593725+0000","last_became_active":"2023-09-23T21:03:30.268996+0000","last_became_peered":"2023-09-23T21:03:30.268996+0000","last_unstale":"2023-09-23T21:04:39.593725+0000","last_undegraded":"2023-09-23T21:04:39.593725+0000","last_fullsized":"2023-09-23T21:04:39.593725+0000","mapping_epoch":254,"log_start":"258'1600","ondisk_log_start":"258'1600","created":254,"last_epoch_clean":282,"parent":"0.0","parent_split_bits":0,"last_scrub":"284'8731","last_scrub_stamp":"2023-09-23T21:03:38.452464+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:01:24.325318+0000","last_clean_scrub_stamp":"2023-09-23T21:03:38.452464+0000","objects_scrubbed":2585,"log_size":8403,"log_dups_size":0,"ondisk_log_size":8403,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:01:42.038382+0000","scrub_duration":0.313423428,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":86048768,"num_objects":1313,"num_object_clones":0,"num_object_copies":3939,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1313,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1272,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,2],"acting":[1,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.8","version":"79'161","reported_seq":565,"reported_epoch":321,"state":"active+clean","last_fresh":"2023-09-23T21:04:33.410221+0000","last_change":"2023-09-23T20:56:57.505737+0000","last_active":"2023-09-23T21:04:33.410221+0000","last_peered":"2023-09-23T21:04:33.410221+0000","last_clean":"2023-09-23T21:04:33.410221+0000","last_became_active":"2023-09-23T20:56:04.982345+0000","last_became_peered":"2023-09-23T20:56:04.982345+0000","last_unstale":"2023-09-23T21:04:33.410221+0000","last_undegraded":"2023-09-23T21:04:33.410221+0000","last_fullsized":"2023-09-23T21:04:33.410221+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:56:57.505663+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:56:57.505663+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:34:30.109724+0000","scrub_duration":0.026284108,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":283,"reported_epoch":320,"state":"active+clean","last_fresh":"2023-09-23T21:04:32.401139+0000","last_change":"2023-09-23T20:55:35.959942+0000","last_active":"2023-09-23T21:04:32.401139+0000","last_peered":"2023-09-23T21:04:32.401139+0000","last_clean":"2023-09-23T21:04:32.401139+0000","last_became_active":"2023-09-23T20:55:35.957071+0000","last_became_peered":"2023-09-23T20:55:35.957071+0000","last_unstale":"2023-09-23T21:04:32.401139+0000","last_undegraded":"2023-09-23T21:04:32.401139+0000","last_fullsized":"2023-09-23T21:04:32.401139+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:44:30.308036+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":199,"reported_epoch":321,"state":"active+clean","last_fresh":"2023-09-23T21:04:33.412197+0000","last_change":"2023-09-23T20:54:13.951459+0000","last_active":"2023-09-23T21:04:33.412197+0000","last_peered":"2023-09-23T21:04:33.412197+0000","last_clean":"2023-09-23T21:04:33.412197+0000","last_became_active":"2023-09-23T20:54:13.948409+0000","last_became_peered":"2023-09-23T20:54:13.948409+0000","last_unstale":"2023-09-23T21:04:33.412197+0000","last_undegraded":"2023-09-23T21:04:33.412197+0000","last_fullsized":"2023-09-23T21:04:33.412197+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:38:10.148061+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.2","version":"322'10062","reported_seq":10173,"reported_epoch":322,"state":"active+clean","last_fresh":"2023-09-23T21:04:42.271755+0000","last_change":"2023-09-23T21:04:21.253437+0000","last_active":"2023-09-23T21:04:42.271755+0000","last_peered":"2023-09-23T21:04:42.271755+0000","last_clean":"2023-09-23T21:04:42.271755+0000","last_became_active":"2023-09-23T21:03:30.254079+0000","last_became_peered":"2023-09-23T21:03:30.254079+0000","last_unstale":"2023-09-23T21:04:42.271755+0000","last_undegraded":"2023-09-23T21:04:42.271755+0000","last_fullsized":"2023-09-23T21:04:42.271755+0000","mapping_epoch":254,"log_start":"258'1700","ondisk_log_start":"258'1700","created":254,"last_epoch_clean":282,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:01:24.325318+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:01:24.325318+0000","last_clean_scrub_stamp":"2023-09-23T21:01:24.325318+0000","objects_scrubbed":0,"log_size":8362,"log_dups_size":0,"ondisk_log_size":8362,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":77037568,"num_objects":1175,"num_object_clones":0,"num_object_copies":3525,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1175,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5852,"num_write_kb":224896,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,5],"acting":[4,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.b","version":"79'161","reported_seq":606,"reported_epoch":320,"state":"active+clean","last_fresh":"2023-09-23T21:04:32.428278+0000","last_change":"2023-09-23T20:57:27.320172+0000","last_active":"2023-09-23T21:04:32.428278+0000","last_peered":"2023-09-23T21:04:32.428278+0000","last_clean":"2023-09-23T21:04:32.428278+0000","last_became_active":"2023-09-23T20:57:16.009630+0000","last_became_peered":"2023-09-23T20:57:16.009630+0000","last_unstale":"2023-09-23T21:04:32.428278+0000","last_undegraded":"2023-09-23T21:04:32.428278+0000","last_fullsized":"2023-09-23T21:04:32.428278+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:27.320022+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:27.320022+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:31:35.221558+0000","scrub_duration":0.0043894679999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,5],"acting":[1,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":199,"reported_epoch":321,"state":"active+clean","last_fresh":"2023-09-23T21:04:33.411483+0000","last_change":"2023-09-23T20:54:13.950254+0000","last_active":"2023-09-23T21:04:33.411483+0000","last_peered":"2023-09-23T21:04:33.411483+0000","last_clean":"2023-09-23T21:04:33.411483+0000","last_became_active":"2023-09-23T20:54:13.949934+0000","last_became_peered":"2023-09-23T20:54:13.949934+0000","last_unstale":"2023-09-23T21:04:33.411483+0000","last_undegraded":"2023-09-23T21:04:33.411483+0000","last_fullsized":"2023-09-23T21:04:33.411483+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:30:32.847167+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":216,"reported_epoch":321,"state":"active+clean","last_fresh":"2023-09-23T21:04:33.412216+0000","last_change":"2023-09-23T20:55:35.956153+0000","last_active":"2023-09-23T21:04:33.412216+0000","last_peered":"2023-09-23T21:04:33.412216+0000","last_clean":"2023-09-23T21:04:33.412216+0000","last_became_active":"2023-09-23T20:55:35.955630+0000","last_became_peered":"2023-09-23T20:55:35.955630+0000","last_unstale":"2023-09-23T21:04:33.412216+0000","last_undegraded":"2023-09-23T21:04:33.412216+0000","last_fullsized":"2023-09-23T21:04:33.412216+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:45:20.282929+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.3","version":"321'9534","reported_seq":9666,"reported_epoch":321,"state":"active+clean","last_fresh":"2023-09-23T21:04:39.577184+0000","last_change":"2023-09-23T21:04:25.370748+0000","last_active":"2023-09-23T21:04:39.577184+0000","last_peered":"2023-09-23T21:04:39.577184+0000","last_clean":"2023-09-23T21:04:39.577184+0000","last_became_active":"2023-09-23T21:03:30.287720+0000","last_became_peered":"2023-09-23T21:03:30.287720+0000","last_unstale":"2023-09-23T21:04:39.577184+0000","last_undegraded":"2023-09-23T21:04:39.577184+0000","last_fullsized":"2023-09-23T21:04:39.577184+0000","mapping_epoch":254,"log_start":"258'1200","ondisk_log_start":"258'1200","created":254,"last_epoch_clean":282,"parent":"0.0","parent_split_bits":0,"last_scrub":"315'9153","last_scrub_stamp":"2023-09-23T21:04:25.370609+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:01:24.325318+0000","last_clean_scrub_stamp":"2023-09-23T21:04:25.370609+0000","objects_scrubbed":1550,"log_size":8334,"log_dups_size":0,"ondisk_log_size":8334,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:46:54.982966+0000","scrub_duration":0.20200885099999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":76611584,"num_objects":1169,"num_object_clones":0,"num_object_copies":3507,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1169,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":381,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.a","version":"79'161","reported_seq":565,"reported_epoch":321,"state":"active+clean","last_fresh":"2023-09-23T21:04:33.410571+0000","last_change":"2023-09-23T20:56:44.487778+0000","last_active":"2023-09-23T21:04:33.410571+0000","last_peered":"2023-09-23T21:04:33.410571+0000","last_clean":"2023-09-23T21:04:33.410571+0000","last_became_active":"2023-09-23T20:56:05.006740+0000","last_became_peered":"2023-09-23T20:56:05.006740+0000","last_unstale":"2023-09-23T21:04:33.410571+0000","last_undegraded":"2023-09-23T21:04:33.410571+0000","last_fullsized":"2023-09-23T21:04:33.410571+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:56:44.487678+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:56:44.487678+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:04:50.086749+0000","scrub_duration":0.0017408899999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":112,"reported_epoch":320,"state":"active+clean","last_fresh":"2023-09-23T21:04:32.423794+0000","last_change":"2023-09-23T20:57:44.982513+0000","last_active":"2023-09-23T21:04:32.423794+0000","last_peered":"2023-09-23T21:04:32.423794+0000","last_clean":"2023-09-23T21:04:32.423794+0000","last_became_active":"2023-09-23T20:57:44.982112+0000","last_became_peered":"2023-09-23T20:57:44.982112+0000","last_unstale":"2023-09-23T21:04:32.423794+0000","last_undegraded":"2023-09-23T21:04:32.423794+0000","last_fullsized":"2023-09-23T21:04:32.423794+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:09:05.630041+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":293,"reported_epoch":320,"state":"active+clean","last_fresh":"2023-09-23T21:04:32.428064+0000","last_change":"2023-09-23T20:54:13.940748+0000","last_active":"2023-09-23T21:04:32.428064+0000","last_peered":"2023-09-23T21:04:32.428064+0000","last_clean":"2023-09-23T21:04:32.428064+0000","last_became_active":"2023-09-23T20:54:13.940419+0000","last_became_peered":"2023-09-23T20:54:13.940419+0000","last_unstale":"2023-09-23T21:04:32.428064+0000","last_undegraded":"2023-09-23T21:04:32.428064+0000","last_fullsized":"2023-09-23T21:04:32.428064+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:15:01.923816+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.e","version":"322'11720","reported_seq":11797,"reported_epoch":322,"state":"active+clean","last_fresh":"2023-09-23T21:04:42.185620+0000","last_change":"2023-09-23T21:01:25.430111+0000","last_active":"2023-09-23T21:04:42.185620+0000","last_peered":"2023-09-23T21:04:42.185620+0000","last_clean":"2023-09-23T21:04:42.185620+0000","last_became_active":"2023-09-23T21:01:25.429811+0000","last_became_peered":"2023-09-23T21:01:25.429811+0000","last_unstale":"2023-09-23T21:04:42.185620+0000","last_undegraded":"2023-09-23T21:04:42.185620+0000","last_fullsized":"2023-09-23T21:04:42.185620+0000","mapping_epoch":254,"log_start":"268'4888","ondisk_log_start":"268'4888","created":254,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:01:24.325318+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:01:24.325318+0000","last_clean_scrub_stamp":"2023-09-23T21:01:24.325318+0000","objects_scrubbed":0,"log_size":6832,"log_dups_size":0,"ondisk_log_size":6832,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:18:01.918758+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":152174592,"num_objects":2322,"num_object_clones":0,"num_object_copies":6966,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2322,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11720,"num_write_kb":449344,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.7","version":"79'161","reported_seq":626,"reported_epoch":320,"state":"active+clean","last_fresh":"2023-09-23T21:04:32.400070+0000","last_change":"2023-09-23T20:57:48.243572+0000","last_active":"2023-09-23T21:04:32.400070+0000","last_peered":"2023-09-23T21:04:32.400070+0000","last_clean":"2023-09-23T21:04:32.400070+0000","last_became_active":"2023-09-23T20:57:48.243229+0000","last_became_peered":"2023-09-23T20:57:48.243229+0000","last_unstale":"2023-09-23T21:04:32.400070+0000","last_undegraded":"2023-09-23T21:04:32.400070+0000","last_fullsized":"2023-09-23T21:04:32.400070+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":3,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:25.916867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:25.916867+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:23:19.385831+0000","scrub_duration":0.001665323,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":299,"reported_epoch":321,"state":"active+clean","last_fresh":"2023-09-23T21:04:33.413065+0000","last_change":"2023-09-23T20:50:05.867130+0000","last_active":"2023-09-23T21:04:33.413065+0000","last_peered":"2023-09-23T21:04:33.413065+0000","last_clean":"2023-09-23T21:04:33.413065+0000","last_became_active":"2023-09-23T20:50:05.866758+0000","last_became_peered":"2023-09-23T20:50:05.866758+0000","last_unstale":"2023-09-23T21:04:33.413065+0000","last_undegraded":"2023-09-23T21:04:33.413065+0000","last_fullsized":"2023-09-23T21:04:33.413065+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:55:07.548209+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":195,"reported_epoch":321,"state":"active+clean","last_fresh":"2023-09-23T21:04:33.410989+0000","last_change":"2023-09-23T20:54:31.887604+0000","last_active":"2023-09-23T21:04:33.410989+0000","last_peered":"2023-09-23T21:04:33.410989+0000","last_clean":"2023-09-23T21:04:33.410989+0000","last_became_active":"2023-09-23T20:54:31.886752+0000","last_became_peered":"2023-09-23T20:54:31.886752+0000","last_unstale":"2023-09-23T21:04:33.410989+0000","last_undegraded":"2023-09-23T21:04:33.410989+0000","last_fullsized":"2023-09-23T21:04:33.410989+0000","mapping_epoch":136,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:13:58.872176+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":297,"reported_epoch":321,"state":"active+clean","last_fresh":"2023-09-23T21:04:33.416597+0000","last_change":"2023-09-23T20:54:31.899082+0000","last_active":"2023-09-23T21:04:33.416597+0000","last_peered":"2023-09-23T21:04:33.416597+0000","last_clean":"2023-09-23T21:04:33.416597+0000","last_became_active":"2023-09-23T20:54:31.898521+0000","last_became_peered":"2023-09-23T20:54:31.898521+0000","last_unstale":"2023-09-23T21:04:33.416597+0000","last_undegraded":"2023-09-23T21:04:33.416597+0000","last_fullsized":"2023-09-23T21:04:33.416597+0000","mapping_epoch":136,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:25:36.059711+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.8","version":"322'9913","reported_seq":10035,"reported_epoch":322,"state":"active+clean","last_fresh":"2023-09-23T21:04:41.978017+0000","last_change":"2023-09-23T21:04:31.416444+0000","last_active":"2023-09-23T21:04:41.978017+0000","last_peered":"2023-09-23T21:04:41.978017+0000","last_clean":"2023-09-23T21:04:41.978017+0000","last_became_active":"2023-09-23T21:03:30.259253+0000","last_became_peered":"2023-09-23T21:03:30.259253+0000","last_unstale":"2023-09-23T21:04:41.978017+0000","last_undegraded":"2023-09-23T21:04:41.978017+0000","last_fullsized":"2023-09-23T21:04:41.978017+0000","mapping_epoch":254,"log_start":"258'600","ondisk_log_start":"258'600","created":254,"last_epoch_clean":282,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:01:24.325318+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:01:24.325318+0000","last_clean_scrub_stamp":"2023-09-23T21:01:24.325318+0000","objects_scrubbed":649,"log_size":9313,"log_dups_size":0,"ondisk_log_size":9313,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":76185600,"num_objects":1162,"num_object_clones":0,"num_object_copies":3486,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1162,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5766,"num_write_kb":221728,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[5,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.1","version":"79'161","reported_seq":585,"reported_epoch":321,"state":"active+clean","last_fresh":"2023-09-23T21:04:33.410223+0000","last_change":"2023-09-23T20:57:19.669346+0000","last_active":"2023-09-23T21:04:33.410223+0000","last_peered":"2023-09-23T21:04:33.410223+0000","last_clean":"2023-09-23T21:04:33.410223+0000","last_became_active":"2023-09-23T20:57:08.246046+0000","last_became_peered":"2023-09-23T20:57:08.246046+0000","last_unstale":"2023-09-23T21:04:33.410223+0000","last_undegraded":"2023-09-23T21:04:33.410223+0000","last_fullsized":"2023-09-23T21:04:33.410223+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:19.669259+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:19.669259+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:14:03.435531+0000","scrub_duration":0.0045759559999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":212,"reported_epoch":321,"state":"active+clean","last_fresh":"2023-09-23T21:04:33.410401+0000","last_change":"2023-09-23T20:55:35.959514+0000","last_active":"2023-09-23T21:04:33.410401+0000","last_peered":"2023-09-23T21:04:33.410401+0000","last_clean":"2023-09-23T21:04:33.410401+0000","last_became_active":"2023-09-23T20:55:35.959244+0000","last_became_peered":"2023-09-23T20:55:35.959244+0000","last_unstale":"2023-09-23T21:04:33.410401+0000","last_undegraded":"2023-09-23T21:04:33.410401+0000","last_fullsized":"2023-09-23T21:04:33.410401+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:47:32.665316+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":216,"reported_epoch":321,"state":"active+clean","last_fresh":"2023-09-23T21:04:33.411845+0000","last_change":"2023-09-23T20:55:35.949000+0000","last_active":"2023-09-23T21:04:33.411845+0000","last_peered":"2023-09-23T21:04:33.411845+0000","last_clean":"2023-09-23T21:04:33.411845+0000","last_became_active":"2023-09-23T20:55:35.948410+0000","last_became_peered":"2023-09-23T20:55:35.948410+0000","last_unstale":"2023-09-23T21:04:33.411845+0000","last_undegraded":"2023-09-23T21:04:33.411845+0000","last_fullsized":"2023-09-23T21:04:33.411845+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:26:50.477289+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":291,"reported_epoch":321,"state":"active+clean","last_fresh":"2023-09-23T21:04:33.412847+0000","last_change":"2023-09-23T20:51:10.829889+0000","last_active":"2023-09-23T21:04:33.412847+0000","last_peered":"2023-09-23T21:04:33.412847+0000","last_clean":"2023-09-23T21:04:33.412847+0000","last_became_active":"2023-09-23T20:51:10.829533+0000","last_became_peered":"2023-09-23T20:51:10.829533+0000","last_unstale":"2023-09-23T21:04:33.412847+0000","last_undegraded":"2023-09-23T21:04:33.412847+0000","last_fullsized":"2023-09-23T21:04:33.412847+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:43:04.603460+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.b","version":"321'11547","reported_seq":11624,"reported_epoch":321,"state":"active+clean","last_fresh":"2023-09-23T21:04:39.912106+0000","last_change":"2023-09-23T21:01:25.340569+0000","last_active":"2023-09-23T21:04:39.912106+0000","last_peered":"2023-09-23T21:04:39.912106+0000","last_clean":"2023-09-23T21:04:39.912106+0000","last_became_active":"2023-09-23T21:01:25.340209+0000","last_became_peered":"2023-09-23T21:01:25.340209+0000","last_unstale":"2023-09-23T21:04:39.912106+0000","last_undegraded":"2023-09-23T21:04:39.912106+0000","last_fullsized":"2023-09-23T21:04:39.912106+0000","mapping_epoch":254,"log_start":"271'5600","ondisk_log_start":"271'5600","created":254,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:01:24.325318+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:01:24.325318+0000","last_clean_scrub_stamp":"2023-09-23T21:01:24.325318+0000","objects_scrubbed":0,"log_size":5947,"log_dups_size":0,"ondisk_log_size":5947,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:53:59.553810+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":163643392,"num_objects":2497,"num_object_clones":0,"num_object_copies":7491,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2497,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11547,"num_write_kb":449408,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.2","version":"79'161","reported_seq":554,"reported_epoch":321,"state":"active+clean","last_fresh":"2023-09-23T21:04:33.429220+0000","last_change":"2023-09-23T20:57:24.802586+0000","last_active":"2023-09-23T21:04:33.429220+0000","last_peered":"2023-09-23T21:04:33.429220+0000","last_clean":"2023-09-23T21:04:33.429220+0000","last_became_active":"2023-09-23T20:57:08.236114+0000","last_became_peered":"2023-09-23T20:57:08.236114+0000","last_unstale":"2023-09-23T21:04:33.429220+0000","last_undegraded":"2023-09-23T21:04:33.429220+0000","last_fullsized":"2023-09-23T21:04:33.429220+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:24.802451+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:24.802451+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:06:22.309980+0000","scrub_duration":0.0090861250000000005,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,4],"acting":[0,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":310,"reported_epoch":321,"state":"active+clean","last_fresh":"2023-09-23T21:04:33.418448+0000","last_change":"2023-09-23T20:51:10.838054+0000","last_active":"2023-09-23T21:04:33.418448+0000","last_peered":"2023-09-23T21:04:33.418448+0000","last_clean":"2023-09-23T21:04:33.418448+0000","last_became_active":"2023-09-23T20:51:10.837750+0000","last_became_peered":"2023-09-23T20:51:10.837750+0000","last_unstale":"2023-09-23T21:04:33.418448+0000","last_undegraded":"2023-09-23T21:04:33.418448+0000","last_fullsized":"2023-09-23T21:04:33.418448+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:15:51.648085+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":291,"reported_epoch":320,"state":"active+clean","last_fresh":"2023-09-23T21:04:32.426372+0000","last_change":"2023-09-23T20:55:35.951146+0000","last_active":"2023-09-23T21:04:32.426372+0000","last_peered":"2023-09-23T21:04:32.426372+0000","last_clean":"2023-09-23T21:04:32.426372+0000","last_became_active":"2023-09-23T20:55:35.950874+0000","last_became_peered":"2023-09-23T20:55:35.950874+0000","last_unstale":"2023-09-23T21:04:32.426372+0000","last_undegraded":"2023-09-23T21:04:32.426372+0000","last_fullsized":"2023-09-23T21:04:32.426372+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:58:13.957677+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":311,"reported_epoch":320,"state":"active+clean","last_fresh":"2023-09-23T21:04:32.427995+0000","last_change":"2023-09-23T20:54:31.898870+0000","last_active":"2023-09-23T21:04:32.427995+0000","last_peered":"2023-09-23T21:04:32.427995+0000","last_clean":"2023-09-23T21:04:32.427995+0000","last_became_active":"2023-09-23T20:54:31.898467+0000","last_became_peered":"2023-09-23T20:54:31.898467+0000","last_unstale":"2023-09-23T21:04:32.427995+0000","last_undegraded":"2023-09-23T21:04:32.427995+0000","last_fullsized":"2023-09-23T21:04:32.427995+0000","mapping_epoch":136,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:56:22.483408+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.9","version":"321'9581","reported_seq":9776,"reported_epoch":321,"state":"active+clean","last_fresh":"2023-09-23T21:04:39.081978+0000","last_change":"2023-09-23T21:03:37.216201+0000","last_active":"2023-09-23T21:04:39.081978+0000","last_peered":"2023-09-23T21:04:39.081978+0000","last_clean":"2023-09-23T21:04:39.081978+0000","last_became_active":"2023-09-23T21:03:30.239058+0000","last_became_peered":"2023-09-23T21:03:30.239058+0000","last_unstale":"2023-09-23T21:04:39.081978+0000","last_undegraded":"2023-09-23T21:04:39.081978+0000","last_fullsized":"2023-09-23T21:04:39.081978+0000","mapping_epoch":254,"log_start":"258'700","ondisk_log_start":"258'700","created":254,"last_epoch_clean":282,"parent":"0.0","parent_split_bits":0,"last_scrub":"283'8319","last_scrub_stamp":"2023-09-23T21:03:37.216105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:01:24.325318+0000","last_clean_scrub_stamp":"2023-09-23T21:03:37.216105+0000","objects_scrubbed":2492,"log_size":8881,"log_dups_size":0,"ondisk_log_size":8881,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:58:52.183586+0000","scrub_duration":0.68308542000000005,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":80150528,"num_objects":1223,"num_object_clones":0,"num_object_copies":3669,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1223,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1269,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,0],"acting":[6,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":195,"reported_epoch":321,"state":"active+clean","last_fresh":"2023-09-23T21:04:33.410613+0000","last_change":"2023-09-23T20:54:31.883504+0000","last_active":"2023-09-23T21:04:33.410613+0000","last_peered":"2023-09-23T21:04:33.410613+0000","last_clean":"2023-09-23T21:04:33.410613+0000","last_became_active":"2023-09-23T20:54:31.883172+0000","last_became_peered":"2023-09-23T20:54:31.883172+0000","last_unstale":"2023-09-23T21:04:33.410613+0000","last_undegraded":"2023-09-23T21:04:33.410613+0000","last_fullsized":"2023-09-23T21:04:33.410613+0000","mapping_epoch":136,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:45:40.289120+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"79'161","reported_seq":554,"reported_epoch":321,"state":"active+clean","last_fresh":"2023-09-23T21:04:33.428725+0000","last_change":"2023-09-23T20:57:20.738417+0000","last_active":"2023-09-23T21:04:33.428725+0000","last_peered":"2023-09-23T21:04:33.428725+0000","last_clean":"2023-09-23T21:04:33.428725+0000","last_became_active":"2023-09-23T20:57:08.236842+0000","last_became_peered":"2023-09-23T20:57:08.236842+0000","last_unstale":"2023-09-23T21:04:33.428725+0000","last_undegraded":"2023-09-23T21:04:33.428725+0000","last_fullsized":"2023-09-23T21:04:33.428725+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":0,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:20.738273+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:20.738273+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:18:28.327990+0000","scrub_duration":0.0088336339999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":301,"reported_epoch":320,"state":"active+clean","last_fresh":"2023-09-23T21:04:32.423972+0000","last_change":"2023-09-23T20:55:35.950777+0000","last_active":"2023-09-23T21:04:32.423972+0000","last_peered":"2023-09-23T21:04:32.423972+0000","last_clean":"2023-09-23T21:04:32.423972+0000","last_became_active":"2023-09-23T20:55:35.950427+0000","last_became_peered":"2023-09-23T20:55:35.950427+0000","last_unstale":"2023-09-23T21:04:32.423972+0000","last_undegraded":"2023-09-23T21:04:32.423972+0000","last_fullsized":"2023-09-23T21:04:32.423972+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:20:10.371751+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":265,"reported_epoch":320,"state":"active+clean","last_fresh":"2023-09-23T21:04:32.399611+0000","last_change":"2023-09-23T20:55:35.978140+0000","last_active":"2023-09-23T21:04:32.399611+0000","last_peered":"2023-09-23T21:04:32.399611+0000","last_clean":"2023-09-23T21:04:32.399611+0000","last_became_active":"2023-09-23T20:55:35.977831+0000","last_became_peered":"2023-09-23T20:55:35.977831+0000","last_unstale":"2023-09-23T21:04:32.399611+0000","last_undegraded":"2023-09-23T21:04:32.399611+0000","last_fullsized":"2023-09-23T21:04:32.399611+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:25:47.678023+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.a","version":"321'11265","reported_seq":11339,"reported_epoch":321,"state":"active+clean","last_fresh":"2023-09-23T21:04:39.617187+0000","last_change":"2023-09-23T21:01:25.429899+0000","last_active":"2023-09-23T21:04:39.617187+0000","last_peered":"2023-09-23T21:04:39.617187+0000","last_clean":"2023-09-23T21:04:39.617187+0000","last_became_active":"2023-09-23T21:01:25.429583+0000","last_became_peered":"2023-09-23T21:01:25.429583+0000","last_unstale":"2023-09-23T21:04:39.617187+0000","last_undegraded":"2023-09-23T21:04:39.617187+0000","last_fullsized":"2023-09-23T21:04:39.617187+0000","mapping_epoch":254,"log_start":"269'4969","ondisk_log_start":"269'4969","created":254,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:01:24.325318+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:01:24.325318+0000","last_clean_scrub_stamp":"2023-09-23T21:01:24.325318+0000","objects_scrubbed":0,"log_size":6296,"log_dups_size":0,"ondisk_log_size":6296,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:31:11.594129+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":160890880,"num_objects":2455,"num_object_clones":0,"num_object_copies":7365,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2455,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11265,"num_write_kb":439040,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.3","version":"79'161","reported_seq":590,"reported_epoch":320,"state":"active+clean","last_fresh":"2023-09-23T21:04:32.428324+0000","last_change":"2023-09-23T20:57:20.378531+0000","last_active":"2023-09-23T21:04:32.428324+0000","last_peered":"2023-09-23T21:04:32.428324+0000","last_clean":"2023-09-23T21:04:32.428324+0000","last_became_active":"2023-09-23T20:57:08.238019+0000","last_became_peered":"2023-09-23T20:57:08.238019+0000","last_unstale":"2023-09-23T21:04:32.428324+0000","last_undegraded":"2023-09-23T21:04:32.428324+0000","last_fullsized":"2023-09-23T21:04:32.428324+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":3,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:20.377714+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:20.377714+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:00:08.305602+0000","scrub_duration":0.003015228,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":192,"reported_epoch":320,"state":"active+clean","last_fresh":"2023-09-23T21:04:32.426517+0000","last_change":"2023-09-23T20:56:45.991884+0000","last_active":"2023-09-23T21:04:32.426517+0000","last_peered":"2023-09-23T21:04:32.426517+0000","last_clean":"2023-09-23T21:04:32.426517+0000","last_became_active":"2023-09-23T20:56:45.991568+0000","last_became_peered":"2023-09-23T20:56:45.991568+0000","last_unstale":"2023-09-23T21:04:32.426517+0000","last_undegraded":"2023-09-23T21:04:32.426517+0000","last_fullsized":"2023-09-23T21:04:32.426517+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":205,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:21:20.775511+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,0],"acting":[1,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":318,"reported_epoch":321,"state":"active+clean","last_fresh":"2023-09-23T21:04:33.429393+0000","last_change":"2023-09-23T20:57:44.979950+0000","last_active":"2023-09-23T21:04:33.429393+0000","last_peered":"2023-09-23T21:04:33.429393+0000","last_clean":"2023-09-23T21:04:33.429393+0000","last_became_active":"2023-09-23T20:57:44.979346+0000","last_became_peered":"2023-09-23T20:57:44.979346+0000","last_unstale":"2023-09-23T21:04:33.429393+0000","last_undegraded":"2023-09-23T21:04:33.429393+0000","last_fullsized":"2023-09-23T21:04:33.429393+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:10:24.372800+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":297,"reported_epoch":322,"state":"active+clean","last_fresh":"2023-09-23T21:04:40.235063+0000","last_change":"2023-09-23T20:50:37.982235+0000","last_active":"2023-09-23T21:04:40.235063+0000","last_peered":"2023-09-23T21:04:40.235063+0000","last_clean":"2023-09-23T21:04:40.235063+0000","last_became_active":"2023-09-23T20:50:37.981913+0000","last_became_peered":"2023-09-23T20:50:37.981913+0000","last_unstale":"2023-09-23T21:04:40.235063+0000","last_undegraded":"2023-09-23T21:04:40.235063+0000","last_fullsized":"2023-09-23T21:04:40.235063+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:59:57.122584+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.f","version":"321'11174","reported_seq":11251,"reported_epoch":321,"state":"active+clean","last_fresh":"2023-09-23T21:04:39.922694+0000","last_change":"2023-09-23T21:01:25.432176+0000","last_active":"2023-09-23T21:04:39.922694+0000","last_peered":"2023-09-23T21:04:39.922694+0000","last_clean":"2023-09-23T21:04:39.922694+0000","last_became_active":"2023-09-23T21:01:25.431835+0000","last_became_peered":"2023-09-23T21:01:25.431835+0000","last_unstale":"2023-09-23T21:04:39.922694+0000","last_undegraded":"2023-09-23T21:04:39.922694+0000","last_fullsized":"2023-09-23T21:04:39.922694+0000","mapping_epoch":254,"log_start":"271'5244","ondisk_log_start":"271'5244","created":254,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:01:24.325318+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:01:24.325318+0000","last_clean_scrub_stamp":"2023-09-23T21:01:24.325318+0000","objects_scrubbed":0,"log_size":5930,"log_dups_size":0,"ondisk_log_size":5930,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:47:26.553643+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":162004992,"num_objects":2472,"num_object_clones":0,"num_object_copies":7416,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2472,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11174,"num_write_kb":436672,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,1],"acting":[3,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.6","version":"79'161","reported_seq":603,"reported_epoch":320,"state":"active+clean","last_fresh":"2023-09-23T21:04:32.399738+0000","last_change":"2023-09-23T20:57:26.907505+0000","last_active":"2023-09-23T21:04:32.399738+0000","last_peered":"2023-09-23T21:04:32.399738+0000","last_clean":"2023-09-23T21:04:32.399738+0000","last_became_active":"2023-09-23T20:57:08.240321+0000","last_became_peered":"2023-09-23T20:57:08.240321+0000","last_unstale":"2023-09-23T21:04:32.399738+0000","last_undegraded":"2023-09-23T21:04:32.399738+0000","last_fullsized":"2023-09-23T21:04:32.399738+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":3,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:26.907408+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:26.907408+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:34:17.392928+0000","scrub_duration":0.010337038,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":240,"reported_epoch":321,"state":"active+clean","last_fresh":"2023-09-23T21:04:33.431886+0000","last_change":"2023-09-23T20:56:45.988566+0000","last_active":"2023-09-23T21:04:33.431886+0000","last_peered":"2023-09-23T21:04:33.431886+0000","last_clean":"2023-09-23T21:04:33.431886+0000","last_became_active":"2023-09-23T20:56:45.988249+0000","last_became_peered":"2023-09-23T20:56:45.988249+0000","last_unstale":"2023-09-23T21:04:33.431886+0000","last_undegraded":"2023-09-23T21:04:33.431886+0000","last_fullsized":"2023-09-23T21:04:33.431886+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":205,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:51:36.644764+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,4],"acting":[0,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":294,"reported_epoch":321,"state":"active+clean","last_fresh":"2023-09-23T21:04:33.412972+0000","last_change":"2023-09-23T20:54:31.898965+0000","last_active":"2023-09-23T21:04:33.412972+0000","last_peered":"2023-09-23T21:04:33.412972+0000","last_clean":"2023-09-23T21:04:33.412972+0000","last_became_active":"2023-09-23T20:54:31.898663+0000","last_became_peered":"2023-09-23T20:54:31.898663+0000","last_unstale":"2023-09-23T21:04:33.412972+0000","last_undegraded":"2023-09-23T21:04:33.412972+0000","last_fullsized":"2023-09-23T21:04:33.412972+0000","mapping_epoch":136,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:53:05.058087+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":285,"reported_epoch":321,"state":"active+clean","last_fresh":"2023-09-23T21:04:33.428920+0000","last_change":"2023-09-23T20:54:31.884717+0000","last_active":"2023-09-23T21:04:33.428920+0000","last_peered":"2023-09-23T21:04:33.428920+0000","last_clean":"2023-09-23T21:04:33.428920+0000","last_became_active":"2023-09-23T20:54:31.884345+0000","last_became_peered":"2023-09-23T20:54:31.884345+0000","last_unstale":"2023-09-23T21:04:33.428920+0000","last_undegraded":"2023-09-23T21:04:33.428920+0000","last_fullsized":"2023-09-23T21:04:33.428920+0000","mapping_epoch":136,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:02:26.393868+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.c","version":"322'11654","reported_seq":11732,"reported_epoch":322,"state":"active+clean","last_fresh":"2023-09-23T21:04:42.191067+0000","last_change":"2023-09-23T21:01:25.560907+0000","last_active":"2023-09-23T21:04:42.191067+0000","last_peered":"2023-09-23T21:04:42.191067+0000","last_clean":"2023-09-23T21:04:42.191067+0000","last_became_active":"2023-09-23T21:01:25.560592+0000","last_became_peered":"2023-09-23T21:01:25.560592+0000","last_unstale":"2023-09-23T21:04:42.191067+0000","last_undegraded":"2023-09-23T21:04:42.191067+0000","last_fullsized":"2023-09-23T21:04:42.191067+0000","mapping_epoch":254,"log_start":"268'4759","ondisk_log_start":"268'4759","created":254,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:01:24.325318+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:01:24.325318+0000","last_clean_scrub_stamp":"2023-09-23T21:01:24.325318+0000","objects_scrubbed":0,"log_size":6895,"log_dups_size":0,"ondisk_log_size":6895,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:14:22.102068+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":150863872,"num_objects":2302,"num_object_clones":0,"num_object_copies":6906,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2302,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11654,"num_write_kb":446592,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.5","version":"79'161","reported_seq":585,"reported_epoch":321,"state":"active+clean","last_fresh":"2023-09-23T21:04:33.411950+0000","last_change":"2023-09-23T20:57:30.029479+0000","last_active":"2023-09-23T21:04:33.411950+0000","last_peered":"2023-09-23T21:04:33.411950+0000","last_clean":"2023-09-23T21:04:33.411950+0000","last_became_active":"2023-09-23T20:57:08.249909+0000","last_became_peered":"2023-09-23T20:57:08.249909+0000","last_unstale":"2023-09-23T21:04:33.411950+0000","last_undegraded":"2023-09-23T21:04:33.411950+0000","last_fullsized":"2023-09-23T21:04:33.411950+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":3,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:30.029396+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:30.029396+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:16:46.083145+0000","scrub_duration":0.012914682,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":287,"reported_epoch":320,"state":"active+clean","last_fresh":"2023-09-23T21:04:32.423689+0000","last_change":"2023-09-23T20:51:10.836910+0000","last_active":"2023-09-23T21:04:32.423689+0000","last_peered":"2023-09-23T21:04:32.423689+0000","last_clean":"2023-09-23T21:04:32.423689+0000","last_became_active":"2023-09-23T20:51:10.836557+0000","last_became_peered":"2023-09-23T20:51:10.836557+0000","last_unstale":"2023-09-23T21:04:32.423689+0000","last_undegraded":"2023-09-23T21:04:32.423689+0000","last_fullsized":"2023-09-23T21:04:32.423689+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:51:54.019969+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":293,"reported_epoch":320,"state":"active+clean","last_fresh":"2023-09-23T21:04:32.427608+0000","last_change":"2023-09-23T20:55:35.953444+0000","last_active":"2023-09-23T21:04:32.427608+0000","last_peered":"2023-09-23T21:04:32.427608+0000","last_clean":"2023-09-23T21:04:32.427608+0000","last_became_active":"2023-09-23T20:55:35.953151+0000","last_became_peered":"2023-09-23T20:55:35.953151+0000","last_unstale":"2023-09-23T21:04:32.427608+0000","last_undegraded":"2023-09-23T21:04:32.427608+0000","last_fullsized":"2023-09-23T21:04:32.427608+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:39:12.404391+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":112,"reported_epoch":320,"state":"active+clean","last_fresh":"2023-09-23T21:04:32.423836+0000","last_change":"2023-09-23T20:57:44.982509+0000","last_active":"2023-09-23T21:04:32.423836+0000","last_peered":"2023-09-23T21:04:32.423836+0000","last_clean":"2023-09-23T21:04:32.423836+0000","last_became_active":"2023-09-23T20:57:44.982175+0000","last_became_peered":"2023-09-23T20:57:44.982175+0000","last_unstale":"2023-09-23T21:04:32.423836+0000","last_undegraded":"2023-09-23T21:04:32.423836+0000","last_fullsized":"2023-09-23T21:04:32.423836+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:37:49.215488+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.d","version":"322'11355","reported_seq":11432,"reported_epoch":322,"state":"active+clean","last_fresh":"2023-09-23T21:04:42.099155+0000","last_change":"2023-09-23T21:01:25.560531+0000","last_active":"2023-09-23T21:04:42.099155+0000","last_peered":"2023-09-23T21:04:42.099155+0000","last_clean":"2023-09-23T21:04:42.099155+0000","last_became_active":"2023-09-23T21:01:25.560228+0000","last_became_peered":"2023-09-23T21:01:25.560228+0000","last_unstale":"2023-09-23T21:04:42.099155+0000","last_undegraded":"2023-09-23T21:04:42.099155+0000","last_fullsized":"2023-09-23T21:04:42.099155+0000","mapping_epoch":254,"log_start":"269'4819","ondisk_log_start":"269'4819","created":254,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:01:24.325318+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:01:24.325318+0000","last_clean_scrub_stamp":"2023-09-23T21:01:24.325318+0000","objects_scrubbed":0,"log_size":6536,"log_dups_size":0,"ondisk_log_size":6536,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:19:04.504855+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":152829976,"num_objects":2333,"num_object_clones":0,"num_object_copies":6999,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2333,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":11355,"num_write_kb":437953,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,2],"acting":[5,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.4","version":"79'161","reported_seq":602,"reported_epoch":320,"state":"active+clean","last_fresh":"2023-09-23T21:04:32.424542+0000","last_change":"2023-09-23T20:57:20.012904+0000","last_active":"2023-09-23T21:04:32.424542+0000","last_peered":"2023-09-23T21:04:32.424542+0000","last_clean":"2023-09-23T21:04:32.424542+0000","last_became_active":"2023-09-23T20:57:08.237996+0000","last_became_peered":"2023-09-23T20:57:08.237996+0000","last_unstale":"2023-09-23T21:04:32.424542+0000","last_undegraded":"2023-09-23T21:04:32.424542+0000","last_fullsized":"2023-09-23T21:04:32.424542+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":3,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:20.012764+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:20.012764+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:51:28.796946+0000","scrub_duration":0.0045251919999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":270,"reported_epoch":320,"state":"active+clean","last_fresh":"2023-09-23T21:04:32.399915+0000","last_change":"2023-09-23T20:55:35.949541+0000","last_active":"2023-09-23T21:04:32.399915+0000","last_peered":"2023-09-23T21:04:32.399915+0000","last_clean":"2023-09-23T21:04:32.399915+0000","last_became_active":"2023-09-23T20:55:35.944635+0000","last_became_peered":"2023-09-23T20:55:35.944635+0000","last_unstale":"2023-09-23T21:04:32.399915+0000","last_undegraded":"2023-09-23T21:04:32.399915+0000","last_fullsized":"2023-09-23T21:04:32.399915+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:30:36.033585+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"70'1","reported_seq":313,"reported_epoch":322,"state":"active+clean","last_fresh":"2023-09-23T21:04:40.235157+0000","last_change":"2023-09-23T20:54:14.009723+0000","last_active":"2023-09-23T21:04:40.235157+0000","last_peered":"2023-09-23T21:04:40.235157+0000","last_clean":"2023-09-23T21:04:40.235157+0000","last_became_active":"2023-09-23T20:54:13.941771+0000","last_became_peered":"2023-09-23T20:54:13.941771+0000","last_unstale":"2023-09-23T21:04:40.235157+0000","last_undegraded":"2023-09-23T21:04:40.235157+0000","last_fullsized":"2023-09-23T21:04:40.235157+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:48:25.588116+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":226,"reported_epoch":321,"state":"active+clean","last_fresh":"2023-09-23T21:04:33.411568+0000","last_change":"2023-09-23T20:55:35.949780+0000","last_active":"2023-09-23T21:04:33.411568+0000","last_peered":"2023-09-23T21:04:33.411568+0000","last_clean":"2023-09-23T21:04:33.411568+0000","last_became_active":"2023-09-23T20:55:35.949497+0000","last_became_peered":"2023-09-23T20:55:35.949497+0000","last_unstale":"2023-09-23T21:04:33.411568+0000","last_undegraded":"2023-09-23T21:04:33.411568+0000","last_fullsized":"2023-09-23T21:04:33.411568+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:09:37.420882+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.10","version":"322'9918","reported_seq":17430,"reported_epoch":322,"state":"active+clean","last_fresh":"2023-09-23T21:04:42.054184+0000","last_change":"2023-09-23T21:03:46.218495+0000","last_active":"2023-09-23T21:04:42.054184+0000","last_peered":"2023-09-23T21:04:42.054184+0000","last_clean":"2023-09-23T21:04:42.054184+0000","last_became_active":"2023-09-23T21:03:46.185478+0000","last_became_peered":"2023-09-23T21:03:46.185478+0000","last_unstale":"2023-09-23T21:04:42.054184+0000","last_undegraded":"2023-09-23T21:04:42.054184+0000","last_fullsized":"2023-09-23T21:04:42.054184+0000","mapping_epoch":292,"log_start":"258'1700","ondisk_log_start":"258'1700","created":281,"last_epoch_clean":293,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:01:24.325318+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:01:24.325318+0000","last_clean_scrub_stamp":"2023-09-23T21:01:24.325318+0000","objects_scrubbed":0,"log_size":8218,"log_dups_size":0,"ondisk_log_size":8218,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":72482816,"num_objects":1106,"num_object_clones":0,"num_object_copies":3318,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1106,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5778,"num_write_kb":220288,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2466,"num_bytes_recovered":161611776,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,4],"acting":[5,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":287,"reported_epoch":321,"state":"active+clean","last_fresh":"2023-09-23T21:04:33.428841+0000","last_change":"2023-09-23T20:54:13.929196+0000","last_active":"2023-09-23T21:04:33.428841+0000","last_peered":"2023-09-23T21:04:33.428841+0000","last_clean":"2023-09-23T21:04:33.428841+0000","last_became_active":"2023-09-23T20:54:13.928831+0000","last_became_peered":"2023-09-23T20:54:13.928831+0000","last_unstale":"2023-09-23T21:04:33.428841+0000","last_undegraded":"2023-09-23T21:04:33.428841+0000","last_fullsized":"2023-09-23T21:04:33.428841+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:28:44.316851+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":292,"reported_epoch":321,"state":"active+clean","last_fresh":"2023-09-23T21:04:33.414998+0000","last_change":"2023-09-23T20:52:12.855430+0000","last_active":"2023-09-23T21:04:33.414998+0000","last_peered":"2023-09-23T21:04:33.414998+0000","last_clean":"2023-09-23T21:04:33.414998+0000","last_became_active":"2023-09-23T20:52:12.855115+0000","last_became_peered":"2023-09-23T20:52:12.855115+0000","last_unstale":"2023-09-23T21:04:33.414998+0000","last_undegraded":"2023-09-23T21:04:33.414998+0000","last_fullsized":"2023-09-23T21:04:33.414998+0000","mapping_epoch":100,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":101,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:33:23.802214+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"8.17","version":"322'9734","reported_seq":14753,"reported_epoch":322,"state":"active+clean","last_fresh":"2023-09-23T21:04:42.192369+0000","last_change":"2023-09-23T21:04:30.691764+0000","last_active":"2023-09-23T21:04:42.192369+0000","last_peered":"2023-09-23T21:04:42.192369+0000","last_clean":"2023-09-23T21:04:42.192369+0000","last_became_active":"2023-09-23T21:04:25.357991+0000","last_became_peered":"2023-09-23T21:04:25.357991+0000","last_unstale":"2023-09-23T21:04:42.192369+0000","last_undegraded":"2023-09-23T21:04:42.192369+0000","last_fullsized":"2023-09-23T21:04:42.192369+0000","mapping_epoch":315,"log_start":"258'1300","ondisk_log_start":"258'1300","created":281,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:01:24.325318+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:01:24.325318+0000","last_clean_scrub_stamp":"2023-09-23T21:01:24.325318+0000","objects_scrubbed":0,"log_size":8434,"log_dups_size":0,"ondisk_log_size":8434,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":78774272,"num_objects":1202,"num_object_clones":0,"num_object_copies":3606,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1202,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5629,"num_write_kb":218592,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1636,"num_bytes_recovered":107216896,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":199,"reported_epoch":321,"state":"active+clean","last_fresh":"2023-09-23T21:04:33.410960+0000","last_change":"2023-09-23T20:54:31.902946+0000","last_active":"2023-09-23T21:04:33.410960+0000","last_peered":"2023-09-23T21:04:33.410960+0000","last_clean":"2023-09-23T21:04:33.410960+0000","last_became_active":"2023-09-23T20:54:31.889484+0000","last_became_peered":"2023-09-23T20:54:31.889484+0000","last_unstale":"2023-09-23T21:04:33.410960+0000","last_undegraded":"2023-09-23T21:04:33.410960+0000","last_fullsized":"2023-09-23T21:04:33.410960+0000","mapping_epoch":136,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:44:44.278398+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":199,"reported_epoch":321,"state":"active+clean","last_fresh":"2023-09-23T21:04:33.411969+0000","last_change":"2023-09-23T20:54:13.950351+0000","last_active":"2023-09-23T21:04:33.411969+0000","last_peered":"2023-09-23T21:04:33.411969+0000","last_clean":"2023-09-23T21:04:33.411969+0000","last_became_active":"2023-09-23T20:54:13.949987+0000","last_became_peered":"2023-09-23T20:54:13.949987+0000","last_unstale":"2023-09-23T21:04:33.411969+0000","last_undegraded":"2023-09-23T21:04:33.411969+0000","last_fullsized":"2023-09-23T21:04:33.411969+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:46:17.765134+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.16","version":"321'10121","reported_seq":17437,"reported_epoch":321,"state":"active+clean","last_fresh":"2023-09-23T21:04:39.876573+0000","last_change":"2023-09-23T21:04:29.781752+0000","last_active":"2023-09-23T21:04:39.876573+0000","last_peered":"2023-09-23T21:04:39.876573+0000","last_clean":"2023-09-23T21:04:39.876573+0000","last_became_active":"2023-09-23T21:04:24.343644+0000","last_became_peered":"2023-09-23T21:04:24.343644+0000","last_unstale":"2023-09-23T21:04:39.876573+0000","last_undegraded":"2023-09-23T21:04:39.876573+0000","last_fullsized":"2023-09-23T21:04:39.876573+0000","mapping_epoch":314,"log_start":"258'600","ondisk_log_start":"258'600","created":281,"last_epoch_clean":315,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:01:24.325318+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:01:24.325318+0000","last_clean_scrub_stamp":"2023-09-23T21:01:24.325318+0000","objects_scrubbed":0,"log_size":9521,"log_dups_size":0,"ondisk_log_size":9521,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":82706432,"num_objects":1262,"num_object_clones":0,"num_object_copies":3786,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1262,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5843,"num_write_kb":227360,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1801,"num_bytes_recovered":118030336,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,5],"acting":[3,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":230,"reported_epoch":321,"state":"active+clean","last_fresh":"2023-09-23T21:04:33.411068+0000","last_change":"2023-09-23T20:56:18.141728+0000","last_active":"2023-09-23T21:04:33.411068+0000","last_peered":"2023-09-23T21:04:33.411068+0000","last_clean":"2023-09-23T21:04:33.411068+0000","last_became_active":"2023-09-23T20:56:18.137979+0000","last_became_peered":"2023-09-23T20:56:18.137979+0000","last_unstale":"2023-09-23T21:04:33.411068+0000","last_undegraded":"2023-09-23T21:04:33.411068+0000","last_fullsized":"2023-09-23T21:04:33.411068+0000","mapping_epoch":192,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":193,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:58:05.658443+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":199,"reported_epoch":321,"state":"active+clean","last_fresh":"2023-09-23T21:04:33.411737+0000","last_change":"2023-09-23T20:54:13.935307+0000","last_active":"2023-09-23T21:04:33.411737+0000","last_peered":"2023-09-23T21:04:33.411737+0000","last_clean":"2023-09-23T21:04:33.411737+0000","last_became_active":"2023-09-23T20:54:13.935000+0000","last_became_peered":"2023-09-23T20:54:13.935000+0000","last_unstale":"2023-09-23T21:04:33.411737+0000","last_undegraded":"2023-09-23T21:04:33.411737+0000","last_fullsized":"2023-09-23T21:04:33.411737+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:47:40.974082+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.15","version":"321'9955","reported_seq":13718,"reported_epoch":321,"state":"active+clean","last_fresh":"2023-09-23T21:04:39.588715+0000","last_change":"2023-09-23T21:04:30.663986+0000","last_active":"2023-09-23T21:04:39.588715+0000","last_peered":"2023-09-23T21:04:39.588715+0000","last_clean":"2023-09-23T21:04:39.588715+0000","last_became_active":"2023-09-23T21:04:18.282767+0000","last_became_peered":"2023-09-23T21:04:18.282767+0000","last_unstale":"2023-09-23T21:04:39.588715+0000","last_undegraded":"2023-09-23T21:04:39.588715+0000","last_fullsized":"2023-09-23T21:04:39.588715+0000","mapping_epoch":310,"log_start":"258'1000","ondisk_log_start":"258'1000","created":281,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":5,"last_scrub":"318'9682","last_scrub_stamp":"2023-09-23T21:04:30.663825+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:01:24.325318+0000","last_clean_scrub_stamp":"2023-09-23T21:04:30.663825+0000","objects_scrubbed":1516,"log_size":8955,"log_dups_size":0,"ondisk_log_size":8955,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:01:18.334239+0000","scrub_duration":0.41809476499999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":81330176,"num_objects":1241,"num_object_clones":0,"num_object_copies":3723,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1241,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":275,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,6],"acting":[1,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":310,"reported_epoch":320,"state":"active+clean","last_fresh":"2023-09-23T21:04:32.424305+0000","last_change":"2023-09-23T20:55:35.954870+0000","last_active":"2023-09-23T21:04:32.424305+0000","last_peered":"2023-09-23T21:04:32.424305+0000","last_clean":"2023-09-23T21:04:32.424305+0000","last_became_active":"2023-09-23T20:55:35.954568+0000","last_became_peered":"2023-09-23T20:55:35.954568+0000","last_unstale":"2023-09-23T21:04:32.424305+0000","last_undegraded":"2023-09-23T21:04:32.424305+0000","last_fullsized":"2023-09-23T21:04:32.424305+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:06:22.423566+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":309,"reported_epoch":320,"state":"active+clean","last_fresh":"2023-09-23T21:04:32.423130+0000","last_change":"2023-09-23T20:51:10.838334+0000","last_active":"2023-09-23T21:04:32.423130+0000","last_peered":"2023-09-23T21:04:32.423130+0000","last_clean":"2023-09-23T21:04:32.423130+0000","last_became_active":"2023-09-23T20:51:10.838026+0000","last_became_peered":"2023-09-23T20:51:10.838026+0000","last_unstale":"2023-09-23T21:04:32.423130+0000","last_undegraded":"2023-09-23T21:04:32.423130+0000","last_fullsized":"2023-09-23T21:04:32.423130+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:04:37.009457+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.14","version":"322'9883","reported_seq":13749,"reported_epoch":322,"state":"active+clean","last_fresh":"2023-09-23T21:04:42.311366+0000","last_change":"2023-09-23T21:04:23.270430+0000","last_active":"2023-09-23T21:04:42.311366+0000","last_peered":"2023-09-23T21:04:42.311366+0000","last_clean":"2023-09-23T21:04:42.311366+0000","last_became_active":"2023-09-23T21:04:14.227313+0000","last_became_peered":"2023-09-23T21:04:14.227313+0000","last_unstale":"2023-09-23T21:04:42.311366+0000","last_undegraded":"2023-09-23T21:04:42.311366+0000","last_fullsized":"2023-09-23T21:04:42.311366+0000","mapping_epoch":307,"log_start":"258'500","ondisk_log_start":"258'500","created":281,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:01:24.325318+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:01:24.325318+0000","last_clean_scrub_stamp":"2023-09-23T21:01:24.325318+0000","objects_scrubbed":0,"log_size":9383,"log_dups_size":0,"ondisk_log_size":9383,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":77627392,"num_objects":1185,"num_object_clones":0,"num_object_copies":3555,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1185,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5739,"num_write_kb":221536,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1877,"num_bytes_recovered":123011072,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,2],"acting":[4,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":315,"reported_epoch":321,"state":"active+clean","last_fresh":"2023-09-23T21:04:33.430513+0000","last_change":"2023-09-23T20:54:31.903139+0000","last_active":"2023-09-23T21:04:33.430513+0000","last_peered":"2023-09-23T21:04:33.430513+0000","last_clean":"2023-09-23T21:04:33.430513+0000","last_became_active":"2023-09-23T20:54:31.884520+0000","last_became_peered":"2023-09-23T20:54:31.884520+0000","last_unstale":"2023-09-23T21:04:33.430513+0000","last_undegraded":"2023-09-23T21:04:33.430513+0000","last_fullsized":"2023-09-23T21:04:33.430513+0000","mapping_epoch":136,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:39:34.269705+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1108,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":216,"reported_epoch":321,"state":"active+clean","last_fresh":"2023-09-23T21:04:33.411266+0000","last_change":"2023-09-23T20:55:35.951572+0000","last_active":"2023-09-23T21:04:33.411266+0000","last_peered":"2023-09-23T21:04:33.411266+0000","last_clean":"2023-09-23T21:04:33.411266+0000","last_became_active":"2023-09-23T20:55:35.951294+0000","last_became_peered":"2023-09-23T20:55:35.951294+0000","last_unstale":"2023-09-23T21:04:33.411266+0000","last_undegraded":"2023-09-23T21:04:33.411266+0000","last_fullsized":"2023-09-23T21:04:33.411266+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:36:32.447210+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":22,"num_bytes_recovered":5990,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":8,"num_pg":26,"stat_sum":{"num_bytes":2520547352,"num_objects":38461,"num_object_clones":0,"num_object_copies":115383,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":38461,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":140453,"num_write_kb":5093505,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":15252,"num_bytes_recovered":999292928,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8781836288,"data_stored":8781824072,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":213906,"ondisk_log_size":213906,"up":78,"acting":78,"num_store_stats":8},{"poolid":1,"num_pg":21,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3381,"ondisk_log_size":3381,"up":63,"acting":63,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201571,"num_pgs":50,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1335252,"kb_used_data":1052108,"kb_used_omap":0,"kb_used_meta":283072,"kb_avail":92409900,"statfs":{"total":95995035648,"available":94627737600,"internally_reserved":0,"allocated":1077358592,"data_stored":1076313796,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":289865728},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:03:53 2023","interfaces":[{"interface":"back","average":{"1min":1.653,"5min":1.9730000000000001,"15min":1.5},"min":{"1min":0.22700000000000001,"5min":0.22700000000000001,"15min":0.21099999999999999},"max":{"1min":8.4990000000000006,"5min":33.326000000000001,"15min":43.994999999999997},"last":0.82899999999999996},{"interface":"front","average":{"1min":2.0990000000000002,"5min":2.1819999999999999,"15min":1.639},"min":{"1min":0.314,"5min":0.28599999999999998,"15min":0.218},"max":{"1min":8.5150000000000006,"5min":33.354999999999997,"15min":44.206000000000003},"last":0.84899999999999998}]},{"osd":1,"last update":"Sat Sep 23 21:04:34 2023","interfaces":[{"interface":"back","average":{"1min":1.2250000000000001,"5min":1.381,"15min":1.5009999999999999},"min":{"1min":0.245,"5min":0.245,"15min":0.23300000000000001},"max":{"1min":5.7270000000000003,"5min":12.061,"15min":22.591999999999999},"last":0.35099999999999998},{"interface":"front","average":{"1min":1.28,"5min":1.603,"15min":1.4390000000000001},"min":{"1min":0.27900000000000003,"5min":0.25900000000000001,"15min":0.23699999999999999},"max":{"1min":4.8170000000000002,"5min":13.981,"15min":21.437000000000001},"last":0.41499999999999998}]},{"osd":2,"last update":"Sat Sep 23 21:04:06 2023","interfaces":[{"interface":"back","average":{"1min":1.554,"5min":1.9530000000000001,"15min":1.617},"min":{"1min":0.34200000000000003,"5min":0.30099999999999999,"15min":0.24399999999999999},"max":{"1min":5.5890000000000004,"5min":10.279999999999999,"15min":10.805999999999999},"last":2.5859999999999999},{"interface":"front","average":{"1min":1.5069999999999999,"5min":1.8779999999999999,"15min":1.6240000000000001},"min":{"1min":0.29999999999999999,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":7.1589999999999998,"5min":11.590999999999999,"15min":45.25},"last":0.379}]},{"osd":3,"last update":"Sat Sep 23 21:03:42 2023","interfaces":[{"interface":"back","average":{"1min":2.2189999999999999,"5min":2.419,"15min":2.2069999999999999},"min":{"1min":0.29099999999999998,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":7.8689999999999998,"5min":31.786000000000001,"15min":36.408000000000001},"last":1.9510000000000001},{"interface":"front","average":{"1min":2.1120000000000001,"5min":2.5630000000000002,"15min":2.0680000000000001},"min":{"1min":0.34399999999999997,"5min":0.309,"15min":0.26000000000000001},"max":{"1min":6.6680000000000001,"5min":32.429000000000002,"15min":35.593000000000004},"last":1.849}]},{"osd":4,"last update":"Sat Sep 23 21:04:34 2023","interfaces":[{"interface":"back","average":{"1min":0.55600000000000005,"5min":0.75600000000000001,"15min":0.69899999999999995},"min":{"1min":0.248,"5min":0.248,"15min":0.23699999999999999},"max":{"1min":2.4399999999999999,"5min":5.2640000000000002,"15min":5.8920000000000003},"last":0.33800000000000002},{"interface":"front","average":{"1min":0.66800000000000004,"5min":0.91000000000000003,"15min":0.75},"min":{"1min":0.28399999999999997,"5min":0.28000000000000003,"15min":0.26300000000000001},"max":{"1min":2.113,"5min":6.3979999999999997,"15min":6.3979999999999997},"last":0.40000000000000002}]},{"osd":5,"last update":"Sat Sep 23 21:04:34 2023","interfaces":[{"interface":"back","average":{"1min":0.52100000000000002,"5min":0.78100000000000003,"15min":0.71999999999999997},"min":{"1min":0.31900000000000001,"5min":0.28999999999999998,"15min":0.28499999999999998},"max":{"1min":1.2250000000000001,"5min":5.2089999999999996,"15min":7.0389999999999997},"last":0.38900000000000001},{"interface":"front","average":{"1min":0.63900000000000001,"5min":0.84699999999999998,"15min":0.71899999999999997},"min":{"1min":0.33900000000000002,"5min":0.32100000000000001,"15min":0.27800000000000002},"max":{"1min":1.5169999999999999,"5min":5.8529999999999998,"15min":5.8529999999999998},"last":0.44800000000000001}]},{"osd":6,"last update":"Sat Sep 23 21:03:49 2023","interfaces":[{"interface":"back","average":{"1min":1.2490000000000001,"5min":0.90400000000000003,"15min":0.76200000000000001},"min":{"1min":0.41899999999999998,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":5.4699999999999998,"5min":6.1219999999999999,"15min":6.1219999999999999},"last":0.36499999999999999},{"interface":"front","average":{"1min":1.3400000000000001,"5min":0.94899999999999995,"15min":0.85599999999999998},"min":{"1min":0.38500000000000001,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":6.0880000000000001,"5min":6.0919999999999996,"15min":6.0919999999999996},"last":0.436}]}]},{"osd":6,"up_from":164,"seq":704374636654,"num_pgs":48,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1146488,"kb_used_data":953392,"kb_used_omap":10,"kb_used_meta":193077,"kb_avail":92598664,"statfs":{"total":95995035648,"available":94821031936,"internally_reserved":0,"allocated":976273408,"data_stored":975248661,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":10328,"internal_metadata":197711784},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:03:51 2023","interfaces":[{"interface":"back","average":{"1min":1.621,"5min":1.794,"15min":1.367},"min":{"1min":0.246,"5min":0.23499999999999999,"15min":0.20999999999999999},"max":{"1min":4.6420000000000003,"5min":14.151999999999999,"15min":14.151999999999999},"last":1.833},{"interface":"front","average":{"1min":1.2490000000000001,"5min":1.929,"15min":1.4710000000000001},"min":{"1min":0.25800000000000001,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":4.4450000000000003,"5min":11.023,"15min":11.023},"last":4.6799999999999997}]},{"osd":1,"last update":"Sat Sep 23 21:03:51 2023","interfaces":[{"interface":"back","average":{"1min":2.1070000000000002,"5min":1.696,"15min":1.194},"min":{"1min":0.35199999999999998,"5min":0.26400000000000001,"15min":0.25600000000000001},"max":{"1min":10.026,"5min":10.026,"15min":10.026},"last":2.4910000000000001},{"interface":"front","average":{"1min":2.4009999999999998,"5min":1.972,"15min":1.361},"min":{"1min":0.26200000000000001,"5min":0.26100000000000001,"15min":0.23000000000000001},"max":{"1min":10.218,"5min":10.518000000000001,"15min":10.518000000000001},"last":2.5859999999999999}]},{"osd":2,"last update":"Sat Sep 23 21:03:51 2023","interfaces":[{"interface":"back","average":{"1min":1.696,"5min":2.3780000000000001,"15min":2.0019999999999998},"min":{"1min":0.36299999999999999,"5min":0.28499999999999998,"15min":0.27700000000000002},"max":{"1min":4.54,"5min":14.029999999999999,"15min":14.029999999999999},"last":3.0899999999999999},{"interface":"front","average":{"1min":3.4460000000000002,"5min":2.5950000000000002,"15min":1.7430000000000001},"min":{"1min":0.26500000000000001,"5min":0.247,"15min":0.247},"max":{"1min":11.672000000000001,"5min":20.195,"15min":20.195},"last":3.379}]},{"osd":3,"last update":"Sat Sep 23 21:03:51 2023","interfaces":[{"interface":"back","average":{"1min":4.1580000000000004,"5min":2.7909999999999999,"15min":1.7310000000000001},"min":{"1min":0.28699999999999998,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":43.085999999999999,"5min":43.085999999999999,"15min":43.085999999999999},"last":3.7690000000000001},{"interface":"front","average":{"1min":4.0279999999999996,"5min":2.4079999999999999,"15min":1.917},"min":{"1min":0.313,"5min":0.313,"15min":0.28599999999999998},"max":{"1min":42.405000000000001,"5min":42.405000000000001,"15min":42.405000000000001},"last":3.1499999999999999}]},{"osd":4,"last update":"Sat Sep 23 21:03:51 2023","interfaces":[{"interface":"back","average":{"1min":1.1619999999999999,"5min":0.93700000000000006,"15min":0.94899999999999995},"min":{"1min":0.28799999999999998,"5min":0.24299999999999999,"15min":0.23000000000000001},"max":{"1min":2.242,"5min":5.3920000000000003,"15min":9.0079999999999991},"last":2.5099999999999998},{"interface":"front","average":{"1min":0.61699999999999999,"5min":0.78900000000000003,"15min":0.60899999999999999},"min":{"1min":0.30099999999999999,"5min":0.26600000000000001,"15min":0.25800000000000001},"max":{"1min":1.782,"5min":4.8760000000000003,"15min":4.8760000000000003},"last":3.29}]},{"osd":5,"last update":"Sat Sep 23 21:03:51 2023","interfaces":[{"interface":"back","average":{"1min":1.341,"5min":1.133,"15min":0.748},"min":{"1min":0.34999999999999998,"5min":0.27500000000000002,"15min":0.27300000000000002},"max":{"1min":5.5170000000000003,"5min":5.9000000000000004,"15min":5.9000000000000004},"last":2.5550000000000002},{"interface":"front","average":{"1min":1.403,"5min":1.0169999999999999,"15min":0.871},"min":{"1min":0.33200000000000002,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":8.9610000000000003,"5min":8.9610000000000003,"15min":8.9610000000000003},"last":1.9450000000000001}]},{"osd":7,"last update":"Sat Sep 23 21:03:51 2023","interfaces":[{"interface":"back","average":{"1min":1.3340000000000001,"5min":1.0920000000000001,"15min":0.95799999999999996},"min":{"1min":0.30499999999999999,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":8.9260000000000002,"5min":8.9260000000000002,"15min":8.9260000000000002},"last":1.8700000000000001},{"interface":"front","average":{"1min":1.1719999999999999,"5min":0.88900000000000001,"15min":0.69399999999999995},"min":{"1min":0.379,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":4.6589999999999998,"5min":4.6589999999999998,"15min":8.282},"last":2.3119999999999998}]}]},{"osd":1,"up_from":88,"seq":377957122225,"num_pgs":48,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1547736,"kb_used_data":1158288,"kb_used_omap":9,"kb_used_meta":389366,"kb_avail":92197416,"statfs":{"total":95995035648,"available":94410153984,"internally_reserved":0,"allocated":1186086912,"data_stored":1184570942,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9239,"internal_metadata":398711785},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:04:32 2023","interfaces":[{"interface":"back","average":{"1min":2.3250000000000002,"5min":2.613,"15min":2.274},"min":{"1min":0.215,"5min":0.215,"15min":0.20699999999999999},"max":{"1min":9.7430000000000003,"5min":45.542000000000002,"15min":45.542000000000002},"last":1.032},{"interface":"front","average":{"1min":2.8679999999999999,"5min":3.2570000000000001,"15min":2.6219999999999999},"min":{"1min":0.32200000000000001,"5min":0.248,"15min":0.22500000000000001},"max":{"1min":11.715,"5min":46.353000000000002,"15min":46.353000000000002},"last":3.6389999999999998}]},{"osd":2,"last update":"Sat Sep 23 21:04:04 2023","interfaces":[{"interface":"back","average":{"1min":1.9410000000000001,"5min":2.0350000000000001,"15min":1.7290000000000001},"min":{"1min":0.32200000000000001,"5min":0.20399999999999999,"15min":0.20300000000000001},"max":{"1min":6.9029999999999996,"5min":13.752000000000001,"15min":14.335000000000001},"last":1.0589999999999999},{"interface":"front","average":{"1min":4.444,"5min":3.8159999999999998,"15min":2.835},"min":{"1min":0.315,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":13.589,"5min":18.026,"15min":38.951000000000001},"last":2.633}]},{"osd":3,"last update":"Sat Sep 23 21:04:32 2023","interfaces":[{"interface":"back","average":{"1min":1.528,"5min":2.7690000000000001,"15min":2.3620000000000001},"min":{"1min":0.23999999999999999,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":6.1689999999999996,"5min":22.686,"15min":25.314},"last":2.3119999999999998},{"interface":"front","average":{"1min":1.472,"5min":2.1680000000000001,"15min":1.9970000000000001},"min":{"1min":0.33100000000000002,"5min":0.25600000000000001,"15min":0.20599999999999999},"max":{"1min":5.5679999999999996,"5min":14.473000000000001,"15min":19.881},"last":2.4079999999999999}]},{"osd":4,"last update":"Sat Sep 23 21:04:32 2023","interfaces":[{"interface":"back","average":{"1min":1.581,"5min":2.2709999999999999,"15min":1.7549999999999999},"min":{"1min":0.34399999999999997,"5min":0.28399999999999997,"15min":0.28299999999999997},"max":{"1min":9.7599999999999998,"5min":41.823,"15min":41.823},"last":0.38800000000000001},{"interface":"front","average":{"1min":1.2410000000000001,"5min":1.8340000000000001,"15min":1.4670000000000001},"min":{"1min":0.30199999999999999,"5min":0.30199999999999999,"15min":0.27600000000000002},"max":{"1min":4.7160000000000002,"5min":41.423000000000002,"15min":41.423000000000002},"last":1.081}]},{"osd":5,"last update":"Sat Sep 23 21:04:32 2023","interfaces":[{"interface":"back","average":{"1min":1.6279999999999999,"5min":2.0059999999999998,"15min":1.728},"min":{"1min":0.39100000000000001,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":6.1920000000000002,"5min":12.699999999999999,"15min":15.041},"last":2.71},{"interface":"front","average":{"1min":1.2330000000000001,"5min":1.804,"15min":1.625},"min":{"1min":0.41099999999999998,"5min":0.35999999999999999,"15min":0.32200000000000001},"max":{"1min":4.242,"5min":18.280999999999999,"15min":18.280999999999999},"last":0.42099999999999999}]},{"osd":6,"last update":"Sat Sep 23 21:03:54 2023","interfaces":[{"interface":"back","average":{"1min":2.5110000000000001,"5min":2.0510000000000002,"15min":1.5780000000000001},"min":{"1min":0.40799999999999997,"5min":0.35799999999999998,"15min":0.30299999999999999},"max":{"1min":8.0980000000000008,"5min":18.984999999999999,"15min":18.984999999999999},"last":0.44900000000000001},{"interface":"front","average":{"1min":1.857,"5min":1.6060000000000001,"15min":1.4550000000000001},"min":{"1min":0.40799999999999997,"5min":0.33400000000000002,"15min":0.309},"max":{"1min":6.7430000000000003,"5min":7.6779999999999999,"15min":8.7530000000000001},"last":1.107}]},{"osd":7,"last update":"Sat Sep 23 21:04:32 2023","interfaces":[{"interface":"back","average":{"1min":1.339,"5min":1.6040000000000001,"15min":1.4730000000000001},"min":{"1min":0.41699999999999998,"5min":0.39600000000000002,"15min":0.35699999999999998},"max":{"1min":4.6760000000000002,"5min":9.5429999999999993,"15min":11.186999999999999},"last":1.1279999999999999},{"interface":"front","average":{"1min":1.6579999999999999,"5min":2.0219999999999998,"15min":1.77},"min":{"1min":0.44800000000000001,"5min":0.36499999999999999,"15min":0.34599999999999997},"max":{"1min":6.234,"5min":12.84,"15min":15.098000000000001},"last":2.6709999999999998}]}]},{"osd":0,"up_from":83,"seq":356482285752,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1433400,"kb_used_data":1032304,"kb_used_omap":28,"kb_used_meta":401059,"kb_avail":92311752,"statfs":{"total":95995035648,"available":94527234048,"internally_reserved":0,"allocated":1057079296,"data_stored":1055599994,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":28848,"internal_metadata":410685264},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 21:04:23 2023","interfaces":[{"interface":"back","average":{"1min":2.0190000000000001,"5min":3.4220000000000002,"15min":2.5150000000000001},"min":{"1min":0.216,"5min":0.19500000000000001,"15min":0.191},"max":{"1min":10.778,"5min":40.924999999999997,"15min":40.924999999999997},"last":7.2889999999999997},{"interface":"front","average":{"1min":2.1659999999999999,"5min":3.2290000000000001,"15min":2.4870000000000001},"min":{"1min":0.32000000000000001,"5min":0.24099999999999999,"15min":0.20999999999999999},"max":{"1min":7.5590000000000002,"5min":22.474,"15min":22.474},"last":0.47099999999999997}]},{"osd":2,"last update":"Sat Sep 23 21:04:01 2023","interfaces":[{"interface":"back","average":{"1min":2.3319999999999999,"5min":2.8839999999999999,"15min":2.4590000000000001},"min":{"1min":0.25600000000000001,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":5.673,"5min":21.053000000000001,"15min":21.053000000000001},"last":3.8580000000000001},{"interface":"front","average":{"1min":2.6659999999999999,"5min":3.0219999999999998,"15min":2.6040000000000001},"min":{"1min":0.23499999999999999,"5min":0.23100000000000001,"15min":0.21099999999999999},"max":{"1min":11.064,"5min":13.956,"15min":21.747},"last":7.6299999999999999}]},{"osd":3,"last update":"Sat Sep 23 21:04:23 2023","interfaces":[{"interface":"back","average":{"1min":2.133,"5min":3.1829999999999998,"15min":2.5179999999999998},"min":{"1min":0.28999999999999998,"5min":0.27100000000000002,"15min":0.25800000000000001},"max":{"1min":10.866,"5min":41.079000000000001,"15min":41.079000000000001},"last":6.4909999999999997},{"interface":"front","average":{"1min":2.0190000000000001,"5min":2.9009999999999998,"15min":2.6019999999999999},"min":{"1min":0.30099999999999999,"5min":0.27400000000000002,"15min":0.23300000000000001},"max":{"1min":5.6390000000000002,"5min":12.167999999999999,"15min":21.167000000000002},"last":0.995}]},{"osd":4,"last update":"Sat Sep 23 21:04:01 2023","interfaces":[{"interface":"back","average":{"1min":1.643,"5min":1.9630000000000001,"15min":1.9119999999999999},"min":{"1min":0.28699999999999998,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":11.025,"5min":11.025,"15min":21.594999999999999},"last":1.1759999999999999},{"interface":"front","average":{"1min":1.9319999999999999,"5min":2.105,"15min":1.613},"min":{"1min":0.32600000000000001,"5min":0.32600000000000001,"15min":0.28100000000000003},"max":{"1min":10.907999999999999,"5min":13.904,"15min":18.724},"last":6.25}]},{"osd":5,"last update":"Sat Sep 23 21:04:01 2023","interfaces":[{"interface":"back","average":{"1min":1.909,"5min":2.0670000000000002,"15min":1.6499999999999999},"min":{"1min":0.32300000000000001,"5min":0.309,"15min":0.309},"max":{"1min":5.5640000000000001,"5min":11.707000000000001,"15min":11.707000000000001},"last":0.36699999999999999},{"interface":"front","average":{"1min":2.452,"5min":2.1200000000000001,"15min":1.966},"min":{"1min":0.32700000000000001,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":10.742000000000001,"5min":10.742000000000001,"15min":20.512},"last":1.5740000000000001}]},{"osd":6,"last update":"Sat Sep 23 21:03:56 2023","interfaces":[{"interface":"back","average":{"1min":2.1589999999999998,"5min":2.1600000000000001,"15min":1.903},"min":{"1min":0.36099999999999999,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":11,"5min":13.933,"15min":13.933},"last":7.3289999999999997},{"interface":"front","average":{"1min":2.0129999999999999,"5min":2.1560000000000001,"15min":1.9550000000000001},"min":{"1min":0.41299999999999998,"5min":0.379,"15min":0.377},"max":{"1min":11.066000000000001,"5min":13.962,"15min":13.962},"last":6.2969999999999997}]},{"osd":7,"last update":"Sat Sep 23 21:04:01 2023","interfaces":[{"interface":"back","average":{"1min":1.9910000000000001,"5min":2.1509999999999998,"15min":1.704},"min":{"1min":0.36199999999999999,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":5.5990000000000002,"5min":11.657,"15min":11.657},"last":0.40500000000000003},{"interface":"front","average":{"1min":4.1859999999999999,"5min":2.5979999999999999,"15min":1.804},"min":{"1min":0.47299999999999998,"5min":0.41499999999999998,"15min":0.377},"max":{"1min":40.984999999999999,"5min":40.984999999999999,"15min":40.984999999999999},"last":8.1609999999999996}]}]},{"osd":2,"up_from":93,"seq":399431958699,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1593280,"kb_used_data":1133560,"kb_used_omap":81,"kb_used_meta":459502,"kb_avail":92151872,"statfs":{"total":95995035648,"available":94363516928,"internally_reserved":0,"allocated":1160765440,"data_stored":1159275697,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":83750,"internal_metadata":470530266},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:03:52 2023","interfaces":[{"interface":"back","average":{"1min":1.873,"5min":2.7090000000000001,"15min":1.863},"min":{"1min":0.22600000000000001,"5min":0.189,"15min":0.17999999999999999},"max":{"1min":7.4649999999999999,"5min":29.498000000000001,"15min":29.498000000000001},"last":0.78500000000000003},{"interface":"front","average":{"1min":2.1459999999999999,"5min":2.6150000000000002,"15min":2.1680000000000001},"min":{"1min":0.27000000000000002,"5min":0.193,"15min":0.193},"max":{"1min":7.5839999999999996,"5min":17.251000000000001,"15min":17.251000000000001},"last":9.2880000000000003}]},{"osd":1,"last update":"Sat Sep 23 21:03:52 2023","interfaces":[{"interface":"back","average":{"1min":2.29,"5min":2.3479999999999999,"15min":2.1240000000000001},"min":{"1min":0.27900000000000003,"5min":0.246,"15min":0.20200000000000001},"max":{"1min":10.698,"5min":13.523,"15min":16.081},"last":0.53000000000000003},{"interface":"front","average":{"1min":2.1589999999999998,"5min":3.1749999999999998,"15min":2.6880000000000002},"min":{"1min":0.32200000000000001,"5min":0.26300000000000001,"15min":0.23599999999999999},"max":{"1min":8.8170000000000002,"5min":26.552,"15min":26.552},"last":8.827}]},{"osd":3,"last update":"Sat Sep 23 21:04:38 2023","interfaces":[{"interface":"back","average":{"1min":1.5349999999999999,"5min":3.1760000000000002,"15min":2.8140000000000001},"min":{"1min":0.27000000000000002,"5min":0.251,"15min":0.23300000000000001},"max":{"1min":7.4790000000000001,"5min":41.057000000000002,"15min":41.057000000000002},"last":20.079999999999998},{"interface":"front","average":{"1min":1.7290000000000001,"5min":3.4510000000000001,"15min":3.2170000000000001},"min":{"1min":0.27100000000000002,"5min":0.25,"15min":0.221},"max":{"1min":8.7089999999999996,"5min":41.012,"15min":41.012},"last":3.8940000000000001}]},{"osd":4,"last update":"Sat Sep 23 21:03:52 2023","interfaces":[{"interface":"back","average":{"1min":1.405,"5min":1.4279999999999999,"15min":1.413},"min":{"1min":0.38300000000000001,"5min":0.28399999999999997,"15min":0.26400000000000001},"max":{"1min":4.6470000000000002,"5min":10.045,"15min":10.045},"last":8.7569999999999997},{"interface":"front","average":{"1min":2.3130000000000002,"5min":2.3210000000000002,"15min":1.7769999999999999},"min":{"1min":0.36099999999999999,"5min":0.32400000000000001,"15min":0.313},"max":{"1min":9.6359999999999992,"5min":14.039999999999999,"15min":14.039999999999999},"last":0.45100000000000001}]},{"osd":5,"last update":"Sat Sep 23 21:03:52 2023","interfaces":[{"interface":"back","average":{"1min":1.3149999999999999,"5min":1.488,"15min":1.526},"min":{"1min":0.34499999999999997,"5min":0.30399999999999999,"15min":0.29299999999999998},"max":{"1min":4.6820000000000004,"5min":10.069000000000001,"15min":10.069000000000001},"last":8.907},{"interface":"front","average":{"1min":2.4260000000000002,"5min":2.2050000000000001,"15min":1.766},"min":{"1min":0.38700000000000001,"5min":0.31900000000000001,"15min":0.30299999999999999},"max":{"1min":9.7050000000000001,"5min":13.98,"15min":13.98},"last":0.49299999999999999}]},{"osd":6,"last update":"Sat Sep 23 21:03:50 2023","interfaces":[{"interface":"back","average":{"1min":2.4430000000000001,"5min":2.3969999999999998,"15min":1.96},"min":{"1min":0.44800000000000001,"5min":0.35699999999999998,"15min":0.312},"max":{"1min":9.6820000000000004,"5min":14.016,"15min":14.244999999999999},"last":0.40500000000000003},{"interface":"front","average":{"1min":2.4209999999999998,"5min":2.476,"15min":1.6499999999999999},"min":{"1min":0.41299999999999998,"5min":0.38,"15min":0.374},"max":{"1min":9.734,"5min":14.068,"15min":14.068},"last":0.68999999999999995}]},{"osd":7,"last update":"Sat Sep 23 21:03:52 2023","interfaces":[{"interface":"back","average":{"1min":1.3300000000000001,"5min":1.335,"15min":1.228},"min":{"1min":0.39200000000000002,"5min":0.35899999999999999,"15min":0.315},"max":{"1min":3.5779999999999998,"5min":13.427,"15min":13.427},"last":0.56699999999999995},{"interface":"front","average":{"1min":1.4510000000000001,"5min":1.512,"15min":1.5449999999999999},"min":{"1min":0.41199999999999998,"5min":0.36699999999999999,"15min":0.34499999999999997},"max":{"1min":4.71,"5min":10.092000000000001,"15min":10.092000000000001},"last":8.8689999999999998}]}]},{"osd":3,"up_from":98,"seq":420906795172,"num_pgs":51,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1586392,"kb_used_data":1084944,"kb_used_omap":7,"kb_used_meta":501432,"kb_avail":92158760,"statfs":{"total":95995035648,"available":94370570240,"internally_reserved":0,"allocated":1110982656,"data_stored":1109522834,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7576,"internal_metadata":513466984},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:04:39 2023","interfaces":[{"interface":"back","average":{"1min":2.476,"5min":3.2290000000000001,"15min":2.6880000000000002},"min":{"1min":0.23300000000000001,"5min":0.188,"15min":0.188},"max":{"1min":10.052,"5min":27.962,"15min":27.962},"last":0.65900000000000003},{"interface":"front","average":{"1min":2.5790000000000002,"5min":3.2040000000000002,"15min":2.5739999999999998},"min":{"1min":0.20599999999999999,"5min":0.20499999999999999,"15min":0.19800000000000001},"max":{"1min":11.483000000000001,"5min":28.058,"15min":28.058},"last":0.63800000000000001}]},{"osd":1,"last update":"Sat Sep 23 21:04:39 2023","interfaces":[{"interface":"back","average":{"1min":2.8239999999999998,"5min":3.5630000000000002,"15min":2.677},"min":{"1min":0.28399999999999997,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":10.233000000000001,"5min":74.093000000000004,"15min":74.093000000000004},"last":4.3029999999999999},{"interface":"front","average":{"1min":2.3759999999999999,"5min":3.7170000000000001,"15min":2.7930000000000001},"min":{"1min":0.22600000000000001,"5min":0.22600000000000001,"15min":0.218},"max":{"1min":10.087999999999999,"5min":76.527000000000001,"15min":76.527000000000001},"last":1.246}]},{"osd":2,"last update":"Sat Sep 23 21:04:39 2023","interfaces":[{"interface":"back","average":{"1min":2.4620000000000002,"5min":3.2570000000000001,"15min":2.6749999999999998},"min":{"1min":0.30399999999999999,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":9.5609999999999999,"5min":28.251999999999999,"15min":28.251999999999999},"last":4.3769999999999998},{"interface":"front","average":{"1min":2.516,"5min":3.1779999999999999,"15min":2.7050000000000001},"min":{"1min":0.29399999999999998,"5min":0.22,"15min":0.22},"max":{"1min":10.73,"5min":18.321000000000002,"15min":25.305},"last":4.3940000000000001}]},{"osd":4,"last update":"Sat Sep 23 21:04:39 2023","interfaces":[{"interface":"back","average":{"1min":1.927,"5min":2.2719999999999998,"15min":1.8460000000000001},"min":{"1min":0.34100000000000003,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":7.9109999999999996,"5min":18.239000000000001,"15min":18.239000000000001},"last":3.7989999999999999},{"interface":"front","average":{"1min":2.4900000000000002,"5min":2.351,"15min":1.9099999999999999},"min":{"1min":0.32200000000000001,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":9.9649999999999999,"5min":14.808,"15min":22.853000000000002},"last":1.276}]},{"osd":5,"last update":"Sat Sep 23 21:04:39 2023","interfaces":[{"interface":"back","average":{"1min":2.2069999999999999,"5min":2.4060000000000001,"15min":1.879},"min":{"1min":0.36299999999999999,"5min":0.318,"15min":0.318},"max":{"1min":7.9829999999999997,"5min":18.324999999999999,"15min":18.324999999999999},"last":3.8660000000000001},{"interface":"front","average":{"1min":1.978,"5min":2.7429999999999999,"15min":1.974},"min":{"1min":0.34499999999999997,"5min":0.33700000000000002,"15min":0.33100000000000002},"max":{"1min":6.9640000000000004,"5min":13.529999999999999,"15min":13.529999999999999},"last":2.6659999999999999}]},{"osd":6,"last update":"Sat Sep 23 21:03:56 2023","interfaces":[{"interface":"back","average":{"1min":2.2879999999999998,"5min":2.3050000000000002,"15min":1.4830000000000001},"min":{"1min":0.33900000000000002,"5min":0.33700000000000002,"15min":0.33500000000000002},"max":{"1min":10.016,"5min":14.833,"15min":14.833},"last":3.8220000000000001},{"interface":"front","average":{"1min":2.1989999999999998,"5min":2.8879999999999999,"15min":1.9259999999999999},"min":{"1min":0.38400000000000001,"5min":0.373,"15min":0.373},"max":{"1min":8.5570000000000004,"5min":13.576000000000001,"15min":13.576000000000001},"last":1.508}]},{"osd":7,"last update":"Sat Sep 23 21:04:39 2023","interfaces":[{"interface":"back","average":{"1min":2.5270000000000001,"5min":2.5550000000000002,"15min":2.0870000000000002},"min":{"1min":0.44800000000000001,"5min":0.39200000000000002,"15min":0.36099999999999999},"max":{"1min":10.603,"5min":18.289999999999999,"15min":18.289999999999999},"last":3.8380000000000001},{"interface":"front","average":{"1min":2.3879999999999999,"5min":2.879,"15min":2.0720000000000001},"min":{"1min":0.41899999999999998,"5min":0.38900000000000001,"15min":0.379},"max":{"1min":12.624000000000001,"5min":13.603999999999999,"15min":13.603999999999999},"last":0.59699999999999998}]}]},{"osd":4,"up_from":31,"seq":133143986480,"num_pgs":51,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1197776,"kb_used_data":970312,"kb_used_omap":0,"kb_used_meta":227456,"kb_avail":92547376,"statfs":{"total":95995035648,"available":94768513024,"internally_reserved":0,"allocated":993599488,"data_stored":992559017,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":232914944},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:04:00 2023","interfaces":[{"interface":"back","average":{"1min":1.9550000000000001,"5min":1.53,"15min":1.341},"min":{"1min":0.22800000000000001,"5min":0.218,"15min":0.214},"max":{"1min":10.757,"5min":10.757,"15min":11.784000000000001},"last":2.1379999999999999},{"interface":"front","average":{"1min":2.4009999999999998,"5min":1.7390000000000001,"15min":1.504},"min":{"1min":0.27200000000000002,"5min":0.27200000000000002,"15min":0.25800000000000001},"max":{"1min":10.744,"5min":10.744,"15min":11.766},"last":2.1829999999999998}]},{"osd":1,"last update":"Sat Sep 23 21:04:30 2023","interfaces":[{"interface":"back","average":{"1min":2.0070000000000001,"5min":1.8700000000000001,"15min":1.5569999999999999},"min":{"1min":0.309,"5min":0.255,"15min":0.23599999999999999},"max":{"1min":16.210000000000001,"5min":16.210000000000001,"15min":16.210000000000001},"last":0.252},{"interface":"front","average":{"1min":1.2330000000000001,"5min":1.784,"15min":1.5369999999999999},"min":{"1min":0.30099999999999999,"5min":0.29199999999999998,"15min":0.23200000000000001},"max":{"1min":3.2000000000000002,"5min":12.441000000000001,"15min":12.441000000000001},"last":0.38800000000000001}]},{"osd":2,"last update":"Sat Sep 23 21:04:00 2023","interfaces":[{"interface":"back","average":{"1min":2.532,"5min":2.173,"15min":1.611},"min":{"1min":0.377,"5min":0.309,"15min":0.29099999999999998},"max":{"1min":10.906000000000001,"5min":10.906000000000001,"15min":15.288},"last":8.3610000000000007},{"interface":"front","average":{"1min":2.6179999999999999,"5min":1.9890000000000001,"15min":1.6679999999999999},"min":{"1min":0.307,"5min":0.28599999999999998,"15min":0.28299999999999997},"max":{"1min":8.9329999999999998,"5min":12.855,"15min":15.262},"last":1.2230000000000001}]},{"osd":3,"last update":"Sat Sep 23 21:04:30 2023","interfaces":[{"interface":"back","average":{"1min":1.8460000000000001,"5min":2.5489999999999999,"15min":2.2509999999999999},"min":{"1min":0.30499999999999999,"5min":0.30499999999999999,"15min":0.28899999999999998},"max":{"1min":4.9329999999999998,"5min":12.172000000000001,"15min":31.562999999999999},"last":4.0979999999999999},{"interface":"front","average":{"1min":1.4239999999999999,"5min":2.2789999999999999,"15min":2.1960000000000002},"min":{"1min":0.30199999999999999,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":5.0289999999999999,"5min":12.477,"15min":31.574000000000002},"last":4.1159999999999997}]},{"osd":5,"last update":"Sat Sep 23 21:03:44 2023","interfaces":[{"interface":"back","average":{"1min":1.1950000000000001,"5min":0.67100000000000004,"15min":0.59899999999999998},"min":{"1min":0.253,"5min":0.251,"15min":0.22700000000000001},"max":{"1min":4.327,"5min":6.6470000000000002,"15min":6.6470000000000002},"last":0.32400000000000001},{"interface":"front","average":{"1min":1.153,"5min":0.91800000000000004,"15min":0.72499999999999998},"min":{"1min":0.28999999999999998,"5min":0.27600000000000002,"15min":0.251},"max":{"1min":6.4029999999999996,"5min":6.4029999999999996,"15min":7.7619999999999996},"last":0.374}]},{"osd":6,"last update":"Sat Sep 23 21:03:54 2023","interfaces":[{"interface":"back","average":{"1min":1.9099999999999999,"5min":1.0720000000000001,"15min":0.74399999999999999},"min":{"1min":0.33400000000000002,"5min":0.27300000000000002,"15min":0.26500000000000001},"max":{"1min":7.4139999999999997,"5min":7.4139999999999997,"15min":7.4139999999999997},"last":0.28999999999999998},{"interface":"front","average":{"1min":0.89400000000000002,"5min":0.752,"15min":0.70399999999999996},"min":{"1min":0.33900000000000002,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":7.1740000000000004,"5min":7.1740000000000004,"15min":7.1740000000000004},"last":0.41999999999999998}]},{"osd":7,"last update":"Sat Sep 23 21:04:30 2023","interfaces":[{"interface":"back","average":{"1min":1.2889999999999999,"5min":0.82999999999999996,"15min":0.77900000000000003},"min":{"1min":0.316,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":14.460000000000001,"5min":14.460000000000001,"15min":14.460000000000001},"last":0.34899999999999998},{"interface":"front","average":{"1min":1.373,"5min":0.95499999999999996,"15min":0.80700000000000005},"min":{"1min":0.32900000000000001,"5min":0.32700000000000001,"15min":0.28199999999999997},"max":{"1min":4.9880000000000004,"5min":6.4459999999999997,"15min":7.8310000000000004},"last":0.503}]}]},{"osd":5,"up_from":39,"seq":167503724845,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1562248,"kb_used_data":1325440,"kb_used_omap":0,"kb_used_meta":236736,"kb_avail":92182904,"statfs":{"total":95995035648,"available":94395293696,"internally_reserved":0,"allocated":1357250560,"data_stored":1356214593,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":242417664},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:03:47 2023","interfaces":[{"interface":"back","average":{"1min":1.95,"5min":2.4140000000000001,"15min":1.3660000000000001},"min":{"1min":0.22600000000000001,"5min":0.22500000000000001,"15min":0.20399999999999999},"max":{"1min":10.278,"5min":12.795999999999999,"15min":12.795999999999999},"last":1.365},{"interface":"front","average":{"1min":2.8260000000000001,"5min":2.3439999999999999,"15min":1.5700000000000001},"min":{"1min":0.26700000000000002,"5min":0.23400000000000001,"15min":0.223},"max":{"1min":9.7479999999999993,"5min":16.219999999999999,"15min":16.219999999999999},"last":0.77900000000000003}]},{"osd":1,"last update":"Sat Sep 23 21:04:25 2023","interfaces":[{"interface":"back","average":{"1min":1.5509999999999999,"5min":2.508,"15min":1.7470000000000001},"min":{"1min":0.25800000000000001,"5min":0.25800000000000001,"15min":0.23300000000000001},"max":{"1min":6.4790000000000001,"5min":62.320999999999998,"15min":62.320999999999998},"last":1.708},{"interface":"front","average":{"1min":2.1579999999999999,"5min":2.806,"15min":1.851},"min":{"1min":0.26700000000000002,"5min":0.249,"15min":0.22800000000000001},"max":{"1min":7.7510000000000003,"5min":61.722999999999999,"15min":61.722999999999999},"last":0.32100000000000001}]},{"osd":2,"last update":"Sat Sep 23 21:04:04 2023","interfaces":[{"interface":"back","average":{"1min":1.7270000000000001,"5min":1.9179999999999999,"15min":1.4370000000000001},"min":{"1min":0.28899999999999998,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":6.8079999999999998,"5min":10.371,"15min":12.651},"last":0.44400000000000001},{"interface":"front","average":{"1min":2.4169999999999998,"5min":2.2229999999999999,"15min":1.631},"min":{"1min":0.28899999999999998,"5min":0.28599999999999998,"15min":0.26100000000000001},"max":{"1min":6.9800000000000004,"5min":10.333,"15min":12.300000000000001},"last":0.48799999999999999}]},{"osd":3,"last update":"Sat Sep 23 21:04:25 2023","interfaces":[{"interface":"back","average":{"1min":1.8959999999999999,"5min":3.488,"15min":2.306},"min":{"1min":0.38700000000000001,"5min":0.313,"15min":0.28599999999999998},"max":{"1min":8.3989999999999991,"5min":61.667999999999999,"15min":61.667999999999999},"last":1.0169999999999999},{"interface":"front","average":{"1min":1.109,"5min":3.2050000000000001,"15min":1.9279999999999999},"min":{"1min":0.35699999999999998,"5min":0.32400000000000001,"15min":0.28899999999999998},"max":{"1min":2.7370000000000001,"5min":62.276000000000003,"15min":62.276000000000003},"last":0.88200000000000001}]},{"osd":4,"last update":"Sat Sep 23 21:04:40 2023","interfaces":[{"interface":"back","average":{"1min":1.22,"5min":0.88,"15min":0.68600000000000005},"min":{"1min":0.28899999999999998,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":9.0950000000000006,"5min":9.0950000000000006,"15min":9.0950000000000006},"last":0.38200000000000001},{"interface":"front","average":{"1min":0.94699999999999995,"5min":0.84499999999999997,"15min":0.68799999999999994},"min":{"1min":0.32100000000000001,"5min":0.28000000000000003,"15min":0.23599999999999999},"max":{"1min":4.9589999999999996,"5min":8.4350000000000005,"15min":8.4350000000000005},"last":0.35599999999999998}]},{"osd":6,"last update":"Sat Sep 23 21:03:47 2023","interfaces":[{"interface":"back","average":{"1min":1.1319999999999999,"5min":1,"15min":0.71699999999999997},"min":{"1min":0.27500000000000002,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":4.1289999999999996,"5min":11.128,"15min":11.128},"last":0.28799999999999998},{"interface":"front","average":{"1min":1.2210000000000001,"5min":1.0649999999999999,"15min":0.83599999999999997},"min":{"1min":0.38100000000000001,"5min":0.36799999999999999,"15min":0.33200000000000002},"max":{"1min":4.8099999999999996,"5min":11.112,"15min":11.112},"last":0.46300000000000002}]},{"osd":7,"last update":"Sat Sep 23 21:04:25 2023","interfaces":[{"interface":"back","average":{"1min":1.123,"5min":0.97399999999999998,"15min":0.78000000000000003},"min":{"1min":0.35899999999999999,"5min":0.309,"15min":0.30199999999999999},"max":{"1min":4.1950000000000003,"5min":4.4240000000000004,"15min":8.5649999999999995},"last":0.36799999999999999},{"interface":"front","average":{"1min":0.84699999999999998,"5min":0.88300000000000001,"15min":0.75900000000000001},"min":{"1min":0.32500000000000001,"5min":0.32500000000000001,"15min":0.27700000000000002},"max":{"1min":2.9180000000000001,"5min":4.6479999999999997,"15min":4.6479999999999997},"last":0.42899999999999999}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1040318464,"data_stored":1040318464,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1169948672,"data_stored":1169948672,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1156321280,"data_stored":1156317208,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1101987840,"data_stored":1101987840,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":963510272,"data_stored":963510272,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1322258432,"data_stored":1322254360,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":961806336,"data_stored":961806336,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1065684992,"data_stored":1065680920,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T21:04:46.088 INFO:tasks.cephadm.ceph_manager.ceph:recovered! 2023-09-23T21:04:46.089 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T21:04:46.089 INFO:tasks.thrashosds.thrasher:Removing osd 5, in_osds are: [0, 1, 2, 3, 6, 7, 4, 5] 2023-09-23T21:04:46.089 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph osd out 5 2023-09-23T21:04:46.161 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:04:46.242 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:04:46.352 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:04:46 smithi026 ceph-mon[175151]: from='client.35730 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T21:04:46.352 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:04:46 smithi026 ceph-mon[175151]: pgmap v830: 119 pgs: 119 active+clean; 2.3 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 928 op/s 2023-09-23T21:04:46.352 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:04:46 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:04:46.352 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:04:46 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:04:46.352 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:04:46 smithi026 ceph-mon[179688]: from='client.35730 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T21:04:46.352 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:04:46 smithi026 ceph-mon[179688]: pgmap v830: 119 pgs: 119 active+clean; 2.3 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 928 op/s 2023-09-23T21:04:46.352 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:04:46 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:04:46.352 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:04:46 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:04:46.374 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:04:46.380 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:04:46.382 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T21:04:46.387 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:04:46.387 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:04:46.488 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:04:46.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:04:46 smithi070 ceph-mon[168282]: from='client.35730 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T21:04:46.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:04:46 smithi070 ceph-mon[168282]: pgmap v830: 119 pgs: 119 active+clean; 2.3 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 928 op/s 2023-09-23T21:04:46.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:04:46 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:04:46.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:04:46 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:04:46.631 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:04:46.733 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:04:46.743 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:04:46.744 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:04:46.845 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:04:46.878 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:04:46.920 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:04:46.925 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:04:46.925 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:04:47.026 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:04:47.126 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:04:47.448 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:04:47.453 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:04:47.453 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:04:47.504 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:04:47.555 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:04:47.645 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:04:47.704 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:04:47.722 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:04:47.728 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:04:47.728 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:04:47.829 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:04:47.870 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:04:47.881 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:04:47 smithi026 ceph-mon[179688]: pgmap v831: 119 pgs: 119 active+clean; 2.2 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 935 op/s 2023-09-23T21:04:47.881 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:04:47 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:04:47.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:04:47 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/4066919678' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T21:04:47.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:04:47 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T21:04:47.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:04:47 smithi026 ceph-mon[175151]: pgmap v831: 119 pgs: 119 active+clean; 2.2 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 935 op/s 2023-09-23T21:04:47.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:04:47 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:04:47.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:04:47 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/4066919678' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T21:04:47.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:04:47 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T21:04:47.930 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:04:47.935 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:04:47.935 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:04:48.036 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:04:48.117 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:04:48.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:04:47 smithi070 ceph-mon[168282]: pgmap v831: 119 pgs: 119 active+clean; 2.2 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 935 op/s 2023-09-23T21:04:48.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:04:47 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:04:48.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:04:47 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/4066919678' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T21:04:48.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:04:47 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T21:04:48.237 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T21:04:48.249 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:04:48.250 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:04:48.340 INFO:teuthology.orchestra.run.smithi026.stderr:noscrub is unset 2023-09-23T21:04:48.351 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:04:48.381 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:04:48.431 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:04:48.436 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:04:48.437 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:04:48.537 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:04:48.570 INFO:teuthology.orchestra.run.smithi026.stdout:{ 2023-09-23T21:04:48.570 INFO:teuthology.orchestra.run.smithi026.stdout: "ops": [ 2023-09-23T21:04:48.570 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:48.570 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_repop(client.35469.0:189929 8.e e323/254 8:76c702a4:::benchmark_data_smithi026_301269_object79172:head v 323'12085, mlcod=323'12083)", 2023-09-23T21:04:48.570 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T21:04:48.562732+0000", 2023-09-23T21:04:48.570 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 0.0060656820000000002, 2023-09-23T21:04:48.571 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0060979939999999998, 2023-09-23T21:04:48.571 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:04:48.571 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "started", 2023-09-23T21:04:48.571 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:04:48.571 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:48.571 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:04:48.571 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:48.562732+0000", 2023-09-23T21:04:48.571 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:04:48.571 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:48.571 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:48.571 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:04:48.571 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:48.562732+0000", 2023-09-23T21:04:48.572 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:04:48.572 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:48.572 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:48.572 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:04:48.572 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:48.562731+0000", 2023-09-23T21:04:48.572 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.999999 2023-09-23T21:04:48.572 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:48.572 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:48.572 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:04:48.572 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:48.562740+0000", 2023-09-23T21:04:48.572 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.2839999999999995e-06 2023-09-23T21:04:48.572 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:48.573 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:48.573 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:04:48.573 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:48.562757+0000", 2023-09-23T21:04:48.573 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.7269999999999999e-05 2023-09-23T21:04:48.573 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:48.573 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:48.573 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:04:48.573 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:48.562766+0000", 2023-09-23T21:04:48.573 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.6959999999999994e-06 2023-09-23T21:04:48.573 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:48.573 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:48.573 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:04:48.574 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:48.562843+0000", 2023-09-23T21:04:48.574 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.7612000000000002e-05 2023-09-23T21:04:48.574 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:48.574 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:48.574 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:04:48.574 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:48.562867+0000", 2023-09-23T21:04:48.574 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.3121000000000001e-05 2023-09-23T21:04:48.574 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:04:48.574 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:04:48.574 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:04:48.574 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:48.574 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:48.575 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_repop(client.35469.0:189932 8.16 e323/314 8:6f0ab446:::benchmark_data_smithi026_301269_object79175:head v 323'10367, mlcod=323'10365)", 2023-09-23T21:04:48.575 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T21:04:48.567570+0000", 2023-09-23T21:04:48.575 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 0.0012274899999999999, 2023-09-23T21:04:48.575 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0013298489999999999, 2023-09-23T21:04:48.575 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:04:48.575 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "started", 2023-09-23T21:04:48.575 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:04:48.575 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:48.575 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:04:48.575 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:48.567570+0000", 2023-09-23T21:04:48.575 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:04:48.575 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:48.575 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:48.576 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:04:48.576 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:48.567570+0000", 2023-09-23T21:04:48.576 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:04:48.576 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:48.576 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:48.576 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:04:48.576 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:48.567569+0000", 2023-09-23T21:04:48.576 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.999999 2023-09-23T21:04:48.576 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:48.576 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:48.576 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:04:48.576 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:48.567579+0000", 2023-09-23T21:04:48.576 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0162e-05 2023-09-23T21:04:48.577 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:48.577 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:48.577 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:04:48.577 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:48.567596+0000", 2023-09-23T21:04:48.577 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.6612e-05 2023-09-23T21:04:48.577 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:48.577 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:48.577 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:04:48.577 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:48.567604+0000", 2023-09-23T21:04:48.577 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.3489999999999992e-06 2023-09-23T21:04:48.577 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:48.577 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:48.578 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:04:48.578 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:48.567650+0000", 2023-09-23T21:04:48.578 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.6090000000000001e-05 2023-09-23T21:04:48.578 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:48.578 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:48.578 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:04:48.578 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:48.567665+0000", 2023-09-23T21:04:48.578 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.4979e-05 2023-09-23T21:04:48.578 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:04:48.578 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:04:48.578 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:04:48.578 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:48.578 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:48.579 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_repop(client.35469.0:189937 8.12 e323/307 8:4fe99530:::benchmark_data_smithi026_301269_object79180:head v 323'10198, mlcod=323'10196)", 2023-09-23T21:04:48.579 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T21:04:48.568345+0000", 2023-09-23T21:04:48.579 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 0.00045266499999999999, 2023-09-23T21:04:48.579 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00061727699999999995, 2023-09-23T21:04:48.579 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:04:48.579 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "started", 2023-09-23T21:04:48.579 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:04:48.579 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:48.579 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:04:48.579 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:48.568345+0000", 2023-09-23T21:04:48.579 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:04:48.579 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:48.580 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:48.580 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:04:48.580 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:48.568345+0000", 2023-09-23T21:04:48.580 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:04:48.580 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:48.580 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:48.580 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:04:48.580 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:48.568344+0000", 2023-09-23T21:04:48.580 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T21:04:48.580 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:48.580 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:48.580 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:04:48.580 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:48.568350+0000", 2023-09-23T21:04:48.581 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.2940000000000002e-06 2023-09-23T21:04:48.581 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:48.581 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:48.581 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:04:48.581 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:48.568365+0000", 2023-09-23T21:04:48.581 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.4295e-05 2023-09-23T21:04:48.581 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:48.581 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:48.581 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:04:48.581 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:48.568371+0000", 2023-09-23T21:04:48.581 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.5880000000000003e-06 2023-09-23T21:04:48.581 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:48.582 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:48.582 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:04:48.582 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:48.568429+0000", 2023-09-23T21:04:48.582 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.817e-05 2023-09-23T21:04:48.582 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:04:48.582 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:04:48.582 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:04:48.582 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:04:48.568450+0000", 2023-09-23T21:04:48.582 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.033e-05 2023-09-23T21:04:48.582 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:04:48.582 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:04:48.582 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:04:48.583 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:04:48.583 INFO:teuthology.orchestra.run.smithi026.stdout: ], 2023-09-23T21:04:48.583 INFO:teuthology.orchestra.run.smithi026.stdout: "num_ops": 3 2023-09-23T21:04:48.583 INFO:teuthology.orchestra.run.smithi026.stdout:} 2023-09-23T21:04:48.583 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:04:48.644 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:04:48.653 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:04:48.653 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:04:48.754 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:04:48.805 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:04:48.861 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:04:48.866 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:04:48.866 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:04:48.967 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:04:49.061 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:04:49.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:04:48 smithi070 ceph-mon[168282]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:04:49.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:04:48 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T21:04:49.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:04:48 smithi070 ceph-mon[168282]: osdmap e323: 8 total, 8 up, 8 in 2023-09-23T21:04:49.131 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:04:48 smithi026 ceph-mon[179688]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:04:49.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:04:48 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T21:04:49.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:04:48 smithi026 ceph-mon[179688]: osdmap e323: 8 total, 8 up, 8 in 2023-09-23T21:04:49.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:04:48 smithi026 ceph-mon[175151]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:04:49.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:04:48 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T21:04:49.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:04:48 smithi026 ceph-mon[175151]: osdmap e323: 8 total, 8 up, 8 in 2023-09-23T21:04:49.374 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:04:49.379 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:04:49.379 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:04:49.453 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_blocked_ops 2023-09-23T21:04:49.480 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:04:49.516 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:04:49.562 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:04:49.571 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:04:49.571 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:04:49.672 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:04:49.754 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 21:04:49 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[164029]: ::ffff:172.21.15.70 - - [23/Sep/2023:21:04:49] "GET /metrics HTTP/1.1" 200 36291 "" "Prometheus/2.43.0" 2023-09-23T21:04:49.760 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:04:49.843 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T21:04:49.868 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:04:49.868 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:04:49.969 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:04:50.002 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:04:50.050 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:04:50.053 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:04:50.053 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:04:50.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:04:49 smithi070 ceph-mon[168282]: failed to encode map e323 with expected crc 2023-09-23T21:04:50.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:04:49 smithi070 ceph-mon[168282]: failed to encode map e323 with expected crc 2023-09-23T21:04:50.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:04:49 smithi070 ceph-mon[168282]: failed to encode map e323 with expected crc 2023-09-23T21:04:50.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:04:49 smithi070 ceph-mon[168282]: 8.2 scrub starts 2023-09-23T21:04:50.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:04:49 smithi070 ceph-mon[168282]: 8.13 scrub starts 2023-09-23T21:04:50.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:04:49 smithi070 ceph-mon[168282]: 8.2 scrub ok 2023-09-23T21:04:50.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:04:49 smithi070 ceph-mon[168282]: pgmap v833: 119 pgs: 119 active+clean; 2.1 GiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 813 op/s 2023-09-23T21:04:50.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:04:49 smithi070 ceph-mon[168282]: 8.13 scrub ok 2023-09-23T21:04:50.135 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:04:49 smithi026 ceph-mon[179688]: failed to encode map e323 with expected crc 2023-09-23T21:04:50.135 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:04:49 smithi026 ceph-mon[179688]: failed to encode map e323 with expected crc 2023-09-23T21:04:50.135 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:04:49 smithi026 ceph-mon[179688]: failed to encode map e323 with expected crc 2023-09-23T21:04:50.135 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:04:49 smithi026 ceph-mon[179688]: 8.2 scrub starts 2023-09-23T21:04:50.135 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:04:49 smithi026 ceph-mon[179688]: 8.13 scrub starts 2023-09-23T21:04:50.135 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:04:49 smithi026 ceph-mon[179688]: 8.2 scrub ok 2023-09-23T21:04:50.135 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:04:49 smithi026 ceph-mon[179688]: pgmap v833: 119 pgs: 119 active+clean; 2.1 GiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 813 op/s 2023-09-23T21:04:50.135 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:04:49 smithi026 ceph-mon[179688]: 8.13 scrub ok 2023-09-23T21:04:50.136 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:04:49 smithi026 ceph-mon[175151]: failed to encode map e323 with expected crc 2023-09-23T21:04:50.136 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:04:49 smithi026 ceph-mon[175151]: failed to encode map e323 with expected crc 2023-09-23T21:04:50.136 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:04:49 smithi026 ceph-mon[175151]: failed to encode map e323 with expected crc 2023-09-23T21:04:50.136 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:04:49 smithi026 ceph-mon[175151]: 8.2 scrub starts 2023-09-23T21:04:50.136 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:04:49 smithi026 ceph-mon[175151]: 8.13 scrub starts 2023-09-23T21:04:50.136 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:04:49 smithi026 ceph-mon[175151]: 8.2 scrub ok 2023-09-23T21:04:50.136 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:04:49 smithi026 ceph-mon[175151]: pgmap v833: 119 pgs: 119 active+clean; 2.1 GiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 813 op/s 2023-09-23T21:04:50.136 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:04:49 smithi026 ceph-mon[175151]: 8.13 scrub ok 2023-09-23T21:04:50.155 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:04:50.223 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:04:50.454 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T21:04:50.460 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:04:50.462 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:04:50.462 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:04:50.563 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:04:50.618 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:04:50.672 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:04:50.676 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:04:50.676 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:04:50.777 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:04:50.842 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:04:51.008 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T21:04:51.008 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0: {} 2023-09-23T21:04:51.009 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1: {} 2023-09-23T21:04:51.032 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2: {} 2023-09-23T21:04:51.079 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3: {} 2023-09-23T21:04:51.115 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4: {} 2023-09-23T21:04:51.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:04:50 smithi070 ceph-mon[168282]: 8.16 scrub starts 2023-09-23T21:04:51.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:04:50 smithi070 ceph-mon[168282]: 8.16 scrub ok 2023-09-23T21:04:51.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:04:50 smithi026 ceph-mon[175151]: 8.16 scrub starts 2023-09-23T21:04:51.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:04:50 smithi026 ceph-mon[175151]: 8.16 scrub ok 2023-09-23T21:04:51.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:04:50 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mon-a[175127]: 2023-09-23T21:04:50.867+0000 7f1e2d487700 -1 mon.a@0(leader).osd e323 definitely_dead 0 2023-09-23T21:04:51.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:04:50 smithi026 ceph-mon[179688]: 8.16 scrub starts 2023-09-23T21:04:51.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:04:50 smithi026 ceph-mon[179688]: 8.16 scrub ok 2023-09-23T21:04:51.158 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T21:04:51.202 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6: {} 2023-09-23T21:04:51.262 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T21:04:51.310 INFO:teuthology.orchestra.run.smithi026.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T21:04:51.310 INFO:teuthology.orchestra.run.smithi026.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T21:04:51.310 INFO:teuthology.orchestra.run.smithi026.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T21:04:51.310 INFO:teuthology.orchestra.run.smithi026.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T21:04:51.310 INFO:teuthology.orchestra.run.smithi026.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T21:04:51.310 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T21:04:51.310 INFO:teuthology.orchestra.run.smithi026.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T21:04:51.310 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T21:04:51.585 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T21:04:51.593 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:04:51.594 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:04:51.694 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:04:51.724 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:04:51.800 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:04:51.806 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:04:51.806 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:04:51.907 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:04:51.920 INFO:teuthology.orchestra.run.smithi026.stderr:marked out osd.5. 2023-09-23T21:04:51.941 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:04:52.086 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T21:04:52.089 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:04:52.089 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:04:52.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:04:51 smithi070 ceph-mon[168282]: pgmap v834: 119 pgs: 1 active+clean+scrubbing, 118 active+clean; 1.9 GiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 350 op/s 2023-09-23T21:04:52.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:04:51 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/3678271272' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["5"]}]: dispatch 2023-09-23T21:04:52.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:04:51 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["5"]}]: dispatch 2023-09-23T21:04:52.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:04:51 smithi070 ceph-mon[168282]: Client client.admin marked osd.5 out, while it was still marked up 2023-09-23T21:04:52.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:04:51 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/3501062478' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:04:52.154 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:04:51 smithi026 ceph-mon[179688]: pgmap v834: 119 pgs: 1 active+clean+scrubbing, 118 active+clean; 1.9 GiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 350 op/s 2023-09-23T21:04:52.154 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:04:51 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/3678271272' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["5"]}]: dispatch 2023-09-23T21:04:52.155 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:04:51 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["5"]}]: dispatch 2023-09-23T21:04:52.155 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:04:51 smithi026 ceph-mon[179688]: Client client.admin marked osd.5 out, while it was still marked up 2023-09-23T21:04:52.155 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:04:51 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/3501062478' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:04:52.155 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:04:51 smithi026 ceph-mon[175151]: pgmap v834: 119 pgs: 1 active+clean+scrubbing, 118 active+clean; 1.9 GiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 350 op/s 2023-09-23T21:04:52.155 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:04:51 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/3678271272' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["5"]}]: dispatch 2023-09-23T21:04:52.155 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:04:51 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["5"]}]: dispatch 2023-09-23T21:04:52.155 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:04:51 smithi026 ceph-mon[175151]: Client client.admin marked osd.5 out, while it was still marked up 2023-09-23T21:04:52.155 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:04:51 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/3501062478' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:04:52.190 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:04:52.216 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:04:52.249 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:04:52.251 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:04:52.251 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:04:52.352 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:04:52.379 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:04:52.412 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:04:52.414 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:04:52.415 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:04:52.516 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:04:52.545 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:04:52.740 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:04:52.743 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:04:52.744 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:04:52.845 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:04:52.891 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:04:52.968 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:04:52.988 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:04:52.992 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:04:52.993 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:04:53.004 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:04:52 smithi026 ceph-mon[179688]: 8.4 scrub starts 2023-09-23T21:04:53.004 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:04:52 smithi026 ceph-mon[179688]: 8.4 scrub ok 2023-09-23T21:04:53.004 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:04:52 smithi026 ceph-mon[179688]: failed to encode map e324 with expected crc 2023-09-23T21:04:53.004 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:04:52 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["5"]}]': finished 2023-09-23T21:04:53.004 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:04:52 smithi026 ceph-mon[179688]: osdmap e324: 8 total, 8 up, 7 in 2023-09-23T21:04:53.004 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:04:52 smithi026 ceph-mon[179688]: failed to encode map e324 with expected crc 2023-09-23T21:04:53.004 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:04:52 smithi026 ceph-mon[179688]: failed to encode map e324 with expected crc 2023-09-23T21:04:53.004 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:04:52 smithi026 ceph-mon[179688]: osdmap e325: 8 total, 8 up, 7 in 2023-09-23T21:04:53.005 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:04:52 smithi026 ceph-mon[175151]: 8.4 scrub starts 2023-09-23T21:04:53.005 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:04:52 smithi026 ceph-mon[175151]: 8.4 scrub ok 2023-09-23T21:04:53.005 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:04:52 smithi026 ceph-mon[175151]: failed to encode map e324 with expected crc 2023-09-23T21:04:53.005 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:04:52 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["5"]}]': finished 2023-09-23T21:04:53.005 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:04:52 smithi026 ceph-mon[175151]: osdmap e324: 8 total, 8 up, 7 in 2023-09-23T21:04:53.005 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:04:52 smithi026 ceph-mon[175151]: failed to encode map e324 with expected crc 2023-09-23T21:04:53.005 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:04:52 smithi026 ceph-mon[175151]: failed to encode map e324 with expected crc 2023-09-23T21:04:53.005 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:04:52 smithi026 ceph-mon[175151]: osdmap e325: 8 total, 8 up, 7 in 2023-09-23T21:04:53.093 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:04:53.124 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:04:53.218 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T21:04:53.221 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:04:53.221 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:04:53.283 INFO:teuthology.orchestra.run.smithi026.stdout:ERROR: (22) Invalid argument 2023-09-23T21:04:53.284 INFO:teuthology.orchestra.run.smithi026.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T21:04:53.322 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:04:53.358 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:04:53.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:04:52 smithi070 ceph-mon[168282]: 8.4 scrub starts 2023-09-23T21:04:53.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:04:52 smithi070 ceph-mon[168282]: 8.4 scrub ok 2023-09-23T21:04:53.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:04:52 smithi070 ceph-mon[168282]: failed to encode map e324 with expected crc 2023-09-23T21:04:53.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:04:52 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["5"]}]': finished 2023-09-23T21:04:53.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:04:52 smithi070 ceph-mon[168282]: osdmap e324: 8 total, 8 up, 7 in 2023-09-23T21:04:53.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:04:52 smithi070 ceph-mon[168282]: failed to encode map e324 with expected crc 2023-09-23T21:04:53.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:04:52 smithi070 ceph-mon[168282]: failed to encode map e324 with expected crc 2023-09-23T21:04:53.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:04:52 smithi070 ceph-mon[168282]: osdmap e325: 8 total, 8 up, 7 in 2023-09-23T21:04:53.713 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T21:04:53.716 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:04:53.716 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:04:53.797 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_historic_ops 2023-09-23T21:04:53.818 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:04:53.871 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:04:54.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:04:53 smithi026 ceph-mon[179688]: 8.14 scrub starts 2023-09-23T21:04:54.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:04:53 smithi026 ceph-mon[179688]: 8.14 scrub ok 2023-09-23T21:04:54.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:04:53 smithi026 ceph-mon[179688]: 8.17 scrub starts 2023-09-23T21:04:54.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:04:53 smithi026 ceph-mon[179688]: 8.17 scrub ok 2023-09-23T21:04:54.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:04:53 smithi026 ceph-mon[179688]: pgmap v836: 119 pgs: 11 peering, 1 unknown, 1 active+clean+scrubbing, 106 active+clean; 1.8 GiB data, 7.8 GiB used, 618 GiB / 626 GiB avail; 5.1 KiB/s rd, 2 op/s 2023-09-23T21:04:54.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:04:53 smithi026 ceph-mon[179688]: failed to encode map e325 with expected crc 2023-09-23T21:04:54.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:04:53 smithi026 ceph-mon[179688]: failed to encode map e325 with expected crc 2023-09-23T21:04:54.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:04:53 smithi026 ceph-mon[179688]: failed to encode map e325 with expected crc 2023-09-23T21:04:54.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:04:53 smithi026 ceph-mon[179688]: failed to encode map e325 with expected crc 2023-09-23T21:04:54.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:04:53 smithi026 ceph-mon[179688]: failed to encode map e325 with expected crc 2023-09-23T21:04:54.134 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:04:53 smithi026 ceph-mon[179688]: failed to encode map e325 with expected crc 2023-09-23T21:04:54.134 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:04:53 smithi026 ceph-mon[179688]: failed to encode map e325 with expected crc 2023-09-23T21:04:54.134 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:04:53 smithi026 ceph-mon[179688]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2023-09-23T21:04:54.134 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:04:53 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T21:04:54.134 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:04:53 smithi026 ceph-mon[179688]: osdmap e326: 8 total, 8 up, 7 in 2023-09-23T21:04:54.134 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:04:53 smithi026 ceph-mon[175151]: 8.14 scrub starts 2023-09-23T21:04:54.134 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:04:53 smithi026 ceph-mon[175151]: 8.14 scrub ok 2023-09-23T21:04:54.134 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:04:53 smithi026 ceph-mon[175151]: 8.17 scrub starts 2023-09-23T21:04:54.135 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:04:53 smithi026 ceph-mon[175151]: 8.17 scrub ok 2023-09-23T21:04:54.135 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:04:53 smithi026 ceph-mon[175151]: pgmap v836: 119 pgs: 11 peering, 1 unknown, 1 active+clean+scrubbing, 106 active+clean; 1.8 GiB data, 7.8 GiB used, 618 GiB / 626 GiB avail; 5.1 KiB/s rd, 2 op/s 2023-09-23T21:04:54.135 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:04:53 smithi026 ceph-mon[175151]: failed to encode map e325 with expected crc 2023-09-23T21:04:54.135 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:04:53 smithi026 ceph-mon[175151]: failed to encode map e325 with expected crc 2023-09-23T21:04:54.135 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:04:53 smithi026 ceph-mon[175151]: failed to encode map e325 with expected crc 2023-09-23T21:04:54.135 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:04:53 smithi026 ceph-mon[175151]: failed to encode map e325 with expected crc 2023-09-23T21:04:54.135 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:04:53 smithi026 ceph-mon[175151]: failed to encode map e325 with expected crc 2023-09-23T21:04:54.135 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:04:53 smithi026 ceph-mon[175151]: failed to encode map e325 with expected crc 2023-09-23T21:04:54.135 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:04:53 smithi026 ceph-mon[175151]: failed to encode map e325 with expected crc 2023-09-23T21:04:54.135 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:04:53 smithi026 ceph-mon[175151]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2023-09-23T21:04:54.135 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:04:53 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T21:04:54.135 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:04:53 smithi026 ceph-mon[175151]: osdmap e326: 8 total, 8 up, 7 in 2023-09-23T21:04:54.174 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T21:04:54.181 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:04:54.190 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:04:54.191 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:04:54.291 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:04:54.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:04:53 smithi070 ceph-mon[168282]: 8.14 scrub starts 2023-09-23T21:04:54.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:04:53 smithi070 ceph-mon[168282]: 8.14 scrub ok 2023-09-23T21:04:54.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:04:53 smithi070 ceph-mon[168282]: 8.17 scrub starts 2023-09-23T21:04:54.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:04:53 smithi070 ceph-mon[168282]: 8.17 scrub ok 2023-09-23T21:04:54.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:04:53 smithi070 ceph-mon[168282]: pgmap v836: 119 pgs: 11 peering, 1 unknown, 1 active+clean+scrubbing, 106 active+clean; 1.8 GiB data, 7.8 GiB used, 618 GiB / 626 GiB avail; 5.1 KiB/s rd, 2 op/s 2023-09-23T21:04:54.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:04:53 smithi070 ceph-mon[168282]: failed to encode map e325 with expected crc 2023-09-23T21:04:54.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:04:53 smithi070 ceph-mon[168282]: failed to encode map e325 with expected crc 2023-09-23T21:04:54.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:04:53 smithi070 ceph-mon[168282]: failed to encode map e325 with expected crc 2023-09-23T21:04:54.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:04:53 smithi070 ceph-mon[168282]: failed to encode map e325 with expected crc 2023-09-23T21:04:54.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:04:53 smithi070 ceph-mon[168282]: failed to encode map e325 with expected crc 2023-09-23T21:04:54.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:04:53 smithi070 ceph-mon[168282]: failed to encode map e325 with expected crc 2023-09-23T21:04:54.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:04:53 smithi070 ceph-mon[168282]: failed to encode map e325 with expected crc 2023-09-23T21:04:54.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:04:53 smithi070 ceph-mon[168282]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2023-09-23T21:04:54.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:04:53 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T21:04:54.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:04:53 smithi070 ceph-mon[168282]: osdmap e326: 8 total, 8 up, 7 in 2023-09-23T21:04:54.435 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:04:54.660 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:04:54.706 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:04:54.711 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:04:54.712 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:04:54.812 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:04:54.881 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:04:55.135 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:04:54 smithi026 ceph-mon[179688]: failed to encode map e326 with expected crc 2023-09-23T21:04:55.135 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:04:54 smithi026 ceph-mon[179688]: failed to encode map e326 with expected crc 2023-09-23T21:04:55.135 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:04:54 smithi026 ceph-mon[179688]: failed to encode map e326 with expected crc 2023-09-23T21:04:55.135 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:04:54 smithi026 ceph-mon[179688]: failed to encode map e326 with expected crc 2023-09-23T21:04:55.135 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:04:54 smithi026 ceph-mon[179688]: 8.11 starting backfill to osd.1 from (258'1600,299'9240] MIN to 323'10346 2023-09-23T21:04:55.135 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:04:54 smithi026 ceph-mon[179688]: 8.9 starting backfill to osd.3 from (0'0,0'0] MAX to 323'9917 2023-09-23T21:04:55.135 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:04:54 smithi026 ceph-mon[179688]: failed to encode map e326 with expected crc 2023-09-23T21:04:55.135 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:04:54 smithi026 ceph-mon[179688]: 8.8 starting backfill to osd.0 from (0'0,0'0] MAX to 323'10188 2023-09-23T21:04:55.136 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:04:54 smithi026 ceph-mon[179688]: 8.3 starting backfill to osd.2 from (0'0,0'0] MAX to 323'9870 2023-09-23T21:04:55.136 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:04:54 smithi026 ceph-mon[179688]: 8.10 starting backfill to osd.6 from (0'0,0'0] MAX to 323'10195 2023-09-23T21:04:55.136 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:04:54 smithi026 ceph-mon[179688]: 8.19 starting backfill to osd.6 from (258'700,318'9429] MIN to 323'9987 2023-09-23T21:04:55.136 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:04:54 smithi026 ceph-mon[179688]: 8.4 starting backfill to osd.6 from (0'0,0'0] MAX to 323'10163 2023-09-23T21:04:55.136 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:04:54 smithi026 ceph-mon[179688]: 8.a starting backfill to osd.3 from (0'0,0'0] MAX to 323'11917 2023-09-23T21:04:55.136 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:04:54 smithi026 ceph-mon[179688]: 8.16 starting backfill to osd.0 from (258'600,312'9711] MIN to 323'10492 2023-09-23T21:04:55.136 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:04:54 smithi026 ceph-mon[179688]: 8.d starting backfill to osd.6 from (0'0,0'0] MAX to 323'11922 2023-09-23T21:04:55.136 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:04:54 smithi026 ceph-mon[179688]: 8.2 starting backfill to osd.3 from (0'0,0'0] MAX to 323'10342 2023-09-23T21:04:55.136 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:04:54 smithi026 ceph-mon[179688]: 8.0 starting backfill to osd.4 from (0'0,0'0] MAX to 323'10113 2023-09-23T21:04:55.136 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:04:54 smithi026 ceph-mon[179688]: 8.18 starting backfill to osd.7 from (258'600,316'9572] MIN to 323'10167 2023-09-23T21:04:55.136 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:04:54 smithi026 ceph-mon[179688]: osdmap e327: 8 total, 8 up, 7 in 2023-09-23T21:04:55.137 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:04:54 smithi026 ceph-mon[175151]: failed to encode map e326 with expected crc 2023-09-23T21:04:55.137 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:04:54 smithi026 ceph-mon[175151]: failed to encode map e326 with expected crc 2023-09-23T21:04:55.137 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:04:54 smithi026 ceph-mon[175151]: failed to encode map e326 with expected crc 2023-09-23T21:04:55.137 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:04:54 smithi026 ceph-mon[175151]: failed to encode map e326 with expected crc 2023-09-23T21:04:55.137 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:04:54 smithi026 ceph-mon[175151]: 8.11 starting backfill to osd.1 from (258'1600,299'9240] MIN to 323'10346 2023-09-23T21:04:55.137 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:04:54 smithi026 ceph-mon[175151]: 8.9 starting backfill to osd.3 from (0'0,0'0] MAX to 323'9917 2023-09-23T21:04:55.137 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:04:54 smithi026 ceph-mon[175151]: failed to encode map e326 with expected crc 2023-09-23T21:04:55.137 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:04:54 smithi026 ceph-mon[175151]: 8.8 starting backfill to osd.0 from (0'0,0'0] MAX to 323'10188 2023-09-23T21:04:55.137 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:04:54 smithi026 ceph-mon[175151]: 8.3 starting backfill to osd.2 from (0'0,0'0] MAX to 323'9870 2023-09-23T21:04:55.137 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:04:54 smithi026 ceph-mon[175151]: 8.10 starting backfill to osd.6 from (0'0,0'0] MAX to 323'10195 2023-09-23T21:04:55.138 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:04:54 smithi026 ceph-mon[175151]: 8.19 starting backfill to osd.6 from (258'700,318'9429] MIN to 323'9987 2023-09-23T21:04:55.138 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:04:54 smithi026 ceph-mon[175151]: 8.4 starting backfill to osd.6 from (0'0,0'0] MAX to 323'10163 2023-09-23T21:04:55.138 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:04:54 smithi026 ceph-mon[175151]: 8.a starting backfill to osd.3 from (0'0,0'0] MAX to 323'11917 2023-09-23T21:04:55.138 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:04:54 smithi026 ceph-mon[175151]: 8.16 starting backfill to osd.0 from (258'600,312'9711] MIN to 323'10492 2023-09-23T21:04:55.138 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:04:54 smithi026 ceph-mon[175151]: 8.d starting backfill to osd.6 from (0'0,0'0] MAX to 323'11922 2023-09-23T21:04:55.138 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:04:54 smithi026 ceph-mon[175151]: 8.2 starting backfill to osd.3 from (0'0,0'0] MAX to 323'10342 2023-09-23T21:04:55.138 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:04:54 smithi026 ceph-mon[175151]: 8.0 starting backfill to osd.4 from (0'0,0'0] MAX to 323'10113 2023-09-23T21:04:55.138 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:04:54 smithi026 ceph-mon[175151]: 8.18 starting backfill to osd.7 from (258'600,316'9572] MIN to 323'10167 2023-09-23T21:04:55.138 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:04:54 smithi026 ceph-mon[175151]: osdmap e327: 8 total, 8 up, 7 in 2023-09-23T21:04:55.140 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:04:55.167 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:04:55.167 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:04:55.268 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:04:55.302 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:04:54 smithi070 ceph-mon[168282]: failed to encode map e326 with expected crc 2023-09-23T21:04:55.303 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:04:54 smithi070 ceph-mon[168282]: failed to encode map e326 with expected crc 2023-09-23T21:04:55.303 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:04:54 smithi070 ceph-mon[168282]: failed to encode map e326 with expected crc 2023-09-23T21:04:55.303 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:04:54 smithi070 ceph-mon[168282]: failed to encode map e326 with expected crc 2023-09-23T21:04:55.303 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:04:54 smithi070 ceph-mon[168282]: 8.11 starting backfill to osd.1 from (258'1600,299'9240] MIN to 323'10346 2023-09-23T21:04:55.303 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:04:54 smithi070 ceph-mon[168282]: 8.9 starting backfill to osd.3 from (0'0,0'0] MAX to 323'9917 2023-09-23T21:04:55.303 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:04:54 smithi070 ceph-mon[168282]: failed to encode map e326 with expected crc 2023-09-23T21:04:55.303 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:04:54 smithi070 ceph-mon[168282]: 8.8 starting backfill to osd.0 from (0'0,0'0] MAX to 323'10188 2023-09-23T21:04:55.303 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:04:54 smithi070 ceph-mon[168282]: 8.3 starting backfill to osd.2 from (0'0,0'0] MAX to 323'9870 2023-09-23T21:04:55.303 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:04:54 smithi070 ceph-mon[168282]: 8.10 starting backfill to osd.6 from (0'0,0'0] MAX to 323'10195 2023-09-23T21:04:55.303 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:04:54 smithi070 ceph-mon[168282]: 8.19 starting backfill to osd.6 from (258'700,318'9429] MIN to 323'9987 2023-09-23T21:04:55.303 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:04:54 smithi070 ceph-mon[168282]: 8.4 starting backfill to osd.6 from (0'0,0'0] MAX to 323'10163 2023-09-23T21:04:55.303 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:04:54 smithi070 ceph-mon[168282]: 8.a starting backfill to osd.3 from (0'0,0'0] MAX to 323'11917 2023-09-23T21:04:55.304 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:04:54 smithi070 ceph-mon[168282]: 8.16 starting backfill to osd.0 from (258'600,312'9711] MIN to 323'10492 2023-09-23T21:04:55.304 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:04:54 smithi070 ceph-mon[168282]: 8.d starting backfill to osd.6 from (0'0,0'0] MAX to 323'11922 2023-09-23T21:04:55.304 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:04:54 smithi070 ceph-mon[168282]: 8.2 starting backfill to osd.3 from (0'0,0'0] MAX to 323'10342 2023-09-23T21:04:55.304 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:04:54 smithi070 ceph-mon[168282]: 8.0 starting backfill to osd.4 from (0'0,0'0] MAX to 323'10113 2023-09-23T21:04:55.304 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:04:54 smithi070 ceph-mon[168282]: 8.18 starting backfill to osd.7 from (258'600,316'9572] MIN to 323'10167 2023-09-23T21:04:55.304 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:04:54 smithi070 ceph-mon[168282]: osdmap e327: 8 total, 8 up, 7 in 2023-09-23T21:04:55.306 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:04:55.341 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:04:55.346 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:04:55.347 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:04:55.447 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:04:55.500 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:04:55.559 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:04:55.566 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:04:55.567 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:04:55.580 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:04:55.668 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:04:55.746 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:04:55.924 INFO:teuthology.orchestra.run.smithi026.stderr:nodeep-scrub is unset 2023-09-23T21:04:56.008 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T21:04:56.016 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:04:56.017 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:04:56.117 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:04:56.166 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:04:56.176 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:04:55 smithi026 ceph-mon[179688]: pgmap v839: 119 pgs: 2 active+remapped+backfilling, 2 active+remapped+backfill_wait, 4 active+undersized+remapped, 11 peering, 1 unknown, 1 active+clean+scrubbing, 98 active+clean; 1.8 GiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 21 KiB/s rd, 11 op/s; 4398/88290 objects misplaced (4.981%) 2023-09-23T21:04:56.177 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:04:55 smithi026 ceph-mon[179688]: failed to encode map e327 with expected crc 2023-09-23T21:04:56.177 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:04:55 smithi026 ceph-mon[179688]: failed to encode map e327 with expected crc 2023-09-23T21:04:56.177 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:04:55 smithi026 ceph-mon[179688]: failed to encode map e327 with expected crc 2023-09-23T21:04:56.177 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:04:55 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/2798665182' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:04:56.177 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:04:55 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:04:56.177 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:04:55 smithi026 ceph-mon[179688]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T21:04:56.177 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:04:55 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T21:04:56.177 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:04:55 smithi026 ceph-mon[179688]: osdmap e328: 8 total, 8 up, 7 in 2023-09-23T21:04:56.178 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:04:55 smithi026 ceph-mon[175151]: pgmap v839: 119 pgs: 2 active+remapped+backfilling, 2 active+remapped+backfill_wait, 4 active+undersized+remapped, 11 peering, 1 unknown, 1 active+clean+scrubbing, 98 active+clean; 1.8 GiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 21 KiB/s rd, 11 op/s; 4398/88290 objects misplaced (4.981%) 2023-09-23T21:04:56.178 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:04:55 smithi026 ceph-mon[175151]: failed to encode map e327 with expected crc 2023-09-23T21:04:56.178 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:04:55 smithi026 ceph-mon[175151]: failed to encode map e327 with expected crc 2023-09-23T21:04:56.178 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:04:55 smithi026 ceph-mon[175151]: failed to encode map e327 with expected crc 2023-09-23T21:04:56.178 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:04:55 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/2798665182' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:04:56.179 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:04:55 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:04:56.179 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:04:55 smithi026 ceph-mon[175151]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T21:04:56.179 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:04:55 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T21:04:56.179 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:04:55 smithi026 ceph-mon[175151]: osdmap e328: 8 total, 8 up, 7 in 2023-09-23T21:04:56.211 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:04:56.220 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:04:55 smithi070 ceph-mon[168282]: pgmap v839: 119 pgs: 2 active+remapped+backfilling, 2 active+remapped+backfill_wait, 4 active+undersized+remapped, 11 peering, 1 unknown, 1 active+clean+scrubbing, 98 active+clean; 1.8 GiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 21 KiB/s rd, 11 op/s; 4398/88290 objects misplaced (4.981%) 2023-09-23T21:04:56.220 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:04:55 smithi070 ceph-mon[168282]: failed to encode map e327 with expected crc 2023-09-23T21:04:56.220 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:04:55 smithi070 ceph-mon[168282]: failed to encode map e327 with expected crc 2023-09-23T21:04:56.220 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:04:55 smithi070 ceph-mon[168282]: failed to encode map e327 with expected crc 2023-09-23T21:04:56.220 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:04:55 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/2798665182' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:04:56.220 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:04:55 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:04:56.220 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:04:55 smithi070 ceph-mon[168282]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T21:04:56.220 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:04:55 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T21:04:56.221 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:04:55 smithi070 ceph-mon[168282]: osdmap e328: 8 total, 8 up, 7 in 2023-09-23T21:04:56.221 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:04:56.221 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:04:56.322 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:04:56.404 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:04:56.762 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:04:56.768 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:04:56.768 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:04:56.869 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:04:56.905 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:04:56.956 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:04:56.960 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:04:56.961 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:04:57.062 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:04:57.133 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:04:57.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:04:56 smithi026 ceph-mon[179688]: failed to encode map e328 with expected crc 2023-09-23T21:04:57.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:04:56 smithi026 ceph-mon[179688]: failed to encode map e328 with expected crc 2023-09-23T21:04:57.134 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:04:56 smithi026 ceph-mon[179688]: failed to encode map e328 with expected crc 2023-09-23T21:04:57.134 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:04:56 smithi026 ceph-mon[175151]: failed to encode map e328 with expected crc 2023-09-23T21:04:57.134 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:04:56 smithi026 ceph-mon[175151]: failed to encode map e328 with expected crc 2023-09-23T21:04:57.134 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:04:56 smithi026 ceph-mon[175151]: failed to encode map e328 with expected crc 2023-09-23T21:04:57.204 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:04:57.208 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:04:56 smithi070 ceph-mon[168282]: failed to encode map e328 with expected crc 2023-09-23T21:04:57.208 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:04:56 smithi070 ceph-mon[168282]: failed to encode map e328 with expected crc 2023-09-23T21:04:57.208 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:04:56 smithi070 ceph-mon[168282]: failed to encode map e328 with expected crc 2023-09-23T21:04:57.208 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:04:57.209 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:04:57.310 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:04:57.385 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:04:57.473 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 6, 7, 4] out_osds: [5] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 7, 6] 2023-09-23T21:04:57.473 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T21:04:57.474 INFO:tasks.thrashosds.thrasher:Adding osd 5 2023-09-23T21:04:57.474 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph osd in 5 2023-09-23T21:04:57.881 INFO:teuthology.orchestra.run.smithi026.stdout:ERROR: (22) Invalid argument 2023-09-23T21:04:57.882 INFO:teuthology.orchestra.run.smithi026.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T21:04:58.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:04:57 smithi026 ceph-mon[175151]: pgmap v842: 119 pgs: 4 active+remapped+backfilling, 4 active+remapped+backfill_wait, 4 active+undersized+remapped, 12 peering, 1 unknown, 94 active+clean; 1.7 GiB data, 7.5 GiB used, 618 GiB / 626 GiB avail; 62 KiB/s rd, 33 op/s; 8253/85371 objects misplaced (9.667%) 2023-09-23T21:04:58.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:04:57 smithi026 ceph-mon[179688]: pgmap v842: 119 pgs: 4 active+remapped+backfilling, 4 active+remapped+backfill_wait, 4 active+undersized+remapped, 12 peering, 1 unknown, 94 active+clean; 1.7 GiB data, 7.5 GiB used, 618 GiB / 626 GiB avail; 62 KiB/s rd, 33 op/s; 8253/85371 objects misplaced (9.667%) 2023-09-23T21:04:58.276 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:04:58.289 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:04:58.290 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:04:58.371 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:04:57 smithi070 ceph-mon[168282]: pgmap v842: 119 pgs: 4 active+remapped+backfilling, 4 active+remapped+backfill_wait, 4 active+undersized+remapped, 12 peering, 1 unknown, 94 active+clean; 1.7 GiB data, 7.5 GiB used, 618 GiB / 626 GiB avail; 62 KiB/s rd, 33 op/s; 8253/85371 objects misplaced (9.667%) 2023-09-23T21:04:58.390 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:04:58.463 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:04:58.572 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:04:58.574 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:04:58.575 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:04:58.675 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:04:58.715 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:04:58.769 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:04:58.776 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:04:58.776 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:04:58.877 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:04:58.961 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:04:59.017 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T21:04:59.693 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:04:59.701 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:04:59.702 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:04:59.711 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 21:04:59 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[164029]: ::ffff:172.21.15.70 - - [23/Sep/2023:21:04:59] "GET /metrics HTTP/1.1" 200 36291 "" "Prometheus/2.43.0" 2023-09-23T21:04:59.748 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_ops_in_flight 2023-09-23T21:04:59.803 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:04:59.836 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:04:59.882 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:04:59.886 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:04:59.886 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:04:59.987 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:05:00.030 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:05:00.041 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:05:00.079 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:05:00.084 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:04:59 smithi070 ceph-mon[168282]: pgmap v843: 119 pgs: 4 active+remapped+backfilling, 9 active+remapped+backfill_wait, 4 active+undersized+remapped, 1 peering, 101 active+clean; 1.7 GiB data, 7.4 GiB used, 618 GiB / 626 GiB avail; 42 KiB/s rd, 22 op/s; 12321/82968 objects misplaced (14.850%) 2023-09-23T21:05:00.084 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:04:59 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:05:00.084 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:04:59 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:05:00.084 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:04:59 smithi070 ceph-mon[168282]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2023-09-23T21:05:00.084 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:04:59 smithi070 ceph-mon[168282]: Cluster is now healthy 2023-09-23T21:05:00.085 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:05:00.085 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:05:00.135 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:04:59 smithi026 ceph-mon[175151]: pgmap v843: 119 pgs: 4 active+remapped+backfilling, 9 active+remapped+backfill_wait, 4 active+undersized+remapped, 1 peering, 101 active+clean; 1.7 GiB data, 7.4 GiB used, 618 GiB / 626 GiB avail; 42 KiB/s rd, 22 op/s; 12321/82968 objects misplaced (14.850%) 2023-09-23T21:05:00.135 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:04:59 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:05:00.135 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:04:59 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:05:00.136 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:04:59 smithi026 ceph-mon[175151]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2023-09-23T21:05:00.136 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:04:59 smithi026 ceph-mon[175151]: Cluster is now healthy 2023-09-23T21:05:00.136 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:04:59 smithi026 ceph-mon[179688]: pgmap v843: 119 pgs: 4 active+remapped+backfilling, 9 active+remapped+backfill_wait, 4 active+undersized+remapped, 1 peering, 101 active+clean; 1.7 GiB data, 7.4 GiB used, 618 GiB / 626 GiB avail; 42 KiB/s rd, 22 op/s; 12321/82968 objects misplaced (14.850%) 2023-09-23T21:05:00.136 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:04:59 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:05:00.136 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:04:59 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:05:00.136 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:04:59 smithi026 ceph-mon[179688]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2023-09-23T21:05:00.136 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:04:59 smithi026 ceph-mon[179688]: Cluster is now healthy 2023-09-23T21:05:00.141 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:05:00.186 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:05:00.259 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:05:00.636 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0: {} 2023-09-23T21:05:00.651 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:05:00.663 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:05:00.663 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:05:00.675 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1: {} 2023-09-23T21:05:00.719 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2: {} 2023-09-23T21:05:00.756 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3: {} 2023-09-23T21:05:00.764 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:05:00.793 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4: {} 2023-09-23T21:05:00.799 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:05:00.823 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T21:05:00.851 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:05:00.854 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:05:00.855 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:05:00.858 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6: {} 2023-09-23T21:05:00.904 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T21:05:00.917 INFO:teuthology.orchestra.run.smithi026.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T21:05:00.917 INFO:teuthology.orchestra.run.smithi026.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T21:05:00.917 INFO:teuthology.orchestra.run.smithi026.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T21:05:00.917 INFO:teuthology.orchestra.run.smithi026.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T21:05:00.918 INFO:teuthology.orchestra.run.smithi026.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T21:05:00.918 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T21:05:00.918 INFO:teuthology.orchestra.run.smithi026.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T21:05:00.918 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T21:05:00.955 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:05:00.992 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:05:01.033 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:05:01.036 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:05:01.036 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:05:01.137 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:05:01.185 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:05:01.533 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:05:01 smithi026 ceph-mon[179688]: failed to encode map e329 with expected crc 2023-09-23T21:05:01.533 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:05:01 smithi026 ceph-mon[179688]: osdmap e329: 8 total, 8 up, 7 in 2023-09-23T21:05:01.533 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:05:01 smithi026 ceph-mon[179688]: 1.1 scrub starts 2023-09-23T21:05:01.533 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:05:01 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/3162718901' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:05:01.533 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:05:01 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:05:01.533 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:05:01 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:05:01.533 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:05:01 smithi026 ceph-mon[175151]: failed to encode map e329 with expected crc 2023-09-23T21:05:01.534 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:05:01 smithi026 ceph-mon[175151]: osdmap e329: 8 total, 8 up, 7 in 2023-09-23T21:05:01.534 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:05:01 smithi026 ceph-mon[175151]: 1.1 scrub starts 2023-09-23T21:05:01.534 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:05:01 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/3162718901' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:05:01.534 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:05:01 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:05:01.534 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:05:01 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:05:01.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:05:01 smithi070 ceph-mon[168282]: failed to encode map e329 with expected crc 2023-09-23T21:05:01.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:05:01 smithi070 ceph-mon[168282]: osdmap e329: 8 total, 8 up, 7 in 2023-09-23T21:05:01.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:05:01 smithi070 ceph-mon[168282]: 1.1 scrub starts 2023-09-23T21:05:01.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:05:01 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/3162718901' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:05:01.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:05:01 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:05:01.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:05:01 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:05:01.865 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:05:01.869 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:05:01.869 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:05:01.970 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:05:01.997 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:05:02.029 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:05:02.032 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:05:02.033 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:05:02.134 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:05:02.180 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:05:02.330 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:05:02.346 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:05:02.346 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:05:02.427 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:05:02.431 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:05:02 smithi026 ceph-mon[175151]: pgmap v845: 119 pgs: 4 active+remapped+backfilling, 9 active+remapped+backfill_wait, 106 active+clean; 1.6 GiB data, 7.3 GiB used, 619 GiB / 626 GiB avail; 28 KiB/s rd, 14 op/s; 9948/76416 objects misplaced (13.018%) 2023-09-23T21:05:02.431 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:05:02 smithi026 ceph-mon[175151]: failed to encode map e330 with expected crc 2023-09-23T21:05:02.431 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:05:02 smithi026 ceph-mon[175151]: failed to encode map e330 with expected crc 2023-09-23T21:05:02.431 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:05:02 smithi026 ceph-mon[175151]: osdmap e330: 8 total, 8 up, 7 in 2023-09-23T21:05:02.432 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:05:02 smithi026 ceph-mon[179688]: pgmap v845: 119 pgs: 4 active+remapped+backfilling, 9 active+remapped+backfill_wait, 106 active+clean; 1.6 GiB data, 7.3 GiB used, 619 GiB / 626 GiB avail; 28 KiB/s rd, 14 op/s; 9948/76416 objects misplaced (13.018%) 2023-09-23T21:05:02.432 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:05:02 smithi026 ceph-mon[179688]: failed to encode map e330 with expected crc 2023-09-23T21:05:02.432 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:05:02 smithi026 ceph-mon[179688]: failed to encode map e330 with expected crc 2023-09-23T21:05:02.432 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:05:02 smithi026 ceph-mon[179688]: osdmap e330: 8 total, 8 up, 7 in 2023-09-23T21:05:02.447 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:05:02.483 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:05:02.537 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:05:02.540 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:05:02.541 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:05:02.541 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:05:02 smithi070 ceph-mon[168282]: pgmap v845: 119 pgs: 4 active+remapped+backfilling, 9 active+remapped+backfill_wait, 106 active+clean; 1.6 GiB data, 7.3 GiB used, 619 GiB / 626 GiB avail; 28 KiB/s rd, 14 op/s; 9948/76416 objects misplaced (13.018%) 2023-09-23T21:05:02.541 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:05:02 smithi070 ceph-mon[168282]: failed to encode map e330 with expected crc 2023-09-23T21:05:02.541 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:05:02 smithi070 ceph-mon[168282]: failed to encode map e330 with expected crc 2023-09-23T21:05:02.541 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:05:02 smithi070 ceph-mon[168282]: osdmap e330: 8 total, 8 up, 7 in 2023-09-23T21:05:02.642 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:05:02.706 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:05:02.798 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T21:05:02.804 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:05:02.804 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:05:02.905 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:05:02.956 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:05:03.260 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:05:03.264 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:05:03 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mon-a[175127]: 2023-09-23T21:05:03.181+0000 7f1e2d487700 -1 mon.a@0(leader).osd e331 definitely_dead 0 2023-09-23T21:05:03.265 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:05:03.265 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:05:03.306 INFO:teuthology.orchestra.run.smithi026.stderr:marked in osd.5. 2023-09-23T21:05:03.366 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:05:03.391 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:05:03.423 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:05:03.426 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:05:03.426 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:05:03.527 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:05:03.551 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:05:03 smithi070 ceph-mon[168282]: failed to encode map e331 with expected crc 2023-09-23T21:05:03.551 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:05:03 smithi070 ceph-mon[168282]: failed to encode map e331 with expected crc 2023-09-23T21:05:03.551 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:05:03 smithi070 ceph-mon[168282]: osdmap e331: 8 total, 8 up, 7 in 2023-09-23T21:05:03.551 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:05:03 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/1764903237' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["5"]}]: dispatch 2023-09-23T21:05:03.551 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:05:03 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["5"]}]: dispatch 2023-09-23T21:05:03.553 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:05:03.586 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:05:03.588 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:05:03.589 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:05:03.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:05:03 smithi026 ceph-mon[175151]: failed to encode map e331 with expected crc 2023-09-23T21:05:03.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:05:03 smithi026 ceph-mon[175151]: failed to encode map e331 with expected crc 2023-09-23T21:05:03.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:05:03 smithi026 ceph-mon[175151]: osdmap e331: 8 total, 8 up, 7 in 2023-09-23T21:05:03.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:05:03 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/1764903237' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["5"]}]: dispatch 2023-09-23T21:05:03.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:05:03 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["5"]}]: dispatch 2023-09-23T21:05:03.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:05:03 smithi026 ceph-mon[179688]: failed to encode map e331 with expected crc 2023-09-23T21:05:03.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:05:03 smithi026 ceph-mon[179688]: failed to encode map e331 with expected crc 2023-09-23T21:05:03.633 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:05:03 smithi026 ceph-mon[179688]: osdmap e331: 8 total, 8 up, 7 in 2023-09-23T21:05:03.633 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:05:03 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/1764903237' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["5"]}]: dispatch 2023-09-23T21:05:03.633 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:05:03 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["5"]}]: dispatch 2023-09-23T21:05:03.689 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:05:03.715 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:05:03.745 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:05:03.748 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:05:03.748 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:05:03.849 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:05:03.875 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:05:03.902 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T21:05:03.905 INFO:tasks.thrashosds.thrasher:Added osd 5 2023-09-23T21:05:03.907 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:05:03.910 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:05:03.910 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:05:04.011 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:05:04.040 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:05:04.078 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:05:04.081 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:05:04.082 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:05:04.183 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:05:04.212 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:05:04.255 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:05:04.257 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:05:04.258 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:05:04.297 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:05:04 smithi026 ceph-mon[175151]: pgmap v848: 119 pgs: 1 peering, 4 active+remapped+backfilling, 7 active+remapped+backfill_wait, 107 active+clean; 1.5 GiB data, 7.0 GiB used, 619 GiB / 626 GiB avail; 7266/72267 objects misplaced (10.054%) 2023-09-23T21:05:04.297 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:05:04 smithi026 ceph-mon[175151]: failed to encode map e332 with expected crc 2023-09-23T21:05:04.297 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:05:04 smithi026 ceph-mon[175151]: failed to encode map e332 with expected crc 2023-09-23T21:05:04.297 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:05:04 smithi026 ceph-mon[175151]: failed to encode map e332 with expected crc 2023-09-23T21:05:04.297 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:05:04 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["5"]}]': finished 2023-09-23T21:05:04.297 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:05:04 smithi026 ceph-mon[175151]: osdmap e332: 8 total, 8 up, 8 in 2023-09-23T21:05:04.297 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:05:04 smithi026 ceph-mon[175151]: failed to encode map e332 with expected crc 2023-09-23T21:05:04.297 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:05:04 smithi026 ceph-mon[175151]: failed to encode map e332 with expected crc 2023-09-23T21:05:04.298 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:05:04 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:05:04.301 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:05:04 smithi026 ceph-mon[179688]: pgmap v848: 119 pgs: 1 peering, 4 active+remapped+backfilling, 7 active+remapped+backfill_wait, 107 active+clean; 1.5 GiB data, 7.0 GiB used, 619 GiB / 626 GiB avail; 7266/72267 objects misplaced (10.054%) 2023-09-23T21:05:04.301 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:05:04 smithi026 ceph-mon[179688]: failed to encode map e332 with expected crc 2023-09-23T21:05:04.301 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:05:04 smithi026 ceph-mon[179688]: failed to encode map e332 with expected crc 2023-09-23T21:05:04.301 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:05:04 smithi026 ceph-mon[179688]: failed to encode map e332 with expected crc 2023-09-23T21:05:04.301 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:05:04 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["5"]}]': finished 2023-09-23T21:05:04.301 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:05:04 smithi026 ceph-mon[179688]: osdmap e332: 8 total, 8 up, 8 in 2023-09-23T21:05:04.301 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:05:04 smithi026 ceph-mon[179688]: failed to encode map e332 with expected crc 2023-09-23T21:05:04.302 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:05:04 smithi026 ceph-mon[179688]: failed to encode map e332 with expected crc 2023-09-23T21:05:04.302 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:05:04 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:05:04.302 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:05:04 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:05:04.302 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:05:04 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/3057716534' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T21:05:04.302 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:05:04 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T21:05:04.302 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:05:04 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T21:05:04.302 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:05:04 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T21:05:04.302 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:05:04 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:05:04.302 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:05:04 smithi070 ceph-mon[168282]: pgmap v848: 119 pgs: 1 peering, 4 active+remapped+backfilling, 7 active+remapped+backfill_wait, 107 active+clean; 1.5 GiB data, 7.0 GiB used, 619 GiB / 626 GiB avail; 7266/72267 objects misplaced (10.054%) 2023-09-23T21:05:04.302 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:05:04 smithi070 ceph-mon[168282]: failed to encode map e332 with expected crc 2023-09-23T21:05:04.302 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:05:04 smithi070 ceph-mon[168282]: failed to encode map e332 with expected crc 2023-09-23T21:05:04.303 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:05:04 smithi070 ceph-mon[168282]: failed to encode map e332 with expected crc 2023-09-23T21:05:04.303 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:05:04 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["5"]}]': finished 2023-09-23T21:05:04.303 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:05:04 smithi070 ceph-mon[168282]: osdmap e332: 8 total, 8 up, 8 in 2023-09-23T21:05:04.303 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:05:04 smithi070 ceph-mon[168282]: failed to encode map e332 with expected crc 2023-09-23T21:05:04.303 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:05:04 smithi070 ceph-mon[168282]: failed to encode map e332 with expected crc 2023-09-23T21:05:04.303 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:05:04 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:05:04.303 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:05:04 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:05:04.303 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:05:04 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/3057716534' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T21:05:04.303 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:05:04 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T21:05:04.303 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:05:04 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T21:05:04.303 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:05:04 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T21:05:04.304 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:05:04 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:05:04.305 INFO:teuthology.orchestra.run.smithi026.stderr:noscrub is set 2023-09-23T21:05:04.359 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:05:04.398 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:05:04.463 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:05:04.471 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:05:04.471 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:05:04.572 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:05:04.597 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:05:04.629 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:05:04.631 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:05:04 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:05:04.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:05:04 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/3057716534' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T21:05:04.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:05:04 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T21:05:04.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:05:04 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T21:05:04.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:05:04 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T21:05:04.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:05:04 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:05:04.632 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:05:04.633 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:05:04.734 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:05:04.759 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:05:04.791 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:05:04.794 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:05:04.794 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:05:04.895 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:05:04.921 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:05:04.953 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:05:04.956 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:05:04.956 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:05:05.057 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:05:05.090 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:05:05.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:05:05 smithi026 ceph-mon[179688]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:05:05.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:05:05 smithi026 ceph-mon[179688]: failed to encode map e333 with expected crc 2023-09-23T21:05:05.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:05:05 smithi026 ceph-mon[179688]: failed to encode map e333 with expected crc 2023-09-23T21:05:05.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:05:05 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T21:05:05.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:05:05 smithi026 ceph-mon[179688]: osdmap e333: 8 total, 8 up, 8 in 2023-09-23T21:05:05.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:05:05 smithi026 ceph-mon[179688]: failed to encode map e333 with expected crc 2023-09-23T21:05:05.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:05:05 smithi026 ceph-mon[179688]: failed to encode map e333 with expected crc 2023-09-23T21:05:05.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:05:05 smithi026 ceph-mon[179688]: failed to encode map e333 with expected crc 2023-09-23T21:05:05.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:05:05 smithi026 ceph-mon[179688]: failed to encode map e333 with expected crc 2023-09-23T21:05:05.383 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:05:05 smithi026 ceph-mon[179688]: osdmap e334: 8 total, 8 up, 8 in 2023-09-23T21:05:05.383 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:05:05 smithi026 ceph-mon[175151]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:05:05.383 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:05:05 smithi026 ceph-mon[175151]: failed to encode map e333 with expected crc 2023-09-23T21:05:05.383 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:05:05 smithi026 ceph-mon[175151]: failed to encode map e333 with expected crc 2023-09-23T21:05:05.383 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:05:05 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T21:05:05.383 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:05:05 smithi026 ceph-mon[175151]: osdmap e333: 8 total, 8 up, 8 in 2023-09-23T21:05:05.383 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:05:05 smithi026 ceph-mon[175151]: failed to encode map e333 with expected crc 2023-09-23T21:05:05.383 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:05:05 smithi026 ceph-mon[175151]: failed to encode map e333 with expected crc 2023-09-23T21:05:05.383 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:05:05 smithi026 ceph-mon[175151]: failed to encode map e333 with expected crc 2023-09-23T21:05:05.383 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:05:05 smithi026 ceph-mon[175151]: failed to encode map e333 with expected crc 2023-09-23T21:05:05.384 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:05:05 smithi026 ceph-mon[175151]: osdmap e334: 8 total, 8 up, 8 in 2023-09-23T21:05:05.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:05:05 smithi070 ceph-mon[168282]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:05:05.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:05:05 smithi070 ceph-mon[168282]: failed to encode map e333 with expected crc 2023-09-23T21:05:05.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:05:05 smithi070 ceph-mon[168282]: failed to encode map e333 with expected crc 2023-09-23T21:05:05.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:05:05 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T21:05:05.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:05:05 smithi070 ceph-mon[168282]: osdmap e333: 8 total, 8 up, 8 in 2023-09-23T21:05:05.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:05:05 smithi070 ceph-mon[168282]: failed to encode map e333 with expected crc 2023-09-23T21:05:05.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:05:05 smithi070 ceph-mon[168282]: failed to encode map e333 with expected crc 2023-09-23T21:05:05.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:05:05 smithi070 ceph-mon[168282]: failed to encode map e333 with expected crc 2023-09-23T21:05:05.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:05:05 smithi070 ceph-mon[168282]: failed to encode map e333 with expected crc 2023-09-23T21:05:05.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:05:05 smithi070 ceph-mon[168282]: osdmap e334: 8 total, 8 up, 8 in 2023-09-23T21:05:06.011 INFO:teuthology.orchestra.run.smithi026.stdout:{ 2023-09-23T21:05:06.011 INFO:teuthology.orchestra.run.smithi026.stdout: "ops": [ 2023-09-23T21:05:06.012 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:05:06.012 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_repop(client.35469.0:198956 8.7 e334/281 8:e219012a:::benchmark_data_smithi026_301269_object88199:head v 334'10272, mlcod=334'10269)", 2023-09-23T21:05:06.012 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T21:05:05.762330+0000", 2023-09-23T21:05:06.012 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 0.0048083559999999997, 2023-09-23T21:05:06.012 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0048314839999999996, 2023-09-23T21:05:06.012 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:05:06.012 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "started", 2023-09-23T21:05:06.012 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:05:06.012 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:05:06.012 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:05:06.012 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:05.762330+0000", 2023-09-23T21:05:06.012 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:05:06.013 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:05:06.013 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:05:06.013 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:05:06.013 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:05.762330+0000", 2023-09-23T21:05:06.013 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:05:06.013 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:05:06.013 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:05:06.013 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:05:06.013 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:05.762329+0000", 2023-09-23T21:05:06.013 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T21:05:06.013 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:05:06.013 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:05:06.013 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:05:06.014 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:05.762336+0000", 2023-09-23T21:05:06.014 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.272e-06 2023-09-23T21:05:06.014 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:05:06.014 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:05:06.014 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:05:06.014 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:05.762345+0000", 2023-09-23T21:05:06.014 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.8890000000000008e-06 2023-09-23T21:05:06.014 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:05:06.014 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:05:06.014 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:05:06.014 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:05.762351+0000", 2023-09-23T21:05:06.014 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.9710000000000001e-06 2023-09-23T21:05:06.015 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:05:06.015 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:05:06.015 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:05:06.015 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:05.762420+0000", 2023-09-23T21:05:06.015 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.8979000000000005e-05 2023-09-23T21:05:06.015 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:05:06.015 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:05:06.015 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:05:06.015 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:05.762440+0000", 2023-09-23T21:05:06.015 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.9837999999999999e-05 2023-09-23T21:05:06.015 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:05:06.015 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:05:06.015 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:05:06.016 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:05:06.016 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:05:06.016 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35469.0:198965 8.f 8.33a62b7f (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e334)", 2023-09-23T21:05:06.016 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T21:05:05.763038+0000", 2023-09-23T21:05:06.016 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 0.0041002349999999998, 2023-09-23T21:05:06.016 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0041692659999999996, 2023-09-23T21:05:06.016 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:05:06.016 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "queued for pg", 2023-09-23T21:05:06.016 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T21:05:06.016 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35469", 2023-09-23T21:05:06.016 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/4041424623", 2023-09-23T21:05:06.016 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 198965 2023-09-23T21:05:06.016 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:05:06.017 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:05:06.017 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:05:06.017 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:05:06.017 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:05.763038+0000", 2023-09-23T21:05:06.017 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:05:06.017 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:05:06.017 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:05:06.017 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:05:06.017 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:05.763038+0000", 2023-09-23T21:05:06.017 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:05:06.017 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:05:06.017 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:05:06.018 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:05:06.018 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:05.763037+0000", 2023-09-23T21:05:06.018 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.999999 2023-09-23T21:05:06.018 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:05:06.018 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:05:06.018 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:05:06.018 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:05.763042+0000", 2023-09-23T21:05:06.018 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.6450000000000002e-06 2023-09-23T21:05:06.018 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:05:06.018 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:05:06.018 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:05:06.018 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:05.763058+0000", 2023-09-23T21:05:06.018 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.5336999999999999e-05 2023-09-23T21:05:06.019 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:05:06.019 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:05:06.019 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:05:06.019 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:05.763066+0000", 2023-09-23T21:05:06.019 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.4079999999999998e-06 2023-09-23T21:05:06.019 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:05:06.019 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:05:06.019 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:05:06.019 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:05:06.019 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:05:06.019 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35469.0:198969 8.f 8.b9062f2f (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e334)", 2023-09-23T21:05:06.019 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T21:05:05.763078+0000", 2023-09-23T21:05:06.020 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 0.0040598209999999999, 2023-09-23T21:05:06.020 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0041824160000000004, 2023-09-23T21:05:06.020 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:05:06.020 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "queued for pg", 2023-09-23T21:05:06.020 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T21:05:06.020 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35469", 2023-09-23T21:05:06.020 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/4041424623", 2023-09-23T21:05:06.020 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 198969 2023-09-23T21:05:06.020 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:05:06.020 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:05:06.020 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:05:06.020 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:05:06.020 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:05.763078+0000", 2023-09-23T21:05:06.021 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:05:06.021 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:05:06.021 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:05:06.021 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:05:06.021 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:05.763078+0000", 2023-09-23T21:05:06.021 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:05:06.021 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:05:06.021 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:05:06.021 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:05:06.021 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:05.763078+0000", 2023-09-23T21:05:06.021 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T21:05:06.021 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:05:06.021 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:05:06.022 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:05:06.022 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:05.763081+0000", 2023-09-23T21:05:06.022 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.1350000000000001e-06 2023-09-23T21:05:06.022 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:05:06.022 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:05:06.022 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:05:06.022 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:05.763091+0000", 2023-09-23T21:05:06.022 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.9219999999999992e-06 2023-09-23T21:05:06.022 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:05:06.022 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:05:06.022 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:05:06.022 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:05.763096+0000", 2023-09-23T21:05:06.023 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.8189999999999997e-06 2023-09-23T21:05:06.023 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:05:06.023 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:05:06.023 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:05:06.023 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:05:06.023 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:05:06.023 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_repop(client.35469.0:198962 8.18 e334/332)", 2023-09-23T21:05:06.023 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T21:05:05.766819+0000", 2023-09-23T21:05:06.023 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 0.00031899599999999999, 2023-09-23T21:05:06.023 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00048663500000000002, 2023-09-23T21:05:06.023 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:05:06.023 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "queued for pg", 2023-09-23T21:05:06.023 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:05:06.024 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:05:06.024 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:05:06.024 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:05.766819+0000", 2023-09-23T21:05:06.024 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:05:06.024 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:05:06.024 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:05:06.024 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:05:06.024 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:05.766819+0000", 2023-09-23T21:05:06.024 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:05:06.024 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:05:06.024 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:05:06.024 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:05:06.024 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:05.766818+0000", 2023-09-23T21:05:06.025 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.999999 2023-09-23T21:05:06.025 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:05:06.025 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:05:06.025 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:05:06.025 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:05.766827+0000", 2023-09-23T21:05:06.025 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.5189999999999992e-06 2023-09-23T21:05:06.025 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:05:06.025 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:05:06.025 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:05:06.025 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:05.766841+0000", 2023-09-23T21:05:06.025 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.4608e-05 2023-09-23T21:05:06.025 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:05:06.026 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:05:06.026 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:05:06.026 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:05.766849+0000", 2023-09-23T21:05:06.026 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.6079999999999998e-06 2023-09-23T21:05:06.026 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:05:06.026 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:05:06.026 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:05:06.026 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:05:06.026 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:05:06.026 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35469.0:198937 8.0 8.8f9edba0 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e334)", 2023-09-23T21:05:06.027 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T21:05:05.726870+0000", 2023-09-23T21:05:06.027 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 0.040267878999999999, 2023-09-23T21:05:06.027 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.040466974000000003, 2023-09-23T21:05:06.027 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:05:06.027 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "queued for pg", 2023-09-23T21:05:06.027 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T21:05:06.027 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35469", 2023-09-23T21:05:06.027 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/4041424623", 2023-09-23T21:05:06.028 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 198937 2023-09-23T21:05:06.028 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:05:06.028 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:05:06.028 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:05:06.028 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:05:06.028 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:05.726870+0000", 2023-09-23T21:05:06.028 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:05:06.028 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:05:06.029 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:05:06.029 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:05:06.029 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:05.726870+0000", 2023-09-23T21:05:06.029 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:05:06.029 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:05:06.029 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:05:06.029 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:05:06.029 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:05.726870+0000", 2023-09-23T21:05:06.030 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T21:05:06.030 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:05:06.030 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:05:06.030 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:05:06.030 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:05.726873+0000", 2023-09-23T21:05:06.030 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.2349999999999999e-06 2023-09-23T21:05:06.030 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:05:06.031 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:05:06.031 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:05:06.031 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:05.726882+0000", 2023-09-23T21:05:06.031 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.7450000000000001e-06 2023-09-23T21:05:06.031 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:05:06.031 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:05:06.031 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:05:06.031 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:05.726888+0000", 2023-09-23T21:05:06.032 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.9270000000000002e-06 2023-09-23T21:05:06.032 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:05:06.032 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:05:06.032 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:05:06.032 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:05:06.032 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:05:06.032 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_repop(client.35469.0:198938 8.18 e334/332)", 2023-09-23T21:05:06.032 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T21:05:05.729648+0000", 2023-09-23T21:05:06.033 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 0.037490173000000002, 2023-09-23T21:05:06.033 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.037723531999999997, 2023-09-23T21:05:06.033 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:05:06.033 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "queued for pg", 2023-09-23T21:05:06.033 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:05:06.033 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:05:06.033 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:05:06.033 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:05.729648+0000", 2023-09-23T21:05:06.034 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:05:06.034 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:05:06.034 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:05:06.034 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:05:06.034 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:05.729648+0000", 2023-09-23T21:05:06.034 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:05:06.034 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:05:06.035 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:05:06.035 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:05:06.035 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:05.729647+0000", 2023-09-23T21:05:06.035 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.999999 2023-09-23T21:05:06.035 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:05:06.035 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:05:06.035 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:05:06.035 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:05.729660+0000", 2023-09-23T21:05:06.036 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.3266e-05 2023-09-23T21:05:06.036 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:05:06.036 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:05:06.036 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:05:06.036 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:05.729669+0000", 2023-09-23T21:05:06.036 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.8929999999999994e-06 2023-09-23T21:05:06.036 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:05:06.036 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:05:06.037 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:05:06.037 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:05.729676+0000", 2023-09-23T21:05:06.037 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.7819999999999996e-06 2023-09-23T21:05:06.037 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:05:06.037 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:05:06.037 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:05:06.037 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:05:06.037 INFO:teuthology.orchestra.run.smithi026.stdout: ], 2023-09-23T21:05:06.038 INFO:teuthology.orchestra.run.smithi026.stdout: "num_ops": 6 2023-09-23T21:05:06.038 INFO:teuthology.orchestra.run.smithi026.stdout:} 2023-09-23T21:05:06.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:05:06 smithi070 ceph-mon[168282]: pgmap v851: 119 pgs: 2 remapped+peering, 9 peering, 3 active+remapped+backfilling, 5 active+remapped+backfill_wait, 100 active+clean; 1.4 GiB data, 6.9 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 471 op/s; 4668/70758 objects misplaced (6.597%); 40 MiB/s, 640 objects/s recovering 2023-09-23T21:05:06.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:05:06 smithi070 ceph-mon[168282]: failed to encode map e334 with expected crc 2023-09-23T21:05:06.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:05:06 smithi070 ceph-mon[168282]: failed to encode map e334 with expected crc 2023-09-23T21:05:06.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:05:06 smithi070 ceph-mon[168282]: failed to encode map e334 with expected crc 2023-09-23T21:05:06.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:05:06 smithi070 ceph-mon[168282]: failed to encode map e334 with expected crc 2023-09-23T21:05:06.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:05:06 smithi070 ceph-mon[168282]: failed to encode map e334 with expected crc 2023-09-23T21:05:06.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:05:06 smithi070 ceph-mon[168282]: failed to encode map e334 with expected crc 2023-09-23T21:05:06.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:05:06 smithi070 ceph-mon[168282]: 8.0 starting backfill to osd.5 from (258'2097,329'10281] MIN to 331'10308 2023-09-23T21:05:06.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:05:06 smithi070 ceph-mon[168282]: 8.11 starting backfill to osd.5 from (259'2187,328'10497] MIN to 331'10535 2023-09-23T21:05:06.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:05:06 smithi070 ceph-mon[168282]: 8.4 starting backfill to osd.5 from (258'1799,329'10340] MIN to 331'10356 2023-09-23T21:05:06.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:05:06 smithi070 ceph-mon[168282]: osdmap e335: 8 total, 8 up, 8 in 2023-09-23T21:05:06.631 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:05:06 smithi026 ceph-mon[179688]: pgmap v851: 119 pgs: 2 remapped+peering, 9 peering, 3 active+remapped+backfilling, 5 active+remapped+backfill_wait, 100 active+clean; 1.4 GiB data, 6.9 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 471 op/s; 4668/70758 objects misplaced (6.597%); 40 MiB/s, 640 objects/s recovering 2023-09-23T21:05:06.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:05:06 smithi026 ceph-mon[179688]: failed to encode map e334 with expected crc 2023-09-23T21:05:06.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:05:06 smithi026 ceph-mon[179688]: failed to encode map e334 with expected crc 2023-09-23T21:05:06.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:05:06 smithi026 ceph-mon[179688]: failed to encode map e334 with expected crc 2023-09-23T21:05:06.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:05:06 smithi026 ceph-mon[179688]: failed to encode map e334 with expected crc 2023-09-23T21:05:06.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:05:06 smithi026 ceph-mon[179688]: failed to encode map e334 with expected crc 2023-09-23T21:05:06.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:05:06 smithi026 ceph-mon[179688]: failed to encode map e334 with expected crc 2023-09-23T21:05:06.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:05:06 smithi026 ceph-mon[179688]: 8.0 starting backfill to osd.5 from (258'2097,329'10281] MIN to 331'10308 2023-09-23T21:05:06.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:05:06 smithi026 ceph-mon[179688]: 8.11 starting backfill to osd.5 from (259'2187,328'10497] MIN to 331'10535 2023-09-23T21:05:06.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:05:06 smithi026 ceph-mon[179688]: 8.4 starting backfill to osd.5 from (258'1799,329'10340] MIN to 331'10356 2023-09-23T21:05:06.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:05:06 smithi026 ceph-mon[179688]: osdmap e335: 8 total, 8 up, 8 in 2023-09-23T21:05:06.634 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:05:06 smithi026 ceph-mon[175151]: pgmap v851: 119 pgs: 2 remapped+peering, 9 peering, 3 active+remapped+backfilling, 5 active+remapped+backfill_wait, 100 active+clean; 1.4 GiB data, 6.9 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 471 op/s; 4668/70758 objects misplaced (6.597%); 40 MiB/s, 640 objects/s recovering 2023-09-23T21:05:06.634 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:05:06 smithi026 ceph-mon[175151]: failed to encode map e334 with expected crc 2023-09-23T21:05:06.634 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:05:06 smithi026 ceph-mon[175151]: failed to encode map e334 with expected crc 2023-09-23T21:05:06.634 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:05:06 smithi026 ceph-mon[175151]: failed to encode map e334 with expected crc 2023-09-23T21:05:06.634 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:05:06 smithi026 ceph-mon[175151]: failed to encode map e334 with expected crc 2023-09-23T21:05:06.634 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:05:06 smithi026 ceph-mon[175151]: failed to encode map e334 with expected crc 2023-09-23T21:05:06.635 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:05:06 smithi026 ceph-mon[175151]: failed to encode map e334 with expected crc 2023-09-23T21:05:06.635 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:05:06 smithi026 ceph-mon[175151]: 8.0 starting backfill to osd.5 from (258'2097,329'10281] MIN to 331'10308 2023-09-23T21:05:06.635 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:05:06 smithi026 ceph-mon[175151]: 8.11 starting backfill to osd.5 from (259'2187,328'10497] MIN to 331'10535 2023-09-23T21:05:06.635 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:05:06 smithi026 ceph-mon[175151]: 8.4 starting backfill to osd.5 from (258'1799,329'10340] MIN to 331'10356 2023-09-23T21:05:06.635 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:05:06 smithi026 ceph-mon[175151]: osdmap e335: 8 total, 8 up, 8 in 2023-09-23T21:05:07.414 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:05:07.423 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:05:07.424 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:05:07.525 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:05:07.559 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:05:07.605 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:05:07.607 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:05:07.608 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:05:07.708 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:05:07.734 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:05:07.766 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:05:07.768 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:05:07.768 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:05:07.869 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:05:07.898 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:05:08.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:05:07 smithi070 ceph-mon[168282]: failed to encode map e335 with expected crc 2023-09-23T21:05:08.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:05:07 smithi070 ceph-mon[168282]: failed to encode map e335 with expected crc 2023-09-23T21:05:08.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:05:07 smithi070 ceph-mon[168282]: failed to encode map e335 with expected crc 2023-09-23T21:05:08.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:05:07 smithi070 ceph-mon[168282]: failed to encode map e335 with expected crc 2023-09-23T21:05:08.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:05:07 smithi070 ceph-mon[168282]: failed to encode map e335 with expected crc 2023-09-23T21:05:08.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:05:07 smithi070 ceph-mon[168282]: failed to encode map e335 with expected crc 2023-09-23T21:05:08.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:05:07 smithi070 ceph-mon[168282]: failed to encode map e335 with expected crc 2023-09-23T21:05:08.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:05:07 smithi070 ceph-mon[168282]: pgmap v854: 119 pgs: 1 active+clean+remapped, 4 remapped+peering, 9 peering, 2 active+remapped+backfilling, 4 active+remapped+backfill_wait, 99 active+clean; 1.4 GiB data, 6.9 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 220 op/s; 3757/69618 objects misplaced (5.397%); 16 MiB/s, 248 objects/s recovering 2023-09-23T21:05:08.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:05:07 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:05:08.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:05:07 smithi070 ceph-mon[168282]: osdmap e336: 8 total, 8 up, 8 in 2023-09-23T21:05:08.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:05:07 smithi026 ceph-mon[175151]: failed to encode map e335 with expected crc 2023-09-23T21:05:08.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:05:07 smithi026 ceph-mon[175151]: failed to encode map e335 with expected crc 2023-09-23T21:05:08.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:05:07 smithi026 ceph-mon[175151]: failed to encode map e335 with expected crc 2023-09-23T21:05:08.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:05:07 smithi026 ceph-mon[175151]: failed to encode map e335 with expected crc 2023-09-23T21:05:08.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:05:07 smithi026 ceph-mon[175151]: failed to encode map e335 with expected crc 2023-09-23T21:05:08.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:05:07 smithi026 ceph-mon[175151]: failed to encode map e335 with expected crc 2023-09-23T21:05:08.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:05:07 smithi026 ceph-mon[175151]: failed to encode map e335 with expected crc 2023-09-23T21:05:08.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:05:07 smithi026 ceph-mon[175151]: pgmap v854: 119 pgs: 1 active+clean+remapped, 4 remapped+peering, 9 peering, 2 active+remapped+backfilling, 4 active+remapped+backfill_wait, 99 active+clean; 1.4 GiB data, 6.9 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 220 op/s; 3757/69618 objects misplaced (5.397%); 16 MiB/s, 248 objects/s recovering 2023-09-23T21:05:08.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:05:07 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:05:08.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:05:07 smithi026 ceph-mon[175151]: osdmap e336: 8 total, 8 up, 8 in 2023-09-23T21:05:08.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:05:07 smithi026 ceph-mon[179688]: failed to encode map e335 with expected crc 2023-09-23T21:05:08.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:05:07 smithi026 ceph-mon[179688]: failed to encode map e335 with expected crc 2023-09-23T21:05:08.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:05:07 smithi026 ceph-mon[179688]: failed to encode map e335 with expected crc 2023-09-23T21:05:08.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:05:07 smithi026 ceph-mon[179688]: failed to encode map e335 with expected crc 2023-09-23T21:05:08.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:05:07 smithi026 ceph-mon[179688]: failed to encode map e335 with expected crc 2023-09-23T21:05:08.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:05:07 smithi026 ceph-mon[179688]: failed to encode map e335 with expected crc 2023-09-23T21:05:08.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:05:07 smithi026 ceph-mon[179688]: failed to encode map e335 with expected crc 2023-09-23T21:05:08.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:05:07 smithi026 ceph-mon[179688]: pgmap v854: 119 pgs: 1 active+clean+remapped, 4 remapped+peering, 9 peering, 2 active+remapped+backfilling, 4 active+remapped+backfill_wait, 99 active+clean; 1.4 GiB data, 6.9 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 220 op/s; 3757/69618 objects misplaced (5.397%); 16 MiB/s, 248 objects/s recovering 2023-09-23T21:05:08.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:05:07 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:05:08.134 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:05:07 smithi026 ceph-mon[179688]: osdmap e336: 8 total, 8 up, 8 in 2023-09-23T21:05:08.194 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T21:05:08.198 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:05:08.198 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:05:08.216 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_blocked_ops 2023-09-23T21:05:08.299 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:05:08.333 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:05:08.338 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:05:08.423 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:05:08.437 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:05:08.437 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:05:08.538 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:05:08.574 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:05:08.621 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:05:08.625 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:05:08.625 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:05:08.726 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:05:08.763 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:05:08.820 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:05:08.829 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:05:08.829 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:05:08.907 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 6, 7, 4, 5] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 7, 6] 2023-09-23T21:05:08.907 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T21:05:08.907 INFO:tasks.thrashosds.thrasher:inject_pause on osd.3 2023-09-23T21:05:08.907 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-09-23T21:05:08.907 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-09-23T21:05:08.907 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 config set filestore_inject_stall 3 2023-09-23T21:05:08.930 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:05:09.006 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:05:09.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:05:08 smithi070 ceph-mon[168282]: failed to encode map e336 with expected crc 2023-09-23T21:05:09.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:05:08 smithi070 ceph-mon[168282]: failed to encode map e336 with expected crc 2023-09-23T21:05:09.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:05:08 smithi070 ceph-mon[168282]: failed to encode map e336 with expected crc 2023-09-23T21:05:09.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:05:08 smithi070 ceph-mon[168282]: failed to encode map e336 with expected crc 2023-09-23T21:05:09.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:05:08 smithi070 ceph-mon[168282]: failed to encode map e336 with expected crc 2023-09-23T21:05:09.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:05:08 smithi070 ceph-mon[168282]: failed to encode map e337 with expected crc 2023-09-23T21:05:09.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:05:08 smithi070 ceph-mon[168282]: osdmap e337: 8 total, 8 up, 8 in 2023-09-23T21:05:09.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:05:08 smithi026 ceph-mon[175151]: failed to encode map e336 with expected crc 2023-09-23T21:05:09.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:05:08 smithi026 ceph-mon[175151]: failed to encode map e336 with expected crc 2023-09-23T21:05:09.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:05:08 smithi026 ceph-mon[175151]: failed to encode map e336 with expected crc 2023-09-23T21:05:09.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:05:08 smithi026 ceph-mon[175151]: failed to encode map e336 with expected crc 2023-09-23T21:05:09.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:05:08 smithi026 ceph-mon[175151]: failed to encode map e336 with expected crc 2023-09-23T21:05:09.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:05:08 smithi026 ceph-mon[175151]: failed to encode map e337 with expected crc 2023-09-23T21:05:09.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:05:08 smithi026 ceph-mon[175151]: osdmap e337: 8 total, 8 up, 8 in 2023-09-23T21:05:09.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:05:08 smithi026 ceph-mon[179688]: failed to encode map e336 with expected crc 2023-09-23T21:05:09.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:05:08 smithi026 ceph-mon[179688]: failed to encode map e336 with expected crc 2023-09-23T21:05:09.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:05:08 smithi026 ceph-mon[179688]: failed to encode map e336 with expected crc 2023-09-23T21:05:09.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:05:08 smithi026 ceph-mon[179688]: failed to encode map e336 with expected crc 2023-09-23T21:05:09.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:05:08 smithi026 ceph-mon[179688]: failed to encode map e336 with expected crc 2023-09-23T21:05:09.134 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:05:08 smithi026 ceph-mon[179688]: failed to encode map e337 with expected crc 2023-09-23T21:05:09.134 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:05:08 smithi026 ceph-mon[179688]: osdmap e337: 8 total, 8 up, 8 in 2023-09-23T21:05:09.230 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:05:09.235 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:05:09.236 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:05:09.337 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:05:09.412 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:05:09.883 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:05:09 smithi026 ceph-mon[179688]: failed to encode map e337 with expected crc 2023-09-23T21:05:09.884 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:05:09 smithi026 ceph-mon[179688]: failed to encode map e337 with expected crc 2023-09-23T21:05:09.884 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:05:09 smithi026 ceph-mon[179688]: failed to encode map e337 with expected crc 2023-09-23T21:05:09.884 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:05:09 smithi026 ceph-mon[179688]: pgmap v857: 119 pgs: 1 active+clean+remapped, 4 remapped+peering, 14 peering, 1 active+remapped+backfilling, 99 active+clean; 1.4 GiB data, 7.7 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 290 op/s; 350/67269 objects misplaced (0.520%); 23 MiB/s, 365 objects/s recovering 2023-09-23T21:05:09.884 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:05:09 smithi026 ceph-mon[179688]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2023-09-23T21:05:09.884 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:05:09 smithi026 ceph-mon[175151]: failed to encode map e337 with expected crc 2023-09-23T21:05:09.884 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:05:09 smithi026 ceph-mon[175151]: failed to encode map e337 with expected crc 2023-09-23T21:05:09.884 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:05:09 smithi026 ceph-mon[175151]: failed to encode map e337 with expected crc 2023-09-23T21:05:09.884 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:05:09 smithi026 ceph-mon[175151]: pgmap v857: 119 pgs: 1 active+clean+remapped, 4 remapped+peering, 14 peering, 1 active+remapped+backfilling, 99 active+clean; 1.4 GiB data, 7.7 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 290 op/s; 350/67269 objects misplaced (0.520%); 23 MiB/s, 365 objects/s recovering 2023-09-23T21:05:09.885 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:05:09 smithi026 ceph-mon[175151]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2023-09-23T21:05:09.885 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 21:05:09 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[164029]: ::ffff:172.21.15.70 - - [23/Sep/2023:21:05:09] "GET /metrics HTTP/1.1" 200 36322 "" "Prometheus/2.43.0" 2023-09-23T21:05:09.941 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T21:05:10.118 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:05:10.121 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:05:09 smithi070 ceph-mon[168282]: failed to encode map e337 with expected crc 2023-09-23T21:05:10.121 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:05:09 smithi070 ceph-mon[168282]: failed to encode map e337 with expected crc 2023-09-23T21:05:10.122 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:05:09 smithi070 ceph-mon[168282]: failed to encode map e337 with expected crc 2023-09-23T21:05:10.122 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:05:09 smithi070 ceph-mon[168282]: pgmap v857: 119 pgs: 1 active+clean+remapped, 4 remapped+peering, 14 peering, 1 active+remapped+backfilling, 99 active+clean; 1.4 GiB data, 7.7 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 290 op/s; 350/67269 objects misplaced (0.520%); 23 MiB/s, 365 objects/s recovering 2023-09-23T21:05:10.122 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:05:09 smithi070 ceph-mon[168282]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2023-09-23T21:05:10.125 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:05:10.125 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:05:10.226 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:05:10.308 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:05:10.410 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:05:10.421 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:05:10.422 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:05:10.523 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:05:10.595 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:05:10.835 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:05:10.867 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T21:05:10.872 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:05:10.873 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:05:10.938 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:05:10.973 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:05:11.026 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:05:11.072 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:05:11.074 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:05:11.075 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:05:11.128 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:05:11.176 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:05:11.214 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:05:11.270 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:05:11.273 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:05:11.273 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:05:11.374 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:05:11.465 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:05:11.792 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:05:11.802 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:05:11.802 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:05:11.903 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:05:11.939 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0: {} 2023-09-23T21:05:11.976 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:05:11.983 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1: {} 2023-09-23T21:05:12.044 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2: {} 2023-09-23T21:05:12.096 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3: {} 2023-09-23T21:05:12.132 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4: {} 2023-09-23T21:05:12.170 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T21:05:12.215 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6: {} 2023-09-23T21:05:12.255 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T21:05:12.266 INFO:teuthology.orchestra.run.smithi026.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T21:05:12.266 INFO:teuthology.orchestra.run.smithi026.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T21:05:12.266 INFO:teuthology.orchestra.run.smithi026.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T21:05:12.266 INFO:teuthology.orchestra.run.smithi026.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T21:05:12.266 INFO:teuthology.orchestra.run.smithi026.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T21:05:12.266 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T21:05:12.266 INFO:teuthology.orchestra.run.smithi026.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T21:05:12.266 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T21:05:12.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:05:12 smithi070 ceph-mon[168282]: pgmap v858: 119 pgs: 1 active+clean+remapped, 6 peering, 1 active+remapped+backfilling, 111 active+clean; 1.3 GiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 388 op/s; 350/63330 objects misplaced (0.553%); 15 MiB/s, 248 objects/s recovering 2023-09-23T21:05:12.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:05:12 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/2634216389' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:05:12.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:05:12 smithi026 ceph-mon[179688]: pgmap v858: 119 pgs: 1 active+clean+remapped, 6 peering, 1 active+remapped+backfilling, 111 active+clean; 1.3 GiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 388 op/s; 350/63330 objects misplaced (0.553%); 15 MiB/s, 248 objects/s recovering 2023-09-23T21:05:12.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:05:12 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/2634216389' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:05:12.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:05:12 smithi026 ceph-mon[175151]: pgmap v858: 119 pgs: 1 active+clean+remapped, 6 peering, 1 active+remapped+backfilling, 111 active+clean; 1.3 GiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 388 op/s; 350/63330 objects misplaced (0.553%); 15 MiB/s, 248 objects/s recovering 2023-09-23T21:05:12.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:05:12 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/2634216389' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:05:12.671 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:05:12.678 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:05:12.679 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:05:12.779 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:05:12.820 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:05:12.878 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:05:12.879 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:05:12.880 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:05:12.980 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:05:13.067 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:05:13.460 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:05:13.463 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:05:13.464 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:05:13.564 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:05:13.573 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:05:13 smithi026 ceph-mon[175151]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2023-09-23T21:05:13.574 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:05:13 smithi026 ceph-mon[179688]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2023-09-23T21:05:13.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:05:13 smithi070 ceph-mon[168282]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2023-09-23T21:05:13.669 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:05:13.834 INFO:teuthology.orchestra.run.smithi026.stdout:ERROR: (22) Invalid argument 2023-09-23T21:05:13.834 INFO:teuthology.orchestra.run.smithi026.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T21:05:13.867 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T21:05:13.873 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:05:13.874 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:05:13.975 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:05:14.076 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:05:14.398 INFO:teuthology.orchestra.run.smithi026.stdout:{ 2023-09-23T21:05:14.398 INFO:teuthology.orchestra.run.smithi026.stdout: "success": "filestore_inject_stall = '3' (not observed, change may require restart) " 2023-09-23T21:05:14.398 INFO:teuthology.orchestra.run.smithi026.stdout:} 2023-09-23T21:05:14.544 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:05:14.552 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:05:14.552 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:05:14.553 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_historic_ops 2023-09-23T21:05:14.622 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:05:14 smithi070 ceph-mon[168282]: pgmap v859: 119 pgs: 119 active+clean; 1.2 GiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 652 op/s; 14 MiB/s, 228 objects/s recovering 2023-09-23T21:05:14.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:05:14 smithi026 ceph-mon[175151]: pgmap v859: 119 pgs: 119 active+clean; 1.2 GiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 652 op/s; 14 MiB/s, 228 objects/s recovering 2023-09-23T21:05:14.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:05:14 smithi026 ceph-mon[179688]: pgmap v859: 119 pgs: 119 active+clean; 1.2 GiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 652 op/s; 14 MiB/s, 228 objects/s recovering 2023-09-23T21:05:14.653 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:05:14.783 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:05:14.875 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:05:14.890 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:05:14.891 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:05:14.986 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T21:05:14.991 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:05:15.040 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:05:15.088 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:05:15.090 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:05:15.091 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:05:15.191 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:05:15.239 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:05:15.280 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:05:15.286 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:05:15.287 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:05:15.380 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:05:15.384 INFO:teuthology.orchestra.run.smithi026.stderr:nodeep-scrub is set 2023-09-23T21:05:15.387 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:05:15.426 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:05:15.473 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:05:15.479 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:05:15.480 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:05:15.580 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:05:15.611 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:05:15 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/278648439' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:05:15.611 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:05:15 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:05:15.618 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:05:15.634 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:05:15 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/278648439' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:05:15.634 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:05:15 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:05:15.634 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:05:15 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/278648439' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:05:15.634 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:05:15 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:05:15.675 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:05:15.686 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:05:15.686 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:05:15.787 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:05:15.833 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:05:15.883 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:05:15.886 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:05:15.887 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:05:15.988 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:05:16.066 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:05:16.133 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:05:16.140 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:05:16.140 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:05:16.241 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:05:16.268 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:05:16.317 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:05:16.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:05:16 smithi070 ceph-mon[168282]: pgmap v860: 119 pgs: 119 active+clean; 1.1 GiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 689 op/s; 11 MiB/s, 182 objects/s recovering 2023-09-23T21:05:16.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:05:16 smithi070 ceph-mon[168282]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:05:16.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:05:16 smithi070 ceph-mon[168282]: failed to encode map e338 with expected crc 2023-09-23T21:05:16.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:05:16 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T21:05:16.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:05:16 smithi070 ceph-mon[168282]: osdmap e338: 8 total, 8 up, 8 in 2023-09-23T21:05:16.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:05:16 smithi070 ceph-mon[168282]: failed to encode map e338 with expected crc 2023-09-23T21:05:16.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:05:16 smithi070 ceph-mon[168282]: failed to encode map e338 with expected crc 2023-09-23T21:05:16.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:05:16 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:05:16.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:05:16 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:05:16.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:05:16 smithi026 ceph-mon[179688]: pgmap v860: 119 pgs: 119 active+clean; 1.1 GiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 689 op/s; 11 MiB/s, 182 objects/s recovering 2023-09-23T21:05:16.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:05:16 smithi026 ceph-mon[179688]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:05:16.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:05:16 smithi026 ceph-mon[179688]: failed to encode map e338 with expected crc 2023-09-23T21:05:16.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:05:16 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T21:05:16.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:05:16 smithi026 ceph-mon[179688]: osdmap e338: 8 total, 8 up, 8 in 2023-09-23T21:05:16.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:05:16 smithi026 ceph-mon[179688]: failed to encode map e338 with expected crc 2023-09-23T21:05:16.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:05:16 smithi026 ceph-mon[179688]: failed to encode map e338 with expected crc 2023-09-23T21:05:16.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:05:16 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:05:16.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:05:16 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:05:16.633 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:05:16 smithi026 ceph-mon[175151]: pgmap v860: 119 pgs: 119 active+clean; 1.1 GiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 689 op/s; 11 MiB/s, 182 objects/s recovering 2023-09-23T21:05:16.633 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:05:16 smithi026 ceph-mon[175151]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:05:16.633 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:05:16 smithi026 ceph-mon[175151]: failed to encode map e338 with expected crc 2023-09-23T21:05:16.633 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:05:16 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T21:05:16.633 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:05:16 smithi026 ceph-mon[175151]: osdmap e338: 8 total, 8 up, 8 in 2023-09-23T21:05:16.633 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:05:16 smithi026 ceph-mon[175151]: failed to encode map e338 with expected crc 2023-09-23T21:05:16.633 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:05:16 smithi026 ceph-mon[175151]: failed to encode map e338 with expected crc 2023-09-23T21:05:16.633 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:05:16 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:05:16.633 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:05:16 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:05:16.644 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:05:16.651 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:05:16.651 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:05:16.753 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:05:16.843 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:05:17.209 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:05:17.225 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:05:17.226 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:05:17.327 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:05:17.377 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:05:17.433 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:05:17.442 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:05:17.442 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:05:17.543 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:05:17.581 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:05:17.625 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:05:17.628 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:05:17.628 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:05:17.727 INFO:teuthology.orchestra.run.smithi026.stdout:ERROR: (22) Invalid argument 2023-09-23T21:05:17.727 INFO:teuthology.orchestra.run.smithi026.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T21:05:17.728 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:05:17.829 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:05:17.901 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:05:17.918 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:05:17.919 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:05:17.957 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T21:05:18.020 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:05:18.059 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:05:18.109 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:05:18.117 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:05:18.118 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:05:18.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:05:17 smithi026 ceph-mon[175151]: pgmap v862: 119 pgs: 119 active+clean; 1.0 GiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 676 op/s; 1.1 MiB/s, 17 objects/s recovering 2023-09-23T21:05:18.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:05:17 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:05:18.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:05:17 smithi026 ceph-mon[179688]: pgmap v862: 119 pgs: 119 active+clean; 1.0 GiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 676 op/s; 1.1 MiB/s, 17 objects/s recovering 2023-09-23T21:05:18.134 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:05:17 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:05:18.218 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:05:18.295 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_ops_in_flight 2023-09-23T21:05:18.335 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:05:18.340 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:05:17 smithi070 ceph-mon[168282]: pgmap v862: 119 pgs: 119 active+clean; 1.0 GiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 676 op/s; 1.1 MiB/s, 17 objects/s recovering 2023-09-23T21:05:18.340 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:05:17 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:05:18.420 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:05:18.424 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:05:18.425 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:05:18.525 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:05:18.564 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:05:18.629 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:05:18.633 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:05:18.634 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:05:18.734 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:05:18.736 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0: {} 2023-09-23T21:05:18.779 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1: {} 2023-09-23T21:05:18.801 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:05:18.833 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2: {} 2023-09-23T21:05:18.856 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:05:18.861 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:05:18.861 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3: {} 2023-09-23T21:05:18.866 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:05:18.866 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:05:18.902 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4: {} 2023-09-23T21:05:18.963 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T21:05:18.967 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:05:18.995 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6: {} 2023-09-23T21:05:19.015 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:05:19.037 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T21:05:19.042 INFO:teuthology.orchestra.run.smithi026.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T21:05:19.042 INFO:teuthology.orchestra.run.smithi026.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T21:05:19.042 INFO:teuthology.orchestra.run.smithi026.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T21:05:19.042 INFO:teuthology.orchestra.run.smithi026.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T21:05:19.042 INFO:teuthology.orchestra.run.smithi026.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T21:05:19.043 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T21:05:19.043 INFO:teuthology.orchestra.run.smithi026.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T21:05:19.043 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T21:05:19.045 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T21:05:19.072 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:05:19.077 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:05:19.077 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:05:19.178 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:05:19.179 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:05:18 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/48808797' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:05:19.179 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:05:18 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/48808797' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:05:19.273 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:05:19.322 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:05:18 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/48808797' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:05:19.689 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:05:19.695 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:05:19.695 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:05:19.797 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:05:19.855 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 6, 7, 4, 5] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 7, 6] 2023-09-23T21:05:19.855 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T21:05:19.855 INFO:tasks.thrashosds.thrasher:Killing osd 1, live_osds are [0, 1, 2, 3, 4, 5, 7, 6] 2023-09-23T21:05:19.856 INFO:tasks.cephadm.osd.1:Stopping osd.1... 2023-09-23T21:05:19.856 DEBUG:teuthology.orchestra.run.smithi026:> sudo systemctl stop ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7@osd.1 2023-09-23T21:05:19.863 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:05:19.884 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 21:05:19 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[164029]: ::ffff:172.21.15.70 - - [23/Sep/2023:21:05:19] "GET /metrics HTTP/1.1" 200 36329 "" "Prometheus/2.43.0" 2023-09-23T21:05:19.921 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:05:19.921 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:05:19 smithi070 ceph-mon[168282]: pgmap v863: 119 pgs: 119 active+clean; 893 MiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 810 op/s; 966 KiB/s, 15 objects/s recovering 2023-09-23T21:05:19.922 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:05:19.922 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:05:20.023 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:05:20.069 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:05:20.156 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:05:20.159 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:05:20.159 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:05:20.260 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:05:20.319 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:05:19 smithi026 ceph-mon[179688]: pgmap v863: 119 pgs: 119 active+clean; 893 MiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 810 op/s; 966 KiB/s, 15 objects/s recovering 2023-09-23T21:05:20.319 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:05:19 smithi026 ceph-mon[175151]: pgmap v863: 119 pgs: 119 active+clean; 893 MiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 810 op/s; 966 KiB/s, 15 objects/s recovering 2023-09-23T21:05:20.320 INFO:journalctl@ceph.osd.1.smithi026.stdout:Sep 23 21:05:19 smithi026 systemd[1]: Stopping Ceph osd.1 for b3f8e4ec-5a50-11ee-8db4-212e2dc638e7... 2023-09-23T21:05:20.329 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:05:20.529 INFO:teuthology.orchestra.run.smithi070.stdout:{ 2023-09-23T21:05:20.529 INFO:teuthology.orchestra.run.smithi070.stdout: "ops": [ 2023-09-23T21:05:20.529 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:05:20.529 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_repop(client.35469.0:211659 8.5 e338/281 8:a2776d71:::benchmark_data_smithi026_301269_object100902:head v 338'10838, mlcod=338'10836)", 2023-09-23T21:05:20.529 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:05:20.524604+0000", 2023-09-23T21:05:20.530 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 0.003058926, 2023-09-23T21:05:20.530 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0030875550000000001, 2023-09-23T21:05:20.530 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:05:20.530 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:05:20.530 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:05:20.530 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:05:20.530 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:05:20.531 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:20.524604+0000", 2023-09-23T21:05:20.531 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:05:20.531 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:05:20.531 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:05:20.531 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:05:20.531 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:20.524604+0000", 2023-09-23T21:05:20.531 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:05:20.532 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:05:20.532 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:05:20.532 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:05:20.532 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:20.524603+0000", 2023-09-23T21:05:20.532 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.999999 2023-09-23T21:05:20.532 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:05:20.532 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:05:20.533 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:05:20.533 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:20.524610+0000", 2023-09-23T21:05:20.533 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.0060000000000003e-06 2023-09-23T21:05:20.533 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:05:20.533 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:05:20.533 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:05:20.533 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:20.524621+0000", 2023-09-23T21:05:20.534 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.0814e-05 2023-09-23T21:05:20.534 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:05:20.534 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:05:20.534 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:05:20.534 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:20.524627+0000", 2023-09-23T21:05:20.534 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.7320000000000003e-06 2023-09-23T21:05:20.534 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:05:20.535 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:05:20.535 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:05:20.535 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:20.524670+0000", 2023-09-23T21:05:20.535 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.3417000000000001e-05 2023-09-23T21:05:20.535 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:05:20.535 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:05:20.535 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:05:20.536 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:20.524689+0000", 2023-09-23T21:05:20.536 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.8552000000000001e-05 2023-09-23T21:05:20.536 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:05:20.536 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:05:20.536 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:05:20.536 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:20.527663+0000", 2023-09-23T21:05:20.536 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0029741479999999998 2023-09-23T21:05:20.537 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:05:20.537 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:05:20.537 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:05:20.537 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:20.527695+0000", 2023-09-23T21:05:20.537 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.2200999999999999e-05 2023-09-23T21:05:20.537 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:05:20.537 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:05:20.538 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:05:20.538 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:05:20.538 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:05:20.538 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.35469.0:211663 8.18 8:1bd0671e:::benchmark_data_smithi026_301269_object100906:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e338)", 2023-09-23T21:05:20.538 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:05:20.526804+0000", 2023-09-23T21:05:20.538 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 0.00085922699999999995, 2023-09-23T21:05:20.538 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00097476400000000003, 2023-09-23T21:05:20.539 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:05:20.539 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "waiting for sub ops", 2023-09-23T21:05:20.539 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:05:20.539 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.35469", 2023-09-23T21:05:20.539 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/4041424623", 2023-09-23T21:05:20.539 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 211663 2023-09-23T21:05:20.539 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:05:20.540 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:05:20.540 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:05:20.540 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:05:20.540 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:20.526804+0000", 2023-09-23T21:05:20.540 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:05:20.540 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:05:20.540 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:05:20.541 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:05:20.541 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:20.526804+0000", 2023-09-23T21:05:20.541 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:05:20.541 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:05:20.541 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:05:20.541 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:05:20.541 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:20.526802+0000", 2023-09-23T21:05:20.542 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999981 2023-09-23T21:05:20.542 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:05:20.542 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:05:20.542 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:05:20.542 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:20.526809+0000", 2023-09-23T21:05:20.542 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.4510000000000003e-06 2023-09-23T21:05:20.542 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:05:20.543 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:05:20.543 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:05:20.543 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:20.526822+0000", 2023-09-23T21:05:20.543 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.2308e-05 2023-09-23T21:05:20.543 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:05:20.543 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:05:20.543 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:05:20.544 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:20.526828+0000", 2023-09-23T21:05:20.544 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.2110000000000003e-06 2023-09-23T21:05:20.544 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:05:20.544 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:05:20.544 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:05:20.544 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:20.526874+0000", 2023-09-23T21:05:20.544 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.6316999999999997e-05 2023-09-23T21:05:20.545 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:05:20.545 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:05:20.545 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:05:20.545 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:20.526983+0000", 2023-09-23T21:05:20.545 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000109187 2023-09-23T21:05:20.545 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:05:20.545 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:05:20.546 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 3,5", 2023-09-23T21:05:20.546 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:20.527364+0000", 2023-09-23T21:05:20.546 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000380181 2023-09-23T21:05:20.546 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:05:20.546 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:05:20.546 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:05:20.546 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:05:20.547 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:05:20.547 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_repop(client.35469.0:211661 8.e e338/254 8:7557b451:::benchmark_data_smithi026_301269_object100904:head v 338'13388, mlcod=338'13386)", 2023-09-23T21:05:20.547 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:05:20.527578+0000", 2023-09-23T21:05:20.547 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 8.4965999999999999e-05, 2023-09-23T21:05:20.547 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000277791, 2023-09-23T21:05:20.547 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:05:20.547 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "started", 2023-09-23T21:05:20.548 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:05:20.548 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:05:20.548 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:05:20.548 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:20.527578+0000", 2023-09-23T21:05:20.548 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:05:20.548 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:05:20.548 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:05:20.548 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:05:20.548 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:20.527578+0000", 2023-09-23T21:05:20.548 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:05:20.549 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:05:20.549 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:05:20.549 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:05:20.549 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:20.527577+0000", 2023-09-23T21:05:20.549 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999995 2023-09-23T21:05:20.549 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:05:20.549 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:05:20.549 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:05:20.549 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:20.527584+0000", 2023-09-23T21:05:20.549 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.216e-06 2023-09-23T21:05:20.550 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:05:20.550 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:05:20.550 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:05:20.550 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:20.527595+0000", 2023-09-23T21:05:20.550 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.0399e-05 2023-09-23T21:05:20.550 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:05:20.550 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:05:20.550 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:05:20.550 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:20.527600+0000", 2023-09-23T21:05:20.550 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.1279999999999999e-06 2023-09-23T21:05:20.551 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:05:20.551 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:05:20.551 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:05:20.551 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:20.527750+0000", 2023-09-23T21:05:20.551 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00015017900000000001 2023-09-23T21:05:20.551 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:05:20.551 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:05:20.551 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:05:20.551 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:20.527767+0000", 2023-09-23T21:05:20.551 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.7084000000000001e-05 2023-09-23T21:05:20.552 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:05:20.552 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:05:20.552 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:05:20.552 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:05:20.552 INFO:teuthology.orchestra.run.smithi070.stdout: ], 2023-09-23T21:05:20.552 INFO:teuthology.orchestra.run.smithi070.stdout: "num_ops": 3 2023-09-23T21:05:20.552 INFO:teuthology.orchestra.run.smithi070.stdout:} 2023-09-23T21:05:20.926 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:05:20.934 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:05:20.935 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:05:21.036 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:05:21.099 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:05:21.150 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:05:21.152 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:05:21.153 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:05:21.166 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_blocked_ops 2023-09-23T21:05:21.247 INFO:journalctl@ceph.osd.1.smithi026.stdout:Sep 23 21:05:21 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd-1[196493]: 2023-09-23T21:05:21.132+0000 7f8e8114d700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-osd -n osd.1 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2023-09-23T21:05:21.248 INFO:journalctl@ceph.osd.1.smithi026.stdout:Sep 23 21:05:21 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd-1[196493]: 2023-09-23T21:05:21.132+0000 7f8e8114d700 -1 osd.1 338 *** Got signal Terminated *** 2023-09-23T21:05:21.248 INFO:journalctl@ceph.osd.1.smithi026.stdout:Sep 23 21:05:21 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd-1[196493]: 2023-09-23T21:05:21.132+0000 7f8e8114d700 -1 osd.1 338 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-09-23T21:05:21.248 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:05:21 smithi026 ceph-mon[175151]: osd.1 marked itself down and dead 2023-09-23T21:05:21.253 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:05:21.327 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:05:21.360 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:05:21.362 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:05:21 smithi070 ceph-mon[168282]: osd.1 marked itself down and dead 2023-09-23T21:05:21.363 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:05:21.363 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:05:21.464 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:05:21.495 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:05:21.532 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:05:21.534 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:05:21 smithi026 ceph-mon[179688]: osd.1 marked itself down and dead 2023-09-23T21:05:21.535 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:05:21.535 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:05:21.636 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:05:21.662 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:05:21.693 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:05:21.695 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:05:21.696 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:05:21.768 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T21:05:21.797 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:05:21.840 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:05:21.841 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T21:05:22.131 INFO:journalctl@ceph.osd.1.smithi026.stdout:Sep 23 21:05:21 smithi026 bash[336143]: ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd-1 2023-09-23T21:05:22.174 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:05:22.177 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:05:22.177 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:05:22.278 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:05:22.318 INFO:teuthology.orchestra.run.smithi026.stderr:noscrub is unset 2023-09-23T21:05:22.368 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:05:22.372 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:05:22 smithi070 ceph-mon[168282]: pgmap v864: 119 pgs: 119 active+clean; 742 MiB data, 5.5 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 921 op/s; 787 KiB/s, 11 objects/s recovering 2023-09-23T21:05:22.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:05:22 smithi070 ceph-mon[168282]: Health check failed: 1 osds down (OSD_DOWN) 2023-09-23T21:05:22.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:05:22 smithi070 ceph-mon[168282]: failed to encode map e339 with expected crc 2023-09-23T21:05:22.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:05:22 smithi070 ceph-mon[168282]: osdmap e339: 8 total, 7 up, 8 in 2023-09-23T21:05:22.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:05:22 smithi070 ceph-mon[168282]: failed to encode map e339 with expected crc 2023-09-23T21:05:22.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:05:22 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/492052557' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T21:05:22.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:05:22 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T21:05:22.458 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:05:22 smithi026 ceph-mon[179688]: pgmap v864: 119 pgs: 119 active+clean; 742 MiB data, 5.5 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 921 op/s; 787 KiB/s, 11 objects/s recovering 2023-09-23T21:05:22.458 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:05:22 smithi026 ceph-mon[179688]: Health check failed: 1 osds down (OSD_DOWN) 2023-09-23T21:05:22.459 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:05:22 smithi026 ceph-mon[179688]: failed to encode map e339 with expected crc 2023-09-23T21:05:22.459 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:05:22 smithi026 ceph-mon[179688]: osdmap e339: 8 total, 7 up, 8 in 2023-09-23T21:05:22.459 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:05:22 smithi026 ceph-mon[179688]: failed to encode map e339 with expected crc 2023-09-23T21:05:22.459 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:05:22 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/492052557' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T21:05:22.459 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:05:22 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T21:05:22.459 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:05:22 smithi026 ceph-mon[175151]: pgmap v864: 119 pgs: 119 active+clean; 742 MiB data, 5.5 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 921 op/s; 787 KiB/s, 11 objects/s recovering 2023-09-23T21:05:22.459 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:05:22 smithi026 ceph-mon[175151]: Health check failed: 1 osds down (OSD_DOWN) 2023-09-23T21:05:22.459 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:05:22 smithi026 ceph-mon[175151]: failed to encode map e339 with expected crc 2023-09-23T21:05:22.459 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:05:22 smithi026 ceph-mon[175151]: osdmap e339: 8 total, 7 up, 8 in 2023-09-23T21:05:22.460 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:05:22 smithi026 ceph-mon[175151]: failed to encode map e339 with expected crc 2023-09-23T21:05:22.460 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:05:22 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/492052557' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T21:05:22.460 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:05:22 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T21:05:22.491 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T21:05:22.498 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:05:22.499 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:05:22.600 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:05:22.691 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:05:22.760 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:05:22.826 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:05:22.826 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:05:22.827 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:05:22.928 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:05:23.005 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:05:23.174 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:05:23.181 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:05:23.181 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:05:23.257 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:05:23 smithi070 ceph-mon[168282]: failed to encode map e339 with expected crc 2023-09-23T21:05:23.257 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:05:23 smithi070 ceph-mon[168282]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:05:23.257 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:05:23 smithi070 ceph-mon[168282]: failed to encode map e340 with expected crc 2023-09-23T21:05:23.257 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:05:23 smithi070 ceph-mon[168282]: failed to encode map e340 with expected crc 2023-09-23T21:05:23.257 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:05:23 smithi070 ceph-mon[168282]: failed to encode map e340 with expected crc 2023-09-23T21:05:23.257 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:05:23 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T21:05:23.257 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:05:23 smithi070 ceph-mon[168282]: osdmap e340: 8 total, 7 up, 8 in 2023-09-23T21:05:23.258 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:05:23 smithi070 ceph-mon[168282]: failed to encode map e340 with expected crc 2023-09-23T21:05:23.258 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:05:23 smithi070 ceph-mon[168282]: 8.18 scrub starts 2023-09-23T21:05:23.258 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:05:23 smithi070 ceph-mon[168282]: 8.18 scrub ok 2023-09-23T21:05:23.261 INFO:teuthology.orchestra.run.smithi070.stdout:{ 2023-09-23T21:05:23.261 INFO:teuthology.orchestra.run.smithi070.stdout: "ops": [], 2023-09-23T21:05:23.261 INFO:teuthology.orchestra.run.smithi070.stdout: "complaint_time": 30, 2023-09-23T21:05:23.261 INFO:teuthology.orchestra.run.smithi070.stdout: "num_blocked_ops": 0 2023-09-23T21:05:23.261 INFO:teuthology.orchestra.run.smithi070.stdout:} 2023-09-23T21:05:23.282 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:05:23.366 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:05:23.381 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:05:23 smithi026 ceph-mon[179688]: failed to encode map e339 with expected crc 2023-09-23T21:05:23.381 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:05:23 smithi026 ceph-mon[179688]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:05:23.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:05:23 smithi026 ceph-mon[179688]: failed to encode map e340 with expected crc 2023-09-23T21:05:23.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:05:23 smithi026 ceph-mon[179688]: failed to encode map e340 with expected crc 2023-09-23T21:05:23.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:05:23 smithi026 ceph-mon[179688]: failed to encode map e340 with expected crc 2023-09-23T21:05:23.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:05:23 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T21:05:23.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:05:23 smithi026 ceph-mon[179688]: osdmap e340: 8 total, 7 up, 8 in 2023-09-23T21:05:23.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:05:23 smithi026 ceph-mon[179688]: failed to encode map e340 with expected crc 2023-09-23T21:05:23.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:05:23 smithi026 ceph-mon[179688]: 8.18 scrub starts 2023-09-23T21:05:23.383 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:05:23 smithi026 ceph-mon[179688]: 8.18 scrub ok 2023-09-23T21:05:23.383 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:05:23 smithi026 ceph-mon[175151]: failed to encode map e339 with expected crc 2023-09-23T21:05:23.383 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:05:23 smithi026 ceph-mon[175151]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:05:23.383 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:05:23 smithi026 ceph-mon[175151]: failed to encode map e340 with expected crc 2023-09-23T21:05:23.383 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:05:23 smithi026 ceph-mon[175151]: failed to encode map e340 with expected crc 2023-09-23T21:05:23.384 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:05:23 smithi026 ceph-mon[175151]: failed to encode map e340 with expected crc 2023-09-23T21:05:23.384 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:05:23 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T21:05:23.384 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:05:23 smithi026 ceph-mon[175151]: osdmap e340: 8 total, 7 up, 8 in 2023-09-23T21:05:23.384 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:05:23 smithi026 ceph-mon[175151]: failed to encode map e340 with expected crc 2023-09-23T21:05:23.384 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:05:23 smithi026 ceph-mon[175151]: 8.18 scrub starts 2023-09-23T21:05:23.384 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:05:23 smithi026 ceph-mon[175151]: 8.18 scrub ok 2023-09-23T21:05:23.644 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:05:23.649 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:05:23.649 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:05:23.711 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_historic_ops 2023-09-23T21:05:23.750 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:05:23.837 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:05:23.912 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:05:23.915 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:05:23.915 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:05:24.016 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:05:24.090 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:05:24.261 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:05:24 smithi070 ceph-mon[168282]: failed to encode map e340 with expected crc 2023-09-23T21:05:24.262 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:05:24 smithi070 ceph-mon[168282]: failed to encode map e340 with expected crc 2023-09-23T21:05:24.262 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:05:24 smithi070 ceph-mon[168282]: pgmap v867: 119 pgs: 1 activating+undersized, 1 active+undersized+degraded, 6 active+undersized, 18 peering, 8 stale+active+clean, 85 active+clean; 603 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 938 op/s; 1/29016 objects degraded (0.003%) 2023-09-23T21:05:24.262 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:05:24 smithi070 ceph-mon[168282]: Health check failed: Reduced data availability: 5 pgs inactive, 5 pgs peering (PG_AVAILABILITY) 2023-09-23T21:05:24.262 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:05:24 smithi070 ceph-mon[168282]: Health check failed: Degraded data redundancy: 1/29016 objects degraded (0.003%), 1 pg degraded (PG_DEGRADED) 2023-09-23T21:05:24.262 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:05:24 smithi070 ceph-mon[168282]: 8.19 scrub starts 2023-09-23T21:05:24.262 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:05:24 smithi070 ceph-mon[168282]: 8.19 scrub ok 2023-09-23T21:05:24.316 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T21:05:24.323 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:05:24.324 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:05:24.424 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:05:24.450 DEBUG:teuthology.orchestra.run.smithi026:> sudo pkill -f 'journalctl -f -n 0 -u ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7@osd.1.service' 2023-09-23T21:05:24.483 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:05:24.502 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T21:05:24.556 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:05:24 smithi026 ceph-mon[179688]: failed to encode map e340 with expected crc 2023-09-23T21:05:24.556 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:05:24 smithi026 ceph-mon[179688]: failed to encode map e340 with expected crc 2023-09-23T21:05:24.556 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:05:24 smithi026 ceph-mon[179688]: pgmap v867: 119 pgs: 1 activating+undersized, 1 active+undersized+degraded, 6 active+undersized, 18 peering, 8 stale+active+clean, 85 active+clean; 603 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 938 op/s; 1/29016 objects degraded (0.003%) 2023-09-23T21:05:24.556 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:05:24 smithi026 ceph-mon[179688]: Health check failed: Reduced data availability: 5 pgs inactive, 5 pgs peering (PG_AVAILABILITY) 2023-09-23T21:05:24.556 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:05:24 smithi026 ceph-mon[179688]: Health check failed: Degraded data redundancy: 1/29016 objects degraded (0.003%), 1 pg degraded (PG_DEGRADED) 2023-09-23T21:05:24.556 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:05:24 smithi026 ceph-mon[179688]: 8.19 scrub starts 2023-09-23T21:05:24.556 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:05:24 smithi026 ceph-mon[179688]: 8.19 scrub ok 2023-09-23T21:05:24.557 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:05:24.557 INFO:journalctl@ceph.osd.1.smithi026.stdout:Sep 23 21:05:24 smithi026 systemd[1]: ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7@osd.1.service: Succeeded. 2023-09-23T21:05:24.557 INFO:journalctl@ceph.osd.1.smithi026.stdout:Sep 23 21:05:24 smithi026 systemd[1]: Stopped Ceph osd.1 for b3f8e4ec-5a50-11ee-8db4-212e2dc638e7. 2023-09-23T21:05:24.559 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:05:24 smithi026 ceph-mon[175151]: failed to encode map e340 with expected crc 2023-09-23T21:05:24.559 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:05:24 smithi026 ceph-mon[175151]: failed to encode map e340 with expected crc 2023-09-23T21:05:24.559 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:05:24 smithi026 ceph-mon[175151]: pgmap v867: 119 pgs: 1 activating+undersized, 1 active+undersized+degraded, 6 active+undersized, 18 peering, 8 stale+active+clean, 85 active+clean; 603 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 938 op/s; 1/29016 objects degraded (0.003%) 2023-09-23T21:05:24.559 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:05:24 smithi026 ceph-mon[175151]: Health check failed: Reduced data availability: 5 pgs inactive, 5 pgs peering (PG_AVAILABILITY) 2023-09-23T21:05:24.559 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:05:24 smithi026 ceph-mon[175151]: Health check failed: Degraded data redundancy: 1/29016 objects degraded (0.003%), 1 pg degraded (PG_DEGRADED) 2023-09-23T21:05:24.559 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:05:24 smithi026 ceph-mon[175151]: 8.19 scrub starts 2023-09-23T21:05:24.559 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:05:24 smithi026 ceph-mon[175151]: 8.19 scrub ok 2023-09-23T21:05:24.564 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:05:24.564 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:05:24.593 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-09-23T21:05:24.593 INFO:tasks.cephadm.osd.1:Stopped osd.1 2023-09-23T21:05:24.665 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:05:24.722 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:05:24.779 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:05:24.787 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:05:24.787 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:05:24.863 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0: {} 2023-09-23T21:05:24.875 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T21:05:24.888 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:05:24.902 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2: {} 2023-09-23T21:05:24.958 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3: {} 2023-09-23T21:05:24.979 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:05:25.004 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4: {} 2023-09-23T21:05:25.046 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T21:05:25.080 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:05:25.089 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:05:25.089 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:05:25.101 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6: {} 2023-09-23T21:05:25.141 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T21:05:25.142 INFO:teuthology.orchestra.run.smithi026.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T21:05:25.142 INFO:teuthology.orchestra.run.smithi026.stderr:Error ENXIO: problem getting command descriptions from osd.1 2023-09-23T21:05:25.142 INFO:teuthology.orchestra.run.smithi026.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T21:05:25.142 INFO:teuthology.orchestra.run.smithi026.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T21:05:25.142 INFO:teuthology.orchestra.run.smithi026.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T21:05:25.142 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T21:05:25.142 INFO:teuthology.orchestra.run.smithi026.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T21:05:25.142 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T21:05:25.190 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:05:25.228 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:05:25.393 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:05:25.402 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:05:25.402 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:05:25.503 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:05:25.569 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:05:25 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/2982288141' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:05:25.579 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:05:25.580 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:05:25 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/2982288141' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:05:25.581 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:05:25 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/2982288141' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:05:25.627 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T21:05:25.627 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T21:05:25.702 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:05:25.715 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:05:25.715 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:05:25.811 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:05:25.816 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:05:25.863 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:05:25.972 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:05:25.978 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:05:25.978 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:05:26.069 INFO:teuthology.orchestra.run.smithi070.stdout:ERROR: (22) Invalid argument 2023-09-23T21:05:26.069 INFO:teuthology.orchestra.run.smithi070.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T21:05:26.080 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:05:26.148 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:05:26.415 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:05:26.421 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:05:26.421 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:05:26.422 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:05:26 smithi070 ceph-mon[168282]: pgmap v868: 119 pgs: 1 activating+undersized, 2 active+undersized+degraded, 12 active+undersized, 18 peering, 4 stale+active+clean, 82 active+clean; 573 MiB data, 4.7 GiB used, 710 GiB / 715 GiB avail; 3.4 KiB/s rd, 9.1 KiB/s wr, 937 op/s; 245/27537 objects degraded (0.890%); 6.5 KiB/s, 0 objects/s recovering 2023-09-23T21:05:26.422 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:05:26 smithi070 ceph-mon[168282]: 8.8 scrub starts 2023-09-23T21:05:26.423 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:05:26 smithi070 ceph-mon[168282]: 8.8 scrub ok 2023-09-23T21:05:26.460 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_ops_in_flight 2023-09-23T21:05:26.511 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:05:26 smithi026 ceph-mon[175151]: pgmap v868: 119 pgs: 1 activating+undersized, 2 active+undersized+degraded, 12 active+undersized, 18 peering, 4 stale+active+clean, 82 active+clean; 573 MiB data, 4.7 GiB used, 710 GiB / 715 GiB avail; 3.4 KiB/s rd, 9.1 KiB/s wr, 937 op/s; 245/27537 objects degraded (0.890%); 6.5 KiB/s, 0 objects/s recovering 2023-09-23T21:05:26.511 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:05:26 smithi026 ceph-mon[175151]: 8.8 scrub starts 2023-09-23T21:05:26.511 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:05:26 smithi026 ceph-mon[175151]: 8.8 scrub ok 2023-09-23T21:05:26.511 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:05:26 smithi026 ceph-mon[179688]: pgmap v868: 119 pgs: 1 activating+undersized, 2 active+undersized+degraded, 12 active+undersized, 18 peering, 4 stale+active+clean, 82 active+clean; 573 MiB data, 4.7 GiB used, 710 GiB / 715 GiB avail; 3.4 KiB/s rd, 9.1 KiB/s wr, 937 op/s; 245/27537 objects degraded (0.890%); 6.5 KiB/s, 0 objects/s recovering 2023-09-23T21:05:26.511 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:05:26 smithi026 ceph-mon[179688]: 8.8 scrub starts 2023-09-23T21:05:26.511 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:05:26 smithi026 ceph-mon[179688]: 8.8 scrub ok 2023-09-23T21:05:26.523 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:05:26.575 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:05:26.757 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:05:26.760 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:05:26.761 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:05:26.861 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:05:26.936 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:05:27.011 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:05:27.015 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:05:27.015 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:05:27.116 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:05:27.201 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:05:27.266 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:05:27.279 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:05:27.279 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:05:27.328 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T21:05:27.380 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:05:27.444 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:05:27.576 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:05:27.581 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:05:27.582 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:05:27.628 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T21:05:27.683 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:05:27.781 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:05:27.868 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T21:05:27.872 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:05:27.873 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:05:27.974 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:05:28.032 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:05:28.139 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:05:28.146 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:05:28.147 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:05:28.247 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:05:28.298 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:05:28 smithi026 ceph-mon[175151]: pgmap v869: 119 pgs: 1 activating+undersized, 8 active+undersized+degraded, 24 active+undersized, 13 peering, 73 active+clean; 493 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail; 3.4 KiB/s rd, 9.1 KiB/s wr, 803 op/s; 1934/23697 objects degraded (8.161%); 6.5 KiB/s, 0 objects/s recovering 2023-09-23T21:05:28.298 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:05:28 smithi026 ceph-mon[175151]: 8.10 scrub starts 2023-09-23T21:05:28.299 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:05:28 smithi026 ceph-mon[175151]: 8.10 scrub ok 2023-09-23T21:05:28.299 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:05:28 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/2772009942' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:05:28.306 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:05:28 smithi070 ceph-mon[168282]: pgmap v869: 119 pgs: 1 activating+undersized, 8 active+undersized+degraded, 24 active+undersized, 13 peering, 73 active+clean; 493 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail; 3.4 KiB/s rd, 9.1 KiB/s wr, 803 op/s; 1934/23697 objects degraded (8.161%); 6.5 KiB/s, 0 objects/s recovering 2023-09-23T21:05:28.307 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:05:28 smithi070 ceph-mon[168282]: 8.10 scrub starts 2023-09-23T21:05:28.307 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:05:28 smithi070 ceph-mon[168282]: 8.10 scrub ok 2023-09-23T21:05:28.307 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:05:28 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/2772009942' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:05:28.316 INFO:teuthology.orchestra.run.smithi026.stderr:nodeep-scrub is unset 2023-09-23T21:05:28.343 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:05:28.346 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:05:28.476 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:05:28.481 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:05:28.481 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:05:28.582 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:05:28.625 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:05:28 smithi026 ceph-mon[179688]: pgmap v869: 119 pgs: 1 activating+undersized, 8 active+undersized+degraded, 24 active+undersized, 13 peering, 73 active+clean; 493 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail; 3.4 KiB/s rd, 9.1 KiB/s wr, 803 op/s; 1934/23697 objects degraded (8.161%); 6.5 KiB/s, 0 objects/s recovering 2023-09-23T21:05:28.626 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:05:28 smithi026 ceph-mon[179688]: 8.10 scrub starts 2023-09-23T21:05:28.626 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:05:28 smithi026 ceph-mon[179688]: 8.10 scrub ok 2023-09-23T21:05:28.626 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:05:28 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/2772009942' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:05:28.635 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:05:28.694 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T21:05:28.703 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:05:28.703 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:05:28.805 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:05:28.873 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:05:28.885 INFO:teuthology.orchestra.run.smithi070.stdout:ERROR: (22) Invalid argument 2023-09-23T21:05:28.885 INFO:teuthology.orchestra.run.smithi070.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T21:05:28.914 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:05:28.921 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:05:28.922 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:05:29.023 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:05:29.081 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:05:29.213 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:05:29.217 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:05:29.218 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:05:29.302 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:05:29 smithi026 ceph-mon[175151]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T21:05:29.302 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:05:29 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/2772009942' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T21:05:29.302 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:05:29 smithi026 ceph-mon[175151]: osdmap e341: 8 total, 7 up, 8 in 2023-09-23T21:05:29.302 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:05:29 smithi026 ceph-mon[175151]: failed to encode map e341 with expected crc 2023-09-23T21:05:29.303 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:05:29 smithi026 ceph-mon[175151]: failed to encode map e341 with expected crc 2023-09-23T21:05:29.303 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:05:29 smithi026 ceph-mon[175151]: failed to encode map e341 with expected crc 2023-09-23T21:05:29.303 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:05:29 smithi026 ceph-mon[175151]: failed to encode map e341 with expected crc 2023-09-23T21:05:29.303 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:05:29 smithi026 ceph-mon[175151]: failed to encode map e341 with expected crc 2023-09-23T21:05:29.303 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:05:29 smithi026 ceph-mon[175151]: failed to encode map e341 with expected crc 2023-09-23T21:05:29.303 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:05:29 smithi026 ceph-mon[179688]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T21:05:29.303 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:05:29 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/2772009942' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T21:05:29.303 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:05:29 smithi026 ceph-mon[179688]: osdmap e341: 8 total, 7 up, 8 in 2023-09-23T21:05:29.303 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:05:29 smithi026 ceph-mon[179688]: failed to encode map e341 with expected crc 2023-09-23T21:05:29.304 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:05:29 smithi026 ceph-mon[179688]: failed to encode map e341 with expected crc 2023-09-23T21:05:29.304 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:05:29 smithi026 ceph-mon[179688]: failed to encode map e341 with expected crc 2023-09-23T21:05:29.304 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:05:29 smithi026 ceph-mon[179688]: failed to encode map e341 with expected crc 2023-09-23T21:05:29.304 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:05:29 smithi026 ceph-mon[179688]: failed to encode map e341 with expected crc 2023-09-23T21:05:29.304 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:05:29 smithi026 ceph-mon[179688]: failed to encode map e341 with expected crc 2023-09-23T21:05:29.311 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_blocked_ops 2023-09-23T21:05:29.319 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:05:29.360 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:05:29.419 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:05:29.427 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:05:29.428 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:05:29.529 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:05:29.583 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:05:29.594 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 6, 7, 4, 5] out_osds: [] dead_osds: [1] live_osds: [0, 2, 3, 4, 5, 7, 6] 2023-09-23T21:05:29.594 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T21:05:29.594 INFO:tasks.thrashosds.thrasher:primary_affinity 2023-09-23T21:05:29.594 INFO:tasks.thrashosds.thrasher:Setting osd 3 primary_affinity to 1.000000 2023-09-23T21:05:29.595 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph osd primary-affinity 3 1 2023-09-23T21:05:29.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:05:29 smithi070 ceph-mon[168282]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T21:05:29.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:05:29 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/2772009942' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T21:05:29.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:05:29 smithi070 ceph-mon[168282]: osdmap e341: 8 total, 7 up, 8 in 2023-09-23T21:05:29.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:05:29 smithi070 ceph-mon[168282]: failed to encode map e341 with expected crc 2023-09-23T21:05:29.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:05:29 smithi070 ceph-mon[168282]: failed to encode map e341 with expected crc 2023-09-23T21:05:29.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:05:29 smithi070 ceph-mon[168282]: failed to encode map e341 with expected crc 2023-09-23T21:05:29.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:05:29 smithi070 ceph-mon[168282]: failed to encode map e341 with expected crc 2023-09-23T21:05:29.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:05:29 smithi070 ceph-mon[168282]: failed to encode map e341 with expected crc 2023-09-23T21:05:29.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:05:29 smithi070 ceph-mon[168282]: failed to encode map e341 with expected crc 2023-09-23T21:05:29.654 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T21:05:29.657 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:05:29.657 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:05:29.758 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:05:29.799 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:05:29.848 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 21:05:29 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[164029]: ::ffff:172.21.15.70 - - [23/Sep/2023:21:05:29] "GET /metrics HTTP/1.1" 200 36329 "" "Prometheus/2.43.0" 2023-09-23T21:05:29.882 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:05:29.890 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:05:29.891 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:05:29.992 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:05:30.004 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T21:05:30.060 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:05:30.090 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0: {} 2023-09-23T21:05:30.134 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2: {} 2023-09-23T21:05:30.167 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3: {} 2023-09-23T21:05:30.197 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4: {} 2023-09-23T21:05:30.214 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:05:30.222 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:05:30.223 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:05:30.229 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T21:05:30.267 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6: {} 2023-09-23T21:05:30.307 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:05:30 smithi070 ceph-mon[168282]: pgmap v871: 119 pgs: 13 active+undersized+degraded, 33 active+undersized, 73 active+clean; 311 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 3.6 KiB/s rd, 1 op/s; 2190/14970 objects degraded (14.629%) 2023-09-23T21:05:30.307 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T21:05:30.307 INFO:teuthology.orchestra.run.smithi026.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T21:05:30.308 INFO:teuthology.orchestra.run.smithi026.stderr:Error ENXIO: problem getting command descriptions from osd.1 2023-09-23T21:05:30.308 INFO:teuthology.orchestra.run.smithi026.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T21:05:30.308 INFO:teuthology.orchestra.run.smithi026.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T21:05:30.308 INFO:teuthology.orchestra.run.smithi026.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T21:05:30.308 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T21:05:30.308 INFO:teuthology.orchestra.run.smithi026.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T21:05:30.308 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T21:05:30.324 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:05:30.386 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:05:30.442 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:05:30.446 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:05:30.447 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:05:30.455 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:05:30.547 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:05:30.602 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:05:30 smithi026 ceph-mon[179688]: pgmap v871: 119 pgs: 13 active+undersized+degraded, 33 active+undersized, 73 active+clean; 311 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 3.6 KiB/s rd, 1 op/s; 2190/14970 objects degraded (14.629%) 2023-09-23T21:05:30.603 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:05:30 smithi026 ceph-mon[179688]: Health check update: Degraded data redundancy: 2190/14970 objects degraded (14.629%), 13 pgs degraded (PG_DEGRADED) 2023-09-23T21:05:30.603 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:05:30 smithi026 ceph-mon[179688]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 3 pgs inactive, 3 pgs peering) 2023-09-23T21:05:30.603 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:05:30 smithi026 ceph-mon[179688]: 1.8 scrub starts 2023-09-23T21:05:30.603 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:05:30 smithi026 ceph-mon[179688]: 1.8 scrub ok 2023-09-23T21:05:30.603 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:05:30 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/3788787120' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:05:30.603 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:05:30 smithi026 ceph-mon[175151]: pgmap v871: 119 pgs: 13 active+undersized+degraded, 33 active+undersized, 73 active+clean; 311 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 3.6 KiB/s rd, 1 op/s; 2190/14970 objects degraded (14.629%) 2023-09-23T21:05:30.603 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:05:30 smithi026 ceph-mon[175151]: Health check update: Degraded data redundancy: 2190/14970 objects degraded (14.629%), 13 pgs degraded (PG_DEGRADED) 2023-09-23T21:05:30.603 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:05:30 smithi026 ceph-mon[175151]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 3 pgs inactive, 3 pgs peering) 2023-09-23T21:05:30.604 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:05:30 smithi026 ceph-mon[175151]: 1.8 scrub starts 2023-09-23T21:05:30.604 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:05:30 smithi026 ceph-mon[175151]: 1.8 scrub ok 2023-09-23T21:05:30.604 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:05:30 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/3788787120' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:05:30.609 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:05:30.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:05:30 smithi070 ceph-mon[168282]: Health check update: Degraded data redundancy: 2190/14970 objects degraded (14.629%), 13 pgs degraded (PG_DEGRADED) 2023-09-23T21:05:30.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:05:30 smithi070 ceph-mon[168282]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 3 pgs inactive, 3 pgs peering) 2023-09-23T21:05:30.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:05:30 smithi070 ceph-mon[168282]: 1.8 scrub starts 2023-09-23T21:05:30.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:05:30 smithi070 ceph-mon[168282]: 1.8 scrub ok 2023-09-23T21:05:30.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:05:30 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/3788787120' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:05:30.707 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T21:05:30.708 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T21:05:30.740 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T21:05:30.743 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:05:30.750 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:05:30.750 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:05:30.851 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:05:30.903 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:05:30.978 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:05:30.988 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:05:30.989 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:05:31.001 INFO:tasks.radosbench.radosbench.0.smithi026.stdout:Removed 110754 objects 2023-09-23T21:05:31.002 INFO:tasks.radosbench.radosbench.0.smithi026.stdout:Clean up completed and total clean up time :149.211 2023-09-23T21:05:31.089 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:05:31.117 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:05:31.200 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:05:31.202 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:05:31.203 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:05:31.304 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:05:31.331 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:05:31.364 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:05:31.367 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:05:31.367 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:05:31.411 INFO:teuthology.orchestra.run.smithi070.stdout:{ 2023-09-23T21:05:31.411 INFO:teuthology.orchestra.run.smithi070.stdout: "ops": [], 2023-09-23T21:05:31.411 INFO:teuthology.orchestra.run.smithi070.stdout: "complaint_time": 30, 2023-09-23T21:05:31.411 INFO:teuthology.orchestra.run.smithi070.stdout: "num_blocked_ops": 0 2023-09-23T21:05:31.411 INFO:teuthology.orchestra.run.smithi070.stdout:} 2023-09-23T21:05:31.468 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:05:31.499 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:05:31.600 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:05:31.754 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:05:31.757 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:05:31.758 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:05:31.843 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_historic_ops 2023-09-23T21:05:31.859 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:05:31.912 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:05:31.945 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:05:31.947 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:05:31.948 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:05:32.048 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:05:32.075 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:05:32.108 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:05:32.110 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:05:32.111 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:05:32.212 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:05:32.243 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:05:32.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:05:32 smithi070 ceph-mon[168282]: 1.1 deep-scrub starts 2023-09-23T21:05:32.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:05:32 smithi070 ceph-mon[168282]: 1.1 deep-scrub ok 2023-09-23T21:05:32.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:05:32 smithi070 ceph-mon[168282]: pgmap v872: 119 pgs: 13 active+undersized+degraded, 33 active+undersized, 73 active+clean; 189 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 3.2 KiB/s rd, 1 op/s; 861/9114 objects degraded (9.447%) 2023-09-23T21:05:32.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:05:32 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:05:32.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:05:32 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:05:32.437 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T21:05:32.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:05:32 smithi026 ceph-mon[175151]: 1.1 deep-scrub starts 2023-09-23T21:05:32.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:05:32 smithi026 ceph-mon[175151]: 1.1 deep-scrub ok 2023-09-23T21:05:32.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:05:32 smithi026 ceph-mon[175151]: pgmap v872: 119 pgs: 13 active+undersized+degraded, 33 active+undersized, 73 active+clean; 189 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 3.2 KiB/s rd, 1 op/s; 861/9114 objects degraded (9.447%) 2023-09-23T21:05:32.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:05:32 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:05:32.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:05:32 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:05:32.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:05:32 smithi026 ceph-mon[179688]: 1.1 deep-scrub starts 2023-09-23T21:05:32.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:05:32 smithi026 ceph-mon[179688]: 1.1 deep-scrub ok 2023-09-23T21:05:32.633 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:05:32 smithi026 ceph-mon[179688]: pgmap v872: 119 pgs: 13 active+undersized+degraded, 33 active+undersized, 73 active+clean; 189 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 3.2 KiB/s rd, 1 op/s; 861/9114 objects degraded (9.447%) 2023-09-23T21:05:32.633 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:05:32 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:05:32.633 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:05:32 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:05:32.639 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T21:05:32.643 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:05:32.643 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:05:32.709 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T21:05:32.744 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:05:32.780 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:05:32.825 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T21:05:32.827 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:05:32.828 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:05:32.929 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:05:32.956 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:05:33.150 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:05:33.153 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:05:33.154 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:05:33.222 INFO:teuthology.orchestra.run.smithi026.stderr:set osd.3 primary-affinity to 1 (8655362) 2023-09-23T21:05:33.254 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:05:33.281 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:05:33.335 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:05:33.337 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:05:33.337 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:05:33.438 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:05:33.465 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:05:33 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/2618606194' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 3, "weight": 1.0}]: dispatch 2023-09-23T21:05:33.468 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:05:33.550 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:05:33 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/2618606194' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 3, "weight": 1.0}]: dispatch 2023-09-23T21:05:33.551 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:05:33 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/2618606194' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 3, "weight": 1.0}]: dispatch 2023-09-23T21:05:33.602 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:05:33.605 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:05:33.606 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:05:33.666 INFO:teuthology.orchestra.run.smithi070.stdout:{ 2023-09-23T21:05:33.666 INFO:teuthology.orchestra.run.smithi070.stdout: "size": 20, 2023-09-23T21:05:33.666 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 600, 2023-09-23T21:05:33.666 INFO:teuthology.orchestra.run.smithi070.stdout: "ops": [ 2023-09-23T21:05:33.666 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:05:33.666 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.34797.0:177909 6.13 6:c9b40212:::benchmark_data_smithi026_223868_object76783:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e164)", 2023-09-23T21:05:33.666 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T20:55:34.933784+0000", 2023-09-23T21:05:33.666 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 598.72951518800005, 2023-09-23T21:05:33.667 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.1089723249999999, 2023-09-23T21:05:33.667 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:05:33.667 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:05:33.667 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:05:33.667 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.34797", 2023-09-23T21:05:33.667 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T21:05:33.667 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 177909 2023-09-23T21:05:33.667 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:05:33.667 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:05:33.667 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:05:33.667 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:05:33.667 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:34.933784+0000", 2023-09-23T21:05:33.668 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:05:33.668 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:05:33.668 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:05:33.668 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:05:33.668 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:34.933784+0000", 2023-09-23T21:05:33.668 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:05:33.668 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:05:33.668 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:05:33.668 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:05:33.668 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:34.933782+0000", 2023-09-23T21:05:33.668 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T21:05:33.668 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:05:33.668 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:05:33.669 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:05:33.669 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:34.933791+0000", 2023-09-23T21:05:33.669 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.3259999999999996e-06 2023-09-23T21:05:33.669 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:05:33.669 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:05:33.669 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:05:33.669 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:34.933804+0000", 2023-09-23T21:05:33.669 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.3155e-05 2023-09-23T21:05:33.669 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:05:33.669 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:05:33.669 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:05:33.669 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:34.933811+0000", 2023-09-23T21:05:33.669 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.2919999999999998e-06 2023-09-23T21:05:33.670 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:05:33.670 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:05:33.670 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:05:33.670 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:34.940148+0000", 2023-09-23T21:05:33.670 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0063370350000000004 2023-09-23T21:05:33.670 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:05:33.670 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:05:33.670 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:05:33.670 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:34.940159+0000", 2023-09-23T21:05:33.670 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.0611999999999999e-05 2023-09-23T21:05:33.670 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:05:33.670 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:05:33.671 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:05:33.671 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:35.938118+0000", 2023-09-23T21:05:33.671 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.99795963499999996 2023-09-23T21:05:33.671 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:05:33.671 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:05:33.671 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:05:33.671 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:35.938130+0000", 2023-09-23T21:05:33.671 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.13e-05 2023-09-23T21:05:33.671 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:05:33.671 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:05:33.671 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:05:33.671 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:37.032096+0000", 2023-09-23T21:05:33.672 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.093965928 2023-09-23T21:05:33.672 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:05:33.672 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:05:33.672 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:05:33.672 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:37.032231+0000", 2023-09-23T21:05:33.672 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00013501999999999999 2023-09-23T21:05:33.672 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:05:33.672 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:05:33.672 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 2,6", 2023-09-23T21:05:33.672 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:37.032750+0000", 2023-09-23T21:05:33.672 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00051932900000000001 2023-09-23T21:05:33.672 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:05:33.672 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:05:33.673 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:05:33.673 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:37.040969+0000", 2023-09-23T21:05:33.673 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0082194039999999996 2023-09-23T21:05:33.673 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:05:33.673 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:05:33.673 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:05:33.673 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:37.041526+0000", 2023-09-23T21:05:33.673 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000556347 2023-09-23T21:05:33.673 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:05:33.673 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:05:33.673 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:05:33.673 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:37.042559+0000", 2023-09-23T21:05:33.673 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.001033405 2023-09-23T21:05:33.674 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:05:33.674 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:05:33.674 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:05:33.674 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:37.042611+0000", 2023-09-23T21:05:33.674 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.2101999999999997e-05 2023-09-23T21:05:33.674 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:05:33.674 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:05:33.674 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:05:33.674 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:37.042756+0000", 2023-09-23T21:05:33.674 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00014501499999999999 2023-09-23T21:05:33.674 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:05:33.674 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:05:33.675 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:05:33.675 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:05:33.675 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:05:33.675 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.34797.0:177918 6.13 6:cb5d9774:::benchmark_data_smithi026_223868_object76792:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e164)", 2023-09-23T21:05:33.675 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T20:55:34.943354+0000", 2023-09-23T21:05:33.675 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 598.71994451499995, 2023-09-23T21:05:33.675 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.1104338490000001, 2023-09-23T21:05:33.675 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:05:33.675 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:05:33.675 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:05:33.675 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.34797", 2023-09-23T21:05:33.675 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T21:05:33.675 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 177918 2023-09-23T21:05:33.676 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:05:33.676 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:05:33.676 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:05:33.676 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:05:33.676 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:34.943354+0000", 2023-09-23T21:05:33.676 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:05:33.676 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:05:33.676 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:05:33.676 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:05:33.676 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:34.943354+0000", 2023-09-23T21:05:33.676 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:05:33.676 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:05:33.676 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:05:33.677 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:05:33.677 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:34.943353+0000", 2023-09-23T21:05:33.677 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999981 2023-09-23T21:05:33.677 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:05:33.677 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:05:33.677 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:05:33.677 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:34.943362+0000", 2023-09-23T21:05:33.677 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.5310000000000007e-06 2023-09-23T21:05:33.677 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:05:33.677 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:05:33.677 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:05:33.677 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:34.943381+0000", 2023-09-23T21:05:33.678 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.8284999999999999e-05 2023-09-23T21:05:33.678 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:05:33.678 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:05:33.678 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:05:33.678 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:34.943390+0000", 2023-09-23T21:05:33.678 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.2010000000000005e-06 2023-09-23T21:05:33.678 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:05:33.678 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:05:33.678 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:05:33.678 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:34.943462+0000", 2023-09-23T21:05:33.678 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.2002000000000006e-05 2023-09-23T21:05:33.678 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:05:33.678 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:05:33.679 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:05:33.679 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:34.943472+0000", 2023-09-23T21:05:33.679 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.0356e-05 2023-09-23T21:05:33.679 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:05:33.679 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:05:33.679 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:05:33.679 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:35.938155+0000", 2023-09-23T21:05:33.679 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.99468292700000005 2023-09-23T21:05:33.679 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:05:33.679 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:05:33.679 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:05:33.679 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:35.938162+0000", 2023-09-23T21:05:33.680 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.0589999999999997e-06 2023-09-23T21:05:33.680 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:05:33.680 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:05:33.680 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:05:33.680 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:37.034935+0000", 2023-09-23T21:05:33.680 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.0967725079999999 2023-09-23T21:05:33.680 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:05:33.680 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:05:33.680 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:05:33.680 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:37.035045+0000", 2023-09-23T21:05:33.680 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000110213 2023-09-23T21:05:33.680 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:05:33.680 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:05:33.681 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 2,6", 2023-09-23T21:05:33.681 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:37.035460+0000", 2023-09-23T21:05:33.681 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00041482800000000001 2023-09-23T21:05:33.681 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:05:33.681 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:05:33.681 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:05:33.681 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:37.041023+0000", 2023-09-23T21:05:33.681 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0055634040000000001 2023-09-23T21:05:33.681 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:05:33.681 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:05:33.681 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:05:33.681 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:37.049549+0000", 2023-09-23T21:05:33.682 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0085260170000000003 2023-09-23T21:05:33.682 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:05:33.682 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:05:33.682 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:05:33.682 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:37.053587+0000", 2023-09-23T21:05:33.682 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0040375999999999997 2023-09-23T21:05:33.682 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:05:33.682 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:05:33.682 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:05:33.682 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:37.053643+0000", 2023-09-23T21:05:33.682 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.6524999999999998e-05 2023-09-23T21:05:33.683 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:05:33.683 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:05:33.683 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:05:33.683 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:37.053788+0000", 2023-09-23T21:05:33.683 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00014508699999999999 2023-09-23T21:05:33.683 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:05:33.683 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:05:33.683 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:05:33.683 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:05:33.683 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:05:33.683 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.34797.0:177940 6.13 6:c8fa8a85:::benchmark_data_smithi026_223868_object76814:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e164)", 2023-09-23T21:05:33.683 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T20:55:34.983208+0000", 2023-09-23T21:05:33.683 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 598.68009082599997, 2023-09-23T21:05:33.684 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.0707293249999998, 2023-09-23T21:05:33.684 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:05:33.684 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:05:33.684 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:05:33.684 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.34797", 2023-09-23T21:05:33.684 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T21:05:33.684 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 177940 2023-09-23T21:05:33.684 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:05:33.684 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:05:33.684 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:05:33.684 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:05:33.684 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:34.983208+0000", 2023-09-23T21:05:33.685 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:05:33.685 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:05:33.685 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:05:33.685 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:05:33.685 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:34.983208+0000", 2023-09-23T21:05:33.685 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:05:33.685 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:05:33.685 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:05:33.685 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:05:33.685 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:34.983207+0000", 2023-09-23T21:05:33.685 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.999999 2023-09-23T21:05:33.685 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:05:33.685 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:05:33.686 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:05:33.686 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:34.983212+0000", 2023-09-23T21:05:33.686 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.3419999999999998e-06 2023-09-23T21:05:33.686 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:05:33.686 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:05:33.686 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:05:33.686 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:34.983220+0000", 2023-09-23T21:05:33.686 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.4610000000000002e-06 2023-09-23T21:05:33.686 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:05:33.686 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:05:33.686 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:05:33.686 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:34.983224+0000", 2023-09-23T21:05:33.687 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.4610000000000001e-06 2023-09-23T21:05:33.687 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:05:33.687 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:05:33.687 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:05:33.687 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:34.983289+0000", 2023-09-23T21:05:33.687 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.5041000000000003e-05 2023-09-23T21:05:33.687 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:05:33.687 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:05:33.687 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:05:33.687 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:34.983297+0000", 2023-09-23T21:05:33.687 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.046e-06 2023-09-23T21:05:33.687 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:05:33.688 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:05:33.688 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:05:33.688 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:35.938873+0000", 2023-09-23T21:05:33.688 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.95557668900000003 2023-09-23T21:05:33.688 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:05:33.688 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:05:33.688 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:05:33.688 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:35.938882+0000", 2023-09-23T21:05:33.688 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.5059999999999995e-06 2023-09-23T21:05:33.688 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:05:33.688 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:05:33.688 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:05:33.688 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:37.037926+0000", 2023-09-23T21:05:33.688 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.0990439059999999 2023-09-23T21:05:33.689 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:05:33.689 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:05:33.689 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:05:33.689 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:37.038068+0000", 2023-09-23T21:05:33.689 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00014263599999999999 2023-09-23T21:05:33.689 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:05:33.689 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:05:33.689 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 2,6", 2023-09-23T21:05:33.689 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:37.038541+0000", 2023-09-23T21:05:33.689 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00047294699999999998 2023-09-23T21:05:33.689 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:05:33.689 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:05:33.689 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:05:33.690 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:37.041561+0000", 2023-09-23T21:05:33.690 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0030193889999999999 2023-09-23T21:05:33.690 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:05:33.690 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:05:33.690 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:05:33.690 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:37.049637+0000", 2023-09-23T21:05:33.690 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0080764089999999997 2023-09-23T21:05:33.690 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:05:33.690 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:05:33.690 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:05:33.690 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:37.053824+0000", 2023-09-23T21:05:33.690 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.004186871 2023-09-23T21:05:33.691 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:05:33.691 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:05:33.691 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:05:33.691 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:37.053870+0000", 2023-09-23T21:05:33.691 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.5782000000000003e-05 2023-09-23T21:05:33.691 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:05:33.691 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:05:33.691 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:05:33.691 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:37.053937+0000", 2023-09-23T21:05:33.691 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.7842999999999994e-05 2023-09-23T21:05:33.691 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:05:33.691 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:05:33.691 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:05:33.692 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:05:33.692 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:05:33.692 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.34797.0:177941 6.15 6:ac4602e1:::benchmark_data_smithi026_223868_object76815:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e164)", 2023-09-23T21:05:33.692 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T20:55:34.983250+0000", 2023-09-23T21:05:33.692 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 598.68004920600004, 2023-09-23T21:05:33.692 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.0517934200000001, 2023-09-23T21:05:33.692 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:05:33.692 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:05:33.692 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:05:33.692 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.34797", 2023-09-23T21:05:33.692 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T21:05:33.692 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 177941 2023-09-23T21:05:33.693 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:05:33.693 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:05:33.693 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:05:33.693 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:05:33.693 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:34.983250+0000", 2023-09-23T21:05:33.693 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:05:33.693 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:05:33.693 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:05:33.693 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:05:33.693 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:34.983250+0000", 2023-09-23T21:05:33.693 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:05:33.693 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:05:33.693 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:05:33.694 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:05:33.694 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:34.983249+0000", 2023-09-23T21:05:33.694 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999995 2023-09-23T21:05:33.694 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:05:33.694 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:05:33.694 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:05:33.694 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:34.983252+0000", 2023-09-23T21:05:33.694 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.0149999999999999e-06 2023-09-23T21:05:33.694 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:05:33.694 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:05:33.694 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:05:33.694 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:34.983259+0000", 2023-09-23T21:05:33.695 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.5540000000000003e-06 2023-09-23T21:05:33.695 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:05:33.695 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:05:33.695 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:05:33.695 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:34.983263+0000", 2023-09-23T21:05:33.695 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.1010000000000002e-06 2023-09-23T21:05:33.695 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:05:33.695 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:05:33.695 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:05:33.695 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:34.983303+0000", 2023-09-23T21:05:33.695 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.9650999999999997e-05 2023-09-23T21:05:33.695 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:05:33.695 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:05:33.696 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:05:33.696 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:34.983309+0000", 2023-09-23T21:05:33.696 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.9320000000000001e-06 2023-09-23T21:05:33.696 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:05:33.696 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:05:33.696 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:05:33.696 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:35.939378+0000", 2023-09-23T21:05:33.696 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.95606941999999995 2023-09-23T21:05:33.696 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:05:33.696 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:05:33.696 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:05:33.696 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:35.939387+0000", 2023-09-23T21:05:33.697 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.9740000000000008e-06 2023-09-23T21:05:33.697 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:05:33.697 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:05:33.697 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:05:33.697 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:37.026628+0000", 2023-09-23T21:05:33.697 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.0872411820000001 2023-09-23T21:05:33.697 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:05:33.697 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:05:33.697 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:05:33.697 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:37.026766+0000", 2023-09-23T21:05:33.697 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00013818599999999999 2023-09-23T21:05:33.697 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:05:33.697 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:05:33.698 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 3,6", 2023-09-23T21:05:33.698 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:37.027155+0000", 2023-09-23T21:05:33.698 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00038901000000000002 2023-09-23T21:05:33.698 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:05:33.698 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:05:33.698 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:05:33.698 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:37.034577+0000", 2023-09-23T21:05:33.698 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0074220909999999996 2023-09-23T21:05:33.698 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:05:33.698 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:05:33.698 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:05:33.698 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:37.034818+0000", 2023-09-23T21:05:33.699 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00024096600000000001 2023-09-23T21:05:33.699 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:05:33.699 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:05:33.699 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:05:33.699 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:37.034889+0000", 2023-09-23T21:05:33.699 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.0186999999999997e-05 2023-09-23T21:05:33.699 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:05:33.699 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:05:33.699 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:05:33.699 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:37.034942+0000", 2023-09-23T21:05:33.699 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.3251999999999998e-05 2023-09-23T21:05:33.699 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:05:33.699 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:05:33.700 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:05:33.700 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:37.035043+0000", 2023-09-23T21:05:33.700 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000101322 2023-09-23T21:05:33.700 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:05:33.700 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:05:33.700 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:05:33.700 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:05:33.700 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:05:33.700 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.34797.0:177943 6.15 6:ae2b0147:::benchmark_data_smithi026_223868_object76817:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e164)", 2023-09-23T21:05:33.700 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T20:55:34.984662+0000", 2023-09-23T21:05:33.700 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 598.67863742099996, 2023-09-23T21:05:33.700 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.0517963250000002, 2023-09-23T21:05:33.700 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:05:33.701 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:05:33.701 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:05:33.701 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.34797", 2023-09-23T21:05:33.701 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T21:05:33.701 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 177943 2023-09-23T21:05:33.701 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:05:33.701 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:05:33.701 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:05:33.701 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:05:33.701 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:34.984662+0000", 2023-09-23T21:05:33.701 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:05:33.701 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:05:33.702 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:05:33.702 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:05:33.702 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:34.984662+0000", 2023-09-23T21:05:33.702 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:05:33.702 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:05:33.702 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:05:33.702 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:05:33.702 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:34.984661+0000", 2023-09-23T21:05:33.702 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999995 2023-09-23T21:05:33.702 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:05:33.702 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:05:33.702 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:05:33.703 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:34.984665+0000", 2023-09-23T21:05:33.703 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.0400000000000003e-06 2023-09-23T21:05:33.703 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:05:33.703 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:05:33.703 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:05:33.703 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:34.984673+0000", 2023-09-23T21:05:33.703 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.6590000000000007e-06 2023-09-23T21:05:33.703 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:05:33.703 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:05:33.703 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:05:33.703 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:34.984677+0000", 2023-09-23T21:05:33.703 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.9450000000000003e-06 2023-09-23T21:05:33.703 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:05:33.704 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:05:33.704 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:05:33.704 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:34.984720+0000", 2023-09-23T21:05:33.704 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.3679e-05 2023-09-23T21:05:33.704 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:05:33.704 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:05:33.704 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:05:33.704 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:34.984726+0000", 2023-09-23T21:05:33.704 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.0650000000000004e-06 2023-09-23T21:05:33.704 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:05:33.704 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:05:33.704 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:05:33.704 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:35.939411+0000", 2023-09-23T21:05:33.705 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.95468514100000001 2023-09-23T21:05:33.705 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:05:33.705 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:05:33.705 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:05:33.705 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:35.939419+0000", 2023-09-23T21:05:33.705 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.1169999999999998e-06 2023-09-23T21:05:33.705 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:05:33.705 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:05:33.705 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:05:33.705 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:37.029284+0000", 2023-09-23T21:05:33.705 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.0898655939999999 2023-09-23T21:05:33.705 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:05:33.705 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:05:33.705 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:05:33.706 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:37.029432+0000", 2023-09-23T21:05:33.706 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000147978 2023-09-23T21:05:33.706 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:05:33.706 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:05:33.706 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 3,6", 2023-09-23T21:05:33.706 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:37.029829+0000", 2023-09-23T21:05:33.706 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000397272 2023-09-23T21:05:33.706 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:05:33.706 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:05:33.706 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:05:33.706 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:37.034635+0000", 2023-09-23T21:05:33.706 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0048057719999999998 2023-09-23T21:05:33.707 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:05:33.707 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:05:33.707 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:05:33.707 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:37.034857+0000", 2023-09-23T21:05:33.707 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00022135300000000001 2023-09-23T21:05:33.707 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:05:33.707 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:05:33.707 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:05:33.707 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:37.036289+0000", 2023-09-23T21:05:33.707 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.001431978 2023-09-23T21:05:33.707 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:05:33.707 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:05:33.707 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:05:33.708 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:37.036342+0000", 2023-09-23T21:05:33.708 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.3029999999999999e-05 2023-09-23T21:05:33.708 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:05:33.708 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:05:33.708 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:05:33.708 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:37.036458+0000", 2023-09-23T21:05:33.708 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00011630500000000001 2023-09-23T21:05:33.708 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:05:33.708 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:05:33.708 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:05:33.708 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:05:33.708 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:05:33.708 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.34797.0:177955 6.15 6:aed95e65:::benchmark_data_smithi026_223868_object76829:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e164)", 2023-09-23T21:05:33.709 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T20:55:35.010017+0000", 2023-09-23T21:05:33.709 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 598.65328193300002, 2023-09-23T21:05:33.709 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.026625954, 2023-09-23T21:05:33.709 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:05:33.709 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:05:33.709 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:05:33.709 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.34797", 2023-09-23T21:05:33.709 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T21:05:33.709 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 177955 2023-09-23T21:05:33.709 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:05:33.709 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:05:33.709 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:05:33.709 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:05:33.710 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:35.010017+0000", 2023-09-23T21:05:33.710 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:05:33.710 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:05:33.710 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:05:33.710 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:05:33.710 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:35.010017+0000", 2023-09-23T21:05:33.710 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:05:33.710 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:05:33.710 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:05:33.710 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:05:33.710 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:35.010016+0000", 2023-09-23T21:05:33.710 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999995 2023-09-23T21:05:33.710 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:05:33.711 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:05:33.711 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:05:33.711 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:35.010020+0000", 2023-09-23T21:05:33.711 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.416e-06 2023-09-23T21:05:33.711 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:05:33.711 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:05:33.711 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:05:33.711 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:35.010028+0000", 2023-09-23T21:05:33.711 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.4640000000000006e-06 2023-09-23T21:05:33.711 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:05:33.711 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:05:33.712 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:05:33.712 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:35.010032+0000", 2023-09-23T21:05:33.712 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.985e-06 2023-09-23T21:05:33.712 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:05:33.712 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:05:33.712 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:05:33.712 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:35.010088+0000", 2023-09-23T21:05:33.712 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.5614e-05 2023-09-23T21:05:33.712 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:05:33.712 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:05:33.712 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:05:33.712 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:35.010098+0000", 2023-09-23T21:05:33.712 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.0553e-05 2023-09-23T21:05:33.712 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:05:33.713 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:05:33.713 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:05:33.713 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:35.939934+0000", 2023-09-23T21:05:33.713 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.92983511600000002 2023-09-23T21:05:33.713 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:05:33.713 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:05:33.713 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:05:33.713 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:35.939942+0000", 2023-09-23T21:05:33.713 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.3049999999999993e-06 2023-09-23T21:05:33.713 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:05:33.713 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:05:33.713 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:05:33.714 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:37.032144+0000", 2023-09-23T21:05:33.714 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.09220184 2023-09-23T21:05:33.714 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:05:33.714 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:05:33.714 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:05:33.714 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:37.032273+0000", 2023-09-23T21:05:33.714 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00012959399999999999 2023-09-23T21:05:33.714 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:05:33.714 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:05:33.714 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 3,6", 2023-09-23T21:05:33.714 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:37.032579+0000", 2023-09-23T21:05:33.714 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000306075 2023-09-23T21:05:33.715 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:05:33.715 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:05:33.715 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:05:33.715 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:37.034826+0000", 2023-09-23T21:05:33.715 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0022461669999999999 2023-09-23T21:05:33.715 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:05:33.715 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:05:33.715 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:05:33.715 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:37.036494+0000", 2023-09-23T21:05:33.715 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0016682559999999999 2023-09-23T21:05:33.715 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:05:33.715 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:05:33.715 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:05:33.716 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:37.036520+0000", 2023-09-23T21:05:33.716 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.5797999999999999e-05 2023-09-23T21:05:33.716 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:05:33.716 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:05:33.716 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:05:33.716 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:37.036557+0000", 2023-09-23T21:05:33.716 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.6993000000000003e-05 2023-09-23T21:05:33.716 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:05:33.716 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:05:33.716 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:05:33.716 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:37.036643+0000", 2023-09-23T21:05:33.716 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.6441999999999997e-05 2023-09-23T21:05:33.716 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:05:33.716 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:05:33.717 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:05:33.717 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:05:33.717 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:05:33.717 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.34797.0:183991 6.3 6:c69266c2:::benchmark_data_smithi026_223868_object82865:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e169)", 2023-09-23T21:05:33.717 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T20:55:44.983566+0000", 2023-09-23T21:05:33.717 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 588.679732715, 2023-09-23T21:05:33.717 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.009211377, 2023-09-23T21:05:33.717 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:05:33.717 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:05:33.717 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:05:33.717 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.34797", 2023-09-23T21:05:33.717 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T21:05:33.717 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 183991 2023-09-23T21:05:33.718 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:05:33.718 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:05:33.718 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:05:33.718 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:05:33.718 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:44.983566+0000", 2023-09-23T21:05:33.718 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:05:33.718 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:05:33.718 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:05:33.718 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:05:33.718 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:44.983566+0000", 2023-09-23T21:05:33.718 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:05:33.718 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:05:33.718 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:05:33.719 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:05:33.719 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:44.983566+0000", 2023-09-23T21:05:33.719 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999995 2023-09-23T21:05:33.719 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:05:33.719 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:05:33.719 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:05:33.719 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:44.983570+0000", 2023-09-23T21:05:33.719 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.8160000000000004e-06 2023-09-23T21:05:33.719 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:05:33.719 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:05:33.719 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:05:33.719 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:44.983580+0000", 2023-09-23T21:05:33.720 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.0499999999999999e-05 2023-09-23T21:05:33.720 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:05:33.720 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:05:33.720 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:05:33.720 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:44.983585+0000", 2023-09-23T21:05:33.720 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.933e-06 2023-09-23T21:05:33.720 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:05:33.720 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:05:33.720 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:05:33.720 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:44.983689+0000", 2023-09-23T21:05:33.720 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000104318 2023-09-23T21:05:33.720 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:05:33.720 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:05:33.721 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:05:33.721 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:44.983701+0000", 2023-09-23T21:05:33.721 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.1369000000000001e-05 2023-09-23T21:05:33.721 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:05:33.721 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:05:33.721 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:05:33.721 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:45.952012+0000", 2023-09-23T21:05:33.721 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.968311335 2023-09-23T21:05:33.721 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:05:33.721 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:05:33.721 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:05:33.721 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:45.952141+0000", 2023-09-23T21:05:33.722 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000129121 2023-09-23T21:05:33.722 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:05:33.722 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:05:33.722 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 2,6", 2023-09-23T21:05:33.722 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:45.952615+0000", 2023-09-23T21:05:33.722 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00047399399999999999 2023-09-23T21:05:33.722 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:05:33.722 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:05:33.722 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:05:33.722 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:45.965254+0000", 2023-09-23T21:05:33.722 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.012638666 2023-09-23T21:05:33.722 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:05:33.723 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:05:33.723 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:05:33.723 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:45.992657+0000", 2023-09-23T21:05:33.723 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.027402724999999999 2023-09-23T21:05:33.723 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:05:33.723 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:05:33.723 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:05:33.723 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:45.992700+0000", 2023-09-23T21:05:33.723 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.3248999999999999e-05 2023-09-23T21:05:33.723 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:05:33.723 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:05:33.723 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:05:33.723 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:45.992736+0000", 2023-09-23T21:05:33.724 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.5775000000000002e-05 2023-09-23T21:05:33.724 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:05:33.724 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:05:33.724 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:05:33.724 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:45.992778+0000", 2023-09-23T21:05:33.724 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.2018999999999999e-05 2023-09-23T21:05:33.724 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:05:33.724 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:05:33.724 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:05:33.724 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:05:33.724 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:05:33.724 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.34797.0:184039 6.3 6:c6d77d60:::benchmark_data_smithi026_223868_object82913:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e169)", 2023-09-23T21:05:33.724 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T20:55:45.045261+0000", 2023-09-23T21:05:33.725 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 588.61803827300002, 2023-09-23T21:05:33.725 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.94769778800000004, 2023-09-23T21:05:33.725 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:05:33.725 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:05:33.725 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:05:33.725 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.34797", 2023-09-23T21:05:33.725 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T21:05:33.725 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 184039 2023-09-23T21:05:33.725 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:05:33.725 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:05:33.725 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:05:33.725 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:05:33.726 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:45.045261+0000", 2023-09-23T21:05:33.726 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:05:33.726 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:05:33.726 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:05:33.726 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:05:33.726 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:45.045261+0000", 2023-09-23T21:05:33.726 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:05:33.726 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:05:33.726 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:05:33.726 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:05:33.726 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:45.045260+0000", 2023-09-23T21:05:33.726 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999995 2023-09-23T21:05:33.727 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:05:33.727 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:05:33.727 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:05:33.727 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:45.045265+0000", 2023-09-23T21:05:33.727 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.9169999999999997e-06 2023-09-23T21:05:33.727 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:05:33.727 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:05:33.727 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:05:33.727 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:45.045275+0000", 2023-09-23T21:05:33.727 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.8220000000000002e-06 2023-09-23T21:05:33.727 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:05:33.727 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:05:33.727 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:05:33.728 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:45.045280+0000", 2023-09-23T21:05:33.728 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.694e-06 2023-09-23T21:05:33.728 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:05:33.728 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:05:33.728 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:05:33.728 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:45.046604+0000", 2023-09-23T21:05:33.728 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0013231950000000001 2023-09-23T21:05:33.728 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:05:33.728 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:05:33.728 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:05:33.728 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:45.046612+0000", 2023-09-23T21:05:33.728 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.7099999999999996e-06 2023-09-23T21:05:33.729 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:05:33.729 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:05:33.729 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:05:33.729 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:45.954669+0000", 2023-09-23T21:05:33.729 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.90805630100000001 2023-09-23T21:05:33.729 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:05:33.729 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:05:33.729 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:05:33.729 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:45.954788+0000", 2023-09-23T21:05:33.729 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000118996 2023-09-23T21:05:33.729 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:05:33.729 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:05:33.729 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 2,6", 2023-09-23T21:05:33.730 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:45.955114+0000", 2023-09-23T21:05:33.730 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00032657500000000001 2023-09-23T21:05:33.730 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:05:33.730 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:05:33.730 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:05:33.730 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:45.967963+0000", 2023-09-23T21:05:33.730 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.012848607999999999 2023-09-23T21:05:33.730 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:05:33.730 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:05:33.730 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:05:33.730 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:45.992828+0000", 2023-09-23T21:05:33.730 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.024865343000000002 2023-09-23T21:05:33.731 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:05:33.731 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:05:33.731 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:05:33.731 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:45.992875+0000", 2023-09-23T21:05:33.731 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.6767999999999997e-05 2023-09-23T21:05:33.731 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:05:33.731 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:05:33.731 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:05:33.731 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:45.992921+0000", 2023-09-23T21:05:33.731 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.5787000000000001e-05 2023-09-23T21:05:33.731 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:05:33.731 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:05:33.731 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:05:33.732 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:45.992959+0000", 2023-09-23T21:05:33.732 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.7725000000000002e-05 2023-09-23T21:05:33.732 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:05:33.732 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:05:33.732 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:05:33.732 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:05:33.732 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:05:33.732 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.34797.0:184064 6.3 6:c1be5d02:::benchmark_data_smithi026_223868_object82938:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e169)", 2023-09-23T21:05:33.732 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T20:55:45.083151+0000", 2023-09-23T21:05:33.732 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 588.58014837600001, 2023-09-23T21:05:33.732 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.90993612000000001, 2023-09-23T21:05:33.732 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:05:33.732 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:05:33.733 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:05:33.733 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.34797", 2023-09-23T21:05:33.733 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T21:05:33.733 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 184064 2023-09-23T21:05:33.733 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:05:33.733 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:05:33.733 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:05:33.733 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:05:33.733 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:45.083151+0000", 2023-09-23T21:05:33.733 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:05:33.733 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:05:33.733 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:05:33.734 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:05:33.734 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:45.083151+0000", 2023-09-23T21:05:33.734 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:05:33.734 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:05:33.734 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:05:33.734 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:05:33.734 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:45.083149+0000", 2023-09-23T21:05:33.734 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T21:05:33.734 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:05:33.734 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:05:33.734 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:05:33.734 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:45.083157+0000", 2023-09-23T21:05:33.734 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.8669999999999994e-06 2023-09-23T21:05:33.735 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:05:33.735 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:05:33.735 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:05:33.735 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:45.083171+0000", 2023-09-23T21:05:33.735 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.3759e-05 2023-09-23T21:05:33.735 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:05:33.735 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:05:33.735 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:05:33.735 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:45.083178+0000", 2023-09-23T21:05:33.735 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.1729999999999998e-06 2023-09-23T21:05:33.735 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:05:33.735 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:05:33.736 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:05:33.736 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:45.083258+0000", 2023-09-23T21:05:33.736 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.0216000000000006e-05 2023-09-23T21:05:33.736 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:05:33.736 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:05:33.736 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:05:33.736 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:45.083268+0000", 2023-09-23T21:05:33.736 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.8670000000000006e-06 2023-09-23T21:05:33.736 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:05:33.736 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:05:33.736 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:05:33.736 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:45.957501+0000", 2023-09-23T21:05:33.736 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.87423300400000004 2023-09-23T21:05:33.737 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:05:33.737 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:05:33.737 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:05:33.737 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:45.957638+0000", 2023-09-23T21:05:33.737 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000136828 2023-09-23T21:05:33.737 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:05:33.737 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:05:33.737 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 2,6", 2023-09-23T21:05:33.737 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:45.958030+0000", 2023-09-23T21:05:33.737 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00039222699999999998 2023-09-23T21:05:33.737 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:05:33.737 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:05:33.738 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:05:33.738 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:45.967973+0000", 2023-09-23T21:05:33.738 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0099433109999999998 2023-09-23T21:05:33.738 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:05:33.738 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:05:33.738 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:05:33.738 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:45.993006+0000", 2023-09-23T21:05:33.738 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.025032180000000001 2023-09-23T21:05:33.738 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:05:33.738 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:05:33.738 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:05:33.738 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:45.993029+0000", 2023-09-23T21:05:33.739 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.3254000000000001e-05 2023-09-23T21:05:33.739 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:05:33.739 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:05:33.739 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:05:33.739 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:45.993055+0000", 2023-09-23T21:05:33.739 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.5859000000000001e-05 2023-09-23T21:05:33.739 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:05:33.739 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:05:33.739 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:05:33.739 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:45.993087+0000", 2023-09-23T21:05:33.739 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.2115e-05 2023-09-23T21:05:33.739 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:05:33.740 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:05:33.740 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:05:33.740 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:05:33.740 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:05:33.740 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.34797.0:184090 6.3 6:c64af2e3:::benchmark_data_smithi026_223868_object82964:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e169)", 2023-09-23T21:05:33.740 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T20:55:45.111638+0000", 2023-09-23T21:05:33.740 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 588.55166079499998, 2023-09-23T21:05:33.740 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.881604795, 2023-09-23T21:05:33.740 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:05:33.740 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:05:33.740 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:05:33.740 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.34797", 2023-09-23T21:05:33.740 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T21:05:33.741 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 184090 2023-09-23T21:05:33.741 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:05:33.741 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:05:33.741 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:05:33.741 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:05:33.741 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:45.111638+0000", 2023-09-23T21:05:33.741 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:05:33.741 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:05:33.741 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:05:33.741 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:05:33.741 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:45.111638+0000", 2023-09-23T21:05:33.741 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:05:33.742 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:05:33.742 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:05:33.742 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:05:33.742 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:45.111637+0000", 2023-09-23T21:05:33.742 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.999999 2023-09-23T21:05:33.742 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:05:33.742 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:05:33.742 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:05:33.742 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:45.111642+0000", 2023-09-23T21:05:33.742 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.3210000000000004e-06 2023-09-23T21:05:33.742 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:05:33.742 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:05:33.742 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:05:33.743 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:45.111654+0000", 2023-09-23T21:05:33.743 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.149e-05 2023-09-23T21:05:33.743 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:05:33.743 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:05:33.743 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:05:33.743 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:45.111659+0000", 2023-09-23T21:05:33.743 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.7280000000000003e-06 2023-09-23T21:05:33.743 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:05:33.743 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:05:33.743 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:05:33.743 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:45.111728+0000", 2023-09-23T21:05:33.743 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.9608000000000002e-05 2023-09-23T21:05:33.744 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:05:33.744 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:05:33.744 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:05:33.744 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:45.111741+0000", 2023-09-23T21:05:33.744 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.2393e-05 2023-09-23T21:05:33.744 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:05:33.744 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:05:33.744 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:05:33.744 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:45.960178+0000", 2023-09-23T21:05:33.744 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.84843696700000004 2023-09-23T21:05:33.744 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:05:33.744 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:05:33.745 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:05:33.745 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:45.960306+0000", 2023-09-23T21:05:33.745 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000128641 2023-09-23T21:05:33.745 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:05:33.745 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:05:33.745 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 2,6", 2023-09-23T21:05:33.745 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:45.960651+0000", 2023-09-23T21:05:33.745 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00034484499999999998 2023-09-23T21:05:33.745 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:05:33.745 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:05:33.745 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:05:33.745 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:45.967982+0000", 2023-09-23T21:05:33.745 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0073310069999999996 2023-09-23T21:05:33.746 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:05:33.746 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:05:33.746 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:05:33.746 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:45.993150+0000", 2023-09-23T21:05:33.746 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.025167622000000001 2023-09-23T21:05:33.746 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:05:33.746 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:05:33.746 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:05:33.746 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:45.993181+0000", 2023-09-23T21:05:33.746 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.1446999999999999e-05 2023-09-23T21:05:33.746 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:05:33.746 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:05:33.747 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:05:33.747 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:45.993206+0000", 2023-09-23T21:05:33.747 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.4564000000000001e-05 2023-09-23T21:05:33.747 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:05:33.747 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:05:33.747 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:05:33.747 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:45.993243+0000", 2023-09-23T21:05:33.747 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.7181999999999997e-05 2023-09-23T21:05:33.747 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:05:33.747 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:05:33.747 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:05:33.747 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:05:33.747 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:05:33.748 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.34797.0:184092 6.3 6:c0ffe052:::benchmark_data_smithi026_223868_object82966:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e169)", 2023-09-23T21:05:33.748 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T20:55:45.112343+0000", 2023-09-23T21:05:33.748 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 588.55095578099997, 2023-09-23T21:05:33.748 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.88104986100000005, 2023-09-23T21:05:33.748 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:05:33.748 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:05:33.748 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:05:33.748 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.34797", 2023-09-23T21:05:33.748 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T21:05:33.748 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 184092 2023-09-23T21:05:33.748 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:05:33.748 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:05:33.748 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:05:33.748 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:05:33.749 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:45.112343+0000", 2023-09-23T21:05:33.749 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:05:33.749 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:05:33.749 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:05:33.749 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:05:33.749 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:45.112343+0000", 2023-09-23T21:05:33.749 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:05:33.749 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:05:33.749 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:05:33.749 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:05:33.749 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:45.112342+0000", 2023-09-23T21:05:33.749 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.999999 2023-09-23T21:05:33.749 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:05:33.750 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:05:33.750 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:05:33.750 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:45.112348+0000", 2023-09-23T21:05:33.750 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.3229999999999997e-06 2023-09-23T21:05:33.750 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:05:33.750 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:05:33.750 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:05:33.750 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:45.112364+0000", 2023-09-23T21:05:33.750 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.6351999999999999e-05 2023-09-23T21:05:33.750 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:05:33.750 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:05:33.750 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:05:33.751 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:45.112371+0000", 2023-09-23T21:05:33.751 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.8650000000000003e-06 2023-09-23T21:05:33.751 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:05:33.751 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:05:33.751 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:05:33.751 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:45.112424+0000", 2023-09-23T21:05:33.751 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.3529000000000002e-05 2023-09-23T21:05:33.751 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:05:33.751 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:05:33.751 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:05:33.751 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:45.112433+0000", 2023-09-23T21:05:33.751 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.4100000000000008e-06 2023-09-23T21:05:33.751 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:05:33.752 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:05:33.752 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:05:33.752 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:45.962520+0000", 2023-09-23T21:05:33.752 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.85008725299999999 2023-09-23T21:05:33.752 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:05:33.752 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:05:33.752 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:05:33.752 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:45.962624+0000", 2023-09-23T21:05:33.752 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000104347 2023-09-23T21:05:33.752 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:05:33.752 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:05:33.752 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 2,6", 2023-09-23T21:05:33.752 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:45.962955+0000", 2023-09-23T21:05:33.752 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00033040100000000002 2023-09-23T21:05:33.753 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:05:33.753 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:05:33.753 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:05:33.753 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:45.973201+0000", 2023-09-23T21:05:33.753 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.010246151 2023-09-23T21:05:33.753 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:05:33.753 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:05:33.753 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:05:33.753 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:45.993284+0000", 2023-09-23T21:05:33.753 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.020082724999999999 2023-09-23T21:05:33.753 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:05:33.753 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:05:33.753 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:05:33.753 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:45.993318+0000", 2023-09-23T21:05:33.754 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.3816000000000003e-05 2023-09-23T21:05:33.754 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:05:33.754 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:05:33.754 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:05:33.754 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:45.993360+0000", 2023-09-23T21:05:33.754 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.2348000000000002e-05 2023-09-23T21:05:33.754 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:05:33.754 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:05:33.754 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:05:33.754 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:45.993393+0000", 2023-09-23T21:05:33.754 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.3204999999999998e-05 2023-09-23T21:05:33.754 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:05:33.755 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:05:33.755 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:05:33.755 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:05:33.755 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:05:33.755 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.34797.0:184168 6.3 6:c7a50969:::benchmark_data_smithi026_223868_object83042:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e169)", 2023-09-23T21:05:33.755 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T20:55:45.196004+0000", 2023-09-23T21:05:33.755 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 588.46729496700004, 2023-09-23T21:05:33.755 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.79750679000000002, 2023-09-23T21:05:33.755 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:05:33.755 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:05:33.755 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:05:33.755 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.34797", 2023-09-23T21:05:33.755 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T21:05:33.756 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 184168 2023-09-23T21:05:33.756 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:05:33.756 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:05:33.756 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:05:33.756 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:05:33.756 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:45.196004+0000", 2023-09-23T21:05:33.756 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:05:33.756 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:05:33.756 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:05:33.756 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:05:33.756 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:45.196004+0000", 2023-09-23T21:05:33.756 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:05:33.756 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:05:33.756 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:05:33.757 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:05:33.757 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:45.196002+0000", 2023-09-23T21:05:33.757 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T21:05:33.757 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:05:33.757 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:05:33.757 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:05:33.757 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:45.196010+0000", 2023-09-23T21:05:33.757 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.289e-06 2023-09-23T21:05:33.757 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:05:33.757 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:05:33.757 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:05:33.757 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:45.196025+0000", 2023-09-23T21:05:33.757 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.5281000000000002e-05 2023-09-23T21:05:33.758 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:05:33.758 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:05:33.758 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:05:33.758 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:45.196033+0000", 2023-09-23T21:05:33.758 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.447e-06 2023-09-23T21:05:33.758 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:05:33.758 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:05:33.758 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:05:33.758 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:45.196083+0000", 2023-09-23T21:05:33.758 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.0841999999999999e-05 2023-09-23T21:05:33.758 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:05:33.758 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:05:33.759 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:05:33.759 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:45.196091+0000", 2023-09-23T21:05:33.759 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.8010000000000004e-06 2023-09-23T21:05:33.759 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:05:33.759 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:05:33.759 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:05:33.759 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:45.965291+0000", 2023-09-23T21:05:33.759 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.76920029499999998 2023-09-23T21:05:33.759 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:05:33.759 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:05:33.759 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:05:33.759 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:45.965400+0000", 2023-09-23T21:05:33.759 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00010818400000000001 2023-09-23T21:05:33.759 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:05:33.760 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:05:33.760 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 2,6", 2023-09-23T21:05:33.760 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:45.965675+0000", 2023-09-23T21:05:33.760 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00027548699999999999 2023-09-23T21:05:33.760 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:05:33.760 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:05:33.760 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:05:33.760 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:45.977533+0000", 2023-09-23T21:05:33.760 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.011858172 2023-09-23T21:05:33.760 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:05:33.760 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:05:33.760 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:05:33.761 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:45.993435+0000", 2023-09-23T21:05:33.761 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.015902143000000001 2023-09-23T21:05:33.761 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:05:33.761 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:05:33.761 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:05:33.761 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:45.993455+0000", 2023-09-23T21:05:33.761 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.9777000000000001e-05 2023-09-23T21:05:33.761 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:05:33.761 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:05:33.761 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:05:33.761 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:45.993481+0000", 2023-09-23T21:05:33.761 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.6021e-05 2023-09-23T21:05:33.761 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:05:33.762 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:05:33.762 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:05:33.762 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:45.993511+0000", 2023-09-23T21:05:33.762 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.9578e-05 2023-09-23T21:05:33.762 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:05:33.762 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:05:33.762 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:05:33.762 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:05:33.762 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:05:33.762 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.34797.0:184197 6.3 6:c34c1e61:::benchmark_data_smithi026_223868_object83071:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e169)", 2023-09-23T21:05:33.762 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T20:55:45.233107+0000", 2023-09-23T21:05:33.762 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 588.43019174100004, 2023-09-23T21:05:33.763 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.760519681, 2023-09-23T21:05:33.763 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:05:33.763 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:05:33.763 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:05:33.763 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.34797", 2023-09-23T21:05:33.763 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/279971165", 2023-09-23T21:05:33.763 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 184197 2023-09-23T21:05:33.763 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:05:33.763 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:05:33.763 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:05:33.763 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:05:33.763 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:45.233107+0000", 2023-09-23T21:05:33.763 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:05:33.764 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:05:33.764 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:05:33.764 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:05:33.764 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:45.233107+0000", 2023-09-23T21:05:33.764 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:05:33.764 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:05:33.764 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:05:33.764 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:05:33.764 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:45.233105+0000", 2023-09-23T21:05:33.764 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999981 2023-09-23T21:05:33.764 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:05:33.764 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:05:33.765 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:05:33.765 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:45.233113+0000", 2023-09-23T21:05:33.765 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.4240000000000004e-06 2023-09-23T21:05:33.765 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:05:33.765 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:05:33.765 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:05:33.765 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:45.233130+0000", 2023-09-23T21:05:33.765 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.6966999999999999e-05 2023-09-23T21:05:33.765 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:05:33.765 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:05:33.765 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:05:33.765 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:45.233137+0000", 2023-09-23T21:05:33.766 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.8730000000000001e-06 2023-09-23T21:05:33.766 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:05:33.766 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:05:33.766 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:05:33.766 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:45.233204+0000", 2023-09-23T21:05:33.766 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.7105000000000002e-05 2023-09-23T21:05:33.766 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:05:33.766 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:05:33.766 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:05:33.766 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:45.233215+0000", 2023-09-23T21:05:33.766 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.1371e-05 2023-09-23T21:05:33.766 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:05:33.767 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:05:33.767 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:05:33.767 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:45.968007+0000", 2023-09-23T21:05:33.767 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.73479188900000003 2023-09-23T21:05:33.767 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:05:33.767 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:05:33.767 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:05:33.767 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:45.968089+0000", 2023-09-23T21:05:33.767 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.1660000000000001e-05 2023-09-23T21:05:33.767 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:05:33.767 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:05:33.767 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 2,6", 2023-09-23T21:05:33.768 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:45.968445+0000", 2023-09-23T21:05:33.768 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00035676499999999999 2023-09-23T21:05:33.768 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:05:33.768 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:05:33.768 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:05:33.768 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:45.977542+0000", 2023-09-23T21:05:33.768 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0090968160000000006 2023-09-23T21:05:33.768 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:05:33.768 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:05:33.768 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:05:33.768 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:45.993553+0000", 2023-09-23T21:05:33.768 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.016010277 2023-09-23T21:05:33.768 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:05:33.769 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:05:33.769 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:05:33.769 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:45.993573+0000", 2023-09-23T21:05:33.769 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.0268e-05 2023-09-23T21:05:33.769 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:05:33.769 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:05:33.769 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:05:33.769 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:45.993597+0000", 2023-09-23T21:05:33.769 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.3978e-05 2023-09-23T21:05:33.769 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:05:33.769 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:05:33.769 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:05:33.769 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T20:55:45.993627+0000", 2023-09-23T21:05:33.770 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.0122999999999999e-05 2023-09-23T21:05:33.770 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:05:33.770 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:05:33.770 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:05:33.770 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:05:33.770 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:05:33.770 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_repop(client.35469.0:213155 8.d e340/333 8:bc3212f9:::benchmark_data_smithi026_301269_object102398:head v 340'13182, mlcod=340'13179)", 2023-09-23T21:05:33.770 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:05:23.051103+0000", 2023-09-23T21:05:33.770 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 10.612196014, 2023-09-23T21:05:33.770 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.022629083000000001, 2023-09-23T21:05:33.770 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:05:33.770 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:05:33.770 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:05:33.771 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:05:33.771 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:05:33.771 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:23.051103+0000", 2023-09-23T21:05:33.771 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:05:33.771 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:05:33.771 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:05:33.771 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:05:33.771 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:23.051103+0000", 2023-09-23T21:05:33.771 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:05:33.771 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:05:33.771 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:05:33.771 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:05:33.771 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:23.051102+0000", 2023-09-23T21:05:33.772 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999995 2023-09-23T21:05:33.772 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:05:33.772 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:05:33.772 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:05:33.772 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:23.051110+0000", 2023-09-23T21:05:33.772 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.5220000000000001e-06 2023-09-23T21:05:33.772 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:05:33.772 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:05:33.772 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:05:33.772 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:23.051120+0000", 2023-09-23T21:05:33.772 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.0149999999999999e-05 2023-09-23T21:05:33.772 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:05:33.773 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:05:33.773 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:05:33.773 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:23.051126+0000", 2023-09-23T21:05:33.773 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.7270000000000004e-06 2023-09-23T21:05:33.773 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:05:33.773 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:05:33.773 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:05:33.773 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:23.057364+0000", 2023-09-23T21:05:33.773 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0062383810000000003 2023-09-23T21:05:33.773 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:05:33.773 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:05:33.773 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:05:33.773 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:23.057386+0000", 2023-09-23T21:05:33.774 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.2283999999999999e-05 2023-09-23T21:05:33.774 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:05:33.774 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:05:33.774 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:05:33.774 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:23.073690+0000", 2023-09-23T21:05:33.774 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.016303537999999999 2023-09-23T21:05:33.774 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:05:33.774 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:05:33.774 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:05:33.774 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:23.073732+0000", 2023-09-23T21:05:33.774 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.2182999999999999e-05 2023-09-23T21:05:33.774 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:05:33.774 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:05:33.775 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:05:33.775 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:05:33.775 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:05:33.775 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_repop(client.35469.0:213162 8.d e340/333 8:ba7b9907:::benchmark_data_smithi026_301269_object102405:head v 340'13183, mlcod=340'13179)", 2023-09-23T21:05:33.775 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:05:23.055332+0000", 2023-09-23T21:05:33.775 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 10.607966947, 2023-09-23T21:05:33.775 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.018423620000000002, 2023-09-23T21:05:33.775 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:05:33.775 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:05:33.775 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:05:33.775 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:05:33.775 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:05:33.776 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:23.055332+0000", 2023-09-23T21:05:33.776 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:05:33.776 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:05:33.776 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:05:33.776 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:05:33.776 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:23.055332+0000", 2023-09-23T21:05:33.776 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:05:33.776 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:05:33.776 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:05:33.776 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:05:33.776 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:23.055331+0000", 2023-09-23T21:05:33.776 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.999999 2023-09-23T21:05:33.776 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:05:33.777 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:05:33.777 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:05:33.777 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:23.055339+0000", 2023-09-23T21:05:33.777 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.2239999999999995e-06 2023-09-23T21:05:33.777 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:05:33.777 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:05:33.777 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:05:33.777 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:23.055351+0000", 2023-09-23T21:05:33.777 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.1914e-05 2023-09-23T21:05:33.777 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:05:33.777 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:05:33.777 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:05:33.777 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:23.055357+0000", 2023-09-23T21:05:33.777 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.8289999999999996e-06 2023-09-23T21:05:33.778 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:05:33.778 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:05:33.778 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:05:33.778 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:23.061061+0000", 2023-09-23T21:05:33.778 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0057040629999999997 2023-09-23T21:05:33.778 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:05:33.778 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:05:33.778 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:05:33.778 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:23.061082+0000", 2023-09-23T21:05:33.778 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.0638000000000001e-05 2023-09-23T21:05:33.778 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:05:33.778 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:05:33.779 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:05:33.779 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:23.073735+0000", 2023-09-23T21:05:33.779 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.012652985 2023-09-23T21:05:33.779 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:05:33.779 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:05:33.779 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:05:33.779 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:23.073756+0000", 2023-09-23T21:05:33.779 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.0835e-05 2023-09-23T21:05:33.779 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:05:33.779 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:05:33.779 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:05:33.779 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:05:33.779 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:05:33.779 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_repop(client.35469.0:213528 8.d e340/333 8:b76f9952:::benchmark_data_smithi026_301269_object102771:head v 340'13210, mlcod=340'13207)", 2023-09-23T21:05:33.780 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:05:23.408443+0000", 2023-09-23T21:05:33.780 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 10.254856294, 2023-09-23T21:05:33.780 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.018448396999999998, 2023-09-23T21:05:33.780 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:05:33.780 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:05:33.780 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:05:33.780 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:05:33.780 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:05:33.780 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:23.408443+0000", 2023-09-23T21:05:33.780 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:05:33.780 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:05:33.780 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:05:33.780 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:05:33.781 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:23.408443+0000", 2023-09-23T21:05:33.781 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:05:33.781 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:05:33.781 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:05:33.781 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:05:33.781 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:23.408442+0000", 2023-09-23T21:05:33.781 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.999999 2023-09-23T21:05:33.781 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:05:33.781 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:05:33.781 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:05:33.781 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:23.408450+0000", 2023-09-23T21:05:33.781 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.0779999999999996e-06 2023-09-23T21:05:33.781 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:05:33.782 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:05:33.782 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:05:33.782 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:23.408466+0000", 2023-09-23T21:05:33.782 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.5982999999999999e-05 2023-09-23T21:05:33.782 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:05:33.782 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:05:33.782 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:05:33.782 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:23.408472+0000", 2023-09-23T21:05:33.782 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.7190000000000004e-06 2023-09-23T21:05:33.782 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:05:33.782 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:05:33.782 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:05:33.783 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:23.410796+0000", 2023-09-23T21:05:33.783 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0023237430000000001 2023-09-23T21:05:33.783 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:05:33.783 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:05:33.783 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:05:33.783 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:23.410815+0000", 2023-09-23T21:05:33.783 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.9055e-05 2023-09-23T21:05:33.783 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:05:33.783 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:05:33.783 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:05:33.783 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:23.426850+0000", 2023-09-23T21:05:33.783 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.016034883999999999 2023-09-23T21:05:33.783 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:05:33.783 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:05:33.784 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:05:33.784 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:23.426891+0000", 2023-09-23T21:05:33.784 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.0982999999999997e-05 2023-09-23T21:05:33.784 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:05:33.784 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:05:33.784 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:05:33.784 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:05:33.784 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:05:33.784 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_repop(client.35469.0:213811 8.d e340/333 8:b8b67400:::benchmark_data_smithi026_301269_object103054:head v 340'13224, mlcod=340'13221)", 2023-09-23T21:05:33.784 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:05:23.666277+0000", 2023-09-23T21:05:33.784 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 9.9970214990000006, 2023-09-23T21:05:33.784 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.019376869000000001, 2023-09-23T21:05:33.784 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:05:33.785 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:05:33.785 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:05:33.785 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:05:33.785 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:05:33.785 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:23.666277+0000", 2023-09-23T21:05:33.785 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:05:33.785 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:05:33.785 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:05:33.785 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:05:33.785 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:23.666277+0000", 2023-09-23T21:05:33.785 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:05:33.785 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:05:33.785 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:05:33.786 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:05:33.786 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:23.666277+0000", 2023-09-23T21:05:33.786 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999995 2023-09-23T21:05:33.786 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:05:33.786 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:05:33.786 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:05:33.786 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:23.666283+0000", 2023-09-23T21:05:33.786 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.6780000000000003e-06 2023-09-23T21:05:33.786 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:05:33.786 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:05:33.786 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:05:33.787 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:23.666294+0000", 2023-09-23T21:05:33.787 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.0166999999999999e-05 2023-09-23T21:05:33.787 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:05:33.787 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:05:33.787 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:05:33.787 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:23.666299+0000", 2023-09-23T21:05:33.787 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.4430000000000002e-06 2023-09-23T21:05:33.787 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:05:33.787 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:05:33.787 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:05:33.787 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:23.666430+0000", 2023-09-23T21:05:33.787 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00013097900000000001 2023-09-23T21:05:33.788 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:05:33.788 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:05:33.788 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:05:33.788 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:23.666448+0000", 2023-09-23T21:05:33.788 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.8111e-05 2023-09-23T21:05:33.788 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:05:33.788 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:05:33.788 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:05:33.788 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:23.685611+0000", 2023-09-23T21:05:33.788 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.019163112 2023-09-23T21:05:33.788 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:05:33.788 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:05:33.788 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:05:33.789 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:23.685654+0000", 2023-09-23T21:05:33.789 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.3074999999999997e-05 2023-09-23T21:05:33.789 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:05:33.789 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:05:33.789 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:05:33.789 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:05:33.789 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:05:33.789 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_repop(client.35469.0:213951 8.5 e340/281 8:a7f59add:::benchmark_data_smithi026_301269_object103194:head v 340'10911, mlcod=340'10909)", 2023-09-23T21:05:33.789 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:05:23.799999+0000", 2023-09-23T21:05:33.789 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 9.8633002370000007, 2023-09-23T21:05:33.789 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.020002972000000001, 2023-09-23T21:05:33.789 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:05:33.789 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:05:33.790 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:05:33.790 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:05:33.790 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:05:33.790 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:23.799999+0000", 2023-09-23T21:05:33.790 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:05:33.790 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:05:33.790 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:05:33.790 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:05:33.790 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:23.799999+0000", 2023-09-23T21:05:33.790 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:05:33.790 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:05:33.790 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:05:33.790 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:05:33.791 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:23.799997+0000", 2023-09-23T21:05:33.791 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T21:05:33.791 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:05:33.791 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:05:33.791 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:05:33.791 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:23.800009+0000", 2023-09-23T21:05:33.791 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.1569999999999999e-05 2023-09-23T21:05:33.791 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:05:33.791 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:05:33.791 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:05:33.791 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:23.800024+0000", 2023-09-23T21:05:33.791 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.4664999999999999e-05 2023-09-23T21:05:33.791 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:05:33.792 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:05:33.792 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:05:33.792 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:23.800031+0000", 2023-09-23T21:05:33.792 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.5220000000000001e-06 2023-09-23T21:05:33.792 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:05:33.792 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:05:33.792 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:05:33.792 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:23.800079+0000", 2023-09-23T21:05:33.792 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.8291000000000002e-05 2023-09-23T21:05:33.792 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:05:33.792 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:05:33.792 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:05:33.792 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:23.800102+0000", 2023-09-23T21:05:33.793 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.2670000000000001e-05 2023-09-23T21:05:33.793 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:05:33.793 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:05:33.793 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:05:33.793 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:23.819953+0000", 2023-09-23T21:05:33.793 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.019850550000000002 2023-09-23T21:05:33.793 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:05:33.793 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:05:33.793 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:05:33.793 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:23.820002+0000", 2023-09-23T21:05:33.793 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.9079000000000002e-05 2023-09-23T21:05:33.793 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:05:33.793 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:05:33.793 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:05:33.794 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:05:33.794 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:05:33.794 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_repop(client.35469.0:213956 8.d e340/333 8:bd4e101e:::benchmark_data_smithi026_301269_object103199:head v 340'13234, mlcod=340'13231)", 2023-09-23T21:05:33.794 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:05:23.801325+0000", 2023-09-23T21:05:33.794 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 9.8619739699999993, 2023-09-23T21:05:33.794 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.018698899000000001, 2023-09-23T21:05:33.794 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:05:33.794 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:05:33.794 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:05:33.794 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:05:33.794 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:05:33.794 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:23.801325+0000", 2023-09-23T21:05:33.795 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:05:33.795 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:05:33.795 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:05:33.795 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:05:33.795 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:23.801325+0000", 2023-09-23T21:05:33.795 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:05:33.795 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:05:33.795 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:05:33.795 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:05:33.795 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:23.801324+0000", 2023-09-23T21:05:33.795 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.999999 2023-09-23T21:05:33.795 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:05:33.795 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:05:33.796 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:05:33.796 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:23.801331+0000", 2023-09-23T21:05:33.796 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.0480000000000001e-06 2023-09-23T21:05:33.796 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:05:33.796 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:05:33.796 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:05:33.796 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:23.801345+0000", 2023-09-23T21:05:33.796 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.3485e-05 2023-09-23T21:05:33.796 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:05:33.796 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:05:33.796 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:05:33.796 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:23.801350+0000", 2023-09-23T21:05:33.796 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.6049999999999997e-06 2023-09-23T21:05:33.796 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:05:33.797 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:05:33.797 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:05:33.797 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:23.801435+0000", 2023-09-23T21:05:33.797 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.4295999999999994e-05 2023-09-23T21:05:33.797 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:05:33.797 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:05:33.797 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:05:33.797 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:23.801460+0000", 2023-09-23T21:05:33.797 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.5619e-05 2023-09-23T21:05:33.797 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:05:33.797 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:05:33.797 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:05:33.797 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:23.820005+0000", 2023-09-23T21:05:33.798 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.018544576 2023-09-23T21:05:33.798 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:05:33.798 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:05:33.798 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:05:33.798 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:23.820024+0000", 2023-09-23T21:05:33.798 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.906e-05 2023-09-23T21:05:33.798 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:05:33.798 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:05:33.798 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:05:33.798 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:05:33.798 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:05:33.798 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_repop(client.35469.0:221118 8.c e341/254 8:35c29e02:::benchmark_data_smithi026_301269_object110361:head v 341'13929, mlcod=341'13925)", 2023-09-23T21:05:33.799 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:05:30.624684+0000", 2023-09-23T21:05:33.799 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 3.038615482, 2023-09-23T21:05:33.799 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.019472789000000001, 2023-09-23T21:05:33.799 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:05:33.799 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:05:33.799 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:05:33.799 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:05:33.799 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:05:33.799 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:30.624684+0000", 2023-09-23T21:05:33.799 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:05:33.799 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:05:33.799 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:05:33.799 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:05:33.800 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:30.624684+0000", 2023-09-23T21:05:33.800 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:05:33.800 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:05:33.800 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:05:33.800 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:05:33.800 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:30.624683+0000", 2023-09-23T21:05:33.800 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.999999 2023-09-23T21:05:33.800 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:05:33.800 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:05:33.800 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:05:33.800 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:30.624694+0000", 2023-09-23T21:05:33.800 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.1382e-05 2023-09-23T21:05:33.800 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:05:33.801 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:05:33.801 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:05:33.801 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:30.624710+0000", 2023-09-23T21:05:33.801 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.5508000000000001e-05 2023-09-23T21:05:33.801 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:05:33.801 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:05:33.801 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:05:33.801 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:30.624716+0000", 2023-09-23T21:05:33.801 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.3350000000000003e-06 2023-09-23T21:05:33.801 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:05:33.801 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:05:33.801 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:05:33.801 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:30.629388+0000", 2023-09-23T21:05:33.802 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0046718630000000001 2023-09-23T21:05:33.802 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:05:33.802 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:05:33.802 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:05:33.802 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:30.629408+0000", 2023-09-23T21:05:33.802 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.9851999999999999e-05 2023-09-23T21:05:33.802 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:05:33.802 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:05:33.802 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:05:33.802 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:30.643894+0000", 2023-09-23T21:05:33.802 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.014486716 2023-09-23T21:05:33.802 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:05:33.802 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:05:33.803 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:05:33.803 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:30.644156+0000", 2023-09-23T21:05:33.803 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00026188300000000003 2023-09-23T21:05:33.803 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:05:33.803 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:05:33.803 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:05:33.803 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:05:33.803 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:05:33.803 INFO:teuthology.orchestra.run.smithi070.stdout:} 2023-09-23T21:05:33.803 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:05:33.833 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:05:33.930 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T21:05:33.933 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:05:33.934 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:05:34.035 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:05:34.066 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:05:34.081 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_ops_in_flight 2023-09-23T21:05:34.099 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:05:34.102 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:05:34.102 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:05:34.203 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:05:34.229 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:05:34.263 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:05:34.266 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:05:34.267 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:05:34.358 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:05:34.367 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:05:34.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:05:34 smithi070 ceph-mon[168282]: 1.a deep-scrub starts 2023-09-23T21:05:34.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:05:34 smithi070 ceph-mon[168282]: 1.a deep-scrub ok 2023-09-23T21:05:34.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:05:34 smithi070 ceph-mon[168282]: pgmap v873: 119 pgs: 13 active+undersized+degraded, 33 active+undersized, 73 active+clean; 36 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 323/1782 objects degraded (18.126%) 2023-09-23T21:05:34.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:05:34 smithi070 ceph-mon[168282]: failed to encode map e342 with expected crc 2023-09-23T21:05:34.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:05:34 smithi070 ceph-mon[168282]: failed to encode map e342 with expected crc 2023-09-23T21:05:34.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:05:34 smithi070 ceph-mon[168282]: failed to encode map e342 with expected crc 2023-09-23T21:05:34.371 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:05:34 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/2618606194' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 3, "weight": 1.0}]': finished 2023-09-23T21:05:34.371 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:05:34 smithi070 ceph-mon[168282]: osdmap e342: 8 total, 7 up, 8 in 2023-09-23T21:05:34.393 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:05:34.426 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:05:34.429 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:05:34.429 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:05:34.530 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:05:34.558 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:05:34 smithi026 ceph-mon[175151]: 1.a deep-scrub starts 2023-09-23T21:05:34.559 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:05:34 smithi026 ceph-mon[175151]: 1.a deep-scrub ok 2023-09-23T21:05:34.559 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:05:34 smithi026 ceph-mon[175151]: pgmap v873: 119 pgs: 13 active+undersized+degraded, 33 active+undersized, 73 active+clean; 36 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 323/1782 objects degraded (18.126%) 2023-09-23T21:05:34.559 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:05:34 smithi026 ceph-mon[175151]: failed to encode map e342 with expected crc 2023-09-23T21:05:34.559 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:05:34 smithi026 ceph-mon[175151]: failed to encode map e342 with expected crc 2023-09-23T21:05:34.559 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:05:34 smithi026 ceph-mon[175151]: failed to encode map e342 with expected crc 2023-09-23T21:05:34.559 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:05:34 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/2618606194' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 3, "weight": 1.0}]': finished 2023-09-23T21:05:34.559 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:05:34 smithi026 ceph-mon[175151]: osdmap e342: 8 total, 7 up, 8 in 2023-09-23T21:05:34.559 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:05:34 smithi026 ceph-mon[179688]: 1.a deep-scrub starts 2023-09-23T21:05:34.559 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:05:34 smithi026 ceph-mon[179688]: 1.a deep-scrub ok 2023-09-23T21:05:34.560 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:05:34 smithi026 ceph-mon[179688]: pgmap v873: 119 pgs: 13 active+undersized+degraded, 33 active+undersized, 73 active+clean; 36 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 323/1782 objects degraded (18.126%) 2023-09-23T21:05:34.560 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:05:34 smithi026 ceph-mon[179688]: failed to encode map e342 with expected crc 2023-09-23T21:05:34.560 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:05:34 smithi026 ceph-mon[179688]: failed to encode map e342 with expected crc 2023-09-23T21:05:34.560 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:05:34 smithi026 ceph-mon[179688]: failed to encode map e342 with expected crc 2023-09-23T21:05:34.560 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:05:34 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/2618606194' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 3, "weight": 1.0}]': finished 2023-09-23T21:05:34.560 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:05:34 smithi026 ceph-mon[179688]: osdmap e342: 8 total, 7 up, 8 in 2023-09-23T21:05:34.562 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:05:34.599 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:05:34.602 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:05:34.603 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:05:34.697 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T21:05:34.704 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:05:34.732 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:05:34.816 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:05:34.819 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:05:34.820 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:05:34.920 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:05:34.948 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:05:35.103 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T21:05:35.106 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:05:35.107 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:05:35.207 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:05:35.231 INFO:teuthology.orchestra.run.smithi026.stderr:noscrub is set 2023-09-23T21:05:35.240 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:05:35.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:05:35 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/1085345201' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T21:05:35.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:05:35 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T21:05:35.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:05:35 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/1085345201' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T21:05:35.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:05:35 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T21:05:35.481 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:05:35 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/1085345201' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T21:05:35.482 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:05:35 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T21:05:35.540 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:05:35.543 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:05:35.543 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:05:35.644 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:05:35.676 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:05:35.676 INFO:tasks.cephadm.ceph_manager.ceph:removing pool_name unique_pool_2 2023-09-23T21:05:35.676 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph osd pool rm unique_pool_2 unique_pool_2 --yes-i-really-really-mean-it 2023-09-23T21:05:35.714 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T21:05:35.717 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:05:35.717 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:05:35.818 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:05:35.849 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:05:35.883 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0: {} 2023-09-23T21:05:35.884 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T21:05:35.886 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:05:35.887 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:05:35.914 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2: {} 2023-09-23T21:05:35.937 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3: {} 2023-09-23T21:05:35.956 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4: {} 2023-09-23T21:05:35.975 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T21:05:35.987 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:05:35.994 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6: {} 2023-09-23T21:05:36.013 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T21:05:36.014 INFO:teuthology.orchestra.run.smithi026.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T21:05:36.014 INFO:teuthology.orchestra.run.smithi026.stderr:Error ENXIO: problem getting command descriptions from osd.1 2023-09-23T21:05:36.014 INFO:teuthology.orchestra.run.smithi026.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T21:05:36.014 INFO:teuthology.orchestra.run.smithi026.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T21:05:36.014 INFO:teuthology.orchestra.run.smithi026.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T21:05:36.014 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T21:05:36.014 INFO:teuthology.orchestra.run.smithi026.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T21:05:36.014 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T21:05:36.018 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:05:36.040 INFO:teuthology.orchestra.run.smithi070.stdout:ERROR: (22) Invalid argument 2023-09-23T21:05:36.040 INFO:teuthology.orchestra.run.smithi070.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T21:05:36.052 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:05:36.054 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:05:36.055 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:05:36.155 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:05:36.186 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:05:36.330 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:05:36 smithi026 ceph-mon[175151]: pgmap v875: 119 pgs: 12 active+undersized+degraded, 34 active+undersized, 73 active+clean; 20 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 253/1002 objects degraded (25.250%) 2023-09-23T21:05:36.330 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:05:36 smithi026 ceph-mon[175151]: Health check update: Degraded data redundancy: 253/1002 objects degraded (25.250%), 12 pgs degraded (PG_DEGRADED) 2023-09-23T21:05:36.330 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:05:36 smithi026 ceph-mon[175151]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:05:36.330 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:05:36 smithi026 ceph-mon[175151]: failed to encode map e343 with expected crc 2023-09-23T21:05:36.330 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:05:36 smithi026 ceph-mon[175151]: failed to encode map e343 with expected crc 2023-09-23T21:05:36.330 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:05:36 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T21:05:36.331 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:05:36 smithi026 ceph-mon[175151]: osdmap e343: 8 total, 7 up, 8 in 2023-09-23T21:05:36.331 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:05:36 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/2051679591' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:05:36.331 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:05:36 smithi026 ceph-mon[179688]: pgmap v875: 119 pgs: 12 active+undersized+degraded, 34 active+undersized, 73 active+clean; 20 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 253/1002 objects degraded (25.250%) 2023-09-23T21:05:36.331 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:05:36 smithi026 ceph-mon[179688]: Health check update: Degraded data redundancy: 253/1002 objects degraded (25.250%), 12 pgs degraded (PG_DEGRADED) 2023-09-23T21:05:36.331 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:05:36 smithi026 ceph-mon[179688]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:05:36.331 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:05:36 smithi026 ceph-mon[179688]: failed to encode map e343 with expected crc 2023-09-23T21:05:36.331 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:05:36 smithi026 ceph-mon[179688]: failed to encode map e343 with expected crc 2023-09-23T21:05:36.331 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:05:36 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T21:05:36.331 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:05:36 smithi026 ceph-mon[179688]: osdmap e343: 8 total, 7 up, 8 in 2023-09-23T21:05:36.331 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:05:36 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/2051679591' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:05:36.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:05:36 smithi070 ceph-mon[168282]: pgmap v875: 119 pgs: 12 active+undersized+degraded, 34 active+undersized, 73 active+clean; 20 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 253/1002 objects degraded (25.250%) 2023-09-23T21:05:36.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:05:36 smithi070 ceph-mon[168282]: Health check update: Degraded data redundancy: 253/1002 objects degraded (25.250%), 12 pgs degraded (PG_DEGRADED) 2023-09-23T21:05:36.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:05:36 smithi070 ceph-mon[168282]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:05:36.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:05:36 smithi070 ceph-mon[168282]: failed to encode map e343 with expected crc 2023-09-23T21:05:36.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:05:36 smithi070 ceph-mon[168282]: failed to encode map e343 with expected crc 2023-09-23T21:05:36.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:05:36 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T21:05:36.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:05:36 smithi070 ceph-mon[168282]: osdmap e343: 8 total, 7 up, 8 in 2023-09-23T21:05:36.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:05:36 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/2051679591' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:05:36.396 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:05:36.399 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:05:36.399 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:05:36.450 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_blocked_ops 2023-09-23T21:05:36.500 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:05:36.529 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:05:36.563 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:05:36.565 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:05:36.565 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:05:36.606 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T21:05:36.606 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T21:05:36.621 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:05:36.667 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:05:36.692 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:05:36.724 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:05:36.727 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:05:36.727 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:05:36.828 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:05:36.865 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:05:36.899 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:05:36.902 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:05:36.902 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:05:37.003 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:05:37.028 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:05:37.040 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T21:05:37.061 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:05:37.064 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:05:37.065 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:05:37.165 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:05:37.194 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:05:37.278 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:05:37 smithi070 ceph-mon[168282]: 1.10 scrub starts 2023-09-23T21:05:37.364 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:05:37.367 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:05:37.367 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:05:37.468 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:05:37.497 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:05:37.585 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:05:37.587 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:05:37.587 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:05:37.595 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:05:37 smithi026 ceph-mon[179688]: 1.10 scrub starts 2023-09-23T21:05:37.595 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:05:37 smithi026 ceph-mon[175151]: 1.10 scrub starts 2023-09-23T21:05:37.688 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:05:37.715 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:05:37.774 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T21:05:37.818 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:05:37.820 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:05:37.820 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:05:37.921 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:05:37.949 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:05:37.985 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:05:37.988 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:05:37.989 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:05:38.089 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:05:38.118 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:05:38.204 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T21:05:38.207 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:05:38.207 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:05:38.253 INFO:teuthology.orchestra.run.smithi026.stderr:pool 'unique_pool_2' removed 2023-09-23T21:05:38.309 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:05:38.355 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:05:38.393 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:05:38.395 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:05:38.396 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:05:38.425 INFO:teuthology.orchestra.run.smithi070.stdout:ERROR: (22) Invalid argument 2023-09-23T21:05:38.425 INFO:teuthology.orchestra.run.smithi070.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T21:05:38.446 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:05:38 smithi070 ceph-mon[168282]: pgmap v877: 119 pgs: 6 active+undersized+degraded, 40 active+undersized, 73 active+clean; 2.6 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 4/168 objects degraded (2.381%) 2023-09-23T21:05:38.446 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:05:38 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/4205491280' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_2", "pool2": "unique_pool_2", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-23T21:05:38.446 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:05:38 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_2", "pool2": "unique_pool_2", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-23T21:05:38.497 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:05:38.524 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:05:38 smithi026 ceph-mon[179688]: pgmap v877: 119 pgs: 6 active+undersized+degraded, 40 active+undersized, 73 active+clean; 2.6 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 4/168 objects degraded (2.381%) 2023-09-23T21:05:38.524 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:05:38 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/4205491280' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_2", "pool2": "unique_pool_2", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-23T21:05:38.525 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:05:38 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_2", "pool2": "unique_pool_2", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-23T21:05:38.525 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:05:38 smithi026 ceph-mon[175151]: pgmap v877: 119 pgs: 6 active+undersized+degraded, 40 active+undersized, 73 active+clean; 2.6 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 4/168 objects degraded (2.381%) 2023-09-23T21:05:38.525 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:05:38 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/4205491280' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_2", "pool2": "unique_pool_2", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-23T21:05:38.525 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:05:38 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_2", "pool2": "unique_pool_2", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-23T21:05:38.528 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:05:38.607 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T21:05:38.806 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_historic_ops 2023-09-23T21:05:38.939 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 6, 7, 4, 5] out_osds: [] dead_osds: [1] live_osds: [0, 2, 3, 4, 5, 7, 6] 2023-09-23T21:05:38.939 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T21:05:38.939 INFO:tasks.thrashosds.thrasher:Reviving osd 1 2023-09-23T21:05:38.940 INFO:tasks.cephadm.osd.1:Restarting osd.1 (starting--it wasn't running)... 2023-09-23T21:05:38.940 DEBUG:teuthology.orchestra.run.smithi026:osd.1> sudo journalctl -f -n 0 -u ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7@osd.1.service 2023-09-23T21:05:38.942 DEBUG:teuthology.orchestra.run.smithi026:> sudo systemctl start ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7@osd.1 2023-09-23T21:05:38.970 INFO:journalctl@ceph.osd.1.smithi026.stdout:-- Logs begin at Sat 2023-09-23 20:20:42 UTC. -- 2023-09-23T21:05:39.239 INFO:journalctl@ceph.osd.1.smithi026.stdout:Sep 23 21:05:39 smithi026 systemd[1]: Starting Ceph osd.1 for b3f8e4ec-5a50-11ee-8db4-212e2dc638e7... 2023-09-23T21:05:39.240 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:05:39 smithi026 ceph-mon[175151]: failed to encode map e344 with expected crc 2023-09-23T21:05:39.240 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:05:39 smithi026 ceph-mon[175151]: failed to encode map e344 with expected crc 2023-09-23T21:05:39.240 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:05:39 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_2", "pool2": "unique_pool_2", "yes_i_really_really_mean_it": true}]': finished 2023-09-23T21:05:39.472 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T21:05:39.611 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:05:39 smithi026 ceph-mon[175151]: osdmap e344: 8 total, 7 up, 8 in 2023-09-23T21:05:39.611 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:05:39 smithi026 ceph-mon[175151]: failed to encode map e344 with expected crc 2023-09-23T21:05:39.611 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:05:39 smithi026 ceph-mon[179688]: failed to encode map e344 with expected crc 2023-09-23T21:05:39.611 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:05:39 smithi026 ceph-mon[179688]: failed to encode map e344 with expected crc 2023-09-23T21:05:39.611 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:05:39 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_2", "pool2": "unique_pool_2", "yes_i_really_really_mean_it": true}]': finished 2023-09-23T21:05:39.611 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:05:39 smithi026 ceph-mon[179688]: osdmap e344: 8 total, 7 up, 8 in 2023-09-23T21:05:39.611 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:05:39 smithi026 ceph-mon[179688]: failed to encode map e344 with expected crc 2023-09-23T21:05:39.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:05:39 smithi070 ceph-mon[168282]: failed to encode map e344 with expected crc 2023-09-23T21:05:39.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:05:39 smithi070 ceph-mon[168282]: failed to encode map e344 with expected crc 2023-09-23T21:05:39.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:05:39 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_2", "pool2": "unique_pool_2", "yes_i_really_really_mean_it": true}]': finished 2023-09-23T21:05:39.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:05:39 smithi070 ceph-mon[168282]: osdmap e344: 8 total, 7 up, 8 in 2023-09-23T21:05:39.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:05:39 smithi070 ceph-mon[168282]: failed to encode map e344 with expected crc 2023-09-23T21:05:39.881 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 21:05:39 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[164029]: ::ffff:172.21.15.70 - - [23/Sep/2023:21:05:39] "GET /metrics HTTP/1.1" 200 36315 "" "Prometheus/2.43.0" 2023-09-23T21:05:40.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:05:40 smithi070 ceph-mon[168282]: failed to encode map e344 with expected crc 2023-09-23T21:05:40.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:05:40 smithi070 ceph-mon[168282]: pgmap v879: 93 pgs: 4 active+undersized+degraded, 33 active+undersized, 56 active+clean; 652 KiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 4/75 objects degraded (5.333%) 2023-09-23T21:05:40.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:05:40 smithi026 ceph-mon[175151]: failed to encode map e344 with expected crc 2023-09-23T21:05:40.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:05:40 smithi026 ceph-mon[175151]: pgmap v879: 93 pgs: 4 active+undersized+degraded, 33 active+undersized, 56 active+clean; 652 KiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 4/75 objects degraded (5.333%) 2023-09-23T21:05:40.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:05:40 smithi026 ceph-mon[179688]: failed to encode map e344 with expected crc 2023-09-23T21:05:40.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:05:40 smithi026 ceph-mon[179688]: pgmap v879: 93 pgs: 4 active+undersized+degraded, 33 active+undersized, 56 active+clean; 652 KiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 4/75 objects degraded (5.333%) 2023-09-23T21:05:40.686 INFO:teuthology.task.full_sequential:In full_sequential, running task radosbench... 2023-09-23T21:05:40.687 INFO:tasks.radosbench:Beginning radosbench... 2023-09-23T21:05:40.687 INFO:tasks.cephadm.ceph_manager.ceph:creating pool_name unique_pool_3 2023-09-23T21:05:40.687 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph osd pool create unique_pool_3 16 2023-09-23T21:05:40.711 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T21:05:40.714 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:05:40.715 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:05:40.715 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:05:40.816 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:05:40.853 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:05:41.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:05:41 smithi070 ceph-mon[168282]: Health check update: Degraded data redundancy: 4/75 objects degraded (5.333%), 4 pgs degraded (PG_DEGRADED) 2023-09-23T21:05:41.631 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:05:41 smithi026 ceph-mon[179688]: Health check update: Degraded data redundancy: 4/75 objects degraded (5.333%), 4 pgs degraded (PG_DEGRADED) 2023-09-23T21:05:41.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:05:41 smithi026 ceph-mon[175151]: Health check update: Degraded data redundancy: 4/75 objects degraded (5.333%), 4 pgs degraded (PG_DEGRADED) 2023-09-23T21:05:41.763 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:05:42.470 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:05:42.474 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:05:42.474 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:05:42.541 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:05:42 smithi026 ceph-mon[175151]: pgmap v880: 93 pgs: 4 active+undersized+degraded, 33 active+undersized, 56 active+clean; 652 KiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 4/75 objects degraded (5.333%) 2023-09-23T21:05:42.541 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:05:42 smithi026 ceph-mon[179688]: pgmap v880: 93 pgs: 4 active+undersized+degraded, 33 active+undersized, 56 active+clean; 652 KiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 4/75 objects degraded (5.333%) 2023-09-23T21:05:42.575 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:05:42.602 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:05:42 smithi070 ceph-mon[168282]: pgmap v880: 93 pgs: 4 active+undersized+degraded, 33 active+undersized, 56 active+clean; 652 KiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 4/75 objects degraded (5.333%) 2023-09-23T21:05:42.606 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:05:43.370 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:05:43.373 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:05:43.373 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:05:43.474 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:05:43.506 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:05:43.593 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:05:43.597 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:05:43.598 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:05:43.617 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:05:43.699 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:05:43.732 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:05:43.798 INFO:teuthology.orchestra.run.smithi070.stdout:ERROR: (22) Invalid argument 2023-09-23T21:05:43.798 INFO:teuthology.orchestra.run.smithi070.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T21:05:43.881 INFO:journalctl@ceph.osd.1.smithi026.stdout:Sep 23 21:05:43 smithi026 bash[339203]: --> Failed to activate via raw: did not find any matching OSD to activate 2023-09-23T21:05:43.882 INFO:journalctl@ceph.osd.1.smithi026.stdout:Sep 23 21:05:43 smithi026 bash[339203]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2023-09-23T21:05:43.882 INFO:journalctl@ceph.osd.1.smithi026.stdout:Sep 23 21:05:43 smithi026 bash[339203]: Running command: /usr/bin/ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev /dev/vg_nvme/lv_3 --path /var/lib/ceph/osd/ceph-1 --no-mon-config 2023-09-23T21:05:43.882 INFO:journalctl@ceph.osd.1.smithi026.stdout:Sep 23 21:05:43 smithi026 bash[339203]: Running command: /usr/bin/ln -snf /dev/vg_nvme/lv_3 /var/lib/ceph/osd/ceph-1/block 2023-09-23T21:05:43.882 INFO:journalctl@ceph.osd.1.smithi026.stdout:Sep 23 21:05:43 smithi026 bash[339203]: Running command: /usr/bin/chown -h ceph:ceph /var/lib/ceph/osd/ceph-1/block 2023-09-23T21:05:43.882 INFO:journalctl@ceph.osd.1.smithi026.stdout:Sep 23 21:05:43 smithi026 bash[339203]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2 2023-09-23T21:05:43.882 INFO:journalctl@ceph.osd.1.smithi026.stdout:Sep 23 21:05:43 smithi026 bash[339203]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2023-09-23T21:05:43.882 INFO:journalctl@ceph.osd.1.smithi026.stdout:Sep 23 21:05:43 smithi026 bash[339203]: --> ceph-volume lvm activate successful for osd ID: 1 2023-09-23T21:05:43.920 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:05:43.923 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:05:43.924 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:05:44.024 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:05:44.056 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:05:44.160 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:05:44.163 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:05:44.164 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:05:44.245 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_ops_in_flight 2023-09-23T21:05:44.265 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:05:44.317 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:05:44.532 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:05:44.534 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:05:44 smithi026 ceph-mon[179688]: pgmap v881: 93 pgs: 4 active+undersized+degraded, 33 active+undersized, 56 active+clean; 652 KiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 4/75 objects degraded (5.333%) 2023-09-23T21:05:44.534 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:05:44 smithi026 ceph-mon[175151]: pgmap v881: 93 pgs: 4 active+undersized+degraded, 33 active+undersized, 56 active+clean; 652 KiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 4/75 objects degraded (5.333%) 2023-09-23T21:05:44.535 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:05:44.535 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:05:44.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:05:44 smithi070 ceph-mon[168282]: pgmap v881: 93 pgs: 4 active+undersized+degraded, 33 active+undersized, 56 active+clean; 652 KiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 4/75 objects degraded (5.333%) 2023-09-23T21:05:44.636 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:05:44.665 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:05:44.698 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:05:44.701 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:05:44.701 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:05:44.802 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0: {} 2023-09-23T21:05:44.802 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:05:44.821 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2: {} 2023-09-23T21:05:44.838 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:05:44.847 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3: {} 2023-09-23T21:05:44.876 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4: {} 2023-09-23T21:05:44.895 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T21:05:44.915 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6: {} 2023-09-23T21:05:44.934 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T21:05:44.935 INFO:teuthology.orchestra.run.smithi026.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T21:05:44.935 INFO:teuthology.orchestra.run.smithi026.stderr:Error ENXIO: problem getting command descriptions from osd.1 2023-09-23T21:05:44.935 INFO:teuthology.orchestra.run.smithi026.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T21:05:44.935 INFO:teuthology.orchestra.run.smithi026.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T21:05:44.935 INFO:teuthology.orchestra.run.smithi026.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T21:05:44.935 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T21:05:44.935 INFO:teuthology.orchestra.run.smithi026.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T21:05:44.935 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T21:05:45.133 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T21:05:45.137 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:05:45.137 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:05:45.238 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:05:45.266 INFO:teuthology.orchestra.run.smithi026.stderr:nodeep-scrub is set 2023-09-23T21:05:45.272 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:05:45.445 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:05:45 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/2878085273' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:05:45.445 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:05:45 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:05:45.445 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:05:45 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/1954268348' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:05:45.445 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:05:45 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/2878085273' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:05:45.446 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:05:45 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:05:45.446 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:05:45 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/1954268348' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:05:45.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:05:45 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/2878085273' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:05:45.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:05:45 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:05:45.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:05:45 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/1954268348' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:05:45.769 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T21:05:45.770 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T21:05:45.782 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:05:45.864 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T21:05:45.867 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:05:45.868 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:05:45.969 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:05:46.000 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:05:46.170 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:05:46.176 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:05:46.176 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:05:46.277 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:05:46.311 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:05:46.348 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:05:46.350 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:05:46.351 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:05:46.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:05:46 smithi026 ceph-mon[175151]: pgmap v882: 93 pgs: 4 active+undersized+degraded, 33 active+undersized, 56 active+clean; 652 KiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 4/75 objects degraded (5.333%) 2023-09-23T21:05:46.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:05:46 smithi026 ceph-mon[175151]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:05:46.383 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:05:46 smithi026 ceph-mon[175151]: failed to encode map e345 with expected crc 2023-09-23T21:05:46.383 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:05:46 smithi026 ceph-mon[175151]: failed to encode map e345 with expected crc 2023-09-23T21:05:46.383 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:05:46 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T21:05:46.383 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:05:46 smithi026 ceph-mon[175151]: osdmap e345: 8 total, 7 up, 8 in 2023-09-23T21:05:46.383 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:05:46 smithi026 ceph-mon[175151]: failed to encode map e345 with expected crc 2023-09-23T21:05:46.383 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:05:46 smithi026 ceph-mon[175151]: failed to encode map e345 with expected crc 2023-09-23T21:05:46.383 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:05:46 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:05:46.383 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:05:46 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:05:46.383 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:05:46 smithi026 ceph-mon[179688]: pgmap v882: 93 pgs: 4 active+undersized+degraded, 33 active+undersized, 56 active+clean; 652 KiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 4/75 objects degraded (5.333%) 2023-09-23T21:05:46.384 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:05:46 smithi026 ceph-mon[179688]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:05:46.384 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:05:46 smithi026 ceph-mon[179688]: failed to encode map e345 with expected crc 2023-09-23T21:05:46.384 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:05:46 smithi026 ceph-mon[179688]: failed to encode map e345 with expected crc 2023-09-23T21:05:46.384 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:05:46 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T21:05:46.384 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:05:46 smithi026 ceph-mon[179688]: osdmap e345: 8 total, 7 up, 8 in 2023-09-23T21:05:46.384 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:05:46 smithi026 ceph-mon[179688]: failed to encode map e345 with expected crc 2023-09-23T21:05:46.384 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:05:46 smithi026 ceph-mon[179688]: failed to encode map e345 with expected crc 2023-09-23T21:05:46.384 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:05:46 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:05:46.384 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:05:46 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:05:46.385 INFO:journalctl@ceph.osd.1.smithi026.stdout:Sep 23 21:05:46 smithi026 podman[339961]: 2023-09-23T21:05:46.385 INFO:journalctl@ceph.osd.1.smithi026.stdout:Sep 23 21:05:46 smithi026 ceph-osd[340206]: -- 172.21.15.26:0/3356558788 <== mon.2 v2:172.21.15.26:3301/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x55bd8598c4e0 con 0x55bd8595cc00 2023-09-23T21:05:46.433 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_ops_in_flight 2023-09-23T21:05:46.452 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:05:46.508 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:05:46.561 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:05:46.565 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:05:46.566 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:05:46.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:05:46 smithi070 ceph-mon[168282]: pgmap v882: 93 pgs: 4 active+undersized+degraded, 33 active+undersized, 56 active+clean; 652 KiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 4/75 objects degraded (5.333%) 2023-09-23T21:05:46.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:05:46 smithi070 ceph-mon[168282]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:05:46.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:05:46 smithi070 ceph-mon[168282]: failed to encode map e345 with expected crc 2023-09-23T21:05:46.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:05:46 smithi070 ceph-mon[168282]: failed to encode map e345 with expected crc 2023-09-23T21:05:46.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:05:46 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T21:05:46.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:05:46 smithi070 ceph-mon[168282]: osdmap e345: 8 total, 7 up, 8 in 2023-09-23T21:05:46.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:05:46 smithi070 ceph-mon[168282]: failed to encode map e345 with expected crc 2023-09-23T21:05:46.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:05:46 smithi070 ceph-mon[168282]: failed to encode map e345 with expected crc 2023-09-23T21:05:46.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:05:46 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:05:46.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:05:46 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:05:46.666 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:05:46.693 INFO:journalctl@ceph.osd.1.smithi026.stdout:Sep 23 21:05:46 smithi026 bash[339961]: 4c18795444a1d8e6313af920f10c45a2bc95bb7f4c3fbf5bde4bc1edb8842189 2023-09-23T21:05:46.693 INFO:journalctl@ceph.osd.1.smithi026.stdout:Sep 23 21:05:46 smithi026 systemd[1]: Started Ceph osd.1 for b3f8e4ec-5a50-11ee-8db4-212e2dc638e7. 2023-09-23T21:05:46.696 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:05:46.880 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T21:05:46.883 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:05:46.883 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:05:46.984 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:05:47.011 INFO:journalctl@ceph.osd.1.smithi026.stdout:Sep 23 21:05:46 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd-1[340180]: 2023-09-23T21:05:46.891+0000 7f49cedcf640 -1 Falling back to public interface 2023-09-23T21:05:47.015 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:05:47.098 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:05:47.102 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:05:47.102 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:05:47.203 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:05:47.234 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:05:47.275 INFO:teuthology.orchestra.run.smithi026.stderr:pool 'unique_pool_3' created 2023-09-23T21:05:47.372 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:05:47.376 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:05:47.377 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:05:47.390 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:05:47.431 INFO:teuthology.orchestra.run.smithi026.stdout:{ 2023-09-23T21:05:47.431 INFO:teuthology.orchestra.run.smithi026.stdout: "ops": [], 2023-09-23T21:05:47.431 INFO:teuthology.orchestra.run.smithi026.stdout: "num_ops": 0 2023-09-23T21:05:47.431 INFO:teuthology.orchestra.run.smithi026.stdout:} 2023-09-23T21:05:47.478 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:05:47.510 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:05:47.546 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:05:47.548 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:05:47 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/1278036073' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16}]: dispatch 2023-09-23T21:05:47.549 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:05:47.550 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:05:47.633 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:05:47 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/1278036073' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16}]: dispatch 2023-09-23T21:05:47.633 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:05:47 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/1278036073' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16}]: dispatch 2023-09-23T21:05:47.651 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:05:47.683 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:05:47.771 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T21:05:47.813 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T21:05:48.052 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_blocked_ops 2023-09-23T21:05:48.283 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:05:48.290 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:05:48.290 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:05:48.368 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph osd pool application enable unique_pool_3 rados --yes-i-really-mean-it || true 2023-09-23T21:05:48.391 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:05:48.423 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:05:48.460 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:05:48.462 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:05:48.463 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:05:48.512 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:05:48 smithi026 ceph-mon[179688]: pgmap v884: 93 pgs: 4 active+undersized+degraded, 33 active+undersized, 56 active+clean; 652 KiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 4/75 objects degraded (5.333%) 2023-09-23T21:05:48.512 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:05:48 smithi026 ceph-mon[179688]: failed to encode map e346 with expected crc 2023-09-23T21:05:48.512 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:05:48 smithi026 ceph-mon[179688]: failed to encode map e346 with expected crc 2023-09-23T21:05:48.512 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:05:48 smithi026 ceph-mon[179688]: failed to encode map e346 with expected crc 2023-09-23T21:05:48.512 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:05:48 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/1278036073' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16}]': finished 2023-09-23T21:05:48.512 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:05:48 smithi026 ceph-mon[179688]: osdmap e346: 8 total, 7 up, 8 in 2023-09-23T21:05:48.512 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:05:48 smithi026 ceph-mon[175151]: pgmap v884: 93 pgs: 4 active+undersized+degraded, 33 active+undersized, 56 active+clean; 652 KiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 4/75 objects degraded (5.333%) 2023-09-23T21:05:48.512 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:05:48 smithi026 ceph-mon[175151]: failed to encode map e346 with expected crc 2023-09-23T21:05:48.513 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:05:48 smithi026 ceph-mon[175151]: failed to encode map e346 with expected crc 2023-09-23T21:05:48.513 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:05:48 smithi026 ceph-mon[175151]: failed to encode map e346 with expected crc 2023-09-23T21:05:48.513 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:05:48 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/1278036073' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16}]': finished 2023-09-23T21:05:48.513 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:05:48 smithi026 ceph-mon[175151]: osdmap e346: 8 total, 7 up, 8 in 2023-09-23T21:05:48.564 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:05:48.590 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:05:48 smithi070 ceph-mon[168282]: pgmap v884: 93 pgs: 4 active+undersized+degraded, 33 active+undersized, 56 active+clean; 652 KiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 4/75 objects degraded (5.333%) 2023-09-23T21:05:48.590 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:05:48 smithi070 ceph-mon[168282]: failed to encode map e346 with expected crc 2023-09-23T21:05:48.590 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:05:48 smithi070 ceph-mon[168282]: failed to encode map e346 with expected crc 2023-09-23T21:05:48.590 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:05:48 smithi070 ceph-mon[168282]: failed to encode map e346 with expected crc 2023-09-23T21:05:48.590 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:05:48 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/1278036073' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16}]': finished 2023-09-23T21:05:48.590 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:05:48 smithi070 ceph-mon[168282]: osdmap e346: 8 total, 7 up, 8 in 2023-09-23T21:05:48.592 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:05:48.625 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:05:48.628 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:05:48.628 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:05:48.729 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:05:48.758 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:05:48.806 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:05:48.813 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:05:48.814 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:05:48.888 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:05:48.915 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:05:48.927 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:05:48.948 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:05:49.035 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:05:49.038 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:05:49.039 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:05:49.040 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:05:49.141 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:05:49.171 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:05:49.207 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:05:49.210 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:05:49.210 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:05:49.311 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:05:49.344 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:05:49.401 INFO:teuthology.orchestra.run.smithi026.stderr:no valid command found; 10 closest matches: 2023-09-23T21:05:49.401 INFO:teuthology.orchestra.run.smithi026.stderr:0 2023-09-23T21:05:49.401 INFO:teuthology.orchestra.run.smithi026.stderr:1 2023-09-23T21:05:49.401 INFO:teuthology.orchestra.run.smithi026.stderr:2 2023-09-23T21:05:49.401 INFO:teuthology.orchestra.run.smithi026.stderr:abort 2023-09-23T21:05:49.401 INFO:teuthology.orchestra.run.smithi026.stderr:assert 2023-09-23T21:05:49.402 INFO:teuthology.orchestra.run.smithi026.stderr:bluefs debug_inject_read_zeros 2023-09-23T21:05:49.402 INFO:teuthology.orchestra.run.smithi026.stderr:bluefs files list 2023-09-23T21:05:49.402 INFO:teuthology.orchestra.run.smithi026.stderr:bluefs stats 2023-09-23T21:05:49.402 INFO:teuthology.orchestra.run.smithi026.stderr:bluestore bluefs device info [] 2023-09-23T21:05:49.402 INFO:teuthology.orchestra.run.smithi026.stderr:config diff 2023-09-23T21:05:49.402 INFO:teuthology.orchestra.run.smithi026.stderr:admin_socket: invalid command 2023-09-23T21:05:49.418 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:05:49 smithi026 ceph-mon[175151]: failed to encode map e347 with expected crc 2023-09-23T21:05:49.418 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:05:49 smithi026 ceph-mon[175151]: failed to encode map e347 with expected crc 2023-09-23T21:05:49.418 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:05:49 smithi026 ceph-mon[175151]: failed to encode map e347 with expected crc 2023-09-23T21:05:49.418 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:05:49 smithi026 ceph-mon[175151]: osdmap e347: 8 total, 7 up, 8 in 2023-09-23T21:05:49.418 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:05:49 smithi026 ceph-mon[175151]: failed to encode map e347 with expected crc 2023-09-23T21:05:49.418 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:05:49 smithi026 ceph-mon[175151]: failed to encode map e347 with expected crc 2023-09-23T21:05:49.418 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:05:49 smithi026 ceph-mon[175151]: failed to encode map e347 with expected crc 2023-09-23T21:05:49.419 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:05:49 smithi026 ceph-mon[179688]: failed to encode map e347 with expected crc 2023-09-23T21:05:49.419 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:05:49 smithi026 ceph-mon[179688]: failed to encode map e347 with expected crc 2023-09-23T21:05:49.419 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:05:49 smithi026 ceph-mon[179688]: failed to encode map e347 with expected crc 2023-09-23T21:05:49.419 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:05:49 smithi026 ceph-mon[179688]: osdmap e347: 8 total, 7 up, 8 in 2023-09-23T21:05:49.419 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:05:49 smithi026 ceph-mon[179688]: failed to encode map e347 with expected crc 2023-09-23T21:05:49.419 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:05:49 smithi026 ceph-mon[179688]: failed to encode map e347 with expected crc 2023-09-23T21:05:49.419 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:05:49 smithi026 ceph-mon[179688]: failed to encode map e347 with expected crc 2023-09-23T21:05:49.472 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:05:49.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:05:49 smithi070 ceph-mon[168282]: failed to encode map e347 with expected crc 2023-09-23T21:05:49.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:05:49 smithi070 ceph-mon[168282]: failed to encode map e347 with expected crc 2023-09-23T21:05:49.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:05:49 smithi070 ceph-mon[168282]: failed to encode map e347 with expected crc 2023-09-23T21:05:49.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:05:49 smithi070 ceph-mon[168282]: osdmap e347: 8 total, 7 up, 8 in 2023-09-23T21:05:49.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:05:49 smithi070 ceph-mon[168282]: failed to encode map e347 with expected crc 2023-09-23T21:05:49.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:05:49 smithi070 ceph-mon[168282]: failed to encode map e347 with expected crc 2023-09-23T21:05:49.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:05:49 smithi070 ceph-mon[168282]: failed to encode map e347 with expected crc 2023-09-23T21:05:49.868 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 21:05:49 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[164029]: ::ffff:172.21.15.70 - - [23/Sep/2023:21:05:49] "GET /metrics HTTP/1.1" 200 33961 "" "Prometheus/2.43.0" 2023-09-23T21:05:49.922 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:05:49.930 DEBUG:teuthology.orchestra.run:got remote process result: 22 2023-09-23T21:05:49.931 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:05:49.932 INFO:tasks.cephadm.ceph_manager.ceph:waiting on admin_socket for osd-1, ['dump_ops_in_flight'] 2023-09-23T21:05:49.932 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:05:50.033 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:05:50.071 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:05:50.369 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T21:05:50.370 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:05:50.371 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:05:50.381 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:05:50 smithi026 ceph-mon[179688]: pgmap v887: 109 pgs: 2 creating+peering, 14 unknown, 4 active+undersized+degraded, 33 active+undersized, 56 active+clean; 652 KiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 4/75 objects degraded (5.333%) 2023-09-23T21:05:50.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:05:50 smithi026 ceph-mon[179688]: failed to encode map e348 with expected crc 2023-09-23T21:05:50.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:05:50 smithi026 ceph-mon[179688]: failed to encode map e348 with expected crc 2023-09-23T21:05:50.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:05:50 smithi026 ceph-mon[179688]: failed to encode map e348 with expected crc 2023-09-23T21:05:50.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:05:50 smithi026 ceph-mon[179688]: osdmap e348: 8 total, 7 up, 8 in 2023-09-23T21:05:50.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:05:50 smithi026 ceph-mon[175151]: pgmap v887: 109 pgs: 2 creating+peering, 14 unknown, 4 active+undersized+degraded, 33 active+undersized, 56 active+clean; 652 KiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 4/75 objects degraded (5.333%) 2023-09-23T21:05:50.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:05:50 smithi026 ceph-mon[175151]: failed to encode map e348 with expected crc 2023-09-23T21:05:50.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:05:50 smithi026 ceph-mon[175151]: failed to encode map e348 with expected crc 2023-09-23T21:05:50.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:05:50 smithi026 ceph-mon[175151]: failed to encode map e348 with expected crc 2023-09-23T21:05:50.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:05:50 smithi026 ceph-mon[175151]: osdmap e348: 8 total, 7 up, 8 in 2023-09-23T21:05:50.472 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:05:50.503 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:05:50.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:05:50 smithi070 ceph-mon[168282]: pgmap v887: 109 pgs: 2 creating+peering, 14 unknown, 4 active+undersized+degraded, 33 active+undersized, 56 active+clean; 652 KiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 4/75 objects degraded (5.333%) 2023-09-23T21:05:50.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:05:50 smithi070 ceph-mon[168282]: failed to encode map e348 with expected crc 2023-09-23T21:05:50.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:05:50 smithi070 ceph-mon[168282]: failed to encode map e348 with expected crc 2023-09-23T21:05:50.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:05:50 smithi070 ceph-mon[168282]: failed to encode map e348 with expected crc 2023-09-23T21:05:50.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:05:50 smithi070 ceph-mon[168282]: osdmap e348: 8 total, 7 up, 8 in 2023-09-23T21:05:50.767 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:05:50.771 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:05:50.771 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:05:50.872 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:05:50.903 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:05:50.941 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:05:50.942 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:05:50.943 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:05:51.043 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:05:51.072 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:05:51.105 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:05:51.108 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:05:51.109 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:05:51.209 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:05:51.243 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:05:51.781 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T21:05:51.786 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:05:51.787 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:05:51.872 INFO:teuthology.orchestra.run.smithi026.stdout:{ 2023-09-23T21:05:51.872 INFO:teuthology.orchestra.run.smithi026.stdout: "ops": [], 2023-09-23T21:05:51.872 INFO:teuthology.orchestra.run.smithi026.stdout: "complaint_time": 30, 2023-09-23T21:05:51.872 INFO:teuthology.orchestra.run.smithi026.stdout: "num_blocked_ops": 0 2023-09-23T21:05:51.873 INFO:teuthology.orchestra.run.smithi026.stdout:} 2023-09-23T21:05:51.888 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:05:51.920 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:05:51.957 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:05:51.958 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:05:51.960 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:05:52.060 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:05:52.098 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:05:52.346 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:05:52.348 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:05:52 smithi026 ceph-mon[179688]: pgmap v889: 109 pgs: 2 creating+peering, 8 unknown, 4 active+undersized+degraded, 36 active+undersized, 59 active+clean; 652 KiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 4/75 objects degraded (5.333%) 2023-09-23T21:05:52.349 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:05:52 smithi026 ceph-mon[175151]: pgmap v889: 109 pgs: 2 creating+peering, 8 unknown, 4 active+undersized+degraded, 36 active+undersized, 59 active+clean; 652 KiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 4/75 objects degraded (5.333%) 2023-09-23T21:05:52.349 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:05:52.350 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:05:52.413 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_historic_ops 2023-09-23T21:05:52.450 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:05:52.485 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:05:52.593 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T21:05:52.597 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:05:52.597 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:05:52.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:05:52 smithi070 ceph-mon[168282]: pgmap v889: 109 pgs: 2 creating+peering, 8 unknown, 4 active+undersized+degraded, 36 active+undersized, 59 active+clean; 652 KiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 4/75 objects degraded (5.333%) 2023-09-23T21:05:52.698 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:05:52.732 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:05:52.875 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T21:05:52.878 INFO:journalctl@ceph.osd.1.smithi026.stdout:Sep 23 21:05:52 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd-1[340180]: 2023-09-23T21:05:52.762+0000 7f49cedcf640 -1 osd.1 338 log_to_monitors true 2023-09-23T21:05:52.879 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:05:52.879 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:05:52.896 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0: {} 2023-09-23T21:05:52.916 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2: {} 2023-09-23T21:05:52.937 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3: {} 2023-09-23T21:05:52.962 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4: {} 2023-09-23T21:05:52.980 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:05:52.990 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T21:05:53.010 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:05:53.011 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6: {} 2023-09-23T21:05:53.031 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T21:05:53.031 INFO:teuthology.orchestra.run.smithi026.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T21:05:53.032 INFO:teuthology.orchestra.run.smithi026.stderr:Error ENXIO: problem getting command descriptions from osd.1 2023-09-23T21:05:53.032 INFO:teuthology.orchestra.run.smithi026.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T21:05:53.032 INFO:teuthology.orchestra.run.smithi026.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T21:05:53.032 INFO:teuthology.orchestra.run.smithi026.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T21:05:53.032 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T21:05:53.032 INFO:teuthology.orchestra.run.smithi026.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T21:05:53.032 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T21:05:53.056 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:05:53.059 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:05:53.059 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:05:53.088 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:05:53.160 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:05:53.193 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:05:53.229 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:05:53.232 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:05:53.233 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:05:53.294 INFO:teuthology.orchestra.run.smithi026.stderr:enabled application 'rados' on pool 'unique_pool_3' 2023-09-23T21:05:53.295 INFO:teuthology.orchestra.run.smithi026.stderr:noscrub is unset 2023-09-23T21:05:53.333 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:05:53.362 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:05:53.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:05:53 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/2120564275' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T21:05:53.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:05:53 smithi070 ceph-mon[168282]: from='osd.1 [v2:172.21.15.26:6810/2971312233,v1:172.21.15.26:6811/2971312233]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2023-09-23T21:05:53.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:05:53 smithi070 ceph-mon[168282]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2023-09-23T21:05:53.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:05:53 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/2477416095' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:05:53.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:05:53 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/2243932299' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T21:05:53.631 INFO:journalctl@ceph.osd.1.smithi026.stdout:Sep 23 21:05:53 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd-1[340180]: 2023-09-23T21:05:53.311+0000 7f49bea39700 -1 osd.1 338 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-09-23T21:05:53.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:05:53 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/2120564275' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T21:05:53.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:05:53 smithi026 ceph-mon[175151]: from='osd.1 [v2:172.21.15.26:6810/2971312233,v1:172.21.15.26:6811/2971312233]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2023-09-23T21:05:53.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:05:53 smithi026 ceph-mon[175151]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2023-09-23T21:05:53.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:05:53 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/2477416095' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:05:53.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:05:53 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/2243932299' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T21:05:53.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:05:53 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/2120564275' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T21:05:53.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:05:53 smithi026 ceph-mon[179688]: from='osd.1 [v2:172.21.15.26:6810/2971312233,v1:172.21.15.26:6811/2971312233]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2023-09-23T21:05:53.633 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:05:53 smithi026 ceph-mon[179688]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2023-09-23T21:05:53.633 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:05:53 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/2477416095' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:05:53.633 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:05:53 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/2243932299' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T21:05:54.603 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:05:54 smithi026 ceph-mon[175151]: pgmap v890: 109 pgs: 4 active+undersized+degraded, 42 active+undersized, 63 active+clean; 652 KiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 4/75 objects degraded (5.333%) 2023-09-23T21:05:54.604 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:05:54 smithi026 ceph-mon[175151]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:05:54.604 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:05:54 smithi026 ceph-mon[175151]: failed to encode map e349 with expected crc 2023-09-23T21:05:54.604 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:05:54 smithi026 ceph-mon[175151]: failed to encode map e349 with expected crc 2023-09-23T21:05:54.604 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:05:54 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/2120564275' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T21:05:54.604 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:05:54 smithi026 ceph-mon[175151]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2023-09-23T21:05:54.604 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:05:54 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/2243932299' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-09-23T21:05:54.604 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:05:54 smithi026 ceph-mon[175151]: osdmap e349: 8 total, 7 up, 8 in 2023-09-23T21:05:54.604 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:05:54 smithi026 ceph-mon[175151]: from='osd.1 [v2:172.21.15.26:6810/2971312233,v1:172.21.15.26:6811/2971312233]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi026", "root=default"]}]: dispatch 2023-09-23T21:05:54.604 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:05:54 smithi026 ceph-mon[175151]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi026", "root=default"]}]: dispatch 2023-09-23T21:05:54.604 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:05:54 smithi026 ceph-mon[175151]: 1.10 scrub starts 2023-09-23T21:05:54.604 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:05:54 smithi026 ceph-mon[175151]: 1.10 scrub ok 2023-09-23T21:05:54.605 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:05:54 smithi026 ceph-mon[179688]: pgmap v890: 109 pgs: 4 active+undersized+degraded, 42 active+undersized, 63 active+clean; 652 KiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 4/75 objects degraded (5.333%) 2023-09-23T21:05:54.605 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:05:54 smithi026 ceph-mon[179688]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:05:54.605 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:05:54 smithi026 ceph-mon[179688]: failed to encode map e349 with expected crc 2023-09-23T21:05:54.605 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:05:54 smithi026 ceph-mon[179688]: failed to encode map e349 with expected crc 2023-09-23T21:05:54.605 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:05:54 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/2120564275' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T21:05:54.605 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:05:54 smithi026 ceph-mon[179688]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2023-09-23T21:05:54.605 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:05:54 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/2243932299' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-09-23T21:05:54.605 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:05:54 smithi026 ceph-mon[179688]: osdmap e349: 8 total, 7 up, 8 in 2023-09-23T21:05:54.605 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:05:54 smithi026 ceph-mon[179688]: from='osd.1 [v2:172.21.15.26:6810/2971312233,v1:172.21.15.26:6811/2971312233]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi026", "root=default"]}]: dispatch 2023-09-23T21:05:54.605 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:05:54 smithi026 ceph-mon[179688]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi026", "root=default"]}]: dispatch 2023-09-23T21:05:54.606 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:05:54 smithi026 ceph-mon[179688]: 1.10 scrub starts 2023-09-23T21:05:54.606 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:05:54 smithi026 ceph-mon[179688]: 1.10 scrub ok 2023-09-23T21:05:54.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:05:54 smithi070 ceph-mon[168282]: pgmap v890: 109 pgs: 4 active+undersized+degraded, 42 active+undersized, 63 active+clean; 652 KiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 4/75 objects degraded (5.333%) 2023-09-23T21:05:54.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:05:54 smithi070 ceph-mon[168282]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:05:54.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:05:54 smithi070 ceph-mon[168282]: failed to encode map e349 with expected crc 2023-09-23T21:05:54.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:05:54 smithi070 ceph-mon[168282]: failed to encode map e349 with expected crc 2023-09-23T21:05:54.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:05:54 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/2120564275' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T21:05:54.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:05:54 smithi070 ceph-mon[168282]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2023-09-23T21:05:54.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:05:54 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/2243932299' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-09-23T21:05:54.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:05:54 smithi070 ceph-mon[168282]: osdmap e349: 8 total, 7 up, 8 in 2023-09-23T21:05:54.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:05:54 smithi070 ceph-mon[168282]: from='osd.1 [v2:172.21.15.26:6810/2971312233,v1:172.21.15.26:6811/2971312233]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi026", "root=default"]}]: dispatch 2023-09-23T21:05:54.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:05:54 smithi070 ceph-mon[168282]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi026", "root=default"]}]: dispatch 2023-09-23T21:05:54.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:05:54 smithi070 ceph-mon[168282]: 1.10 scrub starts 2023-09-23T21:05:54.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:05:54 smithi070 ceph-mon[168282]: 1.10 scrub ok 2023-09-23T21:05:54.797 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T21:05:54.800 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:05:54.801 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:05:54.902 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:05:54.933 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_ops_in_flight 2023-09-23T21:05:54.936 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:05:54.958 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T21:05:54.959 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T21:05:55.237 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T21:05:55.239 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:05:55.240 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:05:55.340 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:05:55.370 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:05:55.406 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:05:55.408 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:05:55.408 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:05:55.501 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:05:55 smithi026 ceph-mon[175151]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-09-23T21:05:55.502 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:05:55 smithi026 ceph-mon[175151]: osd.1 [v2:172.21.15.26:6810/2971312233,v1:172.21.15.26:6811/2971312233] boot 2023-09-23T21:05:55.502 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:05:55 smithi026 ceph-mon[175151]: osdmap e350: 8 total, 8 up, 8 in 2023-09-23T21:05:55.502 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:05:55 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-23T21:05:55.502 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:05:55 smithi026 ceph-mon[175151]: failed to encode map e350 with expected crc 2023-09-23T21:05:55.502 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:05:55 smithi026 ceph-mon[175151]: failed to encode map e350 with expected crc 2023-09-23T21:05:55.502 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:05:55 smithi026 ceph-mon[175151]: failed to encode map e350 with expected crc 2023-09-23T21:05:55.502 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:05:55 smithi026 ceph-mon[175151]: failed to encode map e350 with expected crc 2023-09-23T21:05:55.502 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:05:55 smithi026 ceph-mon[179688]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-09-23T21:05:55.502 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:05:55 smithi026 ceph-mon[179688]: osd.1 [v2:172.21.15.26:6810/2971312233,v1:172.21.15.26:6811/2971312233] boot 2023-09-23T21:05:55.502 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:05:55 smithi026 ceph-mon[179688]: osdmap e350: 8 total, 8 up, 8 in 2023-09-23T21:05:55.503 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:05:55 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-23T21:05:55.503 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:05:55 smithi026 ceph-mon[179688]: failed to encode map e350 with expected crc 2023-09-23T21:05:55.503 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:05:55 smithi026 ceph-mon[179688]: failed to encode map e350 with expected crc 2023-09-23T21:05:55.503 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:05:55 smithi026 ceph-mon[179688]: failed to encode map e350 with expected crc 2023-09-23T21:05:55.503 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:05:55 smithi026 ceph-mon[179688]: failed to encode map e350 with expected crc 2023-09-23T21:05:55.509 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:05:55.536 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:05:55 smithi070 ceph-mon[168282]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-09-23T21:05:55.536 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:05:55 smithi070 ceph-mon[168282]: osd.1 [v2:172.21.15.26:6810/2971312233,v1:172.21.15.26:6811/2971312233] boot 2023-09-23T21:05:55.536 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:05:55 smithi070 ceph-mon[168282]: osdmap e350: 8 total, 8 up, 8 in 2023-09-23T21:05:55.536 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:05:55 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-23T21:05:55.536 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:05:55 smithi070 ceph-mon[168282]: failed to encode map e350 with expected crc 2023-09-23T21:05:55.536 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:05:55 smithi070 ceph-mon[168282]: failed to encode map e350 with expected crc 2023-09-23T21:05:55.537 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:05:55 smithi070 ceph-mon[168282]: failed to encode map e350 with expected crc 2023-09-23T21:05:55.537 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:05:55 smithi070 ceph-mon[168282]: failed to encode map e350 with expected crc 2023-09-23T21:05:55.537 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:05:55.571 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:05:55.573 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:05:55.574 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:05:55.674 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:05:55.704 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:05:55.706 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:05:55.744 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:05:55.747 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:05:55.747 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:05:55.848 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:05:55.879 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:05:55.965 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:05:55.968 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:05:55.969 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:05:56.070 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:05:56.077 DEBUG:teuthology.orchestra.run.smithi026:> /bin/sh -c 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados --no-log-to-stderr --name client.0 -b 65536 --object-size 65536 -p unique_pool_3 bench 90 write' 2023-09-23T21:05:56.080 INFO:tasks.radosbench:joining radosbench (timing out after 3000s) 2023-09-23T21:05:56.080 INFO:teuthology.orchestra.run:waiting for 3000 2023-09-23T21:05:56.101 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:05:56.138 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:05:56.141 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:05:56.142 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:05:56.239 INFO:tasks.radosbench.radosbench.0.smithi026.stdout:hints = 1 2023-09-23T21:05:56.240 INFO:tasks.radosbench.radosbench.0.smithi026.stdout:Maintaining 16 concurrent writes of 65536 bytes to objects of size 65536 for up to 90 seconds or 0 objects 2023-09-23T21:05:56.240 INFO:tasks.radosbench.radosbench.0.smithi026.stdout:Object prefix: benchmark_data_smithi026_342732 2023-09-23T21:05:56.240 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T21:05:56.240 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 0 0 0 0 0 0 - 0 2023-09-23T21:05:56.243 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:05:56.288 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:05:56.334 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:05:56.338 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:05:56.339 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:05:56.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:05:56 smithi070 ceph-mon[168282]: pgmap v893: 109 pgs: 4 active+undersized+degraded, 42 active+undersized, 63 active+clean; 652 KiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 4/75 objects degraded (5.333%) 2023-09-23T21:05:56.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:05:56 smithi070 ceph-mon[168282]: failed to encode map e351 with expected crc 2023-09-23T21:05:56.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:05:56 smithi070 ceph-mon[168282]: failed to encode map e351 with expected crc 2023-09-23T21:05:56.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:05:56 smithi070 ceph-mon[168282]: osdmap e351: 8 total, 8 up, 8 in 2023-09-23T21:05:56.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:05:56 smithi070 ceph-mon[168282]: failed to encode map e351 with expected crc 2023-09-23T21:05:56.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:05:56 smithi070 ceph-mon[168282]: failed to encode map e351 with expected crc 2023-09-23T21:05:56.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:05:56 smithi070 ceph-mon[168282]: failed to encode map e351 with expected crc 2023-09-23T21:05:56.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:05:56 smithi070 ceph-mon[168282]: failed to encode map e351 with expected crc 2023-09-23T21:05:56.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:05:56 smithi070 ceph-mon[168282]: failed to encode map e351 with expected crc 2023-09-23T21:05:56.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:05:56 smithi070 ceph-mon[168282]: failed to encode map e351 with expected crc 2023-09-23T21:05:56.439 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:05:56.481 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:05:56.530 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:05:56.534 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:05:56.534 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:05:56.633 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:05:56 smithi026 ceph-mon[175151]: pgmap v893: 109 pgs: 4 active+undersized+degraded, 42 active+undersized, 63 active+clean; 652 KiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 4/75 objects degraded (5.333%) 2023-09-23T21:05:56.633 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:05:56 smithi026 ceph-mon[175151]: failed to encode map e351 with expected crc 2023-09-23T21:05:56.633 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:05:56 smithi026 ceph-mon[175151]: failed to encode map e351 with expected crc 2023-09-23T21:05:56.633 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:05:56 smithi026 ceph-mon[175151]: osdmap e351: 8 total, 8 up, 8 in 2023-09-23T21:05:56.633 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:05:56 smithi026 ceph-mon[175151]: failed to encode map e351 with expected crc 2023-09-23T21:05:56.633 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:05:56 smithi026 ceph-mon[175151]: failed to encode map e351 with expected crc 2023-09-23T21:05:56.634 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:05:56 smithi026 ceph-mon[175151]: failed to encode map e351 with expected crc 2023-09-23T21:05:56.634 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:05:56 smithi026 ceph-mon[175151]: failed to encode map e351 with expected crc 2023-09-23T21:05:56.634 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:05:56 smithi026 ceph-mon[175151]: failed to encode map e351 with expected crc 2023-09-23T21:05:56.634 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:05:56 smithi026 ceph-mon[175151]: failed to encode map e351 with expected crc 2023-09-23T21:05:56.634 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:05:56 smithi026 ceph-mon[179688]: pgmap v893: 109 pgs: 4 active+undersized+degraded, 42 active+undersized, 63 active+clean; 652 KiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 4/75 objects degraded (5.333%) 2023-09-23T21:05:56.634 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:05:56 smithi026 ceph-mon[179688]: failed to encode map e351 with expected crc 2023-09-23T21:05:56.634 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:05:56 smithi026 ceph-mon[179688]: failed to encode map e351 with expected crc 2023-09-23T21:05:56.634 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:05:56 smithi026 ceph-mon[179688]: osdmap e351: 8 total, 8 up, 8 in 2023-09-23T21:05:56.634 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:05:56 smithi026 ceph-mon[179688]: failed to encode map e351 with expected crc 2023-09-23T21:05:56.634 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:05:56 smithi026 ceph-mon[179688]: failed to encode map e351 with expected crc 2023-09-23T21:05:56.635 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:05:56 smithi026 ceph-mon[179688]: failed to encode map e351 with expected crc 2023-09-23T21:05:56.635 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:05:56 smithi026 ceph-mon[179688]: failed to encode map e351 with expected crc 2023-09-23T21:05:56.635 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:05:56 smithi026 ceph-mon[179688]: failed to encode map e351 with expected crc 2023-09-23T21:05:56.635 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:05:56 smithi026 ceph-mon[179688]: failed to encode map e351 with expected crc 2023-09-23T21:05:56.635 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:05:56.671 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:05:56.697 INFO:teuthology.orchestra.run.smithi026.stdout:ERROR: (22) Invalid argument 2023-09-23T21:05:56.697 INFO:teuthology.orchestra.run.smithi026.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T21:05:56.721 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:05:56.725 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:05:56.725 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:05:56.826 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:05:56.894 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:05:56.960 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T21:05:56.963 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:05:56.967 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:05:56.968 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:05:57.069 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:05:57.106 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:05:57.151 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:05:57.154 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:05:57.155 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:05:57.186 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_ops_in_flight 2023-09-23T21:05:57.240 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 1 16 3155 3139 196.176 196.188 0.00732471 0.00507943 2023-09-23T21:05:57.255 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:05:57.285 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T21:05:57.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:05:57 smithi070 ceph-mon[168282]: 1.b scrub starts 2023-09-23T21:05:57.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:05:57 smithi070 ceph-mon[168282]: 1.b scrub ok 2023-09-23T21:05:57.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:05:57 smithi070 ceph-mon[168282]: 1.12 scrub starts 2023-09-23T21:05:57.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:05:57 smithi070 ceph-mon[168282]: 1.12 scrub ok 2023-09-23T21:05:57.375 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:05:57.567 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:05:57.571 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:05:57 smithi026 ceph-mon[179688]: 1.b scrub starts 2023-09-23T21:05:57.571 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:05:57 smithi026 ceph-mon[179688]: 1.b scrub ok 2023-09-23T21:05:57.571 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:05:57 smithi026 ceph-mon[179688]: 1.12 scrub starts 2023-09-23T21:05:57.571 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:05:57 smithi026 ceph-mon[179688]: 1.12 scrub ok 2023-09-23T21:05:57.572 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:05:57.572 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:05:57 smithi026 ceph-mon[175151]: 1.b scrub starts 2023-09-23T21:05:57.572 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:05:57 smithi026 ceph-mon[175151]: 1.b scrub ok 2023-09-23T21:05:57.572 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:05:57 smithi026 ceph-mon[175151]: 1.12 scrub starts 2023-09-23T21:05:57.572 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:05:57 smithi026 ceph-mon[175151]: 1.12 scrub ok 2023-09-23T21:05:57.573 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:05:57.674 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:05:57.764 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:05:57.793 INFO:teuthology.orchestra.run.smithi026.stdout:{ 2023-09-23T21:05:57.794 INFO:teuthology.orchestra.run.smithi026.stdout: "ops": [ 2023-09-23T21:05:57.794 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:05:57.794 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35856.0:4273 9.4 9:2080e3f7:::benchmark_data_smithi026_342732_object4272:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e351)", 2023-09-23T21:05:57.794 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T21:05:57.787684+0000", 2023-09-23T21:05:57.794 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 0.0041569440000000001, 2023-09-23T21:05:57.794 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0042019070000000004, 2023-09-23T21:05:57.794 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:05:57.794 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "waiting for sub ops", 2023-09-23T21:05:57.794 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T21:05:57.794 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35856", 2023-09-23T21:05:57.795 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/4180396258", 2023-09-23T21:05:57.795 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 4273 2023-09-23T21:05:57.795 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:05:57.795 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:05:57.795 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:05:57.795 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:05:57.795 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:57.787684+0000", 2023-09-23T21:05:57.795 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:05:57.795 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:05:57.795 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:05:57.795 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:05:57.796 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:57.787684+0000", 2023-09-23T21:05:57.796 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:05:57.796 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:05:57.796 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:05:57.796 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:05:57.796 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:57.787683+0000", 2023-09-23T21:05:57.796 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T21:05:57.796 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:05:57.796 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:05:57.796 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:05:57.796 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:57.787718+0000", 2023-09-23T21:05:57.796 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.4332999999999998e-05 2023-09-23T21:05:57.797 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:05:57.797 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:05:57.797 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:05:57.797 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:57.787726+0000", 2023-09-23T21:05:57.797 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.5110000000000003e-06 2023-09-23T21:05:57.797 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:05:57.797 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:05:57.797 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:05:57.797 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:57.787732+0000", 2023-09-23T21:05:57.797 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.9610000000000002e-06 2023-09-23T21:05:57.797 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:05:57.798 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:05:57.798 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:05:57.798 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:57.787788+0000", 2023-09-23T21:05:57.798 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.5665000000000003e-05 2023-09-23T21:05:57.798 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:05:57.798 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:05:57.798 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:05:57.798 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:57.787915+0000", 2023-09-23T21:05:57.798 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000127045 2023-09-23T21:05:57.798 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:05:57.798 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:05:57.799 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 3,4", 2023-09-23T21:05:57.799 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:57.788070+0000", 2023-09-23T21:05:57.799 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000155575 2023-09-23T21:05:57.799 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:05:57.799 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:05:57.799 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:05:57.799 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:57.790018+0000", 2023-09-23T21:05:57.799 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0019476649999999999 2023-09-23T21:05:57.799 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:05:57.799 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:05:57.799 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T21:05:57.799 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:57.790200+0000", 2023-09-23T21:05:57.800 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000182084 2023-09-23T21:05:57.800 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:05:57.800 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:05:57.800 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:05:57.800 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:05:57.800 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:05:57.800 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35856.0:4275 9.0 9:021dce5f:::benchmark_data_smithi026_342732_object4274:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e351)", 2023-09-23T21:05:57.800 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T21:05:57.787796+0000", 2023-09-23T21:05:57.800 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 0.0040452580000000004, 2023-09-23T21:05:57.800 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0042107769999999997, 2023-09-23T21:05:57.800 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:05:57.801 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "waiting for sub ops", 2023-09-23T21:05:57.801 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T21:05:57.801 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35856", 2023-09-23T21:05:57.801 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/4180396258", 2023-09-23T21:05:57.801 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 4275 2023-09-23T21:05:57.801 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:05:57.801 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:05:57.801 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:05:57.801 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:05:57.801 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:57.787796+0000", 2023-09-23T21:05:57.801 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:05:57.801 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:05:57.802 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:05:57.802 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:05:57.802 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:57.787796+0000", 2023-09-23T21:05:57.802 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:05:57.802 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:05:57.802 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:05:57.802 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:05:57.802 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:57.787796+0000", 2023-09-23T21:05:57.802 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T21:05:57.802 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:05:57.802 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:05:57.803 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:05:57.803 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:57.787818+0000", 2023-09-23T21:05:57.803 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.2648999999999999e-05 2023-09-23T21:05:57.803 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:05:57.803 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:05:57.803 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:05:57.803 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:57.787826+0000", 2023-09-23T21:05:57.803 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.6820000000000003e-06 2023-09-23T21:05:57.803 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:05:57.803 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:05:57.803 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:05:57.803 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:57.787832+0000", 2023-09-23T21:05:57.804 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.9660000000000001e-06 2023-09-23T21:05:57.804 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:05:57.804 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:05:57.804 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:05:57.804 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:57.788131+0000", 2023-09-23T21:05:57.804 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00029899700000000001 2023-09-23T21:05:57.804 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:05:57.804 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:05:57.804 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:05:57.804 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:57.788254+0000", 2023-09-23T21:05:57.804 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00012296299999999999 2023-09-23T21:05:57.805 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:05:57.805 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:05:57.805 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 2,6", 2023-09-23T21:05:57.805 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:57.788384+0000", 2023-09-23T21:05:57.805 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00012978299999999999 2023-09-23T21:05:57.805 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:05:57.805 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:05:57.805 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:05:57.805 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:57.791867+0000", 2023-09-23T21:05:57.805 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0034834470000000002 2023-09-23T21:05:57.805 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:05:57.805 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:05:57.806 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T21:05:57.806 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:57.791874+0000", 2023-09-23T21:05:57.806 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.4390000000000003e-06 2023-09-23T21:05:57.806 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:05:57.806 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:05:57.806 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:05:57.806 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:05:57.806 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:05:57.806 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35856.0:4286 9.0 9:0a2bd58a:::benchmark_data_smithi026_342732_object4285:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e351)", 2023-09-23T21:05:57.806 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T21:05:57.787845+0000", 2023-09-23T21:05:57.806 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 0.0039964759999999997, 2023-09-23T21:05:57.807 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0042200629999999996, 2023-09-23T21:05:57.807 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:05:57.807 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "waiting for sub ops", 2023-09-23T21:05:57.807 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T21:05:57.807 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35856", 2023-09-23T21:05:57.807 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/4180396258", 2023-09-23T21:05:57.807 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 4286 2023-09-23T21:05:57.807 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:05:57.807 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:05:57.807 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:05:57.807 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:05:57.808 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:57.787845+0000", 2023-09-23T21:05:57.808 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:05:57.808 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:05:57.808 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:05:57.808 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:05:57.808 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:57.787845+0000", 2023-09-23T21:05:57.808 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:05:57.808 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:05:57.808 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:05:57.808 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:05:57.808 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:57.787844+0000", 2023-09-23T21:05:57.809 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T21:05:57.809 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:05:57.809 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:05:57.809 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:05:57.809 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:57.787868+0000", 2023-09-23T21:05:57.809 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.4173999999999999e-05 2023-09-23T21:05:57.809 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:05:57.809 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:05:57.809 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:05:57.809 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:57.787876+0000", 2023-09-23T21:05:57.809 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.7249999999999997e-06 2023-09-23T21:05:57.809 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:05:57.810 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:05:57.810 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:05:57.810 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:57.787884+0000", 2023-09-23T21:05:57.810 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.1780000000000003e-06 2023-09-23T21:05:57.810 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:05:57.810 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:05:57.810 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:05:57.810 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:57.788769+0000", 2023-09-23T21:05:57.810 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00088480600000000003 2023-09-23T21:05:57.810 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:05:57.810 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:05:57.811 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:05:57.811 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:57.788874+0000", 2023-09-23T21:05:57.811 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00010500799999999999 2023-09-23T21:05:57.811 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:05:57.811 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:05:57.811 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 2,6", 2023-09-23T21:05:57.811 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:57.789013+0000", 2023-09-23T21:05:57.811 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00013881200000000001 2023-09-23T21:05:57.811 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:05:57.811 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:05:57.811 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T21:05:57.811 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:57.791880+0000", 2023-09-23T21:05:57.812 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0028665600000000002 2023-09-23T21:05:57.812 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:05:57.812 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:05:57.812 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:05:57.812 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:57.791914+0000", 2023-09-23T21:05:57.812 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.4449999999999997e-05 2023-09-23T21:05:57.812 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:05:57.812 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:05:57.812 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:05:57.812 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:05:57.812 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:05:57.812 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35856.0:4288 9.0 9:06cca1bb:::benchmark_data_smithi026_342732_object4287:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e351)", 2023-09-23T21:05:57.813 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T21:05:57.788022+0000", 2023-09-23T21:05:57.813 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 0.0038192899999999999, 2023-09-23T21:05:57.813 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0040950459999999998, 2023-09-23T21:05:57.813 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:05:57.813 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "waiting for sub ops", 2023-09-23T21:05:57.813 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T21:05:57.813 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35856", 2023-09-23T21:05:57.813 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/4180396258", 2023-09-23T21:05:57.813 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 4288 2023-09-23T21:05:57.813 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:05:57.813 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:05:57.814 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:05:57.814 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:05:57.814 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:57.788022+0000", 2023-09-23T21:05:57.814 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:05:57.814 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:05:57.814 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:05:57.814 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:05:57.814 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:57.788022+0000", 2023-09-23T21:05:57.814 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:05:57.814 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:05:57.814 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:05:57.815 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:05:57.815 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:57.788021+0000", 2023-09-23T21:05:57.815 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.999999 2023-09-23T21:05:57.815 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:05:57.815 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:05:57.815 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:05:57.815 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:57.788045+0000", 2023-09-23T21:05:57.815 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.3893e-05 2023-09-23T21:05:57.815 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:05:57.815 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:05:57.815 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:05:57.816 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:57.788056+0000", 2023-09-23T21:05:57.816 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1268e-05 2023-09-23T21:05:57.816 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:05:57.816 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:05:57.816 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:05:57.816 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:57.788064+0000", 2023-09-23T21:05:57.816 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.7049999999999999e-06 2023-09-23T21:05:57.816 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:05:57.816 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:05:57.816 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:05:57.816 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:57.789378+0000", 2023-09-23T21:05:57.817 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0013138830000000001 2023-09-23T21:05:57.817 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:05:57.817 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:05:57.817 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:05:57.817 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:57.789489+0000", 2023-09-23T21:05:57.817 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000111357 2023-09-23T21:05:57.817 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:05:57.817 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:05:57.817 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 2,6", 2023-09-23T21:05:57.817 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:57.789616+0000", 2023-09-23T21:05:57.817 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00012651399999999999 2023-09-23T21:05:57.817 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:05:57.818 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:05:57.818 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T21:05:57.818 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:57.791884+0000", 2023-09-23T21:05:57.818 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0022685069999999999 2023-09-23T21:05:57.818 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:05:57.818 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:05:57.818 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:05:57.818 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:57.791943+0000", 2023-09-23T21:05:57.818 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.8894000000000002e-05 2023-09-23T21:05:57.819 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:05:57.819 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:05:57.819 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:05:57.819 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:05:57.819 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:05:57.819 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35856.0:4291 9.4 9:232a116e:::benchmark_data_smithi026_342732_object4290:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e351)", 2023-09-23T21:05:57.819 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T21:05:57.790725+0000", 2023-09-23T21:05:57.819 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 0.001115914, 2023-09-23T21:05:57.819 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0014866669999999999, 2023-09-23T21:05:57.819 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:05:57.820 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "waiting for sub ops", 2023-09-23T21:05:57.820 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T21:05:57.820 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35856", 2023-09-23T21:05:57.820 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/4180396258", 2023-09-23T21:05:57.820 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 4291 2023-09-23T21:05:57.820 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:05:57.820 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:05:57.820 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:05:57.820 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:05:57.820 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:57.790725+0000", 2023-09-23T21:05:57.821 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:05:57.821 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:05:57.821 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:05:57.821 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:05:57.821 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:57.790725+0000", 2023-09-23T21:05:57.821 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:05:57.821 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:05:57.821 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:05:57.821 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:05:57.821 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:57.790724+0000", 2023-09-23T21:05:57.821 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T21:05:57.821 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:05:57.822 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:05:57.822 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:05:57.822 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:57.790749+0000", 2023-09-23T21:05:57.822 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.4864000000000001e-05 2023-09-23T21:05:57.822 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:05:57.822 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:05:57.822 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:05:57.822 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:57.790761+0000", 2023-09-23T21:05:57.822 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1881e-05 2023-09-23T21:05:57.822 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:05:57.822 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:05:57.822 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:05:57.822 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:57.790769+0000", 2023-09-23T21:05:57.823 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.1499999999999999e-06 2023-09-23T21:05:57.823 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:05:57.823 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:05:57.823 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:05:57.823 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:57.791090+0000", 2023-09-23T21:05:57.823 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00032157799999999999 2023-09-23T21:05:57.823 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:05:57.823 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:05:57.823 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:05:57.823 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:57.791231+0000", 2023-09-23T21:05:57.823 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000140426 2023-09-23T21:05:57.823 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:05:57.823 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:05:57.824 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 3,4", 2023-09-23T21:05:57.824 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:57.791386+0000", 2023-09-23T21:05:57.824 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000155453 2023-09-23T21:05:57.824 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:05:57.824 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:05:57.824 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:05:57.824 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:05:57.824 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:05:57.824 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35856.0:4294 9.4 9:26e96e81:::benchmark_data_smithi026_342732_object4293:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e351)", 2023-09-23T21:05:57.824 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T21:05:57.790785+0000", 2023-09-23T21:05:57.824 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 0.0010561870000000001, 2023-09-23T21:05:57.824 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0015354990000000001, 2023-09-23T21:05:57.824 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:05:57.825 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "waiting for sub ops", 2023-09-23T21:05:57.825 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T21:05:57.825 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35856", 2023-09-23T21:05:57.825 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/4180396258", 2023-09-23T21:05:57.825 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 4294 2023-09-23T21:05:57.825 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:05:57.825 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:05:57.825 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:05:57.825 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:05:57.825 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:57.790785+0000", 2023-09-23T21:05:57.825 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:05:57.825 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:05:57.825 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:05:57.826 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:05:57.826 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:57.790785+0000", 2023-09-23T21:05:57.826 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:05:57.826 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:05:57.826 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:05:57.826 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:05:57.826 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:57.790785+0000", 2023-09-23T21:05:57.826 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T21:05:57.826 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:05:57.826 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:05:57.826 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:05:57.826 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:57.790807+0000", 2023-09-23T21:05:57.826 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.2651999999999999e-05 2023-09-23T21:05:57.827 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:05:57.827 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:05:57.827 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:05:57.827 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:57.790816+0000", 2023-09-23T21:05:57.827 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.6009999999999995e-06 2023-09-23T21:05:57.827 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:05:57.827 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:05:57.827 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:05:57.827 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:57.790823+0000", 2023-09-23T21:05:57.827 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.6660000000000002e-06 2023-09-23T21:05:57.827 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:05:57.827 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:05:57.827 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:05:57.828 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:57.791810+0000", 2023-09-23T21:05:57.828 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00098737999999999994 2023-09-23T21:05:57.828 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:05:57.828 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:05:57.828 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:05:57.828 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:57.791930+0000", 2023-09-23T21:05:57.828 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000120086 2023-09-23T21:05:57.828 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:05:57.828 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:05:57.828 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 3,4", 2023-09-23T21:05:57.828 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:57.792055+0000", 2023-09-23T21:05:57.828 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000124852 2023-09-23T21:05:57.828 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:05:57.829 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:05:57.829 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:05:57.829 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:05:57.829 INFO:teuthology.orchestra.run.smithi026.stdout: ], 2023-09-23T21:05:57.829 INFO:teuthology.orchestra.run.smithi026.stdout: "num_ops": 6 2023-09-23T21:05:57.829 INFO:teuthology.orchestra.run.smithi026.stdout:} 2023-09-23T21:05:57.841 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T21:05:57.845 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:05:57.845 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:05:57.946 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:05:57.985 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:05:58.033 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:05:58.039 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:05:58.039 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:05:58.140 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:05:58.180 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:05:58.202 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:05:58.225 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:05:58.229 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:05:58.230 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:05:58.242 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 2 16 5550 5534 172.705 149.688 0.00405176 0.00577804 2023-09-23T21:05:58.324 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:05:58 smithi070 ceph-mon[168282]: pgmap v895: 109 pgs: 7 peering, 4 active+undersized+degraded, 34 active+undersized, 64 active+clean; 652 KiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 4/75 objects degraded (5.333%) 2023-09-23T21:05:58.331 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:05:58.359 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:05:58.395 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:05:58.414 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:05:58.449 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:05:58 smithi026 ceph-mon[179688]: pgmap v895: 109 pgs: 7 peering, 4 active+undersized+degraded, 34 active+undersized, 64 active+clean; 652 KiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 4/75 objects degraded (5.333%) 2023-09-23T21:05:58.450 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:05:58 smithi026 ceph-mon[175151]: pgmap v895: 109 pgs: 7 peering, 4 active+undersized+degraded, 34 active+undersized, 64 active+clean; 652 KiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 4/75 objects degraded (5.333%) 2023-09-23T21:05:58.557 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:05:58.562 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:05:58.563 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:05:58.663 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:05:58.717 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:05:58.788 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:05:58.793 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:05:58.794 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:05:58.894 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:05:58.933 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:05:58.977 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:05:58.980 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:05:58.980 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:05:59.081 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:05:59.138 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:05:59.243 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 3 16 8291 8275 172.236 171.312 0.00969364 0.00579195 2023-09-23T21:05:59.407 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:05:59.414 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:05:59.414 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:05:59.515 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:05:59.565 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:05:59 smithi026 ceph-mon[175151]: 1.3 scrub starts 2023-09-23T21:05:59.565 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:05:59 smithi026 ceph-mon[175151]: 1.3 scrub ok 2023-09-23T21:05:59.565 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:05:59 smithi026 ceph-mon[179688]: 1.3 scrub starts 2023-09-23T21:05:59.565 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:05:59 smithi026 ceph-mon[179688]: 1.3 scrub ok 2023-09-23T21:05:59.573 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:05:59.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:05:59 smithi070 ceph-mon[168282]: 1.3 scrub starts 2023-09-23T21:05:59.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:05:59 smithi070 ceph-mon[168282]: 1.3 scrub ok 2023-09-23T21:05:59.636 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T21:05:59.640 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:05:59.640 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:05:59.741 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:05:59.815 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 21:05:59 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[164029]: ::ffff:172.21.15.70 - - [23/Sep/2023:21:05:59] "GET /metrics HTTP/1.1" 200 33961 "" "Prometheus/2.43.0" 2023-09-23T21:05:59.822 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:05:59.980 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:05:59.989 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:05:59.989 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:06:00.090 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:06:00.130 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:06:00.186 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:06:00.190 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:06:00.190 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:06:00.246 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 4 16 11017 11001 171.767 170.375 0.00701883 0.00581246 2023-09-23T21:06:00.291 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:06:00.333 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:06:00.377 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:06:00.381 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:06:00 smithi070 ceph-mon[168282]: pgmap v896: 109 pgs: 7 peering, 3 active+undersized, 99 active+clean; 164 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 27 MiB/s wr, 430 op/s 2023-09-23T21:06:00.381 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:06:00 smithi070 ceph-mon[168282]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 4/75 objects degraded (5.333%), 4 pgs degraded) 2023-09-23T21:06:00.382 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:06:00.383 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:06:00.483 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:06:00.524 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:06:00.579 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:06:00.583 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:06:00.583 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:06:00.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:06:00 smithi026 ceph-mon[179688]: pgmap v896: 109 pgs: 7 peering, 3 active+undersized, 99 active+clean; 164 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 27 MiB/s wr, 430 op/s 2023-09-23T21:06:00.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:06:00 smithi026 ceph-mon[179688]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 4/75 objects degraded (5.333%), 4 pgs degraded) 2023-09-23T21:06:00.633 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:06:00 smithi026 ceph-mon[175151]: pgmap v896: 109 pgs: 7 peering, 3 active+undersized, 99 active+clean; 164 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 27 MiB/s wr, 430 op/s 2023-09-23T21:06:00.633 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:06:00 smithi026 ceph-mon[175151]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 4/75 objects degraded (5.333%), 4 pgs degraded) 2023-09-23T21:06:00.684 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:06:00.740 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:06:01.243 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 5 16 13952 13936 174.097 183.438 0.00949669 0.00573622 2023-09-23T21:06:01.889 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0: {} 2023-09-23T21:06:01.915 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1: {} 2023-09-23T21:06:01.953 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2: {} 2023-09-23T21:06:01.995 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3: {} 2023-09-23T21:06:02.033 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4: {} 2023-09-23T21:06:02.076 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T21:06:02.113 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6: {} 2023-09-23T21:06:02.152 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T21:06:02.160 INFO:teuthology.orchestra.run.smithi026.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T21:06:02.160 INFO:teuthology.orchestra.run.smithi026.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T21:06:02.161 INFO:teuthology.orchestra.run.smithi026.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T21:06:02.161 INFO:teuthology.orchestra.run.smithi026.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T21:06:02.161 INFO:teuthology.orchestra.run.smithi026.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T21:06:02.161 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T21:06:02.161 INFO:teuthology.orchestra.run.smithi026.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T21:06:02.161 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T21:06:02.170 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:06:02.178 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:06:02.178 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:06:02.243 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 6 16 16363 16347 170.195 150.688 0.00318045 0.00586732 2023-09-23T21:06:02.279 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:06:02.316 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:06:02.367 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:06:02.372 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:06:02.372 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:06:02.473 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:06:02.512 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:06:02 smithi070 ceph-mon[168282]: pgmap v897: 109 pgs: 109 active+clean; 358 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 47 MiB/s wr, 755 op/s 2023-09-23T21:06:02.512 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:06:02 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:06:02.512 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:06:02 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:06:02.512 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:06:02 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/1449199232' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:06:02.515 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:06:02.566 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:06:02.569 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:06:02.570 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:06:02.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:06:02 smithi026 ceph-mon[175151]: pgmap v897: 109 pgs: 109 active+clean; 358 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 47 MiB/s wr, 755 op/s 2023-09-23T21:06:02.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:06:02 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:06:02.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:06:02 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:06:02.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:06:02 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/1449199232' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:06:02.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:06:02 smithi026 ceph-mon[179688]: pgmap v897: 109 pgs: 109 active+clean; 358 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 47 MiB/s wr, 755 op/s 2023-09-23T21:06:02.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:06:02 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:06:02.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:06:02 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:06:02.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:06:02 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/1449199232' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:06:02.671 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:06:02.721 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:06:03.169 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 6, 7, 4, 5] out_osds: [] dead_osds: [] live_osds: [0, 2, 3, 4, 5, 7, 6, 1] 2023-09-23T21:06:03.170 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T21:06:03.170 INFO:tasks.thrashosds.thrasher:inject_pause on osd.3 2023-09-23T21:06:03.170 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-09-23T21:06:03.170 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-09-23T21:06:03.170 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 config set filestore_inject_stall 3 2023-09-23T21:06:03.243 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 7 16 18692 18676 166.675 145.562 0.00186655 0.00597169 2023-09-23T21:06:03.284 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T21:06:03.286 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:06:03.287 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:06:03.388 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:06:03.458 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:06:03.802 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:06:03.807 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:06:03.807 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:06:03.908 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:06:03.954 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:06:04.002 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:06:04.007 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:06:04.007 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:06:04.108 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:06:04.152 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:06:04.203 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:06:04.207 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:06:04.207 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:06:04.243 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 8 16 20967 20951 163.614 142.188 0.00380898 0.00610599 2023-09-23T21:06:04.308 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:06:04.361 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:06:04.400 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:06:04.404 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:06:04.405 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:06:04.505 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:06:04.542 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:06:04 smithi070 ceph-mon[168282]: pgmap v898: 109 pgs: 109 active+clean; 775 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 91 MiB/s wr, 1.45k op/s 2023-09-23T21:06:04.542 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:06:04 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T21:06:04.546 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:06:04.600 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:06:04.603 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:06:04.604 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:06:04.634 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:06:04 smithi026 ceph-mon[175151]: pgmap v898: 109 pgs: 109 active+clean; 775 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 91 MiB/s wr, 1.45k op/s 2023-09-23T21:06:04.634 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:06:04 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T21:06:04.634 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:06:04 smithi026 ceph-mon[179688]: pgmap v898: 109 pgs: 109 active+clean; 775 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 91 MiB/s wr, 1.45k op/s 2023-09-23T21:06:04.634 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:06:04 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T21:06:04.704 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:06:04.752 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:06:04.813 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:06:04.815 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:06:04.816 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:06:04.916 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:06:04.980 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:06:05.030 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:06:05.034 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:06:05.035 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:06:05.135 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:06:05.192 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:06:05.243 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 9 16 23105 23089 160.281 133.625 0.0106757 0.00623049 2023-09-23T21:06:05.389 INFO:teuthology.orchestra.run.smithi026.stdout:{ 2023-09-23T21:06:05.390 INFO:teuthology.orchestra.run.smithi026.stdout: "ops": [], 2023-09-23T21:06:05.390 INFO:teuthology.orchestra.run.smithi026.stdout: "num_ops": 0 2023-09-23T21:06:05.390 INFO:teuthology.orchestra.run.smithi026.stdout:} 2023-09-23T21:06:06.229 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:06:06.242 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:06:06.243 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:06:06.245 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 10 16 25042 25026 156.34 121.062 0.00605637 0.00638908 2023-09-23T21:06:06.345 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:06:06.406 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:06:06.523 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:06:06 smithi026 ceph-mon[175151]: pgmap v899: 109 pgs: 109 active+clean; 930 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 93 MiB/s wr, 1.48k op/s 2023-09-23T21:06:06.527 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:06:06 smithi026 ceph-mon[179688]: pgmap v899: 109 pgs: 109 active+clean; 930 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 93 MiB/s wr, 1.48k op/s 2023-09-23T21:06:06.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:06:06 smithi070 ceph-mon[168282]: pgmap v899: 109 pgs: 109 active+clean; 930 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 93 MiB/s wr, 1.48k op/s 2023-09-23T21:06:07.245 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 11 16 27207 27191 154.428 135.312 0.0138903 0.00647028 2023-09-23T21:06:07.690 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T21:06:07.694 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:06:07.695 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:06:07.795 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:06:07.855 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:06:08.113 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:06:08.116 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:06:07 smithi070 ceph-mon[168282]: pgmap v900: 109 pgs: 109 active+clean; 1.1 GiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 102 MiB/s wr, 1.62k op/s 2023-09-23T21:06:08.116 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:06:07 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:06:08.117 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:06:08.118 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:06:08.131 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:06:07 smithi026 ceph-mon[175151]: pgmap v900: 109 pgs: 109 active+clean; 1.1 GiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 102 MiB/s wr, 1.62k op/s 2023-09-23T21:06:08.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:06:07 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:06:08.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:06:07 smithi026 ceph-mon[179688]: pgmap v900: 109 pgs: 109 active+clean; 1.1 GiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 102 MiB/s wr, 1.62k op/s 2023-09-23T21:06:08.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:06:07 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:06:08.218 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:06:08.236 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T21:06:08.247 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 12 16 29276 29260 152.309 129.312 0.00995548 0.00656083 2023-09-23T21:06:08.258 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:06:08.265 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_blocked_ops 2023-09-23T21:06:08.311 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:06:08.316 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:06:08.317 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:06:08.418 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:06:08.453 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:06:08.497 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:06:08.501 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:06:08.502 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:06:08.602 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:06:08.641 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:06:08.686 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:06:08.689 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:06:08.689 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:06:08.790 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:06:08.865 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:06:08.919 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:06:08.951 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:06:08.964 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:06:08.965 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:06:09.065 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:06:09.104 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:06:09.162 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:06:09.167 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:06:09.168 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:06:09.247 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 13 16 30719 30703 147.532 90.1875 0.00927885 0.00677233 2023-09-23T21:06:09.269 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:06:09.343 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:06:09.614 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 21:06:09 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[164029]: ::ffff:172.21.15.70 - - [23/Sep/2023:21:06:09] "GET /metrics HTTP/1.1" 200 36279 "" "Prometheus/2.43.0" 2023-09-23T21:06:09.932 INFO:teuthology.orchestra.run.smithi026.stderr:nodeep-scrub is unset 2023-09-23T21:06:10.247 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 14 16 32526 32510 145.061 112.938 0.00225379 0.00687203 2023-09-23T21:06:10.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:06:09 smithi070 ceph-mon[168282]: pgmap v901: 109 pgs: 109 active+clean; 1.6 GiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 136 MiB/s wr, 2.17k op/s 2023-09-23T21:06:10.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:06:09 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/2820163673' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:06:10.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:06:09 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:06:10.381 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:06:09 smithi026 ceph-mon[179688]: pgmap v901: 109 pgs: 109 active+clean; 1.6 GiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 136 MiB/s wr, 2.17k op/s 2023-09-23T21:06:10.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:06:09 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/2820163673' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:06:10.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:06:09 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:06:10.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:06:09 smithi026 ceph-mon[175151]: pgmap v901: 109 pgs: 109 active+clean; 1.6 GiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 136 MiB/s wr, 2.17k op/s 2023-09-23T21:06:10.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:06:09 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/2820163673' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:06:10.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:06:09 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:06:10.488 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:06:10.494 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:06:10.494 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:06:10.595 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:06:10.659 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:06:10.722 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:06:10.726 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:06:10.726 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:06:10.827 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:06:10.922 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:06:11.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:06:10 smithi026 ceph-mon[175151]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T21:06:11.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:06:10 smithi026 ceph-mon[175151]: Cluster is now healthy 2023-09-23T21:06:11.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:06:10 smithi026 ceph-mon[175151]: failed to encode map e352 with expected crc 2023-09-23T21:06:11.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:06:10 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T21:06:11.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:06:10 smithi026 ceph-mon[175151]: osdmap e352: 8 total, 8 up, 8 in 2023-09-23T21:06:11.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:06:10 smithi026 ceph-mon[175151]: failed to encode map e352 with expected crc 2023-09-23T21:06:11.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:06:10 smithi026 ceph-mon[175151]: failed to encode map e352 with expected crc 2023-09-23T21:06:11.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:06:10 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:06:11.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:06:10 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:06:11.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:06:10 smithi026 ceph-mon[179688]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T21:06:11.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:06:10 smithi026 ceph-mon[179688]: Cluster is now healthy 2023-09-23T21:06:11.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:06:10 smithi026 ceph-mon[179688]: failed to encode map e352 with expected crc 2023-09-23T21:06:11.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:06:10 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T21:06:11.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:06:10 smithi026 ceph-mon[179688]: osdmap e352: 8 total, 8 up, 8 in 2023-09-23T21:06:11.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:06:10 smithi026 ceph-mon[179688]: failed to encode map e352 with expected crc 2023-09-23T21:06:11.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:06:10 smithi026 ceph-mon[179688]: failed to encode map e352 with expected crc 2023-09-23T21:06:11.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:06:10 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:06:11.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:06:10 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:06:11.248 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 15 15 34338 34323 142.933 113.312 0.00423467 0.00699115 2023-09-23T21:06:11.364 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:06:11.370 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:06:11.370 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:06:11.371 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:06:10 smithi070 ceph-mon[168282]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T21:06:11.371 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:06:10 smithi070 ceph-mon[168282]: Cluster is now healthy 2023-09-23T21:06:11.371 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:06:10 smithi070 ceph-mon[168282]: failed to encode map e352 with expected crc 2023-09-23T21:06:11.371 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:06:10 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T21:06:11.371 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:06:10 smithi070 ceph-mon[168282]: osdmap e352: 8 total, 8 up, 8 in 2023-09-23T21:06:11.371 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:06:10 smithi070 ceph-mon[168282]: failed to encode map e352 with expected crc 2023-09-23T21:06:11.371 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:06:10 smithi070 ceph-mon[168282]: failed to encode map e352 with expected crc 2023-09-23T21:06:11.371 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:06:10 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:06:11.371 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:06:10 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:06:11.471 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:06:11.510 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:06:11.547 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:06:11.572 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:06:11.708 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:06:11.713 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:06:11.713 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:06:11.814 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:06:11.850 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:06:11.901 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:06:11.904 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:06:11.904 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:06:12.005 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:06:12.046 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:06:12.100 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:06:12.104 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:06:12.105 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:06:12.206 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:06:12.248 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 16 16 35913 35897 140.149 98.375 0.0101452 0.00712877 2023-09-23T21:06:12.256 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:06:12.300 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:06:12.302 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:06:12.303 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:06:12.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:06:12 smithi070 ceph-mon[168282]: pgmap v903: 109 pgs: 109 active+clean; 1.7 GiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 142 MiB/s wr, 2.27k op/s 2023-09-23T21:06:12.404 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:06:12.469 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:06:12 smithi026 ceph-mon[179688]: pgmap v903: 109 pgs: 109 active+clean; 1.7 GiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 142 MiB/s wr, 2.27k op/s 2023-09-23T21:06:12.469 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:06:12 smithi026 ceph-mon[175151]: pgmap v903: 109 pgs: 109 active+clean; 1.7 GiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 142 MiB/s wr, 2.27k op/s 2023-09-23T21:06:12.480 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:06:12.684 INFO:teuthology.orchestra.run.smithi026.stdout:{ 2023-09-23T21:06:12.684 INFO:teuthology.orchestra.run.smithi026.stdout: "success": "" 2023-09-23T21:06:12.684 INFO:teuthology.orchestra.run.smithi026.stdout:} 2023-09-23T21:06:12.873 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:06:12.881 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:06:12.881 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:06:12.982 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:06:13.050 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:06:13.253 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 17 16 37546 37530 137.874 102.062 0.00723 0.00724826 2023-09-23T21:06:13.325 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:06:13.337 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:06:13.338 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:06:13.432 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T21:06:13.438 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:06:13.557 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:06:13.777 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:06:13.796 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:06:13.796 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:06:13.897 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:06:13.937 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:06:13.987 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:06:13.991 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:06:13.992 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:06:14.092 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:06:14.185 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:06:14.253 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 18 16 38987 38971 135.218 90.0625 0.0074488 0.00738927 2023-09-23T21:06:14.305 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T21:06:14.313 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:06:14.314 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:06:14.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:06:14 smithi026 ceph-mon[175151]: pgmap v904: 109 pgs: 109 active+clean; 2.0 GiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 128 MiB/s wr, 2.04k op/s 2023-09-23T21:06:14.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:06:14 smithi026 ceph-mon[179688]: pgmap v904: 109 pgs: 109 active+clean; 2.0 GiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 128 MiB/s wr, 2.04k op/s 2023-09-23T21:06:14.415 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:06:14.478 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0: {} 2023-09-23T21:06:14.517 INFO:teuthology.orchestra.run.smithi026.stdout:{ 2023-09-23T21:06:14.517 INFO:teuthology.orchestra.run.smithi026.stdout: "ops": [], 2023-09-23T21:06:14.517 INFO:teuthology.orchestra.run.smithi026.stdout: "complaint_time": 30, 2023-09-23T21:06:14.517 INFO:teuthology.orchestra.run.smithi026.stdout: "num_blocked_ops": 0 2023-09-23T21:06:14.517 INFO:teuthology.orchestra.run.smithi026.stdout:} 2023-09-23T21:06:14.532 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1: {} 2023-09-23T21:06:14.538 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:06:14.581 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2: {} 2023-09-23T21:06:14.619 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T21:06:14.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:06:14 smithi070 ceph-mon[168282]: pgmap v904: 109 pgs: 109 active+clean; 2.0 GiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 128 MiB/s wr, 2.04k op/s 2023-09-23T21:06:14.624 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:06:14.624 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:06:14.636 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3: {} 2023-09-23T21:06:14.677 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4: {} 2023-09-23T21:06:14.716 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T21:06:14.725 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:06:14.752 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6: {} 2023-09-23T21:06:14.773 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:06:14.794 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T21:06:14.805 INFO:teuthology.orchestra.run.smithi026.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T21:06:14.805 INFO:teuthology.orchestra.run.smithi026.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T21:06:14.805 INFO:teuthology.orchestra.run.smithi026.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T21:06:14.805 INFO:teuthology.orchestra.run.smithi026.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T21:06:14.805 INFO:teuthology.orchestra.run.smithi026.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T21:06:14.805 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T21:06:14.805 INFO:teuthology.orchestra.run.smithi026.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T21:06:14.805 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T21:06:14.821 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:06:14.826 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:06:14.826 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:06:14.920 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:06:14.927 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:06:14.934 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_historic_ops 2023-09-23T21:06:14.962 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:06:15.010 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:06:15.016 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:06:15.016 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:06:15.117 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:06:15.154 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:06:15.198 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:06:15.201 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:06:15.201 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:06:15.253 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 19 16 40356 40340 132.605 85.5625 0.00606368 0.00753621 2023-09-23T21:06:15.302 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:06:15.337 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:06:15.380 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:06:15.384 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:06:15.384 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:06:15.427 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:06:15 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/1554378273' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:06:15.427 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:06:15 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/1554378273' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:06:15.485 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:06:15.523 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:06:15 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/1554378273' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:06:15.526 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:06:15.568 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:06:15.571 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:06:15.571 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:06:15.672 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:06:15.714 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:06:15.769 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:06:15.772 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:06:15.772 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:06:15.873 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:06:15.947 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:06:16.254 INFO:tasks.radosbench.radosbench.0.smithi026.stdout:2023-09-23T21:06:16.255084+0000 min lat: 0.00137237 max lat: 0.0872851 avg lat: 0.00765185 2023-09-23T21:06:16.254 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T21:06:16.254 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 20 16 41836 41820 130.595 92.5 0.0101874 0.00765185 2023-09-23T21:06:16.287 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:06:16.289 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:06:16.292 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:06:16.292 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:06:16.393 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:06:16.431 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:06:16.474 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:06:16.477 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:06:16.477 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:06:16.578 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:06:16.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:06:16 smithi070 ceph-mon[168282]: pgmap v905: 109 pgs: 109 active+clean; 2.3 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 138 MiB/s wr, 2.21k op/s 2023-09-23T21:06:16.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:06:16 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:06:16.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:06:16 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:06:16.624 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:06:16.633 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:06:16 smithi026 ceph-mon[179688]: pgmap v905: 109 pgs: 109 active+clean; 2.3 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 138 MiB/s wr, 2.21k op/s 2023-09-23T21:06:16.633 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:06:16 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:06:16.633 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:06:16 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:06:16.634 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:06:16 smithi026 ceph-mon[175151]: pgmap v905: 109 pgs: 109 active+clean; 2.3 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 138 MiB/s wr, 2.21k op/s 2023-09-23T21:06:16.634 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:06:16 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:06:16.634 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:06:16 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:06:16.677 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:06:16.680 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:06:16.681 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:06:16.782 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:06:16.822 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:06:16.880 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:06:16.884 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:06:16.885 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:06:16.986 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:06:17.031 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:06:17.075 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:06:17.080 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:06:17.080 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:06:17.181 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:06:17.258 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 21 16 43325 43309 128.788 93.0625 0.01284 0.00775918 2023-09-23T21:06:17.260 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:06:17.614 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:06:17.623 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:06:17.624 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:06:17.653 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T21:06:17.726 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:06:17.860 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:06:18.008 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:06:18.018 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:06:18.019 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:06:18.119 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:06:18.217 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:06:18.262 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 22 16 44619 44603 126.59 80.875 0.01458 0.00789207 2023-09-23T21:06:18.289 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:06:18.296 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:06:18.297 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:06:18.338 INFO:teuthology.orchestra.run.smithi026.stderr:noscrub is set 2023-09-23T21:06:18.397 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:06:18.433 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:06:18.474 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:06:18.479 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:06:18.480 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:06:18.558 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:06:18 smithi026 ceph-mon[175151]: pgmap v906: 109 pgs: 109 active+clean; 2.3 GiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 114 MiB/s wr, 1.82k op/s 2023-09-23T21:06:18.558 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:06:18 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/3553931328' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T21:06:18.558 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:06:18 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T21:06:18.558 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:06:18 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:06:18.558 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:06:18 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:06:18.558 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:06:18 smithi026 ceph-mon[179688]: pgmap v906: 109 pgs: 109 active+clean; 2.3 GiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 114 MiB/s wr, 1.82k op/s 2023-09-23T21:06:18.559 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:06:18 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/3553931328' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T21:06:18.559 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:06:18 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T21:06:18.559 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:06:18 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:06:18.559 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:06:18 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:06:18.580 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:06:18.613 INFO:teuthology.orchestra.run.smithi026.stdout:ERROR: (22) Invalid argument 2023-09-23T21:06:18.613 INFO:teuthology.orchestra.run.smithi026.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T21:06:18.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:06:18 smithi070 ceph-mon[168282]: pgmap v906: 109 pgs: 109 active+clean; 2.3 GiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 114 MiB/s wr, 1.82k op/s 2023-09-23T21:06:18.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:06:18 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/3553931328' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T21:06:18.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:06:18 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T21:06:18.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:06:18 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:06:18.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:06:18 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:06:18.668 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:06:18.798 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:06:18.804 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:06:18.805 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:06:18.824 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 6, 7, 4, 5] out_osds: [] dead_osds: [] live_osds: [0, 2, 3, 4, 5, 7, 6, 1] 2023-09-23T21:06:18.824 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T21:06:18.824 INFO:tasks.thrashosds.thrasher:primary_affinity 2023-09-23T21:06:18.824 INFO:tasks.thrashosds.thrasher:Setting osd 7 primary_affinity to 0.448360 2023-09-23T21:06:18.824 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph osd primary-affinity 7 0.44836012165793127 2023-09-23T21:06:18.906 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:06:18.997 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:06:19.199 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:06:19.206 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:06:19.206 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:06:19.261 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 23 16 45852 45836 124.438 77.0625 0.0139226 0.00802563 2023-09-23T21:06:19.285 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_ops_in_flight 2023-09-23T21:06:19.307 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:06:19.331 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:06:19 smithi026 ceph-mon[175151]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:06:19.332 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:06:19 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T21:06:19.332 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:06:19 smithi026 ceph-mon[175151]: failed to encode map e353 with expected crc 2023-09-23T21:06:19.332 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:06:19 smithi026 ceph-mon[175151]: osdmap e353: 8 total, 8 up, 8 in 2023-09-23T21:06:19.332 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:06:19 smithi026 ceph-mon[175151]: failed to encode map e353 with expected crc 2023-09-23T21:06:19.332 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:06:19 smithi026 ceph-mon[175151]: failed to encode map e353 with expected crc 2023-09-23T21:06:19.332 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:06:19 smithi026 ceph-mon[175151]: failed to encode map e353 with expected crc 2023-09-23T21:06:19.332 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:06:19 smithi026 ceph-mon[175151]: failed to encode map e353 with expected crc 2023-09-23T21:06:19.332 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:06:19 smithi026 ceph-mon[175151]: failed to encode map e353 with expected crc 2023-09-23T21:06:19.332 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:06:19 smithi026 ceph-mon[175151]: failed to encode map e353 with expected crc 2023-09-23T21:06:19.364 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:06:19.433 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:06:19.528 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:06:19.531 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:06:19.532 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:06:19.612 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:06:19 smithi026 ceph-mon[179688]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:06:19.612 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:06:19 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T21:06:19.612 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:06:19 smithi026 ceph-mon[179688]: failed to encode map e353 with expected crc 2023-09-23T21:06:19.612 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:06:19 smithi026 ceph-mon[179688]: osdmap e353: 8 total, 8 up, 8 in 2023-09-23T21:06:19.613 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:06:19 smithi026 ceph-mon[179688]: failed to encode map e353 with expected crc 2023-09-23T21:06:19.613 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:06:19 smithi026 ceph-mon[179688]: failed to encode map e353 with expected crc 2023-09-23T21:06:19.613 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:06:19 smithi026 ceph-mon[179688]: failed to encode map e353 with expected crc 2023-09-23T21:06:19.613 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:06:19 smithi026 ceph-mon[179688]: failed to encode map e353 with expected crc 2023-09-23T21:06:19.613 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:06:19 smithi026 ceph-mon[179688]: failed to encode map e353 with expected crc 2023-09-23T21:06:19.613 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:06:19 smithi026 ceph-mon[179688]: failed to encode map e353 with expected crc 2023-09-23T21:06:19.621 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:06:19 smithi070 ceph-mon[168282]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:06:19.621 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:06:19 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T21:06:19.621 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:06:19 smithi070 ceph-mon[168282]: failed to encode map e353 with expected crc 2023-09-23T21:06:19.621 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:06:19 smithi070 ceph-mon[168282]: osdmap e353: 8 total, 8 up, 8 in 2023-09-23T21:06:19.621 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:06:19 smithi070 ceph-mon[168282]: failed to encode map e353 with expected crc 2023-09-23T21:06:19.621 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:06:19 smithi070 ceph-mon[168282]: failed to encode map e353 with expected crc 2023-09-23T21:06:19.622 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:06:19 smithi070 ceph-mon[168282]: failed to encode map e353 with expected crc 2023-09-23T21:06:19.622 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:06:19 smithi070 ceph-mon[168282]: failed to encode map e353 with expected crc 2023-09-23T21:06:19.622 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:06:19 smithi070 ceph-mon[168282]: failed to encode map e353 with expected crc 2023-09-23T21:06:19.622 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:06:19 smithi070 ceph-mon[168282]: failed to encode map e353 with expected crc 2023-09-23T21:06:19.633 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:06:19.718 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:06:19.784 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:06:19.790 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:06:19.790 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:06:19.881 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 21:06:19 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[164029]: ::ffff:172.21.15.70 - - [23/Sep/2023:21:06:19] "GET /metrics HTTP/1.1" 200 36286 "" "Prometheus/2.43.0" 2023-09-23T21:06:19.891 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:06:19.930 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:06:19.964 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:06:19.988 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:06:19.991 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:06:19.992 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:06:20.092 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:06:20.175 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:06:20.261 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 24 16 47120 47104 122.557 79.25 0.00922581 0.00815377 2023-09-23T21:06:20.565 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:06:20 smithi026 ceph-mon[179688]: pgmap v908: 109 pgs: 109 active+clean; 2.6 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 95 MiB/s wr, 1.52k op/s 2023-09-23T21:06:20.566 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:06:20 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T21:06:20.566 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:06:20 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T21:06:20.566 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:06:20 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:06:20.566 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:06:20 smithi026 ceph-mon[175151]: pgmap v908: 109 pgs: 109 active+clean; 2.6 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 95 MiB/s wr, 1.52k op/s 2023-09-23T21:06:20.566 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:06:20 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T21:06:20.567 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:06:20 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T21:06:20.567 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:06:20 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:06:20.579 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:06:20.585 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:06:20.585 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:06:20.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:06:20 smithi070 ceph-mon[168282]: pgmap v908: 109 pgs: 109 active+clean; 2.6 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 95 MiB/s wr, 1.52k op/s 2023-09-23T21:06:20.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:06:20 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T21:06:20.621 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:06:20 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T21:06:20.621 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:06:20 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:06:20.686 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:06:20.723 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:06:20.727 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:06:20.770 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:06:20.774 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:06:20.774 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:06:20.875 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:06:20.937 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:06:21.074 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T21:06:21.217 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T21:06:21.230 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:06:21.230 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:06:21.263 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 25 16 48587 48571 121.323 91.6875 0.0114015 0.00823692 2023-09-23T21:06:21.331 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:06:21.363 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:06:21.405 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:06:21.409 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:06:21.410 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:06:21.511 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:06:21.521 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0: {} 2023-09-23T21:06:21.563 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1: {} 2023-09-23T21:06:21.582 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:06:21.605 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2: {} 2023-09-23T21:06:21.649 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3: {} 2023-09-23T21:06:21.688 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4: {} 2023-09-23T21:06:21.723 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T21:06:21.758 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6: {} 2023-09-23T21:06:21.797 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T21:06:21.806 INFO:teuthology.orchestra.run.smithi026.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T21:06:21.806 INFO:teuthology.orchestra.run.smithi026.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T21:06:21.806 INFO:teuthology.orchestra.run.smithi026.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T21:06:21.806 INFO:teuthology.orchestra.run.smithi026.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T21:06:21.806 INFO:teuthology.orchestra.run.smithi026.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T21:06:21.806 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T21:06:21.806 INFO:teuthology.orchestra.run.smithi026.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T21:06:21.806 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T21:06:22.262 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 26 16 49925 49909 119.873 83.625 0.0097423 0.00833723 2023-09-23T21:06:22.570 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:06:22 smithi026 ceph-mon[175151]: pgmap v909: 109 pgs: 109 active+clean; 2.7 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 98 MiB/s wr, 1.57k op/s 2023-09-23T21:06:22.570 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:06:22 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/1234993457' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:06:22.570 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:06:22 smithi026 ceph-mon[179688]: pgmap v909: 109 pgs: 109 active+clean; 2.7 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 98 MiB/s wr, 1.57k op/s 2023-09-23T21:06:22.570 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:06:22 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/1234993457' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:06:22.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:06:22 smithi070 ceph-mon[168282]: pgmap v909: 109 pgs: 109 active+clean; 2.7 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 98 MiB/s wr, 1.57k op/s 2023-09-23T21:06:22.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:06:22 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/1234993457' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:06:23.027 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:06:23.032 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:06:23.033 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:06:23.134 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:06:23.211 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:06:23.268 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 27 16 51287 51271 118.561 85.125 0.0239311 0.00842583 2023-09-23T21:06:23.580 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T21:06:23.586 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:06:23.586 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:06:23.589 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:06:23.688 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:06:23.741 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:06:23.800 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:06:23.804 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:06:23.804 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:06:23.905 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:06:23.942 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:06:23.992 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:06:23.996 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:06:23.996 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:06:24.097 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:06:24.201 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:06:24.268 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 28 16 52609 52593 117.279 82.625 0.00518272 0.00852037 2023-09-23T21:06:24.387 INFO:teuthology.orchestra.run.smithi026.stdout:{ 2023-09-23T21:06:24.387 INFO:teuthology.orchestra.run.smithi026.stdout: "ops": [ 2023-09-23T21:06:24.388 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:06:24.388 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_repop(client.35856.0:52724 9.4 e353/350 9:2b12b4f4:::benchmark_data_smithi026_342732_object52723:head v 353'3271, mlcod=353'3268)", 2023-09-23T21:06:24.388 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T21:06:24.358612+0000", 2023-09-23T21:06:24.388 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 0.0024097709999999998, 2023-09-23T21:06:24.388 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00243954, 2023-09-23T21:06:24.388 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:06:24.388 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "started", 2023-09-23T21:06:24.388 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:06:24.388 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:06:24.388 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:06:24.388 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:24.358612+0000", 2023-09-23T21:06:24.388 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:06:24.389 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:06:24.389 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:06:24.389 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:06:24.389 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:24.358612+0000", 2023-09-23T21:06:24.389 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:06:24.389 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:06:24.389 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:06:24.389 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:06:24.389 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:24.358612+0000", 2023-09-23T21:06:24.389 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.999999 2023-09-23T21:06:24.389 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:06:24.389 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:06:24.389 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:06:24.390 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:24.358634+0000", 2023-09-23T21:06:24.390 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.2472000000000001e-05 2023-09-23T21:06:24.390 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:06:24.390 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:06:24.390 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:06:24.390 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:24.358650+0000", 2023-09-23T21:06:24.390 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.6116999999999998e-05 2023-09-23T21:06:24.390 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:06:24.390 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:06:24.390 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:06:24.390 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:24.358658+0000", 2023-09-23T21:06:24.390 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.2880000000000004e-06 2023-09-23T21:06:24.391 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:06:24.391 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:06:24.391 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:06:24.391 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:24.358726+0000", 2023-09-23T21:06:24.391 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.8291000000000001e-05 2023-09-23T21:06:24.391 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:06:24.391 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:06:24.391 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:06:24.391 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:24.358744+0000", 2023-09-23T21:06:24.391 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.8658e-05 2023-09-23T21:06:24.391 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:06:24.391 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:06:24.391 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:06:24.392 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:06:24.392 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:06:24.392 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35856.0:52737 9.8 9:183f499d:::benchmark_data_smithi026_342732_object52736:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e353)", 2023-09-23T21:06:24.392 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T21:06:24.360111+0000", 2023-09-23T21:06:24.392 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 0.00091097200000000002, 2023-09-23T21:06:24.392 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.001026098, 2023-09-23T21:06:24.392 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:06:24.392 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "waiting for sub ops", 2023-09-23T21:06:24.392 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T21:06:24.392 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35856", 2023-09-23T21:06:24.392 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/4180396258", 2023-09-23T21:06:24.392 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 52737 2023-09-23T21:06:24.393 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:06:24.393 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:06:24.393 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:06:24.393 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:06:24.393 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:24.360111+0000", 2023-09-23T21:06:24.393 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:06:24.393 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:06:24.393 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:06:24.393 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:06:24.393 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:24.360111+0000", 2023-09-23T21:06:24.393 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:06:24.393 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:06:24.393 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:06:24.394 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:06:24.394 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:24.360109+0000", 2023-09-23T21:06:24.394 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999976 2023-09-23T21:06:24.394 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:06:24.394 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:06:24.394 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:06:24.394 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:24.360141+0000", 2023-09-23T21:06:24.394 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.1687999999999999e-05 2023-09-23T21:06:24.394 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:06:24.394 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:06:24.394 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:06:24.394 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:24.360160+0000", 2023-09-23T21:06:24.395 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.9293999999999999e-05 2023-09-23T21:06:24.395 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:06:24.395 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:06:24.395 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:06:24.395 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:24.360170+0000", 2023-09-23T21:06:24.395 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.8700000000000004e-06 2023-09-23T21:06:24.395 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:06:24.395 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:06:24.395 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:06:24.395 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:24.360587+0000", 2023-09-23T21:06:24.395 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00041711899999999998 2023-09-23T21:06:24.395 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:06:24.395 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:06:24.396 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:06:24.396 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:24.360747+0000", 2023-09-23T21:06:24.396 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00016001000000000001 2023-09-23T21:06:24.396 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:06:24.396 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:06:24.396 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 0,5", 2023-09-23T21:06:24.396 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:24.360894+0000", 2023-09-23T21:06:24.396 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00014669999999999999 2023-09-23T21:06:24.396 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:06:24.396 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:06:24.396 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:06:24.396 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:06:24.397 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:06:24.397 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35856.0:52738 9.1 9.7527f9f1 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e353)", 2023-09-23T21:06:24.397 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T21:06:24.360187+0000", 2023-09-23T21:06:24.397 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 0.00083540999999999997, 2023-09-23T21:06:24.397 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0010287639999999999, 2023-09-23T21:06:24.397 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:06:24.397 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "queued for pg", 2023-09-23T21:06:24.397 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T21:06:24.397 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35856", 2023-09-23T21:06:24.397 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/4180396258", 2023-09-23T21:06:24.397 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 52738 2023-09-23T21:06:24.397 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:06:24.397 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:06:24.398 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:06:24.398 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:06:24.398 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:24.360187+0000", 2023-09-23T21:06:24.398 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:06:24.398 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:06:24.398 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:06:24.398 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:06:24.398 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:24.360187+0000", 2023-09-23T21:06:24.398 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:06:24.398 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:06:24.398 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:06:24.398 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:06:24.398 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:24.360186+0000", 2023-09-23T21:06:24.399 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T21:06:24.399 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:06:24.399 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:06:24.399 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:06:24.399 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:24.360211+0000", 2023-09-23T21:06:24.399 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.4490000000000001e-05 2023-09-23T21:06:24.399 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:06:24.399 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:06:24.399 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:06:24.399 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:24.360221+0000", 2023-09-23T21:06:24.399 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0397999999999999e-05 2023-09-23T21:06:24.399 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:06:24.399 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:06:24.400 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:06:24.400 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:24.360228+0000", 2023-09-23T21:06:24.400 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.5030000000000002e-06 2023-09-23T21:06:24.400 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:06:24.400 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:06:24.400 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:06:24.400 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:06:24.400 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:06:24.400 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35856.0:52739 9.8 9.8b26f668 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e353)", 2023-09-23T21:06:24.400 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T21:06:24.360240+0000", 2023-09-23T21:06:24.400 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 0.00078173299999999995, 2023-09-23T21:06:24.400 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.001031805, 2023-09-23T21:06:24.401 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:06:24.401 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "queued for pg", 2023-09-23T21:06:24.401 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T21:06:24.401 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35856", 2023-09-23T21:06:24.401 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/4180396258", 2023-09-23T21:06:24.401 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 52739 2023-09-23T21:06:24.401 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:06:24.401 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:06:24.401 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:06:24.401 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:06:24.401 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:24.360240+0000", 2023-09-23T21:06:24.401 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:06:24.401 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:06:24.402 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:06:24.402 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:06:24.402 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:24.360240+0000", 2023-09-23T21:06:24.402 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:06:24.402 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:06:24.402 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:06:24.402 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:06:24.402 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:24.360240+0000", 2023-09-23T21:06:24.402 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T21:06:24.402 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:06:24.402 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:06:24.402 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:06:24.403 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:24.360265+0000", 2023-09-23T21:06:24.403 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.5167000000000001e-05 2023-09-23T21:06:24.403 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:06:24.403 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:06:24.403 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:06:24.403 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:24.360278+0000", 2023-09-23T21:06:24.403 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.2459999999999999e-05 2023-09-23T21:06:24.403 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:06:24.403 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:06:24.403 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:06:24.403 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:24.360301+0000", 2023-09-23T21:06:24.403 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.374e-05 2023-09-23T21:06:24.404 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:06:24.404 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:06:24.404 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:06:24.404 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:06:24.404 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:06:24.404 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35856.0:52740 9.8 9.7a1c97b8 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e353)", 2023-09-23T21:06:24.404 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T21:06:24.360308+0000", 2023-09-23T21:06:24.404 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 0.00071458999999999997, 2023-09-23T21:06:24.404 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0010206130000000001, 2023-09-23T21:06:24.404 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:06:24.404 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "queued for pg", 2023-09-23T21:06:24.404 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T21:06:24.404 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35856", 2023-09-23T21:06:24.405 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/4180396258", 2023-09-23T21:06:24.405 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 52740 2023-09-23T21:06:24.405 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:06:24.405 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:06:24.405 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:06:24.405 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:06:24.405 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:24.360308+0000", 2023-09-23T21:06:24.405 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:06:24.405 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:06:24.405 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:06:24.405 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:06:24.405 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:24.360308+0000", 2023-09-23T21:06:24.405 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:06:24.406 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:06:24.406 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:06:24.406 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:06:24.406 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:24.360307+0000", 2023-09-23T21:06:24.406 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T21:06:24.406 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:06:24.406 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:06:24.406 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:06:24.406 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:24.360375+0000", 2023-09-23T21:06:24.406 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.7696999999999999e-05 2023-09-23T21:06:24.406 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:06:24.406 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:06:24.407 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:06:24.407 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:24.360386+0000", 2023-09-23T21:06:24.407 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1567e-05 2023-09-23T21:06:24.407 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:06:24.407 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:06:24.407 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:06:24.407 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:24.360393+0000", 2023-09-23T21:06:24.407 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.9260000000000003e-06 2023-09-23T21:06:24.407 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:06:24.407 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:06:24.407 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:06:24.407 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:06:24.408 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:06:24.408 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35856.0:52741 9.3 9:cf8c5318:::benchmark_data_smithi026_342732_object52740:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e353)", 2023-09-23T21:06:24.408 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T21:06:24.360399+0000", 2023-09-23T21:06:24.408 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 0.00062306199999999996, 2023-09-23T21:06:24.408 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00099163700000000007, 2023-09-23T21:06:24.408 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:06:24.408 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "waiting for sub ops", 2023-09-23T21:06:24.408 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T21:06:24.408 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35856", 2023-09-23T21:06:24.408 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/4180396258", 2023-09-23T21:06:24.408 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 52741 2023-09-23T21:06:24.408 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:06:24.408 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:06:24.409 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:06:24.409 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:06:24.409 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:24.360399+0000", 2023-09-23T21:06:24.409 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:06:24.409 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:06:24.409 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:06:24.409 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:06:24.409 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:24.360399+0000", 2023-09-23T21:06:24.409 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:06:24.409 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:06:24.409 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:06:24.409 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:06:24.410 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:24.360399+0000", 2023-09-23T21:06:24.410 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T21:06:24.410 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:06:24.410 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:06:24.410 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:06:24.410 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:24.360421+0000", 2023-09-23T21:06:24.410 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.2378000000000002e-05 2023-09-23T21:06:24.410 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:06:24.410 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:06:24.410 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:06:24.410 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:24.360431+0000", 2023-09-23T21:06:24.410 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.713e-06 2023-09-23T21:06:24.411 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:06:24.411 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:06:24.411 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:06:24.411 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:24.360437+0000", 2023-09-23T21:06:24.411 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.0059999999999998e-06 2023-09-23T21:06:24.411 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:06:24.411 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:06:24.411 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:06:24.411 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:24.360497+0000", 2023-09-23T21:06:24.411 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.0341999999999999e-05 2023-09-23T21:06:24.411 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:06:24.411 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:06:24.411 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:06:24.412 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:24.360662+0000", 2023-09-23T21:06:24.413 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000164964 2023-09-23T21:06:24.413 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:06:24.413 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:06:24.413 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 1,7", 2023-09-23T21:06:24.413 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:24.360814+0000", 2023-09-23T21:06:24.413 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000152332 2023-09-23T21:06:24.413 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:06:24.414 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:06:24.414 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:06:24.414 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:06:24.414 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:06:24.414 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35856.0:52727 9.c 9:34e7013c:::benchmark_data_smithi026_342732_object52726:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e353)", 2023-09-23T21:06:24.414 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T21:06:24.351306+0000", 2023-09-23T21:06:24.414 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 0.0097158890000000001, 2023-09-23T21:06:24.414 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.010163521999999999, 2023-09-23T21:06:24.414 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:06:24.414 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "waiting for sub ops", 2023-09-23T21:06:24.414 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T21:06:24.415 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35856", 2023-09-23T21:06:24.415 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/4180396258", 2023-09-23T21:06:24.415 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 52727 2023-09-23T21:06:24.415 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:06:24.415 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:06:24.415 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:06:24.415 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:06:24.415 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:24.351306+0000", 2023-09-23T21:06:24.415 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:06:24.415 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:06:24.415 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:06:24.415 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:06:24.415 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:24.351306+0000", 2023-09-23T21:06:24.416 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:06:24.416 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:06:24.416 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:06:24.416 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:06:24.416 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:24.351306+0000", 2023-09-23T21:06:24.416 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.999999 2023-09-23T21:06:24.416 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:06:24.416 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:06:24.416 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:06:24.416 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:24.351338+0000", 2023-09-23T21:06:24.416 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.2650000000000001e-05 2023-09-23T21:06:24.416 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:06:24.417 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:06:24.417 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:06:24.417 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:24.351353+0000", 2023-09-23T21:06:24.417 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.4977000000000001e-05 2023-09-23T21:06:24.417 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:06:24.417 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:06:24.417 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:06:24.417 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:24.351361+0000", 2023-09-23T21:06:24.417 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.4580000000000004e-06 2023-09-23T21:06:24.417 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:06:24.417 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:06:24.417 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:06:24.417 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:24.355764+0000", 2023-09-23T21:06:24.418 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0044035979999999999 2023-09-23T21:06:24.418 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:06:24.418 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:06:24.418 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:06:24.418 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:24.355911+0000", 2023-09-23T21:06:24.418 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000146477 2023-09-23T21:06:24.418 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:06:24.418 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:06:24.418 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 4,7", 2023-09-23T21:06:24.418 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:24.356057+0000", 2023-09-23T21:06:24.418 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000146138 2023-09-23T21:06:24.418 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:06:24.418 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:06:24.419 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T21:06:24.419 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:24.358099+0000", 2023-09-23T21:06:24.419 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0020425109999999999 2023-09-23T21:06:24.419 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:06:24.419 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:06:24.419 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:06:24.419 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:06:24.419 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:06:24.419 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35856.0:52729 9.3 9:c453b5ce:::benchmark_data_smithi026_342732_object52728:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e353)", 2023-09-23T21:06:24.419 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T21:06:24.351369+0000", 2023-09-23T21:06:24.419 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 0.0096527609999999993, 2023-09-23T21:06:24.419 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.010206463000000001, 2023-09-23T21:06:24.420 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:06:24.420 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "waiting for sub ops", 2023-09-23T21:06:24.420 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T21:06:24.420 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35856", 2023-09-23T21:06:24.420 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/4180396258", 2023-09-23T21:06:24.420 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 52729 2023-09-23T21:06:24.420 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:06:24.420 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:06:24.420 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:06:24.420 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:06:24.420 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:24.351369+0000", 2023-09-23T21:06:24.420 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:06:24.420 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:06:24.421 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:06:24.421 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:06:24.421 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:24.351369+0000", 2023-09-23T21:06:24.421 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:06:24.421 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:06:24.421 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:06:24.421 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:06:24.421 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:24.351369+0000", 2023-09-23T21:06:24.421 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T21:06:24.421 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:06:24.421 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:06:24.421 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:06:24.421 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:24.351392+0000", 2023-09-23T21:06:24.422 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.3e-05 2023-09-23T21:06:24.422 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:06:24.422 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:06:24.422 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:06:24.422 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:24.351399+0000", 2023-09-23T21:06:24.422 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.593e-06 2023-09-23T21:06:24.422 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:06:24.422 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:06:24.422 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:06:24.422 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:24.351405+0000", 2023-09-23T21:06:24.422 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.4689999999999996e-06 2023-09-23T21:06:24.422 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:06:24.423 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:06:24.423 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:06:24.423 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:24.351448+0000", 2023-09-23T21:06:24.423 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.2697999999999997e-05 2023-09-23T21:06:24.423 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:06:24.423 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:06:24.423 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:06:24.423 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:24.351616+0000", 2023-09-23T21:06:24.423 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000168899 2023-09-23T21:06:24.423 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:06:24.423 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:06:24.423 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 1,7", 2023-09-23T21:06:24.424 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:24.351740+0000", 2023-09-23T21:06:24.424 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00012331399999999999 2023-09-23T21:06:24.424 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:06:24.424 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:06:24.424 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:06:24.424 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:24.355611+0000", 2023-09-23T21:06:24.424 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0038716390000000001 2023-09-23T21:06:24.424 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:06:24.424 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:06:24.424 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T21:06:24.424 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:24.357150+0000", 2023-09-23T21:06:24.424 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0015381150000000001 2023-09-23T21:06:24.425 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:06:24.425 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:06:24.425 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:06:24.425 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:06:24.425 INFO:teuthology.orchestra.run.smithi026.stdout: ], 2023-09-23T21:06:24.425 INFO:teuthology.orchestra.run.smithi026.stdout: "num_ops": 8 2023-09-23T21:06:24.425 INFO:teuthology.orchestra.run.smithi026.stdout:} 2023-09-23T21:06:24.425 INFO:teuthology.orchestra.run.smithi026.stderr:set osd.7 primary-affinity to 0.44836 (8293832) 2023-09-23T21:06:24.501 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:06:24 smithi026 ceph-mon[175151]: pgmap v910: 109 pgs: 109 active+clean; 2.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 91 MiB/s wr, 1.46k op/s 2023-09-23T21:06:24.501 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:06:24 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/1587395234' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 7, "weight": 0.44836012165793127}]: dispatch 2023-09-23T21:06:24.501 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:06:24 smithi026 ceph-mon[179688]: pgmap v910: 109 pgs: 109 active+clean; 2.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 91 MiB/s wr, 1.46k op/s 2023-09-23T21:06:24.501 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:06:24 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/1587395234' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 7, "weight": 0.44836012165793127}]: dispatch 2023-09-23T21:06:24.639 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:06:24.642 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:06:24.643 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:06:24.744 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:06:24.767 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:06:24 smithi070 ceph-mon[168282]: pgmap v910: 109 pgs: 109 active+clean; 2.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 91 MiB/s wr, 1.46k op/s 2023-09-23T21:06:24.767 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:06:24 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/1587395234' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 7, "weight": 0.44836012165793127}]: dispatch 2023-09-23T21:06:24.769 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:06:24.803 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:06:24.864 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:06:24.864 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:06:24.965 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:06:24.987 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_blocked_ops 2023-09-23T21:06:24.998 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:06:25.185 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T21:06:25.188 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:06:25.189 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:06:25.218 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T21:06:25.268 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 29 16 53017 53001 114.117 25.5 0.00251323 0.00852526 2023-09-23T21:06:25.289 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:06:25.320 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:06:25.416 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T21:06:25.420 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:06:25 smithi026 ceph-mon[179688]: failed to encode map e354 with expected crc 2023-09-23T21:06:25.420 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:06:25 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/1587395234' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 7, "weight": 0.44836012165793127}]': finished 2023-09-23T21:06:25.420 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:06:25 smithi026 ceph-mon[179688]: osdmap e354: 8 total, 8 up, 8 in 2023-09-23T21:06:25.420 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:06:25 smithi026 ceph-mon[179688]: failed to encode map e354 with expected crc 2023-09-23T21:06:25.420 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:06:25.421 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:06:25 smithi026 ceph-mon[175151]: failed to encode map e354 with expected crc 2023-09-23T21:06:25.421 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:06:25 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/1587395234' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 7, "weight": 0.44836012165793127}]': finished 2023-09-23T21:06:25.421 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:06:25 smithi026 ceph-mon[175151]: osdmap e354: 8 total, 8 up, 8 in 2023-09-23T21:06:25.421 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:06:25 smithi026 ceph-mon[175151]: failed to encode map e354 with expected crc 2023-09-23T21:06:25.421 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:06:25.522 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:06:25.559 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:06:25.613 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:06:25.617 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:06:25.617 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:06:25.682 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:06:25.718 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:06:25.755 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:06:25 smithi070 ceph-mon[168282]: failed to encode map e354 with expected crc 2023-09-23T21:06:25.755 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:06:25 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/1587395234' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 7, "weight": 0.44836012165793127}]': finished 2023-09-23T21:06:25.755 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:06:25 smithi070 ceph-mon[168282]: osdmap e354: 8 total, 8 up, 8 in 2023-09-23T21:06:25.755 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:06:25 smithi070 ceph-mon[168282]: failed to encode map e354 with expected crc 2023-09-23T21:06:25.759 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:06:25.807 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:06:25.811 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:06:25.811 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:06:25.914 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:06:25.978 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:06:25.999 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:06:26.213 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T21:06:26.221 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:06:26.222 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:06:26.268 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 30 16 54060 54044 112.487 65.1875 0.0136706 0.00888525 2023-09-23T21:06:26.322 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:06:26.367 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:06:26.422 INFO:teuthology.orchestra.run.smithi026.stderr:nodeep-scrub is set 2023-09-23T21:06:26.423 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:06:26.427 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:06:26.429 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:06:26.530 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:06:26.591 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:06:26 smithi026 ceph-mon[175151]: pgmap v912: 109 pgs: 109 active+clean; 3.1 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 108 MiB/s wr, 1.73k op/s 2023-09-23T21:06:26.591 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:06:26 smithi026 ceph-mon[175151]: failed to encode map e355 with expected crc 2023-09-23T21:06:26.591 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:06:26 smithi026 ceph-mon[175151]: failed to encode map e355 with expected crc 2023-09-23T21:06:26.591 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:06:26 smithi026 ceph-mon[175151]: osdmap e355: 8 total, 8 up, 8 in 2023-09-23T21:06:26.591 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:06:26 smithi026 ceph-mon[175151]: failed to encode map e355 with expected crc 2023-09-23T21:06:26.591 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:06:26 smithi026 ceph-mon[175151]: failed to encode map e355 with expected crc 2023-09-23T21:06:26.592 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:06:26 smithi026 ceph-mon[175151]: failed to encode map e355 with expected crc 2023-09-23T21:06:26.592 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:06:26 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/3406325982' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:06:26.592 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:06:26 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:06:26.592 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:06:26 smithi026 ceph-mon[179688]: pgmap v912: 109 pgs: 109 active+clean; 3.1 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 108 MiB/s wr, 1.73k op/s 2023-09-23T21:06:26.592 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:06:26 smithi026 ceph-mon[179688]: failed to encode map e355 with expected crc 2023-09-23T21:06:26.592 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:06:26 smithi026 ceph-mon[179688]: failed to encode map e355 with expected crc 2023-09-23T21:06:26.592 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:06:26 smithi026 ceph-mon[179688]: osdmap e355: 8 total, 8 up, 8 in 2023-09-23T21:06:26.592 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:06:26 smithi026 ceph-mon[179688]: failed to encode map e355 with expected crc 2023-09-23T21:06:26.592 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:06:26 smithi026 ceph-mon[179688]: failed to encode map e355 with expected crc 2023-09-23T21:06:26.592 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:06:26 smithi026 ceph-mon[179688]: failed to encode map e355 with expected crc 2023-09-23T21:06:26.592 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:06:26 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/3406325982' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:06:26.593 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:06:26 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:06:26.602 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:06:26.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:06:26 smithi070 ceph-mon[168282]: pgmap v912: 109 pgs: 109 active+clean; 3.1 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 108 MiB/s wr, 1.73k op/s 2023-09-23T21:06:26.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:06:26 smithi070 ceph-mon[168282]: failed to encode map e355 with expected crc 2023-09-23T21:06:26.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:06:26 smithi070 ceph-mon[168282]: failed to encode map e355 with expected crc 2023-09-23T21:06:26.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:06:26 smithi070 ceph-mon[168282]: osdmap e355: 8 total, 8 up, 8 in 2023-09-23T21:06:26.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:06:26 smithi070 ceph-mon[168282]: failed to encode map e355 with expected crc 2023-09-23T21:06:26.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:06:26 smithi070 ceph-mon[168282]: failed to encode map e355 with expected crc 2023-09-23T21:06:26.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:06:26 smithi070 ceph-mon[168282]: failed to encode map e355 with expected crc 2023-09-23T21:06:26.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:06:26 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/3406325982' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:06:26.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:06:26 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:06:26.881 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:06:26.885 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:06:26.885 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:06:26.986 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:06:27.046 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:06:27.268 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 31 16 55440 55424 111.64 86.25 0.00333494 0.00895275 2023-09-23T21:06:27.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:06:27 smithi070 ceph-mon[168282]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:06:27.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:06:27 smithi070 ceph-mon[168282]: failed to encode map e356 with expected crc 2023-09-23T21:06:27.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:06:27 smithi070 ceph-mon[168282]: failed to encode map e356 with expected crc 2023-09-23T21:06:27.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:06:27 smithi070 ceph-mon[168282]: failed to encode map e356 with expected crc 2023-09-23T21:06:27.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:06:27 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T21:06:27.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:06:27 smithi070 ceph-mon[168282]: osdmap e356: 8 total, 8 up, 8 in 2023-09-23T21:06:27.881 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:06:27 smithi026 ceph-mon[179688]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:06:27.881 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:06:27 smithi026 ceph-mon[179688]: failed to encode map e356 with expected crc 2023-09-23T21:06:27.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:06:27 smithi026 ceph-mon[179688]: failed to encode map e356 with expected crc 2023-09-23T21:06:27.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:06:27 smithi026 ceph-mon[179688]: failed to encode map e356 with expected crc 2023-09-23T21:06:27.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:06:27 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T21:06:27.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:06:27 smithi026 ceph-mon[179688]: osdmap e356: 8 total, 8 up, 8 in 2023-09-23T21:06:27.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:06:27 smithi026 ceph-mon[175151]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:06:27.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:06:27 smithi026 ceph-mon[175151]: failed to encode map e356 with expected crc 2023-09-23T21:06:27.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:06:27 smithi026 ceph-mon[175151]: failed to encode map e356 with expected crc 2023-09-23T21:06:27.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:06:27 smithi026 ceph-mon[175151]: failed to encode map e356 with expected crc 2023-09-23T21:06:27.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:06:27 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T21:06:27.883 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:06:27 smithi026 ceph-mon[175151]: osdmap e356: 8 total, 8 up, 8 in 2023-09-23T21:06:28.268 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 32 16 56905 56889 111.013 91.5625 0.00486719 0.00899257 2023-09-23T21:06:28.814 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:06:28 smithi026 ceph-mon[175151]: pgmap v915: 109 pgs: 109 active+clean; 3.1 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 70 MiB/s wr, 1.12k op/s 2023-09-23T21:06:28.814 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:06:28 smithi026 ceph-mon[179688]: pgmap v915: 109 pgs: 109 active+clean; 3.1 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 70 MiB/s wr, 1.12k op/s 2023-09-23T21:06:28.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:06:28 smithi070 ceph-mon[168282]: pgmap v915: 109 pgs: 109 active+clean; 3.1 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 70 MiB/s wr, 1.12k op/s 2023-09-23T21:06:29.268 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 33 16 58306 58290 110.303 87.5625 0.0128643 0.00906157 2023-09-23T21:06:29.885 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 21:06:29 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[164029]: ::ffff:172.21.15.70 - - [23/Sep/2023:21:06:29] "GET /metrics HTTP/1.1" 200 36286 "" "Prometheus/2.43.0" 2023-09-23T21:06:30.268 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 34 16 59788 59772 109.783 92.625 0.00550722 0.00910345 2023-09-23T21:06:30.339 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 6, 7, 4, 5] out_osds: [] dead_osds: [] live_osds: [0, 2, 3, 4, 5, 7, 6, 1] 2023-09-23T21:06:30.339 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T21:06:30.339 INFO:tasks.thrashosds.thrasher:inject_pause on osd.3 2023-09-23T21:06:30.340 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-09-23T21:06:30.340 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-09-23T21:06:30.340 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 config set filestore_inject_stall 3 2023-09-23T21:06:30.871 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:06:30 smithi070 ceph-mon[168282]: pgmap v916: 109 pgs: 109 active+clean; 3.3 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 76 MiB/s wr, 1.22k op/s; 9 B/s, 1 keys/s, 0 objects/s recovering 2023-09-23T21:06:30.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:06:30 smithi026 ceph-mon[179688]: pgmap v916: 109 pgs: 109 active+clean; 3.3 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 76 MiB/s wr, 1.22k op/s; 9 B/s, 1 keys/s, 0 objects/s recovering 2023-09-23T21:06:30.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:06:30 smithi026 ceph-mon[175151]: pgmap v916: 109 pgs: 109 active+clean; 3.3 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 76 MiB/s wr, 1.22k op/s; 9 B/s, 1 keys/s, 0 objects/s recovering 2023-09-23T21:06:30.917 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T21:06:31.270 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 35 16 61074 61058 108.94 80.375 0.00719162 0.00917423 2023-09-23T21:06:31.492 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T21:06:31.497 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:06:31.497 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:06:31.598 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:06:31.648 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:06:31.709 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:06:31.715 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:06:31.715 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:06:31.816 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:06:31.920 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:06:32.270 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 36 16 62169 62153 107.815 68.4375 0.00570232 0.00927087 2023-09-23T21:06:32.381 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:06:32 smithi026 ceph-mon[179688]: pgmap v917: 109 pgs: 109 active+clean; 3.4 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 52 MiB/s wr, 831 op/s; 8 B/s, 1 keys/s, 0 objects/s recovering 2023-09-23T21:06:32.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:06:32 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:06:32.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:06:32 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:06:32.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:06:32 smithi026 ceph-mon[175151]: pgmap v917: 109 pgs: 109 active+clean; 3.4 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 52 MiB/s wr, 831 op/s; 8 B/s, 1 keys/s, 0 objects/s recovering 2023-09-23T21:06:32.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:06:32 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:06:32.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:06:32 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:06:32.621 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:06:32 smithi070 ceph-mon[168282]: pgmap v917: 109 pgs: 109 active+clean; 3.4 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 52 MiB/s wr, 831 op/s; 8 B/s, 1 keys/s, 0 objects/s recovering 2023-09-23T21:06:32.621 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:06:32 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:06:32.621 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:06:32 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:06:32.904 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:06:32.907 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:06:32.908 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:06:33.008 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:06:33.056 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:06:33.114 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:06:33.119 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:06:33.119 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:06:33.151 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:06:33.151 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:06:33.220 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:06:33.265 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:06:33.270 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 37 16 63479 63463 107.114 81.875 0.0113024 0.00933123 2023-09-23T21:06:33.325 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:06:33.330 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:06:33.330 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:06:33.431 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:06:33.482 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:06:33.544 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:06:33.552 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:06:33.552 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:06:33.653 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:06:33.698 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:06:33.765 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:06:33.770 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:06:33.770 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:06:33.871 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:06:33.936 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:06:34.270 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 38 15 64820 64805 106.503 83.875 0.00660978 0.00938519 2023-09-23T21:06:34.552 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:06:34.556 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:06:34 smithi026 ceph-mon[179688]: pgmap v918: 109 pgs: 109 active+clean; 3.7 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 72 MiB/s wr, 1.15k op/s; 6 B/s, 1 keys/s, 0 objects/s recovering 2023-09-23T21:06:34.556 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:06:34 smithi026 ceph-mon[175151]: pgmap v918: 109 pgs: 109 active+clean; 3.7 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 72 MiB/s wr, 1.15k op/s; 6 B/s, 1 keys/s, 0 objects/s recovering 2023-09-23T21:06:34.557 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:06:34.557 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:06:34.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:06:34 smithi070 ceph-mon[168282]: pgmap v918: 109 pgs: 109 active+clean; 3.7 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 72 MiB/s wr, 1.15k op/s; 6 B/s, 1 keys/s, 0 objects/s recovering 2023-09-23T21:06:34.658 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:06:34.694 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0: {} 2023-09-23T21:06:34.698 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:06:34.738 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1: {} 2023-09-23T21:06:34.787 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2: {} 2023-09-23T21:06:34.829 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3: {} 2023-09-23T21:06:34.871 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4: {} 2023-09-23T21:06:34.910 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T21:06:34.960 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6: {} 2023-09-23T21:06:35.018 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T21:06:35.037 INFO:teuthology.orchestra.run.smithi026.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T21:06:35.038 INFO:teuthology.orchestra.run.smithi026.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T21:06:35.038 INFO:teuthology.orchestra.run.smithi026.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T21:06:35.038 INFO:teuthology.orchestra.run.smithi026.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T21:06:35.038 INFO:teuthology.orchestra.run.smithi026.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T21:06:35.038 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T21:06:35.038 INFO:teuthology.orchestra.run.smithi026.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T21:06:35.038 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T21:06:35.140 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T21:06:35.148 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:06:35.148 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:06:35.249 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:06:35.270 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 39 16 65967 65951 105.609 71.625 0.00799703 0.00946355 2023-09-23T21:06:35.322 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:06:35.450 INFO:teuthology.orchestra.run.smithi026.stdout:ERROR: (22) Invalid argument 2023-09-23T21:06:35.450 INFO:teuthology.orchestra.run.smithi026.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T21:06:35.481 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:06:35 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/2023868544' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:06:35.482 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:06:35 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/2023868544' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:06:35.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:06:35 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/2023868544' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:06:35.649 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:06:35.654 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:06:35.654 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:06:35.755 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:06:35.823 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:06:35.946 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:06:35.953 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:06:35.953 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:06:36.054 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:06:36.149 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:06:36.270 INFO:tasks.radosbench.radosbench.0.smithi026.stdout:2023-09-23T21:06:36.271082+0000 min lat: 0.00137237 max lat: 1.03425 avg lat: 0.00953498 2023-09-23T21:06:36.270 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T21:06:36.270 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 40 16 67154 67138 104.824 74.1875 0.0103522 0.00953498 2023-09-23T21:06:36.330 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:06:36 smithi026 ceph-mon[179688]: pgmap v919: 109 pgs: 109 active+clean; 3.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 80 MiB/s wr, 1.28k op/s; 5 B/s, 1 keys/s, 0 objects/s recovering 2023-09-23T21:06:36.331 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:06:36 smithi026 ceph-mon[175151]: pgmap v919: 109 pgs: 109 active+clean; 3.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 80 MiB/s wr, 1.28k op/s; 5 B/s, 1 keys/s, 0 objects/s recovering 2023-09-23T21:06:36.379 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:06:36.386 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:06:36.386 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:06:36.447 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_historic_ops 2023-09-23T21:06:36.487 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:06:36.527 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:06:36 smithi070 ceph-mon[168282]: pgmap v919: 109 pgs: 109 active+clean; 3.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 80 MiB/s wr, 1.28k op/s; 5 B/s, 1 keys/s, 0 objects/s recovering 2023-09-23T21:06:36.529 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:06:36.587 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:06:36.591 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:06:36.592 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:06:36.693 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:06:36.827 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:06:37.000 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T21:06:37.008 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:06:37.009 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:06:37.110 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:06:37.218 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:06:37.232 INFO:teuthology.orchestra.run.smithi026.stdout:{ 2023-09-23T21:06:37.232 INFO:teuthology.orchestra.run.smithi026.stdout: "success": "" 2023-09-23T21:06:37.232 INFO:teuthology.orchestra.run.smithi026.stdout:} 2023-09-23T21:06:37.272 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 41 16 68193 68177 103.846 64.9375 0.0101585 0.00962428 2023-09-23T21:06:37.696 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T21:06:37.707 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:06:37.707 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:06:37.804 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:06:37.808 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:06:37.898 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:06:37.971 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:06:37.977 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:06:37.978 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:06:38.047 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T21:06:38.079 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:06:38.160 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:06:38.230 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:06:38.241 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:06:38.241 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:06:38.272 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 42 16 69276 69260 102.986 67.6875 0.0216928 0.00970555 2023-09-23T21:06:38.333 INFO:teuthology.orchestra.run.smithi026.stderr:noscrub is unset 2023-09-23T21:06:38.342 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:06:38.391 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:06:38.431 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:06:38.438 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:06:38.439 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:06:38.510 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:06:38 smithi026 ceph-mon[175151]: pgmap v920: 109 pgs: 109 active+clean; 3.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 72 MiB/s wr, 1.16k op/s; 5 B/s, 0 keys/s, 0 objects/s recovering 2023-09-23T21:06:38.510 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:06:38 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/1288484080' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T21:06:38.510 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:06:38 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T21:06:38.512 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:06:38 smithi026 ceph-mon[179688]: pgmap v920: 109 pgs: 109 active+clean; 3.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 72 MiB/s wr, 1.16k op/s; 5 B/s, 0 keys/s, 0 objects/s recovering 2023-09-23T21:06:38.512 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:06:38 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/1288484080' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T21:06:38.512 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:06:38 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T21:06:38.539 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:06:38.603 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:06:38.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:06:38 smithi070 ceph-mon[168282]: pgmap v920: 109 pgs: 109 active+clean; 3.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 72 MiB/s wr, 1.16k op/s; 5 B/s, 0 keys/s, 0 objects/s recovering 2023-09-23T21:06:38.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:06:38 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/1288484080' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T21:06:38.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:06:38 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T21:06:38.799 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:06:38.806 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:06:38.806 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:06:38.907 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:06:38.950 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:06:39.004 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:06:39.010 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:06:39.011 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:06:39.111 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:06:39.187 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:06:39.272 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 43 16 70385 70369 102.203 69.3125 0.012484 0.00977888 2023-09-23T21:06:39.316 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:06:39.324 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:06:39 smithi070 ceph-mon[168282]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:06:39.324 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:06:39 smithi070 ceph-mon[168282]: failed to encode map e357 with expected crc 2023-09-23T21:06:39.324 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:06:39 smithi070 ceph-mon[168282]: failed to encode map e357 with expected crc 2023-09-23T21:06:39.324 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:06:39 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T21:06:39.324 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:06:39 smithi070 ceph-mon[168282]: osdmap e357: 8 total, 8 up, 8 in 2023-09-23T21:06:39.324 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:06:39 smithi070 ceph-mon[168282]: failed to encode map e357 with expected crc 2023-09-23T21:06:39.324 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:06:39 smithi070 ceph-mon[168282]: 1.d scrub starts 2023-09-23T21:06:39.325 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:06:39 smithi070 ceph-mon[168282]: 1.d scrub ok 2023-09-23T21:06:39.325 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:06:39.326 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:06:39.426 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:06:39.468 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:06:39.507 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:06:39 smithi026 ceph-mon[175151]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:06:39.507 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:06:39 smithi026 ceph-mon[175151]: failed to encode map e357 with expected crc 2023-09-23T21:06:39.507 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:06:39 smithi026 ceph-mon[175151]: failed to encode map e357 with expected crc 2023-09-23T21:06:39.507 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:06:39 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T21:06:39.507 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:06:39 smithi026 ceph-mon[175151]: osdmap e357: 8 total, 8 up, 8 in 2023-09-23T21:06:39.507 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:06:39 smithi026 ceph-mon[175151]: failed to encode map e357 with expected crc 2023-09-23T21:06:39.507 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:06:39 smithi026 ceph-mon[175151]: 1.d scrub starts 2023-09-23T21:06:39.507 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:06:39 smithi026 ceph-mon[175151]: 1.d scrub ok 2023-09-23T21:06:39.508 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:06:39 smithi026 ceph-mon[179688]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:06:39.508 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:06:39 smithi026 ceph-mon[179688]: failed to encode map e357 with expected crc 2023-09-23T21:06:39.508 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:06:39 smithi026 ceph-mon[179688]: failed to encode map e357 with expected crc 2023-09-23T21:06:39.508 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:06:39 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T21:06:39.508 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:06:39 smithi026 ceph-mon[179688]: osdmap e357: 8 total, 8 up, 8 in 2023-09-23T21:06:39.508 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:06:39 smithi026 ceph-mon[179688]: failed to encode map e357 with expected crc 2023-09-23T21:06:39.508 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:06:39 smithi026 ceph-mon[179688]: 1.d scrub starts 2023-09-23T21:06:39.508 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:06:39 smithi026 ceph-mon[179688]: 1.d scrub ok 2023-09-23T21:06:39.525 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:06:39.562 INFO:teuthology.orchestra.run.smithi026.stdout:ERROR: (22) Invalid argument 2023-09-23T21:06:39.562 INFO:teuthology.orchestra.run.smithi026.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T21:06:39.707 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:06:39.716 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:06:39.717 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:06:39.817 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:06:39.860 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:06:39.881 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 21:06:39 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[164029]: ::ffff:172.21.15.70 - - [23/Sep/2023:21:06:39] "GET /metrics HTTP/1.1" 200 36295 "" "Prometheus/2.43.0" 2023-09-23T21:06:39.909 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:06:39.915 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:06:39.916 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:06:40.016 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:06:40.095 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:06:40.149 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_ops_in_flight 2023-09-23T21:06:40.274 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 44 16 71511 71495 101.477 70.375 0.017946 0.00984944 2023-09-23T21:06:40.326 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:06:40 smithi026 ceph-mon[179688]: pgmap v922: 109 pgs: 109 active+clean; 4.1 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 78 MiB/s wr, 1.25k op/s 2023-09-23T21:06:40.326 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:06:40 smithi026 ceph-mon[175151]: pgmap v922: 109 pgs: 109 active+clean; 4.1 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 78 MiB/s wr, 1.25k op/s 2023-09-23T21:06:40.333 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:06:40.344 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:06:40.344 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:06:40.445 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:06:40.506 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:06:40 smithi070 ceph-mon[168282]: pgmap v922: 109 pgs: 109 active+clean; 4.1 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 78 MiB/s wr, 1.25k op/s 2023-09-23T21:06:40.509 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:06:40.568 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:06:40.573 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:06:40.573 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:06:40.674 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:06:40.739 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:06:40.844 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:06:40.856 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:06:40.857 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:06:40.904 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T21:06:40.957 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:06:41.045 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:06:41.112 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T21:06:41.120 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:06:41.120 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:06:41.178 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T21:06:41.220 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:06:41.271 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:06:41.274 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 45 16 72638 72622 100.788 70.4375 0.0159981 0.00991558 2023-09-23T21:06:41.334 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0: {} 2023-09-23T21:06:41.366 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1: {} 2023-09-23T21:06:41.380 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:06:41.390 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:06:41.390 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:06:41.391 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:06:41 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/3666144929' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:06:41.407 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2: {} 2023-09-23T21:06:41.445 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3: {} 2023-09-23T21:06:41.490 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4: {} 2023-09-23T21:06:41.491 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:06:41.527 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:06:41.527 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T21:06:41.569 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6: {} 2023-09-23T21:06:41.609 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T21:06:41.622 INFO:teuthology.orchestra.run.smithi026.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T21:06:41.623 INFO:teuthology.orchestra.run.smithi026.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T21:06:41.623 INFO:teuthology.orchestra.run.smithi026.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T21:06:41.623 INFO:teuthology.orchestra.run.smithi026.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T21:06:41.623 INFO:teuthology.orchestra.run.smithi026.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T21:06:41.623 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T21:06:41.623 INFO:teuthology.orchestra.run.smithi026.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T21:06:41.623 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T21:06:41.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:06:41 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/3666144929' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:06:41.633 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:06:41 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/3666144929' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:06:41.648 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:06:41.654 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:06:41.654 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:06:41.755 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:06:41.827 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:06:42.061 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:06:42.068 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:06:42.068 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:06:42.169 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:06:42.238 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:06:42.256 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:06:42.274 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 46 16 73711 73695 100.055 67.0625 0.0115438 0.00998916 2023-09-23T21:06:42.303 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:06:42.307 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:06:42.308 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:06:42.311 INFO:teuthology.orchestra.run.smithi070.stdout:{ 2023-09-23T21:06:42.312 INFO:teuthology.orchestra.run.smithi070.stdout: "ops": [], 2023-09-23T21:06:42.312 INFO:teuthology.orchestra.run.smithi070.stdout: "num_ops": 0 2023-09-23T21:06:42.312 INFO:teuthology.orchestra.run.smithi070.stdout:} 2023-09-23T21:06:42.355 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:06:42 smithi026 ceph-mon[175151]: pgmap v923: 109 pgs: 109 active+clean; 4.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 79 MiB/s wr, 1.26k op/s 2023-09-23T21:06:42.355 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:06:42 smithi026 ceph-mon[179688]: pgmap v923: 109 pgs: 109 active+clean; 4.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 79 MiB/s wr, 1.26k op/s 2023-09-23T21:06:42.409 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:06:42.476 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:06:42.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:06:42 smithi070 ceph-mon[168282]: pgmap v923: 109 pgs: 109 active+clean; 4.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 79 MiB/s wr, 1.26k op/s 2023-09-23T21:06:42.664 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_blocked_ops 2023-09-23T21:06:42.790 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 6, 7, 4, 5] out_osds: [] dead_osds: [] live_osds: [0, 2, 3, 4, 5, 7, 6, 1] 2023-09-23T21:06:42.790 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T21:06:42.790 INFO:tasks.thrashosds.thrasher:primary_affinity 2023-09-23T21:06:42.790 INFO:tasks.thrashosds.thrasher:Setting osd 5 primary_affinity to 0.438407 2023-09-23T21:06:42.791 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph osd primary-affinity 5 0.43840672051352914 2023-09-23T21:06:43.274 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 47 16 74848 74832 99.4382 71.0625 0.00786655 0.0100516 2023-09-23T21:06:43.429 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T21:06:43.644 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:06:43.652 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:06:43.653 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:06:43.753 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:06:43.811 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:06:43.879 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T21:06:43.890 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:06:43.891 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:06:43.992 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:06:44.064 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:06:44.082 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T21:06:44.275 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 48 16 75966 75950 98.8219 69.875 0.0164923 0.010114 2023-09-23T21:06:44.362 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:06:44 smithi070 ceph-mon[168282]: pgmap v924: 109 pgs: 109 active+clean; 4.4 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 74 MiB/s wr, 1.19k op/s 2023-09-23T21:06:44.431 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:06:44.437 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:06:44.438 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:06:44.538 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:06:44.604 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:06:44.627 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:06:44 smithi026 ceph-mon[175151]: pgmap v924: 109 pgs: 109 active+clean; 4.4 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 74 MiB/s wr, 1.19k op/s 2023-09-23T21:06:44.627 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:06:44 smithi026 ceph-mon[179688]: pgmap v924: 109 pgs: 109 active+clean; 4.4 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 74 MiB/s wr, 1.19k op/s 2023-09-23T21:06:44.667 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:06:44.679 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:06:44.680 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:06:44.781 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:06:44.828 INFO:teuthology.orchestra.run.smithi070.stdout:{ 2023-09-23T21:06:44.828 INFO:teuthology.orchestra.run.smithi070.stdout: "ops": [], 2023-09-23T21:06:44.828 INFO:teuthology.orchestra.run.smithi070.stdout: "complaint_time": 30, 2023-09-23T21:06:44.828 INFO:teuthology.orchestra.run.smithi070.stdout: "num_blocked_ops": 0 2023-09-23T21:06:44.828 INFO:teuthology.orchestra.run.smithi070.stdout:} 2023-09-23T21:06:44.845 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:06:44.915 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:06:44.922 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:06:44.923 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:06:45.023 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:06:45.089 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:06:45.265 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_historic_ops 2023-09-23T21:06:45.278 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 49 16 77027 77011 98.1517 66.3125 0.00746092 0.0101826 2023-09-23T21:06:45.405 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T21:06:45.409 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:06:45.410 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:06:45.468 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:06:45.510 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:06:45.560 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:06:45.567 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:06:45.615 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:06:45.621 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:06:45.621 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:06:45.722 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:06:45.762 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:06:45.816 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:06:45.820 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:06:45.820 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:06:45.921 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:06:45.932 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T21:06:45.989 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:06:46.093 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:06:46.100 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:06:46.101 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:06:46.202 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:06:46.251 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:06:46.283 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 50 16 78099 78083 97.5225 67 0.0138771 0.0102491 2023-09-23T21:06:46.320 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:06:46.323 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:06:46.324 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:06:46.368 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:06:46 smithi026 ceph-mon[175151]: pgmap v925: 109 pgs: 109 active+clean; 4.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 71 MiB/s wr, 1.14k op/s 2023-09-23T21:06:46.369 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:06:46 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:06:46.369 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:06:46 smithi026 ceph-mon[179688]: pgmap v925: 109 pgs: 109 active+clean; 4.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 71 MiB/s wr, 1.14k op/s 2023-09-23T21:06:46.369 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:06:46 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:06:46.425 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:06:46.485 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:06:46.605 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:06:46.615 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:06:46.615 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:06:46.716 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:06:46.735 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:06:46 smithi070 ceph-mon[168282]: pgmap v925: 109 pgs: 109 active+clean; 4.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 71 MiB/s wr, 1.14k op/s 2023-09-23T21:06:46.735 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:06:46 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:06:46.770 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:06:46.833 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:06:46.841 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:06:46.841 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:06:46.942 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:06:47.046 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:06:47.100 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:06:47.106 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:06:47.107 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:06:47.208 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:06:47.281 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:06:47.282 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 51 16 79076 79060 96.808 61.0625 0.00924539 0.0103244 2023-09-23T21:06:47.335 INFO:teuthology.orchestra.run.smithi070.stdout:{ 2023-09-23T21:06:47.336 INFO:teuthology.orchestra.run.smithi070.stdout: "size": 20, 2023-09-23T21:06:47.336 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 600, 2023-09-23T21:06:47.336 INFO:teuthology.orchestra.run.smithi070.stdout: "ops": [ 2023-09-23T21:06:47.336 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:06:47.336 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.35469.0:133009 8.2 8:43227b1f:::benchmark_data_smithi026_301269_object22252:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e281)", 2023-09-23T21:06:47.336 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:03:29.350078+0000", 2023-09-23T21:06:47.336 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 197.981880093, 2023-09-23T21:06:47.336 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.92559535299999995, 2023-09-23T21:06:47.337 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:06:47.337 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:06:47.337 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:06:47.337 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.35469", 2023-09-23T21:06:47.337 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/4041424623", 2023-09-23T21:06:47.337 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 133009 2023-09-23T21:06:47.337 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:06:47.337 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:06:47.338 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:06:47.338 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:06:47.338 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:29.350078+0000", 2023-09-23T21:06:47.338 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:06:47.338 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:06:47.338 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:06:47.338 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:06:47.338 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:29.350078+0000", 2023-09-23T21:06:47.338 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:06:47.338 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:06:47.339 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:06:47.339 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:06:47.339 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:29.350075+0000", 2023-09-23T21:06:47.339 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999971 2023-09-23T21:06:47.339 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:06:47.339 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:06:47.339 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:06:47.339 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:29.350092+0000", 2023-09-23T21:06:47.340 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.6350000000000001e-05 2023-09-23T21:06:47.340 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:06:47.340 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:06:47.340 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:06:47.340 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:29.350130+0000", 2023-09-23T21:06:47.340 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.8577e-05 2023-09-23T21:06:47.340 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:06:47.340 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:06:47.340 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:06:47.340 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:29.350157+0000", 2023-09-23T21:06:47.340 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.6738999999999999e-05 2023-09-23T21:06:47.340 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:06:47.341 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:06:47.341 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:06:47.341 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:29.508836+0000", 2023-09-23T21:06:47.341 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.158678763 2023-09-23T21:06:47.341 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:06:47.341 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:06:47.341 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:06:47.341 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:29.508848+0000", 2023-09-23T21:06:47.341 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.2469e-05 2023-09-23T21:06:47.341 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:06:47.341 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:06:47.342 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:06:47.342 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:30.262423+0000", 2023-09-23T21:06:47.342 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.75357442699999999 2023-09-23T21:06:47.342 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:06:47.342 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:06:47.342 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:06:47.342 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:30.262553+0000", 2023-09-23T21:06:47.342 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000130748 2023-09-23T21:06:47.342 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:06:47.342 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:06:47.342 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 0,5", 2023-09-23T21:06:47.343 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:30.262945+0000", 2023-09-23T21:06:47.343 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00039175199999999999 2023-09-23T21:06:47.343 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:06:47.343 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:06:47.343 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:06:47.343 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:30.267224+0000", 2023-09-23T21:06:47.343 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0042790399999999996 2023-09-23T21:06:47.343 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:06:47.343 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:06:47.343 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:06:47.343 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:30.267282+0000", 2023-09-23T21:06:47.344 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.7314e-05 2023-09-23T21:06:47.344 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:06:47.344 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:06:47.344 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:06:47.344 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:30.275547+0000", 2023-09-23T21:06:47.344 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0082652219999999992 2023-09-23T21:06:47.344 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:06:47.344 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:06:47.344 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:06:47.344 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:30.275603+0000", 2023-09-23T21:06:47.344 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.5909999999999998e-05 2023-09-23T21:06:47.344 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:06:47.344 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:06:47.345 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:06:47.345 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:30.275674+0000", 2023-09-23T21:06:47.345 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.1094999999999996e-05 2023-09-23T21:06:47.345 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:06:47.345 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:06:47.345 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:06:47.345 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:06:47.345 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:06:47.345 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.35469.0:164165 8.14 8:29df2d77:::benchmark_data_smithi026_301269_object53408:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e307)", 2023-09-23T21:06:47.345 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:04:13.318492+0000", 2023-09-23T21:06:47.345 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 154.01346680399999, 2023-09-23T21:06:47.345 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.95646294899999995, 2023-09-23T21:06:47.346 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:06:47.346 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:06:47.346 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:06:47.346 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.35469", 2023-09-23T21:06:47.346 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/4041424623", 2023-09-23T21:06:47.346 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 164165 2023-09-23T21:06:47.346 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:06:47.346 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:06:47.346 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:06:47.346 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:06:47.346 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:13.318492+0000", 2023-09-23T21:06:47.347 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:06:47.347 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:06:47.347 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:06:47.347 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:06:47.347 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:13.318492+0000", 2023-09-23T21:06:47.347 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:06:47.347 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:06:47.347 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:06:47.347 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:06:47.347 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:13.318491+0000", 2023-09-23T21:06:47.347 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.999999 2023-09-23T21:06:47.347 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:06:47.348 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:06:47.348 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:06:47.348 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:13.318496+0000", 2023-09-23T21:06:47.348 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.1680000000000003e-06 2023-09-23T21:06:47.348 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:06:47.348 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:06:47.348 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:06:47.348 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:13.318506+0000", 2023-09-23T21:06:47.348 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.0454e-05 2023-09-23T21:06:47.348 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:06:47.348 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:06:47.349 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:06:47.349 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:13.318511+0000", 2023-09-23T21:06:47.349 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.4959999999999998e-06 2023-09-23T21:06:47.349 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:06:47.349 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:06:47.349 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:06:47.349 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:13.318553+0000", 2023-09-23T21:06:47.349 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.2230000000000001e-05 2023-09-23T21:06:47.349 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:06:47.349 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:06:47.349 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:06:47.349 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:13.318560+0000", 2023-09-23T21:06:47.349 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.2320000000000004e-06 2023-09-23T21:06:47.350 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:06:47.350 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:06:47.350 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:06:47.350 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:14.243781+0000", 2023-09-23T21:06:47.350 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.92522103200000005 2023-09-23T21:06:47.350 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:06:47.350 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:06:47.350 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:06:47.350 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:14.243936+0000", 2023-09-23T21:06:47.350 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00015447099999999999 2023-09-23T21:06:47.350 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:06:47.350 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:06:47.351 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 0,2", 2023-09-23T21:06:47.351 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:14.244331+0000", 2023-09-23T21:06:47.351 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00039575899999999998 2023-09-23T21:06:47.351 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:06:47.351 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:06:47.351 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:06:47.351 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:14.246713+0000", 2023-09-23T21:06:47.351 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.002382056 2023-09-23T21:06:47.351 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:06:47.351 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:06:47.351 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:06:47.352 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:14.246724+0000", 2023-09-23T21:06:47.352 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.0152999999999999e-05 2023-09-23T21:06:47.352 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:06:47.352 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:06:47.352 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:06:47.352 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:14.274860+0000", 2023-09-23T21:06:47.352 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.028136345 2023-09-23T21:06:47.352 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:06:47.352 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:06:47.352 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:06:47.352 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:14.274913+0000", 2023-09-23T21:06:47.352 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.2827000000000001e-05 2023-09-23T21:06:47.352 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:06:47.353 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:06:47.353 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:06:47.354 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:14.274955+0000", 2023-09-23T21:06:47.354 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.1907999999999999e-05 2023-09-23T21:06:47.354 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:06:47.354 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:06:47.354 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:06:47.354 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:06:47.354 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:06:47.354 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.35469.0:164203 8.14 8:2d1c900f:::benchmark_data_smithi026_301269_object53446:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e307)", 2023-09-23T21:06:47.354 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:04:13.371049+0000", 2023-09-23T21:06:47.354 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 153.960909176, 2023-09-23T21:06:47.355 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.90403997899999999, 2023-09-23T21:06:47.355 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:06:47.355 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:06:47.355 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:06:47.355 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.35469", 2023-09-23T21:06:47.355 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/4041424623", 2023-09-23T21:06:47.355 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 164203 2023-09-23T21:06:47.355 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:06:47.355 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:06:47.355 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:06:47.355 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:06:47.355 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:13.371049+0000", 2023-09-23T21:06:47.356 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:06:47.356 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:06:47.356 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:06:47.356 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:06:47.356 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:13.371049+0000", 2023-09-23T21:06:47.356 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:06:47.356 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:06:47.356 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:06:47.356 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:06:47.356 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:13.371048+0000", 2023-09-23T21:06:47.356 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999981 2023-09-23T21:06:47.356 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:06:47.356 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:06:47.357 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:06:47.357 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:13.371053+0000", 2023-09-23T21:06:47.357 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.8080000000000001e-06 2023-09-23T21:06:47.357 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:06:47.357 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:06:47.357 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:06:47.357 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:13.371064+0000", 2023-09-23T21:06:47.357 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.0781999999999999e-05 2023-09-23T21:06:47.357 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:06:47.357 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:06:47.357 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:06:47.357 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:13.371070+0000", 2023-09-23T21:06:47.358 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.8300000000000001e-06 2023-09-23T21:06:47.358 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:06:47.358 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:06:47.358 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:06:47.358 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:13.371112+0000", 2023-09-23T21:06:47.358 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.2020000000000001e-05 2023-09-23T21:06:47.358 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:06:47.358 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:06:47.358 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:06:47.358 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:13.371119+0000", 2023-09-23T21:06:47.358 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.2110000000000001e-06 2023-09-23T21:06:47.358 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:06:47.359 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:06:47.359 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:06:47.359 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:14.246769+0000", 2023-09-23T21:06:47.359 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.87565003799999996 2023-09-23T21:06:47.359 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:06:47.359 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:06:47.359 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:06:47.359 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:14.246842+0000", 2023-09-23T21:06:47.359 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.2512e-05 2023-09-23T21:06:47.359 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:06:47.360 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:06:47.360 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 0,2", 2023-09-23T21:06:47.360 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:14.247045+0000", 2023-09-23T21:06:47.360 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000203005 2023-09-23T21:06:47.360 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:06:47.360 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:06:47.360 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:06:47.360 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:14.252323+0000", 2023-09-23T21:06:47.360 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.005278235 2023-09-23T21:06:47.360 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:06:47.360 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:06:47.360 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:06:47.360 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:14.252430+0000", 2023-09-23T21:06:47.361 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000106543 2023-09-23T21:06:47.361 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:06:47.361 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:06:47.361 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:06:47.361 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:14.275003+0000", 2023-09-23T21:06:47.361 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.022573473 2023-09-23T21:06:47.361 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:06:47.361 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:06:47.361 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:06:47.361 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:14.275049+0000", 2023-09-23T21:06:47.361 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.5510999999999998e-05 2023-09-23T21:06:47.361 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:06:47.362 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:06:47.362 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:06:47.362 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:14.275089+0000", 2023-09-23T21:06:47.362 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.0738999999999997e-05 2023-09-23T21:06:47.362 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:06:47.362 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:06:47.362 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:06:47.362 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:06:47.362 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:06:47.362 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.35469.0:172321 8.18 8:1f117af2:::benchmark_data_smithi026_301269_object61564:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e317)", 2023-09-23T21:06:47.362 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:04:28.396199+0000", 2023-09-23T21:06:47.362 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 138.93575999000001, 2023-09-23T21:06:47.363 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.0147737569999999, 2023-09-23T21:06:47.363 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:06:47.363 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:06:47.363 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:06:47.363 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.35469", 2023-09-23T21:06:47.363 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/4041424623", 2023-09-23T21:06:47.363 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 172321 2023-09-23T21:06:47.363 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:06:47.363 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:06:47.363 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:06:47.363 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:06:47.363 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:28.396199+0000", 2023-09-23T21:06:47.363 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:06:47.364 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:06:47.364 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:06:47.364 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:06:47.364 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:28.396199+0000", 2023-09-23T21:06:47.364 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:06:47.364 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:06:47.364 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:06:47.364 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:06:47.364 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:28.396198+0000", 2023-09-23T21:06:47.364 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999995 2023-09-23T21:06:47.364 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:06:47.364 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:06:47.365 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:06:47.365 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:28.396200+0000", 2023-09-23T21:06:47.365 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.8419999999999999e-06 2023-09-23T21:06:47.365 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:06:47.365 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:06:47.365 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:06:47.365 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:28.396204+0000", 2023-09-23T21:06:47.365 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.5640000000000001e-06 2023-09-23T21:06:47.365 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:06:47.365 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:06:47.365 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:06:47.365 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:28.396206+0000", 2023-09-23T21:06:47.365 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.2249999999999999e-06 2023-09-23T21:06:47.366 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:06:47.366 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:06:47.366 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:06:47.366 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:28.396256+0000", 2023-09-23T21:06:47.366 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.9913000000000001e-05 2023-09-23T21:06:47.366 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:06:47.366 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:06:47.366 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:06:47.366 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:28.396264+0000", 2023-09-23T21:06:47.366 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.3809999999999999e-06 2023-09-23T21:06:47.366 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:06:47.366 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:06:47.367 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:06:47.367 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:29.402549+0000", 2023-09-23T21:06:47.367 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.0062851639999999 2023-09-23T21:06:47.367 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:06:47.367 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:06:47.367 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:06:47.367 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:29.402686+0000", 2023-09-23T21:06:47.367 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00013646700000000001 2023-09-23T21:06:47.367 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:06:47.367 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:06:47.367 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 3,5", 2023-09-23T21:06:47.367 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:29.403010+0000", 2023-09-23T21:06:47.367 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00032413599999999999 2023-09-23T21:06:47.368 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:06:47.368 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:06:47.368 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:06:47.368 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:29.407637+0000", 2023-09-23T21:06:47.368 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0046273240000000004 2023-09-23T21:06:47.368 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:06:47.368 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:06:47.368 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:06:47.368 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:29.407667+0000", 2023-09-23T21:06:47.368 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.0192000000000002e-05 2023-09-23T21:06:47.368 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:06:47.368 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:06:47.369 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:06:47.369 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:29.410884+0000", 2023-09-23T21:06:47.369 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0032160800000000001 2023-09-23T21:06:47.369 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:06:47.369 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:06:47.369 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:06:47.369 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:29.410929+0000", 2023-09-23T21:06:47.369 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.5364000000000001e-05 2023-09-23T21:06:47.369 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:06:47.369 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:06:47.369 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:06:47.369 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:29.410972+0000", 2023-09-23T21:06:47.369 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.3436999999999998e-05 2023-09-23T21:06:47.370 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:06:47.370 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:06:47.370 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:06:47.370 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:06:47.370 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:06:47.370 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.35469.0:172350 8.18 8:1980b52b:::benchmark_data_smithi026_301269_object61593:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e317)", 2023-09-23T21:06:47.370 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:04:28.433685+0000", 2023-09-23T21:06:47.370 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 138.898273254, 2023-09-23T21:06:47.370 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.97745544200000001, 2023-09-23T21:06:47.370 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:06:47.370 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:06:47.370 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:06:47.371 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.35469", 2023-09-23T21:06:47.371 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/4041424623", 2023-09-23T21:06:47.371 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 172350 2023-09-23T21:06:47.371 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:06:47.371 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:06:47.371 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:06:47.371 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:06:47.371 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:28.433685+0000", 2023-09-23T21:06:47.371 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:06:47.371 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:06:47.371 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:06:47.371 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:06:47.372 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:28.433685+0000", 2023-09-23T21:06:47.372 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:06:47.372 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:06:47.372 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:06:47.372 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:06:47.372 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:28.433683+0000", 2023-09-23T21:06:47.372 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999981 2023-09-23T21:06:47.372 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:06:47.372 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:06:47.372 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:06:47.372 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:28.433690+0000", 2023-09-23T21:06:47.372 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.2199999999999997e-06 2023-09-23T21:06:47.373 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:06:47.373 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:06:47.373 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:06:47.373 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:28.433701+0000", 2023-09-23T21:06:47.373 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.1474999999999999e-05 2023-09-23T21:06:47.373 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:06:47.373 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:06:47.373 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:06:47.373 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:28.433707+0000", 2023-09-23T21:06:47.373 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.6470000000000003e-06 2023-09-23T21:06:47.373 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:06:47.373 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:06:47.373 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:06:47.374 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:28.433761+0000", 2023-09-23T21:06:47.374 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.4412e-05 2023-09-23T21:06:47.374 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:06:47.374 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:06:47.374 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:06:47.374 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:28.433769+0000", 2023-09-23T21:06:47.374 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.2379999999999997e-06 2023-09-23T21:06:47.374 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:06:47.374 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:06:47.374 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:06:47.374 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:29.404626+0000", 2023-09-23T21:06:47.374 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.97085643600000004 2023-09-23T21:06:47.375 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:06:47.375 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:06:47.375 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:06:47.375 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:29.404696+0000", 2023-09-23T21:06:47.375 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.0241999999999996e-05 2023-09-23T21:06:47.375 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:06:47.375 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:06:47.375 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 3,5", 2023-09-23T21:06:47.375 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:29.404851+0000", 2023-09-23T21:06:47.375 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00015510599999999999 2023-09-23T21:06:47.375 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:06:47.375 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:06:47.376 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:06:47.376 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:29.407733+0000", 2023-09-23T21:06:47.376 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0028823120000000002 2023-09-23T21:06:47.376 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:06:47.376 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:06:47.376 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:06:47.376 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:29.411020+0000", 2023-09-23T21:06:47.376 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0032863570000000002 2023-09-23T21:06:47.376 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:06:47.376 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:06:47.376 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:06:47.376 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:29.411055+0000", 2023-09-23T21:06:47.376 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.5271999999999997e-05 2023-09-23T21:06:47.377 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:06:47.377 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:06:47.377 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:06:47.377 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:29.411102+0000", 2023-09-23T21:06:47.377 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.6746999999999998e-05 2023-09-23T21:06:47.377 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:06:47.377 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:06:47.377 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:06:47.377 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:29.411141+0000", 2023-09-23T21:06:47.377 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.8884000000000002e-05 2023-09-23T21:06:47.377 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:06:47.377 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:06:47.378 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:06:47.378 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:06:47.378 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:06:47.378 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.35469.0:172376 8.18 8:1aa907c3:::benchmark_data_smithi026_301269_object61619:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e317)", 2023-09-23T21:06:47.378 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:04:28.453649+0000", 2023-09-23T21:06:47.378 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 138.878309481, 2023-09-23T21:06:47.378 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.96206967399999999, 2023-09-23T21:06:47.378 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:06:47.378 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:06:47.378 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:06:47.378 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.35469", 2023-09-23T21:06:47.378 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/4041424623", 2023-09-23T21:06:47.379 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 172376 2023-09-23T21:06:47.379 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:06:47.379 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:06:47.379 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:06:47.379 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:06:47.379 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:28.453649+0000", 2023-09-23T21:06:47.379 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:06:47.379 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:06:47.379 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:06:47.379 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:06:47.379 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:28.453649+0000", 2023-09-23T21:06:47.379 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:06:47.379 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:06:47.380 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:06:47.380 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:06:47.380 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:28.453648+0000", 2023-09-23T21:06:47.380 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T21:06:47.380 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:06:47.380 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:06:47.380 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:06:47.380 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:28.453654+0000", 2023-09-23T21:06:47.380 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.0070000000000003e-06 2023-09-23T21:06:47.380 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:06:47.380 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:06:47.380 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:06:47.381 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:28.453667+0000", 2023-09-23T21:06:47.381 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.3389e-05 2023-09-23T21:06:47.381 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:06:47.381 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:06:47.381 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:06:47.381 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:28.453672+0000", 2023-09-23T21:06:47.381 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.4039999999999997e-06 2023-09-23T21:06:47.381 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:06:47.381 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:06:47.381 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:06:47.381 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:28.453719+0000", 2023-09-23T21:06:47.381 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.7491999999999999e-05 2023-09-23T21:06:47.381 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:06:47.382 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:06:47.382 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:06:47.382 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:28.453728+0000", 2023-09-23T21:06:47.382 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.1589999999999999e-06 2023-09-23T21:06:47.382 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:06:47.382 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:06:47.382 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:06:47.382 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:29.405990+0000", 2023-09-23T21:06:47.382 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.95226191800000004 2023-09-23T21:06:47.382 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:06:47.382 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:06:47.383 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:06:47.383 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:29.406103+0000", 2023-09-23T21:06:47.383 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00011302300000000001 2023-09-23T21:06:47.383 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:06:47.383 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:06:47.383 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 3,5", 2023-09-23T21:06:47.383 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:29.406346+0000", 2023-09-23T21:06:47.383 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00024300600000000001 2023-09-23T21:06:47.383 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:06:47.383 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:06:47.383 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:06:47.383 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:29.411176+0000", 2023-09-23T21:06:47.384 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0048295949999999999 2023-09-23T21:06:47.384 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:06:47.384 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:06:47.384 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:06:47.384 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:29.412780+0000", 2023-09-23T21:06:47.384 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.001604131 2023-09-23T21:06:47.384 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:06:47.384 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:06:47.384 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:06:47.384 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:29.415622+0000", 2023-09-23T21:06:47.384 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.002842266 2023-09-23T21:06:47.384 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:06:47.385 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:06:47.385 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:06:47.385 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:29.415676+0000", 2023-09-23T21:06:47.385 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.3331000000000002e-05 2023-09-23T21:06:47.385 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:06:47.385 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:06:47.385 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:06:47.385 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:29.415719+0000", 2023-09-23T21:06:47.385 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.3166e-05 2023-09-23T21:06:47.385 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:06:47.385 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:06:47.385 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:06:47.386 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:06:47.386 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:06:47.386 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.35469.0:172468 8.18 8:19eada54:::benchmark_data_smithi026_301269_object61711:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e317)", 2023-09-23T21:06:47.386 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:04:28.581022+0000", 2023-09-23T21:06:47.386 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 138.75093631199999, 2023-09-23T21:06:47.386 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.83865129299999996, 2023-09-23T21:06:47.386 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:06:47.386 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:06:47.386 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:06:47.386 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.35469", 2023-09-23T21:06:47.386 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/4041424623", 2023-09-23T21:06:47.386 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 172468 2023-09-23T21:06:47.387 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:06:47.387 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:06:47.387 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:06:47.387 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:06:47.387 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:28.581022+0000", 2023-09-23T21:06:47.387 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:06:47.387 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:06:47.387 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:06:47.387 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:06:47.387 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:28.581022+0000", 2023-09-23T21:06:47.387 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:06:47.387 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:06:47.388 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:06:47.388 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:06:47.388 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:28.581021+0000", 2023-09-23T21:06:47.388 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T21:06:47.388 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:06:47.388 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:06:47.388 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:06:47.388 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:28.581026+0000", 2023-09-23T21:06:47.388 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.2349999999999999e-06 2023-09-23T21:06:47.388 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:06:47.388 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:06:47.388 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:06:47.389 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:28.581038+0000", 2023-09-23T21:06:47.389 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.1779e-05 2023-09-23T21:06:47.389 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:06:47.389 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:06:47.389 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:06:47.389 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:28.581043+0000", 2023-09-23T21:06:47.389 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.7480000000000001e-06 2023-09-23T21:06:47.389 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:06:47.389 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:06:47.389 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:06:47.389 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:28.581080+0000", 2023-09-23T21:06:47.389 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.7591e-05 2023-09-23T21:06:47.389 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:06:47.390 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:06:47.390 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:06:47.390 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:28.581092+0000", 2023-09-23T21:06:47.390 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.1747e-05 2023-09-23T21:06:47.390 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:06:47.390 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:06:47.390 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:06:47.390 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:29.407757+0000", 2023-09-23T21:06:47.390 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.82666530000000005 2023-09-23T21:06:47.390 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:06:47.390 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:06:47.390 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:06:47.391 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:29.407864+0000", 2023-09-23T21:06:47.391 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000106475 2023-09-23T21:06:47.391 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:06:47.391 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:06:47.391 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 3,5", 2023-09-23T21:06:47.391 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:29.408075+0000", 2023-09-23T21:06:47.391 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00021070400000000001 2023-09-23T21:06:47.391 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:06:47.391 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:06:47.391 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:06:47.391 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:29.415555+0000", 2023-09-23T21:06:47.391 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.007480815 2023-09-23T21:06:47.391 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:06:47.392 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:06:47.392 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:06:47.392 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:29.415756+0000", 2023-09-23T21:06:47.392 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000200838 2023-09-23T21:06:47.392 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:06:47.392 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:06:47.392 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:06:47.392 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:29.419577+0000", 2023-09-23T21:06:47.392 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0038211569999999999 2023-09-23T21:06:47.392 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:06:47.392 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:06:47.392 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:06:47.393 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:29.419629+0000", 2023-09-23T21:06:47.393 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.1646e-05 2023-09-23T21:06:47.393 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:06:47.393 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:06:47.393 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:06:47.393 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:29.419674+0000", 2023-09-23T21:06:47.393 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.4481999999999999e-05 2023-09-23T21:06:47.393 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:06:47.393 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:06:47.393 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:06:47.393 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:06:47.393 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:06:47.394 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.35469.0:172485 8.18 8:1dd2dd78:::benchmark_data_smithi026_301269_object61728:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e317)", 2023-09-23T21:06:47.394 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:04:28.602617+0000", 2023-09-23T21:06:47.394 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 138.72934151499999, 2023-09-23T21:06:47.394 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.81713416000000005, 2023-09-23T21:06:47.394 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:06:47.394 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:06:47.394 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:06:47.394 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.35469", 2023-09-23T21:06:47.394 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/4041424623", 2023-09-23T21:06:47.394 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 172485 2023-09-23T21:06:47.394 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:06:47.394 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:06:47.395 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:06:47.395 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:06:47.395 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:28.602617+0000", 2023-09-23T21:06:47.395 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:06:47.395 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:06:47.395 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:06:47.395 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:06:47.395 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:28.602617+0000", 2023-09-23T21:06:47.395 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:06:47.395 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:06:47.395 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:06:47.395 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:06:47.396 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:28.602616+0000", 2023-09-23T21:06:47.396 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T21:06:47.396 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:06:47.396 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:06:47.396 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:06:47.396 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:28.602622+0000", 2023-09-23T21:06:47.396 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.7590000000000003e-06 2023-09-23T21:06:47.396 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:06:47.396 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:06:47.396 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:06:47.396 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:28.602634+0000", 2023-09-23T21:06:47.396 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.2655e-05 2023-09-23T21:06:47.396 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:06:47.397 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:06:47.397 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:06:47.397 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:28.602641+0000", 2023-09-23T21:06:47.397 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.4629999999999998e-06 2023-09-23T21:06:47.397 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:06:47.397 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:06:47.397 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:06:47.397 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:28.603804+0000", 2023-09-23T21:06:47.397 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0011636750000000001 2023-09-23T21:06:47.397 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:06:47.397 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:06:47.397 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:06:47.398 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:28.603813+0000", 2023-09-23T21:06:47.398 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.3780000000000001e-06 2023-09-23T21:06:47.398 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:06:47.398 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:06:47.398 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:06:47.398 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:29.409257+0000", 2023-09-23T21:06:47.398 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.80544433800000004 2023-09-23T21:06:47.398 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:06:47.398 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:06:47.398 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:06:47.398 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:29.409524+0000", 2023-09-23T21:06:47.398 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00026692799999999999 2023-09-23T21:06:47.399 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:06:47.399 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:06:47.399 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 3,5", 2023-09-23T21:06:47.399 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:29.409682+0000", 2023-09-23T21:06:47.399 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00015761 2023-09-23T21:06:47.399 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:06:47.399 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:06:47.399 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:06:47.399 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:29.415565+0000", 2023-09-23T21:06:47.399 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0058839030000000002 2023-09-23T21:06:47.399 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:06:47.399 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:06:47.399 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:06:47.400 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:29.415794+0000", 2023-09-23T21:06:47.400 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00022896099999999999 2023-09-23T21:06:47.400 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:06:47.400 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:06:47.400 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:06:47.400 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:29.419701+0000", 2023-09-23T21:06:47.400 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0039063769999999999 2023-09-23T21:06:47.400 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:06:47.400 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:06:47.400 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:06:47.400 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:29.419728+0000", 2023-09-23T21:06:47.400 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.6852e-05 2023-09-23T21:06:47.401 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:06:47.401 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:06:47.401 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:06:47.401 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:29.419751+0000", 2023-09-23T21:06:47.401 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.3513999999999999e-05 2023-09-23T21:06:47.401 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:06:47.401 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:06:47.401 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:06:47.401 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:06:47.401 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:06:47.401 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.35469.0:197936 8.4 8:20e4aa53:::benchmark_data_smithi026_301269_object87179:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e330)", 2023-09-23T21:06:47.401 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:05:01.418440+0000", 2023-09-23T21:06:47.402 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 105.913518079, 2023-09-23T21:06:47.402 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.88711867600000005, 2023-09-23T21:06:47.402 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:06:47.402 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:06:47.402 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:06:47.402 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.35469", 2023-09-23T21:06:47.402 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/4041424623", 2023-09-23T21:06:47.402 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 197936 2023-09-23T21:06:47.402 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:06:47.402 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:06:47.402 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:06:47.403 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:06:47.403 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:01.418440+0000", 2023-09-23T21:06:47.403 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:06:47.403 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:06:47.403 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:06:47.403 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:06:47.403 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:01.418440+0000", 2023-09-23T21:06:47.403 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:06:47.403 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:06:47.403 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:06:47.403 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:06:47.403 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:01.418439+0000", 2023-09-23T21:06:47.403 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T21:06:47.404 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:06:47.404 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:06:47.404 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:06:47.404 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:01.418445+0000", 2023-09-23T21:06:47.404 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.3740000000000003e-06 2023-09-23T21:06:47.404 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:06:47.404 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:06:47.404 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:06:47.404 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:01.418458+0000", 2023-09-23T21:06:47.404 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.2938e-05 2023-09-23T21:06:47.404 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:06:47.404 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:06:47.404 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:06:47.405 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:01.418464+0000", 2023-09-23T21:06:47.405 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.0580000000000003e-06 2023-09-23T21:06:47.405 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:06:47.405 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:06:47.405 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:06:47.405 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:01.418507+0000", 2023-09-23T21:06:47.405 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.2752e-05 2023-09-23T21:06:47.405 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:06:47.405 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:06:47.405 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:06:47.405 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:01.418514+0000", 2023-09-23T21:06:47.405 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.6329999999999999e-06 2023-09-23T21:06:47.405 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:06:47.406 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:06:47.406 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:06:47.406 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:02.294859+0000", 2023-09-23T21:06:47.406 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.87634557300000004 2023-09-23T21:06:47.406 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:06:47.406 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:06:47.406 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:06:47.406 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:02.294965+0000", 2023-09-23T21:06:47.406 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00010594000000000001 2023-09-23T21:06:47.406 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:06:47.406 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:06:47.406 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 2,6", 2023-09-23T21:06:47.407 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:02.295386+0000", 2023-09-23T21:06:47.407 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000420908 2023-09-23T21:06:47.407 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:06:47.407 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:06:47.407 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:06:47.407 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:02.300182+0000", 2023-09-23T21:06:47.407 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0047958159999999996 2023-09-23T21:06:47.407 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:06:47.407 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:06:47.407 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:06:47.407 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:02.305430+0000", 2023-09-23T21:06:47.407 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0052482220000000003 2023-09-23T21:06:47.408 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:06:47.408 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:06:47.408 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:06:47.408 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:02.305463+0000", 2023-09-23T21:06:47.408 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.2694999999999998e-05 2023-09-23T21:06:47.408 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:06:47.408 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:06:47.408 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:06:47.408 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:02.305507+0000", 2023-09-23T21:06:47.408 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.4524999999999998e-05 2023-09-23T21:06:47.408 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:06:47.408 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:06:47.408 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:06:47.409 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:02.305559+0000", 2023-09-23T21:06:47.409 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.1663999999999999e-05 2023-09-23T21:06:47.409 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:06:47.409 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:06:47.409 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:06:47.409 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:06:47.409 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:06:47.409 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.35469.0:197976 8.4 8:262cb904:::benchmark_data_smithi026_301269_object87219:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e330)", 2023-09-23T21:06:47.409 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:05:01.473672+0000", 2023-09-23T21:06:47.409 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 105.858286409, 2023-09-23T21:06:47.409 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.83213024899999999, 2023-09-23T21:06:47.409 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:06:47.410 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:06:47.410 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:06:47.410 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.35469", 2023-09-23T21:06:47.410 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/4041424623", 2023-09-23T21:06:47.410 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 197976 2023-09-23T21:06:47.410 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:06:47.410 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:06:47.410 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:06:47.410 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:06:47.410 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:01.473672+0000", 2023-09-23T21:06:47.410 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:06:47.410 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:06:47.411 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:06:47.411 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:06:47.411 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:01.473672+0000", 2023-09-23T21:06:47.411 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:06:47.411 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:06:47.411 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:06:47.411 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:06:47.411 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:01.473672+0000", 2023-09-23T21:06:47.411 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999995 2023-09-23T21:06:47.411 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:06:47.411 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:06:47.412 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:06:47.412 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:01.473675+0000", 2023-09-23T21:06:47.412 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.0699999999999998e-06 2023-09-23T21:06:47.412 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:06:47.412 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:06:47.412 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:06:47.412 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:01.473682+0000", 2023-09-23T21:06:47.412 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.9450000000000004e-06 2023-09-23T21:06:47.412 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:06:47.412 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:06:47.412 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:06:47.412 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:01.473686+0000", 2023-09-23T21:06:47.412 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.493e-06 2023-09-23T21:06:47.413 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:06:47.413 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:06:47.413 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:06:47.413 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:01.473714+0000", 2023-09-23T21:06:47.413 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.7662999999999999e-05 2023-09-23T21:06:47.413 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:06:47.413 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:06:47.413 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:06:47.413 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:01.473723+0000", 2023-09-23T21:06:47.413 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.8410000000000006e-06 2023-09-23T21:06:47.413 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:06:47.413 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:06:47.414 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:06:47.414 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:02.297745+0000", 2023-09-23T21:06:47.414 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.82402238900000002 2023-09-23T21:06:47.414 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:06:47.414 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:06:47.414 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:06:47.414 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:02.297862+0000", 2023-09-23T21:06:47.414 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00011700400000000001 2023-09-23T21:06:47.414 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:06:47.414 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:06:47.414 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 2,6", 2023-09-23T21:06:47.414 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:02.298175+0000", 2023-09-23T21:06:47.415 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00031293199999999998 2023-09-23T21:06:47.415 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:06:47.415 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:06:47.415 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:06:47.415 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:02.305369+0000", 2023-09-23T21:06:47.415 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0071940939999999998 2023-09-23T21:06:47.415 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:06:47.415 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:06:47.415 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:06:47.415 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:02.305660+0000", 2023-09-23T21:06:47.415 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00029076599999999997 2023-09-23T21:06:47.415 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:06:47.415 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:06:47.416 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:06:47.416 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:02.305705+0000", 2023-09-23T21:06:47.416 INFO:teuthology.orchestra.run.smithi026.stderr:nodeep-scrub is unset 2023-09-23T21:06:47.417 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.5556999999999998e-05 2023-09-23T21:06:47.417 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:06:47.417 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:06:47.417 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:06:47.417 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:02.305752+0000", 2023-09-23T21:06:47.417 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.6660999999999999e-05 2023-09-23T21:06:47.418 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:06:47.418 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:06:47.418 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:06:47.418 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:02.305802+0000", 2023-09-23T21:06:47.418 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.0346000000000002e-05 2023-09-23T21:06:47.418 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:06:47.418 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:06:47.418 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:06:47.418 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:06:47.418 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:06:47.418 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.35469.0:197986 8.4 8:24c1b6a6:::benchmark_data_smithi026_301269_object87229:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e330)", 2023-09-23T21:06:47.419 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:05:01.484363+0000", 2023-09-23T21:06:47.419 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 105.84759532699999, 2023-09-23T21:06:47.419 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.82409744600000001, 2023-09-23T21:06:47.419 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:06:47.419 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:06:47.419 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:06:47.419 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.35469", 2023-09-23T21:06:47.419 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/4041424623", 2023-09-23T21:06:47.419 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 197986 2023-09-23T21:06:47.419 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:06:47.419 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:06:47.419 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:06:47.420 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:06:47.420 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:01.484363+0000", 2023-09-23T21:06:47.420 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:06:47.420 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:06:47.420 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:06:47.420 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:06:47.420 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:01.484363+0000", 2023-09-23T21:06:47.420 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:06:47.420 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:06:47.420 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:06:47.420 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:06:47.421 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:01.484362+0000", 2023-09-23T21:06:47.421 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.999999 2023-09-23T21:06:47.421 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:06:47.421 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:06:47.421 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:06:47.421 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:01.484367+0000", 2023-09-23T21:06:47.421 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.8369999999999996e-06 2023-09-23T21:06:47.421 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:06:47.421 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:06:47.421 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:06:47.421 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:01.484376+0000", 2023-09-23T21:06:47.421 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.7029999999999995e-06 2023-09-23T21:06:47.422 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:06:47.422 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:06:47.422 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:06:47.422 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:01.484381+0000", 2023-09-23T21:06:47.422 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.7709999999999997e-06 2023-09-23T21:06:47.422 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:06:47.422 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:06:47.422 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:06:47.422 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:01.484422+0000", 2023-09-23T21:06:47.422 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.1618e-05 2023-09-23T21:06:47.422 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:06:47.423 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:06:47.423 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:06:47.423 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:01.484433+0000", 2023-09-23T21:06:47.423 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.0735e-05 2023-09-23T21:06:47.423 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:06:47.423 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:06:47.423 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:06:47.423 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:02.300211+0000", 2023-09-23T21:06:47.423 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.81577823400000005 2023-09-23T21:06:47.423 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:06:47.423 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:06:47.423 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:06:47.423 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:02.300326+0000", 2023-09-23T21:06:47.424 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000115337 2023-09-23T21:06:47.424 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:06:47.424 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:06:47.424 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 2,6", 2023-09-23T21:06:47.424 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:02.300630+0000", 2023-09-23T21:06:47.424 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00030335399999999998 2023-09-23T21:06:47.424 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:06:47.424 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:06:47.424 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:06:47.424 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:02.305420+0000", 2023-09-23T21:06:47.424 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0047902329999999996 2023-09-23T21:06:47.424 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:06:47.424 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:06:47.425 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:06:47.425 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:02.305669+0000", 2023-09-23T21:06:47.425 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00024875299999999999 2023-09-23T21:06:47.425 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:06:47.425 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:06:47.425 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:06:47.425 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:02.308400+0000", 2023-09-23T21:06:47.425 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0027312730000000002 2023-09-23T21:06:47.425 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:06:47.425 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:06:47.425 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:06:47.425 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:02.308432+0000", 2023-09-23T21:06:47.426 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.1948e-05 2023-09-23T21:06:47.426 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:06:47.426 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:06:47.426 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:06:47.426 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:02.308461+0000", 2023-09-23T21:06:47.426 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.8651000000000001e-05 2023-09-23T21:06:47.426 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:06:47.426 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:06:47.426 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:06:47.426 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:06:47.426 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:06:47.426 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.35469.0:198190 8.19 8.47097259 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e331)", 2023-09-23T21:06:47.427 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:05:02.366655+0000", 2023-09-23T21:06:47.427 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 104.96530379399999, 2023-09-23T21:06:47.427 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.92870424399999996, 2023-09-23T21:06:47.427 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:06:47.427 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "reached pg", 2023-09-23T21:06:47.427 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:06:47.427 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.35469", 2023-09-23T21:06:47.427 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/4041424623", 2023-09-23T21:06:47.427 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 198190 2023-09-23T21:06:47.427 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:06:47.427 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:06:47.428 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:06:47.428 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:06:47.428 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:02.366655+0000", 2023-09-23T21:06:47.428 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:06:47.428 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:06:47.428 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:06:47.428 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:06:47.428 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:02.366655+0000", 2023-09-23T21:06:47.428 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:06:47.428 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:06:47.428 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:06:47.428 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:06:47.428 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:02.366654+0000", 2023-09-23T21:06:47.429 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.999999 2023-09-23T21:06:47.429 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:06:47.429 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:06:47.429 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:06:47.429 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:02.366659+0000", 2023-09-23T21:06:47.429 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.3970000000000002e-06 2023-09-23T21:06:47.429 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:06:47.429 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:06:47.429 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:06:47.429 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:02.366673+0000", 2023-09-23T21:06:47.429 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.3706000000000001e-05 2023-09-23T21:06:47.429 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:06:47.430 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:06:47.430 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:06:47.430 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:02.366696+0000", 2023-09-23T21:06:47.430 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.3054999999999999e-05 2023-09-23T21:06:47.430 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:06:47.430 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:06:47.430 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:06:47.430 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:02.366740+0000", 2023-09-23T21:06:47.430 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.4297e-05 2023-09-23T21:06:47.430 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:06:47.430 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:06:47.430 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:06:47.431 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:02.366748+0000", 2023-09-23T21:06:47.431 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.3529999999999995e-06 2023-09-23T21:06:47.431 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:06:47.431 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:06:47.431 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:06:47.431 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:03.295349+0000", 2023-09-23T21:06:47.431 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.928600855 2023-09-23T21:06:47.431 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:06:47.431 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:06:47.431 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:06:47.431 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:03.295359+0000", 2023-09-23T21:06:47.431 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.6670000000000008e-06 2023-09-23T21:06:47.432 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:06:47.432 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:06:47.432 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:06:47.432 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:06:47.432 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:06:47.432 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.35469.0:198191 8.19 8.9474e8d9 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e331)", 2023-09-23T21:06:47.432 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:05:02.366711+0000", 2023-09-23T21:06:47.432 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 104.965247793, 2023-09-23T21:06:47.432 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.92867729499999996, 2023-09-23T21:06:47.432 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:06:47.432 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "reached pg", 2023-09-23T21:06:47.432 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:06:47.433 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.35469", 2023-09-23T21:06:47.433 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/4041424623", 2023-09-23T21:06:47.433 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 198191 2023-09-23T21:06:47.433 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:06:47.433 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:06:47.433 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:06:47.433 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:06:47.433 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:02.366711+0000", 2023-09-23T21:06:47.433 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:06:47.433 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:06:47.433 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:06:47.433 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:06:47.434 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:02.366711+0000", 2023-09-23T21:06:47.434 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:06:47.434 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:06:47.434 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:06:47.434 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:06:47.434 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:02.366710+0000", 2023-09-23T21:06:47.434 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999995 2023-09-23T21:06:47.434 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:06:47.434 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:06:47.434 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:06:47.434 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:02.366713+0000", 2023-09-23T21:06:47.434 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.9160000000000001e-06 2023-09-23T21:06:47.435 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:06:47.435 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:06:47.435 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:06:47.435 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:02.366720+0000", 2023-09-23T21:06:47.435 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.9229999999999996e-06 2023-09-23T21:06:47.435 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:06:47.435 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:06:47.435 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:06:47.435 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:02.366725+0000", 2023-09-23T21:06:47.435 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.8110000000000002e-06 2023-09-23T21:06:47.435 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:06:47.435 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:06:47.436 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:06:47.436 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:02.368872+0000", 2023-09-23T21:06:47.436 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.002146677 2023-09-23T21:06:47.436 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:06:47.436 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:06:47.436 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:06:47.436 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:02.368878+0000", 2023-09-23T21:06:47.436 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.0619999999999997e-06 2023-09-23T21:06:47.436 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:06:47.436 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:06:47.436 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:06:47.436 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:03.295381+0000", 2023-09-23T21:06:47.437 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.926503352 2023-09-23T21:06:47.437 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:06:47.437 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:06:47.437 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:06:47.437 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:03.295388+0000", 2023-09-23T21:06:47.437 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.9929999999999998e-06 2023-09-23T21:06:47.437 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:06:47.437 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:06:47.437 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:06:47.437 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:06:47.437 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:06:47.437 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.35469.0:198201 8.19 8.fa337379 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e331)", 2023-09-23T21:06:47.437 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:05:02.369766+0000", 2023-09-23T21:06:47.438 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 104.96219220499999, 2023-09-23T21:06:47.438 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.92565394300000003, 2023-09-23T21:06:47.438 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:06:47.438 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "reached pg", 2023-09-23T21:06:47.438 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:06:47.438 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.35469", 2023-09-23T21:06:47.438 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/4041424623", 2023-09-23T21:06:47.438 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 198201 2023-09-23T21:06:47.438 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:06:47.438 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:06:47.438 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:06:47.439 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:06:47.439 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:02.369766+0000", 2023-09-23T21:06:47.439 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:06:47.439 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:06:47.439 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:06:47.439 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:06:47.439 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:02.369766+0000", 2023-09-23T21:06:47.439 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:06:47.439 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:06:47.439 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:06:47.439 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:06:47.439 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:02.369766+0000", 2023-09-23T21:06:47.439 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999995 2023-09-23T21:06:47.440 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:06:47.440 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:06:47.440 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:06:47.440 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:02.369768+0000", 2023-09-23T21:06:47.440 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.5320000000000001e-06 2023-09-23T21:06:47.440 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:06:47.440 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:06:47.440 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:06:47.440 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:02.369781+0000", 2023-09-23T21:06:47.440 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.2091e-05 2023-09-23T21:06:47.440 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:06:47.440 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:06:47.441 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:06:47.441 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:02.369785+0000", 2023-09-23T21:06:47.441 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.498e-06 2023-09-23T21:06:47.441 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:06:47.441 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:06:47.441 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:06:47.441 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:02.369826+0000", 2023-09-23T21:06:47.441 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.1179e-05 2023-09-23T21:06:47.441 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:06:47.441 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:06:47.441 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:06:47.441 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:02.369833+0000", 2023-09-23T21:06:47.442 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.6719999999999998e-06 2023-09-23T21:06:47.442 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:06:47.442 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:06:47.442 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:06:47.442 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:03.295412+0000", 2023-09-23T21:06:47.442 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.92557892600000002 2023-09-23T21:06:47.442 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:06:47.442 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:06:47.442 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:06:47.442 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:03.295420+0000", 2023-09-23T21:06:47.442 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.4579999999999993e-06 2023-09-23T21:06:47.442 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:06:47.443 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:06:47.443 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:06:47.443 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:06:47.443 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:06:47.443 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.35469.0:198229 8.19 8.92797f39 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e331)", 2023-09-23T21:06:47.443 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:05:02.420904+0000", 2023-09-23T21:06:47.443 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 104.91105406, 2023-09-23T21:06:47.443 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.87453987099999997, 2023-09-23T21:06:47.443 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:06:47.443 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "reached pg", 2023-09-23T21:06:47.443 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:06:47.443 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.35469", 2023-09-23T21:06:47.444 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/4041424623", 2023-09-23T21:06:47.444 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 198229 2023-09-23T21:06:47.444 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:06:47.444 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:06:47.444 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:06:47.444 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:06:47.444 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:02.420904+0000", 2023-09-23T21:06:47.444 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:06:47.444 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:06:47.444 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:06:47.444 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:06:47.444 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:02.420904+0000", 2023-09-23T21:06:47.445 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:06:47.445 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:06:47.445 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:06:47.445 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:06:47.445 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:02.420903+0000", 2023-09-23T21:06:47.445 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T21:06:47.445 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:06:47.445 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:06:47.445 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:06:47.445 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:02.420909+0000", 2023-09-23T21:06:47.445 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.3689999999999998e-06 2023-09-23T21:06:47.445 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:06:47.446 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:06:47.446 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:06:47.446 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:02.420921+0000", 2023-09-23T21:06:47.446 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.2771e-05 2023-09-23T21:06:47.446 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:06:47.446 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:06:47.446 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:06:47.446 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:02.420928+0000", 2023-09-23T21:06:47.446 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.0679999999999999e-06 2023-09-23T21:06:47.446 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:06:47.446 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:06:47.446 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:06:47.447 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:02.420973+0000", 2023-09-23T21:06:47.447 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.4675000000000002e-05 2023-09-23T21:06:47.447 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:06:47.447 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:06:47.447 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:06:47.447 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:02.420982+0000", 2023-09-23T21:06:47.447 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.4999999999999999e-06 2023-09-23T21:06:47.447 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:06:47.447 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:06:47.447 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:06:47.447 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:03.295437+0000", 2023-09-23T21:06:47.447 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.87445572000000005 2023-09-23T21:06:47.448 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:06:47.448 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:06:47.448 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:06:47.448 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:03.295444+0000", 2023-09-23T21:06:47.448 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.9739999999999996e-06 2023-09-23T21:06:47.448 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:06:47.448 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:06:47.448 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:06:47.448 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:06:47.448 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:06:47.448 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.35469.0:198232 8.19 8.bb8c9e39 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e331)", 2023-09-23T21:06:47.448 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:05:02.420941+0000", 2023-09-23T21:06:47.449 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 104.911017876, 2023-09-23T21:06:47.449 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.87452782500000004, 2023-09-23T21:06:47.449 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:06:47.449 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "reached pg", 2023-09-23T21:06:47.449 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:06:47.449 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.35469", 2023-09-23T21:06:47.449 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/4041424623", 2023-09-23T21:06:47.449 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 198232 2023-09-23T21:06:47.449 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:06:47.449 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:06:47.449 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:06:47.449 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:06:47.450 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:02.420941+0000", 2023-09-23T21:06:47.450 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:06:47.450 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:06:47.450 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:06:47.450 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:06:47.450 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:02.420941+0000", 2023-09-23T21:06:47.450 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:06:47.450 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:06:47.450 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:06:47.450 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:06:47.450 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:02.420940+0000", 2023-09-23T21:06:47.450 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999995 2023-09-23T21:06:47.451 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:06:47.451 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:06:47.451 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:06:47.451 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:02.420943+0000", 2023-09-23T21:06:47.451 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.5830000000000001e-06 2023-09-23T21:06:47.451 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:06:47.451 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:06:47.451 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:06:47.451 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:02.420948+0000", 2023-09-23T21:06:47.451 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.1760000000000001e-06 2023-09-23T21:06:47.451 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:06:47.451 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:06:47.452 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:06:47.452 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:02.420951+0000", 2023-09-23T21:06:47.452 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.3239999999999999e-06 2023-09-23T21:06:47.452 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:06:47.452 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:06:47.452 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:06:47.452 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:02.421010+0000", 2023-09-23T21:06:47.452 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.9104999999999997e-05 2023-09-23T21:06:47.452 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:06:47.452 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:06:47.452 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:06:47.452 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:02.421016+0000", 2023-09-23T21:06:47.453 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.4029999999999998e-06 2023-09-23T21:06:47.453 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:06:47.453 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:06:47.453 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:06:47.453 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:03.295461+0000", 2023-09-23T21:06:47.453 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.87444563500000005 2023-09-23T21:06:47.453 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:06:47.453 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:06:47.453 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:06:47.453 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:03.295468+0000", 2023-09-23T21:06:47.453 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.1029999999999996e-06 2023-09-23T21:06:47.453 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:06:47.454 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:06:47.454 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:06:47.454 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:06:47.454 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:06:47.454 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.35856.0:18003 9.e 9:75bab473:::benchmark_data_smithi026_342732_object18002:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e351)", 2023-09-23T21:06:47.454 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:06:02.903342+0000", 2023-09-23T21:06:47.454 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 44.428616577, 2023-09-23T21:06:47.454 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.066989517999999998, 2023-09-23T21:06:47.454 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:06:47.454 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:06:47.454 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:06:47.455 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.35856", 2023-09-23T21:06:47.455 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/4180396258", 2023-09-23T21:06:47.455 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 18003 2023-09-23T21:06:47.455 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:06:47.455 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:06:47.455 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:06:47.455 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:06:47.455 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:02.903342+0000", 2023-09-23T21:06:47.455 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:06:47.455 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:06:47.455 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:06:47.456 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:06:47.456 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:02.903342+0000", 2023-09-23T21:06:47.456 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:06:47.456 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:06:47.456 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:06:47.456 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:06:47.456 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:02.903341+0000", 2023-09-23T21:06:47.456 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T21:06:47.456 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:06:47.456 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:06:47.456 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:06:47.456 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:02.903372+0000", 2023-09-23T21:06:47.456 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.1248000000000003e-05 2023-09-23T21:06:47.457 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:06:47.457 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:06:47.457 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:06:47.457 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:02.903384+0000", 2023-09-23T21:06:47.457 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.1864e-05 2023-09-23T21:06:47.457 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:06:47.457 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:06:47.457 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:06:47.457 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:02.903388+0000", 2023-09-23T21:06:47.457 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.6360000000000002e-06 2023-09-23T21:06:47.457 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:06:47.457 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:06:47.458 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:06:47.458 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:02.903431+0000", 2023-09-23T21:06:47.458 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.3161000000000002e-05 2023-09-23T21:06:47.458 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:06:47.458 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:06:47.458 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:06:47.458 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:02.903550+0000", 2023-09-23T21:06:47.458 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00011866000000000001 2023-09-23T21:06:47.458 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:06:47.458 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:06:47.458 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 1,7", 2023-09-23T21:06:47.458 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:02.903690+0000", 2023-09-23T21:06:47.459 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000139808 2023-09-23T21:06:47.459 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:06:47.459 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:06:47.459 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:06:47.459 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:02.905354+0000", 2023-09-23T21:06:47.459 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.001664532 2023-09-23T21:06:47.459 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:06:47.459 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:06:47.459 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:06:47.459 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:02.906166+0000", 2023-09-23T21:06:47.459 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00081162699999999999 2023-09-23T21:06:47.459 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:06:47.460 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:06:47.460 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:06:47.460 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:02.970247+0000", 2023-09-23T21:06:47.460 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.064080654000000001 2023-09-23T21:06:47.460 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:06:47.460 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:06:47.460 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:06:47.460 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:02.970303+0000", 2023-09-23T21:06:47.460 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.6168000000000002e-05 2023-09-23T21:06:47.460 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:06:47.460 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:06:47.460 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:06:47.461 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:02.970331+0000", 2023-09-23T21:06:47.461 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.8592999999999999e-05 2023-09-23T21:06:47.461 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:06:47.461 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:06:47.461 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:06:47.461 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:06:47.461 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:06:47.461 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.35856.0:18007 9.e 9:756187b0:::benchmark_data_smithi026_342732_object18006:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e351)", 2023-09-23T21:06:47.461 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:06:02.903713+0000", 2023-09-23T21:06:47.461 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 44.428245834000002, 2023-09-23T21:06:47.461 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.066775264000000001, 2023-09-23T21:06:47.461 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:06:47.462 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:06:47.462 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:06:47.462 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.35856", 2023-09-23T21:06:47.462 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/4180396258", 2023-09-23T21:06:47.462 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 18007 2023-09-23T21:06:47.462 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:06:47.462 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:06:47.462 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:06:47.462 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:06:47.462 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:02.903713+0000", 2023-09-23T21:06:47.462 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:06:47.462 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:06:47.463 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:06:47.463 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:06:47.463 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:02.903713+0000", 2023-09-23T21:06:47.463 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:06:47.463 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:06:47.463 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:06:47.463 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:06:47.463 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:02.903712+0000", 2023-09-23T21:06:47.463 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.999999 2023-09-23T21:06:47.463 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:06:47.463 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:06:47.463 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:06:47.464 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:02.903822+0000", 2023-09-23T21:06:47.464 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00011025499999999999 2023-09-23T21:06:47.464 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:06:47.464 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:06:47.464 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:06:47.464 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:02.903832+0000", 2023-09-23T21:06:47.464 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.5440000000000004e-06 2023-09-23T21:06:47.464 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:06:47.464 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:06:47.464 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:06:47.464 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:02.903840+0000", 2023-09-23T21:06:47.464 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.8359999999999992e-06 2023-09-23T21:06:47.465 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:06:47.465 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:06:47.465 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:06:47.465 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:02.904458+0000", 2023-09-23T21:06:47.465 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00061893400000000002 2023-09-23T21:06:47.465 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:06:47.465 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:06:47.465 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:06:47.465 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:02.904572+0000", 2023-09-23T21:06:47.465 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000113484 2023-09-23T21:06:47.465 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:06:47.465 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:06:47.465 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 1,7", 2023-09-23T21:06:47.466 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:02.904712+0000", 2023-09-23T21:06:47.466 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000139713 2023-09-23T21:06:47.466 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:06:47.466 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:06:47.466 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:06:47.466 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:02.905524+0000", 2023-09-23T21:06:47.466 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00081235200000000002 2023-09-23T21:06:47.466 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:06:47.466 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:06:47.466 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:06:47.466 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:02.906258+0000", 2023-09-23T21:06:47.467 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00073409899999999999 2023-09-23T21:06:47.467 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:06:47.467 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:06:47.467 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:06:47.467 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:02.970390+0000", 2023-09-23T21:06:47.467 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.064132236999999995 2023-09-23T21:06:47.467 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:06:47.467 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:06:47.467 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:06:47.467 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:02.970443+0000", 2023-09-23T21:06:47.467 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.2981e-05 2023-09-23T21:06:47.467 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:06:47.467 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:06:47.468 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:06:47.468 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:02.970488+0000", 2023-09-23T21:06:47.468 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.4672000000000002e-05 2023-09-23T21:06:47.468 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:06:47.468 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:06:47.468 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:06:47.468 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:06:47.468 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:06:47.468 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.35856.0:18009 9.e 9:7db91d6c:::benchmark_data_smithi026_342732_object18008:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e351)", 2023-09-23T21:06:47.468 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:06:02.906149+0000", 2023-09-23T21:06:47.468 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 44.425809291, 2023-09-23T21:06:47.468 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.064485201000000006, 2023-09-23T21:06:47.469 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:06:47.469 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:06:47.469 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:06:47.469 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.35856", 2023-09-23T21:06:47.469 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/4180396258", 2023-09-23T21:06:47.469 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 18009 2023-09-23T21:06:47.469 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:06:47.469 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:06:47.469 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:06:47.469 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:06:47.469 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:02.906149+0000", 2023-09-23T21:06:47.469 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:06:47.470 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:06:47.470 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:06:47.470 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:06:47.470 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:02.906149+0000", 2023-09-23T21:06:47.470 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:06:47.470 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:06:47.470 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:06:47.470 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:06:47.470 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:02.906147+0000", 2023-09-23T21:06:47.470 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999981 2023-09-23T21:06:47.470 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:06:47.470 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:06:47.471 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:06:47.471 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:02.906198+0000", 2023-09-23T21:06:47.471 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.0232999999999998e-05 2023-09-23T21:06:47.471 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:06:47.471 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:06:47.471 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:06:47.471 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:02.906210+0000", 2023-09-23T21:06:47.471 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.2091e-05 2023-09-23T21:06:47.471 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:06:47.471 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:06:47.471 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:06:47.471 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:02.906216+0000", 2023-09-23T21:06:47.472 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.3040000000000001e-06 2023-09-23T21:06:47.472 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:06:47.472 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:06:47.472 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:06:47.472 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:02.906277+0000", 2023-09-23T21:06:47.472 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.1022e-05 2023-09-23T21:06:47.472 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:06:47.472 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:06:47.472 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:06:47.472 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:02.906381+0000", 2023-09-23T21:06:47.472 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000103628 2023-09-23T21:06:47.472 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:06:47.472 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:06:47.473 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 1,7", 2023-09-23T21:06:47.473 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:02.906477+0000", 2023-09-23T21:06:47.473 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.5938000000000002e-05 2023-09-23T21:06:47.473 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:06:47.473 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:06:47.473 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:06:47.473 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:02.907125+0000", 2023-09-23T21:06:47.473 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00064823099999999998 2023-09-23T21:06:47.473 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:06:47.473 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:06:47.473 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:06:47.473 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:02.907252+0000", 2023-09-23T21:06:47.474 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000127563 2023-09-23T21:06:47.474 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:06:47.474 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:06:47.474 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:06:47.474 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:02.970539+0000", 2023-09-23T21:06:47.474 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.063286707999999997 2023-09-23T21:06:47.474 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:06:47.474 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:06:47.474 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:06:47.474 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:02.970590+0000", 2023-09-23T21:06:47.474 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.0612000000000003e-05 2023-09-23T21:06:47.474 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:06:47.475 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:06:47.475 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:06:47.475 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:02.970634+0000", 2023-09-23T21:06:47.475 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.4709999999999997e-05 2023-09-23T21:06:47.475 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:06:47.475 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:06:47.475 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:06:47.475 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:06:47.475 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:06:47.475 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.35856.0:28272 9.e 9:74dfd733:::benchmark_data_smithi026_342732_object28271:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e351)", 2023-09-23T21:06:47.475 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:06:07.721004+0000", 2023-09-23T21:06:47.475 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 39.610954215, 2023-09-23T21:06:47.476 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.063824743000000003, 2023-09-23T21:06:47.476 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:06:47.476 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:06:47.476 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:06:47.476 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.35856", 2023-09-23T21:06:47.476 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/4180396258", 2023-09-23T21:06:47.476 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 28272 2023-09-23T21:06:47.476 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:06:47.476 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:06:47.476 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:06:47.476 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:06:47.476 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:07.721004+0000", 2023-09-23T21:06:47.477 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:06:47.477 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:06:47.477 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:06:47.477 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:06:47.477 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:07.721004+0000", 2023-09-23T21:06:47.477 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:06:47.477 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:06:47.477 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:06:47.477 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:06:47.477 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:07.721003+0000", 2023-09-23T21:06:47.477 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999981 2023-09-23T21:06:47.477 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:06:47.478 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:06:47.478 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:06:47.478 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:07.721042+0000", 2023-09-23T21:06:47.478 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.9597000000000001e-05 2023-09-23T21:06:47.478 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:06:47.478 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:06:47.478 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:06:47.478 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:07.721060+0000", 2023-09-23T21:06:47.478 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.7790000000000001e-05 2023-09-23T21:06:47.478 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:06:47.478 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:06:47.478 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:06:47.478 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:07.721067+0000", 2023-09-23T21:06:47.479 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.6089999999999997e-06 2023-09-23T21:06:47.479 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:06:47.479 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:06:47.479 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:06:47.479 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:07.722039+0000", 2023-09-23T21:06:47.479 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00097233400000000005 2023-09-23T21:06:47.479 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:06:47.479 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:06:47.479 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:06:47.479 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:07.722161+0000", 2023-09-23T21:06:47.479 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000121859 2023-09-23T21:06:47.479 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:06:47.480 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:06:47.480 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 1,7", 2023-09-23T21:06:47.480 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:07.722301+0000", 2023-09-23T21:06:47.480 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00014057699999999999 2023-09-23T21:06:47.480 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:06:47.480 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:06:47.480 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:06:47.480 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:07.726440+0000", 2023-09-23T21:06:47.480 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0041382540000000001 2023-09-23T21:06:47.480 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:06:47.480 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:06:47.480 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:06:47.481 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:07.727293+0000", 2023-09-23T21:06:47.481 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00085322900000000005 2023-09-23T21:06:47.481 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:06:47.481 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:06:47.481 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:06:47.481 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:07.784726+0000", 2023-09-23T21:06:47.481 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.057432999999999998 2023-09-23T21:06:47.481 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:06:47.481 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:06:47.481 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:06:47.481 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:07.784788+0000", 2023-09-23T21:06:47.481 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.1997999999999998e-05 2023-09-23T21:06:47.482 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:06:47.482 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:06:47.482 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:06:47.482 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:07.784829+0000", 2023-09-23T21:06:47.482 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.1193999999999999e-05 2023-09-23T21:06:47.482 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:06:47.482 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:06:47.482 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:06:47.482 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:06:47.482 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:06:47.482 INFO:teuthology.orchestra.run.smithi070.stdout:} 2023-09-23T21:06:47.483 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:06:47.483 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:06:47.484 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:06:47.584 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:06:47.621 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:06:47 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/3976550991' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:06:47.671 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:06:47 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/3976550991' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:06:47.672 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:06:47 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/3976550991' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:06:47.672 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:06:47.784 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_ops_in_flight 2023-09-23T21:06:47.831 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:06:47.838 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:06:47.839 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:06:47.939 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:06:48.000 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:06:48.062 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:06:48.066 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:06:48.067 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:06:48.167 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:06:48.254 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:06:48.282 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 52 16 80034 80018 96.0978 59.875 0.018747 0.010401 2023-09-23T21:06:48.338 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:06:48.347 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:06:48.347 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:06:48.438 INFO:teuthology.orchestra.run.smithi026.stderr:set osd.5 primary-affinity to 0.438407 (8287312) 2023-09-23T21:06:48.444 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0: {} 2023-09-23T21:06:48.448 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:06:48.487 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1: {} 2023-09-23T21:06:48.515 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:06:48 smithi026 ceph-mon[179688]: pgmap v926: 109 pgs: 109 active+clean; 4.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 71 MiB/s wr, 1.14k op/s 2023-09-23T21:06:48.515 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:06:48 smithi026 ceph-mon[179688]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T21:06:48.515 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:06:48 smithi026 ceph-mon[179688]: Cluster is now healthy 2023-09-23T21:06:48.515 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:06:48 smithi026 ceph-mon[179688]: failed to encode map e358 with expected crc 2023-09-23T21:06:48.516 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:06:48 smithi026 ceph-mon[179688]: failed to encode map e358 with expected crc 2023-09-23T21:06:48.516 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:06:48 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/3976550991' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T21:06:48.516 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:06:48 smithi026 ceph-mon[179688]: osdmap e358: 8 total, 8 up, 8 in 2023-09-23T21:06:48.516 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:06:48 smithi026 ceph-mon[179688]: failed to encode map e358 with expected crc 2023-09-23T21:06:48.516 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:06:48 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/2097086533' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 5, "weight": 0.43840672051352914}]: dispatch 2023-09-23T21:06:48.516 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:06:48 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 5, "weight": 0.43840672051352914}]: dispatch 2023-09-23T21:06:48.516 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:06:48 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/1157828598' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:06:48.516 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:06:48 smithi026 ceph-mon[175151]: pgmap v926: 109 pgs: 109 active+clean; 4.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 71 MiB/s wr, 1.14k op/s 2023-09-23T21:06:48.516 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:06:48 smithi026 ceph-mon[175151]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T21:06:48.516 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:06:48 smithi026 ceph-mon[175151]: Cluster is now healthy 2023-09-23T21:06:48.517 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:06:48 smithi026 ceph-mon[175151]: failed to encode map e358 with expected crc 2023-09-23T21:06:48.517 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:06:48 smithi026 ceph-mon[175151]: failed to encode map e358 with expected crc 2023-09-23T21:06:48.517 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:06:48 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/3976550991' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T21:06:48.517 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:06:48 smithi026 ceph-mon[175151]: osdmap e358: 8 total, 8 up, 8 in 2023-09-23T21:06:48.517 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:06:48 smithi026 ceph-mon[175151]: failed to encode map e358 with expected crc 2023-09-23T21:06:48.517 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:06:48 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/2097086533' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 5, "weight": 0.43840672051352914}]: dispatch 2023-09-23T21:06:48.517 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:06:48 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 5, "weight": 0.43840672051352914}]: dispatch 2023-09-23T21:06:48.517 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:06:48 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/1157828598' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:06:48.524 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:06:48.530 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2: {} 2023-09-23T21:06:48.546 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T21:06:48.566 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3: {} 2023-09-23T21:06:48.577 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:06:48.580 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:06:48.581 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:06:48.590 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4: {} 2023-09-23T21:06:48.608 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T21:06:48.628 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6: {} 2023-09-23T21:06:48.647 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T21:06:48.648 INFO:teuthology.orchestra.run.smithi026.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T21:06:48.648 INFO:teuthology.orchestra.run.smithi026.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T21:06:48.648 INFO:teuthology.orchestra.run.smithi026.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T21:06:48.648 INFO:teuthology.orchestra.run.smithi026.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T21:06:48.648 INFO:teuthology.orchestra.run.smithi026.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T21:06:48.649 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T21:06:48.649 INFO:teuthology.orchestra.run.smithi026.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T21:06:48.649 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T21:06:48.681 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:06:48.709 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:06:48.755 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:06:48 smithi070 ceph-mon[168282]: pgmap v926: 109 pgs: 109 active+clean; 4.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 71 MiB/s wr, 1.14k op/s 2023-09-23T21:06:48.755 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:06:48 smithi070 ceph-mon[168282]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T21:06:48.755 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:06:48 smithi070 ceph-mon[168282]: Cluster is now healthy 2023-09-23T21:06:48.755 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:06:48 smithi070 ceph-mon[168282]: failed to encode map e358 with expected crc 2023-09-23T21:06:48.755 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:06:48 smithi070 ceph-mon[168282]: failed to encode map e358 with expected crc 2023-09-23T21:06:48.756 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:06:48 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/3976550991' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T21:06:48.756 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:06:48 smithi070 ceph-mon[168282]: osdmap e358: 8 total, 8 up, 8 in 2023-09-23T21:06:48.756 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:06:48 smithi070 ceph-mon[168282]: failed to encode map e358 with expected crc 2023-09-23T21:06:48.756 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:06:48 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/2097086533' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 5, "weight": 0.43840672051352914}]: dispatch 2023-09-23T21:06:48.756 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:06:48 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 5, "weight": 0.43840672051352914}]: dispatch 2023-09-23T21:06:48.756 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:06:48 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/1157828598' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:06:49.035 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:06:49.037 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:06:49.038 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:06:49.139 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:06:49.168 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:06:49.251 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:06:49.254 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:06:49.254 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:06:49.282 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 53 16 80272 80256 94.5662 14.875 0.00381111 0.0104149 2023-09-23T21:06:49.356 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:06:49.385 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:06:49.465 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:06:49.471 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:06:49.471 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:06:49.572 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:06:49.636 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:06:49.648 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:06:49 smithi070 ceph-mon[168282]: failed to encode map e359 with expected crc 2023-09-23T21:06:49.648 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:06:49 smithi070 ceph-mon[168282]: failed to encode map e359 with expected crc 2023-09-23T21:06:49.649 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:06:49 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 5, "weight": 0.43840672051352914}]': finished 2023-09-23T21:06:49.649 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:06:49 smithi070 ceph-mon[168282]: osdmap e359: 8 total, 8 up, 8 in 2023-09-23T21:06:49.698 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:06:49.707 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:06:49.707 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:06:49 smithi026 ceph-mon[179688]: failed to encode map e359 with expected crc 2023-09-23T21:06:49.707 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:06:49 smithi026 ceph-mon[179688]: failed to encode map e359 with expected crc 2023-09-23T21:06:49.708 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:06:49 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 5, "weight": 0.43840672051352914}]': finished 2023-09-23T21:06:49.708 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:06:49 smithi026 ceph-mon[179688]: osdmap e359: 8 total, 8 up, 8 in 2023-09-23T21:06:49.708 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:06:49.709 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:06:49 smithi026 ceph-mon[175151]: failed to encode map e359 with expected crc 2023-09-23T21:06:49.709 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:06:49 smithi026 ceph-mon[175151]: failed to encode map e359 with expected crc 2023-09-23T21:06:49.709 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:06:49 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 5, "weight": 0.43840672051352914}]': finished 2023-09-23T21:06:49.709 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:06:49 smithi026 ceph-mon[175151]: osdmap e359: 8 total, 8 up, 8 in 2023-09-23T21:06:49.709 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 21:06:49 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[164029]: ::ffff:172.21.15.70 - - [23/Sep/2023:21:06:49] "GET /metrics HTTP/1.1" 200 36288 "" "Prometheus/2.43.0" 2023-09-23T21:06:49.810 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:06:49.869 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:06:49.890 INFO:teuthology.orchestra.run.smithi070.stdout:ERROR: (22) Invalid argument 2023-09-23T21:06:49.890 INFO:teuthology.orchestra.run.smithi070.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T21:06:49.928 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:06:49.937 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:06:49.938 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:06:49.985 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T21:06:50.039 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:06:50.114 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:06:50.210 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:06:50.223 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:06:50.224 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:06:50.282 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 54 16 81164 81148 93.8478 55.75 0.025429 0.0106499 2023-09-23T21:06:50.303 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_blocked_ops 2023-09-23T21:06:50.325 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:06:50.406 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:06:50.478 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:06:50.484 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:06:50.485 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:06:50.586 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:06:50.621 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:06:50 smithi070 ceph-mon[168282]: pgmap v929: 109 pgs: 109 active+clean; 4.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 74 MiB/s wr, 1.18k op/s 2023-09-23T21:06:50.621 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:06:50 smithi070 ceph-mon[168282]: failed to encode map e360 with expected crc 2023-09-23T21:06:50.621 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:06:50 smithi070 ceph-mon[168282]: failed to encode map e360 with expected crc 2023-09-23T21:06:50.621 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:06:50 smithi070 ceph-mon[168282]: failed to encode map e360 with expected crc 2023-09-23T21:06:50.622 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:06:50 smithi070 ceph-mon[168282]: failed to encode map e360 with expected crc 2023-09-23T21:06:50.622 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:06:50 smithi070 ceph-mon[168282]: osdmap e360: 8 total, 8 up, 8 in 2023-09-23T21:06:50.622 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:06:50 smithi070 ceph-mon[168282]: failed to encode map e360 with expected crc 2023-09-23T21:06:50.622 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:06:50 smithi070 ceph-mon[168282]: failed to encode map e360 with expected crc 2023-09-23T21:06:50.653 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:06:50.738 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:06:50.743 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:06:50.750 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:06:50.750 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:06:50.751 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:06:50 smithi026 ceph-mon[175151]: pgmap v929: 109 pgs: 109 active+clean; 4.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 74 MiB/s wr, 1.18k op/s 2023-09-23T21:06:50.751 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:06:50 smithi026 ceph-mon[175151]: failed to encode map e360 with expected crc 2023-09-23T21:06:50.751 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:06:50 smithi026 ceph-mon[175151]: failed to encode map e360 with expected crc 2023-09-23T21:06:50.751 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:06:50 smithi026 ceph-mon[175151]: failed to encode map e360 with expected crc 2023-09-23T21:06:50.751 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:06:50 smithi026 ceph-mon[175151]: failed to encode map e360 with expected crc 2023-09-23T21:06:50.751 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:06:50 smithi026 ceph-mon[175151]: osdmap e360: 8 total, 8 up, 8 in 2023-09-23T21:06:50.751 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:06:50 smithi026 ceph-mon[175151]: failed to encode map e360 with expected crc 2023-09-23T21:06:50.752 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:06:50 smithi026 ceph-mon[175151]: failed to encode map e360 with expected crc 2023-09-23T21:06:50.752 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:06:50 smithi026 ceph-mon[179688]: pgmap v929: 109 pgs: 109 active+clean; 4.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 74 MiB/s wr, 1.18k op/s 2023-09-23T21:06:50.752 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:06:50 smithi026 ceph-mon[179688]: failed to encode map e360 with expected crc 2023-09-23T21:06:50.752 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:06:50 smithi026 ceph-mon[179688]: failed to encode map e360 with expected crc 2023-09-23T21:06:50.752 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:06:50 smithi026 ceph-mon[179688]: failed to encode map e360 with expected crc 2023-09-23T21:06:50.752 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:06:50 smithi026 ceph-mon[179688]: failed to encode map e360 with expected crc 2023-09-23T21:06:50.752 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:06:50 smithi026 ceph-mon[179688]: osdmap e360: 8 total, 8 up, 8 in 2023-09-23T21:06:50.752 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:06:50 smithi026 ceph-mon[179688]: failed to encode map e360 with expected crc 2023-09-23T21:06:50.753 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:06:50 smithi026 ceph-mon[179688]: failed to encode map e360 with expected crc 2023-09-23T21:06:50.851 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:06:50.933 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:06:51.014 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:06:51.025 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:06:51.025 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:06:51.054 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T21:06:51.117 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T21:06:51.126 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:06:51.241 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:06:51.283 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 55 16 82154 82138 93.2648 61.875 0.0224788 0.0107155 2023-09-23T21:06:51.339 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:06:51.342 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:06:51.343 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:06:51.443 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:06:51.519 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:06:51.601 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:06:51.606 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:06:51.606 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:06:51.608 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:06:51 smithi026 ceph-mon[179688]: 1.10 deep-scrub starts 2023-09-23T21:06:51.608 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:06:51 smithi026 ceph-mon[179688]: 1.10 deep-scrub ok 2023-09-23T21:06:51.608 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:06:51 smithi026 ceph-mon[175151]: 1.10 deep-scrub starts 2023-09-23T21:06:51.609 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:06:51 smithi026 ceph-mon[175151]: 1.10 deep-scrub ok 2023-09-23T21:06:51.707 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:06:51.747 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:06:51 smithi070 ceph-mon[168282]: 1.10 deep-scrub starts 2023-09-23T21:06:51.748 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:06:51 smithi070 ceph-mon[168282]: 1.10 deep-scrub ok 2023-09-23T21:06:51.751 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:06:51.946 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:06:51.950 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:06:51.950 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:06:51.956 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:06:52.051 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:06:52.139 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:06:52.216 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T21:06:52.217 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:06:52.218 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:06:52.283 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 56 16 83126 83110 92.6845 60.75 0.0194899 0.0107847 2023-09-23T21:06:52.319 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:06:52.354 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:06:52.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:06:52 smithi070 ceph-mon[168282]: pgmap v931: 109 pgs: 2 peering, 107 active+clean; 4.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 6.8 KiB/s rd, 55 MiB/s wr, 879 op/s; 13 KiB/s, 0 objects/s recovering 2023-09-23T21:06:52.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:06:52 smithi070 ceph-mon[168282]: 1.a scrub starts 2023-09-23T21:06:52.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:06:52 smithi070 ceph-mon[168282]: 1.a scrub ok 2023-09-23T21:06:52.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:06:52 smithi070 ceph-mon[168282]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2023-09-23T21:06:52.631 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:06:52 smithi026 ceph-mon[175151]: pgmap v931: 109 pgs: 2 peering, 107 active+clean; 4.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 6.8 KiB/s rd, 55 MiB/s wr, 879 op/s; 13 KiB/s, 0 objects/s recovering 2023-09-23T21:06:52.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:06:52 smithi026 ceph-mon[175151]: 1.a scrub starts 2023-09-23T21:06:52.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:06:52 smithi026 ceph-mon[175151]: 1.a scrub ok 2023-09-23T21:06:52.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:06:52 smithi026 ceph-mon[175151]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2023-09-23T21:06:52.633 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:06:52 smithi026 ceph-mon[179688]: pgmap v931: 109 pgs: 2 peering, 107 active+clean; 4.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 6.8 KiB/s rd, 55 MiB/s wr, 879 op/s; 13 KiB/s, 0 objects/s recovering 2023-09-23T21:06:52.634 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:06:52 smithi026 ceph-mon[179688]: 1.a scrub starts 2023-09-23T21:06:52.634 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:06:52 smithi026 ceph-mon[179688]: 1.a scrub ok 2023-09-23T21:06:52.634 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:06:52 smithi026 ceph-mon[179688]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2023-09-23T21:06:52.907 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:06:52.914 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:06:52.914 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:06:53.015 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:06:53.070 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:06:53.131 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:06:53.136 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:06:53.136 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:06:53.237 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:06:53.284 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 57 16 84157 84141 92.1892 64.4375 0.0154232 0.0108407 2023-09-23T21:06:53.286 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:06:53.465 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:06:53.472 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:06:53.473 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:06:53.478 INFO:teuthology.orchestra.run.smithi026.stderr:noscrub is set 2023-09-23T21:06:53.530 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:06:53 smithi070 ceph-mon[168282]: 1.8 scrub starts 2023-09-23T21:06:53.531 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:06:53 smithi070 ceph-mon[168282]: 1.8 scrub ok 2023-09-23T21:06:53.531 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:06:53 smithi070 ceph-mon[168282]: 1.1 scrub starts 2023-09-23T21:06:53.531 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:06:53 smithi070 ceph-mon[168282]: 1.1 scrub ok 2023-09-23T21:06:53.531 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:06:53 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/1919342011' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T21:06:53.531 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:06:53 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T21:06:53.573 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:06:53.602 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:06:53.717 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:06:53.726 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:06:53.727 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:06:53.828 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:06:53.881 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:06:53 smithi026 ceph-mon[179688]: 1.8 scrub starts 2023-09-23T21:06:53.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:06:53 smithi026 ceph-mon[179688]: 1.8 scrub ok 2023-09-23T21:06:53.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:06:53 smithi026 ceph-mon[179688]: 1.1 scrub starts 2023-09-23T21:06:53.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:06:53 smithi026 ceph-mon[179688]: 1.1 scrub ok 2023-09-23T21:06:53.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:06:53 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/1919342011' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T21:06:53.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:06:53 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T21:06:53.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:06:53 smithi026 ceph-mon[175151]: 1.8 scrub starts 2023-09-23T21:06:53.883 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:06:53 smithi026 ceph-mon[175151]: 1.8 scrub ok 2023-09-23T21:06:53.883 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:06:53 smithi026 ceph-mon[175151]: 1.1 scrub starts 2023-09-23T21:06:53.883 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:06:53 smithi026 ceph-mon[175151]: 1.1 scrub ok 2023-09-23T21:06:53.883 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:06:53 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/1919342011' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T21:06:53.883 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:06:53 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T21:06:53.887 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:06:54.009 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:06:54.018 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:06:54.019 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:06:54.106 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 6, 7, 4, 5] out_osds: [] dead_osds: [] live_osds: [0, 2, 3, 4, 5, 7, 6, 1] 2023-09-23T21:06:54.106 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph osd reweight 0 1 2023-09-23T21:06:54.119 INFO:teuthology.orchestra.run.smithi070.stdout:ERROR: (22) Invalid argument 2023-09-23T21:06:54.120 INFO:teuthology.orchestra.run.smithi070.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T21:06:54.120 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:06:54.229 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:06:54.285 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 58 16 85158 85142 91.6764 62.5625 0.0119331 0.0109015 2023-09-23T21:06:54.462 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:06:54 smithi070 ceph-mon[168282]: pgmap v932: 109 pgs: 2 peering, 107 active+clean; 5.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 41 KiB/s rd, 75 MiB/s wr, 1.23k op/s; 164 KiB/s, 0 objects/s recovering 2023-09-23T21:06:54.462 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:06:54 smithi070 ceph-mon[168282]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:06:54.462 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:06:54 smithi070 ceph-mon[168282]: failed to encode map e361 with expected crc 2023-09-23T21:06:54.462 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:06:54 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T21:06:54.462 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:06:54 smithi070 ceph-mon[168282]: osdmap e361: 8 total, 8 up, 8 in 2023-09-23T21:06:54.567 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_historic_ops 2023-09-23T21:06:54.631 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:06:54 smithi026 ceph-mon[175151]: pgmap v932: 109 pgs: 2 peering, 107 active+clean; 5.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 41 KiB/s rd, 75 MiB/s wr, 1.23k op/s; 164 KiB/s, 0 objects/s recovering 2023-09-23T21:06:54.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:06:54 smithi026 ceph-mon[175151]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:06:54.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:06:54 smithi026 ceph-mon[175151]: failed to encode map e361 with expected crc 2023-09-23T21:06:54.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:06:54 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T21:06:54.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:06:54 smithi026 ceph-mon[175151]: osdmap e361: 8 total, 8 up, 8 in 2023-09-23T21:06:54.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:06:54 smithi026 ceph-mon[179688]: pgmap v932: 109 pgs: 2 peering, 107 active+clean; 5.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 41 KiB/s rd, 75 MiB/s wr, 1.23k op/s; 164 KiB/s, 0 objects/s recovering 2023-09-23T21:06:54.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:06:54 smithi026 ceph-mon[179688]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:06:54.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:06:54 smithi026 ceph-mon[179688]: failed to encode map e361 with expected crc 2023-09-23T21:06:54.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:06:54 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T21:06:54.633 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:06:54 smithi026 ceph-mon[179688]: osdmap e361: 8 total, 8 up, 8 in 2023-09-23T21:06:55.285 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 59 16 86268 86252 91.2986 69.375 0.0141 0.010948 2023-09-23T21:06:55.376 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T21:06:56.283 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:06:56 smithi070 ceph-mon[168282]: pgmap v934: 109 pgs: 109 active+clean; 5.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 32 KiB/s rd, 53 MiB/s wr, 869 op/s; 141 KiB/s, 0 objects/s recovering 2023-09-23T21:06:56.284 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:06:56 smithi070 ceph-mon[168282]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2023-09-23T21:06:56.285 INFO:tasks.radosbench.radosbench.0.smithi026.stdout:2023-09-23T21:06:56.286237+0000 min lat: 0.00137237 max lat: 1.03425 avg lat: 0.0110083 2023-09-23T21:06:56.285 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T21:06:56.285 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 60 16 87244 87228 90.7938 61 0.0142673 0.0110083 2023-09-23T21:06:56.631 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:06:56 smithi026 ceph-mon[175151]: pgmap v934: 109 pgs: 109 active+clean; 5.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 32 KiB/s rd, 53 MiB/s wr, 869 op/s; 141 KiB/s, 0 objects/s recovering 2023-09-23T21:06:56.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:06:56 smithi026 ceph-mon[175151]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2023-09-23T21:06:56.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:06:56 smithi026 ceph-mon[179688]: pgmap v934: 109 pgs: 109 active+clean; 5.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 32 KiB/s rd, 53 MiB/s wr, 869 op/s; 141 KiB/s, 0 objects/s recovering 2023-09-23T21:06:56.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:06:56 smithi026 ceph-mon[179688]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2023-09-23T21:06:56.909 INFO:teuthology.orchestra.run.smithi070.stdout:ERROR: (22) Invalid argument 2023-09-23T21:06:56.910 INFO:teuthology.orchestra.run.smithi070.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T21:06:57.290 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 61 16 88360 88344 90.4419 69.75 0.0176978 0.0110525 2023-09-23T21:06:57.291 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T21:06:57.298 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:06:57.299 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:06:57.331 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_ops_in_flight 2023-09-23T21:06:57.400 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:06:57.489 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:06:58.087 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T21:06:58.091 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:06:58.092 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:06:58.096 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T21:06:58.192 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:06:58.248 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:06:58.290 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 62 16 89327 89311 89.9582 60.4375 0.0117009 0.0111108 2023-09-23T21:06:58.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:06:58 smithi026 ceph-mon[175151]: pgmap v935: 109 pgs: 109 active+clean; 5.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 5.1 KiB/s rd, 45 MiB/s wr, 716 op/s; 9.8 KiB/s, 0 objects/s recovering 2023-09-23T21:06:58.384 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:06:58 smithi026 ceph-mon[179688]: pgmap v935: 109 pgs: 109 active+clean; 5.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 5.1 KiB/s rd, 45 MiB/s wr, 716 op/s; 9.8 KiB/s, 0 objects/s recovering 2023-09-23T21:06:58.464 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:06:58.469 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:06:58.470 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:06:58.509 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:06:58 smithi070 ceph-mon[168282]: pgmap v935: 109 pgs: 109 active+clean; 5.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 5.1 KiB/s rd, 45 MiB/s wr, 716 op/s; 9.8 KiB/s, 0 objects/s recovering 2023-09-23T21:06:58.570 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:06:58.640 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:06:58.895 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:06:58.901 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:06:58.901 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:06:59.002 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:06:59.074 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:06:59.139 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:06:59.145 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:06:59.146 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:06:59.155 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:06:59.246 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:06:59.290 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 63 16 90315 90299 89.5107 61.75 0.00483765 0.0111664 2023-09-23T21:06:59.314 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:06:59.613 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 21:06:59 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[164029]: ::ffff:172.21.15.70 - - [23/Sep/2023:21:06:59] "GET /metrics HTTP/1.1" 200 36288 "" "Prometheus/2.43.0" 2023-09-23T21:06:59.740 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:06:59.749 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:06:59.750 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:06:59.760 INFO:teuthology.orchestra.run.smithi070.stdout:ERROR: (22) Invalid argument 2023-09-23T21:06:59.760 INFO:teuthology.orchestra.run.smithi070.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T21:06:59.850 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:06:59.909 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:07:00.082 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T21:07:00.131 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:07:00.142 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:07:00.142 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:07:00.148 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_blocked_ops 2023-09-23T21:07:00.243 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:07:00.290 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 64 16 91250 91234 89.0254 58.4375 0.0190197 0.0112268 2023-09-23T21:07:00.326 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0: {} 2023-09-23T21:07:00.366 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1: {} 2023-09-23T21:07:00.380 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:07:00.431 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2: {} 2023-09-23T21:07:00.501 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3: {} 2023-09-23T21:07:00.528 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:07:00 smithi026 ceph-mon[179688]: pgmap v936: 109 pgs: 109 active+clean; 5.3 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 51 MiB/s wr, 814 op/s; 0 B/s, 0 objects/s recovering 2023-09-23T21:07:00.529 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:07:00 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/1650263825' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:07:00.529 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:07:00 smithi026 ceph-mon[175151]: pgmap v936: 109 pgs: 109 active+clean; 5.3 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 51 MiB/s wr, 814 op/s; 0 B/s, 0 objects/s recovering 2023-09-23T21:07:00.529 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:07:00 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/1650263825' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:07:00.546 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4: {} 2023-09-23T21:07:00.569 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T21:07:00.612 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6: {} 2023-09-23T21:07:00.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:07:00 smithi070 ceph-mon[168282]: pgmap v936: 109 pgs: 109 active+clean; 5.3 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 51 MiB/s wr, 814 op/s; 0 B/s, 0 objects/s recovering 2023-09-23T21:07:00.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:07:00 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/1650263825' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:07:00.661 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T21:07:00.672 INFO:teuthology.orchestra.run.smithi026.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T21:07:00.673 INFO:teuthology.orchestra.run.smithi026.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T21:07:00.673 INFO:teuthology.orchestra.run.smithi026.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T21:07:00.673 INFO:teuthology.orchestra.run.smithi026.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T21:07:00.673 INFO:teuthology.orchestra.run.smithi026.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T21:07:00.673 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T21:07:00.673 INFO:teuthology.orchestra.run.smithi026.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T21:07:00.673 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T21:07:00.804 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T21:07:01.029 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T21:07:01.033 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:07:01.034 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:07:01.135 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:07:01.218 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:07:01.291 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 65 16 92132 92116 88.5041 55.125 0.0315732 0.0112925 2023-09-23T21:07:01.299 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:07:01.318 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:07:01.318 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:07:01.419 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:07:01.474 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:07:01.538 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:07:01.557 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:07:01 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:07:01.558 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:07:01 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:07:01.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:07:01 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:07:01.709 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:07:01.715 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:07:01.716 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:07:01.817 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:07:01.923 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:07:02.009 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T21:07:02.014 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:07:02.015 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:07:02.115 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:07:02.191 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:07:02.236 INFO:teuthology.orchestra.run.smithi070.stdout:{ 2023-09-23T21:07:02.236 INFO:teuthology.orchestra.run.smithi070.stdout: "ops": [], 2023-09-23T21:07:02.236 INFO:teuthology.orchestra.run.smithi070.stdout: "complaint_time": 30, 2023-09-23T21:07:02.237 INFO:teuthology.orchestra.run.smithi070.stdout: "num_blocked_ops": 0 2023-09-23T21:07:02.237 INFO:teuthology.orchestra.run.smithi070.stdout:} 2023-09-23T21:07:02.294 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 66 16 92989 92973 87.9711 53.5625 0.017033 0.0113625 2023-09-23T21:07:02.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:07:02 smithi026 ceph-mon[179688]: pgmap v937: 109 pgs: 109 active+clean; 5.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 58 MiB/s wr, 929 op/s; 0 B/s, 0 objects/s recovering 2023-09-23T21:07:02.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:07:02 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:07:02.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:07:02 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/2791842882' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-09-23T21:07:02.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:07:02 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-09-23T21:07:02.389 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:07:02 smithi026 ceph-mon[175151]: pgmap v937: 109 pgs: 109 active+clean; 5.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 58 MiB/s wr, 929 op/s; 0 B/s, 0 objects/s recovering 2023-09-23T21:07:02.389 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:07:02 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:07:02.389 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:07:02 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/2791842882' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-09-23T21:07:02.389 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:07:02 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-09-23T21:07:02.509 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:07:02.518 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:07:02.518 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:07:02.599 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:07:02 smithi070 ceph-mon[168282]: pgmap v937: 109 pgs: 109 active+clean; 5.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 58 MiB/s wr, 929 op/s; 0 B/s, 0 objects/s recovering 2023-09-23T21:07:02.599 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:07:02 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:07:02.599 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:07:02 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/2791842882' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-09-23T21:07:02.599 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:07:02 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-09-23T21:07:02.619 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:07:02.646 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_historic_ops 2023-09-23T21:07:02.692 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:07:02.819 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:07:02.827 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:07:02.827 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:07:02.928 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:07:03.009 INFO:teuthology.orchestra.run.smithi026.stderr:reweighted osd.0 to 1 (10000) 2023-09-23T21:07:03.011 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:07:03.091 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:07:03.100 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:07:03.100 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:07:03.201 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:07:03.250 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:07:03.294 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 67 16 93908 93892 87.5157 57.4375 0.0280467 0.0114213 2023-09-23T21:07:03.317 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:07:03.320 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:07:03.321 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:07:03.324 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T21:07:03.413 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T21:07:03.422 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:07:03.526 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:07:03.605 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph osd reweight 1 1 2023-09-23T21:07:03.626 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:07:03.633 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:07:03.633 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:07:03.734 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:07:03.831 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:07:03.920 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:07:03.933 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:07:03.933 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:07:03.954 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:07:03 smithi026 ceph-mon[175151]: pgmap v938: 109 pgs: 109 active+clean; 5.6 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 61 MiB/s wr, 964 op/s 2023-09-23T21:07:03.955 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:07:03 smithi026 ceph-mon[179688]: pgmap v938: 109 pgs: 109 active+clean; 5.6 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 61 MiB/s wr, 964 op/s 2023-09-23T21:07:03.955 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:07:03 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-09-23T21:07:03.955 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:07:03 smithi026 ceph-mon[179688]: osdmap e362: 8 total, 8 up, 8 in 2023-09-23T21:07:03.955 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:07:03 smithi026 ceph-mon[179688]: failed to encode map e362 with expected crc 2023-09-23T21:07:03.955 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:07:03 smithi026 ceph-mon[179688]: failed to encode map e362 with expected crc 2023-09-23T21:07:03.955 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:07:03 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/2898739393' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:07:03.956 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:07:03 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:07:04.034 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:07:04.072 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:07:04.229 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:07:04.270 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:07:04.275 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:07:03 smithi070 ceph-mon[168282]: pgmap v938: 109 pgs: 109 active+clean; 5.6 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 61 MiB/s wr, 964 op/s 2023-09-23T21:07:04.275 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:07:03 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-09-23T21:07:04.275 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:07:03 smithi070 ceph-mon[168282]: osdmap e362: 8 total, 8 up, 8 in 2023-09-23T21:07:04.275 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:07:03 smithi070 ceph-mon[168282]: failed to encode map e362 with expected crc 2023-09-23T21:07:04.275 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:07:03 smithi070 ceph-mon[168282]: failed to encode map e362 with expected crc 2023-09-23T21:07:04.275 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:07:03 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/2898739393' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:07:04.275 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:07:03 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:07:04.276 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:07:04.276 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:07:04.294 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 68 16 94730 94714 86.9843 51.375 0.0117031 0.0114915 2023-09-23T21:07:04.366 INFO:teuthology.orchestra.run.smithi026.stderr:nodeep-scrub is set 2023-09-23T21:07:04.377 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:07:04.383 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:07:03 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-09-23T21:07:04.383 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:07:03 smithi026 ceph-mon[175151]: osdmap e362: 8 total, 8 up, 8 in 2023-09-23T21:07:04.383 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:07:03 smithi026 ceph-mon[175151]: failed to encode map e362 with expected crc 2023-09-23T21:07:04.383 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:07:03 smithi026 ceph-mon[175151]: failed to encode map e362 with expected crc 2023-09-23T21:07:04.383 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:07:03 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/2898739393' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:07:04.384 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:07:03 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:07:04.422 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:07:04.436 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:07:04.485 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:07:04.487 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:07:04.488 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:07:04.588 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:07:04.667 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:07:04.772 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:07:04.776 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:07:04.777 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:07:04.878 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:07:04.909 INFO:teuthology.orchestra.run.smithi070.stdout:{ 2023-09-23T21:07:04.909 INFO:teuthology.orchestra.run.smithi070.stdout: "size": 20, 2023-09-23T21:07:04.909 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 600, 2023-09-23T21:07:04.909 INFO:teuthology.orchestra.run.smithi070.stdout: "ops": [ 2023-09-23T21:07:04.909 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:07:04.910 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.35856.0:52751 9.3 9:c51114d0:::benchmark_data_smithi026_342732_object52750:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e354)", 2023-09-23T21:07:04.910 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:06:24.396732+0000", 2023-09-23T21:07:04.910 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 40.509151127999999, 2023-09-23T21:07:04.910 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.006593885, 2023-09-23T21:07:04.910 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:07:04.910 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "reached pg", 2023-09-23T21:07:04.910 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:07:04.910 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.35856", 2023-09-23T21:07:04.910 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/4180396258", 2023-09-23T21:07:04.911 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 52751 2023-09-23T21:07:04.911 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:07:04.911 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:07:04.911 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:07:04.911 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:07:04.911 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:24.396732+0000", 2023-09-23T21:07:04.911 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:07:04.911 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:07:04.911 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:07:04.912 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:07:04.912 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:24.396732+0000", 2023-09-23T21:07:04.912 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:07:04.912 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:07:04.912 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:07:04.912 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:07:04.912 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:24.396731+0000", 2023-09-23T21:07:04.912 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T21:07:04.912 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:07:04.912 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:07:04.912 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:07:04.913 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:24.396868+0000", 2023-09-23T21:07:04.913 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00013730400000000001 2023-09-23T21:07:04.913 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:07:04.913 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:07:04.913 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:07:04.913 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:24.396883+0000", 2023-09-23T21:07:04.913 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.5221999999999999e-05 2023-09-23T21:07:04.913 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:07:04.913 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:07:04.913 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:07:04.913 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:24.396891+0000", 2023-09-23T21:07:04.913 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.3869999999999997e-06 2023-09-23T21:07:04.914 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:07:04.914 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:07:04.914 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:07:04.914 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:24.397030+0000", 2023-09-23T21:07:04.914 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00013967 2023-09-23T21:07:04.914 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:07:04.914 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:07:04.914 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:07:04.914 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:24.397040+0000", 2023-09-23T21:07:04.914 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.9150000000000008e-06 2023-09-23T21:07:04.915 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:07:04.915 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:07:04.915 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:07:04.915 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:25.402653+0000", 2023-09-23T21:07:04.915 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.00561239 2023-09-23T21:07:04.915 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:07:04.915 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:07:04.915 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:07:04.915 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:25.403326+0000", 2023-09-23T21:07:04.916 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00067348199999999999 2023-09-23T21:07:04.916 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:07:04.916 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:07:04.916 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:07:04.916 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:07:04.916 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:07:04.916 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.35856.0:52765 9.3 9:cd654ac2:::benchmark_data_smithi026_342732_object52764:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e354)", 2023-09-23T21:07:04.916 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:06:24.396908+0000", 2023-09-23T21:07:04.917 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 40.508975006999997, 2023-09-23T21:07:04.917 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.0245688509999999, 2023-09-23T21:07:04.917 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:07:04.917 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:07:04.917 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:07:04.917 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.35856", 2023-09-23T21:07:04.917 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/4180396258", 2023-09-23T21:07:04.917 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 52765 2023-09-23T21:07:04.917 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:07:04.917 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:07:04.918 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:07:04.918 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:07:04.918 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:24.396908+0000", 2023-09-23T21:07:04.918 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:07:04.918 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:07:04.918 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:07:04.918 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:07:04.918 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:24.396908+0000", 2023-09-23T21:07:04.919 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:07:04.919 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:07:04.919 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:07:04.919 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:07:04.919 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:24.396908+0000", 2023-09-23T21:07:04.919 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999995 2023-09-23T21:07:04.919 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:07:04.919 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:07:04.919 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:07:04.920 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:24.396943+0000", 2023-09-23T21:07:04.920 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.5349999999999999e-05 2023-09-23T21:07:04.920 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:07:04.920 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:07:04.920 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:07:04.920 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:24.396952+0000", 2023-09-23T21:07:04.920 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.6649999999999992e-06 2023-09-23T21:07:04.920 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:07:04.920 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:07:04.920 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:07:04.920 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:24.396957+0000", 2023-09-23T21:07:04.921 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.3410000000000002e-06 2023-09-23T21:07:04.921 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:07:04.921 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:07:04.921 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:07:04.921 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:24.397054+0000", 2023-09-23T21:07:04.921 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.7220999999999996e-05 2023-09-23T21:07:04.921 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:07:04.921 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:07:04.921 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:07:04.921 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:24.397060+0000", 2023-09-23T21:07:04.921 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.4020000000000001e-06 2023-09-23T21:07:04.921 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:07:04.921 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:07:04.922 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:07:04.922 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:25.403373+0000", 2023-09-23T21:07:04.922 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.006313697 2023-09-23T21:07:04.922 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:07:04.922 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:07:04.922 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:07:04.922 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:25.403624+0000", 2023-09-23T21:07:04.922 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00025100200000000001 2023-09-23T21:07:04.922 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:07:04.922 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:07:04.922 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 1,3", 2023-09-23T21:07:04.923 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:25.403787+0000", 2023-09-23T21:07:04.923 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00016209899999999999 2023-09-23T21:07:04.923 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:07:04.923 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:07:04.923 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:07:04.923 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:25.409004+0000", 2023-09-23T21:07:04.923 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0052175640000000001 2023-09-23T21:07:04.923 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:07:04.923 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:07:04.923 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:07:04.923 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:25.421372+0000", 2023-09-23T21:07:04.923 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.012367507999999999 2023-09-23T21:07:04.923 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:07:04.924 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:07:04.924 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:07:04.924 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:25.421418+0000", 2023-09-23T21:07:04.924 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.5871000000000002e-05 2023-09-23T21:07:04.924 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:07:04.924 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:07:04.924 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:07:04.924 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:25.421453+0000", 2023-09-23T21:07:04.924 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.5199e-05 2023-09-23T21:07:04.924 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:07:04.924 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:07:04.924 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:07:04.925 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:25.421477+0000", 2023-09-23T21:07:04.925 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.4539e-05 2023-09-23T21:07:04.925 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:07:04.925 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:07:04.925 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:07:04.925 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:07:04.925 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:07:04.925 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.35856.0:52801 9.3 9:cf43f7eb:::benchmark_data_smithi026_342732_object52800:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e354)", 2023-09-23T21:07:04.925 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:06:24.416107+0000", 2023-09-23T21:07:04.925 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 40.489776251999999, 2023-09-23T21:07:04.925 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.0054936219999999, 2023-09-23T21:07:04.925 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:07:04.925 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:07:04.926 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:07:04.926 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.35856", 2023-09-23T21:07:04.926 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/4180396258", 2023-09-23T21:07:04.926 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 52801 2023-09-23T21:07:04.926 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:07:04.926 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:07:04.926 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:07:04.926 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:07:04.926 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:24.416107+0000", 2023-09-23T21:07:04.926 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:07:04.926 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:07:04.926 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:07:04.927 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:07:04.927 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:24.416107+0000", 2023-09-23T21:07:04.927 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:07:04.927 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:07:04.927 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:07:04.927 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:07:04.927 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:24.416106+0000", 2023-09-23T21:07:04.927 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T21:07:04.927 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:07:04.927 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:07:04.927 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:07:04.928 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:24.416150+0000", 2023-09-23T21:07:04.928 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.4604999999999997e-05 2023-09-23T21:07:04.928 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:07:04.928 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:07:04.928 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:07:04.928 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:24.416167+0000", 2023-09-23T21:07:04.928 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.6693000000000001e-05 2023-09-23T21:07:04.928 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:07:04.928 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:07:04.928 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:07:04.928 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:24.416174+0000", 2023-09-23T21:07:04.928 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.7120000000000003e-06 2023-09-23T21:07:04.928 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:07:04.929 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:07:04.929 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:07:04.929 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:24.416243+0000", 2023-09-23T21:07:04.929 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.9830000000000001e-05 2023-09-23T21:07:04.929 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:07:04.929 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:07:04.929 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:07:04.929 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:24.416253+0000", 2023-09-23T21:07:04.929 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.02e-06 2023-09-23T21:07:04.929 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:07:04.929 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:07:04.930 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:07:04.930 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:25.405162+0000", 2023-09-23T21:07:04.930 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.98890962100000002 2023-09-23T21:07:04.930 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:07:04.930 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:07:04.930 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:07:04.930 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:25.405288+0000", 2023-09-23T21:07:04.930 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00012563099999999999 2023-09-23T21:07:04.930 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:07:04.930 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:07:04.930 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 1,3", 2023-09-23T21:07:04.930 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:25.405448+0000", 2023-09-23T21:07:04.931 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000159872 2023-09-23T21:07:04.931 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:07:04.931 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:07:04.931 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:07:04.931 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:25.411143+0000", 2023-09-23T21:07:04.931 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0056956359999999996 2023-09-23T21:07:04.931 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:07:04.931 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:07:04.931 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:07:04.931 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:25.421523+0000", 2023-09-23T21:07:04.931 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.010379767999999999 2023-09-23T21:07:04.931 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:07:04.932 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:07:04.932 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:07:04.932 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:25.421555+0000", 2023-09-23T21:07:04.932 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.2403000000000001e-05 2023-09-23T21:07:04.932 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:07:04.932 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:07:04.932 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:07:04.932 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:25.421581+0000", 2023-09-23T21:07:04.932 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.5843e-05 2023-09-23T21:07:04.932 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:07:04.932 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:07:04.933 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:07:04.933 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:25.421601+0000", 2023-09-23T21:07:04.933 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.9510000000000001e-05 2023-09-23T21:07:04.933 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:07:04.933 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:07:04.933 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:07:04.933 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:07:04.933 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:07:04.933 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.35856.0:52825 9.3 9:c99b89ee:::benchmark_data_smithi026_342732_object52824:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e354)", 2023-09-23T21:07:04.933 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:06:24.432929+0000", 2023-09-23T21:07:04.933 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 40.472953937, 2023-09-23T21:07:04.933 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.98885604299999996, 2023-09-23T21:07:04.934 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:07:04.934 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:07:04.934 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:07:04.934 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.35856", 2023-09-23T21:07:04.934 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/4180396258", 2023-09-23T21:07:04.934 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 52825 2023-09-23T21:07:04.934 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:07:04.934 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:07:04.934 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:07:04.934 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:07:04.934 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:24.432929+0000", 2023-09-23T21:07:04.934 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:07:04.935 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:07:04.935 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:07:04.935 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:07:04.935 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:24.432929+0000", 2023-09-23T21:07:04.935 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:07:04.935 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:07:04.935 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:07:04.935 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:07:04.935 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:24.432928+0000", 2023-09-23T21:07:04.935 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T21:07:04.935 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:07:04.935 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:07:04.936 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:07:04.936 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:24.432975+0000", 2023-09-23T21:07:04.936 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.7441999999999997e-05 2023-09-23T21:07:04.936 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:07:04.936 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:07:04.936 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:07:04.936 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:24.432987+0000", 2023-09-23T21:07:04.936 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.1712999999999999e-05 2023-09-23T21:07:04.936 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:07:04.936 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:07:04.936 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:07:04.936 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:24.432993+0000", 2023-09-23T21:07:04.936 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.7640000000000002e-06 2023-09-23T21:07:04.937 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:07:04.937 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:07:04.937 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:07:04.937 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:24.433060+0000", 2023-09-23T21:07:04.937 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.6691000000000002e-05 2023-09-23T21:07:04.937 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:07:04.937 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:07:04.937 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:07:04.937 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:24.433071+0000", 2023-09-23T21:07:04.937 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.1163e-05 2023-09-23T21:07:04.937 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:07:04.937 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:07:04.937 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:07:04.938 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:25.406642+0000", 2023-09-23T21:07:04.938 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.97357167 2023-09-23T21:07:04.938 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:07:04.938 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:07:04.938 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:07:04.938 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:25.406738+0000", 2023-09-23T21:07:04.938 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.5882000000000001e-05 2023-09-23T21:07:04.938 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:07:04.938 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:07:04.938 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 1,3", 2023-09-23T21:07:04.938 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:25.406824+0000", 2023-09-23T21:07:04.938 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.5877999999999993e-05 2023-09-23T21:07:04.939 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:07:04.939 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:07:04.939 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:07:04.939 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:25.411177+0000", 2023-09-23T21:07:04.939 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0043524490000000004 2023-09-23T21:07:04.939 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:07:04.939 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:07:04.939 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:07:04.939 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:25.421631+0000", 2023-09-23T21:07:04.939 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.010454234 2023-09-23T21:07:04.939 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:07:04.940 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:07:04.940 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:07:04.940 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:25.421724+0000", 2023-09-23T21:07:04.940 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.3381000000000006e-05 2023-09-23T21:07:04.940 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:07:04.940 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:07:04.940 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:07:04.940 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:25.421766+0000", 2023-09-23T21:07:04.940 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.1347000000000002e-05 2023-09-23T21:07:04.940 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:07:04.941 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:07:04.941 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:07:04.941 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:25.421786+0000", 2023-09-23T21:07:04.941 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.9942e-05 2023-09-23T21:07:04.941 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:07:04.941 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:07:04.941 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:07:04.941 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:07:04.941 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:07:04.941 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.35856.0:52839 9.3 9:c25308eb:::benchmark_data_smithi026_342732_object52838:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e354)", 2023-09-23T21:07:04.942 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:06:24.442455+0000", 2023-09-23T21:07:04.942 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 40.463428475000001, 2023-09-23T21:07:04.942 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.979453766, 2023-09-23T21:07:04.942 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:07:04.942 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:07:04.942 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:07:04.942 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.35856", 2023-09-23T21:07:04.942 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/4180396258", 2023-09-23T21:07:04.943 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 52839 2023-09-23T21:07:04.943 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:07:04.943 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:07:04.943 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:07:04.943 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:07:04.943 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:24.442455+0000", 2023-09-23T21:07:04.943 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:07:04.943 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:07:04.943 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:07:04.944 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:07:04.944 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:24.442455+0000", 2023-09-23T21:07:04.944 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:07:04.944 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:07:04.944 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:07:04.944 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:07:04.944 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:24.442453+0000", 2023-09-23T21:07:04.944 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T21:07:04.944 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:07:04.945 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:07:04.945 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:07:04.945 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:24.442566+0000", 2023-09-23T21:07:04.945 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00011271400000000001 2023-09-23T21:07:04.945 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:07:04.945 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:07:04.945 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:07:04.945 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:24.442582+0000", 2023-09-23T21:07:04.945 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.579e-05 2023-09-23T21:07:04.945 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:07:04.945 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:07:04.946 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:07:04.946 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:24.442589+0000", 2023-09-23T21:07:04.946 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.1559999999999998e-06 2023-09-23T21:07:04.946 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:07:04.946 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:07:04.946 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:07:04.946 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:24.442645+0000", 2023-09-23T21:07:04.946 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.6020999999999998e-05 2023-09-23T21:07:04.946 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:07:04.946 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:07:04.947 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:07:04.947 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:24.442652+0000", 2023-09-23T21:07:04.947 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.6120000000000004e-06 2023-09-23T21:07:04.947 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:07:04.947 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:07:04.947 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:07:04.947 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:25.408020+0000", 2023-09-23T21:07:04.947 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.96536787999999996 2023-09-23T21:07:04.947 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:07:04.948 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:07:04.948 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:07:04.948 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:25.408105+0000", 2023-09-23T21:07:04.948 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.5282000000000001e-05 2023-09-23T21:07:04.948 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:07:04.948 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:07:04.948 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 1,3", 2023-09-23T21:07:04.948 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:25.408218+0000", 2023-09-23T21:07:04.948 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000112796 2023-09-23T21:07:04.948 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:07:04.949 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:07:04.949 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:07:04.949 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:25.413318+0000", 2023-09-23T21:07:04.949 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0051002160000000003 2023-09-23T21:07:04.949 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:07:04.949 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:07:04.949 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:07:04.949 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:25.421648+0000", 2023-09-23T21:07:04.950 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0083301109999999994 2023-09-23T21:07:04.950 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:07:04.950 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:07:04.950 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:07:04.950 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:25.421828+0000", 2023-09-23T21:07:04.950 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00017987299999999999 2023-09-23T21:07:04.950 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:07:04.951 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:07:04.951 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:07:04.951 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:25.421878+0000", 2023-09-23T21:07:04.951 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.0591999999999999e-05 2023-09-23T21:07:04.951 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:07:04.951 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:07:04.951 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:07:04.951 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:25.421909+0000", 2023-09-23T21:07:04.952 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.0280000000000001e-05 2023-09-23T21:07:04.952 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:07:04.952 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:07:04.952 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:07:04.952 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:07:04.952 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:07:04.952 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.35856.0:52856 9.3 9:cea642a7:::benchmark_data_smithi026_342732_object52855:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e354)", 2023-09-23T21:07:04.952 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:06:24.452334+0000", 2023-09-23T21:07:04.952 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 40.453549291999998, 2023-09-23T21:07:04.952 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.96970934900000005, 2023-09-23T21:07:04.953 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:07:04.953 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:07:04.953 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:07:04.953 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.35856", 2023-09-23T21:07:04.953 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/4180396258", 2023-09-23T21:07:04.953 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 52856 2023-09-23T21:07:04.953 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:07:04.953 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:07:04.953 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:07:04.953 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:07:04.954 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:24.452334+0000", 2023-09-23T21:07:04.954 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:07:04.954 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:07:04.954 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:07:04.954 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:07:04.954 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:24.452334+0000", 2023-09-23T21:07:04.954 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:07:04.954 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:07:04.954 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:07:04.954 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:07:04.954 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:24.452333+0000", 2023-09-23T21:07:04.954 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T21:07:04.955 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:07:04.955 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:07:04.955 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:07:04.955 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:24.452378+0000", 2023-09-23T21:07:04.955 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.5096000000000003e-05 2023-09-23T21:07:04.955 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:07:04.955 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:07:04.955 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:07:04.955 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:24.452394+0000", 2023-09-23T21:07:04.955 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.5888000000000001e-05 2023-09-23T21:07:04.955 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:07:04.955 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:07:04.956 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:07:04.956 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:24.452400+0000", 2023-09-23T21:07:04.956 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.6980000000000001e-06 2023-09-23T21:07:04.956 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:07:04.956 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:07:04.956 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:07:04.956 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:24.452459+0000", 2023-09-23T21:07:04.956 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.8594999999999997e-05 2023-09-23T21:07:04.956 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:07:04.956 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:07:04.956 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:07:04.956 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:24.452468+0000", 2023-09-23T21:07:04.956 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.6929999999999996e-06 2023-09-23T21:07:04.957 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:07:04.957 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:07:04.957 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:07:04.957 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:25.409045+0000", 2023-09-23T21:07:04.957 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.95657733099999998 2023-09-23T21:07:04.957 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:07:04.957 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:07:04.957 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:07:04.957 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:25.409143+0000", 2023-09-23T21:07:04.957 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.8317000000000001e-05 2023-09-23T21:07:04.957 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:07:04.957 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:07:04.958 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 1,3", 2023-09-23T21:07:04.958 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:25.409239+0000", 2023-09-23T21:07:04.958 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.6280000000000007e-05 2023-09-23T21:07:04.958 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:07:04.958 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:07:04.958 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:07:04.958 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:25.413327+0000", 2023-09-23T21:07:04.958 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0040874220000000003 2023-09-23T21:07:04.958 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:07:04.958 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:07:04.958 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:07:04.958 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:25.421696+0000", 2023-09-23T21:07:04.959 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0083695439999999996 2023-09-23T21:07:04.959 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:07:04.959 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:07:04.959 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:07:04.959 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:25.421994+0000", 2023-09-23T21:07:04.959 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00029776299999999999 2023-09-23T21:07:04.959 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:07:04.959 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:07:04.959 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:07:04.959 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:25.422025+0000", 2023-09-23T21:07:04.959 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.0880000000000002e-05 2023-09-23T21:07:04.959 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:07:04.960 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:07:04.960 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:07:04.960 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:25.422043+0000", 2023-09-23T21:07:04.960 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.8410000000000002e-05 2023-09-23T21:07:04.960 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:07:04.960 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:07:04.960 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:07:04.960 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:07:04.960 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:07:04.960 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.35856.0:52876 9.3 9:c116084f:::benchmark_data_smithi026_342732_object52875:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e354)", 2023-09-23T21:07:04.960 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:06:24.468836+0000", 2023-09-23T21:07:04.960 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 40.437047743999997, 2023-09-23T21:07:04.961 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.95333874799999996, 2023-09-23T21:07:04.961 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:07:04.961 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:07:04.961 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:07:04.961 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.35856", 2023-09-23T21:07:04.961 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/4180396258", 2023-09-23T21:07:04.961 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 52876 2023-09-23T21:07:04.961 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:07:04.961 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:07:04.961 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:07:04.961 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:07:04.961 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:24.468836+0000", 2023-09-23T21:07:04.962 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:07:04.962 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:07:04.962 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:07:04.962 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:07:04.962 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:24.468836+0000", 2023-09-23T21:07:04.962 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:07:04.962 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:07:04.962 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:07:04.962 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:07:04.962 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:24.468834+0000", 2023-09-23T21:07:04.962 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T21:07:04.963 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:07:04.963 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:07:04.963 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:07:04.963 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:24.468883+0000", 2023-09-23T21:07:04.963 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.8093000000000002e-05 2023-09-23T21:07:04.963 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:07:04.963 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:07:04.963 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:07:04.963 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:24.468895+0000", 2023-09-23T21:07:04.963 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.2425999999999999e-05 2023-09-23T21:07:04.963 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:07:04.963 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:07:04.963 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:07:04.964 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:24.468902+0000", 2023-09-23T21:07:04.964 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.6529999999999997e-06 2023-09-23T21:07:04.964 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:07:04.964 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:07:04.964 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:07:04.964 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:24.468973+0000", 2023-09-23T21:07:04.964 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.1608999999999998e-05 2023-09-23T21:07:04.964 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:07:04.964 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:07:04.964 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:07:04.964 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:24.468986+0000", 2023-09-23T21:07:04.964 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.225e-05 2023-09-23T21:07:04.964 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:07:04.965 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:07:04.965 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:07:04.965 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:25.410042+0000", 2023-09-23T21:07:04.965 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.94105615099999995 2023-09-23T21:07:04.965 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:07:04.965 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:07:04.965 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:07:04.965 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:25.410150+0000", 2023-09-23T21:07:04.965 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000108099 2023-09-23T21:07:04.965 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:07:04.965 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:07:04.965 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 1,3", 2023-09-23T21:07:04.966 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:25.410307+0000", 2023-09-23T21:07:04.966 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00015735 2023-09-23T21:07:04.966 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:07:04.966 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:07:04.966 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:07:04.966 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:25.415575+0000", 2023-09-23T21:07:04.966 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0052674810000000001 2023-09-23T21:07:04.966 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:07:04.966 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:07:04.966 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:07:04.966 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:25.421961+0000", 2023-09-23T21:07:04.966 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0063868579999999996 2023-09-23T21:07:04.966 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:07:04.967 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:07:04.967 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:07:04.967 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:25.422089+0000", 2023-09-23T21:07:04.967 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00012751799999999999 2023-09-23T21:07:04.967 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:07:04.967 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:07:04.967 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:07:04.967 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:25.422138+0000", 2023-09-23T21:07:04.967 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.8755000000000003e-05 2023-09-23T21:07:04.967 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:07:04.967 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:07:04.967 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:07:04.968 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:25.422174+0000", 2023-09-23T21:07:04.968 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.6712999999999999e-05 2023-09-23T21:07:04.968 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:07:04.968 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:07:04.968 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:07:04.968 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:07:04.968 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:07:04.968 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.35856.0:52877 9.3 9:c2be6f32:::benchmark_data_smithi026_342732_object52876:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e354)", 2023-09-23T21:07:04.968 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:06:24.474477+0000", 2023-09-23T21:07:04.968 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 40.431406406000001, 2023-09-23T21:07:04.968 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.94913964100000003, 2023-09-23T21:07:04.968 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:07:04.969 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:07:04.969 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:07:04.969 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.35856", 2023-09-23T21:07:04.969 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/4180396258", 2023-09-23T21:07:04.969 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 52877 2023-09-23T21:07:04.969 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:07:04.969 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:07:04.969 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:07:04.969 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:07:04.969 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:24.474477+0000", 2023-09-23T21:07:04.969 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:07:04.969 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:07:04.970 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:07:04.970 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:07:04.970 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:24.474477+0000", 2023-09-23T21:07:04.970 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:07:04.970 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:07:04.970 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:07:04.970 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:07:04.970 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:24.474476+0000", 2023-09-23T21:07:04.970 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.999999 2023-09-23T21:07:04.970 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:07:04.970 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:07:04.970 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:07:04.971 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:24.474528+0000", 2023-09-23T21:07:04.971 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.1582999999999997e-05 2023-09-23T21:07:04.971 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:07:04.971 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:07:04.971 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:07:04.971 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:24.474539+0000", 2023-09-23T21:07:04.971 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.1895e-05 2023-09-23T21:07:04.971 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:07:04.971 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:07:04.971 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:07:04.971 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:24.474546+0000", 2023-09-23T21:07:04.971 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.0700000000000001e-06 2023-09-23T21:07:04.971 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:07:04.972 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:07:04.972 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:07:04.972 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:24.474606+0000", 2023-09-23T21:07:04.972 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.9349999999999999e-05 2023-09-23T21:07:04.972 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:07:04.972 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:07:04.972 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:07:04.972 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:24.474614+0000", 2023-09-23T21:07:04.972 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.4889999999999995e-06 2023-09-23T21:07:04.972 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:07:04.973 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:07:04.973 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:07:04.973 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:25.411215+0000", 2023-09-23T21:07:04.973 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.93660084300000002 2023-09-23T21:07:04.973 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:07:04.973 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:07:04.973 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:07:04.973 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:25.411317+0000", 2023-09-23T21:07:04.973 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000102122 2023-09-23T21:07:04.974 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:07:04.974 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:07:04.974 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 1,3", 2023-09-23T21:07:04.974 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:25.411414+0000", 2023-09-23T21:07:04.974 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.6571999999999996e-05 2023-09-23T21:07:04.974 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:07:04.974 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:07:04.974 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:07:04.974 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:25.415584+0000", 2023-09-23T21:07:04.974 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0041698890000000004 2023-09-23T21:07:04.974 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:07:04.975 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:07:04.975 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:07:04.975 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:25.422211+0000", 2023-09-23T21:07:04.975 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0066269459999999999 2023-09-23T21:07:04.975 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:07:04.975 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:07:04.975 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:07:04.975 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:25.423526+0000", 2023-09-23T21:07:04.975 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0013153139999999999 2023-09-23T21:07:04.976 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:07:04.976 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:07:04.976 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:07:04.976 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:25.423581+0000", 2023-09-23T21:07:04.976 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.4602999999999999e-05 2023-09-23T21:07:04.976 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:07:04.976 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:07:04.976 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:07:04.976 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:25.423617+0000", 2023-09-23T21:07:04.976 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.6056000000000002e-05 2023-09-23T21:07:04.976 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:07:04.977 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:07:04.977 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:07:04.977 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:07:04.977 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:07:04.977 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.35856.0:52886 9.3 9:ccc0ebe9:::benchmark_data_smithi026_342732_object52885:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e354)", 2023-09-23T21:07:04.977 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:06:24.484834+0000", 2023-09-23T21:07:04.977 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 40.421049496999998, 2023-09-23T21:07:04.977 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.93888714799999995, 2023-09-23T21:07:04.977 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:07:04.978 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:07:04.978 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:07:04.978 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.35856", 2023-09-23T21:07:04.978 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/4180396258", 2023-09-23T21:07:04.978 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 52886 2023-09-23T21:07:04.978 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:07:04.978 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:07:04.978 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:07:04.978 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:07:04.979 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:24.484834+0000", 2023-09-23T21:07:04.979 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:07:04.979 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:07:04.979 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:07:04.979 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:07:04.979 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:24.484834+0000", 2023-09-23T21:07:04.979 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:07:04.979 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:07:04.979 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:07:04.979 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:07:04.980 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:24.484833+0000", 2023-09-23T21:07:04.980 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T21:07:04.980 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:07:04.980 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:07:04.980 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:07:04.980 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:24.484888+0000", 2023-09-23T21:07:04.980 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.5315000000000001e-05 2023-09-23T21:07:04.980 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:07:04.980 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:07:04.981 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:07:04.981 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:24.484901+0000", 2023-09-23T21:07:04.981 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.2958e-05 2023-09-23T21:07:04.981 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:07:04.981 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:07:04.981 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:07:04.981 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:24.484906+0000", 2023-09-23T21:07:04.981 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.4609999999999999e-06 2023-09-23T21:07:04.981 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:07:04.981 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:07:04.982 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:07:04.982 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:24.484972+0000", 2023-09-23T21:07:04.982 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.5840999999999995e-05 2023-09-23T21:07:04.982 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:07:04.982 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:07:04.982 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:07:04.982 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:24.484984+0000", 2023-09-23T21:07:04.982 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.1969e-05 2023-09-23T21:07:04.982 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:07:04.982 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:07:04.982 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:07:04.982 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:25.412238+0000", 2023-09-23T21:07:04.983 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.92725426899999996 2023-09-23T21:07:04.983 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:07:04.983 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:07:04.983 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:07:04.983 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:25.412359+0000", 2023-09-23T21:07:04.983 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00012057600000000001 2023-09-23T21:07:04.983 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:07:04.983 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:07:04.983 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 1,3", 2023-09-23T21:07:04.983 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:25.412512+0000", 2023-09-23T21:07:04.983 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00015280400000000001 2023-09-23T21:07:04.983 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:07:04.983 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:07:04.984 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:07:04.984 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:25.417774+0000", 2023-09-23T21:07:04.984 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0052620660000000001 2023-09-23T21:07:04.984 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:07:04.984 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:07:04.984 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:07:04.984 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:25.422260+0000", 2023-09-23T21:07:04.984 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0044861800000000002 2023-09-23T21:07:04.984 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:07:04.984 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:07:04.984 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:07:04.984 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:25.423663+0000", 2023-09-23T21:07:04.985 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.001402915 2023-09-23T21:07:04.985 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:07:04.985 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:07:04.985 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:07:04.985 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:25.423698+0000", 2023-09-23T21:07:04.985 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.5021999999999998e-05 2023-09-23T21:07:04.985 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:07:04.985 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:07:04.985 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:07:04.985 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:25.423721+0000", 2023-09-23T21:07:04.986 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.3166000000000001e-05 2023-09-23T21:07:04.986 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:07:04.986 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:07:04.986 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:07:04.986 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:07:04.986 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:07:04.986 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.35856.0:52898 9.3 9:ce13b48e:::benchmark_data_smithi026_342732_object52897:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e354)", 2023-09-23T21:07:04.986 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:06:24.502098+0000", 2023-09-23T21:07:04.986 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 40.403785911, 2023-09-23T21:07:04.987 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.92904386800000005, 2023-09-23T21:07:04.987 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:07:04.987 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:07:04.987 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:07:04.987 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.35856", 2023-09-23T21:07:04.987 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/4180396258", 2023-09-23T21:07:04.987 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 52898 2023-09-23T21:07:04.987 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:07:04.987 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:07:04.988 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:07:04.988 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:07:04.988 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:24.502098+0000", 2023-09-23T21:07:04.988 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:07:04.988 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:07:04.988 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:07:04.988 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:07:04.988 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:24.502098+0000", 2023-09-23T21:07:04.988 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:07:04.988 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:07:04.989 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:07:04.989 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:07:04.989 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:24.502096+0000", 2023-09-23T21:07:04.989 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999981 2023-09-23T21:07:04.989 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:07:04.989 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:07:04.989 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:07:04.989 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:24.502121+0000", 2023-09-23T21:07:04.989 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.5218000000000001e-05 2023-09-23T21:07:04.989 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:07:04.989 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:07:04.989 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:07:04.990 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:24.502135+0000", 2023-09-23T21:07:04.990 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.3606e-05 2023-09-23T21:07:04.990 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:07:04.990 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:07:04.990 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:07:04.990 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:24.502142+0000", 2023-09-23T21:07:04.990 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.2590000000000003e-06 2023-09-23T21:07:04.990 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:07:04.990 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:07:04.990 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:07:04.990 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:24.502203+0000", 2023-09-23T21:07:04.990 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.1614999999999999e-05 2023-09-23T21:07:04.991 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:07:04.991 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:07:04.991 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:07:04.991 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:24.502212+0000", 2023-09-23T21:07:04.991 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.9080000000000001e-06 2023-09-23T21:07:04.991 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:07:04.991 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:07:04.991 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:07:04.991 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:25.413366+0000", 2023-09-23T21:07:04.991 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.91115327199999996 2023-09-23T21:07:04.991 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:07:04.991 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:07:04.991 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:07:04.992 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:25.413462+0000", 2023-09-23T21:07:04.992 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.6717000000000003e-05 2023-09-23T21:07:04.992 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:07:04.992 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:07:04.992 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 1,3", 2023-09-23T21:07:04.992 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:25.413554+0000", 2023-09-23T21:07:04.992 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.1895e-05 2023-09-23T21:07:04.992 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:07:04.992 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:07:04.992 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:07:04.992 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:25.417800+0000", 2023-09-23T21:07:04.992 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0042454440000000001 2023-09-23T21:07:04.993 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:07:04.993 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:07:04.993 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:07:04.993 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:25.423772+0000", 2023-09-23T21:07:04.993 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0059727590000000002 2023-09-23T21:07:04.993 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:07:04.993 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:07:04.993 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:07:04.993 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:25.431044+0000", 2023-09-23T21:07:04.993 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0072718139999999997 2023-09-23T21:07:04.993 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:07:04.993 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:07:04.993 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:07:04.994 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:25.431098+0000", 2023-09-23T21:07:04.994 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.3832999999999998e-05 2023-09-23T21:07:04.994 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:07:04.994 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:07:04.994 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:07:04.994 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:25.431141+0000", 2023-09-23T21:07:04.994 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.3282000000000003e-05 2023-09-23T21:07:04.994 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:07:04.994 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:07:04.994 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:07:04.994 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:07:04.994 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:07:04.995 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.35856.0:52915 9.3 9:ca2a2df5:::benchmark_data_smithi026_342732_object52914:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e354)", 2023-09-23T21:07:04.995 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:06:24.520294+0000", 2023-09-23T21:07:04.995 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 40.385589318999997, 2023-09-23T21:07:04.995 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.91094416700000003, 2023-09-23T21:07:04.995 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:07:04.995 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:07:04.995 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:07:04.995 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.35856", 2023-09-23T21:07:04.995 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/4180396258", 2023-09-23T21:07:04.995 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 52915 2023-09-23T21:07:04.995 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:07:04.995 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:07:04.996 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:07:04.996 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:07:04.996 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:24.520294+0000", 2023-09-23T21:07:04.996 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:07:04.996 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:07:04.996 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:07:04.996 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:07:04.996 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:24.520294+0000", 2023-09-23T21:07:04.996 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:07:04.996 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:07:04.996 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:07:04.996 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:07:04.996 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:24.520293+0000", 2023-09-23T21:07:04.997 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T21:07:04.997 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:07:04.997 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:07:04.997 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:07:04.997 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:24.520346+0000", 2023-09-23T21:07:04.997 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.3087999999999997e-05 2023-09-23T21:07:04.997 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:07:04.997 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:07:04.997 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:07:04.997 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:24.520357+0000", 2023-09-23T21:07:04.997 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.1335000000000001e-05 2023-09-23T21:07:04.997 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:07:04.998 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:07:04.998 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:07:04.998 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:24.520362+0000", 2023-09-23T21:07:04.998 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.5839999999999996e-06 2023-09-23T21:07:04.998 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:07:04.998 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:07:04.998 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:07:04.998 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:24.520410+0000", 2023-09-23T21:07:04.998 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.7945000000000003e-05 2023-09-23T21:07:04.998 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:07:04.998 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:07:04.998 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:07:04.998 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:24.520418+0000", 2023-09-23T21:07:04.999 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.7029999999999995e-06 2023-09-23T21:07:04.999 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:07:04.999 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:07:04.999 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:07:04.999 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:25.414439+0000", 2023-09-23T21:07:04.999 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.89402078900000004 2023-09-23T21:07:04.999 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:07:04.999 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:07:04.999 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:07:04.999 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:25.414565+0000", 2023-09-23T21:07:04.999 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00012614700000000001 2023-09-23T21:07:04.999 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:07:05.000 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:07:05.000 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 1,3", 2023-09-23T21:07:05.000 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:25.414721+0000", 2023-09-23T21:07:05.000 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000155601 2023-09-23T21:07:05.000 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:07:05.000 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:07:05.000 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:07:05.000 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:25.420955+0000", 2023-09-23T21:07:05.000 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0062335200000000002 2023-09-23T21:07:05.000 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:07:05.000 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:07:05.000 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:07:05.000 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:25.423821+0000", 2023-09-23T21:07:05.001 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0028662779999999999 2023-09-23T21:07:05.001 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:07:05.001 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:07:05.001 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:07:05.001 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:25.431172+0000", 2023-09-23T21:07:05.001 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0073511369999999998 2023-09-23T21:07:05.001 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:07:05.001 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:07:05.001 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:07:05.001 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:25.431206+0000", 2023-09-23T21:07:05.001 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.4307000000000002e-05 2023-09-23T21:07:05.001 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:07:05.001 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:07:05.002 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:07:05.002 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:25.431238+0000", 2023-09-23T21:07:05.002 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.2027999999999999e-05 2023-09-23T21:07:05.002 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:07:05.002 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:07:05.002 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:07:05.002 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:07:05.002 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:07:05.002 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.35856.0:52919 9.3 9:c1c34947:::benchmark_data_smithi026_342732_object52918:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e354)", 2023-09-23T21:07:05.002 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:06:24.523819+0000", 2023-09-23T21:07:05.002 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 40.382064251999999, 2023-09-23T21:07:05.003 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.90866214300000003, 2023-09-23T21:07:05.003 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:07:05.003 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:07:05.003 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:07:05.003 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.35856", 2023-09-23T21:07:05.003 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/4180396258", 2023-09-23T21:07:05.003 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 52919 2023-09-23T21:07:05.003 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:07:05.003 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:07:05.003 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:07:05.003 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:07:05.003 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:24.523819+0000", 2023-09-23T21:07:05.003 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:07:05.004 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:07:05.004 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:07:05.004 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:07:05.004 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:24.523819+0000", 2023-09-23T21:07:05.004 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:07:05.004 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:07:05.004 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:07:05.004 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:07:05.004 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:24.523818+0000", 2023-09-23T21:07:05.004 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.999999 2023-09-23T21:07:05.004 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:07:05.004 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:07:05.005 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:07:05.005 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:24.523838+0000", 2023-09-23T21:07:05.005 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.9938000000000001e-05 2023-09-23T21:07:05.005 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:07:05.005 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:07:05.005 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:07:05.005 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:24.523849+0000", 2023-09-23T21:07:05.005 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.0617e-05 2023-09-23T21:07:05.005 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:07:05.005 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:07:05.005 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:07:05.005 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:24.523853+0000", 2023-09-23T21:07:05.005 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.2860000000000001e-06 2023-09-23T21:07:05.006 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:07:05.006 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:07:05.006 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:07:05.006 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:24.523911+0000", 2023-09-23T21:07:05.006 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.8381000000000002e-05 2023-09-23T21:07:05.006 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:07:05.006 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:07:05.006 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:07:05.006 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:24.523920+0000", 2023-09-23T21:07:05.006 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.8279999999999992e-06 2023-09-23T21:07:05.006 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:07:05.006 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:07:05.007 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:07:05.007 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:25.415624+0000", 2023-09-23T21:07:05.007 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.89170378500000003 2023-09-23T21:07:05.007 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:07:05.007 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:07:05.007 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:07:05.007 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:25.415717+0000", 2023-09-23T21:07:05.007 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.3216999999999999e-05 2023-09-23T21:07:05.007 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:07:05.007 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:07:05.007 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 1,3", 2023-09-23T21:07:05.007 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:25.415813+0000", 2023-09-23T21:07:05.008 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.5261999999999997e-05 2023-09-23T21:07:05.008 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:07:05.008 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:07:05.008 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:07:05.008 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:25.420983+0000", 2023-09-23T21:07:05.008 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0051701919999999997 2023-09-23T21:07:05.008 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:07:05.008 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:07:05.008 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:07:05.008 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:25.429934+0000", 2023-09-23T21:07:05.009 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0089515629999999992 2023-09-23T21:07:05.009 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:07:05.009 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:07:05.009 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:07:05.009 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:25.432395+0000", 2023-09-23T21:07:05.009 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0024608130000000001 2023-09-23T21:07:05.009 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:07:05.009 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:07:05.009 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:07:05.009 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:25.432445+0000", 2023-09-23T21:07:05.010 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.0336000000000001e-05 2023-09-23T21:07:05.010 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:07:05.010 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:07:05.010 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:07:05.010 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:25.432481+0000", 2023-09-23T21:07:05.010 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.5899999999999998e-05 2023-09-23T21:07:05.010 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:07:05.010 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:07:05.010 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:07:05.011 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:07:05.011 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:07:05.011 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.35856.0:52927 9.3 9:c0ef7e28:::benchmark_data_smithi026_342732_object52926:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e354)", 2023-09-23T21:07:05.011 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:06:24.534140+0000", 2023-09-23T21:07:05.011 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 40.371743148999997, 2023-09-23T21:07:05.011 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.90076572399999999, 2023-09-23T21:07:05.011 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:07:05.011 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:07:05.011 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:07:05.011 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.35856", 2023-09-23T21:07:05.012 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/4180396258", 2023-09-23T21:07:05.012 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 52927 2023-09-23T21:07:05.012 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:07:05.012 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:07:05.012 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:07:05.012 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:07:05.012 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:24.534140+0000", 2023-09-23T21:07:05.012 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:07:05.012 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:07:05.012 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:07:05.013 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:07:05.013 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:24.534140+0000", 2023-09-23T21:07:05.013 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:07:05.013 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:07:05.013 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:07:05.013 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:07:05.013 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:24.534139+0000", 2023-09-23T21:07:05.013 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T21:07:05.013 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:07:05.013 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:07:05.013 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:07:05.014 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:24.534175+0000", 2023-09-23T21:07:05.014 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.5553999999999999e-05 2023-09-23T21:07:05.014 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:07:05.014 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:07:05.014 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:07:05.014 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:24.534186+0000", 2023-09-23T21:07:05.014 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.1334e-05 2023-09-23T21:07:05.014 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:07:05.014 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:07:05.014 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:07:05.015 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:24.534192+0000", 2023-09-23T21:07:05.015 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.9950000000000002e-06 2023-09-23T21:07:05.015 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:07:05.015 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:07:05.015 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:07:05.015 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:24.534258+0000", 2023-09-23T21:07:05.015 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.6299999999999999e-05 2023-09-23T21:07:05.015 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:07:05.015 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:07:05.015 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:07:05.016 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:24.534275+0000", 2023-09-23T21:07:05.016 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.6691e-05 2023-09-23T21:07:05.016 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:07:05.016 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:07:05.016 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:07:05.016 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:25.416673+0000", 2023-09-23T21:07:05.016 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.88239785500000001 2023-09-23T21:07:05.016 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:07:05.016 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:07:05.017 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:07:05.017 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:25.416787+0000", 2023-09-23T21:07:05.017 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000114636 2023-09-23T21:07:05.017 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:07:05.017 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:07:05.017 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 1,3", 2023-09-23T21:07:05.017 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:25.416936+0000", 2023-09-23T21:07:05.017 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00014905300000000001 2023-09-23T21:07:05.017 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:07:05.017 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:07:05.018 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:07:05.018 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:25.421097+0000", 2023-09-23T21:07:05.018 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0041605940000000001 2023-09-23T21:07:05.018 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:07:05.018 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:07:05.018 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:07:05.018 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:25.433057+0000", 2023-09-23T21:07:05.018 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.011959981999999999 2023-09-23T21:07:05.018 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:07:05.019 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:07:05.019 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:07:05.019 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:25.434817+0000", 2023-09-23T21:07:05.019 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0017596090000000001 2023-09-23T21:07:05.019 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:07:05.019 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:07:05.019 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:07:05.019 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:25.434871+0000", 2023-09-23T21:07:05.019 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.4228000000000003e-05 2023-09-23T21:07:05.019 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:07:05.020 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:07:05.020 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:07:05.020 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:25.434906+0000", 2023-09-23T21:07:05.020 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.5113000000000001e-05 2023-09-23T21:07:05.020 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:07:05.020 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:07:05.020 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:07:05.020 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:07:05.021 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:07:05.021 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.35856.0:52943 9.3 9:cce7983c:::benchmark_data_smithi026_342732_object52942:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e354)", 2023-09-23T21:07:05.021 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:06:24.551421+0000", 2023-09-23T21:07:05.021 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 40.354462839999997, 2023-09-23T21:07:05.021 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.88400061399999996, 2023-09-23T21:07:05.021 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:07:05.021 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:07:05.021 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:07:05.022 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.35856", 2023-09-23T21:07:05.022 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/4180396258", 2023-09-23T21:07:05.022 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 52943 2023-09-23T21:07:05.022 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:07:05.022 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:07:05.022 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:07:05.022 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:07:05.022 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:24.551421+0000", 2023-09-23T21:07:05.022 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:07:05.023 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:07:05.023 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:07:05.023 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:07:05.023 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:24.551421+0000", 2023-09-23T21:07:05.023 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:07:05.023 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:07:05.023 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:07:05.023 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:07:05.023 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:24.551419+0000", 2023-09-23T21:07:05.023 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T21:07:05.023 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:07:05.023 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:07:05.024 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:07:05.024 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:24.551465+0000", 2023-09-23T21:07:05.024 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.5859000000000003e-05 2023-09-23T21:07:05.024 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:07:05.024 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:07:05.024 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:07:05.024 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:24.551478+0000", 2023-09-23T21:07:05.024 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.2401999999999999e-05 2023-09-23T21:07:05.024 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:07:05.024 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:07:05.024 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:07:05.024 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:24.551482+0000", 2023-09-23T21:07:05.024 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.8180000000000003e-06 2023-09-23T21:07:05.025 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:07:05.025 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:07:05.025 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:07:05.025 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:24.551531+0000", 2023-09-23T21:07:05.025 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.8235999999999997e-05 2023-09-23T21:07:05.025 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:07:05.025 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:07:05.025 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:07:05.025 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:24.551539+0000", 2023-09-23T21:07:05.025 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.5790000000000004e-06 2023-09-23T21:07:05.025 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:07:05.025 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:07:05.026 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:07:05.026 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:25.417837+0000", 2023-09-23T21:07:05.026 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.86629784499999996 2023-09-23T21:07:05.026 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:07:05.026 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:07:05.026 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:07:05.026 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:25.417926+0000", 2023-09-23T21:07:05.026 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.8490000000000001e-05 2023-09-23T21:07:05.026 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:07:05.026 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:07:05.026 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 1,3", 2023-09-23T21:07:05.026 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:25.418026+0000", 2023-09-23T21:07:05.027 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000100294 2023-09-23T21:07:05.027 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:07:05.027 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:07:05.027 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:07:05.027 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:25.421105+0000", 2023-09-23T21:07:05.027 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.003079072 2023-09-23T21:07:05.027 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:07:05.027 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:07:05.027 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:07:05.027 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:25.434959+0000", 2023-09-23T21:07:05.027 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.013854083 2023-09-23T21:07:05.027 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:07:05.027 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:07:05.028 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:07:05.028 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:25.435329+0000", 2023-09-23T21:07:05.028 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00036980599999999998 2023-09-23T21:07:05.028 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:07:05.028 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:07:05.028 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:07:05.028 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:25.435383+0000", 2023-09-23T21:07:05.028 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.3835000000000002e-05 2023-09-23T21:07:05.028 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:07:05.028 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:07:05.028 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:07:05.028 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:25.435421+0000", 2023-09-23T21:07:05.029 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.8525000000000001e-05 2023-09-23T21:07:05.029 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:07:05.029 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:07:05.029 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:07:05.029 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:07:05.029 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:07:05.029 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.35856.0:52979 9.3 9:cb868066:::benchmark_data_smithi026_342732_object52978:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e354)", 2023-09-23T21:07:05.029 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:06:24.605028+0000", 2023-09-23T21:07:05.029 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 40.300855165999998, 2023-09-23T21:07:05.029 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.83054075699999996, 2023-09-23T21:07:05.029 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:07:05.029 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:07:05.029 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:07:05.030 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.35856", 2023-09-23T21:07:05.030 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/4180396258", 2023-09-23T21:07:05.030 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 52979 2023-09-23T21:07:05.030 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:07:05.030 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:07:05.030 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:07:05.030 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:07:05.030 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:24.605028+0000", 2023-09-23T21:07:05.030 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:07:05.030 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:07:05.030 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:07:05.030 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:07:05.031 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:24.605028+0000", 2023-09-23T21:07:05.031 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:07:05.031 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:07:05.031 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:07:05.031 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:07:05.031 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:24.605027+0000", 2023-09-23T21:07:05.031 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T21:07:05.031 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:07:05.031 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:07:05.031 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:07:05.031 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:24.605061+0000", 2023-09-23T21:07:05.031 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.4298000000000003e-05 2023-09-23T21:07:05.032 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:07:05.032 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:07:05.032 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:07:05.032 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:24.605073+0000", 2023-09-23T21:07:05.032 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.2225999999999999e-05 2023-09-23T21:07:05.032 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:07:05.032 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:07:05.032 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:07:05.032 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:24.605078+0000", 2023-09-23T21:07:05.032 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.7740000000000004e-06 2023-09-23T21:07:05.032 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:07:05.032 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:07:05.033 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:07:05.033 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:24.605126+0000", 2023-09-23T21:07:05.033 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.7639000000000002e-05 2023-09-23T21:07:05.033 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:07:05.033 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:07:05.033 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:07:05.033 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:24.605132+0000", 2023-09-23T21:07:05.033 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.7270000000000001e-06 2023-09-23T21:07:05.033 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:07:05.033 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:07:05.033 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:07:05.033 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:25.418852+0000", 2023-09-23T21:07:05.034 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.81371994000000003 2023-09-23T21:07:05.034 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:07:05.034 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:07:05.034 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:07:05.034 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:25.418923+0000", 2023-09-23T21:07:05.034 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.0716000000000005e-05 2023-09-23T21:07:05.034 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:07:05.034 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:07:05.034 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 1,3", 2023-09-23T21:07:05.034 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:25.419029+0000", 2023-09-23T21:07:05.034 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000105603 2023-09-23T21:07:05.034 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:07:05.034 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:07:05.035 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:07:05.035 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:25.421113+0000", 2023-09-23T21:07:05.035 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0020848220000000001 2023-09-23T21:07:05.035 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:07:05.035 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:07:05.035 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:07:05.035 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:25.435018+0000", 2023-09-23T21:07:05.035 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.013904158 2023-09-23T21:07:05.035 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:07:05.035 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:07:05.035 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:07:05.035 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:25.435475+0000", 2023-09-23T21:07:05.036 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00045726399999999998 2023-09-23T21:07:05.036 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:07:05.036 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:07:05.036 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:07:05.036 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:25.435527+0000", 2023-09-23T21:07:05.036 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.2583000000000001e-05 2023-09-23T21:07:05.036 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:07:05.036 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:07:05.036 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:07:05.036 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:25.435569+0000", 2023-09-23T21:07:05.036 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.1551999999999998e-05 2023-09-23T21:07:05.036 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:07:05.036 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:07:05.037 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:07:05.037 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:07:05.037 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:07:05.037 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.35856.0:53017 9.3 9:c443da72:::benchmark_data_smithi026_342732_object53016:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e354)", 2023-09-23T21:07:05.037 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:06:24.701314+0000", 2023-09-23T21:07:05.037 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 40.204569786, 2023-09-23T21:07:05.037 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.73438963099999999, 2023-09-23T21:07:05.037 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:07:05.037 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:07:05.037 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:07:05.037 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.35856", 2023-09-23T21:07:05.037 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/4180396258", 2023-09-23T21:07:05.038 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 53017 2023-09-23T21:07:05.038 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:07:05.038 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:07:05.038 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:07:05.038 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:07:05.038 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:24.701314+0000", 2023-09-23T21:07:05.038 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:07:05.038 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:07:05.038 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:07:05.038 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:07:05.038 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:24.701314+0000", 2023-09-23T21:07:05.038 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:07:05.039 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:07:05.039 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:07:05.039 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:07:05.039 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:24.701307+0000", 2023-09-23T21:07:05.039 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999933 2023-09-23T21:07:05.039 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:07:05.039 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:07:05.039 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:07:05.039 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:24.701345+0000", 2023-09-23T21:07:05.039 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.8661999999999997e-05 2023-09-23T21:07:05.039 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:07:05.039 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:07:05.040 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:07:05.040 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:24.701357+0000", 2023-09-23T21:07:05.040 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.1907e-05 2023-09-23T21:07:05.040 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:07:05.040 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:07:05.040 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:07:05.040 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:24.701362+0000", 2023-09-23T21:07:05.040 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.5979999999999999e-06 2023-09-23T21:07:05.040 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:07:05.040 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:07:05.040 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:07:05.041 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:24.701410+0000", 2023-09-23T21:07:05.041 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.8534e-05 2023-09-23T21:07:05.041 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:07:05.041 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:07:05.041 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:07:05.041 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:24.701417+0000", 2023-09-23T21:07:05.041 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.5780000000000004e-06 2023-09-23T21:07:05.041 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:07:05.041 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:07:05.042 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:07:05.042 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:25.419853+0000", 2023-09-23T21:07:05.042 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.71843563300000002 2023-09-23T21:07:05.042 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:07:05.042 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:07:05.042 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:07:05.042 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:25.419985+0000", 2023-09-23T21:07:05.042 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00013251699999999999 2023-09-23T21:07:05.042 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:07:05.042 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:07:05.042 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 1,3", 2023-09-23T21:07:05.042 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:25.420143+0000", 2023-09-23T21:07:05.043 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00015774499999999999 2023-09-23T21:07:05.043 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:07:05.043 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:07:05.043 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:07:05.043 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:25.421530+0000", 2023-09-23T21:07:05.043 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.001387466 2023-09-23T21:07:05.043 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:07:05.043 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:07:05.043 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:07:05.044 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:25.435064+0000", 2023-09-23T21:07:05.044 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.013533880999999999 2023-09-23T21:07:05.044 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:07:05.044 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:07:05.044 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:07:05.044 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:25.435619+0000", 2023-09-23T21:07:05.044 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00055499900000000003 2023-09-23T21:07:05.044 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:07:05.044 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:07:05.044 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:07:05.045 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:25.435668+0000", 2023-09-23T21:07:05.045 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.8532000000000002e-05 2023-09-23T21:07:05.045 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:07:05.045 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:07:05.045 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:07:05.045 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:25.435703+0000", 2023-09-23T21:07:05.045 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.5471e-05 2023-09-23T21:07:05.045 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:07:05.045 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:07:05.046 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:07:05.046 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:07:05.046 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:07:05.046 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.35856.0:56890 9.3 9:c5ce06a9:::benchmark_data_smithi026_342732_object56889:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e356)", 2023-09-23T21:07:05.046 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:06:28.222626+0000", 2023-09-23T21:07:05.046 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 36.683257851999997, 2023-09-23T21:07:05.046 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.10717096199999999, 2023-09-23T21:07:05.046 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:07:05.046 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:07:05.046 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:07:05.047 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.35856", 2023-09-23T21:07:05.047 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/4180396258", 2023-09-23T21:07:05.047 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 56890 2023-09-23T21:07:05.047 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:07:05.047 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:07:05.047 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:07:05.047 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:07:05.047 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:28.222626+0000", 2023-09-23T21:07:05.048 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:07:05.048 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:07:05.048 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:07:05.048 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:07:05.048 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:28.222626+0000", 2023-09-23T21:07:05.048 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:07:05.048 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:07:05.048 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:07:05.048 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:07:05.048 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:28.222624+0000", 2023-09-23T21:07:05.049 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T21:07:05.049 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:07:05.049 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:07:05.049 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:07:05.049 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:28.222678+0000", 2023-09-23T21:07:05.049 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.3791e-05 2023-09-23T21:07:05.049 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:07:05.049 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:07:05.049 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:07:05.049 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:28.222688+0000", 2023-09-23T21:07:05.050 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.0379e-05 2023-09-23T21:07:05.050 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:07:05.050 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:07:05.050 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:07:05.050 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:28.222694+0000", 2023-09-23T21:07:05.050 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.4430000000000002e-06 2023-09-23T21:07:05.050 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:07:05.050 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:07:05.050 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:07:05.051 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:28.222754+0000", 2023-09-23T21:07:05.051 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.0272000000000002e-05 2023-09-23T21:07:05.051 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:07:05.051 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:07:05.051 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:07:05.051 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:28.222892+0000", 2023-09-23T21:07:05.051 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00013820600000000001 2023-09-23T21:07:05.051 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:07:05.051 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:07:05.051 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 1,3", 2023-09-23T21:07:05.052 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:28.223040+0000", 2023-09-23T21:07:05.052 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000147136 2023-09-23T21:07:05.052 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:07:05.052 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:07:05.052 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:07:05.052 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:28.227304+0000", 2023-09-23T21:07:05.052 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0042642050000000001 2023-09-23T21:07:05.052 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:07:05.052 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:07:05.052 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:07:05.053 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:28.229700+0000", 2023-09-23T21:07:05.053 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0023961970000000001 2023-09-23T21:07:05.053 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:07:05.053 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:07:05.053 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:07:05.053 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:28.329703+0000", 2023-09-23T21:07:05.053 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.100003326 2023-09-23T21:07:05.053 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:07:05.053 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:07:05.053 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:07:05.053 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:28.329757+0000", 2023-09-23T21:07:05.053 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.3871e-05 2023-09-23T21:07:05.054 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:07:05.054 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:07:05.054 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:07:05.054 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:28.329797+0000", 2023-09-23T21:07:05.054 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.9415000000000001e-05 2023-09-23T21:07:05.054 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:07:05.054 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:07:05.054 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:07:05.054 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:07:05.055 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:07:05.055 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.35856.0:56893 9.3 9:c8734263:::benchmark_data_smithi026_342732_object56892:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e356)", 2023-09-23T21:07:05.055 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:06:28.222707+0000", 2023-09-23T21:07:05.055 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 36.683176770000003, 2023-09-23T21:07:05.055 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.107248406, 2023-09-23T21:07:05.055 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:07:05.055 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:07:05.055 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:07:05.055 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.35856", 2023-09-23T21:07:05.056 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/4180396258", 2023-09-23T21:07:05.056 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 56893 2023-09-23T21:07:05.056 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:07:05.056 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:07:05.056 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:07:05.056 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:07:05.056 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:28.222707+0000", 2023-09-23T21:07:05.056 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:07:05.056 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:07:05.056 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:07:05.057 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:07:05.057 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:28.222707+0000", 2023-09-23T21:07:05.057 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:07:05.057 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:07:05.057 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:07:05.057 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:07:05.057 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:28.222706+0000", 2023-09-23T21:07:05.057 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999995 2023-09-23T21:07:05.057 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:07:05.057 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:07:05.057 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:07:05.057 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:28.222737+0000", 2023-09-23T21:07:05.057 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.1038999999999999e-05 2023-09-23T21:07:05.058 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:07:05.058 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:07:05.058 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:07:05.058 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:28.222744+0000", 2023-09-23T21:07:05.058 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.7299999999999999e-06 2023-09-23T21:07:05.058 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:07:05.058 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:07:05.058 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:07:05.058 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:28.222748+0000", 2023-09-23T21:07:05.058 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.1289999999999998e-06 2023-09-23T21:07:05.058 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:07:05.058 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:07:05.059 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:07:05.059 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:28.224892+0000", 2023-09-23T21:07:05.059 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.002144254 2023-09-23T21:07:05.059 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:07:05.059 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:07:05.059 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:07:05.059 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:28.225023+0000", 2023-09-23T21:07:05.059 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000131099 2023-09-23T21:07:05.059 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:07:05.059 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:07:05.059 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 1,3", 2023-09-23T21:07:05.059 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:28.225161+0000", 2023-09-23T21:07:05.060 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00013770199999999999 2023-09-23T21:07:05.060 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:07:05.060 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:07:05.060 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:07:05.060 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:28.227328+0000", 2023-09-23T21:07:05.060 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0021671529999999998 2023-09-23T21:07:05.060 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:07:05.060 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:07:05.060 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:07:05.060 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:28.229730+0000", 2023-09-23T21:07:05.060 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0024018970000000001 2023-09-23T21:07:05.060 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:07:05.061 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:07:05.061 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:07:05.061 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:28.329881+0000", 2023-09-23T21:07:05.061 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.10015086500000001 2023-09-23T21:07:05.061 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:07:05.061 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:07:05.061 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:07:05.061 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:28.329922+0000", 2023-09-23T21:07:05.061 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.1134999999999998e-05 2023-09-23T21:07:05.061 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:07:05.061 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:07:05.061 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:07:05.061 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:28.329955+0000", 2023-09-23T21:07:05.062 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.2926999999999998e-05 2023-09-23T21:07:05.062 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:07:05.062 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:07:05.062 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:07:05.062 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:07:05.062 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:07:05.062 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.35856.0:61412 9.3 9:c8c3de8e:::benchmark_data_smithi026_342732_object61411:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e356)", 2023-09-23T21:07:05.062 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:06:31.510214+0000", 2023-09-23T21:07:05.062 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 33.395668946000001, 2023-09-23T21:07:05.062 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.060346836000000001, 2023-09-23T21:07:05.062 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:07:05.062 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:07:05.063 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:07:05.063 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.35856", 2023-09-23T21:07:05.063 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/4180396258", 2023-09-23T21:07:05.063 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 61412 2023-09-23T21:07:05.063 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:07:05.063 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:07:05.063 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:07:05.063 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:07:05.063 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:31.510214+0000", 2023-09-23T21:07:05.063 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:07:05.063 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:07:05.063 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:07:05.064 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:07:05.064 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:31.510214+0000", 2023-09-23T21:07:05.064 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:07:05.064 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:07:05.064 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:07:05.064 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:07:05.064 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:31.510213+0000", 2023-09-23T21:07:05.064 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.999999 2023-09-23T21:07:05.064 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:07:05.064 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:07:05.064 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:07:05.064 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:31.510327+0000", 2023-09-23T21:07:05.064 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000113343 2023-09-23T21:07:05.065 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:07:05.065 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:07:05.065 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:07:05.065 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:31.510342+0000", 2023-09-23T21:07:05.065 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.5464999999999998e-05 2023-09-23T21:07:05.065 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:07:05.065 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:07:05.065 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:07:05.065 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:31.510349+0000", 2023-09-23T21:07:05.065 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.793e-06 2023-09-23T21:07:05.065 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:07:05.065 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:07:05.066 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:07:05.066 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:31.510402+0000", 2023-09-23T21:07:05.066 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.2602999999999998e-05 2023-09-23T21:07:05.066 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:07:05.066 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:07:05.066 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:07:05.066 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:31.510536+0000", 2023-09-23T21:07:05.066 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00013411599999999999 2023-09-23T21:07:05.066 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:07:05.066 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:07:05.066 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 1,3", 2023-09-23T21:07:05.066 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:31.510693+0000", 2023-09-23T21:07:05.066 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00015724700000000001 2023-09-23T21:07:05.067 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:07:05.067 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:07:05.067 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:07:05.067 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:31.512837+0000", 2023-09-23T21:07:05.067 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0021442509999999998 2023-09-23T21:07:05.067 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:07:05.067 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:07:05.067 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:07:05.067 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:31.569616+0000", 2023-09-23T21:07:05.067 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.056778745999999998 2023-09-23T21:07:05.067 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:07:05.067 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:07:05.067 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:07:05.068 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:31.570456+0000", 2023-09-23T21:07:05.068 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00083952199999999999 2023-09-23T21:07:05.068 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:07:05.068 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:07:05.068 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:07:05.068 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:31.570515+0000", 2023-09-23T21:07:05.068 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.9808000000000001e-05 2023-09-23T21:07:05.068 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:07:05.068 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:07:05.068 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:07:05.068 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:31.570561+0000", 2023-09-23T21:07:05.068 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.5989000000000003e-05 2023-09-23T21:07:05.068 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:07:05.069 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:07:05.069 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:07:05.069 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:07:05.069 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:07:05.069 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.35856.0:62082 9.3 9:cca46907:::benchmark_data_smithi026_342732_object62081:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e356)", 2023-09-23T21:07:05.069 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:06:32.139007+0000", 2023-09-23T21:07:05.069 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 32.766876592999999, 2023-09-23T21:07:05.069 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.061872798, 2023-09-23T21:07:05.069 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:07:05.069 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:07:05.069 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:07:05.069 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.35856", 2023-09-23T21:07:05.070 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/4180396258", 2023-09-23T21:07:05.070 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 62082 2023-09-23T21:07:05.070 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:07:05.070 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:07:05.070 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:07:05.070 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:07:05.070 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:32.139007+0000", 2023-09-23T21:07:05.070 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:07:05.070 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:07:05.070 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:07:05.070 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:07:05.070 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:32.139007+0000", 2023-09-23T21:07:05.071 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:07:05.071 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:07:05.071 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:07:05.071 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:07:05.071 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:32.139006+0000", 2023-09-23T21:07:05.071 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.999999 2023-09-23T21:07:05.071 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:07:05.071 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:07:05.071 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:07:05.071 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:32.139101+0000", 2023-09-23T21:07:05.071 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.5730999999999995e-05 2023-09-23T21:07:05.071 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:07:05.071 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:07:05.072 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:07:05.072 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:32.139110+0000", 2023-09-23T21:07:05.072 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.6309999999999992e-06 2023-09-23T21:07:05.072 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:07:05.072 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:07:05.072 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:07:05.072 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:32.139115+0000", 2023-09-23T21:07:05.072 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.5580000000000002e-06 2023-09-23T21:07:05.072 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:07:05.072 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:07:05.072 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:07:05.072 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:32.139515+0000", 2023-09-23T21:07:05.072 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00039988699999999997 2023-09-23T21:07:05.073 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:07:05.073 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:07:05.073 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:07:05.073 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:32.139618+0000", 2023-09-23T21:07:05.073 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000103398 2023-09-23T21:07:05.073 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:07:05.073 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:07:05.073 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 1,3", 2023-09-23T21:07:05.073 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:32.139739+0000", 2023-09-23T21:07:05.073 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00012105800000000001 2023-09-23T21:07:05.073 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:07:05.073 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:07:05.074 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:07:05.074 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:32.142077+0000", 2023-09-23T21:07:05.074 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0023383190000000002 2023-09-23T21:07:05.074 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:07:05.074 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:07:05.074 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:07:05.074 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:32.143993+0000", 2023-09-23T21:07:05.074 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.001915845 2023-09-23T21:07:05.074 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:07:05.074 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:07:05.074 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:07:05.074 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:32.200790+0000", 2023-09-23T21:07:05.075 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.056797154000000002 2023-09-23T21:07:05.075 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:07:05.075 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:07:05.075 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:07:05.075 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:32.200842+0000", 2023-09-23T21:07:05.075 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.2185000000000003e-05 2023-09-23T21:07:05.075 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:07:05.075 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:07:05.075 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:07:05.075 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:32.200880+0000", 2023-09-23T21:07:05.075 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.7162000000000001e-05 2023-09-23T21:07:05.075 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:07:05.076 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:07:05.076 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:07:05.076 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:07:05.076 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:07:05.076 INFO:teuthology.orchestra.run.smithi070.stdout:} 2023-09-23T21:07:05.077 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:07:05.077 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:07:04 smithi026 ceph-mon[175151]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:07:05.077 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:07:04 smithi026 ceph-mon[175151]: failed to encode map e363 with expected crc 2023-09-23T21:07:05.234 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:07:04 smithi070 ceph-mon[168282]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:07:05.234 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:07:04 smithi070 ceph-mon[168282]: failed to encode map e363 with expected crc 2023-09-23T21:07:05.234 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:07:04 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T21:07:05.234 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:07:04 smithi070 ceph-mon[168282]: osdmap e363: 8 total, 8 up, 8 in 2023-09-23T21:07:05.258 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_ops_in_flight 2023-09-23T21:07:05.294 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 69 16 95637 95621 86.5455 56.6875 0.0200886 0.0115495 2023-09-23T21:07:05.383 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:07:04 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T21:07:05.384 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:07:04 smithi026 ceph-mon[175151]: osdmap e363: 8 total, 8 up, 8 in 2023-09-23T21:07:05.384 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:07:04 smithi026 ceph-mon[179688]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:07:05.384 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:07:04 smithi026 ceph-mon[179688]: failed to encode map e363 with expected crc 2023-09-23T21:07:05.384 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:07:04 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T21:07:05.384 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:07:04 smithi026 ceph-mon[179688]: osdmap e363: 8 total, 8 up, 8 in 2023-09-23T21:07:05.415 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:07:05.421 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:07:05.422 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:07:05.522 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:07:05.593 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:07:05.655 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:07:05.658 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:07:05.658 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:07:05.759 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:07:05.817 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:07:05.884 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:07:05.890 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:07:05.890 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:07:05.991 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:07:06.044 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T21:07:06.048 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:07:06.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:07:05 smithi026 ceph-mon[175151]: failed to encode map e363 with expected crc 2023-09-23T21:07:06.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:07:05 smithi026 ceph-mon[175151]: failed to encode map e363 with expected crc 2023-09-23T21:07:06.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:07:05 smithi026 ceph-mon[175151]: pgmap v941: 109 pgs: 109 active+clean; 5.7 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 75 MiB/s wr, 1.19k op/s 2023-09-23T21:07:06.135 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:07:05 smithi026 ceph-mon[179688]: failed to encode map e363 with expected crc 2023-09-23T21:07:06.136 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:07:05 smithi026 ceph-mon[179688]: failed to encode map e363 with expected crc 2023-09-23T21:07:06.136 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:07:05 smithi026 ceph-mon[179688]: pgmap v941: 109 pgs: 109 active+clean; 5.7 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 75 MiB/s wr, 1.19k op/s 2023-09-23T21:07:06.231 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:07:06.234 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:07:06.235 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:07:06.235 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:07:05 smithi070 ceph-mon[168282]: failed to encode map e363 with expected crc 2023-09-23T21:07:06.235 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:07:05 smithi070 ceph-mon[168282]: failed to encode map e363 with expected crc 2023-09-23T21:07:06.235 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:07:05 smithi070 ceph-mon[168282]: pgmap v941: 109 pgs: 109 active+clean; 5.7 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 75 MiB/s wr, 1.19k op/s 2023-09-23T21:07:06.253 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0: {} 2023-09-23T21:07:06.286 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1: {} 2023-09-23T21:07:06.294 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 70 16 96538 96522 86.1138 56.3125 0.0151396 0.0116069 2023-09-23T21:07:06.317 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2: {} 2023-09-23T21:07:06.335 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:07:06.378 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3: {} 2023-09-23T21:07:06.421 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:07:06.423 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4: {} 2023-09-23T21:07:06.485 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T21:07:06.529 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6: {} 2023-09-23T21:07:06.563 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T21:07:06.591 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:07:06.593 INFO:teuthology.orchestra.run.smithi026.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T21:07:06.593 INFO:teuthology.orchestra.run.smithi026.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T21:07:06.593 INFO:teuthology.orchestra.run.smithi026.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T21:07:06.593 INFO:teuthology.orchestra.run.smithi026.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T21:07:06.593 INFO:teuthology.orchestra.run.smithi026.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T21:07:06.593 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T21:07:06.593 INFO:teuthology.orchestra.run.smithi026.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T21:07:06.593 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T21:07:06.596 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:07:06.596 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:07:06.697 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:07:06.739 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:07:06.871 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:07:06.882 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:07:06.883 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:07:06.983 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:07:07.055 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:07:07.151 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:07:07.160 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:07:07.161 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:07:07.235 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:07:06 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/2611694823' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:07:07.262 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:07:07.294 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 71 16 97435 97419 85.6907 56.0625 0.0184231 0.0116654 2023-09-23T21:07:07.323 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:07:06 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/2611694823' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:07:07.324 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:07:06 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/2611694823' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:07:07.331 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:07:07.397 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T21:07:07.408 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:07:07.408 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:07:07.509 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:07:07.516 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T21:07:07.524 INFO:teuthology.orchestra.run.smithi070.stdout:ERROR: (22) Invalid argument 2023-09-23T21:07:07.525 INFO:teuthology.orchestra.run.smithi070.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T21:07:07.560 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:07:07.704 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:07:07.714 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:07:07.715 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:07:07.815 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:07:07.878 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:07:07.978 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:07:07.983 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:07:07.984 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:07:07.989 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_blocked_ops 2023-09-23T21:07:07.995 INFO:teuthology.orchestra.run.smithi026.stderr:reweighted osd.1 to 1 (10000) 2023-09-23T21:07:08.085 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:07:08.149 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:07:07 smithi070 ceph-mon[168282]: pgmap v942: 109 pgs: 109 active+clean; 5.7 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 51 MiB/s wr, 818 op/s 2023-09-23T21:07:08.149 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:07:07 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/722965454' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-09-23T21:07:08.150 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:07:08.217 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:07:08.228 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:07:08.228 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:07:08.267 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:07:07 smithi026 ceph-mon[179688]: pgmap v942: 109 pgs: 109 active+clean; 5.7 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 51 MiB/s wr, 818 op/s 2023-09-23T21:07:08.267 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:07:07 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/722965454' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-09-23T21:07:08.267 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:07:07 smithi026 ceph-mon[175151]: pgmap v942: 109 pgs: 109 active+clean; 5.7 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 51 MiB/s wr, 818 op/s 2023-09-23T21:07:08.267 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:07:07 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/722965454' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-09-23T21:07:08.294 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 72 16 98316 98300 85.2655 55.0625 0.0189352 0.0117229 2023-09-23T21:07:08.328 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:07:08.384 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:07:08.505 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T21:07:08.514 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:07:08.514 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:07:08.523 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph osd reweight 2 1 2023-09-23T21:07:08.616 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:07:08.722 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:07:08.728 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T21:07:08.795 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:07:08.812 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:07:08.817 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:07:08.817 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:07:08.918 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:07:08.980 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:07:09.033 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:07:09.039 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:07:09.039 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:07:09.088 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:07:09 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/722965454' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-09-23T21:07:09.088 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:07:09 smithi026 ceph-mon[175151]: osdmap e364: 8 total, 8 up, 8 in 2023-09-23T21:07:09.088 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:07:09 smithi026 ceph-mon[175151]: failed to encode map e364 with expected crc 2023-09-23T21:07:09.089 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:07:09 smithi026 ceph-mon[175151]: failed to encode map e364 with expected crc 2023-09-23T21:07:09.089 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:07:09 smithi026 ceph-mon[175151]: failed to encode map e364 with expected crc 2023-09-23T21:07:09.089 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:07:09 smithi026 ceph-mon[175151]: failed to encode map e364 with expected crc 2023-09-23T21:07:09.089 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:07:09 smithi026 ceph-mon[175151]: failed to encode map e364 with expected crc 2023-09-23T21:07:09.089 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:07:09 smithi026 ceph-mon[175151]: failed to encode map e364 with expected crc 2023-09-23T21:07:09.089 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:07:09 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/722965454' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-09-23T21:07:09.089 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:07:09 smithi026 ceph-mon[179688]: osdmap e364: 8 total, 8 up, 8 in 2023-09-23T21:07:09.089 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:07:09 smithi026 ceph-mon[179688]: failed to encode map e364 with expected crc 2023-09-23T21:07:09.089 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:07:09 smithi026 ceph-mon[179688]: failed to encode map e364 with expected crc 2023-09-23T21:07:09.089 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:07:09 smithi026 ceph-mon[179688]: failed to encode map e364 with expected crc 2023-09-23T21:07:09.090 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:07:09 smithi026 ceph-mon[179688]: failed to encode map e364 with expected crc 2023-09-23T21:07:09.090 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:07:09 smithi026 ceph-mon[179688]: failed to encode map e364 with expected crc 2023-09-23T21:07:09.090 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:07:09 smithi026 ceph-mon[179688]: failed to encode map e364 with expected crc 2023-09-23T21:07:09.140 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:07:09.162 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T21:07:09.225 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:07:09.287 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:07:09 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/722965454' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-09-23T21:07:09.287 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:07:09 smithi070 ceph-mon[168282]: osdmap e364: 8 total, 8 up, 8 in 2023-09-23T21:07:09.287 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:07:09 smithi070 ceph-mon[168282]: failed to encode map e364 with expected crc 2023-09-23T21:07:09.287 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:07:09 smithi070 ceph-mon[168282]: failed to encode map e364 with expected crc 2023-09-23T21:07:09.288 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:07:09 smithi070 ceph-mon[168282]: failed to encode map e364 with expected crc 2023-09-23T21:07:09.288 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:07:09 smithi070 ceph-mon[168282]: failed to encode map e364 with expected crc 2023-09-23T21:07:09.288 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:07:09 smithi070 ceph-mon[168282]: failed to encode map e364 with expected crc 2023-09-23T21:07:09.288 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:07:09 smithi070 ceph-mon[168282]: failed to encode map e364 with expected crc 2023-09-23T21:07:09.297 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 73 16 99186 99170 84.8391 54.375 0.0202199 0.0117816 2023-09-23T21:07:09.364 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:07:09.370 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:07:09.371 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:07:09.472 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:07:09.536 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:07:09.597 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:07:09.603 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:07:09.603 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:07:09.635 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 21:07:09 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[164029]: ::ffff:172.21.15.70 - - [23/Sep/2023:21:07:09] "GET /metrics HTTP/1.1" 200 36295 "" "Prometheus/2.43.0" 2023-09-23T21:07:09.704 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:07:09.770 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:07:09.777 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:07:09.856 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:07:09.866 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:07:09.867 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:07:09.967 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:07:10.012 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:07:10 smithi026 ceph-mon[179688]: pgmap v944: 109 pgs: 109 active+clean; 5.9 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 55 MiB/s wr, 884 op/s 2023-09-23T21:07:10.012 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:07:10 smithi026 ceph-mon[175151]: pgmap v944: 109 pgs: 109 active+clean; 5.9 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 55 MiB/s wr, 884 op/s 2023-09-23T21:07:10.063 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:07:10.096 INFO:teuthology.orchestra.run.smithi070.stdout:ERROR: (22) Invalid argument 2023-09-23T21:07:10.096 INFO:teuthology.orchestra.run.smithi070.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T21:07:10.143 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:07:10.149 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:07:10.149 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:07:10.250 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:07:10.297 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 74 16 99992 99976 84.3736 50.375 0.0142827 0.0118463 2023-09-23T21:07:10.310 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:07:10.371 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:07:10 smithi070 ceph-mon[168282]: pgmap v944: 109 pgs: 109 active+clean; 5.9 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 55 MiB/s wr, 884 op/s 2023-09-23T21:07:10.587 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_historic_ops 2023-09-23T21:07:10.627 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:07:10.645 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:07:10.653 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:07:10.654 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:07:10.754 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:07:10.838 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:07:10.923 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:07:10.931 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:07:10.931 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:07:11.032 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:07:11.128 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:07:11.220 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:07:11.232 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:07:11.232 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:07:11.298 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 75 16 100822 100806 83.9404 51.875 0.0178633 0.0119085 2023-09-23T21:07:11.329 INFO:teuthology.orchestra.run.smithi026.stderr:noscrub is unset 2023-09-23T21:07:11.333 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:07:11.356 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T21:07:11.396 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:07:11.516 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:07:11 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/806302438' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T21:07:11.516 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:07:11 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T21:07:11.516 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:07:11 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/806302438' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T21:07:11.516 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:07:11 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T21:07:11.566 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:07:11 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/806302438' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T21:07:11.566 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:07:11 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T21:07:11.858 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T21:07:11.872 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:07:11.873 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:07:11.974 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:07:12.038 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:07:12.144 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:07:12.148 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:07:12.148 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:07:12.249 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:07:12.298 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 76 16 101693 101677 83.5521 54.4375 0.0193974 0.0119628 2023-09-23T21:07:12.342 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:07:12.433 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:07:12.447 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:07:12.448 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:07:12.448 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:07:12 smithi026 ceph-mon[179688]: pgmap v945: 109 pgs: 109 active+clean; 6.0 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 53 MiB/s wr, 844 op/s 2023-09-23T21:07:12.448 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:07:12 smithi026 ceph-mon[179688]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:07:12.448 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:07:12 smithi026 ceph-mon[179688]: failed to encode map e365 with expected crc 2023-09-23T21:07:12.448 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:07:12 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T21:07:12.448 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:07:12 smithi026 ceph-mon[179688]: osdmap e365: 8 total, 8 up, 8 in 2023-09-23T21:07:12.448 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:07:12 smithi026 ceph-mon[179688]: failed to encode map e365 with expected crc 2023-09-23T21:07:12.448 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:07:12 smithi026 ceph-mon[179688]: failed to encode map e365 with expected crc 2023-09-23T21:07:12.448 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:07:12 smithi026 ceph-mon[179688]: failed to encode map e365 with expected crc 2023-09-23T21:07:12.449 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:07:12 smithi026 ceph-mon[175151]: pgmap v945: 109 pgs: 109 active+clean; 6.0 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 53 MiB/s wr, 844 op/s 2023-09-23T21:07:12.449 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:07:12 smithi026 ceph-mon[175151]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:07:12.449 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:07:12 smithi026 ceph-mon[175151]: failed to encode map e365 with expected crc 2023-09-23T21:07:12.449 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:07:12 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T21:07:12.449 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:07:12 smithi026 ceph-mon[175151]: osdmap e365: 8 total, 8 up, 8 in 2023-09-23T21:07:12.449 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:07:12 smithi026 ceph-mon[175151]: failed to encode map e365 with expected crc 2023-09-23T21:07:12.449 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:07:12 smithi026 ceph-mon[175151]: failed to encode map e365 with expected crc 2023-09-23T21:07:12.449 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:07:12 smithi026 ceph-mon[175151]: failed to encode map e365 with expected crc 2023-09-23T21:07:12.549 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:07:12.579 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:07:12 smithi070 ceph-mon[168282]: pgmap v945: 109 pgs: 109 active+clean; 6.0 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 53 MiB/s wr, 844 op/s 2023-09-23T21:07:12.579 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:07:12 smithi070 ceph-mon[168282]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:07:12.579 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:07:12 smithi070 ceph-mon[168282]: failed to encode map e365 with expected crc 2023-09-23T21:07:12.579 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:07:12 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T21:07:12.580 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:07:12 smithi070 ceph-mon[168282]: osdmap e365: 8 total, 8 up, 8 in 2023-09-23T21:07:12.580 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:07:12 smithi070 ceph-mon[168282]: failed to encode map e365 with expected crc 2023-09-23T21:07:12.580 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:07:12 smithi070 ceph-mon[168282]: failed to encode map e365 with expected crc 2023-09-23T21:07:12.580 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:07:12 smithi070 ceph-mon[168282]: failed to encode map e365 with expected crc 2023-09-23T21:07:12.627 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:07:12.700 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T21:07:12.716 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:07:12.717 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:07:12.748 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0: {} 2023-09-23T21:07:12.787 INFO:teuthology.orchestra.run.smithi070.stdout:ERROR: (22) Invalid argument 2023-09-23T21:07:12.788 INFO:teuthology.orchestra.run.smithi070.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T21:07:12.797 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1: {} 2023-09-23T21:07:12.818 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:07:12.842 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2: {} 2023-09-23T21:07:12.892 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3: {} 2023-09-23T21:07:12.909 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:07:12.935 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4: {} 2023-09-23T21:07:12.973 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:07:12.974 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T21:07:12.989 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:07:12.989 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:07:13.012 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6: {} 2023-09-23T21:07:13.063 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T21:07:13.068 INFO:teuthology.orchestra.run.smithi026.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T21:07:13.069 INFO:teuthology.orchestra.run.smithi026.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T21:07:13.069 INFO:teuthology.orchestra.run.smithi026.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T21:07:13.069 INFO:teuthology.orchestra.run.smithi026.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T21:07:13.069 INFO:teuthology.orchestra.run.smithi026.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T21:07:13.069 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T21:07:13.069 INFO:teuthology.orchestra.run.smithi026.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T21:07:13.069 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T21:07:13.090 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:07:13.140 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:07:13.188 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:07:13.192 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:07:13.192 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:07:13.273 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_ops_in_flight 2023-09-23T21:07:13.293 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:07:13.298 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 77 16 102498 102482 83.1207 50.3125 0.0133524 0.0120255 2023-09-23T21:07:13.335 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:07:13.335 INFO:teuthology.orchestra.run.smithi026.stderr:reweighted osd.2 to 1 (10000) 2023-09-23T21:07:13.394 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:07:13.399 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:07:13 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/1504623940' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:07:13.399 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:07:13 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/3684157326' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T21:07:13.399 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:07:13 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T21:07:13.400 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:07:13.400 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:07:13.502 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:07:13.510 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:07:13 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/1504623940' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:07:13.510 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:07:13 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/3684157326' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T21:07:13.510 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:07:13 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T21:07:13.511 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:07:13 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/1504623940' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:07:13.511 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:07:13 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/3684157326' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T21:07:13.511 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:07:13 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T21:07:13.531 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:07:13.574 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:07:13.579 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:07:13.579 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:07:13.680 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:07:13.766 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:07:13.818 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:07:13.826 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:07:13.827 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:07:13.844 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph osd reweight 3 1 2023-09-23T21:07:13.928 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:07:14.040 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:07:14.087 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T21:07:14.126 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:07:14.140 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:07:14.141 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:07:14.241 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:07:14.272 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:07:14.292 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:07:14.299 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 78 16 103306 103290 82.7022 50.5 0.0214814 0.0120869 2023-09-23T21:07:14.338 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:07:14.404 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:07:14.405 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:07:14.505 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:07:14.604 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:07:14 smithi026 ceph-mon[179688]: pgmap v947: 109 pgs: 109 active+clean; 6.1 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 51 MiB/s wr, 821 op/s 2023-09-23T21:07:14.604 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:07:14 smithi026 ceph-mon[179688]: failed to encode map e366 with expected crc 2023-09-23T21:07:14.604 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:07:14 smithi026 ceph-mon[179688]: failed to encode map e366 with expected crc 2023-09-23T21:07:14.604 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:07:14 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-09-23T21:07:14.604 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:07:14 smithi026 ceph-mon[179688]: osdmap e366: 8 total, 8 up, 8 in 2023-09-23T21:07:14.604 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:07:14 smithi026 ceph-mon[179688]: failed to encode map e366 with expected crc 2023-09-23T21:07:14.605 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:07:14.607 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:07:14 smithi026 ceph-mon[175151]: pgmap v947: 109 pgs: 109 active+clean; 6.1 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 51 MiB/s wr, 821 op/s 2023-09-23T21:07:14.607 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:07:14 smithi026 ceph-mon[175151]: failed to encode map e366 with expected crc 2023-09-23T21:07:14.607 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:07:14 smithi026 ceph-mon[175151]: failed to encode map e366 with expected crc 2023-09-23T21:07:14.607 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:07:14 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-09-23T21:07:14.607 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:07:14 smithi026 ceph-mon[175151]: osdmap e366: 8 total, 8 up, 8 in 2023-09-23T21:07:14.608 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:07:14 smithi026 ceph-mon[175151]: failed to encode map e366 with expected crc 2023-09-23T21:07:14.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:07:14 smithi070 ceph-mon[168282]: pgmap v947: 109 pgs: 109 active+clean; 6.1 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 51 MiB/s wr, 821 op/s 2023-09-23T21:07:14.621 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:07:14 smithi070 ceph-mon[168282]: failed to encode map e366 with expected crc 2023-09-23T21:07:14.621 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:07:14 smithi070 ceph-mon[168282]: failed to encode map e366 with expected crc 2023-09-23T21:07:14.621 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:07:14 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-09-23T21:07:14.621 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:07:14 smithi070 ceph-mon[168282]: osdmap e366: 8 total, 8 up, 8 in 2023-09-23T21:07:14.621 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:07:14 smithi070 ceph-mon[168282]: failed to encode map e366 with expected crc 2023-09-23T21:07:14.690 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:07:14.694 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:07:14.694 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:07:14.795 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:07:14.845 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:07:14.899 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:07:14.906 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:07:14.907 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:07:14.907 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:07:15.008 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:07:15.057 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:07:15.137 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:07:15.146 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:07:15.146 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:07:15.247 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:07:15.302 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 79 16 104138 104122 82.3102 52 0.0084656 0.0121451 2023-09-23T21:07:15.313 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:07:15.313 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:07:15.512 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T21:07:15.536 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:07:15.537 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:07:15.538 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:07:15.639 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:07:15.949 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:07:16.110 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:07:16.119 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:07:16.119 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:07:16.220 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:07:16.254 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:07:16.302 INFO:tasks.radosbench.radosbench.0.smithi026.stdout:2023-09-23T21:07:16.303155+0000 min lat: 0.00137237 max lat: 1.03425 avg lat: 0.012203 2023-09-23T21:07:16.303 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T21:07:16.303 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 80 16 104955 104939 81.9198 51.0625 0.00864895 0.012203 2023-09-23T21:07:16.324 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:07:16.327 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:07:16.327 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:07:16.428 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:07:16.498 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:07:16 smithi026 ceph-mon[179688]: pgmap v949: 109 pgs: 109 active+clean; 6.2 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 58 MiB/s wr, 925 op/s 2023-09-23T21:07:16.498 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:07:16 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:07:16.499 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:07:16 smithi026 ceph-mon[175151]: pgmap v949: 109 pgs: 109 active+clean; 6.2 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 58 MiB/s wr, 925 op/s 2023-09-23T21:07:16.500 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:07:16 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:07:16.525 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:07:16.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:07:16 smithi070 ceph-mon[168282]: pgmap v949: 109 pgs: 109 active+clean; 6.2 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 58 MiB/s wr, 925 op/s 2023-09-23T21:07:16.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:07:16 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:07:16.896 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T21:07:16.913 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:07:16.914 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:07:17.015 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:07:17.071 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:07:17.142 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:07:17.149 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:07:17.150 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:07:17.251 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:07:17.302 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 81 16 105822 105806 81.5776 54.1875 0.0247041 0.012253 2023-09-23T21:07:17.330 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:07:17.341 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:07:17.391 INFO:teuthology.orchestra.run.smithi026.stdout:{ 2023-09-23T21:07:17.392 INFO:teuthology.orchestra.run.smithi026.stdout: "ops": [ 2023-09-23T21:07:17.392 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:17.392 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35856.0:105863 9.4 9:261f31d1:::benchmark_data_smithi026_342732_object105862:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e366)", 2023-09-23T21:07:17.392 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T21:07:17.379873+0000", 2023-09-23T21:07:17.392 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 0.0094378169999999994, 2023-09-23T21:07:17.392 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0094791189999999994, 2023-09-23T21:07:17.392 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:07:17.392 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "waiting for sub ops", 2023-09-23T21:07:17.392 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T21:07:17.392 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35856", 2023-09-23T21:07:17.393 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/4180396258", 2023-09-23T21:07:17.393 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 105863 2023-09-23T21:07:17.393 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:17.393 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:07:17.393 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:17.393 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:07:17.393 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:07:17.379873+0000", 2023-09-23T21:07:17.393 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:07:17.393 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:17.393 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:17.393 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:07:17.394 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:07:17.379873+0000", 2023-09-23T21:07:17.394 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:07:17.394 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:17.394 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:17.394 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:07:17.394 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:07:17.379872+0000", 2023-09-23T21:07:17.394 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T21:07:17.394 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:17.394 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:17.394 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:07:17.394 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:07:17.379904+0000", 2023-09-23T21:07:17.395 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.2103000000000001e-05 2023-09-23T21:07:17.395 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:17.395 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:17.395 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:07:17.395 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:07:17.379921+0000", 2023-09-23T21:07:17.395 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.7478000000000001e-05 2023-09-23T21:07:17.395 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:17.395 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:17.395 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:07:17.395 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:07:17.379932+0000", 2023-09-23T21:07:17.395 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0268000000000001e-05 2023-09-23T21:07:17.396 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:17.396 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:17.396 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:07:17.396 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:07:17.379988+0000", 2023-09-23T21:07:17.396 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.6147999999999998e-05 2023-09-23T21:07:17.396 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:17.396 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:17.396 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:07:17.396 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:07:17.380152+0000", 2023-09-23T21:07:17.396 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00016391999999999999 2023-09-23T21:07:17.396 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:17.397 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:17.397 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 3,4", 2023-09-23T21:07:17.397 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:07:17.380327+0000", 2023-09-23T21:07:17.397 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000174766 2023-09-23T21:07:17.397 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:17.397 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:17.397 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:07:17.397 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:07:17.388612+0000", 2023-09-23T21:07:17.397 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0082858570000000006 2023-09-23T21:07:17.397 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:17.397 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:17.397 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T21:07:17.398 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:07:17.388643+0000", 2023-09-23T21:07:17.398 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.0145999999999999e-05 2023-09-23T21:07:17.398 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:07:17.398 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:07:17.398 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:07:17.398 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:17.398 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:17.398 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35856.0:105870 9.0 9:0855cb3d:::benchmark_data_smithi026_342732_object105869:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e366)", 2023-09-23T21:07:17.398 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T21:07:17.380003+0000", 2023-09-23T21:07:17.398 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 0.0093077920000000005, 2023-09-23T21:07:17.398 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0094484419999999996, 2023-09-23T21:07:17.399 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:07:17.399 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "waiting for sub ops", 2023-09-23T21:07:17.399 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T21:07:17.399 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35856", 2023-09-23T21:07:17.399 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/4180396258", 2023-09-23T21:07:17.399 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 105870 2023-09-23T21:07:17.399 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:17.399 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:07:17.399 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:17.399 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:07:17.399 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:07:17.380003+0000", 2023-09-23T21:07:17.399 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:07:17.400 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:17.400 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:17.400 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:07:17.400 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:07:17.380003+0000", 2023-09-23T21:07:17.400 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:07:17.400 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:17.400 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:17.400 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:07:17.400 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:07:17.380003+0000", 2023-09-23T21:07:17.400 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T21:07:17.400 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:17.400 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:17.400 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:07:17.401 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:07:17.380031+0000", 2023-09-23T21:07:17.401 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.7678000000000002e-05 2023-09-23T21:07:17.401 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:17.401 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:17.401 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:07:17.401 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:07:17.380036+0000", 2023-09-23T21:07:17.401 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.8220000000000003e-06 2023-09-23T21:07:17.401 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:17.401 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:17.401 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:07:17.401 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:07:17.380040+0000", 2023-09-23T21:07:17.401 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.817e-06 2023-09-23T21:07:17.402 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:17.402 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:17.402 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:07:17.402 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:07:17.383955+0000", 2023-09-23T21:07:17.402 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0039151710000000003 2023-09-23T21:07:17.402 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:17.402 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:17.402 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:07:17.402 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:07:17.384091+0000", 2023-09-23T21:07:17.402 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00013533999999999999 2023-09-23T21:07:17.402 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:17.402 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:17.402 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 2,6", 2023-09-23T21:07:17.403 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:07:17.384270+0000", 2023-09-23T21:07:17.403 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000178906 2023-09-23T21:07:17.403 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:17.403 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:17.403 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T21:07:17.403 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:07:17.388527+0000", 2023-09-23T21:07:17.403 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0042576369999999999 2023-09-23T21:07:17.403 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:07:17.403 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:07:17.403 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:07:17.403 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:17.403 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:17.404 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35856.0:105871 9.4 9:208ccaf2:::benchmark_data_smithi026_342732_object105870:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e366)", 2023-09-23T21:07:17.404 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T21:07:17.384362+0000", 2023-09-23T21:07:17.404 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 0.0049487560000000003, 2023-09-23T21:07:17.404 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0051822969999999998, 2023-09-23T21:07:17.404 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:07:17.404 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "waiting for sub ops", 2023-09-23T21:07:17.404 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T21:07:17.404 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35856", 2023-09-23T21:07:17.404 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/4180396258", 2023-09-23T21:07:17.404 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 105871 2023-09-23T21:07:17.404 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:17.404 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:07:17.404 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:17.405 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:07:17.405 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:07:17.384362+0000", 2023-09-23T21:07:17.405 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:07:17.405 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:17.405 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:17.405 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:07:17.405 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:07:17.384362+0000", 2023-09-23T21:07:17.405 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:07:17.405 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:17.405 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:17.405 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:07:17.405 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:07:17.384361+0000", 2023-09-23T21:07:17.405 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.999999 2023-09-23T21:07:17.406 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:17.406 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:17.406 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:07:17.406 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:07:17.384385+0000", 2023-09-23T21:07:17.406 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.3998e-05 2023-09-23T21:07:17.406 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:17.406 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:17.406 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:07:17.406 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:07:17.384395+0000", 2023-09-23T21:07:17.406 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.6690000000000001e-06 2023-09-23T21:07:17.406 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:17.406 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:17.407 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:07:17.407 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:07:17.384402+0000", 2023-09-23T21:07:17.407 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.7519999999999999e-06 2023-09-23T21:07:17.407 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:17.407 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:17.407 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:07:17.407 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:07:17.384506+0000", 2023-09-23T21:07:17.407 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000104147 2023-09-23T21:07:17.407 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:17.407 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:17.407 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:07:17.407 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:07:17.384615+0000", 2023-09-23T21:07:17.407 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00010861699999999999 2023-09-23T21:07:17.408 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:17.408 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:17.408 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 3,4", 2023-09-23T21:07:17.408 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:07:17.384745+0000", 2023-09-23T21:07:17.408 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00013047300000000001 2023-09-23T21:07:17.408 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:17.408 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:17.408 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T21:07:17.408 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:07:17.388650+0000", 2023-09-23T21:07:17.408 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0039049079999999999 2023-09-23T21:07:17.408 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:07:17.408 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:07:17.409 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:07:17.409 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:17.409 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:17.409 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_repop(client.35856.0:105868 9.2 e366/350 9:492cb00a:::benchmark_data_smithi026_342732_object105867:head v 366'6598, mlcod=366'6596)", 2023-09-23T21:07:17.409 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T21:07:17.388414+0000", 2023-09-23T21:07:17.409 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 0.00089758199999999996, 2023-09-23T21:07:17.409 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0012130019999999999, 2023-09-23T21:07:17.409 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:07:17.409 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "started", 2023-09-23T21:07:17.409 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:07:17.409 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:17.409 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:07:17.409 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:07:17.388414+0000", 2023-09-23T21:07:17.410 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:07:17.410 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:17.410 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:17.410 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:07:17.410 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:07:17.388414+0000", 2023-09-23T21:07:17.410 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:07:17.410 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:17.410 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:17.410 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:07:17.410 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:07:17.388413+0000", 2023-09-23T21:07:17.410 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T21:07:17.410 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:17.410 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:17.411 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:07:17.411 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:07:17.388438+0000", 2023-09-23T21:07:17.411 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.5272000000000001e-05 2023-09-23T21:07:17.411 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:17.411 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:17.411 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:07:17.411 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:07:17.388447+0000", 2023-09-23T21:07:17.411 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.3569999999999998e-06 2023-09-23T21:07:17.411 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:17.411 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:17.411 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:07:17.411 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:07:17.388453+0000", 2023-09-23T21:07:17.412 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.9730000000000002e-06 2023-09-23T21:07:17.412 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:17.412 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:17.412 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:07:17.412 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:07:17.388680+0000", 2023-09-23T21:07:17.412 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00022716300000000001 2023-09-23T21:07:17.412 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:17.412 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:17.412 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:07:17.412 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:07:17.388705+0000", 2023-09-23T21:07:17.412 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.4890000000000001e-05 2023-09-23T21:07:17.412 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:07:17.412 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:07:17.413 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:07:17.413 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:17.413 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:17.413 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35856.0:105856 9.4 9:24c5f20d:::benchmark_data_smithi026_342732_object105855:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e366)", 2023-09-23T21:07:17.413 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T21:07:17.366290+0000", 2023-09-23T21:07:17.413 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 0.023021073999999999, 2023-09-23T21:07:17.413 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.023401737999999998, 2023-09-23T21:07:17.413 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:07:17.413 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "waiting for sub ops", 2023-09-23T21:07:17.413 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T21:07:17.413 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35856", 2023-09-23T21:07:17.413 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/4180396258", 2023-09-23T21:07:17.413 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 105856 2023-09-23T21:07:17.414 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:17.414 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:07:17.414 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:17.414 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:07:17.414 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:07:17.366290+0000", 2023-09-23T21:07:17.414 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:07:17.414 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:17.414 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:17.414 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:07:17.414 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:07:17.366290+0000", 2023-09-23T21:07:17.414 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:07:17.414 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:17.415 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:17.415 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:07:17.415 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:07:17.366289+0000", 2023-09-23T21:07:17.415 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T21:07:17.415 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:17.415 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:17.415 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:07:17.415 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:07:17.366314+0000", 2023-09-23T21:07:17.415 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.5398e-05 2023-09-23T21:07:17.415 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:17.415 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:17.415 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:07:17.416 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:07:17.366333+0000", 2023-09-23T21:07:17.416 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.8726999999999999e-05 2023-09-23T21:07:17.416 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:17.416 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:17.416 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:07:17.416 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:07:17.366342+0000", 2023-09-23T21:07:17.416 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.7580000000000004e-06 2023-09-23T21:07:17.416 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:17.416 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:17.416 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:07:17.416 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:07:17.366431+0000", 2023-09-23T21:07:17.416 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.8460000000000003e-05 2023-09-23T21:07:17.416 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:17.417 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:17.417 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:07:17.417 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:07:17.366558+0000", 2023-09-23T21:07:17.417 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000127198 2023-09-23T21:07:17.417 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:17.417 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:17.417 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 3,4", 2023-09-23T21:07:17.417 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:07:17.366704+0000", 2023-09-23T21:07:17.417 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000146169 2023-09-23T21:07:17.417 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:17.417 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:17.417 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T21:07:17.417 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:07:17.371584+0000", 2023-09-23T21:07:17.418 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0048798280000000001 2023-09-23T21:07:17.418 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:17.418 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:17.418 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:07:17.418 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:07:17.373342+0000", 2023-09-23T21:07:17.418 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0017577599999999999 2023-09-23T21:07:17.418 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:07:17.418 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:07:17.418 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:07:17.418 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:07:17.418 INFO:teuthology.orchestra.run.smithi026.stdout: ], 2023-09-23T21:07:17.418 INFO:teuthology.orchestra.run.smithi026.stdout: "num_ops": 5 2023-09-23T21:07:17.419 INFO:teuthology.orchestra.run.smithi026.stdout:} 2023-09-23T21:07:17.432 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:07:17.436 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:07:17.437 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:07:17.537 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:07:17.624 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:07:17.700 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:07:17.704 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:07:17.704 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:07:17.805 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:07:17.857 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:07:17.905 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:07:17.912 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:07:17.913 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:07:18.013 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:07:18.054 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:07:18.106 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:07:18.113 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:07:18.114 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:07:18.170 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_blocked_ops 2023-09-23T21:07:18.214 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:07:18.259 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:07:18.309 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 82 16 106619 106603 81.1864 49.8125 0.0165341 0.0123117 2023-09-23T21:07:18.310 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:07:18.313 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:07:18.314 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:07:18.340 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:07:18 smithi026 ceph-mon[175151]: pgmap v950: 109 pgs: 109 active+clean; 6.2 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 45 MiB/s wr, 726 op/s 2023-09-23T21:07:18.344 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:07:18 smithi026 ceph-mon[179688]: pgmap v950: 109 pgs: 109 active+clean; 6.2 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 45 MiB/s wr, 726 op/s 2023-09-23T21:07:18.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:07:18 smithi070 ceph-mon[168282]: pgmap v950: 109 pgs: 109 active+clean; 6.2 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 45 MiB/s wr, 726 op/s 2023-09-23T21:07:18.414 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:07:18.453 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:07:18.494 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:07:18.502 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:07:18.503 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:07:18.604 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:07:18.693 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:07:18.766 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:07:18.773 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:07:18.773 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:07:18.874 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:07:18.963 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:07:19.229 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:07:19.235 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:07:19.236 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:07:19.307 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 83 16 107371 107355 80.7748 47 0.0299237 0.0123746 2023-09-23T21:07:19.336 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:07:19.375 INFO:teuthology.orchestra.run.smithi026.stderr:nodeep-scrub is unset 2023-09-23T21:07:19.392 INFO:teuthology.orchestra.run.smithi026.stderr:reweighted osd.3 to 1 (10000) 2023-09-23T21:07:19.434 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:07:19.516 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:07:19.582 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:07:19 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/245313803' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-09-23T21:07:19.582 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:07:19 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-09-23T21:07:19.582 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:07:19 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/207513453' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:07:19.582 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:07:19 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:07:19.582 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:07:19 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/245313803' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-09-23T21:07:19.583 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:07:19 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-09-23T21:07:19.583 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:07:19 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/207513453' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:07:19.583 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:07:19 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:07:19.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:07:19 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/245313803' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-09-23T21:07:19.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:07:19 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-09-23T21:07:19.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:07:19 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/207513453' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:07:19.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:07:19 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:07:19.843 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:07:19.850 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 21:07:19 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[164029]: ::ffff:172.21.15.70 - - [23/Sep/2023:21:07:19] "GET /metrics HTTP/1.1" 200 36294 "" "Prometheus/2.43.0" 2023-09-23T21:07:19.851 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:07:19.851 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:07:19.952 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:07:20.073 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:07:20.201 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:07:20.209 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:07:20.210 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:07:20.289 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph osd reweight 6 1 2023-09-23T21:07:20.307 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 84 16 108158 108142 80.3987 49.1875 0.0315815 0.0124323 2023-09-23T21:07:20.311 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:07:20.385 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:07:20.437 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:07:20.442 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:07:20.443 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:07:20.543 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:07:20.565 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:07:20 smithi026 ceph-mon[179688]: pgmap v951: 109 pgs: 109 active+clean; 6.4 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 55 MiB/s wr, 884 op/s 2023-09-23T21:07:20.565 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:07:20 smithi026 ceph-mon[179688]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T21:07:20.565 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:07:20 smithi026 ceph-mon[179688]: Cluster is now healthy 2023-09-23T21:07:20.565 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:07:20 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-09-23T21:07:20.565 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:07:20 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T21:07:20.565 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:07:20 smithi026 ceph-mon[179688]: osdmap e367: 8 total, 8 up, 8 in 2023-09-23T21:07:20.565 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:07:20 smithi026 ceph-mon[179688]: failed to encode map e367 with expected crc 2023-09-23T21:07:20.566 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:07:20 smithi026 ceph-mon[179688]: failed to encode map e367 with expected crc 2023-09-23T21:07:20.566 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:07:20 smithi026 ceph-mon[179688]: failed to encode map e367 with expected crc 2023-09-23T21:07:20.566 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:07:20 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T21:07:20.566 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:07:20 smithi026 ceph-mon[175151]: pgmap v951: 109 pgs: 109 active+clean; 6.4 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 55 MiB/s wr, 884 op/s 2023-09-23T21:07:20.566 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:07:20 smithi026 ceph-mon[175151]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T21:07:20.566 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:07:20 smithi026 ceph-mon[175151]: Cluster is now healthy 2023-09-23T21:07:20.566 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:07:20 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-09-23T21:07:20.566 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:07:20 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T21:07:20.566 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:07:20 smithi026 ceph-mon[175151]: osdmap e367: 8 total, 8 up, 8 in 2023-09-23T21:07:20.567 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:07:20 smithi026 ceph-mon[175151]: failed to encode map e367 with expected crc 2023-09-23T21:07:20.567 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:07:20 smithi026 ceph-mon[175151]: failed to encode map e367 with expected crc 2023-09-23T21:07:20.567 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:07:20 smithi026 ceph-mon[175151]: failed to encode map e367 with expected crc 2023-09-23T21:07:20.567 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:07:20 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T21:07:20.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:07:20 smithi070 ceph-mon[168282]: pgmap v951: 109 pgs: 109 active+clean; 6.4 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 55 MiB/s wr, 884 op/s 2023-09-23T21:07:20.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:07:20 smithi070 ceph-mon[168282]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T21:07:20.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:07:20 smithi070 ceph-mon[168282]: Cluster is now healthy 2023-09-23T21:07:20.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:07:20 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-09-23T21:07:20.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:07:20 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T21:07:20.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:07:20 smithi070 ceph-mon[168282]: osdmap e367: 8 total, 8 up, 8 in 2023-09-23T21:07:20.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:07:20 smithi070 ceph-mon[168282]: failed to encode map e367 with expected crc 2023-09-23T21:07:20.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:07:20 smithi070 ceph-mon[168282]: failed to encode map e367 with expected crc 2023-09-23T21:07:20.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:07:20 smithi070 ceph-mon[168282]: failed to encode map e367 with expected crc 2023-09-23T21:07:20.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:07:20 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T21:07:20.665 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:07:20.707 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0: {} 2023-09-23T21:07:20.759 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1: {} 2023-09-23T21:07:20.803 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2: {} 2023-09-23T21:07:20.850 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:07:20.860 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:07:20.861 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:07:20.870 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3: {} 2023-09-23T21:07:20.915 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4: {} 2023-09-23T21:07:20.953 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T21:07:20.962 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:07:20.990 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6: {} 2023-09-23T21:07:21.038 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T21:07:21.040 INFO:teuthology.orchestra.run.smithi026.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T21:07:21.041 INFO:teuthology.orchestra.run.smithi026.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T21:07:21.041 INFO:teuthology.orchestra.run.smithi026.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T21:07:21.041 INFO:teuthology.orchestra.run.smithi026.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T21:07:21.041 INFO:teuthology.orchestra.run.smithi026.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T21:07:21.041 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T21:07:21.041 INFO:teuthology.orchestra.run.smithi026.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T21:07:21.041 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T21:07:21.047 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:07:21.307 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 85 16 108900 108884 79.9986 46.375 0.00893674 0.0124935 2023-09-23T21:07:21.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:07:21 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/2283065217' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:07:21.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:07:21 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/2283065217' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:07:21.872 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:07:21 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/2283065217' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:07:22.013 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T21:07:22.316 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 86 16 109803 109787 79.7206 56.4375 0.0142736 0.0125393 2023-09-23T21:07:22.383 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:07:22 smithi026 ceph-mon[175151]: pgmap v953: 109 pgs: 109 active+clean; 6.5 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 48 MiB/s wr, 764 op/s 2023-09-23T21:07:22.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:07:22 smithi070 ceph-mon[168282]: pgmap v953: 109 pgs: 109 active+clean; 6.5 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 48 MiB/s wr, 764 op/s 2023-09-23T21:07:22.883 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:07:22 smithi026 ceph-mon[179688]: pgmap v953: 109 pgs: 109 active+clean; 6.5 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 48 MiB/s wr, 764 op/s 2023-09-23T21:07:23.104 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T21:07:23.109 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:07:23.109 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:07:23.210 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:07:23.293 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:07:23.312 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 87 16 110729 110713 79.4689 57.875 0.0353167 0.0125783 2023-09-23T21:07:23.841 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:07:23.851 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:07:23.852 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:07:23.952 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:07:24.020 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:07:24.083 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:07:24.096 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:07:24.097 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:07:24.197 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:07:24.275 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:07:24.314 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 88 16 111603 111587 79.1852 54.625 0.0208687 0.0126239 2023-09-23T21:07:24.634 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:07:24 smithi026 ceph-mon[179688]: pgmap v954: 109 pgs: 109 active+clean; 6.6 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 53 MiB/s wr, 848 op/s 2023-09-23T21:07:24.635 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:07:24 smithi026 ceph-mon[175151]: pgmap v954: 109 pgs: 109 active+clean; 6.6 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 53 MiB/s wr, 848 op/s 2023-09-23T21:07:24.714 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T21:07:24.717 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:07:24.718 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:07:24.818 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:07:24.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:07:24 smithi070 ceph-mon[168282]: pgmap v954: 109 pgs: 109 active+clean; 6.6 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 53 MiB/s wr, 848 op/s 2023-09-23T21:07:24.900 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:07:24.924 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:07:24.989 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:07:25.314 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 89 16 112494 112478 78.9213 55.6875 0.027398 0.0126662 2023-09-23T21:07:25.865 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T21:07:26.261 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T21:07:26.264 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:07:26.264 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:07:26.314 INFO:tasks.radosbench.radosbench.0.smithi026.stdout:Total time run: 90.0061 2023-09-23T21:07:26.315 INFO:tasks.radosbench.radosbench.0.smithi026.stdout:Total writes made: 113273 2023-09-23T21:07:26.315 INFO:tasks.radosbench.radosbench.0.smithi026.stdout:Write size: 65536 2023-09-23T21:07:26.315 INFO:tasks.radosbench.radosbench.0.smithi026.stdout:Object size: 65536 2023-09-23T21:07:26.315 INFO:tasks.radosbench.radosbench.0.smithi026.stdout:Bandwidth (MB/sec): 78.6565 2023-09-23T21:07:26.315 INFO:tasks.radosbench.radosbench.0.smithi026.stdout:Stddev Bandwidth: 34.4765 2023-09-23T21:07:26.315 INFO:tasks.radosbench.radosbench.0.smithi026.stdout:Max bandwidth (MB/sec): 196.188 2023-09-23T21:07:26.315 INFO:tasks.radosbench.radosbench.0.smithi026.stdout:Min bandwidth (MB/sec): 14.875 2023-09-23T21:07:26.315 INFO:tasks.radosbench.radosbench.0.smithi026.stdout:Average IOPS: 1258 2023-09-23T21:07:26.315 INFO:tasks.radosbench.radosbench.0.smithi026.stdout:Stddev IOPS: 551.624 2023-09-23T21:07:26.315 INFO:tasks.radosbench.radosbench.0.smithi026.stdout:Max IOPS: 3139 2023-09-23T21:07:26.315 INFO:tasks.radosbench.radosbench.0.smithi026.stdout:Min IOPS: 238 2023-09-23T21:07:26.316 INFO:tasks.radosbench.radosbench.0.smithi026.stdout:Average Latency(s): 0.0127096 2023-09-23T21:07:26.316 INFO:tasks.radosbench.radosbench.0.smithi026.stdout:Stddev Latency(s): 0.0173324 2023-09-23T21:07:26.316 INFO:tasks.radosbench.radosbench.0.smithi026.stdout:Max latency(s): 1.03425 2023-09-23T21:07:26.316 INFO:tasks.radosbench.radosbench.0.smithi026.stdout:Min latency(s): 0.00137237 2023-09-23T21:07:26.319 INFO:tasks.radosbench.radosbench.0.smithi026.stdout:Cleaning up (deleting benchmark objects) 2023-09-23T21:07:26.327 INFO:teuthology.orchestra.run.smithi026.stdout:ERROR: (22) Invalid argument 2023-09-23T21:07:26.328 INFO:teuthology.orchestra.run.smithi026.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T21:07:26.365 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:07:26.421 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:07:26.502 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:07:26.506 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:07:26.507 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:07:26.607 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:07:26.631 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:07:26 smithi026 ceph-mon[179688]: pgmap v955: 109 pgs: 109 active+clean; 6.7 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 51 MiB/s wr, 821 op/s 2023-09-23T21:07:26.631 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:07:26 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:07:26.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:07:26 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:07:26.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:07:26 smithi026 ceph-mon[175151]: pgmap v955: 109 pgs: 109 active+clean; 6.7 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 51 MiB/s wr, 821 op/s 2023-09-23T21:07:26.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:07:26 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:07:26.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:07:26 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:07:26.635 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:07:26.680 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:07:26.691 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:07:26.692 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:07:26.793 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:07:26.844 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:07:26.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:07:26 smithi070 ceph-mon[168282]: pgmap v955: 109 pgs: 109 active+clean; 6.7 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 51 MiB/s wr, 821 op/s 2023-09-23T21:07:26.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:07:26 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:07:26.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:07:26 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:07:27.755 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:07:27.768 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:07:27.768 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:07:27.869 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:07:27.965 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:07:28.050 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:07:28.062 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:07:28.063 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:07:28.163 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:07:28.235 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:07:28.407 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:07:28.412 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:07:28.412 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:07:28.513 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:07:28.596 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:07:28.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:07:28 smithi070 ceph-mon[168282]: pgmap v956: 109 pgs: 109 active+clean; 6.8 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 52 MiB/s wr, 824 op/s 2023-09-23T21:07:28.883 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:07:28 smithi026 ceph-mon[175151]: pgmap v956: 109 pgs: 109 active+clean; 6.8 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 52 MiB/s wr, 824 op/s 2023-09-23T21:07:28.886 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:07:28 smithi026 ceph-mon[179688]: pgmap v956: 109 pgs: 109 active+clean; 6.8 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 52 MiB/s wr, 824 op/s 2023-09-23T21:07:28.922 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:07:28.929 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:07:28.929 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:07:29.030 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:07:29.066 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:07:29.098 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_historic_ops 2023-09-23T21:07:29.107 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:07:29.110 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:07:29.110 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:07:29.211 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:07:29.298 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:07:29.484 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:07:29.490 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:07:29.494 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:07:29.494 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:07:29.595 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:07:29.664 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:07:29.740 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 21:07:29 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[164029]: ::ffff:172.21.15.70 - - [23/Sep/2023:21:07:29] "GET /metrics HTTP/1.1" 200 36294 "" "Prometheus/2.43.0" 2023-09-23T21:07:29.809 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T21:07:29.830 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:07:29.830 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:07:29.931 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:07:29.966 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:07:30.016 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:07:30.019 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:07:30.019 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:07:30.120 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:07:30.148 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:07:30.203 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:07:30.207 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:07:30.207 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:07:30.308 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:07:30.347 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:07:30.385 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:07:30.394 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:07:30.397 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:07:30.397 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:07:30.463 INFO:teuthology.orchestra.run.smithi026.stderr:noscrub is set 2023-09-23T21:07:30.498 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:07:30.566 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:07:30.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:07:30 smithi070 ceph-mon[168282]: pgmap v957: 109 pgs: 109 active+clean; 6.8 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 102 B/s rd, 48 MiB/s wr, 828 op/s 2023-09-23T21:07:30.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:07:30 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/1185659424' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T21:07:30.636 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:07:30 smithi026 ceph-mon[175151]: pgmap v957: 109 pgs: 109 active+clean; 6.8 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 102 B/s rd, 48 MiB/s wr, 828 op/s 2023-09-23T21:07:30.636 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:07:30 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/1185659424' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T21:07:30.636 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:07:30 smithi026 ceph-mon[179688]: pgmap v957: 109 pgs: 109 active+clean; 6.8 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 102 B/s rd, 48 MiB/s wr, 828 op/s 2023-09-23T21:07:30.636 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:07:30 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/1185659424' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T21:07:31.403 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:07:31.413 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:07:31.413 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:07:31.465 INFO:teuthology.orchestra.run.smithi026.stderr:reweighted osd.6 to 1 (10000) 2023-09-23T21:07:31.514 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:07:31.554 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:07:31.610 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:07:31.614 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:07:31.614 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:07:31.715 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:07:31.766 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:07:31 smithi070 ceph-mon[168282]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:07:31.766 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:07:31 smithi070 ceph-mon[168282]: failed to encode map e368 with expected crc 2023-09-23T21:07:31.766 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:07:31 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/1185659424' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T21:07:31.767 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:07:31 smithi070 ceph-mon[168282]: osdmap e368: 8 total, 8 up, 8 in 2023-09-23T21:07:31.767 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:07:31 smithi070 ceph-mon[168282]: failed to encode map e368 with expected crc 2023-09-23T21:07:31.767 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:07:31 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/1109714768' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-09-23T21:07:31.767 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:07:31 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-09-23T21:07:31.767 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:07:31 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:07:31.770 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:07:31.817 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:07:31.823 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:07:31.823 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:07:31.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:07:31 smithi026 ceph-mon[179688]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:07:31.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:07:31 smithi026 ceph-mon[179688]: failed to encode map e368 with expected crc 2023-09-23T21:07:31.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:07:31 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/1185659424' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T21:07:31.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:07:31 smithi026 ceph-mon[179688]: osdmap e368: 8 total, 8 up, 8 in 2023-09-23T21:07:31.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:07:31 smithi026 ceph-mon[179688]: failed to encode map e368 with expected crc 2023-09-23T21:07:31.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:07:31 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/1109714768' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-09-23T21:07:31.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:07:31 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-09-23T21:07:31.883 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:07:31 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:07:31.883 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:07:31 smithi026 ceph-mon[175151]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:07:31.883 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:07:31 smithi026 ceph-mon[175151]: failed to encode map e368 with expected crc 2023-09-23T21:07:31.883 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:07:31 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/1185659424' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T21:07:31.883 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:07:31 smithi026 ceph-mon[175151]: osdmap e368: 8 total, 8 up, 8 in 2023-09-23T21:07:31.883 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:07:31 smithi026 ceph-mon[175151]: failed to encode map e368 with expected crc 2023-09-23T21:07:31.883 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:07:31 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/1109714768' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-09-23T21:07:31.883 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:07:31 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-09-23T21:07:31.884 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:07:31 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:07:31.924 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:07:31.981 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:07:32.049 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:07:32.054 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:07:32.055 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:07:32.155 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:07:32.231 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:07:32.811 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:07:32 smithi026 ceph-mon[175151]: pgmap v959: 109 pgs: 109 active+clean; 6.8 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 102 B/s rd, 44 MiB/s wr, 833 op/s 2023-09-23T21:07:32.811 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:07:32 smithi026 ceph-mon[175151]: failed to encode map e369 with expected crc 2023-09-23T21:07:32.811 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:07:32 smithi026 ceph-mon[175151]: failed to encode map e369 with expected crc 2023-09-23T21:07:32.812 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:07:32 smithi026 ceph-mon[175151]: failed to encode map e369 with expected crc 2023-09-23T21:07:32.812 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:07:32 smithi026 ceph-mon[175151]: failed to encode map e369 with expected crc 2023-09-23T21:07:32.812 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:07:32 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-09-23T21:07:32.812 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:07:32 smithi026 ceph-mon[175151]: osdmap e369: 8 total, 8 up, 8 in 2023-09-23T21:07:32.812 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:07:32 smithi026 ceph-mon[175151]: failed to encode map e369 with expected crc 2023-09-23T21:07:32.812 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:07:32 smithi026 ceph-mon[179688]: pgmap v959: 109 pgs: 109 active+clean; 6.8 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 102 B/s rd, 44 MiB/s wr, 833 op/s 2023-09-23T21:07:32.812 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:07:32 smithi026 ceph-mon[179688]: failed to encode map e369 with expected crc 2023-09-23T21:07:32.812 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:07:32 smithi026 ceph-mon[179688]: failed to encode map e369 with expected crc 2023-09-23T21:07:32.812 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:07:32 smithi026 ceph-mon[179688]: failed to encode map e369 with expected crc 2023-09-23T21:07:32.812 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:07:32 smithi026 ceph-mon[179688]: failed to encode map e369 with expected crc 2023-09-23T21:07:32.813 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:07:32 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-09-23T21:07:32.813 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:07:32 smithi026 ceph-mon[179688]: osdmap e369: 8 total, 8 up, 8 in 2023-09-23T21:07:32.813 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:07:32 smithi026 ceph-mon[179688]: failed to encode map e369 with expected crc 2023-09-23T21:07:32.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:07:32 smithi070 ceph-mon[168282]: pgmap v959: 109 pgs: 109 active+clean; 6.8 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 102 B/s rd, 44 MiB/s wr, 833 op/s 2023-09-23T21:07:32.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:07:32 smithi070 ceph-mon[168282]: failed to encode map e369 with expected crc 2023-09-23T21:07:32.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:07:32 smithi070 ceph-mon[168282]: failed to encode map e369 with expected crc 2023-09-23T21:07:32.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:07:32 smithi070 ceph-mon[168282]: failed to encode map e369 with expected crc 2023-09-23T21:07:32.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:07:32 smithi070 ceph-mon[168282]: failed to encode map e369 with expected crc 2023-09-23T21:07:32.875 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:07:32 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-09-23T21:07:32.875 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:07:32 smithi070 ceph-mon[168282]: osdmap e369: 8 total, 8 up, 8 in 2023-09-23T21:07:32.875 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:07:32 smithi070 ceph-mon[168282]: failed to encode map e369 with expected crc 2023-09-23T21:07:32.982 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph osd reweight 7 1 2023-09-23T21:07:33.057 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:07:33.066 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:07:33.066 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:07:33.167 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:07:33.203 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:07:33.262 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:07:33.264 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:07:33.265 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:07:33.365 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:07:33.398 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:07:33.448 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:07:33.451 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:07:33.451 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:07:33.552 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:07:33.563 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0: {} 2023-09-23T21:07:33.598 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1: {} 2023-09-23T21:07:33.646 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2: {} 2023-09-23T21:07:33.681 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3: {} 2023-09-23T21:07:33.711 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:07:33.724 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4: {} 2023-09-23T21:07:33.771 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T21:07:33.805 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6: {} 2023-09-23T21:07:33.841 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T21:07:33.843 INFO:teuthology.orchestra.run.smithi026.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T21:07:33.843 INFO:teuthology.orchestra.run.smithi026.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T21:07:33.843 INFO:teuthology.orchestra.run.smithi026.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T21:07:33.843 INFO:teuthology.orchestra.run.smithi026.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T21:07:33.843 INFO:teuthology.orchestra.run.smithi026.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T21:07:33.843 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T21:07:33.843 INFO:teuthology.orchestra.run.smithi026.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T21:07:33.844 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T21:07:33.986 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:07:33.998 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:07:33.999 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:07:34.100 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:07:34.149 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:07:34.217 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:07:34.222 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:07:34.222 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:07:34.323 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:07:34.398 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:07:34.504 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:07:34.508 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:07:34.509 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:07:34.610 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:07:34.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:07:34 smithi026 ceph-mon[175151]: pgmap v961: 109 pgs: 109 active+clean; 6.7 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 127 B/s rd, 23 MiB/s wr, 779 op/s 2023-09-23T21:07:34.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:07:34 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/3092042001' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:07:34.637 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:07:34 smithi026 ceph-mon[179688]: pgmap v961: 109 pgs: 109 active+clean; 6.7 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 127 B/s rd, 23 MiB/s wr, 779 op/s 2023-09-23T21:07:34.637 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:07:34 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/3092042001' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:07:34.700 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:07:34.794 INFO:teuthology.orchestra.run.smithi026.stdout:{ 2023-09-23T21:07:34.795 INFO:teuthology.orchestra.run.smithi026.stdout: "size": 20, 2023-09-23T21:07:34.795 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 600, 2023-09-23T21:07:34.795 INFO:teuthology.orchestra.run.smithi026.stdout: "ops": [ 2023-09-23T21:07:34.795 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:34.795 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35856.0:14481 9.0 9:0073314f:::benchmark_data_smithi026_342732_object14480:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e351)", 2023-09-23T21:07:34.795 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T21:06:01.428516+0000", 2023-09-23T21:07:34.795 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 93.357668505000007, 2023-09-23T21:07:34.795 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.084287198999999993, 2023-09-23T21:07:34.795 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:07:34.796 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:07:34.796 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T21:07:34.796 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35856", 2023-09-23T21:07:34.796 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/4180396258", 2023-09-23T21:07:34.796 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 14481 2023-09-23T21:07:34.796 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:34.796 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:07:34.796 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:34.796 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:07:34.796 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:01.428516+0000", 2023-09-23T21:07:34.797 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:07:34.797 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:34.797 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:34.797 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:07:34.797 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:01.428516+0000", 2023-09-23T21:07:34.797 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:07:34.797 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:34.797 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:34.797 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:07:34.797 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:01.428515+0000", 2023-09-23T21:07:34.797 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T21:07:34.798 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:34.798 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:34.798 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:07:34.798 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:01.428538+0000", 2023-09-23T21:07:34.798 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.3506000000000002e-05 2023-09-23T21:07:34.798 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:34.798 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:34.798 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:07:34.798 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:01.428550+0000", 2023-09-23T21:07:34.798 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1749000000000001e-05 2023-09-23T21:07:34.798 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:34.798 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:34.799 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:07:34.799 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:01.428557+0000", 2023-09-23T21:07:34.799 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.7009999999999996e-06 2023-09-23T21:07:34.799 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:34.799 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:34.799 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:07:34.799 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:01.428693+0000", 2023-09-23T21:07:34.799 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00013527100000000001 2023-09-23T21:07:34.799 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:34.799 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:34.799 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:07:34.799 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:01.509886+0000", 2023-09-23T21:07:34.800 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.081193455999999997 2023-09-23T21:07:34.800 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:34.800 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:34.800 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 2,6", 2023-09-23T21:07:34.800 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:01.509986+0000", 2023-09-23T21:07:34.800 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.9902999999999995e-05 2023-09-23T21:07:34.800 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:34.800 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:34.800 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:07:34.800 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:01.512331+0000", 2023-09-23T21:07:34.800 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0023445990000000002 2023-09-23T21:07:34.800 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:34.801 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:34.801 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T21:07:34.801 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:01.512338+0000", 2023-09-23T21:07:34.801 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.4159999999999998e-06 2023-09-23T21:07:34.801 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:34.801 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:34.801 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:07:34.801 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:01.512713+0000", 2023-09-23T21:07:34.801 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00037463399999999999 2023-09-23T21:07:34.801 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:34.801 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:34.801 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T21:07:34.802 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:01.512761+0000", 2023-09-23T21:07:34.802 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.8267999999999999e-05 2023-09-23T21:07:34.802 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:34.802 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:34.802 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T21:07:34.802 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:01.512803+0000", 2023-09-23T21:07:34.802 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.2042999999999997e-05 2023-09-23T21:07:34.802 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:07:34.802 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:07:34.802 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:07:34.802 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:34.802 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:34.802 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35856.0:14482 9.4 9:23f9860c:::benchmark_data_smithi026_342732_object14481:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e351)", 2023-09-23T21:07:34.803 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T21:06:01.428575+0000", 2023-09-23T21:07:34.803 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 93.357609765999996, 2023-09-23T21:07:34.803 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.083734059999999999, 2023-09-23T21:07:34.803 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:07:34.803 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:07:34.803 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T21:07:34.803 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35856", 2023-09-23T21:07:34.803 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/4180396258", 2023-09-23T21:07:34.803 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 14482 2023-09-23T21:07:34.803 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:34.803 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:07:34.803 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:34.804 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:07:34.804 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:01.428575+0000", 2023-09-23T21:07:34.804 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:07:34.804 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:34.804 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:34.804 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:07:34.804 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:01.428575+0000", 2023-09-23T21:07:34.804 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:07:34.804 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:34.804 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:34.804 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:07:34.804 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:01.428574+0000", 2023-09-23T21:07:34.805 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T21:07:34.805 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:34.805 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:34.805 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:07:34.805 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:01.428593+0000", 2023-09-23T21:07:34.805 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.9434000000000001e-05 2023-09-23T21:07:34.805 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:34.805 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:34.805 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:07:34.805 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:01.428602+0000", 2023-09-23T21:07:34.805 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.3429999999999996e-06 2023-09-23T21:07:34.805 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:34.806 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:34.806 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:07:34.806 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:01.428608+0000", 2023-09-23T21:07:34.806 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.4470000000000003e-06 2023-09-23T21:07:34.806 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:34.806 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:34.806 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:07:34.806 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:01.428654+0000", 2023-09-23T21:07:34.806 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.5979000000000001e-05 2023-09-23T21:07:34.806 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:34.806 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:34.806 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:07:34.807 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:01.509838+0000", 2023-09-23T21:07:34.807 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.081183721 2023-09-23T21:07:34.807 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:34.807 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:34.807 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 3,4", 2023-09-23T21:07:34.807 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:01.509998+0000", 2023-09-23T21:07:34.807 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00016033599999999999 2023-09-23T21:07:34.807 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:34.807 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:34.807 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:07:34.807 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:01.511608+0000", 2023-09-23T21:07:34.807 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0016094390000000001 2023-09-23T21:07:34.808 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:34.808 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:34.808 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T21:07:34.808 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:01.511857+0000", 2023-09-23T21:07:34.808 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00024955000000000002 2023-09-23T21:07:34.808 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:34.808 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:34.808 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:07:34.808 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:01.512229+0000", 2023-09-23T21:07:34.808 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00037131300000000001 2023-09-23T21:07:34.808 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:34.808 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:34.809 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T21:07:34.809 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:01.512272+0000", 2023-09-23T21:07:34.809 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.3043000000000001e-05 2023-09-23T21:07:34.809 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:34.809 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:34.809 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T21:07:34.809 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:01.512309+0000", 2023-09-23T21:07:34.809 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.7051000000000002e-05 2023-09-23T21:07:34.809 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:07:34.809 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:07:34.809 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:07:34.809 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:34.810 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:34.810 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35856.0:14487 9.0 9:0bcf534c:::benchmark_data_smithi026_342732_object14486:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e351)", 2023-09-23T21:07:34.810 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T21:06:01.428624+0000", 2023-09-23T21:07:34.810 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 93.357560703000004, 2023-09-23T21:07:34.810 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.084429173999999996, 2023-09-23T21:07:34.810 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:07:34.810 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:07:34.810 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T21:07:34.810 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35856", 2023-09-23T21:07:34.810 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/4180396258", 2023-09-23T21:07:34.810 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 14487 2023-09-23T21:07:34.810 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:34.811 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:07:34.811 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:34.811 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:07:34.811 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:01.428624+0000", 2023-09-23T21:07:34.811 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:07:34.811 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:34.811 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:34.811 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:07:34.811 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:01.428624+0000", 2023-09-23T21:07:34.811 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:07:34.811 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:34.811 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:34.812 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:07:34.812 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:01.428623+0000", 2023-09-23T21:07:34.812 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T21:07:34.812 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:34.812 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:34.812 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:07:34.812 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:01.428643+0000", 2023-09-23T21:07:34.812 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.0279e-05 2023-09-23T21:07:34.812 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:34.812 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:34.812 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:07:34.812 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:01.428651+0000", 2023-09-23T21:07:34.813 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.0639999999999994e-06 2023-09-23T21:07:34.813 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:34.813 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:34.813 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:07:34.813 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:01.428658+0000", 2023-09-23T21:07:34.813 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.4280000000000001e-06 2023-09-23T21:07:34.813 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:34.813 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:34.813 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:07:34.813 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:01.510700+0000", 2023-09-23T21:07:34.813 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.082041858999999995 2023-09-23T21:07:34.813 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:34.814 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:34.814 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:07:34.814 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:01.510800+0000", 2023-09-23T21:07:34.814 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.9877000000000006e-05 2023-09-23T21:07:34.814 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:34.814 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:34.814 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 2,6", 2023-09-23T21:07:34.814 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:01.510936+0000", 2023-09-23T21:07:34.814 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00013684 2023-09-23T21:07:34.814 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:34.814 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:34.814 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T21:07:34.815 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:01.512343+0000", 2023-09-23T21:07:34.815 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0014066910000000001 2023-09-23T21:07:34.815 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:34.815 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:34.815 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:07:34.815 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:01.512850+0000", 2023-09-23T21:07:34.815 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00050711500000000004 2023-09-23T21:07:34.815 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:34.815 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:34.815 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:07:34.815 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:01.512980+0000", 2023-09-23T21:07:34.815 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00012995199999999999 2023-09-23T21:07:34.816 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:34.816 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:34.816 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T21:07:34.816 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:01.513019+0000", 2023-09-23T21:07:34.816 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.8554999999999999e-05 2023-09-23T21:07:34.816 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:34.816 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:34.816 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T21:07:34.816 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:01.513053+0000", 2023-09-23T21:07:34.816 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.4097000000000002e-05 2023-09-23T21:07:34.816 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:07:34.816 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:07:34.817 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:07:34.817 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:34.817 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:34.817 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35856.0:14493 9.d 9:b86035f4:::benchmark_data_smithi026_342732_object14492:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e351)", 2023-09-23T21:07:34.817 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T21:06:01.429292+0000", 2023-09-23T21:07:34.817 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 93.356892801000001, 2023-09-23T21:07:34.817 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.082661684999999999, 2023-09-23T21:07:34.817 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:07:34.817 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:07:34.817 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T21:07:34.817 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35856", 2023-09-23T21:07:34.817 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/4180396258", 2023-09-23T21:07:34.818 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 14493 2023-09-23T21:07:34.818 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:34.818 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:07:34.818 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:34.818 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:07:34.818 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:01.429292+0000", 2023-09-23T21:07:34.818 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:07:34.818 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:34.818 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:34.818 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:07:34.818 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:01.429292+0000", 2023-09-23T21:07:34.819 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:07:34.819 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:34.819 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:34.819 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:07:34.819 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:01.429290+0000", 2023-09-23T21:07:34.819 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.999999 2023-09-23T21:07:34.819 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:34.819 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:34.819 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:07:34.819 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:01.429315+0000", 2023-09-23T21:07:34.819 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.4026e-05 2023-09-23T21:07:34.819 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:34.820 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:34.820 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:07:34.820 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:01.509751+0000", 2023-09-23T21:07:34.820 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.080436828000000002 2023-09-23T21:07:34.820 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:34.820 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:34.820 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:07:34.820 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:01.509765+0000", 2023-09-23T21:07:34.820 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.3399e-05 2023-09-23T21:07:34.820 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:34.820 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:34.820 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:07:34.821 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:01.509826+0000", 2023-09-23T21:07:34.821 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.156e-05 2023-09-23T21:07:34.821 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:34.821 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:34.821 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:07:34.821 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:01.509927+0000", 2023-09-23T21:07:34.821 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000100311 2023-09-23T21:07:34.821 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:34.821 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:34.821 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 5,7", 2023-09-23T21:07:34.821 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:01.510033+0000", 2023-09-23T21:07:34.822 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000106548 2023-09-23T21:07:34.822 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:34.823 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:34.823 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:07:34.823 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:01.511281+0000", 2023-09-23T21:07:34.823 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0012477199999999999 2023-09-23T21:07:34.823 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:34.823 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:34.823 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:07:34.823 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:01.511545+0000", 2023-09-23T21:07:34.823 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00026450099999999999 2023-09-23T21:07:34.824 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:34.824 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:34.824 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T21:07:34.824 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:01.511861+0000", 2023-09-23T21:07:34.824 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00031610200000000002 2023-09-23T21:07:34.824 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:34.824 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:34.824 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T21:07:34.824 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:01.511913+0000", 2023-09-23T21:07:34.824 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.1672000000000003e-05 2023-09-23T21:07:34.824 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:34.824 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:34.824 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T21:07:34.825 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:01.511953+0000", 2023-09-23T21:07:34.825 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.0108999999999998e-05 2023-09-23T21:07:34.825 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:07:34.825 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:07:34.825 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:07:34.825 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:34.825 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:34.825 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_repop(client.35856.0:14489 9.f e351/350 9:f0e30255:::benchmark_data_smithi026_342732_object14488:head v 351'933, mlcod=351'930)", 2023-09-23T21:07:34.825 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T21:06:01.429503+0000", 2023-09-23T21:07:34.825 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 93.356681285999997, 2023-09-23T21:07:34.825 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.082382258, 2023-09-23T21:07:34.825 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:07:34.826 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:07:34.826 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:07:34.826 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:34.826 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:07:34.826 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:01.429503+0000", 2023-09-23T21:07:34.826 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:07:34.826 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:34.826 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:34.826 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:07:34.826 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:01.429503+0000", 2023-09-23T21:07:34.826 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:07:34.826 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:34.827 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:34.827 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:07:34.827 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:01.429502+0000", 2023-09-23T21:07:34.827 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.999999 2023-09-23T21:07:34.827 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:34.827 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:34.827 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:07:34.827 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:01.429543+0000", 2023-09-23T21:07:34.827 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.0305000000000001e-05 2023-09-23T21:07:34.827 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:34.827 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:34.827 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:07:34.828 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:01.509747+0000", 2023-09-23T21:07:34.828 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.080204344999999996 2023-09-23T21:07:34.828 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:34.828 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:34.828 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:07:34.828 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:01.509760+0000", 2023-09-23T21:07:34.828 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.3088000000000001e-05 2023-09-23T21:07:34.828 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:34.828 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:34.828 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:07:34.828 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:01.509807+0000", 2023-09-23T21:07:34.828 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.6672000000000003e-05 2023-09-23T21:07:34.829 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:34.829 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:34.829 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:07:34.829 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:01.509831+0000", 2023-09-23T21:07:34.829 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.3954000000000001e-05 2023-09-23T21:07:34.829 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:34.829 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:34.829 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T21:07:34.829 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:01.511797+0000", 2023-09-23T21:07:34.829 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0019667719999999999 2023-09-23T21:07:34.829 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:34.829 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:34.829 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T21:07:34.830 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:01.511885+0000", 2023-09-23T21:07:34.830 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.7854000000000003e-05 2023-09-23T21:07:34.830 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:07:34.830 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:07:34.830 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:07:34.830 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:34.830 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:34.830 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35856.0:17991 9.d 9:b58bb8ac:::benchmark_data_smithi026_342732_object17990:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e351)", 2023-09-23T21:07:34.830 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T21:06:02.899549+0000", 2023-09-23T21:07:34.830 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 91.886635869000003, 2023-09-23T21:07:34.830 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.067838966000000001, 2023-09-23T21:07:34.830 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:07:34.831 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:07:34.831 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T21:07:34.831 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35856", 2023-09-23T21:07:34.831 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/4180396258", 2023-09-23T21:07:34.831 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 17991 2023-09-23T21:07:34.831 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:34.831 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:07:34.831 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:34.831 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:07:34.831 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:02.899549+0000", 2023-09-23T21:07:34.831 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:07:34.831 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:34.832 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:34.832 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:07:34.832 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:02.899549+0000", 2023-09-23T21:07:34.832 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:07:34.832 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:34.832 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:34.832 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:07:34.832 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:02.899547+0000", 2023-09-23T21:07:34.832 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.999999 2023-09-23T21:07:34.832 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:34.832 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:34.832 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:07:34.832 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:02.899592+0000", 2023-09-23T21:07:34.833 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.4671e-05 2023-09-23T21:07:34.833 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:34.833 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:34.833 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:07:34.833 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:02.899604+0000", 2023-09-23T21:07:34.833 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1647999999999999e-05 2023-09-23T21:07:34.833 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:34.833 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:34.833 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:07:34.833 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:02.899611+0000", 2023-09-23T21:07:34.833 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.4159999999999998e-06 2023-09-23T21:07:34.833 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:34.834 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:34.834 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:07:34.834 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:02.899665+0000", 2023-09-23T21:07:34.834 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.3862e-05 2023-09-23T21:07:34.834 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:34.834 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:34.834 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:07:34.834 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:02.899797+0000", 2023-09-23T21:07:34.834 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00013229499999999999 2023-09-23T21:07:34.834 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:34.834 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:34.834 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 5,7", 2023-09-23T21:07:34.835 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:02.899921+0000", 2023-09-23T21:07:34.835 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00012398800000000001 2023-09-23T21:07:34.835 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:34.835 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:34.835 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T21:07:34.835 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:02.900843+0000", 2023-09-23T21:07:34.835 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00092134699999999999 2023-09-23T21:07:34.835 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:34.835 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:34.835 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:07:34.835 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:02.902774+0000", 2023-09-23T21:07:34.835 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.001931363 2023-09-23T21:07:34.836 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:34.836 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:34.836 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:07:34.836 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:02.902827+0000", 2023-09-23T21:07:34.836 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.2701999999999998e-05 2023-09-23T21:07:34.836 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:34.836 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:34.836 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T21:07:34.836 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:02.902867+0000", 2023-09-23T21:07:34.836 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.0374999999999999e-05 2023-09-23T21:07:34.836 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:34.836 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:34.837 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T21:07:34.837 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:02.967387+0000", 2023-09-23T21:07:34.837 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.064520309999999997 2023-09-23T21:07:34.837 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:07:34.837 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:07:34.837 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:07:34.837 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:34.837 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:34.837 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35856.0:17999 9.0 9:094ae900:::benchmark_data_smithi026_342732_object17998:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e351)", 2023-09-23T21:07:34.837 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T21:06:02.902254+0000", 2023-09-23T21:07:34.837 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 91.883930606000007, 2023-09-23T21:07:34.837 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.067497905999999996, 2023-09-23T21:07:34.837 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:07:34.838 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:07:34.838 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T21:07:34.838 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35856", 2023-09-23T21:07:34.838 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/4180396258", 2023-09-23T21:07:34.838 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 17999 2023-09-23T21:07:34.838 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:34.838 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:07:34.838 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:34.838 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:07:34.838 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:02.902254+0000", 2023-09-23T21:07:34.838 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:07:34.838 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:34.839 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:34.839 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:07:34.839 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:02.902254+0000", 2023-09-23T21:07:34.839 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:07:34.839 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:34.839 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:34.839 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:07:34.839 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:02.902252+0000", 2023-09-23T21:07:34.839 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T21:07:34.839 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:34.839 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:34.839 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:07:34.840 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:02.902280+0000", 2023-09-23T21:07:34.840 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.7900000000000001e-05 2023-09-23T21:07:34.840 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:34.840 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:34.840 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:07:34.840 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:02.902290+0000", 2023-09-23T21:07:34.840 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.5040000000000008e-06 2023-09-23T21:07:34.840 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:34.840 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:34.840 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:07:34.840 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:02.902296+0000", 2023-09-23T21:07:34.840 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.4330000000000001e-06 2023-09-23T21:07:34.841 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:34.841 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:34.841 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:07:34.841 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:02.902486+0000", 2023-09-23T21:07:34.841 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000189948 2023-09-23T21:07:34.841 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:34.841 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:34.841 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:07:34.841 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:02.902625+0000", 2023-09-23T21:07:34.841 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000139068 2023-09-23T21:07:34.841 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:34.841 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:34.841 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 2,6", 2023-09-23T21:07:34.842 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:02.902776+0000", 2023-09-23T21:07:34.842 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00015038000000000001 2023-09-23T21:07:34.842 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:34.842 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:34.842 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:07:34.842 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:02.968238+0000", 2023-09-23T21:07:34.842 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.065462612000000003 2023-09-23T21:07:34.842 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:34.842 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:34.842 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:07:34.842 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:02.968271+0000", 2023-09-23T21:07:34.842 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.256e-05 2023-09-23T21:07:34.843 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:34.843 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:34.843 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T21:07:34.843 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:02.969660+0000", 2023-09-23T21:07:34.843 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0013896990000000001 2023-09-23T21:07:34.843 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:34.843 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:34.843 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T21:07:34.843 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:02.969715+0000", 2023-09-23T21:07:34.843 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.4182999999999999e-05 2023-09-23T21:07:34.843 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:34.843 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:34.844 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T21:07:34.844 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:02.969752+0000", 2023-09-23T21:07:34.844 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.7017000000000002e-05 2023-09-23T21:07:34.844 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:07:34.844 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:07:34.844 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:07:34.844 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:34.844 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:34.844 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_repop(client.35856.0:17988 9.a e351/350 9:5c199503:::benchmark_data_smithi026_342732_object17987:head v 351'1130, mlcod=351'1127)", 2023-09-23T21:07:34.844 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T21:06:02.902551+0000", 2023-09-23T21:07:34.844 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 91.883633509000006, 2023-09-23T21:07:34.844 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.067180202999999994, 2023-09-23T21:07:34.845 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:07:34.845 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:07:34.845 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:07:34.845 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:34.845 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:07:34.845 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:02.902551+0000", 2023-09-23T21:07:34.845 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:07:34.845 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:34.845 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:34.845 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:07:34.845 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:02.902551+0000", 2023-09-23T21:07:34.845 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:07:34.846 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:34.846 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:34.846 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:07:34.846 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:02.902550+0000", 2023-09-23T21:07:34.846 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T21:07:34.846 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:34.846 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:34.846 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:07:34.846 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:02.902586+0000", 2023-09-23T21:07:34.846 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.5630000000000003e-05 2023-09-23T21:07:34.846 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:34.846 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:34.847 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:07:34.847 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:02.902597+0000", 2023-09-23T21:07:34.847 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0592e-05 2023-09-23T21:07:34.847 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:34.847 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:34.847 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:07:34.847 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:02.902603+0000", 2023-09-23T21:07:34.847 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.3799999999999999e-06 2023-09-23T21:07:34.847 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:34.847 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:34.847 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:07:34.847 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:02.967317+0000", 2023-09-23T21:07:34.848 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.064713941999999997 2023-09-23T21:07:34.848 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:34.848 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:34.848 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:07:34.848 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:02.967344+0000", 2023-09-23T21:07:34.848 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.6794000000000001e-05 2023-09-23T21:07:34.848 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:34.848 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:34.848 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T21:07:34.848 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:02.969686+0000", 2023-09-23T21:07:34.848 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.002342261 2023-09-23T21:07:34.849 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:34.849 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:34.849 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T21:07:34.849 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:02.969731+0000", 2023-09-23T21:07:34.849 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.5083999999999997e-05 2023-09-23T21:07:34.849 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:07:34.849 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:07:34.849 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:07:34.849 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:34.849 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:34.849 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_repop(client.35856.0:18000 9.3 e351/350 9:cedee64b:::benchmark_data_smithi026_342732_object17999:head v 351'1117, mlcod=351'1113)", 2023-09-23T21:07:34.849 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T21:06:02.903357+0000", 2023-09-23T21:07:34.850 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 91.882827175000003, 2023-09-23T21:07:34.850 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.066530874000000004, 2023-09-23T21:07:34.850 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:07:34.850 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:07:34.850 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:07:34.850 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:34.850 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:07:34.850 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:02.903357+0000", 2023-09-23T21:07:34.850 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:07:34.850 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:34.850 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:34.850 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:07:34.851 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:02.903357+0000", 2023-09-23T21:07:34.851 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:07:34.851 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:34.851 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:34.851 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:07:34.851 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:02.903357+0000", 2023-09-23T21:07:34.851 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T21:07:34.851 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:34.851 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:34.851 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:07:34.851 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:02.903379+0000", 2023-09-23T21:07:34.851 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.2904e-05 2023-09-23T21:07:34.852 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:34.852 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:34.852 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:07:34.852 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:02.967303+0000", 2023-09-23T21:07:34.852 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.063923063000000002 2023-09-23T21:07:34.852 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:34.852 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:34.852 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:07:34.852 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:02.967309+0000", 2023-09-23T21:07:34.852 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.1970000000000001e-06 2023-09-23T21:07:34.852 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:34.852 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:34.853 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:07:34.853 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:02.967535+0000", 2023-09-23T21:07:34.853 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00022615999999999999 2023-09-23T21:07:34.853 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:34.853 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:34.853 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:07:34.853 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:02.967557+0000", 2023-09-23T21:07:34.853 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.2045999999999999e-05 2023-09-23T21:07:34.853 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:34.853 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:34.853 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T21:07:34.853 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:02.969846+0000", 2023-09-23T21:07:34.854 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00228912 2023-09-23T21:07:34.854 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:34.854 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:34.854 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T21:07:34.854 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:02.969888+0000", 2023-09-23T21:07:34.854 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.1993000000000003e-05 2023-09-23T21:07:34.854 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:07:34.854 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:07:34.854 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:07:34.854 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:34.854 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:34.854 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_repop(client.35856.0:21158 9.3 e351/350 9:c23afb70:::benchmark_data_smithi026_342732_object21157:head v 351'1301, mlcod=351'1299)", 2023-09-23T21:07:34.855 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T21:06:04.346523+0000", 2023-09-23T21:07:34.855 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 90.439661325000003, 2023-09-23T21:07:34.855 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.072563047000000006, 2023-09-23T21:07:34.855 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:07:34.855 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:07:34.855 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:07:34.855 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:34.855 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:07:34.855 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:04.346523+0000", 2023-09-23T21:07:34.855 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:07:34.855 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:34.856 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:34.856 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:07:34.856 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:04.346523+0000", 2023-09-23T21:07:34.856 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:07:34.856 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:34.856 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:34.856 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:07:34.856 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:04.346522+0000", 2023-09-23T21:07:34.856 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T21:07:34.856 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:34.856 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:34.856 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:07:34.856 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:04.346548+0000", 2023-09-23T21:07:34.857 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.5261000000000001e-05 2023-09-23T21:07:34.857 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:34.857 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:34.857 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:07:34.857 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:04.346559+0000", 2023-09-23T21:07:34.857 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1073000000000001e-05 2023-09-23T21:07:34.857 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:34.857 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:34.857 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:07:34.857 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:04.346565+0000", 2023-09-23T21:07:34.857 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.6440000000000003e-06 2023-09-23T21:07:34.857 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:34.858 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:34.858 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:07:34.858 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:04.346636+0000", 2023-09-23T21:07:34.858 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.0710000000000006e-05 2023-09-23T21:07:34.858 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:34.858 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:34.858 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:07:34.858 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:04.346655+0000", 2023-09-23T21:07:34.858 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.9219999999999999e-05 2023-09-23T21:07:34.858 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:34.858 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:34.858 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T21:07:34.859 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:04.419046+0000", 2023-09-23T21:07:34.859 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.072390232999999998 2023-09-23T21:07:34.859 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:34.859 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:34.859 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T21:07:34.859 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:04.419086+0000", 2023-09-23T21:07:34.859 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.0596000000000002e-05 2023-09-23T21:07:34.859 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:07:34.859 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:07:34.859 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:07:34.859 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:34.860 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:34.860 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_repop(client.35856.0:21170 9.3 e351/350 9:cece9000:::benchmark_data_smithi026_342732_object21169:head v 351'1302, mlcod=351'1299)", 2023-09-23T21:07:34.860 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T21:06:04.349707+0000", 2023-09-23T21:07:34.860 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 90.436477726999996, 2023-09-23T21:07:34.860 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.069404628999999995, 2023-09-23T21:07:34.860 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:07:34.860 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:07:34.860 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:07:34.860 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:34.860 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:07:34.860 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:04.349707+0000", 2023-09-23T21:07:34.860 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:07:34.861 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:34.861 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:34.861 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:07:34.861 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:04.349707+0000", 2023-09-23T21:07:34.861 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:07:34.861 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:34.861 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:34.861 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:07:34.861 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:04.349706+0000", 2023-09-23T21:07:34.861 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.999999 2023-09-23T21:07:34.861 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:34.861 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:34.862 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:07:34.862 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:04.349739+0000", 2023-09-23T21:07:34.862 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.3043000000000002e-05 2023-09-23T21:07:34.862 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:34.862 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:34.862 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:07:34.862 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:04.349763+0000", 2023-09-23T21:07:34.862 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.4479000000000001e-05 2023-09-23T21:07:34.862 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:34.862 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:34.862 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:07:34.862 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:04.349775+0000", 2023-09-23T21:07:34.863 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1301000000000001e-05 2023-09-23T21:07:34.863 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:34.863 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:34.863 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:07:34.863 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:04.395248+0000", 2023-09-23T21:07:34.863 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.045473062000000002 2023-09-23T21:07:34.863 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:34.863 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:34.863 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:07:34.863 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:04.395269+0000", 2023-09-23T21:07:34.863 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.1060000000000002e-05 2023-09-23T21:07:34.864 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:34.864 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:34.864 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T21:07:34.864 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:04.419089+0000", 2023-09-23T21:07:34.864 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.023820249000000002 2023-09-23T21:07:34.864 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:34.864 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:34.864 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T21:07:34.864 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:04.419111+0000", 2023-09-23T21:07:34.864 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.2154000000000002e-05 2023-09-23T21:07:34.864 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:07:34.864 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:07:34.865 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:07:34.865 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:34.865 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:34.865 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35856.0:28250 9.4 9:2068156a:::benchmark_data_smithi026_342732_object28249:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e351)", 2023-09-23T21:07:34.865 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T21:06:07.706758+0000", 2023-09-23T21:07:34.865 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 87.079426845, 2023-09-23T21:07:34.865 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.074874349000000007, 2023-09-23T21:07:34.865 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:07:34.865 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:07:34.865 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T21:07:34.865 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35856", 2023-09-23T21:07:34.865 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/4180396258", 2023-09-23T21:07:34.865 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 28250 2023-09-23T21:07:34.866 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:34.866 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:07:34.866 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:34.866 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:07:34.866 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:07.706758+0000", 2023-09-23T21:07:34.866 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:07:34.866 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:34.866 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:34.866 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:07:34.866 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:07.706758+0000", 2023-09-23T21:07:34.866 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:07:34.867 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:34.867 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:34.867 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:07:34.867 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:07.706757+0000", 2023-09-23T21:07:34.867 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.999999 2023-09-23T21:07:34.867 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:34.867 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:34.867 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:07:34.867 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:07.706780+0000", 2023-09-23T21:07:34.867 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.2736999999999999e-05 2023-09-23T21:07:34.867 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:34.867 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:34.867 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:07:34.868 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:07.706788+0000", 2023-09-23T21:07:34.868 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.8599999999999999e-06 2023-09-23T21:07:34.868 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:34.868 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:34.868 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:07:34.868 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:07.706794+0000", 2023-09-23T21:07:34.868 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.0240000000000003e-06 2023-09-23T21:07:34.868 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:34.868 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:34.868 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:07:34.868 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:07.706842+0000", 2023-09-23T21:07:34.868 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.7802000000000001e-05 2023-09-23T21:07:34.869 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:34.869 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:34.869 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:07:34.869 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:07.707000+0000", 2023-09-23T21:07:34.869 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000158212 2023-09-23T21:07:34.869 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:34.869 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:34.869 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 3,4", 2023-09-23T21:07:34.869 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:07.708419+0000", 2023-09-23T21:07:34.869 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.001418707 2023-09-23T21:07:34.869 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:34.869 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:34.869 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T21:07:34.870 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:07.709792+0000", 2023-09-23T21:07:34.870 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.001372699 2023-09-23T21:07:34.870 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:34.870 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:34.870 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:07:34.870 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:07.710542+0000", 2023-09-23T21:07:34.870 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00074978000000000002 2023-09-23T21:07:34.870 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:34.870 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:34.870 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:07:34.870 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:07.781546+0000", 2023-09-23T21:07:34.870 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.071004450999999996 2023-09-23T21:07:34.871 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:34.871 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:34.871 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T21:07:34.871 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:07.781597+0000", 2023-09-23T21:07:34.871 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.0676000000000001e-05 2023-09-23T21:07:34.871 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:34.871 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:34.871 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T21:07:34.871 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:07.781632+0000", 2023-09-23T21:07:34.871 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.5120000000000003e-05 2023-09-23T21:07:34.871 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:07:34.871 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:07:34.872 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:07:34.872 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:34.872 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:34.872 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35856.0:56891 9.4 9:2107680b:::benchmark_data_smithi026_342732_object56890:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e356)", 2023-09-23T21:07:34.872 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T21:06:28.222593+0000", 2023-09-23T21:07:34.872 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 66.563591786000003, 2023-09-23T21:07:34.872 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.1132223, 2023-09-23T21:07:34.872 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:07:34.872 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:07:34.872 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T21:07:34.872 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35856", 2023-09-23T21:07:34.872 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/4180396258", 2023-09-23T21:07:34.873 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 56891 2023-09-23T21:07:34.873 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:34.873 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:07:34.873 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:34.873 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:07:34.873 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:28.222593+0000", 2023-09-23T21:07:34.873 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:07:34.873 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:34.873 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:34.873 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:07:34.873 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:28.222593+0000", 2023-09-23T21:07:34.873 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:07:34.874 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:34.874 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:34.874 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:07:34.874 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:28.222591+0000", 2023-09-23T21:07:34.874 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T21:07:34.874 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:34.874 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:34.874 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:07:34.874 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:28.222623+0000", 2023-09-23T21:07:34.874 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.133e-05 2023-09-23T21:07:34.874 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:34.874 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:34.875 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:07:34.875 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:28.222638+0000", 2023-09-23T21:07:34.875 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.5206e-05 2023-09-23T21:07:34.875 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:34.875 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:34.875 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:07:34.875 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:28.222646+0000", 2023-09-23T21:07:34.875 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.3210000000000005e-06 2023-09-23T21:07:34.875 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:34.875 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:34.875 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:07:34.876 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:28.222701+0000", 2023-09-23T21:07:34.876 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.4626999999999997e-05 2023-09-23T21:07:34.876 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:34.876 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:34.876 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:07:34.876 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:28.222823+0000", 2023-09-23T21:07:34.876 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000122213 2023-09-23T21:07:34.876 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:34.876 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:34.876 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 3,4", 2023-09-23T21:07:34.876 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:28.222968+0000", 2023-09-23T21:07:34.876 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000145284 2023-09-23T21:07:34.877 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:34.877 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:34.877 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T21:07:34.877 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:28.229387+0000", 2023-09-23T21:07:34.877 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0064191650000000001 2023-09-23T21:07:34.877 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:34.877 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:34.877 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:07:34.877 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:28.229847+0000", 2023-09-23T21:07:34.877 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00045973900000000002 2023-09-23T21:07:34.877 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:34.877 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:34.877 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:07:34.878 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:28.335721+0000", 2023-09-23T21:07:34.878 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.105873695 2023-09-23T21:07:34.878 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:34.878 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:34.878 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T21:07:34.878 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:28.335777+0000", 2023-09-23T21:07:34.878 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.5909000000000003e-05 2023-09-23T21:07:34.878 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:34.878 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:34.878 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T21:07:34.878 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:28.335815+0000", 2023-09-23T21:07:34.878 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.8059000000000003e-05 2023-09-23T21:07:34.879 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:07:34.879 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:07:34.879 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:07:34.879 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:34.879 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:34.879 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35856.0:56895 9.4 9:25773dd1:::benchmark_data_smithi026_342732_object56894:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e356)", 2023-09-23T21:07:34.879 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T21:06:28.222728+0000", 2023-09-23T21:07:34.879 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 66.563456810000005, 2023-09-23T21:07:34.879 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.113186604, 2023-09-23T21:07:34.879 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:07:34.879 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:07:34.879 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T21:07:34.880 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35856", 2023-09-23T21:07:34.880 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/4180396258", 2023-09-23T21:07:34.880 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 56895 2023-09-23T21:07:34.880 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:34.880 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:07:34.880 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:34.880 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:07:34.880 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:28.222728+0000", 2023-09-23T21:07:34.880 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:07:34.880 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:34.880 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:34.880 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:07:34.881 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:28.222728+0000", 2023-09-23T21:07:34.881 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:07:34.881 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:34.881 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:34.881 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:07:34.881 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:28.222727+0000", 2023-09-23T21:07:34.881 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T21:07:34.881 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:34.881 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:34.881 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:07:34.881 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:28.222749+0000", 2023-09-23T21:07:34.881 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.1985999999999999e-05 2023-09-23T21:07:34.882 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:34.882 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:34.882 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:07:34.882 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:28.222758+0000", 2023-09-23T21:07:34.882 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.6389999999999998e-06 2023-09-23T21:07:34.882 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:34.882 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:34.882 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:07:34.882 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:28.222764+0000", 2023-09-23T21:07:34.882 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.4230000000000002e-06 2023-09-23T21:07:34.882 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:34.883 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:34.883 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:07:34.883 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:28.224999+0000", 2023-09-23T21:07:34.883 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0022348699999999999 2023-09-23T21:07:34.883 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:34.883 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:34.883 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:07:34.883 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:28.225121+0000", 2023-09-23T21:07:34.883 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00012207099999999999 2023-09-23T21:07:34.883 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:34.883 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:34.883 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 3,4", 2023-09-23T21:07:34.884 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:28.225275+0000", 2023-09-23T21:07:34.884 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00015440399999999999 2023-09-23T21:07:34.884 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:34.884 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:34.884 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T21:07:34.884 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:28.229528+0000", 2023-09-23T21:07:34.884 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0042530520000000002 2023-09-23T21:07:34.884 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:34.884 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:34.884 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:07:34.884 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:28.229886+0000", 2023-09-23T21:07:34.884 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00035731199999999998 2023-09-23T21:07:34.885 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:34.885 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:34.885 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:07:34.885 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:28.335849+0000", 2023-09-23T21:07:34.885 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.10596296099999999 2023-09-23T21:07:34.885 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:34.885 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:34.885 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T21:07:34.885 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:28.335885+0000", 2023-09-23T21:07:34.885 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.5849999999999997e-05 2023-09-23T21:07:34.885 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:34.885 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:34.886 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T21:07:34.886 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:28.335914+0000", 2023-09-23T21:07:34.886 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.9671000000000001e-05 2023-09-23T21:07:34.886 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:07:34.886 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:07:34.886 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:07:34.886 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:34.886 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:34.886 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35856.0:56896 9.4 9:2c2f2fe6:::benchmark_data_smithi026_342732_object56895:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e356)", 2023-09-23T21:07:34.886 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T21:06:28.222775+0000", 2023-09-23T21:07:34.886 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 66.563409448000002, 2023-09-23T21:07:34.886 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.113235683, 2023-09-23T21:07:34.887 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:07:34.887 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:07:34.887 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T21:07:34.887 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35856", 2023-09-23T21:07:34.887 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/4180396258", 2023-09-23T21:07:34.887 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 56896 2023-09-23T21:07:34.887 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:34.887 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:07:34.887 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:34.887 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:07:34.887 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:28.222775+0000", 2023-09-23T21:07:34.887 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:07:34.887 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:34.888 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:34.888 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:07:34.888 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:28.222775+0000", 2023-09-23T21:07:34.888 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:07:34.888 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:34.888 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:34.888 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:07:34.888 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:28.222774+0000", 2023-09-23T21:07:34.888 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T21:07:34.888 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:34.888 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:34.889 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:07:34.889 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:28.222796+0000", 2023-09-23T21:07:34.889 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.1447e-05 2023-09-23T21:07:34.889 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:34.889 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:34.889 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:07:34.889 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:28.222803+0000", 2023-09-23T21:07:34.889 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.6380000000000004e-06 2023-09-23T21:07:34.889 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:34.889 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:34.889 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:07:34.889 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:28.222809+0000", 2023-09-23T21:07:34.889 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.3419999999999998e-06 2023-09-23T21:07:34.890 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:34.890 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:34.890 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:07:34.890 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:28.227114+0000", 2023-09-23T21:07:34.890 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0043053980000000002 2023-09-23T21:07:34.890 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:34.890 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:34.890 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:07:34.890 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:28.227243+0000", 2023-09-23T21:07:34.890 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000128777 2023-09-23T21:07:34.890 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:34.891 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:34.891 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 3,4", 2023-09-23T21:07:34.891 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:28.227344+0000", 2023-09-23T21:07:34.891 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000101049 2023-09-23T21:07:34.891 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:34.891 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:34.891 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T21:07:34.891 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:28.229533+0000", 2023-09-23T21:07:34.891 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0021894420000000002 2023-09-23T21:07:34.891 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:34.891 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:34.891 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:07:34.891 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:28.230019+0000", 2023-09-23T21:07:34.892 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000485732 2023-09-23T21:07:34.892 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:34.892 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:34.892 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:07:34.892 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:28.335944+0000", 2023-09-23T21:07:34.892 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.10592515299999999 2023-09-23T21:07:34.892 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:34.892 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:34.892 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T21:07:34.892 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:28.335981+0000", 2023-09-23T21:07:34.892 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.6609999999999997e-05 2023-09-23T21:07:34.892 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:34.893 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:34.893 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T21:07:34.893 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:28.336011+0000", 2023-09-23T21:07:34.893 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.9683e-05 2023-09-23T21:07:34.893 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:07:34.893 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:07:34.893 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:07:34.893 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:34.893 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:34.893 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35856.0:56900 9.4 9:2dd11179:::benchmark_data_smithi026_342732_object56899:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e356)", 2023-09-23T21:07:34.893 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T21:06:28.230608+0000", 2023-09-23T21:07:34.893 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 66.555576539, 2023-09-23T21:07:34.894 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.10945954300000001, 2023-09-23T21:07:34.894 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:07:34.894 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:07:34.894 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T21:07:34.894 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35856", 2023-09-23T21:07:34.894 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/4180396258", 2023-09-23T21:07:34.894 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 56900 2023-09-23T21:07:34.894 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:34.894 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:07:34.894 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:34.894 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:07:34.894 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:28.230608+0000", 2023-09-23T21:07:34.895 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:07:34.895 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:34.895 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:34.895 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:07:34.895 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:28.230608+0000", 2023-09-23T21:07:34.895 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:07:34.895 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:34.895 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:34.895 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:07:34.895 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:28.230606+0000", 2023-09-23T21:07:34.895 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T21:07:34.895 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:34.896 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:34.896 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:07:34.896 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:28.230637+0000", 2023-09-23T21:07:34.896 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.0661999999999999e-05 2023-09-23T21:07:34.896 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:34.896 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:34.896 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:07:34.896 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:28.230647+0000", 2023-09-23T21:07:34.896 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.7909999999999999e-06 2023-09-23T21:07:34.896 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:34.896 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:34.896 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:07:34.896 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:28.230652+0000", 2023-09-23T21:07:34.897 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.5369999999999997e-06 2023-09-23T21:07:34.897 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:34.897 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:34.897 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:07:34.897 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:28.327367+0000", 2023-09-23T21:07:34.897 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.096714404000000004 2023-09-23T21:07:34.897 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:34.897 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:34.897 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:07:34.897 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:28.327530+0000", 2023-09-23T21:07:34.897 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00016266499999999999 2023-09-23T21:07:34.897 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:34.898 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:34.898 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 3,4", 2023-09-23T21:07:34.898 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:28.327694+0000", 2023-09-23T21:07:34.898 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00016473099999999999 2023-09-23T21:07:34.898 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:34.898 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:34.898 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T21:07:34.898 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:28.329989+0000", 2023-09-23T21:07:34.898 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0022946939999999999 2023-09-23T21:07:34.898 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:34.898 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:34.898 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:07:34.899 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:28.336041+0000", 2023-09-23T21:07:34.899 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0060519700000000003 2023-09-23T21:07:34.899 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:34.899 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:34.899 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:07:34.899 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:28.339969+0000", 2023-09-23T21:07:34.899 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0039278550000000001 2023-09-23T21:07:34.899 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:34.899 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:34.899 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T21:07:34.899 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:28.340023+0000", 2023-09-23T21:07:34.899 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.4403000000000001e-05 2023-09-23T21:07:34.899 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:34.900 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:34.900 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T21:07:34.900 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:28.340067+0000", 2023-09-23T21:07:34.900 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.4190999999999998e-05 2023-09-23T21:07:34.900 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:07:34.900 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:07:34.900 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:07:34.900 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:34.900 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:34.900 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_repop(client.35856.0:56904 9.6 e356/350 9:6b4a6fe3:::benchmark_data_smithi026_342732_object56903:head v 356'3655, mlcod=356'3652)", 2023-09-23T21:07:34.900 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T21:06:28.231068+0000", 2023-09-23T21:07:34.900 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 66.555116436000006, 2023-09-23T21:07:34.901 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.098783234999999997, 2023-09-23T21:07:34.901 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:07:34.901 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:07:34.901 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:07:34.901 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:34.901 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:07:34.901 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:28.231068+0000", 2023-09-23T21:07:34.901 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:07:34.901 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:34.901 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:34.901 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:07:34.901 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:28.231068+0000", 2023-09-23T21:07:34.901 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:07:34.902 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:34.902 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:34.902 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:07:34.902 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:28.231067+0000", 2023-09-23T21:07:34.902 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T21:07:34.902 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:34.902 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:34.902 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:07:34.902 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:28.231159+0000", 2023-09-23T21:07:34.902 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.1742000000000004e-05 2023-09-23T21:07:34.902 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:34.902 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:34.903 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:07:34.903 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:28.327237+0000", 2023-09-23T21:07:34.903 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.096078075999999998 2023-09-23T21:07:34.903 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:34.903 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:34.903 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:07:34.903 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:28.327248+0000", 2023-09-23T21:07:34.903 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0798000000000001e-05 2023-09-23T21:07:34.903 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:34.903 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:34.903 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:07:34.903 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:28.327423+0000", 2023-09-23T21:07:34.903 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00017539000000000001 2023-09-23T21:07:34.904 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:34.904 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:34.904 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:07:34.904 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:28.327446+0000", 2023-09-23T21:07:34.904 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.207e-05 2023-09-23T21:07:34.904 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:34.904 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:34.904 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T21:07:34.904 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:28.329828+0000", 2023-09-23T21:07:34.904 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0023820579999999998 2023-09-23T21:07:34.904 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:34.904 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:34.905 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T21:07:34.905 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:28.329851+0000", 2023-09-23T21:07:34.905 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.3600000000000001e-05 2023-09-23T21:07:34.905 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:07:34.905 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:07:34.905 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:07:34.905 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:34.905 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:34.905 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_repop(client.35856.0:61395 9.f e356/350 9:f0a2a06c:::benchmark_data_smithi026_342732_object61394:head v 356'3811, mlcod=356'3808)", 2023-09-23T21:07:34.905 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T21:06:31.499130+0000", 2023-09-23T21:07:34.905 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 63.287053987, 2023-09-23T21:07:34.905 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.067215021999999999, 2023-09-23T21:07:34.905 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:07:34.906 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:07:34.906 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:07:34.906 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:34.906 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:07:34.906 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:31.499130+0000", 2023-09-23T21:07:34.906 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:07:34.906 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:34.906 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:34.906 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:07:34.906 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:31.499130+0000", 2023-09-23T21:07:34.906 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:07:34.906 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:34.907 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:34.907 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:07:34.907 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:31.499130+0000", 2023-09-23T21:07:34.907 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T21:07:34.907 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:34.907 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:34.907 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:07:34.907 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:31.499165+0000", 2023-09-23T21:07:34.907 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.4907000000000003e-05 2023-09-23T21:07:34.907 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:34.907 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:34.908 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:07:34.908 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:31.499173+0000", 2023-09-23T21:07:34.908 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.9130000000000003e-06 2023-09-23T21:07:34.908 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:34.908 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:34.908 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:07:34.908 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:31.499178+0000", 2023-09-23T21:07:34.908 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.3159999999999996e-06 2023-09-23T21:07:34.908 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:34.908 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:34.908 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:07:34.908 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:31.499276+0000", 2023-09-23T21:07:34.909 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.7678999999999998e-05 2023-09-23T21:07:34.909 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:34.909 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:34.909 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:07:34.909 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:31.499296+0000", 2023-09-23T21:07:34.909 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.0123999999999999e-05 2023-09-23T21:07:34.909 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:34.909 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:34.909 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T21:07:34.909 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:31.566305+0000", 2023-09-23T21:07:34.909 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.067009424999999997 2023-09-23T21:07:34.909 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:34.910 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:34.910 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T21:07:34.910 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:31.566345+0000", 2023-09-23T21:07:34.910 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.0059999999999999e-05 2023-09-23T21:07:34.910 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:07:34.910 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:07:34.910 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:07:34.910 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:34.910 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:34.910 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_repop(client.35856.0:61396 9.f e356/350 9:f8403760:::benchmark_data_smithi026_342732_object61395:head v 356'3812, mlcod=356'3808)", 2023-09-23T21:07:34.910 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T21:06:31.501377+0000", 2023-09-23T21:07:34.910 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 63.284807227999998, 2023-09-23T21:07:34.911 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.067027725999999996, 2023-09-23T21:07:34.911 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:07:34.911 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:07:34.911 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:07:34.911 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:34.911 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:07:34.911 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:31.501377+0000", 2023-09-23T21:07:34.911 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:07:34.911 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:34.911 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:34.911 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:07:34.911 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:31.501377+0000", 2023-09-23T21:07:34.912 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:07:34.912 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:34.912 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:34.912 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:07:34.912 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:31.501376+0000", 2023-09-23T21:07:34.912 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T21:07:34.912 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:34.912 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:34.912 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:07:34.912 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:31.501411+0000", 2023-09-23T21:07:34.912 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.4076999999999999e-05 2023-09-23T21:07:34.912 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:34.912 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:34.913 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:07:34.913 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:31.501420+0000", 2023-09-23T21:07:34.913 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.9930000000000001e-06 2023-09-23T21:07:34.913 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:34.913 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:34.913 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:07:34.913 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:31.501426+0000", 2023-09-23T21:07:34.913 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.3199999999999996e-06 2023-09-23T21:07:34.913 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:34.913 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:34.913 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:07:34.913 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:31.507517+0000", 2023-09-23T21:07:34.914 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0060912049999999997 2023-09-23T21:07:34.914 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:34.914 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:34.914 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:07:34.914 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:31.507539+0000", 2023-09-23T21:07:34.914 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.158e-05 2023-09-23T21:07:34.914 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:34.914 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:34.914 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T21:07:34.914 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:31.568367+0000", 2023-09-23T21:07:34.914 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.060828672 2023-09-23T21:07:34.914 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:34.915 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:34.915 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T21:07:34.915 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:31.568405+0000", 2023-09-23T21:07:34.915 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.7586000000000002e-05 2023-09-23T21:07:34.915 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:07:34.915 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:07:34.915 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:07:34.915 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:34.915 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:34.915 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35856.0:62099 9.4 9:206cd93d:::benchmark_data_smithi026_342732_object62098:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e356)", 2023-09-23T21:07:34.915 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T21:06:32.149970+0000", 2023-09-23T21:07:34.915 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 62.636214721999998, 2023-09-23T21:07:34.915 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.092984948999999997, 2023-09-23T21:07:34.916 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:07:34.916 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:07:34.916 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T21:07:34.916 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35856", 2023-09-23T21:07:34.916 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/4180396258", 2023-09-23T21:07:34.916 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 62099 2023-09-23T21:07:34.916 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:34.916 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:07:34.916 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:34.916 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:07:34.916 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:32.149970+0000", 2023-09-23T21:07:34.916 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:07:34.917 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:34.917 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:34.917 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:07:34.917 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:32.149970+0000", 2023-09-23T21:07:34.917 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:07:34.917 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:34.917 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:34.917 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:07:34.917 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:32.149968+0000", 2023-09-23T21:07:34.917 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T21:07:34.917 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:34.917 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:34.917 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:07:34.918 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:32.149990+0000", 2023-09-23T21:07:34.918 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.1460000000000001e-05 2023-09-23T21:07:34.918 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:34.918 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:34.918 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:07:34.918 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:32.149999+0000", 2023-09-23T21:07:34.918 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.5300000000000002e-06 2023-09-23T21:07:34.918 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:34.918 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:34.918 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:07:34.918 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:32.150005+0000", 2023-09-23T21:07:34.918 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.8619999999999999e-06 2023-09-23T21:07:34.919 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:34.919 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:34.919 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:07:34.919 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:32.150044+0000", 2023-09-23T21:07:34.919 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.8908000000000001e-05 2023-09-23T21:07:34.919 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:34.919 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:34.919 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:07:34.919 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:32.150133+0000", 2023-09-23T21:07:34.919 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.8941000000000001e-05 2023-09-23T21:07:34.919 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:34.919 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:34.919 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 3,4", 2023-09-23T21:07:34.920 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:32.150223+0000", 2023-09-23T21:07:34.920 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.9922999999999999e-05 2023-09-23T21:07:34.920 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:34.920 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:34.920 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T21:07:34.920 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:32.151790+0000", 2023-09-23T21:07:34.920 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0015673379999999999 2023-09-23T21:07:34.920 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:34.920 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:34.920 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:07:34.920 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:32.196261+0000", 2023-09-23T21:07:34.920 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.044470770999999999 2023-09-23T21:07:34.921 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:34.921 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:34.921 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:07:34.921 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:32.242852+0000", 2023-09-23T21:07:34.921 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.046590456000000002 2023-09-23T21:07:34.921 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:34.921 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:34.921 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T21:07:34.921 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:32.242909+0000", 2023-09-23T21:07:34.921 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.7581999999999999e-05 2023-09-23T21:07:34.921 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:34.922 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:34.922 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T21:07:34.922 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:32.242955+0000", 2023-09-23T21:07:34.922 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.5485000000000002e-05 2023-09-23T21:07:34.922 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:07:34.922 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:07:34.922 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:07:34.922 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:07:34.922 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:07:34.922 INFO:teuthology.orchestra.run.smithi026.stdout:} 2023-09-23T21:07:34.923 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:07:34 smithi070 ceph-mon[168282]: pgmap v961: 109 pgs: 109 active+clean; 6.7 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 127 B/s rd, 23 MiB/s wr, 779 op/s 2023-09-23T21:07:34.923 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:07:34 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/3092042001' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:07:34.923 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:07:34.924 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:07:34.924 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:07:35.025 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:07:35.061 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:07:35.114 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:07:35.120 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:07:35.120 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:07:35.214 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:07:35.221 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:07:35.285 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:07:35.513 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:07:35.533 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:07:35.534 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:07:35.588 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_ops_in_flight 2023-09-23T21:07:35.635 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:07:35.663 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T21:07:35.717 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:07:35.774 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:07:35.786 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:07:35.787 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:07:35.888 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:07:35.922 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:07:35.961 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:07:35.964 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:07:35.965 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:07:36.066 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:07:36.138 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:07:36.329 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:07:36.351 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:07:36.352 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:07:36.452 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:07:36.466 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T21:07:36.553 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:07:36.631 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:07:36 smithi026 ceph-mon[179688]: pgmap v962: 109 pgs: 109 active+clean; 6.6 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 127 B/s rd, 21 MiB/s wr, 992 op/s 2023-09-23T21:07:36.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:07:36 smithi026 ceph-mon[175151]: pgmap v962: 109 pgs: 109 active+clean; 6.6 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 127 B/s rd, 21 MiB/s wr, 992 op/s 2023-09-23T21:07:36.702 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:07:36.719 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:07:36.720 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:07:36.820 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:07:36.846 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:07:36 smithi070 ceph-mon[168282]: pgmap v962: 109 pgs: 109 active+clean; 6.6 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 127 B/s rd, 21 MiB/s wr, 992 op/s 2023-09-23T21:07:36.860 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:07:36.910 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:07:36.912 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:07:36.912 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:07:37.013 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:07:37.140 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:07:37.154 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:07:37.162 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:07:37.206 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:07:37.212 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:07:37.212 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:07:37.313 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:07:37.342 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:07:37.386 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:07:37.388 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:07:37.389 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:07:37.489 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:07:37.530 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:07:37.533 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:07:37.577 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:07:37.580 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:07:37.580 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:07:37.681 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:07:37.709 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:07:37.757 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:07:37.762 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:07:37.763 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:07:37.863 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:07:37.970 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:07:38.326 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:07:38.345 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:07:38.346 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:07:38.347 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:07:37 smithi026 ceph-mon[175151]: pgmap v963: 109 pgs: 109 active+clean; 6.6 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 5.0 MiB/s wr, 693 op/s 2023-09-23T21:07:38.347 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:07:37 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:07:38.347 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:07:37 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:07:38.347 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:07:37 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/2708071782' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-09-23T21:07:38.347 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:07:37 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-09-23T21:07:38.348 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:07:37 smithi026 ceph-mon[179688]: pgmap v963: 109 pgs: 109 active+clean; 6.6 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 5.0 MiB/s wr, 693 op/s 2023-09-23T21:07:38.348 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:07:37 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:07:38.348 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:07:37 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:07:38.348 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:07:37 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/2708071782' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-09-23T21:07:38.348 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:07:37 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-09-23T21:07:38.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:07:37 smithi070 ceph-mon[168282]: pgmap v963: 109 pgs: 109 active+clean; 6.6 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 5.0 MiB/s wr, 693 op/s 2023-09-23T21:07:38.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:07:37 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:07:38.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:07:37 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:07:38.371 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:07:37 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/2708071782' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-09-23T21:07:38.371 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:07:37 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-09-23T21:07:38.447 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:07:38.512 INFO:teuthology.orchestra.run.smithi026.stderr:reweighted osd.7 to 1 (10000) 2023-09-23T21:07:38.555 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:07:39.293 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:07:38 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T21:07:39.293 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:07:38 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T21:07:39.293 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:07:38 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:07:39.293 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:07:38 smithi026 ceph-mon[175151]: failed to encode map e370 with expected crc 2023-09-23T21:07:39.293 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:07:38 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-09-23T21:07:39.294 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:07:38 smithi026 ceph-mon[175151]: osdmap e370: 8 total, 8 up, 8 in 2023-09-23T21:07:39.294 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:07:38 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T21:07:39.294 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:07:38 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T21:07:39.294 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:07:38 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:07:39.294 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:07:38 smithi026 ceph-mon[179688]: failed to encode map e370 with expected crc 2023-09-23T21:07:39.294 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:07:38 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-09-23T21:07:39.294 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:07:38 smithi026 ceph-mon[179688]: osdmap e370: 8 total, 8 up, 8 in 2023-09-23T21:07:39.348 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph osd reweight 4 1 2023-09-23T21:07:39.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:07:38 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T21:07:39.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:07:38 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T21:07:39.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:07:38 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:07:39.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:07:38 smithi070 ceph-mon[168282]: failed to encode map e370 with expected crc 2023-09-23T21:07:39.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:07:38 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-09-23T21:07:39.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:07:38 smithi070 ceph-mon[168282]: osdmap e370: 8 total, 8 up, 8 in 2023-09-23T21:07:39.421 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:07:39.427 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:07:39.427 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:07:39.528 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:07:39.558 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:07:39.609 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:07:39.614 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:07:39.615 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:07:39.715 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:07:39.745 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:07:39.795 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:07:39.802 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:07:39.802 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:07:39.886 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 21:07:39 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[164029]: ::ffff:172.21.15.70 - - [23/Sep/2023:21:07:39] "GET /metrics HTTP/1.1" 200 36294 "" "Prometheus/2.43.0" 2023-09-23T21:07:39.904 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:07:39.946 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:07:39.949 INFO:teuthology.orchestra.run.smithi026.stdout:{ 2023-09-23T21:07:39.949 INFO:teuthology.orchestra.run.smithi026.stdout: "ops": [ 2023-09-23T21:07:39.949 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:39.949 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35856.0:124118 9.5 9:a4f44fa5:::benchmark_data_smithi026_342732_object10842:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e370)", 2023-09-23T21:07:39.949 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T21:07:39.923976+0000", 2023-09-23T21:07:39.949 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 0.020414561000000001, 2023-09-23T21:07:39.949 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.020448784000000001, 2023-09-23T21:07:39.949 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:07:39.950 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "waiting for sub ops", 2023-09-23T21:07:39.950 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T21:07:39.950 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35856", 2023-09-23T21:07:39.950 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/4180396258", 2023-09-23T21:07:39.950 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 124118 2023-09-23T21:07:39.950 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:39.950 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:07:39.950 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:39.950 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:07:39.950 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:07:39.923976+0000", 2023-09-23T21:07:39.950 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:07:39.950 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:39.951 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:39.951 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:07:39.951 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:07:39.923976+0000", 2023-09-23T21:07:39.951 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:07:39.951 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:39.951 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:39.951 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:07:39.951 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:07:39.923976+0000", 2023-09-23T21:07:39.951 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T21:07:39.951 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:39.951 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:39.951 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:07:39.951 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:07:39.923978+0000", 2023-09-23T21:07:39.952 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.79e-06 2023-09-23T21:07:39.952 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:39.952 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:39.952 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:07:39.952 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:07:39.923989+0000", 2023-09-23T21:07:39.952 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0264e-05 2023-09-23T21:07:39.952 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:39.952 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:39.952 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:07:39.952 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:07:39.923994+0000", 2023-09-23T21:07:39.952 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.4689999999999996e-06 2023-09-23T21:07:39.952 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:39.952 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:39.953 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:07:39.953 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:07:39.936050+0000", 2023-09-23T21:07:39.953 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.012056020000000001 2023-09-23T21:07:39.953 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:39.953 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:39.953 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:07:39.953 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:07:39.936176+0000", 2023-09-23T21:07:39.953 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000125859 2023-09-23T21:07:39.953 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:39.953 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:39.953 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 5,7", 2023-09-23T21:07:39.953 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:07:39.936290+0000", 2023-09-23T21:07:39.953 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000114094 2023-09-23T21:07:39.954 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:39.954 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:39.954 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T21:07:39.954 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:07:39.940256+0000", 2023-09-23T21:07:39.954 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0039661460000000003 2023-09-23T21:07:39.954 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:07:39.954 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:07:39.954 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:07:39.954 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:39.954 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:39.954 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_repop_reply(client.35856.0:124116 9.5 e370/346)", 2023-09-23T21:07:39.954 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T21:07:39.942543+0000", 2023-09-23T21:07:39.955 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 0.001847319, 2023-09-23T21:07:39.955 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0019743669999999999, 2023-09-23T21:07:39.955 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:07:39.955 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "queued for pg", 2023-09-23T21:07:39.955 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:07:39.955 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:39.955 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:07:39.955 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:07:39.942543+0000", 2023-09-23T21:07:39.955 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:07:39.955 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:39.955 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:39.955 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:07:39.955 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:07:39.942543+0000", 2023-09-23T21:07:39.956 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:07:39.956 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:39.956 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:39.956 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:07:39.956 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:07:39.942543+0000", 2023-09-23T21:07:39.956 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T21:07:39.956 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:39.956 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:39.956 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:07:39.956 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:07:39.942548+0000", 2023-09-23T21:07:39.956 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.1730000000000003e-06 2023-09-23T21:07:39.956 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:39.956 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:39.957 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:07:39.957 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:07:39.942567+0000", 2023-09-23T21:07:39.957 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.9114e-05 2023-09-23T21:07:39.957 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:39.957 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:39.957 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:07:39.957 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:07:39.942575+0000", 2023-09-23T21:07:39.957 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.1159999999999999e-06 2023-09-23T21:07:39.957 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:07:39.957 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:07:39.957 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:07:39.957 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:39.957 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:39.958 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35856.0:124131 9.9 9:9eb88e4b:::benchmark_data_smithi026_342732_object10855:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e370)", 2023-09-23T21:07:39.958 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T21:07:39.942644+0000", 2023-09-23T21:07:39.958 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 0.0017465149999999999, 2023-09-23T21:07:39.958 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.001926475, 2023-09-23T21:07:39.958 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:07:39.958 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "waiting for sub ops", 2023-09-23T21:07:39.958 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T21:07:39.958 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35856", 2023-09-23T21:07:39.958 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/4180396258", 2023-09-23T21:07:39.958 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 124131 2023-09-23T21:07:39.958 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:39.958 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:07:39.959 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:39.959 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:07:39.959 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:07:39.942644+0000", 2023-09-23T21:07:39.959 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:07:39.959 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:39.959 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:39.959 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:07:39.959 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:07:39.942644+0000", 2023-09-23T21:07:39.959 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:07:39.959 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:39.959 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:39.959 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:07:39.959 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:07:39.942643+0000", 2023-09-23T21:07:39.960 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T21:07:39.960 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:39.960 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:39.960 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:07:39.960 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:07:39.942650+0000", 2023-09-23T21:07:39.960 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.1049999999999997e-06 2023-09-23T21:07:39.960 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:39.960 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:39.960 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:07:39.960 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:07:39.942667+0000", 2023-09-23T21:07:39.960 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.6864e-05 2023-09-23T21:07:39.960 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:39.961 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:39.961 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:07:39.961 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:07:39.942675+0000", 2023-09-23T21:07:39.961 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.9200000000000004e-06 2023-09-23T21:07:39.961 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:39.961 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:39.961 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:07:39.961 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:07:39.942829+0000", 2023-09-23T21:07:39.961 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00015426200000000001 2023-09-23T21:07:39.961 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:39.961 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:39.961 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:07:39.961 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:07:39.942955+0000", 2023-09-23T21:07:39.962 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000126341 2023-09-23T21:07:39.962 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:39.962 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:39.962 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 5,7", 2023-09-23T21:07:39.962 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:07:39.943055+0000", 2023-09-23T21:07:39.962 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000100253 2023-09-23T21:07:39.962 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:07:39.962 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:07:39.962 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:07:39.962 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:39.962 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:39.962 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35856.0:124136 9.9 9.d4bdacd9 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e370)", 2023-09-23T21:07:39.963 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T21:07:39.942690+0000", 2023-09-23T21:07:39.963 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 0.0017010199999999999, 2023-09-23T21:07:39.963 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0019515680000000001, 2023-09-23T21:07:39.963 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:07:39.963 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "queued for pg", 2023-09-23T21:07:39.963 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T21:07:39.963 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35856", 2023-09-23T21:07:39.963 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/4180396258", 2023-09-23T21:07:39.963 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 124136 2023-09-23T21:07:39.963 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:39.963 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:07:39.963 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:39.963 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:07:39.964 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:07:39.942690+0000", 2023-09-23T21:07:39.964 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:07:39.964 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:39.964 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:39.964 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:07:39.964 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:07:39.942690+0000", 2023-09-23T21:07:39.964 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:07:39.964 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:39.964 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:39.964 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:07:39.964 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:07:39.942689+0000", 2023-09-23T21:07:39.964 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T21:07:39.965 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:39.965 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:39.965 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:07:39.965 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:07:39.942693+0000", 2023-09-23T21:07:39.965 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.5590000000000001e-06 2023-09-23T21:07:39.965 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:39.965 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:39.965 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:07:39.965 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:07:39.942700+0000", 2023-09-23T21:07:39.965 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.2760000000000003e-06 2023-09-23T21:07:39.965 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:39.965 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:39.966 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:07:39.966 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:07:39.942705+0000", 2023-09-23T21:07:39.966 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.3430000000000003e-06 2023-09-23T21:07:39.966 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:07:39.966 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:07:39.966 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:07:39.966 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:39.966 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:39.966 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_repop_reply(client.35856.0:124118 9.5 e370/346)", 2023-09-23T21:07:39.966 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T21:07:39.942719+0000", 2023-09-23T21:07:39.966 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 0.0016718309999999999, 2023-09-23T21:07:39.966 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.001974979, 2023-09-23T21:07:39.967 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:07:39.967 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "queued for pg", 2023-09-23T21:07:39.967 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:07:39.967 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:39.967 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:07:39.967 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:07:39.942719+0000", 2023-09-23T21:07:39.967 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:07:39.967 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:39.967 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:39.967 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:07:39.967 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:07:39.942719+0000", 2023-09-23T21:07:39.967 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:07:39.967 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:39.968 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:39.968 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:07:39.968 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:07:39.942718+0000", 2023-09-23T21:07:39.968 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T21:07:39.968 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:39.968 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:39.968 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:07:39.968 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:07:39.942722+0000", 2023-09-23T21:07:39.968 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.5420000000000001e-06 2023-09-23T21:07:39.968 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:39.968 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:39.968 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:07:39.969 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:07:39.942729+0000", 2023-09-23T21:07:39.969 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.7939999999999997e-06 2023-09-23T21:07:39.969 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:39.969 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:39.969 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:07:39.969 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:07:39.942733+0000", 2023-09-23T21:07:39.969 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.6460000000000001e-06 2023-09-23T21:07:39.969 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:07:39.969 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:07:39.969 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:07:39.969 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:39.969 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:39.970 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35856.0:124116 9.5 9:a8016bed:::benchmark_data_smithi026_342732_object10840:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e370)", 2023-09-23T21:07:39.970 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T21:07:39.923930+0000", 2023-09-23T21:07:39.970 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 0.020460058, 2023-09-23T21:07:39.970 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.020814539999999999, 2023-09-23T21:07:39.970 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:07:39.970 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "waiting for sub ops", 2023-09-23T21:07:39.970 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T21:07:39.970 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35856", 2023-09-23T21:07:39.970 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/4180396258", 2023-09-23T21:07:39.970 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 124116 2023-09-23T21:07:39.970 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:39.970 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:07:39.970 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:39.971 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:07:39.971 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:07:39.923930+0000", 2023-09-23T21:07:39.971 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:07:39.971 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:39.971 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:39.971 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:07:39.971 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:07:39.923930+0000", 2023-09-23T21:07:39.971 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:07:39.971 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:39.971 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:39.971 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:07:39.971 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:07:39.923929+0000", 2023-09-23T21:07:39.971 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T21:07:39.972 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:39.972 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:39.972 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:07:39.972 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:07:39.923936+0000", 2023-09-23T21:07:39.972 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.5599999999999999e-06 2023-09-23T21:07:39.972 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:39.972 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:39.972 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:07:39.972 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:07:39.923952+0000", 2023-09-23T21:07:39.972 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.6090999999999999e-05 2023-09-23T21:07:39.972 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:39.972 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:39.972 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:07:39.973 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:07:39.923960+0000", 2023-09-23T21:07:39.973 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.4319999999999999e-06 2023-09-23T21:07:39.973 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:39.973 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:39.973 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:07:39.973 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:07:39.926993+0000", 2023-09-23T21:07:39.973 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0030327700000000002 2023-09-23T21:07:39.973 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:39.973 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:39.973 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:07:39.973 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:07:39.927127+0000", 2023-09-23T21:07:39.973 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000134268 2023-09-23T21:07:39.974 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:39.974 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:39.974 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 5,7", 2023-09-23T21:07:39.974 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:07:39.932250+0000", 2023-09-23T21:07:39.974 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.005122199 2023-09-23T21:07:39.974 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:39.974 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:39.974 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T21:07:39.974 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:07:39.940126+0000", 2023-09-23T21:07:39.974 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0078763310000000003 2023-09-23T21:07:39.974 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:39.974 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:39.974 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:07:39.975 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:07:39.941093+0000", 2023-09-23T21:07:39.975 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00096678900000000002 2023-09-23T21:07:39.975 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:07:39.975 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:07:39.975 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:07:39.975 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:07:39.975 INFO:teuthology.orchestra.run.smithi026.stdout: ], 2023-09-23T21:07:39.975 INFO:teuthology.orchestra.run.smithi026.stdout: "num_ops": 6 2023-09-23T21:07:39.975 INFO:teuthology.orchestra.run.smithi026.stdout:} 2023-09-23T21:07:40.014 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:07:40.022 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:07:40.022 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:07:40.123 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:07:40.193 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:07:40.318 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:07:39 smithi026 ceph-mon[175151]: failed to encode map e370 with expected crc 2023-09-23T21:07:40.318 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:07:39 smithi026 ceph-mon[175151]: failed to encode map e370 with expected crc 2023-09-23T21:07:40.318 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:07:39 smithi026 ceph-mon[175151]: pgmap v965: 109 pgs: 109 active+clean; 6.4 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 104 KiB/s wr, 839 op/s 2023-09-23T21:07:40.323 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:07:39 smithi026 ceph-mon[179688]: failed to encode map e370 with expected crc 2023-09-23T21:07:40.323 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:07:39 smithi026 ceph-mon[179688]: failed to encode map e370 with expected crc 2023-09-23T21:07:40.323 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:07:39 smithi026 ceph-mon[179688]: pgmap v965: 109 pgs: 109 active+clean; 6.4 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 104 KiB/s wr, 839 op/s 2023-09-23T21:07:40.338 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T21:07:40.351 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:07:40.352 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:07:40.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:07:39 smithi070 ceph-mon[168282]: failed to encode map e370 with expected crc 2023-09-23T21:07:40.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:07:39 smithi070 ceph-mon[168282]: failed to encode map e370 with expected crc 2023-09-23T21:07:40.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:07:39 smithi070 ceph-mon[168282]: pgmap v965: 109 pgs: 109 active+clean; 6.4 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 104 KiB/s wr, 839 op/s 2023-09-23T21:07:40.453 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:07:40.493 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:07:40.548 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:07:40.554 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:07:40.555 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:07:40.655 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:07:40.687 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:07:40.731 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:07:40.735 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:07:40.735 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:07:40.775 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0: {} 2023-09-23T21:07:40.775 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1: {} 2023-09-23T21:07:40.775 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2: {} 2023-09-23T21:07:40.775 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3: {} 2023-09-23T21:07:40.775 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4: {} 2023-09-23T21:07:40.776 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T21:07:40.776 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6: {} 2023-09-23T21:07:40.776 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T21:07:40.776 INFO:teuthology.orchestra.run.smithi026.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T21:07:40.776 INFO:teuthology.orchestra.run.smithi026.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T21:07:40.776 INFO:teuthology.orchestra.run.smithi026.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T21:07:40.776 INFO:teuthology.orchestra.run.smithi026.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T21:07:40.776 INFO:teuthology.orchestra.run.smithi026.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T21:07:40.776 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T21:07:40.776 INFO:teuthology.orchestra.run.smithi026.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T21:07:40.777 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T21:07:40.777 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_blocked_ops 2023-09-23T21:07:40.836 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:07:40.887 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:07:40.944 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:07:41.133 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:07:41.141 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:07:41.142 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:07:41.242 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:07:41.300 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:07:41.349 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:07:41.350 INFO:teuthology.orchestra.run.smithi026.stderr:nodeep-scrub is set 2023-09-23T21:07:41.356 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:07:41.356 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:07:41.423 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:07:41 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/3563954018' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:07:41.423 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:07:41 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/2557142307' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:07:41.423 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:07:41 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/3563954018' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:07:41.423 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:07:41 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/2557142307' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:07:41.457 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:07:41.494 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:07:41.562 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:07:41.568 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:07:41.568 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:07:41.573 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:07:41 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/3563954018' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:07:41.573 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:07:41 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/2557142307' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:07:41.669 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:07:41.712 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:07:41.770 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:07:41.774 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:07:41.775 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:07:41.876 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:07:41.946 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:07:42.034 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:07:42.044 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:07:42.044 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:07:42.145 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:07:42.200 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:07:42.264 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:07:42.270 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:07:42.271 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:07:42.328 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:07:42 smithi026 ceph-mon[175151]: pgmap v966: 109 pgs: 109 active+clean; 6.4 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 807 op/s 2023-09-23T21:07:42.328 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:07:42 smithi026 ceph-mon[175151]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:07:42.328 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:07:42 smithi026 ceph-mon[175151]: failed to encode map e371 with expected crc 2023-09-23T21:07:42.328 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:07:42 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/2557142307' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T21:07:42.328 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:07:42 smithi026 ceph-mon[175151]: osdmap e371: 8 total, 8 up, 8 in 2023-09-23T21:07:42.328 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:07:42 smithi026 ceph-mon[175151]: failed to encode map e371 with expected crc 2023-09-23T21:07:42.328 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:07:42 smithi026 ceph-mon[175151]: failed to encode map e371 with expected crc 2023-09-23T21:07:42.372 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:07:42.407 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:07:42.424 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:07:42.457 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:07:42.467 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:07:42.468 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:07:42.468 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:07:42 smithi070 ceph-mon[168282]: pgmap v966: 109 pgs: 109 active+clean; 6.4 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 807 op/s 2023-09-23T21:07:42.468 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:07:42 smithi070 ceph-mon[168282]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:07:42.468 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:07:42 smithi070 ceph-mon[168282]: failed to encode map e371 with expected crc 2023-09-23T21:07:42.469 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:07:42 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/2557142307' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T21:07:42.469 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:07:42 smithi070 ceph-mon[168282]: osdmap e371: 8 total, 8 up, 8 in 2023-09-23T21:07:42.469 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:07:42 smithi070 ceph-mon[168282]: failed to encode map e371 with expected crc 2023-09-23T21:07:42.469 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:07:42 smithi070 ceph-mon[168282]: failed to encode map e371 with expected crc 2023-09-23T21:07:42.568 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:07:42.638 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:07:42 smithi026 ceph-mon[179688]: pgmap v966: 109 pgs: 109 active+clean; 6.4 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 807 op/s 2023-09-23T21:07:42.638 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:07:42 smithi026 ceph-mon[179688]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:07:42.638 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:07:42 smithi026 ceph-mon[179688]: failed to encode map e371 with expected crc 2023-09-23T21:07:42.638 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:07:42 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/2557142307' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T21:07:42.638 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:07:42 smithi026 ceph-mon[179688]: osdmap e371: 8 total, 8 up, 8 in 2023-09-23T21:07:42.638 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:07:42 smithi026 ceph-mon[179688]: failed to encode map e371 with expected crc 2023-09-23T21:07:42.639 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:07:42 smithi026 ceph-mon[179688]: failed to encode map e371 with expected crc 2023-09-23T21:07:42.658 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:07:42.731 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:07:42.742 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:07:42.742 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:07:42.843 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:07:42.900 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:07:42.969 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:07:42.971 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:07:42.972 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:07:43.072 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:07:43.144 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:07:43.294 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T21:07:43.442 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:07:43.453 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:07:43.454 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:07:43.555 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:07:43.586 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:07:43.623 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:07:43.627 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:07:43.627 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:07:43.728 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:07:43.806 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:07:43.966 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T21:07:44.357 INFO:teuthology.orchestra.run.smithi026.stderr:reweighted osd.4 to 1 (10000) 2023-09-23T21:07:44.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:07:44 smithi070 ceph-mon[168282]: pgmap v968: 109 pgs: 109 active+clean; 6.2 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 734 op/s 2023-09-23T21:07:44.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:07:44 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/3621643792' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-09-23T21:07:44.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:07:44 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-09-23T21:07:44.631 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:07:44 smithi026 ceph-mon[179688]: pgmap v968: 109 pgs: 109 active+clean; 6.2 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 734 op/s 2023-09-23T21:07:44.631 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:07:44 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/3621643792' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-09-23T21:07:44.631 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:07:44 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-09-23T21:07:44.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:07:44 smithi026 ceph-mon[175151]: pgmap v968: 109 pgs: 109 active+clean; 6.2 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 734 op/s 2023-09-23T21:07:44.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:07:44 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/3621643792' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-09-23T21:07:44.633 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:07:44 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-09-23T21:07:45.097 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:07:45.101 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:07:45.101 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:07:45.202 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:07:45.320 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:07:45.383 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:07:45 smithi026 ceph-mon[175151]: failed to encode map e372 with expected crc 2023-09-23T21:07:45.383 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:07:45 smithi026 ceph-mon[175151]: failed to encode map e372 with expected crc 2023-09-23T21:07:45.383 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:07:45 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-09-23T21:07:45.383 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:07:45 smithi026 ceph-mon[175151]: osdmap e372: 8 total, 8 up, 8 in 2023-09-23T21:07:45.384 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:07:45 smithi026 ceph-mon[175151]: failed to encode map e372 with expected crc 2023-09-23T21:07:45.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:07:45 smithi070 ceph-mon[168282]: failed to encode map e372 with expected crc 2023-09-23T21:07:45.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:07:45 smithi070 ceph-mon[168282]: failed to encode map e372 with expected crc 2023-09-23T21:07:45.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:07:45 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-09-23T21:07:45.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:07:45 smithi070 ceph-mon[168282]: osdmap e372: 8 total, 8 up, 8 in 2023-09-23T21:07:45.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:07:45 smithi070 ceph-mon[168282]: failed to encode map e372 with expected crc 2023-09-23T21:07:45.881 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:07:45 smithi026 ceph-mon[179688]: failed to encode map e372 with expected crc 2023-09-23T21:07:45.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:07:45 smithi026 ceph-mon[179688]: failed to encode map e372 with expected crc 2023-09-23T21:07:45.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:07:45 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-09-23T21:07:45.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:07:45 smithi026 ceph-mon[179688]: osdmap e372: 8 total, 8 up, 8 in 2023-09-23T21:07:45.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:07:45 smithi026 ceph-mon[179688]: failed to encode map e372 with expected crc 2023-09-23T21:07:46.010 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:07:46.025 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:07:46.026 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:07:46.126 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:07:46.189 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:07:46.241 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:07:46.245 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:07:46.246 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:07:46.347 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:07:46.419 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:07:46.480 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:07:46.487 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:07:46.488 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:07:46.573 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph osd reweight 5 1 2023-09-23T21:07:46.589 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:07:46.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:07:46 smithi070 ceph-mon[168282]: pgmap v970: 109 pgs: 109 active+clean; 6.1 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 823 op/s 2023-09-23T21:07:46.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:07:46 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:07:46.630 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:07:46.644 INFO:teuthology.orchestra.run.smithi026.stdout:ERROR: (22) Invalid argument 2023-09-23T21:07:46.644 INFO:teuthology.orchestra.run.smithi026.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T21:07:46.647 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:07:46 smithi026 ceph-mon[175151]: pgmap v970: 109 pgs: 109 active+clean; 6.1 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 823 op/s 2023-09-23T21:07:46.648 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:07:46 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:07:46.649 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:07:46 smithi026 ceph-mon[179688]: pgmap v970: 109 pgs: 109 active+clean; 6.1 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 823 op/s 2023-09-23T21:07:46.649 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:07:46 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:07:46.672 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:07:46.678 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:07:46.679 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:07:46.779 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:07:46.819 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:07:46.880 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:07:46.886 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:07:46.887 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:07:46.987 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:07:47.070 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_historic_ops 2023-09-23T21:07:47.078 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:07:47.088 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:07:47.157 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T21:07:47.168 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:07:47.169 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:07:47.174 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:07:47.270 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:07:47.415 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:07:47.782 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:07:47.790 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:07:47.791 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:07:47.801 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:07:47.891 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:07:47.963 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:07:48.209 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:07:48.219 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:07:48.219 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:07:48.320 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:07:48.357 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:07:48.412 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:07:48.416 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:07:48.417 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:07:48.517 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:07:48.550 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:07:48 smithi026 ceph-mon[179688]: pgmap v971: 109 pgs: 109 active+clean; 6.1 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 686 op/s 2023-09-23T21:07:48.550 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:07:48 smithi026 ceph-mon[175151]: pgmap v971: 109 pgs: 109 active+clean; 6.1 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 686 op/s 2023-09-23T21:07:48.621 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:07:48 smithi070 ceph-mon[168282]: pgmap v971: 109 pgs: 109 active+clean; 6.1 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 686 op/s 2023-09-23T21:07:48.623 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:07:48.733 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:07:48.736 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:07:48.737 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:07:48.838 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:07:48.926 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:07:49.014 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:07:49.106 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:07:49.114 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:07:49.114 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:07:49.215 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:07:49.260 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:07:49.308 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:07:49.313 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:07:49.314 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:07:49.320 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0: {} 2023-09-23T21:07:49.362 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1: {} 2023-09-23T21:07:49.401 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2: {} 2023-09-23T21:07:49.415 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:07:49.458 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3: {} 2023-09-23T21:07:49.496 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4: {} 2023-09-23T21:07:49.510 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:07:49.555 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T21:07:49.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:07:49 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/1681954314' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:07:49.620 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6: {} 2023-09-23T21:07:49.668 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 21:07:49 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[164029]: ::ffff:172.21.15.70 - - [23/Sep/2023:21:07:49] "GET /metrics HTTP/1.1" 200 36298 "" "Prometheus/2.43.0" 2023-09-23T21:07:49.668 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:07:49 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/1681954314' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:07:49.669 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:07:49 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/1681954314' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:07:49.669 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T21:07:49.680 INFO:teuthology.orchestra.run.smithi026.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T21:07:49.680 INFO:teuthology.orchestra.run.smithi026.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T21:07:49.680 INFO:teuthology.orchestra.run.smithi026.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T21:07:49.680 INFO:teuthology.orchestra.run.smithi026.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T21:07:49.680 INFO:teuthology.orchestra.run.smithi026.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T21:07:49.680 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T21:07:49.680 INFO:teuthology.orchestra.run.smithi026.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T21:07:49.681 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T21:07:50.023 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:07:50.038 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:07:50.038 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:07:50.139 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:07:50.184 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:07:50.242 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:07:50.247 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:07:50.247 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:07:50.348 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:07:50.455 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:07:50.457 INFO:teuthology.orchestra.run.smithi026.stderr:noscrub is unset 2023-09-23T21:07:50.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:07:50 smithi070 ceph-mon[168282]: pgmap v972: 109 pgs: 109 active+clean; 5.9 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 876 op/s 2023-09-23T21:07:50.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:07:50 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/1075331713' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T21:07:50.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:07:50 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T21:07:50.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:07:50 smithi026 ceph-mon[175151]: pgmap v972: 109 pgs: 109 active+clean; 5.9 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 876 op/s 2023-09-23T21:07:50.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:07:50 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/1075331713' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T21:07:50.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:07:50 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T21:07:50.633 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:07:50 smithi026 ceph-mon[179688]: pgmap v972: 109 pgs: 109 active+clean; 5.9 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 876 op/s 2023-09-23T21:07:50.633 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:07:50 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/1075331713' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T21:07:50.633 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:07:50 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T21:07:50.898 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:07:50.905 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:07:50.905 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:07:51.006 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:07:51.064 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:07:51.127 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:07:51.133 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:07:51.134 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:07:51.234 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:07:51.273 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:07:51.325 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:07:51.328 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:07:51.328 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:07:51.429 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:07:51.449 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:07:51 smithi026 ceph-mon[175151]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:07:51.449 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:07:51 smithi026 ceph-mon[175151]: failed to encode map e373 with expected crc 2023-09-23T21:07:51.449 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:07:51 smithi026 ceph-mon[175151]: failed to encode map e373 with expected crc 2023-09-23T21:07:51.449 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:07:51 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T21:07:51.449 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:07:51 smithi026 ceph-mon[175151]: osdmap e373: 8 total, 8 up, 8 in 2023-09-23T21:07:51.449 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:07:51 smithi026 ceph-mon[175151]: failed to encode map e373 with expected crc 2023-09-23T21:07:51.450 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:07:51 smithi026 ceph-mon[175151]: failed to encode map e373 with expected crc 2023-09-23T21:07:51.450 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:07:51 smithi026 ceph-mon[175151]: failed to encode map e373 with expected crc 2023-09-23T21:07:51.450 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:07:51 smithi026 ceph-mon[175151]: failed to encode map e373 with expected crc 2023-09-23T21:07:51.450 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:07:51 smithi026 ceph-mon[179688]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:07:51.450 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:07:51 smithi026 ceph-mon[179688]: failed to encode map e373 with expected crc 2023-09-23T21:07:51.450 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:07:51 smithi026 ceph-mon[179688]: failed to encode map e373 with expected crc 2023-09-23T21:07:51.450 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:07:51 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T21:07:51.450 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:07:51 smithi026 ceph-mon[179688]: osdmap e373: 8 total, 8 up, 8 in 2023-09-23T21:07:51.450 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:07:51 smithi026 ceph-mon[179688]: failed to encode map e373 with expected crc 2023-09-23T21:07:51.450 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:07:51 smithi026 ceph-mon[179688]: failed to encode map e373 with expected crc 2023-09-23T21:07:51.451 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:07:51 smithi026 ceph-mon[179688]: failed to encode map e373 with expected crc 2023-09-23T21:07:51.451 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:07:51 smithi026 ceph-mon[179688]: failed to encode map e373 with expected crc 2023-09-23T21:07:51.504 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:07:51.568 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:07:51.583 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:07:51.584 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:07:51.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:07:51 smithi070 ceph-mon[168282]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:07:51.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:07:51 smithi070 ceph-mon[168282]: failed to encode map e373 with expected crc 2023-09-23T21:07:51.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:07:51 smithi070 ceph-mon[168282]: failed to encode map e373 with expected crc 2023-09-23T21:07:51.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:07:51 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T21:07:51.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:07:51 smithi070 ceph-mon[168282]: osdmap e373: 8 total, 8 up, 8 in 2023-09-23T21:07:51.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:07:51 smithi070 ceph-mon[168282]: failed to encode map e373 with expected crc 2023-09-23T21:07:51.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:07:51 smithi070 ceph-mon[168282]: failed to encode map e373 with expected crc 2023-09-23T21:07:51.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:07:51 smithi070 ceph-mon[168282]: failed to encode map e373 with expected crc 2023-09-23T21:07:51.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:07:51 smithi070 ceph-mon[168282]: failed to encode map e373 with expected crc 2023-09-23T21:07:51.685 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:07:51.766 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:07:51.851 INFO:teuthology.orchestra.run.smithi026.stdout:{ 2023-09-23T21:07:51.851 INFO:teuthology.orchestra.run.smithi026.stdout: "size": 20, 2023-09-23T21:07:51.852 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 600, 2023-09-23T21:07:51.852 INFO:teuthology.orchestra.run.smithi026.stdout: "ops": [ 2023-09-23T21:07:51.852 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:51.852 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35469.0:141052 8.11 8:89489a5f:::benchmark_data_smithi026_301269_object30295:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e287)", 2023-09-23T21:07:51.852 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T21:03:40.177394+0000", 2023-09-23T21:07:51.852 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 251.67092255200001, 2023-09-23T21:07:51.852 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1791958380000001, 2023-09-23T21:07:51.852 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:07:51.852 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:07:51.852 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T21:07:51.852 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35469", 2023-09-23T21:07:51.853 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/4041424623", 2023-09-23T21:07:51.853 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 141052 2023-09-23T21:07:51.853 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:51.853 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:07:51.853 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:51.853 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:07:51.853 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:03:40.177394+0000", 2023-09-23T21:07:51.853 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:07:51.853 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:51.853 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:51.853 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:07:51.853 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:03:40.177394+0000", 2023-09-23T21:07:51.853 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:07:51.854 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:51.854 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:51.854 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:07:51.854 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:03:40.177394+0000", 2023-09-23T21:07:51.854 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T21:07:51.854 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:51.854 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:51.854 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:07:51.854 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:03:40.177396+0000", 2023-09-23T21:07:51.854 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.5110000000000002e-06 2023-09-23T21:07:51.854 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:51.854 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:51.855 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:07:51.855 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:03:40.177403+0000", 2023-09-23T21:07:51.855 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.3829999999999997e-06 2023-09-23T21:07:51.855 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:51.855 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:51.855 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:07:51.855 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:03:40.177412+0000", 2023-09-23T21:07:51.855 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.3030000000000005e-06 2023-09-23T21:07:51.855 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:51.855 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:51.855 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:07:51.855 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:03:40.177607+0000", 2023-09-23T21:07:51.855 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00019547299999999999 2023-09-23T21:07:51.856 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:51.856 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:51.856 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T21:07:51.856 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:03:40.177612+0000", 2023-09-23T21:07:51.856 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.1919999999999998e-06 2023-09-23T21:07:51.856 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:51.856 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:51.856 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:07:51.856 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:03:41.339975+0000", 2023-09-23T21:07:51.856 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1623630839999999 2023-09-23T21:07:51.856 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:51.856 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:51.856 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:07:51.857 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:03:41.340055+0000", 2023-09-23T21:07:51.857 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.0049000000000006e-05 2023-09-23T21:07:51.857 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:51.857 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:51.857 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 1,2,5,6,7", 2023-09-23T21:07:51.857 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:03:41.340123+0000", 2023-09-23T21:07:51.857 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.8595000000000003e-05 2023-09-23T21:07:51.857 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:51.857 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:51.857 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:07:51.857 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:03:41.344452+0000", 2023-09-23T21:07:51.857 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0043291099999999997 2023-09-23T21:07:51.858 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:51.858 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:51.858 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T21:07:51.858 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:03:41.344463+0000", 2023-09-23T21:07:51.858 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1041e-05 2023-09-23T21:07:51.858 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:51.858 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:51.858 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:07:51.858 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:03:41.348591+0000", 2023-09-23T21:07:51.858 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0041270120000000002 2023-09-23T21:07:51.858 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:51.858 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:51.858 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:07:51.859 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:03:41.351266+0000", 2023-09-23T21:07:51.859 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0026757199999999999 2023-09-23T21:07:51.859 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:51.859 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:51.859 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:07:51.859 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:03:41.352296+0000", 2023-09-23T21:07:51.859 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.001029664 2023-09-23T21:07:51.859 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:51.859 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:51.859 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:07:51.859 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:03:41.356420+0000", 2023-09-23T21:07:51.859 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0041241799999999999 2023-09-23T21:07:51.860 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:51.860 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:51.860 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T21:07:51.860 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:03:41.356451+0000", 2023-09-23T21:07:51.860 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.1276999999999999e-05 2023-09-23T21:07:51.860 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:51.860 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:51.860 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T21:07:51.860 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:03:41.356590+0000", 2023-09-23T21:07:51.860 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000138622 2023-09-23T21:07:51.860 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:07:51.860 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:07:51.860 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:07:51.861 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:51.861 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:51.861 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35469.0:141053 8.11 8:8ac65d6e:::benchmark_data_smithi026_301269_object30296:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e287)", 2023-09-23T21:07:51.861 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T21:03:40.177417+0000", 2023-09-23T21:07:51.861 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 251.670899249, 2023-09-23T21:07:51.861 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1794388659999999, 2023-09-23T21:07:51.861 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:07:51.861 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:07:51.861 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T21:07:51.861 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35469", 2023-09-23T21:07:51.861 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/4041424623", 2023-09-23T21:07:51.861 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 141053 2023-09-23T21:07:51.862 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:51.862 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:07:51.862 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:51.862 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:07:51.862 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:03:40.177417+0000", 2023-09-23T21:07:51.862 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:07:51.862 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:51.862 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:51.862 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:07:51.862 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:03:40.177417+0000", 2023-09-23T21:07:51.862 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:07:51.862 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:51.862 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:51.863 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:07:51.863 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:03:40.177417+0000", 2023-09-23T21:07:51.863 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T21:07:51.863 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:51.863 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:51.863 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:07:51.863 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:03:40.177420+0000", 2023-09-23T21:07:51.863 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.5110000000000002e-06 2023-09-23T21:07:51.863 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:51.863 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:51.863 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:07:51.864 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:03:40.177432+0000", 2023-09-23T21:07:51.864 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.2452e-05 2023-09-23T21:07:51.864 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:51.864 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:51.864 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:07:51.864 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:03:40.177438+0000", 2023-09-23T21:07:51.864 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.9989999999999997e-06 2023-09-23T21:07:51.864 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:51.864 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:51.864 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:07:51.864 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:03:40.177634+0000", 2023-09-23T21:07:51.864 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000195997 2023-09-23T21:07:51.864 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:51.865 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:51.865 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T21:07:51.865 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:03:40.177638+0000", 2023-09-23T21:07:51.865 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.8139999999999998e-06 2023-09-23T21:07:51.865 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:51.865 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:51.865 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:07:51.865 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:03:41.341453+0000", 2023-09-23T21:07:51.865 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.163815233 2023-09-23T21:07:51.866 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:51.866 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:51.866 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:07:51.866 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:03:41.341574+0000", 2023-09-23T21:07:51.866 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000121192 2023-09-23T21:07:51.866 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:51.866 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:51.866 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 1,2,5,6,7", 2023-09-23T21:07:51.866 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:03:41.341694+0000", 2023-09-23T21:07:51.866 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00011974800000000001 2023-09-23T21:07:51.866 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:51.867 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:51.867 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T21:07:51.867 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:03:41.344484+0000", 2023-09-23T21:07:51.867 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.002790276 2023-09-23T21:07:51.867 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:51.867 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:51.867 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:07:51.867 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:03:41.344511+0000", 2023-09-23T21:07:51.867 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.6458e-05 2023-09-23T21:07:51.867 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:51.868 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:51.868 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:07:51.868 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:03:41.348631+0000", 2023-09-23T21:07:51.868 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0041203580000000002 2023-09-23T21:07:51.868 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:51.868 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:51.868 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:07:51.868 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:03:41.352333+0000", 2023-09-23T21:07:51.868 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0037016140000000002 2023-09-23T21:07:51.868 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:51.868 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:51.868 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:07:51.869 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:03:41.353086+0000", 2023-09-23T21:07:51.869 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00075337099999999997 2023-09-23T21:07:51.869 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:51.869 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:51.869 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:07:51.869 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:03:41.356654+0000", 2023-09-23T21:07:51.869 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0035683809999999998 2023-09-23T21:07:51.869 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:51.869 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:51.869 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T21:07:51.869 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:03:41.356715+0000", 2023-09-23T21:07:51.870 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.0259e-05 2023-09-23T21:07:51.870 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:51.870 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:51.870 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T21:07:51.870 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:03:41.356856+0000", 2023-09-23T21:07:51.870 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000141636 2023-09-23T21:07:51.870 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:07:51.870 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:07:51.870 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:07:51.870 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:51.870 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:51.871 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35469.0:141120 8.11 8:8b730a96:::benchmark_data_smithi026_301269_object30363:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e287)", 2023-09-23T21:07:51.871 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T21:03:40.177443+0000", 2023-09-23T21:07:51.871 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 251.67087402600001, 2023-09-23T21:07:51.871 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1796452230000001, 2023-09-23T21:07:51.871 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:07:51.871 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:07:51.871 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T21:07:51.871 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35469", 2023-09-23T21:07:51.871 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/4041424623", 2023-09-23T21:07:51.872 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 141120 2023-09-23T21:07:51.872 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:51.872 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:07:51.872 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:51.872 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:07:51.872 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:03:40.177443+0000", 2023-09-23T21:07:51.872 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:07:51.872 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:51.872 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:51.872 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:07:51.872 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:03:40.177443+0000", 2023-09-23T21:07:51.872 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:07:51.873 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:51.873 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:51.873 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:07:51.873 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:03:40.177442+0000", 2023-09-23T21:07:51.873 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T21:07:51.873 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:51.873 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:51.873 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:07:51.873 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:03:40.177444+0000", 2023-09-23T21:07:51.873 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.6449999999999999e-06 2023-09-23T21:07:51.873 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:51.873 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:51.874 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:07:51.874 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:03:40.177448+0000", 2023-09-23T21:07:51.874 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.602e-06 2023-09-23T21:07:51.874 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:51.874 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:51.874 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:07:51.874 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:03:40.177451+0000", 2023-09-23T21:07:51.874 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.9610000000000001e-06 2023-09-23T21:07:51.874 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:51.874 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:51.874 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:07:51.874 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:03:40.177661+0000", 2023-09-23T21:07:51.874 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000209984 2023-09-23T21:07:51.875 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:51.875 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:51.875 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T21:07:51.875 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:03:40.177665+0000", 2023-09-23T21:07:51.875 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.2189999999999998e-06 2023-09-23T21:07:51.875 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:51.875 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:51.875 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:07:51.875 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:03:41.344534+0000", 2023-09-23T21:07:51.875 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.16686954 2023-09-23T21:07:51.875 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:51.876 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:51.876 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:07:51.876 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:03:41.344663+0000", 2023-09-23T21:07:51.876 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000128441 2023-09-23T21:07:51.876 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:51.876 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:51.876 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 1,2,5,6,7", 2023-09-23T21:07:51.876 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:03:41.344796+0000", 2023-09-23T21:07:51.876 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00013324499999999999 2023-09-23T21:07:51.877 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:51.877 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:51.877 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T21:07:51.877 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:03:41.348599+0000", 2023-09-23T21:07:51.877 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0038027040000000001 2023-09-23T21:07:51.877 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:51.877 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:51.877 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:07:51.877 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:03:41.348747+0000", 2023-09-23T21:07:51.877 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00014788299999999999 2023-09-23T21:07:51.878 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:51.878 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:51.878 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:07:51.878 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:03:41.352359+0000", 2023-09-23T21:07:51.878 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0036121980000000001 2023-09-23T21:07:51.878 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:51.878 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:51.878 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:07:51.879 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:03:41.353175+0000", 2023-09-23T21:07:51.879 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00081627299999999996 2023-09-23T21:07:51.879 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:51.879 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:51.879 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:07:51.879 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:03:41.354019+0000", 2023-09-23T21:07:51.879 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00084438700000000005 2023-09-23T21:07:51.879 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:51.879 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:51.880 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:07:51.880 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:03:41.356905+0000", 2023-09-23T21:07:51.880 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0028856699999999999 2023-09-23T21:07:51.880 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:51.880 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:51.880 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T21:07:51.880 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:03:41.356952+0000", 2023-09-23T21:07:51.880 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.6471999999999998e-05 2023-09-23T21:07:51.880 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:51.880 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:51.880 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T21:07:51.881 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:03:41.357088+0000", 2023-09-23T21:07:51.881 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00013626 2023-09-23T21:07:51.881 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:07:51.881 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:07:51.881 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:07:51.881 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:51.881 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:51.881 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35469.0:141157 8.11 8:88baa66d:::benchmark_data_smithi026_301269_object30400:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e287)", 2023-09-23T21:07:51.881 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T21:03:40.177463+0000", 2023-09-23T21:07:51.881 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 251.67085336100001, 2023-09-23T21:07:51.881 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.181319032, 2023-09-23T21:07:51.881 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:07:51.881 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:07:51.882 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T21:07:51.882 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35469", 2023-09-23T21:07:51.882 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/4041424623", 2023-09-23T21:07:51.882 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 141157 2023-09-23T21:07:51.882 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:51.882 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:07:51.882 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:51.882 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:07:51.882 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:03:40.177463+0000", 2023-09-23T21:07:51.882 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:07:51.882 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:51.882 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:51.883 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:07:51.883 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:03:40.177463+0000", 2023-09-23T21:07:51.883 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:07:51.883 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:51.883 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:51.883 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:07:51.883 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:03:40.177463+0000", 2023-09-23T21:07:51.883 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T21:07:51.883 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:51.883 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:51.883 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:07:51.883 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:03:40.177465+0000", 2023-09-23T21:07:51.883 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.3539999999999998e-06 2023-09-23T21:07:51.884 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:51.884 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:51.884 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:07:51.884 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:03:40.177471+0000", 2023-09-23T21:07:51.884 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.1149999999999999e-06 2023-09-23T21:07:51.884 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:51.884 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:51.884 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:07:51.884 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:03:40.177482+0000", 2023-09-23T21:07:51.884 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0020999999999999e-05 2023-09-23T21:07:51.884 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:51.884 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:51.885 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:07:51.885 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:03:40.177677+0000", 2023-09-23T21:07:51.885 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00019567099999999999 2023-09-23T21:07:51.885 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:51.885 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:51.885 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T21:07:51.885 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:03:40.177681+0000", 2023-09-23T21:07:51.885 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.507e-06 2023-09-23T21:07:51.885 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:51.885 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:51.885 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:07:51.886 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:03:41.346625+0000", 2023-09-23T21:07:51.886 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.168943949 2023-09-23T21:07:51.886 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:51.886 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:51.886 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:07:51.886 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:03:41.346740+0000", 2023-09-23T21:07:51.886 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000114907 2023-09-23T21:07:51.886 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:51.886 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:51.886 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 1,2,5,6,7", 2023-09-23T21:07:51.886 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:03:41.346855+0000", 2023-09-23T21:07:51.886 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000115623 2023-09-23T21:07:51.887 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:51.887 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:51.887 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:07:51.887 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:03:41.351219+0000", 2023-09-23T21:07:51.887 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0043639450000000002 2023-09-23T21:07:51.887 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:51.887 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:51.887 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T21:07:51.887 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:03:41.351277+0000", 2023-09-23T21:07:51.887 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.8063000000000003e-05 2023-09-23T21:07:51.887 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:51.887 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:51.887 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:07:51.888 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:03:41.352384+0000", 2023-09-23T21:07:51.888 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.001107308 2023-09-23T21:07:51.888 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:51.888 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:51.888 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:07:51.888 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:03:41.353975+0000", 2023-09-23T21:07:51.888 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.001590649 2023-09-23T21:07:51.888 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:51.888 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:51.888 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:07:51.888 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:03:41.354058+0000", 2023-09-23T21:07:51.888 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.3090000000000006e-05 2023-09-23T21:07:51.888 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:51.889 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:51.889 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:07:51.889 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:03:41.358570+0000", 2023-09-23T21:07:51.889 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0045119549999999998 2023-09-23T21:07:51.889 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:51.889 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:51.889 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T21:07:51.889 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:03:41.358608+0000", 2023-09-23T21:07:51.889 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.8098e-05 2023-09-23T21:07:51.889 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:51.889 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:51.889 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T21:07:51.890 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:03:41.358782+0000", 2023-09-23T21:07:51.890 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00017408300000000001 2023-09-23T21:07:51.890 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:07:51.890 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:07:51.890 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:07:51.890 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:51.890 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:51.890 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35856.0:28276 9.9 9:94293a39:::benchmark_data_smithi026_342732_object28275:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e351)", 2023-09-23T21:07:51.890 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T21:06:07.719170+0000", 2023-09-23T21:07:51.890 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 104.129146952, 2023-09-23T21:07:51.890 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.062500316, 2023-09-23T21:07:51.890 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:07:51.890 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:07:51.891 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T21:07:51.891 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35856", 2023-09-23T21:07:51.891 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/4180396258", 2023-09-23T21:07:51.891 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 28276 2023-09-23T21:07:51.891 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:51.891 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:07:51.891 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:51.891 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:07:51.891 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:07.719170+0000", 2023-09-23T21:07:51.891 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:07:51.891 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:51.891 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:51.892 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:07:51.892 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:07.719170+0000", 2023-09-23T21:07:51.892 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:07:51.892 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:51.892 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:51.892 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:07:51.892 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:07.719169+0000", 2023-09-23T21:07:51.892 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.999999 2023-09-23T21:07:51.892 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:51.892 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:51.892 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:07:51.892 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:07.719217+0000", 2023-09-23T21:07:51.892 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.8412000000000003e-05 2023-09-23T21:07:51.893 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:51.893 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:51.893 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:07:51.893 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:07.719228+0000", 2023-09-23T21:07:51.893 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0684e-05 2023-09-23T21:07:51.893 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:51.893 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:51.893 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:07:51.893 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:07.719235+0000", 2023-09-23T21:07:51.893 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.1300000000000003e-06 2023-09-23T21:07:51.893 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:51.893 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:51.893 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:07:51.894 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:07.719289+0000", 2023-09-23T21:07:51.894 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.3458999999999998e-05 2023-09-23T21:07:51.894 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:51.894 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:51.894 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:07:51.894 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:07.719371+0000", 2023-09-23T21:07:51.894 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.2118000000000002e-05 2023-09-23T21:07:51.894 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:51.894 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:51.894 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 5,7", 2023-09-23T21:07:51.894 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:07.719455+0000", 2023-09-23T21:07:51.894 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.4099999999999998e-05 2023-09-23T21:07:51.895 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:51.895 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:51.895 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T21:07:51.895 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:07.720430+0000", 2023-09-23T21:07:51.895 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00097545300000000004 2023-09-23T21:07:51.895 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:51.895 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:51.895 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:07:51.895 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:07.723377+0000", 2023-09-23T21:07:51.895 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0029465670000000002 2023-09-23T21:07:51.895 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:51.897 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:51.897 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:07:51.897 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:07.781584+0000", 2023-09-23T21:07:51.897 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.058206846999999999 2023-09-23T21:07:51.897 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:51.897 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:51.897 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T21:07:51.897 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:07.781632+0000", 2023-09-23T21:07:51.897 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.812e-05 2023-09-23T21:07:51.897 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:51.898 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:51.898 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T21:07:51.898 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:07.781670+0000", 2023-09-23T21:07:51.898 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.8232999999999998e-05 2023-09-23T21:07:51.898 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:07:51.898 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:07:51.898 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:07:51.898 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:51.898 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:51.898 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35856.0:28280 9.b 9:dd084843:::benchmark_data_smithi026_342732_object28279:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e351)", 2023-09-23T21:07:51.898 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T21:06:07.721758+0000", 2023-09-23T21:07:51.898 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 104.12655854499999, 2023-09-23T21:07:51.899 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.059731500999999999, 2023-09-23T21:07:51.899 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:07:51.899 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:07:51.899 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T21:07:51.899 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35856", 2023-09-23T21:07:51.899 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/4180396258", 2023-09-23T21:07:51.899 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 28280 2023-09-23T21:07:51.899 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:51.899 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:07:51.899 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:51.899 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:07:51.899 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:07.721758+0000", 2023-09-23T21:07:51.899 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:07:51.900 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:51.900 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:51.900 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:07:51.900 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:07.721758+0000", 2023-09-23T21:07:51.900 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:07:51.900 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:51.900 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:51.900 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:07:51.900 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:07.721757+0000", 2023-09-23T21:07:51.900 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.999999 2023-09-23T21:07:51.900 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:51.900 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:51.900 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:07:51.900 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:07.721781+0000", 2023-09-23T21:07:51.901 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.372e-05 2023-09-23T21:07:51.901 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:51.901 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:51.901 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:07:51.901 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:07.721791+0000", 2023-09-23T21:07:51.901 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0691e-05 2023-09-23T21:07:51.901 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:51.901 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:51.901 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:07:51.901 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:07.721799+0000", 2023-09-23T21:07:51.901 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.8509999999999999e-06 2023-09-23T21:07:51.901 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:51.902 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:51.902 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:07:51.902 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:07.721835+0000", 2023-09-23T21:07:51.902 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.5512999999999997e-05 2023-09-23T21:07:51.902 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:51.902 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:51.902 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:07:51.902 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:07.721941+0000", 2023-09-23T21:07:51.902 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00010663899999999999 2023-09-23T21:07:51.903 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:51.903 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:51.903 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 5,6", 2023-09-23T21:07:51.903 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:07.722020+0000", 2023-09-23T21:07:51.903 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.8491000000000005e-05 2023-09-23T21:07:51.903 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:51.903 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:51.903 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T21:07:51.903 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:07.723129+0000", 2023-09-23T21:07:51.903 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0011089940000000001 2023-09-23T21:07:51.903 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:51.903 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:51.904 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:07:51.904 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:07.781355+0000", 2023-09-23T21:07:51.904 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.058225701999999997 2023-09-23T21:07:51.904 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:51.904 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:51.904 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:07:51.904 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:07.781391+0000", 2023-09-23T21:07:51.904 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.6692e-05 2023-09-23T21:07:51.904 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:51.904 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:51.904 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T21:07:51.905 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:07.781443+0000", 2023-09-23T21:07:51.905 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.2096999999999999e-05 2023-09-23T21:07:51.905 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:51.905 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:51.905 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T21:07:51.905 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:07.781490+0000", 2023-09-23T21:07:51.905 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.6229000000000001e-05 2023-09-23T21:07:51.905 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:07:51.905 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:07:51.905 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:07:51.905 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:51.906 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:51.906 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35856.0:28285 9.b 9:d321f186:::benchmark_data_smithi026_342732_object28284:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e351)", 2023-09-23T21:07:51.906 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T21:06:07.723636+0000", 2023-09-23T21:07:51.906 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 104.1246809, 2023-09-23T21:07:51.906 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.059610578999999997, 2023-09-23T21:07:51.906 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:07:51.906 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:07:51.906 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T21:07:51.906 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35856", 2023-09-23T21:07:51.906 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/4180396258", 2023-09-23T21:07:51.907 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 28285 2023-09-23T21:07:51.907 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:51.907 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:07:51.907 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:51.907 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:07:51.907 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:07.723636+0000", 2023-09-23T21:07:51.907 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:07:51.907 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:51.907 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:51.907 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:07:51.907 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:07.723636+0000", 2023-09-23T21:07:51.908 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:07:51.908 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:51.908 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:51.908 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:07:51.908 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:07.723634+0000", 2023-09-23T21:07:51.908 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999981 2023-09-23T21:07:51.908 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:51.908 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:51.908 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:07:51.908 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:07.723681+0000", 2023-09-23T21:07:51.909 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.6765999999999999e-05 2023-09-23T21:07:51.909 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:51.909 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:51.909 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:07:51.909 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:07.723690+0000", 2023-09-23T21:07:51.909 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.132e-06 2023-09-23T21:07:51.909 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:51.909 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:51.909 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:07:51.909 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:07.723695+0000", 2023-09-23T21:07:51.909 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.3739999999999997e-06 2023-09-23T21:07:51.909 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:51.910 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:51.910 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:07:51.910 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:07.723735+0000", 2023-09-23T21:07:51.910 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.9388999999999998e-05 2023-09-23T21:07:51.910 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:51.910 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:51.910 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:07:51.910 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:07.723846+0000", 2023-09-23T21:07:51.910 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00011164500000000001 2023-09-23T21:07:51.911 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:51.911 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:51.911 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 5,6", 2023-09-23T21:07:51.911 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:07.780019+0000", 2023-09-23T21:07:51.911 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.056173239 2023-09-23T21:07:51.911 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:51.911 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:51.911 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:07:51.911 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:07.782806+0000", 2023-09-23T21:07:51.912 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0027863879999999999 2023-09-23T21:07:51.912 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:51.912 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:51.912 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:07:51.912 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:07.782835+0000", 2023-09-23T21:07:51.912 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.9173e-05 2023-09-23T21:07:51.912 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:51.912 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:51.912 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T21:07:51.912 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:07.783158+0000", 2023-09-23T21:07:51.912 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00032281200000000001 2023-09-23T21:07:51.913 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:51.913 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:51.913 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T21:07:51.913 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:07.783209+0000", 2023-09-23T21:07:51.913 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.1520000000000001e-05 2023-09-23T21:07:51.913 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:51.913 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:51.913 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T21:07:51.913 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:07.783246+0000", 2023-09-23T21:07:51.914 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.6998000000000001e-05 2023-09-23T21:07:51.914 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:07:51.914 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:07:51.914 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:07:51.914 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:51.914 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:51.914 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35856.0:28286 9.9 9:963952f9:::benchmark_data_smithi026_342732_object28285:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e351)", 2023-09-23T21:07:51.914 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T21:06:07.723708+0000", 2023-09-23T21:07:51.914 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 104.12460830000001, 2023-09-23T21:07:51.914 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.060052071999999998, 2023-09-23T21:07:51.915 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:07:51.915 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:07:51.915 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T21:07:51.915 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35856", 2023-09-23T21:07:51.915 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/4180396258", 2023-09-23T21:07:51.915 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 28286 2023-09-23T21:07:51.915 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:51.915 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:07:51.915 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:51.915 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:07:51.915 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:07.723708+0000", 2023-09-23T21:07:51.915 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:07:51.915 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:51.916 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:51.916 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:07:51.916 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:07.723708+0000", 2023-09-23T21:07:51.916 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:07:51.916 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:51.916 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:51.916 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:07:51.916 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:07.723708+0000", 2023-09-23T21:07:51.916 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.999999 2023-09-23T21:07:51.916 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:51.916 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:51.916 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:07:51.916 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:07.723726+0000", 2023-09-23T21:07:51.917 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.8077999999999999e-05 2023-09-23T21:07:51.917 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:51.917 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:51.917 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:07:51.917 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:07.723733+0000", 2023-09-23T21:07:51.917 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.6569999999999997e-06 2023-09-23T21:07:51.917 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:51.917 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:51.917 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:07:51.917 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:07.723738+0000", 2023-09-23T21:07:51.917 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.33e-06 2023-09-23T21:07:51.917 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:51.918 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:51.918 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:07:51.918 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:07.723777+0000", 2023-09-23T21:07:51.918 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.9072999999999997e-05 2023-09-23T21:07:51.918 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:51.918 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:51.918 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:07:51.918 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:07.779896+0000", 2023-09-23T21:07:51.918 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.056119646000000002 2023-09-23T21:07:51.918 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:51.918 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:51.919 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 5,7", 2023-09-23T21:07:51.919 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:07.780048+0000", 2023-09-23T21:07:51.919 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00015186799999999999 2023-09-23T21:07:51.919 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:51.919 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:51.919 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:07:51.919 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:07.782878+0000", 2023-09-23T21:07:51.919 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00282971 2023-09-23T21:07:51.919 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:51.919 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:51.919 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T21:07:51.919 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:07.783263+0000", 2023-09-23T21:07:51.919 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00038528400000000001 2023-09-23T21:07:51.920 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:51.920 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:51.920 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:07:51.920 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:07.783674+0000", 2023-09-23T21:07:51.920 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00041103 2023-09-23T21:07:51.920 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:51.920 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:51.920 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T21:07:51.920 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:07.783724+0000", 2023-09-23T21:07:51.920 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.9891000000000001e-05 2023-09-23T21:07:51.920 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:51.920 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:51.920 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T21:07:51.921 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:07.783760+0000", 2023-09-23T21:07:51.921 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.6282000000000003e-05 2023-09-23T21:07:51.921 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:07:51.921 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:07:51.921 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:07:51.921 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:51.921 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:51.921 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_repop_reply(client.35856.0:28276 9.9 e351/346 ondisk, result = 0)", 2023-09-23T21:07:51.921 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T21:06:07.723751+0000", 2023-09-23T21:07:51.921 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 104.124565567, 2023-09-23T21:07:51.921 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.057926057000000003, 2023-09-23T21:07:51.921 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:07:51.921 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "started", 2023-09-23T21:07:51.922 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:07:51.922 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:51.922 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:07:51.922 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:07.723751+0000", 2023-09-23T21:07:51.922 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:07:51.922 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:51.922 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:51.922 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:07:51.922 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:07.723751+0000", 2023-09-23T21:07:51.922 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:07:51.922 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:51.922 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:51.923 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:07:51.923 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:07.723751+0000", 2023-09-23T21:07:51.923 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T21:07:51.923 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:51.923 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:51.923 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:07:51.923 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:07.723753+0000", 2023-09-23T21:07:51.923 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.6029999999999999e-06 2023-09-23T21:07:51.923 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:51.923 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:51.923 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:07:51.923 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:07.723758+0000", 2023-09-23T21:07:51.923 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.5260000000000004e-06 2023-09-23T21:07:51.924 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:51.924 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:51.924 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:07:51.924 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:07.723761+0000", 2023-09-23T21:07:51.924 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.9660000000000001e-06 2023-09-23T21:07:51.924 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:51.924 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:51.924 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:07:51.924 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:07.781570+0000", 2023-09-23T21:07:51.924 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.057809087000000002 2023-09-23T21:07:51.924 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:51.924 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:51.924 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:07:51.925 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:07.781578+0000", 2023-09-23T21:07:51.925 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.6450000000000005e-06 2023-09-23T21:07:51.925 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:51.925 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:51.925 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T21:07:51.925 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:07.781677+0000", 2023-09-23T21:07:51.925 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.9537999999999995e-05 2023-09-23T21:07:51.925 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:07:51.925 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:07:51.925 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:07:51.925 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:51.925 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:51.925 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_repop_reply(client.35856.0:28280 9.b e351/346 ondisk, result = 0)", 2023-09-23T21:07:51.926 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T21:06:07.723840+0000", 2023-09-23T21:07:51.926 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 104.124476719, 2023-09-23T21:07:51.926 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.057517367999999999, 2023-09-23T21:07:51.926 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:07:51.926 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "started", 2023-09-23T21:07:51.926 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:07:51.926 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:51.926 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:07:51.926 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:07.723840+0000", 2023-09-23T21:07:51.926 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:07:51.926 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:51.926 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:51.927 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:07:51.927 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:07.723840+0000", 2023-09-23T21:07:51.927 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:07:51.927 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:51.927 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:51.927 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:07:51.927 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:07.723840+0000", 2023-09-23T21:07:51.927 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T21:07:51.927 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:51.927 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:51.927 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:07:51.927 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:07.723843+0000", 2023-09-23T21:07:51.927 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.9629999999999998e-06 2023-09-23T21:07:51.928 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:51.928 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:51.928 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:07:51.928 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:07.723850+0000", 2023-09-23T21:07:51.928 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.8460000000000008e-06 2023-09-23T21:07:51.928 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:51.928 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:51.928 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:07:51.928 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:07.723853+0000", 2023-09-23T21:07:51.928 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.0199999999999999e-06 2023-09-23T21:07:51.928 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:51.928 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:51.928 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:07:51.929 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:07.781340+0000", 2023-09-23T21:07:51.929 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.057486186000000002 2023-09-23T21:07:51.929 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:51.929 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:51.929 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:07:51.929 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:07.781348+0000", 2023-09-23T21:07:51.929 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.4400000000000005e-06 2023-09-23T21:07:51.929 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:51.929 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:51.929 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T21:07:51.929 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:07.781357+0000", 2023-09-23T21:07:51.930 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.2809999999999997e-06 2023-09-23T21:07:51.930 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:07:51.930 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:07:51.930 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:07:51.930 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:51.930 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:51.930 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_repop_reply(client.35856.0:28280 9.b e351/346 ondisk, result = 0)", 2023-09-23T21:07:51.930 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T21:06:07.724633+0000", 2023-09-23T21:07:51.930 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 104.123683812, 2023-09-23T21:07:51.930 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.05686389, 2023-09-23T21:07:51.930 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:07:51.930 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "started", 2023-09-23T21:07:51.930 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:07:51.931 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:51.931 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:07:51.931 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:07.724633+0000", 2023-09-23T21:07:51.931 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:07:51.931 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:51.931 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:51.931 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:07:51.931 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:07.724633+0000", 2023-09-23T21:07:51.931 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:07:51.931 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:51.931 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:51.931 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:07:51.932 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:07.724632+0000", 2023-09-23T21:07:51.932 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T21:07:51.932 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:51.932 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:51.932 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:07:51.932 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:07.724636+0000", 2023-09-23T21:07:51.932 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.3220000000000001e-06 2023-09-23T21:07:51.932 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:51.932 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:51.932 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:07:51.932 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:07.779825+0000", 2023-09-23T21:07:51.932 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.055188849999999998 2023-09-23T21:07:51.932 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:51.933 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:51.933 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:07:51.933 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:07.779836+0000", 2023-09-23T21:07:51.933 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1935e-05 2023-09-23T21:07:51.933 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:51.933 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:51.933 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:07:51.933 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:07.781379+0000", 2023-09-23T21:07:51.933 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.001542723 2023-09-23T21:07:51.933 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:51.933 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:51.933 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:07:51.933 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:07.781386+0000", 2023-09-23T21:07:51.934 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.861e-06 2023-09-23T21:07:51.934 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:51.934 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:51.934 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T21:07:51.934 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:06:07.781497+0000", 2023-09-23T21:07:51.934 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000110686 2023-09-23T21:07:51.934 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:07:51.934 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:07:51.934 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:07:51.934 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:51.934 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:51.934 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35856.0:119277 9.b 9:d08a051b:::benchmark_data_smithi026_342732_object6001:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e369)", 2023-09-23T21:07:51.935 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T21:07:33.779427+0000", 2023-09-23T21:07:51.935 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 18.068890151000002, 2023-09-23T21:07:51.935 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.045763980000000003, 2023-09-23T21:07:51.935 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:07:51.935 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:07:51.935 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T21:07:51.935 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35856", 2023-09-23T21:07:51.935 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/4180396258", 2023-09-23T21:07:51.936 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 119277 2023-09-23T21:07:51.936 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:51.936 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:07:51.936 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:51.936 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:07:51.936 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:07:33.779427+0000", 2023-09-23T21:07:51.936 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:07:51.936 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:51.936 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:51.936 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:07:51.936 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:07:33.779427+0000", 2023-09-23T21:07:51.936 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:07:51.936 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:51.937 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:51.937 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:07:51.937 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:07:33.779426+0000", 2023-09-23T21:07:51.937 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T21:07:51.937 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:51.937 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:51.937 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:07:51.937 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:07:33.779430+0000", 2023-09-23T21:07:51.937 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.0389999999999998e-06 2023-09-23T21:07:51.937 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:51.938 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:51.938 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:07:51.938 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:07:33.779439+0000", 2023-09-23T21:07:51.938 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.0149999999999992e-06 2023-09-23T21:07:51.938 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:51.938 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:51.938 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:07:51.938 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:07:33.779447+0000", 2023-09-23T21:07:51.938 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.089e-06 2023-09-23T21:07:51.938 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:51.938 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:51.939 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:07:51.939 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:07:33.794418+0000", 2023-09-23T21:07:51.939 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.014971177 2023-09-23T21:07:51.939 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:51.939 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:51.939 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:07:51.939 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:07:33.794586+0000", 2023-09-23T21:07:51.939 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00016727699999999999 2023-09-23T21:07:51.939 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:51.939 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:51.940 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 5,6", 2023-09-23T21:07:51.940 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:07:33.794728+0000", 2023-09-23T21:07:51.940 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00014255100000000001 2023-09-23T21:07:51.940 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:51.940 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:51.940 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:07:51.940 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:07:33.816792+0000", 2023-09-23T21:07:51.940 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.022064144000000001 2023-09-23T21:07:51.940 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:51.940 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:51.940 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:07:51.940 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:07:33.816844+0000", 2023-09-23T21:07:51.941 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.2252e-05 2023-09-23T21:07:51.941 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:51.941 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:51.941 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T21:07:51.941 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:07:33.825065+0000", 2023-09-23T21:07:51.941 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0082203650000000003 2023-09-23T21:07:51.941 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:51.941 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:51.941 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T21:07:51.941 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:07:33.825141+0000", 2023-09-23T21:07:51.942 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.5995000000000007e-05 2023-09-23T21:07:51.942 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:51.942 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:51.942 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T21:07:51.942 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:07:33.825191+0000", 2023-09-23T21:07:51.942 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.9703000000000002e-05 2023-09-23T21:07:51.942 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:07:51.942 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:07:51.942 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:07:51.942 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:51.943 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:51.943 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35856.0:119280 9.b 9:de4e090e:::benchmark_data_smithi026_342732_object6004:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e369)", 2023-09-23T21:07:51.943 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T21:07:33.779485+0000", 2023-09-23T21:07:51.943 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 18.068831795000001, 2023-09-23T21:07:51.943 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.045842742999999998, 2023-09-23T21:07:51.943 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:07:51.943 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:07:51.943 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T21:07:51.943 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35856", 2023-09-23T21:07:51.943 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/4180396258", 2023-09-23T21:07:51.944 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 119280 2023-09-23T21:07:51.944 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:51.944 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:07:51.944 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:51.944 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:07:51.944 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:07:33.779485+0000", 2023-09-23T21:07:51.944 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:07:51.944 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:51.944 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:51.944 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:07:51.944 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:07:33.779485+0000", 2023-09-23T21:07:51.944 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:07:51.944 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:51.945 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:51.945 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:07:51.945 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:07:33.779484+0000", 2023-09-23T21:07:51.945 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T21:07:51.945 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:51.945 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:51.945 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:07:51.945 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:07:33.779488+0000", 2023-09-23T21:07:51.945 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.7680000000000002e-06 2023-09-23T21:07:51.945 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:51.945 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:51.945 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:07:51.946 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:07:33.779496+0000", 2023-09-23T21:07:51.946 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.6909999999999997e-06 2023-09-23T21:07:51.946 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:51.946 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:51.946 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:07:51.946 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:07:33.779503+0000", 2023-09-23T21:07:51.946 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.7460000000000003e-06 2023-09-23T21:07:51.946 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:51.946 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:51.946 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:07:51.946 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:07:33.800611+0000", 2023-09-23T21:07:51.947 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.021108954999999999 2023-09-23T21:07:51.947 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:51.947 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:51.947 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:07:51.947 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:07:33.800771+0000", 2023-09-23T21:07:51.947 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00015907399999999999 2023-09-23T21:07:51.947 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:51.948 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:51.948 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 5,6", 2023-09-23T21:07:51.948 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:07:33.800920+0000", 2023-09-23T21:07:51.948 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00014893799999999999 2023-09-23T21:07:51.948 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:51.948 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:51.948 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:07:51.948 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:07:33.816880+0000", 2023-09-23T21:07:51.948 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.015960516000000001 2023-09-23T21:07:51.948 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:51.949 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:51.949 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:07:51.949 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:07:33.816919+0000", 2023-09-23T21:07:51.949 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.8951e-05 2023-09-23T21:07:51.949 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:51.949 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:51.949 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T21:07:51.949 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:07:33.825245+0000", 2023-09-23T21:07:51.949 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0083261429999999994 2023-09-23T21:07:51.950 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:51.950 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:51.950 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T21:07:51.950 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:07:33.825289+0000", 2023-09-23T21:07:51.950 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.3464000000000003e-05 2023-09-23T21:07:51.950 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:51.950 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:51.950 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T21:07:51.950 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:07:33.825328+0000", 2023-09-23T21:07:51.950 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.9048999999999998e-05 2023-09-23T21:07:51.950 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:07:51.950 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:07:51.951 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:07:51.951 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:51.951 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:51.951 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35856.0:119281 9.b 9:d43b6aa3:::benchmark_data_smithi026_342732_object6005:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e369)", 2023-09-23T21:07:51.951 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T21:07:33.779508+0000", 2023-09-23T21:07:51.951 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 18.068808705999999, 2023-09-23T21:07:51.951 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.045987717999999997, 2023-09-23T21:07:51.951 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:07:51.951 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:07:51.951 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T21:07:51.951 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35856", 2023-09-23T21:07:51.951 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/4180396258", 2023-09-23T21:07:51.952 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 119281 2023-09-23T21:07:51.952 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:51.952 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:07:51.952 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:51.952 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:07:51.952 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:07:33.779508+0000", 2023-09-23T21:07:51.952 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:07:51.952 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:51.952 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:51.952 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:07:51.952 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:07:33.779508+0000", 2023-09-23T21:07:51.952 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:07:51.953 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:51.953 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:51.953 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:07:51.953 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:07:33.779508+0000", 2023-09-23T21:07:51.953 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T21:07:51.953 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:51.953 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:51.953 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:07:51.953 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:07:33.779539+0000", 2023-09-23T21:07:51.953 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.1442000000000002e-05 2023-09-23T21:07:51.953 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:51.953 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:51.954 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:07:51.954 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:07:33.779548+0000", 2023-09-23T21:07:51.954 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.166e-06 2023-09-23T21:07:51.954 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:51.954 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:51.954 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:07:51.954 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:07:33.779555+0000", 2023-09-23T21:07:51.954 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.1060000000000002e-06 2023-09-23T21:07:51.954 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:51.954 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:51.954 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:07:51.954 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:07:33.812552+0000", 2023-09-23T21:07:51.955 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.032997112000000002 2023-09-23T21:07:51.955 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:51.955 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:51.955 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:07:51.955 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:07:33.812692+0000", 2023-09-23T21:07:51.955 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000139593 2023-09-23T21:07:51.955 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:51.955 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:51.955 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 5,6", 2023-09-23T21:07:51.955 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:07:33.812811+0000", 2023-09-23T21:07:51.955 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000118749 2023-09-23T21:07:51.955 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:51.955 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:51.956 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:07:51.956 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:07:33.825237+0000", 2023-09-23T21:07:51.956 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.012426471999999999 2023-09-23T21:07:51.956 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:51.956 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:51.956 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T21:07:51.956 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:07:33.825335+0000", 2023-09-23T21:07:51.956 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.7676000000000005e-05 2023-09-23T21:07:51.956 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:51.956 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:51.956 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:07:51.956 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:07:33.825413+0000", 2023-09-23T21:07:51.957 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.8436000000000006e-05 2023-09-23T21:07:51.957 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:51.957 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:51.957 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T21:07:51.957 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:07:33.825456+0000", 2023-09-23T21:07:51.957 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.3180000000000003e-05 2023-09-23T21:07:51.957 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:51.957 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:51.957 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T21:07:51.957 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:07:33.825496+0000", 2023-09-23T21:07:51.957 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.9249000000000003e-05 2023-09-23T21:07:51.957 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:07:51.957 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:07:51.958 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:07:51.958 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:51.958 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:51.958 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35856.0:121993 9.b 9:d683ef48:::benchmark_data_smithi026_342732_object8717:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e369)", 2023-09-23T21:07:51.958 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T21:07:37.256240+0000", 2023-09-23T21:07:51.958 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 14.592076191, 2023-09-23T21:07:51.958 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.041528649000000001, 2023-09-23T21:07:51.958 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:07:51.958 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:07:51.958 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T21:07:51.958 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35856", 2023-09-23T21:07:51.958 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/4180396258", 2023-09-23T21:07:51.959 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 121993 2023-09-23T21:07:51.959 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:51.959 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:07:51.959 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:51.959 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:07:51.959 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:07:37.256240+0000", 2023-09-23T21:07:51.959 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:07:51.959 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:51.959 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:51.959 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:07:51.959 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:07:37.256240+0000", 2023-09-23T21:07:51.959 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:07:51.959 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:51.960 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:51.960 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:07:51.960 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:07:37.256238+0000", 2023-09-23T21:07:51.960 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999976 2023-09-23T21:07:51.960 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:51.960 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:51.960 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:07:51.960 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:07:37.256248+0000", 2023-09-23T21:07:51.960 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.9990000000000003e-06 2023-09-23T21:07:51.960 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:51.960 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:51.960 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:07:51.960 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:07:37.256271+0000", 2023-09-23T21:07:51.961 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.2609999999999999e-05 2023-09-23T21:07:51.961 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:51.961 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:51.961 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:07:51.961 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:07:37.256282+0000", 2023-09-23T21:07:51.961 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1022000000000001e-05 2023-09-23T21:07:51.961 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:51.961 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:51.961 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:07:51.961 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:07:37.282674+0000", 2023-09-23T21:07:51.961 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.026391744000000002 2023-09-23T21:07:51.961 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:51.961 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:51.962 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:07:51.962 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:07:37.282826+0000", 2023-09-23T21:07:51.962 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00015270799999999999 2023-09-23T21:07:51.962 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:51.962 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:51.962 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 5,6", 2023-09-23T21:07:51.962 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:07:37.282984+0000", 2023-09-23T21:07:51.962 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000157245 2023-09-23T21:07:51.962 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:51.962 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:51.962 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:07:51.962 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:07:37.287755+0000", 2023-09-23T21:07:51.963 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0047710410000000002 2023-09-23T21:07:51.963 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:51.963 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:51.963 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:07:51.963 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:07:37.287819+0000", 2023-09-23T21:07:51.963 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.4201000000000004e-05 2023-09-23T21:07:51.963 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:51.963 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:51.963 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T21:07:51.963 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:07:37.297678+0000", 2023-09-23T21:07:51.963 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.009858693 2023-09-23T21:07:51.963 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:51.963 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:51.964 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T21:07:51.964 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:07:37.297728+0000", 2023-09-23T21:07:51.964 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.0766000000000001e-05 2023-09-23T21:07:51.964 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:51.964 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:51.964 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T21:07:51.964 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:07:37.297769+0000", 2023-09-23T21:07:51.964 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.0809000000000002e-05 2023-09-23T21:07:51.964 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:07:51.964 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:07:51.964 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:07:51.965 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:51.965 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:51.965 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35856.0:123983 9.b 9:daba1bb5:::benchmark_data_smithi026_342732_object10707:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e370)", 2023-09-23T21:07:51.965 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T21:07:39.736351+0000", 2023-09-23T21:07:51.965 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 12.1119658, 2023-09-23T21:07:51.965 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.043011300000000002, 2023-09-23T21:07:51.965 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:07:51.965 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:07:51.965 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T21:07:51.965 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35856", 2023-09-23T21:07:51.965 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/4180396258", 2023-09-23T21:07:51.965 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 123983 2023-09-23T21:07:51.965 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:51.966 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:07:51.966 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:51.966 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:07:51.966 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:07:39.736351+0000", 2023-09-23T21:07:51.966 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:07:51.966 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:51.966 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:51.966 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:07:51.966 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:07:39.736351+0000", 2023-09-23T21:07:51.966 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:07:51.966 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:51.966 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:51.967 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:07:51.967 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:07:39.736349+0000", 2023-09-23T21:07:51.967 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T21:07:51.967 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:51.967 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:51.967 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:07:51.967 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:07:39.736357+0000", 2023-09-23T21:07:51.967 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.3010000000000001e-06 2023-09-23T21:07:51.967 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:51.967 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:51.967 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:07:51.967 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:07:39.736369+0000", 2023-09-23T21:07:51.968 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1936999999999999e-05 2023-09-23T21:07:51.968 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:51.968 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:51.968 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:07:51.968 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:07:39.736376+0000", 2023-09-23T21:07:51.968 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.3930000000000002e-06 2023-09-23T21:07:51.968 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:51.968 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:51.968 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:07:51.968 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:07:39.736458+0000", 2023-09-23T21:07:51.968 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.2249999999999993e-05 2023-09-23T21:07:51.968 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:51.969 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:51.969 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:07:51.969 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:07:39.736590+0000", 2023-09-23T21:07:51.969 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000131225 2023-09-23T21:07:51.969 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:51.969 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:51.969 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 5,6", 2023-09-23T21:07:51.969 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:07:39.736699+0000", 2023-09-23T21:07:51.969 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000109283 2023-09-23T21:07:51.969 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:51.969 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:51.969 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:07:51.970 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:07:39.752543+0000", 2023-09-23T21:07:51.970 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.015843808000000001 2023-09-23T21:07:51.970 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:51.970 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:51.970 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:07:51.970 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:07:39.752777+0000", 2023-09-23T21:07:51.970 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000234448 2023-09-23T21:07:51.970 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:51.970 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:51.970 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T21:07:51.971 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:07:39.779265+0000", 2023-09-23T21:07:51.971 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.026487720999999999 2023-09-23T21:07:51.971 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:51.971 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:51.971 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T21:07:51.971 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:07:39.779316+0000", 2023-09-23T21:07:51.971 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.1597000000000001e-05 2023-09-23T21:07:51.971 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:51.971 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:51.971 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T21:07:51.971 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:07:39.779362+0000", 2023-09-23T21:07:51.972 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.5788999999999998e-05 2023-09-23T21:07:51.972 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:07:51.972 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:07:51.972 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:07:51.972 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:51.972 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:51.972 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35856.0:133414 9.b 9:d07cb016:::benchmark_data_smithi026_342732_object20138:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e373)", 2023-09-23T21:07:51.972 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T21:07:51.152742+0000", 2023-09-23T21:07:51.972 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 0.69557445500000004, 2023-09-23T21:07:51.972 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.044206370000000002, 2023-09-23T21:07:51.973 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:07:51.973 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:07:51.973 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T21:07:51.973 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35856", 2023-09-23T21:07:51.973 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/4180396258", 2023-09-23T21:07:51.973 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 133414 2023-09-23T21:07:51.973 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:51.973 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:07:51.973 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:51.973 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:07:51.973 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:07:51.152742+0000", 2023-09-23T21:07:51.973 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:07:51.974 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:51.974 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:51.974 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:07:51.974 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:07:51.152742+0000", 2023-09-23T21:07:51.974 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:07:51.974 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:51.974 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:51.974 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:07:51.974 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:07:51.152742+0000", 2023-09-23T21:07:51.974 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T21:07:51.975 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:51.975 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:51.975 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:07:51.975 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:07:51.152745+0000", 2023-09-23T21:07:51.975 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.7829999999999999e-06 2023-09-23T21:07:51.975 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:51.975 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:51.975 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:07:51.975 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:07:51.152751+0000", 2023-09-23T21:07:51.975 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.4280000000000001e-06 2023-09-23T21:07:51.975 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:51.976 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:51.976 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:07:51.976 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:07:51.152756+0000", 2023-09-23T21:07:51.976 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.04e-06 2023-09-23T21:07:51.976 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:51.976 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:51.976 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:07:51.976 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:07:51.155589+0000", 2023-09-23T21:07:51.976 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.002832963 2023-09-23T21:07:51.976 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:51.977 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:51.977 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:07:51.977 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:07:51.155704+0000", 2023-09-23T21:07:51.977 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000115247 2023-09-23T21:07:51.977 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:51.977 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:51.977 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 5,6", 2023-09-23T21:07:51.977 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:07:51.155801+0000", 2023-09-23T21:07:51.977 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.6569000000000003e-05 2023-09-23T21:07:51.977 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:51.977 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:51.978 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T21:07:51.978 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:07:51.168469+0000", 2023-09-23T21:07:51.978 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.012668291999999999 2023-09-23T21:07:51.978 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:51.978 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:51.978 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:07:51.978 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:07:51.174048+0000", 2023-09-23T21:07:51.978 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0055790520000000001 2023-09-23T21:07:51.978 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:51.979 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:51.979 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:07:51.979 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:07:51.196863+0000", 2023-09-23T21:07:51.979 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.022814320999999999 2023-09-23T21:07:51.979 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:51.979 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:51.979 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T21:07:51.979 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:07:51.196911+0000", 2023-09-23T21:07:51.979 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.8371999999999997e-05 2023-09-23T21:07:51.979 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:51.980 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:51.980 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T21:07:51.980 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:07:51.196949+0000", 2023-09-23T21:07:51.980 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.7706000000000001e-05 2023-09-23T21:07:51.980 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:07:51.980 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:07:51.980 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:07:51.980 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:51.980 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:51.981 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35856.0:133416 9.b 9:d2bf3bcc:::benchmark_data_smithi026_342732_object20140:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e373)", 2023-09-23T21:07:51.981 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T21:07:51.152770+0000", 2023-09-23T21:07:51.981 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 0.69554627700000005, 2023-09-23T21:07:51.981 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.044276933999999997, 2023-09-23T21:07:51.981 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:07:51.981 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:07:51.981 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T21:07:51.981 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35856", 2023-09-23T21:07:51.981 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/4180396258", 2023-09-23T21:07:51.981 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 133416 2023-09-23T21:07:51.982 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:51.982 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:07:51.982 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:51.982 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:07:51.982 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:07:51.152770+0000", 2023-09-23T21:07:51.982 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:07:51.982 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:51.982 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:51.982 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:07:51.982 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:07:51.152770+0000", 2023-09-23T21:07:51.983 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:07:51.983 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:51.983 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:51.983 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:07:51.983 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:07:51.152770+0000", 2023-09-23T21:07:51.983 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T21:07:51.983 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:51.983 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:51.983 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:07:51.983 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:07:51.152773+0000", 2023-09-23T21:07:51.983 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.7819999999999998e-06 2023-09-23T21:07:51.983 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:51.984 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:51.984 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:07:51.984 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:07:51.152788+0000", 2023-09-23T21:07:51.984 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.4748e-05 2023-09-23T21:07:51.984 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:51.984 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:51.984 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:07:51.984 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:07:51.152795+0000", 2023-09-23T21:07:51.984 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.5889999999999997e-06 2023-09-23T21:07:51.984 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:51.984 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:51.984 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:07:51.984 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:07:51.159803+0000", 2023-09-23T21:07:51.985 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0070077439999999998 2023-09-23T21:07:51.985 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:51.985 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:51.985 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:07:51.985 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:07:51.159941+0000", 2023-09-23T21:07:51.985 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00013857499999999999 2023-09-23T21:07:51.985 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:51.985 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:51.985 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 5,6", 2023-09-23T21:07:51.985 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:07:51.160057+0000", 2023-09-23T21:07:51.985 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00011553300000000001 2023-09-23T21:07:51.985 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:51.986 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:51.986 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T21:07:51.986 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:07:51.173995+0000", 2023-09-23T21:07:51.986 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.013938015999999999 2023-09-23T21:07:51.986 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:51.986 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:51.986 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:07:51.986 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:07:51.176386+0000", 2023-09-23T21:07:51.986 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0023911399999999999 2023-09-23T21:07:51.986 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:51.986 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:51.986 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:07:51.987 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:07:51.196980+0000", 2023-09-23T21:07:51.987 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0205942 2023-09-23T21:07:51.987 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:51.987 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:51.987 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T21:07:51.987 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:07:51.197016+0000", 2023-09-23T21:07:51.987 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.5837000000000003e-05 2023-09-23T21:07:51.987 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:51.987 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:51.987 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T21:07:51.987 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:07:51.197047+0000", 2023-09-23T21:07:51.987 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.1171999999999999e-05 2023-09-23T21:07:51.987 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:07:51.988 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:07:51.988 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:07:51.988 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:51.988 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:51.988 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35856.0:133419 9.b 9:d0a87b2e:::benchmark_data_smithi026_342732_object20143:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e373)", 2023-09-23T21:07:51.988 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T21:07:51.152821+0000", 2023-09-23T21:07:51.988 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 0.69549563000000003, 2023-09-23T21:07:51.988 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.044320084000000003, 2023-09-23T21:07:51.988 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:07:51.988 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:07:51.988 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T21:07:51.988 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35856", 2023-09-23T21:07:51.988 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/4180396258", 2023-09-23T21:07:51.989 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 133419 2023-09-23T21:07:51.989 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:51.989 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:07:51.989 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:51.989 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:07:51.989 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:07:51.152821+0000", 2023-09-23T21:07:51.989 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:07:51.989 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:51.989 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:51.989 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:07:51.989 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:07:51.152821+0000", 2023-09-23T21:07:51.989 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:07:51.990 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:51.990 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:51.990 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:07:51.990 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:07:51.152821+0000", 2023-09-23T21:07:51.990 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T21:07:51.990 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:51.990 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:51.990 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:07:51.990 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:07:51.152824+0000", 2023-09-23T21:07:51.990 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.8380000000000002e-06 2023-09-23T21:07:51.990 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:51.990 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:51.990 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:07:51.991 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:07:51.152829+0000", 2023-09-23T21:07:51.991 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.8390000000000003e-06 2023-09-23T21:07:51.991 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:51.991 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:51.991 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:07:51.991 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:07:51.152834+0000", 2023-09-23T21:07:51.991 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.8049999999999997e-06 2023-09-23T21:07:51.991 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:51.991 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:51.991 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:07:51.991 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:07:51.164197+0000", 2023-09-23T21:07:51.991 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.011362274 2023-09-23T21:07:51.992 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:51.992 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:51.992 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:07:51.992 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:07:51.164305+0000", 2023-09-23T21:07:51.992 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00010805 2023-09-23T21:07:51.992 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:51.992 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:51.992 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 5,6", 2023-09-23T21:07:51.992 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:07:51.164404+0000", 2023-09-23T21:07:51.992 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.9055999999999995e-05 2023-09-23T21:07:51.992 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:51.992 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:51.992 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T21:07:51.993 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:07:51.174125+0000", 2023-09-23T21:07:51.993 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0097212549999999998 2023-09-23T21:07:51.993 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:51.993 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:51.993 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:07:51.993 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:07:51.176425+0000", 2023-09-23T21:07:51.993 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0023002119999999998 2023-09-23T21:07:51.993 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:51.993 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:51.993 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:07:51.993 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:07:51.197077+0000", 2023-09-23T21:07:51.993 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.020651869999999999 2023-09-23T21:07:51.993 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:51.994 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:51.994 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T21:07:51.994 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:07:51.197111+0000", 2023-09-23T21:07:51.994 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.4345999999999999e-05 2023-09-23T21:07:51.994 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:51.994 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:51.994 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T21:07:51.994 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:07:51.197141+0000", 2023-09-23T21:07:51.994 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.9848999999999998e-05 2023-09-23T21:07:51.994 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:07:51.994 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:07:51.994 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:07:51.995 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:51.995 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:51.995 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35856.0:133422 9.b 9:d95710d5:::benchmark_data_smithi026_342732_object20146:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e373)", 2023-09-23T21:07:51.995 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T21:07:51.152842+0000", 2023-09-23T21:07:51.995 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 0.69547447799999995, 2023-09-23T21:07:51.995 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.044420188999999999, 2023-09-23T21:07:51.995 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:07:51.995 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:07:51.995 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T21:07:51.995 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35856", 2023-09-23T21:07:51.995 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/4180396258", 2023-09-23T21:07:51.995 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 133422 2023-09-23T21:07:51.995 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:51.996 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:07:51.996 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:51.996 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:07:51.996 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:07:51.152842+0000", 2023-09-23T21:07:51.996 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:07:51.996 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:51.996 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:51.996 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:07:51.996 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:07:51.152842+0000", 2023-09-23T21:07:51.996 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:07:51.996 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:51.996 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:51.996 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:07:51.997 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:07:51.152842+0000", 2023-09-23T21:07:51.997 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T21:07:51.997 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:51.997 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:51.997 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:07:51.997 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:07:51.152844+0000", 2023-09-23T21:07:51.997 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.6699999999999998e-06 2023-09-23T21:07:51.997 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:51.997 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:51.997 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:07:51.997 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:07:51.152853+0000", 2023-09-23T21:07:51.997 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.5380000000000002e-06 2023-09-23T21:07:51.997 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:51.998 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:51.998 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:07:51.998 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:07:51.152858+0000", 2023-09-23T21:07:51.998 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.1000000000000003e-06 2023-09-23T21:07:51.998 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:51.998 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:51.998 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:07:51.998 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:07:51.168505+0000", 2023-09-23T21:07:51.998 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.015646891999999999 2023-09-23T21:07:51.998 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:51.998 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:51.998 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:07:51.999 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:07:51.168613+0000", 2023-09-23T21:07:51.999 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000107847 2023-09-23T21:07:51.999 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:51.999 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:51.999 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 5,6", 2023-09-23T21:07:51.999 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:07:51.168713+0000", 2023-09-23T21:07:51.999 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000100037 2023-09-23T21:07:51.999 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:51.999 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:51.999 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T21:07:51.999 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:07:51.174188+0000", 2023-09-23T21:07:51.999 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0054750729999999996 2023-09-23T21:07:52.000 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:52.000 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:52.000 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:07:52.000 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:07:51.197170+0000", 2023-09-23T21:07:52.000 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.022982321999999999 2023-09-23T21:07:52.000 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:52.001 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:52.001 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:07:52.001 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:07:51.197199+0000", 2023-09-23T21:07:52.001 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.8708999999999999e-05 2023-09-23T21:07:52.001 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:52.001 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:52.001 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T21:07:52.001 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:07:51.197231+0000", 2023-09-23T21:07:52.001 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.2341000000000001e-05 2023-09-23T21:07:52.001 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:52.001 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:52.001 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T21:07:52.002 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:07:51.197262+0000", 2023-09-23T21:07:52.002 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.1099000000000002e-05 2023-09-23T21:07:52.002 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:07:52.002 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:07:52.002 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:07:52.002 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:07:52.002 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:07:52.002 INFO:teuthology.orchestra.run.smithi026.stdout:} 2023-09-23T21:07:52.003 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:07:52.003 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:07:52.003 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:07:52.104 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:07:52.155 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:07:52.232 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:07:52.236 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:07:52.237 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:07:52.337 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:07:52.366 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_ops_in_flight 2023-09-23T21:07:52.378 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T21:07:52.405 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:07:52.467 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:07:52.473 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:07:52.473 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:07:52.477 INFO:teuthology.orchestra.run.smithi026.stderr:reweighted osd.5 to 1 (10000) 2023-09-23T21:07:52.575 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:07:52.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:07:52 smithi026 ceph-mon[179688]: pgmap v974: 109 pgs: 109 active+clean; 5.8 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 793 op/s 2023-09-23T21:07:52.633 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:07:52 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/1033061671' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-09-23T21:07:52.633 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:07:52 smithi026 ceph-mon[175151]: pgmap v974: 109 pgs: 109 active+clean; 5.8 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 793 op/s 2023-09-23T21:07:52.633 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:07:52 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/1033061671' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-09-23T21:07:52.696 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:07:52.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:07:52 smithi070 ceph-mon[168282]: pgmap v974: 109 pgs: 109 active+clean; 5.8 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 793 op/s 2023-09-23T21:07:52.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:07:52 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/1033061671' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-09-23T21:07:52.982 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:07:52.993 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:07:52.994 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:07:53.095 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:07:53.113 INFO:tasks.cephadm.ceph_manager.ceph:waiting for recovery to complete 2023-09-23T21:07:53.113 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T21:07:53.181 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:07:53.209 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T21:07:53.249 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:07:53.269 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:07:53.269 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:07:53.358 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:07:53.370 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:07:53.406 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:07:53.441 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:07:53.446 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:07:53.450 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:07:53.450 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:07:53.551 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:07:53.589 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:07:53.633 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:07:53.638 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:07:53 smithi026 ceph-mon[179688]: failed to encode map e374 with expected crc 2023-09-23T21:07:53.638 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:07:53 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/1033061671' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-09-23T21:07:53.638 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:07:53 smithi026 ceph-mon[179688]: osdmap e374: 8 total, 8 up, 8 in 2023-09-23T21:07:53.638 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:07:53 smithi026 ceph-mon[179688]: failed to encode map e374 with expected crc 2023-09-23T21:07:53.638 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:07:53 smithi026 ceph-mon[175151]: failed to encode map e374 with expected crc 2023-09-23T21:07:53.638 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:07:53 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/1033061671' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-09-23T21:07:53.639 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:07:53 smithi026 ceph-mon[175151]: osdmap e374: 8 total, 8 up, 8 in 2023-09-23T21:07:53.639 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:07:53 smithi026 ceph-mon[175151]: failed to encode map e374 with expected crc 2023-09-23T21:07:53.639 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:07:53.640 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:07:53.740 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:07:53.777 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:07:53 smithi070 ceph-mon[168282]: failed to encode map e374 with expected crc 2023-09-23T21:07:53.777 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:07:53 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/1033061671' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-09-23T21:07:53.778 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:07:53 smithi070 ceph-mon[168282]: osdmap e374: 8 total, 8 up, 8 in 2023-09-23T21:07:53.778 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:07:53 smithi070 ceph-mon[168282]: failed to encode map e374 with expected crc 2023-09-23T21:07:53.780 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:07:53.817 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:07:53.820 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:07:53.821 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:07:53.922 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:07:53.996 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:07:54.235 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:07:54.243 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:07:54.243 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:07:54.344 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:07:54.379 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:07:54.434 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:07:54.436 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:07:54.437 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:07:54.538 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:07:54.609 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:07:54.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:07:54 smithi070 ceph-mon[168282]: pgmap v976: 109 pgs: 109 active+clean; 5.7 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 784 op/s 2023-09-23T21:07:54.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:07:54 smithi026 ceph-mon[179688]: pgmap v976: 109 pgs: 109 active+clean; 5.7 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 784 op/s 2023-09-23T21:07:54.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:07:54 smithi026 ceph-mon[175151]: pgmap v976: 109 pgs: 109 active+clean; 5.7 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 784 op/s 2023-09-23T21:07:54.915 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:07:54.920 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:07:54.921 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:07:55.003 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:07:55.003 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:07:55.022 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:07:55.062 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:07:55.108 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:07:55.111 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:07:55.111 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:07:55.212 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:07:55.258 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:07:55.325 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:07:55.334 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:07:55.334 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:07:55.435 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:07:55.488 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:07:55.495 INFO:teuthology.orchestra.run.smithi026.stdout:{ 2023-09-23T21:07:55.495 INFO:teuthology.orchestra.run.smithi026.stdout: "ops": [ 2023-09-23T21:07:55.495 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:55.495 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35856.0:136828 9.6 9:6917a47f:::benchmark_data_smithi026_342732_object23552:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e374)", 2023-09-23T21:07:55.495 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T21:07:55.483504+0000", 2023-09-23T21:07:55.495 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 0.011274586, 2023-09-23T21:07:55.496 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.011309332, 2023-09-23T21:07:55.496 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:07:55.496 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "waiting for sub ops", 2023-09-23T21:07:55.496 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T21:07:55.496 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35856", 2023-09-23T21:07:55.496 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/4180396258", 2023-09-23T21:07:55.496 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 136828 2023-09-23T21:07:55.496 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:55.496 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:07:55.496 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:55.496 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:07:55.497 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:07:55.483504+0000", 2023-09-23T21:07:55.497 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:07:55.497 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:55.497 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:55.497 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:07:55.497 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:07:55.483504+0000", 2023-09-23T21:07:55.497 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:07:55.497 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:55.497 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:55.497 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:07:55.498 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:07:55.483502+0000", 2023-09-23T21:07:55.498 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T21:07:55.498 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:55.498 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:55.498 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:07:55.498 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:07:55.483511+0000", 2023-09-23T21:07:55.498 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.1470000000000001e-06 2023-09-23T21:07:55.498 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:55.498 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:55.498 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:07:55.498 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:07:55.483530+0000", 2023-09-23T21:07:55.499 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.9313999999999998e-05 2023-09-23T21:07:55.499 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:55.499 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:55.499 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:07:55.499 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:07:55.483539+0000", 2023-09-23T21:07:55.499 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.7399999999999993e-06 2023-09-23T21:07:55.499 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:55.499 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:55.499 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:07:55.499 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:07:55.483753+0000", 2023-09-23T21:07:55.500 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00021465199999999999 2023-09-23T21:07:55.500 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:55.500 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:55.500 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:07:55.500 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:07:55.483866+0000", 2023-09-23T21:07:55.500 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000113103 2023-09-23T21:07:55.500 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:55.500 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:55.500 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 1,5", 2023-09-23T21:07:55.500 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:07:55.483965+0000", 2023-09-23T21:07:55.501 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.8336000000000002e-05 2023-09-23T21:07:55.501 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:55.501 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:55.501 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T21:07:55.501 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:07:55.490637+0000", 2023-09-23T21:07:55.501 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0066719759999999996 2023-09-23T21:07:55.501 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:07:55.501 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:07:55.501 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:07:55.501 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:07:55.493593+0000", 2023-09-23T21:07:55.502 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.002956766 2023-09-23T21:07:55.502 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:07:55.502 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:07:55.502 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:07:55.502 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:07:55.502 INFO:teuthology.orchestra.run.smithi026.stdout: ], 2023-09-23T21:07:55.502 INFO:teuthology.orchestra.run.smithi026.stdout: "num_ops": 1 2023-09-23T21:07:55.502 INFO:teuthology.orchestra.run.smithi026.stdout:} 2023-09-23T21:07:55.553 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:07:55.556 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:07:55.557 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:07:55.657 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:07:55.706 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:07:55.761 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:07:55.769 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:07:55.770 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:07:55.871 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:07:55.903 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:07:55.979 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:07:55.984 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:07:55.985 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:07:56.086 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:07:56.213 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:07:56.234 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_blocked_ops 2023-09-23T21:07:56.324 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0: {} 2023-09-23T21:07:56.329 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:07:56.346 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:07:56.347 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:07:56.375 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1: {} 2023-09-23T21:07:56.431 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2: {} 2023-09-23T21:07:56.448 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:07:56.497 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3: {} 2023-09-23T21:07:56.536 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4: {} 2023-09-23T21:07:56.564 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:07:56.573 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T21:07:56.610 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6: {} 2023-09-23T21:07:56.636 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:07:56 smithi026 ceph-mon[175151]: pgmap v977: 109 pgs: 109 active+clean; 5.6 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 1.00k op/s 2023-09-23T21:07:56.636 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:07:56 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/482970410' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:07:56.636 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:07:56 smithi026 ceph-mon[179688]: pgmap v977: 109 pgs: 109 active+clean; 5.6 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 1.00k op/s 2023-09-23T21:07:56.636 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:07:56 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/482970410' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:07:56.649 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T21:07:56.678 INFO:teuthology.orchestra.run.smithi026.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T21:07:56.679 INFO:teuthology.orchestra.run.smithi026.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T21:07:56.679 INFO:teuthology.orchestra.run.smithi026.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T21:07:56.679 INFO:teuthology.orchestra.run.smithi026.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T21:07:56.679 INFO:teuthology.orchestra.run.smithi026.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T21:07:56.679 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T21:07:56.679 INFO:teuthology.orchestra.run.smithi026.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T21:07:56.679 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T21:07:56.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:07:56 smithi070 ceph-mon[168282]: pgmap v977: 109 pgs: 109 active+clean; 5.6 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 1.00k op/s 2023-09-23T21:07:56.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:07:56 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/482970410' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:07:57.340 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:07:57.350 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:07:57.350 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:07:57.451 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:07:57.491 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:07:57.536 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:07:57.538 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:07:57.539 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:07:57.640 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:07:57.673 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:07:57.726 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:07:57.733 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:07:57.733 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:07:57.834 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:07:57.866 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:07:57.920 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:07:57.927 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:07:57.928 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:07:58.028 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:07:58.064 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:07:58.124 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:07:58.133 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:07:58.133 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:07:58.134 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:07:58.234 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:07:58.283 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:07:58.334 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:07:58.336 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:07:58.337 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:07:58.437 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:07:58.475 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:07:58.524 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:07:58.529 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:07:58.530 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:07:58.546 INFO:teuthology.orchestra.run.smithi026.stderr:nodeep-scrub is unset 2023-09-23T21:07:58.592 INFO:teuthology.orchestra.run.smithi026.stdout: 2023-09-23T21:07:58.600 INFO:teuthology.orchestra.run.smithi026.stderr:dumped all 2023-09-23T21:07:58.631 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:07:58.719 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:07:58 smithi026 ceph-mon[175151]: pgmap v978: 109 pgs: 109 active+clean; 5.6 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 699 op/s 2023-09-23T21:07:58.719 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:07:58 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/3626050738' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:07:58.720 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:07:58 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:07:58.720 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:07:58 smithi026 ceph-mon[179688]: pgmap v978: 109 pgs: 109 active+clean; 5.6 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 699 op/s 2023-09-23T21:07:58.720 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:07:58 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/3626050738' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:07:58.720 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:07:58 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:07:58.724 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:07:58.797 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:07:58.805 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:07:58.806 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:07:58.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:07:58 smithi070 ceph-mon[168282]: pgmap v978: 109 pgs: 109 active+clean; 5.6 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 699 op/s 2023-09-23T21:07:58.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:07:58 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/3626050738' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:07:58.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:07:58 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:07:58.907 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:07:58.967 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:07:59.202 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T21:07:59.343 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:07:59.350 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:07:59.351 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:07:59.452 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:07:59.492 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:07:59.532 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:07:59.542 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:07:59.543 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:07:59.643 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:07:59.676 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:07:59.692 INFO:teuthology.orchestra.run.smithi026.stdout:{"pg_ready":true,"pg_map":{"version":979,"stamp":"2023-09-23T21:07:58.533515+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":5832972216,"num_objects":89020,"num_object_clones":0,"num_object_copies":267060,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":89020,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":137613,"num_write_kb":7249502,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":39,"num_bytes_recovered":7154,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":117588,"ondisk_log_size":117588,"up":327,"acting":327,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":327,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":20230464,"kb_used_data":17140032,"kb_used_omap":137,"kb_used_meta":3089654,"kb_avail":729730752,"statfs":{"total":767960285184,"available":747244290048,"internally_reserved":0,"allocated":17551392768,"data_stored":17540509421,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":141000,"internal_metadata":3163806008},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[8,1,3,6,5,1,1],"upper_bound":128},"perf_stat":{"commit_latency_ms":6,"apply_latency_ms":6,"commit_latency_ns":6000000,"apply_latency_ns":6000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-441778176,"num_objects":-6741,"num_object_clones":0,"num_object_copies":-20223,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":-6741,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6741,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"8.122096"},"pg_stats":[{"pgid":"3.1a","version":"68'1","reported_seq":268,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-09-23T21:07:50.438479+0000","last_change":"2023-09-23T21:05:55.322748+0000","last_active":"2023-09-23T21:07:50.438479+0000","last_peered":"2023-09-23T21:07:50.438479+0000","last_clean":"2023-09-23T21:07:50.438479+0000","last_became_active":"2023-09-23T21:05:55.322498+0000","last_became_peered":"2023-09-23T21:05:55.322498+0000","last_unstale":"2023-09-23T21:07:50.438479+0000","last_undegraded":"2023-09-23T21:07:50.438479+0000","last_fullsized":"2023-09-23T21:07:50.438479+0000","mapping_epoch":350,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":351,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:03:28.049826+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":369,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-09-23T21:07:50.428424+0000","last_change":"2023-09-23T20:54:13.942019+0000","last_active":"2023-09-23T21:07:50.428424+0000","last_peered":"2023-09-23T21:07:50.428424+0000","last_clean":"2023-09-23T21:07:50.428424+0000","last_became_active":"2023-09-23T20:54:13.941647+0000","last_became_peered":"2023-09-23T20:54:13.941647+0000","last_unstale":"2023-09-23T21:07:50.428424+0000","last_undegraded":"2023-09-23T21:07:50.428424+0000","last_fullsized":"2023-09-23T21:07:50.428424+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:30:54.400944+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":346,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-09-23T21:07:50.410230+0000","last_change":"2023-09-23T21:05:55.322068+0000","last_active":"2023-09-23T21:07:50.410230+0000","last_peered":"2023-09-23T21:07:50.410230+0000","last_clean":"2023-09-23T21:07:50.410230+0000","last_became_active":"2023-09-23T21:05:55.321763+0000","last_became_peered":"2023-09-23T21:05:55.321763+0000","last_unstale":"2023-09-23T21:07:50.410230+0000","last_undegraded":"2023-09-23T21:07:50.410230+0000","last_fullsized":"2023-09-23T21:07:50.410230+0000","mapping_epoch":350,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":351,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:01:41.294709+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":363,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-09-23T21:07:50.430418+0000","last_change":"2023-09-23T20:57:44.979811+0000","last_active":"2023-09-23T21:07:50.430418+0000","last_peered":"2023-09-23T21:07:50.430418+0000","last_clean":"2023-09-23T21:07:50.430418+0000","last_became_active":"2023-09-23T20:57:44.979515+0000","last_became_peered":"2023-09-23T20:57:44.979515+0000","last_unstale":"2023-09-23T21:07:50.430418+0000","last_undegraded":"2023-09-23T21:07:50.430418+0000","last_fullsized":"2023-09-23T21:07:50.430418+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:38:22.162140+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":375,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-09-23T21:07:50.428160+0000","last_change":"2023-09-23T21:05:55.325654+0000","last_active":"2023-09-23T21:07:50.428160+0000","last_peered":"2023-09-23T21:07:50.428160+0000","last_clean":"2023-09-23T21:07:50.428160+0000","last_became_active":"2023-09-23T21:05:55.325298+0000","last_became_peered":"2023-09-23T21:05:55.325298+0000","last_unstale":"2023-09-23T21:07:50.428160+0000","last_undegraded":"2023-09-23T21:07:50.428160+0000","last_fullsized":"2023-09-23T21:07:50.428160+0000","mapping_epoch":350,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":351,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:33:19.561063+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":376,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-09-23T21:07:50.436578+0000","last_change":"2023-09-23T21:05:55.319892+0000","last_active":"2023-09-23T21:07:50.436578+0000","last_peered":"2023-09-23T21:07:50.436578+0000","last_clean":"2023-09-23T21:07:50.436578+0000","last_became_active":"2023-09-23T21:05:55.319536+0000","last_became_peered":"2023-09-23T21:05:55.319536+0000","last_unstale":"2023-09-23T21:07:50.436578+0000","last_undegraded":"2023-09-23T21:07:50.436578+0000","last_fullsized":"2023-09-23T21:07:50.436578+0000","mapping_epoch":350,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":351,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:14:48.388831+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":370,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-09-23T21:07:50.429121+0000","last_change":"2023-09-23T21:05:04.304358+0000","last_active":"2023-09-23T21:07:50.429121+0000","last_peered":"2023-09-23T21:07:50.429121+0000","last_clean":"2023-09-23T21:07:50.429121+0000","last_became_active":"2023-09-23T21:05:04.303913+0000","last_became_peered":"2023-09-23T21:05:04.303913+0000","last_unstale":"2023-09-23T21:07:50.429121+0000","last_undegraded":"2023-09-23T21:07:50.429121+0000","last_fullsized":"2023-09-23T21:07:50.429121+0000","mapping_epoch":332,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:21:21.860944+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":373,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-09-23T21:07:50.428188+0000","last_change":"2023-09-23T21:05:55.323506+0000","last_active":"2023-09-23T21:07:50.428188+0000","last_peered":"2023-09-23T21:07:50.428188+0000","last_clean":"2023-09-23T21:07:50.428188+0000","last_became_active":"2023-09-23T21:05:55.323167+0000","last_became_peered":"2023-09-23T21:05:55.323167+0000","last_unstale":"2023-09-23T21:07:50.428188+0000","last_undegraded":"2023-09-23T21:07:50.428188+0000","last_fullsized":"2023-09-23T21:07:50.428188+0000","mapping_epoch":350,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":351,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:06:00.628349+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"79'161","reported_seq":664,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-09-23T21:07:50.431583+0000","last_change":"2023-09-23T20:57:25.096559+0000","last_active":"2023-09-23T21:07:50.431583+0000","last_peered":"2023-09-23T21:07:50.431583+0000","last_clean":"2023-09-23T21:07:50.431583+0000","last_became_active":"2023-09-23T20:57:15.975674+0000","last_became_peered":"2023-09-23T20:57:15.975674+0000","last_unstale":"2023-09-23T21:07:50.431583+0000","last_undegraded":"2023-09-23T21:07:50.431583+0000","last_fullsized":"2023-09-23T21:07:50.431583+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:25.096401+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:25.096401+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:05:05.448404+0000","scrub_duration":0.002859393,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":68,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-09-23T21:07:50.415780+0000","last_change":"2023-09-23T21:05:55.325024+0000","last_active":"2023-09-23T21:07:50.415780+0000","last_peered":"2023-09-23T21:07:50.415780+0000","last_clean":"2023-09-23T21:07:50.415780+0000","last_became_active":"2023-09-23T21:05:55.322890+0000","last_became_peered":"2023-09-23T21:05:55.322890+0000","last_unstale":"2023-09-23T21:07:50.415780+0000","last_undegraded":"2023-09-23T21:07:50.415780+0000","last_fullsized":"2023-09-23T21:07:50.415780+0000","mapping_epoch":350,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":351,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:49:56.913769+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":319,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-09-23T21:07:50.430812+0000","last_change":"2023-09-23T20:55:35.952661+0000","last_active":"2023-09-23T21:07:50.430812+0000","last_peered":"2023-09-23T21:07:50.430812+0000","last_clean":"2023-09-23T21:07:50.430812+0000","last_became_active":"2023-09-23T20:55:35.952349+0000","last_became_peered":"2023-09-23T20:55:35.952349+0000","last_unstale":"2023-09-23T21:07:50.430812+0000","last_undegraded":"2023-09-23T21:07:50.430812+0000","last_fullsized":"2023-09-23T21:07:50.430812+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:00:03.110126+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":195,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-09-23T21:07:50.428562+0000","last_change":"2023-09-23T20:56:45.988607+0000","last_active":"2023-09-23T21:07:50.428562+0000","last_peered":"2023-09-23T21:07:50.428562+0000","last_clean":"2023-09-23T21:07:50.428562+0000","last_became_active":"2023-09-23T20:56:45.988206+0000","last_became_peered":"2023-09-23T20:56:45.988206+0000","last_unstale":"2023-09-23T21:07:50.428562+0000","last_undegraded":"2023-09-23T21:07:50.428562+0000","last_fullsized":"2023-09-23T21:07:50.428562+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":205,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:01:19.783688+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,4],"acting":[3,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":46,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-09-23T21:07:50.409909+0000","last_change":"2023-09-23T21:05:55.315641+0000","last_active":"2023-09-23T21:07:50.409909+0000","last_peered":"2023-09-23T21:07:50.409909+0000","last_clean":"2023-09-23T21:07:50.409909+0000","last_became_active":"2023-09-23T21:05:55.315336+0000","last_became_peered":"2023-09-23T21:05:55.315336+0000","last_unstale":"2023-09-23T21:07:50.409909+0000","last_undegraded":"2023-09-23T21:07:50.409909+0000","last_fullsized":"2023-09-23T21:07:50.409909+0000","mapping_epoch":350,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":351,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:16:47.607565+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"79'22","reported_seq":278,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-09-23T21:07:50.439086+0000","last_change":"2023-09-23T20:55:35.949418+0000","last_active":"2023-09-23T21:07:50.439086+0000","last_peered":"2023-09-23T21:07:50.439086+0000","last_clean":"2023-09-23T21:07:50.439086+0000","last_became_active":"2023-09-23T20:55:35.948566+0000","last_became_peered":"2023-09-23T20:55:35.948566+0000","last_unstale":"2023-09-23T21:07:50.439086+0000","last_undegraded":"2023-09-23T21:07:50.439086+0000","last_fullsized":"2023-09-23T21:07:50.439086+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:59:37.433128+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4734,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":285,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-09-23T21:07:50.438757+0000","last_change":"2023-09-23T21:05:55.325203+0000","last_active":"2023-09-23T21:07:50.438757+0000","last_peered":"2023-09-23T21:07:50.438757+0000","last_clean":"2023-09-23T21:07:50.438757+0000","last_became_active":"2023-09-23T21:05:55.323957+0000","last_became_peered":"2023-09-23T21:05:55.323957+0000","last_unstale":"2023-09-23T21:07:50.438757+0000","last_undegraded":"2023-09-23T21:07:50.438757+0000","last_fullsized":"2023-09-23T21:07:50.438757+0000","mapping_epoch":350,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":351,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:03:20.418924+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.11","version":"79'161","reported_seq":640,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-09-23T21:07:50.428528+0000","last_change":"2023-09-23T21:05:07.301125+0000","last_active":"2023-09-23T21:07:50.428528+0000","last_peered":"2023-09-23T21:07:50.428528+0000","last_clean":"2023-09-23T21:07:50.428528+0000","last_became_active":"2023-09-23T21:05:07.300809+0000","last_became_peered":"2023-09-23T21:05:07.300809+0000","last_unstale":"2023-09-23T21:07:50.428528+0000","last_undegraded":"2023-09-23T21:07:50.428528+0000","last_fullsized":"2023-09-23T21:07:50.428528+0000","mapping_epoch":335,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":336,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:28.756634+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:28.756634+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:28:29.079665+0000","scrub_duration":0.0064925119999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":228,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-09-23T21:07:50.429145+0000","last_change":"2023-09-23T21:06:25.394796+0000","last_active":"2023-09-23T21:07:50.429145+0000","last_peered":"2023-09-23T21:07:50.429145+0000","last_clean":"2023-09-23T21:07:50.429145+0000","last_became_active":"2023-09-23T21:06:25.394447+0000","last_became_peered":"2023-09-23T21:06:25.394447+0000","last_unstale":"2023-09-23T21:07:50.429145+0000","last_undegraded":"2023-09-23T21:07:50.429145+0000","last_fullsized":"2023-09-23T21:07:50.429145+0000","mapping_epoch":354,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":355,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:55:25.161895+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":358,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-09-23T21:07:50.428619+0000","last_change":"2023-09-23T21:05:55.324614+0000","last_active":"2023-09-23T21:07:50.428619+0000","last_peered":"2023-09-23T21:07:50.428619+0000","last_clean":"2023-09-23T21:07:50.428619+0000","last_became_active":"2023-09-23T21:05:55.324306+0000","last_became_peered":"2023-09-23T21:05:55.324306+0000","last_unstale":"2023-09-23T21:07:50.428619+0000","last_undegraded":"2023-09-23T21:07:50.428619+0000","last_fullsized":"2023-09-23T21:07:50.428619+0000","mapping_epoch":350,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":351,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:32:58.417049+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.10","version":"79'161","reported_seq":466,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-09-23T21:07:50.436498+0000","last_change":"2023-09-23T21:06:50.337099+0000","last_active":"2023-09-23T21:07:50.436498+0000","last_peered":"2023-09-23T21:07:50.436498+0000","last_clean":"2023-09-23T21:07:50.436498+0000","last_became_active":"2023-09-23T21:06:49.426789+0000","last_became_peered":"2023-09-23T21:06:49.426789+0000","last_unstale":"2023-09-23T21:07:50.436498+0000","last_undegraded":"2023-09-23T21:07:50.436498+0000","last_fullsized":"2023-09-23T21:07:50.436498+0000","mapping_epoch":359,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":360,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:06:50.336992+0000","last_deep_scrub":"79'161","last_deep_scrub_stamp":"2023-09-23T21:06:50.336992+0000","last_clean_scrub_stamp":"2023-09-23T21:06:50.336992+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:26:47.504229+0000","scrub_duration":0.0026150930000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,7],"acting":[6,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":352,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-09-23T21:07:50.430647+0000","last_change":"2023-09-23T20:57:44.979736+0000","last_active":"2023-09-23T21:07:50.430647+0000","last_peered":"2023-09-23T21:07:50.430647+0000","last_clean":"2023-09-23T21:07:50.430647+0000","last_became_active":"2023-09-23T20:57:44.979356+0000","last_became_peered":"2023-09-23T20:57:44.979356+0000","last_unstale":"2023-09-23T21:07:50.430647+0000","last_undegraded":"2023-09-23T21:07:50.430647+0000","last_fullsized":"2023-09-23T21:07:50.430647+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:14:43.848832+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":96,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-09-23T21:07:50.428333+0000","last_change":"2023-09-23T21:06:49.426186+0000","last_active":"2023-09-23T21:07:50.428333+0000","last_peered":"2023-09-23T21:07:50.428333+0000","last_clean":"2023-09-23T21:07:50.428333+0000","last_became_active":"2023-09-23T21:06:49.425867+0000","last_became_peered":"2023-09-23T21:06:49.425867+0000","last_unstale":"2023-09-23T21:07:50.428333+0000","last_undegraded":"2023-09-23T21:07:50.428333+0000","last_fullsized":"2023-09-23T21:07:50.428333+0000","mapping_epoch":359,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":360,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:40:21.708447+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.13","version":"79'161","reported_seq":647,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-09-23T21:07:50.410325+0000","last_change":"2023-09-23T21:05:55.322760+0000","last_active":"2023-09-23T21:07:50.410325+0000","last_peered":"2023-09-23T21:07:50.410325+0000","last_clean":"2023-09-23T21:07:50.410325+0000","last_became_active":"2023-09-23T21:05:55.322483+0000","last_became_peered":"2023-09-23T21:05:55.322483+0000","last_unstale":"2023-09-23T21:07:50.410325+0000","last_undegraded":"2023-09-23T21:07:50.410325+0000","last_fullsized":"2023-09-23T21:07:50.410325+0000","mapping_epoch":350,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":351,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:36.420362+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:36.420362+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:08:05.220846+0000","scrub_duration":0.028838444000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":318,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-09-23T21:07:50.438402+0000","last_change":"2023-09-23T20:56:45.991822+0000","last_active":"2023-09-23T21:07:50.438402+0000","last_peered":"2023-09-23T21:07:50.438402+0000","last_clean":"2023-09-23T21:07:50.438402+0000","last_became_active":"2023-09-23T20:56:45.991460+0000","last_became_peered":"2023-09-23T20:56:45.991460+0000","last_unstale":"2023-09-23T21:07:50.438402+0000","last_undegraded":"2023-09-23T21:07:50.438402+0000","last_fullsized":"2023-09-23T21:07:50.438402+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":205,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:20:37.503317+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":334,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-09-23T21:07:50.430524+0000","last_change":"2023-09-23T20:51:10.840849+0000","last_active":"2023-09-23T21:07:50.430524+0000","last_peered":"2023-09-23T21:07:50.430524+0000","last_clean":"2023-09-23T21:07:50.430524+0000","last_became_active":"2023-09-23T20:51:10.840593+0000","last_became_peered":"2023-09-23T20:51:10.840593+0000","last_unstale":"2023-09-23T21:07:50.430524+0000","last_undegraded":"2023-09-23T21:07:50.430524+0000","last_fullsized":"2023-09-23T21:07:50.430524+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:30:52.297625+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.12","version":"79'161","reported_seq":535,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-09-23T21:07:50.415892+0000","last_change":"2023-09-23T21:05:56.785264+0000","last_active":"2023-09-23T21:07:50.415892+0000","last_peered":"2023-09-23T21:07:50.415892+0000","last_clean":"2023-09-23T21:07:50.415892+0000","last_became_active":"2023-09-23T21:05:55.322810+0000","last_became_peered":"2023-09-23T21:05:55.322810+0000","last_unstale":"2023-09-23T21:07:50.415892+0000","last_undegraded":"2023-09-23T21:07:50.415892+0000","last_fullsized":"2023-09-23T21:07:50.415892+0000","mapping_epoch":350,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":351,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:05:56.785173+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:05:56.785173+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:56:12.932786+0000","scrub_duration":0.0042114370000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":406,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-09-23T21:07:50.436671+0000","last_change":"2023-09-23T21:05:04.332774+0000","last_active":"2023-09-23T21:07:50.436671+0000","last_peered":"2023-09-23T21:07:50.436671+0000","last_clean":"2023-09-23T21:07:50.436671+0000","last_became_active":"2023-09-23T21:05:04.304541+0000","last_became_peered":"2023-09-23T21:05:04.304541+0000","last_unstale":"2023-09-23T21:07:50.436671+0000","last_undegraded":"2023-09-23T21:07:50.436671+0000","last_fullsized":"2023-09-23T21:07:50.436671+0000","mapping_epoch":332,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:27:02.401939+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":181,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-09-23T21:07:50.433190+0000","last_change":"2023-09-23T21:05:55.325421+0000","last_active":"2023-09-23T21:07:50.433190+0000","last_peered":"2023-09-23T21:07:50.433190+0000","last_clean":"2023-09-23T21:07:50.433190+0000","last_became_active":"2023-09-23T21:05:55.325098+0000","last_became_peered":"2023-09-23T21:05:55.325098+0000","last_unstale":"2023-09-23T21:07:50.433190+0000","last_undegraded":"2023-09-23T21:07:50.433190+0000","last_fullsized":"2023-09-23T21:07:50.433190+0000","mapping_epoch":350,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":351,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:57:47.564327+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.5","version":"374'8597","reported_seq":8635,"reported_epoch":374,"state":"active+clean","last_fresh":"2023-09-23T21:07:57.298511+0000","last_change":"2023-09-23T21:05:48.557604+0000","last_active":"2023-09-23T21:07:57.298511+0000","last_peered":"2023-09-23T21:07:57.298511+0000","last_clean":"2023-09-23T21:07:57.298511+0000","last_became_active":"2023-09-23T21:05:48.557179+0000","last_became_peered":"2023-09-23T21:05:48.557179+0000","last_unstale":"2023-09-23T21:07:57.298511+0000","last_undegraded":"2023-09-23T21:07:57.298511+0000","last_fullsized":"2023-09-23T21:07:57.298511+0000","mapping_epoch":346,"log_start":"351'200","ondisk_log_start":"351'200","created":346,"last_epoch_clean":347,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:05:47.261750+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:05:47.261750+0000","last_clean_scrub_stamp":"2023-09-23T21:05:47.261750+0000","objects_scrubbed":0,"log_size":8397,"log_dups_size":0,"ondisk_log_size":8397,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:17:18.270981+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":356712448,"num_objects":5443,"num_object_clones":0,"num_object_copies":16329,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5443,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8597,"num_write_kb":449280,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.d","version":"79'161","reported_seq":511,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-09-23T21:07:50.429553+0000","last_change":"2023-09-23T21:06:38.762807+0000","last_active":"2023-09-23T21:07:50.429553+0000","last_peered":"2023-09-23T21:07:50.429553+0000","last_clean":"2023-09-23T21:07:50.429553+0000","last_became_active":"2023-09-23T21:06:25.394249+0000","last_became_peered":"2023-09-23T21:06:25.394249+0000","last_unstale":"2023-09-23T21:07:50.429553+0000","last_undegraded":"2023-09-23T21:07:50.429553+0000","last_fullsized":"2023-09-23T21:07:50.429553+0000","mapping_epoch":354,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":355,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:06:38.762742+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:06:38.762742+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:28:31.666726+0000","scrub_duration":0.0053529800000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":237,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-09-23T21:07:50.429478+0000","last_change":"2023-09-23T21:06:25.394656+0000","last_active":"2023-09-23T21:07:50.429478+0000","last_peered":"2023-09-23T21:07:50.429478+0000","last_clean":"2023-09-23T21:07:50.429478+0000","last_became_active":"2023-09-23T21:06:25.394290+0000","last_became_peered":"2023-09-23T21:06:25.394290+0000","last_unstale":"2023-09-23T21:07:50.429478+0000","last_undegraded":"2023-09-23T21:07:50.429478+0000","last_fullsized":"2023-09-23T21:07:50.429478+0000","mapping_epoch":354,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":355,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:27:19.996925+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":66,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":51,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-09-23T21:07:50.415373+0000","last_change":"2023-09-23T21:05:04.307967+0000","last_active":"2023-09-23T21:07:50.415373+0000","last_peered":"2023-09-23T21:07:50.415373+0000","last_clean":"2023-09-23T21:07:50.415373+0000","last_became_active":"2023-09-23T21:05:04.307679+0000","last_became_peered":"2023-09-23T21:05:04.307679+0000","last_unstale":"2023-09-23T21:07:50.415373+0000","last_undegraded":"2023-09-23T21:07:50.415373+0000","last_fullsized":"2023-09-23T21:07:50.415373+0000","mapping_epoch":332,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:39:58.603484+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.4","version":"374'8669","reported_seq":8703,"reported_epoch":374,"state":"active+clean","last_fresh":"2023-09-23T21:07:58.426794+0000","last_change":"2023-09-23T21:05:55.320259+0000","last_active":"2023-09-23T21:07:58.426794+0000","last_peered":"2023-09-23T21:07:58.426794+0000","last_clean":"2023-09-23T21:07:58.426794+0000","last_became_active":"2023-09-23T21:05:55.319895+0000","last_became_peered":"2023-09-23T21:05:55.319895+0000","last_unstale":"2023-09-23T21:07:58.426794+0000","last_undegraded":"2023-09-23T21:07:58.426794+0000","last_fullsized":"2023-09-23T21:07:58.426794+0000","mapping_epoch":350,"log_start":"352'2100","ondisk_log_start":"352'2100","created":346,"last_epoch_clean":351,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:05:47.261750+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:05:47.261750+0000","last_clean_scrub_stamp":"2023-09-23T21:05:47.261750+0000","objects_scrubbed":0,"log_size":6569,"log_dups_size":0,"ondisk_log_size":6569,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:39:53.181523+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":350027776,"num_objects":5341,"num_object_clones":0,"num_object_copies":16023,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5341,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8669,"num_write_kb":448320,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.c","version":"79'161","reported_seq":678,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-09-23T21:07:50.428354+0000","last_change":"2023-09-23T20:57:22.041071+0000","last_active":"2023-09-23T21:07:50.428354+0000","last_peered":"2023-09-23T21:07:50.428354+0000","last_clean":"2023-09-23T21:07:50.428354+0000","last_became_active":"2023-09-23T20:57:15.973204+0000","last_became_peered":"2023-09-23T20:57:15.973204+0000","last_unstale":"2023-09-23T21:07:50.428354+0000","last_undegraded":"2023-09-23T21:07:50.428354+0000","last_fullsized":"2023-09-23T21:07:50.428354+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:22.040907+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:22.040907+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:47:02.019978+0000","scrub_duration":0.013021176000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":224,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-09-23T21:07:50.429431+0000","last_change":"2023-09-23T21:06:25.394978+0000","last_active":"2023-09-23T21:07:50.429431+0000","last_peered":"2023-09-23T21:07:50.429431+0000","last_clean":"2023-09-23T21:07:50.429431+0000","last_became_active":"2023-09-23T21:06:25.394599+0000","last_became_peered":"2023-09-23T21:06:25.394599+0000","last_unstale":"2023-09-23T21:07:50.429431+0000","last_undegraded":"2023-09-23T21:07:50.429431+0000","last_fullsized":"2023-09-23T21:07:50.429431+0000","mapping_epoch":354,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":355,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:52:23.477998+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":312,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-09-23T21:07:50.438554+0000","last_change":"2023-09-23T21:05:55.325375+0000","last_active":"2023-09-23T21:07:50.438554+0000","last_peered":"2023-09-23T21:07:50.438554+0000","last_clean":"2023-09-23T21:07:50.438554+0000","last_became_active":"2023-09-23T21:05:55.324170+0000","last_became_peered":"2023-09-23T21:05:55.324170+0000","last_unstale":"2023-09-23T21:07:50.438554+0000","last_undegraded":"2023-09-23T21:07:50.438554+0000","last_fullsized":"2023-09-23T21:07:50.438554+0000","mapping_epoch":350,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":351,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:08:07.299605+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.7","version":"374'8484","reported_seq":8528,"reported_epoch":374,"state":"active+clean","last_fresh":"2023-09-23T21:07:53.875319+0000","last_change":"2023-09-23T21:06:49.432007+0000","last_active":"2023-09-23T21:07:53.875319+0000","last_peered":"2023-09-23T21:07:53.875319+0000","last_clean":"2023-09-23T21:07:53.875319+0000","last_became_active":"2023-09-23T21:06:49.425988+0000","last_became_peered":"2023-09-23T21:06:49.425988+0000","last_unstale":"2023-09-23T21:07:53.875319+0000","last_undegraded":"2023-09-23T21:07:53.875319+0000","last_fullsized":"2023-09-23T21:07:53.875319+0000","mapping_epoch":359,"log_start":"352'2200","ondisk_log_start":"352'2200","created":346,"last_epoch_clean":360,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:05:47.261750+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:05:47.261750+0000","last_clean_scrub_stamp":"2023-09-23T21:05:47.261750+0000","objects_scrubbed":0,"log_size":6284,"log_dups_size":0,"ondisk_log_size":6284,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:37:23.206892+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":370671616,"num_objects":5656,"num_object_clones":0,"num_object_copies":16968,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5656,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8484,"num_write_kb":452480,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.f","version":"79'161","reported_seq":718,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-09-23T21:07:50.436751+0000","last_change":"2023-09-23T21:05:07.299402+0000","last_active":"2023-09-23T21:07:50.436751+0000","last_peered":"2023-09-23T21:07:50.436751+0000","last_clean":"2023-09-23T21:07:50.436751+0000","last_became_active":"2023-09-23T21:05:07.299059+0000","last_became_peered":"2023-09-23T21:05:07.299059+0000","last_unstale":"2023-09-23T21:07:50.436751+0000","last_undegraded":"2023-09-23T21:07:50.436751+0000","last_fullsized":"2023-09-23T21:07:50.436751+0000","mapping_epoch":335,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":336,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:27.880776+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:27.880776+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:51:23.907187+0000","scrub_duration":0.0014004899999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":55,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-09-23T21:07:50.417496+0000","last_change":"2023-09-23T21:05:04.355129+0000","last_active":"2023-09-23T21:07:50.417496+0000","last_peered":"2023-09-23T21:07:50.417496+0000","last_clean":"2023-09-23T21:07:50.417496+0000","last_became_active":"2023-09-23T21:05:04.326482+0000","last_became_peered":"2023-09-23T21:05:04.326482+0000","last_unstale":"2023-09-23T21:07:50.417496+0000","last_undegraded":"2023-09-23T21:07:50.417496+0000","last_fullsized":"2023-09-23T21:07:50.417496+0000","mapping_epoch":332,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:35:10.887749+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":330,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-09-23T21:07:50.436009+0000","last_change":"2023-09-23T21:05:55.321246+0000","last_active":"2023-09-23T21:07:50.436009+0000","last_peered":"2023-09-23T21:07:50.436009+0000","last_clean":"2023-09-23T21:07:50.436009+0000","last_became_active":"2023-09-23T21:05:55.320918+0000","last_became_peered":"2023-09-23T21:05:55.320918+0000","last_unstale":"2023-09-23T21:07:50.436009+0000","last_undegraded":"2023-09-23T21:07:50.436009+0000","last_fullsized":"2023-09-23T21:07:50.436009+0000","mapping_epoch":350,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":351,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:48:50.646324+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"9.6","version":"374'8577","reported_seq":8628,"reported_epoch":374,"state":"active+clean","last_fresh":"2023-09-23T21:07:55.904690+0000","last_change":"2023-09-23T21:06:49.429119+0000","last_active":"2023-09-23T21:07:55.904690+0000","last_peered":"2023-09-23T21:07:55.904690+0000","last_clean":"2023-09-23T21:07:55.904690+0000","last_became_active":"2023-09-23T21:06:49.423016+0000","last_became_peered":"2023-09-23T21:06:49.423016+0000","last_unstale":"2023-09-23T21:07:55.904690+0000","last_undegraded":"2023-09-23T21:07:55.904690+0000","last_fullsized":"2023-09-23T21:07:55.904690+0000","mapping_epoch":359,"log_start":"0'0","ondisk_log_start":"0'0","created":346,"last_epoch_clean":360,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:05:47.261750+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:05:47.261750+0000","last_clean_scrub_stamp":"2023-09-23T21:05:47.261750+0000","objects_scrubbed":0,"log_size":8577,"log_dups_size":0,"ondisk_log_size":8577,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:55:25.118949+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":361824256,"num_objects":5521,"num_object_clones":0,"num_object_copies":16563,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5521,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8577,"num_write_kb":451136,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,1],"acting":[2,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.e","version":"79'161","reported_seq":585,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-09-23T21:07:50.438828+0000","last_change":"2023-09-23T20:57:34.132738+0000","last_active":"2023-09-23T21:07:50.438828+0000","last_peered":"2023-09-23T21:07:50.438828+0000","last_clean":"2023-09-23T21:07:50.438828+0000","last_became_active":"2023-09-23T20:57:15.974318+0000","last_became_peered":"2023-09-23T20:57:15.974318+0000","last_unstale":"2023-09-23T21:07:50.438828+0000","last_undegraded":"2023-09-23T21:07:50.438828+0000","last_fullsized":"2023-09-23T21:07:50.438828+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:34.132660+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:34.132660+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:07:07.739438+0000","scrub_duration":0.005229694,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":139,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-09-23T21:07:50.428812+0000","last_change":"2023-09-23T21:06:49.423445+0000","last_active":"2023-09-23T21:07:50.428812+0000","last_peered":"2023-09-23T21:07:50.428812+0000","last_clean":"2023-09-23T21:07:50.428812+0000","last_became_active":"2023-09-23T21:06:49.423169+0000","last_became_peered":"2023-09-23T21:06:49.423169+0000","last_unstale":"2023-09-23T21:07:50.428812+0000","last_undegraded":"2023-09-23T21:07:50.428812+0000","last_fullsized":"2023-09-23T21:07:50.428812+0000","mapping_epoch":359,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":360,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:47:52.999823+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":366,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-09-23T21:07:50.430802+0000","last_change":"2023-09-23T20:54:13.928715+0000","last_active":"2023-09-23T21:07:50.430802+0000","last_peered":"2023-09-23T21:07:50.430802+0000","last_clean":"2023-09-23T21:07:50.430802+0000","last_became_active":"2023-09-23T20:54:13.928200+0000","last_became_peered":"2023-09-23T20:54:13.928200+0000","last_unstale":"2023-09-23T21:07:50.430802+0000","last_undegraded":"2023-09-23T21:07:50.430802+0000","last_fullsized":"2023-09-23T21:07:50.430802+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:41:16.927513+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.1","version":"374'8590","reported_seq":8628,"reported_epoch":374,"state":"active+clean","last_fresh":"2023-09-23T21:07:53.914332+0000","last_change":"2023-09-23T21:05:48.558005+0000","last_active":"2023-09-23T21:07:53.914332+0000","last_peered":"2023-09-23T21:07:53.914332+0000","last_clean":"2023-09-23T21:07:53.914332+0000","last_became_active":"2023-09-23T21:05:48.557590+0000","last_became_peered":"2023-09-23T21:05:48.557590+0000","last_unstale":"2023-09-23T21:07:53.914332+0000","last_undegraded":"2023-09-23T21:07:53.914332+0000","last_fullsized":"2023-09-23T21:07:53.914332+0000","mapping_epoch":346,"log_start":"352'2300","ondisk_log_start":"352'2300","created":346,"last_epoch_clean":347,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:05:47.261750+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:05:47.261750+0000","last_clean_scrub_stamp":"2023-09-23T21:05:47.261750+0000","objects_scrubbed":0,"log_size":6290,"log_dups_size":0,"ondisk_log_size":6290,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:19:41.388978+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":376832000,"num_objects":5750,"num_object_clones":0,"num_object_copies":17250,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5750,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8590,"num_write_kb":458880,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.9","version":"79'161","reported_seq":751,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-09-23T21:07:50.430737+0000","last_change":"2023-09-23T20:56:46.921939+0000","last_active":"2023-09-23T21:07:50.430737+0000","last_peered":"2023-09-23T21:07:50.430737+0000","last_clean":"2023-09-23T21:07:50.430737+0000","last_became_active":"2023-09-23T20:56:45.989908+0000","last_became_peered":"2023-09-23T20:56:45.989908+0000","last_unstale":"2023-09-23T21:07:50.430737+0000","last_undegraded":"2023-09-23T21:07:50.430737+0000","last_fullsized":"2023-09-23T21:07:50.430737+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":205,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:56:46.921739+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:56:46.921739+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:20:42.635110+0000","scrub_duration":0.0071954109999999996,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":367,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-09-23T21:07:50.429254+0000","last_change":"2023-09-23T20:54:13.961788+0000","last_active":"2023-09-23T21:07:50.429254+0000","last_peered":"2023-09-23T21:07:50.429254+0000","last_clean":"2023-09-23T21:07:50.429254+0000","last_became_active":"2023-09-23T20:54:13.940937+0000","last_became_peered":"2023-09-23T20:54:13.940937+0000","last_unstale":"2023-09-23T21:07:50.429254+0000","last_undegraded":"2023-09-23T21:07:50.429254+0000","last_fullsized":"2023-09-23T21:07:50.429254+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:11:56.785299+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":268,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-09-23T21:07:50.439228+0000","last_change":"2023-09-23T20:55:35.949670+0000","last_active":"2023-09-23T21:07:50.439228+0000","last_peered":"2023-09-23T21:07:50.439228+0000","last_clean":"2023-09-23T21:07:50.439228+0000","last_became_active":"2023-09-23T20:55:35.949140+0000","last_became_peered":"2023-09-23T20:55:35.949140+0000","last_unstale":"2023-09-23T21:07:50.439228+0000","last_undegraded":"2023-09-23T21:07:50.439228+0000","last_fullsized":"2023-09-23T21:07:50.439228+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:55:59.340303+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.0","version":"374'8748","reported_seq":8782,"reported_epoch":374,"state":"active+clean","last_fresh":"2023-09-23T21:07:58.403310+0000","last_change":"2023-09-23T21:05:55.315757+0000","last_active":"2023-09-23T21:07:58.403310+0000","last_peered":"2023-09-23T21:07:58.403310+0000","last_clean":"2023-09-23T21:07:58.403310+0000","last_became_active":"2023-09-23T21:05:55.314884+0000","last_became_peered":"2023-09-23T21:05:55.314884+0000","last_unstale":"2023-09-23T21:07:58.403310+0000","last_undegraded":"2023-09-23T21:07:58.403310+0000","last_fullsized":"2023-09-23T21:07:58.403310+0000","mapping_epoch":350,"log_start":"352'2200","ondisk_log_start":"352'2200","created":346,"last_epoch_clean":351,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:05:47.261750+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:05:47.261750+0000","last_clean_scrub_stamp":"2023-09-23T21:05:47.261750+0000","objects_scrubbed":0,"log_size":6548,"log_dups_size":0,"ondisk_log_size":6548,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:41:19.184012+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":354680832,"num_objects":5412,"num_object_clones":0,"num_object_copies":16236,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5412,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8748,"num_write_kb":453120,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.8","version":"79'161","reported_seq":482,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-09-23T21:07:50.431086+0000","last_change":"2023-09-23T21:06:52.099024+0000","last_active":"2023-09-23T21:07:50.431086+0000","last_peered":"2023-09-23T21:07:50.431086+0000","last_clean":"2023-09-23T21:07:50.431086+0000","last_became_active":"2023-09-23T21:06:49.425012+0000","last_became_peered":"2023-09-23T21:06:49.425012+0000","last_unstale":"2023-09-23T21:07:50.431086+0000","last_undegraded":"2023-09-23T21:07:50.431086+0000","last_fullsized":"2023-09-23T21:07:50.431086+0000","mapping_epoch":359,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":360,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:06:52.098882+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:06:52.098882+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:48:36.414396+0000","scrub_duration":0.0092173299999999993,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":353,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-09-23T21:07:50.436170+0000","last_change":"2023-09-23T21:05:55.321230+0000","last_active":"2023-09-23T21:07:50.436170+0000","last_peered":"2023-09-23T21:07:50.436170+0000","last_clean":"2023-09-23T21:07:50.436170+0000","last_became_active":"2023-09-23T21:05:55.320887+0000","last_became_peered":"2023-09-23T21:05:55.320887+0000","last_unstale":"2023-09-23T21:07:50.436170+0000","last_undegraded":"2023-09-23T21:07:50.436170+0000","last_fullsized":"2023-09-23T21:07:50.436170+0000","mapping_epoch":350,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":351,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:54:10.329947+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":268,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-09-23T21:07:50.438831+0000","last_change":"2023-09-23T21:05:55.322692+0000","last_active":"2023-09-23T21:07:50.438831+0000","last_peered":"2023-09-23T21:07:50.438831+0000","last_clean":"2023-09-23T21:07:50.438831+0000","last_became_active":"2023-09-23T21:05:55.322388+0000","last_became_peered":"2023-09-23T21:05:55.322388+0000","last_unstale":"2023-09-23T21:07:50.438831+0000","last_undegraded":"2023-09-23T21:07:50.438831+0000","last_fullsized":"2023-09-23T21:07:50.438831+0000","mapping_epoch":350,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":351,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:43:27.774990+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.3","version":"374'8541","reported_seq":8592,"reported_epoch":374,"state":"active+clean","last_fresh":"2023-09-23T21:07:57.140332+0000","last_change":"2023-09-23T21:06:25.421271+0000","last_active":"2023-09-23T21:07:57.140332+0000","last_peered":"2023-09-23T21:07:57.140332+0000","last_clean":"2023-09-23T21:07:57.140332+0000","last_became_active":"2023-09-23T21:06:25.400342+0000","last_became_peered":"2023-09-23T21:06:25.400342+0000","last_unstale":"2023-09-23T21:07:57.140332+0000","last_undegraded":"2023-09-23T21:07:57.140332+0000","last_fullsized":"2023-09-23T21:07:57.140332+0000","mapping_epoch":354,"log_start":"352'2100","ondisk_log_start":"352'2100","created":346,"last_epoch_clean":355,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:05:47.261750+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:05:47.261750+0000","last_clean_scrub_stamp":"2023-09-23T21:05:47.261750+0000","objects_scrubbed":0,"log_size":6441,"log_dups_size":0,"ondisk_log_size":6441,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:32:39.644536+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":357892096,"num_objects":5461,"num_object_clones":0,"num_object_copies":16383,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5461,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8541,"num_write_kb":448064,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.b","version":"79'161","reported_seq":703,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-09-23T21:07:50.410461+0000","last_change":"2023-09-23T21:05:55.772369+0000","last_active":"2023-09-23T21:07:50.410461+0000","last_peered":"2023-09-23T21:07:50.410461+0000","last_clean":"2023-09-23T21:07:50.410461+0000","last_became_active":"2023-09-23T21:05:55.316405+0000","last_became_peered":"2023-09-23T21:05:55.316405+0000","last_unstale":"2023-09-23T21:07:50.410461+0000","last_undegraded":"2023-09-23T21:07:50.410461+0000","last_fullsized":"2023-09-23T21:07:50.410461+0000","mapping_epoch":350,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":351,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:05:55.772264+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:05:55.772264+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:14:30.697692+0000","scrub_duration":0.001862507,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,5],"acting":[1,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":251,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-09-23T21:07:50.438454+0000","last_change":"2023-09-23T20:54:13.950254+0000","last_active":"2023-09-23T21:07:50.438454+0000","last_peered":"2023-09-23T21:07:50.438454+0000","last_clean":"2023-09-23T21:07:50.438454+0000","last_became_active":"2023-09-23T20:54:13.949934+0000","last_became_peered":"2023-09-23T20:54:13.949934+0000","last_unstale":"2023-09-23T21:07:50.438454+0000","last_undegraded":"2023-09-23T21:07:50.438454+0000","last_fullsized":"2023-09-23T21:07:50.438454+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:30:32.847167+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":285,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-09-23T21:07:50.439039+0000","last_change":"2023-09-23T21:05:55.325695+0000","last_active":"2023-09-23T21:07:50.439039+0000","last_peered":"2023-09-23T21:07:50.439039+0000","last_clean":"2023-09-23T21:07:50.439039+0000","last_became_active":"2023-09-23T21:05:55.324518+0000","last_became_peered":"2023-09-23T21:05:55.324518+0000","last_unstale":"2023-09-23T21:07:50.439039+0000","last_undegraded":"2023-09-23T21:07:50.439039+0000","last_fullsized":"2023-09-23T21:07:50.439039+0000","mapping_epoch":350,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":351,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:57:18.249965+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.2","version":"374'8439","reported_seq":8481,"reported_epoch":374,"state":"active+clean","last_fresh":"2023-09-23T21:07:53.889415+0000","last_change":"2023-09-23T21:05:55.325629+0000","last_active":"2023-09-23T21:07:53.889415+0000","last_peered":"2023-09-23T21:07:53.889415+0000","last_clean":"2023-09-23T21:07:53.889415+0000","last_became_active":"2023-09-23T21:05:55.325108+0000","last_became_peered":"2023-09-23T21:05:55.325108+0000","last_unstale":"2023-09-23T21:07:53.889415+0000","last_undegraded":"2023-09-23T21:07:53.889415+0000","last_fullsized":"2023-09-23T21:07:53.889415+0000","mapping_epoch":350,"log_start":"352'2100","ondisk_log_start":"352'2100","created":346,"last_epoch_clean":351,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:05:47.261750+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:05:47.261750+0000","last_clean_scrub_stamp":"2023-09-23T21:05:47.261750+0000","objects_scrubbed":0,"log_size":6339,"log_dups_size":0,"ondisk_log_size":6339,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:50:06.272050+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":372834304,"num_objects":5689,"num_object_clones":0,"num_object_copies":17067,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5689,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8439,"num_write_kb":452096,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.a","version":"79'161","reported_seq":474,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-09-23T21:07:50.432360+0000","last_change":"2023-09-23T21:06:51.075617+0000","last_active":"2023-09-23T21:07:50.432360+0000","last_peered":"2023-09-23T21:07:50.432360+0000","last_clean":"2023-09-23T21:07:50.432360+0000","last_became_active":"2023-09-23T21:06:49.423291+0000","last_became_peered":"2023-09-23T21:06:49.423291+0000","last_unstale":"2023-09-23T21:07:50.432360+0000","last_undegraded":"2023-09-23T21:07:50.432360+0000","last_fullsized":"2023-09-23T21:07:50.432360+0000","mapping_epoch":359,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":360,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:06:51.075448+0000","last_deep_scrub":"79'161","last_deep_scrub_stamp":"2023-09-23T21:05:32.630292+0000","last_clean_scrub_stamp":"2023-09-23T21:06:51.075448+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:35:01.883490+0000","scrub_duration":0.0054793890000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":181,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-09-23T21:07:50.428034+0000","last_change":"2023-09-23T21:05:55.325999+0000","last_active":"2023-09-23T21:07:50.428034+0000","last_peered":"2023-09-23T21:07:50.428034+0000","last_clean":"2023-09-23T21:07:50.428034+0000","last_became_active":"2023-09-23T21:05:55.325669+0000","last_became_peered":"2023-09-23T21:05:55.325669+0000","last_unstale":"2023-09-23T21:07:50.428034+0000","last_undegraded":"2023-09-23T21:07:50.428034+0000","last_fullsized":"2023-09-23T21:07:50.428034+0000","mapping_epoch":350,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":351,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:30:53.984244+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":346,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-09-23T21:07:50.411768+0000","last_change":"2023-09-23T21:05:55.322566+0000","last_active":"2023-09-23T21:07:50.411768+0000","last_peered":"2023-09-23T21:07:50.411768+0000","last_clean":"2023-09-23T21:07:50.411768+0000","last_became_active":"2023-09-23T21:05:55.322280+0000","last_became_peered":"2023-09-23T21:05:55.322280+0000","last_unstale":"2023-09-23T21:07:50.411768+0000","last_undegraded":"2023-09-23T21:07:50.411768+0000","last_fullsized":"2023-09-23T21:07:50.411768+0000","mapping_epoch":350,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":351,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:48:14.297230+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.f","version":"374'8608","reported_seq":8651,"reported_epoch":374,"state":"active+clean","last_fresh":"2023-09-23T21:07:57.117878+0000","last_change":"2023-09-23T21:05:55.325083+0000","last_active":"2023-09-23T21:07:57.117878+0000","last_peered":"2023-09-23T21:07:57.117878+0000","last_clean":"2023-09-23T21:07:57.117878+0000","last_became_active":"2023-09-23T21:05:55.324768+0000","last_became_peered":"2023-09-23T21:05:55.324768+0000","last_unstale":"2023-09-23T21:07:57.117878+0000","last_undegraded":"2023-09-23T21:07:57.117878+0000","last_fullsized":"2023-09-23T21:07:57.117878+0000","mapping_epoch":350,"log_start":"351'700","ondisk_log_start":"351'700","created":346,"last_epoch_clean":351,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:05:47.261750+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:05:47.261750+0000","last_clean_scrub_stamp":"2023-09-23T21:05:47.261750+0000","objects_scrubbed":0,"log_size":7908,"log_dups_size":0,"ondisk_log_size":7908,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:25:42.316668+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":363593728,"num_objects":5548,"num_object_clones":0,"num_object_copies":16644,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5548,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8608,"num_write_kb":452992,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.7","version":"79'161","reported_seq":679,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-09-23T21:07:50.436569+0000","last_change":"2023-09-23T20:57:48.243572+0000","last_active":"2023-09-23T21:07:50.436569+0000","last_peered":"2023-09-23T21:07:50.436569+0000","last_clean":"2023-09-23T21:07:50.436569+0000","last_became_active":"2023-09-23T20:57:48.243229+0000","last_became_peered":"2023-09-23T20:57:48.243229+0000","last_unstale":"2023-09-23T21:07:50.436569+0000","last_undegraded":"2023-09-23T21:07:50.436569+0000","last_fullsized":"2023-09-23T21:07:50.436569+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":3,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:25.916867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:25.916867+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:23:19.385831+0000","scrub_duration":0.001665323,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":351,"reported_epoch":374,"state":"active+clean","last_fresh":"2023-09-23T21:07:52.504737+0000","last_change":"2023-09-23T21:05:55.317140+0000","last_active":"2023-09-23T21:07:52.504737+0000","last_peered":"2023-09-23T21:07:52.504737+0000","last_clean":"2023-09-23T21:07:52.504737+0000","last_became_active":"2023-09-23T21:05:55.316187+0000","last_became_peered":"2023-09-23T21:05:55.316187+0000","last_unstale":"2023-09-23T21:07:52.504737+0000","last_undegraded":"2023-09-23T21:07:52.504737+0000","last_fullsized":"2023-09-23T21:07:52.504737+0000","mapping_epoch":350,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":351,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:34:57.248381+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":51,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-09-23T21:07:50.417571+0000","last_change":"2023-09-23T21:05:04.326889+0000","last_active":"2023-09-23T21:07:50.417571+0000","last_peered":"2023-09-23T21:07:50.417571+0000","last_clean":"2023-09-23T21:07:50.417571+0000","last_became_active":"2023-09-23T21:05:04.326032+0000","last_became_peered":"2023-09-23T21:05:04.326032+0000","last_unstale":"2023-09-23T21:07:50.417571+0000","last_undegraded":"2023-09-23T21:07:50.417571+0000","last_fullsized":"2023-09-23T21:07:50.417571+0000","mapping_epoch":332,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:00:17.507889+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":364,"reported_epoch":374,"state":"active+clean","last_fresh":"2023-09-23T21:07:52.492025+0000","last_change":"2023-09-23T21:05:55.319889+0000","last_active":"2023-09-23T21:07:52.492025+0000","last_peered":"2023-09-23T21:07:52.492025+0000","last_clean":"2023-09-23T21:07:52.492025+0000","last_became_active":"2023-09-23T21:05:55.319400+0000","last_became_peered":"2023-09-23T21:05:55.319400+0000","last_unstale":"2023-09-23T21:07:52.492025+0000","last_undegraded":"2023-09-23T21:07:52.492025+0000","last_fullsized":"2023-09-23T21:07:52.492025+0000","mapping_epoch":350,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":351,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:21:17.483215+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.9","version":"374'8769","reported_seq":8806,"reported_epoch":374,"state":"active+clean","last_fresh":"2023-09-23T21:07:57.278207+0000","last_change":"2023-09-23T21:05:48.557743+0000","last_active":"2023-09-23T21:07:57.278207+0000","last_peered":"2023-09-23T21:07:57.278207+0000","last_clean":"2023-09-23T21:07:57.278207+0000","last_became_active":"2023-09-23T21:05:48.557419+0000","last_became_peered":"2023-09-23T21:05:48.557419+0000","last_unstale":"2023-09-23T21:07:57.278207+0000","last_undegraded":"2023-09-23T21:07:57.278207+0000","last_fullsized":"2023-09-23T21:07:57.278207+0000","mapping_epoch":346,"log_start":"351'400","ondisk_log_start":"351'400","created":346,"last_epoch_clean":347,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:05:47.261750+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:05:47.261750+0000","last_clean_scrub_stamp":"2023-09-23T21:05:47.261750+0000","objects_scrubbed":0,"log_size":8369,"log_dups_size":0,"ondisk_log_size":8369,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:34:16.622408+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":366936064,"num_objects":5599,"num_object_clones":0,"num_object_copies":16797,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5599,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8769,"num_write_kb":459776,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,5],"acting":[0,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1","version":"79'161","reported_seq":481,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-09-23T21:07:50.428722+0000","last_change":"2023-09-23T21:06:52.576036+0000","last_active":"2023-09-23T21:07:50.428722+0000","last_peered":"2023-09-23T21:07:50.428722+0000","last_clean":"2023-09-23T21:07:50.428722+0000","last_became_active":"2023-09-23T21:06:49.422806+0000","last_became_peered":"2023-09-23T21:06:49.422806+0000","last_unstale":"2023-09-23T21:07:50.428722+0000","last_undegraded":"2023-09-23T21:07:50.428722+0000","last_fullsized":"2023-09-23T21:07:50.428722+0000","mapping_epoch":359,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":360,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:06:52.575900+0000","last_deep_scrub":"79'161","last_deep_scrub_stamp":"2023-09-23T21:05:30.652189+0000","last_clean_scrub_stamp":"2023-09-23T21:06:52.575900+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:59:24.733331+0000","scrub_duration":0.017723466,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":133,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-09-23T21:07:50.409618+0000","last_change":"2023-09-23T21:06:49.426087+0000","last_active":"2023-09-23T21:07:50.409618+0000","last_peered":"2023-09-23T21:07:50.409618+0000","last_clean":"2023-09-23T21:07:50.409618+0000","last_became_active":"2023-09-23T21:06:49.425698+0000","last_became_peered":"2023-09-23T21:06:49.425698+0000","last_unstale":"2023-09-23T21:07:50.409618+0000","last_undegraded":"2023-09-23T21:07:50.409618+0000","last_fullsized":"2023-09-23T21:07:50.409618+0000","mapping_epoch":359,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":360,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:51:47.815811+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":268,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-09-23T21:07:50.438890+0000","last_change":"2023-09-23T20:55:35.949000+0000","last_active":"2023-09-23T21:07:50.438890+0000","last_peered":"2023-09-23T21:07:50.438890+0000","last_clean":"2023-09-23T21:07:50.438890+0000","last_became_active":"2023-09-23T20:55:35.948410+0000","last_became_peered":"2023-09-23T20:55:35.948410+0000","last_unstale":"2023-09-23T21:07:50.438890+0000","last_undegraded":"2023-09-23T21:07:50.438890+0000","last_fullsized":"2023-09-23T21:07:50.438890+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:26:50.477289+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":311,"reported_epoch":374,"state":"active+clean","last_fresh":"2023-09-23T21:07:52.505333+0000","last_change":"2023-09-23T21:05:55.320314+0000","last_active":"2023-09-23T21:07:52.505333+0000","last_peered":"2023-09-23T21:07:52.505333+0000","last_clean":"2023-09-23T21:07:52.505333+0000","last_became_active":"2023-09-23T21:05:55.319935+0000","last_became_peered":"2023-09-23T21:05:55.319935+0000","last_unstale":"2023-09-23T21:07:52.505333+0000","last_undegraded":"2023-09-23T21:07:52.505333+0000","last_fullsized":"2023-09-23T21:07:52.505333+0000","mapping_epoch":350,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":351,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:52:28.754534+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.a","version":"374'8536","reported_seq":8579,"reported_epoch":374,"state":"active+clean","last_fresh":"2023-09-23T21:07:57.186051+0000","last_change":"2023-09-23T21:05:55.324585+0000","last_active":"2023-09-23T21:07:57.186051+0000","last_peered":"2023-09-23T21:07:57.186051+0000","last_clean":"2023-09-23T21:07:57.186051+0000","last_became_active":"2023-09-23T21:05:55.323754+0000","last_became_peered":"2023-09-23T21:05:55.323754+0000","last_unstale":"2023-09-23T21:07:57.186051+0000","last_undegraded":"2023-09-23T21:07:57.186051+0000","last_fullsized":"2023-09-23T21:07:57.186051+0000","mapping_epoch":350,"log_start":"351'600","ondisk_log_start":"351'600","created":346,"last_epoch_clean":351,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:05:47.261750+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:05:47.261750+0000","last_clean_scrub_stamp":"2023-09-23T21:05:47.261750+0000","objects_scrubbed":0,"log_size":7936,"log_dups_size":0,"ondisk_log_size":7936,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:38:05.236953+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":357302272,"num_objects":5452,"num_object_clones":0,"num_object_copies":16356,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5452,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8536,"num_write_kb":447616,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,7],"acting":[5,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.2","version":"79'161","reported_seq":608,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-09-23T21:07:50.430737+0000","last_change":"2023-09-23T20:57:24.802586+0000","last_active":"2023-09-23T21:07:50.430737+0000","last_peered":"2023-09-23T21:07:50.430737+0000","last_clean":"2023-09-23T21:07:50.430737+0000","last_became_active":"2023-09-23T20:57:08.236114+0000","last_became_peered":"2023-09-23T20:57:08.236114+0000","last_unstale":"2023-09-23T21:07:50.430737+0000","last_undegraded":"2023-09-23T21:07:50.430737+0000","last_fullsized":"2023-09-23T21:07:50.430737+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:24.802451+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:24.802451+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:06:22.309980+0000","scrub_duration":0.0090861250000000005,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,4],"acting":[0,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":364,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-09-23T21:07:50.431348+0000","last_change":"2023-09-23T20:51:10.838054+0000","last_active":"2023-09-23T21:07:50.431348+0000","last_peered":"2023-09-23T21:07:50.431348+0000","last_clean":"2023-09-23T21:07:50.431348+0000","last_became_active":"2023-09-23T20:51:10.837750+0000","last_became_peered":"2023-09-23T20:51:10.837750+0000","last_unstale":"2023-09-23T21:07:50.431348+0000","last_undegraded":"2023-09-23T21:07:50.431348+0000","last_fullsized":"2023-09-23T21:07:50.431348+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:15:51.648085+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":337,"reported_epoch":374,"state":"active+clean","last_fresh":"2023-09-23T21:07:52.489241+0000","last_change":"2023-09-23T21:05:55.315770+0000","last_active":"2023-09-23T21:07:52.489241+0000","last_peered":"2023-09-23T21:07:52.489241+0000","last_clean":"2023-09-23T21:07:52.489241+0000","last_became_active":"2023-09-23T21:05:55.315290+0000","last_became_peered":"2023-09-23T21:05:55.315290+0000","last_unstale":"2023-09-23T21:07:52.489241+0000","last_undegraded":"2023-09-23T21:07:52.489241+0000","last_fullsized":"2023-09-23T21:07:52.489241+0000","mapping_epoch":350,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":351,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:27:40.430262+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":377,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-09-23T21:07:50.411986+0000","last_change":"2023-09-23T21:05:55.318863+0000","last_active":"2023-09-23T21:07:50.411986+0000","last_peered":"2023-09-23T21:07:50.411986+0000","last_clean":"2023-09-23T21:07:50.411986+0000","last_became_active":"2023-09-23T21:05:55.318489+0000","last_became_peered":"2023-09-23T21:05:55.318489+0000","last_unstale":"2023-09-23T21:07:50.411986+0000","last_undegraded":"2023-09-23T21:07:50.411986+0000","last_fullsized":"2023-09-23T21:07:50.411986+0000","mapping_epoch":350,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":351,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:45:05.912661+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.8","version":"374'8365","reported_seq":8402,"reported_epoch":374,"state":"active+clean","last_fresh":"2023-09-23T21:07:53.931655+0000","last_change":"2023-09-23T21:05:48.286303+0000","last_active":"2023-09-23T21:07:53.931655+0000","last_peered":"2023-09-23T21:07:53.931655+0000","last_clean":"2023-09-23T21:07:53.931655+0000","last_became_active":"2023-09-23T21:05:48.286008+0000","last_became_peered":"2023-09-23T21:05:48.286008+0000","last_unstale":"2023-09-23T21:07:53.931655+0000","last_undegraded":"2023-09-23T21:07:53.931655+0000","last_fullsized":"2023-09-23T21:07:53.931655+0000","mapping_epoch":346,"log_start":"352'2100","ondisk_log_start":"352'2100","created":346,"last_epoch_clean":347,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:05:47.261750+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:05:47.261750+0000","last_clean_scrub_stamp":"2023-09-23T21:05:47.261750+0000","objects_scrubbed":0,"log_size":6265,"log_dups_size":0,"ondisk_log_size":6265,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:42:05.759917+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":371130368,"num_objects":5663,"num_object_clones":0,"num_object_copies":16989,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5663,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8365,"num_write_kb":448896,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":138,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-09-23T21:07:50.431385+0000","last_change":"2023-09-23T21:06:49.427675+0000","last_active":"2023-09-23T21:07:50.431385+0000","last_peered":"2023-09-23T21:07:50.431385+0000","last_clean":"2023-09-23T21:07:50.431385+0000","last_became_active":"2023-09-23T21:06:49.426780+0000","last_became_peered":"2023-09-23T21:06:49.426780+0000","last_unstale":"2023-09-23T21:07:50.431385+0000","last_undegraded":"2023-09-23T21:07:50.431385+0000","last_fullsized":"2023-09-23T21:07:50.431385+0000","mapping_epoch":359,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":360,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:06:15.874264+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"79'161","reported_seq":608,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-09-23T21:07:50.428875+0000","last_change":"2023-09-23T20:57:20.738417+0000","last_active":"2023-09-23T21:07:50.428875+0000","last_peered":"2023-09-23T21:07:50.428875+0000","last_clean":"2023-09-23T21:07:50.428875+0000","last_became_active":"2023-09-23T20:57:08.236842+0000","last_became_peered":"2023-09-23T20:57:08.236842+0000","last_unstale":"2023-09-23T21:07:50.428875+0000","last_undegraded":"2023-09-23T21:07:50.428875+0000","last_fullsized":"2023-09-23T21:07:50.428875+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":0,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:20.738273+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:20.738273+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:18:28.327990+0000","scrub_duration":0.0088336339999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":374,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-09-23T21:07:50.430836+0000","last_change":"2023-09-23T21:05:04.310148+0000","last_active":"2023-09-23T21:07:50.430836+0000","last_peered":"2023-09-23T21:07:50.430836+0000","last_clean":"2023-09-23T21:07:50.430836+0000","last_became_active":"2023-09-23T21:05:04.309869+0000","last_became_peered":"2023-09-23T21:05:04.309869+0000","last_unstale":"2023-09-23T21:07:50.430836+0000","last_undegraded":"2023-09-23T21:07:50.430836+0000","last_fullsized":"2023-09-23T21:07:50.430836+0000","mapping_epoch":332,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:12:55.522835+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":318,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-09-23T21:07:50.436403+0000","last_change":"2023-09-23T20:55:35.978140+0000","last_active":"2023-09-23T21:07:50.436403+0000","last_peered":"2023-09-23T21:07:50.436403+0000","last_clean":"2023-09-23T21:07:50.436403+0000","last_became_active":"2023-09-23T20:55:35.977831+0000","last_became_peered":"2023-09-23T20:55:35.977831+0000","last_unstale":"2023-09-23T21:07:50.436403+0000","last_undegraded":"2023-09-23T21:07:50.436403+0000","last_fullsized":"2023-09-23T21:07:50.436403+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:25:47.678023+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"9.b","version":"374'8569","reported_seq":8606,"reported_epoch":374,"state":"active+clean","last_fresh":"2023-09-23T21:07:57.278654+0000","last_change":"2023-09-23T21:05:48.290142+0000","last_active":"2023-09-23T21:07:57.278654+0000","last_peered":"2023-09-23T21:07:57.278654+0000","last_clean":"2023-09-23T21:07:57.278654+0000","last_became_active":"2023-09-23T21:05:48.289861+0000","last_became_peered":"2023-09-23T21:05:48.289861+0000","last_unstale":"2023-09-23T21:07:57.278654+0000","last_undegraded":"2023-09-23T21:07:57.278654+0000","last_fullsized":"2023-09-23T21:07:57.278654+0000","mapping_epoch":346,"log_start":"351'200","ondisk_log_start":"351'200","created":346,"last_epoch_clean":347,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:05:47.261750+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:05:47.261750+0000","last_clean_scrub_stamp":"2023-09-23T21:05:47.261750+0000","objects_scrubbed":0,"log_size":8369,"log_dups_size":0,"ondisk_log_size":8369,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:36:39.064858+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":357367808,"num_objects":5453,"num_object_clones":0,"num_object_copies":16359,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5453,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8569,"num_write_kb":448704,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.3","version":"79'161","reported_seq":617,"reported_epoch":374,"state":"active+clean","last_fresh":"2023-09-23T21:07:52.504850+0000","last_change":"2023-09-23T21:05:57.832471+0000","last_active":"2023-09-23T21:07:52.504850+0000","last_peered":"2023-09-23T21:07:52.504850+0000","last_clean":"2023-09-23T21:07:52.504850+0000","last_became_active":"2023-09-23T21:05:55.318774+0000","last_became_peered":"2023-09-23T21:05:55.318774+0000","last_unstale":"2023-09-23T21:07:52.504850+0000","last_undegraded":"2023-09-23T21:07:52.504850+0000","last_fullsized":"2023-09-23T21:07:52.504850+0000","mapping_epoch":350,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":351,"parent":"0.0","parent_split_bits":3,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:05:57.832302+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:05:57.832302+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:15:55.402312+0000","scrub_duration":0.010206847999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":238,"reported_epoch":374,"state":"active+clean","last_fresh":"2023-09-23T21:07:52.489315+0000","last_change":"2023-09-23T21:05:55.316391+0000","last_active":"2023-09-23T21:07:52.489315+0000","last_peered":"2023-09-23T21:07:52.489315+0000","last_clean":"2023-09-23T21:07:52.489315+0000","last_became_active":"2023-09-23T21:05:55.316069+0000","last_became_peered":"2023-09-23T21:05:55.316069+0000","last_unstale":"2023-09-23T21:07:52.489315+0000","last_undegraded":"2023-09-23T21:07:52.489315+0000","last_fullsized":"2023-09-23T21:07:52.489315+0000","mapping_epoch":350,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":351,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:55:39.499467+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,0],"acting":[1,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":371,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-09-23T21:07:50.428931+0000","last_change":"2023-09-23T20:57:44.979950+0000","last_active":"2023-09-23T21:07:50.428931+0000","last_peered":"2023-09-23T21:07:50.428931+0000","last_clean":"2023-09-23T21:07:50.428931+0000","last_became_active":"2023-09-23T20:57:44.979346+0000","last_became_peered":"2023-09-23T20:57:44.979346+0000","last_unstale":"2023-09-23T21:07:50.428931+0000","last_undegraded":"2023-09-23T21:07:50.428931+0000","last_fullsized":"2023-09-23T21:07:50.428931+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:10:24.372800+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":365,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-09-23T21:07:50.430308+0000","last_change":"2023-09-23T21:05:55.320840+0000","last_active":"2023-09-23T21:07:50.430308+0000","last_peered":"2023-09-23T21:07:50.430308+0000","last_clean":"2023-09-23T21:07:50.430308+0000","last_became_active":"2023-09-23T21:05:55.320280+0000","last_became_peered":"2023-09-23T21:05:55.320280+0000","last_unstale":"2023-09-23T21:07:50.430308+0000","last_undegraded":"2023-09-23T21:07:50.430308+0000","last_fullsized":"2023-09-23T21:07:50.430308+0000","mapping_epoch":350,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":351,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:45:22.368801+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.e","version":"374'8851","reported_seq":8894,"reported_epoch":374,"state":"active+clean","last_fresh":"2023-09-23T21:07:57.352929+0000","last_change":"2023-09-23T21:05:55.326609+0000","last_active":"2023-09-23T21:07:57.352929+0000","last_peered":"2023-09-23T21:07:57.352929+0000","last_clean":"2023-09-23T21:07:57.352929+0000","last_became_active":"2023-09-23T21:05:55.326268+0000","last_became_peered":"2023-09-23T21:05:55.326268+0000","last_unstale":"2023-09-23T21:07:57.352929+0000","last_undegraded":"2023-09-23T21:07:57.352929+0000","last_fullsized":"2023-09-23T21:07:57.352929+0000","mapping_epoch":350,"log_start":"351'1800","ondisk_log_start":"351'1800","created":346,"last_epoch_clean":351,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:05:47.261750+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:05:47.261750+0000","last_clean_scrub_stamp":"2023-09-23T21:05:47.261750+0000","objects_scrubbed":0,"log_size":7051,"log_dups_size":0,"ondisk_log_size":7051,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:20:12.304644+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":373489664,"num_objects":5699,"num_object_clones":0,"num_object_copies":17097,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5699,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8851,"num_write_kb":465600,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.6","version":"79'161","reported_seq":656,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-09-23T21:07:50.436496+0000","last_change":"2023-09-23T20:57:26.907505+0000","last_active":"2023-09-23T21:07:50.436496+0000","last_peered":"2023-09-23T21:07:50.436496+0000","last_clean":"2023-09-23T21:07:50.436496+0000","last_became_active":"2023-09-23T20:57:08.240321+0000","last_became_peered":"2023-09-23T20:57:08.240321+0000","last_unstale":"2023-09-23T21:07:50.436496+0000","last_undegraded":"2023-09-23T21:07:50.436496+0000","last_fullsized":"2023-09-23T21:07:50.436496+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":3,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:26.907408+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:26.907408+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:34:17.392928+0000","scrub_duration":0.010337038,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":218,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-09-23T21:07:50.429611+0000","last_change":"2023-09-23T21:06:25.394432+0000","last_active":"2023-09-23T21:07:50.429611+0000","last_peered":"2023-09-23T21:07:50.429611+0000","last_clean":"2023-09-23T21:07:50.429611+0000","last_became_active":"2023-09-23T21:06:25.394120+0000","last_became_peered":"2023-09-23T21:06:25.394120+0000","last_unstale":"2023-09-23T21:07:50.429611+0000","last_undegraded":"2023-09-23T21:07:50.429611+0000","last_fullsized":"2023-09-23T21:07:50.429611+0000","mapping_epoch":354,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":355,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:07:30.306153+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":361,"reported_epoch":374,"state":"active+clean","last_fresh":"2023-09-23T21:07:52.504839+0000","last_change":"2023-09-23T21:05:55.319513+0000","last_active":"2023-09-23T21:07:52.504839+0000","last_peered":"2023-09-23T21:07:52.504839+0000","last_clean":"2023-09-23T21:07:52.504839+0000","last_became_active":"2023-09-23T21:05:55.319178+0000","last_became_peered":"2023-09-23T21:05:55.319178+0000","last_unstale":"2023-09-23T21:07:52.504839+0000","last_undegraded":"2023-09-23T21:07:52.504839+0000","last_fullsized":"2023-09-23T21:07:52.504839+0000","mapping_epoch":350,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":351,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:47:32.783934+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":356,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-09-23T21:07:50.431023+0000","last_change":"2023-09-23T21:05:04.303493+0000","last_active":"2023-09-23T21:07:50.431023+0000","last_peered":"2023-09-23T21:07:50.431023+0000","last_clean":"2023-09-23T21:07:50.431023+0000","last_became_active":"2023-09-23T21:05:04.303148+0000","last_became_peered":"2023-09-23T21:05:04.303148+0000","last_unstale":"2023-09-23T21:07:50.431023+0000","last_undegraded":"2023-09-23T21:07:50.431023+0000","last_fullsized":"2023-09-23T21:07:50.431023+0000","mapping_epoch":332,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:48:06.836151+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.d","version":"374'8727","reported_seq":8760,"reported_epoch":374,"state":"active+clean","last_fresh":"2023-09-23T21:07:58.426607+0000","last_change":"2023-09-23T21:05:55.318195+0000","last_active":"2023-09-23T21:07:58.426607+0000","last_peered":"2023-09-23T21:07:58.426607+0000","last_clean":"2023-09-23T21:07:58.426607+0000","last_became_active":"2023-09-23T21:05:55.316441+0000","last_became_peered":"2023-09-23T21:05:55.316441+0000","last_unstale":"2023-09-23T21:07:58.426607+0000","last_undegraded":"2023-09-23T21:07:58.426607+0000","last_fullsized":"2023-09-23T21:07:58.426607+0000","mapping_epoch":350,"log_start":"352'2200","ondisk_log_start":"352'2200","created":346,"last_epoch_clean":351,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:05:47.261750+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:05:47.261750+0000","last_clean_scrub_stamp":"2023-09-23T21:05:47.261750+0000","objects_scrubbed":0,"log_size":6527,"log_dups_size":0,"ondisk_log_size":6527,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:31:06.880466+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":362020888,"num_objects":5525,"num_object_clones":0,"num_object_copies":16575,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5525,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":8727,"num_write_kb":456001,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,5],"acting":[1,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"79'161","reported_seq":637,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-09-23T21:07:50.438629+0000","last_change":"2023-09-23T20:57:30.029479+0000","last_active":"2023-09-23T21:07:50.438629+0000","last_peered":"2023-09-23T21:07:50.438629+0000","last_clean":"2023-09-23T21:07:50.438629+0000","last_became_active":"2023-09-23T20:57:08.249909+0000","last_became_peered":"2023-09-23T20:57:08.249909+0000","last_unstale":"2023-09-23T21:07:50.438629+0000","last_undegraded":"2023-09-23T21:07:50.438629+0000","last_fullsized":"2023-09-23T21:07:50.438629+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":3,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:30.029396+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:30.029396+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:16:46.083145+0000","scrub_duration":0.012914682,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":341,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-09-23T21:07:50.433382+0000","last_change":"2023-09-23T20:51:10.836910+0000","last_active":"2023-09-23T21:07:50.433382+0000","last_peered":"2023-09-23T21:07:50.433382+0000","last_clean":"2023-09-23T21:07:50.433382+0000","last_became_active":"2023-09-23T20:51:10.836557+0000","last_became_peered":"2023-09-23T20:51:10.836557+0000","last_unstale":"2023-09-23T21:07:50.433382+0000","last_undegraded":"2023-09-23T21:07:50.433382+0000","last_fullsized":"2023-09-23T21:07:50.433382+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:51:54.019969+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":338,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-09-23T21:07:50.409835+0000","last_change":"2023-09-23T21:05:55.318675+0000","last_active":"2023-09-23T21:07:50.409835+0000","last_peered":"2023-09-23T21:07:50.409835+0000","last_clean":"2023-09-23T21:07:50.409835+0000","last_became_active":"2023-09-23T21:05:55.318363+0000","last_became_peered":"2023-09-23T21:05:55.318363+0000","last_unstale":"2023-09-23T21:07:50.409835+0000","last_undegraded":"2023-09-23T21:07:50.409835+0000","last_fullsized":"2023-09-23T21:07:50.409835+0000","mapping_epoch":350,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":351,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:19:50.065285+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":167,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-09-23T21:07:50.428071+0000","last_change":"2023-09-23T20:57:44.982509+0000","last_active":"2023-09-23T21:07:50.428071+0000","last_peered":"2023-09-23T21:07:50.428071+0000","last_clean":"2023-09-23T21:07:50.428071+0000","last_became_active":"2023-09-23T20:57:44.982175+0000","last_became_peered":"2023-09-23T20:57:44.982175+0000","last_unstale":"2023-09-23T21:07:50.428071+0000","last_undegraded":"2023-09-23T21:07:50.428071+0000","last_fullsized":"2023-09-23T21:07:50.428071+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:37:49.215488+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.c","version":"374'8482","reported_seq":8518,"reported_epoch":374,"state":"active+clean","last_fresh":"2023-09-23T21:07:53.906696+0000","last_change":"2023-09-23T21:05:48.560410+0000","last_active":"2023-09-23T21:07:53.906696+0000","last_peered":"2023-09-23T21:07:53.906696+0000","last_clean":"2023-09-23T21:07:53.906696+0000","last_became_active":"2023-09-23T21:05:48.560135+0000","last_became_peered":"2023-09-23T21:05:48.560135+0000","last_unstale":"2023-09-23T21:07:53.906696+0000","last_undegraded":"2023-09-23T21:07:53.906696+0000","last_fullsized":"2023-09-23T21:07:53.906696+0000","mapping_epoch":346,"log_start":"352'2200","ondisk_log_start":"352'2200","created":346,"last_epoch_clean":347,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:05:47.261750+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:05:47.261750+0000","last_clean_scrub_stamp":"2023-09-23T21:05:47.261750+0000","objects_scrubbed":0,"log_size":6282,"log_dups_size":0,"ondisk_log_size":6282,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:16:44.290638+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":379060224,"num_objects":5784,"num_object_clones":0,"num_object_copies":17352,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5784,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8482,"num_write_kb":456512,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.4","version":"79'161","reported_seq":690,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-09-23T21:07:50.428542+0000","last_change":"2023-09-23T21:05:07.298174+0000","last_active":"2023-09-23T21:07:50.428542+0000","last_peered":"2023-09-23T21:07:50.428542+0000","last_clean":"2023-09-23T21:07:50.428542+0000","last_became_active":"2023-09-23T21:05:07.297862+0000","last_became_peered":"2023-09-23T21:05:07.297862+0000","last_unstale":"2023-09-23T21:07:50.428542+0000","last_undegraded":"2023-09-23T21:07:50.428542+0000","last_fullsized":"2023-09-23T21:07:50.428542+0000","mapping_epoch":335,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":336,"parent":"0.0","parent_split_bits":3,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:20.012764+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:20.012764+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:24:55.897324+0000","scrub_duration":0.0045251919999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":323,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-09-23T21:07:50.436660+0000","last_change":"2023-09-23T20:55:35.949541+0000","last_active":"2023-09-23T21:07:50.436660+0000","last_peered":"2023-09-23T21:07:50.436660+0000","last_clean":"2023-09-23T21:07:50.436660+0000","last_became_active":"2023-09-23T20:55:35.944635+0000","last_became_peered":"2023-09-23T20:55:35.944635+0000","last_unstale":"2023-09-23T21:07:50.436660+0000","last_undegraded":"2023-09-23T21:07:50.436660+0000","last_fullsized":"2023-09-23T21:07:50.436660+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:30:36.033585+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"70'1","reported_seq":381,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-09-23T21:07:50.430364+0000","last_change":"2023-09-23T21:05:55.320603+0000","last_active":"2023-09-23T21:07:50.430364+0000","last_peered":"2023-09-23T21:07:50.430364+0000","last_clean":"2023-09-23T21:07:50.430364+0000","last_became_active":"2023-09-23T21:05:55.319956+0000","last_became_peered":"2023-09-23T21:05:55.319956+0000","last_unstale":"2023-09-23T21:07:50.430364+0000","last_undegraded":"2023-09-23T21:07:50.430364+0000","last_fullsized":"2023-09-23T21:07:50.430364+0000","mapping_epoch":350,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":351,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:41:03.995212+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":298,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-09-23T21:07:50.438473+0000","last_change":"2023-09-23T21:05:04.305174+0000","last_active":"2023-09-23T21:07:50.438473+0000","last_peered":"2023-09-23T21:07:50.438473+0000","last_clean":"2023-09-23T21:07:50.438473+0000","last_became_active":"2023-09-23T21:05:04.304900+0000","last_became_peered":"2023-09-23T21:05:04.304900+0000","last_unstale":"2023-09-23T21:07:50.438473+0000","last_undegraded":"2023-09-23T21:07:50.438473+0000","last_fullsized":"2023-09-23T21:07:50.438473+0000","mapping_epoch":332,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:13:23.024245+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":340,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-09-23T21:07:50.430955+0000","last_change":"2023-09-23T20:54:13.929196+0000","last_active":"2023-09-23T21:07:50.430955+0000","last_peered":"2023-09-23T21:07:50.430955+0000","last_clean":"2023-09-23T21:07:50.430955+0000","last_became_active":"2023-09-23T20:54:13.928831+0000","last_became_peered":"2023-09-23T20:54:13.928831+0000","last_unstale":"2023-09-23T21:07:50.430955+0000","last_undegraded":"2023-09-23T21:07:50.430955+0000","last_fullsized":"2023-09-23T21:07:50.430955+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:28:44.316851+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":360,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-09-23T21:07:50.431739+0000","last_change":"2023-09-23T21:05:55.313495+0000","last_active":"2023-09-23T21:07:50.431739+0000","last_peered":"2023-09-23T21:07:50.431739+0000","last_clean":"2023-09-23T21:07:50.431739+0000","last_became_active":"2023-09-23T21:05:55.313065+0000","last_became_peered":"2023-09-23T21:05:55.313065+0000","last_unstale":"2023-09-23T21:07:50.431739+0000","last_undegraded":"2023-09-23T21:07:50.431739+0000","last_fullsized":"2023-09-23T21:07:50.431739+0000","mapping_epoch":350,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":351,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:05:22.675197+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":64,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-09-23T21:07:50.439162+0000","last_change":"2023-09-23T21:06:49.429520+0000","last_active":"2023-09-23T21:07:50.439162+0000","last_peered":"2023-09-23T21:07:50.439162+0000","last_clean":"2023-09-23T21:07:50.439162+0000","last_became_active":"2023-09-23T21:06:49.429195+0000","last_became_peered":"2023-09-23T21:06:49.429195+0000","last_unstale":"2023-09-23T21:07:50.439162+0000","last_undegraded":"2023-09-23T21:07:50.439162+0000","last_fullsized":"2023-09-23T21:07:50.439162+0000","mapping_epoch":359,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":360,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:34:39.724075+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,1],"acting":[4,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":268,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-09-23T21:07:50.439002+0000","last_change":"2023-09-23T21:05:55.323591+0000","last_active":"2023-09-23T21:07:50.439002+0000","last_peered":"2023-09-23T21:07:50.439002+0000","last_clean":"2023-09-23T21:07:50.439002+0000","last_became_active":"2023-09-23T21:05:55.323220+0000","last_became_peered":"2023-09-23T21:05:55.323220+0000","last_unstale":"2023-09-23T21:07:50.439002+0000","last_undegraded":"2023-09-23T21:07:50.439002+0000","last_fullsized":"2023-09-23T21:07:50.439002+0000","mapping_epoch":350,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":351,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:03:09.849488+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":135,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-09-23T21:07:50.431941+0000","last_change":"2023-09-23T21:06:49.424707+0000","last_active":"2023-09-23T21:07:50.431941+0000","last_peered":"2023-09-23T21:07:50.431941+0000","last_clean":"2023-09-23T21:07:50.431941+0000","last_became_active":"2023-09-23T21:06:49.424418+0000","last_became_peered":"2023-09-23T21:06:49.424418+0000","last_unstale":"2023-09-23T21:07:50.431941+0000","last_undegraded":"2023-09-23T21:07:50.431941+0000","last_fullsized":"2023-09-23T21:07:50.431941+0000","mapping_epoch":359,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":360,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:28:50.350150+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":251,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-09-23T21:07:50.438621+0000","last_change":"2023-09-23T20:54:13.935307+0000","last_active":"2023-09-23T21:07:50.438621+0000","last_peered":"2023-09-23T21:07:50.438621+0000","last_clean":"2023-09-23T21:07:50.438621+0000","last_became_active":"2023-09-23T20:54:13.935000+0000","last_became_peered":"2023-09-23T20:54:13.935000+0000","last_unstale":"2023-09-23T21:07:50.438621+0000","last_undegraded":"2023-09-23T21:07:50.438621+0000","last_fullsized":"2023-09-23T21:07:50.438621+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:47:40.974082+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":365,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-09-23T21:07:50.428973+0000","last_change":"2023-09-23T20:55:35.954870+0000","last_active":"2023-09-23T21:07:50.428973+0000","last_peered":"2023-09-23T21:07:50.428973+0000","last_clean":"2023-09-23T21:07:50.428973+0000","last_became_active":"2023-09-23T20:55:35.954568+0000","last_became_peered":"2023-09-23T20:55:35.954568+0000","last_unstale":"2023-09-23T21:07:50.428973+0000","last_undegraded":"2023-09-23T21:07:50.428973+0000","last_fullsized":"2023-09-23T21:07:50.428973+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:06:22.423566+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":363,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-09-23T21:07:50.430090+0000","last_change":"2023-09-23T20:51:10.838334+0000","last_active":"2023-09-23T21:07:50.430090+0000","last_peered":"2023-09-23T21:07:50.430090+0000","last_clean":"2023-09-23T21:07:50.430090+0000","last_became_active":"2023-09-23T20:51:10.838026+0000","last_became_peered":"2023-09-23T20:51:10.838026+0000","last_unstale":"2023-09-23T21:07:50.430090+0000","last_undegraded":"2023-09-23T21:07:50.430090+0000","last_fullsized":"2023-09-23T21:07:50.430090+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:04:37.009457+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":398,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-09-23T21:07:50.429169+0000","last_change":"2023-09-23T21:05:04.325268+0000","last_active":"2023-09-23T21:07:50.429169+0000","last_peered":"2023-09-23T21:07:50.429169+0000","last_clean":"2023-09-23T21:07:50.429169+0000","last_became_active":"2023-09-23T21:05:04.303430+0000","last_became_peered":"2023-09-23T21:05:04.303430+0000","last_unstale":"2023-09-23T21:07:50.429169+0000","last_undegraded":"2023-09-23T21:07:50.429169+0000","last_fullsized":"2023-09-23T21:07:50.429169+0000","mapping_epoch":332,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:17:44.879586+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":2216,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":268,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-09-23T21:07:50.438306+0000","last_change":"2023-09-23T20:55:35.951572+0000","last_active":"2023-09-23T21:07:50.438306+0000","last_peered":"2023-09-23T21:07:50.438306+0000","last_clean":"2023-09-23T21:07:50.438306+0000","last_became_active":"2023-09-23T20:55:35.951294+0000","last_became_peered":"2023-09-23T20:55:35.951294+0000","last_unstale":"2023-09-23T21:07:50.438306+0000","last_undegraded":"2023-09-23T21:07:50.438306+0000","last_fullsized":"2023-09-23T21:07:50.438306+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:36:32.447210+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":39,"num_bytes_recovered":7154,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5834,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":21,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3381,"ondisk_log_size":3381,"up":63,"acting":63,"num_store_stats":8},{"poolid":9,"num_pg":16,"stat_sum":{"num_bytes":5832376344,"num_objects":88996,"num_object_clones":0,"num_object_copies":266988,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":88996,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":137552,"num_write_kb":7249473,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":17458278400,"data_stored":17458266184,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":114152,"ondisk_log_size":114152,"up":48,"acting":48,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201610,"num_pgs":47,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3601892,"kb_used_data":3215132,"kb_used_omap":0,"kb_used_meta":386752,"kb_avail":90143260,"statfs":{"total":95995035648,"available":92306698240,"internally_reserved":0,"allocated":3292295168,"data_stored":3291203336,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":396034048},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,1,0,2],"upper_bound":16},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:06:58 2023","interfaces":[{"interface":"back","average":{"1min":1.1870000000000001,"5min":1.4610000000000001,"15min":1.627},"min":{"1min":0.22800000000000001,"5min":0.22700000000000001,"15min":0.21099999999999999},"max":{"1min":4.3760000000000003,"5min":8.4990000000000006,"15min":43.994999999999997},"last":4.9969999999999999},{"interface":"front","average":{"1min":1.2549999999999999,"5min":1.6419999999999999,"15min":1.778},"min":{"1min":0.253,"5min":0.253,"15min":0.218},"max":{"1min":3.355,"5min":8.5150000000000006,"15min":44.206000000000003},"last":0.93100000000000005}]},{"osd":1,"last update":"Sat Sep 23 21:06:58 2023","interfaces":[{"interface":"back","average":{"1min":2.2000000000000002,"5min":2.2000000000000002,"15min":2.2000000000000002},"min":{"1min":0.28799999999999998,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":7.8399999999999999,"5min":7.8399999999999999,"15min":7.8399999999999999},"last":7.8010000000000002},{"interface":"front","average":{"1min":2.093,"5min":2.093,"15min":2.093},"min":{"1min":0.30399999999999999,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":8.3859999999999992,"5min":8.3859999999999992,"15min":8.3859999999999992},"last":3.714}]},{"osd":2,"last update":"Sat Sep 23 21:07:14 2023","interfaces":[{"interface":"back","average":{"1min":2.9329999999999998,"5min":1.802,"15min":1.8460000000000001},"min":{"1min":0.438,"5min":0.27200000000000002,"15min":0.24399999999999999},"max":{"1min":14.478999999999999,"5min":14.478999999999999,"15min":14.478999999999999},"last":7.7729999999999997},{"interface":"front","average":{"1min":3.0430000000000001,"5min":1.7010000000000001,"15min":1.8540000000000001},"min":{"1min":0.27400000000000002,"5min":0.27400000000000002,"15min":0.26500000000000001},"max":{"1min":14.135,"5min":14.135,"15min":45.25},"last":3.7559999999999998}]},{"osd":3,"last update":"Sat Sep 23 21:07:49 2023","interfaces":[{"interface":"back","average":{"1min":3.173,"5min":2.1040000000000001,"15min":2.6110000000000002},"min":{"1min":0.30099999999999999,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":9.1449999999999996,"5min":12.359999999999999,"15min":36.408000000000001},"last":7.8239999999999998},{"interface":"front","average":{"1min":3.6000000000000001,"5min":2.1379999999999999,"15min":2.4870000000000001},"min":{"1min":0.40999999999999998,"5min":0.30299999999999999,"15min":0.26000000000000001},"max":{"1min":12.226000000000001,"5min":13.199,"15min":35.593000000000004},"last":2.1779999999999999}]},{"osd":4,"last update":"Sat Sep 23 21:07:43 2023","interfaces":[{"interface":"back","average":{"1min":1.5589999999999999,"5min":0.96899999999999997,"15min":0.78300000000000003},"min":{"1min":0.28100000000000003,"5min":0.248,"15min":0.23699999999999999},"max":{"1min":4.4139999999999997,"5min":5.2640000000000002,"15min":5.8920000000000003},"last":3.653},{"interface":"front","average":{"1min":1.706,"5min":1.0509999999999999,"15min":0.84499999999999997},"min":{"1min":0.311,"5min":0.28399999999999997,"15min":0.26300000000000001},"max":{"1min":7.1420000000000003,"5min":7.1420000000000003,"15min":7.1420000000000003},"last":0.39600000000000002}]},{"osd":5,"last update":"Sat Sep 23 21:07:43 2023","interfaces":[{"interface":"back","average":{"1min":1.758,"5min":0.996,"15min":0.83499999999999996},"min":{"1min":0.33500000000000002,"5min":0.311,"15min":0.28499999999999998},"max":{"1min":4.8929999999999998,"5min":4.8929999999999998,"15min":7.0389999999999997},"last":7.6879999999999997},{"interface":"front","average":{"1min":1.5409999999999999,"5min":1.1000000000000001,"15min":0.81799999999999995},"min":{"1min":0.308,"5min":0.29899999999999999,"15min":0.27800000000000002},"max":{"1min":5.5090000000000003,"5min":5.8529999999999998,"15min":5.8529999999999998},"last":3.6059999999999999}]},{"osd":6,"last update":"Sat Sep 23 21:06:58 2023","interfaces":[{"interface":"back","average":{"1min":0.95699999999999996,"5min":0.96599999999999997,"15min":0.79100000000000004},"min":{"1min":0.35999999999999999,"5min":0.32200000000000001,"15min":0.28899999999999998},"max":{"1min":3.2480000000000002,"5min":6.1219999999999999,"15min":6.1219999999999999},"last":0.43099999999999999},{"interface":"front","average":{"1min":0.92700000000000005,"5min":0.99099999999999999,"15min":0.86399999999999999},"min":{"1min":0.36399999999999999,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":2.6699999999999999,"5min":6.0919999999999996,"15min":6.0919999999999996},"last":2.9390000000000001}]}]},{"osd":6,"up_from":164,"seq":704374636693,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1335760,"kb_used_data":1091784,"kb_used_omap":10,"kb_used_meta":243957,"kb_avail":92409392,"statfs":{"total":95995035648,"available":94627217408,"internally_reserved":0,"allocated":1117986816,"data_stored":1116915029,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":10328,"internal_metadata":249812904},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:07:01 2023","interfaces":[{"interface":"back","average":{"1min":2.1110000000000002,"5min":2.0110000000000001,"15min":1.575},"min":{"1min":0.25,"5min":0.224,"15min":0.20999999999999999},"max":{"1min":8.2750000000000004,"5min":14.151999999999999,"15min":14.151999999999999},"last":5.8140000000000001},{"interface":"front","average":{"1min":1.7330000000000001,"5min":1.7350000000000001,"15min":1.5920000000000001},"min":{"1min":0.224,"5min":0.224,"15min":0.20999999999999999},"max":{"1min":9.2249999999999996,"5min":10.962,"15min":11.023},"last":4.8330000000000002}]},{"osd":1,"last update":"Sat Sep 23 21:06:55 2023","interfaces":[{"interface":"back","average":{"1min":1.601,"5min":1.601,"15min":1.601},"min":{"1min":0.26700000000000002,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":12.904999999999999,"5min":12.904999999999999,"15min":12.904999999999999},"last":4.2720000000000002},{"interface":"front","average":{"1min":1.762,"5min":1.762,"15min":1.762},"min":{"1min":0.27700000000000002,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":6.7569999999999997,"5min":6.7569999999999997,"15min":6.7569999999999997},"last":2.3999999999999999}]},{"osd":2,"last update":"Sat Sep 23 21:07:01 2023","interfaces":[{"interface":"back","average":{"1min":1.76,"5min":1.9359999999999999,"15min":1.9419999999999999},"min":{"1min":0.36499999999999999,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":5.6799999999999997,"5min":14.029999999999999,"15min":14.029999999999999},"last":4.8929999999999998},{"interface":"front","average":{"1min":1.4390000000000001,"5min":2.0739999999999998,"15min":1.794},"min":{"1min":0.32600000000000001,"5min":0.26500000000000001,"15min":0.247},"max":{"1min":12.957000000000001,"5min":12.957000000000001,"15min":20.195},"last":4.2460000000000004}]},{"osd":3,"last update":"Sat Sep 23 21:07:01 2023","interfaces":[{"interface":"back","average":{"1min":2.7429999999999999,"5min":2.7669999999999999,"15min":1.95},"min":{"1min":0.32300000000000001,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":12.535,"5min":43.085999999999999,"15min":43.085999999999999},"last":0.41599999999999998},{"interface":"front","average":{"1min":2.8759999999999999,"5min":2.492,"15min":1.9950000000000001},"min":{"1min":0.307,"5min":0.30499999999999999,"15min":0.28599999999999998},"max":{"1min":12.513,"5min":42.405000000000001,"15min":42.405000000000001},"last":4.8730000000000002}]},{"osd":4,"last update":"Sat Sep 23 21:07:01 2023","interfaces":[{"interface":"back","average":{"1min":0.84099999999999997,"5min":0.94499999999999995,"15min":0.88100000000000001},"min":{"1min":0.28499999999999998,"5min":0.251,"15min":0.23000000000000001},"max":{"1min":3.0249999999999999,"5min":5.3920000000000003,"15min":9.0079999999999991},"last":4.2050000000000001},{"interface":"front","average":{"1min":1.198,"5min":1.036,"15min":0.69299999999999995},"min":{"1min":0.28799999999999998,"5min":0.26600000000000001,"15min":0.25800000000000001},"max":{"1min":5.5060000000000002,"5min":6.4050000000000002,"15min":6.4050000000000002},"last":2.3639999999999999}]},{"osd":5,"last update":"Sat Sep 23 21:07:01 2023","interfaces":[{"interface":"back","average":{"1min":0.92500000000000004,"5min":1.1220000000000001,"15min":0.80100000000000005},"min":{"1min":0.30099999999999999,"5min":0.27800000000000002,"15min":0.27300000000000002},"max":{"1min":2.6829999999999998,"5min":5.9000000000000004,"15min":5.9000000000000004},"last":6.0780000000000003},{"interface":"front","average":{"1min":0.96499999999999997,"5min":1.234,"15min":0.92300000000000004},"min":{"1min":0.308,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":5.5549999999999997,"5min":10.657,"15min":10.657},"last":0.44600000000000001}]},{"osd":7,"last update":"Sat Sep 23 21:07:01 2023","interfaces":[{"interface":"back","average":{"1min":1.1659999999999999,"5min":1.256,"15min":0.97199999999999998},"min":{"1min":0.29499999999999998,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":5.5890000000000004,"5min":8.9260000000000002,"15min":8.9260000000000002},"last":5.782},{"interface":"front","average":{"1min":1.242,"5min":0.96599999999999997,"15min":0.755},"min":{"1min":0.34200000000000003,"5min":0.28299999999999997,"15min":0.28100000000000003},"max":{"1min":5.6230000000000002,"5min":5.6230000000000002,"15min":8.282},"last":4.8559999999999999}]}]},{"osd":1,"up_from":350,"seq":1503238553626,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3342392,"kb_used_data":3150768,"kb_used_omap":10,"kb_used_meta":191477,"kb_avail":90402760,"statfs":{"total":95995035648,"available":92572426240,"internally_reserved":0,"allocated":3226386432,"data_stored":3224704485,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":10498,"internal_metadata":196073214},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:07:57 2023","interfaces":[{"interface":"back","average":{"1min":3.4860000000000002,"5min":2.222,"15min":2.0110000000000001},"min":{"1min":0.29299999999999998,"5min":0.21199999999999999,"15min":0.21199999999999999},"max":{"1min":15.728,"5min":15.728,"15min":15.728},"last":0.41899999999999998},{"interface":"front","average":{"1min":3.407,"5min":2.7290000000000001,"15min":2.6160000000000001},"min":{"1min":0.20999999999999999,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":14.468999999999999,"5min":14.468999999999999,"15min":14.468999999999999},"last":5.7119999999999997}]},{"osd":2,"last update":"Sat Sep 23 21:07:57 2023","interfaces":[{"interface":"back","average":{"1min":3.2509999999999999,"5min":2.5219999999999998,"15min":2.3999999999999999},"min":{"1min":0.24399999999999999,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":10.769,"5min":10.769,"15min":10.769},"last":8.6110000000000007},{"interface":"front","average":{"1min":3.0249999999999999,"5min":2.2120000000000002,"15min":2.0760000000000001},"min":{"1min":0.214,"5min":0.214,"15min":0.214},"max":{"1min":15.683999999999999,"5min":15.683999999999999,"15min":15.683999999999999},"last":1.7270000000000001}]},{"osd":3,"last update":"Sat Sep 23 21:07:57 2023","interfaces":[{"interface":"back","average":{"1min":3.226,"5min":2.843,"15min":2.7789999999999999},"min":{"1min":0.438,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":15.135999999999999,"5min":15.135999999999999,"15min":15.135999999999999},"last":0.70999999999999996},{"interface":"front","average":{"1min":4.1529999999999996,"5min":3.2810000000000001,"15min":3.1360000000000001},"min":{"1min":0.433,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":14.794,"5min":14.794,"15min":14.794},"last":4.7919999999999998}]},{"osd":4,"last update":"Sat Sep 23 21:07:57 2023","interfaces":[{"interface":"back","average":{"1min":2.6440000000000001,"5min":1.929,"15min":1.8100000000000001},"min":{"1min":0.32700000000000001,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":10.391,"5min":10.391,"15min":10.391},"last":2.71},{"interface":"front","average":{"1min":1.893,"5min":1.6599999999999999,"15min":1.621},"min":{"1min":0.39400000000000002,"5min":0.39400000000000002,"15min":0.39400000000000002},"max":{"1min":6.0270000000000001,"5min":6.0270000000000001,"15min":6.0270000000000001},"last":1.639}]},{"osd":5,"last update":"Sat Sep 23 21:07:57 2023","interfaces":[{"interface":"back","average":{"1min":2.2949999999999999,"5min":2.5430000000000001,"15min":2.5840000000000001},"min":{"1min":0.378,"5min":0.378,"15min":0.378},"max":{"1min":5.923,"5min":8.0899999999999999,"15min":8.0899999999999999},"last":4.8300000000000001},{"interface":"front","average":{"1min":3.3479999999999999,"5min":2.7959999999999998,"15min":2.7040000000000002},"min":{"1min":0.35699999999999998,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":10.339,"5min":10.339,"15min":10.339},"last":8.6539999999999999}]},{"osd":6,"last update":"Sat Sep 23 21:07:57 2023","interfaces":[{"interface":"back","average":{"1min":2.464,"5min":2.3639999999999999,"15min":2.3479999999999999},"min":{"1min":0.377,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":5.968,"5min":8.1880000000000006,"15min":8.1880000000000006},"last":4.8620000000000001},{"interface":"front","average":{"1min":3.2360000000000002,"5min":2.585,"15min":2.4769999999999999},"min":{"1min":0.40100000000000002,"5min":0.40100000000000002,"15min":0.40100000000000002},"max":{"1min":10.433999999999999,"5min":10.433999999999999,"15min":10.433999999999999},"last":2.7490000000000001}]},{"osd":7,"last update":"Sat Sep 23 21:07:57 2023","interfaces":[{"interface":"back","average":{"1min":1.9510000000000001,"5min":2.0230000000000001,"15min":2.0350000000000001},"min":{"1min":0.35299999999999998,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":5.992,"5min":6.5570000000000004,"15min":6.5570000000000004},"last":3.0699999999999998},{"interface":"front","average":{"1min":2.2909999999999999,"5min":2.5590000000000002,"15min":2.6040000000000001},"min":{"1min":0.41099999999999998,"5min":0.41099999999999998,"15min":0.41099999999999998},"max":{"1min":5.8899999999999997,"5min":8.1440000000000001,"15min":8.1440000000000001},"last":4.891}]}]},{"osd":0,"up_from":83,"seq":356482285791,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1869180,"kb_used_data":1415796,"kb_used_omap":28,"kb_used_meta":453347,"kb_avail":91875972,"statfs":{"total":95995035648,"available":94080995328,"internally_reserved":0,"allocated":1449775104,"data_stored":1448146935,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":28848,"internal_metadata":464228176},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[3,0,0,0,3],"upper_bound":32},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 21:06:55 2023","interfaces":[{"interface":"back","average":{"1min":1.3160000000000001,"5min":1.3160000000000001,"15min":1.3160000000000001},"min":{"1min":0.192,"5min":0.192,"15min":0.192},"max":{"1min":3.7730000000000001,"5min":3.7730000000000001,"15min":3.7730000000000001},"last":0.60399999999999998},{"interface":"front","average":{"1min":1.274,"5min":1.274,"15min":1.274},"min":{"1min":0.23999999999999999,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":3.7160000000000002,"5min":3.7160000000000002,"15min":3.7160000000000002},"last":14.646000000000001}]},{"osd":2,"last update":"Sat Sep 23 21:07:11 2023","interfaces":[{"interface":"back","average":{"1min":2.1869999999999998,"5min":2.266,"15min":2.7589999999999999},"min":{"1min":0.23799999999999999,"5min":0.183,"15min":0.183},"max":{"1min":13.032,"5min":13.032,"15min":21.053000000000001},"last":0.40000000000000002},{"interface":"front","average":{"1min":1.526,"5min":2.3290000000000002,"15min":2.8999999999999999},"min":{"1min":0.29699999999999999,"5min":0.23499999999999999,"15min":0.21099999999999999},"max":{"1min":8.6479999999999997,"5min":12.423,"15min":21.747},"last":14.558999999999999}]},{"osd":3,"last update":"Sat Sep 23 21:07:33 2023","interfaces":[{"interface":"back","average":{"1min":3.9220000000000002,"5min":2.5499999999999998,"15min":2.8719999999999999},"min":{"1min":0.27400000000000002,"5min":0.252,"15min":0.252},"max":{"1min":18.669,"5min":41.079000000000001,"15min":41.079000000000001},"last":4.891},{"interface":"front","average":{"1min":3.1499999999999999,"5min":2.282,"15min":2.9390000000000001},"min":{"1min":0.40300000000000002,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":19.91,"5min":19.91,"15min":21.167000000000002},"last":0.45700000000000002}]},{"osd":4,"last update":"Sat Sep 23 21:07:11 2023","interfaces":[{"interface":"back","average":{"1min":1.1879999999999999,"5min":1.79,"15min":2.1040000000000001},"min":{"1min":0.32400000000000001,"5min":0.28699999999999998,"15min":0.28599999999999998},"max":{"1min":3.1030000000000002,"5min":12.394,"15min":21.594999999999999},"last":5.0460000000000003},{"interface":"front","average":{"1min":2.8690000000000002,"5min":1.792,"15min":1.8720000000000001},"min":{"1min":0.311,"5min":0.311,"15min":0.28100000000000003},"max":{"1min":18.516999999999999,"5min":18.516999999999999,"15min":18.724},"last":4.7830000000000004}]},{"osd":5,"last update":"Sat Sep 23 21:07:11 2023","interfaces":[{"interface":"back","average":{"1min":2.2770000000000001,"5min":1.913,"15min":1.859},"min":{"1min":0.29299999999999998,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":12.946999999999999,"5min":12.946999999999999,"15min":12.946999999999999},"last":1.9590000000000001},{"interface":"front","average":{"1min":1.2869999999999999,"5min":1.877,"15min":2.1200000000000001},"min":{"1min":0.42299999999999999,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":3.839,"5min":12.353,"15min":20.512},"last":5.077}]},{"osd":6,"last update":"Sat Sep 23 21:07:05 2023","interfaces":[{"interface":"back","average":{"1min":1.841,"5min":1.6870000000000001,"15min":1.8879999999999999},"min":{"1min":0.44700000000000001,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":8.2300000000000004,"5min":11,"15min":13.933},"last":4.8609999999999998},{"interface":"front","average":{"1min":2.157,"5min":1.7030000000000001,"15min":1.9550000000000001},"min":{"1min":0.40300000000000002,"5min":0.40300000000000002,"15min":0.377},"max":{"1min":8.1950000000000003,"5min":11.066000000000001,"15min":13.962},"last":4.9160000000000004}]},{"osd":7,"last update":"Sat Sep 23 21:07:11 2023","interfaces":[{"interface":"back","average":{"1min":2.004,"5min":1.9530000000000001,"15min":1.9079999999999999},"min":{"1min":0.38900000000000001,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":12.907999999999999,"5min":12.907999999999999,"15min":12.907999999999999},"last":0.64000000000000001},{"interface":"front","average":{"1min":3.0209999999999999,"5min":2.3439999999999999,"15min":2.0880000000000001},"min":{"1min":0.45700000000000002,"5min":0.40999999999999998,"15min":0.377},"max":{"1min":18.634,"5min":40.984999999999999,"15min":40.984999999999999},"last":4.8179999999999996}]}]},{"osd":2,"up_from":93,"seq":399431958738,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1650932,"kb_used_data":1078892,"kb_used_omap":81,"kb_used_meta":571886,"kb_avail":92094220,"statfs":{"total":95995035648,"available":94304481280,"internally_reserved":0,"allocated":1104785408,"data_stored":1103162431,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":83750,"internal_metadata":585611482},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,3,0,1,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:07:00 2023","interfaces":[{"interface":"back","average":{"1min":1.5229999999999999,"5min":1.569,"15min":2.0750000000000002},"min":{"1min":0.23000000000000001,"5min":0.22600000000000001,"15min":0.17999999999999999},"max":{"1min":7.5199999999999996,"5min":8.2829999999999995,"15min":29.498000000000001},"last":11.124000000000001},{"interface":"front","average":{"1min":1.4490000000000001,"5min":1.8049999999999999,"15min":2.4119999999999999},"min":{"1min":0.27100000000000002,"5min":0.27000000000000002,"15min":0.193},"max":{"1min":5.2610000000000001,"5min":9.2880000000000003,"15min":17.251000000000001},"last":2.0059999999999998}]},{"osd":1,"last update":"Sat Sep 23 21:06:56 2023","interfaces":[{"interface":"back","average":{"1min":2.9750000000000001,"5min":2.9750000000000001,"15min":2.9750000000000001},"min":{"1min":0.23100000000000001,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":34.396999999999998,"5min":34.396999999999998,"15min":34.396999999999998},"last":11.019},{"interface":"front","average":{"1min":3.673,"5min":3.673,"15min":3.673},"min":{"1min":0.378,"5min":0.378,"15min":0.378},"max":{"1min":34.462000000000003,"5min":34.462000000000003,"15min":34.462000000000003},"last":7.0659999999999998}]},{"osd":3,"last update":"Sat Sep 23 21:07:47 2023","interfaces":[{"interface":"back","average":{"1min":2.2709999999999999,"5min":2.7679999999999998,"15min":3.3079999999999998},"min":{"1min":0.248,"5min":0.23899999999999999,"15min":0.23300000000000001},"max":{"1min":12.083,"5min":35.549999999999997,"15min":41.057000000000002},"last":2.2170000000000001},{"interface":"front","average":{"1min":2.8679999999999999,"5min":3.2690000000000001,"15min":3.7930000000000001},"min":{"1min":0.36199999999999999,"5min":0.253,"15min":0.22800000000000001},"max":{"1min":12.446,"5min":36.951999999999998,"15min":41.012},"last":2.1589999999999998}]},{"osd":4,"last update":"Sat Sep 23 21:07:00 2023","interfaces":[{"interface":"back","average":{"1min":1.4730000000000001,"5min":1.446,"15min":1.5900000000000001},"min":{"1min":0.29199999999999998,"5min":0.29199999999999998,"15min":0.26400000000000001},"max":{"1min":4.8200000000000003,"5min":8.7569999999999997,"15min":10.045},"last":2.1709999999999998},{"interface":"front","average":{"1min":1.7969999999999999,"5min":1.9510000000000001,"15min":1.964},"min":{"1min":0.26700000000000002,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":6.8129999999999997,"5min":9.6359999999999992,"15min":14.039999999999999},"last":2.0819999999999999}]},{"osd":5,"last update":"Sat Sep 23 21:07:00 2023","interfaces":[{"interface":"back","average":{"1min":1.661,"5min":1.6910000000000001,"15min":1.742},"min":{"1min":0.35099999999999998,"5min":0.316,"15min":0.29299999999999998},"max":{"1min":4.8300000000000001,"5min":8.907,"15min":10.069000000000001},"last":1.9750000000000001},{"interface":"front","average":{"1min":1.71,"5min":1.919,"15min":1.9610000000000001},"min":{"1min":0.35799999999999998,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":4.4820000000000002,"5min":11.906000000000001,"15min":13.98},"last":2.04}]},{"osd":6,"last update":"Sat Sep 23 21:07:00 2023","interfaces":[{"interface":"back","average":{"1min":1.5549999999999999,"5min":2.1779999999999999,"15min":2.028},"min":{"1min":0.42199999999999999,"5min":0.35699999999999998,"15min":0.312},"max":{"1min":4.5110000000000001,"5min":9.6820000000000004,"15min":14.244999999999999},"last":2.1070000000000002},{"interface":"front","average":{"1min":2.0870000000000002,"5min":2.3050000000000002,"15min":1.8859999999999999},"min":{"1min":0.374,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":8.4250000000000007,"5min":10.734999999999999,"15min":14.068},"last":2.1339999999999999}]},{"osd":7,"last update":"Sat Sep 23 21:07:00 2023","interfaces":[{"interface":"back","average":{"1min":1.6160000000000001,"5min":1.5609999999999999,"15min":1.4630000000000001},"min":{"1min":0.35799999999999998,"5min":0.34100000000000003,"15min":0.315},"max":{"1min":7.5940000000000003,"5min":8.7599999999999998,"15min":13.427},"last":7.1050000000000004},{"interface":"front","average":{"1min":2.0169999999999999,"5min":1.8360000000000001,"15min":1.8009999999999999},"min":{"1min":0.32200000000000001,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":5.5679999999999996,"5min":8.8689999999999998,"15min":10.092000000000001},"last":1.9359999999999999}]}]},{"osd":3,"up_from":98,"seq":420906795211,"num_pgs":48,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3575084,"kb_used_data":2933476,"kb_used_omap":7,"kb_used_meta":641336,"kb_avail":90170068,"statfs":{"total":95995035648,"available":92334149632,"internally_reserved":0,"allocated":3003879424,"data_stored":3002263299,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7576,"internal_metadata":656728680},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[3,0,0,2,1,0,1],"upper_bound":128},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:07:49 2023","interfaces":[{"interface":"back","average":{"1min":3.0049999999999999,"5min":2.331,"15min":3.0579999999999998},"min":{"1min":0.19900000000000001,"5min":0.189,"15min":0.188},"max":{"1min":10.409000000000001,"5min":11.717000000000001,"15min":27.962},"last":4.5709999999999997},{"interface":"front","average":{"1min":2.399,"5min":2.3479999999999999,"15min":2.9350000000000001},"min":{"1min":0.245,"5min":0.20599999999999999,"15min":0.19800000000000001},"max":{"1min":7.8920000000000003,"5min":11.483000000000001,"15min":28.058},"last":3.9399999999999999}]},{"osd":1,"last update":"Sat Sep 23 21:06:57 2023","interfaces":[{"interface":"back","average":{"1min":2.1429999999999998,"5min":2.1429999999999998,"15min":2.1429999999999998},"min":{"1min":0.25600000000000001,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":6.5789999999999997,"5min":6.5789999999999997,"15min":6.5789999999999997},"last":4.6130000000000004},{"interface":"front","average":{"1min":2.2210000000000001,"5min":2.2210000000000001,"15min":2.2210000000000001},"min":{"1min":0.313,"5min":0.313,"15min":0.313},"max":{"1min":7.0919999999999996,"5min":7.0919999999999996,"15min":7.0919999999999996},"last":7.4489999999999998}]},{"osd":2,"last update":"Sat Sep 23 21:07:49 2023","interfaces":[{"interface":"back","average":{"1min":4.6349999999999998,"5min":2.419,"15min":3.0640000000000001},"min":{"1min":0.307,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":14.287000000000001,"5min":14.287000000000001,"15min":28.251999999999999},"last":0.88700000000000001},{"interface":"front","average":{"1min":3.8639999999999999,"5min":2.6120000000000001,"15min":3.101},"min":{"1min":0.29899999999999999,"5min":0.29399999999999998,"15min":0.22},"max":{"1min":14.186,"5min":18.321000000000002,"15min":25.305},"last":4.6420000000000003}]},{"osd":4,"last update":"Sat Sep 23 21:07:49 2023","interfaces":[{"interface":"back","average":{"1min":2.8239999999999998,"5min":1.8160000000000001,"15min":2.0470000000000002},"min":{"1min":0.379,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":8.7379999999999995,"5min":11.505000000000001,"15min":18.239000000000001},"last":2.3210000000000002},{"interface":"front","average":{"1min":2.2050000000000001,"5min":1.974,"15min":2.105},"min":{"1min":0.35299999999999998,"5min":0.30299999999999999,"15min":0.29499999999999998},"max":{"1min":6.8929999999999998,"5min":10.914,"15min":22.853000000000002},"last":6.8929999999999998}]},{"osd":5,"last update":"Sat Sep 23 21:07:49 2023","interfaces":[{"interface":"back","average":{"1min":3.1339999999999999,"5min":2.0960000000000001,"15min":2.1509999999999998},"min":{"1min":0.32700000000000001,"5min":0.314,"15min":0.314},"max":{"1min":12.815,"5min":12.815,"15min":18.324999999999999},"last":1.022},{"interface":"front","average":{"1min":4.1870000000000003,"5min":2.4980000000000002,"15min":2.3660000000000001},"min":{"1min":0.34200000000000003,"5min":0.33700000000000002,"15min":0.33100000000000002},"max":{"1min":14.029,"5min":14.029,"15min":14.029},"last":14.029}]},{"osd":6,"last update":"Sat Sep 23 21:07:03 2023","interfaces":[{"interface":"back","average":{"1min":2.0030000000000001,"5min":1.948,"15min":1.6160000000000001},"min":{"1min":0.36599999999999999,"5min":0.33900000000000002,"15min":0.33500000000000002},"max":{"1min":5.4320000000000004,"5min":10.994999999999999,"15min":14.833},"last":4.6740000000000004},{"interface":"front","average":{"1min":2.4380000000000002,"5min":2.2440000000000002,"15min":2.0609999999999999},"min":{"1min":0.55600000000000005,"5min":0.373,"15min":0.373},"max":{"1min":7.633,"5min":10.664,"15min":13.576000000000001},"last":3.6760000000000002}]},{"osd":7,"last update":"Sat Sep 23 21:07:49 2023","interfaces":[{"interface":"back","average":{"1min":3.7410000000000001,"5min":2.27,"15min":2.3690000000000002},"min":{"1min":0.90100000000000002,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":14.81,"5min":14.81,"15min":18.289999999999999},"last":1.944},{"interface":"front","average":{"1min":2.7429999999999999,"5min":2.3199999999999998,"15min":2.3860000000000001},"min":{"1min":0.53200000000000003,"5min":0.40300000000000002,"15min":0.379},"max":{"1min":7.8390000000000004,"5min":12.624000000000001,"15min":13.603999999999999},"last":3.7069999999999999}]}]},{"osd":4,"up_from":31,"seq":133143986519,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1344416,"kb_used_data":1079448,"kb_used_omap":0,"kb_used_meta":264832,"kb_avail":92400736,"statfs":{"total":95995035648,"available":94618353664,"internally_reserved":0,"allocated":1105354752,"data_stored":1104267245,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":271187968},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:07:05 2023","interfaces":[{"interface":"back","average":{"1min":1.6379999999999999,"5min":1.3049999999999999,"15min":1.4330000000000001},"min":{"1min":0.24199999999999999,"5min":0.22700000000000001,"15min":0.214},"max":{"1min":11.353999999999999,"5min":11.353999999999999,"15min":11.784000000000001},"last":0.222},{"interface":"front","average":{"1min":2.004,"5min":1.7490000000000001,"15min":1.6679999999999999},"min":{"1min":0.28799999999999998,"5min":0.254,"15min":0.254},"max":{"1min":11.315,"5min":11.315,"15min":11.766},"last":0.28000000000000003}]},{"osd":1,"last update":"Sat Sep 23 21:06:58 2023","interfaces":[{"interface":"back","average":{"1min":3.8199999999999998,"5min":3.8199999999999998,"15min":3.8199999999999998},"min":{"1min":0.28100000000000003,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":37.960000000000001,"5min":37.960000000000001,"15min":37.960000000000001},"last":0.34300000000000003},{"interface":"front","average":{"1min":3.621,"5min":3.621,"15min":3.621},"min":{"1min":0.29899999999999999,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":37.933,"5min":37.933,"15min":37.933},"last":0.42399999999999999}]},{"osd":2,"last update":"Sat Sep 23 21:07:05 2023","interfaces":[{"interface":"back","average":{"1min":1.593,"5min":1.7490000000000001,"15min":1.7450000000000001},"min":{"1min":0.33500000000000002,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":7.1900000000000004,"5min":10.906000000000001,"15min":15.288},"last":0.38200000000000001},{"interface":"front","average":{"1min":1.786,"5min":1.599,"15min":1.77},"min":{"1min":0.35199999999999998,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":5.4749999999999996,"5min":8.9329999999999998,"15min":15.262},"last":4.508}]},{"osd":3,"last update":"Sat Sep 23 21:07:37 2023","interfaces":[{"interface":"back","average":{"1min":3.274,"5min":2.4239999999999999,"15min":2.5670000000000002},"min":{"1min":0.36599999999999999,"5min":0.29799999999999999,"15min":0.28899999999999998},"max":{"1min":15.01,"5min":15.01,"15min":31.562999999999999},"last":0.29799999999999999},{"interface":"front","average":{"1min":3.391,"5min":2.0779999999999998,"15min":2.4489999999999998},"min":{"1min":0.38700000000000001,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":15.004,"5min":15.004,"15min":31.574000000000002},"last":0.45600000000000002}]},{"osd":5,"last update":"Sat Sep 23 21:07:54 2023","interfaces":[{"interface":"back","average":{"1min":1.387,"5min":1.05,"15min":0.72699999999999998},"min":{"1min":0.24299999999999999,"5min":0.24299999999999999,"15min":0.22700000000000001},"max":{"1min":8.2479999999999993,"5min":8.2479999999999993,"15min":8.2479999999999993},"last":0.33400000000000002},{"interface":"front","average":{"1min":0.78100000000000003,"5min":1.1779999999999999,"15min":0.90100000000000002},"min":{"1min":0.255,"5min":0.255,"15min":0.251},"max":{"1min":2.9620000000000002,"5min":7.8700000000000001,"15min":7.8700000000000001},"last":0.35999999999999999}]},{"osd":6,"last update":"Sat Sep 23 21:06:58 2023","interfaces":[{"interface":"back","average":{"1min":0.91100000000000003,"5min":1.4399999999999999,"15min":0.88600000000000001},"min":{"1min":0.33400000000000002,"5min":0.26800000000000002,"15min":0.26500000000000001},"max":{"1min":2.492,"5min":7.4139999999999997,"15min":7.4139999999999997},"last":0.374},{"interface":"front","average":{"1min":0.70699999999999996,"5min":0.90200000000000002,"15min":0.748},"min":{"1min":0.32800000000000001,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":2.0190000000000001,"5min":7.1740000000000004,"15min":7.1740000000000004},"last":0.49099999999999999}]},{"osd":7,"last update":"Sat Sep 23 21:07:37 2023","interfaces":[{"interface":"back","average":{"1min":1.4330000000000001,"5min":1.1319999999999999,"15min":0.871},"min":{"1min":0.312,"5min":0.312,"15min":0.29299999999999998},"max":{"1min":5.4219999999999997,"5min":14.460000000000001,"15min":14.460000000000001},"last":0.47399999999999998},{"interface":"front","average":{"1min":0.93700000000000006,"5min":1.4450000000000001,"15min":0.98599999999999999},"min":{"1min":0.33100000000000002,"5min":0.32200000000000001,"15min":0.28199999999999997},"max":{"1min":4.2830000000000004,"5min":11.170999999999999,"15min":11.170999999999999},"last":0.39900000000000002}]}]},{"osd":5,"up_from":39,"seq":167503724884,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3510808,"kb_used_data":3174736,"kb_used_omap":0,"kb_used_meta":336064,"kb_avail":90234344,"statfs":{"total":95995035648,"available":92399968256,"internally_reserved":0,"allocated":3250929664,"data_stored":3249846661,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":344129536},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,0,2],"upper_bound":16},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:06:55 2023","interfaces":[{"interface":"back","average":{"1min":1.627,"5min":1.794,"15min":1.5900000000000001},"min":{"1min":0.26700000000000002,"5min":0.22500000000000001,"15min":0.20399999999999999},"max":{"1min":5.4710000000000001,"5min":10.664,"15min":12.795999999999999},"last":0.54600000000000004},{"interface":"front","average":{"1min":1.071,"5min":1.76,"15min":1.736},"min":{"1min":0.252,"5min":0.23400000000000001,"15min":0.223},"max":{"1min":4.2009999999999996,"5min":9.7479999999999993,"15min":16.219999999999999},"last":3.496}]},{"osd":1,"last update":"Sat Sep 23 21:06:55 2023","interfaces":[{"interface":"back","average":{"1min":1.7889999999999999,"5min":1.7889999999999999,"15min":1.7889999999999999},"min":{"1min":0.33600000000000002,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":6.4420000000000002,"5min":6.4420000000000002,"15min":6.4420000000000002},"last":5.4189999999999996},{"interface":"front","average":{"1min":1.5529999999999999,"5min":1.5529999999999999,"15min":1.5529999999999999},"min":{"1min":0.252,"5min":0.252,"15min":0.252},"max":{"1min":6.452,"5min":6.452,"15min":6.452},"last":5.4450000000000003}]},{"osd":2,"last update":"Sat Sep 23 21:07:08 2023","interfaces":[{"interface":"back","average":{"1min":1.5309999999999999,"5min":1.6699999999999999,"15min":1.6419999999999999},"min":{"1min":0.25800000000000001,"5min":0.25800000000000001,"15min":0.23499999999999999},"max":{"1min":5.9009999999999998,"5min":12.634,"15min":12.651},"last":0.57299999999999995},{"interface":"front","average":{"1min":1.8759999999999999,"5min":1.8580000000000001,"15min":1.839},"min":{"1min":0.28100000000000003,"5min":0.28100000000000003,"15min":0.26100000000000001},"max":{"1min":4.1680000000000001,"5min":9.7650000000000006,"15min":12.300000000000001},"last":1.5289999999999999}]},{"osd":3,"last update":"Sat Sep 23 21:07:30 2023","interfaces":[{"interface":"back","average":{"1min":3.3740000000000001,"5min":2.2389999999999999,"15min":2.6419999999999999},"min":{"1min":0.29299999999999998,"5min":0.29299999999999998,"15min":0.28599999999999998},"max":{"1min":25.503,"5min":25.503,"15min":61.667999999999999},"last":0.46600000000000003},{"interface":"front","average":{"1min":4.0129999999999999,"5min":2.4710000000000001,"15min":2.3919999999999999},"min":{"1min":0.42399999999999999,"5min":0.33500000000000002,"15min":0.28899999999999998},"max":{"1min":25.466000000000001,"5min":25.466000000000001,"15min":62.276000000000003},"last":5.8550000000000004}]},{"osd":4,"last update":"Sat Sep 23 21:07:44 2023","interfaces":[{"interface":"back","average":{"1min":1.3360000000000001,"5min":1.129,"15min":0.80200000000000005},"min":{"1min":0.28499999999999998,"5min":0.254,"15min":0.23400000000000001},"max":{"1min":5.7569999999999997,"5min":9.0950000000000006,"15min":9.0950000000000006},"last":0.55700000000000005},{"interface":"front","average":{"1min":1.8380000000000001,"5min":1.054,"15min":0.81799999999999995},"min":{"1min":0.34799999999999998,"5min":0.26900000000000002,"15min":0.23599999999999999},"max":{"1min":8.125,"5min":8.125,"15min":8.4350000000000005},"last":3.536}]},{"osd":6,"last update":"Sat Sep 23 21:06:55 2023","interfaces":[{"interface":"back","average":{"1min":1.0920000000000001,"5min":1.22,"15min":0.83699999999999997},"min":{"1min":0.32500000000000001,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":3.7719999999999998,"5min":11.128,"15min":11.128},"last":0.48599999999999999},{"interface":"front","average":{"1min":0.87,"5min":1.1870000000000001,"15min":0.89300000000000002},"min":{"1min":0.35599999999999998,"5min":0.34599999999999997,"15min":0.33200000000000002},"max":{"1min":2.4239999999999999,"5min":11.112,"15min":11.112},"last":0.40899999999999997}]},{"osd":7,"last update":"Sat Sep 23 21:07:30 2023","interfaces":[{"interface":"back","average":{"1min":1.4710000000000001,"5min":1.107,"15min":0.89300000000000002},"min":{"1min":0.35199999999999998,"5min":0.309,"15min":0.30199999999999999},"max":{"1min":5.6500000000000004,"5min":5.6500000000000004,"15min":8.5649999999999995},"last":0.45200000000000001},{"interface":"front","average":{"1min":0.89900000000000002,"5min":0.95299999999999996,"15min":0.80800000000000005},"min":{"1min":0.35999999999999999,"5min":0.32500000000000001,"15min":0.27700000000000002},"max":{"1min":3.4500000000000002,"5min":4.2009999999999996,"15min":4.6479999999999997},"last":0.44400000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1259,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1440940032,"data_stored":1440940032,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":3216576512,"data_stored":3216572440,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1091764224,"data_stored":1091764224,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":2978873344,"data_stored":2978873344,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1094844416,"data_stored":1094844416,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":3246657536,"data_stored":3246653464,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1113784320,"data_stored":1113784320,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":3274838016,"data_stored":3274833944,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T21:07:59.699 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T21:07:59.730 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:07:59.739 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:07:59.740 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:07:59.841 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:07:59.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:07:59 smithi070 ceph-mon[168282]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T21:07:59.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:07:59 smithi070 ceph-mon[168282]: Cluster is now healthy 2023-09-23T21:07:59.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:07:59 smithi070 ceph-mon[168282]: failed to encode map e375 with expected crc 2023-09-23T21:07:59.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:07:59 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T21:07:59.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:07:59 smithi070 ceph-mon[168282]: osdmap e375: 8 total, 8 up, 8 in 2023-09-23T21:07:59.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:07:59 smithi070 ceph-mon[168282]: failed to encode map e375 with expected crc 2023-09-23T21:07:59.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:07:59 smithi070 ceph-mon[168282]: failed to encode map e375 with expected crc 2023-09-23T21:07:59.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:07:59 smithi070 ceph-mon[168282]: failed to encode map e375 with expected crc 2023-09-23T21:07:59.885 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:07:59.886 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:07:59 smithi026 ceph-mon[175151]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T21:07:59.886 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:07:59 smithi026 ceph-mon[175151]: Cluster is now healthy 2023-09-23T21:07:59.886 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:07:59 smithi026 ceph-mon[175151]: failed to encode map e375 with expected crc 2023-09-23T21:07:59.886 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:07:59 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T21:07:59.887 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:07:59 smithi026 ceph-mon[175151]: osdmap e375: 8 total, 8 up, 8 in 2023-09-23T21:07:59.887 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:07:59 smithi026 ceph-mon[175151]: failed to encode map e375 with expected crc 2023-09-23T21:07:59.887 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:07:59 smithi026 ceph-mon[175151]: failed to encode map e375 with expected crc 2023-09-23T21:07:59.887 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:07:59 smithi026 ceph-mon[175151]: failed to encode map e375 with expected crc 2023-09-23T21:07:59.887 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 21:07:59 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[164029]: ::ffff:172.21.15.70 - - [23/Sep/2023:21:07:59] "GET /metrics HTTP/1.1" 200 36298 "" "Prometheus/2.43.0" 2023-09-23T21:07:59.887 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:07:59 smithi026 ceph-mon[179688]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T21:07:59.888 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:07:59 smithi026 ceph-mon[179688]: Cluster is now healthy 2023-09-23T21:07:59.888 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:07:59 smithi026 ceph-mon[179688]: failed to encode map e375 with expected crc 2023-09-23T21:07:59.888 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:07:59 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T21:07:59.888 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:07:59 smithi026 ceph-mon[179688]: osdmap e375: 8 total, 8 up, 8 in 2023-09-23T21:07:59.888 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:07:59 smithi026 ceph-mon[179688]: failed to encode map e375 with expected crc 2023-09-23T21:07:59.888 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:07:59 smithi026 ceph-mon[179688]: failed to encode map e375 with expected crc 2023-09-23T21:07:59.888 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:07:59 smithi026 ceph-mon[179688]: failed to encode map e375 with expected crc 2023-09-23T21:07:59.930 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:07:59.934 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:07:59.935 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:08:00.036 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:08:00.071 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:08:00.113 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:08:00.119 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:08:00.120 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:08:00.167 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:08:00.221 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:08:00.260 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:08:00.262 INFO:teuthology.orchestra.run.smithi026.stdout:ERROR: (22) Invalid argument 2023-09-23T21:08:00.262 INFO:teuthology.orchestra.run.smithi026.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T21:08:00.317 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:08:00.327 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:08:00.328 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:08:00.429 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:08:00.481 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:08:00.559 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:08:00.564 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:08:00.565 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:08:00.572 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:08:00 smithi026 ceph-mon[175151]: from='client.55918 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T21:08:00.572 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:08:00 smithi026 ceph-mon[175151]: pgmap v980: 109 pgs: 109 active+clean; 5.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 842 op/s 2023-09-23T21:08:00.573 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:08:00 smithi026 ceph-mon[179688]: from='client.55918 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T21:08:00.573 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:08:00 smithi026 ceph-mon[179688]: pgmap v980: 109 pgs: 109 active+clean; 5.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 842 op/s 2023-09-23T21:08:00.665 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:08:00.725 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:08:00.797 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:08:00.805 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:08:00.805 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:08:00.817 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_historic_ops 2023-09-23T21:08:00.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:08:00 smithi070 ceph-mon[168282]: from='client.55918 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T21:08:00.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:08:00 smithi070 ceph-mon[168282]: pgmap v980: 109 pgs: 109 active+clean; 5.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 842 op/s 2023-09-23T21:08:00.906 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:08:01.004 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:08:01.046 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:08:01.078 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:08:01.084 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:08:01.085 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:08:01.186 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:08:01.274 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:08:01.456 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:08:01.463 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:08:01.464 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:08:01.565 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:08:01.603 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:08:01.654 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:08:01.661 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:08:01.662 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:08:01.763 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:08:01.817 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T21:08:01.829 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:08:02.200 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T21:08:02.213 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:08:02.214 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:08:02.315 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:08:02.372 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:08:02.426 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:08:02.430 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:08:02.431 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:08:02.532 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:08:02.568 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:08:02 smithi070 ceph-mon[168282]: pgmap v981: 109 pgs: 109 active+clean; 5.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 712 op/s 2023-09-23T21:08:02.568 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:08:02 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:08:02.568 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:08:02 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:08:02.570 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:08:02.612 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:08:02.615 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:08:02.616 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:08:02.631 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:08:02 smithi026 ceph-mon[179688]: pgmap v981: 109 pgs: 109 active+clean; 5.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 712 op/s 2023-09-23T21:08:02.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:08:02 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:08:02.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:08:02 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:08:02.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:08:02 smithi026 ceph-mon[175151]: pgmap v981: 109 pgs: 109 active+clean; 5.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 712 op/s 2023-09-23T21:08:02.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:08:02 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:08:02.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:08:02 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:08:02.716 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:08:02.721 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0: {} 2023-09-23T21:08:02.745 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:08:02.776 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1: {} 2023-09-23T21:08:02.795 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:08:02.802 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:08:02.803 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:08:02.841 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2: {} 2023-09-23T21:08:02.875 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3: {} 2023-09-23T21:08:02.904 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:08:02.908 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4: {} 2023-09-23T21:08:02.945 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:08:02.945 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T21:08:02.992 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6: {} 2023-09-23T21:08:02.994 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:08:02.997 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:08:02.998 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:08:03.026 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T21:08:03.028 INFO:teuthology.orchestra.run.smithi026.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T21:08:03.028 INFO:teuthology.orchestra.run.smithi026.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T21:08:03.028 INFO:teuthology.orchestra.run.smithi026.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T21:08:03.028 INFO:teuthology.orchestra.run.smithi026.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T21:08:03.028 INFO:teuthology.orchestra.run.smithi026.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T21:08:03.028 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T21:08:03.028 INFO:teuthology.orchestra.run.smithi026.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T21:08:03.029 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T21:08:03.099 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:08:03.168 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:08:03.253 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:08:03 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/4116534723' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:08:03.383 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:08:03 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/4116534723' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:08:03.383 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:08:03 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/4116534723' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:08:03.560 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:08:03.572 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:08:03.572 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:08:03.619 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:08:03.671 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:08:03.673 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:08:03.749 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:08:03.985 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T21:08:03.991 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:08:03.992 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:08:04.093 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:08:04.135 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:08:04.184 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:08:04.189 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:08:04.190 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:08:04.252 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:08:04 smithi026 ceph-mon[175151]: pgmap v982: 109 pgs: 109 active+clean; 5.2 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 802 op/s 2023-09-23T21:08:04.291 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:08:04.346 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:08:04.409 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:08:04.411 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:08:04 smithi070 ceph-mon[168282]: pgmap v982: 109 pgs: 109 active+clean; 5.2 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 802 op/s 2023-09-23T21:08:04.412 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:08:04.412 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:08:04.513 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:08:04.515 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:08:04 smithi026 ceph-mon[179688]: pgmap v982: 109 pgs: 109 active+clean; 5.2 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 802 op/s 2023-09-23T21:08:04.558 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:08:04.622 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:08:04.626 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:08:04.627 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:08:04.728 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:08:04.793 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:08:04.992 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T21:08:05.001 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:08:05.001 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:08:05.102 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:08:05.153 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:08:05.202 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:08:05.208 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:08:05.208 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:08:05.309 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:08:05.344 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:08:05.385 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:08:05.390 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:08:05.390 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:08:05.491 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:08:05.524 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:08:05.569 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:08:05.572 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:08:05.572 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:08:05.587 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T21:08:05.673 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:08:05.709 INFO:teuthology.orchestra.run.smithi026.stdout: 2023-09-23T21:08:05.716 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:08:05.732 INFO:teuthology.orchestra.run.smithi026.stderr:dumped all 2023-09-23T21:08:05.768 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:08:05.770 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:08:05.770 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:08:05.871 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:08:05.919 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:08:05.967 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:08:05.972 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:08:05.973 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:08:06.073 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:08:06.197 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:08:06.314 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:08:06.331 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:08:06.332 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:08:06.396 INFO:teuthology.orchestra.run.smithi026.stdout:{"pg_ready":true,"pg_map":{"version":983,"stamp":"2023-09-23T21:08:04.881683+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":5487269816,"num_objects":83745,"num_object_clones":0,"num_object_copies":251235,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":83745,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":142888,"num_write_kb":7249502,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":39,"num_bytes_recovered":7154,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":118163,"ondisk_log_size":118163,"up":327,"acting":327,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":327,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":19316872,"kb_used_data":16189768,"kb_used_omap":137,"kb_used_meta":3126838,"kb_avail":730644344,"statfs":{"total":767960285184,"available":748179808256,"internally_reserved":0,"allocated":16578322432,"data_stored":16567431349,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":141000,"internal_metadata":3201882424},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[3,1,5,7,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":7,"apply_latency_ms":7,"commit_latency_ns":7000000,"apply_latency_ns":7000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-525336576,"num_objects":-8016,"num_object_clones":0,"num_object_copies":-24048,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":-8016,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8016,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.007881"},"pg_stats":[{"pgid":"3.1a","version":"68'1","reported_seq":269,"reported_epoch":374,"state":"active+clean","last_fresh":"2023-09-23T21:07:52.468394+0000","last_change":"2023-09-23T21:05:55.322748+0000","last_active":"2023-09-23T21:07:52.468394+0000","last_peered":"2023-09-23T21:07:52.468394+0000","last_clean":"2023-09-23T21:07:52.468394+0000","last_became_active":"2023-09-23T21:05:55.322498+0000","last_became_peered":"2023-09-23T21:05:55.322498+0000","last_unstale":"2023-09-23T21:07:52.468394+0000","last_undegraded":"2023-09-23T21:07:52.468394+0000","last_fullsized":"2023-09-23T21:07:52.468394+0000","mapping_epoch":350,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":351,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:03:28.049826+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":370,"reported_epoch":374,"state":"active+clean","last_fresh":"2023-09-23T21:07:52.479441+0000","last_change":"2023-09-23T20:54:13.942019+0000","last_active":"2023-09-23T21:07:52.479441+0000","last_peered":"2023-09-23T21:07:52.479441+0000","last_clean":"2023-09-23T21:07:52.479441+0000","last_became_active":"2023-09-23T20:54:13.941647+0000","last_became_peered":"2023-09-23T20:54:13.941647+0000","last_unstale":"2023-09-23T21:07:52.479441+0000","last_undegraded":"2023-09-23T21:07:52.479441+0000","last_fullsized":"2023-09-23T21:07:52.479441+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:30:54.400944+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":347,"reported_epoch":374,"state":"active+clean","last_fresh":"2023-09-23T21:07:52.489953+0000","last_change":"2023-09-23T21:05:55.322068+0000","last_active":"2023-09-23T21:07:52.489953+0000","last_peered":"2023-09-23T21:07:52.489953+0000","last_clean":"2023-09-23T21:07:52.489953+0000","last_became_active":"2023-09-23T21:05:55.321763+0000","last_became_peered":"2023-09-23T21:05:55.321763+0000","last_unstale":"2023-09-23T21:07:52.489953+0000","last_undegraded":"2023-09-23T21:07:52.489953+0000","last_fullsized":"2023-09-23T21:07:52.489953+0000","mapping_epoch":350,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":351,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:01:41.294709+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":364,"reported_epoch":374,"state":"active+clean","last_fresh":"2023-09-23T21:07:52.467969+0000","last_change":"2023-09-23T20:57:44.979811+0000","last_active":"2023-09-23T21:07:52.467969+0000","last_peered":"2023-09-23T21:07:52.467969+0000","last_clean":"2023-09-23T21:07:52.467969+0000","last_became_active":"2023-09-23T20:57:44.979515+0000","last_became_peered":"2023-09-23T20:57:44.979515+0000","last_unstale":"2023-09-23T21:07:52.467969+0000","last_undegraded":"2023-09-23T21:07:52.467969+0000","last_fullsized":"2023-09-23T21:07:52.467969+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:38:22.162140+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":376,"reported_epoch":374,"state":"active+clean","last_fresh":"2023-09-23T21:07:52.478239+0000","last_change":"2023-09-23T21:05:55.325654+0000","last_active":"2023-09-23T21:07:52.478239+0000","last_peered":"2023-09-23T21:07:52.478239+0000","last_clean":"2023-09-23T21:07:52.478239+0000","last_became_active":"2023-09-23T21:05:55.325298+0000","last_became_peered":"2023-09-23T21:05:55.325298+0000","last_unstale":"2023-09-23T21:07:52.478239+0000","last_undegraded":"2023-09-23T21:07:52.478239+0000","last_fullsized":"2023-09-23T21:07:52.478239+0000","mapping_epoch":350,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":351,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:33:19.561063+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":377,"reported_epoch":374,"state":"active+clean","last_fresh":"2023-09-23T21:07:52.456657+0000","last_change":"2023-09-23T21:05:55.319892+0000","last_active":"2023-09-23T21:07:52.456657+0000","last_peered":"2023-09-23T21:07:52.456657+0000","last_clean":"2023-09-23T21:07:52.456657+0000","last_became_active":"2023-09-23T21:05:55.319536+0000","last_became_peered":"2023-09-23T21:05:55.319536+0000","last_unstale":"2023-09-23T21:07:52.456657+0000","last_undegraded":"2023-09-23T21:07:52.456657+0000","last_fullsized":"2023-09-23T21:07:52.456657+0000","mapping_epoch":350,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":351,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:14:48.388831+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":371,"reported_epoch":374,"state":"active+clean","last_fresh":"2023-09-23T21:07:52.468322+0000","last_change":"2023-09-23T21:05:04.304358+0000","last_active":"2023-09-23T21:07:52.468322+0000","last_peered":"2023-09-23T21:07:52.468322+0000","last_clean":"2023-09-23T21:07:52.468322+0000","last_became_active":"2023-09-23T21:05:04.303913+0000","last_became_peered":"2023-09-23T21:05:04.303913+0000","last_unstale":"2023-09-23T21:07:52.468322+0000","last_undegraded":"2023-09-23T21:07:52.468322+0000","last_fullsized":"2023-09-23T21:07:52.468322+0000","mapping_epoch":332,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:21:21.860944+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":374,"reported_epoch":374,"state":"active+clean","last_fresh":"2023-09-23T21:07:52.478325+0000","last_change":"2023-09-23T21:05:55.323506+0000","last_active":"2023-09-23T21:07:52.478325+0000","last_peered":"2023-09-23T21:07:52.478325+0000","last_clean":"2023-09-23T21:07:52.478325+0000","last_became_active":"2023-09-23T21:05:55.323167+0000","last_became_peered":"2023-09-23T21:05:55.323167+0000","last_unstale":"2023-09-23T21:07:52.478325+0000","last_undegraded":"2023-09-23T21:07:52.478325+0000","last_fullsized":"2023-09-23T21:07:52.478325+0000","mapping_epoch":350,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":351,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:06:00.628349+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"79'161","reported_seq":665,"reported_epoch":374,"state":"active+clean","last_fresh":"2023-09-23T21:07:52.471307+0000","last_change":"2023-09-23T20:57:25.096559+0000","last_active":"2023-09-23T21:07:52.471307+0000","last_peered":"2023-09-23T21:07:52.471307+0000","last_clean":"2023-09-23T21:07:52.471307+0000","last_became_active":"2023-09-23T20:57:15.975674+0000","last_became_peered":"2023-09-23T20:57:15.975674+0000","last_unstale":"2023-09-23T21:07:52.471307+0000","last_undegraded":"2023-09-23T21:07:52.471307+0000","last_fullsized":"2023-09-23T21:07:52.471307+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:25.096401+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:25.096401+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:05:05.448404+0000","scrub_duration":0.002859393,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":69,"reported_epoch":374,"state":"active+clean","last_fresh":"2023-09-23T21:07:52.465896+0000","last_change":"2023-09-23T21:05:55.325024+0000","last_active":"2023-09-23T21:07:52.465896+0000","last_peered":"2023-09-23T21:07:52.465896+0000","last_clean":"2023-09-23T21:07:52.465896+0000","last_became_active":"2023-09-23T21:05:55.322890+0000","last_became_peered":"2023-09-23T21:05:55.322890+0000","last_unstale":"2023-09-23T21:07:52.465896+0000","last_undegraded":"2023-09-23T21:07:52.465896+0000","last_fullsized":"2023-09-23T21:07:52.465896+0000","mapping_epoch":350,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":351,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:49:56.913769+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":320,"reported_epoch":374,"state":"active+clean","last_fresh":"2023-09-23T21:07:52.478886+0000","last_change":"2023-09-23T20:55:35.952661+0000","last_active":"2023-09-23T21:07:52.478886+0000","last_peered":"2023-09-23T21:07:52.478886+0000","last_clean":"2023-09-23T21:07:52.478886+0000","last_became_active":"2023-09-23T20:55:35.952349+0000","last_became_peered":"2023-09-23T20:55:35.952349+0000","last_unstale":"2023-09-23T21:07:52.478886+0000","last_undegraded":"2023-09-23T21:07:52.478886+0000","last_fullsized":"2023-09-23T21:07:52.478886+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:00:03.110126+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":196,"reported_epoch":374,"state":"active+clean","last_fresh":"2023-09-23T21:07:52.479566+0000","last_change":"2023-09-23T20:56:45.988607+0000","last_active":"2023-09-23T21:07:52.479566+0000","last_peered":"2023-09-23T21:07:52.479566+0000","last_clean":"2023-09-23T21:07:52.479566+0000","last_became_active":"2023-09-23T20:56:45.988206+0000","last_became_peered":"2023-09-23T20:56:45.988206+0000","last_unstale":"2023-09-23T21:07:52.479566+0000","last_undegraded":"2023-09-23T21:07:52.479566+0000","last_fullsized":"2023-09-23T21:07:52.479566+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":205,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:01:19.783688+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,4],"acting":[3,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":47,"reported_epoch":374,"state":"active+clean","last_fresh":"2023-09-23T21:07:52.491960+0000","last_change":"2023-09-23T21:05:55.315641+0000","last_active":"2023-09-23T21:07:52.491960+0000","last_peered":"2023-09-23T21:07:52.491960+0000","last_clean":"2023-09-23T21:07:52.491960+0000","last_became_active":"2023-09-23T21:05:55.315336+0000","last_became_peered":"2023-09-23T21:05:55.315336+0000","last_unstale":"2023-09-23T21:07:52.491960+0000","last_undegraded":"2023-09-23T21:07:52.491960+0000","last_fullsized":"2023-09-23T21:07:52.491960+0000","mapping_epoch":350,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":351,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:16:47.607565+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"79'22","reported_seq":279,"reported_epoch":374,"state":"active+clean","last_fresh":"2023-09-23T21:07:52.467386+0000","last_change":"2023-09-23T20:55:35.949418+0000","last_active":"2023-09-23T21:07:52.467386+0000","last_peered":"2023-09-23T21:07:52.467386+0000","last_clean":"2023-09-23T21:07:52.467386+0000","last_became_active":"2023-09-23T20:55:35.948566+0000","last_became_peered":"2023-09-23T20:55:35.948566+0000","last_unstale":"2023-09-23T21:07:52.467386+0000","last_undegraded":"2023-09-23T21:07:52.467386+0000","last_fullsized":"2023-09-23T21:07:52.467386+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:59:37.433128+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4734,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":286,"reported_epoch":374,"state":"active+clean","last_fresh":"2023-09-23T21:07:52.467107+0000","last_change":"2023-09-23T21:05:55.325203+0000","last_active":"2023-09-23T21:07:52.467107+0000","last_peered":"2023-09-23T21:07:52.467107+0000","last_clean":"2023-09-23T21:07:52.467107+0000","last_became_active":"2023-09-23T21:05:55.323957+0000","last_became_peered":"2023-09-23T21:05:55.323957+0000","last_unstale":"2023-09-23T21:07:52.467107+0000","last_undegraded":"2023-09-23T21:07:52.467107+0000","last_fullsized":"2023-09-23T21:07:52.467107+0000","mapping_epoch":350,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":351,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:03:20.418924+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.11","version":"79'161","reported_seq":641,"reported_epoch":374,"state":"active+clean","last_fresh":"2023-09-23T21:07:52.467821+0000","last_change":"2023-09-23T21:05:07.301125+0000","last_active":"2023-09-23T21:07:52.467821+0000","last_peered":"2023-09-23T21:07:52.467821+0000","last_clean":"2023-09-23T21:07:52.467821+0000","last_became_active":"2023-09-23T21:05:07.300809+0000","last_became_peered":"2023-09-23T21:05:07.300809+0000","last_unstale":"2023-09-23T21:07:52.467821+0000","last_undegraded":"2023-09-23T21:07:52.467821+0000","last_fullsized":"2023-09-23T21:07:52.467821+0000","mapping_epoch":335,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":336,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:28.756634+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:28.756634+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:28:29.079665+0000","scrub_duration":0.0064925119999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":229,"reported_epoch":374,"state":"active+clean","last_fresh":"2023-09-23T21:07:52.460708+0000","last_change":"2023-09-23T21:06:25.394796+0000","last_active":"2023-09-23T21:07:52.460708+0000","last_peered":"2023-09-23T21:07:52.460708+0000","last_clean":"2023-09-23T21:07:52.460708+0000","last_became_active":"2023-09-23T21:06:25.394447+0000","last_became_peered":"2023-09-23T21:06:25.394447+0000","last_unstale":"2023-09-23T21:07:52.460708+0000","last_undegraded":"2023-09-23T21:07:52.460708+0000","last_fullsized":"2023-09-23T21:07:52.460708+0000","mapping_epoch":354,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":355,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:55:25.161895+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":359,"reported_epoch":374,"state":"active+clean","last_fresh":"2023-09-23T21:07:52.483506+0000","last_change":"2023-09-23T21:05:55.324614+0000","last_active":"2023-09-23T21:07:52.483506+0000","last_peered":"2023-09-23T21:07:52.483506+0000","last_clean":"2023-09-23T21:07:52.483506+0000","last_became_active":"2023-09-23T21:05:55.324306+0000","last_became_peered":"2023-09-23T21:05:55.324306+0000","last_unstale":"2023-09-23T21:07:52.483506+0000","last_undegraded":"2023-09-23T21:07:52.483506+0000","last_fullsized":"2023-09-23T21:07:52.483506+0000","mapping_epoch":350,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":351,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:32:58.417049+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.10","version":"79'161","reported_seq":467,"reported_epoch":374,"state":"active+clean","last_fresh":"2023-09-23T21:07:52.456056+0000","last_change":"2023-09-23T21:06:50.337099+0000","last_active":"2023-09-23T21:07:52.456056+0000","last_peered":"2023-09-23T21:07:52.456056+0000","last_clean":"2023-09-23T21:07:52.456056+0000","last_became_active":"2023-09-23T21:06:49.426789+0000","last_became_peered":"2023-09-23T21:06:49.426789+0000","last_unstale":"2023-09-23T21:07:52.456056+0000","last_undegraded":"2023-09-23T21:07:52.456056+0000","last_fullsized":"2023-09-23T21:07:52.456056+0000","mapping_epoch":359,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":360,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:06:50.336992+0000","last_deep_scrub":"79'161","last_deep_scrub_stamp":"2023-09-23T21:06:50.336992+0000","last_clean_scrub_stamp":"2023-09-23T21:06:50.336992+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:26:47.504229+0000","scrub_duration":0.0026150930000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,7],"acting":[6,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":353,"reported_epoch":374,"state":"active+clean","last_fresh":"2023-09-23T21:07:52.467131+0000","last_change":"2023-09-23T20:57:44.979736+0000","last_active":"2023-09-23T21:07:52.467131+0000","last_peered":"2023-09-23T21:07:52.467131+0000","last_clean":"2023-09-23T21:07:52.467131+0000","last_became_active":"2023-09-23T20:57:44.979356+0000","last_became_peered":"2023-09-23T20:57:44.979356+0000","last_unstale":"2023-09-23T21:07:52.467131+0000","last_undegraded":"2023-09-23T21:07:52.467131+0000","last_fullsized":"2023-09-23T21:07:52.467131+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:14:43.848832+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":97,"reported_epoch":374,"state":"active+clean","last_fresh":"2023-09-23T21:07:52.479943+0000","last_change":"2023-09-23T21:06:49.426186+0000","last_active":"2023-09-23T21:07:52.479943+0000","last_peered":"2023-09-23T21:07:52.479943+0000","last_clean":"2023-09-23T21:07:52.479943+0000","last_became_active":"2023-09-23T21:06:49.425867+0000","last_became_peered":"2023-09-23T21:06:49.425867+0000","last_unstale":"2023-09-23T21:07:52.479943+0000","last_undegraded":"2023-09-23T21:07:52.479943+0000","last_fullsized":"2023-09-23T21:07:52.479943+0000","mapping_epoch":359,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":360,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:40:21.708447+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.13","version":"79'161","reported_seq":648,"reported_epoch":374,"state":"active+clean","last_fresh":"2023-09-23T21:07:52.504353+0000","last_change":"2023-09-23T21:05:55.322760+0000","last_active":"2023-09-23T21:07:52.504353+0000","last_peered":"2023-09-23T21:07:52.504353+0000","last_clean":"2023-09-23T21:07:52.504353+0000","last_became_active":"2023-09-23T21:05:55.322483+0000","last_became_peered":"2023-09-23T21:05:55.322483+0000","last_unstale":"2023-09-23T21:07:52.504353+0000","last_undegraded":"2023-09-23T21:07:52.504353+0000","last_fullsized":"2023-09-23T21:07:52.504353+0000","mapping_epoch":350,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":351,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:36.420362+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:36.420362+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:08:05.220846+0000","scrub_duration":0.028838444000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":319,"reported_epoch":374,"state":"active+clean","last_fresh":"2023-09-23T21:07:52.466872+0000","last_change":"2023-09-23T20:56:45.991822+0000","last_active":"2023-09-23T21:07:52.466872+0000","last_peered":"2023-09-23T21:07:52.466872+0000","last_clean":"2023-09-23T21:07:52.466872+0000","last_became_active":"2023-09-23T20:56:45.991460+0000","last_became_peered":"2023-09-23T20:56:45.991460+0000","last_unstale":"2023-09-23T21:07:52.466872+0000","last_undegraded":"2023-09-23T21:07:52.466872+0000","last_fullsized":"2023-09-23T21:07:52.466872+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":205,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:20:37.503317+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":335,"reported_epoch":374,"state":"active+clean","last_fresh":"2023-09-23T21:07:52.468104+0000","last_change":"2023-09-23T20:51:10.840849+0000","last_active":"2023-09-23T21:07:52.468104+0000","last_peered":"2023-09-23T21:07:52.468104+0000","last_clean":"2023-09-23T21:07:52.468104+0000","last_became_active":"2023-09-23T20:51:10.840593+0000","last_became_peered":"2023-09-23T20:51:10.840593+0000","last_unstale":"2023-09-23T21:07:52.468104+0000","last_undegraded":"2023-09-23T21:07:52.468104+0000","last_fullsized":"2023-09-23T21:07:52.468104+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:30:52.297625+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.12","version":"79'161","reported_seq":536,"reported_epoch":374,"state":"active+clean","last_fresh":"2023-09-23T21:07:52.470176+0000","last_change":"2023-09-23T21:05:56.785264+0000","last_active":"2023-09-23T21:07:52.470176+0000","last_peered":"2023-09-23T21:07:52.470176+0000","last_clean":"2023-09-23T21:07:52.470176+0000","last_became_active":"2023-09-23T21:05:55.322810+0000","last_became_peered":"2023-09-23T21:05:55.322810+0000","last_unstale":"2023-09-23T21:07:52.470176+0000","last_undegraded":"2023-09-23T21:07:52.470176+0000","last_fullsized":"2023-09-23T21:07:52.470176+0000","mapping_epoch":350,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":351,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:05:56.785173+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:05:56.785173+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:56:12.932786+0000","scrub_duration":0.0042114370000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":407,"reported_epoch":374,"state":"active+clean","last_fresh":"2023-09-23T21:07:52.456190+0000","last_change":"2023-09-23T21:05:04.332774+0000","last_active":"2023-09-23T21:07:52.456190+0000","last_peered":"2023-09-23T21:07:52.456190+0000","last_clean":"2023-09-23T21:07:52.456190+0000","last_became_active":"2023-09-23T21:05:04.304541+0000","last_became_peered":"2023-09-23T21:05:04.304541+0000","last_unstale":"2023-09-23T21:07:52.456190+0000","last_undegraded":"2023-09-23T21:07:52.456190+0000","last_fullsized":"2023-09-23T21:07:52.456190+0000","mapping_epoch":332,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:27:02.401939+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":182,"reported_epoch":374,"state":"active+clean","last_fresh":"2023-09-23T21:07:52.484480+0000","last_change":"2023-09-23T21:05:55.325421+0000","last_active":"2023-09-23T21:07:52.484480+0000","last_peered":"2023-09-23T21:07:52.484480+0000","last_clean":"2023-09-23T21:07:52.484480+0000","last_became_active":"2023-09-23T21:05:55.325098+0000","last_became_peered":"2023-09-23T21:05:55.325098+0000","last_unstale":"2023-09-23T21:07:52.484480+0000","last_undegraded":"2023-09-23T21:07:52.484480+0000","last_fullsized":"2023-09-23T21:07:52.484480+0000","mapping_epoch":350,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":351,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:57:47.564327+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.5","version":"375'8857","reported_seq":8896,"reported_epoch":375,"state":"active+clean","last_fresh":"2023-09-23T21:08:02.269972+0000","last_change":"2023-09-23T21:05:48.557604+0000","last_active":"2023-09-23T21:08:02.269972+0000","last_peered":"2023-09-23T21:08:02.269972+0000","last_clean":"2023-09-23T21:08:02.269972+0000","last_became_active":"2023-09-23T21:05:48.557179+0000","last_became_peered":"2023-09-23T21:05:48.557179+0000","last_unstale":"2023-09-23T21:08:02.269972+0000","last_undegraded":"2023-09-23T21:08:02.269972+0000","last_fullsized":"2023-09-23T21:08:02.269972+0000","mapping_epoch":346,"log_start":"351'500","ondisk_log_start":"351'500","created":346,"last_epoch_clean":347,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:05:47.261750+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:05:47.261750+0000","last_clean_scrub_stamp":"2023-09-23T21:05:47.261750+0000","objects_scrubbed":0,"log_size":8357,"log_dups_size":0,"ondisk_log_size":8357,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:17:18.270981+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":339673088,"num_objects":5183,"num_object_clones":0,"num_object_copies":15549,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5183,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8857,"num_write_kb":449280,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.d","version":"79'161","reported_seq":512,"reported_epoch":374,"state":"active+clean","last_fresh":"2023-09-23T21:07:52.462433+0000","last_change":"2023-09-23T21:06:38.762807+0000","last_active":"2023-09-23T21:07:52.462433+0000","last_peered":"2023-09-23T21:07:52.462433+0000","last_clean":"2023-09-23T21:07:52.462433+0000","last_became_active":"2023-09-23T21:06:25.394249+0000","last_became_peered":"2023-09-23T21:06:25.394249+0000","last_unstale":"2023-09-23T21:07:52.462433+0000","last_undegraded":"2023-09-23T21:07:52.462433+0000","last_fullsized":"2023-09-23T21:07:52.462433+0000","mapping_epoch":354,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":355,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:06:38.762742+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:06:38.762742+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:28:31.666726+0000","scrub_duration":0.0053529800000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":238,"reported_epoch":374,"state":"active+clean","last_fresh":"2023-09-23T21:07:52.460712+0000","last_change":"2023-09-23T21:06:25.394656+0000","last_active":"2023-09-23T21:07:52.460712+0000","last_peered":"2023-09-23T21:07:52.460712+0000","last_clean":"2023-09-23T21:07:52.460712+0000","last_became_active":"2023-09-23T21:06:25.394290+0000","last_became_peered":"2023-09-23T21:06:25.394290+0000","last_unstale":"2023-09-23T21:07:52.460712+0000","last_undegraded":"2023-09-23T21:07:52.460712+0000","last_fullsized":"2023-09-23T21:07:52.460712+0000","mapping_epoch":354,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":355,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:27:19.996925+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":66,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":52,"reported_epoch":374,"state":"active+clean","last_fresh":"2023-09-23T21:07:52.465369+0000","last_change":"2023-09-23T21:05:04.307967+0000","last_active":"2023-09-23T21:07:52.465369+0000","last_peered":"2023-09-23T21:07:52.465369+0000","last_clean":"2023-09-23T21:07:52.465369+0000","last_became_active":"2023-09-23T21:05:04.307679+0000","last_became_peered":"2023-09-23T21:05:04.307679+0000","last_unstale":"2023-09-23T21:07:52.465369+0000","last_undegraded":"2023-09-23T21:07:52.465369+0000","last_fullsized":"2023-09-23T21:07:52.465369+0000","mapping_epoch":332,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:39:58.603484+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.4","version":"375'8885","reported_seq":8921,"reported_epoch":375,"state":"active+clean","last_fresh":"2023-09-23T21:08:03.443823+0000","last_change":"2023-09-23T21:05:55.320259+0000","last_active":"2023-09-23T21:08:03.443823+0000","last_peered":"2023-09-23T21:08:03.443823+0000","last_clean":"2023-09-23T21:08:03.443823+0000","last_became_active":"2023-09-23T21:05:55.319895+0000","last_became_peered":"2023-09-23T21:05:55.319895+0000","last_unstale":"2023-09-23T21:08:03.443823+0000","last_undegraded":"2023-09-23T21:08:03.443823+0000","last_fullsized":"2023-09-23T21:08:03.443823+0000","mapping_epoch":350,"log_start":"352'2300","ondisk_log_start":"352'2300","created":346,"last_epoch_clean":351,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:05:47.261750+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:05:47.261750+0000","last_clean_scrub_stamp":"2023-09-23T21:05:47.261750+0000","objects_scrubbed":0,"log_size":6585,"log_dups_size":0,"ondisk_log_size":6585,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:39:53.181523+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":335872000,"num_objects":5125,"num_object_clones":0,"num_object_copies":15375,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5125,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8885,"num_write_kb":448320,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.c","version":"79'161","reported_seq":679,"reported_epoch":374,"state":"active+clean","last_fresh":"2023-09-23T21:07:52.483287+0000","last_change":"2023-09-23T20:57:22.041071+0000","last_active":"2023-09-23T21:07:52.483287+0000","last_peered":"2023-09-23T21:07:52.483287+0000","last_clean":"2023-09-23T21:07:52.483287+0000","last_became_active":"2023-09-23T20:57:15.973204+0000","last_became_peered":"2023-09-23T20:57:15.973204+0000","last_unstale":"2023-09-23T21:07:52.483287+0000","last_undegraded":"2023-09-23T21:07:52.483287+0000","last_fullsized":"2023-09-23T21:07:52.483287+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:22.040907+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:22.040907+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:47:02.019978+0000","scrub_duration":0.013021176000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":225,"reported_epoch":374,"state":"active+clean","last_fresh":"2023-09-23T21:07:52.462898+0000","last_change":"2023-09-23T21:06:25.394978+0000","last_active":"2023-09-23T21:07:52.462898+0000","last_peered":"2023-09-23T21:07:52.462898+0000","last_clean":"2023-09-23T21:07:52.462898+0000","last_became_active":"2023-09-23T21:06:25.394599+0000","last_became_peered":"2023-09-23T21:06:25.394599+0000","last_unstale":"2023-09-23T21:07:52.462898+0000","last_undegraded":"2023-09-23T21:07:52.462898+0000","last_fullsized":"2023-09-23T21:07:52.462898+0000","mapping_epoch":354,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":355,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:52:23.477998+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":313,"reported_epoch":374,"state":"active+clean","last_fresh":"2023-09-23T21:07:52.467058+0000","last_change":"2023-09-23T21:05:55.325375+0000","last_active":"2023-09-23T21:07:52.467058+0000","last_peered":"2023-09-23T21:07:52.467058+0000","last_clean":"2023-09-23T21:07:52.467058+0000","last_became_active":"2023-09-23T21:05:55.324170+0000","last_became_peered":"2023-09-23T21:05:55.324170+0000","last_unstale":"2023-09-23T21:07:52.467058+0000","last_undegraded":"2023-09-23T21:07:52.467058+0000","last_fullsized":"2023-09-23T21:07:52.467058+0000","mapping_epoch":350,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":351,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:08:07.299605+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.7","version":"375'9016","reported_seq":9061,"reported_epoch":375,"state":"active+clean","last_fresh":"2023-09-23T21:08:03.949757+0000","last_change":"2023-09-23T21:06:49.432007+0000","last_active":"2023-09-23T21:08:03.949757+0000","last_peered":"2023-09-23T21:08:03.949757+0000","last_clean":"2023-09-23T21:08:03.949757+0000","last_became_active":"2023-09-23T21:06:49.425988+0000","last_became_peered":"2023-09-23T21:06:49.425988+0000","last_unstale":"2023-09-23T21:08:03.949757+0000","last_undegraded":"2023-09-23T21:08:03.949757+0000","last_fullsized":"2023-09-23T21:08:03.949757+0000","mapping_epoch":359,"log_start":"352'2700","ondisk_log_start":"352'2700","created":346,"last_epoch_clean":360,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:05:47.261750+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:05:47.261750+0000","last_clean_scrub_stamp":"2023-09-23T21:05:47.261750+0000","objects_scrubbed":0,"log_size":6316,"log_dups_size":0,"ondisk_log_size":6316,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:37:23.206892+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":335806464,"num_objects":5124,"num_object_clones":0,"num_object_copies":15372,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5124,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9016,"num_write_kb":452480,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.f","version":"79'161","reported_seq":719,"reported_epoch":374,"state":"active+clean","last_fresh":"2023-09-23T21:07:52.456763+0000","last_change":"2023-09-23T21:05:07.299402+0000","last_active":"2023-09-23T21:07:52.456763+0000","last_peered":"2023-09-23T21:07:52.456763+0000","last_clean":"2023-09-23T21:07:52.456763+0000","last_became_active":"2023-09-23T21:05:07.299059+0000","last_became_peered":"2023-09-23T21:05:07.299059+0000","last_unstale":"2023-09-23T21:07:52.456763+0000","last_undegraded":"2023-09-23T21:07:52.456763+0000","last_fullsized":"2023-09-23T21:07:52.456763+0000","mapping_epoch":335,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":336,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:27.880776+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:27.880776+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:51:23.907187+0000","scrub_duration":0.0014004899999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":56,"reported_epoch":374,"state":"active+clean","last_fresh":"2023-09-23T21:07:52.465818+0000","last_change":"2023-09-23T21:05:04.355129+0000","last_active":"2023-09-23T21:07:52.465818+0000","last_peered":"2023-09-23T21:07:52.465818+0000","last_clean":"2023-09-23T21:07:52.465818+0000","last_became_active":"2023-09-23T21:05:04.326482+0000","last_became_peered":"2023-09-23T21:05:04.326482+0000","last_unstale":"2023-09-23T21:07:52.465818+0000","last_undegraded":"2023-09-23T21:07:52.465818+0000","last_fullsized":"2023-09-23T21:07:52.465818+0000","mapping_epoch":332,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:35:10.887749+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":331,"reported_epoch":374,"state":"active+clean","last_fresh":"2023-09-23T21:07:52.456209+0000","last_change":"2023-09-23T21:05:55.321246+0000","last_active":"2023-09-23T21:07:52.456209+0000","last_peered":"2023-09-23T21:07:52.456209+0000","last_clean":"2023-09-23T21:07:52.456209+0000","last_became_active":"2023-09-23T21:05:55.320918+0000","last_became_peered":"2023-09-23T21:05:55.320918+0000","last_unstale":"2023-09-23T21:07:52.456209+0000","last_undegraded":"2023-09-23T21:07:52.456209+0000","last_fullsized":"2023-09-23T21:07:52.456209+0000","mapping_epoch":350,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":351,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:48:50.646324+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"9.6","version":"375'8824","reported_seq":8876,"reported_epoch":375,"state":"active+clean","last_fresh":"2023-09-23T21:08:01.003126+0000","last_change":"2023-09-23T21:06:49.429119+0000","last_active":"2023-09-23T21:08:01.003126+0000","last_peered":"2023-09-23T21:08:01.003126+0000","last_clean":"2023-09-23T21:08:01.003126+0000","last_became_active":"2023-09-23T21:06:49.423016+0000","last_became_peered":"2023-09-23T21:06:49.423016+0000","last_unstale":"2023-09-23T21:08:01.003126+0000","last_undegraded":"2023-09-23T21:08:01.003126+0000","last_fullsized":"2023-09-23T21:08:01.003126+0000","mapping_epoch":359,"log_start":"0'0","ondisk_log_start":"0'0","created":346,"last_epoch_clean":360,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:05:47.261750+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:05:47.261750+0000","last_clean_scrub_stamp":"2023-09-23T21:05:47.261750+0000","objects_scrubbed":0,"log_size":8824,"log_dups_size":0,"ondisk_log_size":8824,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:55:25.118949+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":345636864,"num_objects":5274,"num_object_clones":0,"num_object_copies":15822,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5274,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8824,"num_write_kb":451136,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,1],"acting":[2,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.e","version":"79'161","reported_seq":586,"reported_epoch":374,"state":"active+clean","last_fresh":"2023-09-23T21:07:52.470282+0000","last_change":"2023-09-23T20:57:34.132738+0000","last_active":"2023-09-23T21:07:52.470282+0000","last_peered":"2023-09-23T21:07:52.470282+0000","last_clean":"2023-09-23T21:07:52.470282+0000","last_became_active":"2023-09-23T20:57:15.974318+0000","last_became_peered":"2023-09-23T20:57:15.974318+0000","last_unstale":"2023-09-23T21:07:52.470282+0000","last_undegraded":"2023-09-23T21:07:52.470282+0000","last_fullsized":"2023-09-23T21:07:52.470282+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:34.132660+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:34.132660+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:07:07.739438+0000","scrub_duration":0.005229694,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":140,"reported_epoch":374,"state":"active+clean","last_fresh":"2023-09-23T21:07:52.483700+0000","last_change":"2023-09-23T21:06:49.423445+0000","last_active":"2023-09-23T21:07:52.483700+0000","last_peered":"2023-09-23T21:07:52.483700+0000","last_clean":"2023-09-23T21:07:52.483700+0000","last_became_active":"2023-09-23T21:06:49.423169+0000","last_became_peered":"2023-09-23T21:06:49.423169+0000","last_unstale":"2023-09-23T21:07:52.483700+0000","last_undegraded":"2023-09-23T21:07:52.483700+0000","last_fullsized":"2023-09-23T21:07:52.483700+0000","mapping_epoch":359,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":360,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:47:52.999823+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":367,"reported_epoch":374,"state":"active+clean","last_fresh":"2023-09-23T21:07:52.467869+0000","last_change":"2023-09-23T20:54:13.928715+0000","last_active":"2023-09-23T21:07:52.467869+0000","last_peered":"2023-09-23T21:07:52.467869+0000","last_clean":"2023-09-23T21:07:52.467869+0000","last_became_active":"2023-09-23T20:54:13.928200+0000","last_became_peered":"2023-09-23T20:54:13.928200+0000","last_unstale":"2023-09-23T21:07:52.467869+0000","last_undegraded":"2023-09-23T21:07:52.467869+0000","last_fullsized":"2023-09-23T21:07:52.467869+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:41:16.927513+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.1","version":"375'9106","reported_seq":9145,"reported_epoch":375,"state":"active+clean","last_fresh":"2023-09-23T21:08:03.953635+0000","last_change":"2023-09-23T21:05:48.558005+0000","last_active":"2023-09-23T21:08:03.953635+0000","last_peered":"2023-09-23T21:08:03.953635+0000","last_clean":"2023-09-23T21:08:03.953635+0000","last_became_active":"2023-09-23T21:05:48.557590+0000","last_became_peered":"2023-09-23T21:05:48.557590+0000","last_unstale":"2023-09-23T21:08:03.953635+0000","last_undegraded":"2023-09-23T21:08:03.953635+0000","last_fullsized":"2023-09-23T21:08:03.953635+0000","mapping_epoch":346,"log_start":"352'2800","ondisk_log_start":"352'2800","created":346,"last_epoch_clean":347,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:05:47.261750+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:05:47.261750+0000","last_clean_scrub_stamp":"2023-09-23T21:05:47.261750+0000","objects_scrubbed":0,"log_size":6306,"log_dups_size":0,"ondisk_log_size":6306,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:19:41.388978+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":343015424,"num_objects":5234,"num_object_clones":0,"num_object_copies":15702,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5234,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9106,"num_write_kb":458880,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.9","version":"79'161","reported_seq":752,"reported_epoch":374,"state":"active+clean","last_fresh":"2023-09-23T21:07:52.478805+0000","last_change":"2023-09-23T20:56:46.921939+0000","last_active":"2023-09-23T21:07:52.478805+0000","last_peered":"2023-09-23T21:07:52.478805+0000","last_clean":"2023-09-23T21:07:52.478805+0000","last_became_active":"2023-09-23T20:56:45.989908+0000","last_became_peered":"2023-09-23T20:56:45.989908+0000","last_unstale":"2023-09-23T21:07:52.478805+0000","last_undegraded":"2023-09-23T21:07:52.478805+0000","last_fullsized":"2023-09-23T21:07:52.478805+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":205,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:56:46.921739+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:56:46.921739+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:20:42.635110+0000","scrub_duration":0.0071954109999999996,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":368,"reported_epoch":374,"state":"active+clean","last_fresh":"2023-09-23T21:07:52.483242+0000","last_change":"2023-09-23T20:54:13.961788+0000","last_active":"2023-09-23T21:07:52.483242+0000","last_peered":"2023-09-23T21:07:52.483242+0000","last_clean":"2023-09-23T21:07:52.483242+0000","last_became_active":"2023-09-23T20:54:13.940937+0000","last_became_peered":"2023-09-23T20:54:13.940937+0000","last_unstale":"2023-09-23T21:07:52.483242+0000","last_undegraded":"2023-09-23T21:07:52.483242+0000","last_fullsized":"2023-09-23T21:07:52.483242+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:11:56.785299+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":269,"reported_epoch":374,"state":"active+clean","last_fresh":"2023-09-23T21:07:52.467526+0000","last_change":"2023-09-23T20:55:35.949670+0000","last_active":"2023-09-23T21:07:52.467526+0000","last_peered":"2023-09-23T21:07:52.467526+0000","last_clean":"2023-09-23T21:07:52.467526+0000","last_became_active":"2023-09-23T20:55:35.949140+0000","last_became_peered":"2023-09-23T20:55:35.949140+0000","last_unstale":"2023-09-23T21:07:52.467526+0000","last_undegraded":"2023-09-23T21:07:52.467526+0000","last_fullsized":"2023-09-23T21:07:52.467526+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:55:59.340303+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.0","version":"375'9008","reported_seq":9044,"reported_epoch":375,"state":"active+clean","last_fresh":"2023-09-23T21:08:03.419023+0000","last_change":"2023-09-23T21:05:55.315757+0000","last_active":"2023-09-23T21:08:03.419023+0000","last_peered":"2023-09-23T21:08:03.419023+0000","last_clean":"2023-09-23T21:08:03.419023+0000","last_became_active":"2023-09-23T21:05:55.314884+0000","last_became_peered":"2023-09-23T21:05:55.314884+0000","last_unstale":"2023-09-23T21:08:03.419023+0000","last_undegraded":"2023-09-23T21:08:03.419023+0000","last_fullsized":"2023-09-23T21:08:03.419023+0000","mapping_epoch":350,"log_start":"352'2400","ondisk_log_start":"352'2400","created":346,"last_epoch_clean":351,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:05:47.261750+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:05:47.261750+0000","last_clean_scrub_stamp":"2023-09-23T21:05:47.261750+0000","objects_scrubbed":0,"log_size":6608,"log_dups_size":0,"ondisk_log_size":6608,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:41:19.184012+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":337641472,"num_objects":5152,"num_object_clones":0,"num_object_copies":15456,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5152,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9008,"num_write_kb":453120,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.8","version":"79'161","reported_seq":483,"reported_epoch":374,"state":"active+clean","last_fresh":"2023-09-23T21:07:52.468935+0000","last_change":"2023-09-23T21:06:52.099024+0000","last_active":"2023-09-23T21:07:52.468935+0000","last_peered":"2023-09-23T21:07:52.468935+0000","last_clean":"2023-09-23T21:07:52.468935+0000","last_became_active":"2023-09-23T21:06:49.425012+0000","last_became_peered":"2023-09-23T21:06:49.425012+0000","last_unstale":"2023-09-23T21:07:52.468935+0000","last_undegraded":"2023-09-23T21:07:52.468935+0000","last_fullsized":"2023-09-23T21:07:52.468935+0000","mapping_epoch":359,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":360,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:06:52.098882+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:06:52.098882+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:48:36.414396+0000","scrub_duration":0.0092173299999999993,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":354,"reported_epoch":374,"state":"active+clean","last_fresh":"2023-09-23T21:07:52.456395+0000","last_change":"2023-09-23T21:05:55.321230+0000","last_active":"2023-09-23T21:07:52.456395+0000","last_peered":"2023-09-23T21:07:52.456395+0000","last_clean":"2023-09-23T21:07:52.456395+0000","last_became_active":"2023-09-23T21:05:55.320887+0000","last_became_peered":"2023-09-23T21:05:55.320887+0000","last_unstale":"2023-09-23T21:07:52.456395+0000","last_undegraded":"2023-09-23T21:07:52.456395+0000","last_fullsized":"2023-09-23T21:07:52.456395+0000","mapping_epoch":350,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":351,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:54:10.329947+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":269,"reported_epoch":374,"state":"active+clean","last_fresh":"2023-09-23T21:07:52.470416+0000","last_change":"2023-09-23T21:05:55.322692+0000","last_active":"2023-09-23T21:07:52.470416+0000","last_peered":"2023-09-23T21:07:52.470416+0000","last_clean":"2023-09-23T21:07:52.470416+0000","last_became_active":"2023-09-23T21:05:55.322388+0000","last_became_peered":"2023-09-23T21:05:55.322388+0000","last_unstale":"2023-09-23T21:07:52.470416+0000","last_undegraded":"2023-09-23T21:07:52.470416+0000","last_fullsized":"2023-09-23T21:07:52.470416+0000","mapping_epoch":350,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":351,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:43:27.774990+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.3","version":"375'8776","reported_seq":8828,"reported_epoch":375,"state":"active+clean","last_fresh":"2023-09-23T21:08:02.129577+0000","last_change":"2023-09-23T21:06:25.421271+0000","last_active":"2023-09-23T21:08:02.129577+0000","last_peered":"2023-09-23T21:08:02.129577+0000","last_clean":"2023-09-23T21:08:02.129577+0000","last_became_active":"2023-09-23T21:06:25.400342+0000","last_became_peered":"2023-09-23T21:06:25.400342+0000","last_unstale":"2023-09-23T21:08:02.129577+0000","last_undegraded":"2023-09-23T21:08:02.129577+0000","last_fullsized":"2023-09-23T21:08:02.129577+0000","mapping_epoch":354,"log_start":"352'2300","ondisk_log_start":"352'2300","created":346,"last_epoch_clean":355,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:05:47.261750+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:05:47.261750+0000","last_clean_scrub_stamp":"2023-09-23T21:05:47.261750+0000","objects_scrubbed":0,"log_size":6476,"log_dups_size":0,"ondisk_log_size":6476,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:32:39.644536+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":342491136,"num_objects":5226,"num_object_clones":0,"num_object_copies":15678,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5226,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8776,"num_write_kb":448064,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.b","version":"79'161","reported_seq":704,"reported_epoch":374,"state":"active+clean","last_fresh":"2023-09-23T21:07:52.504684+0000","last_change":"2023-09-23T21:05:55.772369+0000","last_active":"2023-09-23T21:07:52.504684+0000","last_peered":"2023-09-23T21:07:52.504684+0000","last_clean":"2023-09-23T21:07:52.504684+0000","last_became_active":"2023-09-23T21:05:55.316405+0000","last_became_peered":"2023-09-23T21:05:55.316405+0000","last_unstale":"2023-09-23T21:07:52.504684+0000","last_undegraded":"2023-09-23T21:07:52.504684+0000","last_fullsized":"2023-09-23T21:07:52.504684+0000","mapping_epoch":350,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":351,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:05:55.772264+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:05:55.772264+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:14:30.697692+0000","scrub_duration":0.001862507,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,5],"acting":[1,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":252,"reported_epoch":374,"state":"active+clean","last_fresh":"2023-09-23T21:07:52.466919+0000","last_change":"2023-09-23T20:54:13.950254+0000","last_active":"2023-09-23T21:07:52.466919+0000","last_peered":"2023-09-23T21:07:52.466919+0000","last_clean":"2023-09-23T21:07:52.466919+0000","last_became_active":"2023-09-23T20:54:13.949934+0000","last_became_peered":"2023-09-23T20:54:13.949934+0000","last_unstale":"2023-09-23T21:07:52.466919+0000","last_undegraded":"2023-09-23T21:07:52.466919+0000","last_fullsized":"2023-09-23T21:07:52.466919+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:30:32.847167+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":286,"reported_epoch":374,"state":"active+clean","last_fresh":"2023-09-23T21:07:52.470839+0000","last_change":"2023-09-23T21:05:55.325695+0000","last_active":"2023-09-23T21:07:52.470839+0000","last_peered":"2023-09-23T21:07:52.470839+0000","last_clean":"2023-09-23T21:07:52.470839+0000","last_became_active":"2023-09-23T21:05:55.324518+0000","last_became_peered":"2023-09-23T21:05:55.324518+0000","last_unstale":"2023-09-23T21:07:52.470839+0000","last_undegraded":"2023-09-23T21:07:52.470839+0000","last_fullsized":"2023-09-23T21:07:52.470839+0000","mapping_epoch":350,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":351,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:57:18.249965+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.2","version":"375'8918","reported_seq":8962,"reported_epoch":375,"state":"active+clean","last_fresh":"2023-09-23T21:08:03.948872+0000","last_change":"2023-09-23T21:05:55.325629+0000","last_active":"2023-09-23T21:08:03.948872+0000","last_peered":"2023-09-23T21:08:03.948872+0000","last_clean":"2023-09-23T21:08:03.948872+0000","last_became_active":"2023-09-23T21:05:55.325108+0000","last_became_peered":"2023-09-23T21:05:55.325108+0000","last_unstale":"2023-09-23T21:08:03.948872+0000","last_undegraded":"2023-09-23T21:08:03.948872+0000","last_fullsized":"2023-09-23T21:08:03.948872+0000","mapping_epoch":350,"log_start":"352'2600","ondisk_log_start":"352'2600","created":346,"last_epoch_clean":351,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:05:47.261750+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:05:47.261750+0000","last_clean_scrub_stamp":"2023-09-23T21:05:47.261750+0000","objects_scrubbed":0,"log_size":6318,"log_dups_size":0,"ondisk_log_size":6318,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:50:06.272050+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":341442560,"num_objects":5210,"num_object_clones":0,"num_object_copies":15630,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5210,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8918,"num_write_kb":452096,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.a","version":"79'161","reported_seq":475,"reported_epoch":374,"state":"active+clean","last_fresh":"2023-09-23T21:07:52.472668+0000","last_change":"2023-09-23T21:06:51.075617+0000","last_active":"2023-09-23T21:07:52.472668+0000","last_peered":"2023-09-23T21:07:52.472668+0000","last_clean":"2023-09-23T21:07:52.472668+0000","last_became_active":"2023-09-23T21:06:49.423291+0000","last_became_peered":"2023-09-23T21:06:49.423291+0000","last_unstale":"2023-09-23T21:07:52.472668+0000","last_undegraded":"2023-09-23T21:07:52.472668+0000","last_fullsized":"2023-09-23T21:07:52.472668+0000","mapping_epoch":359,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":360,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:06:51.075448+0000","last_deep_scrub":"79'161","last_deep_scrub_stamp":"2023-09-23T21:05:32.630292+0000","last_clean_scrub_stamp":"2023-09-23T21:06:51.075448+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:35:01.883490+0000","scrub_duration":0.0054793890000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":182,"reported_epoch":374,"state":"active+clean","last_fresh":"2023-09-23T21:07:52.477995+0000","last_change":"2023-09-23T21:05:55.325999+0000","last_active":"2023-09-23T21:07:52.477995+0000","last_peered":"2023-09-23T21:07:52.477995+0000","last_clean":"2023-09-23T21:07:52.477995+0000","last_became_active":"2023-09-23T21:05:55.325669+0000","last_became_peered":"2023-09-23T21:05:55.325669+0000","last_unstale":"2023-09-23T21:07:52.477995+0000","last_undegraded":"2023-09-23T21:07:52.477995+0000","last_fullsized":"2023-09-23T21:07:52.477995+0000","mapping_epoch":350,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":351,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:30:53.984244+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":347,"reported_epoch":374,"state":"active+clean","last_fresh":"2023-09-23T21:07:52.510760+0000","last_change":"2023-09-23T21:05:55.322566+0000","last_active":"2023-09-23T21:07:52.510760+0000","last_peered":"2023-09-23T21:07:52.510760+0000","last_clean":"2023-09-23T21:07:52.510760+0000","last_became_active":"2023-09-23T21:05:55.322280+0000","last_became_peered":"2023-09-23T21:05:55.322280+0000","last_unstale":"2023-09-23T21:07:52.510760+0000","last_undegraded":"2023-09-23T21:07:52.510760+0000","last_fullsized":"2023-09-23T21:07:52.510760+0000","mapping_epoch":350,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":351,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:48:14.297230+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.f","version":"375'8870","reported_seq":8914,"reported_epoch":375,"state":"active+clean","last_fresh":"2023-09-23T21:08:02.184265+0000","last_change":"2023-09-23T21:05:55.325083+0000","last_active":"2023-09-23T21:08:02.184265+0000","last_peered":"2023-09-23T21:08:02.184265+0000","last_clean":"2023-09-23T21:08:02.184265+0000","last_became_active":"2023-09-23T21:05:55.324768+0000","last_became_peered":"2023-09-23T21:05:55.324768+0000","last_unstale":"2023-09-23T21:08:02.184265+0000","last_undegraded":"2023-09-23T21:08:02.184265+0000","last_fullsized":"2023-09-23T21:08:02.184265+0000","mapping_epoch":350,"log_start":"351'900","ondisk_log_start":"351'900","created":346,"last_epoch_clean":351,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:05:47.261750+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:05:47.261750+0000","last_clean_scrub_stamp":"2023-09-23T21:05:47.261750+0000","objects_scrubbed":0,"log_size":7970,"log_dups_size":0,"ondisk_log_size":7970,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:25:42.316668+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":346423296,"num_objects":5286,"num_object_clones":0,"num_object_copies":15858,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5286,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8870,"num_write_kb":452992,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.7","version":"79'161","reported_seq":680,"reported_epoch":374,"state":"active+clean","last_fresh":"2023-09-23T21:07:52.456598+0000","last_change":"2023-09-23T20:57:48.243572+0000","last_active":"2023-09-23T21:07:52.456598+0000","last_peered":"2023-09-23T21:07:52.456598+0000","last_clean":"2023-09-23T21:07:52.456598+0000","last_became_active":"2023-09-23T20:57:48.243229+0000","last_became_peered":"2023-09-23T20:57:48.243229+0000","last_unstale":"2023-09-23T21:07:52.456598+0000","last_undegraded":"2023-09-23T21:07:52.456598+0000","last_fullsized":"2023-09-23T21:07:52.456598+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":3,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:25.916867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:25.916867+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:23:19.385831+0000","scrub_duration":0.001665323,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":352,"reported_epoch":374,"state":"active+clean","last_fresh":"2023-09-23T21:07:58.449004+0000","last_change":"2023-09-23T21:05:55.317140+0000","last_active":"2023-09-23T21:07:58.449004+0000","last_peered":"2023-09-23T21:07:58.449004+0000","last_clean":"2023-09-23T21:07:58.449004+0000","last_became_active":"2023-09-23T21:05:55.316187+0000","last_became_peered":"2023-09-23T21:05:55.316187+0000","last_unstale":"2023-09-23T21:07:58.449004+0000","last_undegraded":"2023-09-23T21:07:58.449004+0000","last_fullsized":"2023-09-23T21:07:58.449004+0000","mapping_epoch":350,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":351,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:34:57.248381+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":52,"reported_epoch":374,"state":"active+clean","last_fresh":"2023-09-23T21:07:52.465922+0000","last_change":"2023-09-23T21:05:04.326889+0000","last_active":"2023-09-23T21:07:52.465922+0000","last_peered":"2023-09-23T21:07:52.465922+0000","last_clean":"2023-09-23T21:07:52.465922+0000","last_became_active":"2023-09-23T21:05:04.326032+0000","last_became_peered":"2023-09-23T21:05:04.326032+0000","last_unstale":"2023-09-23T21:07:52.465922+0000","last_undegraded":"2023-09-23T21:07:52.465922+0000","last_fullsized":"2023-09-23T21:07:52.465922+0000","mapping_epoch":332,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:00:17.507889+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":365,"reported_epoch":374,"state":"active+clean","last_fresh":"2023-09-23T21:07:58.448939+0000","last_change":"2023-09-23T21:05:55.319889+0000","last_active":"2023-09-23T21:07:58.448939+0000","last_peered":"2023-09-23T21:07:58.448939+0000","last_clean":"2023-09-23T21:07:58.448939+0000","last_became_active":"2023-09-23T21:05:55.319400+0000","last_became_peered":"2023-09-23T21:05:55.319400+0000","last_unstale":"2023-09-23T21:07:58.448939+0000","last_undegraded":"2023-09-23T21:07:58.448939+0000","last_fullsized":"2023-09-23T21:07:58.448939+0000","mapping_epoch":350,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":351,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:21:17.483215+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.9","version":"375'9017","reported_seq":9056,"reported_epoch":375,"state":"active+clean","last_fresh":"2023-09-23T21:08:02.305628+0000","last_change":"2023-09-23T21:05:48.557743+0000","last_active":"2023-09-23T21:08:02.305628+0000","last_peered":"2023-09-23T21:08:02.305628+0000","last_clean":"2023-09-23T21:08:02.305628+0000","last_became_active":"2023-09-23T21:05:48.557419+0000","last_became_peered":"2023-09-23T21:05:48.557419+0000","last_unstale":"2023-09-23T21:08:02.305628+0000","last_undegraded":"2023-09-23T21:08:02.305628+0000","last_fullsized":"2023-09-23T21:08:02.305628+0000","mapping_epoch":346,"log_start":"351'600","ondisk_log_start":"351'600","created":346,"last_epoch_clean":347,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:05:47.261750+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:05:47.261750+0000","last_clean_scrub_stamp":"2023-09-23T21:05:47.261750+0000","objects_scrubbed":0,"log_size":8417,"log_dups_size":0,"ondisk_log_size":8417,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:34:16.622408+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":350683136,"num_objects":5351,"num_object_clones":0,"num_object_copies":16053,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5351,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9017,"num_write_kb":459776,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,5],"acting":[0,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1","version":"79'161","reported_seq":482,"reported_epoch":374,"state":"active+clean","last_fresh":"2023-09-23T21:07:52.467919+0000","last_change":"2023-09-23T21:06:52.576036+0000","last_active":"2023-09-23T21:07:52.467919+0000","last_peered":"2023-09-23T21:07:52.467919+0000","last_clean":"2023-09-23T21:07:52.467919+0000","last_became_active":"2023-09-23T21:06:49.422806+0000","last_became_peered":"2023-09-23T21:06:49.422806+0000","last_unstale":"2023-09-23T21:07:52.467919+0000","last_undegraded":"2023-09-23T21:07:52.467919+0000","last_fullsized":"2023-09-23T21:07:52.467919+0000","mapping_epoch":359,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":360,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:06:52.575900+0000","last_deep_scrub":"79'161","last_deep_scrub_stamp":"2023-09-23T21:05:30.652189+0000","last_clean_scrub_stamp":"2023-09-23T21:06:52.575900+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:59:24.733331+0000","scrub_duration":0.017723466,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":134,"reported_epoch":374,"state":"active+clean","last_fresh":"2023-09-23T21:07:52.491679+0000","last_change":"2023-09-23T21:06:49.426087+0000","last_active":"2023-09-23T21:07:52.491679+0000","last_peered":"2023-09-23T21:07:52.491679+0000","last_clean":"2023-09-23T21:07:52.491679+0000","last_became_active":"2023-09-23T21:06:49.425698+0000","last_became_peered":"2023-09-23T21:06:49.425698+0000","last_unstale":"2023-09-23T21:07:52.491679+0000","last_undegraded":"2023-09-23T21:07:52.491679+0000","last_fullsized":"2023-09-23T21:07:52.491679+0000","mapping_epoch":359,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":360,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:51:47.815811+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":269,"reported_epoch":374,"state":"active+clean","last_fresh":"2023-09-23T21:07:52.467203+0000","last_change":"2023-09-23T20:55:35.949000+0000","last_active":"2023-09-23T21:07:52.467203+0000","last_peered":"2023-09-23T21:07:52.467203+0000","last_clean":"2023-09-23T21:07:52.467203+0000","last_became_active":"2023-09-23T20:55:35.948410+0000","last_became_peered":"2023-09-23T20:55:35.948410+0000","last_unstale":"2023-09-23T21:07:52.467203+0000","last_undegraded":"2023-09-23T21:07:52.467203+0000","last_fullsized":"2023-09-23T21:07:52.467203+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:26:50.477289+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":312,"reported_epoch":374,"state":"active+clean","last_fresh":"2023-09-23T21:07:58.449022+0000","last_change":"2023-09-23T21:05:55.320314+0000","last_active":"2023-09-23T21:07:58.449022+0000","last_peered":"2023-09-23T21:07:58.449022+0000","last_clean":"2023-09-23T21:07:58.449022+0000","last_became_active":"2023-09-23T21:05:55.319935+0000","last_became_peered":"2023-09-23T21:05:55.319935+0000","last_unstale":"2023-09-23T21:07:58.449022+0000","last_undegraded":"2023-09-23T21:07:58.449022+0000","last_fullsized":"2023-09-23T21:07:58.449022+0000","mapping_epoch":350,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":351,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:52:28.754534+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.a","version":"375'8788","reported_seq":8831,"reported_epoch":375,"state":"active+clean","last_fresh":"2023-09-23T21:08:02.182907+0000","last_change":"2023-09-23T21:05:55.324585+0000","last_active":"2023-09-23T21:08:02.182907+0000","last_peered":"2023-09-23T21:08:02.182907+0000","last_clean":"2023-09-23T21:08:02.182907+0000","last_became_active":"2023-09-23T21:05:55.323754+0000","last_became_peered":"2023-09-23T21:05:55.323754+0000","last_unstale":"2023-09-23T21:08:02.182907+0000","last_undegraded":"2023-09-23T21:08:02.182907+0000","last_fullsized":"2023-09-23T21:08:02.182907+0000","mapping_epoch":350,"log_start":"351'800","ondisk_log_start":"351'800","created":346,"last_epoch_clean":351,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:05:47.261750+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:05:47.261750+0000","last_clean_scrub_stamp":"2023-09-23T21:05:47.261750+0000","objects_scrubbed":0,"log_size":7988,"log_dups_size":0,"ondisk_log_size":7988,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:38:05.236953+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":340787200,"num_objects":5200,"num_object_clones":0,"num_object_copies":15600,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5200,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8788,"num_write_kb":447616,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,7],"acting":[5,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.2","version":"79'161","reported_seq":609,"reported_epoch":374,"state":"active+clean","last_fresh":"2023-09-23T21:07:52.467590+0000","last_change":"2023-09-23T20:57:24.802586+0000","last_active":"2023-09-23T21:07:52.467590+0000","last_peered":"2023-09-23T21:07:52.467590+0000","last_clean":"2023-09-23T21:07:52.467590+0000","last_became_active":"2023-09-23T20:57:08.236114+0000","last_became_peered":"2023-09-23T20:57:08.236114+0000","last_unstale":"2023-09-23T21:07:52.467590+0000","last_undegraded":"2023-09-23T21:07:52.467590+0000","last_fullsized":"2023-09-23T21:07:52.467590+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:24.802451+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:24.802451+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:06:22.309980+0000","scrub_duration":0.0090861250000000005,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,4],"acting":[0,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":365,"reported_epoch":374,"state":"active+clean","last_fresh":"2023-09-23T21:07:52.470244+0000","last_change":"2023-09-23T20:51:10.838054+0000","last_active":"2023-09-23T21:07:52.470244+0000","last_peered":"2023-09-23T21:07:52.470244+0000","last_clean":"2023-09-23T21:07:52.470244+0000","last_became_active":"2023-09-23T20:51:10.837750+0000","last_became_peered":"2023-09-23T20:51:10.837750+0000","last_unstale":"2023-09-23T21:07:52.470244+0000","last_undegraded":"2023-09-23T21:07:52.470244+0000","last_fullsized":"2023-09-23T21:07:52.470244+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:15:51.648085+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":338,"reported_epoch":374,"state":"active+clean","last_fresh":"2023-09-23T21:07:53.444813+0000","last_change":"2023-09-23T21:05:55.315770+0000","last_active":"2023-09-23T21:07:53.444813+0000","last_peered":"2023-09-23T21:07:53.444813+0000","last_clean":"2023-09-23T21:07:53.444813+0000","last_became_active":"2023-09-23T21:05:55.315290+0000","last_became_peered":"2023-09-23T21:05:55.315290+0000","last_unstale":"2023-09-23T21:07:53.444813+0000","last_undegraded":"2023-09-23T21:07:53.444813+0000","last_fullsized":"2023-09-23T21:07:53.444813+0000","mapping_epoch":350,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":351,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:27:40.430262+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":379,"reported_epoch":375,"state":"active+clean","last_fresh":"2023-09-23T21:07:58.546326+0000","last_change":"2023-09-23T21:05:55.318863+0000","last_active":"2023-09-23T21:07:58.546326+0000","last_peered":"2023-09-23T21:07:58.546326+0000","last_clean":"2023-09-23T21:07:58.546326+0000","last_became_active":"2023-09-23T21:05:55.318489+0000","last_became_peered":"2023-09-23T21:05:55.318489+0000","last_unstale":"2023-09-23T21:07:58.546326+0000","last_undegraded":"2023-09-23T21:07:58.546326+0000","last_fullsized":"2023-09-23T21:07:58.546326+0000","mapping_epoch":350,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":351,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:45:05.912661+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.8","version":"375'8861","reported_seq":8900,"reported_epoch":375,"state":"active+clean","last_fresh":"2023-09-23T21:08:03.941600+0000","last_change":"2023-09-23T21:05:48.286303+0000","last_active":"2023-09-23T21:08:03.941600+0000","last_peered":"2023-09-23T21:08:03.941600+0000","last_clean":"2023-09-23T21:08:03.941600+0000","last_became_active":"2023-09-23T21:05:48.286008+0000","last_became_peered":"2023-09-23T21:05:48.286008+0000","last_unstale":"2023-09-23T21:08:03.941600+0000","last_undegraded":"2023-09-23T21:08:03.941600+0000","last_fullsized":"2023-09-23T21:08:03.941600+0000","mapping_epoch":346,"log_start":"352'2600","ondisk_log_start":"352'2600","created":346,"last_epoch_clean":347,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:05:47.261750+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:05:47.261750+0000","last_clean_scrub_stamp":"2023-09-23T21:05:47.261750+0000","objects_scrubbed":0,"log_size":6261,"log_dups_size":0,"ondisk_log_size":6261,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:42:05.759917+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":338624512,"num_objects":5167,"num_object_clones":0,"num_object_copies":15501,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5167,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8861,"num_write_kb":448896,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":139,"reported_epoch":374,"state":"active+clean","last_fresh":"2023-09-23T21:07:52.470426+0000","last_change":"2023-09-23T21:06:49.427675+0000","last_active":"2023-09-23T21:07:52.470426+0000","last_peered":"2023-09-23T21:07:52.470426+0000","last_clean":"2023-09-23T21:07:52.470426+0000","last_became_active":"2023-09-23T21:06:49.426780+0000","last_became_peered":"2023-09-23T21:06:49.426780+0000","last_unstale":"2023-09-23T21:07:52.470426+0000","last_undegraded":"2023-09-23T21:07:52.470426+0000","last_fullsized":"2023-09-23T21:07:52.470426+0000","mapping_epoch":359,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":360,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:06:15.874264+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"79'161","reported_seq":609,"reported_epoch":374,"state":"active+clean","last_fresh":"2023-09-23T21:07:52.467239+0000","last_change":"2023-09-23T20:57:20.738417+0000","last_active":"2023-09-23T21:07:52.467239+0000","last_peered":"2023-09-23T21:07:52.467239+0000","last_clean":"2023-09-23T21:07:52.467239+0000","last_became_active":"2023-09-23T20:57:08.236842+0000","last_became_peered":"2023-09-23T20:57:08.236842+0000","last_unstale":"2023-09-23T21:07:52.467239+0000","last_undegraded":"2023-09-23T21:07:52.467239+0000","last_fullsized":"2023-09-23T21:07:52.467239+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":0,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:20.738273+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:20.738273+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:18:28.327990+0000","scrub_duration":0.0088336339999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":375,"reported_epoch":374,"state":"active+clean","last_fresh":"2023-09-23T21:07:52.480224+0000","last_change":"2023-09-23T21:05:04.310148+0000","last_active":"2023-09-23T21:07:52.480224+0000","last_peered":"2023-09-23T21:07:52.480224+0000","last_clean":"2023-09-23T21:07:52.480224+0000","last_became_active":"2023-09-23T21:05:04.309869+0000","last_became_peered":"2023-09-23T21:05:04.309869+0000","last_unstale":"2023-09-23T21:07:52.480224+0000","last_undegraded":"2023-09-23T21:07:52.480224+0000","last_fullsized":"2023-09-23T21:07:52.480224+0000","mapping_epoch":332,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:12:55.522835+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":319,"reported_epoch":374,"state":"active+clean","last_fresh":"2023-09-23T21:07:52.456571+0000","last_change":"2023-09-23T20:55:35.978140+0000","last_active":"2023-09-23T21:07:52.456571+0000","last_peered":"2023-09-23T21:07:52.456571+0000","last_clean":"2023-09-23T21:07:52.456571+0000","last_became_active":"2023-09-23T20:55:35.977831+0000","last_became_peered":"2023-09-23T20:55:35.977831+0000","last_unstale":"2023-09-23T21:07:52.456571+0000","last_undegraded":"2023-09-23T21:07:52.456571+0000","last_fullsized":"2023-09-23T21:07:52.456571+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:25:47.678023+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"9.b","version":"375'8812","reported_seq":8850,"reported_epoch":375,"state":"active+clean","last_fresh":"2023-09-23T21:08:02.309406+0000","last_change":"2023-09-23T21:05:48.290142+0000","last_active":"2023-09-23T21:08:02.309406+0000","last_peered":"2023-09-23T21:08:02.309406+0000","last_clean":"2023-09-23T21:08:02.309406+0000","last_became_active":"2023-09-23T21:05:48.289861+0000","last_became_peered":"2023-09-23T21:05:48.289861+0000","last_unstale":"2023-09-23T21:08:02.309406+0000","last_undegraded":"2023-09-23T21:08:02.309406+0000","last_fullsized":"2023-09-23T21:08:02.309406+0000","mapping_epoch":346,"log_start":"351'400","ondisk_log_start":"351'400","created":346,"last_epoch_clean":347,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:05:47.261750+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:05:47.261750+0000","last_clean_scrub_stamp":"2023-09-23T21:05:47.261750+0000","objects_scrubbed":0,"log_size":8412,"log_dups_size":0,"ondisk_log_size":8412,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:36:39.064858+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":341442560,"num_objects":5210,"num_object_clones":0,"num_object_copies":15630,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5210,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8812,"num_write_kb":448704,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.3","version":"79'161","reported_seq":618,"reported_epoch":374,"state":"active+clean","last_fresh":"2023-09-23T21:07:58.448979+0000","last_change":"2023-09-23T21:05:57.832471+0000","last_active":"2023-09-23T21:07:58.448979+0000","last_peered":"2023-09-23T21:07:58.448979+0000","last_clean":"2023-09-23T21:07:58.448979+0000","last_became_active":"2023-09-23T21:05:55.318774+0000","last_became_peered":"2023-09-23T21:05:55.318774+0000","last_unstale":"2023-09-23T21:07:58.448979+0000","last_undegraded":"2023-09-23T21:07:58.448979+0000","last_fullsized":"2023-09-23T21:07:58.448979+0000","mapping_epoch":350,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":351,"parent":"0.0","parent_split_bits":3,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:05:57.832302+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:05:57.832302+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:15:55.402312+0000","scrub_duration":0.010206847999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":239,"reported_epoch":374,"state":"active+clean","last_fresh":"2023-09-23T21:07:53.444839+0000","last_change":"2023-09-23T21:05:55.316391+0000","last_active":"2023-09-23T21:07:53.444839+0000","last_peered":"2023-09-23T21:07:53.444839+0000","last_clean":"2023-09-23T21:07:53.444839+0000","last_became_active":"2023-09-23T21:05:55.316069+0000","last_became_peered":"2023-09-23T21:05:55.316069+0000","last_unstale":"2023-09-23T21:07:53.444839+0000","last_undegraded":"2023-09-23T21:07:53.444839+0000","last_fullsized":"2023-09-23T21:07:53.444839+0000","mapping_epoch":350,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":351,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:55:39.499467+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,0],"acting":[1,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":372,"reported_epoch":374,"state":"active+clean","last_fresh":"2023-09-23T21:07:52.468067+0000","last_change":"2023-09-23T20:57:44.979950+0000","last_active":"2023-09-23T21:07:52.468067+0000","last_peered":"2023-09-23T21:07:52.468067+0000","last_clean":"2023-09-23T21:07:52.468067+0000","last_became_active":"2023-09-23T20:57:44.979346+0000","last_became_peered":"2023-09-23T20:57:44.979346+0000","last_unstale":"2023-09-23T21:07:52.468067+0000","last_undegraded":"2023-09-23T21:07:52.468067+0000","last_fullsized":"2023-09-23T21:07:52.468067+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:10:24.372800+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":366,"reported_epoch":374,"state":"active+clean","last_fresh":"2023-09-23T21:07:52.467793+0000","last_change":"2023-09-23T21:05:55.320840+0000","last_active":"2023-09-23T21:07:52.467793+0000","last_peered":"2023-09-23T21:07:52.467793+0000","last_clean":"2023-09-23T21:07:52.467793+0000","last_became_active":"2023-09-23T21:05:55.320280+0000","last_became_peered":"2023-09-23T21:05:55.320280+0000","last_unstale":"2023-09-23T21:07:52.467793+0000","last_undegraded":"2023-09-23T21:07:52.467793+0000","last_fullsized":"2023-09-23T21:07:52.467793+0000","mapping_epoch":350,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":351,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:45:22.368801+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.e","version":"375'9127","reported_seq":9171,"reported_epoch":375,"state":"active+clean","last_fresh":"2023-09-23T21:08:02.350779+0000","last_change":"2023-09-23T21:05:55.326609+0000","last_active":"2023-09-23T21:08:02.350779+0000","last_peered":"2023-09-23T21:08:02.350779+0000","last_clean":"2023-09-23T21:08:02.350779+0000","last_became_active":"2023-09-23T21:05:55.326268+0000","last_became_peered":"2023-09-23T21:05:55.326268+0000","last_unstale":"2023-09-23T21:08:02.350779+0000","last_undegraded":"2023-09-23T21:08:02.350779+0000","last_fullsized":"2023-09-23T21:08:02.350779+0000","mapping_epoch":350,"log_start":"352'2100","ondisk_log_start":"352'2100","created":346,"last_epoch_clean":351,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:05:47.261750+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:05:47.261750+0000","last_clean_scrub_stamp":"2023-09-23T21:05:47.261750+0000","objects_scrubbed":0,"log_size":7027,"log_dups_size":0,"ondisk_log_size":7027,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:20:12.304644+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":355401728,"num_objects":5423,"num_object_clones":0,"num_object_copies":16269,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5423,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9127,"num_write_kb":465600,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.6","version":"79'161","reported_seq":657,"reported_epoch":374,"state":"active+clean","last_fresh":"2023-09-23T21:07:52.456645+0000","last_change":"2023-09-23T20:57:26.907505+0000","last_active":"2023-09-23T21:07:52.456645+0000","last_peered":"2023-09-23T21:07:52.456645+0000","last_clean":"2023-09-23T21:07:52.456645+0000","last_became_active":"2023-09-23T20:57:08.240321+0000","last_became_peered":"2023-09-23T20:57:08.240321+0000","last_unstale":"2023-09-23T21:07:52.456645+0000","last_undegraded":"2023-09-23T21:07:52.456645+0000","last_fullsized":"2023-09-23T21:07:52.456645+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":3,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:26.907408+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:26.907408+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:34:17.392928+0000","scrub_duration":0.010337038,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":219,"reported_epoch":374,"state":"active+clean","last_fresh":"2023-09-23T21:07:52.462596+0000","last_change":"2023-09-23T21:06:25.394432+0000","last_active":"2023-09-23T21:07:52.462596+0000","last_peered":"2023-09-23T21:07:52.462596+0000","last_clean":"2023-09-23T21:07:52.462596+0000","last_became_active":"2023-09-23T21:06:25.394120+0000","last_became_peered":"2023-09-23T21:06:25.394120+0000","last_unstale":"2023-09-23T21:07:52.462596+0000","last_undegraded":"2023-09-23T21:07:52.462596+0000","last_fullsized":"2023-09-23T21:07:52.462596+0000","mapping_epoch":354,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":355,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:07:30.306153+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":362,"reported_epoch":374,"state":"active+clean","last_fresh":"2023-09-23T21:07:58.449037+0000","last_change":"2023-09-23T21:05:55.319513+0000","last_active":"2023-09-23T21:07:58.449037+0000","last_peered":"2023-09-23T21:07:58.449037+0000","last_clean":"2023-09-23T21:07:58.449037+0000","last_became_active":"2023-09-23T21:05:55.319178+0000","last_became_peered":"2023-09-23T21:05:55.319178+0000","last_unstale":"2023-09-23T21:07:58.449037+0000","last_undegraded":"2023-09-23T21:07:58.449037+0000","last_fullsized":"2023-09-23T21:07:58.449037+0000","mapping_epoch":350,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":351,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:47:32.783934+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":357,"reported_epoch":374,"state":"active+clean","last_fresh":"2023-09-23T21:07:52.468146+0000","last_change":"2023-09-23T21:05:04.303493+0000","last_active":"2023-09-23T21:07:52.468146+0000","last_peered":"2023-09-23T21:07:52.468146+0000","last_clean":"2023-09-23T21:07:52.468146+0000","last_became_active":"2023-09-23T21:05:04.303148+0000","last_became_peered":"2023-09-23T21:05:04.303148+0000","last_unstale":"2023-09-23T21:07:52.468146+0000","last_undegraded":"2023-09-23T21:07:52.468146+0000","last_fullsized":"2023-09-23T21:07:52.468146+0000","mapping_epoch":332,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:48:06.836151+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.d","version":"375'8961","reported_seq":8997,"reported_epoch":375,"state":"active+clean","last_fresh":"2023-09-23T21:08:03.422861+0000","last_change":"2023-09-23T21:05:55.318195+0000","last_active":"2023-09-23T21:08:03.422861+0000","last_peered":"2023-09-23T21:08:03.422861+0000","last_clean":"2023-09-23T21:08:03.422861+0000","last_became_active":"2023-09-23T21:05:55.316441+0000","last_became_peered":"2023-09-23T21:05:55.316441+0000","last_unstale":"2023-09-23T21:08:03.422861+0000","last_undegraded":"2023-09-23T21:08:03.422861+0000","last_fullsized":"2023-09-23T21:08:03.422861+0000","mapping_epoch":350,"log_start":"352'2400","ondisk_log_start":"352'2400","created":346,"last_epoch_clean":351,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:05:47.261750+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:05:47.261750+0000","last_clean_scrub_stamp":"2023-09-23T21:05:47.261750+0000","objects_scrubbed":0,"log_size":6561,"log_dups_size":0,"ondisk_log_size":6561,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:31:06.880466+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":346685464,"num_objects":5291,"num_object_clones":0,"num_object_copies":15873,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5291,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":8961,"num_write_kb":456001,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,5],"acting":[1,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"79'161","reported_seq":638,"reported_epoch":374,"state":"active+clean","last_fresh":"2023-09-23T21:07:52.470491+0000","last_change":"2023-09-23T20:57:30.029479+0000","last_active":"2023-09-23T21:07:52.470491+0000","last_peered":"2023-09-23T21:07:52.470491+0000","last_clean":"2023-09-23T21:07:52.470491+0000","last_became_active":"2023-09-23T20:57:08.249909+0000","last_became_peered":"2023-09-23T20:57:08.249909+0000","last_unstale":"2023-09-23T21:07:52.470491+0000","last_undegraded":"2023-09-23T21:07:52.470491+0000","last_fullsized":"2023-09-23T21:07:52.470491+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":3,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:30.029396+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:30.029396+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:16:46.083145+0000","scrub_duration":0.012914682,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":342,"reported_epoch":374,"state":"active+clean","last_fresh":"2023-09-23T21:07:52.484742+0000","last_change":"2023-09-23T20:51:10.836910+0000","last_active":"2023-09-23T21:07:52.484742+0000","last_peered":"2023-09-23T21:07:52.484742+0000","last_clean":"2023-09-23T21:07:52.484742+0000","last_became_active":"2023-09-23T20:51:10.836557+0000","last_became_peered":"2023-09-23T20:51:10.836557+0000","last_unstale":"2023-09-23T21:07:52.484742+0000","last_undegraded":"2023-09-23T21:07:52.484742+0000","last_fullsized":"2023-09-23T21:07:52.484742+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:51:54.019969+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":340,"reported_epoch":375,"state":"active+clean","last_fresh":"2023-09-23T21:07:58.550764+0000","last_change":"2023-09-23T21:05:55.318675+0000","last_active":"2023-09-23T21:07:58.550764+0000","last_peered":"2023-09-23T21:07:58.550764+0000","last_clean":"2023-09-23T21:07:58.550764+0000","last_became_active":"2023-09-23T21:05:55.318363+0000","last_became_peered":"2023-09-23T21:05:55.318363+0000","last_unstale":"2023-09-23T21:07:58.550764+0000","last_undegraded":"2023-09-23T21:07:58.550764+0000","last_fullsized":"2023-09-23T21:07:58.550764+0000","mapping_epoch":350,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":351,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:19:50.065285+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":168,"reported_epoch":374,"state":"active+clean","last_fresh":"2023-09-23T21:07:52.478077+0000","last_change":"2023-09-23T20:57:44.982509+0000","last_active":"2023-09-23T21:07:52.478077+0000","last_peered":"2023-09-23T21:07:52.478077+0000","last_clean":"2023-09-23T21:07:52.478077+0000","last_became_active":"2023-09-23T20:57:44.982175+0000","last_became_peered":"2023-09-23T20:57:44.982175+0000","last_unstale":"2023-09-23T21:07:52.478077+0000","last_undegraded":"2023-09-23T21:07:52.478077+0000","last_fullsized":"2023-09-23T21:07:52.478077+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:37:49.215488+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.c","version":"375'9001","reported_seq":9038,"reported_epoch":375,"state":"active+clean","last_fresh":"2023-09-23T21:08:03.957605+0000","last_change":"2023-09-23T21:05:48.560410+0000","last_active":"2023-09-23T21:08:03.957605+0000","last_peered":"2023-09-23T21:08:03.957605+0000","last_clean":"2023-09-23T21:08:03.957605+0000","last_became_active":"2023-09-23T21:05:48.560135+0000","last_became_peered":"2023-09-23T21:05:48.560135+0000","last_unstale":"2023-09-23T21:08:03.957605+0000","last_undegraded":"2023-09-23T21:08:03.957605+0000","last_fullsized":"2023-09-23T21:08:03.957605+0000","mapping_epoch":346,"log_start":"352'2700","ondisk_log_start":"352'2700","created":346,"last_epoch_clean":347,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:05:47.261750+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:05:47.261750+0000","last_clean_scrub_stamp":"2023-09-23T21:05:47.261750+0000","objects_scrubbed":0,"log_size":6301,"log_dups_size":0,"ondisk_log_size":6301,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:16:44.290638+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":345047040,"num_objects":5265,"num_object_clones":0,"num_object_copies":15795,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5265,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9001,"num_write_kb":456512,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.4","version":"79'161","reported_seq":691,"reported_epoch":374,"state":"active+clean","last_fresh":"2023-09-23T21:07:52.483427+0000","last_change":"2023-09-23T21:05:07.298174+0000","last_active":"2023-09-23T21:07:52.483427+0000","last_peered":"2023-09-23T21:07:52.483427+0000","last_clean":"2023-09-23T21:07:52.483427+0000","last_became_active":"2023-09-23T21:05:07.297862+0000","last_became_peered":"2023-09-23T21:05:07.297862+0000","last_unstale":"2023-09-23T21:07:52.483427+0000","last_undegraded":"2023-09-23T21:07:52.483427+0000","last_fullsized":"2023-09-23T21:07:52.483427+0000","mapping_epoch":335,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":336,"parent":"0.0","parent_split_bits":3,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:20.012764+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:20.012764+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:24:55.897324+0000","scrub_duration":0.0045251919999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":324,"reported_epoch":374,"state":"active+clean","last_fresh":"2023-09-23T21:07:52.456718+0000","last_change":"2023-09-23T20:55:35.949541+0000","last_active":"2023-09-23T21:07:52.456718+0000","last_peered":"2023-09-23T21:07:52.456718+0000","last_clean":"2023-09-23T21:07:52.456718+0000","last_became_active":"2023-09-23T20:55:35.944635+0000","last_became_peered":"2023-09-23T20:55:35.944635+0000","last_unstale":"2023-09-23T21:07:52.456718+0000","last_undegraded":"2023-09-23T21:07:52.456718+0000","last_fullsized":"2023-09-23T21:07:52.456718+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:30:36.033585+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"70'1","reported_seq":382,"reported_epoch":374,"state":"active+clean","last_fresh":"2023-09-23T21:07:52.467889+0000","last_change":"2023-09-23T21:05:55.320603+0000","last_active":"2023-09-23T21:07:52.467889+0000","last_peered":"2023-09-23T21:07:52.467889+0000","last_clean":"2023-09-23T21:07:52.467889+0000","last_became_active":"2023-09-23T21:05:55.319956+0000","last_became_peered":"2023-09-23T21:05:55.319956+0000","last_unstale":"2023-09-23T21:07:52.467889+0000","last_undegraded":"2023-09-23T21:07:52.467889+0000","last_fullsized":"2023-09-23T21:07:52.467889+0000","mapping_epoch":350,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":351,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:41:03.995212+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":299,"reported_epoch":374,"state":"active+clean","last_fresh":"2023-09-23T21:07:52.466947+0000","last_change":"2023-09-23T21:05:04.305174+0000","last_active":"2023-09-23T21:07:52.466947+0000","last_peered":"2023-09-23T21:07:52.466947+0000","last_clean":"2023-09-23T21:07:52.466947+0000","last_became_active":"2023-09-23T21:05:04.304900+0000","last_became_peered":"2023-09-23T21:05:04.304900+0000","last_unstale":"2023-09-23T21:07:52.466947+0000","last_undegraded":"2023-09-23T21:07:52.466947+0000","last_fullsized":"2023-09-23T21:07:52.466947+0000","mapping_epoch":332,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:13:23.024245+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":341,"reported_epoch":374,"state":"active+clean","last_fresh":"2023-09-23T21:07:52.468086+0000","last_change":"2023-09-23T20:54:13.929196+0000","last_active":"2023-09-23T21:07:52.468086+0000","last_peered":"2023-09-23T21:07:52.468086+0000","last_clean":"2023-09-23T21:07:52.468086+0000","last_became_active":"2023-09-23T20:54:13.928831+0000","last_became_peered":"2023-09-23T20:54:13.928831+0000","last_unstale":"2023-09-23T21:07:52.468086+0000","last_undegraded":"2023-09-23T21:07:52.468086+0000","last_fullsized":"2023-09-23T21:07:52.468086+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:28:44.316851+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":361,"reported_epoch":374,"state":"active+clean","last_fresh":"2023-09-23T21:07:52.471434+0000","last_change":"2023-09-23T21:05:55.313495+0000","last_active":"2023-09-23T21:07:52.471434+0000","last_peered":"2023-09-23T21:07:52.471434+0000","last_clean":"2023-09-23T21:07:52.471434+0000","last_became_active":"2023-09-23T21:05:55.313065+0000","last_became_peered":"2023-09-23T21:05:55.313065+0000","last_unstale":"2023-09-23T21:07:52.471434+0000","last_undegraded":"2023-09-23T21:07:52.471434+0000","last_fullsized":"2023-09-23T21:07:52.471434+0000","mapping_epoch":350,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":351,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:05:22.675197+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":65,"reported_epoch":374,"state":"active+clean","last_fresh":"2023-09-23T21:07:52.467459+0000","last_change":"2023-09-23T21:06:49.429520+0000","last_active":"2023-09-23T21:07:52.467459+0000","last_peered":"2023-09-23T21:07:52.467459+0000","last_clean":"2023-09-23T21:07:52.467459+0000","last_became_active":"2023-09-23T21:06:49.429195+0000","last_became_peered":"2023-09-23T21:06:49.429195+0000","last_unstale":"2023-09-23T21:07:52.467459+0000","last_undegraded":"2023-09-23T21:07:52.467459+0000","last_fullsized":"2023-09-23T21:07:52.467459+0000","mapping_epoch":359,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":360,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:34:39.724075+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,1],"acting":[4,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":269,"reported_epoch":374,"state":"active+clean","last_fresh":"2023-09-23T21:07:52.467253+0000","last_change":"2023-09-23T21:05:55.323591+0000","last_active":"2023-09-23T21:07:52.467253+0000","last_peered":"2023-09-23T21:07:52.467253+0000","last_clean":"2023-09-23T21:07:52.467253+0000","last_became_active":"2023-09-23T21:05:55.323220+0000","last_became_peered":"2023-09-23T21:05:55.323220+0000","last_unstale":"2023-09-23T21:07:52.467253+0000","last_undegraded":"2023-09-23T21:07:52.467253+0000","last_fullsized":"2023-09-23T21:07:52.467253+0000","mapping_epoch":350,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":351,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:03:09.849488+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":136,"reported_epoch":374,"state":"active+clean","last_fresh":"2023-09-23T21:07:52.473761+0000","last_change":"2023-09-23T21:06:49.424707+0000","last_active":"2023-09-23T21:07:52.473761+0000","last_peered":"2023-09-23T21:07:52.473761+0000","last_clean":"2023-09-23T21:07:52.473761+0000","last_became_active":"2023-09-23T21:06:49.424418+0000","last_became_peered":"2023-09-23T21:06:49.424418+0000","last_unstale":"2023-09-23T21:07:52.473761+0000","last_undegraded":"2023-09-23T21:07:52.473761+0000","last_fullsized":"2023-09-23T21:07:52.473761+0000","mapping_epoch":359,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":360,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:28:50.350150+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":252,"reported_epoch":374,"state":"active+clean","last_fresh":"2023-09-23T21:07:52.468455+0000","last_change":"2023-09-23T20:54:13.935307+0000","last_active":"2023-09-23T21:07:52.468455+0000","last_peered":"2023-09-23T21:07:52.468455+0000","last_clean":"2023-09-23T21:07:52.468455+0000","last_became_active":"2023-09-23T20:54:13.935000+0000","last_became_peered":"2023-09-23T20:54:13.935000+0000","last_unstale":"2023-09-23T21:07:52.468455+0000","last_undegraded":"2023-09-23T21:07:52.468455+0000","last_fullsized":"2023-09-23T21:07:52.468455+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:47:40.974082+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":366,"reported_epoch":374,"state":"active+clean","last_fresh":"2023-09-23T21:07:52.480540+0000","last_change":"2023-09-23T20:55:35.954870+0000","last_active":"2023-09-23T21:07:52.480540+0000","last_peered":"2023-09-23T21:07:52.480540+0000","last_clean":"2023-09-23T21:07:52.480540+0000","last_became_active":"2023-09-23T20:55:35.954568+0000","last_became_peered":"2023-09-23T20:55:35.954568+0000","last_unstale":"2023-09-23T21:07:52.480540+0000","last_undegraded":"2023-09-23T21:07:52.480540+0000","last_fullsized":"2023-09-23T21:07:52.480540+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:06:22.423566+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":364,"reported_epoch":374,"state":"active+clean","last_fresh":"2023-09-23T21:07:52.478285+0000","last_change":"2023-09-23T20:51:10.838334+0000","last_active":"2023-09-23T21:07:52.478285+0000","last_peered":"2023-09-23T21:07:52.478285+0000","last_clean":"2023-09-23T21:07:52.478285+0000","last_became_active":"2023-09-23T20:51:10.838026+0000","last_became_peered":"2023-09-23T20:51:10.838026+0000","last_unstale":"2023-09-23T21:07:52.478285+0000","last_undegraded":"2023-09-23T21:07:52.478285+0000","last_fullsized":"2023-09-23T21:07:52.478285+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:04:37.009457+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":399,"reported_epoch":374,"state":"active+clean","last_fresh":"2023-09-23T21:07:52.468451+0000","last_change":"2023-09-23T21:05:04.325268+0000","last_active":"2023-09-23T21:07:52.468451+0000","last_peered":"2023-09-23T21:07:52.468451+0000","last_clean":"2023-09-23T21:07:52.468451+0000","last_became_active":"2023-09-23T21:05:04.303430+0000","last_became_peered":"2023-09-23T21:05:04.303430+0000","last_unstale":"2023-09-23T21:07:52.468451+0000","last_undegraded":"2023-09-23T21:07:52.468451+0000","last_fullsized":"2023-09-23T21:07:52.468451+0000","mapping_epoch":332,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:17:44.879586+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":2216,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":269,"reported_epoch":374,"state":"active+clean","last_fresh":"2023-09-23T21:07:52.466827+0000","last_change":"2023-09-23T20:55:35.951572+0000","last_active":"2023-09-23T21:07:52.466827+0000","last_peered":"2023-09-23T21:07:52.466827+0000","last_clean":"2023-09-23T21:07:52.466827+0000","last_became_active":"2023-09-23T20:55:35.951294+0000","last_became_peered":"2023-09-23T20:55:35.951294+0000","last_unstale":"2023-09-23T21:07:52.466827+0000","last_undegraded":"2023-09-23T21:07:52.466827+0000","last_fullsized":"2023-09-23T21:07:52.466827+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:36:32.447210+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":39,"num_bytes_recovered":7154,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5834,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":21,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3381,"ondisk_log_size":3381,"up":63,"acting":63,"num_store_stats":8},{"poolid":9,"num_pg":16,"stat_sum":{"num_bytes":5486673944,"num_objects":83721,"num_object_clones":0,"num_object_copies":251163,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":83721,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":142827,"num_write_kb":7249473,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":16481595392,"data_stored":16481583176,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":114727,"ondisk_log_size":114727,"up":48,"acting":48,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201611,"num_pgs":47,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3461676,"kb_used_data":3065188,"kb_used_omap":0,"kb_used_meta":396480,"kb_avail":90283476,"statfs":{"total":95995035648,"available":92450279424,"internally_reserved":0,"allocated":3138752512,"data_stored":3137660647,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":405995520},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:07:59 2023","interfaces":[{"interface":"back","average":{"1min":1.968,"5min":1.4239999999999999,"15min":1.696},"min":{"1min":0.23000000000000001,"5min":0.22700000000000001,"15min":0.21099999999999999},"max":{"1min":4.9969999999999999,"5min":8.4990000000000006,"15min":43.994999999999997},"last":0.40799999999999997},{"interface":"front","average":{"1min":2.2749999999999999,"5min":1.6299999999999999,"15min":1.883},"min":{"1min":0.26800000000000002,"5min":0.253,"15min":0.218},"max":{"1min":6.3529999999999998,"5min":8.5150000000000006,"15min":44.206000000000003},"last":0.32300000000000001}]},{"osd":1,"last update":"Sat Sep 23 21:07:59 2023","interfaces":[{"interface":"back","average":{"1min":2.8029999999999999,"5min":2.3199999999999998,"15min":2.2400000000000002},"min":{"1min":0.26700000000000002,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":7.8010000000000002,"5min":7.8399999999999999,"15min":7.8399999999999999},"last":0.34200000000000003},{"interface":"front","average":{"1min":1.9330000000000001,"5min":2.0609999999999999,"15min":2.0819999999999999},"min":{"1min":0.26300000000000001,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":7.577,"5min":8.3859999999999992,"15min":8.3859999999999992},"last":7.577}]},{"osd":2,"last update":"Sat Sep 23 21:07:14 2023","interfaces":[{"interface":"back","average":{"1min":2.9329999999999998,"5min":1.802,"15min":1.8460000000000001},"min":{"1min":0.438,"5min":0.27200000000000002,"15min":0.24399999999999999},"max":{"1min":14.478999999999999,"5min":14.478999999999999,"15min":14.478999999999999},"last":3.0979999999999999},{"interface":"front","average":{"1min":3.0430000000000001,"5min":1.7010000000000001,"15min":1.8540000000000001},"min":{"1min":0.27400000000000002,"5min":0.27400000000000002,"15min":0.26500000000000001},"max":{"1min":14.135,"5min":14.135,"15min":45.25},"last":0.69499999999999995}]},{"osd":3,"last update":"Sat Sep 23 21:07:49 2023","interfaces":[{"interface":"back","average":{"1min":3.173,"5min":2.1040000000000001,"15min":2.6110000000000002},"min":{"1min":0.30099999999999999,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":9.1449999999999996,"5min":12.359999999999999,"15min":36.408000000000001},"last":0.45600000000000002},{"interface":"front","average":{"1min":3.6000000000000001,"5min":2.1379999999999999,"15min":2.4870000000000001},"min":{"1min":0.40999999999999998,"5min":0.30299999999999999,"15min":0.26000000000000001},"max":{"1min":12.226000000000001,"5min":13.199,"15min":35.593000000000004},"last":1.127}]},{"osd":4,"last update":"Sat Sep 23 21:07:43 2023","interfaces":[{"interface":"back","average":{"1min":1.5589999999999999,"5min":0.96899999999999997,"15min":0.78300000000000003},"min":{"1min":0.28100000000000003,"5min":0.248,"15min":0.23699999999999999},"max":{"1min":4.4139999999999997,"5min":5.2640000000000002,"15min":5.8920000000000003},"last":0.44700000000000001},{"interface":"front","average":{"1min":1.706,"5min":1.0509999999999999,"15min":0.84499999999999997},"min":{"1min":0.311,"5min":0.28399999999999997,"15min":0.26300000000000001},"max":{"1min":7.1420000000000003,"5min":7.1420000000000003,"15min":7.1420000000000003},"last":0.377}]},{"osd":5,"last update":"Sat Sep 23 21:07:43 2023","interfaces":[{"interface":"back","average":{"1min":1.758,"5min":0.996,"15min":0.83499999999999996},"min":{"1min":0.33500000000000002,"5min":0.311,"15min":0.28499999999999998},"max":{"1min":4.8929999999999998,"5min":4.8929999999999998,"15min":7.0389999999999997},"last":0.49299999999999999},{"interface":"front","average":{"1min":1.5409999999999999,"5min":1.1000000000000001,"15min":0.81799999999999995},"min":{"1min":0.308,"5min":0.29899999999999999,"15min":0.27800000000000002},"max":{"1min":5.5090000000000003,"5min":5.8529999999999998,"15min":5.8529999999999998},"last":0.63200000000000001}]},{"osd":6,"last update":"Sat Sep 23 21:07:59 2023","interfaces":[{"interface":"back","average":{"1min":1.556,"5min":1.0469999999999999,"15min":0.84999999999999998},"min":{"1min":0.39400000000000002,"5min":0.32200000000000001,"15min":0.28899999999999998},"max":{"1min":6.5430000000000001,"5min":6.5430000000000001,"15min":6.5430000000000001},"last":0.48299999999999998},{"interface":"front","average":{"1min":1.637,"5min":1.103,"15min":0.91900000000000004},"min":{"1min":0.38100000000000001,"5min":0.35699999999999998,"15min":0.32300000000000001},"max":{"1min":6.5170000000000003,"5min":6.5170000000000003,"15min":6.5170000000000003},"last":0.51200000000000001}]}]},{"osd":6,"up_from":164,"seq":704374636695,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1245080,"kb_used_data":991952,"kb_used_omap":10,"kb_used_meta":253109,"kb_avail":92500072,"statfs":{"total":95995035648,"available":94720073728,"internally_reserved":0,"allocated":1015758848,"data_stored":1014687028,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":10328,"internal_metadata":259184552},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:07:01 2023","interfaces":[{"interface":"back","average":{"1min":2.1110000000000002,"5min":2.0110000000000001,"15min":1.575},"min":{"1min":0.25,"5min":0.224,"15min":0.20999999999999999},"max":{"1min":8.2750000000000004,"5min":14.151999999999999,"15min":14.151999999999999},"last":0.28599999999999998},{"interface":"front","average":{"1min":1.7330000000000001,"5min":1.7350000000000001,"15min":1.5920000000000001},"min":{"1min":0.224,"5min":0.224,"15min":0.20999999999999999},"max":{"1min":9.2249999999999996,"5min":10.962,"15min":11.023},"last":0.36599999999999999}]},{"osd":1,"last update":"Sat Sep 23 21:07:56 2023","interfaces":[{"interface":"back","average":{"1min":2.6930000000000001,"5min":1.819,"15min":1.673},"min":{"1min":0.29099999999999998,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":7.5439999999999996,"5min":12.904999999999999,"15min":12.904999999999999},"last":0.83799999999999997},{"interface":"front","average":{"1min":1.27,"5min":1.663,"15min":1.7290000000000001},"min":{"1min":0.30299999999999999,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":4.6280000000000001,"5min":6.7569999999999997,"15min":6.7569999999999997},"last":0.35099999999999998}]},{"osd":2,"last update":"Sat Sep 23 21:07:01 2023","interfaces":[{"interface":"back","average":{"1min":1.76,"5min":1.9359999999999999,"15min":1.9419999999999999},"min":{"1min":0.36499999999999999,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":5.6799999999999997,"5min":14.029999999999999,"15min":14.029999999999999},"last":0.66300000000000003},{"interface":"front","average":{"1min":1.4390000000000001,"5min":2.0739999999999998,"15min":1.794},"min":{"1min":0.32600000000000001,"5min":0.26500000000000001,"15min":0.247},"max":{"1min":12.957000000000001,"5min":12.957000000000001,"15min":20.195},"last":0.86299999999999999}]},{"osd":3,"last update":"Sat Sep 23 21:07:01 2023","interfaces":[{"interface":"back","average":{"1min":2.7429999999999999,"5min":2.7669999999999999,"15min":1.95},"min":{"1min":0.32300000000000001,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":12.535,"5min":43.085999999999999,"15min":43.085999999999999},"last":4.0830000000000002},{"interface":"front","average":{"1min":2.8759999999999999,"5min":2.492,"15min":1.9950000000000001},"min":{"1min":0.307,"5min":0.30499999999999999,"15min":0.28599999999999998},"max":{"1min":12.513,"5min":42.405000000000001,"15min":42.405000000000001},"last":4.7590000000000003}]},{"osd":4,"last update":"Sat Sep 23 21:07:01 2023","interfaces":[{"interface":"back","average":{"1min":0.84099999999999997,"5min":0.94499999999999995,"15min":0.88100000000000001},"min":{"1min":0.28499999999999998,"5min":0.251,"15min":0.23000000000000001},"max":{"1min":3.0249999999999999,"5min":5.3920000000000003,"15min":9.0079999999999991},"last":0.76400000000000001},{"interface":"front","average":{"1min":1.198,"5min":1.036,"15min":0.69299999999999995},"min":{"1min":0.28799999999999998,"5min":0.26600000000000001,"15min":0.25800000000000001},"max":{"1min":5.5060000000000002,"5min":6.4050000000000002,"15min":6.4050000000000002},"last":0.47899999999999998}]},{"osd":5,"last update":"Sat Sep 23 21:07:01 2023","interfaces":[{"interface":"back","average":{"1min":0.92500000000000004,"5min":1.1220000000000001,"15min":0.80100000000000005},"min":{"1min":0.30099999999999999,"5min":0.27800000000000002,"15min":0.27300000000000002},"max":{"1min":2.6829999999999998,"5min":5.9000000000000004,"15min":5.9000000000000004},"last":0.80300000000000005},{"interface":"front","average":{"1min":0.96499999999999997,"5min":1.234,"15min":0.92300000000000004},"min":{"1min":0.308,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":5.5549999999999997,"5min":10.657,"15min":10.657},"last":0.55300000000000005}]},{"osd":7,"last update":"Sat Sep 23 21:07:01 2023","interfaces":[{"interface":"back","average":{"1min":1.1659999999999999,"5min":1.256,"15min":0.97199999999999998},"min":{"1min":0.29499999999999998,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":5.5890000000000004,"5min":8.9260000000000002,"15min":8.9260000000000002},"last":0.52400000000000002},{"interface":"front","average":{"1min":1.242,"5min":0.96599999999999997,"15min":0.755},"min":{"1min":0.34200000000000003,"5min":0.28299999999999997,"15min":0.28100000000000003},"max":{"1min":5.6230000000000002,"5min":5.6230000000000002,"15min":8.282},"last":0.54000000000000004}]}]},{"osd":1,"up_from":350,"seq":1503238553627,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3203520,"kb_used_data":3007928,"kb_used_omap":10,"kb_used_meta":195573,"kb_avail":90541632,"statfs":{"total":95995035648,"available":92714631168,"internally_reserved":0,"allocated":3080118272,"data_stored":3078436292,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":10498,"internal_metadata":200267518},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,1,0,4],"upper_bound":16},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:07:57 2023","interfaces":[{"interface":"back","average":{"1min":3.4860000000000002,"5min":2.222,"15min":2.0110000000000001},"min":{"1min":0.29299999999999998,"5min":0.21199999999999999,"15min":0.21199999999999999},"max":{"1min":15.728,"5min":15.728,"15min":15.728},"last":0.50900000000000001},{"interface":"front","average":{"1min":3.407,"5min":2.7290000000000001,"15min":2.6160000000000001},"min":{"1min":0.20999999999999999,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":14.468999999999999,"5min":14.468999999999999,"15min":14.468999999999999},"last":0.249}]},{"osd":2,"last update":"Sat Sep 23 21:07:57 2023","interfaces":[{"interface":"back","average":{"1min":3.2509999999999999,"5min":2.5219999999999998,"15min":2.3999999999999999},"min":{"1min":0.24399999999999999,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":10.769,"5min":10.769,"15min":10.769},"last":0.27200000000000002},{"interface":"front","average":{"1min":3.0249999999999999,"5min":2.2120000000000002,"15min":2.0760000000000001},"min":{"1min":0.214,"5min":0.214,"15min":0.214},"max":{"1min":15.683999999999999,"5min":15.683999999999999,"15min":15.683999999999999},"last":0.57399999999999995}]},{"osd":3,"last update":"Sat Sep 23 21:07:57 2023","interfaces":[{"interface":"back","average":{"1min":3.226,"5min":2.843,"15min":2.7789999999999999},"min":{"1min":0.438,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":15.135999999999999,"5min":15.135999999999999,"15min":15.135999999999999},"last":1.6779999999999999},{"interface":"front","average":{"1min":4.1529999999999996,"5min":3.2810000000000001,"15min":3.1360000000000001},"min":{"1min":0.433,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":14.794,"5min":14.794,"15min":14.794},"last":1.885}]},{"osd":4,"last update":"Sat Sep 23 21:07:57 2023","interfaces":[{"interface":"back","average":{"1min":2.6440000000000001,"5min":1.929,"15min":1.8100000000000001},"min":{"1min":0.32700000000000001,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":10.391,"5min":10.391,"15min":10.391},"last":0.34999999999999998},{"interface":"front","average":{"1min":1.893,"5min":1.6599999999999999,"15min":1.621},"min":{"1min":0.39400000000000002,"5min":0.39400000000000002,"15min":0.39400000000000002},"max":{"1min":6.0270000000000001,"5min":6.0270000000000001,"15min":6.0270000000000001},"last":0.54100000000000004}]},{"osd":5,"last update":"Sat Sep 23 21:07:57 2023","interfaces":[{"interface":"back","average":{"1min":2.2949999999999999,"5min":2.5430000000000001,"15min":2.5840000000000001},"min":{"1min":0.378,"5min":0.378,"15min":0.378},"max":{"1min":5.923,"5min":8.0899999999999999,"15min":8.0899999999999999},"last":0.39800000000000002},{"interface":"front","average":{"1min":3.3479999999999999,"5min":2.7959999999999998,"15min":2.7040000000000002},"min":{"1min":0.35699999999999998,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":10.339,"5min":10.339,"15min":10.339},"last":0.58299999999999996}]},{"osd":6,"last update":"Sat Sep 23 21:07:57 2023","interfaces":[{"interface":"back","average":{"1min":2.464,"5min":2.3639999999999999,"15min":2.3479999999999999},"min":{"1min":0.377,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":5.968,"5min":8.1880000000000006,"15min":8.1880000000000006},"last":0.43099999999999999},{"interface":"front","average":{"1min":3.2360000000000002,"5min":2.585,"15min":2.4769999999999999},"min":{"1min":0.40100000000000002,"5min":0.40100000000000002,"15min":0.40100000000000002},"max":{"1min":10.433999999999999,"5min":10.433999999999999,"15min":10.433999999999999},"last":1.4319999999999999}]},{"osd":7,"last update":"Sat Sep 23 21:07:57 2023","interfaces":[{"interface":"back","average":{"1min":1.9510000000000001,"5min":2.0230000000000001,"15min":2.0350000000000001},"min":{"1min":0.35299999999999998,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":5.992,"5min":6.5570000000000004,"15min":6.5570000000000004},"last":1.266},{"interface":"front","average":{"1min":2.2909999999999999,"5min":2.5590000000000002,"15min":2.6040000000000001},"min":{"1min":0.41099999999999998,"5min":0.41099999999999998,"15min":0.41099999999999998},"max":{"1min":5.8899999999999997,"5min":8.1440000000000001,"15min":8.1440000000000001},"last":1.411}]}]},{"osd":0,"up_from":83,"seq":356482285792,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1810376,"kb_used_data":1352896,"kb_used_omap":28,"kb_used_meta":457443,"kb_avail":91934776,"statfs":{"total":95995035648,"available":94141210624,"internally_reserved":0,"allocated":1385365504,"data_stored":1383733566,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":28848,"internal_metadata":468422480},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 21:07:58 2023","interfaces":[{"interface":"back","average":{"1min":3.4209999999999998,"5min":1.7370000000000001,"15min":1.456},"min":{"1min":0.20200000000000001,"5min":0.192,"15min":0.192},"max":{"1min":12.994,"5min":12.994,"15min":12.994},"last":0.20200000000000001},{"interface":"front","average":{"1min":2.448,"5min":1.508,"15min":1.3520000000000001},"min":{"1min":0.247,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":14.646000000000001,"5min":14.646000000000001,"15min":14.646000000000001},"last":3.1000000000000001}]},{"osd":2,"last update":"Sat Sep 23 21:07:11 2023","interfaces":[{"interface":"back","average":{"1min":2.1869999999999998,"5min":2.266,"15min":2.7589999999999999},"min":{"1min":0.23799999999999999,"5min":0.183,"15min":0.183},"max":{"1min":13.032,"5min":13.032,"15min":21.053000000000001},"last":0.44400000000000001},{"interface":"front","average":{"1min":1.526,"5min":2.3290000000000002,"15min":2.8999999999999999},"min":{"1min":0.29699999999999999,"5min":0.23499999999999999,"15min":0.21099999999999999},"max":{"1min":8.6479999999999997,"5min":12.423,"15min":21.747},"last":3.1619999999999999}]},{"osd":3,"last update":"Sat Sep 23 21:07:33 2023","interfaces":[{"interface":"back","average":{"1min":3.9220000000000002,"5min":2.5499999999999998,"15min":2.8719999999999999},"min":{"1min":0.27400000000000002,"5min":0.252,"15min":0.252},"max":{"1min":18.669,"5min":41.079000000000001,"15min":41.079000000000001},"last":3.1349999999999998},{"interface":"front","average":{"1min":3.1499999999999999,"5min":2.282,"15min":2.9390000000000001},"min":{"1min":0.40300000000000002,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":19.91,"5min":19.91,"15min":21.167000000000002},"last":0.47599999999999998}]},{"osd":4,"last update":"Sat Sep 23 21:07:11 2023","interfaces":[{"interface":"back","average":{"1min":1.1879999999999999,"5min":1.79,"15min":2.1040000000000001},"min":{"1min":0.32400000000000001,"5min":0.28699999999999998,"15min":0.28599999999999998},"max":{"1min":3.1030000000000002,"5min":12.394,"15min":21.594999999999999},"last":3.194},{"interface":"front","average":{"1min":2.8690000000000002,"5min":1.792,"15min":1.8720000000000001},"min":{"1min":0.311,"5min":0.311,"15min":0.28100000000000003},"max":{"1min":18.516999999999999,"5min":18.516999999999999,"15min":18.724},"last":2.7160000000000002}]},{"osd":5,"last update":"Sat Sep 23 21:07:11 2023","interfaces":[{"interface":"back","average":{"1min":2.2770000000000001,"5min":1.913,"15min":1.859},"min":{"1min":0.29299999999999998,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":12.946999999999999,"5min":12.946999999999999,"15min":12.946999999999999},"last":0.5},{"interface":"front","average":{"1min":1.2869999999999999,"5min":1.877,"15min":2.1200000000000001},"min":{"1min":0.42299999999999999,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":3.839,"5min":12.353,"15min":20.512},"last":3.2240000000000002}]},{"osd":6,"last update":"Sat Sep 23 21:07:05 2023","interfaces":[{"interface":"back","average":{"1min":1.841,"5min":1.6870000000000001,"15min":1.8879999999999999},"min":{"1min":0.44700000000000001,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":8.2300000000000004,"5min":11,"15min":13.933},"last":2.7810000000000001},{"interface":"front","average":{"1min":2.157,"5min":1.7030000000000001,"15min":1.9550000000000001},"min":{"1min":0.40300000000000002,"5min":0.40300000000000002,"15min":0.377},"max":{"1min":8.1950000000000003,"5min":11.066000000000001,"15min":13.962},"last":2.7530000000000001}]},{"osd":7,"last update":"Sat Sep 23 21:07:11 2023","interfaces":[{"interface":"back","average":{"1min":2.004,"5min":1.9530000000000001,"15min":1.9079999999999999},"min":{"1min":0.38900000000000001,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":12.907999999999999,"5min":12.907999999999999,"15min":12.907999999999999},"last":0.52700000000000002},{"interface":"front","average":{"1min":3.0209999999999999,"5min":2.3439999999999999,"15min":2.0880000000000001},"min":{"1min":0.45700000000000002,"5min":0.40999999999999998,"15min":0.377},"max":{"1min":18.634,"5min":40.984999999999999,"15min":40.984999999999999},"last":2.8159999999999998}]}]},{"osd":2,"up_from":93,"seq":399431958739,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1606848,"kb_used_data":1030776,"kb_used_omap":81,"kb_used_meta":575982,"kb_avail":92138304,"statfs":{"total":95995035648,"available":94349623296,"internally_reserved":0,"allocated":1055514624,"data_stored":1053887878,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":83750,"internal_metadata":589805786},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:08:01 2023","interfaces":[{"interface":"back","average":{"1min":2.2029999999999998,"5min":1.6639999999999999,"15min":2.1859999999999999},"min":{"1min":0.23799999999999999,"5min":0.22600000000000001,"15min":0.17999999999999999},"max":{"1min":11.124000000000001,"5min":11.124000000000001,"15min":29.498000000000001},"last":2.2799999999999998},{"interface":"front","average":{"1min":1.6040000000000001,"5min":1.726,"15min":2.484},"min":{"1min":0.26600000000000001,"5min":0.26600000000000001,"15min":0.193},"max":{"1min":5.8070000000000004,"5min":9.2880000000000003,"15min":17.251000000000001},"last":5.8070000000000004}]},{"osd":1,"last update":"Sat Sep 23 21:07:57 2023","interfaces":[{"interface":"back","average":{"1min":4.2770000000000001,"5min":3.2349999999999999,"15min":3.0609999999999999},"min":{"1min":0.309,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":12.593,"5min":34.396999999999998,"15min":34.396999999999998},"last":3.448},{"interface":"front","average":{"1min":2.3439999999999999,"5min":3.407,"15min":3.5840000000000001},"min":{"1min":0.217,"5min":0.217,"15min":0.217},"max":{"1min":7.0659999999999998,"5min":34.462000000000003,"15min":34.462000000000003},"last":1.6020000000000001}]},{"osd":3,"last update":"Sat Sep 23 21:07:47 2023","interfaces":[{"interface":"back","average":{"1min":2.2709999999999999,"5min":2.7679999999999998,"15min":3.3079999999999998},"min":{"1min":0.248,"5min":0.23899999999999999,"15min":0.23300000000000001},"max":{"1min":12.083,"5min":35.549999999999997,"15min":41.057000000000002},"last":6.0110000000000001},{"interface":"front","average":{"1min":2.8679999999999999,"5min":3.2690000000000001,"15min":3.7930000000000001},"min":{"1min":0.36199999999999999,"5min":0.253,"15min":0.22800000000000001},"max":{"1min":12.446,"5min":36.951999999999998,"15min":41.012},"last":3.4649999999999999}]},{"osd":4,"last update":"Sat Sep 23 21:08:01 2023","interfaces":[{"interface":"back","average":{"1min":1.569,"5min":1.4950000000000001,"15min":1.6479999999999999},"min":{"1min":0.33100000000000002,"5min":0.29199999999999998,"15min":0.26400000000000001},"max":{"1min":5.4960000000000004,"5min":8.7569999999999997,"15min":10.045},"last":5.4960000000000004},{"interface":"front","average":{"1min":2.8530000000000002,"5min":1.9970000000000001,"15min":2.101},"min":{"1min":0.40500000000000003,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":12.49,"5min":12.49,"15min":14.039999999999999},"last":3.3759999999999999}]},{"osd":5,"last update":"Sat Sep 23 21:08:01 2023","interfaces":[{"interface":"back","average":{"1min":1.927,"5min":1.6890000000000001,"15min":1.823},"min":{"1min":0.33800000000000002,"5min":0.316,"15min":0.29299999999999998},"max":{"1min":5.4429999999999996,"5min":8.907,"15min":10.069000000000001},"last":5.4429999999999996},{"interface":"front","average":{"1min":2.7530000000000001,"5min":2.004,"15min":2.089},"min":{"1min":0.36599999999999999,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":12.57,"5min":12.57,"15min":13.98},"last":3.3410000000000002}]},{"osd":6,"last update":"Sat Sep 23 21:08:01 2023","interfaces":[{"interface":"back","average":{"1min":2.7919999999999998,"5min":2.2490000000000001,"15min":2.1019999999999999},"min":{"1min":0.432,"5min":0.35699999999999998,"15min":0.312},"max":{"1min":12.518000000000001,"5min":12.518000000000001,"15min":14.244999999999999},"last":3.4009999999999998},{"interface":"front","average":{"1min":2.9900000000000002,"5min":2.407,"15min":2.0169999999999999},"min":{"1min":0.438,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":12.544,"5min":12.544,"15min":14.068},"last":3.4180000000000001}]},{"osd":7,"last update":"Sat Sep 23 21:08:01 2023","interfaces":[{"interface":"back","average":{"1min":1.694,"5min":1.6899999999999999,"15min":1.5269999999999999},"min":{"1min":0.40999999999999998,"5min":0.34100000000000003,"15min":0.315},"max":{"1min":7.1050000000000004,"5min":8.7599999999999998,"15min":13.427},"last":1.1220000000000001},{"interface":"front","average":{"1min":2.218,"5min":1.8979999999999999,"15min":1.899},"min":{"1min":0.42199999999999999,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":10.92,"5min":10.92,"15min":10.92},"last":5.5300000000000002}]}]},{"osd":3,"up_from":98,"seq":420906795213,"num_pgs":48,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3316340,"kb_used_data":2679084,"kb_used_omap":7,"kb_used_meta":637240,"kb_avail":90428812,"statfs":{"total":95995035648,"available":92599103488,"internally_reserved":0,"allocated":2743382016,"data_stored":2741765858,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7576,"internal_metadata":652534376},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,5,3,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:07:49 2023","interfaces":[{"interface":"back","average":{"1min":3.0049999999999999,"5min":2.331,"15min":3.0579999999999998},"min":{"1min":0.19900000000000001,"5min":0.189,"15min":0.188},"max":{"1min":10.409000000000001,"5min":11.717000000000001,"15min":27.962},"last":4.2039999999999997},{"interface":"front","average":{"1min":2.399,"5min":2.3479999999999999,"15min":2.9350000000000001},"min":{"1min":0.245,"5min":0.20599999999999999,"15min":0.19800000000000001},"max":{"1min":7.8920000000000003,"5min":11.483000000000001,"15min":28.058},"last":5.3010000000000002}]},{"osd":1,"last update":"Sat Sep 23 21:07:59 2023","interfaces":[{"interface":"back","average":{"1min":5.6779999999999999,"5min":2.8500000000000001,"15min":2.3780000000000001},"min":{"1min":0.29499999999999998,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":15.897,"5min":15.897,"15min":15.897},"last":5.5910000000000002},{"interface":"front","average":{"1min":4.5999999999999996,"5min":2.6960000000000002,"15min":2.379},"min":{"1min":0.30399999999999999,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":13.811999999999999,"5min":13.811999999999999,"15min":13.811999999999999},"last":5.6740000000000004}]},{"osd":2,"last update":"Sat Sep 23 21:07:49 2023","interfaces":[{"interface":"back","average":{"1min":4.6349999999999998,"5min":2.419,"15min":3.0640000000000001},"min":{"1min":0.307,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":14.287000000000001,"5min":14.287000000000001,"15min":28.251999999999999},"last":0.84199999999999997},{"interface":"front","average":{"1min":3.8639999999999999,"5min":2.6120000000000001,"15min":3.101},"min":{"1min":0.29899999999999999,"5min":0.29399999999999998,"15min":0.22},"max":{"1min":14.186,"5min":18.321000000000002,"15min":25.305},"last":5.2690000000000001}]},{"osd":4,"last update":"Sat Sep 23 21:07:49 2023","interfaces":[{"interface":"back","average":{"1min":2.8239999999999998,"5min":1.8160000000000001,"15min":2.0470000000000002},"min":{"1min":0.379,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":8.7379999999999995,"5min":11.505000000000001,"15min":18.239000000000001},"last":0.88300000000000001},{"interface":"front","average":{"1min":2.2050000000000001,"5min":1.974,"15min":2.105},"min":{"1min":0.35299999999999998,"5min":0.30299999999999999,"15min":0.29499999999999998},"max":{"1min":6.8929999999999998,"5min":10.914,"15min":22.853000000000002},"last":3.2160000000000002}]},{"osd":5,"last update":"Sat Sep 23 21:07:49 2023","interfaces":[{"interface":"back","average":{"1min":3.1339999999999999,"5min":2.0960000000000001,"15min":2.1509999999999998},"min":{"1min":0.32700000000000001,"5min":0.314,"15min":0.314},"max":{"1min":12.815,"5min":12.815,"15min":18.324999999999999},"last":3.8330000000000002},{"interface":"front","average":{"1min":4.1870000000000003,"5min":2.4980000000000002,"15min":2.3660000000000001},"min":{"1min":0.34200000000000003,"5min":0.33700000000000002,"15min":0.33100000000000002},"max":{"1min":14.029,"5min":14.029,"15min":14.029},"last":1.5049999999999999}]},{"osd":6,"last update":"Sat Sep 23 21:07:03 2023","interfaces":[{"interface":"back","average":{"1min":2.0030000000000001,"5min":1.948,"15min":1.6160000000000001},"min":{"1min":0.36599999999999999,"5min":0.33900000000000002,"15min":0.33500000000000002},"max":{"1min":5.4320000000000004,"5min":10.994999999999999,"15min":14.833},"last":3.1840000000000002},{"interface":"front","average":{"1min":2.4380000000000002,"5min":2.2440000000000002,"15min":2.0609999999999999},"min":{"1min":0.55600000000000005,"5min":0.373,"15min":0.373},"max":{"1min":7.633,"5min":10.664,"15min":13.576000000000001},"last":4.056}]},{"osd":7,"last update":"Sat Sep 23 21:07:49 2023","interfaces":[{"interface":"back","average":{"1min":3.7410000000000001,"5min":2.27,"15min":2.3690000000000002},"min":{"1min":0.90100000000000002,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":14.81,"5min":14.81,"15min":18.289999999999999},"last":0.90900000000000003},{"interface":"front","average":{"1min":2.7429999999999999,"5min":2.3199999999999998,"15min":2.3860000000000001},"min":{"1min":0.53200000000000003,"5min":0.40300000000000002,"15min":0.379},"max":{"1min":7.8390000000000004,"5min":12.624000000000001,"15min":13.603999999999999},"last":4.1040000000000001}]}]},{"osd":4,"up_from":31,"seq":133143986520,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1294824,"kb_used_data":1029408,"kb_used_omap":0,"kb_used_meta":265344,"kb_avail":92450328,"statfs":{"total":95995035648,"available":94669135872,"internally_reserved":0,"allocated":1054113792,"data_stored":1053026252,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":271712256},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:07:05 2023","interfaces":[{"interface":"back","average":{"1min":1.6379999999999999,"5min":1.3049999999999999,"15min":1.4330000000000001},"min":{"1min":0.24199999999999999,"5min":0.22700000000000001,"15min":0.214},"max":{"1min":11.353999999999999,"5min":11.353999999999999,"15min":11.784000000000001},"last":11.537000000000001},{"interface":"front","average":{"1min":2.004,"5min":1.7490000000000001,"15min":1.6679999999999999},"min":{"1min":0.28799999999999998,"5min":0.254,"15min":0.254},"max":{"1min":11.315,"5min":11.315,"15min":11.766},"last":5.4269999999999996}]},{"osd":1,"last update":"Sat Sep 23 21:08:01 2023","interfaces":[{"interface":"back","average":{"1min":2.512,"5min":3.5579999999999998,"15min":3.7320000000000002},"min":{"1min":0.26600000000000001,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":9.2449999999999992,"5min":37.960000000000001,"15min":37.960000000000001},"last":0.54300000000000004},{"interface":"front","average":{"1min":2.379,"5min":3.3719999999999999,"15min":3.5379999999999998},"min":{"1min":0.315,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":8.3870000000000005,"5min":37.933,"15min":37.933},"last":2.8119999999999998}]},{"osd":2,"last update":"Sat Sep 23 21:07:05 2023","interfaces":[{"interface":"back","average":{"1min":1.593,"5min":1.7490000000000001,"15min":1.7450000000000001},"min":{"1min":0.33500000000000002,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":7.1900000000000004,"5min":10.906000000000001,"15min":15.288},"last":2.8479999999999999},{"interface":"front","average":{"1min":1.786,"5min":1.599,"15min":1.77},"min":{"1min":0.35199999999999998,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":5.4749999999999996,"5min":8.9329999999999998,"15min":15.262},"last":11.682}]},{"osd":3,"last update":"Sat Sep 23 21:07:37 2023","interfaces":[{"interface":"back","average":{"1min":3.274,"5min":2.4239999999999999,"15min":2.5670000000000002},"min":{"1min":0.36599999999999999,"5min":0.29799999999999999,"15min":0.28899999999999998},"max":{"1min":15.01,"5min":15.01,"15min":31.562999999999999},"last":0.38100000000000001},{"interface":"front","average":{"1min":3.391,"5min":2.0779999999999998,"15min":2.4489999999999998},"min":{"1min":0.38700000000000001,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":15.004,"5min":15.004,"15min":31.574000000000002},"last":11.507999999999999}]},{"osd":5,"last update":"Sat Sep 23 21:07:54 2023","interfaces":[{"interface":"back","average":{"1min":1.387,"5min":1.05,"15min":0.72699999999999998},"min":{"1min":0.24299999999999999,"5min":0.24299999999999999,"15min":0.22700000000000001},"max":{"1min":8.2479999999999993,"5min":8.2479999999999993,"15min":8.2479999999999993},"last":11.467000000000001},{"interface":"front","average":{"1min":0.78100000000000003,"5min":1.1779999999999999,"15min":0.90100000000000002},"min":{"1min":0.255,"5min":0.255,"15min":0.251},"max":{"1min":2.9620000000000002,"5min":7.8700000000000001,"15min":7.8700000000000001},"last":9.8979999999999997}]},{"osd":6,"last update":"Sat Sep 23 21:08:01 2023","interfaces":[{"interface":"back","average":{"1min":1.728,"5min":1.4430000000000001,"15min":0.96899999999999997},"min":{"1min":0.28399999999999997,"5min":0.26800000000000002,"15min":0.26500000000000001},"max":{"1min":11.103999999999999,"5min":11.103999999999999,"15min":11.103999999999999},"last":3.7949999999999999},{"interface":"front","average":{"1min":1.149,"5min":0.92600000000000005,"15min":0.78100000000000003},"min":{"1min":0.34000000000000002,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":8.0749999999999993,"5min":8.0749999999999993,"15min":8.0749999999999993},"last":2.8719999999999999}]},{"osd":7,"last update":"Sat Sep 23 21:07:37 2023","interfaces":[{"interface":"back","average":{"1min":1.4330000000000001,"5min":1.1319999999999999,"15min":0.871},"min":{"1min":0.312,"5min":0.312,"15min":0.29299999999999998},"max":{"1min":5.4219999999999997,"5min":14.460000000000001,"15min":14.460000000000001},"last":3.2389999999999999},{"interface":"front","average":{"1min":0.93700000000000006,"5min":1.4450000000000001,"15min":0.98599999999999999},"min":{"1min":0.33100000000000002,"5min":0.32200000000000001,"15min":0.28199999999999997},"max":{"1min":4.2830000000000004,"5min":11.170999999999999,"15min":11.170999999999999},"last":3.0819999999999999}]}]},{"osd":5,"up_from":39,"seq":167503724885,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3378208,"kb_used_data":3032536,"kb_used_omap":0,"kb_used_meta":345664,"kb_avail":90366944,"statfs":{"total":95995035648,"available":92535750656,"internally_reserved":0,"allocated":3105316864,"data_stored":3104233828,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":353959936},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:07:58 2023","interfaces":[{"interface":"back","average":{"1min":3.3620000000000001,"5min":2.0339999999999998,"15min":1.778},"min":{"1min":0.19700000000000001,"5min":0.19700000000000001,"15min":0.19700000000000001},"max":{"1min":25.222999999999999,"5min":25.222999999999999,"15min":25.222999999999999},"last":0.312},{"interface":"front","average":{"1min":2.7970000000000002,"5min":1.968,"15min":1.8879999999999999},"min":{"1min":0.26200000000000001,"5min":0.23499999999999999,"15min":0.223},"max":{"1min":25.457999999999998,"5min":25.457999999999998,"15min":25.457999999999998},"last":0.78800000000000003}]},{"osd":1,"last update":"Sat Sep 23 21:07:58 2023","interfaces":[{"interface":"back","average":{"1min":3.7749999999999999,"5min":2.1859999999999999,"15min":1.921},"min":{"1min":0.26800000000000002,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":25.239999999999998,"5min":25.239999999999998,"15min":25.239999999999998},"last":2.9060000000000001},{"interface":"front","average":{"1min":3.4580000000000002,"5min":1.9339999999999999,"15min":1.6799999999999999},"min":{"1min":0.27400000000000002,"5min":0.252,"15min":0.252},"max":{"1min":25.253,"5min":25.253,"15min":25.253},"last":2.891}]},{"osd":2,"last update":"Sat Sep 23 21:07:08 2023","interfaces":[{"interface":"back","average":{"1min":1.5309999999999999,"5min":1.6699999999999999,"15min":1.6419999999999999},"min":{"1min":0.25800000000000001,"5min":0.25800000000000001,"15min":0.23499999999999999},"max":{"1min":5.9009999999999998,"5min":12.634,"15min":12.651},"last":3.2749999999999999},{"interface":"front","average":{"1min":1.8759999999999999,"5min":1.8580000000000001,"15min":1.839},"min":{"1min":0.28100000000000003,"5min":0.28100000000000003,"15min":0.26100000000000001},"max":{"1min":4.1680000000000001,"5min":9.7650000000000006,"15min":12.300000000000001},"last":3.4550000000000001}]},{"osd":3,"last update":"Sat Sep 23 21:07:30 2023","interfaces":[{"interface":"back","average":{"1min":3.3740000000000001,"5min":2.2389999999999999,"15min":2.6419999999999999},"min":{"1min":0.29299999999999998,"5min":0.29299999999999998,"15min":0.28599999999999998},"max":{"1min":25.503,"5min":25.503,"15min":61.667999999999999},"last":1.897},{"interface":"front","average":{"1min":4.0129999999999999,"5min":2.4710000000000001,"15min":2.3919999999999999},"min":{"1min":0.42399999999999999,"5min":0.33500000000000002,"15min":0.28899999999999998},"max":{"1min":25.466000000000001,"5min":25.466000000000001,"15min":62.276000000000003},"last":2.5630000000000002}]},{"osd":4,"last update":"Sat Sep 23 21:07:44 2023","interfaces":[{"interface":"back","average":{"1min":1.3360000000000001,"5min":1.129,"15min":0.80200000000000005},"min":{"1min":0.28499999999999998,"5min":0.254,"15min":0.23400000000000001},"max":{"1min":5.7569999999999997,"5min":9.0950000000000006,"15min":9.0950000000000006},"last":0.65500000000000003},{"interface":"front","average":{"1min":1.8380000000000001,"5min":1.054,"15min":0.81799999999999995},"min":{"1min":0.34799999999999998,"5min":0.26900000000000002,"15min":0.23599999999999999},"max":{"1min":8.125,"5min":8.125,"15min":8.4350000000000005},"last":0.60399999999999998}]},{"osd":6,"last update":"Sat Sep 23 21:07:58 2023","interfaces":[{"interface":"back","average":{"1min":1.796,"5min":1.28,"15min":0.91900000000000004},"min":{"1min":0.36599999999999999,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":8.0909999999999993,"5min":8.0909999999999993,"15min":11.128},"last":0.57699999999999996},{"interface":"front","average":{"1min":1.7629999999999999,"5min":1.2150000000000001,"15min":0.96099999999999997},"min":{"1min":0.35999999999999999,"5min":0.34599999999999997,"15min":0.33200000000000002},"max":{"1min":6.351,"5min":6.681,"15min":11.112},"last":0.62}]},{"osd":7,"last update":"Sat Sep 23 21:07:30 2023","interfaces":[{"interface":"back","average":{"1min":1.4710000000000001,"5min":1.107,"15min":0.89300000000000002},"min":{"1min":0.35199999999999998,"5min":0.309,"15min":0.30199999999999999},"max":{"1min":5.6500000000000004,"5min":5.6500000000000004,"15min":8.5649999999999995},"last":3.2930000000000001},{"interface":"front","average":{"1min":0.89900000000000002,"5min":0.95299999999999996,"15min":0.80800000000000005},"min":{"1min":0.35999999999999999,"5min":0.32500000000000001,"15min":0.27700000000000002},"max":{"1min":3.4500000000000002,"5min":4.2009999999999996,"15min":4.6479999999999997},"last":3.427}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1259,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1376518144,"data_stored":1376518144,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":3068596224,"data_stored":3068592152,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1042350080,"data_stored":1042350080,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":2713911296,"data_stored":2713911296,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1044971520,"data_stored":1044971520,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":3099529216,"data_stored":3099525144,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1011154944,"data_stored":1011154944,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":3124563968,"data_stored":3124559896,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T21:08:06.403 INFO:tasks.cephadm.ceph_manager.ceph:recovered! 2023-09-23T21:08:06.403 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T21:08:06.404 INFO:tasks.thrashosds.thrasher:primary_affinity 2023-09-23T21:08:06.404 INFO:tasks.thrashosds.thrasher:Setting osd 1 primary_affinity to 0.972172 2023-09-23T21:08:06.404 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph osd primary-affinity 1 0.9721718104590561 2023-09-23T21:08:06.433 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:08:06.538 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:08:06 smithi026 ceph-mon[175151]: pgmap v983: 109 pgs: 109 active+clean; 5.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 800 op/s 2023-09-23T21:08:06.538 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:08:06 smithi026 ceph-mon[179688]: pgmap v983: 109 pgs: 109 active+clean; 5.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 800 op/s 2023-09-23T21:08:06.599 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:08:06.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:08:06 smithi070 ceph-mon[168282]: pgmap v983: 109 pgs: 109 active+clean; 5.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 800 op/s 2023-09-23T21:08:06.689 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T21:08:06.703 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:08:06.703 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:08:06.798 INFO:teuthology.orchestra.run.smithi026.stdout:{ 2023-09-23T21:08:06.798 INFO:teuthology.orchestra.run.smithi026.stdout: "size": 20, 2023-09-23T21:08:06.798 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 600, 2023-09-23T21:08:06.799 INFO:teuthology.orchestra.run.smithi026.stdout: "ops": [ 2023-09-23T21:08:06.799 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:06.799 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35469.0:199518 8.4 8:2640ea5b:::benchmark_data_smithi026_301269_object88761:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e335)", 2023-09-23T21:08:06.799 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T21:05:06.377277+0000", 2023-09-23T21:08:06.799 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 180.41609534299999, 2023-09-23T21:08:06.799 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.21749188999999999, 2023-09-23T21:08:06.799 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:08:06.799 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:08:06.799 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T21:08:06.799 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35469", 2023-09-23T21:08:06.799 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/4041424623", 2023-09-23T21:08:06.799 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 199518 2023-09-23T21:08:06.800 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:06.800 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:08:06.800 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:06.800 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:08:06.800 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:06.377277+0000", 2023-09-23T21:08:06.800 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:08:06.800 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:06.800 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:06.800 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:08:06.800 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:06.377277+0000", 2023-09-23T21:08:06.800 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:08:06.800 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:06.801 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:06.801 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:08:06.801 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:06.377276+0000", 2023-09-23T21:08:06.801 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T21:08:06.801 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:06.801 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:06.801 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:08:06.801 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:06.377283+0000", 2023-09-23T21:08:06.801 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.3119999999999996e-06 2023-09-23T21:08:06.801 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:06.801 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:06.801 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:08:06.801 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:06.377295+0000", 2023-09-23T21:08:06.802 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.2089999999999999e-05 2023-09-23T21:08:06.802 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:06.802 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:06.802 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:08:06.802 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:06.377304+0000", 2023-09-23T21:08:06.802 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.3319999999999992e-06 2023-09-23T21:08:06.802 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:06.802 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:06.802 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:08:06.802 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:06.589594+0000", 2023-09-23T21:08:06.802 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.21229039599999999 2023-09-23T21:08:06.802 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:06.803 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:06.803 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:08:06.803 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:06.589713+0000", 2023-09-23T21:08:06.803 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00011854 2023-09-23T21:08:06.803 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:06.803 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:06.803 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 4,5,6", 2023-09-23T21:08:06.803 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:06.590123+0000", 2023-09-23T21:08:06.803 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00041040299999999999 2023-09-23T21:08:06.803 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:06.803 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:06.803 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:08:06.804 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:06.594592+0000", 2023-09-23T21:08:06.804 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0044695100000000003 2023-09-23T21:08:06.804 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:06.804 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:06.804 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:08:06.804 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:06.594624+0000", 2023-09-23T21:08:06.804 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.1934000000000003e-05 2023-09-23T21:08:06.804 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:06.804 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:06.804 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:08:06.804 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:06.594664+0000", 2023-09-23T21:08:06.804 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.9663999999999998e-05 2023-09-23T21:08:06.804 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:06.805 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:06.805 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T21:08:06.805 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:06.594675+0000", 2023-09-23T21:08:06.805 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.094e-05 2023-09-23T21:08:06.805 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:06.805 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:06.805 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T21:08:06.805 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:06.594704+0000", 2023-09-23T21:08:06.805 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.9074e-05 2023-09-23T21:08:06.805 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:06.805 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:06.805 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T21:08:06.805 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:06.594769+0000", 2023-09-23T21:08:06.806 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.5173999999999996e-05 2023-09-23T21:08:06.806 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:08:06.806 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:08:06.806 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:08:06.806 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:06.806 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:06.806 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35469.0:199522 8.4 8:222836e4:::benchmark_data_smithi026_301269_object88765:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e335)", 2023-09-23T21:08:06.806 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T21:05:06.382666+0000", 2023-09-23T21:08:06.806 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 180.41070644800001, 2023-09-23T21:08:06.806 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.21769185399999999, 2023-09-23T21:08:06.806 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:08:06.806 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:08:06.807 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T21:08:06.807 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35469", 2023-09-23T21:08:06.807 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/4041424623", 2023-09-23T21:08:06.807 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 199522 2023-09-23T21:08:06.807 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:06.807 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:08:06.807 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:06.807 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:08:06.807 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:06.382666+0000", 2023-09-23T21:08:06.807 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:08:06.807 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:06.807 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:06.808 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:08:06.808 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:06.382666+0000", 2023-09-23T21:08:06.808 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:08:06.808 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:06.808 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:06.808 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:08:06.808 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:06.382664+0000", 2023-09-23T21:08:06.808 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999981 2023-09-23T21:08:06.808 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:06.808 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:06.808 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:08:06.808 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:06.382672+0000", 2023-09-23T21:08:06.809 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.3660000000000003e-06 2023-09-23T21:08:06.809 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:06.809 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:06.809 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:08:06.809 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:06.382686+0000", 2023-09-23T21:08:06.809 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.4143e-05 2023-09-23T21:08:06.809 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:06.809 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:06.809 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:08:06.809 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:06.382692+0000", 2023-09-23T21:08:06.809 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.0809999999999999e-06 2023-09-23T21:08:06.809 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:06.809 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:06.810 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:08:06.810 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:06.592605+0000", 2023-09-23T21:08:06.810 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.209913192 2023-09-23T21:08:06.810 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:06.810 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:06.810 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:08:06.810 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:06.592753+0000", 2023-09-23T21:08:06.810 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00014783900000000001 2023-09-23T21:08:06.810 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:06.810 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:06.810 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 4,5,6", 2023-09-23T21:08:06.810 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:06.593073+0000", 2023-09-23T21:08:06.810 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00032028100000000001 2023-09-23T21:08:06.811 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:06.811 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:06.811 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T21:08:06.811 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:06.600089+0000", 2023-09-23T21:08:06.811 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0070155570000000004 2023-09-23T21:08:06.811 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:06.811 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:06.811 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:08:06.811 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:06.600133+0000", 2023-09-23T21:08:06.811 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.4079999999999998e-05 2023-09-23T21:08:06.811 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:06.811 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:06.811 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:08:06.812 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:06.600186+0000", 2023-09-23T21:08:06.812 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.2917000000000002e-05 2023-09-23T21:08:06.812 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:06.812 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:06.812 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:08:06.812 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:06.600229+0000", 2023-09-23T21:08:06.812 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.3477999999999999e-05 2023-09-23T21:08:06.812 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:06.812 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:06.812 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T21:08:06.812 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:06.600269+0000", 2023-09-23T21:08:06.812 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.9953000000000002e-05 2023-09-23T21:08:06.813 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:06.813 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:06.813 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T21:08:06.813 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:06.600358+0000", 2023-09-23T21:08:06.813 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.8745999999999994e-05 2023-09-23T21:08:06.813 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:08:06.813 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:08:06.813 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:08:06.813 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:06.813 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:06.813 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35469.0:199537 8.4 8:2222700d:::benchmark_data_smithi026_301269_object88780:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e335)", 2023-09-23T21:08:06.813 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T21:05:06.403867+0000", 2023-09-23T21:08:06.813 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 180.38950557800001, 2023-09-23T21:08:06.814 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.23103794599999999, 2023-09-23T21:08:06.814 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:08:06.814 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:08:06.814 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T21:08:06.814 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35469", 2023-09-23T21:08:06.814 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/4041424623", 2023-09-23T21:08:06.814 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 199537 2023-09-23T21:08:06.814 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:06.814 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:08:06.814 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:06.814 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:08:06.814 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:06.403867+0000", 2023-09-23T21:08:06.815 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:08:06.815 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:06.815 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:06.815 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:08:06.815 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:06.403867+0000", 2023-09-23T21:08:06.815 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:08:06.815 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:06.815 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:06.815 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:08:06.815 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:06.403865+0000", 2023-09-23T21:08:06.815 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999981 2023-09-23T21:08:06.815 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:06.815 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:06.816 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:08:06.816 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:06.403873+0000", 2023-09-23T21:08:06.816 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.9540000000000004e-06 2023-09-23T21:08:06.816 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:06.816 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:06.816 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:08:06.816 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:06.403887+0000", 2023-09-23T21:08:06.816 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.4036000000000001e-05 2023-09-23T21:08:06.816 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:06.816 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:06.817 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:08:06.817 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:06.403895+0000", 2023-09-23T21:08:06.817 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.6620000000000005e-06 2023-09-23T21:08:06.817 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:06.817 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:06.817 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:08:06.817 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:06.594799+0000", 2023-09-23T21:08:06.817 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.19090365200000001 2023-09-23T21:08:06.817 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:06.817 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:06.817 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:08:06.818 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:06.594903+0000", 2023-09-23T21:08:06.818 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000103929 2023-09-23T21:08:06.818 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:06.818 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:06.818 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 4,5,6", 2023-09-23T21:08:06.818 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:06.595226+0000", 2023-09-23T21:08:06.818 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00032308400000000001 2023-09-23T21:08:06.818 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:06.818 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:06.818 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T21:08:06.818 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:06.600147+0000", 2023-09-23T21:08:06.819 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0049209190000000002 2023-09-23T21:08:06.819 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:06.819 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:06.819 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:08:06.819 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:06.600399+0000", 2023-09-23T21:08:06.819 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00025212800000000001 2023-09-23T21:08:06.819 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:06.819 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:06.820 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:08:06.820 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:06.600446+0000", 2023-09-23T21:08:06.820 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.6928000000000002e-05 2023-09-23T21:08:06.820 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:06.820 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:06.820 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:08:06.820 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:06.634742+0000", 2023-09-23T21:08:06.820 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.03429658 2023-09-23T21:08:06.820 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:06.820 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:06.820 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T21:08:06.821 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:06.634788+0000", 2023-09-23T21:08:06.821 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.5302000000000001e-05 2023-09-23T21:08:06.821 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:06.821 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:06.821 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T21:08:06.821 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:06.634905+0000", 2023-09-23T21:08:06.821 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000117546 2023-09-23T21:08:06.821 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:08:06.821 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:08:06.821 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:08:06.822 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:06.822 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:06.822 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_repop(client.35469.0:199587 8.14 e335/307 8:2ee8cbc5:::benchmark_data_smithi026_301269_object88830:head v 335'10379, mlcod=334'10377)", 2023-09-23T21:08:06.822 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T21:05:06.459358+0000", 2023-09-23T21:08:06.822 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 180.33401502699999, 2023-09-23T21:08:06.822 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.13529712899999999, 2023-09-23T21:08:06.822 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:08:06.822 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:08:06.822 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:08:06.822 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:06.822 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:08:06.823 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:06.459358+0000", 2023-09-23T21:08:06.823 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:08:06.823 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:06.823 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:06.823 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:08:06.823 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:06.459358+0000", 2023-09-23T21:08:06.823 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:08:06.823 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:06.823 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:06.823 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:08:06.824 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:06.459357+0000", 2023-09-23T21:08:06.824 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.999999 2023-09-23T21:08:06.824 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:06.824 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:06.824 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:08:06.824 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:06.459368+0000", 2023-09-23T21:08:06.824 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1225e-05 2023-09-23T21:08:06.824 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:06.824 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:06.825 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:08:06.825 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:06.459397+0000", 2023-09-23T21:08:06.825 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.94e-05 2023-09-23T21:08:06.825 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:06.825 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:06.825 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:08:06.825 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:06.459406+0000", 2023-09-23T21:08:06.825 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.4470000000000006e-06 2023-09-23T21:08:06.825 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:06.825 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:06.825 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:08:06.825 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:06.589576+0000", 2023-09-23T21:08:06.826 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.13017012 2023-09-23T21:08:06.826 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:06.826 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:06.826 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:08:06.826 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:06.589591+0000", 2023-09-23T21:08:06.826 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.5177000000000001e-05 2023-09-23T21:08:06.826 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:06.826 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:06.826 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T21:08:06.826 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:06.594630+0000", 2023-09-23T21:08:06.826 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00503872 2023-09-23T21:08:06.826 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:06.827 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:06.827 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T21:08:06.827 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:06.594655+0000", 2023-09-23T21:08:06.827 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.4966000000000001e-05 2023-09-23T21:08:06.827 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:08:06.827 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:08:06.827 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:08:06.827 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:06.827 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:06.827 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35469.0:199619 8.4 8:2416ad93:::benchmark_data_smithi026_301269_object88862:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e335)", 2023-09-23T21:08:06.827 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T21:05:06.495932+0000", 2023-09-23T21:08:06.827 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 180.2974404, 2023-09-23T21:08:06.828 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.13928062599999999, 2023-09-23T21:08:06.828 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:08:06.828 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:08:06.828 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T21:08:06.828 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35469", 2023-09-23T21:08:06.828 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/4041424623", 2023-09-23T21:08:06.828 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 199619 2023-09-23T21:08:06.829 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:06.829 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:08:06.829 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:06.829 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:08:06.829 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:06.495932+0000", 2023-09-23T21:08:06.829 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:08:06.829 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:06.830 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:06.830 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:08:06.830 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:06.495932+0000", 2023-09-23T21:08:06.830 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:08:06.830 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:06.830 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:06.830 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:08:06.830 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:06.495931+0000", 2023-09-23T21:08:06.831 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T21:08:06.831 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:06.831 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:06.831 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:08:06.831 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:06.495937+0000", 2023-09-23T21:08:06.831 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.7339999999999996e-06 2023-09-23T21:08:06.831 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:06.832 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:06.832 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:08:06.832 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:06.495950+0000", 2023-09-23T21:08:06.832 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.3553999999999999e-05 2023-09-23T21:08:06.832 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:06.832 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:06.832 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:08:06.832 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:06.495956+0000", 2023-09-23T21:08:06.832 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.2099999999999998e-06 2023-09-23T21:08:06.832 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:06.832 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:06.833 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:08:06.833 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:06.597523+0000", 2023-09-23T21:08:06.833 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.101566618 2023-09-23T21:08:06.833 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:06.833 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:06.833 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:08:06.833 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:06.597650+0000", 2023-09-23T21:08:06.833 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00012691699999999999 2023-09-23T21:08:06.833 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:06.833 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:06.833 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 4,5,6", 2023-09-23T21:08:06.833 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:06.597985+0000", 2023-09-23T21:08:06.834 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00033532800000000003 2023-09-23T21:08:06.834 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:06.834 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:06.834 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T21:08:06.834 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:06.600406+0000", 2023-09-23T21:08:06.834 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0024210239999999999 2023-09-23T21:08:06.834 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:06.834 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:06.834 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:08:06.834 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:06.600468+0000", 2023-09-23T21:08:06.834 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.1872999999999995e-05 2023-09-23T21:08:06.834 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:06.834 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:06.835 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:08:06.835 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:06.634954+0000", 2023-09-23T21:08:06.835 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.034486079000000003 2023-09-23T21:08:06.835 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:06.835 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:06.835 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:08:06.835 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:06.635041+0000", 2023-09-23T21:08:06.835 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.6985999999999997e-05 2023-09-23T21:08:06.835 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:06.835 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:06.835 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T21:08:06.835 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:06.635100+0000", 2023-09-23T21:08:06.836 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.9082000000000001e-05 2023-09-23T21:08:06.836 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:06.836 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:06.836 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T21:08:06.836 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:06.635213+0000", 2023-09-23T21:08:06.836 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00011256000000000001 2023-09-23T21:08:06.836 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:08:06.836 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:08:06.836 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:08:06.836 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:06.836 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:06.836 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35469.0:212417 8.a 8:5385c5a4:::benchmark_data_smithi026_301269_object101660:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e339)", 2023-09-23T21:08:06.836 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T21:05:21.284981+0000", 2023-09-23T21:08:06.837 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 165.50839128000001, 2023-09-23T21:08:06.837 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0517269170000001, 2023-09-23T21:08:06.837 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:08:06.837 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:08:06.837 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T21:08:06.837 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35469", 2023-09-23T21:08:06.837 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/4041424623", 2023-09-23T21:08:06.837 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 212417 2023-09-23T21:08:06.837 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:06.837 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:08:06.837 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:06.837 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:08:06.838 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:21.284981+0000", 2023-09-23T21:08:06.838 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:08:06.838 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:06.838 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:06.838 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:08:06.838 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:21.284981+0000", 2023-09-23T21:08:06.838 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:08:06.838 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:06.838 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:06.838 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:08:06.838 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:21.284979+0000", 2023-09-23T21:08:06.838 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999976 2023-09-23T21:08:06.839 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:06.839 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:06.839 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:08:06.839 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:21.284988+0000", 2023-09-23T21:08:06.839 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.0289999999999994e-06 2023-09-23T21:08:06.839 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:06.839 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:06.839 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:08:06.839 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:21.285002+0000", 2023-09-23T21:08:06.839 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.3541e-05 2023-09-23T21:08:06.839 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:06.839 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:06.839 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:08:06.840 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:21.285012+0000", 2023-09-23T21:08:06.840 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.8940000000000005e-06 2023-09-23T21:08:06.840 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:06.840 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:06.840 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:08:06.840 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:21.285284+0000", 2023-09-23T21:08:06.840 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00027241899999999998 2023-09-23T21:08:06.840 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:06.840 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:06.840 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T21:08:06.840 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:21.285294+0000", 2023-09-23T21:08:06.840 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.6469999999999993e-06 2023-09-23T21:08:06.840 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:06.841 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:06.841 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:08:06.841 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:22.321489+0000", 2023-09-23T21:08:06.841 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.036195727 2023-09-23T21:08:06.841 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:06.841 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:06.841 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:08:06.841 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:22.324294+0000", 2023-09-23T21:08:06.841 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0028048740000000002 2023-09-23T21:08:06.841 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:06.841 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:06.841 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 5", 2023-09-23T21:08:06.841 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:22.324413+0000", 2023-09-23T21:08:06.842 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000118613 2023-09-23T21:08:06.842 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:06.842 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:06.842 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:08:06.842 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:22.336370+0000", 2023-09-23T21:08:06.842 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.011957198 2023-09-23T21:08:06.842 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:06.842 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:06.842 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T21:08:06.842 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:22.336559+0000", 2023-09-23T21:08:06.842 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000188896 2023-09-23T21:08:06.842 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:06.843 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:06.843 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T21:08:06.843 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:22.336609+0000", 2023-09-23T21:08:06.843 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.9707999999999999e-05 2023-09-23T21:08:06.843 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:06.843 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:06.843 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T21:08:06.843 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:22.336708+0000", 2023-09-23T21:08:06.843 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.9535000000000002e-05 2023-09-23T21:08:06.843 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:08:06.843 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:08:06.843 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:08:06.843 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:06.844 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:06.844 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35469.0:212423 8.a 8:5e64cfdd:::benchmark_data_smithi026_301269_object101666:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e339)", 2023-09-23T21:08:06.844 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T21:05:21.293572+0000", 2023-09-23T21:08:06.844 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 165.499800548, 2023-09-23T21:08:06.844 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0445543349999999, 2023-09-23T21:08:06.844 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:08:06.844 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:08:06.844 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T21:08:06.844 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35469", 2023-09-23T21:08:06.844 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/4041424623", 2023-09-23T21:08:06.844 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 212423 2023-09-23T21:08:06.844 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:06.844 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:08:06.845 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:06.845 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:08:06.845 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:21.293572+0000", 2023-09-23T21:08:06.845 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:08:06.845 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:06.845 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:06.845 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:08:06.845 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:21.293572+0000", 2023-09-23T21:08:06.845 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:08:06.845 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:06.845 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:06.845 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:08:06.845 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:21.293570+0000", 2023-09-23T21:08:06.846 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T21:08:06.846 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:06.846 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:06.846 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:08:06.846 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:21.293579+0000", 2023-09-23T21:08:06.846 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.3359999999999995e-06 2023-09-23T21:08:06.846 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:06.846 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:06.846 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:08:06.846 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:21.293596+0000", 2023-09-23T21:08:06.846 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.6935e-05 2023-09-23T21:08:06.846 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:06.847 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:06.847 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:08:06.847 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:21.293604+0000", 2023-09-23T21:08:06.847 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.8150000000000006e-06 2023-09-23T21:08:06.847 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:06.847 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:06.847 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:08:06.847 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:21.293642+0000", 2023-09-23T21:08:06.847 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.8673000000000001e-05 2023-09-23T21:08:06.847 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:06.847 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:06.847 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T21:08:06.847 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:21.293649+0000", 2023-09-23T21:08:06.848 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.9700000000000002e-06 2023-09-23T21:08:06.848 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:06.848 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:06.848 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:08:06.848 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:22.329308+0000", 2023-09-23T21:08:06.848 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0356588680000001 2023-09-23T21:08:06.848 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:06.848 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:06.848 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:08:06.848 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:22.329433+0000", 2023-09-23T21:08:06.848 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000125318 2023-09-23T21:08:06.848 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:06.849 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:06.849 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 5", 2023-09-23T21:08:06.849 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:22.329532+0000", 2023-09-23T21:08:06.849 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.8507999999999999e-05 2023-09-23T21:08:06.849 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:06.849 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:06.849 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T21:08:06.849 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:22.337337+0000", 2023-09-23T21:08:06.849 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0078052490000000002 2023-09-23T21:08:06.849 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:06.849 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:06.849 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:08:06.850 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:22.337982+0000", 2023-09-23T21:08:06.850 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00064439800000000002 2023-09-23T21:08:06.850 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:06.850 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:06.850 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T21:08:06.850 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:22.338031+0000", 2023-09-23T21:08:06.850 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.9329000000000002e-05 2023-09-23T21:08:06.850 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:06.850 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:06.850 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T21:08:06.850 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:22.338126+0000", 2023-09-23T21:08:06.850 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.5598000000000002e-05 2023-09-23T21:08:06.851 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:08:06.851 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:08:06.851 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:08:06.851 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:06.851 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:06.851 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35469.0:212443 8.a 8:564426b3:::benchmark_data_smithi026_301269_object101686:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e339)", 2023-09-23T21:08:06.851 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T21:05:21.320675+0000", 2023-09-23T21:08:06.851 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 165.47269753699999, 2023-09-23T21:08:06.851 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0176459879999999, 2023-09-23T21:08:06.852 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:08:06.852 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:08:06.852 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T21:08:06.852 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35469", 2023-09-23T21:08:06.852 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/4041424623", 2023-09-23T21:08:06.852 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 212443 2023-09-23T21:08:06.852 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:06.852 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:08:06.852 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:06.853 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:08:06.853 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:21.320675+0000", 2023-09-23T21:08:06.853 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:08:06.853 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:06.853 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:06.853 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:08:06.853 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:21.320675+0000", 2023-09-23T21:08:06.853 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:08:06.853 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:06.853 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:06.853 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:08:06.854 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:21.320674+0000", 2023-09-23T21:08:06.854 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T21:08:06.854 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:06.854 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:06.854 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:08:06.854 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:21.320680+0000", 2023-09-23T21:08:06.854 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.6289999999999998e-06 2023-09-23T21:08:06.855 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:06.855 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:06.855 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:08:06.855 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:21.320692+0000", 2023-09-23T21:08:06.855 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.2221999999999999e-05 2023-09-23T21:08:06.855 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:06.855 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:06.855 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:08:06.855 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:21.320700+0000", 2023-09-23T21:08:06.855 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.9109999999999993e-06 2023-09-23T21:08:06.855 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:06.855 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:06.856 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:08:06.856 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:21.320739+0000", 2023-09-23T21:08:06.856 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.9517000000000001e-05 2023-09-23T21:08:06.856 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:06.856 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:06.856 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T21:08:06.856 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:21.320749+0000", 2023-09-23T21:08:06.856 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.4080000000000004e-06 2023-09-23T21:08:06.856 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:06.856 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:06.856 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:08:06.856 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:22.332451+0000", 2023-09-23T21:08:06.857 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0117019330000001 2023-09-23T21:08:06.857 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:06.857 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:06.857 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:08:06.857 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:22.332573+0000", 2023-09-23T21:08:06.857 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000122569 2023-09-23T21:08:06.857 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:06.857 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:06.857 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 5", 2023-09-23T21:08:06.857 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:22.332666+0000", 2023-09-23T21:08:06.857 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.3067999999999997e-05 2023-09-23T21:08:06.857 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:06.858 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:06.858 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:08:06.858 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:22.338171+0000", 2023-09-23T21:08:06.858 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0055052909999999998 2023-09-23T21:08:06.858 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:06.858 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:06.858 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T21:08:06.858 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:22.338182+0000", 2023-09-23T21:08:06.858 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0074e-05 2023-09-23T21:08:06.858 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:06.858 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:06.858 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T21:08:06.858 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:22.338234+0000", 2023-09-23T21:08:06.859 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.2725000000000001e-05 2023-09-23T21:08:06.859 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:06.859 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:06.859 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T21:08:06.859 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:22.338321+0000", 2023-09-23T21:08:06.859 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.6908000000000002e-05 2023-09-23T21:08:06.859 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:08:06.859 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:08:06.859 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:08:06.859 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:06.859 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:06.859 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35856.0:121064 9.6 9:6d42fb72:::benchmark_data_smithi026_342732_object7788:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e369)", 2023-09-23T21:08:06.860 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T21:07:35.936640+0000", 2023-09-23T21:08:06.860 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 30.856732474000001, 2023-09-23T21:08:06.860 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.04896528, 2023-09-23T21:08:06.860 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:08:06.860 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:08:06.860 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T21:08:06.860 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35856", 2023-09-23T21:08:06.860 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/4180396258", 2023-09-23T21:08:06.860 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 121064 2023-09-23T21:08:06.860 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:06.860 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:08:06.860 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:06.860 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:08:06.861 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:07:35.936640+0000", 2023-09-23T21:08:06.861 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:08:06.861 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:06.861 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:06.861 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:08:06.861 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:07:35.936640+0000", 2023-09-23T21:08:06.861 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:08:06.861 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:06.861 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:06.861 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:08:06.861 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:07:35.936639+0000", 2023-09-23T21:08:06.861 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.999999 2023-09-23T21:08:06.862 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:06.862 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:06.862 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:08:06.862 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:07:35.936644+0000", 2023-09-23T21:08:06.862 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.0989999999999998e-06 2023-09-23T21:08:06.862 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:06.862 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:06.862 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:08:06.862 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:07:35.936659+0000", 2023-09-23T21:08:06.862 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.4338000000000001e-05 2023-09-23T21:08:06.862 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:06.862 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:06.862 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:08:06.863 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:07:35.936667+0000", 2023-09-23T21:08:06.863 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.7810000000000005e-06 2023-09-23T21:08:06.863 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:06.863 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:06.863 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:08:06.863 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:07:35.940223+0000", 2023-09-23T21:08:06.863 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0035569270000000001 2023-09-23T21:08:06.863 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:06.863 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:06.863 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:08:06.863 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:07:35.940354+0000", 2023-09-23T21:08:06.863 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000130834 2023-09-23T21:08:06.864 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:06.864 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:06.864 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 1,5", 2023-09-23T21:08:06.864 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:07:35.940469+0000", 2023-09-23T21:08:06.864 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00011451900000000001 2023-09-23T21:08:06.864 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:06.864 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:06.864 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T21:08:06.864 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:07:35.948511+0000", 2023-09-23T21:08:06.864 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0080424829999999996 2023-09-23T21:08:06.864 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:06.864 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:06.865 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:08:06.865 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:07:35.968114+0000", 2023-09-23T21:08:06.865 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.019603157999999999 2023-09-23T21:08:06.865 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:06.865 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:06.865 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:08:06.865 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:07:35.985535+0000", 2023-09-23T21:08:06.865 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.017420548000000001 2023-09-23T21:08:06.865 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:06.865 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:06.865 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T21:08:06.865 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:07:35.985578+0000", 2023-09-23T21:08:06.865 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.2919e-05 2023-09-23T21:08:06.866 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:06.866 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:06.866 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T21:08:06.866 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:07:35.985605+0000", 2023-09-23T21:08:06.866 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.7571000000000001e-05 2023-09-23T21:08:06.866 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:08:06.866 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:08:06.866 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:08:06.866 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:06.866 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:06.866 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35856.0:121094 9.6 9:6408a037:::benchmark_data_smithi026_342732_object7818:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e369)", 2023-09-23T21:08:06.866 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T21:07:36.008522+0000", 2023-09-23T21:08:06.867 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 30.784850232, 2023-09-23T21:08:06.867 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.050012753, 2023-09-23T21:08:06.867 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:08:06.867 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:08:06.867 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T21:08:06.867 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35856", 2023-09-23T21:08:06.867 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/4180396258", 2023-09-23T21:08:06.867 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 121094 2023-09-23T21:08:06.867 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:06.867 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:08:06.867 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:06.867 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:08:06.867 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:07:36.008522+0000", 2023-09-23T21:08:06.867 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:08:06.868 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:06.868 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:06.868 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:08:06.868 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:07:36.008522+0000", 2023-09-23T21:08:06.868 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:08:06.868 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:06.868 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:06.868 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:08:06.868 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:07:36.008521+0000", 2023-09-23T21:08:06.868 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T21:08:06.868 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:06.868 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:06.869 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:08:06.869 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:07:36.008528+0000", 2023-09-23T21:08:06.869 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.6100000000000002e-06 2023-09-23T21:08:06.869 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:06.869 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:06.869 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:08:06.869 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:07:36.008545+0000", 2023-09-23T21:08:06.869 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.7774e-05 2023-09-23T21:08:06.869 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:06.869 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:06.869 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:08:06.869 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:07:36.008555+0000", 2023-09-23T21:08:06.869 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.8560000000000002e-06 2023-09-23T21:08:06.870 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:06.870 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:06.870 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:08:06.870 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:07:36.008616+0000", 2023-09-23T21:08:06.870 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.1074999999999994e-05 2023-09-23T21:08:06.870 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:06.870 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:06.870 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:08:06.870 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:07:36.008776+0000", 2023-09-23T21:08:06.870 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000160018 2023-09-23T21:08:06.870 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:06.870 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:06.871 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 1,5", 2023-09-23T21:08:06.871 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:07:36.008900+0000", 2023-09-23T21:08:06.871 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000123473 2023-09-23T21:08:06.871 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:06.871 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:06.871 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:08:06.871 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:07:36.012872+0000", 2023-09-23T21:08:06.871 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0039720470000000003 2023-09-23T21:08:06.871 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:06.871 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:06.871 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T21:08:06.871 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:07:36.013212+0000", 2023-09-23T21:08:06.871 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00034057400000000001 2023-09-23T21:08:06.872 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:06.872 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:06.872 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:08:06.872 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:07:36.058429+0000", 2023-09-23T21:08:06.872 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.045216245000000002 2023-09-23T21:08:06.872 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:06.872 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:06.872 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T21:08:06.872 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:07:36.058491+0000", 2023-09-23T21:08:06.872 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.2205000000000005e-05 2023-09-23T21:08:06.872 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:06.872 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:06.872 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T21:08:06.873 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:07:36.058535+0000", 2023-09-23T21:08:06.873 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.4341000000000001e-05 2023-09-23T21:08:06.873 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:08:06.873 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:08:06.873 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:08:06.873 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:06.873 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:06.873 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35856.0:121911 9.6 9:6ca9307f:::benchmark_data_smithi026_342732_object8635:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e369)", 2023-09-23T21:08:06.873 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T21:07:37.106840+0000", 2023-09-23T21:08:06.873 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 29.686532351, 2023-09-23T21:08:06.873 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.053750034000000002, 2023-09-23T21:08:06.873 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:08:06.873 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:08:06.874 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T21:08:06.874 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35856", 2023-09-23T21:08:06.874 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/4180396258", 2023-09-23T21:08:06.874 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 121911 2023-09-23T21:08:06.874 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:06.874 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:08:06.874 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:06.874 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:08:06.874 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:07:37.106840+0000", 2023-09-23T21:08:06.874 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:08:06.874 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:06.874 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:06.875 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:08:06.875 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:07:37.106840+0000", 2023-09-23T21:08:06.875 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:08:06.875 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:06.875 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:06.875 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:08:06.875 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:07:37.106839+0000", 2023-09-23T21:08:06.875 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999981 2023-09-23T21:08:06.875 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:06.875 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:06.875 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:08:06.875 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:07:37.106847+0000", 2023-09-23T21:08:06.875 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.4270000000000008e-06 2023-09-23T21:08:06.876 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:06.876 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:06.876 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:08:06.876 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:07:37.106864+0000", 2023-09-23T21:08:06.876 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.7232999999999999e-05 2023-09-23T21:08:06.876 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:06.876 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:06.876 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:08:06.876 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:07:37.106873+0000", 2023-09-23T21:08:06.876 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.9299999999999992e-06 2023-09-23T21:08:06.876 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:06.876 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:06.876 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:08:06.877 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:07:37.106929+0000", 2023-09-23T21:08:06.877 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.6280999999999999e-05 2023-09-23T21:08:06.877 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:06.877 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:06.877 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:08:06.877 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:07:37.107068+0000", 2023-09-23T21:08:06.877 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00013859699999999999 2023-09-23T21:08:06.877 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:06.877 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:06.877 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 1,5", 2023-09-23T21:08:06.877 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:07:37.107180+0000", 2023-09-23T21:08:06.877 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00011183099999999999 2023-09-23T21:08:06.877 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:06.878 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:06.878 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T21:08:06.878 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:07:37.117349+0000", 2023-09-23T21:08:06.878 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.010168787 2023-09-23T21:08:06.878 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:06.878 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:06.878 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:08:06.878 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:07:37.121231+0000", 2023-09-23T21:08:06.878 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0038826529999999998 2023-09-23T21:08:06.878 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:06.878 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:06.878 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:08:06.879 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:07:37.156652+0000", 2023-09-23T21:08:06.879 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.035420563000000002 2023-09-23T21:08:06.879 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:06.879 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:06.879 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T21:08:06.879 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:07:37.160544+0000", 2023-09-23T21:08:06.879 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0038923849999999999 2023-09-23T21:08:06.879 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:06.879 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:06.879 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T21:08:06.879 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:07:37.160590+0000", 2023-09-23T21:08:06.879 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.6125000000000003e-05 2023-09-23T21:08:06.879 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:08:06.880 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:08:06.880 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:08:06.880 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:06.880 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:06.880 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35856.0:121947 9.6 9:697be20b:::benchmark_data_smithi026_342732_object8671:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e369)", 2023-09-23T21:08:06.880 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T21:07:37.156939+0000", 2023-09-23T21:08:06.880 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 29.636433416999999, 2023-09-23T21:08:06.880 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.056811767999999999, 2023-09-23T21:08:06.880 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:08:06.880 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:08:06.880 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T21:08:06.880 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35856", 2023-09-23T21:08:06.881 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/4180396258", 2023-09-23T21:08:06.881 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 121947 2023-09-23T21:08:06.881 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:06.881 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:08:06.881 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:06.881 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:08:06.881 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:07:37.156939+0000", 2023-09-23T21:08:06.881 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:08:06.881 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:06.881 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:06.881 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:08:06.881 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:07:37.156939+0000", 2023-09-23T21:08:06.881 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:08:06.882 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:06.882 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:06.882 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:08:06.882 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:07:37.156939+0000", 2023-09-23T21:08:06.882 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T21:08:06.882 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:06.882 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:06.882 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:08:06.882 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:07:37.156942+0000", 2023-09-23T21:08:06.882 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.7319999999999998e-06 2023-09-23T21:08:06.882 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:06.882 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:06.883 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:08:06.883 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:07:37.156948+0000", 2023-09-23T21:08:06.883 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.93e-06 2023-09-23T21:08:06.883 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:06.883 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:06.883 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:08:06.883 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:07:37.156953+0000", 2023-09-23T21:08:06.883 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.9799999999999998e-06 2023-09-23T21:08:06.883 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:06.883 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:06.883 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:08:06.883 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:07:37.165418+0000", 2023-09-23T21:08:06.884 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0084650650000000008 2023-09-23T21:08:06.884 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:06.884 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:06.884 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:08:06.884 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:07:37.165575+0000", 2023-09-23T21:08:06.884 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00015764699999999999 2023-09-23T21:08:06.884 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:06.884 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:06.884 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 1,5", 2023-09-23T21:08:06.884 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:07:37.165700+0000", 2023-09-23T21:08:06.884 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00012457200000000001 2023-09-23T21:08:06.884 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:06.884 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:06.885 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T21:08:06.885 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:07:37.170015+0000", 2023-09-23T21:08:06.885 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0043152980000000004 2023-09-23T21:08:06.885 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:06.885 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:06.885 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:08:06.885 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:07:37.175956+0000", 2023-09-23T21:08:06.885 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0059410610000000001 2023-09-23T21:08:06.885 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:06.885 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:06.885 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:08:06.885 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:07:37.213646+0000", 2023-09-23T21:08:06.885 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.037689651999999997 2023-09-23T21:08:06.886 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:06.886 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:06.886 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T21:08:06.886 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:07:37.213703+0000", 2023-09-23T21:08:06.886 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.6972000000000003e-05 2023-09-23T21:08:06.886 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:06.886 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:06.886 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T21:08:06.886 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:07:37.213751+0000", 2023-09-23T21:08:06.886 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.8263000000000002e-05 2023-09-23T21:08:06.886 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:08:06.886 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:08:06.887 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:08:06.887 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:06.887 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:06.887 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35856.0:122447 9.6 9:64fe2da7:::benchmark_data_smithi026_342732_object9171:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e369)", 2023-09-23T21:08:06.887 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T21:07:37.884550+0000", 2023-09-23T21:08:06.887 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 28.908822310000001, 2023-09-23T21:08:06.887 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.052168580999999999, 2023-09-23T21:08:06.887 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:08:06.887 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:08:06.887 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T21:08:06.887 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35856", 2023-09-23T21:08:06.887 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/4180396258", 2023-09-23T21:08:06.887 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 122447 2023-09-23T21:08:06.888 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:06.888 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:08:06.888 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:06.888 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:08:06.888 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:07:37.884550+0000", 2023-09-23T21:08:06.888 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:08:06.888 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:06.888 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:06.888 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:08:06.888 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:07:37.884550+0000", 2023-09-23T21:08:06.888 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:08:06.888 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:06.888 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:06.889 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:08:06.889 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:07:37.884549+0000", 2023-09-23T21:08:06.889 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999981 2023-09-23T21:08:06.889 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:06.889 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:06.889 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:08:06.889 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:07:37.884556+0000", 2023-09-23T21:08:06.889 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.481e-06 2023-09-23T21:08:06.889 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:06.889 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:06.889 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:08:06.889 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:07:37.884575+0000", 2023-09-23T21:08:06.889 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.8842e-05 2023-09-23T21:08:06.890 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:06.890 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:06.890 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:08:06.890 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:07:37.884586+0000", 2023-09-23T21:08:06.890 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0739e-05 2023-09-23T21:08:06.890 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:06.890 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:06.890 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:08:06.890 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:07:37.884641+0000", 2023-09-23T21:08:06.890 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.5926e-05 2023-09-23T21:08:06.890 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:06.890 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:06.891 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:08:06.891 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:07:37.884769+0000", 2023-09-23T21:08:06.891 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000127436 2023-09-23T21:08:06.891 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:06.891 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:06.891 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 1,5", 2023-09-23T21:08:06.891 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:07:37.884872+0000", 2023-09-23T21:08:06.891 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000103154 2023-09-23T21:08:06.891 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:06.891 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:06.891 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T21:08:06.891 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:07:37.893649+0000", 2023-09-23T21:08:06.892 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0087770049999999992 2023-09-23T21:08:06.892 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:06.892 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:06.892 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:08:06.892 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:07:37.906176+0000", 2023-09-23T21:08:06.892 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.012527173000000001 2023-09-23T21:08:06.892 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:06.892 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:06.892 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:08:06.892 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:07:37.936606+0000", 2023-09-23T21:08:06.892 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.030429520000000002 2023-09-23T21:08:06.892 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:06.892 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:06.893 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T21:08:06.893 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:07:37.936675+0000", 2023-09-23T21:08:06.893 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.8837000000000005e-05 2023-09-23T21:08:06.893 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:06.893 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:06.893 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T21:08:06.893 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:07:37.936719+0000", 2023-09-23T21:08:06.893 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.4341000000000001e-05 2023-09-23T21:08:06.893 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:08:06.893 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:08:06.893 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:08:06.893 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:06.894 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:06.894 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35856.0:122453 9.6 9:692a8f72:::benchmark_data_smithi026_342732_object9177:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e369)", 2023-09-23T21:08:06.894 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T21:07:37.884614+0000", 2023-09-23T21:08:06.894 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 28.908758363, 2023-09-23T21:08:06.894 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.052204815000000002, 2023-09-23T21:08:06.894 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:08:06.894 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:08:06.894 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T21:08:06.894 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35856", 2023-09-23T21:08:06.894 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/4180396258", 2023-09-23T21:08:06.894 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 122453 2023-09-23T21:08:06.894 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:06.895 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:08:06.895 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:06.895 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:08:06.895 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:07:37.884614+0000", 2023-09-23T21:08:06.895 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:08:06.895 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:06.895 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:06.895 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:08:06.895 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:07:37.884614+0000", 2023-09-23T21:08:06.895 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:08:06.895 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:06.895 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:06.896 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:08:06.896 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:07:37.884614+0000", 2023-09-23T21:08:06.896 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T21:08:06.896 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:06.896 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:06.896 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:08:06.896 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:07:37.884618+0000", 2023-09-23T21:08:06.896 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.1439999999999996e-06 2023-09-23T21:08:06.896 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:06.896 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:06.896 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:08:06.896 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:07:37.884628+0000", 2023-09-23T21:08:06.897 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0246e-05 2023-09-23T21:08:06.897 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:06.897 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:06.897 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:08:06.897 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:07:37.884635+0000", 2023-09-23T21:08:06.897 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.3200000000000002e-06 2023-09-23T21:08:06.897 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:06.897 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:06.897 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:08:06.897 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:07:37.889273+0000", 2023-09-23T21:08:06.897 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0046380019999999996 2023-09-23T21:08:06.897 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:06.897 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:06.897 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:08:06.898 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:07:37.889395+0000", 2023-09-23T21:08:06.898 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00012180000000000001 2023-09-23T21:08:06.898 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:06.898 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:06.898 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 1,5", 2023-09-23T21:08:06.898 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:07:37.889513+0000", 2023-09-23T21:08:06.898 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00011781399999999999 2023-09-23T21:08:06.898 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:06.898 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:06.898 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T21:08:06.898 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:07:37.895646+0000", 2023-09-23T21:08:06.898 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0061332460000000002 2023-09-23T21:08:06.899 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:06.899 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:06.899 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:08:06.899 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:07:37.906214+0000", 2023-09-23T21:08:06.899 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.010568037000000001 2023-09-23T21:08:06.899 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:06.899 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:06.899 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:08:06.899 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:07:37.936753+0000", 2023-09-23T21:08:06.899 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.030538744 2023-09-23T21:08:06.899 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:06.899 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:06.899 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T21:08:06.900 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:07:37.936788+0000", 2023-09-23T21:08:06.900 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.4808999999999998e-05 2023-09-23T21:08:06.900 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:06.900 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:06.900 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T21:08:06.900 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:07:37.936819+0000", 2023-09-23T21:08:06.900 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.1312000000000001e-05 2023-09-23T21:08:06.900 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:08:06.900 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:08:06.900 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:08:06.900 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:06.900 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:06.901 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35856.0:137419 9.6 9:6f24cadc:::benchmark_data_smithi026_342732_object24143:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e374)", 2023-09-23T21:08:06.901 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T21:07:56.198715+0000", 2023-09-23T21:08:06.901 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 10.594657569000001, 2023-09-23T21:08:06.901 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.050566184, 2023-09-23T21:08:06.901 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:08:06.901 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:08:06.901 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T21:08:06.901 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35856", 2023-09-23T21:08:06.901 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/4180396258", 2023-09-23T21:08:06.901 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 137419 2023-09-23T21:08:06.901 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:06.901 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:08:06.902 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:06.902 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:08:06.902 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:07:56.198715+0000", 2023-09-23T21:08:06.902 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:08:06.902 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:06.902 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:06.902 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:08:06.902 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:07:56.198715+0000", 2023-09-23T21:08:06.902 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:08:06.902 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:06.902 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:06.902 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:08:06.902 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:07:56.198713+0000", 2023-09-23T21:08:06.903 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999976 2023-09-23T21:08:06.903 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:06.903 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:06.903 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:08:06.903 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:07:56.198723+0000", 2023-09-23T21:08:06.903 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.059e-05 2023-09-23T21:08:06.903 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:06.903 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:06.903 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:08:06.903 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:07:56.198740+0000", 2023-09-23T21:08:06.903 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.6073999999999999e-05 2023-09-23T21:08:06.903 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:06.903 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:06.904 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:08:06.904 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:07:56.198750+0000", 2023-09-23T21:08:06.904 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.022e-05 2023-09-23T21:08:06.904 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:06.904 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:06.904 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:08:06.904 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:07:56.198812+0000", 2023-09-23T21:08:06.904 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.2605000000000001e-05 2023-09-23T21:08:06.904 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:06.904 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:06.904 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:08:06.904 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:07:56.198972+0000", 2023-09-23T21:08:06.905 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00015989800000000001 2023-09-23T21:08:06.905 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:06.905 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:06.905 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 1,5", 2023-09-23T21:08:06.905 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:07:56.199094+0000", 2023-09-23T21:08:06.905 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00012132 2023-09-23T21:08:06.905 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:06.905 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:06.905 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T21:08:06.905 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:07:56.210505+0000", 2023-09-23T21:08:06.905 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.011411737999999999 2023-09-23T21:08:06.905 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:06.905 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:06.906 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:08:06.906 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:07:56.214588+0000", 2023-09-23T21:08:06.906 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0040831349999999999 2023-09-23T21:08:06.906 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:06.906 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:06.906 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:08:06.906 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:07:56.249191+0000", 2023-09-23T21:08:06.906 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.034603026000000002 2023-09-23T21:08:06.906 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:06.906 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:06.906 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T21:08:06.906 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:07:56.249243+0000", 2023-09-23T21:08:06.907 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.1542000000000002e-05 2023-09-23T21:08:06.907 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:06.907 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:06.907 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T21:08:06.907 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:07:56.249281+0000", 2023-09-23T21:08:06.907 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.8297999999999998e-05 2023-09-23T21:08:06.907 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:08:06.907 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:08:06.907 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:08:06.907 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:06.907 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:06.907 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35856.0:143466 9.6 9:66134494:::benchmark_data_smithi026_342732_object30190:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e375)", 2023-09-23T21:08:06.907 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T21:08:03.705721+0000", 2023-09-23T21:08:06.908 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 3.0876521769999998, 2023-09-23T21:08:06.908 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.048038299999999999, 2023-09-23T21:08:06.908 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:08:06.908 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:08:06.908 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T21:08:06.908 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35856", 2023-09-23T21:08:06.908 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/4180396258", 2023-09-23T21:08:06.908 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 143466 2023-09-23T21:08:06.908 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:06.908 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:08:06.908 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:06.908 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:08:06.908 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:08:03.705721+0000", 2023-09-23T21:08:06.909 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:08:06.909 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:06.909 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:06.909 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:08:06.909 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:08:03.705721+0000", 2023-09-23T21:08:06.909 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:08:06.909 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:06.909 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:06.909 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:08:06.909 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:08:03.705718+0000", 2023-09-23T21:08:06.909 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999981 2023-09-23T21:08:06.909 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:06.910 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:06.910 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:08:06.910 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:08:03.705726+0000", 2023-09-23T21:08:06.910 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.9780000000000005e-06 2023-09-23T21:08:06.910 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:06.910 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:06.910 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:08:06.910 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:08:03.705746+0000", 2023-09-23T21:08:06.910 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.9496000000000001e-05 2023-09-23T21:08:06.910 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:06.910 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:06.910 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:08:06.910 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:08:03.705755+0000", 2023-09-23T21:08:06.911 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.1030000000000008e-06 2023-09-23T21:08:06.911 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:06.911 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:06.911 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:08:06.911 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:08:03.705839+0000", 2023-09-23T21:08:06.911 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.3720000000000005e-05 2023-09-23T21:08:06.911 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:06.911 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:06.911 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:08:06.911 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:08:03.705980+0000", 2023-09-23T21:08:06.911 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00014166600000000001 2023-09-23T21:08:06.911 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:06.912 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:06.912 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 1,5", 2023-09-23T21:08:06.912 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:08:03.706109+0000", 2023-09-23T21:08:06.912 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00012889999999999999 2023-09-23T21:08:06.912 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:06.912 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:06.912 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:08:06.912 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:08:03.718517+0000", 2023-09-23T21:08:06.912 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.012407484999999999 2023-09-23T21:08:06.912 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:06.912 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:06.912 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T21:08:06.912 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:08:03.718524+0000", 2023-09-23T21:08:06.913 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.6650000000000003e-06 2023-09-23T21:08:06.913 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:06.913 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:06.913 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:08:06.913 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:08:03.753638+0000", 2023-09-23T21:08:06.913 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.035113060000000001 2023-09-23T21:08:06.913 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:06.913 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:06.913 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T21:08:06.913 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:08:03.753707+0000", 2023-09-23T21:08:06.913 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.9519999999999998e-05 2023-09-23T21:08:06.913 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:06.913 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:06.914 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T21:08:06.914 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:08:03.753759+0000", 2023-09-23T21:08:06.914 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.1752000000000002e-05 2023-09-23T21:08:06.914 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:08:06.914 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:08:06.914 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:08:06.914 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:06.914 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:06.914 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35856.0:143468 9.6 9:63fa9138:::benchmark_data_smithi026_342732_object30192:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e375)", 2023-09-23T21:08:06.914 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T21:08:03.708171+0000", 2023-09-23T21:08:06.914 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 3.0852017969999999, 2023-09-23T21:08:06.914 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.045734055000000003, 2023-09-23T21:08:06.915 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:08:06.915 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:08:06.915 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T21:08:06.915 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35856", 2023-09-23T21:08:06.915 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/4180396258", 2023-09-23T21:08:06.915 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 143468 2023-09-23T21:08:06.915 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:06.915 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:08:06.915 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:06.915 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:08:06.915 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:08:03.708171+0000", 2023-09-23T21:08:06.915 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:08:06.916 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:06.916 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:06.916 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:08:06.916 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:08:03.708171+0000", 2023-09-23T21:08:06.916 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:08:06.916 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:06.916 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:06.916 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:08:06.916 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:08:03.708170+0000", 2023-09-23T21:08:06.916 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T21:08:06.916 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:06.916 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:06.917 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:08:06.917 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:08:03.708176+0000", 2023-09-23T21:08:06.917 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.5760000000000002e-06 2023-09-23T21:08:06.917 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:06.917 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:06.917 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:08:06.917 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:08:03.708187+0000", 2023-09-23T21:08:06.917 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1131999999999999e-05 2023-09-23T21:08:06.917 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:06.917 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:06.917 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:08:06.917 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:08:03.708195+0000", 2023-09-23T21:08:06.917 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.6739999999999997e-06 2023-09-23T21:08:06.917 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:06.918 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:06.918 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:08:06.918 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:08:03.712821+0000", 2023-09-23T21:08:06.918 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.004625685 2023-09-23T21:08:06.918 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:06.918 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:06.918 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:08:06.918 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:08:03.712983+0000", 2023-09-23T21:08:06.918 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00016247700000000001 2023-09-23T21:08:06.918 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:06.918 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:06.918 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 1,5", 2023-09-23T21:08:06.919 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:08:03.713119+0000", 2023-09-23T21:08:06.919 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000135468 2023-09-23T21:08:06.919 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:06.919 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:06.919 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:08:06.919 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:08:03.733901+0000", 2023-09-23T21:08:06.919 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.020782551 2023-09-23T21:08:06.919 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:06.919 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:06.919 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T21:08:06.919 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:08:03.733915+0000", 2023-09-23T21:08:06.919 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.3816e-05 2023-09-23T21:08:06.919 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:06.920 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:06.920 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:08:06.920 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:08:03.753808+0000", 2023-09-23T21:08:06.920 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.019892909 2023-09-23T21:08:06.920 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:06.920 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:06.920 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T21:08:06.920 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:08:03.753859+0000", 2023-09-23T21:08:06.920 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.0677000000000003e-05 2023-09-23T21:08:06.920 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:06.920 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:06.920 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T21:08:06.920 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:08:03.753905+0000", 2023-09-23T21:08:06.921 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.6397000000000003e-05 2023-09-23T21:08:06.921 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:08:06.921 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:08:06.921 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:08:06.921 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:06.921 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:06.921 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35856.0:143477 9.6 9:6664fd67:::benchmark_data_smithi026_342732_object30201:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e375)", 2023-09-23T21:08:06.921 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T21:08:03.716956+0000", 2023-09-23T21:08:06.921 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 3.0764167800000002, 2023-09-23T21:08:06.921 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.045761894999999997, 2023-09-23T21:08:06.921 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:08:06.921 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:08:06.922 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T21:08:06.922 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35856", 2023-09-23T21:08:06.922 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/4180396258", 2023-09-23T21:08:06.922 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 143477 2023-09-23T21:08:06.922 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:06.922 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:08:06.922 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:06.922 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:08:06.922 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:08:03.716956+0000", 2023-09-23T21:08:06.922 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:08:06.922 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:06.922 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:06.922 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:08:06.923 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:08:03.716956+0000", 2023-09-23T21:08:06.923 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:08:06.923 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:06.923 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:06.923 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:08:06.923 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:08:03.716954+0000", 2023-09-23T21:08:06.923 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999981 2023-09-23T21:08:06.923 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:06.923 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:06.923 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:08:06.923 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:08:03.716968+0000", 2023-09-23T21:08:06.923 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.343e-05 2023-09-23T21:08:06.924 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:06.924 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:06.924 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:08:06.924 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:08:03.716982+0000", 2023-09-23T21:08:06.924 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.3998999999999999e-05 2023-09-23T21:08:06.924 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:06.924 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:06.924 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:08:06.924 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:08:03.716991+0000", 2023-09-23T21:08:06.924 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.6949999999999995e-06 2023-09-23T21:08:06.924 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:06.924 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:06.924 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:08:06.925 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:08:03.724917+0000", 2023-09-23T21:08:06.925 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0079255160000000005 2023-09-23T21:08:06.925 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:06.925 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:06.925 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:08:06.925 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:08:03.725050+0000", 2023-09-23T21:08:06.925 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00013350800000000001 2023-09-23T21:08:06.925 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:06.925 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:06.925 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 1,5", 2023-09-23T21:08:06.925 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:08:03.725168+0000", 2023-09-23T21:08:06.926 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000117618 2023-09-23T21:08:06.926 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:06.926 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:06.926 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:08:06.926 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:08:03.734386+0000", 2023-09-23T21:08:06.926 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0092177439999999999 2023-09-23T21:08:06.926 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:06.926 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:06.926 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T21:08:06.926 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:08:03.741611+0000", 2023-09-23T21:08:06.926 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0072256330000000004 2023-09-23T21:08:06.926 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:06.926 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:06.927 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:08:06.927 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:08:03.762595+0000", 2023-09-23T21:08:06.927 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.020983778000000002 2023-09-23T21:08:06.927 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:06.927 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:06.927 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T21:08:06.927 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:08:03.762674+0000", 2023-09-23T21:08:06.927 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.8375999999999996e-05 2023-09-23T21:08:06.927 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:06.927 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:06.927 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T21:08:06.927 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:08:03.762718+0000", 2023-09-23T21:08:06.928 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.4267000000000002e-05 2023-09-23T21:08:06.928 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:08:06.928 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:08:06.928 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:08:06.928 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:06.928 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:06.928 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35856.0:144088 9.6 9:6fd430c1:::benchmark_data_smithi026_342732_object30812:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e375)", 2023-09-23T21:08:06.928 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T21:08:04.433514+0000", 2023-09-23T21:08:06.928 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 2.3598588559999998, 2023-09-23T21:08:06.928 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.051366380000000003, 2023-09-23T21:08:06.928 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:08:06.928 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:08:06.928 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T21:08:06.929 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35856", 2023-09-23T21:08:06.929 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/4180396258", 2023-09-23T21:08:06.929 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 144088 2023-09-23T21:08:06.929 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:06.929 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:08:06.929 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:06.929 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:08:06.929 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:08:04.433514+0000", 2023-09-23T21:08:06.929 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:08:06.929 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:06.929 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:06.929 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:08:06.929 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:08:04.433514+0000", 2023-09-23T21:08:06.930 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:08:06.930 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:06.930 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:06.930 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:08:06.930 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:08:04.433513+0000", 2023-09-23T21:08:06.930 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T21:08:06.930 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:06.930 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:06.930 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:08:06.930 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:08:04.433517+0000", 2023-09-23T21:08:06.930 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.4170000000000001e-06 2023-09-23T21:08:06.930 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:06.931 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:06.931 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:08:06.931 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:08:04.433524+0000", 2023-09-23T21:08:06.931 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.9709999999999998e-06 2023-09-23T21:08:06.931 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:06.931 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:06.931 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:08:06.931 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:08:04.433528+0000", 2023-09-23T21:08:06.931 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.7219999999999999e-06 2023-09-23T21:08:06.931 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:06.931 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:06.931 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:08:06.932 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:08:04.449935+0000", 2023-09-23T21:08:06.932 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.016406546000000001 2023-09-23T21:08:06.932 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:06.932 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:06.932 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:08:06.932 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:08:04.450080+0000", 2023-09-23T21:08:06.932 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00014526200000000001 2023-09-23T21:08:06.932 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:06.932 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:06.932 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 1,5", 2023-09-23T21:08:06.932 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:08:04.450209+0000", 2023-09-23T21:08:06.933 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00012872799999999999 2023-09-23T21:08:06.933 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:06.933 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:06.933 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T21:08:06.933 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:08:04.458881+0000", 2023-09-23T21:08:06.933 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0086718850000000007 2023-09-23T21:08:06.933 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:06.933 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:06.934 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:08:06.934 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:08:04.466985+0000", 2023-09-23T21:08:06.934 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0081046120000000006 2023-09-23T21:08:06.934 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:06.934 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:06.934 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:08:06.934 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:08:04.484799+0000", 2023-09-23T21:08:06.934 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.017813786000000002 2023-09-23T21:08:06.934 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:06.934 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:06.934 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T21:08:06.935 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:08:04.484835+0000", 2023-09-23T21:08:06.935 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.6103999999999999e-05 2023-09-23T21:08:06.935 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:06.935 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:06.935 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T21:08:06.935 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:08:04.484880+0000", 2023-09-23T21:08:06.935 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.4876000000000002e-05 2023-09-23T21:08:06.935 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:08:06.935 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:08:06.935 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:08:06.935 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:06.935 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:06.935 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35856.0:144741 9.6 9:6599e3d0:::benchmark_data_smithi026_342732_object31465:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e375)", 2023-09-23T21:08:06.936 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T21:08:05.184459+0000", 2023-09-23T21:08:06.936 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 1.608913939, 2023-09-23T21:08:06.936 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.045853186999999997, 2023-09-23T21:08:06.936 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:08:06.936 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:08:06.936 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T21:08:06.936 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35856", 2023-09-23T21:08:06.936 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/4180396258", 2023-09-23T21:08:06.936 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 144741 2023-09-23T21:08:06.936 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:06.936 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:08:06.936 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:06.936 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:08:06.937 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:08:05.184459+0000", 2023-09-23T21:08:06.937 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:08:06.937 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:06.937 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:06.937 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:08:06.937 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:08:05.184459+0000", 2023-09-23T21:08:06.937 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:08:06.937 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:06.937 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:06.937 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:08:06.937 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:08:05.184458+0000", 2023-09-23T21:08:06.937 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T21:08:06.937 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:06.938 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:06.938 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:08:06.938 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:08:05.184462+0000", 2023-09-23T21:08:06.938 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.3249999999999999e-06 2023-09-23T21:08:06.938 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:06.938 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:06.938 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:08:06.938 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:08:05.184469+0000", 2023-09-23T21:08:06.938 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.0310000000000001e-06 2023-09-23T21:08:06.938 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:06.938 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:06.938 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:08:06.939 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:08:05.184479+0000", 2023-09-23T21:08:06.939 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0888e-05 2023-09-23T21:08:06.939 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:06.939 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:06.939 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:08:06.939 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:08:05.189300+0000", 2023-09-23T21:08:06.939 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.004820319 2023-09-23T21:08:06.939 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:06.939 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:06.939 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:08:06.939 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:08:05.189411+0000", 2023-09-23T21:08:06.939 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00011075 2023-09-23T21:08:06.940 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:06.940 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:06.940 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 1,5", 2023-09-23T21:08:06.940 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:08:05.189520+0000", 2023-09-23T21:08:06.940 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000109096 2023-09-23T21:08:06.940 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:06.940 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:06.940 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T21:08:06.940 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:08:05.198856+0000", 2023-09-23T21:08:06.940 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0093366230000000005 2023-09-23T21:08:06.940 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:06.940 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:06.940 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:08:06.941 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:08:05.198889+0000", 2023-09-23T21:08:06.941 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.2345999999999998e-05 2023-09-23T21:08:06.941 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:06.941 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:06.941 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:08:06.941 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:08:05.230228+0000", 2023-09-23T21:08:06.941 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.031339766999999998 2023-09-23T21:08:06.941 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:06.941 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:06.941 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T21:08:06.941 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:08:05.230276+0000", 2023-09-23T21:08:06.941 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.7398999999999998e-05 2023-09-23T21:08:06.941 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:06.942 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:06.942 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T21:08:06.942 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:08:05.230312+0000", 2023-09-23T21:08:06.942 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.6143000000000003e-05 2023-09-23T21:08:06.942 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:08:06.942 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:08:06.942 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:08:06.942 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:08:06.942 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:08:06.942 INFO:teuthology.orchestra.run.smithi026.stdout:} 2023-09-23T21:08:06.943 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:08:06.943 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:08:07.012 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:08:07.203 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:08:07.215 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:08:07.216 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:08:07.317 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:08:07.361 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:08:07.367 INFO:teuthology.orchestra.run.smithi026.stderr:noscrub is set 2023-09-23T21:08:07.408 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:08:07.416 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:08:07.416 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:08:07.517 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:08:07.559 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:08:07.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:08:07 smithi070 ceph-mon[168282]: from='client.46112 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T21:08:07.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:08:07 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/1618393721' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T21:08:07.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:08:07 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T21:08:07.620 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:08:07.622 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:08:07.623 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:08:07.640 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:08:07 smithi026 ceph-mon[175151]: from='client.46112 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T21:08:07.640 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:08:07 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/1618393721' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T21:08:07.640 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:08:07 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T21:08:07.640 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:08:07 smithi026 ceph-mon[179688]: from='client.46112 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T21:08:07.640 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:08:07 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/1618393721' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T21:08:07.641 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:08:07 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T21:08:07.656 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_ops_in_flight 2023-09-23T21:08:07.724 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:08:07.808 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:08:07.926 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:08:08.386 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:08:08.400 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:08:08.401 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:08:08.502 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:08:08.547 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:08:08.603 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:08:08.610 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:08:08.611 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:08:08.611 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:08:08 smithi070 ceph-mon[168282]: pgmap v984: 109 pgs: 109 active+clean; 5.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 800 op/s 2023-09-23T21:08:08.611 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:08:08 smithi070 ceph-mon[168282]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:08:08.611 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:08:08 smithi070 ceph-mon[168282]: failed to encode map e376 with expected crc 2023-09-23T21:08:08.611 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:08:08 smithi070 ceph-mon[168282]: failed to encode map e376 with expected crc 2023-09-23T21:08:08.611 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:08:08 smithi070 ceph-mon[168282]: failed to encode map e376 with expected crc 2023-09-23T21:08:08.612 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:08:08 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T21:08:08.612 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:08:08 smithi070 ceph-mon[168282]: osdmap e376: 8 total, 8 up, 8 in 2023-09-23T21:08:08.633 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:08:08 smithi026 ceph-mon[175151]: pgmap v984: 109 pgs: 109 active+clean; 5.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 800 op/s 2023-09-23T21:08:08.633 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:08:08 smithi026 ceph-mon[175151]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:08:08.633 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:08:08 smithi026 ceph-mon[175151]: failed to encode map e376 with expected crc 2023-09-23T21:08:08.634 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:08:08 smithi026 ceph-mon[175151]: failed to encode map e376 with expected crc 2023-09-23T21:08:08.634 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:08:08 smithi026 ceph-mon[175151]: failed to encode map e376 with expected crc 2023-09-23T21:08:08.634 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:08:08 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T21:08:08.634 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:08:08 smithi026 ceph-mon[175151]: osdmap e376: 8 total, 8 up, 8 in 2023-09-23T21:08:08.634 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:08:08 smithi026 ceph-mon[179688]: pgmap v984: 109 pgs: 109 active+clean; 5.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 800 op/s 2023-09-23T21:08:08.634 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:08:08 smithi026 ceph-mon[179688]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:08:08.634 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:08:08 smithi026 ceph-mon[179688]: failed to encode map e376 with expected crc 2023-09-23T21:08:08.634 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:08:08 smithi026 ceph-mon[179688]: failed to encode map e376 with expected crc 2023-09-23T21:08:08.634 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:08:08 smithi026 ceph-mon[179688]: failed to encode map e376 with expected crc 2023-09-23T21:08:08.634 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:08:08 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T21:08:08.635 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:08:08 smithi026 ceph-mon[179688]: osdmap e376: 8 total, 8 up, 8 in 2023-09-23T21:08:08.712 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:08:08.763 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:08:08.817 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:08:08.826 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:08:08.827 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:08:08.928 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:08:08.992 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:08:09.041 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:08:09.045 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:08:09.045 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:08:09.146 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:08:09.237 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:08:09.371 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:08:09.375 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:08:09.376 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:08:09.476 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:08:09.546 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:08:09.632 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 21:08:09 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[164029]: ::ffff:172.21.15.70 - - [23/Sep/2023:21:08:09] "GET /metrics HTTP/1.1" 200 36299 "" "Prometheus/2.43.0" 2023-09-23T21:08:09.673 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T21:08:09.680 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:08:09.681 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:08:09.781 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:08:09.794 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:08:09.901 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:08:10.043 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T21:08:10.052 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:08:10.052 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:08:10.123 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0: {} 2023-09-23T21:08:10.146 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1: {} 2023-09-23T21:08:10.153 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:08:10.192 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2: {} 2023-09-23T21:08:10.250 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:08:10.251 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3: {} 2023-09-23T21:08:10.284 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4: {} 2023-09-23T21:08:10.331 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T21:08:10.359 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:08:10 smithi026 ceph-mon[175151]: pgmap v986: 109 pgs: 109 active+clean; 4.9 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 805 op/s 2023-09-23T21:08:10.360 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:08:10 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/1310750585' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:08:10.388 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6: {} 2023-09-23T21:08:10.429 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T21:08:10.435 INFO:teuthology.orchestra.run.smithi026.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T21:08:10.435 INFO:teuthology.orchestra.run.smithi026.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T21:08:10.435 INFO:teuthology.orchestra.run.smithi026.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T21:08:10.436 INFO:teuthology.orchestra.run.smithi026.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T21:08:10.436 INFO:teuthology.orchestra.run.smithi026.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T21:08:10.436 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T21:08:10.436 INFO:teuthology.orchestra.run.smithi026.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T21:08:10.436 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T21:08:10.501 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:08:10.505 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:08:10.505 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:08:10.606 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:08:10.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:08:10 smithi070 ceph-mon[168282]: pgmap v986: 109 pgs: 109 active+clean; 4.9 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 805 op/s 2023-09-23T21:08:10.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:08:10 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/1310750585' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:08:10.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:08:10 smithi026 ceph-mon[179688]: pgmap v986: 109 pgs: 109 active+clean; 4.9 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 805 op/s 2023-09-23T21:08:10.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:08:10 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/1310750585' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:08:10.643 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:08:10.692 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:08:10.698 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:08:10.699 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:08:10.800 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:08:10.869 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:08:10.927 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:08:10.937 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:08:10.937 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:08:11.038 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:08:11.146 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:08:11.339 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T21:08:11.353 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:08:11.364 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:08:11.364 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:08:11.465 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:08:11.517 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:08:11.562 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:08:11.564 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:08:11.564 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:08:11.665 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:08:11.698 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:08:11.752 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:08:11.758 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:08:11.758 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:08:11.859 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:08:11.887 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:08:11.943 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:08:11.947 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:08:11.948 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:08:12.040 INFO:teuthology.orchestra.run.smithi026.stdout:ERROR: (22) Invalid argument 2023-09-23T21:08:12.040 INFO:teuthology.orchestra.run.smithi026.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T21:08:12.048 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:08:12.097 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:08:12.154 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:08:12.164 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:08:12.164 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:08:12.265 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:08:12.327 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:08:12.329 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:08:12.376 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:08:12.385 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:08:12.385 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:08:12.397 INFO:teuthology.orchestra.run.smithi026.stderr:set osd.1 primary-affinity to 0.972172 (8637122) 2023-09-23T21:08:12.486 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:08:12.568 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:08:12.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:08:12 smithi070 ceph-mon[168282]: pgmap v987: 109 pgs: 109 active+clean; 4.9 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 806 op/s 2023-09-23T21:08:12.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:08:12 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/432460561' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 1, "weight": 0.9721718104590561}]: dispatch 2023-09-23T21:08:12.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:08:12 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 1, "weight": 0.9721718104590561}]: dispatch 2023-09-23T21:08:12.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:08:12 smithi026 ceph-mon[175151]: pgmap v987: 109 pgs: 109 active+clean; 4.9 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 806 op/s 2023-09-23T21:08:12.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:08:12 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/432460561' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 1, "weight": 0.9721718104590561}]: dispatch 2023-09-23T21:08:12.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:08:12 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 1, "weight": 0.9721718104590561}]: dispatch 2023-09-23T21:08:12.641 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:08:12 smithi026 ceph-mon[179688]: pgmap v987: 109 pgs: 109 active+clean; 4.9 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 806 op/s 2023-09-23T21:08:12.641 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:08:12 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/432460561' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 1, "weight": 0.9721718104590561}]: dispatch 2023-09-23T21:08:12.641 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:08:12 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 1, "weight": 0.9721718104590561}]: dispatch 2023-09-23T21:08:12.929 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_blocked_ops 2023-09-23T21:08:13.148 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:08:13.156 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:08:13.156 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:08:13.257 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:08:13.311 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:08:13.370 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:08:13.372 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:08:13.373 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:08:13.411 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T21:08:13.473 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:08:13.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:08:13 smithi070 ceph-mon[168282]: failed to encode map e377 with expected crc 2023-09-23T21:08:13.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:08:13 smithi070 ceph-mon[168282]: failed to encode map e377 with expected crc 2023-09-23T21:08:13.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:08:13 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 1, "weight": 0.9721718104590561}]': finished 2023-09-23T21:08:13.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:08:13 smithi070 ceph-mon[168282]: osdmap e377: 8 total, 8 up, 8 in 2023-09-23T21:08:13.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:08:13 smithi070 ceph-mon[168282]: failed to encode map e377 with expected crc 2023-09-23T21:08:13.621 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:08:13 smithi026 ceph-mon[175151]: failed to encode map e377 with expected crc 2023-09-23T21:08:13.621 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:08:13 smithi026 ceph-mon[175151]: failed to encode map e377 with expected crc 2023-09-23T21:08:13.621 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:08:13 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 1, "weight": 0.9721718104590561}]': finished 2023-09-23T21:08:13.621 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:08:13 smithi026 ceph-mon[175151]: osdmap e377: 8 total, 8 up, 8 in 2023-09-23T21:08:13.621 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:08:13 smithi026 ceph-mon[175151]: failed to encode map e377 with expected crc 2023-09-23T21:08:13.621 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:08:13 smithi026 ceph-mon[179688]: failed to encode map e377 with expected crc 2023-09-23T21:08:13.621 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:08:13 smithi026 ceph-mon[179688]: failed to encode map e377 with expected crc 2023-09-23T21:08:13.621 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:08:13 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 1, "weight": 0.9721718104590561}]': finished 2023-09-23T21:08:13.621 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:08:13 smithi026 ceph-mon[179688]: osdmap e377: 8 total, 8 up, 8 in 2023-09-23T21:08:13.621 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:08:13 smithi026 ceph-mon[179688]: failed to encode map e377 with expected crc 2023-09-23T21:08:13.625 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:08:13.848 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:08:13.854 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:08:13.855 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:08:13.955 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:08:13.988 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:08:14.045 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:08:14.049 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:08:14.049 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:08:14.052 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:08:14.150 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:08:14.193 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:08:14.254 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:08:14.264 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:08:14.264 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:08:14.365 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:08:14.415 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:08:14.478 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:08:14.485 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:08:14.486 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:08:14.587 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:08:14.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:08:14 smithi070 ceph-mon[168282]: pgmap v989: 109 pgs: 109 active+clean; 4.7 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 753 op/s 2023-09-23T21:08:14.625 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:08:14.679 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:08:14.688 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:08:14.689 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:08:14.690 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:08:14.691 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:08:14 smithi026 ceph-mon[179688]: pgmap v989: 109 pgs: 109 active+clean; 4.7 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 753 op/s 2023-09-23T21:08:14.696 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:08:14 smithi026 ceph-mon[175151]: pgmap v989: 109 pgs: 109 active+clean; 4.7 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 753 op/s 2023-09-23T21:08:14.791 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:08:14.872 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:08:14.992 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T21:08:15.001 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:08:15.001 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:08:15.102 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:08:15.187 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:08:15.364 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T21:08:15.370 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:08:15.370 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:08:15.471 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:08:15.529 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:08:15.590 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:08:15.595 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:08:15.596 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:08:15.697 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:08:15.762 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:08:16.002 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:08:16.010 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:08:16.010 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:08:16.111 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:08:16.152 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:08:16.199 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:08:16.203 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:08:16.203 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:08:16.304 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:08:16.357 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:08:16.389 INFO:teuthology.orchestra.run.smithi026.stderr:nodeep-scrub is set 2023-09-23T21:08:16.410 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:08:16.412 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:08:16 smithi070 ceph-mon[168282]: pgmap v990: 109 pgs: 109 active+clean; 4.6 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 972 op/s 2023-09-23T21:08:16.412 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:08:16 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/2974639958' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:08:16.412 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:08:16 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:08:16.412 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:08:16 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:08:16.413 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:08:16.413 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:08:16.514 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:08:16.621 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:08:16 smithi026 ceph-mon[179688]: pgmap v990: 109 pgs: 109 active+clean; 4.6 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 972 op/s 2023-09-23T21:08:16.621 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:08:16 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/2974639958' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:08:16.621 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:08:16 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:08:16.621 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:08:16 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:08:16.621 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:08:16 smithi026 ceph-mon[175151]: pgmap v990: 109 pgs: 109 active+clean; 4.6 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 972 op/s 2023-09-23T21:08:16.621 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:08:16 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/2974639958' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:08:16.621 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:08:16 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:08:16.621 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:08:16 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:08:16.625 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0: {} 2023-09-23T21:08:16.642 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:08:16.671 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1: {} 2023-09-23T21:08:16.713 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2: {} 2023-09-23T21:08:16.747 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3: {} 2023-09-23T21:08:16.791 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4: {} 2023-09-23T21:08:16.829 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T21:08:16.870 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6: {} 2023-09-23T21:08:16.910 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:08:16.917 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:08:16.917 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:08:16.923 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T21:08:16.935 INFO:teuthology.orchestra.run.smithi026.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T21:08:16.935 INFO:teuthology.orchestra.run.smithi026.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T21:08:16.936 INFO:teuthology.orchestra.run.smithi026.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T21:08:16.936 INFO:teuthology.orchestra.run.smithi026.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T21:08:16.936 INFO:teuthology.orchestra.run.smithi026.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T21:08:16.936 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T21:08:16.936 INFO:teuthology.orchestra.run.smithi026.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T21:08:16.937 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T21:08:17.018 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:08:17.085 INFO:teuthology.orchestra.run.smithi026.stdout:{ 2023-09-23T21:08:17.085 INFO:teuthology.orchestra.run.smithi026.stdout: "ops": [], 2023-09-23T21:08:17.086 INFO:teuthology.orchestra.run.smithi026.stdout: "complaint_time": 30, 2023-09-23T21:08:17.086 INFO:teuthology.orchestra.run.smithi026.stdout: "num_blocked_ops": 0 2023-09-23T21:08:17.086 INFO:teuthology.orchestra.run.smithi026.stdout:} 2023-09-23T21:08:17.118 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:08:17.354 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:08:17 smithi026 ceph-mon[175151]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:08:17.354 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:08:17 smithi026 ceph-mon[175151]: failed to encode map e378 with expected crc 2023-09-23T21:08:17.354 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:08:17 smithi026 ceph-mon[175151]: failed to encode map e378 with expected crc 2023-09-23T21:08:17.354 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:08:17 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/2974639958' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T21:08:17.354 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:08:17 smithi026 ceph-mon[175151]: osdmap e378: 8 total, 8 up, 8 in 2023-09-23T21:08:17.354 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:08:17 smithi026 ceph-mon[175151]: failed to encode map e378 with expected crc 2023-09-23T21:08:17.354 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:08:17 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/2796519222' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:08:17.526 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:08:17.535 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:08:17.536 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:08:17.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:08:17 smithi070 ceph-mon[168282]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:08:17.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:08:17 smithi070 ceph-mon[168282]: failed to encode map e378 with expected crc 2023-09-23T21:08:17.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:08:17 smithi070 ceph-mon[168282]: failed to encode map e378 with expected crc 2023-09-23T21:08:17.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:08:17 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/2974639958' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T21:08:17.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:08:17 smithi070 ceph-mon[168282]: osdmap e378: 8 total, 8 up, 8 in 2023-09-23T21:08:17.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:08:17 smithi070 ceph-mon[168282]: failed to encode map e378 with expected crc 2023-09-23T21:08:17.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:08:17 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/2796519222' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:08:17.634 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:08:17 smithi026 ceph-mon[179688]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:08:17.634 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:08:17 smithi026 ceph-mon[179688]: failed to encode map e378 with expected crc 2023-09-23T21:08:17.634 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:08:17 smithi026 ceph-mon[179688]: failed to encode map e378 with expected crc 2023-09-23T21:08:17.634 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:08:17 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/2974639958' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T21:08:17.634 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:08:17 smithi026 ceph-mon[179688]: osdmap e378: 8 total, 8 up, 8 in 2023-09-23T21:08:17.634 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:08:17 smithi026 ceph-mon[179688]: failed to encode map e378 with expected crc 2023-09-23T21:08:17.634 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:08:17 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/2796519222' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:08:17.637 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:08:17.694 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:08:17.740 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:08:17.743 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:08:17.743 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:08:17.754 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_historic_ops 2023-09-23T21:08:17.844 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:08:17.916 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:08:17.971 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:08:17.984 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:08:17.984 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:08:18.085 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:08:18.140 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 6, 7, 4, 5] out_osds: [] dead_osds: [] live_osds: [0, 2, 3, 4, 5, 7, 6, 1] 2023-09-23T21:08:18.140 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T21:08:18.140 INFO:tasks.thrashosds.thrasher:fixing pg num pool .mgr 2023-09-23T21:08:18.140 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T21:08:18.178 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:08:18.263 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T21:08:18.276 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:08:18.276 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:08:18.377 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:08:18.467 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:08:18 smithi026 ceph-mon[175151]: pgmap v992: 109 pgs: 109 active+clean; 4.6 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 689 op/s 2023-09-23T21:08:18.467 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:08:18 smithi026 ceph-mon[179688]: pgmap v992: 109 pgs: 109 active+clean; 4.6 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 689 op/s 2023-09-23T21:08:18.502 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:08:18.589 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T21:08:18.601 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:08:18.602 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:08:18.621 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:08:18.703 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:08:18.765 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:08:18 smithi070 ceph-mon[168282]: pgmap v992: 109 pgs: 109 active+clean; 4.6 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 689 op/s 2023-09-23T21:08:18.767 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:08:18.813 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:08:18.816 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:08:18.816 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:08:18.917 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:08:18.946 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T21:08:18.959 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:08:19.014 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:08:19.019 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:08:19.019 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:08:19.120 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:08:19.166 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:08:19.224 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:08:19.230 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:08:19.231 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:08:19.331 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:08:19.396 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:08:19.613 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 21:08:19 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[164029]: ::ffff:172.21.15.70 - - [23/Sep/2023:21:08:19] "GET /metrics HTTP/1.1" 200 36295 "" "Prometheus/2.43.0" 2023-09-23T21:08:19.747 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T21:08:20.001 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:08:20.013 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:08:20.014 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:08:20.035 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:08:20.115 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:08:20.173 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:08:20.227 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:08:20.230 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:08:20.231 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:08:20.331 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:08:20.420 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:08:20.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:08:20 smithi070 ceph-mon[168282]: pgmap v993: 109 pgs: 109 active+clean; 4.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 833 op/s 2023-09-23T21:08:20.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:08:20 smithi026 ceph-mon[179688]: pgmap v993: 109 pgs: 109 active+clean; 4.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 833 op/s 2023-09-23T21:08:20.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:08:20 smithi026 ceph-mon[175151]: pgmap v993: 109 pgs: 109 active+clean; 4.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 833 op/s 2023-09-23T21:08:21.849 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T21:08:21.852 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:08:21.852 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:08:21.953 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:08:22.015 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:08:22.825 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:08:22.832 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:08:22.832 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:08:22.833 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:08:22 smithi026 ceph-mon[175151]: pgmap v994: 109 pgs: 109 active+clean; 4.4 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 735 op/s 2023-09-23T21:08:22.833 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:08:22 smithi026 ceph-mon[179688]: pgmap v994: 109 pgs: 109 active+clean; 4.4 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 735 op/s 2023-09-23T21:08:22.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:08:22 smithi070 ceph-mon[168282]: pgmap v994: 109 pgs: 109 active+clean; 4.4 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 735 op/s 2023-09-23T21:08:22.933 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:08:22.981 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:08:23.032 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:08:23.039 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:08:23.039 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:08:23.140 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:08:23.194 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:08:23.248 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:08:23.255 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:08:23.255 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:08:23.357 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:08:23.423 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:08:23.485 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:08:23.491 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:08:23.491 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:08:23.593 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:08:23.654 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:08:23.748 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:08:24.356 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T21:08:24.363 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:08:24.364 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:08:24.465 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:08:24.530 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:08:24.585 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:08:24.588 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:08:24.588 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:08:24.633 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:08:24 smithi026 ceph-mon[175151]: pgmap v995: 109 pgs: 109 active+clean; 4.3 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 796 op/s 2023-09-23T21:08:24.633 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:08:24 smithi026 ceph-mon[179688]: pgmap v995: 109 pgs: 109 active+clean; 4.3 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 796 op/s 2023-09-23T21:08:24.689 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:08:24.748 INFO:teuthology.orchestra.run.smithi026.stdout:{ 2023-09-23T21:08:24.749 INFO:teuthology.orchestra.run.smithi026.stdout: "size": 20, 2023-09-23T21:08:24.749 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 600, 2023-09-23T21:08:24.749 INFO:teuthology.orchestra.run.smithi026.stdout: "ops": [ 2023-09-23T21:08:24.749 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:24.749 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35469.0:197858 8.0 8:07cf2ed3:::benchmark_data_smithi026_301269_object87101:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e330)", 2023-09-23T21:08:24.749 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T21:05:01.326245+0000", 2023-09-23T21:08:24.749 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 203.41735749200001, 2023-09-23T21:08:24.750 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.97320722999999998, 2023-09-23T21:08:24.750 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:08:24.750 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:08:24.750 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T21:08:24.750 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35469", 2023-09-23T21:08:24.750 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/4041424623", 2023-09-23T21:08:24.750 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 197858 2023-09-23T21:08:24.750 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:24.750 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:08:24.751 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:24.751 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:08:24.751 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:01.326245+0000", 2023-09-23T21:08:24.751 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:08:24.751 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:24.751 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:24.751 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:08:24.751 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:01.326245+0000", 2023-09-23T21:08:24.751 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:08:24.751 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:24.751 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:24.752 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:08:24.752 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:01.326243+0000", 2023-09-23T21:08:24.752 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999981 2023-09-23T21:08:24.752 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:24.752 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:24.752 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:08:24.752 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:01.326250+0000", 2023-09-23T21:08:24.752 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.3139999999999998e-06 2023-09-23T21:08:24.752 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:24.752 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:24.752 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:08:24.752 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:01.326266+0000", 2023-09-23T21:08:24.753 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.5894999999999999e-05 2023-09-23T21:08:24.753 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:24.753 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:24.753 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:08:24.753 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:01.326274+0000", 2023-09-23T21:08:24.753 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.96e-06 2023-09-23T21:08:24.753 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:24.753 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:24.753 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:08:24.753 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:01.327831+0000", 2023-09-23T21:08:24.753 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0015563840000000001 2023-09-23T21:08:24.753 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:24.753 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:24.754 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T21:08:24.754 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:01.327837+0000", 2023-09-23T21:08:24.754 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.3559999999999998e-06 2023-09-23T21:08:24.754 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:24.754 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:24.754 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:08:24.754 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:02.290130+0000", 2023-09-23T21:08:24.754 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.96229288000000002 2023-09-23T21:08:24.754 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:24.754 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:24.754 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:08:24.754 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:02.290276+0000", 2023-09-23T21:08:24.755 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000145826 2023-09-23T21:08:24.755 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:24.755 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:24.755 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 1,4", 2023-09-23T21:08:24.755 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:02.290802+0000", 2023-09-23T21:08:24.755 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00052658900000000005 2023-09-23T21:08:24.755 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:24.755 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:24.755 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T21:08:24.755 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:02.295944+0000", 2023-09-23T21:08:24.755 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0051416539999999998 2023-09-23T21:08:24.755 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:24.756 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:24.756 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:08:24.756 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:02.299196+0000", 2023-09-23T21:08:24.756 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0032521749999999999 2023-09-23T21:08:24.756 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:24.756 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:24.756 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:08:24.756 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:02.299346+0000", 2023-09-23T21:08:24.756 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00014977600000000001 2023-09-23T21:08:24.756 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:24.756 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:24.756 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T21:08:24.756 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:02.299403+0000", 2023-09-23T21:08:24.757 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.7009999999999998e-05 2023-09-23T21:08:24.757 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:24.757 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:24.757 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T21:08:24.757 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:02.299452+0000", 2023-09-23T21:08:24.757 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.939e-05 2023-09-23T21:08:24.757 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:08:24.757 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:08:24.757 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:08:24.757 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:24.757 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:24.757 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35469.0:198574 8.0 8:0792e3f1:::benchmark_data_smithi026_301269_object87817:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e333)", 2023-09-23T21:08:24.758 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T21:05:04.351178+0000", 2023-09-23T21:08:24.758 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 200.39242452299999, 2023-09-23T21:08:24.758 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.090151624, 2023-09-23T21:08:24.758 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:08:24.758 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:08:24.758 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T21:08:24.758 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35469", 2023-09-23T21:08:24.758 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/4041424623", 2023-09-23T21:08:24.758 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 198574 2023-09-23T21:08:24.758 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:24.758 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:08:24.758 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:24.758 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:08:24.759 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:04.351178+0000", 2023-09-23T21:08:24.759 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:08:24.759 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:24.759 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:24.759 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:08:24.759 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:04.351178+0000", 2023-09-23T21:08:24.759 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:08:24.759 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:24.759 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:24.759 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:08:24.759 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:04.351176+0000", 2023-09-23T21:08:24.759 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T21:08:24.759 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:24.760 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:24.760 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:08:24.760 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:04.351184+0000", 2023-09-23T21:08:24.760 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.6020000000000002e-06 2023-09-23T21:08:24.760 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:24.760 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:24.760 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:08:24.760 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:04.351203+0000", 2023-09-23T21:08:24.760 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.9086999999999999e-05 2023-09-23T21:08:24.760 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:24.760 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:24.760 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:08:24.761 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:04.351213+0000", 2023-09-23T21:08:24.761 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0036e-05 2023-09-23T21:08:24.761 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:24.761 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:24.761 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:08:24.761 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:04.351287+0000", 2023-09-23T21:08:24.761 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.3664000000000005e-05 2023-09-23T21:08:24.761 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:24.761 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:24.761 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T21:08:24.761 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:04.351300+0000", 2023-09-23T21:08:24.761 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.3607e-05 2023-09-23T21:08:24.761 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:24.762 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:24.762 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:08:24.762 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:05.427723+0000", 2023-09-23T21:08:24.762 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0764224410000001 2023-09-23T21:08:24.762 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:24.762 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:24.762 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:08:24.762 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:05.427826+0000", 2023-09-23T21:08:24.762 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000102997 2023-09-23T21:08:24.762 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:24.762 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:24.762 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 1,4,5", 2023-09-23T21:08:24.763 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:05.428282+0000", 2023-09-23T21:08:24.763 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00045637800000000001 2023-09-23T21:08:24.763 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:24.763 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:24.763 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:08:24.763 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:05.431215+0000", 2023-09-23T21:08:24.763 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0029325509999999998 2023-09-23T21:08:24.763 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:24.763 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:24.763 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:08:24.763 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:05.440771+0000", 2023-09-23T21:08:24.763 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0095558459999999998 2023-09-23T21:08:24.764 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:24.764 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:24.764 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:08:24.764 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:05.440850+0000", 2023-09-23T21:08:24.764 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.9185999999999997e-05 2023-09-23T21:08:24.764 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:24.764 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:24.764 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T21:08:24.764 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:05.441138+0000", 2023-09-23T21:08:24.764 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00028843100000000002 2023-09-23T21:08:24.764 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:24.764 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:24.765 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T21:08:24.765 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:05.441198+0000", 2023-09-23T21:08:24.765 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.0247000000000001e-05 2023-09-23T21:08:24.765 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:24.765 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:24.765 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T21:08:24.765 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:05.441330+0000", 2023-09-23T21:08:24.765 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00013121300000000001 2023-09-23T21:08:24.765 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:08:24.765 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:08:24.765 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:08:24.765 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:24.766 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:24.766 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35469.0:198642 8.0 8:060a72fa:::benchmark_data_smithi026_301269_object87885:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e333)", 2023-09-23T21:08:24.766 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T21:05:04.480208+0000", 2023-09-23T21:08:24.766 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 200.263394527, 2023-09-23T21:08:24.766 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.96130475699999995, 2023-09-23T21:08:24.766 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:08:24.766 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:08:24.766 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T21:08:24.766 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35469", 2023-09-23T21:08:24.766 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/4041424623", 2023-09-23T21:08:24.766 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 198642 2023-09-23T21:08:24.766 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:24.767 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:08:24.767 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:24.767 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:08:24.767 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:04.480208+0000", 2023-09-23T21:08:24.767 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:08:24.767 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:24.767 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:24.767 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:08:24.767 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:04.480208+0000", 2023-09-23T21:08:24.767 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:08:24.767 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:24.767 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:24.767 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:08:24.768 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:04.480207+0000", 2023-09-23T21:08:24.768 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T21:08:24.768 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:24.768 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:24.768 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:08:24.768 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:04.480213+0000", 2023-09-23T21:08:24.768 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.4459999999999998e-06 2023-09-23T21:08:24.768 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:24.768 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:24.768 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:08:24.768 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:04.480226+0000", 2023-09-23T21:08:24.768 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.3297e-05 2023-09-23T21:08:24.768 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:24.769 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:24.769 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:08:24.769 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:04.480233+0000", 2023-09-23T21:08:24.769 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.3879999999999997e-06 2023-09-23T21:08:24.769 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:24.769 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:24.769 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:08:24.769 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:04.480296+0000", 2023-09-23T21:08:24.769 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.3238000000000007e-05 2023-09-23T21:08:24.769 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:24.769 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:24.769 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T21:08:24.770 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:04.480311+0000", 2023-09-23T21:08:24.770 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.4707e-05 2023-09-23T21:08:24.770 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:24.770 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:24.770 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:08:24.770 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:05.431242+0000", 2023-09-23T21:08:24.770 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.95093115900000003 2023-09-23T21:08:24.770 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:24.770 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:24.770 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:08:24.770 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:05.431367+0000", 2023-09-23T21:08:24.770 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00012549600000000001 2023-09-23T21:08:24.770 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:24.771 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:24.771 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 1,4,5", 2023-09-23T21:08:24.771 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:05.431765+0000", 2023-09-23T21:08:24.771 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00039724900000000002 2023-09-23T21:08:24.771 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:24.771 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:24.771 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:08:24.771 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:05.440810+0000", 2023-09-23T21:08:24.771 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0090457639999999995 2023-09-23T21:08:24.771 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:24.771 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:24.771 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:08:24.771 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:05.440888+0000", 2023-09-23T21:08:24.772 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.7342999999999995e-05 2023-09-23T21:08:24.772 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:24.772 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:24.772 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:08:24.772 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:05.440922+0000", 2023-09-23T21:08:24.772 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.4400999999999998e-05 2023-09-23T21:08:24.772 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:24.772 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:24.772 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T21:08:24.772 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:05.441344+0000", 2023-09-23T21:08:24.772 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00042175699999999998 2023-09-23T21:08:24.772 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:24.772 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:24.773 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T21:08:24.773 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:05.441396+0000", 2023-09-23T21:08:24.773 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.2228999999999997e-05 2023-09-23T21:08:24.773 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:24.773 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:24.773 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T21:08:24.773 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:05.441513+0000", 2023-09-23T21:08:24.773 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00011658 2023-09-23T21:08:24.773 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:08:24.773 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:08:24.773 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:08:24.773 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:24.773 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:24.774 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35469.0:199464 8.0 8:06def8ea:::benchmark_data_smithi026_301269_object88707:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e335)", 2023-09-23T21:08:24.774 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T21:05:06.321815+0000", 2023-09-23T21:08:24.774 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 198.42178736700001, 2023-09-23T21:08:24.774 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0038432799999999, 2023-09-23T21:08:24.774 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:08:24.774 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:08:24.774 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T21:08:24.774 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35469", 2023-09-23T21:08:24.774 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/4041424623", 2023-09-23T21:08:24.774 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 199464 2023-09-23T21:08:24.774 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:24.775 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:08:24.775 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:24.775 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:08:24.775 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:06.321815+0000", 2023-09-23T21:08:24.775 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:08:24.775 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:24.775 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:24.775 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:08:24.775 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:06.321815+0000", 2023-09-23T21:08:24.775 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:08:24.775 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:24.775 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:24.775 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:08:24.775 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:06.321814+0000", 2023-09-23T21:08:24.776 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T21:08:24.776 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:24.776 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:24.776 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:08:24.776 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:06.321820+0000", 2023-09-23T21:08:24.776 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.7340000000000002e-06 2023-09-23T21:08:24.776 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:24.776 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:24.776 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:08:24.776 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:06.321839+0000", 2023-09-23T21:08:24.776 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.8113000000000001e-05 2023-09-23T21:08:24.776 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:24.777 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:24.777 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:08:24.777 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:06.321848+0000", 2023-09-23T21:08:24.777 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.397e-06 2023-09-23T21:08:24.777 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:24.777 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:24.777 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:08:24.777 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:06.321904+0000", 2023-09-23T21:08:24.777 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.6202000000000002e-05 2023-09-23T21:08:24.777 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:24.777 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:24.777 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T21:08:24.777 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:06.321916+0000", 2023-09-23T21:08:24.778 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.2160999999999999e-05 2023-09-23T21:08:24.778 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:24.778 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:24.778 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:08:24.778 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:07.318431+0000", 2023-09-23T21:08:24.778 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.99651437099999995 2023-09-23T21:08:24.778 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:24.778 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:24.778 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:08:24.778 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:07.318584+0000", 2023-09-23T21:08:24.778 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00015343099999999999 2023-09-23T21:08:24.778 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:24.778 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:24.779 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 1,5", 2023-09-23T21:08:24.779 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:07.319038+0000", 2023-09-23T21:08:24.779 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00045385800000000002 2023-09-23T21:08:24.779 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:24.779 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:24.779 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:08:24.779 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:07.325552+0000", 2023-09-23T21:08:24.779 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0065137479999999998 2023-09-23T21:08:24.779 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:24.779 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:24.779 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T21:08:24.779 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:07.325563+0000", 2023-09-23T21:08:24.779 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1114000000000001e-05 2023-09-23T21:08:24.780 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:24.780 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:24.780 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:08:24.780 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:07.325588+0000", 2023-09-23T21:08:24.780 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.5534e-05 2023-09-23T21:08:24.780 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:24.780 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:24.780 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T21:08:24.780 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:07.325617+0000", 2023-09-23T21:08:24.780 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.8447000000000001e-05 2023-09-23T21:08:24.780 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:24.780 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:24.781 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T21:08:24.781 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:07.325658+0000", 2023-09-23T21:08:24.781 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.1616000000000003e-05 2023-09-23T21:08:24.781 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:08:24.781 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:08:24.781 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:08:24.781 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:24.781 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:24.781 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35469.0:199475 8.0 8:027c44f4:::benchmark_data_smithi026_301269_object88718:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e335)", 2023-09-23T21:08:24.781 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T21:05:06.334543+0000", 2023-09-23T21:08:24.781 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 198.40905994900001, 2023-09-23T21:08:24.781 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.99135582600000005, 2023-09-23T21:08:24.781 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:08:24.782 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:08:24.782 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T21:08:24.782 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35469", 2023-09-23T21:08:24.782 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/4041424623", 2023-09-23T21:08:24.782 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 199475 2023-09-23T21:08:24.782 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:24.782 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:08:24.782 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:24.782 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:08:24.782 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:06.334543+0000", 2023-09-23T21:08:24.783 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:08:24.783 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:24.783 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:24.783 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:08:24.783 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:06.334543+0000", 2023-09-23T21:08:24.783 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:08:24.783 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:24.783 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:24.783 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:08:24.783 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:06.334542+0000", 2023-09-23T21:08:24.783 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T21:08:24.783 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:24.784 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:24.784 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:08:24.784 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:06.334545+0000", 2023-09-23T21:08:24.784 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.4770000000000002e-06 2023-09-23T21:08:24.784 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:24.784 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:24.784 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:08:24.784 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:06.334551+0000", 2023-09-23T21:08:24.784 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.0410000000000003e-06 2023-09-23T21:08:24.784 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:24.784 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:24.784 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:08:24.784 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:06.334555+0000", 2023-09-23T21:08:24.785 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.5190000000000003e-06 2023-09-23T21:08:24.785 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:24.785 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:24.785 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:08:24.785 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:06.334605+0000", 2023-09-23T21:08:24.785 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.9262999999999999e-05 2023-09-23T21:08:24.785 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:24.785 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:24.785 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T21:08:24.785 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:06.334615+0000", 2023-09-23T21:08:24.785 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0974e-05 2023-09-23T21:08:24.785 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:24.785 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:24.786 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:08:24.786 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:07.321324+0000", 2023-09-23T21:08:24.786 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.98670861399999998 2023-09-23T21:08:24.786 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:24.786 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:24.786 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:08:24.786 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:07.321440+0000", 2023-09-23T21:08:24.786 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000116066 2023-09-23T21:08:24.786 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:24.786 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:24.786 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 1,5", 2023-09-23T21:08:24.786 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:07.321850+0000", 2023-09-23T21:08:24.787 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00040972799999999999 2023-09-23T21:08:24.787 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:24.787 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:24.787 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:08:24.787 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:07.325791+0000", 2023-09-23T21:08:24.787 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0039407289999999996 2023-09-23T21:08:24.787 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:24.787 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:24.787 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:08:24.787 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:07.325825+0000", 2023-09-23T21:08:24.787 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.3895e-05 2023-09-23T21:08:24.787 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:24.787 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:24.788 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T21:08:24.788 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:07.325831+0000", 2023-09-23T21:08:24.788 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.2809999999999997e-06 2023-09-23T21:08:24.788 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:24.788 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:24.788 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T21:08:24.788 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:07.325867+0000", 2023-09-23T21:08:24.788 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.6591999999999998e-05 2023-09-23T21:08:24.788 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:24.788 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:24.788 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T21:08:24.788 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:07.325898+0000", 2023-09-23T21:08:24.788 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.1015e-05 2023-09-23T21:08:24.789 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:08:24.789 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:08:24.789 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:08:24.789 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:24.789 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:24.789 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35469.0:199489 8.0 8:0578354c:::benchmark_data_smithi026_301269_object88732:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e335)", 2023-09-23T21:08:24.789 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T21:05:06.349565+0000", 2023-09-23T21:08:24.789 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 198.39403756999999, 2023-09-23T21:08:24.789 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.97648712400000004, 2023-09-23T21:08:24.789 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:08:24.789 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:08:24.789 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T21:08:24.789 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35469", 2023-09-23T21:08:24.790 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/4041424623", 2023-09-23T21:08:24.790 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 199489 2023-09-23T21:08:24.790 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:24.790 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:08:24.790 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:24.790 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:08:24.790 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:06.349565+0000", 2023-09-23T21:08:24.790 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:08:24.790 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:24.790 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:24.790 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:08:24.790 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:06.349565+0000", 2023-09-23T21:08:24.790 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:08:24.791 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:24.791 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:24.791 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:08:24.791 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:06.349564+0000", 2023-09-23T21:08:24.791 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T21:08:24.791 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:24.791 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:24.791 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:08:24.791 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:06.349568+0000", 2023-09-23T21:08:24.791 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.4359999999999998e-06 2023-09-23T21:08:24.791 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:24.791 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:24.791 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:08:24.791 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:06.349576+0000", 2023-09-23T21:08:24.792 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.7009999999999996e-06 2023-09-23T21:08:24.792 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:24.792 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:24.792 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:08:24.792 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:06.349581+0000", 2023-09-23T21:08:24.792 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.5609999999999998e-06 2023-09-23T21:08:24.792 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:24.792 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:24.792 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:08:24.792 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:06.349622+0000", 2023-09-23T21:08:24.792 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.0432000000000002e-05 2023-09-23T21:08:24.792 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:24.792 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:24.792 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T21:08:24.793 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:06.349630+0000", 2023-09-23T21:08:24.793 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.8720000000000008e-06 2023-09-23T21:08:24.793 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:24.793 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:24.793 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:08:24.793 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:07.323668+0000", 2023-09-23T21:08:24.793 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.97403801599999995 2023-09-23T21:08:24.793 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:24.793 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:24.793 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:08:24.793 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:07.323750+0000", 2023-09-23T21:08:24.793 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.1191000000000002e-05 2023-09-23T21:08:24.793 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:24.793 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:24.794 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 1,5", 2023-09-23T21:08:24.794 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:07.323985+0000", 2023-09-23T21:08:24.794 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000235467 2023-09-23T21:08:24.794 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:24.794 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:24.794 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T21:08:24.794 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:07.325910+0000", 2023-09-23T21:08:24.794 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.001925316 2023-09-23T21:08:24.794 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:24.794 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:24.794 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:08:24.794 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:07.325960+0000", 2023-09-23T21:08:24.795 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.9259999999999999e-05 2023-09-23T21:08:24.795 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:24.795 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:24.795 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:08:24.795 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:07.326005+0000", 2023-09-23T21:08:24.795 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.5287000000000002e-05 2023-09-23T21:08:24.795 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:24.795 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:24.795 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T21:08:24.795 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:07.326032+0000", 2023-09-23T21:08:24.795 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.7529e-05 2023-09-23T21:08:24.795 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:24.795 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:24.795 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T21:08:24.796 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:07.326052+0000", 2023-09-23T21:08:24.796 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.9585999999999998e-05 2023-09-23T21:08:24.796 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:08:24.796 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:08:24.796 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:08:24.796 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:24.796 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:24.796 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35469.0:199506 8.0 8:00e9106b:::benchmark_data_smithi026_301269_object88749:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e335)", 2023-09-23T21:08:24.796 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T21:05:06.366295+0000", 2023-09-23T21:08:24.796 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 198.37730742100001, 2023-09-23T21:08:24.796 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.96812640900000002, 2023-09-23T21:08:24.796 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:08:24.796 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:08:24.797 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T21:08:24.797 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35469", 2023-09-23T21:08:24.797 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/4041424623", 2023-09-23T21:08:24.797 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 199506 2023-09-23T21:08:24.797 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:24.797 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:08:24.797 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:24.797 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:08:24.797 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:06.366295+0000", 2023-09-23T21:08:24.797 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:08:24.797 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:24.797 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:24.797 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:08:24.797 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:06.366295+0000", 2023-09-23T21:08:24.798 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:08:24.798 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:24.798 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:24.798 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:08:24.798 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:06.366294+0000", 2023-09-23T21:08:24.798 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T21:08:24.798 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:24.798 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:24.798 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:08:24.798 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:06.366300+0000", 2023-09-23T21:08:24.798 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.6390000000000003e-06 2023-09-23T21:08:24.798 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:24.798 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:24.799 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:08:24.799 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:06.366313+0000", 2023-09-23T21:08:24.799 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.2245999999999999e-05 2023-09-23T21:08:24.799 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:24.799 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:24.799 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:08:24.799 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:06.366320+0000", 2023-09-23T21:08:24.799 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.9940000000000003e-06 2023-09-23T21:08:24.799 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:24.799 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:24.799 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:08:24.799 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:06.366369+0000", 2023-09-23T21:08:24.799 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.8877e-05 2023-09-23T21:08:24.799 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:24.800 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:24.800 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T21:08:24.800 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:06.366382+0000", 2023-09-23T21:08:24.800 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.3199e-05 2023-09-23T21:08:24.800 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:24.800 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:24.800 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:08:24.800 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:07.326095+0000", 2023-09-23T21:08:24.800 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.95971310600000004 2023-09-23T21:08:24.800 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:24.800 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:24.800 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:08:24.800 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:07.326207+0000", 2023-09-23T21:08:24.800 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000112527 2023-09-23T21:08:24.801 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:24.801 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:24.801 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 1,5", 2023-09-23T21:08:24.801 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:07.326489+0000", 2023-09-23T21:08:24.801 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00028145299999999998 2023-09-23T21:08:24.801 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:24.801 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:24.801 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:08:24.801 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:07.332147+0000", 2023-09-23T21:08:24.801 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0056581039999999997 2023-09-23T21:08:24.801 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:24.801 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:24.801 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T21:08:24.802 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:07.332159+0000", 2023-09-23T21:08:24.802 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.2303999999999999e-05 2023-09-23T21:08:24.802 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:24.802 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:24.802 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:08:24.802 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:07.334367+0000", 2023-09-23T21:08:24.802 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0022081470000000001 2023-09-23T21:08:24.802 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:24.802 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:24.802 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T21:08:24.802 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:07.334397+0000", 2023-09-23T21:08:24.802 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.9995e-05 2023-09-23T21:08:24.802 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:24.803 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:24.803 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T21:08:24.803 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:07.334422+0000", 2023-09-23T21:08:24.803 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.4148999999999999e-05 2023-09-23T21:08:24.803 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:08:24.803 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:08:24.803 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:08:24.803 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:24.803 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:24.803 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35469.0:199513 8.0 8:05febd72:::benchmark_data_smithi026_301269_object88756:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e335)", 2023-09-23T21:08:24.803 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T21:05:06.367555+0000", 2023-09-23T21:08:24.803 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 198.37604722699999, 2023-09-23T21:08:24.803 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.96702458400000002, 2023-09-23T21:08:24.803 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:08:24.804 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:08:24.804 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T21:08:24.804 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35469", 2023-09-23T21:08:24.804 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/4041424623", 2023-09-23T21:08:24.804 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 199513 2023-09-23T21:08:24.804 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:24.804 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:08:24.804 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:24.804 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:08:24.804 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:06.367555+0000", 2023-09-23T21:08:24.804 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:08:24.804 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:24.804 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:24.804 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:08:24.805 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:06.367555+0000", 2023-09-23T21:08:24.805 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:08:24.805 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:24.805 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:24.805 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:08:24.805 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:06.367554+0000", 2023-09-23T21:08:24.805 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.999999 2023-09-23T21:08:24.805 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:24.805 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:24.805 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:08:24.805 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:06.367561+0000", 2023-09-23T21:08:24.805 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.5690000000000001e-06 2023-09-23T21:08:24.805 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:24.805 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:24.806 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:08:24.806 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:06.367573+0000", 2023-09-23T21:08:24.806 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1768e-05 2023-09-23T21:08:24.806 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:24.806 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:24.806 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:08:24.806 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:06.367580+0000", 2023-09-23T21:08:24.806 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.4699999999999996e-06 2023-09-23T21:08:24.806 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:24.806 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:24.806 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:08:24.806 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:06.367625+0000", 2023-09-23T21:08:24.806 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.4694999999999998e-05 2023-09-23T21:08:24.807 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:24.807 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:24.807 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T21:08:24.807 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:06.367633+0000", 2023-09-23T21:08:24.807 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.1510000000000004e-06 2023-09-23T21:08:24.807 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:24.807 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:24.807 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:08:24.807 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:07.328200+0000", 2023-09-23T21:08:24.807 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.96056760200000002 2023-09-23T21:08:24.807 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:24.807 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:24.808 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:08:24.808 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:07.328331+0000", 2023-09-23T21:08:24.808 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00013024 2023-09-23T21:08:24.808 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:24.808 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:24.808 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 1,5", 2023-09-23T21:08:24.808 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:07.328639+0000", 2023-09-23T21:08:24.808 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00030803699999999999 2023-09-23T21:08:24.808 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:24.808 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:24.808 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:08:24.808 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:07.332224+0000", 2023-09-23T21:08:24.809 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.003585146 2023-09-23T21:08:24.809 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:24.809 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:24.809 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T21:08:24.809 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:07.334329+0000", 2023-09-23T21:08:24.809 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0021056040000000001 2023-09-23T21:08:24.809 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:24.809 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:24.809 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:08:24.809 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:07.334489+0000", 2023-09-23T21:08:24.809 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000159357 2023-09-23T21:08:24.809 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:24.809 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:24.810 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T21:08:24.810 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:07.334535+0000", 2023-09-23T21:08:24.810 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.5717000000000003e-05 2023-09-23T21:08:24.810 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:24.810 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:24.810 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T21:08:24.810 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:07.334580+0000", 2023-09-23T21:08:24.810 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.5348e-05 2023-09-23T21:08:24.810 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:08:24.810 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:08:24.810 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:08:24.810 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:24.811 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:24.811 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35469.0:199548 8.0 8:0318cdab:::benchmark_data_smithi026_301269_object88791:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e335)", 2023-09-23T21:08:24.811 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T21:05:06.414968+0000", 2023-09-23T21:08:24.811 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 198.32863431600001, 2023-09-23T21:08:24.811 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.92483092600000005, 2023-09-23T21:08:24.811 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:08:24.811 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:08:24.811 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T21:08:24.811 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35469", 2023-09-23T21:08:24.811 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/4041424623", 2023-09-23T21:08:24.811 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 199548 2023-09-23T21:08:24.811 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:24.811 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:08:24.812 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:24.812 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:08:24.812 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:06.414968+0000", 2023-09-23T21:08:24.812 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:08:24.812 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:24.812 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:24.812 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:08:24.812 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:06.414968+0000", 2023-09-23T21:08:24.812 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:08:24.812 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:24.812 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:24.812 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:08:24.812 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:06.414967+0000", 2023-09-23T21:08:24.813 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.999999 2023-09-23T21:08:24.813 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:24.813 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:24.813 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:08:24.813 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:06.414973+0000", 2023-09-23T21:08:24.813 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.6740000000000002e-06 2023-09-23T21:08:24.813 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:24.813 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:24.813 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:08:24.813 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:06.414984+0000", 2023-09-23T21:08:24.813 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0679000000000001e-05 2023-09-23T21:08:24.813 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:24.813 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:24.814 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:08:24.814 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:06.414992+0000", 2023-09-23T21:08:24.814 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.3000000000000002e-06 2023-09-23T21:08:24.814 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:24.814 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:24.814 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:08:24.814 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:06.415045+0000", 2023-09-23T21:08:24.814 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.2883999999999997e-05 2023-09-23T21:08:24.814 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:24.814 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:24.814 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T21:08:24.814 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:06.415053+0000", 2023-09-23T21:08:24.814 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.6659999999999997e-06 2023-09-23T21:08:24.815 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:24.815 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:24.815 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:08:24.815 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:07.330213+0000", 2023-09-23T21:08:24.815 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.915159693 2023-09-23T21:08:24.815 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:24.815 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:24.815 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:08:24.815 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:07.330333+0000", 2023-09-23T21:08:24.815 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000119911 2023-09-23T21:08:24.815 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:24.815 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:24.816 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 1,5", 2023-09-23T21:08:24.816 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:07.330655+0000", 2023-09-23T21:08:24.816 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00032238699999999999 2023-09-23T21:08:24.816 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:24.816 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:24.816 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T21:08:24.816 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:07.334593+0000", 2023-09-23T21:08:24.816 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0039378399999999997 2023-09-23T21:08:24.816 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:24.816 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:24.816 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:08:24.816 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:07.334643+0000", 2023-09-23T21:08:24.816 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.9845000000000001e-05 2023-09-23T21:08:24.817 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:24.817 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:24.817 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:08:24.817 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:07.339701+0000", 2023-09-23T21:08:24.817 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0050582370000000001 2023-09-23T21:08:24.817 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:24.817 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:24.817 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T21:08:24.817 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:07.339751+0000", 2023-09-23T21:08:24.817 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.9545000000000001e-05 2023-09-23T21:08:24.817 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:24.817 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:24.817 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T21:08:24.818 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:07.339799+0000", 2023-09-23T21:08:24.818 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.8191e-05 2023-09-23T21:08:24.818 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:08:24.818 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:08:24.818 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:08:24.818 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:24.818 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:24.818 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35469.0:212408 8.b 8:da7047a3:::benchmark_data_smithi026_301269_object101651:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e338)", 2023-09-23T21:08:24.818 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T21:05:21.261714+0000", 2023-09-23T21:08:24.818 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 183.48188871400001, 2023-09-23T21:08:24.818 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.139567666, 2023-09-23T21:08:24.818 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:08:24.819 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "reached pg", 2023-09-23T21:08:24.819 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T21:08:24.819 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35469", 2023-09-23T21:08:24.819 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/4041424623", 2023-09-23T21:08:24.819 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 212408 2023-09-23T21:08:24.819 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:24.819 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:08:24.819 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:24.819 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:08:24.819 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:21.261714+0000", 2023-09-23T21:08:24.819 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:08:24.819 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:24.821 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:24.821 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:08:24.821 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:21.261714+0000", 2023-09-23T21:08:24.821 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:08:24.821 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:24.821 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:24.821 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:08:24.821 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:21.261712+0000", 2023-09-23T21:08:24.821 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T21:08:24.821 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:24.822 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:24.822 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:08:24.822 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:21.261720+0000", 2023-09-23T21:08:24.822 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.3679999999999996e-06 2023-09-23T21:08:24.822 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:24.822 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:24.822 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:08:24.822 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:21.261733+0000", 2023-09-23T21:08:24.822 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.3123e-05 2023-09-23T21:08:24.822 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:24.822 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:24.822 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:08:24.823 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:21.261740+0000", 2023-09-23T21:08:24.823 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.5900000000000002e-06 2023-09-23T21:08:24.823 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:24.823 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:24.823 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:08:24.823 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:21.261789+0000", 2023-09-23T21:08:24.823 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.8819000000000001e-05 2023-09-23T21:08:24.823 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:24.823 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:24.823 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:08:24.823 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:21.261929+0000", 2023-09-23T21:08:24.823 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00013985199999999999 2023-09-23T21:08:24.823 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:24.823 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:24.824 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 1,4", 2023-09-23T21:08:24.824 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:21.262063+0000", 2023-09-23T21:08:24.824 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00013385099999999999 2023-09-23T21:08:24.824 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:24.824 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:24.824 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T21:08:24.824 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:21.265708+0000", 2023-09-23T21:08:24.824 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0036452709999999998 2023-09-23T21:08:24.824 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:24.824 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:24.824 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:08:24.824 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:21.294099+0000", 2023-09-23T21:08:24.824 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.028391385000000002 2023-09-23T21:08:24.825 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:24.825 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:24.825 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T21:08:24.825 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:21.294109+0000", 2023-09-23T21:08:24.825 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.7019999999999996e-06 2023-09-23T21:08:24.825 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:24.825 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:24.825 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:08:24.825 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:22.401207+0000", 2023-09-23T21:08:24.825 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.107097679 2023-09-23T21:08:24.825 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:24.825 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:24.825 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T21:08:24.826 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:22.401281+0000", 2023-09-23T21:08:24.826 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.4621000000000003e-05 2023-09-23T21:08:24.826 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:08:24.826 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:08:24.826 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:08:24.826 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:24.826 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:24.826 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35469.0:212409 8.b 8:d3389620:::benchmark_data_smithi026_301269_object101652:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e338)", 2023-09-23T21:08:24.826 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T21:05:21.265789+0000", 2023-09-23T21:08:24.826 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 183.47781365, 2023-09-23T21:08:24.826 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.135914334, 2023-09-23T21:08:24.826 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:08:24.826 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "reached pg", 2023-09-23T21:08:24.827 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T21:08:24.827 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35469", 2023-09-23T21:08:24.827 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/4041424623", 2023-09-23T21:08:24.827 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 212409 2023-09-23T21:08:24.827 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:24.827 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:08:24.827 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:24.827 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:08:24.827 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:21.265789+0000", 2023-09-23T21:08:24.827 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:08:24.827 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:24.827 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:24.827 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:08:24.828 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:21.265789+0000", 2023-09-23T21:08:24.828 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:08:24.828 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:24.828 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:24.828 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:08:24.828 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:21.265787+0000", 2023-09-23T21:08:24.828 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T21:08:24.828 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:24.828 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:24.828 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:08:24.828 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:21.265794+0000", 2023-09-23T21:08:24.828 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.7800000000000003e-06 2023-09-23T21:08:24.828 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:24.829 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:24.829 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:08:24.829 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:21.265805+0000", 2023-09-23T21:08:24.829 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1022999999999999e-05 2023-09-23T21:08:24.829 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:24.829 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:24.829 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:08:24.829 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:21.265813+0000", 2023-09-23T21:08:24.829 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.5850000000000002e-06 2023-09-23T21:08:24.829 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:24.829 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:24.829 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:08:24.830 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:21.265870+0000", 2023-09-23T21:08:24.830 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.6847e-05 2023-09-23T21:08:24.830 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:24.830 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:24.830 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:08:24.830 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:21.266017+0000", 2023-09-23T21:08:24.830 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00014706100000000001 2023-09-23T21:08:24.830 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:24.830 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:24.830 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 1,4", 2023-09-23T21:08:24.830 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:21.266158+0000", 2023-09-23T21:08:24.830 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00014158500000000001 2023-09-23T21:08:24.831 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:24.831 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:24.831 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:08:24.831 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:21.294210+0000", 2023-09-23T21:08:24.831 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.028051336999999999 2023-09-23T21:08:24.831 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:24.831 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:24.831 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T21:08:24.831 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:21.294219+0000", 2023-09-23T21:08:24.831 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.3619999999999995e-06 2023-09-23T21:08:24.831 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:24.831 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:24.831 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:08:24.831 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:22.401648+0000", 2023-09-23T21:08:24.832 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1074286499999999 2023-09-23T21:08:24.832 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:24.832 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:24.832 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T21:08:24.832 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:22.401703+0000", 2023-09-23T21:08:24.832 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.5483999999999999e-05 2023-09-23T21:08:24.832 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:08:24.832 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:08:24.832 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:08:24.832 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:24.832 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:24.832 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35469.0:212410 8.b 8:d3aaf4c5:::benchmark_data_smithi026_301269_object101653:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e338)", 2023-09-23T21:08:24.832 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T21:05:21.265827+0000", 2023-09-23T21:08:24.832 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 183.47777590999999, 2023-09-23T21:08:24.833 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.157048445, 2023-09-23T21:08:24.833 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:08:24.833 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "reached pg", 2023-09-23T21:08:24.833 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T21:08:24.833 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35469", 2023-09-23T21:08:24.833 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/4041424623", 2023-09-23T21:08:24.833 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 212410 2023-09-23T21:08:24.833 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:24.833 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:08:24.833 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:24.833 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:08:24.833 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:21.265827+0000", 2023-09-23T21:08:24.833 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:08:24.834 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:24.834 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:24.834 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:08:24.834 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:21.265827+0000", 2023-09-23T21:08:24.834 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:08:24.834 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:24.834 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:24.834 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:08:24.834 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:21.265826+0000", 2023-09-23T21:08:24.834 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T21:08:24.834 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:24.834 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:24.834 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:08:24.835 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:21.265829+0000", 2023-09-23T21:08:24.835 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.2849999999999999e-06 2023-09-23T21:08:24.835 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:24.835 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:24.835 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:08:24.835 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:21.265836+0000", 2023-09-23T21:08:24.835 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.9619999999999996e-06 2023-09-23T21:08:24.835 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:24.835 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:24.835 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:08:24.835 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:21.265842+0000", 2023-09-23T21:08:24.835 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.9309999999999996e-06 2023-09-23T21:08:24.835 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:24.836 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:24.836 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:08:24.836 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:21.278647+0000", 2023-09-23T21:08:24.836 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.012804489 2023-09-23T21:08:24.836 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:24.836 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:24.836 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:08:24.836 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:21.278795+0000", 2023-09-23T21:08:24.836 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000148484 2023-09-23T21:08:24.836 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:24.836 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:24.836 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 1,4", 2023-09-23T21:08:24.836 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:21.278934+0000", 2023-09-23T21:08:24.836 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00013905100000000001 2023-09-23T21:08:24.837 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:24.837 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:24.837 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:08:24.837 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:21.294252+0000", 2023-09-23T21:08:24.837 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.015317894 2023-09-23T21:08:24.837 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:24.837 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:24.837 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T21:08:24.837 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:21.294260+0000", 2023-09-23T21:08:24.837 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.0989999999999999e-06 2023-09-23T21:08:24.837 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:24.837 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:24.837 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:08:24.838 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:22.401731+0000", 2023-09-23T21:08:24.838 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1074706940000001 2023-09-23T21:08:24.838 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:24.838 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:24.838 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for missing object", 2023-09-23T21:08:24.838 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:22.401831+0000", 2023-09-23T21:08:24.838 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00010001999999999999 2023-09-23T21:08:24.838 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:24.838 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:24.838 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:08:24.838 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:22.422801+0000", 2023-09-23T21:08:24.838 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.020970098999999999 2023-09-23T21:08:24.838 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:24.839 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:24.839 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T21:08:24.839 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:22.422875+0000", 2023-09-23T21:08:24.839 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.3929000000000003e-05 2023-09-23T21:08:24.839 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:08:24.839 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:08:24.839 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:08:24.839 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:24.839 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:24.839 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35469.0:212419 8.f 8:fd805684:::benchmark_data_smithi026_301269_object101662:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e339)", 2023-09-23T21:08:24.839 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T21:05:21.287539+0000", 2023-09-23T21:08:24.839 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 183.45606361500001, 2023-09-23T21:08:24.839 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0681341200000001, 2023-09-23T21:08:24.839 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:08:24.840 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:08:24.840 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T21:08:24.840 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35469", 2023-09-23T21:08:24.840 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/4041424623", 2023-09-23T21:08:24.840 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 212419 2023-09-23T21:08:24.840 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:24.840 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:08:24.840 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:24.840 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:08:24.840 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:21.287539+0000", 2023-09-23T21:08:24.840 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:08:24.840 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:24.840 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:24.840 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:08:24.841 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:21.287539+0000", 2023-09-23T21:08:24.841 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:08:24.841 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:24.841 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:24.841 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:08:24.841 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:21.287537+0000", 2023-09-23T21:08:24.841 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999981 2023-09-23T21:08:24.841 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:24.841 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:24.841 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:08:24.841 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:21.287544+0000", 2023-09-23T21:08:24.841 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.3370000000000002e-06 2023-09-23T21:08:24.841 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:24.842 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:24.842 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:08:24.842 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:21.287555+0000", 2023-09-23T21:08:24.842 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0565e-05 2023-09-23T21:08:24.842 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:24.842 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:24.842 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:08:24.842 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:21.287561+0000", 2023-09-23T21:08:24.842 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.4339999999999997e-06 2023-09-23T21:08:24.842 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:24.842 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:24.842 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:08:24.842 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:21.288134+0000", 2023-09-23T21:08:24.843 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00057250499999999995 2023-09-23T21:08:24.843 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:24.843 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:24.843 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T21:08:24.843 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:21.288142+0000", 2023-09-23T21:08:24.843 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.6240000000000002e-06 2023-09-23T21:08:24.843 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:24.843 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:24.843 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:08:24.843 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:22.345769+0000", 2023-09-23T21:08:24.843 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0576273620000001 2023-09-23T21:08:24.843 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:24.843 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:24.844 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:08:24.844 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:22.345880+0000", 2023-09-23T21:08:24.844 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000111121 2023-09-23T21:08:24.844 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:24.844 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:24.844 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 2", 2023-09-23T21:08:24.844 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:22.345982+0000", 2023-09-23T21:08:24.844 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00010207199999999999 2023-09-23T21:08:24.844 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:24.844 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:24.844 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T21:08:24.844 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:22.352826+0000", 2023-09-23T21:08:24.844 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0068439979999999996 2023-09-23T21:08:24.845 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:24.845 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:24.845 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:08:24.845 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:22.355536+0000", 2023-09-23T21:08:24.845 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0027094609999999998 2023-09-23T21:08:24.845 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:24.845 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:24.845 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T21:08:24.845 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:22.355589+0000", 2023-09-23T21:08:24.845 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.2961999999999999e-05 2023-09-23T21:08:24.845 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:24.845 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:24.845 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T21:08:24.846 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:22.355673+0000", 2023-09-23T21:08:24.846 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.4436000000000003e-05 2023-09-23T21:08:24.846 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:08:24.846 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:08:24.846 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:08:24.846 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:24.846 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:24.846 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35469.0:212436 8.0 8:0298251e:::benchmark_data_smithi026_301269_object101679:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e339)", 2023-09-23T21:08:24.846 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T21:05:21.315860+0000", 2023-09-23T21:08:24.846 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 183.42774259500001, 2023-09-23T21:08:24.846 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.027109716, 2023-09-23T21:08:24.846 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:08:24.847 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:08:24.847 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T21:08:24.847 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35469", 2023-09-23T21:08:24.847 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/4041424623", 2023-09-23T21:08:24.847 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 212436 2023-09-23T21:08:24.847 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:24.847 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:08:24.847 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:24.847 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:08:24.847 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:21.315860+0000", 2023-09-23T21:08:24.847 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:08:24.848 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:24.848 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:24.848 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:08:24.848 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:21.315860+0000", 2023-09-23T21:08:24.848 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:08:24.848 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:24.848 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:24.848 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:08:24.848 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:21.315858+0000", 2023-09-23T21:08:24.848 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999981 2023-09-23T21:08:24.848 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:24.848 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:24.848 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:08:24.849 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:21.315868+0000", 2023-09-23T21:08:24.849 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.7869999999999996e-06 2023-09-23T21:08:24.849 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:24.849 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:24.849 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:08:24.849 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:21.315889+0000", 2023-09-23T21:08:24.849 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.0883e-05 2023-09-23T21:08:24.849 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:24.849 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:24.849 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:08:24.849 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:21.315898+0000", 2023-09-23T21:08:24.849 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.5899999999999997e-06 2023-09-23T21:08:24.849 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:24.850 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:24.850 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:08:24.850 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:21.315962+0000", 2023-09-23T21:08:24.850 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.3885000000000002e-05 2023-09-23T21:08:24.850 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:24.850 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:24.850 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T21:08:24.850 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:21.315974+0000", 2023-09-23T21:08:24.850 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1439e-05 2023-09-23T21:08:24.850 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:24.850 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:24.850 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:08:24.850 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:22.335034+0000", 2023-09-23T21:08:24.851 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.01906053 2023-09-23T21:08:24.851 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:24.851 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:24.851 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:08:24.851 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:22.335188+0000", 2023-09-23T21:08:24.851 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000154287 2023-09-23T21:08:24.851 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:24.851 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:24.851 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 5", 2023-09-23T21:08:24.851 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:22.335752+0000", 2023-09-23T21:08:24.851 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00056333100000000003 2023-09-23T21:08:24.851 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:24.851 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:24.852 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:08:24.852 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:22.339420+0000", 2023-09-23T21:08:24.852 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0036679780000000001 2023-09-23T21:08:24.852 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:24.852 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:24.852 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T21:08:24.852 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:22.342823+0000", 2023-09-23T21:08:24.852 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0034033689999999998 2023-09-23T21:08:24.852 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:24.852 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:24.852 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T21:08:24.852 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:22.342873+0000", 2023-09-23T21:08:24.852 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.9462999999999997e-05 2023-09-23T21:08:24.853 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:24.853 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:24.853 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T21:08:24.853 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:22.342970+0000", 2023-09-23T21:08:24.853 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.7119000000000003e-05 2023-09-23T21:08:24.853 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:08:24.853 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:08:24.853 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:08:24.853 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:24.853 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:24.853 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35469.0:212438 8.0 8:05ba4396:::benchmark_data_smithi026_301269_object101681:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e339)", 2023-09-23T21:08:24.853 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T21:05:21.315925+0000", 2023-09-23T21:08:24.853 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 183.42767762400001, 2023-09-23T21:08:24.853 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0272295789999999, 2023-09-23T21:08:24.854 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:08:24.854 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:08:24.854 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T21:08:24.854 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35469", 2023-09-23T21:08:24.854 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/4041424623", 2023-09-23T21:08:24.854 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 212438 2023-09-23T21:08:24.854 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:24.854 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:08:24.854 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:24.854 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:08:24.854 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:21.315925+0000", 2023-09-23T21:08:24.854 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:08:24.855 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:24.855 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:24.855 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:08:24.855 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:21.315925+0000", 2023-09-23T21:08:24.855 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:08:24.855 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:24.855 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:24.855 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:08:24.855 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:21.315924+0000", 2023-09-23T21:08:24.855 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T21:08:24.855 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:24.855 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:24.855 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:08:24.855 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:21.315928+0000", 2023-09-23T21:08:24.856 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.4249999999999998e-06 2023-09-23T21:08:24.856 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:24.856 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:24.856 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:08:24.856 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:21.315936+0000", 2023-09-23T21:08:24.856 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.4759999999999998e-06 2023-09-23T21:08:24.856 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:24.856 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:24.856 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:08:24.856 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:21.315943+0000", 2023-09-23T21:08:24.856 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.7429999999999996e-06 2023-09-23T21:08:24.856 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:24.856 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:24.857 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:08:24.857 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:21.315997+0000", 2023-09-23T21:08:24.857 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.3767999999999997e-05 2023-09-23T21:08:24.857 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:24.857 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:24.857 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T21:08:24.857 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:21.316003+0000", 2023-09-23T21:08:24.857 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.331e-06 2023-09-23T21:08:24.857 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:24.857 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:24.857 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:08:24.857 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:22.339456+0000", 2023-09-23T21:08:24.857 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0234528839999999 2023-09-23T21:08:24.858 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:24.858 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:24.858 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:08:24.858 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:22.339612+0000", 2023-09-23T21:08:24.858 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00015559600000000001 2023-09-23T21:08:24.858 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:24.858 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:24.858 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 5", 2023-09-23T21:08:24.858 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:22.340163+0000", 2023-09-23T21:08:24.858 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00055139200000000005 2023-09-23T21:08:24.858 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:24.858 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:24.858 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T21:08:24.859 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:22.343000+0000", 2023-09-23T21:08:24.859 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0028368130000000001 2023-09-23T21:08:24.859 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:24.859 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:24.859 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:08:24.859 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:22.343063+0000", 2023-09-23T21:08:24.859 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.3281999999999995e-05 2023-09-23T21:08:24.859 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:24.859 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:24.859 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T21:08:24.859 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:22.343091+0000", 2023-09-23T21:08:24.859 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.7602000000000001e-05 2023-09-23T21:08:24.859 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:24.860 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:24.860 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T21:08:24.860 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:22.343154+0000", 2023-09-23T21:08:24.860 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.3793000000000004e-05 2023-09-23T21:08:24.860 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:08:24.860 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:08:24.860 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:08:24.860 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:24.860 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:24.860 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35469.0:212442 8.b 8:d9438fc9:::benchmark_data_smithi026_301269_object101685:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e339)", 2023-09-23T21:08:24.860 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T21:05:21.319757+0000", 2023-09-23T21:08:24.860 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 183.42384600299999, 2023-09-23T21:08:24.860 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.102493349, 2023-09-23T21:08:24.861 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:08:24.861 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:08:24.861 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T21:08:24.861 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35469", 2023-09-23T21:08:24.861 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/4041424623", 2023-09-23T21:08:24.861 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 212442 2023-09-23T21:08:24.861 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:24.861 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:08:24.861 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:24.861 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:08:24.861 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:21.319757+0000", 2023-09-23T21:08:24.861 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:08:24.861 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:24.861 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:24.862 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:08:24.862 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:21.319757+0000", 2023-09-23T21:08:24.862 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:08:24.862 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:24.862 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:24.862 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:08:24.862 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:21.319755+0000", 2023-09-23T21:08:24.862 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999981 2023-09-23T21:08:24.862 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:24.862 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:24.862 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:08:24.862 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:21.319764+0000", 2023-09-23T21:08:24.863 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.816e-06 2023-09-23T21:08:24.863 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:24.863 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:24.863 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:08:24.863 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:21.319781+0000", 2023-09-23T21:08:24.863 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.7229999999999999e-05 2023-09-23T21:08:24.863 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:24.863 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:24.863 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:08:24.863 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:21.319789+0000", 2023-09-23T21:08:24.863 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.8800000000000008e-06 2023-09-23T21:08:24.863 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:24.863 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:24.864 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:08:24.864 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:21.328982+0000", 2023-09-23T21:08:24.864 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0091934649999999996 2023-09-23T21:08:24.864 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:24.864 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:24.864 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T21:08:24.864 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:21.328993+0000", 2023-09-23T21:08:24.864 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1134999999999999e-05 2023-09-23T21:08:24.864 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:24.864 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:24.864 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:08:24.864 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:22.401873+0000", 2023-09-23T21:08:24.864 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0728796350000001 2023-09-23T21:08:24.865 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:24.865 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:24.865 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:08:24.865 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:22.402000+0000", 2023-09-23T21:08:24.865 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00012707899999999999 2023-09-23T21:08:24.865 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:24.865 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:24.865 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 4", 2023-09-23T21:08:24.865 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:22.402139+0000", 2023-09-23T21:08:24.865 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00013863699999999999 2023-09-23T21:08:24.865 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:24.865 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:24.865 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T21:08:24.866 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:22.417442+0000", 2023-09-23T21:08:24.866 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.015303235 2023-09-23T21:08:24.866 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:24.866 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:24.866 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:08:24.866 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:22.422114+0000", 2023-09-23T21:08:24.866 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00467228 2023-09-23T21:08:24.866 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:24.866 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:24.866 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T21:08:24.866 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:22.422167+0000", 2023-09-23T21:08:24.866 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.2453000000000001e-05 2023-09-23T21:08:24.866 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:24.867 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:24.867 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T21:08:24.867 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:22.422250+0000", 2023-09-23T21:08:24.867 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.3220999999999995e-05 2023-09-23T21:08:24.867 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:08:24.867 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:08:24.867 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:08:24.867 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:24.867 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:24.867 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35469.0:212445 8.13 8:c9e6ee86:::benchmark_data_smithi026_301269_object101688:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e339)", 2023-09-23T21:08:24.867 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T21:05:21.323540+0000", 2023-09-23T21:08:24.867 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 183.420062825, 2023-09-23T21:08:24.867 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.077949351, 2023-09-23T21:08:24.867 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:08:24.868 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:08:24.868 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T21:08:24.868 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35469", 2023-09-23T21:08:24.868 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/4041424623", 2023-09-23T21:08:24.868 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 212445 2023-09-23T21:08:24.868 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:24.868 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:08:24.868 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:24.868 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:08:24.868 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:21.323540+0000", 2023-09-23T21:08:24.868 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:08:24.868 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:24.868 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:24.869 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:08:24.869 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:21.323540+0000", 2023-09-23T21:08:24.869 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:08:24.869 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:24.869 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:24.869 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:08:24.869 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:21.323538+0000", 2023-09-23T21:08:24.869 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999976 2023-09-23T21:08:24.869 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:24.869 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:24.869 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:08:24.869 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:21.323548+0000", 2023-09-23T21:08:24.869 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0773e-05 2023-09-23T21:08:24.869 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:24.870 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:24.870 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:08:24.870 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:21.323572+0000", 2023-09-23T21:08:24.870 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.3756000000000001e-05 2023-09-23T21:08:24.870 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:24.870 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:24.870 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:08:24.870 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:21.323583+0000", 2023-09-23T21:08:24.870 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0672e-05 2023-09-23T21:08:24.870 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:24.870 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:24.870 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:08:24.870 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:21.329020+0000", 2023-09-23T21:08:24.871 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0054372769999999999 2023-09-23T21:08:24.871 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:24.871 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:24.871 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T21:08:24.871 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:21.329026+0000", 2023-09-23T21:08:24.871 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.8619999999999999e-06 2023-09-23T21:08:24.871 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:24.871 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:24.871 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:08:24.871 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:22.346446+0000", 2023-09-23T21:08:24.871 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.017420038 2023-09-23T21:08:24.871 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:24.872 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:24.872 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:08:24.872 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:22.346568+0000", 2023-09-23T21:08:24.872 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000122416 2023-09-23T21:08:24.872 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:24.872 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:24.872 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 2", 2023-09-23T21:08:24.872 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:22.347038+0000", 2023-09-23T21:08:24.872 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00046929099999999999 2023-09-23T21:08:24.872 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:24.872 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:24.872 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T21:08:24.872 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:22.352737+0000", 2023-09-23T21:08:24.872 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0056990590000000002 2023-09-23T21:08:24.873 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:24.873 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:24.873 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:08:24.873 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:22.401389+0000", 2023-09-23T21:08:24.873 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.048652754999999999 2023-09-23T21:08:24.873 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:24.873 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:24.873 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T21:08:24.873 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:22.401428+0000", 2023-09-23T21:08:24.873 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.8770000000000003e-05 2023-09-23T21:08:24.873 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:24.873 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:24.873 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T21:08:24.874 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:22.401489+0000", 2023-09-23T21:08:24.874 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.0853000000000002e-05 2023-09-23T21:08:24.874 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:08:24.874 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:08:24.874 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:08:24.874 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:24.874 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:24.874 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35469.0:212446 8.b 8:da5201d2:::benchmark_data_smithi026_301269_object101689:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e339)", 2023-09-23T21:08:24.874 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T21:05:21.324611+0000", 2023-09-23T21:08:24.874 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 183.41899154199999, 2023-09-23T21:08:24.874 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0977934, 2023-09-23T21:08:24.874 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:08:24.874 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:08:24.875 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T21:08:24.875 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35469", 2023-09-23T21:08:24.875 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/4041424623", 2023-09-23T21:08:24.875 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 212446 2023-09-23T21:08:24.875 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:24.875 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:08:24.875 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:24.875 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:08:24.875 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:21.324611+0000", 2023-09-23T21:08:24.875 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:08:24.875 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:24.875 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:24.875 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:08:24.875 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:21.324611+0000", 2023-09-23T21:08:24.876 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:08:24.876 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:24.876 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:24.876 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:08:24.876 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:21.324610+0000", 2023-09-23T21:08:24.876 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.999999 2023-09-23T21:08:24.876 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:24.876 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:24.876 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:08:24.876 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:21.324615+0000", 2023-09-23T21:08:24.876 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.9049999999999996e-06 2023-09-23T21:08:24.876 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:24.876 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:24.877 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:08:24.877 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:21.324626+0000", 2023-09-23T21:08:24.877 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.111e-05 2023-09-23T21:08:24.877 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:24.877 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:24.877 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:08:24.877 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:21.324633+0000", 2023-09-23T21:08:24.877 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.0419999999999997e-06 2023-09-23T21:08:24.877 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:24.877 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:24.877 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:08:24.877 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:21.329045+0000", 2023-09-23T21:08:24.877 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0044122950000000001 2023-09-23T21:08:24.877 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:24.878 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:24.878 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T21:08:24.878 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:21.329052+0000", 2023-09-23T21:08:24.878 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.5599999999999999e-06 2023-09-23T21:08:24.878 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:24.878 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:24.878 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:08:24.878 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:22.405916+0000", 2023-09-23T21:08:24.878 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0768636300000001 2023-09-23T21:08:24.878 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:24.878 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:24.878 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:08:24.879 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:22.406058+0000", 2023-09-23T21:08:24.879 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000142173 2023-09-23T21:08:24.879 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:24.879 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:24.879 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 4", 2023-09-23T21:08:24.879 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:22.406188+0000", 2023-09-23T21:08:24.879 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00013058199999999999 2023-09-23T21:08:24.879 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:24.879 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:24.879 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T21:08:24.879 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:22.417448+0000", 2023-09-23T21:08:24.879 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.011259692 2023-09-23T21:08:24.879 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:24.880 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:24.880 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:08:24.880 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:22.422291+0000", 2023-09-23T21:08:24.880 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0048431630000000002 2023-09-23T21:08:24.880 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:24.880 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:24.880 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T21:08:24.880 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:22.422330+0000", 2023-09-23T21:08:24.880 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.8448000000000002e-05 2023-09-23T21:08:24.880 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:24.880 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:24.880 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T21:08:24.880 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:22.422404+0000", 2023-09-23T21:08:24.880 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.4745000000000003e-05 2023-09-23T21:08:24.881 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:08:24.881 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:08:24.881 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:08:24.881 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:24.881 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:24.881 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35856.0:157810 9.c 9:3e05b922:::benchmark_data_smithi026_342732_object44534:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e378)", 2023-09-23T21:08:24.881 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T21:08:21.452277+0000", 2023-09-23T21:08:24.881 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 3.2913258409999999, 2023-09-23T21:08:24.881 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.48204857200000001, 2023-09-23T21:08:24.881 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:08:24.881 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:08:24.881 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T21:08:24.881 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35856", 2023-09-23T21:08:24.881 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/4180396258", 2023-09-23T21:08:24.882 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 157810 2023-09-23T21:08:24.882 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:24.882 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:08:24.882 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:24.882 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:08:24.882 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:08:21.452277+0000", 2023-09-23T21:08:24.882 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:08:24.882 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:24.882 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:24.882 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:08:24.882 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:08:21.452277+0000", 2023-09-23T21:08:24.882 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:08:24.883 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:24.883 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:24.883 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:08:24.883 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:08:21.452276+0000", 2023-09-23T21:08:24.883 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T21:08:24.883 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:24.883 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:24.883 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:08:24.883 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:08:21.452279+0000", 2023-09-23T21:08:24.883 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.0529999999999998e-06 2023-09-23T21:08:24.883 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:24.883 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:24.883 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:08:24.884 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:08:21.452286+0000", 2023-09-23T21:08:24.884 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.4200000000000004e-06 2023-09-23T21:08:24.884 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:24.884 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:24.884 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:08:24.884 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:08:21.452290+0000", 2023-09-23T21:08:24.884 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.8980000000000004e-06 2023-09-23T21:08:24.884 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:24.884 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:24.884 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:08:24.884 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:08:21.452367+0000", 2023-09-23T21:08:24.885 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.6280999999999997e-05 2023-09-23T21:08:24.885 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:24.885 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:24.885 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:08:24.885 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:08:21.452495+0000", 2023-09-23T21:08:24.885 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000128467 2023-09-23T21:08:24.885 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:24.885 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:24.885 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 4,7", 2023-09-23T21:08:24.885 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:08:21.452596+0000", 2023-09-23T21:08:24.885 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000100429 2023-09-23T21:08:24.885 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:24.885 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:24.886 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:08:24.886 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:08:21.455644+0000", 2023-09-23T21:08:24.886 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0030482019999999999 2023-09-23T21:08:24.886 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:24.886 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:24.886 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T21:08:24.886 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:08:21.460150+0000", 2023-09-23T21:08:24.886 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0045060589999999998 2023-09-23T21:08:24.886 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:24.886 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:24.886 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:08:24.886 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:08:21.934202+0000", 2023-09-23T21:08:24.886 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.47405230300000001 2023-09-23T21:08:24.887 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:24.887 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:24.887 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T21:08:24.887 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:08:21.934264+0000", 2023-09-23T21:08:24.887 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.1599000000000004e-05 2023-09-23T21:08:24.887 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:24.887 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:24.887 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T21:08:24.887 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:08:21.934325+0000", 2023-09-23T21:08:24.887 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.1421999999999996e-05 2023-09-23T21:08:24.887 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:08:24.887 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:08:24.887 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:08:24.888 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:24.888 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:24.888 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35856.0:157817 9.c 9:3cd83e3a:::benchmark_data_smithi026_342732_object44541:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e378)", 2023-09-23T21:08:24.888 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T21:08:21.456403+0000", 2023-09-23T21:08:24.888 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 3.287199185, 2023-09-23T21:08:24.888 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.48780633699999998, 2023-09-23T21:08:24.888 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:08:24.888 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:08:24.888 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T21:08:24.888 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35856", 2023-09-23T21:08:24.888 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/4180396258", 2023-09-23T21:08:24.888 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 157817 2023-09-23T21:08:24.888 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:24.889 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:08:24.889 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:24.889 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:08:24.889 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:08:21.456403+0000", 2023-09-23T21:08:24.889 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:08:24.889 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:24.889 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:24.889 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:08:24.889 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:08:21.456403+0000", 2023-09-23T21:08:24.889 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:08:24.889 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:24.889 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:24.889 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:08:24.889 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:08:21.456402+0000", 2023-09-23T21:08:24.890 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T21:08:24.890 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:24.890 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:24.890 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:08:24.890 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:08:21.456408+0000", 2023-09-23T21:08:24.890 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.5090000000000001e-06 2023-09-23T21:08:24.890 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:24.890 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:24.890 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:08:24.890 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:08:21.456416+0000", 2023-09-23T21:08:24.890 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.1699999999999997e-06 2023-09-23T21:08:24.890 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:24.891 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:24.891 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:08:24.891 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:08:21.456421+0000", 2023-09-23T21:08:24.891 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.8319999999999996e-06 2023-09-23T21:08:24.891 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:24.891 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:24.891 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:08:24.891 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:08:21.456640+0000", 2023-09-23T21:08:24.891 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00021963199999999999 2023-09-23T21:08:24.891 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:24.891 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:24.891 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:08:24.891 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:08:21.456783+0000", 2023-09-23T21:08:24.892 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00014281400000000001 2023-09-23T21:08:24.892 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:24.892 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:24.892 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 4,7", 2023-09-23T21:08:24.892 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:08:21.456929+0000", 2023-09-23T21:08:24.892 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00014633800000000001 2023-09-23T21:08:24.892 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:24.892 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:24.892 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T21:08:24.892 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:08:21.460316+0000", 2023-09-23T21:08:24.892 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.003386385 2023-09-23T21:08:24.892 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:24.892 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:24.892 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:08:24.893 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:08:21.465501+0000", 2023-09-23T21:08:24.893 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0051855699999999996 2023-09-23T21:08:24.893 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:24.893 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:24.893 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:08:24.893 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:08:21.944153+0000", 2023-09-23T21:08:24.893 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.47865144300000001 2023-09-23T21:08:24.893 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:24.893 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:24.893 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T21:08:24.893 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:08:21.944185+0000", 2023-09-23T21:08:24.893 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.2668e-05 2023-09-23T21:08:24.893 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:08:24.894 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:08:24.894 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T21:08:24.894 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:08:21.944210+0000", 2023-09-23T21:08:24.894 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.4212000000000001e-05 2023-09-23T21:08:24.894 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:08:24.894 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:08:24.894 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:08:24.894 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:08:24.894 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:08:24.894 INFO:teuthology.orchestra.run.smithi026.stdout:} 2023-09-23T21:08:24.895 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:08:24 smithi070 ceph-mon[168282]: pgmap v995: 109 pgs: 109 active+clean; 4.3 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 796 op/s 2023-09-23T21:08:24.895 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:08:24.895 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:08:25.095 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T21:08:25.118 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:08:25.119 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:08:25.220 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:08:25.272 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:08:25.335 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:08:25.341 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:08:25.342 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:08:25.442 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:08:25.511 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:08:25.688 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_ops_in_flight 2023-09-23T21:08:25.967 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:08:25.985 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:08:25.985 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:08:26.086 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:08:26.131 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:08:26.181 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:08:26.185 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:08:26.186 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:08:26.286 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:08:26.367 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T21:08:26.397 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:08:26.529 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:08:26.541 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:08:26.541 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:08:26.633 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:08:26 smithi026 ceph-mon[179688]: pgmap v996: 109 pgs: 109 active+clean; 4.1 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 790 op/s 2023-09-23T21:08:26.635 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:08:26 smithi026 ceph-mon[175151]: pgmap v996: 109 pgs: 109 active+clean; 4.1 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 790 op/s 2023-09-23T21:08:26.642 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:08:26.698 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:08:26 smithi070 ceph-mon[168282]: pgmap v996: 109 pgs: 109 active+clean; 4.1 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 790 op/s 2023-09-23T21:08:26.699 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:08:26.812 INFO:teuthology.orchestra.run.smithi026.stdout: 2023-09-23T21:08:26.828 INFO:teuthology.orchestra.run.smithi026.stderr:dumped all 2023-09-23T21:08:26.917 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:08:26.919 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:08:26.920 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:08:27.000 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0: {} 2023-09-23T21:08:27.021 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:08:27.040 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1: {} 2023-09-23T21:08:27.089 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2: {} 2023-09-23T21:08:27.109 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:08:27.128 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3: {} 2023-09-23T21:08:27.188 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4: {} 2023-09-23T21:08:27.233 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T21:08:27.262 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:08:27.387 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:08:27.387 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:08:27.388 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6: {} 2023-09-23T21:08:27.388 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T21:08:27.388 INFO:teuthology.orchestra.run.smithi026.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T21:08:27.388 INFO:teuthology.orchestra.run.smithi026.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T21:08:27.389 INFO:teuthology.orchestra.run.smithi026.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T21:08:27.389 INFO:teuthology.orchestra.run.smithi026.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T21:08:27.389 INFO:teuthology.orchestra.run.smithi026.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T21:08:27.389 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T21:08:27.389 INFO:teuthology.orchestra.run.smithi026.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T21:08:27.389 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T21:08:27.394 INFO:teuthology.orchestra.run.smithi026.stdout:{"pg_ready":true,"pg_map":{"version":996,"stamp":"2023-09-23T21:08:24.895241+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4442101688,"num_objects":67797,"num_object_clones":0,"num_object_copies":203391,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":67797,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":158836,"num_write_kb":7249502,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":39,"num_bytes_recovered":7154,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":119211,"ondisk_log_size":119211,"up":327,"acting":327,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":327,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":16356428,"kb_used_data":13166732,"kb_used_omap":137,"kb_used_meta":3189494,"kb_avail":733604788,"statfs":{"total":767960285184,"available":751211302912,"internally_reserved":0,"allocated":13482733568,"data_stored":13471837965,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":141000,"internal_metadata":3266042168},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,0,4,10,6,0,0,0,0,2],"upper_bound":1024},"perf_stat":{"commit_latency_ms":5,"apply_latency_ms":5,"commit_latency_ns":5000000,"apply_latency_ns":5000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-518127616,"num_objects":-7906,"num_object_clones":0,"num_object_copies":-23718,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":-7906,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7906,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.006706"},"pg_stats":[{"pgid":"3.1a","version":"68'1","reported_seq":272,"reported_epoch":377,"state":"active+clean","last_fresh":"2023-09-23T21:08:12.368402+0000","last_change":"2023-09-23T21:05:55.322748+0000","last_active":"2023-09-23T21:08:12.368402+0000","last_peered":"2023-09-23T21:08:12.368402+0000","last_clean":"2023-09-23T21:08:12.368402+0000","last_became_active":"2023-09-23T21:05:55.322498+0000","last_became_peered":"2023-09-23T21:05:55.322498+0000","last_unstale":"2023-09-23T21:08:12.368402+0000","last_undegraded":"2023-09-23T21:08:12.368402+0000","last_fullsized":"2023-09-23T21:08:12.368402+0000","mapping_epoch":350,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":351,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:03:28.049826+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":373,"reported_epoch":377,"state":"active+clean","last_fresh":"2023-09-23T21:08:12.384001+0000","last_change":"2023-09-23T20:54:13.942019+0000","last_active":"2023-09-23T21:08:12.384001+0000","last_peered":"2023-09-23T21:08:12.384001+0000","last_clean":"2023-09-23T21:08:12.384001+0000","last_became_active":"2023-09-23T20:54:13.941647+0000","last_became_peered":"2023-09-23T20:54:13.941647+0000","last_unstale":"2023-09-23T21:08:12.384001+0000","last_undegraded":"2023-09-23T21:08:12.384001+0000","last_fullsized":"2023-09-23T21:08:12.384001+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:30:54.400944+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":351,"reported_epoch":378,"state":"active+clean","last_fresh":"2023-09-23T21:08:16.375271+0000","last_change":"2023-09-23T21:05:55.322068+0000","last_active":"2023-09-23T21:08:16.375271+0000","last_peered":"2023-09-23T21:08:16.375271+0000","last_clean":"2023-09-23T21:08:16.375271+0000","last_became_active":"2023-09-23T21:05:55.321763+0000","last_became_peered":"2023-09-23T21:05:55.321763+0000","last_unstale":"2023-09-23T21:08:16.375271+0000","last_undegraded":"2023-09-23T21:08:16.375271+0000","last_fullsized":"2023-09-23T21:08:16.375271+0000","mapping_epoch":350,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":351,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:01:41.294709+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":367,"reported_epoch":377,"state":"active+clean","last_fresh":"2023-09-23T21:08:12.383543+0000","last_change":"2023-09-23T20:57:44.979811+0000","last_active":"2023-09-23T21:08:12.383543+0000","last_peered":"2023-09-23T21:08:12.383543+0000","last_clean":"2023-09-23T21:08:12.383543+0000","last_became_active":"2023-09-23T20:57:44.979515+0000","last_became_peered":"2023-09-23T20:57:44.979515+0000","last_unstale":"2023-09-23T21:08:12.383543+0000","last_undegraded":"2023-09-23T21:08:12.383543+0000","last_fullsized":"2023-09-23T21:08:12.383543+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:38:22.162140+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":380,"reported_epoch":377,"state":"active+clean","last_fresh":"2023-09-23T21:08:12.380919+0000","last_change":"2023-09-23T21:05:55.325654+0000","last_active":"2023-09-23T21:08:12.380919+0000","last_peered":"2023-09-23T21:08:12.380919+0000","last_clean":"2023-09-23T21:08:12.380919+0000","last_became_active":"2023-09-23T21:05:55.325298+0000","last_became_peered":"2023-09-23T21:05:55.325298+0000","last_unstale":"2023-09-23T21:08:12.380919+0000","last_undegraded":"2023-09-23T21:08:12.380919+0000","last_fullsized":"2023-09-23T21:08:12.380919+0000","mapping_epoch":350,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":351,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:33:19.561063+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":380,"reported_epoch":377,"state":"active+clean","last_fresh":"2023-09-23T21:08:12.367822+0000","last_change":"2023-09-23T21:05:55.319892+0000","last_active":"2023-09-23T21:08:12.367822+0000","last_peered":"2023-09-23T21:08:12.367822+0000","last_clean":"2023-09-23T21:08:12.367822+0000","last_became_active":"2023-09-23T21:05:55.319536+0000","last_became_peered":"2023-09-23T21:05:55.319536+0000","last_unstale":"2023-09-23T21:08:12.367822+0000","last_undegraded":"2023-09-23T21:08:12.367822+0000","last_fullsized":"2023-09-23T21:08:12.367822+0000","mapping_epoch":350,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":351,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:14:48.388831+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":374,"reported_epoch":377,"state":"active+clean","last_fresh":"2023-09-23T21:08:12.386318+0000","last_change":"2023-09-23T21:05:04.304358+0000","last_active":"2023-09-23T21:08:12.386318+0000","last_peered":"2023-09-23T21:08:12.386318+0000","last_clean":"2023-09-23T21:08:12.386318+0000","last_became_active":"2023-09-23T21:05:04.303913+0000","last_became_peered":"2023-09-23T21:05:04.303913+0000","last_unstale":"2023-09-23T21:08:12.386318+0000","last_undegraded":"2023-09-23T21:08:12.386318+0000","last_fullsized":"2023-09-23T21:08:12.386318+0000","mapping_epoch":332,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:21:21.860944+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":377,"reported_epoch":377,"state":"active+clean","last_fresh":"2023-09-23T21:08:12.380999+0000","last_change":"2023-09-23T21:05:55.323506+0000","last_active":"2023-09-23T21:08:12.380999+0000","last_peered":"2023-09-23T21:08:12.380999+0000","last_clean":"2023-09-23T21:08:12.380999+0000","last_became_active":"2023-09-23T21:05:55.323167+0000","last_became_peered":"2023-09-23T21:05:55.323167+0000","last_unstale":"2023-09-23T21:08:12.380999+0000","last_undegraded":"2023-09-23T21:08:12.380999+0000","last_fullsized":"2023-09-23T21:08:12.380999+0000","mapping_epoch":350,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":351,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:06:00.628349+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"79'161","reported_seq":668,"reported_epoch":377,"state":"active+clean","last_fresh":"2023-09-23T21:08:12.376417+0000","last_change":"2023-09-23T20:57:25.096559+0000","last_active":"2023-09-23T21:08:12.376417+0000","last_peered":"2023-09-23T21:08:12.376417+0000","last_clean":"2023-09-23T21:08:12.376417+0000","last_became_active":"2023-09-23T20:57:15.975674+0000","last_became_peered":"2023-09-23T20:57:15.975674+0000","last_unstale":"2023-09-23T21:08:12.376417+0000","last_undegraded":"2023-09-23T21:08:12.376417+0000","last_fullsized":"2023-09-23T21:08:12.376417+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:25.096401+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:25.096401+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:05:05.448404+0000","scrub_duration":0.002859393,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":72,"reported_epoch":377,"state":"active+clean","last_fresh":"2023-09-23T21:08:12.384163+0000","last_change":"2023-09-23T21:05:55.325024+0000","last_active":"2023-09-23T21:08:12.384163+0000","last_peered":"2023-09-23T21:08:12.384163+0000","last_clean":"2023-09-23T21:08:12.384163+0000","last_became_active":"2023-09-23T21:05:55.322890+0000","last_became_peered":"2023-09-23T21:05:55.322890+0000","last_unstale":"2023-09-23T21:08:12.384163+0000","last_undegraded":"2023-09-23T21:08:12.384163+0000","last_fullsized":"2023-09-23T21:08:12.384163+0000","mapping_epoch":350,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":351,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:49:56.913769+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":323,"reported_epoch":377,"state":"active+clean","last_fresh":"2023-09-23T21:08:12.381285+0000","last_change":"2023-09-23T20:55:35.952661+0000","last_active":"2023-09-23T21:08:12.381285+0000","last_peered":"2023-09-23T21:08:12.381285+0000","last_clean":"2023-09-23T21:08:12.381285+0000","last_became_active":"2023-09-23T20:55:35.952349+0000","last_became_peered":"2023-09-23T20:55:35.952349+0000","last_unstale":"2023-09-23T21:08:12.381285+0000","last_undegraded":"2023-09-23T21:08:12.381285+0000","last_fullsized":"2023-09-23T21:08:12.381285+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:00:03.110126+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":199,"reported_epoch":377,"state":"active+clean","last_fresh":"2023-09-23T21:08:12.384086+0000","last_change":"2023-09-23T20:56:45.988607+0000","last_active":"2023-09-23T21:08:12.384086+0000","last_peered":"2023-09-23T21:08:12.384086+0000","last_clean":"2023-09-23T21:08:12.384086+0000","last_became_active":"2023-09-23T20:56:45.988206+0000","last_became_peered":"2023-09-23T20:56:45.988206+0000","last_unstale":"2023-09-23T21:08:12.384086+0000","last_undegraded":"2023-09-23T21:08:12.384086+0000","last_fullsized":"2023-09-23T21:08:12.384086+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":205,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:01:19.783688+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,4],"acting":[3,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":51,"reported_epoch":377,"state":"active+clean","last_fresh":"2023-09-23T21:08:13.466231+0000","last_change":"2023-09-23T21:05:55.315641+0000","last_active":"2023-09-23T21:08:13.466231+0000","last_peered":"2023-09-23T21:08:13.466231+0000","last_clean":"2023-09-23T21:08:13.466231+0000","last_became_active":"2023-09-23T21:05:55.315336+0000","last_became_peered":"2023-09-23T21:05:55.315336+0000","last_unstale":"2023-09-23T21:08:13.466231+0000","last_undegraded":"2023-09-23T21:08:13.466231+0000","last_fullsized":"2023-09-23T21:08:13.466231+0000","mapping_epoch":350,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":351,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:16:47.607565+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"79'22","reported_seq":282,"reported_epoch":377,"state":"active+clean","last_fresh":"2023-09-23T21:08:12.368642+0000","last_change":"2023-09-23T20:55:35.949418+0000","last_active":"2023-09-23T21:08:12.368642+0000","last_peered":"2023-09-23T21:08:12.368642+0000","last_clean":"2023-09-23T21:08:12.368642+0000","last_became_active":"2023-09-23T20:55:35.948566+0000","last_became_peered":"2023-09-23T20:55:35.948566+0000","last_unstale":"2023-09-23T21:08:12.368642+0000","last_undegraded":"2023-09-23T21:08:12.368642+0000","last_fullsized":"2023-09-23T21:08:12.368642+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:59:37.433128+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4734,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":289,"reported_epoch":377,"state":"active+clean","last_fresh":"2023-09-23T21:08:12.368610+0000","last_change":"2023-09-23T21:05:55.325203+0000","last_active":"2023-09-23T21:08:12.368610+0000","last_peered":"2023-09-23T21:08:12.368610+0000","last_clean":"2023-09-23T21:08:12.368610+0000","last_became_active":"2023-09-23T21:05:55.323957+0000","last_became_peered":"2023-09-23T21:05:55.323957+0000","last_unstale":"2023-09-23T21:08:12.368610+0000","last_undegraded":"2023-09-23T21:08:12.368610+0000","last_fullsized":"2023-09-23T21:08:12.368610+0000","mapping_epoch":350,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":351,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:03:20.418924+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.11","version":"79'161","reported_seq":644,"reported_epoch":377,"state":"active+clean","last_fresh":"2023-09-23T21:08:12.383223+0000","last_change":"2023-09-23T21:05:07.301125+0000","last_active":"2023-09-23T21:08:12.383223+0000","last_peered":"2023-09-23T21:08:12.383223+0000","last_clean":"2023-09-23T21:08:12.383223+0000","last_became_active":"2023-09-23T21:05:07.300809+0000","last_became_peered":"2023-09-23T21:05:07.300809+0000","last_unstale":"2023-09-23T21:08:12.383223+0000","last_undegraded":"2023-09-23T21:08:12.383223+0000","last_fullsized":"2023-09-23T21:08:12.383223+0000","mapping_epoch":335,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":336,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:28.756634+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:28.756634+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:28:29.079665+0000","scrub_duration":0.0064925119999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":232,"reported_epoch":377,"state":"active+clean","last_fresh":"2023-09-23T21:08:12.378159+0000","last_change":"2023-09-23T21:06:25.394796+0000","last_active":"2023-09-23T21:08:12.378159+0000","last_peered":"2023-09-23T21:08:12.378159+0000","last_clean":"2023-09-23T21:08:12.378159+0000","last_became_active":"2023-09-23T21:06:25.394447+0000","last_became_peered":"2023-09-23T21:06:25.394447+0000","last_unstale":"2023-09-23T21:08:12.378159+0000","last_undegraded":"2023-09-23T21:08:12.378159+0000","last_fullsized":"2023-09-23T21:08:12.378159+0000","mapping_epoch":354,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":355,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:55:25.161895+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":362,"reported_epoch":377,"state":"active+clean","last_fresh":"2023-09-23T21:08:12.387381+0000","last_change":"2023-09-23T21:05:55.324614+0000","last_active":"2023-09-23T21:08:12.387381+0000","last_peered":"2023-09-23T21:08:12.387381+0000","last_clean":"2023-09-23T21:08:12.387381+0000","last_became_active":"2023-09-23T21:05:55.324306+0000","last_became_peered":"2023-09-23T21:05:55.324306+0000","last_unstale":"2023-09-23T21:08:12.387381+0000","last_undegraded":"2023-09-23T21:08:12.387381+0000","last_fullsized":"2023-09-23T21:08:12.387381+0000","mapping_epoch":350,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":351,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:32:58.417049+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.10","version":"79'161","reported_seq":470,"reported_epoch":377,"state":"active+clean","last_fresh":"2023-09-23T21:08:12.367346+0000","last_change":"2023-09-23T21:06:50.337099+0000","last_active":"2023-09-23T21:08:12.367346+0000","last_peered":"2023-09-23T21:08:12.367346+0000","last_clean":"2023-09-23T21:08:12.367346+0000","last_became_active":"2023-09-23T21:06:49.426789+0000","last_became_peered":"2023-09-23T21:06:49.426789+0000","last_unstale":"2023-09-23T21:08:12.367346+0000","last_undegraded":"2023-09-23T21:08:12.367346+0000","last_fullsized":"2023-09-23T21:08:12.367346+0000","mapping_epoch":359,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":360,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:06:50.336992+0000","last_deep_scrub":"79'161","last_deep_scrub_stamp":"2023-09-23T21:06:50.336992+0000","last_clean_scrub_stamp":"2023-09-23T21:06:50.336992+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:26:47.504229+0000","scrub_duration":0.0026150930000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,7],"acting":[6,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":357,"reported_epoch":377,"state":"active+clean","last_fresh":"2023-09-23T21:08:12.383133+0000","last_change":"2023-09-23T20:57:44.979736+0000","last_active":"2023-09-23T21:08:12.383133+0000","last_peered":"2023-09-23T21:08:12.383133+0000","last_clean":"2023-09-23T21:08:12.383133+0000","last_became_active":"2023-09-23T20:57:44.979356+0000","last_became_peered":"2023-09-23T20:57:44.979356+0000","last_unstale":"2023-09-23T21:08:12.383133+0000","last_undegraded":"2023-09-23T21:08:12.383133+0000","last_fullsized":"2023-09-23T21:08:12.383133+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:14:43.848832+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":100,"reported_epoch":377,"state":"active+clean","last_fresh":"2023-09-23T21:08:12.383928+0000","last_change":"2023-09-23T21:06:49.426186+0000","last_active":"2023-09-23T21:08:12.383928+0000","last_peered":"2023-09-23T21:08:12.383928+0000","last_clean":"2023-09-23T21:08:12.383928+0000","last_became_active":"2023-09-23T21:06:49.425867+0000","last_became_peered":"2023-09-23T21:06:49.425867+0000","last_unstale":"2023-09-23T21:08:12.383928+0000","last_undegraded":"2023-09-23T21:08:12.383928+0000","last_fullsized":"2023-09-23T21:08:12.383928+0000","mapping_epoch":359,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":360,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:40:21.708447+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.13","version":"79'161","reported_seq":652,"reported_epoch":377,"state":"active+clean","last_fresh":"2023-09-23T21:08:13.466259+0000","last_change":"2023-09-23T21:05:55.322760+0000","last_active":"2023-09-23T21:08:13.466259+0000","last_peered":"2023-09-23T21:08:13.466259+0000","last_clean":"2023-09-23T21:08:13.466259+0000","last_became_active":"2023-09-23T21:05:55.322483+0000","last_became_peered":"2023-09-23T21:05:55.322483+0000","last_unstale":"2023-09-23T21:08:13.466259+0000","last_undegraded":"2023-09-23T21:08:13.466259+0000","last_fullsized":"2023-09-23T21:08:13.466259+0000","mapping_epoch":350,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":351,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:36.420362+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:36.420362+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:08:05.220846+0000","scrub_duration":0.028838444000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":322,"reported_epoch":377,"state":"active+clean","last_fresh":"2023-09-23T21:08:12.368229+0000","last_change":"2023-09-23T20:56:45.991822+0000","last_active":"2023-09-23T21:08:12.368229+0000","last_peered":"2023-09-23T21:08:12.368229+0000","last_clean":"2023-09-23T21:08:12.368229+0000","last_became_active":"2023-09-23T20:56:45.991460+0000","last_became_peered":"2023-09-23T20:56:45.991460+0000","last_unstale":"2023-09-23T21:08:12.368229+0000","last_undegraded":"2023-09-23T21:08:12.368229+0000","last_fullsized":"2023-09-23T21:08:12.368229+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":205,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:20:37.503317+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":338,"reported_epoch":377,"state":"active+clean","last_fresh":"2023-09-23T21:08:12.383675+0000","last_change":"2023-09-23T20:51:10.840849+0000","last_active":"2023-09-23T21:08:12.383675+0000","last_peered":"2023-09-23T21:08:12.383675+0000","last_clean":"2023-09-23T21:08:12.383675+0000","last_became_active":"2023-09-23T20:51:10.840593+0000","last_became_peered":"2023-09-23T20:51:10.840593+0000","last_unstale":"2023-09-23T21:08:12.383675+0000","last_undegraded":"2023-09-23T21:08:12.383675+0000","last_fullsized":"2023-09-23T21:08:12.383675+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:30:52.297625+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.12","version":"79'161","reported_seq":539,"reported_epoch":377,"state":"active+clean","last_fresh":"2023-09-23T21:08:12.379098+0000","last_change":"2023-09-23T21:05:56.785264+0000","last_active":"2023-09-23T21:08:12.379098+0000","last_peered":"2023-09-23T21:08:12.379098+0000","last_clean":"2023-09-23T21:08:12.379098+0000","last_became_active":"2023-09-23T21:05:55.322810+0000","last_became_peered":"2023-09-23T21:05:55.322810+0000","last_unstale":"2023-09-23T21:08:12.379098+0000","last_undegraded":"2023-09-23T21:08:12.379098+0000","last_fullsized":"2023-09-23T21:08:12.379098+0000","mapping_epoch":350,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":351,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:05:56.785173+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:05:56.785173+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:56:12.932786+0000","scrub_duration":0.0042114370000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":410,"reported_epoch":377,"state":"active+clean","last_fresh":"2023-09-23T21:08:12.367456+0000","last_change":"2023-09-23T21:05:04.332774+0000","last_active":"2023-09-23T21:08:12.367456+0000","last_peered":"2023-09-23T21:08:12.367456+0000","last_clean":"2023-09-23T21:08:12.367456+0000","last_became_active":"2023-09-23T21:05:04.304541+0000","last_became_peered":"2023-09-23T21:05:04.304541+0000","last_unstale":"2023-09-23T21:08:12.367456+0000","last_undegraded":"2023-09-23T21:08:12.367456+0000","last_fullsized":"2023-09-23T21:08:12.367456+0000","mapping_epoch":332,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:27:02.401939+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":185,"reported_epoch":377,"state":"active+clean","last_fresh":"2023-09-23T21:08:12.382867+0000","last_change":"2023-09-23T21:05:55.325421+0000","last_active":"2023-09-23T21:08:12.382867+0000","last_peered":"2023-09-23T21:08:12.382867+0000","last_clean":"2023-09-23T21:08:12.382867+0000","last_became_active":"2023-09-23T21:05:55.325098+0000","last_became_peered":"2023-09-23T21:05:55.325098+0000","last_unstale":"2023-09-23T21:08:12.382867+0000","last_undegraded":"2023-09-23T21:08:12.382867+0000","last_fullsized":"2023-09-23T21:08:12.382867+0000","mapping_epoch":350,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":351,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:57:47.564327+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.5","version":"378'9832","reported_seq":9874,"reported_epoch":378,"state":"active+clean","last_fresh":"2023-09-23T21:08:22.316545+0000","last_change":"2023-09-23T21:05:48.557604+0000","last_active":"2023-09-23T21:08:22.316545+0000","last_peered":"2023-09-23T21:08:22.316545+0000","last_clean":"2023-09-23T21:08:22.316545+0000","last_became_active":"2023-09-23T21:05:48.557179+0000","last_became_peered":"2023-09-23T21:05:48.557179+0000","last_unstale":"2023-09-23T21:08:22.316545+0000","last_undegraded":"2023-09-23T21:08:22.316545+0000","last_fullsized":"2023-09-23T21:08:22.316545+0000","mapping_epoch":346,"log_start":"351'1400","ondisk_log_start":"351'1400","created":346,"last_epoch_clean":347,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:05:47.261750+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:05:47.261750+0000","last_clean_scrub_stamp":"2023-09-23T21:05:47.261750+0000","objects_scrubbed":0,"log_size":8432,"log_dups_size":0,"ondisk_log_size":8432,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:17:18.270981+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":275775488,"num_objects":4208,"num_object_clones":0,"num_object_copies":12624,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4208,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9832,"num_write_kb":449280,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.d","version":"79'161","reported_seq":515,"reported_epoch":377,"state":"active+clean","last_fresh":"2023-09-23T21:08:12.379874+0000","last_change":"2023-09-23T21:06:38.762807+0000","last_active":"2023-09-23T21:08:12.379874+0000","last_peered":"2023-09-23T21:08:12.379874+0000","last_clean":"2023-09-23T21:08:12.379874+0000","last_became_active":"2023-09-23T21:06:25.394249+0000","last_became_peered":"2023-09-23T21:06:25.394249+0000","last_unstale":"2023-09-23T21:08:12.379874+0000","last_undegraded":"2023-09-23T21:08:12.379874+0000","last_fullsized":"2023-09-23T21:08:12.379874+0000","mapping_epoch":354,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":355,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:06:38.762742+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:06:38.762742+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:28:31.666726+0000","scrub_duration":0.0053529800000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":241,"reported_epoch":377,"state":"active+clean","last_fresh":"2023-09-23T21:08:12.379509+0000","last_change":"2023-09-23T21:06:25.394656+0000","last_active":"2023-09-23T21:08:12.379509+0000","last_peered":"2023-09-23T21:08:12.379509+0000","last_clean":"2023-09-23T21:08:12.379509+0000","last_became_active":"2023-09-23T21:06:25.394290+0000","last_became_peered":"2023-09-23T21:06:25.394290+0000","last_unstale":"2023-09-23T21:08:12.379509+0000","last_undegraded":"2023-09-23T21:08:12.379509+0000","last_fullsized":"2023-09-23T21:08:12.379509+0000","mapping_epoch":354,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":355,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:27:19.996925+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":66,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":55,"reported_epoch":377,"state":"active+clean","last_fresh":"2023-09-23T21:08:12.378675+0000","last_change":"2023-09-23T21:05:04.307967+0000","last_active":"2023-09-23T21:08:12.378675+0000","last_peered":"2023-09-23T21:08:12.378675+0000","last_clean":"2023-09-23T21:08:12.378675+0000","last_became_active":"2023-09-23T21:05:04.307679+0000","last_became_peered":"2023-09-23T21:05:04.307679+0000","last_unstale":"2023-09-23T21:08:12.378675+0000","last_undegraded":"2023-09-23T21:08:12.378675+0000","last_fullsized":"2023-09-23T21:08:12.378675+0000","mapping_epoch":332,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:39:58.603484+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.4","version":"378'9859","reported_seq":9898,"reported_epoch":378,"state":"active+clean","last_fresh":"2023-09-23T21:08:23.424576+0000","last_change":"2023-09-23T21:05:55.320259+0000","last_active":"2023-09-23T21:08:23.424576+0000","last_peered":"2023-09-23T21:08:23.424576+0000","last_clean":"2023-09-23T21:08:23.424576+0000","last_became_active":"2023-09-23T21:05:55.319895+0000","last_became_peered":"2023-09-23T21:05:55.319895+0000","last_unstale":"2023-09-23T21:08:23.424576+0000","last_undegraded":"2023-09-23T21:08:23.424576+0000","last_fullsized":"2023-09-23T21:08:23.424576+0000","mapping_epoch":350,"log_start":"355'3300","ondisk_log_start":"355'3300","created":346,"last_epoch_clean":351,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:05:47.261750+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:05:47.261750+0000","last_clean_scrub_stamp":"2023-09-23T21:05:47.261750+0000","objects_scrubbed":0,"log_size":6559,"log_dups_size":0,"ondisk_log_size":6559,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:39:53.181523+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":272039936,"num_objects":4151,"num_object_clones":0,"num_object_copies":12453,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4151,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9859,"num_write_kb":448320,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.c","version":"79'161","reported_seq":683,"reported_epoch":378,"state":"active+clean","last_fresh":"2023-09-23T21:08:16.385489+0000","last_change":"2023-09-23T20:57:22.041071+0000","last_active":"2023-09-23T21:08:16.385489+0000","last_peered":"2023-09-23T21:08:16.385489+0000","last_clean":"2023-09-23T21:08:16.385489+0000","last_became_active":"2023-09-23T20:57:15.973204+0000","last_became_peered":"2023-09-23T20:57:15.973204+0000","last_unstale":"2023-09-23T21:08:16.385489+0000","last_undegraded":"2023-09-23T21:08:16.385489+0000","last_fullsized":"2023-09-23T21:08:16.385489+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:22.040907+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:22.040907+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:47:02.019978+0000","scrub_duration":0.013021176000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":228,"reported_epoch":377,"state":"active+clean","last_fresh":"2023-09-23T21:08:12.379563+0000","last_change":"2023-09-23T21:06:25.394978+0000","last_active":"2023-09-23T21:08:12.379563+0000","last_peered":"2023-09-23T21:08:12.379563+0000","last_clean":"2023-09-23T21:08:12.379563+0000","last_became_active":"2023-09-23T21:06:25.394599+0000","last_became_peered":"2023-09-23T21:06:25.394599+0000","last_unstale":"2023-09-23T21:08:12.379563+0000","last_undegraded":"2023-09-23T21:08:12.379563+0000","last_fullsized":"2023-09-23T21:08:12.379563+0000","mapping_epoch":354,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":355,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:52:23.477998+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":316,"reported_epoch":377,"state":"active+clean","last_fresh":"2023-09-23T21:08:12.368400+0000","last_change":"2023-09-23T21:05:55.325375+0000","last_active":"2023-09-23T21:08:12.368400+0000","last_peered":"2023-09-23T21:08:12.368400+0000","last_clean":"2023-09-23T21:08:12.368400+0000","last_became_active":"2023-09-23T21:05:55.324170+0000","last_became_peered":"2023-09-23T21:05:55.324170+0000","last_unstale":"2023-09-23T21:08:12.368400+0000","last_undegraded":"2023-09-23T21:08:12.368400+0000","last_fullsized":"2023-09-23T21:08:12.368400+0000","mapping_epoch":350,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":351,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:08:07.299605+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.7","version":"378'10043","reported_seq":10091,"reported_epoch":378,"state":"active+clean","last_fresh":"2023-09-23T21:08:23.976609+0000","last_change":"2023-09-23T21:06:49.432007+0000","last_active":"2023-09-23T21:08:23.976609+0000","last_peered":"2023-09-23T21:08:23.976609+0000","last_clean":"2023-09-23T21:08:23.976609+0000","last_became_active":"2023-09-23T21:06:49.425988+0000","last_became_peered":"2023-09-23T21:06:49.425988+0000","last_unstale":"2023-09-23T21:08:23.976609+0000","last_undegraded":"2023-09-23T21:08:23.976609+0000","last_fullsized":"2023-09-23T21:08:23.976609+0000","mapping_epoch":359,"log_start":"356'3700","ondisk_log_start":"356'3700","created":346,"last_epoch_clean":360,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:05:47.261750+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:05:47.261750+0000","last_clean_scrub_stamp":"2023-09-23T21:05:47.261750+0000","objects_scrubbed":0,"log_size":6343,"log_dups_size":0,"ondisk_log_size":6343,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:37:23.206892+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":268500992,"num_objects":4097,"num_object_clones":0,"num_object_copies":12291,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4097,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10043,"num_write_kb":452480,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.f","version":"79'161","reported_seq":722,"reported_epoch":377,"state":"active+clean","last_fresh":"2023-09-23T21:08:12.367949+0000","last_change":"2023-09-23T21:05:07.299402+0000","last_active":"2023-09-23T21:08:12.367949+0000","last_peered":"2023-09-23T21:08:12.367949+0000","last_clean":"2023-09-23T21:08:12.367949+0000","last_became_active":"2023-09-23T21:05:07.299059+0000","last_became_peered":"2023-09-23T21:05:07.299059+0000","last_unstale":"2023-09-23T21:08:12.367949+0000","last_undegraded":"2023-09-23T21:08:12.367949+0000","last_fullsized":"2023-09-23T21:08:12.367949+0000","mapping_epoch":335,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":336,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:27.880776+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:27.880776+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:51:23.907187+0000","scrub_duration":0.0014004899999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":59,"reported_epoch":377,"state":"active+clean","last_fresh":"2023-09-23T21:08:12.379468+0000","last_change":"2023-09-23T21:05:04.355129+0000","last_active":"2023-09-23T21:08:12.379468+0000","last_peered":"2023-09-23T21:08:12.379468+0000","last_clean":"2023-09-23T21:08:12.379468+0000","last_became_active":"2023-09-23T21:05:04.326482+0000","last_became_peered":"2023-09-23T21:05:04.326482+0000","last_unstale":"2023-09-23T21:08:12.379468+0000","last_undegraded":"2023-09-23T21:08:12.379468+0000","last_fullsized":"2023-09-23T21:08:12.379468+0000","mapping_epoch":332,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:35:10.887749+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":334,"reported_epoch":377,"state":"active+clean","last_fresh":"2023-09-23T21:08:12.367571+0000","last_change":"2023-09-23T21:05:55.321246+0000","last_active":"2023-09-23T21:08:12.367571+0000","last_peered":"2023-09-23T21:08:12.367571+0000","last_clean":"2023-09-23T21:08:12.367571+0000","last_became_active":"2023-09-23T21:05:55.320918+0000","last_became_peered":"2023-09-23T21:05:55.320918+0000","last_unstale":"2023-09-23T21:08:12.367571+0000","last_undegraded":"2023-09-23T21:08:12.367571+0000","last_fullsized":"2023-09-23T21:08:12.367571+0000","mapping_epoch":350,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":351,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:48:50.646324+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"9.6","version":"378'9846","reported_seq":9901,"reported_epoch":378,"state":"active+clean","last_fresh":"2023-09-23T21:08:21.051731+0000","last_change":"2023-09-23T21:06:49.429119+0000","last_active":"2023-09-23T21:08:21.051731+0000","last_peered":"2023-09-23T21:08:21.051731+0000","last_clean":"2023-09-23T21:08:21.051731+0000","last_became_active":"2023-09-23T21:06:49.423016+0000","last_became_peered":"2023-09-23T21:06:49.423016+0000","last_unstale":"2023-09-23T21:08:21.051731+0000","last_undegraded":"2023-09-23T21:08:21.051731+0000","last_fullsized":"2023-09-23T21:08:21.051731+0000","mapping_epoch":359,"log_start":"0'0","ondisk_log_start":"0'0","created":346,"last_epoch_clean":360,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:05:47.261750+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:05:47.261750+0000","last_clean_scrub_stamp":"2023-09-23T21:05:47.261750+0000","objects_scrubbed":0,"log_size":9846,"log_dups_size":0,"ondisk_log_size":9846,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:55:25.118949+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":278659072,"num_objects":4252,"num_object_clones":0,"num_object_copies":12756,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4252,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9846,"num_write_kb":451136,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,1],"acting":[2,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.e","version":"79'161","reported_seq":589,"reported_epoch":377,"state":"active+clean","last_fresh":"2023-09-23T21:08:12.368582+0000","last_change":"2023-09-23T20:57:34.132738+0000","last_active":"2023-09-23T21:08:12.368582+0000","last_peered":"2023-09-23T21:08:12.368582+0000","last_clean":"2023-09-23T21:08:12.368582+0000","last_became_active":"2023-09-23T20:57:15.974318+0000","last_became_peered":"2023-09-23T20:57:15.974318+0000","last_unstale":"2023-09-23T21:08:12.368582+0000","last_undegraded":"2023-09-23T21:08:12.368582+0000","last_fullsized":"2023-09-23T21:08:12.368582+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:34.132660+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:34.132660+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:07:07.739438+0000","scrub_duration":0.005229694,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":143,"reported_epoch":377,"state":"active+clean","last_fresh":"2023-09-23T21:08:12.387573+0000","last_change":"2023-09-23T21:06:49.423445+0000","last_active":"2023-09-23T21:08:12.387573+0000","last_peered":"2023-09-23T21:08:12.387573+0000","last_clean":"2023-09-23T21:08:12.387573+0000","last_became_active":"2023-09-23T21:06:49.423169+0000","last_became_peered":"2023-09-23T21:06:49.423169+0000","last_unstale":"2023-09-23T21:08:12.387573+0000","last_undegraded":"2023-09-23T21:08:12.387573+0000","last_fullsized":"2023-09-23T21:08:12.387573+0000","mapping_epoch":359,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":360,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:47:52.999823+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":370,"reported_epoch":377,"state":"active+clean","last_fresh":"2023-09-23T21:08:12.383903+0000","last_change":"2023-09-23T20:54:13.928715+0000","last_active":"2023-09-23T21:08:12.383903+0000","last_peered":"2023-09-23T21:08:12.383903+0000","last_clean":"2023-09-23T21:08:12.383903+0000","last_became_active":"2023-09-23T20:54:13.928200+0000","last_became_peered":"2023-09-23T20:54:13.928200+0000","last_unstale":"2023-09-23T21:08:12.383903+0000","last_undegraded":"2023-09-23T21:08:12.383903+0000","last_fullsized":"2023-09-23T21:08:12.383903+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:41:16.927513+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.1","version":"378'10111","reported_seq":10153,"reported_epoch":378,"state":"active+clean","last_fresh":"2023-09-23T21:08:23.986760+0000","last_change":"2023-09-23T21:05:48.558005+0000","last_active":"2023-09-23T21:08:23.986760+0000","last_peered":"2023-09-23T21:08:23.986760+0000","last_clean":"2023-09-23T21:08:23.986760+0000","last_became_active":"2023-09-23T21:05:48.557590+0000","last_became_peered":"2023-09-23T21:05:48.557590+0000","last_unstale":"2023-09-23T21:08:23.986760+0000","last_undegraded":"2023-09-23T21:08:23.986760+0000","last_fullsized":"2023-09-23T21:08:23.986760+0000","mapping_epoch":346,"log_start":"356'3800","ondisk_log_start":"356'3800","created":346,"last_epoch_clean":347,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:05:47.261750+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:05:47.261750+0000","last_clean_scrub_stamp":"2023-09-23T21:05:47.261750+0000","objects_scrubbed":0,"log_size":6311,"log_dups_size":0,"ondisk_log_size":6311,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:19:41.388978+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":277151744,"num_objects":4229,"num_object_clones":0,"num_object_copies":12687,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4229,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10111,"num_write_kb":458880,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.9","version":"79'161","reported_seq":755,"reported_epoch":377,"state":"active+clean","last_fresh":"2023-09-23T21:08:12.381219+0000","last_change":"2023-09-23T20:56:46.921939+0000","last_active":"2023-09-23T21:08:12.381219+0000","last_peered":"2023-09-23T21:08:12.381219+0000","last_clean":"2023-09-23T21:08:12.381219+0000","last_became_active":"2023-09-23T20:56:45.989908+0000","last_became_peered":"2023-09-23T20:56:45.989908+0000","last_unstale":"2023-09-23T21:08:12.381219+0000","last_undegraded":"2023-09-23T21:08:12.381219+0000","last_fullsized":"2023-09-23T21:08:12.381219+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":205,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:56:46.921739+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:56:46.921739+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:20:42.635110+0000","scrub_duration":0.0071954109999999996,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":371,"reported_epoch":377,"state":"active+clean","last_fresh":"2023-09-23T21:08:12.384335+0000","last_change":"2023-09-23T20:54:13.961788+0000","last_active":"2023-09-23T21:08:12.384335+0000","last_peered":"2023-09-23T21:08:12.384335+0000","last_clean":"2023-09-23T21:08:12.384335+0000","last_became_active":"2023-09-23T20:54:13.940937+0000","last_became_peered":"2023-09-23T20:54:13.940937+0000","last_unstale":"2023-09-23T21:08:12.384335+0000","last_undegraded":"2023-09-23T21:08:12.384335+0000","last_fullsized":"2023-09-23T21:08:12.384335+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:11:56.785299+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":272,"reported_epoch":377,"state":"active+clean","last_fresh":"2023-09-23T21:08:12.368813+0000","last_change":"2023-09-23T20:55:35.949670+0000","last_active":"2023-09-23T21:08:12.368813+0000","last_peered":"2023-09-23T21:08:12.368813+0000","last_clean":"2023-09-23T21:08:12.368813+0000","last_became_active":"2023-09-23T20:55:35.949140+0000","last_became_peered":"2023-09-23T20:55:35.949140+0000","last_unstale":"2023-09-23T21:08:12.368813+0000","last_undegraded":"2023-09-23T21:08:12.368813+0000","last_fullsized":"2023-09-23T21:08:12.368813+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:55:59.340303+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.0","version":"378'10073","reported_seq":10112,"reported_epoch":378,"state":"active+clean","last_fresh":"2023-09-23T21:08:23.467772+0000","last_change":"2023-09-23T21:05:55.315757+0000","last_active":"2023-09-23T21:08:23.467772+0000","last_peered":"2023-09-23T21:08:23.467772+0000","last_clean":"2023-09-23T21:08:23.467772+0000","last_became_active":"2023-09-23T21:05:55.314884+0000","last_became_peered":"2023-09-23T21:05:55.314884+0000","last_unstale":"2023-09-23T21:08:23.467772+0000","last_undegraded":"2023-09-23T21:08:23.467772+0000","last_fullsized":"2023-09-23T21:08:23.467772+0000","mapping_epoch":350,"log_start":"355'3500","ondisk_log_start":"355'3500","created":346,"last_epoch_clean":351,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:05:47.261750+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:05:47.261750+0000","last_clean_scrub_stamp":"2023-09-23T21:05:47.261750+0000","objects_scrubbed":0,"log_size":6573,"log_dups_size":0,"ondisk_log_size":6573,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:41:19.184012+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":267845632,"num_objects":4087,"num_object_clones":0,"num_object_copies":12261,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4087,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10073,"num_write_kb":453120,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.8","version":"79'161","reported_seq":486,"reported_epoch":377,"state":"active+clean","last_fresh":"2023-09-23T21:08:12.375825+0000","last_change":"2023-09-23T21:06:52.099024+0000","last_active":"2023-09-23T21:08:12.375825+0000","last_peered":"2023-09-23T21:08:12.375825+0000","last_clean":"2023-09-23T21:08:12.375825+0000","last_became_active":"2023-09-23T21:06:49.425012+0000","last_became_peered":"2023-09-23T21:06:49.425012+0000","last_unstale":"2023-09-23T21:08:12.375825+0000","last_undegraded":"2023-09-23T21:08:12.375825+0000","last_fullsized":"2023-09-23T21:08:12.375825+0000","mapping_epoch":359,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":360,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:06:52.098882+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:06:52.098882+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:48:36.414396+0000","scrub_duration":0.0092173299999999993,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":357,"reported_epoch":377,"state":"active+clean","last_fresh":"2023-09-23T21:08:12.367722+0000","last_change":"2023-09-23T21:05:55.321230+0000","last_active":"2023-09-23T21:08:12.367722+0000","last_peered":"2023-09-23T21:08:12.367722+0000","last_clean":"2023-09-23T21:08:12.367722+0000","last_became_active":"2023-09-23T21:05:55.320887+0000","last_became_peered":"2023-09-23T21:05:55.320887+0000","last_unstale":"2023-09-23T21:08:12.367722+0000","last_undegraded":"2023-09-23T21:08:12.367722+0000","last_fullsized":"2023-09-23T21:08:12.367722+0000","mapping_epoch":350,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":351,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:54:10.329947+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":272,"reported_epoch":377,"state":"active+clean","last_fresh":"2023-09-23T21:08:12.368869+0000","last_change":"2023-09-23T21:05:55.322692+0000","last_active":"2023-09-23T21:08:12.368869+0000","last_peered":"2023-09-23T21:08:12.368869+0000","last_clean":"2023-09-23T21:08:12.368869+0000","last_became_active":"2023-09-23T21:05:55.322388+0000","last_became_peered":"2023-09-23T21:05:55.322388+0000","last_unstale":"2023-09-23T21:08:12.368869+0000","last_undegraded":"2023-09-23T21:08:12.368869+0000","last_fullsized":"2023-09-23T21:08:12.368869+0000","mapping_epoch":350,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":351,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:43:27.774990+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.3","version":"378'9746","reported_seq":9801,"reported_epoch":378,"state":"active+clean","last_fresh":"2023-09-23T21:08:22.144369+0000","last_change":"2023-09-23T21:06:25.421271+0000","last_active":"2023-09-23T21:08:22.144369+0000","last_peered":"2023-09-23T21:08:22.144369+0000","last_clean":"2023-09-23T21:08:22.144369+0000","last_became_active":"2023-09-23T21:06:25.400342+0000","last_became_peered":"2023-09-23T21:06:25.400342+0000","last_unstale":"2023-09-23T21:08:22.144369+0000","last_undegraded":"2023-09-23T21:08:22.144369+0000","last_fullsized":"2023-09-23T21:08:22.144369+0000","mapping_epoch":354,"log_start":"355'3300","ondisk_log_start":"355'3300","created":346,"last_epoch_clean":355,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:05:47.261750+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:05:47.261750+0000","last_clean_scrub_stamp":"2023-09-23T21:05:47.261750+0000","objects_scrubbed":0,"log_size":6446,"log_dups_size":0,"ondisk_log_size":6446,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:32:39.644536+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":278921216,"num_objects":4256,"num_object_clones":0,"num_object_copies":12768,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4256,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9746,"num_write_kb":448064,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.b","version":"79'161","reported_seq":708,"reported_epoch":377,"state":"active+clean","last_fresh":"2023-09-23T21:08:12.382095+0000","last_change":"2023-09-23T21:05:55.772369+0000","last_active":"2023-09-23T21:08:12.382095+0000","last_peered":"2023-09-23T21:08:12.382095+0000","last_clean":"2023-09-23T21:08:12.382095+0000","last_became_active":"2023-09-23T21:05:55.316405+0000","last_became_peered":"2023-09-23T21:05:55.316405+0000","last_unstale":"2023-09-23T21:08:12.382095+0000","last_undegraded":"2023-09-23T21:08:12.382095+0000","last_fullsized":"2023-09-23T21:08:12.382095+0000","mapping_epoch":350,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":351,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:05:55.772264+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:05:55.772264+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:14:30.697692+0000","scrub_duration":0.001862507,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,5],"acting":[1,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":255,"reported_epoch":377,"state":"active+clean","last_fresh":"2023-09-23T21:08:12.368297+0000","last_change":"2023-09-23T20:54:13.950254+0000","last_active":"2023-09-23T21:08:12.368297+0000","last_peered":"2023-09-23T21:08:12.368297+0000","last_clean":"2023-09-23T21:08:12.368297+0000","last_became_active":"2023-09-23T20:54:13.949934+0000","last_became_peered":"2023-09-23T20:54:13.949934+0000","last_unstale":"2023-09-23T21:08:12.368297+0000","last_undegraded":"2023-09-23T21:08:12.368297+0000","last_fullsized":"2023-09-23T21:08:12.368297+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:30:32.847167+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":289,"reported_epoch":377,"state":"active+clean","last_fresh":"2023-09-23T21:08:12.368680+0000","last_change":"2023-09-23T21:05:55.325695+0000","last_active":"2023-09-23T21:08:12.368680+0000","last_peered":"2023-09-23T21:08:12.368680+0000","last_clean":"2023-09-23T21:08:12.368680+0000","last_became_active":"2023-09-23T21:05:55.324518+0000","last_became_peered":"2023-09-23T21:05:55.324518+0000","last_unstale":"2023-09-23T21:08:12.368680+0000","last_undegraded":"2023-09-23T21:08:12.368680+0000","last_fullsized":"2023-09-23T21:08:12.368680+0000","mapping_epoch":350,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":351,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:57:18.249965+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.2","version":"378'9922","reported_seq":9970,"reported_epoch":378,"state":"active+clean","last_fresh":"2023-09-23T21:08:24.010456+0000","last_change":"2023-09-23T21:05:55.325629+0000","last_active":"2023-09-23T21:08:24.010456+0000","last_peered":"2023-09-23T21:08:24.010456+0000","last_clean":"2023-09-23T21:08:24.010456+0000","last_became_active":"2023-09-23T21:05:55.325108+0000","last_became_peered":"2023-09-23T21:05:55.325108+0000","last_unstale":"2023-09-23T21:08:24.010456+0000","last_undegraded":"2023-09-23T21:08:24.010456+0000","last_fullsized":"2023-09-23T21:08:24.010456+0000","mapping_epoch":350,"log_start":"356'3600","ondisk_log_start":"356'3600","created":346,"last_epoch_clean":351,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:05:47.261750+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:05:47.261750+0000","last_clean_scrub_stamp":"2023-09-23T21:05:47.261750+0000","objects_scrubbed":0,"log_size":6322,"log_dups_size":0,"ondisk_log_size":6322,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:50:06.272050+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":275644416,"num_objects":4206,"num_object_clones":0,"num_object_copies":12618,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4206,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9922,"num_write_kb":452096,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.a","version":"79'161","reported_seq":478,"reported_epoch":377,"state":"active+clean","last_fresh":"2023-09-23T21:08:12.377057+0000","last_change":"2023-09-23T21:06:51.075617+0000","last_active":"2023-09-23T21:08:12.377057+0000","last_peered":"2023-09-23T21:08:12.377057+0000","last_clean":"2023-09-23T21:08:12.377057+0000","last_became_active":"2023-09-23T21:06:49.423291+0000","last_became_peered":"2023-09-23T21:06:49.423291+0000","last_unstale":"2023-09-23T21:08:12.377057+0000","last_undegraded":"2023-09-23T21:08:12.377057+0000","last_fullsized":"2023-09-23T21:08:12.377057+0000","mapping_epoch":359,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":360,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:06:51.075448+0000","last_deep_scrub":"79'161","last_deep_scrub_stamp":"2023-09-23T21:05:32.630292+0000","last_clean_scrub_stamp":"2023-09-23T21:06:51.075448+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:35:01.883490+0000","scrub_duration":0.0054793890000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":185,"reported_epoch":377,"state":"active+clean","last_fresh":"2023-09-23T21:08:12.380806+0000","last_change":"2023-09-23T21:05:55.325999+0000","last_active":"2023-09-23T21:08:12.380806+0000","last_peered":"2023-09-23T21:08:12.380806+0000","last_clean":"2023-09-23T21:08:12.380806+0000","last_became_active":"2023-09-23T21:05:55.325669+0000","last_became_peered":"2023-09-23T21:05:55.325669+0000","last_unstale":"2023-09-23T21:08:12.380806+0000","last_undegraded":"2023-09-23T21:08:12.380806+0000","last_fullsized":"2023-09-23T21:08:12.380806+0000","mapping_epoch":350,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":351,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:30:53.984244+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":351,"reported_epoch":377,"state":"active+clean","last_fresh":"2023-09-23T21:08:12.382396+0000","last_change":"2023-09-23T21:05:55.322566+0000","last_active":"2023-09-23T21:08:12.382396+0000","last_peered":"2023-09-23T21:08:12.382396+0000","last_clean":"2023-09-23T21:08:12.382396+0000","last_became_active":"2023-09-23T21:05:55.322280+0000","last_became_peered":"2023-09-23T21:05:55.322280+0000","last_unstale":"2023-09-23T21:08:12.382396+0000","last_undegraded":"2023-09-23T21:08:12.382396+0000","last_fullsized":"2023-09-23T21:08:12.382396+0000","mapping_epoch":350,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":351,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:48:14.297230+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.f","version":"378'9865","reported_seq":9912,"reported_epoch":378,"state":"active+clean","last_fresh":"2023-09-23T21:08:22.192597+0000","last_change":"2023-09-23T21:05:55.325083+0000","last_active":"2023-09-23T21:08:22.192597+0000","last_peered":"2023-09-23T21:08:22.192597+0000","last_clean":"2023-09-23T21:08:22.192597+0000","last_became_active":"2023-09-23T21:05:55.324768+0000","last_became_peered":"2023-09-23T21:05:55.324768+0000","last_unstale":"2023-09-23T21:08:22.192597+0000","last_undegraded":"2023-09-23T21:08:22.192597+0000","last_fullsized":"2023-09-23T21:08:22.192597+0000","mapping_epoch":350,"log_start":"351'1900","ondisk_log_start":"351'1900","created":346,"last_epoch_clean":351,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:05:47.261750+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:05:47.261750+0000","last_clean_scrub_stamp":"2023-09-23T21:05:47.261750+0000","objects_scrubbed":0,"log_size":7965,"log_dups_size":0,"ondisk_log_size":7965,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:25:42.316668+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":281214976,"num_objects":4291,"num_object_clones":0,"num_object_copies":12873,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4291,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9865,"num_write_kb":452992,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.7","version":"79'161","reported_seq":683,"reported_epoch":377,"state":"active+clean","last_fresh":"2023-09-23T21:08:12.367777+0000","last_change":"2023-09-23T20:57:48.243572+0000","last_active":"2023-09-23T21:08:12.367777+0000","last_peered":"2023-09-23T21:08:12.367777+0000","last_clean":"2023-09-23T21:08:12.367777+0000","last_became_active":"2023-09-23T20:57:48.243229+0000","last_became_peered":"2023-09-23T20:57:48.243229+0000","last_unstale":"2023-09-23T21:08:12.367777+0000","last_undegraded":"2023-09-23T21:08:12.367777+0000","last_fullsized":"2023-09-23T21:08:12.367777+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":3,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:25.916867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:25.916867+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:23:19.385831+0000","scrub_duration":0.001665323,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":355,"reported_epoch":377,"state":"active+clean","last_fresh":"2023-09-23T21:08:12.387666+0000","last_change":"2023-09-23T21:05:55.317140+0000","last_active":"2023-09-23T21:08:12.387666+0000","last_peered":"2023-09-23T21:08:12.387666+0000","last_clean":"2023-09-23T21:08:12.387666+0000","last_became_active":"2023-09-23T21:05:55.316187+0000","last_became_peered":"2023-09-23T21:05:55.316187+0000","last_unstale":"2023-09-23T21:08:12.387666+0000","last_undegraded":"2023-09-23T21:08:12.387666+0000","last_fullsized":"2023-09-23T21:08:12.387666+0000","mapping_epoch":350,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":351,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:34:57.248381+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":55,"reported_epoch":377,"state":"active+clean","last_fresh":"2023-09-23T21:08:12.379545+0000","last_change":"2023-09-23T21:05:04.326889+0000","last_active":"2023-09-23T21:08:12.379545+0000","last_peered":"2023-09-23T21:08:12.379545+0000","last_clean":"2023-09-23T21:08:12.379545+0000","last_became_active":"2023-09-23T21:05:04.326032+0000","last_became_peered":"2023-09-23T21:05:04.326032+0000","last_unstale":"2023-09-23T21:08:12.379545+0000","last_undegraded":"2023-09-23T21:08:12.379545+0000","last_fullsized":"2023-09-23T21:08:12.379545+0000","mapping_epoch":332,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:00:17.507889+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":368,"reported_epoch":377,"state":"active+clean","last_fresh":"2023-09-23T21:08:12.382036+0000","last_change":"2023-09-23T21:05:55.319889+0000","last_active":"2023-09-23T21:08:12.382036+0000","last_peered":"2023-09-23T21:08:12.382036+0000","last_clean":"2023-09-23T21:08:12.382036+0000","last_became_active":"2023-09-23T21:05:55.319400+0000","last_became_peered":"2023-09-23T21:05:55.319400+0000","last_unstale":"2023-09-23T21:08:12.382036+0000","last_undegraded":"2023-09-23T21:08:12.382036+0000","last_fullsized":"2023-09-23T21:08:12.382036+0000","mapping_epoch":350,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":351,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:21:17.483215+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.9","version":"378'9993","reported_seq":10035,"reported_epoch":378,"state":"active+clean","last_fresh":"2023-09-23T21:08:22.293679+0000","last_change":"2023-09-23T21:05:48.557743+0000","last_active":"2023-09-23T21:08:22.293679+0000","last_peered":"2023-09-23T21:08:22.293679+0000","last_clean":"2023-09-23T21:08:22.293679+0000","last_became_active":"2023-09-23T21:05:48.557419+0000","last_became_peered":"2023-09-23T21:05:48.557419+0000","last_unstale":"2023-09-23T21:08:22.293679+0000","last_undegraded":"2023-09-23T21:08:22.293679+0000","last_fullsized":"2023-09-23T21:08:22.293679+0000","mapping_epoch":346,"log_start":"351'1600","ondisk_log_start":"351'1600","created":346,"last_epoch_clean":347,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:05:47.261750+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:05:47.261750+0000","last_clean_scrub_stamp":"2023-09-23T21:05:47.261750+0000","objects_scrubbed":0,"log_size":8393,"log_dups_size":0,"ondisk_log_size":8393,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:34:16.622408+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":286720000,"num_objects":4375,"num_object_clones":0,"num_object_copies":13125,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4375,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9993,"num_write_kb":459776,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,5],"acting":[0,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1","version":"79'161","reported_seq":485,"reported_epoch":377,"state":"active+clean","last_fresh":"2023-09-23T21:08:12.383459+0000","last_change":"2023-09-23T21:06:52.576036+0000","last_active":"2023-09-23T21:08:12.383459+0000","last_peered":"2023-09-23T21:08:12.383459+0000","last_clean":"2023-09-23T21:08:12.383459+0000","last_became_active":"2023-09-23T21:06:49.422806+0000","last_became_peered":"2023-09-23T21:06:49.422806+0000","last_unstale":"2023-09-23T21:08:12.383459+0000","last_undegraded":"2023-09-23T21:08:12.383459+0000","last_fullsized":"2023-09-23T21:08:12.383459+0000","mapping_epoch":359,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":360,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:06:52.575900+0000","last_deep_scrub":"79'161","last_deep_scrub_stamp":"2023-09-23T21:05:30.652189+0000","last_clean_scrub_stamp":"2023-09-23T21:06:52.575900+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:59:24.733331+0000","scrub_duration":0.017723466,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":137,"reported_epoch":377,"state":"active+clean","last_fresh":"2023-09-23T21:08:12.381800+0000","last_change":"2023-09-23T21:06:49.426087+0000","last_active":"2023-09-23T21:08:12.381800+0000","last_peered":"2023-09-23T21:08:12.381800+0000","last_clean":"2023-09-23T21:08:12.381800+0000","last_became_active":"2023-09-23T21:06:49.425698+0000","last_became_peered":"2023-09-23T21:06:49.425698+0000","last_unstale":"2023-09-23T21:08:12.381800+0000","last_undegraded":"2023-09-23T21:08:12.381800+0000","last_fullsized":"2023-09-23T21:08:12.381800+0000","mapping_epoch":359,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":360,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:51:47.815811+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":272,"reported_epoch":377,"state":"active+clean","last_fresh":"2023-09-23T21:08:12.368935+0000","last_change":"2023-09-23T20:55:35.949000+0000","last_active":"2023-09-23T21:08:12.368935+0000","last_peered":"2023-09-23T21:08:12.368935+0000","last_clean":"2023-09-23T21:08:12.368935+0000","last_became_active":"2023-09-23T20:55:35.948410+0000","last_became_peered":"2023-09-23T20:55:35.948410+0000","last_unstale":"2023-09-23T21:08:12.368935+0000","last_undegraded":"2023-09-23T21:08:12.368935+0000","last_fullsized":"2023-09-23T21:08:12.368935+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:26:50.477289+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":315,"reported_epoch":377,"state":"active+clean","last_fresh":"2023-09-23T21:08:12.387666+0000","last_change":"2023-09-23T21:05:55.320314+0000","last_active":"2023-09-23T21:08:12.387666+0000","last_peered":"2023-09-23T21:08:12.387666+0000","last_clean":"2023-09-23T21:08:12.387666+0000","last_became_active":"2023-09-23T21:05:55.319935+0000","last_became_peered":"2023-09-23T21:05:55.319935+0000","last_unstale":"2023-09-23T21:08:12.387666+0000","last_undegraded":"2023-09-23T21:08:12.387666+0000","last_fullsized":"2023-09-23T21:08:12.387666+0000","mapping_epoch":350,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":351,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:52:28.754534+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.a","version":"378'9727","reported_seq":9774,"reported_epoch":378,"state":"active+clean","last_fresh":"2023-09-23T21:08:22.132826+0000","last_change":"2023-09-23T21:05:55.324585+0000","last_active":"2023-09-23T21:08:22.132826+0000","last_peered":"2023-09-23T21:08:22.132826+0000","last_clean":"2023-09-23T21:08:22.132826+0000","last_became_active":"2023-09-23T21:05:55.323754+0000","last_became_peered":"2023-09-23T21:05:55.323754+0000","last_unstale":"2023-09-23T21:08:22.132826+0000","last_undegraded":"2023-09-23T21:08:22.132826+0000","last_fullsized":"2023-09-23T21:08:22.132826+0000","mapping_epoch":350,"log_start":"351'1800","ondisk_log_start":"351'1800","created":346,"last_epoch_clean":351,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:05:47.261750+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:05:47.261750+0000","last_clean_scrub_stamp":"2023-09-23T21:05:47.261750+0000","objects_scrubbed":0,"log_size":7927,"log_dups_size":0,"ondisk_log_size":7927,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:38:05.236953+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":279248896,"num_objects":4261,"num_object_clones":0,"num_object_copies":12783,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4261,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9727,"num_write_kb":447616,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,7],"acting":[5,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.2","version":"79'161","reported_seq":612,"reported_epoch":377,"state":"active+clean","last_fresh":"2023-09-23T21:08:12.383219+0000","last_change":"2023-09-23T20:57:24.802586+0000","last_active":"2023-09-23T21:08:12.383219+0000","last_peered":"2023-09-23T21:08:12.383219+0000","last_clean":"2023-09-23T21:08:12.383219+0000","last_became_active":"2023-09-23T20:57:08.236114+0000","last_became_peered":"2023-09-23T20:57:08.236114+0000","last_unstale":"2023-09-23T21:08:12.383219+0000","last_undegraded":"2023-09-23T21:08:12.383219+0000","last_fullsized":"2023-09-23T21:08:12.383219+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:24.802451+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:24.802451+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:06:22.309980+0000","scrub_duration":0.0090861250000000005,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,4],"acting":[0,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":368,"reported_epoch":377,"state":"active+clean","last_fresh":"2023-09-23T21:08:12.375780+0000","last_change":"2023-09-23T20:51:10.838054+0000","last_active":"2023-09-23T21:08:12.375780+0000","last_peered":"2023-09-23T21:08:12.375780+0000","last_clean":"2023-09-23T21:08:12.375780+0000","last_became_active":"2023-09-23T20:51:10.837750+0000","last_became_peered":"2023-09-23T20:51:10.837750+0000","last_unstale":"2023-09-23T21:08:12.375780+0000","last_undegraded":"2023-09-23T21:08:12.375780+0000","last_fullsized":"2023-09-23T21:08:12.375780+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:15:51.648085+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":341,"reported_epoch":377,"state":"active+clean","last_fresh":"2023-09-23T21:08:12.389778+0000","last_change":"2023-09-23T21:05:55.315770+0000","last_active":"2023-09-23T21:08:12.389778+0000","last_peered":"2023-09-23T21:08:12.389778+0000","last_clean":"2023-09-23T21:08:12.389778+0000","last_became_active":"2023-09-23T21:05:55.315290+0000","last_became_peered":"2023-09-23T21:05:55.315290+0000","last_unstale":"2023-09-23T21:08:12.389778+0000","last_undegraded":"2023-09-23T21:08:12.389778+0000","last_fullsized":"2023-09-23T21:08:12.389778+0000","mapping_epoch":350,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":351,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:27:40.430262+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":382,"reported_epoch":377,"state":"active+clean","last_fresh":"2023-09-23T21:08:12.386466+0000","last_change":"2023-09-23T21:05:55.318863+0000","last_active":"2023-09-23T21:08:12.386466+0000","last_peered":"2023-09-23T21:08:12.386466+0000","last_clean":"2023-09-23T21:08:12.386466+0000","last_became_active":"2023-09-23T21:05:55.318489+0000","last_became_peered":"2023-09-23T21:05:55.318489+0000","last_unstale":"2023-09-23T21:08:12.386466+0000","last_undegraded":"2023-09-23T21:08:12.386466+0000","last_fullsized":"2023-09-23T21:08:12.386466+0000","mapping_epoch":350,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":351,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:45:05.912661+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.8","version":"378'9860","reported_seq":9902,"reported_epoch":378,"state":"active+clean","last_fresh":"2023-09-23T21:08:23.986649+0000","last_change":"2023-09-23T21:05:48.286303+0000","last_active":"2023-09-23T21:08:23.986649+0000","last_peered":"2023-09-23T21:08:23.986649+0000","last_clean":"2023-09-23T21:08:23.986649+0000","last_became_active":"2023-09-23T21:05:48.286008+0000","last_became_peered":"2023-09-23T21:05:48.286008+0000","last_unstale":"2023-09-23T21:08:23.986649+0000","last_undegraded":"2023-09-23T21:08:23.986649+0000","last_fullsized":"2023-09-23T21:08:23.986649+0000","mapping_epoch":346,"log_start":"356'3600","ondisk_log_start":"356'3600","created":346,"last_epoch_clean":347,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:05:47.261750+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:05:47.261750+0000","last_clean_scrub_stamp":"2023-09-23T21:05:47.261750+0000","objects_scrubbed":0,"log_size":6260,"log_dups_size":0,"ondisk_log_size":6260,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:42:05.759917+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":273154048,"num_objects":4168,"num_object_clones":0,"num_object_copies":12504,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4168,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9860,"num_write_kb":448896,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":142,"reported_epoch":377,"state":"active+clean","last_fresh":"2023-09-23T21:08:12.376275+0000","last_change":"2023-09-23T21:06:49.427675+0000","last_active":"2023-09-23T21:08:12.376275+0000","last_peered":"2023-09-23T21:08:12.376275+0000","last_clean":"2023-09-23T21:08:12.376275+0000","last_became_active":"2023-09-23T21:06:49.426780+0000","last_became_peered":"2023-09-23T21:06:49.426780+0000","last_unstale":"2023-09-23T21:08:12.376275+0000","last_undegraded":"2023-09-23T21:08:12.376275+0000","last_fullsized":"2023-09-23T21:08:12.376275+0000","mapping_epoch":359,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":360,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:06:15.874264+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"79'161","reported_seq":612,"reported_epoch":377,"state":"active+clean","last_fresh":"2023-09-23T21:08:12.385658+0000","last_change":"2023-09-23T20:57:20.738417+0000","last_active":"2023-09-23T21:08:12.385658+0000","last_peered":"2023-09-23T21:08:12.385658+0000","last_clean":"2023-09-23T21:08:12.385658+0000","last_became_active":"2023-09-23T20:57:08.236842+0000","last_became_peered":"2023-09-23T20:57:08.236842+0000","last_unstale":"2023-09-23T21:08:12.385658+0000","last_undegraded":"2023-09-23T21:08:12.385658+0000","last_fullsized":"2023-09-23T21:08:12.385658+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":0,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:20.738273+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:20.738273+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:18:28.327990+0000","scrub_duration":0.0088336339999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":378,"reported_epoch":377,"state":"active+clean","last_fresh":"2023-09-23T21:08:12.380994+0000","last_change":"2023-09-23T21:05:04.310148+0000","last_active":"2023-09-23T21:08:12.380994+0000","last_peered":"2023-09-23T21:08:12.380994+0000","last_clean":"2023-09-23T21:08:12.380994+0000","last_became_active":"2023-09-23T21:05:04.309869+0000","last_became_peered":"2023-09-23T21:05:04.309869+0000","last_unstale":"2023-09-23T21:08:12.380994+0000","last_undegraded":"2023-09-23T21:08:12.380994+0000","last_fullsized":"2023-09-23T21:08:12.380994+0000","mapping_epoch":332,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:12:55.522835+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":322,"reported_epoch":377,"state":"active+clean","last_fresh":"2023-09-23T21:08:12.368052+0000","last_change":"2023-09-23T20:55:35.978140+0000","last_active":"2023-09-23T21:08:12.368052+0000","last_peered":"2023-09-23T21:08:12.368052+0000","last_clean":"2023-09-23T21:08:12.368052+0000","last_became_active":"2023-09-23T20:55:35.977831+0000","last_became_peered":"2023-09-23T20:55:35.977831+0000","last_unstale":"2023-09-23T21:08:12.368052+0000","last_undegraded":"2023-09-23T21:08:12.368052+0000","last_fullsized":"2023-09-23T21:08:12.368052+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:25:47.678023+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"9.b","version":"378'9814","reported_seq":9855,"reported_epoch":378,"state":"active+clean","last_fresh":"2023-09-23T21:08:22.327837+0000","last_change":"2023-09-23T21:05:48.290142+0000","last_active":"2023-09-23T21:08:22.327837+0000","last_peered":"2023-09-23T21:08:22.327837+0000","last_clean":"2023-09-23T21:08:22.327837+0000","last_became_active":"2023-09-23T21:05:48.289861+0000","last_became_peered":"2023-09-23T21:05:48.289861+0000","last_unstale":"2023-09-23T21:08:22.327837+0000","last_undegraded":"2023-09-23T21:08:22.327837+0000","last_fullsized":"2023-09-23T21:08:22.327837+0000","mapping_epoch":346,"log_start":"351'1400","ondisk_log_start":"351'1400","created":346,"last_epoch_clean":347,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:05:47.261750+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:05:47.261750+0000","last_clean_scrub_stamp":"2023-09-23T21:05:47.261750+0000","objects_scrubbed":0,"log_size":8414,"log_dups_size":0,"ondisk_log_size":8414,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:36:39.064858+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":275775488,"num_objects":4208,"num_object_clones":0,"num_object_copies":12624,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4208,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9814,"num_write_kb":448704,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.3","version":"79'161","reported_seq":621,"reported_epoch":377,"state":"active+clean","last_fresh":"2023-09-23T21:08:12.382217+0000","last_change":"2023-09-23T21:05:57.832471+0000","last_active":"2023-09-23T21:08:12.382217+0000","last_peered":"2023-09-23T21:08:12.382217+0000","last_clean":"2023-09-23T21:08:12.382217+0000","last_became_active":"2023-09-23T21:05:55.318774+0000","last_became_peered":"2023-09-23T21:05:55.318774+0000","last_unstale":"2023-09-23T21:08:12.382217+0000","last_undegraded":"2023-09-23T21:08:12.382217+0000","last_fullsized":"2023-09-23T21:08:12.382217+0000","mapping_epoch":350,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":351,"parent":"0.0","parent_split_bits":3,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:05:57.832302+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:05:57.832302+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:15:55.402312+0000","scrub_duration":0.010206847999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":242,"reported_epoch":377,"state":"active+clean","last_fresh":"2023-09-23T21:08:12.389846+0000","last_change":"2023-09-23T21:05:55.316391+0000","last_active":"2023-09-23T21:08:12.389846+0000","last_peered":"2023-09-23T21:08:12.389846+0000","last_clean":"2023-09-23T21:08:12.389846+0000","last_became_active":"2023-09-23T21:05:55.316069+0000","last_became_peered":"2023-09-23T21:05:55.316069+0000","last_unstale":"2023-09-23T21:08:12.389846+0000","last_undegraded":"2023-09-23T21:08:12.389846+0000","last_fullsized":"2023-09-23T21:08:12.389846+0000","mapping_epoch":350,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":351,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:55:39.499467+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,0],"acting":[1,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":376,"reported_epoch":378,"state":"active+clean","last_fresh":"2023-09-23T21:08:16.379597+0000","last_change":"2023-09-23T20:57:44.979950+0000","last_active":"2023-09-23T21:08:16.379597+0000","last_peered":"2023-09-23T21:08:16.379597+0000","last_clean":"2023-09-23T21:08:16.379597+0000","last_became_active":"2023-09-23T20:57:44.979346+0000","last_became_peered":"2023-09-23T20:57:44.979346+0000","last_unstale":"2023-09-23T21:08:16.379597+0000","last_undegraded":"2023-09-23T21:08:16.379597+0000","last_fullsized":"2023-09-23T21:08:16.379597+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:10:24.372800+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":369,"reported_epoch":377,"state":"active+clean","last_fresh":"2023-09-23T21:08:12.383363+0000","last_change":"2023-09-23T21:05:55.320840+0000","last_active":"2023-09-23T21:08:12.383363+0000","last_peered":"2023-09-23T21:08:12.383363+0000","last_clean":"2023-09-23T21:08:12.383363+0000","last_became_active":"2023-09-23T21:05:55.320280+0000","last_became_peered":"2023-09-23T21:05:55.320280+0000","last_unstale":"2023-09-23T21:08:12.383363+0000","last_undegraded":"2023-09-23T21:08:12.383363+0000","last_fullsized":"2023-09-23T21:08:12.383363+0000","mapping_epoch":350,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":351,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:45:22.368801+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.e","version":"378'10140","reported_seq":10187,"reported_epoch":378,"state":"active+clean","last_fresh":"2023-09-23T21:08:22.376382+0000","last_change":"2023-09-23T21:05:55.326609+0000","last_active":"2023-09-23T21:08:22.376382+0000","last_peered":"2023-09-23T21:08:22.376382+0000","last_clean":"2023-09-23T21:08:22.376382+0000","last_became_active":"2023-09-23T21:05:55.326268+0000","last_became_peered":"2023-09-23T21:05:55.326268+0000","last_unstale":"2023-09-23T21:08:22.376382+0000","last_undegraded":"2023-09-23T21:08:22.376382+0000","last_fullsized":"2023-09-23T21:08:22.376382+0000","mapping_epoch":350,"log_start":"353'3100","ondisk_log_start":"353'3100","created":346,"last_epoch_clean":351,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:05:47.261750+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:05:47.261750+0000","last_clean_scrub_stamp":"2023-09-23T21:05:47.261750+0000","objects_scrubbed":0,"log_size":7040,"log_dups_size":0,"ondisk_log_size":7040,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:20:12.304644+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":289013760,"num_objects":4410,"num_object_clones":0,"num_object_copies":13230,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4410,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10140,"num_write_kb":465600,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.6","version":"79'161","reported_seq":660,"reported_epoch":377,"state":"active+clean","last_fresh":"2023-09-23T21:08:12.367612+0000","last_change":"2023-09-23T20:57:26.907505+0000","last_active":"2023-09-23T21:08:12.367612+0000","last_peered":"2023-09-23T21:08:12.367612+0000","last_clean":"2023-09-23T21:08:12.367612+0000","last_became_active":"2023-09-23T20:57:08.240321+0000","last_became_peered":"2023-09-23T20:57:08.240321+0000","last_unstale":"2023-09-23T21:08:12.367612+0000","last_undegraded":"2023-09-23T21:08:12.367612+0000","last_fullsized":"2023-09-23T21:08:12.367612+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":3,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:26.907408+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:26.907408+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:34:17.392928+0000","scrub_duration":0.010337038,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":222,"reported_epoch":377,"state":"active+clean","last_fresh":"2023-09-23T21:08:12.380018+0000","last_change":"2023-09-23T21:06:25.394432+0000","last_active":"2023-09-23T21:08:12.380018+0000","last_peered":"2023-09-23T21:08:12.380018+0000","last_clean":"2023-09-23T21:08:12.380018+0000","last_became_active":"2023-09-23T21:06:25.394120+0000","last_became_peered":"2023-09-23T21:06:25.394120+0000","last_unstale":"2023-09-23T21:08:12.380018+0000","last_undegraded":"2023-09-23T21:08:12.380018+0000","last_fullsized":"2023-09-23T21:08:12.380018+0000","mapping_epoch":354,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":355,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:07:30.306153+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":365,"reported_epoch":377,"state":"active+clean","last_fresh":"2023-09-23T21:08:12.387764+0000","last_change":"2023-09-23T21:05:55.319513+0000","last_active":"2023-09-23T21:08:12.387764+0000","last_peered":"2023-09-23T21:08:12.387764+0000","last_clean":"2023-09-23T21:08:12.387764+0000","last_became_active":"2023-09-23T21:05:55.319178+0000","last_became_peered":"2023-09-23T21:05:55.319178+0000","last_unstale":"2023-09-23T21:08:12.387764+0000","last_undegraded":"2023-09-23T21:08:12.387764+0000","last_fullsized":"2023-09-23T21:08:12.387764+0000","mapping_epoch":350,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":351,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:47:32.783934+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":361,"reported_epoch":378,"state":"active+clean","last_fresh":"2023-09-23T21:08:16.380021+0000","last_change":"2023-09-23T21:05:04.303493+0000","last_active":"2023-09-23T21:08:16.380021+0000","last_peered":"2023-09-23T21:08:16.380021+0000","last_clean":"2023-09-23T21:08:16.380021+0000","last_became_active":"2023-09-23T21:05:04.303148+0000","last_became_peered":"2023-09-23T21:05:04.303148+0000","last_unstale":"2023-09-23T21:08:16.380021+0000","last_undegraded":"2023-09-23T21:08:16.380021+0000","last_fullsized":"2023-09-23T21:08:16.380021+0000","mapping_epoch":332,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:48:06.836151+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.d","version":"378'9906","reported_seq":9946,"reported_epoch":378,"state":"active+clean","last_fresh":"2023-09-23T21:08:23.449889+0000","last_change":"2023-09-23T21:05:55.318195+0000","last_active":"2023-09-23T21:08:23.449889+0000","last_peered":"2023-09-23T21:08:23.449889+0000","last_clean":"2023-09-23T21:08:23.449889+0000","last_became_active":"2023-09-23T21:05:55.316441+0000","last_became_peered":"2023-09-23T21:05:55.316441+0000","last_unstale":"2023-09-23T21:08:23.449889+0000","last_undegraded":"2023-09-23T21:08:23.449889+0000","last_fullsized":"2023-09-23T21:08:23.449889+0000","mapping_epoch":350,"log_start":"355'3300","ondisk_log_start":"355'3300","created":346,"last_epoch_clean":351,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:05:47.261750+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:05:47.261750+0000","last_clean_scrub_stamp":"2023-09-23T21:05:47.261750+0000","objects_scrubbed":0,"log_size":6606,"log_dups_size":0,"ondisk_log_size":6606,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:31:06.880466+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":284753944,"num_objects":4346,"num_object_clones":0,"num_object_copies":13038,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4346,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":9906,"num_write_kb":456001,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,5],"acting":[1,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"79'161","reported_seq":641,"reported_epoch":377,"state":"active+clean","last_fresh":"2023-09-23T21:08:12.368586+0000","last_change":"2023-09-23T20:57:30.029479+0000","last_active":"2023-09-23T21:08:12.368586+0000","last_peered":"2023-09-23T21:08:12.368586+0000","last_clean":"2023-09-23T21:08:12.368586+0000","last_became_active":"2023-09-23T20:57:08.249909+0000","last_became_peered":"2023-09-23T20:57:08.249909+0000","last_unstale":"2023-09-23T21:08:12.368586+0000","last_undegraded":"2023-09-23T21:08:12.368586+0000","last_fullsized":"2023-09-23T21:08:12.368586+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":3,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:30.029396+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:30.029396+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:16:46.083145+0000","scrub_duration":0.012914682,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":345,"reported_epoch":377,"state":"active+clean","last_fresh":"2023-09-23T21:08:12.383067+0000","last_change":"2023-09-23T20:51:10.836910+0000","last_active":"2023-09-23T21:08:12.383067+0000","last_peered":"2023-09-23T21:08:12.383067+0000","last_clean":"2023-09-23T21:08:12.383067+0000","last_became_active":"2023-09-23T20:51:10.836557+0000","last_became_peered":"2023-09-23T20:51:10.836557+0000","last_unstale":"2023-09-23T21:08:12.383067+0000","last_undegraded":"2023-09-23T21:08:12.383067+0000","last_fullsized":"2023-09-23T21:08:12.383067+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:51:54.019969+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":343,"reported_epoch":377,"state":"active+clean","last_fresh":"2023-09-23T21:08:12.390730+0000","last_change":"2023-09-23T21:05:55.318675+0000","last_active":"2023-09-23T21:08:12.390730+0000","last_peered":"2023-09-23T21:08:12.390730+0000","last_clean":"2023-09-23T21:08:12.390730+0000","last_became_active":"2023-09-23T21:05:55.318363+0000","last_became_peered":"2023-09-23T21:05:55.318363+0000","last_unstale":"2023-09-23T21:08:12.390730+0000","last_undegraded":"2023-09-23T21:08:12.390730+0000","last_fullsized":"2023-09-23T21:08:12.390730+0000","mapping_epoch":350,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":351,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:19:50.065285+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":171,"reported_epoch":377,"state":"active+clean","last_fresh":"2023-09-23T21:08:12.380825+0000","last_change":"2023-09-23T20:57:44.982509+0000","last_active":"2023-09-23T21:08:12.380825+0000","last_peered":"2023-09-23T21:08:12.380825+0000","last_clean":"2023-09-23T21:08:12.380825+0000","last_became_active":"2023-09-23T20:57:44.982175+0000","last_became_peered":"2023-09-23T20:57:44.982175+0000","last_unstale":"2023-09-23T21:08:12.380825+0000","last_undegraded":"2023-09-23T21:08:12.380825+0000","last_fullsized":"2023-09-23T21:08:12.380825+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:37:49.215488+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.c","version":"378'10038","reported_seq":10078,"reported_epoch":378,"state":"active+clean","last_fresh":"2023-09-23T21:08:24.022333+0000","last_change":"2023-09-23T21:05:48.560410+0000","last_active":"2023-09-23T21:08:24.022333+0000","last_peered":"2023-09-23T21:08:24.022333+0000","last_clean":"2023-09-23T21:08:24.022333+0000","last_became_active":"2023-09-23T21:05:48.560135+0000","last_became_peered":"2023-09-23T21:05:48.560135+0000","last_unstale":"2023-09-23T21:08:24.022333+0000","last_undegraded":"2023-09-23T21:08:24.022333+0000","last_fullsized":"2023-09-23T21:08:24.022333+0000","mapping_epoch":346,"log_start":"356'3700","ondisk_log_start":"356'3700","created":346,"last_epoch_clean":347,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:05:47.261750+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:05:47.261750+0000","last_clean_scrub_stamp":"2023-09-23T21:05:47.261750+0000","objects_scrubbed":0,"log_size":6338,"log_dups_size":0,"ondisk_log_size":6338,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:16:44.290638+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":277086208,"num_objects":4228,"num_object_clones":0,"num_object_copies":12684,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4228,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10038,"num_write_kb":456512,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.4","version":"79'161","reported_seq":694,"reported_epoch":377,"state":"active+clean","last_fresh":"2023-09-23T21:08:12.387271+0000","last_change":"2023-09-23T21:05:07.298174+0000","last_active":"2023-09-23T21:08:12.387271+0000","last_peered":"2023-09-23T21:08:12.387271+0000","last_clean":"2023-09-23T21:08:12.387271+0000","last_became_active":"2023-09-23T21:05:07.297862+0000","last_became_peered":"2023-09-23T21:05:07.297862+0000","last_unstale":"2023-09-23T21:08:12.387271+0000","last_undegraded":"2023-09-23T21:08:12.387271+0000","last_fullsized":"2023-09-23T21:08:12.387271+0000","mapping_epoch":335,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":336,"parent":"0.0","parent_split_bits":3,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:20.012764+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:20.012764+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:24:55.897324+0000","scrub_duration":0.0045251919999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":327,"reported_epoch":377,"state":"active+clean","last_fresh":"2023-09-23T21:08:12.367876+0000","last_change":"2023-09-23T20:55:35.949541+0000","last_active":"2023-09-23T21:08:12.367876+0000","last_peered":"2023-09-23T21:08:12.367876+0000","last_clean":"2023-09-23T21:08:12.367876+0000","last_became_active":"2023-09-23T20:55:35.944635+0000","last_became_peered":"2023-09-23T20:55:35.944635+0000","last_unstale":"2023-09-23T21:08:12.367876+0000","last_undegraded":"2023-09-23T21:08:12.367876+0000","last_fullsized":"2023-09-23T21:08:12.367876+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:30:36.033585+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"70'1","reported_seq":385,"reported_epoch":377,"state":"active+clean","last_fresh":"2023-09-23T21:08:12.383450+0000","last_change":"2023-09-23T21:05:55.320603+0000","last_active":"2023-09-23T21:08:12.383450+0000","last_peered":"2023-09-23T21:08:12.383450+0000","last_clean":"2023-09-23T21:08:12.383450+0000","last_became_active":"2023-09-23T21:05:55.319956+0000","last_became_peered":"2023-09-23T21:05:55.319956+0000","last_unstale":"2023-09-23T21:08:12.383450+0000","last_undegraded":"2023-09-23T21:08:12.383450+0000","last_fullsized":"2023-09-23T21:08:12.383450+0000","mapping_epoch":350,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":351,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:41:03.995212+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":302,"reported_epoch":377,"state":"active+clean","last_fresh":"2023-09-23T21:08:12.368323+0000","last_change":"2023-09-23T21:05:04.305174+0000","last_active":"2023-09-23T21:08:12.368323+0000","last_peered":"2023-09-23T21:08:12.368323+0000","last_clean":"2023-09-23T21:08:12.368323+0000","last_became_active":"2023-09-23T21:05:04.304900+0000","last_became_peered":"2023-09-23T21:05:04.304900+0000","last_unstale":"2023-09-23T21:08:12.368323+0000","last_undegraded":"2023-09-23T21:08:12.368323+0000","last_fullsized":"2023-09-23T21:08:12.368323+0000","mapping_epoch":332,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:13:23.024245+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":344,"reported_epoch":377,"state":"active+clean","last_fresh":"2023-09-23T21:08:12.384042+0000","last_change":"2023-09-23T20:54:13.929196+0000","last_active":"2023-09-23T21:08:12.384042+0000","last_peered":"2023-09-23T21:08:12.384042+0000","last_clean":"2023-09-23T21:08:12.384042+0000","last_became_active":"2023-09-23T20:54:13.928831+0000","last_became_peered":"2023-09-23T20:54:13.928831+0000","last_unstale":"2023-09-23T21:08:12.384042+0000","last_undegraded":"2023-09-23T21:08:12.384042+0000","last_fullsized":"2023-09-23T21:08:12.384042+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:28:44.316851+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":365,"reported_epoch":377,"state":"active+clean","last_fresh":"2023-09-23T21:08:12.376544+0000","last_change":"2023-09-23T21:05:55.313495+0000","last_active":"2023-09-23T21:08:12.376544+0000","last_peered":"2023-09-23T21:08:12.376544+0000","last_clean":"2023-09-23T21:08:12.376544+0000","last_became_active":"2023-09-23T21:05:55.313065+0000","last_became_peered":"2023-09-23T21:05:55.313065+0000","last_unstale":"2023-09-23T21:08:12.376544+0000","last_undegraded":"2023-09-23T21:08:12.376544+0000","last_fullsized":"2023-09-23T21:08:12.376544+0000","mapping_epoch":350,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":351,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:05:22.675197+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":68,"reported_epoch":377,"state":"active+clean","last_fresh":"2023-09-23T21:08:12.368764+0000","last_change":"2023-09-23T21:06:49.429520+0000","last_active":"2023-09-23T21:08:12.368764+0000","last_peered":"2023-09-23T21:08:12.368764+0000","last_clean":"2023-09-23T21:08:12.368764+0000","last_became_active":"2023-09-23T21:06:49.429195+0000","last_became_peered":"2023-09-23T21:06:49.429195+0000","last_unstale":"2023-09-23T21:08:12.368764+0000","last_undegraded":"2023-09-23T21:08:12.368764+0000","last_fullsized":"2023-09-23T21:08:12.368764+0000","mapping_epoch":359,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":360,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:34:39.724075+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,1],"acting":[4,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":272,"reported_epoch":377,"state":"active+clean","last_fresh":"2023-09-23T21:08:12.369029+0000","last_change":"2023-09-23T21:05:55.323591+0000","last_active":"2023-09-23T21:08:12.369029+0000","last_peered":"2023-09-23T21:08:12.369029+0000","last_clean":"2023-09-23T21:08:12.369029+0000","last_became_active":"2023-09-23T21:05:55.323220+0000","last_became_peered":"2023-09-23T21:05:55.323220+0000","last_unstale":"2023-09-23T21:08:12.369029+0000","last_undegraded":"2023-09-23T21:08:12.369029+0000","last_fullsized":"2023-09-23T21:08:12.369029+0000","mapping_epoch":350,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":351,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:03:09.849488+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":139,"reported_epoch":377,"state":"active+clean","last_fresh":"2023-09-23T21:08:12.376731+0000","last_change":"2023-09-23T21:06:49.424707+0000","last_active":"2023-09-23T21:08:12.376731+0000","last_peered":"2023-09-23T21:08:12.376731+0000","last_clean":"2023-09-23T21:08:12.376731+0000","last_became_active":"2023-09-23T21:06:49.424418+0000","last_became_peered":"2023-09-23T21:06:49.424418+0000","last_unstale":"2023-09-23T21:08:12.376731+0000","last_undegraded":"2023-09-23T21:08:12.376731+0000","last_fullsized":"2023-09-23T21:08:12.376731+0000","mapping_epoch":359,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":360,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:28:50.350150+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":255,"reported_epoch":377,"state":"active+clean","last_fresh":"2023-09-23T21:08:12.368480+0000","last_change":"2023-09-23T20:54:13.935307+0000","last_active":"2023-09-23T21:08:12.368480+0000","last_peered":"2023-09-23T21:08:12.368480+0000","last_clean":"2023-09-23T21:08:12.368480+0000","last_became_active":"2023-09-23T20:54:13.935000+0000","last_became_peered":"2023-09-23T20:54:13.935000+0000","last_unstale":"2023-09-23T21:08:12.368480+0000","last_undegraded":"2023-09-23T21:08:12.368480+0000","last_fullsized":"2023-09-23T21:08:12.368480+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:47:40.974082+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":369,"reported_epoch":377,"state":"active+clean","last_fresh":"2023-09-23T21:08:12.383826+0000","last_change":"2023-09-23T20:55:35.954870+0000","last_active":"2023-09-23T21:08:12.383826+0000","last_peered":"2023-09-23T21:08:12.383826+0000","last_clean":"2023-09-23T21:08:12.383826+0000","last_became_active":"2023-09-23T20:55:35.954568+0000","last_became_peered":"2023-09-23T20:55:35.954568+0000","last_unstale":"2023-09-23T21:08:12.383826+0000","last_undegraded":"2023-09-23T21:08:12.383826+0000","last_fullsized":"2023-09-23T21:08:12.383826+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:06:22.423566+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":368,"reported_epoch":377,"state":"active+clean","last_fresh":"2023-09-23T21:08:12.381047+0000","last_change":"2023-09-23T20:51:10.838334+0000","last_active":"2023-09-23T21:08:12.381047+0000","last_peered":"2023-09-23T21:08:12.381047+0000","last_clean":"2023-09-23T21:08:12.381047+0000","last_became_active":"2023-09-23T20:51:10.838026+0000","last_became_peered":"2023-09-23T20:51:10.838026+0000","last_unstale":"2023-09-23T21:08:12.381047+0000","last_undegraded":"2023-09-23T21:08:12.381047+0000","last_fullsized":"2023-09-23T21:08:12.381047+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:04:37.009457+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":403,"reported_epoch":378,"state":"active+clean","last_fresh":"2023-09-23T21:08:16.380220+0000","last_change":"2023-09-23T21:05:04.325268+0000","last_active":"2023-09-23T21:08:16.380220+0000","last_peered":"2023-09-23T21:08:16.380220+0000","last_clean":"2023-09-23T21:08:16.380220+0000","last_became_active":"2023-09-23T21:05:04.303430+0000","last_became_peered":"2023-09-23T21:05:04.303430+0000","last_unstale":"2023-09-23T21:08:16.380220+0000","last_undegraded":"2023-09-23T21:08:16.380220+0000","last_fullsized":"2023-09-23T21:08:16.380220+0000","mapping_epoch":332,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:17:44.879586+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":2216,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":272,"reported_epoch":377,"state":"active+clean","last_fresh":"2023-09-23T21:08:12.368201+0000","last_change":"2023-09-23T20:55:35.951572+0000","last_active":"2023-09-23T21:08:12.368201+0000","last_peered":"2023-09-23T21:08:12.368201+0000","last_clean":"2023-09-23T21:08:12.368201+0000","last_became_active":"2023-09-23T20:55:35.951294+0000","last_became_peered":"2023-09-23T20:55:35.951294+0000","last_unstale":"2023-09-23T21:08:12.368201+0000","last_undegraded":"2023-09-23T21:08:12.368201+0000","last_fullsized":"2023-09-23T21:08:12.368201+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:36:32.447210+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":39,"num_bytes_recovered":7154,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5834,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":21,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3381,"ondisk_log_size":3381,"up":63,"acting":63,"num_store_stats":8},{"poolid":9,"num_pg":16,"stat_sum":{"num_bytes":4441505816,"num_objects":67773,"num_object_clones":0,"num_object_copies":203319,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":67773,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":158775,"num_write_kb":7249473,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":13353299968,"data_stored":13353287752,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":115775,"ondisk_log_size":115775,"up":48,"acting":48,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201615,"num_pgs":47,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2899524,"kb_used_data":2497340,"kb_used_omap":0,"kb_used_meta":402176,"kb_avail":90845628,"statfs":{"total":95995035648,"available":93025923072,"internally_reserved":0,"allocated":2557276160,"data_stored":2556184196,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":411828224},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,2,3],"upper_bound":16},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:07:59 2023","interfaces":[{"interface":"back","average":{"1min":1.968,"5min":1.4239999999999999,"15min":1.696},"min":{"1min":0.23000000000000001,"5min":0.22700000000000001,"15min":0.21099999999999999},"max":{"1min":4.9969999999999999,"5min":8.4990000000000006,"15min":43.994999999999997},"last":3.5209999999999999},{"interface":"front","average":{"1min":2.2749999999999999,"5min":1.6299999999999999,"15min":1.883},"min":{"1min":0.26800000000000002,"5min":0.253,"15min":0.218},"max":{"1min":6.3529999999999998,"5min":8.5150000000000006,"15min":44.206000000000003},"last":0.38200000000000001}]},{"osd":1,"last update":"Sat Sep 23 21:07:59 2023","interfaces":[{"interface":"back","average":{"1min":2.8029999999999999,"5min":2.3199999999999998,"15min":2.2400000000000002},"min":{"1min":0.26700000000000002,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":7.8010000000000002,"5min":7.8399999999999999,"15min":7.8399999999999999},"last":1.044},{"interface":"front","average":{"1min":1.9330000000000001,"5min":2.0609999999999999,"15min":2.0819999999999999},"min":{"1min":0.26300000000000001,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":7.577,"5min":8.3859999999999992,"15min":8.3859999999999992},"last":6.8680000000000003}]},{"osd":2,"last update":"Sat Sep 23 21:08:19 2023","interfaces":[{"interface":"back","average":{"1min":3.1890000000000001,"5min":1.964,"15min":1.9990000000000001},"min":{"1min":0.40100000000000002,"5min":0.27200000000000002,"15min":0.26200000000000001},"max":{"1min":8.8379999999999992,"5min":14.478999999999999,"15min":14.478999999999999},"last":8.8379999999999992},{"interface":"front","average":{"1min":2.7269999999999999,"5min":1.8879999999999999,"15min":1.974},"min":{"1min":0.28499999999999998,"5min":0.27400000000000002,"15min":0.26500000000000001},"max":{"1min":9.0359999999999996,"5min":14.135,"15min":45.25},"last":3.536}]},{"osd":3,"last update":"Sat Sep 23 21:07:49 2023","interfaces":[{"interface":"back","average":{"1min":3.173,"5min":2.1040000000000001,"15min":2.6110000000000002},"min":{"1min":0.30099999999999999,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":9.1449999999999996,"5min":12.359999999999999,"15min":36.408000000000001},"last":8.7989999999999995},{"interface":"front","average":{"1min":3.6000000000000001,"5min":2.1379999999999999,"15min":2.4870000000000001},"min":{"1min":0.40999999999999998,"5min":0.30299999999999999,"15min":0.26000000000000001},"max":{"1min":12.226000000000001,"5min":13.199,"15min":35.593000000000004},"last":4.8760000000000003}]},{"osd":4,"last update":"Sat Sep 23 21:07:43 2023","interfaces":[{"interface":"back","average":{"1min":1.5589999999999999,"5min":0.96899999999999997,"15min":0.78300000000000003},"min":{"1min":0.28100000000000003,"5min":0.248,"15min":0.23699999999999999},"max":{"1min":4.4139999999999997,"5min":5.2640000000000002,"15min":5.8920000000000003},"last":3.476},{"interface":"front","average":{"1min":1.706,"5min":1.0509999999999999,"15min":0.84499999999999997},"min":{"1min":0.311,"5min":0.28399999999999997,"15min":0.26300000000000001},"max":{"1min":7.1420000000000003,"5min":7.1420000000000003,"15min":7.1420000000000003},"last":1.966}]},{"osd":5,"last update":"Sat Sep 23 21:07:43 2023","interfaces":[{"interface":"back","average":{"1min":1.758,"5min":0.996,"15min":0.83499999999999996},"min":{"1min":0.33500000000000002,"5min":0.311,"15min":0.28499999999999998},"max":{"1min":4.8929999999999998,"5min":4.8929999999999998,"15min":7.0389999999999997},"last":2.3639999999999999},{"interface":"front","average":{"1min":1.5409999999999999,"5min":1.1000000000000001,"15min":0.81799999999999995},"min":{"1min":0.308,"5min":0.29899999999999999,"15min":0.27800000000000002},"max":{"1min":5.5090000000000003,"5min":5.8529999999999998,"15min":5.8529999999999998},"last":3.5049999999999999}]},{"osd":6,"last update":"Sat Sep 23 21:07:59 2023","interfaces":[{"interface":"back","average":{"1min":1.556,"5min":1.0469999999999999,"15min":0.84999999999999998},"min":{"1min":0.39400000000000002,"5min":0.32200000000000001,"15min":0.28899999999999998},"max":{"1min":6.5430000000000001,"5min":6.5430000000000001,"15min":6.5430000000000001},"last":2.1899999999999999},{"interface":"front","average":{"1min":1.637,"5min":1.103,"15min":0.91900000000000004},"min":{"1min":0.38100000000000001,"5min":0.35699999999999998,"15min":0.32300000000000001},"max":{"1min":6.5170000000000003,"5min":6.5170000000000003,"15min":6.5170000000000003},"last":1.9950000000000001}]}]},{"osd":6,"up_from":164,"seq":704374636699,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1056240,"kb_used_data":805288,"kb_used_omap":10,"kb_used_meta":250933,"kb_avail":92688912,"statfs":{"total":95995035648,"available":94913445888,"internally_reserved":0,"allocated":824614912,"data_stored":823542993,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":10328,"internal_metadata":256956328},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:08:04 2023","interfaces":[{"interface":"back","average":{"1min":1.883,"5min":1.8460000000000001,"15min":1.643},"min":{"1min":0.26800000000000002,"5min":0.224,"15min":0.20999999999999999},"max":{"1min":9.9890000000000008,"5min":9.9890000000000008,"15min":14.151999999999999},"last":0.66200000000000003},{"interface":"front","average":{"1min":2.0529999999999999,"5min":1.6120000000000001,"15min":1.663},"min":{"1min":0.26500000000000001,"5min":0.224,"15min":0.20999999999999999},"max":{"1min":6.6079999999999997,"5min":9.5009999999999994,"15min":11.023},"last":0.505}]},{"osd":1,"last update":"Sat Sep 23 21:07:56 2023","interfaces":[{"interface":"back","average":{"1min":2.6930000000000001,"5min":1.819,"15min":1.673},"min":{"1min":0.29099999999999998,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":7.5439999999999996,"5min":12.904999999999999,"15min":12.904999999999999},"last":0.65100000000000002},{"interface":"front","average":{"1min":1.27,"5min":1.663,"15min":1.7290000000000001},"min":{"1min":0.30299999999999999,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":4.6280000000000001,"5min":6.7569999999999997,"15min":6.7569999999999997},"last":0.54300000000000004}]},{"osd":2,"last update":"Sat Sep 23 21:08:04 2023","interfaces":[{"interface":"back","average":{"1min":1.714,"5min":1.49,"15min":1.9470000000000001},"min":{"1min":0.312,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":4.8929999999999998,"5min":6.5289999999999999,"15min":14.029999999999999},"last":0.67300000000000004},{"interface":"front","average":{"1min":2.5800000000000001,"5min":2.02,"15min":1.8919999999999999},"min":{"1min":0.29999999999999999,"5min":0.26500000000000001,"15min":0.247},"max":{"1min":13.385999999999999,"5min":13.385999999999999,"15min":20.195},"last":0.495}]},{"osd":3,"last update":"Sat Sep 23 21:08:04 2023","interfaces":[{"interface":"back","average":{"1min":2.0209999999999999,"5min":2.5630000000000002,"15min":2.0099999999999998},"min":{"1min":0.309,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":7.7190000000000003,"5min":43.085999999999999,"15min":43.085999999999999},"last":0.51500000000000001},{"interface":"front","average":{"1min":2.798,"5min":2.5619999999999998,"15min":2.0750000000000002},"min":{"1min":0.36499999999999999,"5min":0.30499999999999999,"15min":0.28599999999999998},"max":{"1min":8.266,"5min":42.405000000000001,"15min":42.405000000000001},"last":0.69799999999999995}]},{"osd":4,"last update":"Sat Sep 23 21:08:04 2023","interfaces":[{"interface":"back","average":{"1min":1.5720000000000001,"5min":0.97399999999999998,"15min":0.91600000000000004},"min":{"1min":0.315,"5min":0.25700000000000001,"15min":0.23000000000000001},"max":{"1min":6.9930000000000003,"5min":6.9930000000000003,"15min":9.0079999999999991},"last":0.33400000000000002},{"interface":"front","average":{"1min":0.98599999999999999,"5min":0.88900000000000001,"15min":0.72299999999999998},"min":{"1min":0.32700000000000001,"5min":0.28799999999999998,"15min":0.25800000000000001},"max":{"1min":5.4589999999999996,"5min":6.4050000000000002,"15min":6.4050000000000002},"last":0.29399999999999998}]},{"osd":5,"last update":"Sat Sep 23 21:08:04 2023","interfaces":[{"interface":"back","average":{"1min":2.4510000000000001,"5min":1.2430000000000001,"15min":0.92800000000000005},"min":{"1min":0.30099999999999999,"5min":0.27800000000000002,"15min":0.27300000000000002},"max":{"1min":7.9710000000000001,"5min":7.9710000000000001,"15min":7.9710000000000001},"last":0.309},{"interface":"front","average":{"1min":1.2849999999999999,"5min":1.179,"15min":0.95399999999999996},"min":{"1min":0.32900000000000001,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":4.7549999999999999,"5min":10.657,"15min":10.657},"last":0.32100000000000001}]},{"osd":7,"last update":"Sat Sep 23 21:08:04 2023","interfaces":[{"interface":"back","average":{"1min":1.927,"5min":1.2509999999999999,"15min":1.038},"min":{"1min":0.34599999999999997,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":7.7510000000000003,"5min":8.9260000000000002,"15min":8.9260000000000002},"last":205.95599999999999},{"interface":"front","average":{"1min":1.6679999999999999,"5min":1.069,"15min":0.83099999999999996},"min":{"1min":0.308,"5min":0.28299999999999997,"15min":0.28100000000000003},"max":{"1min":5.5369999999999999,"5min":5.6230000000000002,"15min":8.282},"last":206.05699999999999}]}]},{"osd":1,"up_from":350,"seq":1503238553631,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2653464,"kb_used_data":2439824,"kb_used_omap":10,"kb_used_meta":213621,"kb_avail":91091688,"statfs":{"total":95995035648,"available":93277888512,"internally_reserved":0,"allocated":2498379776,"data_stored":2496697697,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":10498,"internal_metadata":218748670},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,2,1,4],"upper_bound":32},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:07:57 2023","interfaces":[{"interface":"back","average":{"1min":3.4860000000000002,"5min":2.222,"15min":2.0110000000000001},"min":{"1min":0.29299999999999998,"5min":0.21199999999999999,"15min":0.21199999999999999},"max":{"1min":15.728,"5min":15.728,"15min":15.728},"last":4.6550000000000002},{"interface":"front","average":{"1min":3.407,"5min":2.7290000000000001,"15min":2.6160000000000001},"min":{"1min":0.20999999999999999,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":14.468999999999999,"5min":14.468999999999999,"15min":14.468999999999999},"last":4.694}]},{"osd":2,"last update":"Sat Sep 23 21:07:57 2023","interfaces":[{"interface":"back","average":{"1min":3.2509999999999999,"5min":2.5219999999999998,"15min":2.3999999999999999},"min":{"1min":0.24399999999999999,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":10.769,"5min":10.769,"15min":10.769},"last":1.532},{"interface":"front","average":{"1min":3.0249999999999999,"5min":2.2120000000000002,"15min":2.0760000000000001},"min":{"1min":0.214,"5min":0.214,"15min":0.214},"max":{"1min":15.683999999999999,"5min":15.683999999999999,"15min":15.683999999999999},"last":1.8859999999999999}]},{"osd":3,"last update":"Sat Sep 23 21:07:57 2023","interfaces":[{"interface":"back","average":{"1min":3.226,"5min":2.843,"15min":2.7789999999999999},"min":{"1min":0.438,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":15.135999999999999,"5min":15.135999999999999,"15min":15.135999999999999},"last":1.9990000000000001},{"interface":"front","average":{"1min":4.1529999999999996,"5min":3.2810000000000001,"15min":3.1360000000000001},"min":{"1min":0.433,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":14.794,"5min":14.794,"15min":14.794},"last":1.9299999999999999}]},{"osd":4,"last update":"Sat Sep 23 21:07:57 2023","interfaces":[{"interface":"back","average":{"1min":2.6440000000000001,"5min":1.929,"15min":1.8100000000000001},"min":{"1min":0.32700000000000001,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":10.391,"5min":10.391,"15min":10.391},"last":0.94499999999999995},{"interface":"front","average":{"1min":1.893,"5min":1.6599999999999999,"15min":1.621},"min":{"1min":0.39400000000000002,"5min":0.39400000000000002,"15min":0.39400000000000002},"max":{"1min":6.0270000000000001,"5min":6.0270000000000001,"15min":6.0270000000000001},"last":2.1339999999999999}]},{"osd":5,"last update":"Sat Sep 23 21:07:57 2023","interfaces":[{"interface":"back","average":{"1min":2.2949999999999999,"5min":2.5430000000000001,"15min":2.5840000000000001},"min":{"1min":0.378,"5min":0.378,"15min":0.378},"max":{"1min":5.923,"5min":8.0899999999999999,"15min":8.0899999999999999},"last":1.673},{"interface":"front","average":{"1min":3.3479999999999999,"5min":2.7959999999999998,"15min":2.7040000000000002},"min":{"1min":0.35699999999999998,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":10.339,"5min":10.339,"15min":10.339},"last":0.72299999999999998}]},{"osd":6,"last update":"Sat Sep 23 21:07:57 2023","interfaces":[{"interface":"back","average":{"1min":2.464,"5min":2.3639999999999999,"15min":2.3479999999999999},"min":{"1min":0.377,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":5.968,"5min":8.1880000000000006,"15min":8.1880000000000006},"last":1.631},{"interface":"front","average":{"1min":3.2360000000000002,"5min":2.585,"15min":2.4769999999999999},"min":{"1min":0.40100000000000002,"5min":0.40100000000000002,"15min":0.40100000000000002},"max":{"1min":10.433999999999999,"5min":10.433999999999999,"15min":10.433999999999999},"last":0.75800000000000001}]},{"osd":7,"last update":"Sat Sep 23 21:07:57 2023","interfaces":[{"interface":"back","average":{"1min":1.9510000000000001,"5min":2.0230000000000001,"15min":2.0350000000000001},"min":{"1min":0.35299999999999998,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":5.992,"5min":6.5570000000000004,"15min":6.5570000000000004},"last":2.169},{"interface":"front","average":{"1min":2.2909999999999999,"5min":2.5590000000000002,"15min":2.6040000000000001},"min":{"1min":0.41099999999999998,"5min":0.41099999999999998,"15min":0.41099999999999998},"max":{"1min":5.8899999999999997,"5min":8.1440000000000001,"15min":8.1440000000000001},"last":1.702}]}]},{"osd":0,"up_from":83,"seq":356482285796,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1555424,"kb_used_data":1102040,"kb_used_omap":28,"kb_used_meta":453347,"kb_avail":92189728,"statfs":{"total":95995035648,"available":94402281472,"internally_reserved":0,"allocated":1128488960,"data_stored":1126856923,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":28848,"internal_metadata":464228176},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1],"upper_bound":2},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 21:07:58 2023","interfaces":[{"interface":"back","average":{"1min":3.4209999999999998,"5min":1.7370000000000001,"15min":1.456},"min":{"1min":0.20200000000000001,"5min":0.192,"15min":0.192},"max":{"1min":12.994,"5min":12.994,"15min":12.994},"last":8.234},{"interface":"front","average":{"1min":2.448,"5min":1.508,"15min":1.3520000000000001},"min":{"1min":0.247,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":14.646000000000001,"5min":14.646000000000001,"15min":14.646000000000001},"last":2.4540000000000002}]},{"osd":2,"last update":"Sat Sep 23 21:08:11 2023","interfaces":[{"interface":"back","average":{"1min":3.0819999999999999,"5min":2.2330000000000001,"15min":2.8839999999999999},"min":{"1min":0.40000000000000002,"5min":0.183,"15min":0.183},"max":{"1min":9.4879999999999995,"5min":13.032,"15min":21.053000000000001},"last":10.945},{"interface":"front","average":{"1min":3.2280000000000002,"5min":2.3010000000000002,"15min":3.0550000000000002},"min":{"1min":0.42899999999999999,"5min":0.23499999999999999,"15min":0.21099999999999999},"max":{"1min":14.558999999999999,"5min":14.558999999999999,"15min":21.747},"last":2.4849999999999999}]},{"osd":3,"last update":"Sat Sep 23 21:07:33 2023","interfaces":[{"interface":"back","average":{"1min":3.9220000000000002,"5min":2.5499999999999998,"15min":2.8719999999999999},"min":{"1min":0.27400000000000002,"5min":0.252,"15min":0.252},"max":{"1min":18.669,"5min":41.079000000000001,"15min":41.079000000000001},"last":0.379},{"interface":"front","average":{"1min":3.1499999999999999,"5min":2.282,"15min":2.9390000000000001},"min":{"1min":0.40300000000000002,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":19.91,"5min":19.91,"15min":21.167000000000002},"last":11.287000000000001}]},{"osd":4,"last update":"Sat Sep 23 21:08:11 2023","interfaces":[{"interface":"back","average":{"1min":2.5649999999999999,"5min":1.8069999999999999,"15min":2.2170000000000001},"min":{"1min":0.56000000000000005,"5min":0.28699999999999998,"15min":0.28599999999999998},"max":{"1min":6.657,"5min":12.394,"15min":21.594999999999999},"last":2.5529999999999999},{"interface":"front","average":{"1min":2.669,"5min":2.0979999999999999,"15min":1.9970000000000001},"min":{"1min":0.36599999999999999,"5min":0.311,"15min":0.28100000000000003},"max":{"1min":5.774,"5min":18.516999999999999,"15min":18.724},"last":0.52300000000000002}]},{"osd":5,"last update":"Sat Sep 23 21:08:11 2023","interfaces":[{"interface":"back","average":{"1min":2.306,"5min":1.8480000000000001,"15min":1.9330000000000001},"min":{"1min":0.35099999999999998,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":5.2140000000000004,"5min":12.946999999999999,"15min":12.946999999999999},"last":3.492},{"interface":"front","average":{"1min":2.0870000000000002,"5min":1.774,"15min":2.202},"min":{"1min":0.48899999999999999,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":6.7000000000000002,"5min":12.353,"15min":20.512},"last":2.5089999999999999}]},{"osd":6,"last update":"Sat Sep 23 21:08:08 2023","interfaces":[{"interface":"back","average":{"1min":3.7280000000000002,"5min":2.125,"15min":2.0259999999999998},"min":{"1min":0.56499999999999995,"5min":0.36099999999999999,"15min":0.35899999999999999},"max":{"1min":18.571000000000002,"5min":18.571000000000002,"15min":18.571000000000002},"last":0.41399999999999998},{"interface":"front","average":{"1min":4.0140000000000002,"5min":2.2530000000000001,"15min":2.1070000000000002},"min":{"1min":0.69599999999999995,"5min":0.40300000000000002,"15min":0.377},"max":{"1min":18.603000000000002,"5min":18.603000000000002,"15min":18.603000000000002},"last":0.46000000000000002}]},{"osd":7,"last update":"Sat Sep 23 21:08:11 2023","interfaces":[{"interface":"back","average":{"1min":2.399,"5min":1.8979999999999999,"15min":1.9890000000000001},"min":{"1min":0.38600000000000001,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":6.2450000000000001,"5min":12.907999999999999,"15min":12.907999999999999},"last":3.4590000000000001},{"interface":"front","average":{"1min":3.214,"5min":2.7290000000000001,"15min":2.2429999999999999},"min":{"1min":0.51100000000000001,"5min":0.40999999999999998,"15min":0.377},"max":{"1min":7.5449999999999999,"5min":40.984999999999999,"15min":40.984999999999999},"last":0.64300000000000002}]}]},{"osd":2,"up_from":93,"seq":399431958743,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1421020,"kb_used_data":835092,"kb_used_omap":81,"kb_used_meta":585838,"kb_avail":92324132,"statfs":{"total":95995035648,"available":94539911168,"internally_reserved":0,"allocated":855134208,"data_stored":853503635,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":83750,"internal_metadata":599898330},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:08:01 2023","interfaces":[{"interface":"back","average":{"1min":2.2029999999999998,"5min":1.6639999999999999,"15min":2.1859999999999999},"min":{"1min":0.23799999999999999,"5min":0.22600000000000001,"15min":0.17999999999999999},"max":{"1min":11.124000000000001,"5min":11.124000000000001,"15min":29.498000000000001},"last":7.3319999999999999},{"interface":"front","average":{"1min":1.6040000000000001,"5min":1.726,"15min":2.484},"min":{"1min":0.26600000000000001,"5min":0.26600000000000001,"15min":0.193},"max":{"1min":5.8070000000000004,"5min":9.2880000000000003,"15min":17.251000000000001},"last":4.468}]},{"osd":1,"last update":"Sat Sep 23 21:07:57 2023","interfaces":[{"interface":"back","average":{"1min":4.2770000000000001,"5min":3.2349999999999999,"15min":3.0609999999999999},"min":{"1min":0.309,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":12.593,"5min":34.396999999999998,"15min":34.396999999999998},"last":4.8090000000000002},{"interface":"front","average":{"1min":2.3439999999999999,"5min":3.407,"15min":3.5840000000000001},"min":{"1min":0.217,"5min":0.217,"15min":0.217},"max":{"1min":7.0659999999999998,"5min":34.462000000000003,"15min":34.462000000000003},"last":3.3999999999999999}]},{"osd":3,"last update":"Sat Sep 23 21:07:47 2023","interfaces":[{"interface":"back","average":{"1min":2.2709999999999999,"5min":2.7679999999999998,"15min":3.3079999999999998},"min":{"1min":0.248,"5min":0.23899999999999999,"15min":0.23300000000000001},"max":{"1min":12.083,"5min":35.549999999999997,"15min":41.057000000000002},"last":0.34999999999999998},{"interface":"front","average":{"1min":2.8679999999999999,"5min":3.2690000000000001,"15min":3.7930000000000001},"min":{"1min":0.36199999999999999,"5min":0.253,"15min":0.22800000000000001},"max":{"1min":12.446,"5min":36.951999999999998,"15min":41.012},"last":3.6320000000000001}]},{"osd":4,"last update":"Sat Sep 23 21:08:01 2023","interfaces":[{"interface":"back","average":{"1min":1.569,"5min":1.4950000000000001,"15min":1.6479999999999999},"min":{"1min":0.33100000000000002,"5min":0.29199999999999998,"15min":0.26400000000000001},"max":{"1min":5.4960000000000004,"5min":8.7569999999999997,"15min":10.045},"last":0.38700000000000001},{"interface":"front","average":{"1min":2.8530000000000002,"5min":1.9970000000000001,"15min":2.101},"min":{"1min":0.40500000000000003,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":12.49,"5min":12.49,"15min":14.039999999999999},"last":3.7050000000000001}]},{"osd":5,"last update":"Sat Sep 23 21:08:01 2023","interfaces":[{"interface":"back","average":{"1min":1.927,"5min":1.6890000000000001,"15min":1.823},"min":{"1min":0.33800000000000002,"5min":0.316,"15min":0.29299999999999998},"max":{"1min":5.4429999999999996,"5min":8.907,"15min":10.069000000000001},"last":0.56599999999999995},{"interface":"front","average":{"1min":2.7530000000000001,"5min":2.004,"15min":2.089},"min":{"1min":0.36599999999999999,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":12.57,"5min":12.57,"15min":13.98},"last":3.677}]},{"osd":6,"last update":"Sat Sep 23 21:08:01 2023","interfaces":[{"interface":"back","average":{"1min":2.7919999999999998,"5min":2.2490000000000001,"15min":2.1019999999999999},"min":{"1min":0.432,"5min":0.35699999999999998,"15min":0.312},"max":{"1min":12.518000000000001,"5min":12.518000000000001,"15min":14.244999999999999},"last":4.024},{"interface":"front","average":{"1min":2.9900000000000002,"5min":2.407,"15min":2.0169999999999999},"min":{"1min":0.438,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":12.544,"5min":12.544,"15min":14.068},"last":5.1669999999999998}]},{"osd":7,"last update":"Sat Sep 23 21:08:01 2023","interfaces":[{"interface":"back","average":{"1min":1.694,"5min":1.6899999999999999,"15min":1.5269999999999999},"min":{"1min":0.40999999999999998,"5min":0.34100000000000003,"15min":0.315},"max":{"1min":7.1050000000000004,"5min":8.7599999999999998,"15min":13.427},"last":3.3490000000000002},{"interface":"front","average":{"1min":2.218,"5min":1.8979999999999999,"15min":1.899},"min":{"1min":0.42199999999999999,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":10.92,"5min":10.92,"15min":10.92},"last":0.59899999999999998}]}]},{"osd":3,"up_from":98,"seq":420906795217,"num_pgs":48,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2840844,"kb_used_data":2171588,"kb_used_omap":7,"kb_used_meta":669240,"kb_avail":90904308,"statfs":{"total":95995035648,"available":93086011392,"internally_reserved":0,"allocated":2223706112,"data_stored":2222089855,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7576,"internal_metadata":685302376},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,5,2],"upper_bound":32},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:07:49 2023","interfaces":[{"interface":"back","average":{"1min":3.0049999999999999,"5min":2.331,"15min":3.0579999999999998},"min":{"1min":0.19900000000000001,"5min":0.189,"15min":0.188},"max":{"1min":10.409000000000001,"5min":11.717000000000001,"15min":27.962},"last":0.40500000000000003},{"interface":"front","average":{"1min":2.399,"5min":2.3479999999999999,"15min":2.9350000000000001},"min":{"1min":0.245,"5min":0.20599999999999999,"15min":0.19800000000000001},"max":{"1min":7.8920000000000003,"5min":11.483000000000001,"15min":28.058},"last":5.7969999999999997}]},{"osd":1,"last update":"Sat Sep 23 21:07:59 2023","interfaces":[{"interface":"back","average":{"1min":5.6779999999999999,"5min":2.8500000000000001,"15min":2.3780000000000001},"min":{"1min":0.29499999999999998,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":15.897,"5min":15.897,"15min":15.897},"last":0.51200000000000001},{"interface":"front","average":{"1min":4.5999999999999996,"5min":2.6960000000000002,"15min":2.379},"min":{"1min":0.30399999999999999,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":13.811999999999999,"5min":13.811999999999999,"15min":13.811999999999999},"last":2.1619999999999999}]},{"osd":2,"last update":"Sat Sep 23 21:07:49 2023","interfaces":[{"interface":"back","average":{"1min":4.6349999999999998,"5min":2.419,"15min":3.0640000000000001},"min":{"1min":0.307,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":14.287000000000001,"5min":14.287000000000001,"15min":28.251999999999999},"last":0.41799999999999998},{"interface":"front","average":{"1min":3.8639999999999999,"5min":2.6120000000000001,"15min":3.101},"min":{"1min":0.29899999999999999,"5min":0.29399999999999998,"15min":0.22},"max":{"1min":14.186,"5min":18.321000000000002,"15min":25.305},"last":0.443}]},{"osd":4,"last update":"Sat Sep 23 21:07:49 2023","interfaces":[{"interface":"back","average":{"1min":2.8239999999999998,"5min":1.8160000000000001,"15min":2.0470000000000002},"min":{"1min":0.379,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":8.7379999999999995,"5min":11.505000000000001,"15min":18.239000000000001},"last":11.614000000000001},{"interface":"front","average":{"1min":2.2050000000000001,"5min":1.974,"15min":2.105},"min":{"1min":0.35299999999999998,"5min":0.30299999999999999,"15min":0.29499999999999998},"max":{"1min":6.8929999999999998,"5min":10.914,"15min":22.853000000000002},"last":11.246}]},{"osd":5,"last update":"Sat Sep 23 21:07:49 2023","interfaces":[{"interface":"back","average":{"1min":3.1339999999999999,"5min":2.0960000000000001,"15min":2.1509999999999998},"min":{"1min":0.32700000000000001,"5min":0.314,"15min":0.314},"max":{"1min":12.815,"5min":12.815,"15min":18.324999999999999},"last":0.34999999999999998},{"interface":"front","average":{"1min":4.1870000000000003,"5min":2.4980000000000002,"15min":2.3660000000000001},"min":{"1min":0.34200000000000003,"5min":0.33700000000000002,"15min":0.33100000000000002},"max":{"1min":14.029,"5min":14.029,"15min":14.029},"last":2.2000000000000002}]},{"osd":6,"last update":"Sat Sep 23 21:08:07 2023","interfaces":[{"interface":"back","average":{"1min":3.0339999999999998,"5min":2.0329999999999999,"15min":1.7549999999999999},"min":{"1min":0.40000000000000002,"5min":0.33900000000000002,"15min":0.33500000000000002},"max":{"1min":11.119999999999999,"5min":11.119999999999999,"15min":14.833},"last":0.64900000000000002},{"interface":"front","average":{"1min":3.0859999999999999,"5min":2.3069999999999999,"15min":2.1720000000000002},"min":{"1min":0.496,"5min":0.377,"15min":0.373},"max":{"1min":7.867,"5min":8.5570000000000004,"15min":13.576000000000001},"last":2.2290000000000001}]},{"osd":7,"last update":"Sat Sep 23 21:07:49 2023","interfaces":[{"interface":"back","average":{"1min":3.7410000000000001,"5min":2.27,"15min":2.3690000000000002},"min":{"1min":0.90100000000000002,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":14.81,"5min":14.81,"15min":18.289999999999999},"last":0.46100000000000002},{"interface":"front","average":{"1min":2.7429999999999999,"5min":2.3199999999999998,"15min":2.3860000000000001},"min":{"1min":0.53200000000000003,"5min":0.40300000000000002,"15min":0.379},"max":{"1min":7.8390000000000004,"5min":12.624000000000001,"15min":13.603999999999999},"last":2.2559999999999998}]}]},{"osd":4,"up_from":31,"seq":133143986524,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1099136,"kb_used_data":835896,"kb_used_omap":0,"kb_used_meta":263232,"kb_avail":92646016,"statfs":{"total":95995035648,"available":94869520384,"internally_reserved":0,"allocated":855957504,"data_stored":854869865,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":269549568},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:08:05 2023","interfaces":[{"interface":"back","average":{"1min":2.3210000000000002,"5min":1.5329999999999999,"15min":1.5489999999999999},"min":{"1min":0.222,"5min":0.222,"15min":0.214},"max":{"1min":11.537000000000001,"5min":11.537000000000001,"15min":11.784000000000001},"last":1.22},{"interface":"front","average":{"1min":2.8029999999999999,"5min":2.048,"15min":1.8169999999999999},"min":{"1min":0.28000000000000003,"5min":0.254,"15min":0.254},"max":{"1min":7.7619999999999996,"5min":11.315,"15min":11.766},"last":1.24}]},{"osd":1,"last update":"Sat Sep 23 21:08:01 2023","interfaces":[{"interface":"back","average":{"1min":2.512,"5min":3.5579999999999998,"15min":3.7320000000000002},"min":{"1min":0.26600000000000001,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":9.2449999999999992,"5min":37.960000000000001,"15min":37.960000000000001},"last":0.504},{"interface":"front","average":{"1min":2.379,"5min":3.3719999999999999,"15min":3.5379999999999998},"min":{"1min":0.315,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":8.3870000000000005,"5min":37.933,"15min":37.933},"last":0.47999999999999998}]},{"osd":2,"last update":"Sat Sep 23 21:08:05 2023","interfaces":[{"interface":"back","average":{"1min":2.5129999999999999,"5min":1.821,"15min":1.869},"min":{"1min":0.26700000000000002,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":7.2290000000000001,"5min":10.906000000000001,"15min":15.288},"last":0.89700000000000002},{"interface":"front","average":{"1min":2.9300000000000002,"5min":1.8819999999999999,"15min":1.915},"min":{"1min":0.27800000000000002,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":11.682,"5min":11.682,"15min":15.262},"last":0.308}]},{"osd":3,"last update":"Sat Sep 23 21:07:37 2023","interfaces":[{"interface":"back","average":{"1min":3.274,"5min":2.4239999999999999,"15min":2.5670000000000002},"min":{"1min":0.36599999999999999,"5min":0.29799999999999999,"15min":0.28899999999999998},"max":{"1min":15.01,"5min":15.01,"15min":31.562999999999999},"last":1.3100000000000001},{"interface":"front","average":{"1min":3.391,"5min":2.0779999999999998,"15min":2.4489999999999998},"min":{"1min":0.38700000000000001,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":15.004,"5min":15.004,"15min":31.574000000000002},"last":1.329}]},{"osd":5,"last update":"Sat Sep 23 21:07:54 2023","interfaces":[{"interface":"back","average":{"1min":1.387,"5min":1.05,"15min":0.72699999999999998},"min":{"1min":0.24299999999999999,"5min":0.24299999999999999,"15min":0.22700000000000001},"max":{"1min":8.2479999999999993,"5min":8.2479999999999993,"15min":8.2479999999999993},"last":0.35099999999999998},{"interface":"front","average":{"1min":0.78100000000000003,"5min":1.1779999999999999,"15min":0.90100000000000002},"min":{"1min":0.255,"5min":0.255,"15min":0.251},"max":{"1min":2.9620000000000002,"5min":7.8700000000000001,"15min":7.8700000000000001},"last":0.33500000000000002}]},{"osd":6,"last update":"Sat Sep 23 21:08:01 2023","interfaces":[{"interface":"back","average":{"1min":1.728,"5min":1.4430000000000001,"15min":0.96899999999999997},"min":{"1min":0.28399999999999997,"5min":0.26800000000000002,"15min":0.26500000000000001},"max":{"1min":11.103999999999999,"5min":11.103999999999999,"15min":11.103999999999999},"last":0.46200000000000002},{"interface":"front","average":{"1min":1.149,"5min":0.92600000000000005,"15min":0.78100000000000003},"min":{"1min":0.34000000000000002,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":8.0749999999999993,"5min":8.0749999999999993,"15min":8.0749999999999993},"last":0.44400000000000001}]},{"osd":7,"last update":"Sat Sep 23 21:07:37 2023","interfaces":[{"interface":"back","average":{"1min":1.4330000000000001,"5min":1.1319999999999999,"15min":0.871},"min":{"1min":0.312,"5min":0.312,"15min":0.29299999999999998},"max":{"1min":5.4219999999999997,"5min":14.460000000000001,"15min":14.460000000000001},"last":0.39400000000000002},{"interface":"front","average":{"1min":0.93700000000000006,"5min":1.4450000000000001,"15min":0.98599999999999999},"min":{"1min":0.33100000000000002,"5min":0.32200000000000001,"15min":0.28199999999999997},"max":{"1min":4.2830000000000004,"5min":11.170999999999999,"15min":11.170999999999999},"last":0.434}]}]},{"osd":5,"up_from":39,"seq":167503724889,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2830776,"kb_used_data":2479664,"kb_used_omap":0,"kb_used_meta":351104,"kb_avail":90914376,"statfs":{"total":95995035648,"available":93096321024,"internally_reserved":0,"allocated":2539175936,"data_stored":2538092801,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":359530496},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,0,0,0,0,0,2],"upper_bound":1024},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:07:58 2023","interfaces":[{"interface":"back","average":{"1min":3.3620000000000001,"5min":2.0339999999999998,"15min":1.778},"min":{"1min":0.19700000000000001,"5min":0.19700000000000001,"15min":0.19700000000000001},"max":{"1min":25.222999999999999,"5min":25.222999999999999,"15min":25.222999999999999},"last":0.28999999999999998},{"interface":"front","average":{"1min":2.7970000000000002,"5min":1.968,"15min":1.8879999999999999},"min":{"1min":0.26200000000000001,"5min":0.23499999999999999,"15min":0.223},"max":{"1min":25.457999999999998,"5min":25.457999999999998,"15min":25.457999999999998},"last":0.40300000000000002}]},{"osd":1,"last update":"Sat Sep 23 21:07:58 2023","interfaces":[{"interface":"back","average":{"1min":3.7749999999999999,"5min":2.1859999999999999,"15min":1.921},"min":{"1min":0.26800000000000002,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":25.239999999999998,"5min":25.239999999999998,"15min":25.239999999999998},"last":0.38300000000000001},{"interface":"front","average":{"1min":3.4580000000000002,"5min":1.9339999999999999,"15min":1.6799999999999999},"min":{"1min":0.27400000000000002,"5min":0.252,"15min":0.252},"max":{"1min":25.253,"5min":25.253,"15min":25.253},"last":0.42499999999999999}]},{"osd":2,"last update":"Sat Sep 23 21:08:11 2023","interfaces":[{"interface":"back","average":{"1min":2.3090000000000002,"5min":1.746,"15min":1.76},"min":{"1min":0.33100000000000002,"5min":0.25800000000000001,"15min":0.23499999999999999},"max":{"1min":8.2119999999999997,"5min":12.634,"15min":12.651},"last":0.34699999999999998},{"interface":"front","average":{"1min":2.673,"5min":1.9910000000000001,"15min":1.9630000000000001},"min":{"1min":0.29999999999999999,"5min":0.28100000000000003,"15min":0.26100000000000001},"max":{"1min":8.1549999999999994,"5min":8.1549999999999994,"15min":12.300000000000001},"last":0.316}]},{"osd":3,"last update":"Sat Sep 23 21:07:30 2023","interfaces":[{"interface":"back","average":{"1min":3.3740000000000001,"5min":2.2389999999999999,"15min":2.6419999999999999},"min":{"1min":0.29299999999999998,"5min":0.29299999999999998,"15min":0.28599999999999998},"max":{"1min":25.503,"5min":25.503,"15min":61.667999999999999},"last":0.375},{"interface":"front","average":{"1min":4.0129999999999999,"5min":2.4710000000000001,"15min":2.3919999999999999},"min":{"1min":0.42399999999999999,"5min":0.33500000000000002,"15min":0.28899999999999998},"max":{"1min":25.466000000000001,"5min":25.466000000000001,"15min":62.276000000000003},"last":0.435}]},{"osd":4,"last update":"Sat Sep 23 21:07:44 2023","interfaces":[{"interface":"back","average":{"1min":1.3360000000000001,"5min":1.129,"15min":0.80200000000000005},"min":{"1min":0.28499999999999998,"5min":0.254,"15min":0.23400000000000001},"max":{"1min":5.7569999999999997,"5min":9.0950000000000006,"15min":9.0950000000000006},"last":0.33000000000000002},{"interface":"front","average":{"1min":1.8380000000000001,"5min":1.054,"15min":0.81799999999999995},"min":{"1min":0.34799999999999998,"5min":0.26900000000000002,"15min":0.23599999999999999},"max":{"1min":8.125,"5min":8.125,"15min":8.4350000000000005},"last":0.39300000000000002}]},{"osd":6,"last update":"Sat Sep 23 21:07:58 2023","interfaces":[{"interface":"back","average":{"1min":1.796,"5min":1.28,"15min":0.91900000000000004},"min":{"1min":0.36599999999999999,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":8.0909999999999993,"5min":8.0909999999999993,"15min":11.128},"last":0.36599999999999999},{"interface":"front","average":{"1min":1.7629999999999999,"5min":1.2150000000000001,"15min":0.96099999999999997},"min":{"1min":0.35999999999999999,"5min":0.34599999999999997,"15min":0.33200000000000002},"max":{"1min":6.351,"5min":6.681,"15min":11.112},"last":0.44600000000000001}]},{"osd":7,"last update":"Sat Sep 23 21:07:30 2023","interfaces":[{"interface":"back","average":{"1min":1.4710000000000001,"5min":1.107,"15min":0.89300000000000002},"min":{"1min":0.35199999999999998,"5min":0.309,"15min":0.30199999999999999},"max":{"1min":5.6500000000000004,"5min":5.6500000000000004,"15min":8.5649999999999995},"last":189.405},{"interface":"front","average":{"1min":0.89900000000000002,"5min":0.95299999999999996,"15min":0.80800000000000005},"min":{"1min":0.35999999999999999,"5min":0.32500000000000001,"15min":0.27700000000000002},"max":{"1min":3.4500000000000002,"5min":4.2009999999999996,"15min":4.6479999999999997},"last":189.685}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1259,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1117061120,"data_stored":1117061120,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":2484015104,"data_stored":2484011032,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":838860800,"data_stored":838860800,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":2188967936,"data_stored":2188967936,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":848166912,"data_stored":848166912,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":2520453120,"data_stored":2520449048,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":811663360,"data_stored":811663360,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":2544111616,"data_stored":2544107544,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T21:08:27.404 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph osd pool set .mgr pgp_num 21 2023-09-23T21:08:27.489 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:08:27.552 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:08:27 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/3732006029' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:08:27.552 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:08:27 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/3732006029' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:08:27.558 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:08:27.704 INFO:teuthology.orchestra.run.smithi070.stdout:ERROR: (22) Invalid argument 2023-09-23T21:08:27.704 INFO:teuthology.orchestra.run.smithi070.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T21:08:27.734 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:08:27 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/3732006029' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:08:27.745 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T21:08:27.755 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:08:27.755 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:08:27.856 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:08:27.928 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:08:28.040 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:08:28.043 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:08:28.044 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:08:28.145 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:08:28.195 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_blocked_ops 2023-09-23T21:08:28.199 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:08:28.259 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:08:28.267 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:08:28.267 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:08:28.368 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:08:28.432 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:08:28.443 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:08:28.491 INFO:teuthology.orchestra.run.smithi026.stderr:noscrub is unset 2023-09-23T21:08:28.508 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:08:28.512 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:08:28.512 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:08:28.613 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:08:28.704 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:08:28.764 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:08:28 smithi026 ceph-mon[175151]: from='client.46139 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T21:08:28.764 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:08:28 smithi026 ceph-mon[175151]: pgmap v997: 109 pgs: 109 active+clean; 4.1 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 748 op/s 2023-09-23T21:08:28.764 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:08:28 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/2287239488' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T21:08:28.764 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:08:28 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T21:08:28.765 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:08:28 smithi026 ceph-mon[179688]: from='client.46139 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T21:08:28.765 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:08:28 smithi026 ceph-mon[179688]: pgmap v997: 109 pgs: 109 active+clean; 4.1 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 748 op/s 2023-09-23T21:08:28.765 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:08:28 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/2287239488' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T21:08:28.765 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:08:28 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T21:08:28.875 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:08:28 smithi070 ceph-mon[168282]: from='client.46139 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T21:08:28.876 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:08:28 smithi070 ceph-mon[168282]: pgmap v997: 109 pgs: 109 active+clean; 4.1 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 748 op/s 2023-09-23T21:08:28.876 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:08:28 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/2287239488' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T21:08:28.876 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:08:28 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T21:08:28.973 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:08:28.979 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:08:28.979 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:08:29.000 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T21:08:29.080 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:08:29.123 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:08:29.243 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:08:29.247 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:08:29.247 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:08:29.348 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:08:29.415 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:08:29.494 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:08:29.507 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:08:29.508 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:08:29.608 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:08:29.660 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:08:29 smithi026 ceph-mon[179688]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:08:29.660 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:08:29 smithi026 ceph-mon[179688]: failed to encode map e379 with expected crc 2023-09-23T21:08:29.661 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:08:29 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T21:08:29.661 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:08:29 smithi026 ceph-mon[179688]: osdmap e379: 8 total, 8 up, 8 in 2023-09-23T21:08:29.661 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:08:29 smithi026 ceph-mon[179688]: failed to encode map e379 with expected crc 2023-09-23T21:08:29.661 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:08:29 smithi026 ceph-mon[179688]: failed to encode map e379 with expected crc 2023-09-23T21:08:29.661 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:08:29 smithi026 ceph-mon[179688]: failed to encode map e379 with expected crc 2023-09-23T21:08:29.661 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:08:29 smithi026 ceph-mon[179688]: failed to encode map e379 with expected crc 2023-09-23T21:08:29.661 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:08:29 smithi026 ceph-mon[175151]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:08:29.662 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:08:29 smithi026 ceph-mon[175151]: failed to encode map e379 with expected crc 2023-09-23T21:08:29.662 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:08:29 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T21:08:29.662 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:08:29 smithi026 ceph-mon[175151]: osdmap e379: 8 total, 8 up, 8 in 2023-09-23T21:08:29.662 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:08:29 smithi026 ceph-mon[175151]: failed to encode map e379 with expected crc 2023-09-23T21:08:29.662 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:08:29 smithi026 ceph-mon[175151]: failed to encode map e379 with expected crc 2023-09-23T21:08:29.662 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:08:29 smithi026 ceph-mon[175151]: failed to encode map e379 with expected crc 2023-09-23T21:08:29.662 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:08:29 smithi026 ceph-mon[175151]: failed to encode map e379 with expected crc 2023-09-23T21:08:29.662 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 21:08:29 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[164029]: ::ffff:172.21.15.70 - - [23/Sep/2023:21:08:29] "GET /metrics HTTP/1.1" 200 36295 "" "Prometheus/2.43.0" 2023-09-23T21:08:29.666 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:08:29.731 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:08:29.736 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:08:29.736 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:08:29.796 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:08:29 smithi070 ceph-mon[168282]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:08:29.797 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:08:29 smithi070 ceph-mon[168282]: failed to encode map e379 with expected crc 2023-09-23T21:08:29.797 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:08:29 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T21:08:29.797 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:08:29 smithi070 ceph-mon[168282]: osdmap e379: 8 total, 8 up, 8 in 2023-09-23T21:08:29.797 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:08:29 smithi070 ceph-mon[168282]: failed to encode map e379 with expected crc 2023-09-23T21:08:29.797 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:08:29 smithi070 ceph-mon[168282]: failed to encode map e379 with expected crc 2023-09-23T21:08:29.797 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:08:29 smithi070 ceph-mon[168282]: failed to encode map e379 with expected crc 2023-09-23T21:08:29.797 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:08:29 smithi070 ceph-mon[168282]: failed to encode map e379 with expected crc 2023-09-23T21:08:29.837 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:08:29.917 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:08:29.996 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:08:30.006 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:08:30.006 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:08:30.107 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:08:30.112 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T21:08:30.156 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:08:30.310 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:08:30.319 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:08:30.319 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:08:30.391 INFO:teuthology.orchestra.run.smithi070.stdout:ERROR: (22) Invalid argument 2023-09-23T21:08:30.391 INFO:teuthology.orchestra.run.smithi070.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T21:08:30.420 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:08:30.484 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:08:30.504 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:08:30 smithi026 ceph-mon[179688]: pgmap v999: 109 pgs: 109 active+clean; 4.0 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 785 op/s 2023-09-23T21:08:30.504 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:08:30 smithi026 ceph-mon[175151]: pgmap v999: 109 pgs: 109 active+clean; 4.0 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 785 op/s 2023-09-23T21:08:30.770 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:08:30 smithi070 ceph-mon[168282]: pgmap v999: 109 pgs: 109 active+clean; 4.0 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 785 op/s 2023-09-23T21:08:30.778 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:08:30.784 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:08:30.785 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:08:30.848 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_historic_ops 2023-09-23T21:08:30.886 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:08:30.977 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:08:30.985 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:08:31.051 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:08:31.060 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:08:31.060 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:08:31.161 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:08:31.170 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T21:08:31.281 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:08:31.351 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T21:08:31.357 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:08:31.358 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:08:31.458 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:08:31.528 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:08:31.550 INFO:teuthology.orchestra.run.smithi026.stderr:set pool 1 pgp_num to 21 2023-09-23T21:08:31.607 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T21:08:31.617 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:08:31.624 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:08:31.625 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:08:31.726 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:08:31.783 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:08:31 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/3600244714' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "21"}]: dispatch 2023-09-23T21:08:31.783 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:08:31 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:08:31.783 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:08:31 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/3600244714' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "21"}]: dispatch 2023-09-23T21:08:31.783 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:08:31 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:08:31.791 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:08:31.852 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:08:31 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/3600244714' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "21"}]: dispatch 2023-09-23T21:08:31.852 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:08:31 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:08:32.255 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:08:32.262 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:08:32.262 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:08:32.363 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:08:32.441 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:08:32.633 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:08:32.641 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:08:32.641 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:08:32.742 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:08:32.808 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:08:32 smithi070 ceph-mon[168282]: pgmap v1000: 109 pgs: 109 active+clean; 3.9 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 781 op/s 2023-09-23T21:08:32.808 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:08:32 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/3600244714' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "21"}]': finished 2023-09-23T21:08:32.808 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:08:32 smithi070 ceph-mon[168282]: osdmap e380: 8 total, 8 up, 8 in 2023-09-23T21:08:32.809 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:08:32 smithi070 ceph-mon[168282]: failed to encode map e380 with expected crc 2023-09-23T21:08:32.809 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:08:32 smithi070 ceph-mon[168282]: failed to encode map e380 with expected crc 2023-09-23T21:08:32.809 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:08:32 smithi070 ceph-mon[168282]: failed to encode map e380 with expected crc 2023-09-23T21:08:32.809 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:08:32 smithi070 ceph-mon[168282]: failed to encode map e380 with expected crc 2023-09-23T21:08:32.809 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:08:32 smithi070 ceph-mon[168282]: failed to encode map e380 with expected crc 2023-09-23T21:08:32.828 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:08:32 smithi026 ceph-mon[179688]: pgmap v1000: 109 pgs: 109 active+clean; 3.9 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 781 op/s 2023-09-23T21:08:32.828 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:08:32 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/3600244714' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "21"}]': finished 2023-09-23T21:08:32.828 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:08:32 smithi026 ceph-mon[179688]: osdmap e380: 8 total, 8 up, 8 in 2023-09-23T21:08:32.828 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:08:32 smithi026 ceph-mon[179688]: failed to encode map e380 with expected crc 2023-09-23T21:08:32.828 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:08:32 smithi026 ceph-mon[179688]: failed to encode map e380 with expected crc 2023-09-23T21:08:32.828 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:08:32 smithi026 ceph-mon[179688]: failed to encode map e380 with expected crc 2023-09-23T21:08:32.828 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:08:32 smithi026 ceph-mon[179688]: failed to encode map e380 with expected crc 2023-09-23T21:08:32.828 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:08:32 smithi026 ceph-mon[179688]: failed to encode map e380 with expected crc 2023-09-23T21:08:32.829 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:08:32 smithi026 ceph-mon[175151]: pgmap v1000: 109 pgs: 109 active+clean; 3.9 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 781 op/s 2023-09-23T21:08:32.829 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:08:32 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/3600244714' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "21"}]': finished 2023-09-23T21:08:32.829 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:08:32 smithi026 ceph-mon[175151]: osdmap e380: 8 total, 8 up, 8 in 2023-09-23T21:08:32.829 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:08:32 smithi026 ceph-mon[175151]: failed to encode map e380 with expected crc 2023-09-23T21:08:32.829 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:08:32 smithi026 ceph-mon[175151]: failed to encode map e380 with expected crc 2023-09-23T21:08:32.829 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:08:32 smithi026 ceph-mon[175151]: failed to encode map e380 with expected crc 2023-09-23T21:08:32.829 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:08:32 smithi026 ceph-mon[175151]: failed to encode map e380 with expected crc 2023-09-23T21:08:32.829 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:08:32 smithi026 ceph-mon[175151]: failed to encode map e380 with expected crc 2023-09-23T21:08:32.836 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:08:32.869 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:08:32.931 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T21:08:32.945 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:08:32.946 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:08:32.982 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0: {} 2023-09-23T21:08:33.016 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1: {} 2023-09-23T21:08:33.033 INFO:teuthology.orchestra.run.smithi070.stdout:ERROR: (22) Invalid argument 2023-09-23T21:08:33.034 INFO:teuthology.orchestra.run.smithi070.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T21:08:33.047 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:08:33.061 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2: {} 2023-09-23T21:08:33.095 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:08:33.110 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3: {} 2023-09-23T21:08:33.150 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4: {} 2023-09-23T21:08:33.186 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T21:08:33.224 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6: {} 2023-09-23T21:08:33.265 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T21:08:33.289 INFO:teuthology.orchestra.run.smithi026.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T21:08:33.289 INFO:teuthology.orchestra.run.smithi026.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T21:08:33.289 INFO:teuthology.orchestra.run.smithi026.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T21:08:33.289 INFO:teuthology.orchestra.run.smithi026.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T21:08:33.290 INFO:teuthology.orchestra.run.smithi026.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T21:08:33.290 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T21:08:33.290 INFO:teuthology.orchestra.run.smithi026.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T21:08:33.290 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T21:08:33.393 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:08:33.401 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:08:33.401 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:08:33.454 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_ops_in_flight 2023-09-23T21:08:33.502 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:08:33.578 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:08:33.872 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:08:33 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/355712025' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:08:33.883 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:08:33 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/355712025' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:08:33.884 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:08:33 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/355712025' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:08:34.099 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:08:34.112 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:08:34.112 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:08:34.213 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:08:34.229 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T21:08:34.290 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:08:34.395 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:08:34.401 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:08:34.401 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:08:34.502 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:08:34.576 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:08:34.630 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:08:34.637 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:08:34.637 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:08:34.738 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:08:34.790 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:08:34 smithi070 ceph-mon[168282]: pgmap v1002: 109 pgs: 109 active+clean; 3.8 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 732 op/s 2023-09-23T21:08:34.792 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:08:34.858 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:08:34.868 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:08:34.869 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:08:34.887 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:08:34 smithi026 ceph-mon[175151]: pgmap v1002: 109 pgs: 109 active+clean; 3.8 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 732 op/s 2023-09-23T21:08:34.887 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:08:34 smithi026 ceph-mon[179688]: pgmap v1002: 109 pgs: 109 active+clean; 3.8 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 732 op/s 2023-09-23T21:08:34.970 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:08:35.011 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:08:35.051 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:08:35.055 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:08:35.056 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:08:35.156 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:08:35.261 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:08:35.342 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:08:35.359 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:08:35.360 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:08:35.460 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:08:35.518 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:08:35.601 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:08:35.611 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:08:35.611 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:08:35.712 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:08:35.738 INFO:teuthology.orchestra.run.smithi070.stdout:{ 2023-09-23T21:08:35.738 INFO:teuthology.orchestra.run.smithi070.stdout: "ops": [ 2023-09-23T21:08:35.738 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:08:35.738 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_repop(client.35856.0:168947 9.b e380/346 9:daa8ce18:::benchmark_data_smithi026_342732_object55671:head v 380'10483, mlcod=380'10481)", 2023-09-23T21:08:35.738 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:08:35.726361+0000", 2023-09-23T21:08:35.738 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 0.011586788000000001, 2023-09-23T21:08:35.739 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.011609144, 2023-09-23T21:08:35.739 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:08:35.739 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "started", 2023-09-23T21:08:35.739 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:08:35.739 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:08:35.739 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:08:35.739 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:08:35.726361+0000", 2023-09-23T21:08:35.739 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:08:35.739 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:08:35.739 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:08:35.739 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:08:35.740 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:08:35.726361+0000", 2023-09-23T21:08:35.740 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:08:35.740 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:08:35.740 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:08:35.740 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:08:35.740 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:08:35.726361+0000", 2023-09-23T21:08:35.740 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999995 2023-09-23T21:08:35.740 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:08:35.740 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:08:35.740 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:08:35.740 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:08:35.726367+0000", 2023-09-23T21:08:35.740 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.1500000000000004e-06 2023-09-23T21:08:35.741 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:08:35.741 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:08:35.741 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:08:35.741 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:08:35.726380+0000", 2023-09-23T21:08:35.741 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.3519e-05 2023-09-23T21:08:35.741 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:08:35.741 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:08:35.741 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:08:35.741 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:08:35.726386+0000", 2023-09-23T21:08:35.741 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.7100000000000004e-06 2023-09-23T21:08:35.741 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:08:35.741 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:08:35.742 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:08:35.742 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:08:35.726436+0000", 2023-09-23T21:08:35.742 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.0475e-05 2023-09-23T21:08:35.742 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:08:35.742 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:08:35.742 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:08:35.742 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:08:35.726454+0000", 2023-09-23T21:08:35.742 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.8003999999999999e-05 2023-09-23T21:08:35.742 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:08:35.742 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:08:35.742 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:08:35.743 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:08:35.743 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:08:35.743 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_repop(client.35856.0:168952 9.5 e380/346 9:a5a92760:::benchmark_data_smithi026_342732_object55676:head v 380'10467, mlcod=380'10464)", 2023-09-23T21:08:35.743 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:08:35.730516+0000", 2023-09-23T21:08:35.743 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 0.0074319039999999996, 2023-09-23T21:08:35.743 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00749689, 2023-09-23T21:08:35.743 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:08:35.743 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "started", 2023-09-23T21:08:35.743 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:08:35.743 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:08:35.743 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:08:35.744 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:08:35.730516+0000", 2023-09-23T21:08:35.744 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:08:35.744 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:08:35.744 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:08:35.744 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:08:35.744 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:08:35.730516+0000", 2023-09-23T21:08:35.744 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:08:35.744 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:08:35.744 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:08:35.744 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:08:35.744 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:08:35.730516+0000", 2023-09-23T21:08:35.744 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999995 2023-09-23T21:08:35.745 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:08:35.745 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:08:35.745 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:08:35.745 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:08:35.730522+0000", 2023-09-23T21:08:35.745 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.657e-06 2023-09-23T21:08:35.745 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:08:35.745 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:08:35.745 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:08:35.745 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:08:35.730529+0000", 2023-09-23T21:08:35.745 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.0570000000000004e-06 2023-09-23T21:08:35.745 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:08:35.745 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:08:35.746 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:08:35.746 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:08:35.730534+0000", 2023-09-23T21:08:35.746 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.5859999999999998e-06 2023-09-23T21:08:35.746 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:08:35.746 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:08:35.746 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:08:35.746 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:08:35.730671+0000", 2023-09-23T21:08:35.746 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000137152 2023-09-23T21:08:35.746 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:08:35.746 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:08:35.746 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:08:35.746 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:08:35.730690+0000", 2023-09-23T21:08:35.747 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.9164000000000002e-05 2023-09-23T21:08:35.747 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:08:35.747 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:08:35.747 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:08:35.747 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:08:35.747 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:08:35.747 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_repop(client.35856.0:168948 9.8 e380/346 9:14a18182:::benchmark_data_smithi026_342732_object55672:head v 380'10453, mlcod=380'10451)", 2023-09-23T21:08:35.747 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:08:35.734284+0000", 2023-09-23T21:08:35.747 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 0.0036637000000000002, 2023-09-23T21:08:35.747 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0037642629999999999, 2023-09-23T21:08:35.747 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:08:35.747 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "started", 2023-09-23T21:08:35.747 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:08:35.748 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:08:35.748 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:08:35.748 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:08:35.734284+0000", 2023-09-23T21:08:35.748 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:08:35.748 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:08:35.748 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:08:35.748 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:08:35.748 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:08:35.734284+0000", 2023-09-23T21:08:35.748 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:08:35.748 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:08:35.748 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:08:35.748 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:08:35.749 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:08:35.734284+0000", 2023-09-23T21:08:35.749 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999995 2023-09-23T21:08:35.749 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:08:35.749 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:08:35.749 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:08:35.749 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:08:35.734290+0000", 2023-09-23T21:08:35.749 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.5629999999999999e-06 2023-09-23T21:08:35.749 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:08:35.749 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:08:35.749 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:08:35.749 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:08:35.734295+0000", 2023-09-23T21:08:35.749 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.716e-06 2023-09-23T21:08:35.749 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:08:35.750 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:08:35.750 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:08:35.750 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:08:35.734299+0000", 2023-09-23T21:08:35.750 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.3450000000000002e-06 2023-09-23T21:08:35.750 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:08:35.750 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:08:35.750 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:08:35.750 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:08:35.734341+0000", 2023-09-23T21:08:35.750 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.2268999999999998e-05 2023-09-23T21:08:35.750 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:08:35.750 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:08:35.750 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:08:35.750 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:08:35.734361+0000", 2023-09-23T21:08:35.751 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.9854999999999999e-05 2023-09-23T21:08:35.751 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:08:35.751 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:08:35.751 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:08:35.751 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:08:35.751 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:08:35.751 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_repop(client.35856.0:168954 9.b e380/346 9:d2ebc4fe:::benchmark_data_smithi026_342732_object55678:head v 380'10484, mlcod=380'10481)", 2023-09-23T21:08:35.751 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:08:35.734971+0000", 2023-09-23T21:08:35.751 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 0.0029768110000000002, 2023-09-23T21:08:35.751 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0031113569999999999, 2023-09-23T21:08:35.751 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:08:35.751 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "started", 2023-09-23T21:08:35.751 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:08:35.752 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:08:35.752 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:08:35.752 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:08:35.734971+0000", 2023-09-23T21:08:35.752 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:08:35.752 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:08:35.752 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:08:35.752 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:08:35.752 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:08:35.734971+0000", 2023-09-23T21:08:35.752 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:08:35.752 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:08:35.752 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:08:35.752 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:08:35.752 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:08:35.734971+0000", 2023-09-23T21:08:35.753 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999995 2023-09-23T21:08:35.753 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:08:35.753 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:08:35.753 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:08:35.753 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:08:35.734977+0000", 2023-09-23T21:08:35.753 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.528e-06 2023-09-23T21:08:35.753 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:08:35.753 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:08:35.753 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:08:35.753 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:08:35.734989+0000", 2023-09-23T21:08:35.753 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.1902999999999999e-05 2023-09-23T21:08:35.753 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:08:35.753 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:08:35.754 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:08:35.754 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:08:35.734995+0000", 2023-09-23T21:08:35.754 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.5589999999999996e-06 2023-09-23T21:08:35.754 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:08:35.754 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:08:35.754 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:08:35.754 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:08:35.735055+0000", 2023-09-23T21:08:35.754 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.0690999999999999e-05 2023-09-23T21:08:35.754 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:08:35.754 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:08:35.754 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:08:35.754 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:08:35.735076+0000", 2023-09-23T21:08:35.755 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.0945999999999999e-05 2023-09-23T21:08:35.755 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:08:35.755 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:08:35.755 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:08:35.755 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:08:35.755 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:08:35.755 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_repop(client.35856.0:168953 9.6 e380/359 9:6878e084:::benchmark_data_smithi026_342732_object55677:head v 380'10628, mlcod=380'10625)", 2023-09-23T21:08:35.755 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:08:35.735209+0000", 2023-09-23T21:08:35.755 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 0.0027386020000000001, 2023-09-23T21:08:35.755 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0029080540000000002, 2023-09-23T21:08:35.755 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:08:35.755 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "started", 2023-09-23T21:08:35.755 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:08:35.755 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:08:35.756 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:08:35.756 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:08:35.735209+0000", 2023-09-23T21:08:35.756 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:08:35.756 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:08:35.756 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:08:35.756 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:08:35.756 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:08:35.735209+0000", 2023-09-23T21:08:35.756 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:08:35.756 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:08:35.756 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:08:35.756 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:08:35.756 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:08:35.735209+0000", 2023-09-23T21:08:35.756 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999995 2023-09-23T21:08:35.757 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:08:35.757 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:08:35.757 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:08:35.757 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:08:35.735214+0000", 2023-09-23T21:08:35.757 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.0420000000000002e-06 2023-09-23T21:08:35.757 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:08:35.757 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:08:35.757 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:08:35.757 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:08:35.735226+0000", 2023-09-23T21:08:35.757 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.253e-05 2023-09-23T21:08:35.757 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:08:35.757 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:08:35.757 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:08:35.757 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:08:35.735237+0000", 2023-09-23T21:08:35.758 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.0278000000000001e-05 2023-09-23T21:08:35.758 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:08:35.758 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:08:35.758 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:08:35.758 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:08:35.735280+0000", 2023-09-23T21:08:35.758 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.3662999999999999e-05 2023-09-23T21:08:35.758 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:08:35.758 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:08:35.758 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:08:35.758 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:08:35.735298+0000", 2023-09-23T21:08:35.758 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.7621999999999998e-05 2023-09-23T21:08:35.758 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:08:35.759 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:08:35.759 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:08:35.759 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:08:35.759 INFO:teuthology.orchestra.run.smithi070.stdout: ], 2023-09-23T21:08:35.759 INFO:teuthology.orchestra.run.smithi070.stdout: "num_ops": 5 2023-09-23T21:08:35.759 INFO:teuthology.orchestra.run.smithi070.stdout:} 2023-09-23T21:08:35.775 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:08:35.837 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:08:35.847 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:08:35.847 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:08:35.948 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:08:36.007 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:08:36.061 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T21:08:36.071 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:08:36.077 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:08:36.077 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:08:36.178 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:08:36.180 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_blocked_ops 2023-09-23T21:08:36.271 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:08:36.348 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:08:36 smithi070 ceph-mon[168282]: pgmap v1003: 109 pgs: 109 active+clean; 3.7 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 955 op/s 2023-09-23T21:08:36.348 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:08:36 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/2603721706' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:08:36.348 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:08:36 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:08:36.354 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:08:36.366 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:08:36.367 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:08:36.408 INFO:teuthology.orchestra.run.smithi026.stderr:nodeep-scrub is unset 2023-09-23T21:08:36.468 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:08:36.509 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:08:36.560 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:08:36 smithi026 ceph-mon[179688]: pgmap v1003: 109 pgs: 109 active+clean; 3.7 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 955 op/s 2023-09-23T21:08:36.560 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:08:36 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/2603721706' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:08:36.560 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:08:36 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:08:36.561 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:08:36 smithi026 ceph-mon[175151]: pgmap v1003: 109 pgs: 109 active+clean; 3.7 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 955 op/s 2023-09-23T21:08:36.561 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:08:36 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/2603721706' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:08:36.561 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:08:36 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:08:36.585 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:08:36.593 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:08:36.594 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:08:36.695 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:08:36.780 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:08:36.831 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:08:36.836 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:08:36.836 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:08:36.863 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T21:08:36.936 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:08:37.025 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:08:37.100 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T21:08:37.109 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:08:37.110 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:08:37.210 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:08:37.238 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:08:37.282 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:08:37.353 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:08:37 smithi070 ceph-mon[168282]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T21:08:37.353 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:08:37 smithi070 ceph-mon[168282]: Cluster is now healthy 2023-09-23T21:08:37.353 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:08:37 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T21:08:37.354 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:08:37 smithi070 ceph-mon[168282]: osdmap e381: 8 total, 8 up, 8 in 2023-09-23T21:08:37.354 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:08:37 smithi070 ceph-mon[168282]: failed to encode map e381 with expected crc 2023-09-23T21:08:37.354 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:08:37 smithi070 ceph-mon[168282]: failed to encode map e381 with expected crc 2023-09-23T21:08:37.354 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:08:37 smithi070 ceph-mon[168282]: failed to encode map e381 with expected crc 2023-09-23T21:08:37.383 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 6, 7, 4, 5] out_osds: [] dead_osds: [] live_osds: [0, 2, 3, 4, 5, 7, 6, 1] 2023-09-23T21:08:37.383 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T21:08:37.384 INFO:tasks.thrashosds.thrasher:Removing osd 7, in_osds are: [0, 1, 2, 3, 6, 7, 4, 5] 2023-09-23T21:08:37.384 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph osd out 7 2023-09-23T21:08:37.443 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:08:37.454 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:08:37.455 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:08:37.555 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:08:37.600 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:08:37.636 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:08:37 smithi026 ceph-mon[179688]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T21:08:37.636 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:08:37 smithi026 ceph-mon[179688]: Cluster is now healthy 2023-09-23T21:08:37.636 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:08:37 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T21:08:37.636 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:08:37 smithi026 ceph-mon[179688]: osdmap e381: 8 total, 8 up, 8 in 2023-09-23T21:08:37.636 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:08:37 smithi026 ceph-mon[179688]: failed to encode map e381 with expected crc 2023-09-23T21:08:37.636 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:08:37 smithi026 ceph-mon[179688]: failed to encode map e381 with expected crc 2023-09-23T21:08:37.636 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:08:37 smithi026 ceph-mon[179688]: failed to encode map e381 with expected crc 2023-09-23T21:08:37.640 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:08:37 smithi026 ceph-mon[175151]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T21:08:37.640 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:08:37 smithi026 ceph-mon[175151]: Cluster is now healthy 2023-09-23T21:08:37.641 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:08:37 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T21:08:37.641 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:08:37 smithi026 ceph-mon[175151]: osdmap e381: 8 total, 8 up, 8 in 2023-09-23T21:08:37.641 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:08:37 smithi026 ceph-mon[175151]: failed to encode map e381 with expected crc 2023-09-23T21:08:37.641 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:08:37 smithi026 ceph-mon[175151]: failed to encode map e381 with expected crc 2023-09-23T21:08:37.641 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:08:37 smithi026 ceph-mon[175151]: failed to encode map e381 with expected crc 2023-09-23T21:08:37.653 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:08:37.657 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:08:37.657 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:08:37.758 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:08:37.839 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:08:37.924 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:08:37.940 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:08:37.940 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:08:38.041 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:08:38.093 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:08:38.271 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:08:38.282 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:08:38.283 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:08:38.380 INFO:teuthology.orchestra.run.smithi070.stdout:{ 2023-09-23T21:08:38.380 INFO:teuthology.orchestra.run.smithi070.stdout: "ops": [], 2023-09-23T21:08:38.380 INFO:teuthology.orchestra.run.smithi070.stdout: "complaint_time": 30, 2023-09-23T21:08:38.380 INFO:teuthology.orchestra.run.smithi070.stdout: "num_blocked_ops": 0 2023-09-23T21:08:38.380 INFO:teuthology.orchestra.run.smithi070.stdout:} 2023-09-23T21:08:38.384 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:08:38.436 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:08:38.498 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:08:38.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:08:38 smithi026 ceph-mon[179688]: pgmap v1005: 109 pgs: 109 active+clean; 3.6 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 690 op/s 2023-09-23T21:08:38.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:08:38 smithi026 ceph-mon[175151]: pgmap v1005: 109 pgs: 109 active+clean; 3.6 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 690 op/s 2023-09-23T21:08:38.771 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:08:38.773 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:08:38 smithi070 ceph-mon[168282]: pgmap v1005: 109 pgs: 109 active+clean; 3.6 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 690 op/s 2023-09-23T21:08:38.774 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:08:38.775 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:08:38.875 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:08:38.881 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_historic_ops 2023-09-23T21:08:38.895 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T21:08:38.995 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:08:39.066 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0: {} 2023-09-23T21:08:39.097 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T21:08:39.106 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:08:39.107 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:08:39.127 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1: {} 2023-09-23T21:08:39.169 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2: {} 2023-09-23T21:08:39.207 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:08:39.211 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3: {} 2023-09-23T21:08:39.245 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:08:39.263 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4: {} 2023-09-23T21:08:39.291 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T21:08:39.301 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:08:39.312 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:08:39.312 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:08:39.339 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6: {} 2023-09-23T21:08:39.392 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T21:08:39.412 INFO:teuthology.orchestra.run.smithi026.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T21:08:39.413 INFO:teuthology.orchestra.run.smithi026.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T21:08:39.413 INFO:teuthology.orchestra.run.smithi026.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T21:08:39.413 INFO:teuthology.orchestra.run.smithi026.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T21:08:39.413 INFO:teuthology.orchestra.run.smithi026.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T21:08:39.413 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T21:08:39.413 INFO:teuthology.orchestra.run.smithi026.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T21:08:39.413 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T21:08:39.413 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:08:39.497 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:08:39 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T21:08:39.497 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:08:39 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/824257119' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:08:39.497 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:08:39 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:08:39.498 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:08:39 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:08:39.502 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:08:39 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T21:08:39.502 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:08:39 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/824257119' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:08:39.502 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:08:39 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:08:39.502 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:08:39 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:08:39.516 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:08:39.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:08:39 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T21:08:39.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:08:39 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/824257119' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:08:39.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:08:39 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:08:39.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:08:39 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:08:39.672 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T21:08:39.780 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:08:39.788 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:08:39.789 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 21:08:39 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[164029]: ::ffff:172.21.15.70 - - [23/Sep/2023:21:08:39] "GET /metrics HTTP/1.1" 200 36298 "" "Prometheus/2.43.0" 2023-09-23T21:08:39.789 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:08:39.890 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:08:39.962 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:08:40.074 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:08:40.077 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:08:40.078 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:08:40.178 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:08:40.227 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:08:40.586 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:08:40 smithi070 ceph-mon[168282]: pgmap v1006: 109 pgs: 109 active+clean; 3.5 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 855 op/s 2023-09-23T21:08:40.838 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:08:40.843 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:08:40 smithi026 ceph-mon[175151]: pgmap v1006: 109 pgs: 109 active+clean; 3.5 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 855 op/s 2023-09-23T21:08:40.843 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:08:40 smithi026 ceph-mon[179688]: pgmap v1006: 109 pgs: 109 active+clean; 3.5 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 855 op/s 2023-09-23T21:08:40.852 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:08:40.852 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:08:40.953 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:08:41.029 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:08:41.091 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:08:41.405 INFO:teuthology.orchestra.run.smithi070.stdout:ERROR: (22) Invalid argument 2023-09-23T21:08:41.406 INFO:teuthology.orchestra.run.smithi070.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T21:08:41.882 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_ops_in_flight 2023-09-23T21:08:41.938 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:08:41.948 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:08:41.948 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:08:42.049 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:08:42.076 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T21:08:42.109 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:08:42.178 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:08:42.183 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:08:42.183 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:08:42.284 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:08:42.410 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:08:42.523 INFO:teuthology.orchestra.run.smithi026.stderr:marked out osd.7. 2023-09-23T21:08:42.552 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:08:42 smithi026 ceph-mon[179688]: pgmap v1007: 109 pgs: 109 active+clean; 3.4 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 839 op/s 2023-09-23T21:08:42.553 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:08:42 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/2239758851' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["7"]}]: dispatch 2023-09-23T21:08:42.553 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:08:42 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["7"]}]: dispatch 2023-09-23T21:08:42.553 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:08:42 smithi026 ceph-mon[179688]: Client client.admin marked osd.7 out, while it was still marked up 2023-09-23T21:08:42.553 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:08:42 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mon-a[175127]: 2023-09-23T21:08:42.418+0000 7f1e2d487700 -1 mon.a@0(leader).osd e381 definitely_dead 0 2023-09-23T21:08:42.553 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:08:42 smithi026 ceph-mon[175151]: pgmap v1007: 109 pgs: 109 active+clean; 3.4 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 839 op/s 2023-09-23T21:08:42.553 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:08:42 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/2239758851' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["7"]}]: dispatch 2023-09-23T21:08:42.553 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:08:42 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["7"]}]: dispatch 2023-09-23T21:08:42.553 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:08:42 smithi026 ceph-mon[175151]: Client client.admin marked osd.7 out, while it was still marked up 2023-09-23T21:08:42.665 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T21:08:42.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:08:42 smithi070 ceph-mon[168282]: pgmap v1007: 109 pgs: 109 active+clean; 3.4 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 839 op/s 2023-09-23T21:08:42.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:08:42 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/2239758851' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["7"]}]: dispatch 2023-09-23T21:08:42.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:08:42 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["7"]}]: dispatch 2023-09-23T21:08:42.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:08:42 smithi070 ceph-mon[168282]: Client client.admin marked osd.7 out, while it was still marked up 2023-09-23T21:08:43.272 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:08:43.275 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:08:43.275 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:08:43.376 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:08:43.411 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:08:43.460 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:08:43.551 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:08:43 smithi026 ceph-mon[175151]: failed to encode map e382 with expected crc 2023-09-23T21:08:43.552 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:08:43 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["7"]}]': finished 2023-09-23T21:08:43.552 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:08:43 smithi026 ceph-mon[175151]: osdmap e382: 8 total, 8 up, 7 in 2023-09-23T21:08:43.552 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:08:43 smithi026 ceph-mon[175151]: failed to encode map e382 with expected crc 2023-09-23T21:08:43.552 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:08:43 smithi026 ceph-mon[175151]: failed to encode map e382 with expected crc 2023-09-23T21:08:43.552 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:08:43 smithi026 ceph-mon[179688]: failed to encode map e382 with expected crc 2023-09-23T21:08:43.552 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:08:43 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["7"]}]': finished 2023-09-23T21:08:43.552 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:08:43 smithi026 ceph-mon[179688]: osdmap e382: 8 total, 8 up, 7 in 2023-09-23T21:08:43.552 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:08:43 smithi026 ceph-mon[179688]: failed to encode map e382 with expected crc 2023-09-23T21:08:43.553 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:08:43 smithi026 ceph-mon[179688]: failed to encode map e382 with expected crc 2023-09-23T21:08:43.558 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:08:43.561 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:08:43.562 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:08:43.663 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:08:43.693 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:08:43.801 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:08:43.803 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:08:43 smithi070 ceph-mon[168282]: failed to encode map e382 with expected crc 2023-09-23T21:08:43.803 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:08:43 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["7"]}]': finished 2023-09-23T21:08:43.803 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:08:43 smithi070 ceph-mon[168282]: osdmap e382: 8 total, 8 up, 7 in 2023-09-23T21:08:43.803 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:08:43 smithi070 ceph-mon[168282]: failed to encode map e382 with expected crc 2023-09-23T21:08:43.803 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:08:43 smithi070 ceph-mon[168282]: failed to encode map e382 with expected crc 2023-09-23T21:08:43.804 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:08:43.804 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:08:43.905 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:08:43.934 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:08:43.974 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:08:43.977 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:08:43.977 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:08:44.078 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:08:44.111 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:08:44.144 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T21:08:44.147 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:08:44.148 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:08:44.248 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:08:44.275 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:08:44.309 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:08:44.312 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:08:44.312 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:08:44.333 INFO:teuthology.orchestra.run.smithi070.stdout:ERROR: (22) Invalid argument 2023-09-23T21:08:44.333 INFO:teuthology.orchestra.run.smithi070.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T21:08:44.413 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:08:44.447 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:08:44.508 INFO:teuthology.orchestra.run.smithi026.stderr:noscrub is set 2023-09-23T21:08:44.548 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:08:44.565 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:08:44.566 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:08:44.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:08:44 smithi070 ceph-mon[168282]: pgmap v1009: 109 pgs: 109 active+clean; 3.3 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 754 op/s 2023-09-23T21:08:44.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:08:44 smithi070 ceph-mon[168282]: failed to encode map e383 with expected crc 2023-09-23T21:08:44.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:08:44 smithi070 ceph-mon[168282]: failed to encode map e383 with expected crc 2023-09-23T21:08:44.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:08:44 smithi070 ceph-mon[168282]: osdmap e383: 8 total, 8 up, 7 in 2023-09-23T21:08:44.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:08:44 smithi070 ceph-mon[168282]: failed to encode map e383 with expected crc 2023-09-23T21:08:44.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:08:44 smithi070 ceph-mon[168282]: failed to encode map e383 with expected crc 2023-09-23T21:08:44.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:08:44 smithi070 ceph-mon[168282]: failed to encode map e383 with expected crc 2023-09-23T21:08:44.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:08:44 smithi070 ceph-mon[168282]: failed to encode map e383 with expected crc 2023-09-23T21:08:44.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:08:44 smithi070 ceph-mon[168282]: failed to encode map e383 with expected crc 2023-09-23T21:08:44.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:08:44 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/944809598' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T21:08:44.667 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:08:44.705 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:08:44.759 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_blocked_ops 2023-09-23T21:08:44.794 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:08:44.800 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:08:44 smithi026 ceph-mon[179688]: pgmap v1009: 109 pgs: 109 active+clean; 3.3 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 754 op/s 2023-09-23T21:08:44.800 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:08:44 smithi026 ceph-mon[179688]: failed to encode map e383 with expected crc 2023-09-23T21:08:44.800 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:08:44 smithi026 ceph-mon[179688]: failed to encode map e383 with expected crc 2023-09-23T21:08:44.800 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:08:44 smithi026 ceph-mon[179688]: osdmap e383: 8 total, 8 up, 7 in 2023-09-23T21:08:44.800 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:08:44 smithi026 ceph-mon[179688]: failed to encode map e383 with expected crc 2023-09-23T21:08:44.800 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:08:44 smithi026 ceph-mon[179688]: failed to encode map e383 with expected crc 2023-09-23T21:08:44.800 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:08:44 smithi026 ceph-mon[179688]: failed to encode map e383 with expected crc 2023-09-23T21:08:44.800 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:08:44 smithi026 ceph-mon[179688]: failed to encode map e383 with expected crc 2023-09-23T21:08:44.800 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:08:44 smithi026 ceph-mon[179688]: failed to encode map e383 with expected crc 2023-09-23T21:08:44.800 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:08:44 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/944809598' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T21:08:44.801 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:08:44 smithi026 ceph-mon[175151]: pgmap v1009: 109 pgs: 109 active+clean; 3.3 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 754 op/s 2023-09-23T21:08:44.801 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:08:44 smithi026 ceph-mon[175151]: failed to encode map e383 with expected crc 2023-09-23T21:08:44.801 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:08:44 smithi026 ceph-mon[175151]: failed to encode map e383 with expected crc 2023-09-23T21:08:44.801 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:08:44 smithi026 ceph-mon[175151]: osdmap e383: 8 total, 8 up, 7 in 2023-09-23T21:08:44.802 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:08:44 smithi026 ceph-mon[175151]: failed to encode map e383 with expected crc 2023-09-23T21:08:44.802 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:08:44 smithi026 ceph-mon[175151]: failed to encode map e383 with expected crc 2023-09-23T21:08:44.802 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:08:44 smithi026 ceph-mon[175151]: failed to encode map e383 with expected crc 2023-09-23T21:08:44.802 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:08:44 smithi026 ceph-mon[175151]: failed to encode map e383 with expected crc 2023-09-23T21:08:44.802 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:08:44 smithi026 ceph-mon[175151]: failed to encode map e383 with expected crc 2023-09-23T21:08:44.802 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:08:44 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/944809598' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T21:08:44.802 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:08:44.802 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:08:44.903 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:08:44.972 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:08:45.134 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:08:45.147 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:08:45.147 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:08:45.248 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:08:45.300 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:08:45.335 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0: {} 2023-09-23T21:08:45.368 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:08:45.376 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:08:45.376 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:08:45.396 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1: {} 2023-09-23T21:08:45.450 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2: {} 2023-09-23T21:08:45.477 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:08:45.485 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:08:45 smithi026 ceph-mon[175151]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:08:45.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:08:45 smithi026 ceph-mon[175151]: failed to encode map e384 with expected crc 2023-09-23T21:08:45.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:08:45 smithi026 ceph-mon[175151]: failed to encode map e384 with expected crc 2023-09-23T21:08:45.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:08:45 smithi026 ceph-mon[175151]: failed to encode map e384 with expected crc 2023-09-23T21:08:45.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:08:45 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/944809598' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T21:08:45.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:08:45 smithi026 ceph-mon[175151]: osdmap e384: 8 total, 8 up, 7 in 2023-09-23T21:08:45.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:08:45 smithi026 ceph-mon[175151]: failed to encode map e384 with expected crc 2023-09-23T21:08:45.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:08:45 smithi026 ceph-mon[175151]: 9.1 starting backfill to osd.4 from (0'0,0'0] MAX to 381'11058 2023-09-23T21:08:45.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:08:45 smithi026 ceph-mon[175151]: 9.d starting backfill to osd.4 from (0'0,0'0] MAX to 381'10915 2023-09-23T21:08:45.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:08:45 smithi026 ceph-mon[175151]: 9.a starting backfill to osd.3 from (0'0,0'0] MAX to 381'10703 2023-09-23T21:08:45.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:08:45 smithi026 ceph-mon[175151]: 9.3 starting backfill to osd.5 from (0'0,0'0] MAX to 381'10782 2023-09-23T21:08:45.487 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:08:45 smithi026 ceph-mon[175151]: 9.e starting backfill to osd.0 from (0'0,0'0] MAX to 381'11140 2023-09-23T21:08:45.487 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:08:45 smithi026 ceph-mon[175151]: 9.c starting backfill to osd.0 from (0'0,0'0] MAX to 381'10955 2023-09-23T21:08:45.487 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:08:45 smithi026 ceph-mon[175151]: 9.7 starting backfill to osd.2 from (0'0,0'0] MAX to 381'10964 2023-09-23T21:08:45.487 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:08:45 smithi026 ceph-mon[175151]: 9.9 starting backfill to osd.3 from (0'0,0'0] MAX to 381'10970 2023-09-23T21:08:45.487 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:08:45 smithi026 ceph-mon[175151]: 9.5 starting backfill to osd.1 from (0'0,0'0] MAX to 381'10785 2023-09-23T21:08:45.487 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:08:45 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/1454946420' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:08:45.487 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:08:45 smithi026 ceph-mon[175151]: osdmap e385: 8 total, 8 up, 7 in 2023-09-23T21:08:45.504 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3: {} 2023-09-23T21:08:45.549 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4: {} 2023-09-23T21:08:45.566 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:08:45.574 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T21:08:45.605 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T21:08:45.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:08:45 smithi070 ceph-mon[168282]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:08:45.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:08:45 smithi070 ceph-mon[168282]: failed to encode map e384 with expected crc 2023-09-23T21:08:45.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:08:45 smithi070 ceph-mon[168282]: failed to encode map e384 with expected crc 2023-09-23T21:08:45.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:08:45 smithi070 ceph-mon[168282]: failed to encode map e384 with expected crc 2023-09-23T21:08:45.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:08:45 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/944809598' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T21:08:45.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:08:45 smithi070 ceph-mon[168282]: osdmap e384: 8 total, 8 up, 7 in 2023-09-23T21:08:45.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:08:45 smithi070 ceph-mon[168282]: failed to encode map e384 with expected crc 2023-09-23T21:08:45.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:08:45 smithi070 ceph-mon[168282]: 9.1 starting backfill to osd.4 from (0'0,0'0] MAX to 381'11058 2023-09-23T21:08:45.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:08:45 smithi070 ceph-mon[168282]: 9.d starting backfill to osd.4 from (0'0,0'0] MAX to 381'10915 2023-09-23T21:08:45.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:08:45 smithi070 ceph-mon[168282]: 9.a starting backfill to osd.3 from (0'0,0'0] MAX to 381'10703 2023-09-23T21:08:45.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:08:45 smithi070 ceph-mon[168282]: 9.3 starting backfill to osd.5 from (0'0,0'0] MAX to 381'10782 2023-09-23T21:08:45.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:08:45 smithi070 ceph-mon[168282]: 9.e starting backfill to osd.0 from (0'0,0'0] MAX to 381'11140 2023-09-23T21:08:45.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:08:45 smithi070 ceph-mon[168282]: 9.c starting backfill to osd.0 from (0'0,0'0] MAX to 381'10955 2023-09-23T21:08:45.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:08:45 smithi070 ceph-mon[168282]: 9.7 starting backfill to osd.2 from (0'0,0'0] MAX to 381'10964 2023-09-23T21:08:45.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:08:45 smithi070 ceph-mon[168282]: 9.9 starting backfill to osd.3 from (0'0,0'0] MAX to 381'10970 2023-09-23T21:08:45.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:08:45 smithi070 ceph-mon[168282]: 9.5 starting backfill to osd.1 from (0'0,0'0] MAX to 381'10785 2023-09-23T21:08:45.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:08:45 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/1454946420' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:08:45.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:08:45 smithi070 ceph-mon[168282]: osdmap e385: 8 total, 8 up, 7 in 2023-09-23T21:08:45.646 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6: {} 2023-09-23T21:08:45.666 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T21:08:45.678 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:08:45.678 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:08:45.685 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T21:08:45.701 INFO:teuthology.orchestra.run.smithi026.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T21:08:45.701 INFO:teuthology.orchestra.run.smithi026.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T21:08:45.701 INFO:teuthology.orchestra.run.smithi026.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T21:08:45.701 INFO:teuthology.orchestra.run.smithi026.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T21:08:45.701 INFO:teuthology.orchestra.run.smithi026.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T21:08:45.701 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T21:08:45.701 INFO:teuthology.orchestra.run.smithi026.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T21:08:45.701 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T21:08:45.742 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:08:45 smithi026 ceph-mon[179688]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:08:45.742 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:08:45 smithi026 ceph-mon[179688]: failed to encode map e384 with expected crc 2023-09-23T21:08:45.742 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:08:45 smithi026 ceph-mon[179688]: failed to encode map e384 with expected crc 2023-09-23T21:08:45.742 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:08:45 smithi026 ceph-mon[179688]: failed to encode map e384 with expected crc 2023-09-23T21:08:45.742 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:08:45 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/944809598' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T21:08:45.742 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:08:45 smithi026 ceph-mon[179688]: osdmap e384: 8 total, 8 up, 7 in 2023-09-23T21:08:45.742 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:08:45 smithi026 ceph-mon[179688]: failed to encode map e384 with expected crc 2023-09-23T21:08:45.743 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:08:45 smithi026 ceph-mon[179688]: 9.1 starting backfill to osd.4 from (0'0,0'0] MAX to 381'11058 2023-09-23T21:08:45.743 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:08:45 smithi026 ceph-mon[179688]: 9.d starting backfill to osd.4 from (0'0,0'0] MAX to 381'10915 2023-09-23T21:08:45.743 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:08:45 smithi026 ceph-mon[179688]: 9.a starting backfill to osd.3 from (0'0,0'0] MAX to 381'10703 2023-09-23T21:08:45.743 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:08:45 smithi026 ceph-mon[179688]: 9.3 starting backfill to osd.5 from (0'0,0'0] MAX to 381'10782 2023-09-23T21:08:45.743 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:08:45 smithi026 ceph-mon[179688]: 9.e starting backfill to osd.0 from (0'0,0'0] MAX to 381'11140 2023-09-23T21:08:45.743 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:08:45 smithi026 ceph-mon[179688]: 9.c starting backfill to osd.0 from (0'0,0'0] MAX to 381'10955 2023-09-23T21:08:45.743 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:08:45 smithi026 ceph-mon[179688]: 9.7 starting backfill to osd.2 from (0'0,0'0] MAX to 381'10964 2023-09-23T21:08:45.743 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:08:45 smithi026 ceph-mon[179688]: 9.9 starting backfill to osd.3 from (0'0,0'0] MAX to 381'10970 2023-09-23T21:08:45.743 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:08:45 smithi026 ceph-mon[179688]: 9.5 starting backfill to osd.1 from (0'0,0'0] MAX to 381'10785 2023-09-23T21:08:45.743 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:08:45 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/1454946420' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:08:45.743 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:08:45 smithi026 ceph-mon[179688]: osdmap e385: 8 total, 8 up, 7 in 2023-09-23T21:08:45.779 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:08:45.861 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:08:46.143 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:08:46.149 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:08:46.149 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:08:46.250 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:08:46.381 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:08:46.451 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T21:08:46.456 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:08:46.456 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:08:46.557 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:08:46.601 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:08:46.710 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:08:46.720 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:08:46.721 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:08:46.721 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:08:46 smithi026 ceph-mon[179688]: pgmap v1012: 109 pgs: 2 activating, 9 remapped+peering, 98 active+clean; 3.2 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 768 op/s; 14 B/s, 0 objects/s recovering 2023-09-23T21:08:46.721 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:08:46 smithi026 ceph-mon[179688]: failed to encode map e385 with expected crc 2023-09-23T21:08:46.721 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:08:46 smithi026 ceph-mon[179688]: failed to encode map e385 with expected crc 2023-09-23T21:08:46.721 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:08:46 smithi026 ceph-mon[179688]: failed to encode map e385 with expected crc 2023-09-23T21:08:46.722 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:08:46 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:08:46.722 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:08:46 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:08:46.722 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:08:46 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:08:46.722 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:08:46 smithi026 ceph-mon[179688]: osdmap e386: 8 total, 8 up, 7 in 2023-09-23T21:08:46.722 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:08:46 smithi026 ceph-mon[175151]: pgmap v1012: 109 pgs: 2 activating, 9 remapped+peering, 98 active+clean; 3.2 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 768 op/s; 14 B/s, 0 objects/s recovering 2023-09-23T21:08:46.722 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:08:46 smithi026 ceph-mon[175151]: failed to encode map e385 with expected crc 2023-09-23T21:08:46.722 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:08:46 smithi026 ceph-mon[175151]: failed to encode map e385 with expected crc 2023-09-23T21:08:46.722 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:08:46 smithi026 ceph-mon[175151]: failed to encode map e385 with expected crc 2023-09-23T21:08:46.722 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:08:46 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:08:46.722 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:08:46 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:08:46.723 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:08:46 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:08:46.723 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:08:46 smithi026 ceph-mon[175151]: osdmap e386: 8 total, 8 up, 7 in 2023-09-23T21:08:46.822 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:08:46.827 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:08:46 smithi070 ceph-mon[168282]: pgmap v1012: 109 pgs: 2 activating, 9 remapped+peering, 98 active+clean; 3.2 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 768 op/s; 14 B/s, 0 objects/s recovering 2023-09-23T21:08:46.827 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:08:46 smithi070 ceph-mon[168282]: failed to encode map e385 with expected crc 2023-09-23T21:08:46.827 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:08:46 smithi070 ceph-mon[168282]: failed to encode map e385 with expected crc 2023-09-23T21:08:46.827 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:08:46 smithi070 ceph-mon[168282]: failed to encode map e385 with expected crc 2023-09-23T21:08:46.827 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:08:46 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:08:46.827 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:08:46 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:08:46.827 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:08:46 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:08:46.827 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:08:46 smithi070 ceph-mon[168282]: osdmap e386: 8 total, 8 up, 7 in 2023-09-23T21:08:46.884 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:08:46.952 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:08:46.959 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:08:46.960 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:08:47.060 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:08:47.112 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:08:47.150 INFO:teuthology.orchestra.run.smithi070.stdout:{ 2023-09-23T21:08:47.151 INFO:teuthology.orchestra.run.smithi070.stdout: "ops": [], 2023-09-23T21:08:47.151 INFO:teuthology.orchestra.run.smithi070.stdout: "complaint_time": 30, 2023-09-23T21:08:47.151 INFO:teuthology.orchestra.run.smithi070.stdout: "num_blocked_ops": 0 2023-09-23T21:08:47.151 INFO:teuthology.orchestra.run.smithi070.stdout:} 2023-09-23T21:08:47.173 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:08:47.181 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:08:47.182 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:08:47.217 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T21:08:47.282 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:08:47.388 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:08:47.518 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T21:08:47.525 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:08:47.526 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:08:47.581 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_historic_ops 2023-09-23T21:08:47.626 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:08:47.725 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:08:47.817 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:08:47.822 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:08:47 smithi026 ceph-mon[175151]: failed to encode map e386 with expected crc 2023-09-23T21:08:47.822 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:08:47 smithi026 ceph-mon[175151]: failed to encode map e386 with expected crc 2023-09-23T21:08:47.822 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:08:47 smithi026 ceph-mon[175151]: failed to encode map e386 with expected crc 2023-09-23T21:08:47.822 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:08:47 smithi026 ceph-mon[175151]: failed to encode map e386 with expected crc 2023-09-23T21:08:47.822 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:08:47 smithi026 ceph-mon[179688]: failed to encode map e386 with expected crc 2023-09-23T21:08:47.822 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:08:47 smithi026 ceph-mon[179688]: failed to encode map e386 with expected crc 2023-09-23T21:08:47.822 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:08:47 smithi026 ceph-mon[179688]: failed to encode map e386 with expected crc 2023-09-23T21:08:47.822 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:08:47 smithi026 ceph-mon[179688]: failed to encode map e386 with expected crc 2023-09-23T21:08:47.824 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:08:47.824 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:08:47.872 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:08:47 smithi070 ceph-mon[168282]: failed to encode map e386 with expected crc 2023-09-23T21:08:47.872 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:08:47 smithi070 ceph-mon[168282]: failed to encode map e386 with expected crc 2023-09-23T21:08:47.872 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:08:47 smithi070 ceph-mon[168282]: failed to encode map e386 with expected crc 2023-09-23T21:08:47.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:08:47 smithi070 ceph-mon[168282]: failed to encode map e386 with expected crc 2023-09-23T21:08:47.925 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:08:47.993 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:08:48.062 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T21:08:48.072 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:08:48.073 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:08:48.173 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:08:48.228 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:08:48.251 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T21:08:48.275 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 6, 4, 5] out_osds: [7] dead_osds: [] live_osds: [0, 2, 3, 4, 5, 7, 6, 1] 2023-09-23T21:08:48.275 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T21:08:48.275 INFO:tasks.thrashosds.thrasher:Reweighting osd 6 to 0.628229787685864 2023-09-23T21:08:48.275 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph osd reweight 6 0.628229787685864 2023-09-23T21:08:48.279 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:08:48.283 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:08:48.284 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:08:48.284 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:08:48.309 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T21:08:48.386 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:08:48.420 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:08:48.466 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:08:48.472 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:08:48.473 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:08:48.574 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:08:48.633 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:08:48 smithi026 ceph-mon[175151]: pgmap v1015: 109 pgs: 1 peering, 2 activating, 9 remapped+peering, 97 active+clean; 3.2 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 391 op/s; 22 B/s, 0 objects/s recovering 2023-09-23T21:08:48.633 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:08:48 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:08:48.633 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:08:48 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:08:48.634 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:08:48 smithi026 ceph-mon[179688]: pgmap v1015: 109 pgs: 1 peering, 2 activating, 9 remapped+peering, 97 active+clean; 3.2 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 391 op/s; 22 B/s, 0 objects/s recovering 2023-09-23T21:08:48.634 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:08:48 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:08:48.634 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:08:48 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:08:48.752 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:08:48.868 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:08:48.872 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:08:48 smithi070 ceph-mon[168282]: pgmap v1015: 109 pgs: 1 peering, 2 activating, 9 remapped+peering, 97 active+clean; 3.2 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 391 op/s; 22 B/s, 0 objects/s recovering 2023-09-23T21:08:48.872 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:08:48 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:08:48.872 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:08:48 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:08:48.877 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:08:48.877 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:08:48.978 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:08:49.072 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:08:49.142 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:08:49.153 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:08:49.153 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:08:49.254 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:08:49.347 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:08:49.349 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:08:49.616 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:08:49.623 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:08:49.623 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:08:49.711 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:08:49.724 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:08:49.775 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:08:49 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T21:08:49.775 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:08:49 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T21:08:49.775 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:08:49 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:08:49.776 INFO:teuthology.orchestra.run.smithi070.stdout:{ 2023-09-23T21:08:49.777 INFO:teuthology.orchestra.run.smithi070.stdout: "size": 20, 2023-09-23T21:08:49.777 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 600, 2023-09-23T21:08:49.777 INFO:teuthology.orchestra.run.smithi070.stdout: "ops": [ 2023-09-23T21:08:49.778 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:08:49.778 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.35856.0:52751 9.3 9:c51114d0:::benchmark_data_smithi026_342732_object52750:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e354)", 2023-09-23T21:08:49.778 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:06:24.396732+0000", 2023-09-23T21:08:49.778 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 145.37352507700001, 2023-09-23T21:08:49.778 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.006593885, 2023-09-23T21:08:49.778 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:08:49.778 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "reached pg", 2023-09-23T21:08:49.778 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:08:49.778 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.35856", 2023-09-23T21:08:49.778 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/4180396258", 2023-09-23T21:08:49.778 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 52751 2023-09-23T21:08:49.779 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:08:49.779 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:08:49.779 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:08:49.779 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:08:49.779 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:24.396732+0000", 2023-09-23T21:08:49.779 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:08:49.779 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:08:49.779 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:08:49.779 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:08:49.779 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:24.396732+0000", 2023-09-23T21:08:49.779 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:08:49.779 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:08:49.779 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:08:49.780 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:08:49.780 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:24.396731+0000", 2023-09-23T21:08:49.780 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T21:08:49.780 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:08:49.780 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:08:49.780 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:08:49.780 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:24.396868+0000", 2023-09-23T21:08:49.780 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00013730400000000001 2023-09-23T21:08:49.780 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:08:49.780 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:08:49.780 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:08:49.780 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:24.396883+0000", 2023-09-23T21:08:49.781 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.5221999999999999e-05 2023-09-23T21:08:49.781 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:08:49.781 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:08:49.781 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:08:49.781 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:24.396891+0000", 2023-09-23T21:08:49.781 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.3869999999999997e-06 2023-09-23T21:08:49.781 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:08:49.781 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:08:49.781 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:08:49.781 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:24.397030+0000", 2023-09-23T21:08:49.781 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00013967 2023-09-23T21:08:49.781 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:08:49.781 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:08:49.782 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:08:49.782 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:24.397040+0000", 2023-09-23T21:08:49.782 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.9150000000000008e-06 2023-09-23T21:08:49.782 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:08:49.782 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:08:49.782 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:08:49.782 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:25.402653+0000", 2023-09-23T21:08:49.782 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.00561239 2023-09-23T21:08:49.782 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:08:49.782 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:08:49.782 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:08:49.782 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:25.403326+0000", 2023-09-23T21:08:49.782 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00067348199999999999 2023-09-23T21:08:49.783 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:08:49.783 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:08:49.783 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:08:49.783 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:08:49.783 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:08:49.783 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.35856.0:52765 9.3 9:cd654ac2:::benchmark_data_smithi026_342732_object52764:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e354)", 2023-09-23T21:08:49.783 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:06:24.396908+0000", 2023-09-23T21:08:49.783 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 145.373348956, 2023-09-23T21:08:49.783 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.0245688509999999, 2023-09-23T21:08:49.783 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:08:49.783 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:08:49.783 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:08:49.784 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.35856", 2023-09-23T21:08:49.784 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/4180396258", 2023-09-23T21:08:49.784 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 52765 2023-09-23T21:08:49.784 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:08:49.784 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:08:49.784 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:08:49.784 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:08:49.784 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:24.396908+0000", 2023-09-23T21:08:49.784 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:08:49.784 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:08:49.784 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:08:49.784 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:08:49.785 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:24.396908+0000", 2023-09-23T21:08:49.785 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:08:49.785 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:08:49.785 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:08:49.785 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:08:49.785 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:24.396908+0000", 2023-09-23T21:08:49.785 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999995 2023-09-23T21:08:49.785 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:08:49.785 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:08:49.785 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:08:49.785 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:24.396943+0000", 2023-09-23T21:08:49.785 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.5349999999999999e-05 2023-09-23T21:08:49.786 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:08:49.786 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:08:49.786 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:08:49.786 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:24.396952+0000", 2023-09-23T21:08:49.786 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.6649999999999992e-06 2023-09-23T21:08:49.786 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:08:49.786 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:08:49.786 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:08:49.786 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:24.396957+0000", 2023-09-23T21:08:49.786 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.3410000000000002e-06 2023-09-23T21:08:49.786 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:08:49.786 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:08:49.787 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:08:49.787 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:24.397054+0000", 2023-09-23T21:08:49.787 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.7220999999999996e-05 2023-09-23T21:08:49.787 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:08:49.787 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:08:49.787 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:08:49.787 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:24.397060+0000", 2023-09-23T21:08:49.787 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.4020000000000001e-06 2023-09-23T21:08:49.787 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:08:49.787 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:08:49.787 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:08:49.787 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:25.403373+0000", 2023-09-23T21:08:49.788 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.006313697 2023-09-23T21:08:49.788 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:08:49.788 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:08:49.788 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:08:49.788 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:25.403624+0000", 2023-09-23T21:08:49.788 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00025100200000000001 2023-09-23T21:08:49.788 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:08:49.788 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:08:49.788 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 1,3", 2023-09-23T21:08:49.788 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:25.403787+0000", 2023-09-23T21:08:49.788 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00016209899999999999 2023-09-23T21:08:49.788 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:08:49.789 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:08:49.789 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:08:49.789 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:25.409004+0000", 2023-09-23T21:08:49.789 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0052175640000000001 2023-09-23T21:08:49.789 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:08:49.789 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:08:49.789 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:08:49.789 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:25.421372+0000", 2023-09-23T21:08:49.789 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.012367507999999999 2023-09-23T21:08:49.789 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:08:49.789 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:08:49.789 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:08:49.790 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:25.421418+0000", 2023-09-23T21:08:49.790 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.5871000000000002e-05 2023-09-23T21:08:49.790 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:08:49.790 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:08:49.790 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:08:49.790 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:25.421453+0000", 2023-09-23T21:08:49.790 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.5199e-05 2023-09-23T21:08:49.790 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:08:49.790 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:08:49.790 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:08:49.790 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:25.421477+0000", 2023-09-23T21:08:49.790 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.4539e-05 2023-09-23T21:08:49.791 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:08:49.791 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:08:49.791 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:08:49.791 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:08:49.791 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:08:49.791 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.35856.0:52801 9.3 9:cf43f7eb:::benchmark_data_smithi026_342732_object52800:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e354)", 2023-09-23T21:08:49.791 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:06:24.416107+0000", 2023-09-23T21:08:49.791 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 145.35415020100001, 2023-09-23T21:08:49.791 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.0054936219999999, 2023-09-23T21:08:49.791 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:08:49.791 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:08:49.791 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:08:49.792 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.35856", 2023-09-23T21:08:49.792 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/4180396258", 2023-09-23T21:08:49.792 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 52801 2023-09-23T21:08:49.792 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:08:49.792 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:08:49.792 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:08:49.792 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:08:49.792 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:24.416107+0000", 2023-09-23T21:08:49.792 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:08:49.792 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:08:49.792 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:08:49.792 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:08:49.792 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:24.416107+0000", 2023-09-23T21:08:49.793 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:08:49.793 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:08:49.793 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:08:49.793 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:08:49.793 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:24.416106+0000", 2023-09-23T21:08:49.793 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T21:08:49.793 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:08:49.793 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:08:49.793 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:08:49.793 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:24.416150+0000", 2023-09-23T21:08:49.793 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.4604999999999997e-05 2023-09-23T21:08:49.793 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:08:49.794 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:08:49.794 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:08:49.794 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:24.416167+0000", 2023-09-23T21:08:49.794 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.6693000000000001e-05 2023-09-23T21:08:49.794 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:08:49.794 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:08:49.794 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:08:49.794 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:24.416174+0000", 2023-09-23T21:08:49.794 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.7120000000000003e-06 2023-09-23T21:08:49.794 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:08:49.794 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:08:49.794 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:08:49.795 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:24.416243+0000", 2023-09-23T21:08:49.795 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.9830000000000001e-05 2023-09-23T21:08:49.795 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:08:49.795 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:08:49.795 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:08:49.795 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:24.416253+0000", 2023-09-23T21:08:49.795 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.02e-06 2023-09-23T21:08:49.795 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:08:49.795 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:08:49.795 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:08:49.795 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:25.405162+0000", 2023-09-23T21:08:49.795 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.98890962100000002 2023-09-23T21:08:49.796 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:08:49.796 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:08:49.796 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:08:49.796 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:25.405288+0000", 2023-09-23T21:08:49.796 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00012563099999999999 2023-09-23T21:08:49.796 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:08:49.796 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:08:49.796 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 1,3", 2023-09-23T21:08:49.796 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:25.405448+0000", 2023-09-23T21:08:49.796 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000159872 2023-09-23T21:08:49.796 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:08:49.796 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:08:49.796 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:08:49.797 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:25.411143+0000", 2023-09-23T21:08:49.797 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0056956359999999996 2023-09-23T21:08:49.797 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:08:49.797 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:08:49.797 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:08:49.797 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:25.421523+0000", 2023-09-23T21:08:49.797 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.010379767999999999 2023-09-23T21:08:49.797 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:08:49.797 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:08:49.797 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:08:49.797 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:25.421555+0000", 2023-09-23T21:08:49.797 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.2403000000000001e-05 2023-09-23T21:08:49.797 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:08:49.798 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:08:49.798 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:08:49.798 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:25.421581+0000", 2023-09-23T21:08:49.798 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.5843e-05 2023-09-23T21:08:49.798 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:08:49.798 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:08:49.798 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:08:49.798 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:25.421601+0000", 2023-09-23T21:08:49.798 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.9510000000000001e-05 2023-09-23T21:08:49.798 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:08:49.798 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:08:49.798 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:08:49.799 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:08:49.799 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:08:49.799 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.35856.0:52825 9.3 9:c99b89ee:::benchmark_data_smithi026_342732_object52824:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e354)", 2023-09-23T21:08:49.799 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:06:24.432929+0000", 2023-09-23T21:08:49.799 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 145.337327886, 2023-09-23T21:08:49.799 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.98885604299999996, 2023-09-23T21:08:49.799 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:08:49.799 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:08:49.799 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:08:49.799 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.35856", 2023-09-23T21:08:49.799 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/4180396258", 2023-09-23T21:08:49.799 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 52825 2023-09-23T21:08:49.800 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:08:49.800 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:08:49.800 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:08:49.800 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:08:49.800 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:24.432929+0000", 2023-09-23T21:08:49.800 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:08:49.800 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:08:49.800 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:08:49.800 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:08:49.800 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:24.432929+0000", 2023-09-23T21:08:49.800 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:08:49.800 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:08:49.800 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:08:49.801 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:08:49.801 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:24.432928+0000", 2023-09-23T21:08:49.801 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T21:08:49.801 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:08:49.801 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:08:49.801 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:08:49.801 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:24.432975+0000", 2023-09-23T21:08:49.801 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.7441999999999997e-05 2023-09-23T21:08:49.801 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:08:49.801 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:08:49.801 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:08:49.801 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:24.432987+0000", 2023-09-23T21:08:49.802 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.1712999999999999e-05 2023-09-23T21:08:49.802 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:08:49.802 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:08:49.802 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:08:49.802 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:24.432993+0000", 2023-09-23T21:08:49.802 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.7640000000000002e-06 2023-09-23T21:08:49.802 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:08:49.802 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:08:49.802 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:08:49.802 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:24.433060+0000", 2023-09-23T21:08:49.802 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.6691000000000002e-05 2023-09-23T21:08:49.802 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:08:49.802 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:08:49.803 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:08:49.803 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:24.433071+0000", 2023-09-23T21:08:49.803 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.1163e-05 2023-09-23T21:08:49.803 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:08:49.803 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:08:49.803 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:08:49.803 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:25.406642+0000", 2023-09-23T21:08:49.803 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.97357167 2023-09-23T21:08:49.803 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:08:49.803 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:08:49.803 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:08:49.804 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:25.406738+0000", 2023-09-23T21:08:49.804 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.5882000000000001e-05 2023-09-23T21:08:49.804 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:08:49.804 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:08:49.804 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 1,3", 2023-09-23T21:08:49.804 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:25.406824+0000", 2023-09-23T21:08:49.804 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.5877999999999993e-05 2023-09-23T21:08:49.804 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:08:49.804 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:08:49.804 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:08:49.804 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:25.411177+0000", 2023-09-23T21:08:49.804 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0043524490000000004 2023-09-23T21:08:49.805 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:08:49.805 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:08:49.805 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:08:49.805 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:25.421631+0000", 2023-09-23T21:08:49.805 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.010454234 2023-09-23T21:08:49.805 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:08:49.805 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:08:49.805 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:08:49.805 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:25.421724+0000", 2023-09-23T21:08:49.805 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.3381000000000006e-05 2023-09-23T21:08:49.805 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:08:49.805 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:08:49.805 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:08:49.806 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:25.421766+0000", 2023-09-23T21:08:49.806 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.1347000000000002e-05 2023-09-23T21:08:49.806 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:08:49.806 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:08:49.806 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:08:49.806 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:25.421786+0000", 2023-09-23T21:08:49.806 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.9942e-05 2023-09-23T21:08:49.806 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:08:49.806 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:08:49.806 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:08:49.806 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:08:49.806 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:08:49.807 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.35856.0:52839 9.3 9:c25308eb:::benchmark_data_smithi026_342732_object52838:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e354)", 2023-09-23T21:08:49.807 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:06:24.442455+0000", 2023-09-23T21:08:49.807 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 145.327802424, 2023-09-23T21:08:49.807 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.979453766, 2023-09-23T21:08:49.807 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:08:49.807 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:08:49.807 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:08:49.807 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.35856", 2023-09-23T21:08:49.807 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/4180396258", 2023-09-23T21:08:49.807 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 52839 2023-09-23T21:08:49.807 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:08:49.807 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:08:49.807 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:08:49.808 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:08:49.808 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:24.442455+0000", 2023-09-23T21:08:49.808 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:08:49.808 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:08:49.808 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:08:49.808 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:08:49.808 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:24.442455+0000", 2023-09-23T21:08:49.808 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:08:49.808 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:08:49.808 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:08:49.808 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:08:49.808 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:24.442453+0000", 2023-09-23T21:08:49.809 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T21:08:49.809 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:08:49.809 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:08:49.809 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:08:49.809 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:24.442566+0000", 2023-09-23T21:08:49.809 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00011271400000000001 2023-09-23T21:08:49.809 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:08:49.809 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:08:49.809 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:08:49.809 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:24.442582+0000", 2023-09-23T21:08:49.809 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.579e-05 2023-09-23T21:08:49.809 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:08:49.810 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:08:49.810 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:08:49.810 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:24.442589+0000", 2023-09-23T21:08:49.810 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.1559999999999998e-06 2023-09-23T21:08:49.810 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:08:49.810 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:08:49.810 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:08:49.810 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:24.442645+0000", 2023-09-23T21:08:49.810 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.6020999999999998e-05 2023-09-23T21:08:49.810 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:08:49.810 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:08:49.810 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:08:49.811 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:24.442652+0000", 2023-09-23T21:08:49.811 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.6120000000000004e-06 2023-09-23T21:08:49.811 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:08:49.811 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:08:49.811 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:08:49.811 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:25.408020+0000", 2023-09-23T21:08:49.811 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.96536787999999996 2023-09-23T21:08:49.811 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:08:49.811 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:08:49.811 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:08:49.811 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:25.408105+0000", 2023-09-23T21:08:49.811 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.5282000000000001e-05 2023-09-23T21:08:49.811 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:08:49.812 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:08:49.812 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 1,3", 2023-09-23T21:08:49.812 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:25.408218+0000", 2023-09-23T21:08:49.812 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000112796 2023-09-23T21:08:49.812 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:08:49.812 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:08:49.812 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:08:49.812 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:25.413318+0000", 2023-09-23T21:08:49.812 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0051002160000000003 2023-09-23T21:08:49.812 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:08:49.812 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:08:49.812 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:08:49.813 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:25.421648+0000", 2023-09-23T21:08:49.813 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0083301109999999994 2023-09-23T21:08:49.813 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:08:49.813 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:08:49.813 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:08:49.813 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:25.421828+0000", 2023-09-23T21:08:49.813 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00017987299999999999 2023-09-23T21:08:49.813 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:08:49.813 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:08:49.813 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:08:49.813 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:25.421878+0000", 2023-09-23T21:08:49.813 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.0591999999999999e-05 2023-09-23T21:08:49.813 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:08:49.814 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:08:49.814 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:08:49.814 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:25.421909+0000", 2023-09-23T21:08:49.814 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.0280000000000001e-05 2023-09-23T21:08:49.814 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:08:49.814 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:08:49.814 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:08:49.814 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:08:49.814 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:08:49.814 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.35856.0:52856 9.3 9:cea642a7:::benchmark_data_smithi026_342732_object52855:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e354)", 2023-09-23T21:08:49.814 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:06:24.452334+0000", 2023-09-23T21:08:49.814 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 145.31792324099999, 2023-09-23T21:08:49.815 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.96970934900000005, 2023-09-23T21:08:49.815 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:08:49.815 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:08:49.815 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:08:49.815 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.35856", 2023-09-23T21:08:49.815 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/4180396258", 2023-09-23T21:08:49.815 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 52856 2023-09-23T21:08:49.815 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:08:49.815 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:08:49.815 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:08:49.815 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:08:49.815 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:24.452334+0000", 2023-09-23T21:08:49.816 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:08:49.816 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:08:49.816 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:08:49.816 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:08:49.816 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:24.452334+0000", 2023-09-23T21:08:49.816 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:08:49.816 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:08:49.816 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:08:49.816 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:08:49.816 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:24.452333+0000", 2023-09-23T21:08:49.816 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T21:08:49.816 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:08:49.817 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:08:49.817 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:08:49.817 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:24.452378+0000", 2023-09-23T21:08:49.817 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.5096000000000003e-05 2023-09-23T21:08:49.817 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:08:49.817 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:08:49.817 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:08:49.817 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:24.452394+0000", 2023-09-23T21:08:49.817 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.5888000000000001e-05 2023-09-23T21:08:49.817 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:08:49.817 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:08:49.817 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:08:49.817 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:24.452400+0000", 2023-09-23T21:08:49.818 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.6980000000000001e-06 2023-09-23T21:08:49.818 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:08:49.818 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:08:49.818 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:08:49.818 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:24.452459+0000", 2023-09-23T21:08:49.818 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.8594999999999997e-05 2023-09-23T21:08:49.818 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:08:49.818 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:08:49.818 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:08:49.818 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:24.452468+0000", 2023-09-23T21:08:49.818 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.6929999999999996e-06 2023-09-23T21:08:49.818 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:08:49.819 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:08:49.819 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:08:49.819 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:25.409045+0000", 2023-09-23T21:08:49.819 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.95657733099999998 2023-09-23T21:08:49.819 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:08:49.819 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:08:49.819 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:08:49.819 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:25.409143+0000", 2023-09-23T21:08:49.819 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.8317000000000001e-05 2023-09-23T21:08:49.819 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:08:49.819 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:08:49.819 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 1,3", 2023-09-23T21:08:49.820 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:25.409239+0000", 2023-09-23T21:08:49.820 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.6280000000000007e-05 2023-09-23T21:08:49.820 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:08:49.820 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:08:49.820 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:08:49.820 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:25.413327+0000", 2023-09-23T21:08:49.820 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0040874220000000003 2023-09-23T21:08:49.820 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:08:49.820 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:08:49.820 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:08:49.820 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:25.421696+0000", 2023-09-23T21:08:49.820 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0083695439999999996 2023-09-23T21:08:49.821 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:08:49.821 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:08:49.821 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:08:49.821 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:25.421994+0000", 2023-09-23T21:08:49.821 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00029776299999999999 2023-09-23T21:08:49.821 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:08:49.821 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:08:49.821 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:08:49.821 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:25.422025+0000", 2023-09-23T21:08:49.821 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.0880000000000002e-05 2023-09-23T21:08:49.821 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:08:49.821 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:08:49.821 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:08:49.822 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:25.422043+0000", 2023-09-23T21:08:49.822 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.8410000000000002e-05 2023-09-23T21:08:49.822 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:08:49.822 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:08:49.822 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:08:49.822 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:08:49.822 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:08:49.822 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.35856.0:52876 9.3 9:c116084f:::benchmark_data_smithi026_342732_object52875:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e354)", 2023-09-23T21:08:49.822 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:06:24.468836+0000", 2023-09-23T21:08:49.822 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 145.30142169300001, 2023-09-23T21:08:49.822 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.95333874799999996, 2023-09-23T21:08:49.823 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:08:49.823 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:08:49.823 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:08:49.823 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.35856", 2023-09-23T21:08:49.823 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/4180396258", 2023-09-23T21:08:49.823 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 52876 2023-09-23T21:08:49.823 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:08:49.823 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:08:49.823 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:08:49.823 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:08:49.823 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:24.468836+0000", 2023-09-23T21:08:49.823 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:08:49.824 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:08:49.824 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:08:49.824 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:08:49.824 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:24.468836+0000", 2023-09-23T21:08:49.824 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:08:49.824 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:08:49.824 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:08:49.824 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:08:49.824 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:24.468834+0000", 2023-09-23T21:08:49.824 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T21:08:49.824 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:08:49.824 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:08:49.824 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:08:49.825 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:24.468883+0000", 2023-09-23T21:08:49.825 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.8093000000000002e-05 2023-09-23T21:08:49.825 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:08:49.825 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:08:49.825 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:08:49.825 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:24.468895+0000", 2023-09-23T21:08:49.825 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.2425999999999999e-05 2023-09-23T21:08:49.825 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:08:49.825 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:08:49.825 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:08:49.826 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:24.468902+0000", 2023-09-23T21:08:49.826 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.6529999999999997e-06 2023-09-23T21:08:49.826 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:08:49.826 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:08:49.826 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:08:49.826 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:24.468973+0000", 2023-09-23T21:08:49.826 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.1608999999999998e-05 2023-09-23T21:08:49.826 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:08:49.826 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:08:49.826 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:08:49.826 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:24.468986+0000", 2023-09-23T21:08:49.826 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.225e-05 2023-09-23T21:08:49.827 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:08:49.827 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:08:49.827 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:08:49.827 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:25.410042+0000", 2023-09-23T21:08:49.827 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.94105615099999995 2023-09-23T21:08:49.827 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:08:49.827 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:08:49.827 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:08:49.827 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:25.410150+0000", 2023-09-23T21:08:49.827 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000108099 2023-09-23T21:08:49.827 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:08:49.827 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:08:49.828 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 1,3", 2023-09-23T21:08:49.828 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:25.410307+0000", 2023-09-23T21:08:49.828 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00015735 2023-09-23T21:08:49.828 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:08:49.828 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:08:49.828 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:08:49.828 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:25.415575+0000", 2023-09-23T21:08:49.828 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0052674810000000001 2023-09-23T21:08:49.828 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:08:49.828 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:08:49.828 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:08:49.828 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:25.421961+0000", 2023-09-23T21:08:49.828 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0063868579999999996 2023-09-23T21:08:49.829 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:08:49.829 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:08:49.829 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:08:49.829 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:25.422089+0000", 2023-09-23T21:08:49.829 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00012751799999999999 2023-09-23T21:08:49.829 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:08:49.829 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:08:49.829 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:08:49.829 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:25.422138+0000", 2023-09-23T21:08:49.829 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.8755000000000003e-05 2023-09-23T21:08:49.830 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:08:49.830 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:08:49.830 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:08:49.830 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:25.422174+0000", 2023-09-23T21:08:49.830 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.6712999999999999e-05 2023-09-23T21:08:49.830 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:08:49.830 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:08:49.830 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:08:49.830 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:08:49.830 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:08:49.830 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.35856.0:52877 9.3 9:c2be6f32:::benchmark_data_smithi026_342732_object52876:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e354)", 2023-09-23T21:08:49.830 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:06:24.474477+0000", 2023-09-23T21:08:49.831 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 145.29578035500001, 2023-09-23T21:08:49.831 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.94913964100000003, 2023-09-23T21:08:49.831 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:08:49.831 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:08:49.831 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:08:49.831 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.35856", 2023-09-23T21:08:49.831 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/4180396258", 2023-09-23T21:08:49.831 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 52877 2023-09-23T21:08:49.831 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:08:49.831 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:08:49.831 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:08:49.831 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:08:49.831 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:24.474477+0000", 2023-09-23T21:08:49.832 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:08:49.832 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:08:49.832 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:08:49.832 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:08:49.832 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:24.474477+0000", 2023-09-23T21:08:49.832 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:08:49.832 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:08:49.832 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:08:49.832 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:08:49.832 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:24.474476+0000", 2023-09-23T21:08:49.832 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.999999 2023-09-23T21:08:49.832 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:08:49.833 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:08:49.833 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:08:49.833 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:24.474528+0000", 2023-09-23T21:08:49.833 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.1582999999999997e-05 2023-09-23T21:08:49.833 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:08:49.833 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:08:49.833 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:08:49.833 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:24.474539+0000", 2023-09-23T21:08:49.833 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.1895e-05 2023-09-23T21:08:49.833 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:08:49.833 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:08:49.833 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:08:49.834 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:24.474546+0000", 2023-09-23T21:08:49.834 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.0700000000000001e-06 2023-09-23T21:08:49.834 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:08:49.834 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:08:49.834 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:08:49.834 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:24.474606+0000", 2023-09-23T21:08:49.834 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.9349999999999999e-05 2023-09-23T21:08:49.834 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:08:49.834 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:08:49.834 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:08:49.834 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:24.474614+0000", 2023-09-23T21:08:49.835 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.4889999999999995e-06 2023-09-23T21:08:49.835 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:08:49.835 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:08:49.835 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:08:49.835 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:25.411215+0000", 2023-09-23T21:08:49.835 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.93660084300000002 2023-09-23T21:08:49.835 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:08:49.835 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:08:49.835 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:08:49.835 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:25.411317+0000", 2023-09-23T21:08:49.835 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000102122 2023-09-23T21:08:49.835 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:08:49.836 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:08:49.836 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 1,3", 2023-09-23T21:08:49.836 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:25.411414+0000", 2023-09-23T21:08:49.836 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.6571999999999996e-05 2023-09-23T21:08:49.836 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:08:49.836 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:08:49.836 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:08:49.836 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:25.415584+0000", 2023-09-23T21:08:49.836 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0041698890000000004 2023-09-23T21:08:49.836 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:08:49.836 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:08:49.836 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:08:49.836 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:25.422211+0000", 2023-09-23T21:08:49.837 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0066269459999999999 2023-09-23T21:08:49.837 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:08:49.837 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:08:49.837 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:08:49.837 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:25.423526+0000", 2023-09-23T21:08:49.837 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0013153139999999999 2023-09-23T21:08:49.837 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:08:49.837 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:08:49.837 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:08:49.837 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:25.423581+0000", 2023-09-23T21:08:49.837 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.4602999999999999e-05 2023-09-23T21:08:49.838 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:08:49.838 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:08:49.838 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:08:49.838 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:25.423617+0000", 2023-09-23T21:08:49.838 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.6056000000000002e-05 2023-09-23T21:08:49.838 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:08:49.838 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:08:49.838 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:08:49.838 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:08:49.838 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:08:49.838 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.35856.0:52886 9.3 9:ccc0ebe9:::benchmark_data_smithi026_342732_object52885:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e354)", 2023-09-23T21:08:49.838 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:06:24.484834+0000", 2023-09-23T21:08:49.839 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 145.28542344600001, 2023-09-23T21:08:49.839 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.93888714799999995, 2023-09-23T21:08:49.839 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:08:49.839 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:08:49.839 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:08:49.839 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.35856", 2023-09-23T21:08:49.839 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/4180396258", 2023-09-23T21:08:49.839 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 52886 2023-09-23T21:08:49.839 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:08:49.839 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:08:49.839 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:08:49.839 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:08:49.839 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:24.484834+0000", 2023-09-23T21:08:49.840 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:08:49.840 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:08:49.840 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:08:49.840 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:08:49.840 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:24.484834+0000", 2023-09-23T21:08:49.840 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:08:49.840 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:08:49.840 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:08:49.840 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:08:49.840 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:24.484833+0000", 2023-09-23T21:08:49.840 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T21:08:49.840 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:08:49.841 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:08:49.841 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:08:49.841 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:24.484888+0000", 2023-09-23T21:08:49.841 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.5315000000000001e-05 2023-09-23T21:08:49.841 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:08:49.841 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:08:49.841 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:08:49.841 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:24.484901+0000", 2023-09-23T21:08:49.841 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.2958e-05 2023-09-23T21:08:49.841 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:08:49.841 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:08:49.841 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:08:49.842 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:24.484906+0000", 2023-09-23T21:08:49.842 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.4609999999999999e-06 2023-09-23T21:08:49.842 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:08:49.842 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:08:49.842 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:08:49.842 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:24.484972+0000", 2023-09-23T21:08:49.842 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.5840999999999995e-05 2023-09-23T21:08:49.842 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:08:49.842 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:08:49.842 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:08:49.842 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:24.484984+0000", 2023-09-23T21:08:49.842 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.1969e-05 2023-09-23T21:08:49.842 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:08:49.843 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:08:49.843 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:08:49.843 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:25.412238+0000", 2023-09-23T21:08:49.843 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.92725426899999996 2023-09-23T21:08:49.843 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:08:49.843 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:08:49.843 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:08:49.843 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:25.412359+0000", 2023-09-23T21:08:49.843 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00012057600000000001 2023-09-23T21:08:49.843 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:08:49.843 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:08:49.843 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 1,3", 2023-09-23T21:08:49.844 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:25.412512+0000", 2023-09-23T21:08:49.844 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00015280400000000001 2023-09-23T21:08:49.844 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:08:49.844 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:08:49.844 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:08:49.844 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:25.417774+0000", 2023-09-23T21:08:49.844 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0052620660000000001 2023-09-23T21:08:49.844 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:08:49.844 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:08:49.844 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:08:49.844 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:25.422260+0000", 2023-09-23T21:08:49.844 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0044861800000000002 2023-09-23T21:08:49.845 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:08:49.845 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:08:49.845 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:08:49.845 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:25.423663+0000", 2023-09-23T21:08:49.845 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.001402915 2023-09-23T21:08:49.845 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:08:49.845 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:08:49.845 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:08:49.845 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:25.423698+0000", 2023-09-23T21:08:49.846 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.5021999999999998e-05 2023-09-23T21:08:49.846 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:08:49.846 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:08:49.846 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:08:49.846 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:25.423721+0000", 2023-09-23T21:08:49.846 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.3166000000000001e-05 2023-09-23T21:08:49.846 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:08:49.846 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:08:49.846 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:08:49.846 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:08:49.846 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:08:49.846 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.35856.0:52898 9.3 9:ce13b48e:::benchmark_data_smithi026_342732_object52897:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e354)", 2023-09-23T21:08:49.846 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:06:24.502098+0000", 2023-09-23T21:08:49.847 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 145.26815986, 2023-09-23T21:08:49.847 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.92904386800000005, 2023-09-23T21:08:49.847 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:08:49.847 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:08:49.847 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:08:49.847 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.35856", 2023-09-23T21:08:49.847 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/4180396258", 2023-09-23T21:08:49.847 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 52898 2023-09-23T21:08:49.847 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:08:49.847 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:08:49.847 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:08:49.847 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:08:49.848 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:24.502098+0000", 2023-09-23T21:08:49.848 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:08:49.848 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:08:49.848 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:08:49.848 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:08:49.848 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:24.502098+0000", 2023-09-23T21:08:49.848 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:08:49.848 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:08:49.848 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:08:49.848 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:08:49.848 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:24.502096+0000", 2023-09-23T21:08:49.848 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999981 2023-09-23T21:08:49.849 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:08:49.849 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:08:49.849 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:08:49.849 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:24.502121+0000", 2023-09-23T21:08:49.849 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.5218000000000001e-05 2023-09-23T21:08:49.849 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:08:49.849 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:08:49.849 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:08:49.849 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:24.502135+0000", 2023-09-23T21:08:49.849 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.3606e-05 2023-09-23T21:08:49.849 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:08:49.849 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:08:49.849 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:08:49.850 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:24.502142+0000", 2023-09-23T21:08:49.850 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.2590000000000003e-06 2023-09-23T21:08:49.850 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:08:49.850 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:08:49.850 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:08:49.850 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:24.502203+0000", 2023-09-23T21:08:49.850 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.1614999999999999e-05 2023-09-23T21:08:49.850 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:08:49.850 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:08:49.850 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:08:49.850 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:24.502212+0000", 2023-09-23T21:08:49.850 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.9080000000000001e-06 2023-09-23T21:08:49.851 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:08:49.851 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:08:49.851 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:08:49.851 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:25.413366+0000", 2023-09-23T21:08:49.851 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.91115327199999996 2023-09-23T21:08:49.851 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:08:49.851 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:08:49.851 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:08:49.851 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:25.413462+0000", 2023-09-23T21:08:49.851 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.6717000000000003e-05 2023-09-23T21:08:49.851 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:08:49.851 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:08:49.852 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 1,3", 2023-09-23T21:08:49.852 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:25.413554+0000", 2023-09-23T21:08:49.852 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.1895e-05 2023-09-23T21:08:49.852 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:08:49.852 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:08:49.852 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:08:49.852 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:25.417800+0000", 2023-09-23T21:08:49.852 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0042454440000000001 2023-09-23T21:08:49.852 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:08:49.852 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:08:49.852 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:08:49.852 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:25.423772+0000", 2023-09-23T21:08:49.852 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0059727590000000002 2023-09-23T21:08:49.853 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:08:49.853 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:08:49.853 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:08:49.853 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:25.431044+0000", 2023-09-23T21:08:49.853 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0072718139999999997 2023-09-23T21:08:49.853 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:08:49.853 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:08:49.853 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:08:49.853 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:25.431098+0000", 2023-09-23T21:08:49.853 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.3832999999999998e-05 2023-09-23T21:08:49.853 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:08:49.853 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:08:49.853 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:08:49.853 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:25.431141+0000", 2023-09-23T21:08:49.854 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.3282000000000003e-05 2023-09-23T21:08:49.854 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:08:49.854 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:08:49.854 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:08:49.854 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:08:49.854 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:08:49.854 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.35856.0:52915 9.3 9:ca2a2df5:::benchmark_data_smithi026_342732_object52914:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e354)", 2023-09-23T21:08:49.854 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:06:24.520294+0000", 2023-09-23T21:08:49.854 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 145.24996326799999, 2023-09-23T21:08:49.854 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.91094416700000003, 2023-09-23T21:08:49.854 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:08:49.854 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:08:49.855 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:08:49.855 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.35856", 2023-09-23T21:08:49.855 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/4180396258", 2023-09-23T21:08:49.855 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 52915 2023-09-23T21:08:49.855 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:08:49.855 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:08:49.855 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:08:49.855 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:08:49.855 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:24.520294+0000", 2023-09-23T21:08:49.855 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:08:49.855 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:08:49.855 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:08:49.856 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:08:49.856 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:24.520294+0000", 2023-09-23T21:08:49.856 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:08:49.856 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:08:49.856 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:08:49.856 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:08:49.856 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:24.520293+0000", 2023-09-23T21:08:49.856 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T21:08:49.856 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:08:49.856 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:08:49.856 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:08:49.856 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:24.520346+0000", 2023-09-23T21:08:49.856 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.3087999999999997e-05 2023-09-23T21:08:49.856 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:08:49.857 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:08:49.857 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:08:49.857 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:24.520357+0000", 2023-09-23T21:08:49.857 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.1335000000000001e-05 2023-09-23T21:08:49.857 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:08:49.857 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:08:49.857 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:08:49.857 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:24.520362+0000", 2023-09-23T21:08:49.857 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.5839999999999996e-06 2023-09-23T21:08:49.857 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:08:49.857 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:08:49.857 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:08:49.857 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:24.520410+0000", 2023-09-23T21:08:49.858 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.7945000000000003e-05 2023-09-23T21:08:49.858 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:08:49.858 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:08:49.858 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:08:49.858 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:24.520418+0000", 2023-09-23T21:08:49.858 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.7029999999999995e-06 2023-09-23T21:08:49.858 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:08:49.858 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:08:49.858 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:08:49.858 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:25.414439+0000", 2023-09-23T21:08:49.858 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.89402078900000004 2023-09-23T21:08:49.858 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:08:49.858 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:08:49.859 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:08:49.859 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:25.414565+0000", 2023-09-23T21:08:49.859 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00012614700000000001 2023-09-23T21:08:49.859 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:08:49.859 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:08:49.859 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 1,3", 2023-09-23T21:08:49.859 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:25.414721+0000", 2023-09-23T21:08:49.859 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000155601 2023-09-23T21:08:49.859 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:08:49.859 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:08:49.859 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:08:49.859 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:25.420955+0000", 2023-09-23T21:08:49.859 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0062335200000000002 2023-09-23T21:08:49.860 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:08:49.860 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:08:49.860 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:08:49.860 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:25.423821+0000", 2023-09-23T21:08:49.860 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0028662779999999999 2023-09-23T21:08:49.860 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:08:49.860 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:08:49.860 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:08:49.860 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:25.431172+0000", 2023-09-23T21:08:49.860 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0073511369999999998 2023-09-23T21:08:49.860 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:08:49.860 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:08:49.860 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:08:49.861 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:25.431206+0000", 2023-09-23T21:08:49.861 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.4307000000000002e-05 2023-09-23T21:08:49.861 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:08:49.861 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:08:49.861 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:08:49.861 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:25.431238+0000", 2023-09-23T21:08:49.861 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.2027999999999999e-05 2023-09-23T21:08:49.861 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:08:49.861 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:08:49.861 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:08:49.861 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:08:49.861 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:08:49.861 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.35856.0:52919 9.3 9:c1c34947:::benchmark_data_smithi026_342732_object52918:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e354)", 2023-09-23T21:08:49.862 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:06:24.523819+0000", 2023-09-23T21:08:49.862 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 145.24643820099999, 2023-09-23T21:08:49.862 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.90866214300000003, 2023-09-23T21:08:49.862 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:08:49.862 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:08:49.862 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:08:49.862 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.35856", 2023-09-23T21:08:49.862 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/4180396258", 2023-09-23T21:08:49.862 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 52919 2023-09-23T21:08:49.862 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:08:49.862 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:08:49.862 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:08:49.863 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:08:49.863 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:24.523819+0000", 2023-09-23T21:08:49.863 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:08:49.863 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:08:49.863 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:08:49.863 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:08:49.863 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:24.523819+0000", 2023-09-23T21:08:49.863 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:08:49.863 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:08:49.863 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:08:49.863 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:08:49.863 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:24.523818+0000", 2023-09-23T21:08:49.863 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.999999 2023-09-23T21:08:49.864 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:08:49.864 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:08:49.864 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:08:49.864 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:24.523838+0000", 2023-09-23T21:08:49.864 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.9938000000000001e-05 2023-09-23T21:08:49.864 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:08:49.864 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:08:49.864 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:08:49.864 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:24.523849+0000", 2023-09-23T21:08:49.864 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.0617e-05 2023-09-23T21:08:49.864 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:08:49.864 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:08:49.864 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:08:49.865 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:24.523853+0000", 2023-09-23T21:08:49.865 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.2860000000000001e-06 2023-09-23T21:08:49.865 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:08:49.865 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:08:49.865 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:08:49.865 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:24.523911+0000", 2023-09-23T21:08:49.865 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.8381000000000002e-05 2023-09-23T21:08:49.865 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:08:49.865 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:08:49.865 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:08:49.865 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:24.523920+0000", 2023-09-23T21:08:49.865 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.8279999999999992e-06 2023-09-23T21:08:49.865 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:08:49.866 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:08:49.866 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:08:49.866 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:25.415624+0000", 2023-09-23T21:08:49.866 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.89170378500000003 2023-09-23T21:08:49.866 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:08:49.866 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:08:49.866 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:08:49.866 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:25.415717+0000", 2023-09-23T21:08:49.866 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.3216999999999999e-05 2023-09-23T21:08:49.866 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:08:49.866 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:08:49.866 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 1,3", 2023-09-23T21:08:49.866 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:25.415813+0000", 2023-09-23T21:08:49.867 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.5261999999999997e-05 2023-09-23T21:08:49.867 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:08:49.867 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:08:49.867 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:08:49.867 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:25.420983+0000", 2023-09-23T21:08:49.867 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0051701919999999997 2023-09-23T21:08:49.867 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:08:49.867 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:08:49.867 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:08:49.867 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:25.429934+0000", 2023-09-23T21:08:49.867 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0089515629999999992 2023-09-23T21:08:49.867 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:08:49.867 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:08:49.868 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:08:49.868 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:25.432395+0000", 2023-09-23T21:08:49.868 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0024608130000000001 2023-09-23T21:08:49.868 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:08:49.868 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:08:49.868 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:08:49.868 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:25.432445+0000", 2023-09-23T21:08:49.868 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.0336000000000001e-05 2023-09-23T21:08:49.868 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:08:49.868 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:08:49.868 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:08:49.868 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:25.432481+0000", 2023-09-23T21:08:49.869 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.5899999999999998e-05 2023-09-23T21:08:49.869 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:08:49.869 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:08:49.869 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:08:49.869 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:08:49.869 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:08:49.869 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.35856.0:52927 9.3 9:c0ef7e28:::benchmark_data_smithi026_342732_object52926:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e354)", 2023-09-23T21:08:49.869 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:06:24.534140+0000", 2023-09-23T21:08:49.869 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 145.23611709799999, 2023-09-23T21:08:49.869 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.90076572399999999, 2023-09-23T21:08:49.869 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:08:49.869 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:08:49.869 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:08:49.870 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.35856", 2023-09-23T21:08:49.870 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/4180396258", 2023-09-23T21:08:49.870 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 52927 2023-09-23T21:08:49.870 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:08:49.870 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:08:49.870 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:08:49.870 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:08:49.870 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:24.534140+0000", 2023-09-23T21:08:49.870 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:08:49.870 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:08:49.870 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:08:49.870 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:08:49.871 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:24.534140+0000", 2023-09-23T21:08:49.871 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:08:49.871 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:08:49.871 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:08:49.871 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:08:49.871 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:24.534139+0000", 2023-09-23T21:08:49.871 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T21:08:49.871 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:08:49.871 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:08:49.871 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:08:49.871 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:24.534175+0000", 2023-09-23T21:08:49.871 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.5553999999999999e-05 2023-09-23T21:08:49.872 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:08:49.872 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:08:49.872 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:08:49.872 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:24.534186+0000", 2023-09-23T21:08:49.872 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.1334e-05 2023-09-23T21:08:49.872 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:08:49.872 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:08:49.872 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:08:49.872 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:24.534192+0000", 2023-09-23T21:08:49.872 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.9950000000000002e-06 2023-09-23T21:08:49.872 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:08:49.872 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:08:49.872 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:08:49.873 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:24.534258+0000", 2023-09-23T21:08:49.873 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.6299999999999999e-05 2023-09-23T21:08:49.873 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:08:49.873 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:08:49.873 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:08:49.873 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:24.534275+0000", 2023-09-23T21:08:49.873 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.6691e-05 2023-09-23T21:08:49.873 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:08:49.873 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:08:49.873 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:08:49.873 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:25.416673+0000", 2023-09-23T21:08:49.873 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.88239785500000001 2023-09-23T21:08:49.874 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:08:49.874 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:08:49.874 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:08:49.874 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:25.416787+0000", 2023-09-23T21:08:49.874 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000114636 2023-09-23T21:08:49.874 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:08:49.874 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:08:49.874 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 1,3", 2023-09-23T21:08:49.874 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:25.416936+0000", 2023-09-23T21:08:49.874 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00014905300000000001 2023-09-23T21:08:49.874 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:08:49.874 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:08:49.875 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:08:49.875 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:25.421097+0000", 2023-09-23T21:08:49.875 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0041605940000000001 2023-09-23T21:08:49.875 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:08:49.875 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:08:49.875 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:08:49.875 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:25.433057+0000", 2023-09-23T21:08:49.875 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.011959981999999999 2023-09-23T21:08:49.875 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:08:49.875 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:08:49.875 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:08:49.875 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:25.434817+0000", 2023-09-23T21:08:49.876 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0017596090000000001 2023-09-23T21:08:49.876 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:08:49.876 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:08:49.876 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:08:49.876 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:25.434871+0000", 2023-09-23T21:08:49.876 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.4228000000000003e-05 2023-09-23T21:08:49.876 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:08:49.876 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:08:49.876 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:08:49.876 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:25.434906+0000", 2023-09-23T21:08:49.876 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.5113000000000001e-05 2023-09-23T21:08:49.876 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:08:49.877 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:08:49.877 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:08:49.877 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:08:49.877 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:08:49.877 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.35856.0:52943 9.3 9:cce7983c:::benchmark_data_smithi026_342732_object52942:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e354)", 2023-09-23T21:08:49.877 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:06:24.551421+0000", 2023-09-23T21:08:49.877 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 145.21883678899999, 2023-09-23T21:08:49.877 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.88400061399999996, 2023-09-23T21:08:49.877 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:08:49.877 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:08:49.877 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:08:49.877 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.35856", 2023-09-23T21:08:49.878 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/4180396258", 2023-09-23T21:08:49.878 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 52943 2023-09-23T21:08:49.878 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:08:49.878 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:08:49.878 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:08:49.878 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:08:49.878 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:24.551421+0000", 2023-09-23T21:08:49.878 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:08:49.878 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:08:49.878 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:08:49.878 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:08:49.878 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:24.551421+0000", 2023-09-23T21:08:49.879 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:08:49.879 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:08:49.879 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:08:49.879 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:08:49.879 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:24.551419+0000", 2023-09-23T21:08:49.879 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T21:08:49.879 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:08:49.879 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:08:49.879 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:08:49.879 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:24.551465+0000", 2023-09-23T21:08:49.879 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.5859000000000003e-05 2023-09-23T21:08:49.879 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:08:49.880 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:08:49.880 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:08:49.880 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:24.551478+0000", 2023-09-23T21:08:49.880 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.2401999999999999e-05 2023-09-23T21:08:49.880 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:08:49.880 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:08:49.880 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:08:49.880 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:24.551482+0000", 2023-09-23T21:08:49.880 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.8180000000000003e-06 2023-09-23T21:08:49.880 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:08:49.880 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:08:49.880 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:08:49.880 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:24.551531+0000", 2023-09-23T21:08:49.881 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.8235999999999997e-05 2023-09-23T21:08:49.881 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:08:49.881 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:08:49.881 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:08:49.881 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:24.551539+0000", 2023-09-23T21:08:49.881 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.5790000000000004e-06 2023-09-23T21:08:49.881 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:08:49.881 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:08:49.881 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:08:49.881 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:25.417837+0000", 2023-09-23T21:08:49.881 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.86629784499999996 2023-09-23T21:08:49.881 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:08:49.882 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:08:49.882 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:08:49.882 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:25.417926+0000", 2023-09-23T21:08:49.882 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.8490000000000001e-05 2023-09-23T21:08:49.882 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:08:49.882 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:08:49.882 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 1,3", 2023-09-23T21:08:49.882 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:25.418026+0000", 2023-09-23T21:08:49.882 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000100294 2023-09-23T21:08:49.882 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:08:49.882 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:08:49.882 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:08:49.882 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:25.421105+0000", 2023-09-23T21:08:49.883 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.003079072 2023-09-23T21:08:49.883 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:08:49.883 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:08:49.883 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:08:49.883 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:25.434959+0000", 2023-09-23T21:08:49.883 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.013854083 2023-09-23T21:08:49.883 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:08:49.883 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:08:49.883 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:08:49.883 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:25.435329+0000", 2023-09-23T21:08:49.883 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00036980599999999998 2023-09-23T21:08:49.883 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:08:49.883 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:08:49.884 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:08:49.884 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:25.435383+0000", 2023-09-23T21:08:49.884 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.3835000000000002e-05 2023-09-23T21:08:49.884 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:08:49.884 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:08:49.884 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:08:49.884 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:25.435421+0000", 2023-09-23T21:08:49.884 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.8525000000000001e-05 2023-09-23T21:08:49.884 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:08:49.884 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:08:49.884 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:08:49.884 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:08:49.885 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:08:49.885 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.35856.0:52979 9.3 9:cb868066:::benchmark_data_smithi026_342732_object52978:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e354)", 2023-09-23T21:08:49.885 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:06:24.605028+0000", 2023-09-23T21:08:49.885 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 145.16522911499999, 2023-09-23T21:08:49.885 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.83054075699999996, 2023-09-23T21:08:49.885 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:08:49.885 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:08:49.885 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:08:49.885 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.35856", 2023-09-23T21:08:49.885 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/4180396258", 2023-09-23T21:08:49.885 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 52979 2023-09-23T21:08:49.885 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:08:49.886 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:08:49.886 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:08:49.886 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:08:49.886 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:24.605028+0000", 2023-09-23T21:08:49.886 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:08:49.886 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:08:49.886 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:08:49.886 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:08:49.886 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:24.605028+0000", 2023-09-23T21:08:49.886 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:08:49.886 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:08:49.886 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:08:49.887 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:08:49.887 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:24.605027+0000", 2023-09-23T21:08:49.887 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T21:08:49.887 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:08:49.887 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:08:49.887 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:08:49.887 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:24.605061+0000", 2023-09-23T21:08:49.887 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.4298000000000003e-05 2023-09-23T21:08:49.887 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:08:49.887 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:08:49.887 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:08:49.887 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:24.605073+0000", 2023-09-23T21:08:49.887 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.2225999999999999e-05 2023-09-23T21:08:49.888 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:08:49.888 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:08:49.888 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:08:49.888 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:24.605078+0000", 2023-09-23T21:08:49.888 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.7740000000000004e-06 2023-09-23T21:08:49.888 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:08:49.888 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:08:49.888 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:08:49.888 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:24.605126+0000", 2023-09-23T21:08:49.888 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.7639000000000002e-05 2023-09-23T21:08:49.888 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:08:49.888 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:08:49.888 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:08:49.888 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:24.605132+0000", 2023-09-23T21:08:49.889 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.7270000000000001e-06 2023-09-23T21:08:49.889 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:08:49.889 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:08:49.889 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:08:49.889 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:25.418852+0000", 2023-09-23T21:08:49.889 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.81371994000000003 2023-09-23T21:08:49.889 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:08:49.889 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:08:49.889 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:08:49.889 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:25.418923+0000", 2023-09-23T21:08:49.889 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.0716000000000005e-05 2023-09-23T21:08:49.889 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:08:49.890 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:08:49.890 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 1,3", 2023-09-23T21:08:49.890 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:25.419029+0000", 2023-09-23T21:08:49.890 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000105603 2023-09-23T21:08:49.890 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:08:49.890 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:08:49.890 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:08:49.890 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:25.421113+0000", 2023-09-23T21:08:49.890 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0020848220000000001 2023-09-23T21:08:49.890 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:08:49.890 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:08:49.890 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:08:49.890 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:25.435018+0000", 2023-09-23T21:08:49.891 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.013904158 2023-09-23T21:08:49.891 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:08:49.891 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:08:49.891 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:08:49.891 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:25.435475+0000", 2023-09-23T21:08:49.891 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00045726399999999998 2023-09-23T21:08:49.891 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:08:49.891 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:08:49.891 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:08:49.891 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:25.435527+0000", 2023-09-23T21:08:49.891 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.2583000000000001e-05 2023-09-23T21:08:49.891 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:08:49.892 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:08:49.892 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:08:49.892 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:25.435569+0000", 2023-09-23T21:08:49.892 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.1551999999999998e-05 2023-09-23T21:08:49.892 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:08:49.892 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:08:49.892 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:08:49.892 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:08:49.892 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:08:49.892 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.35856.0:53017 9.3 9:c443da72:::benchmark_data_smithi026_342732_object53016:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e354)", 2023-09-23T21:08:49.892 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:06:24.701314+0000", 2023-09-23T21:08:49.892 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 145.068943735, 2023-09-23T21:08:49.892 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.73438963099999999, 2023-09-23T21:08:49.893 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:08:49.893 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:08:49.893 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:08:49.893 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.35856", 2023-09-23T21:08:49.893 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/4180396258", 2023-09-23T21:08:49.893 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 53017 2023-09-23T21:08:49.893 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:08:49.893 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:08:49.893 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:08:49.893 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:08:49.893 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:24.701314+0000", 2023-09-23T21:08:49.893 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:08:49.893 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:08:49.893 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:08:49.894 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:08:49.894 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:24.701314+0000", 2023-09-23T21:08:49.894 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:08:49.894 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:08:49.894 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:08:49.894 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:08:49.894 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:24.701307+0000", 2023-09-23T21:08:49.894 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999933 2023-09-23T21:08:49.894 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:08:49.894 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:08:49.894 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:08:49.895 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:24.701345+0000", 2023-09-23T21:08:49.895 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.8661999999999997e-05 2023-09-23T21:08:49.895 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:08:49.895 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:08:49.895 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:08:49.895 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:24.701357+0000", 2023-09-23T21:08:49.895 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.1907e-05 2023-09-23T21:08:49.895 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:08:49.895 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:08:49.895 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:08:49.895 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:24.701362+0000", 2023-09-23T21:08:49.896 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.5979999999999999e-06 2023-09-23T21:08:49.896 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:08:49.896 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:08:49.896 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:08:49.896 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:24.701410+0000", 2023-09-23T21:08:49.896 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.8534e-05 2023-09-23T21:08:49.896 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:08:49.896 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:08:49.896 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:08:49.896 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:24.701417+0000", 2023-09-23T21:08:49.896 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.5780000000000004e-06 2023-09-23T21:08:49.896 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:08:49.896 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:08:49.897 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:08:49.897 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:25.419853+0000", 2023-09-23T21:08:49.897 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.71843563300000002 2023-09-23T21:08:49.897 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:08:49.897 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:08:49.897 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:08:49.897 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:25.419985+0000", 2023-09-23T21:08:49.897 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00013251699999999999 2023-09-23T21:08:49.897 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:08:49.897 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:08:49.897 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 1,3", 2023-09-23T21:08:49.897 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:25.420143+0000", 2023-09-23T21:08:49.897 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00015774499999999999 2023-09-23T21:08:49.898 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:08:49.898 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:08:49.898 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:08:49.898 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:25.421530+0000", 2023-09-23T21:08:49.898 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.001387466 2023-09-23T21:08:49.898 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:08:49.898 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:08:49.898 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:08:49.898 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:25.435064+0000", 2023-09-23T21:08:49.898 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.013533880999999999 2023-09-23T21:08:49.898 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:08:49.898 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:08:49.898 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:08:49.899 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:25.435619+0000", 2023-09-23T21:08:49.899 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00055499900000000003 2023-09-23T21:08:49.899 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:08:49.899 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:08:49.899 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:08:49.899 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:25.435668+0000", 2023-09-23T21:08:49.899 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.8532000000000002e-05 2023-09-23T21:08:49.899 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:08:49.899 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:08:49.899 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:08:49.899 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:25.435703+0000", 2023-09-23T21:08:49.899 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.5471e-05 2023-09-23T21:08:49.900 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:08:49.900 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:08:49.900 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:08:49.900 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:08:49.900 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:08:49.900 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.35856.0:157806 9.3 9:c0d66d9b:::benchmark_data_smithi026_342732_object44530:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e378)", 2023-09-23T21:08:49.900 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:08:21.447748+0000", 2023-09-23T21:08:49.900 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 28.322509689, 2023-09-23T21:08:49.900 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.493376816, 2023-09-23T21:08:49.900 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:08:49.900 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:08:49.900 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:08:49.901 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.35856", 2023-09-23T21:08:49.901 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/4180396258", 2023-09-23T21:08:49.901 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 157806 2023-09-23T21:08:49.901 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:08:49.901 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:08:49.901 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:08:49.901 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:08:49.901 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:08:21.447748+0000", 2023-09-23T21:08:49.901 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:08:49.901 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:08:49.901 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:08:49.901 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:08:49.901 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:08:21.447748+0000", 2023-09-23T21:08:49.902 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:08:49.902 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:08:49.902 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:08:49.902 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:08:49.902 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:08:21.447746+0000", 2023-09-23T21:08:49.902 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T21:08:49.902 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:08:49.902 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:08:49.902 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:08:49.902 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:08:21.447753+0000", 2023-09-23T21:08:49.902 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.002e-06 2023-09-23T21:08:49.902 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:08:49.903 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:08:49.903 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:08:49.903 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:08:21.447766+0000", 2023-09-23T21:08:49.903 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.3198e-05 2023-09-23T21:08:49.903 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:08:49.903 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:08:49.903 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:08:49.903 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:08:21.447773+0000", 2023-09-23T21:08:49.903 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.8859999999999998e-06 2023-09-23T21:08:49.903 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:08:49.903 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:08:49.903 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:08:49.903 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:08:21.447825+0000", 2023-09-23T21:08:49.904 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.1208999999999997e-05 2023-09-23T21:08:49.904 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:08:49.904 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:08:49.904 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:08:49.904 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:08:21.447935+0000", 2023-09-23T21:08:49.904 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000110672 2023-09-23T21:08:49.904 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:08:49.904 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:08:49.904 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 1,3", 2023-09-23T21:08:49.904 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:08:21.448041+0000", 2023-09-23T21:08:49.904 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000105346 2023-09-23T21:08:49.904 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:08:49.904 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:08:49.905 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:08:49.905 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:08:21.451135+0000", 2023-09-23T21:08:49.905 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0030945880000000001 2023-09-23T21:08:49.905 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:08:49.905 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:08:49.905 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:08:49.905 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:08:21.455544+0000", 2023-09-23T21:08:49.905 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.004408538 2023-09-23T21:08:49.905 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:08:49.905 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:08:49.905 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:08:49.905 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:08:21.941037+0000", 2023-09-23T21:08:49.906 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.48549373299999998 2023-09-23T21:08:49.906 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:08:49.906 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:08:49.906 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:08:49.906 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:08:21.941085+0000", 2023-09-23T21:08:49.906 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.7577000000000002e-05 2023-09-23T21:08:49.906 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:08:49.906 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:08:49.906 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:08:49.906 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:08:21.941125+0000", 2023-09-23T21:08:49.906 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.9517999999999997e-05 2023-09-23T21:08:49.906 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:08:49.906 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:08:49.907 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:08:49.907 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:08:49.907 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:08:49.907 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.35856.0:157815 9.3 9:c90227bf:::benchmark_data_smithi026_342732_object44539:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e378)", 2023-09-23T21:08:49.907 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:08:21.456381+0000", 2023-09-23T21:08:49.907 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 28.313876695000001, 2023-09-23T21:08:49.907 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.485658015, 2023-09-23T21:08:49.907 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:08:49.907 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:08:49.907 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:08:49.907 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.35856", 2023-09-23T21:08:49.907 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/4180396258", 2023-09-23T21:08:49.907 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 157815 2023-09-23T21:08:49.908 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:08:49.908 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:08:49.908 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:08:49.908 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:08:49.908 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:08:21.456381+0000", 2023-09-23T21:08:49.908 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:08:49.908 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:08:49.908 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:08:49.908 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:08:49.908 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:08:21.456381+0000", 2023-09-23T21:08:49.908 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:08:49.908 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:08:49.908 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:08:49.908 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:08:49.909 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:08:21.456380+0000", 2023-09-23T21:08:49.909 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.999999 2023-09-23T21:08:49.909 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:08:49.909 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:08:49.909 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:08:49.909 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:08:21.456385+0000", 2023-09-23T21:08:49.909 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.9649999999999999e-06 2023-09-23T21:08:49.909 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:08:49.909 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:08:49.909 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:08:49.909 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:08:21.456393+0000", 2023-09-23T21:08:49.909 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.1599999999999998e-06 2023-09-23T21:08:49.909 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:08:49.910 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:08:49.910 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:08:49.910 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:08:21.456397+0000", 2023-09-23T21:08:49.910 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.5929999999999999e-06 2023-09-23T21:08:49.910 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:08:49.910 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:08:49.910 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:08:49.910 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:08:21.456443+0000", 2023-09-23T21:08:49.910 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.6017999999999999e-05 2023-09-23T21:08:49.910 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:08:49.910 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:08:49.910 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:08:49.910 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:08:21.456564+0000", 2023-09-23T21:08:49.911 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000120698 2023-09-23T21:08:49.911 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:08:49.911 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:08:49.911 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 1,3", 2023-09-23T21:08:49.911 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:08:21.456686+0000", 2023-09-23T21:08:49.911 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000121512 2023-09-23T21:08:49.911 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:08:49.911 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:08:49.911 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:08:49.911 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:08:21.941134+0000", 2023-09-23T21:08:49.911 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.48444830999999999 2023-09-23T21:08:49.911 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:08:49.911 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:08:49.912 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:08:49.912 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:08:21.941164+0000", 2023-09-23T21:08:49.912 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.0134999999999999e-05 2023-09-23T21:08:49.912 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:08:49.912 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:08:49.912 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:08:49.912 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:08:21.941961+0000", 2023-09-23T21:08:49.912 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00079654300000000003 2023-09-23T21:08:49.912 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:08:49.912 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:08:49.912 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:08:49.912 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:08:21.942003+0000", 2023-09-23T21:08:49.913 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.2342000000000002e-05 2023-09-23T21:08:49.913 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:08:49.913 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:08:49.913 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:08:49.913 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:08:21.942039+0000", 2023-09-23T21:08:49.913 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.5754999999999999e-05 2023-09-23T21:08:49.913 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:08:49.913 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:08:49.913 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:08:49.913 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:08:49.913 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:08:49.913 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_repop(client.35856.0:157816 9.a e378/350 9:50dbf38a:::benchmark_data_smithi026_342732_object44540:head v 378'9712, mlcod=378'9710)", 2023-09-23T21:08:49.913 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:08:21.456818+0000", 2023-09-23T21:08:49.914 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 28.313439055, 2023-09-23T21:08:49.914 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.489135653, 2023-09-23T21:08:49.914 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:08:49.914 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:08:49.914 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:08:49.914 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:08:49.914 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:08:49.914 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:08:21.456818+0000", 2023-09-23T21:08:49.914 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:08:49.914 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:08:49.914 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:08:49.915 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:08:49.915 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:08:21.456818+0000", 2023-09-23T21:08:49.915 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:08:49.915 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:08:49.915 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:08:49.915 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:08:49.915 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:08:21.456818+0000", 2023-09-23T21:08:49.915 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999995 2023-09-23T21:08:49.915 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:08:49.915 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:08:49.915 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:08:49.915 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:08:21.456824+0000", 2023-09-23T21:08:49.915 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.1840000000000004e-06 2023-09-23T21:08:49.916 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:08:49.916 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:08:49.916 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:08:49.916 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:08:21.933898+0000", 2023-09-23T21:08:49.916 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.47707417600000002 2023-09-23T21:08:49.916 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:08:49.916 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:08:49.916 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:08:49.916 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:08:21.933908+0000", 2023-09-23T21:08:49.916 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.0304e-05 2023-09-23T21:08:49.916 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:08:49.916 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:08:49.917 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:08:49.917 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:08:21.934049+0000", 2023-09-23T21:08:49.917 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000140908 2023-09-23T21:08:49.917 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:08:49.917 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:08:49.917 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:08:49.917 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:08:21.934081+0000", 2023-09-23T21:08:49.917 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.2019e-05 2023-09-23T21:08:49.917 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:08:49.917 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:08:49.917 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:08:49.917 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:08:21.945920+0000", 2023-09-23T21:08:49.918 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.011838384 2023-09-23T21:08:49.918 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:08:49.918 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:08:49.918 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:08:49.918 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:08:21.945954+0000", 2023-09-23T21:08:49.918 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.4320999999999999e-05 2023-09-23T21:08:49.918 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:08:49.918 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:08:49.918 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:08:49.918 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:08:49.918 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:08:49.918 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_repop(client.35856.0:157819 9.5 e378/346 9:a0500f9c:::benchmark_data_smithi026_342732_object44543:head v 378'9807, mlcod=378'9805)", 2023-09-23T21:08:49.918 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:08:21.460865+0000", 2023-09-23T21:08:49.919 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 28.309391996999999, 2023-09-23T21:08:49.919 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.48548613699999998, 2023-09-23T21:08:49.919 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:08:49.919 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:08:49.919 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:08:49.919 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:08:49.919 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:08:49.919 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:08:21.460865+0000", 2023-09-23T21:08:49.919 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:08:49.919 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:08:49.919 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:08:49.919 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:08:49.919 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:08:21.460865+0000", 2023-09-23T21:08:49.920 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:08:49.920 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:08:49.920 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:08:49.920 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:08:49.920 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:08:21.460865+0000", 2023-09-23T21:08:49.920 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.999999 2023-09-23T21:08:49.920 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:08:49.920 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:08:49.920 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:08:49.920 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:08:21.460871+0000", 2023-09-23T21:08:49.920 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.3350000000000003e-06 2023-09-23T21:08:49.920 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:08:49.920 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:08:49.921 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:08:49.921 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:08:21.933883+0000", 2023-09-23T21:08:49.921 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.47301222399999998 2023-09-23T21:08:49.921 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:08:49.921 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:08:49.921 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:08:49.921 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:08:21.933907+0000", 2023-09-23T21:08:49.921 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.3872000000000001e-05 2023-09-23T21:08:49.921 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:08:49.921 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:08:49.921 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:08:49.921 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:08:21.934265+0000", 2023-09-23T21:08:49.921 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00035773199999999998 2023-09-23T21:08:49.922 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:08:49.922 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:08:49.922 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:08:49.922 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:08:21.934283+0000", 2023-09-23T21:08:49.922 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.8647e-05 2023-09-23T21:08:49.922 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:08:49.922 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:08:49.922 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:08:49.922 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:08:21.946219+0000", 2023-09-23T21:08:49.922 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.011935862 2023-09-23T21:08:49.922 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:08:49.922 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:08:49.923 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:08:49.923 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:08:21.946352+0000", 2023-09-23T21:08:49.923 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000132273 2023-09-23T21:08:49.923 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:08:49.923 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:08:49.923 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:08:49.923 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:08:49.923 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:08:49.923 INFO:teuthology.orchestra.run.smithi070.stdout:} 2023-09-23T21:08:49.924 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:08:49 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T21:08:49.924 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:08:49 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T21:08:49.924 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:08:49 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:08:49.924 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 21:08:49 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[164029]: ::ffff:172.21.15.70 - - [23/Sep/2023:21:08:49] "GET /metrics HTTP/1.1" 200 36300 "" "Prometheus/2.43.0" 2023-09-23T21:08:49.925 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:08:49 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T21:08:49.925 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:08:49 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T21:08:49.925 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:08:49 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:08:49.925 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:08:49.926 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:08:49.926 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:08:49.926 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:08:50.027 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:08:50.118 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:08:50.247 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_ops_in_flight 2023-09-23T21:08:50.343 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:08:50.352 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:08:50.352 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:08:50.453 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:08:50.503 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:08:50.557 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:08:50.561 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:08:50.562 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:08:50.663 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:08:50.741 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:08:50.871 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:08:50 smithi070 ceph-mon[168282]: pgmap v1016: 109 pgs: 3 active+remapped+backfilling, 2 active+remapped+backfill_wait, 1 peering, 8 remapped+peering, 95 active+clean; 3.1 GiB data, 10 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 467 op/s; 14781/153978 objects misplaced (9.599%); 16 MiB/s, 263 objects/s recovering 2023-09-23T21:08:50.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:08:50 smithi026 ceph-mon[179688]: pgmap v1016: 109 pgs: 3 active+remapped+backfilling, 2 active+remapped+backfill_wait, 1 peering, 8 remapped+peering, 95 active+clean; 3.1 GiB data, 10 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 467 op/s; 14781/153978 objects misplaced (9.599%); 16 MiB/s, 263 objects/s recovering 2023-09-23T21:08:50.883 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:08:50 smithi026 ceph-mon[175151]: pgmap v1016: 109 pgs: 3 active+remapped+backfilling, 2 active+remapped+backfill_wait, 1 peering, 8 remapped+peering, 95 active+clean; 3.1 GiB data, 10 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 467 op/s; 14781/153978 objects misplaced (9.599%); 16 MiB/s, 263 objects/s recovering 2023-09-23T21:08:50.973 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:08:50.990 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:08:50.990 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:08:51.007 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T21:08:51.091 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:08:51.211 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:08:51.313 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:08:51.321 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:08:51.321 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:08:51.422 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:08:51.463 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:08:51.533 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:08:51.536 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:08:51.537 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:08:51.637 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:08:51.681 INFO:teuthology.orchestra.run.smithi026.stderr:nodeep-scrub is set 2023-09-23T21:08:51.703 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:08:51.800 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T21:08:51.810 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:08:51.810 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:08:51.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:08:51 smithi070 ceph-mon[168282]: pgmap v1017: 109 pgs: 4 active+remapped+backfilling, 5 active+remapped+backfill_wait, 1 peering, 99 active+clean; 3.1 GiB data, 10 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 321 op/s; 26572/151092 objects misplaced (17.587%); 19 MiB/s, 309 objects/s recovering 2023-09-23T21:08:51.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:08:51 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/4120860958' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:08:51.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:08:51 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:08:51.886 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:08:51 smithi026 ceph-mon[175151]: pgmap v1017: 109 pgs: 4 active+remapped+backfilling, 5 active+remapped+backfill_wait, 1 peering, 99 active+clean; 3.1 GiB data, 10 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 321 op/s; 26572/151092 objects misplaced (17.587%); 19 MiB/s, 309 objects/s recovering 2023-09-23T21:08:51.886 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:08:51 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/4120860958' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:08:51.886 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:08:51 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:08:51.887 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:08:51 smithi026 ceph-mon[179688]: pgmap v1017: 109 pgs: 4 active+remapped+backfilling, 5 active+remapped+backfill_wait, 1 peering, 99 active+clean; 3.1 GiB data, 10 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 321 op/s; 26572/151092 objects misplaced (17.587%); 19 MiB/s, 309 objects/s recovering 2023-09-23T21:08:51.887 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:08:51 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/4120860958' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:08:51.887 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:08:51 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:08:51.911 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:08:52.020 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:08:52.096 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0: {} 2023-09-23T21:08:52.107 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:08:52.111 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:08:52.111 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:08:52.156 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1: {} 2023-09-23T21:08:52.203 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2: {} 2023-09-23T21:08:52.212 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:08:52.265 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3: {} 2023-09-23T21:08:52.270 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:08:52.311 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4: {} 2023-09-23T21:08:52.319 INFO:teuthology.orchestra.run.smithi070.stdout:{ 2023-09-23T21:08:52.319 INFO:teuthology.orchestra.run.smithi070.stdout: "ops": [ 2023-09-23T21:08:52.319 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:08:52.319 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_repop(client.35856.0:178932 9.1 e387/383 9:8b5d2066:::benchmark_data_smithi026_342732_object65656:head v 387'11344, mlcod=387'11342)", 2023-09-23T21:08:52.320 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:08:52.319056+0000", 2023-09-23T21:08:52.320 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 0.00028364599999999997, 2023-09-23T21:08:52.320 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00031325400000000001, 2023-09-23T21:08:52.320 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:08:52.320 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "started", 2023-09-23T21:08:52.320 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:08:52.320 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:08:52.320 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:08:52.320 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:08:52.319056+0000", 2023-09-23T21:08:52.320 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:08:52.320 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:08:52.320 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:08:52.320 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:08:52.320 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:08:52.319056+0000", 2023-09-23T21:08:52.321 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:08:52.321 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:08:52.321 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:08:52.321 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:08:52.321 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:08:52.319055+0000", 2023-09-23T21:08:52.321 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T21:08:52.321 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:08:52.321 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:08:52.321 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:08:52.321 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:08:52.319068+0000", 2023-09-23T21:08:52.321 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.3657e-05 2023-09-23T21:08:52.321 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:08:52.321 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:08:52.322 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:08:52.322 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:08:52.319091+0000", 2023-09-23T21:08:52.322 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.2353000000000001e-05 2023-09-23T21:08:52.322 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:08:52.322 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:08:52.322 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:08:52.322 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:08:52.319099+0000", 2023-09-23T21:08:52.322 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.0560000000000005e-06 2023-09-23T21:08:52.322 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:08:52.322 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:08:52.322 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:08:52.322 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:08:52.319145+0000", 2023-09-23T21:08:52.322 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.6616000000000002e-05 2023-09-23T21:08:52.322 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:08:52.323 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:08:52.323 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:08:52.323 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:08:52.319175+0000", 2023-09-23T21:08:52.323 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.9870999999999999e-05 2023-09-23T21:08:52.323 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:08:52.323 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:08:52.323 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:08:52.323 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:08:52.323 INFO:teuthology.orchestra.run.smithi070.stdout: ], 2023-09-23T21:08:52.323 INFO:teuthology.orchestra.run.smithi070.stdout: "num_ops": 1 2023-09-23T21:08:52.323 INFO:teuthology.orchestra.run.smithi070.stdout:} 2023-09-23T21:08:52.338 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:08:52.343 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:08:52.343 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:08:52.362 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T21:08:52.403 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6: {} 2023-09-23T21:08:52.444 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:08:52.455 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T21:08:52.482 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:08:52.494 INFO:teuthology.orchestra.run.smithi026.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T21:08:52.494 INFO:teuthology.orchestra.run.smithi026.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T21:08:52.494 INFO:teuthology.orchestra.run.smithi026.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T21:08:52.494 INFO:teuthology.orchestra.run.smithi026.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T21:08:52.495 INFO:teuthology.orchestra.run.smithi026.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T21:08:52.495 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T21:08:52.495 INFO:teuthology.orchestra.run.smithi026.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T21:08:52.495 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T21:08:52.691 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:08:52.698 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:08:52.699 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:08:52.711 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_blocked_ops 2023-09-23T21:08:52.800 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:08:52.868 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:08:52.883 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:08:52 smithi026 ceph-mon[175151]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:08:52.884 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:08:52 smithi026 ceph-mon[175151]: failed to encode map e387 with expected crc 2023-09-23T21:08:52.884 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:08:52 smithi026 ceph-mon[175151]: failed to encode map e387 with expected crc 2023-09-23T21:08:52.884 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:08:52 smithi026 ceph-mon[175151]: failed to encode map e387 with expected crc 2023-09-23T21:08:52.884 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:08:52 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T21:08:52.884 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:08:52 smithi026 ceph-mon[175151]: osdmap e387: 8 total, 8 up, 7 in 2023-09-23T21:08:52.884 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:08:52 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/1768996164' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:08:52.884 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:08:52 smithi026 ceph-mon[179688]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:08:52.884 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:08:52 smithi026 ceph-mon[179688]: failed to encode map e387 with expected crc 2023-09-23T21:08:52.884 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:08:52 smithi026 ceph-mon[179688]: failed to encode map e387 with expected crc 2023-09-23T21:08:52.884 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:08:52 smithi026 ceph-mon[179688]: failed to encode map e387 with expected crc 2023-09-23T21:08:52.884 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:08:52 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T21:08:52.885 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:08:52 smithi026 ceph-mon[179688]: osdmap e387: 8 total, 8 up, 7 in 2023-09-23T21:08:52.885 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:08:52 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/1768996164' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:08:52.924 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:08:52.927 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:08:52.928 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:08:52.928 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:08:52 smithi070 ceph-mon[168282]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:08:52.928 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:08:52 smithi070 ceph-mon[168282]: failed to encode map e387 with expected crc 2023-09-23T21:08:52.928 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:08:52 smithi070 ceph-mon[168282]: failed to encode map e387 with expected crc 2023-09-23T21:08:52.928 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:08:52 smithi070 ceph-mon[168282]: failed to encode map e387 with expected crc 2023-09-23T21:08:52.928 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:08:52 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T21:08:52.928 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:08:52 smithi070 ceph-mon[168282]: osdmap e387: 8 total, 8 up, 7 in 2023-09-23T21:08:52.928 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:08:52 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/1768996164' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:08:53.028 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:08:53.129 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:08:53.268 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T21:08:53.276 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:08:53.277 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:08:53.377 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:08:53.444 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:08:53.507 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:08:53.511 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:08:53.511 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:08:53.556 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T21:08:53.612 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:08:53.648 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:08:53 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/3173399329' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 0.628229787685864}]: dispatch 2023-09-23T21:08:53.648 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:08:53 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 0.628229787685864}]: dispatch 2023-09-23T21:08:53.648 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:08:53 smithi026 ceph-mon[175151]: pgmap v1019: 109 pgs: 4 active+remapped+backfilling, 5 active+remapped+backfill_wait, 100 active+clean; 3.0 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 456 op/s; 24781/146955 objects misplaced (16.863%); 25 MiB/s, 403 objects/s recovering 2023-09-23T21:08:53.676 INFO:teuthology.orchestra.run.smithi026.stderr:reweighted osd.6 to 0.62823 (a0d3) 2023-09-23T21:08:53.678 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:08:53.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:08:53 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/3173399329' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 0.628229787685864}]: dispatch 2023-09-23T21:08:53.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:08:53 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 0.628229787685864}]: dispatch 2023-09-23T21:08:53.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:08:53 smithi070 ceph-mon[168282]: pgmap v1019: 109 pgs: 4 active+remapped+backfilling, 5 active+remapped+backfill_wait, 100 active+clean; 3.0 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 456 op/s; 24781/146955 objects misplaced (16.863%); 25 MiB/s, 403 objects/s recovering 2023-09-23T21:08:54.069 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:08:53 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/3173399329' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 0.628229787685864}]: dispatch 2023-09-23T21:08:54.069 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:08:53 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 0.628229787685864}]: dispatch 2023-09-23T21:08:54.069 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:08:53 smithi026 ceph-mon[179688]: pgmap v1019: 109 pgs: 4 active+remapped+backfilling, 5 active+remapped+backfill_wait, 100 active+clean; 3.0 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 456 op/s; 24781/146955 objects misplaced (16.863%); 25 MiB/s, 403 objects/s recovering 2023-09-23T21:08:54.761 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:08:54.767 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:08:54.768 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:08:54.868 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:08:54.914 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:08:54 smithi070 ceph-mon[168282]: failed to encode map e388 with expected crc 2023-09-23T21:08:54.915 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:08:54 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 0.628229787685864}]': finished 2023-09-23T21:08:54.915 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:08:54 smithi070 ceph-mon[168282]: osdmap e388: 8 total, 8 up, 7 in 2023-09-23T21:08:54.915 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:08:54 smithi070 ceph-mon[168282]: failed to encode map e388 with expected crc 2023-09-23T21:08:54.916 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:08:55.135 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:08:54 smithi026 ceph-mon[179688]: failed to encode map e388 with expected crc 2023-09-23T21:08:55.135 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:08:54 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 0.628229787685864}]': finished 2023-09-23T21:08:55.135 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:08:54 smithi026 ceph-mon[179688]: osdmap e388: 8 total, 8 up, 7 in 2023-09-23T21:08:55.135 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:08:54 smithi026 ceph-mon[179688]: failed to encode map e388 with expected crc 2023-09-23T21:08:55.136 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:08:54 smithi026 ceph-mon[175151]: failed to encode map e388 with expected crc 2023-09-23T21:08:55.137 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:08:54 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 0.628229787685864}]': finished 2023-09-23T21:08:55.137 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:08:54 smithi026 ceph-mon[175151]: osdmap e388: 8 total, 8 up, 7 in 2023-09-23T21:08:55.137 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:08:54 smithi026 ceph-mon[175151]: failed to encode map e388 with expected crc 2023-09-23T21:08:55.389 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T21:08:55.585 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:08:55.591 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:08:55.591 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:08:55.692 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:08:55.741 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:08:55.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:08:55 smithi070 ceph-mon[168282]: failed to encode map e389 with expected crc 2023-09-23T21:08:55.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:08:55 smithi070 ceph-mon[168282]: osdmap e389: 8 total, 8 up, 7 in 2023-09-23T21:08:55.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:08:55 smithi070 ceph-mon[168282]: failed to encode map e389 with expected crc 2023-09-23T21:08:55.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:08:55 smithi070 ceph-mon[168282]: failed to encode map e389 with expected crc 2023-09-23T21:08:55.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:08:55 smithi070 ceph-mon[168282]: failed to encode map e389 with expected crc 2023-09-23T21:08:55.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:08:55 smithi070 ceph-mon[168282]: pgmap v1022: 109 pgs: 2 peering, 4 active+remapped+backfilling, 5 active+remapped+backfill_wait, 98 active+clean; 2.9 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 713 op/s; 22668/141129 objects misplaced (16.062%); 29 MiB/s, 459 objects/s recovering 2023-09-23T21:08:55.881 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:08:55 smithi026 ceph-mon[179688]: failed to encode map e389 with expected crc 2023-09-23T21:08:55.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:08:55 smithi026 ceph-mon[179688]: osdmap e389: 8 total, 8 up, 7 in 2023-09-23T21:08:55.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:08:55 smithi026 ceph-mon[179688]: failed to encode map e389 with expected crc 2023-09-23T21:08:55.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:08:55 smithi026 ceph-mon[179688]: failed to encode map e389 with expected crc 2023-09-23T21:08:55.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:08:55 smithi026 ceph-mon[179688]: failed to encode map e389 with expected crc 2023-09-23T21:08:55.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:08:55 smithi026 ceph-mon[179688]: pgmap v1022: 109 pgs: 2 peering, 4 active+remapped+backfilling, 5 active+remapped+backfill_wait, 98 active+clean; 2.9 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 713 op/s; 22668/141129 objects misplaced (16.062%); 29 MiB/s, 459 objects/s recovering 2023-09-23T21:08:55.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:08:55 smithi026 ceph-mon[175151]: failed to encode map e389 with expected crc 2023-09-23T21:08:55.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:08:55 smithi026 ceph-mon[175151]: osdmap e389: 8 total, 8 up, 7 in 2023-09-23T21:08:55.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:08:55 smithi026 ceph-mon[175151]: failed to encode map e389 with expected crc 2023-09-23T21:08:55.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:08:55 smithi026 ceph-mon[175151]: failed to encode map e389 with expected crc 2023-09-23T21:08:55.883 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:08:55 smithi026 ceph-mon[175151]: failed to encode map e389 with expected crc 2023-09-23T21:08:55.883 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:08:55 smithi026 ceph-mon[175151]: pgmap v1022: 109 pgs: 2 peering, 4 active+remapped+backfilling, 5 active+remapped+backfill_wait, 98 active+clean; 2.9 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 713 op/s; 22668/141129 objects misplaced (16.062%); 29 MiB/s, 459 objects/s recovering 2023-09-23T21:08:56.169 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:08:56.172 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:08:56.172 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:08:56.273 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:08:56.355 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:08:56.862 INFO:teuthology.orchestra.run.smithi070.stdout:ERROR: (22) Invalid argument 2023-09-23T21:08:56.862 INFO:teuthology.orchestra.run.smithi070.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T21:08:56.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:08:56 smithi070 ceph-mon[168282]: failed to encode map e390 with expected crc 2023-09-23T21:08:56.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:08:56 smithi070 ceph-mon[168282]: failed to encode map e390 with expected crc 2023-09-23T21:08:56.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:08:56 smithi070 ceph-mon[168282]: failed to encode map e390 with expected crc 2023-09-23T21:08:56.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:08:56 smithi070 ceph-mon[168282]: osdmap e390: 8 total, 8 up, 7 in 2023-09-23T21:08:57.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:08:56 smithi026 ceph-mon[175151]: failed to encode map e390 with expected crc 2023-09-23T21:08:57.134 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:08:56 smithi026 ceph-mon[175151]: failed to encode map e390 with expected crc 2023-09-23T21:08:57.134 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:08:56 smithi026 ceph-mon[175151]: failed to encode map e390 with expected crc 2023-09-23T21:08:57.134 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:08:56 smithi026 ceph-mon[175151]: osdmap e390: 8 total, 8 up, 7 in 2023-09-23T21:08:57.134 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:08:56 smithi026 ceph-mon[179688]: failed to encode map e390 with expected crc 2023-09-23T21:08:57.134 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:08:56 smithi026 ceph-mon[179688]: failed to encode map e390 with expected crc 2023-09-23T21:08:57.134 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:08:56 smithi026 ceph-mon[179688]: failed to encode map e390 with expected crc 2023-09-23T21:08:57.135 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:08:56 smithi026 ceph-mon[179688]: osdmap e390: 8 total, 8 up, 7 in 2023-09-23T21:08:57.314 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_historic_ops 2023-09-23T21:08:57.820 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:08:57 smithi026 ceph-mon[175151]: failed to encode map e391 with expected crc 2023-09-23T21:08:57.821 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:08:57 smithi026 ceph-mon[175151]: osdmap e391: 8 total, 8 up, 7 in 2023-09-23T21:08:57.821 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:08:57 smithi026 ceph-mon[175151]: failed to encode map e391 with expected crc 2023-09-23T21:08:57.821 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:08:57 smithi026 ceph-mon[175151]: failed to encode map e391 with expected crc 2023-09-23T21:08:57.821 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:08:57 smithi026 ceph-mon[175151]: failed to encode map e391 with expected crc 2023-09-23T21:08:57.821 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:08:57 smithi026 ceph-mon[175151]: pgmap v1025: 109 pgs: 2 peering, 4 active+remapped+backfilling, 5 active+remapped+backfill_wait, 98 active+clean; 2.9 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 541 op/s; 22668/140460 objects misplaced (16.138%); 18 MiB/s, 285 objects/s recovering 2023-09-23T21:08:57.821 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:08:57 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:08:57.821 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:08:57 smithi026 ceph-mon[179688]: failed to encode map e391 with expected crc 2023-09-23T21:08:57.821 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:08:57 smithi026 ceph-mon[179688]: osdmap e391: 8 total, 8 up, 7 in 2023-09-23T21:08:57.821 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:08:57 smithi026 ceph-mon[179688]: failed to encode map e391 with expected crc 2023-09-23T21:08:57.822 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:08:57 smithi026 ceph-mon[179688]: failed to encode map e391 with expected crc 2023-09-23T21:08:57.822 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:08:57 smithi026 ceph-mon[179688]: failed to encode map e391 with expected crc 2023-09-23T21:08:57.822 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:08:57 smithi026 ceph-mon[179688]: pgmap v1025: 109 pgs: 2 peering, 4 active+remapped+backfilling, 5 active+remapped+backfill_wait, 98 active+clean; 2.9 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 541 op/s; 22668/140460 objects misplaced (16.138%); 18 MiB/s, 285 objects/s recovering 2023-09-23T21:08:57.822 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:08:57 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:08:57.877 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:08:57.881 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:08:57.882 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:08:57.982 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:08:58.001 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T21:08:58.042 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:08:57 smithi070 ceph-mon[168282]: failed to encode map e391 with expected crc 2023-09-23T21:08:58.042 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:08:57 smithi070 ceph-mon[168282]: osdmap e391: 8 total, 8 up, 7 in 2023-09-23T21:08:58.042 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:08:57 smithi070 ceph-mon[168282]: failed to encode map e391 with expected crc 2023-09-23T21:08:58.042 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:08:57 smithi070 ceph-mon[168282]: failed to encode map e391 with expected crc 2023-09-23T21:08:58.042 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:08:57 smithi070 ceph-mon[168282]: failed to encode map e391 with expected crc 2023-09-23T21:08:58.043 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:08:57 smithi070 ceph-mon[168282]: pgmap v1025: 109 pgs: 2 peering, 4 active+remapped+backfilling, 5 active+remapped+backfill_wait, 98 active+clean; 2.9 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 541 op/s; 22668/140460 objects misplaced (16.138%); 18 MiB/s, 285 objects/s recovering 2023-09-23T21:08:58.043 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:08:57 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:08:58.050 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:08:58.108 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:08:58.161 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:08:58.166 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:08:58.167 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:08:58.268 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:08:58.358 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:08:59.122 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:08:58 smithi070 ceph-mon[168282]: failed to encode map e392 with expected crc 2023-09-23T21:08:59.122 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:08:58 smithi070 ceph-mon[168282]: failed to encode map e392 with expected crc 2023-09-23T21:08:59.122 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:08:58 smithi070 ceph-mon[168282]: failed to encode map e392 with expected crc 2023-09-23T21:08:59.122 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:08:58 smithi070 ceph-mon[168282]: osdmap e392: 8 total, 8 up, 7 in 2023-09-23T21:08:59.131 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:08:58 smithi026 ceph-mon[179688]: failed to encode map e392 with expected crc 2023-09-23T21:08:59.131 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:08:58 smithi026 ceph-mon[179688]: failed to encode map e392 with expected crc 2023-09-23T21:08:59.131 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:08:58 smithi026 ceph-mon[179688]: failed to encode map e392 with expected crc 2023-09-23T21:08:59.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:08:58 smithi026 ceph-mon[179688]: osdmap e392: 8 total, 8 up, 7 in 2023-09-23T21:08:59.134 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:08:58 smithi026 ceph-mon[175151]: failed to encode map e392 with expected crc 2023-09-23T21:08:59.134 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:08:58 smithi026 ceph-mon[175151]: failed to encode map e392 with expected crc 2023-09-23T21:08:59.134 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:08:58 smithi026 ceph-mon[175151]: failed to encode map e392 with expected crc 2023-09-23T21:08:59.134 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:08:58 smithi026 ceph-mon[175151]: osdmap e392: 8 total, 8 up, 7 in 2023-09-23T21:08:59.186 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:08:59.197 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:08:59.198 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:08:59.299 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:08:59.392 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:08:59.519 INFO:teuthology.orchestra.run.smithi070.stdout:ERROR: (22) Invalid argument 2023-09-23T21:08:59.520 INFO:teuthology.orchestra.run.smithi070.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T21:08:59.561 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T21:08:59.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:08:59 smithi070 ceph-mon[168282]: pgmap v1027: 109 pgs: 4 peering, 4 active+remapped+backfilling, 5 active+remapped+backfill_wait, 96 active+clean; 2.7 GiB data, 9.8 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 567 op/s; 19931/133884 objects misplaced (14.887%); 27 MiB/s, 425 objects/s recovering 2023-09-23T21:08:59.882 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 21:08:59 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[164029]: ::ffff:172.21.15.70 - - [23/Sep/2023:21:08:59] "GET /metrics HTTP/1.1" 200 36300 "" "Prometheus/2.43.0" 2023-09-23T21:08:59.883 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:08:59 smithi026 ceph-mon[179688]: pgmap v1027: 109 pgs: 4 peering, 4 active+remapped+backfilling, 5 active+remapped+backfill_wait, 96 active+clean; 2.7 GiB data, 9.8 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 567 op/s; 19931/133884 objects misplaced (14.887%); 27 MiB/s, 425 objects/s recovering 2023-09-23T21:08:59.883 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:08:59 smithi026 ceph-mon[175151]: pgmap v1027: 109 pgs: 4 peering, 4 active+remapped+backfilling, 5 active+remapped+backfill_wait, 96 active+clean; 2.7 GiB data, 9.8 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 567 op/s; 19931/133884 objects misplaced (14.887%); 27 MiB/s, 425 objects/s recovering 2023-09-23T21:08:59.968 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_ops_in_flight 2023-09-23T21:09:00.085 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:09:00.092 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:09:00.092 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:09:00.193 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:09:00.234 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:09:00.286 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:09:00.292 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:09:00.292 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:09:00.393 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:09:00.446 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:09:00.500 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:09:00.503 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:09:00.504 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:09:00.604 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:09:00.649 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:09:00.711 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:09:00.714 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:09:00.714 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:09:00.815 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:09:00.856 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:09:00.878 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:09:00.909 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:09:00.916 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:09:00.916 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:09:01.017 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:09:01.065 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:09:01.120 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:09:01.123 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:09:01.124 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:09:01.152 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:09:01.224 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:09:01.328 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:09:01.385 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:09:01 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:09:01.482 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:09:01 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:09:01.482 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:09:01 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:09:01.684 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:09:01.692 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:09:01.692 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:09:01.793 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:09:01.844 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:09:01.888 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:09:01.893 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:09:01.894 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:09:01.994 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:09:02.082 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:09:02.170 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:09:02.181 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:09:02.182 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:09:02.283 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:09:02.332 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:09:02.383 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:09:02.386 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:09:02.387 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:09:02.428 INFO:teuthology.orchestra.run.smithi026.stderr:noscrub is unset 2023-09-23T21:09:02.487 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:09:02.602 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:09:02.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:09:02 smithi070 ceph-mon[168282]: pgmap v1028: 109 pgs: 2 peering, 4 active+remapped+backfilling, 5 active+remapped+backfill_wait, 98 active+clean; 2.6 GiB data, 9.6 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 655 op/s; 18517/129318 objects misplaced (14.319%); 25 MiB/s, 394 objects/s recovering 2023-09-23T21:09:02.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:09:02 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/1873380680' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T21:09:02.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:09:02 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T21:09:02.688 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:09:02 smithi026 ceph-mon[175151]: pgmap v1028: 109 pgs: 2 peering, 4 active+remapped+backfilling, 5 active+remapped+backfill_wait, 98 active+clean; 2.6 GiB data, 9.6 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 655 op/s; 18517/129318 objects misplaced (14.319%); 25 MiB/s, 394 objects/s recovering 2023-09-23T21:09:02.688 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:09:02 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/1873380680' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T21:09:02.688 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:09:02 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T21:09:02.691 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:09:02 smithi026 ceph-mon[179688]: pgmap v1028: 109 pgs: 2 peering, 4 active+remapped+backfilling, 5 active+remapped+backfill_wait, 98 active+clean; 2.6 GiB data, 9.6 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 655 op/s; 18517/129318 objects misplaced (14.319%); 25 MiB/s, 394 objects/s recovering 2023-09-23T21:09:02.691 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:09:02 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/1873380680' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T21:09:02.692 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:09:02 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T21:09:02.825 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 6, 4, 5] out_osds: [7] dead_osds: [] live_osds: [0, 2, 3, 4, 5, 7, 6, 1] 2023-09-23T21:09:02.825 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T21:09:02.825 INFO:tasks.thrashosds.thrasher:Reweighting osd 4 to 0.26752821003437066 2023-09-23T21:09:02.825 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph osd reweight 4 0.26752821003437066 2023-09-23T21:09:03.049 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:09:03.053 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:09:03.054 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:09:03.155 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:09:03.162 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0: {} 2023-09-23T21:09:03.185 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:09:03.217 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1: {} 2023-09-23T21:09:03.225 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:09:03.230 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:09:03.231 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:09:03.270 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2: {} 2023-09-23T21:09:03.332 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:09:03.345 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3: {} 2023-09-23T21:09:03.366 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:09:03.388 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4: {} 2023-09-23T21:09:03.412 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:09:03.417 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:09:03.417 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:09:03.428 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T21:09:03.466 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6: {} 2023-09-23T21:09:03.504 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T21:09:03.506 INFO:teuthology.orchestra.run.smithi026.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T21:09:03.506 INFO:teuthology.orchestra.run.smithi026.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T21:09:03.506 INFO:teuthology.orchestra.run.smithi026.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T21:09:03.507 INFO:teuthology.orchestra.run.smithi026.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T21:09:03.507 INFO:teuthology.orchestra.run.smithi026.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T21:09:03.507 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T21:09:03.507 INFO:teuthology.orchestra.run.smithi026.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T21:09:03.507 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T21:09:03.518 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:09:03.538 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:09:03 smithi026 ceph-mon[179688]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:09:03.538 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:09:03 smithi026 ceph-mon[179688]: failed to encode map e393 with expected crc 2023-09-23T21:09:03.538 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:09:03 smithi026 ceph-mon[179688]: failed to encode map e393 with expected crc 2023-09-23T21:09:03.538 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:09:03 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T21:09:03.538 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:09:03 smithi026 ceph-mon[179688]: osdmap e393: 8 total, 8 up, 7 in 2023-09-23T21:09:03.538 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:09:03 smithi026 ceph-mon[179688]: failed to encode map e393 with expected crc 2023-09-23T21:09:03.538 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:09:03 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/325708054' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:09:03.541 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:09:03 smithi026 ceph-mon[175151]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:09:03.541 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:09:03 smithi026 ceph-mon[175151]: failed to encode map e393 with expected crc 2023-09-23T21:09:03.541 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:09:03 smithi026 ceph-mon[175151]: failed to encode map e393 with expected crc 2023-09-23T21:09:03.541 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:09:03 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T21:09:03.541 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:09:03 smithi026 ceph-mon[175151]: osdmap e393: 8 total, 8 up, 7 in 2023-09-23T21:09:03.541 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:09:03 smithi026 ceph-mon[175151]: failed to encode map e393 with expected crc 2023-09-23T21:09:03.541 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:09:03 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/325708054' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:09:03.621 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:09:03 smithi070 ceph-mon[168282]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:09:03.621 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:09:03 smithi070 ceph-mon[168282]: failed to encode map e393 with expected crc 2023-09-23T21:09:03.621 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:09:03 smithi070 ceph-mon[168282]: failed to encode map e393 with expected crc 2023-09-23T21:09:03.621 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:09:03 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T21:09:03.621 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:09:03 smithi070 ceph-mon[168282]: osdmap e393: 8 total, 8 up, 7 in 2023-09-23T21:09:03.621 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:09:03 smithi070 ceph-mon[168282]: failed to encode map e393 with expected crc 2023-09-23T21:09:03.621 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:09:03 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/325708054' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:09:03.637 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:09:04.011 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:09:04.014 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:09:04.014 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:09:04.115 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:09:04.137 INFO:teuthology.orchestra.run.smithi026.stdout:{ 2023-09-23T21:09:04.137 INFO:teuthology.orchestra.run.smithi026.stdout: "ops": [ 2023-09-23T21:09:04.137 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:09:04.137 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35856.0:187831 9.0 9:00fcf999:::benchmark_data_smithi026_342732_object74555:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e393)", 2023-09-23T21:09:04.137 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T21:09:04.128298+0000", 2023-09-23T21:09:04.137 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 0.008731941, 2023-09-23T21:09:04.137 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0087578510000000005, 2023-09-23T21:09:04.137 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:09:04.137 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "waiting for sub ops", 2023-09-23T21:09:04.137 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T21:09:04.138 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35856", 2023-09-23T21:09:04.138 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/4180396258", 2023-09-23T21:09:04.138 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 187831 2023-09-23T21:09:04.138 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:09:04.138 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:09:04.138 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:09:04.138 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:09:04.138 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:04.128298+0000", 2023-09-23T21:09:04.138 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:09:04.138 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:09:04.138 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:09:04.138 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:09:04.139 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:04.128298+0000", 2023-09-23T21:09:04.139 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:09:04.139 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:09:04.139 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:09:04.139 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:09:04.139 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:04.128297+0000", 2023-09-23T21:09:04.139 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T21:09:04.139 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:09:04.139 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:09:04.139 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:09:04.139 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:04.128303+0000", 2023-09-23T21:09:04.139 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.1489999999999999e-06 2023-09-23T21:09:04.139 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:09:04.139 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:09:04.140 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:09:04.140 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:04.128316+0000", 2023-09-23T21:09:04.140 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.3008e-05 2023-09-23T21:09:04.140 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:09:04.140 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:09:04.140 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:09:04.140 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:04.128325+0000", 2023-09-23T21:09:04.140 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.5269999999999998e-06 2023-09-23T21:09:04.140 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:09:04.140 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:09:04.140 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:09:04.140 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:04.128378+0000", 2023-09-23T21:09:04.140 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.2939000000000003e-05 2023-09-23T21:09:04.140 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:09:04.140 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:09:04.141 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:09:04.141 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:04.128497+0000", 2023-09-23T21:09:04.141 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000119802 2023-09-23T21:09:04.141 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:09:04.141 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:09:04.141 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 2,6", 2023-09-23T21:09:04.141 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:04.128567+0000", 2023-09-23T21:09:04.141 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.9376999999999997e-05 2023-09-23T21:09:04.141 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:09:04.141 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:09:04.141 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T21:09:04.141 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:04.131237+0000", 2023-09-23T21:09:04.141 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0026703149999999999 2023-09-23T21:09:04.142 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:09:04.142 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:09:04.142 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:09:04.142 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:09:04.142 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:09:04.142 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35856.0:187833 9.0 9:01bf724c:::benchmark_data_smithi026_342732_object74557:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e393)", 2023-09-23T21:09:04.142 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T21:09:04.131946+0000", 2023-09-23T21:09:04.142 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 0.0050839120000000003, 2023-09-23T21:09:04.142 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0051644719999999998, 2023-09-23T21:09:04.142 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:09:04.142 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "waiting for sub ops", 2023-09-23T21:09:04.142 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T21:09:04.142 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35856", 2023-09-23T21:09:04.142 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/4180396258", 2023-09-23T21:09:04.143 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 187833 2023-09-23T21:09:04.143 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:09:04.143 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:09:04.143 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:09:04.143 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:09:04.143 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:04.131946+0000", 2023-09-23T21:09:04.143 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:09:04.143 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:09:04.143 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:09:04.143 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:09:04.143 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:04.131946+0000", 2023-09-23T21:09:04.143 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:09:04.143 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:09:04.143 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:09:04.144 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:09:04.144 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:04.131945+0000", 2023-09-23T21:09:04.144 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T21:09:04.144 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:09:04.144 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:09:04.144 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:09:04.144 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:04.131952+0000", 2023-09-23T21:09:04.144 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.1509999999999998e-06 2023-09-23T21:09:04.144 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:09:04.144 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:09:04.144 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:09:04.144 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:04.131968+0000", 2023-09-23T21:09:04.144 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.5192e-05 2023-09-23T21:09:04.145 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:09:04.145 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:09:04.145 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:09:04.145 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:04.131976+0000", 2023-09-23T21:09:04.145 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.1510000000000004e-06 2023-09-23T21:09:04.145 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:09:04.145 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:09:04.145 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:09:04.145 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:04.132024+0000", 2023-09-23T21:09:04.145 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.8687999999999999e-05 2023-09-23T21:09:04.145 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:09:04.145 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:09:04.145 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:09:04.145 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:04.132150+0000", 2023-09-23T21:09:04.146 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00012573799999999999 2023-09-23T21:09:04.146 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:09:04.146 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:09:04.146 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 2,6", 2023-09-23T21:09:04.146 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:04.132264+0000", 2023-09-23T21:09:04.146 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00011409 2023-09-23T21:09:04.146 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:09:04.146 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:09:04.146 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T21:09:04.146 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:04.135908+0000", 2023-09-23T21:09:04.146 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0036435119999999998 2023-09-23T21:09:04.146 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:09:04.146 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:09:04.147 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:09:04.147 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:09:04.147 INFO:teuthology.orchestra.run.smithi026.stdout: ], 2023-09-23T21:09:04.147 INFO:teuthology.orchestra.run.smithi026.stdout: "num_ops": 2 2023-09-23T21:09:04.147 INFO:teuthology.orchestra.run.smithi026.stdout:} 2023-09-23T21:09:04.167 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:09:04.196 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:09:04.227 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:09:04.231 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:09:04.231 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:09:04.332 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:09:04.381 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:09:04.425 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:09:04.428 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:09:04.429 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:09:04.429 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:09:04 smithi070 ceph-mon[168282]: pgmap v1030: 109 pgs: 4 active+remapped+backfilling, 5 active+remapped+backfill_wait, 100 active+clean; 2.5 GiB data, 9.4 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 874 op/s; 16749/124239 objects misplaced (13.481%); 35 MiB/s, 558 objects/s recovering 2023-09-23T21:09:04.530 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:09:04.615 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:09:04 smithi026 ceph-mon[175151]: pgmap v1030: 109 pgs: 4 active+remapped+backfilling, 5 active+remapped+backfill_wait, 100 active+clean; 2.5 GiB data, 9.4 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 874 op/s; 16749/124239 objects misplaced (13.481%); 35 MiB/s, 558 objects/s recovering 2023-09-23T21:09:04.616 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:09:04 smithi026 ceph-mon[179688]: pgmap v1030: 109 pgs: 4 active+remapped+backfilling, 5 active+remapped+backfill_wait, 100 active+clean; 2.5 GiB data, 9.4 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 874 op/s; 16749/124239 objects misplaced (13.481%); 35 MiB/s, 558 objects/s recovering 2023-09-23T21:09:04.638 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:09:04.689 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_blocked_ops 2023-09-23T21:09:04.744 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:09:04.755 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:09:04.756 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:09:04.856 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:09:04.946 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:09:05.033 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:09:05.043 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:09:05.043 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:09:05.144 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:09:05.201 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:09:05.344 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T21:09:05.496 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:09:05.510 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:09:05.511 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:09:05.611 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:09:05.639 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:09:05.677 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:09:05.681 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:09:05.681 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:09:05.782 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:09:05.788 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:09:05.824 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:09:06.013 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:09:06.020 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:09:06.020 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:09:06.121 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:09:06.172 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:09:06.221 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T21:09:06.230 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:09:06.231 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:09:06.236 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T21:09:06.332 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:09:06.358 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:09:06.394 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:09:06.397 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:09:06.397 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:09:06.400 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:09:06.498 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:09:06.526 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:09:06.567 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:09:06.570 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:09:06.571 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:09:06.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:09:06 smithi070 ceph-mon[168282]: pgmap v1031: 109 pgs: 4 active+remapped+backfilling, 5 active+remapped+backfill_wait, 100 active+clean; 2.4 GiB data, 9.2 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 925 op/s; 14754/118245 objects misplaced (12.477%); 36 MiB/s, 579 objects/s recovering 2023-09-23T21:09:06.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:09:06 smithi070 ceph-mon[168282]: osdmap e394: 8 total, 8 up, 7 in 2023-09-23T21:09:06.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:09:06 smithi070 ceph-mon[168282]: failed to encode map e394 with expected crc 2023-09-23T21:09:06.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:09:06 smithi070 ceph-mon[168282]: failed to encode map e394 with expected crc 2023-09-23T21:09:06.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:09:06 smithi070 ceph-mon[168282]: failed to encode map e394 with expected crc 2023-09-23T21:09:06.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:09:06 smithi070 ceph-mon[168282]: failed to encode map e394 with expected crc 2023-09-23T21:09:06.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:09:06 smithi070 ceph-mon[168282]: failed to encode map e394 with expected crc 2023-09-23T21:09:06.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:09:06 smithi070 ceph-mon[168282]: failed to encode map e394 with expected crc 2023-09-23T21:09:06.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:09:06 smithi070 ceph-mon[168282]: failed to encode map e394 with expected crc 2023-09-23T21:09:06.631 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:09:06 smithi026 ceph-mon[179688]: pgmap v1031: 109 pgs: 4 active+remapped+backfilling, 5 active+remapped+backfill_wait, 100 active+clean; 2.4 GiB data, 9.2 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 925 op/s; 14754/118245 objects misplaced (12.477%); 36 MiB/s, 579 objects/s recovering 2023-09-23T21:09:06.631 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:09:06 smithi026 ceph-mon[179688]: osdmap e394: 8 total, 8 up, 7 in 2023-09-23T21:09:06.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:09:06 smithi026 ceph-mon[179688]: failed to encode map e394 with expected crc 2023-09-23T21:09:06.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:09:06 smithi026 ceph-mon[179688]: failed to encode map e394 with expected crc 2023-09-23T21:09:06.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:09:06 smithi026 ceph-mon[179688]: failed to encode map e394 with expected crc 2023-09-23T21:09:06.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:09:06 smithi026 ceph-mon[179688]: failed to encode map e394 with expected crc 2023-09-23T21:09:06.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:09:06 smithi026 ceph-mon[179688]: failed to encode map e394 with expected crc 2023-09-23T21:09:06.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:09:06 smithi026 ceph-mon[179688]: failed to encode map e394 with expected crc 2023-09-23T21:09:06.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:09:06 smithi026 ceph-mon[179688]: failed to encode map e394 with expected crc 2023-09-23T21:09:06.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:09:06 smithi026 ceph-mon[175151]: pgmap v1031: 109 pgs: 4 active+remapped+backfilling, 5 active+remapped+backfill_wait, 100 active+clean; 2.4 GiB data, 9.2 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 925 op/s; 14754/118245 objects misplaced (12.477%); 36 MiB/s, 579 objects/s recovering 2023-09-23T21:09:06.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:09:06 smithi026 ceph-mon[175151]: osdmap e394: 8 total, 8 up, 7 in 2023-09-23T21:09:06.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:09:06 smithi026 ceph-mon[175151]: failed to encode map e394 with expected crc 2023-09-23T21:09:06.633 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:09:06 smithi026 ceph-mon[175151]: failed to encode map e394 with expected crc 2023-09-23T21:09:06.633 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:09:06 smithi026 ceph-mon[175151]: failed to encode map e394 with expected crc 2023-09-23T21:09:06.633 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:09:06 smithi026 ceph-mon[175151]: failed to encode map e394 with expected crc 2023-09-23T21:09:06.633 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:09:06 smithi026 ceph-mon[175151]: failed to encode map e394 with expected crc 2023-09-23T21:09:06.633 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:09:06 smithi026 ceph-mon[175151]: failed to encode map e394 with expected crc 2023-09-23T21:09:06.633 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:09:06 smithi026 ceph-mon[175151]: failed to encode map e394 with expected crc 2023-09-23T21:09:06.671 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:09:06.732 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:09:06.863 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T21:09:06.874 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:09:06.874 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:09:06.975 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:09:07.001 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:09:07.037 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:09:07.040 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:09:07.041 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:09:07.141 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:09:07.186 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:09:07.238 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:09:07.243 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:09:07.243 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:09:07.344 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:09:07.404 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:09:07.408 INFO:teuthology.orchestra.run.smithi026.stderr:reweighted osd.4 to 0.267528 (447c) 2023-09-23T21:09:07.538 INFO:teuthology.orchestra.run.smithi026.stdout:{ 2023-09-23T21:09:07.538 INFO:teuthology.orchestra.run.smithi026.stdout: "ops": [], 2023-09-23T21:09:07.539 INFO:teuthology.orchestra.run.smithi026.stdout: "complaint_time": 30, 2023-09-23T21:09:07.539 INFO:teuthology.orchestra.run.smithi026.stdout: "num_blocked_ops": 0 2023-09-23T21:09:07.539 INFO:teuthology.orchestra.run.smithi026.stdout:} 2023-09-23T21:09:07.564 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:09:07 smithi026 ceph-mon[179688]: failed to encode map e395 with expected crc 2023-09-23T21:09:07.564 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:09:07 smithi026 ceph-mon[179688]: failed to encode map e395 with expected crc 2023-09-23T21:09:07.564 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:09:07 smithi026 ceph-mon[179688]: osdmap e395: 8 total, 8 up, 7 in 2023-09-23T21:09:07.564 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:09:07 smithi026 ceph-mon[179688]: failed to encode map e395 with expected crc 2023-09-23T21:09:07.564 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:09:07 smithi026 ceph-mon[179688]: failed to encode map e395 with expected crc 2023-09-23T21:09:07.564 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:09:07 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/3829794140' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 0.26752821003437066}]: dispatch 2023-09-23T21:09:07.564 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:09:07 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 0.26752821003437066}]: dispatch 2023-09-23T21:09:07.565 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:09:07 smithi026 ceph-mon[175151]: failed to encode map e395 with expected crc 2023-09-23T21:09:07.565 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:09:07 smithi026 ceph-mon[175151]: failed to encode map e395 with expected crc 2023-09-23T21:09:07.565 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:09:07 smithi026 ceph-mon[175151]: osdmap e395: 8 total, 8 up, 7 in 2023-09-23T21:09:07.565 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:09:07 smithi026 ceph-mon[175151]: failed to encode map e395 with expected crc 2023-09-23T21:09:07.565 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:09:07 smithi026 ceph-mon[175151]: failed to encode map e395 with expected crc 2023-09-23T21:09:07.565 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:09:07 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/3829794140' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 0.26752821003437066}]: dispatch 2023-09-23T21:09:07.565 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:09:07 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 0.26752821003437066}]: dispatch 2023-09-23T21:09:07.606 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T21:09:07.609 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:09:07.610 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:09:07.710 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:09:07.749 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:09:07.846 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:09:07.850 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:09:07.850 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:09:07.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:09:07 smithi070 ceph-mon[168282]: failed to encode map e395 with expected crc 2023-09-23T21:09:07.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:09:07 smithi070 ceph-mon[168282]: failed to encode map e395 with expected crc 2023-09-23T21:09:07.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:09:07 smithi070 ceph-mon[168282]: osdmap e395: 8 total, 8 up, 7 in 2023-09-23T21:09:07.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:09:07 smithi070 ceph-mon[168282]: failed to encode map e395 with expected crc 2023-09-23T21:09:07.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:09:07 smithi070 ceph-mon[168282]: failed to encode map e395 with expected crc 2023-09-23T21:09:07.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:09:07 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/3829794140' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 0.26752821003437066}]: dispatch 2023-09-23T21:09:07.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:09:07 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 0.26752821003437066}]: dispatch 2023-09-23T21:09:07.951 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:09:07.977 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:09:08.010 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:09:08.012 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:09:08.013 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:09:08.113 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:09:08.151 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:09:08.294 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:09:08.299 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:09:08.299 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:09:08.300 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_historic_ops 2023-09-23T21:09:08.386 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:09:08.400 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:09:08.431 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:09:08.468 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:09:08.470 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:09:08.470 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:09:08.571 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:09:08.596 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:09:08.629 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:09:08.631 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:09:08.632 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:09:08.732 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:09:08.773 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:09:08 smithi026 ceph-mon[179688]: pgmap v1034: 109 pgs: 4 active+remapped+backfilling, 5 active+remapped+backfill_wait, 100 active+clean; 2.4 GiB data, 9.2 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 648 op/s; 14754/117633 objects misplaced (12.542%); 24 MiB/s, 378 objects/s recovering 2023-09-23T21:09:08.773 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:09:08 smithi026 ceph-mon[179688]: failed to encode map e396 with expected crc 2023-09-23T21:09:08.773 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:09:08 smithi026 ceph-mon[179688]: failed to encode map e396 with expected crc 2023-09-23T21:09:08.773 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:09:08 smithi026 ceph-mon[179688]: failed to encode map e396 with expected crc 2023-09-23T21:09:08.773 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:09:08 smithi026 ceph-mon[179688]: failed to encode map e396 with expected crc 2023-09-23T21:09:08.773 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:09:08 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 0.26752821003437066}]': finished 2023-09-23T21:09:08.773 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:09:08 smithi026 ceph-mon[179688]: osdmap e396: 8 total, 8 up, 7 in 2023-09-23T21:09:08.773 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:09:08 smithi026 ceph-mon[179688]: failed to encode map e396 with expected crc 2023-09-23T21:09:08.773 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:09:08 smithi026 ceph-mon[179688]: failed to encode map e396 with expected crc 2023-09-23T21:09:08.774 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:09:08 smithi026 ceph-mon[175151]: pgmap v1034: 109 pgs: 4 active+remapped+backfilling, 5 active+remapped+backfill_wait, 100 active+clean; 2.4 GiB data, 9.2 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 648 op/s; 14754/117633 objects misplaced (12.542%); 24 MiB/s, 378 objects/s recovering 2023-09-23T21:09:08.774 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:09:08 smithi026 ceph-mon[175151]: failed to encode map e396 with expected crc 2023-09-23T21:09:08.774 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:09:08 smithi026 ceph-mon[175151]: failed to encode map e396 with expected crc 2023-09-23T21:09:08.774 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:09:08 smithi026 ceph-mon[175151]: failed to encode map e396 with expected crc 2023-09-23T21:09:08.774 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:09:08 smithi026 ceph-mon[175151]: failed to encode map e396 with expected crc 2023-09-23T21:09:08.775 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:09:08 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 0.26752821003437066}]': finished 2023-09-23T21:09:08.775 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:09:08 smithi026 ceph-mon[175151]: osdmap e396: 8 total, 8 up, 7 in 2023-09-23T21:09:08.775 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:09:08 smithi026 ceph-mon[175151]: failed to encode map e396 with expected crc 2023-09-23T21:09:08.775 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:09:08 smithi026 ceph-mon[175151]: failed to encode map e396 with expected crc 2023-09-23T21:09:08.778 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:09:08.835 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:09:08.842 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:09:08.842 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:09:08.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:09:08 smithi070 ceph-mon[168282]: pgmap v1034: 109 pgs: 4 active+remapped+backfilling, 5 active+remapped+backfill_wait, 100 active+clean; 2.4 GiB data, 9.2 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 648 op/s; 14754/117633 objects misplaced (12.542%); 24 MiB/s, 378 objects/s recovering 2023-09-23T21:09:08.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:09:08 smithi070 ceph-mon[168282]: failed to encode map e396 with expected crc 2023-09-23T21:09:08.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:09:08 smithi070 ceph-mon[168282]: failed to encode map e396 with expected crc 2023-09-23T21:09:08.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:09:08 smithi070 ceph-mon[168282]: failed to encode map e396 with expected crc 2023-09-23T21:09:08.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:09:08 smithi070 ceph-mon[168282]: failed to encode map e396 with expected crc 2023-09-23T21:09:08.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:09:08 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 0.26752821003437066}]': finished 2023-09-23T21:09:08.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:09:08 smithi070 ceph-mon[168282]: osdmap e396: 8 total, 8 up, 7 in 2023-09-23T21:09:08.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:09:08 smithi070 ceph-mon[168282]: failed to encode map e396 with expected crc 2023-09-23T21:09:08.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:09:08 smithi070 ceph-mon[168282]: failed to encode map e396 with expected crc 2023-09-23T21:09:08.943 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:09:08.970 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:09:09.005 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:09:09.007 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:09:09.008 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:09:09.108 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:09:09.134 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:09:09.167 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:09:09.169 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:09:09.170 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:09:09.270 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:09:09.296 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:09:09.329 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:09:09.332 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:09:09.332 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:09:09.401 INFO:teuthology.orchestra.run.smithi026.stderr:nodeep-scrub is unset 2023-09-23T21:09:09.433 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:09:09.483 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:09:09.502 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:09:09.562 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:09:09.580 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:09:09.581 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:09:09.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:09:09 smithi070 ceph-mon[168282]: failed to encode map e397 with expected crc 2023-09-23T21:09:09.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:09:09 smithi070 ceph-mon[168282]: osdmap e397: 8 total, 8 up, 7 in 2023-09-23T21:09:09.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:09:09 smithi070 ceph-mon[168282]: failed to encode map e397 with expected crc 2023-09-23T21:09:09.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:09:09 smithi070 ceph-mon[168282]: 9.1 starting backfill to osd.2 from (0'0,0'0] MAX to 395'12013 2023-09-23T21:09:09.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:09:09 smithi070 ceph-mon[168282]: failed to encode map e397 with expected crc 2023-09-23T21:09:09.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:09:09 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/3979707290' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:09:09.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:09:09 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:09:09.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:09:09 smithi070 ceph-mon[168282]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T21:09:09.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:09:09 smithi070 ceph-mon[168282]: Cluster is now healthy 2023-09-23T21:09:09.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:09:09 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T21:09:09.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:09:09 smithi070 ceph-mon[168282]: osdmap e398: 8 total, 8 up, 7 in 2023-09-23T21:09:09.681 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:09:09.712 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:09:09.766 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:09:09.769 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:09:09.770 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:09:09.827 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:09:09 smithi026 ceph-mon[179688]: failed to encode map e397 with expected crc 2023-09-23T21:09:09.827 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:09:09 smithi026 ceph-mon[179688]: osdmap e397: 8 total, 8 up, 7 in 2023-09-23T21:09:09.827 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:09:09 smithi026 ceph-mon[179688]: failed to encode map e397 with expected crc 2023-09-23T21:09:09.827 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:09:09 smithi026 ceph-mon[179688]: 9.1 starting backfill to osd.2 from (0'0,0'0] MAX to 395'12013 2023-09-23T21:09:09.827 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:09:09 smithi026 ceph-mon[179688]: failed to encode map e397 with expected crc 2023-09-23T21:09:09.827 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:09:09 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/3979707290' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:09:09.827 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:09:09 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:09:09.827 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:09:09 smithi026 ceph-mon[179688]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T21:09:09.827 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:09:09 smithi026 ceph-mon[179688]: Cluster is now healthy 2023-09-23T21:09:09.828 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:09:09 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T21:09:09.828 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:09:09 smithi026 ceph-mon[179688]: osdmap e398: 8 total, 8 up, 7 in 2023-09-23T21:09:09.828 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 21:09:09 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[164029]: ::ffff:172.21.15.70 - - [23/Sep/2023:21:09:09] "GET /metrics HTTP/1.1" 200 36329 "" "Prometheus/2.43.0" 2023-09-23T21:09:09.830 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:09:09 smithi026 ceph-mon[175151]: failed to encode map e397 with expected crc 2023-09-23T21:09:09.830 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:09:09 smithi026 ceph-mon[175151]: osdmap e397: 8 total, 8 up, 7 in 2023-09-23T21:09:09.830 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:09:09 smithi026 ceph-mon[175151]: failed to encode map e397 with expected crc 2023-09-23T21:09:09.830 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:09:09 smithi026 ceph-mon[175151]: 9.1 starting backfill to osd.2 from (0'0,0'0] MAX to 395'12013 2023-09-23T21:09:09.830 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:09:09 smithi026 ceph-mon[175151]: failed to encode map e397 with expected crc 2023-09-23T21:09:09.830 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:09:09 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/3979707290' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:09:09.830 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:09:09 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:09:09.830 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:09:09 smithi026 ceph-mon[175151]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T21:09:09.830 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:09:09 smithi026 ceph-mon[175151]: Cluster is now healthy 2023-09-23T21:09:09.830 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:09:09 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T21:09:09.830 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:09:09 smithi026 ceph-mon[175151]: osdmap e398: 8 total, 8 up, 7 in 2023-09-23T21:09:09.871 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:09:09.912 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:09:09.949 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:09:09.954 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:09:09.954 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:09:10.054 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:09:10.089 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:09:10.129 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:09:10.135 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:09:10.136 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:09:10.236 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:09:10.327 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:09:10.627 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0: {} 2023-09-23T21:09:10.671 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1: {} 2023-09-23T21:09:10.714 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2: {} 2023-09-23T21:09:10.758 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3: {} 2023-09-23T21:09:10.797 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4: {} 2023-09-23T21:09:10.843 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T21:09:10.871 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:09:10 smithi070 ceph-mon[168282]: pgmap v1037: 109 pgs: 4 remapped+peering, 1 active+clean+remapped, 2 active+remapped+backfilling, 4 active+remapped+backfill_wait, 98 active+clean; 2.3 GiB data, 9.0 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 372 op/s; 12388/113778 objects misplaced (10.888%); 28 MiB/s, 442 objects/s recovering 2023-09-23T21:09:10.871 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:09:10 smithi070 ceph-mon[168282]: failed to encode map e398 with expected crc 2023-09-23T21:09:10.871 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:09:10 smithi070 ceph-mon[168282]: failed to encode map e398 with expected crc 2023-09-23T21:09:10.871 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:09:10 smithi070 ceph-mon[168282]: failed to encode map e398 with expected crc 2023-09-23T21:09:10.871 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:09:10 smithi070 ceph-mon[168282]: failed to encode map e398 with expected crc 2023-09-23T21:09:10.872 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:09:10 smithi070 ceph-mon[168282]: failed to encode map e398 with expected crc 2023-09-23T21:09:10.872 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:09:10 smithi070 ceph-mon[168282]: 9.e starting backfill to osd.5 from (0'0,0'0] MAX to 394'12074 2023-09-23T21:09:10.872 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:09:10 smithi070 ceph-mon[168282]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2023-09-23T21:09:10.872 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:09:10 smithi070 ceph-mon[168282]: osdmap e399: 8 total, 8 up, 7 in 2023-09-23T21:09:10.878 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6: {} 2023-09-23T21:09:10.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:09:10 smithi026 ceph-mon[175151]: pgmap v1037: 109 pgs: 4 remapped+peering, 1 active+clean+remapped, 2 active+remapped+backfilling, 4 active+remapped+backfill_wait, 98 active+clean; 2.3 GiB data, 9.0 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 372 op/s; 12388/113778 objects misplaced (10.888%); 28 MiB/s, 442 objects/s recovering 2023-09-23T21:09:10.883 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:09:10 smithi026 ceph-mon[175151]: failed to encode map e398 with expected crc 2023-09-23T21:09:10.883 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:09:10 smithi026 ceph-mon[175151]: failed to encode map e398 with expected crc 2023-09-23T21:09:10.883 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:09:10 smithi026 ceph-mon[175151]: failed to encode map e398 with expected crc 2023-09-23T21:09:10.883 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:09:10 smithi026 ceph-mon[175151]: failed to encode map e398 with expected crc 2023-09-23T21:09:10.883 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:09:10 smithi026 ceph-mon[175151]: failed to encode map e398 with expected crc 2023-09-23T21:09:10.883 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:09:10 smithi026 ceph-mon[175151]: 9.e starting backfill to osd.5 from (0'0,0'0] MAX to 394'12074 2023-09-23T21:09:10.883 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:09:10 smithi026 ceph-mon[175151]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2023-09-23T21:09:10.883 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:09:10 smithi026 ceph-mon[175151]: osdmap e399: 8 total, 8 up, 7 in 2023-09-23T21:09:10.883 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:09:10 smithi026 ceph-mon[179688]: pgmap v1037: 109 pgs: 4 remapped+peering, 1 active+clean+remapped, 2 active+remapped+backfilling, 4 active+remapped+backfill_wait, 98 active+clean; 2.3 GiB data, 9.0 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 372 op/s; 12388/113778 objects misplaced (10.888%); 28 MiB/s, 442 objects/s recovering 2023-09-23T21:09:10.883 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:09:10 smithi026 ceph-mon[179688]: failed to encode map e398 with expected crc 2023-09-23T21:09:10.884 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:09:10 smithi026 ceph-mon[179688]: failed to encode map e398 with expected crc 2023-09-23T21:09:10.884 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:09:10 smithi026 ceph-mon[179688]: failed to encode map e398 with expected crc 2023-09-23T21:09:10.884 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:09:10 smithi026 ceph-mon[179688]: failed to encode map e398 with expected crc 2023-09-23T21:09:10.884 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:09:10 smithi026 ceph-mon[179688]: failed to encode map e398 with expected crc 2023-09-23T21:09:10.884 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:09:10 smithi026 ceph-mon[179688]: 9.e starting backfill to osd.5 from (0'0,0'0] MAX to 394'12074 2023-09-23T21:09:10.884 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:09:10 smithi026 ceph-mon[179688]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2023-09-23T21:09:10.884 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:09:10 smithi026 ceph-mon[179688]: osdmap e399: 8 total, 8 up, 7 in 2023-09-23T21:09:10.932 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T21:09:10.949 INFO:teuthology.orchestra.run.smithi026.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T21:09:10.949 INFO:teuthology.orchestra.run.smithi026.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T21:09:10.949 INFO:teuthology.orchestra.run.smithi026.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T21:09:10.949 INFO:teuthology.orchestra.run.smithi026.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T21:09:10.949 INFO:teuthology.orchestra.run.smithi026.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T21:09:10.949 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T21:09:10.949 INFO:teuthology.orchestra.run.smithi026.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T21:09:10.949 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T21:09:11.082 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:09:11.091 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:09:11.091 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:09:11.192 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:09:11.297 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:09:11.427 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:09:11.431 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:09:11.431 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:09:11.532 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:09:11.594 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:09:11 smithi026 ceph-mon[175151]: failed to encode map e399 with expected crc 2023-09-23T21:09:11.594 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:09:11 smithi026 ceph-mon[175151]: failed to encode map e399 with expected crc 2023-09-23T21:09:11.594 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:09:11 smithi026 ceph-mon[175151]: failed to encode map e399 with expected crc 2023-09-23T21:09:11.595 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:09:11 smithi026 ceph-mon[175151]: failed to encode map e399 with expected crc 2023-09-23T21:09:11.595 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:09:11 smithi026 ceph-mon[175151]: failed to encode map e399 with expected crc 2023-09-23T21:09:11.595 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:09:11 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/1172031994' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:09:11.595 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:09:11 smithi026 ceph-mon[175151]: osdmap e400: 8 total, 8 up, 7 in 2023-09-23T21:09:11.596 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:09:11 smithi026 ceph-mon[179688]: failed to encode map e399 with expected crc 2023-09-23T21:09:11.596 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:09:11 smithi026 ceph-mon[179688]: failed to encode map e399 with expected crc 2023-09-23T21:09:11.596 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:09:11 smithi026 ceph-mon[179688]: failed to encode map e399 with expected crc 2023-09-23T21:09:11.596 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:09:11 smithi026 ceph-mon[179688]: failed to encode map e399 with expected crc 2023-09-23T21:09:11.596 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:09:11 smithi026 ceph-mon[179688]: failed to encode map e399 with expected crc 2023-09-23T21:09:11.597 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:09:11 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/1172031994' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:09:11.597 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:09:11 smithi026 ceph-mon[179688]: osdmap e400: 8 total, 8 up, 7 in 2023-09-23T21:09:11.605 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:09:11.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:09:11 smithi070 ceph-mon[168282]: failed to encode map e399 with expected crc 2023-09-23T21:09:11.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:09:11 smithi070 ceph-mon[168282]: failed to encode map e399 with expected crc 2023-09-23T21:09:11.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:09:11 smithi070 ceph-mon[168282]: failed to encode map e399 with expected crc 2023-09-23T21:09:11.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:09:11 smithi070 ceph-mon[168282]: failed to encode map e399 with expected crc 2023-09-23T21:09:11.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:09:11 smithi070 ceph-mon[168282]: failed to encode map e399 with expected crc 2023-09-23T21:09:11.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:09:11 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/1172031994' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:09:11.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:09:11 smithi070 ceph-mon[168282]: osdmap e400: 8 total, 8 up, 7 in 2023-09-23T21:09:11.957 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T21:09:12.530 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:09:12.539 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:09:12.540 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:09:12.641 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:09:12.669 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:09:12.714 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:09:12.716 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:09:12 smithi070 ceph-mon[168282]: pgmap v1040: 109 pgs: 6 remapped+peering, 1 active+clean+remapped, 3 active+remapped+backfilling, 2 active+remapped+backfill_wait, 97 active+clean; 2.3 GiB data, 9.0 GiB used, 617 GiB / 626 GiB avail; 10 KiB/s rd, 27 KiB/s wr, 459 op/s; 9952/112212 objects misplaced (8.869%); 62 MiB/s, 984 objects/s recovering 2023-09-23T21:09:12.716 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:09:12 smithi070 ceph-mon[168282]: failed to encode map e400 with expected crc 2023-09-23T21:09:12.716 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:09:12 smithi070 ceph-mon[168282]: failed to encode map e400 with expected crc 2023-09-23T21:09:12.716 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:09:12 smithi070 ceph-mon[168282]: failed to encode map e400 with expected crc 2023-09-23T21:09:12.716 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:09:12 smithi070 ceph-mon[168282]: failed to encode map e400 with expected crc 2023-09-23T21:09:12.716 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:09:12 smithi070 ceph-mon[168282]: failed to encode map e400 with expected crc 2023-09-23T21:09:12.716 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:09:12 smithi070 ceph-mon[168282]: 1.5 scrub starts 2023-09-23T21:09:12.717 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:09:12.717 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:09:12.738 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:09:12 smithi026 ceph-mon[175151]: pgmap v1040: 109 pgs: 6 remapped+peering, 1 active+clean+remapped, 3 active+remapped+backfilling, 2 active+remapped+backfill_wait, 97 active+clean; 2.3 GiB data, 9.0 GiB used, 617 GiB / 626 GiB avail; 10 KiB/s rd, 27 KiB/s wr, 459 op/s; 9952/112212 objects misplaced (8.869%); 62 MiB/s, 984 objects/s recovering 2023-09-23T21:09:12.738 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:09:12 smithi026 ceph-mon[175151]: failed to encode map e400 with expected crc 2023-09-23T21:09:12.738 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:09:12 smithi026 ceph-mon[175151]: failed to encode map e400 with expected crc 2023-09-23T21:09:12.738 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:09:12 smithi026 ceph-mon[175151]: failed to encode map e400 with expected crc 2023-09-23T21:09:12.738 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:09:12 smithi026 ceph-mon[175151]: failed to encode map e400 with expected crc 2023-09-23T21:09:12.738 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:09:12 smithi026 ceph-mon[175151]: failed to encode map e400 with expected crc 2023-09-23T21:09:12.739 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:09:12 smithi026 ceph-mon[175151]: 1.5 scrub starts 2023-09-23T21:09:12.739 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:09:12 smithi026 ceph-mon[179688]: pgmap v1040: 109 pgs: 6 remapped+peering, 1 active+clean+remapped, 3 active+remapped+backfilling, 2 active+remapped+backfill_wait, 97 active+clean; 2.3 GiB data, 9.0 GiB used, 617 GiB / 626 GiB avail; 10 KiB/s rd, 27 KiB/s wr, 459 op/s; 9952/112212 objects misplaced (8.869%); 62 MiB/s, 984 objects/s recovering 2023-09-23T21:09:12.739 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:09:12 smithi026 ceph-mon[179688]: failed to encode map e400 with expected crc 2023-09-23T21:09:12.739 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:09:12 smithi026 ceph-mon[179688]: failed to encode map e400 with expected crc 2023-09-23T21:09:12.739 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:09:12 smithi026 ceph-mon[179688]: failed to encode map e400 with expected crc 2023-09-23T21:09:12.739 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:09:12 smithi026 ceph-mon[179688]: failed to encode map e400 with expected crc 2023-09-23T21:09:12.739 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:09:12 smithi026 ceph-mon[179688]: failed to encode map e400 with expected crc 2023-09-23T21:09:12.739 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:09:12 smithi026 ceph-mon[179688]: 1.5 scrub starts 2023-09-23T21:09:12.818 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:09:12.894 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:09:12.987 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:09:12.990 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:09:12.991 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:09:13.078 INFO:teuthology.orchestra.run.smithi026.stdout:ERROR: (22) Invalid argument 2023-09-23T21:09:13.078 INFO:teuthology.orchestra.run.smithi026.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T21:09:13.092 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:09:13.121 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:09:13.136 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:09:13.162 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:09:13.165 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:09:13.166 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:09:13.266 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:09:13.342 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:09:13.444 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 6, 4, 5] out_osds: [7] dead_osds: [] live_osds: [0, 2, 3, 4, 5, 7, 6, 1] 2023-09-23T21:09:13.444 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T21:09:13.444 INFO:tasks.thrashosds.thrasher:fixing pg num pool .mgr 2023-09-23T21:09:13.444 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T21:09:13.461 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:09:13.476 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:09:13.477 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:09:13.577 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:09:13.633 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_ops_in_flight 2023-09-23T21:09:13.719 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:09:13 smithi026 ceph-mon[175151]: 1.5 scrub starts 2023-09-23T21:09:13.721 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:09:13 smithi026 ceph-mon[179688]: 1.5 scrub starts 2023-09-23T21:09:13.747 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:09:13.858 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:09:13.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:09:13 smithi070 ceph-mon[168282]: 1.5 scrub starts 2023-09-23T21:09:13.869 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:09:13.870 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:09:13.970 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:09:13.999 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:09:14.037 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:09:14.040 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:09:14.040 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:09:14.141 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:09:14.171 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:09:14.209 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:09:14.212 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:09:14.213 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:09:14.313 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:09:14.410 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:09:14.461 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:09:14.506 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T21:09:14.518 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:09:14.519 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:09:14.619 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:09:14.651 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:09:14.670 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T21:09:14.690 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:09:14.695 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:09:14.695 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:09:14.741 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:09:14 smithi026 ceph-mon[175151]: pgmap v1042: 109 pgs: 2 peering, 6 remapped+peering, 3 active+remapped+backfilling, 2 active+remapped+backfill_wait, 96 active+clean; 2.2 GiB data, 9.1 GiB used, 617 GiB / 626 GiB avail; 18 KiB/s rd, 48 KiB/s wr, 408 op/s; 8360/109887 objects misplaced (7.608%); 58 MiB/s, 919 objects/s recovering 2023-09-23T21:09:14.742 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:09:14 smithi026 ceph-mon[179688]: pgmap v1042: 109 pgs: 2 peering, 6 remapped+peering, 3 active+remapped+backfilling, 2 active+remapped+backfill_wait, 96 active+clean; 2.2 GiB data, 9.1 GiB used, 617 GiB / 626 GiB avail; 18 KiB/s rd, 48 KiB/s wr, 408 op/s; 8360/109887 objects misplaced (7.608%); 58 MiB/s, 919 objects/s recovering 2023-09-23T21:09:14.778 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:09:14.796 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:09:14.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:09:14 smithi070 ceph-mon[168282]: pgmap v1042: 109 pgs: 2 peering, 6 remapped+peering, 3 active+remapped+backfilling, 2 active+remapped+backfill_wait, 96 active+clean; 2.2 GiB data, 9.1 GiB used, 617 GiB / 626 GiB avail; 18 KiB/s rd, 48 KiB/s wr, 408 op/s; 8360/109887 objects misplaced (7.608%); 58 MiB/s, 919 objects/s recovering 2023-09-23T21:09:14.920 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:09:15.388 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:09:15.397 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:09:15.398 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:09:15.499 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:09:15.604 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:09:15 smithi026 ceph-mon[175151]: Health check update: Reduced data availability: 2 pgs inactive, 2 pgs peering (PG_AVAILABILITY) 2023-09-23T21:09:15.604 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:09:15 smithi026 ceph-mon[179688]: Health check update: Reduced data availability: 2 pgs inactive, 2 pgs peering (PG_AVAILABILITY) 2023-09-23T21:09:15.604 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:09:15.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:09:15 smithi070 ceph-mon[168282]: Health check update: Reduced data availability: 2 pgs inactive, 2 pgs peering (PG_AVAILABILITY) 2023-09-23T21:09:16.051 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T21:09:16.060 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:09:16.060 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:09:16.161 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:09:16.193 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:09:16.236 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:09:16.239 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:09:16.239 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:09:16.340 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:09:16.368 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:09:16.407 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:09:16.411 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:09:16.411 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:09:16.464 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:09:16.512 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:09:16.521 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:09:16 smithi070 ceph-mon[168282]: pgmap v1043: 109 pgs: 2 peering, 3 active+remapped+backfilling, 3 active+remapped+backfill_wait, 101 active+clean; 2.1 GiB data, 8.9 GiB used, 617 GiB / 626 GiB avail; 14 KiB/s rd, 37 KiB/s wr, 565 op/s; 8581/103788 objects misplaced (8.268%); 55 MiB/s, 873 objects/s recovering 2023-09-23T21:09:16.521 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:09:16 smithi070 ceph-mon[168282]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs inactive, 2 pgs peering) 2023-09-23T21:09:16.521 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:09:16 smithi070 ceph-mon[168282]: Cluster is now healthy 2023-09-23T21:09:16.521 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:09:16 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/2897653360' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T21:09:16.521 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:09:16 smithi070 ceph-mon[168282]: 1.5 scrub starts 2023-09-23T21:09:16.521 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:09:16 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:09:16.521 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:09:16 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:09:16.522 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:09:16 smithi026 ceph-mon[175151]: pgmap v1043: 109 pgs: 2 peering, 3 active+remapped+backfilling, 3 active+remapped+backfill_wait, 101 active+clean; 2.1 GiB data, 8.9 GiB used, 617 GiB / 626 GiB avail; 14 KiB/s rd, 37 KiB/s wr, 565 op/s; 8581/103788 objects misplaced (8.268%); 55 MiB/s, 873 objects/s recovering 2023-09-23T21:09:16.523 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:09:16 smithi026 ceph-mon[175151]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs inactive, 2 pgs peering) 2023-09-23T21:09:16.523 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:09:16 smithi026 ceph-mon[175151]: Cluster is now healthy 2023-09-23T21:09:16.523 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:09:16 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/2897653360' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T21:09:16.523 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:09:16 smithi026 ceph-mon[175151]: 1.5 scrub starts 2023-09-23T21:09:16.523 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:09:16 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:09:16.523 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:09:16 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:09:16.547 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:09:16.584 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:09:16.586 INFO:teuthology.orchestra.run.smithi026.stderr:noscrub is set 2023-09-23T21:09:16.588 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:09:16.588 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:09:16.689 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:09:16.719 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:09:16.767 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:09:16.774 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:09:16.774 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:09:16.789 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:09:16 smithi026 ceph-mon[179688]: pgmap v1043: 109 pgs: 2 peering, 3 active+remapped+backfilling, 3 active+remapped+backfill_wait, 101 active+clean; 2.1 GiB data, 8.9 GiB used, 617 GiB / 626 GiB avail; 14 KiB/s rd, 37 KiB/s wr, 565 op/s; 8581/103788 objects misplaced (8.268%); 55 MiB/s, 873 objects/s recovering 2023-09-23T21:09:16.789 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:09:16 smithi026 ceph-mon[179688]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs inactive, 2 pgs peering) 2023-09-23T21:09:16.789 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:09:16 smithi026 ceph-mon[179688]: Cluster is now healthy 2023-09-23T21:09:16.790 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:09:16 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/2897653360' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T21:09:16.790 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:09:16 smithi026 ceph-mon[179688]: 1.5 scrub starts 2023-09-23T21:09:16.790 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:09:16 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:09:16.790 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:09:16 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:09:16.875 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:09:16.914 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:09:16.964 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:09:16.968 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:09:16.968 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:09:17.069 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:09:17.100 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:09:17.145 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:09:17.149 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:09:17.149 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:09:17.250 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:09:17.345 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:09:17.405 INFO:teuthology.orchestra.run.smithi026.stdout:ERROR: (22) Invalid argument 2023-09-23T21:09:17.405 INFO:teuthology.orchestra.run.smithi026.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T21:09:17.515 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T21:09:17.526 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:09:17.527 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:09:17.627 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:09:17.661 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:09:17.703 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:09:17.708 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:09:17.709 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:09:17.749 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:09:17 smithi026 ceph-mon[179688]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:09:17.749 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:09:17 smithi026 ceph-mon[179688]: failed to encode map e401 with expected crc 2023-09-23T21:09:17.749 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:09:17 smithi026 ceph-mon[179688]: failed to encode map e401 with expected crc 2023-09-23T21:09:17.749 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:09:17 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/2897653360' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T21:09:17.749 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:09:17 smithi026 ceph-mon[179688]: osdmap e401: 8 total, 8 up, 7 in 2023-09-23T21:09:17.749 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:09:17 smithi026 ceph-mon[179688]: failed to encode map e401 with expected crc 2023-09-23T21:09:17.749 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:09:17 smithi026 ceph-mon[175151]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:09:17.749 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:09:17 smithi026 ceph-mon[175151]: failed to encode map e401 with expected crc 2023-09-23T21:09:17.749 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:09:17 smithi026 ceph-mon[175151]: failed to encode map e401 with expected crc 2023-09-23T21:09:17.749 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:09:17 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/2897653360' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T21:09:17.749 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:09:17 smithi026 ceph-mon[175151]: osdmap e401: 8 total, 8 up, 7 in 2023-09-23T21:09:17.750 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:09:17 smithi026 ceph-mon[175151]: failed to encode map e401 with expected crc 2023-09-23T21:09:17.809 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:09:17.838 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:09:17 smithi070 ceph-mon[168282]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:09:17.838 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:09:17 smithi070 ceph-mon[168282]: failed to encode map e401 with expected crc 2023-09-23T21:09:17.838 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:09:17 smithi070 ceph-mon[168282]: failed to encode map e401 with expected crc 2023-09-23T21:09:17.838 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:09:17 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/2897653360' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T21:09:17.838 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:09:17 smithi070 ceph-mon[168282]: osdmap e401: 8 total, 8 up, 7 in 2023-09-23T21:09:17.838 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:09:17 smithi070 ceph-mon[168282]: failed to encode map e401 with expected crc 2023-09-23T21:09:17.845 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:09:17.852 INFO:teuthology.orchestra.run.smithi026.stdout: 2023-09-23T21:09:17.874 INFO:teuthology.orchestra.run.smithi026.stderr:dumped all 2023-09-23T21:09:17.883 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:09:17.887 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:09:17.888 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:09:17.988 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:09:18.026 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:09:18.065 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:09:18.068 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:09:18.068 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:09:18.169 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:09:18.194 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:09:18.234 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:09:18.237 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:09:18.238 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:09:18.338 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:09:18.372 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:09:18.413 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:09:18.416 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:09:18.416 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:09:18.517 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:09:18.597 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:09:18 smithi026 ceph-mon[175151]: pgmap v1045: 109 pgs: 1 active+clean+scrubbing, 3 active+remapped+backfilling, 3 active+remapped+backfill_wait, 102 active+clean; 2.1 GiB data, 8.8 GiB used, 617 GiB / 626 GiB avail; 6.2 KiB/s rd, 17 KiB/s wr, 466 op/s; 8581/103122 objects misplaced (8.321%); 29 MiB/s, 468 objects/s recovering 2023-09-23T21:09:18.597 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:09:18 smithi026 ceph-mon[179688]: pgmap v1045: 109 pgs: 1 active+clean+scrubbing, 3 active+remapped+backfilling, 3 active+remapped+backfill_wait, 102 active+clean; 2.1 GiB data, 8.8 GiB used, 617 GiB / 626 GiB avail; 6.2 KiB/s rd, 17 KiB/s wr, 466 op/s; 8581/103122 objects misplaced (8.321%); 29 MiB/s, 468 objects/s recovering 2023-09-23T21:09:18.603 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:09:18.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:09:18 smithi070 ceph-mon[168282]: pgmap v1045: 109 pgs: 1 active+clean+scrubbing, 3 active+remapped+backfilling, 3 active+remapped+backfill_wait, 102 active+clean; 2.1 GiB data, 8.8 GiB used, 617 GiB / 626 GiB avail; 6.2 KiB/s rd, 17 KiB/s wr, 466 op/s; 8581/103122 objects misplaced (8.321%); 29 MiB/s, 468 objects/s recovering 2023-09-23T21:09:19.029 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:09:19.033 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:09:19.033 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:09:19.134 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:09:19.174 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:09:19.199 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T21:09:19.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:09:19 smithi070 ceph-mon[168282]: from='client.46217 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T21:09:19.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:09:19 smithi070 ceph-mon[168282]: failed to encode map e402 with expected crc 2023-09-23T21:09:19.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:09:19 smithi070 ceph-mon[168282]: failed to encode map e402 with expected crc 2023-09-23T21:09:19.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:09:19 smithi070 ceph-mon[168282]: osdmap e402: 8 total, 8 up, 7 in 2023-09-23T21:09:19.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:09:19 smithi070 ceph-mon[168282]: failed to encode map e402 with expected crc 2023-09-23T21:09:19.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:09:19 smithi070 ceph-mon[168282]: failed to encode map e402 with expected crc 2023-09-23T21:09:19.870 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:09:19 smithi026 ceph-mon[175151]: from='client.46217 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T21:09:19.870 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:09:19 smithi026 ceph-mon[175151]: failed to encode map e402 with expected crc 2023-09-23T21:09:19.870 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:09:19 smithi026 ceph-mon[175151]: failed to encode map e402 with expected crc 2023-09-23T21:09:19.870 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:09:19 smithi026 ceph-mon[175151]: osdmap e402: 8 total, 8 up, 7 in 2023-09-23T21:09:19.870 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:09:19 smithi026 ceph-mon[175151]: failed to encode map e402 with expected crc 2023-09-23T21:09:19.871 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:09:19 smithi026 ceph-mon[175151]: failed to encode map e402 with expected crc 2023-09-23T21:09:19.871 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 21:09:19 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[164029]: ::ffff:172.21.15.70 - - [23/Sep/2023:21:09:19] "GET /metrics HTTP/1.1" 200 36361 "" "Prometheus/2.43.0" 2023-09-23T21:09:19.871 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:09:19 smithi026 ceph-mon[179688]: from='client.46217 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T21:09:19.871 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:09:19 smithi026 ceph-mon[179688]: failed to encode map e402 with expected crc 2023-09-23T21:09:19.871 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:09:19 smithi026 ceph-mon[179688]: failed to encode map e402 with expected crc 2023-09-23T21:09:19.871 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:09:19 smithi026 ceph-mon[179688]: osdmap e402: 8 total, 8 up, 7 in 2023-09-23T21:09:19.871 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:09:19 smithi026 ceph-mon[179688]: failed to encode map e402 with expected crc 2023-09-23T21:09:19.871 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:09:19 smithi026 ceph-mon[179688]: failed to encode map e402 with expected crc 2023-09-23T21:09:19.887 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:09:19.902 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:09:19.902 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:09:20.003 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:09:20.055 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:09:20.504 INFO:teuthology.orchestra.run.smithi026.stdout:{"pg_ready":true,"pg_map":{"version":1045,"stamp":"2023-09-23T21:09:16.925931+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":2251745630,"num_objects":34374,"num_object_clones":0,"num_object_copies":103122,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":8581,"num_objects_unfound":0,"num_objects_dirty":34374,"num_whiteouts":0,"num_read":45,"num_read_kb":84,"num_write":192281,"num_write_kb":7249722,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":15885,"num_bytes_recovered":1038258600,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":86266,"ondisk_log_size":86266,"up":327,"acting":327,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":344,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":9279208,"kb_used_data":6270640,"kb_used_omap":137,"kb_used_meta":3008246,"kb_avail":646936856,"statfs":{"total":671965249536,"available":662463340544,"internally_reserved":0,"allocated":6421135360,"data_stored":6410607958,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":141000,"internal_metadata":3080444216},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[5,0,1,1,2],"upper_bound":32},"perf_stat":{"commit_latency_ms":6,"apply_latency_ms":6,"commit_latency_ns":6000000,"apply_latency_ns":6000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-198547245,"num_objects":-3030,"num_object_clones":0,"num_object_copies":-9090,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":-1371,"num_objects_unfound":0,"num_objects_dirty":-3030,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":3041,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3081,"num_bytes_recovered":202258987,"num_keys_recovered":-10,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"6.570823"},"pg_stats":[{"pgid":"3.1a","version":"68'1","reported_seq":15,"reported_epoch":399,"state":"active+clean","last_fresh":"2023-09-23T21:09:11.098733+0000","last_change":"2023-09-23T21:09:08.416508+0000","last_active":"2023-09-23T21:09:11.098733+0000","last_peered":"2023-09-23T21:09:11.098733+0000","last_clean":"2023-09-23T21:09:11.098733+0000","last_became_active":"2023-09-23T21:09:08.416124+0000","last_became_peered":"2023-09-23T21:09:08.416124+0000","last_unstale":"2023-09-23T21:09:11.098733+0000","last_undegraded":"2023-09-23T21:09:11.098733+0000","last_fullsized":"2023-09-23T21:09:11.098733+0000","mapping_epoch":396,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":397,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:46:38.597918+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,0],"acting":[2,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":396,"reported_epoch":399,"state":"active+clean","last_fresh":"2023-09-23T21:09:10.370982+0000","last_change":"2023-09-23T20:54:13.942019+0000","last_active":"2023-09-23T21:09:10.370982+0000","last_peered":"2023-09-23T21:09:10.370982+0000","last_clean":"2023-09-23T21:09:10.370982+0000","last_became_active":"2023-09-23T20:54:13.941647+0000","last_became_peered":"2023-09-23T20:54:13.941647+0000","last_unstale":"2023-09-23T21:09:10.370982+0000","last_undegraded":"2023-09-23T21:09:10.370982+0000","last_fullsized":"2023-09-23T21:09:10.370982+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:30:54.400944+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":382,"reported_epoch":399,"state":"active+clean","last_fresh":"2023-09-23T21:09:10.368676+0000","last_change":"2023-09-23T21:09:08.428579+0000","last_active":"2023-09-23T21:09:10.368676+0000","last_peered":"2023-09-23T21:09:10.368676+0000","last_clean":"2023-09-23T21:09:10.368676+0000","last_became_active":"2023-09-23T21:09:08.428247+0000","last_became_peered":"2023-09-23T21:09:08.428247+0000","last_unstale":"2023-09-23T21:09:10.368676+0000","last_undegraded":"2023-09-23T21:09:10.368676+0000","last_fullsized":"2023-09-23T21:09:10.368676+0000","mapping_epoch":396,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":397,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:47:56.458062+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":398,"reported_epoch":399,"state":"active+clean","last_fresh":"2023-09-23T21:09:10.363705+0000","last_change":"2023-09-23T21:08:43.513644+0000","last_active":"2023-09-23T21:09:10.363705+0000","last_peered":"2023-09-23T21:09:10.363705+0000","last_clean":"2023-09-23T21:09:10.363705+0000","last_became_active":"2023-09-23T21:08:43.512977+0000","last_became_peered":"2023-09-23T21:08:43.512977+0000","last_unstale":"2023-09-23T21:09:10.363705+0000","last_undegraded":"2023-09-23T21:09:10.363705+0000","last_fullsized":"2023-09-23T21:09:10.363705+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:42:59.825772+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":402,"reported_epoch":399,"state":"active+clean","last_fresh":"2023-09-23T21:09:10.361676+0000","last_change":"2023-09-23T21:05:55.325654+0000","last_active":"2023-09-23T21:09:10.361676+0000","last_peered":"2023-09-23T21:09:10.361676+0000","last_clean":"2023-09-23T21:09:10.361676+0000","last_became_active":"2023-09-23T21:05:55.325298+0000","last_became_peered":"2023-09-23T21:05:55.325298+0000","last_unstale":"2023-09-23T21:09:10.361676+0000","last_undegraded":"2023-09-23T21:09:10.361676+0000","last_fullsized":"2023-09-23T21:09:10.361676+0000","mapping_epoch":350,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":351,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:33:19.561063+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":402,"reported_epoch":399,"state":"active+clean","last_fresh":"2023-09-23T21:09:10.368010+0000","last_change":"2023-09-23T21:05:55.319892+0000","last_active":"2023-09-23T21:09:10.368010+0000","last_peered":"2023-09-23T21:09:10.368010+0000","last_clean":"2023-09-23T21:09:10.368010+0000","last_became_active":"2023-09-23T21:05:55.319536+0000","last_became_peered":"2023-09-23T21:05:55.319536+0000","last_unstale":"2023-09-23T21:09:10.368010+0000","last_undegraded":"2023-09-23T21:09:10.368010+0000","last_fullsized":"2023-09-23T21:09:10.368010+0000","mapping_epoch":350,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":351,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:14:48.388831+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":397,"reported_epoch":399,"state":"active+clean","last_fresh":"2023-09-23T21:09:10.364447+0000","last_change":"2023-09-23T21:05:04.304358+0000","last_active":"2023-09-23T21:09:10.364447+0000","last_peered":"2023-09-23T21:09:10.364447+0000","last_clean":"2023-09-23T21:09:10.364447+0000","last_became_active":"2023-09-23T21:05:04.303913+0000","last_became_peered":"2023-09-23T21:05:04.303913+0000","last_unstale":"2023-09-23T21:09:10.364447+0000","last_undegraded":"2023-09-23T21:09:10.364447+0000","last_fullsized":"2023-09-23T21:09:10.364447+0000","mapping_epoch":332,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:21:21.860944+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":417,"reported_epoch":399,"state":"active+clean","last_fresh":"2023-09-23T21:09:10.361746+0000","last_change":"2023-09-23T21:09:08.423849+0000","last_active":"2023-09-23T21:09:10.361746+0000","last_peered":"2023-09-23T21:09:10.361746+0000","last_clean":"2023-09-23T21:09:10.361746+0000","last_became_active":"2023-09-23T21:09:08.423444+0000","last_became_peered":"2023-09-23T21:09:08.423444+0000","last_unstale":"2023-09-23T21:09:10.361746+0000","last_undegraded":"2023-09-23T21:09:10.361746+0000","last_fullsized":"2023-09-23T21:09:10.361746+0000","mapping_epoch":396,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":397,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:51:39.763421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,5],"acting":[3,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"79'161","reported_seq":725,"reported_epoch":400,"state":"active+clean","last_fresh":"2023-09-23T21:09:11.387858+0000","last_change":"2023-09-23T21:09:11.387858+0000","last_active":"2023-09-23T21:09:11.387858+0000","last_peered":"2023-09-23T21:09:11.387858+0000","last_clean":"2023-09-23T21:09:11.387858+0000","last_became_active":"2023-09-23T21:09:11.387462+0000","last_became_peered":"2023-09-23T21:09:11.387462+0000","last_unstale":"2023-09-23T21:09:11.387858+0000","last_undegraded":"2023-09-23T21:09:11.387858+0000","last_fullsized":"2023-09-23T21:09:11.387858+0000","mapping_epoch":399,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":400,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:25.096401+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:25.096401+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:00:43.986148+0000","scrub_duration":0.002859393,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,0],"acting":[2,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":104,"reported_epoch":399,"state":"active+clean","last_fresh":"2023-09-23T21:09:10.365522+0000","last_change":"2023-09-23T21:08:43.499467+0000","last_active":"2023-09-23T21:09:10.365522+0000","last_peered":"2023-09-23T21:09:10.365522+0000","last_clean":"2023-09-23T21:09:10.365522+0000","last_became_active":"2023-09-23T21:08:43.499142+0000","last_became_peered":"2023-09-23T21:08:43.499142+0000","last_unstale":"2023-09-23T21:09:10.365522+0000","last_undegraded":"2023-09-23T21:09:10.365522+0000","last_fullsized":"2023-09-23T21:09:10.365522+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:25:25.593675+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":354,"reported_epoch":399,"state":"active+clean","last_fresh":"2023-09-23T21:09:10.381500+0000","last_change":"2023-09-23T21:08:43.494810+0000","last_active":"2023-09-23T21:09:10.381500+0000","last_peered":"2023-09-23T21:09:10.381500+0000","last_clean":"2023-09-23T21:09:10.381500+0000","last_became_active":"2023-09-23T21:08:43.494013+0000","last_became_peered":"2023-09-23T21:08:43.494013+0000","last_unstale":"2023-09-23T21:09:10.381500+0000","last_undegraded":"2023-09-23T21:09:10.381500+0000","last_fullsized":"2023-09-23T21:09:10.381500+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:32:22.787562+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":252,"reported_epoch":399,"state":"active+clean","last_fresh":"2023-09-23T21:09:10.371094+0000","last_change":"2023-09-23T21:09:08.475715+0000","last_active":"2023-09-23T21:09:10.371094+0000","last_peered":"2023-09-23T21:09:10.371094+0000","last_clean":"2023-09-23T21:09:10.371094+0000","last_became_active":"2023-09-23T21:09:08.442399+0000","last_became_peered":"2023-09-23T21:09:08.442399+0000","last_unstale":"2023-09-23T21:09:10.371094+0000","last_undegraded":"2023-09-23T21:09:10.371094+0000","last_fullsized":"2023-09-23T21:09:10.371094+0000","mapping_epoch":396,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":397,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:01:52.337007+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,2],"acting":[3,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":82,"reported_epoch":399,"state":"active+clean","last_fresh":"2023-09-23T21:09:10.369646+0000","last_change":"2023-09-23T21:08:54.693560+0000","last_active":"2023-09-23T21:09:10.369646+0000","last_peered":"2023-09-23T21:09:10.369646+0000","last_clean":"2023-09-23T21:09:10.369646+0000","last_became_active":"2023-09-23T21:08:54.693199+0000","last_became_peered":"2023-09-23T21:08:54.693199+0000","last_unstale":"2023-09-23T21:09:10.369646+0000","last_undegraded":"2023-09-23T21:09:10.369646+0000","last_fullsized":"2023-09-23T21:09:10.369646+0000","mapping_epoch":388,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":389,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:09:53.015332+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"79'22","reported_seq":68,"reported_epoch":399,"state":"active+clean","last_fresh":"2023-09-23T21:09:10.367710+0000","last_change":"2023-09-23T21:09:08.463012+0000","last_active":"2023-09-23T21:09:10.367710+0000","last_peered":"2023-09-23T21:09:10.367710+0000","last_clean":"2023-09-23T21:09:10.367710+0000","last_became_active":"2023-09-23T21:09:08.410151+0000","last_became_peered":"2023-09-23T21:09:08.410151+0000","last_unstale":"2023-09-23T21:09:10.367710+0000","last_undegraded":"2023-09-23T21:09:10.367710+0000","last_fullsized":"2023-09-23T21:09:10.367710+0000","mapping_epoch":396,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":397,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:33:24.349490+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4734,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,0],"acting":[6,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":311,"reported_epoch":399,"state":"active+clean","last_fresh":"2023-09-23T21:09:10.370377+0000","last_change":"2023-09-23T21:05:55.325203+0000","last_active":"2023-09-23T21:09:10.370377+0000","last_peered":"2023-09-23T21:09:10.370377+0000","last_clean":"2023-09-23T21:09:10.370377+0000","last_became_active":"2023-09-23T21:05:55.323957+0000","last_became_peered":"2023-09-23T21:05:55.323957+0000","last_unstale":"2023-09-23T21:09:10.370377+0000","last_undegraded":"2023-09-23T21:09:10.370377+0000","last_fullsized":"2023-09-23T21:09:10.370377+0000","mapping_epoch":350,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":351,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:03:20.418924+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.11","version":"79'161","reported_seq":667,"reported_epoch":399,"state":"active+clean","last_fresh":"2023-09-23T21:09:10.367012+0000","last_change":"2023-09-23T21:05:07.301125+0000","last_active":"2023-09-23T21:09:10.367012+0000","last_peered":"2023-09-23T21:09:10.367012+0000","last_clean":"2023-09-23T21:09:10.367012+0000","last_became_active":"2023-09-23T21:05:07.300809+0000","last_became_peered":"2023-09-23T21:05:07.300809+0000","last_unstale":"2023-09-23T21:09:10.367012+0000","last_undegraded":"2023-09-23T21:09:10.367012+0000","last_fullsized":"2023-09-23T21:09:10.367012+0000","mapping_epoch":335,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":336,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:28.756634+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:28.756634+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:28:29.079665+0000","scrub_duration":0.0064925119999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":31,"reported_epoch":399,"state":"active+clean","last_fresh":"2023-09-23T21:09:11.098753+0000","last_change":"2023-09-23T21:09:08.453198+0000","last_active":"2023-09-23T21:09:11.098753+0000","last_peered":"2023-09-23T21:09:11.098753+0000","last_clean":"2023-09-23T21:09:11.098753+0000","last_became_active":"2023-09-23T21:09:08.409264+0000","last_became_peered":"2023-09-23T21:09:08.409264+0000","last_unstale":"2023-09-23T21:09:11.098753+0000","last_undegraded":"2023-09-23T21:09:11.098753+0000","last_fullsized":"2023-09-23T21:09:11.098753+0000","mapping_epoch":396,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":397,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:10:21.176747+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,0],"acting":[2,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":393,"reported_epoch":399,"state":"active+clean","last_fresh":"2023-09-23T21:09:10.371259+0000","last_change":"2023-09-23T21:08:43.495331+0000","last_active":"2023-09-23T21:09:10.371259+0000","last_peered":"2023-09-23T21:09:10.371259+0000","last_clean":"2023-09-23T21:09:10.371259+0000","last_became_active":"2023-09-23T21:08:43.494485+0000","last_became_peered":"2023-09-23T21:08:43.494485+0000","last_unstale":"2023-09-23T21:09:10.371259+0000","last_undegraded":"2023-09-23T21:09:10.371259+0000","last_fullsized":"2023-09-23T21:09:10.371259+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:15:19.762943+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.10","version":"79'161","reported_seq":509,"reported_epoch":400,"state":"active+clean","last_fresh":"2023-09-23T21:09:11.389150+0000","last_change":"2023-09-23T21:09:11.388116+0000","last_active":"2023-09-23T21:09:11.389150+0000","last_peered":"2023-09-23T21:09:11.389150+0000","last_clean":"2023-09-23T21:09:11.389150+0000","last_became_active":"2023-09-23T21:09:11.387118+0000","last_became_peered":"2023-09-23T21:09:11.387118+0000","last_unstale":"2023-09-23T21:09:11.389150+0000","last_undegraded":"2023-09-23T21:09:11.389150+0000","last_fullsized":"2023-09-23T21:09:11.389150+0000","mapping_epoch":399,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":400,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:06:50.336992+0000","last_deep_scrub":"79'161","last_deep_scrub_stamp":"2023-09-23T21:06:50.336992+0000","last_clean_scrub_stamp":"2023-09-23T21:06:50.336992+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:02:03.484838+0000","scrub_duration":0.0026150930000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":388,"reported_epoch":399,"state":"active+clean","last_fresh":"2023-09-23T21:09:10.364660+0000","last_change":"2023-09-23T21:08:43.512749+0000","last_active":"2023-09-23T21:09:10.364660+0000","last_peered":"2023-09-23T21:09:10.364660+0000","last_clean":"2023-09-23T21:09:10.364660+0000","last_became_active":"2023-09-23T21:08:43.512299+0000","last_became_peered":"2023-09-23T21:08:43.512299+0000","last_unstale":"2023-09-23T21:09:10.364660+0000","last_undegraded":"2023-09-23T21:09:10.364660+0000","last_fullsized":"2023-09-23T21:09:10.364660+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:19:58.835209+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":131,"reported_epoch":399,"state":"active+clean","last_fresh":"2023-09-23T21:09:10.370893+0000","last_change":"2023-09-23T21:08:43.496213+0000","last_active":"2023-09-23T21:09:10.370893+0000","last_peered":"2023-09-23T21:09:10.370893+0000","last_clean":"2023-09-23T21:09:10.370893+0000","last_became_active":"2023-09-23T21:08:43.495883+0000","last_became_peered":"2023-09-23T21:08:43.495883+0000","last_unstale":"2023-09-23T21:09:10.370893+0000","last_undegraded":"2023-09-23T21:09:10.370893+0000","last_fullsized":"2023-09-23T21:09:10.370893+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:27:12.483033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.13","version":"79'161","reported_seq":691,"reported_epoch":400,"state":"active+clean","last_fresh":"2023-09-23T21:09:11.416774+0000","last_change":"2023-09-23T21:09:11.416774+0000","last_active":"2023-09-23T21:09:11.416774+0000","last_peered":"2023-09-23T21:09:11.416774+0000","last_clean":"2023-09-23T21:09:11.416774+0000","last_became_active":"2023-09-23T21:09:11.416431+0000","last_became_peered":"2023-09-23T21:09:11.416431+0000","last_unstale":"2023-09-23T21:09:11.416774+0000","last_undegraded":"2023-09-23T21:09:11.416774+0000","last_fullsized":"2023-09-23T21:09:11.416774+0000","mapping_epoch":399,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":400,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:36.420362+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:36.420362+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:51:54.917367+0000","scrub_duration":0.028838444000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,0],"acting":[1,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":354,"reported_epoch":399,"state":"active+clean","last_fresh":"2023-09-23T21:09:10.369624+0000","last_change":"2023-09-23T21:08:43.516055+0000","last_active":"2023-09-23T21:09:10.369624+0000","last_peered":"2023-09-23T21:09:10.369624+0000","last_clean":"2023-09-23T21:09:10.369624+0000","last_became_active":"2023-09-23T21:08:43.515799+0000","last_became_peered":"2023-09-23T21:08:43.515799+0000","last_unstale":"2023-09-23T21:09:10.369624+0000","last_undegraded":"2023-09-23T21:09:10.369624+0000","last_fullsized":"2023-09-23T21:09:10.369624+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:05:28.535911+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":370,"reported_epoch":399,"state":"active+clean","last_fresh":"2023-09-23T21:09:10.363928+0000","last_change":"2023-09-23T21:08:43.513369+0000","last_active":"2023-09-23T21:09:10.363928+0000","last_peered":"2023-09-23T21:09:10.363928+0000","last_clean":"2023-09-23T21:09:10.363928+0000","last_became_active":"2023-09-23T21:08:43.512435+0000","last_became_peered":"2023-09-23T21:08:43.512435+0000","last_unstale":"2023-09-23T21:09:10.363928+0000","last_undegraded":"2023-09-23T21:09:10.363928+0000","last_fullsized":"2023-09-23T21:09:10.363928+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:00:10.970062+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.12","version":"79'161","reported_seq":561,"reported_epoch":399,"state":"active+clean","last_fresh":"2023-09-23T21:09:10.365720+0000","last_change":"2023-09-23T21:05:56.785264+0000","last_active":"2023-09-23T21:09:10.365720+0000","last_peered":"2023-09-23T21:09:10.365720+0000","last_clean":"2023-09-23T21:09:10.365720+0000","last_became_active":"2023-09-23T21:05:55.322810+0000","last_became_peered":"2023-09-23T21:05:55.322810+0000","last_unstale":"2023-09-23T21:09:10.365720+0000","last_undegraded":"2023-09-23T21:09:10.365720+0000","last_fullsized":"2023-09-23T21:09:10.365720+0000","mapping_epoch":350,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":351,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:05:56.785173+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:05:56.785173+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:56:12.932786+0000","scrub_duration":0.0042114370000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":432,"reported_epoch":399,"state":"active+clean","last_fresh":"2023-09-23T21:09:10.367125+0000","last_change":"2023-09-23T21:05:04.332774+0000","last_active":"2023-09-23T21:09:10.367125+0000","last_peered":"2023-09-23T21:09:10.367125+0000","last_clean":"2023-09-23T21:09:10.367125+0000","last_became_active":"2023-09-23T21:05:04.304541+0000","last_became_peered":"2023-09-23T21:05:04.304541+0000","last_unstale":"2023-09-23T21:09:10.367125+0000","last_undegraded":"2023-09-23T21:09:10.367125+0000","last_fullsized":"2023-09-23T21:09:10.367125+0000","mapping_epoch":332,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:27:02.401939+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":216,"reported_epoch":399,"state":"active+clean","last_fresh":"2023-09-23T21:09:10.371272+0000","last_change":"2023-09-23T21:08:43.494461+0000","last_active":"2023-09-23T21:09:10.371272+0000","last_peered":"2023-09-23T21:09:10.371272+0000","last_clean":"2023-09-23T21:09:10.371272+0000","last_became_active":"2023-09-23T21:08:43.493566+0000","last_became_peered":"2023-09-23T21:08:43.493566+0000","last_unstale":"2023-09-23T21:09:10.371272+0000","last_undegraded":"2023-09-23T21:09:10.371272+0000","last_fullsized":"2023-09-23T21:09:10.371272+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:25:54.987910+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.5","version":"400'11851","reported_seq":13517,"reported_epoch":400,"state":"active+remapped+backfilling","last_fresh":"2023-09-23T21:09:12.360837+0000","last_change":"2023-09-23T21:09:08.457860+0000","last_active":"2023-09-23T21:09:12.360837+0000","last_peered":"2023-09-23T21:09:12.360837+0000","last_clean":"2023-09-23T21:08:42.439150+0000","last_became_active":"2023-09-23T21:08:44.748930+0000","last_became_peered":"2023-09-23T21:08:44.748930+0000","last_unstale":"2023-09-23T21:09:12.360837+0000","last_undegraded":"2023-09-23T21:09:12.360837+0000","last_fullsized":"2023-09-23T21:09:12.360837+0000","mapping_epoch":383,"log_start":"366'6379","ondisk_log_start":"366'6379","created":346,"last_epoch_clean":347,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:05:47.261750+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:05:47.261750+0000","last_clean_scrub_stamp":"2023-09-23T21:05:47.261750+0000","objects_scrubbed":0,"log_size":5472,"log_dups_size":0,"ondisk_log_size":5472,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:47:45.757703+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":143458304,"num_objects":2189,"num_object_clones":0,"num_object_copies":6567,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1432,"num_objects_unfound":0,"num_objects_dirty":2189,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11851,"num_write_kb":449280,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":795,"num_bytes_recovered":51970048,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,1],"acting":[0,5,7],"avail_no_missing":["0","5","7"],"object_location_counts":[{"shards":"0,5,7","objects":2189}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.d","version":"79'161","reported_seq":492,"reported_epoch":400,"state":"active+clean","last_fresh":"2023-09-23T21:09:11.395832+0000","last_change":"2023-09-23T21:09:11.395832+0000","last_active":"2023-09-23T21:09:11.395832+0000","last_peered":"2023-09-23T21:09:11.395832+0000","last_clean":"2023-09-23T21:09:11.395832+0000","last_became_active":"2023-09-23T21:09:11.395441+0000","last_became_peered":"2023-09-23T21:09:11.395441+0000","last_unstale":"2023-09-23T21:09:11.395832+0000","last_undegraded":"2023-09-23T21:09:11.395832+0000","last_fullsized":"2023-09-23T21:09:11.395832+0000","mapping_epoch":399,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":400,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:06:38.762742+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:06:38.762742+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.014169637000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":29,"reported_epoch":400,"state":"active+clean","last_fresh":"2023-09-23T21:09:11.379112+0000","last_change":"2023-09-23T21:09:08.451787+0000","last_active":"2023-09-23T21:09:11.379112+0000","last_peered":"2023-09-23T21:09:11.379112+0000","last_clean":"2023-09-23T21:09:11.379112+0000","last_became_active":"2023-09-23T21:09:08.416379+0000","last_became_peered":"2023-09-23T21:09:08.416379+0000","last_unstale":"2023-09-23T21:09:11.379112+0000","last_undegraded":"2023-09-23T21:09:11.379112+0000","last_fullsized":"2023-09-23T21:09:11.379112+0000","mapping_epoch":396,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":397,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:23:42.852988+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,5],"acting":[0,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":97,"reported_epoch":399,"state":"active+clean","last_fresh":"2023-09-23T21:09:10.365123+0000","last_change":"2023-09-23T21:08:54.675136+0000","last_active":"2023-09-23T21:09:10.365123+0000","last_peered":"2023-09-23T21:09:10.365123+0000","last_clean":"2023-09-23T21:09:10.365123+0000","last_became_active":"2023-09-23T21:08:54.674772+0000","last_became_peered":"2023-09-23T21:08:54.674772+0000","last_unstale":"2023-09-23T21:09:10.365123+0000","last_undegraded":"2023-09-23T21:09:10.365123+0000","last_fullsized":"2023-09-23T21:09:10.365123+0000","mapping_epoch":388,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":389,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:16:55.575774+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,3],"acting":[5,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.4","version":"400'11919","reported_seq":11978,"reported_epoch":400,"state":"active+clean","last_fresh":"2023-09-23T21:09:13.506360+0000","last_change":"2023-09-23T21:05:55.320259+0000","last_active":"2023-09-23T21:09:13.506360+0000","last_peered":"2023-09-23T21:09:13.506360+0000","last_clean":"2023-09-23T21:09:13.506360+0000","last_became_active":"2023-09-23T21:05:55.319895+0000","last_became_peered":"2023-09-23T21:05:55.319895+0000","last_unstale":"2023-09-23T21:09:13.506360+0000","last_undegraded":"2023-09-23T21:09:13.506360+0000","last_fullsized":"2023-09-23T21:09:13.506360+0000","mapping_epoch":350,"log_start":"367'7007","ondisk_log_start":"367'7007","created":346,"last_epoch_clean":351,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:05:47.261750+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:05:47.261750+0000","last_clean_scrub_stamp":"2023-09-23T21:05:47.261750+0000","objects_scrubbed":0,"log_size":4912,"log_dups_size":0,"ondisk_log_size":4912,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:39:53.181523+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":137035776,"num_objects":2091,"num_object_clones":0,"num_object_copies":6273,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2091,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11919,"num_write_kb":448320,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.c","version":"79'161","reported_seq":705,"reported_epoch":399,"state":"active+clean","last_fresh":"2023-09-23T21:09:10.370837+0000","last_change":"2023-09-23T20:57:22.041071+0000","last_active":"2023-09-23T21:09:10.370837+0000","last_peered":"2023-09-23T21:09:10.370837+0000","last_clean":"2023-09-23T21:09:10.370837+0000","last_became_active":"2023-09-23T20:57:15.973204+0000","last_became_peered":"2023-09-23T20:57:15.973204+0000","last_unstale":"2023-09-23T21:09:10.370837+0000","last_undegraded":"2023-09-23T21:09:10.370837+0000","last_fullsized":"2023-09-23T21:09:10.370837+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:22.040907+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:22.040907+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:47:02.019978+0000","scrub_duration":0.013021176000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":29,"reported_epoch":399,"state":"active+clean","last_fresh":"2023-09-23T21:09:10.378963+0000","last_change":"2023-09-23T21:09:08.441997+0000","last_active":"2023-09-23T21:09:10.378963+0000","last_peered":"2023-09-23T21:09:10.378963+0000","last_clean":"2023-09-23T21:09:10.378963+0000","last_became_active":"2023-09-23T21:09:08.407779+0000","last_became_peered":"2023-09-23T21:09:08.407779+0000","last_unstale":"2023-09-23T21:09:10.378963+0000","last_undegraded":"2023-09-23T21:09:10.378963+0000","last_fullsized":"2023-09-23T21:09:10.378963+0000","mapping_epoch":396,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":397,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:47:24.353451+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,5],"acting":[1,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":338,"reported_epoch":399,"state":"active+clean","last_fresh":"2023-09-23T21:09:10.369730+0000","last_change":"2023-09-23T21:05:55.325375+0000","last_active":"2023-09-23T21:09:10.369730+0000","last_peered":"2023-09-23T21:09:10.369730+0000","last_clean":"2023-09-23T21:09:10.369730+0000","last_became_active":"2023-09-23T21:05:55.324170+0000","last_became_peered":"2023-09-23T21:05:55.324170+0000","last_unstale":"2023-09-23T21:09:10.369730+0000","last_undegraded":"2023-09-23T21:09:10.369730+0000","last_fullsized":"2023-09-23T21:09:10.369730+0000","mapping_epoch":350,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":351,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:08:07.299605+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.7","version":"400'12074","reported_seq":17475,"reported_epoch":400,"state":"active+clean","last_fresh":"2023-09-23T21:09:13.995737+0000","last_change":"2023-09-23T21:09:08.472231+0000","last_active":"2023-09-23T21:09:13.995737+0000","last_peered":"2023-09-23T21:09:13.995737+0000","last_clean":"2023-09-23T21:09:13.995737+0000","last_became_active":"2023-09-23T21:09:08.440270+0000","last_became_peered":"2023-09-23T21:09:08.440270+0000","last_unstale":"2023-09-23T21:09:13.995737+0000","last_undegraded":"2023-09-23T21:09:13.995737+0000","last_fullsized":"2023-09-23T21:09:13.995737+0000","mapping_epoch":396,"log_start":"367'7141","ondisk_log_start":"367'7141","created":346,"last_epoch_clean":397,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:05:47.261750+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:05:47.261750+0000","last_clean_scrub_stamp":"2023-09-23T21:05:47.261750+0000","objects_scrubbed":0,"log_size":4933,"log_dups_size":0,"ondisk_log_size":4933,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:42:10.000374+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":135397376,"num_objects":2066,"num_object_clones":0,"num_object_copies":6198,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2066,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12074,"num_write_kb":452480,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2654,"num_bytes_recovered":173932544,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,2],"acting":[3,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.f","version":"79'161","reported_seq":764,"reported_epoch":400,"state":"active+clean","last_fresh":"2023-09-23T21:09:11.395511+0000","last_change":"2023-09-23T21:09:11.395511+0000","last_active":"2023-09-23T21:09:11.395511+0000","last_peered":"2023-09-23T21:09:11.395511+0000","last_clean":"2023-09-23T21:09:11.395511+0000","last_became_active":"2023-09-23T21:09:11.395144+0000","last_became_peered":"2023-09-23T21:09:11.395144+0000","last_unstale":"2023-09-23T21:09:11.395511+0000","last_undegraded":"2023-09-23T21:09:11.395511+0000","last_fullsized":"2023-09-23T21:09:11.395511+0000","mapping_epoch":399,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":400,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:27.880776+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:27.880776+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:23:48.675879+0000","scrub_duration":0.0014004899999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,3],"acting":[6,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":95,"reported_epoch":399,"state":"active+clean","last_fresh":"2023-09-23T21:09:10.365366+0000","last_change":"2023-09-23T21:08:43.532612+0000","last_active":"2023-09-23T21:09:10.365366+0000","last_peered":"2023-09-23T21:09:10.365366+0000","last_clean":"2023-09-23T21:09:10.365366+0000","last_became_active":"2023-09-23T21:08:43.499861+0000","last_became_peered":"2023-09-23T21:08:43.499861+0000","last_unstale":"2023-09-23T21:09:10.365366+0000","last_undegraded":"2023-09-23T21:09:10.365366+0000","last_fullsized":"2023-09-23T21:09:10.365366+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:07:50.158275+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":140,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,3],"acting":[5,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":363,"reported_epoch":399,"state":"active+clean","last_fresh":"2023-09-23T21:09:10.369576+0000","last_change":"2023-09-23T21:08:54.699779+0000","last_active":"2023-09-23T21:09:10.369576+0000","last_peered":"2023-09-23T21:09:10.369576+0000","last_clean":"2023-09-23T21:09:10.369576+0000","last_became_active":"2023-09-23T21:08:54.699118+0000","last_became_peered":"2023-09-23T21:08:54.699118+0000","last_unstale":"2023-09-23T21:09:10.369576+0000","last_undegraded":"2023-09-23T21:09:10.369576+0000","last_fullsized":"2023-09-23T21:09:10.369576+0000","mapping_epoch":388,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":389,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:48:03.900670+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.6","version":"400'12158","reported_seq":12236,"reported_epoch":400,"state":"active+clean","last_fresh":"2023-09-23T21:09:16.034015+0000","last_change":"2023-09-23T21:06:49.429119+0000","last_active":"2023-09-23T21:09:16.034015+0000","last_peered":"2023-09-23T21:09:16.034015+0000","last_clean":"2023-09-23T21:09:16.034015+0000","last_became_active":"2023-09-23T21:06:49.423016+0000","last_became_peered":"2023-09-23T21:06:49.423016+0000","last_unstale":"2023-09-23T21:09:16.034015+0000","last_undegraded":"2023-09-23T21:09:16.034015+0000","last_fullsized":"2023-09-23T21:09:16.034015+0000","mapping_epoch":359,"log_start":"361'5395","ondisk_log_start":"361'5395","created":346,"last_epoch_clean":360,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:05:47.261750+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:05:47.261750+0000","last_clean_scrub_stamp":"2023-09-23T21:05:47.261750+0000","objects_scrubbed":0,"log_size":6763,"log_dups_size":0,"ondisk_log_size":6763,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:55:25.118949+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":127139840,"num_objects":1940,"num_object_clones":0,"num_object_copies":5820,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1940,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12158,"num_write_kb":451136,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,1],"acting":[2,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.e","version":"79'161","reported_seq":431,"reported_epoch":400,"state":"active+clean","last_fresh":"2023-09-23T21:09:11.398852+0000","last_change":"2023-09-23T21:09:11.398852+0000","last_active":"2023-09-23T21:09:11.398852+0000","last_peered":"2023-09-23T21:09:11.398852+0000","last_clean":"2023-09-23T21:09:11.398852+0000","last_became_active":"2023-09-23T21:09:11.398440+0000","last_became_peered":"2023-09-23T21:09:11.398440+0000","last_unstale":"2023-09-23T21:09:11.398852+0000","last_undegraded":"2023-09-23T21:09:11.398852+0000","last_fullsized":"2023-09-23T21:09:11.398852+0000","mapping_epoch":399,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":400,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:34.132660+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:34.132660+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,1],"acting":[3,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":165,"reported_epoch":399,"state":"active+clean","last_fresh":"2023-09-23T21:09:10.371509+0000","last_change":"2023-09-23T21:06:49.423445+0000","last_active":"2023-09-23T21:09:10.371509+0000","last_peered":"2023-09-23T21:09:10.371509+0000","last_clean":"2023-09-23T21:09:10.371509+0000","last_became_active":"2023-09-23T21:06:49.423169+0000","last_became_peered":"2023-09-23T21:06:49.423169+0000","last_unstale":"2023-09-23T21:09:10.371509+0000","last_undegraded":"2023-09-23T21:09:10.371509+0000","last_fullsized":"2023-09-23T21:09:10.371509+0000","mapping_epoch":359,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":360,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:47:52.999823+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":392,"reported_epoch":399,"state":"active+clean","last_fresh":"2023-09-23T21:09:10.365515+0000","last_change":"2023-09-23T20:54:13.928715+0000","last_active":"2023-09-23T21:09:10.365515+0000","last_peered":"2023-09-23T21:09:10.365515+0000","last_clean":"2023-09-23T21:09:10.365515+0000","last_became_active":"2023-09-23T20:54:13.928200+0000","last_became_peered":"2023-09-23T20:54:13.928200+0000","last_unstale":"2023-09-23T21:09:10.365515+0000","last_undegraded":"2023-09-23T21:09:10.365515+0000","last_fullsized":"2023-09-23T21:09:10.365515+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:41:16.927513+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.1","version":"400'12231","reported_seq":12317,"reported_epoch":400,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-23T21:09:14.023709+0000","last_change":"2023-09-23T21:09:08.555822+0000","last_active":"2023-09-23T21:09:14.023709+0000","last_peered":"2023-09-23T21:09:14.023709+0000","last_clean":"2023-09-23T21:08:42.471417+0000","last_became_active":"2023-09-23T21:09:08.516782+0000","last_became_peered":"2023-09-23T21:09:08.516782+0000","last_unstale":"2023-09-23T21:09:14.023709+0000","last_undegraded":"2023-09-23T21:09:14.023709+0000","last_fullsized":"2023-09-23T21:09:14.023709+0000","mapping_epoch":396,"log_start":"368'7375","ondisk_log_start":"368'7375","created":346,"last_epoch_clean":347,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:05:47.261750+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:05:47.261750+0000","last_clean_scrub_stamp":"2023-09-23T21:05:47.261750+0000","objects_scrubbed":0,"log_size":4856,"log_dups_size":0,"ondisk_log_size":4856,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:00:48.518312+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":138215424,"num_objects":2109,"num_object_clones":0,"num_object_copies":6327,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":2109,"num_objects_unfound":0,"num_objects_dirty":2109,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12231,"num_write_kb":458880,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,7],"avail_no_missing":["3","6","7"],"object_location_counts":[{"shards":"3,6,7","objects":2109}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.9","version":"79'161","reported_seq":794,"reported_epoch":399,"state":"active+clean","last_fresh":"2023-09-23T21:09:10.380725+0000","last_change":"2023-09-23T21:08:46.395906+0000","last_active":"2023-09-23T21:09:10.380725+0000","last_peered":"2023-09-23T21:09:10.380725+0000","last_clean":"2023-09-23T21:09:10.380725+0000","last_became_active":"2023-09-23T21:08:46.395425+0000","last_became_peered":"2023-09-23T21:08:46.395425+0000","last_unstale":"2023-09-23T21:09:10.380725+0000","last_undegraded":"2023-09-23T21:09:10.380725+0000","last_fullsized":"2023-09-23T21:09:10.380725+0000","mapping_epoch":385,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":386,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:56:46.921739+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:56:46.921739+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:33:29.648468+0000","scrub_duration":0.0071954109999999996,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,5],"acting":[3,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":394,"reported_epoch":399,"state":"active+clean","last_fresh":"2023-09-23T21:09:10.387440+0000","last_change":"2023-09-23T20:54:13.961788+0000","last_active":"2023-09-23T21:09:10.387440+0000","last_peered":"2023-09-23T21:09:10.387440+0000","last_clean":"2023-09-23T21:09:10.387440+0000","last_became_active":"2023-09-23T20:54:13.940937+0000","last_became_peered":"2023-09-23T20:54:13.940937+0000","last_unstale":"2023-09-23T21:09:10.387440+0000","last_undegraded":"2023-09-23T21:09:10.387440+0000","last_fullsized":"2023-09-23T21:09:10.387440+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:11:56.785299+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":84,"reported_epoch":399,"state":"active+clean","last_fresh":"2023-09-23T21:09:10.371412+0000","last_change":"2023-09-23T21:09:08.423918+0000","last_active":"2023-09-23T21:09:10.371412+0000","last_peered":"2023-09-23T21:09:10.371412+0000","last_clean":"2023-09-23T21:09:10.371412+0000","last_became_active":"2023-09-23T21:09:08.423331+0000","last_became_peered":"2023-09-23T21:09:08.423331+0000","last_unstale":"2023-09-23T21:09:10.371412+0000","last_undegraded":"2023-09-23T21:09:10.371412+0000","last_fullsized":"2023-09-23T21:09:10.371412+0000","mapping_epoch":396,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":397,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:18:18.951372+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.0","version":"400'12081","reported_seq":12142,"reported_epoch":400,"state":"active+clean","last_fresh":"2023-09-23T21:09:13.503487+0000","last_change":"2023-09-23T21:05:55.315757+0000","last_active":"2023-09-23T21:09:13.503487+0000","last_peered":"2023-09-23T21:09:13.503487+0000","last_clean":"2023-09-23T21:09:13.503487+0000","last_became_active":"2023-09-23T21:05:55.314884+0000","last_became_peered":"2023-09-23T21:05:55.314884+0000","last_unstale":"2023-09-23T21:09:13.503487+0000","last_undegraded":"2023-09-23T21:09:13.503487+0000","last_fullsized":"2023-09-23T21:09:13.503487+0000","mapping_epoch":350,"log_start":"367'7163","ondisk_log_start":"367'7163","created":346,"last_epoch_clean":351,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:05:47.261750+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:05:47.261750+0000","last_clean_scrub_stamp":"2023-09-23T21:05:47.261750+0000","objects_scrubbed":0,"log_size":4918,"log_dups_size":0,"ondisk_log_size":4918,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:41:19.184012+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":136249344,"num_objects":2079,"num_object_clones":0,"num_object_copies":6237,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2079,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12081,"num_write_kb":453120,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.8","version":"79'161","reported_seq":509,"reported_epoch":400,"state":"active+clean","last_fresh":"2023-09-23T21:09:11.377265+0000","last_change":"2023-09-23T21:06:52.099024+0000","last_active":"2023-09-23T21:09:11.377265+0000","last_peered":"2023-09-23T21:09:11.377265+0000","last_clean":"2023-09-23T21:09:11.377265+0000","last_became_active":"2023-09-23T21:06:49.425012+0000","last_became_peered":"2023-09-23T21:06:49.425012+0000","last_unstale":"2023-09-23T21:09:11.377265+0000","last_undegraded":"2023-09-23T21:09:11.377265+0000","last_fullsized":"2023-09-23T21:09:11.377265+0000","mapping_epoch":359,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":360,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:06:52.098882+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:06:52.098882+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:48:36.414396+0000","scrub_duration":0.0092173299999999993,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":389,"reported_epoch":399,"state":"active+clean","last_fresh":"2023-09-23T21:09:10.367405+0000","last_change":"2023-09-23T21:09:08.417202+0000","last_active":"2023-09-23T21:09:10.367405+0000","last_peered":"2023-09-23T21:09:10.367405+0000","last_clean":"2023-09-23T21:09:10.367405+0000","last_became_active":"2023-09-23T21:09:08.416908+0000","last_became_peered":"2023-09-23T21:09:08.416908+0000","last_unstale":"2023-09-23T21:09:10.367405+0000","last_undegraded":"2023-09-23T21:09:10.367405+0000","last_fullsized":"2023-09-23T21:09:10.367405+0000","mapping_epoch":396,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":397,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:49:02.368080+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":251,"reported_epoch":399,"state":"active+clean","last_fresh":"2023-09-23T21:09:11.098800+0000","last_change":"2023-09-23T21:09:08.410640+0000","last_active":"2023-09-23T21:09:11.098800+0000","last_peered":"2023-09-23T21:09:11.098800+0000","last_clean":"2023-09-23T21:09:11.098800+0000","last_became_active":"2023-09-23T21:09:08.410288+0000","last_became_peered":"2023-09-23T21:09:08.410288+0000","last_unstale":"2023-09-23T21:09:11.098800+0000","last_undegraded":"2023-09-23T21:09:11.098800+0000","last_fullsized":"2023-09-23T21:09:11.098800+0000","mapping_epoch":396,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":397,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:40:08.562402+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"9.3","version":"400'11911","reported_seq":14686,"reported_epoch":400,"state":"active+remapped+backfilling","last_fresh":"2023-09-23T21:09:14.068855+0000","last_change":"2023-09-23T21:09:08.476100+0000","last_active":"2023-09-23T21:09:14.068855+0000","last_peered":"2023-09-23T21:09:14.068855+0000","last_clean":"2023-09-23T21:08:42.472470+0000","last_became_active":"2023-09-23T21:08:44.637503+0000","last_became_peered":"2023-09-23T21:08:44.637503+0000","last_unstale":"2023-09-23T21:09:14.068855+0000","last_undegraded":"2023-09-23T21:09:14.068855+0000","last_fullsized":"2023-09-23T21:09:14.068855+0000","mapping_epoch":383,"log_start":"367'7070","ondisk_log_start":"367'7070","created":346,"last_epoch_clean":355,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:05:47.261750+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:05:47.261750+0000","last_clean_scrub_stamp":"2023-09-23T21:05:47.261750+0000","objects_scrubbed":0,"log_size":4841,"log_dups_size":0,"ondisk_log_size":4841,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:33:26.597138+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":137035776,"num_objects":2091,"num_object_clones":0,"num_object_copies":6273,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":832,"num_objects_unfound":0,"num_objects_dirty":2091,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11911,"num_write_kb":448064,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1341,"num_bytes_recovered":87883776,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,5],"acting":[3,1,7],"avail_no_missing":["3","1","7"],"object_location_counts":[{"shards":"1,3,7","objects":2091}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.b","version":"79'161","reported_seq":730,"reported_epoch":399,"state":"active+clean","last_fresh":"2023-09-23T21:09:10.370580+0000","last_change":"2023-09-23T21:05:55.772369+0000","last_active":"2023-09-23T21:09:10.370580+0000","last_peered":"2023-09-23T21:09:10.370580+0000","last_clean":"2023-09-23T21:09:10.370580+0000","last_became_active":"2023-09-23T21:05:55.316405+0000","last_became_peered":"2023-09-23T21:05:55.316405+0000","last_unstale":"2023-09-23T21:09:10.370580+0000","last_undegraded":"2023-09-23T21:09:10.370580+0000","last_fullsized":"2023-09-23T21:09:10.370580+0000","mapping_epoch":350,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":351,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:05:55.772264+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:05:55.772264+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:14:30.697692+0000","scrub_duration":0.001862507,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,5],"acting":[1,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":276,"reported_epoch":399,"state":"active+clean","last_fresh":"2023-09-23T21:09:11.098690+0000","last_change":"2023-09-23T21:09:08.401941+0000","last_active":"2023-09-23T21:09:11.098690+0000","last_peered":"2023-09-23T21:09:11.098690+0000","last_clean":"2023-09-23T21:09:11.098690+0000","last_became_active":"2023-09-23T21:09:08.401636+0000","last_became_peered":"2023-09-23T21:09:08.401636+0000","last_unstale":"2023-09-23T21:09:11.098690+0000","last_undegraded":"2023-09-23T21:09:11.098690+0000","last_fullsized":"2023-09-23T21:09:11.098690+0000","mapping_epoch":396,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":397,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:12:45.878513+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,5],"acting":[2,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":321,"reported_epoch":399,"state":"active+clean","last_fresh":"2023-09-23T21:09:10.370832+0000","last_change":"2023-09-23T21:08:54.688849+0000","last_active":"2023-09-23T21:09:10.370832+0000","last_peered":"2023-09-23T21:09:10.370832+0000","last_clean":"2023-09-23T21:09:10.370832+0000","last_became_active":"2023-09-23T21:08:54.688571+0000","last_became_peered":"2023-09-23T21:08:54.688571+0000","last_unstale":"2023-09-23T21:09:10.370832+0000","last_undegraded":"2023-09-23T21:09:10.370832+0000","last_fullsized":"2023-09-23T21:09:10.370832+0000","mapping_epoch":388,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":389,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:33:07.896205+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.2","version":"400'12037","reported_seq":12107,"reported_epoch":400,"state":"active+clean","last_fresh":"2023-09-23T21:09:13.991318+0000","last_change":"2023-09-23T21:05:55.325629+0000","last_active":"2023-09-23T21:09:13.991318+0000","last_peered":"2023-09-23T21:09:13.991318+0000","last_clean":"2023-09-23T21:09:13.991318+0000","last_became_active":"2023-09-23T21:05:55.325108+0000","last_became_peered":"2023-09-23T21:05:55.325108+0000","last_unstale":"2023-09-23T21:09:13.991318+0000","last_undegraded":"2023-09-23T21:09:13.991318+0000","last_fullsized":"2023-09-23T21:09:13.991318+0000","mapping_epoch":350,"log_start":"367'7174","ondisk_log_start":"367'7174","created":346,"last_epoch_clean":351,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:05:47.261750+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:05:47.261750+0000","last_clean_scrub_stamp":"2023-09-23T21:05:47.261750+0000","objects_scrubbed":0,"log_size":4863,"log_dups_size":0,"ondisk_log_size":4863,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:50:06.272050+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":137035776,"num_objects":2091,"num_object_clones":0,"num_object_copies":6273,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2091,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12037,"num_write_kb":452096,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.a","version":"79'161","reported_seq":500,"reported_epoch":399,"state":"active+clean","last_fresh":"2023-09-23T21:09:10.361310+0000","last_change":"2023-09-23T21:06:51.075617+0000","last_active":"2023-09-23T21:09:10.361310+0000","last_peered":"2023-09-23T21:09:10.361310+0000","last_clean":"2023-09-23T21:09:10.361310+0000","last_became_active":"2023-09-23T21:06:49.423291+0000","last_became_peered":"2023-09-23T21:06:49.423291+0000","last_unstale":"2023-09-23T21:09:10.361310+0000","last_undegraded":"2023-09-23T21:09:10.361310+0000","last_fullsized":"2023-09-23T21:09:10.361310+0000","mapping_epoch":359,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":360,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:06:51.075448+0000","last_deep_scrub":"79'161","last_deep_scrub_stamp":"2023-09-23T21:05:32.630292+0000","last_clean_scrub_stamp":"2023-09-23T21:06:51.075448+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:35:01.883490+0000","scrub_duration":0.0054793890000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":226,"reported_epoch":399,"state":"active+clean","last_fresh":"2023-09-23T21:09:10.361467+0000","last_change":"2023-09-23T21:09:08.424026+0000","last_active":"2023-09-23T21:09:10.361467+0000","last_peered":"2023-09-23T21:09:10.361467+0000","last_clean":"2023-09-23T21:09:10.361467+0000","last_became_active":"2023-09-23T21:09:08.423098+0000","last_became_peered":"2023-09-23T21:09:08.423098+0000","last_unstale":"2023-09-23T21:09:10.361467+0000","last_undegraded":"2023-09-23T21:09:10.361467+0000","last_fullsized":"2023-09-23T21:09:10.361467+0000","mapping_epoch":396,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":397,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:52:41.551514+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,5],"acting":[3,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":382,"reported_epoch":399,"state":"active+clean","last_fresh":"2023-09-23T21:09:10.375993+0000","last_change":"2023-09-23T21:09:08.432754+0000","last_active":"2023-09-23T21:09:10.375993+0000","last_peered":"2023-09-23T21:09:10.375993+0000","last_clean":"2023-09-23T21:09:10.375993+0000","last_became_active":"2023-09-23T21:09:08.432368+0000","last_became_peered":"2023-09-23T21:09:08.432368+0000","last_unstale":"2023-09-23T21:09:10.375993+0000","last_undegraded":"2023-09-23T21:09:10.375993+0000","last_fullsized":"2023-09-23T21:09:10.375993+0000","mapping_epoch":396,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":397,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:36:05.828847+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,5],"acting":[1,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.f","version":"400'11914","reported_seq":11983,"reported_epoch":400,"state":"active+clean","last_fresh":"2023-09-23T21:09:12.223530+0000","last_change":"2023-09-23T21:05:55.325083+0000","last_active":"2023-09-23T21:09:12.223530+0000","last_peered":"2023-09-23T21:09:12.223530+0000","last_clean":"2023-09-23T21:09:12.223530+0000","last_became_active":"2023-09-23T21:05:55.324768+0000","last_became_peered":"2023-09-23T21:05:55.324768+0000","last_unstale":"2023-09-23T21:09:12.223530+0000","last_undegraded":"2023-09-23T21:09:12.223530+0000","last_fullsized":"2023-09-23T21:09:12.223530+0000","mapping_epoch":350,"log_start":"366'6510","ondisk_log_start":"366'6510","created":346,"last_epoch_clean":351,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:05:47.261750+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:05:47.261750+0000","last_clean_scrub_stamp":"2023-09-23T21:05:47.261750+0000","objects_scrubbed":0,"log_size":5404,"log_dups_size":0,"ondisk_log_size":5404,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:25:42.316668+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":146931712,"num_objects":2242,"num_object_clones":0,"num_object_copies":6726,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2242,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11914,"num_write_kb":452992,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.7","version":"79'161","reported_seq":725,"reported_epoch":399,"state":"active+clean","last_fresh":"2023-09-23T21:09:10.367923+0000","last_change":"2023-09-23T21:08:46.392238+0000","last_active":"2023-09-23T21:09:10.367923+0000","last_peered":"2023-09-23T21:09:10.367923+0000","last_clean":"2023-09-23T21:09:10.367923+0000","last_became_active":"2023-09-23T21:08:46.391848+0000","last_became_peered":"2023-09-23T21:08:46.391848+0000","last_unstale":"2023-09-23T21:09:10.367923+0000","last_undegraded":"2023-09-23T21:09:10.367923+0000","last_fullsized":"2023-09-23T21:09:10.367923+0000","mapping_epoch":385,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":386,"parent":"0.0","parent_split_bits":3,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:25.916867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:25.916867+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:14:19.787602+0000","scrub_duration":0.001665323,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,5],"acting":[6,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":386,"reported_epoch":399,"state":"active+clean","last_fresh":"2023-09-23T21:09:10.369588+0000","last_change":"2023-09-23T21:08:43.506408+0000","last_active":"2023-09-23T21:09:10.369588+0000","last_peered":"2023-09-23T21:09:10.369588+0000","last_clean":"2023-09-23T21:09:10.369588+0000","last_became_active":"2023-09-23T21:08:43.506156+0000","last_became_peered":"2023-09-23T21:08:43.506156+0000","last_unstale":"2023-09-23T21:09:10.369588+0000","last_undegraded":"2023-09-23T21:09:10.369588+0000","last_fullsized":"2023-09-23T21:09:10.369588+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:31:12.790561+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,2],"acting":[1,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":77,"reported_epoch":399,"state":"active+clean","last_fresh":"2023-09-23T21:09:10.365315+0000","last_change":"2023-09-23T21:05:04.326889+0000","last_active":"2023-09-23T21:09:10.365315+0000","last_peered":"2023-09-23T21:09:10.365315+0000","last_clean":"2023-09-23T21:09:10.365315+0000","last_became_active":"2023-09-23T21:05:04.326032+0000","last_became_peered":"2023-09-23T21:05:04.326032+0000","last_unstale":"2023-09-23T21:09:10.365315+0000","last_undegraded":"2023-09-23T21:09:10.365315+0000","last_fullsized":"2023-09-23T21:09:10.365315+0000","mapping_epoch":332,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:00:17.507889+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":399,"reported_epoch":399,"state":"active+clean","last_fresh":"2023-09-23T21:09:10.369713+0000","last_change":"2023-09-23T21:09:08.438066+0000","last_active":"2023-09-23T21:09:10.369713+0000","last_peered":"2023-09-23T21:09:10.369713+0000","last_clean":"2023-09-23T21:09:10.369713+0000","last_became_active":"2023-09-23T21:09:08.437786+0000","last_became_peered":"2023-09-23T21:09:08.437786+0000","last_unstale":"2023-09-23T21:09:10.369713+0000","last_undegraded":"2023-09-23T21:09:10.369713+0000","last_fullsized":"2023-09-23T21:09:10.369713+0000","mapping_epoch":396,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":397,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:49:53.623831+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.9","version":"400'12090","reported_seq":13919,"reported_epoch":400,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-23T21:09:12.323208+0000","last_change":"2023-09-23T21:09:08.417907+0000","last_active":"2023-09-23T21:09:12.323208+0000","last_peered":"2023-09-23T21:09:12.323208+0000","last_clean":"2023-09-23T21:08:42.489035+0000","last_became_active":"2023-09-23T21:08:44.759670+0000","last_became_peered":"2023-09-23T21:08:44.759670+0000","last_unstale":"2023-09-23T21:09:12.323208+0000","last_undegraded":"2023-09-23T21:09:12.323208+0000","last_fullsized":"2023-09-23T21:09:12.323208+0000","mapping_epoch":383,"log_start":"366'6583","ondisk_log_start":"366'6583","created":346,"last_epoch_clean":347,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:05:47.261750+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:05:47.261750+0000","last_clean_scrub_stamp":"2023-09-23T21:05:47.261750+0000","objects_scrubbed":0,"log_size":5507,"log_dups_size":0,"ondisk_log_size":5507,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:31:39.529816+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":149291008,"num_objects":2278,"num_object_clones":0,"num_object_copies":6834,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1468,"num_objects_unfound":0,"num_objects_dirty":2278,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12090,"num_write_kb":459776,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":876,"num_bytes_recovered":57409536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,7],"avail_no_missing":["0","5","7"],"object_location_counts":[{"shards":"0,5,7","objects":2278}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1","version":"79'161","reported_seq":508,"reported_epoch":399,"state":"active+clean","last_fresh":"2023-09-23T21:09:10.367277+0000","last_change":"2023-09-23T21:06:52.576036+0000","last_active":"2023-09-23T21:09:10.367277+0000","last_peered":"2023-09-23T21:09:10.367277+0000","last_clean":"2023-09-23T21:09:10.367277+0000","last_became_active":"2023-09-23T21:06:49.422806+0000","last_became_peered":"2023-09-23T21:06:49.422806+0000","last_unstale":"2023-09-23T21:09:10.367277+0000","last_undegraded":"2023-09-23T21:09:10.367277+0000","last_fullsized":"2023-09-23T21:09:10.367277+0000","mapping_epoch":359,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":360,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:06:52.575900+0000","last_deep_scrub":"79'161","last_deep_scrub_stamp":"2023-09-23T21:05:30.652189+0000","last_clean_scrub_stamp":"2023-09-23T21:06:52.575900+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:59:24.733331+0000","scrub_duration":0.017723466,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":169,"reported_epoch":399,"state":"active+clean","last_fresh":"2023-09-23T21:09:10.369309+0000","last_change":"2023-09-23T21:08:54.699879+0000","last_active":"2023-09-23T21:09:10.369309+0000","last_peered":"2023-09-23T21:09:10.369309+0000","last_clean":"2023-09-23T21:09:10.369309+0000","last_became_active":"2023-09-23T21:08:54.699558+0000","last_became_peered":"2023-09-23T21:08:54.699558+0000","last_unstale":"2023-09-23T21:09:10.369309+0000","last_undegraded":"2023-09-23T21:09:10.369309+0000","last_fullsized":"2023-09-23T21:09:10.369309+0000","mapping_epoch":388,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":389,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:52:11.177493+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":294,"reported_epoch":399,"state":"active+clean","last_fresh":"2023-09-23T21:09:10.370459+0000","last_change":"2023-09-23T20:55:35.949000+0000","last_active":"2023-09-23T21:09:10.370459+0000","last_peered":"2023-09-23T21:09:10.370459+0000","last_clean":"2023-09-23T21:09:10.370459+0000","last_became_active":"2023-09-23T20:55:35.948410+0000","last_became_peered":"2023-09-23T20:55:35.948410+0000","last_unstale":"2023-09-23T21:09:10.370459+0000","last_undegraded":"2023-09-23T21:09:10.370459+0000","last_fullsized":"2023-09-23T21:09:10.370459+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:26:50.477289+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":337,"reported_epoch":399,"state":"active+clean","last_fresh":"2023-09-23T21:09:10.369668+0000","last_change":"2023-09-23T21:05:55.320314+0000","last_active":"2023-09-23T21:09:10.369668+0000","last_peered":"2023-09-23T21:09:10.369668+0000","last_clean":"2023-09-23T21:09:10.369668+0000","last_became_active":"2023-09-23T21:05:55.319935+0000","last_became_peered":"2023-09-23T21:05:55.319935+0000","last_unstale":"2023-09-23T21:09:10.369668+0000","last_undegraded":"2023-09-23T21:09:10.369668+0000","last_fullsized":"2023-09-23T21:09:10.369668+0000","mapping_epoch":350,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":351,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:52:28.754534+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.a","version":"400'11741","reported_seq":17258,"reported_epoch":400,"state":"active+clean","last_fresh":"2023-09-23T21:09:12.119153+0000","last_change":"2023-09-23T21:09:08.440030+0000","last_active":"2023-09-23T21:09:12.119153+0000","last_peered":"2023-09-23T21:09:12.119153+0000","last_clean":"2023-09-23T21:09:12.119153+0000","last_became_active":"2023-09-23T21:09:08.408781+0000","last_became_peered":"2023-09-23T21:09:08.408781+0000","last_unstale":"2023-09-23T21:09:12.119153+0000","last_undegraded":"2023-09-23T21:09:12.119153+0000","last_fullsized":"2023-09-23T21:09:12.119153+0000","mapping_epoch":396,"log_start":"366'6371","ondisk_log_start":"366'6371","created":346,"last_epoch_clean":397,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:05:47.261750+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:05:47.261750+0000","last_clean_scrub_stamp":"2023-09-23T21:05:47.261750+0000","objects_scrubbed":0,"log_size":5370,"log_dups_size":0,"ondisk_log_size":5370,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:56:53.447879+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":147259392,"num_objects":2247,"num_object_clones":0,"num_object_copies":6741,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2247,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11741,"num_write_kb":447616,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2711,"num_bytes_recovered":177668096,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.2","version":"79'161","reported_seq":668,"reported_epoch":400,"state":"active+clean","last_fresh":"2023-09-23T21:09:11.387828+0000","last_change":"2023-09-23T21:09:11.387828+0000","last_active":"2023-09-23T21:09:11.387828+0000","last_peered":"2023-09-23T21:09:11.387828+0000","last_clean":"2023-09-23T21:09:11.387828+0000","last_became_active":"2023-09-23T21:09:11.387486+0000","last_became_peered":"2023-09-23T21:09:11.387486+0000","last_unstale":"2023-09-23T21:09:11.387828+0000","last_undegraded":"2023-09-23T21:09:11.387828+0000","last_fullsized":"2023-09-23T21:09:11.387828+0000","mapping_epoch":399,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":400,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:24.802451+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:24.802451+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:51:42.716025+0000","scrub_duration":0.0090861250000000005,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,5],"acting":[0,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":390,"reported_epoch":399,"state":"active+clean","last_fresh":"2023-09-23T21:09:10.362867+0000","last_change":"2023-09-23T20:51:10.838054+0000","last_active":"2023-09-23T21:09:10.362867+0000","last_peered":"2023-09-23T21:09:10.362867+0000","last_clean":"2023-09-23T21:09:10.362867+0000","last_became_active":"2023-09-23T20:51:10.837750+0000","last_became_peered":"2023-09-23T20:51:10.837750+0000","last_unstale":"2023-09-23T21:09:10.362867+0000","last_undegraded":"2023-09-23T21:09:10.362867+0000","last_fullsized":"2023-09-23T21:09:10.362867+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:15:51.648085+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":363,"reported_epoch":399,"state":"active+clean","last_fresh":"2023-09-23T21:09:10.370068+0000","last_change":"2023-09-23T21:05:55.315770+0000","last_active":"2023-09-23T21:09:10.370068+0000","last_peered":"2023-09-23T21:09:10.370068+0000","last_clean":"2023-09-23T21:09:10.370068+0000","last_became_active":"2023-09-23T21:05:55.315290+0000","last_became_peered":"2023-09-23T21:05:55.315290+0000","last_unstale":"2023-09-23T21:09:10.370068+0000","last_undegraded":"2023-09-23T21:09:10.370068+0000","last_fullsized":"2023-09-23T21:09:10.370068+0000","mapping_epoch":350,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":351,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:27:40.430262+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":404,"reported_epoch":399,"state":"active+clean","last_fresh":"2023-09-23T21:09:10.376148+0000","last_change":"2023-09-23T21:05:55.318863+0000","last_active":"2023-09-23T21:09:10.376148+0000","last_peered":"2023-09-23T21:09:10.376148+0000","last_clean":"2023-09-23T21:09:10.376148+0000","last_became_active":"2023-09-23T21:05:55.318489+0000","last_became_peered":"2023-09-23T21:05:55.318489+0000","last_unstale":"2023-09-23T21:09:10.376148+0000","last_undegraded":"2023-09-23T21:09:10.376148+0000","last_fullsized":"2023-09-23T21:09:10.376148+0000","mapping_epoch":350,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":351,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:45:05.912661+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.8","version":"400'11920","reported_seq":11981,"reported_epoch":400,"state":"active+clean","last_fresh":"2023-09-23T21:09:14.066932+0000","last_change":"2023-09-23T21:05:48.286303+0000","last_active":"2023-09-23T21:09:14.066932+0000","last_peered":"2023-09-23T21:09:14.066932+0000","last_clean":"2023-09-23T21:09:14.066932+0000","last_became_active":"2023-09-23T21:05:48.286008+0000","last_became_peered":"2023-09-23T21:05:48.286008+0000","last_unstale":"2023-09-23T21:09:14.066932+0000","last_undegraded":"2023-09-23T21:09:14.066932+0000","last_fullsized":"2023-09-23T21:09:14.066932+0000","mapping_epoch":346,"log_start":"367'7072","ondisk_log_start":"367'7072","created":346,"last_epoch_clean":347,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:05:47.261750+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:05:47.261750+0000","last_clean_scrub_stamp":"2023-09-23T21:05:47.261750+0000","objects_scrubbed":0,"log_size":4848,"log_dups_size":0,"ondisk_log_size":4848,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:42:05.759917+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":138149888,"num_objects":2108,"num_object_clones":0,"num_object_copies":6324,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2108,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11920,"num_write_kb":448896,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":174,"reported_epoch":399,"state":"active+clean","last_fresh":"2023-09-23T21:09:10.361453+0000","last_change":"2023-09-23T21:08:43.492449+0000","last_active":"2023-09-23T21:09:10.361453+0000","last_peered":"2023-09-23T21:09:10.361453+0000","last_clean":"2023-09-23T21:09:10.361453+0000","last_became_active":"2023-09-23T21:08:43.492116+0000","last_became_peered":"2023-09-23T21:08:43.492116+0000","last_unstale":"2023-09-23T21:09:10.361453+0000","last_undegraded":"2023-09-23T21:09:10.361453+0000","last_fullsized":"2023-09-23T21:09:10.361453+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:04:30.288981+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"79'161","reported_seq":652,"reported_epoch":399,"state":"active+clean","last_fresh":"2023-09-23T21:09:10.364202+0000","last_change":"2023-09-23T21:08:46.407789+0000","last_active":"2023-09-23T21:09:10.364202+0000","last_peered":"2023-09-23T21:09:10.364202+0000","last_clean":"2023-09-23T21:09:10.364202+0000","last_became_active":"2023-09-23T21:08:46.407378+0000","last_became_peered":"2023-09-23T21:08:46.407378+0000","last_unstale":"2023-09-23T21:09:10.364202+0000","last_undegraded":"2023-09-23T21:09:10.364202+0000","last_fullsized":"2023-09-23T21:09:10.364202+0000","mapping_epoch":385,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":386,"parent":"0.0","parent_split_bits":0,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:20.738273+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:20.738273+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:03:45.506154+0000","scrub_duration":0.0088336339999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":400,"reported_epoch":399,"state":"active+clean","last_fresh":"2023-09-23T21:09:10.371025+0000","last_change":"2023-09-23T21:05:04.310148+0000","last_active":"2023-09-23T21:09:10.371025+0000","last_peered":"2023-09-23T21:09:10.371025+0000","last_clean":"2023-09-23T21:09:10.371025+0000","last_became_active":"2023-09-23T21:05:04.309869+0000","last_became_peered":"2023-09-23T21:05:04.309869+0000","last_unstale":"2023-09-23T21:09:10.371025+0000","last_undegraded":"2023-09-23T21:09:10.371025+0000","last_fullsized":"2023-09-23T21:09:10.371025+0000","mapping_epoch":332,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:12:55.522835+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":347,"reported_epoch":400,"state":"active+clean","last_fresh":"2023-09-23T21:09:11.383446+0000","last_change":"2023-09-23T21:09:08.423506+0000","last_active":"2023-09-23T21:09:11.383446+0000","last_peered":"2023-09-23T21:09:11.383446+0000","last_clean":"2023-09-23T21:09:11.383446+0000","last_became_active":"2023-09-23T21:09:08.423187+0000","last_became_peered":"2023-09-23T21:09:08.423187+0000","last_unstale":"2023-09-23T21:09:11.383446+0000","last_undegraded":"2023-09-23T21:09:11.383446+0000","last_fullsized":"2023-09-23T21:09:11.383446+0000","mapping_epoch":396,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":397,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:41:49.562254+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,5],"acting":[0,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.b","version":"400'11834","reported_seq":11898,"reported_epoch":400,"state":"active+clean","last_fresh":"2023-09-23T21:09:12.297206+0000","last_change":"2023-09-23T21:05:48.290142+0000","last_active":"2023-09-23T21:09:12.297206+0000","last_peered":"2023-09-23T21:09:12.297206+0000","last_clean":"2023-09-23T21:09:12.297206+0000","last_became_active":"2023-09-23T21:05:48.289861+0000","last_became_peered":"2023-09-23T21:05:48.289861+0000","last_unstale":"2023-09-23T21:09:12.297206+0000","last_undegraded":"2023-09-23T21:09:12.297206+0000","last_fullsized":"2023-09-23T21:09:12.297206+0000","mapping_epoch":346,"log_start":"365'6333","ondisk_log_start":"365'6333","created":346,"last_epoch_clean":347,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:05:47.261750+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:05:47.261750+0000","last_clean_scrub_stamp":"2023-09-23T21:05:47.261750+0000","objects_scrubbed":0,"log_size":5501,"log_dups_size":0,"ondisk_log_size":5501,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:36:39.064858+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":143392768,"num_objects":2188,"num_object_clones":0,"num_object_copies":6564,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2188,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11834,"num_write_kb":448704,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.3","version":"79'161","reported_seq":677,"reported_epoch":400,"state":"active+clean","last_fresh":"2023-09-23T21:09:11.396811+0000","last_change":"2023-09-23T21:09:11.396811+0000","last_active":"2023-09-23T21:09:11.396811+0000","last_peered":"2023-09-23T21:09:11.396811+0000","last_clean":"2023-09-23T21:09:11.396811+0000","last_became_active":"2023-09-23T21:09:11.396462+0000","last_became_peered":"2023-09-23T21:09:11.396462+0000","last_unstale":"2023-09-23T21:09:11.396811+0000","last_undegraded":"2023-09-23T21:09:11.396811+0000","last_fullsized":"2023-09-23T21:09:11.396811+0000","mapping_epoch":399,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":400,"parent":"0.0","parent_split_bits":3,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:05:57.832302+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:05:57.832302+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:29:03.783133+0000","scrub_duration":0.010206847999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":273,"reported_epoch":399,"state":"active+clean","last_fresh":"2023-09-23T21:09:10.370140+0000","last_change":"2023-09-23T21:08:43.507176+0000","last_active":"2023-09-23T21:09:10.370140+0000","last_peered":"2023-09-23T21:09:10.370140+0000","last_clean":"2023-09-23T21:09:10.370140+0000","last_became_active":"2023-09-23T21:08:43.506832+0000","last_became_peered":"2023-09-23T21:08:43.506832+0000","last_unstale":"2023-09-23T21:09:10.370140+0000","last_undegraded":"2023-09-23T21:09:10.370140+0000","last_fullsized":"2023-09-23T21:09:10.370140+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:24:09.274005+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":407,"reported_epoch":399,"state":"active+clean","last_fresh":"2023-09-23T21:09:10.367444+0000","last_change":"2023-09-23T21:09:08.424118+0000","last_active":"2023-09-23T21:09:10.367444+0000","last_peered":"2023-09-23T21:09:10.367444+0000","last_clean":"2023-09-23T21:09:10.367444+0000","last_became_active":"2023-09-23T21:09:08.423805+0000","last_became_peered":"2023-09-23T21:09:08.423805+0000","last_unstale":"2023-09-23T21:09:10.367444+0000","last_undegraded":"2023-09-23T21:09:10.367444+0000","last_fullsized":"2023-09-23T21:09:10.367444+0000","mapping_epoch":396,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":397,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:47:06.890350+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,5],"acting":[0,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":392,"reported_epoch":399,"state":"active+clean","last_fresh":"2023-09-23T21:09:10.363523+0000","last_change":"2023-09-23T21:05:55.320840+0000","last_active":"2023-09-23T21:09:10.363523+0000","last_peered":"2023-09-23T21:09:10.363523+0000","last_clean":"2023-09-23T21:09:10.363523+0000","last_became_active":"2023-09-23T21:05:55.320280+0000","last_became_peered":"2023-09-23T21:05:55.320280+0000","last_unstale":"2023-09-23T21:09:10.363523+0000","last_undegraded":"2023-09-23T21:09:10.363523+0000","last_fullsized":"2023-09-23T21:09:10.363523+0000","mapping_epoch":350,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":351,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:45:22.368801+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.e","version":"400'12267","reported_seq":17794,"reported_epoch":400,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-23T21:09:13.528123+0000","last_change":"2023-09-23T21:09:09.542336+0000","last_active":"2023-09-23T21:09:13.528123+0000","last_peered":"2023-09-23T21:09:13.528123+0000","last_clean":"2023-09-23T21:08:42.461913+0000","last_became_active":"2023-09-23T21:09:09.474937+0000","last_became_peered":"2023-09-23T21:09:09.474937+0000","last_unstale":"2023-09-23T21:09:13.528123+0000","last_undegraded":"2023-09-23T21:09:13.528123+0000","last_fullsized":"2023-09-23T21:09:13.528123+0000","mapping_epoch":397,"log_start":"367'7336","ondisk_log_start":"367'7336","created":346,"last_epoch_clean":394,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:05:47.261750+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:05:47.261750+0000","last_clean_scrub_stamp":"2023-09-23T21:05:47.261750+0000","objects_scrubbed":0,"log_size":4931,"log_dups_size":0,"ondisk_log_size":4931,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:32:11.417394+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":149618688,"num_objects":2283,"num_object_clones":0,"num_object_copies":6849,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":2283,"num_objects_unfound":0,"num_objects_dirty":2283,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12267,"num_write_kb":465600,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2720,"num_bytes_recovered":178257920,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,5],"acting":[1,0,4],"avail_no_missing":["1","0","4","7"],"object_location_counts":[{"shards":"0,1,4","objects":2283}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"79'161","reported_seq":682,"reported_epoch":399,"state":"active+clean","last_fresh":"2023-09-23T21:09:10.367570+0000","last_change":"2023-09-23T20:57:26.907505+0000","last_active":"2023-09-23T21:09:10.367570+0000","last_peered":"2023-09-23T21:09:10.367570+0000","last_clean":"2023-09-23T21:09:10.367570+0000","last_became_active":"2023-09-23T20:57:08.240321+0000","last_became_peered":"2023-09-23T20:57:08.240321+0000","last_unstale":"2023-09-23T21:09:10.367570+0000","last_undegraded":"2023-09-23T21:09:10.367570+0000","last_fullsized":"2023-09-23T21:09:10.367570+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":3,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:26.907408+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:26.907408+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:34:17.392928+0000","scrub_duration":0.010337038,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":309,"reported_epoch":399,"state":"active+clean","last_fresh":"2023-09-23T21:09:10.365250+0000","last_change":"2023-09-23T21:09:08.420418+0000","last_active":"2023-09-23T21:09:10.365250+0000","last_peered":"2023-09-23T21:09:10.365250+0000","last_clean":"2023-09-23T21:09:10.365250+0000","last_became_active":"2023-09-23T21:09:08.420094+0000","last_became_peered":"2023-09-23T21:09:08.420094+0000","last_unstale":"2023-09-23T21:09:10.365250+0000","last_undegraded":"2023-09-23T21:09:10.365250+0000","last_fullsized":"2023-09-23T21:09:10.365250+0000","mapping_epoch":396,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":397,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:46:59.735031+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,6],"acting":[0,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":387,"reported_epoch":399,"state":"active+clean","last_fresh":"2023-09-23T21:09:10.369757+0000","last_change":"2023-09-23T21:05:55.319513+0000","last_active":"2023-09-23T21:09:10.369757+0000","last_peered":"2023-09-23T21:09:10.369757+0000","last_clean":"2023-09-23T21:09:10.369757+0000","last_became_active":"2023-09-23T21:05:55.319178+0000","last_became_peered":"2023-09-23T21:05:55.319178+0000","last_unstale":"2023-09-23T21:09:10.369757+0000","last_undegraded":"2023-09-23T21:09:10.369757+0000","last_fullsized":"2023-09-23T21:09:10.369757+0000","mapping_epoch":350,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":351,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:47:32.783934+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":392,"reported_epoch":399,"state":"active+clean","last_fresh":"2023-09-23T21:09:10.365757+0000","last_change":"2023-09-23T21:08:43.508269+0000","last_active":"2023-09-23T21:09:10.365757+0000","last_peered":"2023-09-23T21:09:10.365757+0000","last_clean":"2023-09-23T21:09:10.365757+0000","last_became_active":"2023-09-23T21:08:43.507970+0000","last_became_peered":"2023-09-23T21:08:43.507970+0000","last_unstale":"2023-09-23T21:09:10.365757+0000","last_undegraded":"2023-09-23T21:09:10.365757+0000","last_fullsized":"2023-09-23T21:09:10.365757+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:52:15.049815+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,1],"acting":[0,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.d","version":"400'12042","reported_seq":17905,"reported_epoch":400,"state":"active+clean","last_fresh":"2023-09-23T21:09:13.473720+0000","last_change":"2023-09-23T21:09:06.424530+0000","last_active":"2023-09-23T21:09:13.473720+0000","last_peered":"2023-09-23T21:09:13.473720+0000","last_clean":"2023-09-23T21:09:13.473720+0000","last_became_active":"2023-09-23T21:09:06.379456+0000","last_became_peered":"2023-09-23T21:09:06.379456+0000","last_unstale":"2023-09-23T21:09:13.473720+0000","last_undegraded":"2023-09-23T21:09:13.473720+0000","last_fullsized":"2023-09-23T21:09:13.473720+0000","mapping_epoch":394,"log_start":"367'7202","ondisk_log_start":"367'7202","created":346,"last_epoch_clean":395,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:05:47.261750+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:05:47.261750+0000","last_clean_scrub_stamp":"2023-09-23T21:05:47.261750+0000","objects_scrubbed":0,"log_size":4840,"log_dups_size":0,"ondisk_log_size":4840,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:23:21.748619+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":144769048,"num_objects":2210,"num_object_clones":0,"num_object_copies":6630,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2210,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":12042,"num_write_kb":456001,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2889,"num_bytes_recovered":189267992,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"79'161","reported_seq":429,"reported_epoch":400,"state":"active+clean+scrubbing","last_fresh":"2023-09-23T21:09:12.982279+0000","last_change":"2023-09-23T21:09:11.972293+0000","last_active":"2023-09-23T21:09:12.982279+0000","last_peered":"2023-09-23T21:09:12.982279+0000","last_clean":"2023-09-23T21:09:12.982279+0000","last_became_active":"2023-09-23T21:09:11.388460+0000","last_became_peered":"2023-09-23T21:09:11.388460+0000","last_unstale":"2023-09-23T21:09:12.982279+0000","last_undegraded":"2023-09-23T21:09:12.982279+0000","last_fullsized":"2023-09-23T21:09:12.982279+0000","mapping_epoch":399,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":400,"parent":"0.0","parent_split_bits":3,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:30.029396+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:30.029396+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,0],"acting":[2,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":381,"reported_epoch":399,"state":"active+clean","last_fresh":"2023-09-23T21:09:10.381420+0000","last_change":"2023-09-23T21:08:43.535723+0000","last_active":"2023-09-23T21:09:10.381420+0000","last_peered":"2023-09-23T21:09:10.381420+0000","last_clean":"2023-09-23T21:09:10.381420+0000","last_became_active":"2023-09-23T21:08:43.505805+0000","last_became_peered":"2023-09-23T21:08:43.505805+0000","last_unstale":"2023-09-23T21:09:10.381420+0000","last_undegraded":"2023-09-23T21:09:10.381420+0000","last_fullsized":"2023-09-23T21:09:10.381420+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:44:34.883525+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":374,"reported_epoch":399,"state":"active+clean","last_fresh":"2023-09-23T21:09:10.378860+0000","last_change":"2023-09-23T21:09:08.427533+0000","last_active":"2023-09-23T21:09:10.378860+0000","last_peered":"2023-09-23T21:09:10.378860+0000","last_clean":"2023-09-23T21:09:10.378860+0000","last_became_active":"2023-09-23T21:09:08.427187+0000","last_became_peered":"2023-09-23T21:09:08.427187+0000","last_unstale":"2023-09-23T21:09:10.378860+0000","last_undegraded":"2023-09-23T21:09:10.378860+0000","last_fullsized":"2023-09-23T21:09:10.378860+0000","mapping_epoch":396,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":397,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:24:11.334136+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,3],"acting":[1,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":202,"reported_epoch":399,"state":"active+clean","last_fresh":"2023-09-23T21:09:10.361549+0000","last_change":"2023-09-23T21:08:43.506053+0000","last_active":"2023-09-23T21:09:10.361549+0000","last_peered":"2023-09-23T21:09:10.361549+0000","last_clean":"2023-09-23T21:09:10.361549+0000","last_became_active":"2023-09-23T21:08:43.505716+0000","last_became_peered":"2023-09-23T21:08:43.505716+0000","last_unstale":"2023-09-23T21:09:10.361549+0000","last_undegraded":"2023-09-23T21:09:10.361549+0000","last_fullsized":"2023-09-23T21:09:10.361549+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:57:29.702126+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.c","version":"400'12128","reported_seq":15920,"reported_epoch":400,"state":"active+remapped+backfilling","last_fresh":"2023-09-23T21:09:14.069262+0000","last_change":"2023-09-23T21:09:08.477655+0000","last_active":"2023-09-23T21:09:14.069262+0000","last_peered":"2023-09-23T21:09:14.069262+0000","last_clean":"2023-09-23T21:08:42.493898+0000","last_became_active":"2023-09-23T21:08:44.673436+0000","last_became_peered":"2023-09-23T21:08:44.673436+0000","last_unstale":"2023-09-23T21:09:14.069262+0000","last_undegraded":"2023-09-23T21:09:14.069262+0000","last_fullsized":"2023-09-23T21:09:14.069262+0000","mapping_epoch":383,"log_start":"367'7257","ondisk_log_start":"367'7257","created":346,"last_epoch_clean":347,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:05:47.261750+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:05:47.261750+0000","last_clean_scrub_stamp":"2023-09-23T21:05:47.261750+0000","objects_scrubbed":0,"log_size":4871,"log_dups_size":0,"ondisk_log_size":4871,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:27:38.716434+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":140115968,"num_objects":2138,"num_object_clones":0,"num_object_copies":6414,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":457,"num_objects_unfound":0,"num_objects_dirty":2138,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12128,"num_write_kb":456512,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1857,"num_bytes_recovered":121700352,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,7],"avail_no_missing":["3","4","7"],"object_location_counts":[{"shards":"3,4,7","objects":2138}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.4","version":"79'161","reported_seq":734,"reported_epoch":399,"state":"active+clean","last_fresh":"2023-09-23T21:09:10.371169+0000","last_change":"2023-09-23T21:08:46.398346+0000","last_active":"2023-09-23T21:09:10.371169+0000","last_peered":"2023-09-23T21:09:10.371169+0000","last_clean":"2023-09-23T21:09:10.371169+0000","last_became_active":"2023-09-23T21:08:46.397918+0000","last_became_peered":"2023-09-23T21:08:46.397918+0000","last_unstale":"2023-09-23T21:09:10.371169+0000","last_undegraded":"2023-09-23T21:09:10.371169+0000","last_fullsized":"2023-09-23T21:09:10.371169+0000","mapping_epoch":385,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":386,"parent":"0.0","parent_split_bits":3,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:20.012764+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:20.012764+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:03:01.180190+0000","scrub_duration":0.0045251919999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":359,"reported_epoch":399,"state":"active+clean","last_fresh":"2023-09-23T21:09:10.368082+0000","last_change":"2023-09-23T21:08:43.500587+0000","last_active":"2023-09-23T21:09:10.368082+0000","last_peered":"2023-09-23T21:09:10.368082+0000","last_clean":"2023-09-23T21:09:10.368082+0000","last_became_active":"2023-09-23T21:08:43.500272+0000","last_became_peered":"2023-09-23T21:08:43.500272+0000","last_unstale":"2023-09-23T21:09:10.368082+0000","last_undegraded":"2023-09-23T21:09:10.368082+0000","last_fullsized":"2023-09-23T21:09:10.368082+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:42:06.186343+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,3],"acting":[6,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"70'1","reported_seq":408,"reported_epoch":399,"state":"active+clean","last_fresh":"2023-09-23T21:09:10.363618+0000","last_change":"2023-09-23T21:05:55.320603+0000","last_active":"2023-09-23T21:09:10.363618+0000","last_peered":"2023-09-23T21:09:10.363618+0000","last_clean":"2023-09-23T21:09:10.363618+0000","last_became_active":"2023-09-23T21:05:55.319956+0000","last_became_peered":"2023-09-23T21:05:55.319956+0000","last_unstale":"2023-09-23T21:09:10.363618+0000","last_undegraded":"2023-09-23T21:09:10.363618+0000","last_fullsized":"2023-09-23T21:09:10.363618+0000","mapping_epoch":350,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":351,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:41:03.995212+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":264,"reported_epoch":399,"state":"active+clean","last_fresh":"2023-09-23T21:09:10.365217+0000","last_change":"2023-09-23T21:09:08.405598+0000","last_active":"2023-09-23T21:09:10.365217+0000","last_peered":"2023-09-23T21:09:10.365217+0000","last_clean":"2023-09-23T21:09:10.365217+0000","last_became_active":"2023-09-23T21:09:08.405222+0000","last_became_peered":"2023-09-23T21:09:08.405222+0000","last_unstale":"2023-09-23T21:09:10.365217+0000","last_undegraded":"2023-09-23T21:09:10.365217+0000","last_fullsized":"2023-09-23T21:09:10.365217+0000","mapping_epoch":396,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":397,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:21:55.701363+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,0],"acting":[5,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":375,"reported_epoch":399,"state":"active+clean","last_fresh":"2023-09-23T21:09:10.365674+0000","last_change":"2023-09-23T21:08:43.527566+0000","last_active":"2023-09-23T21:09:10.365674+0000","last_peered":"2023-09-23T21:09:10.365674+0000","last_clean":"2023-09-23T21:09:10.365674+0000","last_became_active":"2023-09-23T21:08:43.527246+0000","last_became_peered":"2023-09-23T21:08:43.527246+0000","last_unstale":"2023-09-23T21:09:10.365674+0000","last_undegraded":"2023-09-23T21:09:10.365674+0000","last_fullsized":"2023-09-23T21:09:10.365674+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:06:28.535063+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":387,"reported_epoch":399,"state":"active+clean","last_fresh":"2023-09-23T21:09:10.362178+0000","last_change":"2023-09-23T21:05:55.313495+0000","last_active":"2023-09-23T21:09:10.362178+0000","last_peered":"2023-09-23T21:09:10.362178+0000","last_clean":"2023-09-23T21:09:10.362178+0000","last_became_active":"2023-09-23T21:05:55.313065+0000","last_became_peered":"2023-09-23T21:05:55.313065+0000","last_unstale":"2023-09-23T21:09:10.362178+0000","last_undegraded":"2023-09-23T21:09:10.362178+0000","last_fullsized":"2023-09-23T21:09:10.362178+0000","mapping_epoch":350,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":351,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:05:22.675197+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":53,"reported_epoch":399,"state":"active+clean","last_fresh":"2023-09-23T21:09:10.369912+0000","last_change":"2023-09-23T21:09:08.454444+0000","last_active":"2023-09-23T21:09:10.369912+0000","last_peered":"2023-09-23T21:09:10.369912+0000","last_clean":"2023-09-23T21:09:10.369912+0000","last_became_active":"2023-09-23T21:09:08.411670+0000","last_became_peered":"2023-09-23T21:09:08.411670+0000","last_unstale":"2023-09-23T21:09:10.369912+0000","last_undegraded":"2023-09-23T21:09:10.369912+0000","last_fullsized":"2023-09-23T21:09:10.369912+0000","mapping_epoch":396,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":397,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:43:04.009432+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,2],"acting":[1,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":251,"reported_epoch":399,"state":"active+clean","last_fresh":"2023-09-23T21:09:10.385115+0000","last_change":"2023-09-23T21:09:08.440185+0000","last_active":"2023-09-23T21:09:10.385115+0000","last_peered":"2023-09-23T21:09:10.385115+0000","last_clean":"2023-09-23T21:09:10.385115+0000","last_became_active":"2023-09-23T21:09:08.439562+0000","last_became_peered":"2023-09-23T21:09:08.439562+0000","last_unstale":"2023-09-23T21:09:10.385115+0000","last_undegraded":"2023-09-23T21:09:10.385115+0000","last_fullsized":"2023-09-23T21:09:10.385115+0000","mapping_epoch":396,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":397,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:39:54.771238+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":174,"reported_epoch":399,"state":"active+clean","last_fresh":"2023-09-23T21:09:10.362856+0000","last_change":"2023-09-23T21:08:54.706324+0000","last_active":"2023-09-23T21:09:10.362856+0000","last_peered":"2023-09-23T21:09:10.362856+0000","last_clean":"2023-09-23T21:09:10.362856+0000","last_became_active":"2023-09-23T21:08:54.681312+0000","last_became_peered":"2023-09-23T21:08:54.681312+0000","last_unstale":"2023-09-23T21:09:10.362856+0000","last_undegraded":"2023-09-23T21:09:10.362856+0000","last_fullsized":"2023-09-23T21:09:10.362856+0000","mapping_epoch":388,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":389,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:08:16.335346+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,1],"acting":[2,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":101,"reported_epoch":399,"state":"active+clean","last_fresh":"2023-09-23T21:09:10.371183+0000","last_change":"2023-09-23T21:09:08.439959+0000","last_active":"2023-09-23T21:09:10.371183+0000","last_peered":"2023-09-23T21:09:10.371183+0000","last_clean":"2023-09-23T21:09:10.371183+0000","last_became_active":"2023-09-23T21:09:08.439676+0000","last_became_peered":"2023-09-23T21:09:08.439676+0000","last_unstale":"2023-09-23T21:09:10.371183+0000","last_undegraded":"2023-09-23T21:09:10.371183+0000","last_fullsized":"2023-09-23T21:09:10.371183+0000","mapping_epoch":396,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":397,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:24:22.416220+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,2],"acting":[3,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":401,"reported_epoch":400,"state":"active+clean","last_fresh":"2023-09-23T21:09:11.389207+0000","last_change":"2023-09-23T21:08:54.686159+0000","last_active":"2023-09-23T21:09:11.389207+0000","last_peered":"2023-09-23T21:09:11.389207+0000","last_clean":"2023-09-23T21:09:11.389207+0000","last_became_active":"2023-09-23T21:08:54.685798+0000","last_became_peered":"2023-09-23T21:08:54.685798+0000","last_unstale":"2023-09-23T21:09:11.389207+0000","last_undegraded":"2023-09-23T21:09:11.389207+0000","last_fullsized":"2023-09-23T21:09:11.389207+0000","mapping_epoch":388,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":389,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:02:10.332347+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,1],"acting":[3,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":390,"reported_epoch":399,"state":"active+clean","last_fresh":"2023-09-23T21:09:10.371583+0000","last_change":"2023-09-23T20:51:10.838334+0000","last_active":"2023-09-23T21:09:10.371583+0000","last_peered":"2023-09-23T21:09:10.371583+0000","last_clean":"2023-09-23T21:09:10.371583+0000","last_became_active":"2023-09-23T20:51:10.838026+0000","last_became_peered":"2023-09-23T20:51:10.838026+0000","last_unstale":"2023-09-23T21:09:10.371583+0000","last_undegraded":"2023-09-23T21:09:10.371583+0000","last_fullsized":"2023-09-23T21:09:10.371583+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:04:37.009457+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":425,"reported_epoch":399,"state":"active+clean","last_fresh":"2023-09-23T21:09:10.364524+0000","last_change":"2023-09-23T21:05:04.325268+0000","last_active":"2023-09-23T21:09:10.364524+0000","last_peered":"2023-09-23T21:09:10.364524+0000","last_clean":"2023-09-23T21:09:10.364524+0000","last_became_active":"2023-09-23T21:05:04.303430+0000","last_became_peered":"2023-09-23T21:05:04.303430+0000","last_unstale":"2023-09-23T21:09:10.364524+0000","last_undegraded":"2023-09-23T21:09:10.364524+0000","last_fullsized":"2023-09-23T21:09:10.364524+0000","mapping_epoch":332,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:17:44.879586+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":2216,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":14,"reported_epoch":399,"state":"active+clean","last_fresh":"2023-09-23T21:09:10.370223+0000","last_change":"2023-09-23T21:09:08.428128+0000","last_active":"2023-09-23T21:09:10.370223+0000","last_peered":"2023-09-23T21:09:10.370223+0000","last_clean":"2023-09-23T21:09:10.370223+0000","last_became_active":"2023-09-23T21:09:08.427481+0000","last_became_peered":"2023-09-23T21:09:08.427481+0000","last_unstale":"2023-09-23T21:09:10.370223+0000","last_undegraded":"2023-09-23T21:09:10.370223+0000","last_fullsized":"2023-09-23T21:09:10.370223+0000","mapping_epoch":396,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":397,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:46:06.052816+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,2],"acting":[1,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":40,"num_bytes_recovered":7326,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":126976,"data_stored":21462,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5834,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":21,"stat_sum":{"num_bytes":644038,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":44,"num_read_kb":83,"num_write":22,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":161010,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3381,"ondisk_log_size":3381,"up":63,"acting":63,"num_store_stats":8},{"poolid":9,"num_pg":16,"stat_sum":{"num_bytes":2251096088,"num_objects":34350,"num_object_clones":0,"num_object_copies":103050,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":8581,"num_objects_unfound":0,"num_objects_dirty":34350,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":192198,"num_write_kb":7249473,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":15844,"num_bytes_recovered":1038090264,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":7655342080,"data_stored":7655325792,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":82830,"ondisk_log_size":82830,"up":48,"acting":48,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":0,"seq":236223201625,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":6,"up_from":164,"seq":704374636709,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":671084,"kb_used_data":406564,"kb_used_omap":10,"kb_used_meta":264501,"kb_avail":93074068,"statfs":{"total":95995035648,"available":95307845632,"internally_reserved":0,"allocated":416321536,"data_stored":415177038,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":10328,"internal_metadata":270849960},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:09:04 2023","interfaces":[{"interface":"back","average":{"1min":1.359,"5min":1.7929999999999999,"15min":1.6759999999999999},"min":{"1min":0.253,"5min":0.224,"15min":0.20999999999999999},"max":{"1min":6.0800000000000001,"5min":9.9890000000000008,"15min":14.151999999999999},"last":0.251},{"interface":"front","average":{"1min":2.5430000000000001,"5min":1.871,"15min":1.7669999999999999},"min":{"1min":0.27100000000000002,"5min":0.224,"15min":0.20999999999999999},"max":{"1min":10.868,"5min":10.868,"15min":11.023},"last":4.657}]},{"osd":1,"last update":"Sat Sep 23 21:08:56 2023","interfaces":[{"interface":"back","average":{"1min":2.802,"5min":2.0590000000000002,"15min":1.7529999999999999},"min":{"1min":0.27100000000000002,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":10.218999999999999,"5min":12.904999999999999,"15min":12.904999999999999},"last":3.4470000000000001},{"interface":"front","average":{"1min":1.5940000000000001,"5min":1.6299999999999999,"15min":1.718},"min":{"1min":0.35099999999999998,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":4.7450000000000001,"5min":6.7569999999999997,"15min":6.7569999999999997},"last":2.625}]},{"osd":2,"last update":"Sat Sep 23 21:09:04 2023","interfaces":[{"interface":"back","average":{"1min":3.581,"5min":1.867,"15min":2.0760000000000001},"min":{"1min":0.36699999999999999,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":23.954999999999998,"5min":23.954999999999998,"15min":23.954999999999998},"last":5.0259999999999998},{"interface":"front","average":{"1min":2.4260000000000002,"5min":1.8160000000000001,"15min":1.98},"min":{"1min":0.36899999999999999,"5min":0.27300000000000002,"15min":0.247},"max":{"1min":6.9729999999999999,"5min":13.385999999999999,"15min":20.195},"last":1.9319999999999999}]},{"osd":3,"last update":"Sat Sep 23 21:09:04 2023","interfaces":[{"interface":"back","average":{"1min":1.889,"5min":2.109,"15min":2.0619999999999998},"min":{"1min":0.34300000000000003,"5min":0.309,"15min":0.28699999999999998},"max":{"1min":6.7889999999999997,"5min":20.625,"15min":43.085999999999999},"last":6.3479999999999999},{"interface":"front","average":{"1min":2.3159999999999998,"5min":2.2200000000000002,"15min":2.1219999999999999},"min":{"1min":0.38400000000000001,"5min":0.30499999999999999,"15min":0.28599999999999998},"max":{"1min":7.6189999999999998,"5min":12.513,"15min":42.405000000000001},"last":6.6639999999999997}]},{"osd":4,"last update":"Sat Sep 23 21:09:04 2023","interfaces":[{"interface":"back","average":{"1min":1.1220000000000001,"5min":0.96599999999999997,"15min":0.92000000000000004},"min":{"1min":0.28299999999999997,"5min":0.25700000000000001,"15min":0.23000000000000001},"max":{"1min":7.2759999999999998,"5min":7.2759999999999998,"15min":9.0079999999999991},"last":0.28299999999999997},{"interface":"front","average":{"1min":0.81000000000000005,"5min":0.92800000000000005,"15min":0.74199999999999999},"min":{"1min":0.29399999999999998,"5min":0.28799999999999998,"15min":0.25800000000000001},"max":{"1min":3.6190000000000002,"5min":6.4050000000000002,"15min":6.4050000000000002},"last":0.31}]},{"osd":5,"last update":"Sat Sep 23 21:09:04 2023","interfaces":[{"interface":"back","average":{"1min":1.2769999999999999,"5min":1.2310000000000001,"15min":0.97799999999999998},"min":{"1min":0.309,"5min":0.27800000000000002,"15min":0.27300000000000002},"max":{"1min":6.9370000000000003,"5min":7.9710000000000001,"15min":7.9710000000000001},"last":0.32700000000000001},{"interface":"front","average":{"1min":0.58499999999999996,"5min":1.0149999999999999,"15min":0.93899999999999995},"min":{"1min":0.32100000000000001,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":1.3839999999999999,"5min":10.657,"15min":10.657},"last":0.34999999999999998}]},{"osd":7,"last update":"Sat Sep 23 21:09:04 2023","interfaces":[{"interface":"back","average":{"1min":12.522,"5min":3.488,"15min":1.8109999999999999},"min":{"1min":0.39500000000000002,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":205.95599999999999,"5min":205.95599999999999,"15min":205.95599999999999},"last":0.38300000000000001},{"interface":"front","average":{"1min":13.239000000000001,"5min":3.4830000000000001,"15min":1.6779999999999999},"min":{"1min":0.33000000000000002,"5min":0.28299999999999997,"15min":0.28100000000000003},"max":{"1min":206.05699999999999,"5min":206.05699999999999,"15min":206.05699999999999},"last":0.39900000000000002}]}]},{"osd":1,"up_from":350,"seq":1503238553641,"num_pgs":62,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1536272,"kb_used_data":1304456,"kb_used_omap":10,"kb_used_meta":231669,"kb_avail":92208880,"statfs":{"total":95995035648,"available":94421893120,"internally_reserved":0,"allocated":1335762944,"data_stored":1333955857,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":10498,"internal_metadata":237229822},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:09:01 2023","interfaces":[{"interface":"back","average":{"1min":2.1850000000000001,"5min":2.2770000000000001,"15min":2.0289999999999999},"min":{"1min":0.20499999999999999,"5min":0.20499999999999999,"15min":0.20499999999999999},"max":{"1min":6.9359999999999999,"5min":15.728,"15min":15.728},"last":6.5990000000000002},{"interface":"front","average":{"1min":2.512,"5min":2.7189999999999999,"15min":2.613},"min":{"1min":0.23100000000000001,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":7.8230000000000004,"5min":14.468999999999999,"15min":14.468999999999999},"last":5.484}]},{"osd":2,"last update":"Sat Sep 23 21:09:01 2023","interfaces":[{"interface":"back","average":{"1min":2.2869999999999999,"5min":2.5110000000000001,"15min":2.3969999999999998},"min":{"1min":0.248,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":9.0050000000000008,"5min":10.769,"15min":10.769},"last":3.2330000000000001},{"interface":"front","average":{"1min":1.827,"5min":2.1749999999999998,"15min":2.0640000000000001},"min":{"1min":0.307,"5min":0.214,"15min":0.214},"max":{"1min":5.3719999999999999,"5min":15.683999999999999,"15min":15.683999999999999},"last":9.5429999999999993}]},{"osd":3,"last update":"Sat Sep 23 21:09:01 2023","interfaces":[{"interface":"back","average":{"1min":2.202,"5min":2.734,"15min":2.7429999999999999},"min":{"1min":0.39300000000000002,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":5.0629999999999997,"5min":15.135999999999999,"15min":15.135999999999999},"last":0.55800000000000005},{"interface":"front","average":{"1min":2.1269999999999998,"5min":3.0939999999999999,"15min":3.0739999999999998},"min":{"1min":0.27300000000000002,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":7.7329999999999997,"5min":14.794,"15min":14.794},"last":5.4459999999999997}]},{"osd":4,"last update":"Sat Sep 23 21:09:01 2023","interfaces":[{"interface":"back","average":{"1min":1.9370000000000001,"5min":1.966,"15min":1.8220000000000001},"min":{"1min":0.34999999999999998,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":7.3029999999999999,"5min":10.391,"15min":10.391},"last":0.42699999999999999},{"interface":"front","average":{"1min":2.3620000000000001,"5min":1.8120000000000001,"15min":1.6719999999999999},"min":{"1min":0.33700000000000002,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":7.3780000000000001,"5min":7.3780000000000001,"15min":7.3780000000000001},"last":0.51400000000000001}]},{"osd":5,"last update":"Sat Sep 23 21:09:01 2023","interfaces":[{"interface":"back","average":{"1min":2.706,"5min":2.5630000000000002,"15min":2.5910000000000002},"min":{"1min":0.29799999999999999,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":7.7670000000000003,"5min":8.0899999999999999,"15min":8.0899999999999999},"last":2.6509999999999998},{"interface":"front","average":{"1min":1.6279999999999999,"5min":2.5899999999999999,"15min":2.6360000000000001},"min":{"1min":0.41999999999999998,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":4.226,"5min":10.339,"15min":10.339},"last":3.0150000000000001}]},{"osd":6,"last update":"Sat Sep 23 21:09:01 2023","interfaces":[{"interface":"back","average":{"1min":2.0230000000000001,"5min":2.3010000000000002,"15min":2.327},"min":{"1min":0.38100000000000001,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":7.7960000000000003,"5min":8.1880000000000006,"15min":8.1880000000000006},"last":2.6859999999999999},{"interface":"front","average":{"1min":2.7919999999999998,"5min":2.6589999999999998,"15min":2.5009999999999999},"min":{"1min":0.39200000000000002,"5min":0.39200000000000002,"15min":0.39200000000000002},"max":{"1min":16.494,"5min":16.494,"15min":16.494},"last":0.46999999999999997}]},{"osd":7,"last update":"Sat Sep 23 21:09:01 2023","interfaces":[{"interface":"back","average":{"1min":2.3450000000000002,"5min":2.0840000000000001,"15min":2.056},"min":{"1min":0.44500000000000001,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":5.2130000000000001,"5min":6.5570000000000004,"15min":6.5570000000000004},"last":0.58799999999999997},{"interface":"front","average":{"1min":2.8399999999999999,"5min":2.6019999999999999,"15min":2.6179999999999999},"min":{"1min":0.48199999999999998,"5min":0.41099999999999998,"15min":0.41099999999999998},"max":{"1min":7.9539999999999997,"5min":8.1440000000000001,"15min":8.1440000000000001},"last":2.7189999999999999}]}]},{"osd":0,"up_from":83,"seq":356482285806,"num_pgs":55,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1348324,"kb_used_data":822748,"kb_used_omap":28,"kb_used_meta":525539,"kb_avail":92396828,"statfs":{"total":95995035648,"available":94614351872,"internally_reserved":0,"allocated":842493952,"data_stored":840728886,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":28848,"internal_metadata":538152784},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[5,0,1,1,2],"upper_bound":32},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 21:08:58 2023","interfaces":[{"interface":"back","average":{"1min":2.7250000000000001,"5min":2.0179999999999998,"15min":1.55},"min":{"1min":0.23300000000000001,"5min":0.192,"15min":0.192},"max":{"1min":8.234,"5min":12.994,"15min":12.994},"last":5.6980000000000004},{"interface":"front","average":{"1min":2.8330000000000002,"5min":1.8200000000000001,"15min":1.456},"min":{"1min":0.27600000000000002,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":7.5449999999999999,"5min":14.646000000000001,"15min":14.646000000000001},"last":2.0870000000000002}]},{"osd":2,"last update":"Sat Sep 23 21:08:11 2023","interfaces":[{"interface":"back","average":{"1min":3.0819999999999999,"5min":2.2330000000000001,"15min":2.8839999999999999},"min":{"1min":0.40000000000000002,"5min":0.183,"15min":0.183},"max":{"1min":9.4879999999999995,"5min":13.032,"15min":21.053000000000001},"last":7.8120000000000003},{"interface":"front","average":{"1min":3.2280000000000002,"5min":2.3010000000000002,"15min":3.0550000000000002},"min":{"1min":0.42899999999999999,"5min":0.23499999999999999,"15min":0.21099999999999999},"max":{"1min":14.558999999999999,"5min":14.558999999999999,"15min":21.747},"last":2.169}]},{"osd":3,"last update":"Sat Sep 23 21:08:34 2023","interfaces":[{"interface":"back","average":{"1min":2.141,"5min":2.1579999999999999,"15min":2.9209999999999998},"min":{"1min":0.27700000000000002,"5min":0.252,"15min":0.252},"max":{"1min":5.96,"5min":18.669,"15min":41.079000000000001},"last":1.177},{"interface":"front","average":{"1min":2.7330000000000001,"5min":2.2469999999999999,"15min":3.0059999999999998},"min":{"1min":0.30299999999999999,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":16.960000000000001,"5min":19.91,"15min":21.167000000000002},"last":6.0570000000000004}]},{"osd":4,"last update":"Sat Sep 23 21:08:11 2023","interfaces":[{"interface":"back","average":{"1min":2.5649999999999999,"5min":1.8069999999999999,"15min":2.2170000000000001},"min":{"1min":0.56000000000000005,"5min":0.28699999999999998,"15min":0.28599999999999998},"max":{"1min":6.657,"5min":12.394,"15min":21.594999999999999},"last":2.1160000000000001},{"interface":"front","average":{"1min":2.669,"5min":2.0979999999999999,"15min":1.9970000000000001},"min":{"1min":0.36599999999999999,"5min":0.311,"15min":0.28100000000000003},"max":{"1min":5.774,"5min":18.516999999999999,"15min":18.724},"last":0.54400000000000004}]},{"osd":5,"last update":"Sat Sep 23 21:08:11 2023","interfaces":[{"interface":"back","average":{"1min":2.306,"5min":1.8480000000000001,"15min":1.9330000000000001},"min":{"1min":0.35099999999999998,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":5.2140000000000004,"5min":12.946999999999999,"15min":12.946999999999999},"last":5.6210000000000004},{"interface":"front","average":{"1min":2.0870000000000002,"5min":1.774,"15min":2.202},"min":{"1min":0.48899999999999999,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":6.7000000000000002,"5min":12.353,"15min":20.512},"last":2.1480000000000001}]},{"osd":6,"last update":"Sat Sep 23 21:09:09 2023","interfaces":[{"interface":"back","average":{"1min":1.73,"5min":2.0390000000000001,"15min":2.0310000000000001},"min":{"1min":0.34899999999999998,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":6.992,"5min":18.571000000000002,"15min":18.571000000000002},"last":0.51300000000000001},{"interface":"front","average":{"1min":1.7,"5min":2.1909999999999998,"15min":2.1040000000000001},"min":{"1min":0.42599999999999999,"5min":0.40300000000000002,"15min":0.377},"max":{"1min":5.5599999999999996,"5min":18.603000000000002,"15min":18.603000000000002},"last":0.57099999999999995}]},{"osd":7,"last update":"Sat Sep 23 21:08:11 2023","interfaces":[{"interface":"back","average":{"1min":2.399,"5min":1.8979999999999999,"15min":1.9890000000000001},"min":{"1min":0.38600000000000001,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":6.2450000000000001,"5min":12.907999999999999,"15min":12.907999999999999},"last":5.6680000000000001},{"interface":"front","average":{"1min":3.214,"5min":2.7290000000000001,"15min":2.2429999999999999},"min":{"1min":0.51100000000000001,"5min":0.40999999999999998,"15min":0.377},"max":{"1min":7.5449999999999999,"5min":40.984999999999999,"15min":40.984999999999999},"last":2.992}]}]},{"osd":2,"up_from":93,"seq":399431958754,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1116520,"kb_used_data":512544,"kb_used_omap":81,"kb_used_meta":603886,"kb_avail":92628632,"statfs":{"total":95995035648,"available":94851719168,"internally_reserved":0,"allocated":524845056,"data_stored":523074896,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":83750,"internal_metadata":618379482},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:09:01 2023","interfaces":[{"interface":"back","average":{"1min":2.9769999999999999,"5min":1.885,"15min":2.2210000000000001},"min":{"1min":0.26000000000000001,"5min":0.23000000000000001,"15min":0.17999999999999999},"max":{"1min":9.2720000000000002,"5min":11.124000000000001,"15min":29.498000000000001},"last":7.5259999999999998},{"interface":"front","average":{"1min":1.8999999999999999,"5min":1.677,"15min":2.4750000000000001},"min":{"1min":0.26900000000000002,"5min":0.26600000000000001,"15min":0.193},"max":{"1min":5.1079999999999997,"5min":9.2880000000000003,"15min":17.251000000000001},"last":12.183}]},{"osd":1,"last update":"Sat Sep 23 21:09:01 2023","interfaces":[{"interface":"back","average":{"1min":2.5649999999999999,"5min":3.153,"15min":3.0339999999999998},"min":{"1min":0.23699999999999999,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":6.5359999999999996,"5min":34.396999999999998,"15min":34.396999999999998},"last":12.484999999999999},{"interface":"front","average":{"1min":3.4399999999999999,"5min":3.3599999999999999,"15min":3.5680000000000001},"min":{"1min":0.34999999999999998,"5min":0.217,"15min":0.217},"max":{"1min":9.6579999999999995,"5min":34.462000000000003,"15min":34.462000000000003},"last":0.64100000000000001}]},{"osd":3,"last update":"Sat Sep 23 21:08:48 2023","interfaces":[{"interface":"back","average":{"1min":2.3330000000000002,"5min":2.5049999999999999,"15min":3.2810000000000001},"min":{"1min":0.32600000000000001,"5min":0.23899999999999999,"15min":0.23300000000000001},"max":{"1min":9.0739999999999998,"5min":35.549999999999997,"15min":41.057000000000002},"last":0.61499999999999999},{"interface":"front","average":{"1min":2.4780000000000002,"5min":2.7970000000000002,"15min":3.7650000000000001},"min":{"1min":0.35699999999999998,"5min":0.253,"15min":0.22800000000000001},"max":{"1min":5.423,"5min":36.951999999999998,"15min":41.012},"last":12.76}]},{"osd":4,"last update":"Sat Sep 23 21:09:01 2023","interfaces":[{"interface":"back","average":{"1min":1.8380000000000001,"5min":1.581,"15min":1.667},"min":{"1min":0.31900000000000001,"5min":0.29199999999999998,"15min":0.28399999999999997},"max":{"1min":4.3810000000000002,"5min":8.7569999999999997,"15min":10.045},"last":0.45100000000000001},{"interface":"front","average":{"1min":1.587,"5min":1.8520000000000001,"15min":2.109},"min":{"1min":0.34499999999999997,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":3.7050000000000001,"5min":12.49,"15min":14.039999999999999},"last":8.6780000000000008}]},{"osd":5,"last update":"Sat Sep 23 21:09:01 2023","interfaces":[{"interface":"back","average":{"1min":1.861,"5min":1.798,"15min":1.841},"min":{"1min":0.35499999999999998,"5min":0.316,"15min":0.29299999999999998},"max":{"1min":4.8289999999999997,"5min":8.907,"15min":10.069000000000001},"last":0.48799999999999999},{"interface":"front","average":{"1min":1.992,"5min":1.917,"15min":2.0979999999999999},"min":{"1min":0.46400000000000002,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":4.1029999999999998,"5min":12.57,"15min":13.98},"last":8.7270000000000003}]},{"osd":6,"last update":"Sat Sep 23 21:09:01 2023","interfaces":[{"interface":"back","average":{"1min":2.1629999999999998,"5min":2.1930000000000001,"15min":2.1349999999999998},"min":{"1min":0.39800000000000002,"5min":0.35699999999999998,"15min":0.312},"max":{"1min":8.7309999999999999,"5min":12.518000000000001,"15min":14.244999999999999},"last":8.7680000000000007},{"interface":"front","average":{"1min":2.637,"5min":2.4510000000000001,"15min":2.1240000000000001},"min":{"1min":0.56399999999999995,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":9.3230000000000004,"5min":12.544,"15min":14.068},"last":8.8010000000000002}]},{"osd":7,"last update":"Sat Sep 23 21:09:01 2023","interfaces":[{"interface":"back","average":{"1min":2.5649999999999999,"5min":1.9370000000000001,"15min":1.569},"min":{"1min":0.40799999999999997,"5min":0.34100000000000003,"15min":0.315},"max":{"1min":6.9390000000000001,"5min":8.7599999999999998,"15min":13.427},"last":0.59899999999999998},{"interface":"front","average":{"1min":2.1509999999999998,"5min":2.0379999999999998,"15min":1.9199999999999999},"min":{"1min":0.432,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":5.9740000000000002,"5min":10.92,"15min":10.92},"last":1.665}]}]},{"osd":3,"up_from":98,"seq":420906795227,"num_pgs":62,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1998600,"kb_used_data":1294912,"kb_used_omap":7,"kb_used_meta":703672,"kb_avail":91746552,"statfs":{"total":95995035648,"available":93948469248,"internally_reserved":0,"allocated":1325989888,"data_stored":1324245555,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7576,"internal_metadata":720560744},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:08:51 2023","interfaces":[{"interface":"back","average":{"1min":3.3380000000000001,"5min":2.4950000000000001,"15min":3.02},"min":{"1min":0.28699999999999998,"5min":0.19900000000000001,"15min":0.188},"max":{"1min":14.08,"5min":14.08,"15min":27.962},"last":0.437},{"interface":"front","average":{"1min":2.133,"5min":2.2370000000000001,"15min":2.919},"min":{"1min":0.26800000000000002,"5min":0.20599999999999999,"15min":0.19800000000000001},"max":{"1min":5.7969999999999997,"5min":11.483000000000001,"15min":28.058},"last":6.2450000000000001}]},{"osd":1,"last update":"Sat Sep 23 21:09:02 2023","interfaces":[{"interface":"back","average":{"1min":3.2490000000000001,"5min":3.0710000000000002,"15min":2.452},"min":{"1min":0.44400000000000001,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":16.939,"5min":16.939,"15min":16.939},"last":4.258},{"interface":"front","average":{"1min":2.5950000000000002,"5min":2.7709999999999999,"15min":2.4039999999999999},"min":{"1min":0.29299999999999998,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":7.9409999999999998,"5min":13.811999999999999,"15min":13.811999999999999},"last":6.2770000000000001}]},{"osd":2,"last update":"Sat Sep 23 21:08:51 2023","interfaces":[{"interface":"back","average":{"1min":2.1989999999999998,"5min":2.355,"15min":3.0379999999999998},"min":{"1min":0.32500000000000001,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":9.0169999999999995,"5min":14.287000000000001,"15min":28.251999999999999},"last":0.27100000000000002},{"interface":"front","average":{"1min":2.9750000000000001,"5min":2.548,"15min":3.097},"min":{"1min":0.39400000000000002,"5min":0.29399999999999998,"15min":0.22},"max":{"1min":14.195,"5min":14.195,"15min":25.305},"last":0.47799999999999998}]},{"osd":4,"last update":"Sat Sep 23 21:08:51 2023","interfaces":[{"interface":"back","average":{"1min":2.3889999999999998,"5min":1.843,"15min":2.052},"min":{"1min":0.38900000000000001,"5min":0.28799999999999998,"15min":0.28399999999999997},"max":{"1min":11.614000000000001,"5min":11.614000000000001,"15min":18.239000000000001},"last":0.70599999999999996},{"interface":"front","average":{"1min":3.7090000000000001,"5min":2.202,"15min":2.1779999999999999},"min":{"1min":0.52700000000000002,"5min":0.32100000000000001,"15min":0.29499999999999998},"max":{"1min":14.119,"5min":14.119,"15min":22.853000000000002},"last":0.57699999999999996}]},{"osd":5,"last update":"Sat Sep 23 21:08:51 2023","interfaces":[{"interface":"back","average":{"1min":2.7210000000000001,"5min":2.1890000000000001,"15min":2.2080000000000002},"min":{"1min":0.34999999999999998,"5min":0.314,"15min":0.314},"max":{"1min":8.5440000000000005,"5min":12.815,"15min":18.324999999999999},"last":0.38700000000000001},{"interface":"front","average":{"1min":2.4830000000000001,"5min":2.4500000000000002,"15min":2.391},"min":{"1min":0.44500000000000001,"5min":0.33700000000000002,"15min":0.33100000000000002},"max":{"1min":7.2060000000000004,"5min":14.029,"15min":14.029},"last":6.1239999999999997}]},{"osd":6,"last update":"Sat Sep 23 21:09:08 2023","interfaces":[{"interface":"back","average":{"1min":3.4049999999999998,"5min":2.2559999999999998,"15min":1.9179999999999999},"min":{"1min":0.49299999999999999,"5min":0.34599999999999997,"15min":0.33500000000000002},"max":{"1min":18.323,"5min":18.323,"15min":18.323},"last":0.51900000000000002},{"interface":"front","average":{"1min":1.669,"5min":2.2010000000000001,"15min":2.1890000000000001},"min":{"1min":0.47299999999999998,"5min":0.377,"15min":0.373},"max":{"1min":9.6300000000000008,"5min":9.6300000000000008,"15min":13.576000000000001},"last":0.77800000000000002}]},{"osd":7,"last update":"Sat Sep 23 21:08:51 2023","interfaces":[{"interface":"back","average":{"1min":1.925,"5min":2.1539999999999999,"15min":2.3719999999999999},"min":{"1min":0.40899999999999997,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":7.2939999999999996,"5min":14.81,"15min":18.289999999999999},"last":0.73999999999999999},{"interface":"front","average":{"1min":2.948,"5min":2.4249999999999998,"15min":2.4399999999999999},"min":{"1min":0.48099999999999998,"5min":0.40300000000000002,"15min":0.379},"max":{"1min":7.5229999999999997,"5min":12.624000000000001,"15min":13.603999999999999},"last":0.80800000000000005}]}]},{"osd":4,"up_from":31,"seq":133143986534,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":875748,"kb_used_data":589596,"kb_used_omap":0,"kb_used_meta":286144,"kb_avail":92869404,"statfs":{"total":95995035648,"available":95098269696,"internally_reserved":0,"allocated":603746304,"data_stored":602605772,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":293011456},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:09:08 2023","interfaces":[{"interface":"back","average":{"1min":2.2370000000000001,"5min":1.5900000000000001,"15min":1.6120000000000001},"min":{"1min":0.26200000000000001,"5min":0.222,"15min":0.214},"max":{"1min":6.351,"5min":11.537000000000001,"15min":11.784000000000001},"last":0.33900000000000002},{"interface":"front","average":{"1min":3.8919999999999999,"5min":2.3460000000000001,"15min":1.958},"min":{"1min":0.32500000000000001,"5min":0.254,"15min":0.254},"max":{"1min":19.109999999999999,"5min":19.109999999999999,"15min":19.109999999999999},"last":0.34999999999999998}]},{"osd":1,"last update":"Sat Sep 23 21:09:02 2023","interfaces":[{"interface":"back","average":{"1min":2.8660000000000001,"5min":3.367,"15min":3.669},"min":{"1min":0.33400000000000002,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":14.798999999999999,"5min":37.960000000000001,"15min":37.960000000000001},"last":0.36699999999999999},{"interface":"front","average":{"1min":2.1019999999999999,"5min":3.0680000000000001,"15min":3.4359999999999999},"min":{"1min":0.27000000000000002,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":14.403,"5min":37.933,"15min":37.933},"last":0.41799999999999998}]},{"osd":2,"last update":"Sat Sep 23 21:09:08 2023","interfaces":[{"interface":"back","average":{"1min":3.02,"5min":1.9179999999999999,"15min":1.952},"min":{"1min":0.308,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":16.928000000000001,"5min":16.928000000000001,"15min":16.928000000000001},"last":2.0489999999999999},{"interface":"front","average":{"1min":3.008,"5min":1.96,"15min":1.974},"min":{"1min":0.308,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":14.417999999999999,"5min":14.417999999999999,"15min":15.262},"last":2.9430000000000001}]},{"osd":3,"last update":"Sat Sep 23 21:08:38 2023","interfaces":[{"interface":"back","average":{"1min":2.21,"5min":2.1499999999999999,"15min":2.4249999999999998},"min":{"1min":0.29799999999999999,"5min":0.29799999999999999,"15min":0.28899999999999998},"max":{"1min":11.173,"5min":15.01,"15min":15.916},"last":3.1429999999999998},{"interface":"front","average":{"1min":2.3839999999999999,"5min":1.978,"15min":2.3239999999999998},"min":{"1min":0.32400000000000001,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":11.507999999999999,"5min":15.004,"15min":15.882},"last":3.1549999999999998}]},{"osd":5,"last update":"Sat Sep 23 21:08:57 2023","interfaces":[{"interface":"back","average":{"1min":2.137,"5min":1.238,"15min":0.84099999999999997},"min":{"1min":0.29599999999999999,"5min":0.24299999999999999,"15min":0.22700000000000001},"max":{"1min":14.298,"5min":14.298,"15min":14.298},"last":0.26300000000000001},{"interface":"front","average":{"1min":2.0259999999999998,"5min":1.3520000000000001,"15min":0.98499999999999999},"min":{"1min":0.26600000000000001,"5min":0.255,"15min":0.251},"max":{"1min":9.8979999999999997,"5min":9.8979999999999997,"15min":9.8979999999999997},"last":0.29499999999999998}]},{"osd":6,"last update":"Sat Sep 23 21:09:02 2023","interfaces":[{"interface":"back","average":{"1min":2.218,"5min":1.504,"15min":1.0840000000000001},"min":{"1min":0.29699999999999999,"5min":0.26800000000000002,"15min":0.26500000000000001},"max":{"1min":14.704000000000001,"5min":14.704000000000001,"15min":14.704000000000001},"last":0.32700000000000001},{"interface":"front","average":{"1min":1.6279999999999999,"5min":1.073,"15min":0.84699999999999998},"min":{"1min":0.32500000000000001,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":14.387,"5min":14.387,"15min":14.387},"last":0.438}]},{"osd":7,"last update":"Sat Sep 23 21:08:38 2023","interfaces":[{"interface":"back","average":{"1min":1.2030000000000001,"5min":1.1519999999999999,"15min":0.90800000000000003},"min":{"1min":0.307,"5min":0.307,"15min":0.29299999999999998},"max":{"1min":5.4619999999999997,"5min":14.460000000000001,"15min":14.460000000000001},"last":0.38900000000000001},{"interface":"front","average":{"1min":1.569,"5min":1.4850000000000001,"15min":1.04},"min":{"1min":0.34200000000000003,"5min":0.32200000000000001,"15min":0.28199999999999997},"max":{"1min":7.7229999999999999,"5min":11.170999999999999,"15min":11.170999999999999},"last":0.379}]}]},{"osd":5,"up_from":39,"seq":167503724899,"num_pgs":56,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1732660,"kb_used_data":1339820,"kb_used_omap":0,"kb_used_meta":392832,"kb_avail":92012492,"statfs":{"total":95995035648,"available":94220791808,"internally_reserved":0,"allocated":1371975680,"data_stored":1370819954,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":402259968},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:09:00 2023","interfaces":[{"interface":"back","average":{"1min":1.639,"5min":1.972,"15min":1.8049999999999999},"min":{"1min":0.22700000000000001,"5min":0.19700000000000001,"15min":0.19700000000000001},"max":{"1min":11.859999999999999,"5min":25.222999999999999,"15min":25.222999999999999},"last":0.31},{"interface":"front","average":{"1min":1.907,"5min":1.784,"15min":1.9350000000000001},"min":{"1min":0.26500000000000001,"5min":0.23499999999999999,"15min":0.223},"max":{"1min":7.7350000000000003,"5min":25.457999999999998,"15min":25.457999999999998},"last":0.251}]},{"osd":1,"last update":"Sat Sep 23 21:09:00 2023","interfaces":[{"interface":"back","average":{"1min":2.097,"5min":2.2469999999999999,"15min":1.9410000000000001},"min":{"1min":0.38100000000000001,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":9.1379999999999999,"5min":25.239999999999998,"15min":25.239999999999998},"last":0.44600000000000001},{"interface":"front","average":{"1min":2.476,"5min":2.1179999999999999,"15min":1.7410000000000001},"min":{"1min":0.27400000000000002,"5min":0.252,"15min":0.252},"max":{"1min":9.1530000000000005,"5min":25.253,"15min":25.253},"last":0.45900000000000002}]},{"osd":2,"last update":"Sat Sep 23 21:08:11 2023","interfaces":[{"interface":"back","average":{"1min":2.3090000000000002,"5min":1.746,"15min":1.76},"min":{"1min":0.33100000000000002,"5min":0.25800000000000001,"15min":0.23499999999999999},"max":{"1min":8.2119999999999997,"5min":12.634,"15min":12.651},"last":2.4039999999999999},{"interface":"front","average":{"1min":2.673,"5min":1.9910000000000001,"15min":1.9630000000000001},"min":{"1min":0.29999999999999999,"5min":0.28100000000000003,"15min":0.26100000000000001},"max":{"1min":8.1549999999999994,"5min":8.1549999999999994,"15min":12.300000000000001},"last":0.57199999999999995}]},{"osd":3,"last update":"Sat Sep 23 21:08:33 2023","interfaces":[{"interface":"back","average":{"1min":2.863,"5min":2.331,"15min":2.7410000000000001},"min":{"1min":0.375,"5min":0.29299999999999998,"15min":0.28599999999999998},"max":{"1min":7.194,"5min":25.503,"15min":61.667999999999999},"last":0.33000000000000002},{"interface":"front","average":{"1min":2.6560000000000001,"5min":2.524,"15min":2.5089999999999999},"min":{"1min":0.34999999999999998,"5min":0.33500000000000002,"15min":0.28899999999999998},"max":{"1min":8.2680000000000007,"5min":25.466000000000001,"15min":62.276000000000003},"last":14.396000000000001}]},{"osd":4,"last update":"Sat Sep 23 21:08:50 2023","interfaces":[{"interface":"back","average":{"1min":1.6200000000000001,"5min":1.23,"15min":0.873},"min":{"1min":0.27800000000000002,"5min":0.27800000000000002,"15min":0.23400000000000001},"max":{"1min":7.7800000000000002,"5min":9.0950000000000006,"15min":9.0950000000000006},"last":0.372},{"interface":"front","average":{"1min":2.4239999999999999,"5min":1.3919999999999999,"15min":0.95199999999999996},"min":{"1min":0.37,"5min":0.26900000000000002,"15min":0.23599999999999999},"max":{"1min":11.973000000000001,"5min":11.973000000000001,"15min":11.973000000000001},"last":0.39100000000000001}]},{"osd":6,"last update":"Sat Sep 23 21:09:00 2023","interfaces":[{"interface":"back","average":{"1min":0.95699999999999996,"5min":1.2450000000000001,"15min":0.94499999999999995},"min":{"1min":0.33700000000000002,"5min":0.28799999999999998,"15min":0.27500000000000002},"max":{"1min":4.5789999999999997,"5min":8.0909999999999993,"15min":11.128},"last":0.35899999999999999},{"interface":"front","average":{"1min":1.3129999999999999,"5min":1.234,"15min":0.999},"min":{"1min":0.317,"5min":0.317,"15min":0.317},"max":{"1min":7.8550000000000004,"5min":7.8550000000000004,"15min":11.112},"last":0.42899999999999999}]},{"osd":7,"last update":"Sat Sep 23 21:08:33 2023","interfaces":[{"interface":"back","average":{"1min":13.24,"5min":3.569,"15min":1.742},"min":{"1min":0.40400000000000003,"5min":0.35099999999999998,"15min":0.30199999999999999},"max":{"1min":189.405,"5min":189.405,"15min":189.405},"last":0.41499999999999998},{"interface":"front","average":{"1min":13.374000000000001,"5min":3.3809999999999998,"15min":1.663},"min":{"1min":0.35599999999999998,"5min":0.32500000000000001,"15min":0.27700000000000002},"max":{"1min":189.685,"5min":189.685,"15min":189.685},"last":2.6469999999999998}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":5434,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1259,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5412,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":646,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":4838,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":836435968,"data_stored":836435968,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1319440384,"data_stored":1319436312,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":517537792,"data_stored":517537792,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1295056896,"data_stored":1295056896,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":586485760,"data_stored":586481688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1365839872,"data_stored":1365835800,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":410320896,"data_stored":410320896,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1324224512,"data_stored":1324220440,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T21:09:20.511 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph osd pool set .mgr pgp_num 21 2023-09-23T21:09:20.517 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:09:20.530 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:09:20.530 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:09:20.607 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_blocked_ops 2023-09-23T21:09:20.631 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:09:20.659 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:09:20.694 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:09:20.697 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:09:20.697 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:09:20.798 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:09:20.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:09:20 smithi070 ceph-mon[168282]: pgmap v1047: 109 pgs: 1 active+clean+scrubbing, 3 active+remapped+backfilling, 3 active+remapped+backfill_wait, 102 active+clean; 2.0 GiB data, 8.6 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 660 op/s; 7656/96942 objects misplaced (7.898%); 29 MiB/s, 466 objects/s recovering 2023-09-23T21:09:20.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:09:20 smithi070 ceph-mon[168282]: failed to encode map e403 with expected crc 2023-09-23T21:09:20.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:09:20 smithi070 ceph-mon[168282]: failed to encode map e403 with expected crc 2023-09-23T21:09:20.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:09:20 smithi070 ceph-mon[168282]: failed to encode map e403 with expected crc 2023-09-23T21:09:20.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:09:20 smithi070 ceph-mon[168282]: osdmap e403: 8 total, 8 up, 7 in 2023-09-23T21:09:20.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:09:20 smithi070 ceph-mon[168282]: failed to encode map e403 with expected crc 2023-09-23T21:09:20.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:09:20 smithi070 ceph-mon[168282]: failed to encode map e403 with expected crc 2023-09-23T21:09:20.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:09:20 smithi070 ceph-mon[168282]: osdmap e404: 8 total, 8 up, 7 in 2023-09-23T21:09:20.870 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:09:20 smithi026 ceph-mon[175151]: pgmap v1047: 109 pgs: 1 active+clean+scrubbing, 3 active+remapped+backfilling, 3 active+remapped+backfill_wait, 102 active+clean; 2.0 GiB data, 8.6 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 660 op/s; 7656/96942 objects misplaced (7.898%); 29 MiB/s, 466 objects/s recovering 2023-09-23T21:09:20.870 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:09:20 smithi026 ceph-mon[175151]: failed to encode map e403 with expected crc 2023-09-23T21:09:20.870 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:09:20 smithi026 ceph-mon[175151]: failed to encode map e403 with expected crc 2023-09-23T21:09:20.870 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:09:20 smithi026 ceph-mon[175151]: failed to encode map e403 with expected crc 2023-09-23T21:09:20.870 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:09:20 smithi026 ceph-mon[175151]: osdmap e403: 8 total, 8 up, 7 in 2023-09-23T21:09:20.870 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:09:20 smithi026 ceph-mon[175151]: failed to encode map e403 with expected crc 2023-09-23T21:09:20.870 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:09:20 smithi026 ceph-mon[175151]: failed to encode map e403 with expected crc 2023-09-23T21:09:20.871 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:09:20 smithi026 ceph-mon[175151]: osdmap e404: 8 total, 8 up, 7 in 2023-09-23T21:09:20.872 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:09:20 smithi026 ceph-mon[179688]: pgmap v1047: 109 pgs: 1 active+clean+scrubbing, 3 active+remapped+backfilling, 3 active+remapped+backfill_wait, 102 active+clean; 2.0 GiB data, 8.6 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 660 op/s; 7656/96942 objects misplaced (7.898%); 29 MiB/s, 466 objects/s recovering 2023-09-23T21:09:20.872 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:09:20 smithi026 ceph-mon[179688]: failed to encode map e403 with expected crc 2023-09-23T21:09:20.872 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:09:20 smithi026 ceph-mon[179688]: failed to encode map e403 with expected crc 2023-09-23T21:09:20.872 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:09:20 smithi026 ceph-mon[179688]: failed to encode map e403 with expected crc 2023-09-23T21:09:20.872 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:09:20 smithi026 ceph-mon[179688]: osdmap e403: 8 total, 8 up, 7 in 2023-09-23T21:09:20.872 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:09:20 smithi026 ceph-mon[179688]: failed to encode map e403 with expected crc 2023-09-23T21:09:20.872 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:09:20 smithi026 ceph-mon[179688]: failed to encode map e403 with expected crc 2023-09-23T21:09:20.872 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:09:20 smithi026 ceph-mon[179688]: osdmap e404: 8 total, 8 up, 7 in 2023-09-23T21:09:20.878 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:09:20.902 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:09:20.960 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:09:20.966 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:09:20.967 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:09:21.068 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:09:21.113 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:09:21.273 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0: {} 2023-09-23T21:09:21.308 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1: {} 2023-09-23T21:09:21.321 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T21:09:21.333 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:09:21.333 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:09:21.341 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2: {} 2023-09-23T21:09:21.379 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3: {} 2023-09-23T21:09:21.411 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4: {} 2023-09-23T21:09:21.434 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:09:21.456 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T21:09:21.463 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:09:21.500 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6: {} 2023-09-23T21:09:21.502 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:09:21.505 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:09:21.505 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:09:21.548 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T21:09:21.564 INFO:teuthology.orchestra.run.smithi026.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T21:09:21.564 INFO:teuthology.orchestra.run.smithi026.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T21:09:21.564 INFO:teuthology.orchestra.run.smithi026.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T21:09:21.564 INFO:teuthology.orchestra.run.smithi026.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T21:09:21.564 INFO:teuthology.orchestra.run.smithi026.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T21:09:21.564 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T21:09:21.564 INFO:teuthology.orchestra.run.smithi026.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T21:09:21.565 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T21:09:21.606 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:09:21.614 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:09:21.697 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:09:21.763 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:09:21.775 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:09:21.776 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:09:21.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:09:21 smithi070 ceph-mon[168282]: failed to encode map e404 with expected crc 2023-09-23T21:09:21.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:09:21 smithi070 ceph-mon[168282]: failed to encode map e404 with expected crc 2023-09-23T21:09:21.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:09:21 smithi070 ceph-mon[168282]: failed to encode map e404 with expected crc 2023-09-23T21:09:21.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:09:21 smithi070 ceph-mon[168282]: failed to encode map e404 with expected crc 2023-09-23T21:09:21.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:09:21 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/3632279072' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:09:21.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:09:21 smithi070 ceph-mon[168282]: osdmap e405: 8 total, 8 up, 7 in 2023-09-23T21:09:21.877 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:09:21.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:09:21 smithi026 ceph-mon[175151]: failed to encode map e404 with expected crc 2023-09-23T21:09:21.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:09:21 smithi026 ceph-mon[175151]: failed to encode map e404 with expected crc 2023-09-23T21:09:21.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:09:21 smithi026 ceph-mon[175151]: failed to encode map e404 with expected crc 2023-09-23T21:09:21.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:09:21 smithi026 ceph-mon[175151]: failed to encode map e404 with expected crc 2023-09-23T21:09:21.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:09:21 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/3632279072' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:09:21.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:09:21 smithi026 ceph-mon[175151]: osdmap e405: 8 total, 8 up, 7 in 2023-09-23T21:09:21.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:09:21 smithi026 ceph-mon[179688]: failed to encode map e404 with expected crc 2023-09-23T21:09:21.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:09:21 smithi026 ceph-mon[179688]: failed to encode map e404 with expected crc 2023-09-23T21:09:21.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:09:21 smithi026 ceph-mon[179688]: failed to encode map e404 with expected crc 2023-09-23T21:09:21.883 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:09:21 smithi026 ceph-mon[179688]: failed to encode map e404 with expected crc 2023-09-23T21:09:21.883 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:09:21 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/3632279072' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:09:21.883 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:09:21 smithi026 ceph-mon[179688]: osdmap e405: 8 total, 8 up, 7 in 2023-09-23T21:09:21.945 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:09:22.187 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:09:22.196 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:09:22.196 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:09:22.288 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:09:22.297 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:09:22.436 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:09:22.510 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:09:22.524 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:09:22.525 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:09:22.626 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:09:22.656 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:09:22.680 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:09:22 smithi026 ceph-mon[175151]: pgmap v1050: 109 pgs: 1 peering, 1 active+clean+remapped, 1 active+clean+scrubbing, 2 active+remapped+backfilling, 2 active+remapped+backfill_wait, 102 active+clean; 1.9 GiB data, 8.4 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 762 op/s; 6022/93081 objects misplaced (6.470%); 26 MiB/s, 411 objects/s recovering 2023-09-23T21:09:22.680 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:09:22 smithi026 ceph-mon[175151]: failed to encode map e405 with expected crc 2023-09-23T21:09:22.680 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:09:22 smithi026 ceph-mon[175151]: failed to encode map e405 with expected crc 2023-09-23T21:09:22.680 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:09:22 smithi026 ceph-mon[175151]: failed to encode map e405 with expected crc 2023-09-23T21:09:22.680 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:09:22 smithi026 ceph-mon[179688]: pgmap v1050: 109 pgs: 1 peering, 1 active+clean+remapped, 1 active+clean+scrubbing, 2 active+remapped+backfilling, 2 active+remapped+backfill_wait, 102 active+clean; 1.9 GiB data, 8.4 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 762 op/s; 6022/93081 objects misplaced (6.470%); 26 MiB/s, 411 objects/s recovering 2023-09-23T21:09:22.680 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:09:22 smithi026 ceph-mon[179688]: failed to encode map e405 with expected crc 2023-09-23T21:09:22.680 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:09:22 smithi026 ceph-mon[179688]: failed to encode map e405 with expected crc 2023-09-23T21:09:22.680 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:09:22 smithi026 ceph-mon[179688]: failed to encode map e405 with expected crc 2023-09-23T21:09:22.695 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:09:22.699 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:09:22.699 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:09:22.800 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:09:22.834 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:09:22.871 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:09:22.874 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:09:22.874 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:09:22.975 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:09:23.059 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:09:23.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:09:22 smithi070 ceph-mon[168282]: pgmap v1050: 109 pgs: 1 peering, 1 active+clean+remapped, 1 active+clean+scrubbing, 2 active+remapped+backfilling, 2 active+remapped+backfill_wait, 102 active+clean; 1.9 GiB data, 8.4 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 762 op/s; 6022/93081 objects misplaced (6.470%); 26 MiB/s, 411 objects/s recovering 2023-09-23T21:09:23.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:09:22 smithi070 ceph-mon[168282]: failed to encode map e405 with expected crc 2023-09-23T21:09:23.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:09:22 smithi070 ceph-mon[168282]: failed to encode map e405 with expected crc 2023-09-23T21:09:23.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:09:22 smithi070 ceph-mon[168282]: failed to encode map e405 with expected crc 2023-09-23T21:09:23.404 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:09:23.412 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:09:23.413 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:09:23.513 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:09:23.643 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:09:23.702 INFO:teuthology.orchestra.run.smithi026.stderr:nodeep-scrub is set 2023-09-23T21:09:23.871 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T21:09:23.878 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:09:23.879 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:09:23 smithi026 ceph-mon[175151]: pgmap v1052: 109 pgs: 1 peering, 1 active+clean+remapped, 2 active+remapped+backfilling, 1 active+remapped+backfill_wait, 104 active+clean; 1.8 GiB data, 8.3 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 485 op/s; 4576/90603 objects misplaced (5.051%); 39 MiB/s, 621 objects/s recovering 2023-09-23T21:09:23.879 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:09:23 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/3520454653' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:09:23.879 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:09:23 smithi026 ceph-mon[179688]: pgmap v1052: 109 pgs: 1 peering, 1 active+clean+remapped, 2 active+remapped+backfilling, 1 active+remapped+backfill_wait, 104 active+clean; 1.8 GiB data, 8.3 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 485 op/s; 4576/90603 objects misplaced (5.051%); 39 MiB/s, 621 objects/s recovering 2023-09-23T21:09:23.879 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:09:23 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/3520454653' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:09:23.880 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:09:23.980 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:09:24.004 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:09:23 smithi070 ceph-mon[168282]: pgmap v1052: 109 pgs: 1 peering, 1 active+clean+remapped, 2 active+remapped+backfilling, 1 active+remapped+backfill_wait, 104 active+clean; 1.8 GiB data, 8.3 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 485 op/s; 4576/90603 objects misplaced (5.051%); 39 MiB/s, 621 objects/s recovering 2023-09-23T21:09:24.004 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:09:23 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/3520454653' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:09:24.007 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:09:24.048 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:09:24.050 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:09:24.051 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:09:24.151 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:09:24.231 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:09:24.243 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T21:09:24.456 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:09:24.462 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:09:24.463 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:09:24.563 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:09:24.704 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:09:24.862 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T21:09:24.868 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:09:24 smithi026 ceph-mon[179688]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:09:24.869 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:09:24 smithi026 ceph-mon[179688]: failed to encode map e406 with expected crc 2023-09-23T21:09:24.869 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:09:24 smithi026 ceph-mon[179688]: failed to encode map e406 with expected crc 2023-09-23T21:09:24.869 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:09:24 smithi026 ceph-mon[179688]: failed to encode map e406 with expected crc 2023-09-23T21:09:24.869 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:09:24 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/3520454653' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T21:09:24.869 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:09:24 smithi026 ceph-mon[179688]: osdmap e406: 8 total, 8 up, 7 in 2023-09-23T21:09:24.869 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:09:24 smithi026 ceph-mon[179688]: failed to encode map e406 with expected crc 2023-09-23T21:09:24.870 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:09:24.870 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:09:24.870 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:09:24 smithi026 ceph-mon[175151]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:09:24.870 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:09:24 smithi026 ceph-mon[175151]: failed to encode map e406 with expected crc 2023-09-23T21:09:24.870 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:09:24 smithi026 ceph-mon[175151]: failed to encode map e406 with expected crc 2023-09-23T21:09:24.871 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:09:24 smithi026 ceph-mon[175151]: failed to encode map e406 with expected crc 2023-09-23T21:09:24.871 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:09:24 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/3520454653' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T21:09:24.871 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:09:24 smithi026 ceph-mon[175151]: osdmap e406: 8 total, 8 up, 7 in 2023-09-23T21:09:24.871 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:09:24 smithi026 ceph-mon[175151]: failed to encode map e406 with expected crc 2023-09-23T21:09:24.970 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:09:24.993 INFO:teuthology.orchestra.run.smithi026.stdout:{ 2023-09-23T21:09:24.993 INFO:teuthology.orchestra.run.smithi026.stdout: "ops": [], 2023-09-23T21:09:24.993 INFO:teuthology.orchestra.run.smithi026.stdout: "complaint_time": 30, 2023-09-23T21:09:24.993 INFO:teuthology.orchestra.run.smithi026.stdout: "num_blocked_ops": 0 2023-09-23T21:09:24.993 INFO:teuthology.orchestra.run.smithi026.stdout:} 2023-09-23T21:09:24.996 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:09:24 smithi070 ceph-mon[168282]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:09:24.996 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:09:24 smithi070 ceph-mon[168282]: failed to encode map e406 with expected crc 2023-09-23T21:09:24.996 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:09:24 smithi070 ceph-mon[168282]: failed to encode map e406 with expected crc 2023-09-23T21:09:24.996 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:09:24 smithi070 ceph-mon[168282]: failed to encode map e406 with expected crc 2023-09-23T21:09:24.996 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:09:24 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/3520454653' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T21:09:24.996 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:09:24 smithi070 ceph-mon[168282]: osdmap e406: 8 total, 8 up, 7 in 2023-09-23T21:09:24.996 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:09:24 smithi070 ceph-mon[168282]: failed to encode map e406 with expected crc 2023-09-23T21:09:24.998 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:09:25.037 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:09:25.039 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:09:25.040 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:09:25.140 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:09:25.219 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:09:25.730 INFO:teuthology.orchestra.run.smithi026.stderr:set pool 1 pgp_num to 21 2023-09-23T21:09:25.770 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:09:25 smithi026 ceph-mon[175151]: pgmap v1054: 109 pgs: 3 active+remapped+backfilling, 106 active+clean; 1.8 GiB data, 8.3 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 434 op/s; 1732/86118 objects misplaced (2.011%); 48 MiB/s, 761 objects/s recovering 2023-09-23T21:09:25.771 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:09:25 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/965016022' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "21"}]: dispatch 2023-09-23T21:09:25.771 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:09:25 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "21"}]: dispatch 2023-09-23T21:09:25.771 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:09:25 smithi026 ceph-mon[179688]: pgmap v1054: 109 pgs: 3 active+remapped+backfilling, 106 active+clean; 1.8 GiB data, 8.3 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 434 op/s; 1732/86118 objects misplaced (2.011%); 48 MiB/s, 761 objects/s recovering 2023-09-23T21:09:25.771 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:09:25 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/965016022' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "21"}]: dispatch 2023-09-23T21:09:25.771 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:09:25 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "21"}]: dispatch 2023-09-23T21:09:26.122 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:09:25 smithi070 ceph-mon[168282]: pgmap v1054: 109 pgs: 3 active+remapped+backfilling, 106 active+clean; 1.8 GiB data, 8.3 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 434 op/s; 1732/86118 objects misplaced (2.011%); 48 MiB/s, 761 objects/s recovering 2023-09-23T21:09:26.122 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:09:25 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/965016022' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "21"}]: dispatch 2023-09-23T21:09:26.122 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:09:25 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "21"}]: dispatch 2023-09-23T21:09:26.321 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T21:09:26.706 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:09:26 smithi026 ceph-mon[175151]: failed to encode map e407 with expected crc 2023-09-23T21:09:26.707 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:09:26 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "21"}]': finished 2023-09-23T21:09:26.707 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:09:26 smithi026 ceph-mon[175151]: osdmap e407: 8 total, 8 up, 7 in 2023-09-23T21:09:27.085 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:09:26 smithi026 ceph-mon[175151]: failed to encode map e407 with expected crc 2023-09-23T21:09:27.085 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:09:26 smithi026 ceph-mon[175151]: failed to encode map e407 with expected crc 2023-09-23T21:09:27.085 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:09:26 smithi026 ceph-mon[179688]: failed to encode map e407 with expected crc 2023-09-23T21:09:27.085 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:09:26 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "21"}]': finished 2023-09-23T21:09:27.085 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:09:26 smithi026 ceph-mon[179688]: osdmap e407: 8 total, 8 up, 7 in 2023-09-23T21:09:27.086 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:09:26 smithi026 ceph-mon[179688]: failed to encode map e407 with expected crc 2023-09-23T21:09:27.086 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:09:26 smithi026 ceph-mon[179688]: failed to encode map e407 with expected crc 2023-09-23T21:09:27.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:09:26 smithi070 ceph-mon[168282]: failed to encode map e407 with expected crc 2023-09-23T21:09:27.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:09:26 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "21"}]': finished 2023-09-23T21:09:27.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:09:26 smithi070 ceph-mon[168282]: osdmap e407: 8 total, 8 up, 7 in 2023-09-23T21:09:27.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:09:26 smithi070 ceph-mon[168282]: failed to encode map e407 with expected crc 2023-09-23T21:09:27.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:09:26 smithi070 ceph-mon[168282]: failed to encode map e407 with expected crc 2023-09-23T21:09:28.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:09:27 smithi070 ceph-mon[168282]: failed to encode map e408 with expected crc 2023-09-23T21:09:28.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:09:27 smithi070 ceph-mon[168282]: osdmap e408: 8 total, 8 up, 7 in 2023-09-23T21:09:28.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:09:27 smithi070 ceph-mon[168282]: failed to encode map e408 with expected crc 2023-09-23T21:09:28.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:09:27 smithi070 ceph-mon[168282]: failed to encode map e408 with expected crc 2023-09-23T21:09:28.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:09:27 smithi070 ceph-mon[168282]: failed to encode map e408 with expected crc 2023-09-23T21:09:28.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:09:27 smithi070 ceph-mon[168282]: failed to encode map e408 with expected crc 2023-09-23T21:09:28.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:09:27 smithi070 ceph-mon[168282]: pgmap v1057: 109 pgs: 3 active+remapped+backfilling, 106 active+clean; 1.7 GiB data, 8.2 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 424 op/s; 1732/85563 objects misplaced (2.024%); 46 MiB/s, 731 objects/s recovering 2023-09-23T21:09:28.131 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:09:27 smithi026 ceph-mon[179688]: failed to encode map e408 with expected crc 2023-09-23T21:09:28.131 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:09:27 smithi026 ceph-mon[179688]: osdmap e408: 8 total, 8 up, 7 in 2023-09-23T21:09:28.131 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:09:27 smithi026 ceph-mon[179688]: failed to encode map e408 with expected crc 2023-09-23T21:09:28.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:09:27 smithi026 ceph-mon[179688]: failed to encode map e408 with expected crc 2023-09-23T21:09:28.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:09:27 smithi026 ceph-mon[179688]: failed to encode map e408 with expected crc 2023-09-23T21:09:28.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:09:27 smithi026 ceph-mon[179688]: failed to encode map e408 with expected crc 2023-09-23T21:09:28.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:09:27 smithi026 ceph-mon[179688]: pgmap v1057: 109 pgs: 3 active+remapped+backfilling, 106 active+clean; 1.7 GiB data, 8.2 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 424 op/s; 1732/85563 objects misplaced (2.024%); 46 MiB/s, 731 objects/s recovering 2023-09-23T21:09:28.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:09:27 smithi026 ceph-mon[175151]: failed to encode map e408 with expected crc 2023-09-23T21:09:28.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:09:27 smithi026 ceph-mon[175151]: osdmap e408: 8 total, 8 up, 7 in 2023-09-23T21:09:28.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:09:27 smithi026 ceph-mon[175151]: failed to encode map e408 with expected crc 2023-09-23T21:09:28.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:09:27 smithi026 ceph-mon[175151]: failed to encode map e408 with expected crc 2023-09-23T21:09:28.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:09:27 smithi026 ceph-mon[175151]: failed to encode map e408 with expected crc 2023-09-23T21:09:28.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:09:27 smithi026 ceph-mon[175151]: failed to encode map e408 with expected crc 2023-09-23T21:09:28.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:09:27 smithi026 ceph-mon[175151]: pgmap v1057: 109 pgs: 3 active+remapped+backfilling, 106 active+clean; 1.7 GiB data, 8.2 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 424 op/s; 1732/85563 objects misplaced (2.024%); 46 MiB/s, 731 objects/s recovering 2023-09-23T21:09:28.259 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:09:28.262 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:09:28.263 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:09:28.365 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:09:28.397 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:09:28.412 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:09:28.432 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:09:28.434 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:09:28.435 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:09:28.436 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_historic_ops 2023-09-23T21:09:28.536 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:09:28.563 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:09:28.596 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:09:28.598 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:09:28.599 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:09:28.699 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:09:28.728 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:09:28.761 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:09:28.764 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:09:28.765 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:09:28.865 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:09:28.944 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:09:28 smithi026 ceph-mon[179688]: failed to encode map e409 with expected crc 2023-09-23T21:09:28.944 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:09:28 smithi026 ceph-mon[179688]: failed to encode map e409 with expected crc 2023-09-23T21:09:28.944 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:09:28 smithi026 ceph-mon[179688]: osdmap e409: 8 total, 8 up, 7 in 2023-09-23T21:09:28.945 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:09:28 smithi026 ceph-mon[179688]: failed to encode map e409 with expected crc 2023-09-23T21:09:28.945 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:09:28 smithi026 ceph-mon[179688]: failed to encode map e409 with expected crc 2023-09-23T21:09:28.945 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:09:28 smithi026 ceph-mon[175151]: failed to encode map e409 with expected crc 2023-09-23T21:09:28.945 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:09:28 smithi026 ceph-mon[175151]: failed to encode map e409 with expected crc 2023-09-23T21:09:28.945 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:09:28 smithi026 ceph-mon[175151]: osdmap e409: 8 total, 8 up, 7 in 2023-09-23T21:09:28.945 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:09:28 smithi026 ceph-mon[175151]: failed to encode map e409 with expected crc 2023-09-23T21:09:28.945 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:09:28 smithi026 ceph-mon[175151]: failed to encode map e409 with expected crc 2023-09-23T21:09:28.961 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:09:29.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:09:28 smithi070 ceph-mon[168282]: failed to encode map e409 with expected crc 2023-09-23T21:09:29.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:09:28 smithi070 ceph-mon[168282]: failed to encode map e409 with expected crc 2023-09-23T21:09:29.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:09:28 smithi070 ceph-mon[168282]: osdmap e409: 8 total, 8 up, 7 in 2023-09-23T21:09:29.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:09:28 smithi070 ceph-mon[168282]: failed to encode map e409 with expected crc 2023-09-23T21:09:29.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:09:28 smithi070 ceph-mon[168282]: failed to encode map e409 with expected crc 2023-09-23T21:09:29.438 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:09:29.444 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:09:29.445 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:09:29.545 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:09:29.627 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:09:29.648 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:09:29.731 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 21:09:29 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[164029]: ::ffff:172.21.15.70 - - [23/Sep/2023:21:09:29] "GET /metrics HTTP/1.1" 200 36361 "" "Prometheus/2.43.0" 2023-09-23T21:09:29.733 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:09:29 smithi026 ceph-mon[175151]: failed to encode map e410 with expected crc 2023-09-23T21:09:29.733 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:09:29 smithi026 ceph-mon[175151]: failed to encode map e410 with expected crc 2023-09-23T21:09:29.733 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:09:29 smithi026 ceph-mon[175151]: failed to encode map e410 with expected crc 2023-09-23T21:09:29.733 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:09:29 smithi026 ceph-mon[175151]: osdmap e410: 8 total, 8 up, 7 in 2023-09-23T21:09:29.733 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:09:29 smithi026 ceph-mon[175151]: failed to encode map e410 with expected crc 2023-09-23T21:09:29.733 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:09:29 smithi026 ceph-mon[175151]: failed to encode map e410 with expected crc 2023-09-23T21:09:29.733 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:09:29 smithi026 ceph-mon[175151]: pgmap v1060: 109 pgs: 2 active+remapped+backfilling, 107 active+clean; 1.6 GiB data, 7.9 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 499 op/s; 650/80124 objects misplaced (0.811%); 16 MiB/s, 249 objects/s recovering 2023-09-23T21:09:30.062 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:09:30.070 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:09:29 smithi026 ceph-mon[179688]: failed to encode map e410 with expected crc 2023-09-23T21:09:30.070 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:09:29 smithi026 ceph-mon[179688]: failed to encode map e410 with expected crc 2023-09-23T21:09:30.070 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:09:29 smithi026 ceph-mon[179688]: failed to encode map e410 with expected crc 2023-09-23T21:09:30.070 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:09:29 smithi026 ceph-mon[179688]: osdmap e410: 8 total, 8 up, 7 in 2023-09-23T21:09:30.070 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:09:29 smithi026 ceph-mon[179688]: failed to encode map e410 with expected crc 2023-09-23T21:09:30.070 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:09:29 smithi026 ceph-mon[179688]: failed to encode map e410 with expected crc 2023-09-23T21:09:30.070 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:09:29 smithi026 ceph-mon[179688]: pgmap v1060: 109 pgs: 2 active+remapped+backfilling, 107 active+clean; 1.6 GiB data, 7.9 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 499 op/s; 650/80124 objects misplaced (0.811%); 16 MiB/s, 249 objects/s recovering 2023-09-23T21:09:30.071 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:09:30.071 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:09:30.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:09:29 smithi070 ceph-mon[168282]: failed to encode map e410 with expected crc 2023-09-23T21:09:30.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:09:29 smithi070 ceph-mon[168282]: failed to encode map e410 with expected crc 2023-09-23T21:09:30.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:09:29 smithi070 ceph-mon[168282]: failed to encode map e410 with expected crc 2023-09-23T21:09:30.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:09:29 smithi070 ceph-mon[168282]: osdmap e410: 8 total, 8 up, 7 in 2023-09-23T21:09:30.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:09:29 smithi070 ceph-mon[168282]: failed to encode map e410 with expected crc 2023-09-23T21:09:30.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:09:29 smithi070 ceph-mon[168282]: failed to encode map e410 with expected crc 2023-09-23T21:09:30.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:09:29 smithi070 ceph-mon[168282]: pgmap v1060: 109 pgs: 2 active+remapped+backfilling, 107 active+clean; 1.6 GiB data, 7.9 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 499 op/s; 650/80124 objects misplaced (0.811%); 16 MiB/s, 249 objects/s recovering 2023-09-23T21:09:30.172 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:09:30.253 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:09:30.488 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T21:09:30.494 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:09:30.495 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:09:30.595 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:09:30.683 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:09:30.888 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:09:30.910 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:09:30.919 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:09:30.919 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:09:31.020 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:09:31.097 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:09:31.185 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T21:09:31.194 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:09:31.194 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:09:31.295 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:09:31.381 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:09:31.698 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:09:31.705 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:09:31.706 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:09:31.806 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:09:31.834 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:09:31.873 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:09:31.875 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:09:31.875 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:09:31.976 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:09:32.005 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:09:32.041 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:09:32.043 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:09:32.044 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:09:32.144 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:09:32.260 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:09:32.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:09:32 smithi070 ceph-mon[168282]: pgmap v1061: 109 pgs: 1 active+remapped+backfilling, 108 active+clean; 1.6 GiB data, 7.8 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 576 op/s; 61/77064 objects misplaced (0.079%); 19 MiB/s, 296 objects/s recovering 2023-09-23T21:09:32.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:09:32 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:09:32.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:09:32 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:09:32.637 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:09:32 smithi026 ceph-mon[175151]: pgmap v1061: 109 pgs: 1 active+remapped+backfilling, 108 active+clean; 1.6 GiB data, 7.8 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 576 op/s; 61/77064 objects misplaced (0.079%); 19 MiB/s, 296 objects/s recovering 2023-09-23T21:09:32.637 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:09:32 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:09:32.637 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:09:32 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:09:32.637 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:09:32 smithi026 ceph-mon[179688]: pgmap v1061: 109 pgs: 1 active+remapped+backfilling, 108 active+clean; 1.6 GiB data, 7.8 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 576 op/s; 61/77064 objects misplaced (0.079%); 19 MiB/s, 296 objects/s recovering 2023-09-23T21:09:32.638 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:09:32 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:09:32.638 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:09:32 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:09:32.866 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:09:32.875 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:09:32.876 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:09:32.976 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:09:33.006 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:09:33.047 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:09:33.051 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:09:33.051 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:09:33.152 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:09:33.168 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0: {} 2023-09-23T21:09:33.169 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1: {} 2023-09-23T21:09:33.169 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2: {} 2023-09-23T21:09:33.169 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3: {} 2023-09-23T21:09:33.169 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4: {} 2023-09-23T21:09:33.169 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T21:09:33.169 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6: {} 2023-09-23T21:09:33.169 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T21:09:33.169 INFO:teuthology.orchestra.run.smithi026.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T21:09:33.169 INFO:teuthology.orchestra.run.smithi026.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T21:09:33.170 INFO:teuthology.orchestra.run.smithi026.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T21:09:33.170 INFO:teuthology.orchestra.run.smithi026.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T21:09:33.170 INFO:teuthology.orchestra.run.smithi026.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T21:09:33.170 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T21:09:33.170 INFO:teuthology.orchestra.run.smithi026.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T21:09:33.170 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T21:09:33.179 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:09:33.219 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:09:33.221 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:09:33.222 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:09:33.322 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:09:33.369 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:09:33.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:09:33 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/2421260243' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:09:33.626 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T21:09:33.633 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:09:33 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/2421260243' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:09:33.633 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:09:33 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/2421260243' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:09:33.638 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:09:33.638 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:09:33.675 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 6, 4, 5] out_osds: [7] dead_osds: [] live_osds: [0, 2, 3, 4, 5, 7, 6, 1] 2023-09-23T21:09:33.675 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T21:09:33.675 INFO:tasks.thrashosds.thrasher:fixing pg num pool .mgr 2023-09-23T21:09:33.675 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T21:09:33.739 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:09:33.768 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:09:33.806 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:09:33.810 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:09:33.810 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:09:33.911 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:09:33.939 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:09:33.976 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:09:33.981 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:09:33.981 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:09:34.082 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:09:34.111 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:09:34.151 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:09:34.154 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:09:34.154 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:09:34.255 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:09:34.255 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:09:34 smithi026 ceph-mon[175151]: pgmap v1062: 109 pgs: 109 active+clean; 1.5 GiB data, 7.5 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 653 op/s; 16 MiB/s, 258 objects/s recovering 2023-09-23T21:09:34.256 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:09:34 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/1881335134' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T21:09:34.256 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:09:34 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T21:09:34.256 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:09:34 smithi026 ceph-mon[179688]: pgmap v1062: 109 pgs: 109 active+clean; 1.5 GiB data, 7.5 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 653 op/s; 16 MiB/s, 258 objects/s recovering 2023-09-23T21:09:34.256 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:09:34 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/1881335134' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T21:09:34.256 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:09:34 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T21:09:34.310 INFO:teuthology.orchestra.run.smithi026.stderr:noscrub is unset 2023-09-23T21:09:34.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:09:34 smithi070 ceph-mon[168282]: pgmap v1062: 109 pgs: 109 active+clean; 1.5 GiB data, 7.5 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 653 op/s; 16 MiB/s, 258 objects/s recovering 2023-09-23T21:09:34.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:09:34 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/1881335134' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T21:09:34.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:09:34 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T21:09:34.371 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:09:35.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:09:35 smithi070 ceph-mon[168282]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:09:35.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:09:35 smithi070 ceph-mon[168282]: failed to encode map e411 with expected crc 2023-09-23T21:09:35.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:09:35 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T21:09:35.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:09:35 smithi070 ceph-mon[168282]: osdmap e411: 8 total, 8 up, 7 in 2023-09-23T21:09:35.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:09:35 smithi070 ceph-mon[168282]: failed to encode map e411 with expected crc 2023-09-23T21:09:35.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:09:35 smithi070 ceph-mon[168282]: failed to encode map e411 with expected crc 2023-09-23T21:09:35.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:09:35 smithi070 ceph-mon[168282]: failed to encode map e411 with expected crc 2023-09-23T21:09:35.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:09:35 smithi070 ceph-mon[168282]: failed to encode map e411 with expected crc 2023-09-23T21:09:35.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:09:35 smithi070 ceph-mon[168282]: 1.5 scrub starts 2023-09-23T21:09:35.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:09:35 smithi070 ceph-mon[168282]: 1.5 scrub ok 2023-09-23T21:09:35.631 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:09:35 smithi026 ceph-mon[179688]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:09:35.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:09:35 smithi026 ceph-mon[179688]: failed to encode map e411 with expected crc 2023-09-23T21:09:35.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:09:35 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T21:09:35.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:09:35 smithi026 ceph-mon[179688]: osdmap e411: 8 total, 8 up, 7 in 2023-09-23T21:09:35.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:09:35 smithi026 ceph-mon[179688]: failed to encode map e411 with expected crc 2023-09-23T21:09:35.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:09:35 smithi026 ceph-mon[179688]: failed to encode map e411 with expected crc 2023-09-23T21:09:35.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:09:35 smithi026 ceph-mon[179688]: failed to encode map e411 with expected crc 2023-09-23T21:09:35.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:09:35 smithi026 ceph-mon[179688]: failed to encode map e411 with expected crc 2023-09-23T21:09:35.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:09:35 smithi026 ceph-mon[179688]: 1.5 scrub starts 2023-09-23T21:09:35.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:09:35 smithi026 ceph-mon[179688]: 1.5 scrub ok 2023-09-23T21:09:35.636 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:09:35 smithi026 ceph-mon[175151]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:09:35.636 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:09:35 smithi026 ceph-mon[175151]: failed to encode map e411 with expected crc 2023-09-23T21:09:35.637 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:09:35 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T21:09:35.637 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:09:35 smithi026 ceph-mon[175151]: osdmap e411: 8 total, 8 up, 7 in 2023-09-23T21:09:35.637 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:09:35 smithi026 ceph-mon[175151]: failed to encode map e411 with expected crc 2023-09-23T21:09:35.637 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:09:35 smithi026 ceph-mon[175151]: failed to encode map e411 with expected crc 2023-09-23T21:09:35.637 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:09:35 smithi026 ceph-mon[175151]: failed to encode map e411 with expected crc 2023-09-23T21:09:35.637 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:09:35 smithi026 ceph-mon[175151]: failed to encode map e411 with expected crc 2023-09-23T21:09:35.637 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:09:35 smithi026 ceph-mon[175151]: 1.5 scrub starts 2023-09-23T21:09:35.637 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:09:35 smithi026 ceph-mon[175151]: 1.5 scrub ok 2023-09-23T21:09:36.631 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:09:36 smithi026 ceph-mon[175151]: pgmap v1064: 109 pgs: 109 active+clean; 1.3 GiB data, 7.1 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 778 op/s; 8.5 MiB/s, 136 objects/s recovering 2023-09-23T21:09:36.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:09:36 smithi026 ceph-mon[179688]: pgmap v1064: 109 pgs: 109 active+clean; 1.3 GiB data, 7.1 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 778 op/s; 8.5 MiB/s, 136 objects/s recovering 2023-09-23T21:09:36.635 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:09:36.644 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:09:36.645 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:09:36.746 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:09:36.774 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:09:36 smithi070 ceph-mon[168282]: pgmap v1064: 109 pgs: 109 active+clean; 1.3 GiB data, 7.1 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 778 op/s; 8.5 MiB/s, 136 objects/s recovering 2023-09-23T21:09:36.776 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:09:36.817 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:09:36.822 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:09:36.822 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:09:36.923 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:09:37.003 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:09:37.710 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T21:09:37.763 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:09:37 smithi026 ceph-mon[179688]: 1.d scrub starts 2023-09-23T21:09:37.763 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:09:37 smithi026 ceph-mon[179688]: 1.d scrub ok 2023-09-23T21:09:37.763 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:09:37 smithi026 ceph-mon[175151]: 1.d scrub starts 2023-09-23T21:09:37.763 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:09:37 smithi026 ceph-mon[175151]: 1.d scrub ok 2023-09-23T21:09:37.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:09:37 smithi070 ceph-mon[168282]: 1.d scrub starts 2023-09-23T21:09:37.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:09:37 smithi070 ceph-mon[168282]: 1.d scrub ok 2023-09-23T21:09:37.969 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:09:37.994 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:09:37.995 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:09:38.096 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:09:38.177 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:09:38.596 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T21:09:38.612 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:09:38.613 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:09:38.655 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:09:38 smithi026 ceph-mon[175151]: pgmap v1065: 109 pgs: 109 active+clean; 1.3 GiB data, 7.1 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 633 op/s; 4.7 MiB/s, 74 objects/s recovering 2023-09-23T21:09:38.655 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:09:38 smithi026 ceph-mon[179688]: pgmap v1065: 109 pgs: 109 active+clean; 1.3 GiB data, 7.1 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 633 op/s; 4.7 MiB/s, 74 objects/s recovering 2023-09-23T21:09:38.714 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:09:38.847 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:09:38.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:09:38 smithi070 ceph-mon[168282]: pgmap v1065: 109 pgs: 109 active+clean; 1.3 GiB data, 7.1 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 633 op/s; 4.7 MiB/s, 74 objects/s recovering 2023-09-23T21:09:38.913 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:09:38.940 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T21:09:38.950 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:09:38.951 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:09:38.982 INFO:teuthology.orchestra.run.smithi026.stdout:ERROR: (22) Invalid argument 2023-09-23T21:09:38.982 INFO:teuthology.orchestra.run.smithi026.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T21:09:39.052 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:09:39.081 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:09:39.120 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:09:39.123 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:09:39.123 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:09:39.224 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:09:39.291 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:09:39.612 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 21:09:39 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[164029]: ::ffff:172.21.15.70 - - [23/Sep/2023:21:09:39] "GET /metrics HTTP/1.1" 200 36349 "" "Prometheus/2.43.0" 2023-09-23T21:09:39.652 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:09:39.659 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:09:39.659 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:09:39.676 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_ops_in_flight 2023-09-23T21:09:39.761 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:09:39.839 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:09:40.017 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T21:09:40.068 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:09:40.091 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:09:40.092 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:09:40.193 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:09:40.297 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:09:40.647 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:09:40.651 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:09:40.655 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:09:40.655 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:09:40.756 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:09:40.825 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:09:40 smithi026 ceph-mon[179688]: pgmap v1066: 109 pgs: 109 active+clean; 1.1 GiB data, 6.5 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 787 op/s; 3.8 MiB/s, 61 objects/s recovering 2023-09-23T21:09:40.825 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:09:40 smithi026 ceph-mon[175151]: pgmap v1066: 109 pgs: 109 active+clean; 1.1 GiB data, 6.5 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 787 op/s; 3.8 MiB/s, 61 objects/s recovering 2023-09-23T21:09:40.832 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:09:40.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:09:40 smithi070 ceph-mon[168282]: pgmap v1066: 109 pgs: 109 active+clean; 1.1 GiB data, 6.5 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 787 op/s; 3.8 MiB/s, 61 objects/s recovering 2023-09-23T21:09:42.119 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:09:42.125 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:09:42.125 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:09:42.226 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:09:42.255 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:09:42.296 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:09:42.298 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:09:42.299 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:09:42.399 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:09:42.430 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:09:42.473 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:09:42.475 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:09:42.476 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:09:42.576 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:09:42.603 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:09:42 smithi070 ceph-mon[168282]: pgmap v1067: 109 pgs: 109 active+clean; 1.0 GiB data, 6.3 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 861 op/s; 376 KiB/s, 5 objects/s recovering 2023-09-23T21:09:42.603 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:09:42 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:09:42.606 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:09:42.646 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:09:42.649 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:09:42.649 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:09:42.750 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:09:42.778 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:09:42.821 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:09:42.823 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:09:42.824 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:09:42.881 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:09:42 smithi026 ceph-mon[175151]: pgmap v1067: 109 pgs: 109 active+clean; 1.0 GiB data, 6.3 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 861 op/s; 376 KiB/s, 5 objects/s recovering 2023-09-23T21:09:42.881 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:09:42 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:09:42.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:09:42 smithi026 ceph-mon[179688]: pgmap v1067: 109 pgs: 109 active+clean; 1.0 GiB data, 6.3 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 861 op/s; 376 KiB/s, 5 objects/s recovering 2023-09-23T21:09:42.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:09:42 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:09:42.925 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:09:42.957 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:09:43.000 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:09:43.002 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:09:43.003 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:09:43.103 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:09:43.198 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:09:43.295 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T21:09:43.308 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:09:43.309 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:09:43.410 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:09:43.443 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:09:43.483 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:09:43.485 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:09:43.486 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:09:43.584 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:09:43.584 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:09:43.586 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:09:43.677 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:09:44.612 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:09:44 smithi026 ceph-mon[179688]: pgmap v1068: 109 pgs: 109 active+clean; 953 MiB data, 5.9 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 917 op/s 2023-09-23T21:09:44.612 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:09:44 smithi026 ceph-mon[179688]: 1.e scrub starts 2023-09-23T21:09:44.612 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:09:44 smithi026 ceph-mon[179688]: 1.e scrub ok 2023-09-23T21:09:44.618 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:09:44 smithi026 ceph-mon[175151]: pgmap v1068: 109 pgs: 109 active+clean; 953 MiB data, 5.9 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 917 op/s 2023-09-23T21:09:44.618 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:09:44 smithi026 ceph-mon[175151]: 1.e scrub starts 2023-09-23T21:09:44.618 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:09:44 smithi026 ceph-mon[175151]: 1.e scrub ok 2023-09-23T21:09:44.646 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:09:44.659 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:09:44.659 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:09:44.760 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:09:44.787 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:09:44 smithi070 ceph-mon[168282]: pgmap v1068: 109 pgs: 109 active+clean; 953 MiB data, 5.9 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 917 op/s 2023-09-23T21:09:44.787 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:09:44 smithi070 ceph-mon[168282]: 1.e scrub starts 2023-09-23T21:09:44.787 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:09:44 smithi070 ceph-mon[168282]: 1.e scrub ok 2023-09-23T21:09:44.787 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:09:44.826 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:09:44.829 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:09:44.829 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:09:44.930 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:09:45.044 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:09:45.389 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T21:09:45.397 INFO:teuthology.orchestra.run.smithi026.stdout: 2023-09-23T21:09:45.399 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:09:45.400 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:09:45.417 INFO:teuthology.orchestra.run.smithi026.stderr:dumped all 2023-09-23T21:09:45.501 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:09:45.584 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:09:46.166 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:09:46.173 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:09:46.173 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:09:46.274 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:09:46.342 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:09:46.477 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:09:46 smithi026 ceph-mon[175151]: pgmap v1069: 109 pgs: 109 active+clean; 774 MiB data, 5.4 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 875 op/s 2023-09-23T21:09:46.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:09:46 smithi070 ceph-mon[168282]: pgmap v1069: 109 pgs: 109 active+clean; 774 MiB data, 5.4 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 875 op/s 2023-09-23T21:09:46.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:09:46 smithi070 ceph-mon[168282]: from='client.46244 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T21:09:46.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:09:46 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:09:46.881 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:09:46 smithi026 ceph-mon[179688]: pgmap v1069: 109 pgs: 109 active+clean; 774 MiB data, 5.4 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 875 op/s 2023-09-23T21:09:46.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:09:46 smithi026 ceph-mon[179688]: from='client.46244 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T21:09:46.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:09:46 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:09:46.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:09:46 smithi026 ceph-mon[175151]: from='client.46244 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T21:09:46.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:09:46 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:09:47.962 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:09:47.967 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:09:47.968 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:09:48.068 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:09:48.154 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:09:48.871 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:09:48 smithi070 ceph-mon[168282]: pgmap v1070: 109 pgs: 109 active+clean; 758 MiB data, 5.3 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 798 op/s 2023-09-23T21:09:48.881 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:09:48 smithi026 ceph-mon[179688]: pgmap v1070: 109 pgs: 109 active+clean; 758 MiB data, 5.3 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 798 op/s 2023-09-23T21:09:48.885 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:09:48 smithi026 ceph-mon[175151]: pgmap v1070: 109 pgs: 109 active+clean; 758 MiB data, 5.3 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 798 op/s 2023-09-23T21:09:49.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:09:49 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T21:09:49.883 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:09:49 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T21:09:49.884 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 21:09:49 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[164029]: ::ffff:172.21.15.70 - - [23/Sep/2023:21:09:49] "GET /metrics HTTP/1.1" 200 36315 "" "Prometheus/2.43.0" 2023-09-23T21:09:49.884 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:09:49 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T21:09:50.804 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:09:50.808 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:09:50.809 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:09:50.809 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:09:50 smithi026 ceph-mon[179688]: pgmap v1071: 109 pgs: 109 active+clean; 591 MiB data, 4.8 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 999 op/s 2023-09-23T21:09:50.809 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:09:50 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:09:50.809 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:09:50 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:09:50.809 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:09:50 smithi026 ceph-mon[175151]: pgmap v1071: 109 pgs: 109 active+clean; 591 MiB data, 4.8 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 999 op/s 2023-09-23T21:09:50.809 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:09:50 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:09:50.810 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:09:50 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:09:50.872 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:09:50 smithi070 ceph-mon[168282]: pgmap v1071: 109 pgs: 109 active+clean; 591 MiB data, 4.8 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 999 op/s 2023-09-23T21:09:50.872 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:09:50 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:09:50.872 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:09:50 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:09:50.909 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:09:50.950 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:09:51.045 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:09:51.049 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:09:51.050 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:09:51.150 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:09:51.224 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:09:51.381 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T21:09:51.406 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:09:51.407 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:09:51.449 INFO:teuthology.orchestra.run.smithi026.stdout:{"pg_ready":true,"pg_map":{"version":1069,"stamp":"2023-09-23T21:09:44.939082+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":811866040,"num_objects":12404,"num_object_clones":0,"num_object_copies":37212,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":12404,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":214229,"num_write_kb":7249502,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":23372,"num_bytes_recovered":1528831158,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":86436,"ondisk_log_size":86436,"up":327,"acting":327,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":328,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":5692156,"kb_used_data":2631876,"kb_used_omap":137,"kb_used_meta":3059638,"kb_avail":650523908,"statfs":{"total":671965249536,"available":666136481792,"internally_reserved":0,"allocated":2695041024,"data_stored":2684159515,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":141000,"internal_metadata":3133069624},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":6,"apply_latency_ms":6,"commit_latency_ns":6000000,"apply_latency_ns":6000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-613339558,"num_objects":-9358,"num_object_clones":0,"num_object_copies":-28074,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":-9358,"num_whiteouts":0,"num_read":-44,"num_read_kb":-83,"num_write":9336,"num_write_kb":-220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":-1,"num_bytes_recovered":-161010,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.661089"},"pg_stats":[{"pgid":"3.1a","version":"68'1","reported_seq":26,"reported_epoch":410,"state":"active+clean","last_fresh":"2023-09-23T21:09:28.716872+0000","last_change":"2023-09-23T21:09:08.416508+0000","last_active":"2023-09-23T21:09:28.716872+0000","last_peered":"2023-09-23T21:09:28.716872+0000","last_clean":"2023-09-23T21:09:28.716872+0000","last_became_active":"2023-09-23T21:09:08.416124+0000","last_became_peered":"2023-09-23T21:09:08.416124+0000","last_unstale":"2023-09-23T21:09:28.716872+0000","last_undegraded":"2023-09-23T21:09:28.716872+0000","last_fullsized":"2023-09-23T21:09:28.716872+0000","mapping_epoch":396,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":397,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:46:38.597918+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,0],"acting":[2,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":407,"reported_epoch":410,"state":"active+clean","last_fresh":"2023-09-23T21:09:28.715092+0000","last_change":"2023-09-23T20:54:13.942019+0000","last_active":"2023-09-23T21:09:28.715092+0000","last_peered":"2023-09-23T21:09:28.715092+0000","last_clean":"2023-09-23T21:09:28.715092+0000","last_became_active":"2023-09-23T20:54:13.941647+0000","last_became_peered":"2023-09-23T20:54:13.941647+0000","last_unstale":"2023-09-23T21:09:28.715092+0000","last_undegraded":"2023-09-23T21:09:28.715092+0000","last_fullsized":"2023-09-23T21:09:28.715092+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:30:54.400944+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":393,"reported_epoch":410,"state":"active+clean","last_fresh":"2023-09-23T21:09:28.746384+0000","last_change":"2023-09-23T21:09:08.428579+0000","last_active":"2023-09-23T21:09:28.746384+0000","last_peered":"2023-09-23T21:09:28.746384+0000","last_clean":"2023-09-23T21:09:28.746384+0000","last_became_active":"2023-09-23T21:09:08.428247+0000","last_became_peered":"2023-09-23T21:09:08.428247+0000","last_unstale":"2023-09-23T21:09:28.746384+0000","last_undegraded":"2023-09-23T21:09:28.746384+0000","last_fullsized":"2023-09-23T21:09:28.746384+0000","mapping_epoch":396,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":397,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:47:56.458062+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":410,"reported_epoch":410,"state":"active+clean","last_fresh":"2023-09-23T21:09:32.390907+0000","last_change":"2023-09-23T21:08:43.513644+0000","last_active":"2023-09-23T21:09:32.390907+0000","last_peered":"2023-09-23T21:09:32.390907+0000","last_clean":"2023-09-23T21:09:32.390907+0000","last_became_active":"2023-09-23T21:08:43.512977+0000","last_became_peered":"2023-09-23T21:08:43.512977+0000","last_unstale":"2023-09-23T21:09:32.390907+0000","last_undegraded":"2023-09-23T21:09:32.390907+0000","last_fullsized":"2023-09-23T21:09:32.390907+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:42:59.825772+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":413,"reported_epoch":410,"state":"active+clean","last_fresh":"2023-09-23T21:09:28.712801+0000","last_change":"2023-09-23T21:05:55.325654+0000","last_active":"2023-09-23T21:09:28.712801+0000","last_peered":"2023-09-23T21:09:28.712801+0000","last_clean":"2023-09-23T21:09:28.712801+0000","last_became_active":"2023-09-23T21:05:55.325298+0000","last_became_peered":"2023-09-23T21:05:55.325298+0000","last_unstale":"2023-09-23T21:09:28.712801+0000","last_undegraded":"2023-09-23T21:09:28.712801+0000","last_fullsized":"2023-09-23T21:09:28.712801+0000","mapping_epoch":350,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":351,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:33:19.561063+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":413,"reported_epoch":410,"state":"active+clean","last_fresh":"2023-09-23T21:09:28.716153+0000","last_change":"2023-09-23T21:05:55.319892+0000","last_active":"2023-09-23T21:09:28.716153+0000","last_peered":"2023-09-23T21:09:28.716153+0000","last_clean":"2023-09-23T21:09:28.716153+0000","last_became_active":"2023-09-23T21:05:55.319536+0000","last_became_peered":"2023-09-23T21:05:55.319536+0000","last_unstale":"2023-09-23T21:09:28.716153+0000","last_undegraded":"2023-09-23T21:09:28.716153+0000","last_fullsized":"2023-09-23T21:09:28.716153+0000","mapping_epoch":350,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":351,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:14:48.388831+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":408,"reported_epoch":410,"state":"active+clean","last_fresh":"2023-09-23T21:09:28.715459+0000","last_change":"2023-09-23T21:05:04.304358+0000","last_active":"2023-09-23T21:09:28.715459+0000","last_peered":"2023-09-23T21:09:28.715459+0000","last_clean":"2023-09-23T21:09:28.715459+0000","last_became_active":"2023-09-23T21:05:04.303913+0000","last_became_peered":"2023-09-23T21:05:04.303913+0000","last_unstale":"2023-09-23T21:09:28.715459+0000","last_undegraded":"2023-09-23T21:09:28.715459+0000","last_fullsized":"2023-09-23T21:09:28.715459+0000","mapping_epoch":332,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:21:21.860944+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":428,"reported_epoch":410,"state":"active+clean","last_fresh":"2023-09-23T21:09:28.712860+0000","last_change":"2023-09-23T21:09:08.423849+0000","last_active":"2023-09-23T21:09:28.712860+0000","last_peered":"2023-09-23T21:09:28.712860+0000","last_clean":"2023-09-23T21:09:28.712860+0000","last_became_active":"2023-09-23T21:09:08.423444+0000","last_became_peered":"2023-09-23T21:09:08.423444+0000","last_unstale":"2023-09-23T21:09:28.712860+0000","last_undegraded":"2023-09-23T21:09:28.712860+0000","last_fullsized":"2023-09-23T21:09:28.712860+0000","mapping_epoch":396,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":397,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:51:39.763421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,5],"acting":[3,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"79'161","reported_seq":736,"reported_epoch":410,"state":"active+clean","last_fresh":"2023-09-23T21:09:28.717075+0000","last_change":"2023-09-23T21:09:11.387858+0000","last_active":"2023-09-23T21:09:28.717075+0000","last_peered":"2023-09-23T21:09:28.717075+0000","last_clean":"2023-09-23T21:09:28.717075+0000","last_became_active":"2023-09-23T21:09:11.387462+0000","last_became_peered":"2023-09-23T21:09:11.387462+0000","last_unstale":"2023-09-23T21:09:28.717075+0000","last_undegraded":"2023-09-23T21:09:28.717075+0000","last_fullsized":"2023-09-23T21:09:28.717075+0000","mapping_epoch":399,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":400,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:25.096401+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:25.096401+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:00:43.986148+0000","scrub_duration":0.002859393,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,0],"acting":[2,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":115,"reported_epoch":410,"state":"active+clean","last_fresh":"2023-09-23T21:09:28.718553+0000","last_change":"2023-09-23T21:08:43.499467+0000","last_active":"2023-09-23T21:09:28.718553+0000","last_peered":"2023-09-23T21:09:28.718553+0000","last_clean":"2023-09-23T21:09:28.718553+0000","last_became_active":"2023-09-23T21:08:43.499142+0000","last_became_peered":"2023-09-23T21:08:43.499142+0000","last_unstale":"2023-09-23T21:09:28.718553+0000","last_undegraded":"2023-09-23T21:09:28.718553+0000","last_fullsized":"2023-09-23T21:09:28.718553+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:25:25.593675+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":365,"reported_epoch":410,"state":"active+clean","last_fresh":"2023-09-23T21:09:28.712882+0000","last_change":"2023-09-23T21:08:43.494810+0000","last_active":"2023-09-23T21:09:28.712882+0000","last_peered":"2023-09-23T21:09:28.712882+0000","last_clean":"2023-09-23T21:09:28.712882+0000","last_became_active":"2023-09-23T21:08:43.494013+0000","last_became_peered":"2023-09-23T21:08:43.494013+0000","last_unstale":"2023-09-23T21:09:28.712882+0000","last_undegraded":"2023-09-23T21:09:28.712882+0000","last_fullsized":"2023-09-23T21:09:28.712882+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:32:22.787562+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":263,"reported_epoch":410,"state":"active+clean","last_fresh":"2023-09-23T21:09:28.715196+0000","last_change":"2023-09-23T21:09:08.475715+0000","last_active":"2023-09-23T21:09:28.715196+0000","last_peered":"2023-09-23T21:09:28.715196+0000","last_clean":"2023-09-23T21:09:28.715196+0000","last_became_active":"2023-09-23T21:09:08.442399+0000","last_became_peered":"2023-09-23T21:09:08.442399+0000","last_unstale":"2023-09-23T21:09:28.715196+0000","last_undegraded":"2023-09-23T21:09:28.715196+0000","last_fullsized":"2023-09-23T21:09:28.715196+0000","mapping_epoch":396,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":397,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:01:52.337007+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,2],"acting":[3,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":93,"reported_epoch":410,"state":"active+clean","last_fresh":"2023-09-23T21:09:28.747024+0000","last_change":"2023-09-23T21:08:54.693560+0000","last_active":"2023-09-23T21:09:28.747024+0000","last_peered":"2023-09-23T21:09:28.747024+0000","last_clean":"2023-09-23T21:09:28.747024+0000","last_became_active":"2023-09-23T21:08:54.693199+0000","last_became_peered":"2023-09-23T21:08:54.693199+0000","last_unstale":"2023-09-23T21:09:28.747024+0000","last_undegraded":"2023-09-23T21:09:28.747024+0000","last_fullsized":"2023-09-23T21:09:28.747024+0000","mapping_epoch":388,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":389,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:09:53.015332+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"79'22","reported_seq":79,"reported_epoch":410,"state":"active+clean","last_fresh":"2023-09-23T21:09:28.716315+0000","last_change":"2023-09-23T21:09:08.463012+0000","last_active":"2023-09-23T21:09:28.716315+0000","last_peered":"2023-09-23T21:09:28.716315+0000","last_clean":"2023-09-23T21:09:28.716315+0000","last_became_active":"2023-09-23T21:09:08.410151+0000","last_became_peered":"2023-09-23T21:09:08.410151+0000","last_unstale":"2023-09-23T21:09:28.716315+0000","last_undegraded":"2023-09-23T21:09:28.716315+0000","last_fullsized":"2023-09-23T21:09:28.716315+0000","mapping_epoch":396,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":397,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:33:24.349490+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4734,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,0],"acting":[6,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":322,"reported_epoch":410,"state":"active+clean","last_fresh":"2023-09-23T21:09:28.715201+0000","last_change":"2023-09-23T21:05:55.325203+0000","last_active":"2023-09-23T21:09:28.715201+0000","last_peered":"2023-09-23T21:09:28.715201+0000","last_clean":"2023-09-23T21:09:28.715201+0000","last_became_active":"2023-09-23T21:05:55.323957+0000","last_became_peered":"2023-09-23T21:05:55.323957+0000","last_unstale":"2023-09-23T21:09:28.715201+0000","last_undegraded":"2023-09-23T21:09:28.715201+0000","last_fullsized":"2023-09-23T21:09:28.715201+0000","mapping_epoch":350,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":351,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:03:20.418924+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.11","version":"79'161","reported_seq":678,"reported_epoch":410,"state":"active+clean","last_fresh":"2023-09-23T21:09:28.714568+0000","last_change":"2023-09-23T21:05:07.301125+0000","last_active":"2023-09-23T21:09:28.714568+0000","last_peered":"2023-09-23T21:09:28.714568+0000","last_clean":"2023-09-23T21:09:28.714568+0000","last_became_active":"2023-09-23T21:05:07.300809+0000","last_became_peered":"2023-09-23T21:05:07.300809+0000","last_unstale":"2023-09-23T21:09:28.714568+0000","last_undegraded":"2023-09-23T21:09:28.714568+0000","last_fullsized":"2023-09-23T21:09:28.714568+0000","mapping_epoch":335,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":336,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:28.756634+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:28.756634+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:28:29.079665+0000","scrub_duration":0.0064925119999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":42,"reported_epoch":410,"state":"active+clean","last_fresh":"2023-09-23T21:09:28.716786+0000","last_change":"2023-09-23T21:09:08.453198+0000","last_active":"2023-09-23T21:09:28.716786+0000","last_peered":"2023-09-23T21:09:28.716786+0000","last_clean":"2023-09-23T21:09:28.716786+0000","last_became_active":"2023-09-23T21:09:08.409264+0000","last_became_peered":"2023-09-23T21:09:08.409264+0000","last_unstale":"2023-09-23T21:09:28.716786+0000","last_undegraded":"2023-09-23T21:09:28.716786+0000","last_fullsized":"2023-09-23T21:09:28.716786+0000","mapping_epoch":396,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":397,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:10:21.176747+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,0],"acting":[2,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":404,"reported_epoch":410,"state":"active+clean","last_fresh":"2023-09-23T21:09:28.715442+0000","last_change":"2023-09-23T21:08:43.495331+0000","last_active":"2023-09-23T21:09:28.715442+0000","last_peered":"2023-09-23T21:09:28.715442+0000","last_clean":"2023-09-23T21:09:28.715442+0000","last_became_active":"2023-09-23T21:08:43.494485+0000","last_became_peered":"2023-09-23T21:08:43.494485+0000","last_unstale":"2023-09-23T21:09:28.715442+0000","last_undegraded":"2023-09-23T21:09:28.715442+0000","last_fullsized":"2023-09-23T21:09:28.715442+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:15:19.762943+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.10","version":"79'161","reported_seq":520,"reported_epoch":410,"state":"active+clean","last_fresh":"2023-09-23T21:09:28.714337+0000","last_change":"2023-09-23T21:09:11.388116+0000","last_active":"2023-09-23T21:09:28.714337+0000","last_peered":"2023-09-23T21:09:28.714337+0000","last_clean":"2023-09-23T21:09:28.714337+0000","last_became_active":"2023-09-23T21:09:11.387118+0000","last_became_peered":"2023-09-23T21:09:11.387118+0000","last_unstale":"2023-09-23T21:09:28.714337+0000","last_undegraded":"2023-09-23T21:09:28.714337+0000","last_fullsized":"2023-09-23T21:09:28.714337+0000","mapping_epoch":399,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":400,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:06:50.336992+0000","last_deep_scrub":"79'161","last_deep_scrub_stamp":"2023-09-23T21:06:50.336992+0000","last_clean_scrub_stamp":"2023-09-23T21:06:50.336992+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:02:03.484838+0000","scrub_duration":0.0026150930000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":399,"reported_epoch":410,"state":"active+clean","last_fresh":"2023-09-23T21:09:28.714795+0000","last_change":"2023-09-23T21:08:43.512749+0000","last_active":"2023-09-23T21:09:28.714795+0000","last_peered":"2023-09-23T21:09:28.714795+0000","last_clean":"2023-09-23T21:09:28.714795+0000","last_became_active":"2023-09-23T21:08:43.512299+0000","last_became_peered":"2023-09-23T21:08:43.512299+0000","last_unstale":"2023-09-23T21:09:28.714795+0000","last_undegraded":"2023-09-23T21:09:28.714795+0000","last_fullsized":"2023-09-23T21:09:28.714795+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:19:58.835209+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":143,"reported_epoch":411,"state":"active+clean","last_fresh":"2023-09-23T21:09:34.289719+0000","last_change":"2023-09-23T21:08:43.496213+0000","last_active":"2023-09-23T21:09:34.289719+0000","last_peered":"2023-09-23T21:09:34.289719+0000","last_clean":"2023-09-23T21:09:34.289719+0000","last_became_active":"2023-09-23T21:08:43.495883+0000","last_became_peered":"2023-09-23T21:08:43.495883+0000","last_unstale":"2023-09-23T21:09:34.289719+0000","last_undegraded":"2023-09-23T21:09:34.289719+0000","last_fullsized":"2023-09-23T21:09:34.289719+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:27:12.483033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.13","version":"79'161","reported_seq":702,"reported_epoch":410,"state":"active+clean","last_fresh":"2023-09-23T21:09:28.746661+0000","last_change":"2023-09-23T21:09:11.416774+0000","last_active":"2023-09-23T21:09:28.746661+0000","last_peered":"2023-09-23T21:09:28.746661+0000","last_clean":"2023-09-23T21:09:28.746661+0000","last_became_active":"2023-09-23T21:09:11.416431+0000","last_became_peered":"2023-09-23T21:09:11.416431+0000","last_unstale":"2023-09-23T21:09:28.746661+0000","last_undegraded":"2023-09-23T21:09:28.746661+0000","last_fullsized":"2023-09-23T21:09:28.746661+0000","mapping_epoch":399,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":400,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:36.420362+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:36.420362+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:51:54.917367+0000","scrub_duration":0.028838444000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,0],"acting":[1,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":365,"reported_epoch":410,"state":"active+clean","last_fresh":"2023-09-23T21:09:28.715117+0000","last_change":"2023-09-23T21:08:43.516055+0000","last_active":"2023-09-23T21:09:28.715117+0000","last_peered":"2023-09-23T21:09:28.715117+0000","last_clean":"2023-09-23T21:09:28.715117+0000","last_became_active":"2023-09-23T21:08:43.515799+0000","last_became_peered":"2023-09-23T21:08:43.515799+0000","last_unstale":"2023-09-23T21:09:28.715117+0000","last_undegraded":"2023-09-23T21:09:28.715117+0000","last_fullsized":"2023-09-23T21:09:28.715117+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:05:28.535911+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":381,"reported_epoch":410,"state":"active+clean","last_fresh":"2023-09-23T21:09:28.716882+0000","last_change":"2023-09-23T21:08:43.513369+0000","last_active":"2023-09-23T21:09:28.716882+0000","last_peered":"2023-09-23T21:09:28.716882+0000","last_clean":"2023-09-23T21:09:28.716882+0000","last_became_active":"2023-09-23T21:08:43.512435+0000","last_became_peered":"2023-09-23T21:08:43.512435+0000","last_unstale":"2023-09-23T21:09:28.716882+0000","last_undegraded":"2023-09-23T21:09:28.716882+0000","last_fullsized":"2023-09-23T21:09:28.716882+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:00:10.970062+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.12","version":"79'161","reported_seq":572,"reported_epoch":410,"state":"active+clean","last_fresh":"2023-09-23T21:09:28.718766+0000","last_change":"2023-09-23T21:05:56.785264+0000","last_active":"2023-09-23T21:09:28.718766+0000","last_peered":"2023-09-23T21:09:28.718766+0000","last_clean":"2023-09-23T21:09:28.718766+0000","last_became_active":"2023-09-23T21:05:55.322810+0000","last_became_peered":"2023-09-23T21:05:55.322810+0000","last_unstale":"2023-09-23T21:09:28.718766+0000","last_undegraded":"2023-09-23T21:09:28.718766+0000","last_fullsized":"2023-09-23T21:09:28.718766+0000","mapping_epoch":350,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":351,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:05:56.785173+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:05:56.785173+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:56:12.932786+0000","scrub_duration":0.0042114370000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":443,"reported_epoch":410,"state":"active+clean","last_fresh":"2023-09-23T21:09:28.715751+0000","last_change":"2023-09-23T21:05:04.332774+0000","last_active":"2023-09-23T21:09:28.715751+0000","last_peered":"2023-09-23T21:09:28.715751+0000","last_clean":"2023-09-23T21:09:28.715751+0000","last_became_active":"2023-09-23T21:05:04.304541+0000","last_became_peered":"2023-09-23T21:05:04.304541+0000","last_unstale":"2023-09-23T21:09:28.715751+0000","last_undegraded":"2023-09-23T21:09:28.715751+0000","last_fullsized":"2023-09-23T21:09:28.715751+0000","mapping_epoch":332,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:27:02.401939+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":227,"reported_epoch":410,"state":"active+clean","last_fresh":"2023-09-23T21:09:28.715186+0000","last_change":"2023-09-23T21:08:43.494461+0000","last_active":"2023-09-23T21:09:28.715186+0000","last_peered":"2023-09-23T21:09:28.715186+0000","last_clean":"2023-09-23T21:09:28.715186+0000","last_became_active":"2023-09-23T21:08:43.493566+0000","last_became_peered":"2023-09-23T21:08:43.493566+0000","last_unstale":"2023-09-23T21:09:28.715186+0000","last_undegraded":"2023-09-23T21:09:28.715186+0000","last_fullsized":"2023-09-23T21:09:28.715186+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:25:54.987910+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.5","version":"411'13234","reported_seq":17504,"reported_epoch":411,"state":"active+clean","last_fresh":"2023-09-23T21:09:42.395492+0000","last_change":"2023-09-23T21:09:21.417196+0000","last_active":"2023-09-23T21:09:42.395492+0000","last_peered":"2023-09-23T21:09:42.395492+0000","last_clean":"2023-09-23T21:09:42.395492+0000","last_became_active":"2023-09-23T21:09:21.379729+0000","last_became_peered":"2023-09-23T21:09:21.379729+0000","last_unstale":"2023-09-23T21:09:42.395492+0000","last_undegraded":"2023-09-23T21:09:42.395492+0000","last_fullsized":"2023-09-23T21:09:42.395492+0000","mapping_epoch":404,"log_start":"371'7779","ondisk_log_start":"371'7779","created":346,"last_epoch_clean":405,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:05:47.261750+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:05:47.261750+0000","last_clean_scrub_stamp":"2023-09-23T21:05:47.261750+0000","objects_scrubbed":0,"log_size":5455,"log_dups_size":0,"ondisk_log_size":5455,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:49:21.608856+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":52822016,"num_objects":806,"num_object_clones":0,"num_object_copies":2418,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":806,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13234,"num_write_kb":449280,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2085,"num_bytes_recovered":136511488,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,1],"acting":[0,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.d","version":"79'161","reported_seq":515,"reported_epoch":411,"state":"active+clean","last_fresh":"2023-09-23T21:09:36.858120+0000","last_change":"2023-09-23T21:09:36.858214+0000","last_active":"2023-09-23T21:09:36.858120+0000","last_peered":"2023-09-23T21:09:36.858120+0000","last_clean":"2023-09-23T21:09:36.858120+0000","last_became_active":"2023-09-23T21:09:11.395441+0000","last_became_peered":"2023-09-23T21:09:11.395441+0000","last_unstale":"2023-09-23T21:09:36.858120+0000","last_undegraded":"2023-09-23T21:09:36.858120+0000","last_fullsized":"2023-09-23T21:09:36.858120+0000","mapping_epoch":399,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":400,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:09:36.858077+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:09:36.858077+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:47:50.795688+0000","scrub_duration":0.0024987500000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":40,"reported_epoch":410,"state":"active+clean","last_fresh":"2023-09-23T21:09:28.715329+0000","last_change":"2023-09-23T21:09:08.451787+0000","last_active":"2023-09-23T21:09:28.715329+0000","last_peered":"2023-09-23T21:09:28.715329+0000","last_clean":"2023-09-23T21:09:28.715329+0000","last_became_active":"2023-09-23T21:09:08.416379+0000","last_became_peered":"2023-09-23T21:09:08.416379+0000","last_unstale":"2023-09-23T21:09:28.715329+0000","last_undegraded":"2023-09-23T21:09:28.715329+0000","last_fullsized":"2023-09-23T21:09:28.715329+0000","mapping_epoch":396,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":397,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:23:42.852988+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,5],"acting":[0,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":108,"reported_epoch":410,"state":"active+clean","last_fresh":"2023-09-23T21:09:28.718128+0000","last_change":"2023-09-23T21:08:54.675136+0000","last_active":"2023-09-23T21:09:28.718128+0000","last_peered":"2023-09-23T21:09:28.718128+0000","last_clean":"2023-09-23T21:09:28.718128+0000","last_became_active":"2023-09-23T21:08:54.674772+0000","last_became_peered":"2023-09-23T21:08:54.674772+0000","last_unstale":"2023-09-23T21:09:28.718128+0000","last_undegraded":"2023-09-23T21:09:28.718128+0000","last_fullsized":"2023-09-23T21:09:28.718128+0000","mapping_epoch":388,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":389,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:16:55.575774+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,3],"acting":[5,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.4","version":"411'13233","reported_seq":13305,"reported_epoch":411,"state":"active+clean","last_fresh":"2023-09-23T21:09:43.519563+0000","last_change":"2023-09-23T21:05:55.320259+0000","last_active":"2023-09-23T21:09:43.519563+0000","last_peered":"2023-09-23T21:09:43.519563+0000","last_clean":"2023-09-23T21:09:43.519563+0000","last_became_active":"2023-09-23T21:05:55.319895+0000","last_became_peered":"2023-09-23T21:05:55.319895+0000","last_unstale":"2023-09-23T21:09:43.519563+0000","last_undegraded":"2023-09-23T21:09:43.519563+0000","last_fullsized":"2023-09-23T21:09:43.519563+0000","mapping_epoch":350,"log_start":"373'8307","ondisk_log_start":"373'8307","created":346,"last_epoch_clean":351,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:05:47.261750+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:05:47.261750+0000","last_clean_scrub_stamp":"2023-09-23T21:05:47.261750+0000","objects_scrubbed":0,"log_size":4926,"log_dups_size":0,"ondisk_log_size":4926,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:39:53.181523+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":50921472,"num_objects":777,"num_object_clones":0,"num_object_copies":2331,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":777,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13233,"num_write_kb":448320,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.c","version":"79'161","reported_seq":716,"reported_epoch":410,"state":"active+clean","last_fresh":"2023-09-23T21:09:28.714190+0000","last_change":"2023-09-23T20:57:22.041071+0000","last_active":"2023-09-23T21:09:28.714190+0000","last_peered":"2023-09-23T21:09:28.714190+0000","last_clean":"2023-09-23T21:09:28.714190+0000","last_became_active":"2023-09-23T20:57:15.973204+0000","last_became_peered":"2023-09-23T20:57:15.973204+0000","last_unstale":"2023-09-23T21:09:28.714190+0000","last_undegraded":"2023-09-23T21:09:28.714190+0000","last_fullsized":"2023-09-23T21:09:28.714190+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:22.040907+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:22.040907+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:47:02.019978+0000","scrub_duration":0.013021176000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":40,"reported_epoch":410,"state":"active+clean","last_fresh":"2023-09-23T21:09:28.749096+0000","last_change":"2023-09-23T21:09:08.441997+0000","last_active":"2023-09-23T21:09:28.749096+0000","last_peered":"2023-09-23T21:09:28.749096+0000","last_clean":"2023-09-23T21:09:28.749096+0000","last_became_active":"2023-09-23T21:09:08.407779+0000","last_became_peered":"2023-09-23T21:09:08.407779+0000","last_unstale":"2023-09-23T21:09:28.749096+0000","last_undegraded":"2023-09-23T21:09:28.749096+0000","last_fullsized":"2023-09-23T21:09:28.749096+0000","mapping_epoch":396,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":397,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:47:24.353451+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,5],"acting":[1,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":349,"reported_epoch":410,"state":"active+clean","last_fresh":"2023-09-23T21:09:28.715139+0000","last_change":"2023-09-23T21:05:55.325375+0000","last_active":"2023-09-23T21:09:28.715139+0000","last_peered":"2023-09-23T21:09:28.715139+0000","last_clean":"2023-09-23T21:09:28.715139+0000","last_became_active":"2023-09-23T21:05:55.324170+0000","last_became_peered":"2023-09-23T21:05:55.324170+0000","last_unstale":"2023-09-23T21:09:28.715139+0000","last_undegraded":"2023-09-23T21:09:28.715139+0000","last_fullsized":"2023-09-23T21:09:28.715139+0000","mapping_epoch":350,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":351,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:08:07.299605+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.7","version":"411'13454","reported_seq":18866,"reported_epoch":411,"state":"active+clean","last_fresh":"2023-09-23T21:09:44.082164+0000","last_change":"2023-09-23T21:09:08.472231+0000","last_active":"2023-09-23T21:09:44.082164+0000","last_peered":"2023-09-23T21:09:44.082164+0000","last_clean":"2023-09-23T21:09:44.082164+0000","last_became_active":"2023-09-23T21:09:08.440270+0000","last_became_peered":"2023-09-23T21:09:08.440270+0000","last_unstale":"2023-09-23T21:09:44.082164+0000","last_undegraded":"2023-09-23T21:09:44.082164+0000","last_fullsized":"2023-09-23T21:09:44.082164+0000","mapping_epoch":396,"log_start":"374'8541","ondisk_log_start":"374'8541","created":346,"last_epoch_clean":397,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:05:47.261750+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:05:47.261750+0000","last_clean_scrub_stamp":"2023-09-23T21:05:47.261750+0000","objects_scrubbed":0,"log_size":4913,"log_dups_size":0,"ondisk_log_size":4913,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:42:10.000374+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":44957696,"num_objects":686,"num_object_clones":0,"num_object_copies":2058,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":686,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13454,"num_write_kb":452480,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2654,"num_bytes_recovered":173932544,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,2],"acting":[3,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.f","version":"79'161","reported_seq":775,"reported_epoch":410,"state":"active+clean","last_fresh":"2023-09-23T21:09:28.716305+0000","last_change":"2023-09-23T21:09:11.395511+0000","last_active":"2023-09-23T21:09:28.716305+0000","last_peered":"2023-09-23T21:09:28.716305+0000","last_clean":"2023-09-23T21:09:28.716305+0000","last_became_active":"2023-09-23T21:09:11.395144+0000","last_became_peered":"2023-09-23T21:09:11.395144+0000","last_unstale":"2023-09-23T21:09:28.716305+0000","last_undegraded":"2023-09-23T21:09:28.716305+0000","last_fullsized":"2023-09-23T21:09:28.716305+0000","mapping_epoch":399,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":400,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:27.880776+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:27.880776+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:23:48.675879+0000","scrub_duration":0.0014004899999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,3],"acting":[6,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":106,"reported_epoch":410,"state":"active+clean","last_fresh":"2023-09-23T21:09:28.718558+0000","last_change":"2023-09-23T21:08:43.532612+0000","last_active":"2023-09-23T21:09:28.718558+0000","last_peered":"2023-09-23T21:09:28.718558+0000","last_clean":"2023-09-23T21:09:28.718558+0000","last_became_active":"2023-09-23T21:08:43.499861+0000","last_became_peered":"2023-09-23T21:08:43.499861+0000","last_unstale":"2023-09-23T21:09:28.718558+0000","last_undegraded":"2023-09-23T21:09:28.718558+0000","last_fullsized":"2023-09-23T21:09:28.718558+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:07:50.158275+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":140,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,3],"acting":[5,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":374,"reported_epoch":410,"state":"active+clean","last_fresh":"2023-09-23T21:09:28.747313+0000","last_change":"2023-09-23T21:08:54.699779+0000","last_active":"2023-09-23T21:09:28.747313+0000","last_peered":"2023-09-23T21:09:28.747313+0000","last_clean":"2023-09-23T21:09:28.747313+0000","last_became_active":"2023-09-23T21:08:54.699118+0000","last_became_peered":"2023-09-23T21:08:54.699118+0000","last_unstale":"2023-09-23T21:09:28.747313+0000","last_undegraded":"2023-09-23T21:09:28.747313+0000","last_fullsized":"2023-09-23T21:09:28.747313+0000","mapping_epoch":388,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":389,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:48:03.900670+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.6","version":"411'13235","reported_seq":13323,"reported_epoch":411,"state":"active+clean","last_fresh":"2023-09-23T21:09:41.077554+0000","last_change":"2023-09-23T21:06:49.429119+0000","last_active":"2023-09-23T21:09:41.077554+0000","last_peered":"2023-09-23T21:09:41.077554+0000","last_clean":"2023-09-23T21:09:41.077554+0000","last_became_active":"2023-09-23T21:06:49.423016+0000","last_became_peered":"2023-09-23T21:06:49.423016+0000","last_unstale":"2023-09-23T21:09:41.077554+0000","last_undegraded":"2023-09-23T21:09:41.077554+0000","last_fullsized":"2023-09-23T21:09:41.077554+0000","mapping_epoch":359,"log_start":"366'6495","ondisk_log_start":"366'6495","created":346,"last_epoch_clean":360,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:05:47.261750+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:05:47.261750+0000","last_clean_scrub_stamp":"2023-09-23T21:05:47.261750+0000","objects_scrubbed":0,"log_size":6740,"log_dups_size":0,"ondisk_log_size":6740,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:55:25.118949+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":56557568,"num_objects":863,"num_object_clones":0,"num_object_copies":2589,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":863,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13235,"num_write_kb":451136,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,1],"acting":[2,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.e","version":"79'161","reported_seq":454,"reported_epoch":411,"state":"active+clean","last_fresh":"2023-09-23T21:09:43.439777+0000","last_change":"2023-09-23T21:09:43.439891+0000","last_active":"2023-09-23T21:09:43.439777+0000","last_peered":"2023-09-23T21:09:43.439777+0000","last_clean":"2023-09-23T21:09:43.439777+0000","last_became_active":"2023-09-23T21:09:11.398440+0000","last_became_peered":"2023-09-23T21:09:11.398440+0000","last_unstale":"2023-09-23T21:09:43.439777+0000","last_undegraded":"2023-09-23T21:09:43.439777+0000","last_fullsized":"2023-09-23T21:09:43.439777+0000","mapping_epoch":399,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":400,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:09:43.439752+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:09:43.439752+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:36:10.240369+0000","scrub_duration":0.024789746000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,1],"acting":[3,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":177,"reported_epoch":410,"state":"active+clean","last_fresh":"2023-09-23T21:09:28.715660+0000","last_change":"2023-09-23T21:06:49.423445+0000","last_active":"2023-09-23T21:09:28.715660+0000","last_peered":"2023-09-23T21:09:28.715660+0000","last_clean":"2023-09-23T21:09:28.715660+0000","last_became_active":"2023-09-23T21:06:49.423169+0000","last_became_peered":"2023-09-23T21:06:49.423169+0000","last_unstale":"2023-09-23T21:09:28.715660+0000","last_undegraded":"2023-09-23T21:09:28.715660+0000","last_fullsized":"2023-09-23T21:09:28.715660+0000","mapping_epoch":359,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":360,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:47:52.999823+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":403,"reported_epoch":410,"state":"active+clean","last_fresh":"2023-09-23T21:09:28.716084+0000","last_change":"2023-09-23T20:54:13.928715+0000","last_active":"2023-09-23T21:09:28.716084+0000","last_peered":"2023-09-23T21:09:28.716084+0000","last_clean":"2023-09-23T21:09:28.716084+0000","last_became_active":"2023-09-23T20:54:13.928200+0000","last_became_peered":"2023-09-23T20:54:13.928200+0000","last_unstale":"2023-09-23T21:09:28.716084+0000","last_undegraded":"2023-09-23T21:09:28.716084+0000","last_fullsized":"2023-09-23T21:09:28.716084+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:41:16.927513+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.1","version":"411'13610","reported_seq":17361,"reported_epoch":411,"state":"active+clean","last_fresh":"2023-09-23T21:09:44.095755+0000","last_change":"2023-09-23T21:09:28.744299+0000","last_active":"2023-09-23T21:09:44.095755+0000","last_peered":"2023-09-23T21:09:44.095755+0000","last_clean":"2023-09-23T21:09:44.095755+0000","last_became_active":"2023-09-23T21:09:28.718028+0000","last_became_peered":"2023-09-23T21:09:28.718028+0000","last_unstale":"2023-09-23T21:09:44.095755+0000","last_undegraded":"2023-09-23T21:09:44.095755+0000","last_fullsized":"2023-09-23T21:09:44.095755+0000","mapping_epoch":409,"log_start":"374'8675","ondisk_log_start":"374'8675","created":346,"last_epoch_clean":410,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:05:47.261750+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:05:47.261750+0000","last_clean_scrub_stamp":"2023-09-23T21:05:47.261750+0000","objects_scrubbed":0,"log_size":4935,"log_dups_size":0,"ondisk_log_size":4935,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:11:41.404528+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47841280,"num_objects":730,"num_object_clones":0,"num_object_copies":2190,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":730,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13610,"num_write_kb":458880,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1821,"num_bytes_recovered":119275520,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.9","version":"79'161","reported_seq":805,"reported_epoch":410,"state":"active+clean","last_fresh":"2023-09-23T21:09:28.712798+0000","last_change":"2023-09-23T21:08:46.395906+0000","last_active":"2023-09-23T21:09:28.712798+0000","last_peered":"2023-09-23T21:09:28.712798+0000","last_clean":"2023-09-23T21:09:28.712798+0000","last_became_active":"2023-09-23T21:08:46.395425+0000","last_became_peered":"2023-09-23T21:08:46.395425+0000","last_unstale":"2023-09-23T21:09:28.712798+0000","last_undegraded":"2023-09-23T21:09:28.712798+0000","last_fullsized":"2023-09-23T21:09:28.712798+0000","mapping_epoch":385,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":386,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:56:46.921739+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:56:46.921739+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:33:29.648468+0000","scrub_duration":0.0071954109999999996,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,5],"acting":[3,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":405,"reported_epoch":410,"state":"active+clean","last_fresh":"2023-09-23T21:09:28.713344+0000","last_change":"2023-09-23T20:54:13.961788+0000","last_active":"2023-09-23T21:09:28.713344+0000","last_peered":"2023-09-23T21:09:28.713344+0000","last_clean":"2023-09-23T21:09:28.713344+0000","last_became_active":"2023-09-23T20:54:13.940937+0000","last_became_peered":"2023-09-23T20:54:13.940937+0000","last_unstale":"2023-09-23T21:09:28.713344+0000","last_undegraded":"2023-09-23T21:09:28.713344+0000","last_fullsized":"2023-09-23T21:09:28.713344+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:11:56.785299+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":95,"reported_epoch":410,"state":"active+clean","last_fresh":"2023-09-23T21:09:28.715580+0000","last_change":"2023-09-23T21:09:08.423918+0000","last_active":"2023-09-23T21:09:28.715580+0000","last_peered":"2023-09-23T21:09:28.715580+0000","last_clean":"2023-09-23T21:09:28.715580+0000","last_became_active":"2023-09-23T21:09:08.423331+0000","last_became_peered":"2023-09-23T21:09:08.423331+0000","last_unstale":"2023-09-23T21:09:28.715580+0000","last_undegraded":"2023-09-23T21:09:28.715580+0000","last_fullsized":"2023-09-23T21:09:28.715580+0000","mapping_epoch":396,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":397,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:18:18.951372+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.0","version":"411'13407","reported_seq":13479,"reported_epoch":411,"state":"active+clean","last_fresh":"2023-09-23T21:09:43.561360+0000","last_change":"2023-09-23T21:05:55.315757+0000","last_active":"2023-09-23T21:09:43.561360+0000","last_peered":"2023-09-23T21:09:43.561360+0000","last_clean":"2023-09-23T21:09:43.561360+0000","last_became_active":"2023-09-23T21:05:55.314884+0000","last_became_peered":"2023-09-23T21:05:55.314884+0000","last_unstale":"2023-09-23T21:09:43.561360+0000","last_undegraded":"2023-09-23T21:09:43.561360+0000","last_fullsized":"2023-09-23T21:09:43.561360+0000","mapping_epoch":350,"log_start":"374'8563","ondisk_log_start":"374'8563","created":346,"last_epoch_clean":351,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:05:47.261750+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:05:47.261750+0000","last_clean_scrub_stamp":"2023-09-23T21:05:47.261750+0000","objects_scrubbed":0,"log_size":4844,"log_dups_size":0,"ondisk_log_size":4844,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:41:19.184012+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":49348608,"num_objects":753,"num_object_clones":0,"num_object_copies":2259,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":753,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13407,"num_write_kb":453120,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.8","version":"79'161","reported_seq":520,"reported_epoch":410,"state":"active+clean","last_fresh":"2023-09-23T21:09:28.716387+0000","last_change":"2023-09-23T21:06:52.099024+0000","last_active":"2023-09-23T21:09:28.716387+0000","last_peered":"2023-09-23T21:09:28.716387+0000","last_clean":"2023-09-23T21:09:28.716387+0000","last_became_active":"2023-09-23T21:06:49.425012+0000","last_became_peered":"2023-09-23T21:06:49.425012+0000","last_unstale":"2023-09-23T21:09:28.716387+0000","last_undegraded":"2023-09-23T21:09:28.716387+0000","last_fullsized":"2023-09-23T21:09:28.716387+0000","mapping_epoch":359,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":360,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:06:52.098882+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:06:52.098882+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:48:36.414396+0000","scrub_duration":0.0092173299999999993,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":400,"reported_epoch":410,"state":"active+clean","last_fresh":"2023-09-23T21:09:28.715916+0000","last_change":"2023-09-23T21:09:08.417202+0000","last_active":"2023-09-23T21:09:28.715916+0000","last_peered":"2023-09-23T21:09:28.715916+0000","last_clean":"2023-09-23T21:09:28.715916+0000","last_became_active":"2023-09-23T21:09:08.416908+0000","last_became_peered":"2023-09-23T21:09:08.416908+0000","last_unstale":"2023-09-23T21:09:28.715916+0000","last_undegraded":"2023-09-23T21:09:28.715916+0000","last_fullsized":"2023-09-23T21:09:28.715916+0000","mapping_epoch":396,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":397,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:49:02.368080+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":262,"reported_epoch":410,"state":"active+clean","last_fresh":"2023-09-23T21:09:28.717397+0000","last_change":"2023-09-23T21:09:08.410640+0000","last_active":"2023-09-23T21:09:28.717397+0000","last_peered":"2023-09-23T21:09:28.717397+0000","last_clean":"2023-09-23T21:09:28.717397+0000","last_became_active":"2023-09-23T21:09:08.410288+0000","last_became_peered":"2023-09-23T21:09:08.410288+0000","last_unstale":"2023-09-23T21:09:28.717397+0000","last_undegraded":"2023-09-23T21:09:28.717397+0000","last_fullsized":"2023-09-23T21:09:28.717397+0000","mapping_epoch":396,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":397,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:40:08.562402+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"9.3","version":"411'13283","reported_seq":17658,"reported_epoch":411,"state":"active+clean","last_fresh":"2023-09-23T21:09:44.071492+0000","last_change":"2023-09-23T21:09:20.409170+0000","last_active":"2023-09-23T21:09:44.071492+0000","last_peered":"2023-09-23T21:09:44.071492+0000","last_clean":"2023-09-23T21:09:44.071492+0000","last_became_active":"2023-09-23T21:09:20.373496+0000","last_became_peered":"2023-09-23T21:09:20.373496+0000","last_unstale":"2023-09-23T21:09:44.071492+0000","last_undegraded":"2023-09-23T21:09:44.071492+0000","last_fullsized":"2023-09-23T21:09:44.071492+0000","mapping_epoch":403,"log_start":"374'8370","ondisk_log_start":"374'8370","created":346,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:05:47.261750+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:05:47.261750+0000","last_clean_scrub_stamp":"2023-09-23T21:05:47.261750+0000","objects_scrubbed":0,"log_size":4913,"log_dups_size":0,"ondisk_log_size":4913,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:38:15.630903+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47120384,"num_objects":719,"num_object_clones":0,"num_object_copies":2157,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":719,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13283,"num_write_kb":448064,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2129,"num_bytes_recovered":139526144,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,5],"acting":[3,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.b","version":"79'161","reported_seq":741,"reported_epoch":410,"state":"active+clean","last_fresh":"2023-09-23T21:09:28.746803+0000","last_change":"2023-09-23T21:05:55.772369+0000","last_active":"2023-09-23T21:09:28.746803+0000","last_peered":"2023-09-23T21:09:28.746803+0000","last_clean":"2023-09-23T21:09:28.746803+0000","last_became_active":"2023-09-23T21:05:55.316405+0000","last_became_peered":"2023-09-23T21:05:55.316405+0000","last_unstale":"2023-09-23T21:09:28.746803+0000","last_undegraded":"2023-09-23T21:09:28.746803+0000","last_fullsized":"2023-09-23T21:09:28.746803+0000","mapping_epoch":350,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":351,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:05:55.772264+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:05:55.772264+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:14:30.697692+0000","scrub_duration":0.001862507,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,5],"acting":[1,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":287,"reported_epoch":410,"state":"active+clean","last_fresh":"2023-09-23T21:09:28.716718+0000","last_change":"2023-09-23T21:09:08.401941+0000","last_active":"2023-09-23T21:09:28.716718+0000","last_peered":"2023-09-23T21:09:28.716718+0000","last_clean":"2023-09-23T21:09:28.716718+0000","last_became_active":"2023-09-23T21:09:08.401636+0000","last_became_peered":"2023-09-23T21:09:08.401636+0000","last_unstale":"2023-09-23T21:09:28.716718+0000","last_undegraded":"2023-09-23T21:09:28.716718+0000","last_fullsized":"2023-09-23T21:09:28.716718+0000","mapping_epoch":396,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":397,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:12:45.878513+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,5],"acting":[2,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":332,"reported_epoch":410,"state":"active+clean","last_fresh":"2023-09-23T21:09:28.715262+0000","last_change":"2023-09-23T21:08:54.688849+0000","last_active":"2023-09-23T21:09:28.715262+0000","last_peered":"2023-09-23T21:09:28.715262+0000","last_clean":"2023-09-23T21:09:28.715262+0000","last_became_active":"2023-09-23T21:08:54.688571+0000","last_became_peered":"2023-09-23T21:08:54.688571+0000","last_unstale":"2023-09-23T21:09:28.715262+0000","last_undegraded":"2023-09-23T21:09:28.715262+0000","last_fullsized":"2023-09-23T21:09:28.715262+0000","mapping_epoch":388,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":389,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:33:07.896205+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.2","version":"411'13414","reported_seq":13495,"reported_epoch":411,"state":"active+clean","last_fresh":"2023-09-23T21:09:44.098700+0000","last_change":"2023-09-23T21:05:55.325629+0000","last_active":"2023-09-23T21:09:44.098700+0000","last_peered":"2023-09-23T21:09:44.098700+0000","last_clean":"2023-09-23T21:09:44.098700+0000","last_became_active":"2023-09-23T21:05:55.325108+0000","last_became_peered":"2023-09-23T21:05:55.325108+0000","last_unstale":"2023-09-23T21:09:44.098700+0000","last_undegraded":"2023-09-23T21:09:44.098700+0000","last_fullsized":"2023-09-23T21:09:44.098700+0000","mapping_epoch":350,"log_start":"374'8574","ondisk_log_start":"374'8574","created":346,"last_epoch_clean":351,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:05:47.261750+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:05:47.261750+0000","last_clean_scrub_stamp":"2023-09-23T21:05:47.261750+0000","objects_scrubbed":0,"log_size":4840,"log_dups_size":0,"ondisk_log_size":4840,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:50:06.272050+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":46792704,"num_objects":714,"num_object_clones":0,"num_object_copies":2142,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":714,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13414,"num_write_kb":452096,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.a","version":"79'161","reported_seq":512,"reported_epoch":410,"state":"active+clean","last_fresh":"2023-09-23T21:09:28.716678+0000","last_change":"2023-09-23T21:06:51.075617+0000","last_active":"2023-09-23T21:09:28.716678+0000","last_peered":"2023-09-23T21:09:28.716678+0000","last_clean":"2023-09-23T21:09:28.716678+0000","last_became_active":"2023-09-23T21:06:49.423291+0000","last_became_peered":"2023-09-23T21:06:49.423291+0000","last_unstale":"2023-09-23T21:09:28.716678+0000","last_undegraded":"2023-09-23T21:09:28.716678+0000","last_fullsized":"2023-09-23T21:09:28.716678+0000","mapping_epoch":359,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":360,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:06:51.075448+0000","last_deep_scrub":"79'161","last_deep_scrub_stamp":"2023-09-23T21:05:32.630292+0000","last_clean_scrub_stamp":"2023-09-23T21:06:51.075448+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:35:01.883490+0000","scrub_duration":0.0054793890000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":237,"reported_epoch":410,"state":"active+clean","last_fresh":"2023-09-23T21:09:28.712700+0000","last_change":"2023-09-23T21:09:08.424026+0000","last_active":"2023-09-23T21:09:28.712700+0000","last_peered":"2023-09-23T21:09:28.712700+0000","last_clean":"2023-09-23T21:09:28.712700+0000","last_became_active":"2023-09-23T21:09:08.423098+0000","last_became_peered":"2023-09-23T21:09:08.423098+0000","last_unstale":"2023-09-23T21:09:28.712700+0000","last_undegraded":"2023-09-23T21:09:28.712700+0000","last_fullsized":"2023-09-23T21:09:28.712700+0000","mapping_epoch":396,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":397,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:52:41.551514+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,5],"acting":[3,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":393,"reported_epoch":410,"state":"active+clean","last_fresh":"2023-09-23T21:09:28.749415+0000","last_change":"2023-09-23T21:09:08.432754+0000","last_active":"2023-09-23T21:09:28.749415+0000","last_peered":"2023-09-23T21:09:28.749415+0000","last_clean":"2023-09-23T21:09:28.749415+0000","last_became_active":"2023-09-23T21:09:08.432368+0000","last_became_peered":"2023-09-23T21:09:08.432368+0000","last_unstale":"2023-09-23T21:09:28.749415+0000","last_undegraded":"2023-09-23T21:09:28.749415+0000","last_fullsized":"2023-09-23T21:09:28.749415+0000","mapping_epoch":396,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":397,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:36:05.828847+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,5],"acting":[1,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.f","version":"411'13297","reported_seq":13377,"reported_epoch":411,"state":"active+clean","last_fresh":"2023-09-23T21:09:42.230753+0000","last_change":"2023-09-23T21:05:55.325083+0000","last_active":"2023-09-23T21:09:42.230753+0000","last_peered":"2023-09-23T21:09:42.230753+0000","last_clean":"2023-09-23T21:09:42.230753+0000","last_became_active":"2023-09-23T21:05:55.324768+0000","last_became_peered":"2023-09-23T21:05:55.324768+0000","last_unstale":"2023-09-23T21:09:42.230753+0000","last_undegraded":"2023-09-23T21:09:42.230753+0000","last_fullsized":"2023-09-23T21:09:42.230753+0000","mapping_epoch":350,"log_start":"371'7910","ondisk_log_start":"371'7910","created":346,"last_epoch_clean":351,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:05:47.261750+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:05:47.261750+0000","last_clean_scrub_stamp":"2023-09-23T21:05:47.261750+0000","objects_scrubbed":0,"log_size":5387,"log_dups_size":0,"ondisk_log_size":5387,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:25:42.316668+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":56295424,"num_objects":859,"num_object_clones":0,"num_object_copies":2577,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":859,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13297,"num_write_kb":452992,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.7","version":"79'161","reported_seq":736,"reported_epoch":410,"state":"active+clean","last_fresh":"2023-09-23T21:09:28.716089+0000","last_change":"2023-09-23T21:08:46.392238+0000","last_active":"2023-09-23T21:09:28.716089+0000","last_peered":"2023-09-23T21:09:28.716089+0000","last_clean":"2023-09-23T21:09:28.716089+0000","last_became_active":"2023-09-23T21:08:46.391848+0000","last_became_peered":"2023-09-23T21:08:46.391848+0000","last_unstale":"2023-09-23T21:09:28.716089+0000","last_undegraded":"2023-09-23T21:09:28.716089+0000","last_fullsized":"2023-09-23T21:09:28.716089+0000","mapping_epoch":385,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":386,"parent":"0.0","parent_split_bits":3,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:25.916867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:25.916867+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:14:19.787602+0000","scrub_duration":0.001665323,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,5],"acting":[6,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":397,"reported_epoch":410,"state":"active+clean","last_fresh":"2023-09-23T21:09:28.747248+0000","last_change":"2023-09-23T21:08:43.506408+0000","last_active":"2023-09-23T21:09:28.747248+0000","last_peered":"2023-09-23T21:09:28.747248+0000","last_clean":"2023-09-23T21:09:28.747248+0000","last_became_active":"2023-09-23T21:08:43.506156+0000","last_became_peered":"2023-09-23T21:08:43.506156+0000","last_unstale":"2023-09-23T21:09:28.747248+0000","last_undegraded":"2023-09-23T21:09:28.747248+0000","last_fullsized":"2023-09-23T21:09:28.747248+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:31:12.790561+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,2],"acting":[1,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":88,"reported_epoch":410,"state":"active+clean","last_fresh":"2023-09-23T21:09:28.718463+0000","last_change":"2023-09-23T21:05:04.326889+0000","last_active":"2023-09-23T21:09:28.718463+0000","last_peered":"2023-09-23T21:09:28.718463+0000","last_clean":"2023-09-23T21:09:28.718463+0000","last_became_active":"2023-09-23T21:05:04.326032+0000","last_became_peered":"2023-09-23T21:05:04.326032+0000","last_unstale":"2023-09-23T21:09:28.718463+0000","last_undegraded":"2023-09-23T21:09:28.718463+0000","last_fullsized":"2023-09-23T21:09:28.718463+0000","mapping_epoch":332,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:00:17.507889+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":410,"reported_epoch":410,"state":"active+clean","last_fresh":"2023-09-23T21:09:28.747115+0000","last_change":"2023-09-23T21:09:08.438066+0000","last_active":"2023-09-23T21:09:28.747115+0000","last_peered":"2023-09-23T21:09:28.747115+0000","last_clean":"2023-09-23T21:09:28.747115+0000","last_became_active":"2023-09-23T21:09:08.437786+0000","last_became_peered":"2023-09-23T21:09:08.437786+0000","last_unstale":"2023-09-23T21:09:28.747115+0000","last_undegraded":"2023-09-23T21:09:28.747115+0000","last_fullsized":"2023-09-23T21:09:28.747115+0000","mapping_epoch":396,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":397,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:49:53.623831+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.9","version":"411'13523","reported_seq":17739,"reported_epoch":411,"state":"active+clean","last_fresh":"2023-09-23T21:09:42.386289+0000","last_change":"2023-09-23T21:09:28.755713+0000","last_active":"2023-09-23T21:09:42.386289+0000","last_peered":"2023-09-23T21:09:42.386289+0000","last_clean":"2023-09-23T21:09:42.386289+0000","last_became_active":"2023-09-23T21:09:28.721062+0000","last_became_peered":"2023-09-23T21:09:28.721062+0000","last_unstale":"2023-09-23T21:09:42.386289+0000","last_undegraded":"2023-09-23T21:09:42.386289+0000","last_fullsized":"2023-09-23T21:09:42.386289+0000","mapping_epoch":409,"log_start":"371'7983","ondisk_log_start":"371'7983","created":346,"last_epoch_clean":410,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:05:47.261750+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:05:47.261750+0000","last_clean_scrub_stamp":"2023-09-23T21:05:47.261750+0000","objects_scrubbed":0,"log_size":5540,"log_dups_size":0,"ondisk_log_size":5540,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:38:34.927835+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":55377920,"num_objects":845,"num_object_clones":0,"num_object_copies":2535,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":845,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13523,"num_write_kb":459776,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2057,"num_bytes_recovered":134807552,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1","version":"79'161","reported_seq":519,"reported_epoch":410,"state":"active+clean","last_fresh":"2023-09-23T21:09:28.714896+0000","last_change":"2023-09-23T21:06:52.576036+0000","last_active":"2023-09-23T21:09:28.714896+0000","last_peered":"2023-09-23T21:09:28.714896+0000","last_clean":"2023-09-23T21:09:28.714896+0000","last_became_active":"2023-09-23T21:06:49.422806+0000","last_became_peered":"2023-09-23T21:06:49.422806+0000","last_unstale":"2023-09-23T21:09:28.714896+0000","last_undegraded":"2023-09-23T21:09:28.714896+0000","last_fullsized":"2023-09-23T21:09:28.714896+0000","mapping_epoch":359,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":360,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:06:52.575900+0000","last_deep_scrub":"79'161","last_deep_scrub_stamp":"2023-09-23T21:05:30.652189+0000","last_clean_scrub_stamp":"2023-09-23T21:06:52.575900+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:59:24.733331+0000","scrub_duration":0.017723466,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":180,"reported_epoch":410,"state":"active+clean","last_fresh":"2023-09-23T21:09:28.746703+0000","last_change":"2023-09-23T21:08:54.699879+0000","last_active":"2023-09-23T21:09:28.746703+0000","last_peered":"2023-09-23T21:09:28.746703+0000","last_clean":"2023-09-23T21:09:28.746703+0000","last_became_active":"2023-09-23T21:08:54.699558+0000","last_became_peered":"2023-09-23T21:08:54.699558+0000","last_unstale":"2023-09-23T21:09:28.746703+0000","last_undegraded":"2023-09-23T21:09:28.746703+0000","last_fullsized":"2023-09-23T21:09:28.746703+0000","mapping_epoch":388,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":389,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:52:11.177493+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":305,"reported_epoch":410,"state":"active+clean","last_fresh":"2023-09-23T21:09:28.715331+0000","last_change":"2023-09-23T20:55:35.949000+0000","last_active":"2023-09-23T21:09:28.715331+0000","last_peered":"2023-09-23T21:09:28.715331+0000","last_clean":"2023-09-23T21:09:28.715331+0000","last_became_active":"2023-09-23T20:55:35.948410+0000","last_became_peered":"2023-09-23T20:55:35.948410+0000","last_unstale":"2023-09-23T21:09:28.715331+0000","last_undegraded":"2023-09-23T21:09:28.715331+0000","last_fullsized":"2023-09-23T21:09:28.715331+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:26:50.477289+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":348,"reported_epoch":410,"state":"active+clean","last_fresh":"2023-09-23T21:09:28.747166+0000","last_change":"2023-09-23T21:05:55.320314+0000","last_active":"2023-09-23T21:09:28.747166+0000","last_peered":"2023-09-23T21:09:28.747166+0000","last_clean":"2023-09-23T21:09:28.747166+0000","last_became_active":"2023-09-23T21:05:55.319935+0000","last_became_peered":"2023-09-23T21:05:55.319935+0000","last_unstale":"2023-09-23T21:09:28.747166+0000","last_undegraded":"2023-09-23T21:09:28.747166+0000","last_fullsized":"2023-09-23T21:09:28.747166+0000","mapping_epoch":350,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":351,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:52:28.754534+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.a","version":"411'13178","reported_seq":18706,"reported_epoch":411,"state":"active+clean","last_fresh":"2023-09-23T21:09:42.262971+0000","last_change":"2023-09-23T21:09:08.440030+0000","last_active":"2023-09-23T21:09:42.262971+0000","last_peered":"2023-09-23T21:09:42.262971+0000","last_clean":"2023-09-23T21:09:42.262971+0000","last_became_active":"2023-09-23T21:09:08.408781+0000","last_became_peered":"2023-09-23T21:09:08.408781+0000","last_unstale":"2023-09-23T21:09:42.262971+0000","last_undegraded":"2023-09-23T21:09:42.262971+0000","last_fullsized":"2023-09-23T21:09:42.262971+0000","mapping_epoch":396,"log_start":"371'7771","ondisk_log_start":"371'7771","created":346,"last_epoch_clean":397,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:05:47.261750+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:05:47.261750+0000","last_clean_scrub_stamp":"2023-09-23T21:05:47.261750+0000","objects_scrubbed":0,"log_size":5407,"log_dups_size":0,"ondisk_log_size":5407,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:56:53.447879+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53084160,"num_objects":810,"num_object_clones":0,"num_object_copies":2430,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":810,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13178,"num_write_kb":447616,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2711,"num_bytes_recovered":177668096,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.2","version":"79'161","reported_seq":680,"reported_epoch":410,"state":"active+clean","last_fresh":"2023-09-23T21:09:28.714889+0000","last_change":"2023-09-23T21:09:11.387828+0000","last_active":"2023-09-23T21:09:28.714889+0000","last_peered":"2023-09-23T21:09:28.714889+0000","last_clean":"2023-09-23T21:09:28.714889+0000","last_became_active":"2023-09-23T21:09:11.387486+0000","last_became_peered":"2023-09-23T21:09:11.387486+0000","last_unstale":"2023-09-23T21:09:28.714889+0000","last_undegraded":"2023-09-23T21:09:28.714889+0000","last_fullsized":"2023-09-23T21:09:28.714889+0000","mapping_epoch":399,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":400,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:24.802451+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:24.802451+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:51:42.716025+0000","scrub_duration":0.0090861250000000005,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,5],"acting":[0,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":402,"reported_epoch":410,"state":"active+clean","last_fresh":"2023-09-23T21:09:28.716807+0000","last_change":"2023-09-23T20:51:10.838054+0000","last_active":"2023-09-23T21:09:28.716807+0000","last_peered":"2023-09-23T21:09:28.716807+0000","last_clean":"2023-09-23T21:09:28.716807+0000","last_became_active":"2023-09-23T20:51:10.837750+0000","last_became_peered":"2023-09-23T20:51:10.837750+0000","last_unstale":"2023-09-23T21:09:28.716807+0000","last_undegraded":"2023-09-23T21:09:28.716807+0000","last_fullsized":"2023-09-23T21:09:28.716807+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:15:51.648085+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":374,"reported_epoch":410,"state":"active+clean","last_fresh":"2023-09-23T21:09:28.746420+0000","last_change":"2023-09-23T21:05:55.315770+0000","last_active":"2023-09-23T21:09:28.746420+0000","last_peered":"2023-09-23T21:09:28.746420+0000","last_clean":"2023-09-23T21:09:28.746420+0000","last_became_active":"2023-09-23T21:05:55.315290+0000","last_became_peered":"2023-09-23T21:05:55.315290+0000","last_unstale":"2023-09-23T21:09:28.746420+0000","last_undegraded":"2023-09-23T21:09:28.746420+0000","last_fullsized":"2023-09-23T21:09:28.746420+0000","mapping_epoch":350,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":351,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:27:40.430262+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":415,"reported_epoch":410,"state":"active+clean","last_fresh":"2023-09-23T21:09:28.749543+0000","last_change":"2023-09-23T21:05:55.318863+0000","last_active":"2023-09-23T21:09:28.749543+0000","last_peered":"2023-09-23T21:09:28.749543+0000","last_clean":"2023-09-23T21:09:28.749543+0000","last_became_active":"2023-09-23T21:05:55.318489+0000","last_became_peered":"2023-09-23T21:05:55.318489+0000","last_unstale":"2023-09-23T21:09:28.749543+0000","last_undegraded":"2023-09-23T21:09:28.749543+0000","last_fullsized":"2023-09-23T21:09:28.749543+0000","mapping_epoch":350,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":351,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:45:05.912661+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.8","version":"411'13299","reported_seq":13374,"reported_epoch":411,"state":"active+clean","last_fresh":"2023-09-23T21:09:44.077599+0000","last_change":"2023-09-23T21:05:48.286303+0000","last_active":"2023-09-23T21:09:44.077599+0000","last_peered":"2023-09-23T21:09:44.077599+0000","last_clean":"2023-09-23T21:09:44.077599+0000","last_became_active":"2023-09-23T21:05:48.286008+0000","last_became_peered":"2023-09-23T21:05:48.286008+0000","last_unstale":"2023-09-23T21:09:44.077599+0000","last_undegraded":"2023-09-23T21:09:44.077599+0000","last_fullsized":"2023-09-23T21:09:44.077599+0000","mapping_epoch":346,"log_start":"374'8372","ondisk_log_start":"374'8372","created":346,"last_epoch_clean":347,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:05:47.261750+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:05:47.261750+0000","last_clean_scrub_stamp":"2023-09-23T21:05:47.261750+0000","objects_scrubbed":0,"log_size":4927,"log_dups_size":0,"ondisk_log_size":4927,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:42:05.759917+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47775744,"num_objects":729,"num_object_clones":0,"num_object_copies":2187,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":729,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13299,"num_write_kb":448896,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":185,"reported_epoch":410,"state":"active+clean","last_fresh":"2023-09-23T21:09:28.716838+0000","last_change":"2023-09-23T21:08:43.492449+0000","last_active":"2023-09-23T21:09:28.716838+0000","last_peered":"2023-09-23T21:09:28.716838+0000","last_clean":"2023-09-23T21:09:28.716838+0000","last_became_active":"2023-09-23T21:08:43.492116+0000","last_became_peered":"2023-09-23T21:08:43.492116+0000","last_unstale":"2023-09-23T21:09:28.716838+0000","last_undegraded":"2023-09-23T21:09:28.716838+0000","last_fullsized":"2023-09-23T21:09:28.716838+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:04:30.288981+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"79'161","reported_seq":663,"reported_epoch":410,"state":"active+clean","last_fresh":"2023-09-23T21:09:28.714515+0000","last_change":"2023-09-23T21:08:46.407789+0000","last_active":"2023-09-23T21:09:28.714515+0000","last_peered":"2023-09-23T21:09:28.714515+0000","last_clean":"2023-09-23T21:09:28.714515+0000","last_became_active":"2023-09-23T21:08:46.407378+0000","last_became_peered":"2023-09-23T21:08:46.407378+0000","last_unstale":"2023-09-23T21:09:28.714515+0000","last_undegraded":"2023-09-23T21:09:28.714515+0000","last_fullsized":"2023-09-23T21:09:28.714515+0000","mapping_epoch":385,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":386,"parent":"0.0","parent_split_bits":0,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:20.738273+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:20.738273+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:03:45.506154+0000","scrub_duration":0.0088336339999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":411,"reported_epoch":410,"state":"active+clean","last_fresh":"2023-09-23T21:09:28.712898+0000","last_change":"2023-09-23T21:05:04.310148+0000","last_active":"2023-09-23T21:09:28.712898+0000","last_peered":"2023-09-23T21:09:28.712898+0000","last_clean":"2023-09-23T21:09:28.712898+0000","last_became_active":"2023-09-23T21:05:04.309869+0000","last_became_peered":"2023-09-23T21:05:04.309869+0000","last_unstale":"2023-09-23T21:09:28.712898+0000","last_undegraded":"2023-09-23T21:09:28.712898+0000","last_fullsized":"2023-09-23T21:09:28.712898+0000","mapping_epoch":332,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:12:55.522835+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":358,"reported_epoch":410,"state":"active+clean","last_fresh":"2023-09-23T21:09:28.715397+0000","last_change":"2023-09-23T21:09:08.423506+0000","last_active":"2023-09-23T21:09:28.715397+0000","last_peered":"2023-09-23T21:09:28.715397+0000","last_clean":"2023-09-23T21:09:28.715397+0000","last_became_active":"2023-09-23T21:09:08.423187+0000","last_became_peered":"2023-09-23T21:09:08.423187+0000","last_unstale":"2023-09-23T21:09:28.715397+0000","last_undegraded":"2023-09-23T21:09:28.715397+0000","last_fullsized":"2023-09-23T21:09:28.715397+0000","mapping_epoch":396,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":397,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:41:49.562254+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,5],"acting":[0,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.b","version":"411'13240","reported_seq":13315,"reported_epoch":411,"state":"active+clean","last_fresh":"2023-09-23T21:09:42.356531+0000","last_change":"2023-09-23T21:05:48.290142+0000","last_active":"2023-09-23T21:09:42.356531+0000","last_peered":"2023-09-23T21:09:42.356531+0000","last_clean":"2023-09-23T21:09:42.356531+0000","last_became_active":"2023-09-23T21:05:48.289861+0000","last_became_peered":"2023-09-23T21:05:48.289861+0000","last_unstale":"2023-09-23T21:09:42.356531+0000","last_undegraded":"2023-09-23T21:09:42.356531+0000","last_fullsized":"2023-09-23T21:09:42.356531+0000","mapping_epoch":346,"log_start":"370'7733","ondisk_log_start":"370'7733","created":346,"last_epoch_clean":347,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:05:47.261750+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:05:47.261750+0000","last_clean_scrub_stamp":"2023-09-23T21:05:47.261750+0000","objects_scrubbed":0,"log_size":5507,"log_dups_size":0,"ondisk_log_size":5507,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:36:39.064858+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":51249152,"num_objects":782,"num_object_clones":0,"num_object_copies":2346,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":782,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13240,"num_write_kb":448704,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.3","version":"79'161","reported_seq":688,"reported_epoch":410,"state":"active+clean","last_fresh":"2023-09-23T21:09:28.746980+0000","last_change":"2023-09-23T21:09:11.396811+0000","last_active":"2023-09-23T21:09:28.746980+0000","last_peered":"2023-09-23T21:09:28.746980+0000","last_clean":"2023-09-23T21:09:28.746980+0000","last_became_active":"2023-09-23T21:09:11.396462+0000","last_became_peered":"2023-09-23T21:09:11.396462+0000","last_unstale":"2023-09-23T21:09:28.746980+0000","last_undegraded":"2023-09-23T21:09:28.746980+0000","last_fullsized":"2023-09-23T21:09:28.746980+0000","mapping_epoch":399,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":400,"parent":"0.0","parent_split_bits":3,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:05:57.832302+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:05:57.832302+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:29:03.783133+0000","scrub_duration":0.010206847999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":284,"reported_epoch":410,"state":"active+clean","last_fresh":"2023-09-23T21:09:28.746440+0000","last_change":"2023-09-23T21:08:43.507176+0000","last_active":"2023-09-23T21:09:28.746440+0000","last_peered":"2023-09-23T21:09:28.746440+0000","last_clean":"2023-09-23T21:09:28.746440+0000","last_became_active":"2023-09-23T21:08:43.506832+0000","last_became_peered":"2023-09-23T21:08:43.506832+0000","last_unstale":"2023-09-23T21:09:28.746440+0000","last_undegraded":"2023-09-23T21:09:28.746440+0000","last_fullsized":"2023-09-23T21:09:28.746440+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:24:09.274005+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":418,"reported_epoch":410,"state":"active+clean","last_fresh":"2023-09-23T21:09:28.715098+0000","last_change":"2023-09-23T21:09:08.424118+0000","last_active":"2023-09-23T21:09:28.715098+0000","last_peered":"2023-09-23T21:09:28.715098+0000","last_clean":"2023-09-23T21:09:28.715098+0000","last_became_active":"2023-09-23T21:09:08.423805+0000","last_became_peered":"2023-09-23T21:09:08.423805+0000","last_unstale":"2023-09-23T21:09:28.715098+0000","last_undegraded":"2023-09-23T21:09:28.715098+0000","last_fullsized":"2023-09-23T21:09:28.715098+0000","mapping_epoch":396,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":397,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:47:06.890350+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,5],"acting":[0,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":403,"reported_epoch":410,"state":"active+clean","last_fresh":"2023-09-23T21:09:28.716521+0000","last_change":"2023-09-23T21:05:55.320840+0000","last_active":"2023-09-23T21:09:28.716521+0000","last_peered":"2023-09-23T21:09:28.716521+0000","last_clean":"2023-09-23T21:09:28.716521+0000","last_became_active":"2023-09-23T21:05:55.320280+0000","last_became_peered":"2023-09-23T21:05:55.320280+0000","last_unstale":"2023-09-23T21:09:28.716521+0000","last_undegraded":"2023-09-23T21:09:28.716521+0000","last_fullsized":"2023-09-23T21:09:28.716521+0000","mapping_epoch":350,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":351,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:45:22.368801+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.e","version":"411'13751","reported_seq":23276,"reported_epoch":411,"state":"active+clean","last_fresh":"2023-09-23T21:09:43.555292+0000","last_change":"2023-09-23T21:09:27.747462+0000","last_active":"2023-09-23T21:09:43.555292+0000","last_peered":"2023-09-23T21:09:43.555292+0000","last_clean":"2023-09-23T21:09:43.555292+0000","last_became_active":"2023-09-23T21:09:27.721708+0000","last_became_peered":"2023-09-23T21:09:27.721708+0000","last_unstale":"2023-09-23T21:09:43.555292+0000","last_undegraded":"2023-09-23T21:09:43.555292+0000","last_fullsized":"2023-09-23T21:09:43.555292+0000","mapping_epoch":408,"log_start":"374'8836","ondisk_log_start":"374'8836","created":346,"last_epoch_clean":409,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:05:47.261750+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:05:47.261750+0000","last_clean_scrub_stamp":"2023-09-23T21:05:47.261750+0000","objects_scrubbed":0,"log_size":4915,"log_dups_size":0,"ondisk_log_size":4915,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:57:13.311339+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":52363264,"num_objects":799,"num_object_clones":0,"num_object_copies":2397,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":799,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13751,"num_write_kb":465600,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4706,"num_bytes_recovered":308412416,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,5],"acting":[1,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"79'161","reported_seq":693,"reported_epoch":410,"state":"active+clean","last_fresh":"2023-09-23T21:09:28.716340+0000","last_change":"2023-09-23T20:57:26.907505+0000","last_active":"2023-09-23T21:09:28.716340+0000","last_peered":"2023-09-23T21:09:28.716340+0000","last_clean":"2023-09-23T21:09:28.716340+0000","last_became_active":"2023-09-23T20:57:08.240321+0000","last_became_peered":"2023-09-23T20:57:08.240321+0000","last_unstale":"2023-09-23T21:09:28.716340+0000","last_undegraded":"2023-09-23T21:09:28.716340+0000","last_fullsized":"2023-09-23T21:09:28.716340+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":3,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:26.907408+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:26.907408+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:34:17.392928+0000","scrub_duration":0.010337038,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":320,"reported_epoch":410,"state":"active+clean","last_fresh":"2023-09-23T21:09:28.715660+0000","last_change":"2023-09-23T21:09:08.420418+0000","last_active":"2023-09-23T21:09:28.715660+0000","last_peered":"2023-09-23T21:09:28.715660+0000","last_clean":"2023-09-23T21:09:28.715660+0000","last_became_active":"2023-09-23T21:09:08.420094+0000","last_became_peered":"2023-09-23T21:09:08.420094+0000","last_unstale":"2023-09-23T21:09:28.715660+0000","last_undegraded":"2023-09-23T21:09:28.715660+0000","last_fullsized":"2023-09-23T21:09:28.715660+0000","mapping_epoch":396,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":397,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:46:59.735031+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,6],"acting":[0,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":398,"reported_epoch":410,"state":"active+clean","last_fresh":"2023-09-23T21:09:28.747249+0000","last_change":"2023-09-23T21:05:55.319513+0000","last_active":"2023-09-23T21:09:28.747249+0000","last_peered":"2023-09-23T21:09:28.747249+0000","last_clean":"2023-09-23T21:09:28.747249+0000","last_became_active":"2023-09-23T21:05:55.319178+0000","last_became_peered":"2023-09-23T21:05:55.319178+0000","last_unstale":"2023-09-23T21:09:28.747249+0000","last_undegraded":"2023-09-23T21:09:28.747249+0000","last_fullsized":"2023-09-23T21:09:28.747249+0000","mapping_epoch":350,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":351,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:47:32.783934+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":403,"reported_epoch":410,"state":"active+clean","last_fresh":"2023-09-23T21:09:28.716307+0000","last_change":"2023-09-23T21:08:43.508269+0000","last_active":"2023-09-23T21:09:28.716307+0000","last_peered":"2023-09-23T21:09:28.716307+0000","last_clean":"2023-09-23T21:09:28.716307+0000","last_became_active":"2023-09-23T21:08:43.507970+0000","last_became_peered":"2023-09-23T21:08:43.507970+0000","last_unstale":"2023-09-23T21:09:28.716307+0000","last_undegraded":"2023-09-23T21:09:28.716307+0000","last_fullsized":"2023-09-23T21:09:28.716307+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:52:15.049815+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,1],"acting":[0,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.d","version":"411'13464","reported_seq":19338,"reported_epoch":411,"state":"active+clean","last_fresh":"2023-09-23T21:09:43.564163+0000","last_change":"2023-09-23T21:09:06.424530+0000","last_active":"2023-09-23T21:09:43.564163+0000","last_peered":"2023-09-23T21:09:43.564163+0000","last_clean":"2023-09-23T21:09:43.564163+0000","last_became_active":"2023-09-23T21:09:06.379456+0000","last_became_peered":"2023-09-23T21:09:06.379456+0000","last_unstale":"2023-09-23T21:09:43.564163+0000","last_undegraded":"2023-09-23T21:09:43.564163+0000","last_fullsized":"2023-09-23T21:09:43.564163+0000","mapping_epoch":394,"log_start":"374'8602","ondisk_log_start":"374'8602","created":346,"last_epoch_clean":395,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:05:47.261750+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:05:47.261750+0000","last_clean_scrub_stamp":"2023-09-23T21:05:47.261750+0000","objects_scrubbed":0,"log_size":4862,"log_dups_size":0,"ondisk_log_size":4862,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:23:21.748619+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":51576856,"num_objects":788,"num_object_clones":0,"num_object_copies":2364,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":788,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":13464,"num_write_kb":456001,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2889,"num_bytes_recovered":189267992,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"79'161","reported_seq":452,"reported_epoch":411,"state":"active+clean","last_fresh":"2023-09-23T21:09:34.892973+0000","last_change":"2023-09-23T21:09:34.893095+0000","last_active":"2023-09-23T21:09:34.892973+0000","last_peered":"2023-09-23T21:09:34.892973+0000","last_clean":"2023-09-23T21:09:34.892973+0000","last_became_active":"2023-09-23T21:09:11.388460+0000","last_became_peered":"2023-09-23T21:09:11.388460+0000","last_unstale":"2023-09-23T21:09:34.892973+0000","last_undegraded":"2023-09-23T21:09:34.892973+0000","last_fullsized":"2023-09-23T21:09:34.892973+0000","mapping_epoch":399,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":400,"parent":"0.0","parent_split_bits":3,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:09:34.892947+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:09:34.892947+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:49:55.576750+0000","scrub_duration":0.0081306369999999996,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,0],"acting":[2,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":392,"reported_epoch":410,"state":"active+clean","last_fresh":"2023-09-23T21:09:28.715378+0000","last_change":"2023-09-23T21:08:43.535723+0000","last_active":"2023-09-23T21:09:28.715378+0000","last_peered":"2023-09-23T21:09:28.715378+0000","last_clean":"2023-09-23T21:09:28.715378+0000","last_became_active":"2023-09-23T21:08:43.505805+0000","last_became_peered":"2023-09-23T21:08:43.505805+0000","last_unstale":"2023-09-23T21:09:28.715378+0000","last_undegraded":"2023-09-23T21:09:28.715378+0000","last_fullsized":"2023-09-23T21:09:28.715378+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:44:34.883525+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":385,"reported_epoch":410,"state":"active+clean","last_fresh":"2023-09-23T21:09:28.749008+0000","last_change":"2023-09-23T21:09:08.427533+0000","last_active":"2023-09-23T21:09:28.749008+0000","last_peered":"2023-09-23T21:09:28.749008+0000","last_clean":"2023-09-23T21:09:28.749008+0000","last_became_active":"2023-09-23T21:09:08.427187+0000","last_became_peered":"2023-09-23T21:09:08.427187+0000","last_unstale":"2023-09-23T21:09:28.749008+0000","last_undegraded":"2023-09-23T21:09:28.749008+0000","last_fullsized":"2023-09-23T21:09:28.749008+0000","mapping_epoch":396,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":397,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:24:11.334136+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,3],"acting":[1,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":213,"reported_epoch":410,"state":"active+clean","last_fresh":"2023-09-23T21:09:28.712679+0000","last_change":"2023-09-23T21:08:43.506053+0000","last_active":"2023-09-23T21:09:28.712679+0000","last_peered":"2023-09-23T21:09:28.712679+0000","last_clean":"2023-09-23T21:09:28.712679+0000","last_became_active":"2023-09-23T21:08:43.505716+0000","last_became_peered":"2023-09-23T21:08:43.505716+0000","last_unstale":"2023-09-23T21:09:28.712679+0000","last_undegraded":"2023-09-23T21:09:28.712679+0000","last_fullsized":"2023-09-23T21:09:28.712679+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:57:29.702126+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.c","version":"411'13546","reported_seq":18207,"reported_epoch":411,"state":"active+clean","last_fresh":"2023-09-23T21:09:44.053431+0000","last_change":"2023-09-23T21:09:19.634940+0000","last_active":"2023-09-23T21:09:44.053431+0000","last_peered":"2023-09-23T21:09:44.053431+0000","last_clean":"2023-09-23T21:09:44.053431+0000","last_became_active":"2023-09-23T21:09:19.597938+0000","last_became_peered":"2023-09-23T21:09:19.597938+0000","last_unstale":"2023-09-23T21:09:44.053431+0000","last_undegraded":"2023-09-23T21:09:44.053431+0000","last_fullsized":"2023-09-23T21:09:44.053431+0000","mapping_epoch":402,"log_start":"374'8657","ondisk_log_start":"374'8657","created":346,"last_epoch_clean":403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:05:47.261750+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:05:47.261750+0000","last_clean_scrub_stamp":"2023-09-23T21:05:47.261750+0000","objects_scrubbed":0,"log_size":4889,"log_dups_size":0,"ondisk_log_size":4889,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:53:39.210983+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47185920,"num_objects":720,"num_object_clones":0,"num_object_copies":2160,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":720,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13546,"num_write_kb":456512,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2280,"num_bytes_recovered":149422080,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.4","version":"79'161","reported_seq":745,"reported_epoch":410,"state":"active+clean","last_fresh":"2023-09-23T21:09:28.715361+0000","last_change":"2023-09-23T21:08:46.398346+0000","last_active":"2023-09-23T21:09:28.715361+0000","last_peered":"2023-09-23T21:09:28.715361+0000","last_clean":"2023-09-23T21:09:28.715361+0000","last_became_active":"2023-09-23T21:08:46.397918+0000","last_became_peered":"2023-09-23T21:08:46.397918+0000","last_unstale":"2023-09-23T21:09:28.715361+0000","last_undegraded":"2023-09-23T21:09:28.715361+0000","last_fullsized":"2023-09-23T21:09:28.715361+0000","mapping_epoch":385,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":386,"parent":"0.0","parent_split_bits":3,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:20.012764+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:20.012764+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:03:01.180190+0000","scrub_duration":0.0045251919999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":370,"reported_epoch":410,"state":"active+clean","last_fresh":"2023-09-23T21:09:28.716221+0000","last_change":"2023-09-23T21:08:43.500587+0000","last_active":"2023-09-23T21:09:28.716221+0000","last_peered":"2023-09-23T21:09:28.716221+0000","last_clean":"2023-09-23T21:09:28.716221+0000","last_became_active":"2023-09-23T21:08:43.500272+0000","last_became_peered":"2023-09-23T21:08:43.500272+0000","last_unstale":"2023-09-23T21:09:28.716221+0000","last_undegraded":"2023-09-23T21:09:28.716221+0000","last_fullsized":"2023-09-23T21:09:28.716221+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:42:06.186343+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,3],"acting":[6,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"70'1","reported_seq":419,"reported_epoch":410,"state":"active+clean","last_fresh":"2023-09-23T21:09:28.716608+0000","last_change":"2023-09-23T21:05:55.320603+0000","last_active":"2023-09-23T21:09:28.716608+0000","last_peered":"2023-09-23T21:09:28.716608+0000","last_clean":"2023-09-23T21:09:28.716608+0000","last_became_active":"2023-09-23T21:05:55.319956+0000","last_became_peered":"2023-09-23T21:05:55.319956+0000","last_unstale":"2023-09-23T21:09:28.716608+0000","last_undegraded":"2023-09-23T21:09:28.716608+0000","last_fullsized":"2023-09-23T21:09:28.716608+0000","mapping_epoch":350,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":351,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:41:03.995212+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":275,"reported_epoch":410,"state":"active+clean","last_fresh":"2023-09-23T21:09:28.718502+0000","last_change":"2023-09-23T21:09:08.405598+0000","last_active":"2023-09-23T21:09:28.718502+0000","last_peered":"2023-09-23T21:09:28.718502+0000","last_clean":"2023-09-23T21:09:28.718502+0000","last_became_active":"2023-09-23T21:09:08.405222+0000","last_became_peered":"2023-09-23T21:09:08.405222+0000","last_unstale":"2023-09-23T21:09:28.718502+0000","last_undegraded":"2023-09-23T21:09:28.718502+0000","last_fullsized":"2023-09-23T21:09:28.718502+0000","mapping_epoch":396,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":397,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:21:55.701363+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,0],"acting":[5,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":387,"reported_epoch":410,"state":"active+clean","last_fresh":"2023-09-23T21:09:32.390819+0000","last_change":"2023-09-23T21:08:43.527566+0000","last_active":"2023-09-23T21:09:32.390819+0000","last_peered":"2023-09-23T21:09:32.390819+0000","last_clean":"2023-09-23T21:09:32.390819+0000","last_became_active":"2023-09-23T21:08:43.527246+0000","last_became_peered":"2023-09-23T21:08:43.527246+0000","last_unstale":"2023-09-23T21:09:32.390819+0000","last_undegraded":"2023-09-23T21:09:32.390819+0000","last_fullsized":"2023-09-23T21:09:32.390819+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:06:28.535063+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":398,"reported_epoch":410,"state":"active+clean","last_fresh":"2023-09-23T21:09:28.717216+0000","last_change":"2023-09-23T21:05:55.313495+0000","last_active":"2023-09-23T21:09:28.717216+0000","last_peered":"2023-09-23T21:09:28.717216+0000","last_clean":"2023-09-23T21:09:28.717216+0000","last_became_active":"2023-09-23T21:05:55.313065+0000","last_became_peered":"2023-09-23T21:05:55.313065+0000","last_unstale":"2023-09-23T21:09:28.717216+0000","last_undegraded":"2023-09-23T21:09:28.717216+0000","last_fullsized":"2023-09-23T21:09:28.717216+0000","mapping_epoch":350,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":351,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:05:22.675197+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":64,"reported_epoch":410,"state":"active+clean","last_fresh":"2023-09-23T21:09:28.747357+0000","last_change":"2023-09-23T21:09:08.454444+0000","last_active":"2023-09-23T21:09:28.747357+0000","last_peered":"2023-09-23T21:09:28.747357+0000","last_clean":"2023-09-23T21:09:28.747357+0000","last_became_active":"2023-09-23T21:09:08.411670+0000","last_became_peered":"2023-09-23T21:09:08.411670+0000","last_unstale":"2023-09-23T21:09:28.747357+0000","last_undegraded":"2023-09-23T21:09:28.747357+0000","last_fullsized":"2023-09-23T21:09:28.747357+0000","mapping_epoch":396,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":397,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:43:04.009432+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,2],"acting":[1,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":262,"reported_epoch":410,"state":"active+clean","last_fresh":"2023-09-23T21:09:28.713230+0000","last_change":"2023-09-23T21:09:08.440185+0000","last_active":"2023-09-23T21:09:28.713230+0000","last_peered":"2023-09-23T21:09:28.713230+0000","last_clean":"2023-09-23T21:09:28.713230+0000","last_became_active":"2023-09-23T21:09:08.439562+0000","last_became_peered":"2023-09-23T21:09:08.439562+0000","last_unstale":"2023-09-23T21:09:28.713230+0000","last_undegraded":"2023-09-23T21:09:28.713230+0000","last_fullsized":"2023-09-23T21:09:28.713230+0000","mapping_epoch":396,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":397,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:39:54.771238+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":185,"reported_epoch":410,"state":"active+clean","last_fresh":"2023-09-23T21:09:28.717442+0000","last_change":"2023-09-23T21:08:54.706324+0000","last_active":"2023-09-23T21:09:28.717442+0000","last_peered":"2023-09-23T21:09:28.717442+0000","last_clean":"2023-09-23T21:09:28.717442+0000","last_became_active":"2023-09-23T21:08:54.681312+0000","last_became_peered":"2023-09-23T21:08:54.681312+0000","last_unstale":"2023-09-23T21:09:28.717442+0000","last_undegraded":"2023-09-23T21:09:28.717442+0000","last_fullsized":"2023-09-23T21:09:28.717442+0000","mapping_epoch":388,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":389,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:08:16.335346+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,1],"acting":[2,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":112,"reported_epoch":410,"state":"active+clean","last_fresh":"2023-09-23T21:09:28.713027+0000","last_change":"2023-09-23T21:09:08.439959+0000","last_active":"2023-09-23T21:09:28.713027+0000","last_peered":"2023-09-23T21:09:28.713027+0000","last_clean":"2023-09-23T21:09:28.713027+0000","last_became_active":"2023-09-23T21:09:08.439676+0000","last_became_peered":"2023-09-23T21:09:08.439676+0000","last_unstale":"2023-09-23T21:09:28.713027+0000","last_undegraded":"2023-09-23T21:09:28.713027+0000","last_fullsized":"2023-09-23T21:09:28.713027+0000","mapping_epoch":396,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":397,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:24:22.416220+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,2],"acting":[3,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":412,"reported_epoch":410,"state":"active+clean","last_fresh":"2023-09-23T21:09:28.715453+0000","last_change":"2023-09-23T21:08:54.686159+0000","last_active":"2023-09-23T21:09:28.715453+0000","last_peered":"2023-09-23T21:09:28.715453+0000","last_clean":"2023-09-23T21:09:28.715453+0000","last_became_active":"2023-09-23T21:08:54.685798+0000","last_became_peered":"2023-09-23T21:08:54.685798+0000","last_unstale":"2023-09-23T21:09:28.715453+0000","last_undegraded":"2023-09-23T21:09:28.715453+0000","last_fullsized":"2023-09-23T21:09:28.715453+0000","mapping_epoch":388,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":389,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:02:10.332347+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,1],"acting":[3,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":401,"reported_epoch":410,"state":"active+clean","last_fresh":"2023-09-23T21:09:28.712698+0000","last_change":"2023-09-23T20:51:10.838334+0000","last_active":"2023-09-23T21:09:28.712698+0000","last_peered":"2023-09-23T21:09:28.712698+0000","last_clean":"2023-09-23T21:09:28.712698+0000","last_became_active":"2023-09-23T20:51:10.838026+0000","last_became_peered":"2023-09-23T20:51:10.838026+0000","last_unstale":"2023-09-23T21:09:28.712698+0000","last_undegraded":"2023-09-23T21:09:28.712698+0000","last_fullsized":"2023-09-23T21:09:28.712698+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:04:37.009457+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":436,"reported_epoch":410,"state":"active+clean","last_fresh":"2023-09-23T21:09:28.715550+0000","last_change":"2023-09-23T21:05:04.325268+0000","last_active":"2023-09-23T21:09:28.715550+0000","last_peered":"2023-09-23T21:09:28.715550+0000","last_clean":"2023-09-23T21:09:28.715550+0000","last_became_active":"2023-09-23T21:05:04.303430+0000","last_became_peered":"2023-09-23T21:05:04.303430+0000","last_unstale":"2023-09-23T21:09:28.715550+0000","last_undegraded":"2023-09-23T21:09:28.715550+0000","last_fullsized":"2023-09-23T21:09:28.715550+0000","mapping_epoch":332,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:17:44.879586+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":2216,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":25,"reported_epoch":410,"state":"active+clean","last_fresh":"2023-09-23T21:09:28.746875+0000","last_change":"2023-09-23T21:09:08.428128+0000","last_active":"2023-09-23T21:09:28.746875+0000","last_peered":"2023-09-23T21:09:28.746875+0000","last_clean":"2023-09-23T21:09:28.746875+0000","last_became_active":"2023-09-23T21:09:08.427481+0000","last_became_peered":"2023-09-23T21:09:08.427481+0000","last_unstale":"2023-09-23T21:09:28.746875+0000","last_undegraded":"2023-09-23T21:09:28.746875+0000","last_fullsized":"2023-09-23T21:09:28.746875+0000","mapping_epoch":396,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":397,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:46:06.052816+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,2],"acting":[1,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":40,"num_bytes_recovered":7326,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":126976,"data_stored":21462,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5834,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":21,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3381,"ondisk_log_size":3381,"up":63,"acting":63,"num_store_stats":8},{"poolid":9,"num_pg":16,"stat_sum":{"num_bytes":811270168,"num_objects":12380,"num_object_clones":0,"num_object_copies":37140,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":12380,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":214168,"num_write_kb":7249473,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":23332,"num_bytes_recovered":1528823832,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":3368960000,"data_stored":3368943712,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":83000,"ondisk_log_size":83000,"up":48,"acting":48,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":0,"seq":236223201631,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":6,"up_from":164,"seq":704374636715,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":389732,"kb_used_data":138524,"kb_used_omap":10,"kb_used_meta":251189,"kb_avail":93355420,"statfs":{"total":95995035648,"available":95595950080,"internally_reserved":0,"allocated":141848576,"data_stored":140671904,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":10328,"internal_metadata":257218472},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:09:04 2023","interfaces":[{"interface":"back","average":{"1min":1.359,"5min":1.7929999999999999,"15min":1.6759999999999999},"min":{"1min":0.253,"5min":0.224,"15min":0.20999999999999999},"max":{"1min":6.0800000000000001,"5min":9.9890000000000008,"15min":14.151999999999999},"last":3.9399999999999999},{"interface":"front","average":{"1min":2.5430000000000001,"5min":1.871,"15min":1.7669999999999999},"min":{"1min":0.27100000000000002,"5min":0.224,"15min":0.20999999999999999},"max":{"1min":10.868,"5min":10.868,"15min":11.023},"last":3.7799999999999998}]},{"osd":1,"last update":"Sat Sep 23 21:08:56 2023","interfaces":[{"interface":"back","average":{"1min":2.802,"5min":2.0590000000000002,"15min":1.7529999999999999},"min":{"1min":0.27100000000000002,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":10.218999999999999,"5min":12.904999999999999,"15min":12.904999999999999},"last":0.98399999999999999},{"interface":"front","average":{"1min":1.5940000000000001,"5min":1.6299999999999999,"15min":1.718},"min":{"1min":0.35099999999999998,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":4.7450000000000001,"5min":6.7569999999999997,"15min":6.7569999999999997},"last":0.30499999999999999}]},{"osd":2,"last update":"Sat Sep 23 21:09:04 2023","interfaces":[{"interface":"back","average":{"1min":3.581,"5min":1.867,"15min":2.0760000000000001},"min":{"1min":0.36699999999999999,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":23.954999999999998,"5min":23.954999999999998,"15min":23.954999999999998},"last":3.3119999999999998},{"interface":"front","average":{"1min":2.4260000000000002,"5min":1.8160000000000001,"15min":1.98},"min":{"1min":0.36899999999999999,"5min":0.27300000000000002,"15min":0.247},"max":{"1min":6.9729999999999999,"5min":13.385999999999999,"15min":20.195},"last":2.802}]},{"osd":3,"last update":"Sat Sep 23 21:09:04 2023","interfaces":[{"interface":"back","average":{"1min":1.889,"5min":2.109,"15min":2.0619999999999998},"min":{"1min":0.34300000000000003,"5min":0.309,"15min":0.28699999999999998},"max":{"1min":6.7889999999999997,"5min":20.625,"15min":43.085999999999999},"last":0.46000000000000002},{"interface":"front","average":{"1min":2.3159999999999998,"5min":2.2200000000000002,"15min":2.1219999999999999},"min":{"1min":0.38400000000000001,"5min":0.30499999999999999,"15min":0.28599999999999998},"max":{"1min":7.6189999999999998,"5min":12.513,"15min":42.405000000000001},"last":0.35899999999999999}]},{"osd":4,"last update":"Sat Sep 23 21:09:04 2023","interfaces":[{"interface":"back","average":{"1min":1.1220000000000001,"5min":0.96599999999999997,"15min":0.92000000000000004},"min":{"1min":0.28299999999999997,"5min":0.25700000000000001,"15min":0.23000000000000001},"max":{"1min":7.2759999999999998,"5min":7.2759999999999998,"15min":9.0079999999999991},"last":0.41199999999999998},{"interface":"front","average":{"1min":0.81000000000000005,"5min":0.92800000000000005,"15min":0.74199999999999999},"min":{"1min":0.29399999999999998,"5min":0.28799999999999998,"15min":0.25800000000000001},"max":{"1min":3.6190000000000002,"5min":6.4050000000000002,"15min":6.4050000000000002},"last":0.38100000000000001}]},{"osd":5,"last update":"Sat Sep 23 21:09:04 2023","interfaces":[{"interface":"back","average":{"1min":1.2769999999999999,"5min":1.2310000000000001,"15min":0.97799999999999998},"min":{"1min":0.309,"5min":0.27800000000000002,"15min":0.27300000000000002},"max":{"1min":6.9370000000000003,"5min":7.9710000000000001,"15min":7.9710000000000001},"last":0.47499999999999998},{"interface":"front","average":{"1min":0.58499999999999996,"5min":1.0149999999999999,"15min":0.93899999999999995},"min":{"1min":0.32100000000000001,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":1.3839999999999999,"5min":10.657,"15min":10.657},"last":1.8959999999999999}]},{"osd":7,"last update":"Sat Sep 23 21:09:04 2023","interfaces":[{"interface":"back","average":{"1min":12.522,"5min":3.488,"15min":1.8109999999999999},"min":{"1min":0.39500000000000002,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":205.95599999999999,"5min":205.95599999999999,"15min":205.95599999999999},"last":0.432},{"interface":"front","average":{"1min":13.239000000000001,"5min":3.4830000000000001,"15min":1.6779999999999999},"min":{"1min":0.33000000000000002,"5min":0.28299999999999997,"15min":0.28100000000000003},"max":{"1min":206.05699999999999,"5min":206.05699999999999,"15min":206.05699999999999},"last":0.443}]}]},{"osd":1,"up_from":350,"seq":1503238553647,"num_pgs":62,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":767592,"kb_used_data":517664,"kb_used_omap":10,"kb_used_meta":249717,"kb_avail":92977560,"statfs":{"total":95995035648,"available":95209021440,"internally_reserved":0,"allocated":530087936,"data_stored":528219334,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":10498,"internal_metadata":255710974},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:09:01 2023","interfaces":[{"interface":"back","average":{"1min":2.1850000000000001,"5min":2.2770000000000001,"15min":2.0289999999999999},"min":{"1min":0.20499999999999999,"5min":0.20499999999999999,"15min":0.20499999999999999},"max":{"1min":6.9359999999999999,"5min":15.728,"15min":15.728},"last":0.66800000000000004},{"interface":"front","average":{"1min":2.512,"5min":2.7189999999999999,"15min":2.613},"min":{"1min":0.23100000000000001,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":7.8230000000000004,"5min":14.468999999999999,"15min":14.468999999999999},"last":4.7130000000000001}]},{"osd":2,"last update":"Sat Sep 23 21:09:01 2023","interfaces":[{"interface":"back","average":{"1min":2.2869999999999999,"5min":2.5110000000000001,"15min":2.3969999999999998},"min":{"1min":0.248,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":9.0050000000000008,"5min":10.769,"15min":10.769},"last":0.23999999999999999},{"interface":"front","average":{"1min":1.827,"5min":2.1749999999999998,"15min":2.0640000000000001},"min":{"1min":0.307,"5min":0.214,"15min":0.214},"max":{"1min":5.3719999999999999,"5min":15.683999999999999,"15min":15.683999999999999},"last":0.254}]},{"osd":3,"last update":"Sat Sep 23 21:09:01 2023","interfaces":[{"interface":"back","average":{"1min":2.202,"5min":2.734,"15min":2.7429999999999999},"min":{"1min":0.39300000000000002,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":5.0629999999999997,"5min":15.135999999999999,"15min":15.135999999999999},"last":0.29599999999999999},{"interface":"front","average":{"1min":2.1269999999999998,"5min":3.0939999999999999,"15min":3.0739999999999998},"min":{"1min":0.27300000000000002,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":7.7329999999999997,"5min":14.794,"15min":14.794},"last":2.1890000000000001}]},{"osd":4,"last update":"Sat Sep 23 21:09:01 2023","interfaces":[{"interface":"back","average":{"1min":1.9370000000000001,"5min":1.966,"15min":1.8220000000000001},"min":{"1min":0.34999999999999998,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":7.3029999999999999,"5min":10.391,"15min":10.391},"last":0.27500000000000002},{"interface":"front","average":{"1min":2.3620000000000001,"5min":1.8120000000000001,"15min":1.6719999999999999},"min":{"1min":0.33700000000000002,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":7.3780000000000001,"5min":7.3780000000000001,"15min":7.3780000000000001},"last":0.36199999999999999}]},{"osd":5,"last update":"Sat Sep 23 21:09:01 2023","interfaces":[{"interface":"back","average":{"1min":2.706,"5min":2.5630000000000002,"15min":2.5910000000000002},"min":{"1min":0.29799999999999999,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":7.7670000000000003,"5min":8.0899999999999999,"15min":8.0899999999999999},"last":2.2280000000000002},{"interface":"front","average":{"1min":1.6279999999999999,"5min":2.5899999999999999,"15min":2.6360000000000001},"min":{"1min":0.41999999999999998,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":4.226,"5min":10.339,"15min":10.339},"last":0.35099999999999998}]},{"osd":6,"last update":"Sat Sep 23 21:09:01 2023","interfaces":[{"interface":"back","average":{"1min":2.0230000000000001,"5min":2.3010000000000002,"15min":2.327},"min":{"1min":0.38100000000000001,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":7.7960000000000003,"5min":8.1880000000000006,"15min":8.1880000000000006},"last":2.2549999999999999},{"interface":"front","average":{"1min":2.7919999999999998,"5min":2.6589999999999998,"15min":2.5009999999999999},"min":{"1min":0.39200000000000002,"5min":0.39200000000000002,"15min":0.39200000000000002},"max":{"1min":16.494,"5min":16.494,"15min":16.494},"last":0.378}]},{"osd":7,"last update":"Sat Sep 23 21:09:01 2023","interfaces":[{"interface":"back","average":{"1min":2.3450000000000002,"5min":2.0840000000000001,"15min":2.056},"min":{"1min":0.44500000000000001,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":5.2130000000000001,"5min":6.5570000000000004,"15min":6.5570000000000004},"last":0.504},{"interface":"front","average":{"1min":2.8399999999999999,"5min":2.6019999999999999,"15min":2.6179999999999999},"min":{"1min":0.48199999999999998,"5min":0.41099999999999998,"15min":0.41099999999999998},"max":{"1min":7.9539999999999997,"5min":8.1440000000000001,"15min":8.1440000000000001},"last":2.278}]}]},{"osd":0,"up_from":83,"seq":356482285812,"num_pgs":55,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":853120,"kb_used_data":327544,"kb_used_omap":28,"kb_used_meta":525539,"kb_avail":92892032,"statfs":{"total":95995035648,"available":95121440768,"internally_reserved":0,"allocated":335405056,"data_stored":333574747,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":28848,"internal_metadata":538152784},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 21:08:58 2023","interfaces":[{"interface":"back","average":{"1min":2.7250000000000001,"5min":2.0179999999999998,"15min":1.55},"min":{"1min":0.23300000000000001,"5min":0.192,"15min":0.192},"max":{"1min":8.234,"5min":12.994,"15min":12.994},"last":4.3109999999999999},{"interface":"front","average":{"1min":2.8330000000000002,"5min":1.8200000000000001,"15min":1.456},"min":{"1min":0.27600000000000002,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":7.5449999999999999,"5min":14.646000000000001,"15min":14.646000000000001},"last":6.944}]},{"osd":2,"last update":"Sat Sep 23 21:09:13 2023","interfaces":[{"interface":"back","average":{"1min":2.218,"5min":2.21,"15min":2.7829999999999999},"min":{"1min":0.24299999999999999,"5min":0.183,"15min":0.183},"max":{"1min":10.945,"5min":13.032,"15min":21.053000000000001},"last":0.24399999999999999},{"interface":"front","average":{"1min":2.5169999999999999,"5min":2.2709999999999999,"15min":3.0369999999999999},"min":{"1min":0.29599999999999999,"5min":0.26600000000000001,"15min":0.23100000000000001},"max":{"1min":8.1539999999999999,"5min":14.558999999999999,"15min":21.747},"last":1.2869999999999999}]},{"osd":3,"last update":"Sat Sep 23 21:09:36 2023","interfaces":[{"interface":"back","average":{"1min":5.5759999999999996,"5min":2.8460000000000001,"15min":3.0289999999999999},"min":{"1min":0.28000000000000003,"5min":0.252,"15min":0.252},"max":{"1min":24.048999999999999,"5min":24.048999999999999,"15min":41.079000000000001},"last":0.57199999999999995},{"interface":"front","average":{"1min":4.1219999999999999,"5min":2.6680000000000001,"15min":3.04},"min":{"1min":0.26000000000000001,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":13.281000000000001,"5min":19.91,"15min":21.167000000000002},"last":1.173}]},{"osd":4,"last update":"Sat Sep 23 21:09:13 2023","interfaces":[{"interface":"back","average":{"1min":2.8730000000000002,"5min":2.0529999999999999,"15min":2.3149999999999999},"min":{"1min":0.51200000000000001,"5min":0.32400000000000001,"15min":0.28599999999999998},"max":{"1min":7.9160000000000004,"5min":12.394,"15min":21.594999999999999},"last":1.3180000000000001},{"interface":"front","average":{"1min":1.171,"5min":1.9450000000000001,"15min":2.0099999999999998},"min":{"1min":0.30399999999999999,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":5.0570000000000004,"5min":18.516999999999999,"15min":18.724},"last":0.33100000000000002}]},{"osd":5,"last update":"Sat Sep 23 21:09:13 2023","interfaces":[{"interface":"back","average":{"1min":1.9850000000000001,"5min":1.863,"15min":1.9119999999999999},"min":{"1min":0.29999999999999999,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":8.2810000000000006,"5min":12.946999999999999,"15min":12.946999999999999},"last":0.49099999999999999},{"interface":"front","average":{"1min":2.5099999999999998,"5min":1.7849999999999999,"15min":2.266},"min":{"1min":0.434,"5min":0.373,"15min":0.32700000000000001},"max":{"1min":7.9800000000000004,"5min":12.353,"15min":20.512},"last":1.3460000000000001}]},{"osd":6,"last update":"Sat Sep 23 21:09:09 2023","interfaces":[{"interface":"back","average":{"1min":1.73,"5min":2.0390000000000001,"15min":2.0310000000000001},"min":{"1min":0.34899999999999998,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":6.992,"5min":18.571000000000002,"15min":18.571000000000002},"last":0.38700000000000001},{"interface":"front","average":{"1min":1.7,"5min":2.1909999999999998,"15min":2.1040000000000001},"min":{"1min":0.42599999999999999,"5min":0.40300000000000002,"15min":0.377},"max":{"1min":5.5599999999999996,"5min":18.603000000000002,"15min":18.603000000000002},"last":0.51300000000000001}]},{"osd":7,"last update":"Sat Sep 23 21:09:13 2023","interfaces":[{"interface":"back","average":{"1min":2.0099999999999998,"5min":1.9019999999999999,"15min":1.976},"min":{"1min":0.33400000000000002,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":8.3360000000000003,"5min":12.907999999999999,"15min":12.907999999999999},"last":0.53600000000000003},{"interface":"front","average":{"1min":1.5700000000000001,"5min":2.206,"15min":2.2789999999999999},"min":{"1min":0.38200000000000001,"5min":0.38200000000000001,"15min":0.377},"max":{"1min":7.1390000000000002,"5min":18.634,"15min":40.984999999999999},"last":0.42299999999999999}]}]},{"osd":2,"up_from":93,"seq":399431958759,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":917768,"kb_used_data":295744,"kb_used_omap":81,"kb_used_meta":621934,"kb_avail":92827384,"statfs":{"total":95995035648,"available":95055241216,"internally_reserved":0,"allocated":302841856,"data_stored":301002746,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":83750,"internal_metadata":636860634},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:09:01 2023","interfaces":[{"interface":"back","average":{"1min":2.9769999999999999,"5min":1.885,"15min":2.2210000000000001},"min":{"1min":0.26000000000000001,"5min":0.23000000000000001,"15min":0.17999999999999999},"max":{"1min":9.2720000000000002,"5min":11.124000000000001,"15min":29.498000000000001},"last":3.1219999999999999},{"interface":"front","average":{"1min":1.8999999999999999,"5min":1.677,"15min":2.4750000000000001},"min":{"1min":0.26900000000000002,"5min":0.26600000000000001,"15min":0.193},"max":{"1min":5.1079999999999997,"5min":9.2880000000000003,"15min":17.251000000000001},"last":5.2599999999999998}]},{"osd":1,"last update":"Sat Sep 23 21:09:01 2023","interfaces":[{"interface":"back","average":{"1min":2.5649999999999999,"5min":3.153,"15min":3.0339999999999998},"min":{"1min":0.23699999999999999,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":6.5359999999999996,"5min":34.396999999999998,"15min":34.396999999999998},"last":4.4489999999999998},{"interface":"front","average":{"1min":3.4399999999999999,"5min":3.3599999999999999,"15min":3.5680000000000001},"min":{"1min":0.34999999999999998,"5min":0.217,"15min":0.217},"max":{"1min":9.6579999999999995,"5min":34.462000000000003,"15min":34.462000000000003},"last":0.57299999999999995}]},{"osd":3,"last update":"Sat Sep 23 21:08:48 2023","interfaces":[{"interface":"back","average":{"1min":2.3330000000000002,"5min":2.5049999999999999,"15min":3.2810000000000001},"min":{"1min":0.32600000000000001,"5min":0.23899999999999999,"15min":0.23300000000000001},"max":{"1min":9.0739999999999998,"5min":35.549999999999997,"15min":41.057000000000002},"last":0.69399999999999995},{"interface":"front","average":{"1min":2.4780000000000002,"5min":2.7970000000000002,"15min":3.7650000000000001},"min":{"1min":0.35699999999999998,"5min":0.253,"15min":0.22800000000000001},"max":{"1min":5.423,"5min":36.951999999999998,"15min":41.012},"last":4.4729999999999999}]},{"osd":4,"last update":"Sat Sep 23 21:09:01 2023","interfaces":[{"interface":"back","average":{"1min":1.8380000000000001,"5min":1.581,"15min":1.667},"min":{"1min":0.31900000000000001,"5min":0.29199999999999998,"15min":0.28399999999999997},"max":{"1min":4.3810000000000002,"5min":8.7569999999999997,"15min":10.045},"last":0.55600000000000005},{"interface":"front","average":{"1min":1.587,"5min":1.8520000000000001,"15min":2.109},"min":{"1min":0.34499999999999997,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":3.7050000000000001,"5min":12.49,"15min":14.039999999999999},"last":4.367}]},{"osd":5,"last update":"Sat Sep 23 21:09:01 2023","interfaces":[{"interface":"back","average":{"1min":1.861,"5min":1.798,"15min":1.841},"min":{"1min":0.35499999999999998,"5min":0.316,"15min":0.29299999999999998},"max":{"1min":4.8289999999999997,"5min":8.907,"15min":10.069000000000001},"last":0.59499999999999997},{"interface":"front","average":{"1min":1.992,"5min":1.917,"15min":2.0979999999999999},"min":{"1min":0.46400000000000002,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":4.1029999999999998,"5min":12.57,"15min":13.98},"last":4.3239999999999998}]},{"osd":6,"last update":"Sat Sep 23 21:09:01 2023","interfaces":[{"interface":"back","average":{"1min":2.1629999999999998,"5min":2.1930000000000001,"15min":2.1349999999999998},"min":{"1min":0.39800000000000002,"5min":0.35699999999999998,"15min":0.312},"max":{"1min":8.7309999999999999,"5min":12.518000000000001,"15min":14.244999999999999},"last":4.4240000000000004},{"interface":"front","average":{"1min":2.637,"5min":2.4510000000000001,"15min":2.1240000000000001},"min":{"1min":0.56399999999999995,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":9.3230000000000004,"5min":12.544,"15min":14.068},"last":4.399}]},{"osd":7,"last update":"Sat Sep 23 21:09:01 2023","interfaces":[{"interface":"back","average":{"1min":2.5649999999999999,"5min":1.9370000000000001,"15min":1.569},"min":{"1min":0.40799999999999997,"5min":0.34100000000000003,"15min":0.315},"max":{"1min":6.9390000000000001,"5min":8.7599999999999998,"15min":13.427},"last":0.36899999999999999},{"interface":"front","average":{"1min":2.1509999999999998,"5min":2.0379999999999998,"15min":1.9199999999999999},"min":{"1min":0.432,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":5.9740000000000002,"5min":10.92,"15min":10.92},"last":0.628}]}]},{"osd":3,"up_from":98,"seq":420906795233,"num_pgs":62,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1200736,"kb_used_data":495128,"kb_used_omap":7,"kb_used_meta":705336,"kb_avail":92544416,"statfs":{"total":95995035648,"available":94765481984,"internally_reserved":0,"allocated":507011072,"data_stored":505204862,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7576,"internal_metadata":722264680},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:08:51 2023","interfaces":[{"interface":"back","average":{"1min":3.3380000000000001,"5min":2.4950000000000001,"15min":3.02},"min":{"1min":0.28699999999999998,"5min":0.19900000000000001,"15min":0.188},"max":{"1min":14.08,"5min":14.08,"15min":27.962},"last":2.1840000000000002},{"interface":"front","average":{"1min":2.133,"5min":2.2370000000000001,"15min":2.919},"min":{"1min":0.26800000000000002,"5min":0.20599999999999999,"15min":0.19800000000000001},"max":{"1min":5.7969999999999997,"5min":11.483000000000001,"15min":28.058},"last":3.0739999999999998}]},{"osd":1,"last update":"Sat Sep 23 21:09:02 2023","interfaces":[{"interface":"back","average":{"1min":3.2490000000000001,"5min":3.0710000000000002,"15min":2.452},"min":{"1min":0.44400000000000001,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":16.939,"5min":16.939,"15min":16.939},"last":1.466},{"interface":"front","average":{"1min":2.5950000000000002,"5min":2.7709999999999999,"15min":2.4039999999999999},"min":{"1min":0.29299999999999998,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":7.9409999999999998,"5min":13.811999999999999,"15min":13.811999999999999},"last":3.121}]},{"osd":2,"last update":"Sat Sep 23 21:08:51 2023","interfaces":[{"interface":"back","average":{"1min":2.1989999999999998,"5min":2.355,"15min":3.0379999999999998},"min":{"1min":0.32500000000000001,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":9.0169999999999995,"5min":14.287000000000001,"15min":28.251999999999999},"last":0.71799999999999997},{"interface":"front","average":{"1min":2.9750000000000001,"5min":2.548,"15min":3.097},"min":{"1min":0.39400000000000002,"5min":0.29399999999999998,"15min":0.22},"max":{"1min":14.195,"5min":14.195,"15min":25.305},"last":1.448}]},{"osd":4,"last update":"Sat Sep 23 21:08:51 2023","interfaces":[{"interface":"back","average":{"1min":2.3889999999999998,"5min":1.843,"15min":2.052},"min":{"1min":0.38900000000000001,"5min":0.28799999999999998,"15min":0.28399999999999997},"max":{"1min":11.614000000000001,"5min":11.614000000000001,"15min":18.239000000000001},"last":0.66500000000000004},{"interface":"front","average":{"1min":3.7090000000000001,"5min":2.202,"15min":2.1779999999999999},"min":{"1min":0.52700000000000002,"5min":0.32100000000000001,"15min":0.29499999999999998},"max":{"1min":14.119,"5min":14.119,"15min":22.853000000000002},"last":1.3979999999999999}]},{"osd":5,"last update":"Sat Sep 23 21:08:51 2023","interfaces":[{"interface":"back","average":{"1min":2.7210000000000001,"5min":2.1890000000000001,"15min":2.2080000000000002},"min":{"1min":0.34999999999999998,"5min":0.314,"15min":0.314},"max":{"1min":8.5440000000000005,"5min":12.815,"15min":18.324999999999999},"last":0.625},{"interface":"front","average":{"1min":2.4830000000000001,"5min":2.4500000000000002,"15min":2.391},"min":{"1min":0.44500000000000001,"5min":0.33700000000000002,"15min":0.33100000000000002},"max":{"1min":7.2060000000000004,"5min":14.029,"15min":14.029},"last":3.149}]},{"osd":6,"last update":"Sat Sep 23 21:09:08 2023","interfaces":[{"interface":"back","average":{"1min":3.4049999999999998,"5min":2.2559999999999998,"15min":1.9179999999999999},"min":{"1min":0.49299999999999999,"5min":0.34599999999999997,"15min":0.33500000000000002},"max":{"1min":18.323,"5min":18.323,"15min":18.323},"last":1.4299999999999999},{"interface":"front","average":{"1min":1.669,"5min":2.2010000000000001,"15min":2.1890000000000001},"min":{"1min":0.47299999999999998,"5min":0.377,"15min":0.373},"max":{"1min":9.6300000000000008,"5min":9.6300000000000008,"15min":13.576000000000001},"last":3.198}]},{"osd":7,"last update":"Sat Sep 23 21:08:51 2023","interfaces":[{"interface":"back","average":{"1min":1.925,"5min":2.1539999999999999,"15min":2.3719999999999999},"min":{"1min":0.40899999999999997,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":7.2939999999999996,"5min":14.81,"15min":18.289999999999999},"last":0.69099999999999995},{"interface":"front","average":{"1min":2.948,"5min":2.4249999999999998,"15min":2.4399999999999999},"min":{"1min":0.48099999999999998,"5min":0.40300000000000002,"15min":0.379},"max":{"1min":7.5229999999999997,"5min":12.624000000000001,"15min":13.603999999999999},"last":3.1739999999999999}]}]},{"osd":4,"up_from":31,"seq":133143986540,"num_pgs":15,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":551196,"kb_used_data":256916,"kb_used_omap":0,"kb_used_meta":294272,"kb_avail":93193956,"statfs":{"total":95995035648,"available":95430610944,"internally_reserved":0,"allocated":263081984,"data_stored":261909278,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":301334528},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:09:08 2023","interfaces":[{"interface":"back","average":{"1min":2.2370000000000001,"5min":1.5900000000000001,"15min":1.6120000000000001},"min":{"1min":0.26200000000000001,"5min":0.222,"15min":0.214},"max":{"1min":6.351,"5min":11.537000000000001,"15min":11.784000000000001},"last":0.29299999999999998},{"interface":"front","average":{"1min":3.8919999999999999,"5min":2.3460000000000001,"15min":1.958},"min":{"1min":0.32500000000000001,"5min":0.254,"15min":0.254},"max":{"1min":19.109999999999999,"5min":19.109999999999999,"15min":19.109999999999999},"last":0.63800000000000001}]},{"osd":1,"last update":"Sat Sep 23 21:09:02 2023","interfaces":[{"interface":"back","average":{"1min":2.8660000000000001,"5min":3.367,"15min":3.669},"min":{"1min":0.33400000000000002,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":14.798999999999999,"5min":37.960000000000001,"15min":37.960000000000001},"last":1.371},{"interface":"front","average":{"1min":2.1019999999999999,"5min":3.0680000000000001,"15min":3.4359999999999999},"min":{"1min":0.27000000000000002,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":14.403,"5min":37.933,"15min":37.933},"last":1.3560000000000001}]},{"osd":2,"last update":"Sat Sep 23 21:09:08 2023","interfaces":[{"interface":"back","average":{"1min":3.02,"5min":1.9179999999999999,"15min":1.952},"min":{"1min":0.308,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":16.928000000000001,"5min":16.928000000000001,"15min":16.928000000000001},"last":0.32200000000000001},{"interface":"front","average":{"1min":3.008,"5min":1.96,"15min":1.974},"min":{"1min":0.308,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":14.417999999999999,"5min":14.417999999999999,"15min":15.262},"last":5.2990000000000004}]},{"osd":3,"last update":"Sat Sep 23 21:09:41 2023","interfaces":[{"interface":"back","average":{"1min":3.1960000000000002,"5min":2.4199999999999999,"15min":2.3849999999999998},"min":{"1min":0.39500000000000002,"5min":0.29799999999999999,"15min":0.28899999999999998},"max":{"1min":14.585000000000001,"5min":15.01,"15min":15.01},"last":1.5580000000000001},{"interface":"front","average":{"1min":3.1960000000000002,"5min":2.3330000000000002,"15min":2.3039999999999998},"min":{"1min":0.40699999999999997,"5min":0.32400000000000001,"15min":0.30199999999999999},"max":{"1min":14.366,"5min":15.004,"15min":15.004},"last":1.5469999999999999}]},{"osd":5,"last update":"Sat Sep 23 21:08:57 2023","interfaces":[{"interface":"back","average":{"1min":2.137,"5min":1.238,"15min":0.84099999999999997},"min":{"1min":0.29599999999999999,"5min":0.24299999999999999,"15min":0.22700000000000001},"max":{"1min":14.298,"5min":14.298,"15min":14.298},"last":0.26100000000000001},{"interface":"front","average":{"1min":2.0259999999999998,"5min":1.3520000000000001,"15min":0.98499999999999999},"min":{"1min":0.26600000000000001,"5min":0.255,"15min":0.251},"max":{"1min":9.8979999999999997,"5min":9.8979999999999997,"15min":9.8979999999999997},"last":0.67400000000000004}]},{"osd":6,"last update":"Sat Sep 23 21:09:02 2023","interfaces":[{"interface":"back","average":{"1min":2.218,"5min":1.504,"15min":1.0840000000000001},"min":{"1min":0.29699999999999999,"5min":0.26800000000000002,"15min":0.26500000000000001},"max":{"1min":14.704000000000001,"5min":14.704000000000001,"15min":14.704000000000001},"last":0.82499999999999996},{"interface":"front","average":{"1min":1.6279999999999999,"5min":1.073,"15min":0.84699999999999998},"min":{"1min":0.32500000000000001,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":14.387,"5min":14.387,"15min":14.387},"last":0.34999999999999998}]},{"osd":7,"last update":"Sat Sep 23 21:09:41 2023","interfaces":[{"interface":"back","average":{"1min":1.5569999999999999,"5min":1.206,"15min":0.92900000000000005},"min":{"1min":0.32000000000000001,"5min":0.307,"15min":0.29299999999999998},"max":{"1min":16.890999999999998,"5min":16.890999999999998,"15min":16.890999999999998},"last":1.6439999999999999},{"interface":"front","average":{"1min":1.1399999999999999,"5min":1.4379999999999999,"15min":1.0589999999999999},"min":{"1min":0.378,"5min":0.32200000000000001,"15min":0.28199999999999997},"max":{"1min":8.5830000000000002,"5min":11.170999999999999,"15min":11.170999999999999},"last":0.69999999999999996}]}]},{"osd":5,"up_from":39,"seq":167503724905,"num_pgs":56,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1012012,"kb_used_data":600356,"kb_used_omap":0,"kb_used_meta":411648,"kb_avail":92733140,"statfs":{"total":95995035648,"available":94958735360,"internally_reserved":0,"allocated":614764544,"data_stored":613576644,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":421527552},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:09:00 2023","interfaces":[{"interface":"back","average":{"1min":1.639,"5min":1.972,"15min":1.8049999999999999},"min":{"1min":0.22700000000000001,"5min":0.19700000000000001,"15min":0.19700000000000001},"max":{"1min":11.859999999999999,"5min":25.222999999999999,"15min":25.222999999999999},"last":2.782},{"interface":"front","average":{"1min":1.907,"5min":1.784,"15min":1.9350000000000001},"min":{"1min":0.26500000000000001,"5min":0.23499999999999999,"15min":0.223},"max":{"1min":7.7350000000000003,"5min":25.457999999999998,"15min":25.457999999999998},"last":0.47599999999999998}]},{"osd":1,"last update":"Sat Sep 23 21:09:00 2023","interfaces":[{"interface":"back","average":{"1min":2.097,"5min":2.2469999999999999,"15min":1.9410000000000001},"min":{"1min":0.38100000000000001,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":9.1379999999999999,"5min":25.239999999999998,"15min":25.239999999999998},"last":1.425},{"interface":"front","average":{"1min":2.476,"5min":2.1179999999999999,"15min":1.7410000000000001},"min":{"1min":0.27400000000000002,"5min":0.252,"15min":0.252},"max":{"1min":9.1530000000000005,"5min":25.253,"15min":25.253},"last":1.4410000000000001}]},{"osd":2,"last update":"Sat Sep 23 21:09:15 2023","interfaces":[{"interface":"back","average":{"1min":2.0659999999999998,"5min":1.8140000000000001,"15min":1.7769999999999999},"min":{"1min":0.26200000000000001,"5min":0.25800000000000001,"15min":0.23499999999999999},"max":{"1min":11.939,"5min":12.634,"15min":12.651},"last":2.6680000000000001},{"interface":"front","average":{"1min":1.665,"5min":1.841,"15min":1.9690000000000001},"min":{"1min":0.29499999999999998,"5min":0.28100000000000003,"15min":0.26100000000000001},"max":{"1min":8.8079999999999998,"5min":8.8079999999999998,"15min":12.300000000000001},"last":1.9019999999999999}]},{"osd":3,"last update":"Sat Sep 23 21:09:38 2023","interfaces":[{"interface":"back","average":{"1min":1.8879999999999999,"5min":2.3300000000000001,"15min":2.6760000000000002},"min":{"1min":0.307,"5min":0.29299999999999998,"15min":0.28599999999999998},"max":{"1min":7.8319999999999999,"5min":25.503,"15min":61.667999999999999},"last":1.7549999999999999},{"interface":"front","average":{"1min":3.0470000000000002,"5min":2.9119999999999999,"15min":2.5739999999999998},"min":{"1min":0.30599999999999999,"5min":0.30599999999999999,"15min":0.28899999999999998},"max":{"1min":14.396000000000001,"5min":25.466000000000001,"15min":62.276000000000003},"last":2.5680000000000001}]},{"osd":4,"last update":"Sat Sep 23 21:08:50 2023","interfaces":[{"interface":"back","average":{"1min":1.6200000000000001,"5min":1.23,"15min":0.873},"min":{"1min":0.27800000000000002,"5min":0.27800000000000002,"15min":0.23400000000000001},"max":{"1min":7.7800000000000002,"5min":9.0950000000000006,"15min":9.0950000000000006},"last":0.42699999999999999},{"interface":"front","average":{"1min":2.4239999999999999,"5min":1.3919999999999999,"15min":0.95199999999999996},"min":{"1min":0.37,"5min":0.26900000000000002,"15min":0.23599999999999999},"max":{"1min":11.973000000000001,"5min":11.973000000000001,"15min":11.973000000000001},"last":0.371}]},{"osd":6,"last update":"Sat Sep 23 21:09:00 2023","interfaces":[{"interface":"back","average":{"1min":0.95699999999999996,"5min":1.2450000000000001,"15min":0.94499999999999995},"min":{"1min":0.33700000000000002,"5min":0.28799999999999998,"15min":0.27500000000000002},"max":{"1min":4.5789999999999997,"5min":8.0909999999999993,"15min":11.128},"last":0.39900000000000002},{"interface":"front","average":{"1min":1.3129999999999999,"5min":1.234,"15min":0.999},"min":{"1min":0.317,"5min":0.317,"15min":0.317},"max":{"1min":7.8550000000000004,"5min":7.8550000000000004,"15min":11.112},"last":0.59999999999999998}]},{"osd":7,"last update":"Sat Sep 23 21:09:38 2023","interfaces":[{"interface":"back","average":{"1min":0.68200000000000005,"5min":3.48,"15min":1.752},"min":{"1min":0.33100000000000002,"5min":0.33100000000000002,"15min":0.307},"max":{"1min":4.7629999999999999,"5min":189.405,"15min":189.405},"last":0.46300000000000002},{"interface":"front","average":{"1min":1.0649999999999999,"5min":3.4239999999999999,"15min":1.6859999999999999},"min":{"1min":0.35899999999999999,"5min":0.35499999999999998,"15min":0.29799999999999999},"max":{"1min":7.8079999999999998,"5min":189.685,"15min":189.685},"last":0.63}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":5434,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1259,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5412,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":646,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":4838,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":323223552,"data_stored":323223552,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":499126272,"data_stored":499122200,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":290848768,"data_stored":290848768,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":475594752,"data_stored":475594752,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":248713216,"data_stored":248709144,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":599265280,"data_stored":599261208,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":136904704,"data_stored":136904704,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":795283456,"data_stored":795279384,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T21:09:51.456 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph osd pool set .mgr pgp_num 21 2023-09-23T21:09:51.507 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:09:51.542 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:09:51.586 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:09:51.588 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:09:51.589 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:09:51.690 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:09:51.811 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:09:52.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:09:52 smithi070 ceph-mon[168282]: pgmap v1072: 109 pgs: 109 active+clean; 486 MiB data, 4.6 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 917 op/s 2023-09-23T21:09:52.881 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:09:52 smithi026 ceph-mon[175151]: pgmap v1072: 109 pgs: 109 active+clean; 486 MiB data, 4.6 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 917 op/s 2023-09-23T21:09:52.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:09:52 smithi026 ceph-mon[179688]: pgmap v1072: 109 pgs: 109 active+clean; 486 MiB data, 4.6 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 917 op/s 2023-09-23T21:09:53.678 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:09:53.682 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:09:53.682 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:09:53.783 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:09:53.870 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:09:54.486 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:09:54.486 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:09:54.486 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:09:54.587 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:09:54.631 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:09:54 smithi026 ceph-mon[175151]: pgmap v1073: 109 pgs: 109 active+clean; 379 MiB data, 4.2 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 914 op/s 2023-09-23T21:09:54.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:09:54 smithi026 ceph-mon[179688]: pgmap v1073: 109 pgs: 109 active+clean; 379 MiB data, 4.2 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 914 op/s 2023-09-23T21:09:54.684 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:09:54.687 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0: {} 2023-09-23T21:09:54.743 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1: {} 2023-09-23T21:09:54.799 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2: {} 2023-09-23T21:09:54.817 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:09:54.826 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:09:54.827 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:09:54.851 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3: {} 2023-09-23T21:09:54.853 INFO:teuthology.orchestra.run.smithi026.stdout:{ 2023-09-23T21:09:54.853 INFO:teuthology.orchestra.run.smithi026.stdout: "ops": [ 2023-09-23T21:09:54.853 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:09:54.853 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_repop(client.35856.0:224750 9.2 e411/350 9:499ed071:::benchmark_data_smithi026_342732_object111474:head v 411'14003, mlcod=411'14000)", 2023-09-23T21:09:54.853 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T21:09:54.849774+0000", 2023-09-23T21:09:54.853 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 0.0021466689999999999, 2023-09-23T21:09:54.853 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0021806899999999999, 2023-09-23T21:09:54.853 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:09:54.853 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "started", 2023-09-23T21:09:54.853 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:09:54.853 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:09:54.854 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:09:54.854 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:54.849774+0000", 2023-09-23T21:09:54.854 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:09:54.854 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:09:54.854 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:09:54.854 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:09:54.854 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:54.849774+0000", 2023-09-23T21:09:54.854 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:09:54.854 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:09:54.854 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:09:54.854 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:09:54.855 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:54.849773+0000", 2023-09-23T21:09:54.855 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T21:09:54.855 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:09:54.855 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:09:54.855 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:09:54.855 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:54.849784+0000", 2023-09-23T21:09:54.855 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1267000000000001e-05 2023-09-23T21:09:54.855 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:09:54.855 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:09:54.855 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:09:54.855 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:54.849808+0000", 2023-09-23T21:09:54.855 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.3879999999999998e-05 2023-09-23T21:09:54.856 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:09:54.856 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:09:54.856 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:09:54.856 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:54.849817+0000", 2023-09-23T21:09:54.856 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.5589999999999994e-06 2023-09-23T21:09:54.856 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:09:54.856 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:09:54.856 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:09:54.856 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:54.849917+0000", 2023-09-23T21:09:54.856 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.9684000000000003e-05 2023-09-23T21:09:54.856 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:09:54.856 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:09:54.856 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:09:54.857 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:54.849940+0000", 2023-09-23T21:09:54.857 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.2741000000000001e-05 2023-09-23T21:09:54.857 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:09:54.857 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:09:54.857 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:09:54.857 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:09:54.857 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:09:54.857 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_repop(client.35856.0:224755 9.2 e411/350)", 2023-09-23T21:09:54.857 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T21:09:54.849833+0000", 2023-09-23T21:09:54.857 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 0.0020875009999999999, 2023-09-23T21:09:54.857 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0021859050000000001, 2023-09-23T21:09:54.857 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:09:54.858 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "queued for pg", 2023-09-23T21:09:54.858 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:09:54.858 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:09:54.858 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:09:54.858 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:54.849833+0000", 2023-09-23T21:09:54.858 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:09:54.858 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:09:54.858 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:09:54.858 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:09:54.858 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:54.849833+0000", 2023-09-23T21:09:54.858 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:09:54.858 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:09:54.858 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:09:54.859 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:09:54.859 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:54.849833+0000", 2023-09-23T21:09:54.859 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T21:09:54.859 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:09:54.859 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:09:54.859 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:09:54.859 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:54.849838+0000", 2023-09-23T21:09:54.859 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.4299999999999997e-06 2023-09-23T21:09:54.859 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:09:54.859 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:09:54.859 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:09:54.859 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:54.849845+0000", 2023-09-23T21:09:54.860 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.9949999999999999e-06 2023-09-23T21:09:54.860 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:09:54.860 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:09:54.860 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:09:54.860 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:54.849851+0000", 2023-09-23T21:09:54.860 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.626e-06 2023-09-23T21:09:54.860 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:09:54.860 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:09:54.860 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:09:54.860 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:09:54.860 INFO:teuthology.orchestra.run.smithi026.stdout: ], 2023-09-23T21:09:54.860 INFO:teuthology.orchestra.run.smithi026.stdout: "num_ops": 2 2023-09-23T21:09:54.860 INFO:teuthology.orchestra.run.smithi026.stdout:} 2023-09-23T21:09:54.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:09:54 smithi070 ceph-mon[168282]: pgmap v1073: 109 pgs: 109 active+clean; 379 MiB data, 4.2 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 914 op/s 2023-09-23T21:09:54.914 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4: {} 2023-09-23T21:09:54.928 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:09:54.953 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T21:09:54.962 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:09:55.007 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6: {} 2023-09-23T21:09:55.082 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T21:09:55.099 INFO:teuthology.orchestra.run.smithi026.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T21:09:55.099 INFO:teuthology.orchestra.run.smithi026.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T21:09:55.099 INFO:teuthology.orchestra.run.smithi026.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T21:09:55.099 INFO:teuthology.orchestra.run.smithi026.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T21:09:55.099 INFO:teuthology.orchestra.run.smithi026.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T21:09:55.099 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T21:09:55.099 INFO:teuthology.orchestra.run.smithi026.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T21:09:55.099 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T21:09:55.105 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:09:55.112 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:09:55.112 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:09:55.213 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:09:55.310 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:09:55.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:09:55 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/2893089819' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:09:55.883 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:09:55 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/2893089819' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:09:55.883 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:09:55 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/2893089819' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:09:56.472 INFO:teuthology.orchestra.run.smithi026.stderr:nodeep-scrub is unset 2023-09-23T21:09:56.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:09:56 smithi070 ceph-mon[168282]: pgmap v1074: 109 pgs: 109 active+clean; 199 MiB data, 3.8 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 1.00k op/s 2023-09-23T21:09:56.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:09:56 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/2382559143' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:09:56.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:09:56 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:09:56.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:09:56 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:09:56.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:09:56 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:09:56.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:09:56 smithi070 ceph-mon[168282]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T21:09:56.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:09:56 smithi070 ceph-mon[168282]: Cluster is now healthy 2023-09-23T21:09:56.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:09:56 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T21:09:56.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:09:56 smithi070 ceph-mon[168282]: osdmap e412: 8 total, 8 up, 7 in 2023-09-23T21:09:56.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:09:56 smithi026 ceph-mon[175151]: pgmap v1074: 109 pgs: 109 active+clean; 199 MiB data, 3.8 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 1.00k op/s 2023-09-23T21:09:56.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:09:56 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/2382559143' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:09:56.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:09:56 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:09:56.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:09:56 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:09:56.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:09:56 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:09:56.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:09:56 smithi026 ceph-mon[175151]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T21:09:56.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:09:56 smithi026 ceph-mon[175151]: Cluster is now healthy 2023-09-23T21:09:56.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:09:56 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T21:09:56.883 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:09:56 smithi026 ceph-mon[175151]: osdmap e412: 8 total, 8 up, 7 in 2023-09-23T21:09:56.883 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:09:56 smithi026 ceph-mon[179688]: pgmap v1074: 109 pgs: 109 active+clean; 199 MiB data, 3.8 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 1.00k op/s 2023-09-23T21:09:56.883 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:09:56 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/2382559143' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:09:56.883 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:09:56 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:09:56.883 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:09:56 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:09:56.883 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:09:56 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:09:56.883 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:09:56 smithi026 ceph-mon[179688]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T21:09:56.883 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:09:56 smithi026 ceph-mon[179688]: Cluster is now healthy 2023-09-23T21:09:56.883 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:09:56 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T21:09:56.883 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:09:56 smithi026 ceph-mon[179688]: osdmap e412: 8 total, 8 up, 7 in 2023-09-23T21:09:57.012 INFO:tasks.radosbench.radosbench.0.smithi026.stdout:Removed 113273 objects 2023-09-23T21:09:57.013 INFO:tasks.radosbench.radosbench.0.smithi026.stdout:Clean up completed and total clean up time :150.693 2023-09-23T21:09:57.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:09:57 smithi070 ceph-mon[168282]: failed to encode map e412 with expected crc 2023-09-23T21:09:57.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:09:57 smithi070 ceph-mon[168282]: failed to encode map e412 with expected crc 2023-09-23T21:09:57.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:09:57 smithi070 ceph-mon[168282]: failed to encode map e412 with expected crc 2023-09-23T21:09:57.881 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:09:57 smithi026 ceph-mon[179688]: failed to encode map e412 with expected crc 2023-09-23T21:09:57.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:09:57 smithi026 ceph-mon[179688]: failed to encode map e412 with expected crc 2023-09-23T21:09:57.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:09:57 smithi026 ceph-mon[179688]: failed to encode map e412 with expected crc 2023-09-23T21:09:57.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:09:57 smithi026 ceph-mon[175151]: failed to encode map e412 with expected crc 2023-09-23T21:09:57.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:09:57 smithi026 ceph-mon[175151]: failed to encode map e412 with expected crc 2023-09-23T21:09:57.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:09:57 smithi026 ceph-mon[175151]: failed to encode map e412 with expected crc 2023-09-23T21:09:58.398 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:09:58.403 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:09:58.403 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:09:58.504 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:09:58.538 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:09:58.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:09:58 smithi070 ceph-mon[168282]: pgmap v1076: 109 pgs: 109 active+clean; 183 MiB data, 3.7 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 920 op/s 2023-09-23T21:09:58.881 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:09:58 smithi026 ceph-mon[175151]: pgmap v1076: 109 pgs: 109 active+clean; 183 MiB data, 3.7 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 920 op/s 2023-09-23T21:09:58.881 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:09:58 smithi026 ceph-mon[179688]: pgmap v1076: 109 pgs: 109 active+clean; 183 MiB data, 3.7 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 920 op/s 2023-09-23T21:09:59.718 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_blocked_ops 2023-09-23T21:09:59.881 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 21:09:59 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[164029]: ::ffff:172.21.15.70 - - [23/Sep/2023:21:09:59] "GET /metrics HTTP/1.1" 200 36315 "" "Prometheus/2.43.0" 2023-09-23T21:09:59.937 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:09:59.940 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:09:59.941 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:10:00.012 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:10:00.042 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:10:00.078 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:10:00.116 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:10:00.119 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:10:00.119 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:10:00.220 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:10:00.248 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:10:00.604 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:10:00.607 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:10:00.607 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:10:00.708 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:10:00.743 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:10:00.779 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:10:00.781 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:10:00.782 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:10:00.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:10:00 smithi070 ceph-mon[168282]: pgmap v1077: 109 pgs: 109 active+clean; 60 MiB data, 3.2 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 849 op/s 2023-09-23T21:10:00.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:10:00 smithi070 ceph-mon[168282]: overall HEALTH_OK 2023-09-23T21:10:00.881 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:10:00 smithi026 ceph-mon[179688]: pgmap v1077: 109 pgs: 109 active+clean; 60 MiB data, 3.2 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 849 op/s 2023-09-23T21:10:00.881 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:10:00 smithi026 ceph-mon[179688]: overall HEALTH_OK 2023-09-23T21:10:00.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:10:00 smithi026 ceph-mon[175151]: pgmap v1077: 109 pgs: 109 active+clean; 60 MiB data, 3.2 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 849 op/s 2023-09-23T21:10:00.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:10:00 smithi026 ceph-mon[175151]: overall HEALTH_OK 2023-09-23T21:10:00.883 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:10:00.912 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:10:00.945 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:10:00.948 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:10:00.948 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:10:01.049 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:10:01.076 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:10:01.110 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:10:01.112 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:10:01.113 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:10:01.213 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:10:01.240 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:10:01.272 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:10:01.275 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:10:01.275 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:10:01.376 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:10:01.401 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:10:01.433 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:10:01.435 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:10:01.436 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:10:01.537 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:10:01.562 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:10:01.594 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:10:01.596 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:10:01.596 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:10:01.697 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:10:01.720 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T21:10:01.723 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:10:01.754 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:10:01.756 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:10:01.756 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:10:01.857 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:10:01.860 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:10:01.891 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:10:01.930 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T21:10:02.107 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:10:02.112 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:10:02.113 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:10:02.213 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:10:02.243 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:10:02.375 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:10:02 smithi026 ceph-mon[175151]: pgmap v1078: 109 pgs: 109 active+clean; 3.4 MiB data, 3.1 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 771 op/s 2023-09-23T21:10:02.375 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:10:02 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:10:02.375 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:10:02 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:10:02.375 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:10:02 smithi026 ceph-mon[179688]: pgmap v1078: 109 pgs: 109 active+clean; 3.4 MiB data, 3.1 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 771 op/s 2023-09-23T21:10:02.375 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:10:02 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:10:02.375 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:10:02 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:10:02.401 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:10:02.404 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:10:02.404 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:10:02.505 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:10:02.544 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:10:02.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:10:02 smithi070 ceph-mon[168282]: pgmap v1078: 109 pgs: 109 active+clean; 3.4 MiB data, 3.1 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 771 op/s 2023-09-23T21:10:02.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:10:02 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:10:02.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:10:02 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:10:02.946 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T21:10:02.949 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:10:02.950 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:10:03.051 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:10:03.085 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:10:03.124 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:10:03.127 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:10:03.128 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:10:03.149 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:10:03.228 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:10:03.256 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:10:03.415 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:10:03.417 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:10:03.418 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:10:03.518 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:10:03.551 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:10:03.587 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:10:03.590 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:10:03.590 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:10:03.691 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:10:03.722 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:10:03.796 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:10:03.812 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T21:10:03.817 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:10:03.817 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:10:03.918 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:10:03.948 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:10:04.120 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:10:04.123 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:10:04.124 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:10:04.224 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:10:04.228 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:10:04 smithi026 ceph-mon[179688]: pgmap v1079: 109 pgs: 109 active+clean; 582 KiB data, 3.0 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 604 op/s 2023-09-23T21:10:04.229 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:10:04 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/12879148' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "21"}]: dispatch 2023-09-23T21:10:04.229 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:10:04 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "21"}]: dispatch 2023-09-23T21:10:04.243 INFO:teuthology.orchestra.run.smithi026.stderr:set pool 1 pgp_num to 21 2023-09-23T21:10:04.256 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:10:04.293 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:10:04.295 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:10:04.296 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:10:04.397 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:10:04.426 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:10:04.462 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:10:04.465 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:10:04.465 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:10:04.566 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:10:04.591 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:10:04 smithi070 ceph-mon[168282]: pgmap v1079: 109 pgs: 109 active+clean; 582 KiB data, 3.0 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 604 op/s 2023-09-23T21:10:04.591 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:10:04 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/12879148' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "21"}]: dispatch 2023-09-23T21:10:04.592 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:10:04 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "21"}]: dispatch 2023-09-23T21:10:04.594 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:10:04.618 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:10:04 smithi026 ceph-mon[175151]: pgmap v1079: 109 pgs: 109 active+clean; 582 KiB data, 3.0 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 604 op/s 2023-09-23T21:10:04.618 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:10:04 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/12879148' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "21"}]: dispatch 2023-09-23T21:10:04.618 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:10:04 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "21"}]: dispatch 2023-09-23T21:10:04.627 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:10:04.629 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:10:04.630 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:10:04.730 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:10:04.756 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:10:04.781 INFO:tasks.cephadm.ceph_manager.ceph:removing pool_name unique_pool_3 2023-09-23T21:10:04.781 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph osd pool rm unique_pool_3 unique_pool_3 --yes-i-really-really-mean-it 2023-09-23T21:10:04.788 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:10:04.790 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:10:04.791 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:10:04.891 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:10:04.916 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:10:04.947 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:10:04.950 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:10:04.950 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:10:04.954 INFO:teuthology.orchestra.run.smithi026.stdout:{ 2023-09-23T21:10:04.954 INFO:teuthology.orchestra.run.smithi026.stdout: "ops": [], 2023-09-23T21:10:04.954 INFO:teuthology.orchestra.run.smithi026.stdout: "complaint_time": 30, 2023-09-23T21:10:04.954 INFO:teuthology.orchestra.run.smithi026.stdout: "num_blocked_ops": 0 2023-09-23T21:10:04.954 INFO:teuthology.orchestra.run.smithi026.stdout:} 2023-09-23T21:10:05.050 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:10:05.079 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:10:05.417 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:10:05.422 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:10:05.422 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:10:05.523 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:10:05.549 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:10:05 smithi026 ceph-mon[175151]: failed to encode map e413 with expected crc 2023-09-23T21:10:05.549 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:10:05 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "21"}]': finished 2023-09-23T21:10:05.549 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:10:05 smithi026 ceph-mon[175151]: osdmap e413: 8 total, 8 up, 7 in 2023-09-23T21:10:05.549 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:10:05 smithi026 ceph-mon[175151]: failed to encode map e413 with expected crc 2023-09-23T21:10:05.549 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:10:05 smithi026 ceph-mon[175151]: failed to encode map e413 with expected crc 2023-09-23T21:10:05.549 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:10:05 smithi026 ceph-mon[179688]: failed to encode map e413 with expected crc 2023-09-23T21:10:05.550 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:10:05 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "21"}]': finished 2023-09-23T21:10:05.550 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:10:05 smithi026 ceph-mon[179688]: osdmap e413: 8 total, 8 up, 7 in 2023-09-23T21:10:05.550 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:10:05 smithi026 ceph-mon[179688]: failed to encode map e413 with expected crc 2023-09-23T21:10:05.550 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:10:05 smithi026 ceph-mon[179688]: failed to encode map e413 with expected crc 2023-09-23T21:10:05.552 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:10:05.613 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:10:05.616 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:10:05.617 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:10:05.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:10:05 smithi070 ceph-mon[168282]: failed to encode map e413 with expected crc 2023-09-23T21:10:05.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:10:05 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "21"}]': finished 2023-09-23T21:10:05.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:10:05 smithi070 ceph-mon[168282]: osdmap e413: 8 total, 8 up, 7 in 2023-09-23T21:10:05.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:10:05 smithi070 ceph-mon[168282]: failed to encode map e413 with expected crc 2023-09-23T21:10:05.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:10:05 smithi070 ceph-mon[168282]: failed to encode map e413 with expected crc 2023-09-23T21:10:05.718 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:10:05.746 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:10:05.779 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:10:05.781 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:10:05.782 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:10:05.882 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:10:05.909 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:10:05.942 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:10:05.945 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:10:05.945 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:10:06.046 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:10:06.061 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0: {} 2023-09-23T21:10:06.072 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:10:06.079 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1: {} 2023-09-23T21:10:06.098 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2: {} 2023-09-23T21:10:06.104 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:10:06.107 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:10:06.107 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:10:06.118 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3: {} 2023-09-23T21:10:06.140 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4: {} 2023-09-23T21:10:06.162 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T21:10:06.185 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6: {} 2023-09-23T21:10:06.203 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T21:10:06.205 INFO:teuthology.orchestra.run.smithi026.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T21:10:06.205 INFO:teuthology.orchestra.run.smithi026.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T21:10:06.205 INFO:teuthology.orchestra.run.smithi026.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T21:10:06.205 INFO:teuthology.orchestra.run.smithi026.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T21:10:06.205 INFO:teuthology.orchestra.run.smithi026.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T21:10:06.205 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T21:10:06.205 INFO:teuthology.orchestra.run.smithi026.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T21:10:06.205 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T21:10:06.208 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:10:06.237 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:10:06.323 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:10:06.326 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:10:06.327 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:10:06.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:10:06 smithi070 ceph-mon[168282]: failed to encode map e413 with expected crc 2023-09-23T21:10:06.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:10:06 smithi070 ceph-mon[168282]: pgmap v1081: 109 pgs: 109 active+clean; 582 KiB data, 3.0 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 340 op/s 2023-09-23T21:10:06.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:10:06 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/2284040328' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T21:10:06.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:10:06 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T21:10:06.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:10:06 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/3056646112' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:10:06.376 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_historic_ops 2023-09-23T21:10:06.390 INFO:teuthology.orchestra.run.smithi026.stderr:noscrub is set 2023-09-23T21:10:06.427 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:10:06.458 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:10:06.553 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T21:10:06.555 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:10:06 smithi026 ceph-mon[179688]: failed to encode map e413 with expected crc 2023-09-23T21:10:06.556 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:10:06 smithi026 ceph-mon[179688]: pgmap v1081: 109 pgs: 109 active+clean; 582 KiB data, 3.0 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 340 op/s 2023-09-23T21:10:06.556 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:10:06 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/2284040328' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T21:10:06.556 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:10:06 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T21:10:06.556 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:10:06 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/3056646112' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:10:06.556 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:10:06 smithi026 ceph-mon[175151]: failed to encode map e413 with expected crc 2023-09-23T21:10:06.556 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:10:06 smithi026 ceph-mon[175151]: pgmap v1081: 109 pgs: 109 active+clean; 582 KiB data, 3.0 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 340 op/s 2023-09-23T21:10:06.556 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:10:06 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/2284040328' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T21:10:06.556 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:10:06 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T21:10:06.557 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:10:06 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/3056646112' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:10:06.557 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:10:06.557 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:10:06.658 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:10:06.685 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:10:06.717 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:10:06.720 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:10:06.720 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:10:06.821 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:10:06.849 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:10:07.054 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:10:07.057 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:10:07.057 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:10:07.158 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:10:07.185 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:10:07.217 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:10:07.220 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:10:07.220 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:10:07.321 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:10:07.347 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:10:07.379 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:10:07.381 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:10:07.382 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:10:07.402 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:10:07.466 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:10:07 smithi026 ceph-mon[179688]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:10:07.467 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:10:07 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T21:10:07.467 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:10:07 smithi026 ceph-mon[179688]: osdmap e414: 8 total, 8 up, 7 in 2023-09-23T21:10:07.467 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:10:07 smithi026 ceph-mon[179688]: failed to encode map e414 with expected crc 2023-09-23T21:10:07.467 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:10:07 smithi026 ceph-mon[179688]: failed to encode map e414 with expected crc 2023-09-23T21:10:07.467 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:10:07 smithi026 ceph-mon[175151]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:10:07.467 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:10:07 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T21:10:07.467 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:10:07 smithi026 ceph-mon[175151]: osdmap e414: 8 total, 8 up, 7 in 2023-09-23T21:10:07.467 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:10:07 smithi026 ceph-mon[175151]: failed to encode map e414 with expected crc 2023-09-23T21:10:07.467 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:10:07 smithi026 ceph-mon[175151]: failed to encode map e414 with expected crc 2023-09-23T21:10:07.483 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:10:07.509 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:10:07.566 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:10:07.570 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:10:07.571 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:10:07.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:10:07 smithi070 ceph-mon[168282]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:10:07.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:10:07 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T21:10:07.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:10:07 smithi070 ceph-mon[168282]: osdmap e414: 8 total, 8 up, 7 in 2023-09-23T21:10:07.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:10:07 smithi070 ceph-mon[168282]: failed to encode map e414 with expected crc 2023-09-23T21:10:07.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:10:07 smithi070 ceph-mon[168282]: failed to encode map e414 with expected crc 2023-09-23T21:10:07.672 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:10:07.698 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:10:07.730 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:10:07.732 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:10:07.733 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:10:07.833 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:10:07.859 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:10:07.890 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:10:07.893 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:10:07.893 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:10:07.912 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:10:07.994 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:10:08.019 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:10:08.050 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:10:08.053 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:10:08.053 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:10:08.154 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:10:08.184 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:10:08.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:10:08 smithi070 ceph-mon[168282]: failed to encode map e414 with expected crc 2023-09-23T21:10:08.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:10:08 smithi070 ceph-mon[168282]: failed to encode map e414 with expected crc 2023-09-23T21:10:08.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:10:08 smithi070 ceph-mon[168282]: pgmap v1083: 109 pgs: 109 active+clean; 582 KiB data, 3.0 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 119 op/s 2023-09-23T21:10:08.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:10:08 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:10:08.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:10:08 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:10:08.410 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:10:08.413 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:10:08.413 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:10:08.514 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:10:08.541 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:10:08.574 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:10:08.576 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:10:08.577 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:10:08.585 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:10:08 smithi026 ceph-mon[179688]: failed to encode map e414 with expected crc 2023-09-23T21:10:08.585 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:10:08 smithi026 ceph-mon[179688]: failed to encode map e414 with expected crc 2023-09-23T21:10:08.585 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:10:08 smithi026 ceph-mon[179688]: pgmap v1083: 109 pgs: 109 active+clean; 582 KiB data, 3.0 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 119 op/s 2023-09-23T21:10:08.585 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:10:08 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:10:08.585 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:10:08 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:10:08.586 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:10:08 smithi026 ceph-mon[175151]: failed to encode map e414 with expected crc 2023-09-23T21:10:08.586 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:10:08 smithi026 ceph-mon[175151]: failed to encode map e414 with expected crc 2023-09-23T21:10:08.586 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:10:08 smithi026 ceph-mon[175151]: pgmap v1083: 109 pgs: 109 active+clean; 582 KiB data, 3.0 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 119 op/s 2023-09-23T21:10:08.586 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:10:08 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:10:08.586 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:10:08 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:10:08.677 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:10:08.703 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:10:08.736 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:10:08.738 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:10:08.739 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:10:08.839 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:10:08.865 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:10:08.896 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:10:08.898 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:10:08.898 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:10:08.999 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:10:09.017 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T21:10:09.030 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:10:09.258 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:10:09.261 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:10:09.261 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:10:09.362 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:10:09.397 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:10:09.442 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:10:09.445 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:10:09.445 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:10:09.467 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T21:10:09.510 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:10:09 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T21:10:09.510 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:10:09 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T21:10:09.510 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:10:09 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:10:09.510 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:10:09 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T21:10:09.511 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:10:09 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T21:10:09.511 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:10:09 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:10:09.546 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:10:09.573 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:10:09 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T21:10:09.573 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:10:09 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T21:10:09.573 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:10:09 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:10:09.575 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:10:09.610 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:10:09.612 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:10:09.613 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:10:09.692 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:10:09.713 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:10:09.740 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:10:09.771 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:10:09.773 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:10:09.774 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:10:09.774 INFO:teuthology.orchestra.run.smithi026.stdout:ERROR: (22) Invalid argument 2023-09-23T21:10:09.775 INFO:teuthology.orchestra.run.smithi026.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T21:10:09.782 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 6, 4, 5] out_osds: [7] dead_osds: [] live_osds: [0, 2, 3, 4, 5, 7, 6, 1] 2023-09-23T21:10:09.782 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T21:10:09.782 INFO:tasks.thrashosds.thrasher:Removing osd 6, in_osds are: [0, 1, 2, 3, 6, 4, 5] 2023-09-23T21:10:09.782 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph osd out 6 2023-09-23T21:10:09.799 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 21:10:09 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[164029]: ::ffff:172.21.15.70 - - [23/Sep/2023:21:10:09] "GET /metrics HTTP/1.1" 200 36327 "" "Prometheus/2.43.0" 2023-09-23T21:10:09.875 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:10:09.901 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:10:09.931 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:10:09.933 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:10:09.934 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:10:10.035 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:10:10.060 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:10:10.093 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:10:10.097 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:10:10.098 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:10:10.199 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:10:10.225 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:10:10.270 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:10:10.277 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:10:10.278 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:10:10.295 INFO:teuthology.orchestra.run.smithi026.stderr:pool 'unique_pool_3' removed 2023-09-23T21:10:10.304 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_ops_in_flight 2023-09-23T21:10:10.379 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:10:10.405 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:10:10.437 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:10:10.439 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:10:10.440 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:10:10.470 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:10:10 smithi026 ceph-mon[179688]: pgmap v1084: 109 pgs: 109 active+clean; 582 KiB data, 3.0 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 5 op/s 2023-09-23T21:10:10.470 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:10:10 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/1203661474' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-23T21:10:10.471 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:10:10 smithi026 ceph-mon[175151]: pgmap v1084: 109 pgs: 109 active+clean; 582 KiB data, 3.0 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 5 op/s 2023-09-23T21:10:10.471 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:10:10 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/1203661474' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-23T21:10:10.541 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:10:10.564 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:10:10 smithi070 ceph-mon[168282]: pgmap v1084: 109 pgs: 109 active+clean; 582 KiB data, 3.0 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 5 op/s 2023-09-23T21:10:10.565 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:10:10 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/1203661474' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-23T21:10:10.567 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:10:10.598 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:10:10.601 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:10:10.601 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:10:10.702 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:10:10.731 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:10:10.817 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:10:10.820 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:10:10.820 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:10:10.921 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:10:10.922 INFO:teuthology.task.full_sequential:In full_sequential, running task radosbench... 2023-09-23T21:10:10.923 INFO:tasks.radosbench:Beginning radosbench... 2023-09-23T21:10:10.923 INFO:tasks.cephadm.ceph_manager.ceph:creating pool_name unique_pool_4 2023-09-23T21:10:10.924 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph osd pool create unique_pool_4 16 2023-09-23T21:10:10.996 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:10:11.090 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T21:10:11.093 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:10:11.094 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:10:11.194 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:10:11.225 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:10:11.260 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:10:11.263 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:10:11.263 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:10:11.364 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:10:11.391 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:10:11.392 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:10:11.427 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:10:11.429 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:10:11.430 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:10:11.531 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:10:11.535 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:10:11.582 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:10:11.583 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:10:11 smithi026 ceph-mon[175151]: failed to encode map e415 with expected crc 2023-09-23T21:10:11.583 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:10:11 smithi026 ceph-mon[175151]: failed to encode map e415 with expected crc 2023-09-23T21:10:11.583 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:10:11 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/1203661474' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true}]': finished 2023-09-23T21:10:11.583 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:10:11 smithi026 ceph-mon[175151]: osdmap e415: 8 total, 8 up, 7 in 2023-09-23T21:10:11.583 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:10:11 smithi026 ceph-mon[179688]: failed to encode map e415 with expected crc 2023-09-23T21:10:11.583 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:10:11 smithi026 ceph-mon[179688]: failed to encode map e415 with expected crc 2023-09-23T21:10:11.584 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:10:11 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/1203661474' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true}]': finished 2023-09-23T21:10:11.584 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:10:11 smithi026 ceph-mon[179688]: osdmap e415: 8 total, 8 up, 7 in 2023-09-23T21:10:11.593 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:10:11.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:10:11 smithi070 ceph-mon[168282]: failed to encode map e415 with expected crc 2023-09-23T21:10:11.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:10:11 smithi070 ceph-mon[168282]: failed to encode map e415 with expected crc 2023-09-23T21:10:11.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:10:11 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/1203661474' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true}]': finished 2023-09-23T21:10:11.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:10:11 smithi070 ceph-mon[168282]: osdmap e415: 8 total, 8 up, 7 in 2023-09-23T21:10:11.678 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T21:10:11.682 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:10:11.683 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:10:11.783 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:10:11.811 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:10:11.837 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:10:11.997 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:10:12.001 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:10:12.001 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:10:12.102 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:10:12.130 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:10:12.163 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:10:12.166 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:10:12.166 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:10:12.267 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:10:12.298 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:10:12.577 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:10:12 smithi026 ceph-mon[179688]: pgmap v1086: 93 pgs: 93 active+clean; 582 KiB data, 3.0 GiB used, 623 GiB / 626 GiB avail 2023-09-23T21:10:12.578 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:10:12 smithi026 ceph-mon[175151]: pgmap v1086: 93 pgs: 93 active+clean; 582 KiB data, 3.0 GiB used, 623 GiB / 626 GiB avail 2023-09-23T21:10:12.790 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:10:12.795 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:10:12.795 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:10:12.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:10:12 smithi070 ceph-mon[168282]: pgmap v1086: 93 pgs: 93 active+clean; 582 KiB data, 3.0 GiB used, 623 GiB / 626 GiB avail 2023-09-23T21:10:12.895 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:10:12.926 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:10:13.490 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:10:13 smithi026 ceph-mon[175151]: failed to encode map e415 with expected crc 2023-09-23T21:10:13.490 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:10:13 smithi026 ceph-mon[179688]: failed to encode map e415 with expected crc 2023-09-23T21:10:13.575 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0: {} 2023-09-23T21:10:13.575 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1: {} 2023-09-23T21:10:13.575 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2: {} 2023-09-23T21:10:13.575 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3: {} 2023-09-23T21:10:13.575 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4: {} 2023-09-23T21:10:13.575 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T21:10:13.575 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6: {} 2023-09-23T21:10:13.575 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T21:10:13.576 INFO:teuthology.orchestra.run.smithi026.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T21:10:13.576 INFO:teuthology.orchestra.run.smithi026.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T21:10:13.576 INFO:teuthology.orchestra.run.smithi026.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T21:10:13.576 INFO:teuthology.orchestra.run.smithi026.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T21:10:13.576 INFO:teuthology.orchestra.run.smithi026.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T21:10:13.576 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T21:10:13.576 INFO:teuthology.orchestra.run.smithi026.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T21:10:13.576 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T21:10:13.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:10:13 smithi070 ceph-mon[168282]: failed to encode map e415 with expected crc 2023-09-23T21:10:14.101 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T21:10:14.104 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:10:14.105 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:10:14.205 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:10:14.244 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:10:14.394 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:10:14.399 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:10:14.399 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:10:14.500 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:10:14.531 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:10:14.569 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:10:14.572 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:10:14.572 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:10:14.673 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:10:14.699 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:10:14 smithi070 ceph-mon[168282]: pgmap v1087: 93 pgs: 93 active+clean; 582 KiB data, 3.0 GiB used, 623 GiB / 626 GiB avail 2023-09-23T21:10:14.699 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:10:14 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/3373672377' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:10:14.701 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:10:14.735 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:10:14.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:10:14 smithi026 ceph-mon[179688]: pgmap v1087: 93 pgs: 93 active+clean; 582 KiB data, 3.0 GiB used, 623 GiB / 626 GiB avail 2023-09-23T21:10:14.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:10:14 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/3373672377' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:10:14.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:10:14 smithi026 ceph-mon[175151]: pgmap v1087: 93 pgs: 93 active+clean; 582 KiB data, 3.0 GiB used, 623 GiB / 626 GiB avail 2023-09-23T21:10:14.737 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:10:14 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/3373672377' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:10:14.738 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:10:14.738 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:10:14.839 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:10:14.870 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:10:15.061 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:10:14 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mon-a[175127]: 2023-09-23T21:10:14.735+0000 7f1e2d487700 -1 mon.a@0(leader).osd e415 definitely_dead 0 2023-09-23T21:10:15.061 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:10:15.065 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:10:15.065 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:10:15.166 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:10:15.195 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:10:15.231 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:10:15.234 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:10:15.235 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:10:15.283 INFO:teuthology.orchestra.run.smithi026.stdout:{ 2023-09-23T21:10:15.283 INFO:teuthology.orchestra.run.smithi026.stdout: "ops": [], 2023-09-23T21:10:15.283 INFO:teuthology.orchestra.run.smithi026.stdout: "num_ops": 0 2023-09-23T21:10:15.283 INFO:teuthology.orchestra.run.smithi026.stdout:} 2023-09-23T21:10:15.336 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:10:15.368 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:10:15.404 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:10:15.407 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:10:15.408 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:10:15.508 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:10:15.536 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:10:15.569 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:10:15.571 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:10:15.572 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:10:15.673 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:10:15.694 INFO:teuthology.orchestra.run.smithi026.stderr:nodeep-scrub is set 2023-09-23T21:10:15.697 INFO:teuthology.orchestra.run.smithi026.stderr:pool 'unique_pool_4' created 2023-09-23T21:10:15.700 INFO:teuthology.orchestra.run.smithi026.stderr:marked out osd.6. 2023-09-23T21:10:15.724 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_blocked_ops 2023-09-23T21:10:15.725 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:10:15.752 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:10:15 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/2983261313' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:10:15.752 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:10:15 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:10:15.753 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:10:15 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/3110347550' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["6"]}]: dispatch 2023-09-23T21:10:15.753 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:10:15 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["6"]}]: dispatch 2023-09-23T21:10:15.753 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:10:15 smithi026 ceph-mon[175151]: Client client.admin marked osd.6 out, while it was still marked up 2023-09-23T21:10:15.753 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:10:15 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/2983261313' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:10:15.753 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:10:15 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:10:15.753 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:10:15 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/3110347550' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["6"]}]: dispatch 2023-09-23T21:10:15.753 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:10:15 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["6"]}]: dispatch 2023-09-23T21:10:15.753 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:10:15 smithi026 ceph-mon[179688]: Client client.admin marked osd.6 out, while it was still marked up 2023-09-23T21:10:15.761 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:10:15.763 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:10:15.764 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:10:15.864 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:10:15.893 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:10:16.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:10:15 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/2983261313' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:10:16.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:10:15 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:10:16.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:10:15 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/3110347550' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["6"]}]: dispatch 2023-09-23T21:10:16.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:10:15 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["6"]}]: dispatch 2023-09-23T21:10:16.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:10:15 smithi070 ceph-mon[168282]: Client client.admin marked osd.6 out, while it was still marked up 2023-09-23T21:10:16.163 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:10:16.166 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:10:16.166 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:10:16.267 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:10:16.298 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:10:16.334 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:10:16.337 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:10:16.337 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:10:16.438 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:10:16.466 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:10:16.500 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:10:16.502 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:10:16.503 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:10:16.509 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph osd pool application enable unique_pool_4 rados --yes-i-really-mean-it || true 2023-09-23T21:10:16.604 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:10:16.633 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:10:16.680 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T21:10:16.698 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:10:16.699 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:10:16.800 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:10:16.827 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:10:16.863 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:10:16.866 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:10:16.867 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:10:16.881 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:10:16 smithi026 ceph-mon[175151]: pgmap v1088: 93 pgs: 93 active+clean; 582 KiB data, 3.0 GiB used, 623 GiB / 626 GiB avail 2023-09-23T21:10:16.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:10:16 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/415070865' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_4", "pg_num": 16}]: dispatch 2023-09-23T21:10:16.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:10:16 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_4", "pg_num": 16}]: dispatch 2023-09-23T21:10:16.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:10:16 smithi026 ceph-mon[175151]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:10:16.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:10:16 smithi026 ceph-mon[175151]: failed to encode map e416 with expected crc 2023-09-23T21:10:16.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:10:16 smithi026 ceph-mon[175151]: failed to encode map e416 with expected crc 2023-09-23T21:10:16.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:10:16 smithi026 ceph-mon[175151]: failed to encode map e416 with expected crc 2023-09-23T21:10:16.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:10:16 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T21:10:16.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:10:16 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["6"]}]': finished 2023-09-23T21:10:16.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:10:16 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_4", "pg_num": 16}]': finished 2023-09-23T21:10:16.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:10:16 smithi026 ceph-mon[175151]: osdmap e416: 8 total, 8 up, 6 in 2023-09-23T21:10:16.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:10:16 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:10:16.883 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:10:16 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:10:16.883 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:10:16 smithi026 ceph-mon[179688]: pgmap v1088: 93 pgs: 93 active+clean; 582 KiB data, 3.0 GiB used, 623 GiB / 626 GiB avail 2023-09-23T21:10:16.883 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:10:16 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/415070865' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_4", "pg_num": 16}]: dispatch 2023-09-23T21:10:16.883 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:10:16 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_4", "pg_num": 16}]: dispatch 2023-09-23T21:10:16.883 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:10:16 smithi026 ceph-mon[179688]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:10:16.883 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:10:16 smithi026 ceph-mon[179688]: failed to encode map e416 with expected crc 2023-09-23T21:10:16.883 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:10:16 smithi026 ceph-mon[179688]: failed to encode map e416 with expected crc 2023-09-23T21:10:16.883 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:10:16 smithi026 ceph-mon[179688]: failed to encode map e416 with expected crc 2023-09-23T21:10:16.883 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:10:16 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T21:10:16.884 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:10:16 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["6"]}]': finished 2023-09-23T21:10:16.884 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:10:16 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_4", "pg_num": 16}]': finished 2023-09-23T21:10:16.884 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:10:16 smithi026 ceph-mon[179688]: osdmap e416: 8 total, 8 up, 6 in 2023-09-23T21:10:16.884 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:10:16 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:10:16.884 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:10:16 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:10:16.922 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T21:10:16.934 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:10:16.967 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:10:16.990 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:10:16 smithi070 ceph-mon[168282]: pgmap v1088: 93 pgs: 93 active+clean; 582 KiB data, 3.0 GiB used, 623 GiB / 626 GiB avail 2023-09-23T21:10:16.991 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:10:16 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/415070865' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_4", "pg_num": 16}]: dispatch 2023-09-23T21:10:16.991 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:10:16 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_4", "pg_num": 16}]: dispatch 2023-09-23T21:10:16.991 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:10:16 smithi070 ceph-mon[168282]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:10:16.991 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:10:16 smithi070 ceph-mon[168282]: failed to encode map e416 with expected crc 2023-09-23T21:10:16.991 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:10:16 smithi070 ceph-mon[168282]: failed to encode map e416 with expected crc 2023-09-23T21:10:16.991 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:10:16 smithi070 ceph-mon[168282]: failed to encode map e416 with expected crc 2023-09-23T21:10:16.991 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:10:16 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T21:10:16.991 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:10:16 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["6"]}]': finished 2023-09-23T21:10:16.991 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:10:16 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_4", "pg_num": 16}]': finished 2023-09-23T21:10:16.991 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:10:16 smithi070 ceph-mon[168282]: osdmap e416: 8 total, 8 up, 6 in 2023-09-23T21:10:16.991 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:10:16 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:10:16.992 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:10:16 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:10:16.993 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:10:17.024 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:10:17.026 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:10:17.027 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:10:17.127 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:10:17.153 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:10:17.185 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:10:17.187 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:10:17.188 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:10:17.288 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:10:17.314 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:10:17.346 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:10:17.349 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:10:17.349 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:10:17.353 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:10:17.450 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:10:17.478 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:10:17.695 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:10:17.698 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:10:17.698 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:10:17.770 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:10:17 smithi026 ceph-mon[175151]: failed to encode map e417 with expected crc 2023-09-23T21:10:17.770 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:10:17 smithi026 ceph-mon[175151]: osdmap e417: 8 total, 8 up, 6 in 2023-09-23T21:10:17.770 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:10:17 smithi026 ceph-mon[175151]: failed to encode map e417 with expected crc 2023-09-23T21:10:17.770 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:10:17 smithi026 ceph-mon[175151]: failed to encode map e417 with expected crc 2023-09-23T21:10:17.770 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:10:17 smithi026 ceph-mon[175151]: failed to encode map e417 with expected crc 2023-09-23T21:10:17.770 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:10:17 smithi026 ceph-mon[175151]: failed to encode map e417 with expected crc 2023-09-23T21:10:17.770 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:10:17 smithi026 ceph-mon[175151]: pgmap v1091: 109 pgs: 5 peering, 4 creating+peering, 12 unknown, 88 active+clean; 582 KiB data, 2.8 GiB used, 534 GiB / 536 GiB avail 2023-09-23T21:10:17.771 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:10:17 smithi026 ceph-mon[179688]: failed to encode map e417 with expected crc 2023-09-23T21:10:17.771 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:10:17 smithi026 ceph-mon[179688]: osdmap e417: 8 total, 8 up, 6 in 2023-09-23T21:10:17.771 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:10:17 smithi026 ceph-mon[179688]: failed to encode map e417 with expected crc 2023-09-23T21:10:17.771 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:10:17 smithi026 ceph-mon[179688]: failed to encode map e417 with expected crc 2023-09-23T21:10:17.771 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:10:17 smithi026 ceph-mon[179688]: failed to encode map e417 with expected crc 2023-09-23T21:10:17.771 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:10:17 smithi026 ceph-mon[179688]: failed to encode map e417 with expected crc 2023-09-23T21:10:17.771 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:10:17 smithi026 ceph-mon[179688]: pgmap v1091: 109 pgs: 5 peering, 4 creating+peering, 12 unknown, 88 active+clean; 582 KiB data, 2.8 GiB used, 534 GiB / 536 GiB avail 2023-09-23T21:10:17.800 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:10:17.806 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:10:17.848 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:10:17.915 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:10:17.918 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:10:17.918 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:10:18.019 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:10:18.043 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:10:17 smithi070 ceph-mon[168282]: failed to encode map e417 with expected crc 2023-09-23T21:10:18.043 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:10:17 smithi070 ceph-mon[168282]: osdmap e417: 8 total, 8 up, 6 in 2023-09-23T21:10:18.043 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:10:17 smithi070 ceph-mon[168282]: failed to encode map e417 with expected crc 2023-09-23T21:10:18.043 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:10:17 smithi070 ceph-mon[168282]: failed to encode map e417 with expected crc 2023-09-23T21:10:18.043 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:10:17 smithi070 ceph-mon[168282]: failed to encode map e417 with expected crc 2023-09-23T21:10:18.043 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:10:17 smithi070 ceph-mon[168282]: failed to encode map e417 with expected crc 2023-09-23T21:10:18.043 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:10:17 smithi070 ceph-mon[168282]: pgmap v1091: 109 pgs: 5 peering, 4 creating+peering, 12 unknown, 88 active+clean; 582 KiB data, 2.8 GiB used, 534 GiB / 536 GiB avail 2023-09-23T21:10:18.045 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:10:18.077 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:10:18.079 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:10:18.080 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:10:18.181 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:10:18.214 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:10:18.285 INFO:teuthology.orchestra.run.smithi026.stdout:{ 2023-09-23T21:10:18.285 INFO:teuthology.orchestra.run.smithi026.stdout: "ops": [], 2023-09-23T21:10:18.285 INFO:teuthology.orchestra.run.smithi026.stdout: "complaint_time": 30, 2023-09-23T21:10:18.285 INFO:teuthology.orchestra.run.smithi026.stdout: "num_blocked_ops": 0 2023-09-23T21:10:18.285 INFO:teuthology.orchestra.run.smithi026.stdout:} 2023-09-23T21:10:18.450 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T21:10:18.453 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:10:18.454 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:10:18.554 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:10:18.582 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:10:18.615 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:10:18.617 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:10:18.618 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:10:18.693 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:10:18 smithi026 ceph-mon[179688]: Health check failed: Reduced data availability: 2 pgs inactive, 2 pgs peering (PG_AVAILABILITY) 2023-09-23T21:10:18.693 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:10:18 smithi026 ceph-mon[179688]: failed to encode map e418 with expected crc 2023-09-23T21:10:18.693 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:10:18 smithi026 ceph-mon[179688]: failed to encode map e418 with expected crc 2023-09-23T21:10:18.693 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:10:18 smithi026 ceph-mon[179688]: osdmap e418: 8 total, 8 up, 6 in 2023-09-23T21:10:18.693 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:10:18 smithi026 ceph-mon[179688]: failed to encode map e418 with expected crc 2023-09-23T21:10:18.694 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:10:18 smithi026 ceph-mon[179688]: failed to encode map e418 with expected crc 2023-09-23T21:10:18.694 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:10:18 smithi026 ceph-mon[179688]: failed to encode map e418 with expected crc 2023-09-23T21:10:18.694 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:10:18 smithi026 ceph-mon[179688]: failed to encode map e418 with expected crc 2023-09-23T21:10:18.694 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:10:18 smithi026 ceph-mon[175151]: Health check failed: Reduced data availability: 2 pgs inactive, 2 pgs peering (PG_AVAILABILITY) 2023-09-23T21:10:18.694 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:10:18 smithi026 ceph-mon[175151]: failed to encode map e418 with expected crc 2023-09-23T21:10:18.694 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:10:18 smithi026 ceph-mon[175151]: failed to encode map e418 with expected crc 2023-09-23T21:10:18.694 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:10:18 smithi026 ceph-mon[175151]: osdmap e418: 8 total, 8 up, 6 in 2023-09-23T21:10:18.694 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:10:18 smithi026 ceph-mon[175151]: failed to encode map e418 with expected crc 2023-09-23T21:10:18.694 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:10:18 smithi026 ceph-mon[175151]: failed to encode map e418 with expected crc 2023-09-23T21:10:18.694 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:10:18 smithi026 ceph-mon[175151]: failed to encode map e418 with expected crc 2023-09-23T21:10:18.695 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:10:18 smithi026 ceph-mon[175151]: failed to encode map e418 with expected crc 2023-09-23T21:10:18.719 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:10:18.744 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T21:10:18.749 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:10:18.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:10:18 smithi070 ceph-mon[168282]: Health check failed: Reduced data availability: 2 pgs inactive, 2 pgs peering (PG_AVAILABILITY) 2023-09-23T21:10:18.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:10:18 smithi070 ceph-mon[168282]: failed to encode map e418 with expected crc 2023-09-23T21:10:18.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:10:18 smithi070 ceph-mon[168282]: failed to encode map e418 with expected crc 2023-09-23T21:10:18.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:10:18 smithi070 ceph-mon[168282]: osdmap e418: 8 total, 8 up, 6 in 2023-09-23T21:10:18.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:10:18 smithi070 ceph-mon[168282]: failed to encode map e418 with expected crc 2023-09-23T21:10:18.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:10:18 smithi070 ceph-mon[168282]: failed to encode map e418 with expected crc 2023-09-23T21:10:18.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:10:18 smithi070 ceph-mon[168282]: failed to encode map e418 with expected crc 2023-09-23T21:10:18.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:10:18 smithi070 ceph-mon[168282]: failed to encode map e418 with expected crc 2023-09-23T21:10:19.248 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:10:19.251 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:10:19.251 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:10:19.352 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:10:19.368 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_historic_ops 2023-09-23T21:10:19.380 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:10:19.414 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:10:19.417 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:10:19.417 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:10:19.518 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:10:19.544 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:10:19.577 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:10:19.579 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:10:19.579 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:10:19.680 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:10:19.697 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 21:10:19 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[164029]: ::ffff:172.21.15.70 - - [23/Sep/2023:21:10:19] "GET /metrics HTTP/1.1" 200 36313 "" "Prometheus/2.43.0" 2023-09-23T21:10:19.719 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:10:19.806 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:10:19.809 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:10:19.809 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:10:19.910 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:10:19.936 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:10:19.968 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:10:19.970 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:10:19 smithi070 ceph-mon[168282]: failed to encode map e419 with expected crc 2023-09-23T21:10:19.970 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:10:19 smithi070 ceph-mon[168282]: osdmap e419: 8 total, 8 up, 6 in 2023-09-23T21:10:19.970 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:10:19 smithi070 ceph-mon[168282]: failed to encode map e419 with expected crc 2023-09-23T21:10:19.970 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:10:19 smithi070 ceph-mon[168282]: failed to encode map e419 with expected crc 2023-09-23T21:10:19.970 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:10:19 smithi070 ceph-mon[168282]: failed to encode map e419 with expected crc 2023-09-23T21:10:19.970 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:10:19 smithi070 ceph-mon[168282]: failed to encode map e419 with expected crc 2023-09-23T21:10:19.970 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:10:19 smithi070 ceph-mon[168282]: failed to encode map e418 with expected crc 2023-09-23T21:10:19.971 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:10:19 smithi070 ceph-mon[168282]: pgmap v1094: 109 pgs: 1 active+undersized+remapped, 4 remapped+peering, 5 peering, 4 creating+peering, 2 unknown, 93 active+clean; 608 KiB data, 2.8 GiB used, 534 GiB / 536 GiB avail; 10 KiB/s rd, 27 KiB/s wr, 8 op/s; 20 KiB/s, 0 objects/s recovering 2023-09-23T21:10:19.971 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:10:19 smithi070 ceph-mon[168282]: osdmap e420: 8 total, 8 up, 6 in 2023-09-23T21:10:19.971 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:10:19.972 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:10:20.072 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:10:20.089 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:10:20.099 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:10:19 smithi026 ceph-mon[179688]: failed to encode map e419 with expected crc 2023-09-23T21:10:20.100 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:10:19 smithi026 ceph-mon[179688]: osdmap e419: 8 total, 8 up, 6 in 2023-09-23T21:10:20.100 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:10:19 smithi026 ceph-mon[179688]: failed to encode map e419 with expected crc 2023-09-23T21:10:20.100 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:10:19 smithi026 ceph-mon[179688]: failed to encode map e419 with expected crc 2023-09-23T21:10:20.100 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:10:19 smithi026 ceph-mon[179688]: failed to encode map e419 with expected crc 2023-09-23T21:10:20.100 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:10:19 smithi026 ceph-mon[179688]: failed to encode map e419 with expected crc 2023-09-23T21:10:20.100 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:10:19 smithi026 ceph-mon[179688]: failed to encode map e418 with expected crc 2023-09-23T21:10:20.100 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:10:19 smithi026 ceph-mon[179688]: pgmap v1094: 109 pgs: 1 active+undersized+remapped, 4 remapped+peering, 5 peering, 4 creating+peering, 2 unknown, 93 active+clean; 608 KiB data, 2.8 GiB used, 534 GiB / 536 GiB avail; 10 KiB/s rd, 27 KiB/s wr, 8 op/s; 20 KiB/s, 0 objects/s recovering 2023-09-23T21:10:20.100 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:10:19 smithi026 ceph-mon[179688]: osdmap e420: 8 total, 8 up, 6 in 2023-09-23T21:10:20.100 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:10:19 smithi026 ceph-mon[175151]: failed to encode map e419 with expected crc 2023-09-23T21:10:20.100 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:10:19 smithi026 ceph-mon[175151]: osdmap e419: 8 total, 8 up, 6 in 2023-09-23T21:10:20.101 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:10:19 smithi026 ceph-mon[175151]: failed to encode map e419 with expected crc 2023-09-23T21:10:20.101 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:10:19 smithi026 ceph-mon[175151]: failed to encode map e419 with expected crc 2023-09-23T21:10:20.101 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:10:19 smithi026 ceph-mon[175151]: failed to encode map e419 with expected crc 2023-09-23T21:10:20.101 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:10:19 smithi026 ceph-mon[175151]: failed to encode map e419 with expected crc 2023-09-23T21:10:20.101 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:10:19 smithi026 ceph-mon[175151]: failed to encode map e418 with expected crc 2023-09-23T21:10:20.101 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:10:19 smithi026 ceph-mon[175151]: pgmap v1094: 109 pgs: 1 active+undersized+remapped, 4 remapped+peering, 5 peering, 4 creating+peering, 2 unknown, 93 active+clean; 608 KiB data, 2.8 GiB used, 534 GiB / 536 GiB avail; 10 KiB/s rd, 27 KiB/s wr, 8 op/s; 20 KiB/s, 0 objects/s recovering 2023-09-23T21:10:20.101 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:10:19 smithi026 ceph-mon[175151]: osdmap e420: 8 total, 8 up, 6 in 2023-09-23T21:10:20.103 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:10:20.147 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:10:20.149 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0: {} 2023-09-23T21:10:20.151 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:10:20.151 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:10:20.168 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1: {} 2023-09-23T21:10:20.188 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2: {} 2023-09-23T21:10:20.207 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3: {} 2023-09-23T21:10:20.226 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4: {} 2023-09-23T21:10:20.245 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T21:10:20.252 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:10:20.269 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6: {} 2023-09-23T21:10:20.285 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:10:20.292 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T21:10:20.293 INFO:teuthology.orchestra.run.smithi026.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T21:10:20.293 INFO:teuthology.orchestra.run.smithi026.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T21:10:20.293 INFO:teuthology.orchestra.run.smithi026.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T21:10:20.294 INFO:teuthology.orchestra.run.smithi026.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T21:10:20.294 INFO:teuthology.orchestra.run.smithi026.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T21:10:20.294 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T21:10:20.294 INFO:teuthology.orchestra.run.smithi026.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T21:10:20.294 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T21:10:20.619 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:10:20.706 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:10:20 smithi026 ceph-mon[175151]: failed to encode map e420 with expected crc 2023-09-23T21:10:20.707 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:10:20 smithi026 ceph-mon[175151]: failed to encode map e420 with expected crc 2023-09-23T21:10:20.707 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:10:20 smithi026 ceph-mon[175151]: failed to encode map e420 with expected crc 2023-09-23T21:10:20.707 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:10:20 smithi026 ceph-mon[175151]: failed to encode map e420 with expected crc 2023-09-23T21:10:20.707 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:10:20 smithi026 ceph-mon[175151]: failed to encode map e420 with expected crc 2023-09-23T21:10:20.707 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:10:20 smithi026 ceph-mon[175151]: failed to encode map e420 with expected crc 2023-09-23T21:10:20.707 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:10:20 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/1595401865' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:10:20.707 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:10:20 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/3309809505' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_4", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T21:10:20.707 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:10:20 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_4", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T21:10:20.708 INFO:teuthology.orchestra.run.smithi026.stderr:enabled application 'rados' on pool 'unique_pool_4' 2023-09-23T21:10:21.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:10:20 smithi070 ceph-mon[168282]: failed to encode map e420 with expected crc 2023-09-23T21:10:21.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:10:20 smithi070 ceph-mon[168282]: failed to encode map e420 with expected crc 2023-09-23T21:10:21.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:10:20 smithi070 ceph-mon[168282]: failed to encode map e420 with expected crc 2023-09-23T21:10:21.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:10:20 smithi070 ceph-mon[168282]: failed to encode map e420 with expected crc 2023-09-23T21:10:21.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:10:20 smithi070 ceph-mon[168282]: failed to encode map e420 with expected crc 2023-09-23T21:10:21.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:10:20 smithi070 ceph-mon[168282]: failed to encode map e420 with expected crc 2023-09-23T21:10:21.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:10:20 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/1595401865' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:10:21.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:10:20 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/3309809505' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_4", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T21:10:21.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:10:20 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_4", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T21:10:21.127 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:10:21.129 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:10:20 smithi026 ceph-mon[179688]: failed to encode map e420 with expected crc 2023-09-23T21:10:21.129 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:10:20 smithi026 ceph-mon[179688]: failed to encode map e420 with expected crc 2023-09-23T21:10:21.129 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:10:20 smithi026 ceph-mon[179688]: failed to encode map e420 with expected crc 2023-09-23T21:10:21.129 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:10:20 smithi026 ceph-mon[179688]: failed to encode map e420 with expected crc 2023-09-23T21:10:21.129 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:10:20 smithi026 ceph-mon[179688]: failed to encode map e420 with expected crc 2023-09-23T21:10:21.130 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:10:20 smithi026 ceph-mon[179688]: failed to encode map e420 with expected crc 2023-09-23T21:10:21.130 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:10:20 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/1595401865' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:10:21.130 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:10:20 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/3309809505' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_4", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T21:10:21.130 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:10:20 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_4", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T21:10:21.130 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:10:21.131 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:10:21.231 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:10:21.250 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5] out_osds: [7, 6] dead_osds: [] live_osds: [0, 2, 3, 4, 5, 7, 6, 1] 2023-09-23T21:10:21.251 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T21:10:21.251 INFO:tasks.thrashosds.thrasher:Adding osd 7 2023-09-23T21:10:21.251 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph osd in 7 2023-09-23T21:10:21.258 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:10:21.291 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:10:21.293 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:10:21.294 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:10:21.395 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:10:21.421 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:10:21.452 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:10:21.455 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:10:21.455 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:10:21.556 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:10:21.581 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:10:21.612 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:10:21.615 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:10:21.615 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:10:21.716 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:10:21.746 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:10:21.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:10:21 smithi070 ceph-mon[168282]: failed to encode map e421 with expected crc 2023-09-23T21:10:21.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:10:21 smithi070 ceph-mon[168282]: failed to encode map e421 with expected crc 2023-09-23T21:10:21.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:10:21 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_4", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-09-23T21:10:21.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:10:21 smithi070 ceph-mon[168282]: osdmap e421: 8 total, 8 up, 6 in 2023-09-23T21:10:21.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:10:21 smithi070 ceph-mon[168282]: pgmap v1097: 109 pgs: 1 active+undersized+remapped, 4 remapped+peering, 8 peering, 4 creating+peering, 92 active+clean; 608 KiB data, 2.8 GiB used, 534 GiB / 536 GiB avail; 10 KiB/s rd, 27 KiB/s wr, 8 op/s; 100 KiB/s, 0 objects/s recovering 2023-09-23T21:10:21.929 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:10:21 smithi026 ceph-mon[179688]: failed to encode map e421 with expected crc 2023-09-23T21:10:21.929 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:10:21 smithi026 ceph-mon[179688]: failed to encode map e421 with expected crc 2023-09-23T21:10:21.929 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:10:21 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_4", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-09-23T21:10:21.929 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:10:21 smithi026 ceph-mon[179688]: osdmap e421: 8 total, 8 up, 6 in 2023-09-23T21:10:21.929 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:10:21 smithi026 ceph-mon[179688]: pgmap v1097: 109 pgs: 1 active+undersized+remapped, 4 remapped+peering, 8 peering, 4 creating+peering, 92 active+clean; 608 KiB data, 2.8 GiB used, 534 GiB / 536 GiB avail; 10 KiB/s rd, 27 KiB/s wr, 8 op/s; 100 KiB/s, 0 objects/s recovering 2023-09-23T21:10:21.930 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:10:21 smithi026 ceph-mon[175151]: failed to encode map e421 with expected crc 2023-09-23T21:10:21.930 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:10:21 smithi026 ceph-mon[175151]: failed to encode map e421 with expected crc 2023-09-23T21:10:21.930 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:10:21 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_4", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-09-23T21:10:21.930 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:10:21 smithi026 ceph-mon[175151]: osdmap e421: 8 total, 8 up, 6 in 2023-09-23T21:10:21.930 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:10:21 smithi026 ceph-mon[175151]: pgmap v1097: 109 pgs: 1 active+undersized+remapped, 4 remapped+peering, 8 peering, 4 creating+peering, 92 active+clean; 608 KiB data, 2.8 GiB used, 534 GiB / 536 GiB avail; 10 KiB/s rd, 27 KiB/s wr, 8 op/s; 100 KiB/s, 0 objects/s recovering 2023-09-23T21:10:21.934 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:10:21.936 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:10:21.937 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:10:22.037 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:10:22.065 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:10:22.096 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:10:22.099 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:10:22.099 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:10:22.200 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:10:22.235 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:10:22.302 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T21:10:22.305 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:10:22.306 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:10:22.306 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:10:22.369 DEBUG:teuthology.orchestra.run.smithi026:> /bin/sh -c 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados --no-log-to-stderr --name client.0 -b 65536 --object-size 65536 -p unique_pool_4 bench 90 write' 2023-09-23T21:10:22.372 INFO:tasks.radosbench:joining radosbench (timing out after 3000s) 2023-09-23T21:10:22.372 INFO:teuthology.orchestra.run:waiting for 3000 2023-09-23T21:10:22.407 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:10:22.425 INFO:tasks.radosbench.radosbench.0.smithi026.stdout:hints = 1 2023-09-23T21:10:22.425 INFO:tasks.radosbench.radosbench.0.smithi026.stdout:Maintaining 16 concurrent writes of 65536 bytes to objects of size 65536 for up to 90 seconds or 0 objects 2023-09-23T21:10:22.426 INFO:tasks.radosbench.radosbench.0.smithi026.stdout:Object prefix: benchmark_data_smithi026_379838 2023-09-23T21:10:22.426 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T21:10:22.426 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 0 0 0 0 0 0 - 0 2023-09-23T21:10:22.436 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:10:22.473 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:10:22.476 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:10:22.477 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:10:22.577 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:10:22.606 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:10:22.642 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:10:22.644 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:10:22.645 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:10:22.738 INFO:teuthology.orchestra.run.smithi026.stderr:noscrub is unset 2023-09-23T21:10:22.745 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:10:22.746 INFO:teuthology.orchestra.run.smithi026.stdout:ERROR: (22) Invalid argument 2023-09-23T21:10:22.746 INFO:teuthology.orchestra.run.smithi026.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T21:10:22.762 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T21:10:22.773 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:10:22.804 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:10:22.807 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:10:22.807 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:10:22.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:10:22 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/2529499389' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T21:10:22.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:10:22 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/2529499389' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T21:10:22.908 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:10:22.994 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:10:23.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:10:22 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/2529499389' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T21:10:23.144 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T21:10:23.150 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:10:23.151 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:10:23.251 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:10:23.280 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:10:23.316 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:10:23.318 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:10:23.319 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:10:23.420 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:10:23.427 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 1 16 2259 2243 140.126 140.188 0.00746568 0.00710189 2023-09-23T21:10:23.479 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:10:23.657 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:10:23.662 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:10:23.663 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:10:23.724 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:10:23 smithi026 ceph-mon[175151]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:10:23.724 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:10:23 smithi026 ceph-mon[175151]: failed to encode map e422 with expected crc 2023-09-23T21:10:23.724 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:10:23 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/2529499389' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T21:10:23.724 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:10:23 smithi026 ceph-mon[175151]: osdmap e422: 8 total, 8 up, 6 in 2023-09-23T21:10:23.724 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:10:23 smithi026 ceph-mon[175151]: failed to encode map e422 with expected crc 2023-09-23T21:10:23.724 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:10:23 smithi026 ceph-mon[175151]: failed to encode map e422 with expected crc 2023-09-23T21:10:23.724 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:10:23 smithi026 ceph-mon[175151]: pgmap v1099: 109 pgs: 1 active+undersized+remapped, 3 peering, 105 active+clean; 608 KiB data, 2.8 GiB used, 534 GiB / 536 GiB avail; 76 KiB/s, 2 keys/s, 1 objects/s recovering 2023-09-23T21:10:23.724 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:10:23 smithi026 ceph-mon[179688]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:10:23.725 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:10:23 smithi026 ceph-mon[179688]: failed to encode map e422 with expected crc 2023-09-23T21:10:23.725 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:10:23 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/2529499389' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T21:10:23.725 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:10:23 smithi026 ceph-mon[179688]: osdmap e422: 8 total, 8 up, 6 in 2023-09-23T21:10:23.725 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:10:23 smithi026 ceph-mon[179688]: failed to encode map e422 with expected crc 2023-09-23T21:10:23.725 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:10:23 smithi026 ceph-mon[179688]: failed to encode map e422 with expected crc 2023-09-23T21:10:23.725 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:10:23 smithi026 ceph-mon[179688]: pgmap v1099: 109 pgs: 1 active+undersized+remapped, 3 peering, 105 active+clean; 608 KiB data, 2.8 GiB used, 534 GiB / 536 GiB avail; 76 KiB/s, 2 keys/s, 1 objects/s recovering 2023-09-23T21:10:23.763 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:10:23.779 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_ops_in_flight 2023-09-23T21:10:23.792 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:10:23.831 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:10:23.834 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:10:23.834 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:10:23.935 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:10:23.963 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:10:24.002 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:10:24.004 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:10:23 smithi070 ceph-mon[168282]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:10:24.005 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:10:23 smithi070 ceph-mon[168282]: failed to encode map e422 with expected crc 2023-09-23T21:10:24.005 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:10:23 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/2529499389' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T21:10:24.005 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:10:23 smithi070 ceph-mon[168282]: osdmap e422: 8 total, 8 up, 6 in 2023-09-23T21:10:24.005 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:10:23 smithi070 ceph-mon[168282]: failed to encode map e422 with expected crc 2023-09-23T21:10:24.005 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:10:23 smithi070 ceph-mon[168282]: failed to encode map e422 with expected crc 2023-09-23T21:10:24.005 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:10:23 smithi070 ceph-mon[168282]: pgmap v1099: 109 pgs: 1 active+undersized+remapped, 3 peering, 105 active+clean; 608 KiB data, 2.8 GiB used, 534 GiB / 536 GiB avail; 76 KiB/s, 2 keys/s, 1 objects/s recovering 2023-09-23T21:10:24.006 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:10:24.006 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:10:24.107 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:10:24.151 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:10:24.180 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:10:24.339 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:10:24.347 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:10:24.348 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:10:24.427 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 2 16 4625 4609 143.966 147.875 0.00571374 0.00693115 2023-09-23T21:10:24.433 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T21:10:24.449 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:10:24.483 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:10:24.564 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:10:24.566 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:10:24.567 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:10:24.668 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:10:24.754 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:10:24.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:10:24 smithi070 ceph-mon[168282]: failed to encode map e422 with expected crc 2023-09-23T21:10:24.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:10:24 smithi070 ceph-mon[168282]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 3 pgs inactive, 4 pgs peering) 2023-09-23T21:10:24.881 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:10:24.881 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:10:24 smithi026 ceph-mon[175151]: failed to encode map e422 with expected crc 2023-09-23T21:10:24.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:10:24 smithi026 ceph-mon[175151]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 3 pgs inactive, 4 pgs peering) 2023-09-23T21:10:24.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:10:24 smithi026 ceph-mon[179688]: failed to encode map e422 with expected crc 2023-09-23T21:10:24.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:10:24 smithi026 ceph-mon[179688]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 3 pgs inactive, 4 pgs peering) 2023-09-23T21:10:24.884 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:10:24.885 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:10:24.985 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:10:25.015 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:10:25.055 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:10:25.057 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:10:25.058 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:10:25.159 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:10:25.209 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:10:25 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mon-a[175127]: 2023-09-23T21:10:25.098+0000 7f1e2d487700 -1 mon.a@0(leader).osd e422 definitely_dead 0 2023-09-23T21:10:25.215 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:10:25.281 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:10:25.285 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:10:25.286 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:10:25.387 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:10:25.427 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 3 16 6705 6689 139.276 130 0.00515137 0.00716674 2023-09-23T21:10:25.459 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:10:25.524 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:10:25.529 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:10:25.529 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:10:25.630 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:10:25.663 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:10:25.703 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:10:25.705 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:10:25.706 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:10:25.770 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T21:10:25.786 INFO:teuthology.orchestra.run.smithi026.stderr:marked in osd.7. 2023-09-23T21:10:25.807 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:10:25.822 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:10:25 smithi026 ceph-mon[175151]: 1.6 scrub starts 2023-09-23T21:10:25.823 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:10:25 smithi026 ceph-mon[175151]: 1.6 scrub ok 2023-09-23T21:10:25.823 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:10:25 smithi026 ceph-mon[175151]: pgmap v1100: 109 pgs: 109 active+clean; 66 MiB data, 2.8 GiB used, 534 GiB / 536 GiB avail; 11 MiB/s wr, 174 op/s; 54 KiB/s, 1 keys/s, 1 objects/s recovering 2023-09-23T21:10:25.823 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:10:25 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/569825267' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["7"]}]: dispatch 2023-09-23T21:10:25.823 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:10:25 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["7"]}]: dispatch 2023-09-23T21:10:25.823 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:10:25 smithi026 ceph-mon[179688]: 1.6 scrub starts 2023-09-23T21:10:25.823 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:10:25 smithi026 ceph-mon[179688]: 1.6 scrub ok 2023-09-23T21:10:25.823 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:10:25 smithi026 ceph-mon[179688]: pgmap v1100: 109 pgs: 109 active+clean; 66 MiB data, 2.8 GiB used, 534 GiB / 536 GiB avail; 11 MiB/s wr, 174 op/s; 54 KiB/s, 1 keys/s, 1 objects/s recovering 2023-09-23T21:10:25.823 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:10:25 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/569825267' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["7"]}]: dispatch 2023-09-23T21:10:25.823 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:10:25 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["7"]}]: dispatch 2023-09-23T21:10:25.826 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:10:25 smithi070 ceph-mon[168282]: 1.6 scrub starts 2023-09-23T21:10:25.826 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:10:25 smithi070 ceph-mon[168282]: 1.6 scrub ok 2023-09-23T21:10:25.826 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:10:25 smithi070 ceph-mon[168282]: pgmap v1100: 109 pgs: 109 active+clean; 66 MiB data, 2.8 GiB used, 534 GiB / 536 GiB avail; 11 MiB/s wr, 174 op/s; 54 KiB/s, 1 keys/s, 1 objects/s recovering 2023-09-23T21:10:25.826 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:10:25 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/569825267' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["7"]}]: dispatch 2023-09-23T21:10:25.826 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:10:25 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["7"]}]: dispatch 2023-09-23T21:10:25.849 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:10:25.943 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:10:25.945 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0: {} 2023-09-23T21:10:25.947 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:10:25.948 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:10:25.965 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1: {} 2023-09-23T21:10:25.985 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2: {} 2023-09-23T21:10:25.988 INFO:teuthology.orchestra.run.smithi070.stdout:ERROR: (22) Invalid argument 2023-09-23T21:10:25.988 INFO:teuthology.orchestra.run.smithi070.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T21:10:26.016 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3: {} 2023-09-23T21:10:26.035 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4: {} 2023-09-23T21:10:26.049 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:10:26.063 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T21:10:26.078 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:10:26.085 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6: {} 2023-09-23T21:10:26.107 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T21:10:26.109 INFO:teuthology.orchestra.run.smithi026.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T21:10:26.110 INFO:teuthology.orchestra.run.smithi026.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T21:10:26.110 INFO:teuthology.orchestra.run.smithi026.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T21:10:26.110 INFO:teuthology.orchestra.run.smithi026.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T21:10:26.110 INFO:teuthology.orchestra.run.smithi026.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T21:10:26.110 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T21:10:26.110 INFO:teuthology.orchestra.run.smithi026.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T21:10:26.110 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T21:10:26.222 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:10:26.225 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:10:26.225 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:10:26.305 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_blocked_ops 2023-09-23T21:10:26.312 INFO:tasks.thrashosds.thrasher:Added osd 7 2023-09-23T21:10:26.325 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:10:26.353 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:10:26.427 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 4 16 7415 7399 115.558 44.375 0.00364871 0.00725819 2023-09-23T21:10:26.558 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:10:26.561 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:10:26.562 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:10:26.662 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:10:26.690 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:10:26.699 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:10:26.726 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:10:26.729 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:10:26.729 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:10:26.830 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:10:26.857 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:10:26.904 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T21:10:26.957 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:10:26 smithi026 ceph-mon[179688]: failed to encode map e423 with expected crc 2023-09-23T21:10:26.957 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:10:26 smithi026 ceph-mon[179688]: failed to encode map e423 with expected crc 2023-09-23T21:10:26.957 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:10:26 smithi026 ceph-mon[179688]: failed to encode map e423 with expected crc 2023-09-23T21:10:26.957 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:10:26 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["7"]}]': finished 2023-09-23T21:10:26.957 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:10:26 smithi026 ceph-mon[179688]: osdmap e423: 8 total, 8 up, 7 in 2023-09-23T21:10:26.957 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:10:26 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/994092500' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:10:26.957 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:10:26 smithi026 ceph-mon[175151]: failed to encode map e423 with expected crc 2023-09-23T21:10:26.958 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:10:26 smithi026 ceph-mon[175151]: failed to encode map e423 with expected crc 2023-09-23T21:10:26.958 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:10:26 smithi026 ceph-mon[175151]: failed to encode map e423 with expected crc 2023-09-23T21:10:26.958 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:10:26 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["7"]}]': finished 2023-09-23T21:10:26.958 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:10:26 smithi026 ceph-mon[175151]: osdmap e423: 8 total, 8 up, 7 in 2023-09-23T21:10:26.958 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:10:26 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/994092500' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:10:26.994 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:10:26.998 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:10:26.998 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:10:27.099 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:10:27.106 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:10:26 smithi070 ceph-mon[168282]: failed to encode map e423 with expected crc 2023-09-23T21:10:27.106 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:10:26 smithi070 ceph-mon[168282]: failed to encode map e423 with expected crc 2023-09-23T21:10:27.106 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:10:26 smithi070 ceph-mon[168282]: failed to encode map e423 with expected crc 2023-09-23T21:10:27.107 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:10:26 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["7"]}]': finished 2023-09-23T21:10:27.107 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:10:26 smithi070 ceph-mon[168282]: osdmap e423: 8 total, 8 up, 7 in 2023-09-23T21:10:27.107 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:10:26 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/994092500' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:10:27.126 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:10:27.162 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:10:27.165 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:10:27.165 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:10:27.266 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:10:27.294 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:10:27.377 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:10:27.379 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:10:27.380 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:10:27.428 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 5 16 7415 7399 92.4531 0 - 0.00725819 2023-09-23T21:10:27.480 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:10:27.510 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:10:27.697 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:10:27.700 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:10:27.700 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:10:27.801 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:10:27.831 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:10:27.917 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:10:27.921 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:10:27.921 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:10:28.017 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:10:27 smithi070 ceph-mon[168282]: failed to encode map e424 with expected crc 2023-09-23T21:10:28.017 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:10:27 smithi070 ceph-mon[168282]: failed to encode map e424 with expected crc 2023-09-23T21:10:28.017 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:10:27 smithi070 ceph-mon[168282]: failed to encode map e424 with expected crc 2023-09-23T21:10:28.017 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:10:27 smithi070 ceph-mon[168282]: failed to encode map e424 with expected crc 2023-09-23T21:10:28.017 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:10:27 smithi070 ceph-mon[168282]: osdmap e424: 8 total, 8 up, 7 in 2023-09-23T21:10:28.017 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:10:27 smithi070 ceph-mon[168282]: failed to encode map e424 with expected crc 2023-09-23T21:10:28.018 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:10:27 smithi070 ceph-mon[168282]: failed to encode map e424 with expected crc 2023-09-23T21:10:28.018 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:10:27 smithi070 ceph-mon[168282]: failed to encode map e424 with expected crc 2023-09-23T21:10:28.018 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:10:27 smithi070 ceph-mon[168282]: pgmap v1103: 109 pgs: 5 peering, 104 active+clean; 180 MiB data, 3.1 GiB used, 533 GiB / 536 GiB avail; 30 MiB/s wr, 478 op/s; 794 B/s, 1 keys/s, 0 objects/s recovering 2023-09-23T21:10:28.018 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:10:27 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:10:28.018 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:10:27 smithi070 ceph-mon[168282]: osdmap e425: 8 total, 8 up, 7 in 2023-09-23T21:10:28.022 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:10:28.050 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:10:28.131 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:10:27 smithi026 ceph-mon[175151]: failed to encode map e424 with expected crc 2023-09-23T21:10:28.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:10:27 smithi026 ceph-mon[175151]: failed to encode map e424 with expected crc 2023-09-23T21:10:28.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:10:27 smithi026 ceph-mon[175151]: failed to encode map e424 with expected crc 2023-09-23T21:10:28.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:10:27 smithi026 ceph-mon[175151]: failed to encode map e424 with expected crc 2023-09-23T21:10:28.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:10:27 smithi026 ceph-mon[175151]: osdmap e424: 8 total, 8 up, 7 in 2023-09-23T21:10:28.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:10:27 smithi026 ceph-mon[175151]: failed to encode map e424 with expected crc 2023-09-23T21:10:28.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:10:27 smithi026 ceph-mon[175151]: failed to encode map e424 with expected crc 2023-09-23T21:10:28.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:10:27 smithi026 ceph-mon[175151]: failed to encode map e424 with expected crc 2023-09-23T21:10:28.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:10:27 smithi026 ceph-mon[175151]: pgmap v1103: 109 pgs: 5 peering, 104 active+clean; 180 MiB data, 3.1 GiB used, 533 GiB / 536 GiB avail; 30 MiB/s wr, 478 op/s; 794 B/s, 1 keys/s, 0 objects/s recovering 2023-09-23T21:10:28.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:10:27 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:10:28.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:10:27 smithi026 ceph-mon[175151]: osdmap e425: 8 total, 8 up, 7 in 2023-09-23T21:10:28.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:10:27 smithi026 ceph-mon[179688]: failed to encode map e424 with expected crc 2023-09-23T21:10:28.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:10:27 smithi026 ceph-mon[179688]: failed to encode map e424 with expected crc 2023-09-23T21:10:28.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:10:27 smithi026 ceph-mon[179688]: failed to encode map e424 with expected crc 2023-09-23T21:10:28.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:10:27 smithi026 ceph-mon[179688]: failed to encode map e424 with expected crc 2023-09-23T21:10:28.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:10:27 smithi026 ceph-mon[179688]: osdmap e424: 8 total, 8 up, 7 in 2023-09-23T21:10:28.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:10:27 smithi026 ceph-mon[179688]: failed to encode map e424 with expected crc 2023-09-23T21:10:28.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:10:27 smithi026 ceph-mon[179688]: failed to encode map e424 with expected crc 2023-09-23T21:10:28.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:10:27 smithi026 ceph-mon[179688]: failed to encode map e424 with expected crc 2023-09-23T21:10:28.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:10:27 smithi026 ceph-mon[179688]: pgmap v1103: 109 pgs: 5 peering, 104 active+clean; 180 MiB data, 3.1 GiB used, 533 GiB / 536 GiB avail; 30 MiB/s wr, 478 op/s; 794 B/s, 1 keys/s, 0 objects/s recovering 2023-09-23T21:10:28.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:10:27 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:10:28.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:10:27 smithi026 ceph-mon[179688]: osdmap e425: 8 total, 8 up, 7 in 2023-09-23T21:10:28.134 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:10:28.137 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:10:28.138 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:10:28.239 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:10:28.256 INFO:teuthology.orchestra.run.smithi070.stdout:{ 2023-09-23T21:10:28.256 INFO:teuthology.orchestra.run.smithi070.stdout: "ops": [], 2023-09-23T21:10:28.256 INFO:teuthology.orchestra.run.smithi070.stdout: "complaint_time": 30, 2023-09-23T21:10:28.256 INFO:teuthology.orchestra.run.smithi070.stdout: "num_blocked_ops": 0 2023-09-23T21:10:28.257 INFO:teuthology.orchestra.run.smithi070.stdout:} 2023-09-23T21:10:28.303 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:10:28.428 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 6 16 7970 7954 82.8248 17.3438 0.00431324 0.0120632 2023-09-23T21:10:28.537 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T21:10:28.883 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:10:28 smithi026 ceph-mon[179688]: failed to encode map e425 with expected crc 2023-09-23T21:10:28.883 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:10:28 smithi026 ceph-mon[179688]: failed to encode map e425 with expected crc 2023-09-23T21:10:28.883 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:10:28 smithi026 ceph-mon[179688]: failed to encode map e425 with expected crc 2023-09-23T21:10:28.883 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:10:28 smithi026 ceph-mon[179688]: failed to encode map e425 with expected crc 2023-09-23T21:10:28.883 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:10:28 smithi026 ceph-mon[175151]: failed to encode map e425 with expected crc 2023-09-23T21:10:28.884 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:10:28 smithi026 ceph-mon[175151]: failed to encode map e425 with expected crc 2023-09-23T21:10:28.884 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:10:28 smithi026 ceph-mon[175151]: failed to encode map e425 with expected crc 2023-09-23T21:10:28.884 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:10:28 smithi026 ceph-mon[175151]: failed to encode map e425 with expected crc 2023-09-23T21:10:29.117 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:10:28 smithi070 ceph-mon[168282]: failed to encode map e425 with expected crc 2023-09-23T21:10:29.117 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:10:28 smithi070 ceph-mon[168282]: failed to encode map e425 with expected crc 2023-09-23T21:10:29.117 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:10:28 smithi070 ceph-mon[168282]: failed to encode map e425 with expected crc 2023-09-23T21:10:29.117 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:10:28 smithi070 ceph-mon[168282]: failed to encode map e425 with expected crc 2023-09-23T21:10:29.428 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 7 16 10002 9986 89.1325 127 0.00592536 0.0112075 2023-09-23T21:10:29.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:10:29 smithi026 ceph-mon[175151]: failed to encode map e426 with expected crc 2023-09-23T21:10:29.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:10:29 smithi026 ceph-mon[175151]: osdmap e426: 8 total, 8 up, 7 in 2023-09-23T21:10:29.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:10:29 smithi026 ceph-mon[175151]: failed to encode map e426 with expected crc 2023-09-23T21:10:29.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:10:29 smithi026 ceph-mon[175151]: failed to encode map e426 with expected crc 2023-09-23T21:10:29.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:10:29 smithi026 ceph-mon[175151]: failed to encode map e426 with expected crc 2023-09-23T21:10:29.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:10:29 smithi026 ceph-mon[175151]: failed to encode map e426 with expected crc 2023-09-23T21:10:29.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:10:29 smithi026 ceph-mon[175151]: failed to encode map e426 with expected crc 2023-09-23T21:10:29.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:10:29 smithi026 ceph-mon[175151]: failed to encode map e426 with expected crc 2023-09-23T21:10:29.883 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:10:29 smithi026 ceph-mon[175151]: pgmap v1106: 109 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 8 remapped+peering, 5 peering, 1 active+remapped, 93 active+clean; 444 MiB data, 4.4 GiB used, 621 GiB / 626 GiB avail; 94 MiB/s wr, 1.51k op/s; 942/21336 objects degraded (4.415%); 31 KiB/s, 0 objects/s recovering 2023-09-23T21:10:29.883 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:10:29 smithi026 ceph-mon[179688]: failed to encode map e426 with expected crc 2023-09-23T21:10:29.883 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:10:29 smithi026 ceph-mon[179688]: osdmap e426: 8 total, 8 up, 7 in 2023-09-23T21:10:29.883 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:10:29 smithi026 ceph-mon[179688]: failed to encode map e426 with expected crc 2023-09-23T21:10:29.883 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:10:29 smithi026 ceph-mon[179688]: failed to encode map e426 with expected crc 2023-09-23T21:10:29.883 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:10:29 smithi026 ceph-mon[179688]: failed to encode map e426 with expected crc 2023-09-23T21:10:29.883 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:10:29 smithi026 ceph-mon[179688]: failed to encode map e426 with expected crc 2023-09-23T21:10:29.883 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:10:29 smithi026 ceph-mon[179688]: failed to encode map e426 with expected crc 2023-09-23T21:10:29.883 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:10:29 smithi026 ceph-mon[179688]: failed to encode map e426 with expected crc 2023-09-23T21:10:29.883 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:10:29 smithi026 ceph-mon[179688]: pgmap v1106: 109 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 8 remapped+peering, 5 peering, 1 active+remapped, 93 active+clean; 444 MiB data, 4.4 GiB used, 621 GiB / 626 GiB avail; 94 MiB/s wr, 1.51k op/s; 942/21336 objects degraded (4.415%); 31 KiB/s, 0 objects/s recovering 2023-09-23T21:10:29.884 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 21:10:29 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[164029]: ::ffff:172.21.15.70 - - [23/Sep/2023:21:10:29] "GET /metrics HTTP/1.1" 200 36313 "" "Prometheus/2.43.0" 2023-09-23T21:10:30.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:10:29 smithi070 ceph-mon[168282]: failed to encode map e426 with expected crc 2023-09-23T21:10:30.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:10:29 smithi070 ceph-mon[168282]: osdmap e426: 8 total, 8 up, 7 in 2023-09-23T21:10:30.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:10:29 smithi070 ceph-mon[168282]: failed to encode map e426 with expected crc 2023-09-23T21:10:30.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:10:29 smithi070 ceph-mon[168282]: failed to encode map e426 with expected crc 2023-09-23T21:10:30.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:10:29 smithi070 ceph-mon[168282]: failed to encode map e426 with expected crc 2023-09-23T21:10:30.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:10:29 smithi070 ceph-mon[168282]: failed to encode map e426 with expected crc 2023-09-23T21:10:30.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:10:29 smithi070 ceph-mon[168282]: failed to encode map e426 with expected crc 2023-09-23T21:10:30.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:10:29 smithi070 ceph-mon[168282]: failed to encode map e426 with expected crc 2023-09-23T21:10:30.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:10:29 smithi070 ceph-mon[168282]: pgmap v1106: 109 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 8 remapped+peering, 5 peering, 1 active+remapped, 93 active+clean; 444 MiB data, 4.4 GiB used, 621 GiB / 626 GiB avail; 94 MiB/s wr, 1.51k op/s; 942/21336 objects degraded (4.415%); 31 KiB/s, 0 objects/s recovering 2023-09-23T21:10:30.284 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T21:10:30.287 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:10:30.288 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:10:30.389 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:10:30.431 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 8 16 11803 11787 92.044 112.562 0.0115161 0.0108563 2023-09-23T21:10:30.715 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:10:30.857 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:10:30.859 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_historic_ops 2023-09-23T21:10:30.860 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:10:30.861 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:10:30.962 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:10:30.993 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:10:31.005 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:10:30 smithi070 ceph-mon[168282]: Health check failed: Degraded data redundancy: 942/21336 objects degraded (4.415%), 2 pgs degraded (PG_DEGRADED) 2023-09-23T21:10:31.005 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:10:30 smithi070 ceph-mon[168282]: failed to encode map e427 with expected crc 2023-09-23T21:10:31.005 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:10:30 smithi070 ceph-mon[168282]: failed to encode map e427 with expected crc 2023-09-23T21:10:31.006 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:10:30 smithi070 ceph-mon[168282]: failed to encode map e427 with expected crc 2023-09-23T21:10:31.006 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:10:30 smithi070 ceph-mon[168282]: osdmap e427: 8 total, 8 up, 7 in 2023-09-23T21:10:31.006 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:10:30 smithi070 ceph-mon[168282]: failed to encode map e427 with expected crc 2023-09-23T21:10:31.006 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:10:30 smithi070 ceph-mon[168282]: failed to encode map e427 with expected crc 2023-09-23T21:10:31.006 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:10:30 smithi070 ceph-mon[168282]: osdmap e428: 8 total, 8 up, 7 in 2023-09-23T21:10:31.042 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:10:31.045 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:10:31.046 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:10:31.131 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:10:30 smithi026 ceph-mon[175151]: Health check failed: Degraded data redundancy: 942/21336 objects degraded (4.415%), 2 pgs degraded (PG_DEGRADED) 2023-09-23T21:10:31.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:10:30 smithi026 ceph-mon[175151]: failed to encode map e427 with expected crc 2023-09-23T21:10:31.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:10:30 smithi026 ceph-mon[175151]: failed to encode map e427 with expected crc 2023-09-23T21:10:31.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:10:30 smithi026 ceph-mon[175151]: failed to encode map e427 with expected crc 2023-09-23T21:10:31.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:10:30 smithi026 ceph-mon[175151]: osdmap e427: 8 total, 8 up, 7 in 2023-09-23T21:10:31.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:10:30 smithi026 ceph-mon[175151]: failed to encode map e427 with expected crc 2023-09-23T21:10:31.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:10:30 smithi026 ceph-mon[175151]: failed to encode map e427 with expected crc 2023-09-23T21:10:31.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:10:30 smithi026 ceph-mon[175151]: osdmap e428: 8 total, 8 up, 7 in 2023-09-23T21:10:31.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:10:30 smithi026 ceph-mon[179688]: Health check failed: Degraded data redundancy: 942/21336 objects degraded (4.415%), 2 pgs degraded (PG_DEGRADED) 2023-09-23T21:10:31.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:10:30 smithi026 ceph-mon[179688]: failed to encode map e427 with expected crc 2023-09-23T21:10:31.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:10:30 smithi026 ceph-mon[179688]: failed to encode map e427 with expected crc 2023-09-23T21:10:31.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:10:30 smithi026 ceph-mon[179688]: failed to encode map e427 with expected crc 2023-09-23T21:10:31.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:10:30 smithi026 ceph-mon[179688]: osdmap e427: 8 total, 8 up, 7 in 2023-09-23T21:10:31.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:10:30 smithi026 ceph-mon[179688]: failed to encode map e427 with expected crc 2023-09-23T21:10:31.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:10:30 smithi026 ceph-mon[179688]: failed to encode map e427 with expected crc 2023-09-23T21:10:31.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:10:30 smithi026 ceph-mon[179688]: osdmap e428: 8 total, 8 up, 7 in 2023-09-23T21:10:31.146 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:10:31.172 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:10:31.203 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:10:31.206 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:10:31.206 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:10:31.243 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:10:31.307 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:10:31.313 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 7] out_osds: [6] dead_osds: [] live_osds: [0, 2, 3, 4, 5, 7, 6, 1] 2023-09-23T21:10:31.313 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T21:10:31.314 INFO:tasks.thrashosds.thrasher:primary_affinity 2023-09-23T21:10:31.314 INFO:tasks.thrashosds.thrasher:Setting osd 3 primary_affinity to 0.640320 2023-09-23T21:10:31.314 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph osd primary-affinity 3 0.6403203422890724 2023-09-23T21:10:31.333 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:10:31.367 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:10:31.370 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:10:31.370 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:10:31.430 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 9 16 12173 12157 84.3885 23.125 0.00161811 0.0110614 2023-09-23T21:10:31.471 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:10:31.499 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:10:31.682 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T21:10:32.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:10:31 smithi070 ceph-mon[168282]: failed to encode map e428 with expected crc 2023-09-23T21:10:32.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:10:31 smithi070 ceph-mon[168282]: failed to encode map e428 with expected crc 2023-09-23T21:10:32.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:10:31 smithi070 ceph-mon[168282]: failed to encode map e428 with expected crc 2023-09-23T21:10:32.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:10:31 smithi070 ceph-mon[168282]: pgmap v1109: 109 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 8 remapped+peering, 8 peering, 1 active+remapped, 90 active+clean; 489 MiB data, 4.6 GiB used, 621 GiB / 626 GiB avail; 77 MiB/s wr, 1.24k op/s; 942/23523 objects degraded (4.005%); 31 KiB/s, 0 objects/s recovering 2023-09-23T21:10:32.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:10:31 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:10:32.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:10:31 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:10:32.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:10:31 smithi070 ceph-mon[168282]: osdmap e429: 8 total, 8 up, 7 in 2023-09-23T21:10:32.131 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:10:31 smithi026 ceph-mon[179688]: failed to encode map e428 with expected crc 2023-09-23T21:10:32.131 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:10:31 smithi026 ceph-mon[179688]: failed to encode map e428 with expected crc 2023-09-23T21:10:32.131 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:10:31 smithi026 ceph-mon[179688]: failed to encode map e428 with expected crc 2023-09-23T21:10:32.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:10:31 smithi026 ceph-mon[179688]: pgmap v1109: 109 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 8 remapped+peering, 8 peering, 1 active+remapped, 90 active+clean; 489 MiB data, 4.6 GiB used, 621 GiB / 626 GiB avail; 77 MiB/s wr, 1.24k op/s; 942/23523 objects degraded (4.005%); 31 KiB/s, 0 objects/s recovering 2023-09-23T21:10:32.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:10:31 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:10:32.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:10:31 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:10:32.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:10:31 smithi026 ceph-mon[179688]: osdmap e429: 8 total, 8 up, 7 in 2023-09-23T21:10:32.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:10:31 smithi026 ceph-mon[175151]: failed to encode map e428 with expected crc 2023-09-23T21:10:32.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:10:31 smithi026 ceph-mon[175151]: failed to encode map e428 with expected crc 2023-09-23T21:10:32.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:10:31 smithi026 ceph-mon[175151]: failed to encode map e428 with expected crc 2023-09-23T21:10:32.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:10:31 smithi026 ceph-mon[175151]: pgmap v1109: 109 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 8 remapped+peering, 8 peering, 1 active+remapped, 90 active+clean; 489 MiB data, 4.6 GiB used, 621 GiB / 626 GiB avail; 77 MiB/s wr, 1.24k op/s; 942/23523 objects degraded (4.005%); 31 KiB/s, 0 objects/s recovering 2023-09-23T21:10:32.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:10:31 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:10:32.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:10:31 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:10:32.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:10:31 smithi026 ceph-mon[175151]: osdmap e429: 8 total, 8 up, 7 in 2023-09-23T21:10:32.430 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 10 16 12378 12362 77.2329 12.8125 0.0014816 0.0120691 2023-09-23T21:10:32.827 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:10:32.834 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:10:32.834 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:10:32.935 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:10:32.975 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:10:33.035 INFO:teuthology.orchestra.run.smithi070.stdout:{ 2023-09-23T21:10:33.035 INFO:teuthology.orchestra.run.smithi070.stdout: "size": 20, 2023-09-23T21:10:33.035 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 600, 2023-09-23T21:10:33.035 INFO:teuthology.orchestra.run.smithi070.stdout: "ops": [ 2023-09-23T21:10:33.035 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:33.035 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.35469.0:133009 8.2 8:43227b1f:::benchmark_data_smithi026_301269_object22252:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e281)", 2023-09-23T21:10:33.035 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:03:29.350078+0000", 2023-09-23T21:10:33.035 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 423.68277058799998, 2023-09-23T21:10:33.035 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.92559535299999995, 2023-09-23T21:10:33.036 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:10:33.036 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:10:33.036 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:10:33.036 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.35469", 2023-09-23T21:10:33.036 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/4041424623", 2023-09-23T21:10:33.036 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 133009 2023-09-23T21:10:33.036 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:33.036 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:10:33.036 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:33.036 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:10:33.036 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:29.350078+0000", 2023-09-23T21:10:33.036 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:10:33.036 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:33.037 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:33.037 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:10:33.037 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:29.350078+0000", 2023-09-23T21:10:33.037 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:10:33.037 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:33.037 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:33.037 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:10:33.037 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:29.350075+0000", 2023-09-23T21:10:33.037 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999971 2023-09-23T21:10:33.037 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:33.037 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:33.037 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:10:33.037 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:29.350092+0000", 2023-09-23T21:10:33.038 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.6350000000000001e-05 2023-09-23T21:10:33.038 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:33.038 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:33.038 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:10:33.038 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:29.350130+0000", 2023-09-23T21:10:33.038 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.8577e-05 2023-09-23T21:10:33.038 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:33.038 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:33.038 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:10:33.038 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:29.350157+0000", 2023-09-23T21:10:33.038 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.6738999999999999e-05 2023-09-23T21:10:33.038 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:33.039 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:33.039 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:10:33.039 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:29.508836+0000", 2023-09-23T21:10:33.039 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.158678763 2023-09-23T21:10:33.039 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:33.039 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:33.039 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:10:33.039 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:29.508848+0000", 2023-09-23T21:10:33.039 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.2469e-05 2023-09-23T21:10:33.039 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:33.039 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:33.039 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:10:33.039 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:30.262423+0000", 2023-09-23T21:10:33.040 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.75357442699999999 2023-09-23T21:10:33.040 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:33.040 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:33.040 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:10:33.040 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:30.262553+0000", 2023-09-23T21:10:33.040 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000130748 2023-09-23T21:10:33.040 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:33.040 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:33.040 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 0,5", 2023-09-23T21:10:33.040 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:30.262945+0000", 2023-09-23T21:10:33.040 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00039175199999999999 2023-09-23T21:10:33.040 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:33.041 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:33.041 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:10:33.041 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:30.267224+0000", 2023-09-23T21:10:33.041 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0042790399999999996 2023-09-23T21:10:33.041 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:33.041 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:33.041 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:10:33.041 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:30.267282+0000", 2023-09-23T21:10:33.041 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.7314e-05 2023-09-23T21:10:33.041 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:33.041 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:33.041 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:10:33.041 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:30.275547+0000", 2023-09-23T21:10:33.042 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0082652219999999992 2023-09-23T21:10:33.042 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:33.042 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:33.042 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:10:33.042 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:30.275603+0000", 2023-09-23T21:10:33.042 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.5909999999999998e-05 2023-09-23T21:10:33.042 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:33.042 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:33.042 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:10:33.042 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:30.275674+0000", 2023-09-23T21:10:33.042 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.1094999999999996e-05 2023-09-23T21:10:33.042 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:10:33.042 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:10:33.043 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:10:33.043 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:33.043 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:33.043 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.35469.0:164165 8.14 8:29df2d77:::benchmark_data_smithi026_301269_object53408:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e307)", 2023-09-23T21:10:33.043 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:04:13.318492+0000", 2023-09-23T21:10:33.043 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 379.71435729900003, 2023-09-23T21:10:33.043 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.95646294899999995, 2023-09-23T21:10:33.043 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:10:33.043 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:10:33.043 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:10:33.043 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.35469", 2023-09-23T21:10:33.043 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/4041424623", 2023-09-23T21:10:33.044 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 164165 2023-09-23T21:10:33.044 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:33.044 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:10:33.044 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:33.044 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:10:33.044 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:13.318492+0000", 2023-09-23T21:10:33.044 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:10:33.044 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:33.044 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:33.044 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:10:33.044 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:13.318492+0000", 2023-09-23T21:10:33.044 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:10:33.045 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:33.045 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:33.045 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:10:33.045 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:13.318491+0000", 2023-09-23T21:10:33.045 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.999999 2023-09-23T21:10:33.045 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:33.045 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:33.045 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:10:33.045 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:13.318496+0000", 2023-09-23T21:10:33.045 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.1680000000000003e-06 2023-09-23T21:10:33.045 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:33.045 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:33.045 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:10:33.046 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:13.318506+0000", 2023-09-23T21:10:33.046 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.0454e-05 2023-09-23T21:10:33.046 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:33.046 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:33.046 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:10:33.046 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:13.318511+0000", 2023-09-23T21:10:33.046 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.4959999999999998e-06 2023-09-23T21:10:33.046 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:33.046 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:33.046 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:10:33.046 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:13.318553+0000", 2023-09-23T21:10:33.046 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.2230000000000001e-05 2023-09-23T21:10:33.047 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:33.047 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:33.047 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:10:33.047 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:13.318560+0000", 2023-09-23T21:10:33.047 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.2320000000000004e-06 2023-09-23T21:10:33.047 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:33.047 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:33.047 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:10:33.047 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:14.243781+0000", 2023-09-23T21:10:33.047 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.92522103200000005 2023-09-23T21:10:33.047 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:33.047 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:33.047 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:10:33.048 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:14.243936+0000", 2023-09-23T21:10:33.048 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00015447099999999999 2023-09-23T21:10:33.048 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:33.048 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:33.048 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 0,2", 2023-09-23T21:10:33.048 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:14.244331+0000", 2023-09-23T21:10:33.048 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00039575899999999998 2023-09-23T21:10:33.048 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:33.048 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:33.048 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:10:33.048 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:14.246713+0000", 2023-09-23T21:10:33.048 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.002382056 2023-09-23T21:10:33.048 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:33.049 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:33.049 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:10:33.049 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:14.246724+0000", 2023-09-23T21:10:33.049 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.0152999999999999e-05 2023-09-23T21:10:33.049 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:33.049 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:33.049 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:10:33.049 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:14.274860+0000", 2023-09-23T21:10:33.049 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.028136345 2023-09-23T21:10:33.049 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:33.049 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:33.049 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:10:33.050 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:14.274913+0000", 2023-09-23T21:10:33.050 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.2827000000000001e-05 2023-09-23T21:10:33.050 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:33.050 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:33.050 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:10:33.050 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:14.274955+0000", 2023-09-23T21:10:33.050 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.1907999999999999e-05 2023-09-23T21:10:33.050 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:10:33.050 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:10:33.050 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:10:33.050 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:33.050 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:33.051 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.35469.0:164203 8.14 8:2d1c900f:::benchmark_data_smithi026_301269_object53446:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e307)", 2023-09-23T21:10:33.051 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:04:13.371049+0000", 2023-09-23T21:10:33.051 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 379.66179967099998, 2023-09-23T21:10:33.051 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.90403997899999999, 2023-09-23T21:10:33.051 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:10:33.051 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:10:33.051 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:10:33.051 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.35469", 2023-09-23T21:10:33.051 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/4041424623", 2023-09-23T21:10:33.051 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 164203 2023-09-23T21:10:33.051 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:33.051 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:10:33.051 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:33.051 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:10:33.052 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:13.371049+0000", 2023-09-23T21:10:33.052 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:10:33.052 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:33.052 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:33.052 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:10:33.052 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:13.371049+0000", 2023-09-23T21:10:33.052 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:10:33.052 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:33.052 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:33.052 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:10:33.052 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:13.371048+0000", 2023-09-23T21:10:33.052 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999981 2023-09-23T21:10:33.053 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:33.053 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:33.053 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:10:33.053 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:13.371053+0000", 2023-09-23T21:10:33.053 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.8080000000000001e-06 2023-09-23T21:10:33.053 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:33.053 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:33.053 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:10:33.053 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:13.371064+0000", 2023-09-23T21:10:33.053 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.0781999999999999e-05 2023-09-23T21:10:33.053 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:33.053 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:33.054 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:10:33.054 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:13.371070+0000", 2023-09-23T21:10:33.054 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.8300000000000001e-06 2023-09-23T21:10:33.054 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:33.054 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:33.054 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:10:33.054 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:13.371112+0000", 2023-09-23T21:10:33.054 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.2020000000000001e-05 2023-09-23T21:10:33.054 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:33.054 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:33.054 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:10:33.054 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:13.371119+0000", 2023-09-23T21:10:33.055 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.2110000000000001e-06 2023-09-23T21:10:33.055 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:33.055 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:33.055 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:10:33.055 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:14.246769+0000", 2023-09-23T21:10:33.055 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.87565003799999996 2023-09-23T21:10:33.055 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:33.055 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:33.055 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:10:33.055 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:14.246842+0000", 2023-09-23T21:10:33.055 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.2512e-05 2023-09-23T21:10:33.055 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:33.056 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:33.056 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 0,2", 2023-09-23T21:10:33.056 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:14.247045+0000", 2023-09-23T21:10:33.056 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000203005 2023-09-23T21:10:33.056 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:33.056 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:33.056 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:10:33.056 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:14.252323+0000", 2023-09-23T21:10:33.056 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.005278235 2023-09-23T21:10:33.056 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:33.056 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:33.056 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:10:33.056 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:14.252430+0000", 2023-09-23T21:10:33.057 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000106543 2023-09-23T21:10:33.057 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:33.057 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:33.057 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:10:33.057 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:14.275003+0000", 2023-09-23T21:10:33.057 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.022573473 2023-09-23T21:10:33.057 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:33.057 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:33.057 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:10:33.057 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:14.275049+0000", 2023-09-23T21:10:33.057 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.5510999999999998e-05 2023-09-23T21:10:33.057 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:33.057 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:33.058 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:10:33.058 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:14.275089+0000", 2023-09-23T21:10:33.058 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.0738999999999997e-05 2023-09-23T21:10:33.058 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:10:33.058 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:10:33.058 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:10:33.058 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:33.058 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:33.058 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.35469.0:172321 8.18 8:1f117af2:::benchmark_data_smithi026_301269_object61564:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e317)", 2023-09-23T21:10:33.058 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:04:28.396199+0000", 2023-09-23T21:10:33.058 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 364.63665048500002, 2023-09-23T21:10:33.058 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.0147737569999999, 2023-09-23T21:10:33.059 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:10:33.059 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:10:33.059 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:10:33.059 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.35469", 2023-09-23T21:10:33.059 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/4041424623", 2023-09-23T21:10:33.059 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 172321 2023-09-23T21:10:33.059 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:33.059 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:10:33.059 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:33.059 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:10:33.059 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:28.396199+0000", 2023-09-23T21:10:33.059 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:10:33.060 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:33.060 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:33.060 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:10:33.060 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:28.396199+0000", 2023-09-23T21:10:33.060 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:10:33.060 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:33.060 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:33.060 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:10:33.060 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:28.396198+0000", 2023-09-23T21:10:33.060 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999995 2023-09-23T21:10:33.060 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:33.060 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:33.060 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:10:33.061 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:28.396200+0000", 2023-09-23T21:10:33.061 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.8419999999999999e-06 2023-09-23T21:10:33.061 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:33.061 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:33.061 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:10:33.061 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:28.396204+0000", 2023-09-23T21:10:33.061 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.5640000000000001e-06 2023-09-23T21:10:33.061 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:33.061 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:33.061 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:10:33.061 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:28.396206+0000", 2023-09-23T21:10:33.061 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.2249999999999999e-06 2023-09-23T21:10:33.062 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:33.062 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:33.062 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:10:33.062 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:28.396256+0000", 2023-09-23T21:10:33.062 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.9913000000000001e-05 2023-09-23T21:10:33.062 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:33.062 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:33.062 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:10:33.062 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:28.396264+0000", 2023-09-23T21:10:33.062 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.3809999999999999e-06 2023-09-23T21:10:33.062 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:33.062 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:33.062 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:10:33.063 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:29.402549+0000", 2023-09-23T21:10:33.063 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.0062851639999999 2023-09-23T21:10:33.063 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:33.063 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:33.063 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:10:33.063 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:29.402686+0000", 2023-09-23T21:10:33.063 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00013646700000000001 2023-09-23T21:10:33.063 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:33.063 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:33.063 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 3,5", 2023-09-23T21:10:33.063 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:29.403010+0000", 2023-09-23T21:10:33.063 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00032413599999999999 2023-09-23T21:10:33.063 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:33.064 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:33.064 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:10:33.064 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:29.407637+0000", 2023-09-23T21:10:33.064 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0046273240000000004 2023-09-23T21:10:33.064 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:33.064 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:33.064 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:10:33.064 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:29.407667+0000", 2023-09-23T21:10:33.064 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.0192000000000002e-05 2023-09-23T21:10:33.064 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:33.064 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:33.064 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:10:33.065 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:29.410884+0000", 2023-09-23T21:10:33.065 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0032160800000000001 2023-09-23T21:10:33.065 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:33.065 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:33.065 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:10:33.065 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:29.410929+0000", 2023-09-23T21:10:33.065 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.5364000000000001e-05 2023-09-23T21:10:33.065 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:33.065 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:33.065 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:10:33.065 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:29.410972+0000", 2023-09-23T21:10:33.065 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.3436999999999998e-05 2023-09-23T21:10:33.065 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:10:33.066 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:10:33.066 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:10:33.066 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:33.066 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:33.066 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.35469.0:172350 8.18 8:1980b52b:::benchmark_data_smithi026_301269_object61593:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e317)", 2023-09-23T21:10:33.066 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:04:28.433685+0000", 2023-09-23T21:10:33.066 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 364.59916374900001, 2023-09-23T21:10:33.066 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.97745544200000001, 2023-09-23T21:10:33.066 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:10:33.066 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:10:33.066 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:10:33.066 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.35469", 2023-09-23T21:10:33.067 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/4041424623", 2023-09-23T21:10:33.067 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 172350 2023-09-23T21:10:33.067 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:33.067 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:10:33.067 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:33.067 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:10:33.067 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:28.433685+0000", 2023-09-23T21:10:33.067 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:10:33.067 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:33.067 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:33.067 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:10:33.067 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:28.433685+0000", 2023-09-23T21:10:33.067 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:10:33.067 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:33.068 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:33.068 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:10:33.068 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:28.433683+0000", 2023-09-23T21:10:33.068 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999981 2023-09-23T21:10:33.068 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:33.068 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:33.068 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:10:33.068 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:28.433690+0000", 2023-09-23T21:10:33.068 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.2199999999999997e-06 2023-09-23T21:10:33.068 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:33.068 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:33.068 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:10:33.068 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:28.433701+0000", 2023-09-23T21:10:33.069 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.1474999999999999e-05 2023-09-23T21:10:33.069 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:33.069 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:33.069 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:10:33.069 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:28.433707+0000", 2023-09-23T21:10:33.069 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.6470000000000003e-06 2023-09-23T21:10:33.069 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:33.069 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:33.069 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:10:33.069 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:28.433761+0000", 2023-09-23T21:10:33.069 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.4412e-05 2023-09-23T21:10:33.069 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:33.069 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:33.070 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:10:33.070 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:28.433769+0000", 2023-09-23T21:10:33.070 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.2379999999999997e-06 2023-09-23T21:10:33.070 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:33.070 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:33.070 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:10:33.070 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:29.404626+0000", 2023-09-23T21:10:33.070 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.97085643600000004 2023-09-23T21:10:33.070 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:33.070 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:33.070 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:10:33.070 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:29.404696+0000", 2023-09-23T21:10:33.071 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.0241999999999996e-05 2023-09-23T21:10:33.071 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:33.071 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:33.071 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 3,5", 2023-09-23T21:10:33.071 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:29.404851+0000", 2023-09-23T21:10:33.071 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00015510599999999999 2023-09-23T21:10:33.071 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:33.071 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:33.071 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:10:33.071 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:29.407733+0000", 2023-09-23T21:10:33.071 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0028823120000000002 2023-09-23T21:10:33.071 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:33.072 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:33.072 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:10:33.072 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:29.411020+0000", 2023-09-23T21:10:33.072 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0032863570000000002 2023-09-23T21:10:33.072 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:33.072 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:33.072 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:10:33.072 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:29.411055+0000", 2023-09-23T21:10:33.072 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.5271999999999997e-05 2023-09-23T21:10:33.072 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:33.072 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:33.072 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:10:33.072 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:29.411102+0000", 2023-09-23T21:10:33.072 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.6746999999999998e-05 2023-09-23T21:10:33.073 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:33.073 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:33.073 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:10:33.073 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:29.411141+0000", 2023-09-23T21:10:33.073 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.8884000000000002e-05 2023-09-23T21:10:33.073 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:10:33.073 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:10:33.073 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:10:33.073 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:33.073 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:33.073 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.35469.0:172376 8.18 8:1aa907c3:::benchmark_data_smithi026_301269_object61619:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e317)", 2023-09-23T21:10:33.073 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:04:28.453649+0000", 2023-09-23T21:10:33.074 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 364.57919997599998, 2023-09-23T21:10:33.074 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.96206967399999999, 2023-09-23T21:10:33.074 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:10:33.074 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:10:33.074 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:10:33.074 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.35469", 2023-09-23T21:10:33.074 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/4041424623", 2023-09-23T21:10:33.074 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 172376 2023-09-23T21:10:33.074 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:33.074 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:10:33.074 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:33.074 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:10:33.075 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:28.453649+0000", 2023-09-23T21:10:33.075 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:10:33.075 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:33.075 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:33.075 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:10:33.075 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:28.453649+0000", 2023-09-23T21:10:33.075 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:10:33.075 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:33.075 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:33.075 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:10:33.075 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:28.453648+0000", 2023-09-23T21:10:33.075 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T21:10:33.075 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:33.076 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:33.076 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:10:33.076 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:28.453654+0000", 2023-09-23T21:10:33.076 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.0070000000000003e-06 2023-09-23T21:10:33.076 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:33.076 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:33.076 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:10:33.076 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:28.453667+0000", 2023-09-23T21:10:33.076 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.3389e-05 2023-09-23T21:10:33.076 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:33.076 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:33.076 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:10:33.077 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:28.453672+0000", 2023-09-23T21:10:33.077 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.4039999999999997e-06 2023-09-23T21:10:33.077 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:33.077 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:33.077 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:10:33.077 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:28.453719+0000", 2023-09-23T21:10:33.077 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.7491999999999999e-05 2023-09-23T21:10:33.077 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:33.077 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:33.077 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:10:33.077 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:28.453728+0000", 2023-09-23T21:10:33.077 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.1589999999999999e-06 2023-09-23T21:10:33.078 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:33.078 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:33.078 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:10:33.078 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:29.405990+0000", 2023-09-23T21:10:33.078 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.95226191800000004 2023-09-23T21:10:33.078 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:33.078 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:33.078 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:10:33.078 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:29.406103+0000", 2023-09-23T21:10:33.078 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00011302300000000001 2023-09-23T21:10:33.078 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:33.078 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:33.079 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 3,5", 2023-09-23T21:10:33.079 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:29.406346+0000", 2023-09-23T21:10:33.079 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00024300600000000001 2023-09-23T21:10:33.079 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:33.079 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:33.079 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:10:33.079 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:29.411176+0000", 2023-09-23T21:10:33.079 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0048295949999999999 2023-09-23T21:10:33.079 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:33.079 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:33.079 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:10:33.079 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:29.412780+0000", 2023-09-23T21:10:33.079 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.001604131 2023-09-23T21:10:33.080 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:33.080 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:33.080 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:10:33.080 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:29.415622+0000", 2023-09-23T21:10:33.080 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.002842266 2023-09-23T21:10:33.080 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:33.080 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:33.080 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:10:33.080 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:29.415676+0000", 2023-09-23T21:10:33.080 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.3331000000000002e-05 2023-09-23T21:10:33.080 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:33.080 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:33.080 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:10:33.081 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:29.415719+0000", 2023-09-23T21:10:33.081 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.3166e-05 2023-09-23T21:10:33.081 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:10:33.081 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:10:33.081 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:10:33.081 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:33.081 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:33.081 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.35469.0:172468 8.18 8:19eada54:::benchmark_data_smithi026_301269_object61711:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e317)", 2023-09-23T21:10:33.081 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:04:28.581022+0000", 2023-09-23T21:10:33.081 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 364.45182680699997, 2023-09-23T21:10:33.081 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.83865129299999996, 2023-09-23T21:10:33.081 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:10:33.081 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:10:33.082 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:10:33.082 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.35469", 2023-09-23T21:10:33.082 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/4041424623", 2023-09-23T21:10:33.082 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 172468 2023-09-23T21:10:33.082 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:33.082 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:10:33.082 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:33.082 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:10:33.082 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:28.581022+0000", 2023-09-23T21:10:33.082 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:10:33.082 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:33.083 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:33.083 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:10:33.083 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:28.581022+0000", 2023-09-23T21:10:33.083 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:10:33.083 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:33.083 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:33.083 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:10:33.083 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:28.581021+0000", 2023-09-23T21:10:33.083 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T21:10:33.083 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:33.083 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:33.083 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:10:33.083 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:28.581026+0000", 2023-09-23T21:10:33.084 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.2349999999999999e-06 2023-09-23T21:10:33.084 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:33.084 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:33.084 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:10:33.084 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:28.581038+0000", 2023-09-23T21:10:33.084 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.1779e-05 2023-09-23T21:10:33.084 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:33.084 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:33.084 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:10:33.084 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:28.581043+0000", 2023-09-23T21:10:33.084 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.7480000000000001e-06 2023-09-23T21:10:33.084 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:33.085 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:33.085 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:10:33.085 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:28.581080+0000", 2023-09-23T21:10:33.085 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.7591e-05 2023-09-23T21:10:33.085 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:33.085 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:33.085 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:10:33.085 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:28.581092+0000", 2023-09-23T21:10:33.085 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.1747e-05 2023-09-23T21:10:33.085 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:33.085 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:33.085 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:10:33.086 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:29.407757+0000", 2023-09-23T21:10:33.086 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.82666530000000005 2023-09-23T21:10:33.086 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:33.086 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:33.086 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:10:33.086 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:29.407864+0000", 2023-09-23T21:10:33.086 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000106475 2023-09-23T21:10:33.086 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:33.086 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:33.086 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 3,5", 2023-09-23T21:10:33.086 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:29.408075+0000", 2023-09-23T21:10:33.086 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00021070400000000001 2023-09-23T21:10:33.087 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:33.087 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:33.087 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:10:33.087 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:29.415555+0000", 2023-09-23T21:10:33.087 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.007480815 2023-09-23T21:10:33.087 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:33.087 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:33.087 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:10:33.087 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:29.415756+0000", 2023-09-23T21:10:33.087 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000200838 2023-09-23T21:10:33.087 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:33.087 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:33.087 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:10:33.088 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:29.419577+0000", 2023-09-23T21:10:33.088 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0038211569999999999 2023-09-23T21:10:33.088 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:33.088 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:33.088 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:10:33.088 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:29.419629+0000", 2023-09-23T21:10:33.088 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.1646e-05 2023-09-23T21:10:33.088 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:33.088 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:33.088 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:10:33.088 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:29.419674+0000", 2023-09-23T21:10:33.088 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.4481999999999999e-05 2023-09-23T21:10:33.088 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:10:33.089 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:10:33.089 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:10:33.089 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:33.089 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:33.089 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.35469.0:172485 8.18 8:1dd2dd78:::benchmark_data_smithi026_301269_object61728:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e317)", 2023-09-23T21:10:33.089 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:04:28.602617+0000", 2023-09-23T21:10:33.089 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 364.43023201, 2023-09-23T21:10:33.089 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.81713416000000005, 2023-09-23T21:10:33.089 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:10:33.089 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:10:33.089 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:10:33.089 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.35469", 2023-09-23T21:10:33.089 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/4041424623", 2023-09-23T21:10:33.090 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 172485 2023-09-23T21:10:33.090 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:33.090 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:10:33.090 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:33.090 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:10:33.090 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:28.602617+0000", 2023-09-23T21:10:33.090 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:10:33.090 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:33.090 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:33.090 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:10:33.090 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:28.602617+0000", 2023-09-23T21:10:33.090 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:10:33.091 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:33.091 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:33.091 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:10:33.091 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:28.602616+0000", 2023-09-23T21:10:33.091 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T21:10:33.091 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:33.091 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:33.091 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:10:33.091 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:28.602622+0000", 2023-09-23T21:10:33.091 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.7590000000000003e-06 2023-09-23T21:10:33.091 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:33.092 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:33.092 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:10:33.092 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:28.602634+0000", 2023-09-23T21:10:33.092 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.2655e-05 2023-09-23T21:10:33.092 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:33.092 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:33.092 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:10:33.092 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:28.602641+0000", 2023-09-23T21:10:33.092 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.4629999999999998e-06 2023-09-23T21:10:33.092 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:33.092 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:33.092 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:10:33.092 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:28.603804+0000", 2023-09-23T21:10:33.093 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0011636750000000001 2023-09-23T21:10:33.093 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:33.093 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:33.093 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:10:33.093 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:28.603813+0000", 2023-09-23T21:10:33.093 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.3780000000000001e-06 2023-09-23T21:10:33.093 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:33.093 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:33.093 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:10:33.093 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:29.409257+0000", 2023-09-23T21:10:33.093 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.80544433800000004 2023-09-23T21:10:33.093 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:33.093 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:33.094 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:10:33.094 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:29.409524+0000", 2023-09-23T21:10:33.094 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00026692799999999999 2023-09-23T21:10:33.094 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:33.094 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:33.094 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 3,5", 2023-09-23T21:10:33.094 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:29.409682+0000", 2023-09-23T21:10:33.094 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00015761 2023-09-23T21:10:33.094 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:33.094 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:33.094 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:10:33.095 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:29.415565+0000", 2023-09-23T21:10:33.095 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0058839030000000002 2023-09-23T21:10:33.095 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:33.095 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:33.095 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:10:33.095 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:29.415794+0000", 2023-09-23T21:10:33.095 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00022896099999999999 2023-09-23T21:10:33.095 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:33.095 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:33.095 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:10:33.095 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:29.419701+0000", 2023-09-23T21:10:33.095 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0039063769999999999 2023-09-23T21:10:33.095 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:33.096 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:33.096 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:10:33.096 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:29.419728+0000", 2023-09-23T21:10:33.096 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.6852e-05 2023-09-23T21:10:33.096 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:33.096 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:33.096 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:10:33.096 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:29.419751+0000", 2023-09-23T21:10:33.096 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.3513999999999999e-05 2023-09-23T21:10:33.096 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:10:33.096 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:10:33.096 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:10:33.097 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:33.097 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:33.097 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.35469.0:197936 8.4 8:20e4aa53:::benchmark_data_smithi026_301269_object87179:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e330)", 2023-09-23T21:10:33.097 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:05:01.418440+0000", 2023-09-23T21:10:33.097 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 331.61440857399998, 2023-09-23T21:10:33.097 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.88711867600000005, 2023-09-23T21:10:33.097 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:10:33.097 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:10:33.097 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:10:33.097 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.35469", 2023-09-23T21:10:33.097 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/4041424623", 2023-09-23T21:10:33.097 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 197936 2023-09-23T21:10:33.097 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:33.098 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:10:33.098 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:33.098 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:10:33.098 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:01.418440+0000", 2023-09-23T21:10:33.098 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:10:33.098 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:33.098 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:33.098 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:10:33.098 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:01.418440+0000", 2023-09-23T21:10:33.098 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:10:33.098 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:33.098 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:33.098 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:10:33.099 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:01.418439+0000", 2023-09-23T21:10:33.099 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T21:10:33.099 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:33.099 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:33.099 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:10:33.099 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:01.418445+0000", 2023-09-23T21:10:33.099 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.3740000000000003e-06 2023-09-23T21:10:33.099 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:33.099 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:33.099 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:10:33.099 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:01.418458+0000", 2023-09-23T21:10:33.099 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.2938e-05 2023-09-23T21:10:33.100 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:33.100 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:33.100 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:10:33.100 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:01.418464+0000", 2023-09-23T21:10:33.100 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.0580000000000003e-06 2023-09-23T21:10:33.100 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:33.100 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:33.100 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:10:33.100 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:01.418507+0000", 2023-09-23T21:10:33.100 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.2752e-05 2023-09-23T21:10:33.100 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:33.100 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:33.100 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:10:33.101 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:01.418514+0000", 2023-09-23T21:10:33.101 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.6329999999999999e-06 2023-09-23T21:10:33.101 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:33.101 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:33.101 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:10:33.101 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:02.294859+0000", 2023-09-23T21:10:33.101 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.87634557300000004 2023-09-23T21:10:33.101 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:33.101 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:33.101 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:10:33.101 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:02.294965+0000", 2023-09-23T21:10:33.101 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00010594000000000001 2023-09-23T21:10:33.101 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:33.102 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:33.102 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 2,6", 2023-09-23T21:10:33.102 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:02.295386+0000", 2023-09-23T21:10:33.102 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000420908 2023-09-23T21:10:33.102 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:33.102 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:33.102 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:10:33.102 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:02.300182+0000", 2023-09-23T21:10:33.102 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0047958159999999996 2023-09-23T21:10:33.102 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:33.102 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:33.102 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:10:33.103 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:02.305430+0000", 2023-09-23T21:10:33.103 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0052482220000000003 2023-09-23T21:10:33.103 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:33.103 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:33.103 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:10:33.103 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:02.305463+0000", 2023-09-23T21:10:33.103 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.2694999999999998e-05 2023-09-23T21:10:33.103 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:33.103 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:33.103 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:10:33.103 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:02.305507+0000", 2023-09-23T21:10:33.103 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.4524999999999998e-05 2023-09-23T21:10:33.103 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:33.104 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:33.104 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:10:33.104 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:02.305559+0000", 2023-09-23T21:10:33.104 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.1663999999999999e-05 2023-09-23T21:10:33.104 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:10:33.104 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:10:33.104 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:10:33.104 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:33.104 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:33.104 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.35469.0:197976 8.4 8:262cb904:::benchmark_data_smithi026_301269_object87219:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e330)", 2023-09-23T21:10:33.104 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:05:01.473672+0000", 2023-09-23T21:10:33.104 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 331.55917690400003, 2023-09-23T21:10:33.105 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.83213024899999999, 2023-09-23T21:10:33.105 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:10:33.105 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:10:33.105 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:10:33.105 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.35469", 2023-09-23T21:10:33.105 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/4041424623", 2023-09-23T21:10:33.105 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 197976 2023-09-23T21:10:33.105 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:33.105 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:10:33.105 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:33.105 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:10:33.105 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:01.473672+0000", 2023-09-23T21:10:33.105 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:10:33.106 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:33.106 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:33.106 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:10:33.106 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:01.473672+0000", 2023-09-23T21:10:33.106 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:10:33.106 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:33.106 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:33.106 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:10:33.106 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:01.473672+0000", 2023-09-23T21:10:33.106 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999995 2023-09-23T21:10:33.106 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:33.106 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:33.107 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:10:33.107 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:01.473675+0000", 2023-09-23T21:10:33.107 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.0699999999999998e-06 2023-09-23T21:10:33.107 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:33.107 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:33.107 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:10:33.107 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:01.473682+0000", 2023-09-23T21:10:33.107 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.9450000000000004e-06 2023-09-23T21:10:33.107 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:33.107 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:33.107 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:10:33.107 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:01.473686+0000", 2023-09-23T21:10:33.107 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.493e-06 2023-09-23T21:10:33.108 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:33.108 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:33.108 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:10:33.108 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:01.473714+0000", 2023-09-23T21:10:33.108 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.7662999999999999e-05 2023-09-23T21:10:33.108 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:33.108 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:33.108 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:10:33.108 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:01.473723+0000", 2023-09-23T21:10:33.108 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.8410000000000006e-06 2023-09-23T21:10:33.108 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:33.108 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:33.108 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:10:33.109 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:02.297745+0000", 2023-09-23T21:10:33.109 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.82402238900000002 2023-09-23T21:10:33.109 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:33.109 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:33.109 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:10:33.109 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:02.297862+0000", 2023-09-23T21:10:33.109 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00011700400000000001 2023-09-23T21:10:33.109 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:33.109 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:33.109 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 2,6", 2023-09-23T21:10:33.109 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:02.298175+0000", 2023-09-23T21:10:33.109 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00031293199999999998 2023-09-23T21:10:33.109 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:33.110 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:33.110 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:10:33.110 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:02.305369+0000", 2023-09-23T21:10:33.110 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0071940939999999998 2023-09-23T21:10:33.110 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:33.110 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:33.110 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:10:33.110 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:02.305660+0000", 2023-09-23T21:10:33.110 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00029076599999999997 2023-09-23T21:10:33.110 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:33.110 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:33.110 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:10:33.110 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:02.305705+0000", 2023-09-23T21:10:33.111 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.5556999999999998e-05 2023-09-23T21:10:33.111 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:33.111 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:33.111 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:10:33.111 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:02.305752+0000", 2023-09-23T21:10:33.111 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.6660999999999999e-05 2023-09-23T21:10:33.111 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:33.111 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:33.111 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:10:33.111 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:02.305802+0000", 2023-09-23T21:10:33.111 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.0346000000000002e-05 2023-09-23T21:10:33.111 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:10:33.112 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:10:33.112 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:10:33.112 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:33.112 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:33.112 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.35469.0:197986 8.4 8:24c1b6a6:::benchmark_data_smithi026_301269_object87229:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e330)", 2023-09-23T21:10:33.112 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:05:01.484363+0000", 2023-09-23T21:10:33.112 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 331.54848582199998, 2023-09-23T21:10:33.112 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.82409744600000001, 2023-09-23T21:10:33.112 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:10:33.112 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:10:33.112 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:10:33.112 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.35469", 2023-09-23T21:10:33.112 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/4041424623", 2023-09-23T21:10:33.113 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 197986 2023-09-23T21:10:33.113 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:33.113 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:10:33.113 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:33.113 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:10:33.113 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:01.484363+0000", 2023-09-23T21:10:33.113 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:10:33.113 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:33.113 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:33.113 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:10:33.113 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:01.484363+0000", 2023-09-23T21:10:33.113 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:10:33.113 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:33.114 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:33.114 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:10:33.114 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:01.484362+0000", 2023-09-23T21:10:33.114 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.999999 2023-09-23T21:10:33.114 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:33.114 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:33.114 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:10:33.114 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:01.484367+0000", 2023-09-23T21:10:33.114 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.8369999999999996e-06 2023-09-23T21:10:33.114 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:33.114 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:33.114 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:10:33.115 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:01.484376+0000", 2023-09-23T21:10:33.115 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.7029999999999995e-06 2023-09-23T21:10:33.115 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:33.115 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:33.115 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:10:33.115 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:01.484381+0000", 2023-09-23T21:10:33.115 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.7709999999999997e-06 2023-09-23T21:10:33.115 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:33.115 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:33.115 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:10:33.115 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:01.484422+0000", 2023-09-23T21:10:33.115 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.1618e-05 2023-09-23T21:10:33.116 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:33.116 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:33.116 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:10:33.116 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:01.484433+0000", 2023-09-23T21:10:33.116 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.0735e-05 2023-09-23T21:10:33.116 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:33.116 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:33.116 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:10:33.116 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:02.300211+0000", 2023-09-23T21:10:33.116 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.81577823400000005 2023-09-23T21:10:33.116 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:33.116 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:33.116 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:10:33.117 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:02.300326+0000", 2023-09-23T21:10:33.117 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000115337 2023-09-23T21:10:33.117 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:33.117 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:33.117 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 2,6", 2023-09-23T21:10:33.117 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:02.300630+0000", 2023-09-23T21:10:33.117 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00030335399999999998 2023-09-23T21:10:33.117 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:33.117 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:33.117 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:10:33.117 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:02.305420+0000", 2023-09-23T21:10:33.117 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0047902329999999996 2023-09-23T21:10:33.118 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:33.118 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:33.118 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:10:33.118 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:02.305669+0000", 2023-09-23T21:10:33.118 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00024875299999999999 2023-09-23T21:10:33.118 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:33.118 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:33.118 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:10:33.118 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:02.308400+0000", 2023-09-23T21:10:33.118 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0027312730000000002 2023-09-23T21:10:33.118 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:33.118 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:33.119 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:10:33.119 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:02.308432+0000", 2023-09-23T21:10:33.119 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.1948e-05 2023-09-23T21:10:33.119 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:33.119 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:33.119 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:10:33.119 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:02.308461+0000", 2023-09-23T21:10:33.119 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.8651000000000001e-05 2023-09-23T21:10:33.119 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:10:33.119 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:10:33.119 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:10:33.119 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:33.120 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:33.120 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.35469.0:198190 8.19 8.47097259 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e331)", 2023-09-23T21:10:33.120 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:05:02.366655+0000", 2023-09-23T21:10:33.120 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 330.66619428899998, 2023-09-23T21:10:33.120 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.92870424399999996, 2023-09-23T21:10:33.120 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:10:33.120 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "reached pg", 2023-09-23T21:10:33.120 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:10:33.120 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.35469", 2023-09-23T21:10:33.120 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/4041424623", 2023-09-23T21:10:33.120 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 198190 2023-09-23T21:10:33.120 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:33.120 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:10:33.121 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:33.121 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:10:33.121 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:02.366655+0000", 2023-09-23T21:10:33.121 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:10:33.121 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:33.121 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:33.121 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:10:33.121 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:02.366655+0000", 2023-09-23T21:10:33.121 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:10:33.121 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:33.121 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:33.121 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:10:33.121 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:02.366654+0000", 2023-09-23T21:10:33.122 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.999999 2023-09-23T21:10:33.122 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:33.122 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:33.122 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:10:33.122 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:02.366659+0000", 2023-09-23T21:10:33.122 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.3970000000000002e-06 2023-09-23T21:10:33.122 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:33.122 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:33.122 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:10:33.122 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:02.366673+0000", 2023-09-23T21:10:33.122 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.3706000000000001e-05 2023-09-23T21:10:33.122 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:33.123 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:33.123 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:10:33.123 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:02.366696+0000", 2023-09-23T21:10:33.123 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.3054999999999999e-05 2023-09-23T21:10:33.123 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:33.123 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:33.123 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:10:33.123 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:02.366740+0000", 2023-09-23T21:10:33.123 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.4297e-05 2023-09-23T21:10:33.123 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:33.123 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:33.123 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:10:33.123 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:02.366748+0000", 2023-09-23T21:10:33.124 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.3529999999999995e-06 2023-09-23T21:10:33.124 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:33.124 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:33.124 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:10:33.124 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:03.295349+0000", 2023-09-23T21:10:33.124 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.928600855 2023-09-23T21:10:33.124 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:33.124 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:33.124 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:10:33.124 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:03.295359+0000", 2023-09-23T21:10:33.124 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.6670000000000008e-06 2023-09-23T21:10:33.124 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:10:33.125 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:10:33.125 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:10:33.125 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:33.125 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:33.125 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.35469.0:198191 8.19 8.9474e8d9 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e331)", 2023-09-23T21:10:33.125 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:05:02.366711+0000", 2023-09-23T21:10:33.125 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 330.66613828800001, 2023-09-23T21:10:33.125 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.92867729499999996, 2023-09-23T21:10:33.125 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:10:33.125 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "reached pg", 2023-09-23T21:10:33.125 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:10:33.125 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.35469", 2023-09-23T21:10:33.125 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/4041424623", 2023-09-23T21:10:33.126 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 198191 2023-09-23T21:10:33.126 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:33.126 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:10:33.126 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:33.126 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:10:33.126 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:02.366711+0000", 2023-09-23T21:10:33.126 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:10:33.126 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:33.126 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:33.126 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:10:33.126 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:02.366711+0000", 2023-09-23T21:10:33.127 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:10:33.127 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:33.127 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:33.127 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:10:33.127 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:02.366710+0000", 2023-09-23T21:10:33.127 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999995 2023-09-23T21:10:33.127 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:33.127 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:33.127 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:10:33.127 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:02.366713+0000", 2023-09-23T21:10:33.127 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.9160000000000001e-06 2023-09-23T21:10:33.127 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:33.127 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:33.128 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:10:33.128 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:02.366720+0000", 2023-09-23T21:10:33.128 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.9229999999999996e-06 2023-09-23T21:10:33.128 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:33.128 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:33.128 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:10:33.128 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:02.366725+0000", 2023-09-23T21:10:33.128 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.8110000000000002e-06 2023-09-23T21:10:33.128 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:33.128 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:33.128 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:10:33.128 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:02.368872+0000", 2023-09-23T21:10:33.129 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.002146677 2023-09-23T21:10:33.129 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:33.129 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:33.129 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:10:33.129 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:02.368878+0000", 2023-09-23T21:10:33.129 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.0619999999999997e-06 2023-09-23T21:10:33.129 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:33.129 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:33.129 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:10:33.129 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:03.295381+0000", 2023-09-23T21:10:33.129 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.926503352 2023-09-23T21:10:33.129 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:33.129 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:33.129 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:10:33.130 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:03.295388+0000", 2023-09-23T21:10:33.130 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.9929999999999998e-06 2023-09-23T21:10:33.130 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:10:33.130 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:10:33.130 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:10:33.130 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:33.130 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:33.130 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.35469.0:198201 8.19 8.fa337379 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e331)", 2023-09-23T21:10:33.130 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:05:02.369766+0000", 2023-09-23T21:10:33.130 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 330.66308270000002, 2023-09-23T21:10:33.130 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.92565394300000003, 2023-09-23T21:10:33.130 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:10:33.131 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "reached pg", 2023-09-23T21:10:33.131 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:10:33.131 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.35469", 2023-09-23T21:10:33.131 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/4041424623", 2023-09-23T21:10:33.131 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 198201 2023-09-23T21:10:33.131 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:33.131 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:10:33.131 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:33.131 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:10:33.131 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:02.369766+0000", 2023-09-23T21:10:33.131 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:10:33.131 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:33.131 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:33.132 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:10:33.132 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:02.369766+0000", 2023-09-23T21:10:33.132 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:10:33.132 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:33.132 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:33.132 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:10:33.132 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:02.369766+0000", 2023-09-23T21:10:33.132 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999995 2023-09-23T21:10:33.132 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:33.132 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:33.132 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:10:33.132 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:02.369768+0000", 2023-09-23T21:10:33.132 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.5320000000000001e-06 2023-09-23T21:10:33.133 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:33.133 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:33.133 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:10:33.133 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:02.369781+0000", 2023-09-23T21:10:33.133 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.2091e-05 2023-09-23T21:10:33.133 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:33.133 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:33.133 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:10:33.133 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:02.369785+0000", 2023-09-23T21:10:33.133 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.498e-06 2023-09-23T21:10:33.133 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:33.133 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:33.134 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:10:33.134 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:02.369826+0000", 2023-09-23T21:10:33.134 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.1179e-05 2023-09-23T21:10:33.134 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:33.134 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:33.134 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:10:33.134 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:02.369833+0000", 2023-09-23T21:10:33.134 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.6719999999999998e-06 2023-09-23T21:10:33.134 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:33.134 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:33.134 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:10:33.134 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:03.295412+0000", 2023-09-23T21:10:33.135 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.92557892600000002 2023-09-23T21:10:33.135 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:33.135 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:33.135 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:10:33.135 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:03.295420+0000", 2023-09-23T21:10:33.135 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.4579999999999993e-06 2023-09-23T21:10:33.135 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:10:33.135 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:10:33.135 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:10:33.135 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:33.135 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:33.135 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.35469.0:198229 8.19 8.92797f39 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e331)", 2023-09-23T21:10:33.135 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:05:02.420904+0000", 2023-09-23T21:10:33.136 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 330.61194455499998, 2023-09-23T21:10:33.136 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.87453987099999997, 2023-09-23T21:10:33.136 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:10:33.136 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "reached pg", 2023-09-23T21:10:33.136 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:10:33.136 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.35469", 2023-09-23T21:10:33.136 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/4041424623", 2023-09-23T21:10:33.136 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 198229 2023-09-23T21:10:33.136 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:33.136 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:10:33.136 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:33.136 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:10:33.136 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:02.420904+0000", 2023-09-23T21:10:33.137 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:10:33.137 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:33.137 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:33.137 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:10:33.137 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:02.420904+0000", 2023-09-23T21:10:33.137 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:10:33.137 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:33.137 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:33.137 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:10:33.137 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:02.420903+0000", 2023-09-23T21:10:33.137 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T21:10:33.137 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:33.137 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:33.138 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:10:33.138 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:02.420909+0000", 2023-09-23T21:10:33.138 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.3689999999999998e-06 2023-09-23T21:10:33.138 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:33.138 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:33.138 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:10:33.138 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:02.420921+0000", 2023-09-23T21:10:33.138 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.2771e-05 2023-09-23T21:10:33.138 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:33.138 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:33.138 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:10:33.138 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:02.420928+0000", 2023-09-23T21:10:33.138 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.0679999999999999e-06 2023-09-23T21:10:33.139 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:33.139 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:33.139 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:10:33.139 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:02.420973+0000", 2023-09-23T21:10:33.139 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.4675000000000002e-05 2023-09-23T21:10:33.139 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:33.139 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:33.139 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:10:33.139 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:02.420982+0000", 2023-09-23T21:10:33.139 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.4999999999999999e-06 2023-09-23T21:10:33.139 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:33.139 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:33.139 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:10:33.140 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:03.295437+0000", 2023-09-23T21:10:33.140 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.87445572000000005 2023-09-23T21:10:33.140 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:33.140 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:33.140 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:10:33.140 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:03.295444+0000", 2023-09-23T21:10:33.140 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.9739999999999996e-06 2023-09-23T21:10:33.140 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:10:33.140 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:10:33.140 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:10:33.140 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:33.140 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:33.140 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.35469.0:198232 8.19 8.bb8c9e39 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e331)", 2023-09-23T21:10:33.141 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:05:02.420941+0000", 2023-09-23T21:10:33.141 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 330.61190837100003, 2023-09-23T21:10:33.141 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.87452782500000004, 2023-09-23T21:10:33.141 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:10:33.141 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "reached pg", 2023-09-23T21:10:33.141 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:10:33.141 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.35469", 2023-09-23T21:10:33.141 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/4041424623", 2023-09-23T21:10:33.141 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 198232 2023-09-23T21:10:33.141 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:33.141 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:10:33.141 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:33.141 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:10:33.141 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:02.420941+0000", 2023-09-23T21:10:33.142 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:10:33.142 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:33.142 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:33.142 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:10:33.142 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:02.420941+0000", 2023-09-23T21:10:33.142 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:10:33.142 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:33.142 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:33.142 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:10:33.142 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:02.420940+0000", 2023-09-23T21:10:33.142 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999995 2023-09-23T21:10:33.142 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:33.143 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:33.143 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:10:33.143 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:02.420943+0000", 2023-09-23T21:10:33.143 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.5830000000000001e-06 2023-09-23T21:10:33.143 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:33.143 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:33.143 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:10:33.143 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:02.420948+0000", 2023-09-23T21:10:33.143 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.1760000000000001e-06 2023-09-23T21:10:33.143 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:33.143 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:33.143 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:10:33.143 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:02.420951+0000", 2023-09-23T21:10:33.144 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.3239999999999999e-06 2023-09-23T21:10:33.144 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:33.144 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:33.144 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:10:33.144 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:02.421010+0000", 2023-09-23T21:10:33.144 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.9104999999999997e-05 2023-09-23T21:10:33.144 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:33.144 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:33.144 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:10:33.144 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:02.421016+0000", 2023-09-23T21:10:33.144 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.4029999999999998e-06 2023-09-23T21:10:33.145 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:33.145 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:33.145 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:10:33.145 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:03.295461+0000", 2023-09-23T21:10:33.145 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.87444563500000005 2023-09-23T21:10:33.145 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:33.145 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:33.145 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:10:33.145 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:03.295468+0000", 2023-09-23T21:10:33.145 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.1029999999999996e-06 2023-09-23T21:10:33.145 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:10:33.145 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:10:33.146 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:10:33.146 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:33.146 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:33.146 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.35856.0:188867 9.e 9.6ba19f5e (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e395)", 2023-09-23T21:10:33.146 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:09:06.459905+0000", 2023-09-23T21:10:33.146 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 86.572944508999996, 2023-09-23T21:10:33.146 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.92984451499999998, 2023-09-23T21:10:33.146 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:10:33.146 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "reached pg", 2023-09-23T21:10:33.146 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:10:33.146 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.35856", 2023-09-23T21:10:33.146 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/4180396258", 2023-09-23T21:10:33.146 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 188867 2023-09-23T21:10:33.147 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:33.147 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:10:33.147 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:33.147 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:10:33.147 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:09:06.459905+0000", 2023-09-23T21:10:33.147 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:10:33.147 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:33.147 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:33.147 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:10:33.147 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:09:06.459905+0000", 2023-09-23T21:10:33.147 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:10:33.148 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:33.148 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:33.148 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:10:33.148 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:09:06.459903+0000", 2023-09-23T21:10:33.148 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T21:10:33.148 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:33.148 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:33.148 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:10:33.148 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:09:06.459911+0000", 2023-09-23T21:10:33.148 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.4109999999999999e-06 2023-09-23T21:10:33.148 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:33.148 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:33.148 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:10:33.149 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:09:06.459928+0000", 2023-09-23T21:10:33.149 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.7065e-05 2023-09-23T21:10:33.149 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:33.149 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:33.149 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:10:33.149 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:09:06.459934+0000", 2023-09-23T21:10:33.149 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.4180000000000002e-06 2023-09-23T21:10:33.149 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:33.149 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:33.149 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:10:33.149 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:09:06.459978+0000", 2023-09-23T21:10:33.149 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.4056e-05 2023-09-23T21:10:33.150 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:33.150 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:33.150 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:10:33.150 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:09:06.459987+0000", 2023-09-23T21:10:33.150 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.0659999999999993e-06 2023-09-23T21:10:33.150 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:33.150 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:33.150 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:10:33.150 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:09:07.389738+0000", 2023-09-23T21:10:33.150 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.92975045199999995 2023-09-23T21:10:33.150 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:33.150 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:33.151 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:10:33.151 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:09:07.389749+0000", 2023-09-23T21:10:33.151 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.1473e-05 2023-09-23T21:10:33.151 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:10:33.151 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:10:33.151 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:10:33.151 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:33.151 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:33.151 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.35856.0:188878 9.e 9.f8edd2e (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e395)", 2023-09-23T21:10:33.151 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:09:06.487137+0000", 2023-09-23T21:10:33.151 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 86.545712344999998, 2023-09-23T21:10:33.151 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.90264091099999999, 2023-09-23T21:10:33.152 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:10:33.152 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "reached pg", 2023-09-23T21:10:33.152 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:10:33.152 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.35856", 2023-09-23T21:10:33.152 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/4180396258", 2023-09-23T21:10:33.152 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 188878 2023-09-23T21:10:33.152 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:33.152 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:10:33.152 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:33.152 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:10:33.152 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:09:06.487137+0000", 2023-09-23T21:10:33.152 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:10:33.153 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:33.153 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:33.153 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:10:33.153 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:09:06.487137+0000", 2023-09-23T21:10:33.153 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:10:33.153 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:33.153 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:33.153 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:10:33.153 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:09:06.487136+0000", 2023-09-23T21:10:33.153 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.999999 2023-09-23T21:10:33.153 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:33.153 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:33.153 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:10:33.154 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:09:06.487142+0000", 2023-09-23T21:10:33.154 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.9789999999999998e-06 2023-09-23T21:10:33.154 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:33.154 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:33.154 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:10:33.154 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:09:06.487153+0000", 2023-09-23T21:10:33.154 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.1782e-05 2023-09-23T21:10:33.154 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:33.154 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:33.154 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:10:33.154 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:09:06.487158+0000", 2023-09-23T21:10:33.154 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.4109999999999998e-06 2023-09-23T21:10:33.155 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:33.155 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:33.155 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:10:33.155 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:09:06.487206+0000", 2023-09-23T21:10:33.155 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.7883999999999998e-05 2023-09-23T21:10:33.155 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:33.155 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:33.155 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:10:33.155 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:09:06.487214+0000", 2023-09-23T21:10:33.155 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.4400000000000005e-06 2023-09-23T21:10:33.155 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:33.155 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:33.156 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:10:33.156 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:09:07.389771+0000", 2023-09-23T21:10:33.156 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.90255646 2023-09-23T21:10:33.156 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:33.156 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:33.156 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:10:33.156 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:09:07.389778+0000", 2023-09-23T21:10:33.156 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.0870000000000001e-06 2023-09-23T21:10:33.156 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:10:33.156 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:10:33.156 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:10:33.156 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:33.157 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:33.157 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.35856.0:188898 9.e 9.64a26bde (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e395)", 2023-09-23T21:10:33.157 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:09:06.513632+0000", 2023-09-23T21:10:33.157 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 86.519216546999999, 2023-09-23T21:10:33.157 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.87616996599999997, 2023-09-23T21:10:33.157 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:10:33.157 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "reached pg", 2023-09-23T21:10:33.157 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:10:33.157 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.35856", 2023-09-23T21:10:33.157 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/4180396258", 2023-09-23T21:10:33.157 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 188898 2023-09-23T21:10:33.157 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:33.157 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:10:33.158 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:33.158 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:10:33.158 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:09:06.513632+0000", 2023-09-23T21:10:33.158 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:10:33.158 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:33.158 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:33.158 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:10:33.158 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:09:06.513632+0000", 2023-09-23T21:10:33.158 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:10:33.158 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:33.158 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:33.158 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:10:33.159 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:09:06.513631+0000", 2023-09-23T21:10:33.159 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999981 2023-09-23T21:10:33.159 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:33.159 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:33.159 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:10:33.159 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:09:06.513640+0000", 2023-09-23T21:10:33.159 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.5889999999999992e-06 2023-09-23T21:10:33.159 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:33.159 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:33.159 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:10:33.159 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:09:06.513662+0000", 2023-09-23T21:10:33.159 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.2014999999999998e-05 2023-09-23T21:10:33.159 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:33.160 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:33.160 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:10:33.160 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:09:06.513672+0000", 2023-09-23T21:10:33.160 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.5850000000000006e-06 2023-09-23T21:10:33.160 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:33.160 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:33.160 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:10:33.160 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:09:06.513725+0000", 2023-09-23T21:10:33.160 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.3087000000000002e-05 2023-09-23T21:10:33.160 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:33.160 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:33.160 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:10:33.161 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:09:06.513736+0000", 2023-09-23T21:10:33.161 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.1375e-05 2023-09-23T21:10:33.161 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:33.161 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:33.161 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:10:33.161 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:09:07.389796+0000", 2023-09-23T21:10:33.161 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.87605925200000001 2023-09-23T21:10:33.161 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:33.161 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:33.161 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:10:33.161 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:09:07.389802+0000", 2023-09-23T21:10:33.161 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.7669999999999998e-06 2023-09-23T21:10:33.161 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:10:33.162 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:10:33.162 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:10:33.162 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:33.162 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:33.162 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.35856.0:188916 9.e 9.d1066dbe (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e395)", 2023-09-23T21:10:33.162 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:09:06.545701+0000", 2023-09-23T21:10:33.162 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 86.487147574999995, 2023-09-23T21:10:33.162 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.84412522400000001, 2023-09-23T21:10:33.162 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:10:33.162 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "reached pg", 2023-09-23T21:10:33.162 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:10:33.162 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.35856", 2023-09-23T21:10:33.163 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/4180396258", 2023-09-23T21:10:33.163 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 188916 2023-09-23T21:10:33.163 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:33.163 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:10:33.163 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:33.163 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:10:33.163 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:09:06.545701+0000", 2023-09-23T21:10:33.163 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:10:33.163 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:33.163 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:33.163 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:10:33.163 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:09:06.545701+0000", 2023-09-23T21:10:33.164 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:10:33.164 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:33.164 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:33.164 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:10:33.164 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:09:06.545700+0000", 2023-09-23T21:10:33.164 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T21:10:33.164 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:33.164 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:33.164 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:10:33.164 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:09:06.545709+0000", 2023-09-23T21:10:33.164 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.9290000000000004e-06 2023-09-23T21:10:33.164 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:33.164 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:33.165 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:10:33.165 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:09:06.545728+0000", 2023-09-23T21:10:33.165 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.9199e-05 2023-09-23T21:10:33.165 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:33.165 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:33.165 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:10:33.165 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:09:06.545735+0000", 2023-09-23T21:10:33.165 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.6619999999999999e-06 2023-09-23T21:10:33.165 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:33.165 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:33.165 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:10:33.165 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:09:06.545800+0000", 2023-09-23T21:10:33.166 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.5129999999999995e-05 2023-09-23T21:10:33.166 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:33.166 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:33.166 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:10:33.166 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:09:06.545812+0000", 2023-09-23T21:10:33.166 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.238e-05 2023-09-23T21:10:33.166 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:33.166 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:33.166 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:10:33.166 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:09:07.389820+0000", 2023-09-23T21:10:33.166 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.84400768800000003 2023-09-23T21:10:33.166 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:33.167 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:33.167 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:10:33.167 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:09:07.389827+0000", 2023-09-23T21:10:33.167 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.7240000000000003e-06 2023-09-23T21:10:33.167 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:10:33.167 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:10:33.167 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:10:33.167 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:10:33.167 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:10:33.167 INFO:teuthology.orchestra.run.smithi070.stdout:} 2023-09-23T21:10:33.168 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:10:33.168 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:10:32 smithi070 ceph-mon[168282]: failed to encode map e429 with expected crc 2023-09-23T21:10:33.168 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:10:32 smithi070 ceph-mon[168282]: failed to encode map e429 with expected crc 2023-09-23T21:10:33.168 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:10:32 smithi070 ceph-mon[168282]: failed to encode map e429 with expected crc 2023-09-23T21:10:33.168 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:10:32 smithi070 ceph-mon[168282]: failed to encode map e429 with expected crc 2023-09-23T21:10:33.168 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:10:32 smithi070 ceph-mon[168282]: failed to encode map e429 with expected crc 2023-09-23T21:10:33.169 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:10:32 smithi070 ceph-mon[168282]: osdmap e430: 8 total, 8 up, 7 in 2023-09-23T21:10:33.169 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:10:32 smithi026 ceph-mon[179688]: failed to encode map e429 with expected crc 2023-09-23T21:10:33.169 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:10:32 smithi026 ceph-mon[179688]: failed to encode map e429 with expected crc 2023-09-23T21:10:33.169 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:10:32 smithi026 ceph-mon[179688]: failed to encode map e429 with expected crc 2023-09-23T21:10:33.169 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:10:32 smithi026 ceph-mon[179688]: failed to encode map e429 with expected crc 2023-09-23T21:10:33.169 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:10:32 smithi026 ceph-mon[179688]: failed to encode map e429 with expected crc 2023-09-23T21:10:33.169 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:10:32 smithi026 ceph-mon[179688]: osdmap e430: 8 total, 8 up, 7 in 2023-09-23T21:10:33.169 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:10:32 smithi026 ceph-mon[175151]: failed to encode map e429 with expected crc 2023-09-23T21:10:33.169 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:10:32 smithi026 ceph-mon[175151]: failed to encode map e429 with expected crc 2023-09-23T21:10:33.170 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:10:32 smithi026 ceph-mon[175151]: failed to encode map e429 with expected crc 2023-09-23T21:10:33.170 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:10:32 smithi026 ceph-mon[175151]: failed to encode map e429 with expected crc 2023-09-23T21:10:33.170 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:10:32 smithi026 ceph-mon[175151]: failed to encode map e429 with expected crc 2023-09-23T21:10:33.170 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:10:32 smithi026 ceph-mon[175151]: osdmap e430: 8 total, 8 up, 7 in 2023-09-23T21:10:33.170 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:10:33.171 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:10:33.271 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:10:33.301 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:10:33.430 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 11 16 12769 12753 72.4344 24.4375 0.0015854 0.0130702 2023-09-23T21:10:33.455 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:10:33.457 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:10:33.458 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:10:33.463 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_ops_in_flight 2023-09-23T21:10:33.509 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:10:33.559 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:10:33.588 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:10:33.627 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:10:33.630 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:10:33.631 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:10:33.686 INFO:teuthology.orchestra.run.smithi026.stderr:nodeep-scrub is unset 2023-09-23T21:10:33.731 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:10:33.787 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:10:33.837 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:10:33.840 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:10:33.840 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:10:33.941 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:10:33.970 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:10:33 smithi026 ceph-mon[175151]: failed to encode map e430 with expected crc 2023-09-23T21:10:33.970 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:10:33 smithi026 ceph-mon[175151]: failed to encode map e430 with expected crc 2023-09-23T21:10:33.970 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:10:33 smithi026 ceph-mon[175151]: failed to encode map e430 with expected crc 2023-09-23T21:10:33.970 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:10:33 smithi026 ceph-mon[175151]: failed to encode map e430 with expected crc 2023-09-23T21:10:33.970 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:10:33 smithi026 ceph-mon[175151]: failed to encode map e430 with expected crc 2023-09-23T21:10:33.970 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:10:33 smithi026 ceph-mon[175151]: pgmap v1112: 109 pgs: 1 active+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 4 peering, 100 active+clean; 699 MiB data, 5.4 GiB used, 620 GiB / 626 GiB avail; 64 MiB/s wr, 1.02k op/s; 1901/33582 objects degraded (5.661%); 14 MiB/s, 224 objects/s recovering 2023-09-23T21:10:33.970 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:10:33 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/2614716931' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:10:33.971 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:10:33 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:10:33.971 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:10:33 smithi026 ceph-mon[175151]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T21:10:33.971 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:10:33 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T21:10:33.971 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:10:33 smithi026 ceph-mon[175151]: osdmap e431: 8 total, 8 up, 7 in 2023-09-23T21:10:33.971 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:10:33 smithi026 ceph-mon[179688]: failed to encode map e430 with expected crc 2023-09-23T21:10:33.971 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:10:33 smithi026 ceph-mon[179688]: failed to encode map e430 with expected crc 2023-09-23T21:10:33.971 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:10:33 smithi026 ceph-mon[179688]: failed to encode map e430 with expected crc 2023-09-23T21:10:33.971 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:10:33 smithi026 ceph-mon[179688]: failed to encode map e430 with expected crc 2023-09-23T21:10:33.971 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:10:33 smithi026 ceph-mon[179688]: failed to encode map e430 with expected crc 2023-09-23T21:10:33.971 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:10:33 smithi026 ceph-mon[179688]: pgmap v1112: 109 pgs: 1 active+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 4 peering, 100 active+clean; 699 MiB data, 5.4 GiB used, 620 GiB / 626 GiB avail; 64 MiB/s wr, 1.02k op/s; 1901/33582 objects degraded (5.661%); 14 MiB/s, 224 objects/s recovering 2023-09-23T21:10:33.972 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:10:33 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/2614716931' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:10:33.972 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:10:33 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:10:33.972 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:10:33 smithi026 ceph-mon[179688]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T21:10:33.972 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:10:33 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T21:10:33.972 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:10:33 smithi026 ceph-mon[179688]: osdmap e431: 8 total, 8 up, 7 in 2023-09-23T21:10:33.972 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:10:34.088 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T21:10:34.109 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T21:10:34.112 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:10:34.112 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:10:34.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:10:33 smithi070 ceph-mon[168282]: failed to encode map e430 with expected crc 2023-09-23T21:10:34.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:10:33 smithi070 ceph-mon[168282]: failed to encode map e430 with expected crc 2023-09-23T21:10:34.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:10:33 smithi070 ceph-mon[168282]: failed to encode map e430 with expected crc 2023-09-23T21:10:34.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:10:33 smithi070 ceph-mon[168282]: failed to encode map e430 with expected crc 2023-09-23T21:10:34.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:10:33 smithi070 ceph-mon[168282]: failed to encode map e430 with expected crc 2023-09-23T21:10:34.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:10:33 smithi070 ceph-mon[168282]: pgmap v1112: 109 pgs: 1 active+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 4 peering, 100 active+clean; 699 MiB data, 5.4 GiB used, 620 GiB / 626 GiB avail; 64 MiB/s wr, 1.02k op/s; 1901/33582 objects degraded (5.661%); 14 MiB/s, 224 objects/s recovering 2023-09-23T21:10:34.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:10:33 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/2614716931' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:10:34.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:10:33 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:10:34.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:10:33 smithi070 ceph-mon[168282]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T21:10:34.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:10:33 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T21:10:34.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:10:33 smithi070 ceph-mon[168282]: osdmap e431: 8 total, 8 up, 7 in 2023-09-23T21:10:34.213 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:10:34.244 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:10:34.430 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 12 16 12939 12923 67.2848 10.625 0.00123707 0.0140212 2023-09-23T21:10:34.631 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:10:34.634 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:10:34.635 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:10:34.735 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:10:34.770 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0: {} 2023-09-23T21:10:34.803 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:10:34.821 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1: {} 2023-09-23T21:10:34.858 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2: {} 2023-09-23T21:10:34.861 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:10:34.866 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:10:34.866 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:10:34.905 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3: {} 2023-09-23T21:10:34.936 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4: {} 2023-09-23T21:10:34.958 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T21:10:34.967 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:10:34.977 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6: {} 2023-09-23T21:10:34.996 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T21:10:34.997 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:10:34.998 INFO:teuthology.orchestra.run.smithi026.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T21:10:34.998 INFO:teuthology.orchestra.run.smithi026.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T21:10:34.998 INFO:teuthology.orchestra.run.smithi026.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T21:10:34.998 INFO:teuthology.orchestra.run.smithi026.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T21:10:34.998 INFO:teuthology.orchestra.run.smithi026.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T21:10:34.999 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T21:10:34.999 INFO:teuthology.orchestra.run.smithi026.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T21:10:34.999 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T21:10:35.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:10:34 smithi070 ceph-mon[168282]: failed to encode map e431 with expected crc 2023-09-23T21:10:35.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:10:34 smithi070 ceph-mon[168282]: failed to encode map e431 with expected crc 2023-09-23T21:10:35.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:10:34 smithi070 ceph-mon[168282]: failed to encode map e431 with expected crc 2023-09-23T21:10:35.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:10:34 smithi070 ceph-mon[168282]: failed to encode map e431 with expected crc 2023-09-23T21:10:35.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:10:34 smithi070 ceph-mon[168282]: failed to encode map e431 with expected crc 2023-09-23T21:10:35.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:10:34 smithi070 ceph-mon[168282]: osdmap e432: 8 total, 8 up, 7 in 2023-09-23T21:10:35.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:10:34 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/2918701105' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:10:35.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:10:34 smithi026 ceph-mon[179688]: failed to encode map e431 with expected crc 2023-09-23T21:10:35.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:10:34 smithi026 ceph-mon[179688]: failed to encode map e431 with expected crc 2023-09-23T21:10:35.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:10:34 smithi026 ceph-mon[179688]: failed to encode map e431 with expected crc 2023-09-23T21:10:35.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:10:34 smithi026 ceph-mon[179688]: failed to encode map e431 with expected crc 2023-09-23T21:10:35.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:10:34 smithi026 ceph-mon[179688]: failed to encode map e431 with expected crc 2023-09-23T21:10:35.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:10:34 smithi026 ceph-mon[179688]: osdmap e432: 8 total, 8 up, 7 in 2023-09-23T21:10:35.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:10:34 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/2918701105' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:10:35.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:10:34 smithi026 ceph-mon[175151]: failed to encode map e431 with expected crc 2023-09-23T21:10:35.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:10:34 smithi026 ceph-mon[175151]: failed to encode map e431 with expected crc 2023-09-23T21:10:35.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:10:34 smithi026 ceph-mon[175151]: failed to encode map e431 with expected crc 2023-09-23T21:10:35.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:10:34 smithi026 ceph-mon[175151]: failed to encode map e431 with expected crc 2023-09-23T21:10:35.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:10:34 smithi026 ceph-mon[175151]: failed to encode map e431 with expected crc 2023-09-23T21:10:35.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:10:34 smithi026 ceph-mon[175151]: osdmap e432: 8 total, 8 up, 7 in 2023-09-23T21:10:35.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:10:34 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/2918701105' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:10:35.401 INFO:teuthology.orchestra.run.smithi070.stdout:ERROR: (22) Invalid argument 2023-09-23T21:10:35.401 INFO:teuthology.orchestra.run.smithi070.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T21:10:35.430 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 13 16 13237 13221 63.5424 18.625 0.00189162 0.0150313 2023-09-23T21:10:35.442 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:10:35.446 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:10:35.446 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:10:35.547 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:10:35.577 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:10:35.687 INFO:teuthology.orchestra.run.smithi026.stderr:set osd.3 primary-affinity to 0.64032 (8419642) 2023-09-23T21:10:35.746 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:10:35.749 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:10:35.749 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:10:35.756 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_blocked_ops 2023-09-23T21:10:35.850 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:10:35.878 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:10:35.938 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:10:35.941 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:10:35.942 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:10:36.042 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:10:36.070 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:10:35 smithi026 ceph-mon[175151]: failed to encode map e431 with expected crc 2023-09-23T21:10:36.070 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:10:35 smithi026 ceph-mon[175151]: failed to encode map e432 with expected crc 2023-09-23T21:10:36.071 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:10:35 smithi026 ceph-mon[175151]: failed to encode map e432 with expected crc 2023-09-23T21:10:36.071 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:10:35 smithi026 ceph-mon[175151]: failed to encode map e432 with expected crc 2023-09-23T21:10:36.071 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:10:35 smithi026 ceph-mon[175151]: failed to encode map e432 with expected crc 2023-09-23T21:10:36.071 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:10:35 smithi026 ceph-mon[175151]: failed to encode map e432 with expected crc 2023-09-23T21:10:36.071 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:10:35 smithi026 ceph-mon[175151]: failed to encode map e432 with expected crc 2023-09-23T21:10:36.071 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:10:35 smithi026 ceph-mon[175151]: pgmap v1115: 109 pgs: 2 active+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 1 peering, 104 active+clean; 778 MiB data, 5.7 GiB used, 620 GiB / 626 GiB avail; 72 MiB/s wr, 1.16k op/s; 959/37407 objects degraded (2.564%); 29 MiB/s, 459 objects/s recovering 2023-09-23T21:10:36.071 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:10:35 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/3341576304' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 3, "weight": 0.6403203422890724}]: dispatch 2023-09-23T21:10:36.071 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:10:35 smithi026 ceph-mon[175151]: Health check update: Degraded data redundancy: 1901/33582 objects degraded (5.661%), 4 pgs degraded (PG_DEGRADED) 2023-09-23T21:10:36.071 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:10:35 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/3341576304' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 3, "weight": 0.6403203422890724}]': finished 2023-09-23T21:10:36.071 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:10:35 smithi026 ceph-mon[175151]: osdmap e433: 8 total, 8 up, 7 in 2023-09-23T21:10:36.072 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:10:35 smithi026 ceph-mon[179688]: failed to encode map e431 with expected crc 2023-09-23T21:10:36.072 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:10:35 smithi026 ceph-mon[179688]: failed to encode map e432 with expected crc 2023-09-23T21:10:36.072 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:10:35 smithi026 ceph-mon[179688]: failed to encode map e432 with expected crc 2023-09-23T21:10:36.072 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:10:35 smithi026 ceph-mon[179688]: failed to encode map e432 with expected crc 2023-09-23T21:10:36.072 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:10:35 smithi026 ceph-mon[179688]: failed to encode map e432 with expected crc 2023-09-23T21:10:36.072 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:10:35 smithi026 ceph-mon[179688]: failed to encode map e432 with expected crc 2023-09-23T21:10:36.072 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:10:35 smithi026 ceph-mon[179688]: failed to encode map e432 with expected crc 2023-09-23T21:10:36.072 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:10:35 smithi026 ceph-mon[179688]: pgmap v1115: 109 pgs: 2 active+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 1 peering, 104 active+clean; 778 MiB data, 5.7 GiB used, 620 GiB / 626 GiB avail; 72 MiB/s wr, 1.16k op/s; 959/37407 objects degraded (2.564%); 29 MiB/s, 459 objects/s recovering 2023-09-23T21:10:36.072 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:10:35 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/3341576304' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 3, "weight": 0.6403203422890724}]: dispatch 2023-09-23T21:10:36.072 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:10:35 smithi026 ceph-mon[179688]: Health check update: Degraded data redundancy: 1901/33582 objects degraded (5.661%), 4 pgs degraded (PG_DEGRADED) 2023-09-23T21:10:36.072 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:10:35 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/3341576304' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 3, "weight": 0.6403203422890724}]': finished 2023-09-23T21:10:36.072 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:10:35 smithi026 ceph-mon[179688]: osdmap e433: 8 total, 8 up, 7 in 2023-09-23T21:10:36.073 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:10:36.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:10:35 smithi070 ceph-mon[168282]: failed to encode map e431 with expected crc 2023-09-23T21:10:36.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:10:35 smithi070 ceph-mon[168282]: failed to encode map e432 with expected crc 2023-09-23T21:10:36.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:10:35 smithi070 ceph-mon[168282]: failed to encode map e432 with expected crc 2023-09-23T21:10:36.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:10:35 smithi070 ceph-mon[168282]: failed to encode map e432 with expected crc 2023-09-23T21:10:36.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:10:35 smithi070 ceph-mon[168282]: failed to encode map e432 with expected crc 2023-09-23T21:10:36.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:10:35 smithi070 ceph-mon[168282]: failed to encode map e432 with expected crc 2023-09-23T21:10:36.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:10:35 smithi070 ceph-mon[168282]: failed to encode map e432 with expected crc 2023-09-23T21:10:36.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:10:35 smithi070 ceph-mon[168282]: pgmap v1115: 109 pgs: 2 active+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 1 peering, 104 active+clean; 778 MiB data, 5.7 GiB used, 620 GiB / 626 GiB avail; 72 MiB/s wr, 1.16k op/s; 959/37407 objects degraded (2.564%); 29 MiB/s, 459 objects/s recovering 2023-09-23T21:10:36.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:10:35 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/3341576304' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 3, "weight": 0.6403203422890724}]: dispatch 2023-09-23T21:10:36.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:10:35 smithi070 ceph-mon[168282]: Health check update: Degraded data redundancy: 1901/33582 objects degraded (5.661%), 4 pgs degraded (PG_DEGRADED) 2023-09-23T21:10:36.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:10:35 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/3341576304' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 3, "weight": 0.6403203422890724}]': finished 2023-09-23T21:10:36.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:10:35 smithi070 ceph-mon[168282]: osdmap e433: 8 total, 8 up, 7 in 2023-09-23T21:10:36.184 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T21:10:36.430 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 14 16 13539 13523 60.3524 18.875 0.00127714 0.0158624 2023-09-23T21:10:36.433 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T21:10:37.007 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:10:36 smithi070 ceph-mon[168282]: failed to encode map e433 with expected crc 2023-09-23T21:10:37.007 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:10:36 smithi070 ceph-mon[168282]: failed to encode map e433 with expected crc 2023-09-23T21:10:37.007 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:10:36 smithi070 ceph-mon[168282]: failed to encode map e433 with expected crc 2023-09-23T21:10:37.007 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:10:36 smithi070 ceph-mon[168282]: failed to encode map e433 with expected crc 2023-09-23T21:10:37.007 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:10:36 smithi070 ceph-mon[168282]: failed to encode map e433 with expected crc 2023-09-23T21:10:37.007 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:10:36 smithi070 ceph-mon[168282]: failed to encode map e433 with expected crc 2023-09-23T21:10:37.007 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:10:36 smithi070 ceph-mon[168282]: failed to encode map e433 with expected crc 2023-09-23T21:10:37.008 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:10:36 smithi070 ceph-mon[168282]: osdmap e434: 8 total, 8 up, 7 in 2023-09-23T21:10:37.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:10:36 smithi026 ceph-mon[175151]: failed to encode map e433 with expected crc 2023-09-23T21:10:37.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:10:36 smithi026 ceph-mon[175151]: failed to encode map e433 with expected crc 2023-09-23T21:10:37.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:10:36 smithi026 ceph-mon[175151]: failed to encode map e433 with expected crc 2023-09-23T21:10:37.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:10:36 smithi026 ceph-mon[175151]: failed to encode map e433 with expected crc 2023-09-23T21:10:37.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:10:36 smithi026 ceph-mon[175151]: failed to encode map e433 with expected crc 2023-09-23T21:10:37.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:10:36 smithi026 ceph-mon[175151]: failed to encode map e433 with expected crc 2023-09-23T21:10:37.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:10:36 smithi026 ceph-mon[175151]: failed to encode map e433 with expected crc 2023-09-23T21:10:37.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:10:36 smithi026 ceph-mon[175151]: osdmap e434: 8 total, 8 up, 7 in 2023-09-23T21:10:37.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:10:36 smithi026 ceph-mon[179688]: failed to encode map e433 with expected crc 2023-09-23T21:10:37.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:10:36 smithi026 ceph-mon[179688]: failed to encode map e433 with expected crc 2023-09-23T21:10:37.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:10:36 smithi026 ceph-mon[179688]: failed to encode map e433 with expected crc 2023-09-23T21:10:37.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:10:36 smithi026 ceph-mon[179688]: failed to encode map e433 with expected crc 2023-09-23T21:10:37.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:10:36 smithi026 ceph-mon[179688]: failed to encode map e433 with expected crc 2023-09-23T21:10:37.134 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:10:36 smithi026 ceph-mon[179688]: failed to encode map e433 with expected crc 2023-09-23T21:10:37.134 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:10:36 smithi026 ceph-mon[179688]: failed to encode map e433 with expected crc 2023-09-23T21:10:37.134 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:10:36 smithi026 ceph-mon[179688]: osdmap e434: 8 total, 8 up, 7 in 2023-09-23T21:10:37.430 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 15 16 15071 15055 62.7111 95.75 0.00700528 0.0159394 2023-09-23T21:10:37.817 INFO:teuthology.orchestra.run.smithi070.stdout:ERROR: (22) Invalid argument 2023-09-23T21:10:37.817 INFO:teuthology.orchestra.run.smithi070.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T21:10:38.214 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:10:38 smithi070 ceph-mon[168282]: failed to encode map e434 with expected crc 2023-09-23T21:10:38.214 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:10:38 smithi070 ceph-mon[168282]: failed to encode map e434 with expected crc 2023-09-23T21:10:38.214 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:10:38 smithi070 ceph-mon[168282]: failed to encode map e434 with expected crc 2023-09-23T21:10:38.214 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:10:38 smithi070 ceph-mon[168282]: failed to encode map e434 with expected crc 2023-09-23T21:10:38.214 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:10:38 smithi070 ceph-mon[168282]: failed to encode map e434 with expected crc 2023-09-23T21:10:38.215 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:10:38 smithi070 ceph-mon[168282]: pgmap v1118: 109 pgs: 2 active+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 4 peering, 101 active+clean; 802 MiB data, 5.7 GiB used, 620 GiB / 626 GiB avail; 21 KiB/s rd, 26 MiB/s wr, 427 op/s; 959/38517 objects degraded (2.490%); 15 MiB/s, 235 objects/s recovering 2023-09-23T21:10:38.215 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:10:38 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:10:38.215 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:10:38 smithi070 ceph-mon[168282]: Health check failed: Reduced data availability: 4 pgs inactive, 4 pgs peering (PG_AVAILABILITY) 2023-09-23T21:10:38.216 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_historic_ops 2023-09-23T21:10:38.347 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:10:38.351 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:10:38 smithi026 ceph-mon[179688]: failed to encode map e434 with expected crc 2023-09-23T21:10:38.351 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:10:38 smithi026 ceph-mon[179688]: failed to encode map e434 with expected crc 2023-09-23T21:10:38.351 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:10:38 smithi026 ceph-mon[179688]: failed to encode map e434 with expected crc 2023-09-23T21:10:38.351 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:10:38 smithi026 ceph-mon[179688]: failed to encode map e434 with expected crc 2023-09-23T21:10:38.352 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:10:38 smithi026 ceph-mon[179688]: failed to encode map e434 with expected crc 2023-09-23T21:10:38.352 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:10:38 smithi026 ceph-mon[179688]: pgmap v1118: 109 pgs: 2 active+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 4 peering, 101 active+clean; 802 MiB data, 5.7 GiB used, 620 GiB / 626 GiB avail; 21 KiB/s rd, 26 MiB/s wr, 427 op/s; 959/38517 objects degraded (2.490%); 15 MiB/s, 235 objects/s recovering 2023-09-23T21:10:38.352 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:10:38 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:10:38.352 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:10:38 smithi026 ceph-mon[179688]: Health check failed: Reduced data availability: 4 pgs inactive, 4 pgs peering (PG_AVAILABILITY) 2023-09-23T21:10:38.352 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:10:38 smithi026 ceph-mon[175151]: failed to encode map e434 with expected crc 2023-09-23T21:10:38.352 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:10:38 smithi026 ceph-mon[175151]: failed to encode map e434 with expected crc 2023-09-23T21:10:38.352 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:10:38 smithi026 ceph-mon[175151]: failed to encode map e434 with expected crc 2023-09-23T21:10:38.352 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:10:38 smithi026 ceph-mon[175151]: failed to encode map e434 with expected crc 2023-09-23T21:10:38.352 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:10:38 smithi026 ceph-mon[175151]: failed to encode map e434 with expected crc 2023-09-23T21:10:38.352 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:10:38 smithi026 ceph-mon[175151]: pgmap v1118: 109 pgs: 2 active+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 4 peering, 101 active+clean; 802 MiB data, 5.7 GiB used, 620 GiB / 626 GiB avail; 21 KiB/s rd, 26 MiB/s wr, 427 op/s; 959/38517 objects degraded (2.490%); 15 MiB/s, 235 objects/s recovering 2023-09-23T21:10:38.353 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:10:38 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:10:38.353 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:10:38 smithi026 ceph-mon[175151]: Health check failed: Reduced data availability: 4 pgs inactive, 4 pgs peering (PG_AVAILABILITY) 2023-09-23T21:10:38.354 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:10:38.355 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:10:38.430 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 16 16 17033 17017 66.4538 122.625 0.0120803 0.0150408 2023-09-23T21:10:38.456 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:10:38.496 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:10:38.535 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:10:38.538 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:10:38.538 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:10:38.639 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:10:38.702 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:10:38.895 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T21:10:39.071 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:10:39.075 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:10:39.075 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:10:39.176 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:10:39.246 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:10:39.293 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:10:39.303 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:10:39.307 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:10:39.307 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:10:39.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:10:39 smithi070 ceph-mon[168282]: 1.9 scrub starts 2023-09-23T21:10:39.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:10:39 smithi070 ceph-mon[168282]: 1.9 scrub ok 2023-09-23T21:10:39.381 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:10:39 smithi026 ceph-mon[179688]: 1.9 scrub starts 2023-09-23T21:10:39.381 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:10:39 smithi026 ceph-mon[179688]: 1.9 scrub ok 2023-09-23T21:10:39.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:10:39 smithi026 ceph-mon[175151]: 1.9 scrub starts 2023-09-23T21:10:39.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:10:39 smithi026 ceph-mon[175151]: 1.9 scrub ok 2023-09-23T21:10:39.398 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T21:10:39.408 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:10:39.434 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 17 16 19001 18985 69.7627 123 0.00844377 0.014327 2023-09-23T21:10:39.498 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:10:39.567 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:10:39.578 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:10:39.579 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:10:39.680 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:10:39.739 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:10:39.881 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 21:10:39 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[164029]: ::ffff:172.21.15.70 - - [23/Sep/2023:21:10:39] "GET /metrics HTTP/1.1" 200 36386 "" "Prometheus/2.43.0" 2023-09-23T21:10:40.097 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:10:40.292 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:10:40 smithi026 ceph-mon[175151]: pgmap v1119: 109 pgs: 4 peering, 105 active+clean; 923 MiB data, 6.0 GiB used, 620 GiB / 626 GiB avail; 16 KiB/s rd, 33 MiB/s wr, 532 op/s; 11 MiB/s, 181 objects/s recovering 2023-09-23T21:10:40.292 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:10:40 smithi026 ceph-mon[175151]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 959/38517 objects degraded (2.490%), 2 pgs degraded) 2023-09-23T21:10:40.292 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:10:40 smithi026 ceph-mon[179688]: pgmap v1119: 109 pgs: 4 peering, 105 active+clean; 923 MiB data, 6.0 GiB used, 620 GiB / 626 GiB avail; 16 KiB/s rd, 33 MiB/s wr, 532 op/s; 11 MiB/s, 181 objects/s recovering 2023-09-23T21:10:40.292 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:10:40 smithi026 ceph-mon[179688]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 959/38517 objects degraded (2.490%), 2 pgs degraded) 2023-09-23T21:10:40.437 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 18 16 20238 20222 70.1738 77.3125 0.0111756 0.0142415 2023-09-23T21:10:40.470 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:10:40 smithi070 ceph-mon[168282]: pgmap v1119: 109 pgs: 4 peering, 105 active+clean; 923 MiB data, 6.0 GiB used, 620 GiB / 626 GiB avail; 16 KiB/s rd, 33 MiB/s wr, 532 op/s; 11 MiB/s, 181 objects/s recovering 2023-09-23T21:10:40.471 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:10:40 smithi070 ceph-mon[168282]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 959/38517 objects degraded (2.490%), 2 pgs degraded) 2023-09-23T21:10:41.066 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:10:41.070 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:10:41.070 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:10:41.171 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:10:41.234 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:10:41.275 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:10:41.276 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:10:41 smithi026 ceph-mon[175151]: 1.e scrub starts 2023-09-23T21:10:41.276 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:10:41 smithi026 ceph-mon[175151]: 1.e scrub ok 2023-09-23T21:10:41.277 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:10:41 smithi026 ceph-mon[179688]: 1.e scrub starts 2023-09-23T21:10:41.277 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:10:41 smithi026 ceph-mon[179688]: 1.e scrub ok 2023-09-23T21:10:41.278 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:10:41.279 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:10:41.367 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:10:41 smithi070 ceph-mon[168282]: 1.e scrub starts 2023-09-23T21:10:41.367 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:10:41 smithi070 ceph-mon[168282]: 1.e scrub ok 2023-09-23T21:10:41.379 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:10:41.438 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 19 16 21898 21882 71.9363 103.75 0.00419141 0.0138946 2023-09-23T21:10:41.455 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:10:41.512 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:10:41.516 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:10:41.517 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:10:41.617 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:10:41.631 INFO:teuthology.orchestra.run.smithi070.stdout:ERROR: (22) Invalid argument 2023-09-23T21:10:41.632 INFO:teuthology.orchestra.run.smithi070.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T21:10:41.652 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:10:41.970 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:10:41.975 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:10:41.975 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:10:41.998 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_ops_in_flight 2023-09-23T21:10:42.076 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:10:42.077 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0: {} 2023-09-23T21:10:42.125 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1: {} 2023-09-23T21:10:42.153 INFO:teuthology.orchestra.run.smithi026.stderr:noscrub is set 2023-09-23T21:10:42.165 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:10:42.173 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2: {} 2023-09-23T21:10:42.212 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3: {} 2023-09-23T21:10:42.232 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:10:42.240 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:10:42.240 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:10:42.254 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4: {} 2023-09-23T21:10:42.296 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T21:10:42.334 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6: {} 2023-09-23T21:10:42.342 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:10:42.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:10:42 smithi070 ceph-mon[168282]: pgmap v1120: 109 pgs: 4 peering, 105 active+clean; 966 MiB data, 6.1 GiB used, 620 GiB / 626 GiB avail; 13 KiB/s rd, 30 MiB/s wr, 487 op/s; 9.6 MiB/s, 152 objects/s recovering 2023-09-23T21:10:42.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:10:42 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/2379765625' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T21:10:42.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:10:42 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T21:10:42.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:10:42 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/179569527' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:10:42.372 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T21:10:42.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:10:42 smithi026 ceph-mon[175151]: pgmap v1120: 109 pgs: 4 peering, 105 active+clean; 966 MiB data, 6.1 GiB used, 620 GiB / 626 GiB avail; 13 KiB/s rd, 30 MiB/s wr, 487 op/s; 9.6 MiB/s, 152 objects/s recovering 2023-09-23T21:10:42.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:10:42 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/2379765625' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T21:10:42.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:10:42 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T21:10:42.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:10:42 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/179569527' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:10:42.383 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:10:42 smithi026 ceph-mon[179688]: pgmap v1120: 109 pgs: 4 peering, 105 active+clean; 966 MiB data, 6.1 GiB used, 620 GiB / 626 GiB avail; 13 KiB/s rd, 30 MiB/s wr, 487 op/s; 9.6 MiB/s, 152 objects/s recovering 2023-09-23T21:10:42.383 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:10:42 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/2379765625' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T21:10:42.383 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:10:42 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T21:10:42.383 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:10:42 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/179569527' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:10:42.387 INFO:teuthology.orchestra.run.smithi026.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T21:10:42.388 INFO:teuthology.orchestra.run.smithi026.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T21:10:42.388 INFO:teuthology.orchestra.run.smithi026.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T21:10:42.388 INFO:teuthology.orchestra.run.smithi026.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T21:10:42.388 INFO:teuthology.orchestra.run.smithi026.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T21:10:42.388 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T21:10:42.388 INFO:teuthology.orchestra.run.smithi026.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T21:10:42.388 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T21:10:42.408 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:10:42.437 INFO:tasks.radosbench.radosbench.0.smithi026.stdout:2023-09-23T21:10:42.438574+0000 min lat: 0.00120238 max lat: 2.42702 avg lat: 0.0136359 2023-09-23T21:10:42.438 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T21:10:42.438 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 20 16 23485 23469 73.2977 99.1875 0.00681065 0.0136359 2023-09-23T21:10:42.744 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T21:10:43.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:10:43 smithi026 ceph-mon[179688]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:10:43.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:10:43 smithi026 ceph-mon[179688]: failed to encode map e435 with expected crc 2023-09-23T21:10:43.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:10:43 smithi026 ceph-mon[179688]: failed to encode map e435 with expected crc 2023-09-23T21:10:43.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:10:43 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T21:10:43.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:10:43 smithi026 ceph-mon[179688]: osdmap e435: 8 total, 8 up, 7 in 2023-09-23T21:10:43.383 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:10:43 smithi026 ceph-mon[179688]: failed to encode map e435 with expected crc 2023-09-23T21:10:43.385 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:10:43 smithi026 ceph-mon[175151]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:10:43.386 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:10:43 smithi026 ceph-mon[175151]: failed to encode map e435 with expected crc 2023-09-23T21:10:43.386 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:10:43 smithi026 ceph-mon[175151]: failed to encode map e435 with expected crc 2023-09-23T21:10:43.386 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:10:43 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T21:10:43.386 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:10:43 smithi026 ceph-mon[175151]: osdmap e435: 8 total, 8 up, 7 in 2023-09-23T21:10:43.386 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:10:43 smithi026 ceph-mon[175151]: failed to encode map e435 with expected crc 2023-09-23T21:10:43.438 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 21 16 25312 25296 75.2433 114.188 0.00491687 0.0132846 2023-09-23T21:10:43.507 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:10:43 smithi070 ceph-mon[168282]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:10:43.507 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:10:43 smithi070 ceph-mon[168282]: failed to encode map e435 with expected crc 2023-09-23T21:10:43.507 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:10:43 smithi070 ceph-mon[168282]: failed to encode map e435 with expected crc 2023-09-23T21:10:43.507 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:10:43 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T21:10:43.507 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:10:43 smithi070 ceph-mon[168282]: osdmap e435: 8 total, 8 up, 7 in 2023-09-23T21:10:43.507 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:10:43 smithi070 ceph-mon[168282]: failed to encode map e435 with expected crc 2023-09-23T21:10:44.132 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 7] out_osds: [6] dead_osds: [] live_osds: [0, 2, 3, 4, 5, 7, 6, 1] 2023-09-23T21:10:44.133 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T21:10:44.133 INFO:tasks.thrashosds.thrasher:Growing pool unique_pool_4 2023-09-23T21:10:44.133 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T21:10:44.384 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:10:44 smithi026 ceph-mon[175151]: pgmap v1122: 109 pgs: 109 active+clean; 1.3 GiB data, 7.2 GiB used, 619 GiB / 626 GiB avail; 74 MiB/s wr, 1.19k op/s; 8.2 MiB/s, 131 objects/s recovering 2023-09-23T21:10:44.384 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:10:44 smithi026 ceph-mon[175151]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 4 pgs inactive, 4 pgs peering) 2023-09-23T21:10:44.384 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:10:44 smithi026 ceph-mon[179688]: pgmap v1122: 109 pgs: 109 active+clean; 1.3 GiB data, 7.2 GiB used, 619 GiB / 626 GiB avail; 74 MiB/s wr, 1.19k op/s; 8.2 MiB/s, 131 objects/s recovering 2023-09-23T21:10:44.384 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:10:44 smithi026 ceph-mon[179688]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 4 pgs inactive, 4 pgs peering) 2023-09-23T21:10:44.438 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 22 16 26880 26864 76.2768 98 0.0105134 0.0131035 2023-09-23T21:10:44.599 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:10:44 smithi070 ceph-mon[168282]: pgmap v1122: 109 pgs: 109 active+clean; 1.3 GiB data, 7.2 GiB used, 619 GiB / 626 GiB avail; 74 MiB/s wr, 1.19k op/s; 8.2 MiB/s, 131 objects/s recovering 2023-09-23T21:10:44.599 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:10:44 smithi070 ceph-mon[168282]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 4 pgs inactive, 4 pgs peering) 2023-09-23T21:10:44.843 INFO:teuthology.orchestra.run.smithi070.stdout:{ 2023-09-23T21:10:44.843 INFO:teuthology.orchestra.run.smithi070.stdout: "ops": [], 2023-09-23T21:10:44.843 INFO:teuthology.orchestra.run.smithi070.stdout: "num_ops": 0 2023-09-23T21:10:44.843 INFO:teuthology.orchestra.run.smithi070.stdout:} 2023-09-23T21:10:45.162 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_blocked_ops 2023-09-23T21:10:45.250 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:10:45.260 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:10:45.261 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:10:45.361 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:10:45.424 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:10:45.438 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 23 16 28539 28523 77.4676 103.688 0.0150779 0.0129005 2023-09-23T21:10:45.934 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:10:45.939 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:10:45.939 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:10:45.961 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T21:10:46.004 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T21:10:46.040 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:10:46.169 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:10:46.217 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:10:46.247 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T21:10:46.255 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:10:46.256 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:10:46.256 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:10:46 smithi026 ceph-mon[179688]: pgmap v1123: 109 pgs: 109 active+clean; 1.4 GiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 82 MiB/s wr, 1.31k op/s; 7.2 MiB/s, 115 objects/s recovering 2023-09-23T21:10:46.256 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:10:46 smithi026 ceph-mon[175151]: pgmap v1123: 109 pgs: 109 active+clean; 1.4 GiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 82 MiB/s wr, 1.31k op/s; 7.2 MiB/s, 115 objects/s recovering 2023-09-23T21:10:46.357 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:10:46.434 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:10:46.438 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 24 16 30016 30000 78.0855 92.3125 0.00963931 0.0128008 2023-09-23T21:10:46.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:10:46 smithi070 ceph-mon[168282]: pgmap v1123: 109 pgs: 109 active+clean; 1.4 GiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 82 MiB/s wr, 1.31k op/s; 7.2 MiB/s, 115 objects/s recovering 2023-09-23T21:10:47.109 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:10:47.112 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:10:47.113 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:10:47.214 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:10:47.248 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:10:47.308 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:10:47.312 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:10:47.312 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:10:47.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:10:47 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:10:47.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:10:47 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:10:47.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:10:47 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:10:47.383 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:10:47 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:10:47.383 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:10:47 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:10:47.383 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:10:47 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:10:47.413 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:10:47.439 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 25 16 31518 31502 78.7121 93.875 0.00993875 0.0126973 2023-09-23T21:10:47.448 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:10:47.592 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:10:47.593 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:10:47 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:10:47.593 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:10:47 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:10:47.593 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:10:47 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:10:47.594 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:10:47.594 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:10:47.695 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:10:47.759 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:10:47.853 INFO:teuthology.orchestra.run.smithi070.stdout:{ 2023-09-23T21:10:47.853 INFO:teuthology.orchestra.run.smithi070.stdout: "ops": [], 2023-09-23T21:10:47.853 INFO:teuthology.orchestra.run.smithi070.stdout: "complaint_time": 30, 2023-09-23T21:10:47.853 INFO:teuthology.orchestra.run.smithi070.stdout: "num_blocked_ops": 0 2023-09-23T21:10:47.853 INFO:teuthology.orchestra.run.smithi070.stdout:} 2023-09-23T21:10:47.991 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T21:10:48.257 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_historic_ops 2023-09-23T21:10:48.383 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:10:48 smithi026 ceph-mon[179688]: pgmap v1124: 109 pgs: 109 active+clean; 1.6 GiB data, 7.9 GiB used, 618 GiB / 626 GiB avail; 80 MiB/s wr, 1.28k op/s; 6.0 MiB/s, 95 objects/s recovering 2023-09-23T21:10:48.383 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:10:48 smithi026 ceph-mon[175151]: pgmap v1124: 109 pgs: 109 active+clean; 1.6 GiB data, 7.9 GiB used, 618 GiB / 626 GiB avail; 80 MiB/s wr, 1.28k op/s; 6.0 MiB/s, 95 objects/s recovering 2023-09-23T21:10:48.440 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 26 16 32803 32787 78.7725 80.3125 0.00571056 0.0126883 2023-09-23T21:10:48.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:10:48 smithi070 ceph-mon[168282]: pgmap v1124: 109 pgs: 109 active+clean; 1.6 GiB data, 7.9 GiB used, 618 GiB / 626 GiB avail; 80 MiB/s wr, 1.28k op/s; 6.0 MiB/s, 95 objects/s recovering 2023-09-23T21:10:48.887 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:10:48.894 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:10:48.895 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:10:48.972 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T21:10:48.995 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:10:48.997 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:10:49.029 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:10:49.096 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:10:49.099 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:10:49.099 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:10:49.200 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:10:49.262 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:10:49.341 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:10:49.442 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 27 16 33989 33973 78.5944 74.125 0.00722024 0.0127166 2023-09-23T21:10:49.446 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:10:49.453 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:10:49.454 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:10:49.555 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:10:49.588 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:10:49.815 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 21:10:49 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[164029]: ::ffff:172.21.15.70 - - [23/Sep/2023:21:10:49] "GET /metrics HTTP/1.1" 200 36387 "" "Prometheus/2.43.0" 2023-09-23T21:10:50.149 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:10:50.152 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:10:50.152 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:10:50.253 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:10:50.331 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:10:50.403 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:10:50.409 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:10:50.410 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:10:50.442 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 28 16 35182 35166 78.4502 74.5625 0.0104285 0.012738 2023-09-23T21:10:50.510 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:10:50.537 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:10:50 smithi070 ceph-mon[168282]: pgmap v1125: 109 pgs: 109 active+clean; 1.9 GiB data, 8.9 GiB used, 617 GiB / 626 GiB avail; 99 MiB/s wr, 1.58k op/s 2023-09-23T21:10:50.574 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:10:50 smithi026 ceph-mon[175151]: pgmap v1125: 109 pgs: 109 active+clean; 1.9 GiB data, 8.9 GiB used, 617 GiB / 626 GiB avail; 99 MiB/s wr, 1.58k op/s 2023-09-23T21:10:50.575 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:10:50 smithi026 ceph-mon[179688]: pgmap v1125: 109 pgs: 109 active+clean; 1.9 GiB data, 8.9 GiB used, 617 GiB / 626 GiB avail; 99 MiB/s wr, 1.58k op/s 2023-09-23T21:10:50.581 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:10:50.806 INFO:teuthology.orchestra.run.smithi070.stdout:{ 2023-09-23T21:10:50.806 INFO:teuthology.orchestra.run.smithi070.stdout: "size": 20, 2023-09-23T21:10:50.806 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 600, 2023-09-23T21:10:50.806 INFO:teuthology.orchestra.run.smithi070.stdout: "ops": [ 2023-09-23T21:10:50.806 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:50.806 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.35856.0:52751 9.3 9:c51114d0:::benchmark_data_smithi026_342732_object52750:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e354)", 2023-09-23T21:10:50.806 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:06:24.396732+0000", 2023-09-23T21:10:50.806 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 266.40553289299999, 2023-09-23T21:10:50.806 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.006593885, 2023-09-23T21:10:50.806 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:10:50.807 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "reached pg", 2023-09-23T21:10:50.807 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:10:50.807 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.35856", 2023-09-23T21:10:50.807 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/4180396258", 2023-09-23T21:10:50.807 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 52751 2023-09-23T21:10:50.807 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:50.807 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:10:50.807 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:50.807 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:10:50.807 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:24.396732+0000", 2023-09-23T21:10:50.807 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:10:50.807 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:50.807 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:50.808 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:10:50.808 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:24.396732+0000", 2023-09-23T21:10:50.808 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:10:50.808 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:50.808 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:50.808 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:10:50.808 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:24.396731+0000", 2023-09-23T21:10:50.808 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T21:10:50.808 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:50.808 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:50.808 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:10:50.808 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:24.396868+0000", 2023-09-23T21:10:50.809 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00013730400000000001 2023-09-23T21:10:50.809 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:50.809 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:50.809 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:10:50.809 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:24.396883+0000", 2023-09-23T21:10:50.809 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.5221999999999999e-05 2023-09-23T21:10:50.809 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:50.809 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:50.809 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:10:50.809 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:24.396891+0000", 2023-09-23T21:10:50.809 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.3869999999999997e-06 2023-09-23T21:10:50.809 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:50.809 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:50.809 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:10:50.810 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:24.397030+0000", 2023-09-23T21:10:50.810 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00013967 2023-09-23T21:10:50.810 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:50.810 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:50.810 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:10:50.810 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:24.397040+0000", 2023-09-23T21:10:50.810 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.9150000000000008e-06 2023-09-23T21:10:50.810 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:50.810 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:50.810 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:10:50.810 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:25.402653+0000", 2023-09-23T21:10:50.810 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.00561239 2023-09-23T21:10:50.811 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:50.811 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:50.811 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:10:50.811 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:25.403326+0000", 2023-09-23T21:10:50.811 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00067348199999999999 2023-09-23T21:10:50.811 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:10:50.811 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:10:50.811 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:10:50.811 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:50.811 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:50.811 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.35856.0:52765 9.3 9:cd654ac2:::benchmark_data_smithi026_342732_object52764:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e354)", 2023-09-23T21:10:50.811 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:06:24.396908+0000", 2023-09-23T21:10:50.812 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 266.405356772, 2023-09-23T21:10:50.812 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.0245688509999999, 2023-09-23T21:10:50.812 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:10:50.812 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:10:50.812 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:10:50.812 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.35856", 2023-09-23T21:10:50.812 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/4180396258", 2023-09-23T21:10:50.812 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 52765 2023-09-23T21:10:50.812 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:50.812 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:10:50.812 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:50.812 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:10:50.813 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:24.396908+0000", 2023-09-23T21:10:50.813 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:10:50.813 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:50.813 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:50.813 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:10:50.813 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:24.396908+0000", 2023-09-23T21:10:50.813 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:10:50.813 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:50.813 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:50.813 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:10:50.813 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:24.396908+0000", 2023-09-23T21:10:50.813 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999995 2023-09-23T21:10:50.813 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:50.814 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:50.814 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:10:50.814 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:24.396943+0000", 2023-09-23T21:10:50.814 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.5349999999999999e-05 2023-09-23T21:10:50.814 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:50.814 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:50.814 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:10:50.814 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:24.396952+0000", 2023-09-23T21:10:50.814 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.6649999999999992e-06 2023-09-23T21:10:50.814 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:50.814 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:50.815 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:10:50.815 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:24.396957+0000", 2023-09-23T21:10:50.815 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.3410000000000002e-06 2023-09-23T21:10:50.815 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:50.815 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:50.815 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:10:50.815 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:24.397054+0000", 2023-09-23T21:10:50.815 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.7220999999999996e-05 2023-09-23T21:10:50.815 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:50.815 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:50.815 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:10:50.815 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:24.397060+0000", 2023-09-23T21:10:50.815 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.4020000000000001e-06 2023-09-23T21:10:50.816 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:50.816 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:50.816 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:10:50.816 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:25.403373+0000", 2023-09-23T21:10:50.816 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.006313697 2023-09-23T21:10:50.816 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:50.816 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:50.816 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:10:50.816 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:25.403624+0000", 2023-09-23T21:10:50.816 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00025100200000000001 2023-09-23T21:10:50.816 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:50.816 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:50.817 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 1,3", 2023-09-23T21:10:50.817 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:25.403787+0000", 2023-09-23T21:10:50.817 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00016209899999999999 2023-09-23T21:10:50.817 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:50.817 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:50.817 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:10:50.817 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:25.409004+0000", 2023-09-23T21:10:50.817 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0052175640000000001 2023-09-23T21:10:50.817 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:50.817 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:50.817 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:10:50.817 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:25.421372+0000", 2023-09-23T21:10:50.817 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.012367507999999999 2023-09-23T21:10:50.818 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:50.818 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:50.818 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:10:50.818 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:25.421418+0000", 2023-09-23T21:10:50.818 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.5871000000000002e-05 2023-09-23T21:10:50.818 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:50.818 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:50.818 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:10:50.818 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:25.421453+0000", 2023-09-23T21:10:50.818 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.5199e-05 2023-09-23T21:10:50.818 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:50.818 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:50.819 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:10:50.819 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:25.421477+0000", 2023-09-23T21:10:50.819 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.4539e-05 2023-09-23T21:10:50.819 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:10:50.819 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:10:50.819 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:10:50.819 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:50.819 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:50.819 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.35856.0:52801 9.3 9:cf43f7eb:::benchmark_data_smithi026_342732_object52800:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e354)", 2023-09-23T21:10:50.819 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:06:24.416107+0000", 2023-09-23T21:10:50.819 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 266.38615801700001, 2023-09-23T21:10:50.819 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.0054936219999999, 2023-09-23T21:10:50.819 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:10:50.819 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:10:50.820 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:10:50.820 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.35856", 2023-09-23T21:10:50.820 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/4180396258", 2023-09-23T21:10:50.820 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 52801 2023-09-23T21:10:50.820 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:50.820 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:10:50.820 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:50.820 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:10:50.820 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:24.416107+0000", 2023-09-23T21:10:50.820 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:10:50.820 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:50.820 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:50.820 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:10:50.821 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:24.416107+0000", 2023-09-23T21:10:50.821 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:10:50.821 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:50.821 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:50.821 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:10:50.821 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:24.416106+0000", 2023-09-23T21:10:50.821 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T21:10:50.821 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:50.821 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:50.821 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:10:50.821 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:24.416150+0000", 2023-09-23T21:10:50.821 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.4604999999999997e-05 2023-09-23T21:10:50.822 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:50.822 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:50.822 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:10:50.822 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:24.416167+0000", 2023-09-23T21:10:50.822 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.6693000000000001e-05 2023-09-23T21:10:50.822 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:50.822 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:50.822 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:10:50.822 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:24.416174+0000", 2023-09-23T21:10:50.822 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.7120000000000003e-06 2023-09-23T21:10:50.822 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:50.822 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:50.822 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:10:50.823 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:24.416243+0000", 2023-09-23T21:10:50.823 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.9830000000000001e-05 2023-09-23T21:10:50.823 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:50.823 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:50.823 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:10:50.823 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:24.416253+0000", 2023-09-23T21:10:50.823 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.02e-06 2023-09-23T21:10:50.823 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:50.823 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:50.823 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:10:50.823 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:25.405162+0000", 2023-09-23T21:10:50.823 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.98890962100000002 2023-09-23T21:10:50.824 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:50.824 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:50.824 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:10:50.824 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:25.405288+0000", 2023-09-23T21:10:50.824 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00012563099999999999 2023-09-23T21:10:50.824 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:50.824 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:50.824 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 1,3", 2023-09-23T21:10:50.824 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:25.405448+0000", 2023-09-23T21:10:50.824 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000159872 2023-09-23T21:10:50.824 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:50.824 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:50.825 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:10:50.825 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:25.411143+0000", 2023-09-23T21:10:50.825 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0056956359999999996 2023-09-23T21:10:50.825 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:50.825 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:50.825 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:10:50.825 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:25.421523+0000", 2023-09-23T21:10:50.825 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.010379767999999999 2023-09-23T21:10:50.825 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:50.825 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:50.825 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:10:50.825 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:25.421555+0000", 2023-09-23T21:10:50.826 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.2403000000000001e-05 2023-09-23T21:10:50.826 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:50.826 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:50.826 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:10:50.826 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:25.421581+0000", 2023-09-23T21:10:50.826 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.5843e-05 2023-09-23T21:10:50.826 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:50.826 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:50.826 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:10:50.826 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:25.421601+0000", 2023-09-23T21:10:50.826 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.9510000000000001e-05 2023-09-23T21:10:50.826 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:10:50.826 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:10:50.827 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:10:50.827 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:50.827 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:50.827 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.35856.0:52825 9.3 9:c99b89ee:::benchmark_data_smithi026_342732_object52824:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e354)", 2023-09-23T21:10:50.827 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:06:24.432929+0000", 2023-09-23T21:10:50.827 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 266.369335702, 2023-09-23T21:10:50.827 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.98885604299999996, 2023-09-23T21:10:50.827 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:10:50.827 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:10:50.827 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:10:50.827 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.35856", 2023-09-23T21:10:50.827 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/4180396258", 2023-09-23T21:10:50.827 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 52825 2023-09-23T21:10:50.828 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:50.828 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:10:50.828 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:50.828 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:10:50.828 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:24.432929+0000", 2023-09-23T21:10:50.828 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:10:50.828 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:50.828 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:50.828 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:10:50.828 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:24.432929+0000", 2023-09-23T21:10:50.828 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:10:50.828 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:50.828 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:50.829 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:10:50.829 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:24.432928+0000", 2023-09-23T21:10:50.829 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T21:10:50.829 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:50.829 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:50.829 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:10:50.829 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:24.432975+0000", 2023-09-23T21:10:50.829 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.7441999999999997e-05 2023-09-23T21:10:50.829 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:50.829 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:50.829 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:10:50.829 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:24.432987+0000", 2023-09-23T21:10:50.829 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.1712999999999999e-05 2023-09-23T21:10:50.830 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:50.830 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:50.830 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:10:50.830 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:24.432993+0000", 2023-09-23T21:10:50.830 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.7640000000000002e-06 2023-09-23T21:10:50.830 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:50.830 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:50.830 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:10:50.830 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:24.433060+0000", 2023-09-23T21:10:50.830 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.6691000000000002e-05 2023-09-23T21:10:50.830 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:50.830 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:50.830 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:10:50.831 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:24.433071+0000", 2023-09-23T21:10:50.831 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.1163e-05 2023-09-23T21:10:50.831 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:50.831 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:50.831 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:10:50.831 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:25.406642+0000", 2023-09-23T21:10:50.831 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.97357167 2023-09-23T21:10:50.831 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:50.831 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:50.831 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:10:50.831 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:25.406738+0000", 2023-09-23T21:10:50.831 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.5882000000000001e-05 2023-09-23T21:10:50.831 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:50.832 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:50.832 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 1,3", 2023-09-23T21:10:50.832 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:25.406824+0000", 2023-09-23T21:10:50.832 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.5877999999999993e-05 2023-09-23T21:10:50.832 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:50.832 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:50.832 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:10:50.832 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:25.411177+0000", 2023-09-23T21:10:50.832 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0043524490000000004 2023-09-23T21:10:50.832 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:50.832 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:50.832 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:10:50.833 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:25.421631+0000", 2023-09-23T21:10:50.833 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.010454234 2023-09-23T21:10:50.833 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:50.833 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:50.833 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:10:50.833 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:25.421724+0000", 2023-09-23T21:10:50.833 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.3381000000000006e-05 2023-09-23T21:10:50.833 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:50.833 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:50.833 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:10:50.833 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:25.421766+0000", 2023-09-23T21:10:50.833 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.1347000000000002e-05 2023-09-23T21:10:50.833 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:50.834 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:50.834 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:10:50.834 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:25.421786+0000", 2023-09-23T21:10:50.834 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.9942e-05 2023-09-23T21:10:50.834 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:10:50.834 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:10:50.834 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:10:50.834 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:50.834 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:50.834 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.35856.0:52839 9.3 9:c25308eb:::benchmark_data_smithi026_342732_object52838:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e354)", 2023-09-23T21:10:50.834 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:06:24.442455+0000", 2023-09-23T21:10:50.834 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 266.35981024, 2023-09-23T21:10:50.835 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.979453766, 2023-09-23T21:10:50.835 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:10:50.835 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:10:50.835 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:10:50.835 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.35856", 2023-09-23T21:10:50.835 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/4180396258", 2023-09-23T21:10:50.835 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 52839 2023-09-23T21:10:50.835 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:50.835 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:10:50.835 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:50.835 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:10:50.835 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:24.442455+0000", 2023-09-23T21:10:50.835 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:10:50.836 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:50.836 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:50.836 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:10:50.836 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:24.442455+0000", 2023-09-23T21:10:50.836 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:10:50.836 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:50.836 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:50.836 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:10:50.836 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:24.442453+0000", 2023-09-23T21:10:50.836 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T21:10:50.836 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:50.836 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:50.837 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:10:50.837 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:24.442566+0000", 2023-09-23T21:10:50.837 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00011271400000000001 2023-09-23T21:10:50.837 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:50.837 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:50.837 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:10:50.837 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:24.442582+0000", 2023-09-23T21:10:50.837 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.579e-05 2023-09-23T21:10:50.837 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:50.837 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:50.837 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:10:50.837 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:24.442589+0000", 2023-09-23T21:10:50.837 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.1559999999999998e-06 2023-09-23T21:10:50.838 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:50.838 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:50.838 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:10:50.838 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:24.442645+0000", 2023-09-23T21:10:50.838 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.6020999999999998e-05 2023-09-23T21:10:50.838 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:50.838 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:50.838 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:10:50.838 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:24.442652+0000", 2023-09-23T21:10:50.838 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.6120000000000004e-06 2023-09-23T21:10:50.838 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:50.838 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:50.838 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:10:50.839 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:25.408020+0000", 2023-09-23T21:10:50.839 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.96536787999999996 2023-09-23T21:10:50.839 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:50.839 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:50.839 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:10:50.839 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:25.408105+0000", 2023-09-23T21:10:50.839 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.5282000000000001e-05 2023-09-23T21:10:50.839 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:50.839 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:50.839 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 1,3", 2023-09-23T21:10:50.839 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:25.408218+0000", 2023-09-23T21:10:50.839 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000112796 2023-09-23T21:10:50.839 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:50.840 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:50.840 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:10:50.840 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:25.413318+0000", 2023-09-23T21:10:50.840 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0051002160000000003 2023-09-23T21:10:50.840 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:50.840 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:50.840 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:10:50.840 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:25.421648+0000", 2023-09-23T21:10:50.840 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0083301109999999994 2023-09-23T21:10:50.840 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:50.840 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:50.840 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:10:50.840 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:25.421828+0000", 2023-09-23T21:10:50.841 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00017987299999999999 2023-09-23T21:10:50.841 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:50.841 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:50.841 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:10:50.841 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:25.421878+0000", 2023-09-23T21:10:50.841 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.0591999999999999e-05 2023-09-23T21:10:50.841 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:50.841 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:50.841 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:10:50.841 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:25.421909+0000", 2023-09-23T21:10:50.841 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.0280000000000001e-05 2023-09-23T21:10:50.841 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:10:50.842 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:10:50.842 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:10:50.842 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:50.842 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:50.842 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.35856.0:52856 9.3 9:cea642a7:::benchmark_data_smithi026_342732_object52855:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e354)", 2023-09-23T21:10:50.842 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:06:24.452334+0000", 2023-09-23T21:10:50.842 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 266.34993105699999, 2023-09-23T21:10:50.842 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.96970934900000005, 2023-09-23T21:10:50.842 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:10:50.842 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:10:50.842 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:10:50.842 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.35856", 2023-09-23T21:10:50.843 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/4180396258", 2023-09-23T21:10:50.843 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 52856 2023-09-23T21:10:50.843 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:50.843 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:10:50.843 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:50.843 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:10:50.843 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:24.452334+0000", 2023-09-23T21:10:50.843 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:10:50.843 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:50.843 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:50.843 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:10:50.843 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:24.452334+0000", 2023-09-23T21:10:50.843 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:10:50.844 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:50.844 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:50.844 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:10:50.844 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:24.452333+0000", 2023-09-23T21:10:50.844 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T21:10:50.844 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:50.844 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:50.844 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:10:50.844 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:24.452378+0000", 2023-09-23T21:10:50.844 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.5096000000000003e-05 2023-09-23T21:10:50.844 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:50.844 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:50.845 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:10:50.845 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:24.452394+0000", 2023-09-23T21:10:50.845 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.5888000000000001e-05 2023-09-23T21:10:50.845 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:50.845 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:50.845 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:10:50.845 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:24.452400+0000", 2023-09-23T21:10:50.845 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.6980000000000001e-06 2023-09-23T21:10:50.845 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:50.845 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:50.845 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:10:50.845 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:24.452459+0000", 2023-09-23T21:10:50.846 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.8594999999999997e-05 2023-09-23T21:10:50.846 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:50.846 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:50.846 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:10:50.846 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:24.452468+0000", 2023-09-23T21:10:50.846 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.6929999999999996e-06 2023-09-23T21:10:50.846 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:50.846 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:50.846 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:10:50.846 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:25.409045+0000", 2023-09-23T21:10:50.846 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.95657733099999998 2023-09-23T21:10:50.846 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:50.846 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:50.847 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:10:50.847 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:25.409143+0000", 2023-09-23T21:10:50.847 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.8317000000000001e-05 2023-09-23T21:10:50.847 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:50.847 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:50.847 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 1,3", 2023-09-23T21:10:50.847 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:25.409239+0000", 2023-09-23T21:10:50.847 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.6280000000000007e-05 2023-09-23T21:10:50.847 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:50.847 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:50.847 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:10:50.847 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:25.413327+0000", 2023-09-23T21:10:50.847 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0040874220000000003 2023-09-23T21:10:50.848 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:50.848 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:50.848 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:10:50.848 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:25.421696+0000", 2023-09-23T21:10:50.848 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0083695439999999996 2023-09-23T21:10:50.848 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:50.848 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:50.848 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:10:50.848 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:25.421994+0000", 2023-09-23T21:10:50.848 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00029776299999999999 2023-09-23T21:10:50.848 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:50.848 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:50.849 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:10:50.849 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:25.422025+0000", 2023-09-23T21:10:50.849 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.0880000000000002e-05 2023-09-23T21:10:50.849 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:50.849 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:50.849 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:10:50.849 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:25.422043+0000", 2023-09-23T21:10:50.849 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.8410000000000002e-05 2023-09-23T21:10:50.849 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:10:50.849 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:10:50.849 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:10:50.849 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:50.849 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:50.850 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.35856.0:52876 9.3 9:c116084f:::benchmark_data_smithi026_342732_object52875:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e354)", 2023-09-23T21:10:50.850 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:06:24.468836+0000", 2023-09-23T21:10:50.850 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 266.33342950899998, 2023-09-23T21:10:50.850 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.95333874799999996, 2023-09-23T21:10:50.850 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:10:50.850 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:10:50.850 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:10:50.850 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.35856", 2023-09-23T21:10:50.850 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/4180396258", 2023-09-23T21:10:50.850 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 52876 2023-09-23T21:10:50.850 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:50.850 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:10:50.851 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:50.851 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:10:50.851 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:24.468836+0000", 2023-09-23T21:10:50.851 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:10:50.851 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:50.851 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:50.851 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:10:50.851 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:24.468836+0000", 2023-09-23T21:10:50.851 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:10:50.851 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:50.851 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:50.851 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:10:50.851 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:24.468834+0000", 2023-09-23T21:10:50.852 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T21:10:50.852 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:50.852 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:50.852 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:10:50.852 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:24.468883+0000", 2023-09-23T21:10:50.852 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.8093000000000002e-05 2023-09-23T21:10:50.852 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:50.852 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:50.852 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:10:50.852 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:24.468895+0000", 2023-09-23T21:10:50.852 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.2425999999999999e-05 2023-09-23T21:10:50.852 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:50.853 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:50.853 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:10:50.853 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:24.468902+0000", 2023-09-23T21:10:50.853 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.6529999999999997e-06 2023-09-23T21:10:50.853 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:50.853 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:50.853 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:10:50.853 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:24.468973+0000", 2023-09-23T21:10:50.853 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.1608999999999998e-05 2023-09-23T21:10:50.853 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:50.853 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:50.853 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:10:50.854 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:24.468986+0000", 2023-09-23T21:10:50.854 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.225e-05 2023-09-23T21:10:50.854 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:50.854 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:50.854 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:10:50.854 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:25.410042+0000", 2023-09-23T21:10:50.854 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.94105615099999995 2023-09-23T21:10:50.854 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:50.854 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:50.854 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:10:50.854 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:25.410150+0000", 2023-09-23T21:10:50.855 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000108099 2023-09-23T21:10:50.855 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:50.855 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:50.855 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 1,3", 2023-09-23T21:10:50.855 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:25.410307+0000", 2023-09-23T21:10:50.855 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00015735 2023-09-23T21:10:50.855 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:50.855 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:50.855 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:10:50.855 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:25.415575+0000", 2023-09-23T21:10:50.855 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0052674810000000001 2023-09-23T21:10:50.855 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:50.855 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:50.856 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:10:50.856 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:25.421961+0000", 2023-09-23T21:10:50.856 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0063868579999999996 2023-09-23T21:10:50.856 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:50.856 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:50.856 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:10:50.856 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:25.422089+0000", 2023-09-23T21:10:50.856 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00012751799999999999 2023-09-23T21:10:50.856 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:50.856 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:50.856 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:10:50.856 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:25.422138+0000", 2023-09-23T21:10:50.856 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.8755000000000003e-05 2023-09-23T21:10:50.857 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:50.857 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:50.857 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:10:50.857 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:25.422174+0000", 2023-09-23T21:10:50.857 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.6712999999999999e-05 2023-09-23T21:10:50.857 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:10:50.857 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:10:50.857 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:10:50.857 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:50.857 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:50.857 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.35856.0:52877 9.3 9:c2be6f32:::benchmark_data_smithi026_342732_object52876:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e354)", 2023-09-23T21:10:50.857 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:06:24.474477+0000", 2023-09-23T21:10:50.857 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 266.32778817100001, 2023-09-23T21:10:50.858 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.94913964100000003, 2023-09-23T21:10:50.858 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:10:50.858 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:10:50.858 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:10:50.858 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.35856", 2023-09-23T21:10:50.858 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/4180396258", 2023-09-23T21:10:50.858 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 52877 2023-09-23T21:10:50.858 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:50.858 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:10:50.858 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:50.858 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:10:50.858 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:24.474477+0000", 2023-09-23T21:10:50.858 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:10:50.859 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:50.859 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:50.859 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:10:50.859 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:24.474477+0000", 2023-09-23T21:10:50.859 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:10:50.859 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:50.859 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:50.859 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:10:50.859 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:24.474476+0000", 2023-09-23T21:10:50.859 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.999999 2023-09-23T21:10:50.859 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:50.859 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:50.859 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:10:50.860 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:24.474528+0000", 2023-09-23T21:10:50.860 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.1582999999999997e-05 2023-09-23T21:10:50.860 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:50.860 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:50.860 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:10:50.860 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:24.474539+0000", 2023-09-23T21:10:50.860 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.1895e-05 2023-09-23T21:10:50.860 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:50.860 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:50.860 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:10:50.860 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:24.474546+0000", 2023-09-23T21:10:50.860 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.0700000000000001e-06 2023-09-23T21:10:50.860 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:50.861 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:50.861 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:10:50.861 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:24.474606+0000", 2023-09-23T21:10:50.861 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.9349999999999999e-05 2023-09-23T21:10:50.861 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:50.861 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:50.861 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:10:50.861 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:24.474614+0000", 2023-09-23T21:10:50.861 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.4889999999999995e-06 2023-09-23T21:10:50.861 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:50.861 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:50.861 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:10:50.861 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:25.411215+0000", 2023-09-23T21:10:50.861 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.93660084300000002 2023-09-23T21:10:50.862 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:50.862 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:50.862 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:10:50.862 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:25.411317+0000", 2023-09-23T21:10:50.862 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000102122 2023-09-23T21:10:50.862 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:50.862 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:50.862 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 1,3", 2023-09-23T21:10:50.862 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:25.411414+0000", 2023-09-23T21:10:50.862 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.6571999999999996e-05 2023-09-23T21:10:50.862 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:50.862 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:50.862 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:10:50.862 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:25.415584+0000", 2023-09-23T21:10:50.863 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0041698890000000004 2023-09-23T21:10:50.863 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:50.863 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:50.863 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:10:50.863 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:25.422211+0000", 2023-09-23T21:10:50.863 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0066269459999999999 2023-09-23T21:10:50.863 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:50.863 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:50.863 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:10:50.863 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:25.423526+0000", 2023-09-23T21:10:50.863 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0013153139999999999 2023-09-23T21:10:50.863 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:50.863 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:50.864 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:10:50.864 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:25.423581+0000", 2023-09-23T21:10:50.864 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.4602999999999999e-05 2023-09-23T21:10:50.864 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:50.864 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:50.864 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:10:50.864 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:25.423617+0000", 2023-09-23T21:10:50.864 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.6056000000000002e-05 2023-09-23T21:10:50.864 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:10:50.864 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:10:50.864 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:10:50.864 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:50.864 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:50.865 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.35856.0:52886 9.3 9:ccc0ebe9:::benchmark_data_smithi026_342732_object52885:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e354)", 2023-09-23T21:10:50.865 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:06:24.484834+0000", 2023-09-23T21:10:50.865 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 266.31743126200001, 2023-09-23T21:10:50.865 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.93888714799999995, 2023-09-23T21:10:50.865 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:10:50.865 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:10:50.865 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:10:50.865 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.35856", 2023-09-23T21:10:50.865 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/4180396258", 2023-09-23T21:10:50.865 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 52886 2023-09-23T21:10:50.865 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:50.865 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:10:50.866 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:50.866 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:10:50.866 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:24.484834+0000", 2023-09-23T21:10:50.866 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:10:50.866 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:50.866 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:50.866 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:10:50.866 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:24.484834+0000", 2023-09-23T21:10:50.866 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:10:50.866 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:50.866 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:50.866 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:10:50.867 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:24.484833+0000", 2023-09-23T21:10:50.867 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T21:10:50.867 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:50.867 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:50.867 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:10:50.867 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:24.484888+0000", 2023-09-23T21:10:50.867 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.5315000000000001e-05 2023-09-23T21:10:50.867 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:50.867 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:50.867 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:10:50.867 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:24.484901+0000", 2023-09-23T21:10:50.867 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.2958e-05 2023-09-23T21:10:50.867 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:50.868 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:50.868 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:10:50.868 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:24.484906+0000", 2023-09-23T21:10:50.868 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.4609999999999999e-06 2023-09-23T21:10:50.868 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:50.868 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:50.868 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:10:50.868 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:24.484972+0000", 2023-09-23T21:10:50.868 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.5840999999999995e-05 2023-09-23T21:10:50.868 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:50.868 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:50.868 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:10:50.868 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:24.484984+0000", 2023-09-23T21:10:50.868 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.1969e-05 2023-09-23T21:10:50.869 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:50.869 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:50.869 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:10:50.869 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:25.412238+0000", 2023-09-23T21:10:50.869 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.92725426899999996 2023-09-23T21:10:50.869 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:50.869 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:50.869 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:10:50.869 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:25.412359+0000", 2023-09-23T21:10:50.869 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00012057600000000001 2023-09-23T21:10:50.869 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:50.869 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:50.869 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 1,3", 2023-09-23T21:10:50.869 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:25.412512+0000", 2023-09-23T21:10:50.870 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00015280400000000001 2023-09-23T21:10:50.870 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:50.870 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:50.870 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:10:50.870 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:25.417774+0000", 2023-09-23T21:10:50.870 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0052620660000000001 2023-09-23T21:10:50.870 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:50.870 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:50.870 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:10:50.870 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:25.422260+0000", 2023-09-23T21:10:50.870 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0044861800000000002 2023-09-23T21:10:50.870 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:50.870 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:50.871 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:10:50.871 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:25.423663+0000", 2023-09-23T21:10:50.871 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.001402915 2023-09-23T21:10:50.871 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:50.871 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:50.871 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:10:50.871 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:25.423698+0000", 2023-09-23T21:10:50.871 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.5021999999999998e-05 2023-09-23T21:10:50.871 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:50.871 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:50.871 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:10:50.871 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:25.423721+0000", 2023-09-23T21:10:50.871 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.3166000000000001e-05 2023-09-23T21:10:50.871 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:10:50.872 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:10:50.872 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:10:50.872 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:50.872 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:50.872 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.35856.0:52898 9.3 9:ce13b48e:::benchmark_data_smithi026_342732_object52897:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e354)", 2023-09-23T21:10:50.872 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:06:24.502098+0000", 2023-09-23T21:10:50.872 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 266.300167676, 2023-09-23T21:10:50.872 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.92904386800000005, 2023-09-23T21:10:50.872 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:10:50.872 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:10:50.872 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:10:50.872 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.35856", 2023-09-23T21:10:50.872 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/4180396258", 2023-09-23T21:10:50.872 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 52898 2023-09-23T21:10:50.873 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:50.873 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:10:50.873 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:50.873 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:10:50.873 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:24.502098+0000", 2023-09-23T21:10:50.873 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:10:50.873 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:50.873 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:50.873 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:10:50.873 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:24.502098+0000", 2023-09-23T21:10:50.873 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:10:50.873 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:50.874 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:50.874 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:10:50.874 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:24.502096+0000", 2023-09-23T21:10:50.874 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999981 2023-09-23T21:10:50.874 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:50.874 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:50.874 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:10:50.874 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:24.502121+0000", 2023-09-23T21:10:50.874 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.5218000000000001e-05 2023-09-23T21:10:50.874 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:50.874 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:50.874 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:10:50.874 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:24.502135+0000", 2023-09-23T21:10:50.875 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.3606e-05 2023-09-23T21:10:50.875 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:50.875 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:50.875 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:10:50.875 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:24.502142+0000", 2023-09-23T21:10:50.875 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.2590000000000003e-06 2023-09-23T21:10:50.875 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:50.875 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:50.875 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:10:50.875 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:24.502203+0000", 2023-09-23T21:10:50.875 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.1614999999999999e-05 2023-09-23T21:10:50.875 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:50.875 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:50.875 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:10:50.876 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:24.502212+0000", 2023-09-23T21:10:50.876 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.9080000000000001e-06 2023-09-23T21:10:50.876 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:50.876 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:50.876 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:10:50.876 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:25.413366+0000", 2023-09-23T21:10:50.876 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.91115327199999996 2023-09-23T21:10:50.876 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:50.876 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:50.876 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:10:50.876 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:25.413462+0000", 2023-09-23T21:10:50.876 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.6717000000000003e-05 2023-09-23T21:10:50.876 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:50.877 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:50.877 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 1,3", 2023-09-23T21:10:50.877 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:25.413554+0000", 2023-09-23T21:10:50.877 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.1895e-05 2023-09-23T21:10:50.877 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:50.877 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:50.877 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:10:50.877 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:25.417800+0000", 2023-09-23T21:10:50.877 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0042454440000000001 2023-09-23T21:10:50.877 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:50.877 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:50.877 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:10:50.877 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:25.423772+0000", 2023-09-23T21:10:50.878 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0059727590000000002 2023-09-23T21:10:50.878 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:50.878 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:50.878 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:10:50.878 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:25.431044+0000", 2023-09-23T21:10:50.878 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0072718139999999997 2023-09-23T21:10:50.878 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:50.878 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:50.878 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:10:50.878 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:25.431098+0000", 2023-09-23T21:10:50.878 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.3832999999999998e-05 2023-09-23T21:10:50.878 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:50.878 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:50.879 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:10:50.879 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:25.431141+0000", 2023-09-23T21:10:50.879 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.3282000000000003e-05 2023-09-23T21:10:50.879 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:10:50.879 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:10:50.879 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:10:50.879 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:50.879 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:50.879 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.35856.0:52915 9.3 9:ca2a2df5:::benchmark_data_smithi026_342732_object52914:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e354)", 2023-09-23T21:10:50.879 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:06:24.520294+0000", 2023-09-23T21:10:50.879 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 266.28197108400002, 2023-09-23T21:10:50.879 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.91094416700000003, 2023-09-23T21:10:50.879 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:10:50.879 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:10:50.880 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:10:50.880 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.35856", 2023-09-23T21:10:50.880 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/4180396258", 2023-09-23T21:10:50.880 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 52915 2023-09-23T21:10:50.880 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:50.880 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:10:50.880 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:50.880 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:10:50.880 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:24.520294+0000", 2023-09-23T21:10:50.880 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:10:50.880 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:50.880 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:50.880 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:10:50.880 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:24.520294+0000", 2023-09-23T21:10:50.881 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:10:50.881 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:50.881 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:50.881 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:10:50.881 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:24.520293+0000", 2023-09-23T21:10:50.881 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T21:10:50.881 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:50.881 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:50.881 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:10:50.881 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:24.520346+0000", 2023-09-23T21:10:50.881 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.3087999999999997e-05 2023-09-23T21:10:50.881 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:50.881 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:50.882 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:10:50.882 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:24.520357+0000", 2023-09-23T21:10:50.882 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.1335000000000001e-05 2023-09-23T21:10:50.882 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:50.882 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:50.882 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:10:50.882 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:24.520362+0000", 2023-09-23T21:10:50.882 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.5839999999999996e-06 2023-09-23T21:10:50.882 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:50.882 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:50.882 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:10:50.882 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:24.520410+0000", 2023-09-23T21:10:50.882 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.7945000000000003e-05 2023-09-23T21:10:50.883 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:50.883 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:50.883 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:10:50.883 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:24.520418+0000", 2023-09-23T21:10:50.883 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.7029999999999995e-06 2023-09-23T21:10:50.883 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:50.883 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:50.883 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:10:50.883 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:25.414439+0000", 2023-09-23T21:10:50.883 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.89402078900000004 2023-09-23T21:10:50.883 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:50.883 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:50.883 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:10:50.883 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:25.414565+0000", 2023-09-23T21:10:50.884 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00012614700000000001 2023-09-23T21:10:50.884 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:50.884 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:50.884 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 1,3", 2023-09-23T21:10:50.884 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:25.414721+0000", 2023-09-23T21:10:50.884 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000155601 2023-09-23T21:10:50.884 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:50.884 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:50.884 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:10:50.884 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:25.420955+0000", 2023-09-23T21:10:50.884 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0062335200000000002 2023-09-23T21:10:50.884 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:50.885 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:50.885 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:10:50.885 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:25.423821+0000", 2023-09-23T21:10:50.885 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0028662779999999999 2023-09-23T21:10:50.885 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:50.885 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:50.885 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:10:50.885 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:25.431172+0000", 2023-09-23T21:10:50.885 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0073511369999999998 2023-09-23T21:10:50.885 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:50.885 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:50.885 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:10:50.885 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:25.431206+0000", 2023-09-23T21:10:50.886 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.4307000000000002e-05 2023-09-23T21:10:50.886 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:50.886 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:50.886 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:10:50.886 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:25.431238+0000", 2023-09-23T21:10:50.886 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.2027999999999999e-05 2023-09-23T21:10:50.886 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:10:50.886 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:10:50.886 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:10:50.886 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:50.886 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:50.886 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.35856.0:52919 9.3 9:c1c34947:::benchmark_data_smithi026_342732_object52918:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e354)", 2023-09-23T21:10:50.887 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:06:24.523819+0000", 2023-09-23T21:10:50.887 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 266.27844601700002, 2023-09-23T21:10:50.887 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.90866214300000003, 2023-09-23T21:10:50.887 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:10:50.887 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:10:50.887 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:10:50.887 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.35856", 2023-09-23T21:10:50.887 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/4180396258", 2023-09-23T21:10:50.887 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 52919 2023-09-23T21:10:50.887 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:50.887 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:10:50.887 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:50.887 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:10:50.887 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:24.523819+0000", 2023-09-23T21:10:50.888 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:10:50.888 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:50.888 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:50.888 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:10:50.888 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:24.523819+0000", 2023-09-23T21:10:50.888 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:10:50.888 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:50.888 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:50.888 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:10:50.888 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:24.523818+0000", 2023-09-23T21:10:50.888 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.999999 2023-09-23T21:10:50.888 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:50.888 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:50.888 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:10:50.889 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:24.523838+0000", 2023-09-23T21:10:50.889 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.9938000000000001e-05 2023-09-23T21:10:50.889 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:50.889 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:50.889 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:10:50.889 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:24.523849+0000", 2023-09-23T21:10:50.889 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.0617e-05 2023-09-23T21:10:50.889 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:50.889 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:50.889 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:10:50.889 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:24.523853+0000", 2023-09-23T21:10:50.889 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.2860000000000001e-06 2023-09-23T21:10:50.889 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:50.890 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:50.890 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:10:50.890 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:24.523911+0000", 2023-09-23T21:10:50.890 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.8381000000000002e-05 2023-09-23T21:10:50.890 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:50.890 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:50.890 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:10:50.890 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:24.523920+0000", 2023-09-23T21:10:50.890 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.8279999999999992e-06 2023-09-23T21:10:50.890 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:50.890 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:50.890 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:10:50.890 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:25.415624+0000", 2023-09-23T21:10:50.891 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.89170378500000003 2023-09-23T21:10:50.891 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:50.891 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:50.891 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:10:50.891 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:25.415717+0000", 2023-09-23T21:10:50.891 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.3216999999999999e-05 2023-09-23T21:10:50.891 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:50.891 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:50.891 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 1,3", 2023-09-23T21:10:50.891 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:25.415813+0000", 2023-09-23T21:10:50.891 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.5261999999999997e-05 2023-09-23T21:10:50.891 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:50.891 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:50.892 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:10:50.892 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:25.420983+0000", 2023-09-23T21:10:50.892 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0051701919999999997 2023-09-23T21:10:50.892 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:50.892 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:50.892 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:10:50.892 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:25.429934+0000", 2023-09-23T21:10:50.892 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0089515629999999992 2023-09-23T21:10:50.892 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:50.892 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:50.892 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:10:50.892 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:25.432395+0000", 2023-09-23T21:10:50.892 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0024608130000000001 2023-09-23T21:10:50.892 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:50.893 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:50.893 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:10:50.893 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:25.432445+0000", 2023-09-23T21:10:50.893 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.0336000000000001e-05 2023-09-23T21:10:50.893 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:50.893 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:50.893 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:10:50.893 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:25.432481+0000", 2023-09-23T21:10:50.893 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.5899999999999998e-05 2023-09-23T21:10:50.893 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:10:50.893 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:10:50.893 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:10:50.893 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:50.894 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:50.894 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.35856.0:52927 9.3 9:c0ef7e28:::benchmark_data_smithi026_342732_object52926:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e354)", 2023-09-23T21:10:50.894 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:06:24.534140+0000", 2023-09-23T21:10:50.894 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 266.268124914, 2023-09-23T21:10:50.894 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.90076572399999999, 2023-09-23T21:10:50.894 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:10:50.894 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:10:50.894 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:10:50.894 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.35856", 2023-09-23T21:10:50.894 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/4180396258", 2023-09-23T21:10:50.894 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 52927 2023-09-23T21:10:50.894 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:50.895 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:10:50.895 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:50.895 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:10:50.895 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:24.534140+0000", 2023-09-23T21:10:50.895 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:10:50.895 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:50.895 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:50.895 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:10:50.895 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:24.534140+0000", 2023-09-23T21:10:50.895 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:10:50.895 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:50.895 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:50.895 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:10:50.895 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:24.534139+0000", 2023-09-23T21:10:50.896 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T21:10:50.896 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:50.896 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:50.896 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:10:50.896 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:24.534175+0000", 2023-09-23T21:10:50.896 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.5553999999999999e-05 2023-09-23T21:10:50.896 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:50.896 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:50.896 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:10:50.896 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:24.534186+0000", 2023-09-23T21:10:50.896 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.1334e-05 2023-09-23T21:10:50.896 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:50.896 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:50.897 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:10:50.897 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:24.534192+0000", 2023-09-23T21:10:50.897 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.9950000000000002e-06 2023-09-23T21:10:50.897 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:50.897 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:50.897 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:10:50.897 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:24.534258+0000", 2023-09-23T21:10:50.897 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.6299999999999999e-05 2023-09-23T21:10:50.897 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:50.897 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:50.897 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:10:50.897 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:24.534275+0000", 2023-09-23T21:10:50.897 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.6691e-05 2023-09-23T21:10:50.897 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:50.898 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:50.898 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:10:50.898 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:25.416673+0000", 2023-09-23T21:10:50.898 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.88239785500000001 2023-09-23T21:10:50.898 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:50.898 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:50.898 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:10:50.898 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:25.416787+0000", 2023-09-23T21:10:50.898 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000114636 2023-09-23T21:10:50.898 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:50.898 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:50.898 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 1,3", 2023-09-23T21:10:50.898 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:25.416936+0000", 2023-09-23T21:10:50.898 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00014905300000000001 2023-09-23T21:10:50.899 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:50.899 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:50.899 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:10:50.899 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:25.421097+0000", 2023-09-23T21:10:50.899 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0041605940000000001 2023-09-23T21:10:50.899 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:50.899 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:50.899 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:10:50.899 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:25.433057+0000", 2023-09-23T21:10:50.899 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.011959981999999999 2023-09-23T21:10:50.899 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:50.899 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:50.899 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:10:50.900 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:25.434817+0000", 2023-09-23T21:10:50.900 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0017596090000000001 2023-09-23T21:10:50.900 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:50.900 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:50.900 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:10:50.900 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:25.434871+0000", 2023-09-23T21:10:50.900 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.4228000000000003e-05 2023-09-23T21:10:50.900 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:50.900 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:50.900 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:10:50.900 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:25.434906+0000", 2023-09-23T21:10:50.900 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.5113000000000001e-05 2023-09-23T21:10:50.901 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:10:50.901 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:10:50.901 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:10:50.901 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:50.901 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:50.901 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.35856.0:52943 9.3 9:cce7983c:::benchmark_data_smithi026_342732_object52942:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e354)", 2023-09-23T21:10:50.901 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:06:24.551421+0000", 2023-09-23T21:10:50.901 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 266.250844605, 2023-09-23T21:10:50.901 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.88400061399999996, 2023-09-23T21:10:50.901 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:10:50.901 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:10:50.901 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:10:50.901 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.35856", 2023-09-23T21:10:50.902 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/4180396258", 2023-09-23T21:10:50.902 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 52943 2023-09-23T21:10:50.902 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:50.902 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:10:50.902 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:50.902 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:10:50.902 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:24.551421+0000", 2023-09-23T21:10:50.902 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:10:50.902 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:50.902 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:50.902 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:10:50.902 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:24.551421+0000", 2023-09-23T21:10:50.902 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:10:50.902 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:50.903 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:50.903 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:10:50.903 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:24.551419+0000", 2023-09-23T21:10:50.903 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T21:10:50.903 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:50.903 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:50.903 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:10:50.903 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:24.551465+0000", 2023-09-23T21:10:50.903 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.5859000000000003e-05 2023-09-23T21:10:50.903 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:50.903 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:50.903 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:10:50.903 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:24.551478+0000", 2023-09-23T21:10:50.904 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.2401999999999999e-05 2023-09-23T21:10:50.904 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:50.904 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:50.904 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:10:50.904 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:24.551482+0000", 2023-09-23T21:10:50.904 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.8180000000000003e-06 2023-09-23T21:10:50.904 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:50.904 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:50.904 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:10:50.904 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:24.551531+0000", 2023-09-23T21:10:50.904 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.8235999999999997e-05 2023-09-23T21:10:50.904 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:50.904 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:50.905 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:10:50.905 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:24.551539+0000", 2023-09-23T21:10:50.905 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.5790000000000004e-06 2023-09-23T21:10:50.905 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:50.905 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:50.905 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:10:50.905 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:25.417837+0000", 2023-09-23T21:10:50.905 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.86629784499999996 2023-09-23T21:10:50.905 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:50.905 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:50.905 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:10:50.905 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:25.417926+0000", 2023-09-23T21:10:50.906 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.8490000000000001e-05 2023-09-23T21:10:50.906 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:50.906 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:50.906 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 1,3", 2023-09-23T21:10:50.906 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:25.418026+0000", 2023-09-23T21:10:50.906 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000100294 2023-09-23T21:10:50.906 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:50.906 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:50.906 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:10:50.906 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:25.421105+0000", 2023-09-23T21:10:50.906 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.003079072 2023-09-23T21:10:50.906 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:50.906 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:50.907 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:10:50.907 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:25.434959+0000", 2023-09-23T21:10:50.907 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.013854083 2023-09-23T21:10:50.907 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:50.907 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:50.907 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:10:50.907 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:25.435329+0000", 2023-09-23T21:10:50.907 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00036980599999999998 2023-09-23T21:10:50.907 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:50.907 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:50.907 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:10:50.907 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:25.435383+0000", 2023-09-23T21:10:50.907 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.3835000000000002e-05 2023-09-23T21:10:50.908 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:50.908 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:50.908 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:10:50.908 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:25.435421+0000", 2023-09-23T21:10:50.908 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.8525000000000001e-05 2023-09-23T21:10:50.908 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:10:50.908 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:10:50.908 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:10:50.908 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:50.908 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:50.908 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.46328.0:12430 10.1 10:8e1b46c6:::benchmark_data_smithi026_379838_object12429:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e430)", 2023-09-23T21:10:50.908 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:10:32.729016+0000", 2023-09-23T21:10:50.908 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 18.073248829000001, 2023-09-23T21:10:50.908 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.96804327999999995, 2023-09-23T21:10:50.909 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:10:50.909 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:10:50.909 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:10:50.909 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.46328", 2023-09-23T21:10:50.909 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/3748861181", 2023-09-23T21:10:50.909 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 12430 2023-09-23T21:10:50.909 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:50.909 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:10:50.909 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:50.909 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:10:50.909 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:10:32.729016+0000", 2023-09-23T21:10:50.909 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:10:50.909 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:50.909 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:50.910 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:10:50.910 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:10:32.729016+0000", 2023-09-23T21:10:50.910 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:10:50.910 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:50.910 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:50.910 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:10:50.910 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:10:32.729015+0000", 2023-09-23T21:10:50.910 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T21:10:50.910 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:50.910 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:50.910 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:10:50.910 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:10:32.729127+0000", 2023-09-23T21:10:50.911 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000111895 2023-09-23T21:10:50.911 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:50.911 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:50.911 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:10:50.911 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:10:32.729138+0000", 2023-09-23T21:10:50.911 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.0900999999999999e-05 2023-09-23T21:10:50.911 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:50.911 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:50.911 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:10:50.911 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:10:32.729143+0000", 2023-09-23T21:10:50.911 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.3329999999999996e-06 2023-09-23T21:10:50.911 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:50.911 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:50.912 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:10:50.912 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:10:32.729222+0000", 2023-09-23T21:10:50.912 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.9169e-05 2023-09-23T21:10:50.912 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:50.912 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:50.912 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:10:50.912 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:10:32.729245+0000", 2023-09-23T21:10:50.912 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.3125e-05 2023-09-23T21:10:50.912 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:50.912 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:50.912 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:10:50.912 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:10:33.686993+0000", 2023-09-23T21:10:50.912 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.95774722400000001 2023-09-23T21:10:50.913 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:50.913 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:50.913 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:10:50.913 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:10:33.687143+0000", 2023-09-23T21:10:50.913 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00015004499999999999 2023-09-23T21:10:50.913 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:50.913 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:50.913 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 0,1", 2023-09-23T21:10:50.913 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:10:33.687248+0000", 2023-09-23T21:10:50.913 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000105204 2023-09-23T21:10:50.913 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:50.913 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:50.913 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:10:50.914 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:10:33.689705+0000", 2023-09-23T21:10:50.914 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0024572420000000001 2023-09-23T21:10:50.914 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:50.914 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:50.914 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:10:50.914 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:10:33.696947+0000", 2023-09-23T21:10:50.914 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0072415550000000002 2023-09-23T21:10:50.914 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:50.914 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:50.914 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:10:50.914 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:10:33.696974+0000", 2023-09-23T21:10:50.915 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.7412000000000001e-05 2023-09-23T21:10:50.915 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:50.915 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:50.915 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:10:50.915 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:10:33.697024+0000", 2023-09-23T21:10:50.915 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.9570000000000001e-05 2023-09-23T21:10:50.915 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:50.915 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:50.915 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:10:50.915 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:10:33.697060+0000", 2023-09-23T21:10:50.915 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.6023999999999999e-05 2023-09-23T21:10:50.915 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:10:50.916 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:10:50.916 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:10:50.916 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:50.916 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:50.916 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.46328.0:12477 10.1 10:8cecffa5:::benchmark_data_smithi026_379838_object12476:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e430)", 2023-09-23T21:10:50.916 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:10:32.755815+0000", 2023-09-23T21:10:50.916 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 18.046449726999999, 2023-09-23T21:10:50.916 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.94137344599999995, 2023-09-23T21:10:50.916 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:10:50.916 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:10:50.916 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:10:50.916 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.46328", 2023-09-23T21:10:50.916 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/3748861181", 2023-09-23T21:10:50.917 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 12477 2023-09-23T21:10:50.917 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:50.917 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:10:50.917 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:50.917 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:10:50.917 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:10:32.755815+0000", 2023-09-23T21:10:50.917 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:10:50.917 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:50.917 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:50.917 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:10:50.917 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:10:32.755815+0000", 2023-09-23T21:10:50.917 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:10:50.918 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:50.918 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:50.918 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:10:50.918 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:10:32.755814+0000", 2023-09-23T21:10:50.918 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T21:10:50.918 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:50.918 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:50.918 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:10:50.918 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:10:32.755861+0000", 2023-09-23T21:10:50.918 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.672e-05 2023-09-23T21:10:50.918 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:50.918 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:50.918 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:10:50.919 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:10:32.755871+0000", 2023-09-23T21:10:50.919 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.9550000000000004e-06 2023-09-23T21:10:50.919 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:50.919 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:50.919 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:10:50.919 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:10:32.755876+0000", 2023-09-23T21:10:50.919 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.763e-06 2023-09-23T21:10:50.919 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:50.919 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:50.919 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:10:50.919 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:10:32.755945+0000", 2023-09-23T21:10:50.919 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.9622000000000005e-05 2023-09-23T21:10:50.919 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:50.920 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:50.920 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:10:50.920 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:10:32.755959+0000", 2023-09-23T21:10:50.920 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.3378e-05 2023-09-23T21:10:50.920 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:50.920 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:50.920 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:10:50.920 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:10:33.687722+0000", 2023-09-23T21:10:50.920 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.93176342400000001 2023-09-23T21:10:50.920 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:50.920 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:50.920 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:10:50.920 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:10:33.687805+0000", 2023-09-23T21:10:50.920 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.2688999999999994e-05 2023-09-23T21:10:50.921 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:50.921 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:50.921 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 0,1", 2023-09-23T21:10:50.921 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:10:33.687896+0000", 2023-09-23T21:10:50.921 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.1217000000000004e-05 2023-09-23T21:10:50.921 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:50.921 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:50.921 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:10:50.921 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:10:33.690957+0000", 2023-09-23T21:10:50.921 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0030608549999999999 2023-09-23T21:10:50.921 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:50.921 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:50.921 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:10:50.922 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:10:33.697095+0000", 2023-09-23T21:10:50.922 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0061379900000000003 2023-09-23T21:10:50.922 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:50.922 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:50.922 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:10:50.922 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:10:33.697119+0000", 2023-09-23T21:10:50.922 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.4345000000000002e-05 2023-09-23T21:10:50.922 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:50.922 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:50.922 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:10:50.922 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:10:33.697156+0000", 2023-09-23T21:10:50.922 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.6891000000000003e-05 2023-09-23T21:10:50.922 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:50.922 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:50.923 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:10:50.923 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:10:33.697189+0000", 2023-09-23T21:10:50.923 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.2793000000000003e-05 2023-09-23T21:10:50.923 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:10:50.923 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:10:50.923 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:10:50.923 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:50.923 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:50.923 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.46328.0:12516 10.1 10:89bfe74f:::benchmark_data_smithi026_379838_object12515:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e430)", 2023-09-23T21:10:50.923 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:10:32.778503+0000", 2023-09-23T21:10:50.923 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 18.023761703000002, 2023-09-23T21:10:50.923 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.91881996200000005, 2023-09-23T21:10:50.923 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:10:50.924 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:10:50.924 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:10:50.924 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.46328", 2023-09-23T21:10:50.924 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/3748861181", 2023-09-23T21:10:50.924 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 12516 2023-09-23T21:10:50.924 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:50.924 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:10:50.924 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:50.924 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:10:50.924 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:10:32.778503+0000", 2023-09-23T21:10:50.924 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:10:50.924 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:50.924 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:50.925 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:10:50.925 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:10:32.778503+0000", 2023-09-23T21:10:50.925 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:10:50.925 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:50.925 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:50.925 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:10:50.925 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:10:32.778502+0000", 2023-09-23T21:10:50.925 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.999999 2023-09-23T21:10:50.925 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:50.925 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:50.925 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:10:50.925 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:10:32.778550+0000", 2023-09-23T21:10:50.925 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.7357000000000001e-05 2023-09-23T21:10:50.926 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:50.926 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:50.926 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:10:50.926 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:10:32.778561+0000", 2023-09-23T21:10:50.926 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.1151999999999999e-05 2023-09-23T21:10:50.926 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:50.926 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:50.926 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:10:50.926 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:10:32.778566+0000", 2023-09-23T21:10:50.926 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.152e-06 2023-09-23T21:10:50.926 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:50.926 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:50.927 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:10:50.927 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:10:32.778619+0000", 2023-09-23T21:10:50.927 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.3167000000000001e-05 2023-09-23T21:10:50.927 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:50.927 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:50.927 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:10:50.927 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:10:32.778629+0000", 2023-09-23T21:10:50.927 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.4280000000000002e-06 2023-09-23T21:10:50.927 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:50.927 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:50.927 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:10:50.927 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:10:33.688318+0000", 2023-09-23T21:10:50.927 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.90968943199999996 2023-09-23T21:10:50.928 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:50.928 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:50.928 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:10:50.928 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:10:33.688426+0000", 2023-09-23T21:10:50.928 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00010844 2023-09-23T21:10:50.928 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:50.928 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:50.928 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 0,1", 2023-09-23T21:10:50.928 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:10:33.688509+0000", 2023-09-23T21:10:50.928 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.2058000000000006e-05 2023-09-23T21:10:50.928 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:50.928 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:50.928 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:10:50.929 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:10:33.690965+0000", 2023-09-23T21:10:50.929 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0024561330000000001 2023-09-23T21:10:50.929 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:50.929 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:50.929 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:10:50.929 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:10:33.697219+0000", 2023-09-23T21:10:50.929 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0062539129999999998 2023-09-23T21:10:50.929 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:50.929 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:50.929 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:10:50.929 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:10:33.697252+0000", 2023-09-23T21:10:50.929 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.341e-05 2023-09-23T21:10:50.929 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:50.929 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:50.930 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:10:50.930 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:10:33.697293+0000", 2023-09-23T21:10:50.930 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.0688000000000001e-05 2023-09-23T21:10:50.930 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:50.930 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:50.930 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:10:50.930 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:10:33.697323+0000", 2023-09-23T21:10:50.930 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.0790000000000002e-05 2023-09-23T21:10:50.930 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:10:50.930 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:10:50.930 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:10:50.930 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:50.930 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:50.931 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.46328.0:12530 10.1 10:8ae90501:::benchmark_data_smithi026_379838_object12529:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e430)", 2023-09-23T21:10:50.931 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:10:32.786084+0000", 2023-09-23T21:10:50.931 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 18.016181625000002, 2023-09-23T21:10:50.931 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.91136620899999998, 2023-09-23T21:10:50.931 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:10:50.931 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:10:50.931 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:10:50.931 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.46328", 2023-09-23T21:10:50.931 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/3748861181", 2023-09-23T21:10:50.931 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 12530 2023-09-23T21:10:50.931 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:50.931 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:10:50.931 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:50.932 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:10:50.932 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:10:32.786084+0000", 2023-09-23T21:10:50.932 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:10:50.932 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:50.932 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:50.932 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:10:50.932 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:10:32.786084+0000", 2023-09-23T21:10:50.932 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:10:50.932 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:50.932 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:50.932 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:10:50.932 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:10:32.786082+0000", 2023-09-23T21:10:50.932 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T21:10:50.933 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:50.933 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:50.933 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:10:50.933 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:10:32.786113+0000", 2023-09-23T21:10:50.933 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.1185e-05 2023-09-23T21:10:50.933 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:50.933 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:50.933 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:10:50.933 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:10:32.786127+0000", 2023-09-23T21:10:50.933 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.3412e-05 2023-09-23T21:10:50.933 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:50.933 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:50.933 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:10:50.933 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:10:32.786133+0000", 2023-09-23T21:10:50.934 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.3289999999999999e-06 2023-09-23T21:10:50.934 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:50.934 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:50.934 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:10:50.934 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:10:32.786181+0000", 2023-09-23T21:10:50.934 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.8056000000000002e-05 2023-09-23T21:10:50.934 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:50.934 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:50.934 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:10:50.934 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:10:32.786191+0000", 2023-09-23T21:10:50.934 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.5230000000000001e-06 2023-09-23T21:10:50.934 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:50.935 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:50.935 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:10:50.935 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:10:33.689096+0000", 2023-09-23T21:10:50.935 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.90290496799999997 2023-09-23T21:10:50.935 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:50.935 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:50.935 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:10:50.935 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:10:33.689176+0000", 2023-09-23T21:10:50.935 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.0456000000000004e-05 2023-09-23T21:10:50.935 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:50.935 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:50.935 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 0,1", 2023-09-23T21:10:50.935 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:10:33.689269+0000", 2023-09-23T21:10:50.936 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.3321999999999999e-05 2023-09-23T21:10:50.936 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:50.936 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:50.936 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:10:50.936 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:10:33.691678+0000", 2023-09-23T21:10:50.936 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0024089179999999999 2023-09-23T21:10:50.936 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:50.936 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:50.936 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:10:50.936 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:10:33.697352+0000", 2023-09-23T21:10:50.936 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0056740879999999999 2023-09-23T21:10:50.936 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:50.937 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:50.937 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:10:50.937 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:10:33.697377+0000", 2023-09-23T21:10:50.937 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.4904000000000001e-05 2023-09-23T21:10:50.937 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:50.937 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:50.937 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:10:50.937 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:10:33.697416+0000", 2023-09-23T21:10:50.937 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.8779000000000003e-05 2023-09-23T21:10:50.937 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:50.937 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:50.937 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:10:50.937 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:10:33.697450+0000", 2023-09-23T21:10:50.937 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.3736999999999999e-05 2023-09-23T21:10:50.938 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:10:50.938 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:10:50.938 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:10:50.938 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:50.938 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:50.938 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.46328.0:12547 10.1 10:84d726cb:::benchmark_data_smithi026_379838_object12546:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e430)", 2023-09-23T21:10:50.938 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:10:32.793960+0000", 2023-09-23T21:10:50.938 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 18.008305645, 2023-09-23T21:10:50.938 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.90371857499999997, 2023-09-23T21:10:50.938 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:10:50.938 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:10:50.938 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:10:50.938 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.46328", 2023-09-23T21:10:50.939 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/3748861181", 2023-09-23T21:10:50.939 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 12547 2023-09-23T21:10:50.939 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:50.939 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:10:50.939 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:50.939 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:10:50.939 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:10:32.793960+0000", 2023-09-23T21:10:50.939 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:10:50.939 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:50.939 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:50.939 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:10:50.939 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:10:32.793960+0000", 2023-09-23T21:10:50.939 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:10:50.940 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:50.940 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:50.940 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:10:50.940 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:10:32.793958+0000", 2023-09-23T21:10:50.940 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T21:10:50.940 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:50.940 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:50.940 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:10:50.940 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:10:32.793991+0000", 2023-09-23T21:10:50.940 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.2821999999999998e-05 2023-09-23T21:10:50.940 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:50.940 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:50.940 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:10:50.941 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:10:32.794005+0000", 2023-09-23T21:10:50.941 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.3845e-05 2023-09-23T21:10:50.941 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:50.941 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:50.941 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:10:50.941 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:10:32.794011+0000", 2023-09-23T21:10:50.941 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.0630000000000002e-06 2023-09-23T21:10:50.941 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:50.941 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:50.941 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:10:50.941 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:10:32.794923+0000", 2023-09-23T21:10:50.941 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00091206700000000002 2023-09-23T21:10:50.941 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:50.941 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:50.942 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:10:50.942 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:10:32.794935+0000", 2023-09-23T21:10:50.942 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.2248e-05 2023-09-23T21:10:50.942 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:50.942 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:50.942 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:10:50.942 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:10:33.689751+0000", 2023-09-23T21:10:50.942 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.89481587799999995 2023-09-23T21:10:50.942 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:50.942 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:50.942 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:10:50.942 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:10:33.689844+0000", 2023-09-23T21:10:50.942 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.3568999999999998e-05 2023-09-23T21:10:50.943 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:50.943 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:50.943 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 0,1", 2023-09-23T21:10:50.943 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:10:33.690004+0000", 2023-09-23T21:10:50.943 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000159634 2023-09-23T21:10:50.943 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:50.943 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:50.943 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:10:50.943 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:10:33.691690+0000", 2023-09-23T21:10:50.943 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0016856550000000001 2023-09-23T21:10:50.943 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:50.943 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:50.943 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:10:50.944 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:10:33.697493+0000", 2023-09-23T21:10:50.944 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0058030099999999999 2023-09-23T21:10:50.944 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:50.944 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:50.944 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:10:50.944 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:10:33.697604+0000", 2023-09-23T21:10:50.944 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00011134299999999999 2023-09-23T21:10:50.944 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:50.944 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:50.944 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:10:50.944 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:10:33.697647+0000", 2023-09-23T21:10:50.944 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.2567000000000001e-05 2023-09-23T21:10:50.944 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:50.945 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:50.945 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:10:50.945 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:10:33.697678+0000", 2023-09-23T21:10:50.945 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.1538999999999997e-05 2023-09-23T21:10:50.945 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:10:50.945 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:10:50.945 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:10:50.945 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:50.945 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:50.945 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.46328.0:12549 10.1 10:858c784b:::benchmark_data_smithi026_379838_object12548:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e430)", 2023-09-23T21:10:50.945 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:10:32.794028+0000", 2023-09-23T21:10:50.945 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 18.008237189999999, 2023-09-23T21:10:50.945 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.90377487700000003, 2023-09-23T21:10:50.946 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:10:50.946 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:10:50.946 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:10:50.946 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.46328", 2023-09-23T21:10:50.946 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/3748861181", 2023-09-23T21:10:50.946 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 12549 2023-09-23T21:10:50.946 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:50.946 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:10:50.946 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:50.946 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:10:50.946 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:10:32.794028+0000", 2023-09-23T21:10:50.946 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:10:50.946 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:50.947 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:50.947 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:10:50.947 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:10:32.794028+0000", 2023-09-23T21:10:50.947 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:10:50.947 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:50.947 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:50.947 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:10:50.947 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:10:32.794028+0000", 2023-09-23T21:10:50.947 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999995 2023-09-23T21:10:50.947 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:50.947 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:50.947 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:10:50.948 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:10:32.794054+0000", 2023-09-23T21:10:50.948 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.6534e-05 2023-09-23T21:10:50.948 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:50.948 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:50.948 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:10:50.948 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:10:32.794061+0000", 2023-09-23T21:10:50.948 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.8229999999999997e-06 2023-09-23T21:10:50.948 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:50.948 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:50.948 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:10:50.948 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:10:32.794065+0000", 2023-09-23T21:10:50.948 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.2520000000000001e-06 2023-09-23T21:10:50.948 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:50.948 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:50.949 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:10:50.949 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:10:32.794960+0000", 2023-09-23T21:10:50.949 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00089468100000000004 2023-09-23T21:10:50.949 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:50.949 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:50.949 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:10:50.949 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:10:32.794969+0000", 2023-09-23T21:10:50.949 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.5680000000000005e-06 2023-09-23T21:10:50.949 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:50.949 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:50.949 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:10:50.949 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:10:33.690437+0000", 2023-09-23T21:10:50.949 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.89546786300000003 2023-09-23T21:10:50.950 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:50.950 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:50.950 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:10:50.950 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:10:33.690542+0000", 2023-09-23T21:10:50.950 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00010514999999999999 2023-09-23T21:10:50.950 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:50.950 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:50.950 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 0,1", 2023-09-23T21:10:50.950 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:10:33.690641+0000", 2023-09-23T21:10:50.950 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.8827999999999996e-05 2023-09-23T21:10:50.950 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:50.950 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:50.950 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:10:50.951 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:10:33.692941+0000", 2023-09-23T21:10:50.951 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.002299298 2023-09-23T21:10:50.951 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:50.951 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:50.951 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:10:50.951 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:10:33.697710+0000", 2023-09-23T21:10:50.951 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0047696650000000002 2023-09-23T21:10:50.951 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:50.951 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:50.951 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:10:50.951 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:10:33.697737+0000", 2023-09-23T21:10:50.951 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.7146999999999999e-05 2023-09-23T21:10:50.951 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:50.952 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:50.952 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:10:50.952 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:10:33.697773+0000", 2023-09-23T21:10:50.952 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.54e-05 2023-09-23T21:10:50.952 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:10:50.952 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:10:50.952 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:10:50.952 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:10:33.697803+0000", 2023-09-23T21:10:50.952 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.0128999999999999e-05 2023-09-23T21:10:50.952 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:10:50.952 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:10:50.952 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:10:50.952 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:10:50.953 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:10:50.953 INFO:teuthology.orchestra.run.smithi070.stdout:} 2023-09-23T21:10:50.953 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:10:50.954 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:10:50.954 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:10:51.035 INFO:teuthology.orchestra.run.smithi026.stdout: 2023-09-23T21:10:51.054 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:10:51.055 INFO:teuthology.orchestra.run.smithi026.stderr:dumped all 2023-09-23T21:10:51.086 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:10:51.177 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:10:51.180 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:10:51.180 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:10:51.181 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_ops_in_flight 2023-09-23T21:10:51.282 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:10:51.343 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:10:51.445 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 29 16 36412 36396 78.3886 76.875 0.018087 0.0127479 2023-09-23T21:10:51.702 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:10:51 smithi026 ceph-mon[175151]: pgmap v1126: 109 pgs: 109 active+clean; 1.9 GiB data, 9.0 GiB used, 617 GiB / 626 GiB avail; 100 MiB/s wr, 1.60k op/s 2023-09-23T21:10:51.702 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:10:51 smithi026 ceph-mon[175151]: from='client.56167 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T21:10:51.703 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:10:51 smithi026 ceph-mon[179688]: pgmap v1126: 109 pgs: 109 active+clean; 1.9 GiB data, 9.0 GiB used, 617 GiB / 626 GiB avail; 100 MiB/s wr, 1.60k op/s 2023-09-23T21:10:51.703 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:10:51 smithi026 ceph-mon[179688]: from='client.56167 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T21:10:51.876 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T21:10:51.880 INFO:teuthology.orchestra.run.smithi026.stdout:{"pg_ready":true,"pg_map":{"version":1126,"stamp":"2023-09-23T21:10:50.961371+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":2063079328,"num_objects":31495,"num_object_clones":0,"num_object_copies":94485,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":31495,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":31532,"num_write_kb":2014173,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2841,"num_bytes_recovered":183836058,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":34907,"ondisk_log_size":34907,"up":327,"acting":327,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":334,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":9484044,"kb_used_data":6046164,"kb_used_omap":127,"kb_used_meta":3437696,"kb_avail":646732020,"statfs":{"total":671965249536,"available":662253588480,"internally_reserved":0,"allocated":6191271936,"data_stored":6179830375,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":130672,"internal_metadata":3520201104},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[8,5,7,9,4],"upper_bound":32},"perf_stat":{"commit_latency_ms":4,"apply_latency_ms":4,"commit_latency_ns":4000000,"apply_latency_ns":4000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":1050658183,"num_objects":16033,"num_object_clones":0,"num_object_copies":48099,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":16033,"num_whiteouts":0,"num_read":-67,"num_read_kb":-126,"num_write":16001,"num_write_kb":1025783,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":-1,"num_bytes_recovered":-241514,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.003706"},"pg_stats":[{"pgid":"3.1a","version":"68'1","reported_seq":50,"reported_epoch":434,"state":"active+clean","last_fresh":"2023-09-23T21:10:36.676784+0000","last_change":"2023-09-23T21:09:08.416508+0000","last_active":"2023-09-23T21:10:36.676784+0000","last_peered":"2023-09-23T21:10:36.676784+0000","last_clean":"2023-09-23T21:10:36.676784+0000","last_became_active":"2023-09-23T21:09:08.416124+0000","last_became_peered":"2023-09-23T21:09:08.416124+0000","last_unstale":"2023-09-23T21:10:36.676784+0000","last_undegraded":"2023-09-23T21:10:36.676784+0000","last_fullsized":"2023-09-23T21:10:36.676784+0000","mapping_epoch":396,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":397,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:46:38.597918+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,0],"acting":[2,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":127,"reported_epoch":434,"state":"active+clean","last_fresh":"2023-09-23T21:10:36.685442+0000","last_change":"2023-09-23T21:10:36.683894+0000","last_active":"2023-09-23T21:10:36.685442+0000","last_peered":"2023-09-23T21:10:36.685442+0000","last_clean":"2023-09-23T21:10:36.685442+0000","last_became_active":"2023-09-23T21:10:36.683577+0000","last_became_peered":"2023-09-23T21:10:36.683577+0000","last_unstale":"2023-09-23T21:10:36.685442+0000","last_undegraded":"2023-09-23T21:10:36.685442+0000","last_fullsized":"2023-09-23T21:10:36.685442+0000","mapping_epoch":433,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":434,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:16:00.636335+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":436,"reported_epoch":435,"state":"active+clean","last_fresh":"2023-09-23T21:10:42.142845+0000","last_change":"2023-09-23T21:10:26.766728+0000","last_active":"2023-09-23T21:10:42.142845+0000","last_peered":"2023-09-23T21:10:42.142845+0000","last_clean":"2023-09-23T21:10:42.142845+0000","last_became_active":"2023-09-23T21:10:26.766362+0000","last_became_peered":"2023-09-23T21:10:26.766362+0000","last_unstale":"2023-09-23T21:10:42.142845+0000","last_undegraded":"2023-09-23T21:10:42.142845+0000","last_fullsized":"2023-09-23T21:10:42.142845+0000","mapping_epoch":423,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":424,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:39:58.279587+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":443,"reported_epoch":434,"state":"active+clean","last_fresh":"2023-09-23T21:10:36.682078+0000","last_change":"2023-09-23T21:10:26.757297+0000","last_active":"2023-09-23T21:10:36.682078+0000","last_peered":"2023-09-23T21:10:36.682078+0000","last_clean":"2023-09-23T21:10:36.682078+0000","last_became_active":"2023-09-23T21:10:26.756829+0000","last_became_peered":"2023-09-23T21:10:26.756829+0000","last_unstale":"2023-09-23T21:10:36.682078+0000","last_undegraded":"2023-09-23T21:10:36.682078+0000","last_fullsized":"2023-09-23T21:10:36.682078+0000","mapping_epoch":423,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":424,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:13:21.769484+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":356,"reported_epoch":434,"state":"active+clean","last_fresh":"2023-09-23T21:10:37.413101+0000","last_change":"2023-09-23T21:10:36.683461+0000","last_active":"2023-09-23T21:10:37.413101+0000","last_peered":"2023-09-23T21:10:37.413101+0000","last_clean":"2023-09-23T21:10:37.413101+0000","last_became_active":"2023-09-23T21:10:36.683129+0000","last_became_peered":"2023-09-23T21:10:36.683129+0000","last_unstale":"2023-09-23T21:10:37.413101+0000","last_undegraded":"2023-09-23T21:10:37.413101+0000","last_fullsized":"2023-09-23T21:10:37.413101+0000","mapping_epoch":433,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":434,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:43:58.805578+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":374,"reported_epoch":434,"state":"active+clean","last_fresh":"2023-09-23T21:10:36.675528+0000","last_change":"2023-09-23T21:10:16.699884+0000","last_active":"2023-09-23T21:10:36.675528+0000","last_peered":"2023-09-23T21:10:36.675528+0000","last_clean":"2023-09-23T21:10:36.675528+0000","last_became_active":"2023-09-23T21:10:16.699576+0000","last_became_peered":"2023-09-23T21:10:16.699576+0000","last_unstale":"2023-09-23T21:10:36.675528+0000","last_undegraded":"2023-09-23T21:10:36.675528+0000","last_fullsized":"2023-09-23T21:10:36.675528+0000","mapping_epoch":416,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":417,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:38:04.728577+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":432,"reported_epoch":434,"state":"active+clean","last_fresh":"2023-09-23T21:10:36.681586+0000","last_change":"2023-09-23T21:05:04.304358+0000","last_active":"2023-09-23T21:10:36.681586+0000","last_peered":"2023-09-23T21:10:36.681586+0000","last_clean":"2023-09-23T21:10:36.681586+0000","last_became_active":"2023-09-23T21:05:04.303913+0000","last_became_peered":"2023-09-23T21:05:04.303913+0000","last_unstale":"2023-09-23T21:10:36.681586+0000","last_undegraded":"2023-09-23T21:10:36.681586+0000","last_fullsized":"2023-09-23T21:10:36.681586+0000","mapping_epoch":332,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:21:21.860944+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":453,"reported_epoch":434,"state":"active+clean","last_fresh":"2023-09-23T21:10:36.679838+0000","last_change":"2023-09-23T21:09:08.423849+0000","last_active":"2023-09-23T21:10:36.679838+0000","last_peered":"2023-09-23T21:10:36.679838+0000","last_clean":"2023-09-23T21:10:36.679838+0000","last_became_active":"2023-09-23T21:09:08.423444+0000","last_became_peered":"2023-09-23T21:09:08.423444+0000","last_unstale":"2023-09-23T21:10:36.679838+0000","last_undegraded":"2023-09-23T21:10:36.679838+0000","last_fullsized":"2023-09-23T21:10:36.679838+0000","mapping_epoch":396,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":397,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:51:39.763421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,5],"acting":[3,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"79'161","reported_seq":777,"reported_epoch":434,"state":"active+clean","last_fresh":"2023-09-23T21:10:36.677713+0000","last_change":"2023-09-23T21:10:29.812970+0000","last_active":"2023-09-23T21:10:36.677713+0000","last_peered":"2023-09-23T21:10:36.677713+0000","last_clean":"2023-09-23T21:10:36.677713+0000","last_became_active":"2023-09-23T21:10:29.812616+0000","last_became_peered":"2023-09-23T21:10:29.812616+0000","last_unstale":"2023-09-23T21:10:36.677713+0000","last_undegraded":"2023-09-23T21:10:36.677713+0000","last_fullsized":"2023-09-23T21:10:36.677713+0000","mapping_epoch":426,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":427,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:25.096401+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:25.096401+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:40:11.302067+0000","scrub_duration":0.002859393,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,1],"acting":[2,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":149,"reported_epoch":434,"state":"active+clean","last_fresh":"2023-09-23T21:10:36.678862+0000","last_change":"2023-09-23T21:10:26.755892+0000","last_active":"2023-09-23T21:10:36.678862+0000","last_peered":"2023-09-23T21:10:36.678862+0000","last_clean":"2023-09-23T21:10:36.678862+0000","last_became_active":"2023-09-23T21:10:26.755704+0000","last_became_peered":"2023-09-23T21:10:26.755704+0000","last_unstale":"2023-09-23T21:10:36.678862+0000","last_undegraded":"2023-09-23T21:10:36.678862+0000","last_fullsized":"2023-09-23T21:10:36.678862+0000","mapping_epoch":423,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":424,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:07:38.975955+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":408,"reported_epoch":434,"state":"active+clean","last_fresh":"2023-09-23T21:10:36.680574+0000","last_change":"2023-09-23T21:10:26.758958+0000","last_active":"2023-09-23T21:10:36.680574+0000","last_peered":"2023-09-23T21:10:36.680574+0000","last_clean":"2023-09-23T21:10:36.680574+0000","last_became_active":"2023-09-23T21:10:26.758539+0000","last_became_peered":"2023-09-23T21:10:26.758539+0000","last_unstale":"2023-09-23T21:10:36.680574+0000","last_undegraded":"2023-09-23T21:10:36.680574+0000","last_fullsized":"2023-09-23T21:10:36.680574+0000","mapping_epoch":423,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":424,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:44:42.319189+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":302,"reported_epoch":434,"state":"active+clean","last_fresh":"2023-09-23T21:10:36.681571+0000","last_change":"2023-09-23T21:10:26.776257+0000","last_active":"2023-09-23T21:10:36.681571+0000","last_peered":"2023-09-23T21:10:36.681571+0000","last_clean":"2023-09-23T21:10:36.681571+0000","last_became_active":"2023-09-23T21:10:26.763126+0000","last_became_peered":"2023-09-23T21:10:26.763126+0000","last_unstale":"2023-09-23T21:10:36.681571+0000","last_undegraded":"2023-09-23T21:10:36.681571+0000","last_fullsized":"2023-09-23T21:10:36.681571+0000","mapping_epoch":423,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":424,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:39:01.350573+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":270,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":118,"reported_epoch":434,"state":"active+clean","last_fresh":"2023-09-23T21:10:36.674985+0000","last_change":"2023-09-23T21:08:54.693560+0000","last_active":"2023-09-23T21:10:36.674985+0000","last_peered":"2023-09-23T21:10:36.674985+0000","last_clean":"2023-09-23T21:10:36.674985+0000","last_became_active":"2023-09-23T21:08:54.693199+0000","last_became_peered":"2023-09-23T21:08:54.693199+0000","last_unstale":"2023-09-23T21:10:36.674985+0000","last_undegraded":"2023-09-23T21:10:36.674985+0000","last_fullsized":"2023-09-23T21:10:36.674985+0000","mapping_epoch":388,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":389,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:09:53.015332+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"79'22","reported_seq":31,"reported_epoch":434,"state":"active+clean","last_fresh":"2023-09-23T21:10:36.680791+0000","last_change":"2023-09-23T21:10:26.787894+0000","last_active":"2023-09-23T21:10:36.680791+0000","last_peered":"2023-09-23T21:10:36.680791+0000","last_clean":"2023-09-23T21:10:36.680791+0000","last_became_active":"2023-09-23T21:10:26.769264+0000","last_became_peered":"2023-09-23T21:10:26.769264+0000","last_unstale":"2023-09-23T21:10:36.680791+0000","last_undegraded":"2023-09-23T21:10:36.680791+0000","last_fullsized":"2023-09-23T21:10:36.680791+0000","mapping_epoch":423,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":424,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:49:19.368169+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4734,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,0],"acting":[7,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":356,"reported_epoch":434,"state":"active+clean","last_fresh":"2023-09-23T21:10:36.676733+0000","last_change":"2023-09-23T21:10:16.702475+0000","last_active":"2023-09-23T21:10:36.676733+0000","last_peered":"2023-09-23T21:10:36.676733+0000","last_clean":"2023-09-23T21:10:36.676733+0000","last_became_active":"2023-09-23T21:10:16.702239+0000","last_became_peered":"2023-09-23T21:10:16.702239+0000","last_unstale":"2023-09-23T21:10:36.676733+0000","last_undegraded":"2023-09-23T21:10:36.676733+0000","last_fullsized":"2023-09-23T21:10:36.676733+0000","mapping_epoch":416,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":417,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:12:54.861337+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.11","version":"79'161","reported_seq":702,"reported_epoch":434,"state":"active+clean","last_fresh":"2023-09-23T21:10:36.677463+0000","last_change":"2023-09-23T21:05:07.301125+0000","last_active":"2023-09-23T21:10:36.677463+0000","last_peered":"2023-09-23T21:10:36.677463+0000","last_clean":"2023-09-23T21:10:36.677463+0000","last_became_active":"2023-09-23T21:05:07.300809+0000","last_became_peered":"2023-09-23T21:05:07.300809+0000","last_unstale":"2023-09-23T21:10:36.677463+0000","last_undegraded":"2023-09-23T21:10:36.677463+0000","last_fullsized":"2023-09-23T21:10:36.677463+0000","mapping_epoch":335,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":336,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:28.756634+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:28.756634+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:28:29.079665+0000","scrub_duration":0.0064925119999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":27,"reported_epoch":434,"state":"active+clean","last_fresh":"2023-09-23T21:10:36.680752+0000","last_change":"2023-09-23T21:10:26.782909+0000","last_active":"2023-09-23T21:10:36.680752+0000","last_peered":"2023-09-23T21:10:36.680752+0000","last_clean":"2023-09-23T21:10:36.680752+0000","last_became_active":"2023-09-23T21:10:26.768929+0000","last_became_peered":"2023-09-23T21:10:26.768929+0000","last_unstale":"2023-09-23T21:10:36.680752+0000","last_undegraded":"2023-09-23T21:10:36.680752+0000","last_fullsized":"2023-09-23T21:10:36.680752+0000","mapping_epoch":423,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":424,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:04:49.554395+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,0],"acting":[7,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":431,"reported_epoch":434,"state":"active+clean","last_fresh":"2023-09-23T21:10:38.613320+0000","last_change":"2023-09-23T21:10:36.684387+0000","last_active":"2023-09-23T21:10:38.613320+0000","last_peered":"2023-09-23T21:10:38.613320+0000","last_clean":"2023-09-23T21:10:38.613320+0000","last_became_active":"2023-09-23T21:10:36.684052+0000","last_became_peered":"2023-09-23T21:10:36.684052+0000","last_unstale":"2023-09-23T21:10:38.613320+0000","last_undegraded":"2023-09-23T21:10:38.613320+0000","last_fullsized":"2023-09-23T21:10:38.613320+0000","mapping_epoch":433,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":434,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:16:50.207053+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.10","version":"79'161","reported_seq":561,"reported_epoch":434,"state":"active+clean","last_fresh":"2023-09-23T21:10:36.677383+0000","last_change":"2023-09-23T21:10:29.815249+0000","last_active":"2023-09-23T21:10:36.677383+0000","last_peered":"2023-09-23T21:10:36.677383+0000","last_clean":"2023-09-23T21:10:36.677383+0000","last_became_active":"2023-09-23T21:10:29.814897+0000","last_became_peered":"2023-09-23T21:10:29.814897+0000","last_unstale":"2023-09-23T21:10:36.677383+0000","last_undegraded":"2023-09-23T21:10:36.677383+0000","last_fullsized":"2023-09-23T21:10:36.677383+0000","mapping_epoch":426,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":427,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:06:50.336992+0000","last_deep_scrub":"79'161","last_deep_scrub_stamp":"2023-09-23T21:06:50.336992+0000","last_clean_scrub_stamp":"2023-09-23T21:06:50.336992+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:47:12.289477+0000","scrub_duration":0.0026150930000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":433,"reported_epoch":434,"state":"active+clean","last_fresh":"2023-09-23T21:10:36.682051+0000","last_change":"2023-09-23T21:10:26.765150+0000","last_active":"2023-09-23T21:10:36.682051+0000","last_peered":"2023-09-23T21:10:36.682051+0000","last_clean":"2023-09-23T21:10:36.682051+0000","last_became_active":"2023-09-23T21:10:26.764831+0000","last_became_peered":"2023-09-23T21:10:26.764831+0000","last_unstale":"2023-09-23T21:10:36.682051+0000","last_undegraded":"2023-09-23T21:10:36.682051+0000","last_fullsized":"2023-09-23T21:10:36.682051+0000","mapping_epoch":423,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":424,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:53:29.260075+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":185,"reported_epoch":434,"state":"active+clean","last_fresh":"2023-09-23T21:10:36.681451+0000","last_change":"2023-09-23T21:10:26.758671+0000","last_active":"2023-09-23T21:10:36.681451+0000","last_peered":"2023-09-23T21:10:36.681451+0000","last_clean":"2023-09-23T21:10:36.681451+0000","last_became_active":"2023-09-23T21:10:26.758244+0000","last_became_peered":"2023-09-23T21:10:26.758244+0000","last_unstale":"2023-09-23T21:10:36.681451+0000","last_undegraded":"2023-09-23T21:10:36.681451+0000","last_fullsized":"2023-09-23T21:10:36.681451+0000","mapping_epoch":423,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":424,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:14:02.735609+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.13","version":"79'161","reported_seq":727,"reported_epoch":434,"state":"active+clean","last_fresh":"2023-09-23T21:10:36.673955+0000","last_change":"2023-09-23T21:09:11.416774+0000","last_active":"2023-09-23T21:10:36.673955+0000","last_peered":"2023-09-23T21:10:36.673955+0000","last_clean":"2023-09-23T21:10:36.673955+0000","last_became_active":"2023-09-23T21:09:11.416431+0000","last_became_peered":"2023-09-23T21:09:11.416431+0000","last_unstale":"2023-09-23T21:10:36.673955+0000","last_undegraded":"2023-09-23T21:10:36.673955+0000","last_fullsized":"2023-09-23T21:10:36.673955+0000","mapping_epoch":399,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":400,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:36.420362+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:36.420362+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:51:54.917367+0000","scrub_duration":0.028838444000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,0],"acting":[1,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":409,"reported_epoch":434,"state":"active+clean","last_fresh":"2023-09-23T21:10:36.676492+0000","last_change":"2023-09-23T21:10:26.755199+0000","last_active":"2023-09-23T21:10:36.676492+0000","last_peered":"2023-09-23T21:10:36.676492+0000","last_clean":"2023-09-23T21:10:36.676492+0000","last_became_active":"2023-09-23T21:10:26.754893+0000","last_became_peered":"2023-09-23T21:10:26.754893+0000","last_unstale":"2023-09-23T21:10:36.676492+0000","last_undegraded":"2023-09-23T21:10:36.676492+0000","last_fullsized":"2023-09-23T21:10:36.676492+0000","mapping_epoch":423,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":424,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:08:10.989913+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":414,"reported_epoch":434,"state":"active+clean","last_fresh":"2023-09-23T21:10:36.682238+0000","last_change":"2023-09-23T21:10:26.765465+0000","last_active":"2023-09-23T21:10:36.682238+0000","last_peered":"2023-09-23T21:10:36.682238+0000","last_clean":"2023-09-23T21:10:36.682238+0000","last_became_active":"2023-09-23T21:10:26.765083+0000","last_became_peered":"2023-09-23T21:10:26.765083+0000","last_unstale":"2023-09-23T21:10:36.682238+0000","last_undegraded":"2023-09-23T21:10:36.682238+0000","last_fullsized":"2023-09-23T21:10:36.682238+0000","mapping_epoch":423,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":424,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:29:18.642527+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.12","version":"79'161","reported_seq":636,"reported_epoch":434,"state":"active+clean","last_fresh":"2023-09-23T21:10:36.678349+0000","last_change":"2023-09-23T21:10:29.809555+0000","last_active":"2023-09-23T21:10:36.678349+0000","last_peered":"2023-09-23T21:10:36.678349+0000","last_clean":"2023-09-23T21:10:36.678349+0000","last_became_active":"2023-09-23T21:10:29.809226+0000","last_became_peered":"2023-09-23T21:10:29.809226+0000","last_unstale":"2023-09-23T21:10:36.678349+0000","last_undegraded":"2023-09-23T21:10:36.678349+0000","last_fullsized":"2023-09-23T21:10:36.678349+0000","mapping_epoch":426,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":427,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:05:56.785173+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:05:56.785173+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:55:37.757367+0000","scrub_duration":0.0042114370000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,7],"acting":[5,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":71,"reported_epoch":434,"state":"active+clean","last_fresh":"2023-09-23T21:10:36.677667+0000","last_change":"2023-09-23T21:10:16.727441+0000","last_active":"2023-09-23T21:10:36.677667+0000","last_peered":"2023-09-23T21:10:36.677667+0000","last_clean":"2023-09-23T21:10:36.677667+0000","last_became_active":"2023-09-23T21:10:16.721104+0000","last_became_peered":"2023-09-23T21:10:16.721104+0000","last_unstale":"2023-09-23T21:10:36.677667+0000","last_undegraded":"2023-09-23T21:10:36.677667+0000","last_fullsized":"2023-09-23T21:10:36.677667+0000","mapping_epoch":416,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":417,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:47:48.699168+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":261,"reported_epoch":434,"state":"active+clean","last_fresh":"2023-09-23T21:10:36.681210+0000","last_change":"2023-09-23T21:10:26.769215+0000","last_active":"2023-09-23T21:10:36.681210+0000","last_peered":"2023-09-23T21:10:36.681210+0000","last_clean":"2023-09-23T21:10:36.681210+0000","last_became_active":"2023-09-23T21:10:26.768888+0000","last_became_peered":"2023-09-23T21:10:26.768888+0000","last_unstale":"2023-09-23T21:10:36.681210+0000","last_undegraded":"2023-09-23T21:10:36.681210+0000","last_fullsized":"2023-09-23T21:10:36.681210+0000","mapping_epoch":423,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":424,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:58:57.402503+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.6","version":"435'1981","reported_seq":2009,"reported_epoch":435,"state":"active+clean","last_fresh":"2023-09-23T21:10:47.405378+0000","last_change":"2023-09-23T21:10:16.711752+0000","last_active":"2023-09-23T21:10:47.405378+0000","last_peered":"2023-09-23T21:10:47.405378+0000","last_clean":"2023-09-23T21:10:47.405378+0000","last_became_active":"2023-09-23T21:10:16.711436+0000","last_became_peered":"2023-09-23T21:10:16.711436+0000","last_unstale":"2023-09-23T21:10:47.405378+0000","last_undegraded":"2023-09-23T21:10:47.405378+0000","last_fullsized":"2023-09-23T21:10:47.405378+0000","mapping_epoch":416,"log_start":"0'0","ondisk_log_start":"0'0","created":416,"last_epoch_clean":417,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_clean_scrub_stamp":"2023-09-23T21:10:15.663179+0000","objects_scrubbed":0,"log_size":1981,"log_dups_size":0,"ondisk_log_size":1981,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:03:30.757178+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":129826816,"num_objects":1981,"num_object_clones":0,"num_object_copies":5943,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1981,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1981,"num_write_kb":126784,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,1],"acting":[0,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.d","version":"79'161","reported_seq":548,"reported_epoch":434,"state":"active+clean","last_fresh":"2023-09-23T21:10:36.680065+0000","last_change":"2023-09-23T21:10:29.812819+0000","last_active":"2023-09-23T21:10:36.680065+0000","last_peered":"2023-09-23T21:10:36.680065+0000","last_clean":"2023-09-23T21:10:36.680065+0000","last_became_active":"2023-09-23T21:10:29.812535+0000","last_became_peered":"2023-09-23T21:10:29.812535+0000","last_unstale":"2023-09-23T21:10:36.680065+0000","last_undegraded":"2023-09-23T21:10:36.680065+0000","last_fullsized":"2023-09-23T21:10:36.680065+0000","mapping_epoch":426,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":427,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:09:36.858077+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:09:36.858077+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:10:25.031595+0000","scrub_duration":0.0024987500000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,2],"acting":[7,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":25,"reported_epoch":434,"state":"active+clean","last_fresh":"2023-09-23T21:10:36.680454+0000","last_change":"2023-09-23T21:10:26.770251+0000","last_active":"2023-09-23T21:10:36.680454+0000","last_peered":"2023-09-23T21:10:36.680454+0000","last_clean":"2023-09-23T21:10:36.680454+0000","last_became_active":"2023-09-23T21:10:26.760228+0000","last_became_peered":"2023-09-23T21:10:26.760228+0000","last_unstale":"2023-09-23T21:10:36.680454+0000","last_undegraded":"2023-09-23T21:10:36.680454+0000","last_fullsized":"2023-09-23T21:10:36.680454+0000","mapping_epoch":423,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":424,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:59:46.323722+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":142,"reported_epoch":434,"state":"active+clean","last_fresh":"2023-09-23T21:10:36.677938+0000","last_change":"2023-09-23T21:10:26.750725+0000","last_active":"2023-09-23T21:10:36.677938+0000","last_peered":"2023-09-23T21:10:36.677938+0000","last_clean":"2023-09-23T21:10:36.677938+0000","last_became_active":"2023-09-23T21:10:26.750307+0000","last_became_peered":"2023-09-23T21:10:26.750307+0000","last_unstale":"2023-09-23T21:10:36.677938+0000","last_undegraded":"2023-09-23T21:10:36.677938+0000","last_fullsized":"2023-09-23T21:10:36.677938+0000","mapping_epoch":423,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":424,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:06:22.040548+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,4],"acting":[5,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.7","version":"435'2124","reported_seq":2152,"reported_epoch":435,"state":"active+clean","last_fresh":"2023-09-23T21:10:49.127285+0000","last_change":"2023-09-23T21:10:16.720164+0000","last_active":"2023-09-23T21:10:49.127285+0000","last_peered":"2023-09-23T21:10:49.127285+0000","last_clean":"2023-09-23T21:10:49.127285+0000","last_became_active":"2023-09-23T21:10:16.719565+0000","last_became_peered":"2023-09-23T21:10:16.719565+0000","last_unstale":"2023-09-23T21:10:49.127285+0000","last_undegraded":"2023-09-23T21:10:49.127285+0000","last_fullsized":"2023-09-23T21:10:49.127285+0000","mapping_epoch":416,"log_start":"0'0","ondisk_log_start":"0'0","created":416,"last_epoch_clean":417,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_clean_scrub_stamp":"2023-09-23T21:10:15.663179+0000","objects_scrubbed":0,"log_size":2124,"log_dups_size":0,"ondisk_log_size":2124,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:29:17.753258+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":139198464,"num_objects":2124,"num_object_clones":0,"num_object_copies":6372,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2124,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2124,"num_write_kb":135936,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,2],"acting":[3,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.c","version":"79'161","reported_seq":762,"reported_epoch":434,"state":"active+clean","last_fresh":"2023-09-23T21:10:39.125611+0000","last_change":"2023-09-23T21:10:19.696713+0000","last_active":"2023-09-23T21:10:39.125611+0000","last_peered":"2023-09-23T21:10:39.125611+0000","last_clean":"2023-09-23T21:10:39.125611+0000","last_became_active":"2023-09-23T21:10:19.696283+0000","last_became_peered":"2023-09-23T21:10:19.696283+0000","last_unstale":"2023-09-23T21:10:39.125611+0000","last_undegraded":"2023-09-23T21:10:39.125611+0000","last_fullsized":"2023-09-23T21:10:39.125611+0000","mapping_epoch":419,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":420,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:22.040907+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:22.040907+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:27:44.589354+0000","scrub_duration":0.013021176000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":327680,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":25,"reported_epoch":434,"state":"active+clean","last_fresh":"2023-09-23T21:10:36.680468+0000","last_change":"2023-09-23T21:10:26.778191+0000","last_active":"2023-09-23T21:10:36.680468+0000","last_peered":"2023-09-23T21:10:36.680468+0000","last_clean":"2023-09-23T21:10:36.680468+0000","last_became_active":"2023-09-23T21:10:26.763308+0000","last_became_peered":"2023-09-23T21:10:26.763308+0000","last_unstale":"2023-09-23T21:10:36.680468+0000","last_undegraded":"2023-09-23T21:10:36.680468+0000","last_fullsized":"2023-09-23T21:10:36.680468+0000","mapping_epoch":423,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":424,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:22:42.213658+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":373,"reported_epoch":434,"state":"active+clean","last_fresh":"2023-09-23T21:10:36.676525+0000","last_change":"2023-09-23T21:05:55.325375+0000","last_active":"2023-09-23T21:10:36.676525+0000","last_peered":"2023-09-23T21:10:36.676525+0000","last_clean":"2023-09-23T21:10:36.676525+0000","last_became_active":"2023-09-23T21:05:55.324170+0000","last_became_peered":"2023-09-23T21:05:55.324170+0000","last_unstale":"2023-09-23T21:10:36.676525+0000","last_undegraded":"2023-09-23T21:10:36.676525+0000","last_fullsized":"2023-09-23T21:10:36.676525+0000","mapping_epoch":350,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":351,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:08:07.299605+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.4","version":"435'1861","reported_seq":1889,"reported_epoch":435,"state":"active+clean","last_fresh":"2023-09-23T21:10:47.411567+0000","last_change":"2023-09-23T21:10:16.713791+0000","last_active":"2023-09-23T21:10:47.411567+0000","last_peered":"2023-09-23T21:10:47.411567+0000","last_clean":"2023-09-23T21:10:47.411567+0000","last_became_active":"2023-09-23T21:10:16.713383+0000","last_became_peered":"2023-09-23T21:10:16.713383+0000","last_unstale":"2023-09-23T21:10:47.411567+0000","last_undegraded":"2023-09-23T21:10:47.411567+0000","last_fullsized":"2023-09-23T21:10:47.411567+0000","mapping_epoch":416,"log_start":"0'0","ondisk_log_start":"0'0","created":416,"last_epoch_clean":417,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_clean_scrub_stamp":"2023-09-23T21:10:15.663179+0000","objects_scrubbed":0,"log_size":1861,"log_dups_size":0,"ondisk_log_size":1861,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:21:29.998071+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":121962496,"num_objects":1861,"num_object_clones":0,"num_object_copies":5583,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1861,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1861,"num_write_kb":119104,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.f","version":"79'161","reported_seq":702,"reported_epoch":434,"state":"active+clean","last_fresh":"2023-09-23T21:10:36.678382+0000","last_change":"2023-09-23T21:10:19.697584+0000","last_active":"2023-09-23T21:10:36.678382+0000","last_peered":"2023-09-23T21:10:36.678382+0000","last_clean":"2023-09-23T21:10:36.678382+0000","last_became_active":"2023-09-23T21:10:19.697212+0000","last_became_peered":"2023-09-23T21:10:19.697212+0000","last_unstale":"2023-09-23T21:10:36.678382+0000","last_undegraded":"2023-09-23T21:10:36.678382+0000","last_fullsized":"2023-09-23T21:10:36.678382+0000","mapping_epoch":419,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":420,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:27.880776+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:27.880776+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:16:37.642483+0000","scrub_duration":0.0014004899999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":158,"reported_epoch":434,"state":"active+clean","last_fresh":"2023-09-23T21:10:36.678405+0000","last_change":"2023-09-23T21:10:26.765068+0000","last_active":"2023-09-23T21:10:36.678405+0000","last_peered":"2023-09-23T21:10:36.678405+0000","last_clean":"2023-09-23T21:10:36.678405+0000","last_became_active":"2023-09-23T21:10:26.758364+0000","last_became_peered":"2023-09-23T21:10:26.758364+0000","last_unstale":"2023-09-23T21:10:36.678405+0000","last_undegraded":"2023-09-23T21:10:36.678405+0000","last_fullsized":"2023-09-23T21:10:36.678405+0000","mapping_epoch":423,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":424,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:13:21.057529+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":280,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":407,"reported_epoch":434,"state":"active+clean","last_fresh":"2023-09-23T21:10:36.674906+0000","last_change":"2023-09-23T21:10:26.756554+0000","last_active":"2023-09-23T21:10:36.674906+0000","last_peered":"2023-09-23T21:10:36.674906+0000","last_clean":"2023-09-23T21:10:36.674906+0000","last_became_active":"2023-09-23T21:10:26.756233+0000","last_became_peered":"2023-09-23T21:10:26.756233+0000","last_unstale":"2023-09-23T21:10:36.674906+0000","last_undegraded":"2023-09-23T21:10:36.674906+0000","last_fullsized":"2023-09-23T21:10:36.674906+0000","mapping_epoch":423,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":424,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:15:05.539375+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,5],"acting":[1,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.5","version":"435'1786","reported_seq":1814,"reported_epoch":435,"state":"active+clean","last_fresh":"2023-09-23T21:10:46.079492+0000","last_change":"2023-09-23T21:10:16.717950+0000","last_active":"2023-09-23T21:10:46.079492+0000","last_peered":"2023-09-23T21:10:46.079492+0000","last_clean":"2023-09-23T21:10:46.079492+0000","last_became_active":"2023-09-23T21:10:16.717647+0000","last_became_peered":"2023-09-23T21:10:16.717647+0000","last_unstale":"2023-09-23T21:10:46.079492+0000","last_undegraded":"2023-09-23T21:10:46.079492+0000","last_fullsized":"2023-09-23T21:10:46.079492+0000","mapping_epoch":416,"log_start":"0'0","ondisk_log_start":"0'0","created":416,"last_epoch_clean":417,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_clean_scrub_stamp":"2023-09-23T21:10:15.663179+0000","objects_scrubbed":0,"log_size":1786,"log_dups_size":0,"ondisk_log_size":1786,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:28:11.972698+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":117047296,"num_objects":1786,"num_object_clones":0,"num_object_copies":5358,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1786,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1786,"num_write_kb":114304,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,3],"acting":[2,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.e","version":"79'161","reported_seq":447,"reported_epoch":434,"state":"active+clean","last_fresh":"2023-09-23T21:10:39.567412+0000","last_change":"2023-09-23T21:10:39.567343+0000","last_active":"2023-09-23T21:10:39.567412+0000","last_peered":"2023-09-23T21:10:39.567412+0000","last_clean":"2023-09-23T21:10:39.567412+0000","last_became_active":"2023-09-23T21:10:29.812583+0000","last_became_peered":"2023-09-23T21:10:29.812583+0000","last_unstale":"2023-09-23T21:10:39.567412+0000","last_undegraded":"2023-09-23T21:10:39.567412+0000","last_fullsized":"2023-09-23T21:10:39.567412+0000","mapping_epoch":426,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":427,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:10:39.567232+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:10:39.567232+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:31:46.167827+0000","scrub_duration":0.0051659669999999996,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":210,"reported_epoch":434,"state":"active+clean","last_fresh":"2023-09-23T21:10:36.680245+0000","last_change":"2023-09-23T21:10:16.718749+0000","last_active":"2023-09-23T21:10:36.680245+0000","last_peered":"2023-09-23T21:10:36.680245+0000","last_clean":"2023-09-23T21:10:36.680245+0000","last_became_active":"2023-09-23T21:10:16.718020+0000","last_became_peered":"2023-09-23T21:10:16.718020+0000","last_unstale":"2023-09-23T21:10:36.680245+0000","last_undegraded":"2023-09-23T21:10:36.680245+0000","last_fullsized":"2023-09-23T21:10:36.680245+0000","mapping_epoch":416,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":417,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:33:52.223742+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,1],"acting":[3,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":428,"reported_epoch":434,"state":"active+clean","last_fresh":"2023-09-23T21:10:36.679633+0000","last_change":"2023-09-23T20:54:13.928715+0000","last_active":"2023-09-23T21:10:36.679633+0000","last_peered":"2023-09-23T21:10:36.679633+0000","last_clean":"2023-09-23T21:10:36.679633+0000","last_became_active":"2023-09-23T20:54:13.928200+0000","last_became_peered":"2023-09-23T20:54:13.928200+0000","last_unstale":"2023-09-23T21:10:36.679633+0000","last_undegraded":"2023-09-23T21:10:36.679633+0000","last_fullsized":"2023-09-23T21:10:36.679633+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:41:16.927513+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.2","version":"435'2096","reported_seq":2124,"reported_epoch":435,"state":"active+clean","last_fresh":"2023-09-23T21:10:48.601453+0000","last_change":"2023-09-23T21:10:16.695422+0000","last_active":"2023-09-23T21:10:48.601453+0000","last_peered":"2023-09-23T21:10:48.601453+0000","last_clean":"2023-09-23T21:10:48.601453+0000","last_became_active":"2023-09-23T21:10:16.695143+0000","last_became_peered":"2023-09-23T21:10:16.695143+0000","last_unstale":"2023-09-23T21:10:48.601453+0000","last_undegraded":"2023-09-23T21:10:48.601453+0000","last_fullsized":"2023-09-23T21:10:48.601453+0000","mapping_epoch":416,"log_start":"0'0","ondisk_log_start":"0'0","created":416,"last_epoch_clean":417,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_clean_scrub_stamp":"2023-09-23T21:10:15.663179+0000","objects_scrubbed":0,"log_size":2096,"log_dups_size":0,"ondisk_log_size":2096,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:00:57.177812+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":137363456,"num_objects":2096,"num_object_clones":0,"num_object_copies":6288,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2096,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2096,"num_write_kb":134144,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.9","version":"79'161","reported_seq":578,"reported_epoch":434,"state":"active+clean","last_fresh":"2023-09-23T21:10:37.697046+0000","last_change":"2023-09-23T21:10:37.697046+0000","last_active":"2023-09-23T21:10:37.697046+0000","last_peered":"2023-09-23T21:10:37.697046+0000","last_clean":"2023-09-23T21:10:37.697046+0000","last_became_active":"2023-09-23T21:10:36.684107+0000","last_became_peered":"2023-09-23T21:10:36.684107+0000","last_unstale":"2023-09-23T21:10:37.697046+0000","last_undegraded":"2023-09-23T21:10:37.697046+0000","last_fullsized":"2023-09-23T21:10:37.697046+0000","mapping_epoch":433,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":434,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:10:37.696951+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:10:37.696951+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:41:10.705836+0000","scrub_duration":0.001386504,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,3],"acting":[2,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":32,"reported_epoch":434,"state":"active+clean","last_fresh":"2023-09-23T21:10:37.413209+0000","last_change":"2023-09-23T21:10:36.684921+0000","last_active":"2023-09-23T21:10:37.413209+0000","last_peered":"2023-09-23T21:10:37.413209+0000","last_clean":"2023-09-23T21:10:37.413209+0000","last_became_active":"2023-09-23T21:10:36.684595+0000","last_became_peered":"2023-09-23T21:10:36.684595+0000","last_unstale":"2023-09-23T21:10:37.413209+0000","last_undegraded":"2023-09-23T21:10:37.413209+0000","last_fullsized":"2023-09-23T21:10:37.413209+0000","mapping_epoch":433,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":434,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:09:54.090507+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":130,"reported_epoch":434,"state":"active+clean","last_fresh":"2023-09-23T21:10:38.613256+0000","last_change":"2023-09-23T21:10:36.684330+0000","last_active":"2023-09-23T21:10:38.613256+0000","last_peered":"2023-09-23T21:10:38.613256+0000","last_clean":"2023-09-23T21:10:38.613256+0000","last_became_active":"2023-09-23T21:10:36.684026+0000","last_became_peered":"2023-09-23T21:10:36.684026+0000","last_unstale":"2023-09-23T21:10:38.613256+0000","last_undegraded":"2023-09-23T21:10:38.613256+0000","last_fullsized":"2023-09-23T21:10:38.613256+0000","mapping_epoch":433,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":434,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:20:06.829513+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.3","version":"435'1846","reported_seq":1874,"reported_epoch":435,"state":"active+clean","last_fresh":"2023-09-23T21:10:46.073316+0000","last_change":"2023-09-23T21:10:16.711581+0000","last_active":"2023-09-23T21:10:46.073316+0000","last_peered":"2023-09-23T21:10:46.073316+0000","last_clean":"2023-09-23T21:10:46.073316+0000","last_became_active":"2023-09-23T21:10:16.711284+0000","last_became_peered":"2023-09-23T21:10:16.711284+0000","last_unstale":"2023-09-23T21:10:46.073316+0000","last_undegraded":"2023-09-23T21:10:46.073316+0000","last_fullsized":"2023-09-23T21:10:46.073316+0000","mapping_epoch":416,"log_start":"0'0","ondisk_log_start":"0'0","created":416,"last_epoch_clean":417,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_clean_scrub_stamp":"2023-09-23T21:10:15.663179+0000","objects_scrubbed":0,"log_size":1846,"log_dups_size":0,"ondisk_log_size":1846,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:56:04.130345+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":120979456,"num_objects":1846,"num_object_clones":0,"num_object_copies":5538,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1846,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1846,"num_write_kb":118144,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.8","version":"79'161","reported_seq":544,"reported_epoch":434,"state":"active+clean","last_fresh":"2023-09-23T21:10:36.676382+0000","last_change":"2023-09-23T21:06:52.099024+0000","last_active":"2023-09-23T21:10:36.676382+0000","last_peered":"2023-09-23T21:10:36.676382+0000","last_clean":"2023-09-23T21:10:36.676382+0000","last_became_active":"2023-09-23T21:06:49.425012+0000","last_became_peered":"2023-09-23T21:06:49.425012+0000","last_unstale":"2023-09-23T21:10:36.676382+0000","last_undegraded":"2023-09-23T21:10:36.676382+0000","last_fullsized":"2023-09-23T21:10:36.676382+0000","mapping_epoch":359,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":360,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:06:52.098882+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:06:52.098882+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:48:36.414396+0000","scrub_duration":0.0092173299999999993,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":407,"reported_epoch":434,"state":"active+clean","last_fresh":"2023-09-23T21:10:36.674826+0000","last_change":"2023-09-23T21:10:16.715566+0000","last_active":"2023-09-23T21:10:36.674826+0000","last_peered":"2023-09-23T21:10:36.674826+0000","last_clean":"2023-09-23T21:10:36.674826+0000","last_became_active":"2023-09-23T21:10:16.715218+0000","last_became_peered":"2023-09-23T21:10:16.715218+0000","last_unstale":"2023-09-23T21:10:36.674826+0000","last_undegraded":"2023-09-23T21:10:36.674826+0000","last_fullsized":"2023-09-23T21:10:36.674826+0000","mapping_epoch":416,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":417,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:34:01.785968+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,5],"acting":[1,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":295,"reported_epoch":434,"state":"active+clean","last_fresh":"2023-09-23T21:10:36.678034+0000","last_change":"2023-09-23T21:10:16.702814+0000","last_active":"2023-09-23T21:10:36.678034+0000","last_peered":"2023-09-23T21:10:36.678034+0000","last_clean":"2023-09-23T21:10:36.678034+0000","last_became_active":"2023-09-23T21:10:16.702484+0000","last_became_peered":"2023-09-23T21:10:16.702484+0000","last_unstale":"2023-09-23T21:10:36.678034+0000","last_undegraded":"2023-09-23T21:10:36.678034+0000","last_fullsized":"2023-09-23T21:10:36.678034+0000","mapping_epoch":416,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":417,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:21:26.301395+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,0],"acting":[2,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"10.0","version":"435'2037","reported_seq":3057,"reported_epoch":435,"state":"active+clean","last_fresh":"2023-09-23T21:10:48.453713+0000","last_change":"2023-09-23T21:10:34.690601+0000","last_active":"2023-09-23T21:10:48.453713+0000","last_peered":"2023-09-23T21:10:48.453713+0000","last_clean":"2023-09-23T21:10:48.453713+0000","last_became_active":"2023-09-23T21:10:34.687990+0000","last_became_peered":"2023-09-23T21:10:34.687990+0000","last_unstale":"2023-09-23T21:10:48.453713+0000","last_undegraded":"2023-09-23T21:10:48.453713+0000","last_fullsized":"2023-09-23T21:10:48.453713+0000","mapping_epoch":431,"log_start":"0'0","ondisk_log_start":"0'0","created":416,"last_epoch_clean":432,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_clean_scrub_stamp":"2023-09-23T21:10:15.663179+0000","objects_scrubbed":0,"log_size":2037,"log_dups_size":0,"ondisk_log_size":2037,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:34:34.313312+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":133496832,"num_objects":2037,"num_object_clones":0,"num_object_copies":6111,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2037,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2037,"num_write_kb":130368,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":486,"num_bytes_recovered":31850496,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,0],"acting":[1,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.b","version":"79'161","reported_seq":766,"reported_epoch":434,"state":"active+clean","last_fresh":"2023-09-23T21:10:36.674046+0000","last_change":"2023-09-23T21:05:55.772369+0000","last_active":"2023-09-23T21:10:36.674046+0000","last_peered":"2023-09-23T21:10:36.674046+0000","last_clean":"2023-09-23T21:10:36.674046+0000","last_became_active":"2023-09-23T21:05:55.316405+0000","last_became_peered":"2023-09-23T21:05:55.316405+0000","last_unstale":"2023-09-23T21:10:36.674046+0000","last_undegraded":"2023-09-23T21:10:36.674046+0000","last_fullsized":"2023-09-23T21:10:36.674046+0000","mapping_epoch":350,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":351,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:05:55.772264+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:05:55.772264+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:14:30.697692+0000","scrub_duration":0.001862507,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,5],"acting":[1,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":320,"reported_epoch":434,"state":"active+clean","last_fresh":"2023-09-23T21:10:36.676609+0000","last_change":"2023-09-23T21:10:26.769668+0000","last_active":"2023-09-23T21:10:36.676609+0000","last_peered":"2023-09-23T21:10:36.676609+0000","last_clean":"2023-09-23T21:10:36.676609+0000","last_became_active":"2023-09-23T21:10:26.769333+0000","last_became_peered":"2023-09-23T21:10:26.769333+0000","last_unstale":"2023-09-23T21:10:36.676609+0000","last_undegraded":"2023-09-23T21:10:36.676609+0000","last_fullsized":"2023-09-23T21:10:36.676609+0000","mapping_epoch":423,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":424,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:39:34.218877+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,1],"acting":[2,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":356,"reported_epoch":434,"state":"active+clean","last_fresh":"2023-09-23T21:10:36.676608+0000","last_change":"2023-09-23T21:08:54.688849+0000","last_active":"2023-09-23T21:10:36.676608+0000","last_peered":"2023-09-23T21:10:36.676608+0000","last_clean":"2023-09-23T21:10:36.676608+0000","last_became_active":"2023-09-23T21:08:54.688571+0000","last_became_peered":"2023-09-23T21:08:54.688571+0000","last_unstale":"2023-09-23T21:10:36.676608+0000","last_undegraded":"2023-09-23T21:10:36.676608+0000","last_fullsized":"2023-09-23T21:10:36.676608+0000","mapping_epoch":388,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":389,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:33:07.896205+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.1","version":"435'1925","reported_seq":2591,"reported_epoch":435,"state":"active+clean","last_fresh":"2023-09-23T21:10:47.224641+0000","last_change":"2023-09-23T21:10:33.696852+0000","last_active":"2023-09-23T21:10:47.224641+0000","last_peered":"2023-09-23T21:10:47.224641+0000","last_clean":"2023-09-23T21:10:47.224641+0000","last_became_active":"2023-09-23T21:10:33.686207+0000","last_became_peered":"2023-09-23T21:10:33.686207+0000","last_unstale":"2023-09-23T21:10:47.224641+0000","last_undegraded":"2023-09-23T21:10:47.224641+0000","last_fullsized":"2023-09-23T21:10:47.224641+0000","mapping_epoch":430,"log_start":"0'0","ondisk_log_start":"0'0","created":416,"last_epoch_clean":431,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_clean_scrub_stamp":"2023-09-23T21:10:15.663179+0000","objects_scrubbed":0,"log_size":1925,"log_dups_size":0,"ondisk_log_size":1925,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:10:39.329817+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":126156800,"num_objects":1925,"num_object_clones":0,"num_object_copies":5775,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1925,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1925,"num_write_kb":123200,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":424,"num_bytes_recovered":27787264,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.a","version":"79'161","reported_seq":553,"reported_epoch":434,"state":"active+clean","last_fresh":"2023-09-23T21:10:36.676714+0000","last_change":"2023-09-23T21:10:19.694995+0000","last_active":"2023-09-23T21:10:36.676714+0000","last_peered":"2023-09-23T21:10:36.676714+0000","last_clean":"2023-09-23T21:10:36.676714+0000","last_became_active":"2023-09-23T21:10:19.694636+0000","last_became_peered":"2023-09-23T21:10:19.694636+0000","last_unstale":"2023-09-23T21:10:36.676714+0000","last_undegraded":"2023-09-23T21:10:36.676714+0000","last_fullsized":"2023-09-23T21:10:36.676714+0000","mapping_epoch":419,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":420,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:06:51.075448+0000","last_deep_scrub":"79'161","last_deep_scrub_stamp":"2023-09-23T21:05:32.630292+0000","last_clean_scrub_stamp":"2023-09-23T21:06:51.075448+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:34:07.672523+0000","scrub_duration":0.0054793890000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,0],"acting":[2,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":271,"reported_epoch":435,"state":"active+clean","last_fresh":"2023-09-23T21:10:42.135740+0000","last_change":"2023-09-23T21:10:26.762544+0000","last_active":"2023-09-23T21:10:42.135740+0000","last_peered":"2023-09-23T21:10:42.135740+0000","last_clean":"2023-09-23T21:10:42.135740+0000","last_became_active":"2023-09-23T21:10:26.762187+0000","last_became_peered":"2023-09-23T21:10:26.762187+0000","last_unstale":"2023-09-23T21:10:42.135740+0000","last_undegraded":"2023-09-23T21:10:42.135740+0000","last_fullsized":"2023-09-23T21:10:42.135740+0000","mapping_epoch":423,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":424,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:16:07.181837+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":427,"reported_epoch":434,"state":"active+clean","last_fresh":"2023-09-23T21:10:36.675446+0000","last_change":"2023-09-23T21:10:26.769450+0000","last_active":"2023-09-23T21:10:36.675446+0000","last_peered":"2023-09-23T21:10:36.675446+0000","last_clean":"2023-09-23T21:10:36.675446+0000","last_became_active":"2023-09-23T21:10:26.769147+0000","last_became_peered":"2023-09-23T21:10:26.769147+0000","last_unstale":"2023-09-23T21:10:36.675446+0000","last_undegraded":"2023-09-23T21:10:36.675446+0000","last_fullsized":"2023-09-23T21:10:36.675446+0000","mapping_epoch":423,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":424,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:27:59.188324+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.c","version":"435'2013","reported_seq":3051,"reported_epoch":435,"state":"active+clean","last_fresh":"2023-09-23T21:10:47.224615+0000","last_change":"2023-09-23T21:10:36.695294+0000","last_active":"2023-09-23T21:10:47.224615+0000","last_peered":"2023-09-23T21:10:47.224615+0000","last_clean":"2023-09-23T21:10:47.224615+0000","last_became_active":"2023-09-23T21:10:36.683034+0000","last_became_peered":"2023-09-23T21:10:36.683034+0000","last_unstale":"2023-09-23T21:10:47.224615+0000","last_undegraded":"2023-09-23T21:10:47.224615+0000","last_fullsized":"2023-09-23T21:10:47.224615+0000","mapping_epoch":433,"log_start":"0'0","ondisk_log_start":"0'0","created":416,"last_epoch_clean":434,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_clean_scrub_stamp":"2023-09-23T21:10:15.663179+0000","objects_scrubbed":0,"log_size":2013,"log_dups_size":0,"ondisk_log_size":2013,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:56:40.959275+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":131923968,"num_objects":2013,"num_object_clones":0,"num_object_copies":6039,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2013,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2013,"num_write_kb":128832,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":497,"num_bytes_recovered":32571392,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.7","version":"79'161","reported_seq":567,"reported_epoch":434,"state":"active+clean","last_fresh":"2023-09-23T21:10:36.681158+0000","last_change":"2023-09-23T21:10:29.807491+0000","last_active":"2023-09-23T21:10:36.681158+0000","last_peered":"2023-09-23T21:10:36.681158+0000","last_clean":"2023-09-23T21:10:36.681158+0000","last_became_active":"2023-09-23T21:10:29.807080+0000","last_became_peered":"2023-09-23T21:10:29.807080+0000","last_unstale":"2023-09-23T21:10:36.681158+0000","last_undegraded":"2023-09-23T21:10:36.681158+0000","last_fullsized":"2023-09-23T21:10:36.681158+0000","mapping_epoch":426,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":427,"parent":"0.0","parent_split_bits":3,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:25.916867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:25.916867+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:22:14.525296+0000","scrub_duration":0.001665323,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":431,"reported_epoch":434,"state":"active+clean","last_fresh":"2023-09-23T21:10:36.674376+0000","last_change":"2023-09-23T21:10:26.761724+0000","last_active":"2023-09-23T21:10:36.674376+0000","last_peered":"2023-09-23T21:10:36.674376+0000","last_clean":"2023-09-23T21:10:36.674376+0000","last_became_active":"2023-09-23T21:10:26.761407+0000","last_became_peered":"2023-09-23T21:10:26.761407+0000","last_unstale":"2023-09-23T21:10:36.674376+0000","last_undegraded":"2023-09-23T21:10:36.674376+0000","last_fullsized":"2023-09-23T21:10:36.674376+0000","mapping_epoch":423,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":424,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:50:10.757587+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":112,"reported_epoch":434,"state":"active+clean","last_fresh":"2023-09-23T21:10:36.678333+0000","last_change":"2023-09-23T21:05:04.326889+0000","last_active":"2023-09-23T21:10:36.678333+0000","last_peered":"2023-09-23T21:10:36.678333+0000","last_clean":"2023-09-23T21:10:36.678333+0000","last_became_active":"2023-09-23T21:05:04.326032+0000","last_became_peered":"2023-09-23T21:05:04.326032+0000","last_unstale":"2023-09-23T21:10:36.678333+0000","last_undegraded":"2023-09-23T21:10:36.678333+0000","last_fullsized":"2023-09-23T21:10:36.678333+0000","mapping_epoch":332,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:00:17.507889+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":435,"reported_epoch":434,"state":"active+clean","last_fresh":"2023-09-23T21:10:36.675058+0000","last_change":"2023-09-23T21:09:08.438066+0000","last_active":"2023-09-23T21:10:36.675058+0000","last_peered":"2023-09-23T21:10:36.675058+0000","last_clean":"2023-09-23T21:10:36.675058+0000","last_became_active":"2023-09-23T21:09:08.437786+0000","last_became_peered":"2023-09-23T21:09:08.437786+0000","last_unstale":"2023-09-23T21:10:36.675058+0000","last_undegraded":"2023-09-23T21:10:36.675058+0000","last_fullsized":"2023-09-23T21:10:36.675058+0000","mapping_epoch":396,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":397,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:49:53.623831+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.a","version":"435'1960","reported_seq":1988,"reported_epoch":435,"state":"active+clean","last_fresh":"2023-09-23T21:10:47.405672+0000","last_change":"2023-09-23T21:10:16.690560+0000","last_active":"2023-09-23T21:10:47.405672+0000","last_peered":"2023-09-23T21:10:47.405672+0000","last_clean":"2023-09-23T21:10:47.405672+0000","last_became_active":"2023-09-23T21:10:16.690221+0000","last_became_peered":"2023-09-23T21:10:16.690221+0000","last_unstale":"2023-09-23T21:10:47.405672+0000","last_undegraded":"2023-09-23T21:10:47.405672+0000","last_fullsized":"2023-09-23T21:10:47.405672+0000","mapping_epoch":416,"log_start":"0'0","ondisk_log_start":"0'0","created":416,"last_epoch_clean":417,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_clean_scrub_stamp":"2023-09-23T21:10:15.663179+0000","objects_scrubbed":0,"log_size":1960,"log_dups_size":0,"ondisk_log_size":1960,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:47:06.888677+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":128450560,"num_objects":1960,"num_object_clones":0,"num_object_copies":5880,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1960,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1960,"num_write_kb":125440,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1","version":"79'161","reported_seq":543,"reported_epoch":434,"state":"active+clean","last_fresh":"2023-09-23T21:10:36.677538+0000","last_change":"2023-09-23T21:06:52.576036+0000","last_active":"2023-09-23T21:10:36.677538+0000","last_peered":"2023-09-23T21:10:36.677538+0000","last_clean":"2023-09-23T21:10:36.677538+0000","last_became_active":"2023-09-23T21:06:49.422806+0000","last_became_peered":"2023-09-23T21:06:49.422806+0000","last_unstale":"2023-09-23T21:10:36.677538+0000","last_undegraded":"2023-09-23T21:10:36.677538+0000","last_fullsized":"2023-09-23T21:10:36.677538+0000","mapping_epoch":359,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":360,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:06:52.575900+0000","last_deep_scrub":"79'161","last_deep_scrub_stamp":"2023-09-23T21:05:30.652189+0000","last_clean_scrub_stamp":"2023-09-23T21:06:52.575900+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:59:24.733331+0000","scrub_duration":0.017723466,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":204,"reported_epoch":434,"state":"active+clean","last_fresh":"2023-09-23T21:10:36.674507+0000","last_change":"2023-09-23T21:08:54.699879+0000","last_active":"2023-09-23T21:10:36.674507+0000","last_peered":"2023-09-23T21:10:36.674507+0000","last_clean":"2023-09-23T21:10:36.674507+0000","last_became_active":"2023-09-23T21:08:54.699558+0000","last_became_peered":"2023-09-23T21:08:54.699558+0000","last_unstale":"2023-09-23T21:10:36.674507+0000","last_undegraded":"2023-09-23T21:10:36.674507+0000","last_fullsized":"2023-09-23T21:10:36.674507+0000","mapping_epoch":388,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":389,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:52:11.177493+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":339,"reported_epoch":434,"state":"active+clean","last_fresh":"2023-09-23T21:10:36.676823+0000","last_change":"2023-09-23T21:10:16.702585+0000","last_active":"2023-09-23T21:10:36.676823+0000","last_peered":"2023-09-23T21:10:36.676823+0000","last_clean":"2023-09-23T21:10:36.676823+0000","last_became_active":"2023-09-23T21:10:16.702260+0000","last_became_peered":"2023-09-23T21:10:16.702260+0000","last_unstale":"2023-09-23T21:10:36.676823+0000","last_undegraded":"2023-09-23T21:10:36.676823+0000","last_fullsized":"2023-09-23T21:10:36.676823+0000","mapping_epoch":416,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":417,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:45:49.430300+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":373,"reported_epoch":434,"state":"active+clean","last_fresh":"2023-09-23T21:10:36.674411+0000","last_change":"2023-09-23T21:05:55.320314+0000","last_active":"2023-09-23T21:10:36.674411+0000","last_peered":"2023-09-23T21:10:36.674411+0000","last_clean":"2023-09-23T21:10:36.674411+0000","last_became_active":"2023-09-23T21:05:55.319935+0000","last_became_peered":"2023-09-23T21:05:55.319935+0000","last_unstale":"2023-09-23T21:10:36.674411+0000","last_undegraded":"2023-09-23T21:10:36.674411+0000","last_fullsized":"2023-09-23T21:10:36.674411+0000","mapping_epoch":350,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":351,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:52:28.754534+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.9","version":"435'1861","reported_seq":2856,"reported_epoch":435,"state":"active+clean","last_fresh":"2023-09-23T21:10:46.094941+0000","last_change":"2023-09-23T21:10:33.099915+0000","last_active":"2023-09-23T21:10:46.094941+0000","last_peered":"2023-09-23T21:10:46.094941+0000","last_clean":"2023-09-23T21:10:46.094941+0000","last_became_active":"2023-09-23T21:10:31.684163+0000","last_became_peered":"2023-09-23T21:10:31.684163+0000","last_unstale":"2023-09-23T21:10:46.094941+0000","last_undegraded":"2023-09-23T21:10:46.094941+0000","last_fullsized":"2023-09-23T21:10:46.094941+0000","mapping_epoch":428,"log_start":"0'0","ondisk_log_start":"0'0","created":416,"last_epoch_clean":430,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_clean_scrub_stamp":"2023-09-23T21:10:15.663179+0000","objects_scrubbed":0,"log_size":1861,"log_dups_size":0,"ondisk_log_size":1861,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:52:38.161708+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":121962496,"num_objects":1861,"num_object_clones":0,"num_object_copies":5583,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1861,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1861,"num_write_kb":119104,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":473,"num_bytes_recovered":30998528,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.2","version":"79'161","reported_seq":721,"reported_epoch":434,"state":"active+clean","last_fresh":"2023-09-23T21:10:36.682140+0000","last_change":"2023-09-23T21:10:29.822693+0000","last_active":"2023-09-23T21:10:36.682140+0000","last_peered":"2023-09-23T21:10:36.682140+0000","last_clean":"2023-09-23T21:10:36.682140+0000","last_became_active":"2023-09-23T21:10:29.822301+0000","last_became_peered":"2023-09-23T21:10:29.822301+0000","last_unstale":"2023-09-23T21:10:36.682140+0000","last_undegraded":"2023-09-23T21:10:36.682140+0000","last_fullsized":"2023-09-23T21:10:36.682140+0000","mapping_epoch":426,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":427,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:24.802451+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:24.802451+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:09:02.755366+0000","scrub_duration":0.0090861250000000005,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,2],"acting":[0,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":426,"reported_epoch":434,"state":"active+clean","last_fresh":"2023-09-23T21:10:36.676811+0000","last_change":"2023-09-23T20:51:10.838054+0000","last_active":"2023-09-23T21:10:36.676811+0000","last_peered":"2023-09-23T21:10:36.676811+0000","last_clean":"2023-09-23T21:10:36.676811+0000","last_became_active":"2023-09-23T20:51:10.837750+0000","last_became_peered":"2023-09-23T20:51:10.837750+0000","last_unstale":"2023-09-23T21:10:36.676811+0000","last_undegraded":"2023-09-23T21:10:36.676811+0000","last_fullsized":"2023-09-23T21:10:36.676811+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:15:51.648085+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":408,"reported_epoch":434,"state":"active+clean","last_fresh":"2023-09-23T21:10:36.673709+0000","last_change":"2023-09-23T21:10:16.697207+0000","last_active":"2023-09-23T21:10:36.673709+0000","last_peered":"2023-09-23T21:10:36.673709+0000","last_clean":"2023-09-23T21:10:36.673709+0000","last_became_active":"2023-09-23T21:10:16.696883+0000","last_became_peered":"2023-09-23T21:10:16.696883+0000","last_unstale":"2023-09-23T21:10:36.673709+0000","last_undegraded":"2023-09-23T21:10:36.673709+0000","last_fullsized":"2023-09-23T21:10:36.673709+0000","mapping_epoch":416,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":417,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:57:29.038857+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":440,"reported_epoch":434,"state":"active+clean","last_fresh":"2023-09-23T21:10:36.675579+0000","last_change":"2023-09-23T21:05:55.318863+0000","last_active":"2023-09-23T21:10:36.675579+0000","last_peered":"2023-09-23T21:10:36.675579+0000","last_clean":"2023-09-23T21:10:36.675579+0000","last_became_active":"2023-09-23T21:05:55.318489+0000","last_became_peered":"2023-09-23T21:05:55.318489+0000","last_unstale":"2023-09-23T21:10:36.675579+0000","last_undegraded":"2023-09-23T21:10:36.675579+0000","last_fullsized":"2023-09-23T21:10:36.675579+0000","mapping_epoch":350,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":351,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:45:05.912661+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.b","version":"435'2055","reported_seq":3019,"reported_epoch":435,"state":"active+clean","last_fresh":"2023-09-23T21:10:48.483419+0000","last_change":"2023-09-23T21:10:32.688759+0000","last_active":"2023-09-23T21:10:48.483419+0000","last_peered":"2023-09-23T21:10:48.483419+0000","last_clean":"2023-09-23T21:10:48.483419+0000","last_became_active":"2023-09-23T21:10:32.686253+0000","last_became_peered":"2023-09-23T21:10:32.686253+0000","last_unstale":"2023-09-23T21:10:48.483419+0000","last_undegraded":"2023-09-23T21:10:48.483419+0000","last_fullsized":"2023-09-23T21:10:48.483419+0000","mapping_epoch":429,"log_start":"0'0","ondisk_log_start":"0'0","created":416,"last_epoch_clean":430,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_clean_scrub_stamp":"2023-09-23T21:10:15.663179+0000","objects_scrubbed":0,"log_size":2055,"log_dups_size":0,"ondisk_log_size":2055,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:51:29.302943+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":134676480,"num_objects":2055,"num_object_clones":0,"num_object_copies":6165,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2055,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2055,"num_write_kb":131520,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":458,"num_bytes_recovered":30015488,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":218,"reported_epoch":434,"state":"active+clean","last_fresh":"2023-09-23T21:10:36.677108+0000","last_change":"2023-09-23T21:10:26.769764+0000","last_active":"2023-09-23T21:10:36.677108+0000","last_peered":"2023-09-23T21:10:36.677108+0000","last_clean":"2023-09-23T21:10:36.677108+0000","last_became_active":"2023-09-23T21:10:26.769326+0000","last_became_peered":"2023-09-23T21:10:26.769326+0000","last_unstale":"2023-09-23T21:10:36.677108+0000","last_undegraded":"2023-09-23T21:10:36.677108+0000","last_fullsized":"2023-09-23T21:10:36.677108+0000","mapping_epoch":423,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":424,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:57:45.219837+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"79'161","reported_seq":722,"reported_epoch":435,"state":"active+clean","last_fresh":"2023-09-23T21:10:42.140082+0000","last_change":"2023-09-23T21:10:29.813770+0000","last_active":"2023-09-23T21:10:42.140082+0000","last_peered":"2023-09-23T21:10:42.140082+0000","last_clean":"2023-09-23T21:10:42.140082+0000","last_became_active":"2023-09-23T21:10:29.813448+0000","last_became_peered":"2023-09-23T21:10:29.813448+0000","last_unstale":"2023-09-23T21:10:42.140082+0000","last_undegraded":"2023-09-23T21:10:42.140082+0000","last_fullsized":"2023-09-23T21:10:42.140082+0000","mapping_epoch":426,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":427,"parent":"0.0","parent_split_bits":0,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:20.738273+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:20.738273+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:47:59.399121+0000","scrub_duration":0.0088336339999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":432,"reported_epoch":434,"state":"active+clean","last_fresh":"2023-09-23T21:10:36.681490+0000","last_change":"2023-09-23T21:10:36.681307+0000","last_active":"2023-09-23T21:10:36.681490+0000","last_peered":"2023-09-23T21:10:36.681490+0000","last_clean":"2023-09-23T21:10:36.681490+0000","last_became_active":"2023-09-23T21:10:36.681091+0000","last_became_peered":"2023-09-23T21:10:36.681091+0000","last_unstale":"2023-09-23T21:10:36.681490+0000","last_undegraded":"2023-09-23T21:10:36.681490+0000","last_fullsized":"2023-09-23T21:10:36.681490+0000","mapping_epoch":433,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":434,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:19:15.538356+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,1],"acting":[5,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":21,"reported_epoch":434,"state":"active+clean","last_fresh":"2023-09-23T21:10:36.680192+0000","last_change":"2023-09-23T21:10:26.759712+0000","last_active":"2023-09-23T21:10:36.680192+0000","last_peered":"2023-09-23T21:10:36.680192+0000","last_clean":"2023-09-23T21:10:36.680192+0000","last_became_active":"2023-09-23T21:10:26.759419+0000","last_became_peered":"2023-09-23T21:10:26.759419+0000","last_unstale":"2023-09-23T21:10:36.680192+0000","last_undegraded":"2023-09-23T21:10:36.680192+0000","last_fullsized":"2023-09-23T21:10:36.680192+0000","mapping_epoch":423,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":424,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:05:03.664183+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"10.8","version":"435'1937","reported_seq":2907,"reported_epoch":435,"state":"active+clean","last_fresh":"2023-09-23T21:10:47.393893+0000","last_change":"2023-09-23T21:10:35.688323+0000","last_active":"2023-09-23T21:10:47.393893+0000","last_peered":"2023-09-23T21:10:47.393893+0000","last_clean":"2023-09-23T21:10:47.393893+0000","last_became_active":"2023-09-23T21:10:35.685937+0000","last_became_peered":"2023-09-23T21:10:35.685937+0000","last_unstale":"2023-09-23T21:10:47.393893+0000","last_undegraded":"2023-09-23T21:10:47.393893+0000","last_fullsized":"2023-09-23T21:10:47.393893+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":416,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_clean_scrub_stamp":"2023-09-23T21:10:15.663179+0000","objects_scrubbed":0,"log_size":1937,"log_dups_size":0,"ondisk_log_size":1937,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:52:28.602228+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":126943232,"num_objects":1937,"num_object_clones":0,"num_object_copies":5811,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1937,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1937,"num_write_kb":123968,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":462,"num_bytes_recovered":30277632,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,7],"acting":[0,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.3","version":"79'161","reported_seq":713,"reported_epoch":434,"state":"active+clean","last_fresh":"2023-09-23T21:10:36.674203+0000","last_change":"2023-09-23T21:09:11.396811+0000","last_active":"2023-09-23T21:10:36.674203+0000","last_peered":"2023-09-23T21:10:36.674203+0000","last_clean":"2023-09-23T21:10:36.674203+0000","last_became_active":"2023-09-23T21:09:11.396462+0000","last_became_peered":"2023-09-23T21:09:11.396462+0000","last_unstale":"2023-09-23T21:10:36.674203+0000","last_undegraded":"2023-09-23T21:10:36.674203+0000","last_fullsized":"2023-09-23T21:10:36.674203+0000","mapping_epoch":399,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":400,"parent":"0.0","parent_split_bits":3,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:05:57.832302+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:05:57.832302+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:29:03.783133+0000","scrub_duration":0.010206847999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":318,"reported_epoch":434,"state":"active+clean","last_fresh":"2023-09-23T21:10:36.673843+0000","last_change":"2023-09-23T21:10:26.760973+0000","last_active":"2023-09-23T21:10:36.673843+0000","last_peered":"2023-09-23T21:10:36.673843+0000","last_clean":"2023-09-23T21:10:36.673843+0000","last_became_active":"2023-09-23T21:10:26.760663+0000","last_became_peered":"2023-09-23T21:10:26.760663+0000","last_unstale":"2023-09-23T21:10:36.673843+0000","last_undegraded":"2023-09-23T21:10:36.673843+0000","last_fullsized":"2023-09-23T21:10:36.673843+0000","mapping_epoch":423,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":424,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:16:09.757099+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,0],"acting":[1,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":443,"reported_epoch":434,"state":"active+clean","last_fresh":"2023-09-23T21:10:36.677706+0000","last_change":"2023-09-23T21:09:08.424118+0000","last_active":"2023-09-23T21:10:36.677706+0000","last_peered":"2023-09-23T21:10:36.677706+0000","last_clean":"2023-09-23T21:10:36.677706+0000","last_became_active":"2023-09-23T21:09:08.423805+0000","last_became_peered":"2023-09-23T21:09:08.423805+0000","last_unstale":"2023-09-23T21:10:36.677706+0000","last_undegraded":"2023-09-23T21:10:36.677706+0000","last_fullsized":"2023-09-23T21:10:36.677706+0000","mapping_epoch":396,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":397,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:47:06.890350+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,5],"acting":[0,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":427,"reported_epoch":434,"state":"active+clean","last_fresh":"2023-09-23T21:10:36.681904+0000","last_change":"2023-09-23T21:05:55.320840+0000","last_active":"2023-09-23T21:10:36.681904+0000","last_peered":"2023-09-23T21:10:36.681904+0000","last_clean":"2023-09-23T21:10:36.681904+0000","last_became_active":"2023-09-23T21:05:55.320280+0000","last_became_peered":"2023-09-23T21:05:55.320280+0000","last_unstale":"2023-09-23T21:10:36.681904+0000","last_undegraded":"2023-09-23T21:10:36.681904+0000","last_fullsized":"2023-09-23T21:10:36.681904+0000","mapping_epoch":350,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":351,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:45:22.368801+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.d","version":"435'1947","reported_seq":1974,"reported_epoch":435,"state":"active+clean","last_fresh":"2023-09-23T21:10:47.392830+0000","last_change":"2023-09-23T21:10:16.717752+0000","last_active":"2023-09-23T21:10:47.392830+0000","last_peered":"2023-09-23T21:10:47.392830+0000","last_clean":"2023-09-23T21:10:47.392830+0000","last_became_active":"2023-09-23T21:10:16.717482+0000","last_became_peered":"2023-09-23T21:10:16.717482+0000","last_unstale":"2023-09-23T21:10:47.392830+0000","last_undegraded":"2023-09-23T21:10:47.392830+0000","last_fullsized":"2023-09-23T21:10:47.392830+0000","mapping_epoch":416,"log_start":"0'0","ondisk_log_start":"0'0","created":416,"last_epoch_clean":417,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_clean_scrub_stamp":"2023-09-23T21:10:15.663179+0000","objects_scrubbed":0,"log_size":1947,"log_dups_size":0,"ondisk_log_size":1947,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:36:40.031342+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":127598592,"num_objects":1947,"num_object_clones":0,"num_object_copies":5841,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1947,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1947,"num_write_kb":124608,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.6","version":"79'161","reported_seq":451,"reported_epoch":434,"state":"active+clean","last_fresh":"2023-09-23T21:10:36.682159+0000","last_change":"2023-09-23T21:10:24.269645+0000","last_active":"2023-09-23T21:10:36.682159+0000","last_peered":"2023-09-23T21:10:36.682159+0000","last_clean":"2023-09-23T21:10:36.682159+0000","last_became_active":"2023-09-23T21:10:19.696602+0000","last_became_peered":"2023-09-23T21:10:19.696602+0000","last_unstale":"2023-09-23T21:10:36.682159+0000","last_undegraded":"2023-09-23T21:10:36.682159+0000","last_fullsized":"2023-09-23T21:10:36.682159+0000","mapping_epoch":419,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":420,"parent":"0.0","parent_split_bits":3,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:10:24.268662+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:10:24.268662+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:09:43.881632+0000","scrub_duration":0.0065149539999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":21,"reported_epoch":434,"state":"active+clean","last_fresh":"2023-09-23T21:10:36.680121+0000","last_change":"2023-09-23T21:10:26.762740+0000","last_active":"2023-09-23T21:10:36.680121+0000","last_peered":"2023-09-23T21:10:36.680121+0000","last_clean":"2023-09-23T21:10:36.680121+0000","last_became_active":"2023-09-23T21:10:26.762442+0000","last_became_peered":"2023-09-23T21:10:26.762442+0000","last_unstale":"2023-09-23T21:10:36.680121+0000","last_undegraded":"2023-09-23T21:10:36.680121+0000","last_fullsized":"2023-09-23T21:10:36.680121+0000","mapping_epoch":423,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":424,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:47:24.723637+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":423,"reported_epoch":434,"state":"active+clean","last_fresh":"2023-09-23T21:10:36.674462+0000","last_change":"2023-09-23T21:05:55.319513+0000","last_active":"2023-09-23T21:10:36.674462+0000","last_peered":"2023-09-23T21:10:36.674462+0000","last_clean":"2023-09-23T21:10:36.674462+0000","last_became_active":"2023-09-23T21:05:55.319178+0000","last_became_peered":"2023-09-23T21:05:55.319178+0000","last_unstale":"2023-09-23T21:10:36.674462+0000","last_undegraded":"2023-09-23T21:10:36.674462+0000","last_fullsized":"2023-09-23T21:10:36.674462+0000","mapping_epoch":350,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":351,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:47:32.783934+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":437,"reported_epoch":434,"state":"active+clean","last_fresh":"2023-09-23T21:10:36.679819+0000","last_change":"2023-09-23T21:10:26.762283+0000","last_active":"2023-09-23T21:10:36.679819+0000","last_peered":"2023-09-23T21:10:36.679819+0000","last_clean":"2023-09-23T21:10:36.679819+0000","last_became_active":"2023-09-23T21:10:26.761725+0000","last_became_peered":"2023-09-23T21:10:26.761725+0000","last_unstale":"2023-09-23T21:10:36.679819+0000","last_undegraded":"2023-09-23T21:10:36.679819+0000","last_fullsized":"2023-09-23T21:10:36.679819+0000","mapping_epoch":423,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":424,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:58:15.366645+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.e","version":"435'1887","reported_seq":1915,"reported_epoch":435,"state":"active+clean","last_fresh":"2023-09-23T21:10:46.099395+0000","last_change":"2023-09-23T21:10:16.713653+0000","last_active":"2023-09-23T21:10:46.099395+0000","last_peered":"2023-09-23T21:10:46.099395+0000","last_clean":"2023-09-23T21:10:46.099395+0000","last_became_active":"2023-09-23T21:10:16.713374+0000","last_became_peered":"2023-09-23T21:10:16.713374+0000","last_unstale":"2023-09-23T21:10:46.099395+0000","last_undegraded":"2023-09-23T21:10:46.099395+0000","last_fullsized":"2023-09-23T21:10:46.099395+0000","mapping_epoch":416,"log_start":"0'0","ondisk_log_start":"0'0","created":416,"last_epoch_clean":417,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_clean_scrub_stamp":"2023-09-23T21:10:15.663179+0000","objects_scrubbed":0,"log_size":1887,"log_dups_size":0,"ondisk_log_size":1887,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:27:49.513673+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":123666432,"num_objects":1887,"num_object_clones":0,"num_object_copies":5661,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1887,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1887,"num_write_kb":120768,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.5","version":"79'161","reported_seq":476,"reported_epoch":434,"state":"active+clean","last_fresh":"2023-09-23T21:10:36.678198+0000","last_change":"2023-09-23T21:09:34.893095+0000","last_active":"2023-09-23T21:10:36.678198+0000","last_peered":"2023-09-23T21:10:36.678198+0000","last_clean":"2023-09-23T21:10:36.678198+0000","last_became_active":"2023-09-23T21:09:11.388460+0000","last_became_peered":"2023-09-23T21:09:11.388460+0000","last_unstale":"2023-09-23T21:10:36.678198+0000","last_undegraded":"2023-09-23T21:10:36.678198+0000","last_fullsized":"2023-09-23T21:10:36.678198+0000","mapping_epoch":399,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":400,"parent":"0.0","parent_split_bits":3,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:09:34.892947+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:09:34.892947+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:49:55.576750+0000","scrub_duration":0.0081306369999999996,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,0],"acting":[2,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":23,"reported_epoch":434,"state":"active+clean","last_fresh":"2023-09-23T21:10:37.413339+0000","last_change":"2023-09-23T21:10:36.685545+0000","last_active":"2023-09-23T21:10:37.413339+0000","last_peered":"2023-09-23T21:10:37.413339+0000","last_clean":"2023-09-23T21:10:37.413339+0000","last_became_active":"2023-09-23T21:10:36.685193+0000","last_became_peered":"2023-09-23T21:10:36.685193+0000","last_unstale":"2023-09-23T21:10:37.413339+0000","last_undegraded":"2023-09-23T21:10:37.413339+0000","last_fullsized":"2023-09-23T21:10:37.413339+0000","mapping_epoch":433,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":434,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:04:31.439180+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":419,"reported_epoch":434,"state":"active+clean","last_fresh":"2023-09-23T21:10:36.675048+0000","last_change":"2023-09-23T21:10:16.714910+0000","last_active":"2023-09-23T21:10:36.675048+0000","last_peered":"2023-09-23T21:10:36.675048+0000","last_clean":"2023-09-23T21:10:36.675048+0000","last_became_active":"2023-09-23T21:10:16.714562+0000","last_became_peered":"2023-09-23T21:10:16.714562+0000","last_unstale":"2023-09-23T21:10:36.675048+0000","last_undegraded":"2023-09-23T21:10:36.675048+0000","last_fullsized":"2023-09-23T21:10:36.675048+0000","mapping_epoch":416,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":417,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:37:16.706543+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,0],"acting":[1,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":247,"reported_epoch":434,"state":"active+clean","last_fresh":"2023-09-23T21:10:36.679706+0000","last_change":"2023-09-23T21:10:26.768185+0000","last_active":"2023-09-23T21:10:36.679706+0000","last_peered":"2023-09-23T21:10:36.679706+0000","last_clean":"2023-09-23T21:10:36.679706+0000","last_became_active":"2023-09-23T21:10:26.767857+0000","last_became_peered":"2023-09-23T21:10:26.767857+0000","last_unstale":"2023-09-23T21:10:36.679706+0000","last_undegraded":"2023-09-23T21:10:36.679706+0000","last_fullsized":"2023-09-23T21:10:36.679706+0000","mapping_epoch":423,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":424,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:17:05.719649+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.f","version":"435'2155","reported_seq":2183,"reported_epoch":435,"state":"active+clean","last_fresh":"2023-09-23T21:10:49.127399+0000","last_change":"2023-09-23T21:10:16.720337+0000","last_active":"2023-09-23T21:10:49.127399+0000","last_peered":"2023-09-23T21:10:49.127399+0000","last_clean":"2023-09-23T21:10:49.127399+0000","last_became_active":"2023-09-23T21:10:16.719957+0000","last_became_peered":"2023-09-23T21:10:16.719957+0000","last_unstale":"2023-09-23T21:10:49.127399+0000","last_undegraded":"2023-09-23T21:10:49.127399+0000","last_fullsized":"2023-09-23T21:10:49.127399+0000","mapping_epoch":416,"log_start":"0'0","ondisk_log_start":"0'0","created":416,"last_epoch_clean":417,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_clean_scrub_stamp":"2023-09-23T21:10:15.663179+0000","objects_scrubbed":0,"log_size":2155,"log_dups_size":0,"ondisk_log_size":2155,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:20:27.180134+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":141230080,"num_objects":2155,"num_object_clones":0,"num_object_copies":6465,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2155,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2155,"num_write_kb":137920,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,1],"acting":[3,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.4","version":"79'161","reported_seq":804,"reported_epoch":435,"state":"active+clean","last_fresh":"2023-09-23T21:10:42.142011+0000","last_change":"2023-09-23T21:10:29.807785+0000","last_active":"2023-09-23T21:10:42.142011+0000","last_peered":"2023-09-23T21:10:42.142011+0000","last_clean":"2023-09-23T21:10:42.142011+0000","last_became_active":"2023-09-23T21:10:29.807366+0000","last_became_peered":"2023-09-23T21:10:29.807366+0000","last_unstale":"2023-09-23T21:10:42.142011+0000","last_undegraded":"2023-09-23T21:10:42.142011+0000","last_fullsized":"2023-09-23T21:10:42.142011+0000","mapping_epoch":426,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":427,"parent":"0.0","parent_split_bits":3,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:20.012764+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:20.012764+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:24:45.763834+0000","scrub_duration":0.0045251919999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":372,"reported_epoch":434,"state":"active+clean","last_fresh":"2023-09-23T21:10:36.678607+0000","last_change":"2023-09-23T21:10:26.768364+0000","last_active":"2023-09-23T21:10:36.678607+0000","last_peered":"2023-09-23T21:10:36.678607+0000","last_clean":"2023-09-23T21:10:36.678607+0000","last_became_active":"2023-09-23T21:10:26.768052+0000","last_became_peered":"2023-09-23T21:10:26.768052+0000","last_unstale":"2023-09-23T21:10:36.678607+0000","last_undegraded":"2023-09-23T21:10:36.678607+0000","last_fullsized":"2023-09-23T21:10:36.678607+0000","mapping_epoch":423,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":424,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:22:42.201200+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,3],"acting":[2,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.6","version":"70'1","reported_seq":443,"reported_epoch":434,"state":"active+clean","last_fresh":"2023-09-23T21:10:36.681997+0000","last_change":"2023-09-23T21:05:55.320603+0000","last_active":"2023-09-23T21:10:36.681997+0000","last_peered":"2023-09-23T21:10:36.681997+0000","last_clean":"2023-09-23T21:10:36.681997+0000","last_became_active":"2023-09-23T21:05:55.319956+0000","last_became_peered":"2023-09-23T21:05:55.319956+0000","last_unstale":"2023-09-23T21:10:36.681997+0000","last_undegraded":"2023-09-23T21:10:36.681997+0000","last_fullsized":"2023-09-23T21:10:36.681997+0000","mapping_epoch":350,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":351,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:41:03.995212+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":319,"reported_epoch":434,"state":"active+clean","last_fresh":"2023-09-23T21:10:36.678169+0000","last_change":"2023-09-23T21:10:26.754542+0000","last_active":"2023-09-23T21:10:36.678169+0000","last_peered":"2023-09-23T21:10:36.678169+0000","last_clean":"2023-09-23T21:10:36.678169+0000","last_became_active":"2023-09-23T21:10:26.754208+0000","last_became_peered":"2023-09-23T21:10:26.754208+0000","last_unstale":"2023-09-23T21:10:36.678169+0000","last_undegraded":"2023-09-23T21:10:36.678169+0000","last_fullsized":"2023-09-23T21:10:36.678169+0000","mapping_epoch":423,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":424,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:38:07.602905+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,7],"acting":[5,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":420,"reported_epoch":434,"state":"active+clean","last_fresh":"2023-09-23T21:10:36.679738+0000","last_change":"2023-09-23T21:10:26.762673+0000","last_active":"2023-09-23T21:10:36.679738+0000","last_peered":"2023-09-23T21:10:36.679738+0000","last_clean":"2023-09-23T21:10:36.679738+0000","last_became_active":"2023-09-23T21:10:26.761275+0000","last_became_peered":"2023-09-23T21:10:26.761275+0000","last_unstale":"2023-09-23T21:10:36.679738+0000","last_undegraded":"2023-09-23T21:10:36.679738+0000","last_fullsized":"2023-09-23T21:10:36.679738+0000","mapping_epoch":423,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":424,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:51:16.027606+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":423,"reported_epoch":434,"state":"active+clean","last_fresh":"2023-09-23T21:10:36.677851+0000","last_change":"2023-09-23T21:05:55.313495+0000","last_active":"2023-09-23T21:10:36.677851+0000","last_peered":"2023-09-23T21:10:36.677851+0000","last_clean":"2023-09-23T21:10:36.677851+0000","last_became_active":"2023-09-23T21:05:55.313065+0000","last_became_peered":"2023-09-23T21:05:55.313065+0000","last_unstale":"2023-09-23T21:10:36.677851+0000","last_undegraded":"2023-09-23T21:10:36.677851+0000","last_fullsized":"2023-09-23T21:10:36.677851+0000","mapping_epoch":350,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":351,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:05:22.675197+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":101,"reported_epoch":434,"state":"active+clean","last_fresh":"2023-09-23T21:10:36.674597+0000","last_change":"2023-09-23T21:10:26.771985+0000","last_active":"2023-09-23T21:10:36.674597+0000","last_peered":"2023-09-23T21:10:36.674597+0000","last_clean":"2023-09-23T21:10:36.674597+0000","last_became_active":"2023-09-23T21:10:26.755509+0000","last_became_peered":"2023-09-23T21:10:26.755509+0000","last_unstale":"2023-09-23T21:10:36.674597+0000","last_undegraded":"2023-09-23T21:10:36.674597+0000","last_fullsized":"2023-09-23T21:10:36.674597+0000","mapping_epoch":423,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":424,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:59:47.931163+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,7],"acting":[1,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":286,"reported_epoch":434,"state":"active+clean","last_fresh":"2023-09-23T21:10:36.679940+0000","last_change":"2023-09-23T21:09:08.440185+0000","last_active":"2023-09-23T21:10:36.679940+0000","last_peered":"2023-09-23T21:10:36.679940+0000","last_clean":"2023-09-23T21:10:36.679940+0000","last_became_active":"2023-09-23T21:09:08.439562+0000","last_became_peered":"2023-09-23T21:09:08.439562+0000","last_unstale":"2023-09-23T21:10:36.679940+0000","last_undegraded":"2023-09-23T21:10:36.679940+0000","last_fullsized":"2023-09-23T21:10:36.679940+0000","mapping_epoch":396,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":397,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:39:54.771238+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":210,"reported_epoch":434,"state":"active+clean","last_fresh":"2023-09-23T21:10:36.678101+0000","last_change":"2023-09-23T21:08:54.706324+0000","last_active":"2023-09-23T21:10:36.678101+0000","last_peered":"2023-09-23T21:10:36.678101+0000","last_clean":"2023-09-23T21:10:36.678101+0000","last_became_active":"2023-09-23T21:08:54.681312+0000","last_became_peered":"2023-09-23T21:08:54.681312+0000","last_unstale":"2023-09-23T21:10:36.678101+0000","last_undegraded":"2023-09-23T21:10:36.678101+0000","last_fullsized":"2023-09-23T21:10:36.678101+0000","mapping_epoch":388,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":389,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:08:16.335346+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,1],"acting":[2,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":101,"reported_epoch":434,"state":"active+clean","last_fresh":"2023-09-23T21:10:37.413177+0000","last_change":"2023-09-23T21:10:36.686732+0000","last_active":"2023-09-23T21:10:37.413177+0000","last_peered":"2023-09-23T21:10:37.413177+0000","last_clean":"2023-09-23T21:10:37.413177+0000","last_became_active":"2023-09-23T21:10:36.686405+0000","last_became_peered":"2023-09-23T21:10:36.686405+0000","last_unstale":"2023-09-23T21:10:37.413177+0000","last_undegraded":"2023-09-23T21:10:37.413177+0000","last_fullsized":"2023-09-23T21:10:37.413177+0000","mapping_epoch":433,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":434,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:02:05.143760+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":13,"reported_epoch":434,"state":"active+clean","last_fresh":"2023-09-23T21:10:36.685552+0000","last_change":"2023-09-23T21:10:36.684138+0000","last_active":"2023-09-23T21:10:36.685552+0000","last_peered":"2023-09-23T21:10:36.685552+0000","last_clean":"2023-09-23T21:10:36.685552+0000","last_became_active":"2023-09-23T21:10:36.683827+0000","last_became_peered":"2023-09-23T21:10:36.683827+0000","last_unstale":"2023-09-23T21:10:36.685552+0000","last_undegraded":"2023-09-23T21:10:36.685552+0000","last_fullsized":"2023-09-23T21:10:36.685552+0000","mapping_epoch":433,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":434,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:33:11.259135+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,1],"acting":[2,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":86,"reported_epoch":434,"state":"active+clean","last_fresh":"2023-09-23T21:10:36.686761+0000","last_change":"2023-09-23T21:10:36.686535+0000","last_active":"2023-09-23T21:10:36.686761+0000","last_peered":"2023-09-23T21:10:36.686761+0000","last_clean":"2023-09-23T21:10:36.686761+0000","last_became_active":"2023-09-23T21:10:36.686220+0000","last_became_peered":"2023-09-23T21:10:36.686220+0000","last_unstale":"2023-09-23T21:10:36.686761+0000","last_undegraded":"2023-09-23T21:10:36.686761+0000","last_fullsized":"2023-09-23T21:10:36.686761+0000","mapping_epoch":433,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":434,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:45:15.133396+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":461,"reported_epoch":434,"state":"active+clean","last_fresh":"2023-09-23T21:10:36.681631+0000","last_change":"2023-09-23T21:05:04.325268+0000","last_active":"2023-09-23T21:10:36.681631+0000","last_peered":"2023-09-23T21:10:36.681631+0000","last_clean":"2023-09-23T21:10:36.681631+0000","last_became_active":"2023-09-23T21:05:04.303430+0000","last_became_peered":"2023-09-23T21:05:04.303430+0000","last_unstale":"2023-09-23T21:10:36.681631+0000","last_undegraded":"2023-09-23T21:10:36.681631+0000","last_fullsized":"2023-09-23T21:10:36.681631+0000","mapping_epoch":332,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:17:44.879586+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":2216,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":49,"reported_epoch":434,"state":"active+clean","last_fresh":"2023-09-23T21:10:36.673799+0000","last_change":"2023-09-23T21:09:08.428128+0000","last_active":"2023-09-23T21:10:36.673799+0000","last_peered":"2023-09-23T21:10:36.673799+0000","last_clean":"2023-09-23T21:10:36.673799+0000","last_became_active":"2023-09-23T21:09:08.427481+0000","last_became_peered":"2023-09-23T21:09:08.427481+0000","last_unstale":"2023-09-23T21:10:36.673799+0000","last_undegraded":"2023-09-23T21:10:36.673799+0000","last_fullsized":"2023-09-23T21:10:36.673799+0000","mapping_epoch":396,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":397,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:46:06.052816+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,2],"acting":[1,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":10,"num_pg":16,"stat_sum":{"num_bytes":2062483456,"num_objects":31471,"num_object_clones":0,"num_object_copies":94413,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":31471,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":31471,"num_write_kb":2014144,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2800,"num_bytes_recovered":183500800,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":6244925440,"data_stored":6244925440,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":31471,"ondisk_log_size":31471,"up":48,"acting":48,"num_store_stats":7},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":40,"num_bytes_recovered":7578,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":118784,"data_stored":21350,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5834,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":21,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":327680,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2109440,"data_stored":2098784,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3381,"ondisk_log_size":3381,"up":63,"acting":63,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201644,"num_pgs":51,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1171484,"kb_used_data":736980,"kb_used_omap":0,"kb_used_meta":434496,"kb_avail":92573668,"statfs":{"total":95995035648,"available":94795436032,"internally_reserved":0,"allocated":754667520,"data_stored":753435293,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":444923904},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,4,0,2],"upper_bound":16},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:10:02 2023","interfaces":[{"interface":"back","average":{"1min":1.0840000000000001,"5min":1.385,"15min":1.7030000000000001},"min":{"1min":0.253,"5min":0.22800000000000001,"15min":0.21099999999999999},"max":{"1min":5.4050000000000002,"5min":7.0170000000000003,"15min":43.994999999999997},"last":0.38},{"interface":"front","average":{"1min":1.0629999999999999,"5min":1.4650000000000001,"15min":1.8520000000000001},"min":{"1min":0.29499999999999998,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":5.4189999999999996,"5min":8.1120000000000001,"15min":44.206000000000003},"last":0.29499999999999998}]},{"osd":1,"last update":"Sat Sep 23 21:10:02 2023","interfaces":[{"interface":"back","average":{"1min":1.5289999999999999,"5min":2.5539999999999998,"15min":2.3180000000000001},"min":{"1min":0.27100000000000002,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":7.218,"5min":11.193,"15min":11.193},"last":0.27700000000000002},{"interface":"front","average":{"1min":1.5429999999999999,"5min":2.1970000000000001,"15min":2.1269999999999998},"min":{"1min":0.29199999999999998,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":6.4850000000000003,"5min":9.9199999999999999,"15min":9.9199999999999999},"last":0.54500000000000004}]},{"osd":2,"last update":"Sat Sep 23 21:10:25 2023","interfaces":[{"interface":"back","average":{"1min":0.91600000000000004,"5min":1.9870000000000001,"15min":1.968},"min":{"1min":0.29499999999999998,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":3.4729999999999999,"5min":14.478999999999999,"15min":14.478999999999999},"last":1.278},{"interface":"front","average":{"1min":1.056,"5min":1.9099999999999999,"15min":1.9510000000000001},"min":{"1min":0.30599999999999999,"5min":0.27400000000000002,"15min":0.26500000000000001},"max":{"1min":3.5230000000000001,"5min":14.135,"15min":45.25},"last":0.81200000000000006}]},{"osd":3,"last update":"Sat Sep 23 21:09:53 2023","interfaces":[{"interface":"back","average":{"1min":1.6990000000000001,"5min":2.5369999999999999,"15min":2.4020000000000001},"min":{"1min":0.316,"5min":0.30099999999999999,"15min":0.29099999999999998},"max":{"1min":4.6790000000000003,"5min":12.359999999999999,"15min":31.786000000000001},"last":1.321},{"interface":"front","average":{"1min":2.226,"5min":2.3370000000000002,"15min":2.3119999999999998},"min":{"1min":0.309,"5min":0.30299999999999999,"15min":0.26000000000000001},"max":{"1min":11.244999999999999,"5min":13.199,"15min":32.429000000000002},"last":0.60799999999999998}]},{"osd":4,"last update":"Sat Sep 23 21:09:47 2023","interfaces":[{"interface":"back","average":{"1min":1.3169999999999999,"5min":1.1950000000000001,"15min":0.91200000000000003},"min":{"1min":0.255,"5min":0.255,"15min":0.23799999999999999},"max":{"1min":7.1189999999999998,"5min":8.5570000000000004,"15min":8.5570000000000004},"last":0.45100000000000001},{"interface":"front","average":{"1min":0.72399999999999998,"5min":1.1639999999999999,"15min":0.94999999999999996},"min":{"1min":0.26800000000000002,"5min":0.26800000000000002,"15min":0.26300000000000001},"max":{"1min":3.331,"5min":12.752000000000001,"15min":12.752000000000001},"last":0.70299999999999996}]},{"osd":5,"last update":"Sat Sep 23 21:09:47 2023","interfaces":[{"interface":"back","average":{"1min":0.64300000000000002,"5min":1.2450000000000001,"15min":0.93600000000000005},"min":{"1min":0.33300000000000002,"5min":0.311,"15min":0.28499999999999998},"max":{"1min":1.7370000000000001,"5min":7.6879999999999997,"15min":7.6879999999999997},"last":0.35899999999999999},{"interface":"front","average":{"1min":1.145,"5min":1.2010000000000001,"15min":0.92400000000000004},"min":{"1min":0.36699999999999999,"5min":0.29699999999999999,"15min":0.27800000000000002},"max":{"1min":5.5090000000000003,"5min":5.5090000000000003,"15min":5.8529999999999998},"last":0.59299999999999997}]},{"osd":6,"last update":"Sat Sep 23 21:10:02 2023","interfaces":[{"interface":"back","average":{"1min":0.72099999999999997,"5min":1.0129999999999999,"15min":0.88700000000000001},"min":{"1min":0.34999999999999998,"5min":0.32000000000000001,"15min":0.28899999999999998},"max":{"1min":3.0619999999999998,"5min":6.5430000000000001,"15min":6.5430000000000001},"last":0.67100000000000004},{"interface":"front","average":{"1min":0.74199999999999999,"5min":1.3160000000000001,"15min":1.026},"min":{"1min":0.377,"5min":0.36399999999999999,"15min":0.32300000000000001},"max":{"1min":3.0249999999999999,"5min":12.715,"15min":12.715},"last":0.64100000000000001}]}]},{"osd":6,"up_from":0,"seq":704374636728,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":1,"up_from":350,"seq":1503238553660,"num_pgs":56,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1169168,"kb_used_data":912072,"kb_used_omap":10,"kb_used_meta":257077,"kb_avail":92575984,"statfs":{"total":95995035648,"available":94797807616,"internally_reserved":0,"allocated":933961728,"data_stored":931996104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":10498,"internal_metadata":263247614},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[3,0,1,5],"upper_bound":16},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:10:04 2023","interfaces":[{"interface":"back","average":{"1min":2.7519999999999998,"5min":2.4470000000000001,"15min":2.0859999999999999},"min":{"1min":0.255,"5min":0.20499999999999999,"15min":0.20499999999999999},"max":{"1min":8.9290000000000003,"5min":15.728,"15min":15.728},"last":0.78500000000000003},{"interface":"front","average":{"1min":2.9049999999999998,"5min":2.7879999999999998,"15min":2.6360000000000001},"min":{"1min":0.27200000000000002,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":15.16,"5min":15.16,"15min":15.16},"last":0.54600000000000004}]},{"osd":2,"last update":"Sat Sep 23 21:10:04 2023","interfaces":[{"interface":"back","average":{"1min":1.048,"5min":2.2530000000000001,"15min":2.3109999999999999},"min":{"1min":0.23999999999999999,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":3.2330000000000001,"5min":10.769,"15min":10.769},"last":0.93400000000000005},{"interface":"front","average":{"1min":1.887,"5min":2.1509999999999998,"15min":2.056},"min":{"1min":0.254,"5min":0.214,"15min":0.214},"max":{"1min":9.5429999999999993,"5min":15.683999999999999,"15min":15.683999999999999},"last":1.1619999999999999}]},{"osd":3,"last update":"Sat Sep 23 21:10:04 2023","interfaces":[{"interface":"back","average":{"1min":2.8740000000000001,"5min":2.7589999999999999,"15min":2.7509999999999999},"min":{"1min":0.29599999999999999,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":9.2550000000000008,"5min":15.135999999999999,"15min":15.135999999999999},"last":1.325},{"interface":"front","average":{"1min":3.3500000000000001,"5min":3.1509999999999998,"15min":3.093},"min":{"1min":0.29299999999999998,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":15.237,"5min":15.237,"15min":15.237},"last":0.82999999999999996}]},{"osd":4,"last update":"Sat Sep 23 21:10:04 2023","interfaces":[{"interface":"back","average":{"1min":0.69999999999999996,"5min":1.756,"15min":1.752},"min":{"1min":0.27500000000000002,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":2.6059999999999999,"5min":10.391,"15min":10.391},"last":0.47599999999999998},{"interface":"front","average":{"1min":1.6200000000000001,"5min":1.8149999999999999,"15min":1.673},"min":{"1min":0.36199999999999999,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":8.8859999999999992,"5min":8.8859999999999992,"15min":8.8859999999999992},"last":1.0329999999999999}]},{"osd":5,"last update":"Sat Sep 23 21:10:04 2023","interfaces":[{"interface":"back","average":{"1min":2.1400000000000001,"5min":2.4700000000000002,"15min":2.5600000000000001},"min":{"1min":0.39200000000000002,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":14.813000000000001,"5min":14.813000000000001,"15min":14.813000000000001},"last":0.60799999999999998},{"interface":"front","average":{"1min":0.874,"5min":2.2330000000000001,"15min":2.5169999999999999},"min":{"1min":0.32500000000000001,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":3.0150000000000001,"5min":10.339,"15min":10.339},"last":0.437}]},{"osd":6,"last update":"Sat Sep 23 21:10:04 2023","interfaces":[{"interface":"back","average":{"1min":2.1230000000000002,"5min":2.258,"15min":2.3119999999999998},"min":{"1min":0.42499999999999999,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":14.859999999999999,"5min":14.859999999999999,"15min":14.859999999999999},"last":0.64500000000000002},{"interface":"front","average":{"1min":0.72399999999999998,"5min":2.319,"15min":2.3879999999999999},"min":{"1min":0.377,"5min":0.377,"15min":0.377},"max":{"1min":2.6600000000000001,"5min":16.494,"15min":16.494},"last":0.505}]},{"osd":7,"last update":"Sat Sep 23 21:10:04 2023","interfaces":[{"interface":"back","average":{"1min":1.7490000000000001,"5min":2.0249999999999999,"15min":2.036},"min":{"1min":0.504,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":8.8520000000000003,"5min":8.8520000000000003,"15min":8.8520000000000003},"last":1.0069999999999999},{"interface":"front","average":{"1min":2.1520000000000001,"5min":2.5070000000000001,"15min":2.5870000000000002},"min":{"1min":0.44400000000000001,"5min":0.41099999999999998,"15min":0.41099999999999998},"max":{"1min":14.891999999999999,"5min":14.891999999999999,"15min":14.891999999999999},"last":0.67300000000000004}]}]},{"osd":0,"up_from":83,"seq":356482285825,"num_pgs":63,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2003252,"kb_used_data":1379244,"kb_used_omap":28,"kb_used_meta":623971,"kb_avail":91741900,"statfs":{"total":95995035648,"available":93943705600,"internally_reserved":0,"allocated":1412345856,"data_stored":1410407021,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":28848,"internal_metadata":638947152},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,0,0,1,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 21:09:58 2023","interfaces":[{"interface":"back","average":{"1min":3.7149999999999999,"5min":2.4980000000000002,"15min":1.71},"min":{"1min":0.29399999999999998,"5min":0.192,"15min":0.192},"max":{"1min":8.1950000000000003,"5min":12.994,"15min":12.994},"last":0.61399999999999999},{"interface":"front","average":{"1min":3.4199999999999999,"5min":2.2490000000000001,"15min":1.599},"min":{"1min":0.253,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":12.278,"5min":14.646000000000001,"15min":14.646000000000001},"last":1.075}]},{"osd":2,"last update":"Sat Sep 23 21:10:18 2023","interfaces":[{"interface":"back","average":{"1min":1.569,"5min":2.0009999999999999,"15min":2.6829999999999998},"min":{"1min":0.24399999999999999,"5min":0.183,"15min":0.183},"max":{"1min":8.0310000000000006,"5min":13.032,"15min":21.053000000000001},"last":0.77200000000000002},{"interface":"front","average":{"1min":2.1179999999999999,"5min":2.0630000000000002,"15min":2.9100000000000001},"min":{"1min":0.27500000000000002,"5min":0.26600000000000001,"15min":0.23100000000000001},"max":{"1min":7.6760000000000002,"5min":14.558999999999999,"15min":21.747},"last":1.1140000000000001}]},{"osd":3,"last update":"Sat Sep 23 21:10:37 2023","interfaces":[{"interface":"back","average":{"1min":1.913,"5min":2.847,"15min":2.9910000000000001},"min":{"1min":0.27300000000000002,"5min":0.253,"15min":0.252},"max":{"1min":8.9220000000000006,"5min":24.048999999999999,"15min":41.079000000000001},"last":0.442},{"interface":"front","average":{"1min":1.6850000000000001,"5min":2.5259999999999998,"15min":2.9780000000000002},"min":{"1min":0.34999999999999998,"5min":0.26000000000000001,"15min":0.23200000000000001},"max":{"1min":7.508,"5min":19.91,"15min":21.167000000000002},"last":0.36799999999999999}]},{"osd":4,"last update":"Sat Sep 23 21:10:18 2023","interfaces":[{"interface":"back","average":{"1min":1.5960000000000001,"5min":1.837,"15min":2.2570000000000001},"min":{"1min":0.377,"5min":0.32400000000000001,"15min":0.28599999999999998},"max":{"1min":5.2889999999999997,"5min":7.9160000000000004,"15min":21.594999999999999},"last":0.81699999999999995},{"interface":"front","average":{"1min":1.6659999999999999,"5min":1.851,"15min":1.931},"min":{"1min":0.33100000000000002,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":10.253,"5min":18.516999999999999,"15min":18.516999999999999},"last":0.40200000000000002}]},{"osd":5,"last update":"Sat Sep 23 21:10:18 2023","interfaces":[{"interface":"back","average":{"1min":1.712,"5min":1.909,"15min":1.9299999999999999},"min":{"1min":0.48999999999999999,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":7.9459999999999997,"5min":12.946999999999999,"15min":12.946999999999999},"last":0.54600000000000004},{"interface":"front","average":{"1min":1.583,"5min":1.71,"15min":2.2000000000000002},"min":{"1min":0.41799999999999998,"5min":0.373,"15min":0.32700000000000001},"max":{"1min":5.3280000000000003,"5min":7.9800000000000004,"15min":20.512},"last":1}]},{"osd":6,"last update":"Sat Sep 23 21:10:10 2023","interfaces":[{"interface":"back","average":{"1min":1.907,"5min":2.024,"15min":2.0470000000000002},"min":{"1min":0.36399999999999999,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":10.282,"5min":18.571000000000002,"15min":18.571000000000002},"last":0.496},{"interface":"front","average":{"1min":2.194,"5min":2.181,"15min":2.133},"min":{"1min":0.46800000000000003,"5min":0.40300000000000002,"15min":0.377},"max":{"1min":10.223000000000001,"5min":18.603000000000002,"15min":18.603000000000002},"last":0.53000000000000003}]},{"osd":7,"last update":"Sat Sep 23 21:10:18 2023","interfaces":[{"interface":"back","average":{"1min":1.7210000000000001,"5min":1.925,"15min":1.994},"min":{"1min":0.45400000000000001,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":7.9790000000000001,"5min":12.907999999999999,"15min":12.907999999999999},"last":0.58999999999999997},{"interface":"front","average":{"1min":2.133,"5min":2.169,"15min":2.242},"min":{"1min":0.42299999999999999,"5min":0.38200000000000001,"15min":0.377},"max":{"1min":10.183,"5min":18.634,"15min":40.984999999999999},"last":0.64100000000000001}]}]},{"osd":2,"up_from":93,"seq":399431958772,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1601216,"kb_used_data":947192,"kb_used_omap":81,"kb_used_meta":653934,"kb_avail":92143936,"statfs":{"total":95995035648,"available":94355390464,"internally_reserved":0,"allocated":969924608,"data_stored":967973878,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":83750,"internal_metadata":669628634},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,5,0,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:10:03 2023","interfaces":[{"interface":"back","average":{"1min":2.5470000000000002,"5min":2.0590000000000002,"15min":2.2759999999999998},"min":{"1min":0.26300000000000001,"5min":0.23000000000000001,"15min":0.17999999999999999},"max":{"1min":7.5259999999999998,"5min":11.124000000000001,"15min":29.498000000000001},"last":3.3610000000000002},{"interface":"front","average":{"1min":2.7040000000000002,"5min":1.8660000000000001,"15min":2.4540000000000002},"min":{"1min":0.193,"5min":0.193,"15min":0.193},"max":{"1min":12.183,"5min":12.183,"15min":17.251000000000001},"last":0.40600000000000003}]},{"osd":1,"last update":"Sat Sep 23 21:10:03 2023","interfaces":[{"interface":"back","average":{"1min":3.109,"5min":3.1800000000000002,"15min":3.0430000000000001},"min":{"1min":0.26100000000000001,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":12.484999999999999,"5min":34.396999999999998,"15min":34.396999999999998},"last":0.53000000000000003},{"interface":"front","average":{"1min":2.3149999999999999,"5min":3.089,"15min":3.4780000000000002},"min":{"1min":0.24099999999999999,"5min":0.217,"15min":0.217},"max":{"1min":8.4420000000000002,"5min":34.462000000000003,"15min":34.462000000000003},"last":3.403}]},{"osd":3,"last update":"Sat Sep 23 21:09:49 2023","interfaces":[{"interface":"back","average":{"1min":3.593,"5min":2.9159999999999999,"15min":3.25},"min":{"1min":0.28699999999999998,"5min":0.23899999999999999,"15min":0.23300000000000001},"max":{"1min":12.951000000000001,"5min":35.549999999999997,"15min":41.057000000000002},"last":0.71499999999999997},{"interface":"front","average":{"1min":3.7250000000000001,"5min":3.1960000000000002,"15min":3.6269999999999998},"min":{"1min":0.23499999999999999,"5min":0.23499999999999999,"15min":0.22800000000000001},"max":{"1min":12.76,"5min":36.951999999999998,"15min":41.012},"last":0.68100000000000005}]},{"osd":4,"last update":"Sat Sep 23 21:10:03 2023","interfaces":[{"interface":"back","average":{"1min":1.4870000000000001,"5min":1.649,"15min":1.589},"min":{"1min":0.307,"5min":0.29199999999999998,"15min":0.28399999999999997},"max":{"1min":5.3520000000000003,"5min":8.2650000000000006,"15min":10.045},"last":0.44600000000000001},{"interface":"front","average":{"1min":1.8140000000000001,"5min":1.9259999999999999,"15min":2.0510000000000002},"min":{"1min":0.35599999999999998,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":8.6780000000000008,"5min":12.49,"15min":14.039999999999999},"last":0.58199999999999996}]},{"osd":5,"last update":"Sat Sep 23 21:10:03 2023","interfaces":[{"interface":"back","average":{"1min":1.4770000000000001,"5min":1.7889999999999999,"15min":1.762},"min":{"1min":0.32200000000000001,"5min":0.32200000000000001,"15min":0.29299999999999998},"max":{"1min":5.3079999999999998,"5min":8.4039999999999999,"15min":10.069000000000001},"last":0.47299999999999998},{"interface":"front","average":{"1min":1.829,"5min":1.9630000000000001,"15min":2.0379999999999998},"min":{"1min":0.32700000000000001,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":8.7270000000000003,"5min":12.57,"15min":13.98},"last":0.55300000000000005}]},{"osd":6,"last update":"Sat Sep 23 21:10:03 2023","interfaces":[{"interface":"back","average":{"1min":1.913,"5min":2.181,"15min":2.1520000000000001},"min":{"1min":0.40899999999999997,"5min":0.35699999999999998,"15min":0.312},"max":{"1min":8.7680000000000007,"5min":12.518000000000001,"15min":14.244999999999999},"last":0.61899999999999999},{"interface":"front","average":{"1min":1.9299999999999999,"5min":2.4399999999999999,"15min":2.1840000000000002},"min":{"1min":0.45600000000000002,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":8.8010000000000002,"5min":12.544,"15min":14.068},"last":0.64900000000000002}]},{"osd":7,"last update":"Sat Sep 23 21:10:03 2023","interfaces":[{"interface":"back","average":{"1min":1.77,"5min":1.911,"15min":1.611},"min":{"1min":0.36899999999999999,"5min":0.34100000000000003,"15min":0.315},"max":{"1min":6.4240000000000004,"5min":8.7599999999999998,"15min":13.427},"last":3.4529999999999998},{"interface":"front","average":{"1min":1.6140000000000001,"5min":2.052,"15min":1.841},"min":{"1min":0.36899999999999999,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":5.3780000000000001,"5min":10.92,"15min":10.92},"last":0.496}]}]},{"osd":3,"up_from":98,"seq":420906795246,"num_pgs":58,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1656280,"kb_used_data":928784,"kb_used_omap":7,"kb_used_meta":727480,"kb_avail":92088872,"statfs":{"total":95995035648,"available":94299004928,"internally_reserved":0,"allocated":951074816,"data_stored":949156347,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7576,"internal_metadata":744940136},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,2],"upper_bound":32},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:09:52 2023","interfaces":[{"interface":"back","average":{"1min":2.641,"5min":2.528,"15min":2.7189999999999999},"min":{"1min":0.221,"5min":0.19900000000000001,"15min":0.188},"max":{"1min":12.071999999999999,"5min":14.08,"15min":27.962},"last":0.42399999999999999},{"interface":"front","average":{"1min":2.0920000000000001,"5min":2.1400000000000001,"15min":2.8180000000000001},"min":{"1min":0.20999999999999999,"5min":0.20999999999999999,"15min":0.19800000000000001},"max":{"1min":6.2450000000000001,"5min":9.9139999999999997,"15min":28.058},"last":1.3049999999999999}]},{"osd":1,"last update":"Sat Sep 23 21:10:02 2023","interfaces":[{"interface":"back","average":{"1min":2.8889999999999998,"5min":3.2200000000000002,"15min":2.5019999999999998},"min":{"1min":0.28000000000000003,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":10.115,"5min":16.939,"15min":16.939},"last":0.39200000000000002},{"interface":"front","average":{"1min":2.6280000000000001,"5min":2.8530000000000002,"15min":2.431},"min":{"1min":0.33600000000000002,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":12.458,"5min":13.811999999999999,"15min":13.811999999999999},"last":3.9199999999999999}]},{"osd":2,"last update":"Sat Sep 23 21:09:52 2023","interfaces":[{"interface":"back","average":{"1min":1.7110000000000001,"5min":2.2050000000000001,"15min":2.8690000000000002},"min":{"1min":0.26700000000000002,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":10.470000000000001,"5min":14.287000000000001,"15min":28.251999999999999},"last":1.0780000000000001},{"interface":"front","average":{"1min":2.4159999999999999,"5min":2.528,"15min":2.7280000000000002},"min":{"1min":0.28399999999999997,"5min":0.28399999999999997,"15min":0.22},"max":{"1min":9.0739999999999998,"5min":14.195,"15min":18.321000000000002},"last":0.58899999999999997}]},{"osd":4,"last update":"Sat Sep 23 21:09:52 2023","interfaces":[{"interface":"back","average":{"1min":1.1839999999999999,"5min":1.694,"15min":1.9990000000000001},"min":{"1min":0.35299999999999998,"5min":0.28799999999999998,"15min":0.28399999999999997},"max":{"1min":3.5870000000000002,"5min":11.614000000000001,"15min":18.239000000000001},"last":1.0409999999999999},{"interface":"front","average":{"1min":1.881,"5min":2.0800000000000001,"15min":1.9299999999999999},"min":{"1min":0.36799999999999999,"5min":0.32100000000000001,"15min":0.29499999999999998},"max":{"1min":8.0540000000000003,"5min":14.119,"15min":14.808},"last":0.625}]},{"osd":5,"last update":"Sat Sep 23 21:09:52 2023","interfaces":[{"interface":"back","average":{"1min":1.2070000000000001,"5min":1.9890000000000001,"15min":2.149},"min":{"1min":0.32300000000000001,"5min":0.314,"15min":0.314},"max":{"1min":3.552,"5min":12.815,"15min":18.324999999999999},"last":1.135},{"interface":"front","average":{"1min":1.613,"5min":2.3769999999999998,"15min":2.3570000000000002},"min":{"1min":0.40400000000000003,"5min":0.33700000000000002,"15min":0.33100000000000002},"max":{"1min":6.1239999999999997,"5min":14.029,"15min":14.029},"last":1.3899999999999999}]},{"osd":6,"last update":"Sat Sep 23 21:10:08 2023","interfaces":[{"interface":"back","average":{"1min":2.1920000000000002,"5min":2.3940000000000001,"15min":2.0009999999999999},"min":{"1min":0.40799999999999997,"5min":0.36599999999999999,"15min":0.33500000000000002},"max":{"1min":12.028,"5min":18.323,"15min":18.323},"last":0.65100000000000002},{"interface":"front","average":{"1min":1.26,"5min":1.9550000000000001,"15min":2.1789999999999998},"min":{"1min":0.434,"5min":0.434,"15min":0.373},"max":{"1min":4.1879999999999997,"5min":9.6300000000000008,"15min":13.576000000000001},"last":1.452}]},{"osd":7,"last update":"Sat Sep 23 21:09:52 2023","interfaces":[{"interface":"back","average":{"1min":1.2490000000000001,"5min":1.8979999999999999,"15min":2.2789999999999999},"min":{"1min":0.38400000000000001,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":3.6139999999999999,"5min":14.81,"15min":18.289999999999999},"last":1.109},{"interface":"front","average":{"1min":1.4279999999999999,"5min":2.2330000000000001,"15min":2.3959999999999999},"min":{"1min":0.47599999999999998,"5min":0.40300000000000002,"15min":0.379},"max":{"1min":4.1559999999999997,"5min":9.9529999999999994,"15min":13.603999999999999},"last":1.4239999999999999}]}]},{"osd":4,"up_from":31,"seq":133143986553,"num_pgs":15,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":687812,"kb_used_data":384508,"kb_used_omap":0,"kb_used_meta":303296,"kb_avail":93057340,"statfs":{"total":95995035648,"available":95290716160,"internally_reserved":0,"allocated":393736192,"data_stored":392523914,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":310575104},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:10:10 2023","interfaces":[{"interface":"back","average":{"1min":1.2310000000000001,"5min":1.635,"15min":1.617},"min":{"1min":0.251,"5min":0.222,"15min":0.214},"max":{"1min":7.0140000000000002,"5min":11.537000000000001,"15min":11.784000000000001},"last":1.0700000000000001},{"interface":"front","average":{"1min":1.4259999999999999,"5min":2.3620000000000001,"15min":1.968},"min":{"1min":0.29899999999999999,"5min":0.28000000000000003,"15min":0.254},"max":{"1min":7.0720000000000001,"5min":19.109999999999999,"15min":19.109999999999999},"last":0.47999999999999998}]},{"osd":1,"last update":"Sat Sep 23 21:10:03 2023","interfaces":[{"interface":"back","average":{"1min":1.4350000000000001,"5min":2.8900000000000001,"15min":3.5099999999999998},"min":{"1min":0.36699999999999999,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":3.6789999999999998,"5min":37.960000000000001,"15min":37.960000000000001},"last":0.40200000000000002},{"interface":"front","average":{"1min":1.323,"5min":2.609,"15min":3.2829999999999999},"min":{"1min":0.36499999999999999,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":3.698,"5min":37.933,"15min":37.933},"last":1.091}]},{"osd":2,"last update":"Sat Sep 23 21:10:10 2023","interfaces":[{"interface":"back","average":{"1min":1.29,"5min":1.8520000000000001,"15min":1.859},"min":{"1min":0.32200000000000001,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":5.1520000000000001,"5min":16.928000000000001,"15min":16.928000000000001},"last":1.0169999999999999},{"interface":"front","average":{"1min":1.9079999999999999,"5min":2.101,"15min":1.865},"min":{"1min":0.371,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":5.2990000000000004,"5min":14.417999999999999,"15min":14.417999999999999},"last":1.04}]},{"osd":3,"last update":"Sat Sep 23 21:10:42 2023","interfaces":[{"interface":"back","average":{"1min":1.04,"5min":2.2970000000000002,"15min":2.2709999999999999},"min":{"1min":0.36599999999999999,"5min":0.29799999999999999,"15min":0.28899999999999998},"max":{"1min":3.7549999999999999,"5min":15.01,"15min":15.01},"last":0.47299999999999998},{"interface":"front","average":{"1min":0.98799999999999999,"5min":2.302,"15min":2.1869999999999998},"min":{"1min":0.502,"5min":0.32400000000000001,"15min":0.30199999999999999},"max":{"1min":3.7639999999999998,"5min":15.004,"15min":15.004},"last":1.141}]},{"osd":5,"last update":"Sat Sep 23 21:09:57 2023","interfaces":[{"interface":"back","average":{"1min":0.44,"5min":1.1639999999999999,"15min":0.84199999999999997},"min":{"1min":0.26100000000000001,"5min":0.24299999999999999,"15min":0.22700000000000001},"max":{"1min":1.5489999999999999,"5min":14.298,"15min":14.298},"last":1.1619999999999999},{"interface":"front","average":{"1min":0.442,"5min":1.2370000000000001,"15min":0.97899999999999998},"min":{"1min":0.29499999999999998,"5min":0.255,"15min":0.251},"max":{"1min":0.94299999999999995,"5min":9.8979999999999997,"15min":9.8979999999999997},"last":0.437}]},{"osd":6,"last update":"Sat Sep 23 21:10:03 2023","interfaces":[{"interface":"back","average":{"1min":0.624,"5min":1.325,"15min":1.0940000000000001},"min":{"1min":0.29699999999999999,"5min":0.28399999999999997,"15min":0.26500000000000001},"max":{"1min":2.395,"5min":14.704000000000001,"15min":14.704000000000001},"last":0.373},{"interface":"front","average":{"1min":0.52600000000000002,"5min":0.93700000000000006,"15min":0.83899999999999997},"min":{"1min":0.34999999999999998,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":1.341,"5min":14.387,"15min":14.387},"last":1.1220000000000001}]},{"osd":7,"last update":"Sat Sep 23 21:10:42 2023","interfaces":[{"interface":"back","average":{"1min":0.68600000000000005,"5min":1.1919999999999999,"15min":0.94099999999999995},"min":{"1min":0.32500000000000001,"5min":0.307,"15min":0.29299999999999998},"max":{"1min":3.3839999999999999,"5min":16.890999999999998,"15min":16.890999999999998},"last":1.1080000000000001},{"interface":"front","average":{"1min":0.48099999999999998,"5min":1.0549999999999999,"15min":1.0509999999999999},"min":{"1min":0.34000000000000002,"5min":0.33100000000000002,"15min":0.28199999999999997},"max":{"1min":0.63900000000000001,"5min":8.5830000000000002,"15min":11.170999999999999},"last":0.58999999999999997}]}]},{"osd":5,"up_from":39,"seq":167503724918,"num_pgs":47,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1194832,"kb_used_data":757384,"kb_used_omap":0,"kb_used_meta":437440,"kb_avail":92550320,"statfs":{"total":95995035648,"available":94771527680,"internally_reserved":0,"allocated":775561216,"data_stored":774337818,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":447938560},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2],"upper_bound":2},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:10:00 2023","interfaces":[{"interface":"back","average":{"1min":1.3560000000000001,"5min":1.9710000000000001,"15min":1.823},"min":{"1min":0.26400000000000001,"5min":0.19700000000000001,"15min":0.19700000000000001},"max":{"1min":5.7300000000000004,"5min":25.222999999999999,"15min":25.222999999999999},"last":0.24199999999999999},{"interface":"front","average":{"1min":1.883,"5min":1.8320000000000001,"15min":1.968},"min":{"1min":0.251,"5min":0.25,"15min":0.223},"max":{"1min":9.0090000000000003,"5min":25.457999999999998,"15min":25.457999999999998},"last":1.2}]},{"osd":1,"last update":"Sat Sep 23 21:10:00 2023","interfaces":[{"interface":"back","average":{"1min":2.145,"5min":2.319,"15min":1.9650000000000001},"min":{"1min":0.27100000000000002,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":9.4429999999999996,"5min":25.239999999999998,"15min":25.239999999999998},"last":1.647},{"interface":"front","average":{"1min":2.1680000000000001,"5min":2.2410000000000001,"15min":1.782},"min":{"1min":0.33800000000000002,"5min":0.252,"15min":0.252},"max":{"1min":9.4290000000000003,"5min":25.253,"15min":25.253},"last":1.6639999999999999}]},{"osd":2,"last update":"Sat Sep 23 21:10:18 2023","interfaces":[{"interface":"back","average":{"1min":1.7190000000000001,"5min":1.873,"15min":1.736},"min":{"1min":0.26700000000000002,"5min":0.25800000000000001,"15min":0.23499999999999999},"max":{"1min":8.0530000000000008,"5min":12.634,"15min":12.634},"last":1.6259999999999999},{"interface":"front","average":{"1min":1.8029999999999999,"5min":1.8440000000000001,"15min":1.9990000000000001},"min":{"1min":0.38800000000000001,"5min":0.28100000000000003,"15min":0.26100000000000001},"max":{"1min":6.3049999999999997,"5min":8.8079999999999998,"15min":12.300000000000001},"last":0.55200000000000005}]},{"osd":3,"last update":"Sat Sep 23 21:10:41 2023","interfaces":[{"interface":"back","average":{"1min":0.81899999999999995,"5min":2.0590000000000002,"15min":2.6179999999999999},"min":{"1min":0.36099999999999999,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":2.6240000000000001,"5min":25.503,"15min":61.667999999999999},"last":1.976},{"interface":"front","average":{"1min":1.27,"5min":2.4289999999999998,"15min":2.5830000000000002},"min":{"1min":0.35499999999999998,"5min":0.30599999999999999,"15min":0.28899999999999998},"max":{"1min":3.5699999999999998,"5min":25.466000000000001,"15min":62.276000000000003},"last":1.375}]},{"osd":4,"last update":"Sat Sep 23 21:09:50 2023","interfaces":[{"interface":"back","average":{"1min":0.38100000000000001,"5min":1.0620000000000001,"15min":0.85799999999999998},"min":{"1min":0.27000000000000002,"5min":0.27000000000000002,"15min":0.23400000000000001},"max":{"1min":0.73599999999999999,"5min":7.7800000000000002,"15min":9.0950000000000006},"last":0.28899999999999998},{"interface":"front","average":{"1min":0.55000000000000004,"5min":1.3129999999999999,"15min":0.94399999999999995},"min":{"1min":0.27000000000000002,"5min":0.26900000000000002,"15min":0.23599999999999999},"max":{"1min":1.46,"5min":11.973000000000001,"15min":11.973000000000001},"last":0.33300000000000002}]},{"osd":6,"last update":"Sat Sep 23 21:10:00 2023","interfaces":[{"interface":"back","average":{"1min":0.47099999999999997,"5min":1.0409999999999999,"15min":0.93899999999999995},"min":{"1min":0.25900000000000001,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":0.878,"5min":8.0909999999999993,"15min":11.128},"last":0.48599999999999999},{"interface":"front","average":{"1min":0.47099999999999997,"5min":1.1080000000000001,"15min":0.98099999999999998},"min":{"1min":0.32400000000000001,"5min":0.317,"15min":0.317},"max":{"1min":0.85699999999999998,"5min":7.8550000000000004,"15min":11.112},"last":0.46000000000000002}]},{"osd":7,"last update":"Sat Sep 23 21:10:41 2023","interfaces":[{"interface":"back","average":{"1min":0.52700000000000002,"5min":3.3969999999999998,"15min":1.722},"min":{"1min":0.36299999999999999,"5min":0.33100000000000002,"15min":0.307},"max":{"1min":0.82999999999999996,"5min":189.405,"15min":189.405},"last":0.58599999999999997},{"interface":"front","average":{"1min":0.5,"5min":3.359,"15min":1.659},"min":{"1min":0.373,"5min":0.35499999999999998,"15min":0.29799999999999999},"max":{"1min":0.68899999999999995,"5min":189.685,"15min":189.685},"last":0.42499999999999999}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":5434,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1259,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5322,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":4838,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1407582208,"data_stored":1407582208,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":941883392,"data_stored":941883392,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":981401600,"data_stored":981401600,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":966066176,"data_stored":966066176,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":391446528,"data_stored":391446528,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":788463616,"data_stored":788463616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":768081920,"data_stored":768081920,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T21:10:51.886 INFO:tasks.cephadm.ceph_manager.ceph:increase pool size by 10 2023-09-23T21:10:51.887 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph osd pool set unique_pool_4 pg_num 26 2023-09-23T21:10:51.951 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:10:51.955 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:10:51.955 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:10:52.056 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:10:52.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:10:51 smithi070 ceph-mon[168282]: pgmap v1126: 109 pgs: 109 active+clean; 1.9 GiB data, 9.0 GiB used, 617 GiB / 626 GiB avail; 100 MiB/s wr, 1.60k op/s 2023-09-23T21:10:52.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:10:51 smithi070 ceph-mon[168282]: from='client.56167 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T21:10:52.197 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:10:52.290 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:10:52.299 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:10:52.299 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:10:52.372 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0: {} 2023-09-23T21:10:52.400 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:10:52.411 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1: {} 2023-09-23T21:10:52.445 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 30 16 37523 37507 78.0902 69.4375 0.0175637 0.0127973 2023-09-23T21:10:52.457 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2: {} 2023-09-23T21:10:52.470 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:10:52.508 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3: {} 2023-09-23T21:10:52.547 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T21:10:52.550 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4: {} 2023-09-23T21:10:52.555 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:10:52.555 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:10:52.594 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T21:10:52.628 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6: {} 2023-09-23T21:10:52.656 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:10:52.663 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T21:10:52.677 INFO:teuthology.orchestra.run.smithi026.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T21:10:52.678 INFO:teuthology.orchestra.run.smithi026.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T21:10:52.678 INFO:teuthology.orchestra.run.smithi026.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T21:10:52.678 INFO:teuthology.orchestra.run.smithi026.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T21:10:52.678 INFO:teuthology.orchestra.run.smithi026.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T21:10:52.678 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T21:10:52.678 INFO:teuthology.orchestra.run.smithi026.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T21:10:52.678 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T21:10:52.689 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:10:52.711 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:10:52 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/3617108963' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:10:52.711 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:10:52 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:10:52.711 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:10:52 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/3990113323' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:10:52.726 INFO:teuthology.orchestra.run.smithi026.stderr:nodeep-scrub is set 2023-09-23T21:10:52.738 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:10:52.743 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:10:52.743 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:10:52.808 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:10:52.843 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:10:52.887 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:10:52 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/3617108963' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:10:52.888 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:10:52 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:10:52.888 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:10:52 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/3990113323' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:10:52.888 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:10:52 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/3617108963' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:10:52.888 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:10:52 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:10:52.888 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:10:52 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/3990113323' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:10:52.927 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:10:53.445 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 31 16 38565 38549 77.672 65.125 0.00886306 0.0128686 2023-09-23T21:10:53.604 INFO:teuthology.orchestra.run.smithi070.stdout:ERROR: (22) Invalid argument 2023-09-23T21:10:53.604 INFO:teuthology.orchestra.run.smithi070.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T21:10:53.909 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:10:53 smithi070 ceph-mon[168282]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:10:53.910 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:10:53 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T21:10:53.910 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:10:53 smithi070 ceph-mon[168282]: osdmap e436: 8 total, 8 up, 7 in 2023-09-23T21:10:53.910 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:10:53 smithi070 ceph-mon[168282]: failed to encode map e436 with expected crc 2023-09-23T21:10:53.910 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:10:53 smithi070 ceph-mon[168282]: failed to encode map e436 with expected crc 2023-09-23T21:10:53.910 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:10:53 smithi070 ceph-mon[168282]: failed to encode map e436 with expected crc 2023-09-23T21:10:53.910 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:10:53 smithi070 ceph-mon[168282]: pgmap v1128: 109 pgs: 109 active+clean; 2.2 GiB data, 9.8 GiB used, 616 GiB / 626 GiB avail; 91 MiB/s wr, 1.46k op/s 2023-09-23T21:10:53.962 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_blocked_ops 2023-09-23T21:10:54.044 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:10:53 smithi026 ceph-mon[179688]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:10:54.044 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:10:53 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T21:10:54.044 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:10:53 smithi026 ceph-mon[179688]: osdmap e436: 8 total, 8 up, 7 in 2023-09-23T21:10:54.044 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:10:53 smithi026 ceph-mon[179688]: failed to encode map e436 with expected crc 2023-09-23T21:10:54.044 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:10:53 smithi026 ceph-mon[179688]: failed to encode map e436 with expected crc 2023-09-23T21:10:54.044 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:10:53 smithi026 ceph-mon[179688]: failed to encode map e436 with expected crc 2023-09-23T21:10:54.044 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:10:53 smithi026 ceph-mon[179688]: pgmap v1128: 109 pgs: 109 active+clean; 2.2 GiB data, 9.8 GiB used, 616 GiB / 626 GiB avail; 91 MiB/s wr, 1.46k op/s 2023-09-23T21:10:54.044 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:10:53 smithi026 ceph-mon[175151]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:10:54.044 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:10:53 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T21:10:54.045 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:10:53 smithi026 ceph-mon[175151]: osdmap e436: 8 total, 8 up, 7 in 2023-09-23T21:10:54.045 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:10:53 smithi026 ceph-mon[175151]: failed to encode map e436 with expected crc 2023-09-23T21:10:54.045 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:10:53 smithi026 ceph-mon[175151]: failed to encode map e436 with expected crc 2023-09-23T21:10:54.045 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:10:53 smithi026 ceph-mon[175151]: failed to encode map e436 with expected crc 2023-09-23T21:10:54.045 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:10:53 smithi026 ceph-mon[175151]: pgmap v1128: 109 pgs: 109 active+clean; 2.2 GiB data, 9.8 GiB used, 616 GiB / 626 GiB avail; 91 MiB/s wr, 1.46k op/s 2023-09-23T21:10:54.341 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:10:54.345 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:10:54.346 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:10:54.445 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 32 16 39671 39655 77.4049 69.125 0.013278 0.0129129 2023-09-23T21:10:54.447 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:10:54.523 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:10:54.595 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T21:10:54.873 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:10:54.887 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:10:54.887 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:10:54.978 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:10:54 smithi026 ceph-mon[179688]: failed to encode map e436 with expected crc 2023-09-23T21:10:54.978 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:10:54 smithi026 ceph-mon[175151]: failed to encode map e436 with expected crc 2023-09-23T21:10:54.985 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:10:54 smithi070 ceph-mon[168282]: failed to encode map e436 with expected crc 2023-09-23T21:10:54.989 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:10:55.020 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:10:55.068 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:10:55.072 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:10:55.072 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:10:55.173 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:10:55.229 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:10:55.359 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:10:55.370 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:10:55.371 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:10:55.447 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 33 16 41004 40988 77.5789 83.3125 0.0128549 0.012883 2023-09-23T21:10:55.472 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:10:55.503 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:10:55.741 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:10:55.744 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:10:55.744 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:10:55.845 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:10:55.878 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:10:55.942 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:10:55.946 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:10:55.946 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:10:56.029 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:10:55 smithi070 ceph-mon[168282]: pgmap v1129: 109 pgs: 109 active+clean; 2.3 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 86 MiB/s wr, 1.38k op/s 2023-09-23T21:10:56.047 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:10:56.085 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:10:56.127 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:10:56.130 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:10:56.130 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:10:56.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:10:55 smithi026 ceph-mon[175151]: pgmap v1129: 109 pgs: 109 active+clean; 2.3 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 86 MiB/s wr, 1.38k op/s 2023-09-23T21:10:56.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:10:55 smithi026 ceph-mon[179688]: pgmap v1129: 109 pgs: 109 active+clean; 2.3 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 86 MiB/s wr, 1.38k op/s 2023-09-23T21:10:56.231 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:10:56.285 INFO:teuthology.orchestra.run.smithi070.stdout:ERROR: (22) Invalid argument 2023-09-23T21:10:56.285 INFO:teuthology.orchestra.run.smithi070.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T21:10:56.310 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:10:56.387 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T21:10:56.392 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:10:56.392 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:10:56.447 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 34 16 42271 42255 77.626 79.1875 0.0134474 0.0128751 2023-09-23T21:10:56.493 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:10:56.526 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:10:56.671 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:10:56.674 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:10:56.675 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:10:56.773 INFO:teuthology.orchestra.run.smithi026.stderr:set pool 10 pg_num to 26 2023-09-23T21:10:56.775 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:10:56.781 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_historic_ops 2023-09-23T21:10:56.806 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:10:56.848 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:10:56.852 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:10:56.852 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:10:56.953 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:10:57.000 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:10:56 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/3269159464' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pg_num", "val": "26"}]: dispatch 2023-09-23T21:10:57.000 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:10:56 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pg_num", "val": "26"}]: dispatch 2023-09-23T21:10:57.000 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:10:56 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/3269159464' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pg_num", "val": "26"}]: dispatch 2023-09-23T21:10:57.000 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:10:56 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pg_num", "val": "26"}]: dispatch 2023-09-23T21:10:57.013 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T21:10:57.036 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:10:57.038 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T21:10:57.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:10:56 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/3269159464' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pg_num", "val": "26"}]: dispatch 2023-09-23T21:10:57.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:10:56 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pg_num", "val": "26"}]: dispatch 2023-09-23T21:10:57.447 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 35 16 43452 43436 77.5169 73.8125 0.02214 0.0128942 2023-09-23T21:10:57.471 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T21:10:58.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:10:57 smithi070 ceph-mon[168282]: failed to encode map e437 with expected crc 2023-09-23T21:10:58.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:10:57 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pg_num", "val": "26"}]': finished 2023-09-23T21:10:58.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:10:57 smithi070 ceph-mon[168282]: osdmap e437: 8 total, 8 up, 7 in 2023-09-23T21:10:58.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:10:57 smithi070 ceph-mon[168282]: failed to encode map e437 with expected crc 2023-09-23T21:10:58.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:10:57 smithi070 ceph-mon[168282]: failed to encode map e437 with expected crc 2023-09-23T21:10:58.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:10:57 smithi070 ceph-mon[168282]: failed to encode map e437 with expected crc 2023-09-23T21:10:58.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:10:57 smithi070 ceph-mon[168282]: pgmap v1131: 109 pgs: 109 active+clean; 2.4 GiB data, 10 GiB used, 615 GiB / 626 GiB avail; 66 MiB/s wr, 1.05k op/s 2023-09-23T21:10:58.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:10:57 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pg_num_actual", "val": "26"}]: dispatch 2023-09-23T21:10:58.127 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:10:58.129 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:10:57 smithi026 ceph-mon[175151]: failed to encode map e437 with expected crc 2023-09-23T21:10:58.129 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:10:57 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pg_num", "val": "26"}]': finished 2023-09-23T21:10:58.129 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:10:57 smithi026 ceph-mon[175151]: osdmap e437: 8 total, 8 up, 7 in 2023-09-23T21:10:58.129 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:10:57 smithi026 ceph-mon[175151]: failed to encode map e437 with expected crc 2023-09-23T21:10:58.129 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:10:57 smithi026 ceph-mon[175151]: failed to encode map e437 with expected crc 2023-09-23T21:10:58.129 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:10:57 smithi026 ceph-mon[175151]: failed to encode map e437 with expected crc 2023-09-23T21:10:58.129 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:10:57 smithi026 ceph-mon[175151]: pgmap v1131: 109 pgs: 109 active+clean; 2.4 GiB data, 10 GiB used, 615 GiB / 626 GiB avail; 66 MiB/s wr, 1.05k op/s 2023-09-23T21:10:58.129 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:10:57 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pg_num_actual", "val": "26"}]: dispatch 2023-09-23T21:10:58.129 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:10:57 smithi026 ceph-mon[179688]: failed to encode map e437 with expected crc 2023-09-23T21:10:58.130 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:10:57 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pg_num", "val": "26"}]': finished 2023-09-23T21:10:58.130 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:10:57 smithi026 ceph-mon[179688]: osdmap e437: 8 total, 8 up, 7 in 2023-09-23T21:10:58.130 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:10:57 smithi026 ceph-mon[179688]: failed to encode map e437 with expected crc 2023-09-23T21:10:58.130 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:10:57 smithi026 ceph-mon[179688]: failed to encode map e437 with expected crc 2023-09-23T21:10:58.130 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:10:57 smithi026 ceph-mon[179688]: failed to encode map e437 with expected crc 2023-09-23T21:10:58.130 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:10:57 smithi026 ceph-mon[179688]: pgmap v1131: 109 pgs: 109 active+clean; 2.4 GiB data, 10 GiB used, 615 GiB / 626 GiB avail; 66 MiB/s wr, 1.05k op/s 2023-09-23T21:10:58.130 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:10:57 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pg_num_actual", "val": "26"}]: dispatch 2023-09-23T21:10:58.130 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:10:58.131 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:10:58.232 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:10:58.338 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:10:58.447 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 36 16 43895 43879 76.1333 27.6875 0.00428735 0.0129002 2023-09-23T21:10:58.464 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:10:58.683 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:10:58.685 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:10:58.686 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:10:58.787 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:10:58.836 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:10:58.888 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:10:58.909 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:10:58.915 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:10:58.915 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:10:59.003 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:10:58 smithi070 ceph-mon[168282]: failed to encode map e438 with expected crc 2023-09-23T21:10:59.004 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:10:58 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pg_num_actual", "val": "26"}]': finished 2023-09-23T21:10:59.004 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:10:58 smithi070 ceph-mon[168282]: osdmap e438: 8 total, 8 up, 7 in 2023-09-23T21:10:59.004 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:10:58 smithi070 ceph-mon[168282]: failed to encode map e438 with expected crc 2023-09-23T21:10:59.004 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:10:58 smithi070 ceph-mon[168282]: failed to encode map e438 with expected crc 2023-09-23T21:10:59.016 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:10:59.052 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:10:58 smithi026 ceph-mon[179688]: failed to encode map e438 with expected crc 2023-09-23T21:10:59.052 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:10:58 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pg_num_actual", "val": "26"}]': finished 2023-09-23T21:10:59.052 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:10:58 smithi026 ceph-mon[179688]: osdmap e438: 8 total, 8 up, 7 in 2023-09-23T21:10:59.052 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:10:58 smithi026 ceph-mon[179688]: failed to encode map e438 with expected crc 2023-09-23T21:10:59.052 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:10:58 smithi026 ceph-mon[179688]: failed to encode map e438 with expected crc 2023-09-23T21:10:59.054 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:10:59.056 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:10:58 smithi026 ceph-mon[175151]: failed to encode map e438 with expected crc 2023-09-23T21:10:59.056 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:10:58 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pg_num_actual", "val": "26"}]': finished 2023-09-23T21:10:59.056 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:10:58 smithi026 ceph-mon[175151]: osdmap e438: 8 total, 8 up, 7 in 2023-09-23T21:10:59.056 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:10:58 smithi026 ceph-mon[175151]: failed to encode map e438 with expected crc 2023-09-23T21:10:59.057 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:10:58 smithi026 ceph-mon[175151]: failed to encode map e438 with expected crc 2023-09-23T21:10:59.145 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:10:59.148 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:10:59.149 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:10:59.237 INFO:teuthology.orchestra.run.smithi070.stdout:ERROR: (22) Invalid argument 2023-09-23T21:10:59.237 INFO:teuthology.orchestra.run.smithi070.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T21:10:59.250 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:10:59.312 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:10:59.436 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:10:59.442 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:10:59.442 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:10:59.449 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 37 16 44714 44698 75.4558 51.1875 0.0103303 0.0132488 2023-09-23T21:10:59.543 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:10:59.575 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:10:59.615 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:10:59.618 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:10:59.619 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:10:59.653 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_ops_in_flight 2023-09-23T21:10:59.720 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:10:59.750 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:10:59.787 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:10:59.790 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:10:59.791 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:10:59.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:10:59 smithi070 ceph-mon[168282]: failed to encode map e439 with expected crc 2023-09-23T21:10:59.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:10:59 smithi070 ceph-mon[168282]: failed to encode map e439 with expected crc 2023-09-23T21:10:59.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:10:59 smithi070 ceph-mon[168282]: osdmap e439: 8 total, 8 up, 7 in 2023-09-23T21:10:59.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:10:59 smithi070 ceph-mon[168282]: failed to encode map e439 with expected crc 2023-09-23T21:10:59.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:10:59 smithi070 ceph-mon[168282]: failed to encode map e439 with expected crc 2023-09-23T21:10:59.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:10:59 smithi070 ceph-mon[168282]: failed to encode map e439 with expected crc 2023-09-23T21:10:59.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:10:59 smithi070 ceph-mon[168282]: failed to encode map e439 with expected crc 2023-09-23T21:10:59.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:10:59 smithi070 ceph-mon[168282]: pgmap v1134: 119 pgs: 2 peering, 10 unknown, 107 active+clean; 2.6 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 72 MiB/s wr, 1.15k op/s 2023-09-23T21:10:59.882 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 21:10:59 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[164029]: ::ffff:172.21.15.70 - - [23/Sep/2023:21:10:59] "GET /metrics HTTP/1.1" 200 36387 "" "Prometheus/2.43.0" 2023-09-23T21:10:59.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:10:59 smithi026 ceph-mon[175151]: failed to encode map e439 with expected crc 2023-09-23T21:10:59.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:10:59 smithi026 ceph-mon[175151]: failed to encode map e439 with expected crc 2023-09-23T21:10:59.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:10:59 smithi026 ceph-mon[175151]: osdmap e439: 8 total, 8 up, 7 in 2023-09-23T21:10:59.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:10:59 smithi026 ceph-mon[175151]: failed to encode map e439 with expected crc 2023-09-23T21:10:59.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:10:59 smithi026 ceph-mon[175151]: failed to encode map e439 with expected crc 2023-09-23T21:10:59.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:10:59 smithi026 ceph-mon[175151]: failed to encode map e439 with expected crc 2023-09-23T21:10:59.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:10:59 smithi026 ceph-mon[175151]: failed to encode map e439 with expected crc 2023-09-23T21:10:59.883 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:10:59 smithi026 ceph-mon[175151]: pgmap v1134: 119 pgs: 2 peering, 10 unknown, 107 active+clean; 2.6 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 72 MiB/s wr, 1.15k op/s 2023-09-23T21:10:59.884 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:10:59 smithi026 ceph-mon[179688]: failed to encode map e439 with expected crc 2023-09-23T21:10:59.884 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:10:59 smithi026 ceph-mon[179688]: failed to encode map e439 with expected crc 2023-09-23T21:10:59.884 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:10:59 smithi026 ceph-mon[179688]: osdmap e439: 8 total, 8 up, 7 in 2023-09-23T21:10:59.884 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:10:59 smithi026 ceph-mon[179688]: failed to encode map e439 with expected crc 2023-09-23T21:10:59.884 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:10:59 smithi026 ceph-mon[179688]: failed to encode map e439 with expected crc 2023-09-23T21:10:59.884 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:10:59 smithi026 ceph-mon[179688]: failed to encode map e439 with expected crc 2023-09-23T21:10:59.884 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:10:59 smithi026 ceph-mon[179688]: failed to encode map e439 with expected crc 2023-09-23T21:10:59.884 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:10:59 smithi026 ceph-mon[179688]: pgmap v1134: 119 pgs: 2 peering, 10 unknown, 107 active+clean; 2.6 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 72 MiB/s wr, 1.15k op/s 2023-09-23T21:10:59.892 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:10:59.919 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:10:59.957 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:10:59.960 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:10:59.960 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:11:00.061 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:11:00.132 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:11:00.208 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:11:00.219 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:11:00.219 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:11:00.320 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:11:00.388 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:11:00.453 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 38 16 46022 46006 75.6143 81.75 0.00728405 0.013219 2023-09-23T21:11:00.680 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:11:00.682 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T21:11:00.694 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:11:00.695 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:11:00.795 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:11:00.884 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:11:01.058 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:11:01.065 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:11:01.066 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:11:01.085 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0: {} 2023-09-23T21:11:01.123 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1: {} 2023-09-23T21:11:01.167 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:11:01.174 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2: {} 2023-09-23T21:11:01.217 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3: {} 2023-09-23T21:11:01.267 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:11:01.269 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4: {} 2023-09-23T21:11:01.304 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T21:11:01.335 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6: {} 2023-09-23T21:11:01.377 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T21:11:01.387 INFO:teuthology.orchestra.run.smithi026.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T21:11:01.387 INFO:teuthology.orchestra.run.smithi026.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T21:11:01.388 INFO:teuthology.orchestra.run.smithi026.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T21:11:01.388 INFO:teuthology.orchestra.run.smithi026.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T21:11:01.388 INFO:teuthology.orchestra.run.smithi026.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T21:11:01.388 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T21:11:01.388 INFO:teuthology.orchestra.run.smithi026.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T21:11:01.388 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T21:11:01.453 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 39 16 47246 47230 75.6368 76.5 0.00700581 0.0132158 2023-09-23T21:11:01.706 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T21:11:01.714 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:11:01.715 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:11:01.720 INFO:teuthology.orchestra.run.smithi026.stderr:noscrub is unset 2023-09-23T21:11:01.815 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:11:01.842 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:11:01.877 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:11:01.880 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:11:01.880 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:11:01.981 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:11:02.007 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:11:01 smithi070 ceph-mon[168282]: pgmap v1135: 119 pgs: 2 peering, 9 unknown, 108 active+clean; 2.6 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 59 MiB/s wr, 951 op/s; 11 KiB/s, 0 objects/s recovering 2023-09-23T21:11:02.007 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:11:01 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/2774894391' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:11:02.007 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:11:01 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/1749214955' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T21:11:02.007 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:11:01 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T21:11:02.007 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:11:01 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:11:02.009 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:11:02.049 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:11:02.052 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:11:02.052 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:11:02.121 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:11:01 smithi026 ceph-mon[179688]: pgmap v1135: 119 pgs: 2 peering, 9 unknown, 108 active+clean; 2.6 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 59 MiB/s wr, 951 op/s; 11 KiB/s, 0 objects/s recovering 2023-09-23T21:11:02.121 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:11:01 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/2774894391' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:11:02.121 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:11:01 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/1749214955' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T21:11:02.121 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:11:01 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T21:11:02.121 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:11:01 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:11:02.121 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:11:01 smithi026 ceph-mon[175151]: pgmap v1135: 119 pgs: 2 peering, 9 unknown, 108 active+clean; 2.6 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 59 MiB/s wr, 951 op/s; 11 KiB/s, 0 objects/s recovering 2023-09-23T21:11:02.121 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:11:01 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/2774894391' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:11:02.121 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:11:01 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/1749214955' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T21:11:02.122 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:11:01 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T21:11:02.122 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:11:01 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:11:02.153 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:11:02.254 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:11:02.412 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:11:02.420 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:11:02.420 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:11:02.462 INFO:tasks.radosbench.radosbench.0.smithi026.stdout:2023-09-23T21:11:02.462507+0000 min lat: 0.00120238 max lat: 2.42702 avg lat: 0.0131712 2023-09-23T21:11:02.462 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T21:11:02.462 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 40 16 48629 48613 75.8902 86.4375 0.0126224 0.0131712 2023-09-23T21:11:02.522 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:11:02.552 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:11:02.589 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:11:02.592 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:11:02.592 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:11:02.693 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:11:02.697 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:11:02 smithi026 ceph-mon[179688]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:11:02.697 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:11:02 smithi026 ceph-mon[179688]: failed to encode map e440 with expected crc 2023-09-23T21:11:02.697 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:11:02 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T21:11:02.697 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:11:02 smithi026 ceph-mon[179688]: osdmap e440: 8 total, 8 up, 7 in 2023-09-23T21:11:02.697 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:11:02 smithi026 ceph-mon[179688]: failed to encode map e440 with expected crc 2023-09-23T21:11:02.697 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:11:02 smithi026 ceph-mon[179688]: failed to encode map e440 with expected crc 2023-09-23T21:11:02.698 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:11:02 smithi026 ceph-mon[179688]: failed to encode map e440 with expected crc 2023-09-23T21:11:02.698 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:11:02 smithi026 ceph-mon[179688]: 10.3 scrub starts 2023-09-23T21:11:02.698 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:11:02 smithi026 ceph-mon[175151]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:11:02.698 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:11:02 smithi026 ceph-mon[175151]: failed to encode map e440 with expected crc 2023-09-23T21:11:02.698 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:11:02 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T21:11:02.698 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:11:02 smithi026 ceph-mon[175151]: osdmap e440: 8 total, 8 up, 7 in 2023-09-23T21:11:02.698 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:11:02 smithi026 ceph-mon[175151]: failed to encode map e440 with expected crc 2023-09-23T21:11:02.698 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:11:02 smithi026 ceph-mon[175151]: failed to encode map e440 with expected crc 2023-09-23T21:11:02.698 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:11:02 smithi026 ceph-mon[175151]: failed to encode map e440 with expected crc 2023-09-23T21:11:02.698 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:11:02 smithi026 ceph-mon[175151]: 10.3 scrub starts 2023-09-23T21:11:02.721 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:11:02.761 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:11:02.764 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:11:02.764 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:11:02.865 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:11:02.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:11:02 smithi070 ceph-mon[168282]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:11:02.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:11:02 smithi070 ceph-mon[168282]: failed to encode map e440 with expected crc 2023-09-23T21:11:02.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:11:02 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T21:11:02.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:11:02 smithi070 ceph-mon[168282]: osdmap e440: 8 total, 8 up, 7 in 2023-09-23T21:11:02.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:11:02 smithi070 ceph-mon[168282]: failed to encode map e440 with expected crc 2023-09-23T21:11:02.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:11:02 smithi070 ceph-mon[168282]: failed to encode map e440 with expected crc 2023-09-23T21:11:02.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:11:02 smithi070 ceph-mon[168282]: failed to encode map e440 with expected crc 2023-09-23T21:11:02.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:11:02 smithi070 ceph-mon[168282]: 10.3 scrub starts 2023-09-23T21:11:02.875 INFO:teuthology.orchestra.run.smithi026.stdout:{ 2023-09-23T21:11:02.876 INFO:teuthology.orchestra.run.smithi026.stdout: "ops": [ 2023-09-23T21:11:02.876 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:02.876 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.46328.0:49195 10.b 10:dcbca5d3:::benchmark_data_smithi026_379838_object49194:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e440)", 2023-09-23T21:11:02.876 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T21:11:02.872705+0000", 2023-09-23T21:11:02.876 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 0.0028945099999999999, 2023-09-23T21:11:02.876 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0029306089999999998, 2023-09-23T21:11:02.876 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:11:02.876 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "waiting for sub ops", 2023-09-23T21:11:02.876 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T21:11:02.876 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.46328", 2023-09-23T21:11:02.876 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/3748861181", 2023-09-23T21:11:02.876 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 49195 2023-09-23T21:11:02.877 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:02.877 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:11:02.877 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:02.877 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:11:02.877 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:11:02.872705+0000", 2023-09-23T21:11:02.877 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:11:02.877 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:02.877 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:02.877 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:11:02.877 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:11:02.872705+0000", 2023-09-23T21:11:02.877 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:11:02.877 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:02.877 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:02.877 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:11:02.878 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:11:02.872703+0000", 2023-09-23T21:11:02.878 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T21:11:02.878 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:02.878 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:02.878 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:11:02.878 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:11:02.872730+0000", 2023-09-23T21:11:02.878 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.6443000000000001e-05 2023-09-23T21:11:02.878 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:02.878 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:02.878 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:11:02.878 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:11:02.872747+0000", 2023-09-23T21:11:02.878 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.7028e-05 2023-09-23T21:11:02.879 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:02.879 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:02.879 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:11:02.879 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:11:02.872754+0000", 2023-09-23T21:11:02.879 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.8019999999999992e-06 2023-09-23T21:11:02.879 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:02.879 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:02.879 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:11:02.879 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:11:02.872926+0000", 2023-09-23T21:11:02.879 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00017103600000000001 2023-09-23T21:11:02.879 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:02.879 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:02.879 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:11:02.880 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:11:02.873072+0000", 2023-09-23T21:11:02.880 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00014691199999999999 2023-09-23T21:11:02.880 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:02.880 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:02.880 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 0,7", 2023-09-23T21:11:02.880 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:11:02.873215+0000", 2023-09-23T21:11:02.880 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000142786 2023-09-23T21:11:02.880 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:02.880 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:02.880 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T21:11:02.880 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:11:02.875457+0000", 2023-09-23T21:11:02.880 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0022416599999999999 2023-09-23T21:11:02.880 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:11:02.881 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:11:02.881 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:11:02.881 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:11:02.881 INFO:teuthology.orchestra.run.smithi026.stdout: ], 2023-09-23T21:11:02.881 INFO:teuthology.orchestra.run.smithi026.stdout: "num_ops": 1 2023-09-23T21:11:02.881 INFO:teuthology.orchestra.run.smithi026.stdout:} 2023-09-23T21:11:02.893 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:11:02.932 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:11:02.935 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:11:02.935 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:11:03.036 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:11:03.105 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:11:03.143 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 7] out_osds: [6] dead_osds: [] live_osds: [0, 2, 3, 4, 5, 7, 6, 1] 2023-09-23T21:11:03.143 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T21:11:03.143 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '3.0', 'overage': '1000', 'type': 'reweight-by-utilization'} 2023-09-23T21:11:03.143 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph osd reweight-by-utilization 1000 3.0 2023-09-23T21:11:03.274 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:11:03.282 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:11:03.283 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:11:03.365 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_blocked_ops 2023-09-23T21:11:03.384 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:11:03.412 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:11:03.449 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:11:03.452 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:11:03.452 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:11:03.462 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 41 16 49931 49915 76.0236 81.375 0.0141857 0.0131475 2023-09-23T21:11:03.553 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:11:03.580 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:11:03.615 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:11:03.618 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:11:03.619 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:11:03.719 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:11:03.807 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:11:03.887 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:11:03.893 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:11:03.894 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:11:03.995 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:11:04.007 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:11:04.022 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:11:03 smithi070 ceph-mon[168282]: 10.3 scrub ok 2023-09-23T21:11:04.022 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:11:03 smithi070 ceph-mon[168282]: 10.5 scrub starts 2023-09-23T21:11:04.022 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:11:03 smithi070 ceph-mon[168282]: pgmap v1137: 119 pgs: 2 peering, 2 unknown, 115 active+clean; 2.9 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 80 MiB/s wr, 1.28k op/s; 43 KiB/s, 0 objects/s recovering 2023-09-23T21:11:04.025 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:11:04.062 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:11:04.065 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:11:04.066 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:11:04.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:11:03 smithi026 ceph-mon[175151]: 10.3 scrub ok 2023-09-23T21:11:04.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:11:03 smithi026 ceph-mon[175151]: 10.5 scrub starts 2023-09-23T21:11:04.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:11:03 smithi026 ceph-mon[175151]: pgmap v1137: 119 pgs: 2 peering, 2 unknown, 115 active+clean; 2.9 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 80 MiB/s wr, 1.28k op/s; 43 KiB/s, 0 objects/s recovering 2023-09-23T21:11:04.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:11:03 smithi026 ceph-mon[179688]: 10.3 scrub ok 2023-09-23T21:11:04.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:11:03 smithi026 ceph-mon[179688]: 10.5 scrub starts 2023-09-23T21:11:04.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:11:03 smithi026 ceph-mon[179688]: pgmap v1137: 119 pgs: 2 peering, 2 unknown, 115 active+clean; 2.9 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 80 MiB/s wr, 1.28k op/s; 43 KiB/s, 0 objects/s recovering 2023-09-23T21:11:04.148 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T21:11:04.167 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:11:04.194 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:11:04.231 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:11:04.233 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:11:04.234 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:11:04.282 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:11:04.334 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:11:04.410 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:11:04.463 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 42 16 51039 51023 75.8606 69.25 0.0289821 0.0131758 2023-09-23T21:11:04.473 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T21:11:04.480 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:11:04.480 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:11:04.483 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T21:11:04.582 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:11:04.706 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:11:04.732 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:11:04 smithi026 ceph-mon[175151]: 10.5 scrub ok 2023-09-23T21:11:04.732 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:11:04 smithi026 ceph-mon[175151]: 10.4 scrub starts 2023-09-23T21:11:04.733 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:11:04 smithi026 ceph-mon[175151]: 10.1 scrub starts 2023-09-23T21:11:04.733 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:11:04 smithi026 ceph-mon[175151]: 10.4 scrub ok 2023-09-23T21:11:04.733 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:11:04 smithi026 ceph-mon[175151]: 10.0 scrub starts 2023-09-23T21:11:04.733 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:11:04 smithi026 ceph-mon[179688]: 10.5 scrub ok 2023-09-23T21:11:04.733 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:11:04 smithi026 ceph-mon[179688]: 10.4 scrub starts 2023-09-23T21:11:04.733 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:11:04 smithi026 ceph-mon[179688]: 10.1 scrub starts 2023-09-23T21:11:04.733 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:11:04 smithi026 ceph-mon[179688]: 10.4 scrub ok 2023-09-23T21:11:04.733 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:11:04 smithi026 ceph-mon[179688]: 10.0 scrub starts 2023-09-23T21:11:04.834 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T21:11:04.845 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:11:04.845 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:11:04.946 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:11:05.042 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:11:05.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:11:04 smithi070 ceph-mon[168282]: 10.5 scrub ok 2023-09-23T21:11:05.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:11:04 smithi070 ceph-mon[168282]: 10.4 scrub starts 2023-09-23T21:11:05.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:11:04 smithi070 ceph-mon[168282]: 10.1 scrub starts 2023-09-23T21:11:05.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:11:04 smithi070 ceph-mon[168282]: 10.4 scrub ok 2023-09-23T21:11:05.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:11:04 smithi070 ceph-mon[168282]: 10.0 scrub starts 2023-09-23T21:11:05.458 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:11:05.463 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 43 16 52157 52141 75.7214 69.875 0.0213738 0.0131975 2023-09-23T21:11:05.464 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:11:05.465 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:11:05.489 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:11:05.566 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:11:05.653 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:11:05.767 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:11:05.918 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:11:05.929 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:11:05 smithi026 ceph-mon[175151]: 10.0 scrub ok 2023-09-23T21:11:05.929 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:11:05 smithi026 ceph-mon[175151]: pgmap v1138: 119 pgs: 119 active+clean; 3.0 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 61 MiB/s wr, 973 op/s; 36 KiB/s, 0 objects/s recovering 2023-09-23T21:11:05.929 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:11:05 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num_actual", "val": "17"}]: dispatch 2023-09-23T21:11:05.929 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:11:05 smithi026 ceph-mon[175151]: 10.2 scrub starts 2023-09-23T21:11:05.929 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:11:05 smithi026 ceph-mon[179688]: 10.0 scrub ok 2023-09-23T21:11:05.930 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:11:05 smithi026 ceph-mon[179688]: pgmap v1138: 119 pgs: 119 active+clean; 3.0 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 61 MiB/s wr, 973 op/s; 36 KiB/s, 0 objects/s recovering 2023-09-23T21:11:05.930 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:11:05 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num_actual", "val": "17"}]: dispatch 2023-09-23T21:11:05.930 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:11:05 smithi026 ceph-mon[179688]: 10.2 scrub starts 2023-09-23T21:11:05.930 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:11:05.931 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:11:06.031 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:11:06.086 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:11:06.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:11:05 smithi070 ceph-mon[168282]: 10.0 scrub ok 2023-09-23T21:11:06.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:11:05 smithi070 ceph-mon[168282]: pgmap v1138: 119 pgs: 119 active+clean; 3.0 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 61 MiB/s wr, 973 op/s; 36 KiB/s, 0 objects/s recovering 2023-09-23T21:11:06.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:11:05 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num_actual", "val": "17"}]: dispatch 2023-09-23T21:11:06.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:11:05 smithi070 ceph-mon[168282]: 10.2 scrub starts 2023-09-23T21:11:06.463 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 44 16 52865 52849 75.0065 44.25 0.00178276 0.0131926 2023-09-23T21:11:06.679 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T21:11:06.683 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:11:06.683 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:11:06.784 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:11:06.856 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:11:06.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:11:06 smithi026 ceph-mon[175151]: 10.2 scrub ok 2023-09-23T21:11:06.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:11:06 smithi026 ceph-mon[175151]: 10.7 scrub starts 2023-09-23T21:11:06.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:11:06 smithi026 ceph-mon[175151]: failed to encode map e441 with expected crc 2023-09-23T21:11:06.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:11:06 smithi026 ceph-mon[175151]: failed to encode map e441 with expected crc 2023-09-23T21:11:06.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:11:06 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num_actual", "val": "17"}]': finished 2023-09-23T21:11:06.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:11:06 smithi026 ceph-mon[175151]: osdmap e441: 8 total, 8 up, 7 in 2023-09-23T21:11:06.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:11:06 smithi026 ceph-mon[175151]: 10.11 scrub starts 2023-09-23T21:11:06.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:11:06 smithi026 ceph-mon[175151]: 10.7 scrub ok 2023-09-23T21:11:06.883 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:11:06 smithi026 ceph-mon[175151]: 10.11 scrub ok 2023-09-23T21:11:06.883 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:11:06 smithi026 ceph-mon[179688]: 10.2 scrub ok 2023-09-23T21:11:06.883 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:11:06 smithi026 ceph-mon[179688]: 10.7 scrub starts 2023-09-23T21:11:06.883 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:11:06 smithi026 ceph-mon[179688]: failed to encode map e441 with expected crc 2023-09-23T21:11:06.883 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:11:06 smithi026 ceph-mon[179688]: failed to encode map e441 with expected crc 2023-09-23T21:11:06.883 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:11:06 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num_actual", "val": "17"}]': finished 2023-09-23T21:11:06.883 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:11:06 smithi026 ceph-mon[179688]: osdmap e441: 8 total, 8 up, 7 in 2023-09-23T21:11:06.883 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:11:06 smithi026 ceph-mon[179688]: 10.11 scrub starts 2023-09-23T21:11:06.883 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:11:06 smithi026 ceph-mon[179688]: 10.7 scrub ok 2023-09-23T21:11:06.883 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:11:06 smithi026 ceph-mon[179688]: 10.11 scrub ok 2023-09-23T21:11:06.901 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:11:06.904 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:11:06.905 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:11:07.005 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:11:07.017 INFO:teuthology.orchestra.run.smithi026.stdout:{ 2023-09-23T21:11:07.017 INFO:teuthology.orchestra.run.smithi026.stdout: "ops": [], 2023-09-23T21:11:07.017 INFO:teuthology.orchestra.run.smithi026.stdout: "complaint_time": 30, 2023-09-23T21:11:07.017 INFO:teuthology.orchestra.run.smithi026.stdout: "num_blocked_ops": 0 2023-09-23T21:11:07.017 INFO:teuthology.orchestra.run.smithi026.stdout:} 2023-09-23T21:11:07.037 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:11:07.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:11:06 smithi070 ceph-mon[168282]: 10.2 scrub ok 2023-09-23T21:11:07.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:11:06 smithi070 ceph-mon[168282]: 10.7 scrub starts 2023-09-23T21:11:07.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:11:06 smithi070 ceph-mon[168282]: failed to encode map e441 with expected crc 2023-09-23T21:11:07.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:11:06 smithi070 ceph-mon[168282]: failed to encode map e441 with expected crc 2023-09-23T21:11:07.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:11:06 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num_actual", "val": "17"}]': finished 2023-09-23T21:11:07.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:11:06 smithi070 ceph-mon[168282]: osdmap e441: 8 total, 8 up, 7 in 2023-09-23T21:11:07.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:11:06 smithi070 ceph-mon[168282]: 10.11 scrub starts 2023-09-23T21:11:07.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:11:06 smithi070 ceph-mon[168282]: 10.7 scrub ok 2023-09-23T21:11:07.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:11:06 smithi070 ceph-mon[168282]: 10.11 scrub ok 2023-09-23T21:11:07.463 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 45 16 52865 52849 73.3408 0 - 0.0131926 2023-09-23T21:11:07.755 INFO:teuthology.orchestra.run.smithi026.stdout:moved 19 / 357 (5.32213%) 2023-09-23T21:11:07.756 INFO:teuthology.orchestra.run.smithi026.stdout:avg 51 2023-09-23T21:11:07.756 INFO:teuthology.orchestra.run.smithi026.stdout:stddev 15.9194 -> 8.75051 (expected baseline 6.61168) 2023-09-23T21:11:07.756 INFO:teuthology.orchestra.run.smithi026.stdout:min osd.4 with 16 -> 35 pgs (0.313726 -> 0.686275 * mean) 2023-09-23T21:11:07.756 INFO:teuthology.orchestra.run.smithi026.stdout:max osd.0 with 70 -> 65 pgs (1.37255 -> 1.27451 * mean) 2023-09-23T21:11:07.756 INFO:teuthology.orchestra.run.smithi026.stdout: 2023-09-23T21:11:07.756 INFO:teuthology.orchestra.run.smithi026.stdout:oload 1000 2023-09-23T21:11:07.756 INFO:teuthology.orchestra.run.smithi026.stdout:max_change 3 2023-09-23T21:11:07.756 INFO:teuthology.orchestra.run.smithi026.stdout:max_change_osds 4 2023-09-23T21:11:07.756 INFO:teuthology.orchestra.run.smithi026.stdout:average_utilization 0.0200 2023-09-23T21:11:07.756 INFO:teuthology.orchestra.run.smithi026.stdout:overload_utilization 0.1996 2023-09-23T21:11:07.757 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4 weight 0.2675 -> 0.5638 2023-09-23T21:11:08.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:11:07 smithi070 ceph-mon[168282]: failed to encode map e442 with expected crc 2023-09-23T21:11:08.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:11:07 smithi070 ceph-mon[168282]: failed to encode map e442 with expected crc 2023-09-23T21:11:08.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:11:07 smithi070 ceph-mon[168282]: failed to encode map e442 with expected crc 2023-09-23T21:11:08.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:11:07 smithi070 ceph-mon[168282]: osdmap e442: 8 total, 8 up, 7 in 2023-09-23T21:11:08.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:11:07 smithi070 ceph-mon[168282]: failed to encode map e442 with expected crc 2023-09-23T21:11:08.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:11:07 smithi070 ceph-mon[168282]: pgmap v1141: 119 pgs: 119 active+clean; 3.1 GiB data, 12 GiB used, 613 GiB / 626 GiB avail; 41 MiB/s wr, 648 op/s; 32 KiB/s, 0 objects/s recovering 2023-09-23T21:11:08.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:11:07 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num_actual", "val": "18"}]: dispatch 2023-09-23T21:11:08.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:11:07 smithi070 ceph-mon[168282]: from='client.36243 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 1000, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T21:11:08.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:11:07 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'4':'36950'}"}]: dispatch 2023-09-23T21:11:08.131 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:11:07 smithi026 ceph-mon[179688]: failed to encode map e442 with expected crc 2023-09-23T21:11:08.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:11:07 smithi026 ceph-mon[179688]: failed to encode map e442 with expected crc 2023-09-23T21:11:08.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:11:07 smithi026 ceph-mon[179688]: failed to encode map e442 with expected crc 2023-09-23T21:11:08.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:11:07 smithi026 ceph-mon[179688]: osdmap e442: 8 total, 8 up, 7 in 2023-09-23T21:11:08.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:11:07 smithi026 ceph-mon[179688]: failed to encode map e442 with expected crc 2023-09-23T21:11:08.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:11:07 smithi026 ceph-mon[179688]: pgmap v1141: 119 pgs: 119 active+clean; 3.1 GiB data, 12 GiB used, 613 GiB / 626 GiB avail; 41 MiB/s wr, 648 op/s; 32 KiB/s, 0 objects/s recovering 2023-09-23T21:11:08.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:11:07 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num_actual", "val": "18"}]: dispatch 2023-09-23T21:11:08.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:11:07 smithi026 ceph-mon[179688]: from='client.36243 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 1000, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T21:11:08.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:11:07 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'4':'36950'}"}]: dispatch 2023-09-23T21:11:08.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:11:07 smithi026 ceph-mon[175151]: failed to encode map e442 with expected crc 2023-09-23T21:11:08.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:11:07 smithi026 ceph-mon[175151]: failed to encode map e442 with expected crc 2023-09-23T21:11:08.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:11:07 smithi026 ceph-mon[175151]: failed to encode map e442 with expected crc 2023-09-23T21:11:08.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:11:07 smithi026 ceph-mon[175151]: osdmap e442: 8 total, 8 up, 7 in 2023-09-23T21:11:08.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:11:07 smithi026 ceph-mon[175151]: failed to encode map e442 with expected crc 2023-09-23T21:11:08.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:11:07 smithi026 ceph-mon[175151]: pgmap v1141: 119 pgs: 119 active+clean; 3.1 GiB data, 12 GiB used, 613 GiB / 626 GiB avail; 41 MiB/s wr, 648 op/s; 32 KiB/s, 0 objects/s recovering 2023-09-23T21:11:08.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:11:07 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num_actual", "val": "18"}]: dispatch 2023-09-23T21:11:08.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:11:07 smithi026 ceph-mon[175151]: from='client.36243 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 1000, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T21:11:08.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:11:07 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'4':'36950'}"}]: dispatch 2023-09-23T21:11:08.463 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 46 16 52930 52914 71.8358 2.03125 0.0020226 0.0137486 2023-09-23T21:11:09.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:11:08 smithi070 ceph-mon[168282]: failed to encode map e443 with expected crc 2023-09-23T21:11:09.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:11:08 smithi070 ceph-mon[168282]: failed to encode map e443 with expected crc 2023-09-23T21:11:09.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:11:08 smithi070 ceph-mon[168282]: failed to encode map e443 with expected crc 2023-09-23T21:11:09.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:11:08 smithi070 ceph-mon[168282]: failed to encode map e443 with expected crc 2023-09-23T21:11:09.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:11:08 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num_actual", "val": "18"}]': finished 2023-09-23T21:11:09.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:11:08 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'4':'36950'}"}]': finished 2023-09-23T21:11:09.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:11:08 smithi070 ceph-mon[168282]: osdmap e443: 8 total, 8 up, 7 in 2023-09-23T21:11:09.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:11:08 smithi070 ceph-mon[168282]: failed to encode map e443 with expected crc 2023-09-23T21:11:09.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:11:08 smithi070 ceph-mon[168282]: failed to encode map e443 with expected crc 2023-09-23T21:11:09.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:11:08 smithi070 ceph-mon[168282]: failed to encode map e443 with expected crc 2023-09-23T21:11:09.121 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:11:08 smithi070 ceph-mon[168282]: failed to encode map e443 with expected crc 2023-09-23T21:11:09.121 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:11:08 smithi070 ceph-mon[168282]: 10.9 scrub starts 2023-09-23T21:11:09.121 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:11:08 smithi070 ceph-mon[168282]: 10.9 scrub ok 2023-09-23T21:11:09.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:11:08 smithi026 ceph-mon[175151]: failed to encode map e443 with expected crc 2023-09-23T21:11:09.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:11:08 smithi026 ceph-mon[175151]: failed to encode map e443 with expected crc 2023-09-23T21:11:09.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:11:08 smithi026 ceph-mon[175151]: failed to encode map e443 with expected crc 2023-09-23T21:11:09.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:11:08 smithi026 ceph-mon[175151]: failed to encode map e443 with expected crc 2023-09-23T21:11:09.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:11:08 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num_actual", "val": "18"}]': finished 2023-09-23T21:11:09.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:11:08 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'4':'36950'}"}]': finished 2023-09-23T21:11:09.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:11:08 smithi026 ceph-mon[175151]: osdmap e443: 8 total, 8 up, 7 in 2023-09-23T21:11:09.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:11:08 smithi026 ceph-mon[175151]: failed to encode map e443 with expected crc 2023-09-23T21:11:09.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:11:08 smithi026 ceph-mon[175151]: failed to encode map e443 with expected crc 2023-09-23T21:11:09.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:11:08 smithi026 ceph-mon[175151]: failed to encode map e443 with expected crc 2023-09-23T21:11:09.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:11:08 smithi026 ceph-mon[175151]: failed to encode map e443 with expected crc 2023-09-23T21:11:09.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:11:08 smithi026 ceph-mon[175151]: 10.9 scrub starts 2023-09-23T21:11:09.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:11:08 smithi026 ceph-mon[175151]: 10.9 scrub ok 2023-09-23T21:11:09.134 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:11:08 smithi026 ceph-mon[179688]: failed to encode map e443 with expected crc 2023-09-23T21:11:09.134 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:11:08 smithi026 ceph-mon[179688]: failed to encode map e443 with expected crc 2023-09-23T21:11:09.134 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:11:08 smithi026 ceph-mon[179688]: failed to encode map e443 with expected crc 2023-09-23T21:11:09.134 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:11:08 smithi026 ceph-mon[179688]: failed to encode map e443 with expected crc 2023-09-23T21:11:09.134 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:11:08 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num_actual", "val": "18"}]': finished 2023-09-23T21:11:09.134 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:11:08 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'4':'36950'}"}]': finished 2023-09-23T21:11:09.134 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:11:08 smithi026 ceph-mon[179688]: osdmap e443: 8 total, 8 up, 7 in 2023-09-23T21:11:09.134 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:11:08 smithi026 ceph-mon[179688]: failed to encode map e443 with expected crc 2023-09-23T21:11:09.134 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:11:08 smithi026 ceph-mon[179688]: failed to encode map e443 with expected crc 2023-09-23T21:11:09.135 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:11:08 smithi026 ceph-mon[179688]: failed to encode map e443 with expected crc 2023-09-23T21:11:09.135 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:11:08 smithi026 ceph-mon[179688]: failed to encode map e443 with expected crc 2023-09-23T21:11:09.135 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:11:08 smithi026 ceph-mon[179688]: 10.9 scrub starts 2023-09-23T21:11:09.135 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:11:08 smithi026 ceph-mon[179688]: 10.9 scrub ok 2023-09-23T21:11:09.463 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 47 16 52930 52914 70.3085 0 - 0.0137486 2023-09-23T21:11:09.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:11:09 smithi026 ceph-mon[175151]: 10.17 scrub starts 2023-09-23T21:11:09.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:11:09 smithi026 ceph-mon[175151]: failed to encode map e444 with expected crc 2023-09-23T21:11:09.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:11:09 smithi026 ceph-mon[175151]: failed to encode map e444 with expected crc 2023-09-23T21:11:09.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:11:09 smithi026 ceph-mon[175151]: failed to encode map e444 with expected crc 2023-09-23T21:11:09.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:11:09 smithi026 ceph-mon[175151]: 10.13 scrub starts 2023-09-23T21:11:09.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:11:09 smithi026 ceph-mon[175151]: osdmap e444: 8 total, 8 up, 7 in 2023-09-23T21:11:09.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:11:09 smithi026 ceph-mon[175151]: failed to encode map e444 with expected crc 2023-09-23T21:11:09.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:11:09 smithi026 ceph-mon[175151]: failed to encode map e444 with expected crc 2023-09-23T21:11:09.883 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:11:09 smithi026 ceph-mon[175151]: failed to encode map e444 with expected crc 2023-09-23T21:11:09.883 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:11:09 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T21:11:09.883 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:11:09 smithi026 ceph-mon[175151]: pgmap v1144: 119 pgs: 2 peering, 1 remapped+peering, 116 active+clean; 3.2 GiB data, 13 GiB used, 613 GiB / 626 GiB avail 2023-09-23T21:11:09.883 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:11:09 smithi026 ceph-mon[179688]: 10.17 scrub starts 2023-09-23T21:11:09.883 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:11:09 smithi026 ceph-mon[179688]: failed to encode map e444 with expected crc 2023-09-23T21:11:09.883 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:11:09 smithi026 ceph-mon[179688]: failed to encode map e444 with expected crc 2023-09-23T21:11:09.883 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:11:09 smithi026 ceph-mon[179688]: failed to encode map e444 with expected crc 2023-09-23T21:11:09.883 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:11:09 smithi026 ceph-mon[179688]: 10.13 scrub starts 2023-09-23T21:11:09.883 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:11:09 smithi026 ceph-mon[179688]: osdmap e444: 8 total, 8 up, 7 in 2023-09-23T21:11:09.884 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:11:09 smithi026 ceph-mon[179688]: failed to encode map e444 with expected crc 2023-09-23T21:11:09.884 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:11:09 smithi026 ceph-mon[179688]: failed to encode map e444 with expected crc 2023-09-23T21:11:09.884 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:11:09 smithi026 ceph-mon[179688]: failed to encode map e444 with expected crc 2023-09-23T21:11:09.884 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:11:09 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T21:11:09.884 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:11:09 smithi026 ceph-mon[179688]: pgmap v1144: 119 pgs: 2 peering, 1 remapped+peering, 116 active+clean; 3.2 GiB data, 13 GiB used, 613 GiB / 626 GiB avail 2023-09-23T21:11:09.884 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 21:11:09 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[164029]: ::ffff:172.21.15.70 - - [23/Sep/2023:21:11:09] "GET /metrics HTTP/1.1" 200 36378 "" "Prometheus/2.43.0" 2023-09-23T21:11:10.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:11:09 smithi070 ceph-mon[168282]: 10.17 scrub starts 2023-09-23T21:11:10.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:11:09 smithi070 ceph-mon[168282]: failed to encode map e444 with expected crc 2023-09-23T21:11:10.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:11:09 smithi070 ceph-mon[168282]: failed to encode map e444 with expected crc 2023-09-23T21:11:10.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:11:09 smithi070 ceph-mon[168282]: failed to encode map e444 with expected crc 2023-09-23T21:11:10.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:11:09 smithi070 ceph-mon[168282]: 10.13 scrub starts 2023-09-23T21:11:10.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:11:09 smithi070 ceph-mon[168282]: osdmap e444: 8 total, 8 up, 7 in 2023-09-23T21:11:10.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:11:09 smithi070 ceph-mon[168282]: failed to encode map e444 with expected crc 2023-09-23T21:11:10.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:11:09 smithi070 ceph-mon[168282]: failed to encode map e444 with expected crc 2023-09-23T21:11:10.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:11:09 smithi070 ceph-mon[168282]: failed to encode map e444 with expected crc 2023-09-23T21:11:10.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:11:09 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T21:11:10.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:11:09 smithi070 ceph-mon[168282]: pgmap v1144: 119 pgs: 2 peering, 1 remapped+peering, 116 active+clean; 3.2 GiB data, 13 GiB used, 613 GiB / 626 GiB avail 2023-09-23T21:11:10.412 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:11:10.417 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:11:10.417 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:11:10.467 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 48 16 53672 53656 69.8048 23.1875 0.00708781 0.0143195 2023-09-23T21:11:10.480 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_historic_ops 2023-09-23T21:11:10.519 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:11:10.623 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:11:10.706 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:11:10.709 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:11:10.710 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:11:10.767 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:11:10 smithi026 ceph-mon[179688]: 10.17 scrub starts 2023-09-23T21:11:10.767 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:11:10 smithi026 ceph-mon[179688]: failed to encode map e445 with expected crc 2023-09-23T21:11:10.767 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:11:10 smithi026 ceph-mon[179688]: failed to encode map e445 with expected crc 2023-09-23T21:11:10.767 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:11:10 smithi026 ceph-mon[179688]: failed to encode map e445 with expected crc 2023-09-23T21:11:10.767 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:11:10 smithi026 ceph-mon[179688]: osdmap e445: 8 total, 8 up, 7 in 2023-09-23T21:11:10.767 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:11:10 smithi026 ceph-mon[179688]: failed to encode map e445 with expected crc 2023-09-23T21:11:10.767 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:11:10 smithi026 ceph-mon[179688]: osdmap e446: 8 total, 8 up, 7 in 2023-09-23T21:11:10.770 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:11:10 smithi026 ceph-mon[175151]: 10.17 scrub starts 2023-09-23T21:11:10.770 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:11:10 smithi026 ceph-mon[175151]: failed to encode map e445 with expected crc 2023-09-23T21:11:10.770 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:11:10 smithi026 ceph-mon[175151]: failed to encode map e445 with expected crc 2023-09-23T21:11:10.770 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:11:10 smithi026 ceph-mon[175151]: failed to encode map e445 with expected crc 2023-09-23T21:11:10.770 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:11:10 smithi026 ceph-mon[175151]: osdmap e445: 8 total, 8 up, 7 in 2023-09-23T21:11:10.770 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:11:10 smithi026 ceph-mon[175151]: failed to encode map e445 with expected crc 2023-09-23T21:11:10.770 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:11:10 smithi026 ceph-mon[175151]: osdmap e446: 8 total, 8 up, 7 in 2023-09-23T21:11:10.810 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:11:10.868 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:11:10 smithi070 ceph-mon[168282]: 10.17 scrub starts 2023-09-23T21:11:10.868 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:11:10 smithi070 ceph-mon[168282]: failed to encode map e445 with expected crc 2023-09-23T21:11:10.868 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:11:10 smithi070 ceph-mon[168282]: failed to encode map e445 with expected crc 2023-09-23T21:11:10.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:11:10 smithi070 ceph-mon[168282]: failed to encode map e445 with expected crc 2023-09-23T21:11:10.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:11:10 smithi070 ceph-mon[168282]: osdmap e445: 8 total, 8 up, 7 in 2023-09-23T21:11:10.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:11:10 smithi070 ceph-mon[168282]: failed to encode map e445 with expected crc 2023-09-23T21:11:10.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:11:10 smithi070 ceph-mon[168282]: osdmap e446: 8 total, 8 up, 7 in 2023-09-23T21:11:10.870 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:11:10.918 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:11:10.921 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:11:10.922 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:11:11.023 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:11:11.109 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:11:11.179 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T21:11:11.184 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:11:11.185 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:11:11.286 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:11:11.319 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:11:11.375 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:11:11.379 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:11:11.379 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:11:11.468 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 49 16 54772 54756 69.7819 68.75 0.00889959 0.0143246 2023-09-23T21:11:11.481 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:11:11.527 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:11:11.579 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:11:11.584 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:11:11.585 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:11:11.686 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:11:11.757 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:11:11.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:11:11 smithi070 ceph-mon[168282]: failed to encode map e446 with expected crc 2023-09-23T21:11:11.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:11:11 smithi070 ceph-mon[168282]: failed to encode map e446 with expected crc 2023-09-23T21:11:11.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:11:11 smithi070 ceph-mon[168282]: pgmap v1147: 119 pgs: 2 peering, 2 remapped+peering, 115 active+clean; 3.2 GiB data, 13 GiB used, 613 GiB / 626 GiB avail 2023-09-23T21:11:11.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:11:11 smithi070 ceph-mon[168282]: osdmap e447: 8 total, 8 up, 7 in 2023-09-23T21:11:12.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:11:11 smithi026 ceph-mon[179688]: failed to encode map e446 with expected crc 2023-09-23T21:11:12.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:11:11 smithi026 ceph-mon[179688]: failed to encode map e446 with expected crc 2023-09-23T21:11:12.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:11:11 smithi026 ceph-mon[179688]: pgmap v1147: 119 pgs: 2 peering, 2 remapped+peering, 115 active+clean; 3.2 GiB data, 13 GiB used, 613 GiB / 626 GiB avail 2023-09-23T21:11:12.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:11:11 smithi026 ceph-mon[179688]: osdmap e447: 8 total, 8 up, 7 in 2023-09-23T21:11:12.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:11:11 smithi026 ceph-mon[175151]: failed to encode map e446 with expected crc 2023-09-23T21:11:12.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:11:11 smithi026 ceph-mon[175151]: failed to encode map e446 with expected crc 2023-09-23T21:11:12.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:11:11 smithi026 ceph-mon[175151]: pgmap v1147: 119 pgs: 2 peering, 2 remapped+peering, 115 active+clean; 3.2 GiB data, 13 GiB used, 613 GiB / 626 GiB avail 2023-09-23T21:11:12.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:11:11 smithi026 ceph-mon[175151]: osdmap e447: 8 total, 8 up, 7 in 2023-09-23T21:11:12.468 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 50 16 55898 55882 69.7936 70.375 0.0123377 0.0143225 2023-09-23T21:11:12.610 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:11:12.617 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:11:12.618 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:11:12.656 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '0.05', 'overage': '110', 'type': 'test-reweight-by-utilization'} 2023-09-23T21:11:12.656 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph osd test-reweight-by-utilization 110 0.05 2023-09-23T21:11:12.706 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0: {} 2023-09-23T21:11:12.718 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:11:12.757 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1: {} 2023-09-23T21:11:12.801 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2: {} 2023-09-23T21:11:12.827 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:11:12.848 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3: {} 2023-09-23T21:11:12.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:11:12 smithi026 ceph-mon[175151]: failed to encode map e447 with expected crc 2023-09-23T21:11:12.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:11:12 smithi026 ceph-mon[175151]: failed to encode map e447 with expected crc 2023-09-23T21:11:12.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:11:12 smithi026 ceph-mon[175151]: failed to encode map e447 with expected crc 2023-09-23T21:11:12.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:11:12 smithi026 ceph-mon[175151]: failed to encode map e447 with expected crc 2023-09-23T21:11:12.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:11:12 smithi026 ceph-mon[175151]: failed to encode map e447 with expected crc 2023-09-23T21:11:12.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:11:12 smithi026 ceph-mon[175151]: 10.12 scrub starts 2023-09-23T21:11:12.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:11:12 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:11:12.883 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:11:12 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/13228642' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:11:12.883 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:11:12 smithi026 ceph-mon[179688]: failed to encode map e447 with expected crc 2023-09-23T21:11:12.883 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:11:12 smithi026 ceph-mon[179688]: failed to encode map e447 with expected crc 2023-09-23T21:11:12.883 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:11:12 smithi026 ceph-mon[179688]: failed to encode map e447 with expected crc 2023-09-23T21:11:12.883 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:11:12 smithi026 ceph-mon[179688]: failed to encode map e447 with expected crc 2023-09-23T21:11:12.883 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:11:12 smithi026 ceph-mon[179688]: failed to encode map e447 with expected crc 2023-09-23T21:11:12.883 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:11:12 smithi026 ceph-mon[179688]: 10.12 scrub starts 2023-09-23T21:11:12.883 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:11:12 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:11:12.883 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:11:12 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/13228642' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:11:12.899 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4: {} 2023-09-23T21:11:12.965 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T21:11:12.972 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:11:12.983 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:11:12.983 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:11:13.001 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6: {} 2023-09-23T21:11:13.031 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T21:11:13.052 INFO:teuthology.orchestra.run.smithi026.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T21:11:13.053 INFO:teuthology.orchestra.run.smithi026.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T21:11:13.053 INFO:teuthology.orchestra.run.smithi026.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T21:11:13.053 INFO:teuthology.orchestra.run.smithi026.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T21:11:13.053 INFO:teuthology.orchestra.run.smithi026.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T21:11:13.053 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T21:11:13.053 INFO:teuthology.orchestra.run.smithi026.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T21:11:13.053 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T21:11:13.084 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:11:13.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:11:12 smithi070 ceph-mon[168282]: failed to encode map e447 with expected crc 2023-09-23T21:11:13.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:11:12 smithi070 ceph-mon[168282]: failed to encode map e447 with expected crc 2023-09-23T21:11:13.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:11:12 smithi070 ceph-mon[168282]: failed to encode map e447 with expected crc 2023-09-23T21:11:13.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:11:12 smithi070 ceph-mon[168282]: failed to encode map e447 with expected crc 2023-09-23T21:11:13.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:11:12 smithi070 ceph-mon[168282]: failed to encode map e447 with expected crc 2023-09-23T21:11:13.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:11:12 smithi070 ceph-mon[168282]: 10.12 scrub starts 2023-09-23T21:11:13.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:11:12 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:11:13.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:11:12 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/13228642' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:11:13.123 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:11:13.211 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:11:13.214 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:11:13.215 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:11:13.240 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:11:13.315 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:11:13.360 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:11:13.455 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:11:13.460 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:11:13.461 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:11:13.471 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 51 16 56889 56873 69.6365 61.9375 0.0115143 0.0143545 2023-09-23T21:11:13.562 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:11:13.610 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:11:13.652 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:11:13.657 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:11:13.658 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:11:13.758 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:11:13.792 INFO:teuthology.orchestra.run.smithi026.stderr:nodeep-scrub is unset 2023-09-23T21:11:13.872 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:11:14.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:11:13 smithi070 ceph-mon[168282]: pgmap v1149: 119 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 3 peering, 1 remapped+peering, 108 active+clean; 3.3 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 11693/164160 objects degraded (7.123%); 796/164160 objects misplaced (0.485%); 8.9 MiB/s, 143 objects/s recovering 2023-09-23T21:11:14.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:11:13 smithi070 ceph-mon[168282]: 10.12 scrub starts 2023-09-23T21:11:14.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:11:13 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/3528876295' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:11:14.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:11:13 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:11:14.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:11:13 smithi026 ceph-mon[175151]: pgmap v1149: 119 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 3 peering, 1 remapped+peering, 108 active+clean; 3.3 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 11693/164160 objects degraded (7.123%); 796/164160 objects misplaced (0.485%); 8.9 MiB/s, 143 objects/s recovering 2023-09-23T21:11:14.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:11:13 smithi026 ceph-mon[175151]: 10.12 scrub starts 2023-09-23T21:11:14.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:11:13 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/3528876295' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:11:14.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:11:13 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:11:14.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:11:13 smithi026 ceph-mon[179688]: pgmap v1149: 119 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 3 peering, 1 remapped+peering, 108 active+clean; 3.3 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 11693/164160 objects degraded (7.123%); 796/164160 objects misplaced (0.485%); 8.9 MiB/s, 143 objects/s recovering 2023-09-23T21:11:14.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:11:13 smithi026 ceph-mon[179688]: 10.12 scrub starts 2023-09-23T21:11:14.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:11:13 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/3528876295' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:11:14.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:11:13 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:11:14.310 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:11:14.319 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:11:14.320 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:11:14.421 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:11:14.471 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 52 16 57837 57821 69.4363 59.25 0.00711523 0.0143957 2023-09-23T21:11:14.486 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:11:15.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:11:14 smithi070 ceph-mon[168282]: Health check failed: Degraded data redundancy: 11693/164160 objects degraded (7.123%), 6 pgs degraded (PG_DEGRADED) 2023-09-23T21:11:15.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:11:14 smithi070 ceph-mon[168282]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T21:11:15.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:11:14 smithi070 ceph-mon[168282]: Cluster is now healthy 2023-09-23T21:11:15.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:11:14 smithi070 ceph-mon[168282]: failed to encode map e448 with expected crc 2023-09-23T21:11:15.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:11:14 smithi070 ceph-mon[168282]: failed to encode map e448 with expected crc 2023-09-23T21:11:15.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:11:14 smithi070 ceph-mon[168282]: failed to encode map e448 with expected crc 2023-09-23T21:11:15.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:11:14 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T21:11:15.121 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:11:14 smithi070 ceph-mon[168282]: osdmap e448: 8 total, 8 up, 7 in 2023-09-23T21:11:15.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:11:14 smithi026 ceph-mon[175151]: Health check failed: Degraded data redundancy: 11693/164160 objects degraded (7.123%), 6 pgs degraded (PG_DEGRADED) 2023-09-23T21:11:15.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:11:14 smithi026 ceph-mon[175151]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T21:11:15.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:11:14 smithi026 ceph-mon[175151]: Cluster is now healthy 2023-09-23T21:11:15.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:11:14 smithi026 ceph-mon[175151]: failed to encode map e448 with expected crc 2023-09-23T21:11:15.134 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:11:14 smithi026 ceph-mon[175151]: failed to encode map e448 with expected crc 2023-09-23T21:11:15.134 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:11:14 smithi026 ceph-mon[175151]: failed to encode map e448 with expected crc 2023-09-23T21:11:15.134 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:11:14 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T21:11:15.134 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:11:14 smithi026 ceph-mon[175151]: osdmap e448: 8 total, 8 up, 7 in 2023-09-23T21:11:15.134 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:11:14 smithi026 ceph-mon[179688]: Health check failed: Degraded data redundancy: 11693/164160 objects degraded (7.123%), 6 pgs degraded (PG_DEGRADED) 2023-09-23T21:11:15.134 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:11:14 smithi026 ceph-mon[179688]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T21:11:15.135 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:11:14 smithi026 ceph-mon[179688]: Cluster is now healthy 2023-09-23T21:11:15.135 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:11:14 smithi026 ceph-mon[179688]: failed to encode map e448 with expected crc 2023-09-23T21:11:15.135 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:11:14 smithi026 ceph-mon[179688]: failed to encode map e448 with expected crc 2023-09-23T21:11:15.135 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:11:14 smithi026 ceph-mon[179688]: failed to encode map e448 with expected crc 2023-09-23T21:11:15.135 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:11:14 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T21:11:15.135 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:11:14 smithi026 ceph-mon[179688]: osdmap e448: 8 total, 8 up, 7 in 2023-09-23T21:11:15.323 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:11:15.327 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:11:15.328 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:11:15.428 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:11:15.466 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:11:15.471 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 53 16 59130 59114 69.6507 80.8125 0.0108529 0.0143527 2023-09-23T21:11:15.523 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:11:15.530 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:11:15.530 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:11:15.631 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:11:15.704 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:11:15.776 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:11:15.789 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:11:15.789 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:11:15.808 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:11:15.890 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:11:15.941 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:11:16.007 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:11:16.014 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:11:16.015 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:11:16.115 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:11:16.144 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:11:16.190 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:11:16.193 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:11:16.193 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:11:16.293 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:11:16.305 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T21:11:16.419 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:11:16.471 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 54 16 60252 60236 69.6593 70.125 0.0179573 0.0143487 2023-09-23T21:11:16.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:11:16 smithi070 ceph-mon[168282]: pgmap v1151: 119 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 1 peering, 110 active+clean; 3.4 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 16 MiB/s wr, 259 op/s; 15020/167880 objects degraded (8.947%); 796/167880 objects misplaced (0.474%); 7.2 MiB/s, 115 objects/s recovering 2023-09-23T21:11:16.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:11:16 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:11:16.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:11:16 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:11:16.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:11:16 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:11:16.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:11:16 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:11:16.645 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:11:16 smithi026 ceph-mon[179688]: pgmap v1151: 119 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 1 peering, 110 active+clean; 3.4 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 16 MiB/s wr, 259 op/s; 15020/167880 objects degraded (8.947%); 796/167880 objects misplaced (0.474%); 7.2 MiB/s, 115 objects/s recovering 2023-09-23T21:11:16.645 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:11:16 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:11:16.646 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:11:16 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:11:16.646 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:11:16 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:11:16.646 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:11:16 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:11:16.646 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:11:16 smithi026 ceph-mon[175151]: pgmap v1151: 119 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 1 peering, 110 active+clean; 3.4 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 16 MiB/s wr, 259 op/s; 15020/167880 objects degraded (8.947%); 796/167880 objects misplaced (0.474%); 7.2 MiB/s, 115 objects/s recovering 2023-09-23T21:11:16.646 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:11:16 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:11:16.646 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:11:16 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:11:16.646 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:11:16 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:11:16.646 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:11:16 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:11:16.711 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:11:16.723 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:11:16.724 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:11:16.825 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:11:16.860 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:11:16.905 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:11:16.909 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:11:16.909 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:11:17.010 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:11:17.086 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:11:17.360 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:11:17.365 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:11:17.366 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:11:17.467 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:11:17.471 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 55 16 61386 61370 69.6813 70.875 0.0125812 0.0143457 2023-09-23T21:11:17.501 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:11:17.542 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:11:17.550 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:11:17.550 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:11:17.651 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:11:17.722 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T21:11:17.736 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:11:17.814 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:11:17.825 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:11:17.825 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:11:17.874 INFO:teuthology.orchestra.run.smithi026.stdout:ERROR: (22) Invalid argument 2023-09-23T21:11:17.874 INFO:teuthology.orchestra.run.smithi026.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T21:11:17.926 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:11:18.085 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:11:18.299 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:11:18.313 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:11:18.313 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:11:18.414 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:11:18.473 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 56 16 62364 62348 69.5257 61.125 0.0254704 0.0143775 2023-09-23T21:11:18.501 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:11:18.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:11:18 smithi026 ceph-mon[175151]: pgmap v1152: 119 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 111 active+clean; 3.5 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 27 MiB/s wr, 438 op/s; 15020/172068 objects degraded (8.729%); 796/172068 objects misplaced (0.463%); 6.0 MiB/s, 96 objects/s recovering 2023-09-23T21:11:18.633 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:11:18 smithi026 ceph-mon[175151]: 10.12 scrub starts 2023-09-23T21:11:18.633 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:11:18 smithi026 ceph-mon[179688]: pgmap v1152: 119 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 111 active+clean; 3.5 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 27 MiB/s wr, 438 op/s; 15020/172068 objects degraded (8.729%); 796/172068 objects misplaced (0.463%); 6.0 MiB/s, 96 objects/s recovering 2023-09-23T21:11:18.633 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:11:18 smithi026 ceph-mon[179688]: 10.12 scrub starts 2023-09-23T21:11:18.646 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:11:18.656 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:11:18.656 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:11:18.755 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:11:18.757 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:11:18.782 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:11:18 smithi070 ceph-mon[168282]: pgmap v1152: 119 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 111 active+clean; 3.5 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 27 MiB/s wr, 438 op/s; 15020/172068 objects degraded (8.729%); 796/172068 objects misplaced (0.463%); 6.0 MiB/s, 96 objects/s recovering 2023-09-23T21:11:18.782 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:11:18 smithi070 ceph-mon[168282]: 10.12 scrub starts 2023-09-23T21:11:18.785 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:11:18.792 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_ops_in_flight 2023-09-23T21:11:18.825 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:11:18.827 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:11:18.828 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:11:18.929 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:11:18.984 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:11:19.159 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:11:19.175 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:11:19.178 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:11:19.178 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:11:19.279 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:11:19.306 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:11:19.340 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:11:19.342 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:11:19.343 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:11:19.422 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:11:19 smithi026 ceph-mon[179688]: failed to encode map e449 with expected crc 2023-09-23T21:11:19.437 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:11:19 smithi026 ceph-mon[179688]: osdmap e449: 8 total, 8 up, 7 in 2023-09-23T21:11:19.437 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:11:19 smithi026 ceph-mon[179688]: failed to encode map e449 with expected crc 2023-09-23T21:11:19.437 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:11:19 smithi026 ceph-mon[179688]: failed to encode map e449 with expected crc 2023-09-23T21:11:19.437 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:11:19 smithi026 ceph-mon[179688]: failed to encode map e449 with expected crc 2023-09-23T21:11:19.437 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:11:19 smithi026 ceph-mon[179688]: 10.1 scrub starts 2023-09-23T21:11:19.437 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:11:19 smithi026 ceph-mon[179688]: 10.12 scrub starts 2023-09-23T21:11:19.437 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:11:19 smithi026 ceph-mon[175151]: failed to encode map e449 with expected crc 2023-09-23T21:11:19.438 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:11:19 smithi026 ceph-mon[175151]: osdmap e449: 8 total, 8 up, 7 in 2023-09-23T21:11:19.438 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:11:19 smithi026 ceph-mon[175151]: failed to encode map e449 with expected crc 2023-09-23T21:11:19.438 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:11:19 smithi026 ceph-mon[175151]: failed to encode map e449 with expected crc 2023-09-23T21:11:19.438 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:11:19 smithi026 ceph-mon[175151]: failed to encode map e449 with expected crc 2023-09-23T21:11:19.438 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:11:19 smithi026 ceph-mon[175151]: 10.1 scrub starts 2023-09-23T21:11:19.438 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:11:19 smithi026 ceph-mon[175151]: 10.12 scrub starts 2023-09-23T21:11:19.443 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:11:19.474 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 57 16 62800 62784 68.7845 27.25 0.0106126 0.0145159 2023-09-23T21:11:19.475 INFO:teuthology.orchestra.run.smithi026.stdout:moved 11 / 357 (3.08123%) 2023-09-23T21:11:19.475 INFO:teuthology.orchestra.run.smithi026.stdout:avg 51 2023-09-23T21:11:19.475 INFO:teuthology.orchestra.run.smithi026.stdout:stddev 8.73417 -> 6.59004 (expected baseline 6.61168) 2023-09-23T21:11:19.475 INFO:teuthology.orchestra.run.smithi026.stdout:min osd.4 with 35 -> 44 pgs (0.686275 -> 0.862745 * mean) 2023-09-23T21:11:19.475 INFO:teuthology.orchestra.run.smithi026.stdout:max osd.0 with 65 -> 62 pgs (1.27451 -> 1.21569 * mean) 2023-09-23T21:11:19.476 INFO:teuthology.orchestra.run.smithi026.stdout: 2023-09-23T21:11:19.476 INFO:teuthology.orchestra.run.smithi026.stdout:oload 110 2023-09-23T21:11:19.476 INFO:teuthology.orchestra.run.smithi026.stdout:max_change 0.05 2023-09-23T21:11:19.476 INFO:teuthology.orchestra.run.smithi026.stdout:max_change_osds 4 2023-09-23T21:11:19.476 INFO:teuthology.orchestra.run.smithi026.stdout:average_utilization 0.0235 2023-09-23T21:11:19.476 INFO:teuthology.orchestra.run.smithi026.stdout:overload_utilization 0.0258 2023-09-23T21:11:19.476 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0 weight 1.0000 -> 0.9500 2023-09-23T21:11:19.476 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4 weight 0.5638 -> 0.6138 2023-09-23T21:11:19.476 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2 weight 1.0000 -> 0.9500 2023-09-23T21:11:19.476 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3 weight 1.0000 -> 0.9500 2023-09-23T21:11:19.501 INFO:teuthology.orchestra.run.smithi026.stderr:no change 2023-09-23T21:11:19.547 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:11:19.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:11:19 smithi070 ceph-mon[168282]: failed to encode map e449 with expected crc 2023-09-23T21:11:19.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:11:19 smithi070 ceph-mon[168282]: osdmap e449: 8 total, 8 up, 7 in 2023-09-23T21:11:19.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:11:19 smithi070 ceph-mon[168282]: failed to encode map e449 with expected crc 2023-09-23T21:11:19.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:11:19 smithi070 ceph-mon[168282]: failed to encode map e449 with expected crc 2023-09-23T21:11:19.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:11:19 smithi070 ceph-mon[168282]: failed to encode map e449 with expected crc 2023-09-23T21:11:19.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:11:19 smithi070 ceph-mon[168282]: 10.1 scrub starts 2023-09-23T21:11:19.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:11:19 smithi070 ceph-mon[168282]: 10.12 scrub starts 2023-09-23T21:11:19.827 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 21:11:19 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[164029]: 2023-09-23T21:11:19.468+0000 7fd59efc8700 -1 mgr.server reply reply (4) Interrupted system call no change 2023-09-23T21:11:19.827 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 21:11:19 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[164029]: ::ffff:172.21.15.70 - - [23/Sep/2023:21:11:19] "GET /metrics HTTP/1.1" 200 36400 "" "Prometheus/2.43.0" 2023-09-23T21:11:20.063 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T21:11:20.073 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:11:20.074 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:11:20.174 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:11:20.204 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:11:20.247 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:11:20.250 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:11:20.251 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:11:20.260 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '1.0', 'overage': '110', 'type': 'reweight-by-utilization'} 2023-09-23T21:11:20.260 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph osd reweight-by-utilization 110 1.0 2023-09-23T21:11:20.281 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:11:20.351 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:11:20.442 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:11:20 smithi026 ceph-mon[175151]: pgmap v1154: 119 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 111 active+clean; 3.7 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 64 MiB/s wr, 1.02k op/s; 15020/182223 objects degraded (8.243%); 66/182223 objects misplaced (0.036%); 13 MiB/s, 216 objects/s recovering 2023-09-23T21:11:20.442 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:11:20 smithi026 ceph-mon[175151]: Health check update: Degraded data redundancy: 15020/182223 objects degraded (8.243%), 7 pgs degraded (PG_DEGRADED) 2023-09-23T21:11:20.442 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:11:20 smithi026 ceph-mon[175151]: failed to encode map e450 with expected crc 2023-09-23T21:11:20.442 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:11:20 smithi026 ceph-mon[175151]: failed to encode map e450 with expected crc 2023-09-23T21:11:20.442 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:11:20 smithi026 ceph-mon[175151]: osdmap e450: 8 total, 8 up, 7 in 2023-09-23T21:11:20.442 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:11:20 smithi026 ceph-mon[175151]: failed to encode map e450 with expected crc 2023-09-23T21:11:20.442 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:11:20 smithi026 ceph-mon[175151]: failed to encode map e450 with expected crc 2023-09-23T21:11:20.442 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:11:20 smithi026 ceph-mon[175151]: from='client.46397 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 110, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T21:11:20.443 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:11:20 smithi026 ceph-mon[179688]: pgmap v1154: 119 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 111 active+clean; 3.7 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 64 MiB/s wr, 1.02k op/s; 15020/182223 objects degraded (8.243%); 66/182223 objects misplaced (0.036%); 13 MiB/s, 216 objects/s recovering 2023-09-23T21:11:20.443 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:11:20 smithi026 ceph-mon[179688]: Health check update: Degraded data redundancy: 15020/182223 objects degraded (8.243%), 7 pgs degraded (PG_DEGRADED) 2023-09-23T21:11:20.443 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:11:20 smithi026 ceph-mon[179688]: failed to encode map e450 with expected crc 2023-09-23T21:11:20.469 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:11:20.475 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 58 16 63838 63822 68.7152 64.875 0.0249301 0.014546 2023-09-23T21:11:20.607 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T21:11:20.615 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:11:20.616 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:11:20.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:11:20 smithi070 ceph-mon[168282]: pgmap v1154: 119 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 111 active+clean; 3.7 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 64 MiB/s wr, 1.02k op/s; 15020/182223 objects degraded (8.243%); 66/182223 objects misplaced (0.036%); 13 MiB/s, 216 objects/s recovering 2023-09-23T21:11:20.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:11:20 smithi070 ceph-mon[168282]: Health check update: Degraded data redundancy: 15020/182223 objects degraded (8.243%), 7 pgs degraded (PG_DEGRADED) 2023-09-23T21:11:20.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:11:20 smithi070 ceph-mon[168282]: failed to encode map e450 with expected crc 2023-09-23T21:11:20.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:11:20 smithi070 ceph-mon[168282]: failed to encode map e450 with expected crc 2023-09-23T21:11:20.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:11:20 smithi070 ceph-mon[168282]: osdmap e450: 8 total, 8 up, 7 in 2023-09-23T21:11:20.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:11:20 smithi070 ceph-mon[168282]: failed to encode map e450 with expected crc 2023-09-23T21:11:20.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:11:20 smithi070 ceph-mon[168282]: failed to encode map e450 with expected crc 2023-09-23T21:11:20.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:11:20 smithi070 ceph-mon[168282]: from='client.46397 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 110, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T21:11:20.716 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:11:20.747 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:11:20.766 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:11:20 smithi026 ceph-mon[179688]: failed to encode map e450 with expected crc 2023-09-23T21:11:20.767 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:11:20 smithi026 ceph-mon[179688]: osdmap e450: 8 total, 8 up, 7 in 2023-09-23T21:11:20.767 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:11:20 smithi026 ceph-mon[179688]: failed to encode map e450 with expected crc 2023-09-23T21:11:20.767 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:11:20 smithi026 ceph-mon[179688]: failed to encode map e450 with expected crc 2023-09-23T21:11:20.767 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:11:20 smithi026 ceph-mon[179688]: from='client.46397 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 110, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T21:11:20.787 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:11:20.792 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:11:20.792 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:11:20.893 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:11:20.999 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:11:21.103 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:11:21.107 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:11:21.108 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:11:21.189 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0: {} 2023-09-23T21:11:21.209 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:11:21.245 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1: {} 2023-09-23T21:11:21.279 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2: {} 2023-09-23T21:11:21.312 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:11:21.321 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3: {} 2023-09-23T21:11:21.360 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4: {} 2023-09-23T21:11:21.393 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:11:21.400 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T21:11:21.402 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:11:21.402 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:11:21.440 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6: {} 2023-09-23T21:11:21.476 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 59 16 64740 64724 68.5047 56.375 0.0291973 0.0145922 2023-09-23T21:11:21.478 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T21:11:21.485 INFO:teuthology.orchestra.run.smithi026.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T21:11:21.486 INFO:teuthology.orchestra.run.smithi026.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T21:11:21.486 INFO:teuthology.orchestra.run.smithi026.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T21:11:21.486 INFO:teuthology.orchestra.run.smithi026.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T21:11:21.486 INFO:teuthology.orchestra.run.smithi026.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T21:11:21.486 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T21:11:21.486 INFO:teuthology.orchestra.run.smithi026.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T21:11:21.486 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T21:11:21.503 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:11:21.536 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:11:21.578 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:11:21.583 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:11:21.583 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:11:21.684 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:11:21.716 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:11:21.727 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:11:21 smithi026 ceph-mon[175151]: pgmap v1156: 119 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 111 active+clean; 3.7 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 53 MiB/s wr, 844 op/s; 15020/184194 objects degraded (8.154%); 66/184194 objects misplaced (0.036%); 6.3 MiB/s, 101 objects/s recovering 2023-09-23T21:11:21.727 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:11:21 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/1183120367' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:11:21.727 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:11:21 smithi026 ceph-mon[179688]: pgmap v1156: 119 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 111 active+clean; 3.7 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 53 MiB/s wr, 844 op/s; 15020/184194 objects degraded (8.154%); 66/184194 objects misplaced (0.036%); 6.3 MiB/s, 101 objects/s recovering 2023-09-23T21:11:21.727 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:11:21 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/1183120367' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:11:21.759 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:11:21.762 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:11:21.763 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:11:21.863 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:11:21.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:11:21 smithi070 ceph-mon[168282]: pgmap v1156: 119 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 111 active+clean; 3.7 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 53 MiB/s wr, 844 op/s; 15020/184194 objects degraded (8.154%); 66/184194 objects misplaced (0.036%); 6.3 MiB/s, 101 objects/s recovering 2023-09-23T21:11:21.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:11:21 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/1183120367' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:11:21.896 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:11:21.939 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:11:21.941 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:11:21.942 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:11:22.042 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:11:22.149 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:11:22.477 INFO:tasks.radosbench.radosbench.0.smithi026.stdout:2023-09-23T21:11:22.477620+0000 min lat: 0.00120238 max lat: 2.42702 avg lat: 0.0146204 2023-09-23T21:11:22.477 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T21:11:22.477 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 60 16 65709 65693 68.3724 60.5625 0.00985115 0.0146204 2023-09-23T21:11:22.671 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:11:22.678 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:11:22.678 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:11:22.772 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:11:22.779 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:11:22.858 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:11:22.966 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:11:22.979 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:11:22.980 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:11:23.080 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:11:23.163 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:11:23.199 INFO:teuthology.orchestra.run.smithi026.stderr:noscrub is set 2023-09-23T21:11:23.237 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:11:23 smithi026 ceph-mon[179688]: 10.12 scrub starts 2023-09-23T21:11:23.237 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:11:23 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:11:23.237 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:11:23 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/1773122161' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T21:11:23.237 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:11:23 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T21:11:23.237 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:11:23 smithi026 ceph-mon[175151]: 10.12 scrub starts 2023-09-23T21:11:23.237 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:11:23 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:11:23.237 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:11:23 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/1773122161' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T21:11:23.237 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:11:23 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T21:11:23.318 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:11:23.324 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:11:23.324 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:11:23.425 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:11:23.478 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 61 16 66688 66672 68.2546 61.1875 0.010643 0.0146455 2023-09-23T21:11:23.556 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:11:23.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:11:23 smithi070 ceph-mon[168282]: 10.12 scrub starts 2023-09-23T21:11:23.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:11:23 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:11:23.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:11:23 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/1773122161' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T21:11:23.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:11:23 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T21:11:23.747 INFO:teuthology.orchestra.run.smithi026.stdout:{ 2023-09-23T21:11:23.747 INFO:teuthology.orchestra.run.smithi026.stdout: "ops": [ 2023-09-23T21:11:23.747 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:23.748 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_repop(client.46328.0:66883 10.b e451/429 10:dac1ba01:::benchmark_data_smithi026_379838_object66882:head v 451'4206, mlcod=451'4203)", 2023-09-23T21:11:23.748 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T21:11:23.735494+0000", 2023-09-23T21:11:23.748 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 0.011398574, 2023-09-23T21:11:23.748 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.011428278, 2023-09-23T21:11:23.748 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:11:23.748 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "started", 2023-09-23T21:11:23.748 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:11:23.748 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:23.748 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:11:23.748 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:11:23.735494+0000", 2023-09-23T21:11:23.749 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:11:23.749 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:23.749 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:23.749 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:11:23.749 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:11:23.735494+0000", 2023-09-23T21:11:23.749 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:11:23.749 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:23.749 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:23.749 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:11:23.749 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:11:23.735493+0000", 2023-09-23T21:11:23.750 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T21:11:23.750 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:23.750 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:23.750 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:11:23.750 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:11:23.735515+0000", 2023-09-23T21:11:23.750 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.1718000000000001e-05 2023-09-23T21:11:23.750 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:23.750 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:23.750 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:11:23.750 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:11:23.735525+0000", 2023-09-23T21:11:23.751 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.5060000000000001e-06 2023-09-23T21:11:23.751 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:23.751 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:23.751 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:11:23.751 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:11:23.735531+0000", 2023-09-23T21:11:23.751 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.0870000000000003e-06 2023-09-23T21:11:23.751 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:23.751 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:23.751 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:11:23.751 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:11:23.742270+0000", 2023-09-23T21:11:23.752 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.006739213 2023-09-23T21:11:23.752 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:23.752 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:23.752 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:11:23.752 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:11:23.742296+0000", 2023-09-23T21:11:23.752 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.5541000000000002e-05 2023-09-23T21:11:23.752 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:11:23.752 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:11:23.752 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:11:23.752 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:23.752 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:23.753 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_repop(client.46328.0:66887 10.b e451/429 10:dc40aa42:::benchmark_data_smithi026_379838_object66886:head v 451'4207, mlcod=451'4203)", 2023-09-23T21:11:23.753 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T21:11:23.744630+0000", 2023-09-23T21:11:23.753 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 0.002262338, 2023-09-23T21:11:23.753 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.002359505, 2023-09-23T21:11:23.753 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:11:23.753 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "started", 2023-09-23T21:11:23.753 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:11:23.753 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:23.753 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:11:23.753 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:11:23.744630+0000", 2023-09-23T21:11:23.754 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:11:23.754 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:23.754 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:23.754 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:11:23.754 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:11:23.744630+0000", 2023-09-23T21:11:23.754 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:11:23.754 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:23.754 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:23.754 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:11:23.754 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:11:23.744630+0000", 2023-09-23T21:11:23.755 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T21:11:23.755 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:23.755 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:23.755 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:11:23.755 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:11:23.744658+0000", 2023-09-23T21:11:23.755 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.7889999999999999e-05 2023-09-23T21:11:23.755 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:23.755 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:23.755 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:11:23.755 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:11:23.744668+0000", 2023-09-23T21:11:23.756 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0288999999999999e-05 2023-09-23T21:11:23.756 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:23.756 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:23.756 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:11:23.756 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:11:23.744675+0000", 2023-09-23T21:11:23.756 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.8000000000000001e-06 2023-09-23T21:11:23.756 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:23.756 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:23.756 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:11:23.756 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:11:23.744872+0000", 2023-09-23T21:11:23.756 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00019683 2023-09-23T21:11:23.757 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:23.757 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:23.757 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:11:23.757 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:11:23.744891+0000", 2023-09-23T21:11:23.757 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.9726999999999999e-05 2023-09-23T21:11:23.757 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:11:23.757 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:11:23.757 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:11:23.757 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:23.757 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:23.758 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_repop(client.46328.0:66895 10.b e451/429)", 2023-09-23T21:11:23.758 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T21:11:23.744684+0000", 2023-09-23T21:11:23.758 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 0.0022090489999999998, 2023-09-23T21:11:23.758 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0023622959999999998, 2023-09-23T21:11:23.758 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:11:23.758 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "queued for pg", 2023-09-23T21:11:23.758 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:11:23.758 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:23.758 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:11:23.758 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:11:23.744684+0000", 2023-09-23T21:11:23.759 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:11:23.759 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:23.759 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:23.759 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:11:23.759 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:11:23.744684+0000", 2023-09-23T21:11:23.759 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:11:23.759 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:23.759 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:23.759 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:11:23.759 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:11:23.744683+0000", 2023-09-23T21:11:23.759 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T21:11:23.760 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:23.760 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:23.760 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:11:23.760 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:11:23.744707+0000", 2023-09-23T21:11:23.760 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.3286e-05 2023-09-23T21:11:23.760 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:23.760 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:23.760 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:11:23.760 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:11:23.744713+0000", 2023-09-23T21:11:23.760 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.0220000000000001e-06 2023-09-23T21:11:23.760 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:23.761 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:23.761 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:11:23.761 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:11:23.744717+0000", 2023-09-23T21:11:23.761 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.6580000000000001e-06 2023-09-23T21:11:23.761 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:11:23.761 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:11:23.761 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:11:23.761 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:23.761 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:23.761 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_repop_reply(client.46328.0:66892 10.8 e451/438)", 2023-09-23T21:11:23.762 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T21:11:23.744758+0000", 2023-09-23T21:11:23.762 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 0.0021345700000000001, 2023-09-23T21:11:23.762 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0023328569999999998, 2023-09-23T21:11:23.762 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:11:23.762 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "queued for pg", 2023-09-23T21:11:23.762 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:11:23.762 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:23.762 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:11:23.762 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:11:23.744758+0000", 2023-09-23T21:11:23.762 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:11:23.762 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:23.763 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:23.763 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:11:23.763 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:11:23.744758+0000", 2023-09-23T21:11:23.763 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:11:23.763 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:23.763 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:23.763 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:11:23.763 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:11:23.744758+0000", 2023-09-23T21:11:23.763 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T21:11:23.763 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:23.763 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:23.764 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:11:23.764 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:11:23.744761+0000", 2023-09-23T21:11:23.764 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.5279999999999999e-06 2023-09-23T21:11:23.764 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:23.764 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:23.764 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:11:23.764 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:11:23.744770+0000", 2023-09-23T21:11:23.764 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.2509999999999994e-06 2023-09-23T21:11:23.764 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:23.764 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:23.765 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:11:23.765 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:11:23.744775+0000", 2023-09-23T21:11:23.765 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.2700000000000004e-06 2023-09-23T21:11:23.765 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:11:23.765 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:11:23.765 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:11:23.765 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:23.765 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:23.765 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_repop(client.46328.0:66901 10.b e451/429)", 2023-09-23T21:11:23.765 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T21:11:23.746485+0000", 2023-09-23T21:11:23.765 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 0.00040806700000000002, 2023-09-23T21:11:23.766 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00065058200000000003, 2023-09-23T21:11:23.766 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:11:23.766 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "queued for pg", 2023-09-23T21:11:23.766 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:11:23.766 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:23.766 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:11:23.766 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:11:23.746485+0000", 2023-09-23T21:11:23.766 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:11:23.766 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:23.766 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:23.767 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:11:23.767 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:11:23.746485+0000", 2023-09-23T21:11:23.767 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:11:23.767 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:23.767 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:23.767 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:11:23.767 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:11:23.746484+0000", 2023-09-23T21:11:23.767 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.999999 2023-09-23T21:11:23.767 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:23.767 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:23.767 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:11:23.768 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:11:23.746509+0000", 2023-09-23T21:11:23.768 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.514e-05 2023-09-23T21:11:23.768 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:23.768 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:23.768 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:11:23.768 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:11:23.746526+0000", 2023-09-23T21:11:23.768 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.7047000000000001e-05 2023-09-23T21:11:23.768 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:23.768 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:23.768 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:11:23.768 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:11:23.746534+0000", 2023-09-23T21:11:23.769 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.3410000000000003e-06 2023-09-23T21:11:23.769 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:11:23.769 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:11:23.769 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:11:23.769 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:23.769 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:23.769 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.46328.0:66892 10.8 10:16840661:::benchmark_data_smithi026_379838_object66891:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e451)", 2023-09-23T21:11:23.769 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T21:11:23.727628+0000", 2023-09-23T21:11:23.769 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 0.019264752, 2023-09-23T21:11:23.769 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.019564617999999999, 2023-09-23T21:11:23.770 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:11:23.770 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "waiting for sub ops", 2023-09-23T21:11:23.770 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T21:11:23.770 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.46328", 2023-09-23T21:11:23.770 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/3748861181", 2023-09-23T21:11:23.770 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 66892 2023-09-23T21:11:23.770 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:23.770 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:11:23.770 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:23.770 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:11:23.770 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:11:23.727628+0000", 2023-09-23T21:11:23.771 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:11:23.771 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:23.771 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:23.771 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:11:23.771 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:11:23.727628+0000", 2023-09-23T21:11:23.771 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:11:23.771 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:23.771 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:23.771 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:11:23.771 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:11:23.727627+0000", 2023-09-23T21:11:23.771 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T21:11:23.772 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:23.772 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:23.772 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:11:23.772 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:11:23.727652+0000", 2023-09-23T21:11:23.772 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.4919e-05 2023-09-23T21:11:23.772 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:23.772 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:23.772 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:11:23.772 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:11:23.727661+0000", 2023-09-23T21:11:23.772 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.5480000000000002e-06 2023-09-23T21:11:23.773 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:23.773 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:23.773 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:11:23.773 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:11:23.727667+0000", 2023-09-23T21:11:23.773 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.2480000000000001e-06 2023-09-23T21:11:23.773 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:23.773 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:23.773 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:11:23.773 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:11:23.729131+0000", 2023-09-23T21:11:23.773 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.001464262 2023-09-23T21:11:23.773 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:23.773 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:23.774 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:11:23.774 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:11:23.729263+0000", 2023-09-23T21:11:23.774 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000132056 2023-09-23T21:11:23.774 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:23.774 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:23.774 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 2,7", 2023-09-23T21:11:23.774 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:11:23.729381+0000", 2023-09-23T21:11:23.774 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000117254 2023-09-23T21:11:23.774 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:23.774 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:23.774 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:11:23.775 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:11:23.731304+0000", 2023-09-23T21:11:23.775 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.001923412 2023-09-23T21:11:23.775 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:23.775 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:23.775 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T21:11:23.775 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:11:23.742427+0000", 2023-09-23T21:11:23.775 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.011122728 2023-09-23T21:11:23.775 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:11:23.775 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:11:23.775 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:11:23.775 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:11:23.776 INFO:teuthology.orchestra.run.smithi026.stdout: ], 2023-09-23T21:11:23.776 INFO:teuthology.orchestra.run.smithi026.stdout: "num_ops": 6 2023-09-23T21:11:23.776 INFO:teuthology.orchestra.run.smithi026.stdout:} 2023-09-23T21:11:24.163 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T21:11:24.261 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:11:24.271 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:11:24.271 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:11:24.372 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:11:24.450 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:11:24 smithi026 ceph-mon[179688]: pgmap v1157: 119 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 3.9 GiB data, 15 GiB used, 610 GiB / 626 GiB avail; 67 MiB/s wr, 1.08k op/s; 11608/193806 objects degraded (5.989%); 3244/193806 objects misplaced (1.674%); 7.5 MiB/s, 120 objects/s recovering 2023-09-23T21:11:24.450 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:11:24 smithi026 ceph-mon[179688]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:11:24.450 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:11:24 smithi026 ceph-mon[179688]: failed to encode map e451 with expected crc 2023-09-23T21:11:24.450 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:11:24 smithi026 ceph-mon[179688]: failed to encode map e451 with expected crc 2023-09-23T21:11:24.450 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:11:24 smithi026 ceph-mon[179688]: failed to encode map e451 with expected crc 2023-09-23T21:11:24.450 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:11:24 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T21:11:24.450 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:11:24 smithi026 ceph-mon[179688]: osdmap e451: 8 total, 8 up, 7 in 2023-09-23T21:11:24.451 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:11:24 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:11:24.451 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:11:24 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:11:24.457 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:11:24 smithi026 ceph-mon[175151]: pgmap v1157: 119 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 3.9 GiB data, 15 GiB used, 610 GiB / 626 GiB avail; 67 MiB/s wr, 1.08k op/s; 11608/193806 objects degraded (5.989%); 3244/193806 objects misplaced (1.674%); 7.5 MiB/s, 120 objects/s recovering 2023-09-23T21:11:24.458 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:11:24 smithi026 ceph-mon[175151]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:11:24.458 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:11:24 smithi026 ceph-mon[175151]: failed to encode map e451 with expected crc 2023-09-23T21:11:24.458 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:11:24 smithi026 ceph-mon[175151]: failed to encode map e451 with expected crc 2023-09-23T21:11:24.458 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:11:24 smithi026 ceph-mon[175151]: failed to encode map e451 with expected crc 2023-09-23T21:11:24.458 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:11:24 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T21:11:24.458 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:11:24 smithi026 ceph-mon[175151]: osdmap e451: 8 total, 8 up, 7 in 2023-09-23T21:11:24.458 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:11:24 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:11:24.458 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:11:24 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:11:24.460 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:11:24.477 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 62 16 67621 67605 68.0942 58.3125 0.0119846 0.0146795 2023-09-23T21:11:24.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:11:24 smithi070 ceph-mon[168282]: pgmap v1157: 119 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 3.9 GiB data, 15 GiB used, 610 GiB / 626 GiB avail; 67 MiB/s wr, 1.08k op/s; 11608/193806 objects degraded (5.989%); 3244/193806 objects misplaced (1.674%); 7.5 MiB/s, 120 objects/s recovering 2023-09-23T21:11:24.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:11:24 smithi070 ceph-mon[168282]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:11:24.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:11:24 smithi070 ceph-mon[168282]: failed to encode map e451 with expected crc 2023-09-23T21:11:24.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:11:24 smithi070 ceph-mon[168282]: failed to encode map e451 with expected crc 2023-09-23T21:11:24.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:11:24 smithi070 ceph-mon[168282]: failed to encode map e451 with expected crc 2023-09-23T21:11:24.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:11:24 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T21:11:24.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:11:24 smithi070 ceph-mon[168282]: osdmap e451: 8 total, 8 up, 7 in 2023-09-23T21:11:24.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:11:24 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:11:24.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:11:24 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:11:24.629 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T21:11:24.642 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:11:24.643 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:11:24.679 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_blocked_ops 2023-09-23T21:11:24.743 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:11:24.790 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:11:24.831 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:11:24.834 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:11:24.834 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:11:24.935 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:11:25.083 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:11:25.173 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:11:25.182 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:11:25.182 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:11:25.283 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:11:25.310 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:11:25.358 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:11:25 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T21:11:25.359 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:11:25 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T21:11:25.359 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:11:25 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:11:25.359 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:11:25 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T21:11:25.359 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:11:25 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T21:11:25.359 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:11:25 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:11:25.369 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:11:25.477 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 63 16 68419 68403 67.8052 49.875 0.0171155 0.014743 2023-09-23T21:11:25.477 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T21:11:25.490 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:11:25.490 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:11:25.591 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:11:25.616 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:11:25 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T21:11:25.616 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:11:25 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T21:11:25.616 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:11:25 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:11:25.619 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:11:25.669 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:11:25.673 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:11:25.674 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:11:25.774 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:11:25.816 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:11:25.866 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:11:25.870 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:11:25.871 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:11:25.871 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:11:25.972 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:11:26.034 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:11:26.241 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:11:26.259 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:11:26.259 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:11:26.272 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T21:11:26.335 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:11:26 smithi026 ceph-mon[179688]: pgmap v1159: 119 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 4.0 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 56 MiB/s wr, 903 op/s; 11608/197520 objects degraded (5.877%); 3244/197520 objects misplaced (1.642%); 2.2 MiB/s, 35 objects/s recovering 2023-09-23T21:11:26.335 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:11:26 smithi026 ceph-mon[179688]: Health check update: Degraded data redundancy: 11608/197520 objects degraded (5.877%), 6 pgs degraded (PG_DEGRADED) 2023-09-23T21:11:26.335 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:11:26 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'0':'43533','2':'55721','3':'58832','4':'64326'}"}]: dispatch 2023-09-23T21:11:26.335 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:11:26 smithi026 ceph-mon[175151]: pgmap v1159: 119 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 4.0 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 56 MiB/s wr, 903 op/s; 11608/197520 objects degraded (5.877%); 3244/197520 objects misplaced (1.642%); 2.2 MiB/s, 35 objects/s recovering 2023-09-23T21:11:26.335 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:11:26 smithi026 ceph-mon[175151]: Health check update: Degraded data redundancy: 11608/197520 objects degraded (5.877%), 6 pgs degraded (PG_DEGRADED) 2023-09-23T21:11:26.336 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:11:26 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'0':'43533','2':'55721','3':'58832','4':'64326'}"}]: dispatch 2023-09-23T21:11:26.360 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:11:26.391 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:11:26.430 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:11:26.433 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:11:26.434 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:11:26.483 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 64 16 69456 69440 67.7536 64.8125 0.0267393 0.0147537 2023-09-23T21:11:26.534 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:11:26.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:11:26 smithi070 ceph-mon[168282]: pgmap v1159: 119 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 4.0 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 56 MiB/s wr, 903 op/s; 11608/197520 objects degraded (5.877%); 3244/197520 objects misplaced (1.642%); 2.2 MiB/s, 35 objects/s recovering 2023-09-23T21:11:26.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:11:26 smithi070 ceph-mon[168282]: Health check update: Degraded data redundancy: 11608/197520 objects degraded (5.877%), 6 pgs degraded (PG_DEGRADED) 2023-09-23T21:11:26.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:11:26 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'0':'43533','2':'55721','3':'58832','4':'64326'}"}]: dispatch 2023-09-23T21:11:26.659 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:11:26.728 INFO:teuthology.orchestra.run.smithi026.stdout:moved 54 / 357 (15.126%) 2023-09-23T21:11:26.729 INFO:teuthology.orchestra.run.smithi026.stdout:avg 51 2023-09-23T21:11:26.729 INFO:teuthology.orchestra.run.smithi026.stdout:stddev 8.73417 -> 9.27362 (expected baseline 6.61168) 2023-09-23T21:11:26.729 INFO:teuthology.orchestra.run.smithi026.stdout:min osd.4 with 35 -> 67 pgs (0.686275 -> 1.31373 * mean) 2023-09-23T21:11:26.729 INFO:teuthology.orchestra.run.smithi026.stdout:max osd.0 with 65 -> 40 pgs (1.27451 -> 0.784314 * mean) 2023-09-23T21:11:26.729 INFO:teuthology.orchestra.run.smithi026.stdout: 2023-09-23T21:11:26.729 INFO:teuthology.orchestra.run.smithi026.stdout:oload 110 2023-09-23T21:11:26.729 INFO:teuthology.orchestra.run.smithi026.stdout:max_change 1 2023-09-23T21:11:26.729 INFO:teuthology.orchestra.run.smithi026.stdout:max_change_osds 4 2023-09-23T21:11:26.729 INFO:teuthology.orchestra.run.smithi026.stdout:average_utilization 0.0249 2023-09-23T21:11:26.729 INFO:teuthology.orchestra.run.smithi026.stdout:overload_utilization 0.0274 2023-09-23T21:11:26.730 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0 weight 1.0000 -> 0.6643 2023-09-23T21:11:26.730 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4 weight 0.5638 -> 0.9815 2023-09-23T21:11:26.730 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2 weight 1.0000 -> 0.8502 2023-09-23T21:11:26.730 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3 weight 1.0000 -> 0.8977 2023-09-23T21:11:27.130 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0: {} 2023-09-23T21:11:27.156 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1: {} 2023-09-23T21:11:27.180 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2: {} 2023-09-23T21:11:27.181 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:11:27.185 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:11:27.186 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:11:27.200 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3: {} 2023-09-23T21:11:27.222 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4: {} 2023-09-23T21:11:27.241 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T21:11:27.260 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6: {} 2023-09-23T21:11:27.280 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T21:11:27.281 INFO:teuthology.orchestra.run.smithi026.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T21:11:27.282 INFO:teuthology.orchestra.run.smithi026.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T21:11:27.282 INFO:teuthology.orchestra.run.smithi026.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T21:11:27.282 INFO:teuthology.orchestra.run.smithi026.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T21:11:27.282 INFO:teuthology.orchestra.run.smithi026.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T21:11:27.282 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T21:11:27.282 INFO:teuthology.orchestra.run.smithi026.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T21:11:27.282 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T21:11:27.287 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:11:27.320 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:11:27.482 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 65 16 69671 69655 66.9186 13.4375 0.00393945 0.0147717 2023-09-23T21:11:27.561 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:11:27 smithi026 ceph-mon[175151]: from='client.46412 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 110, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T21:11:27.561 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:11:27 smithi026 ceph-mon[175151]: failed to encode map e452 with expected crc 2023-09-23T21:11:27.561 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:11:27 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'0':'43533','2':'55721','3':'58832','4':'64326'}"}]': finished 2023-09-23T21:11:27.561 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:11:27 smithi026 ceph-mon[175151]: osdmap e452: 8 total, 8 up, 7 in 2023-09-23T21:11:27.561 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:11:27 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/1957142598' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:11:27.562 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:11:27 smithi026 ceph-mon[179688]: from='client.46412 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 110, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T21:11:27.562 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:11:27 smithi026 ceph-mon[179688]: failed to encode map e452 with expected crc 2023-09-23T21:11:27.562 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:11:27 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'0':'43533','2':'55721','3':'58832','4':'64326'}"}]': finished 2023-09-23T21:11:27.562 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:11:27 smithi026 ceph-mon[179688]: osdmap e452: 8 total, 8 up, 7 in 2023-09-23T21:11:27.562 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:11:27 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/1957142598' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:11:27.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:11:27 smithi070 ceph-mon[168282]: from='client.46412 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 110, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T21:11:27.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:11:27 smithi070 ceph-mon[168282]: failed to encode map e452 with expected crc 2023-09-23T21:11:27.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:11:27 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'0':'43533','2':'55721','3':'58832','4':'64326'}"}]': finished 2023-09-23T21:11:27.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:11:27 smithi070 ceph-mon[168282]: osdmap e452: 8 total, 8 up, 7 in 2023-09-23T21:11:27.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:11:27 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/1957142598' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:11:27.812 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:11:27.816 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:11:27.817 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:11:27.833 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '0.05', 'overage': '110', 'type': 'test-reweight-by-utilization'} 2023-09-23T21:11:27.834 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph osd test-reweight-by-utilization 110 0.05 2023-09-23T21:11:27.918 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:11:27.948 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:11:28.009 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:11:28.014 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:11:28.014 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:11:28.116 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:11:28.142 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:11:28.175 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:11:28.177 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:11:28.178 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:11:28.278 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:11:28.304 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:11:28.310 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:11:28.330 INFO:teuthology.orchestra.run.smithi026.stdout:ERROR: (22) Invalid argument 2023-09-23T21:11:28.330 INFO:teuthology.orchestra.run.smithi026.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T21:11:28.348 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:11:28.351 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:11:28.352 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:11:28.358 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:11:28 smithi026 ceph-mon[175151]: failed to encode map e452 with expected crc 2023-09-23T21:11:28.358 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:11:28 smithi026 ceph-mon[175151]: failed to encode map e452 with expected crc 2023-09-23T21:11:28.358 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:11:28 smithi026 ceph-mon[175151]: pgmap v1161: 119 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 4.1 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 46 MiB/s wr, 741 op/s; 11608/201021 objects degraded (5.775%); 3025/201021 objects misplaced (1.505%); 3.7 MiB/s, 59 objects/s recovering 2023-09-23T21:11:28.359 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:11:28 smithi026 ceph-mon[175151]: failed to encode map e453 with expected crc 2023-09-23T21:11:28.359 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:11:28 smithi026 ceph-mon[175151]: osdmap e453: 8 total, 8 up, 7 in 2023-09-23T21:11:28.359 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:11:28 smithi026 ceph-mon[179688]: failed to encode map e452 with expected crc 2023-09-23T21:11:28.359 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:11:28 smithi026 ceph-mon[179688]: failed to encode map e452 with expected crc 2023-09-23T21:11:28.359 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:11:28 smithi026 ceph-mon[179688]: pgmap v1161: 119 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 4.1 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 46 MiB/s wr, 741 op/s; 11608/201021 objects degraded (5.775%); 3025/201021 objects misplaced (1.505%); 3.7 MiB/s, 59 objects/s recovering 2023-09-23T21:11:28.359 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:11:28 smithi026 ceph-mon[179688]: failed to encode map e453 with expected crc 2023-09-23T21:11:28.359 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:11:28 smithi026 ceph-mon[179688]: osdmap e453: 8 total, 8 up, 7 in 2023-09-23T21:11:28.453 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:11:28.476 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:11:28 smithi070 ceph-mon[168282]: failed to encode map e452 with expected crc 2023-09-23T21:11:28.476 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:11:28 smithi070 ceph-mon[168282]: failed to encode map e452 with expected crc 2023-09-23T21:11:28.476 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:11:28 smithi070 ceph-mon[168282]: pgmap v1161: 119 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 4.1 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 46 MiB/s wr, 741 op/s; 11608/201021 objects degraded (5.775%); 3025/201021 objects misplaced (1.505%); 3.7 MiB/s, 59 objects/s recovering 2023-09-23T21:11:28.476 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:11:28 smithi070 ceph-mon[168282]: failed to encode map e453 with expected crc 2023-09-23T21:11:28.476 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:11:28 smithi070 ceph-mon[168282]: osdmap e453: 8 total, 8 up, 7 in 2023-09-23T21:11:28.478 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:11:28.482 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 66 16 69671 69655 65.905 0 - 0.0147717 2023-09-23T21:11:28.511 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:11:28.513 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:11:28.514 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:11:28.614 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:11:28.647 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:11:28.793 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:11:28.804 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:11:28.804 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:11:28.905 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:11:28.966 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_historic_ops 2023-09-23T21:11:28.994 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:11:29.135 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:11:29.211 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:11:29 smithi026 ceph-mon[175151]: failed to encode map e453 with expected crc 2023-09-23T21:11:29.211 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:11:29 smithi026 ceph-mon[175151]: failed to encode map e453 with expected crc 2023-09-23T21:11:29.211 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:11:29 smithi026 ceph-mon[175151]: failed to encode map e453 with expected crc 2023-09-23T21:11:29.212 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:11:29 smithi026 ceph-mon[175151]: failed to encode map e453 with expected crc 2023-09-23T21:11:29.212 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:11:29 smithi026 ceph-mon[179688]: failed to encode map e453 with expected crc 2023-09-23T21:11:29.212 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:11:29 smithi026 ceph-mon[179688]: failed to encode map e453 with expected crc 2023-09-23T21:11:29.212 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:11:29 smithi026 ceph-mon[179688]: failed to encode map e453 with expected crc 2023-09-23T21:11:29.212 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:11:29 smithi026 ceph-mon[179688]: failed to encode map e453 with expected crc 2023-09-23T21:11:29.212 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:11:29 smithi026 ceph-mon[179688]: failed to encode map e453 with expected crc 2023-09-23T21:11:29.441 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:11:29.444 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:11:29.445 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:11:29.482 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 67 16 69790 69774 65.033 3.71875 0.00677299 0.0152107 2023-09-23T21:11:29.545 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:11:29.578 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:11:29 smithi026 ceph-mon[179688]: failed to encode map e453 with expected crc 2023-09-23T21:11:29.579 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:11:29 smithi026 ceph-mon[179688]: failed to encode map e453 with expected crc 2023-09-23T21:11:29.579 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:11:29 smithi026 ceph-mon[179688]: failed to encode map e453 with expected crc 2023-09-23T21:11:29.579 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:11:29 smithi026 ceph-mon[179688]: failed to encode map e453 with expected crc 2023-09-23T21:11:29.579 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:11:29 smithi026 ceph-mon[179688]: failed to encode map e453 with expected crc 2023-09-23T21:11:29.579 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:11:29 smithi026 ceph-mon[179688]: failed to encode map e454 with expected crc 2023-09-23T21:11:29.579 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:11:29 smithi026 ceph-mon[179688]: osdmap e454: 8 total, 8 up, 7 in 2023-09-23T21:11:29.579 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:11:29 smithi026 ceph-mon[175151]: failed to encode map e453 with expected crc 2023-09-23T21:11:29.580 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:11:29 smithi026 ceph-mon[175151]: failed to encode map e453 with expected crc 2023-09-23T21:11:29.580 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:11:29 smithi026 ceph-mon[175151]: failed to encode map e453 with expected crc 2023-09-23T21:11:29.580 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:11:29 smithi026 ceph-mon[175151]: failed to encode map e453 with expected crc 2023-09-23T21:11:29.580 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:11:29 smithi026 ceph-mon[175151]: failed to encode map e453 with expected crc 2023-09-23T21:11:29.580 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:11:29 smithi026 ceph-mon[175151]: failed to encode map e453 with expected crc 2023-09-23T21:11:29.580 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:11:29 smithi026 ceph-mon[175151]: failed to encode map e454 with expected crc 2023-09-23T21:11:29.580 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:11:29 smithi026 ceph-mon[175151]: osdmap e454: 8 total, 8 up, 7 in 2023-09-23T21:11:29.582 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:11:29.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:11:29 smithi070 ceph-mon[168282]: failed to encode map e453 with expected crc 2023-09-23T21:11:29.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:11:29 smithi070 ceph-mon[168282]: failed to encode map e453 with expected crc 2023-09-23T21:11:29.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:11:29 smithi070 ceph-mon[168282]: failed to encode map e453 with expected crc 2023-09-23T21:11:29.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:11:29 smithi070 ceph-mon[168282]: failed to encode map e453 with expected crc 2023-09-23T21:11:29.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:11:29 smithi070 ceph-mon[168282]: failed to encode map e453 with expected crc 2023-09-23T21:11:29.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:11:29 smithi070 ceph-mon[168282]: failed to encode map e453 with expected crc 2023-09-23T21:11:29.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:11:29 smithi070 ceph-mon[168282]: failed to encode map e453 with expected crc 2023-09-23T21:11:29.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:11:29 smithi070 ceph-mon[168282]: failed to encode map e453 with expected crc 2023-09-23T21:11:29.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:11:29 smithi070 ceph-mon[168282]: failed to encode map e453 with expected crc 2023-09-23T21:11:29.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:11:29 smithi070 ceph-mon[168282]: failed to encode map e453 with expected crc 2023-09-23T21:11:29.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:11:29 smithi070 ceph-mon[168282]: failed to encode map e454 with expected crc 2023-09-23T21:11:29.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:11:29 smithi070 ceph-mon[168282]: osdmap e454: 8 total, 8 up, 7 in 2023-09-23T21:11:29.679 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:11:29.681 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:11:29.682 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:11:29.783 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:11:29.813 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:11:29.881 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 21:11:29 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[164029]: ::ffff:172.21.15.70 - - [23/Sep/2023:21:11:29] "GET /metrics HTTP/1.1" 200 36400 "" "Prometheus/2.43.0" 2023-09-23T21:11:29.953 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:11:29.957 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:11:29.957 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:11:30.044 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T21:11:30.059 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:11:30.119 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:11:30.160 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:11:30.166 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:11:30.166 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:11:30.267 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:11:30.296 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:11:30.334 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:11:30.337 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:11:30.338 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:11:30.338 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:11:30.439 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:11:30.470 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:11:30 smithi026 ceph-mon[179688]: failed to encode map e454 with expected crc 2023-09-23T21:11:30.471 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:11:30 smithi026 ceph-mon[179688]: failed to encode map e454 with expected crc 2023-09-23T21:11:30.471 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:11:30 smithi026 ceph-mon[179688]: failed to encode map e454 with expected crc 2023-09-23T21:11:30.471 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:11:30 smithi026 ceph-mon[179688]: failed to encode map e454 with expected crc 2023-09-23T21:11:30.471 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:11:30 smithi026 ceph-mon[179688]: failed to encode map e454 with expected crc 2023-09-23T21:11:30.471 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:11:30 smithi026 ceph-mon[179688]: pgmap v1164: 119 pgs: 8 remapped+peering, 20 peering, 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 4.2 GiB data, 16 GiB used, 609 GiB / 626 GiB avail; 46 MiB/s wr, 725 op/s; 6701/206490 objects degraded (3.245%); 3025/206490 objects misplaced (1.465%); 2.4 MiB/s, 35 objects/s recovering 2023-09-23T21:11:30.471 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:11:30 smithi026 ceph-mon[179688]: Health check failed: Reduced data availability: 4 pgs inactive, 4 pgs peering (PG_AVAILABILITY) 2023-09-23T21:11:30.471 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:11:30 smithi026 ceph-mon[179688]: osdmap e455: 8 total, 8 up, 7 in 2023-09-23T21:11:30.471 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:11:30 smithi026 ceph-mon[175151]: failed to encode map e454 with expected crc 2023-09-23T21:11:30.471 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:11:30 smithi026 ceph-mon[175151]: failed to encode map e454 with expected crc 2023-09-23T21:11:30.471 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:11:30 smithi026 ceph-mon[175151]: failed to encode map e454 with expected crc 2023-09-23T21:11:30.472 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:11:30 smithi026 ceph-mon[175151]: failed to encode map e454 with expected crc 2023-09-23T21:11:30.472 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:11:30 smithi026 ceph-mon[175151]: failed to encode map e454 with expected crc 2023-09-23T21:11:30.472 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:11:30 smithi026 ceph-mon[175151]: pgmap v1164: 119 pgs: 8 remapped+peering, 20 peering, 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 4.2 GiB data, 16 GiB used, 609 GiB / 626 GiB avail; 46 MiB/s wr, 725 op/s; 6701/206490 objects degraded (3.245%); 3025/206490 objects misplaced (1.465%); 2.4 MiB/s, 35 objects/s recovering 2023-09-23T21:11:30.472 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:11:30 smithi026 ceph-mon[175151]: Health check failed: Reduced data availability: 4 pgs inactive, 4 pgs peering (PG_AVAILABILITY) 2023-09-23T21:11:30.472 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:11:30 smithi026 ceph-mon[175151]: osdmap e455: 8 total, 8 up, 7 in 2023-09-23T21:11:30.473 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:11:30.482 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 68 16 69790 69774 64.0773 0 - 0.0152107 2023-09-23T21:11:30.590 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:11:30.593 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:11:30.594 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:11:30.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:11:30 smithi070 ceph-mon[168282]: failed to encode map e454 with expected crc 2023-09-23T21:11:30.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:11:30 smithi070 ceph-mon[168282]: failed to encode map e454 with expected crc 2023-09-23T21:11:30.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:11:30 smithi070 ceph-mon[168282]: failed to encode map e454 with expected crc 2023-09-23T21:11:30.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:11:30 smithi070 ceph-mon[168282]: failed to encode map e454 with expected crc 2023-09-23T21:11:30.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:11:30 smithi070 ceph-mon[168282]: failed to encode map e454 with expected crc 2023-09-23T21:11:30.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:11:30 smithi070 ceph-mon[168282]: pgmap v1164: 119 pgs: 8 remapped+peering, 20 peering, 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 4.2 GiB data, 16 GiB used, 609 GiB / 626 GiB avail; 46 MiB/s wr, 725 op/s; 6701/206490 objects degraded (3.245%); 3025/206490 objects misplaced (1.465%); 2.4 MiB/s, 35 objects/s recovering 2023-09-23T21:11:30.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:11:30 smithi070 ceph-mon[168282]: Health check failed: Reduced data availability: 4 pgs inactive, 4 pgs peering (PG_AVAILABILITY) 2023-09-23T21:11:30.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:11:30 smithi070 ceph-mon[168282]: osdmap e455: 8 total, 8 up, 7 in 2023-09-23T21:11:30.694 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:11:30.721 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:11:30.748 INFO:teuthology.orchestra.run.smithi026.stderr:nodeep-scrub is set 2023-09-23T21:11:30.768 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:11:30.771 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:11:30.772 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:11:30.789 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:11:30.872 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:11:30.905 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:11:31.381 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:11:31 smithi026 ceph-mon[179688]: failed to encode map e455 with expected crc 2023-09-23T21:11:31.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:11:31 smithi026 ceph-mon[179688]: failed to encode map e455 with expected crc 2023-09-23T21:11:31.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:11:31 smithi026 ceph-mon[179688]: failed to encode map e455 with expected crc 2023-09-23T21:11:31.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:11:31 smithi026 ceph-mon[179688]: failed to encode map e455 with expected crc 2023-09-23T21:11:31.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:11:31 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/449733874' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:11:31.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:11:31 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:11:31.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:11:31 smithi026 ceph-mon[179688]: Health check update: Degraded data redundancy: 6701/206490 objects degraded (3.245%), 3 pgs degraded (PG_DEGRADED) 2023-09-23T21:11:31.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:11:31 smithi026 ceph-mon[179688]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:11:31.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:11:31 smithi026 ceph-mon[179688]: failed to encode map e456 with expected crc 2023-09-23T21:11:31.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:11:31 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T21:11:31.383 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:11:31 smithi026 ceph-mon[179688]: osdmap e456: 8 total, 8 up, 7 in 2023-09-23T21:11:31.383 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:11:31 smithi026 ceph-mon[175151]: failed to encode map e455 with expected crc 2023-09-23T21:11:31.383 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:11:31 smithi026 ceph-mon[175151]: failed to encode map e455 with expected crc 2023-09-23T21:11:31.383 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:11:31 smithi026 ceph-mon[175151]: failed to encode map e455 with expected crc 2023-09-23T21:11:31.383 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:11:31 smithi026 ceph-mon[175151]: failed to encode map e455 with expected crc 2023-09-23T21:11:31.383 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:11:31 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/449733874' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:11:31.383 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:11:31 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:11:31.383 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:11:31 smithi026 ceph-mon[175151]: Health check update: Degraded data redundancy: 6701/206490 objects degraded (3.245%), 3 pgs degraded (PG_DEGRADED) 2023-09-23T21:11:31.383 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:11:31 smithi026 ceph-mon[175151]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:11:31.383 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:11:31 smithi026 ceph-mon[175151]: failed to encode map e456 with expected crc 2023-09-23T21:11:31.384 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:11:31 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T21:11:31.384 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:11:31 smithi026 ceph-mon[175151]: osdmap e456: 8 total, 8 up, 7 in 2023-09-23T21:11:31.482 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 69 16 70062 70046 63.3955 8.5 0.0532271 0.0157526 2023-09-23T21:11:31.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:11:31 smithi070 ceph-mon[168282]: failed to encode map e455 with expected crc 2023-09-23T21:11:31.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:11:31 smithi070 ceph-mon[168282]: failed to encode map e455 with expected crc 2023-09-23T21:11:31.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:11:31 smithi070 ceph-mon[168282]: failed to encode map e455 with expected crc 2023-09-23T21:11:31.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:11:31 smithi070 ceph-mon[168282]: failed to encode map e455 with expected crc 2023-09-23T21:11:31.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:11:31 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/449733874' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:11:31.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:11:31 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:11:31.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:11:31 smithi070 ceph-mon[168282]: Health check update: Degraded data redundancy: 6701/206490 objects degraded (3.245%), 3 pgs degraded (PG_DEGRADED) 2023-09-23T21:11:31.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:11:31 smithi070 ceph-mon[168282]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:11:31.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:11:31 smithi070 ceph-mon[168282]: failed to encode map e456 with expected crc 2023-09-23T21:11:31.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:11:31 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T21:11:31.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:11:31 smithi070 ceph-mon[168282]: osdmap e456: 8 total, 8 up, 7 in 2023-09-23T21:11:31.707 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:11:31.713 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:11:31.713 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:11:31.814 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:11:31.847 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:11:31.887 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:11:31.891 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:11:31.891 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:11:31.992 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:11:32.086 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:11:32.156 INFO:teuthology.orchestra.run.smithi026.stdout:moved 4 / 357 (1.12045%) 2023-09-23T21:11:32.157 INFO:teuthology.orchestra.run.smithi026.stdout:avg 51 2023-09-23T21:11:32.157 INFO:teuthology.orchestra.run.smithi026.stdout:stddev 9.27362 -> 9.95705 (expected baseline 6.61168) 2023-09-23T21:11:32.157 INFO:teuthology.orchestra.run.smithi026.stdout:min osd.2 with 38 -> 39 pgs (0.745098 -> 0.764706 * mean) 2023-09-23T21:11:32.157 INFO:teuthology.orchestra.run.smithi026.stdout:max osd.4 with 67 -> 67 pgs (1.31373 -> 1.31373 * mean) 2023-09-23T21:11:32.157 INFO:teuthology.orchestra.run.smithi026.stdout: 2023-09-23T21:11:32.157 INFO:teuthology.orchestra.run.smithi026.stdout:oload 110 2023-09-23T21:11:32.157 INFO:teuthology.orchestra.run.smithi026.stdout:max_change 0.05 2023-09-23T21:11:32.157 INFO:teuthology.orchestra.run.smithi026.stdout:max_change_osds 4 2023-09-23T21:11:32.157 INFO:teuthology.orchestra.run.smithi026.stdout:average_utilization 0.0263 2023-09-23T21:11:32.157 INFO:teuthology.orchestra.run.smithi026.stdout:overload_utilization 0.0289 2023-09-23T21:11:32.157 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0 weight 0.6643 -> 0.6143 2023-09-23T21:11:32.157 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4 weight 0.9815 -> 1.0000 2023-09-23T21:11:32.158 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2 weight 0.8502 -> 0.8002 2023-09-23T21:11:32.159 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 21:11:32 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[164029]: 2023-09-23T21:11:32.154+0000 7fd59efc8700 -1 mgr.server reply reply (3) No such process no change 2023-09-23T21:11:32.163 INFO:teuthology.orchestra.run.smithi026.stderr:no change 2023-09-23T21:11:32.284 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T21:11:32.293 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:11:32.293 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:11:32.394 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:11:32.432 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:11:32.476 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:11:32.483 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 70 16 70719 70703 63.0767 41.0625 0.00689106 0.0158409 2023-09-23T21:11:32.483 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:11:32.483 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:11:32.510 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:11:32 smithi026 ceph-mon[175151]: failed to encode map e456 with expected crc 2023-09-23T21:11:32.510 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:11:32 smithi026 ceph-mon[175151]: failed to encode map e456 with expected crc 2023-09-23T21:11:32.510 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:11:32 smithi026 ceph-mon[175151]: failed to encode map e456 with expected crc 2023-09-23T21:11:32.510 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:11:32 smithi026 ceph-mon[175151]: failed to encode map e456 with expected crc 2023-09-23T21:11:32.510 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:11:32 smithi026 ceph-mon[175151]: failed to encode map e456 with expected crc 2023-09-23T21:11:32.510 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:11:32 smithi026 ceph-mon[175151]: pgmap v1167: 119 pgs: 1 activating+undersized+degraded+remapped, 8 remapped+peering, 20 peering, 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped, 83 active+clean; 4.2 GiB data, 16 GiB used, 609 GiB / 626 GiB avail; 33 MiB/s wr, 524 op/s; 11026/207372 objects degraded (5.317%); 3025/207372 objects misplaced (1.459%); 14 KiB/s, 0 objects/s recovering 2023-09-23T21:11:32.510 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:11:32 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:11:32.510 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:11:32 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:11:32.510 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:11:32 smithi026 ceph-mon[175151]: failed to encode map e457 with expected crc 2023-09-23T21:11:32.510 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:11:32 smithi026 ceph-mon[175151]: osdmap e457: 8 total, 8 up, 7 in 2023-09-23T21:11:32.511 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:11:32 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:11:32.511 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:11:32 smithi026 ceph-mon[179688]: failed to encode map e456 with expected crc 2023-09-23T21:11:32.511 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:11:32 smithi026 ceph-mon[179688]: failed to encode map e456 with expected crc 2023-09-23T21:11:32.511 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:11:32 smithi026 ceph-mon[179688]: failed to encode map e456 with expected crc 2023-09-23T21:11:32.511 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:11:32 smithi026 ceph-mon[179688]: failed to encode map e456 with expected crc 2023-09-23T21:11:32.511 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:11:32 smithi026 ceph-mon[179688]: failed to encode map e456 with expected crc 2023-09-23T21:11:32.511 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:11:32 smithi026 ceph-mon[179688]: pgmap v1167: 119 pgs: 1 activating+undersized+degraded+remapped, 8 remapped+peering, 20 peering, 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped, 83 active+clean; 4.2 GiB data, 16 GiB used, 609 GiB / 626 GiB avail; 33 MiB/s wr, 524 op/s; 11026/207372 objects degraded (5.317%); 3025/207372 objects misplaced (1.459%); 14 KiB/s, 0 objects/s recovering 2023-09-23T21:11:32.511 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:11:32 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:11:32.511 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:11:32 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:11:32.511 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:11:32 smithi026 ceph-mon[179688]: failed to encode map e457 with expected crc 2023-09-23T21:11:32.512 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:11:32 smithi026 ceph-mon[179688]: osdmap e457: 8 total, 8 up, 7 in 2023-09-23T21:11:32.512 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:11:32 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:11:32.584 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:11:32.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:11:32 smithi070 ceph-mon[168282]: failed to encode map e456 with expected crc 2023-09-23T21:11:32.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:11:32 smithi070 ceph-mon[168282]: failed to encode map e456 with expected crc 2023-09-23T21:11:32.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:11:32 smithi070 ceph-mon[168282]: failed to encode map e456 with expected crc 2023-09-23T21:11:32.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:11:32 smithi070 ceph-mon[168282]: failed to encode map e456 with expected crc 2023-09-23T21:11:32.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:11:32 smithi070 ceph-mon[168282]: failed to encode map e456 with expected crc 2023-09-23T21:11:32.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:11:32 smithi070 ceph-mon[168282]: pgmap v1167: 119 pgs: 1 activating+undersized+degraded+remapped, 8 remapped+peering, 20 peering, 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped, 83 active+clean; 4.2 GiB data, 16 GiB used, 609 GiB / 626 GiB avail; 33 MiB/s wr, 524 op/s; 11026/207372 objects degraded (5.317%); 3025/207372 objects misplaced (1.459%); 14 KiB/s, 0 objects/s recovering 2023-09-23T21:11:32.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:11:32 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:11:32.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:11:32 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:11:32.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:11:32 smithi070 ceph-mon[168282]: failed to encode map e457 with expected crc 2023-09-23T21:11:32.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:11:32 smithi070 ceph-mon[168282]: osdmap e457: 8 total, 8 up, 7 in 2023-09-23T21:11:32.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:11:32 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:11:32.625 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:11:32.671 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:11:32.677 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:11:32.678 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:11:32.778 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:11:32.824 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:11:33.253 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:11:33.257 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:11:33.257 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:11:33.335 INFO:teuthology.orchestra.run.smithi026.stdout:ERROR: (22) Invalid argument 2023-09-23T21:11:33.335 INFO:teuthology.orchestra.run.smithi026.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T21:11:33.358 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:11:33.402 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:11:33.455 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:11:33.461 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:11:33.462 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:11:33.482 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 71 16 71427 71411 62.8116 44.25 0.00440216 0.0159034 2023-09-23T21:11:33.563 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:11:33.617 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:11:33 smithi026 ceph-mon[179688]: failed to encode map e457 with expected crc 2023-09-23T21:11:33.617 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:11:33 smithi026 ceph-mon[179688]: failed to encode map e457 with expected crc 2023-09-23T21:11:33.618 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:11:33 smithi026 ceph-mon[179688]: failed to encode map e457 with expected crc 2023-09-23T21:11:33.618 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:11:33 smithi026 ceph-mon[179688]: from='client.36276 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 110, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T21:11:33.618 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:11:33 smithi026 ceph-mon[179688]: failed to encode map e458 with expected crc 2023-09-23T21:11:33.618 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:11:33 smithi026 ceph-mon[179688]: osdmap e458: 8 total, 8 up, 7 in 2023-09-23T21:11:33.618 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:11:33 smithi026 ceph-mon[175151]: failed to encode map e457 with expected crc 2023-09-23T21:11:33.618 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:11:33 smithi026 ceph-mon[175151]: failed to encode map e457 with expected crc 2023-09-23T21:11:33.618 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:11:33 smithi026 ceph-mon[175151]: failed to encode map e457 with expected crc 2023-09-23T21:11:33.618 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:11:33 smithi026 ceph-mon[175151]: from='client.36276 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 110, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T21:11:33.618 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:11:33 smithi026 ceph-mon[175151]: failed to encode map e458 with expected crc 2023-09-23T21:11:33.618 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:11:33 smithi026 ceph-mon[175151]: osdmap e458: 8 total, 8 up, 7 in 2023-09-23T21:11:33.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:11:33 smithi070 ceph-mon[168282]: failed to encode map e457 with expected crc 2023-09-23T21:11:33.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:11:33 smithi070 ceph-mon[168282]: failed to encode map e457 with expected crc 2023-09-23T21:11:33.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:11:33 smithi070 ceph-mon[168282]: failed to encode map e457 with expected crc 2023-09-23T21:11:33.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:11:33 smithi070 ceph-mon[168282]: from='client.36276 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 110, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T21:11:33.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:11:33 smithi070 ceph-mon[168282]: failed to encode map e458 with expected crc 2023-09-23T21:11:33.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:11:33 smithi070 ceph-mon[168282]: osdmap e458: 8 total, 8 up, 7 in 2023-09-23T21:11:33.625 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T21:11:33.635 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:11:33.688 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '0.05', 'overage': '110', 'type': 'reweight-by-utilization'} 2023-09-23T21:11:33.689 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph osd reweight-by-utilization 110 0.05 2023-09-23T21:11:33.982 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:11:33.994 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:11:33.994 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:11:34.069 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_ops_in_flight 2023-09-23T21:11:34.096 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:11:34.129 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:11:34.179 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:11:34.187 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:11:34.188 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:11:34.289 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:11:34.332 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:11:34.377 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:11:34.378 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:11:34 smithi070 ceph-mon[168282]: failed to encode map e458 with expected crc 2023-09-23T21:11:34.379 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:11:34 smithi070 ceph-mon[168282]: failed to encode map e458 with expected crc 2023-09-23T21:11:34.379 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:11:34 smithi070 ceph-mon[168282]: failed to encode map e458 with expected crc 2023-09-23T21:11:34.379 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:11:34 smithi070 ceph-mon[168282]: failed to encode map e458 with expected crc 2023-09-23T21:11:34.379 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:11:34 smithi070 ceph-mon[168282]: pgmap v1170: 119 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+remapped, 1 activating+undersized+degraded+remapped, 5 remapped+peering, 1 active+undersized+remapped, 11 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped, 97 active+clean; 4.3 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 10 KiB/s rd, 22 MiB/s wr, 369 op/s; 28996/210765 objects degraded (13.758%); 9330/210765 objects misplaced (4.427%); 18 MiB/s, 2 keys/s, 296 objects/s recovering 2023-09-23T21:11:34.379 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:11:34.380 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:11:34.381 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:11:34 smithi026 ceph-mon[179688]: failed to encode map e458 with expected crc 2023-09-23T21:11:34.381 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:11:34 smithi026 ceph-mon[179688]: failed to encode map e458 with expected crc 2023-09-23T21:11:34.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:11:34 smithi026 ceph-mon[179688]: failed to encode map e458 with expected crc 2023-09-23T21:11:34.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:11:34 smithi026 ceph-mon[179688]: failed to encode map e458 with expected crc 2023-09-23T21:11:34.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:11:34 smithi026 ceph-mon[179688]: pgmap v1170: 119 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+remapped, 1 activating+undersized+degraded+remapped, 5 remapped+peering, 1 active+undersized+remapped, 11 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped, 97 active+clean; 4.3 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 10 KiB/s rd, 22 MiB/s wr, 369 op/s; 28996/210765 objects degraded (13.758%); 9330/210765 objects misplaced (4.427%); 18 MiB/s, 2 keys/s, 296 objects/s recovering 2023-09-23T21:11:34.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:11:34 smithi026 ceph-mon[175151]: failed to encode map e458 with expected crc 2023-09-23T21:11:34.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:11:34 smithi026 ceph-mon[175151]: failed to encode map e458 with expected crc 2023-09-23T21:11:34.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:11:34 smithi026 ceph-mon[175151]: failed to encode map e458 with expected crc 2023-09-23T21:11:34.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:11:34 smithi026 ceph-mon[175151]: failed to encode map e458 with expected crc 2023-09-23T21:11:34.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:11:34 smithi026 ceph-mon[175151]: pgmap v1170: 119 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+remapped, 1 activating+undersized+degraded+remapped, 5 remapped+peering, 1 active+undersized+remapped, 11 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped, 97 active+clean; 4.3 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 10 KiB/s rd, 22 MiB/s wr, 369 op/s; 28996/210765 objects degraded (13.758%); 9330/210765 objects misplaced (4.427%); 18 MiB/s, 2 keys/s, 296 objects/s recovering 2023-09-23T21:11:34.481 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:11:34.483 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 72 16 72152 72136 62.5686 45.3125 0.0752299 0.0159748 2023-09-23T21:11:34.519 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:11:34.523 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0: {} 2023-09-23T21:11:34.568 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1: {} 2023-09-23T21:11:34.571 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:11:34.575 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:11:34.582 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:11:34.582 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:11:34.617 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2: {} 2023-09-23T21:11:34.661 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3: {} 2023-09-23T21:11:34.683 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:11:34.705 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4: {} 2023-09-23T21:11:34.738 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T21:11:34.748 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:11:34.761 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:11:34.786 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6: {} 2023-09-23T21:11:34.826 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T21:11:34.842 INFO:teuthology.orchestra.run.smithi026.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T21:11:34.842 INFO:teuthology.orchestra.run.smithi026.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T21:11:34.843 INFO:teuthology.orchestra.run.smithi026.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T21:11:34.843 INFO:teuthology.orchestra.run.smithi026.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T21:11:34.843 INFO:teuthology.orchestra.run.smithi026.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T21:11:34.843 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T21:11:34.843 INFO:teuthology.orchestra.run.smithi026.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T21:11:34.843 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T21:11:35.011 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:11:35.018 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:11:35.018 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:11:35.119 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:11:35.193 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:11:35.353 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:11:35 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/1241912250' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:11:35.353 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:11:35 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/1241912250' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:11:35.417 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:11:35.432 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:11:35.433 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:11:35.483 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 73 16 72910 72894 62.3605 47.375 0.00616977 0.016026 2023-09-23T21:11:35.533 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:11:35.572 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:11:35 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/1241912250' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:11:35.580 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:11:35.638 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:11:35.642 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:11:35.642 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:11:35.668 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:11:35.743 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:11:35.785 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:11:35.837 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:11:35.843 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:11:35.843 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:11:35.944 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:11:35.977 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:11:36.017 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:11:36.024 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:11:36.025 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:11:36.125 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:11:36.181 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:11:36.307 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:11:36 smithi026 ceph-mon[175151]: pgmap v1171: 119 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+remapped, 1 active+undersized+remapped, 14 active+recovery_wait+undersized+degraded+remapped, 101 active+clean; 4.3 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 7.8 KiB/s rd, 20 MiB/s wr, 326 op/s; 35542/212361 objects degraded (16.737%); 9330/212361 objects misplaced (4.393%); 14 MiB/s, 1 keys/s, 225 objects/s recovering 2023-09-23T21:11:36.308 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:11:36 smithi026 ceph-mon[175151]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2023-09-23T21:11:36.308 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:11:36 smithi026 ceph-mon[175151]: Health check update: Degraded data redundancy: 35542/212361 objects degraded (16.737%), 14 pgs degraded (PG_DEGRADED) 2023-09-23T21:11:36.308 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:11:36 smithi026 ceph-mon[179688]: pgmap v1171: 119 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+remapped, 1 active+undersized+remapped, 14 active+recovery_wait+undersized+degraded+remapped, 101 active+clean; 4.3 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 7.8 KiB/s rd, 20 MiB/s wr, 326 op/s; 35542/212361 objects degraded (16.737%); 9330/212361 objects misplaced (4.393%); 14 MiB/s, 1 keys/s, 225 objects/s recovering 2023-09-23T21:11:36.308 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:11:36 smithi026 ceph-mon[179688]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2023-09-23T21:11:36.308 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:11:36 smithi026 ceph-mon[179688]: Health check update: Degraded data redundancy: 35542/212361 objects degraded (16.737%), 14 pgs degraded (PG_DEGRADED) 2023-09-23T21:11:36.372 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T21:11:36.376 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:11:36.376 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:11:36.477 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:11:36.483 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 74 16 73613 73597 62.1115 43.9375 0.0122289 0.0160889 2023-09-23T21:11:36.557 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:11:36.622 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:11:36 smithi070 ceph-mon[168282]: pgmap v1171: 119 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+remapped, 1 active+undersized+remapped, 14 active+recovery_wait+undersized+degraded+remapped, 101 active+clean; 4.3 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 7.8 KiB/s rd, 20 MiB/s wr, 326 op/s; 35542/212361 objects degraded (16.737%); 9330/212361 objects misplaced (4.393%); 14 MiB/s, 1 keys/s, 225 objects/s recovering 2023-09-23T21:11:36.622 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:11:36 smithi070 ceph-mon[168282]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2023-09-23T21:11:36.622 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:11:36 smithi070 ceph-mon[168282]: Health check update: Degraded data redundancy: 35542/212361 objects degraded (16.737%), 14 pgs degraded (PG_DEGRADED) 2023-09-23T21:11:36.674 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:11:36.683 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:11:36.683 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:11:36.784 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:11:36.866 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:11:36.958 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T21:11:36.962 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:11:36.963 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:11:37.063 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:11:37.145 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:11:37.391 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:11:37.394 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:11:37.395 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:11:37.443 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T21:11:37.485 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 75 16 74255 74239 61.8185 40.125 0.0128283 0.0161112 2023-09-23T21:11:37.495 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:11:37.527 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:11:37.571 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:11:37.574 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:11:37.574 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:11:37.675 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:11:37.704 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:11:37.725 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:11:37 smithi026 ceph-mon[179688]: failed to encode map e459 with expected crc 2023-09-23T21:11:37.725 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:11:37 smithi026 ceph-mon[179688]: osdmap e459: 8 total, 8 up, 7 in 2023-09-23T21:11:37.726 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:11:37 smithi026 ceph-mon[175151]: failed to encode map e459 with expected crc 2023-09-23T21:11:37.726 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:11:37 smithi026 ceph-mon[175151]: osdmap e459: 8 total, 8 up, 7 in 2023-09-23T21:11:37.726 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:11:37 smithi026 ceph-mon[175151]: failed to encode map e459 with expected crc 2023-09-23T21:11:37.732 INFO:teuthology.orchestra.run.smithi026.stdout:moved 4 / 357 (1.12045%) 2023-09-23T21:11:37.733 INFO:teuthology.orchestra.run.smithi026.stdout:avg 51 2023-09-23T21:11:37.733 INFO:teuthology.orchestra.run.smithi026.stdout:stddev 9.27362 -> 9.95705 (expected baseline 6.61168) 2023-09-23T21:11:37.733 INFO:teuthology.orchestra.run.smithi026.stdout:min osd.2 with 38 -> 39 pgs (0.745098 -> 0.764706 * mean) 2023-09-23T21:11:37.733 INFO:teuthology.orchestra.run.smithi026.stdout:max osd.4 with 67 -> 67 pgs (1.31373 -> 1.31373 * mean) 2023-09-23T21:11:37.733 INFO:teuthology.orchestra.run.smithi026.stdout: 2023-09-23T21:11:37.733 INFO:teuthology.orchestra.run.smithi026.stdout:oload 110 2023-09-23T21:11:37.733 INFO:teuthology.orchestra.run.smithi026.stdout:max_change 0.05 2023-09-23T21:11:37.733 INFO:teuthology.orchestra.run.smithi026.stdout:max_change_osds 4 2023-09-23T21:11:37.733 INFO:teuthology.orchestra.run.smithi026.stdout:average_utilization 0.0269 2023-09-23T21:11:37.733 INFO:teuthology.orchestra.run.smithi026.stdout:overload_utilization 0.0296 2023-09-23T21:11:37.733 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0 weight 0.6643 -> 0.6143 2023-09-23T21:11:37.734 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4 weight 0.9815 -> 1.0000 2023-09-23T21:11:37.734 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2 weight 0.8502 -> 0.8002 2023-09-23T21:11:37.752 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:11:37.755 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:11:37.755 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:11:37.755 INFO:teuthology.orchestra.run.smithi026.stderr:noscrub is unset 2023-09-23T21:11:37.856 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:11:37.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:11:37 smithi070 ceph-mon[168282]: failed to encode map e459 with expected crc 2023-09-23T21:11:37.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:11:37 smithi070 ceph-mon[168282]: osdmap e459: 8 total, 8 up, 7 in 2023-09-23T21:11:37.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:11:37 smithi070 ceph-mon[168282]: failed to encode map e459 with expected crc 2023-09-23T21:11:37.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:11:37 smithi070 ceph-mon[168282]: failed to encode map e459 with expected crc 2023-09-23T21:11:37.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:11:37 smithi070 ceph-mon[168282]: failed to encode map e459 with expected crc 2023-09-23T21:11:37.871 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:11:37 smithi070 ceph-mon[168282]: pgmap v1173: 119 pgs: 1 active+recovery_wait+remapped, 3 active+recovering+undersized+remapped, 13 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 4.4 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 6.8 KiB/s rd, 29 MiB/s wr, 469 op/s; 32680/215670 objects degraded (15.153%); 12192/215670 objects misplaced (5.653%); 12 MiB/s, 1 keys/s, 197 objects/s recovering 2023-09-23T21:11:37.871 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:11:37 smithi070 ceph-mon[168282]: from='client.36279 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 110, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T21:11:37.871 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:11:37 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'0':'40257','2':'52445','4':'65536'}"}]: dispatch 2023-09-23T21:11:37.871 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:11:37 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/3815146473' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T21:11:37.886 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:11:37.930 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:11:37.935 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:11:37.936 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:11:38.036 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:11:38.066 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:11:38.108 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:11:38.112 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:11:38.113 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:11:38.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:11:37 smithi026 ceph-mon[179688]: failed to encode map e459 with expected crc 2023-09-23T21:11:38.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:11:37 smithi026 ceph-mon[179688]: failed to encode map e459 with expected crc 2023-09-23T21:11:38.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:11:37 smithi026 ceph-mon[179688]: failed to encode map e459 with expected crc 2023-09-23T21:11:38.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:11:37 smithi026 ceph-mon[179688]: pgmap v1173: 119 pgs: 1 active+recovery_wait+remapped, 3 active+recovering+undersized+remapped, 13 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 4.4 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 6.8 KiB/s rd, 29 MiB/s wr, 469 op/s; 32680/215670 objects degraded (15.153%); 12192/215670 objects misplaced (5.653%); 12 MiB/s, 1 keys/s, 197 objects/s recovering 2023-09-23T21:11:38.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:11:37 smithi026 ceph-mon[179688]: from='client.36279 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 110, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T21:11:38.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:11:37 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'0':'40257','2':'52445','4':'65536'}"}]: dispatch 2023-09-23T21:11:38.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:11:37 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/3815146473' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T21:11:38.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:11:37 smithi026 ceph-mon[175151]: failed to encode map e459 with expected crc 2023-09-23T21:11:38.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:11:37 smithi026 ceph-mon[175151]: failed to encode map e459 with expected crc 2023-09-23T21:11:38.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:11:37 smithi026 ceph-mon[175151]: pgmap v1173: 119 pgs: 1 active+recovery_wait+remapped, 3 active+recovering+undersized+remapped, 13 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 4.4 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 6.8 KiB/s rd, 29 MiB/s wr, 469 op/s; 32680/215670 objects degraded (15.153%); 12192/215670 objects misplaced (5.653%); 12 MiB/s, 1 keys/s, 197 objects/s recovering 2023-09-23T21:11:38.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:11:37 smithi026 ceph-mon[175151]: from='client.36279 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 110, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T21:11:38.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:11:37 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'0':'40257','2':'52445','4':'65536'}"}]: dispatch 2023-09-23T21:11:38.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:11:37 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/3815146473' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T21:11:38.213 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:11:38.241 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:11:38.287 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:11:38.291 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:11:38.292 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:11:38.335 INFO:teuthology.orchestra.run.smithi026.stdout:{ 2023-09-23T21:11:38.335 INFO:teuthology.orchestra.run.smithi026.stdout: "ops": [ 2023-09-23T21:11:38.335 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:38.335 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.46328.0:74542 10.a 10:5f8f2f67:::benchmark_data_smithi026_379838_object74541:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e460)", 2023-09-23T21:11:38.335 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T21:11:38.332909+0000", 2023-09-23T21:11:38.335 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 0.0022512169999999998, 2023-09-23T21:11:38.335 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0022810550000000001, 2023-09-23T21:11:38.335 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:11:38.335 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "waiting for sub ops", 2023-09-23T21:11:38.335 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T21:11:38.336 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.46328", 2023-09-23T21:11:38.336 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/3748861181", 2023-09-23T21:11:38.336 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 74542 2023-09-23T21:11:38.336 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:38.336 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:11:38.336 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:38.336 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:11:38.336 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:11:38.332909+0000", 2023-09-23T21:11:38.336 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:11:38.336 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:38.336 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:38.337 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:11:38.337 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:11:38.332909+0000", 2023-09-23T21:11:38.337 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:11:38.337 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:38.337 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:38.337 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:11:38.337 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:11:38.332907+0000", 2023-09-23T21:11:38.337 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999976 2023-09-23T21:11:38.337 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:38.337 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:38.337 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:11:38.337 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:11:38.332943+0000", 2023-09-23T21:11:38.338 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.5512999999999997e-05 2023-09-23T21:11:38.338 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:38.338 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:38.338 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:11:38.338 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:11:38.332963+0000", 2023-09-23T21:11:38.338 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.0608e-05 2023-09-23T21:11:38.338 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:38.338 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:38.338 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:11:38.338 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:11:38.332976+0000", 2023-09-23T21:11:38.338 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.2775999999999999e-05 2023-09-23T21:11:38.339 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:38.339 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:38.339 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:11:38.339 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:11:38.333032+0000", 2023-09-23T21:11:38.339 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.6369000000000002e-05 2023-09-23T21:11:38.339 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:38.339 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:38.339 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:11:38.339 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:11:38.333185+0000", 2023-09-23T21:11:38.339 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00015265799999999999 2023-09-23T21:11:38.339 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:38.340 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:38.340 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 3,5", 2023-09-23T21:11:38.340 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:11:38.333303+0000", 2023-09-23T21:11:38.340 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000118355 2023-09-23T21:11:38.340 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:11:38.340 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:11:38.340 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:11:38.340 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:11:38.340 INFO:teuthology.orchestra.run.smithi026.stdout: ], 2023-09-23T21:11:38.340 INFO:teuthology.orchestra.run.smithi026.stdout: "num_ops": 1 2023-09-23T21:11:38.340 INFO:teuthology.orchestra.run.smithi026.stdout:} 2023-09-23T21:11:38.393 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:11:38.423 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:11:38.458 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:11:38.461 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:11:38.461 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:11:38.483 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 76 16 74569 74553 61.2636 19.625 0.00446235 0.0162044 2023-09-23T21:11:38.562 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:11:38.605 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:11:38.881 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:11:38 smithi026 ceph-mon[179688]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:11:38.881 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:11:38 smithi026 ceph-mon[179688]: failed to encode map e460 with expected crc 2023-09-23T21:11:38.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:11:38 smithi026 ceph-mon[179688]: failed to encode map e460 with expected crc 2023-09-23T21:11:38.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:11:38 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'0':'40257','2':'52445','4':'65536'}"}]': finished 2023-09-23T21:11:38.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:11:38 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/3815146473' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T21:11:38.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:11:38 smithi026 ceph-mon[179688]: osdmap e460: 8 total, 8 up, 7 in 2023-09-23T21:11:38.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:11:38 smithi026 ceph-mon[175151]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:11:38.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:11:38 smithi026 ceph-mon[175151]: failed to encode map e460 with expected crc 2023-09-23T21:11:38.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:11:38 smithi026 ceph-mon[175151]: failed to encode map e460 with expected crc 2023-09-23T21:11:38.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:11:38 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'0':'40257','2':'52445','4':'65536'}"}]': finished 2023-09-23T21:11:38.883 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:11:38 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/3815146473' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T21:11:38.883 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:11:38 smithi026 ceph-mon[175151]: osdmap e460: 8 total, 8 up, 7 in 2023-09-23T21:11:39.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:11:38 smithi070 ceph-mon[168282]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:11:39.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:11:38 smithi070 ceph-mon[168282]: failed to encode map e460 with expected crc 2023-09-23T21:11:39.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:11:38 smithi070 ceph-mon[168282]: failed to encode map e460 with expected crc 2023-09-23T21:11:39.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:11:38 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'0':'40257','2':'52445','4':'65536'}"}]': finished 2023-09-23T21:11:39.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:11:38 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/3815146473' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T21:11:39.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:11:38 smithi070 ceph-mon[168282]: osdmap e460: 8 total, 8 up, 7 in 2023-09-23T21:11:39.216 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:11:39.220 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:11:39.220 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:11:39.271 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_blocked_ops 2023-09-23T21:11:39.316 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:11:39.321 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:11:39.349 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:11:39.380 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:11:39.383 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:11:39.384 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:11:39.483 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 77 16 74636 74620 60.5229 4.1875 0.0297498 0.0161949 2023-09-23T21:11:39.485 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:11:39.518 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:11:39.881 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 21:11:39 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[164029]: ::ffff:172.21.15.70 - - [23/Sep/2023:21:11:39] "GET /metrics HTTP/1.1" 200 36429 "" "Prometheus/2.43.0" 2023-09-23T21:11:39.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:11:39 smithi026 ceph-mon[175151]: failed to encode map e461 with expected crc 2023-09-23T21:11:39.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:11:39 smithi026 ceph-mon[175151]: failed to encode map e461 with expected crc 2023-09-23T21:11:39.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:11:39 smithi026 ceph-mon[175151]: failed to encode map e461 with expected crc 2023-09-23T21:11:39.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:11:39 smithi026 ceph-mon[175151]: osdmap e461: 8 total, 8 up, 7 in 2023-09-23T21:11:39.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:11:39 smithi026 ceph-mon[175151]: failed to encode map e461 with expected crc 2023-09-23T21:11:39.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:11:39 smithi026 ceph-mon[175151]: pgmap v1176: 119 pgs: 1 peering, 1 active+recovery_wait+remapped, 2 active+recovering+undersized+remapped, 13 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 4.5 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 37 MiB/s wr, 594 op/s; 32680/221478 objects degraded (14.755%); 11503/221478 objects misplaced (5.194%); 7.2 MiB/s, 114 objects/s recovering 2023-09-23T21:11:39.883 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:11:39 smithi026 ceph-mon[179688]: failed to encode map e461 with expected crc 2023-09-23T21:11:39.883 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:11:39 smithi026 ceph-mon[179688]: failed to encode map e461 with expected crc 2023-09-23T21:11:39.883 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:11:39 smithi026 ceph-mon[179688]: failed to encode map e461 with expected crc 2023-09-23T21:11:39.883 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:11:39 smithi026 ceph-mon[179688]: osdmap e461: 8 total, 8 up, 7 in 2023-09-23T21:11:39.883 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:11:39 smithi026 ceph-mon[179688]: failed to encode map e461 with expected crc 2023-09-23T21:11:39.883 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:11:39 smithi026 ceph-mon[179688]: pgmap v1176: 119 pgs: 1 peering, 1 active+recovery_wait+remapped, 2 active+recovering+undersized+remapped, 13 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 4.5 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 37 MiB/s wr, 594 op/s; 32680/221478 objects degraded (14.755%); 11503/221478 objects misplaced (5.194%); 7.2 MiB/s, 114 objects/s recovering 2023-09-23T21:11:40.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:11:39 smithi070 ceph-mon[168282]: failed to encode map e461 with expected crc 2023-09-23T21:11:40.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:11:39 smithi070 ceph-mon[168282]: failed to encode map e461 with expected crc 2023-09-23T21:11:40.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:11:39 smithi070 ceph-mon[168282]: failed to encode map e461 with expected crc 2023-09-23T21:11:40.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:11:39 smithi070 ceph-mon[168282]: osdmap e461: 8 total, 8 up, 7 in 2023-09-23T21:11:40.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:11:39 smithi070 ceph-mon[168282]: failed to encode map e461 with expected crc 2023-09-23T21:11:40.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:11:39 smithi070 ceph-mon[168282]: pgmap v1176: 119 pgs: 1 peering, 1 active+recovery_wait+remapped, 2 active+recovering+undersized+remapped, 13 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 4.5 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 37 MiB/s wr, 594 op/s; 32680/221478 objects degraded (14.755%); 11503/221478 objects misplaced (5.194%); 7.2 MiB/s, 114 objects/s recovering 2023-09-23T21:11:40.323 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:11:40.329 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:11:40.330 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:11:40.423 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T21:11:40.431 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:11:40.484 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 78 16 74907 74891 59.9641 16.9375 0.0112067 0.0166682 2023-09-23T21:11:40.569 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:11:41.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:11:40 smithi070 ceph-mon[168282]: failed to encode map e462 with expected crc 2023-09-23T21:11:41.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:11:40 smithi070 ceph-mon[168282]: failed to encode map e462 with expected crc 2023-09-23T21:11:41.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:11:40 smithi070 ceph-mon[168282]: osdmap e462: 8 total, 8 up, 7 in 2023-09-23T21:11:41.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:11:40 smithi070 ceph-mon[168282]: 10.d starting backfill to osd.7 from (0'0,0'0] MAX to 459'4640 2023-09-23T21:11:41.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:11:40 smithi070 ceph-mon[168282]: Health check update: Degraded data redundancy: 32680/221478 objects degraded (14.755%), 13 pgs degraded (PG_DEGRADED) 2023-09-23T21:11:41.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:11:40 smithi070 ceph-mon[168282]: osdmap e463: 8 total, 8 up, 7 in 2023-09-23T21:11:41.131 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:11:40 smithi026 ceph-mon[179688]: failed to encode map e462 with expected crc 2023-09-23T21:11:41.131 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:11:40 smithi026 ceph-mon[179688]: failed to encode map e462 with expected crc 2023-09-23T21:11:41.131 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:11:40 smithi026 ceph-mon[179688]: osdmap e462: 8 total, 8 up, 7 in 2023-09-23T21:11:41.131 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:11:40 smithi026 ceph-mon[179688]: 10.d starting backfill to osd.7 from (0'0,0'0] MAX to 459'4640 2023-09-23T21:11:41.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:11:40 smithi026 ceph-mon[179688]: Health check update: Degraded data redundancy: 32680/221478 objects degraded (14.755%), 13 pgs degraded (PG_DEGRADED) 2023-09-23T21:11:41.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:11:40 smithi026 ceph-mon[179688]: osdmap e463: 8 total, 8 up, 7 in 2023-09-23T21:11:41.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:11:40 smithi026 ceph-mon[175151]: failed to encode map e462 with expected crc 2023-09-23T21:11:41.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:11:40 smithi026 ceph-mon[175151]: failed to encode map e462 with expected crc 2023-09-23T21:11:41.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:11:40 smithi026 ceph-mon[175151]: osdmap e462: 8 total, 8 up, 7 in 2023-09-23T21:11:41.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:11:40 smithi026 ceph-mon[175151]: 10.d starting backfill to osd.7 from (0'0,0'0] MAX to 459'4640 2023-09-23T21:11:41.134 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:11:40 smithi026 ceph-mon[175151]: Health check update: Degraded data redundancy: 32680/221478 objects degraded (14.755%), 13 pgs degraded (PG_DEGRADED) 2023-09-23T21:11:41.134 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:11:40 smithi026 ceph-mon[175151]: osdmap e463: 8 total, 8 up, 7 in 2023-09-23T21:11:41.396 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:11:41.407 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:11:41.408 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:11:41.484 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 79 16 75707 75691 59.838 50 0.00266846 0.0167011 2023-09-23T21:11:41.509 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:11:41.577 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:11:41.636 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:11:41.640 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:11:41.640 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:11:41.741 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:11:41.792 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:11:41.841 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:11:41.844 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:11:41.845 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:11:41.945 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:11:41.989 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:11:42.036 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:11:42.039 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:11:41 smithi070 ceph-mon[168282]: failed to encode map e463 with expected crc 2023-09-23T21:11:42.039 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:11:41 smithi070 ceph-mon[168282]: failed to encode map e463 with expected crc 2023-09-23T21:11:42.039 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:11:41 smithi070 ceph-mon[168282]: failed to encode map e463 with expected crc 2023-09-23T21:11:42.039 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:11:41 smithi070 ceph-mon[168282]: failed to encode map e463 with expected crc 2023-09-23T21:11:42.039 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:11:41 smithi070 ceph-mon[168282]: pgmap v1179: 119 pgs: 1 peering, 1 active+recovery_wait+remapped, 2 active+recovering+undersized+remapped, 13 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 4.5 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 36 MiB/s wr, 568 op/s; 32680/222489 objects degraded (14.688%); 11503/222489 objects misplaced (5.170%); 11 MiB/s, 171 objects/s recovering 2023-09-23T21:11:42.039 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:11:41 smithi070 ceph-mon[168282]: osdmap e464: 8 total, 8 up, 7 in 2023-09-23T21:11:42.040 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:11:42.040 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:11:42.086 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:11:41 smithi026 ceph-mon[175151]: failed to encode map e463 with expected crc 2023-09-23T21:11:42.086 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:11:41 smithi026 ceph-mon[175151]: failed to encode map e463 with expected crc 2023-09-23T21:11:42.086 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:11:41 smithi026 ceph-mon[175151]: failed to encode map e463 with expected crc 2023-09-23T21:11:42.086 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:11:41 smithi026 ceph-mon[175151]: failed to encode map e463 with expected crc 2023-09-23T21:11:42.086 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:11:41 smithi026 ceph-mon[175151]: pgmap v1179: 119 pgs: 1 peering, 1 active+recovery_wait+remapped, 2 active+recovering+undersized+remapped, 13 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 4.5 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 36 MiB/s wr, 568 op/s; 32680/222489 objects degraded (14.688%); 11503/222489 objects misplaced (5.170%); 11 MiB/s, 171 objects/s recovering 2023-09-23T21:11:42.086 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:11:41 smithi026 ceph-mon[175151]: osdmap e464: 8 total, 8 up, 7 in 2023-09-23T21:11:42.087 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:11:41 smithi026 ceph-mon[179688]: failed to encode map e463 with expected crc 2023-09-23T21:11:42.087 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:11:41 smithi026 ceph-mon[179688]: failed to encode map e463 with expected crc 2023-09-23T21:11:42.087 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:11:41 smithi026 ceph-mon[179688]: failed to encode map e463 with expected crc 2023-09-23T21:11:42.087 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:11:41 smithi026 ceph-mon[179688]: failed to encode map e463 with expected crc 2023-09-23T21:11:42.087 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:11:41 smithi026 ceph-mon[179688]: pgmap v1179: 119 pgs: 1 peering, 1 active+recovery_wait+remapped, 2 active+recovering+undersized+remapped, 13 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 4.5 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 36 MiB/s wr, 568 op/s; 32680/222489 objects degraded (14.688%); 11503/222489 objects misplaced (5.170%); 11 MiB/s, 171 objects/s recovering 2023-09-23T21:11:42.087 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:11:41 smithi026 ceph-mon[179688]: osdmap e464: 8 total, 8 up, 7 in 2023-09-23T21:11:42.141 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:11:42.224 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:11:42.239 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:11:42.484 INFO:tasks.radosbench.radosbench.0.smithi026.stdout:2023-09-23T21:11:42.485146+0000 min lat: 0.00120238 max lat: 4.38755 avg lat: 0.016742 2023-09-23T21:11:42.484 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T21:11:42.484 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 80 16 76464 76448 59.6815 47.3125 0.00760175 0.016742 2023-09-23T21:11:42.565 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T21:11:42.571 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:11:42.572 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:11:42.672 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:11:42.694 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:11:42.708 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:11:42.754 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:11:42.758 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:11:42.759 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:11:42.835 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:11:42 smithi026 ceph-mon[175151]: failed to encode map e464 with expected crc 2023-09-23T21:11:42.835 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:11:42 smithi026 ceph-mon[175151]: failed to encode map e464 with expected crc 2023-09-23T21:11:42.835 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:11:42 smithi026 ceph-mon[175151]: failed to encode map e464 with expected crc 2023-09-23T21:11:42.835 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:11:42 smithi026 ceph-mon[175151]: failed to encode map e464 with expected crc 2023-09-23T21:11:42.835 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:11:42 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:11:42.835 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:11:42 smithi026 ceph-mon[179688]: failed to encode map e464 with expected crc 2023-09-23T21:11:42.835 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:11:42 smithi026 ceph-mon[179688]: failed to encode map e464 with expected crc 2023-09-23T21:11:42.835 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:11:42 smithi026 ceph-mon[179688]: failed to encode map e464 with expected crc 2023-09-23T21:11:42.835 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:11:42 smithi026 ceph-mon[179688]: failed to encode map e464 with expected crc 2023-09-23T21:11:42.836 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:11:42 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:11:42.860 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:11:42.919 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:11:43.105 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T21:11:43.108 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:11:43.109 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:11:43.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:11:42 smithi070 ceph-mon[168282]: failed to encode map e464 with expected crc 2023-09-23T21:11:43.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:11:42 smithi070 ceph-mon[168282]: failed to encode map e464 with expected crc 2023-09-23T21:11:43.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:11:42 smithi070 ceph-mon[168282]: failed to encode map e464 with expected crc 2023-09-23T21:11:43.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:11:42 smithi070 ceph-mon[168282]: failed to encode map e464 with expected crc 2023-09-23T21:11:43.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:11:42 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:11:43.209 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:11:43.296 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:11:43.484 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 81 16 77214 77198 59.5235 46.875 0.00982021 0.0167915 2023-09-23T21:11:43.784 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:11:43 smithi026 ceph-mon[175151]: pgmap v1181: 119 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 13 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 4.6 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 9.6 KiB/s rd, 28 MiB/s wr, 448 op/s; 32680/227112 objects degraded (14.389%); 15684/227112 objects misplaced (6.906%); 8.8 MiB/s, 140 objects/s recovering 2023-09-23T21:11:44.001 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 7] out_osds: [6] dead_osds: [] live_osds: [0, 2, 3, 4, 5, 7, 6, 1] 2023-09-23T21:11:44.001 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T21:11:44.001 INFO:tasks.thrashosds.thrasher:Killing osd 0, live_osds are [0, 2, 3, 4, 5, 7, 6, 1] 2023-09-23T21:11:44.001 INFO:tasks.cephadm.osd.0:Stopping osd.0... 2023-09-23T21:11:44.001 DEBUG:teuthology.orchestra.run.smithi026:> sudo systemctl stop ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7@osd.0 2023-09-23T21:11:44.061 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:11:43 smithi026 ceph-mon[179688]: pgmap v1181: 119 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 13 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 4.6 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 9.6 KiB/s rd, 28 MiB/s wr, 448 op/s; 32680/227112 objects degraded (14.389%); 15684/227112 objects misplaced (6.906%); 8.8 MiB/s, 140 objects/s recovering 2023-09-23T21:11:44.121 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:11:43 smithi070 ceph-mon[168282]: pgmap v1181: 119 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 13 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 4.6 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 9.6 KiB/s rd, 28 MiB/s wr, 448 op/s; 32680/227112 objects degraded (14.389%); 15684/227112 objects misplaced (6.906%); 8.8 MiB/s, 140 objects/s recovering 2023-09-23T21:11:44.192 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:11:44.197 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:11:44.198 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:11:44.298 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:11:44.317 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0: {} 2023-09-23T21:11:44.339 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:11:44.360 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1: {} 2023-09-23T21:11:44.376 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:11:44.380 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:11:44.380 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:11:44.382 INFO:journalctl@ceph.osd.0.smithi026.stdout:Sep 23 21:11:44 smithi026 systemd[1]: Stopping Ceph osd.0 for b3f8e4ec-5a50-11ee-8db4-212e2dc638e7... 2023-09-23T21:11:44.414 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2: {} 2023-09-23T21:11:44.466 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3: {} 2023-09-23T21:11:44.482 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:11:44.485 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 82 16 77965 77949 59.3694 46.9375 0.0586651 0.0168348 2023-09-23T21:11:44.506 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4: {} 2023-09-23T21:11:44.516 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:11:44.545 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T21:11:44.572 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:11:44.575 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:11:44.575 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:11:44.581 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6: {} 2023-09-23T21:11:44.616 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T21:11:44.626 INFO:teuthology.orchestra.run.smithi026.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T21:11:44.626 INFO:teuthology.orchestra.run.smithi026.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T21:11:44.626 INFO:teuthology.orchestra.run.smithi026.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T21:11:44.626 INFO:teuthology.orchestra.run.smithi026.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T21:11:44.626 INFO:teuthology.orchestra.run.smithi026.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T21:11:44.627 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T21:11:44.627 INFO:teuthology.orchestra.run.smithi026.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T21:11:44.627 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T21:11:44.676 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:11:44.717 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:11:44.765 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:11:44.771 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:11:44.771 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:11:44.872 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:11:44.909 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:11:45.031 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:11:45.032 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:11:45.032 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:11:45.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:11:44 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/3116641256' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:11:45.187 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:11:45.188 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:11:44 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/3116641256' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:11:45.188 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:11:44 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/3116641256' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:11:45.225 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:11:45.340 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:11:45.341 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:11:45.341 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:11:45.442 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:11:45.485 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 83 16 78596 78580 59.1294 39.4375 0.0457953 0.016896 2023-09-23T21:11:45.511 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:11:45.632 INFO:journalctl@ceph.osd.0.smithi026.stdout:Sep 23 21:11:45 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd-0[190439]: 2023-09-23T21:11:45.319+0000 7f279e6ad700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-osd -n osd.0 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2023-09-23T21:11:46.120 INFO:journalctl@ceph.osd.0.smithi026.stdout:Sep 23 21:11:45 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd-0[190439]: 2023-09-23T21:11:45.319+0000 7f279e6ad700 -1 osd.0 464 *** Got signal Terminated *** 2023-09-23T21:11:46.120 INFO:journalctl@ceph.osd.0.smithi026.stdout:Sep 23 21:11:45 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd-0[190439]: 2023-09-23T21:11:45.319+0000 7f279e6ad700 -1 osd.0 464 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-09-23T21:11:46.121 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:11:45 smithi070 ceph-mon[168282]: pgmap v1182: 119 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 13 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 4.7 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 6.8 KiB/s rd, 28 MiB/s wr, 458 op/s; 32680/229629 objects degraded (14.232%); 15684/229629 objects misplaced (6.830%); 6.2 MiB/s, 99 objects/s recovering 2023-09-23T21:11:46.121 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:11:45 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/506142113' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:11:46.121 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:11:45 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:11:46.121 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:11:45 smithi070 ceph-mon[168282]: osd.0 marked itself down and dead 2023-09-23T21:11:46.121 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:11:45 smithi070 ceph-mon[168282]: Health check update: Degraded data redundancy: 32680/227112 objects degraded (14.389%), 13 pgs degraded (PG_DEGRADED) 2023-09-23T21:11:46.121 INFO:teuthology.orchestra.run.smithi026.stdout:ERROR: (22) Invalid argument 2023-09-23T21:11:46.121 INFO:teuthology.orchestra.run.smithi026.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T21:11:46.122 INFO:teuthology.orchestra.run.smithi026.stderr:nodeep-scrub is unset 2023-09-23T21:11:46.131 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:11:45 smithi026 ceph-mon[175151]: pgmap v1182: 119 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 13 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 4.7 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 6.8 KiB/s rd, 28 MiB/s wr, 458 op/s; 32680/229629 objects degraded (14.232%); 15684/229629 objects misplaced (6.830%); 6.2 MiB/s, 99 objects/s recovering 2023-09-23T21:11:46.131 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:11:45 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/506142113' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:11:46.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:11:45 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:11:46.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:11:45 smithi026 ceph-mon[175151]: osd.0 marked itself down and dead 2023-09-23T21:11:46.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:11:45 smithi026 ceph-mon[175151]: Health check update: Degraded data redundancy: 32680/227112 objects degraded (14.389%), 13 pgs degraded (PG_DEGRADED) 2023-09-23T21:11:46.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:11:45 smithi026 ceph-mon[179688]: pgmap v1182: 119 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 13 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 4.7 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 6.8 KiB/s rd, 28 MiB/s wr, 458 op/s; 32680/229629 objects degraded (14.232%); 15684/229629 objects misplaced (6.830%); 6.2 MiB/s, 99 objects/s recovering 2023-09-23T21:11:46.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:11:45 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/506142113' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:11:46.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:11:45 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:11:46.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:11:45 smithi026 ceph-mon[179688]: osd.0 marked itself down and dead 2023-09-23T21:11:46.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:11:45 smithi026 ceph-mon[179688]: Health check update: Degraded data redundancy: 32680/227112 objects degraded (14.389%), 13 pgs degraded (PG_DEGRADED) 2023-09-23T21:11:46.485 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 84 16 78859 78843 58.6215 16.4375 0.00534678 0.0169297 2023-09-23T21:11:47.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:11:46 smithi070 ceph-mon[168282]: Health check failed: 1 osds down (OSD_DOWN) 2023-09-23T21:11:47.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:11:46 smithi070 ceph-mon[168282]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T21:11:47.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:11:46 smithi070 ceph-mon[168282]: failed to encode map e465 with expected crc 2023-09-23T21:11:47.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:11:46 smithi070 ceph-mon[168282]: failed to encode map e465 with expected crc 2023-09-23T21:11:47.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:11:46 smithi070 ceph-mon[168282]: failed to encode map e465 with expected crc 2023-09-23T21:11:47.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:11:46 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T21:11:47.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:11:46 smithi070 ceph-mon[168282]: osdmap e465: 8 total, 7 up, 7 in 2023-09-23T21:11:47.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:11:46 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:11:47.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:11:46 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:11:47.131 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:11:46 smithi026 ceph-mon[179688]: Health check failed: 1 osds down (OSD_DOWN) 2023-09-23T21:11:47.131 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:11:46 smithi026 ceph-mon[179688]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T21:11:47.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:11:46 smithi026 ceph-mon[179688]: failed to encode map e465 with expected crc 2023-09-23T21:11:47.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:11:46 smithi026 ceph-mon[179688]: failed to encode map e465 with expected crc 2023-09-23T21:11:47.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:11:46 smithi026 ceph-mon[179688]: failed to encode map e465 with expected crc 2023-09-23T21:11:47.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:11:46 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T21:11:47.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:11:46 smithi026 ceph-mon[179688]: osdmap e465: 8 total, 7 up, 7 in 2023-09-23T21:11:47.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:11:46 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:11:47.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:11:46 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:11:47.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:11:46 smithi026 ceph-mon[175151]: Health check failed: 1 osds down (OSD_DOWN) 2023-09-23T21:11:47.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:11:46 smithi026 ceph-mon[175151]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T21:11:47.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:11:46 smithi026 ceph-mon[175151]: failed to encode map e465 with expected crc 2023-09-23T21:11:47.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:11:46 smithi026 ceph-mon[175151]: failed to encode map e465 with expected crc 2023-09-23T21:11:47.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:11:46 smithi026 ceph-mon[175151]: failed to encode map e465 with expected crc 2023-09-23T21:11:47.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:11:46 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T21:11:47.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:11:46 smithi026 ceph-mon[175151]: osdmap e465: 8 total, 7 up, 7 in 2023-09-23T21:11:47.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:11:46 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:11:47.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:11:46 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:11:47.485 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 85 16 78894 78878 57.9579 2.1875 0.00454105 0.0170258 2023-09-23T21:11:48.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:11:47 smithi070 ceph-mon[168282]: failed to encode map e466 with expected crc 2023-09-23T21:11:48.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:11:47 smithi070 ceph-mon[168282]: failed to encode map e466 with expected crc 2023-09-23T21:11:48.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:11:47 smithi070 ceph-mon[168282]: osdmap e466: 8 total, 7 up, 7 in 2023-09-23T21:11:48.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:11:47 smithi070 ceph-mon[168282]: failed to encode map e466 with expected crc 2023-09-23T21:11:48.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:11:47 smithi070 ceph-mon[168282]: failed to encode map e466 with expected crc 2023-09-23T21:11:48.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:11:47 smithi070 ceph-mon[168282]: failed to encode map e466 with expected crc 2023-09-23T21:11:48.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:11:47 smithi070 ceph-mon[168282]: failed to encode map e466 with expected crc 2023-09-23T21:11:48.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:11:47 smithi070 ceph-mon[168282]: failed to encode map e466 with expected crc 2023-09-23T21:11:48.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:11:47 smithi070 ceph-mon[168282]: 10.d continuing backfill to osd.7 from (422'400,464'4922] MIN to 464'4922 2023-09-23T21:11:48.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:11:47 smithi070 ceph-mon[168282]: failed to encode map e466 with expected crc 2023-09-23T21:11:48.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:11:47 smithi070 ceph-mon[168282]: failed to encode map e466 with expected crc 2023-09-23T21:11:48.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:11:47 smithi070 ceph-mon[168282]: pgmap v1185: 119 pgs: 2 remapped+peering, 8 peering, 1 stale+active+recovery_wait+remapped, 2 stale+active+recovery_wait+undersized+degraded+remapped, 8 stale+active+clean, 2 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 9 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 4.7 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 14 KiB/s rd, 38 MiB/s wr, 613 op/s; 28300/233334 objects degraded (12.129%); 10948/233334 objects misplaced (4.692%); 10 MiB/s, 158 objects/s recovering 2023-09-23T21:11:48.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:11:47 smithi026 ceph-mon[175151]: failed to encode map e466 with expected crc 2023-09-23T21:11:48.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:11:47 smithi026 ceph-mon[175151]: failed to encode map e466 with expected crc 2023-09-23T21:11:48.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:11:47 smithi026 ceph-mon[175151]: osdmap e466: 8 total, 7 up, 7 in 2023-09-23T21:11:48.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:11:47 smithi026 ceph-mon[175151]: failed to encode map e466 with expected crc 2023-09-23T21:11:48.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:11:47 smithi026 ceph-mon[175151]: failed to encode map e466 with expected crc 2023-09-23T21:11:48.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:11:47 smithi026 ceph-mon[175151]: failed to encode map e466 with expected crc 2023-09-23T21:11:48.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:11:47 smithi026 ceph-mon[175151]: failed to encode map e466 with expected crc 2023-09-23T21:11:48.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:11:47 smithi026 ceph-mon[175151]: failed to encode map e466 with expected crc 2023-09-23T21:11:48.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:11:47 smithi026 ceph-mon[175151]: 10.d continuing backfill to osd.7 from (422'400,464'4922] MIN to 464'4922 2023-09-23T21:11:48.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:11:47 smithi026 ceph-mon[175151]: failed to encode map e466 with expected crc 2023-09-23T21:11:48.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:11:47 smithi026 ceph-mon[175151]: failed to encode map e466 with expected crc 2023-09-23T21:11:48.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:11:47 smithi026 ceph-mon[175151]: pgmap v1185: 119 pgs: 2 remapped+peering, 8 peering, 1 stale+active+recovery_wait+remapped, 2 stale+active+recovery_wait+undersized+degraded+remapped, 8 stale+active+clean, 2 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 9 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 4.7 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 14 KiB/s rd, 38 MiB/s wr, 613 op/s; 28300/233334 objects degraded (12.129%); 10948/233334 objects misplaced (4.692%); 10 MiB/s, 158 objects/s recovering 2023-09-23T21:11:48.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:11:47 smithi026 ceph-mon[179688]: failed to encode map e466 with expected crc 2023-09-23T21:11:48.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:11:47 smithi026 ceph-mon[179688]: failed to encode map e466 with expected crc 2023-09-23T21:11:48.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:11:47 smithi026 ceph-mon[179688]: osdmap e466: 8 total, 7 up, 7 in 2023-09-23T21:11:48.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:11:47 smithi026 ceph-mon[179688]: failed to encode map e466 with expected crc 2023-09-23T21:11:48.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:11:47 smithi026 ceph-mon[179688]: failed to encode map e466 with expected crc 2023-09-23T21:11:48.134 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:11:47 smithi026 ceph-mon[179688]: failed to encode map e466 with expected crc 2023-09-23T21:11:48.134 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:11:47 smithi026 ceph-mon[179688]: failed to encode map e466 with expected crc 2023-09-23T21:11:48.134 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:11:47 smithi026 ceph-mon[179688]: failed to encode map e466 with expected crc 2023-09-23T21:11:48.134 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:11:47 smithi026 ceph-mon[179688]: 10.d continuing backfill to osd.7 from (422'400,464'4922] MIN to 464'4922 2023-09-23T21:11:48.134 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:11:47 smithi026 ceph-mon[179688]: failed to encode map e466 with expected crc 2023-09-23T21:11:48.134 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:11:47 smithi026 ceph-mon[179688]: failed to encode map e466 with expected crc 2023-09-23T21:11:48.134 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:11:47 smithi026 ceph-mon[179688]: pgmap v1185: 119 pgs: 2 remapped+peering, 8 peering, 1 stale+active+recovery_wait+remapped, 2 stale+active+recovery_wait+undersized+degraded+remapped, 8 stale+active+clean, 2 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 9 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 4.7 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 14 KiB/s rd, 38 MiB/s wr, 613 op/s; 28300/233334 objects degraded (12.129%); 10948/233334 objects misplaced (4.692%); 10 MiB/s, 158 objects/s recovering 2023-09-23T21:11:48.485 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 86 16 78894 78878 57.2844 0 - 0.0170258 2023-09-23T21:11:48.631 INFO:journalctl@ceph.osd.0.smithi026.stdout:Sep 23 21:11:48 smithi026 bash[391239]: ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd-0 2023-09-23T21:11:49.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:11:48 smithi070 ceph-mon[168282]: Health check failed: Reduced data availability: 2 pgs inactive, 2 pgs peering (PG_AVAILABILITY) 2023-09-23T21:11:49.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:11:48 smithi070 ceph-mon[168282]: failed to encode map e467 with expected crc 2023-09-23T21:11:49.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:11:48 smithi070 ceph-mon[168282]: failed to encode map e467 with expected crc 2023-09-23T21:11:49.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:11:48 smithi070 ceph-mon[168282]: failed to encode map e467 with expected crc 2023-09-23T21:11:49.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:11:48 smithi070 ceph-mon[168282]: osdmap e467: 8 total, 7 up, 7 in 2023-09-23T21:11:49.131 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:11:48 smithi026 ceph-mon[179688]: Health check failed: Reduced data availability: 2 pgs inactive, 2 pgs peering (PG_AVAILABILITY) 2023-09-23T21:11:49.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:11:48 smithi026 ceph-mon[179688]: failed to encode map e467 with expected crc 2023-09-23T21:11:49.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:11:48 smithi026 ceph-mon[179688]: failed to encode map e467 with expected crc 2023-09-23T21:11:49.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:11:48 smithi026 ceph-mon[179688]: failed to encode map e467 with expected crc 2023-09-23T21:11:49.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:11:48 smithi026 ceph-mon[179688]: osdmap e467: 8 total, 7 up, 7 in 2023-09-23T21:11:49.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:11:48 smithi026 ceph-mon[175151]: Health check failed: Reduced data availability: 2 pgs inactive, 2 pgs peering (PG_AVAILABILITY) 2023-09-23T21:11:49.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:11:48 smithi026 ceph-mon[175151]: failed to encode map e467 with expected crc 2023-09-23T21:11:49.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:11:48 smithi026 ceph-mon[175151]: failed to encode map e467 with expected crc 2023-09-23T21:11:49.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:11:48 smithi026 ceph-mon[175151]: failed to encode map e467 with expected crc 2023-09-23T21:11:49.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:11:48 smithi026 ceph-mon[175151]: osdmap e467: 8 total, 7 up, 7 in 2023-09-23T21:11:49.333 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T21:11:49.486 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 87 16 78911 78895 56.6386 0.53125 0.0268614 0.0171144 2023-09-23T21:11:49.631 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 21:11:49 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[164029]: ::ffff:172.21.15.70 - - [23/Sep/2023:21:11:49] "GET /metrics HTTP/1.1" 200 36450 "" "Prometheus/2.43.0" 2023-09-23T21:11:50.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:11:49 smithi070 ceph-mon[168282]: pgmap v1187: 119 pgs: 1 activating+degraded+remapped, 5 remapped+peering, 9 peering, 1 stale+active+clean, 1 active+undersized+degraded+remapped+backfill_wait, 17 active+undersized, 1 active+recovering+undersized+remapped, 3 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 72 active+clean; 4.8 GiB data, 18 GiB used, 607 GiB / 626 GiB avail; 6.8 KiB/s rd, 33 MiB/s wr, 525 op/s; 41764/236484 objects degraded (17.660%); 6392/236484 objects misplaced (2.703%); 4.6 MiB/s, 73 objects/s recovering 2023-09-23T21:11:50.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:11:49 smithi026 ceph-mon[175151]: pgmap v1187: 119 pgs: 1 activating+degraded+remapped, 5 remapped+peering, 9 peering, 1 stale+active+clean, 1 active+undersized+degraded+remapped+backfill_wait, 17 active+undersized, 1 active+recovering+undersized+remapped, 3 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 72 active+clean; 4.8 GiB data, 18 GiB used, 607 GiB / 626 GiB avail; 6.8 KiB/s rd, 33 MiB/s wr, 525 op/s; 41764/236484 objects degraded (17.660%); 6392/236484 objects misplaced (2.703%); 4.6 MiB/s, 73 objects/s recovering 2023-09-23T21:11:50.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:11:49 smithi026 ceph-mon[179688]: pgmap v1187: 119 pgs: 1 activating+degraded+remapped, 5 remapped+peering, 9 peering, 1 stale+active+clean, 1 active+undersized+degraded+remapped+backfill_wait, 17 active+undersized, 1 active+recovering+undersized+remapped, 3 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 72 active+clean; 4.8 GiB data, 18 GiB used, 607 GiB / 626 GiB avail; 6.8 KiB/s rd, 33 MiB/s wr, 525 op/s; 41764/236484 objects degraded (17.660%); 6392/236484 objects misplaced (2.703%); 4.6 MiB/s, 73 objects/s recovering 2023-09-23T21:11:50.200 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:11:50.204 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:11:50.204 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:11:50.305 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:11:50.338 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:11:50.486 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 88 16 79017 79001 56.0706 6.625 0.0032945 0.0176265 2023-09-23T21:11:51.097 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:11:50 smithi026 ceph-mon[179688]: Health check update: Degraded data redundancy: 41764/236484 objects degraded (17.660%), 14 pgs degraded (PG_DEGRADED) 2023-09-23T21:11:51.097 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:11:50 smithi026 ceph-mon[175151]: Health check update: Degraded data redundancy: 41764/236484 objects degraded (17.660%), 14 pgs degraded (PG_DEGRADED) 2023-09-23T21:11:51.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:11:50 smithi070 ceph-mon[168282]: Health check update: Degraded data redundancy: 41764/236484 objects degraded (17.660%), 14 pgs degraded (PG_DEGRADED) 2023-09-23T21:11:51.122 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_historic_ops 2023-09-23T21:11:51.285 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:11:51.299 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:11:51.300 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:11:51.401 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:11:51.440 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:11:51.486 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 89 16 79487 79471 55.7708 29.375 0.202256 0.0179141 2023-09-23T21:11:51.526 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:11:51.536 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:11:51.537 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:11:51.567 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:11:51.638 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:11:51.687 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:11:51.742 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:11:51.747 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:11:51.747 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:11:51.848 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:11:51.945 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:11:51 smithi026 ceph-mon[179688]: pgmap v1188: 119 pgs: 1 activating+degraded+remapped, 5 remapped+peering, 9 peering, 1 active+undersized+degraded+remapped+backfill_wait, 17 active+undersized, 1 active+recovering+undersized+remapped, 4 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 72 active+clean; 4.8 GiB data, 18 GiB used, 607 GiB / 626 GiB avail; 6.8 KiB/s rd, 25 MiB/s wr, 410 op/s; 41765/236922 objects degraded (17.628%); 6393/236922 objects misplaced (2.698%); 4.6 MiB/s, 73 objects/s recovering 2023-09-23T21:11:51.946 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:11:51 smithi026 ceph-mon[175151]: pgmap v1188: 119 pgs: 1 activating+degraded+remapped, 5 remapped+peering, 9 peering, 1 active+undersized+degraded+remapped+backfill_wait, 17 active+undersized, 1 active+recovering+undersized+remapped, 4 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 72 active+clean; 4.8 GiB data, 18 GiB used, 607 GiB / 626 GiB avail; 6.8 KiB/s rd, 25 MiB/s wr, 410 op/s; 41765/236922 objects degraded (17.628%); 6393/236922 objects misplaced (2.698%); 4.6 MiB/s, 73 objects/s recovering 2023-09-23T21:11:51.975 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:11:52.034 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:11:52.038 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:11:52.039 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:11:52.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:11:51 smithi070 ceph-mon[168282]: pgmap v1188: 119 pgs: 1 activating+degraded+remapped, 5 remapped+peering, 9 peering, 1 active+undersized+degraded+remapped+backfill_wait, 17 active+undersized, 1 active+recovering+undersized+remapped, 4 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 72 active+clean; 4.8 GiB data, 18 GiB used, 607 GiB / 626 GiB avail; 6.8 KiB/s rd, 25 MiB/s wr, 410 op/s; 41765/236922 objects degraded (17.628%); 6393/236922 objects misplaced (2.698%); 4.6 MiB/s, 73 objects/s recovering 2023-09-23T21:11:52.139 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:11:52.140 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:11:52.197 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:11:52.260 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:11:52.266 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:11:52.266 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:11:52.367 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:11:52.435 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:11:52.475 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:11:52.478 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:11:52.479 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:11:52.486 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 90 3 79954 79951 55.4845 30 0.105867 0.0180093 2023-09-23T21:11:52.579 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:11:52.605 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:11:52.638 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:11:52.641 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:11:52.642 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:11:52.742 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:11:52.779 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:11:52.972 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:11:52.976 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:11:52.977 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:11:53.077 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:11:53.113 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:11:53.319 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T21:11:53.347 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:11:53 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:11:53.347 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:11:53 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:11:53.357 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:11:53.362 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:11:53.362 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:11:53.463 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:11:53.486 INFO:tasks.radosbench.radosbench.0.smithi026.stdout:Total time run: 90.071 2023-09-23T21:11:53.486 INFO:tasks.radosbench.radosbench.0.smithi026.stdout:Total writes made: 79954 2023-09-23T21:11:53.486 INFO:tasks.radosbench.radosbench.0.smithi026.stdout:Write size: 65536 2023-09-23T21:11:53.486 INFO:tasks.radosbench.radosbench.0.smithi026.stdout:Object size: 65536 2023-09-23T21:11:53.487 INFO:tasks.radosbench.radosbench.0.smithi026.stdout:Bandwidth (MB/sec): 55.4798 2023-09-23T21:11:53.487 INFO:tasks.radosbench.radosbench.0.smithi026.stdout:Stddev Bandwidth: 37.1906 2023-09-23T21:11:53.487 INFO:tasks.radosbench.radosbench.0.smithi026.stdout:Max bandwidth (MB/sec): 147.875 2023-09-23T21:11:53.487 INFO:tasks.radosbench.radosbench.0.smithi026.stdout:Min bandwidth (MB/sec): 0 2023-09-23T21:11:53.487 INFO:tasks.radosbench.radosbench.0.smithi026.stdout:Average IOPS: 887 2023-09-23T21:11:53.487 INFO:tasks.radosbench.radosbench.0.smithi026.stdout:Stddev IOPS: 595.079 2023-09-23T21:11:53.487 INFO:tasks.radosbench.radosbench.0.smithi026.stdout:Max IOPS: 2366 2023-09-23T21:11:53.487 INFO:tasks.radosbench.radosbench.0.smithi026.stdout:Min IOPS: 0 2023-09-23T21:11:53.487 INFO:tasks.radosbench.radosbench.0.smithi026.stdout:Average Latency(s): 0.0180129 2023-09-23T21:11:53.487 INFO:tasks.radosbench.radosbench.0.smithi026.stdout:Stddev Latency(s): 0.0989498 2023-09-23T21:11:53.487 INFO:tasks.radosbench.radosbench.0.smithi026.stdout:Max latency(s): 4.6563 2023-09-23T21:11:53.487 INFO:tasks.radosbench.radosbench.0.smithi026.stdout:Min latency(s): 0.00120238 2023-09-23T21:11:53.488 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:11:53 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:11:53.489 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:11:53.490 INFO:tasks.radosbench.radosbench.0.smithi026.stdout:Cleaning up (deleting benchmark objects) 2023-09-23T21:11:53.559 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:11:53.564 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:11:53.564 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:11:53.665 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:11:53.737 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:11:53.823 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1: {} 2023-09-23T21:11:53.827 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:11:53.834 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:11:53.834 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:11:53.856 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2: {} 2023-09-23T21:11:53.889 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3: {} 2023-09-23T21:11:53.911 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4: {} 2023-09-23T21:11:53.935 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:11:53.944 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T21:11:53.974 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6: {} 2023-09-23T21:11:53.978 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:11:54.005 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T21:11:54.005 INFO:teuthology.orchestra.run.smithi026.stderr:Error ENXIO: problem getting command descriptions from osd.0 2023-09-23T21:11:54.005 INFO:teuthology.orchestra.run.smithi026.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T21:11:54.005 INFO:teuthology.orchestra.run.smithi026.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T21:11:54.005 INFO:teuthology.orchestra.run.smithi026.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T21:11:54.005 INFO:teuthology.orchestra.run.smithi026.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T21:11:54.005 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T21:11:54.006 INFO:teuthology.orchestra.run.smithi026.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T21:11:54.006 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T21:11:54.037 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:11:54.049 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:11:54.050 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:11:54.098 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:11:54.150 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:11:54.200 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:11:54.252 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:11:54.260 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:11:54.260 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:11:54.361 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:11:54.407 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:11:54.492 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:11:54 smithi026 ceph-mon[179688]: pgmap v1189: 119 pgs: 1 active+recovery_wait+degraded, 1 activating+degraded+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 21 active+undersized, 1 active+recovering+undersized+degraded+remapped, 8 active+undersized+degraded, 8 active+recovery_wait+undersized+degraded+remapped, 6 active+recovery_wait+degraded+remapped, 72 active+clean; 4.9 GiB data, 18 GiB used, 607 GiB / 626 GiB avail; 5.7 KiB/s rd, 26 MiB/s wr, 423 op/s; 65509/238668 objects degraded (27.448%); 15078/238668 objects misplaced (6.318%); 8.0 MiB/s, 128 objects/s recovering 2023-09-23T21:11:54.493 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:11:54 smithi026 ceph-mon[179688]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs inactive, 2 pgs peering) 2023-09-23T21:11:54.493 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:11:54 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/2161579782' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:11:54.493 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:11:54 smithi026 ceph-mon[175151]: pgmap v1189: 119 pgs: 1 active+recovery_wait+degraded, 1 activating+degraded+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 21 active+undersized, 1 active+recovering+undersized+degraded+remapped, 8 active+undersized+degraded, 8 active+recovery_wait+undersized+degraded+remapped, 6 active+recovery_wait+degraded+remapped, 72 active+clean; 4.9 GiB data, 18 GiB used, 607 GiB / 626 GiB avail; 5.7 KiB/s rd, 26 MiB/s wr, 423 op/s; 65509/238668 objects degraded (27.448%); 15078/238668 objects misplaced (6.318%); 8.0 MiB/s, 128 objects/s recovering 2023-09-23T21:11:54.493 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:11:54 smithi026 ceph-mon[175151]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs inactive, 2 pgs peering) 2023-09-23T21:11:54.493 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:11:54 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/2161579782' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:11:54.622 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:11:54 smithi070 ceph-mon[168282]: pgmap v1189: 119 pgs: 1 active+recovery_wait+degraded, 1 activating+degraded+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 21 active+undersized, 1 active+recovering+undersized+degraded+remapped, 8 active+undersized+degraded, 8 active+recovery_wait+undersized+degraded+remapped, 6 active+recovery_wait+degraded+remapped, 72 active+clean; 4.9 GiB data, 18 GiB used, 607 GiB / 626 GiB avail; 5.7 KiB/s rd, 26 MiB/s wr, 423 op/s; 65509/238668 objects degraded (27.448%); 15078/238668 objects misplaced (6.318%); 8.0 MiB/s, 128 objects/s recovering 2023-09-23T21:11:54.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:11:54 smithi070 ceph-mon[168282]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs inactive, 2 pgs peering) 2023-09-23T21:11:54.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:11:54 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/2161579782' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:11:54.661 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:11:54.666 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:11:54.667 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:11:54.697 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T21:11:54.697 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T21:11:54.768 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:11:54.825 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:11:54.855 INFO:teuthology.orchestra.run.smithi026.stdout:{ 2023-09-23T21:11:54.856 INFO:teuthology.orchestra.run.smithi026.stdout: "size": 20, 2023-09-23T21:11:54.856 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 600, 2023-09-23T21:11:54.856 INFO:teuthology.orchestra.run.smithi026.stdout: "ops": [ 2023-09-23T21:11:54.856 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:54.856 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35469.0:212417 8.a 8:5385c5a4:::benchmark_data_smithi026_301269_object101660:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e339)", 2023-09-23T21:11:54.856 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T21:05:21.284981+0000", 2023-09-23T21:11:54.856 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 393.56700193299997, 2023-09-23T21:11:54.856 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0517269170000001, 2023-09-23T21:11:54.856 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:11:54.856 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:11:54.857 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T21:11:54.857 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35469", 2023-09-23T21:11:54.857 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/4041424623", 2023-09-23T21:11:54.857 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 212417 2023-09-23T21:11:54.857 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:54.857 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:11:54.857 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:54.857 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:11:54.857 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:21.284981+0000", 2023-09-23T21:11:54.857 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:11:54.857 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:54.857 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:54.857 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:11:54.857 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:21.284981+0000", 2023-09-23T21:11:54.858 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:11:54.858 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:54.858 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:54.858 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:11:54.858 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:21.284979+0000", 2023-09-23T21:11:54.858 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999976 2023-09-23T21:11:54.858 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:54.858 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:54.858 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:11:54.858 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:21.284988+0000", 2023-09-23T21:11:54.858 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.0289999999999994e-06 2023-09-23T21:11:54.858 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:54.859 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:54.859 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:11:54.859 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:21.285002+0000", 2023-09-23T21:11:54.859 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.3541e-05 2023-09-23T21:11:54.859 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:54.859 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:54.859 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:11:54.859 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:21.285012+0000", 2023-09-23T21:11:54.859 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.8940000000000005e-06 2023-09-23T21:11:54.859 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:54.859 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:54.859 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:11:54.859 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:21.285284+0000", 2023-09-23T21:11:54.860 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00027241899999999998 2023-09-23T21:11:54.860 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:54.860 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:54.860 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T21:11:54.860 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:21.285294+0000", 2023-09-23T21:11:54.860 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.6469999999999993e-06 2023-09-23T21:11:54.860 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:54.860 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:54.860 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:11:54.860 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:22.321489+0000", 2023-09-23T21:11:54.860 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.036195727 2023-09-23T21:11:54.860 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:54.860 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:54.860 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:11:54.861 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:22.324294+0000", 2023-09-23T21:11:54.861 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0028048740000000002 2023-09-23T21:11:54.861 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:54.861 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:54.861 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 5", 2023-09-23T21:11:54.861 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:22.324413+0000", 2023-09-23T21:11:54.861 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000118613 2023-09-23T21:11:54.861 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:54.861 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:54.861 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:11:54.861 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:22.336370+0000", 2023-09-23T21:11:54.861 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.011957198 2023-09-23T21:11:54.861 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:54.862 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:54.862 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T21:11:54.862 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:22.336559+0000", 2023-09-23T21:11:54.862 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000188896 2023-09-23T21:11:54.862 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:54.862 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:54.862 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T21:11:54.862 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:22.336609+0000", 2023-09-23T21:11:54.862 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.9707999999999999e-05 2023-09-23T21:11:54.862 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:54.862 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:54.862 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T21:11:54.863 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:22.336708+0000", 2023-09-23T21:11:54.863 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.9535000000000002e-05 2023-09-23T21:11:54.863 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:11:54.863 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:11:54.863 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:11:54.863 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:54.863 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:54.863 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35469.0:212423 8.a 8:5e64cfdd:::benchmark_data_smithi026_301269_object101666:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e339)", 2023-09-23T21:11:54.863 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T21:05:21.293572+0000", 2023-09-23T21:11:54.863 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 393.55841120100001, 2023-09-23T21:11:54.863 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0445543349999999, 2023-09-23T21:11:54.863 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:11:54.864 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:11:54.864 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T21:11:54.864 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35469", 2023-09-23T21:11:54.864 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/4041424623", 2023-09-23T21:11:54.864 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 212423 2023-09-23T21:11:54.864 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:54.864 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:11:54.864 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:54.864 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:11:54.864 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:21.293572+0000", 2023-09-23T21:11:54.864 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:11:54.865 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:54.865 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:54.865 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:11:54.865 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:21.293572+0000", 2023-09-23T21:11:54.865 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:11:54.865 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:54.865 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:54.865 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:11:54.865 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:21.293570+0000", 2023-09-23T21:11:54.865 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T21:11:54.865 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:54.865 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:54.866 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:11:54.866 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:21.293579+0000", 2023-09-23T21:11:54.866 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.3359999999999995e-06 2023-09-23T21:11:54.866 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:54.866 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:54.866 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:11:54.866 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:21.293596+0000", 2023-09-23T21:11:54.866 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.6935e-05 2023-09-23T21:11:54.866 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:54.866 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:54.866 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:11:54.866 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:21.293604+0000", 2023-09-23T21:11:54.867 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.8150000000000006e-06 2023-09-23T21:11:54.867 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:54.867 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:54.867 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:11:54.867 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:21.293642+0000", 2023-09-23T21:11:54.867 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.8673000000000001e-05 2023-09-23T21:11:54.867 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:54.867 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:54.867 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T21:11:54.867 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:21.293649+0000", 2023-09-23T21:11:54.867 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.9700000000000002e-06 2023-09-23T21:11:54.867 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:54.867 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:54.867 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:11:54.868 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:22.329308+0000", 2023-09-23T21:11:54.868 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0356588680000001 2023-09-23T21:11:54.868 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:54.868 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:54.868 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:11:54.868 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:22.329433+0000", 2023-09-23T21:11:54.868 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000125318 2023-09-23T21:11:54.868 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:54.868 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:54.868 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 5", 2023-09-23T21:11:54.868 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:22.329532+0000", 2023-09-23T21:11:54.868 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.8507999999999999e-05 2023-09-23T21:11:54.868 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:54.868 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:54.869 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T21:11:54.869 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:22.337337+0000", 2023-09-23T21:11:54.869 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0078052490000000002 2023-09-23T21:11:54.869 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:54.869 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:54.869 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:11:54.869 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:22.337982+0000", 2023-09-23T21:11:54.869 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00064439800000000002 2023-09-23T21:11:54.869 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:54.869 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:54.869 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T21:11:54.869 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:22.338031+0000", 2023-09-23T21:11:54.869 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.9329000000000002e-05 2023-09-23T21:11:54.869 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:54.870 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:54.870 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T21:11:54.870 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:22.338126+0000", 2023-09-23T21:11:54.870 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.5598000000000002e-05 2023-09-23T21:11:54.870 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:11:54.870 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:11:54.870 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:11:54.870 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:54.870 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:54.870 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35469.0:212443 8.a 8:564426b3:::benchmark_data_smithi026_301269_object101686:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e339)", 2023-09-23T21:11:54.870 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T21:05:21.320675+0000", 2023-09-23T21:11:54.871 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 393.53130819, 2023-09-23T21:11:54.871 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0176459879999999, 2023-09-23T21:11:54.871 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:11:54.871 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:11:54.871 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T21:11:54.871 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35469", 2023-09-23T21:11:54.871 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/4041424623", 2023-09-23T21:11:54.871 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 212443 2023-09-23T21:11:54.871 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:54.871 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:11:54.871 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:54.871 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:11:54.871 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:21.320675+0000", 2023-09-23T21:11:54.872 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:11:54.872 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:54.872 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:54.872 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:11:54.872 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:21.320675+0000", 2023-09-23T21:11:54.872 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:11:54.872 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:54.872 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:54.872 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:11:54.872 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:21.320674+0000", 2023-09-23T21:11:54.872 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T21:11:54.872 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:54.872 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:54.873 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:11:54.873 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:21.320680+0000", 2023-09-23T21:11:54.873 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.6289999999999998e-06 2023-09-23T21:11:54.873 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:54.873 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:54.873 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:11:54.873 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:21.320692+0000", 2023-09-23T21:11:54.873 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.2221999999999999e-05 2023-09-23T21:11:54.873 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:54.873 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:54.873 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:11:54.873 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:21.320700+0000", 2023-09-23T21:11:54.873 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.9109999999999993e-06 2023-09-23T21:11:54.873 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:54.874 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:54.874 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:11:54.874 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:21.320739+0000", 2023-09-23T21:11:54.874 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.9517000000000001e-05 2023-09-23T21:11:54.874 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:54.874 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:54.874 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T21:11:54.874 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:21.320749+0000", 2023-09-23T21:11:54.874 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.4080000000000004e-06 2023-09-23T21:11:54.874 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:54.874 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:54.874 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:11:54.875 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:22.332451+0000", 2023-09-23T21:11:54.875 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0117019330000001 2023-09-23T21:11:54.875 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:54.875 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:54.875 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:11:54.875 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:22.332573+0000", 2023-09-23T21:11:54.875 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000122569 2023-09-23T21:11:54.875 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:54.875 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:54.875 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 5", 2023-09-23T21:11:54.875 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:22.332666+0000", 2023-09-23T21:11:54.875 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.3067999999999997e-05 2023-09-23T21:11:54.875 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:54.876 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:54.876 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:11:54.876 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:22.338171+0000", 2023-09-23T21:11:54.876 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0055052909999999998 2023-09-23T21:11:54.876 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:54.876 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:54.876 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T21:11:54.876 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:22.338182+0000", 2023-09-23T21:11:54.876 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0074e-05 2023-09-23T21:11:54.876 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:54.876 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:54.876 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T21:11:54.876 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:22.338234+0000", 2023-09-23T21:11:54.876 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.2725000000000001e-05 2023-09-23T21:11:54.876 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:54.877 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:54.877 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T21:11:54.877 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:05:22.338321+0000", 2023-09-23T21:11:54.877 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.6908000000000002e-05 2023-09-23T21:11:54.877 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:11:54.877 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:11:54.877 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:11:54.877 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:54.877 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:54.877 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.46328.0:11972 10.9 10:9118a5c9:::benchmark_data_smithi026_379838_object11971:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e427)", 2023-09-23T21:11:54.877 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T21:10:30.552674+0000", 2023-09-23T21:11:54.877 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 84.299309734000005, 2023-09-23T21:11:54.877 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.135766673, 2023-09-23T21:11:54.877 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:11:54.878 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "reached pg", 2023-09-23T21:11:54.878 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T21:11:54.878 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.46328", 2023-09-23T21:11:54.878 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/3748861181", 2023-09-23T21:11:54.878 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 11972 2023-09-23T21:11:54.878 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:54.878 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:11:54.878 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:54.878 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:11:54.878 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:10:30.552674+0000", 2023-09-23T21:11:54.878 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:11:54.878 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:54.879 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:54.879 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:11:54.879 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:10:30.552674+0000", 2023-09-23T21:11:54.879 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:11:54.879 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:54.879 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:54.879 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:11:54.879 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:10:30.552672+0000", 2023-09-23T21:11:54.879 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999981 2023-09-23T21:11:54.879 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:54.879 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:54.879 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:11:54.879 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:10:30.552724+0000", 2023-09-23T21:11:54.880 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.2654000000000001e-05 2023-09-23T21:11:54.880 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:54.880 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:54.880 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:11:54.880 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:10:30.754288+0000", 2023-09-23T21:11:54.880 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.20156365600000001 2023-09-23T21:11:54.880 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:54.880 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:54.880 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:11:54.880 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:10:30.754301+0000", 2023-09-23T21:11:54.880 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.3254e-05 2023-09-23T21:11:54.880 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:54.880 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:54.881 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:11:54.881 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:10:30.754456+0000", 2023-09-23T21:11:54.881 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000154807 2023-09-23T21:11:54.881 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:54.881 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:54.881 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:11:54.881 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:10:30.754642+0000", 2023-09-23T21:11:54.881 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000185508 2023-09-23T21:11:54.881 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:54.881 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:54.881 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 5,7", 2023-09-23T21:11:54.881 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:10:30.754808+0000", 2023-09-23T21:11:54.881 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00016601699999999999 2023-09-23T21:11:54.882 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:54.882 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:54.882 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:11:54.882 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:10:30.756892+0000", 2023-09-23T21:11:54.882 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0020849079999999999 2023-09-23T21:11:54.882 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:54.882 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:54.882 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T21:11:54.882 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:10:30.756916+0000", 2023-09-23T21:11:54.882 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.3941999999999999e-05 2023-09-23T21:11:54.882 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:54.882 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:54.883 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:11:54.883 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:10:30.760017+0000", 2023-09-23T21:11:54.883 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.003100298 2023-09-23T21:11:54.883 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:54.883 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:54.883 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T21:11:54.883 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:10:30.760029+0000", 2023-09-23T21:11:54.883 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.202e-05 2023-09-23T21:11:54.883 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:54.883 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:54.883 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:11:54.883 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:10:31.684961+0000", 2023-09-23T21:11:54.883 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.92493212999999996 2023-09-23T21:11:54.883 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:54.884 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:54.884 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for degraded object", 2023-09-23T21:11:54.884 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:10:31.685253+0000", 2023-09-23T21:11:54.884 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00029236099999999999 2023-09-23T21:11:54.884 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:54.884 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:54.884 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:11:54.884 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:10:31.688338+0000", 2023-09-23T21:11:54.884 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0030842949999999999 2023-09-23T21:11:54.884 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:54.884 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:54.885 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T21:11:54.885 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:10:31.688440+0000", 2023-09-23T21:11:54.885 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00010275199999999999 2023-09-23T21:11:54.885 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:11:54.885 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:11:54.885 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:11:54.885 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:54.885 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:54.885 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.46328.0:11990 10.9 10:9e1ea7b0:::benchmark_data_smithi026_379838_object11989:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e427)", 2023-09-23T21:11:54.885 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T21:10:30.758212+0000", 2023-09-23T21:11:54.885 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 84.093771329000006, 2023-09-23T21:11:54.885 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.93055809300000003, 2023-09-23T21:11:54.886 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:11:54.886 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:11:54.886 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T21:11:54.886 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.46328", 2023-09-23T21:11:54.886 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/3748861181", 2023-09-23T21:11:54.886 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 11990 2023-09-23T21:11:54.886 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:54.886 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:11:54.886 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:54.886 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:11:54.886 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:10:30.758212+0000", 2023-09-23T21:11:54.887 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:11:54.887 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:54.887 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:54.887 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:11:54.887 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:10:30.758212+0000", 2023-09-23T21:11:54.887 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:11:54.887 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:54.887 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:54.887 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:11:54.887 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:10:30.758210+0000", 2023-09-23T21:11:54.887 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T21:11:54.887 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:54.888 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:54.888 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:11:54.888 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:10:30.758237+0000", 2023-09-23T21:11:54.888 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.6860999999999999e-05 2023-09-23T21:11:54.888 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:54.888 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:54.888 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:11:54.888 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:10:30.758708+0000", 2023-09-23T21:11:54.888 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00047092199999999997 2023-09-23T21:11:54.888 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:54.888 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:54.888 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:11:54.888 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:10:30.758717+0000", 2023-09-23T21:11:54.889 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.7280000000000001e-06 2023-09-23T21:11:54.889 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:54.889 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:54.889 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:11:54.889 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:10:30.760057+0000", 2023-09-23T21:11:54.889 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0013395880000000001 2023-09-23T21:11:54.889 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:54.889 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:54.889 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T21:11:54.889 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:10:30.760067+0000", 2023-09-23T21:11:54.889 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.9280000000000005e-06 2023-09-23T21:11:54.889 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:54.889 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:54.889 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:11:54.890 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:10:31.685555+0000", 2023-09-23T21:11:54.890 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.92548829200000005 2023-09-23T21:11:54.890 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:54.890 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:54.890 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:11:54.890 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:10:31.685662+0000", 2023-09-23T21:11:54.890 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00010688000000000001 2023-09-23T21:11:54.890 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:54.890 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:54.890 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 5,7", 2023-09-23T21:11:54.890 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:10:31.685823+0000", 2023-09-23T21:11:54.890 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00016115699999999999 2023-09-23T21:11:54.890 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:54.891 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:54.891 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T21:11:54.891 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:10:31.688278+0000", 2023-09-23T21:11:54.891 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0024550650000000002 2023-09-23T21:11:54.891 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:54.891 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:54.891 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:11:54.891 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:10:31.688482+0000", 2023-09-23T21:11:54.891 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00020412199999999999 2023-09-23T21:11:54.891 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:54.891 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:54.892 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:11:54.892 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:10:31.688635+0000", 2023-09-23T21:11:54.892 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00015325899999999999 2023-09-23T21:11:54.892 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:54.892 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:54.892 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T21:11:54.892 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:10:31.688681+0000", 2023-09-23T21:11:54.892 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.5732999999999997e-05 2023-09-23T21:11:54.892 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:54.892 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:54.892 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T21:11:54.892 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:10:31.688770+0000", 2023-09-23T21:11:54.893 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.9072000000000004e-05 2023-09-23T21:11:54.893 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:11:54.893 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:11:54.893 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:11:54.893 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:54.893 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:54.893 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.46328.0:12014 10.9 10:901b719a:::benchmark_data_smithi026_379838_object12013:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e428)", 2023-09-23T21:11:54.893 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T21:10:30.775698+0000", 2023-09-23T21:11:54.893 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 84.076285404999993, 2023-09-23T21:11:54.893 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.91339289800000001, 2023-09-23T21:11:54.893 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:11:54.893 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:11:54.894 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T21:11:54.894 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.46328", 2023-09-23T21:11:54.894 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/3748861181", 2023-09-23T21:11:54.894 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 12014 2023-09-23T21:11:54.894 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:54.894 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:11:54.894 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:54.894 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:11:54.894 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:10:30.775698+0000", 2023-09-23T21:11:54.894 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:11:54.894 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:54.894 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:54.895 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:11:54.895 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:10:30.775698+0000", 2023-09-23T21:11:54.895 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:11:54.895 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:54.895 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:54.895 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:11:54.895 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:10:30.775697+0000", 2023-09-23T21:11:54.895 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T21:11:54.895 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:54.895 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:54.895 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:11:54.895 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:10:30.775722+0000", 2023-09-23T21:11:54.896 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.4882999999999999e-05 2023-09-23T21:11:54.896 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:54.896 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:54.896 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:11:54.896 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:10:30.775734+0000", 2023-09-23T21:11:54.896 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.2303000000000001e-05 2023-09-23T21:11:54.896 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:54.896 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:54.896 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:11:54.896 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:10:30.775742+0000", 2023-09-23T21:11:54.896 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.2719999999999997e-06 2023-09-23T21:11:54.896 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:54.896 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:54.896 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:11:54.897 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:10:30.775807+0000", 2023-09-23T21:11:54.897 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.4700000000000001e-05 2023-09-23T21:11:54.897 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:54.897 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:54.897 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T21:11:54.897 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:10:30.775820+0000", 2023-09-23T21:11:54.897 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.2713e-05 2023-09-23T21:11:54.897 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:54.897 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:54.897 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:11:54.897 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:10:31.686485+0000", 2023-09-23T21:11:54.897 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.91066531100000003 2023-09-23T21:11:54.897 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:54.898 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:54.898 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:11:54.898 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:10:31.686599+0000", 2023-09-23T21:11:54.898 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000114533 2023-09-23T21:11:54.898 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:54.898 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:54.898 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 5,7", 2023-09-23T21:11:54.898 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:10:31.686698+0000", 2023-09-23T21:11:54.898 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.8282000000000005e-05 2023-09-23T21:11:54.898 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:54.898 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:54.899 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T21:11:54.899 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:10:31.688291+0000", 2023-09-23T21:11:54.899 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0015937830000000001 2023-09-23T21:11:54.899 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:54.899 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:54.899 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:11:54.899 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:10:31.688817+0000", 2023-09-23T21:11:54.899 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00052526000000000005 2023-09-23T21:11:54.899 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:54.899 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:54.899 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:11:54.899 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:10:31.688874+0000", 2023-09-23T21:11:54.899 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.6866000000000001e-05 2023-09-23T21:11:54.900 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:54.900 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:54.900 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T21:11:54.900 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:10:31.688978+0000", 2023-09-23T21:11:54.900 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000104832 2023-09-23T21:11:54.900 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:54.900 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:54.900 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T21:11:54.900 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:10:31.689091+0000", 2023-09-23T21:11:54.900 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000112452 2023-09-23T21:11:54.901 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:11:54.901 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:11:54.901 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:11:54.901 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:54.901 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:54.901 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.46328.0:12017 10.9 10:902bb015:::benchmark_data_smithi026_379838_object12016:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e428)", 2023-09-23T21:11:54.901 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T21:10:30.777179+0000", 2023-09-23T21:11:54.901 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 84.074804517000004, 2023-09-23T21:11:54.901 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.91218087599999997, 2023-09-23T21:11:54.901 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:11:54.901 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:11:54.901 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T21:11:54.901 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.46328", 2023-09-23T21:11:54.902 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/3748861181", 2023-09-23T21:11:54.902 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 12017 2023-09-23T21:11:54.902 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:54.902 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:11:54.902 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:54.902 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:11:54.902 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:10:30.777179+0000", 2023-09-23T21:11:54.902 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:11:54.902 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:54.902 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:54.902 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:11:54.902 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:10:30.777179+0000", 2023-09-23T21:11:54.902 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:11:54.903 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:54.903 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:54.903 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:11:54.903 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:10:30.777178+0000", 2023-09-23T21:11:54.903 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T21:11:54.903 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:54.903 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:54.903 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:11:54.903 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:10:30.777202+0000", 2023-09-23T21:11:54.903 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.4437999999999999e-05 2023-09-23T21:11:54.903 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:54.903 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:54.903 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:11:54.903 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:10:30.777214+0000", 2023-09-23T21:11:54.904 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.2132e-05 2023-09-23T21:11:54.904 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:54.904 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:54.904 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:11:54.904 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:10:30.777223+0000", 2023-09-23T21:11:54.904 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.0329999999999997e-06 2023-09-23T21:11:54.904 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:54.904 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:54.904 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:11:54.904 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:10:30.777279+0000", 2023-09-23T21:11:54.904 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.533e-05 2023-09-23T21:11:54.904 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:54.905 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:54.905 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T21:11:54.905 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:10:30.777289+0000", 2023-09-23T21:11:54.905 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0349e-05 2023-09-23T21:11:54.905 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:54.905 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:54.905 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:11:54.905 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:10:31.687155+0000", 2023-09-23T21:11:54.905 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.90986638900000005 2023-09-23T21:11:54.905 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:54.905 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:54.906 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:11:54.906 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:10:31.687300+0000", 2023-09-23T21:11:54.906 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00014477700000000001 2023-09-23T21:11:54.906 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:54.906 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:54.906 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 5,7", 2023-09-23T21:11:54.906 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:10:31.687489+0000", 2023-09-23T21:11:54.906 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00018891000000000001 2023-09-23T21:11:54.906 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:54.906 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:54.906 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T21:11:54.906 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:10:31.688492+0000", 2023-09-23T21:11:54.906 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0010032629999999999 2023-09-23T21:11:54.907 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:54.907 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:54.907 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:11:54.907 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:10:31.689162+0000", 2023-09-23T21:11:54.907 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00067002299999999995 2023-09-23T21:11:54.907 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:54.907 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:54.907 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:11:54.907 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:10:31.689226+0000", 2023-09-23T21:11:54.907 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.3718000000000002e-05 2023-09-23T21:11:54.907 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:54.907 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:54.908 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T21:11:54.908 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:10:31.689275+0000", 2023-09-23T21:11:54.908 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.8779999999999997e-05 2023-09-23T21:11:54.908 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:54.908 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:54.908 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T21:11:54.908 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:10:31.689360+0000", 2023-09-23T21:11:54.908 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.4963000000000006e-05 2023-09-23T21:11:54.908 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:11:54.908 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:11:54.908 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:11:54.908 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:54.909 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:54.909 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.46328.0:12041 10.9 10:9d932200:::benchmark_data_smithi026_379838_object12040:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e428)", 2023-09-23T21:11:54.909 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T21:10:30.791292+0000", 2023-09-23T21:11:54.909 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 84.060691484000003, 2023-09-23T21:11:54.909 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.90054857700000002, 2023-09-23T21:11:54.909 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:11:54.909 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:11:54.909 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T21:11:54.909 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.46328", 2023-09-23T21:11:54.909 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/3748861181", 2023-09-23T21:11:54.909 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 12041 2023-09-23T21:11:54.909 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:54.909 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:11:54.909 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:54.910 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:11:54.910 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:10:30.791292+0000", 2023-09-23T21:11:54.910 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:11:54.910 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:54.910 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:54.910 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:11:54.910 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:10:30.791292+0000", 2023-09-23T21:11:54.910 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:11:54.910 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:54.910 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:54.910 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:11:54.910 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:10:30.791291+0000", 2023-09-23T21:11:54.910 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T21:11:54.911 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:54.911 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:54.911 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:11:54.911 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:10:30.791310+0000", 2023-09-23T21:11:54.911 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.8573999999999999e-05 2023-09-23T21:11:54.911 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:54.911 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:54.911 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:11:54.911 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:10:30.791319+0000", 2023-09-23T21:11:54.911 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.5899999999999997e-06 2023-09-23T21:11:54.911 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:54.911 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:54.911 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:11:54.911 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:10:30.791326+0000", 2023-09-23T21:11:54.912 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.72e-06 2023-09-23T21:11:54.912 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:54.912 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:54.912 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:11:54.912 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:10:30.791379+0000", 2023-09-23T21:11:54.912 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.3424000000000002e-05 2023-09-23T21:11:54.912 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:54.912 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:54.912 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T21:11:54.912 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:10:30.791395+0000", 2023-09-23T21:11:54.912 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.5634e-05 2023-09-23T21:11:54.912 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:54.912 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:54.912 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:11:54.913 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:10:31.689405+0000", 2023-09-23T21:11:54.913 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.89801024799999996 2023-09-23T21:11:54.913 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:54.913 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:54.913 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:11:54.913 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:10:31.689537+0000", 2023-09-23T21:11:54.913 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00013122000000000001 2023-09-23T21:11:54.913 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:54.913 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:54.913 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 5,7", 2023-09-23T21:11:54.913 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:10:31.689673+0000", 2023-09-23T21:11:54.913 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00013614399999999999 2023-09-23T21:11:54.913 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:54.914 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:54.914 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:11:54.914 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:10:31.691688+0000", 2023-09-23T21:11:54.914 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0020155699999999999 2023-09-23T21:11:54.914 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:54.914 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:54.914 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:11:54.914 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:10:31.691726+0000", 2023-09-23T21:11:54.914 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.7589000000000002e-05 2023-09-23T21:11:54.914 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:54.914 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:54.915 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T21:11:54.915 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:10:31.691731+0000", 2023-09-23T21:11:54.915 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.327e-06 2023-09-23T21:11:54.915 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:54.915 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:54.915 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T21:11:54.915 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:10:31.691760+0000", 2023-09-23T21:11:54.915 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.9062999999999999e-05 2023-09-23T21:11:54.915 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:54.915 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:54.915 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T21:11:54.915 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:10:31.691840+0000", 2023-09-23T21:11:54.915 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.0164e-05 2023-09-23T21:11:54.916 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:11:54.916 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:11:54.916 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:11:54.916 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:54.916 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:54.916 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.46328.0:12064 10.9 10:99cfecd5:::benchmark_data_smithi026_379838_object12063:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e428)", 2023-09-23T21:11:54.916 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T21:10:30.850676+0000", 2023-09-23T21:11:54.916 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 84.001307202000007, 2023-09-23T21:11:54.916 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.843835424, 2023-09-23T21:11:54.916 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:11:54.916 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:11:54.916 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T21:11:54.916 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.46328", 2023-09-23T21:11:54.916 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/3748861181", 2023-09-23T21:11:54.917 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 12064 2023-09-23T21:11:54.917 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:54.917 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:11:54.917 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:54.917 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:11:54.917 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:10:30.850676+0000", 2023-09-23T21:11:54.917 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:11:54.917 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:54.917 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:54.917 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:11:54.917 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:10:30.850676+0000", 2023-09-23T21:11:54.917 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:11:54.917 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:54.918 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:54.918 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:11:54.918 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:10:30.850675+0000", 2023-09-23T21:11:54.918 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T21:11:54.918 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:54.918 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:54.918 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:11:54.918 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:10:30.850698+0000", 2023-09-23T21:11:54.918 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.3074999999999998e-05 2023-09-23T21:11:54.918 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:54.918 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:54.918 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:11:54.918 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:10:30.850713+0000", 2023-09-23T21:11:54.919 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.5264000000000002e-05 2023-09-23T21:11:54.919 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:54.919 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:54.919 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:11:54.919 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:10:30.850721+0000", 2023-09-23T21:11:54.919 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.072e-06 2023-09-23T21:11:54.919 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:54.919 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:54.919 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:11:54.919 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:10:30.850782+0000", 2023-09-23T21:11:54.919 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.1173000000000005e-05 2023-09-23T21:11:54.919 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:54.920 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:54.920 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T21:11:54.920 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:10:30.850795+0000", 2023-09-23T21:11:54.920 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.2506e-05 2023-09-23T21:11:54.920 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:54.920 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:54.920 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:11:54.920 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:10:31.690258+0000", 2023-09-23T21:11:54.920 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.83946299899999999 2023-09-23T21:11:54.920 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:54.920 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:54.920 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:11:54.920 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:10:31.690386+0000", 2023-09-23T21:11:54.921 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000128202 2023-09-23T21:11:54.921 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:54.921 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:54.921 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 5,7", 2023-09-23T21:11:54.921 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:10:31.690554+0000", 2023-09-23T21:11:54.921 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00016841600000000001 2023-09-23T21:11:54.921 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:54.921 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:54.921 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T21:11:54.921 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:10:31.691855+0000", 2023-09-23T21:11:54.921 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.001300743 2023-09-23T21:11:54.921 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:54.921 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:54.921 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:11:54.922 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:10:31.691887+0000", 2023-09-23T21:11:54.922 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.1572999999999997e-05 2023-09-23T21:11:54.922 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:54.922 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:54.922 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:11:54.922 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:10:31.694355+0000", 2023-09-23T21:11:54.922 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0024681389999999998 2023-09-23T21:11:54.922 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:54.922 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:54.922 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T21:11:54.922 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:10:31.694404+0000", 2023-09-23T21:11:54.922 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.9126999999999999e-05 2023-09-23T21:11:54.922 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:54.923 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:54.923 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T21:11:54.923 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:10:31.694512+0000", 2023-09-23T21:11:54.923 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000107481 2023-09-23T21:11:54.923 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:11:54.923 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:11:54.923 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:11:54.923 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:54.923 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:54.923 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.46328.0:12066 10.9 10:9c9a9051:::benchmark_data_smithi026_379838_object12065:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e428)", 2023-09-23T21:11:54.923 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T21:10:30.852009+0000", 2023-09-23T21:11:54.923 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 83.999974753999993, 2023-09-23T21:11:54.923 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.84270132900000005, 2023-09-23T21:11:54.924 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:11:54.924 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:11:54.924 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T21:11:54.924 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.46328", 2023-09-23T21:11:54.924 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/3748861181", 2023-09-23T21:11:54.924 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 12066 2023-09-23T21:11:54.924 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:54.924 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:11:54.924 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:54.924 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:11:54.924 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:10:30.852009+0000", 2023-09-23T21:11:54.924 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:11:54.924 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:54.924 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:54.925 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:11:54.925 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:10:30.852009+0000", 2023-09-23T21:11:54.925 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:11:54.925 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:54.925 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:54.925 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:11:54.925 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:10:30.852007+0000", 2023-09-23T21:11:54.925 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T21:11:54.925 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:54.925 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:54.925 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:11:54.925 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:10:30.852031+0000", 2023-09-23T21:11:54.925 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.4071999999999999e-05 2023-09-23T21:11:54.926 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:54.926 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:54.926 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:11:54.926 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:10:30.852050+0000", 2023-09-23T21:11:54.926 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.8474999999999998e-05 2023-09-23T21:11:54.926 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:54.926 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:54.926 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:11:54.926 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:10:30.852057+0000", 2023-09-23T21:11:54.926 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.8620000000000003e-06 2023-09-23T21:11:54.926 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:54.926 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:54.926 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:11:54.927 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:10:30.852113+0000", 2023-09-23T21:11:54.927 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.5856999999999997e-05 2023-09-23T21:11:54.927 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:54.927 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:54.927 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T21:11:54.927 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:10:30.852125+0000", 2023-09-23T21:11:54.927 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1688999999999999e-05 2023-09-23T21:11:54.927 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:54.927 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:54.927 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:11:54.927 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:10:31.691057+0000", 2023-09-23T21:11:54.927 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.83893214400000005 2023-09-23T21:11:54.927 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:54.928 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:54.928 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:11:54.928 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:10:31.691152+0000", 2023-09-23T21:11:54.928 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.5029000000000001e-05 2023-09-23T21:11:54.928 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:54.928 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:54.928 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 5,7", 2023-09-23T21:11:54.928 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:10:31.691239+0000", 2023-09-23T21:11:54.928 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.6961999999999999e-05 2023-09-23T21:11:54.928 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:54.928 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:54.928 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T21:11:54.928 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:10:31.691894+0000", 2023-09-23T21:11:54.928 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00065505800000000003 2023-09-23T21:11:54.929 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:54.929 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:54.929 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:11:54.929 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:10:31.694296+0000", 2023-09-23T21:11:54.929 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0024017230000000001 2023-09-23T21:11:54.929 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:54.929 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:54.929 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:11:54.929 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:10:31.694586+0000", 2023-09-23T21:11:54.929 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000289776 2023-09-23T21:11:54.929 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:54.929 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:54.930 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T21:11:54.930 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:10:31.694632+0000", 2023-09-23T21:11:54.930 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.6425999999999999e-05 2023-09-23T21:11:54.930 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:54.930 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:54.930 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T21:11:54.930 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:10:31.694710+0000", 2023-09-23T21:11:54.930 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.7788999999999997e-05 2023-09-23T21:11:54.930 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:11:54.930 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:11:54.930 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:11:54.930 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:54.930 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:54.931 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.46328.0:12083 10.9 10:910f9ce9:::benchmark_data_smithi026_379838_object12082:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e428)", 2023-09-23T21:11:54.931 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T21:10:30.862065+0000", 2023-09-23T21:11:54.931 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 83.989918697999997, 2023-09-23T21:11:54.931 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.83285090100000003, 2023-09-23T21:11:54.931 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:11:54.931 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:11:54.931 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T21:11:54.931 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.46328", 2023-09-23T21:11:54.931 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/3748861181", 2023-09-23T21:11:54.931 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 12083 2023-09-23T21:11:54.931 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:54.931 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:11:54.931 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:54.932 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:11:54.932 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:10:30.862065+0000", 2023-09-23T21:11:54.932 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:11:54.932 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:54.932 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:54.932 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:11:54.932 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:10:30.862065+0000", 2023-09-23T21:11:54.932 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:11:54.932 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:54.932 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:54.932 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:11:54.932 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:10:30.862063+0000", 2023-09-23T21:11:54.933 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T21:11:54.933 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:54.933 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:54.933 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:11:54.933 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:10:30.862085+0000", 2023-09-23T21:11:54.933 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.1926e-05 2023-09-23T21:11:54.933 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:54.933 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:54.933 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:11:54.933 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:10:30.862102+0000", 2023-09-23T21:11:54.934 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.7091999999999998e-05 2023-09-23T21:11:54.934 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:54.934 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:54.934 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:11:54.934 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:10:30.862111+0000", 2023-09-23T21:11:54.934 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.1730000000000001e-06 2023-09-23T21:11:54.934 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:54.934 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:54.934 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:11:54.934 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:10:30.862213+0000", 2023-09-23T21:11:54.934 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000101332 2023-09-23T21:11:54.934 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:54.935 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:54.935 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T21:11:54.935 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:10:30.862235+0000", 2023-09-23T21:11:54.935 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.1795999999999999e-05 2023-09-23T21:11:54.935 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:54.935 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:54.935 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:11:54.935 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:10:31.691919+0000", 2023-09-23T21:11:54.935 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.82968449700000002 2023-09-23T21:11:54.935 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:54.935 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:54.935 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:11:54.936 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:10:31.692040+0000", 2023-09-23T21:11:54.936 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000120419 2023-09-23T21:11:54.936 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:54.936 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:54.936 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 5,7", 2023-09-23T21:11:54.936 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:10:31.692177+0000", 2023-09-23T21:11:54.936 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000137102 2023-09-23T21:11:54.936 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:54.936 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:54.936 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T21:11:54.937 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:10:31.694227+0000", 2023-09-23T21:11:54.937 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0020501730000000002 2023-09-23T21:11:54.937 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:54.937 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:54.937 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:11:54.937 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:10:31.694559+0000", 2023-09-23T21:11:54.937 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00033181999999999998 2023-09-23T21:11:54.937 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:54.937 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:54.937 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:11:54.937 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:10:31.694792+0000", 2023-09-23T21:11:54.937 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000233019 2023-09-23T21:11:54.937 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:54.938 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:54.938 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T21:11:54.938 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:10:31.694821+0000", 2023-09-23T21:11:54.938 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.8873999999999998e-05 2023-09-23T21:11:54.938 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:54.938 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:54.938 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T21:11:54.938 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:10:31.694916+0000", 2023-09-23T21:11:54.938 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.5024000000000004e-05 2023-09-23T21:11:54.938 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:11:54.938 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:11:54.938 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:11:54.939 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:54.939 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:54.939 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.46328.0:12085 10.9 10:9f6bd2d3:::benchmark_data_smithi026_379838_object12084:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e428)", 2023-09-23T21:11:54.939 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T21:10:30.862834+0000", 2023-09-23T21:11:54.939 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 83.989149217999994, 2023-09-23T21:11:54.939 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.83231058800000002, 2023-09-23T21:11:54.939 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:11:54.939 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:11:54.939 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T21:11:54.939 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.46328", 2023-09-23T21:11:54.939 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/3748861181", 2023-09-23T21:11:54.939 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 12085 2023-09-23T21:11:54.939 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:54.940 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:11:54.940 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:54.940 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:11:54.940 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:10:30.862834+0000", 2023-09-23T21:11:54.940 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:11:54.940 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:54.940 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:54.940 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:11:54.940 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:10:30.862834+0000", 2023-09-23T21:11:54.940 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:11:54.940 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:54.940 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:54.940 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:11:54.941 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:10:30.862833+0000", 2023-09-23T21:11:54.941 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.999999 2023-09-23T21:11:54.941 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:54.941 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:54.941 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:11:54.941 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:10:30.862856+0000", 2023-09-23T21:11:54.941 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.2841999999999999e-05 2023-09-23T21:11:54.941 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:54.941 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:54.941 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:11:54.941 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:10:30.862872+0000", 2023-09-23T21:11:54.941 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.6141999999999999e-05 2023-09-23T21:11:54.941 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:54.941 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:54.942 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:11:54.942 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:10:30.862881+0000", 2023-09-23T21:11:54.942 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.8780000000000004e-06 2023-09-23T21:11:54.942 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:54.942 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:54.942 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:11:54.942 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:10:30.862943+0000", 2023-09-23T21:11:54.942 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.1873999999999997e-05 2023-09-23T21:11:54.942 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:54.942 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:54.942 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T21:11:54.942 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:10:30.862959+0000", 2023-09-23T21:11:54.943 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.6412000000000001e-05 2023-09-23T21:11:54.943 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:54.943 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:54.943 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:11:54.943 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:10:31.692665+0000", 2023-09-23T21:11:54.943 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.82970556299999998 2023-09-23T21:11:54.943 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:54.943 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:54.943 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:11:54.943 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:10:31.692785+0000", 2023-09-23T21:11:54.943 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00012055100000000001 2023-09-23T21:11:54.943 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:54.943 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:54.943 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 5,7", 2023-09-23T21:11:54.944 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:10:31.692947+0000", 2023-09-23T21:11:54.944 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00016191299999999999 2023-09-23T21:11:54.944 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:54.944 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:54.944 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T21:11:54.944 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:10:31.694528+0000", 2023-09-23T21:11:54.944 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00158091 2023-09-23T21:11:54.944 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:54.944 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:54.944 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:11:54.944 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:10:31.694756+0000", 2023-09-23T21:11:54.944 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00022785899999999999 2023-09-23T21:11:54.944 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:54.945 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:54.945 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:11:54.945 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:10:31.694993+0000", 2023-09-23T21:11:54.945 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00023648699999999999 2023-09-23T21:11:54.945 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:54.945 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:54.945 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T21:11:54.945 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:10:31.695051+0000", 2023-09-23T21:11:54.945 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.8227999999999999e-05 2023-09-23T21:11:54.945 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:54.945 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:54.946 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T21:11:54.946 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:10:31.695145+0000", 2023-09-23T21:11:54.946 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.3948000000000002e-05 2023-09-23T21:11:54.946 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:11:54.946 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:11:54.946 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:11:54.946 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:54.946 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:54.946 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.46328.0:12097 10.9 10:9bc5accb:::benchmark_data_smithi026_379838_object12096:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e428)", 2023-09-23T21:11:54.946 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T21:10:30.869358+0000", 2023-09-23T21:11:54.946 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 83.982625686000006, 2023-09-23T21:11:54.946 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.82595051799999997, 2023-09-23T21:11:54.947 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:11:54.947 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:11:54.947 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T21:11:54.947 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.46328", 2023-09-23T21:11:54.947 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/3748861181", 2023-09-23T21:11:54.947 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 12097 2023-09-23T21:11:54.947 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:54.947 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:11:54.947 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:54.947 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:11:54.947 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:10:30.869358+0000", 2023-09-23T21:11:54.947 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:11:54.948 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:54.948 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:54.948 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:11:54.948 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:10:30.869358+0000", 2023-09-23T21:11:54.948 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:11:54.948 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:54.948 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:54.948 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:11:54.948 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:10:30.869356+0000", 2023-09-23T21:11:54.948 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.999999 2023-09-23T21:11:54.948 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:54.948 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:54.948 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:11:54.948 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:10:30.869386+0000", 2023-09-23T21:11:54.949 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.9754e-05 2023-09-23T21:11:54.949 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:54.949 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:54.949 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:11:54.949 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:10:30.869404+0000", 2023-09-23T21:11:54.949 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.7414999999999998e-05 2023-09-23T21:11:54.949 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:54.949 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:54.949 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:11:54.949 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:10:30.869411+0000", 2023-09-23T21:11:54.949 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.1269999999999997e-06 2023-09-23T21:11:54.949 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:54.950 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:54.950 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:11:54.950 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:10:30.869502+0000", 2023-09-23T21:11:54.950 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.1398999999999997e-05 2023-09-23T21:11:54.950 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:54.950 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:54.950 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T21:11:54.950 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:10:30.869511+0000", 2023-09-23T21:11:54.950 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.8610000000000004e-06 2023-09-23T21:11:54.950 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:54.950 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:54.950 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:11:54.951 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:10:31.693497+0000", 2023-09-23T21:11:54.951 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.82398618199999996 2023-09-23T21:11:54.951 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:54.951 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:54.951 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:11:54.951 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:10:31.693621+0000", 2023-09-23T21:11:54.951 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00012345300000000001 2023-09-23T21:11:54.951 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:54.951 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:54.951 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 5,7", 2023-09-23T21:11:54.951 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:10:31.693769+0000", 2023-09-23T21:11:54.951 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000148703 2023-09-23T21:11:54.952 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:54.952 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:54.952 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T21:11:54.952 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:10:31.694763+0000", 2023-09-23T21:11:54.952 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0009940839999999999 2023-09-23T21:11:54.952 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:54.952 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:54.952 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:11:54.952 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:10:31.694957+0000", 2023-09-23T21:11:54.952 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00019378099999999999 2023-09-23T21:11:54.952 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:54.952 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:54.953 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:11:54.953 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:10:31.695188+0000", 2023-09-23T21:11:54.953 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000230969 2023-09-23T21:11:54.953 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:54.953 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:54.953 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T21:11:54.953 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:10:31.695225+0000", 2023-09-23T21:11:54.953 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.7079000000000002e-05 2023-09-23T21:11:54.953 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:54.953 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:54.953 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T21:11:54.953 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:10:31.695308+0000", 2023-09-23T21:11:54.954 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.2892999999999994e-05 2023-09-23T21:11:54.954 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:11:54.954 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:11:54.954 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:11:54.954 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:54.954 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:54.954 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.46328.0:12110 10.9 10:9be78f62:::benchmark_data_smithi026_379838_object12109:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e428)", 2023-09-23T21:11:54.954 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T21:10:30.876223+0000", 2023-09-23T21:11:54.954 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 83.975760039999997, 2023-09-23T21:11:54.954 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.82485409700000001, 2023-09-23T21:11:54.954 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:11:54.954 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:11:54.955 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T21:11:54.955 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.46328", 2023-09-23T21:11:54.955 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/3748861181", 2023-09-23T21:11:54.955 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 12110 2023-09-23T21:11:54.955 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:54.955 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:11:54.955 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:54.955 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:11:54.955 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:10:30.876223+0000", 2023-09-23T21:11:54.955 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:11:54.955 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:54.955 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:54.956 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:11:54.956 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:10:30.876223+0000", 2023-09-23T21:11:54.956 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:11:54.956 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:54.956 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:54.956 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:11:54.956 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:10:30.876222+0000", 2023-09-23T21:11:54.956 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T21:11:54.956 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:54.956 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:54.956 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:11:54.956 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:10:30.876240+0000", 2023-09-23T21:11:54.956 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.7952999999999999e-05 2023-09-23T21:11:54.957 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:54.957 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:54.957 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:11:54.957 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:10:30.876256+0000", 2023-09-23T21:11:54.957 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.6053999999999999e-05 2023-09-23T21:11:54.957 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:54.957 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:54.957 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:11:54.957 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:10:30.876266+0000", 2023-09-23T21:11:54.957 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.961e-06 2023-09-23T21:11:54.958 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:54.958 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:54.958 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:11:54.958 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:10:30.876329+0000", 2023-09-23T21:11:54.958 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.2459000000000006e-05 2023-09-23T21:11:54.958 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:54.958 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:54.958 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T21:11:54.958 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:10:30.876342+0000", 2023-09-23T21:11:54.958 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.3264999999999999e-05 2023-09-23T21:11:54.958 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:54.959 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:54.959 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:11:54.959 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:10:31.695339+0000", 2023-09-23T21:11:54.959 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.81899745800000001 2023-09-23T21:11:54.959 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:54.959 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:54.959 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:11:54.959 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:10:31.695486+0000", 2023-09-23T21:11:54.959 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000147205 2023-09-23T21:11:54.959 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:54.959 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:54.960 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 5,7", 2023-09-23T21:11:54.960 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:10:31.695620+0000", 2023-09-23T21:11:54.960 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00013331899999999999 2023-09-23T21:11:54.960 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:54.960 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:54.960 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T21:11:54.960 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:10:31.697687+0000", 2023-09-23T21:11:54.960 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0020672529999999998 2023-09-23T21:11:54.960 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:54.960 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:54.960 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:11:54.960 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:10:31.697729+0000", 2023-09-23T21:11:54.960 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.1499999999999999e-05 2023-09-23T21:11:54.961 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:54.961 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:54.961 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:11:54.961 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:10:31.700922+0000", 2023-09-23T21:11:54.961 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.003193432 2023-09-23T21:11:54.961 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:54.961 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:54.961 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T21:11:54.961 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:10:31.700977+0000", 2023-09-23T21:11:54.961 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.5226999999999998e-05 2023-09-23T21:11:54.961 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:54.961 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:54.961 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T21:11:54.962 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:10:31.701077+0000", 2023-09-23T21:11:54.962 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00010022900000000001 2023-09-23T21:11:54.962 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:11:54.962 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:11:54.962 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:11:54.962 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:54.962 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:54.962 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.46328.0:12142 10.9 10:9a171347:::benchmark_data_smithi026_379838_object12141:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e428)", 2023-09-23T21:11:54.962 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T21:10:30.893637+0000", 2023-09-23T21:11:54.962 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 83.958346453999994, 2023-09-23T21:11:54.962 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.80775173899999997, 2023-09-23T21:11:54.962 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:11:54.962 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:11:54.963 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T21:11:54.963 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.46328", 2023-09-23T21:11:54.963 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/3748861181", 2023-09-23T21:11:54.963 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 12142 2023-09-23T21:11:54.963 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:54.963 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:11:54.963 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:54.963 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:11:54.963 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:10:30.893637+0000", 2023-09-23T21:11:54.963 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:11:54.963 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:54.963 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:54.963 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:11:54.963 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:10:30.893637+0000", 2023-09-23T21:11:54.964 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:11:54.964 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:54.964 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:54.964 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:11:54.964 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:10:30.893636+0000", 2023-09-23T21:11:54.964 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T21:11:54.964 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:54.964 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:54.964 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:11:54.964 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:10:30.893656+0000", 2023-09-23T21:11:54.964 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.0075999999999999e-05 2023-09-23T21:11:54.964 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:54.964 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:54.965 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:11:54.965 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:10:30.893671+0000", 2023-09-23T21:11:54.965 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.5389000000000001e-05 2023-09-23T21:11:54.965 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:54.965 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:54.965 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:11:54.965 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:10:30.893680+0000", 2023-09-23T21:11:54.965 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.4069999999999993e-06 2023-09-23T21:11:54.965 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:54.965 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:54.965 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:11:54.965 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:10:30.893749+0000", 2023-09-23T21:11:54.965 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.9147999999999996e-05 2023-09-23T21:11:54.965 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:54.966 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:54.966 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T21:11:54.966 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:10:30.893767+0000", 2023-09-23T21:11:54.966 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.7889000000000001e-05 2023-09-23T21:11:54.966 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:54.966 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:54.966 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:11:54.966 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:10:31.696174+0000", 2023-09-23T21:11:54.966 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.80240772199999999 2023-09-23T21:11:54.966 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:54.966 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:54.966 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:11:54.967 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:10:31.696298+0000", 2023-09-23T21:11:54.967 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00012364599999999999 2023-09-23T21:11:54.967 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:54.967 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:54.967 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 5,7", 2023-09-23T21:11:54.967 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:10:31.696447+0000", 2023-09-23T21:11:54.967 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00014930100000000001 2023-09-23T21:11:54.967 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:54.967 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:54.967 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T21:11:54.967 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:10:31.697735+0000", 2023-09-23T21:11:54.968 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0012874290000000001 2023-09-23T21:11:54.968 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:54.968 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:54.968 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:11:54.968 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:10:31.699351+0000", 2023-09-23T21:11:54.968 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0016163849999999999 2023-09-23T21:11:54.968 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:54.968 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:54.968 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:11:54.968 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:10:31.701122+0000", 2023-09-23T21:11:54.969 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0017709220000000001 2023-09-23T21:11:54.969 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:54.969 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:54.969 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T21:11:54.969 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:10:31.701302+0000", 2023-09-23T21:11:54.969 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000179777 2023-09-23T21:11:54.969 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:54.969 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:54.969 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T21:11:54.969 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:10:31.701389+0000", 2023-09-23T21:11:54.969 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.6885000000000006e-05 2023-09-23T21:11:54.969 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:11:54.970 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:11:54.970 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:11:54.970 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:54.970 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:54.970 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.46328.0:12144 10.9 10:9c2848d8:::benchmark_data_smithi026_379838_object12143:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e428)", 2023-09-23T21:11:54.970 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T21:10:30.894589+0000", 2023-09-23T21:11:54.970 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 83.957394671000003, 2023-09-23T21:11:54.970 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.80861499299999995, 2023-09-23T21:11:54.970 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:11:54.970 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:11:54.970 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T21:11:54.970 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.46328", 2023-09-23T21:11:54.971 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/3748861181", 2023-09-23T21:11:54.971 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 12144 2023-09-23T21:11:54.971 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:54.971 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:11:54.971 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:54.971 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:11:54.971 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:10:30.894589+0000", 2023-09-23T21:11:54.971 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:11:54.971 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:54.971 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:54.971 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:11:54.971 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:10:30.894589+0000", 2023-09-23T21:11:54.971 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:11:54.972 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:54.972 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:54.972 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:11:54.972 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:10:30.894588+0000", 2023-09-23T21:11:54.972 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.999999 2023-09-23T21:11:54.972 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:54.972 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:54.972 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:11:54.972 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:10:30.894613+0000", 2023-09-23T21:11:54.972 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.4926000000000002e-05 2023-09-23T21:11:54.972 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:54.972 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:54.972 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:11:54.972 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:10:30.894625+0000", 2023-09-23T21:11:54.973 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1936999999999999e-05 2023-09-23T21:11:54.973 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:54.973 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:54.973 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:11:54.973 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:10:30.894633+0000", 2023-09-23T21:11:54.973 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.0099999999999995e-06 2023-09-23T21:11:54.973 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:54.973 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:54.973 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:11:54.973 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:10:30.894681+0000", 2023-09-23T21:11:54.973 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.7855000000000002e-05 2023-09-23T21:11:54.973 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:54.973 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:54.974 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T21:11:54.974 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:10:30.894688+0000", 2023-09-23T21:11:54.974 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.7330000000000003e-06 2023-09-23T21:11:54.974 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:54.974 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:54.974 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:11:54.974 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:10:31.696942+0000", 2023-09-23T21:11:54.974 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.80225412399999996 2023-09-23T21:11:54.974 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:54.974 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:54.974 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:11:54.974 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:10:31.697083+0000", 2023-09-23T21:11:54.974 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000140588 2023-09-23T21:11:54.975 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:54.975 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:54.975 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 5,7", 2023-09-23T21:11:54.975 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:10:31.697224+0000", 2023-09-23T21:11:54.975 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00014085700000000001 2023-09-23T21:11:54.975 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:54.975 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:54.975 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:11:54.975 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:10:31.701435+0000", 2023-09-23T21:11:54.975 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0042109959999999998 2023-09-23T21:11:54.975 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:54.975 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:54.975 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:11:54.976 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:10:31.701534+0000", 2023-09-23T21:11:54.976 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.8870999999999996e-05 2023-09-23T21:11:54.976 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:54.976 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:54.976 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T21:11:54.976 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:10:31.703103+0000", 2023-09-23T21:11:54.976 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0015688399999999999 2023-09-23T21:11:54.976 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:54.976 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:54.976 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T21:11:54.976 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:10:31.703147+0000", 2023-09-23T21:11:54.976 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.4166999999999999e-05 2023-09-23T21:11:54.977 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:54.977 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:54.977 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T21:11:54.977 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:10:31.703204+0000", 2023-09-23T21:11:54.977 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.6975000000000003e-05 2023-09-23T21:11:54.977 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:11:54.977 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:11:54.977 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:11:54.977 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:54.977 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:54.977 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.46328.0:52889 10.e 10:729c6d6c:::benchmark_data_smithi026_379838_object52888:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e444)", 2023-09-23T21:11:54.977 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T21:11:08.756518+0000", 2023-09-23T21:11:54.977 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 46.095465425, 2023-09-23T21:11:54.977 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.165692414, 2023-09-23T21:11:54.978 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:11:54.978 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:11:54.978 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T21:11:54.978 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.46328", 2023-09-23T21:11:54.978 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/3748861181", 2023-09-23T21:11:54.978 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 52889 2023-09-23T21:11:54.978 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:54.978 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:11:54.978 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:54.978 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:11:54.978 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:11:08.756518+0000", 2023-09-23T21:11:54.978 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:11:54.979 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:54.979 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:54.979 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:11:54.979 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:11:08.756518+0000", 2023-09-23T21:11:54.979 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:11:54.979 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:54.979 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:54.979 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:11:54.979 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:11:08.756516+0000", 2023-09-23T21:11:54.979 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999981 2023-09-23T21:11:54.979 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:54.979 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:54.979 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:11:54.980 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:11:08.756547+0000", 2023-09-23T21:11:54.980 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.1328999999999998e-05 2023-09-23T21:11:54.980 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:54.980 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:54.980 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:11:54.980 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:11:08.756561+0000", 2023-09-23T21:11:54.980 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.3161e-05 2023-09-23T21:11:54.980 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:54.980 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:54.980 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:11:54.980 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:11:08.756572+0000", 2023-09-23T21:11:54.980 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1625e-05 2023-09-23T21:11:54.980 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:54.980 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:54.981 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:11:54.981 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:11:08.756904+0000", 2023-09-23T21:11:54.981 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00033169500000000002 2023-09-23T21:11:54.981 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:54.981 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:54.981 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T21:11:54.981 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:11:08.756918+0000", 2023-09-23T21:11:54.981 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.3752e-05 2023-09-23T21:11:54.981 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:54.981 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:54.981 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:11:54.981 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:11:09.908767+0000", 2023-09-23T21:11:54.981 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1518492060000001 2023-09-23T21:11:54.981 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:54.982 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:54.982 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:11:54.982 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:11:09.908906+0000", 2023-09-23T21:11:54.982 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00013930999999999999 2023-09-23T21:11:54.982 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:54.982 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:54.982 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 4,5", 2023-09-23T21:11:54.982 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:11:09.909071+0000", 2023-09-23T21:11:54.982 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00016437 2023-09-23T21:11:54.982 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:54.982 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:54.983 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:11:54.983 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:11:09.915692+0000", 2023-09-23T21:11:54.983 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0066215859999999996 2023-09-23T21:11:54.983 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:54.983 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:54.983 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:11:54.983 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:11:09.915735+0000", 2023-09-23T21:11:54.983 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.2787999999999998e-05 2023-09-23T21:11:54.983 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:54.983 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:54.983 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T21:11:54.983 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:11:09.919777+0000", 2023-09-23T21:11:54.983 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0040422000000000001 2023-09-23T21:11:54.983 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:54.984 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:54.984 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T21:11:54.984 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:11:09.919835+0000", 2023-09-23T21:11:54.984 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.7528000000000002e-05 2023-09-23T21:11:54.984 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:54.984 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:54.984 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T21:11:54.984 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:11:09.922210+0000", 2023-09-23T21:11:54.984 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0023757019999999999 2023-09-23T21:11:54.984 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:11:54.984 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:11:54.985 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:11:54.985 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:54.985 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:54.985 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.46328.0:52897 10.e 10:7a5f33c8:::benchmark_data_smithi026_379838_object52896:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e444)", 2023-09-23T21:11:54.985 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T21:11:08.756659+0000", 2023-09-23T21:11:54.985 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 46.095323839000002, 2023-09-23T21:11:54.985 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1730620599999999, 2023-09-23T21:11:54.985 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:11:54.985 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:11:54.985 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T21:11:54.985 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.46328", 2023-09-23T21:11:54.985 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/3748861181", 2023-09-23T21:11:54.986 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 52897 2023-09-23T21:11:54.986 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:54.986 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:11:54.986 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:54.986 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:11:54.986 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:11:08.756659+0000", 2023-09-23T21:11:54.986 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:11:54.986 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:54.986 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:54.986 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:11:54.986 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:11:08.756659+0000", 2023-09-23T21:11:54.987 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:11:54.987 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:54.987 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:54.987 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:11:54.987 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:11:08.756659+0000", 2023-09-23T21:11:54.987 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T21:11:54.987 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:54.987 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:54.987 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:11:54.987 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:11:08.756684+0000", 2023-09-23T21:11:54.987 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.5196999999999999e-05 2023-09-23T21:11:54.987 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:54.987 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:54.988 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:11:54.988 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:11:08.756695+0000", 2023-09-23T21:11:54.988 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0797e-05 2023-09-23T21:11:54.988 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:54.988 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:54.988 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:11:54.988 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:11:08.756704+0000", 2023-09-23T21:11:54.988 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.9449999999999999e-06 2023-09-23T21:11:54.988 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:54.988 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:54.988 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:11:54.988 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:11:08.756952+0000", 2023-09-23T21:11:54.988 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00024845200000000003 2023-09-23T21:11:54.989 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:54.989 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:54.989 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T21:11:54.989 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:11:08.756963+0000", 2023-09-23T21:11:54.989 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0265000000000001e-05 2023-09-23T21:11:54.989 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:54.989 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:54.989 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:11:54.989 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:11:09.913098+0000", 2023-09-23T21:11:54.989 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1561355529999999 2023-09-23T21:11:54.989 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:54.989 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:54.989 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:11:54.990 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:11:09.913288+0000", 2023-09-23T21:11:54.990 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00018984500000000001 2023-09-23T21:11:54.990 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:54.990 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:54.990 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 4,5", 2023-09-23T21:11:54.990 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:11:09.913591+0000", 2023-09-23T21:11:54.990 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00030295499999999999 2023-09-23T21:11:54.990 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:54.990 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:54.990 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:11:54.990 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:11:09.922304+0000", 2023-09-23T21:11:54.990 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0087127899999999998 2023-09-23T21:11:54.991 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:54.991 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:54.991 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:11:54.991 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:11:09.927322+0000", 2023-09-23T21:11:54.991 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.005018649 2023-09-23T21:11:54.991 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:54.991 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:54.991 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T21:11:54.991 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:11:09.927340+0000", 2023-09-23T21:11:54.991 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.7771e-05 2023-09-23T21:11:54.991 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:54.991 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:54.991 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T21:11:54.992 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:11:09.927401+0000", 2023-09-23T21:11:54.992 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.0587000000000001e-05 2023-09-23T21:11:54.992 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:54.992 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:54.992 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T21:11:54.992 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:11:09.929722+0000", 2023-09-23T21:11:54.992 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0023208700000000001 2023-09-23T21:11:54.992 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:11:54.992 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:11:54.992 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:11:54.992 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:54.993 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:54.993 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.46328.0:52900 10.e 10:78038d51:::benchmark_data_smithi026_379838_object52899:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e444)", 2023-09-23T21:11:54.993 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T21:11:08.756712+0000", 2023-09-23T21:11:54.993 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 46.095271122, 2023-09-23T21:11:54.993 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1751611849999999, 2023-09-23T21:11:54.993 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:11:54.993 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:11:54.993 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T21:11:54.993 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.46328", 2023-09-23T21:11:54.993 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/3748861181", 2023-09-23T21:11:54.993 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 52900 2023-09-23T21:11:54.993 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:54.993 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:11:54.994 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:54.994 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:11:54.994 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:11:08.756712+0000", 2023-09-23T21:11:54.994 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:11:54.994 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:54.994 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:54.994 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:11:54.994 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:11:08.756712+0000", 2023-09-23T21:11:54.994 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:11:54.994 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:54.994 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:54.994 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:11:54.995 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:11:08.756711+0000", 2023-09-23T21:11:54.995 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.999999 2023-09-23T21:11:54.995 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:54.995 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:54.995 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:11:54.995 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:11:08.756737+0000", 2023-09-23T21:11:54.995 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.5412999999999999e-05 2023-09-23T21:11:54.995 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:54.995 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:54.995 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:11:54.995 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:11:08.756747+0000", 2023-09-23T21:11:54.996 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.6460000000000005e-06 2023-09-23T21:11:54.996 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:54.996 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:54.996 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:11:54.996 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:11:08.756754+0000", 2023-09-23T21:11:54.996 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.435e-06 2023-09-23T21:11:54.996 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:54.996 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:54.996 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:11:54.996 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:11:08.756994+0000", 2023-09-23T21:11:54.996 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000239643 2023-09-23T21:11:54.996 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:54.997 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:54.997 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T21:11:54.997 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:11:08.757004+0000", 2023-09-23T21:11:54.997 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0473e-05 2023-09-23T21:11:54.997 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:54.997 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:54.997 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:11:54.997 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:11:09.915846+0000", 2023-09-23T21:11:54.997 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1588421680000001 2023-09-23T21:11:54.997 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:54.997 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:54.997 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:11:54.997 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:11:09.916007+0000", 2023-09-23T21:11:54.998 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00016045600000000001 2023-09-23T21:11:54.998 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:54.998 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:54.998 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 4,5", 2023-09-23T21:11:54.998 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:11:09.916214+0000", 2023-09-23T21:11:54.998 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000206825 2023-09-23T21:11:54.998 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:54.998 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:54.998 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:11:54.998 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:11:09.922355+0000", 2023-09-23T21:11:54.998 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0061417590000000001 2023-09-23T21:11:54.998 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:54.998 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:54.999 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T21:11:54.999 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:11:09.929733+0000", 2023-09-23T21:11:54.999 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0073777219999999998 2023-09-23T21:11:54.999 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:54.999 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:54.999 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:11:54.999 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:11:09.929790+0000", 2023-09-23T21:11:54.999 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.6552000000000003e-05 2023-09-23T21:11:54.999 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:54.999 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:54.999 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T21:11:54.999 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:11:09.929859+0000", 2023-09-23T21:11:54.999 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.9441000000000001e-05 2023-09-23T21:11:54.999 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:55.000 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:55.000 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T21:11:55.000 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:11:09.931873+0000", 2023-09-23T21:11:55.000 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0020144 2023-09-23T21:11:55.000 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:11:55.000 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:11:55.000 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:11:55.000 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:55.000 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:55.000 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.46328.0:52916 10.e 10:7de9f19e:::benchmark_data_smithi026_379838_object52915:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e444)", 2023-09-23T21:11:55.000 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T21:11:08.756761+0000", 2023-09-23T21:11:55.000 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 46.095222337000003, 2023-09-23T21:11:55.000 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1771335249999999, 2023-09-23T21:11:55.000 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:11:55.001 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:11:55.001 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T21:11:55.001 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.46328", 2023-09-23T21:11:55.001 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/3748861181", 2023-09-23T21:11:55.001 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 52916 2023-09-23T21:11:55.001 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:55.001 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:11:55.001 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:55.001 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:11:55.001 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:11:08.756761+0000", 2023-09-23T21:11:55.001 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:11:55.001 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:55.001 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:55.001 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:11:55.002 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:11:08.756761+0000", 2023-09-23T21:11:55.002 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:11:55.002 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:55.002 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:55.002 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:11:55.002 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:11:08.756761+0000", 2023-09-23T21:11:55.002 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T21:11:55.002 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:55.002 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:55.002 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:11:55.002 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:11:08.756785+0000", 2023-09-23T21:11:55.002 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.4335999999999999e-05 2023-09-23T21:11:55.003 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:55.003 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:55.003 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:11:55.003 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:11:08.756794+0000", 2023-09-23T21:11:55.003 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.3039999999999994e-06 2023-09-23T21:11:55.003 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:55.003 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:55.003 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:11:55.003 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:11:08.756802+0000", 2023-09-23T21:11:55.003 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.0949999999999996e-06 2023-09-23T21:11:55.003 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:55.003 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:55.003 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:11:55.004 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:11:08.757054+0000", 2023-09-23T21:11:55.004 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00025205699999999998 2023-09-23T21:11:55.004 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:55.004 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:55.004 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T21:11:55.004 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:11:08.757070+0000", 2023-09-23T21:11:55.004 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.5373999999999999e-05 2023-09-23T21:11:55.004 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:55.004 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:55.004 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:11:55.004 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:11:09.917924+0000", 2023-09-23T21:11:55.004 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.160854163 2023-09-23T21:11:55.004 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:55.004 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:55.005 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:11:55.005 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:11:09.918077+0000", 2023-09-23T21:11:55.005 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000152693 2023-09-23T21:11:55.005 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:55.005 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:55.005 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 4,5", 2023-09-23T21:11:55.005 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:11:09.918210+0000", 2023-09-23T21:11:55.005 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000133017 2023-09-23T21:11:55.005 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:55.005 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:55.005 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:11:55.005 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:11:09.922406+0000", 2023-09-23T21:11:55.005 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0041964080000000004 2023-09-23T21:11:55.006 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:55.006 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:55.006 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T21:11:55.006 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:11:09.929742+0000", 2023-09-23T21:11:55.006 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0073364249999999997 2023-09-23T21:11:55.006 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:55.006 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:55.006 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:11:55.006 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:11:09.931977+0000", 2023-09-23T21:11:55.006 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0022349309999999999 2023-09-23T21:11:55.006 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:55.006 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:55.007 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T21:11:55.007 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:11:09.932033+0000", 2023-09-23T21:11:55.007 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.5231e-05 2023-09-23T21:11:55.007 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:11:55.007 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:11:55.007 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T21:11:55.007 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:11:09.933895+0000", 2023-09-23T21:11:55.007 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.001861936 2023-09-23T21:11:55.007 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:11:55.007 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:11:55.007 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:11:55.007 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:11:55.007 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:11:55.007 INFO:teuthology.orchestra.run.smithi026.stdout:} 2023-09-23T21:11:55.178 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:11:55.181 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:11:55.182 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:11:55.282 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:11:55.320 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:11:55.504 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:11:55.507 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:11:55.507 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:11:55.540 DEBUG:teuthology.orchestra.run.smithi026:> sudo pkill -f 'journalctl -f -n 0 -u ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7@osd.0.service' 2023-09-23T21:11:55.608 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:11:55.612 INFO:journalctl@ceph.osd.0.smithi026.stdout:Sep 23 21:11:55 smithi026 systemd[1]: ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7@osd.0.service: Succeeded. 2023-09-23T21:11:55.612 INFO:journalctl@ceph.osd.0.smithi026.stdout:Sep 23 21:11:55 smithi026 systemd[1]: Stopped Ceph osd.0 for b3f8e4ec-5a50-11ee-8db4-212e2dc638e7. 2023-09-23T21:11:55.633 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-09-23T21:11:55.633 INFO:tasks.cephadm.osd.0:Stopped osd.0 2023-09-23T21:11:55.655 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:11:55.715 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:11:55.718 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:11:55.719 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:11:55.820 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:11:55.847 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:11:55.885 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:11:55.890 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:11:55.890 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:11:55.893 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_ops_in_flight 2023-09-23T21:11:55.991 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:11:56.027 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:11:56.076 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:11:56.079 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:11:56.080 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:11:56.180 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:11:56.218 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:11:56.239 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:11:56 smithi070 ceph-mon[168282]: pgmap v1190: 119 pgs: 1 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 21 active+undersized, 1 active+recovering+undersized+degraded+remapped, 8 active+undersized+degraded, 8 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+degraded+remapped, 6 active+recovery_wait+degraded+remapped, 72 active+clean; 4.9 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 16 MiB/s wr, 257 op/s; 65653/239499 objects degraded (27.413%); 15077/239499 objects misplaced (6.295%); 6.2 MiB/s, 100 objects/s recovering 2023-09-23T21:11:56.239 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:11:56 smithi070 ceph-mon[168282]: Health check update: Degraded data redundancy: 65653/239499 objects degraded (27.413%), 26 pgs degraded (PG_DEGRADED) 2023-09-23T21:11:56.265 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:11:56.271 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:11:56.272 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:11:56.372 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:11:56.449 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:11:56.526 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:11:56.537 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:11:56.538 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:11:56.631 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:11:56 smithi026 ceph-mon[175151]: pgmap v1190: 119 pgs: 1 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 21 active+undersized, 1 active+recovering+undersized+degraded+remapped, 8 active+undersized+degraded, 8 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+degraded+remapped, 6 active+recovery_wait+degraded+remapped, 72 active+clean; 4.9 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 16 MiB/s wr, 257 op/s; 65653/239499 objects degraded (27.413%); 15077/239499 objects misplaced (6.295%); 6.2 MiB/s, 100 objects/s recovering 2023-09-23T21:11:56.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:11:56 smithi026 ceph-mon[175151]: Health check update: Degraded data redundancy: 65653/239499 objects degraded (27.413%), 26 pgs degraded (PG_DEGRADED) 2023-09-23T21:11:56.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:11:56 smithi026 ceph-mon[179688]: pgmap v1190: 119 pgs: 1 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 21 active+undersized, 1 active+recovering+undersized+degraded+remapped, 8 active+undersized+degraded, 8 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+degraded+remapped, 6 active+recovery_wait+degraded+remapped, 72 active+clean; 4.9 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 16 MiB/s wr, 257 op/s; 65653/239499 objects degraded (27.413%); 15077/239499 objects misplaced (6.295%); 6.2 MiB/s, 100 objects/s recovering 2023-09-23T21:11:56.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:11:56 smithi026 ceph-mon[179688]: Health check update: Degraded data redundancy: 65653/239499 objects degraded (27.413%), 26 pgs degraded (PG_DEGRADED) 2023-09-23T21:11:56.639 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:11:56.659 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T21:11:56.674 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:11:56.699 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T21:11:56.715 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:11:56.718 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:11:56.719 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:11:56.819 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:11:56.855 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:11:56.901 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:11:56.909 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:11:56.909 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:11:57.010 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:11:57.064 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:11:57.156 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:11:57.162 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:11:57.162 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:11:57.263 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:11:57.264 INFO:teuthology.orchestra.run.smithi026.stderr:noscrub is set 2023-09-23T21:11:57.294 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:11:57.304 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:11:57.353 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:11:57.356 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:11:57.356 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:11:57.457 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:11:57.489 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:11:57 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/2812931074' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T21:11:57.490 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:11:57 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/2812931074' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T21:11:57.490 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:11:57.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:11:57 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/2812931074' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T21:11:58.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:11:58 smithi070 ceph-mon[168282]: pgmap v1191: 119 pgs: 1 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 21 active+undersized, 1 active+recovering+undersized+degraded+remapped, 8 active+undersized+degraded, 8 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+degraded+remapped, 6 active+recovery_wait+degraded+remapped, 72 active+clean; 4.9 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 14 MiB/s wr, 255 op/s; 65269/239331 objects degraded (27.271%); 15077/239331 objects misplaced (6.300%); 7.4 MiB/s, 118 objects/s recovering 2023-09-23T21:11:58.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:11:58 smithi070 ceph-mon[168282]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:11:58.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:11:58 smithi070 ceph-mon[168282]: failed to encode map e468 with expected crc 2023-09-23T21:11:58.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:11:58 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/2812931074' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T21:11:58.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:11:58 smithi070 ceph-mon[168282]: osdmap e468: 8 total, 7 up, 7 in 2023-09-23T21:11:58.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:11:58 smithi070 ceph-mon[168282]: failed to encode map e468 with expected crc 2023-09-23T21:11:58.465 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:11:58 smithi026 ceph-mon[179688]: pgmap v1191: 119 pgs: 1 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 21 active+undersized, 1 active+recovering+undersized+degraded+remapped, 8 active+undersized+degraded, 8 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+degraded+remapped, 6 active+recovery_wait+degraded+remapped, 72 active+clean; 4.9 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 14 MiB/s wr, 255 op/s; 65269/239331 objects degraded (27.271%); 15077/239331 objects misplaced (6.300%); 7.4 MiB/s, 118 objects/s recovering 2023-09-23T21:11:58.466 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:11:58 smithi026 ceph-mon[179688]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:11:58.466 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:11:58 smithi026 ceph-mon[179688]: failed to encode map e468 with expected crc 2023-09-23T21:11:58.466 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:11:58 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/2812931074' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T21:11:58.466 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:11:58 smithi026 ceph-mon[179688]: osdmap e468: 8 total, 7 up, 7 in 2023-09-23T21:11:58.466 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:11:58 smithi026 ceph-mon[179688]: failed to encode map e468 with expected crc 2023-09-23T21:11:58.466 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:11:58 smithi026 ceph-mon[175151]: pgmap v1191: 119 pgs: 1 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 21 active+undersized, 1 active+recovering+undersized+degraded+remapped, 8 active+undersized+degraded, 8 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+degraded+remapped, 6 active+recovery_wait+degraded+remapped, 72 active+clean; 4.9 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 14 MiB/s wr, 255 op/s; 65269/239331 objects degraded (27.271%); 15077/239331 objects misplaced (6.300%); 7.4 MiB/s, 118 objects/s recovering 2023-09-23T21:11:58.466 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:11:58 smithi026 ceph-mon[175151]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:11:58.466 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:11:58 smithi026 ceph-mon[175151]: failed to encode map e468 with expected crc 2023-09-23T21:11:58.467 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:11:58 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/2812931074' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T21:11:58.467 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:11:58 smithi026 ceph-mon[175151]: osdmap e468: 8 total, 7 up, 7 in 2023-09-23T21:11:58.467 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:11:58 smithi026 ceph-mon[175151]: failed to encode map e468 with expected crc 2023-09-23T21:11:58.488 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:11:58.492 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:11:58.492 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:11:58.593 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:11:58.638 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:11:58.848 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:11:58.853 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:11:58.853 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:11:58.869 INFO:teuthology.orchestra.run.smithi070.stdout:{ 2023-09-23T21:11:58.869 INFO:teuthology.orchestra.run.smithi070.stdout: "ops": [ 2023-09-23T21:11:58.869 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:11:58.870 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.46328.0:81405 10.4 10:264be6f7:::benchmark_data_smithi026_379838_object1448:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e468)", 2023-09-23T21:11:58.870 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:11:58.847615+0000", 2023-09-23T21:11:58.870 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 0.019708099999999999, 2023-09-23T21:11:58.870 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.019740623999999998, 2023-09-23T21:11:58.870 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:11:58.870 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "waiting for sub ops", 2023-09-23T21:11:58.870 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:11:58.870 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.46328", 2023-09-23T21:11:58.871 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/3748861181", 2023-09-23T21:11:58.871 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 81405 2023-09-23T21:11:58.871 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:11:58.871 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:11:58.871 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:11:58.871 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:11:58.871 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:58.847615+0000", 2023-09-23T21:11:58.871 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:11:58.871 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:11:58.872 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:11:58.872 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:11:58.872 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:58.847615+0000", 2023-09-23T21:11:58.872 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:11:58.872 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:11:58.872 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:11:58.872 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:11:58.872 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:58.847613+0000", 2023-09-23T21:11:58.873 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T21:11:58.873 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:11:58.873 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:11:58.873 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:11:58.873 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:58.847620+0000", 2023-09-23T21:11:58.873 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.674e-06 2023-09-23T21:11:58.873 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:11:58.873 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:11:58.873 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:11:58.874 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:58.847631+0000", 2023-09-23T21:11:58.874 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.0725e-05 2023-09-23T21:11:58.874 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:11:58.874 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:11:58.874 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:11:58.874 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:58.847637+0000", 2023-09-23T21:11:58.874 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.993e-06 2023-09-23T21:11:58.874 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:11:58.875 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:11:58.875 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:11:58.875 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:58.848310+0000", 2023-09-23T21:11:58.875 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00067305000000000002 2023-09-23T21:11:58.875 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:11:58.875 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:11:58.875 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:11:58.875 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:58.853128+0000", 2023-09-23T21:11:58.876 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.004817928 2023-09-23T21:11:58.876 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:11:58.876 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:11:58.876 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 1,3", 2023-09-23T21:11:58.876 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:58.858848+0000", 2023-09-23T21:11:58.876 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0057202370000000004 2023-09-23T21:11:58.876 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:11:58.876 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:11:58.876 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:11:58.877 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:58.865504+0000", 2023-09-23T21:11:58.877 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0066566220000000001 2023-09-23T21:11:58.877 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:11:58.877 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:11:58.877 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:11:58.877 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:58.866007+0000", 2023-09-23T21:11:58.877 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00050287499999999996 2023-09-23T21:11:58.877 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:11:58.877 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:11:58.877 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:11:58.877 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:58.867263+0000", 2023-09-23T21:11:58.878 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.001255241 2023-09-23T21:11:58.878 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:11:58.878 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:11:58.878 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:11:58.878 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:11:58.878 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:11:58.878 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.46328.0:81389 10.1 10:80b364c7:::benchmark_data_smithi026_379838_object1432:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e468)", 2023-09-23T21:11:58.878 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:11:58.806805+0000", 2023-09-23T21:11:58.878 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 0.060517697000000002, 2023-09-23T21:11:58.879 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.060656969999999998, 2023-09-23T21:11:58.879 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:11:58.879 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "delayed", 2023-09-23T21:11:58.879 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:11:58.879 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.46328", 2023-09-23T21:11:58.879 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/3748861181", 2023-09-23T21:11:58.879 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 81389 2023-09-23T21:11:58.879 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:11:58.880 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:11:58.880 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:11:58.880 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:11:58.880 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:58.806805+0000", 2023-09-23T21:11:58.880 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:11:58.880 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:11:58.880 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:11:58.881 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:11:58.881 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:58.806805+0000", 2023-09-23T21:11:58.881 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:11:58.881 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:11:58.881 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:11:58.881 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:11:58.882 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:58.806804+0000", 2023-09-23T21:11:58.882 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.999999 2023-09-23T21:11:58.882 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:11:58.882 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:11:58.882 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:11:58.882 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:58.806809+0000", 2023-09-23T21:11:58.883 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.425e-06 2023-09-23T21:11:58.883 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:11:58.883 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:11:58.883 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:11:58.883 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:58.806817+0000", 2023-09-23T21:11:58.883 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.5520000000000005e-06 2023-09-23T21:11:58.883 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:11:58.883 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:11:58.883 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:11:58.883 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:58.806822+0000", 2023-09-23T21:11:58.884 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.6770000000000003e-06 2023-09-23T21:11:58.884 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:11:58.884 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:11:58.884 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:11:58.884 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:58.832128+0000", 2023-09-23T21:11:58.884 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.025306227000000001 2023-09-23T21:11:58.884 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:11:58.884 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:11:58.884 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for missing object", 2023-09-23T21:11:58.884 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:58.836692+0000", 2023-09-23T21:11:58.884 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0045642579999999999 2023-09-23T21:11:58.885 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:11:58.885 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:11:58.885 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:11:58.885 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:11:58.885 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:11:58.885 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.46328.0:81407 10.6 10.84625626 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e468)", 2023-09-23T21:11:58.885 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:11:58.848455+0000", 2023-09-23T21:11:58.885 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 0.018867993, 2023-09-23T21:11:58.885 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.019074023999999998, 2023-09-23T21:11:58.885 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:11:58.886 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "queued for pg", 2023-09-23T21:11:58.886 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:11:58.886 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.46328", 2023-09-23T21:11:58.886 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/3748861181", 2023-09-23T21:11:58.886 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 81407 2023-09-23T21:11:58.886 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:11:58.886 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:11:58.886 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:11:58.886 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:11:58.886 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:58.848455+0000", 2023-09-23T21:11:58.887 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:11:58.887 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:11:58.887 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:11:58.887 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:11:58.887 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:58.848455+0000", 2023-09-23T21:11:58.887 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:11:58.887 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:11:58.887 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:11:58.887 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:11:58.887 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:58.848454+0000", 2023-09-23T21:11:58.887 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.999999 2023-09-23T21:11:58.888 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:11:58.888 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:11:58.888 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:11:58.888 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:58.848462+0000", 2023-09-23T21:11:58.888 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.8909999999999995e-06 2023-09-23T21:11:58.888 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:11:58.888 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:11:58.888 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:11:58.888 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:58.848473+0000", 2023-09-23T21:11:58.888 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.1068e-05 2023-09-23T21:11:58.889 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:11:58.889 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:11:58.889 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:11:58.889 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:58.848478+0000", 2023-09-23T21:11:58.889 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.6359999999999999e-06 2023-09-23T21:11:58.889 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:11:58.889 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:11:58.889 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:11:58.889 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:11:58.889 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:11:58.890 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "MOSDPGPush(10.b 468/466 [PushOp(10:de333d53:::benchmark_data_smithi026_379838_object1449:head, version: 422'110, data_included: [0~65536], data_size: 65536, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(10:de333d53:::benchmark_data_smithi026_379838_object1449:head@422'110, size: 65536, copy_subset: [0~65536], clone_subset: {}, snapset: 0=[]:{}, object_exist: 0), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:65536, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false, error:false))])", 2023-09-23T21:11:58.890 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:11:58.852284+0000", 2023-09-23T21:11:58.890 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 0.015039327999999999, 2023-09-23T21:11:58.890 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.015324346000000001, 2023-09-23T21:11:58.890 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:11:58.890 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "queued for pg", 2023-09-23T21:11:58.890 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:11:58.890 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:11:58.890 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:11:58.890 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:58.852284+0000", 2023-09-23T21:11:58.891 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:11:58.891 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:11:58.891 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:11:58.891 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:11:58.891 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:58.852284+0000", 2023-09-23T21:11:58.891 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:11:58.891 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:11:58.891 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:11:58.891 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:11:58.891 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:58.852283+0000", 2023-09-23T21:11:58.892 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.999999 2023-09-23T21:11:58.892 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:11:58.892 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:11:58.892 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:11:58.892 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:58.852366+0000", 2023-09-23T21:11:58.892 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.3317000000000002e-05 2023-09-23T21:11:58.892 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:11:58.892 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:11:58.892 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:11:58.892 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:58.852386+0000", 2023-09-23T21:11:58.892 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.9936999999999999e-05 2023-09-23T21:11:58.893 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:11:58.893 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:11:58.893 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:11:58.893 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:58.852398+0000", 2023-09-23T21:11:58.893 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.2269999999999999e-05 2023-09-23T21:11:58.893 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:11:58.893 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:11:58.893 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:11:58.893 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:11:58.893 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:11:58.894 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_repop(client.46328.0:81408 10.0 e468/453 10:0024c53e:::benchmark_data_smithi026_379838_object1451:head v 468'3892, mlcod=0'0)", 2023-09-23T21:11:58.894 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:11:58.852411+0000", 2023-09-23T21:11:58.894 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 0.014911944999999999, 2023-09-23T21:11:58.894 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.015246776, 2023-09-23T21:11:58.894 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:11:58.894 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "started", 2023-09-23T21:11:58.894 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:11:58.894 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:11:58.894 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:11:58.895 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:58.852411+0000", 2023-09-23T21:11:58.895 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:11:58.895 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:11:58.895 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:11:58.895 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:11:58.895 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:58.852411+0000", 2023-09-23T21:11:58.895 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:11:58.895 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:11:58.895 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:11:58.895 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:11:58.895 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:58.852410+0000", 2023-09-23T21:11:58.896 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.999999 2023-09-23T21:11:58.896 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:11:58.896 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:11:58.896 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:11:58.896 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:58.852416+0000", 2023-09-23T21:11:58.896 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.3580000000000002e-06 2023-09-23T21:11:58.896 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:11:58.896 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:11:58.896 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:11:58.897 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:58.852422+0000", 2023-09-23T21:11:58.897 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.652e-06 2023-09-23T21:11:58.897 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:11:58.897 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:11:58.897 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:11:58.897 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:58.852427+0000", 2023-09-23T21:11:58.897 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.2889999999999999e-06 2023-09-23T21:11:58.897 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:11:58.897 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:11:58.897 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:11:58.897 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:58.852568+0000", 2023-09-23T21:11:58.898 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000141194 2023-09-23T21:11:58.898 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:11:58.898 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:11:58.898 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:11:58.898 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:58.852595+0000", 2023-09-23T21:11:58.898 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.7589e-05 2023-09-23T21:11:58.898 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:11:58.898 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:11:58.898 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:11:58.898 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:11:58.898 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:11:58.899 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_repop(client.46328.0:81410 10.0 e468/453 10:0741fb3a:::benchmark_data_smithi026_379838_object1453:head v 468'3893, mlcod=0'0)", 2023-09-23T21:11:58.899 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:11:58.860765+0000", 2023-09-23T21:11:58.899 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 0.0065577279999999996, 2023-09-23T21:11:58.899 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0069524679999999998, 2023-09-23T21:11:58.899 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:11:58.899 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "started", 2023-09-23T21:11:58.899 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:11:58.899 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:11:58.899 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:11:58.899 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:58.860765+0000", 2023-09-23T21:11:58.899 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:11:58.899 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:11:58.900 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:11:58.900 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:11:58.900 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:58.860765+0000", 2023-09-23T21:11:58.900 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:11:58.900 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:11:58.900 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:11:58.900 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:11:58.900 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:58.860763+0000", 2023-09-23T21:11:58.900 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999981 2023-09-23T21:11:58.901 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:11:58.901 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:11:58.901 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:11:58.901 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:58.860774+0000", 2023-09-23T21:11:58.901 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.0407e-05 2023-09-23T21:11:58.901 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:11:58.901 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:11:58.901 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:11:58.901 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:58.860785+0000", 2023-09-23T21:11:58.902 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.1416999999999999e-05 2023-09-23T21:11:58.902 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:11:58.902 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:11:58.902 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:11:58.902 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:58.860792+0000", 2023-09-23T21:11:58.902 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.8499999999999996e-06 2023-09-23T21:11:58.902 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:11:58.902 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:11:58.902 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:11:58.903 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:58.860836+0000", 2023-09-23T21:11:58.903 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.3628999999999999e-05 2023-09-23T21:11:58.903 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:11:58.903 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:11:58.903 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:11:58.903 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:58.860860+0000", 2023-09-23T21:11:58.903 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.4643000000000001e-05 2023-09-23T21:11:58.904 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:11:58.904 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:11:58.904 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:11:58.904 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:11:58.904 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:11:58.904 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.46328.0:81368 10.10 10:0ac1533e:::benchmark_data_smithi026_379838_object1411:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e468)", 2023-09-23T21:11:58.904 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:11:58.774916+0000", 2023-09-23T21:11:58.904 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 0.092406813000000004, 2023-09-23T21:11:58.905 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.092866071999999994, 2023-09-23T21:11:58.905 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:11:58.905 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "delayed", 2023-09-23T21:11:58.905 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:11:58.905 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.46328", 2023-09-23T21:11:58.905 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/3748861181", 2023-09-23T21:11:58.905 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 81368 2023-09-23T21:11:58.906 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:11:58.906 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:11:58.906 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:11:58.906 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:11:58.906 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:58.774916+0000", 2023-09-23T21:11:58.906 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:11:58.906 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:11:58.906 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:11:58.906 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:11:58.906 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:58.774916+0000", 2023-09-23T21:11:58.907 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:11:58.907 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:11:58.907 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:11:58.907 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:11:58.907 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:58.774915+0000", 2023-09-23T21:11:58.907 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T21:11:58.907 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:11:58.907 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:11:58.908 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:11:58.908 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:58.774926+0000", 2023-09-23T21:11:58.908 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.1338000000000001e-05 2023-09-23T21:11:58.908 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:11:58.908 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:11:58.908 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:11:58.908 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:58.774940+0000", 2023-09-23T21:11:58.908 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.3859999999999999e-05 2023-09-23T21:11:58.908 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:11:58.909 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:11:58.909 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:11:58.909 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:58.774947+0000", 2023-09-23T21:11:58.909 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.2609999999999996e-06 2023-09-23T21:11:58.909 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:11:58.909 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:11:58.909 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:11:58.909 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:58.812701+0000", 2023-09-23T21:11:58.910 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.037754230999999999 2023-09-23T21:11:58.910 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:11:58.910 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:11:58.910 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for missing object", 2023-09-23T21:11:58.910 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:58.816913+0000", 2023-09-23T21:11:58.910 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0042113580000000001 2023-09-23T21:11:58.910 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:11:58.910 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:11:58.910 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:11:58.911 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:58.846944+0000", 2023-09-23T21:11:58.911 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.030031803999999999 2023-09-23T21:11:58.911 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:11:58.911 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:11:58.911 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for rw locks", 2023-09-23T21:11:58.911 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:58.851796+0000", 2023-09-23T21:11:58.911 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0048515590000000001 2023-09-23T21:11:58.911 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:11:58.912 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:11:58.912 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:11:58.912 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:11:58.912 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:11:58.912 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.46328.0:81411 10.1 10.f184d81 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e468)", 2023-09-23T21:11:58.912 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:11:58.864551+0000", 2023-09-23T21:11:58.912 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 0.002772222, 2023-09-23T21:11:58.913 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0033108590000000002, 2023-09-23T21:11:58.913 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:11:58.913 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "queued for pg", 2023-09-23T21:11:58.913 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:11:58.913 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.46328", 2023-09-23T21:11:58.913 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/3748861181", 2023-09-23T21:11:58.913 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 81411 2023-09-23T21:11:58.914 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:11:58.914 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:11:58.914 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:11:58.914 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:11:58.914 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:58.864551+0000", 2023-09-23T21:11:58.915 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:11:58.915 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:11:58.915 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:11:58.915 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:11:58.915 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:58.864551+0000", 2023-09-23T21:11:58.915 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:11:58.915 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:11:58.916 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:11:58.916 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:11:58.916 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:58.864549+0000", 2023-09-23T21:11:58.916 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999981 2023-09-23T21:11:58.916 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:11:58.916 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:11:58.916 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:11:58.916 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:58.864557+0000", 2023-09-23T21:11:58.916 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.9230000000000002e-06 2023-09-23T21:11:58.916 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:11:58.916 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:11:58.917 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:11:58.917 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:58.864571+0000", 2023-09-23T21:11:58.917 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.4411e-05 2023-09-23T21:11:58.917 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:11:58.917 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:11:58.917 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:11:58.917 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:58.864580+0000", 2023-09-23T21:11:58.917 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.3489999999999992e-06 2023-09-23T21:11:58.917 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:11:58.917 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:11:58.918 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:11:58.918 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:11:58.918 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:11:58.918 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_repop(client.46328.0:81373 10.b e468/466)", 2023-09-23T21:11:58.918 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:11:58.864753+0000", 2023-09-23T21:11:58.918 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 0.0025700789999999999, 2023-09-23T21:11:58.918 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0031605299999999999, 2023-09-23T21:11:58.918 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:11:58.918 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "queued for pg", 2023-09-23T21:11:58.918 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:11:58.918 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:11:58.919 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:11:58.919 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:58.864753+0000", 2023-09-23T21:11:58.919 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:11:58.919 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:11:58.919 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:11:58.919 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:11:58.919 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:58.864753+0000", 2023-09-23T21:11:58.919 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:11:58.919 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:11:58.919 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:11:58.920 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:11:58.920 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:58.864752+0000", 2023-09-23T21:11:58.920 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.999999 2023-09-23T21:11:58.920 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:11:58.920 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:11:58.920 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:11:58.920 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:58.864760+0000", 2023-09-23T21:11:58.920 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.2840000000000006e-06 2023-09-23T21:11:58.920 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:11:58.920 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:11:58.920 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:11:58.921 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:58.864775+0000", 2023-09-23T21:11:58.921 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.4745e-05 2023-09-23T21:11:58.921 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:11:58.921 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:11:58.921 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:11:58.921 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:58.864782+0000", 2023-09-23T21:11:58.921 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.6780000000000003e-06 2023-09-23T21:11:58.921 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:11:58.921 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:11:58.921 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:11:58.922 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:11:58.922 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:11:58.922 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.46328.0:81370 10.3 10:c62c5dec:::benchmark_data_smithi026_379838_object1413:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e468)", 2023-09-23T21:11:58.922 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:11:58.778701+0000", 2023-09-23T21:11:58.922 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 0.088622065999999999, 2023-09-23T21:11:58.922 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.089263179999999998, 2023-09-23T21:11:58.922 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:11:58.922 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "delayed", 2023-09-23T21:11:58.922 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:11:58.922 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.46328", 2023-09-23T21:11:58.923 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/3748861181", 2023-09-23T21:11:58.923 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 81370 2023-09-23T21:11:58.923 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:11:58.923 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:11:58.923 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:11:58.923 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:11:58.923 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:58.778701+0000", 2023-09-23T21:11:58.923 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:11:58.923 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:11:58.923 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:11:58.924 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:11:58.924 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:58.778701+0000", 2023-09-23T21:11:58.924 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:11:58.924 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:11:58.924 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:11:58.924 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:11:58.924 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:58.778700+0000", 2023-09-23T21:11:58.924 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T21:11:58.924 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:11:58.924 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:11:58.924 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:11:58.925 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:58.778705+0000", 2023-09-23T21:11:58.925 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.9699999999999998e-06 2023-09-23T21:11:58.925 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:11:58.925 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:11:58.925 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:11:58.925 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:58.778714+0000", 2023-09-23T21:11:58.925 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.0119999999999994e-06 2023-09-23T21:11:58.925 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:11:58.925 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:11:58.925 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:11:58.926 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:58.778718+0000", 2023-09-23T21:11:58.926 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.8169999999999998e-06 2023-09-23T21:11:58.926 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:11:58.926 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:11:58.926 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:11:58.926 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:58.786055+0000", 2023-09-23T21:11:58.926 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0073361030000000001 2023-09-23T21:11:58.926 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:11:58.926 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:11:58.926 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for missing object", 2023-09-23T21:11:58.926 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:58.790783+0000", 2023-09-23T21:11:58.927 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0047283270000000001 2023-09-23T21:11:58.927 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:11:58.927 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:11:58.927 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:11:58.927 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:11:58.927 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:11:58.927 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.46328.0:81393 10.6 10:65d51974:::benchmark_data_smithi026_379838_object1436:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e468)", 2023-09-23T21:11:58.927 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:11:58.818294+0000", 2023-09-23T21:11:58.927 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 0.049028536999999997, 2023-09-23T21:11:58.927 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.049740623999999997, 2023-09-23T21:11:58.927 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:11:58.928 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "reached pg", 2023-09-23T21:11:58.928 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:11:58.928 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.46328", 2023-09-23T21:11:58.928 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/3748861181", 2023-09-23T21:11:58.928 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 81393 2023-09-23T21:11:58.928 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:11:58.928 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:11:58.928 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:11:58.928 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:11:58.928 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:58.818294+0000", 2023-09-23T21:11:58.928 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:11:58.929 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:11:58.929 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:11:58.929 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:11:58.929 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:58.818294+0000", 2023-09-23T21:11:58.929 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:11:58.929 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:11:58.929 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:11:58.929 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:11:58.929 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:58.818293+0000", 2023-09-23T21:11:58.929 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999981 2023-09-23T21:11:58.929 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:11:58.930 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:11:58.930 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:11:58.930 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:58.818301+0000", 2023-09-23T21:11:58.930 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.738e-06 2023-09-23T21:11:58.930 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:11:58.930 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:11:58.930 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:11:58.930 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:58.818316+0000", 2023-09-23T21:11:58.930 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.4131000000000001e-05 2023-09-23T21:11:58.930 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:11:58.931 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:11:58.931 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:11:58.931 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:58.818324+0000", 2023-09-23T21:11:58.931 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.9370000000000004e-06 2023-09-23T21:11:58.931 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:11:58.931 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:11:58.931 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:11:58.931 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:58.819471+0000", 2023-09-23T21:11:58.931 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00114726 2023-09-23T21:11:58.932 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:11:58.932 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:11:58.932 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for missing object", 2023-09-23T21:11:58.932 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:58.827604+0000", 2023-09-23T21:11:58.932 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0081328019999999997 2023-09-23T21:11:58.932 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:11:58.932 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:11:58.932 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:11:58.932 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:58.864482+0000", 2023-09-23T21:11:58.932 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.036878637999999998 2023-09-23T21:11:58.933 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:11:58.933 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:11:58.933 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:11:58.933 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:11:58.933 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:11:58.933 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_repop_reply(client.46328.0:81405 10.4 e468/453 ondisk, result = 0)", 2023-09-23T21:11:58.933 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:11:58.866006+0000", 2023-09-23T21:11:58.933 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 0.001317202, 2023-09-23T21:11:58.933 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.002108614, 2023-09-23T21:11:58.933 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:11:58.933 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "started", 2023-09-23T21:11:58.934 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:11:58.934 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:11:58.934 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:11:58.934 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:58.866006+0000", 2023-09-23T21:11:58.934 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:11:58.934 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:11:58.934 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:11:58.934 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:11:58.935 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:58.866006+0000", 2023-09-23T21:11:58.935 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:11:58.935 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:11:58.935 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:11:58.935 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:11:58.935 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:58.866005+0000", 2023-09-23T21:11:58.935 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.999999 2023-09-23T21:11:58.935 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:11:58.935 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:11:58.936 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:11:58.936 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:58.866012+0000", 2023-09-23T21:11:58.936 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.2840000000000001e-06 2023-09-23T21:11:58.936 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:11:58.936 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:11:58.936 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:11:58.936 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:58.866022+0000", 2023-09-23T21:11:58.936 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.027e-05 2023-09-23T21:11:58.936 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:11:58.936 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:11:58.936 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:11:58.937 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:58.866028+0000", 2023-09-23T21:11:58.937 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.4729999999999999e-06 2023-09-23T21:11:58.937 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:11:58.937 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:11:58.937 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:11:58.937 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:58.866478+0000", 2023-09-23T21:11:58.938 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00044999500000000002 2023-09-23T21:11:58.938 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:11:58.938 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:11:58.938 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:11:58.938 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:58.866865+0000", 2023-09-23T21:11:58.938 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00038751399999999999 2023-09-23T21:11:58.938 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:11:58.938 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:11:58.938 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:11:58.938 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:11:58.939 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:11:58.939 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_repop_reply(client.46328.0:81332 10.3 e468/466 ondisk, result = 0)", 2023-09-23T21:11:58.939 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:11:58.823170+0000", 2023-09-23T21:11:58.939 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 0.044153082000000003, 2023-09-23T21:11:58.939 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.045004124, 2023-09-23T21:11:58.939 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:11:58.939 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "started", 2023-09-23T21:11:58.940 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:11:58.940 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:11:58.940 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:11:58.940 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:58.823170+0000", 2023-09-23T21:11:58.940 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:11:58.940 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:11:58.940 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:11:58.940 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:11:58.940 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:58.823170+0000", 2023-09-23T21:11:58.940 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:11:58.941 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:11:58.941 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:11:58.941 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:11:58.941 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:58.823169+0000", 2023-09-23T21:11:58.941 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.999999 2023-09-23T21:11:58.941 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:11:58.941 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:11:58.942 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:11:58.942 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:58.823178+0000", 2023-09-23T21:11:58.942 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.5699999999999999e-06 2023-09-23T21:11:58.942 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:11:58.942 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:11:58.942 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:11:58.942 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:58.823219+0000", 2023-09-23T21:11:58.942 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.1056000000000001e-05 2023-09-23T21:11:58.943 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:11:58.943 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:11:58.943 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:11:58.943 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:58.823226+0000", 2023-09-23T21:11:58.943 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.195e-06 2023-09-23T21:11:58.943 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:11:58.943 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:11:58.943 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:11:58.943 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:58.866111+0000", 2023-09-23T21:11:58.944 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.042885026 2023-09-23T21:11:58.944 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:11:58.944 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:11:58.944 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:11:58.944 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:58.866560+0000", 2023-09-23T21:11:58.944 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000449636 2023-09-23T21:11:58.944 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:11:58.945 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:11:58.945 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:11:58.945 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:11:58.945 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:11:58.945 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.46328.0:81378 10.10 10:0c8a72a8:::benchmark_data_smithi026_379838_object1421:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e468)", 2023-09-23T21:11:58.945 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:11:58.790042+0000", 2023-09-23T21:11:58.945 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 0.077280847999999999, 2023-09-23T21:11:58.945 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.078215976000000006, 2023-09-23T21:11:58.945 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:11:58.945 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "delayed", 2023-09-23T21:11:58.946 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:11:58.946 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.46328", 2023-09-23T21:11:58.946 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/3748861181", 2023-09-23T21:11:58.946 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 81378 2023-09-23T21:11:58.946 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:11:58.946 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:11:58.946 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:11:58.946 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:11:58.946 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:58.790042+0000", 2023-09-23T21:11:58.946 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:11:58.947 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:11:58.947 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:11:58.947 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:11:58.947 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:58.790042+0000", 2023-09-23T21:11:58.947 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:11:58.947 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:11:58.947 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:11:58.947 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:11:58.947 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:58.790041+0000", 2023-09-23T21:11:58.948 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T21:11:58.948 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:11:58.948 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:11:58.948 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:11:58.948 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:58.790048+0000", 2023-09-23T21:11:58.948 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.5460000000000003e-06 2023-09-23T21:11:58.949 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:11:58.949 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:11:58.949 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:11:58.949 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:58.790061+0000", 2023-09-23T21:11:58.949 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.3210999999999999e-05 2023-09-23T21:11:58.949 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:11:58.949 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:11:58.950 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:11:58.950 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:58.790068+0000", 2023-09-23T21:11:58.950 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.4030000000000004e-06 2023-09-23T21:11:58.950 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:11:58.950 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:11:58.950 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:11:58.950 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:58.817681+0000", 2023-09-23T21:11:58.950 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.027612960999999998 2023-09-23T21:11:58.951 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:11:58.951 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:11:58.951 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for missing object", 2023-09-23T21:11:58.951 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:58.827063+0000", 2023-09-23T21:11:58.951 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0093824459999999991 2023-09-23T21:11:58.951 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:11:58.951 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:11:58.951 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:11:58.951 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:11:58.951 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:11:58.952 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.46328.0:81380 10.1 10:84b684e5:::benchmark_data_smithi026_379838_object1423:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e468)", 2023-09-23T21:11:58.952 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:11:58.790074+0000", 2023-09-23T21:11:58.952 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 0.077248937000000004, 2023-09-23T21:11:58.952 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.078252274999999996, 2023-09-23T21:11:58.952 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:11:58.952 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "delayed", 2023-09-23T21:11:58.952 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:11:58.952 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.46328", 2023-09-23T21:11:58.952 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/3748861181", 2023-09-23T21:11:58.952 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 81380 2023-09-23T21:11:58.952 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:11:58.953 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:11:58.953 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:11:58.953 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:11:58.953 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:58.790074+0000", 2023-09-23T21:11:58.953 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:11:58.953 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:11:58.953 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:11:58.953 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:11:58.953 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:58.790074+0000", 2023-09-23T21:11:58.953 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:11:58.954 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:11:58.954 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:11:58.954 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:11:58.954 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:58.790074+0000", 2023-09-23T21:11:58.954 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999995 2023-09-23T21:11:58.954 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:11:58.954 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:11:58.954 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:11:58.954 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:58.790077+0000", 2023-09-23T21:11:58.954 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.8849999999999999e-06 2023-09-23T21:11:58.955 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:11:58.955 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:11:58.955 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:11:58.955 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:58.790082+0000", 2023-09-23T21:11:58.955 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.4260000000000002e-06 2023-09-23T21:11:58.955 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:11:58.955 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:11:58.955 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:11:58.955 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:58.790086+0000", 2023-09-23T21:11:58.955 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.704e-06 2023-09-23T21:11:58.955 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:11:58.956 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:11:58.956 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:11:58.956 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:58.827399+0000", 2023-09-23T21:11:58.956 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.037312866 2023-09-23T21:11:58.956 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:11:58.956 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:11:58.956 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for missing object", 2023-09-23T21:11:58.956 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:58.831628+0000", 2023-09-23T21:11:58.956 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0042290510000000002 2023-09-23T21:11:58.956 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:11:58.957 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:11:58.957 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:11:58.957 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:58.853231+0000", 2023-09-23T21:11:58.957 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.021603391999999999 2023-09-23T21:11:58.957 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:11:58.957 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:11:58.957 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for rw locks", 2023-09-23T21:11:58.957 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:58.856797+0000", 2023-09-23T21:11:58.957 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0035660819999999999 2023-09-23T21:11:58.958 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:11:58.958 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:11:58.958 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:11:58.958 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:11:58.958 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:11:58.958 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.46328.0:81332 10.3 10:c0611fbd:::benchmark_data_smithi026_379838_object1375:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e468)", 2023-09-23T21:11:58.958 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:11:58.673859+0000", 2023-09-23T21:11:58.958 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 0.19346383, 2023-09-23T21:11:58.958 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.194550205, 2023-09-23T21:11:58.959 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:11:58.959 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "waiting for sub ops", 2023-09-23T21:11:58.959 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:11:58.959 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.46328", 2023-09-23T21:11:58.959 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/3748861181", 2023-09-23T21:11:58.959 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 81332 2023-09-23T21:11:58.959 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:11:58.959 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:11:58.959 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:11:58.959 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:11:58.960 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:58.673859+0000", 2023-09-23T21:11:58.960 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:11:58.960 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:11:58.960 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:11:58.960 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:11:58.960 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:58.673859+0000", 2023-09-23T21:11:58.960 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:11:58.960 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:11:58.960 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:11:58.961 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:11:58.961 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:58.673858+0000", 2023-09-23T21:11:58.961 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999995 2023-09-23T21:11:58.961 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:11:58.961 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:11:58.961 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:11:58.961 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:58.673863+0000", 2023-09-23T21:11:58.961 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.34e-06 2023-09-23T21:11:58.961 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:11:58.961 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:11:58.962 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:11:58.962 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:58.673871+0000", 2023-09-23T21:11:58.962 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.4509999999999992e-06 2023-09-23T21:11:58.962 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:11:58.962 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:11:58.962 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:11:58.962 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:58.673876+0000", 2023-09-23T21:11:58.962 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.8470000000000003e-06 2023-09-23T21:11:58.962 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:11:58.962 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:11:58.962 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:11:58.963 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:58.689618+0000", 2023-09-23T21:11:58.963 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.015741501000000001 2023-09-23T21:11:58.963 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:11:58.963 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:11:58.963 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for missing object", 2023-09-23T21:11:58.963 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:58.694994+0000", 2023-09-23T21:11:58.963 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0053765410000000003 2023-09-23T21:11:58.963 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:11:58.963 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:11:58.963 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:11:58.964 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:58.733656+0000", 2023-09-23T21:11:58.964 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.038662344000000001 2023-09-23T21:11:58.964 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:11:58.964 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:11:58.964 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for rw locks", 2023-09-23T21:11:58.964 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:58.739317+0000", 2023-09-23T21:11:58.964 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0056609809999999998 2023-09-23T21:11:58.964 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:11:58.964 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:11:58.964 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:11:58.965 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:58.809064+0000", 2023-09-23T21:11:58.965 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.069746541999999995 2023-09-23T21:11:58.965 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:11:58.965 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:11:58.965 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:11:58.965 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:58.813327+0000", 2023-09-23T21:11:58.965 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0042629219999999997 2023-09-23T21:11:58.965 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:11:58.965 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:11:58.965 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 2,5", 2023-09-23T21:11:58.966 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:58.820740+0000", 2023-09-23T21:11:58.966 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0074133999999999997 2023-09-23T21:11:58.966 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:11:58.966 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:11:58.966 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:11:58.966 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:58.860569+0000", 2023-09-23T21:11:58.966 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.039828802000000003 2023-09-23T21:11:58.966 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:11:58.966 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:11:58.967 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:11:58.967 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:58.861197+0000", 2023-09-23T21:11:58.967 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000627974 2023-09-23T21:11:58.967 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:11:58.967 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:11:58.967 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:11:58.967 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:58.866961+0000", 2023-09-23T21:11:58.967 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0057643349999999998 2023-09-23T21:11:58.967 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:11:58.967 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:11:58.967 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:11:58.968 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:11:58.968 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:11:58.968 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "MOSDPGPush(10.3 468/466 [PushOp(10:c62c5dec:::benchmark_data_smithi026_379838_object1413:head, version: 422'88, data_included: [0~65536], data_size: 65536, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(10:c62c5dec:::benchmark_data_smithi026_379838_object1413:head@422'88, size: 65536, copy_subset: [0~65536], clone_subset: {}, snapset: 0=[]:{}, object_exist: 0), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:65536, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false, error:false))])", 2023-09-23T21:11:58.968 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:11:58.793714+0000", 2023-09-23T21:11:58.968 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 0.073608452000000005, 2023-09-23T21:11:58.968 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.074826714000000003, 2023-09-23T21:11:58.968 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:11:58.968 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "queued for pg", 2023-09-23T21:11:58.968 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:11:58.968 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:11:58.969 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:11:58.969 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:58.793714+0000", 2023-09-23T21:11:58.969 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:11:58.969 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:11:58.969 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:11:58.969 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:11:58.969 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:58.793714+0000", 2023-09-23T21:11:58.969 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:11:58.970 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:11:58.970 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:11:58.970 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:11:58.970 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:58.793714+0000", 2023-09-23T21:11:58.970 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999995 2023-09-23T21:11:58.970 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:11:58.970 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:11:58.970 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:11:58.970 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:58.793782+0000", 2023-09-23T21:11:58.971 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.8297e-05 2023-09-23T21:11:58.971 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:11:58.971 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:11:58.971 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:11:58.971 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:58.793799+0000", 2023-09-23T21:11:58.971 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.6467e-05 2023-09-23T21:11:58.971 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:11:58.971 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:11:58.972 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:11:58.972 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:58.793807+0000", 2023-09-23T21:11:58.972 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.8400000000000001e-06 2023-09-23T21:11:58.972 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:11:58.972 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:11:58.972 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:11:58.972 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:11:58.972 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:11:58.972 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "MOSDPGPush(10.1 468/466 [PushOp(10:80b364c7:::benchmark_data_smithi026_379838_object1432:head, version: 422'91, data_included: [0~65536], data_size: 65536, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(10:80b364c7:::benchmark_data_smithi026_379838_object1432:head@422'91, size: 65536, copy_subset: [0~65536], clone_subset: {}, snapset: 0=[]:{}, object_exist: 0), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:65536, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false, error:false))])", 2023-09-23T21:11:58.973 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:11:58.837996+0000", 2023-09-23T21:11:58.973 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 0.02932742, 2023-09-23T21:11:58.973 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.030616391, 2023-09-23T21:11:58.973 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:11:58.973 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "started", 2023-09-23T21:11:58.973 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:11:58.973 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:11:58.973 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:11:58.974 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:58.837996+0000", 2023-09-23T21:11:58.974 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:11:58.974 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:11:58.974 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:11:58.974 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:11:58.974 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:58.837996+0000", 2023-09-23T21:11:58.974 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:11:58.974 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:11:58.975 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:11:58.975 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:11:58.975 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:58.837995+0000", 2023-09-23T21:11:58.975 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999995 2023-09-23T21:11:58.975 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:11:58.975 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:11:58.975 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:11:58.975 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:58.838061+0000", 2023-09-23T21:11:58.975 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.6100999999999996e-05 2023-09-23T21:11:58.976 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:11:58.976 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:11:58.976 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:11:58.976 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:58.838076+0000", 2023-09-23T21:11:58.976 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.5149e-05 2023-09-23T21:11:58.976 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:11:58.976 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:11:58.976 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:11:58.976 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:58.838086+0000", 2023-09-23T21:11:58.977 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.4369999999999996e-06 2023-09-23T21:11:58.977 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:11:58.977 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:11:58.977 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:11:58.977 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:58.858045+0000", 2023-09-23T21:11:58.977 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.019959352 2023-09-23T21:11:58.977 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:11:58.978 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:11:58.978 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:11:58.978 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:58.858412+0000", 2023-09-23T21:11:58.978 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00036717999999999999 2023-09-23T21:11:58.978 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:11:58.978 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:11:58.978 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:11:58.979 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:11:58.979 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:11:58.979 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.46328.0:81359 10.3 10:c7f371a8:::benchmark_data_smithi026_379838_object1402:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e468)", 2023-09-23T21:11:58.979 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:11:58.748519+0000", 2023-09-23T21:11:58.979 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 0.118803513, 2023-09-23T21:11:58.979 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.120153818, 2023-09-23T21:11:58.979 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:11:58.979 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "delayed", 2023-09-23T21:11:58.979 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:11:58.980 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.46328", 2023-09-23T21:11:58.980 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/3748861181", 2023-09-23T21:11:58.980 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 81359 2023-09-23T21:11:58.980 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:11:58.980 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:11:58.980 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:11:58.980 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:11:58.981 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:58.748519+0000", 2023-09-23T21:11:58.981 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:11:58.981 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:11:58.981 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:11:58.981 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:11:58.981 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:58.748519+0000", 2023-09-23T21:11:58.981 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:11:58.982 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:11:58.982 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:11:58.982 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:11:58.982 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:58.748518+0000", 2023-09-23T21:11:58.982 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T21:11:58.982 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:11:58.982 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:11:58.982 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:11:58.983 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:58.748525+0000", 2023-09-23T21:11:58.983 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.1930000000000004e-06 2023-09-23T21:11:58.983 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:11:58.983 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:11:58.983 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:11:58.983 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:58.748537+0000", 2023-09-23T21:11:58.983 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.1428e-05 2023-09-23T21:11:58.983 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:11:58.983 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:11:58.983 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:11:58.984 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:58.748543+0000", 2023-09-23T21:11:58.984 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.2079999999999997e-06 2023-09-23T21:11:58.984 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:11:58.984 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:11:58.984 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:11:58.984 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:58.780064+0000", 2023-09-23T21:11:58.984 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.031521149999999998 2023-09-23T21:11:58.984 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:11:58.984 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:11:58.984 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for missing object", 2023-09-23T21:11:58.984 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:58.785599+0000", 2023-09-23T21:11:58.985 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0055352969999999998 2023-09-23T21:11:58.985 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:11:58.985 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:11:58.985 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:11:58.985 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:58.851421+0000", 2023-09-23T21:11:58.985 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.065821424000000003 2023-09-23T21:11:58.985 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:11:58.985 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:11:58.985 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for rw locks", 2023-09-23T21:11:58.985 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:58.858759+0000", 2023-09-23T21:11:58.986 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0073382029999999997 2023-09-23T21:11:58.986 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:11:58.986 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:11:58.986 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:11:58.986 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:11:58.986 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:11:58.986 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.46328.0:81385 10.3 10.5af00e83 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e468)", 2023-09-23T21:11:58.986 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:11:58.797459+0000", 2023-09-23T21:11:58.986 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 0.069863842999999995, 2023-09-23T21:11:58.986 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.071295084999999994, 2023-09-23T21:11:58.987 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:11:58.987 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "queued for pg", 2023-09-23T21:11:58.987 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:11:58.987 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.46328", 2023-09-23T21:11:58.987 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/3748861181", 2023-09-23T21:11:58.987 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 81385 2023-09-23T21:11:58.987 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:11:58.987 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:11:58.987 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:11:58.987 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:11:58.988 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:58.797459+0000", 2023-09-23T21:11:58.988 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:11:58.988 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:11:58.988 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:11:58.988 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:11:58.988 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:58.797459+0000", 2023-09-23T21:11:58.988 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:11:58.988 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:11:58.988 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:11:58.988 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:11:58.989 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:58.797459+0000", 2023-09-23T21:11:58.989 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999995 2023-09-23T21:11:58.989 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:11:58.989 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:11:58.989 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:11:58.989 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:58.797462+0000", 2023-09-23T21:11:58.989 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.1520000000000001e-06 2023-09-23T21:11:58.989 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:11:58.989 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:11:58.989 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:11:58.989 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:58.797469+0000", 2023-09-23T21:11:58.990 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.1720000000000001e-06 2023-09-23T21:11:58.990 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:11:58.990 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:11:58.990 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:11:58.990 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:58.797473+0000", 2023-09-23T21:11:58.990 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.4599999999999996e-06 2023-09-23T21:11:58.990 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:11:58.990 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:11:58.990 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:11:58.990 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:11:58.991 INFO:teuthology.orchestra.run.smithi070.stdout: ], 2023-09-23T21:11:58.991 INFO:teuthology.orchestra.run.smithi070.stdout: "num_ops": 20 2023-09-23T21:11:58.991 INFO:teuthology.orchestra.run.smithi070.stdout:} 2023-09-23T21:11:58.991 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:11:59.112 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:11:59.299 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:11:59.310 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:11:59.311 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:11:59.356 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_blocked_ops 2023-09-23T21:11:59.412 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:11:59.446 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:11:59.644 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:11:59.648 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:11:59.649 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:11:59.749 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:11:59.783 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 21:11:59 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[164029]: ::ffff:172.21.15.70 - - [23/Sep/2023:21:11:59] "GET /metrics HTTP/1.1" 200 36450 "" "Prometheus/2.43.0" 2023-09-23T21:11:59.787 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:11:59.984 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:11:59.990 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:11:59.990 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:12:00.091 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:12:00.148 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T21:12:00.200 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:12:00.322 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:12:00.326 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:12:00.326 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:12:00.427 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:12:00.491 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:12:00 smithi070 ceph-mon[168282]: pgmap v1193: 119 pgs: 1 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 21 active+undersized, 1 active+recovering+undersized+degraded+remapped, 8 active+undersized+degraded, 8 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+degraded+remapped, 6 active+recovery_wait+degraded+remapped, 72 active+clean; 4.8 GiB data, 18 GiB used, 607 GiB / 626 GiB avail; 102 B/s rd, 7.7 MiB/s wr, 212 op/s; 64713/237525 objects degraded (27.245%); 14819/237525 objects misplaced (6.239%); 11 MiB/s, 169 objects/s recovering 2023-09-23T21:12:00.496 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T21:12:00.505 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:12:00.578 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:12:00.583 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:12:00.583 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:12:00.631 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:12:00 smithi026 ceph-mon[175151]: pgmap v1193: 119 pgs: 1 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 21 active+undersized, 1 active+recovering+undersized+degraded+remapped, 8 active+undersized+degraded, 8 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+degraded+remapped, 6 active+recovery_wait+degraded+remapped, 72 active+clean; 4.8 GiB data, 18 GiB used, 607 GiB / 626 GiB avail; 102 B/s rd, 7.7 MiB/s wr, 212 op/s; 64713/237525 objects degraded (27.245%); 14819/237525 objects misplaced (6.239%); 11 MiB/s, 169 objects/s recovering 2023-09-23T21:12:00.631 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:12:00 smithi026 ceph-mon[179688]: pgmap v1193: 119 pgs: 1 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 21 active+undersized, 1 active+recovering+undersized+degraded+remapped, 8 active+undersized+degraded, 8 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+degraded+remapped, 6 active+recovery_wait+degraded+remapped, 72 active+clean; 4.8 GiB data, 18 GiB used, 607 GiB / 626 GiB avail; 102 B/s rd, 7.7 MiB/s wr, 212 op/s; 64713/237525 objects degraded (27.245%); 14819/237525 objects misplaced (6.239%); 11 MiB/s, 169 objects/s recovering 2023-09-23T21:12:00.634 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 7] out_osds: [6] dead_osds: [0] live_osds: [2, 3, 4, 5, 7, 6, 1] 2023-09-23T21:12:00.634 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T21:12:00.634 INFO:tasks.thrashosds.thrasher:Removing osd 1, in_osds are: [0, 1, 2, 3, 4, 5, 7] 2023-09-23T21:12:00.634 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph osd out 1 2023-09-23T21:12:00.684 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:12:00.743 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:12:00.812 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:12:00.816 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:12:00.817 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:12:00.917 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:12:00.990 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:12:01.459 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:12:01.477 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:12:01.478 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:12:01.578 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:12:01.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:12:01 smithi070 ceph-mon[168282]: Health check update: Degraded data redundancy: 64713/237525 objects degraded (27.245%), 26 pgs degraded (PG_DEGRADED) 2023-09-23T21:12:01.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:12:01 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:12:01.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:12:01 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:12:01.631 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:12:01 smithi026 ceph-mon[175151]: Health check update: Degraded data redundancy: 64713/237525 objects degraded (27.245%), 26 pgs degraded (PG_DEGRADED) 2023-09-23T21:12:01.631 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:12:01 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:12:01.631 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:12:01 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:12:01.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:12:01 smithi026 ceph-mon[179688]: Health check update: Degraded data redundancy: 64713/237525 objects degraded (27.245%), 26 pgs degraded (PG_DEGRADED) 2023-09-23T21:12:01.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:12:01 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:12:01.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:12:01 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:12:01.757 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:12:01.820 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:12:01.827 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:12:01.827 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:12:01.928 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:12:01.939 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1: {} 2023-09-23T21:12:01.968 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2: {} 2023-09-23T21:12:01.994 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3: {} 2023-09-23T21:12:02.001 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:12:02.030 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4: {} 2023-09-23T21:12:02.068 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:12:02.069 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T21:12:02.087 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:12:02.088 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:12:02.098 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6: {} 2023-09-23T21:12:02.103 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:12:02.123 INFO:teuthology.orchestra.run.smithi070.stdout:ERROR: (22) Invalid argument 2023-09-23T21:12:02.123 INFO:teuthology.orchestra.run.smithi070.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T21:12:02.131 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T21:12:02.134 INFO:teuthology.orchestra.run.smithi026.stderr:Error ENXIO: problem getting command descriptions from osd.0 2023-09-23T21:12:02.135 INFO:teuthology.orchestra.run.smithi026.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T21:12:02.135 INFO:teuthology.orchestra.run.smithi026.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T21:12:02.135 INFO:teuthology.orchestra.run.smithi026.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T21:12:02.135 INFO:teuthology.orchestra.run.smithi026.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T21:12:02.135 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T21:12:02.135 INFO:teuthology.orchestra.run.smithi026.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T21:12:02.135 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T21:12:02.188 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:12:02.220 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:12:02.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:12:02 smithi070 ceph-mon[168282]: pgmap v1194: 119 pgs: 1 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 21 active+undersized, 1 active+recovering+undersized+degraded+remapped, 8 active+undersized+degraded, 8 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+degraded+remapped, 6 active+recovery_wait+degraded+remapped, 72 active+clean; 4.8 GiB data, 18 GiB used, 607 GiB / 626 GiB avail; 102 B/s rd, 6.8 MiB/s wr, 214 op/s; 64713/237009 objects degraded (27.304%); 14819/237009 objects misplaced (6.253%); 11 MiB/s, 169 objects/s recovering 2023-09-23T21:12:02.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:12:02 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/2844653936' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:12:02.631 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:12:02 smithi026 ceph-mon[175151]: pgmap v1194: 119 pgs: 1 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 21 active+undersized, 1 active+recovering+undersized+degraded+remapped, 8 active+undersized+degraded, 8 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+degraded+remapped, 6 active+recovery_wait+degraded+remapped, 72 active+clean; 4.8 GiB data, 18 GiB used, 607 GiB / 626 GiB avail; 102 B/s rd, 6.8 MiB/s wr, 214 op/s; 64713/237009 objects degraded (27.304%); 14819/237009 objects misplaced (6.253%); 11 MiB/s, 169 objects/s recovering 2023-09-23T21:12:02.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:12:02 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/2844653936' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:12:02.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:12:02 smithi026 ceph-mon[179688]: pgmap v1194: 119 pgs: 1 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 21 active+undersized, 1 active+recovering+undersized+degraded+remapped, 8 active+undersized+degraded, 8 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+degraded+remapped, 6 active+recovery_wait+degraded+remapped, 72 active+clean; 4.8 GiB data, 18 GiB used, 607 GiB / 626 GiB avail; 102 B/s rd, 6.8 MiB/s wr, 214 op/s; 64713/237009 objects degraded (27.304%); 14819/237009 objects misplaced (6.253%); 11 MiB/s, 169 objects/s recovering 2023-09-23T21:12:02.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:12:02 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/2844653936' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:12:02.634 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_historic_ops 2023-09-23T21:12:03.029 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:12:03.033 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:12:03.037 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:12:03.037 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:12:03.138 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:12:03.199 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:12:03.252 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:12:03.258 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:12:03.259 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:12:03.359 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:12:03.400 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:12:03.454 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:12:03.463 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:12:03.463 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:12:03.499 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T21:12:03.564 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:12:03.606 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:12:03.802 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:12:03.805 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:12:03.805 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:12:03.906 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:12:03.959 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:12:04.070 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:12:04.079 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:12:04.079 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:12:04.180 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:12:04.254 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:12:04.295 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T21:12:04.296 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T21:12:04.369 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:12:04.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:12:04 smithi070 ceph-mon[168282]: pgmap v1195: 119 pgs: 1 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 21 active+undersized, 1 active+recovering+undersized+degraded+remapped, 8 active+undersized+degraded, 8 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+degraded+remapped, 6 active+recovery_wait+degraded+remapped, 72 active+clean; 4.8 GiB data, 18 GiB used, 607 GiB / 626 GiB avail; 102 B/s rd, 3.2 MiB/s wr, 253 op/s; 63612/234087 objects degraded (27.175%); 14592/234087 objects misplaced (6.234%); 12 MiB/s, 195 objects/s recovering 2023-09-23T21:12:04.375 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:12:04.376 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:12:04.476 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:12:04.557 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:12:04.627 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:12:04.631 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:12:04 smithi026 ceph-mon[179688]: pgmap v1195: 119 pgs: 1 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 21 active+undersized, 1 active+recovering+undersized+degraded+remapped, 8 active+undersized+degraded, 8 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+degraded+remapped, 6 active+recovery_wait+degraded+remapped, 72 active+clean; 4.8 GiB data, 18 GiB used, 607 GiB / 626 GiB avail; 102 B/s rd, 3.2 MiB/s wr, 253 op/s; 63612/234087 objects degraded (27.175%); 14592/234087 objects misplaced (6.234%); 12 MiB/s, 195 objects/s recovering 2023-09-23T21:12:04.631 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:12:04 smithi026 ceph-mon[175151]: pgmap v1195: 119 pgs: 1 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 21 active+undersized, 1 active+recovering+undersized+degraded+remapped, 8 active+undersized+degraded, 8 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+degraded+remapped, 6 active+recovery_wait+degraded+remapped, 72 active+clean; 4.8 GiB data, 18 GiB used, 607 GiB / 626 GiB avail; 102 B/s rd, 3.2 MiB/s wr, 253 op/s; 63612/234087 objects degraded (27.175%); 14592/234087 objects misplaced (6.234%); 12 MiB/s, 195 objects/s recovering 2023-09-23T21:12:04.633 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:12:04.633 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:12:04.734 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:12:04.786 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:12:04.851 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:12:04.858 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:12:04.859 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:12:04.959 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:12:05.043 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:12:05.052 INFO:teuthology.orchestra.run.smithi070.stdout:ERROR: (22) Invalid argument 2023-09-23T21:12:05.053 INFO:teuthology.orchestra.run.smithi070.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T21:12:05.083 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:12:05.088 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:12:05.088 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:12:05.189 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:12:05.244 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:12:05.398 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:12:05.405 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:12:05.405 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:12:05.470 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_ops_in_flight 2023-09-23T21:12:05.506 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:12:05.761 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:12:05.843 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:12:05.849 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:12:05.850 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:12:05.951 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:12:06.030 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:12:06.097 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:12:06.102 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:12:06.103 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:12:06.204 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:12:06.272 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:12:06.297 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T21:12:06.324 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:12:06.328 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:12:06.328 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:12:06.330 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:12:06 smithi070 ceph-mon[168282]: pgmap v1196: 119 pgs: 1 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 21 active+undersized, 1 active+recovering+undersized+degraded+remapped, 8 active+undersized+degraded, 8 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+degraded+remapped, 6 active+recovery_wait+degraded+remapped, 72 active+clean; 4.7 GiB data, 18 GiB used, 607 GiB / 626 GiB avail; 102 B/s rd, 1.2 MiB/s wr, 261 op/s; 63436/232851 objects degraded (27.243%); 14511/232851 objects misplaced (6.232%); 12 MiB/s, 187 objects/s recovering 2023-09-23T21:12:06.330 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:12:06 smithi070 ceph-mon[168282]: Health check update: Degraded data redundancy: 63436/232851 objects degraded (27.243%), 26 pgs degraded (PG_DEGRADED) 2023-09-23T21:12:06.343 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T21:12:06.429 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:12:06.502 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:12:06.631 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:12:06 smithi026 ceph-mon[179688]: pgmap v1196: 119 pgs: 1 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 21 active+undersized, 1 active+recovering+undersized+degraded+remapped, 8 active+undersized+degraded, 8 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+degraded+remapped, 6 active+recovery_wait+degraded+remapped, 72 active+clean; 4.7 GiB data, 18 GiB used, 607 GiB / 626 GiB avail; 102 B/s rd, 1.2 MiB/s wr, 261 op/s; 63436/232851 objects degraded (27.243%); 14511/232851 objects misplaced (6.232%); 12 MiB/s, 187 objects/s recovering 2023-09-23T21:12:06.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:12:06 smithi026 ceph-mon[179688]: Health check update: Degraded data redundancy: 63436/232851 objects degraded (27.243%), 26 pgs degraded (PG_DEGRADED) 2023-09-23T21:12:06.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:12:06 smithi026 ceph-mon[175151]: pgmap v1196: 119 pgs: 1 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 21 active+undersized, 1 active+recovering+undersized+degraded+remapped, 8 active+undersized+degraded, 8 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+degraded+remapped, 6 active+recovery_wait+degraded+remapped, 72 active+clean; 4.7 GiB data, 18 GiB used, 607 GiB / 626 GiB avail; 102 B/s rd, 1.2 MiB/s wr, 261 op/s; 63436/232851 objects degraded (27.243%); 14511/232851 objects misplaced (6.232%); 12 MiB/s, 187 objects/s recovering 2023-09-23T21:12:06.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:12:06 smithi026 ceph-mon[175151]: Health check update: Degraded data redundancy: 63436/232851 objects degraded (27.243%), 26 pgs degraded (PG_DEGRADED) 2023-09-23T21:12:07.115 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:12:07.119 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:12:07.121 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:12:07.222 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:12:07.303 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:12:07.357 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:12:07.365 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:12:07.366 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:12:07.466 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:12:07.516 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:12:07.573 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:12:07.577 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:12:07.577 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:12:07.678 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:12:07.768 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:12:07.863 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:12:07.867 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:12:07.867 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:12:07.917 INFO:teuthology.orchestra.run.smithi070.stdout:ERROR: (22) Invalid argument 2023-09-23T21:12:07.918 INFO:teuthology.orchestra.run.smithi070.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T21:12:07.968 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:12:08.002 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:12:08.292 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:12:08.295 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:12:08.295 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:12:08.318 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_blocked_ops 2023-09-23T21:12:08.360 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:12:08 smithi070 ceph-mon[168282]: pgmap v1197: 119 pgs: 1 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 21 active+undersized, 1 active+recovering+undersized+degraded+remapped, 8 active+undersized+degraded, 8 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+degraded+remapped, 6 active+recovery_wait+degraded+remapped, 72 active+clean; 4.7 GiB data, 18 GiB used, 607 GiB / 626 GiB avail; 102 B/s rd, 45 KiB/s wr, 259 op/s; 62755/231597 objects degraded (27.097%); 14511/231597 objects misplaced (6.266%); 13 MiB/s, 203 objects/s recovering 2023-09-23T21:12:08.396 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:12:08.431 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:12:08.571 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:12:08 smithi026 ceph-mon[179688]: pgmap v1197: 119 pgs: 1 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 21 active+undersized, 1 active+recovering+undersized+degraded+remapped, 8 active+undersized+degraded, 8 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+degraded+remapped, 6 active+recovery_wait+degraded+remapped, 72 active+clean; 4.7 GiB data, 18 GiB used, 607 GiB / 626 GiB avail; 102 B/s rd, 45 KiB/s wr, 259 op/s; 62755/231597 objects degraded (27.097%); 14511/231597 objects misplaced (6.266%); 13 MiB/s, 203 objects/s recovering 2023-09-23T21:12:08.571 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:12:08 smithi026 ceph-mon[175151]: pgmap v1197: 119 pgs: 1 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 21 active+undersized, 1 active+recovering+undersized+degraded+remapped, 8 active+undersized+degraded, 8 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+degraded+remapped, 6 active+recovery_wait+degraded+remapped, 72 active+clean; 4.7 GiB data, 18 GiB used, 607 GiB / 626 GiB avail; 102 B/s rd, 45 KiB/s wr, 259 op/s; 62755/231597 objects degraded (27.097%); 14511/231597 objects misplaced (6.266%); 13 MiB/s, 203 objects/s recovering 2023-09-23T21:12:08.716 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:12:08.719 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:12:08.720 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:12:08.820 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:12:08.846 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:12:08.858 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:12:09.111 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T21:12:09.184 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:12:09.188 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:12:09.189 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:12:09.289 INFO:teuthology.orchestra.run.smithi026.stderr:nodeep-scrub is set 2023-09-23T21:12:09.290 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:12:09.349 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:12:09.498 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:12:09.501 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:12:09.502 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:12:09.502 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:12:09 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/1367528967' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:12:09.502 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:12:09 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:12:09.503 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:12:09 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/1367528967' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:12:09.503 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:12:09 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:12:09.603 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:12:09.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:12:09 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/1367528967' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:12:09.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:12:09 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:12:09.637 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:12:09.881 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:12:09 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mon-a[175127]: 2023-09-23T21:12:09.518+0000 7f1e2d487700 -1 mon.a@0(leader).osd e469 definitely_dead 0 2023-09-23T21:12:09.882 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 21:12:09 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[164029]: ::ffff:172.21.15.70 - - [23/Sep/2023:21:12:09] "GET /metrics HTTP/1.1" 200 36451 "" "Prometheus/2.43.0" 2023-09-23T21:12:09.930 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:12:09.934 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:12:09.934 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:12:10.035 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:12:10.094 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:12:10.236 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:12:10.240 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:12:10.240 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:12:10.273 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:12:10 smithi026 ceph-mon[175151]: pgmap v1198: 119 pgs: 1 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 21 active+undersized, 1 active+recovering+undersized+degraded+remapped, 8 active+undersized+degraded, 8 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+degraded+remapped, 6 active+recovery_wait+degraded+remapped, 72 active+clean; 4.7 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 87 B/s rd, 87 B/s wr, 278 op/s; 62071/229269 objects degraded (27.073%); 14208/229269 objects misplaced (6.197%); 13 MiB/s, 203 objects/s recovering 2023-09-23T21:12:10.273 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:12:10 smithi026 ceph-mon[175151]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:12:10.273 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:12:10 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T21:12:10.273 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:12:10 smithi026 ceph-mon[175151]: osdmap e469: 8 total, 7 up, 7 in 2023-09-23T21:12:10.273 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:12:10 smithi026 ceph-mon[175151]: failed to encode map e469 with expected crc 2023-09-23T21:12:10.273 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:12:10 smithi026 ceph-mon[175151]: failed to encode map e469 with expected crc 2023-09-23T21:12:10.288 INFO:teuthology.orchestra.run.smithi026.stderr:marked out osd.1. 2023-09-23T21:12:10.341 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:12:10.432 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:12:10.551 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:12:10 smithi026 ceph-mon[179688]: pgmap v1198: 119 pgs: 1 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 21 active+undersized, 1 active+recovering+undersized+degraded+remapped, 8 active+undersized+degraded, 8 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+degraded+remapped, 6 active+recovery_wait+degraded+remapped, 72 active+clean; 4.7 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 87 B/s rd, 87 B/s wr, 278 op/s; 62071/229269 objects degraded (27.073%); 14208/229269 objects misplaced (6.197%); 13 MiB/s, 203 objects/s recovering 2023-09-23T21:12:10.551 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:12:10 smithi026 ceph-mon[179688]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:12:10.551 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:12:10 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T21:12:10.551 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:12:10 smithi026 ceph-mon[179688]: osdmap e469: 8 total, 7 up, 7 in 2023-09-23T21:12:10.551 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:12:10 smithi026 ceph-mon[179688]: failed to encode map e469 with expected crc 2023-09-23T21:12:10.551 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:12:10 smithi026 ceph-mon[179688]: failed to encode map e469 with expected crc 2023-09-23T21:12:10.551 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:12:10 smithi026 ceph-mon[179688]: failed to encode map e469 with expected crc 2023-09-23T21:12:10.551 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:12:10 smithi026 ceph-mon[179688]: failed to encode map e469 with expected crc 2023-09-23T21:12:10.552 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:12:10 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/507705213' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["1"]}]: dispatch 2023-09-23T21:12:10.552 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:12:10 smithi026 ceph-mon[179688]: Client client.admin marked osd.1 out, while it was still marked up 2023-09-23T21:12:10.552 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:12:10 smithi026 ceph-mon[175151]: failed to encode map e469 with expected crc 2023-09-23T21:12:10.552 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:12:10 smithi026 ceph-mon[175151]: failed to encode map e469 with expected crc 2023-09-23T21:12:10.552 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:12:10 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/507705213' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["1"]}]: dispatch 2023-09-23T21:12:10.552 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:12:10 smithi026 ceph-mon[175151]: Client client.admin marked osd.1 out, while it was still marked up 2023-09-23T21:12:10.589 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:12:10.594 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:12:10 smithi070 ceph-mon[168282]: pgmap v1198: 119 pgs: 1 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 21 active+undersized, 1 active+recovering+undersized+degraded+remapped, 8 active+undersized+degraded, 8 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+degraded+remapped, 6 active+recovery_wait+degraded+remapped, 72 active+clean; 4.7 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 87 B/s rd, 87 B/s wr, 278 op/s; 62071/229269 objects degraded (27.073%); 14208/229269 objects misplaced (6.197%); 13 MiB/s, 203 objects/s recovering 2023-09-23T21:12:10.594 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:12:10 smithi070 ceph-mon[168282]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:12:10.594 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:12:10 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T21:12:10.594 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:12:10 smithi070 ceph-mon[168282]: osdmap e469: 8 total, 7 up, 7 in 2023-09-23T21:12:10.594 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:12:10 smithi070 ceph-mon[168282]: failed to encode map e469 with expected crc 2023-09-23T21:12:10.594 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:12:10 smithi070 ceph-mon[168282]: failed to encode map e469 with expected crc 2023-09-23T21:12:10.595 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:12:10 smithi070 ceph-mon[168282]: failed to encode map e469 with expected crc 2023-09-23T21:12:10.595 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:12:10 smithi070 ceph-mon[168282]: failed to encode map e469 with expected crc 2023-09-23T21:12:10.595 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:12:10 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/507705213' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["1"]}]: dispatch 2023-09-23T21:12:10.595 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:12:10 smithi070 ceph-mon[168282]: Client client.admin marked osd.1 out, while it was still marked up 2023-09-23T21:12:10.595 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:12:10.596 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:12:10.609 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1: {} 2023-09-23T21:12:10.640 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2: {} 2023-09-23T21:12:10.670 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3: {} 2023-09-23T21:12:10.688 INFO:teuthology.orchestra.run.smithi070.stdout:ERROR: (22) Invalid argument 2023-09-23T21:12:10.688 INFO:teuthology.orchestra.run.smithi070.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T21:12:10.695 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4: {} 2023-09-23T21:12:10.696 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:12:10.721 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T21:12:10.746 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:12:10.762 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6: {} 2023-09-23T21:12:10.790 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T21:12:10.790 INFO:teuthology.orchestra.run.smithi026.stderr:Error ENXIO: problem getting command descriptions from osd.0 2023-09-23T21:12:10.790 INFO:teuthology.orchestra.run.smithi026.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T21:12:10.791 INFO:teuthology.orchestra.run.smithi026.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T21:12:10.791 INFO:teuthology.orchestra.run.smithi026.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T21:12:10.791 INFO:teuthology.orchestra.run.smithi026.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T21:12:10.791 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T21:12:10.791 INFO:teuthology.orchestra.run.smithi026.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T21:12:10.791 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T21:12:10.987 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:12:10.991 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:12:10.992 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:12:11.074 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_historic_ops 2023-09-23T21:12:11.093 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:12:11.145 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:12:11.183 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:12:11.187 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:12:11.188 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:12:11.189 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T21:12:11.189 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T21:12:11.288 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:12:11.318 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:12:11.358 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:12:11.364 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:12:11.364 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:12:11.381 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:12:11 smithi026 ceph-mon[175151]: failed to encode map e470 with expected crc 2023-09-23T21:12:11.381 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:12:11 smithi026 ceph-mon[175151]: failed to encode map e470 with expected crc 2023-09-23T21:12:11.381 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:12:11 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/507705213' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["1"]}]': finished 2023-09-23T21:12:11.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:12:11 smithi026 ceph-mon[175151]: osdmap e470: 8 total, 7 up, 6 in 2023-09-23T21:12:11.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:12:11 smithi026 ceph-mon[175151]: failed to encode map e470 with expected crc 2023-09-23T21:12:11.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:12:11 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/4121462290' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:12:11.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:12:11 smithi026 ceph-mon[175151]: Health check update: Degraded data redundancy: 62071/229269 objects degraded (27.073%), 26 pgs degraded (PG_DEGRADED) 2023-09-23T21:12:11.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:12:11 smithi026 ceph-mon[175151]: failed to encode map e471 with expected crc 2023-09-23T21:12:11.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:12:11 smithi026 ceph-mon[175151]: osdmap e471: 8 total, 7 up, 6 in 2023-09-23T21:12:11.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:12:11 smithi026 ceph-mon[175151]: failed to encode map e471 with expected crc 2023-09-23T21:12:11.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:12:11 smithi026 ceph-mon[175151]: failed to encode map e471 with expected crc 2023-09-23T21:12:11.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:12:11 smithi026 ceph-mon[175151]: failed to encode map e471 with expected crc 2023-09-23T21:12:11.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:12:11 smithi026 ceph-mon[179688]: failed to encode map e470 with expected crc 2023-09-23T21:12:11.383 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:12:11 smithi026 ceph-mon[179688]: failed to encode map e470 with expected crc 2023-09-23T21:12:11.383 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:12:11 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/507705213' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["1"]}]': finished 2023-09-23T21:12:11.383 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:12:11 smithi026 ceph-mon[179688]: osdmap e470: 8 total, 7 up, 6 in 2023-09-23T21:12:11.383 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:12:11 smithi026 ceph-mon[179688]: failed to encode map e470 with expected crc 2023-09-23T21:12:11.383 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:12:11 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/4121462290' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:12:11.383 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:12:11 smithi026 ceph-mon[179688]: Health check update: Degraded data redundancy: 62071/229269 objects degraded (27.073%), 26 pgs degraded (PG_DEGRADED) 2023-09-23T21:12:11.383 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:12:11 smithi026 ceph-mon[179688]: failed to encode map e471 with expected crc 2023-09-23T21:12:11.383 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:12:11 smithi026 ceph-mon[179688]: osdmap e471: 8 total, 7 up, 6 in 2023-09-23T21:12:11.383 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:12:11 smithi026 ceph-mon[179688]: failed to encode map e471 with expected crc 2023-09-23T21:12:11.383 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:12:11 smithi026 ceph-mon[179688]: failed to encode map e471 with expected crc 2023-09-23T21:12:11.383 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:12:11 smithi026 ceph-mon[179688]: failed to encode map e471 with expected crc 2023-09-23T21:12:11.465 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:12:11.497 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:12:11.535 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:12:11.538 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:12:11.539 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:12:11.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:12:11 smithi070 ceph-mon[168282]: failed to encode map e470 with expected crc 2023-09-23T21:12:11.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:12:11 smithi070 ceph-mon[168282]: failed to encode map e470 with expected crc 2023-09-23T21:12:11.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:12:11 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/507705213' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["1"]}]': finished 2023-09-23T21:12:11.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:12:11 smithi070 ceph-mon[168282]: osdmap e470: 8 total, 7 up, 6 in 2023-09-23T21:12:11.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:12:11 smithi070 ceph-mon[168282]: failed to encode map e470 with expected crc 2023-09-23T21:12:11.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:12:11 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/4121462290' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:12:11.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:12:11 smithi070 ceph-mon[168282]: Health check update: Degraded data redundancy: 62071/229269 objects degraded (27.073%), 26 pgs degraded (PG_DEGRADED) 2023-09-23T21:12:11.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:12:11 smithi070 ceph-mon[168282]: failed to encode map e471 with expected crc 2023-09-23T21:12:11.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:12:11 smithi070 ceph-mon[168282]: osdmap e471: 8 total, 7 up, 6 in 2023-09-23T21:12:11.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:12:11 smithi070 ceph-mon[168282]: failed to encode map e471 with expected crc 2023-09-23T21:12:11.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:12:11 smithi070 ceph-mon[168282]: failed to encode map e471 with expected crc 2023-09-23T21:12:11.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:12:11 smithi070 ceph-mon[168282]: failed to encode map e471 with expected crc 2023-09-23T21:12:11.639 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:12:11.670 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:12:11.713 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:12:11.716 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:12:11.717 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:12:11.800 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T21:12:11.818 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:12:11.854 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:12:11.899 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:12:11.902 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:12:11.903 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:12:11.983 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T21:12:12.004 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:12:12.082 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:12:12.151 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:12:12.164 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:12:12.164 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:12:12.265 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:12:12.295 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:12:12.331 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:12:12.334 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:12:12.334 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:12:12.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:12:12 smithi070 ceph-mon[168282]: failed to encode map e471 with expected crc 2023-09-23T21:12:12.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:12:12 smithi070 ceph-mon[168282]: failed to encode map e471 with expected crc 2023-09-23T21:12:12.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:12:12 smithi070 ceph-mon[168282]: failed to encode map e471 with expected crc 2023-09-23T21:12:12.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:12:12 smithi070 ceph-mon[168282]: failed to encode map e471 with expected crc 2023-09-23T21:12:12.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:12:12 smithi070 ceph-mon[168282]: pgmap v1202: 119 pgs: 1 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 21 active+undersized, 1 active+recovering+undersized+degraded+remapped, 8 active+undersized+degraded, 8 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+degraded+remapped, 6 active+recovery_wait+degraded+remapped, 72 active+clean; 4.7 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 229 op/s; 62071/228714 objects degraded (27.139%); 14208/228714 objects misplaced (6.212%); 9.5 MiB/s, 151 objects/s recovering 2023-09-23T21:12:12.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:12:12 smithi070 ceph-mon[168282]: failed to encode map e472 with expected crc 2023-09-23T21:12:12.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:12:12 smithi070 ceph-mon[168282]: failed to encode map e472 with expected crc 2023-09-23T21:12:12.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:12:12 smithi070 ceph-mon[168282]: failed to encode map e472 with expected crc 2023-09-23T21:12:12.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:12:12 smithi070 ceph-mon[168282]: osdmap e472: 8 total, 7 up, 6 in 2023-09-23T21:12:12.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:12:12 smithi070 ceph-mon[168282]: failed to encode map e472 with expected crc 2023-09-23T21:12:12.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:12:12 smithi070 ceph-mon[168282]: failed to encode map e472 with expected crc 2023-09-23T21:12:12.435 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:12:12.466 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:12:12.616 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:12:12.631 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:12:12 smithi026 ceph-mon[175151]: failed to encode map e471 with expected crc 2023-09-23T21:12:12.631 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:12:12 smithi026 ceph-mon[175151]: failed to encode map e471 with expected crc 2023-09-23T21:12:12.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:12:12 smithi026 ceph-mon[175151]: failed to encode map e471 with expected crc 2023-09-23T21:12:12.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:12:12 smithi026 ceph-mon[175151]: failed to encode map e471 with expected crc 2023-09-23T21:12:12.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:12:12 smithi026 ceph-mon[175151]: pgmap v1202: 119 pgs: 1 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 21 active+undersized, 1 active+recovering+undersized+degraded+remapped, 8 active+undersized+degraded, 8 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+degraded+remapped, 6 active+recovery_wait+degraded+remapped, 72 active+clean; 4.7 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 229 op/s; 62071/228714 objects degraded (27.139%); 14208/228714 objects misplaced (6.212%); 9.5 MiB/s, 151 objects/s recovering 2023-09-23T21:12:12.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:12:12 smithi026 ceph-mon[175151]: failed to encode map e472 with expected crc 2023-09-23T21:12:12.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:12:12 smithi026 ceph-mon[175151]: failed to encode map e472 with expected crc 2023-09-23T21:12:12.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:12:12 smithi026 ceph-mon[175151]: failed to encode map e472 with expected crc 2023-09-23T21:12:12.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:12:12 smithi026 ceph-mon[175151]: osdmap e472: 8 total, 7 up, 6 in 2023-09-23T21:12:12.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:12:12 smithi026 ceph-mon[175151]: failed to encode map e472 with expected crc 2023-09-23T21:12:12.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:12:12 smithi026 ceph-mon[175151]: failed to encode map e472 with expected crc 2023-09-23T21:12:12.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:12:12 smithi026 ceph-mon[179688]: failed to encode map e471 with expected crc 2023-09-23T21:12:12.633 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:12:12 smithi026 ceph-mon[179688]: failed to encode map e471 with expected crc 2023-09-23T21:12:12.633 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:12:12 smithi026 ceph-mon[179688]: failed to encode map e471 with expected crc 2023-09-23T21:12:12.633 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:12:12 smithi026 ceph-mon[179688]: failed to encode map e471 with expected crc 2023-09-23T21:12:12.633 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:12:12 smithi026 ceph-mon[179688]: pgmap v1202: 119 pgs: 1 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 21 active+undersized, 1 active+recovering+undersized+degraded+remapped, 8 active+undersized+degraded, 8 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+degraded+remapped, 6 active+recovery_wait+degraded+remapped, 72 active+clean; 4.7 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 229 op/s; 62071/228714 objects degraded (27.139%); 14208/228714 objects misplaced (6.212%); 9.5 MiB/s, 151 objects/s recovering 2023-09-23T21:12:12.633 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:12:12 smithi026 ceph-mon[179688]: failed to encode map e472 with expected crc 2023-09-23T21:12:12.633 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:12:12 smithi026 ceph-mon[179688]: failed to encode map e472 with expected crc 2023-09-23T21:12:12.633 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:12:12 smithi026 ceph-mon[179688]: failed to encode map e472 with expected crc 2023-09-23T21:12:12.633 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:12:12 smithi026 ceph-mon[179688]: osdmap e472: 8 total, 7 up, 6 in 2023-09-23T21:12:12.633 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:12:12 smithi026 ceph-mon[179688]: failed to encode map e472 with expected crc 2023-09-23T21:12:12.633 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:12:12 smithi026 ceph-mon[179688]: failed to encode map e472 with expected crc 2023-09-23T21:12:12.812 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:12:12.816 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:12:12.817 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:12:12.917 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:12:12.952 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:12:13.104 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:12:13.108 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:12:13.109 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:12:13.191 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T21:12:13.210 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:12:13.262 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:12:13.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:12:13 smithi070 ceph-mon[168282]: failed to encode map e472 with expected crc 2023-09-23T21:12:13.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:12:13 smithi070 ceph-mon[168282]: failed to encode map e472 with expected crc 2023-09-23T21:12:13.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:12:13 smithi070 ceph-mon[168282]: failed to encode map e473 with expected crc 2023-09-23T21:12:13.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:12:13 smithi070 ceph-mon[168282]: osdmap e473: 8 total, 7 up, 6 in 2023-09-23T21:12:13.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:12:13 smithi070 ceph-mon[168282]: failed to encode map e473 with expected crc 2023-09-23T21:12:13.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:12:13 smithi070 ceph-mon[168282]: failed to encode map e473 with expected crc 2023-09-23T21:12:13.507 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:12:13.510 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:12:13 smithi026 ceph-mon[179688]: failed to encode map e472 with expected crc 2023-09-23T21:12:13.510 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:12:13 smithi026 ceph-mon[179688]: failed to encode map e472 with expected crc 2023-09-23T21:12:13.511 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:12:13 smithi026 ceph-mon[179688]: failed to encode map e473 with expected crc 2023-09-23T21:12:13.511 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:12:13 smithi026 ceph-mon[179688]: osdmap e473: 8 total, 7 up, 6 in 2023-09-23T21:12:13.511 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:12:13 smithi026 ceph-mon[179688]: failed to encode map e473 with expected crc 2023-09-23T21:12:13.511 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:12:13 smithi026 ceph-mon[179688]: failed to encode map e473 with expected crc 2023-09-23T21:12:13.511 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:12:13 smithi026 ceph-mon[175151]: failed to encode map e472 with expected crc 2023-09-23T21:12:13.511 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:12:13 smithi026 ceph-mon[175151]: failed to encode map e472 with expected crc 2023-09-23T21:12:13.511 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:12:13 smithi026 ceph-mon[175151]: failed to encode map e473 with expected crc 2023-09-23T21:12:13.511 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:12:13 smithi026 ceph-mon[175151]: osdmap e473: 8 total, 7 up, 6 in 2023-09-23T21:12:13.511 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:12:13 smithi026 ceph-mon[175151]: failed to encode map e473 with expected crc 2023-09-23T21:12:13.512 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:12:13 smithi026 ceph-mon[175151]: failed to encode map e473 with expected crc 2023-09-23T21:12:13.512 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:12:13.512 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:12:13.613 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:12:13.641 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:12:13.771 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:12:13.776 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:12:13.777 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:12:13.877 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:12:13.929 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:12:13.981 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:12:13.991 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:12:13.991 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:12:14.092 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:12:14.131 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:12:14.183 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:12:14.187 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:12:14.188 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:12:14.235 INFO:teuthology.orchestra.run.smithi070.stdout:{ 2023-09-23T21:12:14.236 INFO:teuthology.orchestra.run.smithi070.stdout: "size": 20, 2023-09-23T21:12:14.236 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 600, 2023-09-23T21:12:14.236 INFO:teuthology.orchestra.run.smithi070.stdout: "ops": [ 2023-09-23T21:12:14.236 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:14.236 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.35469.0:198190 8.19 8:9a4e90e2:::benchmark_data_smithi026_301269_object87433:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e332)", 2023-09-23T21:12:14.237 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:05:03.300673+0000", 2023-09-23T21:12:14.237 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 430.93173375999999, 2023-09-23T21:12:14.237 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.05033683, 2023-09-23T21:12:14.237 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:12:14.237 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:12:14.237 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:12:14.238 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.35469", 2023-09-23T21:12:14.238 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/4041424623", 2023-09-23T21:12:14.238 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 198190 2023-09-23T21:12:14.238 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:14.238 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:12:14.238 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:14.239 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:12:14.239 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:03.300673+0000", 2023-09-23T21:12:14.239 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:12:14.239 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:14.239 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:14.239 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:12:14.240 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:03.300673+0000", 2023-09-23T21:12:14.240 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:12:14.240 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:14.240 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:14.240 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:12:14.241 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:03.300671+0000", 2023-09-23T21:12:14.241 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999981 2023-09-23T21:12:14.241 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:14.241 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:14.241 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:12:14.241 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:03.300677+0000", 2023-09-23T21:12:14.241 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.964e-06 2023-09-23T21:12:14.241 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:14.242 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:14.242 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:12:14.242 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:03.300688+0000", 2023-09-23T21:12:14.242 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.117e-05 2023-09-23T21:12:14.242 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:14.242 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:14.243 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:12:14.243 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:03.300696+0000", 2023-09-23T21:12:14.243 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.294e-06 2023-09-23T21:12:14.243 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:14.243 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:14.243 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:12:14.243 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:03.300788+0000", 2023-09-23T21:12:14.244 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.2665999999999997e-05 2023-09-23T21:12:14.244 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:14.244 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:14.244 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:12:14.244 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:03.300798+0000", 2023-09-23T21:12:14.244 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.4800000000000007e-06 2023-09-23T21:12:14.245 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:14.245 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:14.245 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:12:14.245 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:04.333049+0000", 2023-09-23T21:12:14.245 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.0322515969999999 2023-09-23T21:12:14.245 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:14.246 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:14.246 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:12:14.246 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:04.333186+0000", 2023-09-23T21:12:14.246 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00013625000000000001 2023-09-23T21:12:14.246 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:14.246 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:14.246 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 0,4", 2023-09-23T21:12:14.247 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:04.333637+0000", 2023-09-23T21:12:14.247 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00045131799999999998 2023-09-23T21:12:14.247 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:14.247 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:14.247 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:12:14.247 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:04.342090+0000", 2023-09-23T21:12:14.247 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0084528899999999994 2023-09-23T21:12:14.247 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:14.248 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:14.248 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:12:14.248 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:04.342101+0000", 2023-09-23T21:12:14.248 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.1275e-05 2023-09-23T21:12:14.248 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:14.248 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:14.248 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:12:14.249 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:04.350919+0000", 2023-09-23T21:12:14.249 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0088174489999999998 2023-09-23T21:12:14.249 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:14.249 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:14.249 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:12:14.249 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:04.350968+0000", 2023-09-23T21:12:14.249 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.9651000000000003e-05 2023-09-23T21:12:14.249 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:14.250 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:14.250 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:12:14.250 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:04.351010+0000", 2023-09-23T21:12:14.250 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.1616000000000003e-05 2023-09-23T21:12:14.250 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:12:14.250 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:12:14.250 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:12:14.251 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:14.251 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:14.251 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.35469.0:198232 8.19 8:9c7931dd:::benchmark_data_smithi026_301269_object87475:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e332)", 2023-09-23T21:12:14.251 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:05:03.301139+0000", 2023-09-23T21:12:14.251 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 430.93126772300002, 2023-09-23T21:12:14.251 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.055824807, 2023-09-23T21:12:14.251 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:12:14.251 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:12:14.252 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:12:14.252 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.35469", 2023-09-23T21:12:14.252 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/4041424623", 2023-09-23T21:12:14.252 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 198232 2023-09-23T21:12:14.252 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:14.252 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:12:14.252 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:14.253 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:12:14.253 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:03.301139+0000", 2023-09-23T21:12:14.253 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:12:14.253 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:14.253 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:14.253 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:12:14.253 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:03.301139+0000", 2023-09-23T21:12:14.254 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:12:14.254 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:14.254 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:14.254 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:12:14.254 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:03.301139+0000", 2023-09-23T21:12:14.254 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999995 2023-09-23T21:12:14.254 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:14.255 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:14.255 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:12:14.255 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:03.301141+0000", 2023-09-23T21:12:14.255 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.1739999999999999e-06 2023-09-23T21:12:14.255 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:14.255 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:14.255 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:12:14.256 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:03.301147+0000", 2023-09-23T21:12:14.256 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.4740000000000002e-06 2023-09-23T21:12:14.256 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:14.256 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:14.256 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:12:14.256 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:03.301151+0000", 2023-09-23T21:12:14.256 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.0500000000000002e-06 2023-09-23T21:12:14.257 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:14.257 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:14.257 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:12:14.257 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:03.301245+0000", 2023-09-23T21:12:14.257 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.3345999999999997e-05 2023-09-23T21:12:14.257 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:14.257 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:14.258 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:12:14.258 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:03.301250+0000", 2023-09-23T21:12:14.258 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.2920000000000003e-06 2023-09-23T21:12:14.258 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:14.258 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:14.258 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:12:14.258 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:04.342298+0000", 2023-09-23T21:12:14.259 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.0410477309999999 2023-09-23T21:12:14.259 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:14.259 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:14.259 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:12:14.259 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:04.342402+0000", 2023-09-23T21:12:14.259 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000104706 2023-09-23T21:12:14.259 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:14.259 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:14.260 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 0,4", 2023-09-23T21:12:14.260 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:04.342653+0000", 2023-09-23T21:12:14.260 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00025070499999999999 2023-09-23T21:12:14.260 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:14.260 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:14.260 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:12:14.260 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:04.350875+0000", 2023-09-23T21:12:14.261 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0082216129999999991 2023-09-23T21:12:14.261 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:14.261 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:14.261 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:12:14.261 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:04.351422+0000", 2023-09-23T21:12:14.261 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00054713199999999998 2023-09-23T21:12:14.261 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:14.262 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:14.262 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:12:14.262 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:04.356875+0000", 2023-09-23T21:12:14.262 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.005452854 2023-09-23T21:12:14.262 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:14.262 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:14.263 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:12:14.263 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:04.356917+0000", 2023-09-23T21:12:14.263 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.2585e-05 2023-09-23T21:12:14.263 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:14.263 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:14.263 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:12:14.263 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:04.356964+0000", 2023-09-23T21:12:14.264 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.6529000000000001e-05 2023-09-23T21:12:14.264 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:12:14.264 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:12:14.264 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:12:14.264 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:14.264 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:14.264 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.35469.0:198310 8.19 8:9d32f5d8:::benchmark_data_smithi026_301269_object87553:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e332)", 2023-09-23T21:12:14.265 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:05:03.301156+0000", 2023-09-23T21:12:14.265 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 430.93125046400002, 2023-09-23T21:12:14.265 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.055943659, 2023-09-23T21:12:14.265 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:12:14.265 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:12:14.265 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:12:14.265 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.35469", 2023-09-23T21:12:14.265 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/4041424623", 2023-09-23T21:12:14.266 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 198310 2023-09-23T21:12:14.266 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:14.266 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:12:14.266 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:14.266 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:12:14.266 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:03.301156+0000", 2023-09-23T21:12:14.266 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:12:14.267 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:14.267 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:14.267 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:12:14.267 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:03.301156+0000", 2023-09-23T21:12:14.267 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:12:14.267 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:14.267 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:14.267 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:12:14.268 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:03.301156+0000", 2023-09-23T21:12:14.268 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999995 2023-09-23T21:12:14.268 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:14.268 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:14.268 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:12:14.268 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:03.301158+0000", 2023-09-23T21:12:14.268 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.114e-06 2023-09-23T21:12:14.269 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:14.269 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:14.269 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:12:14.269 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:03.301165+0000", 2023-09-23T21:12:14.269 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.1770000000000001e-06 2023-09-23T21:12:14.269 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:14.269 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:14.270 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:12:14.270 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:03.301170+0000", 2023-09-23T21:12:14.270 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.6530000000000002e-06 2023-09-23T21:12:14.270 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:14.270 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:14.270 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:12:14.270 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:03.301277+0000", 2023-09-23T21:12:14.271 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000106681 2023-09-23T21:12:14.271 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:14.271 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:14.271 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:12:14.271 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:03.301285+0000", 2023-09-23T21:12:14.271 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.0029999999999994e-06 2023-09-23T21:12:14.271 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:14.272 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:14.272 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:12:14.272 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:04.344519+0000", 2023-09-23T21:12:14.272 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.04323455 2023-09-23T21:12:14.272 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:14.272 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:14.272 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:12:14.273 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:04.344610+0000", 2023-09-23T21:12:14.273 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.0427999999999995e-05 2023-09-23T21:12:14.273 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:14.273 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:14.273 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 0,4", 2023-09-23T21:12:14.273 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:04.344853+0000", 2023-09-23T21:12:14.273 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00024303199999999999 2023-09-23T21:12:14.274 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:14.274 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:14.274 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:12:14.274 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:04.351020+0000", 2023-09-23T21:12:14.274 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0061668469999999996 2023-09-23T21:12:14.274 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:14.274 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:14.275 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:12:14.275 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:04.351454+0000", 2023-09-23T21:12:14.275 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00043414499999999998 2023-09-23T21:12:14.275 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:14.275 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:14.275 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:12:14.276 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:04.357055+0000", 2023-09-23T21:12:14.276 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0056014849999999998 2023-09-23T21:12:14.276 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:14.276 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:14.276 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:12:14.276 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:04.357080+0000", 2023-09-23T21:12:14.276 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.5134999999999999e-05 2023-09-23T21:12:14.277 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:14.277 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:14.277 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:12:14.277 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:04.357100+0000", 2023-09-23T21:12:14.277 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.9724e-05 2023-09-23T21:12:14.277 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:12:14.277 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:12:14.277 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:12:14.278 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:14.278 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:14.278 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.35469.0:198331 8.19 8:9d0af17f:::benchmark_data_smithi026_301269_object87574:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e332)", 2023-09-23T21:12:14.278 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:05:03.301175+0000", 2023-09-23T21:12:14.278 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 430.93123145099997, 2023-09-23T21:12:14.278 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.0560283180000001, 2023-09-23T21:12:14.281 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:12:14.281 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:12:14.281 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:12:14.281 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.35469", 2023-09-23T21:12:14.282 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/4041424623", 2023-09-23T21:12:14.282 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 198331 2023-09-23T21:12:14.282 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:14.282 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:12:14.282 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:14.282 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:12:14.282 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:03.301175+0000", 2023-09-23T21:12:14.282 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:12:14.282 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:14.282 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:14.282 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:12:14.282 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:03.301175+0000", 2023-09-23T21:12:14.283 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:12:14.283 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:14.283 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:14.283 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:12:14.283 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:03.301175+0000", 2023-09-23T21:12:14.283 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999995 2023-09-23T21:12:14.283 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:14.283 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:14.283 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:12:14.283 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:03.301177+0000", 2023-09-23T21:12:14.283 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.148e-06 2023-09-23T21:12:14.283 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:14.284 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:14.284 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:12:14.284 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:03.301183+0000", 2023-09-23T21:12:14.284 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.9429999999999997e-06 2023-09-23T21:12:14.284 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:14.284 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:14.284 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:12:14.284 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:03.301187+0000", 2023-09-23T21:12:14.284 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.5279999999999999e-06 2023-09-23T21:12:14.284 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:14.284 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:14.284 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:12:14.285 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:03.301315+0000", 2023-09-23T21:12:14.285 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000128264 2023-09-23T21:12:14.285 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:14.285 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:14.285 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:12:14.285 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:03.301321+0000", 2023-09-23T21:12:14.285 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.852e-06 2023-09-23T21:12:14.285 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:14.285 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:14.285 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:12:14.285 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:04.346735+0000", 2023-09-23T21:12:14.285 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.0454144299999999 2023-09-23T21:12:14.286 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:14.286 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:14.286 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:12:14.286 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:04.346844+0000", 2023-09-23T21:12:14.286 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00010883 2023-09-23T21:12:14.286 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:14.286 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:14.286 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 0,4", 2023-09-23T21:12:14.286 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:04.347122+0000", 2023-09-23T21:12:14.286 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000278297 2023-09-23T21:12:14.286 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:14.286 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:14.287 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:12:14.287 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:04.351024+0000", 2023-09-23T21:12:14.287 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0039017230000000002 2023-09-23T21:12:14.287 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:14.287 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:14.287 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:12:14.287 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:04.356836+0000", 2023-09-23T21:12:14.287 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.005812004 2023-09-23T21:12:14.287 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:14.287 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:14.287 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:12:14.287 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:04.357160+0000", 2023-09-23T21:12:14.287 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00032355099999999999 2023-09-23T21:12:14.288 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:14.288 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:14.288 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:12:14.288 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:04.357184+0000", 2023-09-23T21:12:14.288 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.3943000000000001e-05 2023-09-23T21:12:14.288 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:14.288 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:14.288 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:12:14.288 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:04.357204+0000", 2023-09-23T21:12:14.288 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.0129999999999999e-05 2023-09-23T21:12:14.288 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:12:14.289 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:12:14.289 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:12:14.289 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:14.289 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:14.289 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.35469.0:198336 8.19 8:9cb16749:::benchmark_data_smithi026_301269_object87579:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e332)", 2023-09-23T21:12:14.289 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:05:03.301263+0000", 2023-09-23T21:12:14.289 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 430.93114333099999, 2023-09-23T21:12:14.289 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.056033843, 2023-09-23T21:12:14.289 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:12:14.289 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:12:14.289 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:12:14.289 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.35469", 2023-09-23T21:12:14.290 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/4041424623", 2023-09-23T21:12:14.290 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 198336 2023-09-23T21:12:14.290 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:14.290 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:12:14.290 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:14.290 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:12:14.290 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:03.301263+0000", 2023-09-23T21:12:14.290 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:12:14.290 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:14.290 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:14.290 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:12:14.290 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:03.301263+0000", 2023-09-23T21:12:14.291 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:12:14.291 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:14.291 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:14.291 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:12:14.291 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:03.301263+0000", 2023-09-23T21:12:14.291 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999995 2023-09-23T21:12:14.291 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:14.291 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:14.291 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:12:14.291 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:03.301267+0000", 2023-09-23T21:12:14.291 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.6229999999999999e-06 2023-09-23T21:12:14.292 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:14.292 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:14.292 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:12:14.292 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:03.301274+0000", 2023-09-23T21:12:14.292 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.6559999999999992e-06 2023-09-23T21:12:14.292 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:14.292 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:14.292 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:12:14.292 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:03.301280+0000", 2023-09-23T21:12:14.292 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.003e-06 2023-09-23T21:12:14.292 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:14.292 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:14.293 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:12:14.293 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:03.301344+0000", 2023-09-23T21:12:14.293 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.4073999999999997e-05 2023-09-23T21:12:14.293 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:14.293 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:14.293 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:12:14.293 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:03.301350+0000", 2023-09-23T21:12:14.293 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.8869999999999997e-06 2023-09-23T21:12:14.293 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:14.293 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:14.293 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:12:14.293 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:04.349006+0000", 2023-09-23T21:12:14.293 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.047656132 2023-09-23T21:12:14.294 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:14.294 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:14.294 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:12:14.294 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:04.349116+0000", 2023-09-23T21:12:14.294 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000109882 2023-09-23T21:12:14.294 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:14.294 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:14.294 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 0,4", 2023-09-23T21:12:14.294 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:04.349418+0000", 2023-09-23T21:12:14.294 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00030181400000000002 2023-09-23T21:12:14.294 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:14.294 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:14.295 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:12:14.295 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:04.351050+0000", 2023-09-23T21:12:14.295 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.001631785 2023-09-23T21:12:14.295 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:14.295 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:14.295 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:12:14.295 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:04.357031+0000", 2023-09-23T21:12:14.295 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0059814009999999999 2023-09-23T21:12:14.295 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:14.295 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:14.295 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:12:14.295 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:04.357254+0000", 2023-09-23T21:12:14.295 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000223293 2023-09-23T21:12:14.296 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:14.296 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:14.296 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:12:14.296 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:04.357278+0000", 2023-09-23T21:12:14.296 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.3859e-05 2023-09-23T21:12:14.296 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:14.296 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:14.296 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:12:14.296 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:04.357297+0000", 2023-09-23T21:12:14.296 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.9029e-05 2023-09-23T21:12:14.296 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:12:14.297 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:12:14.297 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:12:14.297 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:14.297 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:14.297 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.35469.0:198386 8.19 8:9890a619:::benchmark_data_smithi026_301269_object87629:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e332)", 2023-09-23T21:12:14.297 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:05:03.301286+0000", 2023-09-23T21:12:14.297 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 430.93112049000001, 2023-09-23T21:12:14.297 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.0560852510000001, 2023-09-23T21:12:14.297 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:12:14.297 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:12:14.297 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:12:14.297 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.35469", 2023-09-23T21:12:14.297 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/4041424623", 2023-09-23T21:12:14.298 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 198386 2023-09-23T21:12:14.298 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:14.298 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:12:14.298 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:14.298 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:12:14.298 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:03.301286+0000", 2023-09-23T21:12:14.298 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:12:14.298 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:14.298 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:14.298 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:12:14.298 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:03.301286+0000", 2023-09-23T21:12:14.298 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:12:14.299 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:14.299 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:14.299 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:12:14.299 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:03.301286+0000", 2023-09-23T21:12:14.299 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999995 2023-09-23T21:12:14.299 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:14.299 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:14.299 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:12:14.299 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:03.301288+0000", 2023-09-23T21:12:14.299 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.4449999999999999e-06 2023-09-23T21:12:14.299 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:14.299 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:14.300 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:12:14.300 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:03.301294+0000", 2023-09-23T21:12:14.300 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.9710000000000001e-06 2023-09-23T21:12:14.300 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:14.300 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:14.300 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:12:14.300 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:03.301298+0000", 2023-09-23T21:12:14.300 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.8600000000000003e-06 2023-09-23T21:12:14.300 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:14.300 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:14.300 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:12:14.300 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:03.301366+0000", 2023-09-23T21:12:14.301 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.7865000000000002e-05 2023-09-23T21:12:14.301 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:14.301 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:14.301 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:12:14.301 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:03.301371+0000", 2023-09-23T21:12:14.301 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.6530000000000002e-06 2023-09-23T21:12:14.301 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:14.301 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:14.301 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:12:14.301 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:04.351477+0000", 2023-09-23T21:12:14.301 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.0501064470000001 2023-09-23T21:12:14.301 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:14.301 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:14.302 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:12:14.302 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:04.351588+0000", 2023-09-23T21:12:14.302 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000110401 2023-09-23T21:12:14.302 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:14.302 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:14.302 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 0,4", 2023-09-23T21:12:14.302 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:04.351871+0000", 2023-09-23T21:12:14.302 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00028389700000000002 2023-09-23T21:12:14.302 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:14.302 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:14.302 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:12:14.303 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:04.356979+0000", 2023-09-23T21:12:14.303 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0051072510000000002 2023-09-23T21:12:14.303 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:14.303 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:14.303 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:12:14.303 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:04.357138+0000", 2023-09-23T21:12:14.303 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00015906399999999999 2023-09-23T21:12:14.303 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:14.303 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:14.303 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:12:14.303 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:04.357323+0000", 2023-09-23T21:12:14.303 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00018522100000000001 2023-09-23T21:12:14.304 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:14.304 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:14.304 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:12:14.304 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:04.357348+0000", 2023-09-23T21:12:14.304 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.5401999999999998e-05 2023-09-23T21:12:14.304 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:14.304 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:14.304 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:12:14.304 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:04.357372+0000", 2023-09-23T21:12:14.304 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.3170999999999999e-05 2023-09-23T21:12:14.304 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:12:14.305 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:12:14.305 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:12:14.305 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:14.305 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:14.305 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.35469.0:198473 8.19 8:9dfe5eb1:::benchmark_data_smithi026_301269_object87716:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e332)", 2023-09-23T21:12:14.305 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:05:03.301346+0000", 2023-09-23T21:12:14.305 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 430.93106081600001, 2023-09-23T21:12:14.305 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.0560949470000001, 2023-09-23T21:12:14.305 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:12:14.305 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:12:14.305 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:12:14.305 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.35469", 2023-09-23T21:12:14.306 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/4041424623", 2023-09-23T21:12:14.306 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 198473 2023-09-23T21:12:14.306 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:14.306 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:12:14.306 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:14.306 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:12:14.306 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:03.301346+0000", 2023-09-23T21:12:14.306 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:12:14.306 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:14.306 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:14.306 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:12:14.306 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:03.301346+0000", 2023-09-23T21:12:14.307 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:12:14.307 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:14.307 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:14.307 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:12:14.307 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:03.301345+0000", 2023-09-23T21:12:14.307 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999995 2023-09-23T21:12:14.307 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:14.307 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:14.307 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:12:14.307 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:03.301349+0000", 2023-09-23T21:12:14.307 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.8800000000000001e-06 2023-09-23T21:12:14.307 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:14.307 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:14.308 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:12:14.308 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:03.301358+0000", 2023-09-23T21:12:14.308 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.3930000000000008e-06 2023-09-23T21:12:14.308 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:14.308 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:14.308 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:12:14.308 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:03.301364+0000", 2023-09-23T21:12:14.308 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.2530000000000001e-06 2023-09-23T21:12:14.308 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:14.308 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:14.308 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:12:14.308 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:03.301384+0000", 2023-09-23T21:12:14.308 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.9896000000000001e-05 2023-09-23T21:12:14.309 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:14.309 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:14.309 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:12:14.309 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:03.301386+0000", 2023-09-23T21:12:14.309 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.6879999999999999e-06 2023-09-23T21:12:14.309 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:14.309 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:14.309 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:12:14.309 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:04.354056+0000", 2023-09-23T21:12:14.309 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.0526691889999999 2023-09-23T21:12:14.309 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:14.309 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:14.309 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:12:14.310 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:04.354187+0000", 2023-09-23T21:12:14.310 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000131192 2023-09-23T21:12:14.310 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:14.310 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:14.310 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 0,4", 2023-09-23T21:12:14.310 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:04.354563+0000", 2023-09-23T21:12:14.310 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000375798 2023-09-23T21:12:14.310 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:14.310 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:14.310 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:12:14.310 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:04.357109+0000", 2023-09-23T21:12:14.311 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0025466719999999998 2023-09-23T21:12:14.311 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:14.311 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:14.311 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:12:14.311 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:04.357230+0000", 2023-09-23T21:12:14.311 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000120228 2023-09-23T21:12:14.311 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:14.311 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:14.311 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:12:14.311 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:04.357401+0000", 2023-09-23T21:12:14.312 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000171146 2023-09-23T21:12:14.312 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:14.312 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:14.312 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:12:14.312 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:04.357423+0000", 2023-09-23T21:12:14.312 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.1885000000000001e-05 2023-09-23T21:12:14.312 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:14.312 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:14.312 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:12:14.312 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:04.357441+0000", 2023-09-23T21:12:14.312 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.8423e-05 2023-09-23T21:12:14.312 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:12:14.313 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:12:14.313 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:12:14.313 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:14.313 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:14.313 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.35469.0:198478 8.19 8:9d4307fd:::benchmark_data_smithi026_301269_object87721:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e332)", 2023-09-23T21:12:14.313 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:05:03.301369+0000", 2023-09-23T21:12:14.313 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 430.93103793, 2023-09-23T21:12:14.313 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.066125588, 2023-09-23T21:12:14.313 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:12:14.313 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:12:14.313 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:12:14.313 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.35469", 2023-09-23T21:12:14.313 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/4041424623", 2023-09-23T21:12:14.314 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 198478 2023-09-23T21:12:14.314 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:14.314 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:12:14.314 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:14.314 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:12:14.314 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:03.301369+0000", 2023-09-23T21:12:14.314 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:12:14.314 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:14.314 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:14.314 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:12:14.314 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:03.301369+0000", 2023-09-23T21:12:14.315 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:12:14.315 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:14.315 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:14.315 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:12:14.315 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:03.301368+0000", 2023-09-23T21:12:14.315 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999995 2023-09-23T21:12:14.315 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:14.315 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:14.315 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:12:14.315 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:03.301371+0000", 2023-09-23T21:12:14.315 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.548e-06 2023-09-23T21:12:14.315 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:14.315 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:14.316 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:12:14.316 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:03.301378+0000", 2023-09-23T21:12:14.316 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.2450000000000001e-06 2023-09-23T21:12:14.316 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:14.316 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:14.316 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:12:14.316 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:03.301383+0000", 2023-09-23T21:12:14.316 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.2350000000000001e-06 2023-09-23T21:12:14.316 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:14.316 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:14.316 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:12:14.316 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:03.301396+0000", 2023-09-23T21:12:14.317 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.3183999999999999e-05 2023-09-23T21:12:14.317 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:14.317 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:14.317 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:12:14.317 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:03.301398+0000", 2023-09-23T21:12:14.317 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.745e-06 2023-09-23T21:12:14.317 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:14.317 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:14.317 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:12:14.317 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:04.357463+0000", 2023-09-23T21:12:14.317 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.0560645749999999 2023-09-23T21:12:14.317 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:14.317 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:14.318 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:12:14.318 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:04.357537+0000", 2023-09-23T21:12:14.318 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.4395999999999997e-05 2023-09-23T21:12:14.318 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:14.318 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:14.318 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 0,4", 2023-09-23T21:12:14.318 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:04.357798+0000", 2023-09-23T21:12:14.318 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00026083399999999999 2023-09-23T21:12:14.318 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:14.318 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:14.318 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:12:14.318 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:04.367326+0000", 2023-09-23T21:12:14.319 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0095279379999999997 2023-09-23T21:12:14.319 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:14.319 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:14.319 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:12:14.319 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:04.367342+0000", 2023-09-23T21:12:14.319 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.5982999999999999e-05 2023-09-23T21:12:14.319 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:14.319 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:14.319 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:12:14.319 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:04.367396+0000", 2023-09-23T21:12:14.319 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.4110999999999997e-05 2023-09-23T21:12:14.319 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:14.320 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:14.320 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:12:14.320 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:04.367446+0000", 2023-09-23T21:12:14.320 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.9784999999999999e-05 2023-09-23T21:12:14.320 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:14.320 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:14.320 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:12:14.320 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:04.367494+0000", 2023-09-23T21:12:14.320 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.8433000000000002e-05 2023-09-23T21:12:14.320 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:12:14.320 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:12:14.321 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:12:14.321 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:14.321 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:14.321 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.35469.0:198487 8.19 8:9ba99d26:::benchmark_data_smithi026_301269_object87730:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e332)", 2023-09-23T21:12:14.321 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:05:03.301387+0000", 2023-09-23T21:12:14.321 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 430.93101953500002, 2023-09-23T21:12:14.321 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.1069283030000001, 2023-09-23T21:12:14.321 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:12:14.321 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:12:14.321 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:12:14.321 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.35469", 2023-09-23T21:12:14.321 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/4041424623", 2023-09-23T21:12:14.322 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 198487 2023-09-23T21:12:14.322 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:14.322 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:12:14.322 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:14.322 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:12:14.322 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:03.301387+0000", 2023-09-23T21:12:14.322 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:12:14.322 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:14.322 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:14.322 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:12:14.322 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:03.301387+0000", 2023-09-23T21:12:14.322 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:12:14.323 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:14.323 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:14.323 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:12:14.323 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:03.301387+0000", 2023-09-23T21:12:14.323 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999995 2023-09-23T21:12:14.323 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:14.323 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:14.323 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:12:14.323 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:03.301393+0000", 2023-09-23T21:12:14.323 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.1210000000000003e-06 2023-09-23T21:12:14.323 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:14.323 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:14.324 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:12:14.324 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:03.301400+0000", 2023-09-23T21:12:14.324 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.7469999999999999e-06 2023-09-23T21:12:14.324 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:14.324 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:14.324 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:12:14.324 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:03.301404+0000", 2023-09-23T21:12:14.324 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.4020000000000004e-06 2023-09-23T21:12:14.324 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:14.324 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:14.324 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:12:14.324 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:03.301482+0000", 2023-09-23T21:12:14.324 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.8005999999999998e-05 2023-09-23T21:12:14.325 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:14.325 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:14.325 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:12:14.325 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:03.301488+0000", 2023-09-23T21:12:14.325 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.9789999999999998e-06 2023-09-23T21:12:14.325 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:14.325 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:14.325 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:12:14.325 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:04.359818+0000", 2023-09-23T21:12:14.325 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.058330124 2023-09-23T21:12:14.325 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:14.325 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:14.326 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:12:14.326 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:04.359934+0000", 2023-09-23T21:12:14.326 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00011566299999999999 2023-09-23T21:12:14.326 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:14.326 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:14.326 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 0,4", 2023-09-23T21:12:14.326 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:04.360253+0000", 2023-09-23T21:12:14.326 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00031915200000000001 2023-09-23T21:12:14.326 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:14.326 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:14.326 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:12:14.326 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:04.367349+0000", 2023-09-23T21:12:14.327 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0070960470000000003 2023-09-23T21:12:14.327 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:14.327 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:14.327 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:12:14.327 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:04.367551+0000", 2023-09-23T21:12:14.327 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000201745 2023-09-23T21:12:14.327 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:14.327 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:14.327 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:12:14.327 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:04.408261+0000", 2023-09-23T21:12:14.327 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.040710206999999998 2023-09-23T21:12:14.327 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:14.328 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:14.328 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:12:14.328 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:04.408293+0000", 2023-09-23T21:12:14.328 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.1405000000000002e-05 2023-09-23T21:12:14.328 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:14.328 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:14.328 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:12:14.328 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:04.408316+0000", 2023-09-23T21:12:14.328 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.3054e-05 2023-09-23T21:12:14.328 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:12:14.328 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:12:14.328 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:12:14.329 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:14.329 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:14.329 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.35469.0:198494 8.19 8:9dcb4a10:::benchmark_data_smithi026_301269_object87737:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e332)", 2023-09-23T21:12:14.329 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:05:03.301427+0000", 2023-09-23T21:12:14.329 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 430.93098016599998, 2023-09-23T21:12:14.329 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.106992956, 2023-09-23T21:12:14.329 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:12:14.329 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:12:14.329 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:12:14.329 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.35469", 2023-09-23T21:12:14.329 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/4041424623", 2023-09-23T21:12:14.329 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 198494 2023-09-23T21:12:14.329 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:14.330 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:12:14.330 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:14.330 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:12:14.330 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:03.301427+0000", 2023-09-23T21:12:14.330 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:12:14.330 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:14.330 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:14.330 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:12:14.330 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:03.301427+0000", 2023-09-23T21:12:14.330 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:12:14.330 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:14.330 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:14.331 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:12:14.331 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:03.301426+0000", 2023-09-23T21:12:14.331 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.999999 2023-09-23T21:12:14.331 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:14.331 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:14.331 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:12:14.331 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:03.301430+0000", 2023-09-23T21:12:14.331 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.7910000000000002e-06 2023-09-23T21:12:14.331 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:14.331 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:14.331 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:12:14.331 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:03.301438+0000", 2023-09-23T21:12:14.331 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.4360000000000002e-06 2023-09-23T21:12:14.332 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:14.332 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:14.332 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:12:14.332 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:03.301443+0000", 2023-09-23T21:12:14.332 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.9880000000000004e-06 2023-09-23T21:12:14.332 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:14.332 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:14.332 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:12:14.332 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:03.301507+0000", 2023-09-23T21:12:14.332 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.3439e-05 2023-09-23T21:12:14.332 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:14.332 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:14.333 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:12:14.333 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:03.301511+0000", 2023-09-23T21:12:14.333 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.6500000000000004e-06 2023-09-23T21:12:14.333 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:14.333 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:14.333 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:12:14.333 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:04.362136+0000", 2023-09-23T21:12:14.333 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.060624912 2023-09-23T21:12:14.333 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:14.333 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:14.333 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:12:14.333 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:04.362265+0000", 2023-09-23T21:12:14.333 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00012852800000000001 2023-09-23T21:12:14.334 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:14.334 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:14.334 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 0,4", 2023-09-23T21:12:14.334 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:04.362536+0000", 2023-09-23T21:12:14.334 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00027105100000000002 2023-09-23T21:12:14.334 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:14.334 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:14.334 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:12:14.334 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:04.367560+0000", 2023-09-23T21:12:14.334 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0050248089999999999 2023-09-23T21:12:14.334 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:14.334 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:14.334 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:12:14.335 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:04.372309+0000", 2023-09-23T21:12:14.335 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0047485729999999999 2023-09-23T21:12:14.335 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:14.335 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:14.335 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:12:14.335 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:04.408353+0000", 2023-09-23T21:12:14.335 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.036044377000000002 2023-09-23T21:12:14.335 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:14.335 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:14.335 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:12:14.335 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:04.408385+0000", 2023-09-23T21:12:14.335 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.1905e-05 2023-09-23T21:12:14.336 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:14.336 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:14.336 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:12:14.336 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:04.408420+0000", 2023-09-23T21:12:14.336 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.4273999999999997e-05 2023-09-23T21:12:14.336 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:12:14.336 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:12:14.336 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:12:14.336 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:14.336 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:14.336 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.35469.0:198545 8.19 8:9ee3d0ba:::benchmark_data_smithi026_301269_object87788:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e332)", 2023-09-23T21:12:14.336 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:05:03.301449+0000", 2023-09-23T21:12:14.336 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 430.93095806500003, 2023-09-23T21:12:14.337 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.1070379939999999, 2023-09-23T21:12:14.337 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:12:14.337 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:12:14.337 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:12:14.337 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.35469", 2023-09-23T21:12:14.337 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/4041424623", 2023-09-23T21:12:14.337 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 198545 2023-09-23T21:12:14.337 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:14.337 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:12:14.337 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:14.337 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:12:14.337 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:03.301449+0000", 2023-09-23T21:12:14.338 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:12:14.338 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:14.338 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:14.338 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:12:14.338 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:03.301449+0000", 2023-09-23T21:12:14.338 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:12:14.338 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:14.338 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:14.338 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:12:14.338 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:03.301448+0000", 2023-09-23T21:12:14.338 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999995 2023-09-23T21:12:14.338 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:14.339 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:14.339 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:12:14.339 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:03.301451+0000", 2023-09-23T21:12:14.339 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.4650000000000001e-06 2023-09-23T21:12:14.339 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:14.339 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:14.339 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:12:14.339 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:03.301456+0000", 2023-09-23T21:12:14.339 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.2089999999999996e-06 2023-09-23T21:12:14.339 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:14.339 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:14.339 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:12:14.340 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:03.301460+0000", 2023-09-23T21:12:14.340 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.9720000000000003e-06 2023-09-23T21:12:14.340 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:14.340 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:14.340 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:12:14.340 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:03.301528+0000", 2023-09-23T21:12:14.340 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.8028000000000007e-05 2023-09-23T21:12:14.340 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:14.340 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:14.340 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:12:14.340 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:03.301533+0000", 2023-09-23T21:12:14.340 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.0679999999999996e-06 2023-09-23T21:12:14.341 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:14.341 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:14.341 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:12:14.341 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:04.364665+0000", 2023-09-23T21:12:14.341 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.0631317149999999 2023-09-23T21:12:14.341 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:14.341 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:14.341 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:12:14.341 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:04.364799+0000", 2023-09-23T21:12:14.341 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00013458 2023-09-23T21:12:14.341 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:14.341 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:14.341 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 0,4", 2023-09-23T21:12:14.342 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:04.365119+0000", 2023-09-23T21:12:14.342 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00032011000000000002 2023-09-23T21:12:14.342 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:14.342 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:14.342 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:12:14.342 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:04.370011+0000", 2023-09-23T21:12:14.342 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0048917969999999998 2023-09-23T21:12:14.342 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:14.342 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:14.342 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:12:14.342 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:04.372346+0000", 2023-09-23T21:12:14.342 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0023348700000000002 2023-09-23T21:12:14.343 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:14.343 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:14.343 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:12:14.343 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:04.408444+0000", 2023-09-23T21:12:14.343 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.036097603999999998 2023-09-23T21:12:14.343 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:14.343 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:14.343 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:12:14.343 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:04.408467+0000", 2023-09-23T21:12:14.343 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.3538000000000001e-05 2023-09-23T21:12:14.343 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:14.343 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:14.344 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:12:14.344 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:04.408487+0000", 2023-09-23T21:12:14.344 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.9451000000000001e-05 2023-09-23T21:12:14.344 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:12:14.344 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:12:14.344 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:12:14.344 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:14.344 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:14.344 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.35469.0:198567 8.19 8:980f1725:::benchmark_data_smithi026_301269_object87810:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e332)", 2023-09-23T21:12:14.344 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:05:03.301478+0000", 2023-09-23T21:12:14.344 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 430.93092842200002, 2023-09-23T21:12:14.344 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.1099435849999999, 2023-09-23T21:12:14.345 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:12:14.345 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:12:14.345 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:12:14.345 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.35469", 2023-09-23T21:12:14.345 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/4041424623", 2023-09-23T21:12:14.345 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 198567 2023-09-23T21:12:14.345 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:14.345 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:12:14.345 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:14.345 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:12:14.345 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:03.301478+0000", 2023-09-23T21:12:14.346 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:12:14.346 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:14.346 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:14.346 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:12:14.346 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:03.301478+0000", 2023-09-23T21:12:14.346 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:12:14.346 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:14.346 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:14.346 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:12:14.346 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:03.301478+0000", 2023-09-23T21:12:14.346 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999995 2023-09-23T21:12:14.346 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:14.347 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:14.347 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:12:14.347 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:03.301480+0000", 2023-09-23T21:12:14.347 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.1090000000000001e-06 2023-09-23T21:12:14.347 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:14.347 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:14.347 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:12:14.347 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:03.301484+0000", 2023-09-23T21:12:14.347 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.2719999999999999e-06 2023-09-23T21:12:14.347 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:14.348 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:14.348 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:12:14.348 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:03.301487+0000", 2023-09-23T21:12:14.348 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.6599999999999999e-06 2023-09-23T21:12:14.348 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:14.348 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:14.348 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:12:14.348 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:03.301550+0000", 2023-09-23T21:12:14.348 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.3187000000000003e-05 2023-09-23T21:12:14.348 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:14.348 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:14.349 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:12:14.349 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:03.301555+0000", 2023-09-23T21:12:14.349 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.1959999999999999e-06 2023-09-23T21:12:14.349 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:14.349 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:14.349 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:12:14.349 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:04.367597+0000", 2023-09-23T21:12:14.349 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.06604136 2023-09-23T21:12:14.349 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:14.349 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:14.349 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:12:14.349 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:04.367702+0000", 2023-09-23T21:12:14.350 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000105509 2023-09-23T21:12:14.350 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:14.350 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:14.350 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 0,4", 2023-09-23T21:12:14.350 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:04.368038+0000", 2023-09-23T21:12:14.350 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00033594000000000001 2023-09-23T21:12:14.350 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:14.350 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:14.350 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:12:14.350 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:04.372443+0000", 2023-09-23T21:12:14.350 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0044051350000000001 2023-09-23T21:12:14.351 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:14.351 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:14.351 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:12:14.351 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:04.372452+0000", 2023-09-23T21:12:14.351 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.3140000000000004e-06 2023-09-23T21:12:14.351 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:14.351 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:14.351 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:12:14.351 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:04.411340+0000", 2023-09-23T21:12:14.351 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.038888058000000003 2023-09-23T21:12:14.351 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:14.351 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:14.352 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:12:14.352 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:04.411390+0000", 2023-09-23T21:12:14.352 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.0124999999999999e-05 2023-09-23T21:12:14.352 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:14.352 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:14.352 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:12:14.352 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:04.411422+0000", 2023-09-23T21:12:14.352 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.2061999999999999e-05 2023-09-23T21:12:14.352 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:12:14.352 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:12:14.352 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:12:14.353 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:14.353 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:14.353 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.35469.0:212440 8.3 8:c05433da:::benchmark_data_smithi026_301269_object101683:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e339)", 2023-09-23T21:12:14.353 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:05:21.317641+0000", 2023-09-23T21:12:14.353 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 412.914766069, 2023-09-23T21:12:14.353 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.98662722700000005, 2023-09-23T21:12:14.353 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:12:14.353 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:12:14.353 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:12:14.354 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.35469", 2023-09-23T21:12:14.354 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/4041424623", 2023-09-23T21:12:14.354 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 212440 2023-09-23T21:12:14.354 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:14.354 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:12:14.354 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:14.354 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:12:14.354 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:21.317641+0000", 2023-09-23T21:12:14.354 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:12:14.354 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:14.354 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:14.355 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:12:14.355 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:21.317641+0000", 2023-09-23T21:12:14.355 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:12:14.355 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:14.355 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:14.355 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:12:14.355 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:21.317639+0000", 2023-09-23T21:12:14.355 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999981 2023-09-23T21:12:14.355 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:14.355 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:14.355 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:12:14.355 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:21.317647+0000", 2023-09-23T21:12:14.355 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.1499999999999999e-06 2023-09-23T21:12:14.356 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:14.356 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:14.356 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:12:14.356 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:21.317664+0000", 2023-09-23T21:12:14.356 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.7238999999999999e-05 2023-09-23T21:12:14.356 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:14.356 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:14.356 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:12:14.356 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:21.317672+0000", 2023-09-23T21:12:14.356 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.644e-06 2023-09-23T21:12:14.356 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:14.356 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:14.357 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:12:14.357 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:21.317732+0000", 2023-09-23T21:12:14.357 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.9895999999999997e-05 2023-09-23T21:12:14.357 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:14.357 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:14.357 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:12:14.357 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:21.317743+0000", 2023-09-23T21:12:14.357 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.1093e-05 2023-09-23T21:12:14.357 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:14.357 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:14.357 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:12:14.357 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:22.298663+0000", 2023-09-23T21:12:14.358 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.980920032 2023-09-23T21:12:14.358 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:14.358 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:14.358 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:12:14.358 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:22.298776+0000", 2023-09-23T21:12:14.358 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000113408 2023-09-23T21:12:14.358 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:14.358 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:14.358 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 6", 2023-09-23T21:12:14.358 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:22.299290+0000", 2023-09-23T21:12:14.358 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00051313900000000002 2023-09-23T21:12:14.358 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:14.359 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:14.359 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:12:14.359 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:22.302224+0000", 2023-09-23T21:12:14.359 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0029344409999999999 2023-09-23T21:12:14.359 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:14.359 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:14.359 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:12:14.359 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:22.304159+0000", 2023-09-23T21:12:14.359 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00193462 2023-09-23T21:12:14.359 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:14.359 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:14.359 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:12:14.360 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:22.304199+0000", 2023-09-23T21:12:14.360 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.9855999999999999e-05 2023-09-23T21:12:14.360 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:14.360 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:14.360 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:12:14.360 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:22.304268+0000", 2023-09-23T21:12:14.360 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.9455999999999994e-05 2023-09-23T21:12:14.360 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:12:14.360 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:12:14.360 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:12:14.360 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:14.360 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:14.361 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.35856.0:56886 9.f 9:fc8547fd:::benchmark_data_smithi026_342732_object56885:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e356)", 2023-09-23T21:12:14.361 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:06:28.222591+0000", 2023-09-23T21:12:14.361 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 346.00981536099999, 2023-09-23T21:12:14.361 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.10711602100000001, 2023-09-23T21:12:14.361 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:12:14.361 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:12:14.361 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:12:14.361 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.35856", 2023-09-23T21:12:14.361 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/4180396258", 2023-09-23T21:12:14.361 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 56886 2023-09-23T21:12:14.361 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:14.361 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:12:14.361 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:14.362 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:12:14.362 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:28.222591+0000", 2023-09-23T21:12:14.362 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:12:14.362 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:14.362 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:14.362 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:12:14.362 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:28.222591+0000", 2023-09-23T21:12:14.362 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:12:14.362 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:14.362 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:14.362 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:12:14.362 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:28.222590+0000", 2023-09-23T21:12:14.363 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T21:12:14.363 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:14.363 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:14.363 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:12:14.363 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:28.222671+0000", 2023-09-23T21:12:14.363 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.0764000000000001e-05 2023-09-23T21:12:14.363 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:14.363 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:14.363 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:12:14.363 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:28.222691+0000", 2023-09-23T21:12:14.363 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.0203999999999998e-05 2023-09-23T21:12:14.363 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:14.364 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:14.364 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:12:14.364 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:28.222696+0000", 2023-09-23T21:12:14.364 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.831e-06 2023-09-23T21:12:14.364 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:14.364 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:14.364 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:12:14.364 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:28.222755+0000", 2023-09-23T21:12:14.364 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.9246999999999997e-05 2023-09-23T21:12:14.364 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:14.364 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:14.364 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:12:14.365 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:28.222890+0000", 2023-09-23T21:12:14.365 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000135215 2023-09-23T21:12:14.365 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:14.365 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:14.365 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 1,3", 2023-09-23T21:12:14.365 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:28.223040+0000", 2023-09-23T21:12:14.365 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000149154 2023-09-23T21:12:14.365 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:14.365 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:14.365 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:12:14.365 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:28.229464+0000", 2023-09-23T21:12:14.365 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0064243260000000002 2023-09-23T21:12:14.366 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:14.366 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:14.366 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:12:14.366 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:28.229622+0000", 2023-09-23T21:12:14.366 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00015850400000000001 2023-09-23T21:12:14.366 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:14.366 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:14.366 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:12:14.366 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:28.329620+0000", 2023-09-23T21:12:14.366 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.099997878999999998 2023-09-23T21:12:14.366 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:14.366 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:14.367 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:12:14.367 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:28.329667+0000", 2023-09-23T21:12:14.367 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.6542999999999998e-05 2023-09-23T21:12:14.367 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:14.367 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:14.367 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:12:14.367 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:06:28.329707+0000", 2023-09-23T21:12:14.367 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.0708999999999999e-05 2023-09-23T21:12:14.367 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:12:14.367 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:12:14.367 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:12:14.367 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:14.368 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:14.368 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.35856.0:157816 9.a 9:50dbf38a:::benchmark_data_smithi026_342732_object44540:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e378)", 2023-09-23T21:12:14.368 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:08:21.456399+0000", 2023-09-23T21:12:14.368 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 232.77600799199999, 2023-09-23T21:12:14.368 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.49007018499999999, 2023-09-23T21:12:14.368 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:12:14.368 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:12:14.368 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:12:14.368 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.35856", 2023-09-23T21:12:14.368 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/4180396258", 2023-09-23T21:12:14.368 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 157816 2023-09-23T21:12:14.368 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:14.369 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:12:14.369 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:14.369 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:12:14.369 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:08:21.456399+0000", 2023-09-23T21:12:14.369 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:12:14.369 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:14.369 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:14.369 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:12:14.369 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:08:21.456399+0000", 2023-09-23T21:12:14.369 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:12:14.369 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:14.369 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:14.369 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:12:14.370 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:08:21.456397+0000", 2023-09-23T21:12:14.370 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T21:12:14.370 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:14.370 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:14.370 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:12:14.370 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:08:21.456404+0000", 2023-09-23T21:12:14.370 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.4930000000000003e-06 2023-09-23T21:12:14.370 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:14.370 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:14.370 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:12:14.370 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:08:21.456420+0000", 2023-09-23T21:12:14.370 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.6214000000000001e-05 2023-09-23T21:12:14.371 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:14.371 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:14.371 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:12:14.371 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:08:21.456427+0000", 2023-09-23T21:12:14.371 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.5819999999999998e-06 2023-09-23T21:12:14.371 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:14.371 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:14.371 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:12:14.371 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:08:21.456473+0000", 2023-09-23T21:12:14.371 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.5942999999999997e-05 2023-09-23T21:12:14.371 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:14.371 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:14.372 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:12:14.372 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:08:21.456585+0000", 2023-09-23T21:12:14.372 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000112427 2023-09-23T21:12:14.372 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:14.372 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:14.372 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 1,7", 2023-09-23T21:12:14.372 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:08:21.456701+0000", 2023-09-23T21:12:14.372 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000115827 2023-09-23T21:12:14.372 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:14.372 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:14.372 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:12:14.373 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:08:21.460226+0000", 2023-09-23T21:12:14.373 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0035251530000000001 2023-09-23T21:12:14.373 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:14.373 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:14.373 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:12:14.373 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:08:21.464859+0000", 2023-09-23T21:12:14.373 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0046332430000000004 2023-09-23T21:12:14.373 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:14.373 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:14.373 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:12:14.373 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:08:21.946381+0000", 2023-09-23T21:12:14.373 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.48152162700000001 2023-09-23T21:12:14.374 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:14.374 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:14.374 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:12:14.374 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:08:21.946430+0000", 2023-09-23T21:12:14.374 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.9178000000000002e-05 2023-09-23T21:12:14.374 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:14.374 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:14.374 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:12:14.374 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:08:21.946469+0000", 2023-09-23T21:12:14.374 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.9062e-05 2023-09-23T21:12:14.374 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:12:14.374 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:12:14.375 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:12:14.375 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:14.375 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:14.375 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.35856.0:157828 9.a 9:5b408325:::benchmark_data_smithi026_342732_object44552:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e378)", 2023-09-23T21:12:14.375 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:08:21.471593+0000", 2023-09-23T21:12:14.375 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 232.76081347799999, 2023-09-23T21:12:14.375 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.47497405999999998, 2023-09-23T21:12:14.375 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:12:14.375 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:12:14.375 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:12:14.375 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.35856", 2023-09-23T21:12:14.375 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/4180396258", 2023-09-23T21:12:14.376 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 157828 2023-09-23T21:12:14.376 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:14.376 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:12:14.376 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:14.376 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:12:14.376 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:08:21.471593+0000", 2023-09-23T21:12:14.376 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:12:14.376 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:14.376 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:14.376 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:12:14.376 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:08:21.471593+0000", 2023-09-23T21:12:14.376 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:12:14.377 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:14.377 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:14.377 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:12:14.377 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:08:21.471592+0000", 2023-09-23T21:12:14.377 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T21:12:14.377 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:14.377 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:14.377 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:12:14.377 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:08:21.471600+0000", 2023-09-23T21:12:14.377 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.1559999999999995e-06 2023-09-23T21:12:14.377 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:14.377 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:14.377 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:12:14.378 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:08:21.471614+0000", 2023-09-23T21:12:14.378 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.4090000000000001e-05 2023-09-23T21:12:14.378 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:14.378 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:14.378 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:12:14.378 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:08:21.471621+0000", 2023-09-23T21:12:14.378 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.1019999999999999e-06 2023-09-23T21:12:14.378 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:14.378 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:14.378 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:12:14.378 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:08:21.471665+0000", 2023-09-23T21:12:14.378 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.3878000000000002e-05 2023-09-23T21:12:14.379 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:14.379 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:14.379 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:12:14.379 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:08:21.471793+0000", 2023-09-23T21:12:14.379 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00012794400000000001 2023-09-23T21:12:14.379 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:14.379 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:14.379 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 1,7", 2023-09-23T21:12:14.379 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:08:21.471915+0000", 2023-09-23T21:12:14.379 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000121799 2023-09-23T21:12:14.379 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:14.379 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:14.380 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:12:14.380 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:08:21.475680+0000", 2023-09-23T21:12:14.380 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0037653169999999998 2023-09-23T21:12:14.380 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:14.380 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:14.380 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:12:14.380 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:08:21.475792+0000", 2023-09-23T21:12:14.380 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000111488 2023-09-23T21:12:14.380 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:14.380 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:14.380 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:12:14.380 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:08:21.946503+0000", 2023-09-23T21:12:14.381 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.47071068399999999 2023-09-23T21:12:14.381 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:14.381 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:14.381 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:12:14.381 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:08:21.946537+0000", 2023-09-23T21:12:14.381 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.434e-05 2023-09-23T21:12:14.381 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:14.381 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:14.381 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:12:14.381 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:08:21.946567+0000", 2023-09-23T21:12:14.381 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.0550999999999999e-05 2023-09-23T21:12:14.381 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:12:14.382 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:12:14.382 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:12:14.382 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:14.382 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:14.382 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.35856.0:175235 9.a 9:56d48211:::benchmark_data_smithi026_342732_object61959:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e386)", 2023-09-23T21:12:14.382 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:08:46.427375+0000", 2023-09-23T21:12:14.382 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 207.80503225199999, 2023-09-23T21:12:14.382 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.22407644700000001, 2023-09-23T21:12:14.382 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:12:14.382 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:12:14.382 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:12:14.382 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.35856", 2023-09-23T21:12:14.383 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/4180396258", 2023-09-23T21:12:14.383 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 175235 2023-09-23T21:12:14.383 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:14.383 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:12:14.383 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:14.383 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:12:14.383 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:08:46.427375+0000", 2023-09-23T21:12:14.383 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:12:14.383 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:14.383 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:14.383 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:12:14.383 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:08:46.427375+0000", 2023-09-23T21:12:14.384 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:12:14.384 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:14.384 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:14.384 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:12:14.384 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:08:46.427374+0000", 2023-09-23T21:12:14.384 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999995 2023-09-23T21:12:14.384 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:14.384 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:14.384 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:12:14.384 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:08:46.427376+0000", 2023-09-23T21:12:14.384 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.2309999999999999e-06 2023-09-23T21:12:14.384 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:14.385 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:14.385 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:12:14.385 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:08:46.427382+0000", 2023-09-23T21:12:14.385 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.3170000000000001e-06 2023-09-23T21:12:14.385 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:14.385 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:14.385 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:12:14.385 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:08:46.427385+0000", 2023-09-23T21:12:14.385 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.4230000000000001e-06 2023-09-23T21:12:14.385 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:14.385 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:14.385 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:12:14.385 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:08:46.441224+0000", 2023-09-23T21:12:14.386 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.013838886 2023-09-23T21:12:14.386 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:14.386 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:14.386 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:12:14.386 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:08:46.441980+0000", 2023-09-23T21:12:14.386 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00075593899999999996 2023-09-23T21:12:14.386 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:14.386 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:14.386 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 1,3,7", 2023-09-23T21:12:14.386 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:08:46.442107+0000", 2023-09-23T21:12:14.386 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000127138 2023-09-23T21:12:14.386 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:14.387 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:14.387 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:12:14.387 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:08:46.446132+0000", 2023-09-23T21:12:14.387 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0040243700000000002 2023-09-23T21:12:14.387 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:14.387 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:14.387 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:12:14.387 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:08:46.448391+0000", 2023-09-23T21:12:14.387 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0022593610000000001 2023-09-23T21:12:14.387 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:14.387 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:14.387 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:12:14.388 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:08:46.449663+0000", 2023-09-23T21:12:14.388 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0012723700000000001 2023-09-23T21:12:14.388 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:14.388 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:14.388 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:12:14.388 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:08:46.651281+0000", 2023-09-23T21:12:14.388 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.201617307 2023-09-23T21:12:14.388 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:14.388 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:14.388 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:12:14.388 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:08:46.651333+0000", 2023-09-23T21:12:14.388 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.2005000000000001e-05 2023-09-23T21:12:14.388 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:14.389 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:14.389 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:12:14.389 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:08:46.651451+0000", 2023-09-23T21:12:14.389 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000118442 2023-09-23T21:12:14.389 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:12:14.389 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:12:14.389 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:12:14.389 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:14.389 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:14.389 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.35856.0:226079 9.f 9:fe193b3f:::benchmark_data_smithi026_342732_object112803:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e411)", 2023-09-23T21:12:14.389 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:09:56.376863+0000", 2023-09-23T21:12:14.389 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 137.855543651, 2023-09-23T21:12:14.390 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.12671870299999999, 2023-09-23T21:12:14.390 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:12:14.390 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:12:14.390 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:12:14.390 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.35856", 2023-09-23T21:12:14.390 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/4180396258", 2023-09-23T21:12:14.390 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 226079 2023-09-23T21:12:14.390 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:14.390 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:12:14.390 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:14.390 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:12:14.390 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:09:56.376863+0000", 2023-09-23T21:12:14.391 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:12:14.391 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:14.391 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:14.391 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:12:14.391 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:09:56.376863+0000", 2023-09-23T21:12:14.391 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:12:14.391 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:14.391 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:14.391 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:12:14.391 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:09:56.376863+0000", 2023-09-23T21:12:14.391 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999995 2023-09-23T21:12:14.391 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:14.392 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:14.392 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:12:14.392 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:09:56.376865+0000", 2023-09-23T21:12:14.392 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.61e-06 2023-09-23T21:12:14.392 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:14.392 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:14.392 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:12:14.392 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:09:56.376870+0000", 2023-09-23T21:12:14.392 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.7690000000000004e-06 2023-09-23T21:12:14.392 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:14.392 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:14.392 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:12:14.393 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:09:56.376873+0000", 2023-09-23T21:12:14.393 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.971e-06 2023-09-23T21:12:14.393 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:14.393 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:14.393 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:12:14.393 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:09:56.376905+0000", 2023-09-23T21:12:14.393 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.1789999999999999e-05 2023-09-23T21:12:14.393 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:14.393 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:14.393 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:12:14.393 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:09:56.376989+0000", 2023-09-23T21:12:14.393 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.3953000000000001e-05 2023-09-23T21:12:14.394 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:14.394 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:14.394 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 1,3", 2023-09-23T21:12:14.394 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:09:56.377062+0000", 2023-09-23T21:12:14.394 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.3267000000000002e-05 2023-09-23T21:12:14.394 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:14.394 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:14.394 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:12:14.394 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:09:56.379902+0000", 2023-09-23T21:12:14.394 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0028397689999999998 2023-09-23T21:12:14.394 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:14.394 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:14.395 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:12:14.395 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:09:56.388432+0000", 2023-09-23T21:12:14.395 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0085297459999999995 2023-09-23T21:12:14.395 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:14.395 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:14.395 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:12:14.395 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:09:56.503500+0000", 2023-09-23T21:12:14.395 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.11506807300000001 2023-09-23T21:12:14.395 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:14.395 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:14.395 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:12:14.395 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:09:56.503543+0000", 2023-09-23T21:12:14.396 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.2976999999999999e-05 2023-09-23T21:12:14.396 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:14.396 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:14.396 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:12:14.396 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:09:56.503582+0000", 2023-09-23T21:12:14.396 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.9100000000000002e-05 2023-09-23T21:12:14.396 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:12:14.396 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:12:14.396 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:12:14.396 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:14.396 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:14.396 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.35856.0:226081 9.f 9:ff777e4f:::benchmark_data_smithi026_342732_object112805:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e411)", 2023-09-23T21:12:14.397 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:09:56.380528+0000", 2023-09-23T21:12:14.397 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 137.85187853799999, 2023-09-23T21:12:14.397 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.12314288399999999, 2023-09-23T21:12:14.397 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:12:14.397 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:12:14.397 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:12:14.397 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.35856", 2023-09-23T21:12:14.397 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/4180396258", 2023-09-23T21:12:14.397 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 226081 2023-09-23T21:12:14.397 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:14.397 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:12:14.397 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:14.397 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:12:14.398 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:09:56.380528+0000", 2023-09-23T21:12:14.398 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:12:14.398 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:14.398 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:14.398 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:12:14.398 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:09:56.380528+0000", 2023-09-23T21:12:14.398 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:12:14.398 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:14.398 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:14.398 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:12:14.398 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:09:56.380528+0000", 2023-09-23T21:12:14.398 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.999999 2023-09-23T21:12:14.399 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:14.399 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:14.399 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:12:14.399 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:09:56.380532+0000", 2023-09-23T21:12:14.399 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.8509999999999998e-06 2023-09-23T21:12:14.399 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:14.399 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:14.399 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:12:14.399 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:09:56.380545+0000", 2023-09-23T21:12:14.399 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.3098999999999999e-05 2023-09-23T21:12:14.399 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:14.399 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:14.399 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:12:14.400 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:09:56.380551+0000", 2023-09-23T21:12:14.400 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.8660000000000002e-06 2023-09-23T21:12:14.400 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:14.400 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:14.400 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:12:14.400 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:09:56.380611+0000", 2023-09-23T21:12:14.400 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.9332e-05 2023-09-23T21:12:14.400 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:14.400 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:14.400 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:12:14.400 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:09:56.380735+0000", 2023-09-23T21:12:14.400 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00012407299999999999 2023-09-23T21:12:14.401 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:14.401 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:14.401 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 1,3", 2023-09-23T21:12:14.401 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:09:56.380858+0000", 2023-09-23T21:12:14.401 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00012301099999999999 2023-09-23T21:12:14.401 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:14.401 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:14.401 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:12:14.401 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:09:56.382779+0000", 2023-09-23T21:12:14.401 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.001921664 2023-09-23T21:12:14.401 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:14.401 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:14.401 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:12:14.402 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:09:56.481269+0000", 2023-09-23T21:12:14.402 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.098489937999999999 2023-09-23T21:12:14.402 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:14.402 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:14.402 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:12:14.402 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:09:56.503614+0000", 2023-09-23T21:12:14.402 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.022344692999999999 2023-09-23T21:12:14.402 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:14.402 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:14.402 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:12:14.402 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:09:56.503646+0000", 2023-09-23T21:12:14.402 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.2095999999999999e-05 2023-09-23T21:12:14.403 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:14.403 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:14.403 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:12:14.403 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:09:56.503671+0000", 2023-09-23T21:12:14.403 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.5015999999999999e-05 2023-09-23T21:12:14.403 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:12:14.403 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:12:14.403 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:12:14.403 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:14.403 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:14.403 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.35856.0:226084 9.f 9:f7691591:::benchmark_data_smithi026_342732_object112808:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e411)", 2023-09-23T21:12:14.403 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:09:56.383648+0000", 2023-09-23T21:12:14.403 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 137.84875926999999, 2023-09-23T21:12:14.404 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.12777032599999999, 2023-09-23T21:12:14.404 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:12:14.404 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:12:14.404 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:12:14.404 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.35856", 2023-09-23T21:12:14.404 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/4180396258", 2023-09-23T21:12:14.404 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 226084 2023-09-23T21:12:14.404 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:14.404 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:12:14.404 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:14.404 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:12:14.404 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:09:56.383648+0000", 2023-09-23T21:12:14.405 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:12:14.405 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:14.405 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:14.405 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:12:14.405 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:09:56.383648+0000", 2023-09-23T21:12:14.405 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:12:14.405 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:14.405 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:14.405 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:12:14.405 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:09:56.383646+0000", 2023-09-23T21:12:14.405 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.999999 2023-09-23T21:12:14.405 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:14.405 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:14.406 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:12:14.406 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:09:56.383653+0000", 2023-09-23T21:12:14.406 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.0549999999999996e-06 2023-09-23T21:12:14.406 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:14.406 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:14.406 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:12:14.406 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:09:56.383667+0000", 2023-09-23T21:12:14.406 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.4578e-05 2023-09-23T21:12:14.406 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:14.406 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:14.406 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:12:14.406 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:09:56.383672+0000", 2023-09-23T21:12:14.407 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.972e-06 2023-09-23T21:12:14.407 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:14.407 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:14.407 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:12:14.407 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:09:56.383721+0000", 2023-09-23T21:12:14.407 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.8816000000000002e-05 2023-09-23T21:12:14.407 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:14.407 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:14.407 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:12:14.407 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:09:56.383838+0000", 2023-09-23T21:12:14.407 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000117577 2023-09-23T21:12:14.407 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:14.407 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:14.408 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 1,3", 2023-09-23T21:12:14.408 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:09:56.383961+0000", 2023-09-23T21:12:14.408 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000122254 2023-09-23T21:12:14.408 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:14.408 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:14.408 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:12:14.408 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:09:56.386631+0000", 2023-09-23T21:12:14.408 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0026705029999999999 2023-09-23T21:12:14.408 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:14.408 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:14.408 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:12:14.408 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:09:56.485585+0000", 2023-09-23T21:12:14.408 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.098953381000000007 2023-09-23T21:12:14.409 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:14.409 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:14.409 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:12:14.409 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:09:56.511329+0000", 2023-09-23T21:12:14.409 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.025744132999999999 2023-09-23T21:12:14.409 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:14.409 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:14.409 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:12:14.409 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:09:56.511377+0000", 2023-09-23T21:12:14.409 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.8621000000000001e-05 2023-09-23T21:12:14.409 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:14.409 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:14.410 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:12:14.410 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:09:56.511418+0000", 2023-09-23T21:12:14.410 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.0515000000000001e-05 2023-09-23T21:12:14.410 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:12:14.410 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:12:14.410 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:12:14.410 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:12:14.410 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:12:14.410 INFO:teuthology.orchestra.run.smithi070.stdout:} 2023-09-23T21:12:14.410 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:12:14.452 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:12:14 smithi070 ceph-mon[168282]: failed to encode map e473 with expected crc 2023-09-23T21:12:14.452 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:12:14 smithi070 ceph-mon[168282]: failed to encode map e473 with expected crc 2023-09-23T21:12:14.452 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:12:14 smithi070 ceph-mon[168282]: pgmap v1205: 119 pgs: 6 active+clean+remapped, 1 activating, 2 remapped+peering, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 16 active+undersized, 2 active+undersized+remapped, 8 active+undersized+degraded, 8 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+degraded+remapped, 6 active+recovery_wait+degraded+remapped, 66 active+clean; 4.6 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 119 op/s; 59614/227385 objects degraded (26.217%); 15846/227385 objects misplaced (6.969%); 10 MiB/s, 163 objects/s recovering 2023-09-23T21:12:14.452 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:12:14 smithi070 ceph-mon[168282]: failed to encode map e473 with expected crc 2023-09-23T21:12:14.452 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:12:14 smithi070 ceph-mon[168282]: Health check failed: Reduced data availability: 2 pgs inactive, 1 pg peering (PG_AVAILABILITY) 2023-09-23T21:12:14.452 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:12:14 smithi070 ceph-mon[168282]: osdmap e474: 8 total, 7 up, 6 in 2023-09-23T21:12:14.452 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:12:14 smithi070 ceph-mon[168282]: failed to encode map e474 with expected crc 2023-09-23T21:12:14.452 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:12:14 smithi070 ceph-mon[168282]: failed to encode map e474 with expected crc 2023-09-23T21:12:14.460 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:12:14.631 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:12:14 smithi026 ceph-mon[179688]: failed to encode map e473 with expected crc 2023-09-23T21:12:14.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:12:14 smithi026 ceph-mon[179688]: failed to encode map e473 with expected crc 2023-09-23T21:12:14.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:12:14 smithi026 ceph-mon[179688]: pgmap v1205: 119 pgs: 6 active+clean+remapped, 1 activating, 2 remapped+peering, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 16 active+undersized, 2 active+undersized+remapped, 8 active+undersized+degraded, 8 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+degraded+remapped, 6 active+recovery_wait+degraded+remapped, 66 active+clean; 4.6 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 119 op/s; 59614/227385 objects degraded (26.217%); 15846/227385 objects misplaced (6.969%); 10 MiB/s, 163 objects/s recovering 2023-09-23T21:12:14.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:12:14 smithi026 ceph-mon[179688]: failed to encode map e473 with expected crc 2023-09-23T21:12:14.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:12:14 smithi026 ceph-mon[179688]: Health check failed: Reduced data availability: 2 pgs inactive, 1 pg peering (PG_AVAILABILITY) 2023-09-23T21:12:14.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:12:14 smithi026 ceph-mon[179688]: osdmap e474: 8 total, 7 up, 6 in 2023-09-23T21:12:14.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:12:14 smithi026 ceph-mon[179688]: failed to encode map e474 with expected crc 2023-09-23T21:12:14.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:12:14 smithi026 ceph-mon[179688]: failed to encode map e474 with expected crc 2023-09-23T21:12:14.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:12:14 smithi026 ceph-mon[175151]: failed to encode map e473 with expected crc 2023-09-23T21:12:14.633 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:12:14 smithi026 ceph-mon[175151]: failed to encode map e473 with expected crc 2023-09-23T21:12:14.633 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:12:14 smithi026 ceph-mon[175151]: pgmap v1205: 119 pgs: 6 active+clean+remapped, 1 activating, 2 remapped+peering, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 16 active+undersized, 2 active+undersized+remapped, 8 active+undersized+degraded, 8 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+degraded+remapped, 6 active+recovery_wait+degraded+remapped, 66 active+clean; 4.6 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 119 op/s; 59614/227385 objects degraded (26.217%); 15846/227385 objects misplaced (6.969%); 10 MiB/s, 163 objects/s recovering 2023-09-23T21:12:14.633 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:12:14 smithi026 ceph-mon[175151]: failed to encode map e473 with expected crc 2023-09-23T21:12:14.633 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:12:14 smithi026 ceph-mon[175151]: Health check failed: Reduced data availability: 2 pgs inactive, 1 pg peering (PG_AVAILABILITY) 2023-09-23T21:12:14.633 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:12:14 smithi026 ceph-mon[175151]: osdmap e474: 8 total, 7 up, 6 in 2023-09-23T21:12:14.633 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:12:14 smithi026 ceph-mon[175151]: failed to encode map e474 with expected crc 2023-09-23T21:12:14.633 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:12:14 smithi026 ceph-mon[175151]: failed to encode map e474 with expected crc 2023-09-23T21:12:14.667 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:12:14.901 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:12:14.907 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:12:14.907 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:12:14.956 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_ops_in_flight 2023-09-23T21:12:15.008 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:12:15.040 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:12:15.296 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:12:15 smithi026 ceph-mon[175151]: failed to encode map e474 with expected crc 2023-09-23T21:12:15.296 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:12:15 smithi026 ceph-mon[175151]: failed to encode map e474 with expected crc 2023-09-23T21:12:15.296 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:12:15 smithi026 ceph-mon[179688]: failed to encode map e474 with expected crc 2023-09-23T21:12:15.296 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:12:15 smithi026 ceph-mon[179688]: failed to encode map e474 with expected crc 2023-09-23T21:12:15.297 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:12:15 smithi070 ceph-mon[168282]: failed to encode map e474 with expected crc 2023-09-23T21:12:15.297 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:12:15 smithi070 ceph-mon[168282]: failed to encode map e474 with expected crc 2023-09-23T21:12:15.726 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T21:12:15.851 INFO:tasks.thrashosds.thrasher:in_osds: [0, 2, 3, 4, 5, 7] out_osds: [6, 1] dead_osds: [0] live_osds: [2, 3, 4, 5, 7, 6, 1] 2023-09-23T21:12:15.851 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T21:12:15.851 INFO:tasks.thrashosds.thrasher:Growing pool .mgr 2023-09-23T21:12:15.851 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T21:12:16.128 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:12:16.131 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:12:16.132 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:12:16.232 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:12:16.316 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:12:16.381 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:12:16 smithi026 ceph-mon[175151]: pgmap v1207: 119 pgs: 14 active+clean+remapped, 2 activating+undersized+degraded+remapped, 2 activating, 2 remapped+peering, 4 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+remapped, 12 active+undersized, 2 active+undersized+remapped, 6 active+undersized+degraded, 9 active+recovery_wait+undersized+degraded+remapped, 6 active+recovery_wait+degraded+remapped, 58 active+clean; 4.6 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 195 op/s; 64892/226209 objects degraded (28.687%); 21317/226209 objects misplaced (9.424%); 14 MiB/s, 230 objects/s recovering 2023-09-23T21:12:16.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:12:16 smithi026 ceph-mon[175151]: failed to encode map e475 with expected crc 2023-09-23T21:12:16.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:12:16 smithi026 ceph-mon[175151]: failed to encode map e475 with expected crc 2023-09-23T21:12:16.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:12:16 smithi026 ceph-mon[175151]: failed to encode map e475 with expected crc 2023-09-23T21:12:16.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:12:16 smithi026 ceph-mon[175151]: osdmap e475: 8 total, 7 up, 6 in 2023-09-23T21:12:16.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:12:16 smithi026 ceph-mon[175151]: failed to encode map e475 with expected crc 2023-09-23T21:12:16.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:12:16 smithi026 ceph-mon[175151]: Health check update: Degraded data redundancy: 64892/226209 objects degraded (28.687%), 28 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T21:12:16.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:12:16 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:12:16.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:12:16 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:12:16.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:12:16 smithi026 ceph-mon[179688]: pgmap v1207: 119 pgs: 14 active+clean+remapped, 2 activating+undersized+degraded+remapped, 2 activating, 2 remapped+peering, 4 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+remapped, 12 active+undersized, 2 active+undersized+remapped, 6 active+undersized+degraded, 9 active+recovery_wait+undersized+degraded+remapped, 6 active+recovery_wait+degraded+remapped, 58 active+clean; 4.6 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 195 op/s; 64892/226209 objects degraded (28.687%); 21317/226209 objects misplaced (9.424%); 14 MiB/s, 230 objects/s recovering 2023-09-23T21:12:16.383 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:12:16 smithi026 ceph-mon[179688]: failed to encode map e475 with expected crc 2023-09-23T21:12:16.383 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:12:16 smithi026 ceph-mon[179688]: failed to encode map e475 with expected crc 2023-09-23T21:12:16.383 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:12:16 smithi026 ceph-mon[179688]: failed to encode map e475 with expected crc 2023-09-23T21:12:16.383 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:12:16 smithi026 ceph-mon[179688]: osdmap e475: 8 total, 7 up, 6 in 2023-09-23T21:12:16.383 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:12:16 smithi026 ceph-mon[179688]: failed to encode map e475 with expected crc 2023-09-23T21:12:16.383 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:12:16 smithi026 ceph-mon[179688]: Health check update: Degraded data redundancy: 64892/226209 objects degraded (28.687%), 28 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T21:12:16.383 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:12:16 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:12:16.383 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:12:16 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:12:16.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:12:16 smithi070 ceph-mon[168282]: pgmap v1207: 119 pgs: 14 active+clean+remapped, 2 activating+undersized+degraded+remapped, 2 activating, 2 remapped+peering, 4 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+remapped, 12 active+undersized, 2 active+undersized+remapped, 6 active+undersized+degraded, 9 active+recovery_wait+undersized+degraded+remapped, 6 active+recovery_wait+degraded+remapped, 58 active+clean; 4.6 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 195 op/s; 64892/226209 objects degraded (28.687%); 21317/226209 objects misplaced (9.424%); 14 MiB/s, 230 objects/s recovering 2023-09-23T21:12:16.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:12:16 smithi070 ceph-mon[168282]: failed to encode map e475 with expected crc 2023-09-23T21:12:16.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:12:16 smithi070 ceph-mon[168282]: failed to encode map e475 with expected crc 2023-09-23T21:12:16.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:12:16 smithi070 ceph-mon[168282]: failed to encode map e475 with expected crc 2023-09-23T21:12:16.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:12:16 smithi070 ceph-mon[168282]: osdmap e475: 8 total, 7 up, 6 in 2023-09-23T21:12:16.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:12:16 smithi070 ceph-mon[168282]: failed to encode map e475 with expected crc 2023-09-23T21:12:16.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:12:16 smithi070 ceph-mon[168282]: Health check update: Degraded data redundancy: 64892/226209 objects degraded (28.687%), 28 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T21:12:16.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:12:16 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:12:16.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:12:16 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:12:16.625 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:12:16.635 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:12:16.636 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:12:16.737 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:12:16.787 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:12:16.945 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:12:16.950 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:12:16.951 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:12:17.051 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:12:17.134 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:12:17.248 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:12:17.253 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:12:17.253 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:12:17.354 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:12:17.386 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:12:17.435 INFO:teuthology.orchestra.run.smithi070.stdout:{ 2023-09-23T21:12:17.435 INFO:teuthology.orchestra.run.smithi070.stdout: "ops": [ 2023-09-23T21:12:17.435 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:17.436 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_repop(client.46328.0:85475 10.19 e476/438 10:9fb15a1d:::benchmark_data_smithi026_379838_object5518:head v 476'4121, mlcod=476'4119)", 2023-09-23T21:12:17.436 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:12:17.430667+0000", 2023-09-23T21:12:17.436 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 0.0045351380000000002, 2023-09-23T21:12:17.436 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0045592130000000003, 2023-09-23T21:12:17.436 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:12:17.436 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "started", 2023-09-23T21:12:17.436 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:12:17.436 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:17.436 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:12:17.436 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:12:17.430667+0000", 2023-09-23T21:12:17.436 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:12:17.437 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:17.437 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:17.437 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:12:17.437 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:12:17.430667+0000", 2023-09-23T21:12:17.437 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:12:17.437 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:17.437 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:17.437 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:12:17.437 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:12:17.430666+0000", 2023-09-23T21:12:17.437 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.999999 2023-09-23T21:12:17.437 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:17.437 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:17.438 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:12:17.438 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:12:17.430677+0000", 2023-09-23T21:12:17.438 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.0331e-05 2023-09-23T21:12:17.438 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:17.438 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:17.438 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:12:17.438 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:12:17.430693+0000", 2023-09-23T21:12:17.438 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.6861999999999999e-05 2023-09-23T21:12:17.438 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:17.438 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:17.438 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:12:17.438 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:12:17.430701+0000", 2023-09-23T21:12:17.439 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.3089999999999998e-06 2023-09-23T21:12:17.439 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:17.439 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:17.439 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:12:17.439 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:12:17.431278+0000", 2023-09-23T21:12:17.439 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00057698000000000005 2023-09-23T21:12:17.439 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:17.439 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:17.439 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:12:17.439 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:12:17.431303+0000", 2023-09-23T21:12:17.439 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.5361999999999999e-05 2023-09-23T21:12:17.439 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:12:17.439 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:12:17.440 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:12:17.440 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:12:17.440 INFO:teuthology.orchestra.run.smithi070.stdout: ], 2023-09-23T21:12:17.440 INFO:teuthology.orchestra.run.smithi070.stdout: "num_ops": 1 2023-09-23T21:12:17.440 INFO:teuthology.orchestra.run.smithi070.stdout:} 2023-09-23T21:12:17.576 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:12:17 smithi070 ceph-mon[168282]: failed to encode map e476 with expected crc 2023-09-23T21:12:17.576 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:12:17 smithi070 ceph-mon[168282]: failed to encode map e476 with expected crc 2023-09-23T21:12:17.576 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:12:17 smithi070 ceph-mon[168282]: osdmap e476: 8 total, 7 up, 6 in 2023-09-23T21:12:17.576 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:12:17 smithi070 ceph-mon[168282]: failed to encode map e476 with expected crc 2023-09-23T21:12:17.577 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:12:17 smithi070 ceph-mon[168282]: failed to encode map e476 with expected crc 2023-09-23T21:12:17.577 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:12:17 smithi070 ceph-mon[168282]: failed to encode map e476 with expected crc 2023-09-23T21:12:17.631 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:12:17 smithi026 ceph-mon[175151]: failed to encode map e476 with expected crc 2023-09-23T21:12:17.631 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:12:17 smithi026 ceph-mon[175151]: failed to encode map e476 with expected crc 2023-09-23T21:12:17.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:12:17 smithi026 ceph-mon[175151]: osdmap e476: 8 total, 7 up, 6 in 2023-09-23T21:12:17.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:12:17 smithi026 ceph-mon[175151]: failed to encode map e476 with expected crc 2023-09-23T21:12:17.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:12:17 smithi026 ceph-mon[175151]: failed to encode map e476 with expected crc 2023-09-23T21:12:17.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:12:17 smithi026 ceph-mon[175151]: failed to encode map e476 with expected crc 2023-09-23T21:12:17.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:12:17 smithi026 ceph-mon[179688]: failed to encode map e476 with expected crc 2023-09-23T21:12:17.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:12:17 smithi026 ceph-mon[179688]: failed to encode map e476 with expected crc 2023-09-23T21:12:17.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:12:17 smithi026 ceph-mon[179688]: osdmap e476: 8 total, 7 up, 6 in 2023-09-23T21:12:17.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:12:17 smithi026 ceph-mon[179688]: failed to encode map e476 with expected crc 2023-09-23T21:12:17.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:12:17 smithi026 ceph-mon[179688]: failed to encode map e476 with expected crc 2023-09-23T21:12:17.633 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:12:17 smithi026 ceph-mon[179688]: failed to encode map e476 with expected crc 2023-09-23T21:12:17.866 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_blocked_ops 2023-09-23T21:12:17.957 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:12:17.961 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:12:17.961 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:12:18.062 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:12:18.119 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:12:18.196 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:12:18.209 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:12:18.209 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:12:18.310 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:12:18.317 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:12:18.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:12:18 smithi070 ceph-mon[168282]: pgmap v1210: 119 pgs: 1 peering, 15 active+clean+remapped, 2 activating+undersized+degraded+remapped, 1 activating, 4 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 12 active+undersized, 2 active+undersized+remapped, 6 active+undersized+degraded, 9 active+recovery_wait+undersized+degraded+remapped, 6 active+recovery_wait+degraded+remapped, 59 active+clean; 4.6 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 135 op/s; 64813/225654 objects degraded (28.722%); 20688/225654 objects misplaced (9.168%); 15 MiB/s, 237 objects/s recovering 2023-09-23T21:12:18.377 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:12:18.454 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:12:18.459 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:12:18.459 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:12:18.560 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:12:18.590 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:12:18 smithi026 ceph-mon[175151]: pgmap v1210: 119 pgs: 1 peering, 15 active+clean+remapped, 2 activating+undersized+degraded+remapped, 1 activating, 4 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 12 active+undersized, 2 active+undersized+remapped, 6 active+undersized+degraded, 9 active+recovery_wait+undersized+degraded+remapped, 6 active+recovery_wait+degraded+remapped, 59 active+clean; 4.6 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 135 op/s; 64813/225654 objects degraded (28.722%); 20688/225654 objects misplaced (9.168%); 15 MiB/s, 237 objects/s recovering 2023-09-23T21:12:18.590 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:12:18 smithi026 ceph-mon[179688]: pgmap v1210: 119 pgs: 1 peering, 15 active+clean+remapped, 2 activating+undersized+degraded+remapped, 1 activating, 4 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 12 active+undersized, 2 active+undersized+remapped, 6 active+undersized+degraded, 9 active+recovery_wait+undersized+degraded+remapped, 6 active+recovery_wait+degraded+remapped, 59 active+clean; 4.6 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 135 op/s; 64813/225654 objects degraded (28.722%); 20688/225654 objects misplaced (9.168%); 15 MiB/s, 237 objects/s recovering 2023-09-23T21:12:18.594 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:12:18.722 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T21:12:18.937 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:12:18.941 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:12:18.942 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:12:19.042 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:12:19.075 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:12:19.197 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:12:19.200 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:12:19.201 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:12:19.301 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:12:19.314 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:12:19 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/3593518311' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T21:12:19.315 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:12:19 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T21:12:19.328 INFO:teuthology.orchestra.run.smithi026.stderr:noscrub is unset 2023-09-23T21:12:19.380 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:12:19.505 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:12:19 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/3593518311' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T21:12:19.506 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:12:19 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T21:12:19.506 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:12:19 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/3593518311' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T21:12:19.506 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:12:19 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T21:12:19.881 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 21:12:19 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[164029]: ::ffff:172.21.15.70 - - [23/Sep/2023:21:12:19] "GET /metrics HTTP/1.1" 200 36448 "" "Prometheus/2.43.0" 2023-09-23T21:12:20.318 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:12:20.337 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:12:20.337 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:12:20.438 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:12:20.508 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:12:20.573 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:12:20.588 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:12:20.588 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:12:20.588 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:12:20 smithi070 ceph-mon[168282]: pgmap v1211: 119 pgs: 1 peering, 16 active+clean+remapped, 2 activating+undersized+degraded+remapped, 1 activating, 4 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 12 active+undersized, 1 active+recovering+undersized+degraded+remapped, 6 active+undersized+degraded, 8 active+recovery_wait+undersized+degraded+remapped, 6 active+recovery_wait+degraded+remapped, 60 active+clean; 4.6 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 120 op/s; 65319/225219 objects degraded (29.002%); 19885/225219 objects misplaced (8.829%); 12 MiB/s, 193 objects/s recovering 2023-09-23T21:12:20.589 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:12:20 smithi070 ceph-mon[168282]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:12:20.589 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:12:20 smithi070 ceph-mon[168282]: failed to encode map e477 with expected crc 2023-09-23T21:12:20.589 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:12:20 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T21:12:20.589 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:12:20 smithi070 ceph-mon[168282]: osdmap e477: 8 total, 7 up, 6 in 2023-09-23T21:12:20.589 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:12:20 smithi070 ceph-mon[168282]: failed to encode map e477 with expected crc 2023-09-23T21:12:20.589 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:12:20 smithi070 ceph-mon[168282]: failed to encode map e477 with expected crc 2023-09-23T21:12:20.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:12:20 smithi026 ceph-mon[179688]: pgmap v1211: 119 pgs: 1 peering, 16 active+clean+remapped, 2 activating+undersized+degraded+remapped, 1 activating, 4 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 12 active+undersized, 1 active+recovering+undersized+degraded+remapped, 6 active+undersized+degraded, 8 active+recovery_wait+undersized+degraded+remapped, 6 active+recovery_wait+degraded+remapped, 60 active+clean; 4.6 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 120 op/s; 65319/225219 objects degraded (29.002%); 19885/225219 objects misplaced (8.829%); 12 MiB/s, 193 objects/s recovering 2023-09-23T21:12:20.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:12:20 smithi026 ceph-mon[179688]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:12:20.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:12:20 smithi026 ceph-mon[179688]: failed to encode map e477 with expected crc 2023-09-23T21:12:20.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:12:20 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T21:12:20.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:12:20 smithi026 ceph-mon[179688]: osdmap e477: 8 total, 7 up, 6 in 2023-09-23T21:12:20.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:12:20 smithi026 ceph-mon[179688]: failed to encode map e477 with expected crc 2023-09-23T21:12:20.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:12:20 smithi026 ceph-mon[179688]: failed to encode map e477 with expected crc 2023-09-23T21:12:20.633 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:12:20 smithi026 ceph-mon[175151]: pgmap v1211: 119 pgs: 1 peering, 16 active+clean+remapped, 2 activating+undersized+degraded+remapped, 1 activating, 4 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 12 active+undersized, 1 active+recovering+undersized+degraded+remapped, 6 active+undersized+degraded, 8 active+recovery_wait+undersized+degraded+remapped, 6 active+recovery_wait+degraded+remapped, 60 active+clean; 4.6 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 120 op/s; 65319/225219 objects degraded (29.002%); 19885/225219 objects misplaced (8.829%); 12 MiB/s, 193 objects/s recovering 2023-09-23T21:12:20.633 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:12:20 smithi026 ceph-mon[175151]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:12:20.633 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:12:20 smithi026 ceph-mon[175151]: failed to encode map e477 with expected crc 2023-09-23T21:12:20.633 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:12:20 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T21:12:20.633 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:12:20 smithi026 ceph-mon[175151]: osdmap e477: 8 total, 7 up, 6 in 2023-09-23T21:12:20.633 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:12:20 smithi026 ceph-mon[175151]: failed to encode map e477 with expected crc 2023-09-23T21:12:20.633 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:12:20 smithi026 ceph-mon[175151]: failed to encode map e477 with expected crc 2023-09-23T21:12:20.689 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:12:20.720 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:12:20.989 INFO:teuthology.orchestra.run.smithi070.stdout:{ 2023-09-23T21:12:20.990 INFO:teuthology.orchestra.run.smithi070.stdout: "ops": [], 2023-09-23T21:12:20.990 INFO:teuthology.orchestra.run.smithi070.stdout: "complaint_time": 30, 2023-09-23T21:12:20.990 INFO:teuthology.orchestra.run.smithi070.stdout: "num_blocked_ops": 0 2023-09-23T21:12:20.990 INFO:teuthology.orchestra.run.smithi070.stdout:} 2023-09-23T21:12:21.313 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:12:21 smithi070 ceph-mon[168282]: Health check update: Reduced data availability: 1 pg inactive (PG_AVAILABILITY) 2023-09-23T21:12:21.314 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:12:21 smithi070 ceph-mon[168282]: Health check update: Degraded data redundancy: 65319/225219 objects degraded (29.002%), 28 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T21:12:21.493 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_historic_ops 2023-09-23T21:12:21.631 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:12:21 smithi026 ceph-mon[175151]: Health check update: Reduced data availability: 1 pg inactive (PG_AVAILABILITY) 2023-09-23T21:12:21.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:12:21 smithi026 ceph-mon[175151]: Health check update: Degraded data redundancy: 65319/225219 objects degraded (29.002%), 28 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T21:12:21.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:12:21 smithi026 ceph-mon[179688]: Health check update: Reduced data availability: 1 pg inactive (PG_AVAILABILITY) 2023-09-23T21:12:21.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:12:21 smithi026 ceph-mon[179688]: Health check update: Degraded data redundancy: 65319/225219 objects degraded (29.002%), 28 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T21:12:22.126 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T21:12:22.356 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:12:22.360 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:12:22.360 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:12:22.461 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:12:22.554 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:12:22.626 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:12:22 smithi070 ceph-mon[168282]: pgmap v1213: 119 pgs: 1 peering, 16 active+clean+remapped, 4 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+remapped, 12 active+undersized, 1 active+recovering+undersized+degraded+remapped, 6 active+undersized+degraded, 10 active+recovery_wait+undersized+degraded+remapped, 6 active+recovery_wait+degraded+remapped, 61 active+clean; 4.5 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 169 op/s; 64860/223161 objects degraded (29.064%); 19693/223161 objects misplaced (8.825%); 11 MiB/s, 169 objects/s recovering 2023-09-23T21:12:22.626 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:12:22 smithi070 ceph-mon[168282]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive) 2023-09-23T21:12:22.631 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:12:22 smithi026 ceph-mon[179688]: pgmap v1213: 119 pgs: 1 peering, 16 active+clean+remapped, 4 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+remapped, 12 active+undersized, 1 active+recovering+undersized+degraded+remapped, 6 active+undersized+degraded, 10 active+recovery_wait+undersized+degraded+remapped, 6 active+recovery_wait+degraded+remapped, 61 active+clean; 4.5 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 169 op/s; 64860/223161 objects degraded (29.064%); 19693/223161 objects misplaced (8.825%); 11 MiB/s, 169 objects/s recovering 2023-09-23T21:12:22.631 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:12:22 smithi026 ceph-mon[179688]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive) 2023-09-23T21:12:22.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:12:22 smithi026 ceph-mon[175151]: pgmap v1213: 119 pgs: 1 peering, 16 active+clean+remapped, 4 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+remapped, 12 active+undersized, 1 active+recovering+undersized+degraded+remapped, 6 active+undersized+degraded, 10 active+recovery_wait+undersized+degraded+remapped, 6 active+recovery_wait+degraded+remapped, 61 active+clean; 4.5 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 169 op/s; 64860/223161 objects degraded (29.064%); 19693/223161 objects misplaced (8.825%); 11 MiB/s, 169 objects/s recovering 2023-09-23T21:12:22.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:12:22 smithi026 ceph-mon[175151]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive) 2023-09-23T21:12:22.922 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:12:22.928 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:12:22.929 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:12:23.029 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:12:23.064 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:12:23.177 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1: {} 2023-09-23T21:12:23.208 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2: {} 2023-09-23T21:12:23.228 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3: {} 2023-09-23T21:12:23.252 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4: {} 2023-09-23T21:12:23.274 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T21:12:23.299 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6: {} 2023-09-23T21:12:23.326 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T21:12:23.326 INFO:teuthology.orchestra.run.smithi026.stderr:Error ENXIO: problem getting command descriptions from osd.0 2023-09-23T21:12:23.326 INFO:teuthology.orchestra.run.smithi026.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T21:12:23.327 INFO:teuthology.orchestra.run.smithi026.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T21:12:23.327 INFO:teuthology.orchestra.run.smithi026.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T21:12:23.327 INFO:teuthology.orchestra.run.smithi026.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T21:12:23.327 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T21:12:23.327 INFO:teuthology.orchestra.run.smithi026.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T21:12:23.327 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T21:12:23.333 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:12:23 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/1451467494' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:12:23.334 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:12:23 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/1451467494' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:12:23.456 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:12:23.460 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:12:23.460 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:12:23.474 INFO:teuthology.orchestra.run.smithi026.stdout: 2023-09-23T21:12:23.476 INFO:teuthology.orchestra.run.smithi026.stderr:dumped all 2023-09-23T21:12:23.518 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:12:23 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/1451467494' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:12:23.561 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:12:23.626 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:12:23.687 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T21:12:23.688 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T21:12:23.740 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:12:23.753 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:12:23.753 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:12:23.854 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:12:23.906 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:12:24.017 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:12:24.025 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:12:24.025 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:12:24.051 INFO:teuthology.orchestra.run.smithi026.stdout:{"pg_ready":true,"pg_map":{"version":1214,"stamp":"2023-09-23T21:12:23.001706+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4841755185,"num_objects":73896,"num_object_clones":0,"num_object_copies":221688,"num_objects_missing_on_primary":12912,"num_objects_missing":12912,"num_objects_degraded":64548,"num_objects_misplaced":19579,"num_objects_unfound":0,"num_objects_dirty":73896,"num_whiteouts":0,"num_read":67,"num_read_kb":124,"num_write":73596,"num_write_kb":4309672,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12760,"num_bytes_recovered":832851948,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":112623,"ondisk_log_size":112623,"up":314,"acting":326,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":400,"num_osds":6,"num_per_pool_osds":6,"num_per_pool_omap_osds":6,"kb":562470912,"kb_used":16860604,"kb_used_data":13279820,"kb_used_omap":117,"kb_used_meta":3580554,"kb_avail":545610308,"statfs":{"total":575970213888,"available":558704955392,"internally_reserved":0,"allocated":13598535680,"data_stored":13588064850,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":120174,"internal_metadata":3666487954},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,1,0,1,1,1,5,5,5,2],"upper_bound":1024},"perf_stat":{"commit_latency_ms":4,"apply_latency_ms":4,"commit_latency_ns":4000000,"apply_latency_ns":4000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-98762752,"num_objects":-1507,"num_object_clones":0,"num_object_copies":-4521,"num_objects_missing_on_primary":-236,"num_objects_missing":-236,"num_objects_degraded":-344,"num_objects_misplaced":-1738,"num_objects_unfound":0,"num_objects_dirty":-1507,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1507,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1173,"num_bytes_recovered":76873728,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"7.706843"},"pg_stats":[{"pgid":"10.13","version":"476'4055","reported_seq":4490,"reported_epoch":477,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2023-09-23T21:12:19.458792+0000","last_change":"2023-09-23T21:11:49.511106+0000","last_active":"2023-09-23T21:12:19.458792+0000","last_peered":"2023-09-23T21:12:19.458792+0000","last_clean":"2023-09-23T21:11:26.718194+0000","last_became_active":"2023-09-23T21:11:49.450819+0000","last_became_peered":"2023-09-23T21:11:49.450819+0000","last_unstale":"2023-09-23T21:12:19.458792+0000","last_undegraded":"2023-09-23T21:11:47.824462+0000","last_fullsized":"2023-09-23T21:12:19.458792+0000","mapping_epoch":466,"log_start":"0'0","ondisk_log_start":"0'0","created":438,"last_epoch_clean":439,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_clean_scrub_stamp":"2023-09-23T21:10:15.663179+0000","objects_scrubbed":0,"log_size":4055,"log_dups_size":0,"ondisk_log_size":4055,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":153780224,"num_objects":2347,"num_object_clones":0,"num_object_copies":7041,"num_objects_missing_on_primary":1977,"num_objects_missing":1977,"num_objects_degraded":1977,"num_objects_misplaced":2347,"num_objects_unfound":0,"num_objects_dirty":2347,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2687,"num_write_kb":161056,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":171,"num_bytes_recovered":11206656,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5],"acting":[4,5,2],"avail_no_missing":["2","3","5"],"object_location_counts":[{"shards":"2,3,5","objects":1977},{"shards":"2,4,5","objects":370}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1a","version":"68'1","reported_seq":55,"reported_epoch":476,"state":"active+clean+remapped","last_fresh":"2023-09-23T21:12:16.337549+0000","last_change":"2023-09-23T21:12:11.763972+0000","last_active":"2023-09-23T21:12:16.337549+0000","last_peered":"2023-09-23T21:12:16.337549+0000","last_clean":"2023-09-23T21:12:16.337549+0000","last_became_active":"2023-09-23T21:12:11.763522+0000","last_became_peered":"2023-09-23T21:12:11.763522+0000","last_unstale":"2023-09-23T21:12:16.337549+0000","last_undegraded":"2023-09-23T21:12:16.337549+0000","last_fullsized":"2023-09-23T21:12:16.337549+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:05:59.397574+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2],"acting":[4,2,1],"avail_no_missing":["4","1","2"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":136,"reported_epoch":476,"state":"active+clean","last_fresh":"2023-09-23T21:12:16.325460+0000","last_change":"2023-09-23T21:12:10.767963+0000","last_active":"2023-09-23T21:12:16.325460+0000","last_peered":"2023-09-23T21:12:16.325460+0000","last_clean":"2023-09-23T21:12:16.325460+0000","last_became_active":"2023-09-23T21:12:10.767126+0000","last_became_peered":"2023-09-23T21:12:10.767126+0000","last_unstale":"2023-09-23T21:12:16.325460+0000","last_undegraded":"2023-09-23T21:12:16.325460+0000","last_fullsized":"2023-09-23T21:12:16.325460+0000","mapping_epoch":470,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:45:47.138277+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,5],"acting":[4,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.10","version":"476'4046","reported_seq":8814,"reported_epoch":476,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-23T21:12:19.128189+0000","last_change":"2023-09-23T21:11:48.818658+0000","last_active":"2023-09-23T21:12:19.128189+0000","last_peered":"2023-09-23T21:12:19.128189+0000","last_clean":"2023-09-23T21:11:26.716051+0000","last_became_active":"2023-09-23T21:11:48.768606+0000","last_became_peered":"2023-09-23T21:11:48.768606+0000","last_unstale":"2023-09-23T21:12:19.128189+0000","last_undegraded":"2023-09-23T21:11:47.824709+0000","last_fullsized":"2023-09-23T21:12:19.128189+0000","mapping_epoch":466,"log_start":"0'0","ondisk_log_start":"0'0","created":438,"last_epoch_clean":450,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_clean_scrub_stamp":"2023-09-23T21:10:15.663179+0000","objects_scrubbed":0,"log_size":4046,"log_dups_size":0,"ondisk_log_size":4046,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":149782528,"num_objects":2286,"num_object_clones":0,"num_object_copies":6858,"num_objects_missing_on_primary":1948,"num_objects_missing":1948,"num_objects_degraded":1948,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2286,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2682,"num_write_kb":158944,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2075,"num_bytes_recovered":135987200,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,5],"acting":[4,7,5],"avail_no_missing":["1","5","7"],"object_location_counts":[{"shards":"4,5,7","objects":338},{"shards":"5,7","objects":1948}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":471,"reported_epoch":475,"state":"active+clean","last_fresh":"2023-09-23T21:12:15.341190+0000","last_change":"2023-09-23T21:12:10.824116+0000","last_active":"2023-09-23T21:12:15.341190+0000","last_peered":"2023-09-23T21:12:15.341190+0000","last_clean":"2023-09-23T21:12:15.341190+0000","last_became_active":"2023-09-23T21:12:10.823849+0000","last_became_peered":"2023-09-23T21:12:10.823849+0000","last_unstale":"2023-09-23T21:12:15.341190+0000","last_undegraded":"2023-09-23T21:12:15.341190+0000","last_fullsized":"2023-09-23T21:12:15.341190+0000","mapping_epoch":470,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:54:13.777603+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":18,"reported_epoch":476,"state":"active+clean+remapped","last_fresh":"2023-09-23T21:12:16.308508+0000","last_change":"2023-09-23T21:12:11.756254+0000","last_active":"2023-09-23T21:12:16.308508+0000","last_peered":"2023-09-23T21:12:16.308508+0000","last_clean":"2023-09-23T21:12:16.308508+0000","last_became_active":"2023-09-23T21:12:11.755857+0000","last_became_peered":"2023-09-23T21:12:11.755857+0000","last_unstale":"2023-09-23T21:12:16.308508+0000","last_undegraded":"2023-09-23T21:12:16.308508+0000","last_fullsized":"2023-09-23T21:12:16.308508+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:05:33.764259+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7],"acting":[5,7,1],"avail_no_missing":["5","1","7"],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.11","version":"477'3980","reported_seq":5336,"reported_epoch":477,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T21:12:22.236737+0000","last_change":"2023-09-23T21:12:10.834663+0000","last_active":"2023-09-23T21:12:22.236737+0000","last_peered":"2023-09-23T21:12:22.236737+0000","last_clean":"2023-09-23T21:11:06.133399+0000","last_became_active":"2023-09-23T21:11:28.891380+0000","last_became_peered":"2023-09-23T21:11:28.891380+0000","last_unstale":"2023-09-23T21:12:22.236737+0000","last_undegraded":"2023-09-23T21:11:28.793297+0000","last_fullsized":"2023-09-23T21:11:28.730707+0000","mapping_epoch":453,"log_start":"0'0","ondisk_log_start":"0'0","created":438,"last_epoch_clean":439,"parent":"0.0","parent_split_bits":5,"last_scrub":"441'2974","last_scrub_stamp":"2023-09-23T21:11:06.133218+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_clean_scrub_stamp":"2023-09-23T21:11:06.133218+0000","objects_scrubbed":1646,"log_size":3980,"log_dups_size":0,"ondisk_log_size":3980,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:54:46.651039+0000","scrub_duration":0.3173955,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":150142976,"num_objects":2291,"num_object_clones":0,"num_object_copies":6873,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3436,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2291,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1009,"num_write_kb":52928,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":183,"num_bytes_recovered":11993088,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,2],"acting":[7,5],"avail_no_missing":["7","1"],"object_location_counts":[{"shards":"1,7","objects":1467},{"shards":"5,7","objects":322}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":365,"reported_epoch":476,"state":"active+clean","last_fresh":"2023-09-23T21:12:16.297439+0000","last_change":"2023-09-23T21:12:10.778896+0000","last_active":"2023-09-23T21:12:16.297439+0000","last_peered":"2023-09-23T21:12:16.297439+0000","last_clean":"2023-09-23T21:12:16.297439+0000","last_became_active":"2023-09-23T21:12:10.778566+0000","last_became_peered":"2023-09-23T21:12:10.778566+0000","last_unstale":"2023-09-23T21:12:16.297439+0000","last_undegraded":"2023-09-23T21:12:16.297439+0000","last_fullsized":"2023-09-23T21:12:16.297439+0000","mapping_epoch":470,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:29:40.272108+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,7],"acting":[2,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":18,"reported_epoch":476,"state":"active+clean+remapped","last_fresh":"2023-09-23T21:12:16.311035+0000","last_change":"2023-09-23T21:12:11.745494+0000","last_active":"2023-09-23T21:12:16.311035+0000","last_peered":"2023-09-23T21:12:16.311035+0000","last_clean":"2023-09-23T21:12:16.311035+0000","last_became_active":"2023-09-23T21:12:11.745114+0000","last_became_peered":"2023-09-23T21:12:11.745114+0000","last_unstale":"2023-09-23T21:12:16.311035+0000","last_undegraded":"2023-09-23T21:12:16.311035+0000","last_fullsized":"2023-09-23T21:12:16.311035+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:04:33.072687+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5],"acting":[7,5,1],"avail_no_missing":["7","1","5"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":340,"reported_epoch":476,"state":"active+undersized","last_fresh":"2023-09-23T21:12:16.317178+0000","last_change":"2023-09-23T21:11:46.815348+0000","last_active":"2023-09-23T21:12:16.317178+0000","last_peered":"2023-09-23T21:12:16.317178+0000","last_clean":"2023-09-23T21:05:02.271349+0000","last_became_active":"2023-09-23T21:11:46.815348+0000","last_became_peered":"2023-09-23T21:11:46.815348+0000","last_unstale":"2023-09-23T21:12:16.317178+0000","last_undegraded":"2023-09-23T21:12:16.317178+0000","last_fullsized":"2023-09-23T21:11:46.810217+0000","mapping_epoch":465,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:40:20.118349+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3],"acting":[5,3],"avail_no_missing":["5","3"],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":513,"reported_epoch":475,"state":"active+clean","last_fresh":"2023-09-23T21:12:15.341132+0000","last_change":"2023-09-23T21:12:10.790349+0000","last_active":"2023-09-23T21:12:15.341132+0000","last_peered":"2023-09-23T21:12:15.341132+0000","last_clean":"2023-09-23T21:12:15.341132+0000","last_became_active":"2023-09-23T21:12:10.789977+0000","last_became_peered":"2023-09-23T21:12:10.789977+0000","last_unstale":"2023-09-23T21:12:15.341132+0000","last_undegraded":"2023-09-23T21:12:15.341132+0000","last_fullsized":"2023-09-23T21:12:15.341132+0000","mapping_epoch":470,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:49:40.191147+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,5],"acting":[3,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"79'161","reported_seq":837,"reported_epoch":476,"state":"active+clean","last_fresh":"2023-09-23T21:12:16.299420+0000","last_change":"2023-09-23T21:11:32.760292+0000","last_active":"2023-09-23T21:12:16.299420+0000","last_peered":"2023-09-23T21:12:16.299420+0000","last_clean":"2023-09-23T21:12:16.299420+0000","last_became_active":"2023-09-23T21:11:32.759909+0000","last_became_peered":"2023-09-23T21:11:32.759909+0000","last_unstale":"2023-09-23T21:12:16.299420+0000","last_undegraded":"2023-09-23T21:12:16.299420+0000","last_fullsized":"2023-09-23T21:12:16.299420+0000","mapping_epoch":457,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":458,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:25.096401+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:25.096401+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:13:09.225685+0000","scrub_duration":0.002859393,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":201,"reported_epoch":476,"state":"active+clean","last_fresh":"2023-09-23T21:12:16.308293+0000","last_change":"2023-09-23T21:12:10.779468+0000","last_active":"2023-09-23T21:12:16.308293+0000","last_peered":"2023-09-23T21:12:16.308293+0000","last_clean":"2023-09-23T21:12:16.308293+0000","last_became_active":"2023-09-23T21:12:10.779282+0000","last_became_peered":"2023-09-23T21:12:10.779282+0000","last_unstale":"2023-09-23T21:12:16.308293+0000","last_undegraded":"2023-09-23T21:12:16.308293+0000","last_fullsized":"2023-09-23T21:12:16.308293+0000","mapping_epoch":470,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:36:35.417986+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":450,"reported_epoch":475,"state":"active+clean","last_fresh":"2023-09-23T21:12:15.340936+0000","last_change":"2023-09-23T21:10:26.758958+0000","last_active":"2023-09-23T21:12:15.340936+0000","last_peered":"2023-09-23T21:12:15.340936+0000","last_clean":"2023-09-23T21:12:15.340936+0000","last_became_active":"2023-09-23T21:10:26.758539+0000","last_became_peered":"2023-09-23T21:10:26.758539+0000","last_unstale":"2023-09-23T21:12:15.340936+0000","last_undegraded":"2023-09-23T21:12:15.340936+0000","last_fullsized":"2023-09-23T21:12:15.340936+0000","mapping_epoch":423,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":424,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:44:42.319189+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":359,"reported_epoch":475,"state":"active+clean","last_fresh":"2023-09-23T21:12:15.341805+0000","last_change":"2023-09-23T21:11:08.766045+0000","last_active":"2023-09-23T21:12:15.341805+0000","last_peered":"2023-09-23T21:12:15.341805+0000","last_clean":"2023-09-23T21:12:15.341805+0000","last_became_active":"2023-09-23T21:11:08.758504+0000","last_became_peered":"2023-09-23T21:11:08.758504+0000","last_unstale":"2023-09-23T21:12:15.341805+0000","last_undegraded":"2023-09-23T21:12:15.341805+0000","last_fullsized":"2023-09-23T21:12:15.341805+0000","mapping_epoch":443,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":444,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:02:19.482574+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":360,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,4],"acting":[3,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":154,"reported_epoch":476,"state":"active+clean","last_fresh":"2023-09-23T21:12:16.301912+0000","last_change":"2023-09-23T21:12:10.769105+0000","last_active":"2023-09-23T21:12:16.301912+0000","last_peered":"2023-09-23T21:12:16.301912+0000","last_clean":"2023-09-23T21:12:16.301912+0000","last_became_active":"2023-09-23T21:12:10.768824+0000","last_became_peered":"2023-09-23T21:12:10.768824+0000","last_unstale":"2023-09-23T21:12:16.301912+0000","last_undegraded":"2023-09-23T21:12:16.301912+0000","last_fullsized":"2023-09-23T21:12:16.301912+0000","mapping_epoch":470,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:50:57.246204+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.14","version":"79'22","reported_seq":40,"reported_epoch":476,"state":"active+clean","last_fresh":"2023-09-23T21:12:16.325762+0000","last_change":"2023-09-23T21:11:27.770162+0000","last_active":"2023-09-23T21:12:16.325762+0000","last_peered":"2023-09-23T21:12:16.325762+0000","last_clean":"2023-09-23T21:12:16.325762+0000","last_became_active":"2023-09-23T21:11:27.752275+0000","last_became_peered":"2023-09-23T21:11:27.752275+0000","last_unstale":"2023-09-23T21:12:16.325762+0000","last_undegraded":"2023-09-23T21:12:16.325762+0000","last_fullsized":"2023-09-23T21:12:16.325762+0000","mapping_epoch":452,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":453,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:54:44.755597+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4734,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,2],"acting":[4,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":408,"reported_epoch":476,"state":"active+clean","last_fresh":"2023-09-23T21:12:16.347880+0000","last_change":"2023-09-23T21:12:10.839271+0000","last_active":"2023-09-23T21:12:16.347880+0000","last_peered":"2023-09-23T21:12:16.347880+0000","last_clean":"2023-09-23T21:12:16.347880+0000","last_became_active":"2023-09-23T21:12:10.839042+0000","last_became_peered":"2023-09-23T21:12:10.839042+0000","last_unstale":"2023-09-23T21:12:16.347880+0000","last_undegraded":"2023-09-23T21:12:16.347880+0000","last_fullsized":"2023-09-23T21:12:16.347880+0000","mapping_epoch":470,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:10:26.414737+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,2],"acting":[4,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.11","version":"79'161","reported_seq":650,"reported_epoch":476,"state":"active+clean","last_fresh":"2023-09-23T21:12:16.307284+0000","last_change":"2023-09-23T21:12:13.793477+0000","last_active":"2023-09-23T21:12:16.307284+0000","last_peered":"2023-09-23T21:12:16.307284+0000","last_clean":"2023-09-23T21:12:16.307284+0000","last_became_active":"2023-09-23T21:12:13.793060+0000","last_became_peered":"2023-09-23T21:12:13.793060+0000","last_unstale":"2023-09-23T21:12:16.307284+0000","last_undegraded":"2023-09-23T21:12:16.307284+0000","last_fullsized":"2023-09-23T21:12:16.307284+0000","mapping_epoch":473,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":474,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:28.756634+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:28.756634+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:43:58.564606+0000","scrub_duration":0.0064925119999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":85,"reported_epoch":476,"state":"active+clean","last_fresh":"2023-09-23T21:12:16.311956+0000","last_change":"2023-09-23T21:11:27.777154+0000","last_active":"2023-09-23T21:12:16.311956+0000","last_peered":"2023-09-23T21:12:16.311956+0000","last_clean":"2023-09-23T21:12:16.311956+0000","last_became_active":"2023-09-23T21:11:27.754438+0000","last_became_peered":"2023-09-23T21:11:27.754438+0000","last_unstale":"2023-09-23T21:12:16.311956+0000","last_undegraded":"2023-09-23T21:12:16.311956+0000","last_fullsized":"2023-09-23T21:12:16.311956+0000","mapping_epoch":452,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":453,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:27:53.743513+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":17,"reported_epoch":476,"state":"active+clean","last_fresh":"2023-09-23T21:12:16.299115+0000","last_change":"2023-09-23T21:12:10.764264+0000","last_active":"2023-09-23T21:12:16.299115+0000","last_peered":"2023-09-23T21:12:16.299115+0000","last_clean":"2023-09-23T21:12:16.299115+0000","last_became_active":"2023-09-23T21:12:10.763486+0000","last_became_peered":"2023-09-23T21:12:10.763486+0000","last_unstale":"2023-09-23T21:12:16.299115+0000","last_undegraded":"2023-09-23T21:12:16.299115+0000","last_fullsized":"2023-09-23T21:12:16.299115+0000","mapping_epoch":470,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:32:27.577036+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,7],"acting":[2,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.10","version":"79'161","reported_seq":612,"reported_epoch":476,"state":"active+clean","last_fresh":"2023-09-23T21:12:16.365336+0000","last_change":"2023-09-23T21:11:11.716703+0000","last_active":"2023-09-23T21:12:16.365336+0000","last_peered":"2023-09-23T21:12:16.365336+0000","last_clean":"2023-09-23T21:12:16.365336+0000","last_became_active":"2023-09-23T21:11:11.716385+0000","last_became_peered":"2023-09-23T21:11:11.716385+0000","last_unstale":"2023-09-23T21:12:16.365336+0000","last_undegraded":"2023-09-23T21:12:16.365336+0000","last_fullsized":"2023-09-23T21:12:16.365336+0000","mapping_epoch":446,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":447,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:06:50.336992+0000","last_deep_scrub":"79'161","last_deep_scrub_stamp":"2023-09-23T21:06:50.336992+0000","last_clean_scrub_stamp":"2023-09-23T21:06:50.336992+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:17:58.387892+0000","scrub_duration":0.0026150930000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,7],"acting":[4,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":433,"reported_epoch":476,"state":"active+undersized","last_fresh":"2023-09-23T21:12:16.309034+0000","last_change":"2023-09-23T21:11:46.830213+0000","last_active":"2023-09-23T21:12:16.309034+0000","last_peered":"2023-09-23T21:12:16.309034+0000","last_clean":"2023-09-23T21:10:22.735898+0000","last_became_active":"2023-09-23T21:11:46.830213+0000","last_became_peered":"2023-09-23T21:11:46.830213+0000","last_unstale":"2023-09-23T21:12:16.309034+0000","last_undegraded":"2023-09-23T21:12:16.309034+0000","last_fullsized":"2023-09-23T21:11:46.824664+0000","mapping_epoch":465,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":424,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:07:08.645234+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3],"acting":[7,3],"avail_no_missing":["7","3"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":227,"reported_epoch":475,"state":"active+clean","last_fresh":"2023-09-23T21:12:15.341687+0000","last_change":"2023-09-23T21:10:26.758671+0000","last_active":"2023-09-23T21:12:15.341687+0000","last_peered":"2023-09-23T21:12:15.341687+0000","last_clean":"2023-09-23T21:12:15.341687+0000","last_became_active":"2023-09-23T21:10:26.758244+0000","last_became_peered":"2023-09-23T21:10:26.758244+0000","last_unstale":"2023-09-23T21:12:15.341687+0000","last_undegraded":"2023-09-23T21:12:15.341687+0000","last_fullsized":"2023-09-23T21:12:15.341687+0000","mapping_epoch":423,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":424,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:14:02.735609+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.18","version":"477'4078","reported_seq":5105,"reported_epoch":477,"state":"active+undersized+degraded","last_fresh":"2023-09-23T21:12:21.207446+0000","last_change":"2023-09-23T21:11:46.821736+0000","last_active":"2023-09-23T21:12:21.207446+0000","last_peered":"2023-09-23T21:12:21.207446+0000","last_clean":"2023-09-23T21:11:45.720015+0000","last_became_active":"2023-09-23T21:11:46.821736+0000","last_became_peered":"2023-09-23T21:11:46.821736+0000","last_unstale":"2023-09-23T21:12:21.207446+0000","last_undegraded":"2023-09-23T21:11:46.803948+0000","last_fullsized":"2023-09-23T21:11:46.803818+0000","mapping_epoch":465,"log_start":"0'0","ondisk_log_start":"0'0","created":438,"last_epoch_clean":439,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_clean_scrub_stamp":"2023-09-23T21:10:15.663179+0000","objects_scrubbed":0,"log_size":4078,"log_dups_size":0,"ondisk_log_size":4078,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":151420928,"num_objects":2311,"num_object_clones":0,"num_object_copies":6933,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2311,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2311,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2729,"num_write_kb":161248,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":231,"num_bytes_recovered":15138816,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7],"acting":[2,7],"avail_no_missing":["2","7"],"object_location_counts":[{"shards":"2,7","objects":2311}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.13","version":"79'161","reported_seq":542,"reported_epoch":475,"state":"active+clean+remapped","last_fresh":"2023-09-23T21:12:15.341247+0000","last_change":"2023-09-23T21:12:11.813287+0000","last_active":"2023-09-23T21:12:15.341247+0000","last_peered":"2023-09-23T21:12:15.341247+0000","last_clean":"2023-09-23T21:12:15.341247+0000","last_became_active":"2023-09-23T21:12:11.812679+0000","last_became_peered":"2023-09-23T21:12:11.812679+0000","last_unstale":"2023-09-23T21:12:15.341247+0000","last_undegraded":"2023-09-23T21:12:15.341247+0000","last_fullsized":"2023-09-23T21:12:15.341247+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:36.420362+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:36.420362+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:52:15.460156+0000","scrub_duration":0.028838444000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4],"acting":[3,4,1],"avail_no_missing":["3","1","4"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":458,"reported_epoch":476,"state":"active+undersized","last_fresh":"2023-09-23T21:12:16.333870+0000","last_change":"2023-09-23T21:11:46.837134+0000","last_active":"2023-09-23T21:12:16.333870+0000","last_peered":"2023-09-23T21:12:16.333870+0000","last_clean":"2023-09-23T21:11:41.736108+0000","last_became_active":"2023-09-23T21:11:46.837134+0000","last_became_peered":"2023-09-23T21:11:46.837134+0000","last_unstale":"2023-09-23T21:12:16.333870+0000","last_undegraded":"2023-09-23T21:12:16.333870+0000","last_fullsized":"2023-09-23T21:11:46.814012+0000","mapping_epoch":465,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":424,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:48:04.007742+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7],"acting":[4,7],"avail_no_missing":["4","7"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":421,"reported_epoch":475,"state":"active+clean","last_fresh":"2023-09-23T21:12:15.341815+0000","last_change":"2023-09-23T21:11:38.750669+0000","last_active":"2023-09-23T21:12:15.341815+0000","last_peered":"2023-09-23T21:12:15.341815+0000","last_clean":"2023-09-23T21:12:15.341815+0000","last_became_active":"2023-09-23T21:11:38.750299+0000","last_became_peered":"2023-09-23T21:11:38.750299+0000","last_unstale":"2023-09-23T21:12:15.341815+0000","last_undegraded":"2023-09-23T21:12:15.341815+0000","last_fullsized":"2023-09-23T21:12:15.341815+0000","mapping_epoch":460,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":461,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:50:16.788442+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.19","version":"477'4143","reported_seq":5190,"reported_epoch":477,"state":"active+clean","last_fresh":"2023-09-23T21:12:20.605834+0000","last_change":"2023-09-23T21:10:58.790716+0000","last_active":"2023-09-23T21:12:20.605834+0000","last_peered":"2023-09-23T21:12:20.605834+0000","last_clean":"2023-09-23T21:12:20.605834+0000","last_became_active":"2023-09-23T21:10:58.787975+0000","last_became_peered":"2023-09-23T21:10:58.787975+0000","last_unstale":"2023-09-23T21:12:20.605834+0000","last_undegraded":"2023-09-23T21:12:20.605834+0000","last_fullsized":"2023-09-23T21:12:20.605834+0000","mapping_epoch":438,"log_start":"0'0","ondisk_log_start":"0'0","created":438,"last_epoch_clean":439,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_clean_scrub_stamp":"2023-09-23T21:10:15.663179+0000","objects_scrubbed":0,"log_size":4143,"log_dups_size":0,"ondisk_log_size":4143,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":153059328,"num_objects":2336,"num_object_clones":0,"num_object_copies":7008,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2336,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2756,"num_write_kb":162912,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":237,"num_bytes_recovered":15499264,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.12","version":"79'161","reported_seq":690,"reported_epoch":476,"state":"active+clean+remapped","last_fresh":"2023-09-23T21:12:16.307920+0000","last_change":"2023-09-23T21:12:11.756911+0000","last_active":"2023-09-23T21:12:16.307920+0000","last_peered":"2023-09-23T21:12:16.307920+0000","last_clean":"2023-09-23T21:12:16.307920+0000","last_became_active":"2023-09-23T21:12:11.756496+0000","last_became_peered":"2023-09-23T21:12:11.756496+0000","last_unstale":"2023-09-23T21:12:16.307920+0000","last_undegraded":"2023-09-23T21:12:16.307920+0000","last_fullsized":"2023-09-23T21:12:16.307920+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:05:56.785173+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:05:56.785173+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:37:58.942652+0000","scrub_duration":0.0042114370000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7],"acting":[5,7,1],"avail_no_missing":["5","1","7"],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":78,"reported_epoch":475,"state":"active+clean","last_fresh":"2023-09-23T21:12:15.340808+0000","last_change":"2023-09-23T21:11:27.774709+0000","last_active":"2023-09-23T21:12:15.340808+0000","last_peered":"2023-09-23T21:12:15.340808+0000","last_clean":"2023-09-23T21:12:15.340808+0000","last_became_active":"2023-09-23T21:11:27.755028+0000","last_became_peered":"2023-09-23T21:11:27.755028+0000","last_unstale":"2023-09-23T21:12:15.340808+0000","last_undegraded":"2023-09-23T21:12:15.340808+0000","last_fullsized":"2023-09-23T21:12:15.340808+0000","mapping_epoch":452,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":453,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:29:02.058314+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,2],"acting":[3,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":312,"reported_epoch":475,"state":"active+clean","last_fresh":"2023-09-23T21:12:15.344012+0000","last_change":"2023-09-23T21:12:10.765780+0000","last_active":"2023-09-23T21:12:15.344012+0000","last_peered":"2023-09-23T21:12:15.344012+0000","last_clean":"2023-09-23T21:12:15.344012+0000","last_became_active":"2023-09-23T21:12:10.765372+0000","last_became_peered":"2023-09-23T21:12:10.765372+0000","last_unstale":"2023-09-23T21:12:15.344012+0000","last_undegraded":"2023-09-23T21:12:15.344012+0000","last_fullsized":"2023-09-23T21:12:15.344012+0000","mapping_epoch":470,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:56:08.446099+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.6","version":"476'4132","reported_seq":4617,"reported_epoch":476,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2023-09-23T21:12:19.340744+0000","last_change":"2023-09-23T21:11:50.442215+0000","last_active":"2023-09-23T21:12:19.340744+0000","last_peered":"2023-09-23T21:12:19.340744+0000","last_clean":"2023-09-23T21:11:06.744076+0000","last_became_active":"2023-09-23T21:11:50.408881+0000","last_became_peered":"2023-09-23T21:11:50.408881+0000","last_unstale":"2023-09-23T21:12:19.340744+0000","last_undegraded":"2023-09-23T21:11:50.378184+0000","last_fullsized":"2023-09-23T21:12:19.340744+0000","mapping_epoch":466,"log_start":"0'0","ondisk_log_start":"0'0","created":416,"last_epoch_clean":439,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_clean_scrub_stamp":"2023-09-23T21:10:15.663179+0000","objects_scrubbed":0,"log_size":4132,"log_dups_size":0,"ondisk_log_size":4132,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":153354240,"num_objects":2340,"num_object_clones":0,"num_object_copies":7020,"num_objects_missing_on_primary":1511,"num_objects_missing":1511,"num_objects_degraded":2327,"num_objects_misplaced":1524,"num_objects_unfound":0,"num_objects_dirty":2340,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2726,"num_write_kb":162112,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":194,"num_bytes_recovered":12713984,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5],"acting":[4,5,1],"avail_no_missing":["5"],"object_location_counts":[{"shards":"1,4,5","objects":13},{"shards":"1,5","objects":1511},{"shards":"4,5","objects":816}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.d","version":"79'161","reported_seq":620,"reported_epoch":476,"state":"active+clean","last_fresh":"2023-09-23T21:12:16.310152+0000","last_change":"2023-09-23T21:12:10.781851+0000","last_active":"2023-09-23T21:12:16.310152+0000","last_peered":"2023-09-23T21:12:16.310152+0000","last_clean":"2023-09-23T21:12:16.310152+0000","last_became_active":"2023-09-23T21:12:10.781148+0000","last_became_peered":"2023-09-23T21:12:10.781148+0000","last_unstale":"2023-09-23T21:12:16.310152+0000","last_undegraded":"2023-09-23T21:12:16.310152+0000","last_fullsized":"2023-09-23T21:12:16.310152+0000","mapping_epoch":470,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":471,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:09:36.858077+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:09:36.858077+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:32:28.941440+0000","scrub_duration":0.0024987500000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":109,"reported_epoch":476,"state":"active+clean","last_fresh":"2023-09-23T21:12:16.311523+0000","last_change":"2023-09-23T21:12:10.807879+0000","last_active":"2023-09-23T21:12:16.311523+0000","last_peered":"2023-09-23T21:12:16.311523+0000","last_clean":"2023-09-23T21:12:16.311523+0000","last_became_active":"2023-09-23T21:12:10.784639+0000","last_became_peered":"2023-09-23T21:12:10.784639+0000","last_unstale":"2023-09-23T21:12:16.311523+0000","last_undegraded":"2023-09-23T21:12:16.311523+0000","last_fullsized":"2023-09-23T21:12:16.311523+0000","mapping_epoch":470,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:31:27.566762+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":110,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,5],"acting":[7,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":184,"reported_epoch":476,"state":"active+clean","last_fresh":"2023-09-23T21:12:16.309100+0000","last_change":"2023-09-23T21:10:26.750725+0000","last_active":"2023-09-23T21:12:16.309100+0000","last_peered":"2023-09-23T21:12:16.309100+0000","last_clean":"2023-09-23T21:12:16.309100+0000","last_became_active":"2023-09-23T21:10:26.750307+0000","last_became_peered":"2023-09-23T21:10:26.750307+0000","last_unstale":"2023-09-23T21:12:16.309100+0000","last_undegraded":"2023-09-23T21:12:16.309100+0000","last_fullsized":"2023-09-23T21:12:16.309100+0000","mapping_epoch":423,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":424,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:06:22.040548+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,4],"acting":[5,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.7","version":"476'4038","reported_seq":4172,"reported_epoch":476,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T21:12:19.028247+0000","last_change":"2023-09-23T21:11:28.907785+0000","last_active":"2023-09-23T21:12:19.028247+0000","last_peered":"2023-09-23T21:12:19.028247+0000","last_clean":"2023-09-23T21:11:26.699871+0000","last_became_active":"2023-09-23T21:11:28.902567+0000","last_became_peered":"2023-09-23T21:11:28.902567+0000","last_unstale":"2023-09-23T21:12:19.028247+0000","last_undegraded":"2023-09-23T21:11:28.826221+0000","last_fullsized":"2023-09-23T21:11:28.754011+0000","mapping_epoch":453,"log_start":"0'0","ondisk_log_start":"0'0","created":416,"last_epoch_clean":439,"parent":"0.0","parent_split_bits":0,"last_scrub":"441'2973","last_scrub_stamp":"2023-09-23T21:11:06.079326+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_clean_scrub_stamp":"2023-09-23T21:11:06.079326+0000","objects_scrubbed":1647,"log_size":4038,"log_dups_size":0,"ondisk_log_size":4038,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:57:42.965147+0000","scrub_duration":0.44319714500000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":151060480,"num_objects":2305,"num_object_clones":0,"num_object_copies":6915,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2213,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2305,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1070,"num_write_kb":55296,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,4],"acting":[3,5],"avail_no_missing":["3","2","5"],"object_location_counts":[{"shards":"2,3,5","objects":2007},{"shards":"3,5","objects":92}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.c","version":"79'161","reported_seq":820,"reported_epoch":475,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2023-09-23T21:12:15.341505+0000","last_change":"2023-09-23T21:11:47.851317+0000","last_active":"2023-09-23T21:12:15.341505+0000","last_peered":"2023-09-23T21:12:15.341505+0000","last_clean":"2023-09-23T21:11:23.200766+0000","last_became_active":"2023-09-23T21:11:47.850897+0000","last_became_peered":"2023-09-23T21:11:47.850897+0000","last_unstale":"2023-09-23T21:12:15.341505+0000","last_undegraded":"2023-09-23T21:11:47.814123+0000","last_fullsized":"2023-09-23T21:12:15.341505+0000","mapping_epoch":466,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":420,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:22.040907+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:22.040907+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:19:02.707924+0000","scrub_duration":0.013021176000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":1,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":327680,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4],"acting":[3,4,5],"avail_no_missing":["3","5"],"object_location_counts":[{"shards":"3,5","objects":1}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":95,"reported_epoch":476,"state":"active+clean","last_fresh":"2023-09-23T21:12:16.309408+0000","last_change":"2023-09-23T21:12:10.838102+0000","last_active":"2023-09-23T21:12:16.309408+0000","last_peered":"2023-09-23T21:12:16.309408+0000","last_clean":"2023-09-23T21:12:16.309408+0000","last_became_active":"2023-09-23T21:12:10.785015+0000","last_became_peered":"2023-09-23T21:12:10.785015+0000","last_unstale":"2023-09-23T21:12:16.309408+0000","last_undegraded":"2023-09-23T21:12:16.309408+0000","last_fullsized":"2023-09-23T21:12:16.309408+0000","mapping_epoch":470,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:34:16.521069+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,5],"acting":[7,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":427,"reported_epoch":476,"state":"active+clean+remapped","last_fresh":"2023-09-23T21:12:16.333975+0000","last_change":"2023-09-23T21:12:11.783157+0000","last_active":"2023-09-23T21:12:16.333975+0000","last_peered":"2023-09-23T21:12:16.333975+0000","last_clean":"2023-09-23T21:12:16.333975+0000","last_became_active":"2023-09-23T21:12:11.782625+0000","last_became_peered":"2023-09-23T21:12:11.782625+0000","last_unstale":"2023-09-23T21:12:16.333975+0000","last_undegraded":"2023-09-23T21:12:16.333975+0000","last_fullsized":"2023-09-23T21:12:16.333975+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:35:10.580469+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3],"acting":[4,3,1],"avail_no_missing":["4","1","3"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.4","version":"476'4010","reported_seq":4802,"reported_epoch":476,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T21:12:19.147344+0000","last_change":"2023-09-23T21:12:14.321747+0000","last_active":"2023-09-23T21:12:19.147344+0000","last_peered":"2023-09-23T21:12:19.147344+0000","last_clean":"2023-09-23T21:11:06.743842+0000","last_became_active":"2023-09-23T21:12:14.317183+0000","last_became_peered":"2023-09-23T21:12:14.317183+0000","last_unstale":"2023-09-23T21:12:19.147344+0000","last_undegraded":"2023-09-23T21:12:13.030767+0000","last_fullsized":"2023-09-23T21:12:12.933770+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":416,"last_epoch_clean":439,"parent":"0.0","parent_split_bits":0,"last_scrub":"440'2861","last_scrub_stamp":"2023-09-23T21:11:03.842926+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_clean_scrub_stamp":"2023-09-23T21:11:03.842926+0000","objects_scrubbed":1526,"log_size":4010,"log_dups_size":0,"ondisk_log_size":4010,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:14:56.781562+0000","scrub_duration":0.39378383900000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":151453696,"num_objects":2311,"num_object_clones":0,"num_object_copies":6933,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3789,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2311,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1155,"num_write_kb":62080,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":324,"num_bytes_recovered":21233664,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[3,4],"avail_no_missing":["3"],"object_location_counts":[{"shards":"1,3","objects":1293},{"shards":"1,3,4","objects":499},{"shards":"3,4","objects":-53}],"blocked_by":[],"up_primary":4,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.f","version":"79'161","reported_seq":764,"reported_epoch":476,"state":"active+clean","last_fresh":"2023-09-23T21:12:16.317046+0000","last_change":"2023-09-23T21:11:30.750338+0000","last_active":"2023-09-23T21:12:16.317046+0000","last_peered":"2023-09-23T21:12:16.317046+0000","last_clean":"2023-09-23T21:12:16.317046+0000","last_became_active":"2023-09-23T21:11:30.749953+0000","last_became_peered":"2023-09-23T21:11:30.749953+0000","last_unstale":"2023-09-23T21:12:16.317046+0000","last_undegraded":"2023-09-23T21:12:16.317046+0000","last_fullsized":"2023-09-23T21:12:16.317046+0000","mapping_epoch":455,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":456,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:27.880776+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:27.880776+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:07:10.405723+0000","scrub_duration":0.0014004899999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,3],"acting":[5,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":200,"reported_epoch":476,"state":"active+clean","last_fresh":"2023-09-23T21:12:16.316557+0000","last_change":"2023-09-23T21:10:26.765068+0000","last_active":"2023-09-23T21:12:16.316557+0000","last_peered":"2023-09-23T21:12:16.316557+0000","last_clean":"2023-09-23T21:12:16.316557+0000","last_became_active":"2023-09-23T21:10:26.758364+0000","last_became_peered":"2023-09-23T21:10:26.758364+0000","last_unstale":"2023-09-23T21:12:16.316557+0000","last_undegraded":"2023-09-23T21:12:16.316557+0000","last_fullsized":"2023-09-23T21:12:16.316557+0000","mapping_epoch":423,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":424,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:13:21.057529+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":280,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":406,"reported_epoch":476,"state":"active+clean","last_fresh":"2023-09-23T21:12:16.307461+0000","last_change":"2023-09-23T21:12:10.777014+0000","last_active":"2023-09-23T21:12:16.307461+0000","last_peered":"2023-09-23T21:12:16.307461+0000","last_clean":"2023-09-23T21:12:16.307461+0000","last_became_active":"2023-09-23T21:12:10.776675+0000","last_became_peered":"2023-09-23T21:12:10.776675+0000","last_unstale":"2023-09-23T21:12:16.307461+0000","last_undegraded":"2023-09-23T21:12:16.307461+0000","last_fullsized":"2023-09-23T21:12:16.307461+0000","mapping_epoch":470,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:14:16.927368+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.5","version":"477'4016","reported_seq":4142,"reported_epoch":477,"state":"active+undersized+degraded","last_fresh":"2023-09-23T21:12:20.772548+0000","last_change":"2023-09-23T21:11:46.814428+0000","last_active":"2023-09-23T21:12:20.772548+0000","last_peered":"2023-09-23T21:12:20.772548+0000","last_clean":"2023-09-23T21:11:45.768408+0000","last_became_active":"2023-09-23T21:11:46.814428+0000","last_became_peered":"2023-09-23T21:11:46.814428+0000","last_unstale":"2023-09-23T21:12:20.772548+0000","last_undegraded":"2023-09-23T21:11:46.804461+0000","last_fullsized":"2023-09-23T21:11:46.804345+0000","mapping_epoch":465,"log_start":"0'0","ondisk_log_start":"0'0","created":416,"last_epoch_clean":439,"parent":"0.0","parent_split_bits":0,"last_scrub":"440'2840","last_scrub_stamp":"2023-09-23T21:11:03.208833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_clean_scrub_stamp":"2023-09-23T21:11:03.208833+0000","objects_scrubbed":1498,"log_size":4016,"log_dups_size":0,"ondisk_log_size":4016,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:21:59.031975+0000","scrub_duration":0.45608471699999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":153812992,"num_objects":2347,"num_object_clones":0,"num_object_copies":7041,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2347,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2347,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1185,"num_write_kb":65088,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3],"acting":[2,3],"avail_no_missing":["2","3"],"object_location_counts":[{"shards":"2,3","objects":2347}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.e","version":"79'161","reported_seq":635,"reported_epoch":476,"state":"active+clean","last_fresh":"2023-09-23T21:12:16.339572+0000","last_change":"2023-09-23T21:11:30.745762+0000","last_active":"2023-09-23T21:12:16.339572+0000","last_peered":"2023-09-23T21:12:16.339572+0000","last_clean":"2023-09-23T21:12:16.339572+0000","last_became_active":"2023-09-23T21:11:30.745412+0000","last_became_peered":"2023-09-23T21:11:30.745412+0000","last_unstale":"2023-09-23T21:12:16.339572+0000","last_undegraded":"2023-09-23T21:12:16.339572+0000","last_fullsized":"2023-09-23T21:12:16.339572+0000","mapping_epoch":455,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":456,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:10:39.567232+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:10:39.567232+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:52:31.221250+0000","scrub_duration":0.005229694,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":262,"reported_epoch":475,"state":"active+clean+remapped","last_fresh":"2023-09-23T21:12:15.341999+0000","last_change":"2023-09-23T21:12:11.738579+0000","last_active":"2023-09-23T21:12:15.341999+0000","last_peered":"2023-09-23T21:12:15.341999+0000","last_clean":"2023-09-23T21:12:15.341999+0000","last_became_active":"2023-09-23T21:12:11.738193+0000","last_became_peered":"2023-09-23T21:12:11.738193+0000","last_unstale":"2023-09-23T21:12:15.341999+0000","last_undegraded":"2023-09-23T21:12:15.341999+0000","last_fullsized":"2023-09-23T21:12:15.341999+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:32:07.590510+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5],"acting":[3,5,1],"avail_no_missing":["3","1","5"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":134,"reported_epoch":476,"state":"active+clean+remapped","last_fresh":"2023-09-23T21:12:16.348152+0000","last_change":"2023-09-23T21:12:11.776069+0000","last_active":"2023-09-23T21:12:16.348152+0000","last_peered":"2023-09-23T21:12:16.348152+0000","last_clean":"2023-09-23T21:12:16.348152+0000","last_became_active":"2023-09-23T21:12:11.773988+0000","last_became_peered":"2023-09-23T21:12:11.773988+0000","last_unstale":"2023-09-23T21:12:16.348152+0000","last_undegraded":"2023-09-23T21:12:16.348152+0000","last_fullsized":"2023-09-23T21:12:16.348152+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:31:12.442179+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3],"acting":[4,3,2],"avail_no_missing":["4","2","3"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.2","version":"477'4066","reported_seq":4205,"reported_epoch":477,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T21:12:19.343205+0000","last_change":"2023-09-23T21:12:11.841617+0000","last_active":"2023-09-23T21:12:19.343205+0000","last_peered":"2023-09-23T21:12:19.343205+0000","last_clean":"2023-09-23T21:11:45.752822+0000","last_became_active":"2023-09-23T21:12:11.835588+0000","last_became_peered":"2023-09-23T21:12:11.835588+0000","last_unstale":"2023-09-23T21:12:19.343205+0000","last_undegraded":"2023-09-23T21:12:11.758776+0000","last_fullsized":"2023-09-23T21:12:11.733443+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":416,"last_epoch_clean":439,"parent":"0.0","parent_split_bits":0,"last_scrub":"440'3007","last_scrub_stamp":"2023-09-23T21:11:05.553661+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_clean_scrub_stamp":"2023-09-23T21:11:05.553661+0000","objects_scrubbed":1622,"log_size":4066,"log_dups_size":0,"ondisk_log_size":4066,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:57:19.490141+0000","scrub_duration":0.39156952499999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":151912448,"num_objects":2318,"num_object_clones":0,"num_object_copies":6954,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2318,"num_objects_misplaced":2502,"num_objects_unfound":0,"num_objects_dirty":2318,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1064,"num_write_kb":56320,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3],"acting":[4,1],"avail_no_missing":["4","1"],"object_location_counts":[{"shards":"1,4","objects":-33}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.9","version":"79'161","reported_seq":620,"reported_epoch":476,"state":"active+clean","last_fresh":"2023-09-23T21:12:16.298250+0000","last_change":"2023-09-23T21:10:37.697046+0000","last_active":"2023-09-23T21:12:16.298250+0000","last_peered":"2023-09-23T21:12:16.298250+0000","last_clean":"2023-09-23T21:12:16.298250+0000","last_became_active":"2023-09-23T21:10:36.684107+0000","last_became_peered":"2023-09-23T21:10:36.684107+0000","last_unstale":"2023-09-23T21:12:16.298250+0000","last_undegraded":"2023-09-23T21:12:16.298250+0000","last_fullsized":"2023-09-23T21:12:16.298250+0000","mapping_epoch":433,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":434,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:10:37.696951+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:10:37.696951+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:41:10.705836+0000","scrub_duration":0.001386504,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,3],"acting":[2,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":156,"reported_epoch":476,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-23T21:12:16.347743+0000","last_change":"2023-09-23T21:12:10.783913+0000","last_active":"2023-09-23T21:12:16.347743+0000","last_peered":"2023-09-23T21:12:16.347743+0000","last_clean":"2023-09-23T21:12:09.328275+0000","last_became_active":"2023-09-23T21:12:10.781915+0000","last_became_peered":"2023-09-23T21:12:10.781915+0000","last_unstale":"2023-09-23T21:12:16.347743+0000","last_undegraded":"2023-09-23T21:12:10.753452+0000","last_fullsized":"2023-09-23T21:12:16.347743+0000","mapping_epoch":470,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":453,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:29:02.341535+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,7],"acting":[4,3,7],"avail_no_missing":["4","1","3"],"object_location_counts":[{"shards":"1,3,4","objects":1}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":50,"reported_epoch":476,"state":"active+clean","last_fresh":"2023-09-23T21:12:16.324879+0000","last_change":"2023-09-23T21:12:14.313466+0000","last_active":"2023-09-23T21:12:16.324879+0000","last_peered":"2023-09-23T21:12:16.324879+0000","last_clean":"2023-09-23T21:12:16.324879+0000","last_became_active":"2023-09-23T21:12:14.209837+0000","last_became_peered":"2023-09-23T21:12:14.209837+0000","last_unstale":"2023-09-23T21:12:16.324879+0000","last_undegraded":"2023-09-23T21:12:16.324879+0000","last_fullsized":"2023-09-23T21:12:16.324879+0000","mapping_epoch":470,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":474,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:09:40.375241+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,7],"acting":[4,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.3","version":"477'4076","reported_seq":4593,"reported_epoch":477,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2023-09-23T21:12:19.467420+0000","last_change":"2023-09-23T21:11:49.514516+0000","last_active":"2023-09-23T21:12:19.467420+0000","last_peered":"2023-09-23T21:12:19.467420+0000","last_clean":"2023-09-23T21:11:26.700990+0000","last_became_active":"2023-09-23T21:11:49.467619+0000","last_became_peered":"2023-09-23T21:11:49.467619+0000","last_unstale":"2023-09-23T21:12:19.467420+0000","last_undegraded":"2023-09-23T21:11:47.824044+0000","last_fullsized":"2023-09-23T21:12:19.467420+0000","mapping_epoch":466,"log_start":"0'0","ondisk_log_start":"0'0","created":416,"last_epoch_clean":439,"parent":"0.0","parent_split_bits":0,"last_scrub":"440'2873","last_scrub_stamp":"2023-09-23T21:11:02.114783+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_clean_scrub_stamp":"2023-09-23T21:11:02.114783+0000","objects_scrubbed":1532,"log_size":4076,"log_dups_size":0,"ondisk_log_size":4076,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:21:07.466676+0000","scrub_duration":0.32435022600000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":154599424,"num_objects":2359,"num_object_clones":0,"num_object_copies":7077,"num_objects_missing_on_primary":2040,"num_objects_missing":2040,"num_objects_degraded":2040,"num_objects_misplaced":2359,"num_objects_unfound":0,"num_objects_dirty":2359,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1209,"num_write_kb":65152,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":192,"num_bytes_recovered":12582912,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5],"acting":[4,5,2],"avail_no_missing":["2","3","5"],"object_location_counts":[{"shards":"2,3,5","objects":2039},{"shards":"2,4,5","objects":319},{"shards":"2,5","objects":1}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.8","version":"79'161","reported_seq":587,"reported_epoch":476,"state":"active+clean","last_fresh":"2023-09-23T21:12:16.298300+0000","last_change":"2023-09-23T21:06:52.099024+0000","last_active":"2023-09-23T21:12:16.298300+0000","last_peered":"2023-09-23T21:12:16.298300+0000","last_clean":"2023-09-23T21:12:16.298300+0000","last_became_active":"2023-09-23T21:06:49.425012+0000","last_became_peered":"2023-09-23T21:06:49.425012+0000","last_unstale":"2023-09-23T21:12:16.298300+0000","last_undegraded":"2023-09-23T21:12:16.298300+0000","last_fullsized":"2023-09-23T21:12:16.298300+0000","mapping_epoch":359,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":360,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:06:52.098882+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:06:52.098882+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:48:36.414396+0000","scrub_duration":0.0092173299999999993,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":53,"reported_epoch":476,"state":"active+clean","last_fresh":"2023-09-23T21:12:16.334396+0000","last_change":"2023-09-23T21:12:10.823380+0000","last_active":"2023-09-23T21:12:16.334396+0000","last_peered":"2023-09-23T21:12:16.334396+0000","last_clean":"2023-09-23T21:12:16.334396+0000","last_became_active":"2023-09-23T21:12:10.823044+0000","last_became_peered":"2023-09-23T21:12:10.823044+0000","last_unstale":"2023-09-23T21:12:16.334396+0000","last_undegraded":"2023-09-23T21:12:16.334396+0000","last_fullsized":"2023-09-23T21:12:16.334396+0000","mapping_epoch":470,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:53:29.846536+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,5],"acting":[4,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":44,"reported_epoch":476,"state":"active+clean","last_fresh":"2023-09-23T21:12:16.324941+0000","last_change":"2023-09-23T21:12:10.767810+0000","last_active":"2023-09-23T21:12:16.324941+0000","last_peered":"2023-09-23T21:12:16.324941+0000","last_clean":"2023-09-23T21:12:16.324941+0000","last_became_active":"2023-09-23T21:12:10.767576+0000","last_became_peered":"2023-09-23T21:12:10.767576+0000","last_unstale":"2023-09-23T21:12:16.324941+0000","last_undegraded":"2023-09-23T21:12:16.324941+0000","last_fullsized":"2023-09-23T21:12:16.324941+0000","mapping_epoch":470,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:19:49.293648+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.0","version":"477'4045","reported_seq":5302,"reported_epoch":477,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T21:12:19.474944+0000","last_change":"2023-09-23T21:12:14.489327+0000","last_active":"2023-09-23T21:12:19.474944+0000","last_peered":"2023-09-23T21:12:19.474944+0000","last_clean":"2023-09-23T21:11:26.716395+0000","last_became_active":"2023-09-23T21:12:14.462407+0000","last_became_peered":"2023-09-23T21:12:14.462407+0000","last_unstale":"2023-09-23T21:12:19.474944+0000","last_undegraded":"2023-09-23T21:12:14.363293+0000","last_fullsized":"2023-09-23T21:12:11.733443+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":416,"last_epoch_clean":439,"parent":"0.0","parent_split_bits":0,"last_scrub":"440'2961","last_scrub_stamp":"2023-09-23T21:11:04.564028+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_clean_scrub_stamp":"2023-09-23T21:11:04.564028+0000","objects_scrubbed":1608,"log_size":4045,"log_dups_size":0,"ondisk_log_size":4045,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:56:40.236880+0000","scrub_duration":0.41858158299999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":151322624,"num_objects":2309,"num_object_clones":0,"num_object_copies":6927,"num_objects_missing_on_primary":2133,"num_objects_missing":2133,"num_objects_degraded":4593,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2309,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1087,"num_write_kb":57216,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":42,"num_bytes_recovered":2752512,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,5],"acting":[4,7],"avail_no_missing":["1","7"],"object_location_counts":[{"shards":"1,4,7","objects":327},{"shards":"1,7","objects":1982}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.b","version":"79'161","reported_seq":666,"reported_epoch":476,"state":"active+clean+remapped","last_fresh":"2023-09-23T21:12:16.308662+0000","last_change":"2023-09-23T21:12:13.768223+0000","last_active":"2023-09-23T21:12:16.308662+0000","last_peered":"2023-09-23T21:12:16.308662+0000","last_clean":"2023-09-23T21:12:16.308662+0000","last_became_active":"2023-09-23T21:12:13.767831+0000","last_became_peered":"2023-09-23T21:12:13.767831+0000","last_unstale":"2023-09-23T21:12:16.308662+0000","last_undegraded":"2023-09-23T21:12:16.308662+0000","last_fullsized":"2023-09-23T21:12:16.308662+0000","mapping_epoch":473,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":474,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:05:55.772264+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:05:55.772264+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:11:55.359256+0000","scrub_duration":0.0043894679999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2],"acting":[5,2,1],"avail_no_missing":["5","1","2"],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":34,"reported_epoch":476,"state":"active+clean","last_fresh":"2023-09-23T21:12:16.333794+0000","last_change":"2023-09-23T21:11:27.754717+0000","last_active":"2023-09-23T21:12:16.333794+0000","last_peered":"2023-09-23T21:12:16.333794+0000","last_clean":"2023-09-23T21:12:16.333794+0000","last_became_active":"2023-09-23T21:11:27.754376+0000","last_became_peered":"2023-09-23T21:11:27.754376+0000","last_unstale":"2023-09-23T21:12:16.333794+0000","last_undegraded":"2023-09-23T21:12:16.333794+0000","last_fullsized":"2023-09-23T21:12:16.333794+0000","mapping_epoch":452,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":453,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:26:41.734541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":408,"reported_epoch":476,"state":"active+clean","last_fresh":"2023-09-23T21:12:16.363090+0000","last_change":"2023-09-23T21:12:10.811698+0000","last_active":"2023-09-23T21:12:16.363090+0000","last_peered":"2023-09-23T21:12:16.363090+0000","last_clean":"2023-09-23T21:12:16.363090+0000","last_became_active":"2023-09-23T21:12:10.789656+0000","last_became_peered":"2023-09-23T21:12:10.789656+0000","last_unstale":"2023-09-23T21:12:16.363090+0000","last_undegraded":"2023-09-23T21:12:16.363090+0000","last_fullsized":"2023-09-23T21:12:16.363090+0000","mapping_epoch":470,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:39:15.112091+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,5],"acting":[4,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.1","version":"476'4007","reported_seq":5373,"reported_epoch":476,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2023-09-23T21:12:19.160039+0000","last_change":"2023-09-23T21:12:14.329738+0000","last_active":"2023-09-23T21:12:19.160039+0000","last_peered":"2023-09-23T21:12:19.160039+0000","last_clean":"2023-09-23T21:11:26.647513+0000","last_became_active":"2023-09-23T21:11:49.761469+0000","last_became_peered":"2023-09-23T21:11:49.761469+0000","last_unstale":"2023-09-23T21:12:19.160039+0000","last_undegraded":"2023-09-23T21:11:49.719241+0000","last_fullsized":"2023-09-23T21:12:19.160039+0000","mapping_epoch":466,"log_start":"0'0","ondisk_log_start":"0'0","created":416,"last_epoch_clean":439,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_clean_scrub_stamp":"2023-09-23T21:10:15.663179+0000","objects_scrubbed":0,"log_size":4007,"log_dups_size":0,"ondisk_log_size":4007,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":149749760,"num_objects":2285,"num_object_clones":0,"num_object_copies":6855,"num_objects_missing_on_primary":1829,"num_objects_missing":1829,"num_objects_degraded":2102,"num_objects_misplaced":2012,"num_objects_unfound":0,"num_objects_dirty":2285,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2659,"num_write_kb":158208,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":524,"num_bytes_recovered":34340864,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7],"acting":[4,7,1],"avail_no_missing":["7"],"object_location_counts":[{"shards":"1,4,7","objects":183},{"shards":"1,7","objects":1829},{"shards":"4,7","objects":273}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.a","version":"79'161","reported_seq":613,"reported_epoch":476,"state":"active+clean","last_fresh":"2023-09-23T21:12:16.301754+0000","last_change":"2023-09-23T21:11:11.721311+0000","last_active":"2023-09-23T21:12:16.301754+0000","last_peered":"2023-09-23T21:12:16.301754+0000","last_clean":"2023-09-23T21:12:16.301754+0000","last_became_active":"2023-09-23T21:11:11.716202+0000","last_became_peered":"2023-09-23T21:11:11.716202+0000","last_unstale":"2023-09-23T21:12:16.301754+0000","last_undegraded":"2023-09-23T21:12:16.301754+0000","last_fullsized":"2023-09-23T21:12:16.301754+0000","mapping_epoch":446,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":447,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:06:51.075448+0000","last_deep_scrub":"79'161","last_deep_scrub_stamp":"2023-09-23T21:05:32.630292+0000","last_clean_scrub_stamp":"2023-09-23T21:06:51.075448+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:53:03.839438+0000","scrub_duration":0.0054793890000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,4],"acting":[2,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":321,"reported_epoch":475,"state":"active+clean","last_fresh":"2023-09-23T21:12:15.340958+0000","last_change":"2023-09-23T21:12:10.792918+0000","last_active":"2023-09-23T21:12:15.340958+0000","last_peered":"2023-09-23T21:12:15.340958+0000","last_clean":"2023-09-23T21:12:15.340958+0000","last_became_active":"2023-09-23T21:12:10.792679+0000","last_became_peered":"2023-09-23T21:12:10.792679+0000","last_unstale":"2023-09-23T21:12:15.340958+0000","last_undegraded":"2023-09-23T21:12:15.340958+0000","last_fullsized":"2023-09-23T21:12:15.340958+0000","mapping_epoch":470,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:30:48.008690+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,4],"acting":[3,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":453,"reported_epoch":475,"state":"active+clean","last_fresh":"2023-09-23T21:12:15.344202+0000","last_change":"2023-09-23T21:12:10.790812+0000","last_active":"2023-09-23T21:12:15.344202+0000","last_peered":"2023-09-23T21:12:15.344202+0000","last_clean":"2023-09-23T21:12:15.344202+0000","last_became_active":"2023-09-23T21:12:10.790492+0000","last_became_peered":"2023-09-23T21:12:10.790492+0000","last_unstale":"2023-09-23T21:12:15.344202+0000","last_undegraded":"2023-09-23T21:12:15.344202+0000","last_fullsized":"2023-09-23T21:12:15.344202+0000","mapping_epoch":470,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:02:19.863118+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.c","version":"477'5465","reported_seq":6553,"reported_epoch":477,"state":"active+undersized+degraded","last_fresh":"2023-09-23T21:12:22.164358+0000","last_change":"2023-09-23T21:11:46.842543+0000","last_active":"2023-09-23T21:12:22.164358+0000","last_peered":"2023-09-23T21:12:22.164358+0000","last_clean":"2023-09-23T21:11:45.750376+0000","last_became_active":"2023-09-23T21:11:46.842543+0000","last_became_peered":"2023-09-23T21:11:46.842543+0000","last_unstale":"2023-09-23T21:12:22.164358+0000","last_undegraded":"2023-09-23T21:11:46.829931+0000","last_fullsized":"2023-09-23T21:11:46.829830+0000","mapping_epoch":465,"log_start":"427'748","ondisk_log_start":"427'748","created":416,"last_epoch_clean":434,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_clean_scrub_stamp":"2023-09-23T21:10:15.663179+0000","objects_scrubbed":0,"log_size":4717,"log_dups_size":0,"ondisk_log_size":4717,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:33:25.304585+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":299696128,"num_objects":4573,"num_object_clones":0,"num_object_copies":13719,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4573,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4573,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5465,"num_write_kb":321216,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":497,"num_bytes_recovered":32571392,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4],"acting":[7,4],"avail_no_missing":["7","4"],"object_location_counts":[{"shards":"4,7","objects":4573}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.7","version":"79'161","reported_seq":562,"reported_epoch":476,"state":"active+clean+remapped","last_fresh":"2023-09-23T21:12:16.311254+0000","last_change":"2023-09-23T21:12:11.745567+0000","last_active":"2023-09-23T21:12:16.311254+0000","last_peered":"2023-09-23T21:12:16.311254+0000","last_clean":"2023-09-23T21:12:16.311254+0000","last_became_active":"2023-09-23T21:12:11.745108+0000","last_became_peered":"2023-09-23T21:12:11.745108+0000","last_unstale":"2023-09-23T21:12:16.311254+0000","last_undegraded":"2023-09-23T21:12:16.311254+0000","last_fullsized":"2023-09-23T21:12:16.311254+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":3,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:25.916867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:25.916867+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:33:08.734034+0000","scrub_duration":0.001665323,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5],"acting":[7,5,1],"avail_no_missing":["7","1","5"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":483,"reported_epoch":476,"state":"active+clean+remapped","last_fresh":"2023-09-23T21:12:16.309794+0000","last_change":"2023-09-23T21:12:11.747478+0000","last_active":"2023-09-23T21:12:16.309794+0000","last_peered":"2023-09-23T21:12:16.309794+0000","last_clean":"2023-09-23T21:12:16.309794+0000","last_became_active":"2023-09-23T21:12:11.747122+0000","last_became_peered":"2023-09-23T21:12:11.747122+0000","last_unstale":"2023-09-23T21:12:16.309794+0000","last_undegraded":"2023-09-23T21:12:16.309794+0000","last_fullsized":"2023-09-23T21:12:16.309794+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:10:53.216547+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2],"acting":[7,2,1],"avail_no_missing":["7","1","2"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":154,"reported_epoch":476,"state":"active+clean","last_fresh":"2023-09-23T21:12:16.316471+0000","last_change":"2023-09-23T21:05:04.326889+0000","last_active":"2023-09-23T21:12:16.316471+0000","last_peered":"2023-09-23T21:12:16.316471+0000","last_clean":"2023-09-23T21:12:16.316471+0000","last_became_active":"2023-09-23T21:05:04.326032+0000","last_became_peered":"2023-09-23T21:05:04.326032+0000","last_unstale":"2023-09-23T21:12:16.316471+0000","last_undegraded":"2023-09-23T21:12:16.316471+0000","last_fullsized":"2023-09-23T21:12:16.316471+0000","mapping_epoch":332,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:00:17.507889+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":474,"reported_epoch":476,"state":"active+clean+remapped","last_fresh":"2023-09-23T21:12:16.307614+0000","last_change":"2023-09-23T21:12:11.764199+0000","last_active":"2023-09-23T21:12:16.307614+0000","last_peered":"2023-09-23T21:12:16.307614+0000","last_clean":"2023-09-23T21:12:16.307614+0000","last_became_active":"2023-09-23T21:12:11.763820+0000","last_became_peered":"2023-09-23T21:12:11.763820+0000","last_unstale":"2023-09-23T21:12:16.307614+0000","last_undegraded":"2023-09-23T21:12:16.307614+0000","last_fullsized":"2023-09-23T21:12:16.307614+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:09:31.002009+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4],"acting":[5,4,1],"avail_no_missing":["5","1","4"],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.a","version":"477'5349","reported_seq":9267,"reported_epoch":477,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T21:12:21.211737+0000","last_change":"2023-09-23T21:12:10.763654+0000","last_active":"2023-09-23T21:12:21.211737+0000","last_peered":"2023-09-23T21:12:21.211737+0000","last_clean":"2023-09-23T21:11:26.703041+0000","last_became_active":"2023-09-23T21:11:28.955843+0000","last_became_peered":"2023-09-23T21:11:28.955843+0000","last_unstale":"2023-09-23T21:12:21.211737+0000","last_undegraded":"2023-09-23T21:11:28.770144+0000","last_fullsized":"2023-09-23T21:11:28.725682+0000","mapping_epoch":453,"log_start":"422'247","ondisk_log_start":"422'247","created":416,"last_epoch_clean":417,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_clean_scrub_stamp":"2023-09-23T21:10:15.663179+0000","objects_scrubbed":0,"log_size":5102,"log_dups_size":0,"ondisk_log_size":5102,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:01:51.913972+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":296419328,"num_objects":4523,"num_object_clones":0,"num_object_copies":13569,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2374,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4523,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5349,"num_write_kb":315904,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1919,"num_bytes_recovered":125763584,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5],"avail_no_missing":["2","5"],"object_location_counts":[{"shards":"2,5","objects":2149}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.1","version":"79'161","reported_seq":423,"reported_epoch":476,"state":"active+undersized","last_fresh":"2023-09-23T21:12:16.298357+0000","last_change":"2023-09-23T21:11:46.816594+0000","last_active":"2023-09-23T21:12:16.298357+0000","last_peered":"2023-09-23T21:12:16.298357+0000","last_clean":"2023-09-23T20:45:16.177630+0000","last_became_active":"2023-09-23T21:11:46.816594+0000","last_became_peered":"2023-09-23T21:11:46.816594+0000","last_unstale":"2023-09-23T21:12:16.298357+0000","last_undegraded":"2023-09-23T21:12:16.298357+0000","last_fullsized":"2023-09-23T21:11:46.803998+0000","mapping_epoch":465,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":360,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:06:52.575900+0000","last_deep_scrub":"79'161","last_deep_scrub_stamp":"2023-09-23T21:05:30.652189+0000","last_clean_scrub_stamp":"2023-09-23T21:06:52.575900+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5],"acting":[2,5],"avail_no_missing":["2","5"],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":17,"reported_epoch":475,"state":"active+clean+remapped","last_fresh":"2023-09-23T21:12:15.341162+0000","last_change":"2023-09-23T21:12:11.742774+0000","last_active":"2023-09-23T21:12:15.341162+0000","last_peered":"2023-09-23T21:12:15.341162+0000","last_clean":"2023-09-23T21:12:15.341162+0000","last_became_active":"2023-09-23T21:12:11.742366+0000","last_became_peered":"2023-09-23T21:12:11.742366+0000","last_unstale":"2023-09-23T21:12:15.341162+0000","last_undegraded":"2023-09-23T21:12:15.341162+0000","last_fullsized":"2023-09-23T21:12:15.341162+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:33:56.794635+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5],"acting":[3,5,1],"avail_no_missing":["3","1","5"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":400,"reported_epoch":476,"state":"active+clean+remapped","last_fresh":"2023-09-23T21:12:16.348037+0000","last_change":"2023-09-23T21:12:11.823049+0000","last_active":"2023-09-23T21:12:16.348037+0000","last_peered":"2023-09-23T21:12:16.348037+0000","last_clean":"2023-09-23T21:12:16.348037+0000","last_became_active":"2023-09-23T21:12:11.822620+0000","last_became_peered":"2023-09-23T21:12:11.822620+0000","last_unstale":"2023-09-23T21:12:16.348037+0000","last_undegraded":"2023-09-23T21:12:16.348037+0000","last_fullsized":"2023-09-23T21:12:16.348037+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:21:37.599381+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5],"acting":[4,5,1],"avail_no_missing":["4","1","5"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":296,"reported_epoch":476,"state":"active+clean","last_fresh":"2023-09-23T21:12:16.299320+0000","last_change":"2023-09-23T21:12:10.764503+0000","last_active":"2023-09-23T21:12:16.299320+0000","last_peered":"2023-09-23T21:12:16.299320+0000","last_clean":"2023-09-23T21:12:16.299320+0000","last_became_active":"2023-09-23T21:12:10.763933+0000","last_became_peered":"2023-09-23T21:12:10.763933+0000","last_unstale":"2023-09-23T21:12:16.299320+0000","last_undegraded":"2023-09-23T21:12:16.299320+0000","last_fullsized":"2023-09-23T21:12:16.299320+0000","mapping_epoch":470,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:28:33.202825+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,7],"acting":[2,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"10.9","version":"477'4088","reported_seq":5179,"reported_epoch":477,"state":"active+clean","last_fresh":"2023-09-23T21:12:21.152809+0000","last_change":"2023-09-23T21:11:08.062606+0000","last_active":"2023-09-23T21:12:21.152809+0000","last_peered":"2023-09-23T21:12:21.152809+0000","last_clean":"2023-09-23T21:12:21.152809+0000","last_became_active":"2023-09-23T21:10:58.780961+0000","last_became_peered":"2023-09-23T21:10:58.780961+0000","last_unstale":"2023-09-23T21:12:21.152809+0000","last_undegraded":"2023-09-23T21:12:21.152809+0000","last_fullsized":"2023-09-23T21:12:21.152809+0000","mapping_epoch":428,"log_start":"0'0","ondisk_log_start":"0'0","created":416,"last_epoch_clean":439,"parent":"0.0","parent_split_bits":0,"last_scrub":"443'3049","last_scrub_stamp":"2023-09-23T21:11:08.062479+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_clean_scrub_stamp":"2023-09-23T21:11:08.062479+0000","objects_scrubbed":1623,"log_size":4088,"log_dups_size":0,"ondisk_log_size":4088,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:56:22.338088+0000","scrub_duration":0.28207626200000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":148439040,"num_objects":2265,"num_object_clones":0,"num_object_copies":6795,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2265,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1040,"num_write_kb":53824,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.2","version":"79'161","reported_seq":763,"reported_epoch":476,"state":"active+undersized","last_fresh":"2023-09-23T21:12:16.324178+0000","last_change":"2023-09-23T21:11:46.834618+0000","last_active":"2023-09-23T21:12:16.324178+0000","last_peered":"2023-09-23T21:12:16.324178+0000","last_clean":"2023-09-23T21:11:23.208111+0000","last_became_active":"2023-09-23T21:11:46.834618+0000","last_became_peered":"2023-09-23T21:11:46.834618+0000","last_unstale":"2023-09-23T21:12:16.324178+0000","last_undegraded":"2023-09-23T21:12:16.324178+0000","last_fullsized":"2023-09-23T21:11:46.809248+0000","mapping_epoch":465,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":456,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:24.802451+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:24.802451+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:43:16.734450+0000","scrub_duration":0.0090861250000000005,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7],"acting":[4,7],"avail_no_missing":["4","2","7"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":475,"reported_epoch":476,"state":"active+undersized","last_fresh":"2023-09-23T21:12:16.298670+0000","last_change":"2023-09-23T21:11:46.819864+0000","last_active":"2023-09-23T21:12:16.298670+0000","last_peered":"2023-09-23T21:12:16.298670+0000","last_clean":"2023-09-23T21:11:41.737520+0000","last_became_active":"2023-09-23T21:11:46.819864+0000","last_became_peered":"2023-09-23T21:11:46.819864+0000","last_unstale":"2023-09-23T21:12:16.298670+0000","last_undegraded":"2023-09-23T21:12:16.298670+0000","last_fullsized":"2023-09-23T21:11:46.804595+0000","mapping_epoch":465,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:48:26.359499+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3],"acting":[2,3],"avail_no_missing":["2","3"],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":458,"reported_epoch":476,"state":"active+clean+remapped","last_fresh":"2023-09-23T21:12:16.298437+0000","last_change":"2023-09-23T21:12:11.738200+0000","last_active":"2023-09-23T21:12:16.298437+0000","last_peered":"2023-09-23T21:12:16.298437+0000","last_clean":"2023-09-23T21:12:16.298437+0000","last_became_active":"2023-09-23T21:12:11.737743+0000","last_became_peered":"2023-09-23T21:12:11.737743+0000","last_unstale":"2023-09-23T21:12:16.298437+0000","last_undegraded":"2023-09-23T21:12:16.298437+0000","last_fullsized":"2023-09-23T21:12:16.298437+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:00:45.100786+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5],"acting":[2,5,1],"avail_no_missing":["2","1","5"],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":16,"reported_epoch":475,"state":"active+clean","last_fresh":"2023-09-23T21:12:15.344062+0000","last_change":"2023-09-23T21:12:10.765951+0000","last_active":"2023-09-23T21:12:15.344062+0000","last_peered":"2023-09-23T21:12:15.344062+0000","last_clean":"2023-09-23T21:12:15.344062+0000","last_became_active":"2023-09-23T21:12:10.765488+0000","last_became_peered":"2023-09-23T21:12:10.765488+0000","last_unstale":"2023-09-23T21:12:15.344062+0000","last_undegraded":"2023-09-23T21:12:15.344062+0000","last_fullsized":"2023-09-23T21:12:15.344062+0000","mapping_epoch":470,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:30:54.653752+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.b","version":"476'5416","reported_seq":8528,"reported_epoch":476,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-09-23T21:12:18.543005+0000","last_change":"2023-09-23T21:12:14.329339+0000","last_active":"2023-09-23T21:12:18.543005+0000","last_peered":"2023-09-23T21:12:18.543005+0000","last_clean":"2023-09-23T21:11:26.732248+0000","last_became_active":"2023-09-23T21:12:11.816422+0000","last_became_peered":"2023-09-23T21:12:11.816422+0000","last_unstale":"2023-09-23T21:12:18.543005+0000","last_undegraded":"2023-09-23T21:12:11.807865+0000","last_fullsized":"2023-09-23T21:12:11.729298+0000","mapping_epoch":471,"log_start":"426'600","ondisk_log_start":"426'600","created":416,"last_epoch_clean":430,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_clean_scrub_stamp":"2023-09-23T21:10:15.663179+0000","objects_scrubbed":0,"log_size":4816,"log_dups_size":0,"ondisk_log_size":4816,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:18:12.291589+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":306446336,"num_objects":4676,"num_object_clones":0,"num_object_copies":14028,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4676,"num_objects_misplaced":3933,"num_objects_unfound":0,"num_objects_dirty":4676,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5416,"num_write_kb":322944,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1498,"num_bytes_recovered":98041856,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7],"acting":[1,7],"avail_no_missing":["1","7"],"object_location_counts":[{"shards":"1,7","objects":743}],"blocked_by":[],"up_primary":4,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":261,"reported_epoch":476,"state":"active+clean","last_fresh":"2023-09-23T21:12:16.298868+0000","last_change":"2023-09-23T21:10:26.769764+0000","last_active":"2023-09-23T21:12:16.298868+0000","last_peered":"2023-09-23T21:12:16.298868+0000","last_clean":"2023-09-23T21:12:16.298868+0000","last_became_active":"2023-09-23T21:10:26.769326+0000","last_became_peered":"2023-09-23T21:10:26.769326+0000","last_unstale":"2023-09-23T21:12:16.298868+0000","last_undegraded":"2023-09-23T21:12:16.298868+0000","last_fullsized":"2023-09-23T21:12:16.298868+0000","mapping_epoch":423,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":424,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:57:45.219837+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"79'161","reported_seq":650,"reported_epoch":476,"state":"active+clean","last_fresh":"2023-09-23T21:12:16.382336+0000","last_change":"2023-09-23T21:12:16.382036+0000","last_active":"2023-09-23T21:12:16.382336+0000","last_peered":"2023-09-23T21:12:16.382336+0000","last_clean":"2023-09-23T21:12:16.382336+0000","last_became_active":"2023-09-23T21:12:16.381791+0000","last_became_peered":"2023-09-23T21:12:16.381791+0000","last_unstale":"2023-09-23T21:12:16.382336+0000","last_undegraded":"2023-09-23T21:12:16.382336+0000","last_fullsized":"2023-09-23T21:12:16.382336+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:20.738273+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:20.738273+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:23:15.854576+0000","scrub_duration":0.0088336339999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":484,"reported_epoch":476,"state":"active+clean","last_fresh":"2023-09-23T21:12:16.307542+0000","last_change":"2023-09-23T21:11:08.760417+0000","last_active":"2023-09-23T21:12:16.307542+0000","last_peered":"2023-09-23T21:12:16.307542+0000","last_clean":"2023-09-23T21:12:16.307542+0000","last_became_active":"2023-09-23T21:11:08.760150+0000","last_became_peered":"2023-09-23T21:11:08.760150+0000","last_unstale":"2023-09-23T21:12:16.307542+0000","last_undegraded":"2023-09-23T21:12:16.307542+0000","last_fullsized":"2023-09-23T21:12:16.307542+0000","mapping_epoch":443,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":444,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:23:10.849768+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":41,"reported_epoch":476,"state":"active+undersized","last_fresh":"2023-09-23T21:12:16.325093+0000","last_change":"2023-09-23T21:11:46.832968+0000","last_active":"2023-09-23T21:12:16.325093+0000","last_peered":"2023-09-23T21:12:16.325093+0000","last_clean":"2023-09-23T21:11:41.736720+0000","last_became_active":"2023-09-23T21:11:46.832968+0000","last_became_peered":"2023-09-23T21:11:46.832968+0000","last_unstale":"2023-09-23T21:12:16.325093+0000","last_undegraded":"2023-09-23T21:12:16.325093+0000","last_fullsized":"2023-09-23T21:11:46.814439+0000","mapping_epoch":465,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":453,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:58:06.042745+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7],"acting":[4,7],"avail_no_missing":["4","7"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.8","version":"477'3946","reported_seq":4973,"reported_epoch":477,"state":"active+undersized+degraded","last_fresh":"2023-09-23T21:12:20.891346+0000","last_change":"2023-09-23T21:11:46.821191+0000","last_active":"2023-09-23T21:12:20.891346+0000","last_peered":"2023-09-23T21:12:20.891346+0000","last_clean":"2023-09-23T21:11:45.772970+0000","last_became_active":"2023-09-23T21:11:46.821191+0000","last_became_peered":"2023-09-23T21:11:46.821191+0000","last_unstale":"2023-09-23T21:12:20.891346+0000","last_undegraded":"2023-09-23T21:11:46.804035+0000","last_fullsized":"2023-09-23T21:11:46.803901+0000","mapping_epoch":465,"log_start":"0'0","ondisk_log_start":"0'0","created":416,"last_epoch_clean":439,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_clean_scrub_stamp":"2023-09-23T21:10:15.663179+0000","objects_scrubbed":0,"log_size":3946,"log_dups_size":0,"ondisk_log_size":3946,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":146309120,"num_objects":2232,"num_object_clones":0,"num_object_copies":6696,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2232,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2232,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2596,"num_write_kb":154528,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":231,"num_bytes_recovered":15138816,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7],"acting":[2,7],"avail_no_missing":["2","7"],"object_location_counts":[{"shards":"2,7","objects":2232}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"79'161","reported_seq":762,"reported_epoch":476,"state":"active+clean","last_fresh":"2023-09-23T21:12:16.324405+0000","last_change":"2023-09-23T21:12:13.772069+0000","last_active":"2023-09-23T21:12:16.324405+0000","last_peered":"2023-09-23T21:12:16.324405+0000","last_clean":"2023-09-23T21:12:16.324405+0000","last_became_active":"2023-09-23T21:12:13.770055+0000","last_became_peered":"2023-09-23T21:12:13.770055+0000","last_unstale":"2023-09-23T21:12:16.324405+0000","last_undegraded":"2023-09-23T21:12:16.324405+0000","last_fullsized":"2023-09-23T21:12:16.324405+0000","mapping_epoch":473,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":474,"parent":"0.0","parent_split_bits":3,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:05:57.832302+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:05:57.832302+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:52:45.974582+0000","scrub_duration":0.010206847999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,3],"acting":[4,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":17,"reported_epoch":475,"state":"active+clean+remapped","last_fresh":"2023-09-23T21:12:15.340798+0000","last_change":"2023-09-23T21:12:11.739941+0000","last_active":"2023-09-23T21:12:15.340798+0000","last_peered":"2023-09-23T21:12:15.340798+0000","last_clean":"2023-09-23T21:12:15.340798+0000","last_became_active":"2023-09-23T21:12:11.739411+0000","last_became_peered":"2023-09-23T21:12:11.739411+0000","last_unstale":"2023-09-23T21:12:15.340798+0000","last_undegraded":"2023-09-23T21:12:15.340798+0000","last_fullsized":"2023-09-23T21:12:15.340798+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:34:00.765961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7],"acting":[3,7,1],"avail_no_missing":["3","1","7"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":482,"reported_epoch":476,"state":"active+undersized","last_fresh":"2023-09-23T21:12:16.324464+0000","last_change":"2023-09-23T21:11:46.830150+0000","last_active":"2023-09-23T21:12:16.324464+0000","last_peered":"2023-09-23T21:12:16.324464+0000","last_clean":"2023-09-23T21:11:23.208635+0000","last_became_active":"2023-09-23T21:11:46.830150+0000","last_became_peered":"2023-09-23T21:11:46.830150+0000","last_unstale":"2023-09-23T21:12:16.324464+0000","last_undegraded":"2023-09-23T21:12:16.324464+0000","last_fullsized":"2023-09-23T21:11:46.811574+0000","mapping_epoch":465,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":453,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:28:27.977549+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5],"acting":[4,5],"avail_no_missing":["4","5"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":377,"reported_epoch":476,"state":"active+clean","last_fresh":"2023-09-23T21:12:16.299498+0000","last_change":"2023-09-23T21:12:10.778963+0000","last_active":"2023-09-23T21:12:16.299498+0000","last_peered":"2023-09-23T21:12:16.299498+0000","last_clean":"2023-09-23T21:12:16.299498+0000","last_became_active":"2023-09-23T21:12:10.778599+0000","last_became_peered":"2023-09-23T21:12:10.778599+0000","last_unstale":"2023-09-23T21:12:16.299498+0000","last_undegraded":"2023-09-23T21:12:16.299498+0000","last_fullsized":"2023-09-23T21:12:16.299498+0000","mapping_epoch":470,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:02:49.717752+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,5],"acting":[2,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"10.d","version":"477'5347","reported_seq":5436,"reported_epoch":477,"state":"active+undersized+degraded+remapped+backfill_wait","last_fresh":"2023-09-23T21:12:19.341835+0000","last_change":"2023-09-23T21:11:46.836790+0000","last_active":"2023-09-23T21:12:19.341835+0000","last_peered":"2023-09-23T21:12:19.341835+0000","last_clean":"2023-09-23T21:11:37.659929+0000","last_became_active":"2023-09-23T21:11:46.833285+0000","last_became_peered":"2023-09-23T21:11:46.833285+0000","last_unstale":"2023-09-23T21:12:19.341835+0000","last_undegraded":"2023-09-23T21:11:46.812100+0000","last_fullsized":"2023-09-23T21:11:46.810940+0000","mapping_epoch":465,"log_start":"434'1079","ondisk_log_start":"434'1079","created":416,"last_epoch_clean":417,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_clean_scrub_stamp":"2023-09-23T21:10:15.663179+0000","objects_scrubbed":0,"log_size":4268,"log_dups_size":0,"ondisk_log_size":4268,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:19:23.440938+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":305004568,"num_objects":4655,"num_object_clones":0,"num_object_copies":13965,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4655,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4655,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":5347,"num_write_kb":320001,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,7],"acting":[4,3],"avail_no_missing":["4","3"],"object_location_counts":[{"shards":"3,4","objects":4655}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.6","version":"79'161","reported_seq":445,"reported_epoch":476,"state":"active+clean","last_fresh":"2023-09-23T21:12:16.363016+0000","last_change":"2023-09-23T21:11:41.744194+0000","last_active":"2023-09-23T21:12:16.363016+0000","last_peered":"2023-09-23T21:12:16.363016+0000","last_clean":"2023-09-23T21:12:16.363016+0000","last_became_active":"2023-09-23T21:11:41.743871+0000","last_became_peered":"2023-09-23T21:11:41.743871+0000","last_unstale":"2023-09-23T21:12:16.363016+0000","last_undegraded":"2023-09-23T21:12:16.363016+0000","last_fullsized":"2023-09-23T21:12:16.363016+0000","mapping_epoch":463,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":464,"parent":"0.0","parent_split_bits":3,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:10:24.268662+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:10:24.268662+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80504,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,7],"acting":[4,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":93,"reported_epoch":476,"state":"active+clean","last_fresh":"2023-09-23T21:12:16.310251+0000","last_change":"2023-09-23T21:12:10.781452+0000","last_active":"2023-09-23T21:12:16.310251+0000","last_peered":"2023-09-23T21:12:16.310251+0000","last_clean":"2023-09-23T21:12:16.310251+0000","last_became_active":"2023-09-23T21:12:10.780751+0000","last_became_peered":"2023-09-23T21:12:10.780751+0000","last_unstale":"2023-09-23T21:12:16.310251+0000","last_undegraded":"2023-09-23T21:12:16.310251+0000","last_fullsized":"2023-09-23T21:12:16.310251+0000","mapping_epoch":470,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:24:28.813675+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,3],"acting":[7,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":350,"reported_epoch":476,"state":"active+clean","last_fresh":"2023-09-23T21:12:16.299520+0000","last_change":"2023-09-23T21:12:14.210956+0000","last_active":"2023-09-23T21:12:16.299520+0000","last_peered":"2023-09-23T21:12:16.299520+0000","last_clean":"2023-09-23T21:12:16.299520+0000","last_became_active":"2023-09-23T21:12:14.210543+0000","last_became_peered":"2023-09-23T21:12:14.210543+0000","last_unstale":"2023-09-23T21:12:16.299520+0000","last_undegraded":"2023-09-23T21:12:16.299520+0000","last_fullsized":"2023-09-23T21:12:16.299520+0000","mapping_epoch":470,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":474,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:21:13.029295+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,4],"acting":[2,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":438,"reported_epoch":476,"state":"active+undersized","last_fresh":"2023-09-23T21:12:16.308534+0000","last_change":"2023-09-23T21:11:46.830171+0000","last_active":"2023-09-23T21:12:16.308534+0000","last_peered":"2023-09-23T21:12:16.308534+0000","last_clean":"2023-09-23T21:10:22.736238+0000","last_became_active":"2023-09-23T21:11:46.830171+0000","last_became_peered":"2023-09-23T21:11:46.830171+0000","last_unstale":"2023-09-23T21:12:16.308534+0000","last_undegraded":"2023-09-23T21:12:16.308534+0000","last_fullsized":"2023-09-23T21:11:46.810928+0000","mapping_epoch":465,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":424,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:10:04.471875+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7],"acting":[5,7],"avail_no_missing":["5","7"],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.e","version":"477'5515","reported_seq":10630,"reported_epoch":477,"state":"active+recovering+undersized+remapped","last_fresh":"2023-09-23T21:12:21.236648+0000","last_change":"2023-09-23T21:12:10.849863+0000","last_active":"2023-09-23T21:12:21.236648+0000","last_peered":"2023-09-23T21:12:21.236648+0000","last_clean":"2023-09-23T21:11:06.775834+0000","last_became_active":"2023-09-23T21:11:09.903191+0000","last_became_peered":"2023-09-23T21:11:09.903191+0000","last_unstale":"2023-09-23T21:12:21.236648+0000","last_undegraded":"2023-09-23T21:12:21.236648+0000","last_fullsized":"2023-09-23T21:11:09.761490+0000","mapping_epoch":444,"log_start":"422'335","ondisk_log_start":"422'335","created":416,"last_epoch_clean":417,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_clean_scrub_stamp":"2023-09-23T21:10:15.663179+0000","objects_scrubbed":0,"log_size":5180,"log_dups_size":0,"ondisk_log_size":5180,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:41:00.068289+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":308609024,"num_objects":4709,"num_object_clones":0,"num_object_copies":14127,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":895,"num_objects_unfound":0,"num_objects_dirty":4709,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5515,"num_write_kb":327168,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2517,"num_bytes_recovered":164954112,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[2,5],"avail_no_missing":["2","3","5"],"object_location_counts":[{"shards":"2,3,5","objects":895},{"shards":"2,5","objects":3814}],"blocked_by":[],"up_primary":4,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.5","version":"79'161","reported_seq":464,"reported_epoch":476,"state":"active+clean","last_fresh":"2023-09-23T21:12:16.325019+0000","last_change":"2023-09-23T21:11:41.740899+0000","last_active":"2023-09-23T21:12:16.325019+0000","last_peered":"2023-09-23T21:12:16.325019+0000","last_clean":"2023-09-23T21:12:16.325019+0000","last_became_active":"2023-09-23T21:11:41.740507+0000","last_became_peered":"2023-09-23T21:11:41.740507+0000","last_unstale":"2023-09-23T21:12:16.325019+0000","last_undegraded":"2023-09-23T21:12:16.325019+0000","last_fullsized":"2023-09-23T21:12:16.325019+0000","mapping_epoch":463,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":464,"parent":"0.0","parent_split_bits":3,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:09:34.892947+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:09:34.892947+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,3],"acting":[4,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":444,"reported_epoch":475,"state":"active+undersized+degraded","last_fresh":"2023-09-23T21:12:15.344268+0000","last_change":"2023-09-23T21:11:46.830062+0000","last_active":"2023-09-23T21:12:15.344268+0000","last_peered":"2023-09-23T21:12:15.344268+0000","last_clean":"2023-09-23T21:10:34.706419+0000","last_became_active":"2023-09-23T21:11:46.830062+0000","last_became_peered":"2023-09-23T21:11:46.830062+0000","last_unstale":"2023-09-23T21:12:15.344268+0000","last_undegraded":"2023-09-23T21:11:46.812950+0000","last_fullsized":"2023-09-23T21:11:46.812868+0000","mapping_epoch":465,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":434,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:35:42.073916+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7],"acting":[3,7],"avail_no_missing":["3","7"],"object_location_counts":[{"shards":"3,7","objects":1}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":447,"reported_epoch":476,"state":"active+clean","last_fresh":"2023-09-23T21:12:16.362920+0000","last_change":"2023-09-23T21:12:10.771141+0000","last_active":"2023-09-23T21:12:16.362920+0000","last_peered":"2023-09-23T21:12:16.362920+0000","last_clean":"2023-09-23T21:12:16.362920+0000","last_became_active":"2023-09-23T21:12:10.770882+0000","last_became_peered":"2023-09-23T21:12:10.770882+0000","last_unstale":"2023-09-23T21:12:16.362920+0000","last_undegraded":"2023-09-23T21:12:16.362920+0000","last_fullsized":"2023-09-23T21:12:16.362920+0000","mapping_epoch":470,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:35:28.579234+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":307,"reported_epoch":475,"state":"active+clean","last_fresh":"2023-09-23T21:12:15.340995+0000","last_change":"2023-09-23T21:12:10.758758+0000","last_active":"2023-09-23T21:12:15.340995+0000","last_peered":"2023-09-23T21:12:15.340995+0000","last_clean":"2023-09-23T21:12:15.340995+0000","last_became_active":"2023-09-23T21:12:10.758400+0000","last_became_peered":"2023-09-23T21:12:10.758400+0000","last_unstale":"2023-09-23T21:12:15.340995+0000","last_undegraded":"2023-09-23T21:12:15.340995+0000","last_fullsized":"2023-09-23T21:12:15.340995+0000","mapping_epoch":470,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:48:32.753383+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.f","version":"476'5458","reported_seq":5605,"reported_epoch":476,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T21:12:19.130640+0000","last_change":"2023-09-23T21:12:10.800464+0000","last_active":"2023-09-23T21:12:19.130640+0000","last_peered":"2023-09-23T21:12:19.130640+0000","last_clean":"2023-09-23T21:11:07.652290+0000","last_became_active":"2023-09-23T21:11:09.897463+0000","last_became_peered":"2023-09-23T21:11:09.897463+0000","last_unstale":"2023-09-23T21:12:19.130640+0000","last_undegraded":"2023-09-23T21:11:27.781677+0000","last_fullsized":"2023-09-23T21:11:09.761652+0000","mapping_epoch":444,"log_start":"434'1194","ondisk_log_start":"434'1194","created":416,"last_epoch_clean":417,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_clean_scrub_stamp":"2023-09-23T21:10:15.663179+0000","objects_scrubbed":0,"log_size":4264,"log_dups_size":0,"ondisk_log_size":4264,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:42:51.588441+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":305135616,"num_objects":4656,"num_object_clones":0,"num_object_copies":13968,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3295,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4656,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5458,"num_write_kb":323648,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":32,"num_bytes_recovered":2097152,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2],"avail_no_missing":["3","1","2"],"object_location_counts":[{"shards":"1,2,3","objects":2926},{"shards":"2,3","objects":1361}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.4","version":"79'161","reported_seq":845,"reported_epoch":475,"state":"active+clean","last_fresh":"2023-09-23T21:12:15.341703+0000","last_change":"2023-09-23T21:10:29.807785+0000","last_active":"2023-09-23T21:12:15.341703+0000","last_peered":"2023-09-23T21:12:15.341703+0000","last_clean":"2023-09-23T21:12:15.341703+0000","last_became_active":"2023-09-23T21:10:29.807366+0000","last_became_peered":"2023-09-23T21:10:29.807366+0000","last_unstale":"2023-09-23T21:12:15.341703+0000","last_undegraded":"2023-09-23T21:12:15.341703+0000","last_fullsized":"2023-09-23T21:12:15.341703+0000","mapping_epoch":426,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":427,"parent":"0.0","parent_split_bits":3,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:20.012764+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:20.012764+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:24:45.763834+0000","scrub_duration":0.0045251919999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":414,"reported_epoch":476,"state":"active+clean","last_fresh":"2023-09-23T21:12:16.299085+0000","last_change":"2023-09-23T21:10:26.768364+0000","last_active":"2023-09-23T21:12:16.299085+0000","last_peered":"2023-09-23T21:12:16.299085+0000","last_clean":"2023-09-23T21:12:16.299085+0000","last_became_active":"2023-09-23T21:10:26.768052+0000","last_became_peered":"2023-09-23T21:10:26.768052+0000","last_unstale":"2023-09-23T21:12:16.299085+0000","last_undegraded":"2023-09-23T21:12:16.299085+0000","last_fullsized":"2023-09-23T21:12:16.299085+0000","mapping_epoch":423,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":424,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:22:42.201200+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,3],"acting":[2,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.6","version":"70'1","reported_seq":123,"reported_epoch":476,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-23T21:12:16.363191+0000","last_change":"2023-09-23T21:12:10.789858+0000","last_active":"2023-09-23T21:12:16.363191+0000","last_peered":"2023-09-23T21:12:16.363191+0000","last_clean":"2023-09-23T20:54:11.906586+0000","last_became_active":"2023-09-23T21:12:10.789194+0000","last_became_peered":"2023-09-23T21:12:10.789194+0000","last_unstale":"2023-09-23T21:12:16.363191+0000","last_undegraded":"2023-09-23T21:12:10.750324+0000","last_fullsized":"2023-09-23T21:12:16.363191+0000","mapping_epoch":470,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":453,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:19:41.468999+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,5],"acting":[4,7,5],"avail_no_missing":["4","1","7"],"object_location_counts":[{"shards":"1,4,7","objects":1}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":41,"reported_epoch":476,"state":"active+undersized","last_fresh":"2023-09-23T21:12:16.333671+0000","last_change":"2023-09-23T21:11:46.830938+0000","last_active":"2023-09-23T21:12:16.333671+0000","last_peered":"2023-09-23T21:12:16.333671+0000","last_clean":"2023-09-23T21:11:41.735953+0000","last_became_active":"2023-09-23T21:11:46.830938+0000","last_became_peered":"2023-09-23T21:11:46.830938+0000","last_unstale":"2023-09-23T21:12:16.333671+0000","last_undegraded":"2023-09-23T21:12:16.333671+0000","last_fullsized":"2023-09-23T21:11:46.811985+0000","mapping_epoch":465,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":453,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:41:33.479200+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5],"acting":[4,5],"avail_no_missing":["4","5"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.12","version":"477'4026","reported_seq":4126,"reported_epoch":477,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T21:12:19.340512+0000","last_change":"2023-09-23T21:12:11.841578+0000","last_active":"2023-09-23T21:12:19.340512+0000","last_peered":"2023-09-23T21:12:19.340512+0000","last_clean":"2023-09-23T21:11:45.774771+0000","last_became_active":"2023-09-23T21:12:11.835919+0000","last_became_peered":"2023-09-23T21:12:11.835919+0000","last_unstale":"2023-09-23T21:12:19.340512+0000","last_undegraded":"2023-09-23T21:12:11.761910+0000","last_fullsized":"2023-09-23T21:12:11.739805+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":438,"last_epoch_clean":439,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_clean_scrub_stamp":"2023-09-23T21:10:15.663179+0000","objects_scrubbed":0,"log_size":4026,"log_dups_size":0,"ondisk_log_size":4026,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":149880832,"num_objects":2287,"num_object_clones":0,"num_object_copies":6861,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2287,"num_objects_misplaced":2471,"num_objects_unfound":0,"num_objects_dirty":2287,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2649,"num_write_kb":157952,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3],"acting":[4,1],"avail_no_missing":["4","1"],"object_location_counts":[{"shards":"1,4","objects":-47}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":131,"reported_epoch":476,"state":"active+clean","last_fresh":"2023-09-23T21:12:16.347958+0000","last_change":"2023-09-23T21:12:10.784156+0000","last_active":"2023-09-23T21:12:16.347958+0000","last_peered":"2023-09-23T21:12:16.347958+0000","last_clean":"2023-09-23T21:12:16.347958+0000","last_became_active":"2023-09-23T21:12:10.783484+0000","last_became_peered":"2023-09-23T21:12:10.783484+0000","last_unstale":"2023-09-23T21:12:16.347958+0000","last_undegraded":"2023-09-23T21:12:16.347958+0000","last_fullsized":"2023-09-23T21:12:16.347958+0000","mapping_epoch":470,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:47:38.124943+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":347,"reported_epoch":475,"state":"active+clean","last_fresh":"2023-09-23T21:12:15.341866+0000","last_change":"2023-09-23T21:12:14.210932+0000","last_active":"2023-09-23T21:12:15.341866+0000","last_peered":"2023-09-23T21:12:15.341866+0000","last_clean":"2023-09-23T21:12:15.341866+0000","last_became_active":"2023-09-23T21:12:14.210596+0000","last_became_peered":"2023-09-23T21:12:14.210596+0000","last_unstale":"2023-09-23T21:12:15.341866+0000","last_undegraded":"2023-09-23T21:12:15.341866+0000","last_fullsized":"2023-09-23T21:12:15.341866+0000","mapping_epoch":470,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":474,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:06:46.330643+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,5],"acting":[3,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.15","version":"477'4012","reported_seq":4096,"reported_epoch":477,"state":"active+undersized+degraded","last_fresh":"2023-09-23T21:12:21.152447+0000","last_change":"2023-09-23T21:11:46.814238+0000","last_active":"2023-09-23T21:12:21.152447+0000","last_peered":"2023-09-23T21:12:21.152447+0000","last_clean":"2023-09-23T21:11:45.765730+0000","last_became_active":"2023-09-23T21:11:46.814238+0000","last_became_peered":"2023-09-23T21:11:46.814238+0000","last_unstale":"2023-09-23T21:12:21.152447+0000","last_undegraded":"2023-09-23T21:11:46.804145+0000","last_fullsized":"2023-09-23T21:11:46.804041+0000","mapping_epoch":465,"log_start":"0'0","ondisk_log_start":"0'0","created":438,"last_epoch_clean":439,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_clean_scrub_stamp":"2023-09-23T21:10:15.663179+0000","objects_scrubbed":0,"log_size":4012,"log_dups_size":0,"ondisk_log_size":4012,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":148766720,"num_objects":2270,"num_object_clones":0,"num_object_copies":6810,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2270,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2270,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2678,"num_write_kb":158336,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3],"acting":[2,3],"avail_no_missing":["2","3"],"object_location_counts":[{"shards":"2,3","objects":2270}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":52,"reported_epoch":476,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-23T21:12:16.324988+0000","last_change":"2023-09-23T21:12:14.313202+0000","last_active":"2023-09-23T21:12:16.324988+0000","last_peered":"2023-09-23T21:12:16.324988+0000","last_clean":"2023-09-23T21:12:09.299355+0000","last_became_active":"2023-09-23T21:12:14.209408+0000","last_became_peered":"2023-09-23T21:12:14.209408+0000","last_unstale":"2023-09-23T21:12:16.324988+0000","last_undegraded":"2023-09-23T21:12:12.933857+0000","last_fullsized":"2023-09-23T21:12:16.324988+0000","mapping_epoch":470,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":444,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:45:40.142413+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,7],"acting":[4,5,7],"avail_no_missing":["4","1","5"],"object_location_counts":[{"shards":"1,4,5","objects":1}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":44,"reported_epoch":476,"state":"active+clean","last_fresh":"2023-09-23T21:12:16.337003+0000","last_change":"2023-09-23T21:12:10.770474+0000","last_active":"2023-09-23T21:12:16.337003+0000","last_peered":"2023-09-23T21:12:16.337003+0000","last_clean":"2023-09-23T21:12:16.337003+0000","last_became_active":"2023-09-23T21:12:10.770184+0000","last_became_peered":"2023-09-23T21:12:10.770184+0000","last_unstale":"2023-09-23T21:12:16.337003+0000","last_undegraded":"2023-09-23T21:12:16.337003+0000","last_fullsized":"2023-09-23T21:12:16.337003+0000","mapping_epoch":470,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:05:09.407151+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.14","version":"477'3953","reported_seq":4424,"reported_epoch":477,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T21:12:19.460542+0000","last_change":"2023-09-23T21:12:14.317101+0000","last_active":"2023-09-23T21:12:19.460542+0000","last_peered":"2023-09-23T21:12:19.460542+0000","last_clean":"2023-09-23T21:11:06.743945+0000","last_became_active":"2023-09-23T21:12:14.227492+0000","last_became_peered":"2023-09-23T21:12:14.227492+0000","last_unstale":"2023-09-23T21:12:19.460542+0000","last_undegraded":"2023-09-23T21:12:12.924098+0000","last_fullsized":"2023-09-23T21:12:10.748734+0000","mapping_epoch":470,"log_start":"0'0","ondisk_log_start":"0'0","created":438,"last_epoch_clean":439,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_clean_scrub_stamp":"2023-09-23T21:10:15.663179+0000","objects_scrubbed":0,"log_size":3953,"log_dups_size":0,"ondisk_log_size":3953,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":150142976,"num_objects":2291,"num_object_clones":0,"num_object_copies":6873,"num_objects_missing_on_primary":1474,"num_objects_missing":1474,"num_objects_degraded":3889,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2291,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2619,"num_write_kb":157120,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":166,"num_bytes_recovered":10813440,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3],"avail_no_missing":["3"],"object_location_counts":[{"shards":"1,3","objects":1474},{"shards":"1,3,4","objects":289},{"shards":"3,4","objects":5}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":265,"reported_epoch":476,"state":"active+clean","last_fresh":"2023-09-23T21:12:16.299041+0000","last_change":"2023-09-23T21:12:10.771160+0000","last_active":"2023-09-23T21:12:16.299041+0000","last_peered":"2023-09-23T21:12:16.299041+0000","last_clean":"2023-09-23T21:12:16.299041+0000","last_became_active":"2023-09-23T21:12:10.757808+0000","last_became_peered":"2023-09-23T21:12:10.757808+0000","last_unstale":"2023-09-23T21:12:16.299041+0000","last_undegraded":"2023-09-23T21:12:16.299041+0000","last_fullsized":"2023-09-23T21:12:16.299041+0000","mapping_epoch":470,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:30:50.987139+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":41,"reported_epoch":476,"state":"active+undersized","last_fresh":"2023-09-23T21:12:16.324158+0000","last_change":"2023-09-23T21:11:46.816682+0000","last_active":"2023-09-23T21:12:16.324158+0000","last_peered":"2023-09-23T21:12:16.324158+0000","last_clean":"2023-09-23T21:11:41.736396+0000","last_became_active":"2023-09-23T21:11:46.816682+0000","last_became_peered":"2023-09-23T21:11:46.816682+0000","last_unstale":"2023-09-23T21:12:16.324158+0000","last_undegraded":"2023-09-23T21:12:16.324158+0000","last_fullsized":"2023-09-23T21:11:46.808899+0000","mapping_epoch":465,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":453,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:24:58.262409+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3],"acting":[4,3],"avail_no_missing":["4","2","3"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.17","version":"476'4025","reported_seq":4113,"reported_epoch":476,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T21:12:19.027390+0000","last_change":"2023-09-23T21:11:28.906469+0000","last_active":"2023-09-23T21:12:19.027390+0000","last_peered":"2023-09-23T21:12:19.027390+0000","last_clean":"2023-09-23T21:11:26.717207+0000","last_became_active":"2023-09-23T21:11:28.894634+0000","last_became_peered":"2023-09-23T21:11:28.894634+0000","last_unstale":"2023-09-23T21:12:19.027390+0000","last_undegraded":"2023-09-23T21:11:28.806294+0000","last_fullsized":"2023-09-23T21:11:28.754714+0000","mapping_epoch":453,"log_start":"0'0","ondisk_log_start":"0'0","created":438,"last_epoch_clean":439,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_clean_scrub_stamp":"2023-09-23T21:10:15.663179+0000","objects_scrubbed":0,"log_size":4025,"log_dups_size":0,"ondisk_log_size":4025,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":149127168,"num_objects":2276,"num_object_clones":0,"num_object_copies":6828,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2112,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2276,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2660,"num_write_kb":157920,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,4],"acting":[3,5],"avail_no_missing":["3","2","5"],"object_location_counts":[{"shards":"2,3,5","objects":1920},{"shards":"3,5","objects":164}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":64,"reported_epoch":476,"state":"active+clean","last_fresh":"2023-09-23T21:12:16.299631+0000","last_change":"2023-09-23T21:12:10.779066+0000","last_active":"2023-09-23T21:12:16.299631+0000","last_peered":"2023-09-23T21:12:16.299631+0000","last_clean":"2023-09-23T21:12:16.299631+0000","last_became_active":"2023-09-23T21:12:10.778691+0000","last_became_peered":"2023-09-23T21:12:10.778691+0000","last_unstale":"2023-09-23T21:12:16.299631+0000","last_undegraded":"2023-09-23T21:12:16.299631+0000","last_fullsized":"2023-09-23T21:12:16.299631+0000","mapping_epoch":470,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:07:22.043268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,7],"acting":[2,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":134,"reported_epoch":476,"state":"active+undersized","last_fresh":"2023-09-23T21:12:16.298797+0000","last_change":"2023-09-23T21:11:46.819868+0000","last_active":"2023-09-23T21:12:16.298797+0000","last_peered":"2023-09-23T21:12:16.298797+0000","last_clean":"2023-09-23T21:11:41.737588+0000","last_became_active":"2023-09-23T21:11:46.819868+0000","last_became_peered":"2023-09-23T21:11:46.819868+0000","last_unstale":"2023-09-23T21:12:16.298797+0000","last_undegraded":"2023-09-23T21:12:16.298797+0000","last_fullsized":"2023-09-23T21:11:46.804646+0000","mapping_epoch":465,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":434,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:30:11.969273+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3],"acting":[2,3],"avail_no_missing":["2","3"],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"10.16","version":"477'4098","reported_seq":7545,"reported_epoch":477,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2023-09-23T21:12:19.420491+0000","last_change":"2023-09-23T21:11:50.401531+0000","last_active":"2023-09-23T21:12:19.420491+0000","last_peered":"2023-09-23T21:12:19.420491+0000","last_clean":"2023-09-23T21:11:45.652255+0000","last_became_active":"2023-09-23T21:11:50.394506+0000","last_became_peered":"2023-09-23T21:11:50.394506+0000","last_unstale":"2023-09-23T21:12:19.420491+0000","last_undegraded":"2023-09-23T21:11:47.824896+0000","last_fullsized":"2023-09-23T21:12:19.420491+0000","mapping_epoch":466,"log_start":"0'0","ondisk_log_start":"0'0","created":438,"last_epoch_clean":460,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_clean_scrub_stamp":"2023-09-23T21:10:15.663179+0000","objects_scrubbed":0,"log_size":4098,"log_dups_size":0,"ondisk_log_size":4098,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":151650304,"num_objects":2314,"num_object_clones":0,"num_object_copies":6942,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":779,"num_objects_misplaced":1535,"num_objects_unfound":0,"num_objects_dirty":2314,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2692,"num_write_kb":160192,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1670,"num_bytes_recovered":109445120,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5],"acting":[4,5,1],"avail_no_missing":["4","5"],"object_location_counts":[{"shards":"1,4,5","objects":1535},{"shards":"4,5","objects":779}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":42,"reported_epoch":476,"state":"active+clean","last_fresh":"2023-09-23T21:12:16.325262+0000","last_change":"2023-09-23T21:11:27.787804+0000","last_active":"2023-09-23T21:12:16.325262+0000","last_peered":"2023-09-23T21:12:16.325262+0000","last_clean":"2023-09-23T21:12:16.325262+0000","last_became_active":"2023-09-23T21:11:27.755170+0000","last_became_peered":"2023-09-23T21:11:27.755170+0000","last_unstale":"2023-09-23T21:12:16.325262+0000","last_undegraded":"2023-09-23T21:12:16.325262+0000","last_fullsized":"2023-09-23T21:12:16.325262+0000","mapping_epoch":452,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":453,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:32:07.902721+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":2770,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,3],"acting":[4,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":52,"reported_epoch":476,"state":"active+clean","last_fresh":"2023-09-23T21:12:16.365259+0000","last_change":"2023-09-23T21:12:10.793610+0000","last_active":"2023-09-23T21:12:16.365259+0000","last_peered":"2023-09-23T21:12:16.365259+0000","last_clean":"2023-09-23T21:12:16.365259+0000","last_became_active":"2023-09-23T21:12:10.793081+0000","last_became_peered":"2023-09-23T21:12:10.793081+0000","last_unstale":"2023-09-23T21:12:16.365259+0000","last_undegraded":"2023-09-23T21:12:16.365259+0000","last_fullsized":"2023-09-23T21:12:16.365259+0000","mapping_epoch":470,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:34:36.291550+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":86,"acting":91,"num_store_stats":8},{"poolid":10,"num_pg":26,"stat_sum":{"num_bytes":4841078808,"num_objects":73872,"num_object_clones":0,"num_object_copies":221616,"num_objects_missing_on_primary":12912,"num_objects_missing":12912,"num_objects_degraded":64543,"num_objects_misplaced":19578,"num_objects_unfound":0,"num_objects_dirty":73872,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":73502,"num_write_kb":4309313,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12703,"num_bytes_recovered":832274432,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":15324815360,"data_stored":15324807216,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":109187,"ondisk_log_size":109187,"up":65,"acting":60,"num_store_stats":7},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":56,"num_bytes_recovered":8322,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":147456,"data_stored":27248,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5834,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":87,"acting":91,"num_store_stats":8},{"poolid":1,"num_pg":21,"stat_sum":{"num_bytes":670873,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":1,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":123,"num_write":33,"num_write_kb":330,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":569194,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2109440,"data_stored":2098784,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3381,"ondisk_log_size":3381,"up":56,"acting":61,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":20,"acting":23,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201663,"num_pgs":64,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2233588,"kb_used_data":1764460,"kb_used_omap":0,"kb_used_meta":469120,"kb_avail":91511564,"statfs":{"total":95995035648,"available":93707841536,"internally_reserved":0,"allocated":1806807040,"data_stored":1805453478,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":480378880},"hb_peers":[1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:11:02 2023","interfaces":[{"interface":"back","average":{"1min":0.65600000000000003,"5min":1.3340000000000001,"15min":1.6699999999999999},"min":{"1min":0.23599999999999999,"5min":0.22800000000000001,"15min":0.216},"max":{"1min":1.966,"5min":7.0170000000000003,"15min":43.994999999999997},"last":2.3599999999999999},{"interface":"front","average":{"1min":0.68400000000000005,"5min":1.4179999999999999,"15min":1.7869999999999999},"min":{"1min":0.26800000000000002,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":1.9770000000000001,"5min":8.1120000000000001,"15min":44.206000000000003},"last":2.2450000000000001}]},{"osd":1,"last update":"Sat Sep 23 21:12:07 2023","interfaces":[{"interface":"back","average":{"1min":1.8660000000000001,"5min":2.177,"15min":2.1920000000000002},"min":{"1min":0.316,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":14.034000000000001,"5min":14.034000000000001,"15min":14.034000000000001},"last":0.63300000000000001},{"interface":"front","average":{"1min":2.0129999999999999,"5min":1.883,"15min":2.0230000000000001},"min":{"1min":0.35799999999999998,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":11.226000000000001,"5min":11.226000000000001,"15min":11.226000000000001},"last":1.1060000000000001}]},{"osd":2,"last update":"Sat Sep 23 21:11:29 2023","interfaces":[{"interface":"back","average":{"1min":1.28,"5min":2.1200000000000001,"15min":1.8839999999999999},"min":{"1min":0.28499999999999998,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":5.1369999999999996,"5min":14.478999999999999,"15min":14.478999999999999},"last":0.66700000000000004},{"interface":"front","average":{"1min":1.0800000000000001,"5min":1.962,"15min":1.8700000000000001},"min":{"1min":0.40200000000000002,"5min":0.27400000000000002,"15min":0.26500000000000001},"max":{"1min":4.1189999999999998,"5min":14.135,"15min":45.25},"last":1.0780000000000001}]},{"osd":3,"last update":"Sat Sep 23 21:11:56 2023","interfaces":[{"interface":"back","average":{"1min":1.3839999999999999,"5min":2.2650000000000001,"15min":2.2120000000000002},"min":{"1min":0.39400000000000002,"5min":0.30099999999999999,"15min":0.29099999999999998},"max":{"1min":7.5190000000000001,"5min":9.1449999999999996,"15min":31.786000000000001},"last":0.69299999999999995},{"interface":"front","average":{"1min":1.1859999999999999,"5min":2.0699999999999998,"15min":2.1379999999999999},"min":{"1min":0.35799999999999998,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":6.343,"5min":12.226000000000001,"15min":32.429000000000002},"last":0.77800000000000002}]},{"osd":4,"last update":"Sat Sep 23 21:11:51 2023","interfaces":[{"interface":"back","average":{"1min":0.67000000000000004,"5min":1.1439999999999999,"15min":0.83799999999999997},"min":{"1min":0.26200000000000001,"5min":0.255,"15min":0.248},"max":{"1min":3.1970000000000001,"5min":8.5570000000000004,"15min":8.5570000000000004},"last":2.1379999999999999},{"interface":"front","average":{"1min":0.68799999999999994,"5min":1.1419999999999999,"15min":0.89400000000000002},"min":{"1min":0.27400000000000002,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":4.1879999999999997,"5min":12.752000000000001,"15min":12.752000000000001},"last":2.2690000000000001}]},{"osd":5,"last update":"Sat Sep 23 21:11:51 2023","interfaces":[{"interface":"back","average":{"1min":0.55000000000000004,"5min":1.0940000000000001,"15min":0.85699999999999998},"min":{"1min":0.26000000000000001,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":1.9450000000000001,"5min":7.6879999999999997,"15min":7.6879999999999997},"last":0.71399999999999997},{"interface":"front","average":{"1min":0.629,"5min":1.0629999999999999,"15min":0.88400000000000001},"min":{"1min":0.30499999999999999,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":2.4489999999999998,"5min":5.5090000000000003,"15min":5.8529999999999998},"last":1.044}]},{"osd":6,"last update":"Sat Sep 23 21:12:07 2023","interfaces":[{"interface":"back","average":{"1min":1.2709999999999999,"5min":1.042,"15min":0.91600000000000004},"min":{"1min":0.376,"5min":0.30099999999999999,"15min":0.28899999999999998},"max":{"1min":7.6219999999999999,"5min":7.6219999999999999,"15min":7.6219999999999999},"last":0.93000000000000005},{"interface":"front","average":{"1min":1.2230000000000001,"5min":1.319,"15min":1.0349999999999999},"min":{"1min":0.34100000000000003,"5min":0.33100000000000002,"15min":0.32300000000000001},"max":{"1min":7.7670000000000003,"5min":12.715,"15min":12.715},"last":0.98299999999999998}]}]},{"osd":6,"up_from":0,"seq":704374636746,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":1,"up_from":0,"seq":1503238553678,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":0,"up_from":83,"seq":356482285836,"num_pgs":69,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3979320,"kb_used_data":3283120,"kb_used_omap":28,"kb_used_meta":696163,"kb_avail":89765832,"statfs":{"total":95995035648,"available":91920211968,"internally_reserved":0,"allocated":3361914880,"data_stored":3359826765,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":28848,"internal_metadata":712871760},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 21:11:01 2023","interfaces":[{"interface":"back","average":{"1min":1.7190000000000001,"5min":2.5790000000000002,"15min":1.7370000000000001},"min":{"1min":0.214,"5min":0.192,"15min":0.192},"max":{"1min":15.741,"5min":15.741,"15min":15.741},"last":0.378},{"interface":"front","average":{"1min":2.9420000000000002,"5min":2.5830000000000002,"15min":1.71},"min":{"1min":0.35299999999999998,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":17.337,"5min":17.337,"15min":17.337},"last":0.29899999999999999}]},{"osd":2,"last update":"Sat Sep 23 21:11:22 2023","interfaces":[{"interface":"back","average":{"1min":1.8129999999999999,"5min":2.173,"15min":2.6699999999999999},"min":{"1min":0.27300000000000002,"5min":0.23799999999999999,"15min":0.183},"max":{"1min":8.0280000000000005,"5min":13.032,"15min":21.053000000000001},"last":0.32600000000000001},{"interface":"front","average":{"1min":2.528,"5min":2.383,"15min":2.9009999999999998},"min":{"1min":0.27700000000000002,"5min":0.27500000000000002,"15min":0.23100000000000001},"max":{"1min":12.754,"5min":14.558999999999999,"15min":21.747},"last":0.34000000000000002}]},{"osd":3,"last update":"Sat Sep 23 21:11:40 2023","interfaces":[{"interface":"back","average":{"1min":2.3690000000000002,"5min":3.1840000000000002,"15min":2.9750000000000001},"min":{"1min":0.32400000000000001,"5min":0.27300000000000002,"15min":0.252},"max":{"1min":13.403,"5min":24.048999999999999,"15min":41.079000000000001},"last":0.35799999999999998},{"interface":"front","average":{"1min":1.698,"5min":2.677,"15min":2.8180000000000001},"min":{"1min":0.34000000000000002,"5min":0.26000000000000001,"15min":0.23200000000000001},"max":{"1min":8.3780000000000001,"5min":19.91,"15min":21.167000000000002},"last":0.42699999999999999}]},{"osd":4,"last update":"Sat Sep 23 21:11:22 2023","interfaces":[{"interface":"back","average":{"1min":1.248,"5min":1.8939999999999999,"15min":2.1890000000000001},"min":{"1min":0.313,"5min":0.313,"15min":0.28599999999999998},"max":{"1min":8.8360000000000003,"5min":8.8360000000000003,"15min":21.594999999999999},"last":0.51500000000000001},{"interface":"front","average":{"1min":1.6830000000000001,"5min":2.0110000000000001,"15min":1.931},"min":{"1min":0.29999999999999999,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":6.9740000000000002,"5min":18.516999999999999,"15min":18.516999999999999},"last":0.41199999999999998}]},{"osd":5,"last update":"Sat Sep 23 21:11:22 2023","interfaces":[{"interface":"back","average":{"1min":1.081,"5min":1.8720000000000001,"15min":1.897},"min":{"1min":0.39300000000000002,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":4.3170000000000002,"5min":12.946999999999999,"15min":12.946999999999999},"last":0.46800000000000003},{"interface":"front","average":{"1min":1.2929999999999999,"5min":1.752,"15min":2.1110000000000002},"min":{"1min":0.378,"5min":0.378,"15min":0.32700000000000001},"max":{"1min":8.8900000000000006,"5min":8.8900000000000006,"15min":20.512},"last":0.39400000000000002}]},{"osd":6,"last update":"Sat Sep 23 21:11:10 2023","interfaces":[{"interface":"back","average":{"1min":1.5860000000000001,"5min":2.1579999999999999,"15min":2.0430000000000001},"min":{"1min":0.42399999999999999,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":7.0460000000000003,"5min":18.571000000000002,"15min":18.571000000000002},"last":0.44800000000000001},{"interface":"front","average":{"1min":1.647,"5min":2.3420000000000001,"15min":2.1269999999999998},"min":{"1min":0.44400000000000001,"5min":0.40300000000000002,"15min":0.377},"max":{"1min":7.8490000000000002,"5min":18.603000000000002,"15min":18.603000000000002},"last":0.53700000000000003}]},{"osd":7,"last update":"Sat Sep 23 21:11:22 2023","interfaces":[{"interface":"back","average":{"1min":1.0980000000000001,"5min":1.8460000000000001,"15min":1.9610000000000001},"min":{"1min":0.45800000000000002,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":4.2709999999999999,"5min":12.907999999999999,"15min":12.907999999999999},"last":0.59999999999999998},{"interface":"front","average":{"1min":1.774,"5min":2.3420000000000001,"15min":2.246},"min":{"1min":0.50600000000000001,"5min":0.38200000000000001,"15min":0.377},"max":{"1min":7.016,"5min":18.634,"15min":40.984999999999999},"last":0.5}]}]},{"osd":2,"up_from":93,"seq":399431958791,"num_pgs":59,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3106320,"kb_used_data":2372744,"kb_used_omap":81,"kb_used_meta":733486,"kb_avail":90638832,"statfs":{"total":95995035648,"available":92814163968,"internally_reserved":0,"allocated":2429689856,"data_stored":2427510017,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":83750,"internal_metadata":751089882},"hb_peers":[1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,0,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:11:05 2023","interfaces":[{"interface":"back","average":{"1min":0.61499999999999999,"5min":1.9730000000000001,"15min":2.1869999999999998},"min":{"1min":0.20200000000000001,"5min":0.20200000000000001,"15min":0.17999999999999999},"max":{"1min":3.3610000000000002,"5min":11.124000000000001,"15min":29.498000000000001},"last":4.2599999999999998},{"interface":"front","average":{"1min":0.49299999999999999,"5min":1.6299999999999999,"15min":2.3170000000000002},"min":{"1min":0.26300000000000001,"5min":0.193,"15min":0.193},"max":{"1min":1.377,"5min":12.183,"15min":17.251000000000001},"last":4.2759999999999998}]},{"osd":1,"last update":"Sat Sep 23 21:12:09 2023","interfaces":[{"interface":"back","average":{"1min":1.532,"5min":2.5990000000000002,"15min":2.8490000000000002},"min":{"1min":0.20000000000000001,"5min":0.20000000000000001,"15min":0.20000000000000001},"max":{"1min":6.4279999999999999,"5min":12.593,"15min":34.396999999999998},"last":0.24099999999999999},{"interface":"front","average":{"1min":0.89700000000000002,"5min":2.089,"15min":3.145},"min":{"1min":0.22800000000000001,"5min":0.217,"15min":0.217},"max":{"1min":2.4980000000000002,"5min":11.289999999999999,"15min":34.462000000000003},"last":0.26800000000000002}]},{"osd":3,"last update":"Sat Sep 23 21:11:54 2023","interfaces":[{"interface":"back","average":{"1min":2.5150000000000001,"5min":2.2519999999999998,"15min":3.0379999999999998},"min":{"1min":0.28799999999999998,"5min":0.248,"15min":0.23300000000000001},"max":{"1min":9.4399999999999995,"5min":12.951000000000001,"15min":41.057000000000002},"last":0.61699999999999999},{"interface":"front","average":{"1min":2.762,"5min":2.5070000000000001,"15min":3.3340000000000001},"min":{"1min":0.30399999999999999,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":13.159000000000001,"5min":13.159000000000001,"15min":41.012},"last":0.63600000000000001}]},{"osd":4,"last update":"Sat Sep 23 21:12:09 2023","interfaces":[{"interface":"back","average":{"1min":1.9850000000000001,"5min":1.49,"15min":1.5469999999999999},"min":{"1min":0.249,"5min":0.249,"15min":0.249},"max":{"1min":9.4749999999999996,"5min":9.4749999999999996,"15min":10.045},"last":0.76900000000000002},{"interface":"front","average":{"1min":1.7749999999999999,"5min":1.849,"15min":1.9850000000000001},"min":{"1min":0.35799999999999998,"5min":0.34499999999999997,"15min":0.26700000000000002},"max":{"1min":5.1360000000000001,"5min":12.49,"15min":14.039999999999999},"last":6.7720000000000002}]},{"osd":5,"last update":"Sat Sep 23 21:12:09 2023","interfaces":[{"interface":"back","average":{"1min":2.1469999999999998,"5min":1.6319999999999999,"15min":1.653},"min":{"1min":0.28000000000000003,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":9.5039999999999996,"5min":9.5039999999999996,"15min":10.069000000000001},"last":0.65400000000000003},{"interface":"front","average":{"1min":1.716,"5min":1.9019999999999999,"15min":1.964},"min":{"1min":0.34799999999999998,"5min":0.32700000000000001,"15min":0.29399999999999998},"max":{"1min":5.7030000000000003,"5min":12.57,"15min":13.98},"last":2.2290000000000001}]},{"osd":6,"last update":"Sat Sep 23 21:12:09 2023","interfaces":[{"interface":"back","average":{"1min":1.9590000000000001,"5min":2.0099999999999998,"15min":2.1419999999999999},"min":{"1min":0.40999999999999998,"5min":0.39800000000000002,"15min":0.312},"max":{"1min":6.3680000000000003,"5min":12.518000000000001,"15min":14.016},"last":6.6109999999999998},{"interface":"front","average":{"1min":2.048,"5min":2.1699999999999999,"15min":2.266},"min":{"1min":0.38100000000000001,"5min":0.38100000000000001,"15min":0.35599999999999998},"max":{"1min":6.4000000000000004,"5min":12.544,"15min":14.068},"last":4.4199999999999999}]},{"osd":7,"last update":"Sat Sep 23 21:12:09 2023","interfaces":[{"interface":"back","average":{"1min":1.3260000000000001,"5min":1.647,"15min":1.548},"min":{"1min":0.29799999999999999,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":7.0060000000000002,"5min":7.1050000000000004,"15min":13.427},"last":2.4630000000000001},{"interface":"front","average":{"1min":1.853,"5min":1.6910000000000001,"15min":1.7230000000000001},"min":{"1min":0.41099999999999998,"5min":0.36299999999999999,"15min":0.32200000000000001},"max":{"1min":9.5440000000000005,"5min":10.92,"15min":10.92},"last":0.40100000000000002}]}]},{"osd":3,"up_from":98,"seq":420906795264,"num_pgs":72,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2942232,"kb_used_data":2126160,"kb_used_omap":7,"kb_used_meta":816056,"kb_avail":90802920,"statfs":{"total":95995035648,"available":92982190080,"internally_reserved":0,"allocated":2177187840,"data_stored":2175048256,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7576,"internal_metadata":835641960},"hb_peers":[1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:10:53 2023","interfaces":[{"interface":"back","average":{"1min":0.59699999999999998,"5min":2.173,"15min":2.6000000000000001},"min":{"1min":0.223,"5min":0.19900000000000001,"15min":0.188},"max":{"1min":3.032,"5min":14.08,"15min":27.962},"last":0.34200000000000003},{"interface":"front","average":{"1min":0.85699999999999998,"5min":1.8580000000000001,"15min":2.6779999999999999},"min":{"1min":0.26000000000000001,"5min":0.20999999999999999,"15min":0.19800000000000001},"max":{"1min":4.0090000000000003,"5min":7.8920000000000003,"15min":28.058},"last":0.44500000000000001}]},{"osd":1,"last update":"Sat Sep 23 21:12:07 2023","interfaces":[{"interface":"back","average":{"1min":2.2210000000000001,"5min":2.9609999999999999,"15min":2.415},"min":{"1min":0.216,"5min":0.216,"15min":0.216},"max":{"1min":22.577999999999999,"5min":22.577999999999999,"15min":22.577999999999999},"last":0.26200000000000001},{"interface":"front","average":{"1min":3.2709999999999999,"5min":2.8109999999999999,"15min":2.4169999999999998},"min":{"1min":0.23699999999999999,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":21.617999999999999,"5min":21.617999999999999,"15min":21.617999999999999},"last":0.28299999999999997}]},{"osd":2,"last update":"Sat Sep 23 21:11:55 2023","interfaces":[{"interface":"back","average":{"1min":2.9729999999999999,"5min":2.4590000000000001,"15min":2.8479999999999999},"min":{"1min":0.27600000000000002,"5min":0.26700000000000002,"15min":0.23000000000000001},"max":{"1min":22.879999999999999,"5min":22.879999999999999,"15min":28.251999999999999},"last":0.35199999999999998},{"interface":"front","average":{"1min":3.0859999999999999,"5min":2.6379999999999999,"15min":2.7050000000000001},"min":{"1min":0.30299999999999999,"5min":0.28399999999999997,"15min":0.22},"max":{"1min":22.759,"5min":22.759,"15min":22.759},"last":0.32900000000000001}]},{"osd":4,"last update":"Sat Sep 23 21:11:55 2023","interfaces":[{"interface":"back","average":{"1min":2.5670000000000002,"5min":1.944,"15min":2.04},"min":{"1min":0.316,"5min":0.316,"15min":0.28399999999999997},"max":{"1min":21.478999999999999,"5min":21.478999999999999,"15min":21.478999999999999},"last":0.38400000000000001},{"interface":"front","average":{"1min":2.5510000000000002,"5min":2.222,"15min":1.9850000000000001},"min":{"1min":0.36199999999999999,"5min":0.35299999999999998,"15min":0.29499999999999998},"max":{"1min":22.402000000000001,"5min":22.402000000000001,"15min":22.402000000000001},"last":10.237}]},{"osd":5,"last update":"Sat Sep 23 21:11:55 2023","interfaces":[{"interface":"back","average":{"1min":2.5579999999999998,"5min":2.081,"15min":2.1960000000000002},"min":{"1min":0.35599999999999998,"5min":0.32300000000000001,"15min":0.314},"max":{"1min":21.562000000000001,"5min":21.562000000000001,"15min":21.562000000000001},"last":0.40400000000000003},{"interface":"front","average":{"1min":2.069,"5min":2.2480000000000002,"15min":2.3679999999999999},"min":{"1min":0.36599999999999999,"5min":0.34200000000000003,"15min":0.33300000000000002},"max":{"1min":5.7009999999999996,"5min":14.029,"15min":14.029},"last":0.45200000000000001}]},{"osd":6,"last update":"Sat Sep 23 21:12:13 2023","interfaces":[{"interface":"back","average":{"1min":2.3479999999999999,"5min":2.3319999999999999,"15min":2.077},"min":{"1min":0.41599999999999998,"5min":0.40000000000000002,"15min":0.33500000000000002},"max":{"1min":22.446999999999999,"5min":22.446999999999999,"15min":22.446999999999999},"last":10.102},{"interface":"front","average":{"1min":2.2829999999999999,"5min":2.032,"15min":2.2679999999999998},"min":{"1min":0.39000000000000001,"5min":0.39000000000000001,"15min":0.373},"max":{"1min":7.0309999999999997,"5min":12.375999999999999,"15min":13.576000000000001},"last":10.148}]},{"osd":7,"last update":"Sat Sep 23 21:11:55 2023","interfaces":[{"interface":"back","average":{"1min":2.556,"5min":2.0590000000000002,"15min":2.2309999999999999},"min":{"1min":0.42299999999999999,"5min":0.38400000000000001,"15min":0.33900000000000002},"max":{"1min":21.518999999999998,"5min":21.518999999999998,"15min":21.518999999999998},"last":10.002000000000001},{"interface":"front","average":{"1min":2.351,"5min":2.0800000000000001,"15min":2.3929999999999998},"min":{"1min":0.46200000000000002,"5min":0.46200000000000002,"15min":0.38900000000000001},"max":{"1min":6.2190000000000003,"5min":7.8390000000000004,"15min":13.603999999999999},"last":9.9879999999999995}]}]},{"osd":4,"up_from":31,"seq":133143986571,"num_pgs":71,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2154748,"kb_used_data":1765428,"kb_used_omap":0,"kb_used_meta":389312,"kb_avail":91590404,"statfs":{"total":95995035648,"available":93788573696,"internally_reserved":0,"allocated":1807798272,"data_stored":1806437349,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":398655488},"hb_peers":[1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,1,1,5,5,5,2],"upper_bound":1024},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:11:11 2023","interfaces":[{"interface":"back","average":{"1min":0.98499999999999999,"5min":1.6819999999999999,"15min":1.5549999999999999},"min":{"1min":0.20200000000000001,"5min":0.20200000000000001,"15min":0.20200000000000001},"max":{"1min":4.944,"5min":11.537000000000001,"15min":11.784000000000001},"last":3.8500000000000001},{"interface":"front","average":{"1min":1.1499999999999999,"5min":2.2549999999999999,"15min":1.887},"min":{"1min":0.28799999999999998,"5min":0.28000000000000003,"15min":0.254},"max":{"1min":4.9749999999999996,"5min":19.109999999999999,"15min":19.109999999999999},"last":3.839}]},{"osd":1,"last update":"Sat Sep 23 21:12:05 2023","interfaces":[{"interface":"back","average":{"1min":0.91000000000000003,"5min":1.704,"15min":3.1139999999999999},"min":{"1min":0.24399999999999999,"5min":0.221,"15min":0.221},"max":{"1min":3.4340000000000002,"5min":14.798999999999999,"15min":37.960000000000001},"last":0.57599999999999996},{"interface":"front","average":{"1min":1.4890000000000001,"5min":1.653,"15min":2.9649999999999999},"min":{"1min":0.34599999999999997,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":7.1349999999999998,"5min":14.403,"15min":37.933},"last":0.68300000000000005}]},{"osd":2,"last update":"Sat Sep 23 21:12:11 2023","interfaces":[{"interface":"back","average":{"1min":1.821,"5min":1.867,"15min":1.8480000000000001},"min":{"1min":0.34399999999999997,"5min":0.247,"15min":0.247},"max":{"1min":10.464,"5min":16.928000000000001,"15min":16.928000000000001},"last":0.70899999999999996},{"interface":"front","average":{"1min":2.0550000000000002,"5min":2.1499999999999999,"15min":1.8720000000000001},"min":{"1min":0.34100000000000003,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":7.3659999999999997,"5min":14.417999999999999,"15min":14.417999999999999},"last":0.63400000000000001}]},{"osd":3,"last update":"Sat Sep 23 21:11:42 2023","interfaces":[{"interface":"back","average":{"1min":1.3540000000000001,"5min":2.214,"15min":2.1970000000000001},"min":{"1min":0.38100000000000001,"5min":0.29799999999999999,"15min":0.28899999999999998},"max":{"1min":7.5979999999999999,"5min":15.01,"15min":15.01},"last":0.61499999999999999},{"interface":"front","average":{"1min":1.413,"5min":2.274,"15min":2.1000000000000001},"min":{"1min":0.40000000000000002,"5min":0.32400000000000001,"15min":0.30199999999999999},"max":{"1min":7.5549999999999997,"5min":15.004,"15min":15.004},"last":0.65800000000000003}]},{"osd":5,"last update":"Sat Sep 23 21:12:01 2023","interfaces":[{"interface":"back","average":{"1min":0.872,"5min":1.0669999999999999,"15min":0.86099999999999999},"min":{"1min":0.28999999999999998,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":2.9500000000000002,"5min":14.298,"15min":14.298},"last":0.378},{"interface":"front","average":{"1min":0.84399999999999997,"5min":0.93300000000000005,"15min":0.97599999999999998},"min":{"1min":0.29999999999999999,"5min":0.255,"15min":0.255},"max":{"1min":2.718,"5min":9.8979999999999997,"15min":9.8979999999999997},"last":0.48999999999999999}]},{"osd":6,"last update":"Sat Sep 23 21:12:05 2023","interfaces":[{"interface":"back","average":{"1min":0.755,"5min":1.147,"15min":1.107},"min":{"1min":0.31,"5min":0.28399999999999997,"15min":0.26800000000000002},"max":{"1min":2.6389999999999998,"5min":14.704000000000001,"15min":14.704000000000001},"last":1.3400000000000001},{"interface":"front","average":{"1min":1.4590000000000001,"5min":1.0640000000000001,"15min":0.88800000000000001},"min":{"1min":0.41599999999999998,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":7.3920000000000003,"5min":14.387,"15min":14.387},"last":1.375}]},{"osd":7,"last update":"Sat Sep 23 21:11:42 2023","interfaces":[{"interface":"back","average":{"1min":0.82399999999999995,"5min":1.1399999999999999,"15min":0.93100000000000005},"min":{"1min":0.30099999999999999,"5min":0.30099999999999999,"15min":0.29299999999999998},"max":{"1min":2.661,"5min":16.890999999999998,"15min":16.890999999999998},"last":0.60099999999999998},{"interface":"front","average":{"1min":0.90600000000000003,"5min":1.006,"15min":1.042},"min":{"1min":0.27400000000000002,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":2.8450000000000002,"5min":8.5830000000000002,"15min":11.170999999999999},"last":1.4119999999999999}]}]},{"osd":5,"up_from":39,"seq":167503724937,"num_pgs":65,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2444396,"kb_used_data":1967908,"kb_used_omap":0,"kb_used_meta":476416,"kb_avail":91300756,"statfs":{"total":95995035648,"available":93491974144,"internally_reserved":0,"allocated":2015137792,"data_stored":2013788985,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":487849984},"hb_peers":[1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1],"upper_bound":4},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:11:02 2023","interfaces":[{"interface":"back","average":{"1min":0.627,"5min":1.722,"15min":1.8},"min":{"1min":0.222,"5min":0.19700000000000001,"15min":0.19700000000000001},"max":{"1min":2.0840000000000001,"5min":25.222999999999999,"15min":25.222999999999999},"last":2.9079999999999999},{"interface":"front","average":{"1min":0.67200000000000004,"5min":1.6659999999999999,"15min":1.861},"min":{"1min":0.247,"5min":0.247,"15min":0.23400000000000001},"max":{"1min":2.1970000000000001,"5min":25.457999999999998,"15min":25.457999999999998},"last":3.4449999999999998}]},{"osd":1,"last update":"Sat Sep 23 21:12:02 2023","interfaces":[{"interface":"back","average":{"1min":1.419,"5min":2.0710000000000002,"15min":1.883},"min":{"1min":0.26600000000000001,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":6.0789999999999997,"5min":25.239999999999998,"15min":25.239999999999998},"last":0.27900000000000003},{"interface":"front","average":{"1min":2.1909999999999998,"5min":2.254,"15min":1.786},"min":{"1min":0.374,"5min":0.27400000000000002,"15min":0.252},"max":{"1min":10.686,"5min":25.253,"15min":25.253},"last":2.3530000000000002}]},{"osd":2,"last update":"Sat Sep 23 21:11:19 2023","interfaces":[{"interface":"back","average":{"1min":0.70099999999999996,"5min":1.665,"15min":1.677},"min":{"1min":0.28399999999999997,"5min":0.25800000000000001,"15min":0.23499999999999999},"max":{"1min":2.1960000000000002,"5min":11.939,"15min":12.634},"last":0.65600000000000003},{"interface":"front","average":{"1min":1.1399999999999999,"5min":1.831,"15min":1.909},"min":{"1min":0.25700000000000001,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":2.7879999999999998,"5min":8.8079999999999998,"15min":12.300000000000001},"last":0.318}]},{"osd":3,"last update":"Sat Sep 23 21:11:42 2023","interfaces":[{"interface":"back","average":{"1min":2.54,"5min":2.2959999999999998,"15min":2.6429999999999998},"min":{"1min":0.311,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":14.242000000000001,"5min":25.503,"15min":61.667999999999999},"last":2.3039999999999998},{"interface":"front","average":{"1min":2.4500000000000002,"5min":2.6869999999999998,"15min":2.589},"min":{"1min":0.46700000000000003,"5min":0.30599999999999999,"15min":0.28899999999999998},"max":{"1min":12.44,"5min":25.466000000000001,"15min":62.276000000000003},"last":0.86099999999999999}]},{"osd":4,"last update":"Sat Sep 23 21:11:54 2023","interfaces":[{"interface":"back","average":{"1min":1.01,"5min":0.94799999999999995,"15min":0.86099999999999999},"min":{"1min":0.27200000000000002,"5min":0.27000000000000002,"15min":0.23400000000000001},"max":{"1min":10.75,"5min":10.75,"15min":10.75},"last":3.4750000000000001},{"interface":"front","average":{"1min":0.69499999999999995,"5min":1.1870000000000001,"15min":0.91500000000000004},"min":{"1min":0.28899999999999998,"5min":0.27000000000000002,"15min":0.23599999999999999},"max":{"1min":3.3959999999999999,"5min":11.973000000000001,"15min":11.973000000000001},"last":0.83299999999999996}]},{"osd":6,"last update":"Sat Sep 23 21:12:02 2023","interfaces":[{"interface":"back","average":{"1min":1.1739999999999999,"5min":0.98099999999999998,"15min":0.97699999999999998},"min":{"1min":0.33800000000000002,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":5.6749999999999998,"5min":8.0909999999999993,"15min":11.128},"last":0.60499999999999998},{"interface":"front","average":{"1min":1.704,"5min":1.1659999999999999,"15min":1.034},"min":{"1min":0.38800000000000001,"5min":0.317,"15min":0.317},"max":{"1min":10.718999999999999,"5min":10.718999999999999,"15min":11.112},"last":2.4340000000000002}]},{"osd":7,"last update":"Sat Sep 23 21:11:42 2023","interfaces":[{"interface":"back","average":{"1min":0.57799999999999996,"5min":3.2989999999999999,"15min":1.7},"min":{"1min":0.31,"5min":0.31,"15min":0.307},"max":{"1min":1.853,"5min":189.405,"15min":189.405},"last":1.4570000000000001},{"interface":"front","average":{"1min":0.64800000000000002,"5min":3.2970000000000002,"15min":1.653},"min":{"1min":0.35399999999999998,"5min":0.35399999999999998,"15min":0.32500000000000001},"max":{"1min":2.6909999999999998,"5min":189.685,"15min":189.685},"last":2.4009999999999998}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5344,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1259,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5322,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":714,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":5434,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":646,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":4838,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":3377659904,"data_stored":3377659904,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1751384064,"data_stored":1751384064,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":2421293056,"data_stored":2421293056,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":2168262656,"data_stored":2168258584,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1802899456,"data_stored":1802895384,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":2005860352,"data_stored":2005860352,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1797455872,"data_stored":1797455872,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T21:12:24.059 INFO:tasks.cephadm.ceph_manager.ceph:increase pool size by 10 2023-09-23T21:12:24.059 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph osd pool set .mgr pg_num 31 2023-09-23T21:12:24.126 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:12:24.175 INFO:teuthology.orchestra.run.smithi070.stdout:ERROR: (22) Invalid argument 2023-09-23T21:12:24.175 INFO:teuthology.orchestra.run.smithi070.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T21:12:24.182 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:12:24.261 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:12:24.274 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:12:24.274 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:12:24.375 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:12:24.407 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:12:24.447 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:12:24.450 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:12:24.450 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:12:24.551 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:12:24.606 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:12:24 smithi070 ceph-mon[168282]: pgmap v1214: 119 pgs: 16 active+clean+remapped, 4 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 12 active+undersized, 1 active+recovering+undersized+degraded+remapped, 6 active+undersized+degraded, 10 active+recovery_wait+undersized+degraded+remapped, 6 active+recovery_wait+degraded+remapped, 62 active+clean; 4.5 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 195 op/s; 64548/221688 objects degraded (29.117%); 19579/221688 objects misplaced (8.832%); 9.5 MiB/s, 152 objects/s recovering 2023-09-23T21:12:24.606 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:12:24 smithi070 ceph-mon[168282]: from='client.46487 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T21:12:24.617 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:12:24.631 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:12:24 smithi026 ceph-mon[175151]: pgmap v1214: 119 pgs: 16 active+clean+remapped, 4 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 12 active+undersized, 1 active+recovering+undersized+degraded+remapped, 6 active+undersized+degraded, 10 active+recovery_wait+undersized+degraded+remapped, 6 active+recovery_wait+degraded+remapped, 62 active+clean; 4.5 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 195 op/s; 64548/221688 objects degraded (29.117%); 19579/221688 objects misplaced (8.832%); 9.5 MiB/s, 152 objects/s recovering 2023-09-23T21:12:24.631 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:12:24 smithi026 ceph-mon[175151]: from='client.46487 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T21:12:24.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:12:24 smithi026 ceph-mon[179688]: pgmap v1214: 119 pgs: 16 active+clean+remapped, 4 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 12 active+undersized, 1 active+recovering+undersized+degraded+remapped, 6 active+undersized+degraded, 10 active+recovery_wait+undersized+degraded+remapped, 6 active+recovery_wait+degraded+remapped, 62 active+clean; 4.5 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 195 op/s; 64548/221688 objects degraded (29.117%); 19579/221688 objects misplaced (8.832%); 9.5 MiB/s, 152 objects/s recovering 2023-09-23T21:12:24.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:12:24 smithi026 ceph-mon[179688]: from='client.46487 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T21:12:24.676 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:12:24.687 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:12:24.687 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:12:24.704 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:12:24.709 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_ops_in_flight 2023-09-23T21:12:24.788 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:12:24.841 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:12:24.869 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T21:12:24.894 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:12:24.905 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:12:24.906 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:12:25.007 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:12:25.078 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:12:25.150 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:12:25.158 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:12:25.159 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:12:25.260 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:12:25.310 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:12:25.376 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:12:25.382 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:12:25.382 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:12:25.483 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:12:25.524 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:12:25.561 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:12:25.567 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:12:25.568 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:12:25.573 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T21:12:25.669 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:12:25.688 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T21:12:25.703 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:12:25.756 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:12:25.759 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:12:25.761 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:12:25.761 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:12:25.862 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:12:25.944 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:12:26.000 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:12:26.005 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:12:26.005 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:12:26.106 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:12:26.147 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:12:26.357 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:12:26 smithi026 ceph-mon[179688]: pgmap v1215: 119 pgs: 16 active+clean+remapped, 4 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+remapped, 12 active+undersized, 1 active+recovering+undersized+degraded+remapped, 6 active+undersized+degraded, 10 active+recovery_wait+undersized+degraded+remapped, 6 active+recovery_wait+degraded+remapped, 62 active+clean; 4.5 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 235 op/s; 64011/219513 objects degraded (29.160%); 19239/219513 objects misplaced (8.764%); 6.7 MiB/s, 106 objects/s recovering 2023-09-23T21:12:26.358 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:12:26 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T21:12:26.358 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:12:26 smithi026 ceph-mon[179688]: Health check update: Degraded data redundancy: 64011/219513 objects degraded (29.160%), 28 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T21:12:26.358 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:12:26 smithi026 ceph-mon[175151]: pgmap v1215: 119 pgs: 16 active+clean+remapped, 4 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+remapped, 12 active+undersized, 1 active+recovering+undersized+degraded+remapped, 6 active+undersized+degraded, 10 active+recovery_wait+undersized+degraded+remapped, 6 active+recovery_wait+degraded+remapped, 62 active+clean; 4.5 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 235 op/s; 64011/219513 objects degraded (29.160%); 19239/219513 objects misplaced (8.764%); 6.7 MiB/s, 106 objects/s recovering 2023-09-23T21:12:26.358 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:12:26 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T21:12:26.358 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:12:26 smithi026 ceph-mon[175151]: Health check update: Degraded data redundancy: 64011/219513 objects degraded (29.160%), 28 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T21:12:26.412 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:12:26 smithi070 ceph-mon[168282]: pgmap v1215: 119 pgs: 16 active+clean+remapped, 4 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+remapped, 12 active+undersized, 1 active+recovering+undersized+degraded+remapped, 6 active+undersized+degraded, 10 active+recovery_wait+undersized+degraded+remapped, 6 active+recovery_wait+degraded+remapped, 62 active+clean; 4.5 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 235 op/s; 64011/219513 objects degraded (29.160%); 19239/219513 objects misplaced (8.764%); 6.7 MiB/s, 106 objects/s recovering 2023-09-23T21:12:26.413 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:12:26 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T21:12:26.413 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:12:26 smithi070 ceph-mon[168282]: Health check update: Degraded data redundancy: 64011/219513 objects degraded (29.160%), 28 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T21:12:26.463 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:12:26.475 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:12:26.476 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:12:26.576 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:12:26.640 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:12:26.990 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:12:26.998 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:12:26.998 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:12:27.099 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:12:27.134 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:12:27.186 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:12:27.190 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:12:27.190 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:12:27.218 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:12:27.293 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:12:27.371 INFO:teuthology.orchestra.run.smithi026.stderr:set pool 1 pg_num to 31 2023-09-23T21:12:27.381 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:12:27 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/193389919' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "31"}]: dispatch 2023-09-23T21:12:27.381 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:12:27 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/193389919' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "31"}]: dispatch 2023-09-23T21:12:27.397 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:12:27.474 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:12:27.483 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:12:27.483 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:12:27.583 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:12:27.621 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:12:27 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/193389919' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "31"}]: dispatch 2023-09-23T21:12:27.669 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:12:27.756 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:12:27.765 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:12:27.765 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:12:27.778 INFO:teuthology.orchestra.run.smithi070.stdout:ERROR: (22) Invalid argument 2023-09-23T21:12:27.778 INFO:teuthology.orchestra.run.smithi070.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T21:12:27.866 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:12:27.952 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:12:28.362 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:12:28 smithi026 ceph-mon[179688]: pgmap v1216: 119 pgs: 16 active+clean+remapped, 4 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 12 active+undersized, 1 active+recovering+undersized+degraded+remapped, 6 active+undersized+degraded, 10 active+recovery_wait+undersized+degraded+remapped, 6 active+recovery_wait+degraded+remapped, 62 active+clean; 4.4 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 245 op/s; 63854/218283 objects degraded (29.253%); 18907/218283 objects misplaced (8.662%); 7.9 MiB/s, 126 objects/s recovering 2023-09-23T21:12:28.362 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:12:28 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/193389919' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "31"}]': finished 2023-09-23T21:12:28.362 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:12:28 smithi026 ceph-mon[179688]: osdmap e478: 8 total, 7 up, 6 in 2023-09-23T21:12:28.362 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:12:28 smithi026 ceph-mon[179688]: failed to encode map e478 with expected crc 2023-09-23T21:12:28.362 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:12:28 smithi070 ceph-mon[168282]: pgmap v1216: 119 pgs: 16 active+clean+remapped, 4 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 12 active+undersized, 1 active+recovering+undersized+degraded+remapped, 6 active+undersized+degraded, 10 active+recovery_wait+undersized+degraded+remapped, 6 active+recovery_wait+degraded+remapped, 62 active+clean; 4.4 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 245 op/s; 63854/218283 objects degraded (29.253%); 18907/218283 objects misplaced (8.662%); 7.9 MiB/s, 126 objects/s recovering 2023-09-23T21:12:28.363 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:12:28 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/193389919' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "31"}]': finished 2023-09-23T21:12:28.363 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:12:28 smithi070 ceph-mon[168282]: osdmap e478: 8 total, 7 up, 6 in 2023-09-23T21:12:28.363 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:12:28 smithi070 ceph-mon[168282]: failed to encode map e478 with expected crc 2023-09-23T21:12:28.363 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:12:28 smithi070 ceph-mon[168282]: failed to encode map e478 with expected crc 2023-09-23T21:12:28.363 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:12:28 smithi070 ceph-mon[168282]: failed to encode map e478 with expected crc 2023-09-23T21:12:28.363 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:12:28 smithi070 ceph-mon[168282]: failed to encode map e478 with expected crc 2023-09-23T21:12:28.578 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_blocked_ops 2023-09-23T21:12:28.604 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:12:28.616 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:12:28.616 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:12:28.631 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:12:28 smithi026 ceph-mon[179688]: failed to encode map e478 with expected crc 2023-09-23T21:12:28.631 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:12:28 smithi026 ceph-mon[179688]: failed to encode map e478 with expected crc 2023-09-23T21:12:28.631 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:12:28 smithi026 ceph-mon[179688]: failed to encode map e478 with expected crc 2023-09-23T21:12:28.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:12:28 smithi026 ceph-mon[175151]: pgmap v1216: 119 pgs: 16 active+clean+remapped, 4 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 12 active+undersized, 1 active+recovering+undersized+degraded+remapped, 6 active+undersized+degraded, 10 active+recovery_wait+undersized+degraded+remapped, 6 active+recovery_wait+degraded+remapped, 62 active+clean; 4.4 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 245 op/s; 63854/218283 objects degraded (29.253%); 18907/218283 objects misplaced (8.662%); 7.9 MiB/s, 126 objects/s recovering 2023-09-23T21:12:28.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:12:28 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/193389919' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "31"}]': finished 2023-09-23T21:12:28.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:12:28 smithi026 ceph-mon[175151]: osdmap e478: 8 total, 7 up, 6 in 2023-09-23T21:12:28.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:12:28 smithi026 ceph-mon[175151]: failed to encode map e478 with expected crc 2023-09-23T21:12:28.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:12:28 smithi026 ceph-mon[175151]: failed to encode map e478 with expected crc 2023-09-23T21:12:28.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:12:28 smithi026 ceph-mon[175151]: failed to encode map e478 with expected crc 2023-09-23T21:12:28.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:12:28 smithi026 ceph-mon[175151]: failed to encode map e478 with expected crc 2023-09-23T21:12:28.718 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:12:28.753 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:12:28.947 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:12:28.950 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:12:28.951 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:12:29.051 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:12:29.094 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:12:29.158 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1: {} 2023-09-23T21:12:29.178 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2: {} 2023-09-23T21:12:29.201 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3: {} 2023-09-23T21:12:29.224 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4: {} 2023-09-23T21:12:29.255 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T21:12:29.279 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:12:29.283 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:12:29.284 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:12:29.290 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6: {} 2023-09-23T21:12:29.312 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T21:12:29.313 INFO:teuthology.orchestra.run.smithi026.stderr:Error ENXIO: problem getting command descriptions from osd.0 2023-09-23T21:12:29.314 INFO:teuthology.orchestra.run.smithi026.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T21:12:29.314 INFO:teuthology.orchestra.run.smithi026.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T21:12:29.314 INFO:teuthology.orchestra.run.smithi026.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T21:12:29.314 INFO:teuthology.orchestra.run.smithi026.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T21:12:29.314 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T21:12:29.314 INFO:teuthology.orchestra.run.smithi026.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T21:12:29.314 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T21:12:29.377 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:12:29 smithi070 ceph-mon[168282]: failed to encode map e478 with expected crc 2023-09-23T21:12:29.377 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:12:29 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/2591837010' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:12:29.378 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:12:29 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "31"}]: dispatch 2023-09-23T21:12:29.378 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:12:29 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/361290918' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:12:29.385 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:12:29.393 INFO:teuthology.orchestra.run.smithi026.stderr:nodeep-scrub is unset 2023-09-23T21:12:29.476 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:12:29.540 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T21:12:29.611 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:12:29 smithi026 ceph-mon[175151]: failed to encode map e478 with expected crc 2023-09-23T21:12:29.611 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:12:29 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/2591837010' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:12:29.611 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:12:29 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "31"}]: dispatch 2023-09-23T21:12:29.611 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:12:29 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/361290918' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:12:29.612 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:12:29 smithi026 ceph-mon[179688]: failed to encode map e478 with expected crc 2023-09-23T21:12:29.612 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:12:29 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/2591837010' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:12:29.612 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:12:29 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "31"}]: dispatch 2023-09-23T21:12:29.612 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:12:29 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/361290918' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:12:29.677 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:12:29.682 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:12:29.682 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:12:29.783 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:12:29.819 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:12:29.881 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 21:12:29 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[164029]: ::ffff:172.21.15.70 - - [23/Sep/2023:21:12:29] "GET /metrics HTTP/1.1" 200 36448 "" "Prometheus/2.43.0" 2023-09-23T21:12:30.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:12:30 smithi070 ceph-mon[168282]: pgmap v1218: 119 pgs: 16 active+clean+remapped, 4 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 12 active+undersized, 1 active+recovering+undersized+degraded+remapped, 6 active+undersized+degraded, 10 active+recovery_wait+undersized+degraded+remapped, 6 active+recovery_wait+degraded+remapped, 62 active+clean; 4.4 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 236 op/s; 63607/217647 objects degraded (29.225%); 18828/217647 objects misplaced (8.651%); 7.9 MiB/s, 126 objects/s recovering 2023-09-23T21:12:30.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:12:30 smithi070 ceph-mon[168282]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T21:12:30.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:12:30 smithi070 ceph-mon[168282]: failed to encode map e479 with expected crc 2023-09-23T21:12:30.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:12:30 smithi070 ceph-mon[168282]: failed to encode map e479 with expected crc 2023-09-23T21:12:30.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:12:30 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/2591837010' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T21:12:30.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:12:30 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "31"}]': finished 2023-09-23T21:12:30.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:12:30 smithi070 ceph-mon[168282]: osdmap e479: 8 total, 7 up, 6 in 2023-09-23T21:12:30.621 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:12:30 smithi070 ceph-mon[168282]: failed to encode map e479 with expected crc 2023-09-23T21:12:30.621 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:12:30 smithi070 ceph-mon[168282]: failed to encode map e479 with expected crc 2023-09-23T21:12:30.881 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:12:30 smithi026 ceph-mon[175151]: pgmap v1218: 119 pgs: 16 active+clean+remapped, 4 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 12 active+undersized, 1 active+recovering+undersized+degraded+remapped, 6 active+undersized+degraded, 10 active+recovery_wait+undersized+degraded+remapped, 6 active+recovery_wait+degraded+remapped, 62 active+clean; 4.4 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 236 op/s; 63607/217647 objects degraded (29.225%); 18828/217647 objects misplaced (8.651%); 7.9 MiB/s, 126 objects/s recovering 2023-09-23T21:12:30.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:12:30 smithi026 ceph-mon[175151]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T21:12:30.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:12:30 smithi026 ceph-mon[175151]: failed to encode map e479 with expected crc 2023-09-23T21:12:30.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:12:30 smithi026 ceph-mon[175151]: failed to encode map e479 with expected crc 2023-09-23T21:12:30.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:12:30 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/2591837010' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T21:12:30.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:12:30 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "31"}]': finished 2023-09-23T21:12:30.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:12:30 smithi026 ceph-mon[175151]: osdmap e479: 8 total, 7 up, 6 in 2023-09-23T21:12:30.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:12:30 smithi026 ceph-mon[175151]: failed to encode map e479 with expected crc 2023-09-23T21:12:30.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:12:30 smithi026 ceph-mon[175151]: failed to encode map e479 with expected crc 2023-09-23T21:12:30.883 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:12:30 smithi026 ceph-mon[179688]: pgmap v1218: 119 pgs: 16 active+clean+remapped, 4 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 12 active+undersized, 1 active+recovering+undersized+degraded+remapped, 6 active+undersized+degraded, 10 active+recovery_wait+undersized+degraded+remapped, 6 active+recovery_wait+degraded+remapped, 62 active+clean; 4.4 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 236 op/s; 63607/217647 objects degraded (29.225%); 18828/217647 objects misplaced (8.651%); 7.9 MiB/s, 126 objects/s recovering 2023-09-23T21:12:30.883 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:12:30 smithi026 ceph-mon[179688]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T21:12:30.883 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:12:30 smithi026 ceph-mon[179688]: failed to encode map e479 with expected crc 2023-09-23T21:12:30.883 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:12:30 smithi026 ceph-mon[179688]: failed to encode map e479 with expected crc 2023-09-23T21:12:30.883 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:12:30 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/2591837010' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T21:12:30.883 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:12:30 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "31"}]': finished 2023-09-23T21:12:30.883 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:12:30 smithi026 ceph-mon[179688]: osdmap e479: 8 total, 7 up, 6 in 2023-09-23T21:12:30.883 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:12:30 smithi026 ceph-mon[179688]: failed to encode map e479 with expected crc 2023-09-23T21:12:30.883 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:12:30 smithi026 ceph-mon[179688]: failed to encode map e479 with expected crc 2023-09-23T21:12:31.501 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T21:12:31.501 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T21:12:31.554 INFO:teuthology.orchestra.run.smithi070.stdout:{ 2023-09-23T21:12:31.554 INFO:teuthology.orchestra.run.smithi070.stdout: "ops": [], 2023-09-23T21:12:31.555 INFO:teuthology.orchestra.run.smithi070.stdout: "complaint_time": 30, 2023-09-23T21:12:31.555 INFO:teuthology.orchestra.run.smithi070.stdout: "num_blocked_ops": 0 2023-09-23T21:12:31.555 INFO:teuthology.orchestra.run.smithi070.stdout:} 2023-09-23T21:12:31.581 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:12:31 smithi070 ceph-mon[168282]: osdmap e480: 8 total, 7 up, 6 in 2023-09-23T21:12:31.581 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:12:31 smithi070 ceph-mon[168282]: failed to encode map e480 with expected crc 2023-09-23T21:12:31.581 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:12:31 smithi070 ceph-mon[168282]: failed to encode map e480 with expected crc 2023-09-23T21:12:31.581 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:12:31 smithi070 ceph-mon[168282]: failed to encode map e480 with expected crc 2023-09-23T21:12:31.581 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:12:31 smithi070 ceph-mon[168282]: failed to encode map e480 with expected crc 2023-09-23T21:12:31.581 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:12:31 smithi070 ceph-mon[168282]: failed to encode map e480 with expected crc 2023-09-23T21:12:31.581 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:12:31 smithi070 ceph-mon[168282]: failed to encode map e480 with expected crc 2023-09-23T21:12:31.581 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:12:31 smithi070 ceph-mon[168282]: failed to encode map e480 with expected crc 2023-09-23T21:12:31.581 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:12:31 smithi070 ceph-mon[168282]: failed to encode map e480 with expected crc 2023-09-23T21:12:31.581 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:12:31 smithi070 ceph-mon[168282]: failed to encode map e480 with expected crc 2023-09-23T21:12:31.582 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:12:31 smithi070 ceph-mon[168282]: failed to encode map e480 with expected crc 2023-09-23T21:12:31.582 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:12:31 smithi070 ceph-mon[168282]: failed to encode map e480 with expected crc 2023-09-23T21:12:31.582 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:12:31 smithi070 ceph-mon[168282]: Health check update: Degraded data redundancy: 63607/217647 objects degraded (29.225%), 28 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T21:12:31.582 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:12:31 smithi070 ceph-mon[168282]: failed to encode map e481 with expected crc 2023-09-23T21:12:31.582 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:12:31 smithi070 ceph-mon[168282]: osdmap e481: 8 total, 7 up, 6 in 2023-09-23T21:12:31.582 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:12:31 smithi070 ceph-mon[168282]: failed to encode map e481 with expected crc 2023-09-23T21:12:31.582 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:12:31 smithi070 ceph-mon[168282]: failed to encode map e481 with expected crc 2023-09-23T21:12:31.582 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:12:31 smithi070 ceph-mon[168282]: failed to encode map e481 with expected crc 2023-09-23T21:12:31.582 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:12:31 smithi070 ceph-mon[168282]: failed to encode map e481 with expected crc 2023-09-23T21:12:31.582 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:12:31 smithi070 ceph-mon[168282]: failed to encode map e481 with expected crc 2023-09-23T21:12:31.582 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:12:31 smithi070 ceph-mon[168282]: failed to encode map e481 with expected crc 2023-09-23T21:12:31.582 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:12:31 smithi070 ceph-mon[168282]: failed to encode map e481 with expected crc 2023-09-23T21:12:31.583 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:12:31 smithi070 ceph-mon[168282]: failed to encode map e481 with expected crc 2023-09-23T21:12:31.583 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:12:31 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:12:31.583 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:12:31 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:12:31.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:12:31 smithi026 ceph-mon[179688]: osdmap e480: 8 total, 7 up, 6 in 2023-09-23T21:12:31.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:12:31 smithi026 ceph-mon[179688]: failed to encode map e480 with expected crc 2023-09-23T21:12:31.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:12:31 smithi026 ceph-mon[179688]: failed to encode map e480 with expected crc 2023-09-23T21:12:31.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:12:31 smithi026 ceph-mon[179688]: failed to encode map e480 with expected crc 2023-09-23T21:12:31.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:12:31 smithi026 ceph-mon[179688]: failed to encode map e480 with expected crc 2023-09-23T21:12:31.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:12:31 smithi026 ceph-mon[179688]: failed to encode map e480 with expected crc 2023-09-23T21:12:31.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:12:31 smithi026 ceph-mon[179688]: failed to encode map e480 with expected crc 2023-09-23T21:12:31.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:12:31 smithi026 ceph-mon[179688]: failed to encode map e480 with expected crc 2023-09-23T21:12:31.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:12:31 smithi026 ceph-mon[179688]: failed to encode map e480 with expected crc 2023-09-23T21:12:31.883 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:12:31 smithi026 ceph-mon[179688]: failed to encode map e480 with expected crc 2023-09-23T21:12:31.883 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:12:31 smithi026 ceph-mon[179688]: failed to encode map e480 with expected crc 2023-09-23T21:12:31.883 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:12:31 smithi026 ceph-mon[179688]: failed to encode map e480 with expected crc 2023-09-23T21:12:31.883 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:12:31 smithi026 ceph-mon[179688]: Health check update: Degraded data redundancy: 63607/217647 objects degraded (29.225%), 28 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T21:12:31.883 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:12:31 smithi026 ceph-mon[179688]: failed to encode map e481 with expected crc 2023-09-23T21:12:31.883 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:12:31 smithi026 ceph-mon[179688]: osdmap e481: 8 total, 7 up, 6 in 2023-09-23T21:12:31.883 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:12:31 smithi026 ceph-mon[179688]: failed to encode map e481 with expected crc 2023-09-23T21:12:31.883 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:12:31 smithi026 ceph-mon[179688]: failed to encode map e481 with expected crc 2023-09-23T21:12:31.884 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:12:31 smithi026 ceph-mon[179688]: failed to encode map e481 with expected crc 2023-09-23T21:12:31.885 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:12:31 smithi026 ceph-mon[179688]: failed to encode map e481 with expected crc 2023-09-23T21:12:31.885 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:12:31 smithi026 ceph-mon[179688]: failed to encode map e481 with expected crc 2023-09-23T21:12:31.885 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:12:31 smithi026 ceph-mon[179688]: failed to encode map e481 with expected crc 2023-09-23T21:12:31.885 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:12:31 smithi026 ceph-mon[179688]: failed to encode map e481 with expected crc 2023-09-23T21:12:31.885 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:12:31 smithi026 ceph-mon[179688]: failed to encode map e481 with expected crc 2023-09-23T21:12:31.886 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:12:31 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:12:31.886 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:12:31 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:12:31.886 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:12:31 smithi026 ceph-mon[175151]: osdmap e480: 8 total, 7 up, 6 in 2023-09-23T21:12:31.886 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:12:31 smithi026 ceph-mon[175151]: failed to encode map e480 with expected crc 2023-09-23T21:12:31.887 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:12:31 smithi026 ceph-mon[175151]: failed to encode map e480 with expected crc 2023-09-23T21:12:31.887 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:12:31 smithi026 ceph-mon[175151]: failed to encode map e480 with expected crc 2023-09-23T21:12:31.887 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:12:31 smithi026 ceph-mon[175151]: failed to encode map e480 with expected crc 2023-09-23T21:12:31.887 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:12:31 smithi026 ceph-mon[175151]: failed to encode map e480 with expected crc 2023-09-23T21:12:31.887 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:12:31 smithi026 ceph-mon[175151]: failed to encode map e480 with expected crc 2023-09-23T21:12:31.887 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:12:31 smithi026 ceph-mon[175151]: failed to encode map e480 with expected crc 2023-09-23T21:12:31.888 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:12:31 smithi026 ceph-mon[175151]: failed to encode map e480 with expected crc 2023-09-23T21:12:31.888 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:12:31 smithi026 ceph-mon[175151]: failed to encode map e480 with expected crc 2023-09-23T21:12:31.888 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:12:31 smithi026 ceph-mon[175151]: failed to encode map e480 with expected crc 2023-09-23T21:12:31.888 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:12:31 smithi026 ceph-mon[175151]: failed to encode map e480 with expected crc 2023-09-23T21:12:31.888 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:12:31 smithi026 ceph-mon[175151]: Health check update: Degraded data redundancy: 63607/217647 objects degraded (29.225%), 28 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T21:12:31.888 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:12:31 smithi026 ceph-mon[175151]: failed to encode map e481 with expected crc 2023-09-23T21:12:31.888 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:12:31 smithi026 ceph-mon[175151]: osdmap e481: 8 total, 7 up, 6 in 2023-09-23T21:12:31.888 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:12:31 smithi026 ceph-mon[175151]: failed to encode map e481 with expected crc 2023-09-23T21:12:31.889 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:12:31 smithi026 ceph-mon[175151]: failed to encode map e481 with expected crc 2023-09-23T21:12:31.889 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:12:31 smithi026 ceph-mon[175151]: failed to encode map e481 with expected crc 2023-09-23T21:12:31.889 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:12:31 smithi026 ceph-mon[175151]: failed to encode map e481 with expected crc 2023-09-23T21:12:31.889 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:12:31 smithi026 ceph-mon[175151]: failed to encode map e481 with expected crc 2023-09-23T21:12:31.889 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:12:31 smithi026 ceph-mon[175151]: failed to encode map e481 with expected crc 2023-09-23T21:12:31.889 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:12:31 smithi026 ceph-mon[175151]: failed to encode map e481 with expected crc 2023-09-23T21:12:31.889 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:12:31 smithi026 ceph-mon[175151]: failed to encode map e481 with expected crc 2023-09-23T21:12:31.889 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:12:31 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:12:31.890 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:12:31 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:12:32.023 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_historic_ops 2023-09-23T21:12:32.468 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:12:32.471 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:12:32.472 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:12:32.573 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:12:32.607 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:12:32.838 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T21:12:32.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:12:32 smithi070 ceph-mon[168282]: pgmap v1222: 129 pgs: 3 peering, 10 unknown, 16 active+clean+remapped, 4 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+remapped, 12 active+undersized, 1 active+recovering+undersized+degraded+remapped, 6 active+undersized+degraded, 10 active+recovery_wait+undersized+degraded+remapped, 6 active+recovery_wait+degraded+remapped, 59 active+clean; 4.4 GiB data, 16 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 222 op/s; 63154/215574 objects degraded (29.296%); 18626/215574 objects misplaced (8.640%); 5.3 MiB/s, 85 objects/s recovering 2023-09-23T21:12:32.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:12:32 smithi070 ceph-mon[168282]: osdmap e482: 8 total, 7 up, 6 in 2023-09-23T21:12:32.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:12:32 smithi070 ceph-mon[168282]: failed to encode map e482 with expected crc 2023-09-23T21:12:32.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:12:32 smithi070 ceph-mon[168282]: failed to encode map e482 with expected crc 2023-09-23T21:12:32.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:12:32 smithi070 ceph-mon[168282]: failed to encode map e482 with expected crc 2023-09-23T21:12:32.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:12:32 smithi070 ceph-mon[168282]: failed to encode map e482 with expected crc 2023-09-23T21:12:32.871 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:12:32 smithi070 ceph-mon[168282]: failed to encode map e482 with expected crc 2023-09-23T21:12:32.871 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:12:32 smithi070 ceph-mon[168282]: failed to encode map e482 with expected crc 2023-09-23T21:12:32.871 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:12:32 smithi070 ceph-mon[168282]: failed to encode map e482 with expected crc 2023-09-23T21:12:32.871 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:12:32 smithi070 ceph-mon[168282]: failed to encode map e482 with expected crc 2023-09-23T21:12:32.871 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:12:32 smithi070 ceph-mon[168282]: failed to encode map e482 with expected crc 2023-09-23T21:12:32.871 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:12:32 smithi070 ceph-mon[168282]: failed to encode map e482 with expected crc 2023-09-23T21:12:32.881 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:12:32 smithi026 ceph-mon[179688]: pgmap v1222: 129 pgs: 3 peering, 10 unknown, 16 active+clean+remapped, 4 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+remapped, 12 active+undersized, 1 active+recovering+undersized+degraded+remapped, 6 active+undersized+degraded, 10 active+recovery_wait+undersized+degraded+remapped, 6 active+recovery_wait+degraded+remapped, 59 active+clean; 4.4 GiB data, 16 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 222 op/s; 63154/215574 objects degraded (29.296%); 18626/215574 objects misplaced (8.640%); 5.3 MiB/s, 85 objects/s recovering 2023-09-23T21:12:32.881 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:12:32 smithi026 ceph-mon[179688]: osdmap e482: 8 total, 7 up, 6 in 2023-09-23T21:12:32.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:12:32 smithi026 ceph-mon[179688]: failed to encode map e482 with expected crc 2023-09-23T21:12:32.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:12:32 smithi026 ceph-mon[179688]: failed to encode map e482 with expected crc 2023-09-23T21:12:32.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:12:32 smithi026 ceph-mon[179688]: failed to encode map e482 with expected crc 2023-09-23T21:12:32.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:12:32 smithi026 ceph-mon[179688]: failed to encode map e482 with expected crc 2023-09-23T21:12:32.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:12:32 smithi026 ceph-mon[179688]: failed to encode map e482 with expected crc 2023-09-23T21:12:32.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:12:32 smithi026 ceph-mon[179688]: failed to encode map e482 with expected crc 2023-09-23T21:12:32.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:12:32 smithi026 ceph-mon[179688]: failed to encode map e482 with expected crc 2023-09-23T21:12:32.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:12:32 smithi026 ceph-mon[179688]: failed to encode map e482 with expected crc 2023-09-23T21:12:32.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:12:32 smithi026 ceph-mon[179688]: failed to encode map e482 with expected crc 2023-09-23T21:12:32.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:12:32 smithi026 ceph-mon[179688]: failed to encode map e482 with expected crc 2023-09-23T21:12:32.883 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:12:32 smithi026 ceph-mon[175151]: pgmap v1222: 129 pgs: 3 peering, 10 unknown, 16 active+clean+remapped, 4 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+remapped, 12 active+undersized, 1 active+recovering+undersized+degraded+remapped, 6 active+undersized+degraded, 10 active+recovery_wait+undersized+degraded+remapped, 6 active+recovery_wait+degraded+remapped, 59 active+clean; 4.4 GiB data, 16 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 222 op/s; 63154/215574 objects degraded (29.296%); 18626/215574 objects misplaced (8.640%); 5.3 MiB/s, 85 objects/s recovering 2023-09-23T21:12:32.883 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:12:32 smithi026 ceph-mon[175151]: osdmap e482: 8 total, 7 up, 6 in 2023-09-23T21:12:32.883 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:12:32 smithi026 ceph-mon[175151]: failed to encode map e482 with expected crc 2023-09-23T21:12:32.883 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:12:32 smithi026 ceph-mon[175151]: failed to encode map e482 with expected crc 2023-09-23T21:12:32.883 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:12:32 smithi026 ceph-mon[175151]: failed to encode map e482 with expected crc 2023-09-23T21:12:32.883 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:12:32 smithi026 ceph-mon[175151]: failed to encode map e482 with expected crc 2023-09-23T21:12:32.883 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:12:32 smithi026 ceph-mon[175151]: failed to encode map e482 with expected crc 2023-09-23T21:12:32.883 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:12:32 smithi026 ceph-mon[175151]: failed to encode map e482 with expected crc 2023-09-23T21:12:32.883 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:12:32 smithi026 ceph-mon[175151]: failed to encode map e482 with expected crc 2023-09-23T21:12:32.883 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:12:32 smithi026 ceph-mon[175151]: failed to encode map e482 with expected crc 2023-09-23T21:12:32.883 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:12:32 smithi026 ceph-mon[175151]: failed to encode map e482 with expected crc 2023-09-23T21:12:32.883 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:12:32 smithi026 ceph-mon[175151]: failed to encode map e482 with expected crc 2023-09-23T21:12:32.892 INFO:tasks.thrashosds.thrasher:in_osds: [0, 2, 3, 4, 5, 7] out_osds: [6, 1] dead_osds: [0] live_osds: [2, 3, 4, 5, 7, 6, 1] 2023-09-23T21:12:32.892 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T21:12:32.892 INFO:tasks.thrashosds.thrasher:inject_pause on osd.4 2023-09-23T21:12:32.892 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-09-23T21:12:32.892 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-09-23T21:12:32.892 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 config set filestore_inject_stall 3 2023-09-23T21:12:33.313 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:12:33.316 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:12:33.316 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:12:33.418 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:12:33.503 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T21:12:33.521 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:12:33.885 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:12:33.891 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:12:33.891 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:12:33.992 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:12:34.058 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:12:34.179 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:12:34.293 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:12:34.302 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T21:12:34.307 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:12:34.308 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:12:34.408 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:12:34.443 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:12:34.613 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:12:34 smithi070 ceph-mon[168282]: pgmap v1224: 129 pgs: 3 peering, 4 unknown, 18 active+clean+remapped, 4 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 12 active+undersized, 1 active+recovering+undersized+degraded+remapped, 6 active+undersized+degraded, 10 active+recovery_wait+undersized+degraded+remapped, 6 active+recovery_wait+degraded+remapped, 63 active+clean; 4.4 GiB data, 16 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 300 op/s; 62836/214005 objects degraded (29.362%); 18375/214005 objects misplaced (8.586%); 8.1 MiB/s, 130 objects/s recovering 2023-09-23T21:12:34.614 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:12:34 smithi070 ceph-mon[168282]: failed to encode map e482 with expected crc 2023-09-23T21:12:34.634 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:12:34.639 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:12:34.640 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:12:34.741 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:12:34.792 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:12:34 smithi026 ceph-mon[179688]: pgmap v1224: 129 pgs: 3 peering, 4 unknown, 18 active+clean+remapped, 4 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 12 active+undersized, 1 active+recovering+undersized+degraded+remapped, 6 active+undersized+degraded, 10 active+recovery_wait+undersized+degraded+remapped, 6 active+recovery_wait+degraded+remapped, 63 active+clean; 4.4 GiB data, 16 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 300 op/s; 62836/214005 objects degraded (29.362%); 18375/214005 objects misplaced (8.586%); 8.1 MiB/s, 130 objects/s recovering 2023-09-23T21:12:34.792 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:12:34 smithi026 ceph-mon[179688]: failed to encode map e482 with expected crc 2023-09-23T21:12:34.792 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:12:34 smithi026 ceph-mon[175151]: pgmap v1224: 129 pgs: 3 peering, 4 unknown, 18 active+clean+remapped, 4 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 12 active+undersized, 1 active+recovering+undersized+degraded+remapped, 6 active+undersized+degraded, 10 active+recovery_wait+undersized+degraded+remapped, 6 active+recovery_wait+degraded+remapped, 63 active+clean; 4.4 GiB data, 16 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 300 op/s; 62836/214005 objects degraded (29.362%); 18375/214005 objects misplaced (8.586%); 8.1 MiB/s, 130 objects/s recovering 2023-09-23T21:12:34.793 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:12:34 smithi026 ceph-mon[175151]: failed to encode map e482 with expected crc 2023-09-23T21:12:34.853 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:12:34.985 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:12:34.992 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:12:34.992 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:12:35.093 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:12:35.124 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:12:35.165 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:12:35.171 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:12:35.171 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:12:35.272 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:12:35.302 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:12:35.413 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T21:12:35.438 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:12:35.440 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:12:35.441 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:12:35.484 INFO:teuthology.orchestra.run.smithi070.stdout:{ 2023-09-23T21:12:35.484 INFO:teuthology.orchestra.run.smithi070.stdout: "size": 20, 2023-09-23T21:12:35.484 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 600, 2023-09-23T21:12:35.484 INFO:teuthology.orchestra.run.smithi070.stdout: "ops": [ 2023-09-23T21:12:35.485 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:35.485 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.35469.0:132991 8.19 8:9cbd96dc:::benchmark_data_smithi026_301269_object22234:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e281)", 2023-09-23T21:12:35.485 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:03:29.311270+0000", 2023-09-23T21:12:35.485 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 546.16584007999995, 2023-09-23T21:12:35.485 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.004743988, 2023-09-23T21:12:35.485 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:12:35.485 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:12:35.485 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:12:35.485 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.35469", 2023-09-23T21:12:35.485 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/4041424623", 2023-09-23T21:12:35.485 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 132991 2023-09-23T21:12:35.486 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:35.486 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:12:35.486 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:35.486 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:12:35.486 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:29.311270+0000", 2023-09-23T21:12:35.486 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:12:35.486 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:35.486 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:35.486 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:12:35.486 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:29.311270+0000", 2023-09-23T21:12:35.486 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:12:35.486 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:35.487 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:35.487 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:12:35.487 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:29.311267+0000", 2023-09-23T21:12:35.487 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999971 2023-09-23T21:12:35.487 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:35.487 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:35.487 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:12:35.487 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:29.311282+0000", 2023-09-23T21:12:35.487 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.4018e-05 2023-09-23T21:12:35.487 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:35.487 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:35.487 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:12:35.488 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:29.311313+0000", 2023-09-23T21:12:35.488 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.1189000000000002e-05 2023-09-23T21:12:35.488 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:35.488 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:35.488 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:12:35.488 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:29.311325+0000", 2023-09-23T21:12:35.488 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.1951e-05 2023-09-23T21:12:35.488 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:35.488 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:35.488 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:12:35.488 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:29.453380+0000", 2023-09-23T21:12:35.488 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.14205537900000001 2023-09-23T21:12:35.488 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:35.489 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:35.489 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:12:35.489 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:29.453388+0000", 2023-09-23T21:12:35.489 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.5630000000000003e-06 2023-09-23T21:12:35.489 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:35.489 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:35.489 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:12:35.489 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:30.311125+0000", 2023-09-23T21:12:35.489 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.85773704500000003 2023-09-23T21:12:35.489 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:35.489 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:35.489 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:12:35.490 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:30.311242+0000", 2023-09-23T21:12:35.490 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000117271 2023-09-23T21:12:35.490 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:35.490 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:35.490 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 0,5", 2023-09-23T21:12:35.490 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:30.311447+0000", 2023-09-23T21:12:35.490 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00020477200000000001 2023-09-23T21:12:35.490 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:35.490 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:35.490 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:12:35.490 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:30.315503+0000", 2023-09-23T21:12:35.490 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0040560320000000002 2023-09-23T21:12:35.490 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:35.491 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:35.491 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:12:35.491 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:30.315868+0000", 2023-09-23T21:12:35.491 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00036542900000000001 2023-09-23T21:12:35.491 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:35.491 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:35.491 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:12:35.491 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:30.315916+0000", 2023-09-23T21:12:35.491 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.7395000000000003e-05 2023-09-23T21:12:35.491 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:35.491 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:35.491 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:12:35.492 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:30.315975+0000", 2023-09-23T21:12:35.492 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.9741000000000003e-05 2023-09-23T21:12:35.492 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:35.492 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:35.492 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:12:35.492 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:30.316014+0000", 2023-09-23T21:12:35.492 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.9153000000000003e-05 2023-09-23T21:12:35.492 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:12:35.492 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:12:35.492 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:12:35.492 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:35.492 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:35.492 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.35469.0:133002 8.19 8:9f35daf1:::benchmark_data_smithi026_301269_object22245:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e281)", 2023-09-23T21:12:35.493 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:03:29.311341+0000", 2023-09-23T21:12:35.493 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 546.16576970400001, 2023-09-23T21:12:35.493 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.0085733610000001, 2023-09-23T21:12:35.493 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:12:35.493 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:12:35.493 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:12:35.493 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.35469", 2023-09-23T21:12:35.493 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/4041424623", 2023-09-23T21:12:35.493 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 133002 2023-09-23T21:12:35.493 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:35.493 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:12:35.493 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:35.494 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:12:35.494 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:29.311341+0000", 2023-09-23T21:12:35.494 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:12:35.494 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:35.494 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:35.494 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:12:35.494 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:29.311341+0000", 2023-09-23T21:12:35.494 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:12:35.494 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:35.494 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:35.494 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:12:35.494 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:29.311340+0000", 2023-09-23T21:12:35.495 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999995 2023-09-23T21:12:35.495 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:35.495 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:35.495 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:12:35.495 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:29.311343+0000", 2023-09-23T21:12:35.495 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.8940000000000002e-06 2023-09-23T21:12:35.495 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:35.495 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:35.495 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:12:35.495 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:29.311349+0000", 2023-09-23T21:12:35.495 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.1199999999999999e-06 2023-09-23T21:12:35.495 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:35.496 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:35.496 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:12:35.496 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:29.311353+0000", 2023-09-23T21:12:35.496 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.805e-06 2023-09-23T21:12:35.496 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:35.496 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:35.496 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:12:35.496 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:29.453406+0000", 2023-09-23T21:12:35.496 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.14205331299999999 2023-09-23T21:12:35.496 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:35.496 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:35.496 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:12:35.497 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:29.453411+0000", 2023-09-23T21:12:35.497 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.9509999999999997e-06 2023-09-23T21:12:35.497 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:35.497 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:35.497 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:12:35.497 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:30.312451+0000", 2023-09-23T21:12:35.497 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.85903911499999996 2023-09-23T21:12:35.497 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:35.497 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:35.497 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:12:35.497 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:30.312530+0000", 2023-09-23T21:12:35.497 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.9346999999999998e-05 2023-09-23T21:12:35.498 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:35.498 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:35.498 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 0,5", 2023-09-23T21:12:35.498 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:30.312649+0000", 2023-09-23T21:12:35.498 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000118867 2023-09-23T21:12:35.498 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:35.498 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:35.498 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:12:35.498 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:30.315510+0000", 2023-09-23T21:12:35.498 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0028615839999999999 2023-09-23T21:12:35.498 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:35.498 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:35.499 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:12:35.499 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:30.316097+0000", 2023-09-23T21:12:35.499 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00058688200000000005 2023-09-23T21:12:35.499 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:35.499 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:35.499 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:12:35.499 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:30.319819+0000", 2023-09-23T21:12:35.499 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0037219219999999999 2023-09-23T21:12:35.499 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:35.499 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:35.499 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:12:35.499 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:30.319871+0000", 2023-09-23T21:12:35.499 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.1753999999999999e-05 2023-09-23T21:12:35.500 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:35.500 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:35.500 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:12:35.500 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:30.319914+0000", 2023-09-23T21:12:35.500 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.3278000000000001e-05 2023-09-23T21:12:35.500 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:12:35.500 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:12:35.500 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:12:35.500 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:35.500 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:35.500 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.35469.0:133004 8.19 8:98907399:::benchmark_data_smithi026_301269_object22247:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e281)", 2023-09-23T21:12:35.500 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:03:29.311357+0000", 2023-09-23T21:12:35.501 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 546.16575305100002, 2023-09-23T21:12:35.501 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.0087018400000001, 2023-09-23T21:12:35.501 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:12:35.501 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:12:35.501 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:12:35.501 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.35469", 2023-09-23T21:12:35.501 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/4041424623", 2023-09-23T21:12:35.501 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 133004 2023-09-23T21:12:35.501 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:35.501 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:12:35.501 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:35.501 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:12:35.502 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:29.311357+0000", 2023-09-23T21:12:35.502 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:12:35.502 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:35.502 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:35.502 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:12:35.502 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:29.311357+0000", 2023-09-23T21:12:35.502 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:12:35.502 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:35.502 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:35.502 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:12:35.502 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:29.311357+0000", 2023-09-23T21:12:35.502 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999995 2023-09-23T21:12:35.503 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:35.503 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:35.503 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:12:35.503 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:29.311359+0000", 2023-09-23T21:12:35.503 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.1509999999999998e-06 2023-09-23T21:12:35.503 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:35.503 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:35.503 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:12:35.503 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:29.311364+0000", 2023-09-23T21:12:35.503 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.9169999999999997e-06 2023-09-23T21:12:35.503 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:35.503 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:35.504 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:12:35.504 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:29.311367+0000", 2023-09-23T21:12:35.504 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.1870000000000001e-06 2023-09-23T21:12:35.504 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:35.504 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:35.504 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:12:35.504 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:29.453430+0000", 2023-09-23T21:12:35.504 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.142062514 2023-09-23T21:12:35.504 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:35.504 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:35.504 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:12:35.504 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:29.453435+0000", 2023-09-23T21:12:35.505 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.8729999999999998e-06 2023-09-23T21:12:35.505 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:35.505 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:35.505 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:12:35.505 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:30.313361+0000", 2023-09-23T21:12:35.505 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.85992620799999997 2023-09-23T21:12:35.505 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:35.505 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:35.505 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:12:35.505 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:30.313428+0000", 2023-09-23T21:12:35.505 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.7355999999999997e-05 2023-09-23T21:12:35.505 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:35.506 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:35.506 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 0,5", 2023-09-23T21:12:35.506 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:30.313551+0000", 2023-09-23T21:12:35.506 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00012231000000000001 2023-09-23T21:12:35.506 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:35.506 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:35.506 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:12:35.506 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:30.315819+0000", 2023-09-23T21:12:35.506 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.002268267 2023-09-23T21:12:35.506 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:35.506 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:35.506 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:12:35.507 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:30.316740+0000", 2023-09-23T21:12:35.507 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00092082700000000004 2023-09-23T21:12:35.507 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:35.507 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:35.507 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:12:35.507 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:30.319974+0000", 2023-09-23T21:12:35.507 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0032337220000000001 2023-09-23T21:12:35.507 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:35.507 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:35.507 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:12:35.507 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:30.320024+0000", 2023-09-23T21:12:35.507 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.0667999999999997e-05 2023-09-23T21:12:35.508 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:35.508 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:35.508 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:12:35.508 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:30.320059+0000", 2023-09-23T21:12:35.508 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.5120999999999998e-05 2023-09-23T21:12:35.508 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:12:35.508 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:12:35.508 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:12:35.508 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:35.508 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:35.508 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.35469.0:133011 8.5 8:a1043694:::benchmark_data_smithi026_301269_object22254:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e281)", 2023-09-23T21:12:35.508 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:03:29.556262+0000", 2023-09-23T21:12:35.509 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 545.92084816399995, 2023-09-23T21:12:35.509 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.74415403300000005, 2023-09-23T21:12:35.509 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:12:35.509 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:12:35.509 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:12:35.509 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.35469", 2023-09-23T21:12:35.509 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/4041424623", 2023-09-23T21:12:35.509 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 133011 2023-09-23T21:12:35.509 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:35.509 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:12:35.509 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:35.509 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:12:35.510 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:29.556262+0000", 2023-09-23T21:12:35.510 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:12:35.510 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:35.510 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:35.510 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:12:35.510 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:29.556262+0000", 2023-09-23T21:12:35.510 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:12:35.510 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:35.510 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:35.510 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:12:35.510 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:29.556260+0000", 2023-09-23T21:12:35.510 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999976 2023-09-23T21:12:35.510 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:35.511 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:35.511 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:12:35.511 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:29.556273+0000", 2023-09-23T21:12:35.511 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.2782000000000001e-05 2023-09-23T21:12:35.511 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:35.511 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:35.511 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:12:35.511 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:29.556299+0000", 2023-09-23T21:12:35.511 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.6855999999999998e-05 2023-09-23T21:12:35.511 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:35.511 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:35.511 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:12:35.512 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:29.556313+0000", 2023-09-23T21:12:35.512 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.3550000000000001e-05 2023-09-23T21:12:35.512 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:35.512 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:35.512 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:12:35.512 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:29.556374+0000", 2023-09-23T21:12:35.512 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.0738e-05 2023-09-23T21:12:35.512 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:35.512 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:35.512 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:12:35.512 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:29.556386+0000", 2023-09-23T21:12:35.512 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.1806e-05 2023-09-23T21:12:35.513 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:35.513 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:35.513 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:12:35.513 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:30.279558+0000", 2023-09-23T21:12:35.513 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.72317220800000004 2023-09-23T21:12:35.513 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:35.513 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:35.513 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:12:35.513 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:30.279690+0000", 2023-09-23T21:12:35.513 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00013265400000000001 2023-09-23T21:12:35.513 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:35.513 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:35.514 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 4,7", 2023-09-23T21:12:35.514 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:30.280070+0000", 2023-09-23T21:12:35.514 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00037984200000000003 2023-09-23T21:12:35.514 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:35.514 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:35.514 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:12:35.514 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:30.285413+0000", 2023-09-23T21:12:35.514 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0053427259999999999 2023-09-23T21:12:35.514 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:35.514 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:35.514 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:12:35.514 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:30.287185+0000", 2023-09-23T21:12:35.515 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.001771658 2023-09-23T21:12:35.515 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:35.515 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:35.515 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:12:35.515 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:30.300329+0000", 2023-09-23T21:12:35.515 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.013144718 2023-09-23T21:12:35.515 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:35.515 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:35.515 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:12:35.515 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:30.300379+0000", 2023-09-23T21:12:35.515 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.9703999999999997e-05 2023-09-23T21:12:35.515 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:35.516 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:35.516 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:12:35.516 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:30.300416+0000", 2023-09-23T21:12:35.516 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.7321999999999999e-05 2023-09-23T21:12:35.516 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:12:35.516 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:12:35.516 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:12:35.516 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:35.516 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:35.516 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.35469.0:133012 8.9 8:95bb4905:::benchmark_data_smithi026_301269_object22255:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e281)", 2023-09-23T21:12:35.516 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:03:29.606840+0000", 2023-09-23T21:12:35.516 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 545.87027091699997, 2023-09-23T21:12:35.517 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.68457533800000003, 2023-09-23T21:12:35.517 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:12:35.517 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:12:35.517 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:12:35.517 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.35469", 2023-09-23T21:12:35.517 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/4041424623", 2023-09-23T21:12:35.517 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 133012 2023-09-23T21:12:35.517 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:35.517 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:12:35.517 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:35.517 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:12:35.517 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:29.606840+0000", 2023-09-23T21:12:35.517 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:12:35.518 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:35.518 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:35.518 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:12:35.518 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:29.606840+0000", 2023-09-23T21:12:35.518 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:12:35.518 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:35.518 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:35.518 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:12:35.518 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:29.606838+0000", 2023-09-23T21:12:35.518 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999981 2023-09-23T21:12:35.518 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:35.518 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:35.519 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:12:35.519 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:29.606846+0000", 2023-09-23T21:12:35.519 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.2109999999999998e-06 2023-09-23T21:12:35.519 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:35.519 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:35.519 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:12:35.519 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:29.606880+0000", 2023-09-23T21:12:35.519 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.4286999999999999e-05 2023-09-23T21:12:35.519 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:35.519 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:35.519 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:12:35.519 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:29.606889+0000", 2023-09-23T21:12:35.520 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.6659999999999997e-06 2023-09-23T21:12:35.520 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:35.520 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:35.520 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:12:35.520 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:29.606945+0000", 2023-09-23T21:12:35.520 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.6005000000000003e-05 2023-09-23T21:12:35.520 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:35.520 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:35.520 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:12:35.520 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:29.606956+0000", 2023-09-23T21:12:35.520 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.1122e-05 2023-09-23T21:12:35.520 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:35.521 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:35.521 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:12:35.521 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:30.247854+0000", 2023-09-23T21:12:35.521 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.64089811100000005 2023-09-23T21:12:35.521 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:35.521 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:35.521 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:12:35.521 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:30.248009+0000", 2023-09-23T21:12:35.521 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00015453 2023-09-23T21:12:35.521 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:35.521 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:35.521 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 0,5", 2023-09-23T21:12:35.521 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:30.248490+0000", 2023-09-23T21:12:35.522 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00048119599999999998 2023-09-23T21:12:35.522 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:35.522 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:35.522 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:12:35.522 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:30.251137+0000", 2023-09-23T21:12:35.522 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0026472409999999998 2023-09-23T21:12:35.522 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:35.522 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:35.522 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:12:35.522 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:30.286568+0000", 2023-09-23T21:12:35.522 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.035431129999999998 2023-09-23T21:12:35.522 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:35.523 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:35.523 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:12:35.523 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:30.291324+0000", 2023-09-23T21:12:35.523 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.004755881 2023-09-23T21:12:35.523 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:35.523 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:35.523 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:12:35.523 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:30.291374+0000", 2023-09-23T21:12:35.523 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.9885000000000001e-05 2023-09-23T21:12:35.523 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:35.523 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:35.523 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:12:35.524 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:30.291415+0000", 2023-09-23T21:12:35.524 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.0858000000000001e-05 2023-09-23T21:12:35.524 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:12:35.524 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:12:35.524 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:12:35.524 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:35.524 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:35.524 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.35469.0:141176 8.12 8.59eb0f52 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e288)", 2023-09-23T21:12:35.524 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:03:41.352623+0000", 2023-09-23T21:12:35.524 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 534.12448708399995, 2023-09-23T21:12:35.524 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.829076496, 2023-09-23T21:12:35.524 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:12:35.525 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "reached pg", 2023-09-23T21:12:35.525 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:12:35.525 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.35469", 2023-09-23T21:12:35.525 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/4041424623", 2023-09-23T21:12:35.525 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 141176 2023-09-23T21:12:35.525 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:35.525 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:12:35.525 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:35.525 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:12:35.525 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:41.352623+0000", 2023-09-23T21:12:35.525 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:12:35.525 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:35.525 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:35.526 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:12:35.526 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:41.352623+0000", 2023-09-23T21:12:35.526 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:12:35.526 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:35.526 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:35.526 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:12:35.526 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:41.352623+0000", 2023-09-23T21:12:35.526 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999995 2023-09-23T21:12:35.526 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:35.526 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:35.526 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:12:35.526 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:41.352626+0000", 2023-09-23T21:12:35.527 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.7980000000000001e-06 2023-09-23T21:12:35.527 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:35.527 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:35.527 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:12:35.527 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:41.352636+0000", 2023-09-23T21:12:35.527 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.8630000000000003e-06 2023-09-23T21:12:35.527 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:35.527 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:35.527 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:12:35.527 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:41.352639+0000", 2023-09-23T21:12:35.527 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.8109999999999999e-06 2023-09-23T21:12:35.527 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:35.528 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:35.528 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:12:35.528 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:41.352683+0000", 2023-09-23T21:12:35.528 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.4365e-05 2023-09-23T21:12:35.528 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:35.528 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:35.528 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:12:35.528 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:41.352688+0000", 2023-09-23T21:12:35.528 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.164e-06 2023-09-23T21:12:35.528 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:35.528 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:35.528 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:12:35.529 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:42.181691+0000", 2023-09-23T21:12:35.529 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.82900300699999996 2023-09-23T21:12:35.529 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:35.529 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:35.529 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:12:35.529 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:42.181700+0000", 2023-09-23T21:12:35.529 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.8890000000000008e-06 2023-09-23T21:12:35.529 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:12:35.529 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:12:35.529 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:12:35.529 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:35.529 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:35.530 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.35469.0:141263 8.12 8.15bb0532 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e288)", 2023-09-23T21:12:35.530 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:03:41.434595+0000", 2023-09-23T21:12:35.530 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 534.04251544199997, 2023-09-23T21:12:35.530 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.74712263400000001, 2023-09-23T21:12:35.530 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:12:35.530 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "reached pg", 2023-09-23T21:12:35.530 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:12:35.530 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.35469", 2023-09-23T21:12:35.530 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/4041424623", 2023-09-23T21:12:35.530 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 141263 2023-09-23T21:12:35.530 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:35.530 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:12:35.531 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:35.531 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:12:35.531 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:41.434595+0000", 2023-09-23T21:12:35.531 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:12:35.531 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:35.531 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:35.531 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:12:35.531 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:41.434595+0000", 2023-09-23T21:12:35.531 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:12:35.531 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:35.531 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:35.531 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:12:35.532 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:41.434594+0000", 2023-09-23T21:12:35.532 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T21:12:35.532 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:35.532 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:35.532 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:12:35.532 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:41.434601+0000", 2023-09-23T21:12:35.532 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.8859999999999998e-06 2023-09-23T21:12:35.532 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:35.532 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:35.532 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:12:35.532 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:41.434613+0000", 2023-09-23T21:12:35.532 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.1922999999999999e-05 2023-09-23T21:12:35.533 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:35.533 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:35.533 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:12:35.533 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:41.434619+0000", 2023-09-23T21:12:35.533 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.6039999999999998e-06 2023-09-23T21:12:35.533 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:35.533 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:35.533 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:12:35.533 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:41.434729+0000", 2023-09-23T21:12:35.533 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000109573 2023-09-23T21:12:35.533 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:35.533 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:35.534 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:12:35.534 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:41.434736+0000", 2023-09-23T21:12:35.534 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.2250000000000003e-06 2023-09-23T21:12:35.534 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:35.534 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:35.534 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:12:35.534 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:42.181714+0000", 2023-09-23T21:12:35.534 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.74697798999999998 2023-09-23T21:12:35.534 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:35.534 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:35.534 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:12:35.534 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:42.181718+0000", 2023-09-23T21:12:35.535 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.749e-06 2023-09-23T21:12:35.535 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:12:35.535 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:12:35.535 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:12:35.535 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:35.535 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:35.535 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.35469.0:141302 8.12 8.247f1cf2 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e288)", 2023-09-23T21:12:35.535 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:03:41.472759+0000", 2023-09-23T21:12:35.535 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 534.00435188999995, 2023-09-23T21:12:35.535 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.70897215400000002, 2023-09-23T21:12:35.535 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:12:35.535 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "reached pg", 2023-09-23T21:12:35.536 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:12:35.536 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.35469", 2023-09-23T21:12:35.536 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/4041424623", 2023-09-23T21:12:35.536 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 141302 2023-09-23T21:12:35.536 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:35.536 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:12:35.536 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:35.536 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:12:35.536 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:41.472759+0000", 2023-09-23T21:12:35.536 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:12:35.536 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:35.536 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:35.537 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:12:35.537 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:41.472759+0000", 2023-09-23T21:12:35.537 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:12:35.537 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:35.537 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:35.537 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:12:35.537 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:41.472757+0000", 2023-09-23T21:12:35.537 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T21:12:35.537 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:35.537 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:35.537 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:12:35.537 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:41.472765+0000", 2023-09-23T21:12:35.538 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.3889999999999999e-06 2023-09-23T21:12:35.538 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:35.538 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:35.538 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:12:35.538 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:41.472784+0000", 2023-09-23T21:12:35.538 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.8918000000000001e-05 2023-09-23T21:12:35.538 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:35.538 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:35.538 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:12:35.538 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:41.472791+0000", 2023-09-23T21:12:35.538 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.5009999999999998e-06 2023-09-23T21:12:35.538 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:35.539 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:35.539 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:12:35.539 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:41.478507+0000", 2023-09-23T21:12:35.539 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0057159840000000003 2023-09-23T21:12:35.539 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:35.539 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:35.539 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:12:35.539 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:41.478517+0000", 2023-09-23T21:12:35.539 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.4900000000000006e-06 2023-09-23T21:12:35.539 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:35.539 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:35.539 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:12:35.540 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:42.181727+0000", 2023-09-23T21:12:35.540 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.70321072900000003 2023-09-23T21:12:35.540 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:35.540 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:35.540 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:12:35.540 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:42.181731+0000", 2023-09-23T21:12:35.540 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.4300000000000002e-06 2023-09-23T21:12:35.540 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:12:35.540 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:12:35.540 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:12:35.540 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:35.540 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:35.541 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.35469.0:141327 8.12 8.e4f6e4d2 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e288)", 2023-09-23T21:12:35.541 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:03:41.504514+0000", 2023-09-23T21:12:35.541 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 533.97259628899997, 2023-09-23T21:12:35.541 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.67722902600000001, 2023-09-23T21:12:35.541 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:12:35.541 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "reached pg", 2023-09-23T21:12:35.541 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:12:35.541 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.35469", 2023-09-23T21:12:35.541 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/4041424623", 2023-09-23T21:12:35.541 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 141327 2023-09-23T21:12:35.541 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:35.541 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:12:35.541 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:35.542 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:12:35.542 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:41.504514+0000", 2023-09-23T21:12:35.542 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:12:35.542 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:35.542 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:35.542 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:12:35.542 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:41.504514+0000", 2023-09-23T21:12:35.542 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:12:35.542 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:35.542 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:35.542 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:12:35.543 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:41.504513+0000", 2023-09-23T21:12:35.543 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T21:12:35.543 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:35.543 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:35.543 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:12:35.543 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:41.504519+0000", 2023-09-23T21:12:35.543 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.3690000000000003e-06 2023-09-23T21:12:35.543 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:35.543 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:35.543 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:12:35.543 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:41.504532+0000", 2023-09-23T21:12:35.543 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.2979999999999999e-05 2023-09-23T21:12:35.544 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:35.544 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:35.544 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:12:35.544 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:41.504539+0000", 2023-09-23T21:12:35.544 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.494e-06 2023-09-23T21:12:35.544 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:35.544 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:35.544 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:12:35.544 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:41.504615+0000", 2023-09-23T21:12:35.544 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.5727000000000002e-05 2023-09-23T21:12:35.544 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:35.544 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:35.545 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:12:35.545 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:41.504622+0000", 2023-09-23T21:12:35.545 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.8569999999999995e-06 2023-09-23T21:12:35.545 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:35.545 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:35.545 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:12:35.545 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:42.181740+0000", 2023-09-23T21:12:35.545 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.67711736499999997 2023-09-23T21:12:35.545 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:35.545 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:35.545 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:12:35.545 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:42.181743+0000", 2023-09-23T21:12:35.545 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.4350000000000001e-06 2023-09-23T21:12:35.546 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:12:35.546 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:12:35.546 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:12:35.546 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:35.546 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:35.546 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.35469.0:141331 8.12 8.f3501032 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e288)", 2023-09-23T21:12:35.546 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:03:41.504554+0000", 2023-09-23T21:12:35.546 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 533.97255623499996, 2023-09-23T21:12:35.546 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.67720773599999995, 2023-09-23T21:12:35.546 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:12:35.546 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "reached pg", 2023-09-23T21:12:35.546 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:12:35.547 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.35469", 2023-09-23T21:12:35.547 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/4041424623", 2023-09-23T21:12:35.547 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 141331 2023-09-23T21:12:35.547 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:35.547 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:12:35.547 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:35.547 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:12:35.547 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:41.504554+0000", 2023-09-23T21:12:35.547 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:12:35.547 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:35.547 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:35.547 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:12:35.548 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:41.504554+0000", 2023-09-23T21:12:35.548 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:12:35.548 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:35.548 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:35.548 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:12:35.548 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:41.504554+0000", 2023-09-23T21:12:35.548 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999995 2023-09-23T21:12:35.548 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:35.548 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:35.548 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:12:35.548 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:41.504557+0000", 2023-09-23T21:12:35.548 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.399e-06 2023-09-23T21:12:35.548 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:35.549 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:35.549 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:12:35.549 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:41.504564+0000", 2023-09-23T21:12:35.549 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.2189999999999998e-06 2023-09-23T21:12:35.549 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:35.549 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:35.549 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:12:35.549 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:41.504569+0000", 2023-09-23T21:12:35.549 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.2370000000000003e-06 2023-09-23T21:12:35.549 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:35.549 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:35.549 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:12:35.549 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:41.506362+0000", 2023-09-23T21:12:35.550 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0017935379999999999 2023-09-23T21:12:35.550 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:35.550 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:35.550 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:12:35.550 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:41.506369+0000", 2023-09-23T21:12:35.550 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.0299999999999996e-06 2023-09-23T21:12:35.550 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:35.550 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:35.550 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:12:35.550 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:42.181756+0000", 2023-09-23T21:12:35.550 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.67538684500000001 2023-09-23T21:12:35.550 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:35.551 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:35.551 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:12:35.551 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:42.181762+0000", 2023-09-23T21:12:35.551 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.8860000000000001e-06 2023-09-23T21:12:35.551 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:12:35.551 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:12:35.551 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:12:35.551 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:35.551 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:35.551 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.35469.0:141336 8.12 8.541a9572 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e288)", 2023-09-23T21:12:35.551 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:03:41.506356+0000", 2023-09-23T21:12:35.551 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 533.97075408199998, 2023-09-23T21:12:35.551 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.67542309499999997, 2023-09-23T21:12:35.552 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:12:35.552 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "reached pg", 2023-09-23T21:12:35.552 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:12:35.552 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.35469", 2023-09-23T21:12:35.552 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/4041424623", 2023-09-23T21:12:35.552 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 141336 2023-09-23T21:12:35.552 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:35.552 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:12:35.552 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:35.552 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:12:35.552 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:41.506356+0000", 2023-09-23T21:12:35.552 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:12:35.553 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:35.553 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:35.553 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:12:35.553 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:41.506356+0000", 2023-09-23T21:12:35.553 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:12:35.553 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:35.553 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:35.553 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:12:35.553 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:41.506355+0000", 2023-09-23T21:12:35.553 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.999999 2023-09-23T21:12:35.553 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:35.553 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:35.553 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:12:35.554 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:41.506360+0000", 2023-09-23T21:12:35.554 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.2089999999999996e-06 2023-09-23T21:12:35.554 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:35.554 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:35.554 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:12:35.554 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:41.506372+0000", 2023-09-23T21:12:35.554 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.1656e-05 2023-09-23T21:12:35.554 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:35.554 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:35.554 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:12:35.554 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:41.506378+0000", 2023-09-23T21:12:35.554 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.3180000000000003e-06 2023-09-23T21:12:35.554 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:35.555 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:35.555 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:12:35.555 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:41.506451+0000", 2023-09-23T21:12:35.555 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.2465999999999994e-05 2023-09-23T21:12:35.555 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:35.555 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:35.555 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:12:35.555 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:41.506456+0000", 2023-09-23T21:12:35.555 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.9989999999999999e-06 2023-09-23T21:12:35.555 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:35.555 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:35.555 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:12:35.556 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:42.181776+0000", 2023-09-23T21:12:35.556 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.675320013 2023-09-23T21:12:35.556 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:35.556 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:35.556 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:12:35.556 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:42.181780+0000", 2023-09-23T21:12:35.556 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.614e-06 2023-09-23T21:12:35.556 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:12:35.556 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:12:35.556 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:12:35.556 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:35.556 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:35.556 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.35469.0:141369 8.12 8.5d876312 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e288)", 2023-09-23T21:12:35.557 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:03:41.546526+0000", 2023-09-23T21:12:35.557 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 533.93058492800003, 2023-09-23T21:12:35.557 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.63526674999999999, 2023-09-23T21:12:35.557 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:12:35.557 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "reached pg", 2023-09-23T21:12:35.557 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:12:35.557 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.35469", 2023-09-23T21:12:35.557 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/4041424623", 2023-09-23T21:12:35.557 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 141369 2023-09-23T21:12:35.557 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:35.557 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:12:35.557 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:35.557 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:12:35.558 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:41.546526+0000", 2023-09-23T21:12:35.558 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:12:35.558 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:35.558 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:35.558 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:12:35.558 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:41.546526+0000", 2023-09-23T21:12:35.558 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:12:35.558 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:35.558 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:35.558 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:12:35.558 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:41.546525+0000", 2023-09-23T21:12:35.558 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999995 2023-09-23T21:12:35.559 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:35.559 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:35.559 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:12:35.559 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:41.546528+0000", 2023-09-23T21:12:35.559 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.6730000000000001e-06 2023-09-23T21:12:35.559 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:35.559 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:35.559 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:12:35.559 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:41.548135+0000", 2023-09-23T21:12:35.559 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0016073050000000001 2023-09-23T21:12:35.559 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:35.559 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:35.559 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:12:35.560 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:41.548143+0000", 2023-09-23T21:12:35.560 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.3210000000000005e-06 2023-09-23T21:12:35.560 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:35.560 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:35.560 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:12:35.560 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:41.548192+0000", 2023-09-23T21:12:35.560 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.8665999999999998e-05 2023-09-23T21:12:35.560 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:35.560 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:35.560 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:12:35.560 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:41.548200+0000", 2023-09-23T21:12:35.560 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.0539999999999995e-06 2023-09-23T21:12:35.561 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:35.561 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:35.561 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:12:35.561 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:42.181789+0000", 2023-09-23T21:12:35.561 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.63358875199999998 2023-09-23T21:12:35.561 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:35.561 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:35.561 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:12:35.561 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:42.181792+0000", 2023-09-23T21:12:35.561 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.4029999999999999e-06 2023-09-23T21:12:35.561 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:12:35.561 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:12:35.561 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:12:35.562 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:35.562 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:35.562 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.35469.0:141420 8.12 8.1a2d0b12 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e288)", 2023-09-23T21:12:35.562 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:03:41.604591+0000", 2023-09-23T21:12:35.562 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 533.87251906799997, 2023-09-23T21:12:35.562 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.57721354700000005, 2023-09-23T21:12:35.562 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:12:35.562 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "reached pg", 2023-09-23T21:12:35.562 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:12:35.562 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.35469", 2023-09-23T21:12:35.562 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/4041424623", 2023-09-23T21:12:35.562 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 141420 2023-09-23T21:12:35.563 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:35.563 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:12:35.563 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:35.563 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:12:35.563 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:41.604591+0000", 2023-09-23T21:12:35.563 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:12:35.563 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:35.563 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:35.563 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:12:35.563 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:41.604591+0000", 2023-09-23T21:12:35.563 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:12:35.563 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:35.563 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:35.564 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:12:35.564 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:41.604590+0000", 2023-09-23T21:12:35.564 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T21:12:35.564 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:35.564 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:35.564 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:12:35.564 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:41.604597+0000", 2023-09-23T21:12:35.564 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.4740000000000002e-06 2023-09-23T21:12:35.564 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:35.564 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:35.564 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:12:35.564 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:41.604612+0000", 2023-09-23T21:12:35.565 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.5313999999999999e-05 2023-09-23T21:12:35.565 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:35.565 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:35.565 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:12:35.565 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:41.604619+0000", 2023-09-23T21:12:35.565 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.6980000000000001e-06 2023-09-23T21:12:35.565 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:35.565 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:35.565 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:12:35.565 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:41.604676+0000", 2023-09-23T21:12:35.565 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.7535999999999999e-05 2023-09-23T21:12:35.565 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:35.565 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:35.566 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:12:35.566 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:41.604684+0000", 2023-09-23T21:12:35.566 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.7079999999999997e-06 2023-09-23T21:12:35.566 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:35.566 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:35.566 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:12:35.566 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:42.181802+0000", 2023-09-23T21:12:35.566 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.57711783100000003 2023-09-23T21:12:35.566 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:35.566 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:35.566 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:12:35.566 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:42.181805+0000", 2023-09-23T21:12:35.567 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.3869999999999999e-06 2023-09-23T21:12:35.567 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:12:35.567 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:12:35.567 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:12:35.567 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:35.567 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:35.567 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.35469.0:141482 8.12 8.7bcd7e92 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e288)", 2023-09-23T21:12:35.567 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:03:41.685265+0000", 2023-09-23T21:12:35.567 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 533.79184553300001, 2023-09-23T21:12:35.567 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.496552573, 2023-09-23T21:12:35.567 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:12:35.567 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "reached pg", 2023-09-23T21:12:35.568 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:12:35.568 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.35469", 2023-09-23T21:12:35.568 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/4041424623", 2023-09-23T21:12:35.568 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 141482 2023-09-23T21:12:35.568 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:35.568 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:12:35.568 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:35.568 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:12:35.568 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:41.685265+0000", 2023-09-23T21:12:35.568 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:12:35.568 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:35.568 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:35.568 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:12:35.569 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:41.685265+0000", 2023-09-23T21:12:35.569 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:12:35.569 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:35.569 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:35.569 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:12:35.569 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:41.685264+0000", 2023-09-23T21:12:35.569 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.999999 2023-09-23T21:12:35.569 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:35.569 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:35.569 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:12:35.569 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:41.685270+0000", 2023-09-23T21:12:35.569 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.2559999999999999e-06 2023-09-23T21:12:35.569 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:35.570 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:35.570 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:12:35.570 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:41.685285+0000", 2023-09-23T21:12:35.570 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.4681999999999999e-05 2023-09-23T21:12:35.570 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:35.570 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:35.570 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:12:35.570 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:41.685291+0000", 2023-09-23T21:12:35.570 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.4439999999999996e-06 2023-09-23T21:12:35.570 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:35.570 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:35.570 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:12:35.571 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:41.685501+0000", 2023-09-23T21:12:35.571 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00021014100000000001 2023-09-23T21:12:35.571 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:35.571 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:35.571 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:12:35.571 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:41.685508+0000", 2023-09-23T21:12:35.571 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.4690000000000002e-06 2023-09-23T21:12:35.571 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:35.571 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:35.571 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:12:35.571 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:42.181814+0000", 2023-09-23T21:12:35.572 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.496306303 2023-09-23T21:12:35.572 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:35.572 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:35.572 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:12:35.572 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:03:42.181818+0000", 2023-09-23T21:12:35.572 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.3850000000000002e-06 2023-09-23T21:12:35.572 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:12:35.572 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:12:35.572 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:12:35.572 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:35.572 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:35.572 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.35469.0:164178 8.12 8:4fdafa29:::benchmark_data_smithi026_301269_object53421:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e307)", 2023-09-23T21:12:35.573 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:04:13.339306+0000", 2023-09-23T21:12:35.573 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 502.13780444899999, 2023-09-23T21:12:35.573 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.90960294200000003, 2023-09-23T21:12:35.573 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:12:35.573 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:12:35.573 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:12:35.573 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.35469", 2023-09-23T21:12:35.573 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/4041424623", 2023-09-23T21:12:35.573 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 164178 2023-09-23T21:12:35.573 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:35.573 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:12:35.573 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:35.573 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:12:35.574 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:13.339306+0000", 2023-09-23T21:12:35.574 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:12:35.574 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:35.574 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:35.574 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:12:35.574 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:13.339306+0000", 2023-09-23T21:12:35.574 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:12:35.574 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:35.574 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:35.574 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:12:35.574 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:13.339305+0000", 2023-09-23T21:12:35.574 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T21:12:35.575 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:35.575 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:35.575 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:12:35.575 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:13.339312+0000", 2023-09-23T21:12:35.575 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.0620000000000003e-06 2023-09-23T21:12:35.575 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:35.575 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:35.575 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:12:35.575 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:13.339327+0000", 2023-09-23T21:12:35.575 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.5359999999999999e-05 2023-09-23T21:12:35.575 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:35.575 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:35.576 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:12:35.576 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:13.339333+0000", 2023-09-23T21:12:35.576 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.913e-06 2023-09-23T21:12:35.576 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:35.576 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:35.576 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:12:35.576 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:13.339388+0000", 2023-09-23T21:12:35.576 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.4833000000000002e-05 2023-09-23T21:12:35.576 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:35.576 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:35.576 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:12:35.576 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:13.339397+0000", 2023-09-23T21:12:35.577 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.1509999999999993e-06 2023-09-23T21:12:35.577 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:35.577 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:35.577 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:12:35.577 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:14.241184+0000", 2023-09-23T21:12:35.577 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.90178749899999999 2023-09-23T21:12:35.577 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:35.577 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:35.577 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:12:35.577 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:14.241335+0000", 2023-09-23T21:12:35.577 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00015069700000000001 2023-09-23T21:12:35.577 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:35.578 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:35.578 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 2,7", 2023-09-23T21:12:35.578 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:14.241640+0000", 2023-09-23T21:12:35.578 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00030523300000000002 2023-09-23T21:12:35.578 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:35.578 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:35.578 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:12:35.578 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:14.248356+0000", 2023-09-23T21:12:35.578 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0067158019999999999 2023-09-23T21:12:35.578 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:35.578 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:35.579 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:12:35.579 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:14.248758+0000", 2023-09-23T21:12:35.579 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00040140800000000002 2023-09-23T21:12:35.579 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:35.579 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:35.579 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:12:35.579 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:14.248825+0000", 2023-09-23T21:12:35.579 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.6898999999999998e-05 2023-09-23T21:12:35.579 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:35.579 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:35.579 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:12:35.579 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:14.248872+0000", 2023-09-23T21:12:35.580 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.7113000000000001e-05 2023-09-23T21:12:35.580 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:35.580 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:35.580 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:12:35.580 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:14.248909+0000", 2023-09-23T21:12:35.580 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.7397000000000001e-05 2023-09-23T21:12:35.580 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:12:35.580 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:12:35.580 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:12:35.580 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:35.580 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:35.580 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.35469.0:164261 8.12 8:4e1b18c7:::benchmark_data_smithi026_301269_object53504:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e307)", 2023-09-23T21:12:35.581 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:04:13.426681+0000", 2023-09-23T21:12:35.581 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 502.05042994799999, 2023-09-23T21:12:35.581 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.82230143600000005, 2023-09-23T21:12:35.581 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:12:35.581 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:12:35.581 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:12:35.581 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.35469", 2023-09-23T21:12:35.581 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/4041424623", 2023-09-23T21:12:35.581 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 164261 2023-09-23T21:12:35.581 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:35.581 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:12:35.581 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:35.582 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:12:35.582 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:13.426681+0000", 2023-09-23T21:12:35.582 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:12:35.582 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:35.582 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:35.582 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:12:35.582 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:13.426681+0000", 2023-09-23T21:12:35.582 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:12:35.582 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:35.582 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:35.582 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:12:35.582 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:13.426680+0000", 2023-09-23T21:12:35.582 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999995 2023-09-23T21:12:35.583 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:35.583 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:35.583 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:12:35.583 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:13.426684+0000", 2023-09-23T21:12:35.583 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.693e-06 2023-09-23T21:12:35.583 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:35.583 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:35.583 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:12:35.583 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:13.426691+0000", 2023-09-23T21:12:35.583 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.576e-06 2023-09-23T21:12:35.583 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:35.583 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:35.584 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:12:35.584 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:13.426696+0000", 2023-09-23T21:12:35.584 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.8799999999999999e-06 2023-09-23T21:12:35.584 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:35.584 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:35.584 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:12:35.584 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:13.428620+0000", 2023-09-23T21:12:35.584 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.001923545 2023-09-23T21:12:35.584 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:35.584 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:35.584 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:12:35.584 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:13.428628+0000", 2023-09-23T21:12:35.584 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.106e-06 2023-09-23T21:12:35.585 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:35.585 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:35.585 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:12:35.585 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:14.242955+0000", 2023-09-23T21:12:35.585 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.81432692500000003 2023-09-23T21:12:35.585 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:35.585 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:35.585 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:12:35.585 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:14.243064+0000", 2023-09-23T21:12:35.585 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000109474 2023-09-23T21:12:35.585 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:35.585 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:35.586 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 2,7", 2023-09-23T21:12:35.586 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:14.243268+0000", 2023-09-23T21:12:35.586 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000203947 2023-09-23T21:12:35.586 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:35.586 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:35.586 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:12:35.586 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:14.248363+0000", 2023-09-23T21:12:35.586 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0050952489999999996 2023-09-23T21:12:35.586 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:35.586 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:35.586 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:12:35.586 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:14.248792+0000", 2023-09-23T21:12:35.587 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000428837 2023-09-23T21:12:35.587 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:35.587 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:35.587 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:12:35.587 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:14.248936+0000", 2023-09-23T21:12:35.587 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00014367899999999999 2023-09-23T21:12:35.587 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:35.587 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:35.587 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:12:35.587 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:14.248962+0000", 2023-09-23T21:12:35.587 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.6135999999999998e-05 2023-09-23T21:12:35.587 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:35.588 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:35.588 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:12:35.588 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:14.248982+0000", 2023-09-23T21:12:35.588 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.9938000000000001e-05 2023-09-23T21:12:35.588 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:12:35.588 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:12:35.588 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:12:35.588 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:35.588 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:35.588 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.35469.0:164284 8.12 8:4e7b7a22:::benchmark_data_smithi026_301269_object53527:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e307)", 2023-09-23T21:12:35.588 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:04:13.453160+0000", 2023-09-23T21:12:35.588 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 502.02395077800003, 2023-09-23T21:12:35.588 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.79592301499999996, 2023-09-23T21:12:35.589 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:12:35.589 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:12:35.589 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:12:35.589 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.35469", 2023-09-23T21:12:35.589 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/4041424623", 2023-09-23T21:12:35.589 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 164284 2023-09-23T21:12:35.589 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:35.589 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:12:35.589 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:35.589 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:12:35.589 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:13.453160+0000", 2023-09-23T21:12:35.589 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:12:35.590 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:35.590 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:35.590 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:12:35.590 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:13.453160+0000", 2023-09-23T21:12:35.590 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:12:35.590 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:35.590 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:35.590 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:12:35.590 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:13.453159+0000", 2023-09-23T21:12:35.590 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.999999 2023-09-23T21:12:35.590 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:35.590 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:35.591 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:12:35.591 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:13.453164+0000", 2023-09-23T21:12:35.591 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.2959999999999998e-06 2023-09-23T21:12:35.591 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:35.591 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:35.591 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:12:35.591 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:13.453176+0000", 2023-09-23T21:12:35.591 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.2211999999999999e-05 2023-09-23T21:12:35.591 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:35.591 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:35.591 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:12:35.591 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:13.453183+0000", 2023-09-23T21:12:35.591 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.686e-06 2023-09-23T21:12:35.592 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:35.592 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:35.592 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:12:35.592 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:13.453256+0000", 2023-09-23T21:12:35.592 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.2648e-05 2023-09-23T21:12:35.592 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:35.592 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:35.592 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:12:35.592 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:13.453263+0000", 2023-09-23T21:12:35.592 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.6680000000000001e-06 2023-09-23T21:12:35.592 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:35.592 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:35.592 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:12:35.593 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:14.244530+0000", 2023-09-23T21:12:35.593 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.79126670600000004 2023-09-23T21:12:35.593 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:35.593 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:35.593 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:12:35.593 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:14.244618+0000", 2023-09-23T21:12:35.593 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.8208000000000007e-05 2023-09-23T21:12:35.593 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:35.593 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:35.593 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 2,7", 2023-09-23T21:12:35.593 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:14.244775+0000", 2023-09-23T21:12:35.593 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000157114 2023-09-23T21:12:35.593 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:35.594 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:35.594 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:12:35.594 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:14.248447+0000", 2023-09-23T21:12:35.594 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.003671091 2023-09-23T21:12:35.594 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:35.594 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:35.594 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:12:35.594 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:14.249016+0000", 2023-09-23T21:12:35.594 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00056920099999999997 2023-09-23T21:12:35.594 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:35.594 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:35.594 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:12:35.595 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:14.249037+0000", 2023-09-23T21:12:35.595 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.0795e-05 2023-09-23T21:12:35.595 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:35.595 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:35.595 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:12:35.595 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:14.249063+0000", 2023-09-23T21:12:35.595 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.6128000000000001e-05 2023-09-23T21:12:35.595 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:35.595 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:35.595 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:12:35.595 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:14.249083+0000", 2023-09-23T21:12:35.595 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.0047999999999998e-05 2023-09-23T21:12:35.596 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:12:35.596 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:12:35.596 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:12:35.596 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:35.596 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:35.596 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.35469.0:164316 8.12 8:4f1a1a72:::benchmark_data_smithi026_301269_object53559:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e307)", 2023-09-23T21:12:35.596 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:04:13.490488+0000", 2023-09-23T21:12:35.596 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 501.98662286299998, 2023-09-23T21:12:35.596 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.75886759699999995, 2023-09-23T21:12:35.596 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:12:35.596 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:12:35.596 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:12:35.596 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.35469", 2023-09-23T21:12:35.597 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/4041424623", 2023-09-23T21:12:35.597 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 164316 2023-09-23T21:12:35.597 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:35.597 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:12:35.597 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:35.597 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:12:35.597 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:13.490488+0000", 2023-09-23T21:12:35.597 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:12:35.597 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:35.597 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:35.597 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:12:35.597 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:13.490488+0000", 2023-09-23T21:12:35.598 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:12:35.598 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:35.598 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:35.598 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:12:35.598 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:13.490486+0000", 2023-09-23T21:12:35.598 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.999999 2023-09-23T21:12:35.598 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:35.598 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:35.598 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:12:35.598 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:13.490492+0000", 2023-09-23T21:12:35.598 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.2900000000000002e-06 2023-09-23T21:12:35.598 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:35.598 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:35.599 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:12:35.599 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:13.490503+0000", 2023-09-23T21:12:35.599 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.1691e-05 2023-09-23T21:12:35.599 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:35.599 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:35.599 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:12:35.599 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:13.490508+0000", 2023-09-23T21:12:35.599 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.9849999999999997e-06 2023-09-23T21:12:35.599 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:35.599 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:35.599 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:12:35.599 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:13.490565+0000", 2023-09-23T21:12:35.600 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.6393e-05 2023-09-23T21:12:35.600 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:35.600 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:35.600 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:12:35.600 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:13.490575+0000", 2023-09-23T21:12:35.600 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.0541e-05 2023-09-23T21:12:35.600 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:35.600 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:35.600 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:12:35.600 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:14.245708+0000", 2023-09-23T21:12:35.600 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.75513238800000004 2023-09-23T21:12:35.600 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:35.601 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:35.601 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:12:35.601 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:14.245798+0000", 2023-09-23T21:12:35.601 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.0285999999999996e-05 2023-09-23T21:12:35.601 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:35.601 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:35.601 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 2,7", 2023-09-23T21:12:35.601 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:14.245936+0000", 2023-09-23T21:12:35.601 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00013843699999999999 2023-09-23T21:12:35.601 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:35.601 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:35.601 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:12:35.601 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:14.248455+0000", 2023-09-23T21:12:35.602 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0025187590000000002 2023-09-23T21:12:35.602 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:35.602 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:35.602 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:12:35.602 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:14.249200+0000", 2023-09-23T21:12:35.602 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00074436399999999996 2023-09-23T21:12:35.602 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:35.602 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:35.602 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:12:35.602 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:14.249278+0000", 2023-09-23T21:12:35.602 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.8651000000000003e-05 2023-09-23T21:12:35.602 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:35.603 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:35.603 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:12:35.603 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:14.249321+0000", 2023-09-23T21:12:35.603 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.3012000000000001e-05 2023-09-23T21:12:35.603 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:35.603 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:35.603 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:12:35.603 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:14.249355+0000", 2023-09-23T21:12:35.603 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.3973000000000002e-05 2023-09-23T21:12:35.603 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:12:35.603 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:12:35.603 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:12:35.603 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:35.604 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:35.604 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.35469.0:164325 8.12 8:49f268c0:::benchmark_data_smithi026_301269_object53568:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e307)", 2023-09-23T21:12:35.604 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:04:13.512173+0000", 2023-09-23T21:12:35.604 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 501.96493771199999, 2023-09-23T21:12:35.604 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.73732217, 2023-09-23T21:12:35.604 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:12:35.604 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:12:35.604 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:12:35.604 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.35469", 2023-09-23T21:12:35.604 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/4041424623", 2023-09-23T21:12:35.604 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 164325 2023-09-23T21:12:35.604 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:35.605 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:12:35.605 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:35.605 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:12:35.605 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:13.512173+0000", 2023-09-23T21:12:35.605 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:12:35.605 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:35.605 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:35.605 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:12:35.605 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:13.512173+0000", 2023-09-23T21:12:35.605 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:12:35.605 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:35.605 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:35.606 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:12:35.606 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:13.512171+0000", 2023-09-23T21:12:35.606 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T21:12:35.606 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:35.606 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:35.606 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:12:35.606 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:13.512181+0000", 2023-09-23T21:12:35.606 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.9669999999999996e-06 2023-09-23T21:12:35.606 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:35.606 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:35.606 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:12:35.606 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:13.512200+0000", 2023-09-23T21:12:35.607 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.8493000000000001e-05 2023-09-23T21:12:35.607 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:35.607 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:35.607 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:12:35.607 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:13.512207+0000", 2023-09-23T21:12:35.607 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.798e-06 2023-09-23T21:12:35.607 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:35.607 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:35.607 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:12:35.607 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:13.512264+0000", 2023-09-23T21:12:35.607 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.7809000000000002e-05 2023-09-23T21:12:35.607 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:35.607 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:35.608 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:12:35.608 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:13.512273+0000", 2023-09-23T21:12:35.608 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.6090000000000001e-06 2023-09-23T21:12:35.608 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:35.608 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:35.608 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:12:35.608 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:14.247094+0000", 2023-09-23T21:12:35.608 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.73482105499999995 2023-09-23T21:12:35.608 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:35.608 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:35.608 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:12:35.608 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:14.247205+0000", 2023-09-23T21:12:35.609 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000110849 2023-09-23T21:12:35.609 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:35.609 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:35.609 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 2,7", 2023-09-23T21:12:35.609 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:14.247437+0000", 2023-09-23T21:12:35.609 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00023193199999999999 2023-09-23T21:12:35.609 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:35.609 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:35.609 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:12:35.609 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:14.248483+0000", 2023-09-23T21:12:35.609 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0010465509999999999 2023-09-23T21:12:35.609 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:35.609 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:35.610 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:12:35.610 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:14.249249+0000", 2023-09-23T21:12:35.610 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00076557199999999995 2023-09-23T21:12:35.610 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:35.610 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:35.610 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:12:35.610 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:14.249416+0000", 2023-09-23T21:12:35.610 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00016682200000000001 2023-09-23T21:12:35.610 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:35.610 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:35.610 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:12:35.610 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:14.249458+0000", 2023-09-23T21:12:35.611 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.2305999999999998e-05 2023-09-23T21:12:35.611 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:35.611 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:35.611 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:12:35.611 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:04:14.249495+0000", 2023-09-23T21:12:35.611 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.6837e-05 2023-09-23T21:12:35.611 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:12:35.611 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:12:35.611 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:12:35.611 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:35.611 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:35.611 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.35469.0:212413 8.15 8:adc28bc3:::benchmark_data_smithi026_301269_object101656:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e339)", 2023-09-23T21:12:35.612 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:05:21.284869+0000", 2023-09-23T21:12:35.612 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 434.19224161900001, 2023-09-23T21:12:35.612 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.019788304, 2023-09-23T21:12:35.612 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:12:35.612 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:12:35.612 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:12:35.612 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.35469", 2023-09-23T21:12:35.612 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/4041424623", 2023-09-23T21:12:35.612 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 212413 2023-09-23T21:12:35.612 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:35.612 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:12:35.612 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:35.612 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:12:35.613 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:21.284869+0000", 2023-09-23T21:12:35.613 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:12:35.613 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:35.613 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:35.613 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:12:35.613 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:21.284869+0000", 2023-09-23T21:12:35.613 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:12:35.613 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:35.613 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:35.613 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:12:35.613 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:21.284867+0000", 2023-09-23T21:12:35.613 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T21:12:35.614 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:35.614 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:35.614 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:12:35.614 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:21.284875+0000", 2023-09-23T21:12:35.614 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.8310000000000001e-06 2023-09-23T21:12:35.614 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:35.614 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:35.614 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:12:35.614 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:21.284893+0000", 2023-09-23T21:12:35.614 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.7419999999999999e-05 2023-09-23T21:12:35.614 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:35.614 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:35.615 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:12:35.615 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:21.284900+0000", 2023-09-23T21:12:35.615 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.5020000000000003e-06 2023-09-23T21:12:35.615 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:35.615 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:35.615 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:12:35.615 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:21.284949+0000", 2023-09-23T21:12:35.615 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.9403000000000001e-05 2023-09-23T21:12:35.615 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:35.615 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:35.615 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:12:35.615 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:21.284959+0000", 2023-09-23T21:12:35.615 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.1759999999999999e-06 2023-09-23T21:12:35.616 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:35.616 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:35.616 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:12:35.616 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:22.297832+0000", 2023-09-23T21:12:35.616 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.012873634 2023-09-23T21:12:35.616 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:35.616 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:35.616 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:12:35.616 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:22.297968+0000", 2023-09-23T21:12:35.616 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000135928 2023-09-23T21:12:35.616 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:35.616 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:35.617 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 7", 2023-09-23T21:12:35.617 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:22.298508+0000", 2023-09-23T21:12:35.617 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00053965700000000003 2023-09-23T21:12:35.617 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:35.617 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:35.617 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:12:35.617 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:22.301926+0000", 2023-09-23T21:12:35.617 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0034177819999999999 2023-09-23T21:12:35.617 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:35.617 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:35.617 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:12:35.617 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:22.304483+0000", 2023-09-23T21:12:35.617 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0025570219999999999 2023-09-23T21:12:35.618 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:35.618 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:35.618 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:12:35.618 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:22.304548+0000", 2023-09-23T21:12:35.618 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.4927999999999999e-05 2023-09-23T21:12:35.618 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:12:35.618 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:12:35.618 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:12:35.618 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:22.304657+0000", 2023-09-23T21:12:35.618 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000109664 2023-09-23T21:12:35.618 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:12:35.618 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:12:35.619 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:12:35.619 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:12:35.619 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:12:35.619 INFO:teuthology.orchestra.run.smithi070.stdout:} 2023-09-23T21:12:35.619 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:12:35.650 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:12:35.717 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:12:35 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:12:35.718 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:12:35 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:12:35.718 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:12:35 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:12:35.718 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:12:35 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:12:35.811 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:12:35.814 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:12:35.814 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:12:35.872 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:12:35 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:12:35.872 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:12:35 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:12:35.915 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:12:35.956 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:12:36.025 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_ops_in_flight 2023-09-23T21:12:36.391 INFO:teuthology.orchestra.run.smithi070.stdout:{ 2023-09-23T21:12:36.391 INFO:teuthology.orchestra.run.smithi070.stdout: "success": "" 2023-09-23T21:12:36.392 INFO:teuthology.orchestra.run.smithi070.stdout:} 2023-09-23T21:12:36.771 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:12:36 smithi070 ceph-mon[168282]: pgmap v1225: 129 pgs: 19 active+clean+remapped, 1 active+recovery_wait+remapped, 4 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+remapped, 12 active+undersized, 1 active+recovering+undersized+degraded+remapped, 6 active+undersized+degraded, 10 active+recovery_wait+undersized+degraded+remapped, 5 active+recovery_wait+degraded+remapped, 69 active+clean; 4.3 GiB data, 16 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 310 op/s; 62283/211731 objects degraded (29.416%); 18012/211731 objects misplaced (8.507%); 10 MiB/s, 163 objects/s recovering 2023-09-23T21:12:36.771 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:12:36 smithi070 ceph-mon[168282]: Health check update: Degraded data redundancy: 62283/211731 objects degraded (29.416%), 27 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T21:12:36.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:12:36 smithi026 ceph-mon[175151]: pgmap v1225: 129 pgs: 19 active+clean+remapped, 1 active+recovery_wait+remapped, 4 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+remapped, 12 active+undersized, 1 active+recovering+undersized+degraded+remapped, 6 active+undersized+degraded, 10 active+recovery_wait+undersized+degraded+remapped, 5 active+recovery_wait+degraded+remapped, 69 active+clean; 4.3 GiB data, 16 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 310 op/s; 62283/211731 objects degraded (29.416%); 18012/211731 objects misplaced (8.507%); 10 MiB/s, 163 objects/s recovering 2023-09-23T21:12:36.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:12:36 smithi026 ceph-mon[175151]: Health check update: Degraded data redundancy: 62283/211731 objects degraded (29.416%), 27 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T21:12:36.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:12:36 smithi026 ceph-mon[179688]: pgmap v1225: 129 pgs: 19 active+clean+remapped, 1 active+recovery_wait+remapped, 4 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+remapped, 12 active+undersized, 1 active+recovering+undersized+degraded+remapped, 6 active+undersized+degraded, 10 active+recovery_wait+undersized+degraded+remapped, 5 active+recovery_wait+degraded+remapped, 69 active+clean; 4.3 GiB data, 16 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 310 op/s; 62283/211731 objects degraded (29.416%); 18012/211731 objects misplaced (8.507%); 10 MiB/s, 163 objects/s recovering 2023-09-23T21:12:36.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:12:36 smithi026 ceph-mon[179688]: Health check update: Degraded data redundancy: 62283/211731 objects degraded (29.416%), 27 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T21:12:37.643 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:12:37.647 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:12:37.648 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:12:37.748 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:12:37.794 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:12:37.837 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:12:37.843 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:12:37.844 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:12:37.893 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:12:37.945 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:12:37.977 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:12:38.030 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:12:38.039 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:12:38.039 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:12:38.140 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:12:38.172 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:12:38.580 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:12:38 smithi026 ceph-mon[175151]: pgmap v1226: 129 pgs: 19 active+clean+remapped, 1 active+recovery_wait+remapped, 4 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 12 active+undersized, 1 active+recovering+undersized+degraded+remapped, 6 active+undersized+degraded, 10 active+recovery_wait+undersized+degraded+remapped, 5 active+recovery_wait+degraded+remapped, 69 active+clean; 4.3 GiB data, 16 GiB used, 521 GiB / 536 GiB avail; 6.3 KiB/s rd, 17 KiB/s wr, 259 op/s; 62117/210513 objects degraded (29.507%); 17840/210513 objects misplaced (8.475%); 8.2 MiB/s, 131 objects/s recovering 2023-09-23T21:12:38.580 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:12:38 smithi026 ceph-mon[179688]: pgmap v1226: 129 pgs: 19 active+clean+remapped, 1 active+recovery_wait+remapped, 4 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 12 active+undersized, 1 active+recovering+undersized+degraded+remapped, 6 active+undersized+degraded, 10 active+recovery_wait+undersized+degraded+remapped, 5 active+recovery_wait+degraded+remapped, 69 active+clean; 4.3 GiB data, 16 GiB used, 521 GiB / 536 GiB avail; 6.3 KiB/s rd, 17 KiB/s wr, 259 op/s; 62117/210513 objects degraded (29.507%); 17840/210513 objects misplaced (8.475%); 8.2 MiB/s, 131 objects/s recovering 2023-09-23T21:12:38.580 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:12:38 smithi026 ceph-mon[179688]: osdmap e483: 8 total, 7 up, 6 in 2023-09-23T21:12:38.684 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:12:38.689 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:12:38.690 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:12:38.791 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:12:38.806 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:12:38.825 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:12:38.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:12:38 smithi070 ceph-mon[168282]: pgmap v1226: 129 pgs: 19 active+clean+remapped, 1 active+recovery_wait+remapped, 4 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 12 active+undersized, 1 active+recovering+undersized+degraded+remapped, 6 active+undersized+degraded, 10 active+recovery_wait+undersized+degraded+remapped, 5 active+recovery_wait+degraded+remapped, 69 active+clean; 4.3 GiB data, 16 GiB used, 521 GiB / 536 GiB avail; 6.3 KiB/s rd, 17 KiB/s wr, 259 op/s; 62117/210513 objects degraded (29.507%); 17840/210513 objects misplaced (8.475%); 8.2 MiB/s, 131 objects/s recovering 2023-09-23T21:12:38.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:12:38 smithi070 ceph-mon[168282]: osdmap e483: 8 total, 7 up, 6 in 2023-09-23T21:12:38.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:12:38 smithi070 ceph-mon[168282]: failed to encode map e483 with expected crc 2023-09-23T21:12:38.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:12:38 smithi070 ceph-mon[168282]: failed to encode map e483 with expected crc 2023-09-23T21:12:38.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:12:38 smithi070 ceph-mon[168282]: failed to encode map e483 with expected crc 2023-09-23T21:12:38.881 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:12:38 smithi026 ceph-mon[179688]: failed to encode map e483 with expected crc 2023-09-23T21:12:38.881 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:12:38 smithi026 ceph-mon[179688]: failed to encode map e483 with expected crc 2023-09-23T21:12:38.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:12:38 smithi026 ceph-mon[179688]: failed to encode map e483 with expected crc 2023-09-23T21:12:38.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:12:38 smithi026 ceph-mon[175151]: osdmap e483: 8 total, 7 up, 6 in 2023-09-23T21:12:38.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:12:38 smithi026 ceph-mon[175151]: failed to encode map e483 with expected crc 2023-09-23T21:12:38.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:12:38 smithi026 ceph-mon[175151]: failed to encode map e483 with expected crc 2023-09-23T21:12:38.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:12:38 smithi026 ceph-mon[175151]: failed to encode map e483 with expected crc 2023-09-23T21:12:39.750 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1: {} 2023-09-23T21:12:39.750 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2: {} 2023-09-23T21:12:39.777 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3: {} 2023-09-23T21:12:39.834 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4: {} 2023-09-23T21:12:39.859 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T21:12:39.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:12:39 smithi070 ceph-mon[168282]: failed to encode map e484 with expected crc 2023-09-23T21:12:39.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:12:39 smithi070 ceph-mon[168282]: failed to encode map e484 with expected crc 2023-09-23T21:12:39.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:12:39 smithi070 ceph-mon[168282]: failed to encode map e484 with expected crc 2023-09-23T21:12:39.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:12:39 smithi070 ceph-mon[168282]: osdmap e484: 8 total, 7 up, 6 in 2023-09-23T21:12:39.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:12:39 smithi070 ceph-mon[168282]: failed to encode map e484 with expected crc 2023-09-23T21:12:39.882 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 21:12:39 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[164029]: ::ffff:172.21.15.70 - - [23/Sep/2023:21:12:39] "GET /metrics HTTP/1.1" 200 36445 "" "Prometheus/2.43.0" 2023-09-23T21:12:39.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:12:39 smithi026 ceph-mon[179688]: failed to encode map e484 with expected crc 2023-09-23T21:12:39.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:12:39 smithi026 ceph-mon[179688]: failed to encode map e484 with expected crc 2023-09-23T21:12:39.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:12:39 smithi026 ceph-mon[179688]: failed to encode map e484 with expected crc 2023-09-23T21:12:39.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:12:39 smithi026 ceph-mon[179688]: osdmap e484: 8 total, 7 up, 6 in 2023-09-23T21:12:39.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:12:39 smithi026 ceph-mon[179688]: failed to encode map e484 with expected crc 2023-09-23T21:12:39.883 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:12:39 smithi026 ceph-mon[175151]: failed to encode map e484 with expected crc 2023-09-23T21:12:39.883 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:12:39 smithi026 ceph-mon[175151]: failed to encode map e484 with expected crc 2023-09-23T21:12:39.883 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:12:39 smithi026 ceph-mon[175151]: failed to encode map e484 with expected crc 2023-09-23T21:12:39.883 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:12:39 smithi026 ceph-mon[175151]: osdmap e484: 8 total, 7 up, 6 in 2023-09-23T21:12:39.883 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:12:39 smithi026 ceph-mon[175151]: failed to encode map e484 with expected crc 2023-09-23T21:12:39.884 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6: {} 2023-09-23T21:12:39.908 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T21:12:39.909 INFO:teuthology.orchestra.run.smithi026.stderr:Error ENXIO: problem getting command descriptions from osd.0 2023-09-23T21:12:39.909 INFO:teuthology.orchestra.run.smithi026.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T21:12:39.909 INFO:teuthology.orchestra.run.smithi026.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T21:12:39.909 INFO:teuthology.orchestra.run.smithi026.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T21:12:39.909 INFO:teuthology.orchestra.run.smithi026.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T21:12:39.909 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T21:12:39.909 INFO:teuthology.orchestra.run.smithi026.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T21:12:39.909 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T21:12:40.038 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:12:40.041 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:12:40.042 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:12:40.142 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:12:40.189 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:12:40.251 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:12:40.258 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:12:40.259 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:12:40.360 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:12:40.411 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:12:40.456 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:12:40.462 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:12:40.463 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:12:40.563 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:12:40.635 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:12:40.682 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:12:40.686 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:12:40.687 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:12:40.787 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:12:40.833 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:12:40.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:12:40 smithi070 ceph-mon[168282]: pgmap v1229: 129 pgs: 19 active+clean+remapped, 1 active+recovery_wait+remapped, 4 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 12 active+undersized, 1 active+recovering+undersized+degraded+remapped, 6 active+undersized+degraded, 10 active+recovery_wait+undersized+degraded+remapped, 5 active+recovery_wait+degraded+remapped, 69 active+clean; 4.3 GiB data, 16 GiB used, 521 GiB / 536 GiB avail; 5.8 KiB/s rd, 15 KiB/s wr, 212 op/s; 61849/209841 objects degraded (29.474%); 17686/209841 objects misplaced (8.428%); 7.0 MiB/s, 112 objects/s recovering 2023-09-23T21:12:40.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:12:40 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/3181257930' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:12:40.881 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:12:40 smithi026 ceph-mon[175151]: pgmap v1229: 129 pgs: 19 active+clean+remapped, 1 active+recovery_wait+remapped, 4 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 12 active+undersized, 1 active+recovering+undersized+degraded+remapped, 6 active+undersized+degraded, 10 active+recovery_wait+undersized+degraded+remapped, 5 active+recovery_wait+degraded+remapped, 69 active+clean; 4.3 GiB data, 16 GiB used, 521 GiB / 536 GiB avail; 5.8 KiB/s rd, 15 KiB/s wr, 212 op/s; 61849/209841 objects degraded (29.474%); 17686/209841 objects misplaced (8.428%); 7.0 MiB/s, 112 objects/s recovering 2023-09-23T21:12:40.881 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:12:40 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/3181257930' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:12:40.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:12:40 smithi026 ceph-mon[179688]: pgmap v1229: 129 pgs: 19 active+clean+remapped, 1 active+recovery_wait+remapped, 4 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 12 active+undersized, 1 active+recovering+undersized+degraded+remapped, 6 active+undersized+degraded, 10 active+recovery_wait+undersized+degraded+remapped, 5 active+recovery_wait+degraded+remapped, 69 active+clean; 4.3 GiB data, 16 GiB used, 521 GiB / 536 GiB avail; 5.8 KiB/s rd, 15 KiB/s wr, 212 op/s; 61849/209841 objects degraded (29.474%); 17686/209841 objects misplaced (8.428%); 7.0 MiB/s, 112 objects/s recovering 2023-09-23T21:12:40.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:12:40 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/3181257930' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:12:41.585 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:12:41 smithi026 ceph-mon[175151]: Health check update: Degraded data redundancy: 61849/209841 objects degraded (29.474%), 27 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T21:12:41.586 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:12:41 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:12:41.586 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:12:41 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:12:41.802 INFO:tasks.thrashosds.thrasher:in_osds: [0, 2, 3, 4, 5, 7] out_osds: [6, 1] dead_osds: [0] live_osds: [2, 3, 4, 5, 7, 6, 1] 2023-09-23T21:12:41.802 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T21:12:41.802 INFO:tasks.thrashosds.thrasher:inject_pause on osd.2 2023-09-23T21:12:41.802 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-09-23T21:12:41.803 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-09-23T21:12:41.803 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 config set filestore_inject_stall 3 2023-09-23T21:12:41.835 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:12:41 smithi026 ceph-mon[179688]: Health check update: Degraded data redundancy: 61849/209841 objects degraded (29.474%), 27 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T21:12:41.835 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:12:41 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:12:41.836 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:12:41 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:12:41.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:12:41 smithi070 ceph-mon[168282]: Health check update: Degraded data redundancy: 61849/209841 objects degraded (29.474%), 27 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T21:12:41.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:12:41 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:12:41.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:12:41 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:12:42.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:12:42 smithi070 ceph-mon[168282]: pgmap v1230: 129 pgs: 19 active+clean+remapped, 1 active+recovery_wait+remapped, 4 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 12 active+undersized, 2 active+recovering+undersized+degraded+remapped, 6 active+undersized+degraded, 9 active+recovery_wait+undersized+degraded+remapped, 5 active+recovery_wait+degraded+remapped, 70 active+clean; 4.2 GiB data, 16 GiB used, 521 GiB / 536 GiB avail; 5.2 KiB/s rd, 14 KiB/s wr, 267 op/s; 59522/207696 objects degraded (28.658%); 18924/207696 objects misplaced (9.111%); 11 MiB/s, 176 objects/s recovering 2023-09-23T21:12:42.881 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:12:42 smithi026 ceph-mon[175151]: pgmap v1230: 129 pgs: 19 active+clean+remapped, 1 active+recovery_wait+remapped, 4 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 12 active+undersized, 2 active+recovering+undersized+degraded+remapped, 6 active+undersized+degraded, 9 active+recovery_wait+undersized+degraded+remapped, 5 active+recovery_wait+degraded+remapped, 70 active+clean; 4.2 GiB data, 16 GiB used, 521 GiB / 536 GiB avail; 5.2 KiB/s rd, 14 KiB/s wr, 267 op/s; 59522/207696 objects degraded (28.658%); 18924/207696 objects misplaced (9.111%); 11 MiB/s, 176 objects/s recovering 2023-09-23T21:12:42.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:12:42 smithi026 ceph-mon[179688]: pgmap v1230: 129 pgs: 19 active+clean+remapped, 1 active+recovery_wait+remapped, 4 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 12 active+undersized, 2 active+recovering+undersized+degraded+remapped, 6 active+undersized+degraded, 9 active+recovery_wait+undersized+degraded+remapped, 5 active+recovery_wait+degraded+remapped, 70 active+clean; 4.2 GiB data, 16 GiB used, 521 GiB / 536 GiB avail; 5.2 KiB/s rd, 14 KiB/s wr, 267 op/s; 59522/207696 objects degraded (28.658%); 18924/207696 objects misplaced (9.111%); 11 MiB/s, 176 objects/s recovering 2023-09-23T21:12:43.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:12:43 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T21:12:43.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:12:43 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T21:12:43.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:12:43 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:12:43.881 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:12:43 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T21:12:43.881 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:12:43 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T21:12:43.881 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:12:43 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:12:43.881 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:12:43 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T21:12:43.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:12:43 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T21:12:43.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:12:43 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:12:44.346 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:12:44.352 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:12:44.353 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:12:44.453 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:12:44.506 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:12:44.569 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:12:44.575 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:12:44.575 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:12:44.588 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T21:12:44.588 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T21:12:44.676 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:12:44.725 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:12:44.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:12:44 smithi070 ceph-mon[168282]: pgmap v1231: 129 pgs: 19 active+clean+remapped, 1 active+recovery_wait+remapped, 4 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 12 active+undersized, 2 active+recovering+undersized+degraded+remapped, 6 active+undersized+degraded, 9 active+recovery_wait+undersized+degraded+remapped, 5 active+recovery_wait+degraded+remapped, 70 active+clean; 4.2 GiB data, 16 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 227 op/s; 59173/206268 objects degraded (28.687%); 18924/206268 objects misplaced (9.174%); 8.2 MiB/s, 130 objects/s recovering 2023-09-23T21:12:44.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:12:44 smithi026 ceph-mon[179688]: pgmap v1231: 129 pgs: 19 active+clean+remapped, 1 active+recovery_wait+remapped, 4 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 12 active+undersized, 2 active+recovering+undersized+degraded+remapped, 6 active+undersized+degraded, 9 active+recovery_wait+undersized+degraded+remapped, 5 active+recovery_wait+degraded+remapped, 70 active+clean; 4.2 GiB data, 16 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 227 op/s; 59173/206268 objects degraded (28.687%); 18924/206268 objects misplaced (9.174%); 8.2 MiB/s, 130 objects/s recovering 2023-09-23T21:12:44.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:12:44 smithi026 ceph-mon[175151]: pgmap v1231: 129 pgs: 19 active+clean+remapped, 1 active+recovery_wait+remapped, 4 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 12 active+undersized, 2 active+recovering+undersized+degraded+remapped, 6 active+undersized+degraded, 9 active+recovery_wait+undersized+degraded+remapped, 5 active+recovery_wait+degraded+remapped, 70 active+clean; 4.2 GiB data, 16 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 227 op/s; 59173/206268 objects degraded (28.687%); 18924/206268 objects misplaced (9.174%); 8.2 MiB/s, 130 objects/s recovering 2023-09-23T21:12:44.922 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:12:44.925 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:12:44.926 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:12:45.027 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:12:45.067 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:12:45.127 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:12:45.137 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:12:45.138 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:12:45.170 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:12:45.239 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:12:45.302 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:12:45.384 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:12:45.393 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:12:45.393 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:12:45.494 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:12:45.531 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:12:45.591 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:12:45.599 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:12:45.600 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:12:45.624 INFO:teuthology.orchestra.run.smithi026.stderr:noscrub is set 2023-09-23T21:12:45.700 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:12:45.751 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:12:45 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/3969214471' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T21:12:45.755 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:12:45.806 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:12:45.812 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:12:45.812 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:12:45.881 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:12:45 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/3969214471' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T21:12:45.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:12:45 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/3969214471' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T21:12:45.913 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:12:45.918 INFO:teuthology.orchestra.run.smithi026.stdout:ERROR: (22) Invalid argument 2023-09-23T21:12:45.918 INFO:teuthology.orchestra.run.smithi026.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T21:12:45.951 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:12:46.590 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T21:12:46.633 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:12:46 smithi026 ceph-mon[179688]: pgmap v1232: 129 pgs: 19 active+clean+remapped, 1 active+recovery_wait+remapped, 4 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 12 active+undersized, 2 active+recovering+undersized+degraded+remapped, 6 active+undersized+degraded, 9 active+recovery_wait+undersized+degraded+remapped, 5 active+recovery_wait+degraded+remapped, 70 active+clean; 4.1 GiB data, 16 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 299 op/s; 58395/203322 objects degraded (28.720%); 17943/203322 objects misplaced (8.825%); 15 MiB/s, 245 objects/s recovering 2023-09-23T21:12:46.633 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:12:46 smithi026 ceph-mon[179688]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:12:46.633 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:12:46 smithi026 ceph-mon[179688]: failed to encode map e485 with expected crc 2023-09-23T21:12:46.633 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:12:46 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/3969214471' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T21:12:46.634 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:12:46 smithi026 ceph-mon[179688]: osdmap e485: 8 total, 7 up, 6 in 2023-09-23T21:12:46.634 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:12:46 smithi026 ceph-mon[179688]: failed to encode map e485 with expected crc 2023-09-23T21:12:46.634 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:12:46 smithi026 ceph-mon[179688]: failed to encode map e485 with expected crc 2023-09-23T21:12:46.634 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:12:46 smithi026 ceph-mon[179688]: failed to encode map e485 with expected crc 2023-09-23T21:12:46.634 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:12:46 smithi026 ceph-mon[179688]: failed to encode map e485 with expected crc 2023-09-23T21:12:46.634 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:12:46 smithi026 ceph-mon[179688]: Health check update: Degraded data redundancy: 58395/203322 objects degraded (28.720%), 27 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T21:12:46.634 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:12:46 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:12:46.634 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:12:46 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:12:46.635 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:12:46 smithi026 ceph-mon[175151]: pgmap v1232: 129 pgs: 19 active+clean+remapped, 1 active+recovery_wait+remapped, 4 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 12 active+undersized, 2 active+recovering+undersized+degraded+remapped, 6 active+undersized+degraded, 9 active+recovery_wait+undersized+degraded+remapped, 5 active+recovery_wait+degraded+remapped, 70 active+clean; 4.1 GiB data, 16 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 299 op/s; 58395/203322 objects degraded (28.720%); 17943/203322 objects misplaced (8.825%); 15 MiB/s, 245 objects/s recovering 2023-09-23T21:12:46.635 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:12:46 smithi026 ceph-mon[175151]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:12:46.635 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:12:46 smithi026 ceph-mon[175151]: failed to encode map e485 with expected crc 2023-09-23T21:12:46.635 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:12:46 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/3969214471' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T21:12:46.635 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:12:46 smithi026 ceph-mon[175151]: osdmap e485: 8 total, 7 up, 6 in 2023-09-23T21:12:46.635 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:12:46 smithi026 ceph-mon[175151]: failed to encode map e485 with expected crc 2023-09-23T21:12:46.635 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:12:46 smithi026 ceph-mon[175151]: failed to encode map e485 with expected crc 2023-09-23T21:12:46.635 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:12:46 smithi026 ceph-mon[175151]: failed to encode map e485 with expected crc 2023-09-23T21:12:46.635 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:12:46 smithi026 ceph-mon[175151]: failed to encode map e485 with expected crc 2023-09-23T21:12:46.635 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:12:46 smithi026 ceph-mon[175151]: Health check update: Degraded data redundancy: 58395/203322 objects degraded (28.720%), 27 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T21:12:46.636 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:12:46 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:12:46.636 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:12:46 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:12:46.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:12:46 smithi070 ceph-mon[168282]: pgmap v1232: 129 pgs: 19 active+clean+remapped, 1 active+recovery_wait+remapped, 4 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 12 active+undersized, 2 active+recovering+undersized+degraded+remapped, 6 active+undersized+degraded, 9 active+recovery_wait+undersized+degraded+remapped, 5 active+recovery_wait+degraded+remapped, 70 active+clean; 4.1 GiB data, 16 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 299 op/s; 58395/203322 objects degraded (28.720%); 17943/203322 objects misplaced (8.825%); 15 MiB/s, 245 objects/s recovering 2023-09-23T21:12:46.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:12:46 smithi070 ceph-mon[168282]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:12:46.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:12:46 smithi070 ceph-mon[168282]: failed to encode map e485 with expected crc 2023-09-23T21:12:46.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:12:46 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/3969214471' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T21:12:46.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:12:46 smithi070 ceph-mon[168282]: osdmap e485: 8 total, 7 up, 6 in 2023-09-23T21:12:46.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:12:46 smithi070 ceph-mon[168282]: failed to encode map e485 with expected crc 2023-09-23T21:12:46.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:12:46 smithi070 ceph-mon[168282]: failed to encode map e485 with expected crc 2023-09-23T21:12:46.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:12:46 smithi070 ceph-mon[168282]: failed to encode map e485 with expected crc 2023-09-23T21:12:46.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:12:46 smithi070 ceph-mon[168282]: failed to encode map e485 with expected crc 2023-09-23T21:12:46.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:12:46 smithi070 ceph-mon[168282]: Health check update: Degraded data redundancy: 58395/203322 objects degraded (28.720%), 27 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T21:12:46.871 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:12:46 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:12:46.871 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:12:46 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:12:48.354 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:12:48.358 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:12:48.358 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:12:48.459 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:12:48.514 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:12:48.742 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:12:48.743 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_blocked_ops 2023-09-23T21:12:48.744 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:12:48 smithi026 ceph-mon[179688]: pgmap v1234: 129 pgs: 19 active+clean+remapped, 1 active+recovery_wait+remapped, 4 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 12 active+undersized, 2 active+recovering+undersized+degraded+remapped, 6 active+undersized+degraded, 9 active+recovery_wait+undersized+degraded+remapped, 5 active+recovery_wait+degraded+remapped, 70 active+clean; 4.1 GiB data, 16 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 306 op/s; 58247/202314 objects degraded (28.790%); 17943/202314 objects misplaced (8.869%); 14 MiB/s, 227 objects/s recovering 2023-09-23T21:12:48.744 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:12:48.744 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:12:48 smithi026 ceph-mon[175151]: pgmap v1234: 129 pgs: 19 active+clean+remapped, 1 active+recovery_wait+remapped, 4 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 12 active+undersized, 2 active+recovering+undersized+degraded+remapped, 6 active+undersized+degraded, 9 active+recovery_wait+undersized+degraded+remapped, 5 active+recovery_wait+degraded+remapped, 70 active+clean; 4.1 GiB data, 16 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 306 op/s; 58247/202314 objects degraded (28.790%); 17943/202314 objects misplaced (8.869%); 14 MiB/s, 227 objects/s recovering 2023-09-23T21:12:48.745 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:12:48.845 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:12:48.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:12:48 smithi070 ceph-mon[168282]: pgmap v1234: 129 pgs: 19 active+clean+remapped, 1 active+recovery_wait+remapped, 4 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 12 active+undersized, 2 active+recovering+undersized+degraded+remapped, 6 active+undersized+degraded, 9 active+recovery_wait+undersized+degraded+remapped, 5 active+recovery_wait+degraded+remapped, 70 active+clean; 4.1 GiB data, 16 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 306 op/s; 58247/202314 objects degraded (28.790%); 17943/202314 objects misplaced (8.869%); 14 MiB/s, 227 objects/s recovering 2023-09-23T21:12:48.898 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:12:48.952 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:12:48.956 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:12:48.957 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:12:49.058 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:12:49.095 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:12:49.149 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:12:49.154 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:12:49.154 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:12:49.169 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:12:49.255 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:12:49.287 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:12:49.332 INFO:teuthology.orchestra.run.smithi026.stdout:{ 2023-09-23T21:12:49.333 INFO:teuthology.orchestra.run.smithi026.stdout: "success": "filestore_inject_stall = '3' (not observed, change may require restart) " 2023-09-23T21:12:49.333 INFO:teuthology.orchestra.run.smithi026.stdout:} 2023-09-23T21:12:49.335 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:12:49.341 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:12:49.342 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:12:49.443 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:12:49.481 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:12:49.630 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:12:49.633 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:12:49.633 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:12:49.734 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:12:49.788 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:12:49.860 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:12:49.867 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:12:49.867 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:12:49.881 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 21:12:49 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[164029]: ::ffff:172.21.15.70 - - [23/Sep/2023:21:12:49] "GET /metrics HTTP/1.1" 200 36448 "" "Prometheus/2.43.0" 2023-09-23T21:12:49.924 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:12:49.967 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:12:50.059 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:12:50.139 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:12:50.142 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:12:50.142 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:12:50.243 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:12:50.279 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:12:50.388 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:12:50.392 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:12:50.393 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:12:50.493 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:12:50.576 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:12:50.637 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:12:50.640 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:12:50.640 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:12:50.741 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:12:50.793 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1: {} 2023-09-23T21:12:50.800 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:12:50.804 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:12:50 smithi026 ceph-mon[175151]: pgmap v1235: 129 pgs: 19 active+clean+remapped, 1 active+recovery_wait+remapped, 4 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 12 active+undersized, 2 active+recovering+undersized+degraded+remapped, 6 active+undersized+degraded, 9 active+recovery_wait+undersized+degraded+remapped, 5 active+recovery_wait+degraded+remapped, 70 active+clean; 4.1 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 273 op/s; 57972/201645 objects degraded (28.750%); 17769/201645 objects misplaced (8.812%); 12 MiB/s, 199 objects/s recovering 2023-09-23T21:12:50.804 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:12:50 smithi026 ceph-mon[179688]: pgmap v1235: 129 pgs: 19 active+clean+remapped, 1 active+recovery_wait+remapped, 4 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 12 active+undersized, 2 active+recovering+undersized+degraded+remapped, 6 active+undersized+degraded, 9 active+recovery_wait+undersized+degraded+remapped, 5 active+recovery_wait+degraded+remapped, 70 active+clean; 4.1 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 273 op/s; 57972/201645 objects degraded (28.750%); 17769/201645 objects misplaced (8.812%); 12 MiB/s, 199 objects/s recovering 2023-09-23T21:12:50.819 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2: {} 2023-09-23T21:12:50.840 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3: {} 2023-09-23T21:12:50.857 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:12:50.863 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4: {} 2023-09-23T21:12:50.866 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:12:50 smithi070 ceph-mon[168282]: pgmap v1235: 129 pgs: 19 active+clean+remapped, 1 active+recovery_wait+remapped, 4 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 12 active+undersized, 2 active+recovering+undersized+degraded+remapped, 6 active+undersized+degraded, 9 active+recovery_wait+undersized+degraded+remapped, 5 active+recovery_wait+degraded+remapped, 70 active+clean; 4.1 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 273 op/s; 57972/201645 objects degraded (28.750%); 17769/201645 objects misplaced (8.812%); 12 MiB/s, 199 objects/s recovering 2023-09-23T21:12:50.867 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:12:50.868 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:12:50.883 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T21:12:50.884 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T21:12:50.911 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6: {} 2023-09-23T21:12:50.936 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T21:12:50.936 INFO:teuthology.orchestra.run.smithi026.stderr:Error ENXIO: problem getting command descriptions from osd.0 2023-09-23T21:12:50.936 INFO:teuthology.orchestra.run.smithi026.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T21:12:50.936 INFO:teuthology.orchestra.run.smithi026.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T21:12:50.936 INFO:teuthology.orchestra.run.smithi026.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T21:12:50.937 INFO:teuthology.orchestra.run.smithi026.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T21:12:50.937 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T21:12:50.937 INFO:teuthology.orchestra.run.smithi026.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T21:12:50.937 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T21:12:50.968 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:12:51.011 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:12:51.076 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:12:51.081 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:12:51.082 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:12:51.183 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:12:51.215 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:12:51.368 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:12:51.373 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:12:51.373 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:12:51.384 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T21:12:51.384 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T21:12:51.474 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:12:51.523 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:12:51.619 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:12:51.628 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:12:51.629 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:12:51.730 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:12:51.786 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:12:51.793 INFO:teuthology.orchestra.run.smithi026.stdout:{ 2023-09-23T21:12:51.793 INFO:teuthology.orchestra.run.smithi026.stdout: "ops": [], 2023-09-23T21:12:51.793 INFO:teuthology.orchestra.run.smithi026.stdout: "complaint_time": 30, 2023-09-23T21:12:51.793 INFO:teuthology.orchestra.run.smithi026.stdout: "num_blocked_ops": 0 2023-09-23T21:12:51.793 INFO:teuthology.orchestra.run.smithi026.stdout:} 2023-09-23T21:12:51.826 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:12:51.829 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:12:51.830 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:12:51.850 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:12:51.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:12:51 smithi070 ceph-mon[168282]: Health check update: Degraded data redundancy: 57972/201645 objects degraded (28.750%), 27 pgs degraded, 24 pgs undersized (PG_DEGRADED) 2023-09-23T21:12:51.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:12:51 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/3443725626' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:12:51.931 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:12:51.979 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:12:52.022 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:12:51 smithi026 ceph-mon[175151]: Health check update: Degraded data redundancy: 57972/201645 objects degraded (28.750%), 27 pgs degraded, 24 pgs undersized (PG_DEGRADED) 2023-09-23T21:12:52.023 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:12:51 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/3443725626' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:12:52.023 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:12:51 smithi026 ceph-mon[179688]: Health check update: Degraded data redundancy: 57972/201645 objects degraded (28.750%), 27 pgs degraded, 24 pgs undersized (PG_DEGRADED) 2023-09-23T21:12:52.023 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:12:51 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/3443725626' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:12:52.046 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:12:52.060 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:12:52.060 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:12:52.161 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:12:52.245 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:12:52.296 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:12:52.300 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:12:52.300 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:12:52.401 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:12:52.443 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:12:52.553 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_historic_ops 2023-09-23T21:12:52.675 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:12:52.679 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:12:52.680 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:12:52.780 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:12:52.841 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:12:52 smithi026 ceph-mon[175151]: pgmap v1236: 129 pgs: 19 active+clean+remapped, 1 active+recovery_wait+remapped, 4 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 12 active+undersized, 2 active+recovering+undersized+degraded+remapped, 6 active+undersized+degraded, 9 active+recovery_wait+undersized+degraded+remapped, 5 active+recovery_wait+degraded+remapped, 70 active+clean; 4.1 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 280 op/s; 57365/199272 objects degraded (28.787%); 16714/199272 objects misplaced (8.388%); 16 MiB/s, 253 objects/s recovering 2023-09-23T21:12:52.842 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:12:52 smithi026 ceph-mon[175151]: failed to encode map e486 with expected crc 2023-09-23T21:12:52.842 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:12:52 smithi026 ceph-mon[175151]: failed to encode map e486 with expected crc 2023-09-23T21:12:52.842 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:12:52 smithi026 ceph-mon[175151]: osdmap e486: 8 total, 7 up, 6 in 2023-09-23T21:12:52.842 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:12:52 smithi026 ceph-mon[175151]: failed to encode map e486 with expected crc 2023-09-23T21:12:52.842 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:12:52 smithi026 ceph-mon[179688]: pgmap v1236: 129 pgs: 19 active+clean+remapped, 1 active+recovery_wait+remapped, 4 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 12 active+undersized, 2 active+recovering+undersized+degraded+remapped, 6 active+undersized+degraded, 9 active+recovery_wait+undersized+degraded+remapped, 5 active+recovery_wait+degraded+remapped, 70 active+clean; 4.1 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 280 op/s; 57365/199272 objects degraded (28.787%); 16714/199272 objects misplaced (8.388%); 16 MiB/s, 253 objects/s recovering 2023-09-23T21:12:52.842 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:12:52 smithi026 ceph-mon[179688]: failed to encode map e486 with expected crc 2023-09-23T21:12:52.843 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:12:52 smithi026 ceph-mon[179688]: failed to encode map e486 with expected crc 2023-09-23T21:12:52.843 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:12:52 smithi026 ceph-mon[179688]: osdmap e486: 8 total, 7 up, 6 in 2023-09-23T21:12:52.843 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:12:52 smithi026 ceph-mon[179688]: failed to encode map e486 with expected crc 2023-09-23T21:12:52.849 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:12:52.918 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:12:52.928 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:12:52.928 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:12:53.029 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:12:53.078 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:12:53.130 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:12:52 smithi070 ceph-mon[168282]: pgmap v1236: 129 pgs: 19 active+clean+remapped, 1 active+recovery_wait+remapped, 4 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 12 active+undersized, 2 active+recovering+undersized+degraded+remapped, 6 active+undersized+degraded, 9 active+recovery_wait+undersized+degraded+remapped, 5 active+recovery_wait+degraded+remapped, 70 active+clean; 4.1 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 280 op/s; 57365/199272 objects degraded (28.787%); 16714/199272 objects misplaced (8.388%); 16 MiB/s, 253 objects/s recovering 2023-09-23T21:12:53.130 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:12:52 smithi070 ceph-mon[168282]: failed to encode map e486 with expected crc 2023-09-23T21:12:53.130 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:12:52 smithi070 ceph-mon[168282]: failed to encode map e486 with expected crc 2023-09-23T21:12:53.130 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:12:52 smithi070 ceph-mon[168282]: osdmap e486: 8 total, 7 up, 6 in 2023-09-23T21:12:53.131 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:12:52 smithi070 ceph-mon[168282]: failed to encode map e486 with expected crc 2023-09-23T21:12:53.220 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:12:53.223 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:12:53.224 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:12:53.325 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:12:53.381 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:12:53.385 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T21:12:53.422 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:12:53.436 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:12:53.444 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:12:53.445 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:12:53.546 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:12:53.597 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:12:53.648 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:12:53.652 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:12:53.652 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:12:53.753 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:12:53.780 INFO:teuthology.orchestra.run.smithi026.stderr:nodeep-scrub is set 2023-09-23T21:12:53.792 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:12:53.986 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:12:53.988 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:12:53 smithi026 ceph-mon[179688]: failed to encode map e487 with expected crc 2023-09-23T21:12:53.988 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:12:53 smithi026 ceph-mon[179688]: failed to encode map e487 with expected crc 2023-09-23T21:12:53.988 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:12:53 smithi026 ceph-mon[179688]: osdmap e487: 8 total, 7 up, 6 in 2023-09-23T21:12:53.988 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:12:53 smithi026 ceph-mon[179688]: failed to encode map e487 with expected crc 2023-09-23T21:12:53.988 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:12:53 smithi026 ceph-mon[179688]: failed to encode map e487 with expected crc 2023-09-23T21:12:53.988 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:12:53 smithi026 ceph-mon[179688]: pgmap v1239: 129 pgs: 19 active+clean+remapped, 1 active+recovery_wait+remapped, 4 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 12 active+undersized, 3 active+recovering+undersized+degraded+remapped, 6 active+undersized+degraded, 8 active+recovery_wait+undersized+degraded+remapped, 5 active+recovery_wait+degraded+remapped, 70 active+clean; 4.0 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 249 op/s; 55738/197775 objects degraded (28.183%); 17854/197775 objects misplaced (9.027%); 13 MiB/s, 204 objects/s recovering 2023-09-23T21:12:53.988 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:12:53 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/1089111062' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:12:53.989 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:12:53 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:12:53.989 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:12:53 smithi026 ceph-mon[175151]: failed to encode map e487 with expected crc 2023-09-23T21:12:53.989 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:12:53 smithi026 ceph-mon[175151]: failed to encode map e487 with expected crc 2023-09-23T21:12:53.989 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:12:53 smithi026 ceph-mon[175151]: osdmap e487: 8 total, 7 up, 6 in 2023-09-23T21:12:53.989 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:12:53 smithi026 ceph-mon[175151]: failed to encode map e487 with expected crc 2023-09-23T21:12:53.989 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:12:53 smithi026 ceph-mon[175151]: failed to encode map e487 with expected crc 2023-09-23T21:12:53.989 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:12:53 smithi026 ceph-mon[175151]: pgmap v1239: 129 pgs: 19 active+clean+remapped, 1 active+recovery_wait+remapped, 4 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 12 active+undersized, 3 active+recovering+undersized+degraded+remapped, 6 active+undersized+degraded, 8 active+recovery_wait+undersized+degraded+remapped, 5 active+recovery_wait+degraded+remapped, 70 active+clean; 4.0 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 249 op/s; 55738/197775 objects degraded (28.183%); 17854/197775 objects misplaced (9.027%); 13 MiB/s, 204 objects/s recovering 2023-09-23T21:12:53.989 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:12:53 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/1089111062' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:12:53.989 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:12:53 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:12:53.990 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:12:53.990 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:12:54.091 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:12:54.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:12:53 smithi070 ceph-mon[168282]: failed to encode map e487 with expected crc 2023-09-23T21:12:54.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:12:53 smithi070 ceph-mon[168282]: failed to encode map e487 with expected crc 2023-09-23T21:12:54.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:12:53 smithi070 ceph-mon[168282]: osdmap e487: 8 total, 7 up, 6 in 2023-09-23T21:12:54.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:12:53 smithi070 ceph-mon[168282]: failed to encode map e487 with expected crc 2023-09-23T21:12:54.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:12:53 smithi070 ceph-mon[168282]: failed to encode map e487 with expected crc 2023-09-23T21:12:54.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:12:53 smithi070 ceph-mon[168282]: pgmap v1239: 129 pgs: 19 active+clean+remapped, 1 active+recovery_wait+remapped, 4 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 12 active+undersized, 3 active+recovering+undersized+degraded+remapped, 6 active+undersized+degraded, 8 active+recovery_wait+undersized+degraded+remapped, 5 active+recovery_wait+degraded+remapped, 70 active+clean; 4.0 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 249 op/s; 55738/197775 objects degraded (28.183%); 17854/197775 objects misplaced (9.027%); 13 MiB/s, 204 objects/s recovering 2023-09-23T21:12:54.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:12:53 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/1089111062' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:12:54.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:12:53 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:12:54.160 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:12:54.266 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:12:54.271 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:12:54.271 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:12:54.372 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:12:54.460 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:12:54.526 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:12:54.534 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:12:54.534 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:12:54.635 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:12:54.699 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:12:54.771 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:12:54.772 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:12:54 smithi070 ceph-mon[168282]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:12:54.772 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:12:54 smithi070 ceph-mon[168282]: failed to encode map e488 with expected crc 2023-09-23T21:12:54.772 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:12:54 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T21:12:54.777 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:12:54.777 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:12:54.878 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:12:54.950 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:12:54.959 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:12:54 smithi026 ceph-mon[175151]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:12:54.960 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:12:54 smithi026 ceph-mon[175151]: failed to encode map e488 with expected crc 2023-09-23T21:12:54.960 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:12:54 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T21:12:54.960 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:12:54 smithi026 ceph-mon[175151]: osdmap e488: 8 total, 7 up, 6 in 2023-09-23T21:12:54.960 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:12:54 smithi026 ceph-mon[175151]: failed to encode map e488 with expected crc 2023-09-23T21:12:54.960 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:12:54 smithi026 ceph-mon[175151]: failed to encode map e488 with expected crc 2023-09-23T21:12:54.960 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:12:54 smithi026 ceph-mon[175151]: failed to encode map e488 with expected crc 2023-09-23T21:12:54.960 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:12:54 smithi026 ceph-mon[179688]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:12:54.961 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:12:54 smithi026 ceph-mon[179688]: failed to encode map e488 with expected crc 2023-09-23T21:12:54.961 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:12:54 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T21:12:54.961 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:12:54 smithi026 ceph-mon[179688]: osdmap e488: 8 total, 7 up, 6 in 2023-09-23T21:12:54.961 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:12:54 smithi026 ceph-mon[179688]: failed to encode map e488 with expected crc 2023-09-23T21:12:54.961 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:12:54 smithi026 ceph-mon[179688]: failed to encode map e488 with expected crc 2023-09-23T21:12:54.961 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:12:54 smithi026 ceph-mon[179688]: failed to encode map e488 with expected crc 2023-09-23T21:12:54.974 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:12:55.041 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:12:55.050 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:12:55.051 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:12:55.051 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:12:54 smithi070 ceph-mon[168282]: osdmap e488: 8 total, 7 up, 6 in 2023-09-23T21:12:55.051 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:12:54 smithi070 ceph-mon[168282]: failed to encode map e488 with expected crc 2023-09-23T21:12:55.051 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:12:54 smithi070 ceph-mon[168282]: failed to encode map e488 with expected crc 2023-09-23T21:12:55.052 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:12:54 smithi070 ceph-mon[168282]: failed to encode map e488 with expected crc 2023-09-23T21:12:55.152 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:12:55.196 INFO:teuthology.orchestra.run.smithi026.stdout:{ 2023-09-23T21:12:55.196 INFO:teuthology.orchestra.run.smithi026.stdout: "size": 20, 2023-09-23T21:12:55.196 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 600, 2023-09-23T21:12:55.196 INFO:teuthology.orchestra.run.smithi026.stdout: "ops": [ 2023-09-23T21:12:55.196 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.197 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35856.0:188719 9.d 9:b90b65eb:::benchmark_data_smithi026_342732_object75443:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e393)", 2023-09-23T21:12:55.197 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T21:09:05.341761+0000", 2023-09-23T21:12:55.197 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 229.85162924400001, 2023-09-23T21:12:55.197 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0824474660000001, 2023-09-23T21:12:55.197 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:12:55.197 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "reached pg", 2023-09-23T21:12:55.197 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T21:12:55.197 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35856", 2023-09-23T21:12:55.197 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/4180396258", 2023-09-23T21:12:55.197 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 188719 2023-09-23T21:12:55.197 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.197 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:12:55.197 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.198 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:12:55.198 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:05.341761+0000", 2023-09-23T21:12:55.198 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:12:55.198 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.198 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.198 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:12:55.198 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:05.341761+0000", 2023-09-23T21:12:55.198 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:12:55.198 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.198 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.198 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:12:55.198 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:05.341761+0000", 2023-09-23T21:12:55.199 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T21:12:55.199 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.199 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.199 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:12:55.199 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:05.341763+0000", 2023-09-23T21:12:55.199 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.2800000000000002e-06 2023-09-23T21:12:55.199 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.199 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.199 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:12:55.199 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:05.341769+0000", 2023-09-23T21:12:55.199 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.6200000000000004e-06 2023-09-23T21:12:55.199 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.200 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.200 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:12:55.200 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:05.341774+0000", 2023-09-23T21:12:55.200 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.3090000000000003e-06 2023-09-23T21:12:55.200 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.200 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.200 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:12:55.200 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:05.347772+0000", 2023-09-23T21:12:55.200 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0059983110000000001 2023-09-23T21:12:55.200 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.200 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.200 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:12:55.200 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:05.347890+0000", 2023-09-23T21:12:55.201 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00011803600000000001 2023-09-23T21:12:55.201 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.201 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.201 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 4,5,7", 2023-09-23T21:12:55.201 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:05.348008+0000", 2023-09-23T21:12:55.201 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000118013 2023-09-23T21:12:55.201 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.201 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.201 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T21:12:55.201 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:05.351154+0000", 2023-09-23T21:12:55.201 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0031462370000000001 2023-09-23T21:12:55.201 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.201 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.202 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:12:55.202 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:05.364608+0000", 2023-09-23T21:12:55.202 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.013453120000000001 2023-09-23T21:12:55.202 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.202 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.202 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:12:55.202 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:05.366833+0000", 2023-09-23T21:12:55.202 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0022257090000000002 2023-09-23T21:12:55.202 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.202 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.202 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:12:55.202 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:05.369392+0000", 2023-09-23T21:12:55.202 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0025588059999999998 2023-09-23T21:12:55.203 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.203 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.203 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T21:12:55.203 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:05.369401+0000", 2023-09-23T21:12:55.203 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.0299999999999999e-06 2023-09-23T21:12:55.203 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.203 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.203 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:12:55.203 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:06.424122+0000", 2023-09-23T21:12:55.203 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0547212210000001 2023-09-23T21:12:55.203 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.203 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.203 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T21:12:55.204 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:06.424208+0000", 2023-09-23T21:12:55.204 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.6105000000000003e-05 2023-09-23T21:12:55.204 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:12:55.204 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:12:55.204 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:12:55.204 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.204 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.204 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35856.0:188729 9.d 9:b7268ae6:::benchmark_data_smithi026_342732_object75453:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e393)", 2023-09-23T21:12:55.204 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T21:09:05.351457+0000", 2023-09-23T21:12:55.204 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 229.84193289000001, 2023-09-23T21:12:55.204 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0733328019999999, 2023-09-23T21:12:55.204 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:12:55.205 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "reached pg", 2023-09-23T21:12:55.205 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T21:12:55.205 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35856", 2023-09-23T21:12:55.205 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/4180396258", 2023-09-23T21:12:55.205 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 188729 2023-09-23T21:12:55.205 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.205 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:12:55.205 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.205 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:12:55.205 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:05.351457+0000", 2023-09-23T21:12:55.205 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:12:55.205 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.205 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.205 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:12:55.206 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:05.351457+0000", 2023-09-23T21:12:55.206 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:12:55.206 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.206 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.206 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:12:55.206 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:05.351457+0000", 2023-09-23T21:12:55.206 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T21:12:55.206 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.206 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.206 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:12:55.206 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:05.351460+0000", 2023-09-23T21:12:55.206 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.8430000000000001e-06 2023-09-23T21:12:55.207 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.207 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.207 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:12:55.207 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:05.351466+0000", 2023-09-23T21:12:55.207 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.472e-06 2023-09-23T21:12:55.207 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.207 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.207 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:12:55.207 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:05.351471+0000", 2023-09-23T21:12:55.207 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.7160000000000002e-06 2023-09-23T21:12:55.207 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.207 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.207 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:12:55.208 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:05.361407+0000", 2023-09-23T21:12:55.208 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0099358329999999998 2023-09-23T21:12:55.208 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.208 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.208 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:12:55.208 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:05.361591+0000", 2023-09-23T21:12:55.208 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00018443999999999999 2023-09-23T21:12:55.208 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.208 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.208 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 4,5,7", 2023-09-23T21:12:55.208 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:05.361744+0000", 2023-09-23T21:12:55.208 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00015254099999999999 2023-09-23T21:12:55.208 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.209 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.209 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:12:55.209 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:05.370334+0000", 2023-09-23T21:12:55.209 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0085906060000000006 2023-09-23T21:12:55.209 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.209 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.209 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T21:12:55.209 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:05.370346+0000", 2023-09-23T21:12:55.209 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1887e-05 2023-09-23T21:12:55.209 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.209 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.209 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:12:55.209 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:06.424728+0000", 2023-09-23T21:12:55.210 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0543822119999999 2023-09-23T21:12:55.210 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.210 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.210 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T21:12:55.210 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:06.424790+0000", 2023-09-23T21:12:55.210 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.1736999999999995e-05 2023-09-23T21:12:55.210 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:12:55.210 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:12:55.210 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:12:55.210 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.210 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.210 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35856.0:188731 9.d 9:b0b5a20e:::benchmark_data_smithi026_342732_object75455:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e393)", 2023-09-23T21:12:55.210 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T21:09:05.370224+0000", 2023-09-23T21:12:55.211 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 229.823165684, 2023-09-23T21:12:55.211 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.076281139, 2023-09-23T21:12:55.211 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:12:55.211 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:12:55.211 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T21:12:55.211 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35856", 2023-09-23T21:12:55.211 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/4180396258", 2023-09-23T21:12:55.211 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 188731 2023-09-23T21:12:55.211 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.211 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:12:55.211 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.211 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:12:55.212 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:05.370224+0000", 2023-09-23T21:12:55.212 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:12:55.212 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.212 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.212 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:12:55.212 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:05.370224+0000", 2023-09-23T21:12:55.212 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:12:55.212 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.212 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.212 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:12:55.212 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:05.370223+0000", 2023-09-23T21:12:55.212 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T21:12:55.212 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.213 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.213 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:12:55.213 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:05.370229+0000", 2023-09-23T21:12:55.213 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.8799999999999996e-06 2023-09-23T21:12:55.213 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.213 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.213 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:12:55.213 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:05.370246+0000", 2023-09-23T21:12:55.213 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.7632e-05 2023-09-23T21:12:55.213 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.213 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.213 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:12:55.213 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:05.370254+0000", 2023-09-23T21:12:55.213 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.5009999999999998e-06 2023-09-23T21:12:55.214 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.214 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.214 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:12:55.214 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:05.370532+0000", 2023-09-23T21:12:55.214 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000278118 2023-09-23T21:12:55.214 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.214 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.214 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T21:12:55.214 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:05.370542+0000", 2023-09-23T21:12:55.214 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.4269999999999997e-06 2023-09-23T21:12:55.214 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.214 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.215 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:12:55.215 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:06.424815+0000", 2023-09-23T21:12:55.215 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0542737360000001 2023-09-23T21:12:55.215 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.215 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.215 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:12:55.215 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:06.424930+0000", 2023-09-23T21:12:55.215 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000114504 2023-09-23T21:12:55.215 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.215 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.215 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 4,5", 2023-09-23T21:12:55.215 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:06.425060+0000", 2023-09-23T21:12:55.215 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00012982799999999999 2023-09-23T21:12:55.216 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.216 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.216 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T21:12:55.216 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:06.446261+0000", 2023-09-23T21:12:55.216 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.021201748999999999 2023-09-23T21:12:55.216 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.216 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.216 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:12:55.216 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:06.446320+0000", 2023-09-23T21:12:55.216 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.855e-05 2023-09-23T21:12:55.216 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.216 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.216 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:12:55.217 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:06.446395+0000", 2023-09-23T21:12:55.217 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.4777000000000006e-05 2023-09-23T21:12:55.217 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.217 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.217 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T21:12:55.217 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:06.446451+0000", 2023-09-23T21:12:55.217 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.5917e-05 2023-09-23T21:12:55.217 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.217 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.217 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T21:12:55.217 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:06.446506+0000", 2023-09-23T21:12:55.217 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.5031999999999998e-05 2023-09-23T21:12:55.217 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:12:55.218 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:12:55.218 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:12:55.218 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.218 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.218 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35856.0:188734 9.d 9:b773d88e:::benchmark_data_smithi026_342732_object75458:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e393)", 2023-09-23T21:12:55.218 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T21:09:05.370264+0000", 2023-09-23T21:12:55.218 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 229.823126355, 2023-09-23T21:12:55.218 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.076373246, 2023-09-23T21:12:55.218 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:12:55.218 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:12:55.218 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T21:12:55.218 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35856", 2023-09-23T21:12:55.218 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/4180396258", 2023-09-23T21:12:55.219 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 188734 2023-09-23T21:12:55.219 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.219 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:12:55.219 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.219 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:12:55.219 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:05.370264+0000", 2023-09-23T21:12:55.219 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:12:55.219 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.219 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.219 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:12:55.219 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:05.370264+0000", 2023-09-23T21:12:55.219 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:12:55.219 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.220 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.220 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:12:55.220 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:05.370263+0000", 2023-09-23T21:12:55.220 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T21:12:55.220 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.220 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.220 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:12:55.220 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:05.370267+0000", 2023-09-23T21:12:55.220 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.3349999999999998e-06 2023-09-23T21:12:55.220 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.220 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.220 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:12:55.220 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:05.370274+0000", 2023-09-23T21:12:55.221 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.5480000000000004e-06 2023-09-23T21:12:55.221 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.221 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.221 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:12:55.221 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:05.370280+0000", 2023-09-23T21:12:55.221 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.806e-06 2023-09-23T21:12:55.221 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.221 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.221 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:12:55.221 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:05.370565+0000", 2023-09-23T21:12:55.221 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00028462700000000002 2023-09-23T21:12:55.221 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.222 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.222 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T21:12:55.222 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:05.370572+0000", 2023-09-23T21:12:55.222 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.8589999999999999e-06 2023-09-23T21:12:55.222 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.222 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.222 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:12:55.222 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:06.428050+0000", 2023-09-23T21:12:55.222 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.057478607 2023-09-23T21:12:55.222 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.222 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.222 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:12:55.222 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:06.428181+0000", 2023-09-23T21:12:55.223 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00013093800000000001 2023-09-23T21:12:55.223 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.223 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.223 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 4,5", 2023-09-23T21:12:55.223 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:06.428320+0000", 2023-09-23T21:12:55.223 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00013870199999999999 2023-09-23T21:12:55.223 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.223 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.223 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T21:12:55.223 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:06.446270+0000", 2023-09-23T21:12:55.223 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.017950161999999999 2023-09-23T21:12:55.223 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.223 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.224 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:12:55.224 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:06.446358+0000", 2023-09-23T21:12:55.224 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.7860000000000002e-05 2023-09-23T21:12:55.224 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.224 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.224 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:12:55.224 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:06.446559+0000", 2023-09-23T21:12:55.224 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00020098499999999999 2023-09-23T21:12:55.224 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.224 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.224 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T21:12:55.224 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:06.446600+0000", 2023-09-23T21:12:55.224 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.1307999999999998e-05 2023-09-23T21:12:55.225 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.225 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.225 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T21:12:55.225 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:06.446637+0000", 2023-09-23T21:12:55.225 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.7002999999999998e-05 2023-09-23T21:12:55.225 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:12:55.225 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:12:55.225 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:12:55.225 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.225 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.225 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35856.0:188867 9.e 9:7af985d6:::benchmark_data_smithi026_342732_object75591:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e396)", 2023-09-23T21:12:55.225 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T21:09:07.394276+0000", 2023-09-23T21:12:55.225 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 227.79911426300001, 2023-09-23T21:12:55.226 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.157474551, 2023-09-23T21:12:55.226 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:12:55.226 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:12:55.226 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T21:12:55.226 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35856", 2023-09-23T21:12:55.226 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/4180396258", 2023-09-23T21:12:55.226 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 188867 2023-09-23T21:12:55.226 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.226 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:12:55.226 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.226 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:12:55.226 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:07.394276+0000", 2023-09-23T21:12:55.227 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:12:55.227 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.227 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.227 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:12:55.227 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:07.394276+0000", 2023-09-23T21:12:55.227 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:12:55.227 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.227 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.227 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:12:55.227 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:07.394274+0000", 2023-09-23T21:12:55.227 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T21:12:55.227 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.227 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.228 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:12:55.228 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:07.394282+0000", 2023-09-23T21:12:55.228 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.5320000000000001e-06 2023-09-23T21:12:55.228 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.228 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.228 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:12:55.228 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:07.396944+0000", 2023-09-23T21:12:55.228 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0026620440000000001 2023-09-23T21:12:55.228 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.228 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.228 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:12:55.228 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:07.396961+0000", 2023-09-23T21:12:55.228 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.7005000000000001e-05 2023-09-23T21:12:55.229 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.229 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.229 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:12:55.229 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:07.402354+0000", 2023-09-23T21:12:55.229 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0053932590000000001 2023-09-23T21:12:55.229 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.229 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.229 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T21:12:55.229 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:07.402376+0000", 2023-09-23T21:12:55.229 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.2246e-05 2023-09-23T21:12:55.229 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.229 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.230 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:12:55.230 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:08.395392+0000", 2023-09-23T21:12:55.230 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.99301544900000005 2023-09-23T21:12:55.230 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.230 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.230 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T21:12:55.230 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:08.395405+0000", 2023-09-23T21:12:55.230 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.3638000000000001e-05 2023-09-23T21:12:55.230 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.230 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.230 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:12:55.230 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:09.522891+0000", 2023-09-23T21:12:55.230 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.127485616 2023-09-23T21:12:55.231 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.231 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.231 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:12:55.231 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:09.525704+0000", 2023-09-23T21:12:55.231 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0028126359999999999 2023-09-23T21:12:55.231 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.231 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.231 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 0,4,5", 2023-09-23T21:12:55.231 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:09.526082+0000", 2023-09-23T21:12:55.231 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00037785199999999997 2023-09-23T21:12:55.231 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.231 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.231 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:12:55.232 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:09.542468+0000", 2023-09-23T21:12:55.232 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.016386563999999999 2023-09-23T21:12:55.232 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.232 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.232 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:12:55.232 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:09.542529+0000", 2023-09-23T21:12:55.232 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.1137999999999996e-05 2023-09-23T21:12:55.232 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.232 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.232 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T21:12:55.232 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:09.548525+0000", 2023-09-23T21:12:55.232 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0059955060000000003 2023-09-23T21:12:55.233 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.233 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.233 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:12:55.233 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:09.551567+0000", 2023-09-23T21:12:55.233 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0030419209999999999 2023-09-23T21:12:55.233 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.233 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.233 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T21:12:55.233 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:09.551623+0000", 2023-09-23T21:12:55.233 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.5937000000000003e-05 2023-09-23T21:12:55.233 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.233 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.233 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T21:12:55.234 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:09.551750+0000", 2023-09-23T21:12:55.234 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00012785199999999999 2023-09-23T21:12:55.234 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:12:55.234 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:12:55.234 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:12:55.234 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.234 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.234 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35856.0:188878 9.e 9:74bb71f0:::benchmark_data_smithi026_342732_object75602:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e396)", 2023-09-23T21:12:55.234 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T21:09:07.396973+0000", 2023-09-23T21:12:55.234 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 227.796417478, 2023-09-23T21:12:55.234 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.1550696789999999, 2023-09-23T21:12:55.234 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:12:55.235 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:12:55.235 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T21:12:55.235 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35856", 2023-09-23T21:12:55.235 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/4180396258", 2023-09-23T21:12:55.235 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 188878 2023-09-23T21:12:55.235 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.235 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:12:55.235 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.235 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:12:55.235 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:07.396973+0000", 2023-09-23T21:12:55.235 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:12:55.235 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.236 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.236 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:12:55.236 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:07.396973+0000", 2023-09-23T21:12:55.236 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:12:55.236 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.236 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.236 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:12:55.236 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:07.396972+0000", 2023-09-23T21:12:55.236 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.999999 2023-09-23T21:12:55.236 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.236 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.236 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:12:55.237 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:07.396978+0000", 2023-09-23T21:12:55.237 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.3110000000000002e-06 2023-09-23T21:12:55.237 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.237 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.237 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:12:55.237 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:07.396988+0000", 2023-09-23T21:12:55.237 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.8320000000000001e-06 2023-09-23T21:12:55.237 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.237 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.237 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:12:55.237 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:07.396994+0000", 2023-09-23T21:12:55.237 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.3389999999999998e-06 2023-09-23T21:12:55.237 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.238 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.238 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:12:55.238 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:07.402401+0000", 2023-09-23T21:12:55.238 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00540647 2023-09-23T21:12:55.238 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.238 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.238 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T21:12:55.238 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:07.402408+0000", 2023-09-23T21:12:55.238 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.2690000000000002e-06 2023-09-23T21:12:55.238 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.238 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.239 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:12:55.239 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:08.395431+0000", 2023-09-23T21:12:55.239 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.99302307400000001 2023-09-23T21:12:55.239 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.239 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.239 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T21:12:55.239 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:08.395439+0000", 2023-09-23T21:12:55.239 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.2349999999999999e-06 2023-09-23T21:12:55.239 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.239 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.239 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:12:55.239 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:09.542558+0000", 2023-09-23T21:12:55.239 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1471185479999999 2023-09-23T21:12:55.240 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.240 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.240 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:12:55.240 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:09.542701+0000", 2023-09-23T21:12:55.240 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00014349299999999999 2023-09-23T21:12:55.240 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.240 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.240 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 0,4,5", 2023-09-23T21:12:55.240 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:09.542855+0000", 2023-09-23T21:12:55.240 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00015409899999999999 2023-09-23T21:12:55.240 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.240 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.241 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T21:12:55.241 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:09.551436+0000", 2023-09-23T21:12:55.241 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0085808550000000001 2023-09-23T21:12:55.241 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.241 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.241 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:12:55.241 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:09.551486+0000", 2023-09-23T21:12:55.241 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.9679000000000003e-05 2023-09-23T21:12:55.241 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.241 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.241 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:12:55.241 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:09.551531+0000", 2023-09-23T21:12:55.242 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.5185000000000002e-05 2023-09-23T21:12:55.242 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.242 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.242 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:12:55.242 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:09.551848+0000", 2023-09-23T21:12:55.242 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00031644000000000003 2023-09-23T21:12:55.242 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.242 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.242 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T21:12:55.242 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:09.551907+0000", 2023-09-23T21:12:55.242 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.9713000000000002e-05 2023-09-23T21:12:55.242 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.243 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.243 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T21:12:55.243 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:09.552042+0000", 2023-09-23T21:12:55.243 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00013509700000000001 2023-09-23T21:12:55.243 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:12:55.243 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:12:55.243 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:12:55.243 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.243 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.243 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35856.0:188898 9.e 9:7bd64526:::benchmark_data_smithi026_342732_object75622:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e396)", 2023-09-23T21:12:55.243 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T21:09:07.397000+0000", 2023-09-23T21:12:55.243 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 227.79639066600001, 2023-09-23T21:12:55.244 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.1683603769999999, 2023-09-23T21:12:55.244 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:12:55.244 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:12:55.244 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T21:12:55.244 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35856", 2023-09-23T21:12:55.244 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/4180396258", 2023-09-23T21:12:55.244 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 188898 2023-09-23T21:12:55.244 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.244 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:12:55.244 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.244 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:12:55.244 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:07.397000+0000", 2023-09-23T21:12:55.245 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:12:55.245 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.245 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.245 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:12:55.245 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:07.397000+0000", 2023-09-23T21:12:55.245 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:12:55.245 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.245 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.245 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:12:55.245 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:07.396999+0000", 2023-09-23T21:12:55.245 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T21:12:55.246 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.246 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.246 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:12:55.246 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:07.397002+0000", 2023-09-23T21:12:55.246 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.4020000000000002e-06 2023-09-23T21:12:55.246 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.246 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.246 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:12:55.246 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:07.397010+0000", 2023-09-23T21:12:55.246 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.2339999999999997e-06 2023-09-23T21:12:55.246 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.247 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.247 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:12:55.247 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:07.397015+0000", 2023-09-23T21:12:55.247 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.6749999999999999e-06 2023-09-23T21:12:55.247 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.247 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.247 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:12:55.247 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:07.402430+0000", 2023-09-23T21:12:55.247 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0054143790000000004 2023-09-23T21:12:55.247 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.247 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.248 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T21:12:55.248 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:07.402436+0000", 2023-09-23T21:12:55.248 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.5459999999999997e-06 2023-09-23T21:12:55.248 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.248 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.248 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:12:55.248 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:08.395464+0000", 2023-09-23T21:12:55.248 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.99302792500000003 2023-09-23T21:12:55.248 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.248 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.248 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T21:12:55.248 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:08.395480+0000", 2023-09-23T21:12:55.249 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.6220000000000001e-05 2023-09-23T21:12:55.249 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.249 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.249 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:12:55.249 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:09.545709+0000", 2023-09-23T21:12:55.249 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1502284659999999 2023-09-23T21:12:55.249 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.249 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.249 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:12:55.249 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:09.545860+0000", 2023-09-23T21:12:55.249 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00015087 2023-09-23T21:12:55.249 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.249 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.250 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 0,4,5", 2023-09-23T21:12:55.250 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:09.546018+0000", 2023-09-23T21:12:55.250 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000158324 2023-09-23T21:12:55.250 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.250 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.250 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T21:12:55.250 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:09.551495+0000", 2023-09-23T21:12:55.250 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0054766429999999998 2023-09-23T21:12:55.250 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.250 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.250 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:12:55.250 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:09.551808+0000", 2023-09-23T21:12:55.251 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00031368299999999998 2023-09-23T21:12:55.251 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.251 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.251 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:12:55.251 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:09.558471+0000", 2023-09-23T21:12:55.251 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0066625859999999999 2023-09-23T21:12:55.251 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.251 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.251 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:12:55.251 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:09.565161+0000", 2023-09-23T21:12:55.251 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0066899250000000002 2023-09-23T21:12:55.251 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.252 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.252 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T21:12:55.252 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:09.565222+0000", 2023-09-23T21:12:55.252 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.0640999999999998e-05 2023-09-23T21:12:55.252 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.252 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.252 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T21:12:55.252 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:09.565360+0000", 2023-09-23T21:12:55.252 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000138279 2023-09-23T21:12:55.252 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:12:55.252 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:12:55.252 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:12:55.253 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.253 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.253 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35856.0:188916 9.e 9:7db6608b:::benchmark_data_smithi026_342732_object75640:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e396)", 2023-09-23T21:12:55.253 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T21:09:07.397020+0000", 2023-09-23T21:12:55.253 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 227.796370422, 2023-09-23T21:12:55.253 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.1686653690000002, 2023-09-23T21:12:55.253 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:12:55.253 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:12:55.253 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T21:12:55.253 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35856", 2023-09-23T21:12:55.253 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/4180396258", 2023-09-23T21:12:55.253 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 188916 2023-09-23T21:12:55.254 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.254 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:12:55.254 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.254 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:12:55.254 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:07.397020+0000", 2023-09-23T21:12:55.254 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:12:55.254 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.254 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.254 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:12:55.254 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:07.397020+0000", 2023-09-23T21:12:55.254 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:12:55.254 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.254 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.255 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:12:55.255 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:07.397019+0000", 2023-09-23T21:12:55.255 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T21:12:55.255 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.255 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.255 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:12:55.255 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:07.397022+0000", 2023-09-23T21:12:55.255 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.2309999999999999e-06 2023-09-23T21:12:55.255 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.255 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.255 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:12:55.255 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:07.397027+0000", 2023-09-23T21:12:55.256 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.5690000000000004e-06 2023-09-23T21:12:55.256 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.256 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.256 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:12:55.256 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:07.397032+0000", 2023-09-23T21:12:55.256 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.3120000000000004e-06 2023-09-23T21:12:55.256 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.256 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.256 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:12:55.256 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:07.402469+0000", 2023-09-23T21:12:55.256 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0054371130000000004 2023-09-23T21:12:55.256 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.256 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.257 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T21:12:55.257 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:07.402488+0000", 2023-09-23T21:12:55.257 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.8858000000000001e-05 2023-09-23T21:12:55.257 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.257 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.257 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:12:55.257 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:08.395505+0000", 2023-09-23T21:12:55.257 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.99301743399999998 2023-09-23T21:12:55.257 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.257 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.257 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T21:12:55.258 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:08.395513+0000", 2023-09-23T21:12:55.258 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.8869999999999992e-06 2023-09-23T21:12:55.258 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.258 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.258 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:12:55.258 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:09.548556+0000", 2023-09-23T21:12:55.258 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.153043475 2023-09-23T21:12:55.258 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.258 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.258 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:12:55.258 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:09.548673+0000", 2023-09-23T21:12:55.258 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000116536 2023-09-23T21:12:55.259 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.259 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.259 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 0,4,5", 2023-09-23T21:12:55.259 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:09.548765+0000", 2023-09-23T21:12:55.259 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.2607000000000003e-05 2023-09-23T21:12:55.259 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.259 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.259 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T21:12:55.259 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:09.551762+0000", 2023-09-23T21:12:55.259 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0029969060000000001 2023-09-23T21:12:55.259 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.259 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.260 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:12:55.260 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:09.552102+0000", 2023-09-23T21:12:55.260 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00033987500000000001 2023-09-23T21:12:55.260 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.260 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.260 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:12:55.260 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:09.558781+0000", 2023-09-23T21:12:55.260 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0066787050000000001 2023-09-23T21:12:55.260 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.260 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.260 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:12:55.260 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:09.565510+0000", 2023-09-23T21:12:55.260 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0067290680000000004 2023-09-23T21:12:55.261 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.261 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.261 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T21:12:55.261 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:09.565562+0000", 2023-09-23T21:12:55.261 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.2034999999999999e-05 2023-09-23T21:12:55.261 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.261 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.261 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T21:12:55.261 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:09.565685+0000", 2023-09-23T21:12:55.261 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00012306799999999999 2023-09-23T21:12:55.261 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:12:55.261 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:12:55.262 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:12:55.262 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.262 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.262 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35856.0:188936 9.e 9:7020b174:::benchmark_data_smithi026_342732_object75660:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e396)", 2023-09-23T21:12:55.262 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T21:09:07.397035+0000", 2023-09-23T21:12:55.262 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 227.79635507, 2023-09-23T21:12:55.262 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.181647866, 2023-09-23T21:12:55.262 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:12:55.262 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:12:55.262 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T21:12:55.262 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35856", 2023-09-23T21:12:55.262 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/4180396258", 2023-09-23T21:12:55.263 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 188936 2023-09-23T21:12:55.263 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.263 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:12:55.263 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.263 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:12:55.263 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:07.397035+0000", 2023-09-23T21:12:55.263 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:12:55.263 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.263 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.263 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:12:55.263 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:07.397035+0000", 2023-09-23T21:12:55.263 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:12:55.264 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.264 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.264 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:12:55.264 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:07.397035+0000", 2023-09-23T21:12:55.264 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T21:12:55.264 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.264 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.264 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:12:55.264 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:07.397037+0000", 2023-09-23T21:12:55.264 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.4090000000000001e-06 2023-09-23T21:12:55.264 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.264 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.264 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:12:55.264 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:07.397043+0000", 2023-09-23T21:12:55.265 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.9880000000000001e-06 2023-09-23T21:12:55.265 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.265 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.265 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:12:55.265 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:07.397048+0000", 2023-09-23T21:12:55.265 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.0810000000000001e-06 2023-09-23T21:12:55.265 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.265 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.265 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:12:55.265 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:07.402510+0000", 2023-09-23T21:12:55.265 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.005461367 2023-09-23T21:12:55.265 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.265 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.266 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T21:12:55.266 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:07.402517+0000", 2023-09-23T21:12:55.266 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.9140000000000002e-06 2023-09-23T21:12:55.266 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.266 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.266 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:12:55.266 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:08.395537+0000", 2023-09-23T21:12:55.266 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.99302077200000005 2023-09-23T21:12:55.266 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.266 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.266 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T21:12:55.266 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:08.395545+0000", 2023-09-23T21:12:55.267 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.8120000000000008e-06 2023-09-23T21:12:55.267 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.267 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.267 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:12:55.267 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:09.552130+0000", 2023-09-23T21:12:55.267 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1565847460000001 2023-09-23T21:12:55.267 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.267 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.267 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:12:55.267 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:09.552284+0000", 2023-09-23T21:12:55.267 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00015413699999999999 2023-09-23T21:12:55.267 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.267 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.268 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 0,4,5", 2023-09-23T21:12:55.268 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:09.552445+0000", 2023-09-23T21:12:55.268 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000161049 2023-09-23T21:12:55.268 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.268 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.268 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T21:12:55.268 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:09.558498+0000", 2023-09-23T21:12:55.268 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.006053381 2023-09-23T21:12:55.268 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.268 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.268 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:12:55.268 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:09.558837+0000", 2023-09-23T21:12:55.268 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00033902500000000001 2023-09-23T21:12:55.269 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.269 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.269 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:12:55.269 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:09.565923+0000", 2023-09-23T21:12:55.269 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0070852090000000003 2023-09-23T21:12:55.269 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.269 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.269 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:12:55.269 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:09.578541+0000", 2023-09-23T21:12:55.269 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.012618369000000001 2023-09-23T21:12:55.269 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.269 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.269 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T21:12:55.270 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:09.578597+0000", 2023-09-23T21:12:55.270 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.5949999999999998e-05 2023-09-23T21:12:55.270 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.270 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.270 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T21:12:55.270 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:09.578683+0000", 2023-09-23T21:12:55.270 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.5983000000000007e-05 2023-09-23T21:12:55.270 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:12:55.270 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:12:55.270 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:12:55.270 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.270 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.270 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35856.0:188956 9.e 9:78624dbf:::benchmark_data_smithi026_342732_object75680:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e396)", 2023-09-23T21:12:55.271 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T21:09:07.397052+0000", 2023-09-23T21:12:55.271 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 227.796337921, 2023-09-23T21:12:55.271 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.1819009989999998, 2023-09-23T21:12:55.271 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:12:55.271 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:12:55.271 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T21:12:55.271 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35856", 2023-09-23T21:12:55.271 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/4180396258", 2023-09-23T21:12:55.271 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 188956 2023-09-23T21:12:55.271 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.271 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:12:55.271 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.271 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:12:55.272 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:07.397052+0000", 2023-09-23T21:12:55.272 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:12:55.272 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.272 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.272 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:12:55.272 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:07.397052+0000", 2023-09-23T21:12:55.272 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:12:55.272 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.272 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.272 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:12:55.272 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:07.397052+0000", 2023-09-23T21:12:55.272 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T21:12:55.273 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.273 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.273 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:12:55.273 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:07.397054+0000", 2023-09-23T21:12:55.273 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.3259999999999998e-06 2023-09-23T21:12:55.273 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.273 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.273 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:12:55.273 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:07.397060+0000", 2023-09-23T21:12:55.273 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.1299999999999998e-06 2023-09-23T21:12:55.273 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.273 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.274 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:12:55.274 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:07.397065+0000", 2023-09-23T21:12:55.274 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.8659999999999997e-06 2023-09-23T21:12:55.274 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.274 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.274 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:12:55.274 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:07.402545+0000", 2023-09-23T21:12:55.274 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.005479696 2023-09-23T21:12:55.274 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.274 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.274 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T21:12:55.274 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:07.402552+0000", 2023-09-23T21:12:55.275 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.9330000000000004e-06 2023-09-23T21:12:55.275 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.275 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.275 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:12:55.275 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:08.395568+0000", 2023-09-23T21:12:55.275 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.99301622000000001 2023-09-23T21:12:55.275 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.275 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.275 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T21:12:55.275 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:08.395576+0000", 2023-09-23T21:12:55.275 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.2889999999999998e-06 2023-09-23T21:12:55.275 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.275 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.276 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:12:55.276 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:09.555297+0000", 2023-09-23T21:12:55.276 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.159720909 2023-09-23T21:12:55.276 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.276 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.276 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:12:55.276 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:09.555464+0000", 2023-09-23T21:12:55.276 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000166756 2023-09-23T21:12:55.276 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.276 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.276 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 0,4,5", 2023-09-23T21:12:55.276 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:09.555627+0000", 2023-09-23T21:12:55.276 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000162697 2023-09-23T21:12:55.277 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.277 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.277 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:12:55.277 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:09.558888+0000", 2023-09-23T21:12:55.277 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0032610009999999999 2023-09-23T21:12:55.277 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.277 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.277 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T21:12:55.277 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:09.569432+0000", 2023-09-23T21:12:55.277 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.010544661 2023-09-23T21:12:55.277 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.277 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.277 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:12:55.278 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:09.572340+0000", 2023-09-23T21:12:55.278 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0029073549999999999 2023-09-23T21:12:55.278 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.278 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.278 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:12:55.278 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:09.578784+0000", 2023-09-23T21:12:55.278 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0064438840000000004 2023-09-23T21:12:55.278 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.278 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.278 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T21:12:55.278 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:09.578830+0000", 2023-09-23T21:12:55.278 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.6789000000000002e-05 2023-09-23T21:12:55.279 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.279 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.279 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T21:12:55.279 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:09.578953+0000", 2023-09-23T21:12:55.279 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000122774 2023-09-23T21:12:55.279 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:12:55.279 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:12:55.279 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:12:55.279 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.279 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.279 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35856.0:188959 9.e 9:7368e502:::benchmark_data_smithi026_342732_object75683:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e396)", 2023-09-23T21:12:55.279 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T21:09:07.397070+0000", 2023-09-23T21:12:55.280 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 227.796320522, 2023-09-23T21:12:55.280 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.182075142, 2023-09-23T21:12:55.280 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:12:55.280 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:12:55.280 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T21:12:55.280 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35856", 2023-09-23T21:12:55.280 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/4180396258", 2023-09-23T21:12:55.280 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 188959 2023-09-23T21:12:55.280 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.280 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:12:55.280 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.280 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:12:55.280 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:07.397070+0000", 2023-09-23T21:12:55.281 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:12:55.281 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.281 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.281 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:12:55.281 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:07.397070+0000", 2023-09-23T21:12:55.281 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:12:55.281 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.281 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.281 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:12:55.281 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:07.397069+0000", 2023-09-23T21:12:55.281 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T21:12:55.281 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.281 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.282 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:12:55.282 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:07.397072+0000", 2023-09-23T21:12:55.282 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.5409999999999999e-06 2023-09-23T21:12:55.282 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.282 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.282 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:12:55.282 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:07.397078+0000", 2023-09-23T21:12:55.282 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.7590000000000003e-06 2023-09-23T21:12:55.282 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.282 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.282 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:12:55.282 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:07.397083+0000", 2023-09-23T21:12:55.283 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.2469999999999999e-06 2023-09-23T21:12:55.283 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.283 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.283 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:12:55.283 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:07.402573+0000", 2023-09-23T21:12:55.283 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0054898539999999997 2023-09-23T21:12:55.283 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.283 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.283 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T21:12:55.283 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:07.402578+0000", 2023-09-23T21:12:55.283 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.6960000000000002e-06 2023-09-23T21:12:55.283 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.283 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.284 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:12:55.284 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:08.395600+0000", 2023-09-23T21:12:55.284 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.99302183499999996 2023-09-23T21:12:55.284 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.284 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.284 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T21:12:55.284 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:08.395608+0000", 2023-09-23T21:12:55.284 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.5630000000000003e-06 2023-09-23T21:12:55.284 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.284 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.284 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:12:55.284 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:09.558920+0000", 2023-09-23T21:12:55.284 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1633124770000001 2023-09-23T21:12:55.285 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.285 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.285 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:12:55.285 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:09.559075+0000", 2023-09-23T21:12:55.285 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000154878 2023-09-23T21:12:55.285 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.285 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.285 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 0,4,5", 2023-09-23T21:12:55.285 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:09.559251+0000", 2023-09-23T21:12:55.285 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00017559800000000001 2023-09-23T21:12:55.285 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.285 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.286 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:12:55.286 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:09.565431+0000", 2023-09-23T21:12:55.286 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0061802020000000001 2023-09-23T21:12:55.286 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.286 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.286 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T21:12:55.286 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:09.569439+0000", 2023-09-23T21:12:55.286 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.004008419 2023-09-23T21:12:55.286 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.286 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.286 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:12:55.286 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:09.572363+0000", 2023-09-23T21:12:55.287 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.002923967 2023-09-23T21:12:55.287 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.287 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.287 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:12:55.287 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:09.579035+0000", 2023-09-23T21:12:55.287 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0066717160000000003 2023-09-23T21:12:55.287 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.287 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.287 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T21:12:55.287 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:09.579080+0000", 2023-09-23T21:12:55.287 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.4631e-05 2023-09-23T21:12:55.287 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.288 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.288 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T21:12:55.288 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:09.579145+0000", 2023-09-23T21:12:55.288 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.5166000000000006e-05 2023-09-23T21:12:55.288 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:12:55.288 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:12:55.288 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:12:55.288 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.288 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.288 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35856.0:188967 9.e 9:7be9207b:::benchmark_data_smithi026_342732_object75691:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e396)", 2023-09-23T21:12:55.288 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T21:09:07.397087+0000", 2023-09-23T21:12:55.288 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 227.79630332799999, 2023-09-23T21:12:55.289 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.1822950090000002, 2023-09-23T21:12:55.289 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:12:55.289 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:12:55.289 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T21:12:55.289 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35856", 2023-09-23T21:12:55.289 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/4180396258", 2023-09-23T21:12:55.289 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 188967 2023-09-23T21:12:55.289 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.289 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:12:55.289 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.289 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:12:55.289 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:07.397087+0000", 2023-09-23T21:12:55.290 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:12:55.290 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.290 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.290 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:12:55.290 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:07.397087+0000", 2023-09-23T21:12:55.290 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:12:55.290 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.290 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.290 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:12:55.290 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:07.397086+0000", 2023-09-23T21:12:55.290 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T21:12:55.290 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.291 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.291 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:12:55.291 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:07.397089+0000", 2023-09-23T21:12:55.291 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.4990000000000001e-06 2023-09-23T21:12:55.291 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.291 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.291 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:12:55.291 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:07.397095+0000", 2023-09-23T21:12:55.291 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.947e-06 2023-09-23T21:12:55.291 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.291 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.292 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:12:55.292 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:07.397106+0000", 2023-09-23T21:12:55.292 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1267000000000001e-05 2023-09-23T21:12:55.292 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.292 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.292 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:12:55.292 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:07.402597+0000", 2023-09-23T21:12:55.292 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0054908120000000003 2023-09-23T21:12:55.292 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.292 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.292 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T21:12:55.292 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:07.402603+0000", 2023-09-23T21:12:55.293 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.4699999999999999e-06 2023-09-23T21:12:55.293 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.293 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.293 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:12:55.293 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:08.395631+0000", 2023-09-23T21:12:55.293 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.99302780300000004 2023-09-23T21:12:55.293 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.293 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.293 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T21:12:55.293 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:08.395639+0000", 2023-09-23T21:12:55.293 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.4780000000000002e-06 2023-09-23T21:12:55.293 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.294 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.294 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:12:55.294 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:09.562140+0000", 2023-09-23T21:12:55.294 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1665013959999999 2023-09-23T21:12:55.294 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.294 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.294 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:12:55.294 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:09.562309+0000", 2023-09-23T21:12:55.294 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00016833199999999999 2023-09-23T21:12:55.294 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.294 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.295 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 0,4,5", 2023-09-23T21:12:55.295 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:09.562455+0000", 2023-09-23T21:12:55.295 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00014627299999999999 2023-09-23T21:12:55.295 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.295 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.295 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:12:55.295 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:09.565732+0000", 2023-09-23T21:12:55.295 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0032774319999999998 2023-09-23T21:12:55.295 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.295 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.295 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T21:12:55.295 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:09.572296+0000", 2023-09-23T21:12:55.296 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0065640430000000003 2023-09-23T21:12:55.296 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.296 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.296 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:12:55.296 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:09.572382+0000", 2023-09-23T21:12:55.296 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.5730999999999996e-05 2023-09-23T21:12:55.296 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.296 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.296 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:12:55.296 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:09.579214+0000", 2023-09-23T21:12:55.296 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0068315529999999998 2023-09-23T21:12:55.296 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.297 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.297 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T21:12:55.297 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:09.579263+0000", 2023-09-23T21:12:55.297 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.9546000000000003e-05 2023-09-23T21:12:55.297 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.297 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.297 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T21:12:55.297 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:09.579382+0000", 2023-09-23T21:12:55.297 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000118781 2023-09-23T21:12:55.297 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:12:55.297 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:12:55.298 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:12:55.298 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.298 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.298 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35856.0:189010 9.e 9:754d4c4d:::benchmark_data_smithi026_342732_object75734:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e396)", 2023-09-23T21:12:55.298 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T21:09:07.397111+0000", 2023-09-23T21:12:55.298 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 227.79627935799999, 2023-09-23T21:12:55.298 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.188492697, 2023-09-23T21:12:55.298 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:12:55.298 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:12:55.298 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T21:12:55.298 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35856", 2023-09-23T21:12:55.298 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/4180396258", 2023-09-23T21:12:55.298 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 189010 2023-09-23T21:12:55.299 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.299 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:12:55.299 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.299 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:12:55.299 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:07.397111+0000", 2023-09-23T21:12:55.299 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:12:55.299 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.299 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.299 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:12:55.299 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:07.397111+0000", 2023-09-23T21:12:55.299 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:12:55.299 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.300 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.300 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:12:55.300 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:07.397110+0000", 2023-09-23T21:12:55.300 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T21:12:55.300 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.300 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.300 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:12:55.300 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:07.397113+0000", 2023-09-23T21:12:55.300 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.6919999999999998e-06 2023-09-23T21:12:55.300 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.300 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.300 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:12:55.300 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:07.397119+0000", 2023-09-23T21:12:55.301 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.1079999999999998e-06 2023-09-23T21:12:55.301 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.301 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.301 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:12:55.301 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:07.397125+0000", 2023-09-23T21:12:55.301 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.3809999999999998e-06 2023-09-23T21:12:55.301 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.301 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.301 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:12:55.301 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:07.402624+0000", 2023-09-23T21:12:55.301 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0054991320000000003 2023-09-23T21:12:55.301 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.301 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.302 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T21:12:55.302 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:07.402630+0000", 2023-09-23T21:12:55.302 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.0759999999999999e-06 2023-09-23T21:12:55.302 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.302 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.302 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:12:55.302 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:08.395662+0000", 2023-09-23T21:12:55.302 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.99303197700000001 2023-09-23T21:12:55.302 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.302 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.302 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T21:12:55.302 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:08.395669+0000", 2023-09-23T21:12:55.303 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.559e-06 2023-09-23T21:12:55.303 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.303 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.303 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:12:55.303 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:09.566338+0000", 2023-09-23T21:12:55.303 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.170668745 2023-09-23T21:12:55.303 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.303 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.303 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:12:55.303 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:09.566468+0000", 2023-09-23T21:12:55.303 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00012997799999999999 2023-09-23T21:12:55.303 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.303 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.304 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 0,4,5", 2023-09-23T21:12:55.304 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:09.566614+0000", 2023-09-23T21:12:55.304 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000145444 2023-09-23T21:12:55.304 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.304 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.304 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:12:55.304 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:09.572403+0000", 2023-09-23T21:12:55.304 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0057895969999999996 2023-09-23T21:12:55.304 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.304 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.304 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T21:12:55.304 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:09.578691+0000", 2023-09-23T21:12:55.305 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0062879920000000001 2023-09-23T21:12:55.305 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.305 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.305 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:12:55.305 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:09.578723+0000", 2023-09-23T21:12:55.305 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.1377999999999997e-05 2023-09-23T21:12:55.305 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.305 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.305 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:12:55.305 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:09.585401+0000", 2023-09-23T21:12:55.305 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0066785459999999996 2023-09-23T21:12:55.305 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.305 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.306 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T21:12:55.306 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:09.585454+0000", 2023-09-23T21:12:55.306 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.2787999999999997e-05 2023-09-23T21:12:55.306 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.306 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.306 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T21:12:55.306 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:09.585604+0000", 2023-09-23T21:12:55.306 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000149686 2023-09-23T21:12:55.306 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:12:55.306 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:12:55.306 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:12:55.306 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.307 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.307 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35856.0:189039 9.e 9:707fd1cc:::benchmark_data_smithi026_342732_object75763:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e396)", 2023-09-23T21:12:55.307 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T21:09:07.397129+0000", 2023-09-23T21:12:55.307 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 227.79626160999999, 2023-09-23T21:12:55.307 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.1886985710000002, 2023-09-23T21:12:55.307 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:12:55.307 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:12:55.307 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T21:12:55.307 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35856", 2023-09-23T21:12:55.307 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/4180396258", 2023-09-23T21:12:55.307 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 189039 2023-09-23T21:12:55.307 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.308 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:12:55.308 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.308 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:12:55.308 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:07.397129+0000", 2023-09-23T21:12:55.308 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:12:55.308 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.308 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.308 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:12:55.308 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:07.397129+0000", 2023-09-23T21:12:55.308 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:12:55.308 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.308 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.309 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:12:55.309 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:07.397128+0000", 2023-09-23T21:12:55.309 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T21:12:55.309 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.309 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.309 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:12:55.309 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:07.397131+0000", 2023-09-23T21:12:55.309 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.3910000000000001e-06 2023-09-23T21:12:55.309 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.309 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.309 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:12:55.309 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:07.397137+0000", 2023-09-23T21:12:55.309 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.3400000000000003e-06 2023-09-23T21:12:55.310 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.310 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.310 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:12:55.310 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:07.397142+0000", 2023-09-23T21:12:55.310 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.4410000000000001e-06 2023-09-23T21:12:55.310 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.310 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.310 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:12:55.310 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:07.402651+0000", 2023-09-23T21:12:55.310 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0055080709999999998 2023-09-23T21:12:55.310 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.310 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.311 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T21:12:55.311 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:07.402657+0000", 2023-09-23T21:12:55.311 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.3160000000000002e-06 2023-09-23T21:12:55.311 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.311 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.311 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:12:55.311 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:08.395693+0000", 2023-09-23T21:12:55.311 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.99303612500000005 2023-09-23T21:12:55.311 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.311 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.311 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T21:12:55.311 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:08.395700+0000", 2023-09-23T21:12:55.312 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.5170000000000002e-06 2023-09-23T21:12:55.312 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.312 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.312 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:12:55.312 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:09.569468+0000", 2023-09-23T21:12:55.312 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.173767797 2023-09-23T21:12:55.312 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.312 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.312 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:12:55.312 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:09.569598+0000", 2023-09-23T21:12:55.312 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00012940799999999999 2023-09-23T21:12:55.312 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.313 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.313 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 0,4,5", 2023-09-23T21:12:55.313 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:09.569730+0000", 2023-09-23T21:12:55.313 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00013209800000000001 2023-09-23T21:12:55.313 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.313 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.313 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T21:12:55.313 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:09.578695+0000", 2023-09-23T21:12:55.313 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0089652740000000005 2023-09-23T21:12:55.313 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.313 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.313 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:12:55.314 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:09.578748+0000", 2023-09-23T21:12:55.314 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.2713999999999997e-05 2023-09-23T21:12:55.314 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.314 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.314 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:12:55.314 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:09.578996+0000", 2023-09-23T21:12:55.314 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00024833199999999998 2023-09-23T21:12:55.314 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.314 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.314 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:12:55.314 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:09.585652+0000", 2023-09-23T21:12:55.314 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0066561609999999998 2023-09-23T21:12:55.315 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.315 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.315 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T21:12:55.315 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:09.585704+0000", 2023-09-23T21:12:55.315 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.1337e-05 2023-09-23T21:12:55.315 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.315 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.315 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T21:12:55.315 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:09.585827+0000", 2023-09-23T21:12:55.315 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00012356100000000001 2023-09-23T21:12:55.315 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:12:55.315 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:12:55.316 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:12:55.316 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.316 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.316 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35856.0:189079 9.e 9:736d7ce0:::benchmark_data_smithi026_342732_object75803:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e396)", 2023-09-23T21:12:55.316 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T21:09:07.397147+0000", 2023-09-23T21:12:55.316 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 227.796243567, 2023-09-23T21:12:55.316 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.1889064239999998, 2023-09-23T21:12:55.316 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:12:55.316 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:12:55.316 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T21:12:55.316 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35856", 2023-09-23T21:12:55.316 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/4180396258", 2023-09-23T21:12:55.316 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 189079 2023-09-23T21:12:55.317 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.317 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:12:55.317 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.317 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:12:55.317 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:07.397147+0000", 2023-09-23T21:12:55.317 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:12:55.317 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.317 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.317 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:12:55.317 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:07.397147+0000", 2023-09-23T21:12:55.317 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:12:55.317 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.318 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.318 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:12:55.318 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:07.397146+0000", 2023-09-23T21:12:55.318 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T21:12:55.318 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.318 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.318 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:12:55.318 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:07.397149+0000", 2023-09-23T21:12:55.318 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.446e-06 2023-09-23T21:12:55.318 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.318 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.319 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:12:55.319 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:07.397155+0000", 2023-09-23T21:12:55.319 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.3369999999999996e-06 2023-09-23T21:12:55.319 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.319 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.319 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:12:55.319 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:07.397160+0000", 2023-09-23T21:12:55.319 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.9749999999999998e-06 2023-09-23T21:12:55.319 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.319 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.319 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:12:55.319 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:07.402677+0000", 2023-09-23T21:12:55.320 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0055166549999999996 2023-09-23T21:12:55.320 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.320 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.320 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T21:12:55.320 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:07.402682+0000", 2023-09-23T21:12:55.320 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.5539999999999997e-06 2023-09-23T21:12:55.320 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.320 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.320 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:12:55.320 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:08.395724+0000", 2023-09-23T21:12:55.320 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.99304205300000004 2023-09-23T21:12:55.320 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.320 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.321 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T21:12:55.321 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:08.395732+0000", 2023-09-23T21:12:55.321 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.2009999999999999e-06 2023-09-23T21:12:55.321 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.321 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.321 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:12:55.321 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:09.572418+0000", 2023-09-23T21:12:55.321 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1766850739999999 2023-09-23T21:12:55.321 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.321 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.321 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:12:55.321 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:09.572528+0000", 2023-09-23T21:12:55.321 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000110073 2023-09-23T21:12:55.322 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.322 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.322 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 0,4,5", 2023-09-23T21:12:55.322 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:09.572664+0000", 2023-09-23T21:12:55.322 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00013662200000000001 2023-09-23T21:12:55.322 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.322 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.322 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T21:12:55.322 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:09.578699+0000", 2023-09-23T21:12:55.322 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0060349169999999999 2023-09-23T21:12:55.322 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.322 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.323 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:12:55.323 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:09.579176+0000", 2023-09-23T21:12:55.323 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000477042 2023-09-23T21:12:55.323 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.323 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.323 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:12:55.323 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:09.585344+0000", 2023-09-23T21:12:55.323 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0061675410000000003 2023-09-23T21:12:55.323 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.323 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.323 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:12:55.323 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:09.585882+0000", 2023-09-23T21:12:55.323 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00053863299999999995 2023-09-23T21:12:55.324 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.324 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.324 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T21:12:55.324 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:09.585929+0000", 2023-09-23T21:12:55.324 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.7030999999999997e-05 2023-09-23T21:12:55.324 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.324 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.324 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T21:12:55.324 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:09.586053+0000", 2023-09-23T21:12:55.324 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00012363000000000001 2023-09-23T21:12:55.324 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:12:55.324 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:12:55.324 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:12:55.325 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.325 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.325 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35856.0:189134 9.e 9:723254c9:::benchmark_data_smithi026_342732_object75858:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e396)", 2023-09-23T21:12:55.325 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T21:09:07.397164+0000", 2023-09-23T21:12:55.325 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 227.79622605399999, 2023-09-23T21:12:55.325 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.2140037029999999, 2023-09-23T21:12:55.325 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:12:55.325 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:12:55.325 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T21:12:55.325 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35856", 2023-09-23T21:12:55.325 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/4180396258", 2023-09-23T21:12:55.325 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 189134 2023-09-23T21:12:55.325 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.326 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:12:55.326 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.326 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:12:55.326 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:07.397164+0000", 2023-09-23T21:12:55.326 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:12:55.326 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.326 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.326 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:12:55.326 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:07.397164+0000", 2023-09-23T21:12:55.326 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:12:55.326 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.326 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.326 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:12:55.327 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:07.397164+0000", 2023-09-23T21:12:55.327 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T21:12:55.327 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.327 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.327 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:12:55.327 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:07.397172+0000", 2023-09-23T21:12:55.327 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.225e-06 2023-09-23T21:12:55.327 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.327 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.327 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:12:55.327 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:07.397179+0000", 2023-09-23T21:12:55.327 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.5320000000000003e-06 2023-09-23T21:12:55.327 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.328 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.328 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:12:55.328 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:07.397184+0000", 2023-09-23T21:12:55.328 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.2279999999999998e-06 2023-09-23T21:12:55.328 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.328 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.328 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:12:55.328 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:07.402701+0000", 2023-09-23T21:12:55.328 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0055169479999999998 2023-09-23T21:12:55.328 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.328 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.328 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T21:12:55.328 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:07.402707+0000", 2023-09-23T21:12:55.329 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.3500000000000002e-06 2023-09-23T21:12:55.329 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.329 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.329 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:12:55.329 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:08.395757+0000", 2023-09-23T21:12:55.329 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.99305015799999996 2023-09-23T21:12:55.329 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.329 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.329 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T21:12:55.329 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:08.395765+0000", 2023-09-23T21:12:55.329 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.5870000000000004e-06 2023-09-23T21:12:55.329 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.329 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.330 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:12:55.330 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:09.575078+0000", 2023-09-23T21:12:55.330 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.179313482 2023-09-23T21:12:55.330 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.330 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.330 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:12:55.330 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:09.575192+0000", 2023-09-23T21:12:55.330 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00011385199999999999 2023-09-23T21:12:55.330 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.330 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.330 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 0,4,5", 2023-09-23T21:12:55.330 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:09.575323+0000", 2023-09-23T21:12:55.330 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000130524 2023-09-23T21:12:55.331 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.331 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.331 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T21:12:55.331 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:09.578727+0000", 2023-09-23T21:12:55.331 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0034039579999999999 2023-09-23T21:12:55.331 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.331 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.331 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:12:55.331 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:09.579430+0000", 2023-09-23T21:12:55.331 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00070300600000000005 2023-09-23T21:12:55.331 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.331 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.332 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:12:55.332 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:09.586100+0000", 2023-09-23T21:12:55.332 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0066703910000000003 2023-09-23T21:12:55.332 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.332 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.332 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:12:55.332 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:09.610991+0000", 2023-09-23T21:12:55.332 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.024891309 2023-09-23T21:12:55.332 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.332 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.332 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T21:12:55.332 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:09.611043+0000", 2023-09-23T21:12:55.332 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.1882999999999998e-05 2023-09-23T21:12:55.332 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.333 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.333 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T21:12:55.333 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:09.611168+0000", 2023-09-23T21:12:55.333 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000124623 2023-09-23T21:12:55.333 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:12:55.333 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:12:55.333 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:12:55.333 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.333 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.333 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35856.0:189139 9.e 9:71d43f8f:::benchmark_data_smithi026_342732_object75863:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e396)", 2023-09-23T21:12:55.333 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T21:09:07.397188+0000", 2023-09-23T21:12:55.333 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 227.79620240899999, 2023-09-23T21:12:55.334 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.2142602409999999, 2023-09-23T21:12:55.334 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:12:55.334 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:12:55.334 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T21:12:55.334 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35856", 2023-09-23T21:12:55.334 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/4180396258", 2023-09-23T21:12:55.334 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 189139 2023-09-23T21:12:55.334 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.334 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:12:55.334 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.334 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:12:55.335 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:07.397188+0000", 2023-09-23T21:12:55.335 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:12:55.335 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.335 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.335 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:12:55.335 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:07.397188+0000", 2023-09-23T21:12:55.335 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:12:55.335 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.335 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.335 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:12:55.335 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:07.397187+0000", 2023-09-23T21:12:55.335 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T21:12:55.336 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.336 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.336 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:12:55.336 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:07.397190+0000", 2023-09-23T21:12:55.336 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.1119999999999999e-06 2023-09-23T21:12:55.336 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.336 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.336 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:12:55.336 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:07.397195+0000", 2023-09-23T21:12:55.336 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.8000000000000004e-06 2023-09-23T21:12:55.336 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.336 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.337 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:12:55.337 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:07.397200+0000", 2023-09-23T21:12:55.337 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.6639999999999997e-06 2023-09-23T21:12:55.337 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.337 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.337 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:12:55.337 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:07.402726+0000", 2023-09-23T21:12:55.337 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0055261060000000002 2023-09-23T21:12:55.337 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.337 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.337 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T21:12:55.337 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:07.402732+0000", 2023-09-23T21:12:55.337 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.2990000000000002e-06 2023-09-23T21:12:55.338 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.338 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.338 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:12:55.338 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:08.395789+0000", 2023-09-23T21:12:55.338 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.99305684100000002 2023-09-23T21:12:55.338 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.338 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.338 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T21:12:55.338 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:08.395797+0000", 2023-09-23T21:12:55.338 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.6930000000000007e-06 2023-09-23T21:12:55.338 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.338 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.339 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:12:55.339 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:09.579458+0000", 2023-09-23T21:12:55.339 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.183661047 2023-09-23T21:12:55.339 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.339 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.339 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:12:55.339 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:09.579599+0000", 2023-09-23T21:12:55.339 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000140731 2023-09-23T21:12:55.339 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.339 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.339 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 0,4,5", 2023-09-23T21:12:55.339 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:09.579718+0000", 2023-09-23T21:12:55.339 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000119183 2023-09-23T21:12:55.340 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.340 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.340 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T21:12:55.340 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:09.585354+0000", 2023-09-23T21:12:55.340 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0056359349999999999 2023-09-23T21:12:55.340 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.340 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.340 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:12:55.340 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:09.610947+0000", 2023-09-23T21:12:55.340 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.025592658000000001 2023-09-23T21:12:55.340 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.340 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.340 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:12:55.341 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:09.611210+0000", 2023-09-23T21:12:55.341 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00026378600000000001 2023-09-23T21:12:55.341 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.341 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.341 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:12:55.341 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:09.611283+0000", 2023-09-23T21:12:55.341 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.2791000000000002e-05 2023-09-23T21:12:55.341 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.341 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.341 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T21:12:55.341 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:09.611331+0000", 2023-09-23T21:12:55.341 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.8251000000000003e-05 2023-09-23T21:12:55.342 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.342 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.342 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T21:12:55.342 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:09.611448+0000", 2023-09-23T21:12:55.342 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000116659 2023-09-23T21:12:55.342 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:12:55.342 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:12:55.342 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:12:55.342 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.342 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.342 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35856.0:189144 9.e 9:7527167d:::benchmark_data_smithi026_342732_object75868:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e396)", 2023-09-23T21:12:55.342 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T21:09:07.397204+0000", 2023-09-23T21:12:55.343 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 227.796186038, 2023-09-23T21:12:55.343 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.214518773, 2023-09-23T21:12:55.343 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:12:55.343 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:12:55.343 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T21:12:55.343 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35856", 2023-09-23T21:12:55.343 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/4180396258", 2023-09-23T21:12:55.343 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 189144 2023-09-23T21:12:55.343 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.343 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:12:55.343 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.343 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:12:55.343 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:07.397204+0000", 2023-09-23T21:12:55.344 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:12:55.344 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.344 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.344 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:12:55.344 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:07.397204+0000", 2023-09-23T21:12:55.344 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:12:55.344 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.344 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.344 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:12:55.344 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:07.397204+0000", 2023-09-23T21:12:55.344 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T21:12:55.344 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.344 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.345 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:12:55.345 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:07.397206+0000", 2023-09-23T21:12:55.345 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.39e-06 2023-09-23T21:12:55.345 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.345 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.345 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:12:55.345 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:07.397212+0000", 2023-09-23T21:12:55.345 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.2009999999999996e-06 2023-09-23T21:12:55.345 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.345 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.345 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:12:55.346 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:07.397217+0000", 2023-09-23T21:12:55.346 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.6190000000000002e-06 2023-09-23T21:12:55.346 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.346 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.346 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:12:55.346 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:07.402753+0000", 2023-09-23T21:12:55.346 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0055363239999999996 2023-09-23T21:12:55.346 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.346 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.346 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T21:12:55.346 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:07.402759+0000", 2023-09-23T21:12:55.346 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.1249999999999998e-06 2023-09-23T21:12:55.346 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.347 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.347 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:12:55.347 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:08.395821+0000", 2023-09-23T21:12:55.347 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.99306115800000005 2023-09-23T21:12:55.347 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.347 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.347 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T21:12:55.347 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:08.395828+0000", 2023-09-23T21:12:55.347 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.7069999999999992e-06 2023-09-23T21:12:55.347 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.347 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.347 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:12:55.348 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:09.582488+0000", 2023-09-23T21:12:55.348 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1866593400000001 2023-09-23T21:12:55.348 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.348 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.348 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:12:55.348 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:09.582642+0000", 2023-09-23T21:12:55.348 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000154349 2023-09-23T21:12:55.348 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.348 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.348 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 0,4,5", 2023-09-23T21:12:55.348 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:09.582788+0000", 2023-09-23T21:12:55.348 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00014585600000000001 2023-09-23T21:12:55.348 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.349 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.349 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T21:12:55.349 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:09.585841+0000", 2023-09-23T21:12:55.349 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00305273 2023-09-23T21:12:55.349 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.349 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.349 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:12:55.349 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:09.611247+0000", 2023-09-23T21:12:55.349 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.025406245000000001 2023-09-23T21:12:55.349 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.349 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.349 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:12:55.350 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:09.611497+0000", 2023-09-23T21:12:55.350 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000250275 2023-09-23T21:12:55.350 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.350 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.350 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:12:55.350 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:09.611563+0000", 2023-09-23T21:12:55.350 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.6006000000000005e-05 2023-09-23T21:12:55.350 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.350 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.350 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T21:12:55.350 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:09.611604+0000", 2023-09-23T21:12:55.350 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.0479000000000003e-05 2023-09-23T21:12:55.351 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.351 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.351 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T21:12:55.351 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:09.611723+0000", 2023-09-23T21:12:55.351 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000119325 2023-09-23T21:12:55.351 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:12:55.351 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:12:55.351 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:12:55.351 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.351 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.351 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35856.0:189187 9.e 9:7021101b:::benchmark_data_smithi026_342732_object75911:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e396)", 2023-09-23T21:12:55.351 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T21:09:07.397221+0000", 2023-09-23T21:12:55.352 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 227.796169378, 2023-09-23T21:12:55.352 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.2150392590000001, 2023-09-23T21:12:55.352 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:12:55.352 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:12:55.352 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T21:12:55.352 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35856", 2023-09-23T21:12:55.352 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/4180396258", 2023-09-23T21:12:55.352 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 189187 2023-09-23T21:12:55.352 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.352 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:12:55.352 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.352 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:12:55.353 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:07.397221+0000", 2023-09-23T21:12:55.353 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:12:55.353 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.353 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.353 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:12:55.353 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:07.397221+0000", 2023-09-23T21:12:55.353 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:12:55.353 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.353 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.353 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:12:55.353 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:07.397220+0000", 2023-09-23T21:12:55.353 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T21:12:55.354 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.354 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.354 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:12:55.354 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:07.397223+0000", 2023-09-23T21:12:55.354 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.0949999999999998e-06 2023-09-23T21:12:55.354 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.354 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.354 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:12:55.354 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:07.397228+0000", 2023-09-23T21:12:55.354 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.1789999999999999e-06 2023-09-23T21:12:55.354 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.354 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.355 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:12:55.355 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:07.397233+0000", 2023-09-23T21:12:55.355 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.9740000000000001e-06 2023-09-23T21:12:55.355 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.355 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.355 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:12:55.355 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:07.402781+0000", 2023-09-23T21:12:55.355 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0055482609999999996 2023-09-23T21:12:55.355 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.355 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.355 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T21:12:55.355 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:07.402788+0000", 2023-09-23T21:12:55.356 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.5350000000000001e-06 2023-09-23T21:12:55.356 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.356 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.356 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:12:55.356 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:08.395853+0000", 2023-09-23T21:12:55.356 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.99306562899999995 2023-09-23T21:12:55.356 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.356 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.356 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T21:12:55.356 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:08.395861+0000", 2023-09-23T21:12:55.356 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.7680000000000008e-06 2023-09-23T21:12:55.356 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.357 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.357 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:12:55.357 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:09.586126+0000", 2023-09-23T21:12:55.357 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1902646290000001 2023-09-23T21:12:55.357 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.357 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.357 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:12:55.357 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:09.586267+0000", 2023-09-23T21:12:55.357 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00014103 2023-09-23T21:12:55.357 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.357 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.357 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 0,4,5", 2023-09-23T21:12:55.358 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:09.586415+0000", 2023-09-23T21:12:55.358 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000147971 2023-09-23T21:12:55.358 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.358 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.358 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T21:12:55.358 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:09.610957+0000", 2023-09-23T21:12:55.358 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.024542325 2023-09-23T21:12:55.358 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.358 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.358 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:12:55.358 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:09.611531+0000", 2023-09-23T21:12:55.358 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00057401399999999997 2023-09-23T21:12:55.359 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.359 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.359 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:12:55.359 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:09.611761+0000", 2023-09-23T21:12:55.359 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000230304 2023-09-23T21:12:55.359 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.359 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.359 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:12:55.359 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:09.612100+0000", 2023-09-23T21:12:55.359 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00033894699999999998 2023-09-23T21:12:55.359 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.359 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.360 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T21:12:55.360 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:09.612144+0000", 2023-09-23T21:12:55.360 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.3738e-05 2023-09-23T21:12:55.360 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.360 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.360 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T21:12:55.360 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:09.612260+0000", 2023-09-23T21:12:55.360 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000116191 2023-09-23T21:12:55.360 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:12:55.360 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:12:55.360 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:12:55.361 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.361 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.361 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35856.0:189207 9.e 9:77e2cdae:::benchmark_data_smithi026_342732_object75931:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e396)", 2023-09-23T21:12:55.361 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T21:09:07.397237+0000", 2023-09-23T21:12:55.361 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 227.796153524, 2023-09-23T21:12:55.361 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.2290301229999998, 2023-09-23T21:12:55.361 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:12:55.361 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:12:55.361 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T21:12:55.361 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35856", 2023-09-23T21:12:55.361 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/4180396258", 2023-09-23T21:12:55.361 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 189207 2023-09-23T21:12:55.361 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.362 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:12:55.362 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.362 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:12:55.362 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:07.397237+0000", 2023-09-23T21:12:55.362 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:12:55.362 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.362 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.362 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:12:55.362 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:07.397237+0000", 2023-09-23T21:12:55.362 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:12:55.362 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.362 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.362 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:12:55.363 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:07.397236+0000", 2023-09-23T21:12:55.363 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T21:12:55.363 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.363 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.363 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:12:55.363 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:07.397238+0000", 2023-09-23T21:12:55.363 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.1919999999999999e-06 2023-09-23T21:12:55.363 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.363 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.363 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:12:55.363 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:07.397244+0000", 2023-09-23T21:12:55.363 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.6969999999999998e-06 2023-09-23T21:12:55.363 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.363 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.363 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:12:55.364 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:07.397249+0000", 2023-09-23T21:12:55.365 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.3819999999999997e-06 2023-09-23T21:12:55.365 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.365 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.365 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:12:55.365 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:07.402809+0000", 2023-09-23T21:12:55.365 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0055603400000000004 2023-09-23T21:12:55.365 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.365 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.365 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T21:12:55.366 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:07.402815+0000", 2023-09-23T21:12:55.366 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.0290000000000002e-06 2023-09-23T21:12:55.366 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.366 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.366 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:12:55.366 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:08.395886+0000", 2023-09-23T21:12:55.366 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.99307133599999997 2023-09-23T21:12:55.366 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.366 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.366 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T21:12:55.366 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:08.395894+0000", 2023-09-23T21:12:55.366 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.2109999999999998e-06 2023-09-23T21:12:55.367 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.367 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.367 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:12:55.367 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:09.594412+0000", 2023-09-23T21:12:55.367 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.198517316 2023-09-23T21:12:55.367 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.367 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.367 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:12:55.367 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:09.594555+0000", 2023-09-23T21:12:55.367 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000143366 2023-09-23T21:12:55.367 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.367 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.367 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 0,4,5", 2023-09-23T21:12:55.367 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:09.608184+0000", 2023-09-23T21:12:55.368 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.013628707 2023-09-23T21:12:55.368 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.368 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.368 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:12:55.368 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:09.615278+0000", 2023-09-23T21:12:55.368 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0070938980000000004 2023-09-23T21:12:55.368 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.368 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.368 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:12:55.368 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:09.615309+0000", 2023-09-23T21:12:55.368 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.0870000000000001e-05 2023-09-23T21:12:55.368 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.369 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.369 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:12:55.369 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:09.618125+0000", 2023-09-23T21:12:55.369 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.002816416 2023-09-23T21:12:55.369 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.369 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.369 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T21:12:55.369 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:09.626083+0000", 2023-09-23T21:12:55.369 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0079576569999999999 2023-09-23T21:12:55.369 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.369 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.369 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T21:12:55.369 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:09.626140+0000", 2023-09-23T21:12:55.369 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.7599999999999997e-05 2023-09-23T21:12:55.370 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:12:55.370 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:12:55.370 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T21:12:55.370 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:09.626267+0000", 2023-09-23T21:12:55.370 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00012646599999999999 2023-09-23T21:12:55.370 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:12:55.370 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:12:55.370 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:12:55.370 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:12:55.370 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:12:55.370 INFO:teuthology.orchestra.run.smithi026.stdout:} 2023-09-23T21:12:55.371 INFO:tasks.thrashosds.thrasher:in_osds: [0, 2, 3, 4, 5, 7] out_osds: [6, 1] dead_osds: [0] live_osds: [2, 3, 4, 5, 7, 6, 1] 2023-09-23T21:12:55.371 INFO:tasks.thrashosds.thrasher:Reviving osd 0 2023-09-23T21:12:55.371 INFO:tasks.cephadm.osd.0:Restarting osd.0 (starting--it wasn't running)... 2023-09-23T21:12:55.371 DEBUG:teuthology.orchestra.run.smithi026:osd.0> sudo journalctl -f -n 0 -u ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7@osd.0.service 2023-09-23T21:12:55.372 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:12:55.373 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:12:55.373 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:12:55.373 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:12:55.373 DEBUG:teuthology.orchestra.run.smithi026:> sudo systemctl start ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7@osd.0 2023-09-23T21:12:55.410 INFO:journalctl@ceph.osd.0.smithi026.stdout:-- Logs begin at Sat 2023-09-23 20:20:42 UTC. -- 2023-09-23T21:12:55.474 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:12:55.576 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:12:55.598 INFO:journalctl@ceph.osd.0.smithi026.stdout:Sep 23 21:12:55 smithi026 systemd[1]: Starting Ceph osd.0 for b3f8e4ec-5a50-11ee-8db4-212e2dc638e7... 2023-09-23T21:12:55.640 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:12:55.645 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:12:55.645 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:12:55.746 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:12:55.782 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:12:55.801 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_ops_in_flight 2023-09-23T21:12:55.836 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:12:55.841 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:12:55 smithi070 ceph-mon[168282]: pgmap v1241: 129 pgs: 19 active+clean+remapped, 1 active+recovery_wait+remapped, 4 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 12 active+undersized, 2 active+recovering+undersized+degraded+remapped, 6 active+undersized+degraded, 8 active+recovery_wait+undersized+degraded+remapped, 5 active+recovery_wait+degraded+remapped, 71 active+clean; 4.0 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 359 op/s; 54717/195174 objects degraded (28.035%); 17414/195174 objects misplaced (8.922%); 19 MiB/s, 364 objects/s recovering 2023-09-23T21:12:55.841 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:12:55 smithi070 ceph-mon[168282]: Health check update: Degraded data redundancy: 55738/197775 objects degraded (28.183%), 27 pgs degraded, 24 pgs undersized (PG_DEGRADED) 2023-09-23T21:12:55.842 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:12:55.842 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:12:55.881 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:12:55 smithi026 ceph-mon[179688]: pgmap v1241: 129 pgs: 19 active+clean+remapped, 1 active+recovery_wait+remapped, 4 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 12 active+undersized, 2 active+recovering+undersized+degraded+remapped, 6 active+undersized+degraded, 8 active+recovery_wait+undersized+degraded+remapped, 5 active+recovery_wait+degraded+remapped, 71 active+clean; 4.0 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 359 op/s; 54717/195174 objects degraded (28.035%); 17414/195174 objects misplaced (8.922%); 19 MiB/s, 364 objects/s recovering 2023-09-23T21:12:55.881 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:12:55 smithi026 ceph-mon[179688]: Health check update: Degraded data redundancy: 55738/197775 objects degraded (28.183%), 27 pgs degraded, 24 pgs undersized (PG_DEGRADED) 2023-09-23T21:12:55.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:12:55 smithi026 ceph-mon[175151]: pgmap v1241: 129 pgs: 19 active+clean+remapped, 1 active+recovery_wait+remapped, 4 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 12 active+undersized, 2 active+recovering+undersized+degraded+remapped, 6 active+undersized+degraded, 8 active+recovery_wait+undersized+degraded+remapped, 5 active+recovery_wait+degraded+remapped, 71 active+clean; 4.0 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 359 op/s; 54717/195174 objects degraded (28.035%); 17414/195174 objects misplaced (8.922%); 19 MiB/s, 364 objects/s recovering 2023-09-23T21:12:55.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:12:55 smithi026 ceph-mon[175151]: Health check update: Degraded data redundancy: 55738/197775 objects degraded (28.183%), 27 pgs degraded, 24 pgs undersized (PG_DEGRADED) 2023-09-23T21:12:55.943 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:12:56.030 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:12:56.094 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:12:56.103 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:12:56.104 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:12:56.205 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:12:56.260 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:12:56.314 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:12:56.318 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:12:56.318 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:12:56.419 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:12:56.479 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:12:56.583 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:12:56.597 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:12:56.598 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:12:56.656 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T21:12:56.699 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:12:56.745 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1: {} 2023-09-23T21:12:56.766 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2: {} 2023-09-23T21:12:56.778 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:12:56.788 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3: {} 2023-09-23T21:12:56.823 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4: {} 2023-09-23T21:12:56.832 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:12:56.855 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T21:12:56.874 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:12:56.882 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6: {} 2023-09-23T21:12:56.885 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:12:56.886 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:12:56.912 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T21:12:56.912 INFO:teuthology.orchestra.run.smithi026.stderr:Error ENXIO: problem getting command descriptions from osd.0 2023-09-23T21:12:56.912 INFO:teuthology.orchestra.run.smithi026.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T21:12:56.912 INFO:teuthology.orchestra.run.smithi026.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T21:12:56.912 INFO:teuthology.orchestra.run.smithi026.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T21:12:56.912 INFO:teuthology.orchestra.run.smithi026.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T21:12:56.912 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T21:12:56.912 INFO:teuthology.orchestra.run.smithi026.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T21:12:56.913 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T21:12:56.987 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:12:57.030 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:12:56 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/2426119096' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:12:57.033 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:12:57.123 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:12:57.131 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:12:56 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/2426119096' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:12:57.131 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:12:56 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/2426119096' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:12:57.132 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:12:57.133 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:12:57.234 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:12:57.299 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:12:57.391 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:12:57.401 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:12:57.402 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:12:57.503 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:12:57.547 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:12:57.813 INFO:journalctl@ceph.osd.0.smithi026.stdout:Sep 23 21:12:57 smithi026 bash[398995]: --> Failed to activate via raw: did not find any matching OSD to activate 2023-09-23T21:12:57.813 INFO:journalctl@ceph.osd.0.smithi026.stdout:Sep 23 21:12:57 smithi026 bash[398995]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2023-09-23T21:12:57.813 INFO:journalctl@ceph.osd.0.smithi026.stdout:Sep 23 21:12:57 smithi026 bash[398995]: Running command: /usr/bin/ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev /dev/vg_nvme/lv_4 --path /var/lib/ceph/osd/ceph-0 --no-mon-config 2023-09-23T21:12:57.814 INFO:journalctl@ceph.osd.0.smithi026.stdout:Sep 23 21:12:57 smithi026 bash[398995]: Running command: /usr/bin/ln -snf /dev/vg_nvme/lv_4 /var/lib/ceph/osd/ceph-0/block 2023-09-23T21:12:57.814 INFO:journalctl@ceph.osd.0.smithi026.stdout:Sep 23 21:12:57 smithi026 bash[398995]: Running command: /usr/bin/chown -h ceph:ceph /var/lib/ceph/osd/ceph-0/block 2023-09-23T21:12:57.814 INFO:journalctl@ceph.osd.0.smithi026.stdout:Sep 23 21:12:57 smithi026 bash[398995]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2023-09-23T21:12:57.814 INFO:journalctl@ceph.osd.0.smithi026.stdout:Sep 23 21:12:57 smithi026 bash[398995]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2023-09-23T21:12:57.814 INFO:journalctl@ceph.osd.0.smithi026.stdout:Sep 23 21:12:57 smithi026 bash[398995]: --> ceph-volume lvm activate successful for osd ID: 0 2023-09-23T21:12:58.125 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:12:57 smithi070 ceph-mon[168282]: pgmap v1242: 129 pgs: 19 active+clean+remapped, 1 active+recovery_wait+remapped, 4 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 12 active+undersized, 2 active+recovering+undersized+degraded+remapped, 6 active+undersized+degraded, 8 active+recovery_wait+undersized+degraded+remapped, 5 active+recovery_wait+degraded+remapped, 71 active+clean; 4.0 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 281 op/s; 54538/194208 objects degraded (28.082%); 17414/194208 objects misplaced (8.967%); 7.0 MiB/s, 167 objects/s recovering 2023-09-23T21:12:58.131 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:12:57 smithi026 ceph-mon[175151]: pgmap v1242: 129 pgs: 19 active+clean+remapped, 1 active+recovery_wait+remapped, 4 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 12 active+undersized, 2 active+recovering+undersized+degraded+remapped, 6 active+undersized+degraded, 8 active+recovery_wait+undersized+degraded+remapped, 5 active+recovery_wait+degraded+remapped, 71 active+clean; 4.0 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 281 op/s; 54538/194208 objects degraded (28.082%); 17414/194208 objects misplaced (8.967%); 7.0 MiB/s, 167 objects/s recovering 2023-09-23T21:12:58.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:12:57 smithi026 ceph-mon[179688]: pgmap v1242: 129 pgs: 19 active+clean+remapped, 1 active+recovery_wait+remapped, 4 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 12 active+undersized, 2 active+recovering+undersized+degraded+remapped, 6 active+undersized+degraded, 8 active+recovery_wait+undersized+degraded+remapped, 5 active+recovery_wait+degraded+remapped, 71 active+clean; 4.0 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 281 op/s; 54538/194208 objects degraded (28.082%); 17414/194208 objects misplaced (8.967%); 7.0 MiB/s, 167 objects/s recovering 2023-09-23T21:12:58.295 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:12:58.298 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:12:58.299 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:12:58.399 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:12:58.435 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:12:58.661 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T21:12:58.661 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T21:12:59.082 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:12:59.085 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:12:59.085 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:12:59.186 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:12:59.227 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:12:59.277 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:12:59.287 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:12:59.287 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:12:59.388 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:12:59.481 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:12:59.570 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:12:59.584 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:12:59.585 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:12:59.685 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:12:59.778 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:12:59.856 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:12:59.862 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:12:59.862 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:12:59.881 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 21:12:59 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[164029]: ::ffff:172.21.15.70 - - [23/Sep/2023:21:12:59] "GET /metrics HTTP/1.1" 200 36448 "" "Prometheus/2.43.0" 2023-09-23T21:12:59.963 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:12:59.997 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:13:00.082 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:13:00 smithi070 ceph-mon[168282]: pgmap v1243: 129 pgs: 19 active+clean+remapped, 1 active+recovery_wait+remapped, 4 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 12 active+undersized, 2 active+recovering+undersized+degraded+remapped, 6 active+undersized+degraded, 8 active+recovery_wait+undersized+degraded+remapped, 5 active+recovery_wait+degraded+remapped, 71 active+clean; 3.9 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 212 op/s; 53983/193548 objects degraded (27.891%); 16938/193548 objects misplaced (8.751%); 9.9 MiB/s, 204 objects/s recovering 2023-09-23T21:13:00.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:13:00 smithi026 ceph-mon[175151]: pgmap v1243: 129 pgs: 19 active+clean+remapped, 1 active+recovery_wait+remapped, 4 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 12 active+undersized, 2 active+recovering+undersized+degraded+remapped, 6 active+undersized+degraded, 8 active+recovery_wait+undersized+degraded+remapped, 5 active+recovery_wait+degraded+remapped, 71 active+clean; 3.9 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 212 op/s; 53983/193548 objects degraded (27.891%); 16938/193548 objects misplaced (8.751%); 9.9 MiB/s, 204 objects/s recovering 2023-09-23T21:13:00.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:13:00 smithi026 ceph-mon[179688]: pgmap v1243: 129 pgs: 19 active+clean+remapped, 1 active+recovery_wait+remapped, 4 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 12 active+undersized, 2 active+recovering+undersized+degraded+remapped, 6 active+undersized+degraded, 8 active+recovery_wait+undersized+degraded+remapped, 5 active+recovery_wait+degraded+remapped, 71 active+clean; 3.9 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 212 op/s; 53983/193548 objects degraded (27.891%); 16938/193548 objects misplaced (8.751%); 9.9 MiB/s, 204 objects/s recovering 2023-09-23T21:13:00.563 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:13:00.567 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:13:00.567 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:13:00.662 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T21:13:00.668 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:13:00.735 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:13:00.803 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:13:00.812 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:13:00.813 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:13:00.914 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:13:00.984 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:13:00.997 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:13:01.044 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:13:01.049 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:13:01.049 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:13:01.150 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:13:01.179 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:13:01 smithi026 ceph-mon[175151]: Health check update: Degraded data redundancy: 53983/193548 objects degraded (27.891%), 26 pgs degraded, 23 pgs undersized (PG_DEGRADED) 2023-09-23T21:13:01.180 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:13:01 smithi026 ceph-mon[179688]: Health check update: Degraded data redundancy: 53983/193548 objects degraded (27.891%), 26 pgs degraded, 23 pgs undersized (PG_DEGRADED) 2023-09-23T21:13:01.182 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:13:01.298 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:13:01.301 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:13:01.302 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:13:01.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:13:01 smithi070 ceph-mon[168282]: Health check update: Degraded data redundancy: 53983/193548 objects degraded (27.891%), 26 pgs degraded, 23 pgs undersized (PG_DEGRADED) 2023-09-23T21:13:01.402 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:13:01.432 INFO:journalctl@ceph.osd.0.smithi026.stdout:Sep 23 21:13:01 smithi026 podman[399673]: 2023-09-23T21:13:01.436 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:13:01.476 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:13:01.479 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:13:01.479 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:13:01.580 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:13:01.621 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:13:01.657 INFO:teuthology.orchestra.run.smithi026.stdout:ERROR: (22) Invalid argument 2023-09-23T21:13:01.657 INFO:teuthology.orchestra.run.smithi026.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T21:13:01.672 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:13:01.675 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:13:01.675 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:13:01.776 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:13:01.815 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:13:01.820 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_ops_in_flight 2023-09-23T21:13:01.872 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:13:01.882 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:13:01.882 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:13:01.983 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:13:02.037 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:13:02.084 INFO:journalctl@ceph.osd.0.smithi026.stdout:Sep 23 21:13:01 smithi026 ceph-osd[399902]: -- 172.21.15.26:0/2544485422 <== mon.2 v2:172.21.15.26:3301/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x5630401be4e0 con 0x56304018ec00 2023-09-23T21:13:02.084 INFO:journalctl@ceph.osd.0.smithi026.stdout:Sep 23 21:13:01 smithi026 bash[399673]: e0fd0f16226788c4c0c26e037f576e4bef06571ef89e4aa05a6d58cacc9074a6 2023-09-23T21:13:02.085 INFO:journalctl@ceph.osd.0.smithi026.stdout:Sep 23 21:13:01 smithi026 systemd[1]: Started Ceph osd.0 for b3f8e4ec-5a50-11ee-8db4-212e2dc638e7. 2023-09-23T21:13:02.085 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:13:02 smithi070 ceph-mon[168282]: pgmap v1244: 129 pgs: 19 active+clean+remapped, 1 active+recovery_wait+remapped, 4 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 12 active+undersized, 2 active+recovering+undersized+degraded+remapped, 6 active+undersized+degraded, 8 active+recovery_wait+undersized+degraded+remapped, 5 active+recovery_wait+degraded+remapped, 71 active+clean; 3.9 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 275 op/s; 53309/190950 objects degraded (27.918%); 16693/190950 objects misplaced (8.742%); 11 MiB/s, 210 objects/s recovering 2023-09-23T21:13:02.085 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:13:02 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:13:02.107 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:13:02.113 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:13:02.113 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:13:02.214 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:13:02.246 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:13:02.382 INFO:journalctl@ceph.osd.0.smithi026.stdout:Sep 23 21:13:02 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd-0[399811]: 2023-09-23T21:13:02.292+0000 7fe4a7098640 -1 Falling back to public interface 2023-09-23T21:13:02.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:13:02 smithi026 ceph-mon[179688]: pgmap v1244: 129 pgs: 19 active+clean+remapped, 1 active+recovery_wait+remapped, 4 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 12 active+undersized, 2 active+recovering+undersized+degraded+remapped, 6 active+undersized+degraded, 8 active+recovery_wait+undersized+degraded+remapped, 5 active+recovery_wait+degraded+remapped, 71 active+clean; 3.9 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 275 op/s; 53309/190950 objects degraded (27.918%); 16693/190950 objects misplaced (8.742%); 11 MiB/s, 210 objects/s recovering 2023-09-23T21:13:02.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:13:02 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:13:02.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:13:02 smithi026 ceph-mon[175151]: pgmap v1244: 129 pgs: 19 active+clean+remapped, 1 active+recovery_wait+remapped, 4 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 12 active+undersized, 2 active+recovering+undersized+degraded+remapped, 6 active+undersized+degraded, 8 active+recovery_wait+undersized+degraded+remapped, 5 active+recovery_wait+degraded+remapped, 71 active+clean; 3.9 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 275 op/s; 53309/190950 objects degraded (27.918%); 16693/190950 objects misplaced (8.742%); 11 MiB/s, 210 objects/s recovering 2023-09-23T21:13:02.383 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:13:02 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:13:02.552 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:13:02.555 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_blocked_ops 2023-09-23T21:13:02.556 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:13:02.556 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:13:02.599 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:13:02.657 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:13:02.689 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:13:02.745 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:13:02.751 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:13:02.752 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:13:02.824 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:13:02.853 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:13:02.897 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:13:03.091 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:13:03.094 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:13:03.095 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:13:03.196 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:13:03.240 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:13:04.113 INFO:teuthology.orchestra.run.smithi026.stderr:noscrub is unset 2023-09-23T21:13:04.279 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:13:04.282 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:13:04.283 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:13:04.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:13:04 smithi070 ceph-mon[168282]: pgmap v1245: 129 pgs: 19 active+clean+remapped, 1 active+recovery_wait+remapped, 4 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 12 active+undersized, 2 active+recovering+undersized+degraded+remapped, 6 active+undersized+degraded, 8 active+recovery_wait+undersized+degraded+remapped, 5 active+recovery_wait+degraded+remapped, 71 active+clean; 3.9 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 279 op/s; 52562/189387 objects degraded (27.754%); 16228/189387 objects misplaced (8.569%); 12 MiB/s, 219 objects/s recovering 2023-09-23T21:13:04.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:13:04 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/903965519' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T21:13:04.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:13:04 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T21:13:04.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:13:04 smithi026 ceph-mon[179688]: pgmap v1245: 129 pgs: 19 active+clean+remapped, 1 active+recovery_wait+remapped, 4 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 12 active+undersized, 2 active+recovering+undersized+degraded+remapped, 6 active+undersized+degraded, 8 active+recovery_wait+undersized+degraded+remapped, 5 active+recovery_wait+degraded+remapped, 71 active+clean; 3.9 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 279 op/s; 52562/189387 objects degraded (27.754%); 16228/189387 objects misplaced (8.569%); 12 MiB/s, 219 objects/s recovering 2023-09-23T21:13:04.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:13:04 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/903965519' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T21:13:04.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:13:04 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T21:13:04.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:13:04 smithi026 ceph-mon[175151]: pgmap v1245: 129 pgs: 19 active+clean+remapped, 1 active+recovery_wait+remapped, 4 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 12 active+undersized, 2 active+recovering+undersized+degraded+remapped, 6 active+undersized+degraded, 8 active+recovery_wait+undersized+degraded+remapped, 5 active+recovery_wait+degraded+remapped, 71 active+clean; 3.9 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 279 op/s; 52562/189387 objects degraded (27.754%); 16228/189387 objects misplaced (8.569%); 12 MiB/s, 219 objects/s recovering 2023-09-23T21:13:04.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:13:04 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/903965519' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T21:13:04.383 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:13:04 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T21:13:04.384 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:13:04.492 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:13:04.563 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:13:04.568 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:13:04.568 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:13:04.669 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:13:04.708 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:13:04.758 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:13:04.762 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:13:04.763 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:13:04.835 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:13:04.864 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:13:04.904 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:13:05.149 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:13:05.152 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:13:05.153 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:13:05.254 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:13:05.339 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1: {} 2023-09-23T21:13:05.349 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:13:05 smithi026 ceph-mon[179688]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:13:05.350 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:13:05 smithi026 ceph-mon[179688]: failed to encode map e489 with expected crc 2023-09-23T21:13:05.350 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:13:05 smithi026 ceph-mon[179688]: failed to encode map e489 with expected crc 2023-09-23T21:13:05.350 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:13:05 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T21:13:05.350 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:13:05 smithi026 ceph-mon[179688]: osdmap e489: 8 total, 7 up, 6 in 2023-09-23T21:13:05.350 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:13:05 smithi026 ceph-mon[179688]: failed to encode map e489 with expected crc 2023-09-23T21:13:05.350 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:13:05 smithi026 ceph-mon[179688]: failed to encode map e489 with expected crc 2023-09-23T21:13:05.350 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:13:05 smithi026 ceph-mon[175151]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:13:05.350 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:13:05 smithi026 ceph-mon[175151]: failed to encode map e489 with expected crc 2023-09-23T21:13:05.350 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:13:05 smithi026 ceph-mon[175151]: failed to encode map e489 with expected crc 2023-09-23T21:13:05.350 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:13:05 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T21:13:05.351 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:13:05 smithi026 ceph-mon[175151]: osdmap e489: 8 total, 7 up, 6 in 2023-09-23T21:13:05.351 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:13:05 smithi026 ceph-mon[175151]: failed to encode map e489 with expected crc 2023-09-23T21:13:05.351 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:13:05 smithi026 ceph-mon[175151]: failed to encode map e489 with expected crc 2023-09-23T21:13:05.351 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:13:05 smithi070 ceph-mon[168282]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:13:05.351 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:13:05 smithi070 ceph-mon[168282]: failed to encode map e489 with expected crc 2023-09-23T21:13:05.351 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:13:05 smithi070 ceph-mon[168282]: failed to encode map e489 with expected crc 2023-09-23T21:13:05.351 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:13:05 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T21:13:05.351 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:13:05 smithi070 ceph-mon[168282]: osdmap e489: 8 total, 7 up, 6 in 2023-09-23T21:13:05.352 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:13:05 smithi070 ceph-mon[168282]: failed to encode map e489 with expected crc 2023-09-23T21:13:05.352 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:13:05 smithi070 ceph-mon[168282]: failed to encode map e489 with expected crc 2023-09-23T21:13:05.352 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:13:05.368 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2: {} 2023-09-23T21:13:05.393 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3: {} 2023-09-23T21:13:05.415 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:13:05.422 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:13:05.422 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:13:05.424 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4: {} 2023-09-23T21:13:05.467 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T21:13:05.481 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6: {} 2023-09-23T21:13:05.517 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T21:13:05.517 INFO:teuthology.orchestra.run.smithi026.stderr:Error ENXIO: problem getting command descriptions from osd.0 2023-09-23T21:13:05.517 INFO:teuthology.orchestra.run.smithi026.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T21:13:05.517 INFO:teuthology.orchestra.run.smithi026.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T21:13:05.517 INFO:teuthology.orchestra.run.smithi026.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T21:13:05.517 INFO:teuthology.orchestra.run.smithi026.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T21:13:05.517 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T21:13:05.517 INFO:teuthology.orchestra.run.smithi026.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T21:13:05.518 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T21:13:05.523 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:13:05.616 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:13:05.695 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:13:05.702 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:13:05.702 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:13:05.803 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:13:05.862 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:13:05.924 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:13:05.938 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:13:05.939 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:13:06.039 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:13:06.116 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:13:06.209 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:13:06.220 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:13:06.220 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:13:06.321 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:13:06 smithi026 ceph-mon[175151]: pgmap v1247: 129 pgs: 19 active+clean+remapped, 1 active+recovery_wait+remapped, 4 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 12 active+undersized, 2 active+recovering+undersized+degraded+remapped, 6 active+undersized+degraded, 8 active+recovery_wait+undersized+degraded+remapped, 5 active+recovery_wait+degraded+remapped, 71 active+clean; 3.8 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 218 op/s; 52399/188607 objects degraded (27.782%); 16154/188607 objects misplaced (8.565%); 8.8 MiB/s, 141 objects/s recovering 2023-09-23T21:13:06.321 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:13:06 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/1432280638' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:13:06.321 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:13:06 smithi026 ceph-mon[175151]: Health check update: Degraded data redundancy: 52399/188607 objects degraded (27.782%), 26 pgs degraded, 23 pgs undersized (PG_DEGRADED) 2023-09-23T21:13:06.321 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:13:06 smithi026 ceph-mon[179688]: pgmap v1247: 129 pgs: 19 active+clean+remapped, 1 active+recovery_wait+remapped, 4 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 12 active+undersized, 2 active+recovering+undersized+degraded+remapped, 6 active+undersized+degraded, 8 active+recovery_wait+undersized+degraded+remapped, 5 active+recovery_wait+degraded+remapped, 71 active+clean; 3.8 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 218 op/s; 52399/188607 objects degraded (27.782%); 16154/188607 objects misplaced (8.565%); 8.8 MiB/s, 141 objects/s recovering 2023-09-23T21:13:06.321 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:13:06 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/1432280638' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:13:06.321 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:13:06 smithi026 ceph-mon[179688]: Health check update: Degraded data redundancy: 52399/188607 objects degraded (27.782%), 26 pgs degraded, 23 pgs undersized (PG_DEGRADED) 2023-09-23T21:13:06.322 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:13:06.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:13:06 smithi070 ceph-mon[168282]: pgmap v1247: 129 pgs: 19 active+clean+remapped, 1 active+recovery_wait+remapped, 4 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 12 active+undersized, 2 active+recovering+undersized+degraded+remapped, 6 active+undersized+degraded, 8 active+recovery_wait+undersized+degraded+remapped, 5 active+recovery_wait+degraded+remapped, 71 active+clean; 3.8 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 218 op/s; 52399/188607 objects degraded (27.782%); 16154/188607 objects misplaced (8.565%); 8.8 MiB/s, 141 objects/s recovering 2023-09-23T21:13:06.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:13:06 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/1432280638' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:13:06.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:13:06 smithi070 ceph-mon[168282]: Health check update: Degraded data redundancy: 52399/188607 objects degraded (27.782%), 26 pgs degraded, 23 pgs undersized (PG_DEGRADED) 2023-09-23T21:13:06.403 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:13:06.451 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:13:06.466 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:13:06.466 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:13:06.567 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:13:06.677 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:13:06.761 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:13:06.775 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:13:06.776 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:13:06.856 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T21:13:06.857 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T21:13:06.867 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T21:13:06.877 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:13:06.946 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:13:07.058 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:13:07.061 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:13:07.062 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:13:07.163 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:13:07.212 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:13:07.303 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:13:07.311 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:13:07.312 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:13:07.318 INFO:teuthology.orchestra.run.smithi026.stderr:no valid command found; 10 closest matches: 2023-09-23T21:13:07.318 INFO:teuthology.orchestra.run.smithi026.stderr:0 2023-09-23T21:13:07.318 INFO:teuthology.orchestra.run.smithi026.stderr:1 2023-09-23T21:13:07.318 INFO:teuthology.orchestra.run.smithi026.stderr:2 2023-09-23T21:13:07.318 INFO:teuthology.orchestra.run.smithi026.stderr:abort 2023-09-23T21:13:07.318 INFO:teuthology.orchestra.run.smithi026.stderr:assert 2023-09-23T21:13:07.318 INFO:teuthology.orchestra.run.smithi026.stderr:bluefs debug_inject_read_zeros 2023-09-23T21:13:07.318 INFO:teuthology.orchestra.run.smithi026.stderr:bluefs files list 2023-09-23T21:13:07.318 INFO:teuthology.orchestra.run.smithi026.stderr:bluefs stats 2023-09-23T21:13:07.318 INFO:teuthology.orchestra.run.smithi026.stderr:bluestore allocator dump block 2023-09-23T21:13:07.319 INFO:teuthology.orchestra.run.smithi026.stderr:bluestore allocator fragmentation block 2023-09-23T21:13:07.319 INFO:teuthology.orchestra.run.smithi026.stderr:admin_socket: invalid command 2023-09-23T21:13:07.413 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:13:07.453 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:13:07.821 DEBUG:teuthology.orchestra.run:got remote process result: 22 2023-09-23T21:13:07.822 INFO:tasks.cephadm.ceph_manager.ceph:waiting on admin_socket for osd-0, ['dump_ops_in_flight'] 2023-09-23T21:13:07.918 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:13:07.923 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:13:07.924 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:13:08.011 INFO:teuthology.orchestra.run.smithi026.stdout:{ 2023-09-23T21:13:08.011 INFO:teuthology.orchestra.run.smithi026.stdout: "ops": [], 2023-09-23T21:13:08.012 INFO:teuthology.orchestra.run.smithi026.stdout: "complaint_time": 30, 2023-09-23T21:13:08.012 INFO:teuthology.orchestra.run.smithi026.stdout: "num_blocked_ops": 0 2023-09-23T21:13:08.012 INFO:teuthology.orchestra.run.smithi026.stdout:} 2023-09-23T21:13:08.025 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:13:08.037 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:13:08.079 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:13:08.140 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:13:08.152 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:13:08.153 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:13:08.254 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:13:08.308 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:13:08.351 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:13:08 smithi026 ceph-mon[179688]: pgmap v1248: 129 pgs: 19 active+clean+remapped, 1 active+recovery_wait+remapped, 4 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 12 active+undersized, 2 active+recovering+undersized+degraded+remapped, 6 active+undersized+degraded, 8 active+recovery_wait+undersized+degraded+remapped, 5 active+recovery_wait+degraded+remapped, 71 active+clean; 3.8 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 277 op/s; 51674/185877 objects degraded (27.800%); 15915/185877 objects misplaced (8.562%); 11 MiB/s, 174 objects/s recovering 2023-09-23T21:13:08.353 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:13:08.354 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:13:08 smithi026 ceph-mon[175151]: pgmap v1248: 129 pgs: 19 active+clean+remapped, 1 active+recovery_wait+remapped, 4 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 12 active+undersized, 2 active+recovering+undersized+degraded+remapped, 6 active+undersized+degraded, 8 active+recovery_wait+undersized+degraded+remapped, 5 active+recovery_wait+degraded+remapped, 71 active+clean; 3.8 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 277 op/s; 51674/185877 objects degraded (27.800%); 15915/185877 objects misplaced (8.562%); 11 MiB/s, 174 objects/s recovering 2023-09-23T21:13:08.361 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:13:08 smithi070 ceph-mon[168282]: pgmap v1248: 129 pgs: 19 active+clean+remapped, 1 active+recovery_wait+remapped, 4 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 12 active+undersized, 2 active+recovering+undersized+degraded+remapped, 6 active+undersized+degraded, 8 active+recovery_wait+undersized+degraded+remapped, 5 active+recovery_wait+degraded+remapped, 71 active+clean; 3.8 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 277 op/s; 51674/185877 objects degraded (27.800%); 15915/185877 objects misplaced (8.562%); 11 MiB/s, 174 objects/s recovering 2023-09-23T21:13:08.362 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:13:08.362 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:13:08.463 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:13:08.505 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:13:08.566 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:13:08.571 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:13:08.571 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:13:08.672 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:13:08.724 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:13:08.777 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:13:08.780 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:13:08.781 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:13:08.796 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_historic_ops 2023-09-23T21:13:08.858 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T21:13:08.882 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:13:08.910 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:13:08.955 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:13:08.960 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:13:08.961 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:13:09.062 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:13:09.095 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:13:09.133 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:13:09.136 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:13:09.136 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:13:09.237 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:13:09.296 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:13:09.372 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:13:09 smithi070 ceph-mon[168282]: failed to encode map e490 with expected crc 2023-09-23T21:13:09.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:13:09 smithi070 ceph-mon[168282]: failed to encode map e490 with expected crc 2023-09-23T21:13:09.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:13:09 smithi070 ceph-mon[168282]: osdmap e490: 8 total, 7 up, 6 in 2023-09-23T21:13:09.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:13:09 smithi070 ceph-mon[168282]: failed to encode map e490 with expected crc 2023-09-23T21:13:09.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:13:09 smithi070 ceph-mon[168282]: failed to encode map e490 with expected crc 2023-09-23T21:13:09.381 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:13:09 smithi026 ceph-mon[179688]: failed to encode map e490 with expected crc 2023-09-23T21:13:09.381 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:13:09 smithi026 ceph-mon[179688]: failed to encode map e490 with expected crc 2023-09-23T21:13:09.381 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:13:09 smithi026 ceph-mon[179688]: osdmap e490: 8 total, 7 up, 6 in 2023-09-23T21:13:09.381 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:13:09 smithi026 ceph-mon[179688]: failed to encode map e490 with expected crc 2023-09-23T21:13:09.381 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:13:09 smithi026 ceph-mon[179688]: failed to encode map e490 with expected crc 2023-09-23T21:13:09.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:13:09 smithi026 ceph-mon[175151]: failed to encode map e490 with expected crc 2023-09-23T21:13:09.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:13:09 smithi026 ceph-mon[175151]: failed to encode map e490 with expected crc 2023-09-23T21:13:09.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:13:09 smithi026 ceph-mon[175151]: osdmap e490: 8 total, 7 up, 6 in 2023-09-23T21:13:09.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:13:09 smithi026 ceph-mon[175151]: failed to encode map e490 with expected crc 2023-09-23T21:13:09.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:13:09 smithi026 ceph-mon[175151]: failed to encode map e490 with expected crc 2023-09-23T21:13:09.601 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:13:09.610 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:13:09.610 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:13:09.711 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:13:09.749 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:13:09.794 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:13:09.798 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:13:09.798 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:13:09.881 INFO:journalctl@ceph.osd.0.smithi026.stdout:Sep 23 21:13:09 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd-0[399811]: 2023-09-23T21:13:09.718+0000 7fe4a7098640 -1 osd.0 464 log_to_monitors true 2023-09-23T21:13:09.882 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 21:13:09 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[164029]: ::ffff:172.21.15.70 - - [23/Sep/2023:21:13:09] "GET /metrics HTTP/1.1" 200 36444 "" "Prometheus/2.43.0" 2023-09-23T21:13:09.899 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:13:09.983 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:13:10.079 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:13:10.084 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:13:10.085 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:13:10.186 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:13:10.235 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:13:10 smithi070 ceph-mon[168282]: pgmap v1250: 129 pgs: 19 active+clean+remapped, 1 active+recovery_wait+remapped, 4 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 12 active+undersized, 1 active+recovering+undersized+degraded+remapped, 1 active+undersized+remapped, 6 active+undersized+degraded, 8 active+recovery_wait+undersized+degraded+remapped, 5 active+recovery_wait+degraded+remapped, 71 active+clean; 3.8 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 240 op/s; 50707/185178 objects degraded (27.383%); 15519/185178 objects misplaced (8.381%); 14 MiB/s, 225 objects/s recovering 2023-09-23T21:13:10.235 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:13:10 smithi070 ceph-mon[168282]: failed to encode map e491 with expected crc 2023-09-23T21:13:10.236 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:13:10 smithi070 ceph-mon[168282]: failed to encode map e491 with expected crc 2023-09-23T21:13:10.236 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:13:10 smithi070 ceph-mon[168282]: failed to encode map e491 with expected crc 2023-09-23T21:13:10.236 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:13:10 smithi070 ceph-mon[168282]: osdmap e491: 8 total, 7 up, 6 in 2023-09-23T21:13:10.236 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:13:10 smithi070 ceph-mon[168282]: failed to encode map e491 with expected crc 2023-09-23T21:13:10.236 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:13:10 smithi070 ceph-mon[168282]: from='osd.0 [v2:172.21.15.26:6802/1823091461,v1:172.21.15.26:6803/1823091461]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2023-09-23T21:13:10.238 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:13:10.301 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:13:10.305 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:13:10.305 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:13:10.406 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:13:10.445 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:13:10.477 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:13:10.494 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:13:10.498 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:13:10.498 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:13:10.599 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:13:10.631 INFO:journalctl@ceph.osd.0.smithi026.stdout:Sep 23 21:13:10 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd-0[399811]: 2023-09-23T21:13:10.163+0000 7fe4954ff700 -1 osd.0 464 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-09-23T21:13:10.631 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:13:10 smithi026 ceph-mon[179688]: pgmap v1250: 129 pgs: 19 active+clean+remapped, 1 active+recovery_wait+remapped, 4 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 12 active+undersized, 1 active+recovering+undersized+degraded+remapped, 1 active+undersized+remapped, 6 active+undersized+degraded, 8 active+recovery_wait+undersized+degraded+remapped, 5 active+recovery_wait+degraded+remapped, 71 active+clean; 3.8 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 240 op/s; 50707/185178 objects degraded (27.383%); 15519/185178 objects misplaced (8.381%); 14 MiB/s, 225 objects/s recovering 2023-09-23T21:13:10.631 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:13:10 smithi026 ceph-mon[179688]: failed to encode map e491 with expected crc 2023-09-23T21:13:10.631 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:13:10 smithi026 ceph-mon[179688]: failed to encode map e491 with expected crc 2023-09-23T21:13:10.631 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:13:10 smithi026 ceph-mon[179688]: failed to encode map e491 with expected crc 2023-09-23T21:13:10.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:13:10 smithi026 ceph-mon[179688]: osdmap e491: 8 total, 7 up, 6 in 2023-09-23T21:13:10.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:13:10 smithi026 ceph-mon[179688]: failed to encode map e491 with expected crc 2023-09-23T21:13:10.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:13:10 smithi026 ceph-mon[179688]: from='osd.0 [v2:172.21.15.26:6802/1823091461,v1:172.21.15.26:6803/1823091461]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2023-09-23T21:13:10.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:13:10 smithi026 ceph-mon[175151]: pgmap v1250: 129 pgs: 19 active+clean+remapped, 1 active+recovery_wait+remapped, 4 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 12 active+undersized, 1 active+recovering+undersized+degraded+remapped, 1 active+undersized+remapped, 6 active+undersized+degraded, 8 active+recovery_wait+undersized+degraded+remapped, 5 active+recovery_wait+degraded+remapped, 71 active+clean; 3.8 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 240 op/s; 50707/185178 objects degraded (27.383%); 15519/185178 objects misplaced (8.381%); 14 MiB/s, 225 objects/s recovering 2023-09-23T21:13:10.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:13:10 smithi026 ceph-mon[175151]: failed to encode map e491 with expected crc 2023-09-23T21:13:10.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:13:10 smithi026 ceph-mon[175151]: failed to encode map e491 with expected crc 2023-09-23T21:13:10.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:13:10 smithi026 ceph-mon[175151]: failed to encode map e491 with expected crc 2023-09-23T21:13:10.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:13:10 smithi026 ceph-mon[175151]: osdmap e491: 8 total, 7 up, 6 in 2023-09-23T21:13:10.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:13:10 smithi026 ceph-mon[175151]: failed to encode map e491 with expected crc 2023-09-23T21:13:10.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:13:10 smithi026 ceph-mon[175151]: from='osd.0 [v2:172.21.15.26:6802/1823091461,v1:172.21.15.26:6803/1823091461]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2023-09-23T21:13:10.636 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:13:10.686 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:13:10.691 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:13:10.691 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:13:10.756 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:13:10.793 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:13:10.883 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:13:10.945 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:13:10.948 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:13:10.948 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:13:11.049 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:13:11.093 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:13:11.163 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:13:11.172 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:13:11 smithi070 ceph-mon[168282]: failed to encode map e492 with expected crc 2023-09-23T21:13:11.173 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:13:11 smithi070 ceph-mon[168282]: failed to encode map e492 with expected crc 2023-09-23T21:13:11.173 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:13:11 smithi070 ceph-mon[168282]: from='osd.0 [v2:172.21.15.26:6802/1823091461,v1:172.21.15.26:6803/1823091461]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2023-09-23T21:13:11.173 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:13:11 smithi070 ceph-mon[168282]: osdmap e492: 8 total, 7 up, 6 in 2023-09-23T21:13:11.173 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:13:11 smithi070 ceph-mon[168282]: failed to encode map e492 with expected crc 2023-09-23T21:13:11.173 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:13:11 smithi070 ceph-mon[168282]: from='osd.0 [v2:172.21.15.26:6802/1823091461,v1:172.21.15.26:6803/1823091461]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi026", "root=default"]}]: dispatch 2023-09-23T21:13:11.173 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:13:11 smithi070 ceph-mon[168282]: Health check update: Degraded data redundancy: 50707/185178 objects degraded (27.383%), 25 pgs degraded, 23 pgs undersized (PG_DEGRADED) 2023-09-23T21:13:11.173 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:13:11 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/1272454766' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:13:11.173 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:13:11 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:13:11.174 INFO:teuthology.orchestra.run.smithi026.stderr:nodeep-scrub is unset 2023-09-23T21:13:11.181 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:13:11.182 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:13:11.263 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:13:11 smithi026 ceph-mon[175151]: failed to encode map e492 with expected crc 2023-09-23T21:13:11.263 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:13:11 smithi026 ceph-mon[175151]: failed to encode map e492 with expected crc 2023-09-23T21:13:11.263 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:13:11 smithi026 ceph-mon[175151]: from='osd.0 [v2:172.21.15.26:6802/1823091461,v1:172.21.15.26:6803/1823091461]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2023-09-23T21:13:11.263 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:13:11 smithi026 ceph-mon[175151]: osdmap e492: 8 total, 7 up, 6 in 2023-09-23T21:13:11.263 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:13:11 smithi026 ceph-mon[175151]: failed to encode map e492 with expected crc 2023-09-23T21:13:11.263 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:13:11 smithi026 ceph-mon[175151]: from='osd.0 [v2:172.21.15.26:6802/1823091461,v1:172.21.15.26:6803/1823091461]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi026", "root=default"]}]: dispatch 2023-09-23T21:13:11.264 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:13:11 smithi026 ceph-mon[175151]: Health check update: Degraded data redundancy: 50707/185178 objects degraded (27.383%), 25 pgs degraded, 23 pgs undersized (PG_DEGRADED) 2023-09-23T21:13:11.264 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:13:11 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/1272454766' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:13:11.264 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:13:11 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:13:11.264 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:13:11 smithi026 ceph-mon[179688]: failed to encode map e492 with expected crc 2023-09-23T21:13:11.264 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:13:11 smithi026 ceph-mon[179688]: failed to encode map e492 with expected crc 2023-09-23T21:13:11.264 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:13:11 smithi026 ceph-mon[179688]: from='osd.0 [v2:172.21.15.26:6802/1823091461,v1:172.21.15.26:6803/1823091461]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2023-09-23T21:13:11.264 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:13:11 smithi026 ceph-mon[179688]: osdmap e492: 8 total, 7 up, 6 in 2023-09-23T21:13:11.264 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:13:11 smithi026 ceph-mon[179688]: failed to encode map e492 with expected crc 2023-09-23T21:13:11.264 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:13:11 smithi026 ceph-mon[179688]: from='osd.0 [v2:172.21.15.26:6802/1823091461,v1:172.21.15.26:6803/1823091461]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi026", "root=default"]}]: dispatch 2023-09-23T21:13:11.265 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:13:11 smithi026 ceph-mon[179688]: Health check update: Degraded data redundancy: 50707/185178 objects degraded (27.383%), 25 pgs degraded, 23 pgs undersized (PG_DEGRADED) 2023-09-23T21:13:11.265 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:13:11 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/1272454766' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:13:11.265 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:13:11 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:13:11.282 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:13:11.310 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:13:11.350 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:13:11.353 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:13:11.354 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:13:11.455 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:13:11.481 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:13:11.523 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:13:11.525 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:13:11.526 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:13:11.626 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:13:11.652 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:13:11.685 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:13:11.687 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:13:11.688 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:13:11.789 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:13:11.823 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:13:12.551 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:13:12 smithi026 ceph-mon[175151]: pgmap v1253: 129 pgs: 19 active+clean+remapped, 1 active+recovery_wait+remapped, 4 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 12 active+undersized, 1 active+recovering+undersized+degraded+remapped, 1 active+undersized+remapped, 6 active+undersized+degraded, 8 active+recovery_wait+undersized+degraded+remapped, 5 active+recovery_wait+degraded+remapped, 71 active+clean; 3.7 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 348 op/s; 49988/182331 objects degraded (27.416%); 15247/182331 objects misplaced (8.362%); 17 MiB/s, 268 objects/s recovering 2023-09-23T21:13:12.552 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:13:12 smithi026 ceph-mon[175151]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T21:13:12.552 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:13:12 smithi026 ceph-mon[175151]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-09-23T21:13:12.552 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:13:12 smithi026 ceph-mon[175151]: failed to encode map e493 with expected crc 2023-09-23T21:13:12.552 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:13:12 smithi026 ceph-mon[175151]: osd.0 [v2:172.21.15.26:6802/1823091461,v1:172.21.15.26:6803/1823091461] boot 2023-09-23T21:13:12.552 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:13:12 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T21:13:12.552 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:13:12 smithi026 ceph-mon[175151]: osdmap e493: 8 total, 8 up, 6 in 2023-09-23T21:13:12.552 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:13:12 smithi026 ceph-mon[175151]: failed to encode map e493 with expected crc 2023-09-23T21:13:12.552 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:13:12 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-23T21:13:12.552 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:13:12 smithi026 ceph-mon[179688]: pgmap v1253: 129 pgs: 19 active+clean+remapped, 1 active+recovery_wait+remapped, 4 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 12 active+undersized, 1 active+recovering+undersized+degraded+remapped, 1 active+undersized+remapped, 6 active+undersized+degraded, 8 active+recovery_wait+undersized+degraded+remapped, 5 active+recovery_wait+degraded+remapped, 71 active+clean; 3.7 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 348 op/s; 49988/182331 objects degraded (27.416%); 15247/182331 objects misplaced (8.362%); 17 MiB/s, 268 objects/s recovering 2023-09-23T21:13:12.553 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:13:12 smithi026 ceph-mon[179688]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T21:13:12.553 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:13:12 smithi026 ceph-mon[179688]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-09-23T21:13:12.553 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:13:12 smithi026 ceph-mon[179688]: failed to encode map e493 with expected crc 2023-09-23T21:13:12.553 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:13:12 smithi026 ceph-mon[179688]: osd.0 [v2:172.21.15.26:6802/1823091461,v1:172.21.15.26:6803/1823091461] boot 2023-09-23T21:13:12.553 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:13:12 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T21:13:12.553 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:13:12 smithi026 ceph-mon[179688]: osdmap e493: 8 total, 8 up, 6 in 2023-09-23T21:13:12.553 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:13:12 smithi026 ceph-mon[179688]: failed to encode map e493 with expected crc 2023-09-23T21:13:12.553 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:13:12 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-23T21:13:12.615 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:13:12.619 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:13:12.619 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:13:12.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:13:12 smithi070 ceph-mon[168282]: pgmap v1253: 129 pgs: 19 active+clean+remapped, 1 active+recovery_wait+remapped, 4 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 12 active+undersized, 1 active+recovering+undersized+degraded+remapped, 1 active+undersized+remapped, 6 active+undersized+degraded, 8 active+recovery_wait+undersized+degraded+remapped, 5 active+recovery_wait+degraded+remapped, 71 active+clean; 3.7 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 348 op/s; 49988/182331 objects degraded (27.416%); 15247/182331 objects misplaced (8.362%); 17 MiB/s, 268 objects/s recovering 2023-09-23T21:13:12.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:13:12 smithi070 ceph-mon[168282]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T21:13:12.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:13:12 smithi070 ceph-mon[168282]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-09-23T21:13:12.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:13:12 smithi070 ceph-mon[168282]: failed to encode map e493 with expected crc 2023-09-23T21:13:12.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:13:12 smithi070 ceph-mon[168282]: osd.0 [v2:172.21.15.26:6802/1823091461,v1:172.21.15.26:6803/1823091461] boot 2023-09-23T21:13:12.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:13:12 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T21:13:12.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:13:12 smithi070 ceph-mon[168282]: osdmap e493: 8 total, 8 up, 6 in 2023-09-23T21:13:12.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:13:12 smithi070 ceph-mon[168282]: failed to encode map e493 with expected crc 2023-09-23T21:13:12.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:13:12 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-23T21:13:12.720 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:13:12.754 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:13:12.823 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_ops_in_flight 2023-09-23T21:13:12.901 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:13:12.906 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:13:12.906 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:13:13.007 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:13:13.033 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:13:13.064 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:13:13.067 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:13:13.067 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:13:13.168 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:13:13.260 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:13:13.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:13:13 smithi070 ceph-mon[168282]: failed to encode map e494 with expected crc 2023-09-23T21:13:13.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:13:13 smithi070 ceph-mon[168282]: failed to encode map e494 with expected crc 2023-09-23T21:13:13.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:13:13 smithi070 ceph-mon[168282]: osdmap e494: 8 total, 8 up, 6 in 2023-09-23T21:13:13.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:13:13 smithi070 ceph-mon[168282]: failed to encode map e494 with expected crc 2023-09-23T21:13:13.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:13:13 smithi070 ceph-mon[168282]: failed to encode map e494 with expected crc 2023-09-23T21:13:13.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:13:13 smithi070 ceph-mon[168282]: failed to encode map e494 with expected crc 2023-09-23T21:13:13.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:13:13 smithi070 ceph-mon[168282]: failed to encode map e494 with expected crc 2023-09-23T21:13:13.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:13:13 smithi070 ceph-mon[168282]: failed to encode map e494 with expected crc 2023-09-23T21:13:13.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:13:13 smithi070 ceph-mon[168282]: failed to encode map e494 with expected crc 2023-09-23T21:13:13.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:13:13 smithi070 ceph-mon[168282]: failed to encode map e494 with expected crc 2023-09-23T21:13:13.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:13:13 smithi070 ceph-mon[168282]: failed to encode map e494 with expected crc 2023-09-23T21:13:13.631 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:13:13 smithi026 ceph-mon[175151]: failed to encode map e494 with expected crc 2023-09-23T21:13:13.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:13:13 smithi026 ceph-mon[175151]: failed to encode map e494 with expected crc 2023-09-23T21:13:13.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:13:13 smithi026 ceph-mon[175151]: osdmap e494: 8 total, 8 up, 6 in 2023-09-23T21:13:13.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:13:13 smithi026 ceph-mon[175151]: failed to encode map e494 with expected crc 2023-09-23T21:13:13.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:13:13 smithi026 ceph-mon[175151]: failed to encode map e494 with expected crc 2023-09-23T21:13:13.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:13:13 smithi026 ceph-mon[175151]: failed to encode map e494 with expected crc 2023-09-23T21:13:13.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:13:13 smithi026 ceph-mon[175151]: failed to encode map e494 with expected crc 2023-09-23T21:13:13.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:13:13 smithi026 ceph-mon[175151]: failed to encode map e494 with expected crc 2023-09-23T21:13:13.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:13:13 smithi026 ceph-mon[175151]: failed to encode map e494 with expected crc 2023-09-23T21:13:13.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:13:13 smithi026 ceph-mon[175151]: failed to encode map e494 with expected crc 2023-09-23T21:13:13.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:13:13 smithi026 ceph-mon[175151]: failed to encode map e494 with expected crc 2023-09-23T21:13:13.633 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:13:13 smithi026 ceph-mon[179688]: failed to encode map e494 with expected crc 2023-09-23T21:13:13.633 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:13:13 smithi026 ceph-mon[179688]: failed to encode map e494 with expected crc 2023-09-23T21:13:13.633 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:13:13 smithi026 ceph-mon[179688]: osdmap e494: 8 total, 8 up, 6 in 2023-09-23T21:13:13.633 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:13:13 smithi026 ceph-mon[179688]: failed to encode map e494 with expected crc 2023-09-23T21:13:13.633 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:13:13 smithi026 ceph-mon[179688]: failed to encode map e494 with expected crc 2023-09-23T21:13:13.633 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:13:13 smithi026 ceph-mon[179688]: failed to encode map e494 with expected crc 2023-09-23T21:13:13.633 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:13:13 smithi026 ceph-mon[179688]: failed to encode map e494 with expected crc 2023-09-23T21:13:13.633 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:13:13 smithi026 ceph-mon[179688]: failed to encode map e494 with expected crc 2023-09-23T21:13:13.633 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:13:13 smithi026 ceph-mon[179688]: failed to encode map e494 with expected crc 2023-09-23T21:13:13.634 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:13:13 smithi026 ceph-mon[179688]: failed to encode map e494 with expected crc 2023-09-23T21:13:13.634 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:13:13 smithi026 ceph-mon[179688]: failed to encode map e494 with expected crc 2023-09-23T21:13:14.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:13:14 smithi070 ceph-mon[168282]: pgmap v1256: 129 pgs: 7 remapped+peering, 5 peering, 14 active+clean+remapped, 1 active+recovery_wait+remapped, 3 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 10 active+undersized, 2 active+recovering+undersized+degraded+remapped, 3 active+undersized+degraded, 6 active+recovery_wait+undersized+degraded+remapped, 5 active+recovery_wait+degraded+remapped, 72 active+clean; 3.7 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 332 op/s; 36582/181185 objects degraded (20.190%); 15247/181185 objects misplaced (8.415%); 13 MiB/s, 209 objects/s recovering 2023-09-23T21:13:14.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:13:14 smithi070 ceph-mon[168282]: failed to encode map e495 with expected crc 2023-09-23T21:13:14.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:13:14 smithi070 ceph-mon[168282]: failed to encode map e495 with expected crc 2023-09-23T21:13:14.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:13:14 smithi070 ceph-mon[168282]: osdmap e495: 8 total, 8 up, 6 in 2023-09-23T21:13:14.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:13:14 smithi070 ceph-mon[168282]: failed to encode map e495 with expected crc 2023-09-23T21:13:14.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:13:14 smithi070 ceph-mon[168282]: failed to encode map e495 with expected crc 2023-09-23T21:13:14.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:13:14 smithi070 ceph-mon[168282]: 10.d continuing backfill to osd.7 from (435'2195,492'6187] MIN to 492'6187 2023-09-23T21:13:14.631 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:13:14 smithi026 ceph-mon[179688]: pgmap v1256: 129 pgs: 7 remapped+peering, 5 peering, 14 active+clean+remapped, 1 active+recovery_wait+remapped, 3 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 10 active+undersized, 2 active+recovering+undersized+degraded+remapped, 3 active+undersized+degraded, 6 active+recovery_wait+undersized+degraded+remapped, 5 active+recovery_wait+degraded+remapped, 72 active+clean; 3.7 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 332 op/s; 36582/181185 objects degraded (20.190%); 15247/181185 objects misplaced (8.415%); 13 MiB/s, 209 objects/s recovering 2023-09-23T21:13:14.631 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:13:14 smithi026 ceph-mon[179688]: failed to encode map e495 with expected crc 2023-09-23T21:13:14.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:13:14 smithi026 ceph-mon[179688]: failed to encode map e495 with expected crc 2023-09-23T21:13:14.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:13:14 smithi026 ceph-mon[179688]: osdmap e495: 8 total, 8 up, 6 in 2023-09-23T21:13:14.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:13:14 smithi026 ceph-mon[179688]: failed to encode map e495 with expected crc 2023-09-23T21:13:14.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:13:14 smithi026 ceph-mon[179688]: failed to encode map e495 with expected crc 2023-09-23T21:13:14.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:13:14 smithi026 ceph-mon[179688]: 10.d continuing backfill to osd.7 from (435'2195,492'6187] MIN to 492'6187 2023-09-23T21:13:14.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:13:14 smithi026 ceph-mon[175151]: pgmap v1256: 129 pgs: 7 remapped+peering, 5 peering, 14 active+clean+remapped, 1 active+recovery_wait+remapped, 3 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 10 active+undersized, 2 active+recovering+undersized+degraded+remapped, 3 active+undersized+degraded, 6 active+recovery_wait+undersized+degraded+remapped, 5 active+recovery_wait+degraded+remapped, 72 active+clean; 3.7 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 332 op/s; 36582/181185 objects degraded (20.190%); 15247/181185 objects misplaced (8.415%); 13 MiB/s, 209 objects/s recovering 2023-09-23T21:13:14.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:13:14 smithi026 ceph-mon[175151]: failed to encode map e495 with expected crc 2023-09-23T21:13:14.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:13:14 smithi026 ceph-mon[175151]: failed to encode map e495 with expected crc 2023-09-23T21:13:14.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:13:14 smithi026 ceph-mon[175151]: osdmap e495: 8 total, 8 up, 6 in 2023-09-23T21:13:14.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:13:14 smithi026 ceph-mon[175151]: failed to encode map e495 with expected crc 2023-09-23T21:13:14.633 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:13:14 smithi026 ceph-mon[175151]: failed to encode map e495 with expected crc 2023-09-23T21:13:14.633 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:13:14 smithi026 ceph-mon[175151]: 10.d continuing backfill to osd.7 from (435'2195,492'6187] MIN to 492'6187 2023-09-23T21:13:14.763 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:13:14.767 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:13:14.768 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:13:14.868 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:13:14.902 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:13:15.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:13:15 smithi070 ceph-mon[168282]: failed to encode map e496 with expected crc 2023-09-23T21:13:15.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:13:15 smithi070 ceph-mon[168282]: failed to encode map e496 with expected crc 2023-09-23T21:13:15.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:13:15 smithi070 ceph-mon[168282]: failed to encode map e496 with expected crc 2023-09-23T21:13:15.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:13:15 smithi070 ceph-mon[168282]: osdmap e496: 8 total, 8 up, 6 in 2023-09-23T21:13:15.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:13:15 smithi070 ceph-mon[168282]: failed to encode map e496 with expected crc 2023-09-23T21:13:15.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:13:15 smithi070 ceph-mon[168282]: failed to encode map e496 with expected crc 2023-09-23T21:13:15.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:13:15 smithi070 ceph-mon[168282]: failed to encode map e496 with expected crc 2023-09-23T21:13:15.631 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:13:15 smithi026 ceph-mon[179688]: failed to encode map e496 with expected crc 2023-09-23T21:13:15.631 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:13:15 smithi026 ceph-mon[179688]: failed to encode map e496 with expected crc 2023-09-23T21:13:15.631 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:13:15 smithi026 ceph-mon[179688]: failed to encode map e496 with expected crc 2023-09-23T21:13:15.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:13:15 smithi026 ceph-mon[179688]: osdmap e496: 8 total, 8 up, 6 in 2023-09-23T21:13:15.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:13:15 smithi026 ceph-mon[179688]: failed to encode map e496 with expected crc 2023-09-23T21:13:15.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:13:15 smithi026 ceph-mon[179688]: failed to encode map e496 with expected crc 2023-09-23T21:13:15.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:13:15 smithi026 ceph-mon[179688]: failed to encode map e496 with expected crc 2023-09-23T21:13:15.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:13:15 smithi026 ceph-mon[175151]: failed to encode map e496 with expected crc 2023-09-23T21:13:15.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:13:15 smithi026 ceph-mon[175151]: failed to encode map e496 with expected crc 2023-09-23T21:13:15.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:13:15 smithi026 ceph-mon[175151]: failed to encode map e496 with expected crc 2023-09-23T21:13:15.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:13:15 smithi026 ceph-mon[175151]: osdmap e496: 8 total, 8 up, 6 in 2023-09-23T21:13:15.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:13:15 smithi026 ceph-mon[175151]: failed to encode map e496 with expected crc 2023-09-23T21:13:15.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:13:15 smithi026 ceph-mon[175151]: failed to encode map e496 with expected crc 2023-09-23T21:13:15.633 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:13:15 smithi026 ceph-mon[175151]: failed to encode map e496 with expected crc 2023-09-23T21:13:16.476 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:13:16 smithi026 ceph-mon[175151]: pgmap v1259: 129 pgs: 1 activating, 1 activating+degraded+remapped, 7 remapped+peering, 5 peering, 11 active+clean+remapped, 4 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 10 active+undersized, 2 active+recovering+undersized+degraded+remapped, 3 active+undersized+degraded, 5 active+recovery_wait+undersized+degraded+remapped, 5 active+recovery_wait+degraded+remapped, 74 active+clean; 3.7 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 131 op/s; 30998/180753 objects degraded (17.149%); 17574/180753 objects misplaced (9.723%); 13 MiB/s, 206 objects/s recovering 2023-09-23T21:13:16.476 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:13:16 smithi026 ceph-mon[175151]: failed to encode map e497 with expected crc 2023-09-23T21:13:16.476 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:13:16 smithi026 ceph-mon[175151]: failed to encode map e497 with expected crc 2023-09-23T21:13:16.476 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:13:16 smithi026 ceph-mon[175151]: osdmap e497: 8 total, 8 up, 6 in 2023-09-23T21:13:16.476 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:13:16 smithi026 ceph-mon[175151]: failed to encode map e497 with expected crc 2023-09-23T21:13:16.476 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:13:16 smithi026 ceph-mon[175151]: failed to encode map e497 with expected crc 2023-09-23T21:13:16.476 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:13:16 smithi026 ceph-mon[175151]: failed to encode map e497 with expected crc 2023-09-23T21:13:16.476 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:13:16 smithi026 ceph-mon[175151]: Health check update: Degraded data redundancy: 30998/180753 objects degraded (17.149%), 21 pgs degraded, 21 pgs undersized (PG_DEGRADED) 2023-09-23T21:13:16.477 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:13:16 smithi026 ceph-mon[179688]: pgmap v1259: 129 pgs: 1 activating, 1 activating+degraded+remapped, 7 remapped+peering, 5 peering, 11 active+clean+remapped, 4 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 10 active+undersized, 2 active+recovering+undersized+degraded+remapped, 3 active+undersized+degraded, 5 active+recovery_wait+undersized+degraded+remapped, 5 active+recovery_wait+degraded+remapped, 74 active+clean; 3.7 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 131 op/s; 30998/180753 objects degraded (17.149%); 17574/180753 objects misplaced (9.723%); 13 MiB/s, 206 objects/s recovering 2023-09-23T21:13:16.477 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:13:16 smithi026 ceph-mon[179688]: failed to encode map e497 with expected crc 2023-09-23T21:13:16.477 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:13:16 smithi026 ceph-mon[179688]: failed to encode map e497 with expected crc 2023-09-23T21:13:16.477 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:13:16 smithi026 ceph-mon[179688]: osdmap e497: 8 total, 8 up, 6 in 2023-09-23T21:13:16.477 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:13:16 smithi026 ceph-mon[179688]: failed to encode map e497 with expected crc 2023-09-23T21:13:16.477 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:13:16 smithi026 ceph-mon[179688]: failed to encode map e497 with expected crc 2023-09-23T21:13:16.477 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:13:16 smithi026 ceph-mon[179688]: failed to encode map e497 with expected crc 2023-09-23T21:13:16.477 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:13:16 smithi026 ceph-mon[179688]: Health check update: Degraded data redundancy: 30998/180753 objects degraded (17.149%), 21 pgs degraded, 21 pgs undersized (PG_DEGRADED) 2023-09-23T21:13:16.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:13:16 smithi070 ceph-mon[168282]: pgmap v1259: 129 pgs: 1 activating, 1 activating+degraded+remapped, 7 remapped+peering, 5 peering, 11 active+clean+remapped, 4 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 10 active+undersized, 2 active+recovering+undersized+degraded+remapped, 3 active+undersized+degraded, 5 active+recovery_wait+undersized+degraded+remapped, 5 active+recovery_wait+degraded+remapped, 74 active+clean; 3.7 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 131 op/s; 30998/180753 objects degraded (17.149%); 17574/180753 objects misplaced (9.723%); 13 MiB/s, 206 objects/s recovering 2023-09-23T21:13:16.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:13:16 smithi070 ceph-mon[168282]: failed to encode map e497 with expected crc 2023-09-23T21:13:16.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:13:16 smithi070 ceph-mon[168282]: failed to encode map e497 with expected crc 2023-09-23T21:13:16.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:13:16 smithi070 ceph-mon[168282]: osdmap e497: 8 total, 8 up, 6 in 2023-09-23T21:13:16.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:13:16 smithi070 ceph-mon[168282]: failed to encode map e497 with expected crc 2023-09-23T21:13:16.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:13:16 smithi070 ceph-mon[168282]: failed to encode map e497 with expected crc 2023-09-23T21:13:16.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:13:16 smithi070 ceph-mon[168282]: failed to encode map e497 with expected crc 2023-09-23T21:13:16.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:13:16 smithi070 ceph-mon[168282]: Health check update: Degraded data redundancy: 30998/180753 objects degraded (17.149%), 21 pgs degraded, 21 pgs undersized (PG_DEGRADED) 2023-09-23T21:13:16.818 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:13:16.825 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:13:16.826 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:13:16.927 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:13:16.958 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:13:16.966 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0: {} 2023-09-23T21:13:16.978 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:13:16.995 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1: {} 2023-09-23T21:13:17.029 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2: {} 2023-09-23T21:13:17.056 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3: {} 2023-09-23T21:13:17.086 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4: {} 2023-09-23T21:13:17.118 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T21:13:17.140 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6: {} 2023-09-23T21:13:17.167 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T21:13:17.172 INFO:teuthology.orchestra.run.smithi026.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T21:13:17.172 INFO:teuthology.orchestra.run.smithi026.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T21:13:17.172 INFO:teuthology.orchestra.run.smithi026.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T21:13:17.172 INFO:teuthology.orchestra.run.smithi026.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T21:13:17.172 INFO:teuthology.orchestra.run.smithi026.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T21:13:17.172 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T21:13:17.172 INFO:teuthology.orchestra.run.smithi026.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T21:13:17.172 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T21:13:17.264 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:13:17.269 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:13:17.269 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:13:17.366 INFO:teuthology.orchestra.run.smithi026.stdout:ERROR: (22) Invalid argument 2023-09-23T21:13:17.367 INFO:teuthology.orchestra.run.smithi026.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T21:13:17.370 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:13:17.371 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:13:17 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:13:17.371 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:13:17 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:13:17.371 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:13:17 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/4030215383' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:13:17.371 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:13:17 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:13:17.371 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:13:17 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:13:17.371 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:13:17 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/4030215383' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:13:17.412 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:13:17.471 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:13:17.481 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:13:17.482 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:13:17.582 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:13:17.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:13:17 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:13:17.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:13:17 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:13:17.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:13:17 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/4030215383' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:13:17.623 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:13:17.678 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:13:17.686 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:13:17.687 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:13:17.788 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:13:17.837 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:13:18.097 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:13:18.101 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:13:18.101 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:13:18.202 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:13:18.237 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:13:18.383 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:13:18 smithi026 ceph-mon[179688]: pgmap v1261: 129 pgs: 1 activating+undersized+degraded+remapped, 1 activating, 1 activating+degraded+remapped, 3 remapped+peering, 3 peering, 4 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 14 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 99 active+clean; 3.7 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 117 op/s; 33861/179817 objects degraded (18.831%); 10999/179817 objects misplaced (6.117%); 16 MiB/s, 254 objects/s recovering 2023-09-23T21:13:18.383 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:13:18 smithi026 ceph-mon[179688]: Health check failed: Reduced data availability: 2 pgs inactive, 2 pgs peering (PG_AVAILABILITY) 2023-09-23T21:13:18.383 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:13:18 smithi026 ceph-mon[175151]: pgmap v1261: 129 pgs: 1 activating+undersized+degraded+remapped, 1 activating, 1 activating+degraded+remapped, 3 remapped+peering, 3 peering, 4 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 14 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 99 active+clean; 3.7 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 117 op/s; 33861/179817 objects degraded (18.831%); 10999/179817 objects misplaced (6.117%); 16 MiB/s, 254 objects/s recovering 2023-09-23T21:13:18.383 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:13:18 smithi026 ceph-mon[175151]: Health check failed: Reduced data availability: 2 pgs inactive, 2 pgs peering (PG_AVAILABILITY) 2023-09-23T21:13:18.431 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:13:18.436 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:13:18.437 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:13:18.537 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:13:18.580 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:13:18 smithi070 ceph-mon[168282]: pgmap v1261: 129 pgs: 1 activating+undersized+degraded+remapped, 1 activating, 1 activating+degraded+remapped, 3 remapped+peering, 3 peering, 4 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 14 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 99 active+clean; 3.7 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 117 op/s; 33861/179817 objects degraded (18.831%); 10999/179817 objects misplaced (6.117%); 16 MiB/s, 254 objects/s recovering 2023-09-23T21:13:18.580 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:13:18 smithi070 ceph-mon[168282]: Health check failed: Reduced data availability: 2 pgs inactive, 2 pgs peering (PG_AVAILABILITY) 2023-09-23T21:13:18.582 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:13:18.636 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:13:18.647 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:13:18.647 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:13:18.748 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:13:18.795 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:13:18.814 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T21:13:18.849 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:13:18.855 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:13:18.855 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:13:18.956 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:13:19.026 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:13:19.076 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:13:19.087 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:13:19.088 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:13:19.189 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:13:19.220 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:13:19.273 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:13:19.281 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:13:19.282 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:13:19.382 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:13:19.419 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:13:19.882 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 21:13:19 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[164029]: ::ffff:172.21.15.70 - - [23/Sep/2023:21:13:19] "GET /metrics HTTP/1.1" 200 36449 "" "Prometheus/2.43.0" 2023-09-23T21:13:19.884 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T21:13:20.238 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:13:20 smithi026 ceph-mon[175151]: pgmap v1262: 129 pgs: 1 activating+undersized+degraded+remapped, 1 activating, 2 peering, 4 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 16 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 101 active+clean; 3.7 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 94 op/s; 36710/179478 objects degraded (20.454%); 10999/179478 objects misplaced (6.128%); 13 MiB/s, 204 objects/s recovering 2023-09-23T21:13:20.622 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:13:20 smithi070 ceph-mon[168282]: pgmap v1262: 129 pgs: 1 activating+undersized+degraded+remapped, 1 activating, 2 peering, 4 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 16 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 101 active+clean; 3.7 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 94 op/s; 36710/179478 objects degraded (20.454%); 10999/179478 objects misplaced (6.128%); 13 MiB/s, 204 objects/s recovering 2023-09-23T21:13:20.637 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:13:20 smithi026 ceph-mon[179688]: pgmap v1262: 129 pgs: 1 activating+undersized+degraded+remapped, 1 activating, 2 peering, 4 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 16 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 101 active+clean; 3.7 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 94 op/s; 36710/179478 objects degraded (20.454%); 10999/179478 objects misplaced (6.128%); 13 MiB/s, 204 objects/s recovering 2023-09-23T21:13:21.565 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:13:21.568 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:13:21 smithi026 ceph-mon[179688]: Health check update: Degraded data redundancy: 36710/179478 objects degraded (20.454%), 24 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T21:13:21.568 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:13:21 smithi026 ceph-mon[175151]: Health check update: Degraded data redundancy: 36710/179478 objects degraded (20.454%), 24 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T21:13:21.569 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:13:21.569 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:13:21.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:13:21 smithi070 ceph-mon[168282]: Health check update: Degraded data redundancy: 36710/179478 objects degraded (20.454%), 24 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T21:13:21.670 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:13:21.704 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:13:21.744 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:13:21.747 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:13:21.748 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:13:21.848 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:13:21.895 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:13:21.896 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_ops_in_flight 2023-09-23T21:13:21.955 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:13:21.960 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:13:21.960 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:13:22.061 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:13:22.117 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:13:22.190 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:13:22.203 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:13:22.204 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:13:22.305 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:13:22.349 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:13:22.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:13:22 smithi070 ceph-mon[168282]: pgmap v1263: 129 pgs: 4 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 17 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 104 active+clean; 3.6 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 177 op/s; 36197/177009 objects degraded (20.449%); 10013/177009 objects misplaced (5.657%); 16 MiB/s, 257 objects/s recovering 2023-09-23T21:13:22.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:13:22 smithi070 ceph-mon[168282]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs inactive, 2 pgs peering) 2023-09-23T21:13:22.378 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:13:22.381 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:13:22 smithi026 ceph-mon[179688]: pgmap v1263: 129 pgs: 4 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 17 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 104 active+clean; 3.6 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 177 op/s; 36197/177009 objects degraded (20.449%); 10013/177009 objects misplaced (5.657%); 16 MiB/s, 257 objects/s recovering 2023-09-23T21:13:22.381 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:13:22 smithi026 ceph-mon[179688]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs inactive, 2 pgs peering) 2023-09-23T21:13:22.381 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:13:22 smithi026 ceph-mon[175151]: pgmap v1263: 129 pgs: 4 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 17 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 104 active+clean; 3.6 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 177 op/s; 36197/177009 objects degraded (20.449%); 10013/177009 objects misplaced (5.657%); 16 MiB/s, 257 objects/s recovering 2023-09-23T21:13:22.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:13:22 smithi026 ceph-mon[175151]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs inactive, 2 pgs peering) 2023-09-23T21:13:22.394 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:13:22.414 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:13:22.423 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:13:22.423 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:13:22.524 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:13:22.530 INFO:teuthology.orchestra.run.smithi026.stdout:ERROR: (22) Invalid argument 2023-09-23T21:13:22.530 INFO:teuthology.orchestra.run.smithi026.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T21:13:22.574 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:13:22.669 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:13:22.677 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:13:22.678 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:13:22.778 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:13:22.809 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:13:22.873 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:13:22.877 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:13:22.877 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:13:22.978 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:13:23.031 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:13:23.082 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:13:23.088 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:13:23.089 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:13:23.190 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:13:23.221 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:13:24.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:13:24 smithi070 ceph-mon[168282]: pgmap v1264: 129 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 17 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 104 active+clean; 3.6 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 234 op/s; 36073/174846 objects degraded (20.631%); 9055/174846 objects misplaced (5.179%); 21 MiB/s, 333 objects/s recovering 2023-09-23T21:13:24.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:13:24 smithi070 ceph-mon[168282]: failed to encode map e498 with expected crc 2023-09-23T21:13:24.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:13:24 smithi070 ceph-mon[168282]: failed to encode map e498 with expected crc 2023-09-23T21:13:24.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:13:24 smithi070 ceph-mon[168282]: osdmap e498: 8 total, 8 up, 6 in 2023-09-23T21:13:24.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:13:24 smithi070 ceph-mon[168282]: failed to encode map e498 with expected crc 2023-09-23T21:13:24.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:13:24 smithi070 ceph-mon[168282]: failed to encode map e498 with expected crc 2023-09-23T21:13:24.631 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:13:24 smithi026 ceph-mon[175151]: pgmap v1264: 129 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 17 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 104 active+clean; 3.6 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 234 op/s; 36073/174846 objects degraded (20.631%); 9055/174846 objects misplaced (5.179%); 21 MiB/s, 333 objects/s recovering 2023-09-23T21:13:24.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:13:24 smithi026 ceph-mon[175151]: failed to encode map e498 with expected crc 2023-09-23T21:13:24.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:13:24 smithi026 ceph-mon[175151]: failed to encode map e498 with expected crc 2023-09-23T21:13:24.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:13:24 smithi026 ceph-mon[175151]: osdmap e498: 8 total, 8 up, 6 in 2023-09-23T21:13:24.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:13:24 smithi026 ceph-mon[175151]: failed to encode map e498 with expected crc 2023-09-23T21:13:24.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:13:24 smithi026 ceph-mon[175151]: failed to encode map e498 with expected crc 2023-09-23T21:13:24.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:13:24 smithi026 ceph-mon[179688]: pgmap v1264: 129 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 17 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 104 active+clean; 3.6 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 234 op/s; 36073/174846 objects degraded (20.631%); 9055/174846 objects misplaced (5.179%); 21 MiB/s, 333 objects/s recovering 2023-09-23T21:13:24.633 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:13:24 smithi026 ceph-mon[179688]: failed to encode map e498 with expected crc 2023-09-23T21:13:24.633 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:13:24 smithi026 ceph-mon[179688]: failed to encode map e498 with expected crc 2023-09-23T21:13:24.633 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:13:24 smithi026 ceph-mon[179688]: osdmap e498: 8 total, 8 up, 6 in 2023-09-23T21:13:24.633 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:13:24 smithi026 ceph-mon[179688]: failed to encode map e498 with expected crc 2023-09-23T21:13:24.633 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:13:24 smithi026 ceph-mon[179688]: failed to encode map e498 with expected crc 2023-09-23T21:13:24.961 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph osd reweight 0 1 2023-09-23T21:13:25.186 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:13:25.192 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:13:25.192 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:13:25.241 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:13:25.293 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:13:25.356 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:13:25.419 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:13:25.422 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:13:25.423 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:13:25.524 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:13:25.555 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:13:25 smithi026 ceph-mon[175151]: failed to encode map e499 with expected crc 2023-09-23T21:13:25.555 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:13:25 smithi026 ceph-mon[175151]: failed to encode map e499 with expected crc 2023-09-23T21:13:25.555 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:13:25 smithi026 ceph-mon[175151]: osdmap e499: 8 total, 8 up, 6 in 2023-09-23T21:13:25.555 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:13:25 smithi026 ceph-mon[175151]: failed to encode map e499 with expected crc 2023-09-23T21:13:25.555 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:13:25 smithi026 ceph-mon[175151]: failed to encode map e499 with expected crc 2023-09-23T21:13:25.555 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:13:25 smithi026 ceph-mon[175151]: failed to encode map e499 with expected crc 2023-09-23T21:13:25.556 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:13:25 smithi026 ceph-mon[179688]: failed to encode map e499 with expected crc 2023-09-23T21:13:25.556 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:13:25 smithi026 ceph-mon[179688]: failed to encode map e499 with expected crc 2023-09-23T21:13:25.556 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:13:25 smithi026 ceph-mon[179688]: osdmap e499: 8 total, 8 up, 6 in 2023-09-23T21:13:25.556 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:13:25 smithi026 ceph-mon[179688]: failed to encode map e499 with expected crc 2023-09-23T21:13:25.556 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:13:25 smithi026 ceph-mon[179688]: failed to encode map e499 with expected crc 2023-09-23T21:13:25.556 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:13:25 smithi026 ceph-mon[179688]: failed to encode map e499 with expected crc 2023-09-23T21:13:25.558 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:13:25.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:13:25 smithi070 ceph-mon[168282]: failed to encode map e499 with expected crc 2023-09-23T21:13:25.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:13:25 smithi070 ceph-mon[168282]: failed to encode map e499 with expected crc 2023-09-23T21:13:25.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:13:25 smithi070 ceph-mon[168282]: osdmap e499: 8 total, 8 up, 6 in 2023-09-23T21:13:25.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:13:25 smithi070 ceph-mon[168282]: failed to encode map e499 with expected crc 2023-09-23T21:13:25.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:13:25 smithi070 ceph-mon[168282]: failed to encode map e499 with expected crc 2023-09-23T21:13:25.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:13:25 smithi070 ceph-mon[168282]: failed to encode map e499 with expected crc 2023-09-23T21:13:26.036 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:13:26.050 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:13:26.051 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:13:26.110 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0: {} 2023-09-23T21:13:26.146 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1: {} 2023-09-23T21:13:26.152 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:13:26.187 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2: {} 2023-09-23T21:13:26.218 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:13:26.230 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3: {} 2023-09-23T21:13:26.277 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:13:26.283 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:13:26.283 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:13:26.286 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4: {} 2023-09-23T21:13:26.321 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T21:13:26.347 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6: {} 2023-09-23T21:13:26.378 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T21:13:26.382 INFO:teuthology.orchestra.run.smithi026.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T21:13:26.382 INFO:teuthology.orchestra.run.smithi026.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T21:13:26.382 INFO:teuthology.orchestra.run.smithi026.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T21:13:26.382 INFO:teuthology.orchestra.run.smithi026.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T21:13:26.382 INFO:teuthology.orchestra.run.smithi026.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T21:13:26.382 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T21:13:26.383 INFO:teuthology.orchestra.run.smithi026.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T21:13:26.383 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T21:13:26.384 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:13:26.437 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:13:26.489 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:13:26.497 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:13:26.497 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:13:26.598 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:13:26.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:13:26 smithi070 ceph-mon[168282]: pgmap v1267: 129 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 17 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 104 active+clean; 3.5 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 260 op/s; 36073/173556 objects degraded (20.785%); 8447/173556 objects misplaced (4.867%); 17 MiB/s, 266 objects/s recovering 2023-09-23T21:13:26.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:13:26 smithi070 ceph-mon[168282]: Health check update: Degraded data redundancy: 36073/173556 objects degraded (20.785%), 24 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T21:13:26.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:13:26 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/2497326650' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:13:26.631 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:13:26 smithi026 ceph-mon[175151]: pgmap v1267: 129 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 17 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 104 active+clean; 3.5 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 260 op/s; 36073/173556 objects degraded (20.785%); 8447/173556 objects misplaced (4.867%); 17 MiB/s, 266 objects/s recovering 2023-09-23T21:13:26.631 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:13:26 smithi026 ceph-mon[175151]: Health check update: Degraded data redundancy: 36073/173556 objects degraded (20.785%), 24 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T21:13:26.631 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:13:26 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/2497326650' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:13:26.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:13:26 smithi026 ceph-mon[179688]: pgmap v1267: 129 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 17 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 104 active+clean; 3.5 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 260 op/s; 36073/173556 objects degraded (20.785%); 8447/173556 objects misplaced (4.867%); 17 MiB/s, 266 objects/s recovering 2023-09-23T21:13:26.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:13:26 smithi026 ceph-mon[179688]: Health check update: Degraded data redundancy: 36073/173556 objects degraded (20.785%), 24 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T21:13:26.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:13:26 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/2497326650' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:13:26.642 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:13:26.904 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:13:26.909 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:13:26.910 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:13:27.011 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:13:27.025 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:13:27.066 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:13:27.181 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:13:27.189 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:13:27.189 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:13:27.290 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:13:27.344 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:13:27.350 INFO:teuthology.orchestra.run.smithi026.stderr:noscrub is set 2023-09-23T21:13:27.415 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:13:27.422 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:13:27 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/3790135612' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T21:13:27.422 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:13:27 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T21:13:27.422 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:13:27 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/3790135612' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T21:13:27.422 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:13:27 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T21:13:27.423 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:13:27.423 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:13:27.524 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:13:27.552 INFO:teuthology.orchestra.run.smithi026.stdout:{ 2023-09-23T21:13:27.552 INFO:teuthology.orchestra.run.smithi026.stdout: "ops": [ 2023-09-23T21:13:27.552 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:13:27.552 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.46328.0:104488 10.f 10:f8e011b8:::benchmark_data_smithi026_379838_object24531:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e500)", 2023-09-23T21:13:27.553 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T21:13:27.540527+0000", 2023-09-23T21:13:27.553 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 0.0092051449999999996, 2023-09-23T21:13:27.553 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0092407400000000008, 2023-09-23T21:13:27.553 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:13:27.553 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:13:27.553 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T21:13:27.553 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.46328", 2023-09-23T21:13:27.553 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/3748861181", 2023-09-23T21:13:27.553 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 104488 2023-09-23T21:13:27.553 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:13:27.553 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:13:27.553 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:13:27.554 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:13:27.554 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:27.540527+0000", 2023-09-23T21:13:27.554 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:13:27.554 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:13:27.554 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:13:27.554 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:13:27.554 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:27.540527+0000", 2023-09-23T21:13:27.554 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:13:27.554 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:13:27.554 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:13:27.554 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:13:27.554 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:27.540525+0000", 2023-09-23T21:13:27.555 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999981 2023-09-23T21:13:27.555 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:13:27.555 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:13:27.555 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:13:27.555 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:27.540534+0000", 2023-09-23T21:13:27.555 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.8910000000000001e-06 2023-09-23T21:13:27.555 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:13:27.555 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:13:27.555 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:13:27.555 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:27.540549+0000", 2023-09-23T21:13:27.555 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.5116999999999999e-05 2023-09-23T21:13:27.555 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:13:27.556 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:13:27.556 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:13:27.556 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:27.540559+0000", 2023-09-23T21:13:27.556 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.4790000000000002e-06 2023-09-23T21:13:27.556 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:13:27.556 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:13:27.556 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:13:27.556 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:27.541363+0000", 2023-09-23T21:13:27.556 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00080438000000000005 2023-09-23T21:13:27.556 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:13:27.556 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:13:27.556 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:13:27.557 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:27.541550+0000", 2023-09-23T21:13:27.557 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00018678299999999999 2023-09-23T21:13:27.557 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:13:27.557 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:13:27.557 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 2,4", 2023-09-23T21:13:27.557 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:27.541716+0000", 2023-09-23T21:13:27.557 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00016629399999999999 2023-09-23T21:13:27.557 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:13:27.557 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:13:27.557 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T21:13:27.557 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:27.545642+0000", 2023-09-23T21:13:27.557 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00392568 2023-09-23T21:13:27.558 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:13:27.558 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:13:27.558 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:13:27.558 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:27.546115+0000", 2023-09-23T21:13:27.558 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000473351 2023-09-23T21:13:27.558 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:13:27.558 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:13:27.558 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:13:27.558 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:27.548572+0000", 2023-09-23T21:13:27.558 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0024572140000000001 2023-09-23T21:13:27.558 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:13:27.558 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:13:27.559 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T21:13:27.559 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:27.548637+0000", 2023-09-23T21:13:27.559 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.4397e-05 2023-09-23T21:13:27.559 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:13:27.559 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:13:27.559 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:13:27.559 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:13:27.559 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:13:27.559 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "MOSDPGPush(10.12 500/494 [PushOp(10:4c74ea0e:::benchmark_data_smithi026_379838_object24534:head, version: 435'1540, data_included: [0~65536], data_size: 65536, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(10:4c74ea0e:::benchmark_data_smithi026_379838_object24534:head@435'1540, size: 65536, copy_subset: [0~65536], clone_subset: {}, snapset: 0=[]:{}, object_exist: 0), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:65536, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false, error:false))])", 2023-09-23T21:13:27.559 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T21:13:27.546848+0000", 2023-09-23T21:13:27.559 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 0.0028836489999999998, 2023-09-23T21:13:27.559 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.003046286, 2023-09-23T21:13:27.560 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:13:27.560 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "started", 2023-09-23T21:13:27.560 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:13:27.560 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:13:27.560 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:13:27.560 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:27.546848+0000", 2023-09-23T21:13:27.560 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:13:27.560 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:13:27.560 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:13:27.560 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:13:27.560 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:27.546848+0000", 2023-09-23T21:13:27.560 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:13:27.561 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:13:27.561 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:13:27.561 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:13:27.561 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:27.546848+0000", 2023-09-23T21:13:27.561 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T21:13:27.561 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:13:27.561 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:13:27.561 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:13:27.561 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:27.546900+0000", 2023-09-23T21:13:27.561 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.2519000000000003e-05 2023-09-23T21:13:27.561 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:13:27.561 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:13:27.562 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:13:27.562 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:27.546916+0000", 2023-09-23T21:13:27.562 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.5458999999999999e-05 2023-09-23T21:13:27.562 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:13:27.562 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:13:27.562 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:13:27.562 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:27.546926+0000", 2023-09-23T21:13:27.562 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0710000000000001e-05 2023-09-23T21:13:27.562 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:13:27.562 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:13:27.562 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:13:27.562 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:27.549284+0000", 2023-09-23T21:13:27.563 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.002357961 2023-09-23T21:13:27.563 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:13:27.563 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:13:27.563 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:13:27.563 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:27.549297+0000", 2023-09-23T21:13:27.563 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.2303000000000001e-05 2023-09-23T21:13:27.563 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:13:27.563 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:13:27.563 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:13:27.563 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:13:27.563 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:13:27.563 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_repop_reply(client.46328.0:104488 10.f e500/444 ondisk, result = 0)", 2023-09-23T21:13:27.564 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T21:13:27.548425+0000", 2023-09-23T21:13:27.564 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 0.0013071789999999999, 2023-09-23T21:13:27.564 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0015283440000000001, 2023-09-23T21:13:27.564 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:13:27.564 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "started", 2023-09-23T21:13:27.564 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:13:27.564 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:13:27.564 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:13:27.564 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:27.548425+0000", 2023-09-23T21:13:27.564 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:13:27.564 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:13:27.564 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:13:27.565 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:13:27.565 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:27.548425+0000", 2023-09-23T21:13:27.565 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:13:27.565 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:13:27.565 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:13:27.565 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:13:27.565 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:27.548424+0000", 2023-09-23T21:13:27.565 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.999999 2023-09-23T21:13:27.565 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:13:27.565 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:13:27.565 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:13:27.565 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:27.548429+0000", 2023-09-23T21:13:27.566 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.2619999999999998e-06 2023-09-23T21:13:27.566 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:13:27.566 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:13:27.566 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:13:27.566 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:27.548440+0000", 2023-09-23T21:13:27.566 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1012000000000001e-05 2023-09-23T21:13:27.566 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:13:27.566 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:13:27.566 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:13:27.566 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:27.548445+0000", 2023-09-23T21:13:27.566 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.1660000000000002e-06 2023-09-23T21:13:27.566 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:13:27.567 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:13:27.567 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:13:27.567 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:27.548540+0000", 2023-09-23T21:13:27.567 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.4180999999999998e-05 2023-09-23T21:13:27.567 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:13:27.567 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:13:27.567 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:13:27.567 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:27.548561+0000", 2023-09-23T21:13:27.567 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.1403000000000002e-05 2023-09-23T21:13:27.567 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:13:27.567 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:13:27.567 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:13:27.568 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:13:27.568 INFO:teuthology.orchestra.run.smithi026.stdout: ], 2023-09-23T21:13:27.568 INFO:teuthology.orchestra.run.smithi026.stdout: "num_ops": 3 2023-09-23T21:13:27.568 INFO:teuthology.orchestra.run.smithi026.stdout:} 2023-09-23T21:13:27.583 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:13:27.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:13:27 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/3790135612' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T21:13:27.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:13:27 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T21:13:28.085 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_blocked_ops 2023-09-23T21:13:28.299 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:13:28.305 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:13:28.306 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:13:28.406 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:13:28.459 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:13:28.568 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:13:28.574 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:13:28 smithi026 ceph-mon[175151]: pgmap v1268: 129 pgs: 4 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 17 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 105 active+clean; 3.5 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 399 op/s; 35448/169881 objects degraded (20.866%); 7983/169881 objects misplaced (4.699%); 22 MiB/s, 355 objects/s recovering 2023-09-23T21:13:28.574 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:13:28 smithi026 ceph-mon[175151]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:13:28.574 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:13:28 smithi026 ceph-mon[175151]: failed to encode map e500 with expected crc 2023-09-23T21:13:28.574 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:13:28 smithi026 ceph-mon[175151]: failed to encode map e500 with expected crc 2023-09-23T21:13:28.575 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:13:28 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T21:13:28.575 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:13:28 smithi026 ceph-mon[175151]: osdmap e500: 8 total, 8 up, 6 in 2023-09-23T21:13:28.575 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:13:28 smithi026 ceph-mon[175151]: failed to encode map e500 with expected crc 2023-09-23T21:13:28.575 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:13:28 smithi026 ceph-mon[175151]: failed to encode map e500 with expected crc 2023-09-23T21:13:28.575 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:13:28 smithi026 ceph-mon[179688]: pgmap v1268: 129 pgs: 4 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 17 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 105 active+clean; 3.5 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 399 op/s; 35448/169881 objects degraded (20.866%); 7983/169881 objects misplaced (4.699%); 22 MiB/s, 355 objects/s recovering 2023-09-23T21:13:28.575 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:13:28 smithi026 ceph-mon[179688]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:13:28.575 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:13:28 smithi026 ceph-mon[179688]: failed to encode map e500 with expected crc 2023-09-23T21:13:28.575 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:13:28 smithi026 ceph-mon[179688]: failed to encode map e500 with expected crc 2023-09-23T21:13:28.576 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:13:28 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T21:13:28.576 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:13:28 smithi026 ceph-mon[179688]: osdmap e500: 8 total, 8 up, 6 in 2023-09-23T21:13:28.576 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:13:28 smithi026 ceph-mon[179688]: failed to encode map e500 with expected crc 2023-09-23T21:13:28.576 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:13:28 smithi026 ceph-mon[179688]: failed to encode map e500 with expected crc 2023-09-23T21:13:28.576 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:13:28.577 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:13:28.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:13:28 smithi070 ceph-mon[168282]: pgmap v1268: 129 pgs: 4 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 17 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 105 active+clean; 3.5 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 399 op/s; 35448/169881 objects degraded (20.866%); 7983/169881 objects misplaced (4.699%); 22 MiB/s, 355 objects/s recovering 2023-09-23T21:13:28.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:13:28 smithi070 ceph-mon[168282]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:13:28.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:13:28 smithi070 ceph-mon[168282]: failed to encode map e500 with expected crc 2023-09-23T21:13:28.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:13:28 smithi070 ceph-mon[168282]: failed to encode map e500 with expected crc 2023-09-23T21:13:28.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:13:28 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T21:13:28.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:13:28 smithi070 ceph-mon[168282]: osdmap e500: 8 total, 8 up, 6 in 2023-09-23T21:13:28.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:13:28 smithi070 ceph-mon[168282]: failed to encode map e500 with expected crc 2023-09-23T21:13:28.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:13:28 smithi070 ceph-mon[168282]: failed to encode map e500 with expected crc 2023-09-23T21:13:28.677 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:13:28.735 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:13:28.833 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:13:28.837 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:13:28.837 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:13:28.938 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:13:28.990 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T21:13:29.005 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:13:29.063 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:13:29.067 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:13:29.067 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:13:29.148 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:13:29.168 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:13:29.228 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:13:29.375 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:13:29.383 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:13:29.384 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:13:29.485 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:13:29.537 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:13:29.597 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:13:29.602 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:13:29.603 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:13:29.704 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:13:29.759 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 21:13:29 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[164029]: ::ffff:172.21.15.70 - - [23/Sep/2023:21:13:29] "GET /metrics HTTP/1.1" 200 36449 "" "Prometheus/2.43.0" 2023-09-23T21:13:29.768 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:13:29.947 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:13:29.952 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:13:29.952 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:13:30.004 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:13:30.053 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:13:30.098 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:13:30.174 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T21:13:30.180 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:13:30.181 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:13:30.281 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:13:30.315 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:13:30.371 INFO:teuthology.orchestra.run.smithi026.stderr:reweighted osd.0 to 1 (10000) 2023-09-23T21:13:30.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:13:30 smithi026 ceph-mon[175151]: pgmap v1270: 129 pgs: 4 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 17 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 105 active+clean; 3.4 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 339 op/s; 35385/168726 objects degraded (20.972%); 7983/168726 objects misplaced (4.731%); 11 MiB/s, 182 objects/s recovering 2023-09-23T21:13:30.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:13:30 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/3307336541' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-09-23T21:13:30.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:13:30 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-09-23T21:13:30.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:13:30 smithi026 ceph-mon[179688]: pgmap v1270: 129 pgs: 4 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 17 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 105 active+clean; 3.4 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 339 op/s; 35385/168726 objects degraded (20.972%); 7983/168726 objects misplaced (4.731%); 11 MiB/s, 182 objects/s recovering 2023-09-23T21:13:30.383 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:13:30 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/3307336541' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-09-23T21:13:30.383 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:13:30 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-09-23T21:13:30.385 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:13:30.389 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:13:30.390 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:13:30.397 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T21:13:30.490 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:13:30.516 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:13:30.549 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:13:30.551 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:13:30.552 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:13:30.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:13:30 smithi070 ceph-mon[168282]: pgmap v1270: 129 pgs: 4 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 17 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 105 active+clean; 3.4 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 339 op/s; 35385/168726 objects degraded (20.972%); 7983/168726 objects misplaced (4.731%); 11 MiB/s, 182 objects/s recovering 2023-09-23T21:13:30.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:13:30 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/3307336541' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-09-23T21:13:30.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:13:30 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-09-23T21:13:30.652 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:13:30.691 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:13:30.833 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:13:30.840 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:13:30.841 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:13:30.941 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:13:30.971 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:13:31.204 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T21:13:31.207 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:13:31.207 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:13:31.305 INFO:teuthology.orchestra.run.smithi026.stdout:{ 2023-09-23T21:13:31.305 INFO:teuthology.orchestra.run.smithi026.stdout: "ops": [], 2023-09-23T21:13:31.305 INFO:teuthology.orchestra.run.smithi026.stdout: "complaint_time": 30, 2023-09-23T21:13:31.305 INFO:teuthology.orchestra.run.smithi026.stdout: "num_blocked_ops": 0 2023-09-23T21:13:31.305 INFO:teuthology.orchestra.run.smithi026.stdout:} 2023-09-23T21:13:31.308 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:13:31.333 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:13:31.371 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:13:31.374 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:13:31.374 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:13:31.475 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:13:31.508 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:13:31.574 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:13:31 smithi026 ceph-mon[179688]: failed to encode map e501 with expected crc 2023-09-23T21:13:31.574 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:13:31 smithi026 ceph-mon[179688]: failed to encode map e501 with expected crc 2023-09-23T21:13:31.574 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:13:31 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-09-23T21:13:31.574 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:13:31 smithi026 ceph-mon[179688]: osdmap e501: 8 total, 8 up, 6 in 2023-09-23T21:13:31.574 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:13:31 smithi026 ceph-mon[179688]: failed to encode map e501 with expected crc 2023-09-23T21:13:31.575 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:13:31 smithi026 ceph-mon[179688]: Health check update: Degraded data redundancy: 35385/168726 objects degraded (20.972%), 24 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-23T21:13:31.575 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:13:31 smithi026 ceph-mon[179688]: failed to encode map e502 with expected crc 2023-09-23T21:13:31.575 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:13:31 smithi026 ceph-mon[179688]: failed to encode map e502 with expected crc 2023-09-23T21:13:31.575 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:13:31 smithi026 ceph-mon[179688]: osdmap e502: 8 total, 8 up, 6 in 2023-09-23T21:13:31.575 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:13:31 smithi026 ceph-mon[179688]: failed to encode map e502 with expected crc 2023-09-23T21:13:31.575 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:13:31 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:13:31.575 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:13:31 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:13:31.576 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:13:31 smithi026 ceph-mon[175151]: failed to encode map e501 with expected crc 2023-09-23T21:13:31.576 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:13:31 smithi026 ceph-mon[175151]: failed to encode map e501 with expected crc 2023-09-23T21:13:31.576 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:13:31 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-09-23T21:13:31.576 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:13:31 smithi026 ceph-mon[175151]: osdmap e501: 8 total, 8 up, 6 in 2023-09-23T21:13:31.576 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:13:31 smithi026 ceph-mon[175151]: failed to encode map e501 with expected crc 2023-09-23T21:13:31.576 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:13:31 smithi026 ceph-mon[175151]: Health check update: Degraded data redundancy: 35385/168726 objects degraded (20.972%), 24 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-23T21:13:31.576 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:13:31 smithi026 ceph-mon[175151]: failed to encode map e502 with expected crc 2023-09-23T21:13:31.576 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:13:31 smithi026 ceph-mon[175151]: failed to encode map e502 with expected crc 2023-09-23T21:13:31.576 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:13:31 smithi026 ceph-mon[175151]: osdmap e502: 8 total, 8 up, 6 in 2023-09-23T21:13:31.577 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:13:31 smithi026 ceph-mon[175151]: failed to encode map e502 with expected crc 2023-09-23T21:13:31.577 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:13:31 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:13:31.577 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:13:31 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:13:31.578 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph osd reweight 2 1 2023-09-23T21:13:31.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:13:31 smithi070 ceph-mon[168282]: failed to encode map e501 with expected crc 2023-09-23T21:13:31.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:13:31 smithi070 ceph-mon[168282]: failed to encode map e501 with expected crc 2023-09-23T21:13:31.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:13:31 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-09-23T21:13:31.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:13:31 smithi070 ceph-mon[168282]: osdmap e501: 8 total, 8 up, 6 in 2023-09-23T21:13:31.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:13:31 smithi070 ceph-mon[168282]: failed to encode map e501 with expected crc 2023-09-23T21:13:31.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:13:31 smithi070 ceph-mon[168282]: Health check update: Degraded data redundancy: 35385/168726 objects degraded (20.972%), 24 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-23T21:13:31.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:13:31 smithi070 ceph-mon[168282]: failed to encode map e502 with expected crc 2023-09-23T21:13:31.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:13:31 smithi070 ceph-mon[168282]: failed to encode map e502 with expected crc 2023-09-23T21:13:31.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:13:31 smithi070 ceph-mon[168282]: osdmap e502: 8 total, 8 up, 6 in 2023-09-23T21:13:31.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:13:31 smithi070 ceph-mon[168282]: failed to encode map e502 with expected crc 2023-09-23T21:13:31.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:13:31 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:13:31.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:13:31 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:13:31.790 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:13:31.800 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:13:31.801 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:13:31.880 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_historic_ops 2023-09-23T21:13:31.902 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:13:31.978 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:13:32.047 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:13:32.053 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:13:32.054 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:13:32.155 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:13:32.200 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:13:32.259 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T21:13:32.263 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:13:32.263 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:13:32.264 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:13:32.307 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0: {} 2023-09-23T21:13:32.336 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1: {} 2023-09-23T21:13:32.361 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2: {} 2023-09-23T21:13:32.365 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:13:32.383 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3: {} 2023-09-23T21:13:32.392 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:13:32.408 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4: {} 2023-09-23T21:13:32.428 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:13:32.429 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T21:13:32.432 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:13:32.433 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:13:32.450 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6: {} 2023-09-23T21:13:32.477 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T21:13:32.479 INFO:teuthology.orchestra.run.smithi026.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T21:13:32.479 INFO:teuthology.orchestra.run.smithi026.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T21:13:32.479 INFO:teuthology.orchestra.run.smithi026.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T21:13:32.479 INFO:teuthology.orchestra.run.smithi026.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T21:13:32.479 INFO:teuthology.orchestra.run.smithi026.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T21:13:32.479 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T21:13:32.479 INFO:teuthology.orchestra.run.smithi026.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T21:13:32.479 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T21:13:32.504 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:13:32 smithi026 ceph-mon[179688]: failed to encode map e502 with expected crc 2023-09-23T21:13:32.504 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:13:32 smithi026 ceph-mon[179688]: failed to encode map e502 with expected crc 2023-09-23T21:13:32.504 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:13:32 smithi026 ceph-mon[179688]: pgmap v1273: 129 pgs: 1 active+recovering+undersized+remapped, 3 remapped+peering, 5 peering, 4 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 14 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 100 active+clean; 3.4 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 456 op/s; 25355/165342 objects degraded (15.335%); 6189/165342 objects misplaced (3.743%); 22 MiB/s, 348 objects/s recovering 2023-09-23T21:13:32.505 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:13:32 smithi026 ceph-mon[179688]: failed to encode map e502 with expected crc 2023-09-23T21:13:32.505 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:13:32 smithi026 ceph-mon[179688]: failed to encode map e503 with expected crc 2023-09-23T21:13:32.505 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:13:32 smithi026 ceph-mon[179688]: failed to encode map e503 with expected crc 2023-09-23T21:13:32.505 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:13:32 smithi026 ceph-mon[179688]: osdmap e503: 8 total, 8 up, 6 in 2023-09-23T21:13:32.505 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:13:32 smithi026 ceph-mon[179688]: failed to encode map e503 with expected crc 2023-09-23T21:13:32.505 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:13:32 smithi026 ceph-mon[179688]: 10.11 starting backfill to osd.0 from (0'0,451'3477] MIN to 500'4599 2023-09-23T21:13:32.505 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:13:32 smithi026 ceph-mon[179688]: failed to encode map e503 with expected crc 2023-09-23T21:13:32.505 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:13:32 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/3335627355' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:13:32.505 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:13:32 smithi026 ceph-mon[175151]: failed to encode map e502 with expected crc 2023-09-23T21:13:32.505 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:13:32 smithi026 ceph-mon[175151]: failed to encode map e502 with expected crc 2023-09-23T21:13:32.506 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:13:32 smithi026 ceph-mon[175151]: pgmap v1273: 129 pgs: 1 active+recovering+undersized+remapped, 3 remapped+peering, 5 peering, 4 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 14 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 100 active+clean; 3.4 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 456 op/s; 25355/165342 objects degraded (15.335%); 6189/165342 objects misplaced (3.743%); 22 MiB/s, 348 objects/s recovering 2023-09-23T21:13:32.506 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:13:32 smithi026 ceph-mon[175151]: failed to encode map e502 with expected crc 2023-09-23T21:13:32.506 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:13:32 smithi026 ceph-mon[175151]: failed to encode map e503 with expected crc 2023-09-23T21:13:32.506 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:13:32 smithi026 ceph-mon[175151]: failed to encode map e503 with expected crc 2023-09-23T21:13:32.506 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:13:32 smithi026 ceph-mon[175151]: osdmap e503: 8 total, 8 up, 6 in 2023-09-23T21:13:32.506 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:13:32 smithi026 ceph-mon[175151]: failed to encode map e503 with expected crc 2023-09-23T21:13:32.506 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:13:32 smithi026 ceph-mon[175151]: 10.11 starting backfill to osd.0 from (0'0,451'3477] MIN to 500'4599 2023-09-23T21:13:32.506 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:13:32 smithi026 ceph-mon[175151]: failed to encode map e503 with expected crc 2023-09-23T21:13:32.506 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:13:32 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/3335627355' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:13:32.507 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:13:32.534 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:13:32.563 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:13:32.603 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:13:32.605 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:13:32 smithi070 ceph-mon[168282]: failed to encode map e502 with expected crc 2023-09-23T21:13:32.605 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:13:32 smithi070 ceph-mon[168282]: failed to encode map e502 with expected crc 2023-09-23T21:13:32.606 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:13:32 smithi070 ceph-mon[168282]: pgmap v1273: 129 pgs: 1 active+recovering+undersized+remapped, 3 remapped+peering, 5 peering, 4 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 14 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 100 active+clean; 3.4 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 456 op/s; 25355/165342 objects degraded (15.335%); 6189/165342 objects misplaced (3.743%); 22 MiB/s, 348 objects/s recovering 2023-09-23T21:13:32.606 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:13:32 smithi070 ceph-mon[168282]: failed to encode map e502 with expected crc 2023-09-23T21:13:32.606 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:13:32 smithi070 ceph-mon[168282]: failed to encode map e503 with expected crc 2023-09-23T21:13:32.606 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:13:32 smithi070 ceph-mon[168282]: failed to encode map e503 with expected crc 2023-09-23T21:13:32.606 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:13:32 smithi070 ceph-mon[168282]: osdmap e503: 8 total, 8 up, 6 in 2023-09-23T21:13:32.606 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:13:32 smithi070 ceph-mon[168282]: failed to encode map e503 with expected crc 2023-09-23T21:13:32.606 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:13:32 smithi070 ceph-mon[168282]: 10.11 starting backfill to osd.0 from (0'0,451'3477] MIN to 500'4599 2023-09-23T21:13:32.606 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:13:32 smithi070 ceph-mon[168282]: failed to encode map e503 with expected crc 2023-09-23T21:13:32.606 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:13:32 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/3335627355' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:13:32.607 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:13:32.607 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:13:32.708 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:13:32.734 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:13:32.786 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:13:32.793 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:13:32.793 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:13:32.894 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:13:32.926 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:13:33.420 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:13:33 smithi026 ceph-mon[179688]: failed to encode map e503 with expected crc 2023-09-23T21:13:33.420 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:13:33 smithi026 ceph-mon[179688]: failed to encode map e504 with expected crc 2023-09-23T21:13:33.420 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:13:33 smithi026 ceph-mon[179688]: osdmap e504: 8 total, 8 up, 6 in 2023-09-23T21:13:33.420 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:13:33 smithi026 ceph-mon[179688]: failed to encode map e504 with expected crc 2023-09-23T21:13:33.420 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:13:33 smithi026 ceph-mon[179688]: failed to encode map e504 with expected crc 2023-09-23T21:13:33.420 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:13:33 smithi026 ceph-mon[179688]: failed to encode map e504 with expected crc 2023-09-23T21:13:33.420 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:13:33 smithi026 ceph-mon[179688]: failed to encode map e504 with expected crc 2023-09-23T21:13:33.421 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:13:33 smithi026 ceph-mon[175151]: failed to encode map e503 with expected crc 2023-09-23T21:13:33.421 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:13:33 smithi026 ceph-mon[175151]: failed to encode map e504 with expected crc 2023-09-23T21:13:33.421 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:13:33 smithi026 ceph-mon[175151]: osdmap e504: 8 total, 8 up, 6 in 2023-09-23T21:13:33.421 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:13:33 smithi026 ceph-mon[175151]: failed to encode map e504 with expected crc 2023-09-23T21:13:33.421 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:13:33 smithi026 ceph-mon[175151]: failed to encode map e504 with expected crc 2023-09-23T21:13:33.421 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:13:33 smithi026 ceph-mon[175151]: failed to encode map e504 with expected crc 2023-09-23T21:13:33.421 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:13:33 smithi026 ceph-mon[175151]: failed to encode map e504 with expected crc 2023-09-23T21:13:33.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:13:33 smithi070 ceph-mon[168282]: failed to encode map e503 with expected crc 2023-09-23T21:13:33.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:13:33 smithi070 ceph-mon[168282]: failed to encode map e504 with expected crc 2023-09-23T21:13:33.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:13:33 smithi070 ceph-mon[168282]: osdmap e504: 8 total, 8 up, 6 in 2023-09-23T21:13:33.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:13:33 smithi070 ceph-mon[168282]: failed to encode map e504 with expected crc 2023-09-23T21:13:33.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:13:33 smithi070 ceph-mon[168282]: failed to encode map e504 with expected crc 2023-09-23T21:13:33.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:13:33 smithi070 ceph-mon[168282]: failed to encode map e504 with expected crc 2023-09-23T21:13:33.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:13:33 smithi070 ceph-mon[168282]: failed to encode map e504 with expected crc 2023-09-23T21:13:34.344 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T21:13:34.353 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:13:34.353 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:13:34.454 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:13:34.486 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:13:34.518 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:13:34.529 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:13:34.532 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:13:34 smithi026 ceph-mon[179688]: failed to encode map e504 with expected crc 2023-09-23T21:13:34.532 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:13:34 smithi026 ceph-mon[179688]: pgmap v1276: 129 pgs: 1 active+recovering+undersized+remapped, 6 remapped+peering, 5 peering, 6 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 14 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 95 active+clean; 3.3 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 439 op/s; 25295/163449 objects degraded (15.476%); 6128/163449 objects misplaced (3.749%); 21 MiB/s, 339 objects/s recovering 2023-09-23T21:13:34.532 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:13:34 smithi026 ceph-mon[179688]: failed to encode map e505 with expected crc 2023-09-23T21:13:34.532 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:13:34 smithi026 ceph-mon[179688]: failed to encode map e505 with expected crc 2023-09-23T21:13:34.533 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:13:34 smithi026 ceph-mon[179688]: failed to encode map e505 with expected crc 2023-09-23T21:13:34.533 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:13:34 smithi026 ceph-mon[179688]: osdmap e505: 8 total, 8 up, 6 in 2023-09-23T21:13:34.533 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:13:34 smithi026 ceph-mon[179688]: failed to encode map e505 with expected crc 2023-09-23T21:13:34.533 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:13:34 smithi026 ceph-mon[175151]: failed to encode map e504 with expected crc 2023-09-23T21:13:34.533 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:13:34 smithi026 ceph-mon[175151]: pgmap v1276: 129 pgs: 1 active+recovering+undersized+remapped, 6 remapped+peering, 5 peering, 6 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 14 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 95 active+clean; 3.3 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 439 op/s; 25295/163449 objects degraded (15.476%); 6128/163449 objects misplaced (3.749%); 21 MiB/s, 339 objects/s recovering 2023-09-23T21:13:34.533 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:13:34 smithi026 ceph-mon[175151]: failed to encode map e505 with expected crc 2023-09-23T21:13:34.533 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:13:34 smithi026 ceph-mon[175151]: failed to encode map e505 with expected crc 2023-09-23T21:13:34.534 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:13:34 smithi026 ceph-mon[175151]: failed to encode map e505 with expected crc 2023-09-23T21:13:34.534 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:13:34 smithi026 ceph-mon[175151]: osdmap e505: 8 total, 8 up, 6 in 2023-09-23T21:13:34.534 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:13:34 smithi026 ceph-mon[175151]: failed to encode map e505 with expected crc 2023-09-23T21:13:34.534 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:13:34.534 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:13:34.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:13:34 smithi070 ceph-mon[168282]: failed to encode map e504 with expected crc 2023-09-23T21:13:34.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:13:34 smithi070 ceph-mon[168282]: pgmap v1276: 129 pgs: 1 active+recovering+undersized+remapped, 6 remapped+peering, 5 peering, 6 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 14 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 95 active+clean; 3.3 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 439 op/s; 25295/163449 objects degraded (15.476%); 6128/163449 objects misplaced (3.749%); 21 MiB/s, 339 objects/s recovering 2023-09-23T21:13:34.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:13:34 smithi070 ceph-mon[168282]: failed to encode map e505 with expected crc 2023-09-23T21:13:34.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:13:34 smithi070 ceph-mon[168282]: failed to encode map e505 with expected crc 2023-09-23T21:13:34.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:13:34 smithi070 ceph-mon[168282]: failed to encode map e505 with expected crc 2023-09-23T21:13:34.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:13:34 smithi070 ceph-mon[168282]: osdmap e505: 8 total, 8 up, 6 in 2023-09-23T21:13:34.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:13:34 smithi070 ceph-mon[168282]: failed to encode map e505 with expected crc 2023-09-23T21:13:34.635 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:13:34.667 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:13:34.806 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:13:34.810 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:13:34.811 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:13:34.911 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:13:34.938 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:13:34.974 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:13:34.977 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:13:34.978 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:13:35.079 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:13:35.115 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:13:35.158 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:13:35.161 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:13:35.161 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:13:35.262 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:13:35.296 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:13:35.346 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:13:35 smithi070 ceph-mon[168282]: failed to encode map e505 with expected crc 2023-09-23T21:13:35.347 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:13:35 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/3839892000' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:13:35.347 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:13:35 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:13:35.363 INFO:teuthology.orchestra.run.smithi026.stderr:nodeep-scrub is set 2023-09-23T21:13:35.456 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T21:13:35.607 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:13:35 smithi026 ceph-mon[175151]: failed to encode map e505 with expected crc 2023-09-23T21:13:35.607 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:13:35 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/3839892000' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:13:35.607 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:13:35 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:13:35.607 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:13:35 smithi026 ceph-mon[179688]: failed to encode map e505 with expected crc 2023-09-23T21:13:35.607 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:13:35 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/3839892000' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:13:35.608 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:13:35 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:13:36.237 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T21:13:36.242 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:13:36.242 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:13:36.343 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:13:36.376 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:13:36.418 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:13:36.421 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:13:36.422 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:13:36.522 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:13:36.569 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:13:36.611 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:13:36.616 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:13:36 smithi070 ceph-mon[168282]: pgmap v1278: 129 pgs: 1 active+recovering+undersized+remapped, 5 remapped+peering, 5 peering, 6 active+recovery_wait+degraded, 16 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 95 active+clean; 3.3 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 187 op/s; 28150/162936 objects degraded (17.277%); 6054/162936 objects misplaced (3.716%); 2.1 MiB/s, 86 objects/s recovering 2023-09-23T21:13:36.616 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:13:36 smithi070 ceph-mon[168282]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:13:36.616 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:13:36 smithi070 ceph-mon[168282]: failed to encode map e506 with expected crc 2023-09-23T21:13:36.617 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:13:36 smithi070 ceph-mon[168282]: failed to encode map e506 with expected crc 2023-09-23T21:13:36.617 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:13:36 smithi070 ceph-mon[168282]: failed to encode map e506 with expected crc 2023-09-23T21:13:36.617 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:13:36 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T21:13:36.617 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:13:36 smithi070 ceph-mon[168282]: osdmap e506: 8 total, 8 up, 6 in 2023-09-23T21:13:36.617 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:13:36 smithi070 ceph-mon[168282]: failed to encode map e506 with expected crc 2023-09-23T21:13:36.617 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:13:36 smithi070 ceph-mon[168282]: Health check update: Degraded data redundancy: 28150/162936 objects degraded (17.277%), 23 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T21:13:36.617 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:13:36.618 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:13:36.633 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:13:36 smithi026 ceph-mon[175151]: pgmap v1278: 129 pgs: 1 active+recovering+undersized+remapped, 5 remapped+peering, 5 peering, 6 active+recovery_wait+degraded, 16 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 95 active+clean; 3.3 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 187 op/s; 28150/162936 objects degraded (17.277%); 6054/162936 objects misplaced (3.716%); 2.1 MiB/s, 86 objects/s recovering 2023-09-23T21:13:36.634 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:13:36 smithi026 ceph-mon[175151]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:13:36.634 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:13:36 smithi026 ceph-mon[175151]: failed to encode map e506 with expected crc 2023-09-23T21:13:36.634 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:13:36 smithi026 ceph-mon[175151]: failed to encode map e506 with expected crc 2023-09-23T21:13:36.634 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:13:36 smithi026 ceph-mon[175151]: failed to encode map e506 with expected crc 2023-09-23T21:13:36.634 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:13:36 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T21:13:36.634 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:13:36 smithi026 ceph-mon[175151]: osdmap e506: 8 total, 8 up, 6 in 2023-09-23T21:13:36.634 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:13:36 smithi026 ceph-mon[175151]: failed to encode map e506 with expected crc 2023-09-23T21:13:36.634 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:13:36 smithi026 ceph-mon[175151]: Health check update: Degraded data redundancy: 28150/162936 objects degraded (17.277%), 23 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T21:13:36.634 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:13:36 smithi026 ceph-mon[179688]: pgmap v1278: 129 pgs: 1 active+recovering+undersized+remapped, 5 remapped+peering, 5 peering, 6 active+recovery_wait+degraded, 16 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 95 active+clean; 3.3 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 187 op/s; 28150/162936 objects degraded (17.277%); 6054/162936 objects misplaced (3.716%); 2.1 MiB/s, 86 objects/s recovering 2023-09-23T21:13:36.635 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:13:36 smithi026 ceph-mon[179688]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:13:36.635 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:13:36 smithi026 ceph-mon[179688]: failed to encode map e506 with expected crc 2023-09-23T21:13:36.635 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:13:36 smithi026 ceph-mon[179688]: failed to encode map e506 with expected crc 2023-09-23T21:13:36.635 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:13:36 smithi026 ceph-mon[179688]: failed to encode map e506 with expected crc 2023-09-23T21:13:36.635 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:13:36 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T21:13:36.635 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:13:36 smithi026 ceph-mon[179688]: osdmap e506: 8 total, 8 up, 6 in 2023-09-23T21:13:36.635 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:13:36 smithi026 ceph-mon[179688]: failed to encode map e506 with expected crc 2023-09-23T21:13:36.635 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:13:36 smithi026 ceph-mon[179688]: Health check update: Degraded data redundancy: 28150/162936 objects degraded (17.277%), 23 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T21:13:36.719 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:13:36.750 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:13:36.793 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:13:36.796 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:13:36.796 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:13:36.897 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:13:36.943 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:13:37.368 INFO:teuthology.orchestra.run.smithi026.stderr:reweighted osd.2 to 1 (10000) 2023-09-23T21:13:37.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:13:37 smithi070 ceph-mon[168282]: failed to encode map e506 with expected crc 2023-09-23T21:13:37.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:13:37 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/2676632516' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T21:13:37.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:13:37 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T21:13:37.631 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:13:37 smithi026 ceph-mon[175151]: failed to encode map e506 with expected crc 2023-09-23T21:13:37.631 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:13:37 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/2676632516' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T21:13:37.631 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:13:37 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T21:13:37.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:13:37 smithi026 ceph-mon[179688]: failed to encode map e506 with expected crc 2023-09-23T21:13:37.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:13:37 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/2676632516' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T21:13:37.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:13:37 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T21:13:38.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:13:38 smithi070 ceph-mon[168282]: pgmap v1280: 129 pgs: 2 active+recovering+undersized+remapped, 2 active+undersized+degraded+remapped, 7 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 15 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 101 active+clean; 3.3 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 7.9 KiB/s rd, 21 KiB/s wr, 85 op/s; 33516/162498 objects degraded (20.625%); 8235/162498 objects misplaced (5.068%); 6.1 MiB/s, 263 objects/s recovering 2023-09-23T21:13:38.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:13:38 smithi070 ceph-mon[168282]: failed to encode map e507 with expected crc 2023-09-23T21:13:38.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:13:38 smithi070 ceph-mon[168282]: failed to encode map e507 with expected crc 2023-09-23T21:13:38.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:13:38 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-09-23T21:13:38.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:13:38 smithi070 ceph-mon[168282]: osdmap e507: 8 total, 8 up, 6 in 2023-09-23T21:13:38.631 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:13:38 smithi026 ceph-mon[175151]: pgmap v1280: 129 pgs: 2 active+recovering+undersized+remapped, 2 active+undersized+degraded+remapped, 7 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 15 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 101 active+clean; 3.3 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 7.9 KiB/s rd, 21 KiB/s wr, 85 op/s; 33516/162498 objects degraded (20.625%); 8235/162498 objects misplaced (5.068%); 6.1 MiB/s, 263 objects/s recovering 2023-09-23T21:13:38.631 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:13:38 smithi026 ceph-mon[175151]: failed to encode map e507 with expected crc 2023-09-23T21:13:38.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:13:38 smithi026 ceph-mon[175151]: failed to encode map e507 with expected crc 2023-09-23T21:13:38.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:13:38 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-09-23T21:13:38.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:13:38 smithi026 ceph-mon[175151]: osdmap e507: 8 total, 8 up, 6 in 2023-09-23T21:13:38.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:13:38 smithi026 ceph-mon[179688]: pgmap v1280: 129 pgs: 2 active+recovering+undersized+remapped, 2 active+undersized+degraded+remapped, 7 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 15 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 101 active+clean; 3.3 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 7.9 KiB/s rd, 21 KiB/s wr, 85 op/s; 33516/162498 objects degraded (20.625%); 8235/162498 objects misplaced (5.068%); 6.1 MiB/s, 263 objects/s recovering 2023-09-23T21:13:38.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:13:38 smithi026 ceph-mon[179688]: failed to encode map e507 with expected crc 2023-09-23T21:13:38.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:13:38 smithi026 ceph-mon[179688]: failed to encode map e507 with expected crc 2023-09-23T21:13:38.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:13:38 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-09-23T21:13:38.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:13:38 smithi026 ceph-mon[179688]: osdmap e507: 8 total, 8 up, 6 in 2023-09-23T21:13:39.611 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:13:39 smithi026 ceph-mon[179688]: failed to encode map e508 with expected crc 2023-09-23T21:13:39.611 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:13:39 smithi026 ceph-mon[179688]: failed to encode map e508 with expected crc 2023-09-23T21:13:39.611 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:13:39 smithi026 ceph-mon[179688]: osdmap e508: 8 total, 8 up, 6 in 2023-09-23T21:13:39.611 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:13:39 smithi026 ceph-mon[179688]: failed to encode map e508 with expected crc 2023-09-23T21:13:39.612 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:13:39 smithi026 ceph-mon[179688]: failed to encode map e508 with expected crc 2023-09-23T21:13:39.612 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:13:39 smithi026 ceph-mon[179688]: failed to encode map e508 with expected crc 2023-09-23T21:13:39.612 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:13:39 smithi026 ceph-mon[179688]: failed to encode map e508 with expected crc 2023-09-23T21:13:39.612 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:13:39 smithi026 ceph-mon[175151]: failed to encode map e508 with expected crc 2023-09-23T21:13:39.612 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:13:39 smithi026 ceph-mon[175151]: failed to encode map e508 with expected crc 2023-09-23T21:13:39.612 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:13:39 smithi026 ceph-mon[175151]: osdmap e508: 8 total, 8 up, 6 in 2023-09-23T21:13:39.612 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:13:39 smithi026 ceph-mon[175151]: failed to encode map e508 with expected crc 2023-09-23T21:13:39.612 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:13:39 smithi026 ceph-mon[175151]: failed to encode map e508 with expected crc 2023-09-23T21:13:39.612 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:13:39 smithi026 ceph-mon[175151]: failed to encode map e508 with expected crc 2023-09-23T21:13:39.613 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:13:39 smithi026 ceph-mon[175151]: failed to encode map e508 with expected crc 2023-09-23T21:13:39.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:13:39 smithi070 ceph-mon[168282]: failed to encode map e508 with expected crc 2023-09-23T21:13:39.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:13:39 smithi070 ceph-mon[168282]: failed to encode map e508 with expected crc 2023-09-23T21:13:39.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:13:39 smithi070 ceph-mon[168282]: osdmap e508: 8 total, 8 up, 6 in 2023-09-23T21:13:39.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:13:39 smithi070 ceph-mon[168282]: failed to encode map e508 with expected crc 2023-09-23T21:13:39.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:13:39 smithi070 ceph-mon[168282]: failed to encode map e508 with expected crc 2023-09-23T21:13:39.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:13:39 smithi070 ceph-mon[168282]: failed to encode map e508 with expected crc 2023-09-23T21:13:39.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:13:39 smithi070 ceph-mon[168282]: failed to encode map e508 with expected crc 2023-09-23T21:13:39.635 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:13:39.639 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:13:39.639 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:13:39.740 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:13:39.753 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph osd reweight 3 1 2023-09-23T21:13:39.775 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:13:39.881 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 21:13:39 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[164029]: ::ffff:172.21.15.70 - - [23/Sep/2023:21:13:39] "GET /metrics HTTP/1.1" 200 36435 "" "Prometheus/2.43.0" 2023-09-23T21:13:39.915 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T21:13:39.918 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:13:39.918 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:13:39.982 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:13:40.020 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:13:40.045 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:13:40.046 INFO:teuthology.orchestra.run.smithi026.stdout:ERROR: (22) Invalid argument 2023-09-23T21:13:40.046 INFO:teuthology.orchestra.run.smithi026.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T21:13:40.083 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:13:40.086 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:13:40.089 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:13:40.190 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:13:40.219 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:13:40.455 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:13:40 smithi026 ceph-mon[175151]: pgmap v1283: 129 pgs: 2 active+recovering+undersized+remapped, 2 active+undersized+degraded+remapped, 7 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 15 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 101 active+clean; 3.3 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 8.0 KiB/s rd, 21 KiB/s wr, 69 op/s; 33516/162306 objects degraded (20.650%); 8235/162306 objects misplaced (5.074%); 6.1 MiB/s, 263 objects/s recovering 2023-09-23T21:13:40.456 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:13:40 smithi026 ceph-mon[175151]: failed to encode map e509 with expected crc 2023-09-23T21:13:40.456 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:13:40 smithi026 ceph-mon[175151]: failed to encode map e509 with expected crc 2023-09-23T21:13:40.456 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:13:40 smithi026 ceph-mon[175151]: osdmap e509: 8 total, 8 up, 6 in 2023-09-23T21:13:40.456 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:13:40 smithi026 ceph-mon[175151]: failed to encode map e509 with expected crc 2023-09-23T21:13:40.456 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:13:40 smithi026 ceph-mon[175151]: failed to encode map e509 with expected crc 2023-09-23T21:13:40.456 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:13:40 smithi026 ceph-mon[175151]: failed to encode map e509 with expected crc 2023-09-23T21:13:40.456 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:13:40 smithi026 ceph-mon[179688]: pgmap v1283: 129 pgs: 2 active+recovering+undersized+remapped, 2 active+undersized+degraded+remapped, 7 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 15 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 101 active+clean; 3.3 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 8.0 KiB/s rd, 21 KiB/s wr, 69 op/s; 33516/162306 objects degraded (20.650%); 8235/162306 objects misplaced (5.074%); 6.1 MiB/s, 263 objects/s recovering 2023-09-23T21:13:40.456 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:13:40 smithi026 ceph-mon[179688]: failed to encode map e509 with expected crc 2023-09-23T21:13:40.457 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:13:40 smithi026 ceph-mon[179688]: failed to encode map e509 with expected crc 2023-09-23T21:13:40.457 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:13:40 smithi026 ceph-mon[179688]: osdmap e509: 8 total, 8 up, 6 in 2023-09-23T21:13:40.457 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:13:40 smithi026 ceph-mon[179688]: failed to encode map e509 with expected crc 2023-09-23T21:13:40.457 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:13:40 smithi026 ceph-mon[179688]: failed to encode map e509 with expected crc 2023-09-23T21:13:40.457 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:13:40 smithi026 ceph-mon[179688]: failed to encode map e509 with expected crc 2023-09-23T21:13:40.485 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:13:40.488 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:13:40.488 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:13:40.512 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_ops_in_flight 2023-09-23T21:13:40.589 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:13:40.619 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:13:40.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:13:40 smithi070 ceph-mon[168282]: pgmap v1283: 129 pgs: 2 active+recovering+undersized+remapped, 2 active+undersized+degraded+remapped, 7 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 15 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 101 active+clean; 3.3 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 8.0 KiB/s rd, 21 KiB/s wr, 69 op/s; 33516/162306 objects degraded (20.650%); 8235/162306 objects misplaced (5.074%); 6.1 MiB/s, 263 objects/s recovering 2023-09-23T21:13:40.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:13:40 smithi070 ceph-mon[168282]: failed to encode map e509 with expected crc 2023-09-23T21:13:40.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:13:40 smithi070 ceph-mon[168282]: failed to encode map e509 with expected crc 2023-09-23T21:13:40.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:13:40 smithi070 ceph-mon[168282]: osdmap e509: 8 total, 8 up, 6 in 2023-09-23T21:13:40.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:13:40 smithi070 ceph-mon[168282]: failed to encode map e509 with expected crc 2023-09-23T21:13:40.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:13:40 smithi070 ceph-mon[168282]: failed to encode map e509 with expected crc 2023-09-23T21:13:40.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:13:40 smithi070 ceph-mon[168282]: failed to encode map e509 with expected crc 2023-09-23T21:13:40.805 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:13:40.808 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:13:40.808 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:13:40.832 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:13:40.909 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:13:40.938 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:13:40.972 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:13:40.975 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:13:40.975 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:13:41.076 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:13:41.103 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:13:41.137 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:13:41.139 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:13:41.140 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:13:41.214 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T21:13:41.240 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:13:41.268 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:13:41.352 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:13:41.357 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:13:41.358 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:13:41.458 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:13:41.488 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:13:41.540 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:13:41.547 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:13:41.547 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:13:41.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:13:41 smithi026 ceph-mon[175151]: failed to encode map e510 with expected crc 2023-09-23T21:13:41.633 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:13:41 smithi026 ceph-mon[175151]: failed to encode map e510 with expected crc 2023-09-23T21:13:41.633 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:13:41 smithi026 ceph-mon[175151]: osdmap e510: 8 total, 8 up, 6 in 2023-09-23T21:13:41.633 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:13:41 smithi026 ceph-mon[175151]: failed to encode map e510 with expected crc 2023-09-23T21:13:41.633 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:13:41 smithi026 ceph-mon[175151]: failed to encode map e510 with expected crc 2023-09-23T21:13:41.633 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:13:41 smithi026 ceph-mon[175151]: failed to encode map e510 with expected crc 2023-09-23T21:13:41.633 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:13:41 smithi026 ceph-mon[175151]: failed to encode map e510 with expected crc 2023-09-23T21:13:41.633 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:13:41 smithi026 ceph-mon[175151]: failed to encode map e510 with expected crc 2023-09-23T21:13:41.633 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:13:41 smithi026 ceph-mon[175151]: failed to encode map e510 with expected crc 2023-09-23T21:13:41.633 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:13:41 smithi026 ceph-mon[175151]: Health check update: Degraded data redundancy: 33516/162306 objects degraded (20.650%), 25 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T21:13:41.633 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:13:41 smithi026 ceph-mon[179688]: failed to encode map e510 with expected crc 2023-09-23T21:13:41.634 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:13:41 smithi026 ceph-mon[179688]: failed to encode map e510 with expected crc 2023-09-23T21:13:41.634 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:13:41 smithi026 ceph-mon[179688]: osdmap e510: 8 total, 8 up, 6 in 2023-09-23T21:13:41.634 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:13:41 smithi026 ceph-mon[179688]: failed to encode map e510 with expected crc 2023-09-23T21:13:41.634 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:13:41 smithi026 ceph-mon[179688]: failed to encode map e510 with expected crc 2023-09-23T21:13:41.634 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:13:41 smithi026 ceph-mon[179688]: failed to encode map e510 with expected crc 2023-09-23T21:13:41.634 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:13:41 smithi026 ceph-mon[179688]: failed to encode map e510 with expected crc 2023-09-23T21:13:41.634 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:13:41 smithi026 ceph-mon[179688]: failed to encode map e510 with expected crc 2023-09-23T21:13:41.634 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:13:41 smithi026 ceph-mon[179688]: failed to encode map e510 with expected crc 2023-09-23T21:13:41.634 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:13:41 smithi026 ceph-mon[179688]: Health check update: Degraded data redundancy: 33516/162306 objects degraded (20.650%), 25 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T21:13:41.648 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:13:41.670 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T21:13:41.688 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:13:41.729 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:13:41 smithi070 ceph-mon[168282]: failed to encode map e510 with expected crc 2023-09-23T21:13:41.730 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:13:41 smithi070 ceph-mon[168282]: failed to encode map e510 with expected crc 2023-09-23T21:13:41.730 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:13:41 smithi070 ceph-mon[168282]: osdmap e510: 8 total, 8 up, 6 in 2023-09-23T21:13:41.730 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:13:41 smithi070 ceph-mon[168282]: failed to encode map e510 with expected crc 2023-09-23T21:13:41.730 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:13:41 smithi070 ceph-mon[168282]: failed to encode map e510 with expected crc 2023-09-23T21:13:41.730 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:13:41 smithi070 ceph-mon[168282]: failed to encode map e510 with expected crc 2023-09-23T21:13:41.730 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:13:41 smithi070 ceph-mon[168282]: failed to encode map e510 with expected crc 2023-09-23T21:13:41.730 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:13:41 smithi070 ceph-mon[168282]: failed to encode map e510 with expected crc 2023-09-23T21:13:41.730 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:13:41 smithi070 ceph-mon[168282]: failed to encode map e510 with expected crc 2023-09-23T21:13:41.730 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:13:41 smithi070 ceph-mon[168282]: Health check update: Degraded data redundancy: 33516/162306 objects degraded (20.650%), 25 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T21:13:41.946 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:13:41.951 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:13:41.951 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:13:42.052 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:13:42.102 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:13:42.272 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:13:42.283 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:13:42.284 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:13:42.297 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0: {} 2023-09-23T21:13:42.319 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1: {} 2023-09-23T21:13:42.343 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2: {} 2023-09-23T21:13:42.364 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3: {} 2023-09-23T21:13:42.385 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:13:42.391 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4: {} 2023-09-23T21:13:42.415 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T21:13:42.421 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:13:42.442 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6: {} 2023-09-23T21:13:42.470 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T21:13:42.472 INFO:teuthology.orchestra.run.smithi026.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T21:13:42.472 INFO:teuthology.orchestra.run.smithi026.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T21:13:42.472 INFO:teuthology.orchestra.run.smithi026.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T21:13:42.473 INFO:teuthology.orchestra.run.smithi026.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T21:13:42.473 INFO:teuthology.orchestra.run.smithi026.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T21:13:42.473 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T21:13:42.473 INFO:teuthology.orchestra.run.smithi026.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T21:13:42.473 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T21:13:42.510 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:13:42.515 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:13:42.515 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:13:42.597 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:13:42 smithi070 ceph-mon[168282]: pgmap v1286: 129 pgs: 1 remapped+peering, 1 active+recovering+undersized+remapped, 2 active+undersized+degraded+remapped, 5 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 15 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped, 1 active+recovery_wait+degraded+remapped, 101 active+clean; 3.3 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 48 op/s; 32451/161916 objects degraded (20.042%); 7377/161916 objects misplaced (4.556%); 14 MiB/s, 221 objects/s recovering 2023-09-23T21:13:42.597 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:13:42 smithi070 ceph-mon[168282]: failed to encode map e511 with expected crc 2023-09-23T21:13:42.598 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:13:42 smithi070 ceph-mon[168282]: failed to encode map e511 with expected crc 2023-09-23T21:13:42.598 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:13:42 smithi070 ceph-mon[168282]: osdmap e511: 8 total, 8 up, 6 in 2023-09-23T21:13:42.598 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:13:42 smithi070 ceph-mon[168282]: failed to encode map e511 with expected crc 2023-09-23T21:13:42.598 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:13:42 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/3805616460' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:13:42.616 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:13:42.631 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:13:42 smithi026 ceph-mon[175151]: pgmap v1286: 129 pgs: 1 remapped+peering, 1 active+recovering+undersized+remapped, 2 active+undersized+degraded+remapped, 5 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 15 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped, 1 active+recovery_wait+degraded+remapped, 101 active+clean; 3.3 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 48 op/s; 32451/161916 objects degraded (20.042%); 7377/161916 objects misplaced (4.556%); 14 MiB/s, 221 objects/s recovering 2023-09-23T21:13:42.631 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:13:42 smithi026 ceph-mon[175151]: failed to encode map e511 with expected crc 2023-09-23T21:13:42.631 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:13:42 smithi026 ceph-mon[175151]: failed to encode map e511 with expected crc 2023-09-23T21:13:42.631 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:13:42 smithi026 ceph-mon[175151]: osdmap e511: 8 total, 8 up, 6 in 2023-09-23T21:13:42.631 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:13:42 smithi026 ceph-mon[175151]: failed to encode map e511 with expected crc 2023-09-23T21:13:42.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:13:42 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/3805616460' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:13:42.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:13:42 smithi026 ceph-mon[179688]: pgmap v1286: 129 pgs: 1 remapped+peering, 1 active+recovering+undersized+remapped, 2 active+undersized+degraded+remapped, 5 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 15 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped, 1 active+recovery_wait+degraded+remapped, 101 active+clean; 3.3 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 48 op/s; 32451/161916 objects degraded (20.042%); 7377/161916 objects misplaced (4.556%); 14 MiB/s, 221 objects/s recovering 2023-09-23T21:13:42.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:13:42 smithi026 ceph-mon[179688]: failed to encode map e511 with expected crc 2023-09-23T21:13:42.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:13:42 smithi026 ceph-mon[179688]: failed to encode map e511 with expected crc 2023-09-23T21:13:42.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:13:42 smithi026 ceph-mon[179688]: osdmap e511: 8 total, 8 up, 6 in 2023-09-23T21:13:42.633 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:13:42 smithi026 ceph-mon[179688]: failed to encode map e511 with expected crc 2023-09-23T21:13:42.633 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:13:42 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/3805616460' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:13:42.637 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:13:42.655 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:13:42.775 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:13:42.779 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:13:42.779 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:13:42.852 INFO:teuthology.orchestra.run.smithi070.stdout:{ 2023-09-23T21:13:42.852 INFO:teuthology.orchestra.run.smithi070.stdout: "ops": [ 2023-09-23T21:13:42.853 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:13:42.853 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_repop_reply(client.46328.0:107048 10.13 e511/508)", 2023-09-23T21:13:42.853 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:13:42.832790+0000", 2023-09-23T21:13:42.853 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 0.016972458999999999, 2023-09-23T21:13:42.853 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.016990073000000001, 2023-09-23T21:13:42.853 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:13:42.853 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "queued for pg", 2023-09-23T21:13:42.853 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:13:42.853 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:13:42.853 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:13:42.853 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:42.832790+0000", 2023-09-23T21:13:42.853 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:13:42.854 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:13:42.854 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:13:42.854 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:13:42.854 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:42.832790+0000", 2023-09-23T21:13:42.854 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:13:42.854 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:13:42.854 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:13:42.854 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:13:42.854 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:42.832789+0000", 2023-09-23T21:13:42.854 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.999999 2023-09-23T21:13:42.854 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:13:42.855 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:13:42.855 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:13:42.855 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:42.832795+0000", 2023-09-23T21:13:42.855 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.7300000000000002e-06 2023-09-23T21:13:42.855 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:13:42.855 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:13:42.855 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:13:42.855 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:42.832806+0000", 2023-09-23T21:13:42.855 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.0856000000000001e-05 2023-09-23T21:13:42.855 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:13:42.855 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:13:42.855 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:13:42.855 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:42.832810+0000", 2023-09-23T21:13:42.856 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.532e-06 2023-09-23T21:13:42.856 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:13:42.856 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:13:42.856 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:13:42.856 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:13:42.856 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:13:42.856 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.46328.0:106961 10.10 10:0c9e5708:::benchmark_data_smithi026_379838_object27004:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e511)", 2023-09-23T21:13:42.856 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:13:42.509399+0000", 2023-09-23T21:13:42.856 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 0.340363586, 2023-09-23T21:13:42.856 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.340417684, 2023-09-23T21:13:42.856 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:13:42.857 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "delayed", 2023-09-23T21:13:42.857 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:13:42.857 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.46328", 2023-09-23T21:13:42.857 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/3748861181", 2023-09-23T21:13:42.857 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 106961 2023-09-23T21:13:42.857 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:13:42.857 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:13:42.857 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:13:42.857 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:13:42.857 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:42.509399+0000", 2023-09-23T21:13:42.857 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:13:42.857 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:13:42.858 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:13:42.858 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:13:42.858 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:42.509399+0000", 2023-09-23T21:13:42.858 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:13:42.858 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:13:42.858 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:13:42.858 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:13:42.858 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:42.509398+0000", 2023-09-23T21:13:42.858 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.999999 2023-09-23T21:13:42.858 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:13:42.858 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:13:42.858 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:13:42.859 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:42.509403+0000", 2023-09-23T21:13:42.859 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.5379999999999996e-06 2023-09-23T21:13:42.859 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:13:42.859 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:13:42.859 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:13:42.859 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:42.509413+0000", 2023-09-23T21:13:42.859 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.0028e-05 2023-09-23T21:13:42.859 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:13:42.859 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:13:42.859 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:13:42.859 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:42.509417+0000", 2023-09-23T21:13:42.859 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.8520000000000003e-06 2023-09-23T21:13:42.859 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:13:42.860 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:13:42.860 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:13:42.860 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:42.527662+0000", 2023-09-23T21:13:42.860 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.018245022999999999 2023-09-23T21:13:42.860 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:13:42.860 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:13:42.860 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for missing object", 2023-09-23T21:13:42.860 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:42.530408+0000", 2023-09-23T21:13:42.860 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0027456939999999999 2023-09-23T21:13:42.860 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:13:42.860 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:13:42.860 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:13:42.860 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:42.554950+0000", 2023-09-23T21:13:42.861 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.024541457999999999 2023-09-23T21:13:42.861 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:13:42.861 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:13:42.861 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for rw locks", 2023-09-23T21:13:42.861 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:42.557408+0000", 2023-09-23T21:13:42.861 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0024582950000000001 2023-09-23T21:13:42.861 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:13:42.861 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:13:42.861 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:13:42.861 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:13:42.861 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:13:42.861 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_repop_reply(client.46328.0:107048 10.13 e511/508 ondisk, result = 0)", 2023-09-23T21:13:42.862 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:13:42.834432+0000", 2023-09-23T21:13:42.862 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 0.01533053, 2023-09-23T21:13:42.862 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.015431656, 2023-09-23T21:13:42.862 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:13:42.862 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "started", 2023-09-23T21:13:42.862 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:13:42.862 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:13:42.862 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:13:42.862 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:42.834432+0000", 2023-09-23T21:13:42.862 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:13:42.862 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:13:42.863 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:13:42.863 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:13:42.863 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:42.834432+0000", 2023-09-23T21:13:42.863 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:13:42.863 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:13:42.863 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:13:42.863 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:13:42.863 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:42.834431+0000", 2023-09-23T21:13:42.863 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999995 2023-09-23T21:13:42.863 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:13:42.863 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:13:42.863 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:13:42.864 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:42.834436+0000", 2023-09-23T21:13:42.864 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.9699999999999998e-06 2023-09-23T21:13:42.864 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:13:42.864 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:13:42.864 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:13:42.864 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:42.834449+0000", 2023-09-23T21:13:42.864 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.3303e-05 2023-09-23T21:13:42.864 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:13:42.864 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:13:42.864 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:13:42.864 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:42.834455+0000", 2023-09-23T21:13:42.864 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.3040000000000004e-06 2023-09-23T21:13:42.865 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:13:42.865 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:13:42.865 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:13:42.865 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:42.848957+0000", 2023-09-23T21:13:42.865 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.014502525 2023-09-23T21:13:42.865 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:13:42.865 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:13:42.865 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:13:42.865 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:42.849274+0000", 2023-09-23T21:13:42.865 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000316538 2023-09-23T21:13:42.865 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:13:42.865 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:13:42.865 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:13:42.866 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:13:42.866 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:13:42.866 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.46328.0:107067 10.0 10:076ef3f4:::benchmark_data_smithi026_379838_object27110:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e511)", 2023-09-23T21:13:42.866 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:13:42.840052+0000", 2023-09-23T21:13:42.866 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 0.0097100350000000005, 2023-09-23T21:13:42.866 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0098556809999999998, 2023-09-23T21:13:42.866 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:13:42.866 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "reached pg", 2023-09-23T21:13:42.866 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:13:42.866 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.46328", 2023-09-23T21:13:42.866 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/3748861181", 2023-09-23T21:13:42.866 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 107067 2023-09-23T21:13:42.867 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:13:42.867 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:13:42.867 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:13:42.867 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:13:42.867 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:42.840052+0000", 2023-09-23T21:13:42.867 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:13:42.867 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:13:42.867 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:13:42.867 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:13:42.867 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:42.840052+0000", 2023-09-23T21:13:42.867 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:13:42.867 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:13:42.867 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:13:42.868 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:13:42.868 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:42.840051+0000", 2023-09-23T21:13:42.868 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999981 2023-09-23T21:13:42.868 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:13:42.868 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:13:42.868 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:13:42.868 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:42.840058+0000", 2023-09-23T21:13:42.868 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.6899999999999992e-06 2023-09-23T21:13:42.868 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:13:42.868 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:13:42.868 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:13:42.868 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:42.840073+0000", 2023-09-23T21:13:42.869 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.4273e-05 2023-09-23T21:13:42.869 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:13:42.869 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:13:42.869 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:13:42.869 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:42.840082+0000", 2023-09-23T21:13:42.869 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.3370000000000006e-06 2023-09-23T21:13:42.869 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:13:42.869 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:13:42.869 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:13:42.869 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:42.847709+0000", 2023-09-23T21:13:42.869 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0076276479999999999 2023-09-23T21:13:42.869 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:13:42.869 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:13:42.870 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:13:42.870 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:13:42.870 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:13:42.870 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.46328.0:107048 10.13 10:ce991469:::benchmark_data_smithi026_379838_object27091:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e511)", 2023-09-23T21:13:42.870 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:13:42.761839+0000", 2023-09-23T21:13:42.870 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 0.087923715999999999, 2023-09-23T21:13:42.870 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.088106925000000003, 2023-09-23T21:13:42.870 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:13:42.870 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "waiting for sub ops", 2023-09-23T21:13:42.870 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:13:42.870 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.46328", 2023-09-23T21:13:42.871 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/3748861181", 2023-09-23T21:13:42.871 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 107048 2023-09-23T21:13:42.871 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:13:42.871 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:13:42.871 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:13:42.871 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:13:42.871 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:42.761839+0000", 2023-09-23T21:13:42.871 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:13:42.871 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:13:42.871 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:13:42.871 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:13:42.871 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:42.761839+0000", 2023-09-23T21:13:42.872 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:13:42.872 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:13:42.872 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:13:42.872 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:13:42.872 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:42.761838+0000", 2023-09-23T21:13:42.872 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.999999 2023-09-23T21:13:42.872 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:13:42.872 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:13:42.872 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:13:42.872 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:42.761843+0000", 2023-09-23T21:13:42.872 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.3329999999999996e-06 2023-09-23T21:13:42.872 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:13:42.872 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:13:42.873 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:13:42.873 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:42.761854+0000", 2023-09-23T21:13:42.873 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.1338999999999999e-05 2023-09-23T21:13:42.873 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:13:42.873 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:13:42.873 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:13:42.873 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:42.761860+0000", 2023-09-23T21:13:42.873 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.6269999999999997e-06 2023-09-23T21:13:42.873 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:13:42.873 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:13:42.873 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:13:42.873 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:42.765638+0000", 2023-09-23T21:13:42.873 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0037778899999999999 2023-09-23T21:13:42.874 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:13:42.874 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:13:42.874 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for missing object", 2023-09-23T21:13:42.874 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:42.768023+0000", 2023-09-23T21:13:42.874 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0023846520000000001 2023-09-23T21:13:42.874 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:13:42.874 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:13:42.874 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:13:42.874 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:42.801504+0000", 2023-09-23T21:13:42.874 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.033481294000000002 2023-09-23T21:13:42.874 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:13:42.874 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:13:42.875 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for rw locks", 2023-09-23T21:13:42.875 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:42.804926+0000", 2023-09-23T21:13:42.875 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.003421856 2023-09-23T21:13:42.875 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:13:42.875 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:13:42.875 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:13:42.875 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:42.824320+0000", 2023-09-23T21:13:42.875 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.019394426999999999 2023-09-23T21:13:42.875 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:13:42.875 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:13:42.875 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:13:42.875 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:42.826320+0000", 2023-09-23T21:13:42.875 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0019994549999999998 2023-09-23T21:13:42.876 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:13:42.876 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:13:42.876 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 2,5", 2023-09-23T21:13:42.876 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:42.830648+0000", 2023-09-23T21:13:42.876 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0043282809999999998 2023-09-23T21:13:42.876 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:13:42.876 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:13:42.876 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:13:42.876 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:42.849579+0000", 2023-09-23T21:13:42.876 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.018930755 2023-09-23T21:13:42.876 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:13:42.876 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:13:42.877 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:13:42.877 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:42.849890+0000", 2023-09-23T21:13:42.877 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000311143 2023-09-23T21:13:42.877 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:13:42.877 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:13:42.877 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:13:42.877 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:13:42.877 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:13:42.877 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.46328.0:107070 10.1 10:82b1f41a:::benchmark_data_smithi026_379838_object27113:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e511)", 2023-09-23T21:13:42.877 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:13:42.840148+0000", 2023-09-23T21:13:42.877 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 0.0096140089999999997, 2023-09-23T21:13:42.877 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0098572720000000003, 2023-09-23T21:13:42.878 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:13:42.878 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "waiting for sub ops", 2023-09-23T21:13:42.878 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:13:42.878 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.46328", 2023-09-23T21:13:42.878 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/3748861181", 2023-09-23T21:13:42.878 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 107070 2023-09-23T21:13:42.878 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:13:42.878 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:13:42.878 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:13:42.878 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:13:42.878 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:42.840148+0000", 2023-09-23T21:13:42.879 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:13:42.879 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:13:42.879 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:13:42.879 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:13:42.879 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:42.840148+0000", 2023-09-23T21:13:42.879 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:13:42.879 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:13:42.879 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:13:42.879 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:13:42.879 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:42.840148+0000", 2023-09-23T21:13:42.879 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999995 2023-09-23T21:13:42.879 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:13:42.879 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:13:42.880 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:13:42.880 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:42.840152+0000", 2023-09-23T21:13:42.880 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.8089999999999999e-06 2023-09-23T21:13:42.880 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:13:42.880 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:13:42.880 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:13:42.880 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:42.840162+0000", 2023-09-23T21:13:42.880 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.0161e-05 2023-09-23T21:13:42.880 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:13:42.880 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:13:42.880 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:13:42.880 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:42.840168+0000", 2023-09-23T21:13:42.881 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.0070000000000003e-06 2023-09-23T21:13:42.881 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:13:42.881 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:13:42.881 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:13:42.881 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:42.840544+0000", 2023-09-23T21:13:42.881 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000376012 2023-09-23T21:13:42.881 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:13:42.882 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:13:42.882 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:13:42.882 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:42.842575+0000", 2023-09-23T21:13:42.882 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0020309159999999998 2023-09-23T21:13:42.882 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:13:42.882 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:13:42.882 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 0,7", 2023-09-23T21:13:42.882 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:42.845321+0000", 2023-09-23T21:13:42.882 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0027467020000000002 2023-09-23T21:13:42.882 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:13:42.883 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:13:42.883 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:13:42.883 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:42.848801+0000", 2023-09-23T21:13:42.883 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.003479822 2023-09-23T21:13:42.883 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:13:42.883 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:13:42.883 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:13:42.883 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:42.849183+0000", 2023-09-23T21:13:42.883 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00038208600000000001 2023-09-23T21:13:42.883 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:13:42.883 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:13:42.884 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:13:42.884 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:42.849368+0000", 2023-09-23T21:13:42.884 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00018431599999999999 2023-09-23T21:13:42.884 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:13:42.884 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:13:42.884 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:13:42.884 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:13:42.884 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:13:42.884 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_repop(client.46328.0:107068 10.b e511/494)", 2023-09-23T21:13:42.884 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:13:42.840602+0000", 2023-09-23T21:13:42.884 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 0.0091599660000000003, 2023-09-23T21:13:42.885 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0094516189999999996, 2023-09-23T21:13:42.885 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:13:42.885 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "queued for pg", 2023-09-23T21:13:42.885 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:13:42.885 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:13:42.885 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:13:42.885 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:42.840602+0000", 2023-09-23T21:13:42.885 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:13:42.885 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:13:42.885 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:13:42.885 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:13:42.886 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:42.840602+0000", 2023-09-23T21:13:42.886 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:13:42.886 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:13:42.886 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:13:42.886 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:13:42.886 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:42.840602+0000", 2023-09-23T21:13:42.886 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.999999 2023-09-23T21:13:42.886 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:13:42.886 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:13:42.886 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:13:42.886 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:42.840610+0000", 2023-09-23T21:13:42.887 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.7679999999999997e-06 2023-09-23T21:13:42.887 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:13:42.887 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:13:42.887 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:13:42.887 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:42.840628+0000", 2023-09-23T21:13:42.887 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.7329999999999998e-05 2023-09-23T21:13:42.887 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:13:42.887 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:13:42.887 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:13:42.887 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:42.840635+0000", 2023-09-23T21:13:42.887 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.8940000000000004e-06 2023-09-23T21:13:42.887 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:13:42.888 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:13:42.888 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:13:42.888 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:13:42.888 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:13:42.888 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "MOSDPGPush(10.3 511/508 [PushOp(10:c7a2cd9f:::benchmark_data_smithi026_379838_object27112:head, version: 435'1684, data_included: [0~65536], data_size: 65536, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(10:c7a2cd9f:::benchmark_data_smithi026_379838_object27112:head@435'1684, size: 65536, copy_subset: [0~65536], clone_subset: {}, snapset: 0=[]:{}, object_exist: 0), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:65536, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false, error:false))])", 2023-09-23T21:13:42.888 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:13:42.840546+0000", 2023-09-23T21:13:42.888 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 0.0092164699999999992, 2023-09-23T21:13:42.888 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0095516909999999993, 2023-09-23T21:13:42.888 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:13:42.888 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "queued for pg", 2023-09-23T21:13:42.888 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:13:42.888 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:13:42.888 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:13:42.889 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:42.840546+0000", 2023-09-23T21:13:42.889 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:13:42.889 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:13:42.889 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:13:42.889 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:13:42.889 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:42.840546+0000", 2023-09-23T21:13:42.889 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:13:42.889 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:13:42.889 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:13:42.889 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:13:42.889 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:42.840545+0000", 2023-09-23T21:13:42.889 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999995 2023-09-23T21:13:42.890 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:13:42.890 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:13:42.890 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:13:42.890 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:42.840610+0000", 2023-09-23T21:13:42.890 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.5105999999999996e-05 2023-09-23T21:13:42.890 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:13:42.890 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:13:42.890 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:13:42.890 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:42.840629+0000", 2023-09-23T21:13:42.890 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.8309999999999999e-05 2023-09-23T21:13:42.890 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:13:42.890 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:13:42.891 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:13:42.891 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:42.840639+0000", 2023-09-23T21:13:42.891 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.0380999999999999e-05 2023-09-23T21:13:42.891 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:13:42.891 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:13:42.891 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:13:42.891 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:13:42.891 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:13:42.891 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.46328.0:106963 10.0 10:0197b016:::benchmark_data_smithi026_379838_object27006:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e511)", 2023-09-23T21:13:42.891 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:13:42.512664+0000", 2023-09-23T21:13:42.891 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 0.33709835700000002, 2023-09-23T21:13:42.891 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.33746238899999997, 2023-09-23T21:13:42.892 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:13:42.892 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "delayed", 2023-09-23T21:13:42.892 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:13:42.892 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.46328", 2023-09-23T21:13:42.892 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/3748861181", 2023-09-23T21:13:42.892 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 106963 2023-09-23T21:13:42.892 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:13:42.892 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:13:42.892 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:13:42.892 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:13:42.892 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:42.512664+0000", 2023-09-23T21:13:42.892 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:13:42.892 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:13:42.893 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:13:42.893 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:13:42.893 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:42.512664+0000", 2023-09-23T21:13:42.893 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:13:42.893 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:13:42.893 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:13:42.893 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:13:42.893 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:42.512664+0000", 2023-09-23T21:13:42.893 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999995 2023-09-23T21:13:42.893 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:13:42.893 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:13:42.893 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:13:42.894 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:42.512666+0000", 2023-09-23T21:13:42.894 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.2869999999999999e-06 2023-09-23T21:13:42.894 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:13:42.894 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:13:42.894 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:13:42.894 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:42.512677+0000", 2023-09-23T21:13:42.894 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.1282e-05 2023-09-23T21:13:42.894 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:13:42.894 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:13:42.894 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:13:42.894 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:42.512681+0000", 2023-09-23T21:13:42.894 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.6799999999999999e-06 2023-09-23T21:13:42.895 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:13:42.895 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:13:42.895 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:13:42.895 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:42.527941+0000", 2023-09-23T21:13:42.895 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.015260546999999999 2023-09-23T21:13:42.895 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:13:42.895 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:13:42.895 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for missing object", 2023-09-23T21:13:42.895 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:42.531065+0000", 2023-09-23T21:13:42.895 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0031238529999999998 2023-09-23T21:13:42.895 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:13:42.895 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:13:42.896 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:13:42.896 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:42.555140+0000", 2023-09-23T21:13:42.896 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.024074476000000001 2023-09-23T21:13:42.896 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:13:42.896 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:13:42.896 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for rw locks", 2023-09-23T21:13:42.896 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:42.558376+0000", 2023-09-23T21:13:42.896 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0032365039999999999 2023-09-23T21:13:42.896 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:13:42.896 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:13:42.896 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:13:42.896 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:13:42.897 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:13:42.897 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.46328.0:106988 10.10 10:0c356368:::benchmark_data_smithi026_379838_object27031:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e511)", 2023-09-23T21:13:42.897 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:13:42.579070+0000", 2023-09-23T21:13:42.897 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 0.27069197699999997, 2023-09-23T21:13:42.897 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.27110111799999997, 2023-09-23T21:13:42.897 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:13:42.897 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "delayed", 2023-09-23T21:13:42.897 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:13:42.897 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.46328", 2023-09-23T21:13:42.897 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/3748861181", 2023-09-23T21:13:42.897 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 106988 2023-09-23T21:13:42.897 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:13:42.897 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:13:42.898 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:13:42.898 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:13:42.898 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:42.579070+0000", 2023-09-23T21:13:42.898 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:13:42.898 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:13:42.898 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:13:42.898 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:13:42.898 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:42.579070+0000", 2023-09-23T21:13:42.898 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:13:42.898 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:13:42.898 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:13:42.898 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:13:42.899 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:42.579069+0000", 2023-09-23T21:13:42.899 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T21:13:42.899 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:13:42.899 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:13:42.899 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:13:42.899 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:42.579075+0000", 2023-09-23T21:13:42.899 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.3539999999999996e-06 2023-09-23T21:13:42.899 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:13:42.899 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:13:42.899 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:13:42.899 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:42.579092+0000", 2023-09-23T21:13:42.899 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.6229999999999999e-05 2023-09-23T21:13:42.900 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:13:42.900 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:13:42.900 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:13:42.900 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:42.579098+0000", 2023-09-23T21:13:42.900 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.5810000000000002e-06 2023-09-23T21:13:42.900 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:13:42.900 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:13:42.900 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:13:42.900 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:42.614940+0000", 2023-09-23T21:13:42.900 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.035841533000000002 2023-09-23T21:13:42.900 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:13:42.900 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:13:42.901 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for missing object", 2023-09-23T21:13:42.901 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:42.617434+0000", 2023-09-23T21:13:42.901 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.002494343 2023-09-23T21:13:42.901 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:13:42.901 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:13:42.901 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:13:42.901 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:42.646121+0000", 2023-09-23T21:13:42.901 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.028687395000000001 2023-09-23T21:13:42.901 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:13:42.901 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:13:42.901 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for rw locks", 2023-09-23T21:13:42.901 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:42.648758+0000", 2023-09-23T21:13:42.901 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0026363860000000001 2023-09-23T21:13:42.902 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:13:42.902 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:13:42.902 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:13:42.902 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:13:42.902 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:13:42.902 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.46328.0:107030 10.10 10:0af8e778:::benchmark_data_smithi026_379838_object27073:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e511)", 2023-09-23T21:13:42.902 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:13:42.729393+0000", 2023-09-23T21:13:42.902 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 0.12036891299999999, 2023-09-23T21:13:42.902 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.120823099, 2023-09-23T21:13:42.902 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:13:42.902 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "delayed", 2023-09-23T21:13:42.903 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:13:42.903 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.46328", 2023-09-23T21:13:42.903 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/3748861181", 2023-09-23T21:13:42.903 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 107030 2023-09-23T21:13:42.903 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:13:42.903 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:13:42.903 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:13:42.903 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:13:42.903 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:42.729393+0000", 2023-09-23T21:13:42.903 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:13:42.903 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:13:42.903 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:13:42.903 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:13:42.904 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:42.729393+0000", 2023-09-23T21:13:42.904 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:13:42.904 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:13:42.904 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:13:42.904 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:13:42.904 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:42.729392+0000", 2023-09-23T21:13:42.904 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.999999 2023-09-23T21:13:42.904 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:13:42.904 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:13:42.904 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:13:42.904 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:42.729397+0000", 2023-09-23T21:13:42.904 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.1719999999999998e-06 2023-09-23T21:13:42.905 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:13:42.905 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:13:42.905 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:13:42.905 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:42.729406+0000", 2023-09-23T21:13:42.905 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.6160000000000002e-06 2023-09-23T21:13:42.905 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:13:42.905 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:13:42.905 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:13:42.905 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:42.729410+0000", 2023-09-23T21:13:42.905 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.3139999999999997e-06 2023-09-23T21:13:42.905 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:13:42.905 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:13:42.905 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:13:42.906 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:42.737164+0000", 2023-09-23T21:13:42.906 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0077539590000000004 2023-09-23T21:13:42.906 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:13:42.906 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:13:42.906 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for missing object", 2023-09-23T21:13:42.906 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:42.740460+0000", 2023-09-23T21:13:42.906 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0032959310000000002 2023-09-23T21:13:42.906 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:13:42.906 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:13:42.906 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:13:42.906 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:42.791810+0000", 2023-09-23T21:13:42.906 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.051349702999999997 2023-09-23T21:13:42.907 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:13:42.907 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:13:42.907 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for rw locks", 2023-09-23T21:13:42.907 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:42.794301+0000", 2023-09-23T21:13:42.907 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0024906440000000002 2023-09-23T21:13:42.907 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:13:42.907 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:13:42.907 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:13:42.907 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:13:42.907 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:13:42.907 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.46328.0:107052 10.13 10:c888dce3:::benchmark_data_smithi026_379838_object27095:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e511)", 2023-09-23T21:13:42.907 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:13:42.779451+0000", 2023-09-23T21:13:42.908 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 0.070311531999999996, 2023-09-23T21:13:42.908 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.070818979000000004, 2023-09-23T21:13:42.908 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:13:42.908 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "delayed", 2023-09-23T21:13:42.908 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:13:42.908 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.46328", 2023-09-23T21:13:42.908 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/3748861181", 2023-09-23T21:13:42.908 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 107052 2023-09-23T21:13:42.908 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:13:42.908 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:13:42.908 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:13:42.908 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:13:42.908 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:42.779451+0000", 2023-09-23T21:13:42.909 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:13:42.909 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:13:42.909 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:13:42.909 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:13:42.909 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:42.779451+0000", 2023-09-23T21:13:42.909 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:13:42.909 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:13:42.909 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:13:42.909 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:13:42.909 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:42.779449+0000", 2023-09-23T21:13:42.909 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T21:13:42.909 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:13:42.910 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:13:42.910 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:13:42.910 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:42.779457+0000", 2023-09-23T21:13:42.910 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.9100000000000005e-06 2023-09-23T21:13:42.910 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:13:42.910 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:13:42.910 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:13:42.910 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:42.779471+0000", 2023-09-23T21:13:42.910 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.3339999999999999e-05 2023-09-23T21:13:42.910 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:13:42.910 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:13:42.910 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:13:42.911 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:42.779480+0000", 2023-09-23T21:13:42.911 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.0729999999999994e-06 2023-09-23T21:13:42.911 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:13:42.911 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:13:42.911 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:13:42.911 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:42.815514+0000", 2023-09-23T21:13:42.911 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.036033902999999999 2023-09-23T21:13:42.911 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:13:42.911 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:13:42.911 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for missing object", 2023-09-23T21:13:42.911 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:42.817961+0000", 2023-09-23T21:13:42.911 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0024475009999999999 2023-09-23T21:13:42.912 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:13:42.912 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:13:42.912 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:13:42.912 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:42.845433+0000", 2023-09-23T21:13:42.912 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.027472068999999998 2023-09-23T21:13:42.912 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:13:42.912 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:13:42.912 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for rw locks", 2023-09-23T21:13:42.912 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:42.848625+0000", 2023-09-23T21:13:42.912 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0031913530000000001 2023-09-23T21:13:42.912 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:13:42.912 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:13:42.913 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:13:42.913 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:13:42.913 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:13:42.913 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.46328.0:107033 10.0 10:051b8581:::benchmark_data_smithi026_379838_object27076:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e511)", 2023-09-23T21:13:42.913 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:13:42.734703+0000", 2023-09-23T21:13:42.913 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 0.11505983, 2023-09-23T21:13:42.913 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.115611809, 2023-09-23T21:13:42.913 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:13:42.913 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "delayed", 2023-09-23T21:13:42.913 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:13:42.913 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.46328", 2023-09-23T21:13:42.913 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/3748861181", 2023-09-23T21:13:42.913 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 107033 2023-09-23T21:13:42.914 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:13:42.914 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:13:42.914 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:13:42.914 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:13:42.914 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:42.734703+0000", 2023-09-23T21:13:42.914 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:13:42.914 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:13:42.914 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:13:42.914 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:13:42.914 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:42.734703+0000", 2023-09-23T21:13:42.914 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:13:42.914 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:13:42.914 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:13:42.915 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:13:42.915 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:42.734701+0000", 2023-09-23T21:13:42.915 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T21:13:42.915 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:13:42.915 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:13:42.915 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:13:42.915 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:42.734706+0000", 2023-09-23T21:13:42.915 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.259e-06 2023-09-23T21:13:42.915 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:13:42.915 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:13:42.915 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:13:42.915 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:42.734714+0000", 2023-09-23T21:13:42.916 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.7219999999999999e-06 2023-09-23T21:13:42.916 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:13:42.916 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:13:42.916 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:13:42.916 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:42.734718+0000", 2023-09-23T21:13:42.916 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.9389999999999999e-06 2023-09-23T21:13:42.916 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:13:42.916 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:13:42.916 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:13:42.916 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:42.747094+0000", 2023-09-23T21:13:42.916 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.012375499999999999 2023-09-23T21:13:42.916 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:13:42.917 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:13:42.917 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for missing object", 2023-09-23T21:13:42.917 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:42.751006+0000", 2023-09-23T21:13:42.917 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0039120659999999996 2023-09-23T21:13:42.917 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:13:42.917 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:13:42.917 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:13:42.917 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:42.787575+0000", 2023-09-23T21:13:42.917 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.036569496999999999 2023-09-23T21:13:42.917 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:13:42.917 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:13:42.917 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for rw locks", 2023-09-23T21:13:42.918 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:42.790628+0000", 2023-09-23T21:13:42.918 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0030532020000000001 2023-09-23T21:13:42.918 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:13:42.918 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:13:42.918 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:13:42.918 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:13:42.918 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:13:42.918 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.46328.0:106991 10.0 10:018972d7:::benchmark_data_smithi026_379838_object27034:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e511)", 2023-09-23T21:13:42.918 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:13:42.613300+0000", 2023-09-23T21:13:42.918 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 0.23646286699999999, 2023-09-23T21:13:42.918 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.23706296099999999, 2023-09-23T21:13:42.918 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:13:42.919 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "delayed", 2023-09-23T21:13:42.919 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:13:42.919 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.46328", 2023-09-23T21:13:42.919 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/3748861181", 2023-09-23T21:13:42.919 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 106991 2023-09-23T21:13:42.919 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:13:42.919 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:13:42.919 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:13:42.919 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:13:42.919 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:42.613300+0000", 2023-09-23T21:13:42.919 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:13:42.919 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:13:42.920 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:13:42.920 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:13:42.920 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:42.613300+0000", 2023-09-23T21:13:42.920 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:13:42.920 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:13:42.920 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:13:42.920 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:13:42.920 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:42.613298+0000", 2023-09-23T21:13:42.920 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T21:13:42.920 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:13:42.920 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:13:42.920 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:13:42.921 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:42.613306+0000", 2023-09-23T21:13:42.921 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.8520000000000004e-06 2023-09-23T21:13:42.921 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:13:42.921 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:13:42.921 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:13:42.921 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:42.613318+0000", 2023-09-23T21:13:42.921 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.1705000000000001e-05 2023-09-23T21:13:42.921 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:13:42.921 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:13:42.921 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:13:42.921 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:42.613324+0000", 2023-09-23T21:13:42.921 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.7920000000000004e-06 2023-09-23T21:13:42.922 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:13:42.922 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:13:42.922 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:13:42.922 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:42.624173+0000", 2023-09-23T21:13:42.922 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.010848122999999999 2023-09-23T21:13:42.922 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:13:42.922 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:13:42.922 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for missing object", 2023-09-23T21:13:42.922 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:42.627037+0000", 2023-09-23T21:13:42.922 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0028645649999999999 2023-09-23T21:13:42.922 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:13:42.922 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:13:42.923 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:13:42.923 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:42.642281+0000", 2023-09-23T21:13:42.923 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.015244057 2023-09-23T21:13:42.923 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:13:42.923 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:13:42.923 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for rw locks", 2023-09-23T21:13:42.923 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:42.645523+0000", 2023-09-23T21:13:42.923 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.003241701 2023-09-23T21:13:42.923 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:13:42.923 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:13:42.923 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:13:42.923 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:13:42.924 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:13:42.924 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.46328.0:106930 10.10 10:0fcca17d:::benchmark_data_smithi026_379838_object26973:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e511)", 2023-09-23T21:13:42.924 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:13:42.438500+0000", 2023-09-23T21:13:42.924 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 0.41126252299999999, 2023-09-23T21:13:42.924 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.41190589100000002, 2023-09-23T21:13:42.924 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:13:42.924 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "started", 2023-09-23T21:13:42.924 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:13:42.924 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.46328", 2023-09-23T21:13:42.924 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/3748861181", 2023-09-23T21:13:42.924 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 106930 2023-09-23T21:13:42.924 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:13:42.925 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:13:42.925 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:13:42.925 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:13:42.925 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:42.438500+0000", 2023-09-23T21:13:42.925 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:13:42.925 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:13:42.925 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:13:42.925 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:13:42.925 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:42.438500+0000", 2023-09-23T21:13:42.925 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:13:42.925 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:13:42.925 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:13:42.925 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:13:42.926 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:42.438498+0000", 2023-09-23T21:13:42.926 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T21:13:42.926 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:13:42.926 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:13:42.926 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:13:42.926 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:42.438504+0000", 2023-09-23T21:13:42.926 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.0909999999999998e-06 2023-09-23T21:13:42.926 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:13:42.926 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:13:42.926 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:13:42.926 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:42.438514+0000", 2023-09-23T21:13:42.926 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.6919999999999997e-06 2023-09-23T21:13:42.927 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:13:42.927 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:13:42.927 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:13:42.927 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:42.438520+0000", 2023-09-23T21:13:42.927 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.0469999999999999e-06 2023-09-23T21:13:42.927 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:13:42.927 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:13:42.927 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:13:42.927 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:42.500366+0000", 2023-09-23T21:13:42.927 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.061845913000000002 2023-09-23T21:13:42.927 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:13:42.927 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:13:42.928 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for missing object", 2023-09-23T21:13:42.928 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:42.502602+0000", 2023-09-23T21:13:42.928 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0022359749999999999 2023-09-23T21:13:42.928 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:13:42.928 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:13:42.928 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:13:42.928 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:42.521872+0000", 2023-09-23T21:13:42.928 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.019269531999999999 2023-09-23T21:13:42.928 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:13:42.928 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:13:42.928 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for rw locks", 2023-09-23T21:13:42.928 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:42.524558+0000", 2023-09-23T21:13:42.928 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0026867100000000001 2023-09-23T21:13:42.929 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:13:42.929 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:13:42.929 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:13:42.929 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:42.847492+0000", 2023-09-23T21:13:42.929 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.32293408400000001 2023-09-23T21:13:42.929 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:13:42.929 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:13:42.929 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:13:42.929 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:42.849918+0000", 2023-09-23T21:13:42.929 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0024256049999999999 2023-09-23T21:13:42.929 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:13:42.929 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:13:42.929 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:13:42.930 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:13:42.930 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:13:42.930 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.46328.0:106953 10.10 10:0bfcaf29:::benchmark_data_smithi026_379838_object26996:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e511)", 2023-09-23T21:13:42.930 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:13:42.495711+0000", 2023-09-23T21:13:42.930 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 0.35405161099999999, 2023-09-23T21:13:42.930 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.35474718399999999, 2023-09-23T21:13:42.930 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:13:42.930 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "delayed", 2023-09-23T21:13:42.930 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:13:42.930 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.46328", 2023-09-23T21:13:42.930 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/3748861181", 2023-09-23T21:13:42.930 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 106953 2023-09-23T21:13:42.931 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:13:42.931 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:13:42.931 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:13:42.931 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:13:42.931 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:42.495711+0000", 2023-09-23T21:13:42.931 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:13:42.931 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:13:42.931 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:13:42.931 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:13:42.931 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:42.495711+0000", 2023-09-23T21:13:42.931 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:13:42.932 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:13:42.932 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:13:42.932 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:13:42.932 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:42.495710+0000", 2023-09-23T21:13:42.932 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.999999 2023-09-23T21:13:42.932 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:13:42.932 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:13:42.932 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:13:42.932 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:42.495715+0000", 2023-09-23T21:13:42.932 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.7090000000000001e-06 2023-09-23T21:13:42.932 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:13:42.932 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:13:42.933 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:13:42.933 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:42.495727+0000", 2023-09-23T21:13:42.933 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.217e-05 2023-09-23T21:13:42.933 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:13:42.933 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:13:42.933 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:13:42.933 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:42.495733+0000", 2023-09-23T21:13:42.933 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.7490000000000004e-06 2023-09-23T21:13:42.933 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:13:42.933 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:13:42.933 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:13:42.933 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:42.524920+0000", 2023-09-23T21:13:42.933 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.029187682999999999 2023-09-23T21:13:42.934 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:13:42.934 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:13:42.934 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for missing object", 2023-09-23T21:13:42.934 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:42.527251+0000", 2023-09-23T21:13:42.934 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0023311809999999999 2023-09-23T21:13:42.934 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:13:42.934 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:13:42.934 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:13:42.934 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:42.541475+0000", 2023-09-23T21:13:42.934 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.014224086 2023-09-23T21:13:42.934 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:13:42.935 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:13:42.935 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for rw locks", 2023-09-23T21:13:42.935 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:42.544000+0000", 2023-09-23T21:13:42.935 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.002524732 2023-09-23T21:13:42.935 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:13:42.935 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:13:42.935 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:13:42.935 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:13:42.935 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:13:42.935 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.46328.0:106996 10.0 10:065d0050:::benchmark_data_smithi026_379838_object27039:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e511)", 2023-09-23T21:13:42.935 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:13:42.623582+0000", 2023-09-23T21:13:42.935 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 0.226179978, 2023-09-23T21:13:42.935 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.22691971799999999, 2023-09-23T21:13:42.936 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:13:42.936 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "delayed", 2023-09-23T21:13:42.936 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:13:42.936 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.46328", 2023-09-23T21:13:42.936 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/3748861181", 2023-09-23T21:13:42.936 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 106996 2023-09-23T21:13:42.936 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:13:42.936 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:13:42.936 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:13:42.936 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:13:42.936 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:42.623582+0000", 2023-09-23T21:13:42.936 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:13:42.936 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:13:42.937 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:13:42.937 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:13:42.937 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:42.623582+0000", 2023-09-23T21:13:42.937 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:13:42.937 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:13:42.937 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:13:42.937 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:13:42.937 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:42.623582+0000", 2023-09-23T21:13:42.937 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999995 2023-09-23T21:13:42.937 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:13:42.937 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:13:42.937 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:13:42.938 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:42.623585+0000", 2023-09-23T21:13:42.938 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.3160000000000001e-06 2023-09-23T21:13:42.938 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:13:42.938 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:13:42.938 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:13:42.938 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:42.623593+0000", 2023-09-23T21:13:42.938 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.8509999999999999e-06 2023-09-23T21:13:42.938 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:13:42.938 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:13:42.938 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:13:42.938 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:42.623597+0000", 2023-09-23T21:13:42.938 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.5269999999999999e-06 2023-09-23T21:13:42.938 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:13:42.939 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:13:42.939 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:13:42.939 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:42.627350+0000", 2023-09-23T21:13:42.939 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0037537809999999999 2023-09-23T21:13:42.939 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:13:42.939 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:13:42.939 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for missing object", 2023-09-23T21:13:42.939 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:42.630257+0000", 2023-09-23T21:13:42.939 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.002907111 2023-09-23T21:13:42.939 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:13:42.939 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:13:42.939 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:13:42.940 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:42.666100+0000", 2023-09-23T21:13:42.940 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.035842801000000001 2023-09-23T21:13:42.940 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:13:42.940 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:13:42.940 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for rw locks", 2023-09-23T21:13:42.940 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:42.669129+0000", 2023-09-23T21:13:42.940 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.003028354 2023-09-23T21:13:42.940 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:13:42.940 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:13:42.940 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:13:42.940 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:13:42.940 INFO:teuthology.orchestra.run.smithi070.stdout: ], 2023-09-23T21:13:42.940 INFO:teuthology.orchestra.run.smithi070.stdout: "num_ops": 17 2023-09-23T21:13:42.941 INFO:teuthology.orchestra.run.smithi070.stdout:} 2023-09-23T21:13:42.941 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:13:43.012 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:13:43.247 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:13:43.253 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:13:43.254 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:13:43.315 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_blocked_ops 2023-09-23T21:13:43.354 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:13:43.379 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:13:43 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/229682940' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-09-23T21:13:43.379 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:13:43 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-09-23T21:13:43.380 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:13:43 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T21:13:43.380 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:13:43 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/229682940' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-09-23T21:13:43.380 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:13:43 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-09-23T21:13:43.380 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:13:43 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T21:13:43.398 INFO:teuthology.orchestra.run.smithi026.stderr:reweighted osd.3 to 1 (10000) 2023-09-23T21:13:43.399 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:13:43.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:13:43 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/229682940' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-09-23T21:13:43.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:13:43 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-09-23T21:13:43.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:13:43 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T21:13:43.906 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:13:43.908 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:13:43.909 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:13:43.989 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph osd reweight 4 1 2023-09-23T21:13:44.009 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:13:44.024 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T21:13:44.063 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:13:44.284 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:13:44.287 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:13:44.288 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:13:44.388 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:13:44.430 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:13:44.474 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:13:44.478 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:13:44.478 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:13:44.579 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:13:44.607 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:13:44 smithi070 ceph-mon[168282]: pgmap v1288: 129 pgs: 1 active+recovering+undersized+remapped, 2 peering, 1 remapped+peering, 6 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 16 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 101 active+clean; 3.3 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 127 op/s; 30543/160644 objects degraded (19.013%); 4918/160644 objects misplaced (3.061%); 18 MiB/s, 287 objects/s recovering 2023-09-23T21:13:44.607 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:13:44 smithi070 ceph-mon[168282]: failed to encode map e512 with expected crc 2023-09-23T21:13:44.607 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:13:44 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-09-23T21:13:44.607 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:13:44 smithi070 ceph-mon[168282]: osdmap e512: 8 total, 8 up, 6 in 2023-09-23T21:13:44.607 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:13:44 smithi070 ceph-mon[168282]: failed to encode map e512 with expected crc 2023-09-23T21:13:44.608 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:13:44 smithi070 ceph-mon[168282]: failed to encode map e512 with expected crc 2023-09-23T21:13:44.611 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:13:44.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:13:44 smithi026 ceph-mon[179688]: pgmap v1288: 129 pgs: 1 active+recovering+undersized+remapped, 2 peering, 1 remapped+peering, 6 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 16 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 101 active+clean; 3.3 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 127 op/s; 30543/160644 objects degraded (19.013%); 4918/160644 objects misplaced (3.061%); 18 MiB/s, 287 objects/s recovering 2023-09-23T21:13:44.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:13:44 smithi026 ceph-mon[179688]: failed to encode map e512 with expected crc 2023-09-23T21:13:44.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:13:44 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-09-23T21:13:44.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:13:44 smithi026 ceph-mon[179688]: osdmap e512: 8 total, 8 up, 6 in 2023-09-23T21:13:44.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:13:44 smithi026 ceph-mon[179688]: failed to encode map e512 with expected crc 2023-09-23T21:13:44.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:13:44 smithi026 ceph-mon[179688]: failed to encode map e512 with expected crc 2023-09-23T21:13:44.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:13:44 smithi026 ceph-mon[175151]: pgmap v1288: 129 pgs: 1 active+recovering+undersized+remapped, 2 peering, 1 remapped+peering, 6 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 16 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 101 active+clean; 3.3 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 127 op/s; 30543/160644 objects degraded (19.013%); 4918/160644 objects misplaced (3.061%); 18 MiB/s, 287 objects/s recovering 2023-09-23T21:13:44.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:13:44 smithi026 ceph-mon[175151]: failed to encode map e512 with expected crc 2023-09-23T21:13:44.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:13:44 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-09-23T21:13:44.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:13:44 smithi026 ceph-mon[175151]: osdmap e512: 8 total, 8 up, 6 in 2023-09-23T21:13:44.633 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:13:44 smithi026 ceph-mon[175151]: failed to encode map e512 with expected crc 2023-09-23T21:13:44.633 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:13:44 smithi026 ceph-mon[175151]: failed to encode map e512 with expected crc 2023-09-23T21:13:44.686 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:13:44.699 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:13:44.702 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:13:44.703 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:13:44.803 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:13:44.830 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:13:45.076 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:13:45.083 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:13:45.083 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:13:45.184 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:13:45.187 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T21:13:45.216 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:13:45.310 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:13:45.313 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:13:45.313 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:13:45.414 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:13:45.415 INFO:teuthology.orchestra.run.smithi026.stderr:noscrub is unset 2023-09-23T21:13:45.462 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:13:45.606 INFO:teuthology.orchestra.run.smithi070.stdout:{ 2023-09-23T21:13:45.607 INFO:teuthology.orchestra.run.smithi070.stdout: "ops": [], 2023-09-23T21:13:45.607 INFO:teuthology.orchestra.run.smithi070.stdout: "complaint_time": 30, 2023-09-23T21:13:45.607 INFO:teuthology.orchestra.run.smithi070.stdout: "num_blocked_ops": 0 2023-09-23T21:13:45.607 INFO:teuthology.orchestra.run.smithi070.stdout:} 2023-09-23T21:13:45.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:13:45 smithi070 ceph-mon[168282]: failed to encode map e513 with expected crc 2023-09-23T21:13:45.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:13:45 smithi070 ceph-mon[168282]: failed to encode map e513 with expected crc 2023-09-23T21:13:45.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:13:45 smithi070 ceph-mon[168282]: failed to encode map e513 with expected crc 2023-09-23T21:13:45.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:13:45 smithi070 ceph-mon[168282]: osdmap e513: 8 total, 8 up, 6 in 2023-09-23T21:13:45.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:13:45 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/303205615' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T21:13:45.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:13:45 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T21:13:45.631 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:13:45 smithi026 ceph-mon[179688]: failed to encode map e513 with expected crc 2023-09-23T21:13:45.631 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:13:45 smithi026 ceph-mon[179688]: failed to encode map e513 with expected crc 2023-09-23T21:13:45.631 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:13:45 smithi026 ceph-mon[179688]: failed to encode map e513 with expected crc 2023-09-23T21:13:45.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:13:45 smithi026 ceph-mon[179688]: osdmap e513: 8 total, 8 up, 6 in 2023-09-23T21:13:45.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:13:45 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/303205615' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T21:13:45.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:13:45 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T21:13:45.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:13:45 smithi026 ceph-mon[175151]: failed to encode map e513 with expected crc 2023-09-23T21:13:45.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:13:45 smithi026 ceph-mon[175151]: failed to encode map e513 with expected crc 2023-09-23T21:13:45.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:13:45 smithi026 ceph-mon[175151]: failed to encode map e513 with expected crc 2023-09-23T21:13:45.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:13:45 smithi026 ceph-mon[175151]: osdmap e513: 8 total, 8 up, 6 in 2023-09-23T21:13:45.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:13:45 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/303205615' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T21:13:45.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:13:45 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T21:13:46.296 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_historic_ops 2023-09-23T21:13:46.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:13:46 smithi070 ceph-mon[168282]: pgmap v1291: 129 pgs: 1 active+recovering+undersized+remapped, 2 peering, 6 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 17 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 101 active+clean; 3.3 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 149 op/s; 31619/159825 objects degraded (19.784%); 4918/159825 objects misplaced (3.077%); 6.6 MiB/s, 105 objects/s recovering 2023-09-23T21:13:46.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:13:46 smithi070 ceph-mon[168282]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:13:46.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:13:46 smithi070 ceph-mon[168282]: failed to encode map e514 with expected crc 2023-09-23T21:13:46.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:13:46 smithi070 ceph-mon[168282]: failed to encode map e514 with expected crc 2023-09-23T21:13:46.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:13:46 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T21:13:46.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:13:46 smithi070 ceph-mon[168282]: osdmap e514: 8 total, 8 up, 6 in 2023-09-23T21:13:46.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:13:46 smithi070 ceph-mon[168282]: failed to encode map e514 with expected crc 2023-09-23T21:13:46.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:13:46 smithi070 ceph-mon[168282]: failed to encode map e514 with expected crc 2023-09-23T21:13:46.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:13:46 smithi070 ceph-mon[168282]: Health check update: Degraded data redundancy: 31619/159825 objects degraded (19.784%), 24 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T21:13:46.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:13:46 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:13:46.631 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:13:46 smithi026 ceph-mon[179688]: pgmap v1291: 129 pgs: 1 active+recovering+undersized+remapped, 2 peering, 6 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 17 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 101 active+clean; 3.3 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 149 op/s; 31619/159825 objects degraded (19.784%); 4918/159825 objects misplaced (3.077%); 6.6 MiB/s, 105 objects/s recovering 2023-09-23T21:13:46.631 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:13:46 smithi026 ceph-mon[179688]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:13:46.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:13:46 smithi026 ceph-mon[179688]: failed to encode map e514 with expected crc 2023-09-23T21:13:46.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:13:46 smithi026 ceph-mon[179688]: failed to encode map e514 with expected crc 2023-09-23T21:13:46.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:13:46 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T21:13:46.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:13:46 smithi026 ceph-mon[179688]: osdmap e514: 8 total, 8 up, 6 in 2023-09-23T21:13:46.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:13:46 smithi026 ceph-mon[179688]: failed to encode map e514 with expected crc 2023-09-23T21:13:46.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:13:46 smithi026 ceph-mon[179688]: failed to encode map e514 with expected crc 2023-09-23T21:13:46.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:13:46 smithi026 ceph-mon[179688]: Health check update: Degraded data redundancy: 31619/159825 objects degraded (19.784%), 24 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T21:13:46.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:13:46 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:13:46.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:13:46 smithi026 ceph-mon[175151]: pgmap v1291: 129 pgs: 1 active+recovering+undersized+remapped, 2 peering, 6 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 17 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 101 active+clean; 3.3 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 149 op/s; 31619/159825 objects degraded (19.784%); 4918/159825 objects misplaced (3.077%); 6.6 MiB/s, 105 objects/s recovering 2023-09-23T21:13:46.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:13:46 smithi026 ceph-mon[175151]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:13:46.633 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:13:46 smithi026 ceph-mon[175151]: failed to encode map e514 with expected crc 2023-09-23T21:13:46.633 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:13:46 smithi026 ceph-mon[175151]: failed to encode map e514 with expected crc 2023-09-23T21:13:46.633 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:13:46 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T21:13:46.633 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:13:46 smithi026 ceph-mon[175151]: osdmap e514: 8 total, 8 up, 6 in 2023-09-23T21:13:46.633 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:13:46 smithi026 ceph-mon[175151]: failed to encode map e514 with expected crc 2023-09-23T21:13:46.633 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:13:46 smithi026 ceph-mon[175151]: failed to encode map e514 with expected crc 2023-09-23T21:13:46.633 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:13:46 smithi026 ceph-mon[175151]: Health check update: Degraded data redundancy: 31619/159825 objects degraded (19.784%), 24 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T21:13:46.633 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:13:46 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:13:46.726 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:13:46.732 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:13:46.733 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:13:46.833 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:13:46.866 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:13:46.926 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:13:46.931 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:13:46.932 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:13:46.932 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:13:46.974 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T21:13:47.033 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:13:47.060 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:13:47.166 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:13:47.170 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:13:47.170 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:13:47.271 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:13:47.301 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:13:47.392 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T21:13:47.395 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:13:47.396 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:13:47.496 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:13:47.526 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:13:48.087 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:13:48.090 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:13:48.090 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:13:48.191 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:13:48.223 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:13:48.418 INFO:teuthology.orchestra.run.smithi026.stderr:reweighted osd.4 to 1 (10000) 2023-09-23T21:13:48.603 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0: {} 2023-09-23T21:13:48.621 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1: {} 2023-09-23T21:13:48.628 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:13:48 smithi026 ceph-mon[179688]: pgmap v1293: 129 pgs: 1 active+recovering+undersized+remapped, 2 peering, 5 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 18 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 101 active+clean; 3.2 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 171 op/s; 31410/158565 objects degraded (19.809%); 4876/158565 objects misplaced (3.075%); 5.6 MiB/s, 89 objects/s recovering 2023-09-23T21:13:48.629 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:13:48 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/2428952537' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-09-23T21:13:48.629 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:13:48 smithi026 ceph-mon[175151]: pgmap v1293: 129 pgs: 1 active+recovering+undersized+remapped, 2 peering, 5 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 18 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 101 active+clean; 3.2 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 171 op/s; 31410/158565 objects degraded (19.809%); 4876/158565 objects misplaced (3.075%); 5.6 MiB/s, 89 objects/s recovering 2023-09-23T21:13:48.629 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:13:48 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/2428952537' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-09-23T21:13:48.639 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2: {} 2023-09-23T21:13:48.661 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3: {} 2023-09-23T21:13:48.680 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4: {} 2023-09-23T21:13:48.700 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T21:13:48.722 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6: {} 2023-09-23T21:13:48.737 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T21:13:48.741 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T21:13:48.741 INFO:teuthology.orchestra.run.smithi026.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T21:13:48.742 INFO:teuthology.orchestra.run.smithi026.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T21:13:48.742 INFO:teuthology.orchestra.run.smithi026.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T21:13:48.742 INFO:teuthology.orchestra.run.smithi026.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T21:13:48.742 INFO:teuthology.orchestra.run.smithi026.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T21:13:48.742 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T21:13:48.742 INFO:teuthology.orchestra.run.smithi026.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T21:13:48.742 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T21:13:48.770 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:13:48.772 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:13:48 smithi070 ceph-mon[168282]: pgmap v1293: 129 pgs: 1 active+recovering+undersized+remapped, 2 peering, 5 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 18 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 101 active+clean; 3.2 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 171 op/s; 31410/158565 objects degraded (19.809%); 4876/158565 objects misplaced (3.075%); 5.6 MiB/s, 89 objects/s recovering 2023-09-23T21:13:48.772 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:13:48 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/2428952537' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-09-23T21:13:48.772 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:13:48.772 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:13:48.873 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:13:48.907 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:13:48.952 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:13:48.955 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:13:48.956 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:13:49.057 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:13:49.071 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph osd reweight 5 1 2023-09-23T21:13:49.091 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:13:49.132 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:13:49.138 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:13:49.138 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:13:49.239 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:13:49.274 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:13:49.489 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:13:49.493 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:13:49 smithi026 ceph-mon[175151]: failed to encode map e515 with expected crc 2023-09-23T21:13:49.494 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:13:49 smithi026 ceph-mon[175151]: failed to encode map e515 with expected crc 2023-09-23T21:13:49.494 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:13:49 smithi026 ceph-mon[175151]: failed to encode map e515 with expected crc 2023-09-23T21:13:49.494 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:13:49 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/2428952537' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-09-23T21:13:49.494 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:13:49 smithi026 ceph-mon[175151]: osdmap e515: 8 total, 8 up, 6 in 2023-09-23T21:13:49.494 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:13:49 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/224828593' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:13:49.494 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:13:49 smithi026 ceph-mon[179688]: failed to encode map e515 with expected crc 2023-09-23T21:13:49.494 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:13:49 smithi026 ceph-mon[179688]: failed to encode map e515 with expected crc 2023-09-23T21:13:49.494 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:13:49 smithi026 ceph-mon[179688]: failed to encode map e515 with expected crc 2023-09-23T21:13:49.494 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:13:49 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/2428952537' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-09-23T21:13:49.495 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:13:49 smithi026 ceph-mon[179688]: osdmap e515: 8 total, 8 up, 6 in 2023-09-23T21:13:49.495 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:13:49 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/224828593' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:13:49.495 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:13:49.496 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:13:49.597 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:13:49.630 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:13:49.707 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:13:49.718 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:13:49.724 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:13:49.724 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:13:49.825 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:13:49.853 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 21:13:49 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[164029]: ::ffff:172.21.15.70 - - [23/Sep/2023:21:13:49] "GET /metrics HTTP/1.1" 200 36399 "" "Prometheus/2.43.0" 2023-09-23T21:13:49.857 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:13:49.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:13:49 smithi070 ceph-mon[168282]: failed to encode map e515 with expected crc 2023-09-23T21:13:49.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:13:49 smithi070 ceph-mon[168282]: failed to encode map e515 with expected crc 2023-09-23T21:13:49.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:13:49 smithi070 ceph-mon[168282]: failed to encode map e515 with expected crc 2023-09-23T21:13:49.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:13:49 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/2428952537' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-09-23T21:13:49.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:13:49 smithi070 ceph-mon[168282]: osdmap e515: 8 total, 8 up, 6 in 2023-09-23T21:13:49.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:13:49 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/224828593' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:13:49.973 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:13:50.185 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:13:50.191 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:13:50.191 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:13:50.292 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:13:50.322 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:13:50.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:13:50 smithi070 ceph-mon[168282]: pgmap v1295: 129 pgs: 1 activating+undersized+remapped, 1 active+recovering+undersized+remapped, 2 peering, 5 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 17 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 101 active+clean; 3.2 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 208 op/s; 31073/157104 objects degraded (19.779%); 4211/157104 objects misplaced (2.680%); 10 MiB/s, 161 objects/s recovering 2023-09-23T21:13:50.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:13:50 smithi070 ceph-mon[168282]: failed to encode map e516 with expected crc 2023-09-23T21:13:50.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:13:50 smithi070 ceph-mon[168282]: failed to encode map e516 with expected crc 2023-09-23T21:13:50.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:13:50 smithi070 ceph-mon[168282]: failed to encode map e516 with expected crc 2023-09-23T21:13:50.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:13:50 smithi070 ceph-mon[168282]: osdmap e516: 8 total, 8 up, 6 in 2023-09-23T21:13:50.634 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:13:50 smithi026 ceph-mon[175151]: pgmap v1295: 129 pgs: 1 activating+undersized+remapped, 1 active+recovering+undersized+remapped, 2 peering, 5 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 17 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 101 active+clean; 3.2 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 208 op/s; 31073/157104 objects degraded (19.779%); 4211/157104 objects misplaced (2.680%); 10 MiB/s, 161 objects/s recovering 2023-09-23T21:13:50.634 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:13:50 smithi026 ceph-mon[175151]: failed to encode map e516 with expected crc 2023-09-23T21:13:50.635 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:13:50 smithi026 ceph-mon[175151]: failed to encode map e516 with expected crc 2023-09-23T21:13:50.635 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:13:50 smithi026 ceph-mon[175151]: failed to encode map e516 with expected crc 2023-09-23T21:13:50.635 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:13:50 smithi026 ceph-mon[175151]: osdmap e516: 8 total, 8 up, 6 in 2023-09-23T21:13:50.635 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:13:50 smithi026 ceph-mon[179688]: pgmap v1295: 129 pgs: 1 activating+undersized+remapped, 1 active+recovering+undersized+remapped, 2 peering, 5 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 17 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 101 active+clean; 3.2 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 208 op/s; 31073/157104 objects degraded (19.779%); 4211/157104 objects misplaced (2.680%); 10 MiB/s, 161 objects/s recovering 2023-09-23T21:13:50.635 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:13:50 smithi026 ceph-mon[179688]: failed to encode map e516 with expected crc 2023-09-23T21:13:50.635 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:13:50 smithi026 ceph-mon[179688]: failed to encode map e516 with expected crc 2023-09-23T21:13:50.635 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:13:50 smithi026 ceph-mon[179688]: failed to encode map e516 with expected crc 2023-09-23T21:13:50.635 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:13:50 smithi026 ceph-mon[179688]: osdmap e516: 8 total, 8 up, 6 in 2023-09-23T21:13:50.962 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:13:50.966 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:13:50.967 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:13:51.068 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:13:51.107 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:13:51.269 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:13:51.274 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:13:51.275 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:13:51.376 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:13:51.412 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:13:51.497 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T21:13:51.558 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:13:51 smithi026 ceph-mon[179688]: failed to encode map e517 with expected crc 2023-09-23T21:13:51.559 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:13:51 smithi026 ceph-mon[179688]: failed to encode map e517 with expected crc 2023-09-23T21:13:51.559 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:13:51 smithi026 ceph-mon[179688]: failed to encode map e517 with expected crc 2023-09-23T21:13:51.559 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:13:51 smithi026 ceph-mon[179688]: osdmap e517: 8 total, 8 up, 6 in 2023-09-23T21:13:51.559 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:13:51 smithi026 ceph-mon[179688]: failed to encode map e517 with expected crc 2023-09-23T21:13:51.559 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:13:51 smithi026 ceph-mon[179688]: Health check update: Degraded data redundancy: 31073/157104 objects degraded (19.779%), 23 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T21:13:51.559 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:13:51 smithi026 ceph-mon[175151]: failed to encode map e517 with expected crc 2023-09-23T21:13:51.560 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:13:51 smithi026 ceph-mon[175151]: failed to encode map e517 with expected crc 2023-09-23T21:13:51.560 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:13:51 smithi026 ceph-mon[175151]: failed to encode map e517 with expected crc 2023-09-23T21:13:51.560 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:13:51 smithi026 ceph-mon[175151]: osdmap e517: 8 total, 8 up, 6 in 2023-09-23T21:13:51.560 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:13:51 smithi026 ceph-mon[175151]: failed to encode map e517 with expected crc 2023-09-23T21:13:51.560 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:13:51 smithi026 ceph-mon[175151]: Health check update: Degraded data redundancy: 31073/157104 objects degraded (19.779%), 23 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T21:13:51.565 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:13:51.587 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:13:51.588 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:13:51.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:13:51 smithi070 ceph-mon[168282]: failed to encode map e517 with expected crc 2023-09-23T21:13:51.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:13:51 smithi070 ceph-mon[168282]: failed to encode map e517 with expected crc 2023-09-23T21:13:51.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:13:51 smithi070 ceph-mon[168282]: failed to encode map e517 with expected crc 2023-09-23T21:13:51.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:13:51 smithi070 ceph-mon[168282]: osdmap e517: 8 total, 8 up, 6 in 2023-09-23T21:13:51.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:13:51 smithi070 ceph-mon[168282]: failed to encode map e517 with expected crc 2023-09-23T21:13:51.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:13:51 smithi070 ceph-mon[168282]: Health check update: Degraded data redundancy: 31073/157104 objects degraded (19.779%), 23 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T21:13:51.689 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:13:51.716 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:13:51.831 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:13:51.836 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:13:51.836 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:13:51.937 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:13:51.971 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:13:52.114 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:13:52.120 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:13:52.121 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:13:52.221 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:13:52.263 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:13:52.416 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:13:52.426 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:13:52.427 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:13:52.433 INFO:teuthology.orchestra.run.smithi070.stdout:ERROR: (22) Invalid argument 2023-09-23T21:13:52.433 INFO:teuthology.orchestra.run.smithi070.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T21:13:52.528 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:13:52.566 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:13:52.588 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:13:52 smithi070 ceph-mon[168282]: pgmap v1298: 129 pgs: 1 activating+undersized+remapped, 1 active+recovering+undersized+remapped, 5 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 17 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 103 active+clean; 3.2 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 159 op/s; 31072/157011 objects degraded (19.790%); 4211/157011 objects misplaced (2.682%); 9.8 MiB/s, 156 objects/s recovering 2023-09-23T21:13:52.588 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:13:52 smithi070 ceph-mon[168282]: failed to encode map e518 with expected crc 2023-09-23T21:13:52.588 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:13:52 smithi070 ceph-mon[168282]: failed to encode map e518 with expected crc 2023-09-23T21:13:52.588 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:13:52 smithi070 ceph-mon[168282]: failed to encode map e518 with expected crc 2023-09-23T21:13:52.588 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:13:52 smithi070 ceph-mon[168282]: osdmap e518: 8 total, 8 up, 6 in 2023-09-23T21:13:52.588 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:13:52 smithi070 ceph-mon[168282]: failed to encode map e518 with expected crc 2023-09-23T21:13:52.588 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:13:52 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:13:52.588 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:13:52 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:13:52.716 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T21:13:52.721 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:13:52 smithi026 ceph-mon[179688]: pgmap v1298: 129 pgs: 1 activating+undersized+remapped, 1 active+recovering+undersized+remapped, 5 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 17 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 103 active+clean; 3.2 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 159 op/s; 31072/157011 objects degraded (19.790%); 4211/157011 objects misplaced (2.682%); 9.8 MiB/s, 156 objects/s recovering 2023-09-23T21:13:52.721 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:13:52 smithi026 ceph-mon[179688]: failed to encode map e518 with expected crc 2023-09-23T21:13:52.721 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:13:52 smithi026 ceph-mon[179688]: failed to encode map e518 with expected crc 2023-09-23T21:13:52.721 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:13:52 smithi026 ceph-mon[179688]: failed to encode map e518 with expected crc 2023-09-23T21:13:52.721 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:13:52 smithi026 ceph-mon[179688]: osdmap e518: 8 total, 8 up, 6 in 2023-09-23T21:13:52.721 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:13:52 smithi026 ceph-mon[179688]: failed to encode map e518 with expected crc 2023-09-23T21:13:52.721 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:13:52 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:13:52.722 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:13:52 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:13:52.722 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:13:52.723 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:13:52.723 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:13:52 smithi026 ceph-mon[175151]: pgmap v1298: 129 pgs: 1 activating+undersized+remapped, 1 active+recovering+undersized+remapped, 5 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 17 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 103 active+clean; 3.2 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 159 op/s; 31072/157011 objects degraded (19.790%); 4211/157011 objects misplaced (2.682%); 9.8 MiB/s, 156 objects/s recovering 2023-09-23T21:13:52.723 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:13:52 smithi026 ceph-mon[175151]: failed to encode map e518 with expected crc 2023-09-23T21:13:52.723 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:13:52 smithi026 ceph-mon[175151]: failed to encode map e518 with expected crc 2023-09-23T21:13:52.723 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:13:52 smithi026 ceph-mon[175151]: failed to encode map e518 with expected crc 2023-09-23T21:13:52.723 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:13:52 smithi026 ceph-mon[175151]: osdmap e518: 8 total, 8 up, 6 in 2023-09-23T21:13:52.723 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:13:52 smithi026 ceph-mon[175151]: failed to encode map e518 with expected crc 2023-09-23T21:13:52.723 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:13:52 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:13:52.724 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:13:52 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:13:52.823 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:13:52.855 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:13:52.857 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:13:52.917 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:13:52.924 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:13:52.925 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:13:52.943 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_ops_in_flight 2023-09-23T21:13:53.025 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:13:53.071 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:13:53.452 INFO:teuthology.orchestra.run.smithi026.stderr:nodeep-scrub is unset 2023-09-23T21:13:53.454 INFO:teuthology.orchestra.run.smithi026.stderr:reweighted osd.5 to 1 (10000) 2023-09-23T21:13:53.767 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T21:13:53.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:13:53 smithi070 ceph-mon[168282]: failed to encode map e519 with expected crc 2023-09-23T21:13:53.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:13:53 smithi070 ceph-mon[168282]: failed to encode map e519 with expected crc 2023-09-23T21:13:53.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:13:53 smithi070 ceph-mon[168282]: osdmap e519: 8 total, 8 up, 6 in 2023-09-23T21:13:53.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:13:53 smithi070 ceph-mon[168282]: failed to encode map e519 with expected crc 2023-09-23T21:13:53.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:13:53 smithi070 ceph-mon[168282]: failed to encode map e519 with expected crc 2023-09-23T21:13:53.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:13:53 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/723659324' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-09-23T21:13:53.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:13:53 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/3457819685' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:13:53.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:13:53 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:13:53.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:13:53 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:13:53.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:13:53 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:13:53.881 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:13:53 smithi026 ceph-mon[179688]: failed to encode map e519 with expected crc 2023-09-23T21:13:53.881 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:13:53 smithi026 ceph-mon[179688]: failed to encode map e519 with expected crc 2023-09-23T21:13:53.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:13:53 smithi026 ceph-mon[179688]: osdmap e519: 8 total, 8 up, 6 in 2023-09-23T21:13:53.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:13:53 smithi026 ceph-mon[179688]: failed to encode map e519 with expected crc 2023-09-23T21:13:53.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:13:53 smithi026 ceph-mon[179688]: failed to encode map e519 with expected crc 2023-09-23T21:13:53.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:13:53 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/723659324' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-09-23T21:13:53.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:13:53 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/3457819685' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:13:53.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:13:53 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:13:53.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:13:53 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:13:53.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:13:53 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:13:53.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:13:53 smithi026 ceph-mon[175151]: failed to encode map e519 with expected crc 2023-09-23T21:13:53.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:13:53 smithi026 ceph-mon[175151]: failed to encode map e519 with expected crc 2023-09-23T21:13:53.883 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:13:53 smithi026 ceph-mon[175151]: osdmap e519: 8 total, 8 up, 6 in 2023-09-23T21:13:53.883 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:13:53 smithi026 ceph-mon[175151]: failed to encode map e519 with expected crc 2023-09-23T21:13:53.883 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:13:53 smithi026 ceph-mon[175151]: failed to encode map e519 with expected crc 2023-09-23T21:13:53.883 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:13:53 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/723659324' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-09-23T21:13:53.883 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:13:53 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/3457819685' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:13:53.883 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:13:53 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:13:53.883 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:13:53 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:13:53.883 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:13:53 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:13:54.568 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:13:54 smithi070 ceph-mon[168282]: pgmap v1301: 129 pgs: 1 activating+undersized+remapped, 1 active+recovering+undersized+remapped, 5 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 17 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 103 active+clean; 3.2 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 57 op/s; 31072/156417 objects degraded (19.865%); 4198/156417 objects misplaced (2.684%); 16 KiB/s, 0 objects/s recovering 2023-09-23T21:13:54.568 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:13:54 smithi070 ceph-mon[168282]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T21:13:54.568 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:13:54 smithi070 ceph-mon[168282]: failed to encode map e520 with expected crc 2023-09-23T21:13:54.568 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:13:54 smithi070 ceph-mon[168282]: failed to encode map e520 with expected crc 2023-09-23T21:13:54.568 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:13:54 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/723659324' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-09-23T21:13:54.568 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:13:54 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T21:13:54.568 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:13:54 smithi070 ceph-mon[168282]: osdmap e520: 8 total, 8 up, 6 in 2023-09-23T21:13:54.569 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:13:54 smithi070 ceph-mon[168282]: failed to encode map e520 with expected crc 2023-09-23T21:13:54.569 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:13:54 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T21:13:54.569 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:13:54 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T21:13:54.569 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:13:54 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:13:54.696 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:13:54.699 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:13:54 smithi026 ceph-mon[175151]: pgmap v1301: 129 pgs: 1 activating+undersized+remapped, 1 active+recovering+undersized+remapped, 5 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 17 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 103 active+clean; 3.2 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 57 op/s; 31072/156417 objects degraded (19.865%); 4198/156417 objects misplaced (2.684%); 16 KiB/s, 0 objects/s recovering 2023-09-23T21:13:54.700 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:13:54 smithi026 ceph-mon[175151]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T21:13:54.700 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:13:54 smithi026 ceph-mon[175151]: failed to encode map e520 with expected crc 2023-09-23T21:13:54.700 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:13:54 smithi026 ceph-mon[175151]: failed to encode map e520 with expected crc 2023-09-23T21:13:54.700 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:13:54 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/723659324' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-09-23T21:13:54.700 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:13:54 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T21:13:54.700 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:13:54 smithi026 ceph-mon[175151]: osdmap e520: 8 total, 8 up, 6 in 2023-09-23T21:13:54.700 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:13:54 smithi026 ceph-mon[175151]: failed to encode map e520 with expected crc 2023-09-23T21:13:54.700 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:13:54 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T21:13:54.700 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:13:54 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T21:13:54.700 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:13:54 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:13:54.701 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:13:54 smithi026 ceph-mon[179688]: pgmap v1301: 129 pgs: 1 activating+undersized+remapped, 1 active+recovering+undersized+remapped, 5 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 17 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 103 active+clean; 3.2 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 57 op/s; 31072/156417 objects degraded (19.865%); 4198/156417 objects misplaced (2.684%); 16 KiB/s, 0 objects/s recovering 2023-09-23T21:13:54.701 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:13:54 smithi026 ceph-mon[179688]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T21:13:54.701 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:13:54 smithi026 ceph-mon[179688]: failed to encode map e520 with expected crc 2023-09-23T21:13:54.701 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:13:54 smithi026 ceph-mon[179688]: failed to encode map e520 with expected crc 2023-09-23T21:13:54.701 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:13:54 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/723659324' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-09-23T21:13:54.701 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:13:54 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T21:13:54.701 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:13:54 smithi026 ceph-mon[179688]: osdmap e520: 8 total, 8 up, 6 in 2023-09-23T21:13:54.701 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:13:54 smithi026 ceph-mon[179688]: failed to encode map e520 with expected crc 2023-09-23T21:13:54.701 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:13:54 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T21:13:54.701 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:13:54 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T21:13:54.701 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:13:54 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:13:54.702 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:13:54.702 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:13:54.803 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:13:54.842 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:13:54.927 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:13:54.931 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:13:54.931 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:13:55.032 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:13:55.071 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:13:55.183 INFO:teuthology.orchestra.run.smithi070.stdout:ERROR: (22) Invalid argument 2023-09-23T21:13:55.183 INFO:teuthology.orchestra.run.smithi070.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T21:13:55.270 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:13:55.275 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph osd reweight 7 1 2023-09-23T21:13:55.278 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:13:55.278 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:13:55.379 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:13:55.420 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:13:55.643 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:13:55.648 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:13:55.648 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:13:55.749 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:13:55.751 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_blocked_ops 2023-09-23T21:13:55.793 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:13:55.842 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:13:55.846 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:13:55.847 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:13:55.948 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:13:55.989 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:13:56.052 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:13:56.056 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:13:56.057 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:13:56.157 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:13:56.215 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:13:56.271 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T21:13:56.283 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:13:56.283 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:13:56.360 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0: {} 2023-09-23T21:13:56.384 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:13:56.385 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:13:56.393 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1: {} 2023-09-23T21:13:56.425 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:13:56.426 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2: {} 2023-09-23T21:13:56.461 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3: {} 2023-09-23T21:13:56.467 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:13:56.470 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:13:56.471 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:13:56.493 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4: {} 2023-09-23T21:13:56.507 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T21:13:56.526 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T21:13:56.558 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6: {} 2023-09-23T21:13:56.571 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:13:56.589 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T21:13:56.594 INFO:teuthology.orchestra.run.smithi026.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T21:13:56.594 INFO:teuthology.orchestra.run.smithi026.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T21:13:56.594 INFO:teuthology.orchestra.run.smithi026.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T21:13:56.595 INFO:teuthology.orchestra.run.smithi026.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T21:13:56.595 INFO:teuthology.orchestra.run.smithi026.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T21:13:56.595 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T21:13:56.595 INFO:teuthology.orchestra.run.smithi026.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T21:13:56.595 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T21:13:56.604 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:13:56.622 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:13:56 smithi026 ceph-mon[175151]: pgmap v1303: 129 pgs: 1 active+recovering+undersized+remapped, 1 peering, 5 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 17 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 103 active+clean; 3.1 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 216 op/s; 29064/154014 objects degraded (18.871%); 4621/154014 objects misplaced (3.000%); 12 MiB/s, 327 objects/s recovering 2023-09-23T21:13:56.622 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:13:56 smithi026 ceph-mon[175151]: Health check update: Degraded data redundancy: 29064/154014 objects degraded (18.871%), 23 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T21:13:56.622 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:13:56 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/2879271438' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:13:56.622 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:13:56 smithi026 ceph-mon[179688]: pgmap v1303: 129 pgs: 1 active+recovering+undersized+remapped, 1 peering, 5 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 17 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 103 active+clean; 3.1 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 216 op/s; 29064/154014 objects degraded (18.871%); 4621/154014 objects misplaced (3.000%); 12 MiB/s, 327 objects/s recovering 2023-09-23T21:13:56.623 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:13:56 smithi026 ceph-mon[179688]: Health check update: Degraded data redundancy: 29064/154014 objects degraded (18.871%), 23 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T21:13:56.623 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:13:56 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/2879271438' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:13:56.644 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:13:56.647 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:13:56.647 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:13:56.731 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:13:56 smithi070 ceph-mon[168282]: pgmap v1303: 129 pgs: 1 active+recovering+undersized+remapped, 1 peering, 5 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 17 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 103 active+clean; 3.1 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 216 op/s; 29064/154014 objects degraded (18.871%); 4621/154014 objects misplaced (3.000%); 12 MiB/s, 327 objects/s recovering 2023-09-23T21:13:56.731 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:13:56 smithi070 ceph-mon[168282]: Health check update: Degraded data redundancy: 29064/154014 objects degraded (18.871%), 23 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T21:13:56.731 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:13:56 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/2879271438' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:13:56.749 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:13:56.785 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:13:56.832 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:13:56.838 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:13:56.839 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:13:56.939 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:13:56.968 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:13:57.376 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T21:13:58.061 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:13:58.198 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:13:58.207 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:13:58.208 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:13:58.308 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:13:58.355 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:13:58.405 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T21:13:58.410 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:13:58.410 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:13:58.465 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:13:58 smithi070 ceph-mon[168282]: pgmap v1304: 129 pgs: 1 active+recovering+undersized+remapped, 1 peering, 5 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 17 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 103 active+clean; 3.1 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 288 op/s; 29064/151809 objects degraded (19.145%); 4552/151809 objects misplaced (2.999%); 9.3 MiB/s, 252 objects/s recovering 2023-09-23T21:13:58.465 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:13:58 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/1497861503' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-09-23T21:13:58.466 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:13:58 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-09-23T21:13:58.474 INFO:teuthology.orchestra.run.smithi026.stderr:reweighted osd.7 to 1 (10000) 2023-09-23T21:13:58.512 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:13:58.544 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:13:58.631 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:13:58 smithi026 ceph-mon[179688]: pgmap v1304: 129 pgs: 1 active+recovering+undersized+remapped, 1 peering, 5 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 17 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 103 active+clean; 3.1 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 288 op/s; 29064/151809 objects degraded (19.145%); 4552/151809 objects misplaced (2.999%); 9.3 MiB/s, 252 objects/s recovering 2023-09-23T21:13:58.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:13:58 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/1497861503' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-09-23T21:13:58.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:13:58 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-09-23T21:13:58.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:13:58 smithi026 ceph-mon[175151]: pgmap v1304: 129 pgs: 1 active+recovering+undersized+remapped, 1 peering, 5 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 17 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 103 active+clean; 3.1 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 288 op/s; 29064/151809 objects degraded (19.145%); 4552/151809 objects misplaced (2.999%); 9.3 MiB/s, 252 objects/s recovering 2023-09-23T21:13:58.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:13:58 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/1497861503' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-09-23T21:13:58.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:13:58 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-09-23T21:13:58.775 INFO:teuthology.orchestra.run.smithi070.stdout:{ 2023-09-23T21:13:58.775 INFO:teuthology.orchestra.run.smithi070.stdout: "ops": [], 2023-09-23T21:13:58.775 INFO:teuthology.orchestra.run.smithi070.stdout: "complaint_time": 30, 2023-09-23T21:13:58.775 INFO:teuthology.orchestra.run.smithi070.stdout: "num_blocked_ops": 0 2023-09-23T21:13:58.775 INFO:teuthology.orchestra.run.smithi070.stdout:} 2023-09-23T21:13:59.112 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T21:13:59.215 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_historic_ops 2023-09-23T21:13:59.440 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:13:59.443 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:13:59.444 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:13:59.545 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:13:59.578 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:13:59.728 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:13:59.728 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:13:59 smithi026 ceph-mon[175151]: failed to encode map e521 with expected crc 2023-09-23T21:13:59.728 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:13:59 smithi026 ceph-mon[175151]: failed to encode map e521 with expected crc 2023-09-23T21:13:59.729 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:13:59 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-09-23T21:13:59.729 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:13:59 smithi026 ceph-mon[175151]: osdmap e521: 8 total, 8 up, 6 in 2023-09-23T21:13:59.729 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:13:59 smithi026 ceph-mon[175151]: failed to encode map e521 with expected crc 2023-09-23T21:13:59.729 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 21:13:59 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[164029]: ::ffff:172.21.15.70 - - [23/Sep/2023:21:13:59] "GET /metrics HTTP/1.1" 200 36399 "" "Prometheus/2.43.0" 2023-09-23T21:13:59.731 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:13:59 smithi026 ceph-mon[179688]: failed to encode map e521 with expected crc 2023-09-23T21:13:59.732 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:13:59 smithi026 ceph-mon[179688]: failed to encode map e521 with expected crc 2023-09-23T21:13:59.732 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:13:59 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-09-23T21:13:59.732 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:13:59 smithi026 ceph-mon[179688]: osdmap e521: 8 total, 8 up, 6 in 2023-09-23T21:13:59.732 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:13:59 smithi026 ceph-mon[179688]: failed to encode map e521 with expected crc 2023-09-23T21:13:59.739 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:13:59.739 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:13:59.840 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:13:59.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:13:59 smithi070 ceph-mon[168282]: failed to encode map e521 with expected crc 2023-09-23T21:13:59.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:13:59 smithi070 ceph-mon[168282]: failed to encode map e521 with expected crc 2023-09-23T21:13:59.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:13:59 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-09-23T21:13:59.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:13:59 smithi070 ceph-mon[168282]: osdmap e521: 8 total, 8 up, 6 in 2023-09-23T21:13:59.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:13:59 smithi070 ceph-mon[168282]: failed to encode map e521 with expected crc 2023-09-23T21:13:59.882 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:13:59.931 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:13:59.936 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:13:59.936 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:13:59.942 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T21:14:00.038 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:14:00.080 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:14:00.488 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:14:00 smithi070 ceph-mon[168282]: failed to encode map e521 with expected crc 2023-09-23T21:14:00.488 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:14:00 smithi070 ceph-mon[168282]: pgmap v1306: 129 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 17 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 104 active+clean; 3.0 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 395 op/s; 28763/148584 objects degraded (19.358%); 4425/148584 objects misplaced (2.978%); 13 MiB/s, 303 objects/s recovering 2023-09-23T21:14:00.881 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:14:00 smithi026 ceph-mon[179688]: failed to encode map e521 with expected crc 2023-09-23T21:14:00.881 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:14:00 smithi026 ceph-mon[179688]: pgmap v1306: 129 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 17 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 104 active+clean; 3.0 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 395 op/s; 28763/148584 objects degraded (19.358%); 4425/148584 objects misplaced (2.978%); 13 MiB/s, 303 objects/s recovering 2023-09-23T21:14:00.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:14:00 smithi026 ceph-mon[175151]: failed to encode map e521 with expected crc 2023-09-23T21:14:00.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:14:00 smithi026 ceph-mon[175151]: pgmap v1306: 129 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 17 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 104 active+clean; 3.0 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 395 op/s; 28763/148584 objects degraded (19.358%); 4425/148584 objects misplaced (2.978%); 13 MiB/s, 303 objects/s recovering 2023-09-23T21:14:01.228 INFO:tasks.cephadm.ceph_manager.ceph:waiting for recovery to complete 2023-09-23T21:14:01.229 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T21:14:01.230 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:14:01.234 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:14:01.235 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:14:01.336 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:14:01.338 INFO:teuthology.orchestra.run.smithi070.stdout:{ 2023-09-23T21:14:01.338 INFO:teuthology.orchestra.run.smithi070.stdout: "size": 20, 2023-09-23T21:14:01.338 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 600, 2023-09-23T21:14:01.339 INFO:teuthology.orchestra.run.smithi070.stdout: "ops": [ 2023-09-23T21:14:01.339 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:14:01.339 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.35469.0:198190 8.19 8:9a4e90e2:::benchmark_data_smithi026_301269_object87433:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e332)", 2023-09-23T21:14:01.339 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:05:03.300673+0000", 2023-09-23T21:14:01.339 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 538.03521304399999, 2023-09-23T21:14:01.339 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.05033683, 2023-09-23T21:14:01.339 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:14:01.339 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:14:01.339 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:14:01.339 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.35469", 2023-09-23T21:14:01.339 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/4041424623", 2023-09-23T21:14:01.340 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 198190 2023-09-23T21:14:01.340 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:14:01.340 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:14:01.340 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:14:01.340 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:14:01.340 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:03.300673+0000", 2023-09-23T21:14:01.340 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:14:01.340 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:14:01.340 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:14:01.340 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:14:01.340 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:03.300673+0000", 2023-09-23T21:14:01.341 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:14:01.341 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:14:01.341 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:14:01.341 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:14:01.341 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:03.300671+0000", 2023-09-23T21:14:01.341 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999981 2023-09-23T21:14:01.341 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:14:01.341 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:14:01.341 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:14:01.341 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:03.300677+0000", 2023-09-23T21:14:01.341 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.964e-06 2023-09-23T21:14:01.341 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:14:01.342 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:14:01.342 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:14:01.342 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:03.300688+0000", 2023-09-23T21:14:01.342 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.117e-05 2023-09-23T21:14:01.342 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:14:01.342 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:14:01.342 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:14:01.342 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:03.300696+0000", 2023-09-23T21:14:01.342 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.294e-06 2023-09-23T21:14:01.342 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:14:01.342 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:14:01.342 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:14:01.343 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:03.300788+0000", 2023-09-23T21:14:01.343 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.2665999999999997e-05 2023-09-23T21:14:01.343 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:14:01.343 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:14:01.343 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:14:01.343 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:03.300798+0000", 2023-09-23T21:14:01.343 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.4800000000000007e-06 2023-09-23T21:14:01.343 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:14:01.343 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:14:01.343 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:14:01.343 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:04.333049+0000", 2023-09-23T21:14:01.343 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.0322515969999999 2023-09-23T21:14:01.344 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:14:01.344 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:14:01.344 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:14:01.344 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:04.333186+0000", 2023-09-23T21:14:01.344 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00013625000000000001 2023-09-23T21:14:01.344 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:14:01.344 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:14:01.344 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 0,4", 2023-09-23T21:14:01.344 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:04.333637+0000", 2023-09-23T21:14:01.344 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00045131799999999998 2023-09-23T21:14:01.344 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:14:01.344 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:14:01.345 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:14:01.345 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:04.342090+0000", 2023-09-23T21:14:01.345 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0084528899999999994 2023-09-23T21:14:01.345 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:14:01.345 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:14:01.345 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:14:01.345 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:04.342101+0000", 2023-09-23T21:14:01.345 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.1275e-05 2023-09-23T21:14:01.345 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:14:01.345 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:14:01.345 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:14:01.345 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:04.350919+0000", 2023-09-23T21:14:01.346 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0088174489999999998 2023-09-23T21:14:01.346 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:14:01.346 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:14:01.346 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:14:01.346 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:04.350968+0000", 2023-09-23T21:14:01.346 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.9651000000000003e-05 2023-09-23T21:14:01.346 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:14:01.346 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:14:01.346 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:14:01.346 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:04.351010+0000", 2023-09-23T21:14:01.346 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.1616000000000003e-05 2023-09-23T21:14:01.346 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:14:01.347 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:14:01.347 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:14:01.347 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:14:01.347 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:14:01.347 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.35469.0:198232 8.19 8:9c7931dd:::benchmark_data_smithi026_301269_object87475:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e332)", 2023-09-23T21:14:01.347 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:05:03.301139+0000", 2023-09-23T21:14:01.347 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 538.03474700699996, 2023-09-23T21:14:01.347 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.055824807, 2023-09-23T21:14:01.347 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:14:01.347 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:14:01.347 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:14:01.347 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.35469", 2023-09-23T21:14:01.348 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/4041424623", 2023-09-23T21:14:01.348 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 198232 2023-09-23T21:14:01.348 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:14:01.348 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:14:01.348 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:14:01.348 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:14:01.348 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:03.301139+0000", 2023-09-23T21:14:01.348 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:14:01.348 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:14:01.348 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:14:01.348 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:14:01.348 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:03.301139+0000", 2023-09-23T21:14:01.349 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:14:01.349 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:14:01.349 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:14:01.349 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:14:01.349 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:03.301139+0000", 2023-09-23T21:14:01.349 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999995 2023-09-23T21:14:01.349 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:14:01.349 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:14:01.349 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:14:01.349 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:03.301141+0000", 2023-09-23T21:14:01.349 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.1739999999999999e-06 2023-09-23T21:14:01.349 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:14:01.350 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:14:01.350 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:14:01.350 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:03.301147+0000", 2023-09-23T21:14:01.350 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.4740000000000002e-06 2023-09-23T21:14:01.350 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:14:01.350 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:14:01.350 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:14:01.350 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:03.301151+0000", 2023-09-23T21:14:01.350 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.0500000000000002e-06 2023-09-23T21:14:01.350 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:14:01.350 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:14:01.350 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:14:01.350 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:03.301245+0000", 2023-09-23T21:14:01.351 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.3345999999999997e-05 2023-09-23T21:14:01.351 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:14:01.351 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:14:01.351 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:14:01.351 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:03.301250+0000", 2023-09-23T21:14:01.351 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.2920000000000003e-06 2023-09-23T21:14:01.351 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:14:01.351 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:14:01.351 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:14:01.351 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:04.342298+0000", 2023-09-23T21:14:01.351 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.0410477309999999 2023-09-23T21:14:01.351 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:14:01.353 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:14:01.353 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:14:01.353 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:04.342402+0000", 2023-09-23T21:14:01.353 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000104706 2023-09-23T21:14:01.353 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:14:01.353 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:14:01.354 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 0,4", 2023-09-23T21:14:01.354 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:04.342653+0000", 2023-09-23T21:14:01.354 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00025070499999999999 2023-09-23T21:14:01.354 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:14:01.354 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:14:01.354 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:14:01.354 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:04.350875+0000", 2023-09-23T21:14:01.354 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0082216129999999991 2023-09-23T21:14:01.354 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:14:01.354 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:14:01.354 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:14:01.354 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:04.351422+0000", 2023-09-23T21:14:01.355 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00054713199999999998 2023-09-23T21:14:01.355 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:14:01.355 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:14:01.355 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:14:01.355 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:04.356875+0000", 2023-09-23T21:14:01.355 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.005452854 2023-09-23T21:14:01.355 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:14:01.355 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:14:01.355 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:14:01.355 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:04.356917+0000", 2023-09-23T21:14:01.355 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.2585e-05 2023-09-23T21:14:01.355 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:14:01.355 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:14:01.356 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:14:01.356 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:04.356964+0000", 2023-09-23T21:14:01.356 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.6529000000000001e-05 2023-09-23T21:14:01.356 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:14:01.356 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:14:01.356 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:14:01.356 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:14:01.356 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:14:01.356 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.35469.0:198310 8.19 8:9d32f5d8:::benchmark_data_smithi026_301269_object87553:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e332)", 2023-09-23T21:14:01.356 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:05:03.301156+0000", 2023-09-23T21:14:01.356 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 538.03472974800002, 2023-09-23T21:14:01.356 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.055943659, 2023-09-23T21:14:01.357 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:14:01.357 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:14:01.357 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:14:01.357 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.35469", 2023-09-23T21:14:01.357 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/4041424623", 2023-09-23T21:14:01.357 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 198310 2023-09-23T21:14:01.357 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:14:01.357 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:14:01.357 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:14:01.357 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:14:01.357 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:03.301156+0000", 2023-09-23T21:14:01.357 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:14:01.358 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:14:01.358 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:14:01.358 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:14:01.358 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:03.301156+0000", 2023-09-23T21:14:01.358 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:14:01.358 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:14:01.358 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:14:01.358 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:14:01.358 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:03.301156+0000", 2023-09-23T21:14:01.358 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999995 2023-09-23T21:14:01.358 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:14:01.358 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:14:01.359 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:14:01.359 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:03.301158+0000", 2023-09-23T21:14:01.359 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.114e-06 2023-09-23T21:14:01.359 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:14:01.359 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:14:01.359 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:14:01.359 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:03.301165+0000", 2023-09-23T21:14:01.359 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.1770000000000001e-06 2023-09-23T21:14:01.359 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:14:01.359 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:14:01.359 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:14:01.359 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:03.301170+0000", 2023-09-23T21:14:01.360 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.6530000000000002e-06 2023-09-23T21:14:01.360 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:14:01.360 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:14:01.360 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:14:01.360 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:03.301277+0000", 2023-09-23T21:14:01.360 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000106681 2023-09-23T21:14:01.360 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:14:01.360 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:14:01.360 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:14:01.360 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:03.301285+0000", 2023-09-23T21:14:01.360 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.0029999999999994e-06 2023-09-23T21:14:01.360 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:14:01.360 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:14:01.361 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:14:01.361 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:04.344519+0000", 2023-09-23T21:14:01.361 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.04323455 2023-09-23T21:14:01.361 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:14:01.361 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:14:01.361 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:14:01.361 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:04.344610+0000", 2023-09-23T21:14:01.361 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.0427999999999995e-05 2023-09-23T21:14:01.361 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:14:01.361 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:14:01.361 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 0,4", 2023-09-23T21:14:01.361 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:04.344853+0000", 2023-09-23T21:14:01.362 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00024303199999999999 2023-09-23T21:14:01.362 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:14:01.362 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:14:01.362 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:14:01.362 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:04.351020+0000", 2023-09-23T21:14:01.362 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0061668469999999996 2023-09-23T21:14:01.362 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:14:01.362 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:14:01.362 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:14:01.362 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:04.351454+0000", 2023-09-23T21:14:01.362 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00043414499999999998 2023-09-23T21:14:01.362 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:14:01.362 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:14:01.363 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:14:01.363 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:04.357055+0000", 2023-09-23T21:14:01.363 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0056014849999999998 2023-09-23T21:14:01.363 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:14:01.363 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:14:01.363 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:14:01.363 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:04.357080+0000", 2023-09-23T21:14:01.363 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.5134999999999999e-05 2023-09-23T21:14:01.363 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:14:01.363 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:14:01.363 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:14:01.363 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:04.357100+0000", 2023-09-23T21:14:01.364 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.9724e-05 2023-09-23T21:14:01.364 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:14:01.364 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:14:01.364 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:14:01.364 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:14:01.364 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:14:01.364 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.35469.0:198331 8.19 8:9d0af17f:::benchmark_data_smithi026_301269_object87574:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e332)", 2023-09-23T21:14:01.364 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:05:03.301175+0000", 2023-09-23T21:14:01.364 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 538.03471073499998, 2023-09-23T21:14:01.364 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.0560283180000001, 2023-09-23T21:14:01.364 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:14:01.364 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:14:01.364 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:14:01.365 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.35469", 2023-09-23T21:14:01.365 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/4041424623", 2023-09-23T21:14:01.365 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 198331 2023-09-23T21:14:01.365 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:14:01.365 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:14:01.365 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:14:01.365 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:14:01.365 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:03.301175+0000", 2023-09-23T21:14:01.365 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:14:01.365 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:14:01.365 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:14:01.365 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:14:01.366 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:03.301175+0000", 2023-09-23T21:14:01.366 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:14:01.366 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:14:01.366 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:14:01.366 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:14:01.366 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:03.301175+0000", 2023-09-23T21:14:01.366 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999995 2023-09-23T21:14:01.366 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:14:01.366 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:14:01.366 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:14:01.366 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:03.301177+0000", 2023-09-23T21:14:01.366 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.148e-06 2023-09-23T21:14:01.367 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:14:01.367 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:14:01.367 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:14:01.367 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:03.301183+0000", 2023-09-23T21:14:01.367 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.9429999999999997e-06 2023-09-23T21:14:01.367 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:14:01.367 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:14:01.367 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:14:01.367 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:03.301187+0000", 2023-09-23T21:14:01.367 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.5279999999999999e-06 2023-09-23T21:14:01.367 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:14:01.367 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:14:01.367 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:14:01.368 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:03.301315+0000", 2023-09-23T21:14:01.368 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000128264 2023-09-23T21:14:01.368 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:14:01.368 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:14:01.368 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:14:01.368 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:03.301321+0000", 2023-09-23T21:14:01.368 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.852e-06 2023-09-23T21:14:01.368 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:14:01.368 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:14:01.368 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:14:01.368 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:04.346735+0000", 2023-09-23T21:14:01.368 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.0454144299999999 2023-09-23T21:14:01.369 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:14:01.369 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:14:01.369 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:14:01.369 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:04.346844+0000", 2023-09-23T21:14:01.369 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00010883 2023-09-23T21:14:01.369 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:14:01.369 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:14:01.369 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 0,4", 2023-09-23T21:14:01.369 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:04.347122+0000", 2023-09-23T21:14:01.369 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000278297 2023-09-23T21:14:01.369 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:14:01.369 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:14:01.369 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:14:01.370 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:04.351024+0000", 2023-09-23T21:14:01.370 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0039017230000000002 2023-09-23T21:14:01.370 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:14:01.370 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:14:01.370 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:14:01.370 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:04.356836+0000", 2023-09-23T21:14:01.370 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.005812004 2023-09-23T21:14:01.370 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:14:01.370 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:14:01.370 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:14:01.370 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:04.357160+0000", 2023-09-23T21:14:01.370 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00032355099999999999 2023-09-23T21:14:01.371 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:14:01.371 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:14:01.371 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:14:01.371 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:04.357184+0000", 2023-09-23T21:14:01.371 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.3943000000000001e-05 2023-09-23T21:14:01.371 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:14:01.371 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:14:01.371 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:14:01.371 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:04.357204+0000", 2023-09-23T21:14:01.371 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.0129999999999999e-05 2023-09-23T21:14:01.371 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:14:01.371 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:14:01.372 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:14:01.372 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:14:01.372 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:14:01.372 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.35469.0:198336 8.19 8:9cb16749:::benchmark_data_smithi026_301269_object87579:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e332)", 2023-09-23T21:14:01.372 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:05:03.301263+0000", 2023-09-23T21:14:01.372 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 538.03462261499999, 2023-09-23T21:14:01.372 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.056033843, 2023-09-23T21:14:01.372 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:14:01.372 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:14:01.372 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:14:01.372 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.35469", 2023-09-23T21:14:01.372 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/4041424623", 2023-09-23T21:14:01.373 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 198336 2023-09-23T21:14:01.373 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:14:01.373 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:14:01.373 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:14:01.373 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:14:01.373 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:03.301263+0000", 2023-09-23T21:14:01.373 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:14:01.373 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:14:01.373 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:14:01.373 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:14:01.373 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:03.301263+0000", 2023-09-23T21:14:01.373 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:14:01.373 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:14:01.374 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:14:01.374 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:14:01.374 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:03.301263+0000", 2023-09-23T21:14:01.374 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999995 2023-09-23T21:14:01.374 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:14:01.374 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:14:01.374 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:14:01.374 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:03.301267+0000", 2023-09-23T21:14:01.374 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.6229999999999999e-06 2023-09-23T21:14:01.374 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:14:01.374 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:14:01.374 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:14:01.374 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:03.301274+0000", 2023-09-23T21:14:01.375 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.6559999999999992e-06 2023-09-23T21:14:01.375 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:14:01.375 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:14:01.375 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:14:01.375 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:03.301280+0000", 2023-09-23T21:14:01.375 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.003e-06 2023-09-23T21:14:01.375 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:14:01.375 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:14:01.375 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:14:01.375 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:03.301344+0000", 2023-09-23T21:14:01.375 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.4073999999999997e-05 2023-09-23T21:14:01.375 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:14:01.376 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:14:01.376 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:14:01.376 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:03.301350+0000", 2023-09-23T21:14:01.376 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.8869999999999997e-06 2023-09-23T21:14:01.376 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:14:01.376 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:14:01.376 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:14:01.376 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:04.349006+0000", 2023-09-23T21:14:01.376 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.047656132 2023-09-23T21:14:01.376 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:14:01.376 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:14:01.376 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:14:01.376 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:04.349116+0000", 2023-09-23T21:14:01.377 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000109882 2023-09-23T21:14:01.377 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:14:01.377 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:14:01.377 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 0,4", 2023-09-23T21:14:01.377 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:04.349418+0000", 2023-09-23T21:14:01.377 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00030181400000000002 2023-09-23T21:14:01.377 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:14:01.377 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:14:01.377 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:14:01.377 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:04.351050+0000", 2023-09-23T21:14:01.377 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.001631785 2023-09-23T21:14:01.377 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:14:01.377 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:14:01.378 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:14:01.378 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:04.357031+0000", 2023-09-23T21:14:01.378 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0059814009999999999 2023-09-23T21:14:01.378 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:14:01.378 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:14:01.378 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:14:01.378 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:04.357254+0000", 2023-09-23T21:14:01.378 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000223293 2023-09-23T21:14:01.378 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:14:01.378 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:14:01.378 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:14:01.378 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:04.357278+0000", 2023-09-23T21:14:01.378 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.3859e-05 2023-09-23T21:14:01.379 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:14:01.379 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:14:01.379 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:14:01.379 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:04.357297+0000", 2023-09-23T21:14:01.379 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.9029e-05 2023-09-23T21:14:01.379 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:14:01.379 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:14:01.379 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:14:01.379 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:14:01.379 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:14:01.379 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.35469.0:198386 8.19 8:9890a619:::benchmark_data_smithi026_301269_object87629:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e332)", 2023-09-23T21:14:01.379 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:05:03.301286+0000", 2023-09-23T21:14:01.380 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 538.03459977399996, 2023-09-23T21:14:01.380 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.0560852510000001, 2023-09-23T21:14:01.380 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:14:01.380 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:14:01.380 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:14:01.380 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.35469", 2023-09-23T21:14:01.380 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/4041424623", 2023-09-23T21:14:01.380 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 198386 2023-09-23T21:14:01.380 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:14:01.380 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:14:01.380 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:14:01.380 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:14:01.380 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:03.301286+0000", 2023-09-23T21:14:01.381 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:14:01.381 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:14:01.381 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:14:01.381 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:14:01.381 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:03.301286+0000", 2023-09-23T21:14:01.381 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:14:01.381 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:14:01.381 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:14:01.381 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:14:01.381 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:03.301286+0000", 2023-09-23T21:14:01.381 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999995 2023-09-23T21:14:01.381 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:14:01.381 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:14:01.382 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:14:01.382 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:03.301288+0000", 2023-09-23T21:14:01.382 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.4449999999999999e-06 2023-09-23T21:14:01.382 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:14:01.382 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:14:01.382 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:14:01.382 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:03.301294+0000", 2023-09-23T21:14:01.382 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.9710000000000001e-06 2023-09-23T21:14:01.382 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:14:01.382 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:14:01.382 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:14:01.382 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:03.301298+0000", 2023-09-23T21:14:01.382 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.8600000000000003e-06 2023-09-23T21:14:01.383 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:14:01.383 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:14:01.383 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:14:01.383 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:03.301366+0000", 2023-09-23T21:14:01.383 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.7865000000000002e-05 2023-09-23T21:14:01.383 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:14:01.383 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:14:01.383 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:14:01.383 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:03.301371+0000", 2023-09-23T21:14:01.383 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.6530000000000002e-06 2023-09-23T21:14:01.383 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:14:01.383 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:14:01.383 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:14:01.384 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:04.351477+0000", 2023-09-23T21:14:01.384 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.0501064470000001 2023-09-23T21:14:01.384 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:14:01.384 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:14:01.384 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:14:01.384 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:04.351588+0000", 2023-09-23T21:14:01.384 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000110401 2023-09-23T21:14:01.384 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:14:01.384 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:14:01.384 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 0,4", 2023-09-23T21:14:01.384 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:04.351871+0000", 2023-09-23T21:14:01.384 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00028389700000000002 2023-09-23T21:14:01.384 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:14:01.385 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:14:01.385 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:14:01.385 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:04.356979+0000", 2023-09-23T21:14:01.385 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0051072510000000002 2023-09-23T21:14:01.385 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:14:01.385 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:14:01.385 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:14:01.385 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:04.357138+0000", 2023-09-23T21:14:01.385 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00015906399999999999 2023-09-23T21:14:01.385 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:14:01.385 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:14:01.385 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:14:01.385 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:04.357323+0000", 2023-09-23T21:14:01.386 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00018522100000000001 2023-09-23T21:14:01.386 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:14:01.386 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:14:01.386 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:14:01.386 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:04.357348+0000", 2023-09-23T21:14:01.386 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.5401999999999998e-05 2023-09-23T21:14:01.386 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:14:01.386 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:14:01.386 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:14:01.386 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:04.357372+0000", 2023-09-23T21:14:01.386 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.3170999999999999e-05 2023-09-23T21:14:01.386 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:14:01.387 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:14:01.387 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:14:01.387 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:14:01.387 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:14:01.387 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.35469.0:198473 8.19 8:9dfe5eb1:::benchmark_data_smithi026_301269_object87716:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e332)", 2023-09-23T21:14:01.387 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:05:03.301346+0000", 2023-09-23T21:14:01.387 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 538.03454009999996, 2023-09-23T21:14:01.387 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.0560949470000001, 2023-09-23T21:14:01.387 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:14:01.387 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:14:01.387 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:14:01.387 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.35469", 2023-09-23T21:14:01.388 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/4041424623", 2023-09-23T21:14:01.388 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 198473 2023-09-23T21:14:01.388 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:14:01.388 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:14:01.388 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:14:01.388 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:14:01.388 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:03.301346+0000", 2023-09-23T21:14:01.388 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:14:01.388 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:14:01.388 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:14:01.388 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:14:01.388 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:03.301346+0000", 2023-09-23T21:14:01.388 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:14:01.389 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:14:01.389 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:14:01.389 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:14:01.389 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:03.301345+0000", 2023-09-23T21:14:01.389 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999995 2023-09-23T21:14:01.389 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:14:01.389 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:14:01.389 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:14:01.389 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:03.301349+0000", 2023-09-23T21:14:01.389 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.8800000000000001e-06 2023-09-23T21:14:01.389 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:14:01.389 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:14:01.389 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:14:01.390 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:03.301358+0000", 2023-09-23T21:14:01.390 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.3930000000000008e-06 2023-09-23T21:14:01.390 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:14:01.390 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:14:01.390 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:14:01.390 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:03.301364+0000", 2023-09-23T21:14:01.390 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.2530000000000001e-06 2023-09-23T21:14:01.390 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:14:01.390 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:14:01.390 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:14:01.390 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:03.301384+0000", 2023-09-23T21:14:01.390 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.9896000000000001e-05 2023-09-23T21:14:01.391 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:14:01.391 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:14:01.391 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:14:01.391 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:03.301386+0000", 2023-09-23T21:14:01.391 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.6879999999999999e-06 2023-09-23T21:14:01.391 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:14:01.391 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:14:01.391 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:14:01.391 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:04.354056+0000", 2023-09-23T21:14:01.391 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.0526691889999999 2023-09-23T21:14:01.391 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:14:01.391 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:14:01.391 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:14:01.392 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:04.354187+0000", 2023-09-23T21:14:01.392 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000131192 2023-09-23T21:14:01.392 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:14:01.392 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:14:01.392 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 0,4", 2023-09-23T21:14:01.392 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:04.354563+0000", 2023-09-23T21:14:01.392 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000375798 2023-09-23T21:14:01.392 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:14:01.392 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:14:01.392 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:14:01.392 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:04.357109+0000", 2023-09-23T21:14:01.392 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0025466719999999998 2023-09-23T21:14:01.392 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:14:01.393 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:14:01.393 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:14:01.393 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:04.357230+0000", 2023-09-23T21:14:01.393 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000120228 2023-09-23T21:14:01.393 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:14:01.393 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:14:01.393 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:14:01.393 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:04.357401+0000", 2023-09-23T21:14:01.393 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000171146 2023-09-23T21:14:01.393 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:14:01.393 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:14:01.393 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:14:01.394 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:04.357423+0000", 2023-09-23T21:14:01.394 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.1885000000000001e-05 2023-09-23T21:14:01.394 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:14:01.394 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:14:01.394 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:14:01.394 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:04.357441+0000", 2023-09-23T21:14:01.394 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.8423e-05 2023-09-23T21:14:01.394 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:14:01.394 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:14:01.394 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:14:01.394 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:14:01.394 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:14:01.394 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.35469.0:198478 8.19 8:9d4307fd:::benchmark_data_smithi026_301269_object87721:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e332)", 2023-09-23T21:14:01.395 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:05:03.301369+0000", 2023-09-23T21:14:01.395 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 538.03451721399995, 2023-09-23T21:14:01.395 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.066125588, 2023-09-23T21:14:01.395 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:14:01.395 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:14:01.395 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:14:01.395 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.35469", 2023-09-23T21:14:01.395 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/4041424623", 2023-09-23T21:14:01.395 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 198478 2023-09-23T21:14:01.395 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:14:01.395 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:14:01.395 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:14:01.395 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:14:01.396 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:03.301369+0000", 2023-09-23T21:14:01.396 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:14:01.396 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:14:01.396 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:14:01.396 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:14:01.396 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:03.301369+0000", 2023-09-23T21:14:01.396 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:14:01.396 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:14:01.396 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:14:01.396 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:14:01.396 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:03.301368+0000", 2023-09-23T21:14:01.396 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999995 2023-09-23T21:14:01.396 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:14:01.397 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:14:01.397 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:14:01.397 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:03.301371+0000", 2023-09-23T21:14:01.397 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.548e-06 2023-09-23T21:14:01.397 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:14:01.397 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:14:01.397 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:14:01.397 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:03.301378+0000", 2023-09-23T21:14:01.397 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.2450000000000001e-06 2023-09-23T21:14:01.397 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:14:01.397 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:14:01.397 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:14:01.397 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:03.301383+0000", 2023-09-23T21:14:01.398 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.2350000000000001e-06 2023-09-23T21:14:01.398 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:14:01.398 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:14:01.398 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:14:01.398 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:03.301396+0000", 2023-09-23T21:14:01.398 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.3183999999999999e-05 2023-09-23T21:14:01.398 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:14:01.398 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:14:01.398 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:14:01.398 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:03.301398+0000", 2023-09-23T21:14:01.398 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.745e-06 2023-09-23T21:14:01.398 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:14:01.399 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:14:01.399 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:14:01.399 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:04.357463+0000", 2023-09-23T21:14:01.399 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.0560645749999999 2023-09-23T21:14:01.399 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:14:01.399 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:14:01.399 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:14:01.399 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:04.357537+0000", 2023-09-23T21:14:01.399 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.4395999999999997e-05 2023-09-23T21:14:01.399 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:14:01.399 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:14:01.399 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 0,4", 2023-09-23T21:14:01.399 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:04.357798+0000", 2023-09-23T21:14:01.400 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00026083399999999999 2023-09-23T21:14:01.400 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:14:01.400 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:14:01.400 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:14:01.400 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:04.367326+0000", 2023-09-23T21:14:01.400 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0095279379999999997 2023-09-23T21:14:01.400 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:14:01.400 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:14:01.400 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:14:01.400 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:04.367342+0000", 2023-09-23T21:14:01.400 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.5982999999999999e-05 2023-09-23T21:14:01.400 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:14:01.400 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:14:01.401 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:14:01.401 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:04.367396+0000", 2023-09-23T21:14:01.401 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.4110999999999997e-05 2023-09-23T21:14:01.401 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:14:01.401 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:14:01.401 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:14:01.401 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:04.367446+0000", 2023-09-23T21:14:01.401 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.9784999999999999e-05 2023-09-23T21:14:01.401 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:14:01.401 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:14:01.401 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:14:01.401 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:04.367494+0000", 2023-09-23T21:14:01.401 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.8433000000000002e-05 2023-09-23T21:14:01.402 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:14:01.402 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:14:01.402 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:14:01.402 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:14:01.402 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:14:01.402 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.35469.0:198487 8.19 8:9ba99d26:::benchmark_data_smithi026_301269_object87730:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e332)", 2023-09-23T21:14:01.402 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:05:03.301387+0000", 2023-09-23T21:14:01.402 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 538.03449881899996, 2023-09-23T21:14:01.402 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.1069283030000001, 2023-09-23T21:14:01.402 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:14:01.402 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:14:01.402 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:14:01.403 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.35469", 2023-09-23T21:14:01.403 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/4041424623", 2023-09-23T21:14:01.403 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 198487 2023-09-23T21:14:01.403 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:14:01.403 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:14:01.403 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:14:01.403 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:14:01.403 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:03.301387+0000", 2023-09-23T21:14:01.403 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:14:01.403 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:14:01.403 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:14:01.403 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:14:01.403 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:03.301387+0000", 2023-09-23T21:14:01.404 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:14:01.404 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:14:01.404 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:14:01.404 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:14:01.404 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:03.301387+0000", 2023-09-23T21:14:01.404 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999995 2023-09-23T21:14:01.404 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:14:01.404 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:14:01.404 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:14:01.404 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:03.301393+0000", 2023-09-23T21:14:01.404 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.1210000000000003e-06 2023-09-23T21:14:01.404 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:14:01.405 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:14:01.405 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:14:01.405 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:03.301400+0000", 2023-09-23T21:14:01.405 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.7469999999999999e-06 2023-09-23T21:14:01.405 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:14:01.405 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:14:01.405 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:14:01.405 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:03.301404+0000", 2023-09-23T21:14:01.405 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.4020000000000004e-06 2023-09-23T21:14:01.405 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:14:01.405 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:14:01.405 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:14:01.405 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:03.301482+0000", 2023-09-23T21:14:01.406 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.8005999999999998e-05 2023-09-23T21:14:01.406 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:14:01.406 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:14:01.406 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:14:01.406 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:03.301488+0000", 2023-09-23T21:14:01.406 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.9789999999999998e-06 2023-09-23T21:14:01.406 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:14:01.406 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:14:01.406 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:14:01.406 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:04.359818+0000", 2023-09-23T21:14:01.406 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.058330124 2023-09-23T21:14:01.406 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:14:01.407 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:14:01.407 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:14:01.407 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:04.359934+0000", 2023-09-23T21:14:01.407 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00011566299999999999 2023-09-23T21:14:01.407 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:14:01.407 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:14:01.407 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 0,4", 2023-09-23T21:14:01.407 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:04.360253+0000", 2023-09-23T21:14:01.407 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00031915200000000001 2023-09-23T21:14:01.407 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:14:01.407 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:14:01.407 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:14:01.407 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:04.367349+0000", 2023-09-23T21:14:01.408 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0070960470000000003 2023-09-23T21:14:01.408 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:14:01.408 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:14:01.408 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:14:01.408 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:04.367551+0000", 2023-09-23T21:14:01.408 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000201745 2023-09-23T21:14:01.408 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:14:01.408 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:14:01.408 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:14:01.408 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:04.408261+0000", 2023-09-23T21:14:01.408 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.040710206999999998 2023-09-23T21:14:01.408 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:14:01.408 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:14:01.409 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:14:01.409 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:04.408293+0000", 2023-09-23T21:14:01.409 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.1405000000000002e-05 2023-09-23T21:14:01.409 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:14:01.409 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:14:01.409 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:14:01.409 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:04.408316+0000", 2023-09-23T21:14:01.409 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.3054e-05 2023-09-23T21:14:01.409 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:14:01.409 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:14:01.409 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:14:01.409 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:14:01.409 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:14:01.410 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.35469.0:198494 8.19 8:9dcb4a10:::benchmark_data_smithi026_301269_object87737:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e332)", 2023-09-23T21:14:01.410 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:05:03.301427+0000", 2023-09-23T21:14:01.410 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 538.03445944999999, 2023-09-23T21:14:01.410 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.106992956, 2023-09-23T21:14:01.410 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:14:01.410 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:14:01.410 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:14:01.410 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.35469", 2023-09-23T21:14:01.410 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/4041424623", 2023-09-23T21:14:01.410 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 198494 2023-09-23T21:14:01.410 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:14:01.410 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:14:01.411 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:14:01.411 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:14:01.411 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:03.301427+0000", 2023-09-23T21:14:01.411 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:14:01.411 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:14:01.411 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:14:01.411 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:14:01.411 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:03.301427+0000", 2023-09-23T21:14:01.411 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:14:01.411 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:14:01.411 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:14:01.411 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:14:01.411 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:03.301426+0000", 2023-09-23T21:14:01.412 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.999999 2023-09-23T21:14:01.412 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:14:01.412 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:14:01.412 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:14:01.412 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:03.301430+0000", 2023-09-23T21:14:01.412 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.7910000000000002e-06 2023-09-23T21:14:01.412 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:14:01.412 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:14:01.412 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:14:01.412 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:03.301438+0000", 2023-09-23T21:14:01.412 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.4360000000000002e-06 2023-09-23T21:14:01.412 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:14:01.412 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:14:01.413 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:14:01.413 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:03.301443+0000", 2023-09-23T21:14:01.413 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.9880000000000004e-06 2023-09-23T21:14:01.413 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:14:01.413 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:14:01.413 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:14:01.413 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:03.301507+0000", 2023-09-23T21:14:01.413 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.3439e-05 2023-09-23T21:14:01.413 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:14:01.413 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:14:01.413 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:14:01.413 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:03.301511+0000", 2023-09-23T21:14:01.413 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.6500000000000004e-06 2023-09-23T21:14:01.414 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:14:01.414 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:14:01.414 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:14:01.414 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:04.362136+0000", 2023-09-23T21:14:01.414 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.060624912 2023-09-23T21:14:01.414 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:14:01.414 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:14:01.414 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:14:01.414 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:04.362265+0000", 2023-09-23T21:14:01.414 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00012852800000000001 2023-09-23T21:14:01.414 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:14:01.414 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:14:01.415 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 0,4", 2023-09-23T21:14:01.415 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:04.362536+0000", 2023-09-23T21:14:01.415 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00027105100000000002 2023-09-23T21:14:01.415 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:14:01.415 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:14:01.415 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:14:01.415 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:04.367560+0000", 2023-09-23T21:14:01.415 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0050248089999999999 2023-09-23T21:14:01.415 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:14:01.415 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:14:01.415 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:14:01.415 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:04.372309+0000", 2023-09-23T21:14:01.415 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0047485729999999999 2023-09-23T21:14:01.416 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:14:01.416 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:14:01.416 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:14:01.416 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:04.408353+0000", 2023-09-23T21:14:01.416 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.036044377000000002 2023-09-23T21:14:01.416 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:14:01.416 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:14:01.416 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:14:01.416 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:04.408385+0000", 2023-09-23T21:14:01.416 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.1905e-05 2023-09-23T21:14:01.416 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:14:01.416 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:14:01.417 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:14:01.417 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:04.408420+0000", 2023-09-23T21:14:01.417 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.4273999999999997e-05 2023-09-23T21:14:01.417 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:14:01.417 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:14:01.417 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:14:01.417 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:14:01.417 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:14:01.417 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.35469.0:198545 8.19 8:9ee3d0ba:::benchmark_data_smithi026_301269_object87788:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e332)", 2023-09-23T21:14:01.417 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:05:03.301449+0000", 2023-09-23T21:14:01.417 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 538.03443734899997, 2023-09-23T21:14:01.417 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.1070379939999999, 2023-09-23T21:14:01.417 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:14:01.418 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:14:01.418 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:14:01.418 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.35469", 2023-09-23T21:14:01.418 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/4041424623", 2023-09-23T21:14:01.418 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 198545 2023-09-23T21:14:01.418 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:14:01.418 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:14:01.418 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:14:01.418 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:14:01.418 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:03.301449+0000", 2023-09-23T21:14:01.418 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:14:01.418 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:14:01.419 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:14:01.419 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:14:01.419 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:03.301449+0000", 2023-09-23T21:14:01.419 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:14:01.419 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:14:01.419 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:14:01.419 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:14:01.419 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:03.301448+0000", 2023-09-23T21:14:01.419 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999995 2023-09-23T21:14:01.419 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:14:01.419 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:14:01.419 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:14:01.420 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:03.301451+0000", 2023-09-23T21:14:01.420 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.4650000000000001e-06 2023-09-23T21:14:01.420 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:14:01.420 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:14:01.420 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:14:01.420 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:03.301456+0000", 2023-09-23T21:14:01.420 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.2089999999999996e-06 2023-09-23T21:14:01.420 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:14:01.420 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:14:01.420 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:14:01.420 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:03.301460+0000", 2023-09-23T21:14:01.420 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.9720000000000003e-06 2023-09-23T21:14:01.421 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:14:01.421 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:14:01.421 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:14:01.421 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:03.301528+0000", 2023-09-23T21:14:01.421 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.8028000000000007e-05 2023-09-23T21:14:01.421 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:14:01.421 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:14:01.421 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:14:01.421 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:03.301533+0000", 2023-09-23T21:14:01.421 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.0679999999999996e-06 2023-09-23T21:14:01.421 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:14:01.421 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:14:01.421 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:14:01.422 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:04.364665+0000", 2023-09-23T21:14:01.422 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.0631317149999999 2023-09-23T21:14:01.422 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:14:01.422 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:14:01.422 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:14:01.422 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:04.364799+0000", 2023-09-23T21:14:01.422 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00013458 2023-09-23T21:14:01.422 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:14:01.422 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:14:01.422 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 0,4", 2023-09-23T21:14:01.422 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:04.365119+0000", 2023-09-23T21:14:01.422 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00032011000000000002 2023-09-23T21:14:01.423 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:14:01.423 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:14:01.423 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:14:01.423 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:04.370011+0000", 2023-09-23T21:14:01.423 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0048917969999999998 2023-09-23T21:14:01.423 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:14:01.423 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:14:01.423 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:14:01.423 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:04.372346+0000", 2023-09-23T21:14:01.423 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0023348700000000002 2023-09-23T21:14:01.423 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:14:01.423 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:14:01.424 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:14:01.424 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:04.408444+0000", 2023-09-23T21:14:01.424 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.036097603999999998 2023-09-23T21:14:01.424 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:14:01.424 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:14:01.424 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:14:01.424 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:04.408467+0000", 2023-09-23T21:14:01.424 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.3538000000000001e-05 2023-09-23T21:14:01.424 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:14:01.424 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:14:01.424 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:14:01.424 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:04.408487+0000", 2023-09-23T21:14:01.424 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.9451000000000001e-05 2023-09-23T21:14:01.425 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:14:01.425 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:14:01.425 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:14:01.425 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:14:01.425 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:14:01.425 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.35469.0:198567 8.19 8:980f1725:::benchmark_data_smithi026_301269_object87810:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e332)", 2023-09-23T21:14:01.425 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:05:03.301478+0000", 2023-09-23T21:14:01.425 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 538.03440770600002, 2023-09-23T21:14:01.425 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.1099435849999999, 2023-09-23T21:14:01.425 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:14:01.425 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:14:01.425 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:14:01.426 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.35469", 2023-09-23T21:14:01.426 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/4041424623", 2023-09-23T21:14:01.426 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 198567 2023-09-23T21:14:01.426 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:14:01.426 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:14:01.426 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:14:01.426 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:14:01.426 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:03.301478+0000", 2023-09-23T21:14:01.426 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:14:01.426 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:14:01.426 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:14:01.426 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:14:01.427 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:03.301478+0000", 2023-09-23T21:14:01.427 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:14:01.427 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:14:01.427 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:14:01.427 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:14:01.427 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:03.301478+0000", 2023-09-23T21:14:01.427 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999995 2023-09-23T21:14:01.427 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:14:01.427 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:14:01.428 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:14:01.428 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:03.301480+0000", 2023-09-23T21:14:01.428 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.1090000000000001e-06 2023-09-23T21:14:01.428 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:14:01.428 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:14:01.428 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:14:01.428 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:03.301484+0000", 2023-09-23T21:14:01.429 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.2719999999999999e-06 2023-09-23T21:14:01.429 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:14:01.429 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:14:01.429 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:14:01.429 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:03.301487+0000", 2023-09-23T21:14:01.429 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.6599999999999999e-06 2023-09-23T21:14:01.429 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:14:01.429 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:14:01.429 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:14:01.429 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:03.301550+0000", 2023-09-23T21:14:01.429 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.3187000000000003e-05 2023-09-23T21:14:01.429 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:14:01.429 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:14:01.430 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:14:01.430 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:03.301555+0000", 2023-09-23T21:14:01.430 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.1959999999999999e-06 2023-09-23T21:14:01.430 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:14:01.430 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:14:01.430 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:14:01.430 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:04.367597+0000", 2023-09-23T21:14:01.430 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.06604136 2023-09-23T21:14:01.430 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:14:01.430 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:14:01.430 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:14:01.430 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:04.367702+0000", 2023-09-23T21:14:01.431 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000105509 2023-09-23T21:14:01.431 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:14:01.431 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:14:01.431 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 0,4", 2023-09-23T21:14:01.431 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:04.368038+0000", 2023-09-23T21:14:01.431 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00033594000000000001 2023-09-23T21:14:01.431 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:14:01.431 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:14:01.431 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:14:01.431 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:04.372443+0000", 2023-09-23T21:14:01.431 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0044051350000000001 2023-09-23T21:14:01.431 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:14:01.432 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:14:01.432 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:14:01.432 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:04.372452+0000", 2023-09-23T21:14:01.432 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.3140000000000004e-06 2023-09-23T21:14:01.432 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:14:01.432 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:14:01.432 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:14:01.432 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:04.411340+0000", 2023-09-23T21:14:01.432 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.038888058000000003 2023-09-23T21:14:01.432 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:14:01.432 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:14:01.432 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:14:01.433 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:04.411390+0000", 2023-09-23T21:14:01.433 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.0124999999999999e-05 2023-09-23T21:14:01.433 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:14:01.433 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:14:01.433 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:14:01.433 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:04.411422+0000", 2023-09-23T21:14:01.433 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.2061999999999999e-05 2023-09-23T21:14:01.433 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:14:01.433 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:14:01.433 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:14:01.433 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:14:01.433 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:14:01.433 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.35469.0:212440 8.3 8:c05433da:::benchmark_data_smithi026_301269_object101683:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e339)", 2023-09-23T21:14:01.434 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:05:21.317641+0000", 2023-09-23T21:14:01.434 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 520.018245353, 2023-09-23T21:14:01.434 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.98662722700000005, 2023-09-23T21:14:01.434 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:14:01.434 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:14:01.434 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:14:01.434 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.35469", 2023-09-23T21:14:01.434 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/4041424623", 2023-09-23T21:14:01.434 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 212440 2023-09-23T21:14:01.434 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:14:01.434 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:14:01.434 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:14:01.435 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:14:01.435 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:21.317641+0000", 2023-09-23T21:14:01.435 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:14:01.435 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:14:01.435 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:14:01.435 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:14:01.435 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:21.317641+0000", 2023-09-23T21:14:01.435 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:14:01.435 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:14:01.435 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:14:01.435 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:14:01.435 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:21.317639+0000", 2023-09-23T21:14:01.436 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999981 2023-09-23T21:14:01.436 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:14:01.436 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:14:01.436 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:14:01.436 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:21.317647+0000", 2023-09-23T21:14:01.436 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.1499999999999999e-06 2023-09-23T21:14:01.436 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:14:01.436 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:14:01.436 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:14:01.436 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:21.317664+0000", 2023-09-23T21:14:01.436 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.7238999999999999e-05 2023-09-23T21:14:01.436 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:14:01.436 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:14:01.437 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:14:01.437 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:21.317672+0000", 2023-09-23T21:14:01.437 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.644e-06 2023-09-23T21:14:01.437 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:14:01.437 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:14:01.437 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:14:01.437 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:21.317732+0000", 2023-09-23T21:14:01.437 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.9895999999999997e-05 2023-09-23T21:14:01.437 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:14:01.437 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:14:01.437 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:14:01.437 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:21.317743+0000", 2023-09-23T21:14:01.438 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.1093e-05 2023-09-23T21:14:01.438 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:14:01.438 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:14:01.438 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:14:01.438 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:22.298663+0000", 2023-09-23T21:14:01.438 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.980920032 2023-09-23T21:14:01.438 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:14:01.438 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:14:01.438 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:14:01.438 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:22.298776+0000", 2023-09-23T21:14:01.438 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000113408 2023-09-23T21:14:01.438 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:14:01.438 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:14:01.439 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 6", 2023-09-23T21:14:01.439 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:22.299290+0000", 2023-09-23T21:14:01.439 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00051313900000000002 2023-09-23T21:14:01.439 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:14:01.439 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:14:01.439 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:14:01.439 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:22.302224+0000", 2023-09-23T21:14:01.439 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0029344409999999999 2023-09-23T21:14:01.439 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:14:01.439 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:14:01.439 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:14:01.439 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:22.304159+0000", 2023-09-23T21:14:01.440 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00193462 2023-09-23T21:14:01.440 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:14:01.440 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:14:01.440 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:14:01.440 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:22.304199+0000", 2023-09-23T21:14:01.440 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.9855999999999999e-05 2023-09-23T21:14:01.440 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:14:01.440 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:14:01.440 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:14:01.440 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:05:22.304268+0000", 2023-09-23T21:14:01.440 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.9455999999999994e-05 2023-09-23T21:14:01.440 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:14:01.440 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:14:01.441 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:14:01.441 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:14:01.441 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:14:01.441 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.46328.0:100001 10.10 10:0c7ac00a:::benchmark_data_smithi026_379838_object20044:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e495)", 2023-09-23T21:14:01.441 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:13:13.465471+0000", 2023-09-23T21:14:01.441 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 47.870414996000001, 2023-09-23T21:14:01.441 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.4773325069999999, 2023-09-23T21:14:01.441 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:14:01.441 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:14:01.441 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:14:01.441 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.46328", 2023-09-23T21:14:01.442 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/3748861181", 2023-09-23T21:14:01.442 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 100001 2023-09-23T21:14:01.442 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:14:01.442 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:14:01.442 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:14:01.442 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:14:01.442 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:13.465471+0000", 2023-09-23T21:14:01.442 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:14:01.442 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:14:01.442 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:14:01.442 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:14:01.442 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:13.465471+0000", 2023-09-23T21:14:01.442 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:14:01.443 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:14:01.443 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:14:01.443 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:14:01.443 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:13.465470+0000", 2023-09-23T21:14:01.443 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T21:14:01.443 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:14:01.443 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:14:01.443 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:14:01.443 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:13.465477+0000", 2023-09-23T21:14:01.443 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.8259999999999993e-06 2023-09-23T21:14:01.443 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:14:01.443 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:14:01.444 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:14:01.444 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:13.465496+0000", 2023-09-23T21:14:01.444 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.8465999999999999e-05 2023-09-23T21:14:01.444 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:14:01.444 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:14:01.444 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:14:01.444 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:13.465504+0000", 2023-09-23T21:14:01.444 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.1869999999999997e-06 2023-09-23T21:14:01.444 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:14:01.444 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:14:01.444 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:14:01.444 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:13.465570+0000", 2023-09-23T21:14:01.444 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.5831e-05 2023-09-23T21:14:01.445 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:14:01.445 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:14:01.445 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:14:01.445 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:13.465583+0000", 2023-09-23T21:14:01.445 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.2883e-05 2023-09-23T21:14:01.445 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:14:01.445 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:14:01.445 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:14:01.445 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:16.722777+0000", 2023-09-23T21:14:01.445 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.2571938070000002 2023-09-23T21:14:01.445 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:14:01.446 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:14:01.446 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for degraded object", 2023-09-23T21:14:01.446 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:16.723120+0000", 2023-09-23T21:14:01.446 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00034327099999999997 2023-09-23T21:14:01.446 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:14:01.446 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:14:01.446 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:14:01.446 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:16.875534+0000", 2023-09-23T21:14:01.446 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.152413828 2023-09-23T21:14:01.446 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:14:01.446 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:14:01.446 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:14:01.446 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:16.875609+0000", 2023-09-23T21:14:01.447 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.4875000000000004e-05 2023-09-23T21:14:01.447 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:14:01.447 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:14:01.447 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 4,7", 2023-09-23T21:14:01.447 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:16.875890+0000", 2023-09-23T21:14:01.447 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00028097300000000001 2023-09-23T21:14:01.447 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:14:01.447 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:14:01.447 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:14:01.447 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:16.881216+0000", 2023-09-23T21:14:01.447 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0053265589999999998 2023-09-23T21:14:01.447 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:14:01.448 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:14:01.448 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:14:01.448 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:16.893186+0000", 2023-09-23T21:14:01.448 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.011969762 2023-09-23T21:14:01.448 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:14:01.448 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:14:01.448 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:14:01.448 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:16.941662+0000", 2023-09-23T21:14:01.448 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.048476545000000003 2023-09-23T21:14:01.448 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:14:01.448 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:14:01.448 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:14:01.448 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:16.941724+0000", 2023-09-23T21:14:01.449 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.1719999999999999e-05 2023-09-23T21:14:01.449 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:14:01.449 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:14:01.449 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:14:01.449 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:16.942804+0000", 2023-09-23T21:14:01.449 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.001079483 2023-09-23T21:14:01.449 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:14:01.449 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:14:01.449 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:14:01.449 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:14:01.449 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:14:01.449 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.46328.0:100035 10.10 10:091f2b4d:::benchmark_data_smithi026_379838_object20078:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e496)", 2023-09-23T21:14:01.450 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:13:15.077410+0000", 2023-09-23T21:14:01.450 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 46.25847632, 2023-09-23T21:14:01.450 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.866327224, 2023-09-23T21:14:01.450 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:14:01.450 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:14:01.450 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:14:01.450 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.46328", 2023-09-23T21:14:01.450 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/3748861181", 2023-09-23T21:14:01.450 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 100035 2023-09-23T21:14:01.450 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:14:01.450 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:14:01.450 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:14:01.451 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:14:01.451 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:15.077410+0000", 2023-09-23T21:14:01.451 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:14:01.451 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:14:01.451 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:14:01.451 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:14:01.451 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:15.077410+0000", 2023-09-23T21:14:01.451 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:14:01.451 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:14:01.451 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:14:01.451 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:14:01.451 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:15.077408+0000", 2023-09-23T21:14:01.451 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T21:14:01.452 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:14:01.452 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:14:01.452 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:14:01.452 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:15.077416+0000", 2023-09-23T21:14:01.452 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.6799999999999993e-06 2023-09-23T21:14:01.452 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:14:01.452 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:14:01.452 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:14:01.452 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:15.077434+0000", 2023-09-23T21:14:01.452 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.7634000000000001e-05 2023-09-23T21:14:01.452 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:14:01.452 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:14:01.453 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:14:01.453 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:15.077441+0000", 2023-09-23T21:14:01.453 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.8329999999999994e-06 2023-09-23T21:14:01.453 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:14:01.453 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:14:01.453 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:14:01.453 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:15.077507+0000", 2023-09-23T21:14:01.453 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.5270000000000004e-05 2023-09-23T21:14:01.453 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:14:01.453 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:14:01.453 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:14:01.453 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:15.077520+0000", 2023-09-23T21:14:01.453 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.3301e-05 2023-09-23T21:14:01.454 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:14:01.454 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:14:01.454 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:14:01.454 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:16.723157+0000", 2023-09-23T21:14:01.454 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.6456375030000001 2023-09-23T21:14:01.454 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:14:01.454 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:14:01.454 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for degraded object", 2023-09-23T21:14:01.454 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:16.723547+0000", 2023-09-23T21:14:01.454 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000389126 2023-09-23T21:14:01.454 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:14:01.454 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:14:01.455 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:14:01.455 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:16.878482+0000", 2023-09-23T21:14:01.455 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.15493558900000001 2023-09-23T21:14:01.455 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:14:01.455 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:14:01.455 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:14:01.455 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:16.878543+0000", 2023-09-23T21:14:01.455 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.0544000000000002e-05 2023-09-23T21:14:01.455 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:14:01.455 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:14:01.455 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 4,7", 2023-09-23T21:14:01.455 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:16.878732+0000", 2023-09-23T21:14:01.455 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000189736 2023-09-23T21:14:01.456 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:14:01.456 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:14:01.456 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:14:01.456 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:16.883755+0000", 2023-09-23T21:14:01.456 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0050220610000000004 2023-09-23T21:14:01.456 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:14:01.456 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:14:01.456 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:14:01.456 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:16.895687+0000", 2023-09-23T21:14:01.456 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.011932720000000001 2023-09-23T21:14:01.456 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:14:01.456 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:14:01.456 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:14:01.457 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:16.942854+0000", 2023-09-23T21:14:01.457 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.047166300000000001 2023-09-23T21:14:01.457 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:14:01.457 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:14:01.457 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:14:01.457 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:16.942900+0000", 2023-09-23T21:14:01.457 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.6932999999999999e-05 2023-09-23T21:14:01.457 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:14:01.457 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:14:01.457 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:14:01.457 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:16.943737+0000", 2023-09-23T21:14:01.457 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00083654400000000005 2023-09-23T21:14:01.457 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:14:01.458 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:14:01.458 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:14:01.458 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:14:01.458 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:14:01.458 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.46328.0:100041 10.10 10:0d6c00f5:::benchmark_data_smithi026_379838_object20084:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e497)", 2023-09-23T21:14:01.458 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:13:15.390855+0000", 2023-09-23T21:14:01.458 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 45.945031338, 2023-09-23T21:14:01.458 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.5537436790000001, 2023-09-23T21:14:01.458 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:14:01.458 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:14:01.458 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:14:01.458 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.46328", 2023-09-23T21:14:01.459 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/3748861181", 2023-09-23T21:14:01.459 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 100041 2023-09-23T21:14:01.459 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:14:01.459 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:14:01.459 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:14:01.459 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:14:01.459 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:15.390855+0000", 2023-09-23T21:14:01.459 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:14:01.459 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:14:01.459 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:14:01.459 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:14:01.459 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:15.390855+0000", 2023-09-23T21:14:01.459 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:14:01.460 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:14:01.460 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:14:01.460 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:14:01.460 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:15.390853+0000", 2023-09-23T21:14:01.460 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T21:14:01.460 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:14:01.460 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:14:01.460 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:14:01.460 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:15.390861+0000", 2023-09-23T21:14:01.460 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.6559999999999992e-06 2023-09-23T21:14:01.460 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:14:01.460 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:14:01.461 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:14:01.461 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:15.390875+0000", 2023-09-23T21:14:01.461 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.4440000000000001e-05 2023-09-23T21:14:01.461 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:14:01.461 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:14:01.461 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:14:01.461 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:15.390882+0000", 2023-09-23T21:14:01.461 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.3249999999999996e-06 2023-09-23T21:14:01.461 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:14:01.461 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:14:01.461 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:14:01.461 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:15.390949+0000", 2023-09-23T21:14:01.461 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.7775999999999996e-05 2023-09-23T21:14:01.462 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:14:01.462 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:14:01.462 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:14:01.462 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:15.390963+0000", 2023-09-23T21:14:01.462 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.3139000000000001e-05 2023-09-23T21:14:01.462 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:14:01.462 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:14:01.462 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:14:01.462 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:16.723581+0000", 2023-09-23T21:14:01.462 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.332618455 2023-09-23T21:14:01.462 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:14:01.463 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:14:01.463 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for degraded object", 2023-09-23T21:14:01.463 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:16.723937+0000", 2023-09-23T21:14:01.463 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00035614099999999998 2023-09-23T21:14:01.463 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:14:01.463 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:14:01.463 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:14:01.463 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:16.881249+0000", 2023-09-23T21:14:01.463 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.15731209800000001 2023-09-23T21:14:01.463 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:14:01.463 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:14:01.463 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:14:01.464 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:16.881317+0000", 2023-09-23T21:14:01.464 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.7847000000000003e-05 2023-09-23T21:14:01.464 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:14:01.464 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:14:01.464 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 4,7", 2023-09-23T21:14:01.464 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:16.881502+0000", 2023-09-23T21:14:01.464 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00018458999999999999 2023-09-23T21:14:01.464 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:14:01.464 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:14:01.464 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:14:01.464 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:16.886249+0000", 2023-09-23T21:14:01.464 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0047471739999999998 2023-09-23T21:14:01.464 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:14:01.465 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:14:01.465 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:14:01.465 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:16.895740+0000", 2023-09-23T21:14:01.465 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0094913419999999998 2023-09-23T21:14:01.465 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:14:01.465 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:14:01.465 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:14:01.465 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:16.943800+0000", 2023-09-23T21:14:01.465 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.048059719000000001 2023-09-23T21:14:01.465 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:14:01.465 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:14:01.465 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:14:01.465 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:16.943851+0000", 2023-09-23T21:14:01.466 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.1156999999999998e-05 2023-09-23T21:14:01.466 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:14:01.466 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:14:01.466 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:14:01.466 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:16.944598+0000", 2023-09-23T21:14:01.466 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00074733099999999995 2023-09-23T21:14:01.466 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:14:01.466 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:14:01.466 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:14:01.466 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:14:01.466 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:14:01.466 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.46328.0:100102 10.10 10:0b658828:::benchmark_data_smithi026_379838_object20145:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e497)", 2023-09-23T21:14:01.467 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:13:15.673805+0000", 2023-09-23T21:14:01.467 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 45.662080951, 2023-09-23T21:14:01.467 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.2715812390000001, 2023-09-23T21:14:01.467 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:14:01.467 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:14:01.467 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:14:01.467 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.46328", 2023-09-23T21:14:01.467 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/3748861181", 2023-09-23T21:14:01.467 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 100102 2023-09-23T21:14:01.467 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:14:01.467 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:14:01.467 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:14:01.468 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:14:01.468 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:15.673805+0000", 2023-09-23T21:14:01.468 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:14:01.468 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:14:01.468 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:14:01.468 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:14:01.468 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:15.673805+0000", 2023-09-23T21:14:01.468 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:14:01.468 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:14:01.468 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:14:01.468 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:14:01.468 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:15.673804+0000", 2023-09-23T21:14:01.469 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T21:14:01.469 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:14:01.469 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:14:01.469 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:14:01.469 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:15.673810+0000", 2023-09-23T21:14:01.469 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.7379999999999997e-06 2023-09-23T21:14:01.469 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:14:01.469 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:14:01.469 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:14:01.469 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:15.673828+0000", 2023-09-23T21:14:01.469 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.7768999999999999e-05 2023-09-23T21:14:01.469 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:14:01.469 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:14:01.470 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:14:01.470 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:15.673836+0000", 2023-09-23T21:14:01.470 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.6399999999999997e-06 2023-09-23T21:14:01.470 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:14:01.470 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:14:01.470 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:14:01.470 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:15.673897+0000", 2023-09-23T21:14:01.470 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.1247000000000005e-05 2023-09-23T21:14:01.470 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:14:01.470 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:14:01.470 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:14:01.470 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:15.673909+0000", 2023-09-23T21:14:01.471 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.1698e-05 2023-09-23T21:14:01.471 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:14:01.471 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:14:01.471 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:14:01.471 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:16.723986+0000", 2023-09-23T21:14:01.471 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.0500774100000001 2023-09-23T21:14:01.471 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:14:01.471 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:14:01.471 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for degraded object", 2023-09-23T21:14:01.471 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:16.724288+0000", 2023-09-23T21:14:01.471 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000301472 2023-09-23T21:14:01.471 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:14:01.471 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:14:01.472 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:14:01.472 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:16.883809+0000", 2023-09-23T21:14:01.472 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.15952158999999999 2023-09-23T21:14:01.472 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:14:01.472 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:14:01.472 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:14:01.472 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:16.883893+0000", 2023-09-23T21:14:01.472 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.4110000000000006e-05 2023-09-23T21:14:01.472 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:14:01.472 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:14:01.472 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 4,7", 2023-09-23T21:14:01.472 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:16.884072+0000", 2023-09-23T21:14:01.473 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00017881400000000001 2023-09-23T21:14:01.473 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:14:01.473 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:14:01.473 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:14:01.473 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:16.890278+0000", 2023-09-23T21:14:01.473 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.006205398 2023-09-23T21:14:01.473 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:14:01.473 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:14:01.473 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:14:01.473 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:16.895778+0000", 2023-09-23T21:14:01.473 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0055007260000000001 2023-09-23T21:14:01.473 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:14:01.473 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:14:01.474 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:14:01.474 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:16.944655+0000", 2023-09-23T21:14:01.474 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.048876997999999998 2023-09-23T21:14:01.474 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:14:01.474 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:14:01.474 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:14:01.474 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:16.944704+0000", 2023-09-23T21:14:01.474 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.8625999999999998e-05 2023-09-23T21:14:01.474 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:14:01.474 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:14:01.474 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:14:01.474 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:16.945386+0000", 2023-09-23T21:14:01.475 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00068246099999999998 2023-09-23T21:14:01.475 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:14:01.475 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:14:01.475 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:14:01.475 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:14:01.475 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:14:01.475 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.46328.0:100156 10.10 10:0d90ee30:::benchmark_data_smithi026_379838_object20199:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e497)", 2023-09-23T21:14:01.475 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:13:15.866363+0000", 2023-09-23T21:14:01.475 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 45.469523561999999, 2023-09-23T21:14:01.475 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.0797975689999999, 2023-09-23T21:14:01.475 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:14:01.475 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:14:01.475 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:14:01.476 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.46328", 2023-09-23T21:14:01.476 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/3748861181", 2023-09-23T21:14:01.476 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 100156 2023-09-23T21:14:01.476 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:14:01.476 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:14:01.476 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:14:01.476 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:14:01.476 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:15.866363+0000", 2023-09-23T21:14:01.476 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:14:01.476 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:14:01.476 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:14:01.476 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:14:01.476 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:15.866363+0000", 2023-09-23T21:14:01.477 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:14:01.477 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:14:01.477 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:14:01.477 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:14:01.477 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:15.866361+0000", 2023-09-23T21:14:01.477 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T21:14:01.477 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:14:01.477 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:14:01.477 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:14:01.477 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:15.866368+0000", 2023-09-23T21:14:01.477 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.0759999999999997e-06 2023-09-23T21:14:01.478 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:14:01.478 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:14:01.478 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:14:01.478 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:15.866384+0000", 2023-09-23T21:14:01.478 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.6299000000000001e-05 2023-09-23T21:14:01.478 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:14:01.478 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:14:01.478 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:14:01.478 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:15.866391+0000", 2023-09-23T21:14:01.478 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.5579999999999997e-06 2023-09-23T21:14:01.478 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:14:01.478 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:14:01.479 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:14:01.479 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:15.866446+0000", 2023-09-23T21:14:01.479 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.5102999999999997e-05 2023-09-23T21:14:01.479 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:14:01.479 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:14:01.479 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:14:01.479 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:15.866456+0000", 2023-09-23T21:14:01.479 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.5179999999999993e-06 2023-09-23T21:14:01.479 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:14:01.479 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:14:01.479 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:14:01.479 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:16.724330+0000", 2023-09-23T21:14:01.479 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.85787401799999996 2023-09-23T21:14:01.480 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:14:01.480 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:14:01.480 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for degraded object", 2023-09-23T21:14:01.480 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:16.724694+0000", 2023-09-23T21:14:01.480 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00036395399999999999 2023-09-23T21:14:01.480 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:14:01.480 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:14:01.480 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:14:01.480 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:16.886296+0000", 2023-09-23T21:14:01.480 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.16160285299999999 2023-09-23T21:14:01.480 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:14:01.480 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:14:01.481 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:14:01.481 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:16.886397+0000", 2023-09-23T21:14:01.481 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000100832 2023-09-23T21:14:01.481 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:14:01.481 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:14:01.481 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 4,7", 2023-09-23T21:14:01.481 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:16.886638+0000", 2023-09-23T21:14:01.481 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000240444 2023-09-23T21:14:01.481 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:14:01.481 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:14:01.481 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:14:01.481 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:16.893114+0000", 2023-09-23T21:14:01.481 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0064762530000000004 2023-09-23T21:14:01.482 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:14:01.482 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:14:01.482 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:14:01.482 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:16.895818+0000", 2023-09-23T21:14:01.482 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0027042379999999999 2023-09-23T21:14:01.482 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:14:01.482 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:14:01.482 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:14:01.482 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:16.945430+0000", 2023-09-23T21:14:01.482 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.049611719999999998 2023-09-23T21:14:01.482 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:14:01.482 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:14:01.483 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:14:01.483 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:16.945478+0000", 2023-09-23T21:14:01.483 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.8433000000000002e-05 2023-09-23T21:14:01.483 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:14:01.483 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:14:01.483 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:14:01.483 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:16.946160+0000", 2023-09-23T21:14:01.483 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00068183899999999995 2023-09-23T21:14:01.483 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:14:01.483 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:14:01.483 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:14:01.483 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:14:01.483 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:14:01.484 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.46328.0:100162 10.10 10:0dbd73eb:::benchmark_data_smithi026_379838_object20205:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e497)", 2023-09-23T21:14:01.484 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:13:15.889020+0000", 2023-09-23T21:14:01.484 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 45.446865625000001, 2023-09-23T21:14:01.484 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.057916241, 2023-09-23T21:14:01.484 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:14:01.484 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:14:01.484 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:14:01.484 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.46328", 2023-09-23T21:14:01.484 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/3748861181", 2023-09-23T21:14:01.484 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 100162 2023-09-23T21:14:01.484 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:14:01.484 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:14:01.485 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:14:01.485 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:14:01.485 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:15.889020+0000", 2023-09-23T21:14:01.485 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:14:01.485 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:14:01.485 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:14:01.485 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:14:01.485 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:15.889020+0000", 2023-09-23T21:14:01.485 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:14:01.485 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:14:01.485 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:14:01.485 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:14:01.485 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:15.889019+0000", 2023-09-23T21:14:01.486 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.999999 2023-09-23T21:14:01.486 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:14:01.486 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:14:01.486 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:14:01.486 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:15.889025+0000", 2023-09-23T21:14:01.486 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.2950000000000001e-06 2023-09-23T21:14:01.486 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:14:01.486 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:14:01.486 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:14:01.486 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:15.889037+0000", 2023-09-23T21:14:01.486 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.2515e-05 2023-09-23T21:14:01.486 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:14:01.487 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:14:01.487 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:14:01.487 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:15.889042+0000", 2023-09-23T21:14:01.487 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.259e-06 2023-09-23T21:14:01.487 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:14:01.487 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:14:01.487 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:14:01.487 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:15.889091+0000", 2023-09-23T21:14:01.487 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.8362000000000002e-05 2023-09-23T21:14:01.487 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:14:01.487 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:14:01.487 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:14:01.487 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:15.889097+0000", 2023-09-23T21:14:01.488 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.5320000000000003e-06 2023-09-23T21:14:01.488 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:14:01.488 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:14:01.488 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:14:01.488 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:16.724728+0000", 2023-09-23T21:14:01.488 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.835630977 2023-09-23T21:14:01.488 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:14:01.488 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:14:01.488 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for degraded object", 2023-09-23T21:14:01.488 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:16.725083+0000", 2023-09-23T21:14:01.488 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00035435300000000001 2023-09-23T21:14:01.489 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:14:01.489 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:14:01.489 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:14:01.489 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:16.888992+0000", 2023-09-23T21:14:01.489 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.163909412 2023-09-23T21:14:01.489 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:14:01.489 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:14:01.489 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:14:01.489 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:16.889058+0000", 2023-09-23T21:14:01.489 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.6261999999999997e-05 2023-09-23T21:14:01.489 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:14:01.489 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:14:01.489 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 4,7", 2023-09-23T21:14:01.490 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:16.889271+0000", 2023-09-23T21:14:01.490 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00021296299999999999 2023-09-23T21:14:01.490 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:14:01.490 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:14:01.490 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:14:01.490 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:16.894373+0000", 2023-09-23T21:14:01.490 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0051016830000000001 2023-09-23T21:14:01.490 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:14:01.490 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:14:01.490 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:14:01.490 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:16.895865+0000", 2023-09-23T21:14:01.490 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0014915620000000001 2023-09-23T21:14:01.491 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:14:01.491 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:14:01.491 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:14:01.491 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:16.946201+0000", 2023-09-23T21:14:01.491 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.050336233000000001 2023-09-23T21:14:01.491 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:14:01.491 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:14:01.491 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:14:01.491 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:16.946250+0000", 2023-09-23T21:14:01.491 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.9400999999999997e-05 2023-09-23T21:14:01.491 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:14:01.491 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:14:01.491 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:14:01.492 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:16.946937+0000", 2023-09-23T21:14:01.492 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00068656700000000001 2023-09-23T21:14:01.492 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:14:01.492 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:14:01.492 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:14:01.492 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:14:01.492 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:14:01.492 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.46328.0:100182 10.10 10:0d89bf4d:::benchmark_data_smithi026_379838_object20225:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e497)", 2023-09-23T21:14:01.492 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:13:15.978413+0000", 2023-09-23T21:14:01.492 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 45.357472680000001, 2023-09-23T21:14:01.492 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.96927643900000005, 2023-09-23T21:14:01.492 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:14:01.493 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:14:01.493 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:14:01.493 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.46328", 2023-09-23T21:14:01.493 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/3748861181", 2023-09-23T21:14:01.493 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 100182 2023-09-23T21:14:01.493 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:14:01.493 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:14:01.493 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:14:01.493 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:14:01.493 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:15.978413+0000", 2023-09-23T21:14:01.493 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:14:01.493 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:14:01.493 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:14:01.494 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:14:01.494 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:15.978413+0000", 2023-09-23T21:14:01.494 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:14:01.494 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:14:01.494 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:14:01.494 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:14:01.494 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:15.978412+0000", 2023-09-23T21:14:01.494 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.999999 2023-09-23T21:14:01.494 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:14:01.494 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:14:01.494 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:14:01.494 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:15.978417+0000", 2023-09-23T21:14:01.495 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.0599999999999998e-06 2023-09-23T21:14:01.495 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:14:01.495 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:14:01.495 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:14:01.495 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:15.978429+0000", 2023-09-23T21:14:01.495 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.1386000000000001e-05 2023-09-23T21:14:01.495 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:14:01.495 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:14:01.495 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:14:01.495 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:15.978434+0000", 2023-09-23T21:14:01.495 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.2989999999999996e-06 2023-09-23T21:14:01.495 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:14:01.495 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:14:01.496 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:14:01.496 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:15.978481+0000", 2023-09-23T21:14:01.496 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.6622999999999997e-05 2023-09-23T21:14:01.496 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:14:01.496 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:14:01.496 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:14:01.496 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:15.978487+0000", 2023-09-23T21:14:01.496 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.5950000000000004e-06 2023-09-23T21:14:01.496 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:14:01.496 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:14:01.496 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:14:01.496 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:16.725115+0000", 2023-09-23T21:14:01.496 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.74662764199999998 2023-09-23T21:14:01.497 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:14:01.497 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:14:01.497 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for degraded object", 2023-09-23T21:14:01.497 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:16.725478+0000", 2023-09-23T21:14:01.497 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00036277599999999998 2023-09-23T21:14:01.497 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:14:01.497 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:14:01.497 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:14:01.497 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:16.891492+0000", 2023-09-23T21:14:01.497 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.166013891 2023-09-23T21:14:01.497 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:14:01.497 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:14:01.498 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:14:01.498 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:16.891595+0000", 2023-09-23T21:14:01.498 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000103737 2023-09-23T21:14:01.498 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:14:01.498 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:14:01.498 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 4,7", 2023-09-23T21:14:01.498 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:16.891852+0000", 2023-09-23T21:14:01.498 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00025655900000000001 2023-09-23T21:14:01.498 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:14:01.498 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:14:01.498 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:14:01.498 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:16.895701+0000", 2023-09-23T21:14:01.499 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0038490569999999999 2023-09-23T21:14:01.499 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:14:01.499 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:14:01.499 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:14:01.499 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:16.895891+0000", 2023-09-23T21:14:01.499 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00019018299999999999 2023-09-23T21:14:01.499 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:14:01.499 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:14:01.499 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:14:01.499 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:16.946983+0000", 2023-09-23T21:14:01.499 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.051091717000000002 2023-09-23T21:14:01.499 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:14:01.499 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:14:01.500 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:14:01.500 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:16.947035+0000", 2023-09-23T21:14:01.500 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.2031999999999999e-05 2023-09-23T21:14:01.500 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:14:01.500 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:14:01.500 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:14:01.500 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:16.947690+0000", 2023-09-23T21:14:01.500 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000655063 2023-09-23T21:14:01.500 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:14:01.500 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:14:01.500 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:14:01.500 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:14:01.500 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:14:01.501 INFO:teuthology.orchestra.run.smithi070.stdout:} 2023-09-23T21:14:01.502 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:14:01.502 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:14:01.502 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:14:01.503 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:14:01.603 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:14:01.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:14:01 smithi070 ceph-mon[168282]: failed to encode map e522 with expected crc 2023-09-23T21:14:01.621 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:14:01 smithi070 ceph-mon[168282]: failed to encode map e522 with expected crc 2023-09-23T21:14:01.621 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:14:01 smithi070 ceph-mon[168282]: osdmap e522: 8 total, 8 up, 6 in 2023-09-23T21:14:01.621 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:14:01 smithi070 ceph-mon[168282]: failed to encode map e522 with expected crc 2023-09-23T21:14:01.621 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:14:01 smithi070 ceph-mon[168282]: failed to encode map e522 with expected crc 2023-09-23T21:14:01.621 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:14:01 smithi070 ceph-mon[168282]: failed to encode map e522 with expected crc 2023-09-23T21:14:01.621 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:14:01 smithi070 ceph-mon[168282]: Health check update: Degraded data redundancy: 28763/148584 objects degraded (19.358%), 23 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T21:14:01.621 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:14:01 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:14:01.621 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:14:01 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:14:01.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:14:01 smithi026 ceph-mon[175151]: failed to encode map e522 with expected crc 2023-09-23T21:14:01.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:14:01 smithi026 ceph-mon[175151]: failed to encode map e522 with expected crc 2023-09-23T21:14:01.633 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:14:01 smithi026 ceph-mon[175151]: osdmap e522: 8 total, 8 up, 6 in 2023-09-23T21:14:01.633 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:14:01 smithi026 ceph-mon[175151]: failed to encode map e522 with expected crc 2023-09-23T21:14:01.633 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:14:01 smithi026 ceph-mon[175151]: failed to encode map e522 with expected crc 2023-09-23T21:14:01.633 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:14:01 smithi026 ceph-mon[175151]: failed to encode map e522 with expected crc 2023-09-23T21:14:01.633 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:14:01 smithi026 ceph-mon[175151]: Health check update: Degraded data redundancy: 28763/148584 objects degraded (19.358%), 23 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T21:14:01.633 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:14:01 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:14:01.633 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:14:01 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:14:01.635 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:14:01 smithi026 ceph-mon[179688]: failed to encode map e522 with expected crc 2023-09-23T21:14:01.635 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:14:01 smithi026 ceph-mon[179688]: failed to encode map e522 with expected crc 2023-09-23T21:14:01.636 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:14:01 smithi026 ceph-mon[179688]: osdmap e522: 8 total, 8 up, 6 in 2023-09-23T21:14:01.636 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:14:01 smithi026 ceph-mon[179688]: failed to encode map e522 with expected crc 2023-09-23T21:14:01.636 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:14:01 smithi026 ceph-mon[179688]: failed to encode map e522 with expected crc 2023-09-23T21:14:01.636 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:14:01 smithi026 ceph-mon[179688]: failed to encode map e522 with expected crc 2023-09-23T21:14:01.636 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:14:01 smithi026 ceph-mon[179688]: Health check update: Degraded data redundancy: 28763/148584 objects degraded (19.358%), 23 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T21:14:01.636 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:14:01 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:14:01.636 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:14:01 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:14:01.672 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:14:01.719 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:14:01.780 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:14:01.793 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:14:01.793 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:14:01.894 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:14:01.948 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:14:02.001 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:14:02.007 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:14:02.007 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:14:02.108 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:14:02.147 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:14:02.440 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:14:02.453 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:14:02.458 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:14:02.458 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:14:02.549 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_ops_in_flight 2023-09-23T21:14:02.559 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:14:02.648 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:14:02.697 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:14:02 smithi026 ceph-mon[179688]: pgmap v1308: 129 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 17 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 104 active+clean; 3.0 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 350 op/s; 28763/147300 objects degraded (19.527%); 3944/147300 objects misplaced (2.678%); 14 MiB/s, 281 objects/s recovering 2023-09-23T21:14:02.697 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:14:02 smithi026 ceph-mon[179688]: failed to encode map e523 with expected crc 2023-09-23T21:14:02.697 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:14:02 smithi026 ceph-mon[179688]: failed to encode map e523 with expected crc 2023-09-23T21:14:02.697 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:14:02 smithi026 ceph-mon[179688]: osdmap e523: 8 total, 8 up, 6 in 2023-09-23T21:14:02.697 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:14:02 smithi026 ceph-mon[175151]: pgmap v1308: 129 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 17 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 104 active+clean; 3.0 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 350 op/s; 28763/147300 objects degraded (19.527%); 3944/147300 objects misplaced (2.678%); 14 MiB/s, 281 objects/s recovering 2023-09-23T21:14:02.697 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:14:02 smithi026 ceph-mon[175151]: failed to encode map e523 with expected crc 2023-09-23T21:14:02.698 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:14:02 smithi026 ceph-mon[175151]: failed to encode map e523 with expected crc 2023-09-23T21:14:02.698 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:14:02 smithi026 ceph-mon[175151]: osdmap e523: 8 total, 8 up, 6 in 2023-09-23T21:14:02.726 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:14:02.734 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:14:02 smithi070 ceph-mon[168282]: pgmap v1308: 129 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 17 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 104 active+clean; 3.0 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 350 op/s; 28763/147300 objects degraded (19.527%); 3944/147300 objects misplaced (2.678%); 14 MiB/s, 281 objects/s recovering 2023-09-23T21:14:02.734 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:14:02 smithi070 ceph-mon[168282]: failed to encode map e523 with expected crc 2023-09-23T21:14:02.734 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:14:02 smithi070 ceph-mon[168282]: failed to encode map e523 with expected crc 2023-09-23T21:14:02.734 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:14:02 smithi070 ceph-mon[168282]: osdmap e523: 8 total, 8 up, 6 in 2023-09-23T21:14:02.735 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:14:02.735 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:14:02.774 INFO:teuthology.orchestra.run.smithi026.stderr:noscrub is set 2023-09-23T21:14:02.835 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:14:02.888 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:14:03.208 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:14:03.212 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:14:03.212 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:14:03.294 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T21:14:03.313 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:14:03.356 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:14:03.508 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:14:03 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/1241366304' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T21:14:03.509 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:14:03 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T21:14:03.509 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:14:03 smithi026 ceph-mon[175151]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:14:03.509 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:14:03 smithi026 ceph-mon[175151]: failed to encode map e524 with expected crc 2023-09-23T21:14:03.509 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:14:03 smithi026 ceph-mon[175151]: failed to encode map e524 with expected crc 2023-09-23T21:14:03.509 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:14:03 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T21:14:03.509 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:14:03 smithi026 ceph-mon[175151]: osdmap e524: 8 total, 8 up, 6 in 2023-09-23T21:14:03.509 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:14:03 smithi026 ceph-mon[175151]: failed to encode map e524 with expected crc 2023-09-23T21:14:03.509 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:14:03 smithi026 ceph-mon[175151]: failed to encode map e524 with expected crc 2023-09-23T21:14:03.649 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0: {} 2023-09-23T21:14:03.656 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:14:03.660 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:14:03.660 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:14:03.680 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1: {} 2023-09-23T21:14:03.708 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2: {} 2023-09-23T21:14:03.736 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3: {} 2023-09-23T21:14:03.760 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:14:03.767 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4: {} 2023-09-23T21:14:03.788 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:14:03 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/1241366304' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T21:14:03.788 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:14:03 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T21:14:03.789 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:14:03 smithi070 ceph-mon[168282]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:14:03.789 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:14:03 smithi070 ceph-mon[168282]: failed to encode map e524 with expected crc 2023-09-23T21:14:03.789 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:14:03 smithi070 ceph-mon[168282]: failed to encode map e524 with expected crc 2023-09-23T21:14:03.789 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:14:03 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T21:14:03.789 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:14:03 smithi070 ceph-mon[168282]: osdmap e524: 8 total, 8 up, 6 in 2023-09-23T21:14:03.789 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:14:03 smithi070 ceph-mon[168282]: failed to encode map e524 with expected crc 2023-09-23T21:14:03.789 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:14:03 smithi070 ceph-mon[168282]: failed to encode map e524 with expected crc 2023-09-23T21:14:03.798 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T21:14:03.800 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:14:03.807 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:14:03 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/1241366304' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T21:14:03.808 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:14:03 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T21:14:03.808 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:14:03 smithi026 ceph-mon[179688]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:14:03.808 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:14:03 smithi026 ceph-mon[179688]: failed to encode map e524 with expected crc 2023-09-23T21:14:03.808 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:14:03 smithi026 ceph-mon[179688]: failed to encode map e524 with expected crc 2023-09-23T21:14:03.808 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:14:03 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T21:14:03.808 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:14:03 smithi026 ceph-mon[179688]: osdmap e524: 8 total, 8 up, 6 in 2023-09-23T21:14:03.808 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:14:03 smithi026 ceph-mon[179688]: failed to encode map e524 with expected crc 2023-09-23T21:14:03.808 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:14:03 smithi026 ceph-mon[179688]: failed to encode map e524 with expected crc 2023-09-23T21:14:03.830 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6: {} 2023-09-23T21:14:03.850 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:14:03.854 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:14:03.855 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:14:03.861 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T21:14:03.867 INFO:teuthology.orchestra.run.smithi026.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T21:14:03.867 INFO:teuthology.orchestra.run.smithi026.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T21:14:03.867 INFO:teuthology.orchestra.run.smithi026.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T21:14:03.867 INFO:teuthology.orchestra.run.smithi026.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T21:14:03.867 INFO:teuthology.orchestra.run.smithi026.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T21:14:03.867 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T21:14:03.867 INFO:teuthology.orchestra.run.smithi026.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T21:14:03.868 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T21:14:03.955 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:14:03.995 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:14:04.516 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:14:04.519 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:14:04.520 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:14:04.605 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:14:04 smithi026 ceph-mon[179688]: pgmap v1311: 129 pgs: 2 active+recovering+undersized+remapped, 3 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 16 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 106 active+clean; 2.9 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 292 op/s; 28112/144942 objects degraded (19.395%); 3877/144942 objects misplaced (2.675%); 8.7 MiB/s, 334 objects/s recovering 2023-09-23T21:14:04.605 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:14:04 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/1007664438' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:14:04.605 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:14:04 smithi026 ceph-mon[179688]: failed to encode map e525 with expected crc 2023-09-23T21:14:04.605 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:14:04 smithi026 ceph-mon[179688]: osdmap e525: 8 total, 8 up, 6 in 2023-09-23T21:14:04.605 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:14:04 smithi026 ceph-mon[179688]: failed to encode map e525 with expected crc 2023-09-23T21:14:04.605 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:14:04 smithi026 ceph-mon[179688]: failed to encode map e525 with expected crc 2023-09-23T21:14:04.606 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:14:04 smithi026 ceph-mon[179688]: failed to encode map e525 with expected crc 2023-09-23T21:14:04.606 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:14:04 smithi026 ceph-mon[175151]: pgmap v1311: 129 pgs: 2 active+recovering+undersized+remapped, 3 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 16 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 106 active+clean; 2.9 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 292 op/s; 28112/144942 objects degraded (19.395%); 3877/144942 objects misplaced (2.675%); 8.7 MiB/s, 334 objects/s recovering 2023-09-23T21:14:04.606 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:14:04 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/1007664438' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:14:04.606 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:14:04 smithi026 ceph-mon[175151]: failed to encode map e525 with expected crc 2023-09-23T21:14:04.606 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:14:04 smithi026 ceph-mon[175151]: osdmap e525: 8 total, 8 up, 6 in 2023-09-23T21:14:04.606 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:14:04 smithi026 ceph-mon[175151]: failed to encode map e525 with expected crc 2023-09-23T21:14:04.606 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:14:04 smithi026 ceph-mon[175151]: failed to encode map e525 with expected crc 2023-09-23T21:14:04.606 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:14:04 smithi026 ceph-mon[175151]: failed to encode map e525 with expected crc 2023-09-23T21:14:04.621 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:14:04.658 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:14:04.760 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T21:14:04.763 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:14:04.763 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:14:04.829 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:14:04 smithi070 ceph-mon[168282]: pgmap v1311: 129 pgs: 2 active+recovering+undersized+remapped, 3 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 16 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 106 active+clean; 2.9 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 292 op/s; 28112/144942 objects degraded (19.395%); 3877/144942 objects misplaced (2.675%); 8.7 MiB/s, 334 objects/s recovering 2023-09-23T21:14:04.829 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:14:04 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/1007664438' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:14:04.829 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:14:04 smithi070 ceph-mon[168282]: failed to encode map e525 with expected crc 2023-09-23T21:14:04.829 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:14:04 smithi070 ceph-mon[168282]: osdmap e525: 8 total, 8 up, 6 in 2023-09-23T21:14:04.829 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:14:04 smithi070 ceph-mon[168282]: failed to encode map e525 with expected crc 2023-09-23T21:14:04.829 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:14:04 smithi070 ceph-mon[168282]: failed to encode map e525 with expected crc 2023-09-23T21:14:04.830 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:14:04 smithi070 ceph-mon[168282]: failed to encode map e525 with expected crc 2023-09-23T21:14:04.864 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:14:04.908 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:14:04.956 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:14:04.963 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:14:04.964 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:14:05.064 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:14:05.106 INFO:teuthology.orchestra.run.smithi026.stdout: 2023-09-23T21:14:05.109 INFO:teuthology.orchestra.run.smithi026.stderr:dumped all 2023-09-23T21:14:05.111 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:14:05.147 INFO:teuthology.orchestra.run.smithi070.stdout:ERROR: (22) Invalid argument 2023-09-23T21:14:05.147 INFO:teuthology.orchestra.run.smithi070.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T21:14:05.160 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T21:14:05.164 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:14:05.164 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:14:05.265 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:14:05.310 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:14:05.556 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:14:05.559 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:14:05.560 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:14:05.599 INFO:teuthology.orchestra.run.smithi026.stdout:{"pg_ready":true,"pg_map":{"version":1313,"stamp":"2023-09-23T21:14:05.042465+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":3084696324,"num_objects":47086,"num_object_clones":0,"num_object_copies":141258,"num_objects_missing_on_primary":3546,"num_objects_missing":3546,"num_objects_degraded":25394,"num_objects_misplaced":5223,"num_objects_unfound":0,"num_objects_dirty":47086,"num_whiteouts":0,"num_read":90,"num_read_kb":166,"num_write":100418,"num_write_kb":4309782,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":35590,"num_bytes_recovered":2149551269,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":129487,"ondisk_log_size":129487,"up":387,"acting":369,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":435,"num_osds":6,"num_per_pool_osds":6,"num_per_pool_omap_osds":6,"kb":562470912,"kb_used":13506788,"kb_used_data":10344820,"kb_used_omap":125,"kb_used_meta":3161666,"kb_avail":548964124,"statfs":{"total":575970213888,"available":562139262976,"internally_reserved":0,"allocated":10593095680,"data_stored":10581266224,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":128662,"internal_metadata":3237546346},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[3,2,3,2,0,0,4,3],"upper_bound":256},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-131989504,"num_objects":-2014,"num_object_clones":0,"num_object_copies":-6042,"num_objects_missing_on_primary":-197,"num_objects_missing":-197,"num_objects_degraded":-3369,"num_objects_misplaced":1279,"num_objects_unfound":0,"num_objects_dirty":-2014,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2014,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":929,"num_bytes_recovered":33226774,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"4.552995"},"pg_stats":[{"pgid":"1.18","version":"79'161","reported_seq":642,"reported_epoch":521,"state":"active+clean","last_fresh":"2023-09-23T21:13:58.464827+0000","last_change":"2023-09-23T21:12:30.438733+0000","last_active":"2023-09-23T21:13:58.464827+0000","last_peered":"2023-09-23T21:13:58.464827+0000","last_clean":"2023-09-23T21:13:58.464827+0000","last_became_active":"2023-09-23T21:12:30.438356+0000","last_became_peered":"2023-09-23T21:12:30.438356+0000","last_unstale":"2023-09-23T21:13:58.464827+0000","last_undegraded":"2023-09-23T21:13:58.464827+0000","last_fullsized":"2023-09-23T21:13:58.464827+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":479,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:06:52.098882+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:06:52.098882+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0092173299999999993,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":131344,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"10.13","version":"524'4895","reported_seq":7291,"reported_epoch":524,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T21:14:03.180531+0000","last_change":"2023-09-23T21:13:49.840403+0000","last_active":"2023-09-23T21:14:03.180531+0000","last_peered":"2023-09-23T21:14:03.180531+0000","last_clean":"2023-09-23T21:11:26.718194+0000","last_became_active":"2023-09-23T21:13:48.181112+0000","last_became_peered":"2023-09-23T21:13:48.181112+0000","last_unstale":"2023-09-23T21:14:03.180531+0000","last_undegraded":"2023-09-23T21:13:48.139049+0000","last_fullsized":"2023-09-23T21:13:45.405990+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":438,"last_epoch_clean":439,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_clean_scrub_stamp":"2023-09-23T21:10:15.663179+0000","objects_scrubbed":0,"log_size":4895,"log_dups_size":0,"ondisk_log_size":4895,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":98729984,"num_objects":1507,"num_object_clones":0,"num_object_copies":4521,"num_objects_missing_on_primary":1137,"num_objects_missing":1137,"num_objects_degraded":1337,"num_objects_misplaced":30,"num_objects_unfound":0,"num_objects_dirty":1507,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3527,"num_write_kb":161056,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1011,"num_bytes_recovered":66191360,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[4,3],"avail_no_missing":["5"],"object_location_counts":[{"shards":"0,2,4,5","objects":307},{"shards":"2,3,5","objects":1137},{"shards":"2,4,5","objects":28},{"shards":"3,4","objects":35}],"blocked_by":[],"up_primary":2,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1a","version":"68'1","reported_seq":114,"reported_epoch":523,"state":"active+clean","last_fresh":"2023-09-23T21:14:01.500016+0000","last_change":"2023-09-23T21:13:13.291403+0000","last_active":"2023-09-23T21:14:01.500016+0000","last_peered":"2023-09-23T21:14:01.500016+0000","last_clean":"2023-09-23T21:14:01.500016+0000","last_became_active":"2023-09-23T21:13:13.291123+0000","last_became_peered":"2023-09-23T21:13:13.291123+0000","last_unstale":"2023-09-23T21:14:01.500016+0000","last_undegraded":"2023-09-23T21:14:01.500016+0000","last_fullsized":"2023-09-23T21:14:01.500016+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":495,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:45:15.427430+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,0],"acting":[4,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":183,"reported_epoch":523,"state":"active+clean","last_fresh":"2023-09-23T21:14:01.499679+0000","last_change":"2023-09-23T21:12:10.767963+0000","last_active":"2023-09-23T21:14:01.499679+0000","last_peered":"2023-09-23T21:14:01.499679+0000","last_clean":"2023-09-23T21:14:01.499679+0000","last_became_active":"2023-09-23T21:12:10.767126+0000","last_became_peered":"2023-09-23T21:12:10.767126+0000","last_unstale":"2023-09-23T21:14:01.499679+0000","last_undegraded":"2023-09-23T21:14:01.499679+0000","last_fullsized":"2023-09-23T21:14:01.499679+0000","mapping_epoch":470,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:45:47.138277+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,5],"acting":[4,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1b","version":"79'161","reported_seq":524,"reported_epoch":520,"state":"active+clean","last_fresh":"2023-09-23T21:13:53.463747+0000","last_change":"2023-09-23T21:13:13.373548+0000","last_active":"2023-09-23T21:13:53.463747+0000","last_peered":"2023-09-23T21:13:53.463747+0000","last_clean":"2023-09-23T21:13:53.463747+0000","last_became_active":"2023-09-23T21:13:13.372845+0000","last_became_peered":"2023-09-23T21:13:13.372845+0000","last_unstale":"2023-09-23T21:13:53.463747+0000","last_undegraded":"2023-09-23T21:13:53.463747+0000","last_fullsized":"2023-09-23T21:13:53.463747+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":479,"last_epoch_clean":495,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:05:55.772264+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:05:55.772264+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.001862507,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.10","version":"524'4873","reported_seq":11364,"reported_epoch":524,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T21:14:03.165245+0000","last_change":"2023-09-23T21:13:36.071975+0000","last_active":"2023-09-23T21:14:03.165245+0000","last_peered":"2023-09-23T21:14:03.165245+0000","last_clean":"2023-09-23T21:11:26.716051+0000","last_became_active":"2023-09-23T21:13:33.619010+0000","last_became_peered":"2023-09-23T21:13:33.619010+0000","last_unstale":"2023-09-23T21:14:03.165245+0000","last_undegraded":"2023-09-23T21:13:33.443511+0000","last_fullsized":"2023-09-23T21:13:31.753181+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":438,"last_epoch_clean":450,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_clean_scrub_stamp":"2023-09-23T21:10:15.663179+0000","objects_scrubbed":0,"log_size":4873,"log_dups_size":0,"ondisk_log_size":4873,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":95584256,"num_objects":1459,"num_object_clones":0,"num_object_copies":4377,"num_objects_missing_on_primary":1115,"num_objects_missing":1115,"num_objects_degraded":1941,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1459,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3509,"num_write_kb":158944,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2902,"num_bytes_recovered":190185472,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7],"avail_no_missing":["1","5","7"],"object_location_counts":[{"shards":"0,4,5,7","objects":6},{"shards":"0,5,7","objects":1115},{"shards":"4,5,7","objects":29},{"shards":"4,7","objects":309}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":520,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-09-23T21:14:02.776046+0000","last_change":"2023-09-23T21:12:10.824116+0000","last_active":"2023-09-23T21:14:02.776046+0000","last_peered":"2023-09-23T21:14:02.776046+0000","last_clean":"2023-09-23T21:14:02.776046+0000","last_became_active":"2023-09-23T21:12:10.823849+0000","last_became_peered":"2023-09-23T21:12:10.823849+0000","last_unstale":"2023-09-23T21:14:02.776046+0000","last_undegraded":"2023-09-23T21:14:02.776046+0000","last_fullsized":"2023-09-23T21:14:02.776046+0000","mapping_epoch":470,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:54:13.777603+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":517,"reported_epoch":520,"state":"active+clean","last_fresh":"2023-09-23T21:13:53.465141+0000","last_change":"2023-09-23T21:13:44.413713+0000","last_active":"2023-09-23T21:13:53.465141+0000","last_peered":"2023-09-23T21:13:53.465141+0000","last_clean":"2023-09-23T21:13:53.465141+0000","last_became_active":"2023-09-23T21:13:44.413366+0000","last_became_peered":"2023-09-23T21:13:44.413366+0000","last_unstale":"2023-09-23T21:13:53.465141+0000","last_undegraded":"2023-09-23T21:13:53.465141+0000","last_fullsized":"2023-09-23T21:13:53.465141+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:31:56.349530+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1a","version":"79'161","reported_seq":667,"reported_epoch":521,"state":"active+clean","last_fresh":"2023-09-23T21:13:58.465632+0000","last_change":"2023-09-23T21:12:30.444665+0000","last_active":"2023-09-23T21:13:58.465632+0000","last_peered":"2023-09-23T21:13:58.465632+0000","last_clean":"2023-09-23T21:13:58.465632+0000","last_became_active":"2023-09-23T21:12:30.444326+0000","last_became_peered":"2023-09-23T21:12:30.444326+0000","last_unstale":"2023-09-23T21:13:58.465632+0000","last_undegraded":"2023-09-23T21:13:58.465632+0000","last_fullsized":"2023-09-23T21:13:58.465632+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":479,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:06:51.075448+0000","last_deep_scrub":"79'161","last_deep_scrub_stamp":"2023-09-23T21:05:32.630292+0000","last_clean_scrub_stamp":"2023-09-23T21:06:51.075448+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0054793890000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,4],"acting":[2,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"10.11","version":"522'4815","reported_seq":11639,"reported_epoch":522,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-23T21:14:00.806634+0000","last_change":"2023-09-23T21:13:31.864763+0000","last_active":"2023-09-23T21:14:00.806634+0000","last_peered":"2023-09-23T21:14:00.806634+0000","last_clean":"2023-09-23T21:13:30.058478+0000","last_became_active":"2023-09-23T21:13:31.850860+0000","last_became_peered":"2023-09-23T21:13:31.850860+0000","last_unstale":"2023-09-23T21:14:00.806634+0000","last_undegraded":"2023-09-23T21:14:00.806634+0000","last_fullsized":"2023-09-23T21:14:00.806634+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":438,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":5,"last_scrub":"441'2974","last_scrub_stamp":"2023-09-23T21:11:06.133218+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_clean_scrub_stamp":"2023-09-23T21:11:06.133218+0000","objects_scrubbed":1646,"log_size":4815,"log_dups_size":0,"ondisk_log_size":4815,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:54:59.366711+0000","scrub_duration":0.3173955,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":95420416,"num_objects":1456,"num_object_clones":0,"num_object_copies":4368,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1456,"num_objects_unfound":0,"num_objects_dirty":1456,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1844,"num_write_kb":52928,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2152,"num_bytes_recovered":141033472,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[2,5,7],"avail_no_missing":["2","1","5","7"],"object_location_counts":[{"shards":"2,5,7","objects":1456}],"blocked_by":[],"up_primary":0,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":30,"reported_epoch":520,"state":"active+clean","last_fresh":"2023-09-23T21:13:53.464239+0000","last_change":"2023-09-23T21:13:30.806084+0000","last_active":"2023-09-23T21:13:53.464239+0000","last_peered":"2023-09-23T21:13:53.464239+0000","last_clean":"2023-09-23T21:13:53.464239+0000","last_became_active":"2023-09-23T21:13:30.805743+0000","last_became_peered":"2023-09-23T21:13:30.805743+0000","last_unstale":"2023-09-23T21:13:53.464239+0000","last_undegraded":"2023-09-23T21:13:53.464239+0000","last_fullsized":"2023-09-23T21:13:53.464239+0000","mapping_epoch":501,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:48:10.849281+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":394,"reported_epoch":520,"state":"active+clean","last_fresh":"2023-09-23T21:13:53.464607+0000","last_change":"2023-09-23T21:13:44.410921+0000","last_active":"2023-09-23T21:13:53.464607+0000","last_peered":"2023-09-23T21:13:53.464607+0000","last_clean":"2023-09-23T21:13:53.464607+0000","last_became_active":"2023-09-23T21:13:44.410633+0000","last_became_peered":"2023-09-23T21:13:44.410633+0000","last_unstale":"2023-09-23T21:13:53.464607+0000","last_undegraded":"2023-09-23T21:13:53.464607+0000","last_fullsized":"2023-09-23T21:13:53.464607+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:57:10.116786+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.15","version":"79'161","reported_seq":550,"reported_epoch":523,"state":"active+clean","last_fresh":"2023-09-23T21:14:01.499224+0000","last_change":"2023-09-23T21:13:38.379453+0000","last_active":"2023-09-23T21:14:01.499224+0000","last_peered":"2023-09-23T21:14:01.499224+0000","last_clean":"2023-09-23T21:14:01.499224+0000","last_became_active":"2023-09-23T21:13:38.376499+0000","last_became_peered":"2023-09-23T21:13:38.376499+0000","last_unstale":"2023-09-23T21:14:01.499224+0000","last_undegraded":"2023-09-23T21:14:01.499224+0000","last_fullsized":"2023-09-23T21:14:01.499224+0000","mapping_epoch":507,"log_start":"0'0","ondisk_log_start":"0'0","created":479,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:09:34.892947+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:09:34.892947+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13418,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":11,"num_read_kb":21,"num_write":6,"num_write_kb":55,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":40253,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":495,"reported_epoch":520,"state":"active+clean","last_fresh":"2023-09-23T21:13:53.464698+0000","last_change":"2023-09-23T21:13:12.235728+0000","last_active":"2023-09-23T21:13:53.464698+0000","last_peered":"2023-09-23T21:13:53.464698+0000","last_clean":"2023-09-23T21:13:53.464698+0000","last_became_active":"2023-09-23T21:13:12.216549+0000","last_became_peered":"2023-09-23T21:13:12.216549+0000","last_unstale":"2023-09-23T21:13:53.464698+0000","last_undegraded":"2023-09-23T21:13:53.464698+0000","last_fullsized":"2023-09-23T21:13:53.464698+0000","mapping_epoch":493,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:39:33.324239+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":562,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-09-23T21:14:02.775595+0000","last_change":"2023-09-23T21:12:10.790349+0000","last_active":"2023-09-23T21:14:02.775595+0000","last_peered":"2023-09-23T21:14:02.775595+0000","last_clean":"2023-09-23T21:14:02.775595+0000","last_became_active":"2023-09-23T21:12:10.789977+0000","last_became_peered":"2023-09-23T21:12:10.789977+0000","last_unstale":"2023-09-23T21:14:02.775595+0000","last_undegraded":"2023-09-23T21:14:02.775595+0000","last_fullsized":"2023-09-23T21:14:02.775595+0000","mapping_epoch":470,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:49:40.191147+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,5],"acting":[3,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"79'161","reported_seq":883,"reported_epoch":521,"state":"active+clean","last_fresh":"2023-09-23T21:13:58.465737+0000","last_change":"2023-09-23T21:11:32.760292+0000","last_active":"2023-09-23T21:13:58.465737+0000","last_peered":"2023-09-23T21:13:58.465737+0000","last_clean":"2023-09-23T21:13:58.465737+0000","last_became_active":"2023-09-23T21:11:32.759909+0000","last_became_peered":"2023-09-23T21:11:32.759909+0000","last_unstale":"2023-09-23T21:13:58.465737+0000","last_undegraded":"2023-09-23T21:13:58.465737+0000","last_fullsized":"2023-09-23T21:13:58.465737+0000","mapping_epoch":457,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":458,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:25.096401+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:25.096401+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:13:09.225685+0000","scrub_duration":0.002859393,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":247,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-09-23T21:14:00.508890+0000","last_change":"2023-09-23T21:12:10.779468+0000","last_active":"2023-09-23T21:14:00.508890+0000","last_peered":"2023-09-23T21:14:00.508890+0000","last_clean":"2023-09-23T21:14:00.508890+0000","last_became_active":"2023-09-23T21:12:10.779282+0000","last_became_peered":"2023-09-23T21:12:10.779282+0000","last_unstale":"2023-09-23T21:14:00.508890+0000","last_undegraded":"2023-09-23T21:14:00.508890+0000","last_fullsized":"2023-09-23T21:14:00.508890+0000","mapping_epoch":470,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:36:35.417986+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":499,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-09-23T21:14:02.774912+0000","last_change":"2023-09-23T21:10:26.758958+0000","last_active":"2023-09-23T21:14:02.774912+0000","last_peered":"2023-09-23T21:14:02.774912+0000","last_clean":"2023-09-23T21:14:02.774912+0000","last_became_active":"2023-09-23T21:10:26.758539+0000","last_became_peered":"2023-09-23T21:10:26.758539+0000","last_unstale":"2023-09-23T21:14:02.774912+0000","last_undegraded":"2023-09-23T21:14:02.774912+0000","last_fullsized":"2023-09-23T21:14:02.774912+0000","mapping_epoch":423,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":424,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:44:42.319189+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.17","version":"79'161","reported_seq":617,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-09-23T21:14:02.782314+0000","last_change":"2023-09-23T21:13:44.416768+0000","last_active":"2023-09-23T21:14:02.782314+0000","last_peered":"2023-09-23T21:14:02.782314+0000","last_clean":"2023-09-23T21:14:02.782314+0000","last_became_active":"2023-09-23T21:13:44.416156+0000","last_became_peered":"2023-09-23T21:13:44.416156+0000","last_unstale":"2023-09-23T21:14:02.782314+0000","last_undegraded":"2023-09-23T21:14:02.782314+0000","last_fullsized":"2023-09-23T21:14:02.782314+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":479,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:25.916867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:25.916867+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.001665323,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":409,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-09-23T21:14:02.781698+0000","last_change":"2023-09-23T21:11:08.766045+0000","last_active":"2023-09-23T21:14:02.781698+0000","last_peered":"2023-09-23T21:14:02.781698+0000","last_clean":"2023-09-23T21:14:02.781698+0000","last_became_active":"2023-09-23T21:11:08.758504+0000","last_became_peered":"2023-09-23T21:11:08.758504+0000","last_unstale":"2023-09-23T21:14:02.781698+0000","last_undegraded":"2023-09-23T21:14:02.781698+0000","last_fullsized":"2023-09-23T21:14:02.781698+0000","mapping_epoch":443,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":444,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:02:19.482574+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":360,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,4],"acting":[3,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":208,"reported_epoch":521,"state":"active+clean","last_fresh":"2023-09-23T21:13:58.467280+0000","last_change":"2023-09-23T21:13:30.821489+0000","last_active":"2023-09-23T21:13:58.467280+0000","last_peered":"2023-09-23T21:13:58.467280+0000","last_clean":"2023-09-23T21:13:58.467280+0000","last_became_active":"2023-09-23T21:13:30.821125+0000","last_became_peered":"2023-09-23T21:13:30.821125+0000","last_unstale":"2023-09-23T21:13:58.467280+0000","last_undegraded":"2023-09-23T21:13:58.467280+0000","last_fullsized":"2023-09-23T21:13:58.467280+0000","mapping_epoch":501,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:47:18.553417+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,5],"acting":[2,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.16","version":"79'161","reported_seq":439,"reported_epoch":520,"state":"active+clean","last_fresh":"2023-09-23T21:13:53.465069+0000","last_change":"2023-09-23T21:13:33.770962+0000","last_active":"2023-09-23T21:13:53.465069+0000","last_peered":"2023-09-23T21:13:53.465069+0000","last_clean":"2023-09-23T21:13:53.465069+0000","last_became_active":"2023-09-23T21:13:33.770407+0000","last_became_peered":"2023-09-23T21:13:33.770407+0000","last_unstale":"2023-09-23T21:13:53.465069+0000","last_undegraded":"2023-09-23T21:13:53.465069+0000","last_fullsized":"2023-09-23T21:13:53.465069+0000","mapping_epoch":504,"log_start":"0'0","ondisk_log_start":"0'0","created":479,"last_epoch_clean":505,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:10:24.268662+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:10:24.268662+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13418,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":11,"num_read_kb":21,"num_write":6,"num_write_kb":55,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":40252,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.14","version":"79'22","reported_seq":87,"reported_epoch":523,"state":"active+clean","last_fresh":"2023-09-23T21:14:01.499029+0000","last_change":"2023-09-23T21:11:27.770162+0000","last_active":"2023-09-23T21:14:01.499029+0000","last_peered":"2023-09-23T21:14:01.499029+0000","last_clean":"2023-09-23T21:14:01.499029+0000","last_became_active":"2023-09-23T21:11:27.752275+0000","last_became_peered":"2023-09-23T21:11:27.752275+0000","last_unstale":"2023-09-23T21:14:01.499029+0000","last_undegraded":"2023-09-23T21:14:01.499029+0000","last_fullsized":"2023-09-23T21:14:01.499029+0000","mapping_epoch":452,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":453,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:05:11.769428+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4734,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,2],"acting":[4,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":452,"reported_epoch":523,"state":"active+clean","last_fresh":"2023-09-23T21:14:01.516604+0000","last_change":"2023-09-23T21:12:10.839271+0000","last_active":"2023-09-23T21:14:01.516604+0000","last_peered":"2023-09-23T21:14:01.516604+0000","last_clean":"2023-09-23T21:14:01.516604+0000","last_became_active":"2023-09-23T21:12:10.839042+0000","last_became_peered":"2023-09-23T21:12:10.839042+0000","last_unstale":"2023-09-23T21:14:01.516604+0000","last_undegraded":"2023-09-23T21:14:01.516604+0000","last_fullsized":"2023-09-23T21:14:01.516604+0000","mapping_epoch":470,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:10:26.414737+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,2],"acting":[4,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.11","version":"79'161","reported_seq":444,"reported_epoch":520,"state":"active+clean","last_fresh":"2023-09-23T21:13:53.463521+0000","last_change":"2023-09-23T21:13:33.767406+0000","last_active":"2023-09-23T21:13:53.463521+0000","last_peered":"2023-09-23T21:13:53.463521+0000","last_clean":"2023-09-23T21:13:53.463521+0000","last_became_active":"2023-09-23T21:13:33.767067+0000","last_became_peered":"2023-09-23T21:13:33.767067+0000","last_unstale":"2023-09-23T21:13:53.463521+0000","last_undegraded":"2023-09-23T21:13:53.463521+0000","last_fullsized":"2023-09-23T21:13:53.463521+0000","mapping_epoch":504,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":505,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:28.756634+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:28.756634+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":23,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":131,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-09-23T21:14:00.483797+0000","last_change":"2023-09-23T21:11:27.777154+0000","last_active":"2023-09-23T21:14:00.483797+0000","last_peered":"2023-09-23T21:14:00.483797+0000","last_clean":"2023-09-23T21:14:00.483797+0000","last_became_active":"2023-09-23T21:11:27.754438+0000","last_became_peered":"2023-09-23T21:11:27.754438+0000","last_unstale":"2023-09-23T21:14:00.483797+0000","last_undegraded":"2023-09-23T21:14:00.483797+0000","last_fullsized":"2023-09-23T21:14:00.483797+0000","mapping_epoch":452,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":453,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:33:02.845882+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":62,"reported_epoch":521,"state":"active+clean","last_fresh":"2023-09-23T21:13:58.465433+0000","last_change":"2023-09-23T21:12:10.764264+0000","last_active":"2023-09-23T21:13:58.465433+0000","last_peered":"2023-09-23T21:13:58.465433+0000","last_clean":"2023-09-23T21:13:58.465433+0000","last_became_active":"2023-09-23T21:12:10.763486+0000","last_became_peered":"2023-09-23T21:12:10.763486+0000","last_unstale":"2023-09-23T21:13:58.465433+0000","last_undegraded":"2023-09-23T21:13:58.465433+0000","last_fullsized":"2023-09-23T21:13:58.465433+0000","mapping_epoch":470,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:32:27.577036+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,7],"acting":[2,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.10","version":"79'161","reported_seq":658,"reported_epoch":523,"state":"active+clean","last_fresh":"2023-09-23T21:14:01.503850+0000","last_change":"2023-09-23T21:11:11.716703+0000","last_active":"2023-09-23T21:14:01.503850+0000","last_peered":"2023-09-23T21:14:01.503850+0000","last_clean":"2023-09-23T21:14:01.503850+0000","last_became_active":"2023-09-23T21:11:11.716385+0000","last_became_peered":"2023-09-23T21:11:11.716385+0000","last_unstale":"2023-09-23T21:14:01.503850+0000","last_undegraded":"2023-09-23T21:14:01.503850+0000","last_fullsized":"2023-09-23T21:14:01.503850+0000","mapping_epoch":446,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":447,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:06:50.336992+0000","last_deep_scrub":"79'161","last_deep_scrub_stamp":"2023-09-23T21:06:50.336992+0000","last_clean_scrub_stamp":"2023-09-23T21:06:50.336992+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:44:29.538890+0000","scrub_duration":0.0026150930000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,7],"acting":[4,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":461,"reported_epoch":520,"state":"active+clean","last_fresh":"2023-09-23T21:13:53.464133+0000","last_change":"2023-09-23T21:13:12.270990+0000","last_active":"2023-09-23T21:13:53.464133+0000","last_peered":"2023-09-23T21:13:53.464133+0000","last_clean":"2023-09-23T21:13:53.464133+0000","last_became_active":"2023-09-23T21:13:12.237201+0000","last_became_peered":"2023-09-23T21:13:12.237201+0000","last_unstale":"2023-09-23T21:13:53.464133+0000","last_undegraded":"2023-09-23T21:13:53.464133+0000","last_fullsized":"2023-09-23T21:13:53.464133+0000","mapping_epoch":493,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:59:54.523877+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":276,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-09-23T21:14:02.781515+0000","last_change":"2023-09-23T21:10:26.758671+0000","last_active":"2023-09-23T21:14:02.781515+0000","last_peered":"2023-09-23T21:14:02.781515+0000","last_clean":"2023-09-23T21:14:02.781515+0000","last_became_active":"2023-09-23T21:10:26.758244+0000","last_became_peered":"2023-09-23T21:10:26.758244+0000","last_unstale":"2023-09-23T21:14:02.781515+0000","last_undegraded":"2023-09-23T21:14:02.781515+0000","last_fullsized":"2023-09-23T21:14:02.781515+0000","mapping_epoch":423,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":424,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:14:02.735609+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.18","version":"522'4858","reported_seq":5936,"reported_epoch":522,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T21:14:01.212273+0000","last_change":"2023-09-23T21:13:13.412097+0000","last_active":"2023-09-23T21:14:01.212273+0000","last_peered":"2023-09-23T21:14:01.212273+0000","last_clean":"2023-09-23T21:11:45.720015+0000","last_became_active":"2023-09-23T21:13:13.395632+0000","last_became_peered":"2023-09-23T21:13:13.395632+0000","last_unstale":"2023-09-23T21:14:01.212273+0000","last_undegraded":"2023-09-23T21:13:13.220971+0000","last_fullsized":"2023-09-23T21:13:13.183825+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":438,"last_epoch_clean":439,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_clean_scrub_stamp":"2023-09-23T21:10:15.663179+0000","objects_scrubbed":0,"log_size":4858,"log_dups_size":0,"ondisk_log_size":4858,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":100302848,"num_objects":1531,"num_object_clones":0,"num_object_copies":4593,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":661,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1531,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3509,"num_write_kb":161248,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":231,"num_bytes_recovered":15138816,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,7],"acting":[2,7],"avail_no_missing":["2","7"],"object_location_counts":[{"shards":"2,7","objects":1497}],"blocked_by":[],"up_primary":0,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.13","version":"79'161","reported_seq":601,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-09-23T21:14:02.776185+0000","last_change":"2023-09-23T21:13:14.933096+0000","last_active":"2023-09-23T21:14:02.776185+0000","last_peered":"2023-09-23T21:14:02.776185+0000","last_clean":"2023-09-23T21:14:02.776185+0000","last_became_active":"2023-09-23T21:13:14.932703+0000","last_became_peered":"2023-09-23T21:13:14.932703+0000","last_unstale":"2023-09-23T21:14:02.776185+0000","last_undegraded":"2023-09-23T21:14:02.776185+0000","last_fullsized":"2023-09-23T21:14:02.776185+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:36.420362+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:36.420362+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:29:42.163911+0000","scrub_duration":0.028838444000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":515,"reported_epoch":523,"state":"active+clean","last_fresh":"2023-09-23T21:14:01.498105+0000","last_change":"2023-09-23T21:13:12.216467+0000","last_active":"2023-09-23T21:14:01.498105+0000","last_peered":"2023-09-23T21:14:01.498105+0000","last_clean":"2023-09-23T21:14:01.498105+0000","last_became_active":"2023-09-23T21:13:12.215259+0000","last_became_peered":"2023-09-23T21:13:12.215259+0000","last_unstale":"2023-09-23T21:14:01.498105+0000","last_undegraded":"2023-09-23T21:14:01.498105+0000","last_fullsized":"2023-09-23T21:14:01.498105+0000","mapping_epoch":493,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:35:33.987839+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":30,"reported_epoch":520,"state":"active+clean","last_fresh":"2023-09-23T21:13:53.464833+0000","last_change":"2023-09-23T21:13:30.804108+0000","last_active":"2023-09-23T21:13:53.464833+0000","last_peered":"2023-09-23T21:13:53.464833+0000","last_clean":"2023-09-23T21:13:53.464833+0000","last_became_active":"2023-09-23T21:13:30.796505+0000","last_became_peered":"2023-09-23T21:13:30.796505+0000","last_unstale":"2023-09-23T21:13:53.464833+0000","last_undegraded":"2023-09-23T21:13:53.464833+0000","last_fullsized":"2023-09-23T21:13:53.464833+0000","mapping_epoch":501,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:19:33.508496+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.19","version":"522'4992","reported_seq":6085,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-09-23T21:14:01.248656+0000","last_change":"2023-09-23T21:10:58.790716+0000","last_active":"2023-09-23T21:14:01.248656+0000","last_peered":"2023-09-23T21:14:01.248656+0000","last_clean":"2023-09-23T21:14:01.248656+0000","last_became_active":"2023-09-23T21:10:58.787975+0000","last_became_peered":"2023-09-23T21:10:58.787975+0000","last_unstale":"2023-09-23T21:14:01.248656+0000","last_undegraded":"2023-09-23T21:14:01.248656+0000","last_fullsized":"2023-09-23T21:14:01.248656+0000","mapping_epoch":438,"log_start":"0'0","ondisk_log_start":"0'0","created":438,"last_epoch_clean":439,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_clean_scrub_stamp":"2023-09-23T21:10:15.663179+0000","objects_scrubbed":0,"log_size":4992,"log_dups_size":0,"ondisk_log_size":4992,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":97419264,"num_objects":1487,"num_object_clones":0,"num_object_copies":4461,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1487,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3605,"num_write_kb":162912,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":237,"num_bytes_recovered":15499264,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.12","version":"79'161","reported_seq":756,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-09-23T21:14:00.508896+0000","last_change":"2023-09-23T21:13:15.206574+0000","last_active":"2023-09-23T21:14:00.508896+0000","last_peered":"2023-09-23T21:14:00.508896+0000","last_clean":"2023-09-23T21:14:00.508896+0000","last_became_active":"2023-09-23T21:13:15.206186+0000","last_became_peered":"2023-09-23T21:13:15.206186+0000","last_unstale":"2023-09-23T21:14:00.508896+0000","last_undegraded":"2023-09-23T21:14:00.508896+0000","last_fullsized":"2023-09-23T21:14:00.508896+0000","mapping_epoch":496,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":497,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:05:56.785173+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:05:56.785173+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:34:29.221809+0000","scrub_duration":0.0042114370000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,0],"acting":[5,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":40,"reported_epoch":520,"state":"active+clean","last_fresh":"2023-09-23T21:13:53.464434+0000","last_change":"2023-09-23T21:13:38.048791+0000","last_active":"2023-09-23T21:13:53.464434+0000","last_peered":"2023-09-23T21:13:53.464434+0000","last_clean":"2023-09-23T21:13:53.464434+0000","last_became_active":"2023-09-23T21:13:30.801386+0000","last_became_peered":"2023-09-23T21:13:30.801386+0000","last_unstale":"2023-09-23T21:13:53.464434+0000","last_undegraded":"2023-09-23T21:13:53.464434+0000","last_fullsized":"2023-09-23T21:13:53.464434+0000","mapping_epoch":501,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:52:49.170414+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":361,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-09-23T21:14:02.778671+0000","last_change":"2023-09-23T21:12:10.765780+0000","last_active":"2023-09-23T21:14:02.778671+0000","last_peered":"2023-09-23T21:14:02.778671+0000","last_clean":"2023-09-23T21:14:02.778671+0000","last_became_active":"2023-09-23T21:12:10.765372+0000","last_became_peered":"2023-09-23T21:12:10.765372+0000","last_unstale":"2023-09-23T21:14:02.778671+0000","last_undegraded":"2023-09-23T21:14:02.778671+0000","last_fullsized":"2023-09-23T21:14:02.778671+0000","mapping_epoch":470,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:56:08.446099+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.6","version":"521'4953","reported_seq":7863,"reported_epoch":521,"state":"active+clean","last_fresh":"2023-09-23T21:14:00.026668+0000","last_change":"2023-09-23T21:13:39.390285+0000","last_active":"2023-09-23T21:14:00.026668+0000","last_peered":"2023-09-23T21:14:00.026668+0000","last_clean":"2023-09-23T21:14:00.026668+0000","last_became_active":"2023-09-23T21:13:39.378284+0000","last_became_peered":"2023-09-23T21:13:39.378284+0000","last_unstale":"2023-09-23T21:14:00.026668+0000","last_undegraded":"2023-09-23T21:14:00.026668+0000","last_fullsized":"2023-09-23T21:14:00.026668+0000","mapping_epoch":508,"log_start":"0'0","ondisk_log_start":"0'0","created":416,"last_epoch_clean":509,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_clean_scrub_stamp":"2023-09-23T21:10:15.663179+0000","objects_scrubbed":0,"log_size":4953,"log_dups_size":0,"ondisk_log_size":4953,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":99549184,"num_objects":1519,"num_object_clones":0,"num_object_copies":4557,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1519,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3547,"num_write_kb":162112,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2399,"num_bytes_recovered":113967104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,4],"acting":[0,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.d","version":"79'161","reported_seq":676,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-09-23T21:14:00.483837+0000","last_change":"2023-09-23T21:12:30.444461+0000","last_active":"2023-09-23T21:14:00.483837+0000","last_peered":"2023-09-23T21:14:00.483837+0000","last_clean":"2023-09-23T21:14:00.483837+0000","last_became_active":"2023-09-23T21:12:30.443979+0000","last_became_peered":"2023-09-23T21:12:30.443979+0000","last_unstale":"2023-09-23T21:14:00.483837+0000","last_undegraded":"2023-09-23T21:14:00.483837+0000","last_fullsized":"2023-09-23T21:14:00.483837+0000","mapping_epoch":470,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:09:36.858077+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:09:36.858077+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0024987500000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":169,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-09-23T21:14:00.484621+0000","last_change":"2023-09-23T21:13:59.872834+0000","last_active":"2023-09-23T21:14:00.484621+0000","last_peered":"2023-09-23T21:14:00.484621+0000","last_clean":"2023-09-23T21:14:00.484621+0000","last_became_active":"2023-09-23T21:13:30.770556+0000","last_became_peered":"2023-09-23T21:13:30.770556+0000","last_unstale":"2023-09-23T21:14:00.484621+0000","last_undegraded":"2023-09-23T21:14:00.484621+0000","last_fullsized":"2023-09-23T21:14:00.484621+0000","mapping_epoch":501,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":521,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:57:55.460052+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":132,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":230,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-09-23T21:14:00.508262+0000","last_change":"2023-09-23T21:10:26.750725+0000","last_active":"2023-09-23T21:14:00.508262+0000","last_peered":"2023-09-23T21:14:00.508262+0000","last_clean":"2023-09-23T21:14:00.508262+0000","last_became_active":"2023-09-23T21:10:26.750307+0000","last_became_peered":"2023-09-23T21:10:26.750307+0000","last_unstale":"2023-09-23T21:14:00.508262+0000","last_undegraded":"2023-09-23T21:14:00.508262+0000","last_fullsized":"2023-09-23T21:14:00.508262+0000","mapping_epoch":423,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":424,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:06:22.040548+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,4],"acting":[5,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.7","version":"525'4959","reported_seq":5151,"reported_epoch":525,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T21:14:04.172115+0000","last_change":"2023-09-23T21:13:38.417178+0000","last_active":"2023-09-23T21:14:04.172115+0000","last_peered":"2023-09-23T21:14:04.172115+0000","last_clean":"2023-09-23T21:11:26.699871+0000","last_became_active":"2023-09-23T21:13:38.408523+0000","last_became_peered":"2023-09-23T21:13:38.408523+0000","last_unstale":"2023-09-23T21:14:04.172115+0000","last_undegraded":"2023-09-23T21:13:38.369833+0000","last_fullsized":"2023-09-23T21:13:38.358198+0000","mapping_epoch":507,"log_start":"0'0","ondisk_log_start":"0'0","created":416,"last_epoch_clean":439,"parent":"0.0","parent_split_bits":0,"last_scrub":"441'2973","last_scrub_stamp":"2023-09-23T21:11:06.079326+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_clean_scrub_stamp":"2023-09-23T21:11:06.079326+0000","objects_scrubbed":1647,"log_size":4959,"log_dups_size":0,"ondisk_log_size":4959,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:27:59.805526+0000","scrub_duration":0.44319714500000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90701824,"num_objects":1384,"num_object_clones":0,"num_object_copies":4152,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1165,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1384,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1991,"num_write_kb":55296,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,2],"acting":[3,5],"avail_no_missing":["3","5"],"object_location_counts":[{"shards":"3,4,5","objects":298},{"shards":"3,5","objects":1086}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.c","version":"79'161","reported_seq":887,"reported_epoch":524,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-23T21:14:02.776776+0000","last_change":"2023-09-23T21:13:13.402530+0000","last_active":"2023-09-23T21:14:02.776776+0000","last_peered":"2023-09-23T21:14:02.776776+0000","last_clean":"2023-09-23T21:11:23.200766+0000","last_became_active":"2023-09-23T21:13:13.401526+0000","last_became_peered":"2023-09-23T21:13:13.401526+0000","last_unstale":"2023-09-23T21:14:02.776776+0000","last_undegraded":"2023-09-23T21:13:13.204992+0000","last_fullsized":"2023-09-23T21:14:02.776776+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":420,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:22.040907+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:22.040907+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.013021176000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":163840,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":163840,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":["3","0","5"],"object_location_counts":[{"shards":"0,3,4","objects":-1},{"shards":"0,3,5","objects":1}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":155,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-09-23T21:14:00.485683+0000","last_change":"2023-09-23T21:13:59.884113+0000","last_active":"2023-09-23T21:14:00.485683+0000","last_peered":"2023-09-23T21:14:00.485683+0000","last_clean":"2023-09-23T21:14:00.485683+0000","last_became_active":"2023-09-23T21:13:30.806778+0000","last_became_peered":"2023-09-23T21:13:30.806778+0000","last_unstale":"2023-09-23T21:14:00.485683+0000","last_undegraded":"2023-09-23T21:14:00.485683+0000","last_fullsized":"2023-09-23T21:14:00.485683+0000","mapping_epoch":501,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":521,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:36:15.284465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":486,"reported_epoch":523,"state":"active+clean","last_fresh":"2023-09-23T21:14:01.498196+0000","last_change":"2023-09-23T21:13:15.152792+0000","last_active":"2023-09-23T21:14:01.498196+0000","last_peered":"2023-09-23T21:14:01.498196+0000","last_clean":"2023-09-23T21:14:01.498196+0000","last_became_active":"2023-09-23T21:13:14.515084+0000","last_became_peered":"2023-09-23T21:13:14.515084+0000","last_unstale":"2023-09-23T21:14:01.498196+0000","last_undegraded":"2023-09-23T21:14:01.498196+0000","last_fullsized":"2023-09-23T21:14:01.498196+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:08:16.702072+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.4","version":"525'4823","reported_seq":11764,"reported_epoch":525,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T21:14:04.178423+0000","last_change":"2023-09-23T21:13:30.847170+0000","last_active":"2023-09-23T21:14:04.178423+0000","last_peered":"2023-09-23T21:14:04.178423+0000","last_clean":"2023-09-23T21:11:06.743842+0000","last_became_active":"2023-09-23T21:13:30.834806+0000","last_became_peered":"2023-09-23T21:13:30.834806+0000","last_unstale":"2023-09-23T21:14:04.178423+0000","last_undegraded":"2023-09-23T21:13:30.760230+0000","last_fullsized":"2023-09-23T21:13:30.749974+0000","mapping_epoch":501,"log_start":"0'0","ondisk_log_start":"0'0","created":416,"last_epoch_clean":439,"parent":"0.0","parent_split_bits":0,"last_scrub":"440'2861","last_scrub_stamp":"2023-09-23T21:11:03.842926+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_clean_scrub_stamp":"2023-09-23T21:11:03.842926+0000","objects_scrubbed":1526,"log_size":4823,"log_dups_size":0,"ondisk_log_size":4823,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:32:29.379756+0000","scrub_duration":0.39378383900000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":98172928,"num_objects":1498,"num_object_clones":0,"num_object_copies":4494,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1130,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1498,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1968,"num_write_kb":62080,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2723,"num_bytes_recovered":163577856,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[3,4],"avail_no_missing":["3","4"],"object_location_counts":[{"shards":"1,2,3,4","objects":360},{"shards":"3,4","objects":1138}],"blocked_by":[],"up_primary":0,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.f","version":"79'161","reported_seq":810,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-09-23T21:14:00.509349+0000","last_change":"2023-09-23T21:11:30.750338+0000","last_active":"2023-09-23T21:14:00.509349+0000","last_peered":"2023-09-23T21:14:00.509349+0000","last_clean":"2023-09-23T21:14:00.509349+0000","last_became_active":"2023-09-23T21:11:30.749953+0000","last_became_peered":"2023-09-23T21:11:30.749953+0000","last_unstale":"2023-09-23T21:14:00.509349+0000","last_undegraded":"2023-09-23T21:14:00.509349+0000","last_fullsized":"2023-09-23T21:14:00.509349+0000","mapping_epoch":455,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":456,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:27.880776+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:27.880776+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:07:10.405723+0000","scrub_duration":0.0014004899999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,3],"acting":[5,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":246,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-09-23T21:14:00.508918+0000","last_change":"2023-09-23T21:10:26.765068+0000","last_active":"2023-09-23T21:14:00.508918+0000","last_peered":"2023-09-23T21:14:00.508918+0000","last_clean":"2023-09-23T21:14:00.508918+0000","last_became_active":"2023-09-23T21:10:26.758364+0000","last_became_peered":"2023-09-23T21:10:26.758364+0000","last_unstale":"2023-09-23T21:14:00.508918+0000","last_undegraded":"2023-09-23T21:14:00.508918+0000","last_fullsized":"2023-09-23T21:14:00.508918+0000","mapping_epoch":423,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":424,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:13:21.057529+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":280,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":462,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-09-23T21:14:00.508588+0000","last_change":"2023-09-23T21:13:30.770746+0000","last_active":"2023-09-23T21:14:00.508588+0000","last_peered":"2023-09-23T21:14:00.508588+0000","last_clean":"2023-09-23T21:14:00.508588+0000","last_became_active":"2023-09-23T21:13:30.770437+0000","last_became_peered":"2023-09-23T21:13:30.770437+0000","last_unstale":"2023-09-23T21:14:00.508588+0000","last_undegraded":"2023-09-23T21:14:00.508588+0000","last_fullsized":"2023-09-23T21:14:00.508588+0000","mapping_epoch":501,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:33:48.325730+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,0],"acting":[5,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.5","version":"522'4829","reported_seq":5009,"reported_epoch":522,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T21:14:01.183625+0000","last_change":"2023-09-23T21:13:13.436794+0000","last_active":"2023-09-23T21:14:01.183625+0000","last_peered":"2023-09-23T21:14:01.183625+0000","last_clean":"2023-09-23T21:11:45.768408+0000","last_became_active":"2023-09-23T21:13:13.424939+0000","last_became_peered":"2023-09-23T21:13:13.424939+0000","last_unstale":"2023-09-23T21:14:01.183625+0000","last_undegraded":"2023-09-23T21:13:13.225067+0000","last_fullsized":"2023-09-23T21:13:13.198194+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":416,"last_epoch_clean":439,"parent":"0.0","parent_split_bits":0,"last_scrub":"440'2840","last_scrub_stamp":"2023-09-23T21:11:03.208833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_clean_scrub_stamp":"2023-09-23T21:11:03.208833+0000","objects_scrubbed":1498,"log_size":4829,"log_dups_size":0,"ondisk_log_size":4829,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:56:40.574477+0000","scrub_duration":0.45608471699999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":100532224,"num_objects":1534,"num_object_clones":0,"num_object_copies":4602,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":644,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1534,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1998,"num_write_kb":65088,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,3],"acting":[2,3],"avail_no_missing":["2","3"],"object_location_counts":[{"shards":"2,3","objects":1488}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.e","version":"79'161","reported_seq":692,"reported_epoch":523,"state":"active+clean","last_fresh":"2023-09-23T21:14:01.499520+0000","last_change":"2023-09-23T21:12:30.440952+0000","last_active":"2023-09-23T21:14:01.499520+0000","last_peered":"2023-09-23T21:14:01.499520+0000","last_clean":"2023-09-23T21:14:01.499520+0000","last_became_active":"2023-09-23T21:12:30.440463+0000","last_became_peered":"2023-09-23T21:12:30.440463+0000","last_unstale":"2023-09-23T21:14:01.499520+0000","last_undegraded":"2023-09-23T21:14:01.499520+0000","last_fullsized":"2023-09-23T21:14:01.499520+0000","mapping_epoch":455,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:10:39.567232+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:10:39.567232+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.005229694,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":322,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-09-23T21:14:02.777152+0000","last_change":"2023-09-23T21:13:13.402418+0000","last_active":"2023-09-23T21:14:02.777152+0000","last_peered":"2023-09-23T21:14:02.777152+0000","last_clean":"2023-09-23T21:14:02.777152+0000","last_became_active":"2023-09-23T21:13:13.400984+0000","last_became_peered":"2023-09-23T21:13:13.400984+0000","last_unstale":"2023-09-23T21:14:02.777152+0000","last_undegraded":"2023-09-23T21:14:02.777152+0000","last_fullsized":"2023-09-23T21:14:02.777152+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":495,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:28:01.411191+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,0],"acting":[3,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":493,"reported_epoch":520,"state":"active+clean","last_fresh":"2023-09-23T21:13:53.464256+0000","last_change":"2023-09-23T21:13:14.339670+0000","last_active":"2023-09-23T21:13:53.464256+0000","last_peered":"2023-09-23T21:13:53.464256+0000","last_clean":"2023-09-23T21:13:53.464256+0000","last_became_active":"2023-09-23T21:13:14.339341+0000","last_became_peered":"2023-09-23T21:13:14.339341+0000","last_unstale":"2023-09-23T21:13:53.464256+0000","last_undegraded":"2023-09-23T21:13:53.464256+0000","last_fullsized":"2023-09-23T21:13:53.464256+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:23:19.411368+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.2","version":"524'4946","reported_seq":6013,"reported_epoch":524,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T21:14:03.161557+0000","last_change":"2023-09-23T21:13:13.407366+0000","last_active":"2023-09-23T21:14:03.161557+0000","last_peered":"2023-09-23T21:14:03.161557+0000","last_clean":"2023-09-23T21:11:45.752822+0000","last_became_active":"2023-09-23T21:13:13.398616+0000","last_became_peered":"2023-09-23T21:13:13.398616+0000","last_unstale":"2023-09-23T21:14:03.161557+0000","last_undegraded":"2023-09-23T21:13:13.289815+0000","last_fullsized":"2023-09-23T21:13:13.190270+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":416,"last_epoch_clean":439,"parent":"0.0","parent_split_bits":0,"last_scrub":"440'3007","last_scrub_stamp":"2023-09-23T21:11:05.553661+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_clean_scrub_stamp":"2023-09-23T21:11:05.553661+0000","objects_scrubbed":1622,"log_size":4946,"log_dups_size":0,"ondisk_log_size":4946,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:51:21.839818+0000","scrub_duration":0.39156952499999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":94240768,"num_objects":1438,"num_object_clones":0,"num_object_copies":4314,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2730,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1438,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1944,"num_write_kb":56320,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":435,"num_bytes_recovered":28508160,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,3],"avail_no_missing":["4","1"],"object_location_counts":[{"shards":"0,1,4","objects":1404},{"shards":"1,4","objects":34}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.9","version":"79'161","reported_seq":675,"reported_epoch":521,"state":"active+clean","last_fresh":"2023-09-23T21:13:58.465311+0000","last_change":"2023-09-23T21:12:30.436118+0000","last_active":"2023-09-23T21:13:58.465311+0000","last_peered":"2023-09-23T21:13:58.465311+0000","last_clean":"2023-09-23T21:13:58.465311+0000","last_became_active":"2023-09-23T21:12:30.435779+0000","last_became_peered":"2023-09-23T21:12:30.435779+0000","last_unstale":"2023-09-23T21:13:58.465311+0000","last_undegraded":"2023-09-23T21:13:58.465311+0000","last_fullsized":"2023-09-23T21:13:58.465311+0000","mapping_epoch":433,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:10:37.696951+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:10:37.696951+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.001386504,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,3],"acting":[2,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":79,"reported_epoch":520,"state":"active+clean","last_fresh":"2023-09-23T21:13:53.464363+0000","last_change":"2023-09-23T21:13:30.793273+0000","last_active":"2023-09-23T21:13:53.464363+0000","last_peered":"2023-09-23T21:13:53.464363+0000","last_clean":"2023-09-23T21:13:53.464363+0000","last_became_active":"2023-09-23T21:13:30.791269+0000","last_became_peered":"2023-09-23T21:13:30.791269+0000","last_unstale":"2023-09-23T21:13:53.464363+0000","last_undegraded":"2023-09-23T21:13:53.464363+0000","last_fullsized":"2023-09-23T21:13:53.464363+0000","mapping_epoch":501,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:38:07.142290+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":97,"reported_epoch":523,"state":"active+clean","last_fresh":"2023-09-23T21:14:01.498503+0000","last_change":"2023-09-23T21:12:14.313466+0000","last_active":"2023-09-23T21:14:01.498503+0000","last_peered":"2023-09-23T21:14:01.498503+0000","last_clean":"2023-09-23T21:14:01.498503+0000","last_became_active":"2023-09-23T21:12:14.209837+0000","last_became_peered":"2023-09-23T21:12:14.209837+0000","last_unstale":"2023-09-23T21:14:01.498503+0000","last_undegraded":"2023-09-23T21:14:01.498503+0000","last_fullsized":"2023-09-23T21:14:01.498503+0000","mapping_epoch":470,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":474,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:09:40.375241+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,7],"acting":[4,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.3","version":"522'4934","reported_seq":7059,"reported_epoch":522,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T21:14:01.262189+0000","last_change":"2023-09-23T21:13:45.480705+0000","last_active":"2023-09-23T21:14:01.262189+0000","last_peered":"2023-09-23T21:14:01.262189+0000","last_clean":"2023-09-23T21:11:26.700990+0000","last_became_active":"2023-09-23T21:13:45.471702+0000","last_became_peered":"2023-09-23T21:13:45.471702+0000","last_unstale":"2023-09-23T21:14:01.262189+0000","last_undegraded":"2023-09-23T21:13:45.412179+0000","last_fullsized":"2023-09-23T21:13:45.401701+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":416,"last_epoch_clean":439,"parent":"0.0","parent_split_bits":0,"last_scrub":"440'2873","last_scrub_stamp":"2023-09-23T21:11:02.114783+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_clean_scrub_stamp":"2023-09-23T21:11:02.114783+0000","objects_scrubbed":1532,"log_size":4934,"log_dups_size":0,"ondisk_log_size":4934,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:42:48.549370+0000","scrub_duration":0.32435022600000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":98369536,"num_objects":1501,"num_object_clones":0,"num_object_copies":4503,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1184,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1501,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2067,"num_write_kb":65152,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":865,"num_bytes_recovered":56688640,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5],"avail_no_missing":["2","5"],"object_location_counts":[{"shards":"0,2,4,5","objects":270},{"shards":"2,4,5","objects":48},{"shards":"2,5","objects":1182}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.8","version":"79'161","reported_seq":642,"reported_epoch":521,"state":"active+clean","last_fresh":"2023-09-23T21:13:58.466030+0000","last_change":"2023-09-23T21:12:30.433561+0000","last_active":"2023-09-23T21:13:58.466030+0000","last_peered":"2023-09-23T21:13:58.466030+0000","last_clean":"2023-09-23T21:13:58.466030+0000","last_became_active":"2023-09-23T21:12:30.433217+0000","last_became_peered":"2023-09-23T21:12:30.433217+0000","last_unstale":"2023-09-23T21:13:58.466030+0000","last_undegraded":"2023-09-23T21:13:58.466030+0000","last_fullsized":"2023-09-23T21:13:58.466030+0000","mapping_epoch":359,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:06:52.098882+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:06:52.098882+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0092173299999999993,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":131344,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":100,"reported_epoch":523,"state":"active+clean","last_fresh":"2023-09-23T21:14:01.498661+0000","last_change":"2023-09-23T21:12:10.823380+0000","last_active":"2023-09-23T21:14:01.498661+0000","last_peered":"2023-09-23T21:14:01.498661+0000","last_clean":"2023-09-23T21:14:01.498661+0000","last_became_active":"2023-09-23T21:12:10.823044+0000","last_became_peered":"2023-09-23T21:12:10.823044+0000","last_unstale":"2023-09-23T21:14:01.498661+0000","last_undegraded":"2023-09-23T21:14:01.498661+0000","last_fullsized":"2023-09-23T21:14:01.498661+0000","mapping_epoch":470,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:53:29.846536+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,5],"acting":[4,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":101,"reported_epoch":523,"state":"active+clean","last_fresh":"2023-09-23T21:14:01.499354+0000","last_change":"2023-09-23T21:13:30.772451+0000","last_active":"2023-09-23T21:14:01.499354+0000","last_peered":"2023-09-23T21:14:01.499354+0000","last_clean":"2023-09-23T21:14:01.499354+0000","last_became_active":"2023-09-23T21:13:30.772040+0000","last_became_peered":"2023-09-23T21:13:30.772040+0000","last_unstale":"2023-09-23T21:14:01.499354+0000","last_undegraded":"2023-09-23T21:14:01.499354+0000","last_fullsized":"2023-09-23T21:14:01.499354+0000","mapping_epoch":501,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:24:49.246727+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,0],"acting":[4,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.0","version":"524'4881","reported_seq":8488,"reported_epoch":524,"state":"active+recovering+undersized+remapped","last_fresh":"2023-09-23T21:14:03.176326+0000","last_change":"2023-09-23T21:14:02.130787+0000","last_active":"2023-09-23T21:14:03.176326+0000","last_peered":"2023-09-23T21:14:03.176326+0000","last_clean":"2023-09-23T21:11:26.716395+0000","last_became_active":"2023-09-23T21:13:34.557827+0000","last_became_peered":"2023-09-23T21:13:34.557827+0000","last_unstale":"2023-09-23T21:14:03.176326+0000","last_undegraded":"2023-09-23T21:14:03.176326+0000","last_fullsized":"2023-09-23T21:13:31.752840+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":416,"last_epoch_clean":439,"parent":"0.0","parent_split_bits":0,"last_scrub":"440'2961","last_scrub_stamp":"2023-09-23T21:11:04.564028+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_clean_scrub_stamp":"2023-09-23T21:11:04.564028+0000","objects_scrubbed":1608,"log_size":4881,"log_dups_size":0,"ondisk_log_size":4881,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:43:09.815808+0000","scrub_duration":0.41858158299999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":96534528,"num_objects":1473,"num_object_clones":0,"num_object_copies":4419,"num_objects_missing_on_primary":1294,"num_objects_missing":1294,"num_objects_degraded":0,"num_objects_misplaced":1469,"num_objects_unfound":0,"num_objects_dirty":1473,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1923,"num_write_kb":57216,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":874,"num_bytes_recovered":57212928,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,0],"avail_no_missing":["1","7"],"object_location_counts":[{"shards":"0,1,4,7","objects":4},{"shards":"0,1,7","objects":1143},{"shards":"1,4,7","objects":326}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.b","version":"79'161","reported_seq":524,"reported_epoch":520,"state":"active+clean","last_fresh":"2023-09-23T21:13:53.463914+0000","last_change":"2023-09-23T21:13:13.373313+0000","last_active":"2023-09-23T21:13:53.463914+0000","last_peered":"2023-09-23T21:13:53.463914+0000","last_clean":"2023-09-23T21:13:53.463914+0000","last_became_active":"2023-09-23T21:13:13.372328+0000","last_became_peered":"2023-09-23T21:13:13.372328+0000","last_unstale":"2023-09-23T21:13:53.463914+0000","last_undegraded":"2023-09-23T21:13:53.463914+0000","last_fullsized":"2023-09-23T21:13:53.463914+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":495,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:05:55.772264+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:05:55.772264+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.001862507,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":81,"reported_epoch":523,"state":"active+clean","last_fresh":"2023-09-23T21:14:01.498040+0000","last_change":"2023-09-23T21:11:27.754717+0000","last_active":"2023-09-23T21:14:01.498040+0000","last_peered":"2023-09-23T21:14:01.498040+0000","last_clean":"2023-09-23T21:14:01.498040+0000","last_became_active":"2023-09-23T21:11:27.754376+0000","last_became_peered":"2023-09-23T21:11:27.754376+0000","last_unstale":"2023-09-23T21:14:01.498040+0000","last_undegraded":"2023-09-23T21:14:01.498040+0000","last_fullsized":"2023-09-23T21:14:01.498040+0000","mapping_epoch":452,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":453,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:26:41.734541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":455,"reported_epoch":523,"state":"active+clean","last_fresh":"2023-09-23T21:14:01.500220+0000","last_change":"2023-09-23T21:12:10.811698+0000","last_active":"2023-09-23T21:14:01.500220+0000","last_peered":"2023-09-23T21:14:01.500220+0000","last_clean":"2023-09-23T21:14:01.500220+0000","last_became_active":"2023-09-23T21:12:10.789656+0000","last_became_peered":"2023-09-23T21:12:10.789656+0000","last_unstale":"2023-09-23T21:14:01.500220+0000","last_undegraded":"2023-09-23T21:14:01.500220+0000","last_fullsized":"2023-09-23T21:14:01.500220+0000","mapping_epoch":470,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:39:15.112091+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,5],"acting":[4,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.1","version":"524'4828","reported_seq":11238,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-09-23T21:14:03.139483+0000","last_change":"2023-09-23T21:13:53.483736+0000","last_active":"2023-09-23T21:14:03.139483+0000","last_peered":"2023-09-23T21:14:03.139483+0000","last_clean":"2023-09-23T21:14:03.139483+0000","last_became_active":"2023-09-23T21:13:53.472685+0000","last_became_peered":"2023-09-23T21:13:53.472685+0000","last_unstale":"2023-09-23T21:14:03.139483+0000","last_undegraded":"2023-09-23T21:14:03.139483+0000","last_fullsized":"2023-09-23T21:14:03.139483+0000","mapping_epoch":519,"log_start":"0'0","ondisk_log_start":"0'0","created":416,"last_epoch_clean":520,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_clean_scrub_stamp":"2023-09-23T21:10:15.663179+0000","objects_scrubbed":0,"log_size":4828,"log_dups_size":0,"ondisk_log_size":4828,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":95944704,"num_objects":1464,"num_object_clones":0,"num_object_copies":4392,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1464,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3480,"num_write_kb":158208,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3005,"num_bytes_recovered":156499968,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.a","version":"79'161","reported_seq":668,"reported_epoch":521,"state":"active+clean","last_fresh":"2023-09-23T21:13:58.467117+0000","last_change":"2023-09-23T21:12:30.437790+0000","last_active":"2023-09-23T21:13:58.467117+0000","last_peered":"2023-09-23T21:13:58.467117+0000","last_clean":"2023-09-23T21:13:58.467117+0000","last_became_active":"2023-09-23T21:12:30.437427+0000","last_became_peered":"2023-09-23T21:12:30.437427+0000","last_unstale":"2023-09-23T21:13:58.467117+0000","last_undegraded":"2023-09-23T21:13:58.467117+0000","last_fullsized":"2023-09-23T21:13:58.467117+0000","mapping_epoch":446,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:06:51.075448+0000","last_deep_scrub":"79'161","last_deep_scrub_stamp":"2023-09-23T21:05:32.630292+0000","last_clean_scrub_stamp":"2023-09-23T21:06:51.075448+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0054793890000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,4],"acting":[2,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":371,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-09-23T21:14:02.775316+0000","last_change":"2023-09-23T21:12:10.792918+0000","last_active":"2023-09-23T21:14:02.775316+0000","last_peered":"2023-09-23T21:14:02.775316+0000","last_clean":"2023-09-23T21:14:02.775316+0000","last_became_active":"2023-09-23T21:12:10.792679+0000","last_became_peered":"2023-09-23T21:12:10.792679+0000","last_unstale":"2023-09-23T21:14:02.775316+0000","last_undegraded":"2023-09-23T21:14:02.775316+0000","last_fullsized":"2023-09-23T21:14:02.775316+0000","mapping_epoch":470,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:30:48.008690+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,4],"acting":[3,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":502,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-09-23T21:14:02.785832+0000","last_change":"2023-09-23T21:12:10.790812+0000","last_active":"2023-09-23T21:14:02.785832+0000","last_peered":"2023-09-23T21:14:02.785832+0000","last_clean":"2023-09-23T21:14:02.785832+0000","last_became_active":"2023-09-23T21:12:10.790492+0000","last_became_peered":"2023-09-23T21:12:10.790492+0000","last_unstale":"2023-09-23T21:14:02.785832+0000","last_undegraded":"2023-09-23T21:14:02.785832+0000","last_fullsized":"2023-09-23T21:14:02.785832+0000","mapping_epoch":470,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:02:19.863118+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.c","version":"523'7134","reported_seq":8284,"reported_epoch":523,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T21:14:02.252177+0000","last_change":"2023-09-23T21:13:30.822215+0000","last_active":"2023-09-23T21:14:02.252177+0000","last_peered":"2023-09-23T21:14:02.252177+0000","last_clean":"2023-09-23T21:11:45.750376+0000","last_became_active":"2023-09-23T21:13:13.404327+0000","last_became_peered":"2023-09-23T21:13:13.404327+0000","last_unstale":"2023-09-23T21:14:02.252177+0000","last_undegraded":"2023-09-23T21:13:13.200907+0000","last_fullsized":"2023-09-23T21:13:13.193701+0000","mapping_epoch":494,"log_start":"437'2751","ondisk_log_start":"437'2751","created":416,"last_epoch_clean":434,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_clean_scrub_stamp":"2023-09-23T21:10:15.663179+0000","objects_scrubbed":0,"log_size":4383,"log_dups_size":0,"ondisk_log_size":4383,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:33:42.971375+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":190316544,"num_objects":2904,"num_object_clones":0,"num_object_copies":8712,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1339,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2904,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7134,"num_write_kb":321216,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":498,"num_bytes_recovered":32636928,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4],"avail_no_missing":["7","4"],"object_location_counts":[{"shards":"4,7","objects":2850}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.7","version":"79'161","reported_seq":617,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-09-23T21:14:02.785536+0000","last_change":"2023-09-23T21:13:44.416548+0000","last_active":"2023-09-23T21:14:02.785536+0000","last_peered":"2023-09-23T21:14:02.785536+0000","last_clean":"2023-09-23T21:14:02.785536+0000","last_became_active":"2023-09-23T21:13:44.415617+0000","last_became_peered":"2023-09-23T21:13:44.415617+0000","last_unstale":"2023-09-23T21:14:02.785536+0000","last_undegraded":"2023-09-23T21:14:02.785536+0000","last_fullsized":"2023-09-23T21:14:02.785536+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":3,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:25.916867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:25.916867+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.001665323,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":450,"reported_epoch":520,"state":"active+clean","last_fresh":"2023-09-23T21:13:53.465818+0000","last_change":"2023-09-23T21:13:13.211959+0000","last_active":"2023-09-23T21:13:53.465818+0000","last_peered":"2023-09-23T21:13:53.465818+0000","last_clean":"2023-09-23T21:13:53.465818+0000","last_became_active":"2023-09-23T21:13:13.211712+0000","last_became_peered":"2023-09-23T21:13:13.211712+0000","last_unstale":"2023-09-23T21:13:53.465818+0000","last_undegraded":"2023-09-23T21:13:53.465818+0000","last_fullsized":"2023-09-23T21:13:53.465818+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":495,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:37:21.655775+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,2],"acting":[0,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":200,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-09-23T21:14:00.508827+0000","last_change":"2023-09-23T21:05:04.326889+0000","last_active":"2023-09-23T21:14:00.508827+0000","last_peered":"2023-09-23T21:14:00.508827+0000","last_clean":"2023-09-23T21:14:00.508827+0000","last_became_active":"2023-09-23T21:05:04.326032+0000","last_became_peered":"2023-09-23T21:05:04.326032+0000","last_unstale":"2023-09-23T21:14:00.508827+0000","last_undegraded":"2023-09-23T21:14:00.508827+0000","last_fullsized":"2023-09-23T21:14:00.508827+0000","mapping_epoch":332,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:00:17.507889+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":532,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-09-23T21:14:00.509654+0000","last_change":"2023-09-23T21:13:13.285608+0000","last_active":"2023-09-23T21:14:00.509654+0000","last_peered":"2023-09-23T21:14:00.509654+0000","last_clean":"2023-09-23T21:14:00.509654+0000","last_became_active":"2023-09-23T21:13:13.285255+0000","last_became_peered":"2023-09-23T21:13:13.285255+0000","last_unstale":"2023-09-23T21:14:00.509654+0000","last_undegraded":"2023-09-23T21:14:00.509654+0000","last_fullsized":"2023-09-23T21:14:00.509654+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":495,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:42:30.087599+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.a","version":"522'6941","reported_seq":15670,"reported_epoch":522,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T21:14:01.239118+0000","last_change":"2023-09-23T21:13:31.927282+0000","last_active":"2023-09-23T21:14:01.239118+0000","last_peered":"2023-09-23T21:14:01.239118+0000","last_clean":"2023-09-23T21:13:30.353360+0000","last_became_active":"2023-09-23T21:13:31.896923+0000","last_became_peered":"2023-09-23T21:13:31.896923+0000","last_unstale":"2023-09-23T21:14:01.239118+0000","last_undegraded":"2023-09-23T21:13:31.776904+0000","last_fullsized":"2023-09-23T21:13:31.751334+0000","mapping_epoch":502,"log_start":"436'2512","ondisk_log_start":"436'2512","created":416,"last_epoch_clean":499,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_clean_scrub_stamp":"2023-09-23T21:10:15.663179+0000","objects_scrubbed":0,"log_size":4429,"log_dups_size":0,"ondisk_log_size":4429,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:23:14.477602+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":192086016,"num_objects":2931,"num_object_clones":0,"num_object_copies":8793,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2243,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2931,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6941,"num_write_kb":315904,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4293,"num_bytes_recovered":281346048,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[2,5],"avail_no_missing":["2","3","5"],"object_location_counts":[{"shards":"2,3,5","objects":643},{"shards":"2,5","objects":2288}],"blocked_by":[],"up_primary":0,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.1","version":"79'161","reported_seq":591,"reported_epoch":520,"state":"active+clean","last_fresh":"2023-09-23T21:13:53.463928+0000","last_change":"2023-09-23T21:13:12.204135+0000","last_active":"2023-09-23T21:13:53.463928+0000","last_peered":"2023-09-23T21:13:53.463928+0000","last_clean":"2023-09-23T21:13:53.463928+0000","last_became_active":"2023-09-23T21:13:12.203380+0000","last_became_peered":"2023-09-23T21:13:12.203380+0000","last_unstale":"2023-09-23T21:13:53.463928+0000","last_undegraded":"2023-09-23T21:13:53.463928+0000","last_fullsized":"2023-09-23T21:13:53.463928+0000","mapping_epoch":493,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:06:52.575900+0000","last_deep_scrub":"79'161","last_deep_scrub_stamp":"2023-09-23T21:05:30.652189+0000","last_clean_scrub_stamp":"2023-09-23T21:06:52.575900+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":187,"reported_epoch":520,"state":"active+clean","last_fresh":"2023-09-23T21:13:53.463981+0000","last_change":"2023-09-23T21:13:13.238617+0000","last_active":"2023-09-23T21:13:53.463981+0000","last_peered":"2023-09-23T21:13:53.463981+0000","last_clean":"2023-09-23T21:13:53.463981+0000","last_became_active":"2023-09-23T21:13:13.232926+0000","last_became_peered":"2023-09-23T21:13:13.232926+0000","last_unstale":"2023-09-23T21:13:53.463981+0000","last_undegraded":"2023-09-23T21:13:53.463981+0000","last_fullsized":"2023-09-23T21:13:53.463981+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":495,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:17:21.823772+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":457,"reported_epoch":523,"state":"active+clean","last_fresh":"2023-09-23T21:14:01.516761+0000","last_change":"2023-09-23T21:13:14.144918+0000","last_active":"2023-09-23T21:14:01.516761+0000","last_peered":"2023-09-23T21:14:01.516761+0000","last_clean":"2023-09-23T21:14:01.516761+0000","last_became_active":"2023-09-23T21:13:14.132388+0000","last_became_peered":"2023-09-23T21:13:14.132388+0000","last_unstale":"2023-09-23T21:14:01.516761+0000","last_undegraded":"2023-09-23T21:14:01.516761+0000","last_fullsized":"2023-09-23T21:14:01.516761+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":495,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:56:02.619905+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,5],"acting":[4,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":341,"reported_epoch":521,"state":"active+clean","last_fresh":"2023-09-23T21:13:58.465654+0000","last_change":"2023-09-23T21:12:10.764503+0000","last_active":"2023-09-23T21:13:58.465654+0000","last_peered":"2023-09-23T21:13:58.465654+0000","last_clean":"2023-09-23T21:13:58.465654+0000","last_became_active":"2023-09-23T21:12:10.763933+0000","last_became_peered":"2023-09-23T21:12:10.763933+0000","last_unstale":"2023-09-23T21:13:58.465654+0000","last_undegraded":"2023-09-23T21:13:58.465654+0000","last_fullsized":"2023-09-23T21:13:58.465654+0000","mapping_epoch":470,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:28:33.202825+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,7],"acting":[2,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"10.9","version":"522'4897","reported_seq":6034,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-09-23T21:14:01.241468+0000","last_change":"2023-09-23T21:11:08.062606+0000","last_active":"2023-09-23T21:14:01.241468+0000","last_peered":"2023-09-23T21:14:01.241468+0000","last_clean":"2023-09-23T21:14:01.241468+0000","last_became_active":"2023-09-23T21:10:58.780961+0000","last_became_peered":"2023-09-23T21:10:58.780961+0000","last_unstale":"2023-09-23T21:14:01.241468+0000","last_undegraded":"2023-09-23T21:14:01.241468+0000","last_fullsized":"2023-09-23T21:14:01.241468+0000","mapping_epoch":428,"log_start":"0'0","ondisk_log_start":"0'0","created":416,"last_epoch_clean":439,"parent":"0.0","parent_split_bits":0,"last_scrub":"443'3049","last_scrub_stamp":"2023-09-23T21:11:08.062479+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_clean_scrub_stamp":"2023-09-23T21:11:08.062479+0000","objects_scrubbed":1623,"log_size":4897,"log_dups_size":0,"ondisk_log_size":4897,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:56:22.338088+0000","scrub_duration":0.28207626200000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":95420416,"num_objects":1456,"num_object_clones":0,"num_object_copies":4368,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1456,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1849,"num_write_kb":53824,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.2","version":"79'161","reported_seq":800,"reported_epoch":520,"state":"active+clean","last_fresh":"2023-09-23T21:13:53.463782+0000","last_change":"2023-09-23T21:13:12.209907+0000","last_active":"2023-09-23T21:13:53.463782+0000","last_peered":"2023-09-23T21:13:53.463782+0000","last_clean":"2023-09-23T21:13:53.463782+0000","last_became_active":"2023-09-23T21:13:12.205664+0000","last_became_peered":"2023-09-23T21:13:12.205664+0000","last_unstale":"2023-09-23T21:13:53.463782+0000","last_undegraded":"2023-09-23T21:13:53.463782+0000","last_fullsized":"2023-09-23T21:13:53.463782+0000","mapping_epoch":493,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:24.802451+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:24.802451+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:26:16.035788+0000","scrub_duration":0.0090861250000000005,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,4],"acting":[0,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":530,"reported_epoch":521,"state":"active+clean","last_fresh":"2023-09-23T21:13:58.465667+0000","last_change":"2023-09-23T21:13:12.206980+0000","last_active":"2023-09-23T21:13:58.465667+0000","last_peered":"2023-09-23T21:13:58.465667+0000","last_clean":"2023-09-23T21:13:58.465667+0000","last_became_active":"2023-09-23T21:13:12.206650+0000","last_became_peered":"2023-09-23T21:13:12.206650+0000","last_unstale":"2023-09-23T21:13:58.465667+0000","last_undegraded":"2023-09-23T21:13:58.465667+0000","last_fullsized":"2023-09-23T21:13:58.465667+0000","mapping_epoch":493,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:10:01.892936+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":513,"reported_epoch":521,"state":"active+clean","last_fresh":"2023-09-23T21:13:58.466193+0000","last_change":"2023-09-23T21:13:13.407073+0000","last_active":"2023-09-23T21:13:58.466193+0000","last_peered":"2023-09-23T21:13:58.466193+0000","last_clean":"2023-09-23T21:13:58.466193+0000","last_became_active":"2023-09-23T21:13:13.395047+0000","last_became_peered":"2023-09-23T21:13:13.395047+0000","last_unstale":"2023-09-23T21:13:58.466193+0000","last_undegraded":"2023-09-23T21:13:58.466193+0000","last_fullsized":"2023-09-23T21:13:58.466193+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":495,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:04:43.384190+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,5],"acting":[2,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":30,"reported_epoch":520,"state":"active+clean","last_fresh":"2023-09-23T21:13:53.464635+0000","last_change":"2023-09-23T21:13:30.785054+0000","last_active":"2023-09-23T21:13:53.464635+0000","last_peered":"2023-09-23T21:13:53.464635+0000","last_clean":"2023-09-23T21:13:53.464635+0000","last_became_active":"2023-09-23T21:13:30.784701+0000","last_became_peered":"2023-09-23T21:13:30.784701+0000","last_unstale":"2023-09-23T21:13:53.464635+0000","last_undegraded":"2023-09-23T21:13:53.464635+0000","last_fullsized":"2023-09-23T21:13:53.464635+0000","mapping_epoch":501,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:19:20.152252+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.b","version":"524'7155","reported_seq":13660,"reported_epoch":524,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2023-09-23T21:14:03.696892+0000","last_change":"2023-09-23T21:13:59.891994+0000","last_active":"2023-09-23T21:14:03.696892+0000","last_peered":"2023-09-23T21:14:03.696892+0000","last_clean":"2023-09-23T21:11:26.732248+0000","last_became_active":"2023-09-23T21:13:14.937458+0000","last_became_peered":"2023-09-23T21:13:14.937458+0000","last_unstale":"2023-09-23T21:14:03.696892+0000","last_undegraded":"2023-09-23T21:13:59.884123+0000","last_fullsized":"2023-09-23T21:14:03.696892+0000","mapping_epoch":494,"log_start":"426'600","ondisk_log_start":"426'600","created":416,"last_epoch_clean":430,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_clean_scrub_stamp":"2023-09-23T21:10:15.663179+0000","objects_scrubbed":0,"log_size":6555,"log_dups_size":0,"ondisk_log_size":6555,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:36:15.760947+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":192479232,"num_objects":2937,"num_object_clones":0,"num_object_copies":8811,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1337,"num_objects_misplaced":2268,"num_objects_unfound":0,"num_objects_dirty":2937,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7155,"num_write_kb":322944,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3164,"num_bytes_recovered":207159296,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[1,4,7],"avail_no_missing":["1","7"],"object_location_counts":[{"shards":"0,1,7","objects":2260},{"shards":"1,4,7","objects":602}],"blocked_by":[],"up_primary":0,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":307,"reported_epoch":521,"state":"active+clean","last_fresh":"2023-09-23T21:13:58.467879+0000","last_change":"2023-09-23T21:10:26.769764+0000","last_active":"2023-09-23T21:13:58.467879+0000","last_peered":"2023-09-23T21:13:58.467879+0000","last_clean":"2023-09-23T21:13:58.467879+0000","last_became_active":"2023-09-23T21:10:26.769326+0000","last_became_peered":"2023-09-23T21:10:26.769326+0000","last_unstale":"2023-09-23T21:13:58.467879+0000","last_undegraded":"2023-09-23T21:13:58.467879+0000","last_fullsized":"2023-09-23T21:13:58.467879+0000","mapping_epoch":423,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":424,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:57:45.219837+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"79'161","reported_seq":703,"reported_epoch":520,"state":"active+clean","last_fresh":"2023-09-23T21:13:53.463453+0000","last_change":"2023-09-23T21:13:33.766895+0000","last_active":"2023-09-23T21:13:53.463453+0000","last_peered":"2023-09-23T21:13:53.463453+0000","last_clean":"2023-09-23T21:13:53.463453+0000","last_became_active":"2023-09-23T21:13:33.766538+0000","last_became_peered":"2023-09-23T21:13:33.766538+0000","last_unstale":"2023-09-23T21:13:53.463453+0000","last_undegraded":"2023-09-23T21:13:53.463453+0000","last_fullsized":"2023-09-23T21:13:53.463453+0000","mapping_epoch":504,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":505,"parent":"0.0","parent_split_bits":0,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:20.738273+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:20.738273+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:35:35.855987+0000","scrub_duration":0.0088336339999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,2],"acting":[0,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":530,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-09-23T21:14:00.508602+0000","last_change":"2023-09-23T21:11:08.760417+0000","last_active":"2023-09-23T21:14:00.508602+0000","last_peered":"2023-09-23T21:14:00.508602+0000","last_clean":"2023-09-23T21:14:00.508602+0000","last_became_active":"2023-09-23T21:11:08.760150+0000","last_became_peered":"2023-09-23T21:11:08.760150+0000","last_unstale":"2023-09-23T21:14:00.508602+0000","last_undegraded":"2023-09-23T21:14:00.508602+0000","last_fullsized":"2023-09-23T21:14:00.508602+0000","mapping_epoch":443,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":444,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:23:10.849768+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":98,"reported_epoch":523,"state":"active+clean","last_fresh":"2023-09-23T21:14:01.499490+0000","last_change":"2023-09-23T21:13:12.216020+0000","last_active":"2023-09-23T21:14:01.499490+0000","last_peered":"2023-09-23T21:14:01.499490+0000","last_clean":"2023-09-23T21:14:01.499490+0000","last_became_active":"2023-09-23T21:13:12.215180+0000","last_became_peered":"2023-09-23T21:13:12.215180+0000","last_unstale":"2023-09-23T21:14:01.499490+0000","last_undegraded":"2023-09-23T21:14:01.499490+0000","last_fullsized":"2023-09-23T21:14:01.499490+0000","mapping_epoch":493,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:26:18.704684+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.8","version":"522'4766","reported_seq":6740,"reported_epoch":522,"state":"active+recovering+undersized+remapped","last_fresh":"2023-09-23T21:14:01.265227+0000","last_change":"2023-09-23T21:14:00.002164+0000","last_active":"2023-09-23T21:14:01.265227+0000","last_peered":"2023-09-23T21:14:01.265227+0000","last_clean":"2023-09-23T21:11:45.772970+0000","last_became_active":"2023-09-23T21:13:13.399180+0000","last_became_peered":"2023-09-23T21:13:13.399180+0000","last_unstale":"2023-09-23T21:14:01.265227+0000","last_undegraded":"2023-09-23T21:14:01.265227+0000","last_fullsized":"2023-09-23T21:13:13.185457+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":416,"last_epoch_clean":439,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_clean_scrub_stamp":"2023-09-23T21:10:15.663179+0000","objects_scrubbed":0,"log_size":4766,"log_dups_size":0,"ondisk_log_size":4766,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":92569600,"num_objects":1412,"num_object_clones":0,"num_object_copies":4236,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1412,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3416,"num_write_kb":154528,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":678,"num_bytes_recovered":16908288,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,7],"acting":[2,7],"avail_no_missing":["2","7"],"object_location_counts":[{"shards":"2,7","objects":1412}],"blocked_by":[],"up_primary":0,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"79'161","reported_seq":826,"reported_epoch":523,"state":"active+clean","last_fresh":"2023-09-23T21:14:01.505286+0000","last_change":"2023-09-23T21:13:33.771279+0000","last_active":"2023-09-23T21:14:01.505286+0000","last_peered":"2023-09-23T21:14:01.505286+0000","last_clean":"2023-09-23T21:14:01.505286+0000","last_became_active":"2023-09-23T21:13:33.770980+0000","last_became_peered":"2023-09-23T21:13:33.770980+0000","last_unstale":"2023-09-23T21:14:01.505286+0000","last_undegraded":"2023-09-23T21:14:01.505286+0000","last_fullsized":"2023-09-23T21:14:01.505286+0000","mapping_epoch":504,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":505,"parent":"0.0","parent_split_bits":3,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:05:57.832302+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:05:57.832302+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:19:42.309604+0000","scrub_duration":0.010206847999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":337,"reported_epoch":520,"state":"active+clean","last_fresh":"2023-09-23T21:13:53.464528+0000","last_change":"2023-09-23T21:13:13.212775+0000","last_active":"2023-09-23T21:13:53.464528+0000","last_peered":"2023-09-23T21:13:53.464528+0000","last_clean":"2023-09-23T21:13:53.464528+0000","last_became_active":"2023-09-23T21:13:13.212460+0000","last_became_peered":"2023-09-23T21:13:13.212460+0000","last_unstale":"2023-09-23T21:13:53.464528+0000","last_undegraded":"2023-09-23T21:13:53.464528+0000","last_fullsized":"2023-09-23T21:13:53.464528+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":495,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:27:10.853970+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":519,"reported_epoch":520,"state":"active+clean","last_fresh":"2023-09-23T21:13:53.463814+0000","last_change":"2023-09-23T21:13:44.409212+0000","last_active":"2023-09-23T21:13:53.463814+0000","last_peered":"2023-09-23T21:13:53.463814+0000","last_clean":"2023-09-23T21:13:53.463814+0000","last_became_active":"2023-09-23T21:13:44.408958+0000","last_became_peered":"2023-09-23T21:13:44.408958+0000","last_unstale":"2023-09-23T21:13:53.463814+0000","last_undegraded":"2023-09-23T21:13:53.463814+0000","last_fullsized":"2023-09-23T21:13:53.463814+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:03:52.749964+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":30,"reported_epoch":520,"state":"active+clean","last_fresh":"2023-09-23T21:13:53.464989+0000","last_change":"2023-09-23T21:13:30.804379+0000","last_active":"2023-09-23T21:13:53.464989+0000","last_peered":"2023-09-23T21:13:53.464989+0000","last_clean":"2023-09-23T21:13:53.464989+0000","last_became_active":"2023-09-23T21:13:30.799907+0000","last_became_peered":"2023-09-23T21:13:30.799907+0000","last_unstale":"2023-09-23T21:13:53.464989+0000","last_undegraded":"2023-09-23T21:13:53.464989+0000","last_fullsized":"2023-09-23T21:13:53.464989+0000","mapping_epoch":501,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:49:42.859212+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,7],"acting":[0,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.d","version":"524'7067","reported_seq":7224,"reported_epoch":524,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-23T21:14:03.182723+0000","last_change":"2023-09-23T21:13:31.801486+0000","last_active":"2023-09-23T21:14:03.182723+0000","last_peered":"2023-09-23T21:14:03.182723+0000","last_clean":"2023-09-23T21:11:37.659929+0000","last_became_active":"2023-09-23T21:13:31.786626+0000","last_became_peered":"2023-09-23T21:13:31.786626+0000","last_unstale":"2023-09-23T21:14:03.182723+0000","last_undegraded":"2023-09-23T21:13:31.760908+0000","last_fullsized":"2023-09-23T21:14:03.182723+0000","mapping_epoch":502,"log_start":"440'3095","ondisk_log_start":"440'3095","created":416,"last_epoch_clean":417,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_clean_scrub_stamp":"2023-09-23T21:10:15.663179+0000","objects_scrubbed":0,"log_size":3972,"log_dups_size":0,"ondisk_log_size":3972,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:46:02.424493+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":192282648,"num_objects":2935,"num_object_clones":0,"num_object_copies":8805,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1265,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2935,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":7067,"num_write_kb":320001,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,3],"avail_no_missing":["4","3","7"],"object_location_counts":[{"shards":"0,3,4","objects":2856},{"shards":"3,4","objects":79}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.6","version":"79'161","reported_seq":439,"reported_epoch":520,"state":"active+clean","last_fresh":"2023-09-23T21:13:53.464917+0000","last_change":"2023-09-23T21:13:33.770715+0000","last_active":"2023-09-23T21:13:53.464917+0000","last_peered":"2023-09-23T21:13:53.464917+0000","last_clean":"2023-09-23T21:13:53.464917+0000","last_became_active":"2023-09-23T21:13:33.769303+0000","last_became_peered":"2023-09-23T21:13:33.769303+0000","last_unstale":"2023-09-23T21:13:53.464917+0000","last_undegraded":"2023-09-23T21:13:53.464917+0000","last_fullsized":"2023-09-23T21:13:53.464917+0000","mapping_epoch":504,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":505,"parent":"0.0","parent_split_bits":3,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:10:24.268662+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:10:24.268662+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13417,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":11,"num_read_kb":20,"num_write":5,"num_write_kb":55,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":40252,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":149,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-09-23T21:14:00.484028+0000","last_change":"2023-09-23T21:13:30.770826+0000","last_active":"2023-09-23T21:14:00.484028+0000","last_peered":"2023-09-23T21:14:00.484028+0000","last_clean":"2023-09-23T21:14:00.484028+0000","last_became_active":"2023-09-23T21:13:30.770512+0000","last_became_peered":"2023-09-23T21:13:30.770512+0000","last_unstale":"2023-09-23T21:14:00.484028+0000","last_undegraded":"2023-09-23T21:14:00.484028+0000","last_fullsized":"2023-09-23T21:14:00.484028+0000","mapping_epoch":501,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:44:23.470538+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":404,"reported_epoch":521,"state":"active+clean","last_fresh":"2023-09-23T21:13:58.465892+0000","last_change":"2023-09-23T21:13:30.790590+0000","last_active":"2023-09-23T21:13:58.465892+0000","last_peered":"2023-09-23T21:13:58.465892+0000","last_clean":"2023-09-23T21:13:58.465892+0000","last_became_active":"2023-09-23T21:13:30.790253+0000","last_became_peered":"2023-09-23T21:13:30.790253+0000","last_unstale":"2023-09-23T21:13:58.465892+0000","last_undegraded":"2023-09-23T21:13:58.465892+0000","last_fullsized":"2023-09-23T21:13:58.465892+0000","mapping_epoch":501,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:23:41.843250+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,0],"acting":[2,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":466,"reported_epoch":520,"state":"active+clean","last_fresh":"2023-09-23T21:13:53.464789+0000","last_change":"2023-09-23T21:13:12.210546+0000","last_active":"2023-09-23T21:13:53.464789+0000","last_peered":"2023-09-23T21:13:53.464789+0000","last_clean":"2023-09-23T21:13:53.464789+0000","last_became_active":"2023-09-23T21:13:12.205934+0000","last_became_peered":"2023-09-23T21:13:12.205934+0000","last_unstale":"2023-09-23T21:13:53.464789+0000","last_undegraded":"2023-09-23T21:13:53.464789+0000","last_fullsized":"2023-09-23T21:13:53.464789+0000","mapping_epoch":493,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:21:18.299428+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.e","version":"524'7241","reported_seq":14207,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-09-23T21:14:03.166070+0000","last_change":"2023-09-23T21:12:38.623611+0000","last_active":"2023-09-23T21:14:03.166070+0000","last_peered":"2023-09-23T21:14:03.166070+0000","last_clean":"2023-09-23T21:14:03.166070+0000","last_became_active":"2023-09-23T21:12:38.610155+0000","last_became_peered":"2023-09-23T21:12:38.610155+0000","last_unstale":"2023-09-23T21:14:03.166070+0000","last_undegraded":"2023-09-23T21:14:03.166070+0000","last_fullsized":"2023-09-23T21:14:03.166070+0000","mapping_epoch":483,"log_start":"440'3293","ondisk_log_start":"440'3293","created":416,"last_epoch_clean":484,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_clean_scrub_stamp":"2023-09-23T21:10:15.663179+0000","objects_scrubbed":0,"log_size":3948,"log_dups_size":0,"ondisk_log_size":3948,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:19:27.097170+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":195493888,"num_objects":2983,"num_object_clones":0,"num_object_copies":8949,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2983,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7241,"num_write_kb":327168,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3412,"num_bytes_recovered":223608832,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.5","version":"79'161","reported_seq":551,"reported_epoch":523,"state":"active+clean","last_fresh":"2023-09-23T21:14:01.499423+0000","last_change":"2023-09-23T21:13:38.373969+0000","last_active":"2023-09-23T21:14:01.499423+0000","last_peered":"2023-09-23T21:14:01.499423+0000","last_clean":"2023-09-23T21:14:01.499423+0000","last_became_active":"2023-09-23T21:13:38.373633+0000","last_became_peered":"2023-09-23T21:13:38.373633+0000","last_unstale":"2023-09-23T21:14:01.499423+0000","last_undegraded":"2023-09-23T21:14:01.499423+0000","last_fullsized":"2023-09-23T21:14:01.499423+0000","mapping_epoch":507,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":3,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:09:34.892947+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:09:34.892947+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13417,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":11,"num_read_kb":20,"num_write":5,"num_write_kb":55,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":40252,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":61,"reported_epoch":520,"state":"active+clean","last_fresh":"2023-09-23T21:13:53.464734+0000","last_change":"2023-09-23T21:13:12.235743+0000","last_active":"2023-09-23T21:13:53.464734+0000","last_peered":"2023-09-23T21:13:53.464734+0000","last_clean":"2023-09-23T21:13:53.464734+0000","last_became_active":"2023-09-23T21:13:12.216241+0000","last_became_peered":"2023-09-23T21:13:12.216241+0000","last_unstale":"2023-09-23T21:13:53.464734+0000","last_undegraded":"2023-09-23T21:13:53.464734+0000","last_fullsized":"2023-09-23T21:13:53.464734+0000","mapping_epoch":493,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:12:19.090710+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":504,"reported_epoch":523,"state":"active+clean","last_fresh":"2023-09-23T21:14:01.500083+0000","last_change":"2023-09-23T21:13:30.807520+0000","last_active":"2023-09-23T21:14:01.500083+0000","last_peered":"2023-09-23T21:14:01.500083+0000","last_clean":"2023-09-23T21:14:01.500083+0000","last_became_active":"2023-09-23T21:13:30.807222+0000","last_became_peered":"2023-09-23T21:13:30.807222+0000","last_unstale":"2023-09-23T21:14:01.500083+0000","last_undegraded":"2023-09-23T21:14:01.500083+0000","last_fullsized":"2023-09-23T21:14:01.500083+0000","mapping_epoch":501,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:40:31.989749+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":366,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-09-23T21:14:02.775387+0000","last_change":"2023-09-23T21:13:30.794095+0000","last_active":"2023-09-23T21:14:02.775387+0000","last_peered":"2023-09-23T21:14:02.775387+0000","last_clean":"2023-09-23T21:14:02.775387+0000","last_became_active":"2023-09-23T21:13:30.793791+0000","last_became_peered":"2023-09-23T21:13:30.793791+0000","last_unstale":"2023-09-23T21:14:02.775387+0000","last_undegraded":"2023-09-23T21:14:02.775387+0000","last_fullsized":"2023-09-23T21:14:02.775387+0000","mapping_epoch":501,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:36:28.707362+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.f","version":"525'7224","reported_seq":7424,"reported_epoch":525,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T21:14:04.156122+0000","last_change":"2023-09-23T21:13:59.874192+0000","last_active":"2023-09-23T21:14:04.156122+0000","last_peered":"2023-09-23T21:14:04.156122+0000","last_clean":"2023-09-23T21:11:07.652290+0000","last_became_active":"2023-09-23T21:11:09.897463+0000","last_became_peered":"2023-09-23T21:11:09.897463+0000","last_unstale":"2023-09-23T21:14:04.156122+0000","last_undegraded":"2023-09-23T21:11:27.781677+0000","last_fullsized":"2023-09-23T21:11:09.761652+0000","mapping_epoch":444,"log_start":"447'3509","ondisk_log_start":"447'3509","created":416,"last_epoch_clean":417,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_clean_scrub_stamp":"2023-09-23T21:10:15.663179+0000","objects_scrubbed":0,"log_size":3715,"log_dups_size":0,"ondisk_log_size":3715,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:42:51.588441+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":189399040,"num_objects":2890,"num_object_clones":0,"num_object_copies":8670,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3295,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2890,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7224,"num_write_kb":323648,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":32,"num_bytes_recovered":2097152,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2],"avail_no_missing":["3","1","2"],"object_location_counts":[{"shards":"1,2,3","objects":1160},{"shards":"2,3","objects":-405}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.4","version":"79'161","reported_seq":895,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-09-23T21:14:02.776948+0000","last_change":"2023-09-23T21:10:29.807785+0000","last_active":"2023-09-23T21:14:02.776948+0000","last_peered":"2023-09-23T21:14:02.776948+0000","last_clean":"2023-09-23T21:14:02.776948+0000","last_became_active":"2023-09-23T21:10:29.807366+0000","last_became_peered":"2023-09-23T21:10:29.807366+0000","last_unstale":"2023-09-23T21:14:02.776948+0000","last_undegraded":"2023-09-23T21:14:02.776948+0000","last_fullsized":"2023-09-23T21:14:02.776948+0000","mapping_epoch":426,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":427,"parent":"0.0","parent_split_bits":3,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:20.012764+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:20.012764+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:24:45.763834+0000","scrub_duration":0.0045251919999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":460,"reported_epoch":521,"state":"active+clean","last_fresh":"2023-09-23T21:13:58.468260+0000","last_change":"2023-09-23T21:10:26.768364+0000","last_active":"2023-09-23T21:13:58.468260+0000","last_peered":"2023-09-23T21:13:58.468260+0000","last_clean":"2023-09-23T21:13:58.468260+0000","last_became_active":"2023-09-23T21:10:26.768052+0000","last_became_peered":"2023-09-23T21:10:26.768052+0000","last_unstale":"2023-09-23T21:13:58.468260+0000","last_undegraded":"2023-09-23T21:13:58.468260+0000","last_fullsized":"2023-09-23T21:13:58.468260+0000","mapping_epoch":423,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":424,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:22:42.201200+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,3],"acting":[2,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.6","version":"70'1","reported_seq":491,"reported_epoch":520,"state":"active+clean","last_fresh":"2023-09-23T21:13:53.465222+0000","last_change":"2023-09-23T21:13:30.793290+0000","last_active":"2023-09-23T21:13:53.465222+0000","last_peered":"2023-09-23T21:13:53.465222+0000","last_clean":"2023-09-23T21:13:53.465222+0000","last_became_active":"2023-09-23T21:13:30.786079+0000","last_became_peered":"2023-09-23T21:13:30.786079+0000","last_unstale":"2023-09-23T21:13:53.465222+0000","last_undegraded":"2023-09-23T21:13:53.465222+0000","last_fullsized":"2023-09-23T21:13:53.465222+0000","mapping_epoch":501,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:34:43.281095+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":98,"reported_epoch":523,"state":"active+clean","last_fresh":"2023-09-23T21:14:01.497975+0000","last_change":"2023-09-23T21:13:12.216895+0000","last_active":"2023-09-23T21:14:01.497975+0000","last_peered":"2023-09-23T21:14:01.497975+0000","last_clean":"2023-09-23T21:14:01.497975+0000","last_became_active":"2023-09-23T21:13:12.216654+0000","last_became_peered":"2023-09-23T21:13:12.216654+0000","last_unstale":"2023-09-23T21:14:01.497975+0000","last_undegraded":"2023-09-23T21:14:01.497975+0000","last_fullsized":"2023-09-23T21:14:01.497975+0000","mapping_epoch":493,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:02:40.430096+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.19","version":"79'161","reported_seq":674,"reported_epoch":521,"state":"active+clean","last_fresh":"2023-09-23T21:13:58.465093+0000","last_change":"2023-09-23T21:12:30.442914+0000","last_active":"2023-09-23T21:13:58.465093+0000","last_peered":"2023-09-23T21:13:58.465093+0000","last_clean":"2023-09-23T21:13:58.465093+0000","last_became_active":"2023-09-23T21:12:30.442578+0000","last_became_peered":"2023-09-23T21:12:30.442578+0000","last_unstale":"2023-09-23T21:13:58.465093+0000","last_undegraded":"2023-09-23T21:13:58.465093+0000","last_fullsized":"2023-09-23T21:13:58.465093+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":479,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:10:37.696951+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:10:37.696951+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.001386504,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,3],"acting":[2,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"10.12","version":"524'4903","reported_seq":5925,"reported_epoch":524,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T21:14:03.033460+0000","last_change":"2023-09-23T21:13:13.419184+0000","last_active":"2023-09-23T21:14:03.033460+0000","last_peered":"2023-09-23T21:14:03.033460+0000","last_clean":"2023-09-23T21:11:45.774771+0000","last_became_active":"2023-09-23T21:13:13.409281+0000","last_became_peered":"2023-09-23T21:13:13.409281+0000","last_unstale":"2023-09-23T21:14:03.033460+0000","last_undegraded":"2023-09-23T21:13:13.284063+0000","last_fullsized":"2023-09-23T21:13:13.191117+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":438,"last_epoch_clean":439,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_clean_scrub_stamp":"2023-09-23T21:10:15.663179+0000","objects_scrubbed":0,"log_size":4903,"log_dups_size":0,"ondisk_log_size":4903,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":92405760,"num_objects":1410,"num_object_clones":0,"num_object_copies":4230,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2706,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1410,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3526,"num_write_kb":157952,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":432,"num_bytes_recovered":28311552,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,3],"avail_no_missing":["4","1"],"object_location_counts":[{"shards":"0,1,4","objects":1372},{"shards":"1,4","objects":41},{"shards":"3,4","objects":-3}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":30,"reported_epoch":520,"state":"active+clean","last_fresh":"2023-09-23T21:13:53.464364+0000","last_change":"2023-09-23T21:13:30.791297+0000","last_active":"2023-09-23T21:13:53.464364+0000","last_peered":"2023-09-23T21:13:53.464364+0000","last_clean":"2023-09-23T21:13:53.464364+0000","last_became_active":"2023-09-23T21:13:30.788721+0000","last_became_peered":"2023-09-23T21:13:30.788721+0000","last_unstale":"2023-09-23T21:13:53.464364+0000","last_undegraded":"2023-09-23T21:13:53.464364+0000","last_fullsized":"2023-09-23T21:13:53.464364+0000","mapping_epoch":501,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:31:18.147883+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":25,"reported_epoch":521,"state":"active+clean","last_fresh":"2023-09-23T21:13:58.465814+0000","last_change":"2023-09-23T21:13:38.373814+0000","last_active":"2023-09-23T21:13:58.465814+0000","last_peered":"2023-09-23T21:13:58.465814+0000","last_clean":"2023-09-23T21:13:58.465814+0000","last_became_active":"2023-09-23T21:13:38.373458+0000","last_became_peered":"2023-09-23T21:13:38.373458+0000","last_unstale":"2023-09-23T21:13:58.465814+0000","last_undegraded":"2023-09-23T21:13:58.465814+0000","last_fullsized":"2023-09-23T21:13:58.465814+0000","mapping_epoch":507,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:23:08.288401+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.1e","version":"79'161","reported_seq":691,"reported_epoch":523,"state":"active+clean","last_fresh":"2023-09-23T21:14:01.499243+0000","last_change":"2023-09-23T21:12:30.448761+0000","last_active":"2023-09-23T21:14:01.499243+0000","last_peered":"2023-09-23T21:14:01.499243+0000","last_clean":"2023-09-23T21:14:01.499243+0000","last_became_active":"2023-09-23T21:12:30.448043+0000","last_became_peered":"2023-09-23T21:12:30.448043+0000","last_unstale":"2023-09-23T21:14:01.499243+0000","last_undegraded":"2023-09-23T21:14:01.499243+0000","last_fullsized":"2023-09-23T21:14:01.499243+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":479,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:10:39.567232+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:10:39.567232+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.005229694,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.15","version":"522'4777","reported_seq":4916,"reported_epoch":522,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T21:14:01.227399+0000","last_change":"2023-09-23T21:13:13.436807+0000","last_active":"2023-09-23T21:14:01.227399+0000","last_peered":"2023-09-23T21:14:01.227399+0000","last_clean":"2023-09-23T21:11:45.765730+0000","last_became_active":"2023-09-23T21:13:13.429206+0000","last_became_peered":"2023-09-23T21:13:13.429206+0000","last_unstale":"2023-09-23T21:14:01.227399+0000","last_undegraded":"2023-09-23T21:13:13.246273+0000","last_fullsized":"2023-09-23T21:13:13.203825+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":438,"last_epoch_clean":439,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_clean_scrub_stamp":"2023-09-23T21:10:15.663179+0000","objects_scrubbed":0,"log_size":4777,"log_dups_size":0,"ondisk_log_size":4777,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":98631680,"num_objects":1505,"num_object_clones":0,"num_object_copies":4515,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":643,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1505,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3443,"num_write_kb":158336,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,3],"acting":[2,3],"avail_no_missing":["2","3"],"object_location_counts":[{"shards":"2,3","objects":1469}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":99,"reported_epoch":523,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-23T21:14:01.498603+0000","last_change":"2023-09-23T21:12:14.313202+0000","last_active":"2023-09-23T21:14:01.498603+0000","last_peered":"2023-09-23T21:14:01.498603+0000","last_clean":"2023-09-23T21:12:09.299355+0000","last_became_active":"2023-09-23T21:12:14.209408+0000","last_became_peered":"2023-09-23T21:12:14.209408+0000","last_unstale":"2023-09-23T21:14:01.498603+0000","last_undegraded":"2023-09-23T21:12:12.933857+0000","last_fullsized":"2023-09-23T21:14:01.498603+0000","mapping_epoch":470,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":444,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:45:40.142413+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,7],"acting":[4,5,7],"avail_no_missing":["4","1","5"],"object_location_counts":[{"shards":"1,4,5","objects":1}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":88,"reported_epoch":523,"state":"active+clean","last_fresh":"2023-09-23T21:14:01.506286+0000","last_change":"2023-09-23T21:12:10.770474+0000","last_active":"2023-09-23T21:14:01.506286+0000","last_peered":"2023-09-23T21:14:01.506286+0000","last_clean":"2023-09-23T21:14:01.506286+0000","last_became_active":"2023-09-23T21:12:10.770184+0000","last_became_peered":"2023-09-23T21:12:10.770184+0000","last_unstale":"2023-09-23T21:14:01.506286+0000","last_undegraded":"2023-09-23T21:14:01.506286+0000","last_fullsized":"2023-09-23T21:14:01.506286+0000","mapping_epoch":470,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:05:09.407151+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.14","version":"521'4715","reported_seq":10284,"reported_epoch":521,"state":"active+recovering+undersized+remapped","last_fresh":"2023-09-23T21:13:59.195287+0000","last_change":"2023-09-23T21:13:49.401303+0000","last_active":"2023-09-23T21:13:59.195287+0000","last_peered":"2023-09-23T21:13:59.195287+0000","last_clean":"2023-09-23T21:11:06.743945+0000","last_became_active":"2023-09-23T21:13:31.902039+0000","last_became_peered":"2023-09-23T21:13:31.902039+0000","last_unstale":"2023-09-23T21:13:59.195287+0000","last_undegraded":"2023-09-23T21:13:59.195287+0000","last_fullsized":"2023-09-23T21:13:31.752922+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":438,"last_epoch_clean":439,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_clean_scrub_stamp":"2023-09-23T21:10:15.663179+0000","objects_scrubbed":0,"log_size":4715,"log_dups_size":0,"ondisk_log_size":4715,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":100204544,"num_objects":1529,"num_object_clones":0,"num_object_copies":4587,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1529,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3381,"num_write_kb":157120,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2404,"num_bytes_recovered":126156800,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[3,4],"avail_no_missing":["3","4"],"object_location_counts":[{"shards":"3,4","objects":1529}],"blocked_by":[],"up_primary":0,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":311,"reported_epoch":521,"state":"active+clean","last_fresh":"2023-09-23T21:13:58.467644+0000","last_change":"2023-09-23T21:12:10.771160+0000","last_active":"2023-09-23T21:13:58.467644+0000","last_peered":"2023-09-23T21:13:58.467644+0000","last_clean":"2023-09-23T21:13:58.467644+0000","last_became_active":"2023-09-23T21:12:10.757808+0000","last_became_peered":"2023-09-23T21:12:10.757808+0000","last_unstale":"2023-09-23T21:13:58.467644+0000","last_undegraded":"2023-09-23T21:13:58.467644+0000","last_fullsized":"2023-09-23T21:13:58.467644+0000","mapping_epoch":470,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:30:50.987139+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":98,"reported_epoch":523,"state":"active+clean","last_fresh":"2023-09-23T21:14:01.498464+0000","last_change":"2023-09-23T21:13:12.216602+0000","last_active":"2023-09-23T21:14:01.498464+0000","last_peered":"2023-09-23T21:14:01.498464+0000","last_clean":"2023-09-23T21:14:01.498464+0000","last_became_active":"2023-09-23T21:13:12.216204+0000","last_became_peered":"2023-09-23T21:13:12.216204+0000","last_unstale":"2023-09-23T21:14:01.498464+0000","last_undegraded":"2023-09-23T21:14:01.498464+0000","last_fullsized":"2023-09-23T21:14:01.498464+0000","mapping_epoch":493,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:23:37.043390+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1c","version":"79'161","reported_seq":890,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-09-23T21:14:02.775904+0000","last_change":"2023-09-23T21:13:13.402773+0000","last_active":"2023-09-23T21:14:02.775904+0000","last_peered":"2023-09-23T21:14:02.775904+0000","last_clean":"2023-09-23T21:14:02.775904+0000","last_became_active":"2023-09-23T21:13:13.402025+0000","last_became_peered":"2023-09-23T21:13:13.402025+0000","last_unstale":"2023-09-23T21:14:02.775904+0000","last_undegraded":"2023-09-23T21:14:02.775904+0000","last_fullsized":"2023-09-23T21:14:02.775904+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":479,"last_epoch_clean":495,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:22.040907+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:22.040907+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.013021176000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":163840,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":163840,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.17","version":"525'4850","reported_seq":9232,"reported_epoch":525,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T21:14:04.127104+0000","last_change":"2023-09-23T21:13:39.447969+0000","last_active":"2023-09-23T21:14:04.127104+0000","last_peered":"2023-09-23T21:14:04.127104+0000","last_clean":"2023-09-23T21:13:37.065694+0000","last_became_active":"2023-09-23T21:13:39.439739+0000","last_became_peered":"2023-09-23T21:13:39.439739+0000","last_unstale":"2023-09-23T21:14:04.127104+0000","last_undegraded":"2023-09-23T21:13:39.411853+0000","last_fullsized":"2023-09-23T21:13:39.404657+0000","mapping_epoch":508,"log_start":"0'0","ondisk_log_start":"0'0","created":438,"last_epoch_clean":487,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_clean_scrub_stamp":"2023-09-23T21:10:15.663179+0000","objects_scrubbed":0,"log_size":4850,"log_dups_size":0,"ondisk_log_size":4850,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":95059968,"num_objects":1451,"num_object_clones":0,"num_object_copies":4353,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1109,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1451,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3485,"num_write_kb":157920,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2112,"num_bytes_recovered":116719616,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,2],"acting":[3,5],"avail_no_missing":["3","4","5"],"object_location_counts":[{"shards":"3,4,5","objects":311},{"shards":"3,5","objects":1140}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":110,"reported_epoch":521,"state":"active+clean","last_fresh":"2023-09-23T21:13:58.466051+0000","last_change":"2023-09-23T21:12:10.779066+0000","last_active":"2023-09-23T21:13:58.466051+0000","last_peered":"2023-09-23T21:13:58.466051+0000","last_clean":"2023-09-23T21:13:58.466051+0000","last_became_active":"2023-09-23T21:12:10.778691+0000","last_became_peered":"2023-09-23T21:12:10.778691+0000","last_unstale":"2023-09-23T21:13:58.466051+0000","last_undegraded":"2023-09-23T21:13:58.466051+0000","last_fullsized":"2023-09-23T21:13:58.466051+0000","mapping_epoch":470,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:07:22.043268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,7],"acting":[2,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":189,"reported_epoch":521,"state":"active+clean","last_fresh":"2023-09-23T21:13:58.465769+0000","last_change":"2023-09-23T21:13:12.200767+0000","last_active":"2023-09-23T21:13:58.465769+0000","last_peered":"2023-09-23T21:13:58.465769+0000","last_clean":"2023-09-23T21:13:58.465769+0000","last_became_active":"2023-09-23T21:13:12.200435+0000","last_became_peered":"2023-09-23T21:13:12.200435+0000","last_unstale":"2023-09-23T21:13:58.465769+0000","last_undegraded":"2023-09-23T21:13:58.465769+0000","last_fullsized":"2023-09-23T21:13:58.465769+0000","mapping_epoch":493,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:15:44.937886+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.1d","version":"79'161","reported_seq":675,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-09-23T21:14:00.483617+0000","last_change":"2023-09-23T21:12:30.444331+0000","last_active":"2023-09-23T21:14:00.483617+0000","last_peered":"2023-09-23T21:14:00.483617+0000","last_clean":"2023-09-23T21:14:00.483617+0000","last_became_active":"2023-09-23T21:12:30.444001+0000","last_became_peered":"2023-09-23T21:12:30.444001+0000","last_unstale":"2023-09-23T21:14:00.483617+0000","last_undegraded":"2023-09-23T21:14:00.483617+0000","last_fullsized":"2023-09-23T21:14:00.483617+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":479,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:09:36.858077+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:09:36.858077+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0024987500000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"10.16","version":"524'4945","reported_seq":8450,"reported_epoch":524,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T21:14:03.093738+0000","last_change":"2023-09-23T21:13:14.424843+0000","last_active":"2023-09-23T21:14:03.093738+0000","last_peered":"2023-09-23T21:14:03.093738+0000","last_clean":"2023-09-23T21:11:45.652255+0000","last_became_active":"2023-09-23T21:13:14.414710+0000","last_became_peered":"2023-09-23T21:13:14.414710+0000","last_unstale":"2023-09-23T21:14:03.093738+0000","last_undegraded":"2023-09-23T21:13:13.211957+0000","last_fullsized":"2023-09-23T21:13:13.188859+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":438,"last_epoch_clean":460,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_clean_scrub_stamp":"2023-09-23T21:10:15.663179+0000","objects_scrubbed":0,"log_size":4945,"log_dups_size":0,"ondisk_log_size":4945,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":96141312,"num_objects":1467,"num_object_clones":0,"num_object_copies":4401,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":663,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1467,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3539,"num_write_kb":160192,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1670,"num_bytes_recovered":109445120,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,4],"acting":[4,5],"avail_no_missing":["4","5"],"object_location_counts":[{"shards":"1,4,5","objects":33},{"shards":"4,5","objects":1434}],"blocked_by":[],"up_primary":0,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":47,"reported_epoch":520,"state":"active+clean","last_fresh":"2023-09-23T21:13:53.464518+0000","last_change":"2023-09-23T21:13:38.380574+0000","last_active":"2023-09-23T21:13:53.464518+0000","last_peered":"2023-09-23T21:13:53.464518+0000","last_clean":"2023-09-23T21:13:53.464518+0000","last_became_active":"2023-09-23T21:13:38.373252+0000","last_became_peered":"2023-09-23T21:13:38.373252+0000","last_unstale":"2023-09-23T21:13:53.464518+0000","last_undegraded":"2023-09-23T21:13:53.464518+0000","last_fullsized":"2023-09-23T21:13:53.464518+0000","mapping_epoch":507,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:58:14.788031+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":11,"num_bytes_recovered":3324,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":98,"reported_epoch":523,"state":"active+clean","last_fresh":"2023-09-23T21:14:01.503772+0000","last_change":"2023-09-23T21:12:10.793610+0000","last_active":"2023-09-23T21:14:01.503772+0000","last_peered":"2023-09-23T21:14:01.503772+0000","last_clean":"2023-09-23T21:14:01.503772+0000","last_became_active":"2023-09-23T21:12:10.793081+0000","last_became_peered":"2023-09-23T21:12:10.793081+0000","last_unstale":"2023-09-23T21:14:01.503772+0000","last_undegraded":"2023-09-23T21:14:01.503772+0000","last_fullsized":"2023-09-23T21:14:01.503772+0000","mapping_epoch":470,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:34:36.291550+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":10,"num_pg":26,"stat_sum":{"num_bytes":3083993112,"num_objects":47061,"num_object_clones":0,"num_object_copies":141183,"num_objects_missing_on_primary":3546,"num_objects_missing":3546,"num_objects_degraded":25392,"num_objects_misplaced":5223,"num_objects_unfound":0,"num_objects_dirty":47061,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":100313,"num_write_kb":4309313,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":35529,"num_bytes_recovered":2148892672,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":11937193984,"data_stored":11937185840,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":124441,"ondisk_log_size":124441,"up":78,"acting":60,"num_store_stats":7},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":59,"num_bytes_recovered":8898,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":126976,"data_stored":22458,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":14322,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":31,"stat_sum":{"num_bytes":697708,"num_objects":3,"num_object_clones":0,"num_object_copies":9,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3,"num_whiteouts":0,"num_read":89,"num_read_kb":165,"num_write":44,"num_write_kb":440,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":649699,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2109440,"data_stored":2098784,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4991,"ondisk_log_size":4991,"up":93,"acting":93,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201683,"num_pgs":69,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1613980,"kb_used_data":1134676,"kb_used_omap":0,"kb_used_meta":479232,"kb_avail":92131172,"statfs":{"total":95995035648,"available":94342320128,"internally_reserved":0,"allocated":1161908224,"data_stored":1160397791,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":490733568},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:11:02 2023","interfaces":[{"interface":"back","average":{"1min":0.65600000000000003,"5min":1.3340000000000001,"15min":1.6699999999999999},"min":{"1min":0.23599999999999999,"5min":0.22800000000000001,"15min":0.216},"max":{"1min":1.966,"5min":7.0170000000000003,"15min":43.994999999999997},"last":0.29099999999999998},{"interface":"front","average":{"1min":0.68400000000000005,"5min":1.4179999999999999,"15min":1.7869999999999999},"min":{"1min":0.26800000000000002,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":1.9770000000000001,"5min":8.1120000000000001,"15min":44.206000000000003},"last":1.2410000000000001}]},{"osd":1,"last update":"Sat Sep 23 21:13:07 2023","interfaces":[{"interface":"back","average":{"1min":1.1739999999999999,"5min":1.851,"15min":2.1230000000000002},"min":{"1min":0.27500000000000002,"5min":0.27100000000000002,"15min":0.26700000000000002},"max":{"1min":6.4649999999999999,"5min":14.034000000000001,"15min":14.034000000000001},"last":0.50800000000000001},{"interface":"front","average":{"1min":1.966,"5min":1.889,"15min":2.0139999999999998},"min":{"1min":0.38,"5min":0.27100000000000002,"15min":0.26300000000000001},"max":{"1min":9.234,"5min":11.226000000000001,"15min":11.226000000000001},"last":3.137}]},{"osd":2,"last update":"Sat Sep 23 21:13:29 2023","interfaces":[{"interface":"back","average":{"1min":1.254,"5min":1.3839999999999999,"15min":1.9119999999999999},"min":{"1min":0.26400000000000001,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":6.4219999999999997,"5min":6.5469999999999997,"15min":14.478999999999999},"last":1.173},{"interface":"front","average":{"1min":1.298,"5min":1.5129999999999999,"15min":1.819},"min":{"1min":0.26700000000000002,"5min":0.26700000000000002,"15min":0.26500000000000001},"max":{"1min":9.1839999999999993,"5min":11.198,"15min":14.135},"last":0.37}]},{"osd":3,"last update":"Sat Sep 23 21:13:57 2023","interfaces":[{"interface":"back","average":{"1min":1.022,"5min":1.296,"15min":2.0590000000000002},"min":{"1min":0.33000000000000002,"5min":0.316,"15min":0.29099999999999998},"max":{"1min":6.4930000000000003,"5min":7.5190000000000001,"15min":31.786000000000001},"last":0.30599999999999999},{"interface":"front","average":{"1min":1.1160000000000001,"5min":1.3859999999999999,"15min":2.0409999999999999},"min":{"1min":0.37,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":5.9329999999999998,"5min":11.244999999999999,"15min":32.429000000000002},"last":3.9860000000000002}]},{"osd":4,"last update":"Sat Sep 23 21:13:53 2023","interfaces":[{"interface":"back","average":{"1min":1.5980000000000001,"5min":1.5209999999999999,"15min":1.097},"min":{"1min":0.28999999999999998,"5min":0.255,"15min":0.248},"max":{"1min":7.6609999999999996,"5min":11.297000000000001,"15min":11.297000000000001},"last":0.32500000000000001},{"interface":"front","average":{"1min":1.387,"5min":1.1679999999999999,"15min":1.0669999999999999},"min":{"1min":0.314,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":8.6059999999999999,"5min":11.093999999999999,"15min":12.752000000000001},"last":1.6930000000000001}]},{"osd":5,"last update":"Sat Sep 23 21:13:53 2023","interfaces":[{"interface":"back","average":{"1min":1.4670000000000001,"5min":1.0549999999999999,"15min":1.014},"min":{"1min":0.29899999999999999,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":7.5910000000000002,"5min":9.4789999999999992,"15min":9.4789999999999992},"last":1.575},{"interface":"front","average":{"1min":1.4950000000000001,"5min":1.3680000000000001,"15min":1.099},"min":{"1min":0.39700000000000002,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":7.6929999999999996,"5min":11.257,"15min":11.257},"last":0.34699999999999998}]},{"osd":6,"last update":"Sat Sep 23 21:13:07 2023","interfaces":[{"interface":"back","average":{"1min":2.6240000000000001,"5min":1.256,"15min":1.0429999999999999},"min":{"1min":0.46500000000000002,"5min":0.30099999999999999,"15min":0.28899999999999998},"max":{"1min":10.941000000000001,"5min":10.941000000000001,"15min":10.941000000000001},"last":0.46400000000000002},{"interface":"front","average":{"1min":2.0249999999999999,"5min":1.397,"15min":1.1200000000000001},"min":{"1min":0.40100000000000002,"5min":0.33100000000000002,"15min":0.32300000000000001},"max":{"1min":8.923,"5min":12.715,"15min":12.715},"last":0.42499999999999999}]}]},{"osd":6,"up_from":0,"seq":704374636767,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":1,"up_from":0,"seq":1503238553699,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":0,"up_from":493,"seq":2117418876939,"num_pgs":75,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2977612,"kb_used_data":2837380,"kb_used_omap":36,"kb_used_meta":140187,"kb_avail":90767540,"statfs":{"total":95995035648,"available":92945960960,"internally_reserved":0,"allocated":2905477120,"data_stored":2903048570,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":37336,"internal_metadata":143552040},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,2],"upper_bound":8},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.58199999999999996}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.52400000000000002}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.373}]},{"osd":4,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":2.9159999999999999}]},{"osd":5,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":2.5819999999999999}]},{"osd":6,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":1.278}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":1.1599999999999999}]}]},{"osd":2,"up_from":93,"seq":399431958811,"num_pgs":69,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2580576,"kb_used_data":1814936,"kb_used_omap":81,"kb_used_meta":765486,"kb_avail":91164576,"statfs":{"total":95995035648,"available":93352525824,"internally_reserved":0,"allocated":1858494464,"data_stored":1856053430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":83750,"internal_metadata":783857882},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,0,0,0,0,4],"upper_bound":128},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:11:05 2023","interfaces":[{"interface":"back","average":{"1min":0.61499999999999999,"5min":1.9730000000000001,"15min":2.1869999999999998},"min":{"1min":0.20200000000000001,"5min":0.20200000000000001,"15min":0.17999999999999999},"max":{"1min":3.3610000000000002,"5min":11.124000000000001,"15min":29.498000000000001},"last":0.59099999999999997},{"interface":"front","average":{"1min":0.49299999999999999,"5min":1.6299999999999999,"15min":2.3170000000000002},"min":{"1min":0.26300000000000001,"5min":0.193,"15min":0.193},"max":{"1min":1.377,"5min":12.183,"15min":17.251000000000001},"last":0.61899999999999999}]},{"osd":1,"last update":"Sat Sep 23 21:13:11 2023","interfaces":[{"interface":"back","average":{"1min":0.26400000000000001,"5min":1.7969999999999999,"15min":2.669},"min":{"1min":0.188,"5min":0.188,"15min":0.188},"max":{"1min":0.377,"5min":12.484999999999999,"15min":34.396999999999998},"last":0.47799999999999998},{"interface":"front","average":{"1min":0.30599999999999999,"5min":1.6819999999999999,"15min":2.9199999999999999},"min":{"1min":0.217,"5min":0.217,"15min":0.217},"max":{"1min":0.57099999999999995,"5min":11.289999999999999,"15min":34.462000000000003},"last":0.86599999999999999}]},{"osd":3,"last update":"Sat Sep 23 21:13:56 2023","interfaces":[{"interface":"back","average":{"1min":0.495,"5min":1.5429999999999999,"15min":2.6419999999999999},"min":{"1min":0.20599999999999999,"5min":0.20599999999999999,"15min":0.20599999999999999},"max":{"1min":1.679,"5min":12.951000000000001,"15min":41.057000000000002},"last":1.145},{"interface":"front","average":{"1min":0.749,"5min":1.661,"15min":2.915},"min":{"1min":0.28499999999999998,"5min":0.21299999999999999,"15min":0.21299999999999999},"max":{"1min":3.6989999999999998,"5min":13.159000000000001,"15min":41.012},"last":1.1679999999999999}]},{"osd":4,"last update":"Sat Sep 23 21:13:11 2023","interfaces":[{"interface":"back","average":{"1min":1.5900000000000001,"5min":1.494,"15min":1.5289999999999999},"min":{"1min":0.30599999999999999,"5min":0.249,"15min":0.249},"max":{"1min":10.266999999999999,"5min":10.266999999999999,"15min":10.266999999999999},"last":1.546},{"interface":"front","average":{"1min":1.476,"5min":1.573,"15min":1.952},"min":{"1min":0.29399999999999998,"5min":0.29399999999999998,"15min":0.26700000000000002},"max":{"1min":6.7720000000000002,"5min":8.6780000000000008,"15min":14.039999999999999},"last":1.2}]},{"osd":5,"last update":"Sat Sep 23 21:13:11 2023","interfaces":[{"interface":"back","average":{"1min":1.0660000000000001,"5min":1.46,"15min":1.603},"min":{"1min":0.34399999999999997,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":3.4390000000000001,"5min":9.5039999999999996,"15min":10.069000000000001},"last":2.512},{"interface":"front","average":{"1min":0.97299999999999998,"5min":1.546,"15min":1.8939999999999999},"min":{"1min":0.29699999999999999,"5min":0.29699999999999999,"15min":0.29399999999999998},"max":{"1min":7.4969999999999999,"5min":8.7270000000000003,"15min":13.98},"last":1.1339999999999999}]},{"osd":6,"last update":"Sat Sep 23 21:13:11 2023","interfaces":[{"interface":"back","average":{"1min":1.7989999999999999,"5min":1.8120000000000001,"15min":2.1320000000000001},"min":{"1min":0.373,"5min":0.373,"15min":0.35699999999999998},"max":{"1min":7.3310000000000004,"5min":8.7680000000000007,"15min":14.016},"last":0.78800000000000003},{"interface":"front","average":{"1min":2.3940000000000001,"5min":2.0510000000000002,"15min":2.2959999999999998},"min":{"1min":0.40699999999999997,"5min":0.38100000000000001,"15min":0.35599999999999998},"max":{"1min":9.3469999999999995,"5min":9.3469999999999995,"15min":14.068},"last":0.84499999999999997}]},{"osd":7,"last update":"Sat Sep 23 21:13:11 2023","interfaces":[{"interface":"back","average":{"1min":1.609,"5min":1.6299999999999999,"15min":1.5620000000000001},"min":{"1min":0.42899999999999999,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":7.694,"5min":7.694,"15min":13.427},"last":0.89600000000000002},{"interface":"front","average":{"1min":2.0499999999999998,"5min":1.657,"15min":1.74},"min":{"1min":0.34799999999999998,"5min":0.34799999999999998,"15min":0.32200000000000001},"max":{"1min":10.359,"5min":10.359,"15min":10.92},"last":1.1000000000000001}]}]},{"osd":3,"up_from":98,"seq":420906795285,"num_pgs":82,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2464360,"kb_used_data":1617952,"kb_used_omap":7,"kb_used_meta":846392,"kb_avail":91280792,"statfs":{"total":95995035648,"available":93471531008,"internally_reserved":0,"allocated":1656782848,"data_stored":1654352807,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7576,"internal_metadata":866706024},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,2],"upper_bound":4},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:10:53 2023","interfaces":[{"interface":"back","average":{"1min":0.59699999999999998,"5min":2.173,"15min":2.6000000000000001},"min":{"1min":0.223,"5min":0.19900000000000001,"15min":0.188},"max":{"1min":3.032,"5min":14.08,"15min":27.962},"last":0.32300000000000001},{"interface":"front","average":{"1min":0.85699999999999998,"5min":1.8580000000000001,"15min":2.6779999999999999},"min":{"1min":0.26000000000000001,"5min":0.20999999999999999,"15min":0.19800000000000001},"max":{"1min":4.0090000000000003,"5min":7.8920000000000003,"15min":28.058},"last":0.69999999999999996}]},{"osd":1,"last update":"Sat Sep 23 21:13:11 2023","interfaces":[{"interface":"back","average":{"1min":0.40200000000000002,"5min":1.9059999999999999,"15min":2.2989999999999999},"min":{"1min":0.20000000000000001,"5min":0.20000000000000001,"15min":0.20000000000000001},"max":{"1min":2.27,"5min":22.577999999999999,"15min":22.577999999999999},"last":2.7629999999999999},{"interface":"front","average":{"1min":0.33100000000000002,"5min":1.9570000000000001,"15min":2.2909999999999999},"min":{"1min":0.216,"5min":0.216,"15min":0.216},"max":{"1min":1.268,"5min":21.617999999999999,"15min":21.617999999999999},"last":1.3009999999999999}]},{"osd":2,"last update":"Sat Sep 23 21:14:00 2023","interfaces":[{"interface":"back","average":{"1min":3.6669999999999998,"5min":1.9279999999999999,"15min":2.6640000000000001},"min":{"1min":0.221,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":56.317999999999998,"5min":56.317999999999998,"15min":56.317999999999998},"last":0.248},{"interface":"front","average":{"1min":3.8479999999999999,"5min":2.1499999999999999,"15min":2.702},"min":{"1min":0.29699999999999999,"5min":0.216,"15min":0.216},"max":{"1min":55.988,"5min":55.988,"15min":55.988},"last":2.7869999999999999}]},{"osd":4,"last update":"Sat Sep 23 21:14:00 2023","interfaces":[{"interface":"back","average":{"1min":0.71499999999999997,"5min":1.3029999999999999,"15min":1.8779999999999999},"min":{"1min":0.30299999999999999,"5min":0.30299999999999999,"15min":0.28399999999999997},"max":{"1min":5.0819999999999999,"5min":21.478999999999999,"15min":21.478999999999999},"last":2.0760000000000001},{"interface":"front","average":{"1min":1.3009999999999999,"5min":1.5760000000000001,"15min":2.0070000000000001},"min":{"1min":0.33100000000000002,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":4.1909999999999998,"5min":22.402000000000001,"15min":22.402000000000001},"last":3.4750000000000001}]},{"osd":5,"last update":"Sat Sep 23 21:14:00 2023","interfaces":[{"interface":"back","average":{"1min":0.93600000000000005,"5min":1.3120000000000001,"15min":2.012},"min":{"1min":0.32200000000000001,"5min":0.311,"15min":0.311},"max":{"1min":5.04,"5min":21.562000000000001,"15min":21.562000000000001},"last":0.375},{"interface":"front","average":{"1min":0.83699999999999997,"5min":1.339,"15min":2.2839999999999998},"min":{"1min":0.34300000000000003,"5min":0.31,"15min":0.31},"max":{"1min":3.4780000000000002,"5min":8.8450000000000006,"15min":14.029},"last":1.367}]},{"osd":6,"last update":"Sat Sep 23 21:13:14 2023","interfaces":[{"interface":"back","average":{"1min":1.48,"5min":2.0219999999999998,"15min":2.097},"min":{"1min":0.34999999999999998,"5min":0.34999999999999998,"15min":0.33500000000000002},"max":{"1min":10.102,"5min":22.446999999999999,"15min":22.446999999999999},"last":3.0510000000000002},{"interface":"front","average":{"1min":2.0790000000000002,"5min":1.831,"15min":2.294},"min":{"1min":0.39900000000000002,"5min":0.39000000000000001,"15min":0.373},"max":{"1min":10.148,"5min":12.375999999999999,"15min":13.576000000000001},"last":1.3340000000000001}]},{"osd":7,"last update":"Sat Sep 23 21:14:00 2023","interfaces":[{"interface":"back","average":{"1min":1.032,"5min":1.6120000000000001,"15min":2.1299999999999999},"min":{"1min":0.41199999999999998,"5min":0.38400000000000001,"15min":0.33900000000000002},"max":{"1min":5.0090000000000003,"5min":21.518999999999998,"15min":21.518999999999998},"last":1.7909999999999999},{"interface":"front","average":{"1min":0.92900000000000005,"5min":1.7050000000000001,"15min":2.4199999999999999},"min":{"1min":0.45800000000000002,"5min":0.442,"15min":0.38900000000000001},"max":{"1min":3.2679999999999998,"5min":9.9879999999999995,"15min":13.603999999999999},"last":1.8100000000000001}]}]},{"osd":4,"up_from":31,"seq":133143986591,"num_pgs":77,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2037604,"kb_used_data":1593884,"kb_used_omap":0,"kb_used_meta":443712,"kb_avail":91707548,"statfs":{"total":95995035648,"available":93908529152,"internally_reserved":0,"allocated":1632137216,"data_stored":1630619422,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":454361088},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,1,2,0,0,0,3],"upper_bound":256},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:11:11 2023","interfaces":[{"interface":"back","average":{"1min":0.98499999999999999,"5min":1.6819999999999999,"15min":1.5549999999999999},"min":{"1min":0.20200000000000001,"5min":0.20200000000000001,"15min":0.20200000000000001},"max":{"1min":4.944,"5min":11.537000000000001,"15min":11.784000000000001},"last":1.2589999999999999},{"interface":"front","average":{"1min":1.1499999999999999,"5min":2.2549999999999999,"15min":1.887},"min":{"1min":0.28799999999999998,"5min":0.28000000000000003,"15min":0.254},"max":{"1min":4.9749999999999996,"5min":19.109999999999999,"15min":19.109999999999999},"last":2.1179999999999999}]},{"osd":1,"last update":"Sat Sep 23 21:13:10 2023","interfaces":[{"interface":"back","average":{"1min":1.389,"5min":1.48,"15min":2.952},"min":{"1min":0.251,"5min":0.221,"15min":0.221},"max":{"1min":7.8609999999999998,"5min":14.798999999999999,"15min":37.960000000000001},"last":0.90500000000000003},{"interface":"front","average":{"1min":1.081,"5min":1.393,"15min":2.7949999999999999},"min":{"1min":0.32700000000000001,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":4.3639999999999999,"5min":14.403,"15min":37.933},"last":0.89400000000000002}]},{"osd":2,"last update":"Sat Sep 23 21:13:13 2023","interfaces":[{"interface":"back","average":{"1min":1.1399999999999999,"5min":1.5920000000000001,"15min":1.8180000000000001},"min":{"1min":0.26000000000000001,"5min":0.247,"15min":0.247},"max":{"1min":4.5019999999999998,"5min":16.928000000000001,"15min":16.928000000000001},"last":0.47899999999999998},{"interface":"front","average":{"1min":1.304,"5min":1.825,"15min":1.833},"min":{"1min":0.28599999999999998,"5min":0.28599999999999998,"15min":0.27800000000000002},"max":{"1min":4.5570000000000004,"5min":14.417999999999999,"15min":14.417999999999999},"last":0.33500000000000002}]},{"osd":3,"last update":"Sat Sep 23 21:13:46 2023","interfaces":[{"interface":"back","average":{"1min":1.0229999999999999,"5min":1.6299999999999999,"15min":2.1339999999999999},"min":{"1min":0.25600000000000001,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":4.2480000000000002,"5min":14.585000000000001,"15min":15.01},"last":0.60899999999999999},{"interface":"front","average":{"1min":1.135,"5min":1.647,"15min":2.0249999999999999},"min":{"1min":0.36699999999999999,"5min":0.36099999999999999,"15min":0.30199999999999999},"max":{"1min":4.4649999999999999,"5min":14.366,"15min":15.004},"last":0.61899999999999999}]},{"osd":5,"last update":"Sat Sep 23 21:13:02 2023","interfaces":[{"interface":"back","average":{"1min":2.1099999999999999,"5min":1.212,"15min":0.95199999999999996},"min":{"1min":0.222,"5min":0.222,"15min":0.222},"max":{"1min":9.4909999999999997,"5min":14.298,"15min":14.298},"last":0.30099999999999999},{"interface":"front","average":{"1min":1.9179999999999999,"5min":1.1599999999999999,"15min":1.0600000000000001},"min":{"1min":0.27200000000000002,"5min":0.26600000000000001,"15min":0.255},"max":{"1min":6.8730000000000002,"5min":9.8979999999999997,"15min":9.8979999999999997},"last":0.42099999999999999}]},{"osd":6,"last update":"Sat Sep 23 21:13:10 2023","interfaces":[{"interface":"back","average":{"1min":2.923,"5min":1.3859999999999999,"15min":1.2330000000000001},"min":{"1min":0.34499999999999997,"5min":0.28599999999999998,"15min":0.26800000000000002},"max":{"1min":24.452999999999999,"5min":24.452999999999999,"15min":24.452999999999999},"last":0.34399999999999997},{"interface":"front","average":{"1min":1.6859999999999999,"5min":1.171,"15min":0.94499999999999995},"min":{"1min":0.38800000000000001,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":11.409000000000001,"5min":14.387,"15min":14.387},"last":0.44700000000000001}]},{"osd":7,"last update":"Sat Sep 23 21:13:46 2023","interfaces":[{"interface":"back","average":{"1min":1.294,"5min":1.268,"15min":1.0529999999999999},"min":{"1min":0.375,"5min":0.30099999999999999,"15min":0.29299999999999998},"max":{"1min":4.532,"5min":16.890999999999998,"15min":16.890999999999998},"last":0.40999999999999998},{"interface":"front","average":{"1min":1.387,"5min":1.155,"15min":1.1559999999999999},"min":{"1min":0.375,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":7.1879999999999997,"5min":8.5830000000000002,"15min":11.170999999999999},"last":0.55500000000000005}]}]},{"osd":5,"up_from":39,"seq":167503724957,"num_pgs":63,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1832656,"kb_used_data":1345992,"kb_used_omap":0,"kb_used_meta":486656,"kb_avail":91912496,"statfs":{"total":95995035648,"available":94118395904,"internally_reserved":0,"allocated":1378295808,"data_stored":1376794204,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":498335744},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:11:02 2023","interfaces":[{"interface":"back","average":{"1min":0.627,"5min":1.722,"15min":1.8},"min":{"1min":0.222,"5min":0.19700000000000001,"15min":0.19700000000000001},"max":{"1min":2.0840000000000001,"5min":25.222999999999999,"15min":25.222999999999999},"last":0.26200000000000001},{"interface":"front","average":{"1min":0.67200000000000004,"5min":1.6659999999999999,"15min":1.861},"min":{"1min":0.247,"5min":0.247,"15min":0.23400000000000001},"max":{"1min":2.1970000000000001,"5min":25.457999999999998,"15min":25.457999999999998},"last":0.28499999999999998}]},{"osd":1,"last update":"Sat Sep 23 21:13:04 2023","interfaces":[{"interface":"back","average":{"1min":0.60099999999999998,"5min":1.4370000000000001,"15min":1.804},"min":{"1min":0.23499999999999999,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":2.0379999999999998,"5min":9.4429999999999996,"15min":25.239999999999998},"last":0.78700000000000003},{"interface":"front","average":{"1min":1.6040000000000001,"5min":1.883,"15min":1.79},"min":{"1min":0.29699999999999999,"5min":0.27400000000000002,"15min":0.252},"max":{"1min":3.956,"5min":10.686,"15min":25.253},"last":0.35599999999999998}]},{"osd":2,"last update":"Sat Sep 23 21:13:26 2023","interfaces":[{"interface":"back","average":{"1min":1.871,"5min":1.6679999999999999,"15min":1.7909999999999999},"min":{"1min":0.30299999999999999,"5min":0.26200000000000001,"15min":0.23499999999999999},"max":{"1min":6.085,"5min":11.939,"15min":12.634},"last":0.432},{"interface":"front","average":{"1min":0.71699999999999997,"5min":1.3979999999999999,"15min":1.915},"min":{"1min":0.27600000000000002,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":3.165,"5min":9.7170000000000005,"15min":12.300000000000001},"last":0.84099999999999997}]},{"osd":3,"last update":"Sat Sep 23 21:13:45 2023","interfaces":[{"interface":"back","average":{"1min":1.4279999999999999,"5min":1.8089999999999999,"15min":2.6560000000000001},"min":{"1min":0.33400000000000002,"5min":0.307,"15min":0.29299999999999998},"max":{"1min":6.1859999999999999,"5min":14.242000000000001,"15min":61.667999999999999},"last":1.169},{"interface":"front","average":{"1min":1.4390000000000001,"5min":2.0089999999999999,"15min":2.6429999999999998},"min":{"1min":0.34899999999999998,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":6.1200000000000001,"5min":14.396000000000001,"15min":62.276000000000003},"last":0.58599999999999997}]},{"osd":4,"last update":"Sat Sep 23 21:13:59 2023","interfaces":[{"interface":"back","average":{"1min":1.452,"5min":1.387,"15min":1.139},"min":{"1min":0.28799999999999998,"5min":0.27000000000000002,"15min":0.23400000000000001},"max":{"1min":11.156000000000001,"5min":12.776,"15min":12.776},"last":0.32600000000000001},{"interface":"front","average":{"1min":1.5109999999999999,"5min":1.1799999999999999,"15min":1.1319999999999999},"min":{"1min":0.28000000000000003,"5min":0.27000000000000002,"15min":0.26900000000000002},"max":{"1min":9.7119999999999997,"5min":9.7119999999999997,"15min":11.973000000000001},"last":0.39400000000000002}]},{"osd":6,"last update":"Sat Sep 23 21:13:04 2023","interfaces":[{"interface":"back","average":{"1min":1.8340000000000001,"5min":0.98799999999999999,"15min":1.0449999999999999},"min":{"1min":0.313,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":7.7030000000000003,"5min":7.7030000000000003,"15min":11.128},"last":0.72899999999999998},{"interface":"front","average":{"1min":3.004,"5min":1.4139999999999999,"15min":1.1830000000000001},"min":{"1min":0.51000000000000001,"5min":0.317,"15min":0.317},"max":{"1min":10.026999999999999,"5min":10.718999999999999,"15min":11.112},"last":0.51100000000000001}]},{"osd":7,"last update":"Sat Sep 23 21:13:45 2023","interfaces":[{"interface":"back","average":{"1min":1.3,"5min":0.95099999999999996,"15min":1.804},"min":{"1min":0.34300000000000003,"5min":0.31,"15min":0.309},"max":{"1min":5.5659999999999998,"5min":5.5659999999999998,"15min":189.405},"last":0.81399999999999995},{"interface":"front","average":{"1min":2.004,"5min":1.5009999999999999,"15min":1.9159999999999999},"min":{"1min":0.40200000000000002,"5min":0.35399999999999998,"15min":0.32500000000000001},"max":{"1min":11.183999999999999,"5min":11.183999999999999,"15min":189.685},"last":0.66100000000000003}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8488,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1259,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5322,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":714,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":5434,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":4838,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":2875916288,"data_stored":2875916288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1434714112,"data_stored":1434714112,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1850867712,"data_stored":1850867712,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1649217536,"data_stored":1649213464,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1613631488,"data_stored":1613627416,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1360068608,"data_stored":1360068608,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1152778240,"data_stored":1152778240,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T21:14:05.607 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T21:14:05.608 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_blocked_ops 2023-09-23T21:14:05.638 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T21:14:05.661 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:14:05.721 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:14:05.775 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:14:05.780 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:14:05.781 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:14:05.881 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:14:05.920 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:14:05.964 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:14:05.970 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:14:05.971 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:14:06.071 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:14:06.125 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:14:06.161 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:14:06.164 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:14:06.164 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:14:06.265 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:14:06.300 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:14:06.327 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T21:14:06.353 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T21:14:06.355 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:14:06.359 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:14:06.359 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:14:06.391 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:14:06.393 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:14:06.459 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:14:06.501 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:14:06.566 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:14:06 smithi070 ceph-mon[168282]: pgmap v1313: 129 pgs: 3 active+recovering+undersized+remapped, 3 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 15 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 106 active+clean; 2.9 GiB data, 13 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 442 op/s; 25394/141258 objects degraded (17.977%); 5223/141258 objects misplaced (3.697%); 7.0 MiB/s, 204 objects/s recovering 2023-09-23T21:14:06.567 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:14:06 smithi070 ceph-mon[168282]: from='client.46652 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T21:14:06.567 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:14:06 smithi070 ceph-mon[168282]: Health check update: Degraded data redundancy: 25394/141258 objects degraded (17.977%), 19 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T21:14:06.600 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:14:06.604 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:14:06.605 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:14:06.705 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:14:06.738 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:14:06 smithi026 ceph-mon[179688]: pgmap v1313: 129 pgs: 3 active+recovering+undersized+remapped, 3 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 15 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 106 active+clean; 2.9 GiB data, 13 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 442 op/s; 25394/141258 objects degraded (17.977%); 5223/141258 objects misplaced (3.697%); 7.0 MiB/s, 204 objects/s recovering 2023-09-23T21:14:06.738 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:14:06 smithi026 ceph-mon[179688]: from='client.46652 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T21:14:06.738 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:14:06 smithi026 ceph-mon[179688]: Health check update: Degraded data redundancy: 25394/141258 objects degraded (17.977%), 19 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T21:14:06.738 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:14:06 smithi026 ceph-mon[175151]: pgmap v1313: 129 pgs: 3 active+recovering+undersized+remapped, 3 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 15 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 106 active+clean; 2.9 GiB data, 13 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 442 op/s; 25394/141258 objects degraded (17.977%); 5223/141258 objects misplaced (3.697%); 7.0 MiB/s, 204 objects/s recovering 2023-09-23T21:14:06.738 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:14:06 smithi026 ceph-mon[175151]: from='client.46652 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T21:14:06.738 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:14:06 smithi026 ceph-mon[175151]: Health check update: Degraded data redundancy: 25394/141258 objects degraded (17.977%), 19 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T21:14:06.750 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:14:06.809 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:14:06.815 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:14:06.816 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:14:06.917 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:14:06.972 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:14:07.083 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:14:07.086 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:14:07.086 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:14:07.187 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:14:07.220 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:14:07.284 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:14:07.288 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:14:07.289 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:14:07.389 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:14:07.419 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:14:07.468 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T21:14:07.474 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:14:07.475 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:14:07.576 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:14:07.621 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:14:07.685 INFO:teuthology.orchestra.run.smithi070.stdout:ERROR: (22) Invalid argument 2023-09-23T21:14:07.685 INFO:teuthology.orchestra.run.smithi070.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T21:14:07.700 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:14:07.709 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:14:07.710 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:14:07.810 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:14:07.851 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:14:07.851 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:14:08.089 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:14:08.093 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:14:08.093 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:14:08.130 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_historic_ops 2023-09-23T21:14:08.194 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:14:08.251 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:14:08.409 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:14:08.415 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:14:08.416 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:14:08.505 INFO:teuthology.orchestra.run.smithi026.stdout: 2023-09-23T21:14:08.510 INFO:teuthology.orchestra.run.smithi026.stderr:dumped all 2023-09-23T21:14:08.517 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:14:08.527 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:14:08 smithi026 ceph-mon[175151]: pgmap v1314: 129 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 15 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 108 active+clean; 2.8 GiB data, 13 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 479 op/s; 25394/138666 objects degraded (18.313%); 5155/138666 objects misplaced (3.718%); 5.3 MiB/s, 231 objects/s recovering 2023-09-23T21:14:08.527 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:14:08 smithi026 ceph-mon[179688]: pgmap v1314: 129 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 15 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 108 active+clean; 2.8 GiB data, 13 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 479 op/s; 25394/138666 objects degraded (18.313%); 5155/138666 objects misplaced (3.718%); 5.3 MiB/s, 231 objects/s recovering 2023-09-23T21:14:08.557 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:14:08.601 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:14:08.609 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:14:08.609 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:14:08.710 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:14:08.751 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:14:08.802 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:14:08.805 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:14:08.806 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:14:08.814 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:14:08 smithi070 ceph-mon[168282]: pgmap v1314: 129 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 15 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 108 active+clean; 2.8 GiB data, 13 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 479 op/s; 25394/138666 objects degraded (18.313%); 5155/138666 objects misplaced (3.718%); 5.3 MiB/s, 231 objects/s recovering 2023-09-23T21:14:08.907 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:14:08.910 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T21:14:08.961 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:14:09.028 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T21:14:09.036 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:14:09.037 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:14:09.137 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:14:09.174 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:14:09.538 INFO:teuthology.orchestra.run.smithi026.stderr:nodeep-scrub is set 2023-09-23T21:14:09.750 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:14:09 smithi070 ceph-mon[168282]: from='client.56401 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T21:14:09.750 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:14:09 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/1061583611' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:14:09.881 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 21:14:09 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[164029]: ::ffff:172.21.15.70 - - [23/Sep/2023:21:14:09] "GET /metrics HTTP/1.1" 200 36407 "" "Prometheus/2.43.0" 2023-09-23T21:14:09.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:14:09 smithi026 ceph-mon[175151]: from='client.56401 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T21:14:09.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:14:09 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/1061583611' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:14:09.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:14:09 smithi026 ceph-mon[179688]: from='client.56401 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T21:14:09.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:14:09 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/1061583611' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:14:10.355 INFO:teuthology.orchestra.run.smithi070.stdout:ERROR: (22) Invalid argument 2023-09-23T21:14:10.356 INFO:teuthology.orchestra.run.smithi070.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T21:14:10.805 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:14:10 smithi070 ceph-mon[168282]: pgmap v1315: 129 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 15 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 108 active+clean; 2.8 GiB data, 13 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 429 op/s; 25135/135597 objects degraded (18.537%); 4995/135597 objects misplaced (3.684%); 8.0 MiB/s, 188 objects/s recovering 2023-09-23T21:14:10.805 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:14:10 smithi070 ceph-mon[168282]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:14:10.805 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:14:10 smithi070 ceph-mon[168282]: failed to encode map e526 with expected crc 2023-09-23T21:14:10.805 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:14:10 smithi070 ceph-mon[168282]: failed to encode map e526 with expected crc 2023-09-23T21:14:10.805 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:14:10 smithi070 ceph-mon[168282]: failed to encode map e526 with expected crc 2023-09-23T21:14:10.805 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:14:10 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/1061583611' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T21:14:10.805 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:14:10 smithi070 ceph-mon[168282]: osdmap e526: 8 total, 8 up, 6 in 2023-09-23T21:14:10.827 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_ops_in_flight 2023-09-23T21:14:10.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:14:10 smithi026 ceph-mon[179688]: pgmap v1315: 129 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 15 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 108 active+clean; 2.8 GiB data, 13 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 429 op/s; 25135/135597 objects degraded (18.537%); 4995/135597 objects misplaced (3.684%); 8.0 MiB/s, 188 objects/s recovering 2023-09-23T21:14:10.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:14:10 smithi026 ceph-mon[179688]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:14:10.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:14:10 smithi026 ceph-mon[179688]: failed to encode map e526 with expected crc 2023-09-23T21:14:10.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:14:10 smithi026 ceph-mon[179688]: failed to encode map e526 with expected crc 2023-09-23T21:14:10.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:14:10 smithi026 ceph-mon[179688]: failed to encode map e526 with expected crc 2023-09-23T21:14:10.883 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:14:10 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/1061583611' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T21:14:10.883 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:14:10 smithi026 ceph-mon[179688]: osdmap e526: 8 total, 8 up, 6 in 2023-09-23T21:14:10.883 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:14:10 smithi026 ceph-mon[175151]: pgmap v1315: 129 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 15 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 108 active+clean; 2.8 GiB data, 13 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 429 op/s; 25135/135597 objects degraded (18.537%); 4995/135597 objects misplaced (3.684%); 8.0 MiB/s, 188 objects/s recovering 2023-09-23T21:14:10.883 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:14:10 smithi026 ceph-mon[175151]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:14:10.883 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:14:10 smithi026 ceph-mon[175151]: failed to encode map e526 with expected crc 2023-09-23T21:14:10.883 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:14:10 smithi026 ceph-mon[175151]: failed to encode map e526 with expected crc 2023-09-23T21:14:10.883 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:14:10 smithi026 ceph-mon[175151]: failed to encode map e526 with expected crc 2023-09-23T21:14:10.883 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:14:10 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/1061583611' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T21:14:10.883 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:14:10 smithi026 ceph-mon[175151]: osdmap e526: 8 total, 8 up, 6 in 2023-09-23T21:14:10.891 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:14:10.892 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:14:10.893 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:14:10.938 INFO:teuthology.orchestra.run.smithi026.stdout:{"pg_ready":true,"pg_map":{"version":1314,"stamp":"2023-09-23T21:14:07.044739+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":3028073220,"num_objects":46222,"num_object_clones":0,"num_object_copies":138666,"num_objects_missing_on_primary":3546,"num_objects_missing":3546,"num_objects_degraded":25394,"num_objects_misplaced":5155,"num_objects_unfound":0,"num_objects_dirty":46222,"num_whiteouts":0,"num_read":90,"num_read_kb":166,"num_write":101282,"num_write_kb":4309782,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":36048,"num_bytes_recovered":2149551269,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":130151,"ondisk_log_size":130151,"up":387,"acting":371,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":435,"num_osds":6,"num_per_pool_osds":6,"num_per_pool_omap_osds":6,"kb":562470912,"kb_used":13289496,"kb_used_data":10127592,"kb_used_omap":125,"kb_used_meta":3161666,"kb_avail":549181416,"statfs":{"total":575970213888,"available":562361769984,"internally_reserved":0,"allocated":10370654208,"data_stored":10358768321,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":128662,"internal_metadata":3237546346},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,2,1,2,0,0,1,4,3,3,2],"upper_bound":2048},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-188612608,"num_objects":-2878,"num_object_clones":0,"num_object_copies":-8634,"num_objects_missing_on_primary":-197,"num_objects_missing":-197,"num_objects_degraded":-3369,"num_objects_misplaced":1211,"num_objects_unfound":0,"num_objects_dirty":-2878,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2878,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1387,"num_bytes_recovered":33226774,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"6.003508"},"pg_stats":[{"pgid":"1.18","version":"79'161","reported_seq":645,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-09-23T21:14:02.766700+0000","last_change":"2023-09-23T21:12:30.438733+0000","last_active":"2023-09-23T21:14:02.766700+0000","last_peered":"2023-09-23T21:14:02.766700+0000","last_clean":"2023-09-23T21:14:02.766700+0000","last_became_active":"2023-09-23T21:12:30.438356+0000","last_became_peered":"2023-09-23T21:12:30.438356+0000","last_unstale":"2023-09-23T21:14:02.766700+0000","last_undegraded":"2023-09-23T21:14:02.766700+0000","last_fullsized":"2023-09-23T21:14:02.766700+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":479,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:06:52.098882+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:06:52.098882+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0092173299999999993,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":131344,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"10.13","version":"524'4895","reported_seq":7291,"reported_epoch":524,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T21:14:03.180531+0000","last_change":"2023-09-23T21:13:49.840403+0000","last_active":"2023-09-23T21:14:03.180531+0000","last_peered":"2023-09-23T21:14:03.180531+0000","last_clean":"2023-09-23T21:11:26.718194+0000","last_became_active":"2023-09-23T21:13:48.181112+0000","last_became_peered":"2023-09-23T21:13:48.181112+0000","last_unstale":"2023-09-23T21:14:03.180531+0000","last_undegraded":"2023-09-23T21:13:48.139049+0000","last_fullsized":"2023-09-23T21:13:45.405990+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":438,"last_epoch_clean":439,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_clean_scrub_stamp":"2023-09-23T21:10:15.663179+0000","objects_scrubbed":0,"log_size":4895,"log_dups_size":0,"ondisk_log_size":4895,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":98729984,"num_objects":1507,"num_object_clones":0,"num_object_copies":4521,"num_objects_missing_on_primary":1137,"num_objects_missing":1137,"num_objects_degraded":1337,"num_objects_misplaced":30,"num_objects_unfound":0,"num_objects_dirty":1507,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3527,"num_write_kb":161056,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1011,"num_bytes_recovered":66191360,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[4,3],"avail_no_missing":["5"],"object_location_counts":[{"shards":"0,2,4,5","objects":307},{"shards":"2,3,5","objects":1137},{"shards":"2,4,5","objects":28},{"shards":"3,4","objects":35}],"blocked_by":[],"up_primary":2,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1a","version":"68'1","reported_seq":114,"reported_epoch":523,"state":"active+clean","last_fresh":"2023-09-23T21:14:01.500016+0000","last_change":"2023-09-23T21:13:13.291403+0000","last_active":"2023-09-23T21:14:01.500016+0000","last_peered":"2023-09-23T21:14:01.500016+0000","last_clean":"2023-09-23T21:14:01.500016+0000","last_became_active":"2023-09-23T21:13:13.291123+0000","last_became_peered":"2023-09-23T21:13:13.291123+0000","last_unstale":"2023-09-23T21:14:01.500016+0000","last_undegraded":"2023-09-23T21:14:01.500016+0000","last_fullsized":"2023-09-23T21:14:01.500016+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":495,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:45:15.427430+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,0],"acting":[4,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":183,"reported_epoch":523,"state":"active+clean","last_fresh":"2023-09-23T21:14:01.499679+0000","last_change":"2023-09-23T21:12:10.767963+0000","last_active":"2023-09-23T21:14:01.499679+0000","last_peered":"2023-09-23T21:14:01.499679+0000","last_clean":"2023-09-23T21:14:01.499679+0000","last_became_active":"2023-09-23T21:12:10.767126+0000","last_became_peered":"2023-09-23T21:12:10.767126+0000","last_unstale":"2023-09-23T21:14:01.499679+0000","last_undegraded":"2023-09-23T21:14:01.499679+0000","last_fullsized":"2023-09-23T21:14:01.499679+0000","mapping_epoch":470,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:45:47.138277+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,5],"acting":[4,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1b","version":"79'161","reported_seq":528,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-09-23T21:14:02.771426+0000","last_change":"2023-09-23T21:13:13.373548+0000","last_active":"2023-09-23T21:14:02.771426+0000","last_peered":"2023-09-23T21:14:02.771426+0000","last_clean":"2023-09-23T21:14:02.771426+0000","last_became_active":"2023-09-23T21:13:13.372845+0000","last_became_peered":"2023-09-23T21:13:13.372845+0000","last_unstale":"2023-09-23T21:14:02.771426+0000","last_undegraded":"2023-09-23T21:14:02.771426+0000","last_fullsized":"2023-09-23T21:14:02.771426+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":479,"last_epoch_clean":495,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:05:55.772264+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:05:55.772264+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.001862507,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.10","version":"524'4873","reported_seq":11364,"reported_epoch":524,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T21:14:03.165245+0000","last_change":"2023-09-23T21:13:36.071975+0000","last_active":"2023-09-23T21:14:03.165245+0000","last_peered":"2023-09-23T21:14:03.165245+0000","last_clean":"2023-09-23T21:11:26.716051+0000","last_became_active":"2023-09-23T21:13:33.619010+0000","last_became_peered":"2023-09-23T21:13:33.619010+0000","last_unstale":"2023-09-23T21:14:03.165245+0000","last_undegraded":"2023-09-23T21:13:33.443511+0000","last_fullsized":"2023-09-23T21:13:31.753181+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":438,"last_epoch_clean":450,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_clean_scrub_stamp":"2023-09-23T21:10:15.663179+0000","objects_scrubbed":0,"log_size":4873,"log_dups_size":0,"ondisk_log_size":4873,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":95584256,"num_objects":1459,"num_object_clones":0,"num_object_copies":4377,"num_objects_missing_on_primary":1115,"num_objects_missing":1115,"num_objects_degraded":1941,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1459,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3509,"num_write_kb":158944,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2902,"num_bytes_recovered":190185472,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7],"avail_no_missing":["1","5","7"],"object_location_counts":[{"shards":"0,4,5,7","objects":6},{"shards":"0,5,7","objects":1115},{"shards":"4,5,7","objects":29},{"shards":"4,7","objects":309}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":520,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-09-23T21:14:02.776046+0000","last_change":"2023-09-23T21:12:10.824116+0000","last_active":"2023-09-23T21:14:02.776046+0000","last_peered":"2023-09-23T21:14:02.776046+0000","last_clean":"2023-09-23T21:14:02.776046+0000","last_became_active":"2023-09-23T21:12:10.823849+0000","last_became_peered":"2023-09-23T21:12:10.823849+0000","last_unstale":"2023-09-23T21:14:02.776046+0000","last_undegraded":"2023-09-23T21:14:02.776046+0000","last_fullsized":"2023-09-23T21:14:02.776046+0000","mapping_epoch":470,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:54:13.777603+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":521,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-09-23T21:14:02.771415+0000","last_change":"2023-09-23T21:13:44.413713+0000","last_active":"2023-09-23T21:14:02.771415+0000","last_peered":"2023-09-23T21:14:02.771415+0000","last_clean":"2023-09-23T21:14:02.771415+0000","last_became_active":"2023-09-23T21:13:44.413366+0000","last_became_peered":"2023-09-23T21:13:44.413366+0000","last_unstale":"2023-09-23T21:14:02.771415+0000","last_undegraded":"2023-09-23T21:14:02.771415+0000","last_fullsized":"2023-09-23T21:14:02.771415+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:31:56.349530+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1a","version":"79'161","reported_seq":670,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-09-23T21:14:02.769089+0000","last_change":"2023-09-23T21:12:30.444665+0000","last_active":"2023-09-23T21:14:02.769089+0000","last_peered":"2023-09-23T21:14:02.769089+0000","last_clean":"2023-09-23T21:14:02.769089+0000","last_became_active":"2023-09-23T21:12:30.444326+0000","last_became_peered":"2023-09-23T21:12:30.444326+0000","last_unstale":"2023-09-23T21:14:02.769089+0000","last_undegraded":"2023-09-23T21:14:02.769089+0000","last_fullsized":"2023-09-23T21:14:02.769089+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":479,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:06:51.075448+0000","last_deep_scrub":"79'161","last_deep_scrub_stamp":"2023-09-23T21:05:32.630292+0000","last_clean_scrub_stamp":"2023-09-23T21:06:51.075448+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0054793890000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,4],"acting":[2,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"10.11","version":"525'4883","reported_seq":11710,"reported_epoch":525,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-23T21:14:06.244971+0000","last_change":"2023-09-23T21:13:31.864763+0000","last_active":"2023-09-23T21:14:06.244971+0000","last_peered":"2023-09-23T21:14:06.244971+0000","last_clean":"2023-09-23T21:13:30.058478+0000","last_became_active":"2023-09-23T21:13:31.850860+0000","last_became_peered":"2023-09-23T21:13:31.850860+0000","last_unstale":"2023-09-23T21:14:06.244971+0000","last_undegraded":"2023-09-23T21:14:06.244971+0000","last_fullsized":"2023-09-23T21:14:06.244971+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":438,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":5,"last_scrub":"441'2974","last_scrub_stamp":"2023-09-23T21:11:06.133218+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_clean_scrub_stamp":"2023-09-23T21:11:06.133218+0000","objects_scrubbed":1646,"log_size":4883,"log_dups_size":0,"ondisk_log_size":4883,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:54:59.366711+0000","scrub_duration":0.3173955,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90963968,"num_objects":1388,"num_object_clones":0,"num_object_copies":4164,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1388,"num_objects_unfound":0,"num_objects_dirty":1388,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1912,"num_write_kb":52928,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2152,"num_bytes_recovered":141033472,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[2,5,7],"avail_no_missing":["2","1","5","7"],"object_location_counts":[{"shards":"2,5,7","objects":1388}],"blocked_by":[],"up_primary":0,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":34,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-09-23T21:14:02.771956+0000","last_change":"2023-09-23T21:13:30.806084+0000","last_active":"2023-09-23T21:14:02.771956+0000","last_peered":"2023-09-23T21:14:02.771956+0000","last_clean":"2023-09-23T21:14:02.771956+0000","last_became_active":"2023-09-23T21:13:30.805743+0000","last_became_peered":"2023-09-23T21:13:30.805743+0000","last_unstale":"2023-09-23T21:14:02.771956+0000","last_undegraded":"2023-09-23T21:14:02.771956+0000","last_fullsized":"2023-09-23T21:14:02.771956+0000","mapping_epoch":501,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:48:10.849281+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":398,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-09-23T21:14:02.770994+0000","last_change":"2023-09-23T21:13:44.410921+0000","last_active":"2023-09-23T21:14:02.770994+0000","last_peered":"2023-09-23T21:14:02.770994+0000","last_clean":"2023-09-23T21:14:02.770994+0000","last_became_active":"2023-09-23T21:13:44.410633+0000","last_became_peered":"2023-09-23T21:13:44.410633+0000","last_unstale":"2023-09-23T21:14:02.770994+0000","last_undegraded":"2023-09-23T21:14:02.770994+0000","last_fullsized":"2023-09-23T21:14:02.770994+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:57:10.116786+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.15","version":"79'161","reported_seq":550,"reported_epoch":523,"state":"active+clean","last_fresh":"2023-09-23T21:14:01.499224+0000","last_change":"2023-09-23T21:13:38.379453+0000","last_active":"2023-09-23T21:14:01.499224+0000","last_peered":"2023-09-23T21:14:01.499224+0000","last_clean":"2023-09-23T21:14:01.499224+0000","last_became_active":"2023-09-23T21:13:38.376499+0000","last_became_peered":"2023-09-23T21:13:38.376499+0000","last_unstale":"2023-09-23T21:14:01.499224+0000","last_undegraded":"2023-09-23T21:14:01.499224+0000","last_fullsized":"2023-09-23T21:14:01.499224+0000","mapping_epoch":507,"log_start":"0'0","ondisk_log_start":"0'0","created":479,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:09:34.892947+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:09:34.892947+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13418,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":11,"num_read_kb":21,"num_write":6,"num_write_kb":55,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":40253,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":499,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-09-23T21:14:02.771133+0000","last_change":"2023-09-23T21:13:12.235728+0000","last_active":"2023-09-23T21:14:02.771133+0000","last_peered":"2023-09-23T21:14:02.771133+0000","last_clean":"2023-09-23T21:14:02.771133+0000","last_became_active":"2023-09-23T21:13:12.216549+0000","last_became_peered":"2023-09-23T21:13:12.216549+0000","last_unstale":"2023-09-23T21:14:02.771133+0000","last_undegraded":"2023-09-23T21:14:02.771133+0000","last_fullsized":"2023-09-23T21:14:02.771133+0000","mapping_epoch":493,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:39:33.324239+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":562,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-09-23T21:14:02.775595+0000","last_change":"2023-09-23T21:12:10.790349+0000","last_active":"2023-09-23T21:14:02.775595+0000","last_peered":"2023-09-23T21:14:02.775595+0000","last_clean":"2023-09-23T21:14:02.775595+0000","last_became_active":"2023-09-23T21:12:10.789977+0000","last_became_peered":"2023-09-23T21:12:10.789977+0000","last_unstale":"2023-09-23T21:14:02.775595+0000","last_undegraded":"2023-09-23T21:14:02.775595+0000","last_fullsized":"2023-09-23T21:14:02.775595+0000","mapping_epoch":470,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:49:40.191147+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,5],"acting":[3,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"79'161","reported_seq":886,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-09-23T21:14:02.768416+0000","last_change":"2023-09-23T21:11:32.760292+0000","last_active":"2023-09-23T21:14:02.768416+0000","last_peered":"2023-09-23T21:14:02.768416+0000","last_clean":"2023-09-23T21:14:02.768416+0000","last_became_active":"2023-09-23T21:11:32.759909+0000","last_became_peered":"2023-09-23T21:11:32.759909+0000","last_unstale":"2023-09-23T21:14:02.768416+0000","last_undegraded":"2023-09-23T21:14:02.768416+0000","last_fullsized":"2023-09-23T21:14:02.768416+0000","mapping_epoch":457,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":458,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:25.096401+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:25.096401+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:13:09.225685+0000","scrub_duration":0.002859393,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":247,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-09-23T21:14:00.508890+0000","last_change":"2023-09-23T21:12:10.779468+0000","last_active":"2023-09-23T21:14:00.508890+0000","last_peered":"2023-09-23T21:14:00.508890+0000","last_clean":"2023-09-23T21:14:00.508890+0000","last_became_active":"2023-09-23T21:12:10.779282+0000","last_became_peered":"2023-09-23T21:12:10.779282+0000","last_unstale":"2023-09-23T21:14:00.508890+0000","last_undegraded":"2023-09-23T21:14:00.508890+0000","last_fullsized":"2023-09-23T21:14:00.508890+0000","mapping_epoch":470,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:36:35.417986+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":499,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-09-23T21:14:02.774912+0000","last_change":"2023-09-23T21:10:26.758958+0000","last_active":"2023-09-23T21:14:02.774912+0000","last_peered":"2023-09-23T21:14:02.774912+0000","last_clean":"2023-09-23T21:14:02.774912+0000","last_became_active":"2023-09-23T21:10:26.758539+0000","last_became_peered":"2023-09-23T21:10:26.758539+0000","last_unstale":"2023-09-23T21:14:02.774912+0000","last_undegraded":"2023-09-23T21:14:02.774912+0000","last_fullsized":"2023-09-23T21:14:02.774912+0000","mapping_epoch":423,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":424,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:44:42.319189+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.17","version":"79'161","reported_seq":617,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-09-23T21:14:02.782314+0000","last_change":"2023-09-23T21:13:44.416768+0000","last_active":"2023-09-23T21:14:02.782314+0000","last_peered":"2023-09-23T21:14:02.782314+0000","last_clean":"2023-09-23T21:14:02.782314+0000","last_became_active":"2023-09-23T21:13:44.416156+0000","last_became_peered":"2023-09-23T21:13:44.416156+0000","last_unstale":"2023-09-23T21:14:02.782314+0000","last_undegraded":"2023-09-23T21:14:02.782314+0000","last_fullsized":"2023-09-23T21:14:02.782314+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":479,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:25.916867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:25.916867+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.001665323,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":409,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-09-23T21:14:02.781698+0000","last_change":"2023-09-23T21:11:08.766045+0000","last_active":"2023-09-23T21:14:02.781698+0000","last_peered":"2023-09-23T21:14:02.781698+0000","last_clean":"2023-09-23T21:14:02.781698+0000","last_became_active":"2023-09-23T21:11:08.758504+0000","last_became_peered":"2023-09-23T21:11:08.758504+0000","last_unstale":"2023-09-23T21:14:02.781698+0000","last_undegraded":"2023-09-23T21:14:02.781698+0000","last_fullsized":"2023-09-23T21:14:02.781698+0000","mapping_epoch":443,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":444,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:02:19.482574+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":360,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,4],"acting":[3,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":211,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-09-23T21:14:02.769421+0000","last_change":"2023-09-23T21:13:30.821489+0000","last_active":"2023-09-23T21:14:02.769421+0000","last_peered":"2023-09-23T21:14:02.769421+0000","last_clean":"2023-09-23T21:14:02.769421+0000","last_became_active":"2023-09-23T21:13:30.821125+0000","last_became_peered":"2023-09-23T21:13:30.821125+0000","last_unstale":"2023-09-23T21:14:02.769421+0000","last_undegraded":"2023-09-23T21:14:02.769421+0000","last_fullsized":"2023-09-23T21:14:02.769421+0000","mapping_epoch":501,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:47:18.553417+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,5],"acting":[2,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.16","version":"79'161","reported_seq":443,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-09-23T21:14:02.771334+0000","last_change":"2023-09-23T21:13:33.770962+0000","last_active":"2023-09-23T21:14:02.771334+0000","last_peered":"2023-09-23T21:14:02.771334+0000","last_clean":"2023-09-23T21:14:02.771334+0000","last_became_active":"2023-09-23T21:13:33.770407+0000","last_became_peered":"2023-09-23T21:13:33.770407+0000","last_unstale":"2023-09-23T21:14:02.771334+0000","last_undegraded":"2023-09-23T21:14:02.771334+0000","last_fullsized":"2023-09-23T21:14:02.771334+0000","mapping_epoch":504,"log_start":"0'0","ondisk_log_start":"0'0","created":479,"last_epoch_clean":505,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:10:24.268662+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:10:24.268662+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13418,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":11,"num_read_kb":21,"num_write":6,"num_write_kb":55,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":40252,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.14","version":"79'22","reported_seq":87,"reported_epoch":523,"state":"active+clean","last_fresh":"2023-09-23T21:14:01.499029+0000","last_change":"2023-09-23T21:11:27.770162+0000","last_active":"2023-09-23T21:14:01.499029+0000","last_peered":"2023-09-23T21:14:01.499029+0000","last_clean":"2023-09-23T21:14:01.499029+0000","last_became_active":"2023-09-23T21:11:27.752275+0000","last_became_peered":"2023-09-23T21:11:27.752275+0000","last_unstale":"2023-09-23T21:14:01.499029+0000","last_undegraded":"2023-09-23T21:14:01.499029+0000","last_fullsized":"2023-09-23T21:14:01.499029+0000","mapping_epoch":452,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":453,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:05:11.769428+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4734,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,2],"acting":[4,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":452,"reported_epoch":523,"state":"active+clean","last_fresh":"2023-09-23T21:14:01.516604+0000","last_change":"2023-09-23T21:12:10.839271+0000","last_active":"2023-09-23T21:14:01.516604+0000","last_peered":"2023-09-23T21:14:01.516604+0000","last_clean":"2023-09-23T21:14:01.516604+0000","last_became_active":"2023-09-23T21:12:10.839042+0000","last_became_peered":"2023-09-23T21:12:10.839042+0000","last_unstale":"2023-09-23T21:14:01.516604+0000","last_undegraded":"2023-09-23T21:14:01.516604+0000","last_fullsized":"2023-09-23T21:14:01.516604+0000","mapping_epoch":470,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:10:26.414737+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,2],"acting":[4,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.11","version":"79'161","reported_seq":448,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-09-23T21:14:02.770364+0000","last_change":"2023-09-23T21:13:33.767406+0000","last_active":"2023-09-23T21:14:02.770364+0000","last_peered":"2023-09-23T21:14:02.770364+0000","last_clean":"2023-09-23T21:14:02.770364+0000","last_became_active":"2023-09-23T21:13:33.767067+0000","last_became_peered":"2023-09-23T21:13:33.767067+0000","last_unstale":"2023-09-23T21:14:02.770364+0000","last_undegraded":"2023-09-23T21:14:02.770364+0000","last_fullsized":"2023-09-23T21:14:02.770364+0000","mapping_epoch":504,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":505,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:28.756634+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:28.756634+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":23,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":131,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-09-23T21:14:00.483797+0000","last_change":"2023-09-23T21:11:27.777154+0000","last_active":"2023-09-23T21:14:00.483797+0000","last_peered":"2023-09-23T21:14:00.483797+0000","last_clean":"2023-09-23T21:14:00.483797+0000","last_became_active":"2023-09-23T21:11:27.754438+0000","last_became_peered":"2023-09-23T21:11:27.754438+0000","last_unstale":"2023-09-23T21:14:00.483797+0000","last_undegraded":"2023-09-23T21:14:00.483797+0000","last_fullsized":"2023-09-23T21:14:00.483797+0000","mapping_epoch":452,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":453,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:33:02.845882+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":65,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-09-23T21:14:02.768125+0000","last_change":"2023-09-23T21:12:10.764264+0000","last_active":"2023-09-23T21:14:02.768125+0000","last_peered":"2023-09-23T21:14:02.768125+0000","last_clean":"2023-09-23T21:14:02.768125+0000","last_became_active":"2023-09-23T21:12:10.763486+0000","last_became_peered":"2023-09-23T21:12:10.763486+0000","last_unstale":"2023-09-23T21:14:02.768125+0000","last_undegraded":"2023-09-23T21:14:02.768125+0000","last_fullsized":"2023-09-23T21:14:02.768125+0000","mapping_epoch":470,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:32:27.577036+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,7],"acting":[2,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.10","version":"79'161","reported_seq":658,"reported_epoch":523,"state":"active+clean","last_fresh":"2023-09-23T21:14:01.503850+0000","last_change":"2023-09-23T21:11:11.716703+0000","last_active":"2023-09-23T21:14:01.503850+0000","last_peered":"2023-09-23T21:14:01.503850+0000","last_clean":"2023-09-23T21:14:01.503850+0000","last_became_active":"2023-09-23T21:11:11.716385+0000","last_became_peered":"2023-09-23T21:11:11.716385+0000","last_unstale":"2023-09-23T21:14:01.503850+0000","last_undegraded":"2023-09-23T21:14:01.503850+0000","last_fullsized":"2023-09-23T21:14:01.503850+0000","mapping_epoch":446,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":447,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:06:50.336992+0000","last_deep_scrub":"79'161","last_deep_scrub_stamp":"2023-09-23T21:06:50.336992+0000","last_clean_scrub_stamp":"2023-09-23T21:06:50.336992+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:44:29.538890+0000","scrub_duration":0.0026150930000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,7],"acting":[4,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":465,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-09-23T21:14:02.771201+0000","last_change":"2023-09-23T21:13:12.270990+0000","last_active":"2023-09-23T21:14:02.771201+0000","last_peered":"2023-09-23T21:14:02.771201+0000","last_clean":"2023-09-23T21:14:02.771201+0000","last_became_active":"2023-09-23T21:13:12.237201+0000","last_became_peered":"2023-09-23T21:13:12.237201+0000","last_unstale":"2023-09-23T21:14:02.771201+0000","last_undegraded":"2023-09-23T21:14:02.771201+0000","last_fullsized":"2023-09-23T21:14:02.771201+0000","mapping_epoch":493,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:59:54.523877+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":276,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-09-23T21:14:02.781515+0000","last_change":"2023-09-23T21:10:26.758671+0000","last_active":"2023-09-23T21:14:02.781515+0000","last_peered":"2023-09-23T21:14:02.781515+0000","last_clean":"2023-09-23T21:14:02.781515+0000","last_became_active":"2023-09-23T21:10:26.758244+0000","last_became_peered":"2023-09-23T21:10:26.758244+0000","last_unstale":"2023-09-23T21:14:02.781515+0000","last_undegraded":"2023-09-23T21:14:02.781515+0000","last_fullsized":"2023-09-23T21:14:02.781515+0000","mapping_epoch":423,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":424,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:14:02.735609+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.18","version":"525'4931","reported_seq":6012,"reported_epoch":525,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T21:14:05.899670+0000","last_change":"2023-09-23T21:13:13.412097+0000","last_active":"2023-09-23T21:14:05.899670+0000","last_peered":"2023-09-23T21:14:05.899670+0000","last_clean":"2023-09-23T21:11:45.720015+0000","last_became_active":"2023-09-23T21:13:13.395632+0000","last_became_peered":"2023-09-23T21:13:13.395632+0000","last_unstale":"2023-09-23T21:14:05.899670+0000","last_undegraded":"2023-09-23T21:13:13.220971+0000","last_fullsized":"2023-09-23T21:13:13.183825+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":438,"last_epoch_clean":439,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_clean_scrub_stamp":"2023-09-23T21:10:15.663179+0000","objects_scrubbed":0,"log_size":4931,"log_dups_size":0,"ondisk_log_size":4931,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":95518720,"num_objects":1458,"num_object_clones":0,"num_object_copies":4374,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":661,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1458,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3582,"num_write_kb":161248,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":231,"num_bytes_recovered":15138816,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,7],"acting":[2,7],"avail_no_missing":["2","7"],"object_location_counts":[{"shards":"2,7","objects":1424}],"blocked_by":[],"up_primary":0,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.13","version":"79'161","reported_seq":601,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-09-23T21:14:02.776185+0000","last_change":"2023-09-23T21:13:14.933096+0000","last_active":"2023-09-23T21:14:02.776185+0000","last_peered":"2023-09-23T21:14:02.776185+0000","last_clean":"2023-09-23T21:14:02.776185+0000","last_became_active":"2023-09-23T21:13:14.932703+0000","last_became_peered":"2023-09-23T21:13:14.932703+0000","last_unstale":"2023-09-23T21:14:02.776185+0000","last_undegraded":"2023-09-23T21:14:02.776185+0000","last_fullsized":"2023-09-23T21:14:02.776185+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:36.420362+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:36.420362+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:29:42.163911+0000","scrub_duration":0.028838444000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":515,"reported_epoch":523,"state":"active+clean","last_fresh":"2023-09-23T21:14:01.498105+0000","last_change":"2023-09-23T21:13:12.216467+0000","last_active":"2023-09-23T21:14:01.498105+0000","last_peered":"2023-09-23T21:14:01.498105+0000","last_clean":"2023-09-23T21:14:01.498105+0000","last_became_active":"2023-09-23T21:13:12.215259+0000","last_became_peered":"2023-09-23T21:13:12.215259+0000","last_unstale":"2023-09-23T21:14:01.498105+0000","last_undegraded":"2023-09-23T21:14:01.498105+0000","last_fullsized":"2023-09-23T21:14:01.498105+0000","mapping_epoch":493,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:35:33.987839+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":34,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-09-23T21:14:02.771082+0000","last_change":"2023-09-23T21:13:30.804108+0000","last_active":"2023-09-23T21:14:02.771082+0000","last_peered":"2023-09-23T21:14:02.771082+0000","last_clean":"2023-09-23T21:14:02.771082+0000","last_became_active":"2023-09-23T21:13:30.796505+0000","last_became_peered":"2023-09-23T21:13:30.796505+0000","last_unstale":"2023-09-23T21:14:02.771082+0000","last_undegraded":"2023-09-23T21:14:02.771082+0000","last_fullsized":"2023-09-23T21:14:02.771082+0000","mapping_epoch":501,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:19:33.508496+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.19","version":"525'5076","reported_seq":6172,"reported_epoch":525,"state":"active+clean","last_fresh":"2023-09-23T21:14:06.223971+0000","last_change":"2023-09-23T21:10:58.790716+0000","last_active":"2023-09-23T21:14:06.223971+0000","last_peered":"2023-09-23T21:14:06.223971+0000","last_clean":"2023-09-23T21:14:06.223971+0000","last_became_active":"2023-09-23T21:10:58.787975+0000","last_became_peered":"2023-09-23T21:10:58.787975+0000","last_unstale":"2023-09-23T21:14:06.223971+0000","last_undegraded":"2023-09-23T21:14:06.223971+0000","last_fullsized":"2023-09-23T21:14:06.223971+0000","mapping_epoch":438,"log_start":"0'0","ondisk_log_start":"0'0","created":438,"last_epoch_clean":439,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_clean_scrub_stamp":"2023-09-23T21:10:15.663179+0000","objects_scrubbed":0,"log_size":5076,"log_dups_size":0,"ondisk_log_size":5076,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":91914240,"num_objects":1403,"num_object_clones":0,"num_object_copies":4209,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1403,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3689,"num_write_kb":162912,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":237,"num_bytes_recovered":15499264,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.12","version":"79'161","reported_seq":756,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-09-23T21:14:00.508896+0000","last_change":"2023-09-23T21:13:15.206574+0000","last_active":"2023-09-23T21:14:00.508896+0000","last_peered":"2023-09-23T21:14:00.508896+0000","last_clean":"2023-09-23T21:14:00.508896+0000","last_became_active":"2023-09-23T21:13:15.206186+0000","last_became_peered":"2023-09-23T21:13:15.206186+0000","last_unstale":"2023-09-23T21:14:00.508896+0000","last_undegraded":"2023-09-23T21:14:00.508896+0000","last_fullsized":"2023-09-23T21:14:00.508896+0000","mapping_epoch":496,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":497,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:05:56.785173+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:05:56.785173+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:34:29.221809+0000","scrub_duration":0.0042114370000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,0],"acting":[5,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":44,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-09-23T21:14:02.772044+0000","last_change":"2023-09-23T21:13:38.048791+0000","last_active":"2023-09-23T21:14:02.772044+0000","last_peered":"2023-09-23T21:14:02.772044+0000","last_clean":"2023-09-23T21:14:02.772044+0000","last_became_active":"2023-09-23T21:13:30.801386+0000","last_became_peered":"2023-09-23T21:13:30.801386+0000","last_unstale":"2023-09-23T21:14:02.772044+0000","last_undegraded":"2023-09-23T21:14:02.772044+0000","last_fullsized":"2023-09-23T21:14:02.772044+0000","mapping_epoch":501,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:52:49.170414+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":361,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-09-23T21:14:02.778671+0000","last_change":"2023-09-23T21:12:10.765780+0000","last_active":"2023-09-23T21:14:02.778671+0000","last_peered":"2023-09-23T21:14:02.778671+0000","last_clean":"2023-09-23T21:14:02.778671+0000","last_became_active":"2023-09-23T21:12:10.765372+0000","last_became_peered":"2023-09-23T21:12:10.765372+0000","last_unstale":"2023-09-23T21:14:02.778671+0000","last_undegraded":"2023-09-23T21:14:02.778671+0000","last_fullsized":"2023-09-23T21:14:02.778671+0000","mapping_epoch":470,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:56:08.446099+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.6","version":"525'5036","reported_seq":7950,"reported_epoch":525,"state":"active+clean","last_fresh":"2023-09-23T21:14:05.110352+0000","last_change":"2023-09-23T21:13:39.390285+0000","last_active":"2023-09-23T21:14:05.110352+0000","last_peered":"2023-09-23T21:14:05.110352+0000","last_clean":"2023-09-23T21:14:05.110352+0000","last_became_active":"2023-09-23T21:13:39.378284+0000","last_became_peered":"2023-09-23T21:13:39.378284+0000","last_unstale":"2023-09-23T21:14:05.110352+0000","last_undegraded":"2023-09-23T21:14:05.110352+0000","last_fullsized":"2023-09-23T21:14:05.110352+0000","mapping_epoch":508,"log_start":"0'0","ondisk_log_start":"0'0","created":416,"last_epoch_clean":509,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_clean_scrub_stamp":"2023-09-23T21:10:15.663179+0000","objects_scrubbed":0,"log_size":5036,"log_dups_size":0,"ondisk_log_size":5036,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":94109696,"num_objects":1436,"num_object_clones":0,"num_object_copies":4308,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1436,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3630,"num_write_kb":162112,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2399,"num_bytes_recovered":113967104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,4],"acting":[0,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.d","version":"79'161","reported_seq":676,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-09-23T21:14:00.483837+0000","last_change":"2023-09-23T21:12:30.444461+0000","last_active":"2023-09-23T21:14:00.483837+0000","last_peered":"2023-09-23T21:14:00.483837+0000","last_clean":"2023-09-23T21:14:00.483837+0000","last_became_active":"2023-09-23T21:12:30.443979+0000","last_became_peered":"2023-09-23T21:12:30.443979+0000","last_unstale":"2023-09-23T21:14:00.483837+0000","last_undegraded":"2023-09-23T21:14:00.483837+0000","last_fullsized":"2023-09-23T21:14:00.483837+0000","mapping_epoch":470,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:09:36.858077+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:09:36.858077+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0024987500000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":169,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-09-23T21:14:00.484621+0000","last_change":"2023-09-23T21:13:59.872834+0000","last_active":"2023-09-23T21:14:00.484621+0000","last_peered":"2023-09-23T21:14:00.484621+0000","last_clean":"2023-09-23T21:14:00.484621+0000","last_became_active":"2023-09-23T21:13:30.770556+0000","last_became_peered":"2023-09-23T21:13:30.770556+0000","last_unstale":"2023-09-23T21:14:00.484621+0000","last_undegraded":"2023-09-23T21:14:00.484621+0000","last_fullsized":"2023-09-23T21:14:00.484621+0000","mapping_epoch":501,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":521,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:57:55.460052+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":132,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":230,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-09-23T21:14:00.508262+0000","last_change":"2023-09-23T21:10:26.750725+0000","last_active":"2023-09-23T21:14:00.508262+0000","last_peered":"2023-09-23T21:14:00.508262+0000","last_clean":"2023-09-23T21:14:00.508262+0000","last_became_active":"2023-09-23T21:10:26.750307+0000","last_became_peered":"2023-09-23T21:10:26.750307+0000","last_unstale":"2023-09-23T21:14:00.508262+0000","last_undegraded":"2023-09-23T21:14:00.508262+0000","last_fullsized":"2023-09-23T21:14:00.508262+0000","mapping_epoch":423,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":424,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:06:22.040548+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,4],"acting":[5,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.7","version":"525'4959","reported_seq":5151,"reported_epoch":525,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T21:14:04.172115+0000","last_change":"2023-09-23T21:13:38.417178+0000","last_active":"2023-09-23T21:14:04.172115+0000","last_peered":"2023-09-23T21:14:04.172115+0000","last_clean":"2023-09-23T21:11:26.699871+0000","last_became_active":"2023-09-23T21:13:38.408523+0000","last_became_peered":"2023-09-23T21:13:38.408523+0000","last_unstale":"2023-09-23T21:14:04.172115+0000","last_undegraded":"2023-09-23T21:13:38.369833+0000","last_fullsized":"2023-09-23T21:13:38.358198+0000","mapping_epoch":507,"log_start":"0'0","ondisk_log_start":"0'0","created":416,"last_epoch_clean":439,"parent":"0.0","parent_split_bits":0,"last_scrub":"441'2973","last_scrub_stamp":"2023-09-23T21:11:06.079326+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_clean_scrub_stamp":"2023-09-23T21:11:06.079326+0000","objects_scrubbed":1647,"log_size":4959,"log_dups_size":0,"ondisk_log_size":4959,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:27:59.805526+0000","scrub_duration":0.44319714500000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90701824,"num_objects":1384,"num_object_clones":0,"num_object_copies":4152,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1165,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1384,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1991,"num_write_kb":55296,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,2],"acting":[3,5],"avail_no_missing":["3","5"],"object_location_counts":[{"shards":"3,4,5","objects":298},{"shards":"3,5","objects":1086}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.c","version":"79'161","reported_seq":887,"reported_epoch":524,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-23T21:14:02.776776+0000","last_change":"2023-09-23T21:13:13.402530+0000","last_active":"2023-09-23T21:14:02.776776+0000","last_peered":"2023-09-23T21:14:02.776776+0000","last_clean":"2023-09-23T21:11:23.200766+0000","last_became_active":"2023-09-23T21:13:13.401526+0000","last_became_peered":"2023-09-23T21:13:13.401526+0000","last_unstale":"2023-09-23T21:14:02.776776+0000","last_undegraded":"2023-09-23T21:13:13.204992+0000","last_fullsized":"2023-09-23T21:14:02.776776+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":420,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:22.040907+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:22.040907+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.013021176000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":163840,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":163840,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":["3","0","5"],"object_location_counts":[{"shards":"0,3,4","objects":-1},{"shards":"0,3,5","objects":1}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":155,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-09-23T21:14:00.485683+0000","last_change":"2023-09-23T21:13:59.884113+0000","last_active":"2023-09-23T21:14:00.485683+0000","last_peered":"2023-09-23T21:14:00.485683+0000","last_clean":"2023-09-23T21:14:00.485683+0000","last_became_active":"2023-09-23T21:13:30.806778+0000","last_became_peered":"2023-09-23T21:13:30.806778+0000","last_unstale":"2023-09-23T21:14:00.485683+0000","last_undegraded":"2023-09-23T21:14:00.485683+0000","last_fullsized":"2023-09-23T21:14:00.485683+0000","mapping_epoch":501,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":521,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:36:15.284465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":486,"reported_epoch":523,"state":"active+clean","last_fresh":"2023-09-23T21:14:01.498196+0000","last_change":"2023-09-23T21:13:15.152792+0000","last_active":"2023-09-23T21:14:01.498196+0000","last_peered":"2023-09-23T21:14:01.498196+0000","last_clean":"2023-09-23T21:14:01.498196+0000","last_became_active":"2023-09-23T21:13:14.515084+0000","last_became_peered":"2023-09-23T21:13:14.515084+0000","last_unstale":"2023-09-23T21:14:01.498196+0000","last_undegraded":"2023-09-23T21:14:01.498196+0000","last_fullsized":"2023-09-23T21:14:01.498196+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:08:16.702072+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.4","version":"525'4823","reported_seq":11764,"reported_epoch":525,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T21:14:04.178423+0000","last_change":"2023-09-23T21:13:30.847170+0000","last_active":"2023-09-23T21:14:04.178423+0000","last_peered":"2023-09-23T21:14:04.178423+0000","last_clean":"2023-09-23T21:11:06.743842+0000","last_became_active":"2023-09-23T21:13:30.834806+0000","last_became_peered":"2023-09-23T21:13:30.834806+0000","last_unstale":"2023-09-23T21:14:04.178423+0000","last_undegraded":"2023-09-23T21:13:30.760230+0000","last_fullsized":"2023-09-23T21:13:30.749974+0000","mapping_epoch":501,"log_start":"0'0","ondisk_log_start":"0'0","created":416,"last_epoch_clean":439,"parent":"0.0","parent_split_bits":0,"last_scrub":"440'2861","last_scrub_stamp":"2023-09-23T21:11:03.842926+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_clean_scrub_stamp":"2023-09-23T21:11:03.842926+0000","objects_scrubbed":1526,"log_size":4823,"log_dups_size":0,"ondisk_log_size":4823,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:32:29.379756+0000","scrub_duration":0.39378383900000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":98172928,"num_objects":1498,"num_object_clones":0,"num_object_copies":4494,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1130,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1498,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1968,"num_write_kb":62080,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2723,"num_bytes_recovered":163577856,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[3,4],"avail_no_missing":["3","4"],"object_location_counts":[{"shards":"1,2,3,4","objects":360},{"shards":"3,4","objects":1138}],"blocked_by":[],"up_primary":0,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.f","version":"79'161","reported_seq":810,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-09-23T21:14:00.509349+0000","last_change":"2023-09-23T21:11:30.750338+0000","last_active":"2023-09-23T21:14:00.509349+0000","last_peered":"2023-09-23T21:14:00.509349+0000","last_clean":"2023-09-23T21:14:00.509349+0000","last_became_active":"2023-09-23T21:11:30.749953+0000","last_became_peered":"2023-09-23T21:11:30.749953+0000","last_unstale":"2023-09-23T21:14:00.509349+0000","last_undegraded":"2023-09-23T21:14:00.509349+0000","last_fullsized":"2023-09-23T21:14:00.509349+0000","mapping_epoch":455,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":456,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:27.880776+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:27.880776+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:07:10.405723+0000","scrub_duration":0.0014004899999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,3],"acting":[5,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":246,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-09-23T21:14:00.508918+0000","last_change":"2023-09-23T21:10:26.765068+0000","last_active":"2023-09-23T21:14:00.508918+0000","last_peered":"2023-09-23T21:14:00.508918+0000","last_clean":"2023-09-23T21:14:00.508918+0000","last_became_active":"2023-09-23T21:10:26.758364+0000","last_became_peered":"2023-09-23T21:10:26.758364+0000","last_unstale":"2023-09-23T21:14:00.508918+0000","last_undegraded":"2023-09-23T21:14:00.508918+0000","last_fullsized":"2023-09-23T21:14:00.508918+0000","mapping_epoch":423,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":424,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:13:21.057529+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":280,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":462,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-09-23T21:14:00.508588+0000","last_change":"2023-09-23T21:13:30.770746+0000","last_active":"2023-09-23T21:14:00.508588+0000","last_peered":"2023-09-23T21:14:00.508588+0000","last_clean":"2023-09-23T21:14:00.508588+0000","last_became_active":"2023-09-23T21:13:30.770437+0000","last_became_peered":"2023-09-23T21:13:30.770437+0000","last_unstale":"2023-09-23T21:14:00.508588+0000","last_undegraded":"2023-09-23T21:14:00.508588+0000","last_fullsized":"2023-09-23T21:14:00.508588+0000","mapping_epoch":501,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:33:48.325730+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,0],"acting":[5,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.5","version":"525'4920","reported_seq":5103,"reported_epoch":525,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T21:14:06.193767+0000","last_change":"2023-09-23T21:13:13.436794+0000","last_active":"2023-09-23T21:14:06.193767+0000","last_peered":"2023-09-23T21:14:06.193767+0000","last_clean":"2023-09-23T21:11:45.768408+0000","last_became_active":"2023-09-23T21:13:13.424939+0000","last_became_peered":"2023-09-23T21:13:13.424939+0000","last_unstale":"2023-09-23T21:14:06.193767+0000","last_undegraded":"2023-09-23T21:13:13.225067+0000","last_fullsized":"2023-09-23T21:13:13.198194+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":416,"last_epoch_clean":439,"parent":"0.0","parent_split_bits":0,"last_scrub":"440'2840","last_scrub_stamp":"2023-09-23T21:11:03.208833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_clean_scrub_stamp":"2023-09-23T21:11:03.208833+0000","objects_scrubbed":1498,"log_size":4920,"log_dups_size":0,"ondisk_log_size":4920,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:56:40.574477+0000","scrub_duration":0.45608471699999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":94568448,"num_objects":1443,"num_object_clones":0,"num_object_copies":4329,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":644,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1443,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2089,"num_write_kb":65088,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,3],"acting":[2,3],"avail_no_missing":["2","3"],"object_location_counts":[{"shards":"2,3","objects":1397}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.e","version":"79'161","reported_seq":692,"reported_epoch":523,"state":"active+clean","last_fresh":"2023-09-23T21:14:01.499520+0000","last_change":"2023-09-23T21:12:30.440952+0000","last_active":"2023-09-23T21:14:01.499520+0000","last_peered":"2023-09-23T21:14:01.499520+0000","last_clean":"2023-09-23T21:14:01.499520+0000","last_became_active":"2023-09-23T21:12:30.440463+0000","last_became_peered":"2023-09-23T21:12:30.440463+0000","last_unstale":"2023-09-23T21:14:01.499520+0000","last_undegraded":"2023-09-23T21:14:01.499520+0000","last_fullsized":"2023-09-23T21:14:01.499520+0000","mapping_epoch":455,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:10:39.567232+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:10:39.567232+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.005229694,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":322,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-09-23T21:14:02.777152+0000","last_change":"2023-09-23T21:13:13.402418+0000","last_active":"2023-09-23T21:14:02.777152+0000","last_peered":"2023-09-23T21:14:02.777152+0000","last_clean":"2023-09-23T21:14:02.777152+0000","last_became_active":"2023-09-23T21:13:13.400984+0000","last_became_peered":"2023-09-23T21:13:13.400984+0000","last_unstale":"2023-09-23T21:14:02.777152+0000","last_undegraded":"2023-09-23T21:14:02.777152+0000","last_fullsized":"2023-09-23T21:14:02.777152+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":495,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:28:01.411191+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,0],"acting":[3,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":497,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-09-23T21:14:02.771778+0000","last_change":"2023-09-23T21:13:14.339670+0000","last_active":"2023-09-23T21:14:02.771778+0000","last_peered":"2023-09-23T21:14:02.771778+0000","last_clean":"2023-09-23T21:14:02.771778+0000","last_became_active":"2023-09-23T21:13:14.339341+0000","last_became_peered":"2023-09-23T21:13:14.339341+0000","last_unstale":"2023-09-23T21:14:02.771778+0000","last_undegraded":"2023-09-23T21:14:02.771778+0000","last_fullsized":"2023-09-23T21:14:02.771778+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:23:19.411368+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.2","version":"524'4946","reported_seq":6013,"reported_epoch":524,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T21:14:03.161557+0000","last_change":"2023-09-23T21:13:13.407366+0000","last_active":"2023-09-23T21:14:03.161557+0000","last_peered":"2023-09-23T21:14:03.161557+0000","last_clean":"2023-09-23T21:11:45.752822+0000","last_became_active":"2023-09-23T21:13:13.398616+0000","last_became_peered":"2023-09-23T21:13:13.398616+0000","last_unstale":"2023-09-23T21:14:03.161557+0000","last_undegraded":"2023-09-23T21:13:13.289815+0000","last_fullsized":"2023-09-23T21:13:13.190270+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":416,"last_epoch_clean":439,"parent":"0.0","parent_split_bits":0,"last_scrub":"440'3007","last_scrub_stamp":"2023-09-23T21:11:05.553661+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_clean_scrub_stamp":"2023-09-23T21:11:05.553661+0000","objects_scrubbed":1622,"log_size":4946,"log_dups_size":0,"ondisk_log_size":4946,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:51:21.839818+0000","scrub_duration":0.39156952499999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":94240768,"num_objects":1438,"num_object_clones":0,"num_object_copies":4314,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2730,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1438,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1944,"num_write_kb":56320,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":435,"num_bytes_recovered":28508160,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,3],"avail_no_missing":["4","1"],"object_location_counts":[{"shards":"0,1,4","objects":1404},{"shards":"1,4","objects":34}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.9","version":"79'161","reported_seq":678,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-09-23T21:14:02.767616+0000","last_change":"2023-09-23T21:12:30.436118+0000","last_active":"2023-09-23T21:14:02.767616+0000","last_peered":"2023-09-23T21:14:02.767616+0000","last_clean":"2023-09-23T21:14:02.767616+0000","last_became_active":"2023-09-23T21:12:30.435779+0000","last_became_peered":"2023-09-23T21:12:30.435779+0000","last_unstale":"2023-09-23T21:14:02.767616+0000","last_undegraded":"2023-09-23T21:14:02.767616+0000","last_fullsized":"2023-09-23T21:14:02.767616+0000","mapping_epoch":433,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:10:37.696951+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:10:37.696951+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.001386504,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,3],"acting":[2,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":83,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-09-23T21:14:02.771887+0000","last_change":"2023-09-23T21:13:30.793273+0000","last_active":"2023-09-23T21:14:02.771887+0000","last_peered":"2023-09-23T21:14:02.771887+0000","last_clean":"2023-09-23T21:14:02.771887+0000","last_became_active":"2023-09-23T21:13:30.791269+0000","last_became_peered":"2023-09-23T21:13:30.791269+0000","last_unstale":"2023-09-23T21:14:02.771887+0000","last_undegraded":"2023-09-23T21:14:02.771887+0000","last_fullsized":"2023-09-23T21:14:02.771887+0000","mapping_epoch":501,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:38:07.142290+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":97,"reported_epoch":523,"state":"active+clean","last_fresh":"2023-09-23T21:14:01.498503+0000","last_change":"2023-09-23T21:12:14.313466+0000","last_active":"2023-09-23T21:14:01.498503+0000","last_peered":"2023-09-23T21:14:01.498503+0000","last_clean":"2023-09-23T21:14:01.498503+0000","last_became_active":"2023-09-23T21:12:14.209837+0000","last_became_peered":"2023-09-23T21:12:14.209837+0000","last_unstale":"2023-09-23T21:14:01.498503+0000","last_undegraded":"2023-09-23T21:14:01.498503+0000","last_fullsized":"2023-09-23T21:14:01.498503+0000","mapping_epoch":470,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":474,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:09:40.375241+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,7],"acting":[4,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.3","version":"525'5005","reported_seq":7133,"reported_epoch":525,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T21:14:06.238744+0000","last_change":"2023-09-23T21:13:45.480705+0000","last_active":"2023-09-23T21:14:06.238744+0000","last_peered":"2023-09-23T21:14:06.238744+0000","last_clean":"2023-09-23T21:11:26.700990+0000","last_became_active":"2023-09-23T21:13:45.471702+0000","last_became_peered":"2023-09-23T21:13:45.471702+0000","last_unstale":"2023-09-23T21:14:06.238744+0000","last_undegraded":"2023-09-23T21:13:45.412179+0000","last_fullsized":"2023-09-23T21:13:45.401701+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":416,"last_epoch_clean":439,"parent":"0.0","parent_split_bits":0,"last_scrub":"440'2873","last_scrub_stamp":"2023-09-23T21:11:02.114783+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_clean_scrub_stamp":"2023-09-23T21:11:02.114783+0000","objects_scrubbed":1532,"log_size":5005,"log_dups_size":0,"ondisk_log_size":5005,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:42:48.549370+0000","scrub_duration":0.32435022600000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":93716480,"num_objects":1430,"num_object_clones":0,"num_object_copies":4290,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1184,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1430,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2138,"num_write_kb":65152,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":865,"num_bytes_recovered":56688640,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5],"avail_no_missing":["2","5"],"object_location_counts":[{"shards":"0,2,4,5","objects":270},{"shards":"2,4,5","objects":48},{"shards":"2,5","objects":1111}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.8","version":"79'161","reported_seq":645,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-09-23T21:14:02.767029+0000","last_change":"2023-09-23T21:12:30.433561+0000","last_active":"2023-09-23T21:14:02.767029+0000","last_peered":"2023-09-23T21:14:02.767029+0000","last_clean":"2023-09-23T21:14:02.767029+0000","last_became_active":"2023-09-23T21:12:30.433217+0000","last_became_peered":"2023-09-23T21:12:30.433217+0000","last_unstale":"2023-09-23T21:14:02.767029+0000","last_undegraded":"2023-09-23T21:14:02.767029+0000","last_fullsized":"2023-09-23T21:14:02.767029+0000","mapping_epoch":359,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:06:52.098882+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:06:52.098882+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0092173299999999993,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":131344,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":100,"reported_epoch":523,"state":"active+clean","last_fresh":"2023-09-23T21:14:01.498661+0000","last_change":"2023-09-23T21:12:10.823380+0000","last_active":"2023-09-23T21:14:01.498661+0000","last_peered":"2023-09-23T21:14:01.498661+0000","last_clean":"2023-09-23T21:14:01.498661+0000","last_became_active":"2023-09-23T21:12:10.823044+0000","last_became_peered":"2023-09-23T21:12:10.823044+0000","last_unstale":"2023-09-23T21:14:01.498661+0000","last_undegraded":"2023-09-23T21:14:01.498661+0000","last_fullsized":"2023-09-23T21:14:01.498661+0000","mapping_epoch":470,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:53:29.846536+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,5],"acting":[4,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":101,"reported_epoch":523,"state":"active+clean","last_fresh":"2023-09-23T21:14:01.499354+0000","last_change":"2023-09-23T21:13:30.772451+0000","last_active":"2023-09-23T21:14:01.499354+0000","last_peered":"2023-09-23T21:14:01.499354+0000","last_clean":"2023-09-23T21:14:01.499354+0000","last_became_active":"2023-09-23T21:13:30.772040+0000","last_became_peered":"2023-09-23T21:13:30.772040+0000","last_unstale":"2023-09-23T21:14:01.499354+0000","last_undegraded":"2023-09-23T21:14:01.499354+0000","last_fullsized":"2023-09-23T21:14:01.499354+0000","mapping_epoch":501,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:24:49.246727+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,0],"acting":[4,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.0","version":"524'4881","reported_seq":8488,"reported_epoch":524,"state":"active+recovering+undersized+remapped","last_fresh":"2023-09-23T21:14:03.176326+0000","last_change":"2023-09-23T21:14:02.130787+0000","last_active":"2023-09-23T21:14:03.176326+0000","last_peered":"2023-09-23T21:14:03.176326+0000","last_clean":"2023-09-23T21:11:26.716395+0000","last_became_active":"2023-09-23T21:13:34.557827+0000","last_became_peered":"2023-09-23T21:13:34.557827+0000","last_unstale":"2023-09-23T21:14:03.176326+0000","last_undegraded":"2023-09-23T21:14:03.176326+0000","last_fullsized":"2023-09-23T21:13:31.752840+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":416,"last_epoch_clean":439,"parent":"0.0","parent_split_bits":0,"last_scrub":"440'2961","last_scrub_stamp":"2023-09-23T21:11:04.564028+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_clean_scrub_stamp":"2023-09-23T21:11:04.564028+0000","objects_scrubbed":1608,"log_size":4881,"log_dups_size":0,"ondisk_log_size":4881,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:43:09.815808+0000","scrub_duration":0.41858158299999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":96534528,"num_objects":1473,"num_object_clones":0,"num_object_copies":4419,"num_objects_missing_on_primary":1294,"num_objects_missing":1294,"num_objects_degraded":0,"num_objects_misplaced":1469,"num_objects_unfound":0,"num_objects_dirty":1473,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1923,"num_write_kb":57216,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":874,"num_bytes_recovered":57212928,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,0],"avail_no_missing":["1","7"],"object_location_counts":[{"shards":"0,1,4,7","objects":4},{"shards":"0,1,7","objects":1143},{"shards":"1,4,7","objects":326}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.b","version":"79'161","reported_seq":528,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-09-23T21:14:02.771527+0000","last_change":"2023-09-23T21:13:13.373313+0000","last_active":"2023-09-23T21:14:02.771527+0000","last_peered":"2023-09-23T21:14:02.771527+0000","last_clean":"2023-09-23T21:14:02.771527+0000","last_became_active":"2023-09-23T21:13:13.372328+0000","last_became_peered":"2023-09-23T21:13:13.372328+0000","last_unstale":"2023-09-23T21:14:02.771527+0000","last_undegraded":"2023-09-23T21:14:02.771527+0000","last_fullsized":"2023-09-23T21:14:02.771527+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":495,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:05:55.772264+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:05:55.772264+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.001862507,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":81,"reported_epoch":523,"state":"active+clean","last_fresh":"2023-09-23T21:14:01.498040+0000","last_change":"2023-09-23T21:11:27.754717+0000","last_active":"2023-09-23T21:14:01.498040+0000","last_peered":"2023-09-23T21:14:01.498040+0000","last_clean":"2023-09-23T21:14:01.498040+0000","last_became_active":"2023-09-23T21:11:27.754376+0000","last_became_peered":"2023-09-23T21:11:27.754376+0000","last_unstale":"2023-09-23T21:14:01.498040+0000","last_undegraded":"2023-09-23T21:14:01.498040+0000","last_fullsized":"2023-09-23T21:14:01.498040+0000","mapping_epoch":452,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":453,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:26:41.734541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":455,"reported_epoch":523,"state":"active+clean","last_fresh":"2023-09-23T21:14:01.500220+0000","last_change":"2023-09-23T21:12:10.811698+0000","last_active":"2023-09-23T21:14:01.500220+0000","last_peered":"2023-09-23T21:14:01.500220+0000","last_clean":"2023-09-23T21:14:01.500220+0000","last_became_active":"2023-09-23T21:12:10.789656+0000","last_became_peered":"2023-09-23T21:12:10.789656+0000","last_unstale":"2023-09-23T21:14:01.500220+0000","last_undegraded":"2023-09-23T21:14:01.500220+0000","last_fullsized":"2023-09-23T21:14:01.500220+0000","mapping_epoch":470,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:39:15.112091+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,5],"acting":[4,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.1","version":"524'4828","reported_seq":11238,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-09-23T21:14:03.139483+0000","last_change":"2023-09-23T21:13:53.483736+0000","last_active":"2023-09-23T21:14:03.139483+0000","last_peered":"2023-09-23T21:14:03.139483+0000","last_clean":"2023-09-23T21:14:03.139483+0000","last_became_active":"2023-09-23T21:13:53.472685+0000","last_became_peered":"2023-09-23T21:13:53.472685+0000","last_unstale":"2023-09-23T21:14:03.139483+0000","last_undegraded":"2023-09-23T21:14:03.139483+0000","last_fullsized":"2023-09-23T21:14:03.139483+0000","mapping_epoch":519,"log_start":"0'0","ondisk_log_start":"0'0","created":416,"last_epoch_clean":520,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_clean_scrub_stamp":"2023-09-23T21:10:15.663179+0000","objects_scrubbed":0,"log_size":4828,"log_dups_size":0,"ondisk_log_size":4828,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":95944704,"num_objects":1464,"num_object_clones":0,"num_object_copies":4392,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1464,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3480,"num_write_kb":158208,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3005,"num_bytes_recovered":156499968,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.a","version":"79'161","reported_seq":671,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-09-23T21:14:02.769291+0000","last_change":"2023-09-23T21:12:30.437790+0000","last_active":"2023-09-23T21:14:02.769291+0000","last_peered":"2023-09-23T21:14:02.769291+0000","last_clean":"2023-09-23T21:14:02.769291+0000","last_became_active":"2023-09-23T21:12:30.437427+0000","last_became_peered":"2023-09-23T21:12:30.437427+0000","last_unstale":"2023-09-23T21:14:02.769291+0000","last_undegraded":"2023-09-23T21:14:02.769291+0000","last_fullsized":"2023-09-23T21:14:02.769291+0000","mapping_epoch":446,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:06:51.075448+0000","last_deep_scrub":"79'161","last_deep_scrub_stamp":"2023-09-23T21:05:32.630292+0000","last_clean_scrub_stamp":"2023-09-23T21:06:51.075448+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0054793890000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,4],"acting":[2,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":371,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-09-23T21:14:02.775316+0000","last_change":"2023-09-23T21:12:10.792918+0000","last_active":"2023-09-23T21:14:02.775316+0000","last_peered":"2023-09-23T21:14:02.775316+0000","last_clean":"2023-09-23T21:14:02.775316+0000","last_became_active":"2023-09-23T21:12:10.792679+0000","last_became_peered":"2023-09-23T21:12:10.792679+0000","last_unstale":"2023-09-23T21:14:02.775316+0000","last_undegraded":"2023-09-23T21:14:02.775316+0000","last_fullsized":"2023-09-23T21:14:02.775316+0000","mapping_epoch":470,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:30:48.008690+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,4],"acting":[3,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":502,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-09-23T21:14:02.785832+0000","last_change":"2023-09-23T21:12:10.790812+0000","last_active":"2023-09-23T21:14:02.785832+0000","last_peered":"2023-09-23T21:14:02.785832+0000","last_clean":"2023-09-23T21:14:02.785832+0000","last_became_active":"2023-09-23T21:12:10.790492+0000","last_became_peered":"2023-09-23T21:12:10.790492+0000","last_unstale":"2023-09-23T21:14:02.785832+0000","last_undegraded":"2023-09-23T21:14:02.785832+0000","last_fullsized":"2023-09-23T21:14:02.785832+0000","mapping_epoch":470,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:02:19.863118+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.c","version":"523'7134","reported_seq":8284,"reported_epoch":523,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T21:14:02.252177+0000","last_change":"2023-09-23T21:13:30.822215+0000","last_active":"2023-09-23T21:14:02.252177+0000","last_peered":"2023-09-23T21:14:02.252177+0000","last_clean":"2023-09-23T21:11:45.750376+0000","last_became_active":"2023-09-23T21:13:13.404327+0000","last_became_peered":"2023-09-23T21:13:13.404327+0000","last_unstale":"2023-09-23T21:14:02.252177+0000","last_undegraded":"2023-09-23T21:13:13.200907+0000","last_fullsized":"2023-09-23T21:13:13.193701+0000","mapping_epoch":494,"log_start":"437'2751","ondisk_log_start":"437'2751","created":416,"last_epoch_clean":434,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_clean_scrub_stamp":"2023-09-23T21:10:15.663179+0000","objects_scrubbed":0,"log_size":4383,"log_dups_size":0,"ondisk_log_size":4383,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:33:42.971375+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":190316544,"num_objects":2904,"num_object_clones":0,"num_object_copies":8712,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1339,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2904,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7134,"num_write_kb":321216,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":498,"num_bytes_recovered":32636928,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4],"avail_no_missing":["7","4"],"object_location_counts":[{"shards":"4,7","objects":2850}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.7","version":"79'161","reported_seq":617,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-09-23T21:14:02.785536+0000","last_change":"2023-09-23T21:13:44.416548+0000","last_active":"2023-09-23T21:14:02.785536+0000","last_peered":"2023-09-23T21:14:02.785536+0000","last_clean":"2023-09-23T21:14:02.785536+0000","last_became_active":"2023-09-23T21:13:44.415617+0000","last_became_peered":"2023-09-23T21:13:44.415617+0000","last_unstale":"2023-09-23T21:14:02.785536+0000","last_undegraded":"2023-09-23T21:14:02.785536+0000","last_fullsized":"2023-09-23T21:14:02.785536+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":3,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:25.916867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:25.916867+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.001665323,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":454,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-09-23T21:14:02.771317+0000","last_change":"2023-09-23T21:13:13.211959+0000","last_active":"2023-09-23T21:14:02.771317+0000","last_peered":"2023-09-23T21:14:02.771317+0000","last_clean":"2023-09-23T21:14:02.771317+0000","last_became_active":"2023-09-23T21:13:13.211712+0000","last_became_peered":"2023-09-23T21:13:13.211712+0000","last_unstale":"2023-09-23T21:14:02.771317+0000","last_undegraded":"2023-09-23T21:14:02.771317+0000","last_fullsized":"2023-09-23T21:14:02.771317+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":495,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:37:21.655775+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,2],"acting":[0,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":200,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-09-23T21:14:00.508827+0000","last_change":"2023-09-23T21:05:04.326889+0000","last_active":"2023-09-23T21:14:00.508827+0000","last_peered":"2023-09-23T21:14:00.508827+0000","last_clean":"2023-09-23T21:14:00.508827+0000","last_became_active":"2023-09-23T21:05:04.326032+0000","last_became_peered":"2023-09-23T21:05:04.326032+0000","last_unstale":"2023-09-23T21:14:00.508827+0000","last_undegraded":"2023-09-23T21:14:00.508827+0000","last_fullsized":"2023-09-23T21:14:00.508827+0000","mapping_epoch":332,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:00:17.507889+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":532,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-09-23T21:14:00.509654+0000","last_change":"2023-09-23T21:13:13.285608+0000","last_active":"2023-09-23T21:14:00.509654+0000","last_peered":"2023-09-23T21:14:00.509654+0000","last_clean":"2023-09-23T21:14:00.509654+0000","last_became_active":"2023-09-23T21:13:13.285255+0000","last_became_peered":"2023-09-23T21:13:13.285255+0000","last_unstale":"2023-09-23T21:14:00.509654+0000","last_undegraded":"2023-09-23T21:14:00.509654+0000","last_fullsized":"2023-09-23T21:14:00.509654+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":495,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:42:30.087599+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.a","version":"525'7083","reported_seq":15815,"reported_epoch":525,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T21:14:06.265261+0000","last_change":"2023-09-23T21:13:31.927282+0000","last_active":"2023-09-23T21:14:06.265261+0000","last_peered":"2023-09-23T21:14:06.265261+0000","last_clean":"2023-09-23T21:13:30.353360+0000","last_became_active":"2023-09-23T21:13:31.896923+0000","last_became_peered":"2023-09-23T21:13:31.896923+0000","last_unstale":"2023-09-23T21:14:06.265261+0000","last_undegraded":"2023-09-23T21:13:31.776904+0000","last_fullsized":"2023-09-23T21:13:31.751334+0000","mapping_epoch":502,"log_start":"439'2712","ondisk_log_start":"439'2712","created":416,"last_epoch_clean":499,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_clean_scrub_stamp":"2023-09-23T21:10:15.663179+0000","objects_scrubbed":0,"log_size":4371,"log_dups_size":0,"ondisk_log_size":4371,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:23:14.477602+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":182779904,"num_objects":2789,"num_object_clones":0,"num_object_copies":8367,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2243,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2789,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7083,"num_write_kb":315904,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4293,"num_bytes_recovered":281346048,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[2,5],"avail_no_missing":["2","3","5"],"object_location_counts":[{"shards":"2,3,5","objects":643},{"shards":"2,5","objects":2146}],"blocked_by":[],"up_primary":0,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.1","version":"79'161","reported_seq":595,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-09-23T21:14:02.770912+0000","last_change":"2023-09-23T21:13:12.204135+0000","last_active":"2023-09-23T21:14:02.770912+0000","last_peered":"2023-09-23T21:14:02.770912+0000","last_clean":"2023-09-23T21:14:02.770912+0000","last_became_active":"2023-09-23T21:13:12.203380+0000","last_became_peered":"2023-09-23T21:13:12.203380+0000","last_unstale":"2023-09-23T21:14:02.770912+0000","last_undegraded":"2023-09-23T21:14:02.770912+0000","last_fullsized":"2023-09-23T21:14:02.770912+0000","mapping_epoch":493,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:06:52.575900+0000","last_deep_scrub":"79'161","last_deep_scrub_stamp":"2023-09-23T21:05:30.652189+0000","last_clean_scrub_stamp":"2023-09-23T21:06:52.575900+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":191,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-09-23T21:14:02.771021+0000","last_change":"2023-09-23T21:13:13.238617+0000","last_active":"2023-09-23T21:14:02.771021+0000","last_peered":"2023-09-23T21:14:02.771021+0000","last_clean":"2023-09-23T21:14:02.771021+0000","last_became_active":"2023-09-23T21:13:13.232926+0000","last_became_peered":"2023-09-23T21:13:13.232926+0000","last_unstale":"2023-09-23T21:14:02.771021+0000","last_undegraded":"2023-09-23T21:14:02.771021+0000","last_fullsized":"2023-09-23T21:14:02.771021+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":495,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:17:21.823772+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":457,"reported_epoch":523,"state":"active+clean","last_fresh":"2023-09-23T21:14:01.516761+0000","last_change":"2023-09-23T21:13:14.144918+0000","last_active":"2023-09-23T21:14:01.516761+0000","last_peered":"2023-09-23T21:14:01.516761+0000","last_clean":"2023-09-23T21:14:01.516761+0000","last_became_active":"2023-09-23T21:13:14.132388+0000","last_became_peered":"2023-09-23T21:13:14.132388+0000","last_unstale":"2023-09-23T21:14:01.516761+0000","last_undegraded":"2023-09-23T21:14:01.516761+0000","last_fullsized":"2023-09-23T21:14:01.516761+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":495,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:56:02.619905+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,5],"acting":[4,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":344,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-09-23T21:14:02.768329+0000","last_change":"2023-09-23T21:12:10.764503+0000","last_active":"2023-09-23T21:14:02.768329+0000","last_peered":"2023-09-23T21:14:02.768329+0000","last_clean":"2023-09-23T21:14:02.768329+0000","last_became_active":"2023-09-23T21:12:10.763933+0000","last_became_peered":"2023-09-23T21:12:10.763933+0000","last_unstale":"2023-09-23T21:14:02.768329+0000","last_undegraded":"2023-09-23T21:14:02.768329+0000","last_fullsized":"2023-09-23T21:14:02.768329+0000","mapping_epoch":470,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:28:33.202825+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,7],"acting":[2,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"10.9","version":"525'4953","reported_seq":6093,"reported_epoch":525,"state":"active+clean","last_fresh":"2023-09-23T21:14:06.056649+0000","last_change":"2023-09-23T21:11:08.062606+0000","last_active":"2023-09-23T21:14:06.056649+0000","last_peered":"2023-09-23T21:14:06.056649+0000","last_clean":"2023-09-23T21:14:06.056649+0000","last_became_active":"2023-09-23T21:10:58.780961+0000","last_became_peered":"2023-09-23T21:10:58.780961+0000","last_unstale":"2023-09-23T21:14:06.056649+0000","last_undegraded":"2023-09-23T21:14:06.056649+0000","last_fullsized":"2023-09-23T21:14:06.056649+0000","mapping_epoch":428,"log_start":"0'0","ondisk_log_start":"0'0","created":416,"last_epoch_clean":439,"parent":"0.0","parent_split_bits":0,"last_scrub":"443'3049","last_scrub_stamp":"2023-09-23T21:11:08.062479+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_clean_scrub_stamp":"2023-09-23T21:11:08.062479+0000","objects_scrubbed":1623,"log_size":4953,"log_dups_size":0,"ondisk_log_size":4953,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:56:22.338088+0000","scrub_duration":0.28207626200000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":91750400,"num_objects":1400,"num_object_clones":0,"num_object_copies":4200,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1400,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1905,"num_write_kb":53824,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.2","version":"79'161","reported_seq":804,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-09-23T21:14:02.770766+0000","last_change":"2023-09-23T21:13:12.209907+0000","last_active":"2023-09-23T21:14:02.770766+0000","last_peered":"2023-09-23T21:14:02.770766+0000","last_clean":"2023-09-23T21:14:02.770766+0000","last_became_active":"2023-09-23T21:13:12.205664+0000","last_became_peered":"2023-09-23T21:13:12.205664+0000","last_unstale":"2023-09-23T21:14:02.770766+0000","last_undegraded":"2023-09-23T21:14:02.770766+0000","last_fullsized":"2023-09-23T21:14:02.770766+0000","mapping_epoch":493,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:24.802451+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:24.802451+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:26:16.035788+0000","scrub_duration":0.0090861250000000005,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,4],"acting":[0,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":533,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-09-23T21:14:02.768074+0000","last_change":"2023-09-23T21:13:12.206980+0000","last_active":"2023-09-23T21:14:02.768074+0000","last_peered":"2023-09-23T21:14:02.768074+0000","last_clean":"2023-09-23T21:14:02.768074+0000","last_became_active":"2023-09-23T21:13:12.206650+0000","last_became_peered":"2023-09-23T21:13:12.206650+0000","last_unstale":"2023-09-23T21:14:02.768074+0000","last_undegraded":"2023-09-23T21:14:02.768074+0000","last_fullsized":"2023-09-23T21:14:02.768074+0000","mapping_epoch":493,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:10:01.892936+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":516,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-09-23T21:14:02.767188+0000","last_change":"2023-09-23T21:13:13.407073+0000","last_active":"2023-09-23T21:14:02.767188+0000","last_peered":"2023-09-23T21:14:02.767188+0000","last_clean":"2023-09-23T21:14:02.767188+0000","last_became_active":"2023-09-23T21:13:13.395047+0000","last_became_peered":"2023-09-23T21:13:13.395047+0000","last_unstale":"2023-09-23T21:14:02.767188+0000","last_undegraded":"2023-09-23T21:14:02.767188+0000","last_fullsized":"2023-09-23T21:14:02.767188+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":495,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:04:43.384190+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,5],"acting":[2,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":34,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-09-23T21:14:02.771058+0000","last_change":"2023-09-23T21:13:30.785054+0000","last_active":"2023-09-23T21:14:02.771058+0000","last_peered":"2023-09-23T21:14:02.771058+0000","last_clean":"2023-09-23T21:14:02.771058+0000","last_became_active":"2023-09-23T21:13:30.784701+0000","last_became_peered":"2023-09-23T21:13:30.784701+0000","last_unstale":"2023-09-23T21:14:02.771058+0000","last_undegraded":"2023-09-23T21:14:02.771058+0000","last_fullsized":"2023-09-23T21:14:02.771058+0000","mapping_epoch":501,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:19:20.152252+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.b","version":"524'7155","reported_seq":13660,"reported_epoch":524,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2023-09-23T21:14:03.696892+0000","last_change":"2023-09-23T21:13:59.891994+0000","last_active":"2023-09-23T21:14:03.696892+0000","last_peered":"2023-09-23T21:14:03.696892+0000","last_clean":"2023-09-23T21:11:26.732248+0000","last_became_active":"2023-09-23T21:13:14.937458+0000","last_became_peered":"2023-09-23T21:13:14.937458+0000","last_unstale":"2023-09-23T21:14:03.696892+0000","last_undegraded":"2023-09-23T21:13:59.884123+0000","last_fullsized":"2023-09-23T21:14:03.696892+0000","mapping_epoch":494,"log_start":"426'600","ondisk_log_start":"426'600","created":416,"last_epoch_clean":430,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_clean_scrub_stamp":"2023-09-23T21:10:15.663179+0000","objects_scrubbed":0,"log_size":6555,"log_dups_size":0,"ondisk_log_size":6555,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:36:15.760947+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":192479232,"num_objects":2937,"num_object_clones":0,"num_object_copies":8811,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1337,"num_objects_misplaced":2268,"num_objects_unfound":0,"num_objects_dirty":2937,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7155,"num_write_kb":322944,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3164,"num_bytes_recovered":207159296,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[1,4,7],"avail_no_missing":["1","7"],"object_location_counts":[{"shards":"0,1,7","objects":2260},{"shards":"1,4,7","objects":602}],"blocked_by":[],"up_primary":0,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":310,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-09-23T21:14:02.768450+0000","last_change":"2023-09-23T21:10:26.769764+0000","last_active":"2023-09-23T21:14:02.768450+0000","last_peered":"2023-09-23T21:14:02.768450+0000","last_clean":"2023-09-23T21:14:02.768450+0000","last_became_active":"2023-09-23T21:10:26.769326+0000","last_became_peered":"2023-09-23T21:10:26.769326+0000","last_unstale":"2023-09-23T21:14:02.768450+0000","last_undegraded":"2023-09-23T21:14:02.768450+0000","last_fullsized":"2023-09-23T21:14:02.768450+0000","mapping_epoch":423,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":424,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:57:45.219837+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"79'161","reported_seq":707,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-09-23T21:14:02.771755+0000","last_change":"2023-09-23T21:13:33.766895+0000","last_active":"2023-09-23T21:14:02.771755+0000","last_peered":"2023-09-23T21:14:02.771755+0000","last_clean":"2023-09-23T21:14:02.771755+0000","last_became_active":"2023-09-23T21:13:33.766538+0000","last_became_peered":"2023-09-23T21:13:33.766538+0000","last_unstale":"2023-09-23T21:14:02.771755+0000","last_undegraded":"2023-09-23T21:14:02.771755+0000","last_fullsized":"2023-09-23T21:14:02.771755+0000","mapping_epoch":504,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":505,"parent":"0.0","parent_split_bits":0,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:20.738273+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:20.738273+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:35:35.855987+0000","scrub_duration":0.0088336339999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,2],"acting":[0,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":530,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-09-23T21:14:00.508602+0000","last_change":"2023-09-23T21:11:08.760417+0000","last_active":"2023-09-23T21:14:00.508602+0000","last_peered":"2023-09-23T21:14:00.508602+0000","last_clean":"2023-09-23T21:14:00.508602+0000","last_became_active":"2023-09-23T21:11:08.760150+0000","last_became_peered":"2023-09-23T21:11:08.760150+0000","last_unstale":"2023-09-23T21:14:00.508602+0000","last_undegraded":"2023-09-23T21:14:00.508602+0000","last_fullsized":"2023-09-23T21:14:00.508602+0000","mapping_epoch":443,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":444,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:23:10.849768+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":98,"reported_epoch":523,"state":"active+clean","last_fresh":"2023-09-23T21:14:01.499490+0000","last_change":"2023-09-23T21:13:12.216020+0000","last_active":"2023-09-23T21:14:01.499490+0000","last_peered":"2023-09-23T21:14:01.499490+0000","last_clean":"2023-09-23T21:14:01.499490+0000","last_became_active":"2023-09-23T21:13:12.215180+0000","last_became_peered":"2023-09-23T21:13:12.215180+0000","last_unstale":"2023-09-23T21:14:01.499490+0000","last_undegraded":"2023-09-23T21:14:01.499490+0000","last_fullsized":"2023-09-23T21:14:01.499490+0000","mapping_epoch":493,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:26:18.704684+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.8","version":"525'4817","reported_seq":7208,"reported_epoch":525,"state":"active+clean","last_fresh":"2023-09-23T21:14:05.077803+0000","last_change":"2023-09-23T21:14:03.796930+0000","last_active":"2023-09-23T21:14:05.077803+0000","last_peered":"2023-09-23T21:14:05.077803+0000","last_clean":"2023-09-23T21:14:05.077803+0000","last_became_active":"2023-09-23T21:14:03.786177+0000","last_became_peered":"2023-09-23T21:14:03.786177+0000","last_unstale":"2023-09-23T21:14:05.077803+0000","last_undegraded":"2023-09-23T21:14:05.077803+0000","last_fullsized":"2023-09-23T21:14:05.077803+0000","mapping_epoch":524,"log_start":"0'0","ondisk_log_start":"0'0","created":416,"last_epoch_clean":525,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_clean_scrub_stamp":"2023-09-23T21:10:15.663179+0000","objects_scrubbed":0,"log_size":4817,"log_dups_size":0,"ondisk_log_size":4817,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":89227264,"num_objects":1361,"num_object_clones":0,"num_object_copies":4083,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1361,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3467,"num_write_kb":154528,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":880,"num_bytes_recovered":16908288,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,7],"acting":[0,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.3","version":"79'161","reported_seq":826,"reported_epoch":523,"state":"active+clean","last_fresh":"2023-09-23T21:14:01.505286+0000","last_change":"2023-09-23T21:13:33.771279+0000","last_active":"2023-09-23T21:14:01.505286+0000","last_peered":"2023-09-23T21:14:01.505286+0000","last_clean":"2023-09-23T21:14:01.505286+0000","last_became_active":"2023-09-23T21:13:33.770980+0000","last_became_peered":"2023-09-23T21:13:33.770980+0000","last_unstale":"2023-09-23T21:14:01.505286+0000","last_undegraded":"2023-09-23T21:14:01.505286+0000","last_fullsized":"2023-09-23T21:14:01.505286+0000","mapping_epoch":504,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":505,"parent":"0.0","parent_split_bits":3,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:05:57.832302+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:05:57.832302+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:19:42.309604+0000","scrub_duration":0.010206847999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":341,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-09-23T21:14:02.772087+0000","last_change":"2023-09-23T21:13:13.212775+0000","last_active":"2023-09-23T21:14:02.772087+0000","last_peered":"2023-09-23T21:14:02.772087+0000","last_clean":"2023-09-23T21:14:02.772087+0000","last_became_active":"2023-09-23T21:13:13.212460+0000","last_became_peered":"2023-09-23T21:13:13.212460+0000","last_unstale":"2023-09-23T21:14:02.772087+0000","last_undegraded":"2023-09-23T21:14:02.772087+0000","last_fullsized":"2023-09-23T21:14:02.772087+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":495,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:27:10.853970+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":523,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-09-23T21:14:02.770641+0000","last_change":"2023-09-23T21:13:44.409212+0000","last_active":"2023-09-23T21:14:02.770641+0000","last_peered":"2023-09-23T21:14:02.770641+0000","last_clean":"2023-09-23T21:14:02.770641+0000","last_became_active":"2023-09-23T21:13:44.408958+0000","last_became_peered":"2023-09-23T21:13:44.408958+0000","last_unstale":"2023-09-23T21:14:02.770641+0000","last_undegraded":"2023-09-23T21:14:02.770641+0000","last_fullsized":"2023-09-23T21:14:02.770641+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:03:52.749964+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":34,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-09-23T21:14:02.771394+0000","last_change":"2023-09-23T21:13:30.804379+0000","last_active":"2023-09-23T21:14:02.771394+0000","last_peered":"2023-09-23T21:14:02.771394+0000","last_clean":"2023-09-23T21:14:02.771394+0000","last_became_active":"2023-09-23T21:13:30.799907+0000","last_became_peered":"2023-09-23T21:13:30.799907+0000","last_unstale":"2023-09-23T21:14:02.771394+0000","last_undegraded":"2023-09-23T21:14:02.771394+0000","last_fullsized":"2023-09-23T21:14:02.771394+0000","mapping_epoch":501,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:49:42.859212+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,7],"acting":[0,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.d","version":"524'7067","reported_seq":7224,"reported_epoch":524,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-23T21:14:03.182723+0000","last_change":"2023-09-23T21:13:31.801486+0000","last_active":"2023-09-23T21:14:03.182723+0000","last_peered":"2023-09-23T21:14:03.182723+0000","last_clean":"2023-09-23T21:11:37.659929+0000","last_became_active":"2023-09-23T21:13:31.786626+0000","last_became_peered":"2023-09-23T21:13:31.786626+0000","last_unstale":"2023-09-23T21:14:03.182723+0000","last_undegraded":"2023-09-23T21:13:31.760908+0000","last_fullsized":"2023-09-23T21:14:03.182723+0000","mapping_epoch":502,"log_start":"440'3095","ondisk_log_start":"440'3095","created":416,"last_epoch_clean":417,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_clean_scrub_stamp":"2023-09-23T21:10:15.663179+0000","objects_scrubbed":0,"log_size":3972,"log_dups_size":0,"ondisk_log_size":3972,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:46:02.424493+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":192282648,"num_objects":2935,"num_object_clones":0,"num_object_copies":8805,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1265,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2935,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":7067,"num_write_kb":320001,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,3],"avail_no_missing":["4","3","7"],"object_location_counts":[{"shards":"0,3,4","objects":2856},{"shards":"3,4","objects":79}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.6","version":"79'161","reported_seq":443,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-09-23T21:14:02.771177+0000","last_change":"2023-09-23T21:13:33.770715+0000","last_active":"2023-09-23T21:14:02.771177+0000","last_peered":"2023-09-23T21:14:02.771177+0000","last_clean":"2023-09-23T21:14:02.771177+0000","last_became_active":"2023-09-23T21:13:33.769303+0000","last_became_peered":"2023-09-23T21:13:33.769303+0000","last_unstale":"2023-09-23T21:14:02.771177+0000","last_undegraded":"2023-09-23T21:14:02.771177+0000","last_fullsized":"2023-09-23T21:14:02.771177+0000","mapping_epoch":504,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":505,"parent":"0.0","parent_split_bits":3,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:10:24.268662+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:10:24.268662+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13417,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":11,"num_read_kb":20,"num_write":5,"num_write_kb":55,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":40252,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":149,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-09-23T21:14:00.484028+0000","last_change":"2023-09-23T21:13:30.770826+0000","last_active":"2023-09-23T21:14:00.484028+0000","last_peered":"2023-09-23T21:14:00.484028+0000","last_clean":"2023-09-23T21:14:00.484028+0000","last_became_active":"2023-09-23T21:13:30.770512+0000","last_became_peered":"2023-09-23T21:13:30.770512+0000","last_unstale":"2023-09-23T21:14:00.484028+0000","last_undegraded":"2023-09-23T21:14:00.484028+0000","last_fullsized":"2023-09-23T21:14:00.484028+0000","mapping_epoch":501,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:44:23.470538+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":407,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-09-23T21:14:02.768605+0000","last_change":"2023-09-23T21:13:30.790590+0000","last_active":"2023-09-23T21:14:02.768605+0000","last_peered":"2023-09-23T21:14:02.768605+0000","last_clean":"2023-09-23T21:14:02.768605+0000","last_became_active":"2023-09-23T21:13:30.790253+0000","last_became_peered":"2023-09-23T21:13:30.790253+0000","last_unstale":"2023-09-23T21:14:02.768605+0000","last_undegraded":"2023-09-23T21:14:02.768605+0000","last_fullsized":"2023-09-23T21:14:02.768605+0000","mapping_epoch":501,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:23:41.843250+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,0],"acting":[2,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":470,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-09-23T21:14:02.771938+0000","last_change":"2023-09-23T21:13:12.210546+0000","last_active":"2023-09-23T21:14:02.771938+0000","last_peered":"2023-09-23T21:14:02.771938+0000","last_clean":"2023-09-23T21:14:02.771938+0000","last_became_active":"2023-09-23T21:13:12.205934+0000","last_became_peered":"2023-09-23T21:13:12.205934+0000","last_unstale":"2023-09-23T21:14:02.771938+0000","last_undegraded":"2023-09-23T21:14:02.771938+0000","last_fullsized":"2023-09-23T21:14:02.771938+0000","mapping_epoch":493,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:21:18.299428+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.e","version":"524'7241","reported_seq":14207,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-09-23T21:14:03.166070+0000","last_change":"2023-09-23T21:12:38.623611+0000","last_active":"2023-09-23T21:14:03.166070+0000","last_peered":"2023-09-23T21:14:03.166070+0000","last_clean":"2023-09-23T21:14:03.166070+0000","last_became_active":"2023-09-23T21:12:38.610155+0000","last_became_peered":"2023-09-23T21:12:38.610155+0000","last_unstale":"2023-09-23T21:14:03.166070+0000","last_undegraded":"2023-09-23T21:14:03.166070+0000","last_fullsized":"2023-09-23T21:14:03.166070+0000","mapping_epoch":483,"log_start":"440'3293","ondisk_log_start":"440'3293","created":416,"last_epoch_clean":484,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_clean_scrub_stamp":"2023-09-23T21:10:15.663179+0000","objects_scrubbed":0,"log_size":3948,"log_dups_size":0,"ondisk_log_size":3948,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:19:27.097170+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":195493888,"num_objects":2983,"num_object_clones":0,"num_object_copies":8949,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2983,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7241,"num_write_kb":327168,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3412,"num_bytes_recovered":223608832,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.5","version":"79'161","reported_seq":551,"reported_epoch":523,"state":"active+clean","last_fresh":"2023-09-23T21:14:01.499423+0000","last_change":"2023-09-23T21:13:38.373969+0000","last_active":"2023-09-23T21:14:01.499423+0000","last_peered":"2023-09-23T21:14:01.499423+0000","last_clean":"2023-09-23T21:14:01.499423+0000","last_became_active":"2023-09-23T21:13:38.373633+0000","last_became_peered":"2023-09-23T21:13:38.373633+0000","last_unstale":"2023-09-23T21:14:01.499423+0000","last_undegraded":"2023-09-23T21:14:01.499423+0000","last_fullsized":"2023-09-23T21:14:01.499423+0000","mapping_epoch":507,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":3,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:09:34.892947+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:09:34.892947+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13417,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":11,"num_read_kb":20,"num_write":5,"num_write_kb":55,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":40252,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":65,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-09-23T21:14:02.771313+0000","last_change":"2023-09-23T21:13:12.235743+0000","last_active":"2023-09-23T21:14:02.771313+0000","last_peered":"2023-09-23T21:14:02.771313+0000","last_clean":"2023-09-23T21:14:02.771313+0000","last_became_active":"2023-09-23T21:13:12.216241+0000","last_became_peered":"2023-09-23T21:13:12.216241+0000","last_unstale":"2023-09-23T21:14:02.771313+0000","last_undegraded":"2023-09-23T21:14:02.771313+0000","last_fullsized":"2023-09-23T21:14:02.771313+0000","mapping_epoch":493,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:12:19.090710+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":504,"reported_epoch":523,"state":"active+clean","last_fresh":"2023-09-23T21:14:01.500083+0000","last_change":"2023-09-23T21:13:30.807520+0000","last_active":"2023-09-23T21:14:01.500083+0000","last_peered":"2023-09-23T21:14:01.500083+0000","last_clean":"2023-09-23T21:14:01.500083+0000","last_became_active":"2023-09-23T21:13:30.807222+0000","last_became_peered":"2023-09-23T21:13:30.807222+0000","last_unstale":"2023-09-23T21:14:01.500083+0000","last_undegraded":"2023-09-23T21:14:01.500083+0000","last_fullsized":"2023-09-23T21:14:01.500083+0000","mapping_epoch":501,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:40:31.989749+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":366,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-09-23T21:14:02.775387+0000","last_change":"2023-09-23T21:13:30.794095+0000","last_active":"2023-09-23T21:14:02.775387+0000","last_peered":"2023-09-23T21:14:02.775387+0000","last_clean":"2023-09-23T21:14:02.775387+0000","last_became_active":"2023-09-23T21:13:30.793791+0000","last_became_peered":"2023-09-23T21:13:30.793791+0000","last_unstale":"2023-09-23T21:14:02.775387+0000","last_undegraded":"2023-09-23T21:14:02.775387+0000","last_fullsized":"2023-09-23T21:14:02.775387+0000","mapping_epoch":501,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:36:28.707362+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.f","version":"525'7224","reported_seq":7424,"reported_epoch":525,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T21:14:04.156122+0000","last_change":"2023-09-23T21:13:59.874192+0000","last_active":"2023-09-23T21:14:04.156122+0000","last_peered":"2023-09-23T21:14:04.156122+0000","last_clean":"2023-09-23T21:11:07.652290+0000","last_became_active":"2023-09-23T21:11:09.897463+0000","last_became_peered":"2023-09-23T21:11:09.897463+0000","last_unstale":"2023-09-23T21:14:04.156122+0000","last_undegraded":"2023-09-23T21:11:27.781677+0000","last_fullsized":"2023-09-23T21:11:09.761652+0000","mapping_epoch":444,"log_start":"447'3509","ondisk_log_start":"447'3509","created":416,"last_epoch_clean":417,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_clean_scrub_stamp":"2023-09-23T21:10:15.663179+0000","objects_scrubbed":0,"log_size":3715,"log_dups_size":0,"ondisk_log_size":3715,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:42:51.588441+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":189399040,"num_objects":2890,"num_object_clones":0,"num_object_copies":8670,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3295,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2890,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7224,"num_write_kb":323648,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":32,"num_bytes_recovered":2097152,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2],"avail_no_missing":["3","1","2"],"object_location_counts":[{"shards":"1,2,3","objects":1160},{"shards":"2,3","objects":-405}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.4","version":"79'161","reported_seq":895,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-09-23T21:14:02.776948+0000","last_change":"2023-09-23T21:10:29.807785+0000","last_active":"2023-09-23T21:14:02.776948+0000","last_peered":"2023-09-23T21:14:02.776948+0000","last_clean":"2023-09-23T21:14:02.776948+0000","last_became_active":"2023-09-23T21:10:29.807366+0000","last_became_peered":"2023-09-23T21:10:29.807366+0000","last_unstale":"2023-09-23T21:14:02.776948+0000","last_undegraded":"2023-09-23T21:14:02.776948+0000","last_fullsized":"2023-09-23T21:14:02.776948+0000","mapping_epoch":426,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":427,"parent":"0.0","parent_split_bits":3,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:20.012764+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:20.012764+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:24:45.763834+0000","scrub_duration":0.0045251919999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":463,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-09-23T21:14:02.768603+0000","last_change":"2023-09-23T21:10:26.768364+0000","last_active":"2023-09-23T21:14:02.768603+0000","last_peered":"2023-09-23T21:14:02.768603+0000","last_clean":"2023-09-23T21:14:02.768603+0000","last_became_active":"2023-09-23T21:10:26.768052+0000","last_became_peered":"2023-09-23T21:10:26.768052+0000","last_unstale":"2023-09-23T21:14:02.768603+0000","last_undegraded":"2023-09-23T21:14:02.768603+0000","last_fullsized":"2023-09-23T21:14:02.768603+0000","mapping_epoch":423,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":424,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:22:42.201200+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,3],"acting":[2,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.6","version":"70'1","reported_seq":495,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-09-23T21:14:02.771506+0000","last_change":"2023-09-23T21:13:30.793290+0000","last_active":"2023-09-23T21:14:02.771506+0000","last_peered":"2023-09-23T21:14:02.771506+0000","last_clean":"2023-09-23T21:14:02.771506+0000","last_became_active":"2023-09-23T21:13:30.786079+0000","last_became_peered":"2023-09-23T21:13:30.786079+0000","last_unstale":"2023-09-23T21:14:02.771506+0000","last_undegraded":"2023-09-23T21:14:02.771506+0000","last_fullsized":"2023-09-23T21:14:02.771506+0000","mapping_epoch":501,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:34:43.281095+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":98,"reported_epoch":523,"state":"active+clean","last_fresh":"2023-09-23T21:14:01.497975+0000","last_change":"2023-09-23T21:13:12.216895+0000","last_active":"2023-09-23T21:14:01.497975+0000","last_peered":"2023-09-23T21:14:01.497975+0000","last_clean":"2023-09-23T21:14:01.497975+0000","last_became_active":"2023-09-23T21:13:12.216654+0000","last_became_peered":"2023-09-23T21:13:12.216654+0000","last_unstale":"2023-09-23T21:14:01.497975+0000","last_undegraded":"2023-09-23T21:14:01.497975+0000","last_fullsized":"2023-09-23T21:14:01.497975+0000","mapping_epoch":493,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:02:40.430096+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.19","version":"79'161","reported_seq":677,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-09-23T21:14:02.767420+0000","last_change":"2023-09-23T21:12:30.442914+0000","last_active":"2023-09-23T21:14:02.767420+0000","last_peered":"2023-09-23T21:14:02.767420+0000","last_clean":"2023-09-23T21:14:02.767420+0000","last_became_active":"2023-09-23T21:12:30.442578+0000","last_became_peered":"2023-09-23T21:12:30.442578+0000","last_unstale":"2023-09-23T21:14:02.767420+0000","last_undegraded":"2023-09-23T21:14:02.767420+0000","last_fullsized":"2023-09-23T21:14:02.767420+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":479,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:10:37.696951+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:10:37.696951+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.001386504,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,3],"acting":[2,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"10.12","version":"524'4903","reported_seq":5925,"reported_epoch":524,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T21:14:03.033460+0000","last_change":"2023-09-23T21:13:13.419184+0000","last_active":"2023-09-23T21:14:03.033460+0000","last_peered":"2023-09-23T21:14:03.033460+0000","last_clean":"2023-09-23T21:11:45.774771+0000","last_became_active":"2023-09-23T21:13:13.409281+0000","last_became_peered":"2023-09-23T21:13:13.409281+0000","last_unstale":"2023-09-23T21:14:03.033460+0000","last_undegraded":"2023-09-23T21:13:13.284063+0000","last_fullsized":"2023-09-23T21:13:13.191117+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":438,"last_epoch_clean":439,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_clean_scrub_stamp":"2023-09-23T21:10:15.663179+0000","objects_scrubbed":0,"log_size":4903,"log_dups_size":0,"ondisk_log_size":4903,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":92405760,"num_objects":1410,"num_object_clones":0,"num_object_copies":4230,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2706,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1410,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3526,"num_write_kb":157952,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":432,"num_bytes_recovered":28311552,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,3],"avail_no_missing":["4","1"],"object_location_counts":[{"shards":"0,1,4","objects":1372},{"shards":"1,4","objects":41},{"shards":"3,4","objects":-3}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":34,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-09-23T21:14:02.771902+0000","last_change":"2023-09-23T21:13:30.791297+0000","last_active":"2023-09-23T21:14:02.771902+0000","last_peered":"2023-09-23T21:14:02.771902+0000","last_clean":"2023-09-23T21:14:02.771902+0000","last_became_active":"2023-09-23T21:13:30.788721+0000","last_became_peered":"2023-09-23T21:13:30.788721+0000","last_unstale":"2023-09-23T21:14:02.771902+0000","last_undegraded":"2023-09-23T21:14:02.771902+0000","last_fullsized":"2023-09-23T21:14:02.771902+0000","mapping_epoch":501,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:31:18.147883+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":28,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-09-23T21:14:02.768510+0000","last_change":"2023-09-23T21:13:38.373814+0000","last_active":"2023-09-23T21:14:02.768510+0000","last_peered":"2023-09-23T21:14:02.768510+0000","last_clean":"2023-09-23T21:14:02.768510+0000","last_became_active":"2023-09-23T21:13:38.373458+0000","last_became_peered":"2023-09-23T21:13:38.373458+0000","last_unstale":"2023-09-23T21:14:02.768510+0000","last_undegraded":"2023-09-23T21:14:02.768510+0000","last_fullsized":"2023-09-23T21:14:02.768510+0000","mapping_epoch":507,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:23:08.288401+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.1e","version":"79'161","reported_seq":691,"reported_epoch":523,"state":"active+clean","last_fresh":"2023-09-23T21:14:01.499243+0000","last_change":"2023-09-23T21:12:30.448761+0000","last_active":"2023-09-23T21:14:01.499243+0000","last_peered":"2023-09-23T21:14:01.499243+0000","last_clean":"2023-09-23T21:14:01.499243+0000","last_became_active":"2023-09-23T21:12:30.448043+0000","last_became_peered":"2023-09-23T21:12:30.448043+0000","last_unstale":"2023-09-23T21:14:01.499243+0000","last_undegraded":"2023-09-23T21:14:01.499243+0000","last_fullsized":"2023-09-23T21:14:01.499243+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":479,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:10:39.567232+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:10:39.567232+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.005229694,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.15","version":"525'4834","reported_seq":4976,"reported_epoch":525,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T21:14:05.864700+0000","last_change":"2023-09-23T21:13:13.436807+0000","last_active":"2023-09-23T21:14:05.864700+0000","last_peered":"2023-09-23T21:14:05.864700+0000","last_clean":"2023-09-23T21:11:45.765730+0000","last_became_active":"2023-09-23T21:13:13.429206+0000","last_became_peered":"2023-09-23T21:13:13.429206+0000","last_unstale":"2023-09-23T21:14:05.864700+0000","last_undegraded":"2023-09-23T21:13:13.246273+0000","last_fullsized":"2023-09-23T21:13:13.203825+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":438,"last_epoch_clean":439,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_clean_scrub_stamp":"2023-09-23T21:10:15.663179+0000","objects_scrubbed":0,"log_size":4834,"log_dups_size":0,"ondisk_log_size":4834,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":94896128,"num_objects":1448,"num_object_clones":0,"num_object_copies":4344,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":643,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1448,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3500,"num_write_kb":158336,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,3],"acting":[2,3],"avail_no_missing":["2","3"],"object_location_counts":[{"shards":"2,3","objects":1412}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":99,"reported_epoch":523,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-23T21:14:01.498603+0000","last_change":"2023-09-23T21:12:14.313202+0000","last_active":"2023-09-23T21:14:01.498603+0000","last_peered":"2023-09-23T21:14:01.498603+0000","last_clean":"2023-09-23T21:12:09.299355+0000","last_became_active":"2023-09-23T21:12:14.209408+0000","last_became_peered":"2023-09-23T21:12:14.209408+0000","last_unstale":"2023-09-23T21:14:01.498603+0000","last_undegraded":"2023-09-23T21:12:12.933857+0000","last_fullsized":"2023-09-23T21:14:01.498603+0000","mapping_epoch":470,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":444,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:45:40.142413+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,7],"acting":[4,5,7],"avail_no_missing":["4","1","5"],"object_location_counts":[{"shards":"1,4,5","objects":1}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":88,"reported_epoch":523,"state":"active+clean","last_fresh":"2023-09-23T21:14:01.506286+0000","last_change":"2023-09-23T21:12:10.770474+0000","last_active":"2023-09-23T21:14:01.506286+0000","last_peered":"2023-09-23T21:14:01.506286+0000","last_clean":"2023-09-23T21:14:01.506286+0000","last_became_active":"2023-09-23T21:12:10.770184+0000","last_became_peered":"2023-09-23T21:12:10.770184+0000","last_unstale":"2023-09-23T21:14:01.506286+0000","last_undegraded":"2023-09-23T21:14:01.506286+0000","last_fullsized":"2023-09-23T21:14:01.506286+0000","mapping_epoch":470,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:05:09.407151+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.14","version":"525'4803","reported_seq":10900,"reported_epoch":525,"state":"active+clean","last_fresh":"2023-09-23T21:14:05.149179+0000","last_change":"2023-09-23T21:14:01.515957+0000","last_active":"2023-09-23T21:14:05.149179+0000","last_peered":"2023-09-23T21:14:05.149179+0000","last_clean":"2023-09-23T21:14:05.149179+0000","last_became_active":"2023-09-23T21:14:01.506428+0000","last_became_peered":"2023-09-23T21:14:01.506428+0000","last_unstale":"2023-09-23T21:14:05.149179+0000","last_undegraded":"2023-09-23T21:14:05.149179+0000","last_fullsized":"2023-09-23T21:14:05.149179+0000","mapping_epoch":522,"log_start":"0'0","ondisk_log_start":"0'0","created":438,"last_epoch_clean":523,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_clean_scrub_stamp":"2023-09-23T21:10:15.663179+0000","objects_scrubbed":0,"log_size":4803,"log_dups_size":0,"ondisk_log_size":4803,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":94437376,"num_objects":1441,"num_object_clones":0,"num_object_copies":4323,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1441,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3469,"num_write_kb":157120,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2660,"num_bytes_recovered":126156800,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":314,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-09-23T21:14:02.768713+0000","last_change":"2023-09-23T21:12:10.771160+0000","last_active":"2023-09-23T21:14:02.768713+0000","last_peered":"2023-09-23T21:14:02.768713+0000","last_clean":"2023-09-23T21:14:02.768713+0000","last_became_active":"2023-09-23T21:12:10.757808+0000","last_became_peered":"2023-09-23T21:12:10.757808+0000","last_unstale":"2023-09-23T21:14:02.768713+0000","last_undegraded":"2023-09-23T21:14:02.768713+0000","last_fullsized":"2023-09-23T21:14:02.768713+0000","mapping_epoch":470,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:30:50.987139+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":98,"reported_epoch":523,"state":"active+clean","last_fresh":"2023-09-23T21:14:01.498464+0000","last_change":"2023-09-23T21:13:12.216602+0000","last_active":"2023-09-23T21:14:01.498464+0000","last_peered":"2023-09-23T21:14:01.498464+0000","last_clean":"2023-09-23T21:14:01.498464+0000","last_became_active":"2023-09-23T21:13:12.216204+0000","last_became_peered":"2023-09-23T21:13:12.216204+0000","last_unstale":"2023-09-23T21:14:01.498464+0000","last_undegraded":"2023-09-23T21:14:01.498464+0000","last_fullsized":"2023-09-23T21:14:01.498464+0000","mapping_epoch":493,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:23:37.043390+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1c","version":"79'161","reported_seq":890,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-09-23T21:14:02.775904+0000","last_change":"2023-09-23T21:13:13.402773+0000","last_active":"2023-09-23T21:14:02.775904+0000","last_peered":"2023-09-23T21:14:02.775904+0000","last_clean":"2023-09-23T21:14:02.775904+0000","last_became_active":"2023-09-23T21:13:13.402025+0000","last_became_peered":"2023-09-23T21:13:13.402025+0000","last_unstale":"2023-09-23T21:14:02.775904+0000","last_undegraded":"2023-09-23T21:14:02.775904+0000","last_fullsized":"2023-09-23T21:14:02.775904+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":479,"last_epoch_clean":495,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:22.040907+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:22.040907+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.013021176000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":163840,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":163840,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.17","version":"525'4850","reported_seq":9232,"reported_epoch":525,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T21:14:04.127104+0000","last_change":"2023-09-23T21:13:39.447969+0000","last_active":"2023-09-23T21:14:04.127104+0000","last_peered":"2023-09-23T21:14:04.127104+0000","last_clean":"2023-09-23T21:13:37.065694+0000","last_became_active":"2023-09-23T21:13:39.439739+0000","last_became_peered":"2023-09-23T21:13:39.439739+0000","last_unstale":"2023-09-23T21:14:04.127104+0000","last_undegraded":"2023-09-23T21:13:39.411853+0000","last_fullsized":"2023-09-23T21:13:39.404657+0000","mapping_epoch":508,"log_start":"0'0","ondisk_log_start":"0'0","created":438,"last_epoch_clean":487,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_clean_scrub_stamp":"2023-09-23T21:10:15.663179+0000","objects_scrubbed":0,"log_size":4850,"log_dups_size":0,"ondisk_log_size":4850,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":95059968,"num_objects":1451,"num_object_clones":0,"num_object_copies":4353,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1109,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1451,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3485,"num_write_kb":157920,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2112,"num_bytes_recovered":116719616,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,2],"acting":[3,5],"avail_no_missing":["3","4","5"],"object_location_counts":[{"shards":"3,4,5","objects":311},{"shards":"3,5","objects":1140}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":113,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-09-23T21:14:02.767873+0000","last_change":"2023-09-23T21:12:10.779066+0000","last_active":"2023-09-23T21:14:02.767873+0000","last_peered":"2023-09-23T21:14:02.767873+0000","last_clean":"2023-09-23T21:14:02.767873+0000","last_became_active":"2023-09-23T21:12:10.778691+0000","last_became_peered":"2023-09-23T21:12:10.778691+0000","last_unstale":"2023-09-23T21:14:02.767873+0000","last_undegraded":"2023-09-23T21:14:02.767873+0000","last_fullsized":"2023-09-23T21:14:02.767873+0000","mapping_epoch":470,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:07:22.043268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,7],"acting":[2,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":192,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-09-23T21:14:02.768174+0000","last_change":"2023-09-23T21:13:12.200767+0000","last_active":"2023-09-23T21:14:02.768174+0000","last_peered":"2023-09-23T21:14:02.768174+0000","last_clean":"2023-09-23T21:14:02.768174+0000","last_became_active":"2023-09-23T21:13:12.200435+0000","last_became_peered":"2023-09-23T21:13:12.200435+0000","last_unstale":"2023-09-23T21:14:02.768174+0000","last_undegraded":"2023-09-23T21:14:02.768174+0000","last_fullsized":"2023-09-23T21:14:02.768174+0000","mapping_epoch":493,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:15:44.937886+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.1d","version":"79'161","reported_seq":675,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-09-23T21:14:00.483617+0000","last_change":"2023-09-23T21:12:30.444331+0000","last_active":"2023-09-23T21:14:00.483617+0000","last_peered":"2023-09-23T21:14:00.483617+0000","last_clean":"2023-09-23T21:14:00.483617+0000","last_became_active":"2023-09-23T21:12:30.444001+0000","last_became_peered":"2023-09-23T21:12:30.444001+0000","last_unstale":"2023-09-23T21:14:00.483617+0000","last_undegraded":"2023-09-23T21:14:00.483617+0000","last_fullsized":"2023-09-23T21:14:00.483617+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":479,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:09:36.858077+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:09:36.858077+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0024987500000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"10.16","version":"524'4945","reported_seq":8450,"reported_epoch":524,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T21:14:03.093738+0000","last_change":"2023-09-23T21:13:14.424843+0000","last_active":"2023-09-23T21:14:03.093738+0000","last_peered":"2023-09-23T21:14:03.093738+0000","last_clean":"2023-09-23T21:11:45.652255+0000","last_became_active":"2023-09-23T21:13:14.414710+0000","last_became_peered":"2023-09-23T21:13:14.414710+0000","last_unstale":"2023-09-23T21:14:03.093738+0000","last_undegraded":"2023-09-23T21:13:13.211957+0000","last_fullsized":"2023-09-23T21:13:13.188859+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":438,"last_epoch_clean":460,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_clean_scrub_stamp":"2023-09-23T21:10:15.663179+0000","objects_scrubbed":0,"log_size":4945,"log_dups_size":0,"ondisk_log_size":4945,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":96141312,"num_objects":1467,"num_object_clones":0,"num_object_copies":4401,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":663,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1467,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3539,"num_write_kb":160192,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1670,"num_bytes_recovered":109445120,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,4],"acting":[4,5],"avail_no_missing":["4","5"],"object_location_counts":[{"shards":"1,4,5","objects":33},{"shards":"4,5","objects":1434}],"blocked_by":[],"up_primary":0,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":51,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-09-23T21:14:02.770898+0000","last_change":"2023-09-23T21:13:38.380574+0000","last_active":"2023-09-23T21:14:02.770898+0000","last_peered":"2023-09-23T21:14:02.770898+0000","last_clean":"2023-09-23T21:14:02.770898+0000","last_became_active":"2023-09-23T21:13:38.373252+0000","last_became_peered":"2023-09-23T21:13:38.373252+0000","last_unstale":"2023-09-23T21:14:02.770898+0000","last_undegraded":"2023-09-23T21:14:02.770898+0000","last_fullsized":"2023-09-23T21:14:02.770898+0000","mapping_epoch":507,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:58:14.788031+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":11,"num_bytes_recovered":3324,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":98,"reported_epoch":523,"state":"active+clean","last_fresh":"2023-09-23T21:14:01.503772+0000","last_change":"2023-09-23T21:12:10.793610+0000","last_active":"2023-09-23T21:14:01.503772+0000","last_peered":"2023-09-23T21:14:01.503772+0000","last_clean":"2023-09-23T21:14:01.503772+0000","last_became_active":"2023-09-23T21:12:10.793081+0000","last_became_peered":"2023-09-23T21:12:10.793081+0000","last_unstale":"2023-09-23T21:14:01.503772+0000","last_undegraded":"2023-09-23T21:14:01.503772+0000","last_fullsized":"2023-09-23T21:14:01.503772+0000","mapping_epoch":470,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:34:36.291550+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":10,"num_pg":26,"stat_sum":{"num_bytes":3027370008,"num_objects":46197,"num_object_clones":0,"num_object_copies":138591,"num_objects_missing_on_primary":3546,"num_objects_missing":3546,"num_objects_degraded":25392,"num_objects_misplaced":5155,"num_objects_unfound":0,"num_objects_dirty":46197,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":101177,"num_write_kb":4309313,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":35987,"num_bytes_recovered":2148892672,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":11722956800,"data_stored":11722948656,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":125105,"ondisk_log_size":125105,"up":78,"acting":62,"num_store_stats":7},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":59,"num_bytes_recovered":8898,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":126976,"data_stored":22458,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":14322,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":31,"stat_sum":{"num_bytes":697708,"num_objects":3,"num_object_clones":0,"num_object_copies":9,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3,"num_whiteouts":0,"num_read":89,"num_read_kb":165,"num_write":44,"num_write_kb":440,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":649699,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2109440,"data_stored":2098784,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4991,"ondisk_log_size":4991,"up":93,"acting":93,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201683,"num_pgs":69,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1613980,"kb_used_data":1134676,"kb_used_omap":0,"kb_used_meta":479232,"kb_avail":92131172,"statfs":{"total":95995035648,"available":94342320128,"internally_reserved":0,"allocated":1161908224,"data_stored":1160397791,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":490733568},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:11:02 2023","interfaces":[{"interface":"back","average":{"1min":0.65600000000000003,"5min":1.3340000000000001,"15min":1.6699999999999999},"min":{"1min":0.23599999999999999,"5min":0.22800000000000001,"15min":0.216},"max":{"1min":1.966,"5min":7.0170000000000003,"15min":43.994999999999997},"last":0.29099999999999998},{"interface":"front","average":{"1min":0.68400000000000005,"5min":1.4179999999999999,"15min":1.7869999999999999},"min":{"1min":0.26800000000000002,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":1.9770000000000001,"5min":8.1120000000000001,"15min":44.206000000000003},"last":1.2410000000000001}]},{"osd":1,"last update":"Sat Sep 23 21:13:07 2023","interfaces":[{"interface":"back","average":{"1min":1.1739999999999999,"5min":1.851,"15min":2.1230000000000002},"min":{"1min":0.27500000000000002,"5min":0.27100000000000002,"15min":0.26700000000000002},"max":{"1min":6.4649999999999999,"5min":14.034000000000001,"15min":14.034000000000001},"last":0.50800000000000001},{"interface":"front","average":{"1min":1.966,"5min":1.889,"15min":2.0139999999999998},"min":{"1min":0.38,"5min":0.27100000000000002,"15min":0.26300000000000001},"max":{"1min":9.234,"5min":11.226000000000001,"15min":11.226000000000001},"last":3.137}]},{"osd":2,"last update":"Sat Sep 23 21:13:29 2023","interfaces":[{"interface":"back","average":{"1min":1.254,"5min":1.3839999999999999,"15min":1.9119999999999999},"min":{"1min":0.26400000000000001,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":6.4219999999999997,"5min":6.5469999999999997,"15min":14.478999999999999},"last":1.173},{"interface":"front","average":{"1min":1.298,"5min":1.5129999999999999,"15min":1.819},"min":{"1min":0.26700000000000002,"5min":0.26700000000000002,"15min":0.26500000000000001},"max":{"1min":9.1839999999999993,"5min":11.198,"15min":14.135},"last":0.37}]},{"osd":3,"last update":"Sat Sep 23 21:13:57 2023","interfaces":[{"interface":"back","average":{"1min":1.022,"5min":1.296,"15min":2.0590000000000002},"min":{"1min":0.33000000000000002,"5min":0.316,"15min":0.29099999999999998},"max":{"1min":6.4930000000000003,"5min":7.5190000000000001,"15min":31.786000000000001},"last":0.30599999999999999},{"interface":"front","average":{"1min":1.1160000000000001,"5min":1.3859999999999999,"15min":2.0409999999999999},"min":{"1min":0.37,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":5.9329999999999998,"5min":11.244999999999999,"15min":32.429000000000002},"last":3.9860000000000002}]},{"osd":4,"last update":"Sat Sep 23 21:13:53 2023","interfaces":[{"interface":"back","average":{"1min":1.5980000000000001,"5min":1.5209999999999999,"15min":1.097},"min":{"1min":0.28999999999999998,"5min":0.255,"15min":0.248},"max":{"1min":7.6609999999999996,"5min":11.297000000000001,"15min":11.297000000000001},"last":0.32500000000000001},{"interface":"front","average":{"1min":1.387,"5min":1.1679999999999999,"15min":1.0669999999999999},"min":{"1min":0.314,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":8.6059999999999999,"5min":11.093999999999999,"15min":12.752000000000001},"last":1.6930000000000001}]},{"osd":5,"last update":"Sat Sep 23 21:13:53 2023","interfaces":[{"interface":"back","average":{"1min":1.4670000000000001,"5min":1.0549999999999999,"15min":1.014},"min":{"1min":0.29899999999999999,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":7.5910000000000002,"5min":9.4789999999999992,"15min":9.4789999999999992},"last":1.575},{"interface":"front","average":{"1min":1.4950000000000001,"5min":1.3680000000000001,"15min":1.099},"min":{"1min":0.39700000000000002,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":7.6929999999999996,"5min":11.257,"15min":11.257},"last":0.34699999999999998}]},{"osd":6,"last update":"Sat Sep 23 21:13:07 2023","interfaces":[{"interface":"back","average":{"1min":2.6240000000000001,"5min":1.256,"15min":1.0429999999999999},"min":{"1min":0.46500000000000002,"5min":0.30099999999999999,"15min":0.28899999999999998},"max":{"1min":10.941000000000001,"5min":10.941000000000001,"15min":10.941000000000001},"last":0.46400000000000002},{"interface":"front","average":{"1min":2.0249999999999999,"5min":1.397,"15min":1.1200000000000001},"min":{"1min":0.40100000000000002,"5min":0.33100000000000002,"15min":0.32300000000000001},"max":{"1min":8.923,"5min":12.715,"15min":12.715},"last":0.42499999999999999}]}]},{"osd":6,"up_from":0,"seq":704374636767,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":1,"up_from":0,"seq":1503238553699,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":0,"up_from":493,"seq":2117418876940,"num_pgs":75,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2832080,"kb_used_data":2695944,"kb_used_omap":36,"kb_used_meta":136091,"kb_avail":90913072,"statfs":{"total":95995035648,"available":93094985728,"internally_reserved":0,"allocated":2760646656,"data_stored":2758184128,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":37336,"internal_metadata":139357736},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,0,0,1,1,3,3,2],"upper_bound":2048},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.504}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.314}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.79000000000000004}]},{"osd":4,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":2.9169999999999998}]},{"osd":5,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.42799999999999999}]},{"osd":6,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.53600000000000003}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":2.8239999999999998}]}]},{"osd":2,"up_from":93,"seq":399431958812,"num_pgs":69,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2508816,"kb_used_data":1739144,"kb_used_omap":81,"kb_used_meta":769582,"kb_avail":91236336,"statfs":{"total":95995035648,"available":93426008064,"internally_reserved":0,"allocated":1780883456,"data_stored":1778419969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":83750,"internal_metadata":788052186},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:11:05 2023","interfaces":[{"interface":"back","average":{"1min":0.61499999999999999,"5min":1.9730000000000001,"15min":2.1869999999999998},"min":{"1min":0.20200000000000001,"5min":0.20200000000000001,"15min":0.17999999999999999},"max":{"1min":3.3610000000000002,"5min":11.124000000000001,"15min":29.498000000000001},"last":1.006},{"interface":"front","average":{"1min":0.49299999999999999,"5min":1.6299999999999999,"15min":2.3170000000000002},"min":{"1min":0.26300000000000001,"5min":0.193,"15min":0.193},"max":{"1min":1.377,"5min":12.183,"15min":17.251000000000001},"last":0.61699999999999999}]},{"osd":1,"last update":"Sat Sep 23 21:13:11 2023","interfaces":[{"interface":"back","average":{"1min":0.26400000000000001,"5min":1.7969999999999999,"15min":2.669},"min":{"1min":0.188,"5min":0.188,"15min":0.188},"max":{"1min":0.377,"5min":12.484999999999999,"15min":34.396999999999998},"last":0.52900000000000003},{"interface":"front","average":{"1min":0.30599999999999999,"5min":1.6819999999999999,"15min":2.9199999999999999},"min":{"1min":0.217,"5min":0.217,"15min":0.217},"max":{"1min":0.57099999999999995,"5min":11.289999999999999,"15min":34.462000000000003},"last":0.57699999999999996}]},{"osd":3,"last update":"Sat Sep 23 21:13:56 2023","interfaces":[{"interface":"back","average":{"1min":0.495,"5min":1.5429999999999999,"15min":2.6419999999999999},"min":{"1min":0.20599999999999999,"5min":0.20599999999999999,"15min":0.20599999999999999},"max":{"1min":1.679,"5min":12.951000000000001,"15min":41.057000000000002},"last":0.41899999999999998},{"interface":"front","average":{"1min":0.749,"5min":1.661,"15min":2.915},"min":{"1min":0.28499999999999998,"5min":0.21299999999999999,"15min":0.21299999999999999},"max":{"1min":3.6989999999999998,"5min":13.159000000000001,"15min":41.012},"last":0.48099999999999998}]},{"osd":4,"last update":"Sat Sep 23 21:13:11 2023","interfaces":[{"interface":"back","average":{"1min":1.5900000000000001,"5min":1.494,"15min":1.5289999999999999},"min":{"1min":0.30599999999999999,"5min":0.249,"15min":0.249},"max":{"1min":10.266999999999999,"5min":10.266999999999999,"15min":10.266999999999999},"last":0.46300000000000002},{"interface":"front","average":{"1min":1.476,"5min":1.573,"15min":1.952},"min":{"1min":0.29399999999999998,"5min":0.29399999999999998,"15min":0.26700000000000002},"max":{"1min":6.7720000000000002,"5min":8.6780000000000008,"15min":14.039999999999999},"last":0.437}]},{"osd":5,"last update":"Sat Sep 23 21:13:11 2023","interfaces":[{"interface":"back","average":{"1min":1.0660000000000001,"5min":1.46,"15min":1.603},"min":{"1min":0.34399999999999997,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":3.4390000000000001,"5min":9.5039999999999996,"15min":10.069000000000001},"last":0.498},{"interface":"front","average":{"1min":0.97299999999999998,"5min":1.546,"15min":1.8939999999999999},"min":{"1min":0.29699999999999999,"5min":0.29699999999999999,"15min":0.29399999999999998},"max":{"1min":7.4969999999999999,"5min":8.7270000000000003,"15min":13.98},"last":0.56200000000000006}]},{"osd":6,"last update":"Sat Sep 23 21:13:11 2023","interfaces":[{"interface":"back","average":{"1min":1.7989999999999999,"5min":1.8120000000000001,"15min":2.1320000000000001},"min":{"1min":0.373,"5min":0.373,"15min":0.35699999999999998},"max":{"1min":7.3310000000000004,"5min":8.7680000000000007,"15min":14.016},"last":0.625},{"interface":"front","average":{"1min":2.3940000000000001,"5min":2.0510000000000002,"15min":2.2959999999999998},"min":{"1min":0.40699999999999997,"5min":0.38100000000000001,"15min":0.35599999999999998},"max":{"1min":9.3469999999999995,"5min":9.3469999999999995,"15min":14.068},"last":0.59799999999999998}]},{"osd":7,"last update":"Sat Sep 23 21:13:11 2023","interfaces":[{"interface":"back","average":{"1min":1.609,"5min":1.6299999999999999,"15min":1.5620000000000001},"min":{"1min":0.42899999999999999,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":7.694,"5min":7.694,"15min":13.427},"last":0.79600000000000004},{"interface":"front","average":{"1min":2.0499999999999998,"5min":1.657,"15min":1.74},"min":{"1min":0.34799999999999998,"5min":0.34799999999999998,"15min":0.32200000000000001},"max":{"1min":10.359,"5min":10.359,"15min":10.92},"last":0.66900000000000004}]}]},{"osd":3,"up_from":98,"seq":420906795285,"num_pgs":82,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2464360,"kb_used_data":1617952,"kb_used_omap":7,"kb_used_meta":846392,"kb_avail":91280792,"statfs":{"total":95995035648,"available":93471531008,"internally_reserved":0,"allocated":1656782848,"data_stored":1654352807,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7576,"internal_metadata":866706024},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,2],"upper_bound":4},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:10:53 2023","interfaces":[{"interface":"back","average":{"1min":0.59699999999999998,"5min":2.173,"15min":2.6000000000000001},"min":{"1min":0.223,"5min":0.19900000000000001,"15min":0.188},"max":{"1min":3.032,"5min":14.08,"15min":27.962},"last":0.32300000000000001},{"interface":"front","average":{"1min":0.85699999999999998,"5min":1.8580000000000001,"15min":2.6779999999999999},"min":{"1min":0.26000000000000001,"5min":0.20999999999999999,"15min":0.19800000000000001},"max":{"1min":4.0090000000000003,"5min":7.8920000000000003,"15min":28.058},"last":0.69999999999999996}]},{"osd":1,"last update":"Sat Sep 23 21:13:11 2023","interfaces":[{"interface":"back","average":{"1min":0.40200000000000002,"5min":1.9059999999999999,"15min":2.2989999999999999},"min":{"1min":0.20000000000000001,"5min":0.20000000000000001,"15min":0.20000000000000001},"max":{"1min":2.27,"5min":22.577999999999999,"15min":22.577999999999999},"last":2.7629999999999999},{"interface":"front","average":{"1min":0.33100000000000002,"5min":1.9570000000000001,"15min":2.2909999999999999},"min":{"1min":0.216,"5min":0.216,"15min":0.216},"max":{"1min":1.268,"5min":21.617999999999999,"15min":21.617999999999999},"last":1.3009999999999999}]},{"osd":2,"last update":"Sat Sep 23 21:14:00 2023","interfaces":[{"interface":"back","average":{"1min":3.6669999999999998,"5min":1.9279999999999999,"15min":2.6640000000000001},"min":{"1min":0.221,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":56.317999999999998,"5min":56.317999999999998,"15min":56.317999999999998},"last":0.248},{"interface":"front","average":{"1min":3.8479999999999999,"5min":2.1499999999999999,"15min":2.702},"min":{"1min":0.29699999999999999,"5min":0.216,"15min":0.216},"max":{"1min":55.988,"5min":55.988,"15min":55.988},"last":2.7869999999999999}]},{"osd":4,"last update":"Sat Sep 23 21:14:00 2023","interfaces":[{"interface":"back","average":{"1min":0.71499999999999997,"5min":1.3029999999999999,"15min":1.8779999999999999},"min":{"1min":0.30299999999999999,"5min":0.30299999999999999,"15min":0.28399999999999997},"max":{"1min":5.0819999999999999,"5min":21.478999999999999,"15min":21.478999999999999},"last":2.0760000000000001},{"interface":"front","average":{"1min":1.3009999999999999,"5min":1.5760000000000001,"15min":2.0070000000000001},"min":{"1min":0.33100000000000002,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":4.1909999999999998,"5min":22.402000000000001,"15min":22.402000000000001},"last":3.4750000000000001}]},{"osd":5,"last update":"Sat Sep 23 21:14:00 2023","interfaces":[{"interface":"back","average":{"1min":0.93600000000000005,"5min":1.3120000000000001,"15min":2.012},"min":{"1min":0.32200000000000001,"5min":0.311,"15min":0.311},"max":{"1min":5.04,"5min":21.562000000000001,"15min":21.562000000000001},"last":0.375},{"interface":"front","average":{"1min":0.83699999999999997,"5min":1.339,"15min":2.2839999999999998},"min":{"1min":0.34300000000000003,"5min":0.31,"15min":0.31},"max":{"1min":3.4780000000000002,"5min":8.8450000000000006,"15min":14.029},"last":1.367}]},{"osd":6,"last update":"Sat Sep 23 21:13:14 2023","interfaces":[{"interface":"back","average":{"1min":1.48,"5min":2.0219999999999998,"15min":2.097},"min":{"1min":0.34999999999999998,"5min":0.34999999999999998,"15min":0.33500000000000002},"max":{"1min":10.102,"5min":22.446999999999999,"15min":22.446999999999999},"last":3.0510000000000002},{"interface":"front","average":{"1min":2.0790000000000002,"5min":1.831,"15min":2.294},"min":{"1min":0.39900000000000002,"5min":0.39000000000000001,"15min":0.373},"max":{"1min":10.148,"5min":12.375999999999999,"15min":13.576000000000001},"last":1.3340000000000001}]},{"osd":7,"last update":"Sat Sep 23 21:14:00 2023","interfaces":[{"interface":"back","average":{"1min":1.032,"5min":1.6120000000000001,"15min":2.1299999999999999},"min":{"1min":0.41199999999999998,"5min":0.38400000000000001,"15min":0.33900000000000002},"max":{"1min":5.0090000000000003,"5min":21.518999999999998,"15min":21.518999999999998},"last":1.7909999999999999},{"interface":"front","average":{"1min":0.92900000000000005,"5min":1.7050000000000001,"15min":2.4199999999999999},"min":{"1min":0.45800000000000002,"5min":0.442,"15min":0.38900000000000001},"max":{"1min":3.2679999999999998,"5min":9.9879999999999995,"15min":13.603999999999999},"last":1.8100000000000001}]}]},{"osd":4,"up_from":31,"seq":133143986591,"num_pgs":77,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2037604,"kb_used_data":1593884,"kb_used_omap":0,"kb_used_meta":443712,"kb_avail":91707548,"statfs":{"total":95995035648,"available":93908529152,"internally_reserved":0,"allocated":1632137216,"data_stored":1630619422,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":454361088},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,1,2,0,0,0,3],"upper_bound":256},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:11:11 2023","interfaces":[{"interface":"back","average":{"1min":0.98499999999999999,"5min":1.6819999999999999,"15min":1.5549999999999999},"min":{"1min":0.20200000000000001,"5min":0.20200000000000001,"15min":0.20200000000000001},"max":{"1min":4.944,"5min":11.537000000000001,"15min":11.784000000000001},"last":1.2589999999999999},{"interface":"front","average":{"1min":1.1499999999999999,"5min":2.2549999999999999,"15min":1.887},"min":{"1min":0.28799999999999998,"5min":0.28000000000000003,"15min":0.254},"max":{"1min":4.9749999999999996,"5min":19.109999999999999,"15min":19.109999999999999},"last":2.1179999999999999}]},{"osd":1,"last update":"Sat Sep 23 21:13:10 2023","interfaces":[{"interface":"back","average":{"1min":1.389,"5min":1.48,"15min":2.952},"min":{"1min":0.251,"5min":0.221,"15min":0.221},"max":{"1min":7.8609999999999998,"5min":14.798999999999999,"15min":37.960000000000001},"last":0.90500000000000003},{"interface":"front","average":{"1min":1.081,"5min":1.393,"15min":2.7949999999999999},"min":{"1min":0.32700000000000001,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":4.3639999999999999,"5min":14.403,"15min":37.933},"last":0.89400000000000002}]},{"osd":2,"last update":"Sat Sep 23 21:13:13 2023","interfaces":[{"interface":"back","average":{"1min":1.1399999999999999,"5min":1.5920000000000001,"15min":1.8180000000000001},"min":{"1min":0.26000000000000001,"5min":0.247,"15min":0.247},"max":{"1min":4.5019999999999998,"5min":16.928000000000001,"15min":16.928000000000001},"last":0.47899999999999998},{"interface":"front","average":{"1min":1.304,"5min":1.825,"15min":1.833},"min":{"1min":0.28599999999999998,"5min":0.28599999999999998,"15min":0.27800000000000002},"max":{"1min":4.5570000000000004,"5min":14.417999999999999,"15min":14.417999999999999},"last":0.33500000000000002}]},{"osd":3,"last update":"Sat Sep 23 21:13:46 2023","interfaces":[{"interface":"back","average":{"1min":1.0229999999999999,"5min":1.6299999999999999,"15min":2.1339999999999999},"min":{"1min":0.25600000000000001,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":4.2480000000000002,"5min":14.585000000000001,"15min":15.01},"last":0.60899999999999999},{"interface":"front","average":{"1min":1.135,"5min":1.647,"15min":2.0249999999999999},"min":{"1min":0.36699999999999999,"5min":0.36099999999999999,"15min":0.30199999999999999},"max":{"1min":4.4649999999999999,"5min":14.366,"15min":15.004},"last":0.61899999999999999}]},{"osd":5,"last update":"Sat Sep 23 21:13:02 2023","interfaces":[{"interface":"back","average":{"1min":2.1099999999999999,"5min":1.212,"15min":0.95199999999999996},"min":{"1min":0.222,"5min":0.222,"15min":0.222},"max":{"1min":9.4909999999999997,"5min":14.298,"15min":14.298},"last":0.30099999999999999},{"interface":"front","average":{"1min":1.9179999999999999,"5min":1.1599999999999999,"15min":1.0600000000000001},"min":{"1min":0.27200000000000002,"5min":0.26600000000000001,"15min":0.255},"max":{"1min":6.8730000000000002,"5min":9.8979999999999997,"15min":9.8979999999999997},"last":0.42099999999999999}]},{"osd":6,"last update":"Sat Sep 23 21:13:10 2023","interfaces":[{"interface":"back","average":{"1min":2.923,"5min":1.3859999999999999,"15min":1.2330000000000001},"min":{"1min":0.34499999999999997,"5min":0.28599999999999998,"15min":0.26800000000000002},"max":{"1min":24.452999999999999,"5min":24.452999999999999,"15min":24.452999999999999},"last":0.34399999999999997},{"interface":"front","average":{"1min":1.6859999999999999,"5min":1.171,"15min":0.94499999999999995},"min":{"1min":0.38800000000000001,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":11.409000000000001,"5min":14.387,"15min":14.387},"last":0.44700000000000001}]},{"osd":7,"last update":"Sat Sep 23 21:13:46 2023","interfaces":[{"interface":"back","average":{"1min":1.294,"5min":1.268,"15min":1.0529999999999999},"min":{"1min":0.375,"5min":0.30099999999999999,"15min":0.29299999999999998},"max":{"1min":4.532,"5min":16.890999999999998,"15min":16.890999999999998},"last":0.40999999999999998},{"interface":"front","average":{"1min":1.387,"5min":1.155,"15min":1.1559999999999999},"min":{"1min":0.375,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":7.1879999999999997,"5min":8.5830000000000002,"15min":11.170999999999999},"last":0.55500000000000005}]}]},{"osd":5,"up_from":39,"seq":167503724957,"num_pgs":63,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1832656,"kb_used_data":1345992,"kb_used_omap":0,"kb_used_meta":486656,"kb_avail":91912496,"statfs":{"total":95995035648,"available":94118395904,"internally_reserved":0,"allocated":1378295808,"data_stored":1376794204,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":498335744},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:11:02 2023","interfaces":[{"interface":"back","average":{"1min":0.627,"5min":1.722,"15min":1.8},"min":{"1min":0.222,"5min":0.19700000000000001,"15min":0.19700000000000001},"max":{"1min":2.0840000000000001,"5min":25.222999999999999,"15min":25.222999999999999},"last":0.26200000000000001},{"interface":"front","average":{"1min":0.67200000000000004,"5min":1.6659999999999999,"15min":1.861},"min":{"1min":0.247,"5min":0.247,"15min":0.23400000000000001},"max":{"1min":2.1970000000000001,"5min":25.457999999999998,"15min":25.457999999999998},"last":0.28499999999999998}]},{"osd":1,"last update":"Sat Sep 23 21:13:04 2023","interfaces":[{"interface":"back","average":{"1min":0.60099999999999998,"5min":1.4370000000000001,"15min":1.804},"min":{"1min":0.23499999999999999,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":2.0379999999999998,"5min":9.4429999999999996,"15min":25.239999999999998},"last":0.78700000000000003},{"interface":"front","average":{"1min":1.6040000000000001,"5min":1.883,"15min":1.79},"min":{"1min":0.29699999999999999,"5min":0.27400000000000002,"15min":0.252},"max":{"1min":3.956,"5min":10.686,"15min":25.253},"last":0.35599999999999998}]},{"osd":2,"last update":"Sat Sep 23 21:13:26 2023","interfaces":[{"interface":"back","average":{"1min":1.871,"5min":1.6679999999999999,"15min":1.7909999999999999},"min":{"1min":0.30299999999999999,"5min":0.26200000000000001,"15min":0.23499999999999999},"max":{"1min":6.085,"5min":11.939,"15min":12.634},"last":0.432},{"interface":"front","average":{"1min":0.71699999999999997,"5min":1.3979999999999999,"15min":1.915},"min":{"1min":0.27600000000000002,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":3.165,"5min":9.7170000000000005,"15min":12.300000000000001},"last":0.84099999999999997}]},{"osd":3,"last update":"Sat Sep 23 21:13:45 2023","interfaces":[{"interface":"back","average":{"1min":1.4279999999999999,"5min":1.8089999999999999,"15min":2.6560000000000001},"min":{"1min":0.33400000000000002,"5min":0.307,"15min":0.29299999999999998},"max":{"1min":6.1859999999999999,"5min":14.242000000000001,"15min":61.667999999999999},"last":1.169},{"interface":"front","average":{"1min":1.4390000000000001,"5min":2.0089999999999999,"15min":2.6429999999999998},"min":{"1min":0.34899999999999998,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":6.1200000000000001,"5min":14.396000000000001,"15min":62.276000000000003},"last":0.58599999999999997}]},{"osd":4,"last update":"Sat Sep 23 21:13:59 2023","interfaces":[{"interface":"back","average":{"1min":1.452,"5min":1.387,"15min":1.139},"min":{"1min":0.28799999999999998,"5min":0.27000000000000002,"15min":0.23400000000000001},"max":{"1min":11.156000000000001,"5min":12.776,"15min":12.776},"last":0.32600000000000001},{"interface":"front","average":{"1min":1.5109999999999999,"5min":1.1799999999999999,"15min":1.1319999999999999},"min":{"1min":0.28000000000000003,"5min":0.27000000000000002,"15min":0.26900000000000002},"max":{"1min":9.7119999999999997,"5min":9.7119999999999997,"15min":11.973000000000001},"last":0.39400000000000002}]},{"osd":6,"last update":"Sat Sep 23 21:13:04 2023","interfaces":[{"interface":"back","average":{"1min":1.8340000000000001,"5min":0.98799999999999999,"15min":1.0449999999999999},"min":{"1min":0.313,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":7.7030000000000003,"5min":7.7030000000000003,"15min":11.128},"last":0.72899999999999998},{"interface":"front","average":{"1min":3.004,"5min":1.4139999999999999,"15min":1.1830000000000001},"min":{"1min":0.51000000000000001,"5min":0.317,"15min":0.317},"max":{"1min":10.026999999999999,"5min":10.718999999999999,"15min":11.112},"last":0.51100000000000001}]},{"osd":7,"last update":"Sat Sep 23 21:13:45 2023","interfaces":[{"interface":"back","average":{"1min":1.3,"5min":0.95099999999999996,"15min":1.804},"min":{"1min":0.34300000000000003,"5min":0.31,"15min":0.309},"max":{"1min":5.5659999999999998,"5min":5.5659999999999998,"15min":189.405},"last":0.81399999999999995},{"interface":"front","average":{"1min":2.004,"5min":1.5009999999999999,"15min":1.9159999999999999},"min":{"1min":0.40200000000000002,"5min":0.35399999999999998,"15min":0.32500000000000001},"max":{"1min":11.183999999999999,"5min":11.183999999999999,"15min":189.685},"last":0.66100000000000003}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8488,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1259,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5322,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":714,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":5434,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":4838,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":2741043200,"data_stored":2741043200,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1434714112,"data_stored":1434714112,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1771503616,"data_stored":1771503616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1649217536,"data_stored":1649213464,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1613631488,"data_stored":1613627416,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1360068608,"data_stored":1360068608,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1152778240,"data_stored":1152778240,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T21:14:10.948 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph status --format=json 2023-09-23T21:14:10.993 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:14:11.028 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:14:11.099 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:14:11.104 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:14:11.104 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:14:11.205 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:14:11.236 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:14:11.289 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:14:11.293 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:14:11.293 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:14:11.394 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:14:11.427 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:14:11.472 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:14:11.475 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:14:11.476 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:14:11.536 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:14:11 smithi026 ceph-mon[179688]: Health check update: Degraded data redundancy: 25135/135597 objects degraded (18.537%), 19 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T21:14:11.537 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:14:11 smithi026 ceph-mon[175151]: Health check update: Degraded data redundancy: 25135/135597 objects degraded (18.537%), 19 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T21:14:11.576 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:14:11.615 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:14:11.624 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T21:14:11.666 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:14:11.673 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:14:11.673 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:14:11.774 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:14:11.811 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:14:11 smithi070 ceph-mon[168282]: Health check update: Degraded data redundancy: 25135/135597 objects degraded (18.537%), 19 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T21:14:11.813 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:14:12.384 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:14:12.492 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:14:12.493 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0: {} 2023-09-23T21:14:12.500 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:14:12.501 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:14:12.516 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1: {} 2023-09-23T21:14:12.538 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:14:12 smithi026 ceph-mon[179688]: pgmap v1317: 129 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 15 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 108 active+clean; 2.7 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 459 op/s; 25135/133914 objects degraded (18.770%); 4995/133914 objects misplaced (3.730%); 7.5 MiB/s, 177 objects/s recovering 2023-09-23T21:14:12.538 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:14:12 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/3710588311' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:14:12.538 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:14:12 smithi026 ceph-mon[175151]: pgmap v1317: 129 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 15 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 108 active+clean; 2.7 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 459 op/s; 25135/133914 objects degraded (18.770%); 4995/133914 objects misplaced (3.730%); 7.5 MiB/s, 177 objects/s recovering 2023-09-23T21:14:12.538 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:14:12 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/3710588311' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:14:12.541 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2: {} 2023-09-23T21:14:12.569 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3: {} 2023-09-23T21:14:12.599 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4: {} 2023-09-23T21:14:12.601 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:14:12.630 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T21:14:12.658 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6: {} 2023-09-23T21:14:12.665 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:14:12.686 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T21:14:12.689 INFO:teuthology.orchestra.run.smithi026.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T21:14:12.689 INFO:teuthology.orchestra.run.smithi026.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T21:14:12.690 INFO:teuthology.orchestra.run.smithi026.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T21:14:12.690 INFO:teuthology.orchestra.run.smithi026.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T21:14:12.690 INFO:teuthology.orchestra.run.smithi026.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T21:14:12.690 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T21:14:12.690 INFO:teuthology.orchestra.run.smithi026.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T21:14:12.690 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T21:14:12.716 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T21:14:12.721 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:14:12.721 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:14:12.822 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:14:12.831 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:14:12 smithi070 ceph-mon[168282]: pgmap v1317: 129 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 15 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 108 active+clean; 2.7 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 459 op/s; 25135/133914 objects degraded (18.770%); 4995/133914 objects misplaced (3.730%); 7.5 MiB/s, 177 objects/s recovering 2023-09-23T21:14:12.831 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:14:12 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/3710588311' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:14:12.864 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:14:13.073 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:14:13.076 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:14:13.077 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:14:13.178 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:14:13.253 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:14:13.306 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:14:13.314 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:14:13.315 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:14:13.416 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:14:13.459 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:14:13.553 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:14:13.557 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:14:13.558 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:14:13.659 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:14:13.690 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:14:13.735 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:14:13.740 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:14:13.740 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:14:13.838 INFO:teuthology.orchestra.run.smithi070.stdout:{ 2023-09-23T21:14:13.838 INFO:teuthology.orchestra.run.smithi070.stdout: "ops": [], 2023-09-23T21:14:13.838 INFO:teuthology.orchestra.run.smithi070.stdout: "num_ops": 0 2023-09-23T21:14:13.839 INFO:teuthology.orchestra.run.smithi070.stdout:} 2023-09-23T21:14:13.841 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:14:13.883 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:14:13.909 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T21:14:13.992 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:14:13.997 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:14:13.998 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:14:14.098 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:14:14.165 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:14:14.224 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:14:14.227 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:14:14.227 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:14:14.328 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:14:14.371 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:14:14.434 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T21:14:14.445 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:14:14.446 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:14:14.546 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:14:14.598 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:14:14.648 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:14:14.655 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:14:14.656 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:14:14.704 INFO:teuthology.orchestra.run.smithi026.stdout: 2023-09-23T21:14:14.705 INFO:teuthology.orchestra.run.smithi026.stdout:{"fsid":"b3f8e4ec-5a50-11ee-8db4-212e2dc638e7","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"noscrub,nodeep-scrub flag(s) set","count":20},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 25135/131577 objects degraded (19.103%), 19 pgs degraded, 1 pg undersized","count":20},"muted":false}},"mutes":[]},"election_epoch":30,"quorum":[0,1,2],"quorum_names":["a","b","c"],"quorum_age":1573,"monmap":{"epoch":4,"min_mon_release_name":"reef","num_mons":3},"osdmap":{"epoch":526,"num_osds":8,"num_up_osds":8,"osd_up_since":1695503591,"num_in_osds":6,"osd_in_since":1695503530,"num_remapped_pgs":18},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":108},{"state_name":"active+recovery_wait+undersized+degraded+remapped","count":15},{"state_name":"active+recovery_wait+degraded","count":3},{"state_name":"active+recovery_wait+degraded+remapped","count":1},{"state_name":"active+remapped+backfill_wait","count":1},{"state_name":"active+recovering+undersized+remapped","count":1}],"num_pgs":129,"num_pools":5,"num_objects":43859,"data_bytes":2873211652,"bytes_used":13042913280,"bytes_avail":562927300608,"bytes_total":575970213888,"degraded_objects":25135,"degraded_total":131577,"degraded_ratio":0.19102882722664297,"misplaced_objects":4925,"misplaced_total":131577,"misplaced_ratio":0.037430553972198785,"recovering_objects_per_sec":101,"recovering_bytes_per_sec":3431504,"recovering_keys_per_sec":0,"num_objects_recovered":944,"num_bytes_recovered":31850496,"num_keys_recovered":0,"write_bytes_sec":0,"read_op_per_sec":0,"write_op_per_sec":383},"fsmap":{"epoch":8,"id":1,"up":1,"in":1,"max":1,"by_rank":[{"filesystem_id":1,"rank":0,"name":"foo.smithi026.znfpnl","status":"up:active","gid":14658}],"up:standby":1},"mgrmap":{"available":true,"num_standbys":1,"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"services":{"dashboard":"https://172.21.15.26:8443/","prometheus":"http://172.21.15.26:9283/"}},"servicemap":{"epoch":28,"modified":"2023-09-23T21:14:11.047835+0000","services":{}},"progress_events":{"0ed42496-3a31-41bd-a432-1bbd8c010433":{"message":"Global Recovery Event (2m)\n [=======================.....] (remaining: 27s)","progress":0.83720928430557251,"add_to_ceph_s":true},"fc746e9d-3f76-434d-b6d2-8fc9e8a72a5e":{"message":"Upgrade to 18.2.0-460-gbb0daeff (4m)\n [============................] (remaining: 5m)","progress":0.44999998807907104,"add_to_ceph_s":true}}} 2023-09-23T21:14:14.707 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:14:14.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:14:14 smithi026 ceph-mon[175151]: pgmap v1318: 129 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 15 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 108 active+clean; 2.7 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 383 op/s; 25135/131577 objects degraded (19.103%); 4925/131577 objects misplaced (3.743%); 3.3 MiB/s, 101 objects/s recovering 2023-09-23T21:14:14.739 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:14:14 smithi026 ceph-mon[179688]: pgmap v1318: 129 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 15 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 108 active+clean; 2.7 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 383 op/s; 25135/131577 objects degraded (19.103%); 4925/131577 objects misplaced (3.743%); 3.3 MiB/s, 101 objects/s recovering 2023-09-23T21:14:14.749 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_blocked_ops 2023-09-23T21:14:14.756 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:14:14.797 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:14:14.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:14:14 smithi070 ceph-mon[168282]: pgmap v1318: 129 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 15 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 108 active+clean; 2.7 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 383 op/s; 25135/131577 objects degraded (19.103%); 4925/131577 objects misplaced (3.743%); 3.3 MiB/s, 101 objects/s recovering 2023-09-23T21:14:14.892 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:14:14.897 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:14:14.898 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:14:14.998 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:14:15.035 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:14:15.264 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T21:14:15.336 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T21:14:15.343 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:14:15.344 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:14:15.367 INFO:tasks.cephadm.ceph_manager.ceph:making progress, resetting timeout 2023-09-23T21:14:15.367 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T21:14:15.444 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:14:15.445 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T21:14:15.491 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:14:15.721 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:14:15.724 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:14:15.725 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:14:15.825 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:14:15.865 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:14:15 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/1544525634' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-23T21:14:15.872 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:14:15.881 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:14:15 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/1544525634' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-23T21:14:15.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:14:15 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/1544525634' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-23T21:14:16.025 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:14:16.029 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:14:16.030 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:14:16.130 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:14:16.175 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:14:16.283 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:14:16.287 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:14:16.287 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:14:16.371 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:14:16.377 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:14:16.388 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:14:16.468 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:14:16.542 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:14:16.546 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:14:16.546 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:14:16.647 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:14:16.684 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:14:16.769 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:14:16 smithi070 ceph-mon[168282]: pgmap v1319: 129 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 15 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 108 active+clean; 2.6 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 461 op/s; 24836/127395 objects degraded (19.495%); 4748/127395 objects misplaced (3.727%); 6.5 MiB/s, 149 objects/s recovering 2023-09-23T21:14:16.769 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:14:16 smithi070 ceph-mon[168282]: Health check update: Degraded data redundancy: 24836/127395 objects degraded (19.495%), 19 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-09-23T21:14:16.770 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:14:16 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:14:16.770 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:14:16 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:14:16.789 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:14:16 smithi026 ceph-mon[175151]: pgmap v1319: 129 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 15 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 108 active+clean; 2.6 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 461 op/s; 24836/127395 objects degraded (19.495%); 4748/127395 objects misplaced (3.727%); 6.5 MiB/s, 149 objects/s recovering 2023-09-23T21:14:16.789 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:14:16 smithi026 ceph-mon[175151]: Health check update: Degraded data redundancy: 24836/127395 objects degraded (19.495%), 19 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-09-23T21:14:16.789 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:14:16 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:14:16.789 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:14:16 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:14:16.789 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:14:16 smithi026 ceph-mon[179688]: pgmap v1319: 129 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 15 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 108 active+clean; 2.6 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 461 op/s; 24836/127395 objects degraded (19.495%); 4748/127395 objects misplaced (3.727%); 6.5 MiB/s, 149 objects/s recovering 2023-09-23T21:14:16.789 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:14:16 smithi026 ceph-mon[179688]: Health check update: Degraded data redundancy: 24836/127395 objects degraded (19.495%), 19 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-09-23T21:14:16.789 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:14:16 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:14:16.790 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:14:16 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:14:16.791 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:14:16.797 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:14:16.797 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:14:16.898 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:14:16.958 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:14:17.308 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:14:17.311 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:14:17.312 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:14:17.323 INFO:teuthology.orchestra.run.smithi070.stdout:{ 2023-09-23T21:14:17.323 INFO:teuthology.orchestra.run.smithi070.stdout: "ops": [], 2023-09-23T21:14:17.323 INFO:teuthology.orchestra.run.smithi070.stdout: "complaint_time": 30, 2023-09-23T21:14:17.324 INFO:teuthology.orchestra.run.smithi070.stdout: "num_blocked_ops": 0 2023-09-23T21:14:17.324 INFO:teuthology.orchestra.run.smithi070.stdout:} 2023-09-23T21:14:17.412 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:14:17.454 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:14:17.569 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:14:17 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/2033090106' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T21:14:17.575 INFO:teuthology.orchestra.run.smithi026.stderr:noscrub is unset 2023-09-23T21:14:17.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:14:17 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/2033090106' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T21:14:17.745 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_historic_ops 2023-09-23T21:14:17.881 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:14:17 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/2033090106' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T21:14:18.447 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T21:14:18.756 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:14:18.760 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:14:18 smithi026 ceph-mon[175151]: pgmap v1320: 129 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 15 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 108 active+clean; 2.5 GiB data, 12 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 467 op/s; 24836/124629 objects degraded (19.928%); 4688/124629 objects misplaced (3.762%); 6.5 MiB/s, 103 objects/s recovering 2023-09-23T21:14:18.760 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:14:18 smithi026 ceph-mon[175151]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:14:18.760 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:14:18 smithi026 ceph-mon[175151]: failed to encode map e527 with expected crc 2023-09-23T21:14:18.760 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:14:18 smithi026 ceph-mon[175151]: failed to encode map e527 with expected crc 2023-09-23T21:14:18.760 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:14:18 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/2033090106' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T21:14:18.760 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:14:18 smithi026 ceph-mon[175151]: osdmap e527: 8 total, 8 up, 6 in 2023-09-23T21:14:18.760 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:14:18 smithi026 ceph-mon[175151]: failed to encode map e527 with expected crc 2023-09-23T21:14:18.761 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:14:18 smithi026 ceph-mon[179688]: pgmap v1320: 129 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 15 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 108 active+clean; 2.5 GiB data, 12 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 467 op/s; 24836/124629 objects degraded (19.928%); 4688/124629 objects misplaced (3.762%); 6.5 MiB/s, 103 objects/s recovering 2023-09-23T21:14:18.761 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:14:18 smithi026 ceph-mon[179688]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:14:18.761 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:14:18 smithi026 ceph-mon[179688]: failed to encode map e527 with expected crc 2023-09-23T21:14:18.761 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:14:18 smithi026 ceph-mon[179688]: failed to encode map e527 with expected crc 2023-09-23T21:14:18.761 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:14:18 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/2033090106' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T21:14:18.761 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:14:18 smithi026 ceph-mon[179688]: osdmap e527: 8 total, 8 up, 6 in 2023-09-23T21:14:18.761 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:14:18 smithi026 ceph-mon[179688]: failed to encode map e527 with expected crc 2023-09-23T21:14:18.761 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:14:18.762 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:14:18.863 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:14:18.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:14:18 smithi070 ceph-mon[168282]: pgmap v1320: 129 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 15 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 108 active+clean; 2.5 GiB data, 12 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 467 op/s; 24836/124629 objects degraded (19.928%); 4688/124629 objects misplaced (3.762%); 6.5 MiB/s, 103 objects/s recovering 2023-09-23T21:14:18.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:14:18 smithi070 ceph-mon[168282]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:14:18.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:14:18 smithi070 ceph-mon[168282]: failed to encode map e527 with expected crc 2023-09-23T21:14:18.871 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:14:18 smithi070 ceph-mon[168282]: failed to encode map e527 with expected crc 2023-09-23T21:14:18.871 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:14:18 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/2033090106' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T21:14:18.871 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:14:18 smithi070 ceph-mon[168282]: osdmap e527: 8 total, 8 up, 6 in 2023-09-23T21:14:18.871 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:14:18 smithi070 ceph-mon[168282]: failed to encode map e527 with expected crc 2023-09-23T21:14:18.927 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:14:19.106 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:14:19.109 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:14:19.109 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:14:19.210 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:14:19.258 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:14:19.882 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 21:14:19 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[164029]: ::ffff:172.21.15.70 - - [23/Sep/2023:21:14:19] "GET /metrics HTTP/1.1" 200 36406 "" "Prometheus/2.43.0" 2023-09-23T21:14:19.893 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:14:19.905 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:14:19.905 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:14:19.937 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0: {} 2023-09-23T21:14:19.969 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1: {} 2023-09-23T21:14:20.005 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2: {} 2023-09-23T21:14:20.007 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:14:20.039 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3: {} 2023-09-23T21:14:20.066 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:14:20.076 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4: {} 2023-09-23T21:14:20.106 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T21:14:20.137 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:14:20.143 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:14:20.144 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:14:20.146 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6: {} 2023-09-23T21:14:20.180 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T21:14:20.189 INFO:teuthology.orchestra.run.smithi026.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T21:14:20.190 INFO:teuthology.orchestra.run.smithi026.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T21:14:20.190 INFO:teuthology.orchestra.run.smithi026.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T21:14:20.190 INFO:teuthology.orchestra.run.smithi026.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T21:14:20.190 INFO:teuthology.orchestra.run.smithi026.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T21:14:20.190 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T21:14:20.190 INFO:teuthology.orchestra.run.smithi026.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T21:14:20.190 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T21:14:20.245 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:14:20.288 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:14:20.329 INFO:teuthology.orchestra.run.smithi026.stdout: 2023-09-23T21:14:20.345 INFO:teuthology.orchestra.run.smithi026.stderr:dumped all 2023-09-23T21:14:20.345 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:14:20.353 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:14:20.354 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:14:20.455 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:14:20.488 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:14:20.520 INFO:teuthology.orchestra.run.smithi070.stdout:ERROR: (22) Invalid argument 2023-09-23T21:14:20.520 INFO:teuthology.orchestra.run.smithi070.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T21:14:20.544 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 21:14:20 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a[153130]: logger=cleanup t=2023-09-23T21:14:20.219438372Z level=info msg="Completed cleanup jobs" duration=14.165745ms 2023-09-23T21:14:20.631 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:14:20 smithi026 ceph-mon[179688]: pgmap v1322: 129 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 15 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 108 active+clean; 2.5 GiB data, 12 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 474 op/s; 24525/121032 objects degraded (20.263%); 4517/121032 objects misplaced (3.732%); 7.3 MiB/s, 117 objects/s recovering 2023-09-23T21:14:20.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:14:20 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/772397482' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:14:20.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:14:20 smithi026 ceph-mon[175151]: pgmap v1322: 129 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 15 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 108 active+clean; 2.5 GiB data, 12 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 474 op/s; 24525/121032 objects degraded (20.263%); 4517/121032 objects misplaced (3.732%); 7.3 MiB/s, 117 objects/s recovering 2023-09-23T21:14:20.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:14:20 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/772397482' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:14:20.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:14:20 smithi070 ceph-mon[168282]: pgmap v1322: 129 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 15 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 108 active+clean; 2.5 GiB data, 12 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 474 op/s; 24525/121032 objects degraded (20.263%); 4517/121032 objects misplaced (3.732%); 7.3 MiB/s, 117 objects/s recovering 2023-09-23T21:14:20.871 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:14:20 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/772397482' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:14:20.878 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:14:20.885 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:14:20.885 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:14:20.922 INFO:teuthology.orchestra.run.smithi026.stdout:{"pg_ready":true,"pg_map":{"version":1322,"stamp":"2023-09-23T21:14:19.051901+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":2642852612,"num_objects":40344,"num_object_clones":0,"num_object_copies":121032,"num_objects_missing_on_primary":2882,"num_objects_missing":2882,"num_objects_degraded":24525,"num_objects_misplaced":4517,"num_objects_unfound":0,"num_objects_dirty":40344,"num_whiteouts":0,"num_read":90,"num_read_kb":166,"num_write":107160,"num_write_kb":4309782,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":37650,"num_bytes_recovered":2254539941,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":133915,"ondisk_log_size":133915,"up":387,"acting":371,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":433,"num_osds":6,"num_per_pool_osds":6,"num_per_pool_omap_osds":6,"kb":562470912,"kb_used":12297192,"kb_used_data":9129336,"kb_used_omap":125,"kb_used_meta":3167682,"kb_avail":550173720,"statfs":{"total":575970213888,"available":563377889280,"internally_reserved":0,"allocated":9348440064,"data_stored":9336509945,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":128662,"internal_metadata":3243706730},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,2,5,1,5,0,3,5],"upper_bound":512},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-295763968,"num_objects":-4513,"num_object_clones":0,"num_object_copies":-13539,"num_objects_missing_on_primary":-462,"num_objects_missing":-462,"num_objects_degraded":-610,"num_objects_misplaced":-478,"num_objects_unfound":0,"num_objects_dirty":-4513,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4513,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1116,"num_bytes_recovered":73138176,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"9.521077"},"pg_stats":[{"pgid":"1.18","version":"79'161","reported_seq":646,"reported_epoch":525,"state":"active+clean","last_fresh":"2023-09-23T21:14:03.771594+0000","last_change":"2023-09-23T21:12:30.438733+0000","last_active":"2023-09-23T21:14:03.771594+0000","last_peered":"2023-09-23T21:14:03.771594+0000","last_clean":"2023-09-23T21:14:03.771594+0000","last_became_active":"2023-09-23T21:12:30.438356+0000","last_became_peered":"2023-09-23T21:12:30.438356+0000","last_unstale":"2023-09-23T21:14:03.771594+0000","last_undegraded":"2023-09-23T21:14:03.771594+0000","last_fullsized":"2023-09-23T21:14:03.771594+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":479,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:06:52.098882+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:06:52.098882+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0092173299999999993,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":131344,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"10.13","version":"527'5116","reported_seq":7958,"reported_epoch":527,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T21:14:18.227743+0000","last_change":"2023-09-23T21:13:49.840403+0000","last_active":"2023-09-23T21:14:18.227743+0000","last_peered":"2023-09-23T21:14:18.227743+0000","last_clean":"2023-09-23T21:11:26.718194+0000","last_became_active":"2023-09-23T21:13:48.181112+0000","last_became_peered":"2023-09-23T21:13:48.181112+0000","last_unstale":"2023-09-23T21:14:18.227743+0000","last_undegraded":"2023-09-23T21:13:48.139049+0000","last_fullsized":"2023-09-23T21:13:45.405990+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":438,"last_epoch_clean":439,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_clean_scrub_stamp":"2023-09-23T21:10:15.663179+0000","objects_scrubbed":0,"log_size":5116,"log_dups_size":0,"ondisk_log_size":5116,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":84246528,"num_objects":1286,"num_object_clones":0,"num_object_copies":3858,"num_objects_missing_on_primary":914,"num_objects_missing":914,"num_objects_degraded":1114,"num_objects_misplaced":253,"num_objects_unfound":0,"num_objects_dirty":1286,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3748,"num_write_kb":161056,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1231,"num_bytes_recovered":80609280,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[4,3],"avail_no_missing":["5"],"object_location_counts":[{"shards":"0,2,4,5","objects":307},{"shards":"2,3,4,5","objects":3},{"shards":"2,3,5","objects":914},{"shards":"2,4,5","objects":28},{"shards":"3,4","objects":34}],"blocked_by":[],"up_primary":2,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1a","version":"68'1","reported_seq":117,"reported_epoch":526,"state":"active+clean","last_fresh":"2023-09-23T21:14:09.541544+0000","last_change":"2023-09-23T21:13:13.291403+0000","last_active":"2023-09-23T21:14:09.541544+0000","last_peered":"2023-09-23T21:14:09.541544+0000","last_clean":"2023-09-23T21:14:09.541544+0000","last_became_active":"2023-09-23T21:13:13.291123+0000","last_became_peered":"2023-09-23T21:13:13.291123+0000","last_unstale":"2023-09-23T21:14:09.541544+0000","last_undegraded":"2023-09-23T21:14:09.541544+0000","last_fullsized":"2023-09-23T21:14:09.541544+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":495,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:45:15.427430+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,0],"acting":[4,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":186,"reported_epoch":526,"state":"active+clean","last_fresh":"2023-09-23T21:14:09.540663+0000","last_change":"2023-09-23T21:12:10.767963+0000","last_active":"2023-09-23T21:14:09.540663+0000","last_peered":"2023-09-23T21:14:09.540663+0000","last_clean":"2023-09-23T21:14:09.540663+0000","last_became_active":"2023-09-23T21:12:10.767126+0000","last_became_peered":"2023-09-23T21:12:10.767126+0000","last_unstale":"2023-09-23T21:14:09.540663+0000","last_undegraded":"2023-09-23T21:14:09.540663+0000","last_fullsized":"2023-09-23T21:14:09.540663+0000","mapping_epoch":470,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:45:47.138277+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,5],"acting":[4,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1b","version":"79'161","reported_seq":529,"reported_epoch":525,"state":"active+clean","last_fresh":"2023-09-23T21:14:03.770913+0000","last_change":"2023-09-23T21:13:13.373548+0000","last_active":"2023-09-23T21:14:03.770913+0000","last_peered":"2023-09-23T21:14:03.770913+0000","last_clean":"2023-09-23T21:14:03.770913+0000","last_became_active":"2023-09-23T21:13:13.372845+0000","last_became_peered":"2023-09-23T21:13:13.372845+0000","last_unstale":"2023-09-23T21:14:03.770913+0000","last_undegraded":"2023-09-23T21:14:03.770913+0000","last_fullsized":"2023-09-23T21:14:03.770913+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":479,"last_epoch_clean":495,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:05:55.772264+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:05:55.772264+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.001862507,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.10","version":"527'5089","reported_seq":12009,"reported_epoch":527,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T21:14:18.096332+0000","last_change":"2023-09-23T21:13:36.071975+0000","last_active":"2023-09-23T21:14:18.096332+0000","last_peered":"2023-09-23T21:14:18.096332+0000","last_clean":"2023-09-23T21:11:26.716051+0000","last_became_active":"2023-09-23T21:13:33.619010+0000","last_became_peered":"2023-09-23T21:13:33.619010+0000","last_unstale":"2023-09-23T21:14:18.096332+0000","last_undegraded":"2023-09-23T21:13:33.443511+0000","last_fullsized":"2023-09-23T21:13:31.753181+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":438,"last_epoch_clean":450,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_clean_scrub_stamp":"2023-09-23T21:10:15.663179+0000","objects_scrubbed":0,"log_size":5089,"log_dups_size":0,"ondisk_log_size":5089,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":81428480,"num_objects":1243,"num_object_clones":0,"num_object_copies":3729,"num_objects_missing_on_primary":905,"num_objects_missing":905,"num_objects_degraded":1731,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1243,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3725,"num_write_kb":158944,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3118,"num_bytes_recovered":204341248,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7],"avail_no_missing":["1","5","7"],"object_location_counts":[{"shards":"0,5,7","objects":905},{"shards":"4,5,7","objects":29},{"shards":"4,7","objects":309}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":521,"reported_epoch":525,"state":"active+clean","last_fresh":"2023-09-23T21:14:03.785382+0000","last_change":"2023-09-23T21:12:10.824116+0000","last_active":"2023-09-23T21:14:03.785382+0000","last_peered":"2023-09-23T21:14:03.785382+0000","last_clean":"2023-09-23T21:14:03.785382+0000","last_became_active":"2023-09-23T21:12:10.823849+0000","last_became_peered":"2023-09-23T21:12:10.823849+0000","last_unstale":"2023-09-23T21:14:03.785382+0000","last_undegraded":"2023-09-23T21:14:03.785382+0000","last_fullsized":"2023-09-23T21:14:03.785382+0000","mapping_epoch":470,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:54:13.777603+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":522,"reported_epoch":525,"state":"active+clean","last_fresh":"2023-09-23T21:14:03.770644+0000","last_change":"2023-09-23T21:13:44.413713+0000","last_active":"2023-09-23T21:14:03.770644+0000","last_peered":"2023-09-23T21:14:03.770644+0000","last_clean":"2023-09-23T21:14:03.770644+0000","last_became_active":"2023-09-23T21:13:44.413366+0000","last_became_peered":"2023-09-23T21:13:44.413366+0000","last_unstale":"2023-09-23T21:14:03.770644+0000","last_undegraded":"2023-09-23T21:14:03.770644+0000","last_fullsized":"2023-09-23T21:14:03.770644+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:31:56.349530+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1a","version":"79'161","reported_seq":671,"reported_epoch":525,"state":"active+clean","last_fresh":"2023-09-23T21:14:03.772730+0000","last_change":"2023-09-23T21:12:30.444665+0000","last_active":"2023-09-23T21:14:03.772730+0000","last_peered":"2023-09-23T21:14:03.772730+0000","last_clean":"2023-09-23T21:14:03.772730+0000","last_became_active":"2023-09-23T21:12:30.444326+0000","last_became_peered":"2023-09-23T21:12:30.444326+0000","last_unstale":"2023-09-23T21:14:03.772730+0000","last_undegraded":"2023-09-23T21:14:03.772730+0000","last_fullsized":"2023-09-23T21:14:03.772730+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":479,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:06:51.075448+0000","last_deep_scrub":"79'161","last_deep_scrub_stamp":"2023-09-23T21:05:32.630292+0000","last_clean_scrub_stamp":"2023-09-23T21:06:51.075448+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0054793890000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,4],"acting":[2,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"10.11","version":"526'5013","reported_seq":11841,"reported_epoch":526,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-23T21:14:15.965759+0000","last_change":"2023-09-23T21:13:31.864763+0000","last_active":"2023-09-23T21:14:15.965759+0000","last_peered":"2023-09-23T21:14:15.965759+0000","last_clean":"2023-09-23T21:13:30.058478+0000","last_became_active":"2023-09-23T21:13:31.850860+0000","last_became_peered":"2023-09-23T21:13:31.850860+0000","last_unstale":"2023-09-23T21:14:15.965759+0000","last_undegraded":"2023-09-23T21:14:15.965759+0000","last_fullsized":"2023-09-23T21:14:15.965759+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":438,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":5,"last_scrub":"441'2974","last_scrub_stamp":"2023-09-23T21:11:06.133218+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_clean_scrub_stamp":"2023-09-23T21:11:06.133218+0000","objects_scrubbed":1646,"log_size":5013,"log_dups_size":0,"ondisk_log_size":5013,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:54:59.366711+0000","scrub_duration":0.3173955,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":82444288,"num_objects":1258,"num_object_clones":0,"num_object_copies":3774,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1258,"num_objects_unfound":0,"num_objects_dirty":1258,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2042,"num_write_kb":52928,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2152,"num_bytes_recovered":141033472,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[2,5,7],"avail_no_missing":["2","1","5","7"],"object_location_counts":[{"shards":"2,5,7","objects":1258}],"blocked_by":[],"up_primary":0,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":35,"reported_epoch":525,"state":"active+clean","last_fresh":"2023-09-23T21:14:03.768800+0000","last_change":"2023-09-23T21:13:30.806084+0000","last_active":"2023-09-23T21:14:03.768800+0000","last_peered":"2023-09-23T21:14:03.768800+0000","last_clean":"2023-09-23T21:14:03.768800+0000","last_became_active":"2023-09-23T21:13:30.805743+0000","last_became_peered":"2023-09-23T21:13:30.805743+0000","last_unstale":"2023-09-23T21:14:03.768800+0000","last_undegraded":"2023-09-23T21:14:03.768800+0000","last_fullsized":"2023-09-23T21:14:03.768800+0000","mapping_epoch":501,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:48:10.849281+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":399,"reported_epoch":525,"state":"active+clean","last_fresh":"2023-09-23T21:14:03.769720+0000","last_change":"2023-09-23T21:13:44.410921+0000","last_active":"2023-09-23T21:14:03.769720+0000","last_peered":"2023-09-23T21:14:03.769720+0000","last_clean":"2023-09-23T21:14:03.769720+0000","last_became_active":"2023-09-23T21:13:44.410633+0000","last_became_peered":"2023-09-23T21:13:44.410633+0000","last_unstale":"2023-09-23T21:14:03.769720+0000","last_undegraded":"2023-09-23T21:14:03.769720+0000","last_fullsized":"2023-09-23T21:14:03.769720+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:57:10.116786+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.15","version":"79'161","reported_seq":553,"reported_epoch":526,"state":"active+clean","last_fresh":"2023-09-23T21:14:09.539729+0000","last_change":"2023-09-23T21:13:38.379453+0000","last_active":"2023-09-23T21:14:09.539729+0000","last_peered":"2023-09-23T21:14:09.539729+0000","last_clean":"2023-09-23T21:14:09.539729+0000","last_became_active":"2023-09-23T21:13:38.376499+0000","last_became_peered":"2023-09-23T21:13:38.376499+0000","last_unstale":"2023-09-23T21:14:09.539729+0000","last_undegraded":"2023-09-23T21:14:09.539729+0000","last_fullsized":"2023-09-23T21:14:09.539729+0000","mapping_epoch":507,"log_start":"0'0","ondisk_log_start":"0'0","created":479,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:09:34.892947+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:09:34.892947+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13418,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":11,"num_read_kb":21,"num_write":6,"num_write_kb":55,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":40253,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":500,"reported_epoch":525,"state":"active+clean","last_fresh":"2023-09-23T21:14:03.769873+0000","last_change":"2023-09-23T21:13:12.235728+0000","last_active":"2023-09-23T21:14:03.769873+0000","last_peered":"2023-09-23T21:14:03.769873+0000","last_clean":"2023-09-23T21:14:03.769873+0000","last_became_active":"2023-09-23T21:13:12.216549+0000","last_became_peered":"2023-09-23T21:13:12.216549+0000","last_unstale":"2023-09-23T21:14:03.769873+0000","last_undegraded":"2023-09-23T21:14:03.769873+0000","last_fullsized":"2023-09-23T21:14:03.769873+0000","mapping_epoch":493,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:39:33.324239+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":563,"reported_epoch":525,"state":"active+clean","last_fresh":"2023-09-23T21:14:03.784811+0000","last_change":"2023-09-23T21:12:10.790349+0000","last_active":"2023-09-23T21:14:03.784811+0000","last_peered":"2023-09-23T21:14:03.784811+0000","last_clean":"2023-09-23T21:14:03.784811+0000","last_became_active":"2023-09-23T21:12:10.789977+0000","last_became_peered":"2023-09-23T21:12:10.789977+0000","last_unstale":"2023-09-23T21:14:03.784811+0000","last_undegraded":"2023-09-23T21:14:03.784811+0000","last_fullsized":"2023-09-23T21:14:03.784811+0000","mapping_epoch":470,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:49:40.191147+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,5],"acting":[3,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"79'161","reported_seq":887,"reported_epoch":525,"state":"active+clean","last_fresh":"2023-09-23T21:14:03.772239+0000","last_change":"2023-09-23T21:11:32.760292+0000","last_active":"2023-09-23T21:14:03.772239+0000","last_peered":"2023-09-23T21:14:03.772239+0000","last_clean":"2023-09-23T21:14:03.772239+0000","last_became_active":"2023-09-23T21:11:32.759909+0000","last_became_peered":"2023-09-23T21:11:32.759909+0000","last_unstale":"2023-09-23T21:14:03.772239+0000","last_undegraded":"2023-09-23T21:14:03.772239+0000","last_fullsized":"2023-09-23T21:14:03.772239+0000","mapping_epoch":457,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":458,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:25.096401+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:25.096401+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:13:09.225685+0000","scrub_duration":0.002859393,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":250,"reported_epoch":525,"state":"active+clean","last_fresh":"2023-09-23T21:14:03.799534+0000","last_change":"2023-09-23T21:12:10.779468+0000","last_active":"2023-09-23T21:14:03.799534+0000","last_peered":"2023-09-23T21:14:03.799534+0000","last_clean":"2023-09-23T21:14:03.799534+0000","last_became_active":"2023-09-23T21:12:10.779282+0000","last_became_peered":"2023-09-23T21:12:10.779282+0000","last_unstale":"2023-09-23T21:14:03.799534+0000","last_undegraded":"2023-09-23T21:14:03.799534+0000","last_fullsized":"2023-09-23T21:14:03.799534+0000","mapping_epoch":470,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:36:35.417986+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":500,"reported_epoch":525,"state":"active+clean","last_fresh":"2023-09-23T21:14:03.785571+0000","last_change":"2023-09-23T21:10:26.758958+0000","last_active":"2023-09-23T21:14:03.785571+0000","last_peered":"2023-09-23T21:14:03.785571+0000","last_clean":"2023-09-23T21:14:03.785571+0000","last_became_active":"2023-09-23T21:10:26.758539+0000","last_became_peered":"2023-09-23T21:10:26.758539+0000","last_unstale":"2023-09-23T21:14:03.785571+0000","last_undegraded":"2023-09-23T21:14:03.785571+0000","last_fullsized":"2023-09-23T21:14:03.785571+0000","mapping_epoch":423,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":424,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:44:42.319189+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.17","version":"79'161","reported_seq":618,"reported_epoch":525,"state":"active+clean","last_fresh":"2023-09-23T21:14:03.786605+0000","last_change":"2023-09-23T21:13:44.416768+0000","last_active":"2023-09-23T21:14:03.786605+0000","last_peered":"2023-09-23T21:14:03.786605+0000","last_clean":"2023-09-23T21:14:03.786605+0000","last_became_active":"2023-09-23T21:13:44.416156+0000","last_became_peered":"2023-09-23T21:13:44.416156+0000","last_unstale":"2023-09-23T21:14:03.786605+0000","last_undegraded":"2023-09-23T21:14:03.786605+0000","last_fullsized":"2023-09-23T21:14:03.786605+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":479,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:25.916867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:25.916867+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.001665323,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":410,"reported_epoch":525,"state":"active+clean","last_fresh":"2023-09-23T21:14:03.787621+0000","last_change":"2023-09-23T21:11:08.766045+0000","last_active":"2023-09-23T21:14:03.787621+0000","last_peered":"2023-09-23T21:14:03.787621+0000","last_clean":"2023-09-23T21:14:03.787621+0000","last_became_active":"2023-09-23T21:11:08.758504+0000","last_became_peered":"2023-09-23T21:11:08.758504+0000","last_unstale":"2023-09-23T21:14:03.787621+0000","last_undegraded":"2023-09-23T21:14:03.787621+0000","last_fullsized":"2023-09-23T21:14:03.787621+0000","mapping_epoch":443,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":444,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:02:19.482574+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":360,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,4],"acting":[3,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":212,"reported_epoch":525,"state":"active+clean","last_fresh":"2023-09-23T21:14:03.773316+0000","last_change":"2023-09-23T21:13:30.821489+0000","last_active":"2023-09-23T21:14:03.773316+0000","last_peered":"2023-09-23T21:14:03.773316+0000","last_clean":"2023-09-23T21:14:03.773316+0000","last_became_active":"2023-09-23T21:13:30.821125+0000","last_became_peered":"2023-09-23T21:13:30.821125+0000","last_unstale":"2023-09-23T21:14:03.773316+0000","last_undegraded":"2023-09-23T21:14:03.773316+0000","last_fullsized":"2023-09-23T21:14:03.773316+0000","mapping_epoch":501,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:47:18.553417+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,5],"acting":[2,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.16","version":"79'161","reported_seq":444,"reported_epoch":525,"state":"active+clean","last_fresh":"2023-09-23T21:14:03.770564+0000","last_change":"2023-09-23T21:13:33.770962+0000","last_active":"2023-09-23T21:14:03.770564+0000","last_peered":"2023-09-23T21:14:03.770564+0000","last_clean":"2023-09-23T21:14:03.770564+0000","last_became_active":"2023-09-23T21:13:33.770407+0000","last_became_peered":"2023-09-23T21:13:33.770407+0000","last_unstale":"2023-09-23T21:14:03.770564+0000","last_undegraded":"2023-09-23T21:14:03.770564+0000","last_fullsized":"2023-09-23T21:14:03.770564+0000","mapping_epoch":504,"log_start":"0'0","ondisk_log_start":"0'0","created":479,"last_epoch_clean":505,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:10:24.268662+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:10:24.268662+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13418,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":11,"num_read_kb":21,"num_write":6,"num_write_kb":55,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":40252,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.14","version":"79'22","reported_seq":90,"reported_epoch":526,"state":"active+clean","last_fresh":"2023-09-23T21:14:09.539838+0000","last_change":"2023-09-23T21:11:27.770162+0000","last_active":"2023-09-23T21:14:09.539838+0000","last_peered":"2023-09-23T21:14:09.539838+0000","last_clean":"2023-09-23T21:14:09.539838+0000","last_became_active":"2023-09-23T21:11:27.752275+0000","last_became_peered":"2023-09-23T21:11:27.752275+0000","last_unstale":"2023-09-23T21:14:09.539838+0000","last_undegraded":"2023-09-23T21:14:09.539838+0000","last_fullsized":"2023-09-23T21:14:09.539838+0000","mapping_epoch":452,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":453,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:05:11.769428+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4734,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,2],"acting":[4,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":455,"reported_epoch":526,"state":"active+clean","last_fresh":"2023-09-23T21:14:09.555427+0000","last_change":"2023-09-23T21:12:10.839271+0000","last_active":"2023-09-23T21:14:09.555427+0000","last_peered":"2023-09-23T21:14:09.555427+0000","last_clean":"2023-09-23T21:14:09.555427+0000","last_became_active":"2023-09-23T21:12:10.839042+0000","last_became_peered":"2023-09-23T21:12:10.839042+0000","last_unstale":"2023-09-23T21:14:09.555427+0000","last_undegraded":"2023-09-23T21:14:09.555427+0000","last_fullsized":"2023-09-23T21:14:09.555427+0000","mapping_epoch":470,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:10:26.414737+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,2],"acting":[4,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.11","version":"79'161","reported_seq":449,"reported_epoch":525,"state":"active+clean","last_fresh":"2023-09-23T21:14:03.768725+0000","last_change":"2023-09-23T21:13:33.767406+0000","last_active":"2023-09-23T21:14:03.768725+0000","last_peered":"2023-09-23T21:14:03.768725+0000","last_clean":"2023-09-23T21:14:03.768725+0000","last_became_active":"2023-09-23T21:13:33.767067+0000","last_became_peered":"2023-09-23T21:13:33.767067+0000","last_unstale":"2023-09-23T21:14:03.768725+0000","last_undegraded":"2023-09-23T21:14:03.768725+0000","last_fullsized":"2023-09-23T21:14:03.768725+0000","mapping_epoch":504,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":505,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:28.756634+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:28.756634+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":23,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":134,"reported_epoch":525,"state":"active+clean","last_fresh":"2023-09-23T21:14:03.791008+0000","last_change":"2023-09-23T21:11:27.777154+0000","last_active":"2023-09-23T21:14:03.791008+0000","last_peered":"2023-09-23T21:14:03.791008+0000","last_clean":"2023-09-23T21:14:03.791008+0000","last_became_active":"2023-09-23T21:11:27.754438+0000","last_became_peered":"2023-09-23T21:11:27.754438+0000","last_unstale":"2023-09-23T21:14:03.791008+0000","last_undegraded":"2023-09-23T21:14:03.791008+0000","last_fullsized":"2023-09-23T21:14:03.791008+0000","mapping_epoch":452,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":453,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:33:02.845882+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":66,"reported_epoch":525,"state":"active+clean","last_fresh":"2023-09-23T21:14:03.771924+0000","last_change":"2023-09-23T21:12:10.764264+0000","last_active":"2023-09-23T21:14:03.771924+0000","last_peered":"2023-09-23T21:14:03.771924+0000","last_clean":"2023-09-23T21:14:03.771924+0000","last_became_active":"2023-09-23T21:12:10.763486+0000","last_became_peered":"2023-09-23T21:12:10.763486+0000","last_unstale":"2023-09-23T21:14:03.771924+0000","last_undegraded":"2023-09-23T21:14:03.771924+0000","last_fullsized":"2023-09-23T21:14:03.771924+0000","mapping_epoch":470,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:32:27.577036+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,7],"acting":[2,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.10","version":"79'161","reported_seq":661,"reported_epoch":526,"state":"active+clean","last_fresh":"2023-09-23T21:14:09.569033+0000","last_change":"2023-09-23T21:11:11.716703+0000","last_active":"2023-09-23T21:14:09.569033+0000","last_peered":"2023-09-23T21:14:09.569033+0000","last_clean":"2023-09-23T21:14:09.569033+0000","last_became_active":"2023-09-23T21:11:11.716385+0000","last_became_peered":"2023-09-23T21:11:11.716385+0000","last_unstale":"2023-09-23T21:14:09.569033+0000","last_undegraded":"2023-09-23T21:14:09.569033+0000","last_fullsized":"2023-09-23T21:14:09.569033+0000","mapping_epoch":446,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":447,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:06:50.336992+0000","last_deep_scrub":"79'161","last_deep_scrub_stamp":"2023-09-23T21:06:50.336992+0000","last_clean_scrub_stamp":"2023-09-23T21:06:50.336992+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:44:29.538890+0000","scrub_duration":0.0026150930000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,7],"acting":[4,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":466,"reported_epoch":525,"state":"active+clean","last_fresh":"2023-09-23T21:14:03.769762+0000","last_change":"2023-09-23T21:13:12.270990+0000","last_active":"2023-09-23T21:14:03.769762+0000","last_peered":"2023-09-23T21:14:03.769762+0000","last_clean":"2023-09-23T21:14:03.769762+0000","last_became_active":"2023-09-23T21:13:12.237201+0000","last_became_peered":"2023-09-23T21:13:12.237201+0000","last_unstale":"2023-09-23T21:14:03.769762+0000","last_undegraded":"2023-09-23T21:14:03.769762+0000","last_fullsized":"2023-09-23T21:14:03.769762+0000","mapping_epoch":493,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:59:54.523877+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":277,"reported_epoch":525,"state":"active+clean","last_fresh":"2023-09-23T21:14:03.786006+0000","last_change":"2023-09-23T21:10:26.758671+0000","last_active":"2023-09-23T21:14:03.786006+0000","last_peered":"2023-09-23T21:14:03.786006+0000","last_clean":"2023-09-23T21:14:03.786006+0000","last_became_active":"2023-09-23T21:10:26.758244+0000","last_became_peered":"2023-09-23T21:10:26.758244+0000","last_unstale":"2023-09-23T21:14:03.786006+0000","last_undegraded":"2023-09-23T21:14:03.786006+0000","last_fullsized":"2023-09-23T21:14:03.786006+0000","mapping_epoch":423,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":424,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:14:02.735609+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.18","version":"526'5066","reported_seq":6149,"reported_epoch":526,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T21:14:16.190879+0000","last_change":"2023-09-23T21:13:13.412097+0000","last_active":"2023-09-23T21:14:16.190879+0000","last_peered":"2023-09-23T21:14:16.190879+0000","last_clean":"2023-09-23T21:11:45.720015+0000","last_became_active":"2023-09-23T21:13:13.395632+0000","last_became_peered":"2023-09-23T21:13:13.395632+0000","last_unstale":"2023-09-23T21:14:16.190879+0000","last_undegraded":"2023-09-23T21:13:13.220971+0000","last_fullsized":"2023-09-23T21:13:13.183825+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":438,"last_epoch_clean":439,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_clean_scrub_stamp":"2023-09-23T21:10:15.663179+0000","objects_scrubbed":0,"log_size":5066,"log_dups_size":0,"ondisk_log_size":5066,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":86671360,"num_objects":1323,"num_object_clones":0,"num_object_copies":3969,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":661,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1323,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3717,"num_write_kb":161248,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":231,"num_bytes_recovered":15138816,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,7],"acting":[2,7],"avail_no_missing":["2","7"],"object_location_counts":[{"shards":"2,7","objects":1289}],"blocked_by":[],"up_primary":0,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.13","version":"79'161","reported_seq":602,"reported_epoch":525,"state":"active+clean","last_fresh":"2023-09-23T21:14:03.785054+0000","last_change":"2023-09-23T21:13:14.933096+0000","last_active":"2023-09-23T21:14:03.785054+0000","last_peered":"2023-09-23T21:14:03.785054+0000","last_clean":"2023-09-23T21:14:03.785054+0000","last_became_active":"2023-09-23T21:13:14.932703+0000","last_became_peered":"2023-09-23T21:13:14.932703+0000","last_unstale":"2023-09-23T21:14:03.785054+0000","last_undegraded":"2023-09-23T21:14:03.785054+0000","last_fullsized":"2023-09-23T21:14:03.785054+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:36.420362+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:36.420362+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:29:42.163911+0000","scrub_duration":0.028838444000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":518,"reported_epoch":526,"state":"active+clean","last_fresh":"2023-09-23T21:14:09.538836+0000","last_change":"2023-09-23T21:13:12.216467+0000","last_active":"2023-09-23T21:14:09.538836+0000","last_peered":"2023-09-23T21:14:09.538836+0000","last_clean":"2023-09-23T21:14:09.538836+0000","last_became_active":"2023-09-23T21:13:12.215259+0000","last_became_peered":"2023-09-23T21:13:12.215259+0000","last_unstale":"2023-09-23T21:14:09.538836+0000","last_undegraded":"2023-09-23T21:14:09.538836+0000","last_fullsized":"2023-09-23T21:14:09.538836+0000","mapping_epoch":493,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:35:33.987839+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":35,"reported_epoch":525,"state":"active+clean","last_fresh":"2023-09-23T21:14:03.770375+0000","last_change":"2023-09-23T21:13:30.804108+0000","last_active":"2023-09-23T21:14:03.770375+0000","last_peered":"2023-09-23T21:14:03.770375+0000","last_clean":"2023-09-23T21:14:03.770375+0000","last_became_active":"2023-09-23T21:13:30.796505+0000","last_became_peered":"2023-09-23T21:13:30.796505+0000","last_unstale":"2023-09-23T21:14:03.770375+0000","last_undegraded":"2023-09-23T21:14:03.770375+0000","last_fullsized":"2023-09-23T21:14:03.770375+0000","mapping_epoch":501,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:19:33.508496+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.19","version":"526'5234","reported_seq":6331,"reported_epoch":526,"state":"active+clean","last_fresh":"2023-09-23T21:14:16.241168+0000","last_change":"2023-09-23T21:10:58.790716+0000","last_active":"2023-09-23T21:14:16.241168+0000","last_peered":"2023-09-23T21:14:16.241168+0000","last_clean":"2023-09-23T21:14:16.241168+0000","last_became_active":"2023-09-23T21:10:58.787975+0000","last_became_peered":"2023-09-23T21:10:58.787975+0000","last_unstale":"2023-09-23T21:14:16.241168+0000","last_undegraded":"2023-09-23T21:14:16.241168+0000","last_fullsized":"2023-09-23T21:14:16.241168+0000","mapping_epoch":438,"log_start":"0'0","ondisk_log_start":"0'0","created":438,"last_epoch_clean":439,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_clean_scrub_stamp":"2023-09-23T21:10:15.663179+0000","objects_scrubbed":0,"log_size":5234,"log_dups_size":0,"ondisk_log_size":5234,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":81559552,"num_objects":1245,"num_object_clones":0,"num_object_copies":3735,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1245,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3847,"num_write_kb":162912,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":237,"num_bytes_recovered":15499264,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.12","version":"79'161","reported_seq":759,"reported_epoch":525,"state":"active+clean","last_fresh":"2023-09-23T21:14:03.798784+0000","last_change":"2023-09-23T21:13:15.206574+0000","last_active":"2023-09-23T21:14:03.798784+0000","last_peered":"2023-09-23T21:14:03.798784+0000","last_clean":"2023-09-23T21:14:03.798784+0000","last_became_active":"2023-09-23T21:13:15.206186+0000","last_became_peered":"2023-09-23T21:13:15.206186+0000","last_unstale":"2023-09-23T21:14:03.798784+0000","last_undegraded":"2023-09-23T21:14:03.798784+0000","last_fullsized":"2023-09-23T21:14:03.798784+0000","mapping_epoch":496,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":497,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:05:56.785173+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:05:56.785173+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:34:29.221809+0000","scrub_duration":0.0042114370000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,0],"acting":[5,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":45,"reported_epoch":525,"state":"active+clean","last_fresh":"2023-09-23T21:14:03.768902+0000","last_change":"2023-09-23T21:13:38.048791+0000","last_active":"2023-09-23T21:14:03.768902+0000","last_peered":"2023-09-23T21:14:03.768902+0000","last_clean":"2023-09-23T21:14:03.768902+0000","last_became_active":"2023-09-23T21:13:30.801386+0000","last_became_peered":"2023-09-23T21:13:30.801386+0000","last_unstale":"2023-09-23T21:14:03.768902+0000","last_undegraded":"2023-09-23T21:14:03.768902+0000","last_fullsized":"2023-09-23T21:14:03.768902+0000","mapping_epoch":501,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:52:49.170414+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":362,"reported_epoch":525,"state":"active+clean","last_fresh":"2023-09-23T21:14:03.786928+0000","last_change":"2023-09-23T21:12:10.765780+0000","last_active":"2023-09-23T21:14:03.786928+0000","last_peered":"2023-09-23T21:14:03.786928+0000","last_clean":"2023-09-23T21:14:03.786928+0000","last_became_active":"2023-09-23T21:12:10.765372+0000","last_became_peered":"2023-09-23T21:12:10.765372+0000","last_unstale":"2023-09-23T21:14:03.786928+0000","last_undegraded":"2023-09-23T21:14:03.786928+0000","last_fullsized":"2023-09-23T21:14:03.786928+0000","mapping_epoch":470,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:56:08.446099+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.6","version":"526'5195","reported_seq":8110,"reported_epoch":526,"state":"active+clean","last_fresh":"2023-09-23T21:14:15.088315+0000","last_change":"2023-09-23T21:13:39.390285+0000","last_active":"2023-09-23T21:14:15.088315+0000","last_peered":"2023-09-23T21:14:15.088315+0000","last_clean":"2023-09-23T21:14:15.088315+0000","last_became_active":"2023-09-23T21:13:39.378284+0000","last_became_peered":"2023-09-23T21:13:39.378284+0000","last_unstale":"2023-09-23T21:14:15.088315+0000","last_undegraded":"2023-09-23T21:14:15.088315+0000","last_fullsized":"2023-09-23T21:14:15.088315+0000","mapping_epoch":508,"log_start":"0'0","ondisk_log_start":"0'0","created":416,"last_epoch_clean":509,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_clean_scrub_stamp":"2023-09-23T21:10:15.663179+0000","objects_scrubbed":0,"log_size":5195,"log_dups_size":0,"ondisk_log_size":5195,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":83689472,"num_objects":1277,"num_object_clones":0,"num_object_copies":3831,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1277,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3789,"num_write_kb":162112,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2399,"num_bytes_recovered":113967104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,4],"acting":[0,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.d","version":"79'161","reported_seq":679,"reported_epoch":525,"state":"active+clean","last_fresh":"2023-09-23T21:14:03.789913+0000","last_change":"2023-09-23T21:12:30.444461+0000","last_active":"2023-09-23T21:14:03.789913+0000","last_peered":"2023-09-23T21:14:03.789913+0000","last_clean":"2023-09-23T21:14:03.789913+0000","last_became_active":"2023-09-23T21:12:30.443979+0000","last_became_peered":"2023-09-23T21:12:30.443979+0000","last_unstale":"2023-09-23T21:14:03.789913+0000","last_undegraded":"2023-09-23T21:14:03.789913+0000","last_fullsized":"2023-09-23T21:14:03.789913+0000","mapping_epoch":470,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:09:36.858077+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:09:36.858077+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0024987500000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":172,"reported_epoch":525,"state":"active+clean","last_fresh":"2023-09-23T21:14:03.791067+0000","last_change":"2023-09-23T21:13:59.872834+0000","last_active":"2023-09-23T21:14:03.791067+0000","last_peered":"2023-09-23T21:14:03.791067+0000","last_clean":"2023-09-23T21:14:03.791067+0000","last_became_active":"2023-09-23T21:13:30.770556+0000","last_became_peered":"2023-09-23T21:13:30.770556+0000","last_unstale":"2023-09-23T21:14:03.791067+0000","last_undegraded":"2023-09-23T21:14:03.791067+0000","last_fullsized":"2023-09-23T21:14:03.791067+0000","mapping_epoch":501,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":521,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:57:55.460052+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":132,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":233,"reported_epoch":525,"state":"active+clean","last_fresh":"2023-09-23T21:14:03.798116+0000","last_change":"2023-09-23T21:10:26.750725+0000","last_active":"2023-09-23T21:14:03.798116+0000","last_peered":"2023-09-23T21:14:03.798116+0000","last_clean":"2023-09-23T21:14:03.798116+0000","last_became_active":"2023-09-23T21:10:26.750307+0000","last_became_peered":"2023-09-23T21:10:26.750307+0000","last_unstale":"2023-09-23T21:14:03.798116+0000","last_undegraded":"2023-09-23T21:14:03.798116+0000","last_fullsized":"2023-09-23T21:14:03.798116+0000","mapping_epoch":423,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":424,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:06:22.040548+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,4],"acting":[5,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.7","version":"526'5086","reported_seq":5279,"reported_epoch":526,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T21:14:14.060264+0000","last_change":"2023-09-23T21:13:38.417178+0000","last_active":"2023-09-23T21:14:14.060264+0000","last_peered":"2023-09-23T21:14:14.060264+0000","last_clean":"2023-09-23T21:11:26.699871+0000","last_became_active":"2023-09-23T21:13:38.408523+0000","last_became_peered":"2023-09-23T21:13:38.408523+0000","last_unstale":"2023-09-23T21:14:14.060264+0000","last_undegraded":"2023-09-23T21:13:38.369833+0000","last_fullsized":"2023-09-23T21:13:38.358198+0000","mapping_epoch":507,"log_start":"422'214","ondisk_log_start":"422'214","created":416,"last_epoch_clean":439,"parent":"0.0","parent_split_bits":0,"last_scrub":"441'2973","last_scrub_stamp":"2023-09-23T21:11:06.079326+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_clean_scrub_stamp":"2023-09-23T21:11:06.079326+0000","objects_scrubbed":1647,"log_size":4872,"log_dups_size":0,"ondisk_log_size":4872,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:27:59.805526+0000","scrub_duration":0.44319714500000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":82378752,"num_objects":1257,"num_object_clones":0,"num_object_copies":3771,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1165,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1257,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2118,"num_write_kb":55296,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,2],"acting":[3,5],"avail_no_missing":["3","5"],"object_location_counts":[{"shards":"3,4,5","objects":298},{"shards":"3,5","objects":959}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.c","version":"79'161","reported_seq":888,"reported_epoch":525,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-23T21:14:03.785422+0000","last_change":"2023-09-23T21:13:13.402530+0000","last_active":"2023-09-23T21:14:03.785422+0000","last_peered":"2023-09-23T21:14:03.785422+0000","last_clean":"2023-09-23T21:11:23.200766+0000","last_became_active":"2023-09-23T21:13:13.401526+0000","last_became_peered":"2023-09-23T21:13:13.401526+0000","last_unstale":"2023-09-23T21:14:03.785422+0000","last_undegraded":"2023-09-23T21:13:13.204992+0000","last_fullsized":"2023-09-23T21:14:03.785422+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":420,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:22.040907+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:22.040907+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.013021176000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":163840,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":163840,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":["3","0","5"],"object_location_counts":[{"shards":"0,3,4","objects":-1},{"shards":"0,3,5","objects":1}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":158,"reported_epoch":525,"state":"active+clean","last_fresh":"2023-09-23T21:14:03.790541+0000","last_change":"2023-09-23T21:13:59.884113+0000","last_active":"2023-09-23T21:14:03.790541+0000","last_peered":"2023-09-23T21:14:03.790541+0000","last_clean":"2023-09-23T21:14:03.790541+0000","last_became_active":"2023-09-23T21:13:30.806778+0000","last_became_peered":"2023-09-23T21:13:30.806778+0000","last_unstale":"2023-09-23T21:14:03.790541+0000","last_undegraded":"2023-09-23T21:14:03.790541+0000","last_fullsized":"2023-09-23T21:14:03.790541+0000","mapping_epoch":501,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":521,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:36:15.284465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":489,"reported_epoch":526,"state":"active+clean","last_fresh":"2023-09-23T21:14:09.538881+0000","last_change":"2023-09-23T21:13:15.152792+0000","last_active":"2023-09-23T21:14:09.538881+0000","last_peered":"2023-09-23T21:14:09.538881+0000","last_clean":"2023-09-23T21:14:09.538881+0000","last_became_active":"2023-09-23T21:13:14.515084+0000","last_became_peered":"2023-09-23T21:13:14.515084+0000","last_unstale":"2023-09-23T21:14:09.538881+0000","last_undegraded":"2023-09-23T21:14:09.538881+0000","last_fullsized":"2023-09-23T21:14:09.538881+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:08:16.702072+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.4","version":"526'4986","reported_seq":11928,"reported_epoch":526,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T21:14:14.119977+0000","last_change":"2023-09-23T21:13:30.847170+0000","last_active":"2023-09-23T21:14:14.119977+0000","last_peered":"2023-09-23T21:14:14.119977+0000","last_clean":"2023-09-23T21:11:06.743842+0000","last_became_active":"2023-09-23T21:13:30.834806+0000","last_became_peered":"2023-09-23T21:13:30.834806+0000","last_unstale":"2023-09-23T21:14:14.119977+0000","last_undegraded":"2023-09-23T21:13:30.760230+0000","last_fullsized":"2023-09-23T21:13:30.749974+0000","mapping_epoch":501,"log_start":"0'0","ondisk_log_start":"0'0","created":416,"last_epoch_clean":439,"parent":"0.0","parent_split_bits":0,"last_scrub":"440'2861","last_scrub_stamp":"2023-09-23T21:11:03.842926+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_clean_scrub_stamp":"2023-09-23T21:11:03.842926+0000","objects_scrubbed":1526,"log_size":4986,"log_dups_size":0,"ondisk_log_size":4986,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:32:29.379756+0000","scrub_duration":0.39378383900000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":87490560,"num_objects":1335,"num_object_clones":0,"num_object_copies":4005,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1130,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1335,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2131,"num_write_kb":62080,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2723,"num_bytes_recovered":163577856,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[3,4],"avail_no_missing":["3","4"],"object_location_counts":[{"shards":"1,2,3,4","objects":360},{"shards":"3,4","objects":975}],"blocked_by":[],"up_primary":0,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.f","version":"79'161","reported_seq":813,"reported_epoch":525,"state":"active+clean","last_fresh":"2023-09-23T21:14:03.799892+0000","last_change":"2023-09-23T21:11:30.750338+0000","last_active":"2023-09-23T21:14:03.799892+0000","last_peered":"2023-09-23T21:14:03.799892+0000","last_clean":"2023-09-23T21:14:03.799892+0000","last_became_active":"2023-09-23T21:11:30.749953+0000","last_became_peered":"2023-09-23T21:11:30.749953+0000","last_unstale":"2023-09-23T21:14:03.799892+0000","last_undegraded":"2023-09-23T21:14:03.799892+0000","last_fullsized":"2023-09-23T21:14:03.799892+0000","mapping_epoch":455,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":456,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:27.880776+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:27.880776+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:07:10.405723+0000","scrub_duration":0.0014004899999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,3],"acting":[5,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":249,"reported_epoch":525,"state":"active+clean","last_fresh":"2023-09-23T21:14:03.799044+0000","last_change":"2023-09-23T21:10:26.765068+0000","last_active":"2023-09-23T21:14:03.799044+0000","last_peered":"2023-09-23T21:14:03.799044+0000","last_clean":"2023-09-23T21:14:03.799044+0000","last_became_active":"2023-09-23T21:10:26.758364+0000","last_became_peered":"2023-09-23T21:10:26.758364+0000","last_unstale":"2023-09-23T21:14:03.799044+0000","last_undegraded":"2023-09-23T21:14:03.799044+0000","last_fullsized":"2023-09-23T21:14:03.799044+0000","mapping_epoch":423,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":424,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:13:21.057529+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":280,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":465,"reported_epoch":525,"state":"active+clean","last_fresh":"2023-09-23T21:14:03.798497+0000","last_change":"2023-09-23T21:13:30.770746+0000","last_active":"2023-09-23T21:14:03.798497+0000","last_peered":"2023-09-23T21:14:03.798497+0000","last_clean":"2023-09-23T21:14:03.798497+0000","last_became_active":"2023-09-23T21:13:30.770437+0000","last_became_peered":"2023-09-23T21:13:30.770437+0000","last_unstale":"2023-09-23T21:14:03.798497+0000","last_undegraded":"2023-09-23T21:14:03.798497+0000","last_fullsized":"2023-09-23T21:14:03.798497+0000","mapping_epoch":501,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:33:48.325730+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,0],"acting":[5,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.5","version":"526'5067","reported_seq":5251,"reported_epoch":526,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T21:14:16.228462+0000","last_change":"2023-09-23T21:13:13.436794+0000","last_active":"2023-09-23T21:14:16.228462+0000","last_peered":"2023-09-23T21:14:16.228462+0000","last_clean":"2023-09-23T21:11:45.768408+0000","last_became_active":"2023-09-23T21:13:13.424939+0000","last_became_peered":"2023-09-23T21:13:13.424939+0000","last_unstale":"2023-09-23T21:14:16.228462+0000","last_undegraded":"2023-09-23T21:13:13.225067+0000","last_fullsized":"2023-09-23T21:13:13.198194+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":416,"last_epoch_clean":439,"parent":"0.0","parent_split_bits":0,"last_scrub":"440'2840","last_scrub_stamp":"2023-09-23T21:11:03.208833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_clean_scrub_stamp":"2023-09-23T21:11:03.208833+0000","objects_scrubbed":1498,"log_size":5067,"log_dups_size":0,"ondisk_log_size":5067,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:56:40.574477+0000","scrub_duration":0.45608471699999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":84934656,"num_objects":1296,"num_object_clones":0,"num_object_copies":3888,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":644,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1296,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2236,"num_write_kb":65088,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,3],"acting":[2,3],"avail_no_missing":["2","3"],"object_location_counts":[{"shards":"2,3","objects":1250}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.e","version":"79'161","reported_seq":695,"reported_epoch":526,"state":"active+clean","last_fresh":"2023-09-23T21:14:09.539188+0000","last_change":"2023-09-23T21:12:30.440952+0000","last_active":"2023-09-23T21:14:09.539188+0000","last_peered":"2023-09-23T21:14:09.539188+0000","last_clean":"2023-09-23T21:14:09.539188+0000","last_became_active":"2023-09-23T21:12:30.440463+0000","last_became_peered":"2023-09-23T21:12:30.440463+0000","last_unstale":"2023-09-23T21:14:09.539188+0000","last_undegraded":"2023-09-23T21:14:09.539188+0000","last_fullsized":"2023-09-23T21:14:09.539188+0000","mapping_epoch":455,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:10:39.567232+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:10:39.567232+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.005229694,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":323,"reported_epoch":525,"state":"active+clean","last_fresh":"2023-09-23T21:14:03.785833+0000","last_change":"2023-09-23T21:13:13.402418+0000","last_active":"2023-09-23T21:14:03.785833+0000","last_peered":"2023-09-23T21:14:03.785833+0000","last_clean":"2023-09-23T21:14:03.785833+0000","last_became_active":"2023-09-23T21:13:13.400984+0000","last_became_peered":"2023-09-23T21:13:13.400984+0000","last_unstale":"2023-09-23T21:14:03.785833+0000","last_undegraded":"2023-09-23T21:14:03.785833+0000","last_fullsized":"2023-09-23T21:14:03.785833+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":495,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:28:01.411191+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,0],"acting":[3,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":498,"reported_epoch":525,"state":"active+clean","last_fresh":"2023-09-23T21:14:03.771167+0000","last_change":"2023-09-23T21:13:14.339670+0000","last_active":"2023-09-23T21:14:03.771167+0000","last_peered":"2023-09-23T21:14:03.771167+0000","last_clean":"2023-09-23T21:14:03.771167+0000","last_became_active":"2023-09-23T21:13:14.339341+0000","last_became_peered":"2023-09-23T21:13:14.339341+0000","last_unstale":"2023-09-23T21:14:03.771167+0000","last_undegraded":"2023-09-23T21:14:03.771167+0000","last_fullsized":"2023-09-23T21:14:03.771167+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:23:19.411368+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.2","version":"527'5150","reported_seq":6628,"reported_epoch":527,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T21:14:18.231890+0000","last_change":"2023-09-23T21:13:13.407366+0000","last_active":"2023-09-23T21:14:18.231890+0000","last_peered":"2023-09-23T21:14:18.231890+0000","last_clean":"2023-09-23T21:11:45.752822+0000","last_became_active":"2023-09-23T21:13:13.398616+0000","last_became_peered":"2023-09-23T21:13:13.398616+0000","last_unstale":"2023-09-23T21:14:18.231890+0000","last_undegraded":"2023-09-23T21:13:13.289815+0000","last_fullsized":"2023-09-23T21:13:13.190270+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":416,"last_epoch_clean":439,"parent":"0.0","parent_split_bits":0,"last_scrub":"440'3007","last_scrub_stamp":"2023-09-23T21:11:05.553661+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_clean_scrub_stamp":"2023-09-23T21:11:05.553661+0000","objects_scrubbed":1622,"log_size":5150,"log_dups_size":0,"ondisk_log_size":5150,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:51:21.839818+0000","scrub_duration":0.39156952499999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":80871424,"num_objects":1234,"num_object_clones":0,"num_object_copies":3702,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2526,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1234,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2148,"num_write_kb":56320,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":639,"num_bytes_recovered":41877504,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,3],"avail_no_missing":["4","1"],"object_location_counts":[{"shards":"0,1,4","objects":1200},{"shards":"1,4","objects":34}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.9","version":"79'161","reported_seq":679,"reported_epoch":525,"state":"active+clean","last_fresh":"2023-09-23T21:14:03.771555+0000","last_change":"2023-09-23T21:12:30.436118+0000","last_active":"2023-09-23T21:14:03.771555+0000","last_peered":"2023-09-23T21:14:03.771555+0000","last_clean":"2023-09-23T21:14:03.771555+0000","last_became_active":"2023-09-23T21:12:30.435779+0000","last_became_peered":"2023-09-23T21:12:30.435779+0000","last_unstale":"2023-09-23T21:14:03.771555+0000","last_undegraded":"2023-09-23T21:14:03.771555+0000","last_fullsized":"2023-09-23T21:14:03.771555+0000","mapping_epoch":433,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:10:37.696951+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:10:37.696951+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.001386504,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,3],"acting":[2,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":84,"reported_epoch":525,"state":"active+clean","last_fresh":"2023-09-23T21:14:03.771254+0000","last_change":"2023-09-23T21:13:30.793273+0000","last_active":"2023-09-23T21:14:03.771254+0000","last_peered":"2023-09-23T21:14:03.771254+0000","last_clean":"2023-09-23T21:14:03.771254+0000","last_became_active":"2023-09-23T21:13:30.791269+0000","last_became_peered":"2023-09-23T21:13:30.791269+0000","last_unstale":"2023-09-23T21:14:03.771254+0000","last_undegraded":"2023-09-23T21:14:03.771254+0000","last_fullsized":"2023-09-23T21:14:03.771254+0000","mapping_epoch":501,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:38:07.142290+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":100,"reported_epoch":526,"state":"active+clean","last_fresh":"2023-09-23T21:14:09.539315+0000","last_change":"2023-09-23T21:12:14.313466+0000","last_active":"2023-09-23T21:14:09.539315+0000","last_peered":"2023-09-23T21:14:09.539315+0000","last_clean":"2023-09-23T21:14:09.539315+0000","last_became_active":"2023-09-23T21:12:14.209837+0000","last_became_peered":"2023-09-23T21:12:14.209837+0000","last_unstale":"2023-09-23T21:14:09.539315+0000","last_undegraded":"2023-09-23T21:14:09.539315+0000","last_fullsized":"2023-09-23T21:14:09.539315+0000","mapping_epoch":470,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":474,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:09:40.375241+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,7],"acting":[4,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.3","version":"526'5153","reported_seq":7282,"reported_epoch":526,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T21:14:16.206325+0000","last_change":"2023-09-23T21:13:45.480705+0000","last_active":"2023-09-23T21:14:16.206325+0000","last_peered":"2023-09-23T21:14:16.206325+0000","last_clean":"2023-09-23T21:11:26.700990+0000","last_became_active":"2023-09-23T21:13:45.471702+0000","last_became_peered":"2023-09-23T21:13:45.471702+0000","last_unstale":"2023-09-23T21:14:16.206325+0000","last_undegraded":"2023-09-23T21:13:45.412179+0000","last_fullsized":"2023-09-23T21:13:45.401701+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":416,"last_epoch_clean":439,"parent":"0.0","parent_split_bits":0,"last_scrub":"440'2873","last_scrub_stamp":"2023-09-23T21:11:02.114783+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_clean_scrub_stamp":"2023-09-23T21:11:02.114783+0000","objects_scrubbed":1532,"log_size":5153,"log_dups_size":0,"ondisk_log_size":5153,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:42:48.549370+0000","scrub_duration":0.32435022600000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":84017152,"num_objects":1282,"num_object_clones":0,"num_object_copies":3846,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1184,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1282,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2286,"num_write_kb":65152,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":865,"num_bytes_recovered":56688640,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5],"avail_no_missing":["2","5"],"object_location_counts":[{"shards":"0,2,4,5","objects":270},{"shards":"2,4,5","objects":48},{"shards":"2,5","objects":963}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.8","version":"79'161","reported_seq":646,"reported_epoch":525,"state":"active+clean","last_fresh":"2023-09-23T21:14:03.771988+0000","last_change":"2023-09-23T21:12:30.433561+0000","last_active":"2023-09-23T21:14:03.771988+0000","last_peered":"2023-09-23T21:14:03.771988+0000","last_clean":"2023-09-23T21:14:03.771988+0000","last_became_active":"2023-09-23T21:12:30.433217+0000","last_became_peered":"2023-09-23T21:12:30.433217+0000","last_unstale":"2023-09-23T21:14:03.771988+0000","last_undegraded":"2023-09-23T21:14:03.771988+0000","last_fullsized":"2023-09-23T21:14:03.771988+0000","mapping_epoch":359,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:06:52.098882+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:06:52.098882+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0092173299999999993,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":131344,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":103,"reported_epoch":526,"state":"active+clean","last_fresh":"2023-09-23T21:14:09.540391+0000","last_change":"2023-09-23T21:12:10.823380+0000","last_active":"2023-09-23T21:14:09.540391+0000","last_peered":"2023-09-23T21:14:09.540391+0000","last_clean":"2023-09-23T21:14:09.540391+0000","last_became_active":"2023-09-23T21:12:10.823044+0000","last_became_peered":"2023-09-23T21:12:10.823044+0000","last_unstale":"2023-09-23T21:14:09.540391+0000","last_undegraded":"2023-09-23T21:14:09.540391+0000","last_fullsized":"2023-09-23T21:14:09.540391+0000","mapping_epoch":470,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:53:29.846536+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,5],"acting":[4,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":104,"reported_epoch":526,"state":"active+clean","last_fresh":"2023-09-23T21:14:09.539859+0000","last_change":"2023-09-23T21:13:30.772451+0000","last_active":"2023-09-23T21:14:09.539859+0000","last_peered":"2023-09-23T21:14:09.539859+0000","last_clean":"2023-09-23T21:14:09.539859+0000","last_became_active":"2023-09-23T21:13:30.772040+0000","last_became_peered":"2023-09-23T21:13:30.772040+0000","last_unstale":"2023-09-23T21:14:09.539859+0000","last_undegraded":"2023-09-23T21:14:09.539859+0000","last_fullsized":"2023-09-23T21:14:09.539859+0000","mapping_epoch":501,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:24:49.246727+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,0],"acting":[4,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.0","version":"527'5111","reported_seq":9182,"reported_epoch":527,"state":"active+recovering+undersized+remapped","last_fresh":"2023-09-23T21:14:18.243376+0000","last_change":"2023-09-23T21:14:02.130787+0000","last_active":"2023-09-23T21:14:18.243376+0000","last_peered":"2023-09-23T21:14:18.243376+0000","last_clean":"2023-09-23T21:11:26.716395+0000","last_became_active":"2023-09-23T21:13:34.557827+0000","last_became_peered":"2023-09-23T21:13:34.557827+0000","last_unstale":"2023-09-23T21:14:18.243376+0000","last_undegraded":"2023-09-23T21:14:18.243376+0000","last_fullsized":"2023-09-23T21:13:31.752840+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":416,"last_epoch_clean":439,"parent":"0.0","parent_split_bits":0,"last_scrub":"440'2961","last_scrub_stamp":"2023-09-23T21:11:04.564028+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_clean_scrub_stamp":"2023-09-23T21:11:04.564028+0000","objects_scrubbed":1608,"log_size":5111,"log_dups_size":0,"ondisk_log_size":5111,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:43:09.815808+0000","scrub_duration":0.41858158299999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":81461248,"num_objects":1243,"num_object_clones":0,"num_object_copies":3729,"num_objects_missing_on_primary":1063,"num_objects_missing":1063,"num_objects_degraded":0,"num_objects_misplaced":1238,"num_objects_unfound":0,"num_objects_dirty":1243,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2153,"num_write_kb":57216,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1104,"num_bytes_recovered":72286208,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,0],"avail_no_missing":["1","7"],"object_location_counts":[{"shards":"0,1,4,7","objects":5},{"shards":"0,1,7","objects":912},{"shards":"1,4,7","objects":326}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.b","version":"79'161","reported_seq":529,"reported_epoch":525,"state":"active+clean","last_fresh":"2023-09-23T21:14:03.771000+0000","last_change":"2023-09-23T21:13:13.373313+0000","last_active":"2023-09-23T21:14:03.771000+0000","last_peered":"2023-09-23T21:14:03.771000+0000","last_clean":"2023-09-23T21:14:03.771000+0000","last_became_active":"2023-09-23T21:13:13.372328+0000","last_became_peered":"2023-09-23T21:13:13.372328+0000","last_unstale":"2023-09-23T21:14:03.771000+0000","last_undegraded":"2023-09-23T21:14:03.771000+0000","last_fullsized":"2023-09-23T21:14:03.771000+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":495,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:05:55.772264+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:05:55.772264+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.001862507,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":84,"reported_epoch":526,"state":"active+clean","last_fresh":"2023-09-23T21:14:09.538803+0000","last_change":"2023-09-23T21:11:27.754717+0000","last_active":"2023-09-23T21:14:09.538803+0000","last_peered":"2023-09-23T21:14:09.538803+0000","last_clean":"2023-09-23T21:14:09.538803+0000","last_became_active":"2023-09-23T21:11:27.754376+0000","last_became_peered":"2023-09-23T21:11:27.754376+0000","last_unstale":"2023-09-23T21:14:09.538803+0000","last_undegraded":"2023-09-23T21:14:09.538803+0000","last_fullsized":"2023-09-23T21:14:09.538803+0000","mapping_epoch":452,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":453,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:26:41.734541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":458,"reported_epoch":526,"state":"active+clean","last_fresh":"2023-09-23T21:14:09.539860+0000","last_change":"2023-09-23T21:12:10.811698+0000","last_active":"2023-09-23T21:14:09.539860+0000","last_peered":"2023-09-23T21:14:09.539860+0000","last_clean":"2023-09-23T21:14:09.539860+0000","last_became_active":"2023-09-23T21:12:10.789656+0000","last_became_peered":"2023-09-23T21:12:10.789656+0000","last_unstale":"2023-09-23T21:14:09.539860+0000","last_undegraded":"2023-09-23T21:14:09.539860+0000","last_fullsized":"2023-09-23T21:14:09.539860+0000","mapping_epoch":470,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:39:15.112091+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,5],"acting":[4,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.1","version":"527'5035","reported_seq":11448,"reported_epoch":527,"state":"active+clean","last_fresh":"2023-09-23T21:14:18.098836+0000","last_change":"2023-09-23T21:13:53.483736+0000","last_active":"2023-09-23T21:14:18.098836+0000","last_peered":"2023-09-23T21:14:18.098836+0000","last_clean":"2023-09-23T21:14:18.098836+0000","last_became_active":"2023-09-23T21:13:53.472685+0000","last_became_peered":"2023-09-23T21:13:53.472685+0000","last_unstale":"2023-09-23T21:14:18.098836+0000","last_undegraded":"2023-09-23T21:14:18.098836+0000","last_fullsized":"2023-09-23T21:14:18.098836+0000","mapping_epoch":519,"log_start":"0'0","ondisk_log_start":"0'0","created":416,"last_epoch_clean":520,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_clean_scrub_stamp":"2023-09-23T21:10:15.663179+0000","objects_scrubbed":0,"log_size":5035,"log_dups_size":0,"ondisk_log_size":5035,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":82378752,"num_objects":1257,"num_object_clones":0,"num_object_copies":3771,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1257,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3687,"num_write_kb":158208,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3005,"num_bytes_recovered":156499968,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.a","version":"79'161","reported_seq":672,"reported_epoch":525,"state":"active+clean","last_fresh":"2023-09-23T21:14:03.773235+0000","last_change":"2023-09-23T21:12:30.437790+0000","last_active":"2023-09-23T21:14:03.773235+0000","last_peered":"2023-09-23T21:14:03.773235+0000","last_clean":"2023-09-23T21:14:03.773235+0000","last_became_active":"2023-09-23T21:12:30.437427+0000","last_became_peered":"2023-09-23T21:12:30.437427+0000","last_unstale":"2023-09-23T21:14:03.773235+0000","last_undegraded":"2023-09-23T21:14:03.773235+0000","last_fullsized":"2023-09-23T21:14:03.773235+0000","mapping_epoch":446,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:06:51.075448+0000","last_deep_scrub":"79'161","last_deep_scrub_stamp":"2023-09-23T21:05:32.630292+0000","last_clean_scrub_stamp":"2023-09-23T21:06:51.075448+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0054793890000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,4],"acting":[2,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":372,"reported_epoch":525,"state":"active+clean","last_fresh":"2023-09-23T21:14:03.784568+0000","last_change":"2023-09-23T21:12:10.792918+0000","last_active":"2023-09-23T21:14:03.784568+0000","last_peered":"2023-09-23T21:14:03.784568+0000","last_clean":"2023-09-23T21:14:03.784568+0000","last_became_active":"2023-09-23T21:12:10.792679+0000","last_became_peered":"2023-09-23T21:12:10.792679+0000","last_unstale":"2023-09-23T21:14:03.784568+0000","last_undegraded":"2023-09-23T21:14:03.784568+0000","last_fullsized":"2023-09-23T21:14:03.784568+0000","mapping_epoch":470,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:30:48.008690+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,4],"acting":[3,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":503,"reported_epoch":525,"state":"active+clean","last_fresh":"2023-09-23T21:14:03.787192+0000","last_change":"2023-09-23T21:12:10.790812+0000","last_active":"2023-09-23T21:14:03.787192+0000","last_peered":"2023-09-23T21:14:03.787192+0000","last_clean":"2023-09-23T21:14:03.787192+0000","last_became_active":"2023-09-23T21:12:10.790492+0000","last_became_peered":"2023-09-23T21:12:10.790492+0000","last_unstale":"2023-09-23T21:14:03.787192+0000","last_undegraded":"2023-09-23T21:14:03.787192+0000","last_fullsized":"2023-09-23T21:14:03.787192+0000","mapping_epoch":470,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:02:19.863118+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.c","version":"526'7552","reported_seq":8705,"reported_epoch":526,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T21:14:17.225370+0000","last_change":"2023-09-23T21:13:30.822215+0000","last_active":"2023-09-23T21:14:17.225370+0000","last_peered":"2023-09-23T21:14:17.225370+0000","last_clean":"2023-09-23T21:11:45.750376+0000","last_became_active":"2023-09-23T21:13:13.404327+0000","last_became_peered":"2023-09-23T21:13:13.404327+0000","last_unstale":"2023-09-23T21:14:17.225370+0000","last_undegraded":"2023-09-23T21:13:13.200907+0000","last_fullsized":"2023-09-23T21:13:13.193701+0000","mapping_epoch":494,"log_start":"440'3151","ondisk_log_start":"440'3151","created":416,"last_epoch_clean":434,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_clean_scrub_stamp":"2023-09-23T21:10:15.663179+0000","objects_scrubbed":0,"log_size":4401,"log_dups_size":0,"ondisk_log_size":4401,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:33:42.971375+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":162922496,"num_objects":2486,"num_object_clones":0,"num_object_copies":7458,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1339,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2486,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7552,"num_write_kb":321216,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":498,"num_bytes_recovered":32636928,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4],"avail_no_missing":["7","4"],"object_location_counts":[{"shards":"4,7","objects":2432}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.7","version":"79'161","reported_seq":618,"reported_epoch":525,"state":"active+clean","last_fresh":"2023-09-23T21:14:03.786842+0000","last_change":"2023-09-23T21:13:44.416548+0000","last_active":"2023-09-23T21:14:03.786842+0000","last_peered":"2023-09-23T21:14:03.786842+0000","last_clean":"2023-09-23T21:14:03.786842+0000","last_became_active":"2023-09-23T21:13:44.415617+0000","last_became_peered":"2023-09-23T21:13:44.415617+0000","last_unstale":"2023-09-23T21:14:03.786842+0000","last_undegraded":"2023-09-23T21:14:03.786842+0000","last_fullsized":"2023-09-23T21:14:03.786842+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":3,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:25.916867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:25.916867+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.001665323,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":455,"reported_epoch":525,"state":"active+clean","last_fresh":"2023-09-23T21:14:03.769533+0000","last_change":"2023-09-23T21:13:13.211959+0000","last_active":"2023-09-23T21:14:03.769533+0000","last_peered":"2023-09-23T21:14:03.769533+0000","last_clean":"2023-09-23T21:14:03.769533+0000","last_became_active":"2023-09-23T21:13:13.211712+0000","last_became_peered":"2023-09-23T21:13:13.211712+0000","last_unstale":"2023-09-23T21:14:03.769533+0000","last_undegraded":"2023-09-23T21:14:03.769533+0000","last_fullsized":"2023-09-23T21:14:03.769533+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":495,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:37:21.655775+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,2],"acting":[0,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":203,"reported_epoch":525,"state":"active+clean","last_fresh":"2023-09-23T21:14:03.798966+0000","last_change":"2023-09-23T21:05:04.326889+0000","last_active":"2023-09-23T21:14:03.798966+0000","last_peered":"2023-09-23T21:14:03.798966+0000","last_clean":"2023-09-23T21:14:03.798966+0000","last_became_active":"2023-09-23T21:05:04.326032+0000","last_became_peered":"2023-09-23T21:05:04.326032+0000","last_unstale":"2023-09-23T21:14:03.798966+0000","last_undegraded":"2023-09-23T21:14:03.798966+0000","last_fullsized":"2023-09-23T21:14:03.798966+0000","mapping_epoch":332,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:00:17.507889+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":535,"reported_epoch":525,"state":"active+clean","last_fresh":"2023-09-23T21:14:03.798646+0000","last_change":"2023-09-23T21:13:13.285608+0000","last_active":"2023-09-23T21:14:03.798646+0000","last_peered":"2023-09-23T21:14:03.798646+0000","last_clean":"2023-09-23T21:14:03.798646+0000","last_became_active":"2023-09-23T21:13:13.285255+0000","last_became_peered":"2023-09-23T21:13:13.285255+0000","last_unstale":"2023-09-23T21:14:03.798646+0000","last_undegraded":"2023-09-23T21:14:03.798646+0000","last_fullsized":"2023-09-23T21:14:03.798646+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":495,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:42:30.087599+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.a","version":"526'7367","reported_seq":16100,"reported_epoch":526,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T21:14:16.198239+0000","last_change":"2023-09-23T21:13:31.927282+0000","last_active":"2023-09-23T21:14:16.198239+0000","last_peered":"2023-09-23T21:14:16.198239+0000","last_clean":"2023-09-23T21:13:30.353360+0000","last_became_active":"2023-09-23T21:13:31.896923+0000","last_became_peered":"2023-09-23T21:13:31.896923+0000","last_unstale":"2023-09-23T21:14:16.198239+0000","last_undegraded":"2023-09-23T21:13:31.776904+0000","last_fullsized":"2023-09-23T21:13:31.751334+0000","mapping_epoch":502,"log_start":"440'3012","ondisk_log_start":"440'3012","created":416,"last_epoch_clean":499,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_clean_scrub_stamp":"2023-09-23T21:10:15.663179+0000","objects_scrubbed":0,"log_size":4355,"log_dups_size":0,"ondisk_log_size":4355,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:23:14.477602+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":164167680,"num_objects":2505,"num_object_clones":0,"num_object_copies":7515,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2243,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2505,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7367,"num_write_kb":315904,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4293,"num_bytes_recovered":281346048,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[2,5],"avail_no_missing":["2","3","5"],"object_location_counts":[{"shards":"2,3,5","objects":643},{"shards":"2,5","objects":1862}],"blocked_by":[],"up_primary":0,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.1","version":"79'161","reported_seq":596,"reported_epoch":525,"state":"active+clean","last_fresh":"2023-09-23T21:14:03.769087+0000","last_change":"2023-09-23T21:13:12.204135+0000","last_active":"2023-09-23T21:14:03.769087+0000","last_peered":"2023-09-23T21:14:03.769087+0000","last_clean":"2023-09-23T21:14:03.769087+0000","last_became_active":"2023-09-23T21:13:12.203380+0000","last_became_peered":"2023-09-23T21:13:12.203380+0000","last_unstale":"2023-09-23T21:14:03.769087+0000","last_undegraded":"2023-09-23T21:14:03.769087+0000","last_fullsized":"2023-09-23T21:14:03.769087+0000","mapping_epoch":493,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:06:52.575900+0000","last_deep_scrub":"79'161","last_deep_scrub_stamp":"2023-09-23T21:05:30.652189+0000","last_clean_scrub_stamp":"2023-09-23T21:06:52.575900+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":192,"reported_epoch":525,"state":"active+clean","last_fresh":"2023-09-23T21:14:03.769692+0000","last_change":"2023-09-23T21:13:13.238617+0000","last_active":"2023-09-23T21:14:03.769692+0000","last_peered":"2023-09-23T21:14:03.769692+0000","last_clean":"2023-09-23T21:14:03.769692+0000","last_became_active":"2023-09-23T21:13:13.232926+0000","last_became_peered":"2023-09-23T21:13:13.232926+0000","last_unstale":"2023-09-23T21:14:03.769692+0000","last_undegraded":"2023-09-23T21:14:03.769692+0000","last_fullsized":"2023-09-23T21:14:03.769692+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":495,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:17:21.823772+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":460,"reported_epoch":526,"state":"active+clean","last_fresh":"2023-09-23T21:14:09.555574+0000","last_change":"2023-09-23T21:13:14.144918+0000","last_active":"2023-09-23T21:14:09.555574+0000","last_peered":"2023-09-23T21:14:09.555574+0000","last_clean":"2023-09-23T21:14:09.555574+0000","last_became_active":"2023-09-23T21:13:14.132388+0000","last_became_peered":"2023-09-23T21:13:14.132388+0000","last_unstale":"2023-09-23T21:14:09.555574+0000","last_undegraded":"2023-09-23T21:14:09.555574+0000","last_fullsized":"2023-09-23T21:14:09.555574+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":495,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:56:02.619905+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,5],"acting":[4,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":346,"reported_epoch":526,"state":"active+clean","last_fresh":"2023-09-23T21:14:09.556248+0000","last_change":"2023-09-23T21:12:10.764503+0000","last_active":"2023-09-23T21:14:09.556248+0000","last_peered":"2023-09-23T21:14:09.556248+0000","last_clean":"2023-09-23T21:14:09.556248+0000","last_became_active":"2023-09-23T21:12:10.763933+0000","last_became_peered":"2023-09-23T21:12:10.763933+0000","last_unstale":"2023-09-23T21:14:09.556248+0000","last_undegraded":"2023-09-23T21:14:09.556248+0000","last_fullsized":"2023-09-23T21:14:09.556248+0000","mapping_epoch":470,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:28:33.202825+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,7],"acting":[2,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"10.9","version":"526'5101","reported_seq":6242,"reported_epoch":526,"state":"active+clean","last_fresh":"2023-09-23T21:14:16.241209+0000","last_change":"2023-09-23T21:11:08.062606+0000","last_active":"2023-09-23T21:14:16.241209+0000","last_peered":"2023-09-23T21:14:16.241209+0000","last_clean":"2023-09-23T21:14:16.241209+0000","last_became_active":"2023-09-23T21:10:58.780961+0000","last_became_peered":"2023-09-23T21:10:58.780961+0000","last_unstale":"2023-09-23T21:14:16.241209+0000","last_undegraded":"2023-09-23T21:14:16.241209+0000","last_fullsized":"2023-09-23T21:14:16.241209+0000","mapping_epoch":428,"log_start":"0'0","ondisk_log_start":"0'0","created":416,"last_epoch_clean":439,"parent":"0.0","parent_split_bits":0,"last_scrub":"443'3049","last_scrub_stamp":"2023-09-23T21:11:08.062479+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_clean_scrub_stamp":"2023-09-23T21:11:08.062479+0000","objects_scrubbed":1623,"log_size":5101,"log_dups_size":0,"ondisk_log_size":5101,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:56:22.338088+0000","scrub_duration":0.28207626200000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":82051072,"num_objects":1252,"num_object_clones":0,"num_object_copies":3756,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1252,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2053,"num_write_kb":53824,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.2","version":"79'161","reported_seq":805,"reported_epoch":525,"state":"active+clean","last_fresh":"2023-09-23T21:14:03.769417+0000","last_change":"2023-09-23T21:13:12.209907+0000","last_active":"2023-09-23T21:14:03.769417+0000","last_peered":"2023-09-23T21:14:03.769417+0000","last_clean":"2023-09-23T21:14:03.769417+0000","last_became_active":"2023-09-23T21:13:12.205664+0000","last_became_peered":"2023-09-23T21:13:12.205664+0000","last_unstale":"2023-09-23T21:14:03.769417+0000","last_undegraded":"2023-09-23T21:14:03.769417+0000","last_fullsized":"2023-09-23T21:14:03.769417+0000","mapping_epoch":493,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:24.802451+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:24.802451+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:26:16.035788+0000","scrub_duration":0.0090861250000000005,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,4],"acting":[0,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":534,"reported_epoch":525,"state":"active+clean","last_fresh":"2023-09-23T21:14:03.773002+0000","last_change":"2023-09-23T21:13:12.206980+0000","last_active":"2023-09-23T21:14:03.773002+0000","last_peered":"2023-09-23T21:14:03.773002+0000","last_clean":"2023-09-23T21:14:03.773002+0000","last_became_active":"2023-09-23T21:13:12.206650+0000","last_became_peered":"2023-09-23T21:13:12.206650+0000","last_unstale":"2023-09-23T21:14:03.773002+0000","last_undegraded":"2023-09-23T21:14:03.773002+0000","last_fullsized":"2023-09-23T21:14:03.773002+0000","mapping_epoch":493,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:10:01.892936+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":518,"reported_epoch":526,"state":"active+clean","last_fresh":"2023-09-23T21:14:09.555819+0000","last_change":"2023-09-23T21:13:13.407073+0000","last_active":"2023-09-23T21:14:09.555819+0000","last_peered":"2023-09-23T21:14:09.555819+0000","last_clean":"2023-09-23T21:14:09.555819+0000","last_became_active":"2023-09-23T21:13:13.395047+0000","last_became_peered":"2023-09-23T21:13:13.395047+0000","last_unstale":"2023-09-23T21:14:09.555819+0000","last_undegraded":"2023-09-23T21:14:09.555819+0000","last_fullsized":"2023-09-23T21:14:09.555819+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":495,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:04:43.384190+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,5],"acting":[2,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":35,"reported_epoch":525,"state":"active+clean","last_fresh":"2023-09-23T21:14:03.769798+0000","last_change":"2023-09-23T21:13:30.785054+0000","last_active":"2023-09-23T21:14:03.769798+0000","last_peered":"2023-09-23T21:14:03.769798+0000","last_clean":"2023-09-23T21:14:03.769798+0000","last_became_active":"2023-09-23T21:13:30.784701+0000","last_became_peered":"2023-09-23T21:13:30.784701+0000","last_unstale":"2023-09-23T21:14:03.769798+0000","last_undegraded":"2023-09-23T21:14:03.769798+0000","last_fullsized":"2023-09-23T21:14:03.769798+0000","mapping_epoch":501,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:19:20.152252+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.b","version":"527'7656","reported_seq":15163,"reported_epoch":527,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2023-09-23T21:14:18.662004+0000","last_change":"2023-09-23T21:13:59.891994+0000","last_active":"2023-09-23T21:14:18.662004+0000","last_peered":"2023-09-23T21:14:18.662004+0000","last_clean":"2023-09-23T21:11:26.732248+0000","last_became_active":"2023-09-23T21:13:14.937458+0000","last_became_peered":"2023-09-23T21:13:14.937458+0000","last_unstale":"2023-09-23T21:14:18.662004+0000","last_undegraded":"2023-09-23T21:13:59.884123+0000","last_fullsized":"2023-09-23T21:14:18.662004+0000","mapping_epoch":494,"log_start":"426'600","ondisk_log_start":"426'600","created":416,"last_epoch_clean":430,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_clean_scrub_stamp":"2023-09-23T21:10:15.663179+0000","objects_scrubbed":0,"log_size":7056,"log_dups_size":0,"ondisk_log_size":7056,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:36:15.760947+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":159645696,"num_objects":2436,"num_object_clones":0,"num_object_copies":7308,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1337,"num_objects_misplaced":1768,"num_objects_unfound":0,"num_objects_dirty":2436,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7656,"num_write_kb":322944,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3664,"num_bytes_recovered":239927296,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[1,4,7],"avail_no_missing":["1","7"],"object_location_counts":[{"shards":"0,1,7","objects":1760},{"shards":"1,4,7","objects":601}],"blocked_by":[],"up_primary":0,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":311,"reported_epoch":525,"state":"active+clean","last_fresh":"2023-09-23T21:14:03.772352+0000","last_change":"2023-09-23T21:10:26.769764+0000","last_active":"2023-09-23T21:14:03.772352+0000","last_peered":"2023-09-23T21:14:03.772352+0000","last_clean":"2023-09-23T21:14:03.772352+0000","last_became_active":"2023-09-23T21:10:26.769326+0000","last_became_peered":"2023-09-23T21:10:26.769326+0000","last_unstale":"2023-09-23T21:14:03.772352+0000","last_undegraded":"2023-09-23T21:14:03.772352+0000","last_fullsized":"2023-09-23T21:14:03.772352+0000","mapping_epoch":423,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":424,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:57:45.219837+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"79'161","reported_seq":708,"reported_epoch":525,"state":"active+clean","last_fresh":"2023-09-23T21:14:03.768636+0000","last_change":"2023-09-23T21:13:33.766895+0000","last_active":"2023-09-23T21:14:03.768636+0000","last_peered":"2023-09-23T21:14:03.768636+0000","last_clean":"2023-09-23T21:14:03.768636+0000","last_became_active":"2023-09-23T21:13:33.766538+0000","last_became_peered":"2023-09-23T21:13:33.766538+0000","last_unstale":"2023-09-23T21:14:03.768636+0000","last_undegraded":"2023-09-23T21:14:03.768636+0000","last_fullsized":"2023-09-23T21:14:03.768636+0000","mapping_epoch":504,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":505,"parent":"0.0","parent_split_bits":0,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:20.738273+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:20.738273+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:35:35.855987+0000","scrub_duration":0.0088336339999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,2],"acting":[0,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":533,"reported_epoch":525,"state":"active+clean","last_fresh":"2023-09-23T21:14:03.798573+0000","last_change":"2023-09-23T21:11:08.760417+0000","last_active":"2023-09-23T21:14:03.798573+0000","last_peered":"2023-09-23T21:14:03.798573+0000","last_clean":"2023-09-23T21:14:03.798573+0000","last_became_active":"2023-09-23T21:11:08.760150+0000","last_became_peered":"2023-09-23T21:11:08.760150+0000","last_unstale":"2023-09-23T21:14:03.798573+0000","last_undegraded":"2023-09-23T21:14:03.798573+0000","last_fullsized":"2023-09-23T21:14:03.798573+0000","mapping_epoch":443,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":444,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:23:10.849768+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":101,"reported_epoch":526,"state":"active+clean","last_fresh":"2023-09-23T21:14:09.540064+0000","last_change":"2023-09-23T21:13:12.216020+0000","last_active":"2023-09-23T21:14:09.540064+0000","last_peered":"2023-09-23T21:14:09.540064+0000","last_clean":"2023-09-23T21:14:09.540064+0000","last_became_active":"2023-09-23T21:13:12.215180+0000","last_became_peered":"2023-09-23T21:13:12.215180+0000","last_unstale":"2023-09-23T21:14:09.540064+0000","last_undegraded":"2023-09-23T21:14:09.540064+0000","last_fullsized":"2023-09-23T21:14:09.540064+0000","mapping_epoch":493,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:26:18.704684+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.8","version":"526'4967","reported_seq":7359,"reported_epoch":526,"state":"active+clean","last_fresh":"2023-09-23T21:14:15.083660+0000","last_change":"2023-09-23T21:14:03.796930+0000","last_active":"2023-09-23T21:14:15.083660+0000","last_peered":"2023-09-23T21:14:15.083660+0000","last_clean":"2023-09-23T21:14:15.083660+0000","last_became_active":"2023-09-23T21:14:03.786177+0000","last_became_peered":"2023-09-23T21:14:03.786177+0000","last_unstale":"2023-09-23T21:14:15.083660+0000","last_undegraded":"2023-09-23T21:14:15.083660+0000","last_fullsized":"2023-09-23T21:14:15.083660+0000","mapping_epoch":524,"log_start":"0'0","ondisk_log_start":"0'0","created":416,"last_epoch_clean":525,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_clean_scrub_stamp":"2023-09-23T21:10:15.663179+0000","objects_scrubbed":0,"log_size":4967,"log_dups_size":0,"ondisk_log_size":4967,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":79396864,"num_objects":1211,"num_object_clones":0,"num_object_copies":3633,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1211,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3617,"num_write_kb":154528,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":880,"num_bytes_recovered":16908288,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,7],"acting":[0,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.3","version":"79'161","reported_seq":829,"reported_epoch":526,"state":"active+clean","last_fresh":"2023-09-23T21:14:09.540321+0000","last_change":"2023-09-23T21:13:33.771279+0000","last_active":"2023-09-23T21:14:09.540321+0000","last_peered":"2023-09-23T21:14:09.540321+0000","last_clean":"2023-09-23T21:14:09.540321+0000","last_became_active":"2023-09-23T21:13:33.770980+0000","last_became_peered":"2023-09-23T21:13:33.770980+0000","last_unstale":"2023-09-23T21:14:09.540321+0000","last_undegraded":"2023-09-23T21:14:09.540321+0000","last_fullsized":"2023-09-23T21:14:09.540321+0000","mapping_epoch":504,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":505,"parent":"0.0","parent_split_bits":3,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:05:57.832302+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:05:57.832302+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:19:42.309604+0000","scrub_duration":0.010206847999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":342,"reported_epoch":525,"state":"active+clean","last_fresh":"2023-09-23T21:14:03.768901+0000","last_change":"2023-09-23T21:13:13.212775+0000","last_active":"2023-09-23T21:14:03.768901+0000","last_peered":"2023-09-23T21:14:03.768901+0000","last_clean":"2023-09-23T21:14:03.768901+0000","last_became_active":"2023-09-23T21:13:13.212460+0000","last_became_peered":"2023-09-23T21:13:13.212460+0000","last_unstale":"2023-09-23T21:14:03.768901+0000","last_undegraded":"2023-09-23T21:14:03.768901+0000","last_fullsized":"2023-09-23T21:14:03.768901+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":495,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:27:10.853970+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":524,"reported_epoch":525,"state":"active+clean","last_fresh":"2023-09-23T21:14:03.768866+0000","last_change":"2023-09-23T21:13:44.409212+0000","last_active":"2023-09-23T21:14:03.768866+0000","last_peered":"2023-09-23T21:14:03.768866+0000","last_clean":"2023-09-23T21:14:03.768866+0000","last_became_active":"2023-09-23T21:13:44.408958+0000","last_became_peered":"2023-09-23T21:13:44.408958+0000","last_unstale":"2023-09-23T21:14:03.768866+0000","last_undegraded":"2023-09-23T21:14:03.768866+0000","last_fullsized":"2023-09-23T21:14:03.768866+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:03:52.749964+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":35,"reported_epoch":525,"state":"active+clean","last_fresh":"2023-09-23T21:14:03.770544+0000","last_change":"2023-09-23T21:13:30.804379+0000","last_active":"2023-09-23T21:14:03.770544+0000","last_peered":"2023-09-23T21:14:03.770544+0000","last_clean":"2023-09-23T21:14:03.770544+0000","last_became_active":"2023-09-23T21:13:30.799907+0000","last_became_peered":"2023-09-23T21:13:30.799907+0000","last_unstale":"2023-09-23T21:14:03.770544+0000","last_undegraded":"2023-09-23T21:14:03.770544+0000","last_fullsized":"2023-09-23T21:14:03.770544+0000","mapping_epoch":501,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:49:42.859212+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,7],"acting":[0,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.d","version":"527'7540","reported_seq":7700,"reported_epoch":527,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-23T21:14:18.224648+0000","last_change":"2023-09-23T21:13:31.801486+0000","last_active":"2023-09-23T21:14:18.224648+0000","last_peered":"2023-09-23T21:14:18.224648+0000","last_clean":"2023-09-23T21:11:37.659929+0000","last_became_active":"2023-09-23T21:13:31.786626+0000","last_became_peered":"2023-09-23T21:13:31.786626+0000","last_unstale":"2023-09-23T21:14:18.224648+0000","last_undegraded":"2023-09-23T21:13:31.760908+0000","last_fullsized":"2023-09-23T21:14:18.224648+0000","mapping_epoch":502,"log_start":"448'3595","ondisk_log_start":"448'3595","created":416,"last_epoch_clean":417,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_clean_scrub_stamp":"2023-09-23T21:10:15.663179+0000","objects_scrubbed":0,"log_size":3945,"log_dups_size":0,"ondisk_log_size":3945,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:46:02.424493+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":161284120,"num_objects":2462,"num_object_clones":0,"num_object_copies":7386,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1265,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2462,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":7540,"num_write_kb":320001,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,3],"avail_no_missing":["4","3","7"],"object_location_counts":[{"shards":"0,3,4","objects":2383},{"shards":"3,4","objects":79}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.6","version":"79'161","reported_seq":444,"reported_epoch":525,"state":"active+clean","last_fresh":"2023-09-23T21:14:03.770467+0000","last_change":"2023-09-23T21:13:33.770715+0000","last_active":"2023-09-23T21:14:03.770467+0000","last_peered":"2023-09-23T21:14:03.770467+0000","last_clean":"2023-09-23T21:14:03.770467+0000","last_became_active":"2023-09-23T21:13:33.769303+0000","last_became_peered":"2023-09-23T21:13:33.769303+0000","last_unstale":"2023-09-23T21:14:03.770467+0000","last_undegraded":"2023-09-23T21:14:03.770467+0000","last_fullsized":"2023-09-23T21:14:03.770467+0000","mapping_epoch":504,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":505,"parent":"0.0","parent_split_bits":3,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:10:24.268662+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:10:24.268662+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13417,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":11,"num_read_kb":20,"num_write":5,"num_write_kb":55,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":40252,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":152,"reported_epoch":525,"state":"active+clean","last_fresh":"2023-09-23T21:14:03.789998+0000","last_change":"2023-09-23T21:13:30.770826+0000","last_active":"2023-09-23T21:14:03.789998+0000","last_peered":"2023-09-23T21:14:03.789998+0000","last_clean":"2023-09-23T21:14:03.789998+0000","last_became_active":"2023-09-23T21:13:30.770512+0000","last_became_peered":"2023-09-23T21:13:30.770512+0000","last_unstale":"2023-09-23T21:14:03.789998+0000","last_undegraded":"2023-09-23T21:14:03.789998+0000","last_fullsized":"2023-09-23T21:14:03.789998+0000","mapping_epoch":501,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:44:23.470538+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":409,"reported_epoch":526,"state":"active+clean","last_fresh":"2023-09-23T21:14:09.556159+0000","last_change":"2023-09-23T21:13:30.790590+0000","last_active":"2023-09-23T21:14:09.556159+0000","last_peered":"2023-09-23T21:14:09.556159+0000","last_clean":"2023-09-23T21:14:09.556159+0000","last_became_active":"2023-09-23T21:13:30.790253+0000","last_became_peered":"2023-09-23T21:13:30.790253+0000","last_unstale":"2023-09-23T21:14:09.556159+0000","last_undegraded":"2023-09-23T21:14:09.556159+0000","last_fullsized":"2023-09-23T21:14:09.556159+0000","mapping_epoch":501,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:23:41.843250+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,0],"acting":[2,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":471,"reported_epoch":525,"state":"active+clean","last_fresh":"2023-09-23T21:14:03.771329+0000","last_change":"2023-09-23T21:13:12.210546+0000","last_active":"2023-09-23T21:14:03.771329+0000","last_peered":"2023-09-23T21:14:03.771329+0000","last_clean":"2023-09-23T21:14:03.771329+0000","last_became_active":"2023-09-23T21:13:12.205934+0000","last_became_peered":"2023-09-23T21:13:12.205934+0000","last_unstale":"2023-09-23T21:14:03.771329+0000","last_undegraded":"2023-09-23T21:14:03.771329+0000","last_fullsized":"2023-09-23T21:14:03.771329+0000","mapping_epoch":493,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:21:18.299428+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.e","version":"527'7674","reported_seq":14643,"reported_epoch":527,"state":"active+clean","last_fresh":"2023-09-23T21:14:18.157405+0000","last_change":"2023-09-23T21:12:38.623611+0000","last_active":"2023-09-23T21:14:18.157405+0000","last_peered":"2023-09-23T21:14:18.157405+0000","last_clean":"2023-09-23T21:14:18.157405+0000","last_became_active":"2023-09-23T21:12:38.610155+0000","last_became_peered":"2023-09-23T21:12:38.610155+0000","last_unstale":"2023-09-23T21:14:18.157405+0000","last_undegraded":"2023-09-23T21:14:18.157405+0000","last_fullsized":"2023-09-23T21:14:18.157405+0000","mapping_epoch":483,"log_start":"448'3693","ondisk_log_start":"448'3693","created":416,"last_epoch_clean":484,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_clean_scrub_stamp":"2023-09-23T21:10:15.663179+0000","objects_scrubbed":0,"log_size":3981,"log_dups_size":0,"ondisk_log_size":3981,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:19:27.097170+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":167116800,"num_objects":2550,"num_object_clones":0,"num_object_copies":7650,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2550,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7674,"num_write_kb":327168,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3412,"num_bytes_recovered":223608832,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.5","version":"79'161","reported_seq":554,"reported_epoch":526,"state":"active+clean","last_fresh":"2023-09-23T21:14:09.539985+0000","last_change":"2023-09-23T21:13:38.373969+0000","last_active":"2023-09-23T21:14:09.539985+0000","last_peered":"2023-09-23T21:14:09.539985+0000","last_clean":"2023-09-23T21:14:09.539985+0000","last_became_active":"2023-09-23T21:13:38.373633+0000","last_became_peered":"2023-09-23T21:13:38.373633+0000","last_unstale":"2023-09-23T21:14:09.539985+0000","last_undegraded":"2023-09-23T21:14:09.539985+0000","last_fullsized":"2023-09-23T21:14:09.539985+0000","mapping_epoch":507,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":3,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:09:34.892947+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:09:34.892947+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13417,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":11,"num_read_kb":20,"num_write":5,"num_write_kb":55,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":40252,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":66,"reported_epoch":525,"state":"active+clean","last_fresh":"2023-09-23T21:14:03.769956+0000","last_change":"2023-09-23T21:13:12.235743+0000","last_active":"2023-09-23T21:14:03.769956+0000","last_peered":"2023-09-23T21:14:03.769956+0000","last_clean":"2023-09-23T21:14:03.769956+0000","last_became_active":"2023-09-23T21:13:12.216241+0000","last_became_peered":"2023-09-23T21:13:12.216241+0000","last_unstale":"2023-09-23T21:14:03.769956+0000","last_undegraded":"2023-09-23T21:14:03.769956+0000","last_fullsized":"2023-09-23T21:14:03.769956+0000","mapping_epoch":493,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:12:19.090710+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":507,"reported_epoch":526,"state":"active+clean","last_fresh":"2023-09-23T21:14:09.539720+0000","last_change":"2023-09-23T21:13:30.807520+0000","last_active":"2023-09-23T21:14:09.539720+0000","last_peered":"2023-09-23T21:14:09.539720+0000","last_clean":"2023-09-23T21:14:09.539720+0000","last_became_active":"2023-09-23T21:13:30.807222+0000","last_became_peered":"2023-09-23T21:13:30.807222+0000","last_unstale":"2023-09-23T21:14:09.539720+0000","last_undegraded":"2023-09-23T21:14:09.539720+0000","last_fullsized":"2023-09-23T21:14:09.539720+0000","mapping_epoch":501,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:40:31.989749+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":367,"reported_epoch":525,"state":"active+clean","last_fresh":"2023-09-23T21:14:03.784566+0000","last_change":"2023-09-23T21:13:30.794095+0000","last_active":"2023-09-23T21:14:03.784566+0000","last_peered":"2023-09-23T21:14:03.784566+0000","last_clean":"2023-09-23T21:14:03.784566+0000","last_became_active":"2023-09-23T21:13:30.793791+0000","last_became_peered":"2023-09-23T21:13:30.793791+0000","last_unstale":"2023-09-23T21:14:03.784566+0000","last_undegraded":"2023-09-23T21:14:03.784566+0000","last_fullsized":"2023-09-23T21:14:03.784566+0000","mapping_epoch":501,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:36:28.707362+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.f","version":"526'7527","reported_seq":7728,"reported_epoch":526,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T21:14:14.136028+0000","last_change":"2023-09-23T21:13:59.874192+0000","last_active":"2023-09-23T21:14:14.136028+0000","last_peered":"2023-09-23T21:14:14.136028+0000","last_clean":"2023-09-23T21:11:07.652290+0000","last_became_active":"2023-09-23T21:11:09.897463+0000","last_became_peered":"2023-09-23T21:11:09.897463+0000","last_unstale":"2023-09-23T21:14:14.136028+0000","last_undegraded":"2023-09-23T21:11:27.781677+0000","last_fullsized":"2023-09-23T21:11:09.761652+0000","mapping_epoch":444,"log_start":"448'3809","ondisk_log_start":"448'3809","created":416,"last_epoch_clean":417,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_clean_scrub_stamp":"2023-09-23T21:10:15.663179+0000","objects_scrubbed":0,"log_size":3718,"log_dups_size":0,"ondisk_log_size":3718,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:42:51.588441+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":169541632,"num_objects":2587,"num_object_clones":0,"num_object_copies":7761,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3295,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2587,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7527,"num_write_kb":323648,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":32,"num_bytes_recovered":2097152,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2],"avail_no_missing":["3","1","2"],"object_location_counts":[{"shards":"1,2,3","objects":857},{"shards":"2,3","objects":-708}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.4","version":"79'161","reported_seq":896,"reported_epoch":525,"state":"active+clean","last_fresh":"2023-09-23T21:14:03.785609+0000","last_change":"2023-09-23T21:10:29.807785+0000","last_active":"2023-09-23T21:14:03.785609+0000","last_peered":"2023-09-23T21:14:03.785609+0000","last_clean":"2023-09-23T21:14:03.785609+0000","last_became_active":"2023-09-23T21:10:29.807366+0000","last_became_peered":"2023-09-23T21:10:29.807366+0000","last_unstale":"2023-09-23T21:14:03.785609+0000","last_undegraded":"2023-09-23T21:14:03.785609+0000","last_fullsized":"2023-09-23T21:14:03.785609+0000","mapping_epoch":426,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":427,"parent":"0.0","parent_split_bits":3,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:20.012764+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:20.012764+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:24:45.763834+0000","scrub_duration":0.0045251919999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":464,"reported_epoch":525,"state":"active+clean","last_fresh":"2023-09-23T21:14:03.773144+0000","last_change":"2023-09-23T21:10:26.768364+0000","last_active":"2023-09-23T21:14:03.773144+0000","last_peered":"2023-09-23T21:14:03.773144+0000","last_clean":"2023-09-23T21:14:03.773144+0000","last_became_active":"2023-09-23T21:10:26.768052+0000","last_became_peered":"2023-09-23T21:10:26.768052+0000","last_unstale":"2023-09-23T21:14:03.773144+0000","last_undegraded":"2023-09-23T21:14:03.773144+0000","last_fullsized":"2023-09-23T21:14:03.773144+0000","mapping_epoch":423,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":424,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:22:42.201200+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,3],"acting":[2,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.6","version":"70'1","reported_seq":496,"reported_epoch":525,"state":"active+clean","last_fresh":"2023-09-23T21:14:03.770655+0000","last_change":"2023-09-23T21:13:30.793290+0000","last_active":"2023-09-23T21:14:03.770655+0000","last_peered":"2023-09-23T21:14:03.770655+0000","last_clean":"2023-09-23T21:14:03.770655+0000","last_became_active":"2023-09-23T21:13:30.786079+0000","last_became_peered":"2023-09-23T21:13:30.786079+0000","last_unstale":"2023-09-23T21:14:03.770655+0000","last_undegraded":"2023-09-23T21:14:03.770655+0000","last_fullsized":"2023-09-23T21:14:03.770655+0000","mapping_epoch":501,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:34:43.281095+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":101,"reported_epoch":526,"state":"active+clean","last_fresh":"2023-09-23T21:14:09.538729+0000","last_change":"2023-09-23T21:13:12.216895+0000","last_active":"2023-09-23T21:14:09.538729+0000","last_peered":"2023-09-23T21:14:09.538729+0000","last_clean":"2023-09-23T21:14:09.538729+0000","last_became_active":"2023-09-23T21:13:12.216654+0000","last_became_peered":"2023-09-23T21:13:12.216654+0000","last_unstale":"2023-09-23T21:14:09.538729+0000","last_undegraded":"2023-09-23T21:14:09.538729+0000","last_fullsized":"2023-09-23T21:14:09.538729+0000","mapping_epoch":493,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:02:40.430096+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.19","version":"79'161","reported_seq":678,"reported_epoch":525,"state":"active+clean","last_fresh":"2023-09-23T21:14:03.771450+0000","last_change":"2023-09-23T21:12:30.442914+0000","last_active":"2023-09-23T21:14:03.771450+0000","last_peered":"2023-09-23T21:14:03.771450+0000","last_clean":"2023-09-23T21:14:03.771450+0000","last_became_active":"2023-09-23T21:12:30.442578+0000","last_became_peered":"2023-09-23T21:12:30.442578+0000","last_unstale":"2023-09-23T21:14:03.771450+0000","last_undegraded":"2023-09-23T21:14:03.771450+0000","last_fullsized":"2023-09-23T21:14:03.771450+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":479,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:10:37.696951+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:10:37.696951+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.001386504,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,3],"acting":[2,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"10.12","version":"527'5135","reported_seq":6624,"reported_epoch":527,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T21:14:18.212678+0000","last_change":"2023-09-23T21:13:13.419184+0000","last_active":"2023-09-23T21:14:18.212678+0000","last_peered":"2023-09-23T21:14:18.212678+0000","last_clean":"2023-09-23T21:11:45.774771+0000","last_became_active":"2023-09-23T21:13:13.409281+0000","last_became_peered":"2023-09-23T21:13:13.409281+0000","last_unstale":"2023-09-23T21:14:18.212678+0000","last_undegraded":"2023-09-23T21:13:13.284063+0000","last_fullsized":"2023-09-23T21:13:13.191117+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":438,"last_epoch_clean":439,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_clean_scrub_stamp":"2023-09-23T21:10:15.663179+0000","objects_scrubbed":0,"log_size":5135,"log_dups_size":0,"ondisk_log_size":5135,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":77201408,"num_objects":1178,"num_object_clones":0,"num_object_copies":3534,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2474,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1178,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3758,"num_write_kb":157952,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":664,"num_bytes_recovered":43515904,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,3],"avail_no_missing":["4","1"],"object_location_counts":[{"shards":"0,1,4","objects":1140},{"shards":"1,4","objects":41},{"shards":"3,4","objects":-3}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":35,"reported_epoch":525,"state":"active+clean","last_fresh":"2023-09-23T21:14:03.771269+0000","last_change":"2023-09-23T21:13:30.791297+0000","last_active":"2023-09-23T21:14:03.771269+0000","last_peered":"2023-09-23T21:14:03.771269+0000","last_clean":"2023-09-23T21:14:03.771269+0000","last_became_active":"2023-09-23T21:13:30.788721+0000","last_became_peered":"2023-09-23T21:13:30.788721+0000","last_unstale":"2023-09-23T21:14:03.771269+0000","last_undegraded":"2023-09-23T21:14:03.771269+0000","last_fullsized":"2023-09-23T21:14:03.771269+0000","mapping_epoch":501,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:31:18.147883+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":29,"reported_epoch":525,"state":"active+clean","last_fresh":"2023-09-23T21:14:03.772321+0000","last_change":"2023-09-23T21:13:38.373814+0000","last_active":"2023-09-23T21:14:03.772321+0000","last_peered":"2023-09-23T21:14:03.772321+0000","last_clean":"2023-09-23T21:14:03.772321+0000","last_became_active":"2023-09-23T21:13:38.373458+0000","last_became_peered":"2023-09-23T21:13:38.373458+0000","last_unstale":"2023-09-23T21:14:03.772321+0000","last_undegraded":"2023-09-23T21:14:03.772321+0000","last_fullsized":"2023-09-23T21:14:03.772321+0000","mapping_epoch":507,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:23:08.288401+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.1e","version":"79'161","reported_seq":694,"reported_epoch":526,"state":"active+clean","last_fresh":"2023-09-23T21:14:09.538966+0000","last_change":"2023-09-23T21:12:30.448761+0000","last_active":"2023-09-23T21:14:09.538966+0000","last_peered":"2023-09-23T21:14:09.538966+0000","last_clean":"2023-09-23T21:14:09.538966+0000","last_became_active":"2023-09-23T21:12:30.448043+0000","last_became_peered":"2023-09-23T21:12:30.448043+0000","last_unstale":"2023-09-23T21:14:09.538966+0000","last_undegraded":"2023-09-23T21:14:09.538966+0000","last_fullsized":"2023-09-23T21:14:09.538966+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":479,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:10:39.567232+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:10:39.567232+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.005229694,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.15","version":"526'5005","reported_seq":5148,"reported_epoch":526,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T21:14:15.938698+0000","last_change":"2023-09-23T21:13:13.436807+0000","last_active":"2023-09-23T21:14:15.938698+0000","last_peered":"2023-09-23T21:14:15.938698+0000","last_clean":"2023-09-23T21:11:45.765730+0000","last_became_active":"2023-09-23T21:13:13.429206+0000","last_became_peered":"2023-09-23T21:13:13.429206+0000","last_unstale":"2023-09-23T21:14:15.938698+0000","last_undegraded":"2023-09-23T21:13:13.246273+0000","last_fullsized":"2023-09-23T21:13:13.203825+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":438,"last_epoch_clean":439,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_clean_scrub_stamp":"2023-09-23T21:10:15.663179+0000","objects_scrubbed":0,"log_size":5005,"log_dups_size":0,"ondisk_log_size":5005,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":83689472,"num_objects":1277,"num_object_clones":0,"num_object_copies":3831,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":643,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1277,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3671,"num_write_kb":158336,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,3],"acting":[2,3],"avail_no_missing":["2","3"],"object_location_counts":[{"shards":"2,3","objects":1241}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":102,"reported_epoch":526,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-23T21:14:09.539424+0000","last_change":"2023-09-23T21:12:14.313202+0000","last_active":"2023-09-23T21:14:09.539424+0000","last_peered":"2023-09-23T21:14:09.539424+0000","last_clean":"2023-09-23T21:12:09.299355+0000","last_became_active":"2023-09-23T21:12:14.209408+0000","last_became_peered":"2023-09-23T21:12:14.209408+0000","last_unstale":"2023-09-23T21:14:09.539424+0000","last_undegraded":"2023-09-23T21:12:12.933857+0000","last_fullsized":"2023-09-23T21:14:09.539424+0000","mapping_epoch":470,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":444,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:45:40.142413+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,7],"acting":[4,5,7],"avail_no_missing":["4","1","5"],"object_location_counts":[{"shards":"1,4,5","objects":1}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":91,"reported_epoch":526,"state":"active+clean","last_fresh":"2023-09-23T21:14:09.540665+0000","last_change":"2023-09-23T21:12:10.770474+0000","last_active":"2023-09-23T21:14:09.540665+0000","last_peered":"2023-09-23T21:14:09.540665+0000","last_clean":"2023-09-23T21:14:09.540665+0000","last_became_active":"2023-09-23T21:12:10.770184+0000","last_became_peered":"2023-09-23T21:12:10.770184+0000","last_unstale":"2023-09-23T21:14:09.540665+0000","last_undegraded":"2023-09-23T21:14:09.540665+0000","last_fullsized":"2023-09-23T21:14:09.540665+0000","mapping_epoch":470,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:05:09.407151+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.14","version":"526'4956","reported_seq":11054,"reported_epoch":526,"state":"active+clean","last_fresh":"2023-09-23T21:14:14.846457+0000","last_change":"2023-09-23T21:14:01.515957+0000","last_active":"2023-09-23T21:14:14.846457+0000","last_peered":"2023-09-23T21:14:14.846457+0000","last_clean":"2023-09-23T21:14:14.846457+0000","last_became_active":"2023-09-23T21:14:01.506428+0000","last_became_peered":"2023-09-23T21:14:01.506428+0000","last_unstale":"2023-09-23T21:14:14.846457+0000","last_undegraded":"2023-09-23T21:14:14.846457+0000","last_fullsized":"2023-09-23T21:14:14.846457+0000","mapping_epoch":522,"log_start":"0'0","ondisk_log_start":"0'0","created":438,"last_epoch_clean":523,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_clean_scrub_stamp":"2023-09-23T21:10:15.663179+0000","objects_scrubbed":0,"log_size":4956,"log_dups_size":0,"ondisk_log_size":4956,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":84410368,"num_objects":1288,"num_object_clones":0,"num_object_copies":3864,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1288,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3622,"num_write_kb":157120,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2660,"num_bytes_recovered":126156800,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":315,"reported_epoch":525,"state":"active+clean","last_fresh":"2023-09-23T21:14:03.772979+0000","last_change":"2023-09-23T21:12:10.771160+0000","last_active":"2023-09-23T21:14:03.772979+0000","last_peered":"2023-09-23T21:14:03.772979+0000","last_clean":"2023-09-23T21:14:03.772979+0000","last_became_active":"2023-09-23T21:12:10.757808+0000","last_became_peered":"2023-09-23T21:12:10.757808+0000","last_unstale":"2023-09-23T21:14:03.772979+0000","last_undegraded":"2023-09-23T21:14:03.772979+0000","last_fullsized":"2023-09-23T21:14:03.772979+0000","mapping_epoch":470,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:30:50.987139+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":101,"reported_epoch":526,"state":"active+clean","last_fresh":"2023-09-23T21:14:09.540165+0000","last_change":"2023-09-23T21:13:12.216602+0000","last_active":"2023-09-23T21:14:09.540165+0000","last_peered":"2023-09-23T21:14:09.540165+0000","last_clean":"2023-09-23T21:14:09.540165+0000","last_became_active":"2023-09-23T21:13:12.216204+0000","last_became_peered":"2023-09-23T21:13:12.216204+0000","last_unstale":"2023-09-23T21:14:09.540165+0000","last_undegraded":"2023-09-23T21:14:09.540165+0000","last_fullsized":"2023-09-23T21:14:09.540165+0000","mapping_epoch":493,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:23:37.043390+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1c","version":"79'161","reported_seq":891,"reported_epoch":525,"state":"active+clean","last_fresh":"2023-09-23T21:14:03.784737+0000","last_change":"2023-09-23T21:13:13.402773+0000","last_active":"2023-09-23T21:14:03.784737+0000","last_peered":"2023-09-23T21:14:03.784737+0000","last_clean":"2023-09-23T21:14:03.784737+0000","last_became_active":"2023-09-23T21:13:13.402025+0000","last_became_peered":"2023-09-23T21:13:13.402025+0000","last_unstale":"2023-09-23T21:14:03.784737+0000","last_undegraded":"2023-09-23T21:14:03.784737+0000","last_fullsized":"2023-09-23T21:14:03.784737+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":479,"last_epoch_clean":495,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:22.040907+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:22.040907+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.013021176000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":163840,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":163840,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.17","version":"526'4978","reported_seq":9361,"reported_epoch":526,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T21:14:14.120674+0000","last_change":"2023-09-23T21:13:39.447969+0000","last_active":"2023-09-23T21:14:14.120674+0000","last_peered":"2023-09-23T21:14:14.120674+0000","last_clean":"2023-09-23T21:13:37.065694+0000","last_became_active":"2023-09-23T21:13:39.439739+0000","last_became_peered":"2023-09-23T21:13:39.439739+0000","last_unstale":"2023-09-23T21:14:14.120674+0000","last_undegraded":"2023-09-23T21:13:39.411853+0000","last_fullsized":"2023-09-23T21:13:39.404657+0000","mapping_epoch":508,"log_start":"0'0","ondisk_log_start":"0'0","created":438,"last_epoch_clean":487,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_clean_scrub_stamp":"2023-09-23T21:10:15.663179+0000","objects_scrubbed":0,"log_size":4978,"log_dups_size":0,"ondisk_log_size":4978,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":86671360,"num_objects":1323,"num_object_clones":0,"num_object_copies":3969,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1109,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1323,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3613,"num_write_kb":157920,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2112,"num_bytes_recovered":116719616,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,2],"acting":[3,5],"avail_no_missing":["3","4","5"],"object_location_counts":[{"shards":"3,4,5","objects":311},{"shards":"3,5","objects":1012}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":114,"reported_epoch":525,"state":"active+clean","last_fresh":"2023-09-23T21:14:03.771895+0000","last_change":"2023-09-23T21:12:10.779066+0000","last_active":"2023-09-23T21:14:03.771895+0000","last_peered":"2023-09-23T21:14:03.771895+0000","last_clean":"2023-09-23T21:14:03.771895+0000","last_became_active":"2023-09-23T21:12:10.778691+0000","last_became_peered":"2023-09-23T21:12:10.778691+0000","last_unstale":"2023-09-23T21:14:03.771895+0000","last_undegraded":"2023-09-23T21:14:03.771895+0000","last_fullsized":"2023-09-23T21:14:03.771895+0000","mapping_epoch":470,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:07:22.043268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,7],"acting":[2,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":193,"reported_epoch":525,"state":"active+clean","last_fresh":"2023-09-23T21:14:03.773048+0000","last_change":"2023-09-23T21:13:12.200767+0000","last_active":"2023-09-23T21:14:03.773048+0000","last_peered":"2023-09-23T21:14:03.773048+0000","last_clean":"2023-09-23T21:14:03.773048+0000","last_became_active":"2023-09-23T21:13:12.200435+0000","last_became_peered":"2023-09-23T21:13:12.200435+0000","last_unstale":"2023-09-23T21:14:03.773048+0000","last_undegraded":"2023-09-23T21:14:03.773048+0000","last_fullsized":"2023-09-23T21:14:03.773048+0000","mapping_epoch":493,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:15:44.937886+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.1d","version":"79'161","reported_seq":678,"reported_epoch":525,"state":"active+clean","last_fresh":"2023-09-23T21:14:03.789709+0000","last_change":"2023-09-23T21:12:30.444331+0000","last_active":"2023-09-23T21:14:03.789709+0000","last_peered":"2023-09-23T21:14:03.789709+0000","last_clean":"2023-09-23T21:14:03.789709+0000","last_became_active":"2023-09-23T21:12:30.444001+0000","last_became_peered":"2023-09-23T21:12:30.444001+0000","last_unstale":"2023-09-23T21:14:03.789709+0000","last_undegraded":"2023-09-23T21:14:03.789709+0000","last_fullsized":"2023-09-23T21:14:03.789709+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":479,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:09:36.858077+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:09:36.858077+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0024987500000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"10.16","version":"527'5184","reported_seq":8692,"reported_epoch":527,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T21:14:18.136407+0000","last_change":"2023-09-23T21:13:14.424843+0000","last_active":"2023-09-23T21:14:18.136407+0000","last_peered":"2023-09-23T21:14:18.136407+0000","last_clean":"2023-09-23T21:11:45.652255+0000","last_became_active":"2023-09-23T21:13:14.414710+0000","last_became_peered":"2023-09-23T21:13:14.414710+0000","last_unstale":"2023-09-23T21:14:18.136407+0000","last_undegraded":"2023-09-23T21:13:13.211957+0000","last_fullsized":"2023-09-23T21:13:13.188859+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":438,"last_epoch_clean":460,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_clean_scrub_stamp":"2023-09-23T21:10:15.663179+0000","objects_scrubbed":0,"log_size":5184,"log_dups_size":0,"ondisk_log_size":5184,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":80478208,"num_objects":1228,"num_object_clones":0,"num_object_copies":3684,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":663,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1228,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3778,"num_write_kb":160192,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1670,"num_bytes_recovered":109445120,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,4],"acting":[4,5],"avail_no_missing":["4","5"],"object_location_counts":[{"shards":"1,4,5","objects":33},{"shards":"4,5","objects":1195}],"blocked_by":[],"up_primary":0,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":52,"reported_epoch":525,"state":"active+clean","last_fresh":"2023-09-23T21:14:03.769635+0000","last_change":"2023-09-23T21:13:38.380574+0000","last_active":"2023-09-23T21:14:03.769635+0000","last_peered":"2023-09-23T21:14:03.769635+0000","last_clean":"2023-09-23T21:14:03.769635+0000","last_became_active":"2023-09-23T21:13:38.373252+0000","last_became_peered":"2023-09-23T21:13:38.373252+0000","last_unstale":"2023-09-23T21:14:03.769635+0000","last_undegraded":"2023-09-23T21:14:03.769635+0000","last_fullsized":"2023-09-23T21:14:03.769635+0000","mapping_epoch":507,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:58:14.788031+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":11,"num_bytes_recovered":3324,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":101,"reported_epoch":526,"state":"active+clean","last_fresh":"2023-09-23T21:14:09.568942+0000","last_change":"2023-09-23T21:12:10.793610+0000","last_active":"2023-09-23T21:14:09.568942+0000","last_peered":"2023-09-23T21:14:09.568942+0000","last_clean":"2023-09-23T21:14:09.568942+0000","last_became_active":"2023-09-23T21:12:10.793081+0000","last_became_peered":"2023-09-23T21:12:10.793081+0000","last_unstale":"2023-09-23T21:14:09.568942+0000","last_undegraded":"2023-09-23T21:14:09.568942+0000","last_fullsized":"2023-09-23T21:14:09.568942+0000","mapping_epoch":470,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:34:36.291550+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":10,"num_pg":26,"stat_sum":{"num_bytes":2642149400,"num_objects":40319,"num_object_clones":0,"num_object_copies":120957,"num_objects_missing_on_primary":2882,"num_objects_missing":2882,"num_objects_degraded":24523,"num_objects_misplaced":4517,"num_objects_unfound":0,"num_objects_dirty":40319,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":107055,"num_write_kb":4309313,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":37589,"num_bytes_recovered":2253881344,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":10595934208,"data_stored":10595926064,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":128869,"ondisk_log_size":128869,"up":78,"acting":62,"num_store_stats":7},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":59,"num_bytes_recovered":8898,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":126976,"data_stored":22458,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":14322,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":31,"stat_sum":{"num_bytes":697708,"num_objects":3,"num_object_clones":0,"num_object_copies":9,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3,"num_whiteouts":0,"num_read":89,"num_read_kb":165,"num_write":44,"num_write_kb":440,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":649699,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2109440,"data_stored":2098784,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4991,"ondisk_log_size":4991,"up":93,"acting":93,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201686,"num_pgs":69,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1454720,"kb_used_data":964152,"kb_used_omap":0,"kb_used_meta":490560,"kb_avail":92290432,"statfs":{"total":95995035648,"available":94505402368,"internally_reserved":0,"allocated":987291648,"data_stored":985770157,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":502333440},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:14:13 2023","interfaces":[{"interface":"back","average":{"1min":2.4260000000000002,"5min":2.4260000000000002,"15min":2.4260000000000002},"min":{"1min":0.23400000000000001,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":37.207999999999998,"5min":37.207999999999998,"15min":37.207999999999998},"last":0.63},{"interface":"front","average":{"1min":2.234,"5min":2.234,"15min":2.234},"min":{"1min":0.28599999999999998,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":37.122,"5min":37.122,"15min":37.122},"last":0.29699999999999999}]},{"osd":1,"last update":"Sat Sep 23 21:14:09 2023","interfaces":[{"interface":"back","average":{"1min":0.94299999999999995,"5min":1.2310000000000001,"15min":2.04},"min":{"1min":0.30399999999999999,"5min":0.27100000000000002,"15min":0.26700000000000002},"max":{"1min":7.5369999999999999,"5min":14.034000000000001,"15min":14.034000000000001},"last":0.79100000000000004},{"interface":"front","average":{"1min":1.0469999999999999,"5min":1.4339999999999999,"15min":1.944},"min":{"1min":0.30499999999999999,"5min":0.29199999999999998,"15min":0.26300000000000001},"max":{"1min":5.6870000000000003,"5min":11.226000000000001,"15min":11.226000000000001},"last":2.7490000000000001}]},{"osd":2,"last update":"Sat Sep 23 21:13:29 2023","interfaces":[{"interface":"back","average":{"1min":1.254,"5min":1.3839999999999999,"15min":1.9119999999999999},"min":{"1min":0.26400000000000001,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":6.4219999999999997,"5min":6.5469999999999997,"15min":14.478999999999999},"last":0.76400000000000001},{"interface":"front","average":{"1min":1.298,"5min":1.5129999999999999,"15min":1.819},"min":{"1min":0.26700000000000002,"5min":0.26700000000000002,"15min":0.26500000000000001},"max":{"1min":9.1839999999999993,"5min":11.198,"15min":14.135},"last":0.68000000000000005}]},{"osd":3,"last update":"Sat Sep 23 21:13:57 2023","interfaces":[{"interface":"back","average":{"1min":1.022,"5min":1.296,"15min":2.0590000000000002},"min":{"1min":0.33000000000000002,"5min":0.316,"15min":0.29099999999999998},"max":{"1min":6.4930000000000003,"5min":7.5190000000000001,"15min":31.786000000000001},"last":0.68899999999999995},{"interface":"front","average":{"1min":1.1160000000000001,"5min":1.3859999999999999,"15min":2.0409999999999999},"min":{"1min":0.37,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":5.9329999999999998,"5min":11.244999999999999,"15min":32.429000000000002},"last":0.60399999999999998}]},{"osd":4,"last update":"Sat Sep 23 21:13:53 2023","interfaces":[{"interface":"back","average":{"1min":1.5980000000000001,"5min":1.5209999999999999,"15min":1.097},"min":{"1min":0.28999999999999998,"5min":0.255,"15min":0.248},"max":{"1min":7.6609999999999996,"5min":11.297000000000001,"15min":11.297000000000001},"last":2.819},{"interface":"front","average":{"1min":1.387,"5min":1.1679999999999999,"15min":1.0669999999999999},"min":{"1min":0.314,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":8.6059999999999999,"5min":11.093999999999999,"15min":12.752000000000001},"last":0.87}]},{"osd":5,"last update":"Sat Sep 23 21:13:53 2023","interfaces":[{"interface":"back","average":{"1min":1.4670000000000001,"5min":1.0549999999999999,"15min":1.014},"min":{"1min":0.29899999999999999,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":7.5910000000000002,"5min":9.4789999999999992,"15min":9.4789999999999992},"last":0.72999999999999998},{"interface":"front","average":{"1min":1.4950000000000001,"5min":1.3680000000000001,"15min":1.099},"min":{"1min":0.39700000000000002,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":7.6929999999999996,"5min":11.257,"15min":11.257},"last":0.70799999999999996}]},{"osd":6,"last update":"Sat Sep 23 21:14:09 2023","interfaces":[{"interface":"back","average":{"1min":0.79600000000000004,"5min":1.1819999999999999,"15min":1.038},"min":{"1min":0.39800000000000002,"5min":0.30099999999999999,"15min":0.28899999999999998},"max":{"1min":1.667,"5min":10.941000000000001,"15min":10.941000000000001},"last":0.65900000000000003},{"interface":"front","average":{"1min":0.99299999999999999,"5min":1.1020000000000001,"15min":1.1259999999999999},"min":{"1min":0.41699999999999998,"5min":0.33100000000000002,"15min":0.32300000000000001},"max":{"1min":2.4969999999999999,"5min":8.923,"15min":12.715},"last":1.5649999999999999}]}]},{"osd":6,"up_from":0,"seq":704374636769,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":1,"up_from":0,"seq":1503238553702,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":0,"up_from":493,"seq":2117418876942,"num_pgs":75,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2629660,"kb_used_data":2493524,"kb_used_omap":36,"kb_used_meta":136091,"kb_avail":91115492,"statfs":{"total":95995035648,"available":93302263808,"internally_reserved":0,"allocated":2553368576,"data_stored":2550902362,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":37336,"internal_metadata":139357736},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 21:14:12 2023","interfaces":[{"interface":"back","average":{"1min":0.75600000000000001,"5min":0.75600000000000001,"15min":0.75600000000000001},"min":{"1min":0.23000000000000001,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":4.9000000000000004,"5min":4.9000000000000004,"15min":4.9000000000000004},"last":0.83899999999999997},{"interface":"front","average":{"1min":0.77100000000000002,"5min":0.77100000000000002,"15min":0.77100000000000002},"min":{"1min":0.24299999999999999,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":3.444,"5min":3.444,"15min":3.444},"last":0.56299999999999994}]},{"osd":2,"last update":"Sat Sep 23 21:14:12 2023","interfaces":[{"interface":"back","average":{"1min":0.65400000000000003,"5min":0.65400000000000003,"15min":0.65400000000000003},"min":{"1min":0.26900000000000002,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":4.8630000000000004,"5min":4.8630000000000004,"15min":4.8630000000000004},"last":0.88500000000000001},{"interface":"front","average":{"1min":0.96799999999999997,"5min":0.96799999999999997,"15min":0.96799999999999997},"min":{"1min":0.29299999999999998,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":4.649,"5min":4.649,"15min":4.649},"last":0.60299999999999998}]},{"osd":3,"last update":"Sat Sep 23 21:14:12 2023","interfaces":[{"interface":"back","average":{"1min":0.86799999999999999,"5min":0.86799999999999999,"15min":0.86799999999999999},"min":{"1min":0.23400000000000001,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":4.5110000000000001,"5min":4.5110000000000001,"15min":4.5110000000000001},"last":1.1339999999999999},{"interface":"front","average":{"1min":1.1180000000000001,"5min":1.1180000000000001,"15min":1.1180000000000001},"min":{"1min":0.26200000000000001,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":3.7200000000000002,"5min":3.7200000000000002,"15min":3.7200000000000002},"last":3.7200000000000002}]},{"osd":4,"last update":"Sat Sep 23 21:14:12 2023","interfaces":[{"interface":"back","average":{"1min":1.093,"5min":1.093,"15min":1.093},"min":{"1min":0.30199999999999999,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":3.7799999999999998,"5min":3.7799999999999998,"15min":3.7799999999999998},"last":3.7799999999999998},{"interface":"front","average":{"1min":0.996,"5min":0.996,"15min":0.996},"min":{"1min":0.317,"5min":0.317,"15min":0.317},"max":{"1min":5.093,"5min":5.093,"15min":5.093},"last":0.93100000000000005}]},{"osd":5,"last update":"Sat Sep 23 21:14:12 2023","interfaces":[{"interface":"back","average":{"1min":1.236,"5min":1.236,"15min":1.236},"min":{"1min":0.33200000000000002,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":4.5640000000000001,"5min":4.5640000000000001,"15min":4.5640000000000001},"last":0.63800000000000001},{"interface":"front","average":{"1min":1.0640000000000001,"5min":1.0640000000000001,"15min":1.0640000000000001},"min":{"1min":0.33200000000000002,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":4.5599999999999996,"5min":4.5599999999999996,"15min":4.5599999999999996},"last":3.8159999999999998}]},{"osd":6,"last update":"Sat Sep 23 21:14:12 2023","interfaces":[{"interface":"back","average":{"1min":1.127,"5min":1.127,"15min":1.127},"min":{"1min":0.38100000000000001,"5min":0.38100000000000001,"15min":0.38100000000000001},"max":{"1min":6.2750000000000004,"5min":6.2750000000000004,"15min":6.2750000000000004},"last":0.95999999999999996},{"interface":"front","average":{"1min":1.3080000000000001,"5min":1.3080000000000001,"15min":1.3080000000000001},"min":{"1min":0.35599999999999998,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":5.0860000000000003,"5min":5.0860000000000003,"15min":5.0860000000000003},"last":0.66900000000000004}]},{"osd":7,"last update":"Sat Sep 23 21:14:12 2023","interfaces":[{"interface":"back","average":{"1min":1.2090000000000001,"5min":1.2090000000000001,"15min":1.2090000000000001},"min":{"1min":0.41099999999999998,"5min":0.41099999999999998,"15min":0.41099999999999998},"max":{"1min":5.1539999999999999,"5min":5.1539999999999999,"15min":5.1539999999999999},"last":5.1539999999999999},{"interface":"front","average":{"1min":1.1579999999999999,"5min":1.1579999999999999,"15min":1.1579999999999999},"min":{"1min":0.441,"5min":0.441,"15min":0.441},"max":{"1min":5.1289999999999996,"5min":5.1289999999999996,"15min":5.1289999999999996},"last":1.0640000000000001}]}]},{"osd":2,"up_from":93,"seq":399431958814,"num_pgs":69,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2344604,"kb_used_data":1583124,"kb_used_omap":81,"kb_used_meta":761390,"kb_avail":91400548,"statfs":{"total":95995035648,"available":93594161152,"internally_reserved":0,"allocated":1621118976,"data_stored":1618651803,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":83750,"internal_metadata":779663578},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:14:15 2023","interfaces":[{"interface":"back","average":{"1min":0.83299999999999996,"5min":0.83299999999999996,"15min":0.83299999999999996},"min":{"1min":0.20899999999999999,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":3.7290000000000001,"5min":3.7290000000000001,"15min":3.7290000000000001},"last":2.1949999999999998},{"interface":"front","average":{"1min":1.665,"5min":1.665,"15min":1.665},"min":{"1min":0.23499999999999999,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":18.111999999999998,"5min":18.111999999999998,"15min":18.111999999999998},"last":0.29299999999999998}]},{"osd":1,"last update":"Sat Sep 23 21:14:15 2023","interfaces":[{"interface":"back","average":{"1min":0.95099999999999996,"5min":1.474,"15min":2.5339999999999998},"min":{"1min":0.23999999999999999,"5min":0.188,"15min":0.188},"max":{"1min":6.0499999999999998,"5min":12.484999999999999,"15min":34.396999999999998},"last":0.27400000000000002},{"interface":"front","average":{"1min":1.597,"5min":1.3129999999999999,"15min":2.782},"min":{"1min":0.29899999999999999,"5min":0.217,"15min":0.217},"max":{"1min":18.081,"5min":18.081,"15min":34.462000000000003},"last":0.44400000000000001}]},{"osd":3,"last update":"Sat Sep 23 21:13:56 2023","interfaces":[{"interface":"back","average":{"1min":0.495,"5min":1.5429999999999999,"15min":2.6419999999999999},"min":{"1min":0.20599999999999999,"5min":0.20599999999999999,"15min":0.20599999999999999},"max":{"1min":1.679,"5min":12.951000000000001,"15min":41.057000000000002},"last":2.4340000000000002},{"interface":"front","average":{"1min":0.749,"5min":1.661,"15min":2.915},"min":{"1min":0.28499999999999998,"5min":0.21299999999999999,"15min":0.21299999999999999},"max":{"1min":3.6989999999999998,"5min":13.159000000000001,"15min":41.012},"last":2.4529999999999998}]},{"osd":4,"last update":"Sat Sep 23 21:14:15 2023","interfaces":[{"interface":"back","average":{"1min":0.64800000000000002,"5min":1.256,"15min":1.4219999999999999},"min":{"1min":0.34100000000000003,"5min":0.249,"15min":0.249},"max":{"1min":1.7210000000000001,"5min":10.266999999999999,"15min":10.266999999999999},"last":0.34100000000000003},{"interface":"front","average":{"1min":1.0029999999999999,"5min":1.456,"15min":1.8759999999999999},"min":{"1min":0.38300000000000001,"5min":0.29399999999999998,"15min":0.26700000000000002},"max":{"1min":3.5819999999999999,"5min":8.6780000000000008,"15min":14.039999999999999},"last":0.51100000000000001}]},{"osd":5,"last update":"Sat Sep 23 21:14:15 2023","interfaces":[{"interface":"back","average":{"1min":0.94399999999999995,"5min":1.276,"15min":1.5209999999999999},"min":{"1min":0.311,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":3.3759999999999999,"5min":9.5039999999999996,"15min":10.069000000000001},"last":0.378},{"interface":"front","average":{"1min":0.83999999999999997,"5min":1.3149999999999999,"15min":1.8120000000000001},"min":{"1min":0.41499999999999998,"5min":0.29699999999999999,"15min":0.29399999999999998},"max":{"1min":3.629,"5min":8.7270000000000003,"15min":13.98},"last":0.54200000000000004}]},{"osd":6,"last update":"Sat Sep 23 21:14:15 2023","interfaces":[{"interface":"back","average":{"1min":0.94999999999999996,"5min":1.569,"15min":2.0529999999999999},"min":{"1min":0.436,"5min":0.373,"15min":0.35699999999999998},"max":{"1min":3.665,"5min":8.7680000000000007,"15min":14.016},"last":0.56699999999999995},{"interface":"front","average":{"1min":1.2450000000000001,"5min":1.7729999999999999,"15min":2.2330000000000001},"min":{"1min":0.45700000000000002,"5min":0.38100000000000001,"15min":0.35599999999999998},"max":{"1min":3.7610000000000001,"5min":9.3469999999999995,"15min":14.068},"last":0.60199999999999998}]},{"osd":7,"last update":"Sat Sep 23 21:14:15 2023","interfaces":[{"interface":"back","average":{"1min":1.8120000000000001,"5min":1.4790000000000001,"15min":1.5840000000000001},"min":{"1min":0.433,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":18.021999999999998,"5min":18.021999999999998,"15min":18.021999999999998},"last":0.498},{"interface":"front","average":{"1min":1.242,"5min":1.476,"15min":1.675},"min":{"1min":0.39400000000000002,"5min":0.34799999999999998,"15min":0.32200000000000001},"max":{"1min":9.6349999999999998,"5min":10.359,"15min":10.92},"last":0.41099999999999998}]}]},{"osd":3,"up_from":98,"seq":420906795287,"num_pgs":82,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2355892,"kb_used_data":1517676,"kb_used_omap":7,"kb_used_meta":838200,"kb_avail":91389260,"statfs":{"total":95995035648,"available":93582602240,"internally_reserved":0,"allocated":1554100224,"data_stored":1551666497,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7576,"internal_metadata":858317416},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:14:12 2023","interfaces":[{"interface":"back","average":{"1min":0.33800000000000002,"5min":0.33800000000000002,"15min":0.33800000000000002},"min":{"1min":0.20499999999999999,"5min":0.20499999999999999,"15min":0.20499999999999999},"max":{"1min":0.69899999999999995,"5min":0.69899999999999995,"15min":0.69899999999999995},"last":0.32700000000000001},{"interface":"front","average":{"1min":0.66800000000000004,"5min":0.66800000000000004,"15min":0.66800000000000004},"min":{"1min":0.23300000000000001,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":4.5510000000000002,"5min":4.5510000000000002,"15min":4.5510000000000002},"last":0.29199999999999998}]},{"osd":1,"last update":"Sat Sep 23 21:14:12 2023","interfaces":[{"interface":"back","average":{"1min":1.052,"5min":1.466,"15min":2.226},"min":{"1min":0.218,"5min":0.20000000000000001,"15min":0.20000000000000001},"max":{"1min":5.5019999999999998,"5min":22.577999999999999,"15min":22.577999999999999},"last":0.313},{"interface":"front","average":{"1min":0.60099999999999998,"5min":1.5580000000000001,"15min":2.1829999999999998},"min":{"1min":0.249,"5min":0.216,"15min":0.216},"max":{"1min":2.7519999999999998,"5min":21.617999999999999,"15min":21.617999999999999},"last":0.374}]},{"osd":2,"last update":"Sat Sep 23 21:14:00 2023","interfaces":[{"interface":"back","average":{"1min":3.6669999999999998,"5min":1.9279999999999999,"15min":2.6640000000000001},"min":{"1min":0.221,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":56.317999999999998,"5min":56.317999999999998,"15min":56.317999999999998},"last":0.64000000000000001},{"interface":"front","average":{"1min":3.8479999999999999,"5min":2.1499999999999999,"15min":2.702},"min":{"1min":0.29699999999999999,"5min":0.216,"15min":0.216},"max":{"1min":55.988,"5min":55.988,"15min":55.988},"last":0.60199999999999998}]},{"osd":4,"last update":"Sat Sep 23 21:14:00 2023","interfaces":[{"interface":"back","average":{"1min":0.71499999999999997,"5min":1.3029999999999999,"15min":1.8779999999999999},"min":{"1min":0.30299999999999999,"5min":0.30299999999999999,"15min":0.28399999999999997},"max":{"1min":5.0819999999999999,"5min":21.478999999999999,"15min":21.478999999999999},"last":0.53000000000000003},{"interface":"front","average":{"1min":1.3009999999999999,"5min":1.5760000000000001,"15min":2.0070000000000001},"min":{"1min":0.33100000000000002,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":4.1909999999999998,"5min":22.402000000000001,"15min":22.402000000000001},"last":0.77300000000000002}]},{"osd":5,"last update":"Sat Sep 23 21:14:00 2023","interfaces":[{"interface":"back","average":{"1min":0.93600000000000005,"5min":1.3120000000000001,"15min":2.012},"min":{"1min":0.32200000000000001,"5min":0.311,"15min":0.311},"max":{"1min":5.04,"5min":21.562000000000001,"15min":21.562000000000001},"last":0.56699999999999995},{"interface":"front","average":{"1min":0.83699999999999997,"5min":1.339,"15min":2.2839999999999998},"min":{"1min":0.34300000000000003,"5min":0.31,"15min":0.31},"max":{"1min":3.4780000000000002,"5min":8.8450000000000006,"15min":14.029},"last":2.0529999999999999}]},{"osd":6,"last update":"Sat Sep 23 21:13:14 2023","interfaces":[{"interface":"back","average":{"1min":1.48,"5min":2.0219999999999998,"15min":2.097},"min":{"1min":0.34999999999999998,"5min":0.34999999999999998,"15min":0.33500000000000002},"max":{"1min":10.102,"5min":22.446999999999999,"15min":22.446999999999999},"last":0.55400000000000005},{"interface":"front","average":{"1min":2.0790000000000002,"5min":1.831,"15min":2.294},"min":{"1min":0.39900000000000002,"5min":0.39000000000000001,"15min":0.373},"max":{"1min":10.148,"5min":12.375999999999999,"15min":13.576000000000001},"last":0.68200000000000005}]},{"osd":7,"last update":"Sat Sep 23 21:14:00 2023","interfaces":[{"interface":"back","average":{"1min":1.032,"5min":1.6120000000000001,"15min":2.1299999999999999},"min":{"1min":0.41199999999999998,"5min":0.38400000000000001,"15min":0.33900000000000002},"max":{"1min":5.0090000000000003,"5min":21.518999999999998,"15min":21.518999999999998},"last":0.61199999999999999},{"interface":"front","average":{"1min":0.92900000000000005,"5min":1.7050000000000001,"15min":2.4199999999999999},"min":{"1min":0.45800000000000002,"5min":0.442,"15min":0.38900000000000001},"max":{"1min":3.2679999999999998,"5min":9.9879999999999995,"15min":13.603999999999999},"last":0.622}]}]},{"osd":4,"up_from":31,"seq":133143986594,"num_pgs":77,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1852232,"kb_used_data":1408896,"kb_used_omap":0,"kb_used_meta":443328,"kb_avail":91892920,"statfs":{"total":95995035648,"available":94098350080,"internally_reserved":0,"allocated":1442709504,"data_stored":1441180652,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":453967872},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,2,5,1,5,0,3,5],"upper_bound":512},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:14:14 2023","interfaces":[{"interface":"back","average":{"1min":0.90000000000000002,"5min":0.90000000000000002,"15min":0.90000000000000002},"min":{"1min":0.22500000000000001,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":2.6539999999999999,"5min":2.6539999999999999,"15min":2.6539999999999999},"last":0.44400000000000001},{"interface":"front","average":{"1min":1.1339999999999999,"5min":1.1339999999999999,"15min":1.1339999999999999},"min":{"1min":0.246,"5min":0.246,"15min":0.246},"max":{"1min":10.945,"5min":10.945,"15min":10.945},"last":0.378}]},{"osd":1,"last update":"Sat Sep 23 21:14:14 2023","interfaces":[{"interface":"back","average":{"1min":0.77300000000000002,"5min":1.0609999999999999,"15min":2.7490000000000001},"min":{"1min":0.29699999999999999,"5min":0.221,"15min":0.221},"max":{"1min":2.387,"5min":7.8609999999999998,"15min":37.960000000000001},"last":0.40899999999999997},{"interface":"front","average":{"1min":0.80600000000000005,"5min":1.1339999999999999,"15min":2.6080000000000001},"min":{"1min":0.28499999999999998,"5min":0.28499999999999998,"15min":0.27000000000000002},"max":{"1min":3.581,"5min":7.1349999999999998,"15min":37.933},"last":0.30299999999999999}]},{"osd":2,"last update":"Sat Sep 23 21:14:14 2023","interfaces":[{"interface":"back","average":{"1min":0.82099999999999995,"5min":1.153,"15min":1.748},"min":{"1min":0.27400000000000002,"5min":0.247,"15min":0.247},"max":{"1min":3.476,"5min":10.464,"15min":16.928000000000001},"last":0.26400000000000001},{"interface":"front","average":{"1min":0.86099999999999999,"5min":1.3959999999999999,"15min":1.7809999999999999},"min":{"1min":0.311,"5min":0.28599999999999998,"15min":0.27800000000000002},"max":{"1min":3.5129999999999999,"5min":7.3659999999999997,"15min":14.417999999999999},"last":0.33000000000000002}]},{"osd":3,"last update":"Sat Sep 23 21:13:46 2023","interfaces":[{"interface":"back","average":{"1min":1.0229999999999999,"5min":1.6299999999999999,"15min":2.1339999999999999},"min":{"1min":0.25600000000000001,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":4.2480000000000002,"5min":14.585000000000001,"15min":15.01},"last":0.48399999999999999},{"interface":"front","average":{"1min":1.135,"5min":1.647,"15min":2.0249999999999999},"min":{"1min":0.36699999999999999,"5min":0.36099999999999999,"15min":0.30199999999999999},"max":{"1min":4.4649999999999999,"5min":14.366,"15min":15.004},"last":0.499}]},{"osd":5,"last update":"Sat Sep 23 21:14:04 2023","interfaces":[{"interface":"back","average":{"1min":1.169,"5min":1.018,"15min":0.97599999999999998},"min":{"1min":0.30099999999999999,"5min":0.222,"15min":0.222},"max":{"1min":5.1609999999999996,"5min":9.4909999999999997,"15min":14.298},"last":0.36099999999999999},{"interface":"front","average":{"1min":1.0189999999999999,"5min":0.95899999999999996,"15min":1.0760000000000001},"min":{"1min":0.28000000000000003,"5min":0.27200000000000002,"15min":0.255},"max":{"1min":3.0059999999999998,"5min":6.8730000000000002,"15min":9.8979999999999997},"last":0.63500000000000001}]},{"osd":6,"last update":"Sat Sep 23 21:14:14 2023","interfaces":[{"interface":"back","average":{"1min":0.747,"5min":1.0920000000000001,"15min":1.2230000000000001},"min":{"1min":0.317,"5min":0.28599999999999998,"15min":0.26800000000000002},"max":{"1min":2.8650000000000002,"5min":24.452999999999999,"15min":24.452999999999999},"last":8.7420000000000009},{"interface":"front","average":{"1min":0.95099999999999996,"5min":1.036,"15min":0.95399999999999996},"min":{"1min":0.40100000000000002,"5min":0.33500000000000002,"15min":0.32500000000000001},"max":{"1min":3.6600000000000001,"5min":11.409000000000001,"15min":14.387},"last":8.7840000000000007}]},{"osd":7,"last update":"Sat Sep 23 21:13:46 2023","interfaces":[{"interface":"back","average":{"1min":1.294,"5min":1.268,"15min":1.0529999999999999},"min":{"1min":0.375,"5min":0.30099999999999999,"15min":0.29299999999999998},"max":{"1min":4.532,"5min":16.890999999999998,"15min":16.890999999999998},"last":0.39200000000000002},{"interface":"front","average":{"1min":1.387,"5min":1.155,"15min":1.1559999999999999},"min":{"1min":0.375,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":7.1879999999999997,"5min":8.5830000000000002,"15min":11.170999999999999},"last":0.67200000000000004}]}]},{"osd":5,"up_from":39,"seq":167503724960,"num_pgs":61,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1660084,"kb_used_data":1161964,"kb_used_omap":0,"kb_used_meta":498112,"kb_avail":92085068,"statfs":{"total":95995035648,"available":94295109632,"internally_reserved":0,"allocated":1189851136,"data_stored":1188338474,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":510066688},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:14:13 2023","interfaces":[{"interface":"back","average":{"1min":0.71499999999999997,"5min":0.71499999999999997,"15min":0.71499999999999997},"min":{"1min":0.251,"5min":0.251,"15min":0.251},"max":{"1min":3.3889999999999998,"5min":3.3889999999999998,"15min":3.3889999999999998},"last":0.88},{"interface":"front","average":{"1min":0.66900000000000004,"5min":0.66900000000000004,"15min":0.66900000000000004},"min":{"1min":0.22800000000000001,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":3.1150000000000002,"5min":3.1150000000000002,"15min":3.1150000000000002},"last":0.96099999999999997}]},{"osd":1,"last update":"Sat Sep 23 21:14:08 2023","interfaces":[{"interface":"back","average":{"1min":0.65200000000000002,"5min":1.1479999999999999,"15min":1.728},"min":{"1min":0.248,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":3.109,"5min":9.4429999999999996,"15min":25.239999999999998},"last":0.73999999999999999},{"interface":"front","average":{"1min":1.468,"5min":1.681,"15min":1.784},"min":{"1min":0.34999999999999998,"5min":0.28699999999999998,"15min":0.252},"max":{"1min":11.116,"5min":11.116,"15min":25.253},"last":0.69999999999999996}]},{"osd":2,"last update":"Sat Sep 23 21:13:26 2023","interfaces":[{"interface":"back","average":{"1min":1.871,"5min":1.6679999999999999,"15min":1.7909999999999999},"min":{"1min":0.30299999999999999,"5min":0.26200000000000001,"15min":0.23499999999999999},"max":{"1min":6.085,"5min":11.939,"15min":12.634},"last":1.0760000000000001},{"interface":"front","average":{"1min":0.71699999999999997,"5min":1.3979999999999999,"15min":1.915},"min":{"1min":0.27600000000000002,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":3.165,"5min":9.7170000000000005,"15min":12.300000000000001},"last":1.891}]},{"osd":3,"last update":"Sat Sep 23 21:13:45 2023","interfaces":[{"interface":"back","average":{"1min":1.4279999999999999,"5min":1.8089999999999999,"15min":2.6560000000000001},"min":{"1min":0.33400000000000002,"5min":0.307,"15min":0.29299999999999998},"max":{"1min":6.1859999999999999,"5min":14.242000000000001,"15min":61.667999999999999},"last":0.72699999999999998},{"interface":"front","average":{"1min":1.4390000000000001,"5min":2.0089999999999999,"15min":2.6429999999999998},"min":{"1min":0.34899999999999998,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":6.1200000000000001,"5min":14.396000000000001,"15min":62.276000000000003},"last":0.68100000000000005}]},{"osd":4,"last update":"Sat Sep 23 21:13:59 2023","interfaces":[{"interface":"back","average":{"1min":1.452,"5min":1.387,"15min":1.139},"min":{"1min":0.28799999999999998,"5min":0.27000000000000002,"15min":0.23400000000000001},"max":{"1min":11.156000000000001,"5min":12.776,"15min":12.776},"last":0.41099999999999998},{"interface":"front","average":{"1min":1.5109999999999999,"5min":1.1799999999999999,"15min":1.1319999999999999},"min":{"1min":0.28000000000000003,"5min":0.27000000000000002,"15min":0.26900000000000002},"max":{"1min":9.7119999999999997,"5min":9.7119999999999997,"15min":11.973000000000001},"last":0.47899999999999998}]},{"osd":6,"last update":"Sat Sep 23 21:14:08 2023","interfaces":[{"interface":"back","average":{"1min":0.625,"5min":0.92200000000000004,"15min":1.056},"min":{"1min":0.38800000000000001,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":1.448,"5min":7.7030000000000003,"15min":11.128},"last":0.60199999999999998},{"interface":"front","average":{"1min":1.4350000000000001,"5min":1.4390000000000001,"15min":1.246},"min":{"1min":0.42099999999999999,"5min":0.32400000000000001,"15min":0.317},"max":{"1min":10.999000000000001,"5min":10.999000000000001,"15min":11.112},"last":0.626}]},{"osd":7,"last update":"Sat Sep 23 21:13:45 2023","interfaces":[{"interface":"back","average":{"1min":1.3,"5min":0.95099999999999996,"15min":1.804},"min":{"1min":0.34300000000000003,"5min":0.31,"15min":0.309},"max":{"1min":5.5659999999999998,"5min":5.5659999999999998,"15min":189.405},"last":0.57499999999999996},{"interface":"front","average":{"1min":2.004,"5min":1.5009999999999999,"15min":1.9159999999999999},"min":{"1min":0.40200000000000002,"5min":0.35399999999999998,"15min":0.32500000000000001},"max":{"1min":11.183999999999999,"5min":11.183999999999999,"15min":189.685},"last":0.441}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8488,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1259,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5322,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":714,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":5434,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":4838,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":2531459072,"data_stored":2531459072,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1342963712,"data_stored":1342963712,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1610416128,"data_stored":1610416128,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1533808640,"data_stored":1533804568,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1424232448,"data_stored":1424228376,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1174011904,"data_stored":1174011904,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":979042304,"data_stored":979042304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T21:14:20.985 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_ops_in_flight 2023-09-23T21:14:20.986 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:14:21.035 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:14:21.091 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:14:21.100 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:14:21.100 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:14:21.201 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:14:21.251 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:14:21.303 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T21:14:21.307 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:14:21.308 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:14:21.351 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T21:14:21.409 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:14:21.464 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:14:21.512 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:14:21.516 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:14:21.516 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:14:21.617 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:14:21.671 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:14:21.720 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:14:21.724 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:14:21.724 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:14:21.752 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:14:21.826 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:14:21.877 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:14:21 smithi070 ceph-mon[168282]: from='client.46682 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T21:14:21.877 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:14:21 smithi070 ceph-mon[168282]: Health check update: Degraded data redundancy: 24525/121032 objects degraded (20.263%), 19 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-09-23T21:14:21.881 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:14:21 smithi026 ceph-mon[175151]: from='client.46682 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T21:14:21.881 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:14:21 smithi026 ceph-mon[175151]: Health check update: Degraded data redundancy: 24525/121032 objects degraded (20.263%), 19 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-09-23T21:14:21.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:14:21 smithi026 ceph-mon[179688]: from='client.46682 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T21:14:21.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:14:21 smithi026 ceph-mon[179688]: Health check update: Degraded data redundancy: 24525/121032 objects degraded (20.263%), 19 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-09-23T21:14:21.882 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:14:21.932 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:14:21.939 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:14:21.939 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:14:22.040 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:14:22.091 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:14:22.140 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:14:22.148 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:14:22.149 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:14:22.249 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:14:22.297 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:14:22.327 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:14:22.355 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:14:22.360 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:14:22.361 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:14:22.462 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:14:22.519 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:14:22.582 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:14:22 smithi026 ceph-mon[179688]: pgmap v1323: 129 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 15 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 108 active+clean; 2.4 GiB data, 12 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 494 op/s; 24525/119070 objects degraded (20.597%); 4517/119070 objects misplaced (3.794%); 7.0 MiB/s, 111 objects/s recovering 2023-09-23T21:14:22.582 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:14:22 smithi026 ceph-mon[175151]: pgmap v1323: 129 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 15 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 108 active+clean; 2.4 GiB data, 12 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 494 op/s; 24525/119070 objects degraded (20.597%); 4517/119070 objects misplaced (3.794%); 7.0 MiB/s, 111 objects/s recovering 2023-09-23T21:14:22.583 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:14:22 smithi070 ceph-mon[168282]: pgmap v1323: 129 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 15 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 108 active+clean; 2.4 GiB data, 12 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 494 op/s; 24525/119070 objects degraded (20.597%); 4517/119070 objects misplaced (3.794%); 7.0 MiB/s, 111 objects/s recovering 2023-09-23T21:14:22.584 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:14:22.588 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:14:22.588 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:14:22.689 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:14:22.735 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:14:22.838 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T21:14:22.847 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:14:22.848 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:14:22.918 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T21:14:22.949 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:14:23.008 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:14:23.134 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:14:23.137 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:14:23.138 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:14:23.238 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:14:23.276 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:14:23.329 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:14:23.337 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:14:23.338 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:14:23.438 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:14:23.482 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:14:23.527 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:14:23.530 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:14:23.531 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:14:23.631 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:14:23.650 INFO:teuthology.orchestra.run.smithi026.stdout:ERROR: (22) Invalid argument 2023-09-23T21:14:23.650 INFO:teuthology.orchestra.run.smithi026.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T21:14:23.663 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:14:23.707 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:14:23.710 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:14:23.711 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:14:23.811 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:14:23.856 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:14:23.932 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T21:14:24.226 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:14:24.231 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:14:24.231 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:14:24.266 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:14:24.303 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_blocked_ops 2023-09-23T21:14:24.332 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:14:24.415 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:14:24.652 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T21:14:24.655 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:14:24.655 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:14:24.756 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:14:24.794 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:14:24 smithi026 ceph-mon[175151]: pgmap v1324: 129 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 15 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 108 active+clean; 2.4 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 506 op/s; 24525/116367 objects degraded (21.076%); 4430/116367 objects misplaced (3.807%); 7.0 MiB/s, 111 objects/s recovering 2023-09-23T21:14:24.794 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:14:24 smithi026 ceph-mon[179688]: pgmap v1324: 129 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 15 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 108 active+clean; 2.4 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 506 op/s; 24525/116367 objects degraded (21.076%); 4430/116367 objects misplaced (3.807%); 7.0 MiB/s, 111 objects/s recovering 2023-09-23T21:14:24.799 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:14:24.860 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:14:24.866 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:14:24.867 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:14:24.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:14:24 smithi070 ceph-mon[168282]: pgmap v1324: 129 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 15 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 108 active+clean; 2.4 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 506 op/s; 24525/116367 objects degraded (21.076%); 4430/116367 objects misplaced (3.807%); 7.0 MiB/s, 111 objects/s recovering 2023-09-23T21:14:24.968 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:14:25.003 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:14:25.058 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:14:25.062 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:14:25.062 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:14:25.163 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:14:25.218 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:14:25.270 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:14:25.283 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:14:25.284 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:14:25.385 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:14:25.447 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:14:25.494 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:14:25.496 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:14:25.497 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:14:25.593 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:14:25 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/2550457306' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:14:25.598 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:14:25.611 INFO:teuthology.orchestra.run.smithi026.stderr:nodeep-scrub is unset 2023-09-23T21:14:25.654 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:14:25.698 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:14:25.700 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:14:25.701 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:14:25.801 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:14:25.850 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:14:25.881 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:14:25 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/2550457306' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:14:25.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:14:25 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/2550457306' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:14:25.897 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:14:25.900 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:14:25.900 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:14:26.001 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:14:26.040 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:14:26.857 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:14:26.861 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:14:26 smithi026 ceph-mon[179688]: pgmap v1325: 129 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 15 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 108 active+clean; 2.3 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 520 op/s; 24197/111774 objects degraded (21.648%); 4257/111774 objects misplaced (3.809%); 7.2 MiB/s, 114 objects/s recovering 2023-09-23T21:14:26.862 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:14:26 smithi026 ceph-mon[179688]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T21:14:26.862 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:14:26 smithi026 ceph-mon[179688]: failed to encode map e528 with expected crc 2023-09-23T21:14:26.862 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:14:26 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/2550457306' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T21:14:26.862 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:14:26 smithi026 ceph-mon[179688]: osdmap e528: 8 total, 8 up, 6 in 2023-09-23T21:14:26.862 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:14:26 smithi026 ceph-mon[179688]: failed to encode map e528 with expected crc 2023-09-23T21:14:26.862 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:14:26 smithi026 ceph-mon[179688]: failed to encode map e528 with expected crc 2023-09-23T21:14:26.862 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:14:26 smithi026 ceph-mon[179688]: failed to encode map e528 with expected crc 2023-09-23T21:14:26.862 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:14:26 smithi026 ceph-mon[179688]: failed to encode map e528 with expected crc 2023-09-23T21:14:26.862 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:14:26 smithi026 ceph-mon[179688]: Health check update: Degraded data redundancy: 24197/111774 objects degraded (21.648%), 19 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-09-23T21:14:26.863 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:14:26.863 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:14:26.864 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:14:26 smithi026 ceph-mon[175151]: pgmap v1325: 129 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 15 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 108 active+clean; 2.3 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 520 op/s; 24197/111774 objects degraded (21.648%); 4257/111774 objects misplaced (3.809%); 7.2 MiB/s, 114 objects/s recovering 2023-09-23T21:14:26.865 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:14:26 smithi026 ceph-mon[175151]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T21:14:26.865 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:14:26 smithi026 ceph-mon[175151]: failed to encode map e528 with expected crc 2023-09-23T21:14:26.865 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:14:26 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/2550457306' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T21:14:26.865 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:14:26 smithi026 ceph-mon[175151]: osdmap e528: 8 total, 8 up, 6 in 2023-09-23T21:14:26.865 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:14:26 smithi026 ceph-mon[175151]: failed to encode map e528 with expected crc 2023-09-23T21:14:26.865 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:14:26 smithi026 ceph-mon[175151]: failed to encode map e528 with expected crc 2023-09-23T21:14:26.865 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:14:26 smithi026 ceph-mon[175151]: failed to encode map e528 with expected crc 2023-09-23T21:14:26.865 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:14:26 smithi026 ceph-mon[175151]: failed to encode map e528 with expected crc 2023-09-23T21:14:26.865 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:14:26 smithi026 ceph-mon[175151]: Health check update: Degraded data redundancy: 24197/111774 objects degraded (21.648%), 19 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-09-23T21:14:26.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:14:26 smithi070 ceph-mon[168282]: pgmap v1325: 129 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 15 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 108 active+clean; 2.3 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 520 op/s; 24197/111774 objects degraded (21.648%); 4257/111774 objects misplaced (3.809%); 7.2 MiB/s, 114 objects/s recovering 2023-09-23T21:14:26.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:14:26 smithi070 ceph-mon[168282]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T21:14:26.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:14:26 smithi070 ceph-mon[168282]: failed to encode map e528 with expected crc 2023-09-23T21:14:26.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:14:26 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/2550457306' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T21:14:26.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:14:26 smithi070 ceph-mon[168282]: osdmap e528: 8 total, 8 up, 6 in 2023-09-23T21:14:26.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:14:26 smithi070 ceph-mon[168282]: failed to encode map e528 with expected crc 2023-09-23T21:14:26.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:14:26 smithi070 ceph-mon[168282]: failed to encode map e528 with expected crc 2023-09-23T21:14:26.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:14:26 smithi070 ceph-mon[168282]: failed to encode map e528 with expected crc 2023-09-23T21:14:26.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:14:26 smithi070 ceph-mon[168282]: failed to encode map e528 with expected crc 2023-09-23T21:14:26.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:14:26 smithi070 ceph-mon[168282]: Health check update: Degraded data redundancy: 24197/111774 objects degraded (21.648%), 19 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-09-23T21:14:26.964 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:14:26.972 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:14:27.053 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:14:27.177 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:14:27.183 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:14:27.183 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:14:27.184 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:14:27.284 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:14:27.337 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:14:27.551 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:14:27.560 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:14:27.560 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:14:27.661 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:14:27.692 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0: {} 2023-09-23T21:14:27.696 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:14:27.724 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1: {} 2023-09-23T21:14:27.762 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2: {} 2023-09-23T21:14:27.794 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3: {} 2023-09-23T21:14:27.799 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:14:27.810 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:14:27.811 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:14:27.825 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4: {} 2023-09-23T21:14:27.858 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T21:14:27.886 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6: {} 2023-09-23T21:14:27.912 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:14:27.915 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T21:14:27.917 INFO:teuthology.orchestra.run.smithi026.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T21:14:27.917 INFO:teuthology.orchestra.run.smithi026.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T21:14:27.918 INFO:teuthology.orchestra.run.smithi026.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T21:14:27.918 INFO:teuthology.orchestra.run.smithi026.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T21:14:27.918 INFO:teuthology.orchestra.run.smithi026.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T21:14:27.918 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T21:14:27.918 INFO:teuthology.orchestra.run.smithi026.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T21:14:27.918 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T21:14:27.948 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:14:27.996 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:14:27.999 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:14:27.999 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:14:28.100 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:14:28.147 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:14:28.703 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:14:28.707 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:14:28 smithi026 ceph-mon[175151]: pgmap v1327: 129 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 15 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 108 active+clean; 2.2 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 534 op/s; 24197/108930 objects degraded (22.213%); 4164/108930 objects misplaced (3.823%); 7.5 MiB/s, 120 objects/s recovering 2023-09-23T21:14:28.708 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:14:28 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/2136490453' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:14:28.708 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:14:28 smithi026 ceph-mon[179688]: pgmap v1327: 129 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 15 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 108 active+clean; 2.2 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 534 op/s; 24197/108930 objects degraded (22.213%); 4164/108930 objects misplaced (3.823%); 7.5 MiB/s, 120 objects/s recovering 2023-09-23T21:14:28.708 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:14:28 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/2136490453' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:14:28.708 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:14:28.709 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:14:28.810 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:14:28.868 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:14:28 smithi070 ceph-mon[168282]: pgmap v1327: 129 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 15 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 108 active+clean; 2.2 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 534 op/s; 24197/108930 objects degraded (22.213%); 4164/108930 objects misplaced (3.823%); 7.5 MiB/s, 120 objects/s recovering 2023-09-23T21:14:28.868 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:14:28 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/2136490453' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:14:28.869 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:14:28.915 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:14:28.919 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:14:28.919 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:14:29.001 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T21:14:29.020 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:14:29.047 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:14:29.084 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:14:29.087 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:14:29.087 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:14:29.188 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:14:29.243 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:14:29.312 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:14:29.316 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:14:29.317 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:14:29.317 INFO:teuthology.orchestra.run.smithi026.stdout:{ 2023-09-23T21:14:29.317 INFO:teuthology.orchestra.run.smithi026.stdout: "ops": [], 2023-09-23T21:14:29.317 INFO:teuthology.orchestra.run.smithi026.stdout: "complaint_time": 30, 2023-09-23T21:14:29.317 INFO:teuthology.orchestra.run.smithi026.stdout: "num_blocked_ops": 0 2023-09-23T21:14:29.317 INFO:teuthology.orchestra.run.smithi026.stdout:} 2023-09-23T21:14:29.417 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:14:29.478 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:14:29.551 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T21:14:29.557 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:14:29.557 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:14:29.658 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:14:29.703 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:14:29.812 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T21:14:29.816 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 21:14:29 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[164029]: ::ffff:172.21.15.70 - - [23/Sep/2023:21:14:29] "GET /metrics HTTP/1.1" 200 36406 "" "Prometheus/2.43.0" 2023-09-23T21:14:29.818 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:14:29.818 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:14:29.910 INFO:teuthology.orchestra.run.smithi026.stdout: 2023-09-23T21:14:29.919 INFO:teuthology.orchestra.run.smithi026.stderr:dumped all 2023-09-23T21:14:29.919 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:14:29.970 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_historic_ops 2023-09-23T21:14:29.981 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:14:29.982 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:14:30.028 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:14:30.034 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:14:30.035 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:14:30.136 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:14:30.186 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:14:30.286 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T21:14:30.304 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T21:14:30.307 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:14:30.307 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:14:30.408 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:14:30.444 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:14:30.498 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:14:30.501 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:14:30.502 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:14:30.602 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:14:30.637 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:14:30.679 INFO:teuthology.orchestra.run.smithi026.stdout:{"pg_ready":true,"pg_map":{"version":1328,"stamp":"2023-09-23T21:14:29.055868+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":2296953604,"num_objects":35066,"num_object_clones":0,"num_object_copies":105198,"num_objects_missing_on_primary":2360,"num_objects_missing":2360,"num_objects_degraded":23883,"num_objects_misplaced":3977,"num_objects_unfound":0,"num_objects_dirty":35066,"num_whiteouts":0,"num_read":90,"num_read_kb":166,"num_write":112438,"num_write_kb":4309782,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":38811,"num_bytes_recovered":2330627237,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":136588,"ondisk_log_size":136588,"up":387,"acting":371,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":433,"num_osds":6,"num_per_pool_osds":6,"num_per_pool_omap_osds":6,"kb":562470912,"kb_used":11419688,"kb_used_data":8199352,"kb_used_omap":125,"kb_used_meta":3220098,"kb_avail":551051224,"statfs":{"total":575970213888,"available":564276453376,"internally_reserved":0,"allocated":8396136448,"data_stored":8384161823,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":128662,"internal_metadata":3297380714},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-345899008,"num_objects":-5278,"num_object_clones":0,"num_object_copies":-15834,"num_objects_missing_on_primary":-522,"num_objects_missing":-522,"num_objects_degraded":-642,"num_objects_misplaced":-540,"num_objects_unfound":0,"num_objects_dirty":-5278,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5278,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1161,"num_bytes_recovered":76087296,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.003967"},"pg_stats":[{"pgid":"1.18","version":"79'161","reported_seq":648,"reported_epoch":527,"state":"active+clean","last_fresh":"2023-09-23T21:14:17.580895+0000","last_change":"2023-09-23T21:12:30.438733+0000","last_active":"2023-09-23T21:14:17.580895+0000","last_peered":"2023-09-23T21:14:17.580895+0000","last_clean":"2023-09-23T21:14:17.580895+0000","last_became_active":"2023-09-23T21:12:30.438356+0000","last_became_peered":"2023-09-23T21:12:30.438356+0000","last_unstale":"2023-09-23T21:14:17.580895+0000","last_undegraded":"2023-09-23T21:14:17.580895+0000","last_fullsized":"2023-09-23T21:14:17.580895+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":479,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:06:52.098882+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:06:52.098882+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0092173299999999993,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":131344,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"10.13","version":"528'5279","reported_seq":8446,"reported_epoch":528,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T21:14:28.265006+0000","last_change":"2023-09-23T21:13:49.840403+0000","last_active":"2023-09-23T21:14:28.265006+0000","last_peered":"2023-09-23T21:14:28.265006+0000","last_clean":"2023-09-23T21:11:26.718194+0000","last_became_active":"2023-09-23T21:13:48.181112+0000","last_became_peered":"2023-09-23T21:13:48.181112+0000","last_unstale":"2023-09-23T21:14:28.265006+0000","last_undegraded":"2023-09-23T21:13:48.139049+0000","last_fullsized":"2023-09-23T21:13:45.405990+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":438,"last_epoch_clean":439,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_clean_scrub_stamp":"2023-09-23T21:10:15.663179+0000","objects_scrubbed":0,"log_size":5279,"log_dups_size":0,"ondisk_log_size":5279,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":73564160,"num_objects":1123,"num_object_clones":0,"num_object_copies":3369,"num_objects_missing_on_primary":753,"num_objects_missing":753,"num_objects_degraded":953,"num_objects_misplaced":414,"num_objects_unfound":0,"num_objects_dirty":1123,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3911,"num_write_kb":161056,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1394,"num_bytes_recovered":91291648,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[4,3],"avail_no_missing":["5"],"object_location_counts":[{"shards":"0,2,4,5","objects":307},{"shards":"2,3,4,5","objects":1},{"shards":"2,3,5","objects":753},{"shards":"2,4,5","objects":28},{"shards":"3,4","objects":34}],"blocked_by":[],"up_primary":2,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1a","version":"68'1","reported_seq":118,"reported_epoch":527,"state":"active+clean","last_fresh":"2023-09-23T21:14:17.577878+0000","last_change":"2023-09-23T21:13:13.291403+0000","last_active":"2023-09-23T21:14:17.577878+0000","last_peered":"2023-09-23T21:14:17.577878+0000","last_clean":"2023-09-23T21:14:17.577878+0000","last_became_active":"2023-09-23T21:13:13.291123+0000","last_became_peered":"2023-09-23T21:13:13.291123+0000","last_unstale":"2023-09-23T21:14:17.577878+0000","last_undegraded":"2023-09-23T21:14:17.577878+0000","last_fullsized":"2023-09-23T21:14:17.577878+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":495,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:45:15.427430+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,0],"acting":[4,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":187,"reported_epoch":527,"state":"active+clean","last_fresh":"2023-09-23T21:14:17.578682+0000","last_change":"2023-09-23T21:12:10.767963+0000","last_active":"2023-09-23T21:14:17.578682+0000","last_peered":"2023-09-23T21:14:17.578682+0000","last_clean":"2023-09-23T21:14:17.578682+0000","last_became_active":"2023-09-23T21:12:10.767126+0000","last_became_peered":"2023-09-23T21:12:10.767126+0000","last_unstale":"2023-09-23T21:14:17.578682+0000","last_undegraded":"2023-09-23T21:14:17.578682+0000","last_fullsized":"2023-09-23T21:14:17.578682+0000","mapping_epoch":470,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:45:47.138277+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,5],"acting":[4,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1b","version":"79'161","reported_seq":530,"reported_epoch":526,"state":"active+clean","last_fresh":"2023-09-23T21:14:09.544527+0000","last_change":"2023-09-23T21:13:13.373548+0000","last_active":"2023-09-23T21:14:09.544527+0000","last_peered":"2023-09-23T21:14:09.544527+0000","last_clean":"2023-09-23T21:14:09.544527+0000","last_became_active":"2023-09-23T21:13:13.372845+0000","last_became_peered":"2023-09-23T21:13:13.372845+0000","last_unstale":"2023-09-23T21:14:09.544527+0000","last_undegraded":"2023-09-23T21:14:09.544527+0000","last_fullsized":"2023-09-23T21:14:09.544527+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":479,"last_epoch_clean":495,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:05:55.772264+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:05:55.772264+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.001862507,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.10","version":"528'5264","reported_seq":12541,"reported_epoch":528,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T21:14:28.276631+0000","last_change":"2023-09-23T21:13:36.071975+0000","last_active":"2023-09-23T21:14:28.276631+0000","last_peered":"2023-09-23T21:14:28.276631+0000","last_clean":"2023-09-23T21:11:26.716051+0000","last_became_active":"2023-09-23T21:13:33.619010+0000","last_became_peered":"2023-09-23T21:13:33.619010+0000","last_unstale":"2023-09-23T21:14:28.276631+0000","last_undegraded":"2023-09-23T21:13:33.443511+0000","last_fullsized":"2023-09-23T21:13:31.753181+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":438,"last_epoch_clean":450,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_clean_scrub_stamp":"2023-09-23T21:10:15.663179+0000","objects_scrubbed":0,"log_size":5264,"log_dups_size":0,"ondisk_log_size":5264,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":69959680,"num_objects":1068,"num_object_clones":0,"num_object_copies":3204,"num_objects_missing_on_primary":724,"num_objects_missing":724,"num_objects_degraded":1550,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1068,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3900,"num_write_kb":158944,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3294,"num_bytes_recovered":215875584,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7],"avail_no_missing":["1","5","7"],"object_location_counts":[{"shards":"0,4,5,7","objects":5},{"shards":"0,5,7","objects":724},{"shards":"4,5,7","objects":29},{"shards":"4,7","objects":310}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":522,"reported_epoch":526,"state":"active+clean","last_fresh":"2023-09-23T21:14:09.547375+0000","last_change":"2023-09-23T21:12:10.824116+0000","last_active":"2023-09-23T21:14:09.547375+0000","last_peered":"2023-09-23T21:14:09.547375+0000","last_clean":"2023-09-23T21:14:09.547375+0000","last_became_active":"2023-09-23T21:12:10.823849+0000","last_became_peered":"2023-09-23T21:12:10.823849+0000","last_unstale":"2023-09-23T21:14:09.547375+0000","last_undegraded":"2023-09-23T21:14:09.547375+0000","last_fullsized":"2023-09-23T21:14:09.547375+0000","mapping_epoch":470,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:54:13.777603+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":523,"reported_epoch":526,"state":"active+clean","last_fresh":"2023-09-23T21:14:09.544406+0000","last_change":"2023-09-23T21:13:44.413713+0000","last_active":"2023-09-23T21:14:09.544406+0000","last_peered":"2023-09-23T21:14:09.544406+0000","last_clean":"2023-09-23T21:14:09.544406+0000","last_became_active":"2023-09-23T21:13:44.413366+0000","last_became_peered":"2023-09-23T21:13:44.413366+0000","last_unstale":"2023-09-23T21:14:09.544406+0000","last_undegraded":"2023-09-23T21:14:09.544406+0000","last_fullsized":"2023-09-23T21:14:09.544406+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:31:56.349530+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1a","version":"79'161","reported_seq":673,"reported_epoch":527,"state":"active+clean","last_fresh":"2023-09-23T21:14:17.581080+0000","last_change":"2023-09-23T21:12:30.444665+0000","last_active":"2023-09-23T21:14:17.581080+0000","last_peered":"2023-09-23T21:14:17.581080+0000","last_clean":"2023-09-23T21:14:17.581080+0000","last_became_active":"2023-09-23T21:12:30.444326+0000","last_became_peered":"2023-09-23T21:12:30.444326+0000","last_unstale":"2023-09-23T21:14:17.581080+0000","last_undegraded":"2023-09-23T21:14:17.581080+0000","last_fullsized":"2023-09-23T21:14:17.581080+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":479,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:06:51.075448+0000","last_deep_scrub":"79'161","last_deep_scrub_stamp":"2023-09-23T21:05:32.630292+0000","last_clean_scrub_stamp":"2023-09-23T21:06:51.075448+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0054793890000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,4],"acting":[2,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"10.11","version":"528'5193","reported_seq":12023,"reported_epoch":528,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-23T21:14:26.183403+0000","last_change":"2023-09-23T21:13:31.864763+0000","last_active":"2023-09-23T21:14:26.183403+0000","last_peered":"2023-09-23T21:14:26.183403+0000","last_clean":"2023-09-23T21:13:30.058478+0000","last_became_active":"2023-09-23T21:13:31.850860+0000","last_became_peered":"2023-09-23T21:13:31.850860+0000","last_unstale":"2023-09-23T21:14:26.183403+0000","last_undegraded":"2023-09-23T21:14:26.183403+0000","last_fullsized":"2023-09-23T21:14:26.183403+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":438,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":5,"last_scrub":"441'2974","last_scrub_stamp":"2023-09-23T21:11:06.133218+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_clean_scrub_stamp":"2023-09-23T21:11:06.133218+0000","objects_scrubbed":1646,"log_size":5193,"log_dups_size":0,"ondisk_log_size":5193,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:54:59.366711+0000","scrub_duration":0.3173955,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70647808,"num_objects":1078,"num_object_clones":0,"num_object_copies":3234,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1078,"num_objects_unfound":0,"num_objects_dirty":1078,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2222,"num_write_kb":52928,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2152,"num_bytes_recovered":141033472,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[2,5,7],"avail_no_missing":["2","1","5","7"],"object_location_counts":[{"shards":"2,5,7","objects":1078}],"blocked_by":[],"up_primary":0,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":36,"reported_epoch":526,"state":"active+clean","last_fresh":"2023-09-23T21:14:09.544004+0000","last_change":"2023-09-23T21:13:30.806084+0000","last_active":"2023-09-23T21:14:09.544004+0000","last_peered":"2023-09-23T21:14:09.544004+0000","last_clean":"2023-09-23T21:14:09.544004+0000","last_became_active":"2023-09-23T21:13:30.805743+0000","last_became_peered":"2023-09-23T21:13:30.805743+0000","last_unstale":"2023-09-23T21:14:09.544004+0000","last_undegraded":"2023-09-23T21:14:09.544004+0000","last_fullsized":"2023-09-23T21:14:09.544004+0000","mapping_epoch":501,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:48:10.849281+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":400,"reported_epoch":526,"state":"active+clean","last_fresh":"2023-09-23T21:14:09.544725+0000","last_change":"2023-09-23T21:13:44.410921+0000","last_active":"2023-09-23T21:14:09.544725+0000","last_peered":"2023-09-23T21:14:09.544725+0000","last_clean":"2023-09-23T21:14:09.544725+0000","last_became_active":"2023-09-23T21:13:44.410633+0000","last_became_peered":"2023-09-23T21:13:44.410633+0000","last_unstale":"2023-09-23T21:14:09.544725+0000","last_undegraded":"2023-09-23T21:14:09.544725+0000","last_fullsized":"2023-09-23T21:14:09.544725+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:57:10.116786+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.15","version":"79'161","reported_seq":554,"reported_epoch":527,"state":"active+clean","last_fresh":"2023-09-23T21:14:17.577843+0000","last_change":"2023-09-23T21:13:38.379453+0000","last_active":"2023-09-23T21:14:17.577843+0000","last_peered":"2023-09-23T21:14:17.577843+0000","last_clean":"2023-09-23T21:14:17.577843+0000","last_became_active":"2023-09-23T21:13:38.376499+0000","last_became_peered":"2023-09-23T21:13:38.376499+0000","last_unstale":"2023-09-23T21:14:17.577843+0000","last_undegraded":"2023-09-23T21:14:17.577843+0000","last_fullsized":"2023-09-23T21:14:17.577843+0000","mapping_epoch":507,"log_start":"0'0","ondisk_log_start":"0'0","created":479,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:09:34.892947+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:09:34.892947+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13418,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":11,"num_read_kb":21,"num_write":6,"num_write_kb":55,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":40253,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":501,"reported_epoch":526,"state":"active+clean","last_fresh":"2023-09-23T21:14:09.544984+0000","last_change":"2023-09-23T21:13:12.235728+0000","last_active":"2023-09-23T21:14:09.544984+0000","last_peered":"2023-09-23T21:14:09.544984+0000","last_clean":"2023-09-23T21:14:09.544984+0000","last_became_active":"2023-09-23T21:13:12.216549+0000","last_became_peered":"2023-09-23T21:13:12.216549+0000","last_unstale":"2023-09-23T21:14:09.544984+0000","last_undegraded":"2023-09-23T21:14:09.544984+0000","last_fullsized":"2023-09-23T21:14:09.544984+0000","mapping_epoch":493,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:39:33.324239+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":564,"reported_epoch":526,"state":"active+clean","last_fresh":"2023-09-23T21:14:09.547234+0000","last_change":"2023-09-23T21:12:10.790349+0000","last_active":"2023-09-23T21:14:09.547234+0000","last_peered":"2023-09-23T21:14:09.547234+0000","last_clean":"2023-09-23T21:14:09.547234+0000","last_became_active":"2023-09-23T21:12:10.789977+0000","last_became_peered":"2023-09-23T21:12:10.789977+0000","last_unstale":"2023-09-23T21:14:09.547234+0000","last_undegraded":"2023-09-23T21:14:09.547234+0000","last_fullsized":"2023-09-23T21:14:09.547234+0000","mapping_epoch":470,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:49:40.191147+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,5],"acting":[3,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"79'161","reported_seq":889,"reported_epoch":527,"state":"active+clean","last_fresh":"2023-09-23T21:14:17.581863+0000","last_change":"2023-09-23T21:11:32.760292+0000","last_active":"2023-09-23T21:14:17.581863+0000","last_peered":"2023-09-23T21:14:17.581863+0000","last_clean":"2023-09-23T21:14:17.581863+0000","last_became_active":"2023-09-23T21:11:32.759909+0000","last_became_peered":"2023-09-23T21:11:32.759909+0000","last_unstale":"2023-09-23T21:14:17.581863+0000","last_undegraded":"2023-09-23T21:14:17.581863+0000","last_fullsized":"2023-09-23T21:14:17.581863+0000","mapping_epoch":457,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":458,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:25.096401+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:25.096401+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:13:09.225685+0000","scrub_duration":0.002859393,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":252,"reported_epoch":527,"state":"active+clean","last_fresh":"2023-09-23T21:14:17.585209+0000","last_change":"2023-09-23T21:12:10.779468+0000","last_active":"2023-09-23T21:14:17.585209+0000","last_peered":"2023-09-23T21:14:17.585209+0000","last_clean":"2023-09-23T21:14:17.585209+0000","last_became_active":"2023-09-23T21:12:10.779282+0000","last_became_peered":"2023-09-23T21:12:10.779282+0000","last_unstale":"2023-09-23T21:14:17.585209+0000","last_undegraded":"2023-09-23T21:14:17.585209+0000","last_fullsized":"2023-09-23T21:14:17.585209+0000","mapping_epoch":470,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:36:35.417986+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":501,"reported_epoch":526,"state":"active+clean","last_fresh":"2023-09-23T21:14:09.547283+0000","last_change":"2023-09-23T21:10:26.758958+0000","last_active":"2023-09-23T21:14:09.547283+0000","last_peered":"2023-09-23T21:14:09.547283+0000","last_clean":"2023-09-23T21:14:09.547283+0000","last_became_active":"2023-09-23T21:10:26.758539+0000","last_became_peered":"2023-09-23T21:10:26.758539+0000","last_unstale":"2023-09-23T21:14:09.547283+0000","last_undegraded":"2023-09-23T21:14:09.547283+0000","last_fullsized":"2023-09-23T21:14:09.547283+0000","mapping_epoch":423,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":424,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:44:42.319189+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.17","version":"79'161","reported_seq":619,"reported_epoch":526,"state":"active+clean","last_fresh":"2023-09-23T21:14:09.548493+0000","last_change":"2023-09-23T21:13:44.416768+0000","last_active":"2023-09-23T21:14:09.548493+0000","last_peered":"2023-09-23T21:14:09.548493+0000","last_clean":"2023-09-23T21:14:09.548493+0000","last_became_active":"2023-09-23T21:13:44.416156+0000","last_became_peered":"2023-09-23T21:13:44.416156+0000","last_unstale":"2023-09-23T21:14:09.548493+0000","last_undegraded":"2023-09-23T21:14:09.548493+0000","last_fullsized":"2023-09-23T21:14:09.548493+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":479,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:25.916867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:25.916867+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.001665323,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":411,"reported_epoch":526,"state":"active+clean","last_fresh":"2023-09-23T21:14:09.547985+0000","last_change":"2023-09-23T21:11:08.766045+0000","last_active":"2023-09-23T21:14:09.547985+0000","last_peered":"2023-09-23T21:14:09.547985+0000","last_clean":"2023-09-23T21:14:09.547985+0000","last_became_active":"2023-09-23T21:11:08.758504+0000","last_became_peered":"2023-09-23T21:11:08.758504+0000","last_unstale":"2023-09-23T21:14:09.547985+0000","last_undegraded":"2023-09-23T21:14:09.547985+0000","last_fullsized":"2023-09-23T21:14:09.547985+0000","mapping_epoch":443,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":444,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:02:19.482574+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":360,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,4],"acting":[3,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":214,"reported_epoch":527,"state":"active+clean","last_fresh":"2023-09-23T21:14:17.581509+0000","last_change":"2023-09-23T21:13:30.821489+0000","last_active":"2023-09-23T21:14:17.581509+0000","last_peered":"2023-09-23T21:14:17.581509+0000","last_clean":"2023-09-23T21:14:17.581509+0000","last_became_active":"2023-09-23T21:13:30.821125+0000","last_became_peered":"2023-09-23T21:13:30.821125+0000","last_unstale":"2023-09-23T21:14:17.581509+0000","last_undegraded":"2023-09-23T21:14:17.581509+0000","last_fullsized":"2023-09-23T21:14:17.581509+0000","mapping_epoch":501,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:47:18.553417+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,5],"acting":[2,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.16","version":"79'161","reported_seq":445,"reported_epoch":526,"state":"active+clean","last_fresh":"2023-09-23T21:14:09.544336+0000","last_change":"2023-09-23T21:13:33.770962+0000","last_active":"2023-09-23T21:14:09.544336+0000","last_peered":"2023-09-23T21:14:09.544336+0000","last_clean":"2023-09-23T21:14:09.544336+0000","last_became_active":"2023-09-23T21:13:33.770407+0000","last_became_peered":"2023-09-23T21:13:33.770407+0000","last_unstale":"2023-09-23T21:14:09.544336+0000","last_undegraded":"2023-09-23T21:14:09.544336+0000","last_fullsized":"2023-09-23T21:14:09.544336+0000","mapping_epoch":504,"log_start":"0'0","ondisk_log_start":"0'0","created":479,"last_epoch_clean":505,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:10:24.268662+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:10:24.268662+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13418,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":11,"num_read_kb":21,"num_write":6,"num_write_kb":55,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":40252,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.14","version":"79'22","reported_seq":91,"reported_epoch":527,"state":"active+clean","last_fresh":"2023-09-23T21:14:17.577580+0000","last_change":"2023-09-23T21:11:27.770162+0000","last_active":"2023-09-23T21:14:17.577580+0000","last_peered":"2023-09-23T21:14:17.577580+0000","last_clean":"2023-09-23T21:14:17.577580+0000","last_became_active":"2023-09-23T21:11:27.752275+0000","last_became_peered":"2023-09-23T21:11:27.752275+0000","last_unstale":"2023-09-23T21:14:17.577580+0000","last_undegraded":"2023-09-23T21:14:17.577580+0000","last_fullsized":"2023-09-23T21:14:17.577580+0000","mapping_epoch":452,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":453,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:05:11.769428+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4734,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,2],"acting":[4,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":456,"reported_epoch":527,"state":"active+clean","last_fresh":"2023-09-23T21:14:17.581019+0000","last_change":"2023-09-23T21:12:10.839271+0000","last_active":"2023-09-23T21:14:17.581019+0000","last_peered":"2023-09-23T21:14:17.581019+0000","last_clean":"2023-09-23T21:14:17.581019+0000","last_became_active":"2023-09-23T21:12:10.839042+0000","last_became_peered":"2023-09-23T21:12:10.839042+0000","last_unstale":"2023-09-23T21:14:17.581019+0000","last_undegraded":"2023-09-23T21:14:17.581019+0000","last_fullsized":"2023-09-23T21:14:17.581019+0000","mapping_epoch":470,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:10:26.414737+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,2],"acting":[4,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.11","version":"79'161","reported_seq":450,"reported_epoch":526,"state":"active+clean","last_fresh":"2023-09-23T21:14:09.544674+0000","last_change":"2023-09-23T21:13:33.767406+0000","last_active":"2023-09-23T21:14:09.544674+0000","last_peered":"2023-09-23T21:14:09.544674+0000","last_clean":"2023-09-23T21:14:09.544674+0000","last_became_active":"2023-09-23T21:13:33.767067+0000","last_became_peered":"2023-09-23T21:13:33.767067+0000","last_unstale":"2023-09-23T21:14:09.544674+0000","last_undegraded":"2023-09-23T21:14:09.544674+0000","last_fullsized":"2023-09-23T21:14:09.544674+0000","mapping_epoch":504,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":505,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:28.756634+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:28.756634+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":23,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":136,"reported_epoch":527,"state":"active+clean","last_fresh":"2023-09-23T21:14:17.600801+0000","last_change":"2023-09-23T21:11:27.777154+0000","last_active":"2023-09-23T21:14:17.600801+0000","last_peered":"2023-09-23T21:14:17.600801+0000","last_clean":"2023-09-23T21:14:17.600801+0000","last_became_active":"2023-09-23T21:11:27.754438+0000","last_became_peered":"2023-09-23T21:11:27.754438+0000","last_unstale":"2023-09-23T21:14:17.600801+0000","last_undegraded":"2023-09-23T21:14:17.600801+0000","last_fullsized":"2023-09-23T21:14:17.600801+0000","mapping_epoch":452,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":453,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:33:02.845882+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":68,"reported_epoch":527,"state":"active+clean","last_fresh":"2023-09-23T21:14:17.581567+0000","last_change":"2023-09-23T21:12:10.764264+0000","last_active":"2023-09-23T21:14:17.581567+0000","last_peered":"2023-09-23T21:14:17.581567+0000","last_clean":"2023-09-23T21:14:17.581567+0000","last_became_active":"2023-09-23T21:12:10.763486+0000","last_became_peered":"2023-09-23T21:12:10.763486+0000","last_unstale":"2023-09-23T21:14:17.581567+0000","last_undegraded":"2023-09-23T21:14:17.581567+0000","last_fullsized":"2023-09-23T21:14:17.581567+0000","mapping_epoch":470,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:32:27.577036+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,7],"acting":[2,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.10","version":"79'161","reported_seq":662,"reported_epoch":527,"state":"active+clean","last_fresh":"2023-09-23T21:14:17.578957+0000","last_change":"2023-09-23T21:11:11.716703+0000","last_active":"2023-09-23T21:14:17.578957+0000","last_peered":"2023-09-23T21:14:17.578957+0000","last_clean":"2023-09-23T21:14:17.578957+0000","last_became_active":"2023-09-23T21:11:11.716385+0000","last_became_peered":"2023-09-23T21:11:11.716385+0000","last_unstale":"2023-09-23T21:14:17.578957+0000","last_undegraded":"2023-09-23T21:14:17.578957+0000","last_fullsized":"2023-09-23T21:14:17.578957+0000","mapping_epoch":446,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":447,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:06:50.336992+0000","last_deep_scrub":"79'161","last_deep_scrub_stamp":"2023-09-23T21:06:50.336992+0000","last_clean_scrub_stamp":"2023-09-23T21:06:50.336992+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:44:29.538890+0000","scrub_duration":0.0026150930000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,7],"acting":[4,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":467,"reported_epoch":526,"state":"active+clean","last_fresh":"2023-09-23T21:14:09.545798+0000","last_change":"2023-09-23T21:13:12.270990+0000","last_active":"2023-09-23T21:14:09.545798+0000","last_peered":"2023-09-23T21:14:09.545798+0000","last_clean":"2023-09-23T21:14:09.545798+0000","last_became_active":"2023-09-23T21:13:12.237201+0000","last_became_peered":"2023-09-23T21:13:12.237201+0000","last_unstale":"2023-09-23T21:14:09.545798+0000","last_undegraded":"2023-09-23T21:14:09.545798+0000","last_fullsized":"2023-09-23T21:14:09.545798+0000","mapping_epoch":493,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:59:54.523877+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":278,"reported_epoch":526,"state":"active+clean","last_fresh":"2023-09-23T21:14:09.547862+0000","last_change":"2023-09-23T21:10:26.758671+0000","last_active":"2023-09-23T21:14:09.547862+0000","last_peered":"2023-09-23T21:14:09.547862+0000","last_clean":"2023-09-23T21:14:09.547862+0000","last_became_active":"2023-09-23T21:10:26.758244+0000","last_became_peered":"2023-09-23T21:10:26.758244+0000","last_unstale":"2023-09-23T21:14:09.547862+0000","last_undegraded":"2023-09-23T21:14:09.547862+0000","last_fullsized":"2023-09-23T21:14:09.547862+0000","mapping_epoch":423,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":424,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:14:02.735609+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.18","version":"528'5230","reported_seq":6315,"reported_epoch":528,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T21:14:26.161519+0000","last_change":"2023-09-23T21:13:13.412097+0000","last_active":"2023-09-23T21:14:26.161519+0000","last_peered":"2023-09-23T21:14:26.161519+0000","last_clean":"2023-09-23T21:11:45.720015+0000","last_became_active":"2023-09-23T21:13:13.395632+0000","last_became_peered":"2023-09-23T21:13:13.395632+0000","last_unstale":"2023-09-23T21:14:26.161519+0000","last_undegraded":"2023-09-23T21:13:13.220971+0000","last_fullsized":"2023-09-23T21:13:13.183825+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":438,"last_epoch_clean":439,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_clean_scrub_stamp":"2023-09-23T21:10:15.663179+0000","objects_scrubbed":0,"log_size":5230,"log_dups_size":0,"ondisk_log_size":5230,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":75923456,"num_objects":1159,"num_object_clones":0,"num_object_copies":3477,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":661,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1159,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3881,"num_write_kb":161248,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":231,"num_bytes_recovered":15138816,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,7],"acting":[2,7],"avail_no_missing":["2","7"],"object_location_counts":[{"shards":"2,7","objects":1125}],"blocked_by":[],"up_primary":0,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.13","version":"79'161","reported_seq":603,"reported_epoch":526,"state":"active+clean","last_fresh":"2023-09-23T21:14:09.547444+0000","last_change":"2023-09-23T21:13:14.933096+0000","last_active":"2023-09-23T21:14:09.547444+0000","last_peered":"2023-09-23T21:14:09.547444+0000","last_clean":"2023-09-23T21:14:09.547444+0000","last_became_active":"2023-09-23T21:13:14.932703+0000","last_became_peered":"2023-09-23T21:13:14.932703+0000","last_unstale":"2023-09-23T21:14:09.547444+0000","last_undegraded":"2023-09-23T21:14:09.547444+0000","last_fullsized":"2023-09-23T21:14:09.547444+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:36.420362+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:36.420362+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:29:42.163911+0000","scrub_duration":0.028838444000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":519,"reported_epoch":527,"state":"active+clean","last_fresh":"2023-09-23T21:14:17.576722+0000","last_change":"2023-09-23T21:13:12.216467+0000","last_active":"2023-09-23T21:14:17.576722+0000","last_peered":"2023-09-23T21:14:17.576722+0000","last_clean":"2023-09-23T21:14:17.576722+0000","last_became_active":"2023-09-23T21:13:12.215259+0000","last_became_peered":"2023-09-23T21:13:12.215259+0000","last_unstale":"2023-09-23T21:14:17.576722+0000","last_undegraded":"2023-09-23T21:14:17.576722+0000","last_fullsized":"2023-09-23T21:14:17.576722+0000","mapping_epoch":493,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:35:33.987839+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":36,"reported_epoch":526,"state":"active+clean","last_fresh":"2023-09-23T21:14:09.544103+0000","last_change":"2023-09-23T21:13:30.804108+0000","last_active":"2023-09-23T21:14:09.544103+0000","last_peered":"2023-09-23T21:14:09.544103+0000","last_clean":"2023-09-23T21:14:09.544103+0000","last_became_active":"2023-09-23T21:13:30.796505+0000","last_became_peered":"2023-09-23T21:13:30.796505+0000","last_unstale":"2023-09-23T21:14:09.544103+0000","last_undegraded":"2023-09-23T21:14:09.544103+0000","last_fullsized":"2023-09-23T21:14:09.544103+0000","mapping_epoch":501,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:19:33.508496+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.19","version":"528'5377","reported_seq":6476,"reported_epoch":528,"state":"active+clean","last_fresh":"2023-09-23T21:14:26.072956+0000","last_change":"2023-09-23T21:10:58.790716+0000","last_active":"2023-09-23T21:14:26.072956+0000","last_peered":"2023-09-23T21:14:26.072956+0000","last_clean":"2023-09-23T21:14:26.072956+0000","last_became_active":"2023-09-23T21:10:58.787975+0000","last_became_peered":"2023-09-23T21:10:58.787975+0000","last_unstale":"2023-09-23T21:14:26.072956+0000","last_undegraded":"2023-09-23T21:14:26.072956+0000","last_fullsized":"2023-09-23T21:14:26.072956+0000","mapping_epoch":438,"log_start":"0'0","ondisk_log_start":"0'0","created":438,"last_epoch_clean":439,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_clean_scrub_stamp":"2023-09-23T21:10:15.663179+0000","objects_scrubbed":0,"log_size":5377,"log_dups_size":0,"ondisk_log_size":5377,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":72187904,"num_objects":1102,"num_object_clones":0,"num_object_copies":3306,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1102,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3990,"num_write_kb":162912,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":237,"num_bytes_recovered":15499264,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.12","version":"79'161","reported_seq":761,"reported_epoch":527,"state":"active+clean","last_fresh":"2023-09-23T21:14:17.584758+0000","last_change":"2023-09-23T21:13:15.206574+0000","last_active":"2023-09-23T21:14:17.584758+0000","last_peered":"2023-09-23T21:14:17.584758+0000","last_clean":"2023-09-23T21:14:17.584758+0000","last_became_active":"2023-09-23T21:13:15.206186+0000","last_became_peered":"2023-09-23T21:13:15.206186+0000","last_unstale":"2023-09-23T21:14:17.584758+0000","last_undegraded":"2023-09-23T21:14:17.584758+0000","last_fullsized":"2023-09-23T21:14:17.584758+0000","mapping_epoch":496,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":497,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:05:56.785173+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:05:56.785173+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:34:29.221809+0000","scrub_duration":0.0042114370000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,0],"acting":[5,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":46,"reported_epoch":526,"state":"active+clean","last_fresh":"2023-09-23T21:14:09.544181+0000","last_change":"2023-09-23T21:13:38.048791+0000","last_active":"2023-09-23T21:14:09.544181+0000","last_peered":"2023-09-23T21:14:09.544181+0000","last_clean":"2023-09-23T21:14:09.544181+0000","last_became_active":"2023-09-23T21:13:30.801386+0000","last_became_peered":"2023-09-23T21:13:30.801386+0000","last_unstale":"2023-09-23T21:14:09.544181+0000","last_undegraded":"2023-09-23T21:14:09.544181+0000","last_fullsized":"2023-09-23T21:14:09.544181+0000","mapping_epoch":501,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:52:49.170414+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":363,"reported_epoch":526,"state":"active+clean","last_fresh":"2023-09-23T21:14:09.548902+0000","last_change":"2023-09-23T21:12:10.765780+0000","last_active":"2023-09-23T21:14:09.548902+0000","last_peered":"2023-09-23T21:14:09.548902+0000","last_clean":"2023-09-23T21:14:09.548902+0000","last_became_active":"2023-09-23T21:12:10.765372+0000","last_became_peered":"2023-09-23T21:12:10.765372+0000","last_unstale":"2023-09-23T21:14:09.548902+0000","last_undegraded":"2023-09-23T21:14:09.548902+0000","last_fullsized":"2023-09-23T21:14:09.548902+0000","mapping_epoch":470,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:56:08.446099+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.6","version":"527'5372","reported_seq":8288,"reported_epoch":527,"state":"active+clean","last_fresh":"2023-09-23T21:14:25.189907+0000","last_change":"2023-09-23T21:13:39.390285+0000","last_active":"2023-09-23T21:14:25.189907+0000","last_peered":"2023-09-23T21:14:25.189907+0000","last_clean":"2023-09-23T21:14:25.189907+0000","last_became_active":"2023-09-23T21:13:39.378284+0000","last_became_peered":"2023-09-23T21:13:39.378284+0000","last_unstale":"2023-09-23T21:14:25.189907+0000","last_undegraded":"2023-09-23T21:14:25.189907+0000","last_fullsized":"2023-09-23T21:14:25.189907+0000","mapping_epoch":508,"log_start":"0'0","ondisk_log_start":"0'0","created":416,"last_epoch_clean":509,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_clean_scrub_stamp":"2023-09-23T21:10:15.663179+0000","objects_scrubbed":0,"log_size":5372,"log_dups_size":0,"ondisk_log_size":5372,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":72089600,"num_objects":1100,"num_object_clones":0,"num_object_copies":3300,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3966,"num_write_kb":162112,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2399,"num_bytes_recovered":113967104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,4],"acting":[0,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.d","version":"79'161","reported_seq":681,"reported_epoch":527,"state":"active+clean","last_fresh":"2023-09-23T21:14:17.601195+0000","last_change":"2023-09-23T21:12:30.444461+0000","last_active":"2023-09-23T21:14:17.601195+0000","last_peered":"2023-09-23T21:14:17.601195+0000","last_clean":"2023-09-23T21:14:17.601195+0000","last_became_active":"2023-09-23T21:12:30.443979+0000","last_became_peered":"2023-09-23T21:12:30.443979+0000","last_unstale":"2023-09-23T21:14:17.601195+0000","last_undegraded":"2023-09-23T21:14:17.601195+0000","last_fullsized":"2023-09-23T21:14:17.601195+0000","mapping_epoch":470,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:09:36.858077+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:09:36.858077+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0024987500000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":174,"reported_epoch":527,"state":"active+clean","last_fresh":"2023-09-23T21:14:17.601313+0000","last_change":"2023-09-23T21:13:59.872834+0000","last_active":"2023-09-23T21:14:17.601313+0000","last_peered":"2023-09-23T21:14:17.601313+0000","last_clean":"2023-09-23T21:14:17.601313+0000","last_became_active":"2023-09-23T21:13:30.770556+0000","last_became_peered":"2023-09-23T21:13:30.770556+0000","last_unstale":"2023-09-23T21:14:17.601313+0000","last_undegraded":"2023-09-23T21:14:17.601313+0000","last_fullsized":"2023-09-23T21:14:17.601313+0000","mapping_epoch":501,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":521,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:57:55.460052+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":132,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":235,"reported_epoch":527,"state":"active+clean","last_fresh":"2023-09-23T21:14:17.583740+0000","last_change":"2023-09-23T21:10:26.750725+0000","last_active":"2023-09-23T21:14:17.583740+0000","last_peered":"2023-09-23T21:14:17.583740+0000","last_clean":"2023-09-23T21:14:17.583740+0000","last_became_active":"2023-09-23T21:10:26.750307+0000","last_became_peered":"2023-09-23T21:10:26.750307+0000","last_unstale":"2023-09-23T21:14:17.583740+0000","last_undegraded":"2023-09-23T21:14:17.583740+0000","last_fullsized":"2023-09-23T21:14:17.583740+0000","mapping_epoch":423,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":424,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:06:22.040548+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,4],"acting":[5,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.7","version":"527'5230","reported_seq":5424,"reported_epoch":527,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T21:14:24.050640+0000","last_change":"2023-09-23T21:13:38.417178+0000","last_active":"2023-09-23T21:14:24.050640+0000","last_peered":"2023-09-23T21:14:24.050640+0000","last_clean":"2023-09-23T21:11:26.699871+0000","last_became_active":"2023-09-23T21:13:38.408523+0000","last_became_peered":"2023-09-23T21:13:38.408523+0000","last_unstale":"2023-09-23T21:14:24.050640+0000","last_undegraded":"2023-09-23T21:13:38.369833+0000","last_fullsized":"2023-09-23T21:13:38.358198+0000","mapping_epoch":507,"log_start":"422'485","ondisk_log_start":"422'485","created":416,"last_epoch_clean":439,"parent":"0.0","parent_split_bits":0,"last_scrub":"441'2973","last_scrub_stamp":"2023-09-23T21:11:06.079326+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_clean_scrub_stamp":"2023-09-23T21:11:06.079326+0000","objects_scrubbed":1647,"log_size":4745,"log_dups_size":0,"ondisk_log_size":4745,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:27:59.805526+0000","scrub_duration":0.44319714500000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":72941568,"num_objects":1113,"num_object_clones":0,"num_object_copies":3339,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1165,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1113,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2262,"num_write_kb":55296,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,2],"acting":[3,5],"avail_no_missing":["3","5"],"object_location_counts":[{"shards":"3,4,5","objects":298},{"shards":"3,5","objects":815}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.c","version":"79'161","reported_seq":889,"reported_epoch":526,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-23T21:14:09.548383+0000","last_change":"2023-09-23T21:13:13.402530+0000","last_active":"2023-09-23T21:14:09.548383+0000","last_peered":"2023-09-23T21:14:09.548383+0000","last_clean":"2023-09-23T21:11:23.200766+0000","last_became_active":"2023-09-23T21:13:13.401526+0000","last_became_peered":"2023-09-23T21:13:13.401526+0000","last_unstale":"2023-09-23T21:14:09.548383+0000","last_undegraded":"2023-09-23T21:13:13.204992+0000","last_fullsized":"2023-09-23T21:14:09.548383+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":420,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:22.040907+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:22.040907+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.013021176000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":163840,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":163840,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":["3","0","5"],"object_location_counts":[{"shards":"0,3,4","objects":-1},{"shards":"0,3,5","objects":1}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":160,"reported_epoch":527,"state":"active+clean","last_fresh":"2023-09-23T21:14:17.601859+0000","last_change":"2023-09-23T21:13:59.884113+0000","last_active":"2023-09-23T21:14:17.601859+0000","last_peered":"2023-09-23T21:14:17.601859+0000","last_clean":"2023-09-23T21:14:17.601859+0000","last_became_active":"2023-09-23T21:13:30.806778+0000","last_became_peered":"2023-09-23T21:13:30.806778+0000","last_unstale":"2023-09-23T21:14:17.601859+0000","last_undegraded":"2023-09-23T21:14:17.601859+0000","last_fullsized":"2023-09-23T21:14:17.601859+0000","mapping_epoch":501,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":521,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:36:15.284465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":490,"reported_epoch":527,"state":"active+clean","last_fresh":"2023-09-23T21:14:17.576790+0000","last_change":"2023-09-23T21:13:15.152792+0000","last_active":"2023-09-23T21:14:17.576790+0000","last_peered":"2023-09-23T21:14:17.576790+0000","last_clean":"2023-09-23T21:14:17.576790+0000","last_became_active":"2023-09-23T21:13:14.515084+0000","last_became_peered":"2023-09-23T21:13:14.515084+0000","last_unstale":"2023-09-23T21:14:17.576790+0000","last_undegraded":"2023-09-23T21:14:17.576790+0000","last_fullsized":"2023-09-23T21:14:17.576790+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:08:16.702072+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.4","version":"527'5160","reported_seq":12104,"reported_epoch":527,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T21:14:24.100139+0000","last_change":"2023-09-23T21:13:30.847170+0000","last_active":"2023-09-23T21:14:24.100139+0000","last_peered":"2023-09-23T21:14:24.100139+0000","last_clean":"2023-09-23T21:11:06.743842+0000","last_became_active":"2023-09-23T21:13:30.834806+0000","last_became_peered":"2023-09-23T21:13:30.834806+0000","last_unstale":"2023-09-23T21:14:24.100139+0000","last_undegraded":"2023-09-23T21:13:30.760230+0000","last_fullsized":"2023-09-23T21:13:30.749974+0000","mapping_epoch":501,"log_start":"422'326","ondisk_log_start":"422'326","created":416,"last_epoch_clean":439,"parent":"0.0","parent_split_bits":0,"last_scrub":"440'2861","last_scrub_stamp":"2023-09-23T21:11:03.842926+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_clean_scrub_stamp":"2023-09-23T21:11:03.842926+0000","objects_scrubbed":1526,"log_size":4834,"log_dups_size":0,"ondisk_log_size":4834,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:32:29.379756+0000","scrub_duration":0.39378383900000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":76087296,"num_objects":1161,"num_object_clones":0,"num_object_copies":3483,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1130,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1161,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2305,"num_write_kb":62080,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2723,"num_bytes_recovered":163577856,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[3,4],"avail_no_missing":["3","4"],"object_location_counts":[{"shards":"1,2,3,4","objects":360},{"shards":"3,4","objects":801}],"blocked_by":[],"up_primary":0,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.f","version":"79'161","reported_seq":815,"reported_epoch":527,"state":"active+clean","last_fresh":"2023-09-23T21:14:17.585181+0000","last_change":"2023-09-23T21:11:30.750338+0000","last_active":"2023-09-23T21:14:17.585181+0000","last_peered":"2023-09-23T21:14:17.585181+0000","last_clean":"2023-09-23T21:14:17.585181+0000","last_became_active":"2023-09-23T21:11:30.749953+0000","last_became_peered":"2023-09-23T21:11:30.749953+0000","last_unstale":"2023-09-23T21:14:17.585181+0000","last_undegraded":"2023-09-23T21:14:17.585181+0000","last_fullsized":"2023-09-23T21:14:17.585181+0000","mapping_epoch":455,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":456,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:27.880776+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:27.880776+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:07:10.405723+0000","scrub_duration":0.0014004899999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,3],"acting":[5,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":251,"reported_epoch":527,"state":"active+clean","last_fresh":"2023-09-23T21:14:17.584536+0000","last_change":"2023-09-23T21:10:26.765068+0000","last_active":"2023-09-23T21:14:17.584536+0000","last_peered":"2023-09-23T21:14:17.584536+0000","last_clean":"2023-09-23T21:14:17.584536+0000","last_became_active":"2023-09-23T21:10:26.758364+0000","last_became_peered":"2023-09-23T21:10:26.758364+0000","last_unstale":"2023-09-23T21:14:17.584536+0000","last_undegraded":"2023-09-23T21:14:17.584536+0000","last_fullsized":"2023-09-23T21:14:17.584536+0000","mapping_epoch":423,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":424,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:13:21.057529+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":280,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":467,"reported_epoch":527,"state":"active+clean","last_fresh":"2023-09-23T21:14:17.584343+0000","last_change":"2023-09-23T21:13:30.770746+0000","last_active":"2023-09-23T21:14:17.584343+0000","last_peered":"2023-09-23T21:14:17.584343+0000","last_clean":"2023-09-23T21:14:17.584343+0000","last_became_active":"2023-09-23T21:13:30.770437+0000","last_became_peered":"2023-09-23T21:13:30.770437+0000","last_unstale":"2023-09-23T21:14:17.584343+0000","last_undegraded":"2023-09-23T21:14:17.584343+0000","last_fullsized":"2023-09-23T21:14:17.584343+0000","mapping_epoch":501,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:33:48.325730+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,0],"acting":[5,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.5","version":"528'5209","reported_seq":5396,"reported_epoch":528,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T21:14:26.260944+0000","last_change":"2023-09-23T21:13:13.436794+0000","last_active":"2023-09-23T21:14:26.260944+0000","last_peered":"2023-09-23T21:14:26.260944+0000","last_clean":"2023-09-23T21:11:45.768408+0000","last_became_active":"2023-09-23T21:13:13.424939+0000","last_became_peered":"2023-09-23T21:13:13.424939+0000","last_unstale":"2023-09-23T21:14:26.260944+0000","last_undegraded":"2023-09-23T21:13:13.225067+0000","last_fullsized":"2023-09-23T21:13:13.198194+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":416,"last_epoch_clean":439,"parent":"0.0","parent_split_bits":0,"last_scrub":"440'2840","last_scrub_stamp":"2023-09-23T21:11:03.208833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_clean_scrub_stamp":"2023-09-23T21:11:03.208833+0000","objects_scrubbed":1498,"log_size":5209,"log_dups_size":0,"ondisk_log_size":5209,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:56:40.574477+0000","scrub_duration":0.45608471699999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":75628544,"num_objects":1154,"num_object_clones":0,"num_object_copies":3462,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":644,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1154,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2378,"num_write_kb":65088,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,3],"acting":[2,3],"avail_no_missing":["2","3"],"object_location_counts":[{"shards":"2,3","objects":1108}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.e","version":"79'161","reported_seq":696,"reported_epoch":527,"state":"active+clean","last_fresh":"2023-09-23T21:14:17.577102+0000","last_change":"2023-09-23T21:12:30.440952+0000","last_active":"2023-09-23T21:14:17.577102+0000","last_peered":"2023-09-23T21:14:17.577102+0000","last_clean":"2023-09-23T21:14:17.577102+0000","last_became_active":"2023-09-23T21:12:30.440463+0000","last_became_peered":"2023-09-23T21:12:30.440463+0000","last_unstale":"2023-09-23T21:14:17.577102+0000","last_undegraded":"2023-09-23T21:14:17.577102+0000","last_fullsized":"2023-09-23T21:14:17.577102+0000","mapping_epoch":455,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:10:39.567232+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:10:39.567232+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.005229694,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":324,"reported_epoch":526,"state":"active+clean","last_fresh":"2023-09-23T21:14:09.548856+0000","last_change":"2023-09-23T21:13:13.402418+0000","last_active":"2023-09-23T21:14:09.548856+0000","last_peered":"2023-09-23T21:14:09.548856+0000","last_clean":"2023-09-23T21:14:09.548856+0000","last_became_active":"2023-09-23T21:13:13.400984+0000","last_became_peered":"2023-09-23T21:13:13.400984+0000","last_unstale":"2023-09-23T21:14:09.548856+0000","last_undegraded":"2023-09-23T21:14:09.548856+0000","last_fullsized":"2023-09-23T21:14:09.548856+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":495,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:28:01.411191+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,0],"acting":[3,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":499,"reported_epoch":526,"state":"active+clean","last_fresh":"2023-09-23T21:14:09.545240+0000","last_change":"2023-09-23T21:13:14.339670+0000","last_active":"2023-09-23T21:14:09.545240+0000","last_peered":"2023-09-23T21:14:09.545240+0000","last_clean":"2023-09-23T21:14:09.545240+0000","last_became_active":"2023-09-23T21:13:14.339341+0000","last_became_peered":"2023-09-23T21:13:14.339341+0000","last_unstale":"2023-09-23T21:14:09.545240+0000","last_undegraded":"2023-09-23T21:14:09.545240+0000","last_fullsized":"2023-09-23T21:14:09.545240+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:23:19.411368+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.2","version":"528'5307","reported_seq":7100,"reported_epoch":528,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T21:14:28.125100+0000","last_change":"2023-09-23T21:13:13.407366+0000","last_active":"2023-09-23T21:14:28.125100+0000","last_peered":"2023-09-23T21:14:28.125100+0000","last_clean":"2023-09-23T21:11:45.752822+0000","last_became_active":"2023-09-23T21:13:13.398616+0000","last_became_peered":"2023-09-23T21:13:13.398616+0000","last_unstale":"2023-09-23T21:14:28.125100+0000","last_undegraded":"2023-09-23T21:13:13.289815+0000","last_fullsized":"2023-09-23T21:13:13.190270+0000","mapping_epoch":494,"log_start":"422'58","ondisk_log_start":"422'58","created":416,"last_epoch_clean":439,"parent":"0.0","parent_split_bits":0,"last_scrub":"440'3007","last_scrub_stamp":"2023-09-23T21:11:05.553661+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_clean_scrub_stamp":"2023-09-23T21:11:05.553661+0000","objects_scrubbed":1622,"log_size":5249,"log_dups_size":0,"ondisk_log_size":5249,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:51:21.839818+0000","scrub_duration":0.39156952499999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70582272,"num_objects":1077,"num_object_clones":0,"num_object_copies":3231,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2369,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1077,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2305,"num_write_kb":56320,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":796,"num_bytes_recovered":52166656,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,3],"avail_no_missing":["4","1"],"object_location_counts":[{"shards":"0,1,4","objects":1043},{"shards":"1,4","objects":34}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.9","version":"79'161","reported_seq":681,"reported_epoch":527,"state":"active+clean","last_fresh":"2023-09-23T21:14:17.581212+0000","last_change":"2023-09-23T21:12:30.436118+0000","last_active":"2023-09-23T21:14:17.581212+0000","last_peered":"2023-09-23T21:14:17.581212+0000","last_clean":"2023-09-23T21:14:17.581212+0000","last_became_active":"2023-09-23T21:12:30.435779+0000","last_became_peered":"2023-09-23T21:12:30.435779+0000","last_unstale":"2023-09-23T21:14:17.581212+0000","last_undegraded":"2023-09-23T21:14:17.581212+0000","last_fullsized":"2023-09-23T21:14:17.581212+0000","mapping_epoch":433,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:10:37.696951+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:10:37.696951+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.001386504,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,3],"acting":[2,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":85,"reported_epoch":526,"state":"active+clean","last_fresh":"2023-09-23T21:14:09.545402+0000","last_change":"2023-09-23T21:13:30.793273+0000","last_active":"2023-09-23T21:14:09.545402+0000","last_peered":"2023-09-23T21:14:09.545402+0000","last_clean":"2023-09-23T21:14:09.545402+0000","last_became_active":"2023-09-23T21:13:30.791269+0000","last_became_peered":"2023-09-23T21:13:30.791269+0000","last_unstale":"2023-09-23T21:14:09.545402+0000","last_undegraded":"2023-09-23T21:14:09.545402+0000","last_fullsized":"2023-09-23T21:14:09.545402+0000","mapping_epoch":501,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:38:07.142290+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":101,"reported_epoch":527,"state":"active+clean","last_fresh":"2023-09-23T21:14:17.577230+0000","last_change":"2023-09-23T21:12:14.313466+0000","last_active":"2023-09-23T21:14:17.577230+0000","last_peered":"2023-09-23T21:14:17.577230+0000","last_clean":"2023-09-23T21:14:17.577230+0000","last_became_active":"2023-09-23T21:12:14.209837+0000","last_became_peered":"2023-09-23T21:12:14.209837+0000","last_unstale":"2023-09-23T21:14:17.577230+0000","last_undegraded":"2023-09-23T21:14:17.577230+0000","last_fullsized":"2023-09-23T21:14:17.577230+0000","mapping_epoch":470,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":474,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:09:40.375241+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,7],"acting":[4,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.3","version":"528'5318","reported_seq":7449,"reported_epoch":528,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T21:14:26.240022+0000","last_change":"2023-09-23T21:13:45.480705+0000","last_active":"2023-09-23T21:14:26.240022+0000","last_peered":"2023-09-23T21:14:26.240022+0000","last_clean":"2023-09-23T21:11:26.700990+0000","last_became_active":"2023-09-23T21:13:45.471702+0000","last_became_peered":"2023-09-23T21:13:45.471702+0000","last_unstale":"2023-09-23T21:14:26.240022+0000","last_undegraded":"2023-09-23T21:13:45.412179+0000","last_fullsized":"2023-09-23T21:13:45.401701+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":416,"last_epoch_clean":439,"parent":"0.0","parent_split_bits":0,"last_scrub":"440'2873","last_scrub_stamp":"2023-09-23T21:11:02.114783+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_clean_scrub_stamp":"2023-09-23T21:11:02.114783+0000","objects_scrubbed":1532,"log_size":5318,"log_dups_size":0,"ondisk_log_size":5318,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:42:48.549370+0000","scrub_duration":0.32435022600000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":73203712,"num_objects":1117,"num_object_clones":0,"num_object_copies":3351,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1184,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1117,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2451,"num_write_kb":65152,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":865,"num_bytes_recovered":56688640,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5],"avail_no_missing":["2","5"],"object_location_counts":[{"shards":"0,2,4,5","objects":270},{"shards":"2,4,5","objects":48},{"shards":"2,5","objects":798}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.8","version":"79'161","reported_seq":648,"reported_epoch":527,"state":"active+clean","last_fresh":"2023-09-23T21:14:17.581167+0000","last_change":"2023-09-23T21:12:30.433561+0000","last_active":"2023-09-23T21:14:17.581167+0000","last_peered":"2023-09-23T21:14:17.581167+0000","last_clean":"2023-09-23T21:14:17.581167+0000","last_became_active":"2023-09-23T21:12:30.433217+0000","last_became_peered":"2023-09-23T21:12:30.433217+0000","last_unstale":"2023-09-23T21:14:17.581167+0000","last_undegraded":"2023-09-23T21:14:17.581167+0000","last_fullsized":"2023-09-23T21:14:17.581167+0000","mapping_epoch":359,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:06:52.098882+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:06:52.098882+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0092173299999999993,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":131344,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":104,"reported_epoch":527,"state":"active+clean","last_fresh":"2023-09-23T21:14:17.577786+0000","last_change":"2023-09-23T21:12:10.823380+0000","last_active":"2023-09-23T21:14:17.577786+0000","last_peered":"2023-09-23T21:14:17.577786+0000","last_clean":"2023-09-23T21:14:17.577786+0000","last_became_active":"2023-09-23T21:12:10.823044+0000","last_became_peered":"2023-09-23T21:12:10.823044+0000","last_unstale":"2023-09-23T21:14:17.577786+0000","last_undegraded":"2023-09-23T21:14:17.577786+0000","last_fullsized":"2023-09-23T21:14:17.577786+0000","mapping_epoch":470,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:53:29.846536+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,5],"acting":[4,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":105,"reported_epoch":527,"state":"active+clean","last_fresh":"2023-09-23T21:14:17.577966+0000","last_change":"2023-09-23T21:13:30.772451+0000","last_active":"2023-09-23T21:14:17.577966+0000","last_peered":"2023-09-23T21:14:17.577966+0000","last_clean":"2023-09-23T21:14:17.577966+0000","last_became_active":"2023-09-23T21:13:30.772040+0000","last_became_peered":"2023-09-23T21:13:30.772040+0000","last_unstale":"2023-09-23T21:14:17.577966+0000","last_undegraded":"2023-09-23T21:14:17.577966+0000","last_fullsized":"2023-09-23T21:14:17.577966+0000","mapping_epoch":501,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:24:49.246727+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,0],"acting":[4,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.0","version":"528'5292","reported_seq":9725,"reported_epoch":528,"state":"active+recovering+undersized+remapped","last_fresh":"2023-09-23T21:14:28.263596+0000","last_change":"2023-09-23T21:14:02.130787+0000","last_active":"2023-09-23T21:14:28.263596+0000","last_peered":"2023-09-23T21:14:28.263596+0000","last_clean":"2023-09-23T21:11:26.716395+0000","last_became_active":"2023-09-23T21:13:34.557827+0000","last_became_peered":"2023-09-23T21:13:34.557827+0000","last_unstale":"2023-09-23T21:14:28.263596+0000","last_undegraded":"2023-09-23T21:14:28.263596+0000","last_fullsized":"2023-09-23T21:13:31.752840+0000","mapping_epoch":502,"log_start":"422'96","ondisk_log_start":"422'96","created":416,"last_epoch_clean":439,"parent":"0.0","parent_split_bits":0,"last_scrub":"440'2961","last_scrub_stamp":"2023-09-23T21:11:04.564028+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_clean_scrub_stamp":"2023-09-23T21:11:04.564028+0000","objects_scrubbed":1608,"log_size":5196,"log_dups_size":0,"ondisk_log_size":5196,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:43:09.815808+0000","scrub_duration":0.41858158299999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":69599232,"num_objects":1062,"num_object_clones":0,"num_object_copies":3186,"num_objects_missing_on_primary":883,"num_objects_missing":883,"num_objects_degraded":0,"num_objects_misplaced":1058,"num_objects_unfound":0,"num_objects_dirty":1062,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2334,"num_write_kb":57216,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1285,"num_bytes_recovered":84148224,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,0],"avail_no_missing":["1","7"],"object_location_counts":[{"shards":"0,1,4,7","objects":4},{"shards":"0,1,7","objects":732},{"shards":"1,4,7","objects":326}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.b","version":"79'161","reported_seq":530,"reported_epoch":526,"state":"active+clean","last_fresh":"2023-09-23T21:14:09.544740+0000","last_change":"2023-09-23T21:13:13.373313+0000","last_active":"2023-09-23T21:14:09.544740+0000","last_peered":"2023-09-23T21:14:09.544740+0000","last_clean":"2023-09-23T21:14:09.544740+0000","last_became_active":"2023-09-23T21:13:13.372328+0000","last_became_peered":"2023-09-23T21:13:13.372328+0000","last_unstale":"2023-09-23T21:14:09.544740+0000","last_undegraded":"2023-09-23T21:14:09.544740+0000","last_fullsized":"2023-09-23T21:14:09.544740+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":495,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:05:55.772264+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:05:55.772264+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.001862507,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":85,"reported_epoch":527,"state":"active+clean","last_fresh":"2023-09-23T21:14:17.576700+0000","last_change":"2023-09-23T21:11:27.754717+0000","last_active":"2023-09-23T21:14:17.576700+0000","last_peered":"2023-09-23T21:14:17.576700+0000","last_clean":"2023-09-23T21:14:17.576700+0000","last_became_active":"2023-09-23T21:11:27.754376+0000","last_became_peered":"2023-09-23T21:11:27.754376+0000","last_unstale":"2023-09-23T21:14:17.576700+0000","last_undegraded":"2023-09-23T21:14:17.576700+0000","last_fullsized":"2023-09-23T21:14:17.576700+0000","mapping_epoch":452,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":453,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:26:41.734541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":459,"reported_epoch":527,"state":"active+clean","last_fresh":"2023-09-23T21:14:17.578499+0000","last_change":"2023-09-23T21:12:10.811698+0000","last_active":"2023-09-23T21:14:17.578499+0000","last_peered":"2023-09-23T21:14:17.578499+0000","last_clean":"2023-09-23T21:14:17.578499+0000","last_became_active":"2023-09-23T21:12:10.789656+0000","last_became_peered":"2023-09-23T21:12:10.789656+0000","last_unstale":"2023-09-23T21:14:17.578499+0000","last_undegraded":"2023-09-23T21:14:17.578499+0000","last_fullsized":"2023-09-23T21:14:17.578499+0000","mapping_epoch":470,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:39:15.112091+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,5],"acting":[4,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.1","version":"528'5200","reported_seq":11614,"reported_epoch":528,"state":"active+clean","last_fresh":"2023-09-23T21:14:28.011595+0000","last_change":"2023-09-23T21:13:53.483736+0000","last_active":"2023-09-23T21:14:28.011595+0000","last_peered":"2023-09-23T21:14:28.011595+0000","last_clean":"2023-09-23T21:14:28.011595+0000","last_became_active":"2023-09-23T21:13:53.472685+0000","last_became_peered":"2023-09-23T21:13:53.472685+0000","last_unstale":"2023-09-23T21:14:28.011595+0000","last_undegraded":"2023-09-23T21:14:28.011595+0000","last_fullsized":"2023-09-23T21:14:28.011595+0000","mapping_epoch":519,"log_start":"0'0","ondisk_log_start":"0'0","created":416,"last_epoch_clean":520,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_clean_scrub_stamp":"2023-09-23T21:10:15.663179+0000","objects_scrubbed":0,"log_size":5200,"log_dups_size":0,"ondisk_log_size":5200,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":71565312,"num_objects":1092,"num_object_clones":0,"num_object_copies":3276,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1092,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3852,"num_write_kb":158208,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3005,"num_bytes_recovered":156499968,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.a","version":"79'161","reported_seq":674,"reported_epoch":527,"state":"active+clean","last_fresh":"2023-09-23T21:14:17.581321+0000","last_change":"2023-09-23T21:12:30.437790+0000","last_active":"2023-09-23T21:14:17.581321+0000","last_peered":"2023-09-23T21:14:17.581321+0000","last_clean":"2023-09-23T21:14:17.581321+0000","last_became_active":"2023-09-23T21:12:30.437427+0000","last_became_peered":"2023-09-23T21:12:30.437427+0000","last_unstale":"2023-09-23T21:14:17.581321+0000","last_undegraded":"2023-09-23T21:14:17.581321+0000","last_fullsized":"2023-09-23T21:14:17.581321+0000","mapping_epoch":446,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:06:51.075448+0000","last_deep_scrub":"79'161","last_deep_scrub_stamp":"2023-09-23T21:05:32.630292+0000","last_clean_scrub_stamp":"2023-09-23T21:06:51.075448+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0054793890000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,4],"acting":[2,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":373,"reported_epoch":526,"state":"active+clean","last_fresh":"2023-09-23T21:14:09.547078+0000","last_change":"2023-09-23T21:12:10.792918+0000","last_active":"2023-09-23T21:14:09.547078+0000","last_peered":"2023-09-23T21:14:09.547078+0000","last_clean":"2023-09-23T21:14:09.547078+0000","last_became_active":"2023-09-23T21:12:10.792679+0000","last_became_peered":"2023-09-23T21:12:10.792679+0000","last_unstale":"2023-09-23T21:14:09.547078+0000","last_undegraded":"2023-09-23T21:14:09.547078+0000","last_fullsized":"2023-09-23T21:14:09.547078+0000","mapping_epoch":470,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:30:48.008690+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,4],"acting":[3,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":504,"reported_epoch":526,"state":"active+clean","last_fresh":"2023-09-23T21:14:09.549124+0000","last_change":"2023-09-23T21:12:10.790812+0000","last_active":"2023-09-23T21:14:09.549124+0000","last_peered":"2023-09-23T21:14:09.549124+0000","last_clean":"2023-09-23T21:14:09.549124+0000","last_became_active":"2023-09-23T21:12:10.790492+0000","last_became_peered":"2023-09-23T21:12:10.790492+0000","last_unstale":"2023-09-23T21:14:09.549124+0000","last_undegraded":"2023-09-23T21:14:09.549124+0000","last_fullsized":"2023-09-23T21:14:09.549124+0000","mapping_epoch":470,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:02:19.863118+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.c","version":"528'7875","reported_seq":9030,"reported_epoch":528,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T21:14:27.234484+0000","last_change":"2023-09-23T21:13:30.822215+0000","last_active":"2023-09-23T21:14:27.234484+0000","last_peered":"2023-09-23T21:14:27.234484+0000","last_clean":"2023-09-23T21:11:45.750376+0000","last_became_active":"2023-09-23T21:13:13.404327+0000","last_became_peered":"2023-09-23T21:13:13.404327+0000","last_unstale":"2023-09-23T21:14:27.234484+0000","last_undegraded":"2023-09-23T21:13:13.200907+0000","last_fullsized":"2023-09-23T21:13:13.193701+0000","mapping_epoch":494,"log_start":"446'3451","ondisk_log_start":"446'3451","created":416,"last_epoch_clean":434,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_clean_scrub_stamp":"2023-09-23T21:10:15.663179+0000","objects_scrubbed":0,"log_size":4424,"log_dups_size":0,"ondisk_log_size":4424,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:33:42.971375+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":141754368,"num_objects":2163,"num_object_clones":0,"num_object_copies":6489,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1339,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2163,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7875,"num_write_kb":321216,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":498,"num_bytes_recovered":32636928,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4],"avail_no_missing":["7","4"],"object_location_counts":[{"shards":"4,7","objects":2109}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.7","version":"79'161","reported_seq":619,"reported_epoch":526,"state":"active+clean","last_fresh":"2023-09-23T21:14:09.548828+0000","last_change":"2023-09-23T21:13:44.416548+0000","last_active":"2023-09-23T21:14:09.548828+0000","last_peered":"2023-09-23T21:14:09.548828+0000","last_clean":"2023-09-23T21:14:09.548828+0000","last_became_active":"2023-09-23T21:13:44.415617+0000","last_became_peered":"2023-09-23T21:13:44.415617+0000","last_unstale":"2023-09-23T21:14:09.548828+0000","last_undegraded":"2023-09-23T21:14:09.548828+0000","last_fullsized":"2023-09-23T21:14:09.548828+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":3,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:25.916867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:25.916867+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.001665323,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":456,"reported_epoch":526,"state":"active+clean","last_fresh":"2023-09-23T21:14:09.544857+0000","last_change":"2023-09-23T21:13:13.211959+0000","last_active":"2023-09-23T21:14:09.544857+0000","last_peered":"2023-09-23T21:14:09.544857+0000","last_clean":"2023-09-23T21:14:09.544857+0000","last_became_active":"2023-09-23T21:13:13.211712+0000","last_became_peered":"2023-09-23T21:13:13.211712+0000","last_unstale":"2023-09-23T21:14:09.544857+0000","last_undegraded":"2023-09-23T21:14:09.544857+0000","last_fullsized":"2023-09-23T21:14:09.544857+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":495,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:37:21.655775+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,2],"acting":[0,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":205,"reported_epoch":527,"state":"active+clean","last_fresh":"2023-09-23T21:14:17.584448+0000","last_change":"2023-09-23T21:05:04.326889+0000","last_active":"2023-09-23T21:14:17.584448+0000","last_peered":"2023-09-23T21:14:17.584448+0000","last_clean":"2023-09-23T21:14:17.584448+0000","last_became_active":"2023-09-23T21:05:04.326032+0000","last_became_peered":"2023-09-23T21:05:04.326032+0000","last_unstale":"2023-09-23T21:14:17.584448+0000","last_undegraded":"2023-09-23T21:14:17.584448+0000","last_fullsized":"2023-09-23T21:14:17.584448+0000","mapping_epoch":332,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:00:17.507889+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":537,"reported_epoch":527,"state":"active+clean","last_fresh":"2023-09-23T21:14:17.584499+0000","last_change":"2023-09-23T21:13:13.285608+0000","last_active":"2023-09-23T21:14:17.584499+0000","last_peered":"2023-09-23T21:14:17.584499+0000","last_clean":"2023-09-23T21:14:17.584499+0000","last_became_active":"2023-09-23T21:13:13.285255+0000","last_became_peered":"2023-09-23T21:13:13.285255+0000","last_unstale":"2023-09-23T21:14:17.584499+0000","last_undegraded":"2023-09-23T21:14:17.584499+0000","last_fullsized":"2023-09-23T21:14:17.584499+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":495,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:42:30.087599+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.a","version":"528'7664","reported_seq":16399,"reported_epoch":528,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T21:14:26.176695+0000","last_change":"2023-09-23T21:13:31.927282+0000","last_active":"2023-09-23T21:14:26.176695+0000","last_peered":"2023-09-23T21:14:26.176695+0000","last_clean":"2023-09-23T21:13:30.353360+0000","last_became_active":"2023-09-23T21:13:31.896923+0000","last_became_peered":"2023-09-23T21:13:31.896923+0000","last_unstale":"2023-09-23T21:14:26.176695+0000","last_undegraded":"2023-09-23T21:13:31.776904+0000","last_fullsized":"2023-09-23T21:13:31.751334+0000","mapping_epoch":502,"log_start":"446'3312","ondisk_log_start":"446'3312","created":416,"last_epoch_clean":499,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_clean_scrub_stamp":"2023-09-23T21:10:15.663179+0000","objects_scrubbed":0,"log_size":4352,"log_dups_size":0,"ondisk_log_size":4352,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:23:14.477602+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":144703488,"num_objects":2208,"num_object_clones":0,"num_object_copies":6624,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2243,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2208,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7664,"num_write_kb":315904,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4293,"num_bytes_recovered":281346048,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[2,5],"avail_no_missing":["2","3","5"],"object_location_counts":[{"shards":"2,3,5","objects":643},{"shards":"2,5","objects":1565}],"blocked_by":[],"up_primary":0,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.1","version":"79'161","reported_seq":597,"reported_epoch":526,"state":"active+clean","last_fresh":"2023-09-23T21:14:09.545284+0000","last_change":"2023-09-23T21:13:12.204135+0000","last_active":"2023-09-23T21:14:09.545284+0000","last_peered":"2023-09-23T21:14:09.545284+0000","last_clean":"2023-09-23T21:14:09.545284+0000","last_became_active":"2023-09-23T21:13:12.203380+0000","last_became_peered":"2023-09-23T21:13:12.203380+0000","last_unstale":"2023-09-23T21:14:09.545284+0000","last_undegraded":"2023-09-23T21:14:09.545284+0000","last_fullsized":"2023-09-23T21:14:09.545284+0000","mapping_epoch":493,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:06:52.575900+0000","last_deep_scrub":"79'161","last_deep_scrub_stamp":"2023-09-23T21:05:30.652189+0000","last_clean_scrub_stamp":"2023-09-23T21:06:52.575900+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":193,"reported_epoch":526,"state":"active+clean","last_fresh":"2023-09-23T21:14:09.545633+0000","last_change":"2023-09-23T21:13:13.238617+0000","last_active":"2023-09-23T21:14:09.545633+0000","last_peered":"2023-09-23T21:14:09.545633+0000","last_clean":"2023-09-23T21:14:09.545633+0000","last_became_active":"2023-09-23T21:13:13.232926+0000","last_became_peered":"2023-09-23T21:13:13.232926+0000","last_unstale":"2023-09-23T21:14:09.545633+0000","last_undegraded":"2023-09-23T21:14:09.545633+0000","last_fullsized":"2023-09-23T21:14:09.545633+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":495,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:17:21.823772+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":461,"reported_epoch":527,"state":"active+clean","last_fresh":"2023-09-23T21:14:17.581156+0000","last_change":"2023-09-23T21:13:14.144918+0000","last_active":"2023-09-23T21:14:17.581156+0000","last_peered":"2023-09-23T21:14:17.581156+0000","last_clean":"2023-09-23T21:14:17.581156+0000","last_became_active":"2023-09-23T21:13:14.132388+0000","last_became_peered":"2023-09-23T21:13:14.132388+0000","last_unstale":"2023-09-23T21:14:17.581156+0000","last_undegraded":"2023-09-23T21:14:17.581156+0000","last_fullsized":"2023-09-23T21:14:17.581156+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":495,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:56:02.619905+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,5],"acting":[4,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":348,"reported_epoch":527,"state":"active+clean","last_fresh":"2023-09-23T21:14:17.581779+0000","last_change":"2023-09-23T21:12:10.764503+0000","last_active":"2023-09-23T21:14:17.581779+0000","last_peered":"2023-09-23T21:14:17.581779+0000","last_clean":"2023-09-23T21:14:17.581779+0000","last_became_active":"2023-09-23T21:12:10.763933+0000","last_became_peered":"2023-09-23T21:12:10.763933+0000","last_unstale":"2023-09-23T21:14:17.581779+0000","last_undegraded":"2023-09-23T21:14:17.581779+0000","last_fullsized":"2023-09-23T21:14:17.581779+0000","mapping_epoch":470,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:28:33.202825+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,7],"acting":[2,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"10.9","version":"528'5268","reported_seq":6411,"reported_epoch":528,"state":"active+clean","last_fresh":"2023-09-23T21:14:26.255113+0000","last_change":"2023-09-23T21:11:08.062606+0000","last_active":"2023-09-23T21:14:26.255113+0000","last_peered":"2023-09-23T21:14:26.255113+0000","last_clean":"2023-09-23T21:14:26.255113+0000","last_became_active":"2023-09-23T21:10:58.780961+0000","last_became_peered":"2023-09-23T21:10:58.780961+0000","last_unstale":"2023-09-23T21:14:26.255113+0000","last_undegraded":"2023-09-23T21:14:26.255113+0000","last_fullsized":"2023-09-23T21:14:26.255113+0000","mapping_epoch":428,"log_start":"0'0","ondisk_log_start":"0'0","created":416,"last_epoch_clean":439,"parent":"0.0","parent_split_bits":0,"last_scrub":"443'3049","last_scrub_stamp":"2023-09-23T21:11:08.062479+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_clean_scrub_stamp":"2023-09-23T21:11:08.062479+0000","objects_scrubbed":1623,"log_size":5268,"log_dups_size":0,"ondisk_log_size":5268,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:56:22.338088+0000","scrub_duration":0.28207626200000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":71106560,"num_objects":1085,"num_object_clones":0,"num_object_copies":3255,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1085,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2220,"num_write_kb":53824,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.2","version":"79'161","reported_seq":806,"reported_epoch":526,"state":"active+clean","last_fresh":"2023-09-23T21:14:09.545334+0000","last_change":"2023-09-23T21:13:12.209907+0000","last_active":"2023-09-23T21:14:09.545334+0000","last_peered":"2023-09-23T21:14:09.545334+0000","last_clean":"2023-09-23T21:14:09.545334+0000","last_became_active":"2023-09-23T21:13:12.205664+0000","last_became_peered":"2023-09-23T21:13:12.205664+0000","last_unstale":"2023-09-23T21:14:09.545334+0000","last_undegraded":"2023-09-23T21:14:09.545334+0000","last_fullsized":"2023-09-23T21:14:09.545334+0000","mapping_epoch":493,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:24.802451+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:24.802451+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:26:16.035788+0000","scrub_duration":0.0090861250000000005,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,4],"acting":[0,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":536,"reported_epoch":527,"state":"active+clean","last_fresh":"2023-09-23T21:14:17.581572+0000","last_change":"2023-09-23T21:13:12.206980+0000","last_active":"2023-09-23T21:14:17.581572+0000","last_peered":"2023-09-23T21:14:17.581572+0000","last_clean":"2023-09-23T21:14:17.581572+0000","last_became_active":"2023-09-23T21:13:12.206650+0000","last_became_peered":"2023-09-23T21:13:12.206650+0000","last_unstale":"2023-09-23T21:14:17.581572+0000","last_undegraded":"2023-09-23T21:14:17.581572+0000","last_fullsized":"2023-09-23T21:14:17.581572+0000","mapping_epoch":493,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:10:01.892936+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":520,"reported_epoch":527,"state":"active+clean","last_fresh":"2023-09-23T21:14:17.582437+0000","last_change":"2023-09-23T21:13:13.407073+0000","last_active":"2023-09-23T21:14:17.582437+0000","last_peered":"2023-09-23T21:14:17.582437+0000","last_clean":"2023-09-23T21:14:17.582437+0000","last_became_active":"2023-09-23T21:13:13.395047+0000","last_became_peered":"2023-09-23T21:13:13.395047+0000","last_unstale":"2023-09-23T21:14:17.582437+0000","last_undegraded":"2023-09-23T21:14:17.582437+0000","last_fullsized":"2023-09-23T21:14:17.582437+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":495,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:04:43.384190+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,5],"acting":[2,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":36,"reported_epoch":526,"state":"active+clean","last_fresh":"2023-09-23T21:14:09.544811+0000","last_change":"2023-09-23T21:13:30.785054+0000","last_active":"2023-09-23T21:14:09.544811+0000","last_peered":"2023-09-23T21:14:09.544811+0000","last_clean":"2023-09-23T21:14:09.544811+0000","last_became_active":"2023-09-23T21:13:30.784701+0000","last_became_peered":"2023-09-23T21:13:30.784701+0000","last_unstale":"2023-09-23T21:14:09.544811+0000","last_undegraded":"2023-09-23T21:14:09.544811+0000","last_fullsized":"2023-09-23T21:14:09.544811+0000","mapping_epoch":501,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:19:20.152252+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.b","version":"528'7997","reported_seq":16188,"reported_epoch":528,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2023-09-23T21:14:28.690896+0000","last_change":"2023-09-23T21:13:59.891994+0000","last_active":"2023-09-23T21:14:28.690896+0000","last_peered":"2023-09-23T21:14:28.690896+0000","last_clean":"2023-09-23T21:11:26.732248+0000","last_became_active":"2023-09-23T21:13:14.937458+0000","last_became_peered":"2023-09-23T21:13:14.937458+0000","last_unstale":"2023-09-23T21:14:28.690896+0000","last_undegraded":"2023-09-23T21:13:59.884123+0000","last_fullsized":"2023-09-23T21:14:28.690896+0000","mapping_epoch":494,"log_start":"426'600","ondisk_log_start":"426'600","created":416,"last_epoch_clean":430,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_clean_scrub_stamp":"2023-09-23T21:10:15.663179+0000","objects_scrubbed":0,"log_size":7397,"log_dups_size":0,"ondisk_log_size":7397,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:36:15.760947+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":137297920,"num_objects":2095,"num_object_clones":0,"num_object_copies":6285,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1337,"num_objects_misplaced":1427,"num_objects_unfound":0,"num_objects_dirty":2095,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7997,"num_write_kb":322944,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4005,"num_bytes_recovered":262275072,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[1,4,7],"avail_no_missing":["1","7"],"object_location_counts":[{"shards":"0,1,7","objects":1419},{"shards":"1,4,7","objects":601}],"blocked_by":[],"up_primary":0,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":313,"reported_epoch":527,"state":"active+clean","last_fresh":"2023-09-23T21:14:17.584330+0000","last_change":"2023-09-23T21:10:26.769764+0000","last_active":"2023-09-23T21:14:17.584330+0000","last_peered":"2023-09-23T21:14:17.584330+0000","last_clean":"2023-09-23T21:14:17.584330+0000","last_became_active":"2023-09-23T21:10:26.769326+0000","last_became_peered":"2023-09-23T21:10:26.769326+0000","last_unstale":"2023-09-23T21:14:17.584330+0000","last_undegraded":"2023-09-23T21:14:17.584330+0000","last_fullsized":"2023-09-23T21:14:17.584330+0000","mapping_epoch":423,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":424,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:57:45.219837+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"79'161","reported_seq":709,"reported_epoch":526,"state":"active+clean","last_fresh":"2023-09-23T21:14:09.543698+0000","last_change":"2023-09-23T21:13:33.766895+0000","last_active":"2023-09-23T21:14:09.543698+0000","last_peered":"2023-09-23T21:14:09.543698+0000","last_clean":"2023-09-23T21:14:09.543698+0000","last_became_active":"2023-09-23T21:13:33.766538+0000","last_became_peered":"2023-09-23T21:13:33.766538+0000","last_unstale":"2023-09-23T21:14:09.543698+0000","last_undegraded":"2023-09-23T21:14:09.543698+0000","last_fullsized":"2023-09-23T21:14:09.543698+0000","mapping_epoch":504,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":505,"parent":"0.0","parent_split_bits":0,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:20.738273+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:20.738273+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:35:35.855987+0000","scrub_duration":0.0088336339999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,2],"acting":[0,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":535,"reported_epoch":527,"state":"active+clean","last_fresh":"2023-09-23T21:14:17.584425+0000","last_change":"2023-09-23T21:11:08.760417+0000","last_active":"2023-09-23T21:14:17.584425+0000","last_peered":"2023-09-23T21:14:17.584425+0000","last_clean":"2023-09-23T21:14:17.584425+0000","last_became_active":"2023-09-23T21:11:08.760150+0000","last_became_peered":"2023-09-23T21:11:08.760150+0000","last_unstale":"2023-09-23T21:14:17.584425+0000","last_undegraded":"2023-09-23T21:14:17.584425+0000","last_fullsized":"2023-09-23T21:14:17.584425+0000","mapping_epoch":443,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":444,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:23:10.849768+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":102,"reported_epoch":527,"state":"active+clean","last_fresh":"2023-09-23T21:14:17.578107+0000","last_change":"2023-09-23T21:13:12.216020+0000","last_active":"2023-09-23T21:14:17.578107+0000","last_peered":"2023-09-23T21:14:17.578107+0000","last_clean":"2023-09-23T21:14:17.578107+0000","last_became_active":"2023-09-23T21:13:12.215180+0000","last_became_peered":"2023-09-23T21:13:12.215180+0000","last_unstale":"2023-09-23T21:14:17.578107+0000","last_undegraded":"2023-09-23T21:14:17.578107+0000","last_fullsized":"2023-09-23T21:14:17.578107+0000","mapping_epoch":493,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:26:18.704684+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.8","version":"527'5140","reported_seq":7533,"reported_epoch":527,"state":"active+clean","last_fresh":"2023-09-23T21:14:25.142044+0000","last_change":"2023-09-23T21:14:03.796930+0000","last_active":"2023-09-23T21:14:25.142044+0000","last_peered":"2023-09-23T21:14:25.142044+0000","last_clean":"2023-09-23T21:14:25.142044+0000","last_became_active":"2023-09-23T21:14:03.786177+0000","last_became_peered":"2023-09-23T21:14:03.786177+0000","last_unstale":"2023-09-23T21:14:25.142044+0000","last_undegraded":"2023-09-23T21:14:25.142044+0000","last_fullsized":"2023-09-23T21:14:25.142044+0000","mapping_epoch":524,"log_start":"0'0","ondisk_log_start":"0'0","created":416,"last_epoch_clean":525,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_clean_scrub_stamp":"2023-09-23T21:10:15.663179+0000","objects_scrubbed":0,"log_size":5140,"log_dups_size":0,"ondisk_log_size":5140,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":68059136,"num_objects":1038,"num_object_clones":0,"num_object_copies":3114,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1038,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3790,"num_write_kb":154528,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":880,"num_bytes_recovered":16908288,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,7],"acting":[0,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.3","version":"79'161","reported_seq":830,"reported_epoch":527,"state":"active+clean","last_fresh":"2023-09-23T21:14:17.577313+0000","last_change":"2023-09-23T21:13:33.771279+0000","last_active":"2023-09-23T21:14:17.577313+0000","last_peered":"2023-09-23T21:14:17.577313+0000","last_clean":"2023-09-23T21:14:17.577313+0000","last_became_active":"2023-09-23T21:13:33.770980+0000","last_became_peered":"2023-09-23T21:13:33.770980+0000","last_unstale":"2023-09-23T21:14:17.577313+0000","last_undegraded":"2023-09-23T21:14:17.577313+0000","last_fullsized":"2023-09-23T21:14:17.577313+0000","mapping_epoch":504,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":505,"parent":"0.0","parent_split_bits":3,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:05:57.832302+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:05:57.832302+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:19:42.309604+0000","scrub_duration":0.010206847999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":343,"reported_epoch":526,"state":"active+clean","last_fresh":"2023-09-23T21:14:09.544262+0000","last_change":"2023-09-23T21:13:13.212775+0000","last_active":"2023-09-23T21:14:09.544262+0000","last_peered":"2023-09-23T21:14:09.544262+0000","last_clean":"2023-09-23T21:14:09.544262+0000","last_became_active":"2023-09-23T21:13:13.212460+0000","last_became_peered":"2023-09-23T21:13:13.212460+0000","last_unstale":"2023-09-23T21:14:09.544262+0000","last_undegraded":"2023-09-23T21:14:09.544262+0000","last_fullsized":"2023-09-23T21:14:09.544262+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":495,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:27:10.853970+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":525,"reported_epoch":526,"state":"active+clean","last_fresh":"2023-09-23T21:14:09.545031+0000","last_change":"2023-09-23T21:13:44.409212+0000","last_active":"2023-09-23T21:14:09.545031+0000","last_peered":"2023-09-23T21:14:09.545031+0000","last_clean":"2023-09-23T21:14:09.545031+0000","last_became_active":"2023-09-23T21:13:44.408958+0000","last_became_peered":"2023-09-23T21:13:44.408958+0000","last_unstale":"2023-09-23T21:14:09.545031+0000","last_undegraded":"2023-09-23T21:14:09.545031+0000","last_fullsized":"2023-09-23T21:14:09.545031+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:03:52.749964+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":36,"reported_epoch":526,"state":"active+clean","last_fresh":"2023-09-23T21:14:09.544260+0000","last_change":"2023-09-23T21:13:30.804379+0000","last_active":"2023-09-23T21:14:09.544260+0000","last_peered":"2023-09-23T21:14:09.544260+0000","last_clean":"2023-09-23T21:14:09.544260+0000","last_became_active":"2023-09-23T21:13:30.799907+0000","last_became_peered":"2023-09-23T21:13:30.799907+0000","last_unstale":"2023-09-23T21:14:09.544260+0000","last_undegraded":"2023-09-23T21:14:09.544260+0000","last_fullsized":"2023-09-23T21:14:09.544260+0000","mapping_epoch":501,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:49:42.859212+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,7],"acting":[0,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.d","version":"528'7863","reported_seq":8024,"reported_epoch":528,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-23T21:14:28.279271+0000","last_change":"2023-09-23T21:13:31.801486+0000","last_active":"2023-09-23T21:14:28.279271+0000","last_peered":"2023-09-23T21:14:28.279271+0000","last_clean":"2023-09-23T21:11:37.659929+0000","last_became_active":"2023-09-23T21:13:31.786626+0000","last_became_peered":"2023-09-23T21:13:31.786626+0000","last_unstale":"2023-09-23T21:14:28.279271+0000","last_undegraded":"2023-09-23T21:13:31.760908+0000","last_fullsized":"2023-09-23T21:14:28.279271+0000","mapping_epoch":502,"log_start":"448'3895","ondisk_log_start":"448'3895","created":416,"last_epoch_clean":417,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_clean_scrub_stamp":"2023-09-23T21:10:15.663179+0000","objects_scrubbed":0,"log_size":3968,"log_dups_size":0,"ondisk_log_size":3968,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:46:02.424493+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":140115992,"num_objects":2139,"num_object_clones":0,"num_object_copies":6417,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1265,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2139,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":7863,"num_write_kb":320001,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,3],"avail_no_missing":["4","3","7"],"object_location_counts":[{"shards":"0,3,4","objects":2060},{"shards":"3,4","objects":79}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.6","version":"79'161","reported_seq":445,"reported_epoch":526,"state":"active+clean","last_fresh":"2023-09-23T21:14:09.544185+0000","last_change":"2023-09-23T21:13:33.770715+0000","last_active":"2023-09-23T21:14:09.544185+0000","last_peered":"2023-09-23T21:14:09.544185+0000","last_clean":"2023-09-23T21:14:09.544185+0000","last_became_active":"2023-09-23T21:13:33.769303+0000","last_became_peered":"2023-09-23T21:13:33.769303+0000","last_unstale":"2023-09-23T21:14:09.544185+0000","last_undegraded":"2023-09-23T21:14:09.544185+0000","last_fullsized":"2023-09-23T21:14:09.544185+0000","mapping_epoch":504,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":505,"parent":"0.0","parent_split_bits":3,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:10:24.268662+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:10:24.268662+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13417,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":11,"num_read_kb":20,"num_write":5,"num_write_kb":55,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":40252,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":154,"reported_epoch":527,"state":"active+clean","last_fresh":"2023-09-23T21:14:17.601016+0000","last_change":"2023-09-23T21:13:30.770826+0000","last_active":"2023-09-23T21:14:17.601016+0000","last_peered":"2023-09-23T21:14:17.601016+0000","last_clean":"2023-09-23T21:14:17.601016+0000","last_became_active":"2023-09-23T21:13:30.770512+0000","last_became_peered":"2023-09-23T21:13:30.770512+0000","last_unstale":"2023-09-23T21:14:17.601016+0000","last_undegraded":"2023-09-23T21:14:17.601016+0000","last_fullsized":"2023-09-23T21:14:17.601016+0000","mapping_epoch":501,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:44:23.470538+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":411,"reported_epoch":527,"state":"active+clean","last_fresh":"2023-09-23T21:14:17.582023+0000","last_change":"2023-09-23T21:13:30.790590+0000","last_active":"2023-09-23T21:14:17.582023+0000","last_peered":"2023-09-23T21:14:17.582023+0000","last_clean":"2023-09-23T21:14:17.582023+0000","last_became_active":"2023-09-23T21:13:30.790253+0000","last_became_peered":"2023-09-23T21:13:30.790253+0000","last_unstale":"2023-09-23T21:14:17.582023+0000","last_undegraded":"2023-09-23T21:14:17.582023+0000","last_fullsized":"2023-09-23T21:14:17.582023+0000","mapping_epoch":501,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:23:41.843250+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,0],"acting":[2,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":472,"reported_epoch":526,"state":"active+clean","last_fresh":"2023-09-23T21:14:09.545521+0000","last_change":"2023-09-23T21:13:12.210546+0000","last_active":"2023-09-23T21:14:09.545521+0000","last_peered":"2023-09-23T21:14:09.545521+0000","last_clean":"2023-09-23T21:14:09.545521+0000","last_became_active":"2023-09-23T21:13:12.205934+0000","last_became_peered":"2023-09-23T21:13:12.205934+0000","last_unstale":"2023-09-23T21:14:09.545521+0000","last_undegraded":"2023-09-23T21:14:09.545521+0000","last_fullsized":"2023-09-23T21:14:09.545521+0000","mapping_epoch":493,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:21:18.299428+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.e","version":"528'8062","reported_seq":15032,"reported_epoch":528,"state":"active+clean","last_fresh":"2023-09-23T21:14:28.259323+0000","last_change":"2023-09-23T21:12:38.623611+0000","last_active":"2023-09-23T21:14:28.259323+0000","last_peered":"2023-09-23T21:14:28.259323+0000","last_clean":"2023-09-23T21:14:28.259323+0000","last_became_active":"2023-09-23T21:12:38.610155+0000","last_became_peered":"2023-09-23T21:12:38.610155+0000","last_unstale":"2023-09-23T21:14:28.259323+0000","last_undegraded":"2023-09-23T21:14:28.259323+0000","last_fullsized":"2023-09-23T21:14:28.259323+0000","mapping_epoch":483,"log_start":"450'4093","ondisk_log_start":"450'4093","created":416,"last_epoch_clean":484,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_clean_scrub_stamp":"2023-09-23T21:10:15.663179+0000","objects_scrubbed":0,"log_size":3969,"log_dups_size":0,"ondisk_log_size":3969,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:19:27.097170+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":141688832,"num_objects":2162,"num_object_clones":0,"num_object_copies":6486,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2162,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8062,"num_write_kb":327168,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3412,"num_bytes_recovered":223608832,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.5","version":"79'161","reported_seq":555,"reported_epoch":527,"state":"active+clean","last_fresh":"2023-09-23T21:14:17.578037+0000","last_change":"2023-09-23T21:13:38.373969+0000","last_active":"2023-09-23T21:14:17.578037+0000","last_peered":"2023-09-23T21:14:17.578037+0000","last_clean":"2023-09-23T21:14:17.578037+0000","last_became_active":"2023-09-23T21:13:38.373633+0000","last_became_peered":"2023-09-23T21:13:38.373633+0000","last_unstale":"2023-09-23T21:14:17.578037+0000","last_undegraded":"2023-09-23T21:14:17.578037+0000","last_fullsized":"2023-09-23T21:14:17.578037+0000","mapping_epoch":507,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":3,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:09:34.892947+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:09:34.892947+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13417,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":11,"num_read_kb":20,"num_write":5,"num_write_kb":55,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":40252,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":67,"reported_epoch":526,"state":"active+clean","last_fresh":"2023-09-23T21:14:09.545047+0000","last_change":"2023-09-23T21:13:12.235743+0000","last_active":"2023-09-23T21:14:09.545047+0000","last_peered":"2023-09-23T21:14:09.545047+0000","last_clean":"2023-09-23T21:14:09.545047+0000","last_became_active":"2023-09-23T21:13:12.216241+0000","last_became_peered":"2023-09-23T21:13:12.216241+0000","last_unstale":"2023-09-23T21:14:09.545047+0000","last_undegraded":"2023-09-23T21:14:09.545047+0000","last_fullsized":"2023-09-23T21:14:09.545047+0000","mapping_epoch":493,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:12:19.090710+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":508,"reported_epoch":527,"state":"active+clean","last_fresh":"2023-09-23T21:14:17.578382+0000","last_change":"2023-09-23T21:13:30.807520+0000","last_active":"2023-09-23T21:14:17.578382+0000","last_peered":"2023-09-23T21:14:17.578382+0000","last_clean":"2023-09-23T21:14:17.578382+0000","last_became_active":"2023-09-23T21:13:30.807222+0000","last_became_peered":"2023-09-23T21:13:30.807222+0000","last_unstale":"2023-09-23T21:14:17.578382+0000","last_undegraded":"2023-09-23T21:14:17.578382+0000","last_fullsized":"2023-09-23T21:14:17.578382+0000","mapping_epoch":501,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:40:31.989749+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":368,"reported_epoch":526,"state":"active+clean","last_fresh":"2023-09-23T21:14:09.547075+0000","last_change":"2023-09-23T21:13:30.794095+0000","last_active":"2023-09-23T21:14:09.547075+0000","last_peered":"2023-09-23T21:14:09.547075+0000","last_clean":"2023-09-23T21:14:09.547075+0000","last_became_active":"2023-09-23T21:13:30.793791+0000","last_became_peered":"2023-09-23T21:13:30.793791+0000","last_unstale":"2023-09-23T21:14:09.547075+0000","last_undegraded":"2023-09-23T21:14:09.547075+0000","last_fullsized":"2023-09-23T21:14:09.547075+0000","mapping_epoch":501,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:36:28.707362+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.f","version":"527'7840","reported_seq":8042,"reported_epoch":527,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T21:14:24.043875+0000","last_change":"2023-09-23T21:13:59.874192+0000","last_active":"2023-09-23T21:14:24.043875+0000","last_peered":"2023-09-23T21:14:24.043875+0000","last_clean":"2023-09-23T21:11:07.652290+0000","last_became_active":"2023-09-23T21:11:09.897463+0000","last_became_peered":"2023-09-23T21:11:09.897463+0000","last_unstale":"2023-09-23T21:14:24.043875+0000","last_undegraded":"2023-09-23T21:11:27.781677+0000","last_fullsized":"2023-09-23T21:11:09.761652+0000","mapping_epoch":444,"log_start":"450'4109","ondisk_log_start":"450'4109","created":416,"last_epoch_clean":417,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_clean_scrub_stamp":"2023-09-23T21:10:15.663179+0000","objects_scrubbed":0,"log_size":3731,"log_dups_size":0,"ondisk_log_size":3731,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:42:51.588441+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":149028864,"num_objects":2274,"num_object_clones":0,"num_object_copies":6822,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3295,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2274,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7840,"num_write_kb":323648,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":32,"num_bytes_recovered":2097152,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2],"avail_no_missing":["3","1","2"],"object_location_counts":[{"shards":"1,2,3","objects":544},{"shards":"2,3","objects":-1021}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.4","version":"79'161","reported_seq":897,"reported_epoch":526,"state":"active+clean","last_fresh":"2023-09-23T21:14:09.548560+0000","last_change":"2023-09-23T21:10:29.807785+0000","last_active":"2023-09-23T21:14:09.548560+0000","last_peered":"2023-09-23T21:14:09.548560+0000","last_clean":"2023-09-23T21:14:09.548560+0000","last_became_active":"2023-09-23T21:10:29.807366+0000","last_became_peered":"2023-09-23T21:10:29.807366+0000","last_unstale":"2023-09-23T21:14:09.548560+0000","last_undegraded":"2023-09-23T21:14:09.548560+0000","last_fullsized":"2023-09-23T21:14:09.548560+0000","mapping_epoch":426,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":427,"parent":"0.0","parent_split_bits":3,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:20.012764+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:20.012764+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:24:45.763834+0000","scrub_duration":0.0045251919999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":466,"reported_epoch":527,"state":"active+clean","last_fresh":"2023-09-23T21:14:17.582742+0000","last_change":"2023-09-23T21:10:26.768364+0000","last_active":"2023-09-23T21:14:17.582742+0000","last_peered":"2023-09-23T21:14:17.582742+0000","last_clean":"2023-09-23T21:14:17.582742+0000","last_became_active":"2023-09-23T21:10:26.768052+0000","last_became_peered":"2023-09-23T21:10:26.768052+0000","last_unstale":"2023-09-23T21:14:17.582742+0000","last_undegraded":"2023-09-23T21:14:17.582742+0000","last_fullsized":"2023-09-23T21:14:17.582742+0000","mapping_epoch":423,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":424,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:22:42.201200+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,3],"acting":[2,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.6","version":"70'1","reported_seq":497,"reported_epoch":526,"state":"active+clean","last_fresh":"2023-09-23T21:14:09.544498+0000","last_change":"2023-09-23T21:13:30.793290+0000","last_active":"2023-09-23T21:14:09.544498+0000","last_peered":"2023-09-23T21:14:09.544498+0000","last_clean":"2023-09-23T21:14:09.544498+0000","last_became_active":"2023-09-23T21:13:30.786079+0000","last_became_peered":"2023-09-23T21:13:30.786079+0000","last_unstale":"2023-09-23T21:14:09.544498+0000","last_undegraded":"2023-09-23T21:14:09.544498+0000","last_fullsized":"2023-09-23T21:14:09.544498+0000","mapping_epoch":501,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:34:43.281095+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":102,"reported_epoch":527,"state":"active+clean","last_fresh":"2023-09-23T21:14:17.576637+0000","last_change":"2023-09-23T21:13:12.216895+0000","last_active":"2023-09-23T21:14:17.576637+0000","last_peered":"2023-09-23T21:14:17.576637+0000","last_clean":"2023-09-23T21:14:17.576637+0000","last_became_active":"2023-09-23T21:13:12.216654+0000","last_became_peered":"2023-09-23T21:13:12.216654+0000","last_unstale":"2023-09-23T21:14:17.576637+0000","last_undegraded":"2023-09-23T21:14:17.576637+0000","last_fullsized":"2023-09-23T21:14:17.576637+0000","mapping_epoch":493,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:02:40.430096+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.19","version":"79'161","reported_seq":680,"reported_epoch":527,"state":"active+clean","last_fresh":"2023-09-23T21:14:17.580980+0000","last_change":"2023-09-23T21:12:30.442914+0000","last_active":"2023-09-23T21:14:17.580980+0000","last_peered":"2023-09-23T21:14:17.580980+0000","last_clean":"2023-09-23T21:14:17.580980+0000","last_became_active":"2023-09-23T21:12:30.442578+0000","last_became_peered":"2023-09-23T21:12:30.442578+0000","last_unstale":"2023-09-23T21:14:17.580980+0000","last_undegraded":"2023-09-23T21:14:17.580980+0000","last_fullsized":"2023-09-23T21:14:17.580980+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":479,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:10:37.696951+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:10:37.696951+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.001386504,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,3],"acting":[2,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"10.12","version":"528'5278","reported_seq":7054,"reported_epoch":528,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T21:14:28.278702+0000","last_change":"2023-09-23T21:13:13.419184+0000","last_active":"2023-09-23T21:14:28.278702+0000","last_peered":"2023-09-23T21:14:28.278702+0000","last_clean":"2023-09-23T21:11:45.774771+0000","last_became_active":"2023-09-23T21:13:13.409281+0000","last_became_peered":"2023-09-23T21:13:13.409281+0000","last_unstale":"2023-09-23T21:14:28.278702+0000","last_undegraded":"2023-09-23T21:13:13.284063+0000","last_fullsized":"2023-09-23T21:13:13.191117+0000","mapping_epoch":494,"log_start":"421'18","ondisk_log_start":"421'18","created":438,"last_epoch_clean":439,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_clean_scrub_stamp":"2023-09-23T21:10:15.663179+0000","objects_scrubbed":0,"log_size":5260,"log_dups_size":0,"ondisk_log_size":5260,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":67829760,"num_objects":1035,"num_object_clones":0,"num_object_copies":3105,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2331,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1035,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3901,"num_write_kb":157952,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":807,"num_bytes_recovered":52887552,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,3],"avail_no_missing":["4","1"],"object_location_counts":[{"shards":"0,1,4","objects":997},{"shards":"1,4","objects":41},{"shards":"3,4","objects":-3}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":36,"reported_epoch":526,"state":"active+clean","last_fresh":"2023-09-23T21:14:09.545504+0000","last_change":"2023-09-23T21:13:30.791297+0000","last_active":"2023-09-23T21:14:09.545504+0000","last_peered":"2023-09-23T21:14:09.545504+0000","last_clean":"2023-09-23T21:14:09.545504+0000","last_became_active":"2023-09-23T21:13:30.788721+0000","last_became_peered":"2023-09-23T21:13:30.788721+0000","last_unstale":"2023-09-23T21:14:09.545504+0000","last_undegraded":"2023-09-23T21:14:09.545504+0000","last_fullsized":"2023-09-23T21:14:09.545504+0000","mapping_epoch":501,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:31:18.147883+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":31,"reported_epoch":527,"state":"active+clean","last_fresh":"2023-09-23T21:14:17.581939+0000","last_change":"2023-09-23T21:13:38.373814+0000","last_active":"2023-09-23T21:14:17.581939+0000","last_peered":"2023-09-23T21:14:17.581939+0000","last_clean":"2023-09-23T21:14:17.581939+0000","last_became_active":"2023-09-23T21:13:38.373458+0000","last_became_peered":"2023-09-23T21:13:38.373458+0000","last_unstale":"2023-09-23T21:14:17.581939+0000","last_undegraded":"2023-09-23T21:14:17.581939+0000","last_fullsized":"2023-09-23T21:14:17.581939+0000","mapping_epoch":507,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:23:08.288401+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.1e","version":"79'161","reported_seq":695,"reported_epoch":527,"state":"active+clean","last_fresh":"2023-09-23T21:14:17.576885+0000","last_change":"2023-09-23T21:12:30.448761+0000","last_active":"2023-09-23T21:14:17.576885+0000","last_peered":"2023-09-23T21:14:17.576885+0000","last_clean":"2023-09-23T21:14:17.576885+0000","last_became_active":"2023-09-23T21:12:30.448043+0000","last_became_peered":"2023-09-23T21:12:30.448043+0000","last_unstale":"2023-09-23T21:14:17.576885+0000","last_undegraded":"2023-09-23T21:14:17.576885+0000","last_fullsized":"2023-09-23T21:14:17.576885+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":479,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:10:39.567232+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:10:39.567232+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.005229694,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.15","version":"528'5164","reported_seq":5309,"reported_epoch":528,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T21:14:26.058526+0000","last_change":"2023-09-23T21:13:13.436807+0000","last_active":"2023-09-23T21:14:26.058526+0000","last_peered":"2023-09-23T21:14:26.058526+0000","last_clean":"2023-09-23T21:11:45.765730+0000","last_became_active":"2023-09-23T21:13:13.429206+0000","last_became_peered":"2023-09-23T21:13:13.429206+0000","last_unstale":"2023-09-23T21:14:26.058526+0000","last_undegraded":"2023-09-23T21:13:13.246273+0000","last_fullsized":"2023-09-23T21:13:13.203825+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":438,"last_epoch_clean":439,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_clean_scrub_stamp":"2023-09-23T21:10:15.663179+0000","objects_scrubbed":0,"log_size":5164,"log_dups_size":0,"ondisk_log_size":5164,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":73269248,"num_objects":1118,"num_object_clones":0,"num_object_copies":3354,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":643,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1118,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3830,"num_write_kb":158336,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,3],"acting":[2,3],"avail_no_missing":["2","3"],"object_location_counts":[{"shards":"2,3","objects":1082}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":103,"reported_epoch":527,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-23T21:14:17.578231+0000","last_change":"2023-09-23T21:12:14.313202+0000","last_active":"2023-09-23T21:14:17.578231+0000","last_peered":"2023-09-23T21:14:17.578231+0000","last_clean":"2023-09-23T21:12:09.299355+0000","last_became_active":"2023-09-23T21:12:14.209408+0000","last_became_peered":"2023-09-23T21:12:14.209408+0000","last_unstale":"2023-09-23T21:14:17.578231+0000","last_undegraded":"2023-09-23T21:12:12.933857+0000","last_fullsized":"2023-09-23T21:14:17.578231+0000","mapping_epoch":470,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":444,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:45:40.142413+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,7],"acting":[4,5,7],"avail_no_missing":["4","1","5"],"object_location_counts":[{"shards":"1,4,5","objects":1}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":92,"reported_epoch":527,"state":"active+clean","last_fresh":"2023-09-23T21:14:17.580884+0000","last_change":"2023-09-23T21:12:10.770474+0000","last_active":"2023-09-23T21:14:17.580884+0000","last_peered":"2023-09-23T21:14:17.580884+0000","last_clean":"2023-09-23T21:14:17.580884+0000","last_became_active":"2023-09-23T21:12:10.770184+0000","last_became_peered":"2023-09-23T21:12:10.770184+0000","last_unstale":"2023-09-23T21:14:17.580884+0000","last_undegraded":"2023-09-23T21:14:17.580884+0000","last_fullsized":"2023-09-23T21:14:17.580884+0000","mapping_epoch":470,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:05:09.407151+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.14","version":"527'5127","reported_seq":11226,"reported_epoch":527,"state":"active+clean","last_fresh":"2023-09-23T21:14:25.126695+0000","last_change":"2023-09-23T21:14:01.515957+0000","last_active":"2023-09-23T21:14:25.126695+0000","last_peered":"2023-09-23T21:14:25.126695+0000","last_clean":"2023-09-23T21:14:25.126695+0000","last_became_active":"2023-09-23T21:14:01.506428+0000","last_became_peered":"2023-09-23T21:14:01.506428+0000","last_unstale":"2023-09-23T21:14:25.126695+0000","last_undegraded":"2023-09-23T21:14:25.126695+0000","last_fullsized":"2023-09-23T21:14:25.126695+0000","mapping_epoch":522,"log_start":"0'0","ondisk_log_start":"0'0","created":438,"last_epoch_clean":523,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_clean_scrub_stamp":"2023-09-23T21:10:15.663179+0000","objects_scrubbed":0,"log_size":5127,"log_dups_size":0,"ondisk_log_size":5127,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":73203712,"num_objects":1117,"num_object_clones":0,"num_object_copies":3351,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1117,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3793,"num_write_kb":157120,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2660,"num_bytes_recovered":126156800,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":317,"reported_epoch":527,"state":"active+clean","last_fresh":"2023-09-23T21:14:17.582103+0000","last_change":"2023-09-23T21:12:10.771160+0000","last_active":"2023-09-23T21:14:17.582103+0000","last_peered":"2023-09-23T21:14:17.582103+0000","last_clean":"2023-09-23T21:14:17.582103+0000","last_became_active":"2023-09-23T21:12:10.757808+0000","last_became_peered":"2023-09-23T21:12:10.757808+0000","last_unstale":"2023-09-23T21:14:17.582103+0000","last_undegraded":"2023-09-23T21:14:17.582103+0000","last_fullsized":"2023-09-23T21:14:17.582103+0000","mapping_epoch":470,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:30:50.987139+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":102,"reported_epoch":527,"state":"active+clean","last_fresh":"2023-09-23T21:14:17.577668+0000","last_change":"2023-09-23T21:13:12.216602+0000","last_active":"2023-09-23T21:14:17.577668+0000","last_peered":"2023-09-23T21:14:17.577668+0000","last_clean":"2023-09-23T21:14:17.577668+0000","last_became_active":"2023-09-23T21:13:12.216204+0000","last_became_peered":"2023-09-23T21:13:12.216204+0000","last_unstale":"2023-09-23T21:14:17.577668+0000","last_undegraded":"2023-09-23T21:14:17.577668+0000","last_fullsized":"2023-09-23T21:14:17.577668+0000","mapping_epoch":493,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:23:37.043390+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1c","version":"79'161","reported_seq":892,"reported_epoch":526,"state":"active+clean","last_fresh":"2023-09-23T21:14:09.547350+0000","last_change":"2023-09-23T21:13:13.402773+0000","last_active":"2023-09-23T21:14:09.547350+0000","last_peered":"2023-09-23T21:14:09.547350+0000","last_clean":"2023-09-23T21:14:09.547350+0000","last_became_active":"2023-09-23T21:13:13.402025+0000","last_became_peered":"2023-09-23T21:13:13.402025+0000","last_unstale":"2023-09-23T21:14:09.547350+0000","last_undegraded":"2023-09-23T21:14:09.547350+0000","last_fullsized":"2023-09-23T21:14:09.547350+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":479,"last_epoch_clean":495,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:22.040907+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:22.040907+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.013021176000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":163840,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":163840,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.17","version":"527'5150","reported_seq":9534,"reported_epoch":527,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T21:14:23.912775+0000","last_change":"2023-09-23T21:13:39.447969+0000","last_active":"2023-09-23T21:14:23.912775+0000","last_peered":"2023-09-23T21:14:23.912775+0000","last_clean":"2023-09-23T21:13:37.065694+0000","last_became_active":"2023-09-23T21:13:39.439739+0000","last_became_peered":"2023-09-23T21:13:39.439739+0000","last_unstale":"2023-09-23T21:14:23.912775+0000","last_undegraded":"2023-09-23T21:13:39.411853+0000","last_fullsized":"2023-09-23T21:13:39.404657+0000","mapping_epoch":508,"log_start":"422'162","ondisk_log_start":"422'162","created":438,"last_epoch_clean":487,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_clean_scrub_stamp":"2023-09-23T21:10:15.663179+0000","objects_scrubbed":0,"log_size":4988,"log_dups_size":0,"ondisk_log_size":4988,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":75399168,"num_objects":1151,"num_object_clones":0,"num_object_copies":3453,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1109,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1151,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3785,"num_write_kb":157920,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2112,"num_bytes_recovered":116719616,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,2],"acting":[3,5],"avail_no_missing":["3","4","5"],"object_location_counts":[{"shards":"3,4,5","objects":311},{"shards":"3,5","objects":840}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":116,"reported_epoch":527,"state":"active+clean","last_fresh":"2023-09-23T21:14:17.582263+0000","last_change":"2023-09-23T21:12:10.779066+0000","last_active":"2023-09-23T21:14:17.582263+0000","last_peered":"2023-09-23T21:14:17.582263+0000","last_clean":"2023-09-23T21:14:17.582263+0000","last_became_active":"2023-09-23T21:12:10.778691+0000","last_became_peered":"2023-09-23T21:12:10.778691+0000","last_unstale":"2023-09-23T21:14:17.582263+0000","last_undegraded":"2023-09-23T21:14:17.582263+0000","last_fullsized":"2023-09-23T21:14:17.582263+0000","mapping_epoch":470,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:07:22.043268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,7],"acting":[2,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":195,"reported_epoch":527,"state":"active+clean","last_fresh":"2023-09-23T21:14:17.581592+0000","last_change":"2023-09-23T21:13:12.200767+0000","last_active":"2023-09-23T21:14:17.581592+0000","last_peered":"2023-09-23T21:14:17.581592+0000","last_clean":"2023-09-23T21:14:17.581592+0000","last_became_active":"2023-09-23T21:13:12.200435+0000","last_became_peered":"2023-09-23T21:13:12.200435+0000","last_unstale":"2023-09-23T21:14:17.581592+0000","last_undegraded":"2023-09-23T21:14:17.581592+0000","last_fullsized":"2023-09-23T21:14:17.581592+0000","mapping_epoch":493,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:15:44.937886+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.1d","version":"79'161","reported_seq":680,"reported_epoch":527,"state":"active+clean","last_fresh":"2023-09-23T21:14:17.600387+0000","last_change":"2023-09-23T21:12:30.444331+0000","last_active":"2023-09-23T21:14:17.600387+0000","last_peered":"2023-09-23T21:14:17.600387+0000","last_clean":"2023-09-23T21:14:17.600387+0000","last_became_active":"2023-09-23T21:12:30.444001+0000","last_became_peered":"2023-09-23T21:12:30.444001+0000","last_unstale":"2023-09-23T21:14:17.600387+0000","last_undegraded":"2023-09-23T21:14:17.600387+0000","last_fullsized":"2023-09-23T21:14:17.600387+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":479,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:09:36.858077+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:09:36.858077+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0024987500000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"10.16","version":"528'5362","reported_seq":8871,"reported_epoch":528,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T21:14:28.068854+0000","last_change":"2023-09-23T21:13:14.424843+0000","last_active":"2023-09-23T21:14:28.068854+0000","last_peered":"2023-09-23T21:14:28.068854+0000","last_clean":"2023-09-23T21:11:45.652255+0000","last_became_active":"2023-09-23T21:13:14.414710+0000","last_became_peered":"2023-09-23T21:13:14.414710+0000","last_unstale":"2023-09-23T21:14:28.068854+0000","last_undegraded":"2023-09-23T21:13:13.211957+0000","last_fullsized":"2023-09-23T21:13:13.188859+0000","mapping_epoch":494,"log_start":"422'74","ondisk_log_start":"422'74","created":438,"last_epoch_clean":460,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_clean_scrub_stamp":"2023-09-23T21:10:15.663179+0000","objects_scrubbed":0,"log_size":5288,"log_dups_size":0,"ondisk_log_size":5288,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":68812800,"num_objects":1050,"num_object_clones":0,"num_object_copies":3150,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":663,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1050,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3956,"num_write_kb":160192,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1670,"num_bytes_recovered":109445120,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,4],"acting":[4,5],"avail_no_missing":["4","5"],"object_location_counts":[{"shards":"1,4,5","objects":33},{"shards":"4,5","objects":1017}],"blocked_by":[],"up_primary":0,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":53,"reported_epoch":526,"state":"active+clean","last_fresh":"2023-09-23T21:14:09.544626+0000","last_change":"2023-09-23T21:13:38.380574+0000","last_active":"2023-09-23T21:14:09.544626+0000","last_peered":"2023-09-23T21:14:09.544626+0000","last_clean":"2023-09-23T21:14:09.544626+0000","last_became_active":"2023-09-23T21:13:38.373252+0000","last_became_peered":"2023-09-23T21:13:38.373252+0000","last_unstale":"2023-09-23T21:14:09.544626+0000","last_undegraded":"2023-09-23T21:14:09.544626+0000","last_fullsized":"2023-09-23T21:14:09.544626+0000","mapping_epoch":507,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:58:14.788031+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":11,"num_bytes_recovered":3324,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":102,"reported_epoch":527,"state":"active+clean","last_fresh":"2023-09-23T21:14:17.578878+0000","last_change":"2023-09-23T21:12:10.793610+0000","last_active":"2023-09-23T21:14:17.578878+0000","last_peered":"2023-09-23T21:14:17.578878+0000","last_clean":"2023-09-23T21:14:17.578878+0000","last_became_active":"2023-09-23T21:12:10.793081+0000","last_became_peered":"2023-09-23T21:12:10.793081+0000","last_unstale":"2023-09-23T21:14:17.578878+0000","last_undegraded":"2023-09-23T21:14:17.578878+0000","last_fullsized":"2023-09-23T21:14:17.578878+0000","mapping_epoch":470,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:34:36.291550+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":10,"num_pg":26,"stat_sum":{"num_bytes":2296250392,"num_objects":35041,"num_object_clones":0,"num_object_copies":105123,"num_objects_missing_on_primary":2360,"num_objects_missing":2360,"num_objects_degraded":23881,"num_objects_misplaced":3977,"num_objects_unfound":0,"num_objects_dirty":35041,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":112333,"num_write_kb":4309313,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":38750,"num_bytes_recovered":2329968640,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":9586745344,"data_stored":9586737200,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":131542,"ondisk_log_size":131542,"up":78,"acting":62,"num_store_stats":7},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":59,"num_bytes_recovered":8898,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":126976,"data_stored":22458,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":14322,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":31,"stat_sum":{"num_bytes":697708,"num_objects":3,"num_object_clones":0,"num_object_copies":9,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3,"num_whiteouts":0,"num_read":89,"num_read_kb":165,"num_write":44,"num_write_kb":440,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":649699,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2109440,"data_stored":2098784,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4991,"ondisk_log_size":4991,"up":93,"acting":93,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201688,"num_pgs":69,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1322840,"kb_used_data":838160,"kb_used_omap":0,"kb_used_meta":484672,"kb_avail":92422312,"statfs":{"total":95995035648,"available":94640447488,"internally_reserved":0,"allocated":858275840,"data_stored":856746977,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":496304128},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:14:13 2023","interfaces":[{"interface":"back","average":{"1min":2.4260000000000002,"5min":2.4260000000000002,"15min":2.4260000000000002},"min":{"1min":0.23400000000000001,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":37.207999999999998,"5min":37.207999999999998,"15min":37.207999999999998},"last":0.40600000000000003},{"interface":"front","average":{"1min":2.234,"5min":2.234,"15min":2.234},"min":{"1min":0.28599999999999998,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":37.122,"5min":37.122,"15min":37.122},"last":0.253}]},{"osd":1,"last update":"Sat Sep 23 21:14:09 2023","interfaces":[{"interface":"back","average":{"1min":0.94299999999999995,"5min":1.2310000000000001,"15min":2.04},"min":{"1min":0.30399999999999999,"5min":0.27100000000000002,"15min":0.26700000000000002},"max":{"1min":7.5369999999999999,"5min":14.034000000000001,"15min":14.034000000000001},"last":0.67600000000000005},{"interface":"front","average":{"1min":1.0469999999999999,"5min":1.4339999999999999,"15min":1.944},"min":{"1min":0.30499999999999999,"5min":0.29199999999999998,"15min":0.26300000000000001},"max":{"1min":5.6870000000000003,"5min":11.226000000000001,"15min":11.226000000000001},"last":0.54300000000000004}]},{"osd":2,"last update":"Sat Sep 23 21:13:29 2023","interfaces":[{"interface":"back","average":{"1min":1.254,"5min":1.3839999999999999,"15min":1.9119999999999999},"min":{"1min":0.26400000000000001,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":6.4219999999999997,"5min":6.5469999999999997,"15min":14.478999999999999},"last":0.69399999999999995},{"interface":"front","average":{"1min":1.298,"5min":1.5129999999999999,"15min":1.819},"min":{"1min":0.26700000000000002,"5min":0.26700000000000002,"15min":0.26500000000000001},"max":{"1min":9.1839999999999993,"5min":11.198,"15min":14.135},"last":1.829}]},{"osd":3,"last update":"Sat Sep 23 21:13:57 2023","interfaces":[{"interface":"back","average":{"1min":1.022,"5min":1.296,"15min":2.0590000000000002},"min":{"1min":0.33000000000000002,"5min":0.316,"15min":0.29099999999999998},"max":{"1min":6.4930000000000003,"5min":7.5190000000000001,"15min":31.786000000000001},"last":1.992},{"interface":"front","average":{"1min":1.1160000000000001,"5min":1.3859999999999999,"15min":2.0409999999999999},"min":{"1min":0.37,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":5.9329999999999998,"5min":11.244999999999999,"15min":32.429000000000002},"last":0.62}]},{"osd":4,"last update":"Sat Sep 23 21:13:53 2023","interfaces":[{"interface":"back","average":{"1min":1.5980000000000001,"5min":1.5209999999999999,"15min":1.097},"min":{"1min":0.28999999999999998,"5min":0.255,"15min":0.248},"max":{"1min":7.6609999999999996,"5min":11.297000000000001,"15min":11.297000000000001},"last":0.443},{"interface":"front","average":{"1min":1.387,"5min":1.1679999999999999,"15min":1.0669999999999999},"min":{"1min":0.314,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":8.6059999999999999,"5min":11.093999999999999,"15min":12.752000000000001},"last":0.60199999999999998}]},{"osd":5,"last update":"Sat Sep 23 21:13:53 2023","interfaces":[{"interface":"back","average":{"1min":1.4670000000000001,"5min":1.0549999999999999,"15min":1.014},"min":{"1min":0.29899999999999999,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":7.5910000000000002,"5min":9.4789999999999992,"15min":9.4789999999999992},"last":0.81799999999999995},{"interface":"front","average":{"1min":1.4950000000000001,"5min":1.3680000000000001,"15min":1.099},"min":{"1min":0.39700000000000002,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":7.6929999999999996,"5min":11.257,"15min":11.257},"last":0.57999999999999996}]},{"osd":6,"last update":"Sat Sep 23 21:14:09 2023","interfaces":[{"interface":"back","average":{"1min":0.79600000000000004,"5min":1.1819999999999999,"15min":1.038},"min":{"1min":0.39800000000000002,"5min":0.30099999999999999,"15min":0.28899999999999998},"max":{"1min":1.667,"5min":10.941000000000001,"15min":10.941000000000001},"last":0.63600000000000001},{"interface":"front","average":{"1min":0.99299999999999999,"5min":1.1020000000000001,"15min":1.1259999999999999},"min":{"1min":0.41699999999999998,"5min":0.33100000000000002,"15min":0.32300000000000001},"max":{"1min":2.4969999999999999,"5min":8.923,"15min":12.715},"last":0.65000000000000002}]}]},{"osd":6,"up_from":0,"seq":704374636771,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":1,"up_from":0,"seq":1503238553704,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":0,"up_from":493,"seq":2117418876944,"num_pgs":75,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2426216,"kb_used_data":2276128,"kb_used_omap":36,"kb_used_meta":150043,"kb_avail":91318936,"statfs":{"total":95995035648,"available":93510590464,"internally_reserved":0,"allocated":2330755072,"data_stored":2328285172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":37336,"internal_metadata":153644584},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 21:14:12 2023","interfaces":[{"interface":"back","average":{"1min":0.75600000000000001,"5min":0.75600000000000001,"15min":0.75600000000000001},"min":{"1min":0.23000000000000001,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":4.9000000000000004,"5min":4.9000000000000004,"15min":4.9000000000000004},"last":3.3359999999999999},{"interface":"front","average":{"1min":0.77100000000000002,"5min":0.77100000000000002,"15min":0.77100000000000002},"min":{"1min":0.24299999999999999,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":3.444,"5min":3.444,"15min":3.444},"last":0.255}]},{"osd":2,"last update":"Sat Sep 23 21:14:12 2023","interfaces":[{"interface":"back","average":{"1min":0.65400000000000003,"5min":0.65400000000000003,"15min":0.65400000000000003},"min":{"1min":0.26900000000000002,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":4.8630000000000004,"5min":4.8630000000000004,"15min":4.8630000000000004},"last":1.5409999999999999},{"interface":"front","average":{"1min":0.96799999999999997,"5min":0.96799999999999997,"15min":0.96799999999999997},"min":{"1min":0.29299999999999998,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":4.649,"5min":4.649,"15min":4.649},"last":0.39000000000000001}]},{"osd":3,"last update":"Sat Sep 23 21:14:12 2023","interfaces":[{"interface":"back","average":{"1min":0.86799999999999999,"5min":0.86799999999999999,"15min":0.86799999999999999},"min":{"1min":0.23400000000000001,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":4.5110000000000001,"5min":4.5110000000000001,"15min":4.5110000000000001},"last":3.448},{"interface":"front","average":{"1min":1.1180000000000001,"5min":1.1180000000000001,"15min":1.1180000000000001},"min":{"1min":0.26200000000000001,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":3.7200000000000002,"5min":3.7200000000000002,"15min":3.7200000000000002},"last":1.671}]},{"osd":4,"last update":"Sat Sep 23 21:14:12 2023","interfaces":[{"interface":"back","average":{"1min":1.093,"5min":1.093,"15min":1.093},"min":{"1min":0.30199999999999999,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":3.7799999999999998,"5min":3.7799999999999998,"15min":3.7799999999999998},"last":1.6950000000000001},{"interface":"front","average":{"1min":0.996,"5min":0.996,"15min":0.996},"min":{"1min":0.317,"5min":0.317,"15min":0.317},"max":{"1min":5.093,"5min":5.093,"15min":5.093},"last":1.5820000000000001}]},{"osd":5,"last update":"Sat Sep 23 21:14:12 2023","interfaces":[{"interface":"back","average":{"1min":1.236,"5min":1.236,"15min":1.236},"min":{"1min":0.33200000000000002,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":4.5640000000000001,"5min":4.5640000000000001,"15min":4.5640000000000001},"last":0.30299999999999999},{"interface":"front","average":{"1min":1.0640000000000001,"5min":1.0640000000000001,"15min":1.0640000000000001},"min":{"1min":0.33200000000000002,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":4.5599999999999996,"5min":4.5599999999999996,"15min":4.5599999999999996},"last":1.7410000000000001}]},{"osd":6,"last update":"Sat Sep 23 21:14:12 2023","interfaces":[{"interface":"back","average":{"1min":1.127,"5min":1.127,"15min":1.127},"min":{"1min":0.38100000000000001,"5min":0.38100000000000001,"15min":0.38100000000000001},"max":{"1min":6.2750000000000004,"5min":6.2750000000000004,"15min":6.2750000000000004},"last":1.635},{"interface":"front","average":{"1min":1.3080000000000001,"5min":1.3080000000000001,"15min":1.3080000000000001},"min":{"1min":0.35599999999999998,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":5.0860000000000003,"5min":5.0860000000000003,"15min":5.0860000000000003},"last":0.435}]},{"osd":7,"last update":"Sat Sep 23 21:14:12 2023","interfaces":[{"interface":"back","average":{"1min":1.2090000000000001,"5min":1.2090000000000001,"15min":1.2090000000000001},"min":{"1min":0.41099999999999998,"5min":0.41099999999999998,"15min":0.41099999999999998},"max":{"1min":5.1539999999999999,"5min":5.1539999999999999,"15min":5.1539999999999999},"last":1.718},{"interface":"front","average":{"1min":1.1579999999999999,"5min":1.1579999999999999,"15min":1.1579999999999999},"min":{"1min":0.441,"5min":0.441,"15min":0.441},"max":{"1min":5.1289999999999996,"5min":5.1289999999999996,"15min":5.1289999999999996},"last":1.607}]}]},{"osd":2,"up_from":93,"seq":399431958816,"num_pgs":69,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2177336,"kb_used_data":1401904,"kb_used_omap":81,"kb_used_meta":775342,"kb_avail":91567816,"statfs":{"total":95995035648,"available":93765443584,"internally_reserved":0,"allocated":1435549696,"data_stored":1433071328,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":83750,"internal_metadata":793950426},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:14:15 2023","interfaces":[{"interface":"back","average":{"1min":0.83299999999999996,"5min":0.83299999999999996,"15min":0.83299999999999996},"min":{"1min":0.20899999999999999,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":3.7290000000000001,"5min":3.7290000000000001,"15min":3.7290000000000001},"last":0.30299999999999999},{"interface":"front","average":{"1min":1.665,"5min":1.665,"15min":1.665},"min":{"1min":0.23499999999999999,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":18.111999999999998,"5min":18.111999999999998,"15min":18.111999999999998},"last":0.24399999999999999}]},{"osd":1,"last update":"Sat Sep 23 21:14:15 2023","interfaces":[{"interface":"back","average":{"1min":0.95099999999999996,"5min":1.474,"15min":2.5339999999999998},"min":{"1min":0.23999999999999999,"5min":0.188,"15min":0.188},"max":{"1min":6.0499999999999998,"5min":12.484999999999999,"15min":34.396999999999998},"last":0.41799999999999998},{"interface":"front","average":{"1min":1.597,"5min":1.3129999999999999,"15min":2.782},"min":{"1min":0.29899999999999999,"5min":0.217,"15min":0.217},"max":{"1min":18.081,"5min":18.081,"15min":34.462000000000003},"last":0.437}]},{"osd":3,"last update":"Sat Sep 23 21:13:56 2023","interfaces":[{"interface":"back","average":{"1min":0.495,"5min":1.5429999999999999,"15min":2.6419999999999999},"min":{"1min":0.20599999999999999,"5min":0.20599999999999999,"15min":0.20599999999999999},"max":{"1min":1.679,"5min":12.951000000000001,"15min":41.057000000000002},"last":0.38100000000000001},{"interface":"front","average":{"1min":0.749,"5min":1.661,"15min":2.915},"min":{"1min":0.28499999999999998,"5min":0.21299999999999999,"15min":0.21299999999999999},"max":{"1min":3.6989999999999998,"5min":13.159000000000001,"15min":41.012},"last":0.60399999999999998}]},{"osd":4,"last update":"Sat Sep 23 21:14:15 2023","interfaces":[{"interface":"back","average":{"1min":0.64800000000000002,"5min":1.256,"15min":1.4219999999999999},"min":{"1min":0.34100000000000003,"5min":0.249,"15min":0.249},"max":{"1min":1.7210000000000001,"5min":10.266999999999999,"15min":10.266999999999999},"last":0.40699999999999997},{"interface":"front","average":{"1min":1.0029999999999999,"5min":1.456,"15min":1.8759999999999999},"min":{"1min":0.38300000000000001,"5min":0.29399999999999998,"15min":0.26700000000000002},"max":{"1min":3.5819999999999999,"5min":8.6780000000000008,"15min":14.039999999999999},"last":0.48999999999999999}]},{"osd":5,"last update":"Sat Sep 23 21:14:15 2023","interfaces":[{"interface":"back","average":{"1min":0.94399999999999995,"5min":1.276,"15min":1.5209999999999999},"min":{"1min":0.311,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":3.3759999999999999,"5min":9.5039999999999996,"15min":10.069000000000001},"last":0.45000000000000001},{"interface":"front","average":{"1min":0.83999999999999997,"5min":1.3149999999999999,"15min":1.8120000000000001},"min":{"1min":0.41499999999999998,"5min":0.29699999999999999,"15min":0.29399999999999998},"max":{"1min":3.629,"5min":8.7270000000000003,"15min":13.98},"last":0.53300000000000003}]},{"osd":6,"last update":"Sat Sep 23 21:14:15 2023","interfaces":[{"interface":"back","average":{"1min":0.94999999999999996,"5min":1.569,"15min":2.0529999999999999},"min":{"1min":0.436,"5min":0.373,"15min":0.35699999999999998},"max":{"1min":3.665,"5min":8.7680000000000007,"15min":14.016},"last":0.56699999999999995},{"interface":"front","average":{"1min":1.2450000000000001,"5min":1.7729999999999999,"15min":2.2330000000000001},"min":{"1min":0.45700000000000002,"5min":0.38100000000000001,"15min":0.35599999999999998},"max":{"1min":3.7610000000000001,"5min":9.3469999999999995,"15min":14.068},"last":0.63600000000000001}]},{"osd":7,"last update":"Sat Sep 23 21:14:15 2023","interfaces":[{"interface":"back","average":{"1min":1.8120000000000001,"5min":1.4790000000000001,"15min":1.5840000000000001},"min":{"1min":0.433,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":18.021999999999998,"5min":18.021999999999998,"15min":18.021999999999998},"last":0.58499999999999996},{"interface":"front","average":{"1min":1.242,"5min":1.476,"15min":1.675},"min":{"1min":0.39400000000000002,"5min":0.34799999999999998,"15min":0.32200000000000001},"max":{"1min":9.6349999999999998,"5min":10.359,"15min":10.92},"last":0.47399999999999998}]}]},{"osd":3,"up_from":98,"seq":420906795289,"num_pgs":82,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2256580,"kb_used_data":1392124,"kb_used_omap":7,"kb_used_meta":864440,"kb_avail":91488572,"statfs":{"total":95995035648,"available":93684297728,"internally_reserved":0,"allocated":1425534976,"data_stored":1423093740,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7576,"internal_metadata":885187176},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:14:12 2023","interfaces":[{"interface":"back","average":{"1min":0.33800000000000002,"5min":0.33800000000000002,"15min":0.33800000000000002},"min":{"1min":0.20499999999999999,"5min":0.20499999999999999,"15min":0.20499999999999999},"max":{"1min":0.69899999999999995,"5min":0.69899999999999995,"15min":0.69899999999999995},"last":1.639},{"interface":"front","average":{"1min":0.66800000000000004,"5min":0.66800000000000004,"15min":0.66800000000000004},"min":{"1min":0.23300000000000001,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":4.5510000000000002,"5min":4.5510000000000002,"15min":4.5510000000000002},"last":0.44}]},{"osd":1,"last update":"Sat Sep 23 21:14:12 2023","interfaces":[{"interface":"back","average":{"1min":1.052,"5min":1.466,"15min":2.226},"min":{"1min":0.218,"5min":0.20000000000000001,"15min":0.20000000000000001},"max":{"1min":5.5019999999999998,"5min":22.577999999999999,"15min":22.577999999999999},"last":1.242},{"interface":"front","average":{"1min":0.60099999999999998,"5min":1.5580000000000001,"15min":2.1829999999999998},"min":{"1min":0.249,"5min":0.216,"15min":0.216},"max":{"1min":2.7519999999999998,"5min":21.617999999999999,"15min":21.617999999999999},"last":0.79600000000000004}]},{"osd":2,"last update":"Sat Sep 23 21:14:00 2023","interfaces":[{"interface":"back","average":{"1min":3.6669999999999998,"5min":1.9279999999999999,"15min":2.6640000000000001},"min":{"1min":0.221,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":56.317999999999998,"5min":56.317999999999998,"15min":56.317999999999998},"last":0.47499999999999998},{"interface":"front","average":{"1min":3.8479999999999999,"5min":2.1499999999999999,"15min":2.702},"min":{"1min":0.29699999999999999,"5min":0.216,"15min":0.216},"max":{"1min":55.988,"5min":55.988,"15min":55.988},"last":1.27}]},{"osd":4,"last update":"Sat Sep 23 21:14:00 2023","interfaces":[{"interface":"back","average":{"1min":0.71499999999999997,"5min":1.3029999999999999,"15min":1.8779999999999999},"min":{"1min":0.30299999999999999,"5min":0.30299999999999999,"15min":0.28399999999999997},"max":{"1min":5.0819999999999999,"5min":21.478999999999999,"15min":21.478999999999999},"last":0.501},{"interface":"front","average":{"1min":1.3009999999999999,"5min":1.5760000000000001,"15min":2.0070000000000001},"min":{"1min":0.33100000000000002,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":4.1909999999999998,"5min":22.402000000000001,"15min":22.402000000000001},"last":1.3080000000000001}]},{"osd":5,"last update":"Sat Sep 23 21:14:00 2023","interfaces":[{"interface":"back","average":{"1min":0.93600000000000005,"5min":1.3120000000000001,"15min":2.012},"min":{"1min":0.32200000000000001,"5min":0.311,"15min":0.311},"max":{"1min":5.04,"5min":21.562000000000001,"15min":21.562000000000001},"last":0.55200000000000005},{"interface":"front","average":{"1min":0.83699999999999997,"5min":1.339,"15min":2.2839999999999998},"min":{"1min":0.34300000000000003,"5min":0.31,"15min":0.31},"max":{"1min":3.4780000000000002,"5min":8.8450000000000006,"15min":14.029},"last":0.51200000000000001}]},{"osd":6,"last update":"Sat Sep 23 21:14:15 2023","interfaces":[{"interface":"back","average":{"1min":1.254,"5min":1.591,"15min":2.0510000000000002},"min":{"1min":0.38900000000000001,"5min":0.34999999999999998,"15min":0.33700000000000002},"max":{"1min":4.2380000000000004,"5min":22.446999999999999,"15min":22.446999999999999},"last":1.3280000000000001},{"interface":"front","average":{"1min":1.2450000000000001,"5min":1.746,"15min":2.278},"min":{"1min":0.44,"5min":0.39000000000000001,"15min":0.373},"max":{"1min":9.3889999999999993,"5min":12.375999999999999,"15min":13.576000000000001},"last":0.60799999999999998}]},{"osd":7,"last update":"Sat Sep 23 21:14:00 2023","interfaces":[{"interface":"back","average":{"1min":1.032,"5min":1.6120000000000001,"15min":2.1299999999999999},"min":{"1min":0.41199999999999998,"5min":0.38400000000000001,"15min":0.33900000000000002},"max":{"1min":5.0090000000000003,"5min":21.518999999999998,"15min":21.518999999999998},"last":0.52700000000000002},{"interface":"front","average":{"1min":0.92900000000000005,"5min":1.7050000000000001,"15min":2.4199999999999999},"min":{"1min":0.45800000000000002,"5min":0.442,"15min":0.38900000000000001},"max":{"1min":3.2679999999999998,"5min":9.9879999999999995,"15min":13.603999999999999},"last":0.56899999999999995}]}]},{"osd":4,"up_from":31,"seq":133143986596,"num_pgs":77,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1717472,"kb_used_data":1268312,"kb_used_omap":0,"kb_used_meta":449088,"kb_avail":92027680,"statfs":{"total":95995035648,"available":94236344320,"internally_reserved":0,"allocated":1298751488,"data_stored":1297215264,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":459866112},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:14:14 2023","interfaces":[{"interface":"back","average":{"1min":0.90000000000000002,"5min":0.90000000000000002,"15min":0.90000000000000002},"min":{"1min":0.22500000000000001,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":2.6539999999999999,"5min":2.6539999999999999,"15min":2.6539999999999999},"last":0.751},{"interface":"front","average":{"1min":1.1339999999999999,"5min":1.1339999999999999,"15min":1.1339999999999999},"min":{"1min":0.246,"5min":0.246,"15min":0.246},"max":{"1min":10.945,"5min":10.945,"15min":10.945},"last":1.839}]},{"osd":1,"last update":"Sat Sep 23 21:14:14 2023","interfaces":[{"interface":"back","average":{"1min":0.77300000000000002,"5min":1.0609999999999999,"15min":2.7490000000000001},"min":{"1min":0.29699999999999999,"5min":0.221,"15min":0.221},"max":{"1min":2.387,"5min":7.8609999999999998,"15min":37.960000000000001},"last":0.70299999999999996},{"interface":"front","average":{"1min":0.80600000000000005,"5min":1.1339999999999999,"15min":2.6080000000000001},"min":{"1min":0.28499999999999998,"5min":0.28499999999999998,"15min":0.27000000000000002},"max":{"1min":3.581,"5min":7.1349999999999998,"15min":37.933},"last":0.66000000000000003}]},{"osd":2,"last update":"Sat Sep 23 21:14:14 2023","interfaces":[{"interface":"back","average":{"1min":0.82099999999999995,"5min":1.153,"15min":1.748},"min":{"1min":0.27400000000000002,"5min":0.247,"15min":0.247},"max":{"1min":3.476,"5min":10.464,"15min":16.928000000000001},"last":0.47899999999999998},{"interface":"front","average":{"1min":0.86099999999999999,"5min":1.3959999999999999,"15min":1.7809999999999999},"min":{"1min":0.311,"5min":0.28599999999999998,"15min":0.27800000000000002},"max":{"1min":3.5129999999999999,"5min":7.3659999999999997,"15min":14.417999999999999},"last":2.0299999999999998}]},{"osd":3,"last update":"Sat Sep 23 21:13:46 2023","interfaces":[{"interface":"back","average":{"1min":1.0229999999999999,"5min":1.6299999999999999,"15min":2.1339999999999999},"min":{"1min":0.25600000000000001,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":4.2480000000000002,"5min":14.585000000000001,"15min":15.01},"last":2.4359999999999999},{"interface":"front","average":{"1min":1.135,"5min":1.647,"15min":2.0249999999999999},"min":{"1min":0.36699999999999999,"5min":0.36099999999999999,"15min":0.30199999999999999},"max":{"1min":4.4649999999999999,"5min":14.366,"15min":15.004},"last":2.452}]},{"osd":5,"last update":"Sat Sep 23 21:14:04 2023","interfaces":[{"interface":"back","average":{"1min":1.169,"5min":1.018,"15min":0.97599999999999998},"min":{"1min":0.30099999999999999,"5min":0.222,"15min":0.222},"max":{"1min":5.1609999999999996,"5min":9.4909999999999997,"15min":14.298},"last":0.442},{"interface":"front","average":{"1min":1.0189999999999999,"5min":0.95899999999999996,"15min":1.0760000000000001},"min":{"1min":0.28000000000000003,"5min":0.27200000000000002,"15min":0.255},"max":{"1min":3.0059999999999998,"5min":6.8730000000000002,"15min":9.8979999999999997},"last":0.78000000000000003}]},{"osd":6,"last update":"Sat Sep 23 21:14:14 2023","interfaces":[{"interface":"back","average":{"1min":0.747,"5min":1.0920000000000001,"15min":1.2230000000000001},"min":{"1min":0.317,"5min":0.28599999999999998,"15min":0.26800000000000002},"max":{"1min":2.8650000000000002,"5min":24.452999999999999,"15min":24.452999999999999},"last":0.72899999999999998},{"interface":"front","average":{"1min":0.95099999999999996,"5min":1.036,"15min":0.95399999999999996},"min":{"1min":0.40100000000000002,"5min":0.33500000000000002,"15min":0.32500000000000001},"max":{"1min":3.6600000000000001,"5min":11.409000000000001,"15min":14.387},"last":0.63}]},{"osd":7,"last update":"Sat Sep 23 21:13:46 2023","interfaces":[{"interface":"back","average":{"1min":1.294,"5min":1.268,"15min":1.0529999999999999},"min":{"1min":0.375,"5min":0.30099999999999999,"15min":0.29299999999999998},"max":{"1min":4.532,"5min":16.890999999999998,"15min":16.890999999999998},"last":0.68500000000000005},{"interface":"front","average":{"1min":1.387,"5min":1.155,"15min":1.1559999999999999},"min":{"1min":0.375,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":7.1879999999999997,"5min":8.5830000000000002,"15min":11.170999999999999},"last":0.76600000000000001}]}]},{"osd":5,"up_from":39,"seq":167503724962,"num_pgs":61,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1519244,"kb_used_data":1022724,"kb_used_omap":0,"kb_used_meta":496512,"kb_avail":92225908,"statfs":{"total":95995035648,"available":94439329792,"internally_reserved":0,"allocated":1047269376,"data_stored":1045749342,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":508428288},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:14:13 2023","interfaces":[{"interface":"back","average":{"1min":0.71499999999999997,"5min":0.71499999999999997,"15min":0.71499999999999997},"min":{"1min":0.251,"5min":0.251,"15min":0.251},"max":{"1min":3.3889999999999998,"5min":3.3889999999999998,"15min":3.3889999999999998},"last":0.25},{"interface":"front","average":{"1min":0.66900000000000004,"5min":0.66900000000000004,"15min":0.66900000000000004},"min":{"1min":0.22800000000000001,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":3.1150000000000002,"5min":3.1150000000000002,"15min":3.1150000000000002},"last":0.65200000000000002}]},{"osd":1,"last update":"Sat Sep 23 21:14:08 2023","interfaces":[{"interface":"back","average":{"1min":0.65200000000000002,"5min":1.1479999999999999,"15min":1.728},"min":{"1min":0.248,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":3.109,"5min":9.4429999999999996,"15min":25.239999999999998},"last":0.29399999999999998},{"interface":"front","average":{"1min":1.468,"5min":1.681,"15min":1.784},"min":{"1min":0.34999999999999998,"5min":0.28699999999999998,"15min":0.252},"max":{"1min":11.116,"5min":11.116,"15min":25.253},"last":0.35299999999999998}]},{"osd":2,"last update":"Sat Sep 23 21:14:26 2023","interfaces":[{"interface":"back","average":{"1min":0.73099999999999998,"5min":1.401,"15min":1.7110000000000001},"min":{"1min":0.27800000000000002,"5min":0.26700000000000002,"15min":0.23499999999999999},"max":{"1min":4.0490000000000004,"5min":9.2889999999999997,"15min":12.634},"last":0.70899999999999996},{"interface":"front","average":{"1min":0.78600000000000003,"5min":1.222,"15min":1.762},"min":{"1min":0.30199999999999999,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":2.524,"5min":9.7170000000000005,"15min":10.333},"last":0.56299999999999994}]},{"osd":3,"last update":"Sat Sep 23 21:13:45 2023","interfaces":[{"interface":"back","average":{"1min":1.4279999999999999,"5min":1.8089999999999999,"15min":2.6560000000000001},"min":{"1min":0.33400000000000002,"5min":0.307,"15min":0.29299999999999998},"max":{"1min":6.1859999999999999,"5min":14.242000000000001,"15min":61.667999999999999},"last":0.32400000000000001},{"interface":"front","average":{"1min":1.4390000000000001,"5min":2.0089999999999999,"15min":2.6429999999999998},"min":{"1min":0.34899999999999998,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":6.1200000000000001,"5min":14.396000000000001,"15min":62.276000000000003},"last":0.68200000000000005}]},{"osd":4,"last update":"Sat Sep 23 21:13:59 2023","interfaces":[{"interface":"back","average":{"1min":1.452,"5min":1.387,"15min":1.139},"min":{"1min":0.28799999999999998,"5min":0.27000000000000002,"15min":0.23400000000000001},"max":{"1min":11.156000000000001,"5min":12.776,"15min":12.776},"last":0.84399999999999997},{"interface":"front","average":{"1min":1.5109999999999999,"5min":1.1799999999999999,"15min":1.1319999999999999},"min":{"1min":0.28000000000000003,"5min":0.27000000000000002,"15min":0.26900000000000002},"max":{"1min":9.7119999999999997,"5min":9.7119999999999997,"15min":11.973000000000001},"last":0.73699999999999999}]},{"osd":6,"last update":"Sat Sep 23 21:14:08 2023","interfaces":[{"interface":"back","average":{"1min":0.625,"5min":0.92200000000000004,"15min":1.056},"min":{"1min":0.38800000000000001,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":1.448,"5min":7.7030000000000003,"15min":11.128},"last":0.61399999999999999},{"interface":"front","average":{"1min":1.4350000000000001,"5min":1.4390000000000001,"15min":1.246},"min":{"1min":0.42099999999999999,"5min":0.32400000000000001,"15min":0.317},"max":{"1min":10.999000000000001,"5min":10.999000000000001,"15min":11.112},"last":0.502}]},{"osd":7,"last update":"Sat Sep 23 21:13:45 2023","interfaces":[{"interface":"back","average":{"1min":1.3,"5min":0.95099999999999996,"15min":1.804},"min":{"1min":0.34300000000000003,"5min":0.31,"15min":0.309},"max":{"1min":5.5659999999999998,"5min":5.5659999999999998,"15min":189.405},"last":0.58899999999999997},{"interface":"front","average":{"1min":2.004,"5min":1.5009999999999999,"15min":1.9159999999999999},"min":{"1min":0.40200000000000002,"5min":0.35399999999999998,"15min":0.32500000000000001},"max":{"1min":11.183999999999999,"5min":11.183999999999999,"15min":189.685},"last":0.88200000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8488,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1259,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5322,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":714,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":5434,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":4838,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":2308505600,"data_stored":2308505600,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1281294336,"data_stored":1281294336,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1426915328,"data_stored":1426915328,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1407324160,"data_stored":1407320088,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1280643072,"data_stored":1280639000,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1032650752,"data_stored":1032650752,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":849412096,"data_stored":849412096,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T21:14:30.688 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph status --format=json 2023-09-23T21:14:30.688 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:14:30.689 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:14:30.689 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:14:30.790 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:14:30.818 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:14:30 smithi070 ceph-mon[168282]: pgmap v1328: 129 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 15 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 108 active+clean; 2.1 GiB data, 11 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 527 op/s; 23883/105198 objects degraded (22.703%); 3977/105198 objects misplaced (3.780%); 7.3 MiB/s, 116 objects/s recovering 2023-09-23T21:14:30.821 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:14:30.876 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:14:30.881 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:14:30 smithi026 ceph-mon[179688]: pgmap v1328: 129 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 15 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 108 active+clean; 2.1 GiB data, 11 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 527 op/s; 23883/105198 objects degraded (22.703%); 3977/105198 objects misplaced (3.780%); 7.3 MiB/s, 116 objects/s recovering 2023-09-23T21:14:30.881 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:14:30 smithi026 ceph-mon[175151]: pgmap v1328: 129 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 15 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 108 active+clean; 2.1 GiB data, 11 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 527 op/s; 23883/105198 objects degraded (22.703%); 3977/105198 objects misplaced (3.780%); 7.3 MiB/s, 116 objects/s recovering 2023-09-23T21:14:30.886 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:14:30.887 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:14:30.988 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:14:31.073 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:14:31.137 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:14:31.141 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T21:14:31.148 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:14:31.149 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:14:31.249 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:14:31.250 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:14:31.295 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:14:31.351 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:14:31.366 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:14:31.366 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:14:31.467 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:14:31.509 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:14:31.615 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:14:31 smithi026 ceph-mon[179688]: from='client.56452 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T21:14:31.615 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:14:31 smithi026 ceph-mon[179688]: Health check update: Degraded data redundancy: 23883/105198 objects degraded (22.703%), 19 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-09-23T21:14:31.615 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:14:31 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:14:31.615 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:14:31 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:14:31.616 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:14:31 smithi026 ceph-mon[175151]: from='client.56452 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T21:14:31.616 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:14:31 smithi026 ceph-mon[175151]: Health check update: Degraded data redundancy: 23883/105198 objects degraded (22.703%), 19 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-09-23T21:14:31.616 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:14:31 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:14:31.616 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:14:31 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:14:31.871 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:14:31 smithi070 ceph-mon[168282]: from='client.56452 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T21:14:31.871 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:14:31 smithi070 ceph-mon[168282]: Health check update: Degraded data redundancy: 23883/105198 objects degraded (22.703%), 19 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-09-23T21:14:31.871 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:14:31 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:14:31.871 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:14:31 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:14:31.963 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:14:31.971 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:14:31.972 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:14:32.072 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:14:32.129 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:14:32.348 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T21:14:32.352 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:14:32.352 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:14:32.453 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:14:32.456 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:14:32.506 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:14:32.566 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:14:32.577 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:14:32.578 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:14:32.679 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:14:32.726 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:14:32.794 INFO:teuthology.orchestra.run.smithi026.stderr:noscrub is set 2023-09-23T21:14:32.848 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T21:14:32.859 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:14:33.017 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:14:33.017 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:14:32 smithi026 ceph-mon[175151]: pgmap v1329: 129 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 15 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 108 active+clean; 2.1 GiB data, 11 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 524 op/s; 23883/103338 objects degraded (23.112%); 3977/103338 objects misplaced (3.849%); 7.3 MiB/s, 116 objects/s recovering 2023-09-23T21:14:33.017 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:14:32 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/169895610' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T21:14:33.017 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:14:32 smithi026 ceph-mon[179688]: pgmap v1329: 129 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 15 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 108 active+clean; 2.1 GiB data, 11 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 524 op/s; 23883/103338 objects degraded (23.112%); 3977/103338 objects misplaced (3.849%); 7.3 MiB/s, 116 objects/s recovering 2023-09-23T21:14:33.017 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:14:32 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/169895610' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T21:14:33.018 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:14:32 smithi070 ceph-mon[168282]: pgmap v1329: 129 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 15 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 108 active+clean; 2.1 GiB data, 11 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 524 op/s; 23883/103338 objects degraded (23.112%); 3977/103338 objects misplaced (3.849%); 7.3 MiB/s, 116 objects/s recovering 2023-09-23T21:14:33.018 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:14:32 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/169895610' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T21:14:33.117 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:14:33.184 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:14:33.396 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:14:33.400 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:14:33.401 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:14:33.502 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:14:33.551 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:14:33.813 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:14:33 smithi026 ceph-mon[179688]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:14:33.813 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:14:33 smithi026 ceph-mon[179688]: failed to encode map e529 with expected crc 2023-09-23T21:14:33.813 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:14:33 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/169895610' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T21:14:33.813 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:14:33 smithi026 ceph-mon[179688]: osdmap e529: 8 total, 8 up, 6 in 2023-09-23T21:14:33.813 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:14:33 smithi026 ceph-mon[179688]: failed to encode map e529 with expected crc 2023-09-23T21:14:33.813 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:14:33 smithi026 ceph-mon[179688]: failed to encode map e529 with expected crc 2023-09-23T21:14:33.814 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:14:33 smithi026 ceph-mon[175151]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:14:33.815 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:14:33 smithi026 ceph-mon[175151]: failed to encode map e529 with expected crc 2023-09-23T21:14:33.815 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:14:33 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/169895610' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T21:14:33.815 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:14:33 smithi026 ceph-mon[175151]: osdmap e529: 8 total, 8 up, 6 in 2023-09-23T21:14:33.815 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:14:33 smithi026 ceph-mon[175151]: failed to encode map e529 with expected crc 2023-09-23T21:14:33.815 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:14:33 smithi026 ceph-mon[175151]: failed to encode map e529 with expected crc 2023-09-23T21:14:34.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:14:33 smithi070 ceph-mon[168282]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:14:34.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:14:33 smithi070 ceph-mon[168282]: failed to encode map e529 with expected crc 2023-09-23T21:14:34.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:14:33 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/169895610' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T21:14:34.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:14:33 smithi070 ceph-mon[168282]: osdmap e529: 8 total, 8 up, 6 in 2023-09-23T21:14:34.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:14:33 smithi070 ceph-mon[168282]: failed to encode map e529 with expected crc 2023-09-23T21:14:34.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:14:33 smithi070 ceph-mon[168282]: failed to encode map e529 with expected crc 2023-09-23T21:14:34.301 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0: {} 2023-09-23T21:14:34.302 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1: {} 2023-09-23T21:14:34.307 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2: {} 2023-09-23T21:14:34.348 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3: {} 2023-09-23T21:14:34.369 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:14:34.373 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4: {} 2023-09-23T21:14:34.375 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:14:34.375 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:14:34.425 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T21:14:34.466 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6: {} 2023-09-23T21:14:34.476 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:14:34.504 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T21:14:34.508 INFO:teuthology.orchestra.run.smithi026.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T21:14:34.508 INFO:teuthology.orchestra.run.smithi026.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T21:14:34.508 INFO:teuthology.orchestra.run.smithi026.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T21:14:34.508 INFO:teuthology.orchestra.run.smithi026.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T21:14:34.508 INFO:teuthology.orchestra.run.smithi026.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T21:14:34.508 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T21:14:34.509 INFO:teuthology.orchestra.run.smithi026.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T21:14:34.509 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T21:14:34.554 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:14:34.641 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:14:34 smithi026 ceph-mon[175151]: pgmap v1331: 129 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 15 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 108 active+clean; 2.0 GiB data, 11 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 465 op/s; 23883/100587 objects degraded (23.744%); 3887/100587 objects misplaced (3.864%); 4.5 MiB/s, 72 objects/s recovering 2023-09-23T21:14:34.642 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:14:34 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/743399828' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:14:34.643 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:14:34 smithi026 ceph-mon[179688]: pgmap v1331: 129 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 15 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 108 active+clean; 2.0 GiB data, 11 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 465 op/s; 23883/100587 objects degraded (23.744%); 3887/100587 objects misplaced (3.864%); 4.5 MiB/s, 72 objects/s recovering 2023-09-23T21:14:34.643 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:14:34 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/743399828' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:14:34.732 INFO:teuthology.orchestra.run.smithi026.stdout:ERROR: (22) Invalid argument 2023-09-23T21:14:34.733 INFO:teuthology.orchestra.run.smithi026.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T21:14:34.904 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:14:34.906 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:14:34.907 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:14:35.007 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:14:35.043 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:14:35.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:14:34 smithi070 ceph-mon[168282]: pgmap v1331: 129 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 15 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 108 active+clean; 2.0 GiB data, 11 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 465 op/s; 23883/100587 objects degraded (23.744%); 3887/100587 objects misplaced (3.864%); 4.5 MiB/s, 72 objects/s recovering 2023-09-23T21:14:35.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:14:34 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/743399828' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:14:35.310 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:14:35.320 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:14:35.321 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:14:35.422 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:14:35.486 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:14:35.539 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:14:35.546 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:14:35.546 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:14:35.615 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_ops_in_flight 2023-09-23T21:14:35.646 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:14:35.741 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:14:35.826 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:14:35.836 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:14:35.836 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:14:35.873 INFO:teuthology.orchestra.run.smithi026.stdout: 2023-09-23T21:14:35.873 INFO:teuthology.orchestra.run.smithi026.stdout:{"fsid":"b3f8e4ec-5a50-11ee-8db4-212e2dc638e7","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"noscrub flag(s) set","count":7},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 23377/95424 objects degraded (24.498%), 19 pgs degraded, 12 pgs undersized","count":31},"muted":false}},"mutes":[]},"election_epoch":30,"quorum":[0,1,2],"quorum_names":["a","b","c"],"quorum_age":1594,"monmap":{"epoch":4,"min_mon_release_name":"reef","num_mons":3},"osdmap":{"epoch":529,"num_osds":8,"num_up_osds":8,"osd_up_since":1695503591,"num_in_osds":6,"osd_in_since":1695503530,"num_remapped_pgs":18},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":108},{"state_name":"active+recovery_wait+undersized+degraded+remapped","count":15},{"state_name":"active+recovery_wait+degraded","count":3},{"state_name":"active+recovery_wait+degraded+remapped","count":1},{"state_name":"active+remapped+backfill_wait","count":1},{"state_name":"active+recovering+undersized+remapped","count":1}],"num_pgs":129,"num_pools":5,"num_objects":31808,"data_bytes":2083437316,"bytes_used":11158978560,"bytes_avail":564811235328,"bytes_total":575970213888,"degraded_objects":23377,"degraded_total":95424,"degraded_ratio":0.24498029845741112,"misplaced_objects":3799,"misplaced_total":95424,"misplaced_ratio":0.039811787391012744,"recovering_objects_per_sec":135,"recovering_bytes_per_sec":8852155,"recovering_keys_per_sec":0,"num_objects_recovered":1279,"num_bytes_recovered":83820544,"num_keys_recovered":0,"write_bytes_sec":0,"read_op_per_sec":0,"write_op_per_sec":548},"fsmap":{"epoch":8,"id":1,"up":1,"in":1,"max":1,"by_rank":[{"filesystem_id":1,"rank":0,"name":"foo.smithi026.znfpnl","status":"up:active","gid":14658}],"up:standby":1},"mgrmap":{"available":true,"num_standbys":1,"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"services":{"dashboard":"https://172.21.15.26:8443/","prometheus":"http://172.21.15.26:9283/"}},"servicemap":{"epoch":28,"modified":"2023-09-23T21:14:11.047835+0000","services":{}},"progress_events":{"0ed42496-3a31-41bd-a432-1bbd8c010433":{"message":"Global Recovery Event (2m)\n [=======================.....] (remaining: 31s)","progress":0.83720928430557251,"add_to_ceph_s":true},"fc746e9d-3f76-434d-b6d2-8fc9e8a72a5e":{"message":"Upgrade to 18.2.0-460-gbb0daeff (4m)\n [============................] (remaining: 5m)","progress":0.44999998807907104,"add_to_ceph_s":true}}} 2023-09-23T21:14:35.937 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:14:36.029 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:14:36.251 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:14:36.257 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:14:36.257 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:14:36.326 INFO:tasks.cephadm.ceph_manager.ceph:making progress, resetting timeout 2023-09-23T21:14:36.326 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T21:14:36.359 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:14:36.413 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:14:36.428 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T21:14:36.481 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T21:14:36.485 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:14:36.486 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:14:36.586 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:14:36.635 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:14:36.636 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:14:36.692 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:14:36.695 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:14:36.696 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:14:36.796 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:14:36.841 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:14:36.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:14:36 smithi026 ceph-mon[179688]: pgmap v1332: 129 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 15 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 108 active+clean; 1.9 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 548 op/s; 23377/95424 objects degraded (24.498%); 3799/95424 objects misplaced (3.981%); 8.4 MiB/s, 135 objects/s recovering 2023-09-23T21:14:36.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:14:36 smithi026 ceph-mon[179688]: Health check update: Degraded data redundancy: 23377/95424 objects degraded (24.498%), 19 pgs degraded, 12 pgs undersized (PG_DEGRADED) 2023-09-23T21:14:36.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:14:36 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/276588520' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-23T21:14:36.883 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:14:36 smithi026 ceph-mon[175151]: pgmap v1332: 129 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 15 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 108 active+clean; 1.9 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 548 op/s; 23377/95424 objects degraded (24.498%); 3799/95424 objects misplaced (3.981%); 8.4 MiB/s, 135 objects/s recovering 2023-09-23T21:14:36.883 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:14:36 smithi026 ceph-mon[175151]: Health check update: Degraded data redundancy: 23377/95424 objects degraded (24.498%), 19 pgs degraded, 12 pgs undersized (PG_DEGRADED) 2023-09-23T21:14:36.883 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:14:36 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/276588520' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-23T21:14:36.892 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:14:36.896 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:14:36 smithi070 ceph-mon[168282]: pgmap v1332: 129 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 15 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 108 active+clean; 1.9 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 548 op/s; 23377/95424 objects degraded (24.498%); 3799/95424 objects misplaced (3.981%); 8.4 MiB/s, 135 objects/s recovering 2023-09-23T21:14:36.896 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:14:36 smithi070 ceph-mon[168282]: Health check update: Degraded data redundancy: 23377/95424 objects degraded (24.498%), 19 pgs degraded, 12 pgs undersized (PG_DEGRADED) 2023-09-23T21:14:36.896 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:14:36 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/276588520' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-23T21:14:36.900 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:14:36.901 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:14:37.002 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:14:37.055 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:14:37.080 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T21:14:37.172 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:14:37.184 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:14:37.184 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:14:37.285 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:14:37.334 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:14:37.401 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:14:37.402 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:14:37.402 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:14:37.503 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:14:37.558 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:14:37.598 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:14:37.637 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T21:14:37.644 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:14:37.645 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:14:37.745 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:14:37.783 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:14:37.786 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:14:37.844 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:14:37.848 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:14:37.849 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:14:37.884 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:14:37.950 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:14:38.014 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:14:38.068 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:14:38.074 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:14:38.074 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:14:38.175 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:14:38.247 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:14:38.371 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:14:38.376 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:14:38.377 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:14:38.477 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:14:38.537 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:14:38.592 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:14:38.599 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:14:38.600 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:14:38.701 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:14:38.756 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:14:38.762 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:14:38 smithi026 ceph-mon[175151]: pgmap v1333: 129 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 15 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 108 active+clean; 1.9 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 572 op/s; 23377/91758 objects degraded (25.477%); 3694/91758 objects misplaced (4.026%); 8.0 MiB/s, 127 objects/s recovering 2023-09-23T21:14:38.762 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:14:38 smithi026 ceph-mon[179688]: pgmap v1333: 129 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 15 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 108 active+clean; 1.9 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 572 op/s; 23377/91758 objects degraded (25.477%); 3694/91758 objects misplaced (4.026%); 8.0 MiB/s, 127 objects/s recovering 2023-09-23T21:14:38.821 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:14:38.828 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:14:38.829 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:14:38 smithi070 ceph-mon[168282]: pgmap v1333: 129 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 15 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 108 active+clean; 1.9 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 572 op/s; 23377/91758 objects degraded (25.477%); 3694/91758 objects misplaced (4.026%); 8.0 MiB/s, 127 objects/s recovering 2023-09-23T21:14:38.830 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:14:38.930 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:14:38.988 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:14:39.316 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:14:39.326 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:14:39.328 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:14:39.428 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:14:39.471 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:14:39.519 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:14:39.522 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:14:39.523 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:14:39.614 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 21:14:39 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[164029]: ::ffff:172.21.15.70 - - [23/Sep/2023:21:14:39] "GET /metrics HTTP/1.1" 200 36403 "" "Prometheus/2.43.0" 2023-09-23T21:14:39.623 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:14:39.682 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:14:41.035 INFO:teuthology.orchestra.run.smithi026.stdout:ERROR: (22) Invalid argument 2023-09-23T21:14:41.035 INFO:teuthology.orchestra.run.smithi026.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T21:14:41.121 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:14:40 smithi070 ceph-mon[168282]: pgmap v1334: 129 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 15 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 108 active+clean; 1.8 GiB data, 9.9 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 605 op/s; 22773/87012 objects degraded (26.172%); 3684/87012 objects misplaced (4.234%); 8.8 MiB/s, 141 objects/s recovering 2023-09-23T21:14:41.131 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:14:40 smithi026 ceph-mon[175151]: pgmap v1334: 129 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 15 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 108 active+clean; 1.8 GiB data, 9.9 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 605 op/s; 22773/87012 objects degraded (26.172%); 3684/87012 objects misplaced (4.234%); 8.8 MiB/s, 141 objects/s recovering 2023-09-23T21:14:41.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:14:40 smithi026 ceph-mon[179688]: pgmap v1334: 129 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 15 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 108 active+clean; 1.8 GiB data, 9.9 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 605 op/s; 22773/87012 objects degraded (26.172%); 3684/87012 objects misplaced (4.234%); 8.8 MiB/s, 141 objects/s recovering 2023-09-23T21:14:41.870 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:14:41 smithi026 ceph-mon[175151]: Health check update: Degraded data redundancy: 22773/87012 objects degraded (26.172%), 19 pgs degraded, 13 pgs undersized (PG_DEGRADED) 2023-09-23T21:14:41.870 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:14:41 smithi026 ceph-mon[179688]: Health check update: Degraded data redundancy: 22773/87012 objects degraded (26.172%), 19 pgs degraded, 13 pgs undersized (PG_DEGRADED) 2023-09-23T21:14:42.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:14:41 smithi070 ceph-mon[168282]: Health check update: Degraded data redundancy: 22773/87012 objects degraded (26.172%), 19 pgs degraded, 13 pgs undersized (PG_DEGRADED) 2023-09-23T21:14:42.691 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T21:14:42.699 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:14:42.700 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:14:42.800 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:14:42.965 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:14:42 smithi070 ceph-mon[168282]: pgmap v1335: 129 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 15 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 108 active+clean; 1.7 GiB data, 9.7 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 629 op/s; 22773/84444 objects degraded (26.968%); 3684/84444 objects misplaced (4.363%); 8.8 MiB/s, 141 objects/s recovering 2023-09-23T21:14:42.966 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:14:43.023 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:14:43.033 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:14:43.034 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:14:43.131 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:14:42 smithi026 ceph-mon[175151]: pgmap v1335: 129 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 15 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 108 active+clean; 1.7 GiB data, 9.7 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 629 op/s; 22773/84444 objects degraded (26.968%); 3684/84444 objects misplaced (4.363%); 8.8 MiB/s, 141 objects/s recovering 2023-09-23T21:14:43.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:14:42 smithi026 ceph-mon[179688]: pgmap v1335: 129 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 15 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 108 active+clean; 1.7 GiB data, 9.7 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 629 op/s; 22773/84444 objects degraded (26.968%); 3684/84444 objects misplaced (4.363%); 8.8 MiB/s, 141 objects/s recovering 2023-09-23T21:14:43.134 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:14:43.175 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_blocked_ops 2023-09-23T21:14:43.207 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:14:43.319 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:14:43.331 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:14:43.331 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:14:43.432 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:14:43.467 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:14:43.521 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:14:43.526 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:14:43.526 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:14:43.627 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:14:43.700 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:14:43.907 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:14:43.912 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:14:43.912 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:14:44.013 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:14:44.100 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:14:44.698 INFO:teuthology.orchestra.run.smithi026.stderr:nodeep-scrub is set 2023-09-23T21:14:44.710 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:14:44.727 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:14:44.728 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:14:44.746 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0: {} 2023-09-23T21:14:44.792 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1: {} 2023-09-23T21:14:44.828 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:14:44.829 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:14:44.854 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2: {} 2023-09-23T21:14:44.856 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:14:44.898 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:14:44.906 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:14:44.907 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:14:44.910 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3: {} 2023-09-23T21:14:44.944 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4: {} 2023-09-23T21:14:45.001 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T21:14:45.008 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:14:45.034 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6: {} 2023-09-23T21:14:45.042 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:14:44 smithi070 ceph-mon[168282]: pgmap v1336: 129 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 15 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 108 active+clean; 1.6 GiB data, 9.4 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 646 op/s; 22773/80619 objects degraded (28.248%); 3588/80619 objects misplaced (4.451%); 8.6 MiB/s, 137 objects/s recovering 2023-09-23T21:14:45.042 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:14:44 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/3707056521' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:14:45.043 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:14:44 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:14:45.051 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:14:45.071 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T21:14:45.082 INFO:teuthology.orchestra.run.smithi026.stdout: 2023-09-23T21:14:45.082 INFO:teuthology.orchestra.run.smithi026.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T21:14:45.082 INFO:teuthology.orchestra.run.smithi026.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T21:14:45.082 INFO:teuthology.orchestra.run.smithi026.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T21:14:45.082 INFO:teuthology.orchestra.run.smithi026.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T21:14:45.082 INFO:teuthology.orchestra.run.smithi026.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T21:14:45.082 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T21:14:45.083 INFO:teuthology.orchestra.run.smithi026.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T21:14:45.083 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T21:14:45.091 INFO:teuthology.orchestra.run.smithi026.stderr:dumped all 2023-09-23T21:14:45.110 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:14:45.113 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:14:44 smithi026 ceph-mon[175151]: pgmap v1336: 129 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 15 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 108 active+clean; 1.6 GiB data, 9.4 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 646 op/s; 22773/80619 objects degraded (28.248%); 3588/80619 objects misplaced (4.451%); 8.6 MiB/s, 137 objects/s recovering 2023-09-23T21:14:45.114 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:14:44 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/3707056521' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:14:45.114 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:14:44 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:14:45.114 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:14:44 smithi026 ceph-mon[179688]: pgmap v1336: 129 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 15 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 108 active+clean; 1.6 GiB data, 9.4 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 646 op/s; 22773/80619 objects degraded (28.248%); 3588/80619 objects misplaced (4.451%); 8.6 MiB/s, 137 objects/s recovering 2023-09-23T21:14:45.114 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:14:44 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/3707056521' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:14:45.114 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:14:44 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:14:45.115 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:14:45.115 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:14:45.216 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:14:45.286 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:14:45.926 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:14:45.930 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:14:45.930 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:14:45.968 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:14:45 smithi026 ceph-mon[179688]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:14:45.968 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:14:45 smithi026 ceph-mon[179688]: failed to encode map e530 with expected crc 2023-09-23T21:14:45.968 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:14:45 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T21:14:45.968 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:14:45 smithi026 ceph-mon[179688]: osdmap e530: 8 total, 8 up, 6 in 2023-09-23T21:14:45.968 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:14:45 smithi026 ceph-mon[179688]: failed to encode map e530 with expected crc 2023-09-23T21:14:45.969 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:14:45 smithi026 ceph-mon[179688]: failed to encode map e530 with expected crc 2023-09-23T21:14:45.969 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:14:45 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/241882249' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:14:45.969 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:14:45 smithi026 ceph-mon[175151]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:14:45.969 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:14:45 smithi026 ceph-mon[175151]: failed to encode map e530 with expected crc 2023-09-23T21:14:45.969 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:14:45 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T21:14:45.969 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:14:45 smithi026 ceph-mon[175151]: osdmap e530: 8 total, 8 up, 6 in 2023-09-23T21:14:45.969 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:14:45 smithi026 ceph-mon[175151]: failed to encode map e530 with expected crc 2023-09-23T21:14:45.969 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:14:45 smithi026 ceph-mon[175151]: failed to encode map e530 with expected crc 2023-09-23T21:14:45.970 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:14:45 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/241882249' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:14:46.030 INFO:teuthology.orchestra.run.smithi026.stdout:{"pg_ready":true,"pg_map":{"version":1337,"stamp":"2023-09-23T21:14:44.682652+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":1623243524,"num_objects":24786,"num_object_clones":0,"num_object_copies":74358,"num_objects_missing_on_primary":1388,"num_objects_missing":1388,"num_objects_degraded":22120,"num_objects_misplaced":3570,"num_objects_unfound":0,"num_objects_dirty":24786,"num_whiteouts":0,"num_read":90,"num_read_kb":166,"num_write":122718,"num_write_kb":4309782,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":41009,"num_bytes_recovered":2474675365,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":135958,"ondisk_log_size":135958,"up":387,"acting":371,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":433,"num_osds":6,"num_per_pool_osds":6,"num_per_pool_omap_osds":6,"kb":562470912,"kb_used":9637968,"kb_used_data":6404256,"kb_used_omap":125,"kb_used_meta":3233154,"kb_avail":552832944,"statfs":{"total":575970213888,"available":566100934656,"internally_reserved":0,"allocated":6557958144,"data_stored":6545946385,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":128662,"internal_metadata":3310750058},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-572981248,"num_objects":-8743,"num_object_clones":0,"num_object_copies":-26229,"num_objects_missing_on_primary":-972,"num_objects_missing":-972,"num_objects_degraded":-1763,"num_objects_misplaced":-317,"num_objects_unfound":0,"num_objects_dirty":-8743,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8743,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2198,"num_bytes_recovered":144048128,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"11.622081"},"pg_stats":[{"pgid":"1.18","version":"79'161","reported_seq":649,"reported_epoch":528,"state":"active+clean","last_fresh":"2023-09-23T21:14:25.596482+0000","last_change":"2023-09-23T21:12:30.438733+0000","last_active":"2023-09-23T21:14:25.596482+0000","last_peered":"2023-09-23T21:14:25.596482+0000","last_clean":"2023-09-23T21:14:25.596482+0000","last_became_active":"2023-09-23T21:12:30.438356+0000","last_became_peered":"2023-09-23T21:12:30.438356+0000","last_unstale":"2023-09-23T21:14:25.596482+0000","last_undegraded":"2023-09-23T21:14:25.596482+0000","last_fullsized":"2023-09-23T21:14:25.596482+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":479,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:06:52.098882+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:06:52.098882+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0092173299999999993,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":131344,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"10.13","version":"529'5589","reported_seq":9378,"reported_epoch":529,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T21:14:43.304735+0000","last_change":"2023-09-23T21:13:49.840403+0000","last_active":"2023-09-23T21:14:43.304735+0000","last_peered":"2023-09-23T21:14:43.304735+0000","last_clean":"2023-09-23T21:11:26.718194+0000","last_became_active":"2023-09-23T21:13:48.181112+0000","last_became_peered":"2023-09-23T21:13:48.181112+0000","last_unstale":"2023-09-23T21:14:43.304735+0000","last_undegraded":"2023-09-23T21:13:48.139049+0000","last_fullsized":"2023-09-23T21:13:45.405990+0000","mapping_epoch":513,"log_start":"426'622","ondisk_log_start":"426'622","created":438,"last_epoch_clean":439,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_clean_scrub_stamp":"2023-09-23T21:10:15.663179+0000","objects_scrubbed":0,"log_size":4967,"log_dups_size":0,"ondisk_log_size":4967,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53248000,"num_objects":813,"num_object_clones":0,"num_object_copies":2439,"num_objects_missing_on_primary":442,"num_objects_missing":442,"num_objects_degraded":642,"num_objects_misplaced":725,"num_objects_unfound":0,"num_objects_dirty":813,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4221,"num_write_kb":161056,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1704,"num_bytes_recovered":111607808,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[4,3],"avail_no_missing":["5"],"object_location_counts":[{"shards":"0,2,4,5","objects":307},{"shards":"2,3,4,5","objects":2},{"shards":"2,3,5","objects":442},{"shards":"2,4,5","objects":28},{"shards":"3,4","objects":34}],"blocked_by":[],"up_primary":2,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1a","version":"68'1","reported_seq":119,"reported_epoch":528,"state":"active+clean","last_fresh":"2023-09-23T21:14:25.608974+0000","last_change":"2023-09-23T21:13:13.291403+0000","last_active":"2023-09-23T21:14:25.608974+0000","last_peered":"2023-09-23T21:14:25.608974+0000","last_clean":"2023-09-23T21:14:25.608974+0000","last_became_active":"2023-09-23T21:13:13.291123+0000","last_became_peered":"2023-09-23T21:13:13.291123+0000","last_unstale":"2023-09-23T21:14:25.608974+0000","last_undegraded":"2023-09-23T21:14:25.608974+0000","last_fullsized":"2023-09-23T21:14:25.608974+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":495,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:45:15.427430+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,0],"acting":[4,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":188,"reported_epoch":528,"state":"active+clean","last_fresh":"2023-09-23T21:14:25.607950+0000","last_change":"2023-09-23T21:12:10.767963+0000","last_active":"2023-09-23T21:14:25.607950+0000","last_peered":"2023-09-23T21:14:25.607950+0000","last_clean":"2023-09-23T21:14:25.607950+0000","last_became_active":"2023-09-23T21:12:10.767126+0000","last_became_peered":"2023-09-23T21:12:10.767126+0000","last_unstale":"2023-09-23T21:14:25.607950+0000","last_undegraded":"2023-09-23T21:14:25.607950+0000","last_fullsized":"2023-09-23T21:14:25.607950+0000","mapping_epoch":470,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:45:47.138277+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,5],"acting":[4,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1b","version":"79'161","reported_seq":532,"reported_epoch":528,"state":"active+clean","last_fresh":"2023-09-23T21:14:25.604258+0000","last_change":"2023-09-23T21:13:13.373548+0000","last_active":"2023-09-23T21:14:25.604258+0000","last_peered":"2023-09-23T21:14:25.604258+0000","last_clean":"2023-09-23T21:14:25.604258+0000","last_became_active":"2023-09-23T21:13:13.372845+0000","last_became_peered":"2023-09-23T21:13:13.372845+0000","last_unstale":"2023-09-23T21:14:25.604258+0000","last_undegraded":"2023-09-23T21:14:25.604258+0000","last_fullsized":"2023-09-23T21:14:25.604258+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":479,"last_epoch_clean":495,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:05:55.772264+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:05:55.772264+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.001862507,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.10","version":"529'5594","reported_seq":13526,"reported_epoch":529,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T21:14:43.250424+0000","last_change":"2023-09-23T21:13:36.071975+0000","last_active":"2023-09-23T21:14:43.250424+0000","last_peered":"2023-09-23T21:14:43.250424+0000","last_clean":"2023-09-23T21:11:26.716051+0000","last_became_active":"2023-09-23T21:13:33.619010+0000","last_became_peered":"2023-09-23T21:13:33.619010+0000","last_unstale":"2023-09-23T21:14:43.250424+0000","last_undegraded":"2023-09-23T21:13:33.443511+0000","last_fullsized":"2023-09-23T21:13:31.753181+0000","mapping_epoch":502,"log_start":"426'605","ondisk_log_start":"426'605","created":438,"last_epoch_clean":450,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_clean_scrub_stamp":"2023-09-23T21:10:15.663179+0000","objects_scrubbed":0,"log_size":4989,"log_dups_size":0,"ondisk_log_size":4989,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":48332800,"num_objects":738,"num_object_clones":0,"num_object_copies":2214,"num_objects_missing_on_primary":400,"num_objects_missing":400,"num_objects_degraded":1226,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":738,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4230,"num_write_kb":158944,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3623,"num_bytes_recovered":237436928,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7],"avail_no_missing":["1","5","7"],"object_location_counts":[{"shards":"0,5,7","objects":400},{"shards":"4,5,7","objects":29},{"shards":"4,7","objects":309}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":525,"reported_epoch":529,"state":"active+clean","last_fresh":"2023-09-23T21:14:32.778645+0000","last_change":"2023-09-23T21:12:10.824116+0000","last_active":"2023-09-23T21:14:32.778645+0000","last_peered":"2023-09-23T21:14:32.778645+0000","last_clean":"2023-09-23T21:14:32.778645+0000","last_became_active":"2023-09-23T21:12:10.823849+0000","last_became_peered":"2023-09-23T21:12:10.823849+0000","last_unstale":"2023-09-23T21:14:32.778645+0000","last_undegraded":"2023-09-23T21:14:32.778645+0000","last_fullsized":"2023-09-23T21:14:32.778645+0000","mapping_epoch":470,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:54:13.777603+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":525,"reported_epoch":528,"state":"active+clean","last_fresh":"2023-09-23T21:14:25.605703+0000","last_change":"2023-09-23T21:13:44.413713+0000","last_active":"2023-09-23T21:14:25.605703+0000","last_peered":"2023-09-23T21:14:25.605703+0000","last_clean":"2023-09-23T21:14:25.605703+0000","last_became_active":"2023-09-23T21:13:44.413366+0000","last_became_peered":"2023-09-23T21:13:44.413366+0000","last_unstale":"2023-09-23T21:14:25.605703+0000","last_undegraded":"2023-09-23T21:14:25.605703+0000","last_fullsized":"2023-09-23T21:14:25.605703+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:31:56.349530+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1a","version":"79'161","reported_seq":674,"reported_epoch":528,"state":"active+clean","last_fresh":"2023-09-23T21:14:25.596818+0000","last_change":"2023-09-23T21:12:30.444665+0000","last_active":"2023-09-23T21:14:25.596818+0000","last_peered":"2023-09-23T21:14:25.596818+0000","last_clean":"2023-09-23T21:14:25.596818+0000","last_became_active":"2023-09-23T21:12:30.444326+0000","last_became_peered":"2023-09-23T21:12:30.444326+0000","last_unstale":"2023-09-23T21:14:25.596818+0000","last_undegraded":"2023-09-23T21:14:25.596818+0000","last_fullsized":"2023-09-23T21:14:25.596818+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":479,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:06:51.075448+0000","last_deep_scrub":"79'161","last_deep_scrub_stamp":"2023-09-23T21:05:32.630292+0000","last_clean_scrub_stamp":"2023-09-23T21:06:51.075448+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0054793890000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,4],"acting":[2,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"10.11","version":"529'5484","reported_seq":12315,"reported_epoch":529,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-23T21:14:41.182027+0000","last_change":"2023-09-23T21:13:31.864763+0000","last_active":"2023-09-23T21:14:41.182027+0000","last_peered":"2023-09-23T21:14:41.182027+0000","last_clean":"2023-09-23T21:13:30.058478+0000","last_became_active":"2023-09-23T21:13:31.850860+0000","last_became_peered":"2023-09-23T21:13:31.850860+0000","last_unstale":"2023-09-23T21:14:41.182027+0000","last_undegraded":"2023-09-23T21:14:41.182027+0000","last_fullsized":"2023-09-23T21:14:41.182027+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":438,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":5,"last_scrub":"441'2974","last_scrub_stamp":"2023-09-23T21:11:06.133218+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_clean_scrub_stamp":"2023-09-23T21:11:06.133218+0000","objects_scrubbed":1646,"log_size":5484,"log_dups_size":0,"ondisk_log_size":5484,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:54:59.366711+0000","scrub_duration":0.3173955,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":51576832,"num_objects":787,"num_object_clones":0,"num_object_copies":2361,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":787,"num_objects_unfound":0,"num_objects_dirty":787,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2513,"num_write_kb":52928,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2152,"num_bytes_recovered":141033472,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[2,5,7],"avail_no_missing":["2","1","5","7"],"object_location_counts":[{"shards":"2,5,7","objects":787}],"blocked_by":[],"up_primary":0,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":38,"reported_epoch":528,"state":"active+clean","last_fresh":"2023-09-23T21:14:25.602959+0000","last_change":"2023-09-23T21:13:30.806084+0000","last_active":"2023-09-23T21:14:25.602959+0000","last_peered":"2023-09-23T21:14:25.602959+0000","last_clean":"2023-09-23T21:14:25.602959+0000","last_became_active":"2023-09-23T21:13:30.805743+0000","last_became_peered":"2023-09-23T21:13:30.805743+0000","last_unstale":"2023-09-23T21:14:25.602959+0000","last_undegraded":"2023-09-23T21:14:25.602959+0000","last_fullsized":"2023-09-23T21:14:25.602959+0000","mapping_epoch":501,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:48:10.849281+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":402,"reported_epoch":528,"state":"active+clean","last_fresh":"2023-09-23T21:14:25.609192+0000","last_change":"2023-09-23T21:13:44.410921+0000","last_active":"2023-09-23T21:14:25.609192+0000","last_peered":"2023-09-23T21:14:25.609192+0000","last_clean":"2023-09-23T21:14:25.609192+0000","last_became_active":"2023-09-23T21:13:44.410633+0000","last_became_peered":"2023-09-23T21:13:44.410633+0000","last_unstale":"2023-09-23T21:14:25.609192+0000","last_undegraded":"2023-09-23T21:14:25.609192+0000","last_fullsized":"2023-09-23T21:14:25.609192+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:57:10.116786+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.15","version":"79'161","reported_seq":555,"reported_epoch":528,"state":"active+clean","last_fresh":"2023-09-23T21:14:25.607063+0000","last_change":"2023-09-23T21:13:38.379453+0000","last_active":"2023-09-23T21:14:25.607063+0000","last_peered":"2023-09-23T21:14:25.607063+0000","last_clean":"2023-09-23T21:14:25.607063+0000","last_became_active":"2023-09-23T21:13:38.376499+0000","last_became_peered":"2023-09-23T21:13:38.376499+0000","last_unstale":"2023-09-23T21:14:25.607063+0000","last_undegraded":"2023-09-23T21:14:25.607063+0000","last_fullsized":"2023-09-23T21:14:25.607063+0000","mapping_epoch":507,"log_start":"0'0","ondisk_log_start":"0'0","created":479,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:09:34.892947+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:09:34.892947+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13418,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":11,"num_read_kb":21,"num_write":6,"num_write_kb":55,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":40253,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":503,"reported_epoch":528,"state":"active+clean","last_fresh":"2023-09-23T21:14:25.609282+0000","last_change":"2023-09-23T21:13:12.235728+0000","last_active":"2023-09-23T21:14:25.609282+0000","last_peered":"2023-09-23T21:14:25.609282+0000","last_clean":"2023-09-23T21:14:25.609282+0000","last_became_active":"2023-09-23T21:13:12.216549+0000","last_became_peered":"2023-09-23T21:13:12.216549+0000","last_unstale":"2023-09-23T21:14:25.609282+0000","last_undegraded":"2023-09-23T21:14:25.609282+0000","last_fullsized":"2023-09-23T21:14:25.609282+0000","mapping_epoch":493,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:39:33.324239+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":567,"reported_epoch":528,"state":"active+clean","last_fresh":"2023-09-23T21:14:29.204985+0000","last_change":"2023-09-23T21:12:10.790349+0000","last_active":"2023-09-23T21:14:29.204985+0000","last_peered":"2023-09-23T21:14:29.204985+0000","last_clean":"2023-09-23T21:14:29.204985+0000","last_became_active":"2023-09-23T21:12:10.789977+0000","last_became_peered":"2023-09-23T21:12:10.789977+0000","last_unstale":"2023-09-23T21:14:29.204985+0000","last_undegraded":"2023-09-23T21:14:29.204985+0000","last_fullsized":"2023-09-23T21:14:29.204985+0000","mapping_epoch":470,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:49:40.191147+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,5],"acting":[3,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"79'161","reported_seq":890,"reported_epoch":528,"state":"active+clean","last_fresh":"2023-09-23T21:14:25.598419+0000","last_change":"2023-09-23T21:11:32.760292+0000","last_active":"2023-09-23T21:14:25.598419+0000","last_peered":"2023-09-23T21:14:25.598419+0000","last_clean":"2023-09-23T21:14:25.598419+0000","last_became_active":"2023-09-23T21:11:32.759909+0000","last_became_peered":"2023-09-23T21:11:32.759909+0000","last_unstale":"2023-09-23T21:14:25.598419+0000","last_undegraded":"2023-09-23T21:14:25.598419+0000","last_fullsized":"2023-09-23T21:14:25.598419+0000","mapping_epoch":457,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":458,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:25.096401+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:25.096401+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:13:09.225685+0000","scrub_duration":0.002859393,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":253,"reported_epoch":528,"state":"active+clean","last_fresh":"2023-09-23T21:14:25.619060+0000","last_change":"2023-09-23T21:12:10.779468+0000","last_active":"2023-09-23T21:14:25.619060+0000","last_peered":"2023-09-23T21:14:25.619060+0000","last_clean":"2023-09-23T21:14:25.619060+0000","last_became_active":"2023-09-23T21:12:10.779282+0000","last_became_peered":"2023-09-23T21:12:10.779282+0000","last_unstale":"2023-09-23T21:14:25.619060+0000","last_undegraded":"2023-09-23T21:14:25.619060+0000","last_fullsized":"2023-09-23T21:14:25.619060+0000","mapping_epoch":470,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:36:35.417986+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":504,"reported_epoch":528,"state":"active+clean","last_fresh":"2023-09-23T21:14:29.205007+0000","last_change":"2023-09-23T21:10:26.758958+0000","last_active":"2023-09-23T21:14:29.205007+0000","last_peered":"2023-09-23T21:14:29.205007+0000","last_clean":"2023-09-23T21:14:29.205007+0000","last_became_active":"2023-09-23T21:10:26.758539+0000","last_became_peered":"2023-09-23T21:10:26.758539+0000","last_unstale":"2023-09-23T21:14:29.205007+0000","last_undegraded":"2023-09-23T21:14:29.205007+0000","last_fullsized":"2023-09-23T21:14:29.205007+0000","mapping_epoch":423,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":424,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:44:42.319189+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.17","version":"79'161","reported_seq":621,"reported_epoch":528,"state":"active+clean","last_fresh":"2023-09-23T21:14:25.607991+0000","last_change":"2023-09-23T21:13:44.416768+0000","last_active":"2023-09-23T21:14:25.607991+0000","last_peered":"2023-09-23T21:14:25.607991+0000","last_clean":"2023-09-23T21:14:25.607991+0000","last_became_active":"2023-09-23T21:13:44.416156+0000","last_became_peered":"2023-09-23T21:13:44.416156+0000","last_unstale":"2023-09-23T21:14:25.607991+0000","last_undegraded":"2023-09-23T21:14:25.607991+0000","last_fullsized":"2023-09-23T21:14:25.607991+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":479,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:25.916867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:25.916867+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.001665323,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":413,"reported_epoch":528,"state":"active+clean","last_fresh":"2023-09-23T21:14:25.609035+0000","last_change":"2023-09-23T21:11:08.766045+0000","last_active":"2023-09-23T21:14:25.609035+0000","last_peered":"2023-09-23T21:14:25.609035+0000","last_clean":"2023-09-23T21:14:25.609035+0000","last_became_active":"2023-09-23T21:11:08.758504+0000","last_became_peered":"2023-09-23T21:11:08.758504+0000","last_unstale":"2023-09-23T21:14:25.609035+0000","last_undegraded":"2023-09-23T21:14:25.609035+0000","last_fullsized":"2023-09-23T21:14:25.609035+0000","mapping_epoch":443,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":444,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:02:19.482574+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":360,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,4],"acting":[3,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":216,"reported_epoch":528,"state":"active+clean","last_fresh":"2023-09-23T21:14:31.288132+0000","last_change":"2023-09-23T21:13:30.821489+0000","last_active":"2023-09-23T21:14:31.288132+0000","last_peered":"2023-09-23T21:14:31.288132+0000","last_clean":"2023-09-23T21:14:31.288132+0000","last_became_active":"2023-09-23T21:13:30.821125+0000","last_became_peered":"2023-09-23T21:13:30.821125+0000","last_unstale":"2023-09-23T21:14:31.288132+0000","last_undegraded":"2023-09-23T21:14:31.288132+0000","last_fullsized":"2023-09-23T21:14:31.288132+0000","mapping_epoch":501,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:47:18.553417+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,5],"acting":[2,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.16","version":"79'161","reported_seq":447,"reported_epoch":528,"state":"active+clean","last_fresh":"2023-09-23T21:14:25.605639+0000","last_change":"2023-09-23T21:13:33.770962+0000","last_active":"2023-09-23T21:14:25.605639+0000","last_peered":"2023-09-23T21:14:25.605639+0000","last_clean":"2023-09-23T21:14:25.605639+0000","last_became_active":"2023-09-23T21:13:33.770407+0000","last_became_peered":"2023-09-23T21:13:33.770407+0000","last_unstale":"2023-09-23T21:14:25.605639+0000","last_undegraded":"2023-09-23T21:14:25.605639+0000","last_fullsized":"2023-09-23T21:14:25.605639+0000","mapping_epoch":504,"log_start":"0'0","ondisk_log_start":"0'0","created":479,"last_epoch_clean":505,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:10:24.268662+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:10:24.268662+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13418,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":11,"num_read_kb":21,"num_write":6,"num_write_kb":55,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":40252,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.14","version":"79'22","reported_seq":92,"reported_epoch":528,"state":"active+clean","last_fresh":"2023-09-23T21:14:25.608213+0000","last_change":"2023-09-23T21:11:27.770162+0000","last_active":"2023-09-23T21:14:25.608213+0000","last_peered":"2023-09-23T21:14:25.608213+0000","last_clean":"2023-09-23T21:14:25.608213+0000","last_became_active":"2023-09-23T21:11:27.752275+0000","last_became_peered":"2023-09-23T21:11:27.752275+0000","last_unstale":"2023-09-23T21:14:25.608213+0000","last_undegraded":"2023-09-23T21:14:25.608213+0000","last_fullsized":"2023-09-23T21:14:25.608213+0000","mapping_epoch":452,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":453,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:05:11.769428+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4734,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,2],"acting":[4,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":457,"reported_epoch":528,"state":"active+clean","last_fresh":"2023-09-23T21:14:25.610956+0000","last_change":"2023-09-23T21:12:10.839271+0000","last_active":"2023-09-23T21:14:25.610956+0000","last_peered":"2023-09-23T21:14:25.610956+0000","last_clean":"2023-09-23T21:14:25.610956+0000","last_became_active":"2023-09-23T21:12:10.839042+0000","last_became_peered":"2023-09-23T21:12:10.839042+0000","last_unstale":"2023-09-23T21:14:25.610956+0000","last_undegraded":"2023-09-23T21:14:25.610956+0000","last_fullsized":"2023-09-23T21:14:25.610956+0000","mapping_epoch":470,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:10:26.414737+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,2],"acting":[4,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.11","version":"79'161","reported_seq":452,"reported_epoch":528,"state":"active+clean","last_fresh":"2023-09-23T21:14:25.600511+0000","last_change":"2023-09-23T21:13:33.767406+0000","last_active":"2023-09-23T21:14:25.600511+0000","last_peered":"2023-09-23T21:14:25.600511+0000","last_clean":"2023-09-23T21:14:25.600511+0000","last_became_active":"2023-09-23T21:13:33.767067+0000","last_became_peered":"2023-09-23T21:13:33.767067+0000","last_unstale":"2023-09-23T21:14:25.600511+0000","last_undegraded":"2023-09-23T21:14:25.600511+0000","last_fullsized":"2023-09-23T21:14:25.600511+0000","mapping_epoch":504,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":505,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:28.756634+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:28.756634+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":23,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":137,"reported_epoch":528,"state":"active+clean","last_fresh":"2023-09-23T21:14:25.615348+0000","last_change":"2023-09-23T21:11:27.777154+0000","last_active":"2023-09-23T21:14:25.615348+0000","last_peered":"2023-09-23T21:14:25.615348+0000","last_clean":"2023-09-23T21:14:25.615348+0000","last_became_active":"2023-09-23T21:11:27.754438+0000","last_became_peered":"2023-09-23T21:11:27.754438+0000","last_unstale":"2023-09-23T21:14:25.615348+0000","last_undegraded":"2023-09-23T21:14:25.615348+0000","last_fullsized":"2023-09-23T21:14:25.615348+0000","mapping_epoch":452,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":453,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:33:02.845882+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":70,"reported_epoch":528,"state":"active+clean","last_fresh":"2023-09-23T21:14:31.288173+0000","last_change":"2023-09-23T21:12:10.764264+0000","last_active":"2023-09-23T21:14:31.288173+0000","last_peered":"2023-09-23T21:14:31.288173+0000","last_clean":"2023-09-23T21:14:31.288173+0000","last_became_active":"2023-09-23T21:12:10.763486+0000","last_became_peered":"2023-09-23T21:12:10.763486+0000","last_unstale":"2023-09-23T21:14:31.288173+0000","last_undegraded":"2023-09-23T21:14:31.288173+0000","last_fullsized":"2023-09-23T21:14:31.288173+0000","mapping_epoch":470,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:32:27.577036+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,7],"acting":[2,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.10","version":"79'161","reported_seq":663,"reported_epoch":528,"state":"active+clean","last_fresh":"2023-09-23T21:14:25.615947+0000","last_change":"2023-09-23T21:11:11.716703+0000","last_active":"2023-09-23T21:14:25.615947+0000","last_peered":"2023-09-23T21:14:25.615947+0000","last_clean":"2023-09-23T21:14:25.615947+0000","last_became_active":"2023-09-23T21:11:11.716385+0000","last_became_peered":"2023-09-23T21:11:11.716385+0000","last_unstale":"2023-09-23T21:14:25.615947+0000","last_undegraded":"2023-09-23T21:14:25.615947+0000","last_fullsized":"2023-09-23T21:14:25.615947+0000","mapping_epoch":446,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":447,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:06:50.336992+0000","last_deep_scrub":"79'161","last_deep_scrub_stamp":"2023-09-23T21:06:50.336992+0000","last_clean_scrub_stamp":"2023-09-23T21:06:50.336992+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:44:29.538890+0000","scrub_duration":0.0026150930000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,7],"acting":[4,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":469,"reported_epoch":528,"state":"active+clean","last_fresh":"2023-09-23T21:14:25.603870+0000","last_change":"2023-09-23T21:13:12.270990+0000","last_active":"2023-09-23T21:14:25.603870+0000","last_peered":"2023-09-23T21:14:25.603870+0000","last_clean":"2023-09-23T21:14:25.603870+0000","last_became_active":"2023-09-23T21:13:12.237201+0000","last_became_peered":"2023-09-23T21:13:12.237201+0000","last_unstale":"2023-09-23T21:14:25.603870+0000","last_undegraded":"2023-09-23T21:14:25.603870+0000","last_fullsized":"2023-09-23T21:14:25.603870+0000","mapping_epoch":493,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:59:54.523877+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":281,"reported_epoch":529,"state":"active+clean","last_fresh":"2023-09-23T21:14:32.780092+0000","last_change":"2023-09-23T21:10:26.758671+0000","last_active":"2023-09-23T21:14:32.780092+0000","last_peered":"2023-09-23T21:14:32.780092+0000","last_clean":"2023-09-23T21:14:32.780092+0000","last_became_active":"2023-09-23T21:10:26.758244+0000","last_became_peered":"2023-09-23T21:10:26.758244+0000","last_unstale":"2023-09-23T21:14:32.780092+0000","last_undegraded":"2023-09-23T21:14:32.780092+0000","last_fullsized":"2023-09-23T21:14:32.780092+0000","mapping_epoch":423,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":424,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:14:02.735609+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.18","version":"529'5534","reported_seq":6620,"reported_epoch":529,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T21:14:41.277768+0000","last_change":"2023-09-23T21:13:13.412097+0000","last_active":"2023-09-23T21:14:41.277768+0000","last_peered":"2023-09-23T21:14:41.277768+0000","last_clean":"2023-09-23T21:11:45.720015+0000","last_became_active":"2023-09-23T21:13:13.395632+0000","last_became_peered":"2023-09-23T21:13:13.395632+0000","last_unstale":"2023-09-23T21:14:41.277768+0000","last_undegraded":"2023-09-23T21:13:13.220971+0000","last_fullsized":"2023-09-23T21:13:13.183825+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":438,"last_epoch_clean":439,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_clean_scrub_stamp":"2023-09-23T21:10:15.663179+0000","objects_scrubbed":0,"log_size":5534,"log_dups_size":0,"ondisk_log_size":5534,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":56000512,"num_objects":855,"num_object_clones":0,"num_object_copies":2565,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":661,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":855,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4185,"num_write_kb":161248,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":231,"num_bytes_recovered":15138816,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,7],"acting":[2,7],"avail_no_missing":["2","7"],"object_location_counts":[{"shards":"2,7","objects":821}],"blocked_by":[],"up_primary":0,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.13","version":"79'161","reported_seq":606,"reported_epoch":529,"state":"active+clean","last_fresh":"2023-09-23T21:14:32.779992+0000","last_change":"2023-09-23T21:13:14.933096+0000","last_active":"2023-09-23T21:14:32.779992+0000","last_peered":"2023-09-23T21:14:32.779992+0000","last_clean":"2023-09-23T21:14:32.779992+0000","last_became_active":"2023-09-23T21:13:14.932703+0000","last_became_peered":"2023-09-23T21:13:14.932703+0000","last_unstale":"2023-09-23T21:14:32.779992+0000","last_undegraded":"2023-09-23T21:14:32.779992+0000","last_fullsized":"2023-09-23T21:14:32.779992+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:36.420362+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:36.420362+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:29:42.163911+0000","scrub_duration":0.028838444000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":520,"reported_epoch":528,"state":"active+clean","last_fresh":"2023-09-23T21:14:25.606413+0000","last_change":"2023-09-23T21:13:12.216467+0000","last_active":"2023-09-23T21:14:25.606413+0000","last_peered":"2023-09-23T21:14:25.606413+0000","last_clean":"2023-09-23T21:14:25.606413+0000","last_became_active":"2023-09-23T21:13:12.215259+0000","last_became_peered":"2023-09-23T21:13:12.215259+0000","last_unstale":"2023-09-23T21:14:25.606413+0000","last_undegraded":"2023-09-23T21:14:25.606413+0000","last_fullsized":"2023-09-23T21:14:25.606413+0000","mapping_epoch":493,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:35:33.987839+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":38,"reported_epoch":528,"state":"active+clean","last_fresh":"2023-09-23T21:14:25.605199+0000","last_change":"2023-09-23T21:13:30.804108+0000","last_active":"2023-09-23T21:14:25.605199+0000","last_peered":"2023-09-23T21:14:25.605199+0000","last_clean":"2023-09-23T21:14:25.605199+0000","last_became_active":"2023-09-23T21:13:30.796505+0000","last_became_peered":"2023-09-23T21:13:30.796505+0000","last_unstale":"2023-09-23T21:14:25.605199+0000","last_undegraded":"2023-09-23T21:14:25.605199+0000","last_fullsized":"2023-09-23T21:14:25.605199+0000","mapping_epoch":501,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:19:33.508496+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.19","version":"529'5693","reported_seq":6793,"reported_epoch":529,"state":"active+clean","last_fresh":"2023-09-23T21:14:41.281642+0000","last_change":"2023-09-23T21:10:58.790716+0000","last_active":"2023-09-23T21:14:41.281642+0000","last_peered":"2023-09-23T21:14:41.281642+0000","last_clean":"2023-09-23T21:14:41.281642+0000","last_became_active":"2023-09-23T21:10:58.787975+0000","last_became_peered":"2023-09-23T21:10:58.787975+0000","last_unstale":"2023-09-23T21:14:41.281642+0000","last_undegraded":"2023-09-23T21:14:41.281642+0000","last_fullsized":"2023-09-23T21:14:41.281642+0000","mapping_epoch":438,"log_start":"0'0","ondisk_log_start":"0'0","created":438,"last_epoch_clean":439,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_clean_scrub_stamp":"2023-09-23T21:10:15.663179+0000","objects_scrubbed":0,"log_size":5693,"log_dups_size":0,"ondisk_log_size":5693,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":51478528,"num_objects":786,"num_object_clones":0,"num_object_copies":2358,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":786,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4306,"num_write_kb":162912,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":237,"num_bytes_recovered":15499264,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.12","version":"79'161","reported_seq":762,"reported_epoch":528,"state":"active+clean","last_fresh":"2023-09-23T21:14:25.617462+0000","last_change":"2023-09-23T21:13:15.206574+0000","last_active":"2023-09-23T21:14:25.617462+0000","last_peered":"2023-09-23T21:14:25.617462+0000","last_clean":"2023-09-23T21:14:25.617462+0000","last_became_active":"2023-09-23T21:13:15.206186+0000","last_became_peered":"2023-09-23T21:13:15.206186+0000","last_unstale":"2023-09-23T21:14:25.617462+0000","last_undegraded":"2023-09-23T21:14:25.617462+0000","last_fullsized":"2023-09-23T21:14:25.617462+0000","mapping_epoch":496,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":497,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:05:56.785173+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:05:56.785173+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:34:29.221809+0000","scrub_duration":0.0042114370000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,0],"acting":[5,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":48,"reported_epoch":528,"state":"active+clean","last_fresh":"2023-09-23T21:14:25.603104+0000","last_change":"2023-09-23T21:13:38.048791+0000","last_active":"2023-09-23T21:14:25.603104+0000","last_peered":"2023-09-23T21:14:25.603104+0000","last_clean":"2023-09-23T21:14:25.603104+0000","last_became_active":"2023-09-23T21:13:30.801386+0000","last_became_peered":"2023-09-23T21:13:30.801386+0000","last_unstale":"2023-09-23T21:14:25.603104+0000","last_undegraded":"2023-09-23T21:14:25.603104+0000","last_fullsized":"2023-09-23T21:14:25.603104+0000","mapping_epoch":501,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:52:49.170414+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":366,"reported_epoch":529,"state":"active+clean","last_fresh":"2023-09-23T21:14:32.784462+0000","last_change":"2023-09-23T21:12:10.765780+0000","last_active":"2023-09-23T21:14:32.784462+0000","last_peered":"2023-09-23T21:14:32.784462+0000","last_clean":"2023-09-23T21:14:32.784462+0000","last_became_active":"2023-09-23T21:12:10.765372+0000","last_became_peered":"2023-09-23T21:12:10.765372+0000","last_unstale":"2023-09-23T21:14:32.784462+0000","last_undegraded":"2023-09-23T21:14:32.784462+0000","last_fullsized":"2023-09-23T21:14:32.784462+0000","mapping_epoch":470,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:56:08.446099+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.6","version":"529'5667","reported_seq":8585,"reported_epoch":529,"state":"active+clean","last_fresh":"2023-09-23T21:14:40.159415+0000","last_change":"2023-09-23T21:13:39.390285+0000","last_active":"2023-09-23T21:14:40.159415+0000","last_peered":"2023-09-23T21:14:40.159415+0000","last_clean":"2023-09-23T21:14:40.159415+0000","last_became_active":"2023-09-23T21:13:39.378284+0000","last_became_peered":"2023-09-23T21:13:39.378284+0000","last_unstale":"2023-09-23T21:14:40.159415+0000","last_undegraded":"2023-09-23T21:14:40.159415+0000","last_fullsized":"2023-09-23T21:14:40.159415+0000","mapping_epoch":508,"log_start":"426'550","ondisk_log_start":"426'550","created":416,"last_epoch_clean":509,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_clean_scrub_stamp":"2023-09-23T21:10:15.663179+0000","objects_scrubbed":0,"log_size":5117,"log_dups_size":0,"ondisk_log_size":5117,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":52756480,"num_objects":805,"num_object_clones":0,"num_object_copies":2415,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":805,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4261,"num_write_kb":162112,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2399,"num_bytes_recovered":113967104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,4],"acting":[0,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.d","version":"79'161","reported_seq":682,"reported_epoch":528,"state":"active+clean","last_fresh":"2023-09-23T21:14:25.617202+0000","last_change":"2023-09-23T21:12:30.444461+0000","last_active":"2023-09-23T21:14:25.617202+0000","last_peered":"2023-09-23T21:14:25.617202+0000","last_clean":"2023-09-23T21:14:25.617202+0000","last_became_active":"2023-09-23T21:12:30.443979+0000","last_became_peered":"2023-09-23T21:12:30.443979+0000","last_unstale":"2023-09-23T21:14:25.617202+0000","last_undegraded":"2023-09-23T21:14:25.617202+0000","last_fullsized":"2023-09-23T21:14:25.617202+0000","mapping_epoch":470,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:09:36.858077+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:09:36.858077+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0024987500000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":175,"reported_epoch":528,"state":"active+clean","last_fresh":"2023-09-23T21:14:25.618149+0000","last_change":"2023-09-23T21:13:59.872834+0000","last_active":"2023-09-23T21:14:25.618149+0000","last_peered":"2023-09-23T21:14:25.618149+0000","last_clean":"2023-09-23T21:14:25.618149+0000","last_became_active":"2023-09-23T21:13:30.770556+0000","last_became_peered":"2023-09-23T21:13:30.770556+0000","last_unstale":"2023-09-23T21:14:25.618149+0000","last_undegraded":"2023-09-23T21:14:25.618149+0000","last_fullsized":"2023-09-23T21:14:25.618149+0000","mapping_epoch":501,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":521,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:57:55.460052+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":132,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":236,"reported_epoch":528,"state":"active+clean","last_fresh":"2023-09-23T21:14:25.616653+0000","last_change":"2023-09-23T21:10:26.750725+0000","last_active":"2023-09-23T21:14:25.616653+0000","last_peered":"2023-09-23T21:14:25.616653+0000","last_clean":"2023-09-23T21:14:25.616653+0000","last_became_active":"2023-09-23T21:10:26.750307+0000","last_became_peered":"2023-09-23T21:10:26.750307+0000","last_unstale":"2023-09-23T21:14:25.616653+0000","last_undegraded":"2023-09-23T21:14:25.616653+0000","last_fullsized":"2023-09-23T21:14:25.616653+0000","mapping_epoch":423,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":424,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:06:22.040548+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,4],"acting":[5,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.7","version":"529'5622","reported_seq":5819,"reported_epoch":529,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T21:14:44.194902+0000","last_change":"2023-09-23T21:13:38.417178+0000","last_active":"2023-09-23T21:14:44.194902+0000","last_peered":"2023-09-23T21:14:44.194902+0000","last_clean":"2023-09-23T21:11:26.699871+0000","last_became_active":"2023-09-23T21:13:38.408523+0000","last_became_peered":"2023-09-23T21:13:38.408523+0000","last_unstale":"2023-09-23T21:14:44.194902+0000","last_undegraded":"2023-09-23T21:13:38.369833+0000","last_fullsized":"2023-09-23T21:13:38.358198+0000","mapping_epoch":507,"log_start":"434'1247","ondisk_log_start":"434'1247","created":416,"last_epoch_clean":439,"parent":"0.0","parent_split_bits":0,"last_scrub":"441'2973","last_scrub_stamp":"2023-09-23T21:11:06.079326+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_clean_scrub_stamp":"2023-09-23T21:11:06.079326+0000","objects_scrubbed":1647,"log_size":4375,"log_dups_size":0,"ondisk_log_size":4375,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:27:59.805526+0000","scrub_duration":0.44319714500000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47251456,"num_objects":721,"num_object_clones":0,"num_object_copies":2163,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1165,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":721,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2654,"num_write_kb":55296,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,2],"acting":[3,5],"avail_no_missing":["3","5"],"object_location_counts":[{"shards":"3,4,5","objects":298},{"shards":"3,5","objects":423}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.c","version":"79'161","reported_seq":891,"reported_epoch":528,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-23T21:14:25.607333+0000","last_change":"2023-09-23T21:13:13.402530+0000","last_active":"2023-09-23T21:14:25.607333+0000","last_peered":"2023-09-23T21:14:25.607333+0000","last_clean":"2023-09-23T21:11:23.200766+0000","last_became_active":"2023-09-23T21:13:13.401526+0000","last_became_peered":"2023-09-23T21:13:13.401526+0000","last_unstale":"2023-09-23T21:14:25.607333+0000","last_undegraded":"2023-09-23T21:13:13.204992+0000","last_fullsized":"2023-09-23T21:14:25.607333+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":420,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:22.040907+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:22.040907+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.013021176000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":163840,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":163840,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":["3","0","5"],"object_location_counts":[{"shards":"0,3,4","objects":-1},{"shards":"0,3,5","objects":1}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":161,"reported_epoch":528,"state":"active+clean","last_fresh":"2023-09-23T21:14:25.618027+0000","last_change":"2023-09-23T21:13:59.884113+0000","last_active":"2023-09-23T21:14:25.618027+0000","last_peered":"2023-09-23T21:14:25.618027+0000","last_clean":"2023-09-23T21:14:25.618027+0000","last_became_active":"2023-09-23T21:13:30.806778+0000","last_became_peered":"2023-09-23T21:13:30.806778+0000","last_unstale":"2023-09-23T21:14:25.618027+0000","last_undegraded":"2023-09-23T21:14:25.618027+0000","last_fullsized":"2023-09-23T21:14:25.618027+0000","mapping_epoch":501,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":521,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:36:15.284465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":491,"reported_epoch":528,"state":"active+clean","last_fresh":"2023-09-23T21:14:25.606479+0000","last_change":"2023-09-23T21:13:15.152792+0000","last_active":"2023-09-23T21:14:25.606479+0000","last_peered":"2023-09-23T21:14:25.606479+0000","last_clean":"2023-09-23T21:14:25.606479+0000","last_became_active":"2023-09-23T21:13:14.515084+0000","last_became_peered":"2023-09-23T21:13:14.515084+0000","last_unstale":"2023-09-23T21:14:25.606479+0000","last_undegraded":"2023-09-23T21:14:25.606479+0000","last_fullsized":"2023-09-23T21:14:25.606479+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:08:16.702072+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.4","version":"529'5553","reported_seq":12499,"reported_epoch":529,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T21:14:44.052207+0000","last_change":"2023-09-23T21:13:30.847170+0000","last_active":"2023-09-23T21:14:44.052207+0000","last_peered":"2023-09-23T21:14:44.052207+0000","last_clean":"2023-09-23T21:11:06.743842+0000","last_became_active":"2023-09-23T21:13:30.834806+0000","last_became_peered":"2023-09-23T21:13:30.834806+0000","last_unstale":"2023-09-23T21:14:44.052207+0000","last_undegraded":"2023-09-23T21:13:30.760230+0000","last_fullsized":"2023-09-23T21:13:30.749974+0000","mapping_epoch":501,"log_start":"434'1116","ondisk_log_start":"434'1116","created":416,"last_epoch_clean":439,"parent":"0.0","parent_split_bits":0,"last_scrub":"440'2861","last_scrub_stamp":"2023-09-23T21:11:03.842926+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_clean_scrub_stamp":"2023-09-23T21:11:03.842926+0000","objects_scrubbed":1526,"log_size":4437,"log_dups_size":0,"ondisk_log_size":4437,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:32:29.379756+0000","scrub_duration":0.39378383900000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":50331648,"num_objects":768,"num_object_clones":0,"num_object_copies":2304,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1130,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":768,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2698,"num_write_kb":62080,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2723,"num_bytes_recovered":163577856,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[3,4],"avail_no_missing":["3","4"],"object_location_counts":[{"shards":"1,2,3,4","objects":360},{"shards":"3,4","objects":408}],"blocked_by":[],"up_primary":0,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.f","version":"79'161","reported_seq":816,"reported_epoch":528,"state":"active+clean","last_fresh":"2023-09-23T21:14:25.619531+0000","last_change":"2023-09-23T21:11:30.750338+0000","last_active":"2023-09-23T21:14:25.619531+0000","last_peered":"2023-09-23T21:14:25.619531+0000","last_clean":"2023-09-23T21:14:25.619531+0000","last_became_active":"2023-09-23T21:11:30.749953+0000","last_became_peered":"2023-09-23T21:11:30.749953+0000","last_unstale":"2023-09-23T21:14:25.619531+0000","last_undegraded":"2023-09-23T21:14:25.619531+0000","last_fullsized":"2023-09-23T21:14:25.619531+0000","mapping_epoch":455,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":456,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:27.880776+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:27.880776+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:07:10.405723+0000","scrub_duration":0.0014004899999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,3],"acting":[5,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":252,"reported_epoch":528,"state":"active+clean","last_fresh":"2023-09-23T21:14:25.618688+0000","last_change":"2023-09-23T21:10:26.765068+0000","last_active":"2023-09-23T21:14:25.618688+0000","last_peered":"2023-09-23T21:14:25.618688+0000","last_clean":"2023-09-23T21:14:25.618688+0000","last_became_active":"2023-09-23T21:10:26.758364+0000","last_became_peered":"2023-09-23T21:10:26.758364+0000","last_unstale":"2023-09-23T21:14:25.618688+0000","last_undegraded":"2023-09-23T21:14:25.618688+0000","last_fullsized":"2023-09-23T21:14:25.618688+0000","mapping_epoch":423,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":424,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:13:21.057529+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":280,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":468,"reported_epoch":528,"state":"active+clean","last_fresh":"2023-09-23T21:14:25.617043+0000","last_change":"2023-09-23T21:13:30.770746+0000","last_active":"2023-09-23T21:14:25.617043+0000","last_peered":"2023-09-23T21:14:25.617043+0000","last_clean":"2023-09-23T21:14:25.617043+0000","last_became_active":"2023-09-23T21:13:30.770437+0000","last_became_peered":"2023-09-23T21:13:30.770437+0000","last_unstale":"2023-09-23T21:14:25.617043+0000","last_undegraded":"2023-09-23T21:14:25.617043+0000","last_fullsized":"2023-09-23T21:14:25.617043+0000","mapping_epoch":501,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:33:48.325730+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,0],"acting":[5,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.5","version":"529'5499","reported_seq":5687,"reported_epoch":529,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T21:14:41.173426+0000","last_change":"2023-09-23T21:13:13.436794+0000","last_active":"2023-09-23T21:14:41.173426+0000","last_peered":"2023-09-23T21:14:41.173426+0000","last_clean":"2023-09-23T21:11:45.768408+0000","last_became_active":"2023-09-23T21:13:13.424939+0000","last_became_peered":"2023-09-23T21:13:13.424939+0000","last_unstale":"2023-09-23T21:14:41.173426+0000","last_undegraded":"2023-09-23T21:13:13.225067+0000","last_fullsized":"2023-09-23T21:13:13.198194+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":416,"last_epoch_clean":439,"parent":"0.0","parent_split_bits":0,"last_scrub":"440'2840","last_scrub_stamp":"2023-09-23T21:11:03.208833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_clean_scrub_stamp":"2023-09-23T21:11:03.208833+0000","objects_scrubbed":1498,"log_size":5499,"log_dups_size":0,"ondisk_log_size":5499,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:56:40.574477+0000","scrub_duration":0.45608471699999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":56623104,"num_objects":864,"num_object_clones":0,"num_object_copies":2592,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":644,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":864,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2668,"num_write_kb":65088,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,3],"acting":[2,3],"avail_no_missing":["2","3"],"object_location_counts":[{"shards":"2,3","objects":818}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.e","version":"79'161","reported_seq":697,"reported_epoch":528,"state":"active+clean","last_fresh":"2023-09-23T21:14:25.607416+0000","last_change":"2023-09-23T21:12:30.440952+0000","last_active":"2023-09-23T21:14:25.607416+0000","last_peered":"2023-09-23T21:14:25.607416+0000","last_clean":"2023-09-23T21:14:25.607416+0000","last_became_active":"2023-09-23T21:12:30.440463+0000","last_became_peered":"2023-09-23T21:12:30.440463+0000","last_unstale":"2023-09-23T21:14:25.607416+0000","last_undegraded":"2023-09-23T21:14:25.607416+0000","last_fullsized":"2023-09-23T21:14:25.607416+0000","mapping_epoch":455,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:10:39.567232+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:10:39.567232+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.005229694,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":326,"reported_epoch":528,"state":"active+clean","last_fresh":"2023-09-23T21:14:25.607720+0000","last_change":"2023-09-23T21:13:13.402418+0000","last_active":"2023-09-23T21:14:25.607720+0000","last_peered":"2023-09-23T21:14:25.607720+0000","last_clean":"2023-09-23T21:14:25.607720+0000","last_became_active":"2023-09-23T21:13:13.400984+0000","last_became_peered":"2023-09-23T21:13:13.400984+0000","last_unstale":"2023-09-23T21:14:25.607720+0000","last_undegraded":"2023-09-23T21:14:25.607720+0000","last_fullsized":"2023-09-23T21:14:25.607720+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":495,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:28:01.411191+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,0],"acting":[3,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":501,"reported_epoch":528,"state":"active+clean","last_fresh":"2023-09-23T21:14:25.604679+0000","last_change":"2023-09-23T21:13:14.339670+0000","last_active":"2023-09-23T21:14:25.604679+0000","last_peered":"2023-09-23T21:14:25.604679+0000","last_clean":"2023-09-23T21:14:25.604679+0000","last_became_active":"2023-09-23T21:13:14.339341+0000","last_became_peered":"2023-09-23T21:13:14.339341+0000","last_unstale":"2023-09-23T21:14:25.604679+0000","last_undegraded":"2023-09-23T21:14:25.604679+0000","last_fullsized":"2023-09-23T21:14:25.604679+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:23:19.411368+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.2","version":"529'5610","reported_seq":8010,"reported_epoch":529,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T21:14:43.310767+0000","last_change":"2023-09-23T21:13:13.407366+0000","last_active":"2023-09-23T21:14:43.310767+0000","last_peered":"2023-09-23T21:14:43.310767+0000","last_clean":"2023-09-23T21:11:45.752822+0000","last_became_active":"2023-09-23T21:13:13.398616+0000","last_became_peered":"2023-09-23T21:13:13.398616+0000","last_unstale":"2023-09-23T21:14:43.310767+0000","last_undegraded":"2023-09-23T21:13:13.289815+0000","last_fullsized":"2023-09-23T21:13:13.190270+0000","mapping_epoch":494,"log_start":"427'683","ondisk_log_start":"427'683","created":416,"last_epoch_clean":439,"parent":"0.0","parent_split_bits":0,"last_scrub":"440'3007","last_scrub_stamp":"2023-09-23T21:11:05.553661+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_clean_scrub_stamp":"2023-09-23T21:11:05.553661+0000","objects_scrubbed":1622,"log_size":4927,"log_dups_size":0,"ondisk_log_size":4927,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:51:21.839818+0000","scrub_duration":0.39156952499999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":50724864,"num_objects":774,"num_object_clones":0,"num_object_copies":2322,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2066,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":774,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2608,"num_write_kb":56320,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1099,"num_bytes_recovered":72024064,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,3],"avail_no_missing":["4","1"],"object_location_counts":[{"shards":"0,1,4","objects":740},{"shards":"1,4","objects":34}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.9","version":"79'161","reported_seq":682,"reported_epoch":528,"state":"active+clean","last_fresh":"2023-09-23T21:14:25.597443+0000","last_change":"2023-09-23T21:12:30.436118+0000","last_active":"2023-09-23T21:14:25.597443+0000","last_peered":"2023-09-23T21:14:25.597443+0000","last_clean":"2023-09-23T21:14:25.597443+0000","last_became_active":"2023-09-23T21:12:30.435779+0000","last_became_peered":"2023-09-23T21:12:30.435779+0000","last_unstale":"2023-09-23T21:14:25.597443+0000","last_undegraded":"2023-09-23T21:14:25.597443+0000","last_fullsized":"2023-09-23T21:14:25.597443+0000","mapping_epoch":433,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:10:37.696951+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:10:37.696951+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.001386504,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,3],"acting":[2,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":87,"reported_epoch":528,"state":"active+clean","last_fresh":"2023-09-23T21:14:25.604823+0000","last_change":"2023-09-23T21:13:30.793273+0000","last_active":"2023-09-23T21:14:25.604823+0000","last_peered":"2023-09-23T21:14:25.604823+0000","last_clean":"2023-09-23T21:14:25.604823+0000","last_became_active":"2023-09-23T21:13:30.791269+0000","last_became_peered":"2023-09-23T21:13:30.791269+0000","last_unstale":"2023-09-23T21:14:25.604823+0000","last_undegraded":"2023-09-23T21:14:25.604823+0000","last_fullsized":"2023-09-23T21:14:25.604823+0000","mapping_epoch":501,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:38:07.142290+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":102,"reported_epoch":528,"state":"active+clean","last_fresh":"2023-09-23T21:14:25.607710+0000","last_change":"2023-09-23T21:12:14.313466+0000","last_active":"2023-09-23T21:14:25.607710+0000","last_peered":"2023-09-23T21:14:25.607710+0000","last_clean":"2023-09-23T21:14:25.607710+0000","last_became_active":"2023-09-23T21:12:14.209837+0000","last_became_peered":"2023-09-23T21:12:14.209837+0000","last_unstale":"2023-09-23T21:14:25.607710+0000","last_undegraded":"2023-09-23T21:14:25.607710+0000","last_fullsized":"2023-09-23T21:14:25.607710+0000","mapping_epoch":470,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":474,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:09:40.375241+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,7],"acting":[4,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.3","version":"529'5619","reported_seq":7751,"reported_epoch":529,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T21:14:41.195530+0000","last_change":"2023-09-23T21:13:45.480705+0000","last_active":"2023-09-23T21:14:41.195530+0000","last_peered":"2023-09-23T21:14:41.195530+0000","last_clean":"2023-09-23T21:11:26.700990+0000","last_became_active":"2023-09-23T21:13:45.471702+0000","last_became_peered":"2023-09-23T21:13:45.471702+0000","last_unstale":"2023-09-23T21:14:41.195530+0000","last_undegraded":"2023-09-23T21:13:45.412179+0000","last_fullsized":"2023-09-23T21:13:45.401701+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":416,"last_epoch_clean":439,"parent":"0.0","parent_split_bits":0,"last_scrub":"440'2873","last_scrub_stamp":"2023-09-23T21:11:02.114783+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_clean_scrub_stamp":"2023-09-23T21:11:02.114783+0000","objects_scrubbed":1532,"log_size":5619,"log_dups_size":0,"ondisk_log_size":5619,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:42:48.549370+0000","scrub_duration":0.32435022600000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53477376,"num_objects":816,"num_object_clones":0,"num_object_copies":2448,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1184,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":816,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2752,"num_write_kb":65152,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":865,"num_bytes_recovered":56688640,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5],"avail_no_missing":["2","5"],"object_location_counts":[{"shards":"0,2,4,5","objects":270},{"shards":"2,4,5","objects":48},{"shards":"2,5","objects":497}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.8","version":"79'161","reported_seq":649,"reported_epoch":528,"state":"active+clean","last_fresh":"2023-09-23T21:14:25.597122+0000","last_change":"2023-09-23T21:12:30.433561+0000","last_active":"2023-09-23T21:14:25.597122+0000","last_peered":"2023-09-23T21:14:25.597122+0000","last_clean":"2023-09-23T21:14:25.597122+0000","last_became_active":"2023-09-23T21:12:30.433217+0000","last_became_peered":"2023-09-23T21:12:30.433217+0000","last_unstale":"2023-09-23T21:14:25.597122+0000","last_undegraded":"2023-09-23T21:14:25.597122+0000","last_fullsized":"2023-09-23T21:14:25.597122+0000","mapping_epoch":359,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:06:52.098882+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:06:52.098882+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0092173299999999993,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":131344,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":105,"reported_epoch":528,"state":"active+clean","last_fresh":"2023-09-23T21:14:25.607017+0000","last_change":"2023-09-23T21:12:10.823380+0000","last_active":"2023-09-23T21:14:25.607017+0000","last_peered":"2023-09-23T21:14:25.607017+0000","last_clean":"2023-09-23T21:14:25.607017+0000","last_became_active":"2023-09-23T21:12:10.823044+0000","last_became_peered":"2023-09-23T21:12:10.823044+0000","last_unstale":"2023-09-23T21:14:25.607017+0000","last_undegraded":"2023-09-23T21:14:25.607017+0000","last_fullsized":"2023-09-23T21:14:25.607017+0000","mapping_epoch":470,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:53:29.846536+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,5],"acting":[4,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":106,"reported_epoch":528,"state":"active+clean","last_fresh":"2023-09-23T21:14:25.607209+0000","last_change":"2023-09-23T21:13:30.772451+0000","last_active":"2023-09-23T21:14:25.607209+0000","last_peered":"2023-09-23T21:14:25.607209+0000","last_clean":"2023-09-23T21:14:25.607209+0000","last_became_active":"2023-09-23T21:13:30.772040+0000","last_became_peered":"2023-09-23T21:13:30.772040+0000","last_unstale":"2023-09-23T21:14:25.607209+0000","last_undegraded":"2023-09-23T21:14:25.607209+0000","last_fullsized":"2023-09-23T21:14:25.607209+0000","mapping_epoch":501,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:24:49.246727+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,0],"acting":[4,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.0","version":"529'5600","reported_seq":10712,"reported_epoch":529,"state":"active+recovering+undersized+remapped","last_fresh":"2023-09-23T21:14:43.311475+0000","last_change":"2023-09-23T21:14:02.130787+0000","last_active":"2023-09-23T21:14:43.311475+0000","last_peered":"2023-09-23T21:14:43.311475+0000","last_clean":"2023-09-23T21:11:26.716395+0000","last_became_active":"2023-09-23T21:13:34.557827+0000","last_became_peered":"2023-09-23T21:13:34.557827+0000","last_unstale":"2023-09-23T21:14:43.311475+0000","last_undegraded":"2023-09-23T21:14:43.311475+0000","last_fullsized":"2023-09-23T21:13:31.752840+0000","mapping_epoch":502,"log_start":"427'731","ondisk_log_start":"427'731","created":416,"last_epoch_clean":439,"parent":"0.0","parent_split_bits":0,"last_scrub":"440'2961","last_scrub_stamp":"2023-09-23T21:11:04.564028+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_clean_scrub_stamp":"2023-09-23T21:11:04.564028+0000","objects_scrubbed":1608,"log_size":4869,"log_dups_size":0,"ondisk_log_size":4869,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:43:09.815808+0000","scrub_duration":0.41858158299999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":49414144,"num_objects":754,"num_object_clones":0,"num_object_copies":2262,"num_objects_missing_on_primary":546,"num_objects_missing":546,"num_objects_degraded":0,"num_objects_misplaced":721,"num_objects_unfound":0,"num_objects_dirty":754,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2642,"num_write_kb":57216,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1626,"num_bytes_recovered":106496000,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,0],"avail_no_missing":["1","7"],"object_location_counts":[{"shards":"0,1,7","objects":395},{"shards":"0,4","objects":33},{"shards":"1,4,7","objects":326}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.b","version":"79'161","reported_seq":532,"reported_epoch":528,"state":"active+clean","last_fresh":"2023-09-23T21:14:25.604408+0000","last_change":"2023-09-23T21:13:13.373313+0000","last_active":"2023-09-23T21:14:25.604408+0000","last_peered":"2023-09-23T21:14:25.604408+0000","last_clean":"2023-09-23T21:14:25.604408+0000","last_became_active":"2023-09-23T21:13:13.372328+0000","last_became_peered":"2023-09-23T21:13:13.372328+0000","last_unstale":"2023-09-23T21:14:25.604408+0000","last_undegraded":"2023-09-23T21:14:25.604408+0000","last_fullsized":"2023-09-23T21:14:25.604408+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":495,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:05:55.772264+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:05:55.772264+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.001862507,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":86,"reported_epoch":528,"state":"active+clean","last_fresh":"2023-09-23T21:14:25.606411+0000","last_change":"2023-09-23T21:11:27.754717+0000","last_active":"2023-09-23T21:14:25.606411+0000","last_peered":"2023-09-23T21:14:25.606411+0000","last_clean":"2023-09-23T21:14:25.606411+0000","last_became_active":"2023-09-23T21:11:27.754376+0000","last_became_peered":"2023-09-23T21:11:27.754376+0000","last_unstale":"2023-09-23T21:14:25.606411+0000","last_undegraded":"2023-09-23T21:14:25.606411+0000","last_fullsized":"2023-09-23T21:14:25.606411+0000","mapping_epoch":452,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":453,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:26:41.734541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":460,"reported_epoch":528,"state":"active+clean","last_fresh":"2023-09-23T21:14:25.608117+0000","last_change":"2023-09-23T21:12:10.811698+0000","last_active":"2023-09-23T21:14:25.608117+0000","last_peered":"2023-09-23T21:14:25.608117+0000","last_clean":"2023-09-23T21:14:25.608117+0000","last_became_active":"2023-09-23T21:12:10.789656+0000","last_became_peered":"2023-09-23T21:12:10.789656+0000","last_unstale":"2023-09-23T21:14:25.608117+0000","last_undegraded":"2023-09-23T21:14:25.608117+0000","last_fullsized":"2023-09-23T21:14:25.608117+0000","mapping_epoch":470,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:39:15.112091+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,5],"acting":[4,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.1","version":"529'5535","reported_seq":11950,"reported_epoch":529,"state":"active+clean","last_fresh":"2023-09-23T21:14:43.269779+0000","last_change":"2023-09-23T21:13:53.483736+0000","last_active":"2023-09-23T21:14:43.269779+0000","last_peered":"2023-09-23T21:14:43.269779+0000","last_clean":"2023-09-23T21:14:43.269779+0000","last_became_active":"2023-09-23T21:13:53.472685+0000","last_became_peered":"2023-09-23T21:13:53.472685+0000","last_unstale":"2023-09-23T21:14:43.269779+0000","last_undegraded":"2023-09-23T21:14:43.269779+0000","last_fullsized":"2023-09-23T21:14:43.269779+0000","mapping_epoch":519,"log_start":"425'505","ondisk_log_start":"425'505","created":416,"last_epoch_clean":520,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_clean_scrub_stamp":"2023-09-23T21:10:15.663179+0000","objects_scrubbed":0,"log_size":5030,"log_dups_size":0,"ondisk_log_size":5030,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":49610752,"num_objects":757,"num_object_clones":0,"num_object_copies":2271,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":757,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4187,"num_write_kb":158208,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3005,"num_bytes_recovered":156499968,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.a","version":"79'161","reported_seq":675,"reported_epoch":528,"state":"active+clean","last_fresh":"2023-09-23T21:14:25.597939+0000","last_change":"2023-09-23T21:12:30.437790+0000","last_active":"2023-09-23T21:14:25.597939+0000","last_peered":"2023-09-23T21:14:25.597939+0000","last_clean":"2023-09-23T21:14:25.597939+0000","last_became_active":"2023-09-23T21:12:30.437427+0000","last_became_peered":"2023-09-23T21:12:30.437427+0000","last_unstale":"2023-09-23T21:14:25.597939+0000","last_undegraded":"2023-09-23T21:14:25.597939+0000","last_fullsized":"2023-09-23T21:14:25.597939+0000","mapping_epoch":446,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:06:51.075448+0000","last_deep_scrub":"79'161","last_deep_scrub_stamp":"2023-09-23T21:05:32.630292+0000","last_clean_scrub_stamp":"2023-09-23T21:06:51.075448+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0054793890000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,4],"acting":[2,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":375,"reported_epoch":528,"state":"active+clean","last_fresh":"2023-09-23T21:14:25.606051+0000","last_change":"2023-09-23T21:12:10.792918+0000","last_active":"2023-09-23T21:14:25.606051+0000","last_peered":"2023-09-23T21:14:25.606051+0000","last_clean":"2023-09-23T21:14:25.606051+0000","last_became_active":"2023-09-23T21:12:10.792679+0000","last_became_peered":"2023-09-23T21:12:10.792679+0000","last_unstale":"2023-09-23T21:14:25.606051+0000","last_undegraded":"2023-09-23T21:14:25.606051+0000","last_fullsized":"2023-09-23T21:14:25.606051+0000","mapping_epoch":470,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:30:48.008690+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,4],"acting":[3,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":507,"reported_epoch":528,"state":"active+clean","last_fresh":"2023-09-23T21:14:29.206737+0000","last_change":"2023-09-23T21:12:10.790812+0000","last_active":"2023-09-23T21:14:29.206737+0000","last_peered":"2023-09-23T21:14:29.206737+0000","last_clean":"2023-09-23T21:14:29.206737+0000","last_became_active":"2023-09-23T21:12:10.790492+0000","last_became_peered":"2023-09-23T21:12:10.790492+0000","last_unstale":"2023-09-23T21:14:29.206737+0000","last_undegraded":"2023-09-23T21:14:29.206737+0000","last_fullsized":"2023-09-23T21:14:29.206737+0000","mapping_epoch":470,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:02:19.863118+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.c","version":"529'8494","reported_seq":9650,"reported_epoch":529,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T21:14:42.083996+0000","last_change":"2023-09-23T21:13:30.822215+0000","last_active":"2023-09-23T21:14:42.083996+0000","last_peered":"2023-09-23T21:14:42.083996+0000","last_clean":"2023-09-23T21:11:45.750376+0000","last_became_active":"2023-09-23T21:13:13.404327+0000","last_became_peered":"2023-09-23T21:13:13.404327+0000","last_unstale":"2023-09-23T21:14:42.083996+0000","last_undegraded":"2023-09-23T21:13:13.200907+0000","last_fullsized":"2023-09-23T21:13:13.193701+0000","mapping_epoch":494,"log_start":"450'4051","ondisk_log_start":"450'4051","created":416,"last_epoch_clean":434,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_clean_scrub_stamp":"2023-09-23T21:10:15.663179+0000","objects_scrubbed":0,"log_size":4443,"log_dups_size":0,"ondisk_log_size":4443,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:33:42.971375+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":101187584,"num_objects":1544,"num_object_clones":0,"num_object_copies":4632,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1339,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1544,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8494,"num_write_kb":321216,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":498,"num_bytes_recovered":32636928,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4],"avail_no_missing":["7","4"],"object_location_counts":[{"shards":"4,7","objects":1490}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.7","version":"79'161","reported_seq":621,"reported_epoch":528,"state":"active+clean","last_fresh":"2023-09-23T21:14:25.608305+0000","last_change":"2023-09-23T21:13:44.416548+0000","last_active":"2023-09-23T21:14:25.608305+0000","last_peered":"2023-09-23T21:14:25.608305+0000","last_clean":"2023-09-23T21:14:25.608305+0000","last_became_active":"2023-09-23T21:13:44.415617+0000","last_became_peered":"2023-09-23T21:13:44.415617+0000","last_unstale":"2023-09-23T21:14:25.608305+0000","last_undegraded":"2023-09-23T21:14:25.608305+0000","last_fullsized":"2023-09-23T21:14:25.608305+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":3,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:25.916867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:25.916867+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.001665323,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":458,"reported_epoch":528,"state":"active+clean","last_fresh":"2023-09-23T21:14:25.604760+0000","last_change":"2023-09-23T21:13:13.211959+0000","last_active":"2023-09-23T21:14:25.604760+0000","last_peered":"2023-09-23T21:14:25.604760+0000","last_clean":"2023-09-23T21:14:25.604760+0000","last_became_active":"2023-09-23T21:13:13.211712+0000","last_became_peered":"2023-09-23T21:13:13.211712+0000","last_unstale":"2023-09-23T21:14:25.604760+0000","last_undegraded":"2023-09-23T21:14:25.604760+0000","last_fullsized":"2023-09-23T21:14:25.604760+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":495,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:37:21.655775+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,2],"acting":[0,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":206,"reported_epoch":528,"state":"active+clean","last_fresh":"2023-09-23T21:14:25.618600+0000","last_change":"2023-09-23T21:05:04.326889+0000","last_active":"2023-09-23T21:14:25.618600+0000","last_peered":"2023-09-23T21:14:25.618600+0000","last_clean":"2023-09-23T21:14:25.618600+0000","last_became_active":"2023-09-23T21:05:04.326032+0000","last_became_peered":"2023-09-23T21:05:04.326032+0000","last_unstale":"2023-09-23T21:14:25.618600+0000","last_undegraded":"2023-09-23T21:14:25.618600+0000","last_fullsized":"2023-09-23T21:14:25.618600+0000","mapping_epoch":332,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:00:17.507889+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":538,"reported_epoch":528,"state":"active+clean","last_fresh":"2023-09-23T21:14:25.617192+0000","last_change":"2023-09-23T21:13:13.285608+0000","last_active":"2023-09-23T21:14:25.617192+0000","last_peered":"2023-09-23T21:14:25.617192+0000","last_clean":"2023-09-23T21:14:25.617192+0000","last_became_active":"2023-09-23T21:13:13.285255+0000","last_became_peered":"2023-09-23T21:13:13.285255+0000","last_unstale":"2023-09-23T21:14:25.617192+0000","last_undegraded":"2023-09-23T21:14:25.617192+0000","last_fullsized":"2023-09-23T21:14:25.617192+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":495,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:42:30.087599+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.a","version":"529'8267","reported_seq":17003,"reported_epoch":529,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T21:14:41.197421+0000","last_change":"2023-09-23T21:13:31.927282+0000","last_active":"2023-09-23T21:14:41.197421+0000","last_peered":"2023-09-23T21:14:41.197421+0000","last_clean":"2023-09-23T21:13:30.353360+0000","last_became_active":"2023-09-23T21:13:31.896923+0000","last_became_peered":"2023-09-23T21:13:31.896923+0000","last_unstale":"2023-09-23T21:14:41.197421+0000","last_undegraded":"2023-09-23T21:13:31.776904+0000","last_fullsized":"2023-09-23T21:13:31.751334+0000","mapping_epoch":502,"log_start":"450'3912","ondisk_log_start":"450'3912","created":416,"last_epoch_clean":499,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_clean_scrub_stamp":"2023-09-23T21:10:15.663179+0000","objects_scrubbed":0,"log_size":4355,"log_dups_size":0,"ondisk_log_size":4355,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:23:14.477602+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":105185280,"num_objects":1605,"num_object_clones":0,"num_object_copies":4815,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2243,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1605,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8267,"num_write_kb":315904,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4293,"num_bytes_recovered":281346048,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[2,5],"avail_no_missing":["2","3","5"],"object_location_counts":[{"shards":"2,3,5","objects":643},{"shards":"2,5","objects":962}],"blocked_by":[],"up_primary":0,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.1","version":"79'161","reported_seq":599,"reported_epoch":528,"state":"active+clean","last_fresh":"2023-09-23T21:14:25.601002+0000","last_change":"2023-09-23T21:13:12.204135+0000","last_active":"2023-09-23T21:14:25.601002+0000","last_peered":"2023-09-23T21:14:25.601002+0000","last_clean":"2023-09-23T21:14:25.601002+0000","last_became_active":"2023-09-23T21:13:12.203380+0000","last_became_peered":"2023-09-23T21:13:12.203380+0000","last_unstale":"2023-09-23T21:14:25.601002+0000","last_undegraded":"2023-09-23T21:14:25.601002+0000","last_fullsized":"2023-09-23T21:14:25.601002+0000","mapping_epoch":493,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:06:52.575900+0000","last_deep_scrub":"79'161","last_deep_scrub_stamp":"2023-09-23T21:05:30.652189+0000","last_clean_scrub_stamp":"2023-09-23T21:06:52.575900+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":195,"reported_epoch":528,"state":"active+clean","last_fresh":"2023-09-23T21:14:25.603778+0000","last_change":"2023-09-23T21:13:13.238617+0000","last_active":"2023-09-23T21:14:25.603778+0000","last_peered":"2023-09-23T21:14:25.603778+0000","last_clean":"2023-09-23T21:14:25.603778+0000","last_became_active":"2023-09-23T21:13:13.232926+0000","last_became_peered":"2023-09-23T21:13:13.232926+0000","last_unstale":"2023-09-23T21:14:25.603778+0000","last_undegraded":"2023-09-23T21:14:25.603778+0000","last_fullsized":"2023-09-23T21:14:25.603778+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":495,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:17:21.823772+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":462,"reported_epoch":528,"state":"active+clean","last_fresh":"2023-09-23T21:14:25.611080+0000","last_change":"2023-09-23T21:13:14.144918+0000","last_active":"2023-09-23T21:14:25.611080+0000","last_peered":"2023-09-23T21:14:25.611080+0000","last_clean":"2023-09-23T21:14:25.611080+0000","last_became_active":"2023-09-23T21:13:14.132388+0000","last_became_peered":"2023-09-23T21:13:14.132388+0000","last_unstale":"2023-09-23T21:14:25.611080+0000","last_undegraded":"2023-09-23T21:14:25.611080+0000","last_fullsized":"2023-09-23T21:14:25.611080+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":495,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:56:02.619905+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,5],"acting":[4,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":349,"reported_epoch":528,"state":"active+clean","last_fresh":"2023-09-23T21:14:25.598016+0000","last_change":"2023-09-23T21:12:10.764503+0000","last_active":"2023-09-23T21:14:25.598016+0000","last_peered":"2023-09-23T21:14:25.598016+0000","last_clean":"2023-09-23T21:14:25.598016+0000","last_became_active":"2023-09-23T21:12:10.763933+0000","last_became_peered":"2023-09-23T21:12:10.763933+0000","last_unstale":"2023-09-23T21:14:25.598016+0000","last_undegraded":"2023-09-23T21:14:25.598016+0000","last_fullsized":"2023-09-23T21:14:25.598016+0000","mapping_epoch":470,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:28:33.202825+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,7],"acting":[2,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"10.9","version":"529'5555","reported_seq":6700,"reported_epoch":529,"state":"active+clean","last_fresh":"2023-09-23T21:14:41.190619+0000","last_change":"2023-09-23T21:11:08.062606+0000","last_active":"2023-09-23T21:14:41.190619+0000","last_peered":"2023-09-23T21:14:41.190619+0000","last_clean":"2023-09-23T21:14:41.190619+0000","last_became_active":"2023-09-23T21:10:58.780961+0000","last_became_peered":"2023-09-23T21:10:58.780961+0000","last_unstale":"2023-09-23T21:14:41.190619+0000","last_undegraded":"2023-09-23T21:14:41.190619+0000","last_fullsized":"2023-09-23T21:14:41.190619+0000","mapping_epoch":428,"log_start":"0'0","ondisk_log_start":"0'0","created":416,"last_epoch_clean":439,"parent":"0.0","parent_split_bits":0,"last_scrub":"443'3049","last_scrub_stamp":"2023-09-23T21:11:08.062479+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_clean_scrub_stamp":"2023-09-23T21:11:08.062479+0000","objects_scrubbed":1623,"log_size":5555,"log_dups_size":0,"ondisk_log_size":5555,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:56:22.338088+0000","scrub_duration":0.28207626200000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":52297728,"num_objects":798,"num_object_clones":0,"num_object_copies":2394,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":798,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2507,"num_write_kb":53824,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.2","version":"79'161","reported_seq":808,"reported_epoch":528,"state":"active+clean","last_fresh":"2023-09-23T21:14:25.600684+0000","last_change":"2023-09-23T21:13:12.209907+0000","last_active":"2023-09-23T21:14:25.600684+0000","last_peered":"2023-09-23T21:14:25.600684+0000","last_clean":"2023-09-23T21:14:25.600684+0000","last_became_active":"2023-09-23T21:13:12.205664+0000","last_became_peered":"2023-09-23T21:13:12.205664+0000","last_unstale":"2023-09-23T21:14:25.600684+0000","last_undegraded":"2023-09-23T21:14:25.600684+0000","last_fullsized":"2023-09-23T21:14:25.600684+0000","mapping_epoch":493,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:24.802451+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:24.802451+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:26:16.035788+0000","scrub_duration":0.0090861250000000005,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,4],"acting":[0,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":537,"reported_epoch":528,"state":"active+clean","last_fresh":"2023-09-23T21:14:25.598189+0000","last_change":"2023-09-23T21:13:12.206980+0000","last_active":"2023-09-23T21:14:25.598189+0000","last_peered":"2023-09-23T21:14:25.598189+0000","last_clean":"2023-09-23T21:14:25.598189+0000","last_became_active":"2023-09-23T21:13:12.206650+0000","last_became_peered":"2023-09-23T21:13:12.206650+0000","last_unstale":"2023-09-23T21:14:25.598189+0000","last_undegraded":"2023-09-23T21:14:25.598189+0000","last_fullsized":"2023-09-23T21:14:25.598189+0000","mapping_epoch":493,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:10:01.892936+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":521,"reported_epoch":528,"state":"active+clean","last_fresh":"2023-09-23T21:14:25.597264+0000","last_change":"2023-09-23T21:13:13.407073+0000","last_active":"2023-09-23T21:14:25.597264+0000","last_peered":"2023-09-23T21:14:25.597264+0000","last_clean":"2023-09-23T21:14:25.597264+0000","last_became_active":"2023-09-23T21:13:13.395047+0000","last_became_peered":"2023-09-23T21:13:13.395047+0000","last_unstale":"2023-09-23T21:14:25.597264+0000","last_undegraded":"2023-09-23T21:14:25.597264+0000","last_fullsized":"2023-09-23T21:14:25.597264+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":495,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:04:43.384190+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,5],"acting":[2,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":38,"reported_epoch":528,"state":"active+clean","last_fresh":"2023-09-23T21:14:25.609232+0000","last_change":"2023-09-23T21:13:30.785054+0000","last_active":"2023-09-23T21:14:25.609232+0000","last_peered":"2023-09-23T21:14:25.609232+0000","last_clean":"2023-09-23T21:14:25.609232+0000","last_became_active":"2023-09-23T21:13:30.784701+0000","last_became_peered":"2023-09-23T21:13:30.784701+0000","last_unstale":"2023-09-23T21:14:25.609232+0000","last_undegraded":"2023-09-23T21:14:25.609232+0000","last_fullsized":"2023-09-23T21:14:25.609232+0000","mapping_epoch":501,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:19:20.152252+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.b","version":"529'8614","reported_seq":18041,"reported_epoch":529,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2023-09-23T21:14:43.714315+0000","last_change":"2023-09-23T21:13:59.891994+0000","last_active":"2023-09-23T21:14:43.714315+0000","last_peered":"2023-09-23T21:14:43.714315+0000","last_clean":"2023-09-23T21:11:26.732248+0000","last_became_active":"2023-09-23T21:13:14.937458+0000","last_became_peered":"2023-09-23T21:13:14.937458+0000","last_unstale":"2023-09-23T21:14:43.714315+0000","last_undegraded":"2023-09-23T21:13:59.884123+0000","last_fullsized":"2023-09-23T21:14:43.714315+0000","mapping_epoch":494,"log_start":"426'600","ondisk_log_start":"426'600","created":416,"last_epoch_clean":430,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_clean_scrub_stamp":"2023-09-23T21:10:15.663179+0000","objects_scrubbed":0,"log_size":8014,"log_dups_size":0,"ondisk_log_size":8014,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:36:15.760947+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":96862208,"num_objects":1478,"num_object_clones":0,"num_object_copies":4434,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":809,"num_objects_misplaced":1337,"num_objects_unfound":0,"num_objects_dirty":1478,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8614,"num_write_kb":322944,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4623,"num_bytes_recovered":302776320,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[1,4,7],"avail_no_missing":["1","7"],"object_location_counts":[{"shards":"0,1,7","objects":801},{"shards":"1,4,7","objects":602}],"blocked_by":[],"up_primary":0,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":314,"reported_epoch":528,"state":"active+clean","last_fresh":"2023-09-23T21:14:25.599361+0000","last_change":"2023-09-23T21:10:26.769764+0000","last_active":"2023-09-23T21:14:25.599361+0000","last_peered":"2023-09-23T21:14:25.599361+0000","last_clean":"2023-09-23T21:14:25.599361+0000","last_became_active":"2023-09-23T21:10:26.769326+0000","last_became_peered":"2023-09-23T21:10:26.769326+0000","last_unstale":"2023-09-23T21:14:25.599361+0000","last_undegraded":"2023-09-23T21:14:25.599361+0000","last_fullsized":"2023-09-23T21:14:25.599361+0000","mapping_epoch":423,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":424,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:57:45.219837+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"79'161","reported_seq":711,"reported_epoch":528,"state":"active+clean","last_fresh":"2023-09-23T21:14:25.602689+0000","last_change":"2023-09-23T21:13:33.766895+0000","last_active":"2023-09-23T21:14:25.602689+0000","last_peered":"2023-09-23T21:14:25.602689+0000","last_clean":"2023-09-23T21:14:25.602689+0000","last_became_active":"2023-09-23T21:13:33.766538+0000","last_became_peered":"2023-09-23T21:13:33.766538+0000","last_unstale":"2023-09-23T21:14:25.602689+0000","last_undegraded":"2023-09-23T21:14:25.602689+0000","last_fullsized":"2023-09-23T21:14:25.602689+0000","mapping_epoch":504,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":505,"parent":"0.0","parent_split_bits":0,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:20.738273+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:20.738273+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:35:35.855987+0000","scrub_duration":0.0088336339999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,2],"acting":[0,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":536,"reported_epoch":528,"state":"active+clean","last_fresh":"2023-09-23T21:14:25.617120+0000","last_change":"2023-09-23T21:11:08.760417+0000","last_active":"2023-09-23T21:14:25.617120+0000","last_peered":"2023-09-23T21:14:25.617120+0000","last_clean":"2023-09-23T21:14:25.617120+0000","last_became_active":"2023-09-23T21:11:08.760150+0000","last_became_peered":"2023-09-23T21:11:08.760150+0000","last_unstale":"2023-09-23T21:14:25.617120+0000","last_undegraded":"2023-09-23T21:14:25.617120+0000","last_fullsized":"2023-09-23T21:14:25.617120+0000","mapping_epoch":443,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":444,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:23:10.849768+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":103,"reported_epoch":528,"state":"active+clean","last_fresh":"2023-09-23T21:14:25.607373+0000","last_change":"2023-09-23T21:13:12.216020+0000","last_active":"2023-09-23T21:14:25.607373+0000","last_peered":"2023-09-23T21:14:25.607373+0000","last_clean":"2023-09-23T21:14:25.607373+0000","last_became_active":"2023-09-23T21:13:12.215180+0000","last_became_peered":"2023-09-23T21:13:12.215180+0000","last_unstale":"2023-09-23T21:14:25.607373+0000","last_undegraded":"2023-09-23T21:14:25.607373+0000","last_fullsized":"2023-09-23T21:14:25.607373+0000","mapping_epoch":493,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:26:18.704684+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.8","version":"529'5430","reported_seq":7825,"reported_epoch":529,"state":"active+clean","last_fresh":"2023-09-23T21:14:40.127249+0000","last_change":"2023-09-23T21:14:03.796930+0000","last_active":"2023-09-23T21:14:40.127249+0000","last_peered":"2023-09-23T21:14:40.127249+0000","last_clean":"2023-09-23T21:14:40.127249+0000","last_became_active":"2023-09-23T21:14:03.786177+0000","last_became_peered":"2023-09-23T21:14:03.786177+0000","last_unstale":"2023-09-23T21:14:40.127249+0000","last_undegraded":"2023-09-23T21:14:40.127249+0000","last_fullsized":"2023-09-23T21:14:40.127249+0000","mapping_epoch":524,"log_start":"422'182","ondisk_log_start":"422'182","created":416,"last_epoch_clean":525,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_clean_scrub_stamp":"2023-09-23T21:10:15.663179+0000","objects_scrubbed":0,"log_size":5248,"log_dups_size":0,"ondisk_log_size":5248,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":49053696,"num_objects":748,"num_object_clones":0,"num_object_copies":2244,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":748,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4080,"num_write_kb":154528,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":880,"num_bytes_recovered":16908288,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,7],"acting":[0,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.3","version":"79'161","reported_seq":831,"reported_epoch":528,"state":"active+clean","last_fresh":"2023-09-23T21:14:25.610255+0000","last_change":"2023-09-23T21:13:33.771279+0000","last_active":"2023-09-23T21:14:25.610255+0000","last_peered":"2023-09-23T21:14:25.610255+0000","last_clean":"2023-09-23T21:14:25.610255+0000","last_became_active":"2023-09-23T21:13:33.770980+0000","last_became_peered":"2023-09-23T21:13:33.770980+0000","last_unstale":"2023-09-23T21:14:25.610255+0000","last_undegraded":"2023-09-23T21:14:25.610255+0000","last_fullsized":"2023-09-23T21:14:25.610255+0000","mapping_epoch":504,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":505,"parent":"0.0","parent_split_bits":3,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:05:57.832302+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:05:57.832302+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:19:42.309604+0000","scrub_duration":0.010206847999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":345,"reported_epoch":528,"state":"active+clean","last_fresh":"2023-09-23T21:14:25.603174+0000","last_change":"2023-09-23T21:13:13.212775+0000","last_active":"2023-09-23T21:14:25.603174+0000","last_peered":"2023-09-23T21:14:25.603174+0000","last_clean":"2023-09-23T21:14:25.603174+0000","last_became_active":"2023-09-23T21:13:13.212460+0000","last_became_peered":"2023-09-23T21:13:13.212460+0000","last_unstale":"2023-09-23T21:14:25.603174+0000","last_undegraded":"2023-09-23T21:14:25.603174+0000","last_fullsized":"2023-09-23T21:14:25.603174+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":495,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:27:10.853970+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":527,"reported_epoch":528,"state":"active+clean","last_fresh":"2023-09-23T21:14:25.600793+0000","last_change":"2023-09-23T21:13:44.409212+0000","last_active":"2023-09-23T21:14:25.600793+0000","last_peered":"2023-09-23T21:14:25.600793+0000","last_clean":"2023-09-23T21:14:25.600793+0000","last_became_active":"2023-09-23T21:13:44.408958+0000","last_became_peered":"2023-09-23T21:13:44.408958+0000","last_unstale":"2023-09-23T21:14:25.600793+0000","last_undegraded":"2023-09-23T21:14:25.600793+0000","last_fullsized":"2023-09-23T21:14:25.600793+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:03:52.749964+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":38,"reported_epoch":528,"state":"active+clean","last_fresh":"2023-09-23T21:14:25.605616+0000","last_change":"2023-09-23T21:13:30.804379+0000","last_active":"2023-09-23T21:14:25.605616+0000","last_peered":"2023-09-23T21:14:25.605616+0000","last_clean":"2023-09-23T21:14:25.605616+0000","last_became_active":"2023-09-23T21:13:30.799907+0000","last_became_peered":"2023-09-23T21:13:30.799907+0000","last_unstale":"2023-09-23T21:14:25.605616+0000","last_undegraded":"2023-09-23T21:14:25.605616+0000","last_fullsized":"2023-09-23T21:14:25.605616+0000","mapping_epoch":501,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:49:42.859212+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,7],"acting":[0,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.d","version":"529'8501","reported_seq":8663,"reported_epoch":529,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-23T21:14:43.222919+0000","last_change":"2023-09-23T21:13:31.801486+0000","last_active":"2023-09-23T21:14:43.222919+0000","last_peered":"2023-09-23T21:14:43.222919+0000","last_clean":"2023-09-23T21:11:37.659929+0000","last_became_active":"2023-09-23T21:13:31.786626+0000","last_became_peered":"2023-09-23T21:13:31.786626+0000","last_unstale":"2023-09-23T21:14:43.222919+0000","last_undegraded":"2023-09-23T21:13:31.760908+0000","last_fullsized":"2023-09-23T21:14:43.222919+0000","mapping_epoch":502,"log_start":"458'4595","ondisk_log_start":"458'4595","created":416,"last_epoch_clean":417,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_clean_scrub_stamp":"2023-09-23T21:10:15.663179+0000","objects_scrubbed":0,"log_size":3906,"log_dups_size":0,"ondisk_log_size":3906,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:46:02.424493+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":98304024,"num_objects":1501,"num_object_clones":0,"num_object_copies":4503,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1265,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1501,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":8501,"num_write_kb":320001,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,3],"avail_no_missing":["4","3","7"],"object_location_counts":[{"shards":"0,3,4","objects":1422},{"shards":"3,4","objects":79}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.6","version":"79'161","reported_seq":447,"reported_epoch":528,"state":"active+clean","last_fresh":"2023-09-23T21:14:25.605215+0000","last_change":"2023-09-23T21:13:33.770715+0000","last_active":"2023-09-23T21:14:25.605215+0000","last_peered":"2023-09-23T21:14:25.605215+0000","last_clean":"2023-09-23T21:14:25.605215+0000","last_became_active":"2023-09-23T21:13:33.769303+0000","last_became_peered":"2023-09-23T21:13:33.769303+0000","last_unstale":"2023-09-23T21:14:25.605215+0000","last_undegraded":"2023-09-23T21:14:25.605215+0000","last_fullsized":"2023-09-23T21:14:25.605215+0000","mapping_epoch":504,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":505,"parent":"0.0","parent_split_bits":3,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:10:24.268662+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:10:24.268662+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13417,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":11,"num_read_kb":20,"num_write":5,"num_write_kb":55,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":40252,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":155,"reported_epoch":528,"state":"active+clean","last_fresh":"2023-09-23T21:14:25.617322+0000","last_change":"2023-09-23T21:13:30.770826+0000","last_active":"2023-09-23T21:14:25.617322+0000","last_peered":"2023-09-23T21:14:25.617322+0000","last_clean":"2023-09-23T21:14:25.617322+0000","last_became_active":"2023-09-23T21:13:30.770512+0000","last_became_peered":"2023-09-23T21:13:30.770512+0000","last_unstale":"2023-09-23T21:14:25.617322+0000","last_undegraded":"2023-09-23T21:14:25.617322+0000","last_fullsized":"2023-09-23T21:14:25.617322+0000","mapping_epoch":501,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:44:23.470538+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":412,"reported_epoch":528,"state":"active+clean","last_fresh":"2023-09-23T21:14:25.598340+0000","last_change":"2023-09-23T21:13:30.790590+0000","last_active":"2023-09-23T21:14:25.598340+0000","last_peered":"2023-09-23T21:14:25.598340+0000","last_clean":"2023-09-23T21:14:25.598340+0000","last_became_active":"2023-09-23T21:13:30.790253+0000","last_became_peered":"2023-09-23T21:13:30.790253+0000","last_unstale":"2023-09-23T21:14:25.598340+0000","last_undegraded":"2023-09-23T21:14:25.598340+0000","last_fullsized":"2023-09-23T21:14:25.598340+0000","mapping_epoch":501,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:23:41.843250+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,0],"acting":[2,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":474,"reported_epoch":528,"state":"active+clean","last_fresh":"2023-09-23T21:14:25.604980+0000","last_change":"2023-09-23T21:13:12.210546+0000","last_active":"2023-09-23T21:14:25.604980+0000","last_peered":"2023-09-23T21:14:25.604980+0000","last_clean":"2023-09-23T21:14:25.604980+0000","last_became_active":"2023-09-23T21:13:12.205934+0000","last_became_peered":"2023-09-23T21:13:12.205934+0000","last_unstale":"2023-09-23T21:14:25.604980+0000","last_undegraded":"2023-09-23T21:14:25.604980+0000","last_fullsized":"2023-09-23T21:14:25.604980+0000","mapping_epoch":493,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:21:18.299428+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.e","version":"529'8709","reported_seq":15680,"reported_epoch":529,"state":"active+clean","last_fresh":"2023-09-23T21:14:43.307766+0000","last_change":"2023-09-23T21:12:38.623611+0000","last_active":"2023-09-23T21:14:43.307766+0000","last_peered":"2023-09-23T21:14:43.307766+0000","last_clean":"2023-09-23T21:14:43.307766+0000","last_became_active":"2023-09-23T21:12:38.610155+0000","last_became_peered":"2023-09-23T21:12:38.610155+0000","last_unstale":"2023-09-23T21:14:43.307766+0000","last_undegraded":"2023-09-23T21:14:43.307766+0000","last_fullsized":"2023-09-23T21:14:43.307766+0000","mapping_epoch":483,"log_start":"462'4793","ondisk_log_start":"462'4793","created":416,"last_epoch_clean":484,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_clean_scrub_stamp":"2023-09-23T21:10:15.663179+0000","objects_scrubbed":0,"log_size":3916,"log_dups_size":0,"ondisk_log_size":3916,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:19:27.097170+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":99287040,"num_objects":1515,"num_object_clones":0,"num_object_copies":4545,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1515,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8709,"num_write_kb":327168,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3412,"num_bytes_recovered":223608832,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.5","version":"79'161","reported_seq":556,"reported_epoch":528,"state":"active+clean","last_fresh":"2023-09-23T21:14:25.607298+0000","last_change":"2023-09-23T21:13:38.373969+0000","last_active":"2023-09-23T21:14:25.607298+0000","last_peered":"2023-09-23T21:14:25.607298+0000","last_clean":"2023-09-23T21:14:25.607298+0000","last_became_active":"2023-09-23T21:13:38.373633+0000","last_became_peered":"2023-09-23T21:13:38.373633+0000","last_unstale":"2023-09-23T21:14:25.607298+0000","last_undegraded":"2023-09-23T21:14:25.607298+0000","last_fullsized":"2023-09-23T21:14:25.607298+0000","mapping_epoch":507,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":3,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:09:34.892947+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:09:34.892947+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13417,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":11,"num_read_kb":20,"num_write":5,"num_write_kb":55,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":40252,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":69,"reported_epoch":528,"state":"active+clean","last_fresh":"2023-09-23T21:14:25.609312+0000","last_change":"2023-09-23T21:13:12.235743+0000","last_active":"2023-09-23T21:14:25.609312+0000","last_peered":"2023-09-23T21:14:25.609312+0000","last_clean":"2023-09-23T21:14:25.609312+0000","last_became_active":"2023-09-23T21:13:12.216241+0000","last_became_peered":"2023-09-23T21:13:12.216241+0000","last_unstale":"2023-09-23T21:14:25.609312+0000","last_undegraded":"2023-09-23T21:14:25.609312+0000","last_fullsized":"2023-09-23T21:14:25.609312+0000","mapping_epoch":493,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:12:19.090710+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":509,"reported_epoch":528,"state":"active+clean","last_fresh":"2023-09-23T21:14:25.607983+0000","last_change":"2023-09-23T21:13:30.807520+0000","last_active":"2023-09-23T21:14:25.607983+0000","last_peered":"2023-09-23T21:14:25.607983+0000","last_clean":"2023-09-23T21:14:25.607983+0000","last_became_active":"2023-09-23T21:13:30.807222+0000","last_became_peered":"2023-09-23T21:13:30.807222+0000","last_unstale":"2023-09-23T21:14:25.607983+0000","last_undegraded":"2023-09-23T21:14:25.607983+0000","last_fullsized":"2023-09-23T21:14:25.607983+0000","mapping_epoch":501,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:40:31.989749+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":370,"reported_epoch":528,"state":"active+clean","last_fresh":"2023-09-23T21:14:25.606132+0000","last_change":"2023-09-23T21:13:30.794095+0000","last_active":"2023-09-23T21:14:25.606132+0000","last_peered":"2023-09-23T21:14:25.606132+0000","last_clean":"2023-09-23T21:14:25.606132+0000","last_became_active":"2023-09-23T21:13:30.793791+0000","last_became_peered":"2023-09-23T21:13:30.793791+0000","last_unstale":"2023-09-23T21:14:25.606132+0000","last_undegraded":"2023-09-23T21:14:25.606132+0000","last_fullsized":"2023-09-23T21:14:25.606132+0000","mapping_epoch":501,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:36:28.707362+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.f","version":"529'8649","reported_seq":8853,"reported_epoch":529,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T21:14:44.156805+0000","last_change":"2023-09-23T21:13:59.874192+0000","last_active":"2023-09-23T21:14:44.156805+0000","last_peered":"2023-09-23T21:14:44.156805+0000","last_clean":"2023-09-23T21:11:07.652290+0000","last_became_active":"2023-09-23T21:11:09.897463+0000","last_became_peered":"2023-09-23T21:11:09.897463+0000","last_unstale":"2023-09-23T21:14:44.156805+0000","last_undegraded":"2023-09-23T21:11:27.781677+0000","last_fullsized":"2023-09-23T21:11:09.761652+0000","mapping_epoch":444,"log_start":"464'4909","ondisk_log_start":"464'4909","created":416,"last_epoch_clean":417,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_clean_scrub_stamp":"2023-09-23T21:10:15.663179+0000","objects_scrubbed":0,"log_size":3740,"log_dups_size":0,"ondisk_log_size":3740,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:42:51.588441+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":96010240,"num_objects":1465,"num_object_clones":0,"num_object_copies":4395,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3295,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1465,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8649,"num_write_kb":323648,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":32,"num_bytes_recovered":2097152,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2],"avail_no_missing":["3","1","2"],"object_location_counts":[{"shards":"2,3","objects":-1830}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.4","version":"79'161","reported_seq":899,"reported_epoch":528,"state":"active+clean","last_fresh":"2023-09-23T21:14:25.607464+0000","last_change":"2023-09-23T21:10:29.807785+0000","last_active":"2023-09-23T21:14:25.607464+0000","last_peered":"2023-09-23T21:14:25.607464+0000","last_clean":"2023-09-23T21:14:25.607464+0000","last_became_active":"2023-09-23T21:10:29.807366+0000","last_became_peered":"2023-09-23T21:10:29.807366+0000","last_unstale":"2023-09-23T21:14:25.607464+0000","last_undegraded":"2023-09-23T21:14:25.607464+0000","last_fullsized":"2023-09-23T21:14:25.607464+0000","mapping_epoch":426,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":427,"parent":"0.0","parent_split_bits":3,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:20.012764+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:20.012764+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:24:45.763834+0000","scrub_duration":0.0045251919999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":468,"reported_epoch":529,"state":"active+clean","last_fresh":"2023-09-23T21:14:32.797373+0000","last_change":"2023-09-23T21:10:26.768364+0000","last_active":"2023-09-23T21:14:32.797373+0000","last_peered":"2023-09-23T21:14:32.797373+0000","last_clean":"2023-09-23T21:14:32.797373+0000","last_became_active":"2023-09-23T21:10:26.768052+0000","last_became_peered":"2023-09-23T21:10:26.768052+0000","last_unstale":"2023-09-23T21:14:32.797373+0000","last_undegraded":"2023-09-23T21:14:32.797373+0000","last_fullsized":"2023-09-23T21:14:32.797373+0000","mapping_epoch":423,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":424,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:22:42.201200+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,3],"acting":[2,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.6","version":"70'1","reported_seq":499,"reported_epoch":528,"state":"active+clean","last_fresh":"2023-09-23T21:14:25.605745+0000","last_change":"2023-09-23T21:13:30.793290+0000","last_active":"2023-09-23T21:14:25.605745+0000","last_peered":"2023-09-23T21:14:25.605745+0000","last_clean":"2023-09-23T21:14:25.605745+0000","last_became_active":"2023-09-23T21:13:30.786079+0000","last_became_peered":"2023-09-23T21:13:30.786079+0000","last_unstale":"2023-09-23T21:14:25.605745+0000","last_undegraded":"2023-09-23T21:14:25.605745+0000","last_fullsized":"2023-09-23T21:14:25.605745+0000","mapping_epoch":501,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:34:43.281095+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":103,"reported_epoch":528,"state":"active+clean","last_fresh":"2023-09-23T21:14:25.606327+0000","last_change":"2023-09-23T21:13:12.216895+0000","last_active":"2023-09-23T21:14:25.606327+0000","last_peered":"2023-09-23T21:14:25.606327+0000","last_clean":"2023-09-23T21:14:25.606327+0000","last_became_active":"2023-09-23T21:13:12.216654+0000","last_became_peered":"2023-09-23T21:13:12.216654+0000","last_unstale":"2023-09-23T21:14:25.606327+0000","last_undegraded":"2023-09-23T21:14:25.606327+0000","last_fullsized":"2023-09-23T21:14:25.606327+0000","mapping_epoch":493,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:02:40.430096+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.19","version":"79'161","reported_seq":681,"reported_epoch":528,"state":"active+clean","last_fresh":"2023-09-23T21:14:25.597406+0000","last_change":"2023-09-23T21:12:30.442914+0000","last_active":"2023-09-23T21:14:25.597406+0000","last_peered":"2023-09-23T21:14:25.597406+0000","last_clean":"2023-09-23T21:14:25.597406+0000","last_became_active":"2023-09-23T21:12:30.442578+0000","last_became_peered":"2023-09-23T21:12:30.442578+0000","last_unstale":"2023-09-23T21:14:25.597406+0000","last_undegraded":"2023-09-23T21:14:25.597406+0000","last_fullsized":"2023-09-23T21:14:25.597406+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":479,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:10:37.696951+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:10:37.696951+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.001386504,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,3],"acting":[2,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"10.12","version":"529'5575","reported_seq":7946,"reported_epoch":529,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T21:14:43.230284+0000","last_change":"2023-09-23T21:13:13.419184+0000","last_active":"2023-09-23T21:14:43.230284+0000","last_peered":"2023-09-23T21:14:43.230284+0000","last_clean":"2023-09-23T21:11:45.774771+0000","last_became_active":"2023-09-23T21:13:13.409281+0000","last_became_peered":"2023-09-23T21:13:13.409281+0000","last_unstale":"2023-09-23T21:14:43.230284+0000","last_undegraded":"2023-09-23T21:13:13.284063+0000","last_fullsized":"2023-09-23T21:13:13.191117+0000","mapping_epoch":494,"log_start":"426'588","ondisk_log_start":"426'588","created":438,"last_epoch_clean":439,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_clean_scrub_stamp":"2023-09-23T21:10:15.663179+0000","objects_scrubbed":0,"log_size":4987,"log_dups_size":0,"ondisk_log_size":4987,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":48365568,"num_objects":738,"num_object_clones":0,"num_object_copies":2214,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2034,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":738,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4198,"num_write_kb":157952,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1104,"num_bytes_recovered":72351744,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,3],"avail_no_missing":["4","1"],"object_location_counts":[{"shards":"0,1,4","objects":700},{"shards":"1,4","objects":41},{"shards":"3,4","objects":-3}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":38,"reported_epoch":528,"state":"active+clean","last_fresh":"2023-09-23T21:14:25.604903+0000","last_change":"2023-09-23T21:13:30.791297+0000","last_active":"2023-09-23T21:14:25.604903+0000","last_peered":"2023-09-23T21:14:25.604903+0000","last_clean":"2023-09-23T21:14:25.604903+0000","last_became_active":"2023-09-23T21:13:30.788721+0000","last_became_peered":"2023-09-23T21:13:30.788721+0000","last_unstale":"2023-09-23T21:14:25.604903+0000","last_undegraded":"2023-09-23T21:14:25.604903+0000","last_fullsized":"2023-09-23T21:14:25.604903+0000","mapping_epoch":501,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:31:18.147883+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":32,"reported_epoch":528,"state":"active+clean","last_fresh":"2023-09-23T21:14:25.598261+0000","last_change":"2023-09-23T21:13:38.373814+0000","last_active":"2023-09-23T21:14:25.598261+0000","last_peered":"2023-09-23T21:14:25.598261+0000","last_clean":"2023-09-23T21:14:25.598261+0000","last_became_active":"2023-09-23T21:13:38.373458+0000","last_became_peered":"2023-09-23T21:13:38.373458+0000","last_unstale":"2023-09-23T21:14:25.598261+0000","last_undegraded":"2023-09-23T21:14:25.598261+0000","last_fullsized":"2023-09-23T21:14:25.598261+0000","mapping_epoch":507,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:23:08.288401+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.1e","version":"79'161","reported_seq":696,"reported_epoch":528,"state":"active+clean","last_fresh":"2023-09-23T21:14:25.607191+0000","last_change":"2023-09-23T21:12:30.448761+0000","last_active":"2023-09-23T21:14:25.607191+0000","last_peered":"2023-09-23T21:14:25.607191+0000","last_clean":"2023-09-23T21:14:25.607191+0000","last_became_active":"2023-09-23T21:12:30.448043+0000","last_became_peered":"2023-09-23T21:12:30.448043+0000","last_unstale":"2023-09-23T21:14:25.607191+0000","last_undegraded":"2023-09-23T21:14:25.607191+0000","last_fullsized":"2023-09-23T21:14:25.607191+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":479,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:10:39.567232+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:10:39.567232+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.005229694,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.15","version":"529'5460","reported_seq":5606,"reported_epoch":529,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T21:14:41.047724+0000","last_change":"2023-09-23T21:13:13.436807+0000","last_active":"2023-09-23T21:14:41.047724+0000","last_peered":"2023-09-23T21:14:41.047724+0000","last_clean":"2023-09-23T21:11:45.765730+0000","last_became_active":"2023-09-23T21:13:13.429206+0000","last_became_peered":"2023-09-23T21:13:13.429206+0000","last_unstale":"2023-09-23T21:14:41.047724+0000","last_undegraded":"2023-09-23T21:13:13.246273+0000","last_fullsized":"2023-09-23T21:13:13.203825+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":438,"last_epoch_clean":439,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_clean_scrub_stamp":"2023-09-23T21:10:15.663179+0000","objects_scrubbed":0,"log_size":5460,"log_dups_size":0,"ondisk_log_size":5460,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53870592,"num_objects":822,"num_object_clones":0,"num_object_copies":2466,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":643,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":822,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4126,"num_write_kb":158336,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,3],"acting":[2,3],"avail_no_missing":["2","3"],"object_location_counts":[{"shards":"2,3","objects":786}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":104,"reported_epoch":528,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-23T21:14:25.607808+0000","last_change":"2023-09-23T21:12:14.313202+0000","last_active":"2023-09-23T21:14:25.607808+0000","last_peered":"2023-09-23T21:14:25.607808+0000","last_clean":"2023-09-23T21:12:09.299355+0000","last_became_active":"2023-09-23T21:12:14.209408+0000","last_became_peered":"2023-09-23T21:12:14.209408+0000","last_unstale":"2023-09-23T21:14:25.607808+0000","last_undegraded":"2023-09-23T21:12:12.933857+0000","last_fullsized":"2023-09-23T21:14:25.607808+0000","mapping_epoch":470,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":444,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:45:40.142413+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,7],"acting":[4,5,7],"avail_no_missing":["4","1","5"],"object_location_counts":[{"shards":"1,4,5","objects":1}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":93,"reported_epoch":528,"state":"active+clean","last_fresh":"2023-09-23T21:14:25.610601+0000","last_change":"2023-09-23T21:12:10.770474+0000","last_active":"2023-09-23T21:14:25.610601+0000","last_peered":"2023-09-23T21:14:25.610601+0000","last_clean":"2023-09-23T21:14:25.610601+0000","last_became_active":"2023-09-23T21:12:10.770184+0000","last_became_peered":"2023-09-23T21:12:10.770184+0000","last_unstale":"2023-09-23T21:14:25.610601+0000","last_undegraded":"2023-09-23T21:14:25.610601+0000","last_fullsized":"2023-09-23T21:14:25.610601+0000","mapping_epoch":470,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:05:09.407151+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.14","version":"529'5442","reported_seq":11543,"reported_epoch":529,"state":"active+clean","last_fresh":"2023-09-23T21:14:39.999822+0000","last_change":"2023-09-23T21:14:01.515957+0000","last_active":"2023-09-23T21:14:39.999822+0000","last_peered":"2023-09-23T21:14:39.999822+0000","last_clean":"2023-09-23T21:14:39.999822+0000","last_became_active":"2023-09-23T21:14:01.506428+0000","last_became_peered":"2023-09-23T21:14:01.506428+0000","last_unstale":"2023-09-23T21:14:39.999822+0000","last_undegraded":"2023-09-23T21:14:39.999822+0000","last_fullsized":"2023-09-23T21:14:39.999822+0000","mapping_epoch":522,"log_start":"422'217","ondisk_log_start":"422'217","created":438,"last_epoch_clean":523,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_clean_scrub_stamp":"2023-09-23T21:10:15.663179+0000","objects_scrubbed":0,"log_size":5225,"log_dups_size":0,"ondisk_log_size":5225,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":52559872,"num_objects":802,"num_object_clones":0,"num_object_copies":2406,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":802,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4108,"num_write_kb":157120,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2660,"num_bytes_recovered":126156800,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":318,"reported_epoch":528,"state":"active+clean","last_fresh":"2023-09-23T21:14:25.598795+0000","last_change":"2023-09-23T21:12:10.771160+0000","last_active":"2023-09-23T21:14:25.598795+0000","last_peered":"2023-09-23T21:14:25.598795+0000","last_clean":"2023-09-23T21:14:25.598795+0000","last_became_active":"2023-09-23T21:12:10.757808+0000","last_became_peered":"2023-09-23T21:12:10.757808+0000","last_unstale":"2023-09-23T21:14:25.598795+0000","last_undegraded":"2023-09-23T21:14:25.598795+0000","last_fullsized":"2023-09-23T21:14:25.598795+0000","mapping_epoch":470,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:30:50.987139+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":103,"reported_epoch":528,"state":"active+clean","last_fresh":"2023-09-23T21:14:25.606829+0000","last_change":"2023-09-23T21:13:12.216602+0000","last_active":"2023-09-23T21:14:25.606829+0000","last_peered":"2023-09-23T21:14:25.606829+0000","last_clean":"2023-09-23T21:14:25.606829+0000","last_became_active":"2023-09-23T21:13:12.216204+0000","last_became_peered":"2023-09-23T21:13:12.216204+0000","last_unstale":"2023-09-23T21:14:25.606829+0000","last_undegraded":"2023-09-23T21:14:25.606829+0000","last_fullsized":"2023-09-23T21:14:25.606829+0000","mapping_epoch":493,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:23:37.043390+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1c","version":"79'161","reported_seq":894,"reported_epoch":528,"state":"active+clean","last_fresh":"2023-09-23T21:14:25.607120+0000","last_change":"2023-09-23T21:13:13.402773+0000","last_active":"2023-09-23T21:14:25.607120+0000","last_peered":"2023-09-23T21:14:25.607120+0000","last_clean":"2023-09-23T21:14:25.607120+0000","last_became_active":"2023-09-23T21:13:13.402025+0000","last_became_peered":"2023-09-23T21:13:13.402025+0000","last_unstale":"2023-09-23T21:14:25.607120+0000","last_undegraded":"2023-09-23T21:14:25.607120+0000","last_fullsized":"2023-09-23T21:14:25.607120+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":479,"last_epoch_clean":495,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:22.040907+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:22.040907+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.013021176000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":163840,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":163840,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.17","version":"529'5542","reported_seq":9928,"reported_epoch":529,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T21:14:44.182928+0000","last_change":"2023-09-23T21:13:39.447969+0000","last_active":"2023-09-23T21:14:44.182928+0000","last_peered":"2023-09-23T21:14:44.182928+0000","last_clean":"2023-09-23T21:13:37.065694+0000","last_became_active":"2023-09-23T21:13:39.439739+0000","last_became_peered":"2023-09-23T21:13:39.439739+0000","last_unstale":"2023-09-23T21:14:44.182928+0000","last_undegraded":"2023-09-23T21:13:39.411853+0000","last_fullsized":"2023-09-23T21:13:39.404657+0000","mapping_epoch":508,"log_start":"434'983","ondisk_log_start":"434'983","created":438,"last_epoch_clean":487,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_clean_scrub_stamp":"2023-09-23T21:10:15.663179+0000","objects_scrubbed":0,"log_size":4559,"log_dups_size":0,"ondisk_log_size":4559,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":49709056,"num_objects":759,"num_object_clones":0,"num_object_copies":2277,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1109,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":759,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4177,"num_write_kb":157920,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2112,"num_bytes_recovered":116719616,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,2],"acting":[3,5],"avail_no_missing":["3","4","5"],"object_location_counts":[{"shards":"3,4,5","objects":311},{"shards":"3,5","objects":448}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":118,"reported_epoch":529,"state":"active+clean","last_fresh":"2023-09-23T21:14:32.795846+0000","last_change":"2023-09-23T21:12:10.779066+0000","last_active":"2023-09-23T21:14:32.795846+0000","last_peered":"2023-09-23T21:14:32.795846+0000","last_clean":"2023-09-23T21:14:32.795846+0000","last_became_active":"2023-09-23T21:12:10.778691+0000","last_became_peered":"2023-09-23T21:12:10.778691+0000","last_unstale":"2023-09-23T21:14:32.795846+0000","last_undegraded":"2023-09-23T21:14:32.795846+0000","last_fullsized":"2023-09-23T21:14:32.795846+0000","mapping_epoch":470,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:07:22.043268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,7],"acting":[2,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":196,"reported_epoch":528,"state":"active+clean","last_fresh":"2023-09-23T21:14:25.597736+0000","last_change":"2023-09-23T21:13:12.200767+0000","last_active":"2023-09-23T21:14:25.597736+0000","last_peered":"2023-09-23T21:14:25.597736+0000","last_clean":"2023-09-23T21:14:25.597736+0000","last_became_active":"2023-09-23T21:13:12.200435+0000","last_became_peered":"2023-09-23T21:13:12.200435+0000","last_unstale":"2023-09-23T21:14:25.597736+0000","last_undegraded":"2023-09-23T21:14:25.597736+0000","last_fullsized":"2023-09-23T21:14:25.597736+0000","mapping_epoch":493,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:15:44.937886+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.1d","version":"79'161","reported_seq":681,"reported_epoch":528,"state":"active+clean","last_fresh":"2023-09-23T21:14:25.617070+0000","last_change":"2023-09-23T21:12:30.444331+0000","last_active":"2023-09-23T21:14:25.617070+0000","last_peered":"2023-09-23T21:14:25.617070+0000","last_clean":"2023-09-23T21:14:25.617070+0000","last_became_active":"2023-09-23T21:12:30.444001+0000","last_became_peered":"2023-09-23T21:12:30.444001+0000","last_unstale":"2023-09-23T21:14:25.617070+0000","last_undegraded":"2023-09-23T21:14:25.617070+0000","last_fullsized":"2023-09-23T21:14:25.617070+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":479,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:09:36.858077+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:09:36.858077+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0024987500000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"10.16","version":"529'5664","reported_seq":9174,"reported_epoch":529,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T21:14:43.244679+0000","last_change":"2023-09-23T21:13:14.424843+0000","last_active":"2023-09-23T21:14:43.244679+0000","last_peered":"2023-09-23T21:14:43.244679+0000","last_clean":"2023-09-23T21:11:45.652255+0000","last_became_active":"2023-09-23T21:13:14.414710+0000","last_became_peered":"2023-09-23T21:13:14.414710+0000","last_unstale":"2023-09-23T21:14:43.244679+0000","last_undegraded":"2023-09-23T21:13:13.211957+0000","last_fullsized":"2023-09-23T21:13:13.188859+0000","mapping_epoch":494,"log_start":"427'700","ondisk_log_start":"427'700","created":438,"last_epoch_clean":460,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_clean_scrub_stamp":"2023-09-23T21:10:15.663179+0000","objects_scrubbed":0,"log_size":4964,"log_dups_size":0,"ondisk_log_size":4964,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":49020928,"num_objects":748,"num_object_clones":0,"num_object_copies":2244,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":663,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":748,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4258,"num_write_kb":160192,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1670,"num_bytes_recovered":109445120,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,4],"acting":[4,5],"avail_no_missing":["4","5"],"object_location_counts":[{"shards":"1,4,5","objects":33},{"shards":"4,5","objects":715}],"blocked_by":[],"up_primary":0,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":55,"reported_epoch":528,"state":"active+clean","last_fresh":"2023-09-23T21:14:25.609067+0000","last_change":"2023-09-23T21:13:38.380574+0000","last_active":"2023-09-23T21:14:25.609067+0000","last_peered":"2023-09-23T21:14:25.609067+0000","last_clean":"2023-09-23T21:14:25.609067+0000","last_became_active":"2023-09-23T21:13:38.373252+0000","last_became_peered":"2023-09-23T21:13:38.373252+0000","last_unstale":"2023-09-23T21:14:25.609067+0000","last_undegraded":"2023-09-23T21:14:25.609067+0000","last_fullsized":"2023-09-23T21:14:25.609067+0000","mapping_epoch":507,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:58:14.788031+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":11,"num_bytes_recovered":3324,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":103,"reported_epoch":528,"state":"active+clean","last_fresh":"2023-09-23T21:14:25.615859+0000","last_change":"2023-09-23T21:12:10.793610+0000","last_active":"2023-09-23T21:14:25.615859+0000","last_peered":"2023-09-23T21:14:25.615859+0000","last_clean":"2023-09-23T21:14:25.615859+0000","last_became_active":"2023-09-23T21:12:10.793081+0000","last_became_peered":"2023-09-23T21:12:10.793081+0000","last_unstale":"2023-09-23T21:14:25.615859+0000","last_undegraded":"2023-09-23T21:14:25.615859+0000","last_fullsized":"2023-09-23T21:14:25.615859+0000","mapping_epoch":470,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:34:36.291550+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":10,"num_pg":26,"stat_sum":{"num_bytes":1622540312,"num_objects":24761,"num_object_clones":0,"num_object_copies":74283,"num_objects_missing_on_primary":1388,"num_objects_missing":1388,"num_objects_degraded":22118,"num_objects_misplaced":3570,"num_objects_unfound":0,"num_objects_dirty":24761,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":122613,"num_write_kb":4309313,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":40948,"num_bytes_recovered":2474016768,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":7663263744,"data_stored":7663255600,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":130912,"ondisk_log_size":130912,"up":78,"acting":62,"num_store_stats":7},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":59,"num_bytes_recovered":8898,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":126976,"data_stored":22458,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":14322,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":31,"stat_sum":{"num_bytes":697708,"num_objects":3,"num_object_clones":0,"num_object_copies":9,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3,"num_whiteouts":0,"num_read":89,"num_read_kb":165,"num_write":44,"num_write_kb":440,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":649699,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2109440,"data_stored":2098784,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4991,"ondisk_log_size":4991,"up":93,"acting":93,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201691,"num_pgs":69,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1090980,"kb_used_data":601628,"kb_used_omap":0,"kb_used_meta":489280,"kb_avail":92654172,"statfs":{"total":95995035648,"available":94877872128,"internally_reserved":0,"allocated":616067072,"data_stored":614534523,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":501022720},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:14:13 2023","interfaces":[{"interface":"back","average":{"1min":2.4260000000000002,"5min":2.4260000000000002,"15min":2.4260000000000002},"min":{"1min":0.23400000000000001,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":37.207999999999998,"5min":37.207999999999998,"15min":37.207999999999998},"last":4.3070000000000004},{"interface":"front","average":{"1min":2.234,"5min":2.234,"15min":2.234},"min":{"1min":0.28599999999999998,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":37.122,"5min":37.122,"15min":37.122},"last":0.26800000000000002}]},{"osd":1,"last update":"Sat Sep 23 21:14:09 2023","interfaces":[{"interface":"back","average":{"1min":0.94299999999999995,"5min":1.2310000000000001,"15min":2.04},"min":{"1min":0.30399999999999999,"5min":0.27100000000000002,"15min":0.26700000000000002},"max":{"1min":7.5369999999999999,"5min":14.034000000000001,"15min":14.034000000000001},"last":0.69899999999999995},{"interface":"front","average":{"1min":1.0469999999999999,"5min":1.4339999999999999,"15min":1.944},"min":{"1min":0.30499999999999999,"5min":0.29199999999999998,"15min":0.26300000000000001},"max":{"1min":5.6870000000000003,"5min":11.226000000000001,"15min":11.226000000000001},"last":4.2690000000000001}]},{"osd":2,"last update":"Sat Sep 23 21:14:30 2023","interfaces":[{"interface":"back","average":{"1min":0.80900000000000005,"5min":1.0880000000000001,"15min":1.7170000000000001},"min":{"1min":0.29699999999999999,"5min":0.26400000000000001,"15min":0.26200000000000001},"max":{"1min":3.0190000000000001,"5min":6.4219999999999997,"15min":14.478999999999999},"last":0.67000000000000004},{"interface":"front","average":{"1min":1.101,"5min":1.3520000000000001,"15min":1.7330000000000001},"min":{"1min":0.30399999999999999,"5min":0.26700000000000002,"15min":0.26500000000000001},"max":{"1min":5.7640000000000002,"5min":11.198,"15min":14.135},"last":4.1349999999999998}]},{"osd":3,"last update":"Sat Sep 23 21:13:57 2023","interfaces":[{"interface":"back","average":{"1min":1.022,"5min":1.296,"15min":2.0590000000000002},"min":{"1min":0.33000000000000002,"5min":0.316,"15min":0.29099999999999998},"max":{"1min":6.4930000000000003,"5min":7.5190000000000001,"15min":31.786000000000001},"last":0.63},{"interface":"front","average":{"1min":1.1160000000000001,"5min":1.3859999999999999,"15min":2.0409999999999999},"min":{"1min":0.37,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":5.9329999999999998,"5min":11.244999999999999,"15min":32.429000000000002},"last":0.40899999999999997}]},{"osd":4,"last update":"Sat Sep 23 21:13:53 2023","interfaces":[{"interface":"back","average":{"1min":1.5980000000000001,"5min":1.5209999999999999,"15min":1.097},"min":{"1min":0.28999999999999998,"5min":0.255,"15min":0.248},"max":{"1min":7.6609999999999996,"5min":11.297000000000001,"15min":11.297000000000001},"last":4.2130000000000001},{"interface":"front","average":{"1min":1.387,"5min":1.1679999999999999,"15min":1.0669999999999999},"min":{"1min":0.314,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":8.6059999999999999,"5min":11.093999999999999,"15min":12.752000000000001},"last":0.29699999999999999}]},{"osd":5,"last update":"Sat Sep 23 21:13:53 2023","interfaces":[{"interface":"back","average":{"1min":1.4670000000000001,"5min":1.0549999999999999,"15min":1.014},"min":{"1min":0.29899999999999999,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":7.5910000000000002,"5min":9.4789999999999992,"15min":9.4789999999999992},"last":0.73099999999999998},{"interface":"front","average":{"1min":1.4950000000000001,"5min":1.3680000000000001,"15min":1.099},"min":{"1min":0.39700000000000002,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":7.6929999999999996,"5min":11.257,"15min":11.257},"last":4.181}]},{"osd":6,"last update":"Sat Sep 23 21:14:09 2023","interfaces":[{"interface":"back","average":{"1min":0.79600000000000004,"5min":1.1819999999999999,"15min":1.038},"min":{"1min":0.39800000000000002,"5min":0.30099999999999999,"15min":0.28899999999999998},"max":{"1min":1.667,"5min":10.941000000000001,"15min":10.941000000000001},"last":0.48099999999999998},{"interface":"front","average":{"1min":0.99299999999999999,"5min":1.1020000000000001,"15min":1.1259999999999999},"min":{"1min":0.41699999999999998,"5min":0.33100000000000002,"15min":0.32300000000000001},"max":{"1min":2.4969999999999999,"5min":8.923,"15min":12.715},"last":0.53700000000000003}]}]},{"osd":6,"up_from":0,"seq":704374636775,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":1,"up_from":0,"seq":1503238553707,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":0,"up_from":493,"seq":2117418876947,"num_pgs":75,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2050948,"kb_used_data":1892540,"kb_used_omap":36,"kb_used_meta":158235,"kb_avail":91694204,"statfs":{"total":95995035648,"available":93894864896,"internally_reserved":0,"allocated":1937960960,"data_stored":1935479865,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":37336,"internal_metadata":162033192},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 21:14:12 2023","interfaces":[{"interface":"back","average":{"1min":0.75600000000000001,"5min":0.75600000000000001,"15min":0.75600000000000001},"min":{"1min":0.23000000000000001,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":4.9000000000000004,"5min":4.9000000000000004,"15min":4.9000000000000004},"last":0.32100000000000001},{"interface":"front","average":{"1min":0.77100000000000002,"5min":0.77100000000000002,"15min":0.77100000000000002},"min":{"1min":0.24299999999999999,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":3.444,"5min":3.444,"15min":3.444},"last":0.33400000000000002}]},{"osd":2,"last update":"Sat Sep 23 21:14:12 2023","interfaces":[{"interface":"back","average":{"1min":0.65400000000000003,"5min":0.65400000000000003,"15min":0.65400000000000003},"min":{"1min":0.26900000000000002,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":4.8630000000000004,"5min":4.8630000000000004,"15min":4.8630000000000004},"last":0.34699999999999998},{"interface":"front","average":{"1min":0.96799999999999997,"5min":0.96799999999999997,"15min":0.96799999999999997},"min":{"1min":0.29299999999999998,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":4.649,"5min":4.649,"15min":4.649},"last":0.46700000000000003}]},{"osd":3,"last update":"Sat Sep 23 21:14:12 2023","interfaces":[{"interface":"back","average":{"1min":0.86799999999999999,"5min":0.86799999999999999,"15min":0.86799999999999999},"min":{"1min":0.23400000000000001,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":4.5110000000000001,"5min":4.5110000000000001,"15min":4.5110000000000001},"last":0.307},{"interface":"front","average":{"1min":1.1180000000000001,"5min":1.1180000000000001,"15min":1.1180000000000001},"min":{"1min":0.26200000000000001,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":3.7200000000000002,"5min":3.7200000000000002,"15min":3.7200000000000002},"last":1.9930000000000001}]},{"osd":4,"last update":"Sat Sep 23 21:14:12 2023","interfaces":[{"interface":"back","average":{"1min":1.093,"5min":1.093,"15min":1.093},"min":{"1min":0.30199999999999999,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":3.7799999999999998,"5min":3.7799999999999998,"15min":3.7799999999999998},"last":1.944},{"interface":"front","average":{"1min":0.996,"5min":0.996,"15min":0.996},"min":{"1min":0.317,"5min":0.317,"15min":0.317},"max":{"1min":5.093,"5min":5.093,"15min":5.093},"last":0.98799999999999999}]},{"osd":5,"last update":"Sat Sep 23 21:14:12 2023","interfaces":[{"interface":"back","average":{"1min":1.236,"5min":1.236,"15min":1.236},"min":{"1min":0.33200000000000002,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":4.5640000000000001,"5min":4.5640000000000001,"15min":4.5640000000000001},"last":3.6850000000000001},{"interface":"front","average":{"1min":1.0640000000000001,"5min":1.0640000000000001,"15min":1.0640000000000001},"min":{"1min":0.33200000000000002,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":4.5599999999999996,"5min":4.5599999999999996,"15min":4.5599999999999996},"last":0.442}]},{"osd":6,"last update":"Sat Sep 23 21:14:12 2023","interfaces":[{"interface":"back","average":{"1min":1.127,"5min":1.127,"15min":1.127},"min":{"1min":0.38100000000000001,"5min":0.38100000000000001,"15min":0.38100000000000001},"max":{"1min":6.2750000000000004,"5min":6.2750000000000004,"15min":6.2750000000000004},"last":0.55800000000000005},{"interface":"front","average":{"1min":1.3080000000000001,"5min":1.3080000000000001,"15min":1.3080000000000001},"min":{"1min":0.35599999999999998,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":5.0860000000000003,"5min":5.0860000000000003,"15min":5.0860000000000003},"last":4.7850000000000001}]},{"osd":7,"last update":"Sat Sep 23 21:14:12 2023","interfaces":[{"interface":"back","average":{"1min":1.2090000000000001,"5min":1.2090000000000001,"15min":1.2090000000000001},"min":{"1min":0.41099999999999998,"5min":0.41099999999999998,"15min":0.41099999999999998},"max":{"1min":5.1539999999999999,"5min":5.1539999999999999,"15min":5.1539999999999999},"last":3.8969999999999998},{"interface":"front","average":{"1min":1.1579999999999999,"5min":1.1579999999999999,"15min":1.1579999999999999},"min":{"1min":0.441,"5min":0.441,"15min":0.441},"max":{"1min":5.1289999999999996,"5min":5.1289999999999996,"15min":5.1289999999999996},"last":0.74199999999999999}]}]},{"osd":2,"up_from":93,"seq":399431958819,"num_pgs":69,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1859332,"kb_used_data":1083900,"kb_used_omap":81,"kb_used_meta":775342,"kb_avail":91885820,"statfs":{"total":95995035648,"available":94091079680,"internally_reserved":0,"allocated":1109913600,"data_stored":1107431546,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":83750,"internal_metadata":793950426},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:14:15 2023","interfaces":[{"interface":"back","average":{"1min":0.83299999999999996,"5min":0.83299999999999996,"15min":0.83299999999999996},"min":{"1min":0.20899999999999999,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":3.7290000000000001,"5min":3.7290000000000001,"15min":3.7290000000000001},"last":0.20899999999999999},{"interface":"front","average":{"1min":1.665,"5min":1.665,"15min":1.665},"min":{"1min":0.23499999999999999,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":18.111999999999998,"5min":18.111999999999998,"15min":18.111999999999998},"last":0.49099999999999999}]},{"osd":1,"last update":"Sat Sep 23 21:14:15 2023","interfaces":[{"interface":"back","average":{"1min":0.95099999999999996,"5min":1.474,"15min":2.5339999999999998},"min":{"1min":0.23999999999999999,"5min":0.188,"15min":0.188},"max":{"1min":6.0499999999999998,"5min":12.484999999999999,"15min":34.396999999999998},"last":0.32100000000000001},{"interface":"front","average":{"1min":1.597,"5min":1.3129999999999999,"15min":2.782},"min":{"1min":0.29899999999999999,"5min":0.217,"15min":0.217},"max":{"1min":18.081,"5min":18.081,"15min":34.462000000000003},"last":1.206}]},{"osd":3,"last update":"Sat Sep 23 21:13:56 2023","interfaces":[{"interface":"back","average":{"1min":0.495,"5min":1.5429999999999999,"15min":2.6419999999999999},"min":{"1min":0.20599999999999999,"5min":0.20599999999999999,"15min":0.20599999999999999},"max":{"1min":1.679,"5min":12.951000000000001,"15min":41.057000000000002},"last":0.44700000000000001},{"interface":"front","average":{"1min":0.749,"5min":1.661,"15min":2.915},"min":{"1min":0.28499999999999998,"5min":0.21299999999999999,"15min":0.21299999999999999},"max":{"1min":3.6989999999999998,"5min":13.159000000000001,"15min":41.012},"last":0.28599999999999998}]},{"osd":4,"last update":"Sat Sep 23 21:14:15 2023","interfaces":[{"interface":"back","average":{"1min":0.64800000000000002,"5min":1.256,"15min":1.4219999999999999},"min":{"1min":0.34100000000000003,"5min":0.249,"15min":0.249},"max":{"1min":1.7210000000000001,"5min":10.266999999999999,"15min":10.266999999999999},"last":0.73599999999999999},{"interface":"front","average":{"1min":1.0029999999999999,"5min":1.456,"15min":1.8759999999999999},"min":{"1min":0.38300000000000001,"5min":0.29399999999999998,"15min":0.26700000000000002},"max":{"1min":3.5819999999999999,"5min":8.6780000000000008,"15min":14.039999999999999},"last":4.3810000000000002}]},{"osd":5,"last update":"Sat Sep 23 21:14:15 2023","interfaces":[{"interface":"back","average":{"1min":0.94399999999999995,"5min":1.276,"15min":1.5209999999999999},"min":{"1min":0.311,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":3.3759999999999999,"5min":9.5039999999999996,"15min":10.069000000000001},"last":0.66700000000000004},{"interface":"front","average":{"1min":0.83999999999999997,"5min":1.3149999999999999,"15min":1.8120000000000001},"min":{"1min":0.41499999999999998,"5min":0.29699999999999999,"15min":0.29399999999999998},"max":{"1min":3.629,"5min":8.7270000000000003,"15min":13.98},"last":0.59599999999999997}]},{"osd":6,"last update":"Sat Sep 23 21:14:15 2023","interfaces":[{"interface":"back","average":{"1min":0.94999999999999996,"5min":1.569,"15min":2.0529999999999999},"min":{"1min":0.436,"5min":0.373,"15min":0.35699999999999998},"max":{"1min":3.665,"5min":8.7680000000000007,"15min":14.016},"last":0.46300000000000002},{"interface":"front","average":{"1min":1.2450000000000001,"5min":1.7729999999999999,"15min":2.2330000000000001},"min":{"1min":0.45700000000000002,"5min":0.38100000000000001,"15min":0.35599999999999998},"max":{"1min":3.7610000000000001,"5min":9.3469999999999995,"15min":14.068},"last":2.2789999999999999}]},{"osd":7,"last update":"Sat Sep 23 21:14:15 2023","interfaces":[{"interface":"back","average":{"1min":1.8120000000000001,"5min":1.4790000000000001,"15min":1.5840000000000001},"min":{"1min":0.433,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":18.021999999999998,"5min":18.021999999999998,"15min":18.021999999999998},"last":0.92500000000000004},{"interface":"front","average":{"1min":1.242,"5min":1.476,"15min":1.675},"min":{"1min":0.39400000000000002,"5min":0.34799999999999998,"15min":0.32200000000000001},"max":{"1min":9.6349999999999998,"5min":10.359,"15min":10.92},"last":0.878}]}]},{"osd":3,"up_from":98,"seq":420906795293,"num_pgs":82,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1932192,"kb_used_data":1075736,"kb_used_omap":7,"kb_used_meta":856248,"kb_avail":91812960,"statfs":{"total":95995035648,"available":94016471040,"internally_reserved":0,"allocated":1101553664,"data_stored":1099101233,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7576,"internal_metadata":876798568},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:14:12 2023","interfaces":[{"interface":"back","average":{"1min":0.33800000000000002,"5min":0.33800000000000002,"15min":0.33800000000000002},"min":{"1min":0.20499999999999999,"5min":0.20499999999999999,"15min":0.20499999999999999},"max":{"1min":0.69899999999999995,"5min":0.69899999999999995,"15min":0.69899999999999995},"last":147.38900000000001},{"interface":"front","average":{"1min":0.66800000000000004,"5min":0.66800000000000004,"15min":0.66800000000000004},"min":{"1min":0.23300000000000001,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":4.5510000000000002,"5min":4.5510000000000002,"15min":4.5510000000000002},"last":149.989}]},{"osd":1,"last update":"Sat Sep 23 21:14:12 2023","interfaces":[{"interface":"back","average":{"1min":1.052,"5min":1.466,"15min":2.226},"min":{"1min":0.218,"5min":0.20000000000000001,"15min":0.20000000000000001},"max":{"1min":5.5019999999999998,"5min":22.577999999999999,"15min":22.577999999999999},"last":0.23699999999999999},{"interface":"front","average":{"1min":0.60099999999999998,"5min":1.5580000000000001,"15min":2.1829999999999998},"min":{"1min":0.249,"5min":0.216,"15min":0.216},"max":{"1min":2.7519999999999998,"5min":21.617999999999999,"15min":21.617999999999999},"last":0.26100000000000001}]},{"osd":2,"last update":"Sat Sep 23 21:14:00 2023","interfaces":[{"interface":"back","average":{"1min":3.6669999999999998,"5min":1.9279999999999999,"15min":2.6640000000000001},"min":{"1min":0.221,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":56.317999999999998,"5min":56.317999999999998,"15min":56.317999999999998},"last":0.29299999999999998},{"interface":"front","average":{"1min":3.8479999999999999,"5min":2.1499999999999999,"15min":2.702},"min":{"1min":0.29699999999999999,"5min":0.216,"15min":0.216},"max":{"1min":55.988,"5min":55.988,"15min":55.988},"last":0.32900000000000001}]},{"osd":4,"last update":"Sat Sep 23 21:14:00 2023","interfaces":[{"interface":"back","average":{"1min":0.71499999999999997,"5min":1.3029999999999999,"15min":1.8779999999999999},"min":{"1min":0.30299999999999999,"5min":0.30299999999999999,"15min":0.28399999999999997},"max":{"1min":5.0819999999999999,"5min":21.478999999999999,"15min":21.478999999999999},"last":0.54200000000000004},{"interface":"front","average":{"1min":1.3009999999999999,"5min":1.5760000000000001,"15min":2.0070000000000001},"min":{"1min":0.33100000000000002,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":4.1909999999999998,"5min":22.402000000000001,"15min":22.402000000000001},"last":0.37}]},{"osd":5,"last update":"Sat Sep 23 21:14:00 2023","interfaces":[{"interface":"back","average":{"1min":0.93600000000000005,"5min":1.3120000000000001,"15min":2.012},"min":{"1min":0.32200000000000001,"5min":0.311,"15min":0.311},"max":{"1min":5.04,"5min":21.562000000000001,"15min":21.562000000000001},"last":0.38600000000000001},{"interface":"front","average":{"1min":0.83699999999999997,"5min":1.339,"15min":2.2839999999999998},"min":{"1min":0.34300000000000003,"5min":0.31,"15min":0.31},"max":{"1min":3.4780000000000002,"5min":8.8450000000000006,"15min":14.029},"last":0.35199999999999998}]},{"osd":6,"last update":"Sat Sep 23 21:14:15 2023","interfaces":[{"interface":"back","average":{"1min":1.254,"5min":1.591,"15min":2.0510000000000002},"min":{"1min":0.38900000000000001,"5min":0.34999999999999998,"15min":0.33700000000000002},"max":{"1min":4.2380000000000004,"5min":22.446999999999999,"15min":22.446999999999999},"last":0.42099999999999999},{"interface":"front","average":{"1min":1.2450000000000001,"5min":1.746,"15min":2.278},"min":{"1min":0.44,"5min":0.39000000000000001,"15min":0.373},"max":{"1min":9.3889999999999993,"5min":12.375999999999999,"15min":13.576000000000001},"last":0.39900000000000002}]},{"osd":7,"last update":"Sat Sep 23 21:14:00 2023","interfaces":[{"interface":"back","average":{"1min":1.032,"5min":1.6120000000000001,"15min":2.1299999999999999},"min":{"1min":0.41199999999999998,"5min":0.38400000000000001,"15min":0.33900000000000002},"max":{"1min":5.0090000000000003,"5min":21.518999999999998,"15min":21.518999999999998},"last":2.4689999999999999},{"interface":"front","average":{"1min":0.92900000000000005,"5min":1.7050000000000001,"15min":2.4199999999999999},"min":{"1min":0.45800000000000002,"5min":0.442,"15min":0.38900000000000001},"max":{"1min":3.2679999999999998,"5min":9.9879999999999995,"15min":13.603999999999999},"last":2.456}]}]},{"osd":4,"up_from":31,"seq":133143986599,"num_pgs":77,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1435756,"kb_used_data":978788,"kb_used_omap":0,"kb_used_meta":456960,"kb_avail":92309396,"statfs":{"total":95995035648,"available":94524821504,"internally_reserved":0,"allocated":1002278912,"data_stored":1000739002,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":467927040},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:14:14 2023","interfaces":[{"interface":"back","average":{"1min":0.90000000000000002,"5min":0.90000000000000002,"15min":0.90000000000000002},"min":{"1min":0.22500000000000001,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":2.6539999999999999,"5min":2.6539999999999999,"15min":2.6539999999999999},"last":0.45600000000000002},{"interface":"front","average":{"1min":1.1339999999999999,"5min":1.1339999999999999,"15min":1.1339999999999999},"min":{"1min":0.246,"5min":0.246,"15min":0.246},"max":{"1min":10.945,"5min":10.945,"15min":10.945},"last":1.7829999999999999}]},{"osd":1,"last update":"Sat Sep 23 21:14:14 2023","interfaces":[{"interface":"back","average":{"1min":0.77300000000000002,"5min":1.0609999999999999,"15min":2.7490000000000001},"min":{"1min":0.29699999999999999,"5min":0.221,"15min":0.221},"max":{"1min":2.387,"5min":7.8609999999999998,"15min":37.960000000000001},"last":1.8720000000000001},{"interface":"front","average":{"1min":0.80600000000000005,"5min":1.1339999999999999,"15min":2.6080000000000001},"min":{"1min":0.28499999999999998,"5min":0.28499999999999998,"15min":0.27000000000000002},"max":{"1min":3.581,"5min":7.1349999999999998,"15min":37.933},"last":2.6120000000000001}]},{"osd":2,"last update":"Sat Sep 23 21:14:14 2023","interfaces":[{"interface":"back","average":{"1min":0.82099999999999995,"5min":1.153,"15min":1.748},"min":{"1min":0.27400000000000002,"5min":0.247,"15min":0.247},"max":{"1min":3.476,"5min":10.464,"15min":16.928000000000001},"last":2.5830000000000002},{"interface":"front","average":{"1min":0.86099999999999999,"5min":1.3959999999999999,"15min":1.7809999999999999},"min":{"1min":0.311,"5min":0.28599999999999998,"15min":0.27800000000000002},"max":{"1min":3.5129999999999999,"5min":7.3659999999999997,"15min":14.417999999999999},"last":2.7069999999999999}]},{"osd":3,"last update":"Sat Sep 23 21:13:46 2023","interfaces":[{"interface":"back","average":{"1min":1.0229999999999999,"5min":1.6299999999999999,"15min":2.1339999999999999},"min":{"1min":0.25600000000000001,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":4.2480000000000002,"5min":14.585000000000001,"15min":15.01},"last":1.8260000000000001},{"interface":"front","average":{"1min":1.135,"5min":1.647,"15min":2.0249999999999999},"min":{"1min":0.36699999999999999,"5min":0.36099999999999999,"15min":0.30199999999999999},"max":{"1min":4.4649999999999999,"5min":14.366,"15min":15.004},"last":2.7269999999999999}]},{"osd":5,"last update":"Sat Sep 23 21:14:04 2023","interfaces":[{"interface":"back","average":{"1min":1.169,"5min":1.018,"15min":0.97599999999999998},"min":{"1min":0.30099999999999999,"5min":0.222,"15min":0.222},"max":{"1min":5.1609999999999996,"5min":9.4909999999999997,"15min":14.298},"last":2.637},{"interface":"front","average":{"1min":1.0189999999999999,"5min":0.95899999999999996,"15min":1.0760000000000001},"min":{"1min":0.28000000000000003,"5min":0.27200000000000002,"15min":0.255},"max":{"1min":3.0059999999999998,"5min":6.8730000000000002,"15min":9.8979999999999997},"last":0.42799999999999999}]},{"osd":6,"last update":"Sat Sep 23 21:14:14 2023","interfaces":[{"interface":"back","average":{"1min":0.747,"5min":1.0920000000000001,"15min":1.2230000000000001},"min":{"1min":0.317,"5min":0.28599999999999998,"15min":0.26800000000000002},"max":{"1min":2.8650000000000002,"5min":24.452999999999999,"15min":24.452999999999999},"last":1.851},{"interface":"front","average":{"1min":0.95099999999999996,"5min":1.036,"15min":0.95399999999999996},"min":{"1min":0.40100000000000002,"5min":0.33500000000000002,"15min":0.32500000000000001},"max":{"1min":3.6600000000000001,"5min":11.409000000000001,"15min":14.387},"last":2.6619999999999999}]},{"osd":7,"last update":"Sat Sep 23 21:13:46 2023","interfaces":[{"interface":"back","average":{"1min":1.294,"5min":1.268,"15min":1.0529999999999999},"min":{"1min":0.375,"5min":0.30099999999999999,"15min":0.29299999999999998},"max":{"1min":4.532,"5min":16.890999999999998,"15min":16.890999999999998},"last":2.6850000000000001},{"interface":"front","average":{"1min":1.387,"5min":1.155,"15min":1.1559999999999999},"min":{"1min":0.375,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":7.1879999999999997,"5min":8.5830000000000002,"15min":11.170999999999999},"last":0.47699999999999998}]}]},{"osd":5,"up_from":39,"seq":167503724965,"num_pgs":61,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1268760,"kb_used_data":771664,"kb_used_omap":0,"kb_used_meta":497088,"kb_avail":92476392,"statfs":{"total":95995035648,"available":94695825408,"internally_reserved":0,"allocated":790183936,"data_stored":788660216,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":509018112},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:14:13 2023","interfaces":[{"interface":"back","average":{"1min":0.71499999999999997,"5min":0.71499999999999997,"15min":0.71499999999999997},"min":{"1min":0.251,"5min":0.251,"15min":0.251},"max":{"1min":3.3889999999999998,"5min":3.3889999999999998,"15min":3.3889999999999998},"last":1.9450000000000001},{"interface":"front","average":{"1min":0.66900000000000004,"5min":0.66900000000000004,"15min":0.66900000000000004},"min":{"1min":0.22800000000000001,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":3.1150000000000002,"5min":3.1150000000000002,"15min":3.1150000000000002},"last":3.169}]},{"osd":1,"last update":"Sat Sep 23 21:14:08 2023","interfaces":[{"interface":"back","average":{"1min":0.65200000000000002,"5min":1.1479999999999999,"15min":1.728},"min":{"1min":0.248,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":3.109,"5min":9.4429999999999996,"15min":25.239999999999998},"last":0.43099999999999999},{"interface":"front","average":{"1min":1.468,"5min":1.681,"15min":1.784},"min":{"1min":0.34999999999999998,"5min":0.28699999999999998,"15min":0.252},"max":{"1min":11.116,"5min":11.116,"15min":25.253},"last":1.978}]},{"osd":2,"last update":"Sat Sep 23 21:14:26 2023","interfaces":[{"interface":"back","average":{"1min":0.73099999999999998,"5min":1.401,"15min":1.7110000000000001},"min":{"1min":0.27800000000000002,"5min":0.26700000000000002,"15min":0.23499999999999999},"max":{"1min":4.0490000000000004,"5min":9.2889999999999997,"15min":12.634},"last":3.2490000000000001},{"interface":"front","average":{"1min":0.78600000000000003,"5min":1.222,"15min":1.762},"min":{"1min":0.30199999999999999,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":2.524,"5min":9.7170000000000005,"15min":10.333},"last":0.39100000000000001}]},{"osd":3,"last update":"Sat Sep 23 21:13:45 2023","interfaces":[{"interface":"back","average":{"1min":1.4279999999999999,"5min":1.8089999999999999,"15min":2.6560000000000001},"min":{"1min":0.33400000000000002,"5min":0.307,"15min":0.29299999999999998},"max":{"1min":6.1859999999999999,"5min":14.242000000000001,"15min":61.667999999999999},"last":2.0059999999999998},{"interface":"front","average":{"1min":1.4390000000000001,"5min":2.0089999999999999,"15min":2.6429999999999998},"min":{"1min":0.34899999999999998,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":6.1200000000000001,"5min":14.396000000000001,"15min":62.276000000000003},"last":3.2770000000000001}]},{"osd":4,"last update":"Sat Sep 23 21:13:59 2023","interfaces":[{"interface":"back","average":{"1min":1.452,"5min":1.387,"15min":1.139},"min":{"1min":0.28799999999999998,"5min":0.27000000000000002,"15min":0.23400000000000001},"max":{"1min":11.156000000000001,"5min":12.776,"15min":12.776},"last":2.0270000000000001},{"interface":"front","average":{"1min":1.5109999999999999,"5min":1.1799999999999999,"15min":1.1319999999999999},"min":{"1min":0.28000000000000003,"5min":0.27000000000000002,"15min":0.26900000000000002},"max":{"1min":9.7119999999999997,"5min":9.7119999999999997,"15min":11.973000000000001},"last":3.2109999999999999}]},{"osd":6,"last update":"Sat Sep 23 21:14:08 2023","interfaces":[{"interface":"back","average":{"1min":0.625,"5min":0.92200000000000004,"15min":1.056},"min":{"1min":0.38800000000000001,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":1.448,"5min":7.7030000000000003,"15min":11.128},"last":3.7170000000000001},{"interface":"front","average":{"1min":1.4350000000000001,"5min":1.4390000000000001,"15min":1.246},"min":{"1min":0.42099999999999999,"5min":0.32400000000000001,"15min":0.317},"max":{"1min":10.999000000000001,"5min":10.999000000000001,"15min":11.112},"last":3.738}]},{"osd":7,"last update":"Sat Sep 23 21:13:45 2023","interfaces":[{"interface":"back","average":{"1min":1.3,"5min":0.95099999999999996,"15min":1.804},"min":{"1min":0.34300000000000003,"5min":0.31,"15min":0.309},"max":{"1min":5.5659999999999998,"5min":5.5659999999999998,"15min":189.405},"last":0.46300000000000002},{"interface":"front","average":{"1min":2.004,"5min":1.5009999999999999,"15min":1.9159999999999999},"min":{"1min":0.40200000000000002,"5min":0.35399999999999998,"15min":0.32500000000000001},"max":{"1min":11.183999999999999,"5min":11.183999999999999,"15min":189.685},"last":3.2949999999999999}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8488,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1259,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5322,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":714,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":5434,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":4838,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1903493120,"data_stored":1903493120,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1197604864,"data_stored":1197604864,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1096155136,"data_stored":1096155136,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1092358144,"data_stored":1092354072,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":995561472,"data_stored":995557400,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":772079616,"data_stored":772079616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":606011392,"data_stored":606011392,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T21:14:46.038 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:14:46.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:14:45 smithi070 ceph-mon[168282]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:14:46.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:14:45 smithi070 ceph-mon[168282]: failed to encode map e530 with expected crc 2023-09-23T21:14:46.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:14:45 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T21:14:46.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:14:45 smithi070 ceph-mon[168282]: osdmap e530: 8 total, 8 up, 6 in 2023-09-23T21:14:46.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:14:45 smithi070 ceph-mon[168282]: failed to encode map e530 with expected crc 2023-09-23T21:14:46.121 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:14:45 smithi070 ceph-mon[168282]: failed to encode map e530 with expected crc 2023-09-23T21:14:46.121 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:14:45 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/241882249' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:14:46.126 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:14:46.385 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:14:46.399 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:14:46.399 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:14:46.500 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:14:46.546 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:14:46.625 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:14:46.628 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:14:46.629 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:14:46.729 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:14:46.800 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:14:46.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:14:46 smithi070 ceph-mon[168282]: from='client.56476 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T21:14:46.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:14:46 smithi070 ceph-mon[168282]: pgmap v1338: 129 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 15 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 108 active+clean; 1.5 GiB data, 9.2 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 701 op/s; 22120/74358 objects degraded (29.748%); 3570/74358 objects misplaced (4.801%); 9.4 MiB/s, 149 objects/s recovering 2023-09-23T21:14:46.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:14:46 smithi070 ceph-mon[168282]: Health check update: Degraded data redundancy: 22120/74358 objects degraded (29.748%), 19 pgs degraded, 14 pgs undersized (PG_DEGRADED) 2023-09-23T21:14:46.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:14:46 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:14:46.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:14:46 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:14:46.878 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T21:14:46.881 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:14:46 smithi026 ceph-mon[179688]: from='client.56476 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T21:14:46.881 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:14:46 smithi026 ceph-mon[179688]: pgmap v1338: 129 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 15 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 108 active+clean; 1.5 GiB data, 9.2 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 701 op/s; 22120/74358 objects degraded (29.748%); 3570/74358 objects misplaced (4.801%); 9.4 MiB/s, 149 objects/s recovering 2023-09-23T21:14:46.881 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:14:46 smithi026 ceph-mon[179688]: Health check update: Degraded data redundancy: 22120/74358 objects degraded (29.748%), 19 pgs degraded, 14 pgs undersized (PG_DEGRADED) 2023-09-23T21:14:46.881 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:14:46 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:14:46.881 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:14:46 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:14:46.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:14:46 smithi026 ceph-mon[175151]: from='client.56476 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T21:14:46.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:14:46 smithi026 ceph-mon[175151]: pgmap v1338: 129 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 15 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 108 active+clean; 1.5 GiB data, 9.2 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 701 op/s; 22120/74358 objects degraded (29.748%); 3570/74358 objects misplaced (4.801%); 9.4 MiB/s, 149 objects/s recovering 2023-09-23T21:14:46.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:14:46 smithi026 ceph-mon[175151]: Health check update: Degraded data redundancy: 22120/74358 objects degraded (29.748%), 19 pgs degraded, 14 pgs undersized (PG_DEGRADED) 2023-09-23T21:14:46.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:14:46 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:14:46.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:14:46 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:14:46.888 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:14:46.888 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:14:46.989 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:14:47.052 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:14:47.117 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:14:47.125 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:14:47.125 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:14:47.226 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:14:47.278 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:14:47.328 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:14:47.336 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:14:47.337 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:14:47.438 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:14:47.490 INFO:teuthology.orchestra.run.smithi026.stdout:{ 2023-09-23T21:14:47.491 INFO:teuthology.orchestra.run.smithi026.stdout: "ops": [], 2023-09-23T21:14:47.492 INFO:teuthology.orchestra.run.smithi026.stdout: "complaint_time": 30, 2023-09-23T21:14:47.492 INFO:teuthology.orchestra.run.smithi026.stdout: "num_blocked_ops": 0 2023-09-23T21:14:47.492 INFO:teuthology.orchestra.run.smithi026.stdout:} 2023-09-23T21:14:47.492 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:14:47.543 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:14:47.551 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:14:47.552 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:14:47.653 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:14:47.709 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:14:47.724 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T21:14:47.910 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T21:14:47.920 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:14:47.921 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:14:47.986 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_historic_ops 2023-09-23T21:14:48.022 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:14:48.074 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:14:48.138 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:14:48.141 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:14:48.141 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:14:48.242 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:14:48.314 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:14:48.398 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:14:48.402 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:14:48.402 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:14:48.478 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T21:14:48.503 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:14:48.545 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:14:48.556 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:14:48.598 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:14:48.604 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:14:48.605 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:14:48.705 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:14:48.813 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:14:48.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:14:48 smithi070 ceph-mon[168282]: pgmap v1339: 129 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 15 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 108 active+clean; 1.4 GiB data, 8.9 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 720 op/s; 22120/70104 objects degraded (31.553%); 3466/70104 objects misplaced (4.944%); 9.4 MiB/s, 149 objects/s recovering 2023-09-23T21:14:48.887 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T21:14:48.892 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:14:48.892 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:14:48.993 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:14:49.039 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T21:14:49.040 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:14:48 smithi026 ceph-mon[175151]: pgmap v1339: 129 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 15 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 108 active+clean; 1.4 GiB data, 8.9 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 720 op/s; 22120/70104 objects degraded (31.553%); 3466/70104 objects misplaced (4.944%); 9.4 MiB/s, 149 objects/s recovering 2023-09-23T21:14:49.040 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:14:48 smithi026 ceph-mon[179688]: pgmap v1339: 129 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 15 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 108 active+clean; 1.4 GiB data, 8.9 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 720 op/s; 22120/70104 objects degraded (31.553%); 3466/70104 objects misplaced (4.944%); 9.4 MiB/s, 149 objects/s recovering 2023-09-23T21:14:49.043 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:14:49.091 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:14:49.099 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:14:49.107 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:14:49.107 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:14:49.208 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:14:49.314 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:14:49.663 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:14:49.674 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:14:49.674 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:14:49.747 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:14:49.775 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:14:49.821 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:14:49.886 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 21:14:49 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[164029]: ::ffff:172.21.15.70 - - [23/Sep/2023:21:14:49] "GET /metrics HTTP/1.1" 200 36397 "" "Prometheus/2.43.0" 2023-09-23T21:14:49.890 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:14:49.895 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:14:49.896 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:14:49.996 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:14:50.065 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:14:50.140 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:14:50.146 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:14:50.146 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:14:50.247 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:14:50.308 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:14:50.372 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:14:50.380 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:14:50.381 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:14:50.482 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:14:50.549 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:14:50.785 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:14:50.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:14:50 smithi026 ceph-mon[179688]: pgmap v1340: 129 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 15 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 108 active+clean; 1.3 GiB data, 8.7 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 735 op/s; 21439/64914 objects degraded (33.027%); 3366/64914 objects misplaced (5.185%); 10 MiB/s, 167 objects/s recovering 2023-09-23T21:14:50.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:14:50 smithi026 ceph-mon[175151]: pgmap v1340: 129 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 15 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 108 active+clean; 1.3 GiB data, 8.7 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 735 op/s; 21439/64914 objects degraded (33.027%); 3366/64914 objects misplaced (5.185%); 10 MiB/s, 167 objects/s recovering 2023-09-23T21:14:51.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:14:50 smithi070 ceph-mon[168282]: pgmap v1340: 129 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 15 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 108 active+clean; 1.3 GiB data, 8.7 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 735 op/s; 21439/64914 objects degraded (33.027%); 3366/64914 objects misplaced (5.185%); 10 MiB/s, 167 objects/s recovering 2023-09-23T21:14:51.239 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:14:51.245 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:14:51.246 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:14:51.347 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:14:51.416 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:14:51.849 INFO:teuthology.orchestra.run.smithi026.stderr:noscrub is unset 2023-09-23T21:14:51.996 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:14:51 smithi026 ceph-mon[175151]: Health check update: Degraded data redundancy: 21439/64914 objects degraded (33.027%), 19 pgs degraded, 16 pgs undersized (PG_DEGRADED) 2023-09-23T21:14:51.997 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:14:51 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/287709091' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T21:14:51.997 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:14:51 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T21:14:51.997 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:14:51 smithi026 ceph-mon[179688]: Health check update: Degraded data redundancy: 21439/64914 objects degraded (33.027%), 19 pgs degraded, 16 pgs undersized (PG_DEGRADED) 2023-09-23T21:14:51.997 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:14:51 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/287709091' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T21:14:51.997 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:14:51 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T21:14:52.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:14:51 smithi070 ceph-mon[168282]: Health check update: Degraded data redundancy: 21439/64914 objects degraded (33.027%), 19 pgs degraded, 16 pgs undersized (PG_DEGRADED) 2023-09-23T21:14:52.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:14:51 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/287709091' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T21:14:52.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:14:51 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T21:14:52.822 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:14:52.826 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:14:52.827 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:14:52.928 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:14:52.976 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:14:53.028 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:14:53.031 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:14:52 smithi070 ceph-mon[168282]: pgmap v1341: 129 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 15 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 108 active+clean; 1.3 GiB data, 8.5 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 745 op/s; 21439/62058 objects degraded (34.547%); 3366/62058 objects misplaced (5.424%); 10 MiB/s, 167 objects/s recovering 2023-09-23T21:14:53.031 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:14:52 smithi070 ceph-mon[168282]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:14:53.032 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:14:52 smithi070 ceph-mon[168282]: failed to encode map e531 with expected crc 2023-09-23T21:14:53.032 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:14:52 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T21:14:53.032 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:14:52 smithi070 ceph-mon[168282]: osdmap e531: 8 total, 8 up, 6 in 2023-09-23T21:14:53.032 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:14:52 smithi070 ceph-mon[168282]: failed to encode map e531 with expected crc 2023-09-23T21:14:53.032 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:14:52 smithi070 ceph-mon[168282]: failed to encode map e531 with expected crc 2023-09-23T21:14:53.032 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:14:53.033 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:14:53.041 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:14:52 smithi026 ceph-mon[179688]: pgmap v1341: 129 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 15 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 108 active+clean; 1.3 GiB data, 8.5 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 745 op/s; 21439/62058 objects degraded (34.547%); 3366/62058 objects misplaced (5.424%); 10 MiB/s, 167 objects/s recovering 2023-09-23T21:14:53.041 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:14:52 smithi026 ceph-mon[179688]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:14:53.041 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:14:52 smithi026 ceph-mon[179688]: failed to encode map e531 with expected crc 2023-09-23T21:14:53.041 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:14:52 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T21:14:53.041 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:14:52 smithi026 ceph-mon[179688]: osdmap e531: 8 total, 8 up, 6 in 2023-09-23T21:14:53.041 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:14:52 smithi026 ceph-mon[179688]: failed to encode map e531 with expected crc 2023-09-23T21:14:53.041 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:14:52 smithi026 ceph-mon[179688]: failed to encode map e531 with expected crc 2023-09-23T21:14:53.041 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:14:52 smithi026 ceph-mon[175151]: pgmap v1341: 129 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 15 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 108 active+clean; 1.3 GiB data, 8.5 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 745 op/s; 21439/62058 objects degraded (34.547%); 3366/62058 objects misplaced (5.424%); 10 MiB/s, 167 objects/s recovering 2023-09-23T21:14:53.041 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:14:52 smithi026 ceph-mon[175151]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:14:53.042 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:14:52 smithi026 ceph-mon[175151]: failed to encode map e531 with expected crc 2023-09-23T21:14:53.042 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:14:52 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T21:14:53.042 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:14:52 smithi026 ceph-mon[175151]: osdmap e531: 8 total, 8 up, 6 in 2023-09-23T21:14:53.042 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:14:52 smithi026 ceph-mon[175151]: failed to encode map e531 with expected crc 2023-09-23T21:14:53.042 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:14:52 smithi026 ceph-mon[175151]: failed to encode map e531 with expected crc 2023-09-23T21:14:53.065 INFO:teuthology.orchestra.run.smithi026.stdout:{ 2023-09-23T21:14:53.066 INFO:teuthology.orchestra.run.smithi026.stdout: "size": 20, 2023-09-23T21:14:53.066 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 600, 2023-09-23T21:14:53.066 INFO:teuthology.orchestra.run.smithi026.stdout: "ops": [ 2023-09-23T21:14:53.066 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:14:53.066 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.46328.0:99980 10.18 10.aa304a38 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e493)", 2023-09-23T21:14:53.066 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T21:13:11.863061+0000", 2023-09-23T21:14:53.066 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 101.195084815, 2023-09-23T21:14:53.066 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.32472362500000002, 2023-09-23T21:14:53.066 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:14:53.066 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "reached pg", 2023-09-23T21:14:53.067 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T21:14:53.067 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.46328", 2023-09-23T21:14:53.067 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/3748861181", 2023-09-23T21:14:53.067 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 99980 2023-09-23T21:14:53.067 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:14:53.067 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:14:53.067 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:14:53.067 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:14:53.067 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:11.863061+0000", 2023-09-23T21:14:53.067 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:14:53.067 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:14:53.067 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:14:53.067 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:14:53.068 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:11.863061+0000", 2023-09-23T21:14:53.068 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:14:53.068 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:14:53.068 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:14:53.068 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:14:53.068 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:11.863059+0000", 2023-09-23T21:14:53.068 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T21:14:53.068 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:14:53.068 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:14:53.068 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:14:53.068 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:11.863068+0000", 2023-09-23T21:14:53.068 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.3340000000000002e-06 2023-09-23T21:14:53.069 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:14:53.069 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:14:53.069 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:14:53.069 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:11.863078+0000", 2023-09-23T21:14:53.069 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0479000000000001e-05 2023-09-23T21:14:53.069 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:14:53.069 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:14:53.069 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:14:53.069 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:11.863089+0000", 2023-09-23T21:14:53.069 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0808000000000001e-05 2023-09-23T21:14:53.069 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:14:53.069 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:14:53.070 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:14:53.070 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:11.863166+0000", 2023-09-23T21:14:53.070 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.7020000000000005e-05 2023-09-23T21:14:53.070 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:14:53.070 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:14:53.070 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T21:14:53.070 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:11.863177+0000", 2023-09-23T21:14:53.070 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0679999999999999e-05 2023-09-23T21:14:53.070 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:14:53.070 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:14:53.070 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:14:53.070 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:12.187773+0000", 2023-09-23T21:14:53.071 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.32459634900000001 2023-09-23T21:14:53.071 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:14:53.071 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:14:53.071 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T21:14:53.071 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:12.187784+0000", 2023-09-23T21:14:53.071 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.134e-05 2023-09-23T21:14:53.071 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:14:53.071 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:14:53.071 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:14:53.071 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:14:53.071 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:14:53.071 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.46328.0:99982 10.8 10.3f3e48e8 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e493)", 2023-09-23T21:14:53.071 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T21:13:11.871168+0000", 2023-09-23T21:14:53.072 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 101.186977191, 2023-09-23T21:14:53.072 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.31668655600000001, 2023-09-23T21:14:53.072 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:14:53.072 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "reached pg", 2023-09-23T21:14:53.072 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T21:14:53.072 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.46328", 2023-09-23T21:14:53.072 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/3748861181", 2023-09-23T21:14:53.072 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 99982 2023-09-23T21:14:53.072 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:14:53.072 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:14:53.072 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:14:53.072 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:14:53.072 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:11.871168+0000", 2023-09-23T21:14:53.073 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:14:53.073 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:14:53.073 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:14:53.073 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:14:53.073 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:11.871168+0000", 2023-09-23T21:14:53.073 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:14:53.073 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:14:53.073 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:14:53.073 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:14:53.073 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:11.871166+0000", 2023-09-23T21:14:53.073 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999976 2023-09-23T21:14:53.073 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:14:53.074 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:14:53.074 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:14:53.074 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:11.871177+0000", 2023-09-23T21:14:53.074 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1221e-05 2023-09-23T21:14:53.074 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:14:53.074 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:14:53.074 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:14:53.074 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:11.871200+0000", 2023-09-23T21:14:53.074 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.3102999999999999e-05 2023-09-23T21:14:53.074 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:14:53.074 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:14:53.074 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:14:53.075 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:11.871211+0000", 2023-09-23T21:14:53.075 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0624e-05 2023-09-23T21:14:53.075 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:14:53.075 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:14:53.075 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:14:53.075 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:11.871265+0000", 2023-09-23T21:14:53.075 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.3804000000000002e-05 2023-09-23T21:14:53.075 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:14:53.075 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:14:53.075 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T21:14:53.075 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:11.871276+0000", 2023-09-23T21:14:53.075 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1178e-05 2023-09-23T21:14:53.075 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:14:53.076 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:14:53.076 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:14:53.076 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:12.187844+0000", 2023-09-23T21:14:53.076 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.31656841699999999 2023-09-23T21:14:53.076 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:14:53.076 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:14:53.076 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T21:14:53.076 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:12.187855+0000", 2023-09-23T21:14:53.076 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.048e-05 2023-09-23T21:14:53.076 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:14:53.076 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:14:53.076 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:14:53.076 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:14:53.077 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:14:53.077 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.46328.0:99972 10.6 10:650fbbdc:::benchmark_data_smithi026_379838_object20015:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e494)", 2023-09-23T21:14:53.077 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T21:13:12.188907+0000", 2023-09-23T21:14:53.077 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 100.869238446, 2023-09-23T21:14:53.077 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.267026682, 2023-09-23T21:14:53.077 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:14:53.077 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:14:53.077 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T21:14:53.077 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.46328", 2023-09-23T21:14:53.077 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/3748861181", 2023-09-23T21:14:53.077 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 99972 2023-09-23T21:14:53.077 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:14:53.078 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:14:53.078 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:14:53.078 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:14:53.078 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:12.188907+0000", 2023-09-23T21:14:53.078 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:14:53.078 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:14:53.078 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:14:53.078 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:14:53.078 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:12.188907+0000", 2023-09-23T21:14:53.078 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:14:53.078 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:14:53.078 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:14:53.079 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:14:53.079 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:12.188906+0000", 2023-09-23T21:14:53.079 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.999999 2023-09-23T21:14:53.079 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:14:53.079 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:14:53.079 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:14:53.079 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:12.188911+0000", 2023-09-23T21:14:53.079 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.8790000000000002e-06 2023-09-23T21:14:53.079 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:14:53.079 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:14:53.079 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:14:53.079 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:12.188921+0000", 2023-09-23T21:14:53.079 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.8789999999999998e-06 2023-09-23T21:14:53.080 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:14:53.080 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:14:53.080 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:14:53.080 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:12.188931+0000", 2023-09-23T21:14:53.080 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0287e-05 2023-09-23T21:14:53.080 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:14:53.080 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:14:53.080 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:14:53.080 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:12.200174+0000", 2023-09-23T21:14:53.080 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.011242722 2023-09-23T21:14:53.080 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:14:53.080 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:14:53.080 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T21:14:53.081 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:12.200187+0000", 2023-09-23T21:14:53.081 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.3198e-05 2023-09-23T21:14:53.081 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:14:53.081 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:14:53.081 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:14:53.081 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:14.419319+0000", 2023-09-23T21:14:53.081 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.2191318880000002 2023-09-23T21:14:53.081 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:14:53.081 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:14:53.081 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for degraded object", 2023-09-23T21:14:53.081 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:14.422985+0000", 2023-09-23T21:14:53.081 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00366571 2023-09-23T21:14:53.082 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:14:53.082 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:14:53.082 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:14:53.082 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:14.433515+0000", 2023-09-23T21:14:53.082 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.010530688999999999 2023-09-23T21:14:53.082 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:14:53.082 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:14:53.082 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:14:53.082 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:14.434736+0000", 2023-09-23T21:14:53.082 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.001220383 2023-09-23T21:14:53.082 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:14:53.082 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:14:53.083 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 4,5", 2023-09-23T21:14:53.083 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:14.436852+0000", 2023-09-23T21:14:53.083 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.002116845 2023-09-23T21:14:53.083 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:14:53.083 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:14:53.083 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T21:14:53.083 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:14.451217+0000", 2023-09-23T21:14:53.083 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.014364965 2023-09-23T21:14:53.083 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:14:53.083 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:14:53.083 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:14:53.083 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:14.451640+0000", 2023-09-23T21:14:53.084 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00042237600000000002 2023-09-23T21:14:53.084 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:14:53.084 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:14:53.084 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:14:53.084 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:14.452035+0000", 2023-09-23T21:14:53.084 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00039501800000000002 2023-09-23T21:14:53.084 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:14:53.084 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:14:53.084 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T21:14:53.084 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:14.452663+0000", 2023-09-23T21:14:53.084 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00062800300000000002 2023-09-23T21:14:53.084 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:14:53.084 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:14:53.085 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T21:14:53.085 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:14.455934+0000", 2023-09-23T21:14:53.085 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.003270893 2023-09-23T21:14:53.085 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:14:53.085 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:14:53.085 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:14:53.085 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:14:53.085 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:14:53.085 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.46328.0:99999 10.6 10:641f61df:::benchmark_data_smithi026_379838_object20042:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e495)", 2023-09-23T21:14:53.085 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T21:13:13.453310+0000", 2023-09-23T21:14:53.085 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 99.604835473999998, 2023-09-23T21:14:53.085 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0071024850000001, 2023-09-23T21:14:53.086 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:14:53.086 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:14:53.086 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T21:14:53.086 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.46328", 2023-09-23T21:14:53.086 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/3748861181", 2023-09-23T21:14:53.086 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 99999 2023-09-23T21:14:53.086 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:14:53.086 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:14:53.086 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:14:53.086 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:14:53.086 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:13.453310+0000", 2023-09-23T21:14:53.086 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:14:53.086 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:14:53.087 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:14:53.087 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:14:53.087 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:13.453310+0000", 2023-09-23T21:14:53.087 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:14:53.087 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:14:53.087 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:14:53.087 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:14:53.087 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:13.453308+0000", 2023-09-23T21:14:53.087 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999981 2023-09-23T21:14:53.087 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:14:53.087 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:14:53.087 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:14:53.088 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:13.453318+0000", 2023-09-23T21:14:53.088 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0453e-05 2023-09-23T21:14:53.088 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:14:53.088 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:14:53.088 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:14:53.088 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:13.453333+0000", 2023-09-23T21:14:53.088 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.4979e-05 2023-09-23T21:14:53.088 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:14:53.088 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:14:53.088 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:14:53.088 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:13.453344+0000", 2023-09-23T21:14:53.088 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0560999999999999e-05 2023-09-23T21:14:53.089 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:14:53.089 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:14:53.089 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:14:53.089 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:14.423175+0000", 2023-09-23T21:14:53.089 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.96983093399999998 2023-09-23T21:14:53.089 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:14:53.089 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:14:53.089 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for degraded object", 2023-09-23T21:14:53.089 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:14.426649+0000", 2023-09-23T21:14:53.089 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0034738199999999999 2023-09-23T21:14:53.089 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:14:53.089 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:14:53.089 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:14:53.090 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:14.445239+0000", 2023-09-23T21:14:53.090 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.018589859 2023-09-23T21:14:53.090 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:14:53.090 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:14:53.090 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:14:53.090 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:14.446367+0000", 2023-09-23T21:14:53.090 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0011285240000000001 2023-09-23T21:14:53.090 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:14:53.090 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:14:53.090 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 4,5", 2023-09-23T21:14:53.090 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:14.448530+0000", 2023-09-23T21:14:53.090 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.002163257 2023-09-23T21:14:53.091 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:14:53.091 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:14:53.091 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T21:14:53.091 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:14.456072+0000", 2023-09-23T21:14:53.091 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0075418560000000004 2023-09-23T21:14:53.091 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:14:53.091 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:14:53.091 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:14:53.091 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:14.456530+0000", 2023-09-23T21:14:53.091 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00045766099999999999 2023-09-23T21:14:53.091 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:14:53.091 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:14:53.092 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:14:53.092 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:14.457026+0000", 2023-09-23T21:14:53.092 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00049629700000000004 2023-09-23T21:14:53.092 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:14:53.092 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:14:53.092 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T21:14:53.092 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:14.457752+0000", 2023-09-23T21:14:53.092 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00072537999999999997 2023-09-23T21:14:53.092 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:14:53.092 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:14:53.092 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T21:14:53.092 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:14.460413+0000", 2023-09-23T21:14:53.093 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.002660974 2023-09-23T21:14:53.093 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:14:53.093 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:14:53.093 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:14:53.093 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:14:53.093 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:14:53.093 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.46328.0:105700 10.a 10.d45319ea (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e501)", 2023-09-23T21:14:53.093 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T21:13:30.377651+0000", 2023-09-23T21:14:53.093 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 82.680494827999993, 2023-09-23T21:14:53.093 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.37983421699999997, 2023-09-23T21:14:53.093 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:14:53.093 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "reached pg", 2023-09-23T21:14:53.093 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T21:14:53.094 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.46328", 2023-09-23T21:14:53.094 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/3748861181", 2023-09-23T21:14:53.094 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 105700 2023-09-23T21:14:53.094 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:14:53.094 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:14:53.094 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:14:53.094 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:14:53.094 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:30.377651+0000", 2023-09-23T21:14:53.094 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:14:53.094 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:14:53.094 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:14:53.094 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:14:53.095 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:30.377651+0000", 2023-09-23T21:14:53.095 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:14:53.095 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:14:53.095 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:14:53.095 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:14:53.095 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:30.377649+0000", 2023-09-23T21:14:53.095 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T21:14:53.095 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:14:53.095 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:14:53.095 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:14:53.095 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:30.377656+0000", 2023-09-23T21:14:53.095 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.4030000000000004e-06 2023-09-23T21:14:53.096 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:14:53.096 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:14:53.096 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:14:53.096 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:30.377673+0000", 2023-09-23T21:14:53.096 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.7438000000000001e-05 2023-09-23T21:14:53.096 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:14:53.096 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:14:53.096 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:14:53.096 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:30.377683+0000", 2023-09-23T21:14:53.096 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.6889999999999999e-06 2023-09-23T21:14:53.096 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:14:53.096 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:14:53.097 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:14:53.097 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:30.379823+0000", 2023-09-23T21:14:53.097 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0021400099999999999 2023-09-23T21:14:53.097 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:14:53.097 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:14:53.097 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T21:14:53.097 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:30.379836+0000", 2023-09-23T21:14:53.097 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.3652000000000001e-05 2023-09-23T21:14:53.097 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:14:53.097 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:14:53.097 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:14:53.097 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:30.757463+0000", 2023-09-23T21:14:53.098 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.37762642200000002 2023-09-23T21:14:53.098 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:14:53.098 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:14:53.098 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T21:14:53.098 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:30.757485+0000", 2023-09-23T21:14:53.098 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.2050999999999999e-05 2023-09-23T21:14:53.098 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:14:53.098 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:14:53.098 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:14:53.098 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:14:53.098 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:14:53.098 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.46328.0:105702 10.a 10.26b86e5a (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e501)", 2023-09-23T21:14:53.099 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T21:13:30.388674+0000", 2023-09-23T21:14:53.099 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 82.669471834999996, 2023-09-23T21:14:53.099 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.36884504800000001, 2023-09-23T21:14:53.099 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:14:53.099 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "reached pg", 2023-09-23T21:14:53.099 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T21:14:53.099 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.46328", 2023-09-23T21:14:53.099 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/3748861181", 2023-09-23T21:14:53.099 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 105702 2023-09-23T21:14:53.099 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:14:53.099 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:14:53.099 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:14:53.100 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:14:53.101 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:30.388674+0000", 2023-09-23T21:14:53.101 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:14:53.101 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:14:53.101 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:14:53.101 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:14:53.101 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:30.388674+0000", 2023-09-23T21:14:53.101 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:14:53.101 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:14:53.102 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:14:53.102 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:14:53.102 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:30.388672+0000", 2023-09-23T21:14:53.102 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T21:14:53.102 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:14:53.102 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:14:53.102 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:14:53.102 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:30.388680+0000", 2023-09-23T21:14:53.102 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.4399999999999999e-06 2023-09-23T21:14:53.102 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:14:53.102 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:14:53.103 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:14:53.103 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:30.388690+0000", 2023-09-23T21:14:53.103 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0916e-05 2023-09-23T21:14:53.103 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:14:53.103 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:14:53.103 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:14:53.103 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:30.388701+0000", 2023-09-23T21:14:53.103 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0995000000000001e-05 2023-09-23T21:14:53.103 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:14:53.103 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:14:53.103 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:14:53.103 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:30.388769+0000", 2023-09-23T21:14:53.103 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.7804000000000003e-05 2023-09-23T21:14:53.104 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:14:53.104 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:14:53.104 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T21:14:53.104 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:30.388788+0000", 2023-09-23T21:14:53.104 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.8850000000000001e-05 2023-09-23T21:14:53.104 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:14:53.104 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:14:53.104 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:14:53.104 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:30.757509+0000", 2023-09-23T21:14:53.104 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.368720777 2023-09-23T21:14:53.104 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:14:53.104 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:14:53.104 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T21:14:53.105 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:30.757519+0000", 2023-09-23T21:14:53.105 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.8800000000000003e-06 2023-09-23T21:14:53.105 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:14:53.105 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:14:53.105 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:14:53.105 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:14:53.105 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:14:53.105 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.46328.0:105709 10.11 10.8f3eeab1 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e501)", 2023-09-23T21:14:53.105 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T21:13:30.484621+0000", 2023-09-23T21:14:53.105 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 82.573524347000003, 2023-09-23T21:14:53.105 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.30865292100000002, 2023-09-23T21:14:53.105 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:14:53.106 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "reached pg", 2023-09-23T21:14:53.106 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T21:14:53.106 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.46328", 2023-09-23T21:14:53.106 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/3748861181", 2023-09-23T21:14:53.106 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 105709 2023-09-23T21:14:53.106 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:14:53.106 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:14:53.106 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:14:53.106 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:14:53.106 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:30.484621+0000", 2023-09-23T21:14:53.106 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:14:53.106 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:14:53.106 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:14:53.107 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:14:53.107 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:30.484621+0000", 2023-09-23T21:14:53.107 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:14:53.107 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:14:53.107 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:14:53.107 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:14:53.107 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:30.484619+0000", 2023-09-23T21:14:53.107 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999981 2023-09-23T21:14:53.107 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:14:53.107 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:14:53.107 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:14:53.107 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:30.484631+0000", 2023-09-23T21:14:53.107 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1722000000000001e-05 2023-09-23T21:14:53.108 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:14:53.108 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:14:53.108 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:14:53.108 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:30.484652+0000", 2023-09-23T21:14:53.108 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.092e-05 2023-09-23T21:14:53.108 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:14:53.108 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:14:53.108 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:14:53.108 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:30.484665+0000", 2023-09-23T21:14:53.108 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.314e-05 2023-09-23T21:14:53.108 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:14:53.108 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:14:53.108 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:14:53.109 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:30.484738+0000", 2023-09-23T21:14:53.109 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.2749000000000004e-05 2023-09-23T21:14:53.109 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:14:53.109 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:14:53.109 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T21:14:53.109 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:30.484750+0000", 2023-09-23T21:14:53.109 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1959e-05 2023-09-23T21:14:53.109 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:14:53.109 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:14:53.109 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:14:53.109 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:30.793260+0000", 2023-09-23T21:14:53.109 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.30851023399999999 2023-09-23T21:14:53.109 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:14:53.110 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:14:53.110 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T21:14:53.110 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:30.793274+0000", 2023-09-23T21:14:53.110 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.434e-05 2023-09-23T21:14:53.110 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:14:53.110 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:14:53.110 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:14:53.110 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:14:53.110 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:14:53.110 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_repop(client.46328.0:111231 10.0 e521/502 10:059cb734:::benchmark_data_smithi026_379838_object31274:head v 521'4830, mlcod=435'1946)", 2023-09-23T21:14:53.110 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T21:13:58.628366+0000", 2023-09-23T21:14:53.110 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 54.429779222999997, 2023-09-23T21:14:53.111 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.37221736, 2023-09-23T21:14:53.111 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:14:53.111 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:14:53.111 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:14:53.111 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:14:53.111 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:14:53.111 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:58.628366+0000", 2023-09-23T21:14:53.111 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:14:53.111 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:14:53.111 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:14:53.111 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:14:53.111 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:58.628366+0000", 2023-09-23T21:14:53.111 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:14:53.112 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:14:53.112 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:14:53.112 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:14:53.112 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:58.628366+0000", 2023-09-23T21:14:53.112 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T21:14:53.112 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:14:53.112 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:14:53.112 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:14:53.112 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:58.628376+0000", 2023-09-23T21:14:53.112 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0349e-05 2023-09-23T21:14:53.112 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:14:53.112 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:14:53.113 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:14:53.113 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:58.995558+0000", 2023-09-23T21:14:53.113 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.36718158400000001 2023-09-23T21:14:53.113 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:14:53.113 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:14:53.113 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:14:53.113 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:58.995569+0000", 2023-09-23T21:14:53.113 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1681e-05 2023-09-23T21:14:53.113 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:14:53.113 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:14:53.113 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:14:53.113 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:58.995616+0000", 2023-09-23T21:14:53.113 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.6307999999999998e-05 2023-09-23T21:14:53.114 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:14:53.114 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:14:53.114 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:14:53.114 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:58.995652+0000", 2023-09-23T21:14:53.114 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.5957999999999997e-05 2023-09-23T21:14:53.114 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:14:53.114 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:14:53.114 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T21:14:53.114 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:59.000554+0000", 2023-09-23T21:14:53.114 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0049026850000000004 2023-09-23T21:14:53.114 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:14:53.114 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:14:53.115 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T21:14:53.115 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:59.000584+0000", 2023-09-23T21:14:53.115 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.9453000000000001e-05 2023-09-23T21:14:53.115 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:14:53.115 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:14:53.115 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:14:53.115 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:14:53.115 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:14:53.115 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_repop(client.46328.0:111334 10.11 e521/502 10:8e9e8bfc:::benchmark_data_smithi026_379838_object31377:head v 521'4778, mlcod=520'4776)", 2023-09-23T21:14:53.115 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T21:13:58.629174+0000", 2023-09-23T21:14:53.115 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 54.428971079, 2023-09-23T21:14:53.115 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.36777505399999999, 2023-09-23T21:14:53.116 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:14:53.116 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:14:53.116 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:14:53.116 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:14:53.116 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:14:53.116 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:58.629174+0000", 2023-09-23T21:14:53.116 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:14:53.116 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:14:53.116 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:14:53.116 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:14:53.116 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:58.629174+0000", 2023-09-23T21:14:53.116 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:14:53.116 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:14:53.117 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:14:53.117 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:14:53.117 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:58.629174+0000", 2023-09-23T21:14:53.117 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.999999 2023-09-23T21:14:53.117 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:14:53.117 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:14:53.117 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:14:53.117 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:58.629181+0000", 2023-09-23T21:14:53.117 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.0110000000000003e-06 2023-09-23T21:14:53.117 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:14:53.117 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:14:53.117 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:14:53.117 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:58.995526+0000", 2023-09-23T21:14:53.118 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.36634555499999999 2023-09-23T21:14:53.118 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:14:53.118 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:14:53.118 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:14:53.118 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:58.995550+0000", 2023-09-23T21:14:53.118 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.3901999999999999e-05 2023-09-23T21:14:53.118 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:14:53.118 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:14:53.118 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:14:53.118 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:58.995617+0000", 2023-09-23T21:14:53.118 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.6772000000000004e-05 2023-09-23T21:14:53.118 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:14:53.119 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:14:53.119 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:14:53.119 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:58.995644+0000", 2023-09-23T21:14:53.119 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.6709000000000001e-05 2023-09-23T21:14:53.119 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:14:53.119 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:14:53.119 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T21:14:53.119 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:58.996910+0000", 2023-09-23T21:14:53.119 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0012662610000000001 2023-09-23T21:14:53.119 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:14:53.119 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:14:53.119 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T21:14:53.119 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:58.996949+0000", 2023-09-23T21:14:53.120 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.9594000000000001e-05 2023-09-23T21:14:53.120 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:14:53.120 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:14:53.120 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:14:53.120 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:14:53.120 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:14:53.120 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.46328.0:112004 10.14 10:2ab213e5:::benchmark_data_smithi026_379838_object32047:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e522)", 2023-09-23T21:14:53.120 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T21:14:00.500275+0000", 2023-09-23T21:14:53.120 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 52.55787067, 2023-09-23T21:14:53.120 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0387811549999999, 2023-09-23T21:14:53.120 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:14:53.120 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:14:53.120 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T21:14:53.121 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.46328", 2023-09-23T21:14:53.121 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/3748861181", 2023-09-23T21:14:53.121 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 112004 2023-09-23T21:14:53.121 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:14:53.121 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:14:53.121 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:14:53.121 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:14:53.121 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:14:00.500275+0000", 2023-09-23T21:14:53.121 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:14:53.121 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:14:53.121 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:14:53.122 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:14:53.122 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:14:00.500275+0000", 2023-09-23T21:14:53.122 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:14:53.122 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:14:53.122 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:14:53.122 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:14:53.122 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:14:00.500273+0000", 2023-09-23T21:14:53.122 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T21:14:53.122 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:14:53.122 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:14:53.122 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:14:53.122 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:14:00.500280+0000", 2023-09-23T21:14:53.123 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.3300000000000004e-06 2023-09-23T21:14:53.123 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:14:53.123 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:14:53.123 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:14:53.123 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:14:00.500292+0000", 2023-09-23T21:14:53.123 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1822e-05 2023-09-23T21:14:53.123 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:14:53.123 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:14:53.123 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:14:53.123 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:14:00.500299+0000", 2023-09-23T21:14:53.123 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.9079999999999995e-06 2023-09-23T21:14:53.123 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:14:53.123 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:14:53.123 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:14:53.124 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:14:00.500356+0000", 2023-09-23T21:14:53.124 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.7058000000000002e-05 2023-09-23T21:14:53.124 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:14:53.124 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:14:53.124 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T21:14:53.124 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:14:00.500366+0000", 2023-09-23T21:14:53.124 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.73e-06 2023-09-23T21:14:53.124 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:14:53.124 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:14:53.124 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:14:53.124 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:14:01.514070+0000", 2023-09-23T21:14:53.124 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.013704121 2023-09-23T21:14:53.124 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:14:53.124 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:14:53.125 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:14:53.125 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:14:01.514202+0000", 2023-09-23T21:14:53.125 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00013123599999999999 2023-09-23T21:14:53.125 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:14:53.125 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:14:53.125 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 3,4", 2023-09-23T21:14:53.125 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:14:01.514488+0000", 2023-09-23T21:14:53.125 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00028681699999999997 2023-09-23T21:14:53.125 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:14:53.125 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:14:53.125 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:14:53.125 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:14:01.519905+0000", 2023-09-23T21:14:53.125 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0054170959999999997 2023-09-23T21:14:53.125 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:14:53.126 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:14:53.126 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:14:53.126 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:14:01.525371+0000", 2023-09-23T21:14:53.126 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0054652729999999997 2023-09-23T21:14:53.126 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:14:53.126 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:14:53.126 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T21:14:53.126 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:14:01.538952+0000", 2023-09-23T21:14:53.126 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.013580938000000001 2023-09-23T21:14:53.126 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:14:53.126 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:14:53.126 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T21:14:53.126 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:14:01.539012+0000", 2023-09-23T21:14:53.127 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.0356000000000003e-05 2023-09-23T21:14:53.127 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:14:53.127 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:14:53.127 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T21:14:53.127 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:14:01.539056+0000", 2023-09-23T21:14:53.127 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.3961999999999997e-05 2023-09-23T21:14:53.127 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:14:53.127 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:14:53.127 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:14:53.127 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:14:53.127 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:14:53.127 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.46328.0:112018 10.14 10:2860496d:::benchmark_data_smithi026_379838_object32061:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e522)", 2023-09-23T21:14:53.127 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T21:14:00.531660+0000", 2023-09-23T21:14:53.128 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 52.526485243000003, 2023-09-23T21:14:53.128 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0075591260000001, 2023-09-23T21:14:53.128 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:14:53.128 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:14:53.128 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T21:14:53.128 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.46328", 2023-09-23T21:14:53.128 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/3748861181", 2023-09-23T21:14:53.128 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 112018 2023-09-23T21:14:53.128 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:14:53.128 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:14:53.128 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:14:53.128 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:14:53.128 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:14:00.531660+0000", 2023-09-23T21:14:53.129 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:14:53.129 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:14:53.129 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:14:53.129 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:14:53.129 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:14:00.531660+0000", 2023-09-23T21:14:53.129 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:14:53.129 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:14:53.129 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:14:53.129 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:14:53.129 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:14:00.531658+0000", 2023-09-23T21:14:53.129 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999981 2023-09-23T21:14:53.130 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:14:53.130 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:14:53.130 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:14:53.130 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:14:00.531666+0000", 2023-09-23T21:14:53.130 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.3610000000000003e-06 2023-09-23T21:14:53.130 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:14:53.130 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:14:53.130 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:14:53.130 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:14:00.531678+0000", 2023-09-23T21:14:53.130 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.2707999999999999e-05 2023-09-23T21:14:53.130 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:14:53.130 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:14:53.131 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:14:53.131 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:14:00.531687+0000", 2023-09-23T21:14:53.131 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.2360000000000004e-06 2023-09-23T21:14:53.131 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:14:53.131 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:14:53.131 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:14:53.131 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:14:00.531753+0000", 2023-09-23T21:14:53.131 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.6613999999999996e-05 2023-09-23T21:14:53.131 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:14:53.131 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:14:53.131 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T21:14:53.131 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:14:00.531766+0000", 2023-09-23T21:14:53.131 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.272e-05 2023-09-23T21:14:53.132 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:14:53.132 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:14:53.132 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:14:53.132 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:14:01.516057+0000", 2023-09-23T21:14:53.132 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.98429071400000001 2023-09-23T21:14:53.132 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:14:53.132 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:14:53.132 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:14:53.132 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:14:01.516195+0000", 2023-09-23T21:14:53.132 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00013835599999999999 2023-09-23T21:14:53.132 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:14:53.132 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:14:53.133 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 3,4", 2023-09-23T21:14:53.133 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:14:01.516467+0000", 2023-09-23T21:14:53.133 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00027230700000000001 2023-09-23T21:14:53.133 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:14:53.133 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:14:53.133 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:14:53.133 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:14:01.525339+0000", 2023-09-23T21:14:53.133 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.008871416 2023-09-23T21:14:53.133 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:14:53.133 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:14:53.133 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:14:53.133 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:14:01.525401+0000", 2023-09-23T21:14:53.133 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.2712000000000005e-05 2023-09-23T21:14:53.134 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:14:53.134 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:14:53.134 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T21:14:53.134 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:14:01.539151+0000", 2023-09-23T21:14:53.134 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.013749513 2023-09-23T21:14:53.134 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:14:53.134 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:14:53.134 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T21:14:53.134 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:14:01.539187+0000", 2023-09-23T21:14:53.134 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.6109000000000003e-05 2023-09-23T21:14:53.134 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:14:53.134 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:14:53.135 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T21:14:53.135 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:14:01.539219+0000", 2023-09-23T21:14:53.135 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.2370999999999999e-05 2023-09-23T21:14:53.135 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:14:53.135 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:14:53.135 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:14:53.135 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:14:53.135 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:14:53.135 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.46328.0:112020 10.14 10:2ad7942e:::benchmark_data_smithi026_379838_object32063:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e522)", 2023-09-23T21:14:53.135 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T21:14:00.532917+0000", 2023-09-23T21:14:53.135 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 52.525228196, 2023-09-23T21:14:53.135 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0063778370000001, 2023-09-23T21:14:53.135 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:14:53.136 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:14:53.136 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T21:14:53.136 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.46328", 2023-09-23T21:14:53.136 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/3748861181", 2023-09-23T21:14:53.136 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 112020 2023-09-23T21:14:53.136 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:14:53.136 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:14:53.136 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:14:53.136 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:14:53.136 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:14:00.532917+0000", 2023-09-23T21:14:53.136 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:14:53.136 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:14:53.136 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:14:53.137 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:14:53.137 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:14:00.532917+0000", 2023-09-23T21:14:53.137 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:14:53.137 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:14:53.137 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:14:53.137 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:14:53.137 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:14:00.532917+0000", 2023-09-23T21:14:53.137 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.999999 2023-09-23T21:14:53.137 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:14:53.137 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:14:53.137 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:14:53.137 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:14:00.532921+0000", 2023-09-23T21:14:53.137 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.3080000000000001e-06 2023-09-23T21:14:53.138 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:14:53.138 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:14:53.138 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:14:53.138 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:14:00.532932+0000", 2023-09-23T21:14:53.138 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1205e-05 2023-09-23T21:14:53.138 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:14:53.138 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:14:53.138 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:14:53.138 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:14:00.532937+0000", 2023-09-23T21:14:53.138 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.3059999999999997e-06 2023-09-23T21:14:53.138 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:14:53.138 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:14:53.139 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:14:53.139 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:14:00.532998+0000", 2023-09-23T21:14:53.139 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.0504000000000002e-05 2023-09-23T21:14:53.139 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:14:53.139 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:14:53.139 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T21:14:53.139 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:14:00.533009+0000", 2023-09-23T21:14:53.139 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1453e-05 2023-09-23T21:14:53.139 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:14:53.139 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:14:53.139 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:14:53.139 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:14:01.518082+0000", 2023-09-23T21:14:53.139 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.98507318399999999 2023-09-23T21:14:53.140 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:14:53.140 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:14:53.140 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:14:53.140 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:14:01.518190+0000", 2023-09-23T21:14:53.140 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000107796 2023-09-23T21:14:53.140 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:14:53.140 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:14:53.140 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 3,4", 2023-09-23T21:14:53.140 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:14:01.518375+0000", 2023-09-23T21:14:53.140 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00018488700000000001 2023-09-23T21:14:53.140 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:14:53.140 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:14:53.140 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:14:53.141 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:14:01.534073+0000", 2023-09-23T21:14:53.141 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.015698167999999998 2023-09-23T21:14:53.141 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:14:53.141 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:14:53.141 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:14:53.141 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:14:01.534230+0000", 2023-09-23T21:14:53.141 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00015691699999999999 2023-09-23T21:14:53.141 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:14:53.141 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:14:53.141 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T21:14:53.141 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:14:01.539226+0000", 2023-09-23T21:14:53.141 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00499564 2023-09-23T21:14:53.141 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:14:53.142 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:14:53.142 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T21:14:53.142 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:14:01.539263+0000", 2023-09-23T21:14:53.142 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.7036000000000003e-05 2023-09-23T21:14:53.142 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:14:53.142 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:14:53.142 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T21:14:53.142 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:14:01.539295+0000", 2023-09-23T21:14:53.142 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.2211e-05 2023-09-23T21:14:53.142 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:14:53.142 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:14:53.142 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:14:53.143 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:14:53.143 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:14:53.143 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.46328.0:112026 10.14 10:2d0ad96b:::benchmark_data_smithi026_379838_object32069:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e522)", 2023-09-23T21:14:53.143 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T21:14:00.587188+0000", 2023-09-23T21:14:53.143 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 52.470957141, 2023-09-23T21:14:53.143 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.95219126899999995, 2023-09-23T21:14:53.143 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:14:53.143 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:14:53.143 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T21:14:53.143 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.46328", 2023-09-23T21:14:53.143 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/3748861181", 2023-09-23T21:14:53.143 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 112026 2023-09-23T21:14:53.144 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:14:53.144 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:14:53.144 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:14:53.144 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:14:53.144 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:14:00.587188+0000", 2023-09-23T21:14:53.144 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:14:53.144 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:14:53.144 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:14:53.144 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:14:53.144 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:14:00.587188+0000", 2023-09-23T21:14:53.144 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:14:53.144 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:14:53.144 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:14:53.145 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:14:53.145 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:14:00.587187+0000", 2023-09-23T21:14:53.145 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999981 2023-09-23T21:14:53.145 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:14:53.145 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:14:53.145 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:14:53.145 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:14:00.587193+0000", 2023-09-23T21:14:53.145 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.3470000000000004e-06 2023-09-23T21:14:53.145 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:14:53.145 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:14:53.145 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:14:53.145 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:14:00.587206+0000", 2023-09-23T21:14:53.145 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.3137e-05 2023-09-23T21:14:53.146 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:14:53.146 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:14:53.146 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:14:53.146 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:14:00.587212+0000", 2023-09-23T21:14:53.146 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.003e-06 2023-09-23T21:14:53.146 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:14:53.146 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:14:53.146 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:14:53.146 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:14:00.587267+0000", 2023-09-23T21:14:53.146 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.4931e-05 2023-09-23T21:14:53.146 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:14:53.146 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:14:53.147 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T21:14:53.147 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:14:00.587274+0000", 2023-09-23T21:14:53.147 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.5299999999999999e-06 2023-09-23T21:14:53.147 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:14:53.147 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:14:53.147 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:14:53.147 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:14:01.519930+0000", 2023-09-23T21:14:53.147 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.93265571400000002 2023-09-23T21:14:53.147 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:14:53.147 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:14:53.147 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:14:53.147 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:14:01.520049+0000", 2023-09-23T21:14:53.148 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000118936 2023-09-23T21:14:53.148 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:14:53.148 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:14:53.148 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 3,4", 2023-09-23T21:14:53.148 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:14:01.520282+0000", 2023-09-23T21:14:53.148 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00023281300000000001 2023-09-23T21:14:53.148 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:14:53.148 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:14:53.148 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:14:53.148 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:14:01.534105+0000", 2023-09-23T21:14:53.148 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.013822872 2023-09-23T21:14:53.148 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:14:53.148 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:14:53.149 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:14:53.149 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:14:01.534317+0000", 2023-09-23T21:14:53.149 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00021244800000000001 2023-09-23T21:14:53.149 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:14:53.149 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:14:53.149 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T21:14:53.149 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:14:01.539301+0000", 2023-09-23T21:14:53.149 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0049840700000000002 2023-09-23T21:14:53.149 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:14:53.149 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:14:53.149 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T21:14:53.149 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:14:01.539344+0000", 2023-09-23T21:14:53.149 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.2635000000000001e-05 2023-09-23T21:14:53.150 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:14:53.150 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:14:53.150 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T21:14:53.150 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:14:01.539380+0000", 2023-09-23T21:14:53.150 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.5675e-05 2023-09-23T21:14:53.150 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:14:53.150 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:14:53.150 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:14:53.150 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:14:53.150 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:14:53.150 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.46328.0:112070 10.14 10:2bdb93b2:::benchmark_data_smithi026_379838_object32113:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e522)", 2023-09-23T21:14:53.150 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T21:14:00.670023+0000", 2023-09-23T21:14:53.151 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 52.388122963000001, 2023-09-23T21:14:53.151 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.86943998199999994, 2023-09-23T21:14:53.151 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:14:53.151 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:14:53.151 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T21:14:53.151 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.46328", 2023-09-23T21:14:53.151 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/3748861181", 2023-09-23T21:14:53.151 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 112070 2023-09-23T21:14:53.151 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:14:53.151 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:14:53.151 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:14:53.151 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:14:53.151 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:14:00.670023+0000", 2023-09-23T21:14:53.152 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:14:53.152 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:14:53.152 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:14:53.152 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:14:53.152 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:14:00.670023+0000", 2023-09-23T21:14:53.152 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:14:53.152 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:14:53.152 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:14:53.152 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:14:53.152 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:14:00.670020+0000", 2023-09-23T21:14:53.152 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999976 2023-09-23T21:14:53.152 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:14:53.152 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:14:53.153 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:14:53.153 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:14:00.670028+0000", 2023-09-23T21:14:53.153 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.3300000000000001e-06 2023-09-23T21:14:53.153 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:14:53.153 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:14:53.153 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:14:53.153 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:14:00.670040+0000", 2023-09-23T21:14:53.153 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.2071e-05 2023-09-23T21:14:53.153 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:14:53.153 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:14:53.153 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:14:53.153 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:14:00.670048+0000", 2023-09-23T21:14:53.154 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.9179999999999994e-06 2023-09-23T21:14:53.154 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:14:53.154 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:14:53.154 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:14:53.154 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:14:00.670116+0000", 2023-09-23T21:14:53.154 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.8277999999999999e-05 2023-09-23T21:14:53.154 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:14:53.154 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:14:53.154 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T21:14:53.154 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:14:00.670129+0000", 2023-09-23T21:14:53.154 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.3052e-05 2023-09-23T21:14:53.154 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:14:53.155 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:14:53.155 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:14:53.155 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:14:01.521719+0000", 2023-09-23T21:14:53.155 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.85158983099999996 2023-09-23T21:14:53.155 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:14:53.155 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:14:53.155 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:14:53.155 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:14:01.521848+0000", 2023-09-23T21:14:53.155 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000129558 2023-09-23T21:14:53.155 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:14:53.155 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:14:53.155 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 3,4", 2023-09-23T21:14:53.155 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:14:01.522134+0000", 2023-09-23T21:14:53.156 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00028569499999999998 2023-09-23T21:14:53.156 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:14:53.156 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:14:53.156 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:14:53.156 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:14:01.534134+0000", 2023-09-23T21:14:53.156 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.012000022000000001 2023-09-23T21:14:53.156 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:14:53.156 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:14:53.156 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:14:53.156 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:14:01.534348+0000", 2023-09-23T21:14:53.156 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00021344199999999999 2023-09-23T21:14:53.156 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:14:53.156 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:14:53.157 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T21:14:53.157 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:14:01.539386+0000", 2023-09-23T21:14:53.157 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0050384820000000004 2023-09-23T21:14:53.157 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:14:53.157 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:14:53.157 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T21:14:53.157 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:14:01.539428+0000", 2023-09-23T21:14:53.157 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.2148999999999999e-05 2023-09-23T21:14:53.157 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:14:53.157 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:14:53.157 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T21:14:53.157 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:14:01.539463+0000", 2023-09-23T21:14:53.158 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.4351999999999999e-05 2023-09-23T21:14:53.158 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:14:53.158 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:14:53.158 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:14:53.158 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:14:53.158 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:14:53.158 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.46328.0:112097 10.14 10:2eaf8bdc:::benchmark_data_smithi026_379838_object32140:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e522)", 2023-09-23T21:14:53.158 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T21:14:00.712118+0000", 2023-09-23T21:14:53.158 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 52.346027358000001, 2023-09-23T21:14:53.158 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.82743756300000004, 2023-09-23T21:14:53.158 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:14:53.158 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:14:53.159 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T21:14:53.159 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.46328", 2023-09-23T21:14:53.159 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/3748861181", 2023-09-23T21:14:53.159 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 112097 2023-09-23T21:14:53.159 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:14:53.159 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:14:53.159 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:14:53.159 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:14:53.159 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:14:00.712118+0000", 2023-09-23T21:14:53.159 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:14:53.159 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:14:53.159 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:14:53.159 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:14:53.160 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:14:00.712118+0000", 2023-09-23T21:14:53.160 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:14:53.160 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:14:53.160 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:14:53.160 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:14:53.160 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:14:00.712117+0000", 2023-09-23T21:14:53.160 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T21:14:53.160 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:14:53.160 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:14:53.160 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:14:53.160 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:14:00.712123+0000", 2023-09-23T21:14:53.160 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.9730000000000002e-06 2023-09-23T21:14:53.160 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:14:53.161 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:14:53.161 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:14:53.161 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:14:00.712138+0000", 2023-09-23T21:14:53.161 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.4958e-05 2023-09-23T21:14:53.161 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:14:53.161 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:14:53.161 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:14:53.161 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:14:00.712145+0000", 2023-09-23T21:14:53.161 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.5020000000000003e-06 2023-09-23T21:14:53.161 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:14:53.161 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:14:53.161 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:14:53.162 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:14:00.712199+0000", 2023-09-23T21:14:53.162 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.4141999999999998e-05 2023-09-23T21:14:53.162 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:14:53.162 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:14:53.162 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T21:14:53.162 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:14:00.712208+0000", 2023-09-23T21:14:53.162 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.5159999999999994e-06 2023-09-23T21:14:53.162 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:14:53.162 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:14:53.162 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:14:53.162 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:14:01.523550+0000", 2023-09-23T21:14:53.162 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.81134222600000006 2023-09-23T21:14:53.163 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:14:53.163 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:14:53.163 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:14:53.163 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:14:01.523654+0000", 2023-09-23T21:14:53.163 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000103562 2023-09-23T21:14:53.163 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:14:53.163 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:14:53.163 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 3,4", 2023-09-23T21:14:53.163 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:14:01.523893+0000", 2023-09-23T21:14:53.163 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00023928099999999999 2023-09-23T21:14:53.163 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:14:53.163 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:14:53.164 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:14:53.164 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:14:01.534164+0000", 2023-09-23T21:14:53.164 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.010271471000000001 2023-09-23T21:14:53.164 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:14:53.164 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:14:53.164 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:14:53.164 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:14:01.538942+0000", 2023-09-23T21:14:53.164 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0047775320000000001 2023-09-23T21:14:53.164 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:14:53.164 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:14:53.164 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T21:14:53.164 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:14:01.539469+0000", 2023-09-23T21:14:53.165 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00052666900000000001 2023-09-23T21:14:53.165 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:14:53.165 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:14:53.165 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T21:14:53.165 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:14:01.539521+0000", 2023-09-23T21:14:53.165 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.1941000000000003e-05 2023-09-23T21:14:53.165 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:14:53.165 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:14:53.165 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T21:14:53.165 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:14:01.539556+0000", 2023-09-23T21:14:53.165 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.5213000000000003e-05 2023-09-23T21:14:53.165 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:14:53.165 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:14:53.166 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:14:53.166 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:14:53.166 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:14:53.166 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.46328.0:112200 10.14 10:2a5c1230:::benchmark_data_smithi026_379838_object32243:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e522)", 2023-09-23T21:14:53.166 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T21:14:00.941399+0000", 2023-09-23T21:14:53.166 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 52.116746974999998, 2023-09-23T21:14:53.166 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.60795617000000002, 2023-09-23T21:14:53.166 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:14:53.166 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:14:53.166 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T21:14:53.166 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.46328", 2023-09-23T21:14:53.167 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/3748861181", 2023-09-23T21:14:53.167 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 112200 2023-09-23T21:14:53.167 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:14:53.167 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:14:53.167 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:14:53.167 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:14:53.167 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:14:00.941399+0000", 2023-09-23T21:14:53.167 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:14:53.167 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:14:53.167 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:14:53.167 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:14:53.167 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:14:00.941399+0000", 2023-09-23T21:14:53.167 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:14:53.168 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:14:53.168 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:14:53.168 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:14:53.168 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:14:00.941397+0000", 2023-09-23T21:14:53.168 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T21:14:53.168 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:14:53.168 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:14:53.168 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:14:53.168 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:14:00.941403+0000", 2023-09-23T21:14:53.168 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.3629999999999999e-06 2023-09-23T21:14:53.168 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:14:53.168 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:14:53.168 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:14:53.169 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:14:00.941418+0000", 2023-09-23T21:14:53.169 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.5143000000000001e-05 2023-09-23T21:14:53.169 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:14:53.169 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:14:53.169 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:14:53.169 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:14:00.941426+0000", 2023-09-23T21:14:53.169 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.4749999999999996e-06 2023-09-23T21:14:53.169 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:14:53.169 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:14:53.169 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:14:53.169 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:14:00.941491+0000", 2023-09-23T21:14:53.169 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.5562999999999996e-05 2023-09-23T21:14:53.170 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:14:53.170 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:14:53.170 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T21:14:53.170 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:14:00.941503+0000", 2023-09-23T21:14:53.170 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1919000000000001e-05 2023-09-23T21:14:53.170 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:14:53.170 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:14:53.170 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:14:53.170 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:14:01.525423+0000", 2023-09-23T21:14:53.170 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.58391958099999997 2023-09-23T21:14:53.170 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:14:53.170 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:14:53.171 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:14:53.171 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:14:01.525551+0000", 2023-09-23T21:14:53.171 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00012816499999999999 2023-09-23T21:14:53.171 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:14:53.171 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:14:53.171 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 3,4", 2023-09-23T21:14:53.171 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:14:01.525798+0000", 2023-09-23T21:14:53.171 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00024720699999999999 2023-09-23T21:14:53.171 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:14:53.171 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:14:53.171 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:14:53.171 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:14:01.534197+0000", 2023-09-23T21:14:53.172 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0083985180000000006 2023-09-23T21:14:53.172 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:14:53.172 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:14:53.172 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T21:14:53.172 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:14:01.539563+0000", 2023-09-23T21:14:53.172 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0053662090000000003 2023-09-23T21:14:53.172 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:14:53.172 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:14:53.172 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:14:53.172 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:14:01.549281+0000", 2023-09-23T21:14:53.172 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0097184379999999994 2023-09-23T21:14:53.172 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:14:53.172 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:14:53.173 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T21:14:53.173 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:14:01.549317+0000", 2023-09-23T21:14:53.173 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.5324000000000003e-05 2023-09-23T21:14:53.173 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:14:53.173 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:14:53.173 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T21:14:53.173 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:14:01.549355+0000", 2023-09-23T21:14:53.173 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.7929000000000002e-05 2023-09-23T21:14:53.173 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:14:53.173 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:14:53.173 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:14:53.173 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:14:53.174 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:14:53.174 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.46328.0:112293 10.14 10:2b83171c:::benchmark_data_smithi026_379838_object32336:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e522)", 2023-09-23T21:14:53.174 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T21:14:01.182239+0000", 2023-09-23T21:14:53.174 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 51.875906839999999, 2023-09-23T21:14:53.174 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.36771867699999999, 2023-09-23T21:14:53.174 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:14:53.174 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:14:53.174 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T21:14:53.174 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.46328", 2023-09-23T21:14:53.174 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/3748861181", 2023-09-23T21:14:53.174 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 112293 2023-09-23T21:14:53.174 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:14:53.174 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:14:53.175 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:14:53.175 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:14:53.175 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:14:01.182239+0000", 2023-09-23T21:14:53.175 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:14:53.175 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:14:53.175 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:14:53.175 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:14:53.175 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:14:01.182239+0000", 2023-09-23T21:14:53.175 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:14:53.175 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:14:53.175 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:14:53.175 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:14:53.176 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:14:01.182237+0000", 2023-09-23T21:14:53.176 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T21:14:53.176 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:14:53.176 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:14:53.176 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:14:53.176 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:14:01.182244+0000", 2023-09-23T21:14:53.176 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.6390000000000003e-06 2023-09-23T21:14:53.176 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:14:53.176 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:14:53.176 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:14:53.176 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:14:01.182254+0000", 2023-09-23T21:14:53.176 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0662999999999999e-05 2023-09-23T21:14:53.176 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:14:53.177 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:14:53.177 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:14:53.177 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:14:01.182262+0000", 2023-09-23T21:14:53.177 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.6090000000000003e-06 2023-09-23T21:14:53.177 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:14:53.177 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:14:53.177 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:14:53.177 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:14:01.182315+0000", 2023-09-23T21:14:53.177 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.3335999999999999e-05 2023-09-23T21:14:53.177 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:14:53.177 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:14:53.177 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T21:14:53.177 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:14:01.182326+0000", 2023-09-23T21:14:53.178 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0489000000000001e-05 2023-09-23T21:14:53.178 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:14:53.178 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:14:53.178 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:14:53.178 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:14:01.527190+0000", 2023-09-23T21:14:53.178 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.34486373300000001 2023-09-23T21:14:53.178 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:14:53.178 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:14:53.178 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:14:53.178 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:14:01.527315+0000", 2023-09-23T21:14:53.178 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00012556199999999999 2023-09-23T21:14:53.179 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:14:53.179 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:14:53.179 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 3,4", 2023-09-23T21:14:53.179 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:14:01.527576+0000", 2023-09-23T21:14:53.179 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000261086 2023-09-23T21:14:53.179 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:14:53.179 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:14:53.179 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:14:53.179 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:14:01.534259+0000", 2023-09-23T21:14:53.179 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0066831290000000003 2023-09-23T21:14:53.179 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:14:53.179 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:14:53.179 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T21:14:53.180 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:14:01.539568+0000", 2023-09-23T21:14:53.180 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0053088780000000004 2023-09-23T21:14:53.180 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:14:53.180 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:14:53.180 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:14:53.180 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:14:01.549878+0000", 2023-09-23T21:14:53.180 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.010309420999999999 2023-09-23T21:14:53.180 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:14:53.180 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:14:53.180 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T21:14:53.180 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:14:01.549920+0000", 2023-09-23T21:14:53.180 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.1817000000000003e-05 2023-09-23T21:14:53.180 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:14:53.181 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:14:53.181 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T21:14:53.181 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:14:01.549957+0000", 2023-09-23T21:14:53.181 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.7824000000000002e-05 2023-09-23T21:14:53.181 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:14:53.181 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:14:53.181 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:14:53.181 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:14:53.181 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:14:53.181 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.46328.0:112298 10.14 10:2a87c47f:::benchmark_data_smithi026_379838_object32341:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e522)", 2023-09-23T21:14:53.181 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T21:14:01.190720+0000", 2023-09-23T21:14:53.181 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 51.867425099999998, 2023-09-23T21:14:53.182 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.359408432, 2023-09-23T21:14:53.182 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:14:53.182 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:14:53.182 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T21:14:53.182 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.46328", 2023-09-23T21:14:53.182 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/3748861181", 2023-09-23T21:14:53.182 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 112298 2023-09-23T21:14:53.182 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:14:53.182 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:14:53.182 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:14:53.182 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:14:53.182 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:14:01.190720+0000", 2023-09-23T21:14:53.182 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:14:53.183 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:14:53.183 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:14:53.183 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:14:53.183 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:14:01.190720+0000", 2023-09-23T21:14:53.183 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:14:53.183 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:14:53.183 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:14:53.183 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:14:53.183 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:14:01.190719+0000", 2023-09-23T21:14:53.183 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999981 2023-09-23T21:14:53.183 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:14:53.183 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:14:53.184 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:14:53.184 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:14:01.190727+0000", 2023-09-23T21:14:53.184 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.3510000000000002e-06 2023-09-23T21:14:53.184 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:14:53.184 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:14:53.184 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:14:53.184 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:14:01.190746+0000", 2023-09-23T21:14:53.184 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.8822999999999999e-05 2023-09-23T21:14:53.184 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:14:53.184 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:14:53.184 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:14:53.184 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:14:01.190755+0000", 2023-09-23T21:14:53.184 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.5850000000000006e-06 2023-09-23T21:14:53.185 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:14:53.185 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:14:53.185 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:14:53.185 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:14:01.190830+0000", 2023-09-23T21:14:53.185 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.4629000000000007e-05 2023-09-23T21:14:53.185 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:14:53.185 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:14:53.185 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T21:14:53.185 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:14:01.190842+0000", 2023-09-23T21:14:53.185 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1792e-05 2023-09-23T21:14:53.185 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:14:53.185 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:14:53.185 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:14:53.186 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:14:01.528920+0000", 2023-09-23T21:14:53.186 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.33807835200000003 2023-09-23T21:14:53.186 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:14:53.186 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:14:53.186 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:14:53.186 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:14:01.529041+0000", 2023-09-23T21:14:53.186 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000120777 2023-09-23T21:14:53.186 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:14:53.186 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:14:53.186 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 3,4", 2023-09-23T21:14:53.186 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:14:01.529252+0000", 2023-09-23T21:14:53.186 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00021154400000000001 2023-09-23T21:14:53.187 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:14:53.187 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:14:53.187 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:14:53.187 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:14:01.534288+0000", 2023-09-23T21:14:53.187 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0050353739999999996 2023-09-23T21:14:53.187 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:14:53.187 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:14:53.187 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T21:14:53.187 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:14:01.539573+0000", 2023-09-23T21:14:53.187 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0052854779999999997 2023-09-23T21:14:53.187 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:14:53.187 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:14:53.187 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:14:53.188 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:14:01.550038+0000", 2023-09-23T21:14:53.188 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.010464865 2023-09-23T21:14:53.188 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:14:53.188 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:14:53.188 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T21:14:53.188 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:14:01.550087+0000", 2023-09-23T21:14:53.188 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.8566000000000002e-05 2023-09-23T21:14:53.188 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:14:53.188 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:14:53.188 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T21:14:53.188 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:14:01.550129+0000", 2023-09-23T21:14:53.188 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.2206000000000002e-05 2023-09-23T21:14:53.189 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:14:53.189 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:14:53.189 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:14:53.189 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:14:53.189 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:14:53.189 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.46328.0:112309 10.14 10:2f86a8f4:::benchmark_data_smithi026_379838_object32352:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e522)", 2023-09-23T21:14:53.189 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T21:14:01.216557+0000", 2023-09-23T21:14:53.189 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 51.841588754999997, 2023-09-23T21:14:53.189 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.34050878099999998, 2023-09-23T21:14:53.189 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:14:53.189 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:14:53.189 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T21:14:53.189 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.46328", 2023-09-23T21:14:53.190 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/3748861181", 2023-09-23T21:14:53.190 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 112309 2023-09-23T21:14:53.190 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:14:53.190 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:14:53.190 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:14:53.190 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:14:53.190 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:14:01.216557+0000", 2023-09-23T21:14:53.190 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:14:53.190 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:14:53.190 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:14:53.190 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:14:53.190 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:14:01.216557+0000", 2023-09-23T21:14:53.191 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:14:53.191 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:14:53.191 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:14:53.191 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:14:53.191 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:14:01.216555+0000", 2023-09-23T21:14:53.191 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T21:14:53.191 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:14:53.191 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:14:53.191 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:14:53.191 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:14:01.216563+0000", 2023-09-23T21:14:53.191 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.3239999999999997e-06 2023-09-23T21:14:53.191 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:14:53.191 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:14:53.192 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:14:53.192 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:14:01.216574+0000", 2023-09-23T21:14:53.192 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1693e-05 2023-09-23T21:14:53.192 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:14:53.192 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:14:53.192 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:14:53.192 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:14:01.216582+0000", 2023-09-23T21:14:53.192 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.089e-06 2023-09-23T21:14:53.192 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:14:53.192 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:14:53.192 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:14:53.193 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:14:01.216633+0000", 2023-09-23T21:14:53.193 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.0727999999999999e-05 2023-09-23T21:14:53.193 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:14:53.193 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:14:53.193 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T21:14:53.193 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:14:01.216643+0000", 2023-09-23T21:14:53.193 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0029e-05 2023-09-23T21:14:53.193 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:14:53.193 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:14:53.193 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:14:53.193 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:14:01.532419+0000", 2023-09-23T21:14:53.193 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.31577592300000001 2023-09-23T21:14:53.193 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:14:53.194 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:14:53.194 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:14:53.194 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:14:01.532568+0000", 2023-09-23T21:14:53.194 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00014903299999999999 2023-09-23T21:14:53.194 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:14:53.194 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:14:53.194 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 3,4", 2023-09-23T21:14:53.194 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:14:01.532883+0000", 2023-09-23T21:14:53.194 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00031542799999999998 2023-09-23T21:14:53.194 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:14:53.194 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:14:53.194 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:14:53.195 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:14:01.538892+0000", 2023-09-23T21:14:53.195 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.006008457 2023-09-23T21:14:53.195 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:14:53.195 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:14:53.195 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T21:14:53.195 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:14:01.539578+0000", 2023-09-23T21:14:53.195 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00068610700000000004 2023-09-23T21:14:53.195 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:14:53.195 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:14:53.195 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:14:53.195 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:14:01.556973+0000", 2023-09-23T21:14:53.195 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.017395458999999999 2023-09-23T21:14:53.196 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:14:53.196 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:14:53.196 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T21:14:53.196 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:14:01.557023+0000", 2023-09-23T21:14:53.196 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.9175999999999998e-05 2023-09-23T21:14:53.196 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:14:53.196 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:14:53.196 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T21:14:53.196 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:14:01.557066+0000", 2023-09-23T21:14:53.196 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.2904999999999997e-05 2023-09-23T21:14:53.196 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:14:53.196 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:14:53.196 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:14:53.197 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:14:53.197 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:14:53.197 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.46328.0:112317 10.14 10:2c28a9d8:::benchmark_data_smithi026_379838_object32360:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e522)", 2023-09-23T21:14:53.197 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T21:14:01.228376+0000", 2023-09-23T21:14:53.197 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 51.829769222000003, 2023-09-23T21:14:53.197 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.32879608300000002, 2023-09-23T21:14:53.197 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:14:53.197 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:14:53.197 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T21:14:53.197 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.46328", 2023-09-23T21:14:53.197 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/3748861181", 2023-09-23T21:14:53.197 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 112317 2023-09-23T21:14:53.197 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:14:53.198 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:14:53.198 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:14:53.198 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:14:53.198 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:14:01.228376+0000", 2023-09-23T21:14:53.198 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:14:53.198 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:14:53.198 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:14:53.198 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:14:53.198 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:14:01.228376+0000", 2023-09-23T21:14:53.198 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:14:53.198 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:14:53.198 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:14:53.199 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:14:53.199 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:14:01.228375+0000", 2023-09-23T21:14:53.199 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.999999 2023-09-23T21:14:53.199 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:14:53.199 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:14:53.199 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:14:53.199 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:14:01.228381+0000", 2023-09-23T21:14:53.199 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.003e-06 2023-09-23T21:14:53.199 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:14:53.199 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:14:53.199 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:14:53.199 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:14:01.228398+0000", 2023-09-23T21:14:53.199 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.7028e-05 2023-09-23T21:14:53.200 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:14:53.200 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:14:53.200 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:14:53.200 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:14:01.228408+0000", 2023-09-23T21:14:53.200 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.9639999999999998e-06 2023-09-23T21:14:53.200 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:14:53.200 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:14:53.200 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:14:53.200 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:14:01.228486+0000", 2023-09-23T21:14:53.200 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.7732999999999996e-05 2023-09-23T21:14:53.200 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:14:53.200 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:14:53.200 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T21:14:53.201 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:14:01.228499+0000", 2023-09-23T21:14:53.201 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.278e-05 2023-09-23T21:14:53.201 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:14:53.201 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:14:53.201 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:14:53.201 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:14:01.534371+0000", 2023-09-23T21:14:53.201 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.30587188599999998 2023-09-23T21:14:53.201 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:14:53.201 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:14:53.201 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:14:53.201 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:14:01.534466+0000", 2023-09-23T21:14:53.201 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.5261999999999997e-05 2023-09-23T21:14:53.201 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:14:53.202 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:14:53.202 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 3,4", 2023-09-23T21:14:53.202 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:14:01.534657+0000", 2023-09-23T21:14:53.202 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00019150300000000001 2023-09-23T21:14:53.202 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:14:53.202 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:14:53.202 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T21:14:53.202 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:14:01.539583+0000", 2023-09-23T21:14:53.202 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0049253930000000001 2023-09-23T21:14:53.202 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:14:53.202 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:14:53.202 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:14:53.202 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:14:01.546707+0000", 2023-09-23T21:14:53.203 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0071237269999999998 2023-09-23T21:14:53.203 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:14:53.203 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:14:53.203 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:14:53.203 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:14:01.557100+0000", 2023-09-23T21:14:53.203 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.010393062999999999 2023-09-23T21:14:53.203 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:14:53.203 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:14:53.203 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T21:14:53.203 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:14:01.557136+0000", 2023-09-23T21:14:53.203 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.6751000000000001e-05 2023-09-23T21:14:53.203 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:14:53.203 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:14:53.204 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T21:14:53.204 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:14:01.557172+0000", 2023-09-23T21:14:53.204 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.5939000000000003e-05 2023-09-23T21:14:53.204 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:14:53.204 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:14:53.204 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:14:53.204 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:14:53.204 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:14:53.204 INFO:teuthology.orchestra.run.smithi026.stdout:} 2023-09-23T21:14:53.204 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:14:53.249 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:14:53.315 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:14:53.319 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:14:53.319 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:14:53.420 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:14:53.451 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:14:53.506 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:14:53.516 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:14:53.516 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:14:53.617 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:14:53.665 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:14:53.721 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:14:53.724 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:14:53.725 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:14:53.826 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:14:53.889 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:14:53.948 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:14:53.952 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_ops_in_flight 2023-09-23T21:14:53.953 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:14:53.954 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:14:54.055 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:14:54.078 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0: {} 2023-09-23T21:14:54.120 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1: {} 2023-09-23T21:14:54.133 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:14:54.169 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2: {} 2023-09-23T21:14:54.210 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3: {} 2023-09-23T21:14:54.227 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:14:54.240 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:14:54.241 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:14:54.281 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4: {} 2023-09-23T21:14:54.342 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T21:14:54.342 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:14:54.395 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6: {} 2023-09-23T21:14:54.434 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:14:54.442 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T21:14:54.468 INFO:teuthology.orchestra.run.smithi026.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T21:14:54.469 INFO:teuthology.orchestra.run.smithi026.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T21:14:54.469 INFO:teuthology.orchestra.run.smithi026.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T21:14:54.469 INFO:teuthology.orchestra.run.smithi026.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T21:14:54.469 INFO:teuthology.orchestra.run.smithi026.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T21:14:54.469 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T21:14:54.469 INFO:teuthology.orchestra.run.smithi026.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T21:14:54.469 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T21:14:54.517 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:14:54.525 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:14:54.526 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:14:54.627 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:14:54.714 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:14:54.801 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T21:14:54.865 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:14:54 smithi026 ceph-mon[175151]: pgmap v1343: 129 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 15 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 108 active+clean; 1.2 GiB data, 8.2 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 643 op/s; 21439/58140 objects degraded (36.875%); 3237/58140 objects misplaced (5.568%); 6.6 MiB/s, 106 objects/s recovering 2023-09-23T21:14:54.865 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:14:54 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T21:14:54.865 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:14:54 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/1371288863' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:14:54.869 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:14:54 smithi026 ceph-mon[179688]: pgmap v1343: 129 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 15 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 108 active+clean; 1.2 GiB data, 8.2 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 643 op/s; 21439/58140 objects degraded (36.875%); 3237/58140 objects misplaced (5.568%); 6.6 MiB/s, 106 objects/s recovering 2023-09-23T21:14:54.869 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:14:54 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T21:14:54.869 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:14:54 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/1371288863' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:14:54.962 INFO:teuthology.orchestra.run.smithi026.stdout: 2023-09-23T21:14:54.974 INFO:teuthology.orchestra.run.smithi026.stderr:dumped all 2023-09-23T21:14:55.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:14:54 smithi070 ceph-mon[168282]: pgmap v1343: 129 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 15 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 108 active+clean; 1.2 GiB data, 8.2 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 643 op/s; 21439/58140 objects degraded (36.875%); 3237/58140 objects misplaced (5.568%); 6.6 MiB/s, 106 objects/s recovering 2023-09-23T21:14:55.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:14:54 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T21:14:55.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:14:54 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/1371288863' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:14:55.161 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T21:14:55.172 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:14:55.173 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:14:55.179 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:14:55.273 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:14:55.328 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:14:55.432 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:14:55.437 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:14:55.438 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:14:55.538 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:14:55.564 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:14:55.602 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:14:55.607 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:14:55.607 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:14:55.708 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:14:55.763 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:14:55.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:14:55 smithi070 ceph-mon[168282]: from='client.56491 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T21:14:55.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:14:55 smithi070 ceph-mon[168282]: pgmap v1344: 129 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 15 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 108 active+clean; 1.1 GiB data, 8.0 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 752 op/s; 20724/51750 objects degraded (40.046%); 3181/51750 objects misplaced (6.147%); 11 MiB/s, 186 objects/s recovering 2023-09-23T21:14:55.957 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:14:55.965 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:14:55.966 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:14:56.066 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:14:56.129 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:14:55 smithi026 ceph-mon[179688]: from='client.56491 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T21:14:56.129 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:14:55 smithi026 ceph-mon[179688]: pgmap v1344: 129 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 15 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 108 active+clean; 1.1 GiB data, 8.0 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 752 op/s; 20724/51750 objects degraded (40.046%); 3181/51750 objects misplaced (6.147%); 11 MiB/s, 186 objects/s recovering 2023-09-23T21:14:56.129 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:14:55 smithi026 ceph-mon[175151]: from='client.56491 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T21:14:56.129 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:14:55 smithi026 ceph-mon[175151]: pgmap v1344: 129 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 15 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 108 active+clean; 1.1 GiB data, 8.0 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 752 op/s; 20724/51750 objects degraded (40.046%); 3181/51750 objects misplaced (6.147%); 11 MiB/s, 186 objects/s recovering 2023-09-23T21:14:56.146 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:14:57.017 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:14:56 smithi026 ceph-mon[179688]: Health check update: Degraded data redundancy: 20724/51750 objects degraded (40.046%), 19 pgs degraded, 16 pgs undersized (PG_DEGRADED) 2023-09-23T21:14:57.017 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:14:56 smithi026 ceph-mon[175151]: Health check update: Degraded data redundancy: 20724/51750 objects degraded (40.046%), 19 pgs degraded, 16 pgs undersized (PG_DEGRADED) 2023-09-23T21:14:57.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:14:56 smithi070 ceph-mon[168282]: Health check update: Degraded data redundancy: 20724/51750 objects degraded (40.046%), 19 pgs degraded, 16 pgs undersized (PG_DEGRADED) 2023-09-23T21:14:57.161 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T21:14:57.206 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:14:57.218 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:14:57.218 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:14:57.319 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:14:57.431 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:14:57.550 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:14:57.551 INFO:teuthology.orchestra.run.smithi026.stdout:{"pg_ready":true,"pg_map":{"version":1343,"stamp":"2023-09-23T21:14:53.083986+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":1268955908,"num_objects":19380,"num_object_clones":0,"num_object_copies":58140,"num_objects_missing_on_primary":970,"num_objects_missing":970,"num_objects_degraded":21439,"num_objects_misplaced":3237,"num_objects_unfound":0,"num_objects_dirty":19380,"num_whiteouts":0,"num_read":90,"num_read_kb":166,"num_write":128124,"num_write_kb":4309782,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":41900,"num_bytes_recovered":2533067941,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":134788,"ondisk_log_size":134788,"up":387,"acting":371,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":433,"num_osds":6,"num_per_pool_osds":6,"num_per_pool_omap_osds":6,"kb":562470912,"kb_used":8563940,"kb_used_data":5276596,"kb_used_omap":125,"kb_used_meta":3286914,"kb_avail":553906972,"statfs":{"total":575970213888,"available":567200739328,"internally_reserved":0,"allocated":5403234304,"data_stored":5391196743,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":128662,"internal_metadata":3365800298},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[4,3,3,6,6,4,1,0,1],"upper_bound":512},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-354287616,"num_objects":-5406,"num_object_clones":0,"num_object_copies":-16218,"num_objects_missing_on_primary":-418,"num_objects_missing":-418,"num_objects_degraded":-681,"num_objects_misplaced":-333,"num_objects_unfound":0,"num_objects_dirty":-5406,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5406,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":891,"num_bytes_recovered":58392576,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"8.401333"},"pg_stats":[{"pgid":"1.18","version":"79'161","reported_seq":650,"reported_epoch":529,"state":"active+clean","last_fresh":"2023-09-23T21:14:32.788047+0000","last_change":"2023-09-23T21:12:30.438733+0000","last_active":"2023-09-23T21:14:32.788047+0000","last_peered":"2023-09-23T21:14:32.788047+0000","last_clean":"2023-09-23T21:14:32.788047+0000","last_became_active":"2023-09-23T21:12:30.438356+0000","last_became_peered":"2023-09-23T21:12:30.438356+0000","last_unstale":"2023-09-23T21:14:32.788047+0000","last_undegraded":"2023-09-23T21:14:32.788047+0000","last_fullsized":"2023-09-23T21:14:32.788047+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":479,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:06:52.098882+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:06:52.098882+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0092173299999999993,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":131344,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"10.13","version":"530'5700","reported_seq":9712,"reported_epoch":530,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T21:14:48.320777+0000","last_change":"2023-09-23T21:13:49.840403+0000","last_active":"2023-09-23T21:14:48.320777+0000","last_peered":"2023-09-23T21:14:48.320777+0000","last_clean":"2023-09-23T21:11:26.718194+0000","last_became_active":"2023-09-23T21:13:48.181112+0000","last_became_peered":"2023-09-23T21:13:48.181112+0000","last_unstale":"2023-09-23T21:14:48.320777+0000","last_undegraded":"2023-09-23T21:13:48.139049+0000","last_fullsized":"2023-09-23T21:13:45.405990+0000","mapping_epoch":513,"log_start":"434'838","ondisk_log_start":"434'838","created":438,"last_epoch_clean":439,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_clean_scrub_stamp":"2023-09-23T21:10:15.663179+0000","objects_scrubbed":0,"log_size":4862,"log_dups_size":0,"ondisk_log_size":4862,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":45973504,"num_objects":702,"num_object_clones":0,"num_object_copies":2106,"num_objects_missing_on_primary":331,"num_objects_missing":331,"num_objects_degraded":531,"num_objects_misplaced":836,"num_objects_unfound":0,"num_objects_dirty":702,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4332,"num_write_kb":161056,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1816,"num_bytes_recovered":118947840,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[4,3],"avail_no_missing":["5"],"object_location_counts":[{"shards":"0,2,4,5","objects":307},{"shards":"2,3,4,5","objects":1},{"shards":"2,3,5","objects":331},{"shards":"2,4,5","objects":28},{"shards":"3,4","objects":35}],"blocked_by":[],"up_primary":2,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1a","version":"68'1","reported_seq":120,"reported_epoch":529,"state":"active+clean","last_fresh":"2023-09-23T21:14:32.781648+0000","last_change":"2023-09-23T21:13:13.291403+0000","last_active":"2023-09-23T21:14:32.781648+0000","last_peered":"2023-09-23T21:14:32.781648+0000","last_clean":"2023-09-23T21:14:32.781648+0000","last_became_active":"2023-09-23T21:13:13.291123+0000","last_became_peered":"2023-09-23T21:13:13.291123+0000","last_unstale":"2023-09-23T21:14:32.781648+0000","last_undegraded":"2023-09-23T21:14:32.781648+0000","last_fullsized":"2023-09-23T21:14:32.781648+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":495,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:45:15.427430+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,0],"acting":[4,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":189,"reported_epoch":529,"state":"active+clean","last_fresh":"2023-09-23T21:14:32.779843+0000","last_change":"2023-09-23T21:12:10.767963+0000","last_active":"2023-09-23T21:14:32.779843+0000","last_peered":"2023-09-23T21:14:32.779843+0000","last_clean":"2023-09-23T21:14:32.779843+0000","last_became_active":"2023-09-23T21:12:10.767126+0000","last_became_peered":"2023-09-23T21:12:10.767126+0000","last_unstale":"2023-09-23T21:14:32.779843+0000","last_undegraded":"2023-09-23T21:14:32.779843+0000","last_fullsized":"2023-09-23T21:14:32.779843+0000","mapping_epoch":470,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:45:47.138277+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,5],"acting":[4,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1b","version":"79'161","reported_seq":533,"reported_epoch":529,"state":"active+clean","last_fresh":"2023-09-23T21:14:32.786348+0000","last_change":"2023-09-23T21:13:13.373548+0000","last_active":"2023-09-23T21:14:32.786348+0000","last_peered":"2023-09-23T21:14:32.786348+0000","last_clean":"2023-09-23T21:14:32.786348+0000","last_became_active":"2023-09-23T21:13:13.372845+0000","last_became_peered":"2023-09-23T21:13:13.372845+0000","last_unstale":"2023-09-23T21:14:32.786348+0000","last_undegraded":"2023-09-23T21:14:32.786348+0000","last_fullsized":"2023-09-23T21:14:32.786348+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":479,"last_epoch_clean":495,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:05:55.772264+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:05:55.772264+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.001862507,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.10","version":"530'5689","reported_seq":13813,"reported_epoch":530,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T21:14:48.207117+0000","last_change":"2023-09-23T21:13:36.071975+0000","last_active":"2023-09-23T21:14:48.207117+0000","last_peered":"2023-09-23T21:14:48.207117+0000","last_clean":"2023-09-23T21:11:26.716051+0000","last_became_active":"2023-09-23T21:13:33.619010+0000","last_became_peered":"2023-09-23T21:13:33.619010+0000","last_unstale":"2023-09-23T21:14:48.207117+0000","last_undegraded":"2023-09-23T21:13:33.443511+0000","last_fullsized":"2023-09-23T21:13:31.753181+0000","mapping_epoch":502,"log_start":"432'809","ondisk_log_start":"432'809","created":438,"last_epoch_clean":450,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_clean_scrub_stamp":"2023-09-23T21:10:15.663179+0000","objects_scrubbed":0,"log_size":4880,"log_dups_size":0,"ondisk_log_size":4880,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":42106880,"num_objects":643,"num_object_clones":0,"num_object_copies":1929,"num_objects_missing_on_primary":304,"num_objects_missing":304,"num_objects_degraded":1130,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":643,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4325,"num_write_kb":158944,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3718,"num_bytes_recovered":243662848,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7],"avail_no_missing":["1","5","7"],"object_location_counts":[{"shards":"0,4,5,7","objects":1},{"shards":"0,5,7","objects":304},{"shards":"4,5,7","objects":29},{"shards":"4,7","objects":309}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":526,"reported_epoch":529,"state":"active+clean","last_fresh":"2023-09-23T21:14:39.209299+0000","last_change":"2023-09-23T21:12:10.824116+0000","last_active":"2023-09-23T21:14:39.209299+0000","last_peered":"2023-09-23T21:14:39.209299+0000","last_clean":"2023-09-23T21:14:39.209299+0000","last_became_active":"2023-09-23T21:12:10.823849+0000","last_became_peered":"2023-09-23T21:12:10.823849+0000","last_unstale":"2023-09-23T21:14:39.209299+0000","last_undegraded":"2023-09-23T21:14:39.209299+0000","last_fullsized":"2023-09-23T21:14:39.209299+0000","mapping_epoch":470,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:54:13.777603+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":526,"reported_epoch":529,"state":"active+clean","last_fresh":"2023-09-23T21:14:32.787194+0000","last_change":"2023-09-23T21:13:44.413713+0000","last_active":"2023-09-23T21:14:32.787194+0000","last_peered":"2023-09-23T21:14:32.787194+0000","last_clean":"2023-09-23T21:14:32.787194+0000","last_became_active":"2023-09-23T21:13:44.413366+0000","last_became_peered":"2023-09-23T21:13:44.413366+0000","last_unstale":"2023-09-23T21:14:32.787194+0000","last_undegraded":"2023-09-23T21:14:32.787194+0000","last_fullsized":"2023-09-23T21:14:32.787194+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:31:56.349530+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1a","version":"79'161","reported_seq":675,"reported_epoch":529,"state":"active+clean","last_fresh":"2023-09-23T21:14:32.788108+0000","last_change":"2023-09-23T21:12:30.444665+0000","last_active":"2023-09-23T21:14:32.788108+0000","last_peered":"2023-09-23T21:14:32.788108+0000","last_clean":"2023-09-23T21:14:32.788108+0000","last_became_active":"2023-09-23T21:12:30.444326+0000","last_became_peered":"2023-09-23T21:12:30.444326+0000","last_unstale":"2023-09-23T21:14:32.788108+0000","last_undegraded":"2023-09-23T21:14:32.788108+0000","last_fullsized":"2023-09-23T21:14:32.788108+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":479,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:06:51.075448+0000","last_deep_scrub":"79'161","last_deep_scrub_stamp":"2023-09-23T21:05:32.630292+0000","last_clean_scrub_stamp":"2023-09-23T21:06:51.075448+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0054793890000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,4],"acting":[2,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"10.11","version":"530'5717","reported_seq":12549,"reported_epoch":530,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-23T21:14:51.143773+0000","last_change":"2023-09-23T21:13:31.864763+0000","last_active":"2023-09-23T21:14:51.143773+0000","last_peered":"2023-09-23T21:14:51.143773+0000","last_clean":"2023-09-23T21:13:30.058478+0000","last_became_active":"2023-09-23T21:13:31.850860+0000","last_became_peered":"2023-09-23T21:13:31.850860+0000","last_unstale":"2023-09-23T21:14:51.143773+0000","last_undegraded":"2023-09-23T21:14:51.143773+0000","last_fullsized":"2023-09-23T21:14:51.143773+0000","mapping_epoch":502,"log_start":"422'104","ondisk_log_start":"422'104","created":438,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":5,"last_scrub":"441'2974","last_scrub_stamp":"2023-09-23T21:11:06.133218+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_clean_scrub_stamp":"2023-09-23T21:11:06.133218+0000","objects_scrubbed":1646,"log_size":5613,"log_dups_size":0,"ondisk_log_size":5613,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:54:59.366711+0000","scrub_duration":0.3173955,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":36306944,"num_objects":554,"num_object_clones":0,"num_object_copies":1662,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":554,"num_objects_unfound":0,"num_objects_dirty":554,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2746,"num_write_kb":52928,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2152,"num_bytes_recovered":141033472,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[2,5,7],"avail_no_missing":["2","1","5","7"],"object_location_counts":[{"shards":"2,5,7","objects":554}],"blocked_by":[],"up_primary":0,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":39,"reported_epoch":529,"state":"active+clean","last_fresh":"2023-09-23T21:14:32.784544+0000","last_change":"2023-09-23T21:13:30.806084+0000","last_active":"2023-09-23T21:14:32.784544+0000","last_peered":"2023-09-23T21:14:32.784544+0000","last_clean":"2023-09-23T21:14:32.784544+0000","last_became_active":"2023-09-23T21:13:30.805743+0000","last_became_peered":"2023-09-23T21:13:30.805743+0000","last_unstale":"2023-09-23T21:14:32.784544+0000","last_undegraded":"2023-09-23T21:14:32.784544+0000","last_fullsized":"2023-09-23T21:14:32.784544+0000","mapping_epoch":501,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:48:10.849281+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":403,"reported_epoch":529,"state":"active+clean","last_fresh":"2023-09-23T21:14:32.787009+0000","last_change":"2023-09-23T21:13:44.410921+0000","last_active":"2023-09-23T21:14:32.787009+0000","last_peered":"2023-09-23T21:14:32.787009+0000","last_clean":"2023-09-23T21:14:32.787009+0000","last_became_active":"2023-09-23T21:13:44.410633+0000","last_became_peered":"2023-09-23T21:13:44.410633+0000","last_unstale":"2023-09-23T21:14:32.787009+0000","last_undegraded":"2023-09-23T21:14:32.787009+0000","last_fullsized":"2023-09-23T21:14:32.787009+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:57:10.116786+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.15","version":"79'161","reported_seq":556,"reported_epoch":529,"state":"active+clean","last_fresh":"2023-09-23T21:14:32.778548+0000","last_change":"2023-09-23T21:13:38.379453+0000","last_active":"2023-09-23T21:14:32.778548+0000","last_peered":"2023-09-23T21:14:32.778548+0000","last_clean":"2023-09-23T21:14:32.778548+0000","last_became_active":"2023-09-23T21:13:38.376499+0000","last_became_peered":"2023-09-23T21:13:38.376499+0000","last_unstale":"2023-09-23T21:14:32.778548+0000","last_undegraded":"2023-09-23T21:14:32.778548+0000","last_fullsized":"2023-09-23T21:14:32.778548+0000","mapping_epoch":507,"log_start":"0'0","ondisk_log_start":"0'0","created":479,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:09:34.892947+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:09:34.892947+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13418,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":11,"num_read_kb":21,"num_write":6,"num_write_kb":55,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":40253,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":504,"reported_epoch":529,"state":"active+clean","last_fresh":"2023-09-23T21:14:32.787168+0000","last_change":"2023-09-23T21:13:12.235728+0000","last_active":"2023-09-23T21:14:32.787168+0000","last_peered":"2023-09-23T21:14:32.787168+0000","last_clean":"2023-09-23T21:14:32.787168+0000","last_became_active":"2023-09-23T21:13:12.216549+0000","last_became_peered":"2023-09-23T21:13:12.216549+0000","last_unstale":"2023-09-23T21:14:32.787168+0000","last_undegraded":"2023-09-23T21:14:32.787168+0000","last_fullsized":"2023-09-23T21:14:32.787168+0000","mapping_epoch":493,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:39:33.324239+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":568,"reported_epoch":529,"state":"active+clean","last_fresh":"2023-09-23T21:14:32.778231+0000","last_change":"2023-09-23T21:12:10.790349+0000","last_active":"2023-09-23T21:14:32.778231+0000","last_peered":"2023-09-23T21:14:32.778231+0000","last_clean":"2023-09-23T21:14:32.778231+0000","last_became_active":"2023-09-23T21:12:10.789977+0000","last_became_peered":"2023-09-23T21:12:10.789977+0000","last_unstale":"2023-09-23T21:14:32.778231+0000","last_undegraded":"2023-09-23T21:14:32.778231+0000","last_fullsized":"2023-09-23T21:14:32.778231+0000","mapping_epoch":470,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:49:40.191147+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,5],"acting":[3,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"79'161","reported_seq":891,"reported_epoch":529,"state":"active+clean","last_fresh":"2023-09-23T21:14:32.795496+0000","last_change":"2023-09-23T21:11:32.760292+0000","last_active":"2023-09-23T21:14:32.795496+0000","last_peered":"2023-09-23T21:14:32.795496+0000","last_clean":"2023-09-23T21:14:32.795496+0000","last_became_active":"2023-09-23T21:11:32.759909+0000","last_became_peered":"2023-09-23T21:11:32.759909+0000","last_unstale":"2023-09-23T21:14:32.795496+0000","last_undegraded":"2023-09-23T21:14:32.795496+0000","last_fullsized":"2023-09-23T21:14:32.795496+0000","mapping_epoch":457,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":458,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:25.096401+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:25.096401+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:13:09.225685+0000","scrub_duration":0.002859393,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":255,"reported_epoch":530,"state":"active+clean","last_fresh":"2023-09-23T21:14:44.690636+0000","last_change":"2023-09-23T21:12:10.779468+0000","last_active":"2023-09-23T21:14:44.690636+0000","last_peered":"2023-09-23T21:14:44.690636+0000","last_clean":"2023-09-23T21:14:44.690636+0000","last_became_active":"2023-09-23T21:12:10.779282+0000","last_became_peered":"2023-09-23T21:12:10.779282+0000","last_unstale":"2023-09-23T21:14:44.690636+0000","last_undegraded":"2023-09-23T21:14:44.690636+0000","last_fullsized":"2023-09-23T21:14:44.690636+0000","mapping_epoch":470,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:36:35.417986+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":505,"reported_epoch":529,"state":"active+clean","last_fresh":"2023-09-23T21:14:32.778570+0000","last_change":"2023-09-23T21:10:26.758958+0000","last_active":"2023-09-23T21:14:32.778570+0000","last_peered":"2023-09-23T21:14:32.778570+0000","last_clean":"2023-09-23T21:14:32.778570+0000","last_became_active":"2023-09-23T21:10:26.758539+0000","last_became_peered":"2023-09-23T21:10:26.758539+0000","last_unstale":"2023-09-23T21:14:32.778570+0000","last_undegraded":"2023-09-23T21:14:32.778570+0000","last_fullsized":"2023-09-23T21:14:32.778570+0000","mapping_epoch":423,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":424,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:44:42.319189+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.17","version":"79'161","reported_seq":622,"reported_epoch":529,"state":"active+clean","last_fresh":"2023-09-23T21:14:32.781633+0000","last_change":"2023-09-23T21:13:44.416768+0000","last_active":"2023-09-23T21:14:32.781633+0000","last_peered":"2023-09-23T21:14:32.781633+0000","last_clean":"2023-09-23T21:14:32.781633+0000","last_became_active":"2023-09-23T21:13:44.416156+0000","last_became_peered":"2023-09-23T21:13:44.416156+0000","last_unstale":"2023-09-23T21:14:32.781633+0000","last_undegraded":"2023-09-23T21:14:32.781633+0000","last_fullsized":"2023-09-23T21:14:32.781633+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":479,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:25.916867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:25.916867+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.001665323,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":414,"reported_epoch":529,"state":"active+clean","last_fresh":"2023-09-23T21:14:32.780362+0000","last_change":"2023-09-23T21:11:08.766045+0000","last_active":"2023-09-23T21:14:32.780362+0000","last_peered":"2023-09-23T21:14:32.780362+0000","last_clean":"2023-09-23T21:14:32.780362+0000","last_became_active":"2023-09-23T21:11:08.758504+0000","last_became_peered":"2023-09-23T21:11:08.758504+0000","last_unstale":"2023-09-23T21:14:32.780362+0000","last_undegraded":"2023-09-23T21:14:32.780362+0000","last_fullsized":"2023-09-23T21:14:32.780362+0000","mapping_epoch":443,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":444,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:02:19.482574+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":360,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,4],"acting":[3,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":217,"reported_epoch":529,"state":"active+clean","last_fresh":"2023-09-23T21:14:32.798670+0000","last_change":"2023-09-23T21:13:30.821489+0000","last_active":"2023-09-23T21:14:32.798670+0000","last_peered":"2023-09-23T21:14:32.798670+0000","last_clean":"2023-09-23T21:14:32.798670+0000","last_became_active":"2023-09-23T21:13:30.821125+0000","last_became_peered":"2023-09-23T21:13:30.821125+0000","last_unstale":"2023-09-23T21:14:32.798670+0000","last_undegraded":"2023-09-23T21:14:32.798670+0000","last_fullsized":"2023-09-23T21:14:32.798670+0000","mapping_epoch":501,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:47:18.553417+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,5],"acting":[2,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.16","version":"79'161","reported_seq":448,"reported_epoch":529,"state":"active+clean","last_fresh":"2023-09-23T21:14:32.787114+0000","last_change":"2023-09-23T21:13:33.770962+0000","last_active":"2023-09-23T21:14:32.787114+0000","last_peered":"2023-09-23T21:14:32.787114+0000","last_clean":"2023-09-23T21:14:32.787114+0000","last_became_active":"2023-09-23T21:13:33.770407+0000","last_became_peered":"2023-09-23T21:13:33.770407+0000","last_unstale":"2023-09-23T21:14:32.787114+0000","last_undegraded":"2023-09-23T21:14:32.787114+0000","last_fullsized":"2023-09-23T21:14:32.787114+0000","mapping_epoch":504,"log_start":"0'0","ondisk_log_start":"0'0","created":479,"last_epoch_clean":505,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:10:24.268662+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:10:24.268662+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13418,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":11,"num_read_kb":21,"num_write":6,"num_write_kb":55,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":40252,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.14","version":"79'22","reported_seq":93,"reported_epoch":529,"state":"active+clean","last_fresh":"2023-09-23T21:14:32.779681+0000","last_change":"2023-09-23T21:11:27.770162+0000","last_active":"2023-09-23T21:14:32.779681+0000","last_peered":"2023-09-23T21:14:32.779681+0000","last_clean":"2023-09-23T21:14:32.779681+0000","last_became_active":"2023-09-23T21:11:27.752275+0000","last_became_peered":"2023-09-23T21:11:27.752275+0000","last_unstale":"2023-09-23T21:14:32.779681+0000","last_undegraded":"2023-09-23T21:14:32.779681+0000","last_fullsized":"2023-09-23T21:14:32.779681+0000","mapping_epoch":452,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":453,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:05:11.769428+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4734,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,2],"acting":[4,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":458,"reported_epoch":529,"state":"active+clean","last_fresh":"2023-09-23T21:14:32.782066+0000","last_change":"2023-09-23T21:12:10.839271+0000","last_active":"2023-09-23T21:14:32.782066+0000","last_peered":"2023-09-23T21:14:32.782066+0000","last_clean":"2023-09-23T21:14:32.782066+0000","last_became_active":"2023-09-23T21:12:10.839042+0000","last_became_peered":"2023-09-23T21:12:10.839042+0000","last_unstale":"2023-09-23T21:14:32.782066+0000","last_undegraded":"2023-09-23T21:14:32.782066+0000","last_fullsized":"2023-09-23T21:14:32.782066+0000","mapping_epoch":470,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:10:26.414737+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,2],"acting":[4,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.11","version":"79'161","reported_seq":453,"reported_epoch":529,"state":"active+clean","last_fresh":"2023-09-23T21:14:32.782316+0000","last_change":"2023-09-23T21:13:33.767406+0000","last_active":"2023-09-23T21:14:32.782316+0000","last_peered":"2023-09-23T21:14:32.782316+0000","last_clean":"2023-09-23T21:14:32.782316+0000","last_became_active":"2023-09-23T21:13:33.767067+0000","last_became_peered":"2023-09-23T21:13:33.767067+0000","last_unstale":"2023-09-23T21:14:32.782316+0000","last_undegraded":"2023-09-23T21:14:32.782316+0000","last_fullsized":"2023-09-23T21:14:32.782316+0000","mapping_epoch":504,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":505,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:28.756634+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:28.756634+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":23,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":139,"reported_epoch":530,"state":"active+clean","last_fresh":"2023-09-23T21:14:44.710067+0000","last_change":"2023-09-23T21:11:27.777154+0000","last_active":"2023-09-23T21:14:44.710067+0000","last_peered":"2023-09-23T21:14:44.710067+0000","last_clean":"2023-09-23T21:14:44.710067+0000","last_became_active":"2023-09-23T21:11:27.754438+0000","last_became_peered":"2023-09-23T21:11:27.754438+0000","last_unstale":"2023-09-23T21:14:44.710067+0000","last_undegraded":"2023-09-23T21:14:44.710067+0000","last_fullsized":"2023-09-23T21:14:44.710067+0000","mapping_epoch":452,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":453,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:33:02.845882+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":71,"reported_epoch":529,"state":"active+clean","last_fresh":"2023-09-23T21:14:32.795186+0000","last_change":"2023-09-23T21:12:10.764264+0000","last_active":"2023-09-23T21:14:32.795186+0000","last_peered":"2023-09-23T21:14:32.795186+0000","last_clean":"2023-09-23T21:14:32.795186+0000","last_became_active":"2023-09-23T21:12:10.763486+0000","last_became_peered":"2023-09-23T21:12:10.763486+0000","last_unstale":"2023-09-23T21:14:32.795186+0000","last_undegraded":"2023-09-23T21:14:32.795186+0000","last_fullsized":"2023-09-23T21:14:32.795186+0000","mapping_epoch":470,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:32:27.577036+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,7],"acting":[2,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.10","version":"79'161","reported_seq":664,"reported_epoch":529,"state":"active+clean","last_fresh":"2023-09-23T21:14:32.791441+0000","last_change":"2023-09-23T21:11:11.716703+0000","last_active":"2023-09-23T21:14:32.791441+0000","last_peered":"2023-09-23T21:14:32.791441+0000","last_clean":"2023-09-23T21:14:32.791441+0000","last_became_active":"2023-09-23T21:11:11.716385+0000","last_became_peered":"2023-09-23T21:11:11.716385+0000","last_unstale":"2023-09-23T21:14:32.791441+0000","last_undegraded":"2023-09-23T21:14:32.791441+0000","last_fullsized":"2023-09-23T21:14:32.791441+0000","mapping_epoch":446,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":447,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:06:50.336992+0000","last_deep_scrub":"79'161","last_deep_scrub_stamp":"2023-09-23T21:06:50.336992+0000","last_clean_scrub_stamp":"2023-09-23T21:06:50.336992+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:44:29.538890+0000","scrub_duration":0.0026150930000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,7],"acting":[4,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":470,"reported_epoch":529,"state":"active+clean","last_fresh":"2023-09-23T21:14:32.786777+0000","last_change":"2023-09-23T21:13:12.270990+0000","last_active":"2023-09-23T21:14:32.786777+0000","last_peered":"2023-09-23T21:14:32.786777+0000","last_clean":"2023-09-23T21:14:32.786777+0000","last_became_active":"2023-09-23T21:13:12.237201+0000","last_became_peered":"2023-09-23T21:13:12.237201+0000","last_unstale":"2023-09-23T21:14:32.786777+0000","last_undegraded":"2023-09-23T21:14:32.786777+0000","last_fullsized":"2023-09-23T21:14:32.786777+0000","mapping_epoch":493,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:59:54.523877+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":282,"reported_epoch":529,"state":"active+clean","last_fresh":"2023-09-23T21:14:34.208647+0000","last_change":"2023-09-23T21:10:26.758671+0000","last_active":"2023-09-23T21:14:34.208647+0000","last_peered":"2023-09-23T21:14:34.208647+0000","last_clean":"2023-09-23T21:14:34.208647+0000","last_became_active":"2023-09-23T21:10:26.758244+0000","last_became_peered":"2023-09-23T21:10:26.758244+0000","last_unstale":"2023-09-23T21:14:34.208647+0000","last_undegraded":"2023-09-23T21:14:34.208647+0000","last_fullsized":"2023-09-23T21:14:34.208647+0000","mapping_epoch":423,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":424,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:14:02.735609+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.18","version":"530'5771","reported_seq":6858,"reported_epoch":530,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T21:14:51.281443+0000","last_change":"2023-09-23T21:13:13.412097+0000","last_active":"2023-09-23T21:14:51.281443+0000","last_peered":"2023-09-23T21:14:51.281443+0000","last_clean":"2023-09-23T21:11:45.720015+0000","last_became_active":"2023-09-23T21:13:13.395632+0000","last_became_peered":"2023-09-23T21:13:13.395632+0000","last_unstale":"2023-09-23T21:14:51.281443+0000","last_undegraded":"2023-09-23T21:13:13.220971+0000","last_fullsized":"2023-09-23T21:13:13.183825+0000","mapping_epoch":494,"log_start":"422'115","ondisk_log_start":"422'115","created":438,"last_epoch_clean":439,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_clean_scrub_stamp":"2023-09-23T21:10:15.663179+0000","objects_scrubbed":0,"log_size":5656,"log_dups_size":0,"ondisk_log_size":5656,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":40468480,"num_objects":618,"num_object_clones":0,"num_object_copies":1854,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":661,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":618,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4422,"num_write_kb":161248,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":231,"num_bytes_recovered":15138816,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,7],"acting":[2,7],"avail_no_missing":["2","7"],"object_location_counts":[{"shards":"2,7","objects":584}],"blocked_by":[],"up_primary":0,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.13","version":"79'161","reported_seq":607,"reported_epoch":529,"state":"active+clean","last_fresh":"2023-09-23T21:14:34.208573+0000","last_change":"2023-09-23T21:13:14.933096+0000","last_active":"2023-09-23T21:14:34.208573+0000","last_peered":"2023-09-23T21:14:34.208573+0000","last_clean":"2023-09-23T21:14:34.208573+0000","last_became_active":"2023-09-23T21:13:14.932703+0000","last_became_peered":"2023-09-23T21:13:14.932703+0000","last_unstale":"2023-09-23T21:14:34.208573+0000","last_undegraded":"2023-09-23T21:14:34.208573+0000","last_fullsized":"2023-09-23T21:14:34.208573+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:36.420362+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:36.420362+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:29:42.163911+0000","scrub_duration":0.028838444000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":521,"reported_epoch":529,"state":"active+clean","last_fresh":"2023-09-23T21:14:32.778534+0000","last_change":"2023-09-23T21:13:12.216467+0000","last_active":"2023-09-23T21:14:32.778534+0000","last_peered":"2023-09-23T21:14:32.778534+0000","last_clean":"2023-09-23T21:14:32.778534+0000","last_became_active":"2023-09-23T21:13:12.215259+0000","last_became_peered":"2023-09-23T21:13:12.215259+0000","last_unstale":"2023-09-23T21:14:32.778534+0000","last_undegraded":"2023-09-23T21:14:32.778534+0000","last_fullsized":"2023-09-23T21:14:32.778534+0000","mapping_epoch":493,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:35:33.987839+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":39,"reported_epoch":529,"state":"active+clean","last_fresh":"2023-09-23T21:14:32.786859+0000","last_change":"2023-09-23T21:13:30.804108+0000","last_active":"2023-09-23T21:14:32.786859+0000","last_peered":"2023-09-23T21:14:32.786859+0000","last_clean":"2023-09-23T21:14:32.786859+0000","last_became_active":"2023-09-23T21:13:30.796505+0000","last_became_peered":"2023-09-23T21:13:30.796505+0000","last_unstale":"2023-09-23T21:14:32.786859+0000","last_undegraded":"2023-09-23T21:14:32.786859+0000","last_fullsized":"2023-09-23T21:14:32.786859+0000","mapping_epoch":501,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:19:33.508496+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.19","version":"530'5950","reported_seq":7051,"reported_epoch":530,"state":"active+clean","last_fresh":"2023-09-23T21:14:51.278864+0000","last_change":"2023-09-23T21:10:58.790716+0000","last_active":"2023-09-23T21:14:51.278864+0000","last_peered":"2023-09-23T21:14:51.278864+0000","last_clean":"2023-09-23T21:14:51.278864+0000","last_became_active":"2023-09-23T21:10:58.787975+0000","last_became_peered":"2023-09-23T21:10:58.787975+0000","last_unstale":"2023-09-23T21:14:51.278864+0000","last_undegraded":"2023-09-23T21:14:51.278864+0000","last_fullsized":"2023-09-23T21:14:51.278864+0000","mapping_epoch":438,"log_start":"425'500","ondisk_log_start":"425'500","created":438,"last_epoch_clean":439,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_clean_scrub_stamp":"2023-09-23T21:10:15.663179+0000","objects_scrubbed":0,"log_size":5450,"log_dups_size":0,"ondisk_log_size":5450,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34635776,"num_objects":529,"num_object_clones":0,"num_object_copies":1587,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":529,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4563,"num_write_kb":162912,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":237,"num_bytes_recovered":15499264,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.12","version":"79'161","reported_seq":764,"reported_epoch":530,"state":"active+clean","last_fresh":"2023-09-23T21:14:44.691457+0000","last_change":"2023-09-23T21:13:15.206574+0000","last_active":"2023-09-23T21:14:44.691457+0000","last_peered":"2023-09-23T21:14:44.691457+0000","last_clean":"2023-09-23T21:14:44.691457+0000","last_became_active":"2023-09-23T21:13:15.206186+0000","last_became_peered":"2023-09-23T21:13:15.206186+0000","last_unstale":"2023-09-23T21:14:44.691457+0000","last_undegraded":"2023-09-23T21:14:44.691457+0000","last_fullsized":"2023-09-23T21:14:44.691457+0000","mapping_epoch":496,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":497,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:05:56.785173+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:05:56.785173+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:34:29.221809+0000","scrub_duration":0.0042114370000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,0],"acting":[5,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":49,"reported_epoch":529,"state":"active+clean","last_fresh":"2023-09-23T21:14:32.784636+0000","last_change":"2023-09-23T21:13:38.048791+0000","last_active":"2023-09-23T21:14:32.784636+0000","last_peered":"2023-09-23T21:14:32.784636+0000","last_clean":"2023-09-23T21:14:32.784636+0000","last_became_active":"2023-09-23T21:13:30.801386+0000","last_became_peered":"2023-09-23T21:13:30.801386+0000","last_unstale":"2023-09-23T21:14:32.784636+0000","last_undegraded":"2023-09-23T21:14:32.784636+0000","last_fullsized":"2023-09-23T21:14:32.784636+0000","mapping_epoch":501,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:52:49.170414+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":367,"reported_epoch":529,"state":"active+clean","last_fresh":"2023-09-23T21:14:39.209923+0000","last_change":"2023-09-23T21:12:10.765780+0000","last_active":"2023-09-23T21:14:39.209923+0000","last_peered":"2023-09-23T21:14:39.209923+0000","last_clean":"2023-09-23T21:14:39.209923+0000","last_became_active":"2023-09-23T21:12:10.765372+0000","last_became_peered":"2023-09-23T21:12:10.765372+0000","last_unstale":"2023-09-23T21:14:39.209923+0000","last_undegraded":"2023-09-23T21:14:39.209923+0000","last_fullsized":"2023-09-23T21:14:39.209923+0000","mapping_epoch":470,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:56:08.446099+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.6","version":"530'5927","reported_seq":8846,"reported_epoch":530,"state":"active+clean","last_fresh":"2023-09-23T21:14:49.955919+0000","last_change":"2023-09-23T21:13:39.390285+0000","last_active":"2023-09-23T21:14:49.955919+0000","last_peered":"2023-09-23T21:14:49.955919+0000","last_clean":"2023-09-23T21:14:49.955919+0000","last_became_active":"2023-09-23T21:13:39.378284+0000","last_became_peered":"2023-09-23T21:13:39.378284+0000","last_unstale":"2023-09-23T21:14:49.955919+0000","last_undegraded":"2023-09-23T21:14:49.955919+0000","last_fullsized":"2023-09-23T21:14:49.955919+0000","mapping_epoch":508,"log_start":"434'1068","ondisk_log_start":"434'1068","created":416,"last_epoch_clean":509,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_clean_scrub_stamp":"2023-09-23T21:10:15.663179+0000","objects_scrubbed":0,"log_size":4859,"log_dups_size":0,"ondisk_log_size":4859,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":35717120,"num_objects":545,"num_object_clones":0,"num_object_copies":1635,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":545,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4521,"num_write_kb":162112,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2399,"num_bytes_recovered":113967104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,4],"acting":[0,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.d","version":"79'161","reported_seq":684,"reported_epoch":530,"state":"active+clean","last_fresh":"2023-09-23T21:14:44.709352+0000","last_change":"2023-09-23T21:12:30.444461+0000","last_active":"2023-09-23T21:14:44.709352+0000","last_peered":"2023-09-23T21:14:44.709352+0000","last_clean":"2023-09-23T21:14:44.709352+0000","last_became_active":"2023-09-23T21:12:30.443979+0000","last_became_peered":"2023-09-23T21:12:30.443979+0000","last_unstale":"2023-09-23T21:14:44.709352+0000","last_undegraded":"2023-09-23T21:14:44.709352+0000","last_fullsized":"2023-09-23T21:14:44.709352+0000","mapping_epoch":470,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:09:36.858077+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:09:36.858077+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0024987500000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":177,"reported_epoch":530,"state":"active+clean","last_fresh":"2023-09-23T21:14:44.709702+0000","last_change":"2023-09-23T21:13:59.872834+0000","last_active":"2023-09-23T21:14:44.709702+0000","last_peered":"2023-09-23T21:14:44.709702+0000","last_clean":"2023-09-23T21:14:44.709702+0000","last_became_active":"2023-09-23T21:13:30.770556+0000","last_became_peered":"2023-09-23T21:13:30.770556+0000","last_unstale":"2023-09-23T21:14:44.709702+0000","last_undegraded":"2023-09-23T21:14:44.709702+0000","last_fullsized":"2023-09-23T21:14:44.709702+0000","mapping_epoch":501,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":521,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:57:55.460052+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":132,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":238,"reported_epoch":530,"state":"active+clean","last_fresh":"2023-09-23T21:14:44.690452+0000","last_change":"2023-09-23T21:10:26.750725+0000","last_active":"2023-09-23T21:14:44.690452+0000","last_peered":"2023-09-23T21:14:44.690452+0000","last_clean":"2023-09-23T21:14:44.690452+0000","last_became_active":"2023-09-23T21:10:26.750307+0000","last_became_peered":"2023-09-23T21:10:26.750307+0000","last_unstale":"2023-09-23T21:14:44.690452+0000","last_undegraded":"2023-09-23T21:14:44.690452+0000","last_fullsized":"2023-09-23T21:14:44.690452+0000","mapping_epoch":423,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":424,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:06:22.040548+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,4],"acting":[5,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.7","version":"530'5746","reported_seq":5943,"reported_epoch":530,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T21:14:49.150940+0000","last_change":"2023-09-23T21:13:38.417178+0000","last_active":"2023-09-23T21:14:49.150940+0000","last_peered":"2023-09-23T21:14:49.150940+0000","last_clean":"2023-09-23T21:11:26.699871+0000","last_became_active":"2023-09-23T21:13:38.408523+0000","last_became_peered":"2023-09-23T21:13:38.408523+0000","last_unstale":"2023-09-23T21:14:49.150940+0000","last_undegraded":"2023-09-23T21:13:38.369833+0000","last_fullsized":"2023-09-23T21:13:38.358198+0000","mapping_epoch":507,"log_start":"435'1485","ondisk_log_start":"435'1485","created":416,"last_epoch_clean":439,"parent":"0.0","parent_split_bits":0,"last_scrub":"441'2973","last_scrub_stamp":"2023-09-23T21:11:06.079326+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_clean_scrub_stamp":"2023-09-23T21:11:06.079326+0000","objects_scrubbed":1647,"log_size":4261,"log_dups_size":0,"ondisk_log_size":4261,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:27:59.805526+0000","scrub_duration":0.44319714500000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":39124992,"num_objects":597,"num_object_clones":0,"num_object_copies":1791,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1165,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":597,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2778,"num_write_kb":55296,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,2],"acting":[3,5],"avail_no_missing":["3","5"],"object_location_counts":[{"shards":"3,4,5","objects":298},{"shards":"3,5","objects":299}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.c","version":"79'161","reported_seq":892,"reported_epoch":529,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-23T21:14:32.779275+0000","last_change":"2023-09-23T21:13:13.402530+0000","last_active":"2023-09-23T21:14:32.779275+0000","last_peered":"2023-09-23T21:14:32.779275+0000","last_clean":"2023-09-23T21:11:23.200766+0000","last_became_active":"2023-09-23T21:13:13.401526+0000","last_became_peered":"2023-09-23T21:13:13.401526+0000","last_unstale":"2023-09-23T21:14:32.779275+0000","last_undegraded":"2023-09-23T21:13:13.204992+0000","last_fullsized":"2023-09-23T21:14:32.779275+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":420,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:22.040907+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:22.040907+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.013021176000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":163840,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":163840,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":["3","0","5"],"object_location_counts":[{"shards":"0,3,4","objects":-1},{"shards":"0,3,5","objects":1}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":163,"reported_epoch":530,"state":"active+clean","last_fresh":"2023-09-23T21:14:44.710903+0000","last_change":"2023-09-23T21:13:59.884113+0000","last_active":"2023-09-23T21:14:44.710903+0000","last_peered":"2023-09-23T21:14:44.710903+0000","last_clean":"2023-09-23T21:14:44.710903+0000","last_became_active":"2023-09-23T21:13:30.806778+0000","last_became_peered":"2023-09-23T21:13:30.806778+0000","last_unstale":"2023-09-23T21:14:44.710903+0000","last_undegraded":"2023-09-23T21:14:44.710903+0000","last_fullsized":"2023-09-23T21:14:44.710903+0000","mapping_epoch":501,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":521,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:36:15.284465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":492,"reported_epoch":529,"state":"active+clean","last_fresh":"2023-09-23T21:14:32.778620+0000","last_change":"2023-09-23T21:13:15.152792+0000","last_active":"2023-09-23T21:14:32.778620+0000","last_peered":"2023-09-23T21:14:32.778620+0000","last_clean":"2023-09-23T21:14:32.778620+0000","last_became_active":"2023-09-23T21:13:14.515084+0000","last_became_peered":"2023-09-23T21:13:14.515084+0000","last_unstale":"2023-09-23T21:14:32.778620+0000","last_undegraded":"2023-09-23T21:14:32.778620+0000","last_fullsized":"2023-09-23T21:14:32.778620+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:08:16.702072+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.4","version":"530'5667","reported_seq":12614,"reported_epoch":530,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T21:14:49.072325+0000","last_change":"2023-09-23T21:13:30.847170+0000","last_active":"2023-09-23T21:14:49.072325+0000","last_peered":"2023-09-23T21:14:49.072325+0000","last_clean":"2023-09-23T21:11:06.743842+0000","last_became_active":"2023-09-23T21:13:30.834806+0000","last_became_peered":"2023-09-23T21:13:30.834806+0000","last_unstale":"2023-09-23T21:14:49.072325+0000","last_undegraded":"2023-09-23T21:13:30.760230+0000","last_fullsized":"2023-09-23T21:13:30.749974+0000","mapping_epoch":501,"log_start":"434'1331","ondisk_log_start":"434'1331","created":416,"last_epoch_clean":439,"parent":"0.0","parent_split_bits":0,"last_scrub":"440'2861","last_scrub_stamp":"2023-09-23T21:11:03.842926+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_clean_scrub_stamp":"2023-09-23T21:11:03.842926+0000","objects_scrubbed":1526,"log_size":4336,"log_dups_size":0,"ondisk_log_size":4336,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:32:29.379756+0000","scrub_duration":0.39378383900000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":42860544,"num_objects":654,"num_object_clones":0,"num_object_copies":1962,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1130,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":654,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2812,"num_write_kb":62080,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2723,"num_bytes_recovered":163577856,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[3,4],"avail_no_missing":["3","4"],"object_location_counts":[{"shards":"1,2,3,4","objects":360},{"shards":"3,4","objects":294}],"blocked_by":[],"up_primary":0,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.f","version":"79'161","reported_seq":818,"reported_epoch":530,"state":"active+clean","last_fresh":"2023-09-23T21:14:44.690862+0000","last_change":"2023-09-23T21:11:30.750338+0000","last_active":"2023-09-23T21:14:44.690862+0000","last_peered":"2023-09-23T21:14:44.690862+0000","last_clean":"2023-09-23T21:14:44.690862+0000","last_became_active":"2023-09-23T21:11:30.749953+0000","last_became_peered":"2023-09-23T21:11:30.749953+0000","last_unstale":"2023-09-23T21:14:44.690862+0000","last_undegraded":"2023-09-23T21:14:44.690862+0000","last_fullsized":"2023-09-23T21:14:44.690862+0000","mapping_epoch":455,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":456,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:27.880776+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:27.880776+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:07:10.405723+0000","scrub_duration":0.0014004899999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,3],"acting":[5,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":254,"reported_epoch":530,"state":"active+clean","last_fresh":"2023-09-23T21:14:44.692050+0000","last_change":"2023-09-23T21:10:26.765068+0000","last_active":"2023-09-23T21:14:44.692050+0000","last_peered":"2023-09-23T21:14:44.692050+0000","last_clean":"2023-09-23T21:14:44.692050+0000","last_became_active":"2023-09-23T21:10:26.758364+0000","last_became_peered":"2023-09-23T21:10:26.758364+0000","last_unstale":"2023-09-23T21:14:44.692050+0000","last_undegraded":"2023-09-23T21:14:44.692050+0000","last_fullsized":"2023-09-23T21:14:44.692050+0000","mapping_epoch":423,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":424,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:13:21.057529+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":280,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":470,"reported_epoch":530,"state":"active+clean","last_fresh":"2023-09-23T21:14:44.691099+0000","last_change":"2023-09-23T21:13:30.770746+0000","last_active":"2023-09-23T21:14:44.691099+0000","last_peered":"2023-09-23T21:14:44.691099+0000","last_clean":"2023-09-23T21:14:44.691099+0000","last_became_active":"2023-09-23T21:13:30.770437+0000","last_became_peered":"2023-09-23T21:13:30.770437+0000","last_unstale":"2023-09-23T21:14:44.691099+0000","last_undegraded":"2023-09-23T21:14:44.691099+0000","last_fullsized":"2023-09-23T21:14:44.691099+0000","mapping_epoch":501,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:33:48.325730+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,0],"acting":[5,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.5","version":"530'5743","reported_seq":5932,"reported_epoch":530,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T21:14:51.277449+0000","last_change":"2023-09-23T21:13:13.436794+0000","last_active":"2023-09-23T21:14:51.277449+0000","last_peered":"2023-09-23T21:14:51.277449+0000","last_clean":"2023-09-23T21:11:45.768408+0000","last_became_active":"2023-09-23T21:13:13.424939+0000","last_became_peered":"2023-09-23T21:13:13.424939+0000","last_unstale":"2023-09-23T21:14:51.277449+0000","last_undegraded":"2023-09-23T21:13:13.225067+0000","last_fullsized":"2023-09-23T21:13:13.198194+0000","mapping_epoch":494,"log_start":"422'117","ondisk_log_start":"422'117","created":416,"last_epoch_clean":439,"parent":"0.0","parent_split_bits":0,"last_scrub":"440'2840","last_scrub_stamp":"2023-09-23T21:11:03.208833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_clean_scrub_stamp":"2023-09-23T21:11:03.208833+0000","objects_scrubbed":1498,"log_size":5626,"log_dups_size":0,"ondisk_log_size":5626,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:56:40.574477+0000","scrub_duration":0.45608471699999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":40632320,"num_objects":620,"num_object_clones":0,"num_object_copies":1860,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":644,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":620,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2912,"num_write_kb":65088,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,3],"acting":[2,3],"avail_no_missing":["2","3"],"object_location_counts":[{"shards":"2,3","objects":574}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.e","version":"79'161","reported_seq":698,"reported_epoch":529,"state":"active+clean","last_fresh":"2023-09-23T21:14:32.779595+0000","last_change":"2023-09-23T21:12:30.440952+0000","last_active":"2023-09-23T21:14:32.779595+0000","last_peered":"2023-09-23T21:14:32.779595+0000","last_clean":"2023-09-23T21:14:32.779595+0000","last_became_active":"2023-09-23T21:12:30.440463+0000","last_became_peered":"2023-09-23T21:12:30.440463+0000","last_unstale":"2023-09-23T21:14:32.779595+0000","last_undegraded":"2023-09-23T21:14:32.779595+0000","last_fullsized":"2023-09-23T21:14:32.779595+0000","mapping_epoch":455,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:10:39.567232+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:10:39.567232+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.005229694,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":327,"reported_epoch":529,"state":"active+clean","last_fresh":"2023-09-23T21:14:32.779627+0000","last_change":"2023-09-23T21:13:13.402418+0000","last_active":"2023-09-23T21:14:32.779627+0000","last_peered":"2023-09-23T21:14:32.779627+0000","last_clean":"2023-09-23T21:14:32.779627+0000","last_became_active":"2023-09-23T21:13:13.400984+0000","last_became_peered":"2023-09-23T21:13:13.400984+0000","last_unstale":"2023-09-23T21:14:32.779627+0000","last_undegraded":"2023-09-23T21:14:32.779627+0000","last_fullsized":"2023-09-23T21:14:32.779627+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":495,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:28:01.411191+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,0],"acting":[3,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":502,"reported_epoch":529,"state":"active+clean","last_fresh":"2023-09-23T21:14:32.786921+0000","last_change":"2023-09-23T21:13:14.339670+0000","last_active":"2023-09-23T21:14:32.786921+0000","last_peered":"2023-09-23T21:14:32.786921+0000","last_clean":"2023-09-23T21:14:32.786921+0000","last_became_active":"2023-09-23T21:13:14.339341+0000","last_became_peered":"2023-09-23T21:13:14.339341+0000","last_unstale":"2023-09-23T21:14:32.786921+0000","last_undegraded":"2023-09-23T21:14:32.786921+0000","last_fullsized":"2023-09-23T21:14:32.786921+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:23:19.411368+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.2","version":"530'5732","reported_seq":8379,"reported_epoch":530,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T21:14:48.333604+0000","last_change":"2023-09-23T21:13:13.407366+0000","last_active":"2023-09-23T21:14:48.333604+0000","last_peered":"2023-09-23T21:14:48.333604+0000","last_clean":"2023-09-23T21:11:45.752822+0000","last_became_active":"2023-09-23T21:13:13.398616+0000","last_became_peered":"2023-09-23T21:13:13.398616+0000","last_unstale":"2023-09-23T21:14:48.333604+0000","last_undegraded":"2023-09-23T21:13:13.289815+0000","last_fullsized":"2023-09-23T21:13:13.190270+0000","mapping_epoch":494,"log_start":"434'904","ondisk_log_start":"434'904","created":416,"last_epoch_clean":439,"parent":"0.0","parent_split_bits":0,"last_scrub":"440'3007","last_scrub_stamp":"2023-09-23T21:11:05.553661+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_clean_scrub_stamp":"2023-09-23T21:11:05.553661+0000","objects_scrubbed":1622,"log_size":4828,"log_dups_size":0,"ondisk_log_size":4828,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:51:21.839818+0000","scrub_duration":0.39156952499999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":42729472,"num_objects":652,"num_object_clones":0,"num_object_copies":1956,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1943,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":652,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2730,"num_write_kb":56320,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1222,"num_bytes_recovered":80084992,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,3],"avail_no_missing":["4","1"],"object_location_counts":[{"shards":"0,1,4","objects":617},{"shards":"1,4","objects":34},{"shards":"3,4","objects":1}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.9","version":"79'161","reported_seq":683,"reported_epoch":529,"state":"active+clean","last_fresh":"2023-09-23T21:14:32.788694+0000","last_change":"2023-09-23T21:12:30.436118+0000","last_active":"2023-09-23T21:14:32.788694+0000","last_peered":"2023-09-23T21:14:32.788694+0000","last_clean":"2023-09-23T21:14:32.788694+0000","last_became_active":"2023-09-23T21:12:30.435779+0000","last_became_peered":"2023-09-23T21:12:30.435779+0000","last_unstale":"2023-09-23T21:14:32.788694+0000","last_undegraded":"2023-09-23T21:14:32.788694+0000","last_fullsized":"2023-09-23T21:14:32.788694+0000","mapping_epoch":433,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:10:37.696951+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:10:37.696951+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.001386504,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,3],"acting":[2,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":88,"reported_epoch":529,"state":"active+clean","last_fresh":"2023-09-23T21:14:32.787076+0000","last_change":"2023-09-23T21:13:30.793273+0000","last_active":"2023-09-23T21:14:32.787076+0000","last_peered":"2023-09-23T21:14:32.787076+0000","last_clean":"2023-09-23T21:14:32.787076+0000","last_became_active":"2023-09-23T21:13:30.791269+0000","last_became_peered":"2023-09-23T21:13:30.791269+0000","last_unstale":"2023-09-23T21:14:32.787076+0000","last_undegraded":"2023-09-23T21:14:32.787076+0000","last_fullsized":"2023-09-23T21:14:32.787076+0000","mapping_epoch":501,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:38:07.142290+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":103,"reported_epoch":529,"state":"active+clean","last_fresh":"2023-09-23T21:14:32.779354+0000","last_change":"2023-09-23T21:12:14.313466+0000","last_active":"2023-09-23T21:14:32.779354+0000","last_peered":"2023-09-23T21:14:32.779354+0000","last_clean":"2023-09-23T21:14:32.779354+0000","last_became_active":"2023-09-23T21:12:14.209837+0000","last_became_peered":"2023-09-23T21:12:14.209837+0000","last_unstale":"2023-09-23T21:14:32.779354+0000","last_undegraded":"2023-09-23T21:14:32.779354+0000","last_fullsized":"2023-09-23T21:14:32.779354+0000","mapping_epoch":470,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":474,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:09:40.375241+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,7],"acting":[4,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.3","version":"530'5873","reported_seq":8006,"reported_epoch":530,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T21:14:51.286043+0000","last_change":"2023-09-23T21:13:45.480705+0000","last_active":"2023-09-23T21:14:51.286043+0000","last_peered":"2023-09-23T21:14:51.286043+0000","last_clean":"2023-09-23T21:11:26.700990+0000","last_became_active":"2023-09-23T21:13:45.471702+0000","last_became_peered":"2023-09-23T21:13:45.471702+0000","last_unstale":"2023-09-23T21:14:51.286043+0000","last_undegraded":"2023-09-23T21:13:45.412179+0000","last_fullsized":"2023-09-23T21:13:45.401701+0000","mapping_epoch":513,"log_start":"422'359","ondisk_log_start":"422'359","created":416,"last_epoch_clean":439,"parent":"0.0","parent_split_bits":0,"last_scrub":"440'2873","last_scrub_stamp":"2023-09-23T21:11:02.114783+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_clean_scrub_stamp":"2023-09-23T21:11:02.114783+0000","objects_scrubbed":1532,"log_size":5514,"log_dups_size":0,"ondisk_log_size":5514,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:42:48.549370+0000","scrub_duration":0.32435022600000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":36831232,"num_objects":562,"num_object_clones":0,"num_object_copies":1686,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1184,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":562,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3006,"num_write_kb":65152,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":865,"num_bytes_recovered":56688640,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5],"avail_no_missing":["2","5"],"object_location_counts":[{"shards":"0,2,4,5","objects":270},{"shards":"2,4,5","objects":48},{"shards":"2,5","objects":243}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.8","version":"79'161","reported_seq":650,"reported_epoch":529,"state":"active+clean","last_fresh":"2023-09-23T21:14:32.788739+0000","last_change":"2023-09-23T21:12:30.433561+0000","last_active":"2023-09-23T21:14:32.788739+0000","last_peered":"2023-09-23T21:14:32.788739+0000","last_clean":"2023-09-23T21:14:32.788739+0000","last_became_active":"2023-09-23T21:12:30.433217+0000","last_became_peered":"2023-09-23T21:12:30.433217+0000","last_unstale":"2023-09-23T21:14:32.788739+0000","last_undegraded":"2023-09-23T21:14:32.788739+0000","last_fullsized":"2023-09-23T21:14:32.788739+0000","mapping_epoch":359,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:06:52.098882+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:06:52.098882+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0092173299999999993,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":131344,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":106,"reported_epoch":529,"state":"active+clean","last_fresh":"2023-09-23T21:14:32.780493+0000","last_change":"2023-09-23T21:12:10.823380+0000","last_active":"2023-09-23T21:14:32.780493+0000","last_peered":"2023-09-23T21:14:32.780493+0000","last_clean":"2023-09-23T21:14:32.780493+0000","last_became_active":"2023-09-23T21:12:10.823044+0000","last_became_peered":"2023-09-23T21:12:10.823044+0000","last_unstale":"2023-09-23T21:14:32.780493+0000","last_undegraded":"2023-09-23T21:14:32.780493+0000","last_fullsized":"2023-09-23T21:14:32.780493+0000","mapping_epoch":470,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:53:29.846536+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,5],"acting":[4,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":107,"reported_epoch":529,"state":"active+clean","last_fresh":"2023-09-23T21:14:32.778704+0000","last_change":"2023-09-23T21:13:30.772451+0000","last_active":"2023-09-23T21:14:32.778704+0000","last_peered":"2023-09-23T21:14:32.778704+0000","last_clean":"2023-09-23T21:14:32.778704+0000","last_became_active":"2023-09-23T21:13:30.772040+0000","last_became_peered":"2023-09-23T21:13:30.772040+0000","last_unstale":"2023-09-23T21:14:32.778704+0000","last_undegraded":"2023-09-23T21:14:32.778704+0000","last_fullsized":"2023-09-23T21:14:32.778704+0000","mapping_epoch":501,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:24:49.246727+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,0],"acting":[4,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.0","version":"530'5707","reported_seq":11241,"reported_epoch":530,"state":"active+recovering+undersized+remapped","last_fresh":"2023-09-23T21:14:48.310131+0000","last_change":"2023-09-23T21:14:02.130787+0000","last_active":"2023-09-23T21:14:48.310131+0000","last_peered":"2023-09-23T21:14:48.310131+0000","last_clean":"2023-09-23T21:11:26.716395+0000","last_became_active":"2023-09-23T21:13:34.557827+0000","last_became_peered":"2023-09-23T21:13:34.557827+0000","last_unstale":"2023-09-23T21:14:48.310131+0000","last_undegraded":"2023-09-23T21:14:48.310131+0000","last_fullsized":"2023-09-23T21:13:31.752840+0000","mapping_epoch":502,"log_start":"434'931","ondisk_log_start":"434'931","created":416,"last_epoch_clean":439,"parent":"0.0","parent_split_bits":0,"last_scrub":"440'2961","last_scrub_stamp":"2023-09-23T21:11:04.564028+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_clean_scrub_stamp":"2023-09-23T21:11:04.564028+0000","objects_scrubbed":1608,"log_size":4776,"log_dups_size":0,"ondisk_log_size":4776,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:43:09.815808+0000","scrub_duration":0.41858158299999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":42401792,"num_objects":647,"num_object_clones":0,"num_object_copies":1941,"num_objects_missing_on_primary":335,"num_objects_missing":335,"num_objects_degraded":0,"num_objects_misplaced":510,"num_objects_unfound":0,"num_objects_dirty":647,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2749,"num_write_kb":57216,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1836,"num_bytes_recovered":120258560,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,0],"avail_no_missing":["1","7"],"object_location_counts":[{"shards":"0,1,4,7","objects":1},{"shards":"0,1,7","objects":184},{"shards":"0,4","objects":136},{"shards":"1,4,7","objects":326}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.b","version":"79'161","reported_seq":533,"reported_epoch":529,"state":"active+clean","last_fresh":"2023-09-23T21:14:32.786650+0000","last_change":"2023-09-23T21:13:13.373313+0000","last_active":"2023-09-23T21:14:32.786650+0000","last_peered":"2023-09-23T21:14:32.786650+0000","last_clean":"2023-09-23T21:14:32.786650+0000","last_became_active":"2023-09-23T21:13:13.372328+0000","last_became_peered":"2023-09-23T21:13:13.372328+0000","last_unstale":"2023-09-23T21:14:32.786650+0000","last_undegraded":"2023-09-23T21:14:32.786650+0000","last_fullsized":"2023-09-23T21:14:32.786650+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":495,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:05:55.772264+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:05:55.772264+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.001862507,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":87,"reported_epoch":529,"state":"active+clean","last_fresh":"2023-09-23T21:14:32.778447+0000","last_change":"2023-09-23T21:11:27.754717+0000","last_active":"2023-09-23T21:14:32.778447+0000","last_peered":"2023-09-23T21:14:32.778447+0000","last_clean":"2023-09-23T21:14:32.778447+0000","last_became_active":"2023-09-23T21:11:27.754376+0000","last_became_peered":"2023-09-23T21:11:27.754376+0000","last_unstale":"2023-09-23T21:14:32.778447+0000","last_undegraded":"2023-09-23T21:14:32.778447+0000","last_fullsized":"2023-09-23T21:14:32.778447+0000","mapping_epoch":452,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":453,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:26:41.734541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":461,"reported_epoch":529,"state":"active+clean","last_fresh":"2023-09-23T21:14:32.780231+0000","last_change":"2023-09-23T21:12:10.811698+0000","last_active":"2023-09-23T21:14:32.780231+0000","last_peered":"2023-09-23T21:14:32.780231+0000","last_clean":"2023-09-23T21:14:32.780231+0000","last_became_active":"2023-09-23T21:12:10.789656+0000","last_became_peered":"2023-09-23T21:12:10.789656+0000","last_unstale":"2023-09-23T21:14:32.780231+0000","last_undegraded":"2023-09-23T21:14:32.780231+0000","last_fullsized":"2023-09-23T21:14:32.780231+0000","mapping_epoch":470,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:39:15.112091+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,5],"acting":[4,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.1","version":"530'5647","reported_seq":12063,"reported_epoch":530,"state":"active+clean","last_fresh":"2023-09-23T21:14:48.219080+0000","last_change":"2023-09-23T21:13:53.483736+0000","last_active":"2023-09-23T21:14:48.219080+0000","last_peered":"2023-09-23T21:14:48.219080+0000","last_clean":"2023-09-23T21:14:48.219080+0000","last_became_active":"2023-09-23T21:13:53.472685+0000","last_became_peered":"2023-09-23T21:13:53.472685+0000","last_unstale":"2023-09-23T21:14:48.219080+0000","last_undegraded":"2023-09-23T21:14:48.219080+0000","last_fullsized":"2023-09-23T21:14:48.219080+0000","mapping_epoch":519,"log_start":"428'735","ondisk_log_start":"428'735","created":416,"last_epoch_clean":520,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_clean_scrub_stamp":"2023-09-23T21:10:15.663179+0000","objects_scrubbed":0,"log_size":4912,"log_dups_size":0,"ondisk_log_size":4912,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":42270720,"num_objects":645,"num_object_clones":0,"num_object_copies":1935,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":645,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4299,"num_write_kb":158208,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3005,"num_bytes_recovered":156499968,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.a","version":"79'161","reported_seq":676,"reported_epoch":529,"state":"active+clean","last_fresh":"2023-09-23T21:14:32.798492+0000","last_change":"2023-09-23T21:12:30.437790+0000","last_active":"2023-09-23T21:14:32.798492+0000","last_peered":"2023-09-23T21:14:32.798492+0000","last_clean":"2023-09-23T21:14:32.798492+0000","last_became_active":"2023-09-23T21:12:30.437427+0000","last_became_peered":"2023-09-23T21:12:30.437427+0000","last_unstale":"2023-09-23T21:14:32.798492+0000","last_undegraded":"2023-09-23T21:14:32.798492+0000","last_fullsized":"2023-09-23T21:14:32.798492+0000","mapping_epoch":446,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:06:51.075448+0000","last_deep_scrub":"79'161","last_deep_scrub_stamp":"2023-09-23T21:05:32.630292+0000","last_clean_scrub_stamp":"2023-09-23T21:06:51.075448+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0054793890000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,4],"acting":[2,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":376,"reported_epoch":529,"state":"active+clean","last_fresh":"2023-09-23T21:14:32.778085+0000","last_change":"2023-09-23T21:12:10.792918+0000","last_active":"2023-09-23T21:14:32.778085+0000","last_peered":"2023-09-23T21:14:32.778085+0000","last_clean":"2023-09-23T21:14:32.778085+0000","last_became_active":"2023-09-23T21:12:10.792679+0000","last_became_peered":"2023-09-23T21:12:10.792679+0000","last_unstale":"2023-09-23T21:14:32.778085+0000","last_undegraded":"2023-09-23T21:14:32.778085+0000","last_fullsized":"2023-09-23T21:14:32.778085+0000","mapping_epoch":470,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:30:48.008690+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,4],"acting":[3,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":508,"reported_epoch":529,"state":"active+clean","last_fresh":"2023-09-23T21:14:32.784742+0000","last_change":"2023-09-23T21:12:10.790812+0000","last_active":"2023-09-23T21:14:32.784742+0000","last_peered":"2023-09-23T21:14:32.784742+0000","last_clean":"2023-09-23T21:14:32.784742+0000","last_became_active":"2023-09-23T21:12:10.790492+0000","last_became_peered":"2023-09-23T21:12:10.790492+0000","last_unstale":"2023-09-23T21:14:32.784742+0000","last_undegraded":"2023-09-23T21:14:32.784742+0000","last_fullsized":"2023-09-23T21:14:32.784742+0000","mapping_epoch":470,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:02:19.863118+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.c","version":"531'8962","reported_seq":10120,"reported_epoch":531,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T21:14:52.259453+0000","last_change":"2023-09-23T21:13:30.822215+0000","last_active":"2023-09-23T21:14:52.259453+0000","last_peered":"2023-09-23T21:14:52.259453+0000","last_clean":"2023-09-23T21:11:45.750376+0000","last_became_active":"2023-09-23T21:13:13.404327+0000","last_became_peered":"2023-09-23T21:13:13.404327+0000","last_unstale":"2023-09-23T21:14:52.259453+0000","last_undegraded":"2023-09-23T21:13:13.200907+0000","last_fullsized":"2023-09-23T21:13:13.193701+0000","mapping_epoch":494,"log_start":"458'4551","ondisk_log_start":"458'4551","created":416,"last_epoch_clean":434,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_clean_scrub_stamp":"2023-09-23T21:10:15.663179+0000","objects_scrubbed":0,"log_size":4411,"log_dups_size":0,"ondisk_log_size":4411,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:33:42.971375+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70516736,"num_objects":1076,"num_object_clones":0,"num_object_copies":3228,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1339,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1076,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8962,"num_write_kb":321216,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":498,"num_bytes_recovered":32636928,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4],"avail_no_missing":["7","4"],"object_location_counts":[{"shards":"4,7","objects":1022}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.7","version":"79'161","reported_seq":622,"reported_epoch":529,"state":"active+clean","last_fresh":"2023-09-23T21:14:32.784372+0000","last_change":"2023-09-23T21:13:44.416548+0000","last_active":"2023-09-23T21:14:32.784372+0000","last_peered":"2023-09-23T21:14:32.784372+0000","last_clean":"2023-09-23T21:14:32.784372+0000","last_became_active":"2023-09-23T21:13:44.415617+0000","last_became_peered":"2023-09-23T21:13:44.415617+0000","last_unstale":"2023-09-23T21:14:32.784372+0000","last_undegraded":"2023-09-23T21:14:32.784372+0000","last_fullsized":"2023-09-23T21:14:32.784372+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":3,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:25.916867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:25.916867+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.001665323,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":459,"reported_epoch":529,"state":"active+clean","last_fresh":"2023-09-23T21:14:32.787502+0000","last_change":"2023-09-23T21:13:13.211959+0000","last_active":"2023-09-23T21:14:32.787502+0000","last_peered":"2023-09-23T21:14:32.787502+0000","last_clean":"2023-09-23T21:14:32.787502+0000","last_became_active":"2023-09-23T21:13:13.211712+0000","last_became_peered":"2023-09-23T21:13:13.211712+0000","last_unstale":"2023-09-23T21:14:32.787502+0000","last_undegraded":"2023-09-23T21:14:32.787502+0000","last_fullsized":"2023-09-23T21:14:32.787502+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":495,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:37:21.655775+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,2],"acting":[0,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":208,"reported_epoch":530,"state":"active+clean","last_fresh":"2023-09-23T21:14:44.691964+0000","last_change":"2023-09-23T21:05:04.326889+0000","last_active":"2023-09-23T21:14:44.691964+0000","last_peered":"2023-09-23T21:14:44.691964+0000","last_clean":"2023-09-23T21:14:44.691964+0000","last_became_active":"2023-09-23T21:05:04.326032+0000","last_became_peered":"2023-09-23T21:05:04.326032+0000","last_unstale":"2023-09-23T21:14:44.691964+0000","last_undegraded":"2023-09-23T21:14:44.691964+0000","last_fullsized":"2023-09-23T21:14:44.691964+0000","mapping_epoch":332,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:00:17.507889+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":540,"reported_epoch":530,"state":"active+clean","last_fresh":"2023-09-23T21:14:44.691303+0000","last_change":"2023-09-23T21:13:13.285608+0000","last_active":"2023-09-23T21:14:44.691303+0000","last_peered":"2023-09-23T21:14:44.691303+0000","last_clean":"2023-09-23T21:14:44.691303+0000","last_became_active":"2023-09-23T21:13:13.285255+0000","last_became_peered":"2023-09-23T21:13:13.285255+0000","last_unstale":"2023-09-23T21:14:44.691303+0000","last_undegraded":"2023-09-23T21:14:44.691303+0000","last_fullsized":"2023-09-23T21:14:44.691303+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":495,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:42:30.087599+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.a","version":"530'8716","reported_seq":17453,"reported_epoch":530,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T21:14:51.240930+0000","last_change":"2023-09-23T21:13:31.927282+0000","last_active":"2023-09-23T21:14:51.240930+0000","last_peered":"2023-09-23T21:14:51.240930+0000","last_clean":"2023-09-23T21:13:30.353360+0000","last_became_active":"2023-09-23T21:13:31.896923+0000","last_became_peered":"2023-09-23T21:13:31.896923+0000","last_unstale":"2023-09-23T21:14:51.240930+0000","last_undegraded":"2023-09-23T21:13:31.776904+0000","last_fullsized":"2023-09-23T21:13:31.751334+0000","mapping_epoch":502,"log_start":"456'4312","ondisk_log_start":"456'4312","created":416,"last_epoch_clean":499,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_clean_scrub_stamp":"2023-09-23T21:10:15.663179+0000","objects_scrubbed":0,"log_size":4404,"log_dups_size":0,"ondisk_log_size":4404,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:23:14.477602+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":75759616,"num_objects":1156,"num_object_clones":0,"num_object_copies":3468,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2243,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1156,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8716,"num_write_kb":315904,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4293,"num_bytes_recovered":281346048,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[2,5],"avail_no_missing":["2","3","5"],"object_location_counts":[{"shards":"2,3,5","objects":643},{"shards":"2,5","objects":513}],"blocked_by":[],"up_primary":0,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.1","version":"79'161","reported_seq":600,"reported_epoch":529,"state":"active+clean","last_fresh":"2023-09-23T21:14:32.786346+0000","last_change":"2023-09-23T21:13:12.204135+0000","last_active":"2023-09-23T21:14:32.786346+0000","last_peered":"2023-09-23T21:14:32.786346+0000","last_clean":"2023-09-23T21:14:32.786346+0000","last_became_active":"2023-09-23T21:13:12.203380+0000","last_became_peered":"2023-09-23T21:13:12.203380+0000","last_unstale":"2023-09-23T21:14:32.786346+0000","last_undegraded":"2023-09-23T21:14:32.786346+0000","last_fullsized":"2023-09-23T21:14:32.786346+0000","mapping_epoch":493,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:06:52.575900+0000","last_deep_scrub":"79'161","last_deep_scrub_stamp":"2023-09-23T21:05:30.652189+0000","last_clean_scrub_stamp":"2023-09-23T21:06:52.575900+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":196,"reported_epoch":529,"state":"active+clean","last_fresh":"2023-09-23T21:14:32.786693+0000","last_change":"2023-09-23T21:13:13.238617+0000","last_active":"2023-09-23T21:14:32.786693+0000","last_peered":"2023-09-23T21:14:32.786693+0000","last_clean":"2023-09-23T21:14:32.786693+0000","last_became_active":"2023-09-23T21:13:13.232926+0000","last_became_peered":"2023-09-23T21:13:13.232926+0000","last_unstale":"2023-09-23T21:14:32.786693+0000","last_undegraded":"2023-09-23T21:14:32.786693+0000","last_fullsized":"2023-09-23T21:14:32.786693+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":495,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:17:21.823772+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":463,"reported_epoch":529,"state":"active+clean","last_fresh":"2023-09-23T21:14:32.782183+0000","last_change":"2023-09-23T21:13:14.144918+0000","last_active":"2023-09-23T21:14:32.782183+0000","last_peered":"2023-09-23T21:14:32.782183+0000","last_clean":"2023-09-23T21:14:32.782183+0000","last_became_active":"2023-09-23T21:13:14.132388+0000","last_became_peered":"2023-09-23T21:13:14.132388+0000","last_unstale":"2023-09-23T21:14:32.782183+0000","last_undegraded":"2023-09-23T21:14:32.782183+0000","last_fullsized":"2023-09-23T21:14:32.782183+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":495,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:56:02.619905+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,5],"acting":[4,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":350,"reported_epoch":529,"state":"active+clean","last_fresh":"2023-09-23T21:14:32.795401+0000","last_change":"2023-09-23T21:12:10.764503+0000","last_active":"2023-09-23T21:14:32.795401+0000","last_peered":"2023-09-23T21:14:32.795401+0000","last_clean":"2023-09-23T21:14:32.795401+0000","last_became_active":"2023-09-23T21:12:10.763933+0000","last_became_peered":"2023-09-23T21:12:10.763933+0000","last_unstale":"2023-09-23T21:14:32.795401+0000","last_undegraded":"2023-09-23T21:14:32.795401+0000","last_fullsized":"2023-09-23T21:14:32.795401+0000","mapping_epoch":470,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:28:33.202825+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,7],"acting":[2,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"10.9","version":"530'5781","reported_seq":6927,"reported_epoch":530,"state":"active+clean","last_fresh":"2023-09-23T21:14:51.229471+0000","last_change":"2023-09-23T21:11:08.062606+0000","last_active":"2023-09-23T21:14:51.229471+0000","last_peered":"2023-09-23T21:14:51.229471+0000","last_clean":"2023-09-23T21:14:51.229471+0000","last_became_active":"2023-09-23T21:10:58.780961+0000","last_became_peered":"2023-09-23T21:10:58.780961+0000","last_unstale":"2023-09-23T21:14:51.229471+0000","last_undegraded":"2023-09-23T21:14:51.229471+0000","last_fullsized":"2023-09-23T21:14:51.229471+0000","mapping_epoch":428,"log_start":"421'17","ondisk_log_start":"421'17","created":416,"last_epoch_clean":439,"parent":"0.0","parent_split_bits":0,"last_scrub":"443'3049","last_scrub_stamp":"2023-09-23T21:11:08.062479+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_clean_scrub_stamp":"2023-09-23T21:11:08.062479+0000","objects_scrubbed":1623,"log_size":5764,"log_dups_size":0,"ondisk_log_size":5764,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:56:22.338088+0000","scrub_duration":0.28207626200000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":37486592,"num_objects":572,"num_object_clones":0,"num_object_copies":1716,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":572,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2733,"num_write_kb":53824,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.2","version":"79'161","reported_seq":809,"reported_epoch":529,"state":"active+clean","last_fresh":"2023-09-23T21:14:32.786541+0000","last_change":"2023-09-23T21:13:12.209907+0000","last_active":"2023-09-23T21:14:32.786541+0000","last_peered":"2023-09-23T21:14:32.786541+0000","last_clean":"2023-09-23T21:14:32.786541+0000","last_became_active":"2023-09-23T21:13:12.205664+0000","last_became_peered":"2023-09-23T21:13:12.205664+0000","last_unstale":"2023-09-23T21:14:32.786541+0000","last_undegraded":"2023-09-23T21:14:32.786541+0000","last_fullsized":"2023-09-23T21:14:32.786541+0000","mapping_epoch":493,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:24.802451+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:24.802451+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:26:16.035788+0000","scrub_duration":0.0090861250000000005,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,4],"acting":[0,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":538,"reported_epoch":529,"state":"active+clean","last_fresh":"2023-09-23T21:14:32.789025+0000","last_change":"2023-09-23T21:13:12.206980+0000","last_active":"2023-09-23T21:14:32.789025+0000","last_peered":"2023-09-23T21:14:32.789025+0000","last_clean":"2023-09-23T21:14:32.789025+0000","last_became_active":"2023-09-23T21:13:12.206650+0000","last_became_peered":"2023-09-23T21:13:12.206650+0000","last_unstale":"2023-09-23T21:14:32.789025+0000","last_undegraded":"2023-09-23T21:14:32.789025+0000","last_fullsized":"2023-09-23T21:14:32.789025+0000","mapping_epoch":493,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:10:01.892936+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":522,"reported_epoch":529,"state":"active+clean","last_fresh":"2023-09-23T21:14:32.788873+0000","last_change":"2023-09-23T21:13:13.407073+0000","last_active":"2023-09-23T21:14:32.788873+0000","last_peered":"2023-09-23T21:14:32.788873+0000","last_clean":"2023-09-23T21:14:32.788873+0000","last_became_active":"2023-09-23T21:13:13.395047+0000","last_became_peered":"2023-09-23T21:13:13.395047+0000","last_unstale":"2023-09-23T21:14:32.788873+0000","last_undegraded":"2023-09-23T21:14:32.788873+0000","last_fullsized":"2023-09-23T21:14:32.788873+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":495,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:04:43.384190+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,5],"acting":[2,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":39,"reported_epoch":529,"state":"active+clean","last_fresh":"2023-09-23T21:14:32.787084+0000","last_change":"2023-09-23T21:13:30.785054+0000","last_active":"2023-09-23T21:14:32.787084+0000","last_peered":"2023-09-23T21:14:32.787084+0000","last_clean":"2023-09-23T21:14:32.787084+0000","last_became_active":"2023-09-23T21:13:30.784701+0000","last_became_peered":"2023-09-23T21:13:30.784701+0000","last_unstale":"2023-09-23T21:14:32.787084+0000","last_undegraded":"2023-09-23T21:14:32.787084+0000","last_fullsized":"2023-09-23T21:14:32.787084+0000","mapping_epoch":501,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:19:20.152252+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.b","version":"530'8838","reported_seq":18712,"reported_epoch":530,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2023-09-23T21:14:48.715759+0000","last_change":"2023-09-23T21:13:59.891994+0000","last_active":"2023-09-23T21:14:48.715759+0000","last_peered":"2023-09-23T21:14:48.715759+0000","last_clean":"2023-09-23T21:11:26.732248+0000","last_became_active":"2023-09-23T21:13:14.937458+0000","last_became_peered":"2023-09-23T21:13:14.937458+0000","last_unstale":"2023-09-23T21:14:48.715759+0000","last_undegraded":"2023-09-23T21:13:59.884123+0000","last_fullsized":"2023-09-23T21:14:48.715759+0000","mapping_epoch":494,"log_start":"426'600","ondisk_log_start":"426'600","created":416,"last_epoch_clean":430,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_clean_scrub_stamp":"2023-09-23T21:10:15.663179+0000","objects_scrubbed":0,"log_size":8238,"log_dups_size":0,"ondisk_log_size":8238,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:36:15.760947+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":82182144,"num_objects":1254,"num_object_clones":0,"num_object_copies":3762,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":586,"num_objects_misplaced":1337,"num_objects_unfound":0,"num_objects_dirty":1254,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8838,"num_write_kb":322944,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4846,"num_bytes_recovered":317390848,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[1,4,7],"avail_no_missing":["1","7"],"object_location_counts":[{"shards":"0,1,7","objects":578},{"shards":"1,4,7","objects":601}],"blocked_by":[],"up_primary":0,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":315,"reported_epoch":529,"state":"active+clean","last_fresh":"2023-09-23T21:14:32.796531+0000","last_change":"2023-09-23T21:10:26.769764+0000","last_active":"2023-09-23T21:14:32.796531+0000","last_peered":"2023-09-23T21:14:32.796531+0000","last_clean":"2023-09-23T21:14:32.796531+0000","last_became_active":"2023-09-23T21:10:26.769326+0000","last_became_peered":"2023-09-23T21:10:26.769326+0000","last_unstale":"2023-09-23T21:14:32.796531+0000","last_undegraded":"2023-09-23T21:14:32.796531+0000","last_fullsized":"2023-09-23T21:14:32.796531+0000","mapping_epoch":423,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":424,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:57:45.219837+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"79'161","reported_seq":712,"reported_epoch":529,"state":"active+clean","last_fresh":"2023-09-23T21:14:32.784334+0000","last_change":"2023-09-23T21:13:33.766895+0000","last_active":"2023-09-23T21:14:32.784334+0000","last_peered":"2023-09-23T21:14:32.784334+0000","last_clean":"2023-09-23T21:14:32.784334+0000","last_became_active":"2023-09-23T21:13:33.766538+0000","last_became_peered":"2023-09-23T21:13:33.766538+0000","last_unstale":"2023-09-23T21:14:32.784334+0000","last_undegraded":"2023-09-23T21:14:32.784334+0000","last_fullsized":"2023-09-23T21:14:32.784334+0000","mapping_epoch":504,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":505,"parent":"0.0","parent_split_bits":0,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:20.738273+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:20.738273+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:35:35.855987+0000","scrub_duration":0.0088336339999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,2],"acting":[0,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":538,"reported_epoch":530,"state":"active+clean","last_fresh":"2023-09-23T21:14:44.691212+0000","last_change":"2023-09-23T21:11:08.760417+0000","last_active":"2023-09-23T21:14:44.691212+0000","last_peered":"2023-09-23T21:14:44.691212+0000","last_clean":"2023-09-23T21:14:44.691212+0000","last_became_active":"2023-09-23T21:11:08.760150+0000","last_became_peered":"2023-09-23T21:11:08.760150+0000","last_unstale":"2023-09-23T21:14:44.691212+0000","last_undegraded":"2023-09-23T21:14:44.691212+0000","last_fullsized":"2023-09-23T21:14:44.691212+0000","mapping_epoch":443,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":444,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:23:10.849768+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":104,"reported_epoch":529,"state":"active+clean","last_fresh":"2023-09-23T21:14:32.779436+0000","last_change":"2023-09-23T21:13:12.216020+0000","last_active":"2023-09-23T21:14:32.779436+0000","last_peered":"2023-09-23T21:14:32.779436+0000","last_clean":"2023-09-23T21:14:32.779436+0000","last_became_active":"2023-09-23T21:13:12.215180+0000","last_became_peered":"2023-09-23T21:13:12.215180+0000","last_unstale":"2023-09-23T21:14:32.779436+0000","last_undegraded":"2023-09-23T21:14:32.779436+0000","last_fullsized":"2023-09-23T21:14:32.779436+0000","mapping_epoch":493,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:26:18.704684+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.8","version":"530'5666","reported_seq":8062,"reported_epoch":530,"state":"active+clean","last_fresh":"2023-09-23T21:14:50.186488+0000","last_change":"2023-09-23T21:14:03.796930+0000","last_active":"2023-09-23T21:14:50.186488+0000","last_peered":"2023-09-23T21:14:50.186488+0000","last_clean":"2023-09-23T21:14:50.186488+0000","last_became_active":"2023-09-23T21:14:03.786177+0000","last_became_peered":"2023-09-23T21:14:03.786177+0000","last_unstale":"2023-09-23T21:14:50.186488+0000","last_undegraded":"2023-09-23T21:14:50.186488+0000","last_fullsized":"2023-09-23T21:14:50.186488+0000","mapping_epoch":524,"log_start":"427'667","ondisk_log_start":"427'667","created":416,"last_epoch_clean":525,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_clean_scrub_stamp":"2023-09-23T21:10:15.663179+0000","objects_scrubbed":0,"log_size":4999,"log_dups_size":0,"ondisk_log_size":4999,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":33587200,"num_objects":512,"num_object_clones":0,"num_object_copies":1536,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":512,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4316,"num_write_kb":154528,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":880,"num_bytes_recovered":16908288,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,7],"acting":[0,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.3","version":"79'161","reported_seq":832,"reported_epoch":529,"state":"active+clean","last_fresh":"2023-09-23T21:14:32.779351+0000","last_change":"2023-09-23T21:13:33.771279+0000","last_active":"2023-09-23T21:14:32.779351+0000","last_peered":"2023-09-23T21:14:32.779351+0000","last_clean":"2023-09-23T21:14:32.779351+0000","last_became_active":"2023-09-23T21:13:33.770980+0000","last_became_peered":"2023-09-23T21:13:33.770980+0000","last_unstale":"2023-09-23T21:14:32.779351+0000","last_undegraded":"2023-09-23T21:14:32.779351+0000","last_fullsized":"2023-09-23T21:14:32.779351+0000","mapping_epoch":504,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":505,"parent":"0.0","parent_split_bits":3,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:05:57.832302+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:05:57.832302+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:19:42.309604+0000","scrub_duration":0.010206847999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":346,"reported_epoch":529,"state":"active+clean","last_fresh":"2023-09-23T21:14:32.784686+0000","last_change":"2023-09-23T21:13:13.212775+0000","last_active":"2023-09-23T21:14:32.784686+0000","last_peered":"2023-09-23T21:14:32.784686+0000","last_clean":"2023-09-23T21:14:32.784686+0000","last_became_active":"2023-09-23T21:13:13.212460+0000","last_became_peered":"2023-09-23T21:13:13.212460+0000","last_unstale":"2023-09-23T21:14:32.784686+0000","last_undegraded":"2023-09-23T21:14:32.784686+0000","last_fullsized":"2023-09-23T21:14:32.784686+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":495,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:27:10.853970+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":528,"reported_epoch":529,"state":"active+clean","last_fresh":"2023-09-23T21:14:32.786168+0000","last_change":"2023-09-23T21:13:44.409212+0000","last_active":"2023-09-23T21:14:32.786168+0000","last_peered":"2023-09-23T21:14:32.786168+0000","last_clean":"2023-09-23T21:14:32.786168+0000","last_became_active":"2023-09-23T21:13:44.408958+0000","last_became_peered":"2023-09-23T21:13:44.408958+0000","last_unstale":"2023-09-23T21:14:32.786168+0000","last_undegraded":"2023-09-23T21:14:32.786168+0000","last_fullsized":"2023-09-23T21:14:32.786168+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:03:52.749964+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":39,"reported_epoch":529,"state":"active+clean","last_fresh":"2023-09-23T21:14:32.787029+0000","last_change":"2023-09-23T21:13:30.804379+0000","last_active":"2023-09-23T21:14:32.787029+0000","last_peered":"2023-09-23T21:14:32.787029+0000","last_clean":"2023-09-23T21:14:32.787029+0000","last_became_active":"2023-09-23T21:13:30.799907+0000","last_became_peered":"2023-09-23T21:13:30.799907+0000","last_unstale":"2023-09-23T21:14:32.787029+0000","last_undegraded":"2023-09-23T21:14:32.787029+0000","last_fullsized":"2023-09-23T21:14:32.787029+0000","mapping_epoch":501,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:49:42.859212+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,7],"acting":[0,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.d","version":"530'8735","reported_seq":8898,"reported_epoch":530,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-23T21:14:48.316402+0000","last_change":"2023-09-23T21:13:31.801486+0000","last_active":"2023-09-23T21:14:48.316402+0000","last_peered":"2023-09-23T21:14:48.316402+0000","last_clean":"2023-09-23T21:11:37.659929+0000","last_became_active":"2023-09-23T21:13:31.786626+0000","last_became_peered":"2023-09-23T21:13:31.786626+0000","last_unstale":"2023-09-23T21:14:48.316402+0000","last_undegraded":"2023-09-23T21:13:31.760908+0000","last_fullsized":"2023-09-23T21:14:48.316402+0000","mapping_epoch":502,"log_start":"464'4795","ondisk_log_start":"464'4795","created":416,"last_epoch_clean":417,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_clean_scrub_stamp":"2023-09-23T21:10:15.663179+0000","objects_scrubbed":0,"log_size":3940,"log_dups_size":0,"ondisk_log_size":3940,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:46:02.424493+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":82968600,"num_objects":1267,"num_object_clones":0,"num_object_copies":3801,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1265,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1267,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":8735,"num_write_kb":320001,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,3],"avail_no_missing":["4","3","7"],"object_location_counts":[{"shards":"0,3,4","objects":1188},{"shards":"3,4","objects":79}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.6","version":"79'161","reported_seq":448,"reported_epoch":529,"state":"active+clean","last_fresh":"2023-09-23T21:14:32.786945+0000","last_change":"2023-09-23T21:13:33.770715+0000","last_active":"2023-09-23T21:14:32.786945+0000","last_peered":"2023-09-23T21:14:32.786945+0000","last_clean":"2023-09-23T21:14:32.786945+0000","last_became_active":"2023-09-23T21:13:33.769303+0000","last_became_peered":"2023-09-23T21:13:33.769303+0000","last_unstale":"2023-09-23T21:14:32.786945+0000","last_undegraded":"2023-09-23T21:14:32.786945+0000","last_fullsized":"2023-09-23T21:14:32.786945+0000","mapping_epoch":504,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":505,"parent":"0.0","parent_split_bits":3,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:10:24.268662+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:10:24.268662+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13417,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":11,"num_read_kb":20,"num_write":5,"num_write_kb":55,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":40252,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":157,"reported_epoch":530,"state":"active+clean","last_fresh":"2023-09-23T21:14:44.709545+0000","last_change":"2023-09-23T21:13:30.770826+0000","last_active":"2023-09-23T21:14:44.709545+0000","last_peered":"2023-09-23T21:14:44.709545+0000","last_clean":"2023-09-23T21:14:44.709545+0000","last_became_active":"2023-09-23T21:13:30.770512+0000","last_became_peered":"2023-09-23T21:13:30.770512+0000","last_unstale":"2023-09-23T21:14:44.709545+0000","last_undegraded":"2023-09-23T21:14:44.709545+0000","last_fullsized":"2023-09-23T21:14:44.709545+0000","mapping_epoch":501,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:44:23.470538+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":413,"reported_epoch":529,"state":"active+clean","last_fresh":"2023-09-23T21:14:32.795675+0000","last_change":"2023-09-23T21:13:30.790590+0000","last_active":"2023-09-23T21:14:32.795675+0000","last_peered":"2023-09-23T21:14:32.795675+0000","last_clean":"2023-09-23T21:14:32.795675+0000","last_became_active":"2023-09-23T21:13:30.790253+0000","last_became_peered":"2023-09-23T21:13:30.790253+0000","last_unstale":"2023-09-23T21:14:32.795675+0000","last_undegraded":"2023-09-23T21:14:32.795675+0000","last_fullsized":"2023-09-23T21:14:32.795675+0000","mapping_epoch":501,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:23:41.843250+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,0],"acting":[2,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":475,"reported_epoch":529,"state":"active+clean","last_fresh":"2023-09-23T21:14:32.787253+0000","last_change":"2023-09-23T21:13:12.210546+0000","last_active":"2023-09-23T21:14:32.787253+0000","last_peered":"2023-09-23T21:14:32.787253+0000","last_clean":"2023-09-23T21:14:32.787253+0000","last_became_active":"2023-09-23T21:13:12.205934+0000","last_became_peered":"2023-09-23T21:13:12.205934+0000","last_unstale":"2023-09-23T21:14:32.787253+0000","last_undegraded":"2023-09-23T21:14:32.787253+0000","last_fullsized":"2023-09-23T21:14:32.787253+0000","mapping_epoch":493,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:21:18.299428+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.e","version":"530'8941","reported_seq":15913,"reported_epoch":530,"state":"active+clean","last_fresh":"2023-09-23T21:14:48.303307+0000","last_change":"2023-09-23T21:12:38.623611+0000","last_active":"2023-09-23T21:14:48.303307+0000","last_peered":"2023-09-23T21:14:48.303307+0000","last_clean":"2023-09-23T21:14:48.303307+0000","last_became_active":"2023-09-23T21:12:38.610155+0000","last_became_peered":"2023-09-23T21:12:38.610155+0000","last_unstale":"2023-09-23T21:14:48.303307+0000","last_undegraded":"2023-09-23T21:14:48.303307+0000","last_fullsized":"2023-09-23T21:14:48.303307+0000","mapping_epoch":483,"log_start":"464'4993","ondisk_log_start":"464'4993","created":416,"last_epoch_clean":484,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_clean_scrub_stamp":"2023-09-23T21:10:15.663179+0000","objects_scrubbed":0,"log_size":3948,"log_dups_size":0,"ondisk_log_size":3948,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:19:27.097170+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":84082688,"num_objects":1283,"num_object_clones":0,"num_object_copies":3849,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1283,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8941,"num_write_kb":327168,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3412,"num_bytes_recovered":223608832,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.5","version":"79'161","reported_seq":557,"reported_epoch":529,"state":"active+clean","last_fresh":"2023-09-23T21:14:32.778786+0000","last_change":"2023-09-23T21:13:38.373969+0000","last_active":"2023-09-23T21:14:32.778786+0000","last_peered":"2023-09-23T21:14:32.778786+0000","last_clean":"2023-09-23T21:14:32.778786+0000","last_became_active":"2023-09-23T21:13:38.373633+0000","last_became_peered":"2023-09-23T21:13:38.373633+0000","last_unstale":"2023-09-23T21:14:32.778786+0000","last_undegraded":"2023-09-23T21:14:32.778786+0000","last_fullsized":"2023-09-23T21:14:32.778786+0000","mapping_epoch":507,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":3,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:09:34.892947+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:09:34.892947+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13417,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":11,"num_read_kb":20,"num_write":5,"num_write_kb":55,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":40252,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":70,"reported_epoch":529,"state":"active+clean","last_fresh":"2023-09-23T21:14:32.787256+0000","last_change":"2023-09-23T21:13:12.235743+0000","last_active":"2023-09-23T21:14:32.787256+0000","last_peered":"2023-09-23T21:14:32.787256+0000","last_clean":"2023-09-23T21:14:32.787256+0000","last_became_active":"2023-09-23T21:13:12.216241+0000","last_became_peered":"2023-09-23T21:13:12.216241+0000","last_unstale":"2023-09-23T21:14:32.787256+0000","last_undegraded":"2023-09-23T21:14:32.787256+0000","last_fullsized":"2023-09-23T21:14:32.787256+0000","mapping_epoch":493,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:12:19.090710+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":510,"reported_epoch":529,"state":"active+clean","last_fresh":"2023-09-23T21:14:32.780105+0000","last_change":"2023-09-23T21:13:30.807520+0000","last_active":"2023-09-23T21:14:32.780105+0000","last_peered":"2023-09-23T21:14:32.780105+0000","last_clean":"2023-09-23T21:14:32.780105+0000","last_became_active":"2023-09-23T21:13:30.807222+0000","last_became_peered":"2023-09-23T21:13:30.807222+0000","last_unstale":"2023-09-23T21:14:32.780105+0000","last_undegraded":"2023-09-23T21:14:32.780105+0000","last_fullsized":"2023-09-23T21:14:32.780105+0000","mapping_epoch":501,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:40:31.989749+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":371,"reported_epoch":529,"state":"active+clean","last_fresh":"2023-09-23T21:14:32.778098+0000","last_change":"2023-09-23T21:13:30.794095+0000","last_active":"2023-09-23T21:14:32.778098+0000","last_peered":"2023-09-23T21:14:32.778098+0000","last_clean":"2023-09-23T21:14:32.778098+0000","last_became_active":"2023-09-23T21:13:30.793791+0000","last_became_peered":"2023-09-23T21:13:30.793791+0000","last_unstale":"2023-09-23T21:14:32.778098+0000","last_undegraded":"2023-09-23T21:14:32.778098+0000","last_fullsized":"2023-09-23T21:14:32.778098+0000","mapping_epoch":501,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:36:28.707362+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.f","version":"530'8887","reported_seq":9093,"reported_epoch":530,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T21:14:49.196961+0000","last_change":"2023-09-23T21:13:59.874192+0000","last_active":"2023-09-23T21:14:49.196961+0000","last_peered":"2023-09-23T21:14:49.196961+0000","last_clean":"2023-09-23T21:11:07.652290+0000","last_became_active":"2023-09-23T21:11:09.897463+0000","last_became_peered":"2023-09-23T21:11:09.897463+0000","last_unstale":"2023-09-23T21:14:49.196961+0000","last_undegraded":"2023-09-23T21:11:27.781677+0000","last_fullsized":"2023-09-23T21:11:09.761652+0000","mapping_epoch":444,"log_start":"468'5209","ondisk_log_start":"468'5209","created":416,"last_epoch_clean":417,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_clean_scrub_stamp":"2023-09-23T21:10:15.663179+0000","objects_scrubbed":0,"log_size":3678,"log_dups_size":0,"ondisk_log_size":3678,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:42:51.588441+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":80412672,"num_objects":1227,"num_object_clones":0,"num_object_copies":3681,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3295,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1227,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8887,"num_write_kb":323648,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":32,"num_bytes_recovered":2097152,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2],"avail_no_missing":["3","1","2"],"object_location_counts":[{"shards":"2,3","objects":-2068}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.4","version":"79'161","reported_seq":900,"reported_epoch":529,"state":"active+clean","last_fresh":"2023-09-23T21:14:32.779419+0000","last_change":"2023-09-23T21:10:29.807785+0000","last_active":"2023-09-23T21:14:32.779419+0000","last_peered":"2023-09-23T21:14:32.779419+0000","last_clean":"2023-09-23T21:14:32.779419+0000","last_became_active":"2023-09-23T21:10:29.807366+0000","last_became_peered":"2023-09-23T21:10:29.807366+0000","last_unstale":"2023-09-23T21:14:32.779419+0000","last_undegraded":"2023-09-23T21:14:32.779419+0000","last_fullsized":"2023-09-23T21:14:32.779419+0000","mapping_epoch":426,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":427,"parent":"0.0","parent_split_bits":3,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:20.012764+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:20.012764+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:24:45.763834+0000","scrub_duration":0.0045251919999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":469,"reported_epoch":529,"state":"active+clean","last_fresh":"2023-09-23T21:14:36.290134+0000","last_change":"2023-09-23T21:10:26.768364+0000","last_active":"2023-09-23T21:14:36.290134+0000","last_peered":"2023-09-23T21:14:36.290134+0000","last_clean":"2023-09-23T21:14:36.290134+0000","last_became_active":"2023-09-23T21:10:26.768052+0000","last_became_peered":"2023-09-23T21:10:26.768052+0000","last_unstale":"2023-09-23T21:14:36.290134+0000","last_undegraded":"2023-09-23T21:14:36.290134+0000","last_fullsized":"2023-09-23T21:14:36.290134+0000","mapping_epoch":423,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":424,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:22:42.201200+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,3],"acting":[2,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.6","version":"70'1","reported_seq":500,"reported_epoch":529,"state":"active+clean","last_fresh":"2023-09-23T21:14:32.787281+0000","last_change":"2023-09-23T21:13:30.793290+0000","last_active":"2023-09-23T21:14:32.787281+0000","last_peered":"2023-09-23T21:14:32.787281+0000","last_clean":"2023-09-23T21:14:32.787281+0000","last_became_active":"2023-09-23T21:13:30.786079+0000","last_became_peered":"2023-09-23T21:13:30.786079+0000","last_unstale":"2023-09-23T21:14:32.787281+0000","last_undegraded":"2023-09-23T21:14:32.787281+0000","last_fullsized":"2023-09-23T21:14:32.787281+0000","mapping_epoch":501,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:34:43.281095+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":104,"reported_epoch":529,"state":"active+clean","last_fresh":"2023-09-23T21:14:32.778367+0000","last_change":"2023-09-23T21:13:12.216895+0000","last_active":"2023-09-23T21:14:32.778367+0000","last_peered":"2023-09-23T21:14:32.778367+0000","last_clean":"2023-09-23T21:14:32.778367+0000","last_became_active":"2023-09-23T21:13:12.216654+0000","last_became_peered":"2023-09-23T21:13:12.216654+0000","last_unstale":"2023-09-23T21:14:32.778367+0000","last_undegraded":"2023-09-23T21:14:32.778367+0000","last_fullsized":"2023-09-23T21:14:32.778367+0000","mapping_epoch":493,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:02:40.430096+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.19","version":"79'161","reported_seq":682,"reported_epoch":529,"state":"active+clean","last_fresh":"2023-09-23T21:14:32.788514+0000","last_change":"2023-09-23T21:12:30.442914+0000","last_active":"2023-09-23T21:14:32.788514+0000","last_peered":"2023-09-23T21:14:32.788514+0000","last_clean":"2023-09-23T21:14:32.788514+0000","last_became_active":"2023-09-23T21:12:30.442578+0000","last_became_peered":"2023-09-23T21:12:30.442578+0000","last_unstale":"2023-09-23T21:14:32.788514+0000","last_undegraded":"2023-09-23T21:14:32.788514+0000","last_fullsized":"2023-09-23T21:14:32.788514+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":479,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:10:37.696951+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:10:37.696951+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.001386504,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,3],"acting":[2,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"10.12","version":"530'5703","reported_seq":8331,"reported_epoch":530,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T21:14:48.311507+0000","last_change":"2023-09-23T21:13:13.419184+0000","last_active":"2023-09-23T21:14:48.311507+0000","last_peered":"2023-09-23T21:14:48.311507+0000","last_clean":"2023-09-23T21:11:45.774771+0000","last_became_active":"2023-09-23T21:13:13.409281+0000","last_became_peered":"2023-09-23T21:13:13.409281+0000","last_unstale":"2023-09-23T21:14:48.311507+0000","last_undegraded":"2023-09-23T21:13:13.284063+0000","last_fullsized":"2023-09-23T21:13:13.191117+0000","mapping_epoch":494,"log_start":"434'863","ondisk_log_start":"434'863","created":438,"last_epoch_clean":439,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_clean_scrub_stamp":"2023-09-23T21:10:15.663179+0000","objects_scrubbed":0,"log_size":4840,"log_dups_size":0,"ondisk_log_size":4840,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":39976960,"num_objects":610,"num_object_clones":0,"num_object_copies":1830,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1906,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":610,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4326,"num_write_kb":157952,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1232,"num_bytes_recovered":80740352,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,3],"avail_no_missing":["4","1"],"object_location_counts":[{"shards":"0,1,4","objects":572},{"shards":"1,4","objects":41},{"shards":"3,4","objects":-3}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":39,"reported_epoch":529,"state":"active+clean","last_fresh":"2023-09-23T21:14:32.787171+0000","last_change":"2023-09-23T21:13:30.791297+0000","last_active":"2023-09-23T21:14:32.787171+0000","last_peered":"2023-09-23T21:14:32.787171+0000","last_clean":"2023-09-23T21:14:32.787171+0000","last_became_active":"2023-09-23T21:13:30.788721+0000","last_became_peered":"2023-09-23T21:13:30.788721+0000","last_unstale":"2023-09-23T21:14:32.787171+0000","last_undegraded":"2023-09-23T21:14:32.787171+0000","last_fullsized":"2023-09-23T21:14:32.787171+0000","mapping_epoch":501,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:31:18.147883+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":33,"reported_epoch":529,"state":"active+clean","last_fresh":"2023-09-23T21:14:32.795588+0000","last_change":"2023-09-23T21:13:38.373814+0000","last_active":"2023-09-23T21:14:32.795588+0000","last_peered":"2023-09-23T21:14:32.795588+0000","last_clean":"2023-09-23T21:14:32.795588+0000","last_became_active":"2023-09-23T21:13:38.373458+0000","last_became_peered":"2023-09-23T21:13:38.373458+0000","last_unstale":"2023-09-23T21:14:32.795588+0000","last_undegraded":"2023-09-23T21:14:32.795588+0000","last_fullsized":"2023-09-23T21:14:32.795588+0000","mapping_epoch":507,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:23:08.288401+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.1e","version":"79'161","reported_seq":697,"reported_epoch":529,"state":"active+clean","last_fresh":"2023-09-23T21:14:32.779385+0000","last_change":"2023-09-23T21:12:30.448761+0000","last_active":"2023-09-23T21:14:32.779385+0000","last_peered":"2023-09-23T21:14:32.779385+0000","last_clean":"2023-09-23T21:14:32.779385+0000","last_became_active":"2023-09-23T21:12:30.448043+0000","last_became_peered":"2023-09-23T21:12:30.448043+0000","last_unstale":"2023-09-23T21:14:32.779385+0000","last_undegraded":"2023-09-23T21:14:32.779385+0000","last_fullsized":"2023-09-23T21:14:32.779385+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":479,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:10:39.567232+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:10:39.567232+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.005229694,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.15","version":"530'5689","reported_seq":5836,"reported_epoch":530,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T21:14:51.242630+0000","last_change":"2023-09-23T21:13:13.436807+0000","last_active":"2023-09-23T21:14:51.242630+0000","last_peered":"2023-09-23T21:14:51.242630+0000","last_clean":"2023-09-23T21:11:45.765730+0000","last_became_active":"2023-09-23T21:13:13.429206+0000","last_became_peered":"2023-09-23T21:13:13.429206+0000","last_unstale":"2023-09-23T21:14:51.242630+0000","last_undegraded":"2023-09-23T21:13:13.246273+0000","last_fullsized":"2023-09-23T21:13:13.203825+0000","mapping_epoch":494,"log_start":"421'12","ondisk_log_start":"421'12","created":438,"last_epoch_clean":439,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_clean_scrub_stamp":"2023-09-23T21:10:15.663179+0000","objects_scrubbed":0,"log_size":5677,"log_dups_size":0,"ondisk_log_size":5677,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":38862848,"num_objects":593,"num_object_clones":0,"num_object_copies":1779,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":643,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":593,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4355,"num_write_kb":158336,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,3],"acting":[2,3],"avail_no_missing":["2","3"],"object_location_counts":[{"shards":"2,3","objects":557}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":105,"reported_epoch":529,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-23T21:14:32.779436+0000","last_change":"2023-09-23T21:12:14.313202+0000","last_active":"2023-09-23T21:14:32.779436+0000","last_peered":"2023-09-23T21:14:32.779436+0000","last_clean":"2023-09-23T21:12:09.299355+0000","last_became_active":"2023-09-23T21:12:14.209408+0000","last_became_peered":"2023-09-23T21:12:14.209408+0000","last_unstale":"2023-09-23T21:14:32.779436+0000","last_undegraded":"2023-09-23T21:12:12.933857+0000","last_fullsized":"2023-09-23T21:14:32.779436+0000","mapping_epoch":470,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":444,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:45:40.142413+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,7],"acting":[4,5,7],"avail_no_missing":["4","1","5"],"object_location_counts":[{"shards":"1,4,5","objects":1}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":94,"reported_epoch":529,"state":"active+clean","last_fresh":"2023-09-23T21:14:32.781930+0000","last_change":"2023-09-23T21:12:10.770474+0000","last_active":"2023-09-23T21:14:32.781930+0000","last_peered":"2023-09-23T21:14:32.781930+0000","last_clean":"2023-09-23T21:14:32.781930+0000","last_became_active":"2023-09-23T21:12:10.770184+0000","last_became_peered":"2023-09-23T21:12:10.770184+0000","last_unstale":"2023-09-23T21:14:32.781930+0000","last_undegraded":"2023-09-23T21:14:32.781930+0000","last_fullsized":"2023-09-23T21:14:32.781930+0000","mapping_epoch":470,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:05:09.407151+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.14","version":"530'5686","reported_seq":11788,"reported_epoch":530,"state":"active+clean","last_fresh":"2023-09-23T21:14:50.220956+0000","last_change":"2023-09-23T21:14:01.515957+0000","last_active":"2023-09-23T21:14:50.220956+0000","last_peered":"2023-09-23T21:14:50.220956+0000","last_clean":"2023-09-23T21:14:50.220956+0000","last_became_active":"2023-09-23T21:14:01.506428+0000","last_became_peered":"2023-09-23T21:14:01.506428+0000","last_unstale":"2023-09-23T21:14:50.220956+0000","last_undegraded":"2023-09-23T21:14:50.220956+0000","last_fullsized":"2023-09-23T21:14:50.220956+0000","mapping_epoch":522,"log_start":"428'716","ondisk_log_start":"428'716","created":438,"last_epoch_clean":523,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_clean_scrub_stamp":"2023-09-23T21:10:15.663179+0000","objects_scrubbed":0,"log_size":4970,"log_dups_size":0,"ondisk_log_size":4970,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":36569088,"num_objects":558,"num_object_clones":0,"num_object_copies":1674,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":558,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4352,"num_write_kb":157120,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2660,"num_bytes_recovered":126156800,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":319,"reported_epoch":529,"state":"active+clean","last_fresh":"2023-09-23T21:14:32.796408+0000","last_change":"2023-09-23T21:12:10.771160+0000","last_active":"2023-09-23T21:14:32.796408+0000","last_peered":"2023-09-23T21:14:32.796408+0000","last_clean":"2023-09-23T21:14:32.796408+0000","last_became_active":"2023-09-23T21:12:10.757808+0000","last_became_peered":"2023-09-23T21:12:10.757808+0000","last_unstale":"2023-09-23T21:14:32.796408+0000","last_undegraded":"2023-09-23T21:14:32.796408+0000","last_fullsized":"2023-09-23T21:14:32.796408+0000","mapping_epoch":470,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:30:50.987139+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":104,"reported_epoch":529,"state":"active+clean","last_fresh":"2023-09-23T21:14:32.780388+0000","last_change":"2023-09-23T21:13:12.216602+0000","last_active":"2023-09-23T21:14:32.780388+0000","last_peered":"2023-09-23T21:14:32.780388+0000","last_clean":"2023-09-23T21:14:32.780388+0000","last_became_active":"2023-09-23T21:13:12.216204+0000","last_became_peered":"2023-09-23T21:13:12.216204+0000","last_unstale":"2023-09-23T21:14:32.780388+0000","last_undegraded":"2023-09-23T21:14:32.780388+0000","last_fullsized":"2023-09-23T21:14:32.780388+0000","mapping_epoch":493,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:23:37.043390+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1c","version":"79'161","reported_seq":895,"reported_epoch":529,"state":"active+clean","last_fresh":"2023-09-23T21:14:32.778327+0000","last_change":"2023-09-23T21:13:13.402773+0000","last_active":"2023-09-23T21:14:32.778327+0000","last_peered":"2023-09-23T21:14:32.778327+0000","last_clean":"2023-09-23T21:14:32.778327+0000","last_became_active":"2023-09-23T21:13:13.402025+0000","last_became_peered":"2023-09-23T21:13:13.402025+0000","last_unstale":"2023-09-23T21:14:32.778327+0000","last_undegraded":"2023-09-23T21:14:32.778327+0000","last_fullsized":"2023-09-23T21:14:32.778327+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":479,"last_epoch_clean":495,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:22.040907+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:22.040907+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.013021176000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":163840,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":163840,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.17","version":"530'5653","reported_seq":10040,"reported_epoch":530,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T21:14:49.176818+0000","last_change":"2023-09-23T21:13:39.447969+0000","last_active":"2023-09-23T21:14:49.176818+0000","last_peered":"2023-09-23T21:14:49.176818+0000","last_clean":"2023-09-23T21:13:37.065694+0000","last_became_active":"2023-09-23T21:13:39.439739+0000","last_became_peered":"2023-09-23T21:13:39.439739+0000","last_unstale":"2023-09-23T21:14:49.176818+0000","last_undegraded":"2023-09-23T21:13:39.411853+0000","last_fullsized":"2023-09-23T21:13:39.404657+0000","mapping_epoch":508,"log_start":"434'1192","ondisk_log_start":"434'1192","created":438,"last_epoch_clean":487,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_clean_scrub_stamp":"2023-09-23T21:10:15.663179+0000","objects_scrubbed":0,"log_size":4461,"log_dups_size":0,"ondisk_log_size":4461,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":42434560,"num_objects":648,"num_object_clones":0,"num_object_copies":1944,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1109,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":648,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4288,"num_write_kb":157920,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2112,"num_bytes_recovered":116719616,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,2],"acting":[3,5],"avail_no_missing":["3","4","5"],"object_location_counts":[{"shards":"3,4,5","objects":311},{"shards":"3,5","objects":337}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":119,"reported_epoch":529,"state":"active+clean","last_fresh":"2023-09-23T21:14:36.290110+0000","last_change":"2023-09-23T21:12:10.779066+0000","last_active":"2023-09-23T21:14:36.290110+0000","last_peered":"2023-09-23T21:14:36.290110+0000","last_clean":"2023-09-23T21:14:36.290110+0000","last_became_active":"2023-09-23T21:12:10.778691+0000","last_became_peered":"2023-09-23T21:12:10.778691+0000","last_unstale":"2023-09-23T21:14:36.290110+0000","last_undegraded":"2023-09-23T21:14:36.290110+0000","last_fullsized":"2023-09-23T21:14:36.290110+0000","mapping_epoch":470,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:07:22.043268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,7],"acting":[2,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":197,"reported_epoch":529,"state":"active+clean","last_fresh":"2023-09-23T21:14:32.789110+0000","last_change":"2023-09-23T21:13:12.200767+0000","last_active":"2023-09-23T21:14:32.789110+0000","last_peered":"2023-09-23T21:14:32.789110+0000","last_clean":"2023-09-23T21:14:32.789110+0000","last_became_active":"2023-09-23T21:13:12.200435+0000","last_became_peered":"2023-09-23T21:13:12.200435+0000","last_unstale":"2023-09-23T21:14:32.789110+0000","last_undegraded":"2023-09-23T21:14:32.789110+0000","last_fullsized":"2023-09-23T21:14:32.789110+0000","mapping_epoch":493,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:15:44.937886+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.1d","version":"79'161","reported_seq":683,"reported_epoch":530,"state":"active+clean","last_fresh":"2023-09-23T21:14:44.709145+0000","last_change":"2023-09-23T21:12:30.444331+0000","last_active":"2023-09-23T21:14:44.709145+0000","last_peered":"2023-09-23T21:14:44.709145+0000","last_clean":"2023-09-23T21:14:44.709145+0000","last_became_active":"2023-09-23T21:12:30.444001+0000","last_became_peered":"2023-09-23T21:12:30.444001+0000","last_unstale":"2023-09-23T21:14:44.709145+0000","last_undegraded":"2023-09-23T21:14:44.709145+0000","last_fullsized":"2023-09-23T21:14:44.709145+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":479,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:09:36.858077+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:09:36.858077+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0024987500000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"10.16","version":"530'5781","reported_seq":9292,"reported_epoch":530,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T21:14:48.173101+0000","last_change":"2023-09-23T21:13:14.424843+0000","last_active":"2023-09-23T21:14:48.173101+0000","last_peered":"2023-09-23T21:14:48.173101+0000","last_clean":"2023-09-23T21:11:45.652255+0000","last_became_active":"2023-09-23T21:13:14.414710+0000","last_became_peered":"2023-09-23T21:13:14.414710+0000","last_unstale":"2023-09-23T21:14:48.173101+0000","last_undegraded":"2023-09-23T21:13:13.211957+0000","last_fullsized":"2023-09-23T21:13:13.188859+0000","mapping_epoch":494,"log_start":"434'942","ondisk_log_start":"434'942","created":438,"last_epoch_clean":460,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_clean_scrub_stamp":"2023-09-23T21:10:15.663179+0000","objects_scrubbed":0,"log_size":4839,"log_dups_size":0,"ondisk_log_size":4839,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":41353216,"num_objects":631,"num_object_clones":0,"num_object_copies":1893,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":663,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":631,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4375,"num_write_kb":160192,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1670,"num_bytes_recovered":109445120,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,4],"acting":[4,5],"avail_no_missing":["4","5"],"object_location_counts":[{"shards":"1,4,5","objects":33},{"shards":"4,5","objects":598}],"blocked_by":[],"up_primary":0,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":56,"reported_epoch":529,"state":"active+clean","last_fresh":"2023-09-23T21:14:32.786925+0000","last_change":"2023-09-23T21:13:38.380574+0000","last_active":"2023-09-23T21:14:32.786925+0000","last_peered":"2023-09-23T21:14:32.786925+0000","last_clean":"2023-09-23T21:14:32.786925+0000","last_became_active":"2023-09-23T21:13:38.373252+0000","last_became_peered":"2023-09-23T21:13:38.373252+0000","last_unstale":"2023-09-23T21:14:32.786925+0000","last_undegraded":"2023-09-23T21:14:32.786925+0000","last_fullsized":"2023-09-23T21:14:32.786925+0000","mapping_epoch":507,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:58:14.788031+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":11,"num_bytes_recovered":3324,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":104,"reported_epoch":529,"state":"active+clean","last_fresh":"2023-09-23T21:14:32.791348+0000","last_change":"2023-09-23T21:12:10.793610+0000","last_active":"2023-09-23T21:14:32.791348+0000","last_peered":"2023-09-23T21:14:32.791348+0000","last_clean":"2023-09-23T21:14:32.791348+0000","last_became_active":"2023-09-23T21:12:10.793081+0000","last_became_peered":"2023-09-23T21:12:10.793081+0000","last_unstale":"2023-09-23T21:14:32.791348+0000","last_undegraded":"2023-09-23T21:14:32.791348+0000","last_fullsized":"2023-09-23T21:14:32.791348+0000","mapping_epoch":470,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:34:36.291550+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":10,"num_pg":26,"stat_sum":{"num_bytes":1268252696,"num_objects":19355,"num_object_clones":0,"num_object_copies":58065,"num_objects_missing_on_primary":970,"num_objects_missing":970,"num_objects_degraded":21437,"num_objects_misplaced":3237,"num_objects_unfound":0,"num_objects_dirty":19355,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":128019,"num_write_kb":4309313,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":41839,"num_bytes_recovered":2532409344,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":6461661184,"data_stored":6461653040,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":129742,"ondisk_log_size":129742,"up":78,"acting":62,"num_store_stats":7},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":59,"num_bytes_recovered":8898,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":126976,"data_stored":22458,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":14322,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":31,"stat_sum":{"num_bytes":697708,"num_objects":3,"num_object_clones":0,"num_object_copies":9,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3,"num_whiteouts":0,"num_read":89,"num_read_kb":165,"num_write":44,"num_write_kb":440,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":649699,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2109440,"data_stored":2098784,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4991,"ondisk_log_size":4991,"up":93,"acting":93,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201693,"num_pgs":69,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":927484,"kb_used_data":426292,"kb_used_omap":0,"kb_used_meta":501184,"kb_avail":92817668,"statfs":{"total":95995035648,"available":95045292032,"internally_reserved":0,"allocated":436523008,"data_stored":434983087,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":513212416},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1,1,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:14:13 2023","interfaces":[{"interface":"back","average":{"1min":2.4260000000000002,"5min":2.4260000000000002,"15min":2.4260000000000002},"min":{"1min":0.23400000000000001,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":37.207999999999998,"5min":37.207999999999998,"15min":37.207999999999998},"last":2.3980000000000001},{"interface":"front","average":{"1min":2.234,"5min":2.234,"15min":2.234},"min":{"1min":0.28599999999999998,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":37.122,"5min":37.122,"15min":37.122},"last":0.47299999999999998}]},{"osd":1,"last update":"Sat Sep 23 21:14:09 2023","interfaces":[{"interface":"back","average":{"1min":0.94299999999999995,"5min":1.2310000000000001,"15min":2.04},"min":{"1min":0.30399999999999999,"5min":0.27100000000000002,"15min":0.26700000000000002},"max":{"1min":7.5369999999999999,"5min":14.034000000000001,"15min":14.034000000000001},"last":3.133},{"interface":"front","average":{"1min":1.0469999999999999,"5min":1.4339999999999999,"15min":1.944},"min":{"1min":0.30499999999999999,"5min":0.29199999999999998,"15min":0.26300000000000001},"max":{"1min":5.6870000000000003,"5min":11.226000000000001,"15min":11.226000000000001},"last":0.51900000000000002}]},{"osd":2,"last update":"Sat Sep 23 21:14:30 2023","interfaces":[{"interface":"back","average":{"1min":0.80900000000000005,"5min":1.0880000000000001,"15min":1.7170000000000001},"min":{"1min":0.29699999999999999,"5min":0.26400000000000001,"15min":0.26200000000000001},"max":{"1min":3.0190000000000001,"5min":6.4219999999999997,"15min":14.478999999999999},"last":4.7599999999999998},{"interface":"front","average":{"1min":1.101,"5min":1.3520000000000001,"15min":1.7330000000000001},"min":{"1min":0.30399999999999999,"5min":0.26700000000000002,"15min":0.26500000000000001},"max":{"1min":5.7640000000000002,"5min":11.198,"15min":14.135},"last":0.371}]},{"osd":3,"last update":"Sat Sep 23 21:13:57 2023","interfaces":[{"interface":"back","average":{"1min":1.022,"5min":1.296,"15min":2.0590000000000002},"min":{"1min":0.33000000000000002,"5min":0.316,"15min":0.29099999999999998},"max":{"1min":6.4930000000000003,"5min":7.5190000000000001,"15min":31.786000000000001},"last":3.173},{"interface":"front","average":{"1min":1.1160000000000001,"5min":1.3859999999999999,"15min":2.0409999999999999},"min":{"1min":0.37,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":5.9329999999999998,"5min":11.244999999999999,"15min":32.429000000000002},"last":2.2189999999999999}]},{"osd":4,"last update":"Sat Sep 23 21:13:53 2023","interfaces":[{"interface":"back","average":{"1min":1.5980000000000001,"5min":1.5209999999999999,"15min":1.097},"min":{"1min":0.28999999999999998,"5min":0.255,"15min":0.248},"max":{"1min":7.6609999999999996,"5min":11.297000000000001,"15min":11.297000000000001},"last":0.54400000000000004},{"interface":"front","average":{"1min":1.387,"5min":1.1679999999999999,"15min":1.0669999999999999},"min":{"1min":0.314,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":8.6059999999999999,"5min":11.093999999999999,"15min":12.752000000000001},"last":0.44800000000000001}]},{"osd":5,"last update":"Sat Sep 23 21:13:53 2023","interfaces":[{"interface":"back","average":{"1min":1.4670000000000001,"5min":1.0549999999999999,"15min":1.014},"min":{"1min":0.29899999999999999,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":7.5910000000000002,"5min":9.4789999999999992,"15min":9.4789999999999992},"last":3.194},{"interface":"front","average":{"1min":1.4950000000000001,"5min":1.3680000000000001,"15min":1.099},"min":{"1min":0.39700000000000002,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":7.6929999999999996,"5min":11.257,"15min":11.257},"last":0.33300000000000002}]},{"osd":6,"last update":"Sat Sep 23 21:14:09 2023","interfaces":[{"interface":"back","average":{"1min":0.79600000000000004,"5min":1.1819999999999999,"15min":1.038},"min":{"1min":0.39800000000000002,"5min":0.30099999999999999,"15min":0.28899999999999998},"max":{"1min":1.667,"5min":10.941000000000001,"15min":10.941000000000001},"last":0.60999999999999999},{"interface":"front","average":{"1min":0.99299999999999999,"5min":1.1020000000000001,"15min":1.1259999999999999},"min":{"1min":0.41699999999999998,"5min":0.33100000000000002,"15min":0.32300000000000001},"max":{"1min":2.4969999999999999,"5min":8.923,"15min":12.715},"last":0.41699999999999998}]}]},{"osd":6,"up_from":0,"seq":704374636776,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":1,"up_from":0,"seq":1503238553708,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":0,"up_from":493,"seq":2117418876949,"num_pgs":75,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1760592,"kb_used_data":1584264,"kb_used_omap":36,"kb_used_meta":176283,"kb_avail":91984560,"statfs":{"total":95995035648,"available":94192189440,"internally_reserved":0,"allocated":1622286336,"data_stored":1619801555,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":37336,"internal_metadata":180514344},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,1,3],"upper_bound":16},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 21:14:12 2023","interfaces":[{"interface":"back","average":{"1min":0.75600000000000001,"5min":0.75600000000000001,"15min":0.75600000000000001},"min":{"1min":0.23000000000000001,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":4.9000000000000004,"5min":4.9000000000000004,"15min":4.9000000000000004},"last":0.68600000000000005},{"interface":"front","average":{"1min":0.77100000000000002,"5min":0.77100000000000002,"15min":0.77100000000000002},"min":{"1min":0.24299999999999999,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":3.444,"5min":3.444,"15min":3.444},"last":0.20799999999999999}]},{"osd":2,"last update":"Sat Sep 23 21:14:12 2023","interfaces":[{"interface":"back","average":{"1min":0.65400000000000003,"5min":0.65400000000000003,"15min":0.65400000000000003},"min":{"1min":0.26900000000000002,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":4.8630000000000004,"5min":4.8630000000000004,"15min":4.8630000000000004},"last":0.746},{"interface":"front","average":{"1min":0.96799999999999997,"5min":0.96799999999999997,"15min":0.96799999999999997},"min":{"1min":0.29299999999999998,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":4.649,"5min":4.649,"15min":4.649},"last":0.42299999999999999}]},{"osd":3,"last update":"Sat Sep 23 21:14:12 2023","interfaces":[{"interface":"back","average":{"1min":0.86799999999999999,"5min":0.86799999999999999,"15min":0.86799999999999999},"min":{"1min":0.23400000000000001,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":4.5110000000000001,"5min":4.5110000000000001,"15min":4.5110000000000001},"last":0.52400000000000002},{"interface":"front","average":{"1min":1.1180000000000001,"5min":1.1180000000000001,"15min":1.1180000000000001},"min":{"1min":0.26200000000000001,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":3.7200000000000002,"5min":3.7200000000000002,"15min":3.7200000000000002},"last":0.81399999999999995}]},{"osd":4,"last update":"Sat Sep 23 21:14:12 2023","interfaces":[{"interface":"back","average":{"1min":1.093,"5min":1.093,"15min":1.093},"min":{"1min":0.30199999999999999,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":3.7799999999999998,"5min":3.7799999999999998,"15min":3.7799999999999998},"last":4.4409999999999998},{"interface":"front","average":{"1min":0.996,"5min":0.996,"15min":0.996},"min":{"1min":0.317,"5min":0.317,"15min":0.317},"max":{"1min":5.093,"5min":5.093,"15min":5.093},"last":0.79400000000000004}]},{"osd":5,"last update":"Sat Sep 23 21:14:12 2023","interfaces":[{"interface":"back","average":{"1min":1.236,"5min":1.236,"15min":1.236},"min":{"1min":0.33200000000000002,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":4.5640000000000001,"5min":4.5640000000000001,"15min":4.5640000000000001},"last":0.29999999999999999},{"interface":"front","average":{"1min":1.0640000000000001,"5min":1.0640000000000001,"15min":1.0640000000000001},"min":{"1min":0.33200000000000002,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":4.5599999999999996,"5min":4.5599999999999996,"15min":4.5599999999999996},"last":0.84099999999999997}]},{"osd":6,"last update":"Sat Sep 23 21:14:12 2023","interfaces":[{"interface":"back","average":{"1min":1.127,"5min":1.127,"15min":1.127},"min":{"1min":0.38100000000000001,"5min":0.38100000000000001,"15min":0.38100000000000001},"max":{"1min":6.2750000000000004,"5min":6.2750000000000004,"15min":6.2750000000000004},"last":0.83299999999999996},{"interface":"front","average":{"1min":1.3080000000000001,"5min":1.3080000000000001,"15min":1.3080000000000001},"min":{"1min":0.35599999999999998,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":5.0860000000000003,"5min":5.0860000000000003,"15min":5.0860000000000003},"last":0.39600000000000002}]},{"osd":7,"last update":"Sat Sep 23 21:14:12 2023","interfaces":[{"interface":"back","average":{"1min":1.2090000000000001,"5min":1.2090000000000001,"15min":1.2090000000000001},"min":{"1min":0.41099999999999998,"5min":0.41099999999999998,"15min":0.41099999999999998},"max":{"1min":5.1539999999999999,"5min":5.1539999999999999,"15min":5.1539999999999999},"last":3.9540000000000002},{"interface":"front","average":{"1min":1.1579999999999999,"5min":1.1579999999999999,"15min":1.1579999999999999},"min":{"1min":0.441,"5min":0.441,"15min":0.441},"max":{"1min":5.1289999999999996,"5min":5.1289999999999996,"15min":5.1289999999999996},"last":0.86199999999999999}]}]},{"osd":2,"up_from":93,"seq":399431958821,"num_pgs":69,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1597456,"kb_used_data":822024,"kb_used_omap":81,"kb_used_meta":775342,"kb_avail":92147696,"statfs":{"total":95995035648,"available":94359240704,"internally_reserved":0,"allocated":841752576,"data_stored":839266836,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":83750,"internal_metadata":793950426},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,1,3],"upper_bound":32},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:14:15 2023","interfaces":[{"interface":"back","average":{"1min":0.83299999999999996,"5min":0.83299999999999996,"15min":0.83299999999999996},"min":{"1min":0.20899999999999999,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":3.7290000000000001,"5min":3.7290000000000001,"15min":3.7290000000000001},"last":3.915},{"interface":"front","average":{"1min":1.665,"5min":1.665,"15min":1.665},"min":{"1min":0.23499999999999999,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":18.111999999999998,"5min":18.111999999999998,"15min":18.111999999999998},"last":4.4420000000000002}]},{"osd":1,"last update":"Sat Sep 23 21:14:15 2023","interfaces":[{"interface":"back","average":{"1min":0.95099999999999996,"5min":1.474,"15min":2.5339999999999998},"min":{"1min":0.23999999999999999,"5min":0.188,"15min":0.188},"max":{"1min":6.0499999999999998,"5min":12.484999999999999,"15min":34.396999999999998},"last":0.34100000000000003},{"interface":"front","average":{"1min":1.597,"5min":1.3129999999999999,"15min":2.782},"min":{"1min":0.29899999999999999,"5min":0.217,"15min":0.217},"max":{"1min":18.081,"5min":18.081,"15min":34.462000000000003},"last":4.141}]},{"osd":3,"last update":"Sat Sep 23 21:13:56 2023","interfaces":[{"interface":"back","average":{"1min":0.495,"5min":1.5429999999999999,"15min":2.6419999999999999},"min":{"1min":0.20599999999999999,"5min":0.20599999999999999,"15min":0.20599999999999999},"max":{"1min":1.679,"5min":12.951000000000001,"15min":41.057000000000002},"last":1.083},{"interface":"front","average":{"1min":0.749,"5min":1.661,"15min":2.915},"min":{"1min":0.28499999999999998,"5min":0.21299999999999999,"15min":0.21299999999999999},"max":{"1min":3.6989999999999998,"5min":13.159000000000001,"15min":41.012},"last":3.8050000000000002}]},{"osd":4,"last update":"Sat Sep 23 21:14:15 2023","interfaces":[{"interface":"back","average":{"1min":0.64800000000000002,"5min":1.256,"15min":1.4219999999999999},"min":{"1min":0.34100000000000003,"5min":0.249,"15min":0.249},"max":{"1min":1.7210000000000001,"5min":10.266999999999999,"15min":10.266999999999999},"last":7.9770000000000003},{"interface":"front","average":{"1min":1.0029999999999999,"5min":1.456,"15min":1.8759999999999999},"min":{"1min":0.38300000000000001,"5min":0.29399999999999998,"15min":0.26700000000000002},"max":{"1min":3.5819999999999999,"5min":8.6780000000000008,"15min":14.039999999999999},"last":3.6880000000000002}]},{"osd":5,"last update":"Sat Sep 23 21:14:15 2023","interfaces":[{"interface":"back","average":{"1min":0.94399999999999995,"5min":1.276,"15min":1.5209999999999999},"min":{"1min":0.311,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":3.3759999999999999,"5min":9.5039999999999996,"15min":10.069000000000001},"last":0.36499999999999999},{"interface":"front","average":{"1min":0.83999999999999997,"5min":1.3149999999999999,"15min":1.8120000000000001},"min":{"1min":0.41499999999999998,"5min":0.29699999999999999,"15min":0.29399999999999998},"max":{"1min":3.629,"5min":8.7270000000000003,"15min":13.98},"last":0.39500000000000002}]},{"osd":6,"last update":"Sat Sep 23 21:14:15 2023","interfaces":[{"interface":"back","average":{"1min":0.94999999999999996,"5min":1.569,"15min":2.0529999999999999},"min":{"1min":0.436,"5min":0.373,"15min":0.35699999999999998},"max":{"1min":3.665,"5min":8.7680000000000007,"15min":14.016},"last":3.657},{"interface":"front","average":{"1min":1.2450000000000001,"5min":1.7729999999999999,"15min":2.2330000000000001},"min":{"1min":0.45700000000000002,"5min":0.38100000000000001,"15min":0.35599999999999998},"max":{"1min":3.7610000000000001,"5min":9.3469999999999995,"15min":14.068},"last":3.7120000000000002}]},{"osd":7,"last update":"Sat Sep 23 21:14:15 2023","interfaces":[{"interface":"back","average":{"1min":1.8120000000000001,"5min":1.4790000000000001,"15min":1.5840000000000001},"min":{"1min":0.433,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":18.021999999999998,"5min":18.021999999999998,"15min":18.021999999999998},"last":4.1769999999999996},{"interface":"front","average":{"1min":1.242,"5min":1.476,"15min":1.675},"min":{"1min":0.39400000000000002,"5min":0.34799999999999998,"15min":0.32200000000000001},"max":{"1min":9.6349999999999998,"5min":10.359,"15min":10.92},"last":1.0469999999999999}]}]},{"osd":3,"up_from":98,"seq":420906795294,"num_pgs":82,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1849708,"kb_used_data":984996,"kb_used_omap":7,"kb_used_meta":864440,"kb_avail":91895444,"statfs":{"total":95995035648,"available":94100934656,"internally_reserved":0,"allocated":1008635904,"data_stored":1006179787,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7576,"internal_metadata":885187176},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:14:12 2023","interfaces":[{"interface":"back","average":{"1min":0.33800000000000002,"5min":0.33800000000000002,"15min":0.33800000000000002},"min":{"1min":0.20499999999999999,"5min":0.20499999999999999,"15min":0.20499999999999999},"max":{"1min":0.69899999999999995,"5min":0.69899999999999995,"15min":0.69899999999999995},"last":4.3179999999999996},{"interface":"front","average":{"1min":0.66800000000000004,"5min":0.66800000000000004,"15min":0.66800000000000004},"min":{"1min":0.23300000000000001,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":4.5510000000000002,"5min":4.5510000000000002,"15min":4.5510000000000002},"last":1.7090000000000001}]},{"osd":1,"last update":"Sat Sep 23 21:14:12 2023","interfaces":[{"interface":"back","average":{"1min":1.052,"5min":1.466,"15min":2.226},"min":{"1min":0.218,"5min":0.20000000000000001,"15min":0.20000000000000001},"max":{"1min":5.5019999999999998,"5min":22.577999999999999,"15min":22.577999999999999},"last":4.6120000000000001},{"interface":"front","average":{"1min":0.60099999999999998,"5min":1.5580000000000001,"15min":2.1829999999999998},"min":{"1min":0.249,"5min":0.216,"15min":0.216},"max":{"1min":2.7519999999999998,"5min":21.617999999999999,"15min":21.617999999999999},"last":4.2119999999999997}]},{"osd":2,"last update":"Sat Sep 23 21:14:00 2023","interfaces":[{"interface":"back","average":{"1min":3.6669999999999998,"5min":1.9279999999999999,"15min":2.6640000000000001},"min":{"1min":0.221,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":56.317999999999998,"5min":56.317999999999998,"15min":56.317999999999998},"last":3.1659999999999999},{"interface":"front","average":{"1min":3.8479999999999999,"5min":2.1499999999999999,"15min":2.702},"min":{"1min":0.29699999999999999,"5min":0.216,"15min":0.216},"max":{"1min":55.988,"5min":55.988,"15min":55.988},"last":3.1269999999999998}]},{"osd":4,"last update":"Sat Sep 23 21:14:00 2023","interfaces":[{"interface":"back","average":{"1min":0.71499999999999997,"5min":1.3029999999999999,"15min":1.8779999999999999},"min":{"1min":0.30299999999999999,"5min":0.30299999999999999,"15min":0.28399999999999997},"max":{"1min":5.0819999999999999,"5min":21.478999999999999,"15min":21.478999999999999},"last":1.742},{"interface":"front","average":{"1min":1.3009999999999999,"5min":1.5760000000000001,"15min":2.0070000000000001},"min":{"1min":0.33100000000000002,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":4.1909999999999998,"5min":22.402000000000001,"15min":22.402000000000001},"last":1.3340000000000001}]},{"osd":5,"last update":"Sat Sep 23 21:14:00 2023","interfaces":[{"interface":"back","average":{"1min":0.93600000000000005,"5min":1.3120000000000001,"15min":2.012},"min":{"1min":0.32200000000000001,"5min":0.311,"15min":0.311},"max":{"1min":5.04,"5min":21.562000000000001,"15min":21.562000000000001},"last":1.784},{"interface":"front","average":{"1min":0.83699999999999997,"5min":1.339,"15min":2.2839999999999998},"min":{"1min":0.34300000000000003,"5min":0.31,"15min":0.31},"max":{"1min":3.4780000000000002,"5min":8.8450000000000006,"15min":14.029},"last":4.1639999999999997}]},{"osd":6,"last update":"Sat Sep 23 21:14:15 2023","interfaces":[{"interface":"back","average":{"1min":1.254,"5min":1.591,"15min":2.0510000000000002},"min":{"1min":0.38900000000000001,"5min":0.34999999999999998,"15min":0.33700000000000002},"max":{"1min":4.2380000000000004,"5min":22.446999999999999,"15min":22.446999999999999},"last":2.8740000000000001},{"interface":"front","average":{"1min":1.2450000000000001,"5min":1.746,"15min":2.278},"min":{"1min":0.44,"5min":0.39000000000000001,"15min":0.373},"max":{"1min":9.3889999999999993,"5min":12.375999999999999,"15min":13.576000000000001},"last":4.2469999999999999}]},{"osd":7,"last update":"Sat Sep 23 21:14:00 2023","interfaces":[{"interface":"back","average":{"1min":1.032,"5min":1.6120000000000001,"15min":2.1299999999999999},"min":{"1min":0.41199999999999998,"5min":0.38400000000000001,"15min":0.33900000000000002},"max":{"1min":5.0090000000000003,"5min":21.518999999999998,"15min":21.518999999999998},"last":1.8100000000000001},{"interface":"front","average":{"1min":0.92900000000000005,"5min":1.7050000000000001,"15min":2.4199999999999999},"min":{"1min":0.45800000000000002,"5min":0.442,"15min":0.38900000000000001},"max":{"1min":3.2679999999999998,"5min":9.9879999999999995,"15min":13.603999999999999},"last":4.2800000000000002}]}]},{"osd":4,"up_from":31,"seq":133143986600,"num_pgs":77,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1360120,"kb_used_data":887344,"kb_used_omap":0,"kb_used_meta":472768,"kb_avail":92385032,"statfs":{"total":95995035648,"available":94602272768,"internally_reserved":0,"allocated":908640256,"data_stored":907096660,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":484114432},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,0,0,1,2,4,1,0,1],"upper_bound":512},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:14:14 2023","interfaces":[{"interface":"back","average":{"1min":0.90000000000000002,"5min":0.90000000000000002,"15min":0.90000000000000002},"min":{"1min":0.22500000000000001,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":2.6539999999999999,"5min":2.6539999999999999,"15min":2.6539999999999999},"last":3.3340000000000001},{"interface":"front","average":{"1min":1.1339999999999999,"5min":1.1339999999999999,"15min":1.1339999999999999},"min":{"1min":0.246,"5min":0.246,"15min":0.246},"max":{"1min":10.945,"5min":10.945,"15min":10.945},"last":0.23799999999999999}]},{"osd":1,"last update":"Sat Sep 23 21:14:14 2023","interfaces":[{"interface":"back","average":{"1min":0.77300000000000002,"5min":1.0609999999999999,"15min":2.7490000000000001},"min":{"1min":0.29699999999999999,"5min":0.221,"15min":0.221},"max":{"1min":2.387,"5min":7.8609999999999998,"15min":37.960000000000001},"last":0.499},{"interface":"front","average":{"1min":0.80600000000000005,"5min":1.1339999999999999,"15min":2.6080000000000001},"min":{"1min":0.28499999999999998,"5min":0.28499999999999998,"15min":0.27000000000000002},"max":{"1min":3.581,"5min":7.1349999999999998,"15min":37.933},"last":13.983000000000001}]},{"osd":2,"last update":"Sat Sep 23 21:14:14 2023","interfaces":[{"interface":"back","average":{"1min":0.82099999999999995,"5min":1.153,"15min":1.748},"min":{"1min":0.27400000000000002,"5min":0.247,"15min":0.247},"max":{"1min":3.476,"5min":10.464,"15min":16.928000000000001},"last":14.026},{"interface":"front","average":{"1min":0.86099999999999999,"5min":1.3959999999999999,"15min":1.7809999999999999},"min":{"1min":0.311,"5min":0.28599999999999998,"15min":0.27800000000000002},"max":{"1min":3.5129999999999999,"5min":7.3659999999999997,"15min":14.417999999999999},"last":14.339}]},{"osd":3,"last update":"Sat Sep 23 21:14:47 2023","interfaces":[{"interface":"back","average":{"1min":1.0389999999999999,"5min":1.1990000000000001,"15min":2.056},"min":{"1min":0.27900000000000003,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":2.766,"5min":9.3279999999999994,"15min":15.01},"last":0.27900000000000003},{"interface":"front","average":{"1min":1.899,"5min":1.3879999999999999,"15min":2},"min":{"1min":0.32200000000000001,"5min":0.32200000000000001,"15min":0.30199999999999999},"max":{"1min":14.305,"5min":14.305,"15min":15.004},"last":14.305}]},{"osd":5,"last update":"Sat Sep 23 21:14:04 2023","interfaces":[{"interface":"back","average":{"1min":1.169,"5min":1.018,"15min":0.97599999999999998},"min":{"1min":0.30099999999999999,"5min":0.222,"15min":0.222},"max":{"1min":5.1609999999999996,"5min":9.4909999999999997,"15min":14.298},"last":14.398999999999999},{"interface":"front","average":{"1min":1.0189999999999999,"5min":0.95899999999999996,"15min":1.0760000000000001},"min":{"1min":0.28000000000000003,"5min":0.27200000000000002,"15min":0.255},"max":{"1min":3.0059999999999998,"5min":6.8730000000000002,"15min":9.8979999999999997},"last":3.4159999999999999}]},{"osd":6,"last update":"Sat Sep 23 21:14:14 2023","interfaces":[{"interface":"back","average":{"1min":0.747,"5min":1.0920000000000001,"15min":1.2230000000000001},"min":{"1min":0.317,"5min":0.28599999999999998,"15min":0.26800000000000002},"max":{"1min":2.8650000000000002,"5min":24.452999999999999,"15min":24.452999999999999},"last":0.34300000000000003},{"interface":"front","average":{"1min":0.95099999999999996,"5min":1.036,"15min":0.95399999999999996},"min":{"1min":0.40100000000000002,"5min":0.33500000000000002,"15min":0.32500000000000001},"max":{"1min":3.6600000000000001,"5min":11.409000000000001,"15min":14.387},"last":14.374000000000001}]},{"osd":7,"last update":"Sat Sep 23 21:14:47 2023","interfaces":[{"interface":"back","average":{"1min":1.9910000000000001,"5min":1.355,"15min":1.1299999999999999},"min":{"1min":0.38300000000000001,"5min":0.30099999999999999,"15min":0.29299999999999998},"max":{"1min":14.422000000000001,"5min":14.422000000000001,"15min":16.890999999999998},"last":14.422000000000001},{"interface":"front","average":{"1min":1.536,"5min":1.234,"15min":1.2090000000000001},"min":{"1min":0.39200000000000002,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":7.7229999999999999,"5min":7.7229999999999999,"15min":11.170999999999999},"last":3.383}]}]},{"osd":5,"up_from":39,"seq":167503724967,"num_pgs":61,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1068580,"kb_used_data":571676,"kb_used_omap":0,"kb_used_meta":496896,"kb_avail":92676572,"statfs":{"total":95995035648,"available":94900809728,"internally_reserved":0,"allocated":585396224,"data_stored":583868818,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":508821504},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,3],"upper_bound":4},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:14:13 2023","interfaces":[{"interface":"back","average":{"1min":0.71499999999999997,"5min":0.71499999999999997,"15min":0.71499999999999997},"min":{"1min":0.251,"5min":0.251,"15min":0.251},"max":{"1min":3.3889999999999998,"5min":3.3889999999999998,"15min":3.3889999999999998},"last":1.9630000000000001},{"interface":"front","average":{"1min":0.66900000000000004,"5min":0.66900000000000004,"15min":0.66900000000000004},"min":{"1min":0.22800000000000001,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":3.1150000000000002,"5min":3.1150000000000002,"15min":3.1150000000000002},"last":3.9390000000000001}]},{"osd":1,"last update":"Sat Sep 23 21:14:08 2023","interfaces":[{"interface":"back","average":{"1min":0.65200000000000002,"5min":1.1479999999999999,"15min":1.728},"min":{"1min":0.248,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":3.109,"5min":9.4429999999999996,"15min":25.239999999999998},"last":0.44900000000000001},{"interface":"front","average":{"1min":1.468,"5min":1.681,"15min":1.784},"min":{"1min":0.34999999999999998,"5min":0.28699999999999998,"15min":0.252},"max":{"1min":11.116,"5min":11.116,"15min":25.253},"last":2.028}]},{"osd":2,"last update":"Sat Sep 23 21:14:26 2023","interfaces":[{"interface":"back","average":{"1min":0.73099999999999998,"5min":1.401,"15min":1.7110000000000001},"min":{"1min":0.27800000000000002,"5min":0.26700000000000002,"15min":0.23499999999999999},"max":{"1min":4.0490000000000004,"5min":9.2889999999999997,"15min":12.634},"last":2.0430000000000001},{"interface":"front","average":{"1min":0.78600000000000003,"5min":1.222,"15min":1.762},"min":{"1min":0.30199999999999999,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":2.524,"5min":9.7170000000000005,"15min":10.333},"last":2.4340000000000002}]},{"osd":3,"last update":"Sat Sep 23 21:14:48 2023","interfaces":[{"interface":"back","average":{"1min":1.0109999999999999,"5min":1.6339999999999999,"15min":2.484},"min":{"1min":0.32400000000000001,"5min":0.311,"15min":0.29299999999999998},"max":{"1min":3.0390000000000001,"5min":14.242000000000001,"15min":61.667999999999999},"last":4.6790000000000003},{"interface":"front","average":{"1min":1.284,"5min":1.6559999999999999,"15min":2.5910000000000002},"min":{"1min":0.39100000000000001,"5min":0.34899999999999998,"15min":0.30599999999999999},"max":{"1min":6.508,"5min":12.44,"15min":62.276000000000003},"last":2.8919999999999999}]},{"osd":4,"last update":"Sat Sep 23 21:13:59 2023","interfaces":[{"interface":"back","average":{"1min":1.452,"5min":1.387,"15min":1.139},"min":{"1min":0.28799999999999998,"5min":0.27000000000000002,"15min":0.23400000000000001},"max":{"1min":11.156000000000001,"5min":12.776,"15min":12.776},"last":1.9239999999999999},{"interface":"front","average":{"1min":1.5109999999999999,"5min":1.1799999999999999,"15min":1.1319999999999999},"min":{"1min":0.28000000000000003,"5min":0.27000000000000002,"15min":0.26900000000000002},"max":{"1min":9.7119999999999997,"5min":9.7119999999999997,"15min":11.973000000000001},"last":2.008}]},{"osd":6,"last update":"Sat Sep 23 21:14:08 2023","interfaces":[{"interface":"back","average":{"1min":0.625,"5min":0.92200000000000004,"15min":1.056},"min":{"1min":0.38800000000000001,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":1.448,"5min":7.7030000000000003,"15min":11.128},"last":0.41699999999999998},{"interface":"front","average":{"1min":1.4350000000000001,"5min":1.4390000000000001,"15min":1.246},"min":{"1min":0.42099999999999999,"5min":0.32400000000000001,"15min":0.317},"max":{"1min":10.999000000000001,"5min":10.999000000000001,"15min":11.112},"last":1.9890000000000001}]},{"osd":7,"last update":"Sat Sep 23 21:14:48 2023","interfaces":[{"interface":"back","average":{"1min":0.84999999999999998,"5min":0.98499999999999999,"15min":1.8129999999999999},"min":{"1min":0.34499999999999997,"5min":0.31,"15min":0.309},"max":{"1min":3.3039999999999998,"5min":5.5659999999999998,"15min":189.405},"last":0.91900000000000004},{"interface":"front","average":{"1min":0.92000000000000004,"5min":1.472,"15min":1.9259999999999999},"min":{"1min":0.40899999999999997,"5min":0.35399999999999998,"15min":0.32500000000000001},"max":{"1min":3.3239999999999998,"5min":11.183999999999999,"15min":189.685},"last":4.5369999999999999}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8488,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1259,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5322,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":714,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":5434,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":4838,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1585709056,"data_stored":1585709056,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1173159936,"data_stored":1173159936,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":830734336,"data_stored":830734336,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":999886848,"data_stored":999882776,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":882118656,"data_stored":882114584,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":565968896,"data_stored":565968896,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":424083456,"data_stored":424083456,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T21:14:57.559 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph status --format=json 2023-09-23T21:14:57.560 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:14:57.561 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:14:57.661 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:14:57.736 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:14:58.122 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:14:57 smithi070 ceph-mon[168282]: pgmap v1345: 129 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 15 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 108 active+clean; 989 MiB data, 7.7 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 752 op/s; 20724/47514 objects degraded (43.617%); 3079/47514 objects misplaced (6.480%); 11 MiB/s, 186 objects/s recovering 2023-09-23T21:14:58.131 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:14:57 smithi026 ceph-mon[179688]: pgmap v1345: 129 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 15 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 108 active+clean; 989 MiB data, 7.7 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 752 op/s; 20724/47514 objects degraded (43.617%); 3079/47514 objects misplaced (6.480%); 11 MiB/s, 186 objects/s recovering 2023-09-23T21:14:58.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:14:57 smithi026 ceph-mon[175151]: pgmap v1345: 129 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 15 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 108 active+clean; 989 MiB data, 7.7 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 752 op/s; 20724/47514 objects degraded (43.617%); 3079/47514 objects misplaced (6.480%); 11 MiB/s, 186 objects/s recovering 2023-09-23T21:14:58.285 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:14:58.297 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:14:58.298 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:14:58.365 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:14:58.399 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:14:58.489 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:14:58.767 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:14:58.772 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:14:58.773 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:14:58.782 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:14:58.874 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:14:58.897 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:14:58.961 INFO:teuthology.orchestra.run.smithi026.stdout:ERROR: (22) Invalid argument 2023-09-23T21:14:58.961 INFO:teuthology.orchestra.run.smithi026.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T21:14:58.975 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:14:59.178 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T21:14:59.184 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:14:59.184 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:14:59.285 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:14:59.367 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:14:59.692 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T21:14:59.695 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:14:59.695 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:14:59.796 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:14:59.869 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 21:14:59 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[164029]: ::ffff:172.21.15.70 - - [23/Sep/2023:21:14:59] "GET /metrics HTTP/1.1" 200 36397 "" "Prometheus/2.43.0" 2023-09-23T21:14:59.881 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:14:59.954 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:15:00.022 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:15:00.022 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:15:00.039 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_blocked_ops 2023-09-23T21:15:00.123 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:15:00.183 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:15:00.241 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:15:00.249 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:15:00.250 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:15:00.350 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:15:00.389 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:15:00.427 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:15:00 smithi026 ceph-mon[179688]: pgmap v1346: 129 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 15 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 108 active+clean; 883 MiB data, 7.4 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 748 op/s; 20046/42444 objects degraded (47.229%); 2941/42444 objects misplaced (6.929%); 11 MiB/s, 199 objects/s recovering 2023-09-23T21:15:00.427 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:15:00 smithi026 ceph-mon[175151]: pgmap v1346: 129 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 15 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 108 active+clean; 883 MiB data, 7.4 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 748 op/s; 20046/42444 objects degraded (47.229%); 2941/42444 objects misplaced (6.929%); 11 MiB/s, 199 objects/s recovering 2023-09-23T21:15:00.454 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:15:00.460 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:15:00 smithi070 ceph-mon[168282]: pgmap v1346: 129 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 15 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 108 active+clean; 883 MiB data, 7.4 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 748 op/s; 20046/42444 objects degraded (47.229%); 2941/42444 objects misplaced (6.929%); 11 MiB/s, 199 objects/s recovering 2023-09-23T21:15:00.460 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:15:00.461 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:15:00.561 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:15:00.604 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:15:00.666 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:15:00.672 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:15:00.673 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:15:00.774 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:15:00.885 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:15:01.207 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:15:01.224 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:15:01.224 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:15:01.325 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:15:01.360 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:15:01.407 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:15:01.415 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:15:01.415 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:15:01.415 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:15:01 smithi070 ceph-mon[168282]: Health check update: Degraded data redundancy: 20046/42444 objects degraded (47.229%), 19 pgs degraded, 16 pgs undersized (PG_DEGRADED) 2023-09-23T21:15:01.416 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:15:01 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:15:01.416 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:15:01 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:15:01.516 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:15:01.551 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:15:01.607 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:15:01.617 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:15:01 smithi026 ceph-mon[175151]: Health check update: Degraded data redundancy: 20046/42444 objects degraded (47.229%), 19 pgs degraded, 16 pgs undersized (PG_DEGRADED) 2023-09-23T21:15:01.617 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:15:01 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:15:01.617 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:15:01 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:15:01.617 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:15:01 smithi026 ceph-mon[179688]: Health check update: Degraded data redundancy: 20046/42444 objects degraded (47.229%), 19 pgs degraded, 16 pgs undersized (PG_DEGRADED) 2023-09-23T21:15:01.617 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:15:01 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:15:01.617 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:15:01 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:15:01.618 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:15:01.618 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:15:01.719 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:15:01.742 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:15:01.755 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:15:01.799 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:15:01.803 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:15:01.803 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:15:01.904 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:15:01.979 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:15:01.996 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0: {} 2023-09-23T21:15:02.082 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1: {} 2023-09-23T21:15:02.121 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2: {} 2023-09-23T21:15:02.176 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3: {} 2023-09-23T21:15:02.212 INFO:teuthology.orchestra.run.smithi026.stderr:nodeep-scrub is unset 2023-09-23T21:15:02.224 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4: {} 2023-09-23T21:15:02.265 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T21:15:02.300 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6: {} 2023-09-23T21:15:02.310 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:15:02 smithi026 ceph-mon[175151]: pgmap v1347: 129 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 15 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 108 active+clean; 826 MiB data, 7.2 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 744 op/s; 20046/39714 objects degraded (50.476%); 2941/39714 objects misplaced (7.405%); 11 MiB/s, 199 objects/s recovering 2023-09-23T21:15:02.310 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:15:02 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:15:02.311 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:15:02 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/2646530759' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:15:02.311 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:15:02 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:15:02.311 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:15:02 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/632855739' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:15:02.311 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:15:02 smithi026 ceph-mon[179688]: pgmap v1347: 129 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 15 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 108 active+clean; 826 MiB data, 7.2 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 744 op/s; 20046/39714 objects degraded (50.476%); 2941/39714 objects misplaced (7.405%); 11 MiB/s, 199 objects/s recovering 2023-09-23T21:15:02.311 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:15:02 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:15:02.311 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:15:02 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/2646530759' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:15:02.311 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:15:02 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:15:02.311 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:15:02 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/632855739' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:15:02.343 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T21:15:02.428 INFO:teuthology.orchestra.run.smithi026.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T21:15:02.428 INFO:teuthology.orchestra.run.smithi026.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T21:15:02.428 INFO:teuthology.orchestra.run.smithi026.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T21:15:02.428 INFO:teuthology.orchestra.run.smithi026.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T21:15:02.428 INFO:teuthology.orchestra.run.smithi026.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T21:15:02.428 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T21:15:02.428 INFO:teuthology.orchestra.run.smithi026.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T21:15:02.428 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T21:15:02.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:15:02 smithi070 ceph-mon[168282]: pgmap v1347: 129 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 15 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 108 active+clean; 826 MiB data, 7.2 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 744 op/s; 20046/39714 objects degraded (50.476%); 2941/39714 objects misplaced (7.405%); 11 MiB/s, 199 objects/s recovering 2023-09-23T21:15:02.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:15:02 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:15:02.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:15:02 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/2646530759' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:15:02.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:15:02 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:15:02.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:15:02 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/632855739' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:15:03.301 INFO:teuthology.orchestra.run.smithi026.stdout: 2023-09-23T21:15:03.302 INFO:teuthology.orchestra.run.smithi026.stdout:{"fsid":"b3f8e4ec-5a50-11ee-8db4-212e2dc638e7","health":{"status":"HEALTH_WARN","checks":{"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 20046/35742 objects degraded (56.085%), 19 pgs degraded, 16 pgs undersized","count":35},"muted":false}},"mutes":[]},"election_epoch":30,"quorum":[0,1,2],"quorum_names":["a","b","c"],"quorum_age":1621,"monmap":{"epoch":4,"min_mon_release_name":"reef","num_mons":3},"osdmap":{"epoch":532,"num_osds":8,"num_up_osds":8,"osd_up_since":1695503591,"num_in_osds":6,"osd_in_since":1695503530,"num_remapped_pgs":18},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":108},{"state_name":"active+recovery_wait+undersized+degraded+remapped","count":15},{"state_name":"active+recovery_wait+degraded","count":3},{"state_name":"active+recovery_wait+degraded+remapped","count":1},{"state_name":"active+remapped+backfill_wait","count":1},{"state_name":"active+recovering+undersized+remapped","count":1}],"num_pgs":129,"num_pools":5,"num_objects":11914,"data_bytes":779664132,"bytes_used":7427162112,"bytes_avail":568543051776,"bytes_total":575970213888,"degraded_objects":20046,"degraded_total":35742,"degraded_ratio":0.56085277824408264,"misplaced_objects":2821,"misplaced_total":35742,"misplaced_ratio":0.078926752839796319,"recovering_objects_per_sec":199,"recovering_bytes_per_sec":12059694,"recovering_keys_per_sec":0,"num_objects_recovered":1992,"num_bytes_recovered":120651776,"num_keys_recovered":0,"write_bytes_sec":0,"read_op_per_sec":0,"write_op_per_sec":746},"fsmap":{"epoch":8,"id":1,"up":1,"in":1,"max":1,"by_rank":[{"filesystem_id":1,"rank":0,"name":"foo.smithi026.znfpnl","status":"up:active","gid":14658}],"up:standby":1},"mgrmap":{"available":true,"num_standbys":1,"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"services":{"dashboard":"https://172.21.15.26:8443/","prometheus":"http://172.21.15.26:9283/"}},"servicemap":{"epoch":28,"modified":"2023-09-23T21:14:11.047835+0000","services":{}},"progress_events":{"0ed42496-3a31-41bd-a432-1bbd8c010433":{"message":"Global Recovery Event (3m)\n [=======================.....] (remaining: 36s)","progress":0.83720928430557251,"add_to_ceph_s":true},"fc746e9d-3f76-434d-b6d2-8fc9e8a72a5e":{"message":"Upgrade to 18.2.0-460-gbb0daeff (4m)\n [============................] (remaining: 5m)","progress":0.44999998807907104,"add_to_ceph_s":true}}} 2023-09-23T21:15:03.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:15:03 smithi070 ceph-mon[168282]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T21:15:03.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:15:03 smithi070 ceph-mon[168282]: failed to encode map e532 with expected crc 2023-09-23T21:15:03.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:15:03 smithi070 ceph-mon[168282]: failed to encode map e532 with expected crc 2023-09-23T21:15:03.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:15:03 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T21:15:03.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:15:03 smithi070 ceph-mon[168282]: osdmap e532: 8 total, 8 up, 6 in 2023-09-23T21:15:03.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:15:03 smithi070 ceph-mon[168282]: failed to encode map e532 with expected crc 2023-09-23T21:15:03.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:15:03 smithi070 ceph-mon[168282]: failed to encode map e532 with expected crc 2023-09-23T21:15:03.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:15:03 smithi026 ceph-mon[175151]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T21:15:03.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:15:03 smithi026 ceph-mon[175151]: failed to encode map e532 with expected crc 2023-09-23T21:15:03.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:15:03 smithi026 ceph-mon[175151]: failed to encode map e532 with expected crc 2023-09-23T21:15:03.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:15:03 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T21:15:03.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:15:03 smithi026 ceph-mon[175151]: osdmap e532: 8 total, 8 up, 6 in 2023-09-23T21:15:03.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:15:03 smithi026 ceph-mon[175151]: failed to encode map e532 with expected crc 2023-09-23T21:15:03.633 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:15:03 smithi026 ceph-mon[175151]: failed to encode map e532 with expected crc 2023-09-23T21:15:03.633 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:15:03 smithi026 ceph-mon[179688]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T21:15:03.633 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:15:03 smithi026 ceph-mon[179688]: failed to encode map e532 with expected crc 2023-09-23T21:15:03.633 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:15:03 smithi026 ceph-mon[179688]: failed to encode map e532 with expected crc 2023-09-23T21:15:03.633 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:15:03 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T21:15:03.633 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:15:03 smithi026 ceph-mon[179688]: osdmap e532: 8 total, 8 up, 6 in 2023-09-23T21:15:03.633 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:15:03 smithi026 ceph-mon[179688]: failed to encode map e532 with expected crc 2023-09-23T21:15:03.633 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:15:03 smithi026 ceph-mon[179688]: failed to encode map e532 with expected crc 2023-09-23T21:15:04.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:15:04 smithi026 ceph-mon[179688]: pgmap v1349: 129 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 15 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 108 active+clean; 744 MiB data, 6.9 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 746 op/s; 20046/35742 objects degraded (56.085%); 2821/35742 objects misplaced (7.893%); 12 MiB/s, 199 objects/s recovering 2023-09-23T21:15:04.383 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:15:04 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/1498793926' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-23T21:15:04.383 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:15:04 smithi026 ceph-mon[175151]: pgmap v1349: 129 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 15 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 108 active+clean; 744 MiB data, 6.9 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 746 op/s; 20046/35742 objects degraded (56.085%); 2821/35742 objects misplaced (7.893%); 12 MiB/s, 199 objects/s recovering 2023-09-23T21:15:04.383 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:15:04 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/1498793926' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-23T21:15:04.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:15:04 smithi070 ceph-mon[168282]: pgmap v1349: 129 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 15 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 108 active+clean; 744 MiB data, 6.9 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 746 op/s; 20046/35742 objects degraded (56.085%); 2821/35742 objects misplaced (7.893%); 12 MiB/s, 199 objects/s recovering 2023-09-23T21:15:04.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:15:04 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/1498793926' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-23T21:15:04.772 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:15:04.780 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:15:04.781 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:15:04.819 INFO:tasks.cephadm.ceph_manager.ceph:making progress, resetting timeout 2023-09-23T21:15:04.819 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T21:15:04.881 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:15:04.991 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:15:05.170 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:15:05.177 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:15:05.178 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:15:05.279 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:15:05.387 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:15:05.563 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:15:05.578 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:15:05.579 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:15:05.680 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:15:05.782 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:15:06.198 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:15:06.207 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:15:06.208 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:15:06.310 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:15:06.350 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:15:06.383 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:15:06 smithi026 ceph-mon[175151]: pgmap v1350: 129 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 15 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 108 active+clean; 607 MiB data, 6.7 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 750 op/s; 19531/29211 objects degraded (66.862%); 2794/29211 objects misplaced (9.565%); 9.8 MiB/s, 197 objects/s recovering 2023-09-23T21:15:06.383 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:15:06 smithi026 ceph-mon[175151]: Health check update: Degraded data redundancy: 19531/29211 objects degraded (66.862%), 19 pgs degraded, 16 pgs undersized (PG_DEGRADED) 2023-09-23T21:15:06.383 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:15:06 smithi026 ceph-mon[179688]: pgmap v1350: 129 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 15 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 108 active+clean; 607 MiB data, 6.7 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 750 op/s; 19531/29211 objects degraded (66.862%); 2794/29211 objects misplaced (9.565%); 9.8 MiB/s, 197 objects/s recovering 2023-09-23T21:15:06.383 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:15:06 smithi026 ceph-mon[179688]: Health check update: Degraded data redundancy: 19531/29211 objects degraded (66.862%), 19 pgs degraded, 16 pgs undersized (PG_DEGRADED) 2023-09-23T21:15:06.411 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:15:06.415 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:15:06.416 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:15:06.516 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:15:06.550 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:15:06 smithi070 ceph-mon[168282]: pgmap v1350: 129 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 15 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 108 active+clean; 607 MiB data, 6.7 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 750 op/s; 19531/29211 objects degraded (66.862%); 2794/29211 objects misplaced (9.565%); 9.8 MiB/s, 197 objects/s recovering 2023-09-23T21:15:06.550 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:15:06 smithi070 ceph-mon[168282]: Health check update: Degraded data redundancy: 19531/29211 objects degraded (66.862%), 19 pgs degraded, 16 pgs undersized (PG_DEGRADED) 2023-09-23T21:15:06.553 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:15:06.573 INFO:teuthology.orchestra.run.smithi026.stdout:{ 2023-09-23T21:15:06.574 INFO:teuthology.orchestra.run.smithi026.stdout: "ops": [], 2023-09-23T21:15:06.574 INFO:teuthology.orchestra.run.smithi026.stdout: "complaint_time": 30, 2023-09-23T21:15:06.574 INFO:teuthology.orchestra.run.smithi026.stdout: "num_blocked_ops": 0 2023-09-23T21:15:06.574 INFO:teuthology.orchestra.run.smithi026.stdout:} 2023-09-23T21:15:06.591 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:15:06.596 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:15:06.596 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:15:06.655 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:15:06.697 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:15:06.818 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:15:07.179 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T21:15:07.183 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:15:07.183 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:15:07.284 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:15:07.323 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:15:07.336 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_historic_ops 2023-09-23T21:15:07.369 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:15:07.373 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:15:07.373 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:15:07.474 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:15:07.507 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:15:07.547 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:15:07.552 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:15:07.552 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:15:07.615 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T21:15:07.652 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:15:07.728 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:15:07.782 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:15:07.789 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:15:07.790 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:15:07.890 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:15:07.931 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:15:08.026 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T21:15:08.029 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:15:08.029 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:15:08.130 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:15:08.155 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:15:08.190 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:15:08.192 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:15:08.193 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:15:08.214 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:15:08.248 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T21:15:08.286 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:15:08 smithi026 ceph-mon[175151]: pgmap v1351: 129 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 15 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 108 active+clean; 514 MiB data, 6.4 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 759 op/s; 19531/24717 objects degraded (79.018%); 2659/24717 objects misplaced (10.758%); 9.8 MiB/s, 197 objects/s recovering 2023-09-23T21:15:08.286 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:15:08 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:15:08.286 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:15:08 smithi026 ceph-mon[175151]: failed to encode map e533 with expected crc 2023-09-23T21:15:08.286 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:15:08 smithi026 ceph-mon[175151]: failed to encode map e533 with expected crc 2023-09-23T21:15:08.286 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:15:08 smithi026 ceph-mon[175151]: osdmap e533: 8 total, 8 up, 6 in 2023-09-23T21:15:08.286 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:15:08 smithi026 ceph-mon[175151]: failed to encode map e533 with expected crc 2023-09-23T21:15:08.286 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:15:08 smithi026 ceph-mon[175151]: failed to encode map e533 with expected crc 2023-09-23T21:15:08.287 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:15:08 smithi026 ceph-mon[175151]: failed to encode map e533 with expected crc 2023-09-23T21:15:08.287 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:15:08 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:15:08.287 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:15:08 smithi026 ceph-mon[179688]: pgmap v1351: 129 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 15 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 108 active+clean; 514 MiB data, 6.4 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 759 op/s; 19531/24717 objects degraded (79.018%); 2659/24717 objects misplaced (10.758%); 9.8 MiB/s, 197 objects/s recovering 2023-09-23T21:15:08.287 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:15:08 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:15:08.287 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:15:08 smithi026 ceph-mon[179688]: failed to encode map e533 with expected crc 2023-09-23T21:15:08.287 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:15:08 smithi026 ceph-mon[179688]: failed to encode map e533 with expected crc 2023-09-23T21:15:08.287 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:15:08 smithi026 ceph-mon[179688]: osdmap e533: 8 total, 8 up, 6 in 2023-09-23T21:15:08.287 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:15:08 smithi026 ceph-mon[179688]: failed to encode map e533 with expected crc 2023-09-23T21:15:08.287 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:15:08 smithi026 ceph-mon[179688]: failed to encode map e533 with expected crc 2023-09-23T21:15:08.287 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:15:08 smithi026 ceph-mon[179688]: failed to encode map e533 with expected crc 2023-09-23T21:15:08.288 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:15:08 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:15:08.294 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:15:08.428 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:15:08.576 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:15:08.587 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:15:08.588 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:15:08.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:15:08 smithi070 ceph-mon[168282]: pgmap v1351: 129 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 15 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 108 active+clean; 514 MiB data, 6.4 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 759 op/s; 19531/24717 objects degraded (79.018%); 2659/24717 objects misplaced (10.758%); 9.8 MiB/s, 197 objects/s recovering 2023-09-23T21:15:08.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:15:08 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:15:08.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:15:08 smithi070 ceph-mon[168282]: failed to encode map e533 with expected crc 2023-09-23T21:15:08.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:15:08 smithi070 ceph-mon[168282]: failed to encode map e533 with expected crc 2023-09-23T21:15:08.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:15:08 smithi070 ceph-mon[168282]: osdmap e533: 8 total, 8 up, 6 in 2023-09-23T21:15:08.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:15:08 smithi070 ceph-mon[168282]: failed to encode map e533 with expected crc 2023-09-23T21:15:08.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:15:08 smithi070 ceph-mon[168282]: failed to encode map e533 with expected crc 2023-09-23T21:15:08.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:15:08 smithi070 ceph-mon[168282]: failed to encode map e533 with expected crc 2023-09-23T21:15:08.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:15:08 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:15:08.689 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:15:08.805 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:15:08.942 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:15:08.951 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:15:08.951 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:15:09.025 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:15:09.048 INFO:teuthology.orchestra.run.smithi026.stdout: 2023-09-23T21:15:09.053 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:15:09.066 INFO:teuthology.orchestra.run.smithi026.stderr:dumped all 2023-09-23T21:15:09.085 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:15:09.128 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:15:09.137 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:15:09.138 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:15:09.238 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:15:09.305 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:15:09.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:15:09 smithi070 ceph-mon[168282]: failed to encode map e534 with expected crc 2023-09-23T21:15:09.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:15:09 smithi070 ceph-mon[168282]: osdmap e534: 8 total, 8 up, 6 in 2023-09-23T21:15:09.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:15:09 smithi070 ceph-mon[168282]: failed to encode map e534 with expected crc 2023-09-23T21:15:09.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:15:09 smithi070 ceph-mon[168282]: failed to encode map e534 with expected crc 2023-09-23T21:15:09.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:15:09 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T21:15:09.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:15:09 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T21:15:09.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:15:09 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:15:09.384 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:15:09 smithi026 ceph-mon[175151]: failed to encode map e534 with expected crc 2023-09-23T21:15:09.384 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:15:09 smithi026 ceph-mon[175151]: osdmap e534: 8 total, 8 up, 6 in 2023-09-23T21:15:09.384 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:15:09 smithi026 ceph-mon[175151]: failed to encode map e534 with expected crc 2023-09-23T21:15:09.384 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:15:09 smithi026 ceph-mon[175151]: failed to encode map e534 with expected crc 2023-09-23T21:15:09.384 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:15:09 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T21:15:09.384 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:15:09 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T21:15:09.385 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:15:09 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:15:09.385 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:15:09 smithi026 ceph-mon[179688]: failed to encode map e534 with expected crc 2023-09-23T21:15:09.385 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:15:09 smithi026 ceph-mon[179688]: osdmap e534: 8 total, 8 up, 6 in 2023-09-23T21:15:09.385 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:15:09 smithi026 ceph-mon[179688]: failed to encode map e534 with expected crc 2023-09-23T21:15:09.385 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:15:09 smithi026 ceph-mon[179688]: failed to encode map e534 with expected crc 2023-09-23T21:15:09.385 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:15:09 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T21:15:09.385 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:15:09 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T21:15:09.385 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:15:09 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:15:09.742 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 21:15:09 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[164029]: ::ffff:172.21.15.70 - - [23/Sep/2023:21:15:09] "GET /metrics HTTP/1.1" 200 36402 "" "Prometheus/2.43.0" 2023-09-23T21:15:09.743 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:15:09.747 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:15:09.763 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:15:09.764 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:15:09.765 INFO:teuthology.orchestra.run.smithi026.stdout:{"pg_ready":true,"pg_map":{"version":1353,"stamp":"2023-09-23T21:15:08.237330+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":521583364,"num_objects":7976,"num_object_clones":0,"num_object_copies":23928,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":18206,"num_objects_misplaced":2659,"num_objects_unfound":0,"num_objects_dirty":7976,"num_whiteouts":0,"num_read":90,"num_read_kb":166,"num_write":139528,"num_write_kb":4309782,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":44885,"num_bytes_recovered":2697104549,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":128245,"ondisk_log_size":128245,"up":387,"acting":371,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":433,"num_osds":6,"num_per_pool_osds":6,"num_per_pool_omap_osds":6,"kb":562470912,"kb_used":6531440,"kb_used_data":3200064,"kb_used_omap":125,"kb_used_meta":3330946,"kb_avail":555939472,"statfs":{"total":575970213888,"available":569282019328,"internally_reserved":0,"allocated":3276865536,"data_stored":3264779783,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":128662,"internal_metadata":3410889066},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[3,3,1,8,3],"upper_bound":32},"perf_stat":{"commit_latency_ms":6,"apply_latency_ms":6,"commit_latency_ns":6000000,"apply_latency_ns":6000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-344850432,"num_objects":-5262,"num_object_clones":0,"num_object_copies":-15786,"num_objects_missing_on_primary":-155,"num_objects_missing":-155,"num_objects_degraded":-1840,"num_objects_misplaced":-282,"num_objects_unfound":0,"num_objects_dirty":-5262,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5262,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":993,"num_bytes_recovered":43384832,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"7.149629"},"pg_stats":[{"pgid":"1.18","version":"79'161","reported_seq":653,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-09-23T21:14:56.306647+0000","last_change":"2023-09-23T21:12:30.438733+0000","last_active":"2023-09-23T21:14:56.306647+0000","last_peered":"2023-09-23T21:14:56.306647+0000","last_clean":"2023-09-23T21:14:56.306647+0000","last_became_active":"2023-09-23T21:12:30.438356+0000","last_became_peered":"2023-09-23T21:12:30.438356+0000","last_unstale":"2023-09-23T21:14:56.306647+0000","last_undegraded":"2023-09-23T21:14:56.306647+0000","last_fullsized":"2023-09-23T21:14:56.306647+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":479,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:06:52.098882+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:06:52.098882+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0092173299999999993,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":131344,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"10.13","version":"532'6075","reported_seq":10837,"reported_epoch":532,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T21:15:03.318364+0000","last_change":"2023-09-23T21:13:49.840403+0000","last_active":"2023-09-23T21:15:03.318364+0000","last_peered":"2023-09-23T21:15:03.318364+0000","last_clean":"2023-09-23T21:11:26.718194+0000","last_became_active":"2023-09-23T21:13:48.181112+0000","last_became_peered":"2023-09-23T21:13:48.181112+0000","last_unstale":"2023-09-23T21:15:03.318364+0000","last_undegraded":"2023-09-23T21:13:48.139049+0000","last_fullsized":"2023-09-23T21:13:45.405990+0000","mapping_epoch":513,"log_start":"435'1590","ondisk_log_start":"435'1590","created":438,"last_epoch_clean":439,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_clean_scrub_stamp":"2023-09-23T21:10:15.663179+0000","objects_scrubbed":0,"log_size":4485,"log_dups_size":0,"ondisk_log_size":4485,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":21397504,"num_objects":327,"num_object_clones":0,"num_object_copies":981,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":200,"num_objects_misplaced":1125,"num_objects_unfound":0,"num_objects_dirty":327,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4707,"num_write_kb":161056,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2190,"num_bytes_recovered":143458304,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[4,3],"avail_no_missing":["4","5"],"object_location_counts":[{"shards":"0,2,4,5","objects":265},{"shards":"2,4,5","objects":28},{"shards":"3,4","objects":34}],"blocked_by":[],"up_primary":2,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1a","version":"68'1","reported_seq":122,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-09-23T21:14:51.806780+0000","last_change":"2023-09-23T21:13:13.291403+0000","last_active":"2023-09-23T21:14:51.806780+0000","last_peered":"2023-09-23T21:14:51.806780+0000","last_clean":"2023-09-23T21:14:51.806780+0000","last_became_active":"2023-09-23T21:13:13.291123+0000","last_became_peered":"2023-09-23T21:13:13.291123+0000","last_unstale":"2023-09-23T21:14:51.806780+0000","last_undegraded":"2023-09-23T21:14:51.806780+0000","last_fullsized":"2023-09-23T21:14:51.806780+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":495,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:45:15.427430+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,0],"acting":[4,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":191,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-09-23T21:14:51.808276+0000","last_change":"2023-09-23T21:12:10.767963+0000","last_active":"2023-09-23T21:14:51.808276+0000","last_peered":"2023-09-23T21:14:51.808276+0000","last_clean":"2023-09-23T21:14:51.808276+0000","last_became_active":"2023-09-23T21:12:10.767126+0000","last_became_peered":"2023-09-23T21:12:10.767126+0000","last_unstale":"2023-09-23T21:14:51.808276+0000","last_undegraded":"2023-09-23T21:14:51.808276+0000","last_fullsized":"2023-09-23T21:14:51.808276+0000","mapping_epoch":470,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:45:47.138277+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,5],"acting":[4,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1b","version":"79'161","reported_seq":535,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-09-23T21:14:51.815565+0000","last_change":"2023-09-23T21:13:13.373548+0000","last_active":"2023-09-23T21:14:51.815565+0000","last_peered":"2023-09-23T21:14:51.815565+0000","last_clean":"2023-09-23T21:14:51.815565+0000","last_became_active":"2023-09-23T21:13:13.372845+0000","last_became_peered":"2023-09-23T21:13:13.372845+0000","last_unstale":"2023-09-23T21:14:51.815565+0000","last_undegraded":"2023-09-23T21:14:51.815565+0000","last_fullsized":"2023-09-23T21:14:51.815565+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":479,"last_epoch_clean":495,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:05:55.772264+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:05:55.772264+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.001862507,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.10","version":"532'6045","reported_seq":14780,"reported_epoch":532,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T21:15:03.271718+0000","last_change":"2023-09-23T21:13:36.071975+0000","last_active":"2023-09-23T21:15:03.271718+0000","last_peered":"2023-09-23T21:15:03.271718+0000","last_clean":"2023-09-23T21:11:26.716051+0000","last_became_active":"2023-09-23T21:13:33.619010+0000","last_became_peered":"2023-09-23T21:13:33.619010+0000","last_unstale":"2023-09-23T21:15:03.271718+0000","last_undegraded":"2023-09-23T21:13:33.443511+0000","last_fullsized":"2023-09-23T21:13:31.753181+0000","mapping_epoch":502,"log_start":"435'1538","ondisk_log_start":"435'1538","created":438,"last_epoch_clean":450,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_clean_scrub_stamp":"2023-09-23T21:10:15.663179+0000","objects_scrubbed":0,"log_size":4507,"log_dups_size":0,"ondisk_log_size":4507,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":18776064,"num_objects":287,"num_object_clones":0,"num_object_copies":861,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":826,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":287,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4681,"num_write_kb":158944,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4023,"num_bytes_recovered":263651328,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7],"avail_no_missing":["4","1","5","7"],"object_location_counts":[{"shards":"4,5,7","objects":29},{"shards":"4,7","objects":258}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":528,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-09-23T21:14:51.812574+0000","last_change":"2023-09-23T21:12:10.824116+0000","last_active":"2023-09-23T21:14:51.812574+0000","last_peered":"2023-09-23T21:14:51.812574+0000","last_clean":"2023-09-23T21:14:51.812574+0000","last_became_active":"2023-09-23T21:12:10.823849+0000","last_became_peered":"2023-09-23T21:12:10.823849+0000","last_unstale":"2023-09-23T21:14:51.812574+0000","last_undegraded":"2023-09-23T21:14:51.812574+0000","last_fullsized":"2023-09-23T21:14:51.812574+0000","mapping_epoch":470,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:54:13.777603+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":528,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-09-23T21:14:51.817174+0000","last_change":"2023-09-23T21:13:44.413713+0000","last_active":"2023-09-23T21:14:51.817174+0000","last_peered":"2023-09-23T21:14:51.817174+0000","last_clean":"2023-09-23T21:14:51.817174+0000","last_became_active":"2023-09-23T21:13:44.413366+0000","last_became_peered":"2023-09-23T21:13:44.413366+0000","last_unstale":"2023-09-23T21:14:51.817174+0000","last_undegraded":"2023-09-23T21:14:51.817174+0000","last_fullsized":"2023-09-23T21:14:51.817174+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:31:56.349530+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1a","version":"79'161","reported_seq":678,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-09-23T21:15:01.313006+0000","last_change":"2023-09-23T21:12:30.444665+0000","last_active":"2023-09-23T21:15:01.313006+0000","last_peered":"2023-09-23T21:15:01.313006+0000","last_clean":"2023-09-23T21:15:01.313006+0000","last_became_active":"2023-09-23T21:12:30.444326+0000","last_became_peered":"2023-09-23T21:12:30.444326+0000","last_unstale":"2023-09-23T21:15:01.313006+0000","last_undegraded":"2023-09-23T21:15:01.313006+0000","last_fullsized":"2023-09-23T21:15:01.313006+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":479,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:06:51.075448+0000","last_deep_scrub":"79'161","last_deep_scrub_stamp":"2023-09-23T21:05:32.630292+0000","last_clean_scrub_stamp":"2023-09-23T21:06:51.075448+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0054793890000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,4],"acting":[2,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"10.11","version":"532'6074","reported_seq":12908,"reported_epoch":532,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-23T21:15:06.180726+0000","last_change":"2023-09-23T21:13:31.864763+0000","last_active":"2023-09-23T21:15:06.180726+0000","last_peered":"2023-09-23T21:15:06.180726+0000","last_clean":"2023-09-23T21:13:30.058478+0000","last_became_active":"2023-09-23T21:13:31.850860+0000","last_became_peered":"2023-09-23T21:13:31.850860+0000","last_unstale":"2023-09-23T21:15:06.180726+0000","last_undegraded":"2023-09-23T21:15:06.180726+0000","last_fullsized":"2023-09-23T21:15:06.180726+0000","mapping_epoch":502,"log_start":"433'807","ondisk_log_start":"433'807","created":438,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":5,"last_scrub":"441'2974","last_scrub_stamp":"2023-09-23T21:11:06.133218+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_clean_scrub_stamp":"2023-09-23T21:11:06.133218+0000","objects_scrubbed":1646,"log_size":5267,"log_dups_size":0,"ondisk_log_size":5267,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:54:59.366711+0000","scrub_duration":0.3173955,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":12910592,"num_objects":197,"num_object_clones":0,"num_object_copies":591,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":197,"num_objects_unfound":0,"num_objects_dirty":197,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3103,"num_write_kb":52928,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2152,"num_bytes_recovered":141033472,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[2,5,7],"avail_no_missing":["2","1","5","7"],"object_location_counts":[{"shards":"2,5,7","objects":197}],"blocked_by":[],"up_primary":0,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":41,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-09-23T21:14:51.817516+0000","last_change":"2023-09-23T21:13:30.806084+0000","last_active":"2023-09-23T21:14:51.817516+0000","last_peered":"2023-09-23T21:14:51.817516+0000","last_clean":"2023-09-23T21:14:51.817516+0000","last_became_active":"2023-09-23T21:13:30.805743+0000","last_became_peered":"2023-09-23T21:13:30.805743+0000","last_unstale":"2023-09-23T21:14:51.817516+0000","last_undegraded":"2023-09-23T21:14:51.817516+0000","last_fullsized":"2023-09-23T21:14:51.817516+0000","mapping_epoch":501,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:48:10.849281+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":405,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-09-23T21:14:51.816912+0000","last_change":"2023-09-23T21:13:44.410921+0000","last_active":"2023-09-23T21:14:51.816912+0000","last_peered":"2023-09-23T21:14:51.816912+0000","last_clean":"2023-09-23T21:14:51.816912+0000","last_became_active":"2023-09-23T21:13:44.410633+0000","last_became_peered":"2023-09-23T21:13:44.410633+0000","last_unstale":"2023-09-23T21:14:51.816912+0000","last_undegraded":"2023-09-23T21:14:51.816912+0000","last_fullsized":"2023-09-23T21:14:51.816912+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:57:10.116786+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.15","version":"79'161","reported_seq":558,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-09-23T21:14:51.807421+0000","last_change":"2023-09-23T21:13:38.379453+0000","last_active":"2023-09-23T21:14:51.807421+0000","last_peered":"2023-09-23T21:14:51.807421+0000","last_clean":"2023-09-23T21:14:51.807421+0000","last_became_active":"2023-09-23T21:13:38.376499+0000","last_became_peered":"2023-09-23T21:13:38.376499+0000","last_unstale":"2023-09-23T21:14:51.807421+0000","last_undegraded":"2023-09-23T21:14:51.807421+0000","last_fullsized":"2023-09-23T21:14:51.807421+0000","mapping_epoch":507,"log_start":"0'0","ondisk_log_start":"0'0","created":479,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:09:34.892947+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:09:34.892947+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13418,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":11,"num_read_kb":21,"num_write":6,"num_write_kb":55,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":40253,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":506,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-09-23T21:14:51.817059+0000","last_change":"2023-09-23T21:13:12.235728+0000","last_active":"2023-09-23T21:14:51.817059+0000","last_peered":"2023-09-23T21:14:51.817059+0000","last_clean":"2023-09-23T21:14:51.817059+0000","last_became_active":"2023-09-23T21:13:12.216549+0000","last_became_peered":"2023-09-23T21:13:12.216549+0000","last_unstale":"2023-09-23T21:14:51.817059+0000","last_undegraded":"2023-09-23T21:14:51.817059+0000","last_fullsized":"2023-09-23T21:14:51.817059+0000","mapping_epoch":493,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:39:33.324239+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":570,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-09-23T21:14:51.812264+0000","last_change":"2023-09-23T21:12:10.790349+0000","last_active":"2023-09-23T21:14:51.812264+0000","last_peered":"2023-09-23T21:14:51.812264+0000","last_clean":"2023-09-23T21:14:51.812264+0000","last_became_active":"2023-09-23T21:12:10.789977+0000","last_became_peered":"2023-09-23T21:12:10.789977+0000","last_unstale":"2023-09-23T21:14:51.812264+0000","last_undegraded":"2023-09-23T21:14:51.812264+0000","last_fullsized":"2023-09-23T21:14:51.812264+0000","mapping_epoch":470,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:49:40.191147+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,5],"acting":[3,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"79'161","reported_seq":893,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-09-23T21:14:51.826606+0000","last_change":"2023-09-23T21:11:32.760292+0000","last_active":"2023-09-23T21:14:51.826606+0000","last_peered":"2023-09-23T21:14:51.826606+0000","last_clean":"2023-09-23T21:14:51.826606+0000","last_became_active":"2023-09-23T21:11:32.759909+0000","last_became_peered":"2023-09-23T21:11:32.759909+0000","last_unstale":"2023-09-23T21:14:51.826606+0000","last_undegraded":"2023-09-23T21:14:51.826606+0000","last_fullsized":"2023-09-23T21:14:51.826606+0000","mapping_epoch":457,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":458,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:25.096401+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:25.096401+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:13:09.225685+0000","scrub_duration":0.002859393,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":257,"reported_epoch":532,"state":"active+clean","last_fresh":"2023-09-23T21:15:02.202206+0000","last_change":"2023-09-23T21:12:10.779468+0000","last_active":"2023-09-23T21:15:02.202206+0000","last_peered":"2023-09-23T21:15:02.202206+0000","last_clean":"2023-09-23T21:15:02.202206+0000","last_became_active":"2023-09-23T21:12:10.779282+0000","last_became_peered":"2023-09-23T21:12:10.779282+0000","last_unstale":"2023-09-23T21:15:02.202206+0000","last_undegraded":"2023-09-23T21:15:02.202206+0000","last_fullsized":"2023-09-23T21:15:02.202206+0000","mapping_epoch":470,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:36:35.417986+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":507,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-09-23T21:14:51.812495+0000","last_change":"2023-09-23T21:10:26.758958+0000","last_active":"2023-09-23T21:14:51.812495+0000","last_peered":"2023-09-23T21:14:51.812495+0000","last_clean":"2023-09-23T21:14:51.812495+0000","last_became_active":"2023-09-23T21:10:26.758539+0000","last_became_peered":"2023-09-23T21:10:26.758539+0000","last_unstale":"2023-09-23T21:14:51.812495+0000","last_undegraded":"2023-09-23T21:14:51.812495+0000","last_fullsized":"2023-09-23T21:14:51.812495+0000","mapping_epoch":423,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":424,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:44:42.319189+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.17","version":"79'161","reported_seq":624,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-09-23T21:14:51.817253+0000","last_change":"2023-09-23T21:13:44.416768+0000","last_active":"2023-09-23T21:14:51.817253+0000","last_peered":"2023-09-23T21:14:51.817253+0000","last_clean":"2023-09-23T21:14:51.817253+0000","last_became_active":"2023-09-23T21:13:44.416156+0000","last_became_peered":"2023-09-23T21:13:44.416156+0000","last_unstale":"2023-09-23T21:14:51.817253+0000","last_undegraded":"2023-09-23T21:14:51.817253+0000","last_fullsized":"2023-09-23T21:14:51.817253+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":479,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:25.916867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:25.916867+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.001665323,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":416,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-09-23T21:14:51.813820+0000","last_change":"2023-09-23T21:11:08.766045+0000","last_active":"2023-09-23T21:14:51.813820+0000","last_peered":"2023-09-23T21:14:51.813820+0000","last_clean":"2023-09-23T21:14:51.813820+0000","last_became_active":"2023-09-23T21:11:08.758504+0000","last_became_peered":"2023-09-23T21:11:08.758504+0000","last_unstale":"2023-09-23T21:14:51.813820+0000","last_undegraded":"2023-09-23T21:14:51.813820+0000","last_fullsized":"2023-09-23T21:14:51.813820+0000","mapping_epoch":443,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":444,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:02:19.482574+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":360,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,4],"acting":[3,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":219,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-09-23T21:14:51.832333+0000","last_change":"2023-09-23T21:13:30.821489+0000","last_active":"2023-09-23T21:14:51.832333+0000","last_peered":"2023-09-23T21:14:51.832333+0000","last_clean":"2023-09-23T21:14:51.832333+0000","last_became_active":"2023-09-23T21:13:30.821125+0000","last_became_peered":"2023-09-23T21:13:30.821125+0000","last_unstale":"2023-09-23T21:14:51.832333+0000","last_undegraded":"2023-09-23T21:14:51.832333+0000","last_fullsized":"2023-09-23T21:14:51.832333+0000","mapping_epoch":501,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:47:18.553417+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,5],"acting":[2,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.16","version":"79'161","reported_seq":450,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-09-23T21:14:51.817082+0000","last_change":"2023-09-23T21:13:33.770962+0000","last_active":"2023-09-23T21:14:51.817082+0000","last_peered":"2023-09-23T21:14:51.817082+0000","last_clean":"2023-09-23T21:14:51.817082+0000","last_became_active":"2023-09-23T21:13:33.770407+0000","last_became_peered":"2023-09-23T21:13:33.770407+0000","last_unstale":"2023-09-23T21:14:51.817082+0000","last_undegraded":"2023-09-23T21:14:51.817082+0000","last_fullsized":"2023-09-23T21:14:51.817082+0000","mapping_epoch":504,"log_start":"0'0","ondisk_log_start":"0'0","created":479,"last_epoch_clean":505,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:10:24.268662+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:10:24.268662+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13418,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":11,"num_read_kb":21,"num_write":6,"num_write_kb":55,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":40252,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.14","version":"79'22","reported_seq":95,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-09-23T21:14:51.805574+0000","last_change":"2023-09-23T21:11:27.770162+0000","last_active":"2023-09-23T21:14:51.805574+0000","last_peered":"2023-09-23T21:14:51.805574+0000","last_clean":"2023-09-23T21:14:51.805574+0000","last_became_active":"2023-09-23T21:11:27.752275+0000","last_became_peered":"2023-09-23T21:11:27.752275+0000","last_unstale":"2023-09-23T21:14:51.805574+0000","last_undegraded":"2023-09-23T21:14:51.805574+0000","last_fullsized":"2023-09-23T21:14:51.805574+0000","mapping_epoch":452,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":453,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:05:11.769428+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4734,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,2],"acting":[4,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":460,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-09-23T21:14:51.810300+0000","last_change":"2023-09-23T21:12:10.839271+0000","last_active":"2023-09-23T21:14:51.810300+0000","last_peered":"2023-09-23T21:14:51.810300+0000","last_clean":"2023-09-23T21:14:51.810300+0000","last_became_active":"2023-09-23T21:12:10.839042+0000","last_became_peered":"2023-09-23T21:12:10.839042+0000","last_unstale":"2023-09-23T21:14:51.810300+0000","last_undegraded":"2023-09-23T21:14:51.810300+0000","last_fullsized":"2023-09-23T21:14:51.810300+0000","mapping_epoch":470,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:10:26.414737+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,2],"acting":[4,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.11","version":"79'161","reported_seq":455,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-09-23T21:14:51.826288+0000","last_change":"2023-09-23T21:13:33.767406+0000","last_active":"2023-09-23T21:14:51.826288+0000","last_peered":"2023-09-23T21:14:51.826288+0000","last_clean":"2023-09-23T21:14:51.826288+0000","last_became_active":"2023-09-23T21:13:33.767067+0000","last_became_peered":"2023-09-23T21:13:33.767067+0000","last_unstale":"2023-09-23T21:14:51.826288+0000","last_undegraded":"2023-09-23T21:14:51.826288+0000","last_fullsized":"2023-09-23T21:14:51.826288+0000","mapping_epoch":504,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":505,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:28.756634+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:28.756634+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":23,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":141,"reported_epoch":532,"state":"active+clean","last_fresh":"2023-09-23T21:15:02.188351+0000","last_change":"2023-09-23T21:11:27.777154+0000","last_active":"2023-09-23T21:15:02.188351+0000","last_peered":"2023-09-23T21:15:02.188351+0000","last_clean":"2023-09-23T21:15:02.188351+0000","last_became_active":"2023-09-23T21:11:27.754438+0000","last_became_peered":"2023-09-23T21:11:27.754438+0000","last_unstale":"2023-09-23T21:15:02.188351+0000","last_undegraded":"2023-09-23T21:15:02.188351+0000","last_fullsized":"2023-09-23T21:15:02.188351+0000","mapping_epoch":452,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":453,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:33:02.845882+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":73,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-09-23T21:14:51.826239+0000","last_change":"2023-09-23T21:12:10.764264+0000","last_active":"2023-09-23T21:14:51.826239+0000","last_peered":"2023-09-23T21:14:51.826239+0000","last_clean":"2023-09-23T21:14:51.826239+0000","last_became_active":"2023-09-23T21:12:10.763486+0000","last_became_peered":"2023-09-23T21:12:10.763486+0000","last_unstale":"2023-09-23T21:14:51.826239+0000","last_undegraded":"2023-09-23T21:14:51.826239+0000","last_fullsized":"2023-09-23T21:14:51.826239+0000","mapping_epoch":470,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:32:27.577036+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,7],"acting":[2,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.10","version":"79'161","reported_seq":666,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-09-23T21:14:51.817273+0000","last_change":"2023-09-23T21:11:11.716703+0000","last_active":"2023-09-23T21:14:51.817273+0000","last_peered":"2023-09-23T21:14:51.817273+0000","last_clean":"2023-09-23T21:14:51.817273+0000","last_became_active":"2023-09-23T21:11:11.716385+0000","last_became_peered":"2023-09-23T21:11:11.716385+0000","last_unstale":"2023-09-23T21:14:51.817273+0000","last_undegraded":"2023-09-23T21:14:51.817273+0000","last_fullsized":"2023-09-23T21:14:51.817273+0000","mapping_epoch":446,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":447,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:06:50.336992+0000","last_deep_scrub":"79'161","last_deep_scrub_stamp":"2023-09-23T21:06:50.336992+0000","last_clean_scrub_stamp":"2023-09-23T21:06:50.336992+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:44:29.538890+0000","scrub_duration":0.0026150930000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,7],"acting":[4,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":472,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-09-23T21:14:51.833624+0000","last_change":"2023-09-23T21:13:12.270990+0000","last_active":"2023-09-23T21:14:51.833624+0000","last_peered":"2023-09-23T21:14:51.833624+0000","last_clean":"2023-09-23T21:14:51.833624+0000","last_became_active":"2023-09-23T21:13:12.237201+0000","last_became_peered":"2023-09-23T21:13:12.237201+0000","last_unstale":"2023-09-23T21:14:51.833624+0000","last_undegraded":"2023-09-23T21:14:51.833624+0000","last_fullsized":"2023-09-23T21:14:51.833624+0000","mapping_epoch":493,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:59:54.523877+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":284,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-09-23T21:14:51.813695+0000","last_change":"2023-09-23T21:10:26.758671+0000","last_active":"2023-09-23T21:14:51.813695+0000","last_peered":"2023-09-23T21:14:51.813695+0000","last_clean":"2023-09-23T21:14:51.813695+0000","last_became_active":"2023-09-23T21:10:26.758244+0000","last_became_peered":"2023-09-23T21:10:26.758244+0000","last_unstale":"2023-09-23T21:14:51.813695+0000","last_undegraded":"2023-09-23T21:14:51.813695+0000","last_fullsized":"2023-09-23T21:14:51.813695+0000","mapping_epoch":423,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":424,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:14:02.735609+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.18","version":"532'6180","reported_seq":7269,"reported_epoch":532,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T21:15:06.291794+0000","last_change":"2023-09-23T21:13:13.412097+0000","last_active":"2023-09-23T21:15:06.291794+0000","last_peered":"2023-09-23T21:15:06.291794+0000","last_clean":"2023-09-23T21:11:45.720015+0000","last_became_active":"2023-09-23T21:13:13.395632+0000","last_became_peered":"2023-09-23T21:13:13.395632+0000","last_unstale":"2023-09-23T21:15:06.291794+0000","last_undegraded":"2023-09-23T21:13:13.220971+0000","last_fullsized":"2023-09-23T21:13:13.183825+0000","mapping_epoch":494,"log_start":"434'915","ondisk_log_start":"434'915","created":438,"last_epoch_clean":439,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_clean_scrub_stamp":"2023-09-23T21:10:15.663179+0000","objects_scrubbed":0,"log_size":5265,"log_dups_size":0,"ondisk_log_size":5265,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13664256,"num_objects":209,"num_object_clones":0,"num_object_copies":627,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":661,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":209,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4831,"num_write_kb":161248,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":231,"num_bytes_recovered":15138816,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,7],"acting":[2,7],"avail_no_missing":["2","7"],"object_location_counts":[{"shards":"2,7","objects":175}],"blocked_by":[],"up_primary":0,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.13","version":"79'161","reported_seq":609,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-09-23T21:14:51.825029+0000","last_change":"2023-09-23T21:13:14.933096+0000","last_active":"2023-09-23T21:14:51.825029+0000","last_peered":"2023-09-23T21:14:51.825029+0000","last_clean":"2023-09-23T21:14:51.825029+0000","last_became_active":"2023-09-23T21:13:14.932703+0000","last_became_peered":"2023-09-23T21:13:14.932703+0000","last_unstale":"2023-09-23T21:14:51.825029+0000","last_undegraded":"2023-09-23T21:14:51.825029+0000","last_fullsized":"2023-09-23T21:14:51.825029+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:36.420362+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:36.420362+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:29:42.163911+0000","scrub_duration":0.028838444000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":523,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-09-23T21:14:51.804526+0000","last_change":"2023-09-23T21:13:12.216467+0000","last_active":"2023-09-23T21:14:51.804526+0000","last_peered":"2023-09-23T21:14:51.804526+0000","last_clean":"2023-09-23T21:14:51.804526+0000","last_became_active":"2023-09-23T21:13:12.215259+0000","last_became_peered":"2023-09-23T21:13:12.215259+0000","last_unstale":"2023-09-23T21:14:51.804526+0000","last_undegraded":"2023-09-23T21:14:51.804526+0000","last_fullsized":"2023-09-23T21:14:51.804526+0000","mapping_epoch":493,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:35:33.987839+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":41,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-09-23T21:14:51.816805+0000","last_change":"2023-09-23T21:13:30.804108+0000","last_active":"2023-09-23T21:14:51.816805+0000","last_peered":"2023-09-23T21:14:51.816805+0000","last_clean":"2023-09-23T21:14:51.816805+0000","last_became_active":"2023-09-23T21:13:30.796505+0000","last_became_peered":"2023-09-23T21:13:30.796505+0000","last_unstale":"2023-09-23T21:14:51.816805+0000","last_undegraded":"2023-09-23T21:14:51.816805+0000","last_fullsized":"2023-09-23T21:14:51.816805+0000","mapping_epoch":501,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:19:33.508496+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.19","version":"532'6291","reported_seq":7394,"reported_epoch":532,"state":"active+clean","last_fresh":"2023-09-23T21:15:06.200522+0000","last_change":"2023-09-23T21:10:58.790716+0000","last_active":"2023-09-23T21:15:06.200522+0000","last_peered":"2023-09-23T21:15:06.200522+0000","last_clean":"2023-09-23T21:15:06.200522+0000","last_became_active":"2023-09-23T21:10:58.787975+0000","last_became_peered":"2023-09-23T21:10:58.787975+0000","last_unstale":"2023-09-23T21:15:06.200522+0000","last_undegraded":"2023-09-23T21:15:06.200522+0000","last_fullsized":"2023-09-23T21:15:06.200522+0000","mapping_epoch":438,"log_start":"434'1167","ondisk_log_start":"434'1167","created":438,"last_epoch_clean":439,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_clean_scrub_stamp":"2023-09-23T21:10:15.663179+0000","objects_scrubbed":0,"log_size":5124,"log_dups_size":0,"ondisk_log_size":5124,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":12288000,"num_objects":188,"num_object_clones":0,"num_object_copies":564,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":188,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4904,"num_write_kb":162912,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":237,"num_bytes_recovered":15499264,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.12","version":"79'161","reported_seq":766,"reported_epoch":532,"state":"active+clean","last_fresh":"2023-09-23T21:15:02.202336+0000","last_change":"2023-09-23T21:13:15.206574+0000","last_active":"2023-09-23T21:15:02.202336+0000","last_peered":"2023-09-23T21:15:02.202336+0000","last_clean":"2023-09-23T21:15:02.202336+0000","last_became_active":"2023-09-23T21:13:15.206186+0000","last_became_peered":"2023-09-23T21:13:15.206186+0000","last_unstale":"2023-09-23T21:15:02.202336+0000","last_undegraded":"2023-09-23T21:15:02.202336+0000","last_fullsized":"2023-09-23T21:15:02.202336+0000","mapping_epoch":496,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":497,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:05:56.785173+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:05:56.785173+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:34:29.221809+0000","scrub_duration":0.0042114370000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,0],"acting":[5,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":51,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-09-23T21:14:51.817663+0000","last_change":"2023-09-23T21:13:38.048791+0000","last_active":"2023-09-23T21:14:51.817663+0000","last_peered":"2023-09-23T21:14:51.817663+0000","last_clean":"2023-09-23T21:14:51.817663+0000","last_became_active":"2023-09-23T21:13:30.801386+0000","last_became_peered":"2023-09-23T21:13:30.801386+0000","last_unstale":"2023-09-23T21:14:51.817663+0000","last_undegraded":"2023-09-23T21:14:51.817663+0000","last_fullsized":"2023-09-23T21:14:51.817663+0000","mapping_epoch":501,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:52:49.170414+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":369,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-09-23T21:14:51.818965+0000","last_change":"2023-09-23T21:12:10.765780+0000","last_active":"2023-09-23T21:14:51.818965+0000","last_peered":"2023-09-23T21:14:51.818965+0000","last_clean":"2023-09-23T21:14:51.818965+0000","last_became_active":"2023-09-23T21:12:10.765372+0000","last_became_peered":"2023-09-23T21:12:10.765372+0000","last_unstale":"2023-09-23T21:14:51.818965+0000","last_undegraded":"2023-09-23T21:14:51.818965+0000","last_fullsized":"2023-09-23T21:14:51.818965+0000","mapping_epoch":470,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:56:08.446099+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.6","version":"532'6258","reported_seq":9179,"reported_epoch":532,"state":"active+clean","last_fresh":"2023-09-23T21:15:05.142608+0000","last_change":"2023-09-23T21:13:39.390285+0000","last_active":"2023-09-23T21:15:05.142608+0000","last_peered":"2023-09-23T21:15:05.142608+0000","last_clean":"2023-09-23T21:15:05.142608+0000","last_became_active":"2023-09-23T21:13:39.378284+0000","last_became_peered":"2023-09-23T21:13:39.378284+0000","last_unstale":"2023-09-23T21:15:05.142608+0000","last_undegraded":"2023-09-23T21:15:05.142608+0000","last_fullsized":"2023-09-23T21:15:05.142608+0000","mapping_epoch":508,"log_start":"435'1717","ondisk_log_start":"435'1717","created":416,"last_epoch_clean":509,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_clean_scrub_stamp":"2023-09-23T21:10:15.663179+0000","objects_scrubbed":0,"log_size":4541,"log_dups_size":0,"ondisk_log_size":4541,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":14024704,"num_objects":214,"num_object_clones":0,"num_object_copies":642,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":214,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4852,"num_write_kb":162112,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2399,"num_bytes_recovered":113967104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,4],"acting":[0,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.d","version":"79'161","reported_seq":686,"reported_epoch":532,"state":"active+clean","last_fresh":"2023-09-23T21:15:02.189417+0000","last_change":"2023-09-23T21:12:30.444461+0000","last_active":"2023-09-23T21:15:02.189417+0000","last_peered":"2023-09-23T21:15:02.189417+0000","last_clean":"2023-09-23T21:15:02.189417+0000","last_became_active":"2023-09-23T21:12:30.443979+0000","last_became_peered":"2023-09-23T21:12:30.443979+0000","last_unstale":"2023-09-23T21:15:02.189417+0000","last_undegraded":"2023-09-23T21:15:02.189417+0000","last_fullsized":"2023-09-23T21:15:02.189417+0000","mapping_epoch":470,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:09:36.858077+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:09:36.858077+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0024987500000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":179,"reported_epoch":532,"state":"active+clean","last_fresh":"2023-09-23T21:15:02.189969+0000","last_change":"2023-09-23T21:13:59.872834+0000","last_active":"2023-09-23T21:15:02.189969+0000","last_peered":"2023-09-23T21:15:02.189969+0000","last_clean":"2023-09-23T21:15:02.189969+0000","last_became_active":"2023-09-23T21:13:30.770556+0000","last_became_peered":"2023-09-23T21:13:30.770556+0000","last_unstale":"2023-09-23T21:15:02.189969+0000","last_undegraded":"2023-09-23T21:15:02.189969+0000","last_fullsized":"2023-09-23T21:15:02.189969+0000","mapping_epoch":501,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":521,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:57:55.460052+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":132,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":240,"reported_epoch":532,"state":"active+clean","last_fresh":"2023-09-23T21:15:02.201915+0000","last_change":"2023-09-23T21:10:26.750725+0000","last_active":"2023-09-23T21:15:02.201915+0000","last_peered":"2023-09-23T21:15:02.201915+0000","last_clean":"2023-09-23T21:15:02.201915+0000","last_became_active":"2023-09-23T21:10:26.750307+0000","last_became_peered":"2023-09-23T21:10:26.750307+0000","last_unstale":"2023-09-23T21:15:02.201915+0000","last_undegraded":"2023-09-23T21:15:02.201915+0000","last_fullsized":"2023-09-23T21:15:02.201915+0000","mapping_epoch":423,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":424,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:06:22.040548+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,4],"acting":[5,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.7","version":"532'6092","reported_seq":6291,"reported_epoch":532,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T21:15:04.183175+0000","last_change":"2023-09-23T21:13:38.417178+0000","last_active":"2023-09-23T21:15:04.183175+0000","last_peered":"2023-09-23T21:15:04.183175+0000","last_clean":"2023-09-23T21:11:26.699871+0000","last_became_active":"2023-09-23T21:13:38.408523+0000","last_became_peered":"2023-09-23T21:13:38.408523+0000","last_unstale":"2023-09-23T21:15:04.183175+0000","last_undegraded":"2023-09-23T21:13:38.369833+0000","last_fullsized":"2023-09-23T21:13:38.358198+0000","mapping_epoch":507,"log_start":"435'2119","ondisk_log_start":"435'2119","created":416,"last_epoch_clean":439,"parent":"0.0","parent_split_bits":0,"last_scrub":"441'2973","last_scrub_stamp":"2023-09-23T21:11:06.079326+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_clean_scrub_stamp":"2023-09-23T21:11:06.079326+0000","objects_scrubbed":1647,"log_size":3973,"log_dups_size":0,"ondisk_log_size":3973,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:27:59.805526+0000","scrub_duration":0.44319714500000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":16449536,"num_objects":251,"num_object_clones":0,"num_object_copies":753,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1165,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":251,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3124,"num_write_kb":55296,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,2],"acting":[3,5],"avail_no_missing":["3","5"],"object_location_counts":[{"shards":"3,4,5","objects":251},{"shards":"3,5","objects":-47}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.c","version":"79'161","reported_seq":894,"reported_epoch":531,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-23T21:14:51.815056+0000","last_change":"2023-09-23T21:13:13.402530+0000","last_active":"2023-09-23T21:14:51.815056+0000","last_peered":"2023-09-23T21:14:51.815056+0000","last_clean":"2023-09-23T21:11:23.200766+0000","last_became_active":"2023-09-23T21:13:13.401526+0000","last_became_peered":"2023-09-23T21:13:13.401526+0000","last_unstale":"2023-09-23T21:14:51.815056+0000","last_undegraded":"2023-09-23T21:13:13.204992+0000","last_fullsized":"2023-09-23T21:14:51.815056+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":420,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:22.040907+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:22.040907+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.013021176000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":163840,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":163840,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":["3","0","5"],"object_location_counts":[{"shards":"0,3,4","objects":-1},{"shards":"0,3,5","objects":1}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":165,"reported_epoch":532,"state":"active+clean","last_fresh":"2023-09-23T21:15:02.189724+0000","last_change":"2023-09-23T21:13:59.884113+0000","last_active":"2023-09-23T21:15:02.189724+0000","last_peered":"2023-09-23T21:15:02.189724+0000","last_clean":"2023-09-23T21:15:02.189724+0000","last_became_active":"2023-09-23T21:13:30.806778+0000","last_became_peered":"2023-09-23T21:13:30.806778+0000","last_unstale":"2023-09-23T21:15:02.189724+0000","last_undegraded":"2023-09-23T21:15:02.189724+0000","last_fullsized":"2023-09-23T21:15:02.189724+0000","mapping_epoch":501,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":521,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:36:15.284465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":494,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-09-23T21:14:51.804591+0000","last_change":"2023-09-23T21:13:15.152792+0000","last_active":"2023-09-23T21:14:51.804591+0000","last_peered":"2023-09-23T21:14:51.804591+0000","last_clean":"2023-09-23T21:14:51.804591+0000","last_became_active":"2023-09-23T21:13:14.515084+0000","last_became_peered":"2023-09-23T21:13:14.515084+0000","last_unstale":"2023-09-23T21:14:51.804591+0000","last_undegraded":"2023-09-23T21:14:51.804591+0000","last_fullsized":"2023-09-23T21:14:51.804591+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:08:16.702072+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.4","version":"532'6024","reported_seq":12973,"reported_epoch":532,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T21:15:04.141093+0000","last_change":"2023-09-23T21:13:30.847170+0000","last_active":"2023-09-23T21:15:04.141093+0000","last_peered":"2023-09-23T21:15:04.141093+0000","last_clean":"2023-09-23T21:11:06.743842+0000","last_became_active":"2023-09-23T21:13:30.834806+0000","last_became_peered":"2023-09-23T21:13:30.834806+0000","last_unstale":"2023-09-23T21:15:04.141093+0000","last_undegraded":"2023-09-23T21:13:30.760230+0000","last_fullsized":"2023-09-23T21:13:30.749974+0000","mapping_epoch":501,"log_start":"435'2075","ondisk_log_start":"435'2075","created":416,"last_epoch_clean":439,"parent":"0.0","parent_split_bits":0,"last_scrub":"440'2861","last_scrub_stamp":"2023-09-23T21:11:03.842926+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_clean_scrub_stamp":"2023-09-23T21:11:03.842926+0000","objects_scrubbed":1526,"log_size":3949,"log_dups_size":0,"ondisk_log_size":3949,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:32:29.379756+0000","scrub_duration":0.39378383900000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19464192,"num_objects":297,"num_object_clones":0,"num_object_copies":891,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1130,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":297,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3169,"num_write_kb":62080,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2723,"num_bytes_recovered":163577856,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[3,4],"avail_no_missing":["3","4"],"object_location_counts":[{"shards":"1,2,3,4","objects":297},{"shards":"3,4","objects":-63}],"blocked_by":[],"up_primary":0,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.f","version":"79'161","reported_seq":820,"reported_epoch":532,"state":"active+clean","last_fresh":"2023-09-23T21:15:02.203144+0000","last_change":"2023-09-23T21:11:30.750338+0000","last_active":"2023-09-23T21:15:02.203144+0000","last_peered":"2023-09-23T21:15:02.203144+0000","last_clean":"2023-09-23T21:15:02.203144+0000","last_became_active":"2023-09-23T21:11:30.749953+0000","last_became_peered":"2023-09-23T21:11:30.749953+0000","last_unstale":"2023-09-23T21:15:02.203144+0000","last_undegraded":"2023-09-23T21:15:02.203144+0000","last_fullsized":"2023-09-23T21:15:02.203144+0000","mapping_epoch":455,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":456,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:27.880776+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:27.880776+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:07:10.405723+0000","scrub_duration":0.0014004899999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,3],"acting":[5,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":256,"reported_epoch":532,"state":"active+clean","last_fresh":"2023-09-23T21:15:02.202639+0000","last_change":"2023-09-23T21:10:26.765068+0000","last_active":"2023-09-23T21:15:02.202639+0000","last_peered":"2023-09-23T21:15:02.202639+0000","last_clean":"2023-09-23T21:15:02.202639+0000","last_became_active":"2023-09-23T21:10:26.758364+0000","last_became_peered":"2023-09-23T21:10:26.758364+0000","last_unstale":"2023-09-23T21:15:02.202639+0000","last_undegraded":"2023-09-23T21:15:02.202639+0000","last_fullsized":"2023-09-23T21:15:02.202639+0000","mapping_epoch":423,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":424,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:13:21.057529+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":280,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":472,"reported_epoch":532,"state":"active+clean","last_fresh":"2023-09-23T21:15:02.202053+0000","last_change":"2023-09-23T21:13:30.770746+0000","last_active":"2023-09-23T21:15:02.202053+0000","last_peered":"2023-09-23T21:15:02.202053+0000","last_clean":"2023-09-23T21:15:02.202053+0000","last_became_active":"2023-09-23T21:13:30.770437+0000","last_became_peered":"2023-09-23T21:13:30.770437+0000","last_unstale":"2023-09-23T21:15:02.202053+0000","last_undegraded":"2023-09-23T21:15:02.202053+0000","last_fullsized":"2023-09-23T21:15:02.202053+0000","mapping_epoch":501,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:33:48.325730+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,0],"acting":[5,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.5","version":"532'6126","reported_seq":6317,"reported_epoch":532,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T21:15:06.175994+0000","last_change":"2023-09-23T21:13:13.436794+0000","last_active":"2023-09-23T21:15:06.175994+0000","last_peered":"2023-09-23T21:15:06.175994+0000","last_clean":"2023-09-23T21:11:45.768408+0000","last_became_active":"2023-09-23T21:13:13.424939+0000","last_became_peered":"2023-09-23T21:13:13.424939+0000","last_unstale":"2023-09-23T21:15:06.175994+0000","last_undegraded":"2023-09-23T21:13:13.225067+0000","last_fullsized":"2023-09-23T21:13:13.198194+0000","mapping_epoch":494,"log_start":"434'914","ondisk_log_start":"434'914","created":416,"last_epoch_clean":439,"parent":"0.0","parent_split_bits":0,"last_scrub":"440'2840","last_scrub_stamp":"2023-09-23T21:11:03.208833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_clean_scrub_stamp":"2023-09-23T21:11:03.208833+0000","objects_scrubbed":1498,"log_size":5212,"log_dups_size":0,"ondisk_log_size":5212,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:56:40.574477+0000","scrub_duration":0.45608471699999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":15532032,"num_objects":237,"num_object_clones":0,"num_object_copies":711,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":644,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":237,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3295,"num_write_kb":65088,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,3],"acting":[2,3],"avail_no_missing":["2","3"],"object_location_counts":[{"shards":"2,3","objects":191}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.e","version":"79'161","reported_seq":700,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-09-23T21:14:51.805202+0000","last_change":"2023-09-23T21:12:30.440952+0000","last_active":"2023-09-23T21:14:51.805202+0000","last_peered":"2023-09-23T21:14:51.805202+0000","last_clean":"2023-09-23T21:14:51.805202+0000","last_became_active":"2023-09-23T21:12:30.440463+0000","last_became_peered":"2023-09-23T21:12:30.440463+0000","last_unstale":"2023-09-23T21:14:51.805202+0000","last_undegraded":"2023-09-23T21:14:51.805202+0000","last_fullsized":"2023-09-23T21:14:51.805202+0000","mapping_epoch":455,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:10:39.567232+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:10:39.567232+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.005229694,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":329,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-09-23T21:14:51.815517+0000","last_change":"2023-09-23T21:13:13.402418+0000","last_active":"2023-09-23T21:14:51.815517+0000","last_peered":"2023-09-23T21:14:51.815517+0000","last_clean":"2023-09-23T21:14:51.815517+0000","last_became_active":"2023-09-23T21:13:13.400984+0000","last_became_peered":"2023-09-23T21:13:13.400984+0000","last_unstale":"2023-09-23T21:14:51.815517+0000","last_undegraded":"2023-09-23T21:14:51.815517+0000","last_fullsized":"2023-09-23T21:14:51.815517+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":495,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:28:01.411191+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,0],"acting":[3,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":504,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-09-23T21:14:51.815985+0000","last_change":"2023-09-23T21:13:14.339670+0000","last_active":"2023-09-23T21:14:51.815985+0000","last_peered":"2023-09-23T21:14:51.815985+0000","last_clean":"2023-09-23T21:14:51.815985+0000","last_became_active":"2023-09-23T21:13:14.339341+0000","last_became_peered":"2023-09-23T21:13:14.339341+0000","last_unstale":"2023-09-23T21:14:51.815985+0000","last_undegraded":"2023-09-23T21:14:51.815985+0000","last_fullsized":"2023-09-23T21:14:51.815985+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:23:19.411368+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.2","version":"532'6092","reported_seq":9459,"reported_epoch":532,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T21:15:03.339997+0000","last_change":"2023-09-23T21:13:13.407366+0000","last_active":"2023-09-23T21:15:03.339997+0000","last_peered":"2023-09-23T21:15:03.339997+0000","last_clean":"2023-09-23T21:11:45.752822+0000","last_became_active":"2023-09-23T21:13:13.398616+0000","last_became_peered":"2023-09-23T21:13:13.398616+0000","last_unstale":"2023-09-23T21:15:03.339997+0000","last_undegraded":"2023-09-23T21:13:13.289815+0000","last_fullsized":"2023-09-23T21:13:13.190270+0000","mapping_epoch":494,"log_start":"435'1611","ondisk_log_start":"435'1611","created":416,"last_epoch_clean":439,"parent":"0.0","parent_split_bits":0,"last_scrub":"440'3007","last_scrub_stamp":"2023-09-23T21:11:05.553661+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_clean_scrub_stamp":"2023-09-23T21:11:05.553661+0000","objects_scrubbed":1622,"log_size":4481,"log_dups_size":0,"ondisk_log_size":4481,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:51:21.839818+0000","scrub_duration":0.39156952499999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19136512,"num_objects":292,"num_object_clones":0,"num_object_copies":876,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1584,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":292,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3090,"num_write_kb":56320,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1581,"num_bytes_recovered":103612416,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,3],"avail_no_missing":["4","1"],"object_location_counts":[{"shards":"0,1,4","objects":258},{"shards":"1,4","objects":34}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.9","version":"79'161","reported_seq":686,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-09-23T21:14:56.309902+0000","last_change":"2023-09-23T21:12:30.436118+0000","last_active":"2023-09-23T21:14:56.309902+0000","last_peered":"2023-09-23T21:14:56.309902+0000","last_clean":"2023-09-23T21:14:56.309902+0000","last_became_active":"2023-09-23T21:12:30.435779+0000","last_became_peered":"2023-09-23T21:12:30.435779+0000","last_unstale":"2023-09-23T21:14:56.309902+0000","last_undegraded":"2023-09-23T21:14:56.309902+0000","last_fullsized":"2023-09-23T21:14:56.309902+0000","mapping_epoch":433,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:10:37.696951+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:10:37.696951+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.001386504,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,3],"acting":[2,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":90,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-09-23T21:14:51.816119+0000","last_change":"2023-09-23T21:13:30.793273+0000","last_active":"2023-09-23T21:14:51.816119+0000","last_peered":"2023-09-23T21:14:51.816119+0000","last_clean":"2023-09-23T21:14:51.816119+0000","last_became_active":"2023-09-23T21:13:30.791269+0000","last_became_peered":"2023-09-23T21:13:30.791269+0000","last_unstale":"2023-09-23T21:14:51.816119+0000","last_undegraded":"2023-09-23T21:14:51.816119+0000","last_fullsized":"2023-09-23T21:14:51.816119+0000","mapping_epoch":501,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:38:07.142290+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":105,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-09-23T21:14:51.804975+0000","last_change":"2023-09-23T21:12:14.313466+0000","last_active":"2023-09-23T21:14:51.804975+0000","last_peered":"2023-09-23T21:14:51.804975+0000","last_clean":"2023-09-23T21:14:51.804975+0000","last_became_active":"2023-09-23T21:12:14.209837+0000","last_became_peered":"2023-09-23T21:12:14.209837+0000","last_unstale":"2023-09-23T21:14:51.804975+0000","last_undegraded":"2023-09-23T21:14:51.804975+0000","last_fullsized":"2023-09-23T21:14:51.804975+0000","mapping_epoch":470,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":474,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:09:40.375241+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,7],"acting":[4,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.3","version":"532'6219","reported_seq":8354,"reported_epoch":532,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T21:15:06.137319+0000","last_change":"2023-09-23T21:13:45.480705+0000","last_active":"2023-09-23T21:15:06.137319+0000","last_peered":"2023-09-23T21:15:06.137319+0000","last_clean":"2023-09-23T21:11:26.700990+0000","last_became_active":"2023-09-23T21:13:45.471702+0000","last_became_peered":"2023-09-23T21:13:45.471702+0000","last_unstale":"2023-09-23T21:15:06.137319+0000","last_undegraded":"2023-09-23T21:13:45.412179+0000","last_fullsized":"2023-09-23T21:13:45.401701+0000","mapping_epoch":513,"log_start":"434'1038","ondisk_log_start":"434'1038","created":416,"last_epoch_clean":439,"parent":"0.0","parent_split_bits":0,"last_scrub":"440'2873","last_scrub_stamp":"2023-09-23T21:11:02.114783+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_clean_scrub_stamp":"2023-09-23T21:11:02.114783+0000","objects_scrubbed":1532,"log_size":5181,"log_dups_size":0,"ondisk_log_size":5181,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:42:48.549370+0000","scrub_duration":0.32435022600000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":14155776,"num_objects":216,"num_object_clones":0,"num_object_copies":648,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1184,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":216,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3352,"num_write_kb":65152,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":865,"num_bytes_recovered":56688640,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5],"avail_no_missing":["2","5"],"object_location_counts":[{"shards":"0,2,4,5","objects":168},{"shards":"2,4,5","objects":48},{"shards":"2,5","objects":-103}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.8","version":"79'161","reported_seq":652,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-09-23T21:14:51.826690+0000","last_change":"2023-09-23T21:12:30.433561+0000","last_active":"2023-09-23T21:14:51.826690+0000","last_peered":"2023-09-23T21:14:51.826690+0000","last_clean":"2023-09-23T21:14:51.826690+0000","last_became_active":"2023-09-23T21:12:30.433217+0000","last_became_peered":"2023-09-23T21:12:30.433217+0000","last_unstale":"2023-09-23T21:14:51.826690+0000","last_undegraded":"2023-09-23T21:14:51.826690+0000","last_fullsized":"2023-09-23T21:14:51.826690+0000","mapping_epoch":359,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:06:52.098882+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:06:52.098882+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0092173299999999993,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":131344,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":108,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-09-23T21:14:51.805724+0000","last_change":"2023-09-23T21:12:10.823380+0000","last_active":"2023-09-23T21:14:51.805724+0000","last_peered":"2023-09-23T21:14:51.805724+0000","last_clean":"2023-09-23T21:14:51.805724+0000","last_became_active":"2023-09-23T21:12:10.823044+0000","last_became_peered":"2023-09-23T21:12:10.823044+0000","last_unstale":"2023-09-23T21:14:51.805724+0000","last_undegraded":"2023-09-23T21:14:51.805724+0000","last_fullsized":"2023-09-23T21:14:51.805724+0000","mapping_epoch":470,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:53:29.846536+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,5],"acting":[4,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":109,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-09-23T21:14:51.807558+0000","last_change":"2023-09-23T21:13:30.772451+0000","last_active":"2023-09-23T21:14:51.807558+0000","last_peered":"2023-09-23T21:14:51.807558+0000","last_clean":"2023-09-23T21:14:51.807558+0000","last_became_active":"2023-09-23T21:13:30.772040+0000","last_became_peered":"2023-09-23T21:13:30.772040+0000","last_unstale":"2023-09-23T21:14:51.807558+0000","last_undegraded":"2023-09-23T21:14:51.807558+0000","last_fullsized":"2023-09-23T21:14:51.807558+0000","mapping_epoch":501,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:24:49.246727+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,0],"acting":[4,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.0","version":"532'6069","reported_seq":13741,"reported_epoch":532,"state":"active+recovering+undersized+remapped","last_fresh":"2023-09-23T21:15:03.357653+0000","last_change":"2023-09-23T21:14:02.130787+0000","last_active":"2023-09-23T21:15:03.357653+0000","last_peered":"2023-09-23T21:15:03.357653+0000","last_clean":"2023-09-23T21:11:26.716395+0000","last_became_active":"2023-09-23T21:13:34.557827+0000","last_became_peered":"2023-09-23T21:13:34.557827+0000","last_unstale":"2023-09-23T21:15:03.357653+0000","last_undegraded":"2023-09-23T21:15:03.357653+0000","last_fullsized":"2023-09-23T21:13:31.752840+0000","mapping_epoch":502,"log_start":"435'1628","ondisk_log_start":"435'1628","created":416,"last_epoch_clean":439,"parent":"0.0","parent_split_bits":0,"last_scrub":"440'2961","last_scrub_stamp":"2023-09-23T21:11:04.564028+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_clean_scrub_stamp":"2023-09-23T21:11:04.564028+0000","objects_scrubbed":1608,"log_size":4441,"log_dups_size":0,"ondisk_log_size":4441,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:43:09.815808+0000","scrub_duration":0.41858158299999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":18677760,"num_objects":285,"num_object_clones":0,"num_object_copies":855,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":285,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3111,"num_write_kb":57216,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2829,"num_bytes_recovered":153747456,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,0],"avail_no_missing":["4","1","7"],"object_location_counts":[{"shards":"0,4","objects":285}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.b","version":"79'161","reported_seq":535,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-09-23T21:14:51.815721+0000","last_change":"2023-09-23T21:13:13.373313+0000","last_active":"2023-09-23T21:14:51.815721+0000","last_peered":"2023-09-23T21:14:51.815721+0000","last_clean":"2023-09-23T21:14:51.815721+0000","last_became_active":"2023-09-23T21:13:13.372328+0000","last_became_peered":"2023-09-23T21:13:13.372328+0000","last_unstale":"2023-09-23T21:14:51.815721+0000","last_undegraded":"2023-09-23T21:14:51.815721+0000","last_fullsized":"2023-09-23T21:14:51.815721+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":495,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:05:55.772264+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:05:55.772264+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.001862507,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":89,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-09-23T21:14:51.804459+0000","last_change":"2023-09-23T21:11:27.754717+0000","last_active":"2023-09-23T21:14:51.804459+0000","last_peered":"2023-09-23T21:14:51.804459+0000","last_clean":"2023-09-23T21:14:51.804459+0000","last_became_active":"2023-09-23T21:11:27.754376+0000","last_became_peered":"2023-09-23T21:11:27.754376+0000","last_unstale":"2023-09-23T21:14:51.804459+0000","last_undegraded":"2023-09-23T21:14:51.804459+0000","last_fullsized":"2023-09-23T21:14:51.804459+0000","mapping_epoch":452,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":453,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:26:41.734541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":463,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-09-23T21:14:51.805899+0000","last_change":"2023-09-23T21:12:10.811698+0000","last_active":"2023-09-23T21:14:51.805899+0000","last_peered":"2023-09-23T21:14:51.805899+0000","last_clean":"2023-09-23T21:14:51.805899+0000","last_became_active":"2023-09-23T21:12:10.789656+0000","last_became_peered":"2023-09-23T21:12:10.789656+0000","last_unstale":"2023-09-23T21:14:51.805899+0000","last_undegraded":"2023-09-23T21:14:51.805899+0000","last_fullsized":"2023-09-23T21:14:51.805899+0000","mapping_epoch":470,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:39:15.112091+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,5],"acting":[4,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.1","version":"532'5998","reported_seq":12416,"reported_epoch":532,"state":"active+clean","last_fresh":"2023-09-23T21:15:03.318781+0000","last_change":"2023-09-23T21:13:53.483736+0000","last_active":"2023-09-23T21:15:03.318781+0000","last_peered":"2023-09-23T21:15:03.318781+0000","last_clean":"2023-09-23T21:15:03.318781+0000","last_became_active":"2023-09-23T21:13:53.472685+0000","last_became_peered":"2023-09-23T21:13:53.472685+0000","last_unstale":"2023-09-23T21:15:03.318781+0000","last_undegraded":"2023-09-23T21:15:03.318781+0000","last_fullsized":"2023-09-23T21:15:03.318781+0000","mapping_epoch":519,"log_start":"435'1467","ondisk_log_start":"435'1467","created":416,"last_epoch_clean":520,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_clean_scrub_stamp":"2023-09-23T21:10:15.663179+0000","objects_scrubbed":0,"log_size":4531,"log_dups_size":0,"ondisk_log_size":4531,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19267584,"num_objects":294,"num_object_clones":0,"num_object_copies":882,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":294,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4650,"num_write_kb":158208,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3005,"num_bytes_recovered":156499968,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.a","version":"79'161","reported_seq":678,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-09-23T21:14:51.832191+0000","last_change":"2023-09-23T21:12:30.437790+0000","last_active":"2023-09-23T21:14:51.832191+0000","last_peered":"2023-09-23T21:14:51.832191+0000","last_clean":"2023-09-23T21:14:51.832191+0000","last_became_active":"2023-09-23T21:12:30.437427+0000","last_became_peered":"2023-09-23T21:12:30.437427+0000","last_unstale":"2023-09-23T21:14:51.832191+0000","last_undegraded":"2023-09-23T21:14:51.832191+0000","last_fullsized":"2023-09-23T21:14:51.832191+0000","mapping_epoch":446,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:06:51.075448+0000","last_deep_scrub":"79'161","last_deep_scrub_stamp":"2023-09-23T21:05:32.630292+0000","last_clean_scrub_stamp":"2023-09-23T21:06:51.075448+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0054793890000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,4],"acting":[2,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":379,"reported_epoch":532,"state":"active+clean","last_fresh":"2023-09-23T21:15:02.197432+0000","last_change":"2023-09-23T21:12:10.792918+0000","last_active":"2023-09-23T21:15:02.197432+0000","last_peered":"2023-09-23T21:15:02.197432+0000","last_clean":"2023-09-23T21:15:02.197432+0000","last_became_active":"2023-09-23T21:12:10.792679+0000","last_became_peered":"2023-09-23T21:12:10.792679+0000","last_unstale":"2023-09-23T21:15:02.197432+0000","last_undegraded":"2023-09-23T21:15:02.197432+0000","last_fullsized":"2023-09-23T21:15:02.197432+0000","mapping_epoch":470,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:30:48.008690+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,4],"acting":[3,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":510,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-09-23T21:14:51.819182+0000","last_change":"2023-09-23T21:12:10.790812+0000","last_active":"2023-09-23T21:14:51.819182+0000","last_peered":"2023-09-23T21:14:51.819182+0000","last_clean":"2023-09-23T21:14:51.819182+0000","last_became_active":"2023-09-23T21:12:10.790492+0000","last_became_peered":"2023-09-23T21:12:10.790492+0000","last_unstale":"2023-09-23T21:14:51.819182+0000","last_undegraded":"2023-09-23T21:14:51.819182+0000","last_fullsized":"2023-09-23T21:14:51.819182+0000","mapping_epoch":470,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:02:19.863118+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.c","version":"533'9706","reported_seq":10945,"reported_epoch":533,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-09-23T21:15:07.284328+0000","last_change":"2023-09-23T21:15:06.740110+0000","last_active":"2023-09-23T21:15:07.284328+0000","last_peered":"2023-09-23T21:15:07.284328+0000","last_clean":"2023-09-23T21:11:45.750376+0000","last_became_active":"2023-09-23T21:13:13.404327+0000","last_became_peered":"2023-09-23T21:13:13.404327+0000","last_unstale":"2023-09-23T21:15:07.284328+0000","last_undegraded":"2023-09-23T21:13:13.200907+0000","last_fullsized":"2023-09-23T21:13:13.193701+0000","mapping_epoch":494,"log_start":"474'5351","ondisk_log_start":"474'5351","created":416,"last_epoch_clean":434,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_clean_scrub_stamp":"2023-09-23T21:10:15.663179+0000","objects_scrubbed":0,"log_size":4355,"log_dups_size":0,"ondisk_log_size":4355,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:33:42.971375+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":21757952,"num_objects":332,"num_object_clones":0,"num_object_copies":996,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":14,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":332,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9706,"num_write_kb":321216,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":538,"num_bytes_recovered":35258368,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4],"avail_no_missing":["7","4"],"object_location_counts":[{"shards":"4,7","objects":318}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.7","version":"79'161","reported_seq":624,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-09-23T21:14:51.818894+0000","last_change":"2023-09-23T21:13:44.416548+0000","last_active":"2023-09-23T21:14:51.818894+0000","last_peered":"2023-09-23T21:14:51.818894+0000","last_clean":"2023-09-23T21:14:51.818894+0000","last_became_active":"2023-09-23T21:13:44.415617+0000","last_became_peered":"2023-09-23T21:13:44.415617+0000","last_unstale":"2023-09-23T21:14:51.818894+0000","last_undegraded":"2023-09-23T21:14:51.818894+0000","last_fullsized":"2023-09-23T21:14:51.818894+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":3,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:25.916867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:25.916867+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.001665323,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":461,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-09-23T21:14:51.818296+0000","last_change":"2023-09-23T21:13:13.211959+0000","last_active":"2023-09-23T21:14:51.818296+0000","last_peered":"2023-09-23T21:14:51.818296+0000","last_clean":"2023-09-23T21:14:51.818296+0000","last_became_active":"2023-09-23T21:13:13.211712+0000","last_became_peered":"2023-09-23T21:13:13.211712+0000","last_unstale":"2023-09-23T21:14:51.818296+0000","last_undegraded":"2023-09-23T21:14:51.818296+0000","last_fullsized":"2023-09-23T21:14:51.818296+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":495,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:37:21.655775+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,2],"acting":[0,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":210,"reported_epoch":532,"state":"active+clean","last_fresh":"2023-09-23T21:15:02.202561+0000","last_change":"2023-09-23T21:05:04.326889+0000","last_active":"2023-09-23T21:15:02.202561+0000","last_peered":"2023-09-23T21:15:02.202561+0000","last_clean":"2023-09-23T21:15:02.202561+0000","last_became_active":"2023-09-23T21:05:04.326032+0000","last_became_peered":"2023-09-23T21:05:04.326032+0000","last_unstale":"2023-09-23T21:15:02.202561+0000","last_undegraded":"2023-09-23T21:15:02.202561+0000","last_fullsized":"2023-09-23T21:15:02.202561+0000","mapping_epoch":332,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:00:17.507889+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":542,"reported_epoch":532,"state":"active+clean","last_fresh":"2023-09-23T21:15:02.202170+0000","last_change":"2023-09-23T21:13:13.285608+0000","last_active":"2023-09-23T21:15:02.202170+0000","last_peered":"2023-09-23T21:15:02.202170+0000","last_clean":"2023-09-23T21:15:02.202170+0000","last_became_active":"2023-09-23T21:13:13.285255+0000","last_became_peered":"2023-09-23T21:13:13.285255+0000","last_unstale":"2023-09-23T21:15:02.202170+0000","last_undegraded":"2023-09-23T21:15:02.202170+0000","last_fullsized":"2023-09-23T21:15:02.202170+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":495,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:42:30.087599+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.a","version":"532'9467","reported_seq":18206,"reported_epoch":532,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T21:15:06.306500+0000","last_change":"2023-09-23T21:13:31.927282+0000","last_active":"2023-09-23T21:15:06.306500+0000","last_peered":"2023-09-23T21:15:06.306500+0000","last_clean":"2023-09-23T21:13:30.353360+0000","last_became_active":"2023-09-23T21:13:31.896923+0000","last_became_peered":"2023-09-23T21:13:31.896923+0000","last_unstale":"2023-09-23T21:15:06.306500+0000","last_undegraded":"2023-09-23T21:13:31.776904+0000","last_fullsized":"2023-09-23T21:13:31.751334+0000","mapping_epoch":502,"log_start":"468'5112","ondisk_log_start":"468'5112","created":416,"last_epoch_clean":499,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_clean_scrub_stamp":"2023-09-23T21:10:15.663179+0000","objects_scrubbed":0,"log_size":4355,"log_dups_size":0,"ondisk_log_size":4355,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:23:14.477602+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26542080,"num_objects":405,"num_object_clones":0,"num_object_copies":1215,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2243,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":405,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9467,"num_write_kb":315904,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4293,"num_bytes_recovered":281346048,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[2,5],"avail_no_missing":["2","3","5"],"object_location_counts":[{"shards":"2,3,5","objects":405},{"shards":"2,5","objects":-238}],"blocked_by":[],"up_primary":0,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.1","version":"79'161","reported_seq":602,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-09-23T21:14:51.826881+0000","last_change":"2023-09-23T21:13:12.204135+0000","last_active":"2023-09-23T21:14:51.826881+0000","last_peered":"2023-09-23T21:14:51.826881+0000","last_clean":"2023-09-23T21:14:51.826881+0000","last_became_active":"2023-09-23T21:13:12.203380+0000","last_became_peered":"2023-09-23T21:13:12.203380+0000","last_unstale":"2023-09-23T21:14:51.826881+0000","last_undegraded":"2023-09-23T21:14:51.826881+0000","last_fullsized":"2023-09-23T21:14:51.826881+0000","mapping_epoch":493,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:06:52.575900+0000","last_deep_scrub":"79'161","last_deep_scrub_stamp":"2023-09-23T21:05:30.652189+0000","last_clean_scrub_stamp":"2023-09-23T21:06:52.575900+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":198,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-09-23T21:14:51.819722+0000","last_change":"2023-09-23T21:13:13.238617+0000","last_active":"2023-09-23T21:14:51.819722+0000","last_peered":"2023-09-23T21:14:51.819722+0000","last_clean":"2023-09-23T21:14:51.819722+0000","last_became_active":"2023-09-23T21:13:13.232926+0000","last_became_peered":"2023-09-23T21:13:13.232926+0000","last_unstale":"2023-09-23T21:14:51.819722+0000","last_undegraded":"2023-09-23T21:14:51.819722+0000","last_fullsized":"2023-09-23T21:14:51.819722+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":495,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:17:21.823772+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":465,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-09-23T21:14:51.810436+0000","last_change":"2023-09-23T21:13:14.144918+0000","last_active":"2023-09-23T21:14:51.810436+0000","last_peered":"2023-09-23T21:14:51.810436+0000","last_clean":"2023-09-23T21:14:51.810436+0000","last_became_active":"2023-09-23T21:13:14.132388+0000","last_became_peered":"2023-09-23T21:13:14.132388+0000","last_unstale":"2023-09-23T21:14:51.810436+0000","last_undegraded":"2023-09-23T21:14:51.810436+0000","last_fullsized":"2023-09-23T21:14:51.810436+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":495,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:56:02.619905+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,5],"acting":[4,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":352,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-09-23T21:14:51.826512+0000","last_change":"2023-09-23T21:12:10.764503+0000","last_active":"2023-09-23T21:14:51.826512+0000","last_peered":"2023-09-23T21:14:51.826512+0000","last_clean":"2023-09-23T21:14:51.826512+0000","last_became_active":"2023-09-23T21:12:10.763933+0000","last_became_peered":"2023-09-23T21:12:10.763933+0000","last_unstale":"2023-09-23T21:14:51.826512+0000","last_undegraded":"2023-09-23T21:14:51.826512+0000","last_fullsized":"2023-09-23T21:14:51.826512+0000","mapping_epoch":470,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:28:33.202825+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,7],"acting":[2,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"10.9","version":"532'6125","reported_seq":7273,"reported_epoch":532,"state":"active+clean","last_fresh":"2023-09-23T21:15:06.217376+0000","last_change":"2023-09-23T21:11:08.062606+0000","last_active":"2023-09-23T21:15:06.217376+0000","last_peered":"2023-09-23T21:15:06.217376+0000","last_clean":"2023-09-23T21:15:06.217376+0000","last_became_active":"2023-09-23T21:10:58.780961+0000","last_became_peered":"2023-09-23T21:10:58.780961+0000","last_unstale":"2023-09-23T21:15:06.217376+0000","last_undegraded":"2023-09-23T21:15:06.217376+0000","last_fullsized":"2023-09-23T21:15:06.217376+0000","mapping_epoch":428,"log_start":"427'709","ondisk_log_start":"427'709","created":416,"last_epoch_clean":439,"parent":"0.0","parent_split_bits":0,"last_scrub":"443'3049","last_scrub_stamp":"2023-09-23T21:11:08.062479+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_clean_scrub_stamp":"2023-09-23T21:11:08.062479+0000","objects_scrubbed":1623,"log_size":5416,"log_dups_size":0,"ondisk_log_size":5416,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:56:22.338088+0000","scrub_duration":0.28207626200000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":14942208,"num_objects":228,"num_object_clones":0,"num_object_copies":684,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":228,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3077,"num_write_kb":53824,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.2","version":"79'161","reported_seq":811,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-09-23T21:14:51.819308+0000","last_change":"2023-09-23T21:13:12.209907+0000","last_active":"2023-09-23T21:14:51.819308+0000","last_peered":"2023-09-23T21:14:51.819308+0000","last_clean":"2023-09-23T21:14:51.819308+0000","last_became_active":"2023-09-23T21:13:12.205664+0000","last_became_peered":"2023-09-23T21:13:12.205664+0000","last_unstale":"2023-09-23T21:14:51.819308+0000","last_undegraded":"2023-09-23T21:14:51.819308+0000","last_fullsized":"2023-09-23T21:14:51.819308+0000","mapping_epoch":493,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:24.802451+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:24.802451+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:26:16.035788+0000","scrub_duration":0.0090861250000000005,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,4],"acting":[0,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":540,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-09-23T21:14:51.828467+0000","last_change":"2023-09-23T21:13:12.206980+0000","last_active":"2023-09-23T21:14:51.828467+0000","last_peered":"2023-09-23T21:14:51.828467+0000","last_clean":"2023-09-23T21:14:51.828467+0000","last_became_active":"2023-09-23T21:13:12.206650+0000","last_became_peered":"2023-09-23T21:13:12.206650+0000","last_unstale":"2023-09-23T21:14:51.828467+0000","last_undegraded":"2023-09-23T21:14:51.828467+0000","last_fullsized":"2023-09-23T21:14:51.828467+0000","mapping_epoch":493,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:10:01.892936+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":524,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-09-23T21:14:51.826882+0000","last_change":"2023-09-23T21:13:13.407073+0000","last_active":"2023-09-23T21:14:51.826882+0000","last_peered":"2023-09-23T21:14:51.826882+0000","last_clean":"2023-09-23T21:14:51.826882+0000","last_became_active":"2023-09-23T21:13:13.395047+0000","last_became_peered":"2023-09-23T21:13:13.395047+0000","last_unstale":"2023-09-23T21:14:51.826882+0000","last_undegraded":"2023-09-23T21:14:51.826882+0000","last_fullsized":"2023-09-23T21:14:51.826882+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":495,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:04:43.384190+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,5],"acting":[2,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":41,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-09-23T21:14:51.816985+0000","last_change":"2023-09-23T21:13:30.785054+0000","last_active":"2023-09-23T21:14:51.816985+0000","last_peered":"2023-09-23T21:14:51.816985+0000","last_clean":"2023-09-23T21:14:51.816985+0000","last_became_active":"2023-09-23T21:13:30.784701+0000","last_became_peered":"2023-09-23T21:13:30.784701+0000","last_unstale":"2023-09-23T21:14:51.816985+0000","last_undegraded":"2023-09-23T21:14:51.816985+0000","last_fullsized":"2023-09-23T21:14:51.816985+0000","mapping_epoch":501,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:19:20.152252+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.b","version":"532'9512","reported_seq":20546,"reported_epoch":532,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2023-09-23T21:15:03.675982+0000","last_change":"2023-09-23T21:13:59.891994+0000","last_active":"2023-09-23T21:15:03.675982+0000","last_peered":"2023-09-23T21:15:03.675982+0000","last_clean":"2023-09-23T21:11:26.732248+0000","last_became_active":"2023-09-23T21:13:14.937458+0000","last_became_peered":"2023-09-23T21:13:14.937458+0000","last_unstale":"2023-09-23T21:15:03.675982+0000","last_undegraded":"2023-09-23T21:13:59.884123+0000","last_fullsized":"2023-09-23T21:15:03.675982+0000","mapping_epoch":494,"log_start":"426'600","ondisk_log_start":"426'600","created":416,"last_epoch_clean":430,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_clean_scrub_stamp":"2023-09-23T21:10:15.663179+0000","objects_scrubbed":0,"log_size":8912,"log_dups_size":0,"ondisk_log_size":8912,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:36:15.760947+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":38010880,"num_objects":580,"num_object_clones":0,"num_object_copies":1740,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":8,"num_objects_misplaced":1337,"num_objects_unfound":0,"num_objects_dirty":580,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9512,"num_write_kb":322944,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5424,"num_bytes_recovered":355270656,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[1,4,7],"avail_no_missing":["1","7"],"object_location_counts":[{"shards":"1,4,7","objects":505}],"blocked_by":[],"up_primary":0,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":317,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-09-23T21:14:51.831770+0000","last_change":"2023-09-23T21:10:26.769764+0000","last_active":"2023-09-23T21:14:51.831770+0000","last_peered":"2023-09-23T21:14:51.831770+0000","last_clean":"2023-09-23T21:14:51.831770+0000","last_became_active":"2023-09-23T21:10:26.769326+0000","last_became_peered":"2023-09-23T21:10:26.769326+0000","last_unstale":"2023-09-23T21:14:51.831770+0000","last_undegraded":"2023-09-23T21:14:51.831770+0000","last_fullsized":"2023-09-23T21:14:51.831770+0000","mapping_epoch":423,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":424,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:57:45.219837+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"79'161","reported_seq":714,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-09-23T21:14:51.817319+0000","last_change":"2023-09-23T21:13:33.766895+0000","last_active":"2023-09-23T21:14:51.817319+0000","last_peered":"2023-09-23T21:14:51.817319+0000","last_clean":"2023-09-23T21:14:51.817319+0000","last_became_active":"2023-09-23T21:13:33.766538+0000","last_became_peered":"2023-09-23T21:13:33.766538+0000","last_unstale":"2023-09-23T21:14:51.817319+0000","last_undegraded":"2023-09-23T21:14:51.817319+0000","last_fullsized":"2023-09-23T21:14:51.817319+0000","mapping_epoch":504,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":505,"parent":"0.0","parent_split_bits":0,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:20.738273+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:20.738273+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:35:35.855987+0000","scrub_duration":0.0088336339999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,2],"acting":[0,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":540,"reported_epoch":532,"state":"active+clean","last_fresh":"2023-09-23T21:15:02.202124+0000","last_change":"2023-09-23T21:11:08.760417+0000","last_active":"2023-09-23T21:15:02.202124+0000","last_peered":"2023-09-23T21:15:02.202124+0000","last_clean":"2023-09-23T21:15:02.202124+0000","last_became_active":"2023-09-23T21:11:08.760150+0000","last_became_peered":"2023-09-23T21:11:08.760150+0000","last_unstale":"2023-09-23T21:15:02.202124+0000","last_undegraded":"2023-09-23T21:15:02.202124+0000","last_fullsized":"2023-09-23T21:15:02.202124+0000","mapping_epoch":443,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":444,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:23:10.849768+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":106,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-09-23T21:14:51.807714+0000","last_change":"2023-09-23T21:13:12.216020+0000","last_active":"2023-09-23T21:14:51.807714+0000","last_peered":"2023-09-23T21:14:51.807714+0000","last_clean":"2023-09-23T21:14:51.807714+0000","last_became_active":"2023-09-23T21:13:12.215180+0000","last_became_peered":"2023-09-23T21:13:12.215180+0000","last_unstale":"2023-09-23T21:14:51.807714+0000","last_undegraded":"2023-09-23T21:14:51.807714+0000","last_fullsized":"2023-09-23T21:14:51.807714+0000","mapping_epoch":493,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:26:18.704684+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.8","version":"532'5993","reported_seq":8391,"reported_epoch":532,"state":"active+clean","last_fresh":"2023-09-23T21:15:05.202421+0000","last_change":"2023-09-23T21:14:03.796930+0000","last_active":"2023-09-23T21:15:05.202421+0000","last_peered":"2023-09-23T21:15:05.202421+0000","last_clean":"2023-09-23T21:15:05.202421+0000","last_became_active":"2023-09-23T21:14:03.786177+0000","last_became_peered":"2023-09-23T21:14:03.786177+0000","last_unstale":"2023-09-23T21:15:05.202421+0000","last_undegraded":"2023-09-23T21:15:05.202421+0000","last_fullsized":"2023-09-23T21:15:05.202421+0000","mapping_epoch":524,"log_start":"434'1302","ondisk_log_start":"434'1302","created":416,"last_epoch_clean":525,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_clean_scrub_stamp":"2023-09-23T21:10:15.663179+0000","objects_scrubbed":0,"log_size":4691,"log_dups_size":0,"ondisk_log_size":4691,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":12156928,"num_objects":185,"num_object_clones":0,"num_object_copies":555,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":185,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4643,"num_write_kb":154528,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":880,"num_bytes_recovered":16908288,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,7],"acting":[0,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.3","version":"79'161","reported_seq":834,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-09-23T21:14:51.804781+0000","last_change":"2023-09-23T21:13:33.771279+0000","last_active":"2023-09-23T21:14:51.804781+0000","last_peered":"2023-09-23T21:14:51.804781+0000","last_clean":"2023-09-23T21:14:51.804781+0000","last_became_active":"2023-09-23T21:13:33.770980+0000","last_became_peered":"2023-09-23T21:13:33.770980+0000","last_unstale":"2023-09-23T21:14:51.804781+0000","last_undegraded":"2023-09-23T21:14:51.804781+0000","last_fullsized":"2023-09-23T21:14:51.804781+0000","mapping_epoch":504,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":505,"parent":"0.0","parent_split_bits":3,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:05:57.832302+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:05:57.832302+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:19:42.309604+0000","scrub_duration":0.010206847999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":348,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-09-23T21:14:51.817731+0000","last_change":"2023-09-23T21:13:13.212775+0000","last_active":"2023-09-23T21:14:51.817731+0000","last_peered":"2023-09-23T21:14:51.817731+0000","last_clean":"2023-09-23T21:14:51.817731+0000","last_became_active":"2023-09-23T21:13:13.212460+0000","last_became_peered":"2023-09-23T21:13:13.212460+0000","last_unstale":"2023-09-23T21:14:51.817731+0000","last_undegraded":"2023-09-23T21:14:51.817731+0000","last_fullsized":"2023-09-23T21:14:51.817731+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":495,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:27:10.853970+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":530,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-09-23T21:14:51.826628+0000","last_change":"2023-09-23T21:13:44.409212+0000","last_active":"2023-09-23T21:14:51.826628+0000","last_peered":"2023-09-23T21:14:51.826628+0000","last_clean":"2023-09-23T21:14:51.826628+0000","last_became_active":"2023-09-23T21:13:44.408958+0000","last_became_peered":"2023-09-23T21:13:44.408958+0000","last_unstale":"2023-09-23T21:14:51.826628+0000","last_undegraded":"2023-09-23T21:14:51.826628+0000","last_fullsized":"2023-09-23T21:14:51.826628+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:03:52.749964+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":41,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-09-23T21:14:51.816988+0000","last_change":"2023-09-23T21:13:30.804379+0000","last_active":"2023-09-23T21:14:51.816988+0000","last_peered":"2023-09-23T21:14:51.816988+0000","last_clean":"2023-09-23T21:14:51.816988+0000","last_became_active":"2023-09-23T21:13:30.799907+0000","last_became_peered":"2023-09-23T21:13:30.799907+0000","last_unstale":"2023-09-23T21:14:51.816988+0000","last_undegraded":"2023-09-23T21:14:51.816988+0000","last_fullsized":"2023-09-23T21:14:51.816988+0000","mapping_epoch":501,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:49:42.859212+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,7],"acting":[0,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.d","version":"532'9435","reported_seq":9600,"reported_epoch":532,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-23T21:15:03.336118+0000","last_change":"2023-09-23T21:13:31.801486+0000","last_active":"2023-09-23T21:15:03.336118+0000","last_peered":"2023-09-23T21:15:03.336118+0000","last_clean":"2023-09-23T21:11:37.659929+0000","last_became_active":"2023-09-23T21:13:31.786626+0000","last_became_peered":"2023-09-23T21:13:31.786626+0000","last_unstale":"2023-09-23T21:15:03.336118+0000","last_undegraded":"2023-09-23T21:13:31.760908+0000","last_fullsized":"2023-09-23T21:15:03.336118+0000","mapping_epoch":502,"log_start":"479'5495","ondisk_log_start":"479'5495","created":416,"last_epoch_clean":417,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_clean_scrub_stamp":"2023-09-23T21:10:15.663179+0000","objects_scrubbed":0,"log_size":3940,"log_dups_size":0,"ondisk_log_size":3940,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:46:02.424493+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":37093400,"num_objects":567,"num_object_clones":0,"num_object_copies":1701,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1265,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":567,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":9435,"num_write_kb":320001,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,3],"avail_no_missing":["4","3","7"],"object_location_counts":[{"shards":"0,3,4","objects":488},{"shards":"3,4","objects":79}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.6","version":"79'161","reported_seq":450,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-09-23T21:14:51.816901+0000","last_change":"2023-09-23T21:13:33.770715+0000","last_active":"2023-09-23T21:14:51.816901+0000","last_peered":"2023-09-23T21:14:51.816901+0000","last_clean":"2023-09-23T21:14:51.816901+0000","last_became_active":"2023-09-23T21:13:33.769303+0000","last_became_peered":"2023-09-23T21:13:33.769303+0000","last_unstale":"2023-09-23T21:14:51.816901+0000","last_undegraded":"2023-09-23T21:14:51.816901+0000","last_fullsized":"2023-09-23T21:14:51.816901+0000","mapping_epoch":504,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":505,"parent":"0.0","parent_split_bits":3,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:10:24.268662+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:10:24.268662+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13417,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":11,"num_read_kb":20,"num_write":5,"num_write_kb":55,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":40252,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":159,"reported_epoch":532,"state":"active+clean","last_fresh":"2023-09-23T21:15:02.189501+0000","last_change":"2023-09-23T21:13:30.770826+0000","last_active":"2023-09-23T21:15:02.189501+0000","last_peered":"2023-09-23T21:15:02.189501+0000","last_clean":"2023-09-23T21:15:02.189501+0000","last_became_active":"2023-09-23T21:13:30.770512+0000","last_became_peered":"2023-09-23T21:13:30.770512+0000","last_unstale":"2023-09-23T21:15:02.189501+0000","last_undegraded":"2023-09-23T21:15:02.189501+0000","last_fullsized":"2023-09-23T21:15:02.189501+0000","mapping_epoch":501,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:44:23.470538+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":415,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-09-23T21:14:51.826778+0000","last_change":"2023-09-23T21:13:30.790590+0000","last_active":"2023-09-23T21:14:51.826778+0000","last_peered":"2023-09-23T21:14:51.826778+0000","last_clean":"2023-09-23T21:14:51.826778+0000","last_became_active":"2023-09-23T21:13:30.790253+0000","last_became_peered":"2023-09-23T21:13:30.790253+0000","last_unstale":"2023-09-23T21:14:51.826778+0000","last_undegraded":"2023-09-23T21:14:51.826778+0000","last_fullsized":"2023-09-23T21:14:51.826778+0000","mapping_epoch":501,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:23:41.843250+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,0],"acting":[2,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":477,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-09-23T21:14:51.816262+0000","last_change":"2023-09-23T21:13:12.210546+0000","last_active":"2023-09-23T21:14:51.816262+0000","last_peered":"2023-09-23T21:14:51.816262+0000","last_clean":"2023-09-23T21:14:51.816262+0000","last_became_active":"2023-09-23T21:13:12.205934+0000","last_became_peered":"2023-09-23T21:13:12.205934+0000","last_unstale":"2023-09-23T21:14:51.816262+0000","last_undegraded":"2023-09-23T21:14:51.816262+0000","last_fullsized":"2023-09-23T21:14:51.816262+0000","mapping_epoch":493,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:21:18.299428+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.e","version":"532'9673","reported_seq":16647,"reported_epoch":532,"state":"active+clean","last_fresh":"2023-09-23T21:15:03.301957+0000","last_change":"2023-09-23T21:12:38.623611+0000","last_active":"2023-09-23T21:15:03.301957+0000","last_peered":"2023-09-23T21:15:03.301957+0000","last_clean":"2023-09-23T21:15:03.301957+0000","last_became_active":"2023-09-23T21:12:38.610155+0000","last_became_peered":"2023-09-23T21:12:38.610155+0000","last_unstale":"2023-09-23T21:15:03.301957+0000","last_undegraded":"2023-09-23T21:15:03.301957+0000","last_fullsized":"2023-09-23T21:15:03.301957+0000","mapping_epoch":483,"log_start":"481'5693","ondisk_log_start":"481'5693","created":416,"last_epoch_clean":484,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_clean_scrub_stamp":"2023-09-23T21:10:15.663179+0000","objects_scrubbed":0,"log_size":3980,"log_dups_size":0,"ondisk_log_size":3980,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:19:27.097170+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":36110336,"num_objects":551,"num_object_clones":0,"num_object_copies":1653,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":551,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9673,"num_write_kb":327168,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3412,"num_bytes_recovered":223608832,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.5","version":"79'161","reported_seq":559,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-09-23T21:14:51.807636+0000","last_change":"2023-09-23T21:13:38.373969+0000","last_active":"2023-09-23T21:14:51.807636+0000","last_peered":"2023-09-23T21:14:51.807636+0000","last_clean":"2023-09-23T21:14:51.807636+0000","last_became_active":"2023-09-23T21:13:38.373633+0000","last_became_peered":"2023-09-23T21:13:38.373633+0000","last_unstale":"2023-09-23T21:14:51.807636+0000","last_undegraded":"2023-09-23T21:14:51.807636+0000","last_fullsized":"2023-09-23T21:14:51.807636+0000","mapping_epoch":507,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":3,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:09:34.892947+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:09:34.892947+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13417,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":11,"num_read_kb":20,"num_write":5,"num_write_kb":55,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":40252,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":72,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-09-23T21:14:51.817138+0000","last_change":"2023-09-23T21:13:12.235743+0000","last_active":"2023-09-23T21:14:51.817138+0000","last_peered":"2023-09-23T21:14:51.817138+0000","last_clean":"2023-09-23T21:14:51.817138+0000","last_became_active":"2023-09-23T21:13:12.216241+0000","last_became_peered":"2023-09-23T21:13:12.216241+0000","last_unstale":"2023-09-23T21:14:51.817138+0000","last_undegraded":"2023-09-23T21:14:51.817138+0000","last_fullsized":"2023-09-23T21:14:51.817138+0000","mapping_epoch":493,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:12:19.090710+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":512,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-09-23T21:14:51.805775+0000","last_change":"2023-09-23T21:13:30.807520+0000","last_active":"2023-09-23T21:14:51.805775+0000","last_peered":"2023-09-23T21:14:51.805775+0000","last_clean":"2023-09-23T21:14:51.805775+0000","last_became_active":"2023-09-23T21:13:30.807222+0000","last_became_peered":"2023-09-23T21:13:30.807222+0000","last_unstale":"2023-09-23T21:14:51.805775+0000","last_undegraded":"2023-09-23T21:14:51.805775+0000","last_fullsized":"2023-09-23T21:14:51.805775+0000","mapping_epoch":501,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:40:31.989749+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":373,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-09-23T21:14:51.812067+0000","last_change":"2023-09-23T21:13:30.794095+0000","last_active":"2023-09-23T21:14:51.812067+0000","last_peered":"2023-09-23T21:14:51.812067+0000","last_clean":"2023-09-23T21:14:51.812067+0000","last_became_active":"2023-09-23T21:13:30.793791+0000","last_became_peered":"2023-09-23T21:13:30.793791+0000","last_unstale":"2023-09-23T21:14:51.812067+0000","last_undegraded":"2023-09-23T21:14:51.812067+0000","last_fullsized":"2023-09-23T21:14:51.812067+0000","mapping_epoch":501,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:36:28.707362+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.f","version":"532'9573","reported_seq":9781,"reported_epoch":532,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T21:15:04.192231+0000","last_change":"2023-09-23T21:13:59.874192+0000","last_active":"2023-09-23T21:15:04.192231+0000","last_peered":"2023-09-23T21:15:04.192231+0000","last_clean":"2023-09-23T21:11:07.652290+0000","last_became_active":"2023-09-23T21:11:09.897463+0000","last_became_peered":"2023-09-23T21:11:09.897463+0000","last_unstale":"2023-09-23T21:15:04.192231+0000","last_undegraded":"2023-09-23T21:11:27.781677+0000","last_fullsized":"2023-09-23T21:11:09.761652+0000","mapping_epoch":444,"log_start":"485'5909","ondisk_log_start":"485'5909","created":416,"last_epoch_clean":417,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_clean_scrub_stamp":"2023-09-23T21:10:15.663179+0000","objects_scrubbed":0,"log_size":3664,"log_dups_size":0,"ondisk_log_size":3664,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:42:51.588441+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":35454976,"num_objects":541,"num_object_clones":0,"num_object_copies":1623,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3295,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":541,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9573,"num_write_kb":323648,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":32,"num_bytes_recovered":2097152,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2],"avail_no_missing":["3","1","2"],"object_location_counts":[{"shards":"2,3","objects":-2754}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.4","version":"79'161","reported_seq":903,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-09-23T21:14:54.216943+0000","last_change":"2023-09-23T21:10:29.807785+0000","last_active":"2023-09-23T21:14:54.216943+0000","last_peered":"2023-09-23T21:14:54.216943+0000","last_clean":"2023-09-23T21:14:54.216943+0000","last_became_active":"2023-09-23T21:10:29.807366+0000","last_became_peered":"2023-09-23T21:10:29.807366+0000","last_unstale":"2023-09-23T21:14:54.216943+0000","last_undegraded":"2023-09-23T21:14:54.216943+0000","last_fullsized":"2023-09-23T21:14:54.216943+0000","mapping_epoch":426,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":427,"parent":"0.0","parent_split_bits":3,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:20.012764+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:20.012764+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:24:45.763834+0000","scrub_duration":0.0045251919999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":471,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-09-23T21:14:51.827466+0000","last_change":"2023-09-23T21:10:26.768364+0000","last_active":"2023-09-23T21:14:51.827466+0000","last_peered":"2023-09-23T21:14:51.827466+0000","last_clean":"2023-09-23T21:14:51.827466+0000","last_became_active":"2023-09-23T21:10:26.768052+0000","last_became_peered":"2023-09-23T21:10:26.768052+0000","last_unstale":"2023-09-23T21:14:51.827466+0000","last_undegraded":"2023-09-23T21:14:51.827466+0000","last_fullsized":"2023-09-23T21:14:51.827466+0000","mapping_epoch":423,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":424,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:22:42.201200+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,3],"acting":[2,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.6","version":"70'1","reported_seq":502,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-09-23T21:14:51.817267+0000","last_change":"2023-09-23T21:13:30.793290+0000","last_active":"2023-09-23T21:14:51.817267+0000","last_peered":"2023-09-23T21:14:51.817267+0000","last_clean":"2023-09-23T21:14:51.817267+0000","last_became_active":"2023-09-23T21:13:30.786079+0000","last_became_peered":"2023-09-23T21:13:30.786079+0000","last_unstale":"2023-09-23T21:14:51.817267+0000","last_undegraded":"2023-09-23T21:14:51.817267+0000","last_fullsized":"2023-09-23T21:14:51.817267+0000","mapping_epoch":501,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:34:43.281095+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":106,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-09-23T21:14:51.804392+0000","last_change":"2023-09-23T21:13:12.216895+0000","last_active":"2023-09-23T21:14:51.804392+0000","last_peered":"2023-09-23T21:14:51.804392+0000","last_clean":"2023-09-23T21:14:51.804392+0000","last_became_active":"2023-09-23T21:13:12.216654+0000","last_became_peered":"2023-09-23T21:13:12.216654+0000","last_unstale":"2023-09-23T21:14:51.804392+0000","last_undegraded":"2023-09-23T21:14:51.804392+0000","last_fullsized":"2023-09-23T21:14:51.804392+0000","mapping_epoch":493,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:02:40.430096+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.19","version":"79'161","reported_seq":685,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-09-23T21:14:56.306704+0000","last_change":"2023-09-23T21:12:30.442914+0000","last_active":"2023-09-23T21:14:56.306704+0000","last_peered":"2023-09-23T21:14:56.306704+0000","last_clean":"2023-09-23T21:14:56.306704+0000","last_became_active":"2023-09-23T21:12:30.442578+0000","last_became_peered":"2023-09-23T21:12:30.442578+0000","last_unstale":"2023-09-23T21:14:56.306704+0000","last_undegraded":"2023-09-23T21:14:56.306704+0000","last_fullsized":"2023-09-23T21:14:56.306704+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":479,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:10:37.696951+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:10:37.696951+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.001386504,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,3],"acting":[2,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"10.12","version":"532'6038","reported_seq":9340,"reported_epoch":532,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T21:15:03.347033+0000","last_change":"2023-09-23T21:13:13.419184+0000","last_active":"2023-09-23T21:15:03.347033+0000","last_peered":"2023-09-23T21:15:03.347033+0000","last_clean":"2023-09-23T21:11:45.774771+0000","last_became_active":"2023-09-23T21:13:13.409281+0000","last_became_peered":"2023-09-23T21:13:13.409281+0000","last_unstale":"2023-09-23T21:15:03.347033+0000","last_undegraded":"2023-09-23T21:13:13.284063+0000","last_fullsized":"2023-09-23T21:13:13.191117+0000","mapping_epoch":494,"log_start":"435'1550","ondisk_log_start":"435'1550","created":438,"last_epoch_clean":439,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_clean_scrub_stamp":"2023-09-23T21:10:15.663179+0000","objects_scrubbed":0,"log_size":4488,"log_dups_size":0,"ondisk_log_size":4488,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":18022400,"num_objects":275,"num_object_clones":0,"num_object_copies":825,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1570,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":275,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4661,"num_write_kb":157952,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1568,"num_bytes_recovered":102760448,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,3],"avail_no_missing":["4","1"],"object_location_counts":[{"shards":"0,1,4","objects":236},{"shards":"1,4","objects":41},{"shards":"3,4","objects":-2}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":41,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-09-23T21:14:51.816193+0000","last_change":"2023-09-23T21:13:30.791297+0000","last_active":"2023-09-23T21:14:51.816193+0000","last_peered":"2023-09-23T21:14:51.816193+0000","last_clean":"2023-09-23T21:14:51.816193+0000","last_became_active":"2023-09-23T21:13:30.788721+0000","last_became_peered":"2023-09-23T21:13:30.788721+0000","last_unstale":"2023-09-23T21:14:51.816193+0000","last_undegraded":"2023-09-23T21:14:51.816193+0000","last_fullsized":"2023-09-23T21:14:51.816193+0000","mapping_epoch":501,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:31:18.147883+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":35,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-09-23T21:14:51.826691+0000","last_change":"2023-09-23T21:13:38.373814+0000","last_active":"2023-09-23T21:14:51.826691+0000","last_peered":"2023-09-23T21:14:51.826691+0000","last_clean":"2023-09-23T21:14:51.826691+0000","last_became_active":"2023-09-23T21:13:38.373458+0000","last_became_peered":"2023-09-23T21:13:38.373458+0000","last_unstale":"2023-09-23T21:14:51.826691+0000","last_undegraded":"2023-09-23T21:14:51.826691+0000","last_fullsized":"2023-09-23T21:14:51.826691+0000","mapping_epoch":507,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:23:08.288401+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.1e","version":"79'161","reported_seq":699,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-09-23T21:14:51.804981+0000","last_change":"2023-09-23T21:12:30.448761+0000","last_active":"2023-09-23T21:14:51.804981+0000","last_peered":"2023-09-23T21:14:51.804981+0000","last_clean":"2023-09-23T21:14:51.804981+0000","last_became_active":"2023-09-23T21:12:30.448043+0000","last_became_peered":"2023-09-23T21:12:30.448043+0000","last_unstale":"2023-09-23T21:14:51.804981+0000","last_undegraded":"2023-09-23T21:14:51.804981+0000","last_fullsized":"2023-09-23T21:14:51.804981+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":479,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:10:39.567232+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:10:39.567232+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.005229694,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.15","version":"532'6058","reported_seq":6207,"reported_epoch":532,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T21:15:06.301209+0000","last_change":"2023-09-23T21:13:13.436807+0000","last_active":"2023-09-23T21:15:06.301209+0000","last_peered":"2023-09-23T21:15:06.301209+0000","last_clean":"2023-09-23T21:11:45.765730+0000","last_became_active":"2023-09-23T21:13:13.429206+0000","last_became_peered":"2023-09-23T21:13:13.429206+0000","last_unstale":"2023-09-23T21:15:06.301209+0000","last_undegraded":"2023-09-23T21:13:13.246273+0000","last_fullsized":"2023-09-23T21:13:13.203825+0000","mapping_epoch":494,"log_start":"428'724","ondisk_log_start":"428'724","created":438,"last_epoch_clean":439,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_clean_scrub_stamp":"2023-09-23T21:10:15.663179+0000","objects_scrubbed":0,"log_size":5334,"log_dups_size":0,"ondisk_log_size":5334,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":14680064,"num_objects":224,"num_object_clones":0,"num_object_copies":672,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":643,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":224,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4724,"num_write_kb":158336,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,3],"acting":[2,3],"avail_no_missing":["2","3"],"object_location_counts":[{"shards":"2,3","objects":188}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":107,"reported_epoch":531,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-23T21:14:51.805207+0000","last_change":"2023-09-23T21:12:14.313202+0000","last_active":"2023-09-23T21:14:51.805207+0000","last_peered":"2023-09-23T21:14:51.805207+0000","last_clean":"2023-09-23T21:12:09.299355+0000","last_became_active":"2023-09-23T21:12:14.209408+0000","last_became_peered":"2023-09-23T21:12:14.209408+0000","last_unstale":"2023-09-23T21:14:51.805207+0000","last_undegraded":"2023-09-23T21:12:12.933857+0000","last_fullsized":"2023-09-23T21:14:51.805207+0000","mapping_epoch":470,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":444,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:45:40.142413+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,7],"acting":[4,5,7],"avail_no_missing":["4","1","5"],"object_location_counts":[{"shards":"1,4,5","objects":1}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":96,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-09-23T21:14:51.805103+0000","last_change":"2023-09-23T21:12:10.770474+0000","last_active":"2023-09-23T21:14:51.805103+0000","last_peered":"2023-09-23T21:14:51.805103+0000","last_clean":"2023-09-23T21:14:51.805103+0000","last_became_active":"2023-09-23T21:12:10.770184+0000","last_became_peered":"2023-09-23T21:12:10.770184+0000","last_unstale":"2023-09-23T21:14:51.805103+0000","last_undegraded":"2023-09-23T21:14:51.805103+0000","last_fullsized":"2023-09-23T21:14:51.805103+0000","mapping_epoch":470,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:05:09.407151+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.14","version":"532'6047","reported_seq":12151,"reported_epoch":532,"state":"active+clean","last_fresh":"2023-09-23T21:15:05.140367+0000","last_change":"2023-09-23T21:14:01.515957+0000","last_active":"2023-09-23T21:15:05.140367+0000","last_peered":"2023-09-23T21:15:05.140367+0000","last_clean":"2023-09-23T21:15:05.140367+0000","last_became_active":"2023-09-23T21:14:01.506428+0000","last_became_peered":"2023-09-23T21:14:01.506428+0000","last_unstale":"2023-09-23T21:15:05.140367+0000","last_undegraded":"2023-09-23T21:15:05.140367+0000","last_fullsized":"2023-09-23T21:15:05.140367+0000","mapping_epoch":522,"log_start":"435'1452","ondisk_log_start":"435'1452","created":438,"last_epoch_clean":523,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_clean_scrub_stamp":"2023-09-23T21:10:15.663179+0000","objects_scrubbed":0,"log_size":4595,"log_dups_size":0,"ondisk_log_size":4595,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":12910592,"num_objects":197,"num_object_clones":0,"num_object_copies":591,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":197,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4713,"num_write_kb":157120,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2660,"num_bytes_recovered":126156800,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":322,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-09-23T21:14:56.310243+0000","last_change":"2023-09-23T21:12:10.771160+0000","last_active":"2023-09-23T21:14:56.310243+0000","last_peered":"2023-09-23T21:14:56.310243+0000","last_clean":"2023-09-23T21:14:56.310243+0000","last_became_active":"2023-09-23T21:12:10.757808+0000","last_became_peered":"2023-09-23T21:12:10.757808+0000","last_unstale":"2023-09-23T21:14:56.310243+0000","last_undegraded":"2023-09-23T21:14:56.310243+0000","last_fullsized":"2023-09-23T21:14:56.310243+0000","mapping_epoch":470,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:30:50.987139+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":106,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-09-23T21:14:51.805503+0000","last_change":"2023-09-23T21:13:12.216602+0000","last_active":"2023-09-23T21:14:51.805503+0000","last_peered":"2023-09-23T21:14:51.805503+0000","last_clean":"2023-09-23T21:14:51.805503+0000","last_became_active":"2023-09-23T21:13:12.216204+0000","last_became_peered":"2023-09-23T21:13:12.216204+0000","last_unstale":"2023-09-23T21:14:51.805503+0000","last_undegraded":"2023-09-23T21:14:51.805503+0000","last_fullsized":"2023-09-23T21:14:51.805503+0000","mapping_epoch":493,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:23:37.043390+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1c","version":"79'161","reported_seq":898,"reported_epoch":532,"state":"active+clean","last_fresh":"2023-09-23T21:15:02.205229+0000","last_change":"2023-09-23T21:13:13.402773+0000","last_active":"2023-09-23T21:15:02.205229+0000","last_peered":"2023-09-23T21:15:02.205229+0000","last_clean":"2023-09-23T21:15:02.205229+0000","last_became_active":"2023-09-23T21:13:13.402025+0000","last_became_peered":"2023-09-23T21:13:13.402025+0000","last_unstale":"2023-09-23T21:15:02.205229+0000","last_undegraded":"2023-09-23T21:15:02.205229+0000","last_fullsized":"2023-09-23T21:15:02.205229+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":479,"last_epoch_clean":495,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:22.040907+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:22.040907+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.013021176000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":163840,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":163840,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.17","version":"532'6005","reported_seq":10394,"reported_epoch":532,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T21:15:04.190101+0000","last_change":"2023-09-23T21:13:39.447969+0000","last_active":"2023-09-23T21:15:04.190101+0000","last_peered":"2023-09-23T21:15:04.190101+0000","last_clean":"2023-09-23T21:13:37.065694+0000","last_became_active":"2023-09-23T21:13:39.439739+0000","last_became_peered":"2023-09-23T21:13:39.439739+0000","last_unstale":"2023-09-23T21:15:04.190101+0000","last_undegraded":"2023-09-23T21:13:39.411853+0000","last_fullsized":"2023-09-23T21:13:39.404657+0000","mapping_epoch":508,"log_start":"435'1974","ondisk_log_start":"435'1974","created":438,"last_epoch_clean":487,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_clean_scrub_stamp":"2023-09-23T21:10:15.663179+0000","objects_scrubbed":0,"log_size":4031,"log_dups_size":0,"ondisk_log_size":4031,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19365888,"num_objects":296,"num_object_clones":0,"num_object_copies":888,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1109,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":296,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4640,"num_write_kb":157920,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2112,"num_bytes_recovered":116719616,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,2],"acting":[3,5],"avail_no_missing":["3","4","5"],"object_location_counts":[{"shards":"3,4,5","objects":251},{"shards":"3,5","objects":-15}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":121,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-09-23T21:14:51.826695+0000","last_change":"2023-09-23T21:12:10.779066+0000","last_active":"2023-09-23T21:14:51.826695+0000","last_peered":"2023-09-23T21:14:51.826695+0000","last_clean":"2023-09-23T21:14:51.826695+0000","last_became_active":"2023-09-23T21:12:10.778691+0000","last_became_peered":"2023-09-23T21:12:10.778691+0000","last_unstale":"2023-09-23T21:14:51.826695+0000","last_undegraded":"2023-09-23T21:14:51.826695+0000","last_fullsized":"2023-09-23T21:14:51.826695+0000","mapping_epoch":470,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:07:22.043268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,7],"acting":[2,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":199,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-09-23T21:14:51.828554+0000","last_change":"2023-09-23T21:13:12.200767+0000","last_active":"2023-09-23T21:14:51.828554+0000","last_peered":"2023-09-23T21:14:51.828554+0000","last_clean":"2023-09-23T21:14:51.828554+0000","last_became_active":"2023-09-23T21:13:12.200435+0000","last_became_peered":"2023-09-23T21:13:12.200435+0000","last_unstale":"2023-09-23T21:14:51.828554+0000","last_undegraded":"2023-09-23T21:14:51.828554+0000","last_fullsized":"2023-09-23T21:14:51.828554+0000","mapping_epoch":493,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:15:44.937886+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.1d","version":"79'161","reported_seq":685,"reported_epoch":532,"state":"active+clean","last_fresh":"2023-09-23T21:15:02.188246+0000","last_change":"2023-09-23T21:12:30.444331+0000","last_active":"2023-09-23T21:15:02.188246+0000","last_peered":"2023-09-23T21:15:02.188246+0000","last_clean":"2023-09-23T21:15:02.188246+0000","last_became_active":"2023-09-23T21:12:30.444001+0000","last_became_peered":"2023-09-23T21:12:30.444001+0000","last_unstale":"2023-09-23T21:15:02.188246+0000","last_undegraded":"2023-09-23T21:15:02.188246+0000","last_fullsized":"2023-09-23T21:15:02.188246+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":479,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:09:36.858077+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:09:36.858077+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0024987500000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"10.16","version":"532'6136","reported_seq":9649,"reported_epoch":532,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T21:15:03.251284+0000","last_change":"2023-09-23T21:13:14.424843+0000","last_active":"2023-09-23T21:15:03.251284+0000","last_peered":"2023-09-23T21:15:03.251284+0000","last_clean":"2023-09-23T21:11:45.652255+0000","last_became_active":"2023-09-23T21:13:14.414710+0000","last_became_peered":"2023-09-23T21:13:14.414710+0000","last_unstale":"2023-09-23T21:15:03.251284+0000","last_undegraded":"2023-09-23T21:13:13.211957+0000","last_fullsized":"2023-09-23T21:13:13.188859+0000","mapping_epoch":494,"log_start":"435'1655","ondisk_log_start":"435'1655","created":438,"last_epoch_clean":460,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_clean_scrub_stamp":"2023-09-23T21:10:15.663179+0000","objects_scrubbed":0,"log_size":4481,"log_dups_size":0,"ondisk_log_size":4481,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":18087936,"num_objects":276,"num_object_clones":0,"num_object_copies":828,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":663,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":276,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4730,"num_write_kb":160192,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1670,"num_bytes_recovered":109445120,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,4],"acting":[4,5],"avail_no_missing":["4","5"],"object_location_counts":[{"shards":"1,4,5","objects":33},{"shards":"4,5","objects":243}],"blocked_by":[],"up_primary":0,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":58,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-09-23T21:14:51.816831+0000","last_change":"2023-09-23T21:13:38.380574+0000","last_active":"2023-09-23T21:14:51.816831+0000","last_peered":"2023-09-23T21:14:51.816831+0000","last_clean":"2023-09-23T21:14:51.816831+0000","last_became_active":"2023-09-23T21:13:38.373252+0000","last_became_peered":"2023-09-23T21:13:38.373252+0000","last_unstale":"2023-09-23T21:14:51.816831+0000","last_undegraded":"2023-09-23T21:14:51.816831+0000","last_fullsized":"2023-09-23T21:14:51.816831+0000","mapping_epoch":507,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:58:14.788031+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":11,"num_bytes_recovered":3324,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":106,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-09-23T21:14:51.817184+0000","last_change":"2023-09-23T21:12:10.793610+0000","last_active":"2023-09-23T21:14:51.817184+0000","last_peered":"2023-09-23T21:14:51.817184+0000","last_clean":"2023-09-23T21:14:51.817184+0000","last_became_active":"2023-09-23T21:12:10.793081+0000","last_became_peered":"2023-09-23T21:12:10.793081+0000","last_unstale":"2023-09-23T21:14:51.817184+0000","last_undegraded":"2023-09-23T21:14:51.817184+0000","last_fullsized":"2023-09-23T21:14:51.817184+0000","mapping_epoch":470,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:34:36.291550+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":10,"num_pg":26,"stat_sum":{"num_bytes":520880152,"num_objects":7951,"num_object_clones":0,"num_object_copies":23853,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":18204,"num_objects_misplaced":2659,"num_objects_unfound":0,"num_objects_dirty":7951,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":139423,"num_write_kb":4309313,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":44824,"num_bytes_recovered":2696445952,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":4290191360,"data_stored":4290183216,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":123199,"ondisk_log_size":123199,"up":78,"acting":62,"num_store_stats":7},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":59,"num_bytes_recovered":8898,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":126976,"data_stored":22458,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":14322,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":31,"stat_sum":{"num_bytes":697708,"num_objects":3,"num_object_clones":0,"num_object_copies":9,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3,"num_whiteouts":0,"num_read":89,"num_read_kb":165,"num_write":44,"num_write_kb":440,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":649699,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4991,"ondisk_log_size":4991,"up":93,"acting":93,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201696,"num_pgs":69,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":661512,"kb_used_data":154368,"kb_used_omap":0,"kb_used_meta":507136,"kb_avail":93083640,"statfs":{"total":95995035648,"available":95317647360,"internally_reserved":0,"allocated":158072832,"data_stored":156529225,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":519307264},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,2],"upper_bound":16},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:14:13 2023","interfaces":[{"interface":"back","average":{"1min":2.4260000000000002,"5min":2.4260000000000002,"15min":2.4260000000000002},"min":{"1min":0.23400000000000001,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":37.207999999999998,"5min":37.207999999999998,"15min":37.207999999999998},"last":0.91500000000000004},{"interface":"front","average":{"1min":2.234,"5min":2.234,"15min":2.234},"min":{"1min":0.28599999999999998,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":37.122,"5min":37.122,"15min":37.122},"last":0.47899999999999998}]},{"osd":1,"last update":"Sat Sep 23 21:14:09 2023","interfaces":[{"interface":"back","average":{"1min":0.94299999999999995,"5min":1.2310000000000001,"15min":2.04},"min":{"1min":0.30399999999999999,"5min":0.27100000000000002,"15min":0.26700000000000002},"max":{"1min":7.5369999999999999,"5min":14.034000000000001,"15min":14.034000000000001},"last":0.32500000000000001},{"interface":"front","average":{"1min":1.0469999999999999,"5min":1.4339999999999999,"15min":1.944},"min":{"1min":0.30499999999999999,"5min":0.29199999999999998,"15min":0.26300000000000001},"max":{"1min":5.6870000000000003,"5min":11.226000000000001,"15min":11.226000000000001},"last":0.41599999999999998}]},{"osd":2,"last update":"Sat Sep 23 21:14:30 2023","interfaces":[{"interface":"back","average":{"1min":0.80900000000000005,"5min":1.0880000000000001,"15min":1.7170000000000001},"min":{"1min":0.29699999999999999,"5min":0.26400000000000001,"15min":0.26200000000000001},"max":{"1min":3.0190000000000001,"5min":6.4219999999999997,"15min":14.478999999999999},"last":3.298},{"interface":"front","average":{"1min":1.101,"5min":1.3520000000000001,"15min":1.7330000000000001},"min":{"1min":0.30399999999999999,"5min":0.26700000000000002,"15min":0.26500000000000001},"max":{"1min":5.7640000000000002,"5min":11.198,"15min":14.135},"last":0.45500000000000002}]},{"osd":3,"last update":"Sat Sep 23 21:14:58 2023","interfaces":[{"interface":"back","average":{"1min":1.7270000000000001,"5min":1.3009999999999999,"15min":2},"min":{"1min":0.30599999999999999,"5min":0.30599999999999999,"15min":0.29099999999999998},"max":{"1min":4.1619999999999999,"5min":7.5190000000000001,"15min":31.786000000000001},"last":0.438},{"interface":"front","average":{"1min":2.7410000000000001,"5min":1.4890000000000001,"15min":2.0390000000000001},"min":{"1min":0.35599999999999998,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":11.743,"5min":11.743,"15min":32.429000000000002},"last":1.246}]},{"osd":4,"last update":"Sat Sep 23 21:14:58 2023","interfaces":[{"interface":"back","average":{"1min":1.1100000000000001,"5min":1.48,"15min":1.143},"min":{"1min":0.32500000000000001,"5min":0.26200000000000001,"15min":0.248},"max":{"1min":4.2130000000000001,"5min":11.297000000000001,"15min":11.297000000000001},"last":0.49199999999999999},{"interface":"front","average":{"1min":1.085,"5min":1.24,"15min":1.105},"min":{"1min":0.29699999999999999,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":3.7850000000000001,"5min":11.093999999999999,"15min":12.752000000000001},"last":0.50800000000000001}]},{"osd":5,"last update":"Sat Sep 23 21:14:58 2023","interfaces":[{"interface":"back","average":{"1min":1.359,"5min":1.1990000000000001,"15min":1.075},"min":{"1min":0.32300000000000001,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":6.133,"5min":9.4789999999999992,"15min":9.4789999999999992},"last":0.36099999999999999},{"interface":"front","average":{"1min":0.98699999999999999,"5min":1.3360000000000001,"15min":1.1279999999999999},"min":{"1min":0.33300000000000002,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":4.181,"5min":11.257,"15min":11.257},"last":2.778}]},{"osd":6,"last update":"Sat Sep 23 21:14:09 2023","interfaces":[{"interface":"back","average":{"1min":0.79600000000000004,"5min":1.1819999999999999,"15min":1.038},"min":{"1min":0.39800000000000002,"5min":0.30099999999999999,"15min":0.28899999999999998},"max":{"1min":1.667,"5min":10.941000000000001,"15min":10.941000000000001},"last":0.53900000000000003},{"interface":"front","average":{"1min":0.99299999999999999,"5min":1.1020000000000001,"15min":1.1259999999999999},"min":{"1min":0.41699999999999998,"5min":0.33100000000000002,"15min":0.32300000000000001},"max":{"1min":2.4969999999999999,"5min":8.923,"15min":12.715},"last":0.42999999999999999}]}]},{"osd":6,"up_from":0,"seq":704374636779,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":1,"up_from":0,"seq":1503238553711,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":0,"up_from":493,"seq":2117418876952,"num_pgs":75,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1241576,"kb_used_data":1069344,"kb_used_omap":36,"kb_used_meta":172187,"kb_avail":92503576,"statfs":{"total":95995035648,"available":94723661824,"internally_reserved":0,"allocated":1095008256,"data_stored":1092516103,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":37336,"internal_metadata":176320040},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,2],"upper_bound":4},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 21:14:12 2023","interfaces":[{"interface":"back","average":{"1min":0.75600000000000001,"5min":0.75600000000000001,"15min":0.75600000000000001},"min":{"1min":0.23000000000000001,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":4.9000000000000004,"5min":4.9000000000000004,"15min":4.9000000000000004},"last":3.9020000000000001},{"interface":"front","average":{"1min":0.77100000000000002,"5min":0.77100000000000002,"15min":0.77100000000000002},"min":{"1min":0.24299999999999999,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":3.444,"5min":3.444,"15min":3.444},"last":4.1479999999999997}]},{"osd":2,"last update":"Sat Sep 23 21:14:12 2023","interfaces":[{"interface":"back","average":{"1min":0.65400000000000003,"5min":0.65400000000000003,"15min":0.65400000000000003},"min":{"1min":0.26900000000000002,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":4.8630000000000004,"5min":4.8630000000000004,"15min":4.8630000000000004},"last":1.635},{"interface":"front","average":{"1min":0.96799999999999997,"5min":0.96799999999999997,"15min":0.96799999999999997},"min":{"1min":0.29299999999999998,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":4.649,"5min":4.649,"15min":4.649},"last":4.1790000000000003}]},{"osd":3,"last update":"Sat Sep 23 21:14:12 2023","interfaces":[{"interface":"back","average":{"1min":0.86799999999999999,"5min":0.86799999999999999,"15min":0.86799999999999999},"min":{"1min":0.23400000000000001,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":4.5110000000000001,"5min":4.5110000000000001,"15min":4.5110000000000001},"last":4.2039999999999997},{"interface":"front","average":{"1min":1.1180000000000001,"5min":1.1180000000000001,"15min":1.1180000000000001},"min":{"1min":0.26200000000000001,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":3.7200000000000002,"5min":3.7200000000000002,"15min":3.7200000000000002},"last":4.8040000000000003}]},{"osd":4,"last update":"Sat Sep 23 21:14:12 2023","interfaces":[{"interface":"back","average":{"1min":1.093,"5min":1.093,"15min":1.093},"min":{"1min":0.30199999999999999,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":3.7799999999999998,"5min":3.7799999999999998,"15min":3.7799999999999998},"last":0.92100000000000004},{"interface":"front","average":{"1min":0.996,"5min":0.996,"15min":0.996},"min":{"1min":0.317,"5min":0.317,"15min":0.317},"max":{"1min":5.093,"5min":5.093,"15min":5.093},"last":3.9809999999999999}]},{"osd":5,"last update":"Sat Sep 23 21:14:12 2023","interfaces":[{"interface":"back","average":{"1min":1.236,"5min":1.236,"15min":1.236},"min":{"1min":0.33200000000000002,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":4.5640000000000001,"5min":4.5640000000000001,"15min":4.5640000000000001},"last":4.0869999999999997},{"interface":"front","average":{"1min":1.0640000000000001,"5min":1.0640000000000001,"15min":1.0640000000000001},"min":{"1min":0.33200000000000002,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":4.5599999999999996,"5min":4.5599999999999996,"15min":4.5599999999999996},"last":1.0309999999999999}]},{"osd":6,"last update":"Sat Sep 23 21:14:12 2023","interfaces":[{"interface":"back","average":{"1min":1.127,"5min":1.127,"15min":1.127},"min":{"1min":0.38100000000000001,"5min":0.38100000000000001,"15min":0.38100000000000001},"max":{"1min":6.2750000000000004,"5min":6.2750000000000004,"15min":6.2750000000000004},"last":4.0069999999999997},{"interface":"front","average":{"1min":1.3080000000000001,"5min":1.3080000000000001,"15min":1.3080000000000001},"min":{"1min":0.35599999999999998,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":5.0860000000000003,"5min":5.0860000000000003,"15min":5.0860000000000003},"last":4.1200000000000001}]},{"osd":7,"last update":"Sat Sep 23 21:14:12 2023","interfaces":[{"interface":"back","average":{"1min":1.2090000000000001,"5min":1.2090000000000001,"15min":1.2090000000000001},"min":{"1min":0.41099999999999998,"5min":0.41099999999999998,"15min":0.41099999999999998},"max":{"1min":5.1539999999999999,"5min":5.1539999999999999,"15min":5.1539999999999999},"last":1.0580000000000001},{"interface":"front","average":{"1min":1.1579999999999999,"5min":1.1579999999999999,"15min":1.1579999999999999},"min":{"1min":0.441,"5min":0.441,"15min":0.441},"max":{"1min":5.1289999999999996,"5min":5.1289999999999996,"15min":5.1289999999999996},"last":3.9390000000000001}]}]},{"osd":2,"up_from":93,"seq":399431958824,"num_pgs":69,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1238316,"kb_used_data":440740,"kb_used_omap":81,"kb_used_meta":797486,"kb_avail":92506836,"statfs":{"total":95995035648,"available":94727000064,"internally_reserved":0,"allocated":451317760,"data_stored":448820825,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":83750,"internal_metadata":816625882},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,4,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:14:15 2023","interfaces":[{"interface":"back","average":{"1min":0.83299999999999996,"5min":0.83299999999999996,"15min":0.83299999999999996},"min":{"1min":0.20899999999999999,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":3.7290000000000001,"5min":3.7290000000000001,"15min":3.7290000000000001},"last":9.9329999999999998},{"interface":"front","average":{"1min":1.665,"5min":1.665,"15min":1.665},"min":{"1min":0.23499999999999999,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":18.111999999999998,"5min":18.111999999999998,"15min":18.111999999999998},"last":5.2249999999999996}]},{"osd":1,"last update":"Sat Sep 23 21:14:15 2023","interfaces":[{"interface":"back","average":{"1min":0.95099999999999996,"5min":1.474,"15min":2.5339999999999998},"min":{"1min":0.23999999999999999,"5min":0.188,"15min":0.188},"max":{"1min":6.0499999999999998,"5min":12.484999999999999,"15min":34.396999999999998},"last":9.9890000000000008},{"interface":"front","average":{"1min":1.597,"5min":1.3129999999999999,"15min":2.782},"min":{"1min":0.29899999999999999,"5min":0.217,"15min":0.217},"max":{"1min":18.081,"5min":18.081,"15min":34.462000000000003},"last":0.26200000000000001}]},{"osd":3,"last update":"Sat Sep 23 21:14:58 2023","interfaces":[{"interface":"back","average":{"1min":2.246,"5min":1.274,"15min":2.4550000000000001},"min":{"1min":0.37,"5min":0.20599999999999999,"15min":0.20599999999999999},"max":{"1min":14.442,"5min":14.442,"15min":41.057000000000002},"last":0.68600000000000005},{"interface":"front","average":{"1min":2.633,"5min":1.4419999999999999,"15min":2.6970000000000001},"min":{"1min":0.28599999999999998,"5min":0.21299999999999999,"15min":0.21299999999999999},"max":{"1min":14.622,"5min":14.622,"15min":41.012},"last":3.2549999999999999}]},{"osd":4,"last update":"Sat Sep 23 21:14:15 2023","interfaces":[{"interface":"back","average":{"1min":0.64800000000000002,"5min":1.256,"15min":1.4219999999999999},"min":{"1min":0.34100000000000003,"5min":0.249,"15min":0.249},"max":{"1min":1.7210000000000001,"5min":10.266999999999999,"15min":10.266999999999999},"last":0.41099999999999998},{"interface":"front","average":{"1min":1.0029999999999999,"5min":1.456,"15min":1.8759999999999999},"min":{"1min":0.38300000000000001,"5min":0.29399999999999998,"15min":0.26700000000000002},"max":{"1min":3.5819999999999999,"5min":8.6780000000000008,"15min":14.039999999999999},"last":2.9049999999999998}]},{"osd":5,"last update":"Sat Sep 23 21:14:15 2023","interfaces":[{"interface":"back","average":{"1min":0.94399999999999995,"5min":1.276,"15min":1.5209999999999999},"min":{"1min":0.311,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":3.3759999999999999,"5min":9.5039999999999996,"15min":10.069000000000001},"last":0.443},{"interface":"front","average":{"1min":0.83999999999999997,"5min":1.3149999999999999,"15min":1.8120000000000001},"min":{"1min":0.41499999999999998,"5min":0.29699999999999999,"15min":0.29399999999999998},"max":{"1min":3.629,"5min":8.7270000000000003,"15min":13.98},"last":2.9849999999999999}]},{"osd":6,"last update":"Sat Sep 23 21:14:15 2023","interfaces":[{"interface":"back","average":{"1min":0.94999999999999996,"5min":1.569,"15min":2.0529999999999999},"min":{"1min":0.436,"5min":0.373,"15min":0.35699999999999998},"max":{"1min":3.665,"5min":8.7680000000000007,"15min":14.016},"last":3.0129999999999999},{"interface":"front","average":{"1min":1.2450000000000001,"5min":1.7729999999999999,"15min":2.2330000000000001},"min":{"1min":0.45700000000000002,"5min":0.38100000000000001,"15min":0.35599999999999998},"max":{"1min":3.7610000000000001,"5min":9.3469999999999995,"15min":14.068},"last":2.9540000000000002}]},{"osd":7,"last update":"Sat Sep 23 21:14:15 2023","interfaces":[{"interface":"back","average":{"1min":1.8120000000000001,"5min":1.4790000000000001,"15min":1.5840000000000001},"min":{"1min":0.433,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":18.021999999999998,"5min":18.021999999999998,"15min":18.021999999999998},"last":0.35599999999999998},{"interface":"front","average":{"1min":1.242,"5min":1.476,"15min":1.675},"min":{"1min":0.39400000000000002,"5min":0.34799999999999998,"15min":0.32200000000000001},"max":{"1min":9.6349999999999998,"5min":10.359,"15min":10.92},"last":0.46899999999999997}]}]},{"osd":3,"up_from":98,"seq":420906795297,"num_pgs":82,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1589512,"kb_used_data":719232,"kb_used_omap":7,"kb_used_meta":870200,"kb_avail":92155640,"statfs":{"total":95995035648,"available":94367375360,"internally_reserved":0,"allocated":736493568,"data_stored":734026256,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7576,"internal_metadata":891085416},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[3,0,1,2,2],"upper_bound":32},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:14:12 2023","interfaces":[{"interface":"back","average":{"1min":0.33800000000000002,"5min":0.33800000000000002,"15min":0.33800000000000002},"min":{"1min":0.20499999999999999,"5min":0.20499999999999999,"15min":0.20499999999999999},"max":{"1min":0.69899999999999995,"5min":0.69899999999999995,"15min":0.69899999999999995},"last":3.508},{"interface":"front","average":{"1min":0.66800000000000004,"5min":0.66800000000000004,"15min":0.66800000000000004},"min":{"1min":0.23300000000000001,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":4.5510000000000002,"5min":4.5510000000000002,"15min":4.5510000000000002},"last":0.55500000000000005}]},{"osd":1,"last update":"Sat Sep 23 21:14:12 2023","interfaces":[{"interface":"back","average":{"1min":1.052,"5min":1.466,"15min":2.226},"min":{"1min":0.218,"5min":0.20000000000000001,"15min":0.20000000000000001},"max":{"1min":5.5019999999999998,"5min":22.577999999999999,"15min":22.577999999999999},"last":1.2130000000000001},{"interface":"front","average":{"1min":0.60099999999999998,"5min":1.5580000000000001,"15min":2.1829999999999998},"min":{"1min":0.249,"5min":0.216,"15min":0.216},"max":{"1min":2.7519999999999998,"5min":21.617999999999999,"15min":21.617999999999999},"last":3.8439999999999999}]},{"osd":2,"last update":"Sat Sep 23 21:15:00 2023","interfaces":[{"interface":"back","average":{"1min":1.78,"5min":1.9419999999999999,"15min":2.468},"min":{"1min":0.23899999999999999,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":10.946,"5min":56.317999999999998,"15min":56.317999999999998},"last":3.2349999999999999},{"interface":"front","average":{"1min":1.554,"5min":1.978,"15min":2.5609999999999999},"min":{"1min":0.246,"5min":0.216,"15min":0.216},"max":{"1min":6.29,"5min":55.988,"15min":55.988},"last":3.2839999999999998}]},{"osd":4,"last update":"Sat Sep 23 21:15:00 2023","interfaces":[{"interface":"back","average":{"1min":2.2959999999999998,"5min":1.526,"15min":1.831},"min":{"1min":0.29899999999999999,"5min":0.29899999999999999,"15min":0.28399999999999997},"max":{"1min":10.807,"5min":21.478999999999999,"15min":21.478999999999999},"last":0.46200000000000002},{"interface":"front","average":{"1min":1.415,"5min":1.4830000000000001,"15min":1.9710000000000001},"min":{"1min":0.314,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":6.3499999999999996,"5min":22.402000000000001,"15min":22.402000000000001},"last":1.2490000000000001}]},{"osd":5,"last update":"Sat Sep 23 21:15:00 2023","interfaces":[{"interface":"back","average":{"1min":1.925,"5min":1.456,"15min":1.95},"min":{"1min":0.34599999999999997,"5min":0.311,"15min":0.311},"max":{"1min":10.882999999999999,"5min":21.562000000000001,"15min":21.562000000000001},"last":0.52000000000000002},{"interface":"front","average":{"1min":1.429,"5min":1.302,"15min":2.141},"min":{"1min":0.34799999999999998,"5min":0.31,"15min":0.31},"max":{"1min":4.2779999999999996,"5min":8.8450000000000006,"15min":14.029},"last":3.5790000000000002}]},{"osd":6,"last update":"Sat Sep 23 21:14:15 2023","interfaces":[{"interface":"back","average":{"1min":1.254,"5min":1.591,"15min":2.0510000000000002},"min":{"1min":0.38900000000000001,"5min":0.34999999999999998,"15min":0.33700000000000002},"max":{"1min":4.2380000000000004,"5min":22.446999999999999,"15min":22.446999999999999},"last":1.2829999999999999},{"interface":"front","average":{"1min":1.2450000000000001,"5min":1.746,"15min":2.278},"min":{"1min":0.44,"5min":0.39000000000000001,"15min":0.373},"max":{"1min":9.3889999999999993,"5min":12.375999999999999,"15min":13.576000000000001},"last":3.6509999999999998}]},{"osd":7,"last update":"Sat Sep 23 21:15:00 2023","interfaces":[{"interface":"back","average":{"1min":2.4289999999999998,"5min":1.8480000000000001,"15min":2.1000000000000001},"min":{"1min":0.439,"5min":0.38800000000000001,"15min":0.33900000000000002},"max":{"1min":10.914999999999999,"5min":21.518999999999998,"15min":21.518999999999998},"last":0.49399999999999999},{"interface":"front","average":{"1min":2.4220000000000002,"5min":1.903,"15min":2.3380000000000001},"min":{"1min":0.42499999999999999,"5min":0.42499999999999999,"15min":0.38900000000000001},"max":{"1min":8.2590000000000003,"5min":9.9879999999999995,"15min":13.603999999999999},"last":3.6219999999999999}]}]},{"osd":4,"up_from":31,"seq":133143986603,"num_pgs":77,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1018192,"kb_used_data":537032,"kb_used_omap":0,"kb_used_meta":480960,"kb_avail":92726960,"statfs":{"total":95995035648,"available":94952407040,"internally_reserved":0,"allocated":549920768,"data_stored":548369800,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":492503040},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:14:14 2023","interfaces":[{"interface":"back","average":{"1min":0.90000000000000002,"5min":0.90000000000000002,"15min":0.90000000000000002},"min":{"1min":0.22500000000000001,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":2.6539999999999999,"5min":2.6539999999999999,"15min":2.6539999999999999},"last":1.0940000000000001},{"interface":"front","average":{"1min":1.1339999999999999,"5min":1.1339999999999999,"15min":1.1339999999999999},"min":{"1min":0.246,"5min":0.246,"15min":0.246},"max":{"1min":10.945,"5min":10.945,"15min":10.945},"last":0.60799999999999998}]},{"osd":1,"last update":"Sat Sep 23 21:14:14 2023","interfaces":[{"interface":"back","average":{"1min":0.77300000000000002,"5min":1.0609999999999999,"15min":2.7490000000000001},"min":{"1min":0.29699999999999999,"5min":0.221,"15min":0.221},"max":{"1min":2.387,"5min":7.8609999999999998,"15min":37.960000000000001},"last":2.6280000000000001},{"interface":"front","average":{"1min":0.80600000000000005,"5min":1.1339999999999999,"15min":2.6080000000000001},"min":{"1min":0.28499999999999998,"5min":0.28499999999999998,"15min":0.27000000000000002},"max":{"1min":3.581,"5min":7.1349999999999998,"15min":37.933},"last":2.677}]},{"osd":2,"last update":"Sat Sep 23 21:14:14 2023","interfaces":[{"interface":"back","average":{"1min":0.82099999999999995,"5min":1.153,"15min":1.748},"min":{"1min":0.27400000000000002,"5min":0.247,"15min":0.247},"max":{"1min":3.476,"5min":10.464,"15min":16.928000000000001},"last":1.9910000000000001},{"interface":"front","average":{"1min":0.86099999999999999,"5min":1.3959999999999999,"15min":1.7809999999999999},"min":{"1min":0.311,"5min":0.28599999999999998,"15min":0.27800000000000002},"max":{"1min":3.5129999999999999,"5min":7.3659999999999997,"15min":14.417999999999999},"last":1.905}]},{"osd":3,"last update":"Sat Sep 23 21:14:47 2023","interfaces":[{"interface":"back","average":{"1min":1.0389999999999999,"5min":1.1990000000000001,"15min":2.056},"min":{"1min":0.27900000000000003,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":2.766,"5min":9.3279999999999994,"15min":15.01},"last":2.79},{"interface":"front","average":{"1min":1.899,"5min":1.3879999999999999,"15min":2},"min":{"1min":0.32200000000000001,"5min":0.32200000000000001,"15min":0.30199999999999999},"max":{"1min":14.305,"5min":14.305,"15min":15.004},"last":2.8220000000000001}]},{"osd":5,"last update":"Sat Sep 23 21:14:04 2023","interfaces":[{"interface":"back","average":{"1min":1.169,"5min":1.018,"15min":0.97599999999999998},"min":{"1min":0.30099999999999999,"5min":0.222,"15min":0.222},"max":{"1min":5.1609999999999996,"5min":9.4909999999999997,"15min":14.298},"last":1.964},{"interface":"front","average":{"1min":1.0189999999999999,"5min":0.95899999999999996,"15min":1.0760000000000001},"min":{"1min":0.28000000000000003,"5min":0.27200000000000002,"15min":0.255},"max":{"1min":3.0059999999999998,"5min":6.8730000000000002,"15min":9.8979999999999997},"last":2.6160000000000001}]},{"osd":6,"last update":"Sat Sep 23 21:14:14 2023","interfaces":[{"interface":"back","average":{"1min":0.747,"5min":1.0920000000000001,"15min":1.2230000000000001},"min":{"1min":0.317,"5min":0.28599999999999998,"15min":0.26800000000000002},"max":{"1min":2.8650000000000002,"5min":24.452999999999999,"15min":24.452999999999999},"last":2.048},{"interface":"front","average":{"1min":0.95099999999999996,"5min":1.036,"15min":0.95399999999999996},"min":{"1min":0.40100000000000002,"5min":0.33500000000000002,"15min":0.32500000000000001},"max":{"1min":3.6600000000000001,"5min":11.409000000000001,"15min":14.387},"last":2.0649999999999999}]},{"osd":7,"last update":"Sat Sep 23 21:14:47 2023","interfaces":[{"interface":"back","average":{"1min":1.9910000000000001,"5min":1.355,"15min":1.1299999999999999},"min":{"1min":0.38300000000000001,"5min":0.30099999999999999,"15min":0.29299999999999998},"max":{"1min":14.422000000000001,"5min":14.422000000000001,"15min":16.890999999999998},"last":1.9370000000000001},{"interface":"front","average":{"1min":1.536,"5min":1.234,"15min":1.2090000000000001},"min":{"1min":0.39200000000000002,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":7.7229999999999999,"5min":7.7229999999999999,"15min":11.170999999999999},"last":2.8759999999999999}]}]},{"osd":5,"up_from":39,"seq":167503724970,"num_pgs":61,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":782332,"kb_used_data":279348,"kb_used_omap":0,"kb_used_meta":502976,"kb_avail":92962820,"statfs":{"total":95995035648,"available":95193927680,"internally_reserved":0,"allocated":286052352,"data_stored":284517574,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":515047424},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1],"upper_bound":4},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:14:13 2023","interfaces":[{"interface":"back","average":{"1min":0.71499999999999997,"5min":0.71499999999999997,"15min":0.71499999999999997},"min":{"1min":0.251,"5min":0.251,"15min":0.251},"max":{"1min":3.3889999999999998,"5min":3.3889999999999998,"15min":3.3889999999999998},"last":0.26300000000000001},{"interface":"front","average":{"1min":0.66900000000000004,"5min":0.66900000000000004,"15min":0.66900000000000004},"min":{"1min":0.22800000000000001,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":3.1150000000000002,"5min":3.1150000000000002,"15min":3.1150000000000002},"last":0.47499999999999998}]},{"osd":1,"last update":"Sat Sep 23 21:14:08 2023","interfaces":[{"interface":"back","average":{"1min":0.65200000000000002,"5min":1.1479999999999999,"15min":1.728},"min":{"1min":0.248,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":3.109,"5min":9.4429999999999996,"15min":25.239999999999998},"last":0.45600000000000002},{"interface":"front","average":{"1min":1.468,"5min":1.681,"15min":1.784},"min":{"1min":0.34999999999999998,"5min":0.28699999999999998,"15min":0.252},"max":{"1min":11.116,"5min":11.116,"15min":25.253},"last":0.30199999999999999}]},{"osd":2,"last update":"Sat Sep 23 21:14:26 2023","interfaces":[{"interface":"back","average":{"1min":0.73099999999999998,"5min":1.401,"15min":1.7110000000000001},"min":{"1min":0.27800000000000002,"5min":0.26700000000000002,"15min":0.23499999999999999},"max":{"1min":4.0490000000000004,"5min":9.2889999999999997,"15min":12.634},"last":0.434},{"interface":"front","average":{"1min":0.78600000000000003,"5min":1.222,"15min":1.762},"min":{"1min":0.30199999999999999,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":2.524,"5min":9.7170000000000005,"15min":10.333},"last":0.495}]},{"osd":3,"last update":"Sat Sep 23 21:14:48 2023","interfaces":[{"interface":"back","average":{"1min":1.0109999999999999,"5min":1.6339999999999999,"15min":2.484},"min":{"1min":0.32400000000000001,"5min":0.311,"15min":0.29299999999999998},"max":{"1min":3.0390000000000001,"5min":14.242000000000001,"15min":61.667999999999999},"last":1.4790000000000001},{"interface":"front","average":{"1min":1.284,"5min":1.6559999999999999,"15min":2.5910000000000002},"min":{"1min":0.39100000000000001,"5min":0.34899999999999998,"15min":0.30599999999999999},"max":{"1min":6.508,"5min":12.44,"15min":62.276000000000003},"last":0.57999999999999996}]},{"osd":4,"last update":"Sat Sep 23 21:15:00 2023","interfaces":[{"interface":"back","average":{"1min":1.0920000000000001,"5min":1.5289999999999999,"15min":1.157},"min":{"1min":0.253,"5min":0.253,"15min":0.23400000000000001},"max":{"1min":4.7750000000000004,"5min":12.776,"15min":12.776},"last":0.63900000000000001},{"interface":"front","average":{"1min":1.163,"5min":1.3029999999999999,"15min":1.1539999999999999},"min":{"1min":0.35999999999999999,"5min":0.27700000000000002,"15min":0.26900000000000002},"max":{"1min":4.8200000000000003,"5min":9.7119999999999997,"15min":11.973000000000001},"last":1.595}]},{"osd":6,"last update":"Sat Sep 23 21:14:08 2023","interfaces":[{"interface":"back","average":{"1min":0.625,"5min":0.92200000000000004,"15min":1.056},"min":{"1min":0.38800000000000001,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":1.448,"5min":7.7030000000000003,"15min":11.128},"last":0.52400000000000002},{"interface":"front","average":{"1min":1.4350000000000001,"5min":1.4390000000000001,"15min":1.246},"min":{"1min":0.42099999999999999,"5min":0.32400000000000001,"15min":0.317},"max":{"1min":10.999000000000001,"5min":10.999000000000001,"15min":11.112},"last":0.34300000000000003}]},{"osd":7,"last update":"Sat Sep 23 21:14:48 2023","interfaces":[{"interface":"back","average":{"1min":0.84999999999999998,"5min":0.98499999999999999,"15min":1.8129999999999999},"min":{"1min":0.34499999999999997,"5min":0.31,"15min":0.309},"max":{"1min":3.3039999999999998,"5min":5.5659999999999998,"15min":189.405},"last":0.55300000000000005},{"interface":"front","average":{"1min":0.92000000000000004,"5min":1.472,"15min":1.9259999999999999},"min":{"1min":0.40899999999999997,"5min":0.35399999999999998,"15min":0.32500000000000001},"max":{"1min":3.3239999999999998,"5min":11.183999999999999,"15min":189.685},"last":0.56100000000000005}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8488,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1259,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5322,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":714,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":5434,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":4838,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1082589184,"data_stored":1082589184,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1123090432,"data_stored":1123090432,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":441057280,"data_stored":441057280,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":725618688,"data_stored":725614616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":515772416,"data_stored":515768344,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":261423104,"data_stored":261423104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":140640256,"data_stored":140640256,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T21:15:09.865 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:15:09.909 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:15:09.956 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:15:09.962 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:15:09.962 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:15:10.063 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:15:10.108 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:15:10.158 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:15:10.167 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:15:10.168 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:15:10.268 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:15:10.308 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:15:10.346 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:15:10.351 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:15:10.352 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:15:10.418 INFO:teuthology.orchestra.run.smithi026.stdout:{ 2023-09-23T21:15:10.418 INFO:teuthology.orchestra.run.smithi026.stdout: "size": 20, 2023-09-23T21:15:10.418 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 600, 2023-09-23T21:15:10.418 INFO:teuthology.orchestra.run.smithi026.stdout: "ops": [ 2023-09-23T21:15:10.418 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:10.418 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.46328.0:52889 10.e 10:729c6d6c:::benchmark_data_smithi026_379838_object52888:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e444)", 2023-09-23T21:15:10.418 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T21:11:08.756518+0000", 2023-09-23T21:15:10.419 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 241.65845034899999, 2023-09-23T21:15:10.419 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.165692414, 2023-09-23T21:15:10.419 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:15:10.419 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:15:10.419 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T21:15:10.419 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.46328", 2023-09-23T21:15:10.419 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/3748861181", 2023-09-23T21:15:10.419 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 52889 2023-09-23T21:15:10.419 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:10.419 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:15:10.419 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:10.419 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:15:10.419 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:11:08.756518+0000", 2023-09-23T21:15:10.419 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:15:10.420 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:10.420 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:10.420 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:15:10.420 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:11:08.756518+0000", 2023-09-23T21:15:10.420 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:15:10.420 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:10.420 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:10.420 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:15:10.420 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:11:08.756516+0000", 2023-09-23T21:15:10.420 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999981 2023-09-23T21:15:10.420 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:10.420 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:10.420 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:15:10.421 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:11:08.756547+0000", 2023-09-23T21:15:10.421 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.1328999999999998e-05 2023-09-23T21:15:10.421 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:10.421 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:10.421 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:15:10.421 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:11:08.756561+0000", 2023-09-23T21:15:10.421 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.3161e-05 2023-09-23T21:15:10.421 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:10.421 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:10.421 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:15:10.421 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:11:08.756572+0000", 2023-09-23T21:15:10.421 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1625e-05 2023-09-23T21:15:10.421 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:10.422 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:10.422 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:15:10.422 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:11:08.756904+0000", 2023-09-23T21:15:10.422 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00033169500000000002 2023-09-23T21:15:10.422 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:10.422 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:10.422 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T21:15:10.422 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:11:08.756918+0000", 2023-09-23T21:15:10.422 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.3752e-05 2023-09-23T21:15:10.422 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:10.422 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:10.422 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:15:10.422 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:11:09.908767+0000", 2023-09-23T21:15:10.423 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1518492060000001 2023-09-23T21:15:10.423 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:10.423 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:10.423 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:15:10.423 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:11:09.908906+0000", 2023-09-23T21:15:10.423 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00013930999999999999 2023-09-23T21:15:10.423 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:10.423 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:10.423 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 4,5", 2023-09-23T21:15:10.423 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:11:09.909071+0000", 2023-09-23T21:15:10.423 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00016437 2023-09-23T21:15:10.423 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:10.423 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:10.423 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:15:10.424 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:11:09.915692+0000", 2023-09-23T21:15:10.424 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0066215859999999996 2023-09-23T21:15:10.424 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:10.424 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:10.424 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:15:10.424 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:11:09.915735+0000", 2023-09-23T21:15:10.424 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.2787999999999998e-05 2023-09-23T21:15:10.424 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:10.424 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:10.424 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T21:15:10.424 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:11:09.919777+0000", 2023-09-23T21:15:10.424 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0040422000000000001 2023-09-23T21:15:10.424 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:10.425 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:10.425 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T21:15:10.425 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:11:09.919835+0000", 2023-09-23T21:15:10.425 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.7528000000000002e-05 2023-09-23T21:15:10.425 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:10.425 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:10.425 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T21:15:10.425 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:11:09.922210+0000", 2023-09-23T21:15:10.425 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0023757019999999999 2023-09-23T21:15:10.425 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:15:10.425 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:15:10.425 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:15:10.425 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:10.426 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:10.426 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.46328.0:52897 10.e 10:7a5f33c8:::benchmark_data_smithi026_379838_object52896:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e444)", 2023-09-23T21:15:10.426 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T21:11:08.756659+0000", 2023-09-23T21:15:10.426 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 241.65830876300001, 2023-09-23T21:15:10.426 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1730620599999999, 2023-09-23T21:15:10.426 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:15:10.426 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:15:10.426 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T21:15:10.426 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.46328", 2023-09-23T21:15:10.426 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/3748861181", 2023-09-23T21:15:10.426 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 52897 2023-09-23T21:15:10.426 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:10.427 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:15:10.427 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:10.427 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:15:10.427 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:11:08.756659+0000", 2023-09-23T21:15:10.427 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:15:10.427 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:10.427 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:10.427 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:15:10.427 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:11:08.756659+0000", 2023-09-23T21:15:10.427 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:15:10.427 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:10.427 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:10.427 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:15:10.427 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:11:08.756659+0000", 2023-09-23T21:15:10.428 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T21:15:10.428 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:10.428 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:10.428 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:15:10.428 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:11:08.756684+0000", 2023-09-23T21:15:10.428 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.5196999999999999e-05 2023-09-23T21:15:10.428 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:10.428 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:10.428 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:15:10.428 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:11:08.756695+0000", 2023-09-23T21:15:10.428 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0797e-05 2023-09-23T21:15:10.428 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:10.428 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:10.429 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:15:10.429 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:11:08.756704+0000", 2023-09-23T21:15:10.429 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.9449999999999999e-06 2023-09-23T21:15:10.429 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:10.429 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:10.429 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:15:10.429 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:11:08.756952+0000", 2023-09-23T21:15:10.429 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00024845200000000003 2023-09-23T21:15:10.429 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:10.429 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:10.429 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T21:15:10.429 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:11:08.756963+0000", 2023-09-23T21:15:10.430 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0265000000000001e-05 2023-09-23T21:15:10.430 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:10.430 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:10.430 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:15:10.430 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:11:09.913098+0000", 2023-09-23T21:15:10.430 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1561355529999999 2023-09-23T21:15:10.430 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:10.430 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:10.430 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:15:10.430 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:11:09.913288+0000", 2023-09-23T21:15:10.430 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00018984500000000001 2023-09-23T21:15:10.430 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:10.430 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:10.431 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 4,5", 2023-09-23T21:15:10.431 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:11:09.913591+0000", 2023-09-23T21:15:10.431 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00030295499999999999 2023-09-23T21:15:10.431 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:10.431 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:10.431 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:15:10.431 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:11:09.922304+0000", 2023-09-23T21:15:10.431 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0087127899999999998 2023-09-23T21:15:10.431 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:10.431 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:10.431 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:15:10.431 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:11:09.927322+0000", 2023-09-23T21:15:10.431 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.005018649 2023-09-23T21:15:10.432 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:10.432 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:10.432 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T21:15:10.432 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:11:09.927340+0000", 2023-09-23T21:15:10.432 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.7771e-05 2023-09-23T21:15:10.432 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:10.432 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:10.432 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T21:15:10.432 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:11:09.927401+0000", 2023-09-23T21:15:10.432 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.0587000000000001e-05 2023-09-23T21:15:10.432 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:10.432 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:10.432 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T21:15:10.432 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:11:09.929722+0000", 2023-09-23T21:15:10.433 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0023208700000000001 2023-09-23T21:15:10.433 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:15:10.433 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:15:10.433 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:15:10.433 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:10.433 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:10.433 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.46328.0:52900 10.e 10:78038d51:::benchmark_data_smithi026_379838_object52899:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e444)", 2023-09-23T21:15:10.433 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T21:11:08.756712+0000", 2023-09-23T21:15:10.433 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 241.65825604599999, 2023-09-23T21:15:10.433 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1751611849999999, 2023-09-23T21:15:10.433 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:15:10.433 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:15:10.433 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T21:15:10.434 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.46328", 2023-09-23T21:15:10.434 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/3748861181", 2023-09-23T21:15:10.434 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 52900 2023-09-23T21:15:10.434 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:10.434 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:15:10.434 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:10.434 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:15:10.434 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:11:08.756712+0000", 2023-09-23T21:15:10.434 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:15:10.434 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:10.434 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:10.434 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:15:10.434 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:11:08.756712+0000", 2023-09-23T21:15:10.435 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:15:10.435 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:10.435 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:10.435 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:15:10.435 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:11:08.756711+0000", 2023-09-23T21:15:10.435 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.999999 2023-09-23T21:15:10.435 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:10.435 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:10.435 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:15:10.435 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:11:08.756737+0000", 2023-09-23T21:15:10.435 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.5412999999999999e-05 2023-09-23T21:15:10.435 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:10.435 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:10.435 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:15:10.436 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:11:08.756747+0000", 2023-09-23T21:15:10.436 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.6460000000000005e-06 2023-09-23T21:15:10.436 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:10.436 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:10.436 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:15:10.436 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:11:08.756754+0000", 2023-09-23T21:15:10.436 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.435e-06 2023-09-23T21:15:10.436 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:10.436 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:10.436 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:15:10.436 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:11:08.756994+0000", 2023-09-23T21:15:10.436 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000239643 2023-09-23T21:15:10.436 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:10.437 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:10.437 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T21:15:10.437 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:11:08.757004+0000", 2023-09-23T21:15:10.437 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0473e-05 2023-09-23T21:15:10.437 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:10.437 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:10.437 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:15:10.437 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:11:09.915846+0000", 2023-09-23T21:15:10.437 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1588421680000001 2023-09-23T21:15:10.437 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:10.437 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:10.437 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:15:10.437 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:11:09.916007+0000", 2023-09-23T21:15:10.438 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00016045600000000001 2023-09-23T21:15:10.438 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:10.438 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:10.438 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 4,5", 2023-09-23T21:15:10.438 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:11:09.916214+0000", 2023-09-23T21:15:10.438 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000206825 2023-09-23T21:15:10.438 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:10.438 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:10.438 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:15:10.438 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:11:09.922355+0000", 2023-09-23T21:15:10.438 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0061417590000000001 2023-09-23T21:15:10.438 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:10.438 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:10.439 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T21:15:10.439 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:11:09.929733+0000", 2023-09-23T21:15:10.439 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0073777219999999998 2023-09-23T21:15:10.439 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:10.439 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:10.439 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:15:10.439 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:11:09.929790+0000", 2023-09-23T21:15:10.439 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.6552000000000003e-05 2023-09-23T21:15:10.439 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:10.439 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:10.439 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T21:15:10.439 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:11:09.929859+0000", 2023-09-23T21:15:10.439 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.9441000000000001e-05 2023-09-23T21:15:10.439 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:10.440 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:10.440 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T21:15:10.440 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:11:09.931873+0000", 2023-09-23T21:15:10.440 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0020144 2023-09-23T21:15:10.440 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:15:10.440 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:15:10.440 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:15:10.440 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:10.440 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:10.440 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.46328.0:52916 10.e 10:7de9f19e:::benchmark_data_smithi026_379838_object52915:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e444)", 2023-09-23T21:15:10.440 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T21:11:08.756761+0000", 2023-09-23T21:15:10.440 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 241.658207261, 2023-09-23T21:15:10.440 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1771335249999999, 2023-09-23T21:15:10.441 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:15:10.441 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:15:10.441 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T21:15:10.441 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.46328", 2023-09-23T21:15:10.441 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/3748861181", 2023-09-23T21:15:10.441 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 52916 2023-09-23T21:15:10.441 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:10.441 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:15:10.441 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:10.441 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:15:10.441 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:11:08.756761+0000", 2023-09-23T21:15:10.441 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:15:10.441 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:10.442 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:10.442 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:15:10.442 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:11:08.756761+0000", 2023-09-23T21:15:10.442 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:15:10.442 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:10.442 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:10.442 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:15:10.442 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:11:08.756761+0000", 2023-09-23T21:15:10.442 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T21:15:10.442 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:10.442 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:10.442 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:15:10.443 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:11:08.756785+0000", 2023-09-23T21:15:10.443 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.4335999999999999e-05 2023-09-23T21:15:10.443 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:10.443 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:10.443 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:15:10.443 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:11:08.756794+0000", 2023-09-23T21:15:10.443 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.3039999999999994e-06 2023-09-23T21:15:10.443 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:10.443 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:10.443 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:15:10.443 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:11:08.756802+0000", 2023-09-23T21:15:10.443 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.0949999999999996e-06 2023-09-23T21:15:10.443 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:10.443 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:10.444 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:15:10.444 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:11:08.757054+0000", 2023-09-23T21:15:10.444 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00025205699999999998 2023-09-23T21:15:10.444 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:10.444 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:10.444 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T21:15:10.444 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:11:08.757070+0000", 2023-09-23T21:15:10.444 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.5373999999999999e-05 2023-09-23T21:15:10.444 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:10.444 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:10.444 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:15:10.444 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:11:09.917924+0000", 2023-09-23T21:15:10.444 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.160854163 2023-09-23T21:15:10.445 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:10.445 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:10.445 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:15:10.445 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:11:09.918077+0000", 2023-09-23T21:15:10.445 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000152693 2023-09-23T21:15:10.445 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:10.445 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:10.445 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 4,5", 2023-09-23T21:15:10.445 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:11:09.918210+0000", 2023-09-23T21:15:10.445 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000133017 2023-09-23T21:15:10.445 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:10.445 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:10.445 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:15:10.445 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:11:09.922406+0000", 2023-09-23T21:15:10.446 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0041964080000000004 2023-09-23T21:15:10.446 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:10.446 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:10.446 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T21:15:10.446 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:11:09.929742+0000", 2023-09-23T21:15:10.446 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0073364249999999997 2023-09-23T21:15:10.446 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:10.446 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:10.446 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:15:10.446 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:11:09.931977+0000", 2023-09-23T21:15:10.446 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0022349309999999999 2023-09-23T21:15:10.446 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:10.446 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:10.447 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T21:15:10.447 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:11:09.932033+0000", 2023-09-23T21:15:10.447 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.5231e-05 2023-09-23T21:15:10.447 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:10.447 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:10.447 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T21:15:10.447 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:11:09.933895+0000", 2023-09-23T21:15:10.447 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.001861936 2023-09-23T21:15:10.447 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:15:10.447 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:15:10.447 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:15:10.447 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:10.447 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:10.448 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.46328.0:99980 10.18 10:1c520c55:::benchmark_data_smithi026_379838_object20023:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e494)", 2023-09-23T21:15:10.448 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T21:13:12.184008+0000", 2023-09-23T21:15:10.448 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 118.23095989399999, 2023-09-23T21:15:10.448 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.235008101, 2023-09-23T21:15:10.448 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:15:10.448 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:15:10.448 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T21:15:10.448 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.46328", 2023-09-23T21:15:10.448 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/3748861181", 2023-09-23T21:15:10.448 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 99980 2023-09-23T21:15:10.448 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:10.449 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:15:10.449 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:10.449 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:15:10.449 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:12.184008+0000", 2023-09-23T21:15:10.449 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:15:10.449 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:10.449 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:10.449 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:15:10.449 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:12.184008+0000", 2023-09-23T21:15:10.449 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:15:10.449 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:10.449 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:10.449 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:15:10.450 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:12.184007+0000", 2023-09-23T21:15:10.450 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999981 2023-09-23T21:15:10.450 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:10.450 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:10.450 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:15:10.450 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:12.184014+0000", 2023-09-23T21:15:10.450 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.8669999999999994e-06 2023-09-23T21:15:10.450 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:10.450 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:10.450 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:15:10.450 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:12.184028+0000", 2023-09-23T21:15:10.450 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.3249e-05 2023-09-23T21:15:10.450 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:10.451 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:10.451 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:15:10.451 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:12.184039+0000", 2023-09-23T21:15:10.451 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1012999999999999e-05 2023-09-23T21:15:10.451 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:10.451 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:10.451 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:15:10.451 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:12.184097+0000", 2023-09-23T21:15:10.451 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.8523000000000002e-05 2023-09-23T21:15:10.451 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:10.451 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:10.451 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T21:15:10.451 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:12.184109+0000", 2023-09-23T21:15:10.451 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1406999999999999e-05 2023-09-23T21:15:10.452 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:10.452 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:10.452 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:15:10.452 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:13.409512+0000", 2023-09-23T21:15:10.452 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.2254034819999999 2023-09-23T21:15:10.452 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:10.452 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:10.452 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:15:10.452 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:13.409674+0000", 2023-09-23T21:15:10.452 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00016224999999999999 2023-09-23T21:15:10.452 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:10.452 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:10.453 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 0,7", 2023-09-23T21:15:10.453 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:13.409827+0000", 2023-09-23T21:15:10.453 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00015218299999999999 2023-09-23T21:15:10.453 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:10.453 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:10.453 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:15:10.453 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:13.414936+0000", 2023-09-23T21:15:10.453 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0051093589999999999 2023-09-23T21:15:10.453 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:10.453 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:10.453 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T21:15:10.453 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:13.416256+0000", 2023-09-23T21:15:10.453 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0013198330000000001 2023-09-23T21:15:10.453 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:10.454 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:10.454 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:15:10.454 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:13.418817+0000", 2023-09-23T21:15:10.454 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0025614370000000002 2023-09-23T21:15:10.454 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:10.454 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:10.454 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T21:15:10.454 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:13.418856+0000", 2023-09-23T21:15:10.454 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.8642e-05 2023-09-23T21:15:10.454 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:10.454 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:10.454 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T21:15:10.455 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:13.419016+0000", 2023-09-23T21:15:10.455 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00016062800000000001 2023-09-23T21:15:10.455 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:15:10.455 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:15:10.455 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:15:10.455 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:10.455 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:10.455 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.46328.0:99982 10.8 10:17127cfc:::benchmark_data_smithi026_379838_object20025:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e494)", 2023-09-23T21:15:10.455 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T21:13:12.184057+0000", 2023-09-23T21:15:10.455 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 118.230910928, 2023-09-23T21:15:10.455 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.2347244719999999, 2023-09-23T21:15:10.455 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:15:10.455 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:15:10.456 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T21:15:10.456 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.46328", 2023-09-23T21:15:10.456 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/3748861181", 2023-09-23T21:15:10.456 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 99982 2023-09-23T21:15:10.456 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:10.456 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:15:10.456 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:10.456 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:15:10.456 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:12.184057+0000", 2023-09-23T21:15:10.456 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:15:10.456 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:10.456 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:10.457 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:15:10.457 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:12.184057+0000", 2023-09-23T21:15:10.457 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:15:10.457 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:10.457 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:10.457 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:15:10.457 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:12.184057+0000", 2023-09-23T21:15:10.457 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T21:15:10.457 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:10.457 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:10.457 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:15:10.457 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:12.184061+0000", 2023-09-23T21:15:10.457 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.8020000000000002e-06 2023-09-23T21:15:10.457 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:10.458 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:10.458 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:15:10.458 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:12.184069+0000", 2023-09-23T21:15:10.458 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.4670000000000004e-06 2023-09-23T21:15:10.458 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:10.458 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:10.458 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:15:10.458 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:12.184075+0000", 2023-09-23T21:15:10.458 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.2550000000000003e-06 2023-09-23T21:15:10.458 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:10.458 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:10.458 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:15:10.459 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:12.184113+0000", 2023-09-23T21:15:10.459 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.8047999999999999e-05 2023-09-23T21:15:10.459 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:10.459 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:10.459 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T21:15:10.459 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:12.184125+0000", 2023-09-23T21:15:10.459 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1306e-05 2023-09-23T21:15:10.459 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:10.459 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:10.459 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:15:10.459 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:13.412280+0000", 2023-09-23T21:15:10.459 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.2281551589999999 2023-09-23T21:15:10.459 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:10.460 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:10.460 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:15:10.460 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:13.412427+0000", 2023-09-23T21:15:10.460 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00014726600000000001 2023-09-23T21:15:10.460 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:10.460 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:10.460 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 0,7", 2023-09-23T21:15:10.460 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:13.412571+0000", 2023-09-23T21:15:10.460 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00014380999999999999 2023-09-23T21:15:10.460 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:10.460 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:10.460 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T21:15:10.460 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:13.416437+0000", 2023-09-23T21:15:10.460 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0038657729999999999 2023-09-23T21:15:10.461 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:10.461 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:10.461 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:15:10.461 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:13.418223+0000", 2023-09-23T21:15:10.461 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0017864020000000001 2023-09-23T21:15:10.461 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:10.461 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:10.461 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:15:10.461 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:13.418575+0000", 2023-09-23T21:15:10.461 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00035199 2023-09-23T21:15:10.461 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:10.461 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:10.462 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T21:15:10.462 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:13.418629+0000", 2023-09-23T21:15:10.462 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.3567999999999999e-05 2023-09-23T21:15:10.462 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:10.462 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:10.462 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T21:15:10.462 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:13.418782+0000", 2023-09-23T21:15:10.462 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00015318499999999999 2023-09-23T21:15:10.462 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:15:10.462 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:15:10.462 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:15:10.462 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:10.463 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:10.463 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.46328.0:99985 10.18 10:1c8eb92a:::benchmark_data_smithi026_379838_object20028:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e494)", 2023-09-23T21:15:10.463 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T21:13:12.184082+0000", 2023-09-23T21:15:10.463 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 118.23088638500001, 2023-09-23T21:15:10.463 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.2351000889999999, 2023-09-23T21:15:10.463 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:15:10.463 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:15:10.463 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T21:15:10.463 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.46328", 2023-09-23T21:15:10.463 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/3748861181", 2023-09-23T21:15:10.463 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 99985 2023-09-23T21:15:10.463 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:10.463 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:15:10.463 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:10.464 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:15:10.464 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:12.184082+0000", 2023-09-23T21:15:10.464 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:15:10.464 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:10.464 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:10.464 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:15:10.464 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:12.184082+0000", 2023-09-23T21:15:10.464 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:15:10.464 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:10.464 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:10.464 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:15:10.464 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:12.184081+0000", 2023-09-23T21:15:10.464 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T21:15:10.465 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:10.465 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:10.465 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:15:10.465 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:12.184084+0000", 2023-09-23T21:15:10.465 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.729e-06 2023-09-23T21:15:10.465 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:10.465 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:10.465 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:15:10.465 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:12.184090+0000", 2023-09-23T21:15:10.465 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.0650000000000004e-06 2023-09-23T21:15:10.465 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:10.465 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:10.465 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:15:10.466 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:12.184097+0000", 2023-09-23T21:15:10.466 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.7499999999999997e-06 2023-09-23T21:15:10.466 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:10.466 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:10.466 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:15:10.466 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:12.184134+0000", 2023-09-23T21:15:10.466 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.6532999999999997e-05 2023-09-23T21:15:10.466 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:10.466 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:10.466 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T21:15:10.466 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:12.184142+0000", 2023-09-23T21:15:10.466 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.0779999999999996e-06 2023-09-23T21:15:10.466 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:10.467 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:10.467 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:15:10.467 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:13.412825+0000", 2023-09-23T21:15:10.467 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.228683271 2023-09-23T21:15:10.467 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:10.467 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:10.467 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:15:10.467 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:13.412942+0000", 2023-09-23T21:15:10.467 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000116685 2023-09-23T21:15:10.467 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:10.467 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:10.467 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 0,7", 2023-09-23T21:15:10.467 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:13.413057+0000", 2023-09-23T21:15:10.467 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000115621 2023-09-23T21:15:10.468 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:10.468 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:10.468 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T21:15:10.468 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:13.416445+0000", 2023-09-23T21:15:10.468 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.003387349 2023-09-23T21:15:10.468 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:10.468 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:10.468 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:15:10.468 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:13.418262+0000", 2023-09-23T21:15:10.468 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.001817817 2023-09-23T21:15:10.468 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:10.468 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:10.468 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:15:10.469 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:13.419049+0000", 2023-09-23T21:15:10.469 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00078619499999999995 2023-09-23T21:15:10.469 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:10.469 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:10.469 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T21:15:10.469 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:13.419084+0000", 2023-09-23T21:15:10.469 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.5660000000000001e-05 2023-09-23T21:15:10.469 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:10.469 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:10.469 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T21:15:10.469 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:13.419182+0000", 2023-09-23T21:15:10.469 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.7774000000000003e-05 2023-09-23T21:15:10.469 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:15:10.470 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:15:10.470 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:15:10.470 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:10.470 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:10.470 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.46328.0:99988 10.3 10:c4dd81b5:::benchmark_data_smithi026_379838_object20031:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e495)", 2023-09-23T21:15:10.470 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T21:13:13.422814+0000", 2023-09-23T21:15:10.470 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 116.992154223, 2023-09-23T21:15:10.470 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.1547886850000002, 2023-09-23T21:15:10.470 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:15:10.470 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:15:10.470 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T21:15:10.470 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.46328", 2023-09-23T21:15:10.470 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/3748861181", 2023-09-23T21:15:10.471 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 99988 2023-09-23T21:15:10.471 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:10.471 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:15:10.471 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:10.471 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:15:10.471 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:13.422814+0000", 2023-09-23T21:15:10.471 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:15:10.471 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:10.471 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:10.471 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:15:10.471 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:13.422814+0000", 2023-09-23T21:15:10.471 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:15:10.471 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:10.472 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:10.472 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:15:10.472 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:13.422812+0000", 2023-09-23T21:15:10.472 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T21:15:10.472 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:10.472 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:10.472 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:15:10.472 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:13.422822+0000", 2023-09-23T21:15:10.472 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.1339999999999993e-06 2023-09-23T21:15:10.472 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:10.472 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:10.472 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:15:10.472 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:13.422840+0000", 2023-09-23T21:15:10.473 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.8675999999999999e-05 2023-09-23T21:15:10.473 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:10.473 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:10.473 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:15:10.473 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:13.422851+0000", 2023-09-23T21:15:10.473 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0678e-05 2023-09-23T21:15:10.473 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:10.473 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:10.473 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:15:10.473 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:13.422912+0000", 2023-09-23T21:15:10.473 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.0930000000000001e-05 2023-09-23T21:15:10.473 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:10.473 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:10.474 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T21:15:10.474 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:13.422925+0000", 2023-09-23T21:15:10.474 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.2845e-05 2023-09-23T21:15:10.474 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:10.474 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:10.474 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:15:10.474 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:14.984591+0000", 2023-09-23T21:15:10.474 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.561666531 2023-09-23T21:15:10.474 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:10.474 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:10.474 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for degraded object", 2023-09-23T21:15:10.474 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:14.985128+0000", 2023-09-23T21:15:10.474 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00053700600000000003 2023-09-23T21:15:10.475 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:10.475 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:10.475 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:15:10.475 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:15.563363+0000", 2023-09-23T21:15:10.475 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.57823501499999996 2023-09-23T21:15:10.475 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:10.475 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:10.475 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:15:10.475 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:15.563492+0000", 2023-09-23T21:15:10.475 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000129006 2023-09-23T21:15:10.475 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:10.475 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:10.475 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 0,4,5", 2023-09-23T21:15:10.475 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:15.563658+0000", 2023-09-23T21:15:10.476 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00016557900000000001 2023-09-23T21:15:10.476 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:10.476 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:10.476 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T21:15:10.476 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:15.565965+0000", 2023-09-23T21:15:10.476 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0023074020000000001 2023-09-23T21:15:10.476 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:10.476 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:10.476 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:15:10.476 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:15.569932+0000", 2023-09-23T21:15:10.476 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0039669060000000001 2023-09-23T21:15:10.476 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:10.476 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:10.477 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:15:10.477 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:15.569989+0000", 2023-09-23T21:15:10.477 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.7089999999999997e-05 2023-09-23T21:15:10.477 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:10.477 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:10.477 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:15:10.477 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:15.576591+0000", 2023-09-23T21:15:10.477 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0066019370000000004 2023-09-23T21:15:10.477 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:10.477 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:10.477 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T21:15:10.477 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:15.576640+0000", 2023-09-23T21:15:10.477 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.8899000000000001e-05 2023-09-23T21:15:10.477 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:10.478 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:10.478 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T21:15:10.478 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:15.577603+0000", 2023-09-23T21:15:10.478 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00096270999999999998 2023-09-23T21:15:10.478 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:15:10.478 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:15:10.478 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:15:10.478 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:10.478 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:10.478 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.46328.0:99993 10.3 10:c459259a:::benchmark_data_smithi026_379838_object20036:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e495)", 2023-09-23T21:15:10.478 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T21:13:13.433644+0000", 2023-09-23T21:15:10.478 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 116.98132429099999, 2023-09-23T21:15:10.478 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.1512665480000002, 2023-09-23T21:15:10.479 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:15:10.479 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:15:10.479 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T21:15:10.479 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.46328", 2023-09-23T21:15:10.479 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/3748861181", 2023-09-23T21:15:10.479 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 99993 2023-09-23T21:15:10.479 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:10.479 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:15:10.479 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:10.479 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:15:10.479 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:13.433644+0000", 2023-09-23T21:15:10.479 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:15:10.479 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:10.479 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:10.480 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:15:10.480 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:13.433644+0000", 2023-09-23T21:15:10.480 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:15:10.480 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:10.480 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:10.480 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:15:10.480 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:13.433642+0000", 2023-09-23T21:15:10.480 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T21:15:10.480 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:10.480 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:10.480 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:15:10.480 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:13.433650+0000", 2023-09-23T21:15:10.480 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.6260000000000003e-06 2023-09-23T21:15:10.481 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:10.481 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:10.481 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:15:10.481 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:13.433666+0000", 2023-09-23T21:15:10.481 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.5432000000000001e-05 2023-09-23T21:15:10.481 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:10.481 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:10.481 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:15:10.481 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:13.433675+0000", 2023-09-23T21:15:10.481 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.4099999999999997e-06 2023-09-23T21:15:10.481 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:10.481 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:10.481 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:15:10.481 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:13.433729+0000", 2023-09-23T21:15:10.482 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.4011999999999997e-05 2023-09-23T21:15:10.482 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:10.482 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:10.482 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T21:15:10.482 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:13.433741+0000", 2023-09-23T21:15:10.482 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.167e-05 2023-09-23T21:15:10.482 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:10.482 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:10.482 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:15:10.482 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:14.986137+0000", 2023-09-23T21:15:10.482 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.5523959469999999 2023-09-23T21:15:10.482 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:10.483 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:10.483 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for degraded object", 2023-09-23T21:15:10.483 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:14.986648+0000", 2023-09-23T21:15:10.483 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00051177999999999998 2023-09-23T21:15:10.483 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:10.483 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:10.483 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:15:10.483 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:15.567752+0000", 2023-09-23T21:15:10.483 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.58110373800000004 2023-09-23T21:15:10.483 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:10.483 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:10.483 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:15:10.483 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:15.567863+0000", 2023-09-23T21:15:10.484 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00011069 2023-09-23T21:15:10.484 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:10.484 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:10.484 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 0,4,5", 2023-09-23T21:15:10.484 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:15.567957+0000", 2023-09-23T21:15:10.484 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.4568999999999995e-05 2023-09-23T21:15:10.484 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:10.484 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:10.484 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:15:10.484 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:15.570620+0000", 2023-09-23T21:15:10.484 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0026622249999999998 2023-09-23T21:15:10.484 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:10.484 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:10.485 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:15:10.485 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:15.570706+0000", 2023-09-23T21:15:10.485 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.6131000000000006e-05 2023-09-23T21:15:10.485 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:10.485 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:10.485 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T21:15:10.485 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:15.570720+0000", 2023-09-23T21:15:10.485 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.3971999999999999e-05 2023-09-23T21:15:10.485 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:10.485 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:10.485 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:15:10.485 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:15.583861+0000", 2023-09-23T21:15:10.485 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.013141661000000001 2023-09-23T21:15:10.486 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:10.486 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:10.486 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T21:15:10.486 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:15.583922+0000", 2023-09-23T21:15:10.486 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.0577e-05 2023-09-23T21:15:10.486 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:10.486 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:10.486 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T21:15:10.486 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:15.584911+0000", 2023-09-23T21:15:10.486 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00098861799999999992 2023-09-23T21:15:10.486 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:15:10.486 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:15:10.487 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:15:10.487 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:10.487 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:10.487 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.46328.0:105700 10.a 10:5798ca2b:::benchmark_data_smithi026_379838_object25743:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e502)", 2023-09-23T21:15:10.487 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T21:13:30.752996+0000", 2023-09-23T21:15:10.487 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 99.661972614000007, 2023-09-23T21:15:10.487 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1848791160000001, 2023-09-23T21:15:10.487 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:15:10.487 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:15:10.487 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T21:15:10.487 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.46328", 2023-09-23T21:15:10.487 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/3748861181", 2023-09-23T21:15:10.487 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 105700 2023-09-23T21:15:10.488 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:10.488 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:15:10.488 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:10.488 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:15:10.488 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:30.752996+0000", 2023-09-23T21:15:10.488 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:15:10.488 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:10.488 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:10.488 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:15:10.488 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:30.752996+0000", 2023-09-23T21:15:10.488 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:15:10.488 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:10.488 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:10.489 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:15:10.489 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:30.752994+0000", 2023-09-23T21:15:10.489 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T21:15:10.489 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:10.489 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:10.489 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:15:10.489 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:30.753002+0000", 2023-09-23T21:15:10.489 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.1920000000000005e-06 2023-09-23T21:15:10.489 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:10.489 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:10.489 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:15:10.489 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:30.753020+0000", 2023-09-23T21:15:10.490 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.7448e-05 2023-09-23T21:15:10.490 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:10.490 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:10.490 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:15:10.490 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:30.753033+0000", 2023-09-23T21:15:10.490 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.2992e-05 2023-09-23T21:15:10.490 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:10.490 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:10.490 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:15:10.490 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:30.753328+0000", 2023-09-23T21:15:10.490 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00029530499999999999 2023-09-23T21:15:10.490 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:10.491 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:10.491 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op must wait for map", 2023-09-23T21:15:10.491 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:30.753338+0000", 2023-09-23T21:15:10.491 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.6660000000000003e-06 2023-09-23T21:15:10.491 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:10.491 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:10.491 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:15:10.491 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:30.755003+0000", 2023-09-23T21:15:10.491 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0016657200000000001 2023-09-23T21:15:10.491 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:10.491 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:10.491 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T21:15:10.491 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:30.755015+0000", 2023-09-23T21:15:10.492 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1168e-05 2023-09-23T21:15:10.492 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:10.492 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:10.492 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:15:10.492 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:31.924283+0000", 2023-09-23T21:15:10.492 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1692687070000001 2023-09-23T21:15:10.492 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:10.492 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:10.492 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:15:10.492 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:31.924405+0000", 2023-09-23T21:15:10.492 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00012188000000000001 2023-09-23T21:15:10.492 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:10.492 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:10.493 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 0,5", 2023-09-23T21:15:10.493 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:31.924544+0000", 2023-09-23T21:15:10.493 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000139192 2023-09-23T21:15:10.493 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:10.493 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:10.493 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T21:15:10.493 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:31.931263+0000", 2023-09-23T21:15:10.493 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.006718356 2023-09-23T21:15:10.493 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:10.493 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:10.493 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:15:10.493 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:31.932044+0000", 2023-09-23T21:15:10.493 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00078127999999999997 2023-09-23T21:15:10.494 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:10.494 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:10.494 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:15:10.494 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:31.936878+0000", 2023-09-23T21:15:10.494 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0048337440000000001 2023-09-23T21:15:10.494 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:10.494 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:10.494 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T21:15:10.494 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:31.936948+0000", 2023-09-23T21:15:10.494 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.0578000000000001e-05 2023-09-23T21:15:10.494 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:10.494 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:10.495 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T21:15:10.495 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:31.937875+0000", 2023-09-23T21:15:10.495 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00092649099999999999 2023-09-23T21:15:10.495 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:15:10.495 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:15:10.495 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:15:10.495 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:10.495 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:10.495 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.46328.0:105702 10.a 10:5a761d64:::benchmark_data_smithi026_379838_object25745:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e502)", 2023-09-23T21:15:10.495 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T21:13:30.753049+0000", 2023-09-23T21:15:10.495 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 99.661919690999994, 2023-09-23T21:15:10.495 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1857564570000001, 2023-09-23T21:15:10.495 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:15:10.496 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:15:10.496 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T21:15:10.496 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.46328", 2023-09-23T21:15:10.496 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/3748861181", 2023-09-23T21:15:10.496 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 105702 2023-09-23T21:15:10.496 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:10.496 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:15:10.496 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:10.496 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:15:10.496 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:30.753049+0000", 2023-09-23T21:15:10.496 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:15:10.496 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:10.496 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:10.497 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:15:10.497 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:30.753049+0000", 2023-09-23T21:15:10.497 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:15:10.497 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:10.497 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:10.497 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:15:10.497 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:30.753048+0000", 2023-09-23T21:15:10.497 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T21:15:10.497 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:10.497 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:10.497 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:15:10.497 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:30.753051+0000", 2023-09-23T21:15:10.497 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.067e-06 2023-09-23T21:15:10.498 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:10.498 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:10.498 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:15:10.498 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:30.753058+0000", 2023-09-23T21:15:10.498 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.2300000000000002e-06 2023-09-23T21:15:10.498 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:10.498 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:10.498 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:15:10.498 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:30.753065+0000", 2023-09-23T21:15:10.498 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.6069999999999996e-06 2023-09-23T21:15:10.498 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:10.498 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:10.498 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:15:10.499 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:30.753397+0000", 2023-09-23T21:15:10.499 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000331918 2023-09-23T21:15:10.499 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:10.499 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:10.499 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting_for_map not empty", 2023-09-23T21:15:10.499 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:30.753404+0000", 2023-09-23T21:15:10.499 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.5629999999999997e-06 2023-09-23T21:15:10.499 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:10.499 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:10.499 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:15:10.499 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:30.755041+0000", 2023-09-23T21:15:10.499 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.001637325 2023-09-23T21:15:10.499 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:10.500 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:10.500 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T21:15:10.500 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:30.755049+0000", 2023-09-23T21:15:10.500 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.6179999999999995e-06 2023-09-23T21:15:10.500 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:10.500 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:10.500 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:15:10.500 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:31.928564+0000", 2023-09-23T21:15:10.500 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.173514124 2023-09-23T21:15:10.500 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:10.500 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:10.500 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:15:10.500 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:31.928730+0000", 2023-09-23T21:15:10.501 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00016596100000000001 2023-09-23T21:15:10.501 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:10.501 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:10.501 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 0,5", 2023-09-23T21:15:10.501 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:31.928902+0000", 2023-09-23T21:15:10.501 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00017216000000000001 2023-09-23T21:15:10.501 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:10.501 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:10.501 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T21:15:10.501 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:31.931483+0000", 2023-09-23T21:15:10.501 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0025809930000000002 2023-09-23T21:15:10.501 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:10.501 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:10.501 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:15:10.502 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:31.932184+0000", 2023-09-23T21:15:10.502 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00070106300000000003 2023-09-23T21:15:10.502 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:10.502 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:10.502 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:15:10.502 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:31.937936+0000", 2023-09-23T21:15:10.502 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0057521259999999998 2023-09-23T21:15:10.502 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:10.502 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:10.502 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T21:15:10.502 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:31.937991+0000", 2023-09-23T21:15:10.502 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.5009999999999997e-05 2023-09-23T21:15:10.502 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:10.503 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:10.503 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T21:15:10.503 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:31.938805+0000", 2023-09-23T21:15:10.503 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00081414599999999997 2023-09-23T21:15:10.503 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:15:10.503 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:15:10.503 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:15:10.503 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:10.503 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:10.503 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.46328.0:107390 10.3 10:c424f3a1:::benchmark_data_smithi026_379838_object27433:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e512)", 2023-09-23T21:15:10.503 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T21:13:43.521296+0000", 2023-09-23T21:15:10.503 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 86.893672635000001, 2023-09-23T21:15:10.504 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.9701002459999999, 2023-09-23T21:15:10.504 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:15:10.504 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:15:10.504 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T21:15:10.504 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.46328", 2023-09-23T21:15:10.504 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/3748861181", 2023-09-23T21:15:10.504 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 107390 2023-09-23T21:15:10.504 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:10.504 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:15:10.504 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:10.504 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:15:10.504 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:43.521296+0000", 2023-09-23T21:15:10.504 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:15:10.505 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:10.505 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:10.505 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:15:10.505 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:43.521296+0000", 2023-09-23T21:15:10.505 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:15:10.505 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:10.505 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:10.505 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:15:10.505 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:43.521294+0000", 2023-09-23T21:15:10.505 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T21:15:10.505 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:10.505 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:10.506 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:15:10.506 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:43.521301+0000", 2023-09-23T21:15:10.506 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.165e-06 2023-09-23T21:15:10.506 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:10.506 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:10.506 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:15:10.506 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:43.521317+0000", 2023-09-23T21:15:10.506 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.5716e-05 2023-09-23T21:15:10.506 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:10.506 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:10.506 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:15:10.506 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:43.521325+0000", 2023-09-23T21:15:10.506 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.7200000000000006e-06 2023-09-23T21:15:10.507 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:10.507 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:10.507 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:15:10.507 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:43.521392+0000", 2023-09-23T21:15:10.507 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.7413e-05 2023-09-23T21:15:10.507 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:10.507 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:10.507 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T21:15:10.507 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:43.521406+0000", 2023-09-23T21:15:10.507 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.3603e-05 2023-09-23T21:15:10.507 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:10.507 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:10.508 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:15:10.508 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:44.398597+0000", 2023-09-23T21:15:10.508 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.87719106599999996 2023-09-23T21:15:10.508 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:10.508 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:10.508 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T21:15:10.508 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:44.398610+0000", 2023-09-23T21:15:10.508 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.3179e-05 2023-09-23T21:15:10.508 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:10.508 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:10.508 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:15:10.508 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:45.478528+0000", 2023-09-23T21:15:10.508 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.079918468 2023-09-23T21:15:10.509 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:10.509 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:10.509 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:15:10.509 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:45.478686+0000", 2023-09-23T21:15:10.509 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00015774499999999999 2023-09-23T21:15:10.509 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:10.509 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:10.509 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 3,5", 2023-09-23T21:15:10.509 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:45.479025+0000", 2023-09-23T21:15:10.509 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00033855499999999999 2023-09-23T21:15:10.509 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:10.509 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:10.509 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:15:10.509 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:45.486024+0000", 2023-09-23T21:15:10.510 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0069990310000000002 2023-09-23T21:15:10.510 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:10.510 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:10.510 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:15:10.510 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:45.486107+0000", 2023-09-23T21:15:10.510 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.3211e-05 2023-09-23T21:15:10.510 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:10.510 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:10.510 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T21:15:10.510 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:45.490919+0000", 2023-09-23T21:15:10.510 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0048123819999999996 2023-09-23T21:15:10.510 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:10.510 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:10.511 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T21:15:10.511 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:45.490972+0000", 2023-09-23T21:15:10.511 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.2976000000000002e-05 2023-09-23T21:15:10.511 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:10.511 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:10.511 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T21:15:10.511 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:45.491396+0000", 2023-09-23T21:15:10.511 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00042355300000000002 2023-09-23T21:15:10.511 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:15:10.511 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:15:10.511 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:15:10.511 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:10.511 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:10.512 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.46328.0:107402 10.3 10:c088a5ec:::benchmark_data_smithi026_379838_object27445:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e512)", 2023-09-23T21:15:10.512 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T21:13:43.551185+0000", 2023-09-23T21:15:10.512 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 86.863783323999996, 2023-09-23T21:15:10.512 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.944933936, 2023-09-23T21:15:10.512 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:15:10.512 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:15:10.512 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T21:15:10.512 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.46328", 2023-09-23T21:15:10.512 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/3748861181", 2023-09-23T21:15:10.512 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 107402 2023-09-23T21:15:10.512 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:10.512 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:15:10.512 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:10.512 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:15:10.513 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:43.551185+0000", 2023-09-23T21:15:10.513 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:15:10.513 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:10.513 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:10.513 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:15:10.513 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:43.551185+0000", 2023-09-23T21:15:10.513 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:15:10.513 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:10.513 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:10.513 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:15:10.513 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:43.551184+0000", 2023-09-23T21:15:10.513 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T21:15:10.513 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:10.514 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:10.514 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:15:10.514 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:43.551191+0000", 2023-09-23T21:15:10.514 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.0249999999999997e-06 2023-09-23T21:15:10.514 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:10.514 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:10.514 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:15:10.514 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:43.551205+0000", 2023-09-23T21:15:10.514 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.3896e-05 2023-09-23T21:15:10.514 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:10.514 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:10.514 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:15:10.514 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:43.551213+0000", 2023-09-23T21:15:10.514 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.6510000000000007e-06 2023-09-23T21:15:10.515 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:10.515 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:10.515 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:15:10.515 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:43.551270+0000", 2023-09-23T21:15:10.515 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.6585000000000001e-05 2023-09-23T21:15:10.515 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:10.515 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:10.515 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T21:15:10.515 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:43.551278+0000", 2023-09-23T21:15:10.515 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.8660000000000006e-06 2023-09-23T21:15:10.515 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:10.515 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:10.515 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:15:10.516 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:44.398634+0000", 2023-09-23T21:15:10.516 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.84735668099999994 2023-09-23T21:15:10.516 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:10.516 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:10.516 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T21:15:10.516 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:44.398643+0000", 2023-09-23T21:15:10.516 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.3939999999999996e-06 2023-09-23T21:15:10.516 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:10.516 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:10.516 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:15:10.516 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:45.481093+0000", 2023-09-23T21:15:10.516 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0824503219999999 2023-09-23T21:15:10.516 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:10.517 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:10.517 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:15:10.517 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:45.481263+0000", 2023-09-23T21:15:10.517 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00017003600000000001 2023-09-23T21:15:10.517 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:10.517 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:10.517 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 3,5", 2023-09-23T21:15:10.517 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:45.481658+0000", 2023-09-23T21:15:10.517 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00039501100000000002 2023-09-23T21:15:10.517 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:10.517 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:10.517 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:15:10.517 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:45.486065+0000", 2023-09-23T21:15:10.517 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0044072599999999997 2023-09-23T21:15:10.518 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:10.518 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:10.518 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:15:10.518 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:45.486194+0000", 2023-09-23T21:15:10.518 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000128465 2023-09-23T21:15:10.518 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:10.518 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:10.518 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T21:15:10.518 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:45.495610+0000", 2023-09-23T21:15:10.518 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0094165020000000002 2023-09-23T21:15:10.518 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:10.518 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:10.519 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T21:15:10.519 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:45.495669+0000", 2023-09-23T21:15:10.519 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.8387999999999997e-05 2023-09-23T21:15:10.519 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:10.519 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:10.519 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T21:15:10.519 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:45.496119+0000", 2023-09-23T21:15:10.519 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00045012600000000002 2023-09-23T21:15:10.519 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:15:10.519 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:15:10.519 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:15:10.519 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:10.519 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:10.520 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.46328.0:107405 10.3 10:c57b82fd:::benchmark_data_smithi026_379838_object27448:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e512)", 2023-09-23T21:15:10.520 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T21:13:43.555000+0000", 2023-09-23T21:15:10.520 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 86.859968660000007, 2023-09-23T21:15:10.520 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.941392612, 2023-09-23T21:15:10.520 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:15:10.520 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:15:10.520 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T21:15:10.520 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.46328", 2023-09-23T21:15:10.520 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/3748861181", 2023-09-23T21:15:10.520 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 107405 2023-09-23T21:15:10.520 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:10.520 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:15:10.520 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:10.521 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:15:10.521 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:43.555000+0000", 2023-09-23T21:15:10.521 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:15:10.521 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:10.521 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:10.521 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:15:10.521 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:43.555000+0000", 2023-09-23T21:15:10.521 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:15:10.521 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:10.521 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:10.521 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:15:10.521 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:43.554998+0000", 2023-09-23T21:15:10.522 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T21:15:10.522 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:10.522 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:10.522 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:15:10.522 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:43.555005+0000", 2023-09-23T21:15:10.522 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.8789999999999997e-06 2023-09-23T21:15:10.522 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:10.522 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:10.522 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:15:10.522 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:43.555023+0000", 2023-09-23T21:15:10.522 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.7788e-05 2023-09-23T21:15:10.522 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:10.522 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:10.523 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:15:10.523 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:43.555033+0000", 2023-09-23T21:15:10.523 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.8479999999999996e-06 2023-09-23T21:15:10.523 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:10.523 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:10.523 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:15:10.523 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:43.555103+0000", 2023-09-23T21:15:10.523 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.9715999999999995e-05 2023-09-23T21:15:10.523 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:10.523 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:10.523 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T21:15:10.523 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:43.555117+0000", 2023-09-23T21:15:10.524 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.4544e-05 2023-09-23T21:15:10.524 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:10.524 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:10.524 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:15:10.524 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:44.398665+0000", 2023-09-23T21:15:10.524 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.84354806699999996 2023-09-23T21:15:10.524 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:10.524 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:10.524 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T21:15:10.524 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:44.398673+0000", 2023-09-23T21:15:10.524 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.9310000000000008e-06 2023-09-23T21:15:10.524 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:10.524 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:10.525 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:15:10.525 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:45.483663+0000", 2023-09-23T21:15:10.525 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0849898170000001 2023-09-23T21:15:10.525 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:10.525 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:10.525 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:15:10.525 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:45.483846+0000", 2023-09-23T21:15:10.525 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000182592 2023-09-23T21:15:10.525 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:10.525 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:10.525 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 3,5", 2023-09-23T21:15:10.525 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:45.484262+0000", 2023-09-23T21:15:10.526 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00041659599999999998 2023-09-23T21:15:10.526 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:10.526 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:10.526 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:15:10.526 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:45.486149+0000", 2023-09-23T21:15:10.526 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.001887262 2023-09-23T21:15:10.526 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:10.526 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:10.526 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T21:15:10.526 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:45.496125+0000", 2023-09-23T21:15:10.526 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0099756380000000002 2023-09-23T21:15:10.526 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:10.527 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:10.527 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:15:10.527 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:45.496208+0000", 2023-09-23T21:15:10.527 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.2627999999999996e-05 2023-09-23T21:15:10.527 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:10.527 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:10.527 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T21:15:10.527 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:45.496235+0000", 2023-09-23T21:15:10.527 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.7491000000000002e-05 2023-09-23T21:15:10.527 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:10.527 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:10.527 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T21:15:10.527 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:45.496392+0000", 2023-09-23T21:15:10.528 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00015702700000000001 2023-09-23T21:15:10.528 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:15:10.528 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:15:10.528 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:15:10.528 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:10.528 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:10.528 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.46328.0:107407 10.3 10:c7c573cd:::benchmark_data_smithi026_379838_object27450:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e512)", 2023-09-23T21:15:10.528 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T21:13:43.558612+0000", 2023-09-23T21:15:10.528 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 86.856355827000002, 2023-09-23T21:15:10.528 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.938076554, 2023-09-23T21:15:10.528 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:15:10.528 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:15:10.528 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T21:15:10.529 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.46328", 2023-09-23T21:15:10.529 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/3748861181", 2023-09-23T21:15:10.529 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 107407 2023-09-23T21:15:10.529 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:10.529 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:15:10.529 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:10.529 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:15:10.529 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:43.558612+0000", 2023-09-23T21:15:10.529 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:15:10.529 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:10.529 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:10.529 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:15:10.529 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:43.558612+0000", 2023-09-23T21:15:10.529 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:15:10.530 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:10.530 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:10.530 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:15:10.530 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:43.558611+0000", 2023-09-23T21:15:10.530 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.999999 2023-09-23T21:15:10.530 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:10.530 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:10.530 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:15:10.530 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:43.558617+0000", 2023-09-23T21:15:10.530 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.7699999999999998e-06 2023-09-23T21:15:10.530 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:10.530 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:10.531 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:15:10.531 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:43.558629+0000", 2023-09-23T21:15:10.531 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1389000000000001e-05 2023-09-23T21:15:10.531 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:10.531 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:10.531 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:15:10.531 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:43.558634+0000", 2023-09-23T21:15:10.531 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.7780000000000004e-06 2023-09-23T21:15:10.531 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:10.531 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:10.531 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:15:10.531 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:43.558677+0000", 2023-09-23T21:15:10.531 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.2567000000000001e-05 2023-09-23T21:15:10.531 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:10.532 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:10.532 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T21:15:10.532 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:43.558685+0000", 2023-09-23T21:15:10.532 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.3529999999999995e-06 2023-09-23T21:15:10.532 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:10.532 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:10.532 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:15:10.532 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:44.398695+0000", 2023-09-23T21:15:10.532 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.84000955300000002 2023-09-23T21:15:10.532 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:10.532 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:10.532 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T21:15:10.532 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:44.398703+0000", 2023-09-23T21:15:10.533 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.8250000000000005e-06 2023-09-23T21:15:10.533 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:10.533 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:10.533 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:15:10.533 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:45.486224+0000", 2023-09-23T21:15:10.533 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0875214470000001 2023-09-23T21:15:10.533 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:10.533 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:10.533 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:15:10.533 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:45.486405+0000", 2023-09-23T21:15:10.533 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000181201 2023-09-23T21:15:10.533 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:10.533 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:10.533 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 3,5", 2023-09-23T21:15:10.534 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:45.486850+0000", 2023-09-23T21:15:10.534 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00044453100000000002 2023-09-23T21:15:10.534 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:10.534 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:10.534 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T21:15:10.534 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:45.496129+0000", 2023-09-23T21:15:10.534 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0092789220000000002 2023-09-23T21:15:10.534 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:10.534 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:10.534 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:15:10.534 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:45.496156+0000", 2023-09-23T21:15:10.534 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.7268999999999999e-05 2023-09-23T21:15:10.535 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:10.535 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:10.535 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:15:10.535 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:45.496446+0000", 2023-09-23T21:15:10.535 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00028957199999999998 2023-09-23T21:15:10.535 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:10.535 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:10.535 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T21:15:10.535 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:45.496471+0000", 2023-09-23T21:15:10.535 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.5094999999999999e-05 2023-09-23T21:15:10.535 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:10.535 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:10.535 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T21:15:10.535 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:45.496689+0000", 2023-09-23T21:15:10.536 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00021824100000000001 2023-09-23T21:15:10.536 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:15:10.536 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:15:10.536 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:15:10.536 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:10.536 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:10.536 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.46328.0:107416 10.3 10:c1a268c0:::benchmark_data_smithi026_379838_object27459:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e512)", 2023-09-23T21:15:10.536 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T21:13:43.621120+0000", 2023-09-23T21:15:10.536 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 86.793848643000004, 2023-09-23T21:15:10.536 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.8866092810000001, 2023-09-23T21:15:10.536 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:15:10.536 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:15:10.536 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T21:15:10.537 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.46328", 2023-09-23T21:15:10.537 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/3748861181", 2023-09-23T21:15:10.537 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 107416 2023-09-23T21:15:10.537 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:10.537 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:15:10.537 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:10.537 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:15:10.537 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:43.621120+0000", 2023-09-23T21:15:10.537 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:15:10.537 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:10.537 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:10.537 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:15:10.537 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:43.621120+0000", 2023-09-23T21:15:10.537 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:15:10.538 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:10.538 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:10.538 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:15:10.538 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:43.621118+0000", 2023-09-23T21:15:10.538 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.999999 2023-09-23T21:15:10.538 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:10.538 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:10.538 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:15:10.538 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:43.621124+0000", 2023-09-23T21:15:10.538 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.2669999999999997e-06 2023-09-23T21:15:10.538 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:10.538 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:10.538 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:15:10.539 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:43.621132+0000", 2023-09-23T21:15:10.539 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.6100000000000006e-06 2023-09-23T21:15:10.539 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:10.539 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:10.539 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:15:10.539 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:43.621138+0000", 2023-09-23T21:15:10.539 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.5029999999999997e-06 2023-09-23T21:15:10.539 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:10.539 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:10.539 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:15:10.539 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:43.621190+0000", 2023-09-23T21:15:10.539 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.2049999999999998e-05 2023-09-23T21:15:10.539 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:10.540 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:10.540 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T21:15:10.540 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:43.621200+0000", 2023-09-23T21:15:10.540 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0128e-05 2023-09-23T21:15:10.540 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:10.540 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:10.540 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:15:10.540 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:44.398724+0000", 2023-09-23T21:15:10.540 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.77752439900000003 2023-09-23T21:15:10.540 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:10.540 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:10.540 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T21:15:10.540 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:44.398732+0000", 2023-09-23T21:15:10.540 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.7670000000000003e-06 2023-09-23T21:15:10.541 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:10.541 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:10.541 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:15:10.541 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:45.488513+0000", 2023-09-23T21:15:10.541 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.089780795 2023-09-23T21:15:10.541 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:10.541 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:10.541 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:15:10.541 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:45.488683+0000", 2023-09-23T21:15:10.541 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000169753 2023-09-23T21:15:10.541 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:10.541 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:10.541 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 3,5", 2023-09-23T21:15:10.542 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:45.489075+0000", 2023-09-23T21:15:10.542 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000392724 2023-09-23T21:15:10.542 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:10.542 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:10.542 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T21:15:10.542 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:45.496161+0000", 2023-09-23T21:15:10.542 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0070855179999999999 2023-09-23T21:15:10.542 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:10.542 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:10.542 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:15:10.542 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:45.496188+0000", 2023-09-23T21:15:10.542 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.7334e-05 2023-09-23T21:15:10.542 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:10.543 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:10.543 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:15:10.543 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:45.507169+0000", 2023-09-23T21:15:10.543 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.010980978000000001 2023-09-23T21:15:10.543 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:10.543 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:10.543 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T21:15:10.543 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:45.507226+0000", 2023-09-23T21:15:10.543 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.6758000000000001e-05 2023-09-23T21:15:10.543 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:10.543 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:10.543 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T21:15:10.543 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:45.507729+0000", 2023-09-23T21:15:10.544 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00050283700000000001 2023-09-23T21:15:10.544 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:15:10.544 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:15:10.544 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:15:10.544 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:10.544 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:10.544 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.46328.0:107486 10.3 10:c6bc522c:::benchmark_data_smithi026_379838_object27529:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e512)", 2023-09-23T21:15:10.544 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T21:13:43.853389+0000", 2023-09-23T21:15:10.544 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 86.561579472000005, 2023-09-23T21:15:10.544 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.6556707799999999, 2023-09-23T21:15:10.544 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:15:10.544 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:15:10.544 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T21:15:10.544 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.46328", 2023-09-23T21:15:10.545 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/3748861181", 2023-09-23T21:15:10.545 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 107486 2023-09-23T21:15:10.545 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:10.545 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:15:10.545 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:10.545 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:15:10.545 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:43.853389+0000", 2023-09-23T21:15:10.545 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:15:10.545 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:10.545 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:10.545 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:15:10.545 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:43.853389+0000", 2023-09-23T21:15:10.545 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:15:10.545 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:10.546 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:10.546 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:15:10.546 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:43.853387+0000", 2023-09-23T21:15:10.546 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T21:15:10.546 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:10.546 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:10.546 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:15:10.546 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:43.853393+0000", 2023-09-23T21:15:10.546 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.0070000000000003e-06 2023-09-23T21:15:10.546 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:10.546 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:10.546 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:15:10.546 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:43.853406+0000", 2023-09-23T21:15:10.547 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.2252999999999999e-05 2023-09-23T21:15:10.547 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:10.547 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:10.547 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:15:10.547 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:43.853412+0000", 2023-09-23T21:15:10.547 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.8660000000000002e-06 2023-09-23T21:15:10.547 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:10.547 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:10.547 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:15:10.547 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:43.853468+0000", 2023-09-23T21:15:10.547 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.6023999999999998e-05 2023-09-23T21:15:10.547 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:10.547 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:10.547 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T21:15:10.548 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:43.853493+0000", 2023-09-23T21:15:10.548 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.5383000000000001e-05 2023-09-23T21:15:10.548 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:10.548 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:10.548 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:15:10.548 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:44.398754+0000", 2023-09-23T21:15:10.548 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.54526147999999997 2023-09-23T21:15:10.548 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:10.548 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:10.548 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T21:15:10.548 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:44.398763+0000", 2023-09-23T21:15:10.548 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.0310000000000004e-06 2023-09-23T21:15:10.549 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:10.549 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:10.549 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:15:10.549 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:45.491466+0000", 2023-09-23T21:15:10.549 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.092702219 2023-09-23T21:15:10.549 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:10.549 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:10.549 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:15:10.549 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:45.491677+0000", 2023-09-23T21:15:10.549 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00021079599999999999 2023-09-23T21:15:10.549 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:10.549 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:10.549 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 3,5", 2023-09-23T21:15:10.549 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:45.492053+0000", 2023-09-23T21:15:10.550 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00037604499999999998 2023-09-23T21:15:10.550 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:10.550 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:10.550 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T21:15:10.550 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:45.496165+0000", 2023-09-23T21:15:10.550 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0041124170000000002 2023-09-23T21:15:10.550 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:10.550 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:10.550 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:15:10.550 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:45.496424+0000", 2023-09-23T21:15:10.550 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00025910199999999999 2023-09-23T21:15:10.550 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:10.550 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:10.551 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:15:10.551 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:45.508604+0000", 2023-09-23T21:15:10.551 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.012179730999999999 2023-09-23T21:15:10.551 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:10.551 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:10.551 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T21:15:10.551 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:45.508658+0000", 2023-09-23T21:15:10.551 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.4474000000000001e-05 2023-09-23T21:15:10.551 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:10.551 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:10.551 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T21:15:10.551 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:45.509060+0000", 2023-09-23T21:15:10.551 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00040129100000000002 2023-09-23T21:15:10.551 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:15:10.552 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:15:10.552 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:15:10.552 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:10.552 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:10.552 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.46328.0:107514 10.3 10:c2397a05:::benchmark_data_smithi026_379838_object27557:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e512)", 2023-09-23T21:15:10.552 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T21:13:44.024846+0000", 2023-09-23T21:15:10.552 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 86.390122749, 2023-09-23T21:15:10.552 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.4880499569999999, 2023-09-23T21:15:10.552 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:15:10.552 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:15:10.552 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T21:15:10.552 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.46328", 2023-09-23T21:15:10.552 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/3748861181", 2023-09-23T21:15:10.553 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 107514 2023-09-23T21:15:10.553 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:10.553 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:15:10.553 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:10.553 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:15:10.553 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:44.024846+0000", 2023-09-23T21:15:10.553 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:15:10.553 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:10.553 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:10.553 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:15:10.553 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:44.024846+0000", 2023-09-23T21:15:10.553 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:15:10.553 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:10.553 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:10.554 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:15:10.554 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:44.024843+0000", 2023-09-23T21:15:10.554 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999976 2023-09-23T21:15:10.554 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:10.554 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:10.554 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:15:10.554 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:44.024856+0000", 2023-09-23T21:15:10.554 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.3232e-05 2023-09-23T21:15:10.554 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:10.554 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:10.554 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:15:10.554 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:44.024882+0000", 2023-09-23T21:15:10.554 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.5420999999999999e-05 2023-09-23T21:15:10.555 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:10.555 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:10.555 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:15:10.555 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:44.024896+0000", 2023-09-23T21:15:10.555 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.4372999999999999e-05 2023-09-23T21:15:10.555 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:10.555 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:10.555 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:15:10.555 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:44.024951+0000", 2023-09-23T21:15:10.555 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.5145999999999997e-05 2023-09-23T21:15:10.555 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:10.555 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:10.555 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T21:15:10.556 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:44.024963+0000", 2023-09-23T21:15:10.556 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1554e-05 2023-09-23T21:15:10.556 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:10.556 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:10.556 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:15:10.556 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:44.398786+0000", 2023-09-23T21:15:10.556 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.37382335700000002 2023-09-23T21:15:10.556 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:10.556 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:10.556 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T21:15:10.556 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:44.398796+0000", 2023-09-23T21:15:10.556 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0111e-05 2023-09-23T21:15:10.556 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:10.557 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:10.557 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:15:10.557 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:45.493494+0000", 2023-09-23T21:15:10.557 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.094697917 2023-09-23T21:15:10.557 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:10.557 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:10.557 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:15:10.557 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:45.493604+0000", 2023-09-23T21:15:10.557 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000109504 2023-09-23T21:15:10.557 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:10.557 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:10.557 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 3,5", 2023-09-23T21:15:10.557 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:45.493891+0000", 2023-09-23T21:15:10.558 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00028718100000000001 2023-09-23T21:15:10.558 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:10.558 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:10.558 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T21:15:10.558 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:45.496169+0000", 2023-09-23T21:15:10.558 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0022776279999999999 2023-09-23T21:15:10.558 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:10.558 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:10.558 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:15:10.558 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:45.503605+0000", 2023-09-23T21:15:10.558 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.007436425 2023-09-23T21:15:10.558 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:10.558 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:10.558 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:15:10.559 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:45.512378+0000", 2023-09-23T21:15:10.559 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.008773444 2023-09-23T21:15:10.559 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:10.559 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:10.559 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T21:15:10.559 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:45.512437+0000", 2023-09-23T21:15:10.559 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.8684000000000003e-05 2023-09-23T21:15:10.559 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:10.559 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:10.559 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T21:15:10.559 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:45.512895+0000", 2023-09-23T21:15:10.559 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00045832199999999999 2023-09-23T21:15:10.560 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:15:10.560 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:15:10.560 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:15:10.560 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:10.560 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:10.560 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.46328.0:107536 10.3 10:c05d5d75:::benchmark_data_smithi026_379838_object27579:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e512)", 2023-09-23T21:15:10.560 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T21:13:44.141186+0000", 2023-09-23T21:15:10.560 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 86.273782419, 2023-09-23T21:15:10.560 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.3753114790000001, 2023-09-23T21:15:10.560 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:15:10.560 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:15:10.560 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T21:15:10.560 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.46328", 2023-09-23T21:15:10.561 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/3748861181", 2023-09-23T21:15:10.561 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 107536 2023-09-23T21:15:10.561 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:10.561 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:15:10.561 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:10.561 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:15:10.561 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:44.141186+0000", 2023-09-23T21:15:10.561 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:15:10.561 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:10.561 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:10.561 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:15:10.561 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:44.141186+0000", 2023-09-23T21:15:10.561 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:15:10.562 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:10.562 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:10.562 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:15:10.562 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:44.141184+0000", 2023-09-23T21:15:10.562 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T21:15:10.562 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:10.562 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:10.562 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:15:10.562 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:44.141193+0000", 2023-09-23T21:15:10.562 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.3140000000000004e-06 2023-09-23T21:15:10.562 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:10.562 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:10.562 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:15:10.563 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:44.141210+0000", 2023-09-23T21:15:10.563 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.6898999999999999e-05 2023-09-23T21:15:10.563 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:10.563 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:10.563 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:15:10.563 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:44.141221+0000", 2023-09-23T21:15:10.563 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.096e-05 2023-09-23T21:15:10.563 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:10.563 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:10.563 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:15:10.563 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:44.141296+0000", 2023-09-23T21:15:10.563 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.5228000000000005e-05 2023-09-23T21:15:10.564 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:10.564 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:10.564 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T21:15:10.564 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:44.141311+0000", 2023-09-23T21:15:10.564 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.5468000000000002e-05 2023-09-23T21:15:10.564 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:10.564 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:10.564 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:15:10.564 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:44.398819+0000", 2023-09-23T21:15:10.564 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.25750751 2023-09-23T21:15:10.564 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:10.564 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:10.564 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T21:15:10.565 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:44.398827+0000", 2023-09-23T21:15:10.565 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.3620000000000006e-06 2023-09-23T21:15:10.565 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:10.565 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:10.565 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:15:10.565 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:45.496713+0000", 2023-09-23T21:15:10.565 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0978854710000001 2023-09-23T21:15:10.565 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:10.565 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:10.565 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:15:10.565 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:45.496803+0000", 2023-09-23T21:15:10.565 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.0482000000000005e-05 2023-09-23T21:15:10.566 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:10.566 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:10.566 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 3,5", 2023-09-23T21:15:10.566 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:45.497089+0000", 2023-09-23T21:15:10.566 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00028609399999999998 2023-09-23T21:15:10.566 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:10.566 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:10.566 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T21:15:10.566 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:45.500768+0000", 2023-09-23T21:15:10.566 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0036785920000000001 2023-09-23T21:15:10.566 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:10.566 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:10.566 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:15:10.567 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:45.503649+0000", 2023-09-23T21:15:10.567 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0028816670000000001 2023-09-23T21:15:10.567 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:10.567 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:10.567 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:15:10.567 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:45.515977+0000", 2023-09-23T21:15:10.567 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.012327486 2023-09-23T21:15:10.567 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:10.567 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:10.567 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T21:15:10.567 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:45.516038+0000", 2023-09-23T21:15:10.567 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.1285999999999996e-05 2023-09-23T21:15:10.567 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:10.567 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:10.568 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T21:15:10.568 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:45.516497+0000", 2023-09-23T21:15:10.568 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000459121 2023-09-23T21:15:10.568 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:15:10.568 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:15:10.568 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:15:10.568 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:10.568 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:10.568 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.46328.0:107576 10.3 10:c55ae8e3:::benchmark_data_smithi026_379838_object27619:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e512)", 2023-09-23T21:15:10.568 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T21:13:44.319450+0000", 2023-09-23T21:15:10.568 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 86.095518712000001, 2023-09-23T21:15:10.568 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1979831869999999, 2023-09-23T21:15:10.568 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:15:10.569 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:15:10.569 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T21:15:10.569 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.46328", 2023-09-23T21:15:10.569 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/3748861181", 2023-09-23T21:15:10.569 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 107576 2023-09-23T21:15:10.569 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:10.569 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:15:10.569 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:10.569 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:15:10.569 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:44.319450+0000", 2023-09-23T21:15:10.569 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:15:10.569 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:10.569 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:10.569 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:15:10.570 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:44.319450+0000", 2023-09-23T21:15:10.570 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:15:10.570 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:10.570 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:10.570 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:15:10.570 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:44.319447+0000", 2023-09-23T21:15:10.570 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999976 2023-09-23T21:15:10.570 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:10.570 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:10.570 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:15:10.570 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:44.319457+0000", 2023-09-23T21:15:10.570 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.3340000000000008e-06 2023-09-23T21:15:10.570 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:10.571 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:10.571 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:15:10.571 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:44.319483+0000", 2023-09-23T21:15:10.571 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.6613000000000001e-05 2023-09-23T21:15:10.571 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:10.571 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:10.571 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:15:10.571 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:44.319494+0000", 2023-09-23T21:15:10.571 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1212e-05 2023-09-23T21:15:10.571 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:10.571 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:10.571 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:15:10.571 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:44.319563+0000", 2023-09-23T21:15:10.572 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.8459000000000003e-05 2023-09-23T21:15:10.572 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:10.572 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:10.572 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T21:15:10.572 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:44.319575+0000", 2023-09-23T21:15:10.572 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.201e-05 2023-09-23T21:15:10.572 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:10.572 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:10.572 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:15:10.572 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:44.398850+0000", 2023-09-23T21:15:10.572 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.079274905000000007 2023-09-23T21:15:10.572 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:10.572 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:10.573 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T21:15:10.573 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:44.398857+0000", 2023-09-23T21:15:10.573 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.7319999999999998e-06 2023-09-23T21:15:10.573 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:10.573 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:10.573 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:15:10.573 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:45.498888+0000", 2023-09-23T21:15:10.573 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1000307170000001 2023-09-23T21:15:10.573 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:10.573 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:10.573 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:15:10.573 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:45.499047+0000", 2023-09-23T21:15:10.573 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00015881000000000001 2023-09-23T21:15:10.573 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:10.574 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:10.574 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 3,5", 2023-09-23T21:15:10.574 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:45.499354+0000", 2023-09-23T21:15:10.574 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00030677200000000002 2023-09-23T21:15:10.574 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:10.574 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:10.574 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T21:15:10.574 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:45.500945+0000", 2023-09-23T21:15:10.574 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.001590892 2023-09-23T21:15:10.574 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:10.574 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:10.574 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:15:10.574 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:45.503813+0000", 2023-09-23T21:15:10.575 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0028686139999999998 2023-09-23T21:15:10.575 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:10.575 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:10.575 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:15:10.575 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:45.516996+0000", 2023-09-23T21:15:10.575 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.013182256 2023-09-23T21:15:10.575 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:10.575 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:10.575 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T21:15:10.575 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:45.517051+0000", 2023-09-23T21:15:10.575 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.5469e-05 2023-09-23T21:15:10.575 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:10.575 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:10.575 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T21:15:10.576 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:45.517433+0000", 2023-09-23T21:15:10.576 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00038173599999999998 2023-09-23T21:15:10.576 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:15:10.576 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:15:10.576 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:15:10.576 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:15:10.576 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:15:10.576 INFO:teuthology.orchestra.run.smithi026.stdout:} 2023-09-23T21:15:10.576 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:15:10.617 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:15:10 smithi070 ceph-mon[168282]: from='client.46742 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T21:15:10.617 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:15:10 smithi070 ceph-mon[168282]: pgmap v1354: 129 pgs: 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 14 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 110 active+clean; 402 MiB data, 6.1 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 825 op/s; 17952/19350 objects degraded (92.775%); 2524/19350 objects misplaced (13.044%); 9.5 MiB/s, 246 objects/s recovering 2023-09-23T21:15:10.619 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:15:10.633 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:15:10 smithi026 ceph-mon[175151]: from='client.46742 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T21:15:10.634 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:15:10 smithi026 ceph-mon[175151]: pgmap v1354: 129 pgs: 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 14 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 110 active+clean; 402 MiB data, 6.1 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 825 op/s; 17952/19350 objects degraded (92.775%); 2524/19350 objects misplaced (13.044%); 9.5 MiB/s, 246 objects/s recovering 2023-09-23T21:15:10.634 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:15:10 smithi026 ceph-mon[179688]: from='client.46742 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T21:15:10.634 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:15:10 smithi026 ceph-mon[179688]: pgmap v1354: 129 pgs: 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 14 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 110 active+clean; 402 MiB data, 6.1 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 825 op/s; 17952/19350 objects degraded (92.775%); 2524/19350 objects misplaced (13.044%); 9.5 MiB/s, 246 objects/s recovering 2023-09-23T21:15:10.661 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:15:10.668 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:15:10.669 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:15:10.769 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:15:10.802 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:15:10.843 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:15:10.847 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:15:10.848 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:15:10.949 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:15:11.001 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:15:11.039 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:15:11.042 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:15:11.042 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:15:11.143 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:15:11.242 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:15:11.361 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:15:11.369 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:15:11.369 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:15:11.412 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_ops_in_flight 2023-09-23T21:15:11.470 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:15:11.541 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:15:11.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:15:11 smithi070 ceph-mon[168282]: Health check update: Degraded data redundancy: 17952/19350 objects degraded (92.775%), 18 pgs degraded, 15 pgs undersized (PG_DEGRADED) 2023-09-23T21:15:11.635 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:15:11 smithi026 ceph-mon[175151]: Health check update: Degraded data redundancy: 17952/19350 objects degraded (92.775%), 18 pgs degraded, 15 pgs undersized (PG_DEGRADED) 2023-09-23T21:15:11.635 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:15:11 smithi026 ceph-mon[179688]: Health check update: Degraded data redundancy: 17952/19350 objects degraded (92.775%), 18 pgs degraded, 15 pgs undersized (PG_DEGRADED) 2023-09-23T21:15:11.682 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T21:15:11.690 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:15:11.691 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:15:11.792 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:15:11.877 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:15:12.008 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:15:12.024 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:15:12.024 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:15:12.125 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:15:12.200 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0: {} 2023-09-23T21:15:12.237 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:15:12.253 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1: {} 2023-09-23T21:15:12.310 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2: {} 2023-09-23T21:15:12.314 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:15:12.322 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:15:12.323 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:15:12.355 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3: {} 2023-09-23T21:15:12.365 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:15:12.400 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4: {} 2023-09-23T21:15:12.423 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:15:12.446 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T21:15:12.455 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:15:12.497 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:15:12.501 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:15:12.501 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:15:12.550 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6: {} 2023-09-23T21:15:12.551 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T21:15:12.551 INFO:teuthology.orchestra.run.smithi026.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T21:15:12.551 INFO:teuthology.orchestra.run.smithi026.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T21:15:12.551 INFO:teuthology.orchestra.run.smithi026.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T21:15:12.551 INFO:teuthology.orchestra.run.smithi026.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T21:15:12.551 INFO:teuthology.orchestra.run.smithi026.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T21:15:12.551 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T21:15:12.551 INFO:teuthology.orchestra.run.smithi026.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T21:15:12.551 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T21:15:12.580 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:15:12 smithi026 ceph-mon[175151]: pgmap v1355: 129 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 14 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 111 active+clean; 349 MiB data, 5.9 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 788 op/s; 17951/16815 objects degraded (106.756%); 2524/16815 objects misplaced (15.010%); 8.2 MiB/s, 212 objects/s recovering 2023-09-23T21:15:12.581 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:15:12 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/3460052830' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:15:12.581 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:15:12 smithi026 ceph-mon[179688]: pgmap v1355: 129 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 14 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 111 active+clean; 349 MiB data, 5.9 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 788 op/s; 17951/16815 objects degraded (106.756%); 2524/16815 objects misplaced (15.010%); 8.2 MiB/s, 212 objects/s recovering 2023-09-23T21:15:12.581 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:15:12 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/3460052830' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:15:12.602 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:15:12.644 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:15:12 smithi070 ceph-mon[168282]: pgmap v1355: 129 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 14 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 111 active+clean; 349 MiB data, 5.9 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 788 op/s; 17951/16815 objects degraded (106.756%); 2524/16815 objects misplaced (15.010%); 8.2 MiB/s, 212 objects/s recovering 2023-09-23T21:15:12.644 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:15:12 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/3460052830' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:15:12.649 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:15:12.699 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:15:12.703 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:15:12.704 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:15:12.775 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T21:15:12.805 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:15:12.843 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:15:12.893 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:15:12.900 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:15:12.900 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:15:13.001 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:15:13.120 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:15:13.407 INFO:teuthology.orchestra.run.smithi026.stderr:noscrub is set 2023-09-23T21:15:13.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:15:13 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/619946996' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T21:15:13.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:15:13 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T21:15:13.885 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:15:13 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/619946996' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T21:15:13.885 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:15:13 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T21:15:13.885 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:15:13 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/619946996' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T21:15:13.885 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:15:13 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T21:15:14.633 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:15:14.644 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:15:14.644 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:15:14.661 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:15:14 smithi026 ceph-mon[179688]: pgmap v1356: 129 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 14 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 111 active+clean; 271 MiB data, 5.6 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 672 op/s; 17937/13074 objects degraded (137.196%); 2414/13074 objects misplaced (18.464%); 3.5 MiB/s, 149 objects/s recovering 2023-09-23T21:15:14.662 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:15:14 smithi026 ceph-mon[179688]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:15:14.662 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:15:14 smithi026 ceph-mon[179688]: failed to encode map e535 with expected crc 2023-09-23T21:15:14.662 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:15:14 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T21:15:14.662 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:15:14 smithi026 ceph-mon[179688]: osdmap e535: 8 total, 8 up, 6 in 2023-09-23T21:15:14.662 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:15:14 smithi026 ceph-mon[179688]: failed to encode map e535 with expected crc 2023-09-23T21:15:14.662 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:15:14 smithi026 ceph-mon[179688]: failed to encode map e535 with expected crc 2023-09-23T21:15:14.662 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:15:14 smithi026 ceph-mon[175151]: pgmap v1356: 129 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 14 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 111 active+clean; 271 MiB data, 5.6 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 672 op/s; 17937/13074 objects degraded (137.196%); 2414/13074 objects misplaced (18.464%); 3.5 MiB/s, 149 objects/s recovering 2023-09-23T21:15:14.662 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:15:14 smithi026 ceph-mon[175151]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:15:14.662 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:15:14 smithi026 ceph-mon[175151]: failed to encode map e535 with expected crc 2023-09-23T21:15:14.662 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:15:14 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T21:15:14.663 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:15:14 smithi026 ceph-mon[175151]: osdmap e535: 8 total, 8 up, 6 in 2023-09-23T21:15:14.663 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:15:14 smithi026 ceph-mon[175151]: failed to encode map e535 with expected crc 2023-09-23T21:15:14.663 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:15:14 smithi026 ceph-mon[175151]: failed to encode map e535 with expected crc 2023-09-23T21:15:14.745 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:15:14.812 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:15:14.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:15:14 smithi070 ceph-mon[168282]: pgmap v1356: 129 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 14 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 111 active+clean; 271 MiB data, 5.6 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 672 op/s; 17937/13074 objects degraded (137.196%); 2414/13074 objects misplaced (18.464%); 3.5 MiB/s, 149 objects/s recovering 2023-09-23T21:15:14.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:15:14 smithi070 ceph-mon[168282]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:15:14.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:15:14 smithi070 ceph-mon[168282]: failed to encode map e535 with expected crc 2023-09-23T21:15:14.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:15:14 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T21:15:14.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:15:14 smithi070 ceph-mon[168282]: osdmap e535: 8 total, 8 up, 6 in 2023-09-23T21:15:14.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:15:14 smithi070 ceph-mon[168282]: failed to encode map e535 with expected crc 2023-09-23T21:15:14.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:15:14 smithi070 ceph-mon[168282]: failed to encode map e535 with expected crc 2023-09-23T21:15:14.939 INFO:tasks.radosbench.radosbench.0.smithi026.stdout:Removed 79954 objects 2023-09-23T21:15:14.939 INFO:tasks.radosbench.radosbench.0.smithi026.stdout:Clean up completed and total clean up time :201.449 2023-09-23T21:15:14.985 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:15:14.989 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:15:14.989 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:15:15.089 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:15:15.120 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:15:15.155 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:15:15.158 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:15:15.158 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:15:15.232 INFO:tasks.cephadm.ceph_manager.ceph:removing pool_name unique_pool_4 2023-09-23T21:15:15.232 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph osd pool rm unique_pool_4 unique_pool_4 --yes-i-really-really-mean-it 2023-09-23T21:15:15.259 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:15:15.295 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:15:15.328 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:15:15.330 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:15:15.331 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:15:15.431 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:15:15.466 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:15:15.632 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:15:15.635 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:15:15.636 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:15:15.737 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:15:15.766 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:15:15.811 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:15:15.909 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T21:15:15.913 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:15:15.914 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:15:16.014 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:15:16.043 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:15:16.085 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T21:15:16.091 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:15:16.092 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:15:16.192 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:15:16.196 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:15:16.218 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:15:16.253 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:15:16.255 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:15:16.255 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:15:16.275 INFO:teuthology.orchestra.run.smithi026.stdout:ERROR: (22) Invalid argument 2023-09-23T21:15:16.275 INFO:teuthology.orchestra.run.smithi026.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T21:15:16.356 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:15:16.386 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:15:16.565 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:15:16 smithi026 ceph-mon[179688]: pgmap v1358: 129 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 14 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 111 active+clean; 129 MiB data, 5.4 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 782 op/s; 17713/6261 objects degraded (282.910%); 2294/6261 objects misplaced (36.640%); 6.3 MiB/s, 195 objects/s recovering 2023-09-23T21:15:16.565 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:15:16 smithi026 ceph-mon[179688]: Health check update: Degraded data redundancy: 17713/6261 objects degraded (282.910%), 16 pgs degraded, 15 pgs undersized (PG_DEGRADED) 2023-09-23T21:15:16.565 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:15:16 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:15:16.565 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:15:16 smithi026 ceph-mon[175151]: pgmap v1358: 129 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 14 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 111 active+clean; 129 MiB data, 5.4 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 782 op/s; 17713/6261 objects degraded (282.910%); 2294/6261 objects misplaced (36.640%); 6.3 MiB/s, 195 objects/s recovering 2023-09-23T21:15:16.565 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:15:16 smithi026 ceph-mon[175151]: Health check update: Degraded data redundancy: 17713/6261 objects degraded (282.910%), 16 pgs degraded, 15 pgs undersized (PG_DEGRADED) 2023-09-23T21:15:16.565 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:15:16 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:15:16.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:15:16 smithi070 ceph-mon[168282]: pgmap v1358: 129 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 14 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 111 active+clean; 129 MiB data, 5.4 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 782 op/s; 17713/6261 objects degraded (282.910%); 2294/6261 objects misplaced (36.640%); 6.3 MiB/s, 195 objects/s recovering 2023-09-23T21:15:16.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:15:16 smithi070 ceph-mon[168282]: Health check update: Degraded data redundancy: 17713/6261 objects degraded (282.910%), 16 pgs degraded, 15 pgs undersized (PG_DEGRADED) 2023-09-23T21:15:16.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:15:16 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:15:16.945 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_blocked_ops 2023-09-23T21:15:17.007 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T21:15:17.212 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T21:15:17.218 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:15:17.218 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:15:17.319 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:15:17.348 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:15:17.384 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:15:17.387 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:15:17.387 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:15:17.488 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:15:17.503 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T21:15:17.516 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:15:17.551 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:15:17.554 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:15:17.554 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:15:17.655 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:15:17.681 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:15:17.713 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:15:17.716 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:15:17.716 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:15:17.817 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:15:17.843 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:15:17.875 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:15:17.877 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:15:17.878 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:15:17.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:15:17 smithi026 ceph-mon[179688]: failed to encode map e536 with expected crc 2023-09-23T21:15:17.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:15:17 smithi026 ceph-mon[179688]: failed to encode map e536 with expected crc 2023-09-23T21:15:17.883 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:15:17 smithi026 ceph-mon[179688]: failed to encode map e536 with expected crc 2023-09-23T21:15:17.883 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:15:17 smithi026 ceph-mon[179688]: osdmap e536: 8 total, 8 up, 6 in 2023-09-23T21:15:17.883 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:15:17 smithi026 ceph-mon[179688]: failed to encode map e536 with expected crc 2023-09-23T21:15:17.883 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:15:17 smithi026 ceph-mon[179688]: pgmap v1360: 129 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 14 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 111 active+clean; 28 MiB data, 5.1 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 757 op/s; 17467/1392 objects degraded (1254.813%); 2169/1392 objects misplaced (155.819%); 4.5 MiB/s, 125 objects/s recovering 2023-09-23T21:15:17.883 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:15:17 smithi026 ceph-mon[175151]: failed to encode map e536 with expected crc 2023-09-23T21:15:17.883 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:15:17 smithi026 ceph-mon[175151]: failed to encode map e536 with expected crc 2023-09-23T21:15:17.883 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:15:17 smithi026 ceph-mon[175151]: failed to encode map e536 with expected crc 2023-09-23T21:15:17.883 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:15:17 smithi026 ceph-mon[175151]: osdmap e536: 8 total, 8 up, 6 in 2023-09-23T21:15:17.883 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:15:17 smithi026 ceph-mon[175151]: failed to encode map e536 with expected crc 2023-09-23T21:15:17.884 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:15:17 smithi026 ceph-mon[175151]: pgmap v1360: 129 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 14 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 111 active+clean; 28 MiB data, 5.1 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 757 op/s; 17467/1392 objects degraded (1254.813%); 2169/1392 objects misplaced (155.819%); 4.5 MiB/s, 125 objects/s recovering 2023-09-23T21:15:17.979 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:15:18.023 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:15:18.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:15:17 smithi070 ceph-mon[168282]: failed to encode map e536 with expected crc 2023-09-23T21:15:18.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:15:17 smithi070 ceph-mon[168282]: failed to encode map e536 with expected crc 2023-09-23T21:15:18.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:15:17 smithi070 ceph-mon[168282]: failed to encode map e536 with expected crc 2023-09-23T21:15:18.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:15:17 smithi070 ceph-mon[168282]: osdmap e536: 8 total, 8 up, 6 in 2023-09-23T21:15:18.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:15:17 smithi070 ceph-mon[168282]: failed to encode map e536 with expected crc 2023-09-23T21:15:18.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:15:17 smithi070 ceph-mon[168282]: pgmap v1360: 129 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 14 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 111 active+clean; 28 MiB data, 5.1 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 757 op/s; 17467/1392 objects degraded (1254.813%); 2169/1392 objects misplaced (155.819%); 4.5 MiB/s, 125 objects/s recovering 2023-09-23T21:15:18.122 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:15:18.125 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:15:18.126 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:15:18.164 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:15:18.208 INFO:teuthology.orchestra.run.smithi026.stdout: 2023-09-23T21:15:18.210 INFO:teuthology.orchestra.run.smithi026.stderr:dumped all 2023-09-23T21:15:18.213 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:15:18.226 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:15:18.263 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:15:18.312 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:15:18.316 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:15:18.317 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:15:18.417 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:15:18.457 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:15:18.500 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T21:15:18.503 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:15:18.504 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:15:18.604 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:15:18.636 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:15:18.672 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:15:18.674 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:15:18.675 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:15:18.776 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:15:18.802 INFO:teuthology.orchestra.run.smithi026.stderr:pool 'unique_pool_4' removed 2023-09-23T21:15:18.822 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:15:18.839 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:15:19.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:15:18 smithi070 ceph-mon[168282]: failed to encode map e537 with expected crc 2023-09-23T21:15:19.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:15:18 smithi070 ceph-mon[168282]: osdmap e537: 8 total, 8 up, 6 in 2023-09-23T21:15:19.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:15:18 smithi070 ceph-mon[168282]: failed to encode map e537 with expected crc 2023-09-23T21:15:19.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:15:18 smithi070 ceph-mon[168282]: failed to encode map e537 with expected crc 2023-09-23T21:15:19.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:15:18 smithi070 ceph-mon[168282]: from='client.46754 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T21:15:19.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:15:18 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/387007920' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_4", "pool2": "unique_pool_4", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-23T21:15:19.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:15:18 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_4", "pool2": "unique_pool_4", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-23T21:15:19.131 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:15:18 smithi026 ceph-mon[179688]: failed to encode map e537 with expected crc 2023-09-23T21:15:19.131 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:15:18 smithi026 ceph-mon[179688]: osdmap e537: 8 total, 8 up, 6 in 2023-09-23T21:15:19.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:15:18 smithi026 ceph-mon[179688]: failed to encode map e537 with expected crc 2023-09-23T21:15:19.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:15:18 smithi026 ceph-mon[179688]: failed to encode map e537 with expected crc 2023-09-23T21:15:19.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:15:18 smithi026 ceph-mon[179688]: from='client.46754 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T21:15:19.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:15:18 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/387007920' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_4", "pool2": "unique_pool_4", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-23T21:15:19.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:15:18 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_4", "pool2": "unique_pool_4", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-23T21:15:19.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:15:18 smithi026 ceph-mon[175151]: failed to encode map e537 with expected crc 2023-09-23T21:15:19.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:15:18 smithi026 ceph-mon[175151]: osdmap e537: 8 total, 8 up, 6 in 2023-09-23T21:15:19.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:15:18 smithi026 ceph-mon[175151]: failed to encode map e537 with expected crc 2023-09-23T21:15:19.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:15:18 smithi026 ceph-mon[175151]: failed to encode map e537 with expected crc 2023-09-23T21:15:19.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:15:18 smithi026 ceph-mon[175151]: from='client.46754 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T21:15:19.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:15:18 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/387007920' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_4", "pool2": "unique_pool_4", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-23T21:15:19.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:15:18 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_4", "pool2": "unique_pool_4", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-23T21:15:19.744 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 21:15:19 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[164029]: ::ffff:172.21.15.70 - - [23/Sep/2023:21:15:19] "GET /metrics HTTP/1.1" 200 36422 "" "Prometheus/2.43.0" 2023-09-23T21:15:19.751 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:15:19.754 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:15:19.755 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:15:19.855 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:15:19.893 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:15:19.898 INFO:teuthology.orchestra.run.smithi026.stdout:{"pg_ready":true,"pg_map":{"version":1361,"stamp":"2023-09-23T21:15:17.785915+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":21871340,"num_objects":350,"num_object_clones":0,"num_object_copies":1050,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":17467,"num_objects_misplaced":2169,"num_objects_unfound":0,"num_objects_dirty":350,"num_whiteouts":0,"num_read":90,"num_read_kb":166,"num_write":147229,"num_write_kb":4309782,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":47447,"num_bytes_recovered":2759953597,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":123920,"ondisk_log_size":123920,"up":387,"acting":373,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":431,"num_osds":6,"num_per_pool_osds":6,"num_per_pool_omap_osds":6,"kb":562470912,"kb_used":5237032,"kb_used_data":1844536,"kb_used_omap":125,"kb_used_meta":3392322,"kb_avail":557233880,"statfs":{"total":575970213888,"available":570607493120,"internally_reserved":0,"allocated":1888804864,"data_stored":1876629734,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":128662,"internal_metadata":3473738090},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,1,10],"upper_bound":16},"perf_stat":{"commit_latency_ms":6,"apply_latency_ms":6,"commit_latency_ns":6000000,"apply_latency_ns":6000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-344326168,"num_objects":-5255,"num_object_clones":0,"num_object_copies":-15765,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":-484,"num_objects_misplaced":-355,"num_objects_unfound":0,"num_objects_dirty":-5255,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5330,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1852,"num_bytes_recovered":37093400,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"6.693780"},"pg_stats":[{"pgid":"1.18","version":"79'161","reported_seq":656,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-09-23T21:15:08.244700+0000","last_change":"2023-09-23T21:12:30.438733+0000","last_active":"2023-09-23T21:15:08.244700+0000","last_peered":"2023-09-23T21:15:08.244700+0000","last_clean":"2023-09-23T21:15:08.244700+0000","last_became_active":"2023-09-23T21:12:30.438356+0000","last_became_peered":"2023-09-23T21:12:30.438356+0000","last_unstale":"2023-09-23T21:15:08.244700+0000","last_undegraded":"2023-09-23T21:15:08.244700+0000","last_fullsized":"2023-09-23T21:15:08.244700+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":479,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:06:52.098882+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:06:52.098882+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0092173299999999993,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":131344,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"10.13","version":"535'6368","reported_seq":11720,"reported_epoch":535,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T21:15:14.868636+0000","last_change":"2023-09-23T21:13:49.840403+0000","last_active":"2023-09-23T21:15:14.868636+0000","last_peered":"2023-09-23T21:15:14.868636+0000","last_clean":"2023-09-23T21:11:26.718194+0000","last_became_active":"2023-09-23T21:13:48.181112+0000","last_became_peered":"2023-09-23T21:13:48.181112+0000","last_unstale":"2023-09-23T21:15:14.868636+0000","last_undegraded":"2023-09-23T21:13:48.139049+0000","last_fullsized":"2023-09-23T21:13:45.405990+0000","mapping_epoch":513,"log_start":"435'2204","ondisk_log_start":"435'2204","created":438,"last_epoch_clean":439,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_clean_scrub_stamp":"2023-09-23T21:10:15.663179+0000","objects_scrubbed":0,"log_size":4164,"log_dups_size":0,"ondisk_log_size":4164,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":2195456,"num_objects":34,"num_object_clones":0,"num_object_copies":102,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":200,"num_objects_misplaced":832,"num_objects_unfound":0,"num_objects_dirty":34,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5000,"num_write_kb":161056,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2483,"num_bytes_recovered":162660352,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[4,3],"avail_no_missing":["4","5"],"object_location_counts":[{"shards":"3,4","objects":34}],"blocked_by":[],"up_primary":2,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1a","version":"68'1","reported_seq":126,"reported_epoch":535,"state":"active+clean","last_fresh":"2023-09-23T21:15:13.392169+0000","last_change":"2023-09-23T21:13:13.291403+0000","last_active":"2023-09-23T21:15:13.392169+0000","last_peered":"2023-09-23T21:15:13.392169+0000","last_clean":"2023-09-23T21:15:13.392169+0000","last_became_active":"2023-09-23T21:13:13.291123+0000","last_became_peered":"2023-09-23T21:13:13.291123+0000","last_unstale":"2023-09-23T21:15:13.392169+0000","last_undegraded":"2023-09-23T21:15:13.392169+0000","last_fullsized":"2023-09-23T21:15:13.392169+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":495,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:45:15.427430+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,0],"acting":[4,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":195,"reported_epoch":535,"state":"active+clean","last_fresh":"2023-09-23T21:15:13.390930+0000","last_change":"2023-09-23T21:12:10.767963+0000","last_active":"2023-09-23T21:15:13.390930+0000","last_peered":"2023-09-23T21:15:13.390930+0000","last_clean":"2023-09-23T21:15:13.390930+0000","last_became_active":"2023-09-23T21:12:10.767126+0000","last_became_peered":"2023-09-23T21:12:10.767126+0000","last_unstale":"2023-09-23T21:15:13.390930+0000","last_undegraded":"2023-09-23T21:15:13.390930+0000","last_fullsized":"2023-09-23T21:15:13.390930+0000","mapping_epoch":470,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:45:47.138277+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,5],"acting":[4,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1b","version":"79'161","reported_seq":538,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-09-23T21:15:08.250965+0000","last_change":"2023-09-23T21:13:13.373548+0000","last_active":"2023-09-23T21:15:08.250965+0000","last_peered":"2023-09-23T21:15:08.250965+0000","last_clean":"2023-09-23T21:15:08.250965+0000","last_became_active":"2023-09-23T21:13:13.372845+0000","last_became_peered":"2023-09-23T21:13:13.372845+0000","last_unstale":"2023-09-23T21:15:08.250965+0000","last_undegraded":"2023-09-23T21:15:08.250965+0000","last_fullsized":"2023-09-23T21:15:08.250965+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":479,"last_epoch_clean":495,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:05:55.772264+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:05:55.772264+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.001862507,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.10","version":"535'6317","reported_seq":15056,"reported_epoch":535,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T21:15:14.935151+0000","last_change":"2023-09-23T21:13:36.071975+0000","last_active":"2023-09-23T21:15:14.935151+0000","last_peered":"2023-09-23T21:15:14.935151+0000","last_clean":"2023-09-23T21:11:26.716051+0000","last_became_active":"2023-09-23T21:13:33.619010+0000","last_became_peered":"2023-09-23T21:13:33.619010+0000","last_unstale":"2023-09-23T21:15:14.935151+0000","last_undegraded":"2023-09-23T21:13:33.443511+0000","last_fullsized":"2023-09-23T21:13:31.753181+0000","mapping_epoch":502,"log_start":"435'2088","ondisk_log_start":"435'2088","created":438,"last_epoch_clean":450,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_clean_scrub_stamp":"2023-09-23T21:10:15.663179+0000","objects_scrubbed":0,"log_size":4229,"log_dups_size":0,"ondisk_log_size":4229,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":950272,"num_objects":15,"num_object_clones":0,"num_object_copies":45,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":826,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":15,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4953,"num_write_kb":158944,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4023,"num_bytes_recovered":263651328,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7],"avail_no_missing":["4","1","5","7"],"object_location_counts":[{"shards":"4,7","objects":-14}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":531,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-09-23T21:15:08.243187+0000","last_change":"2023-09-23T21:12:10.824116+0000","last_active":"2023-09-23T21:15:08.243187+0000","last_peered":"2023-09-23T21:15:08.243187+0000","last_clean":"2023-09-23T21:15:08.243187+0000","last_became_active":"2023-09-23T21:12:10.823849+0000","last_became_peered":"2023-09-23T21:12:10.823849+0000","last_unstale":"2023-09-23T21:15:08.243187+0000","last_undegraded":"2023-09-23T21:15:08.243187+0000","last_fullsized":"2023-09-23T21:15:08.243187+0000","mapping_epoch":470,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:54:13.777603+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":532,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-09-23T21:15:10.247709+0000","last_change":"2023-09-23T21:13:44.413713+0000","last_active":"2023-09-23T21:15:10.247709+0000","last_peered":"2023-09-23T21:15:10.247709+0000","last_clean":"2023-09-23T21:15:10.247709+0000","last_became_active":"2023-09-23T21:13:44.413366+0000","last_became_peered":"2023-09-23T21:13:44.413366+0000","last_unstale":"2023-09-23T21:15:10.247709+0000","last_undegraded":"2023-09-23T21:15:10.247709+0000","last_fullsized":"2023-09-23T21:15:10.247709+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:31:56.349530+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1a","version":"79'161","reported_seq":681,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-09-23T21:15:08.247147+0000","last_change":"2023-09-23T21:12:30.444665+0000","last_active":"2023-09-23T21:15:08.247147+0000","last_peered":"2023-09-23T21:15:08.247147+0000","last_clean":"2023-09-23T21:15:08.247147+0000","last_became_active":"2023-09-23T21:12:30.444326+0000","last_became_peered":"2023-09-23T21:12:30.444326+0000","last_unstale":"2023-09-23T21:15:08.247147+0000","last_undegraded":"2023-09-23T21:15:08.247147+0000","last_fullsized":"2023-09-23T21:15:08.247147+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":479,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:06:51.075448+0000","last_deep_scrub":"79'161","last_deep_scrub_stamp":"2023-09-23T21:05:32.630292+0000","last_clean_scrub_stamp":"2023-09-23T21:06:51.075448+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0054793890000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,4],"acting":[2,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"10.11","version":"535'6271","reported_seq":13108,"reported_epoch":535,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-23T21:15:14.806148+0000","last_change":"2023-09-23T21:13:31.864763+0000","last_active":"2023-09-23T21:15:14.806148+0000","last_peered":"2023-09-23T21:15:14.806148+0000","last_clean":"2023-09-23T21:13:30.058478+0000","last_became_active":"2023-09-23T21:13:31.850860+0000","last_became_peered":"2023-09-23T21:13:31.850860+0000","last_unstale":"2023-09-23T21:15:14.806148+0000","last_undegraded":"2023-09-23T21:15:14.806148+0000","last_fullsized":"2023-09-23T21:15:14.806148+0000","mapping_epoch":502,"log_start":"434'1181","ondisk_log_start":"434'1181","created":438,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":5,"last_scrub":"441'2974","last_scrub_stamp":"2023-09-23T21:11:06.133218+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_clean_scrub_stamp":"2023-09-23T21:11:06.133218+0000","objects_scrubbed":1646,"log_size":5090,"log_dups_size":0,"ondisk_log_size":5090,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:54:59.366711+0000","scrub_duration":0.3173955,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3300,"num_write_kb":52928,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2152,"num_bytes_recovered":141033472,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[2,5,7],"avail_no_missing":["2","0","1","5","7"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":44,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-09-23T21:15:08.250544+0000","last_change":"2023-09-23T21:13:30.806084+0000","last_active":"2023-09-23T21:15:08.250544+0000","last_peered":"2023-09-23T21:15:08.250544+0000","last_clean":"2023-09-23T21:15:08.250544+0000","last_became_active":"2023-09-23T21:13:30.805743+0000","last_became_peered":"2023-09-23T21:13:30.805743+0000","last_unstale":"2023-09-23T21:15:08.250544+0000","last_undegraded":"2023-09-23T21:15:08.250544+0000","last_fullsized":"2023-09-23T21:15:08.250544+0000","mapping_epoch":501,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:48:10.849281+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":408,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-09-23T21:15:08.250710+0000","last_change":"2023-09-23T21:13:44.410921+0000","last_active":"2023-09-23T21:15:08.250710+0000","last_peered":"2023-09-23T21:15:08.250710+0000","last_clean":"2023-09-23T21:15:08.250710+0000","last_became_active":"2023-09-23T21:13:44.410633+0000","last_became_peered":"2023-09-23T21:13:44.410633+0000","last_unstale":"2023-09-23T21:15:08.250710+0000","last_undegraded":"2023-09-23T21:15:08.250710+0000","last_fullsized":"2023-09-23T21:15:08.250710+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:57:10.116786+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.15","version":"79'161","reported_seq":562,"reported_epoch":535,"state":"active+clean","last_fresh":"2023-09-23T21:15:13.390518+0000","last_change":"2023-09-23T21:13:38.379453+0000","last_active":"2023-09-23T21:15:13.390518+0000","last_peered":"2023-09-23T21:15:13.390518+0000","last_clean":"2023-09-23T21:15:13.390518+0000","last_became_active":"2023-09-23T21:13:38.376499+0000","last_became_peered":"2023-09-23T21:13:38.376499+0000","last_unstale":"2023-09-23T21:15:13.390518+0000","last_undegraded":"2023-09-23T21:15:13.390518+0000","last_fullsized":"2023-09-23T21:15:13.390518+0000","mapping_epoch":507,"log_start":"0'0","ondisk_log_start":"0'0","created":479,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:09:34.892947+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:09:34.892947+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13418,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":11,"num_read_kb":21,"num_write":6,"num_write_kb":55,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":40253,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":509,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-09-23T21:15:08.250868+0000","last_change":"2023-09-23T21:13:12.235728+0000","last_active":"2023-09-23T21:15:08.250868+0000","last_peered":"2023-09-23T21:15:08.250868+0000","last_clean":"2023-09-23T21:15:08.250868+0000","last_became_active":"2023-09-23T21:13:12.216549+0000","last_became_peered":"2023-09-23T21:13:12.216549+0000","last_unstale":"2023-09-23T21:15:08.250868+0000","last_undegraded":"2023-09-23T21:15:08.250868+0000","last_fullsized":"2023-09-23T21:15:08.250868+0000","mapping_epoch":493,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:39:33.324239+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":573,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-09-23T21:15:08.242586+0000","last_change":"2023-09-23T21:12:10.790349+0000","last_active":"2023-09-23T21:15:08.242586+0000","last_peered":"2023-09-23T21:15:08.242586+0000","last_clean":"2023-09-23T21:15:08.242586+0000","last_became_active":"2023-09-23T21:12:10.789977+0000","last_became_peered":"2023-09-23T21:12:10.789977+0000","last_unstale":"2023-09-23T21:15:08.242586+0000","last_undegraded":"2023-09-23T21:15:08.242586+0000","last_fullsized":"2023-09-23T21:15:08.242586+0000","mapping_epoch":470,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:49:40.191147+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,5],"acting":[3,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"79'161","reported_seq":896,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-09-23T21:15:08.244895+0000","last_change":"2023-09-23T21:11:32.760292+0000","last_active":"2023-09-23T21:15:08.244895+0000","last_peered":"2023-09-23T21:15:08.244895+0000","last_clean":"2023-09-23T21:15:08.244895+0000","last_became_active":"2023-09-23T21:11:32.759909+0000","last_became_peered":"2023-09-23T21:11:32.759909+0000","last_unstale":"2023-09-23T21:15:08.244895+0000","last_undegraded":"2023-09-23T21:15:08.244895+0000","last_fullsized":"2023-09-23T21:15:08.244895+0000","mapping_epoch":457,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":458,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:25.096401+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:25.096401+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:13:09.225685+0000","scrub_duration":0.002859393,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":260,"reported_epoch":535,"state":"active+clean","last_fresh":"2023-09-23T21:15:13.396620+0000","last_change":"2023-09-23T21:12:10.779468+0000","last_active":"2023-09-23T21:15:13.396620+0000","last_peered":"2023-09-23T21:15:13.396620+0000","last_clean":"2023-09-23T21:15:13.396620+0000","last_became_active":"2023-09-23T21:12:10.779282+0000","last_became_peered":"2023-09-23T21:12:10.779282+0000","last_unstale":"2023-09-23T21:15:13.396620+0000","last_undegraded":"2023-09-23T21:15:13.396620+0000","last_fullsized":"2023-09-23T21:15:13.396620+0000","mapping_epoch":470,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:36:35.417986+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":510,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-09-23T21:15:08.243080+0000","last_change":"2023-09-23T21:10:26.758958+0000","last_active":"2023-09-23T21:15:08.243080+0000","last_peered":"2023-09-23T21:15:08.243080+0000","last_clean":"2023-09-23T21:15:08.243080+0000","last_became_active":"2023-09-23T21:10:26.758539+0000","last_became_peered":"2023-09-23T21:10:26.758539+0000","last_unstale":"2023-09-23T21:15:08.243080+0000","last_undegraded":"2023-09-23T21:15:08.243080+0000","last_fullsized":"2023-09-23T21:15:08.243080+0000","mapping_epoch":423,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":424,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:44:42.319189+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.17","version":"79'161","reported_seq":627,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-09-23T21:15:08.245016+0000","last_change":"2023-09-23T21:13:44.416768+0000","last_active":"2023-09-23T21:15:08.245016+0000","last_peered":"2023-09-23T21:15:08.245016+0000","last_clean":"2023-09-23T21:15:08.245016+0000","last_became_active":"2023-09-23T21:13:44.416156+0000","last_became_peered":"2023-09-23T21:13:44.416156+0000","last_unstale":"2023-09-23T21:15:08.245016+0000","last_undegraded":"2023-09-23T21:15:08.245016+0000","last_fullsized":"2023-09-23T21:15:08.245016+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":479,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:25.916867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:25.916867+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.001665323,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":419,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-09-23T21:15:08.245526+0000","last_change":"2023-09-23T21:11:08.766045+0000","last_active":"2023-09-23T21:15:08.245526+0000","last_peered":"2023-09-23T21:15:08.245526+0000","last_clean":"2023-09-23T21:15:08.245526+0000","last_became_active":"2023-09-23T21:11:08.758504+0000","last_became_peered":"2023-09-23T21:11:08.758504+0000","last_unstale":"2023-09-23T21:15:08.245526+0000","last_undegraded":"2023-09-23T21:15:08.245526+0000","last_fullsized":"2023-09-23T21:15:08.245526+0000","mapping_epoch":443,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":444,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:02:19.482574+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":360,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,4],"acting":[3,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":222,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-09-23T21:15:08.247635+0000","last_change":"2023-09-23T21:13:30.821489+0000","last_active":"2023-09-23T21:15:08.247635+0000","last_peered":"2023-09-23T21:15:08.247635+0000","last_clean":"2023-09-23T21:15:08.247635+0000","last_became_active":"2023-09-23T21:13:30.821125+0000","last_became_peered":"2023-09-23T21:13:30.821125+0000","last_unstale":"2023-09-23T21:15:08.247635+0000","last_undegraded":"2023-09-23T21:15:08.247635+0000","last_fullsized":"2023-09-23T21:15:08.247635+0000","mapping_epoch":501,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:47:18.553417+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,5],"acting":[2,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.16","version":"79'161","reported_seq":453,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-09-23T21:15:08.251796+0000","last_change":"2023-09-23T21:13:33.770962+0000","last_active":"2023-09-23T21:15:08.251796+0000","last_peered":"2023-09-23T21:15:08.251796+0000","last_clean":"2023-09-23T21:15:08.251796+0000","last_became_active":"2023-09-23T21:13:33.770407+0000","last_became_peered":"2023-09-23T21:13:33.770407+0000","last_unstale":"2023-09-23T21:15:08.251796+0000","last_undegraded":"2023-09-23T21:15:08.251796+0000","last_fullsized":"2023-09-23T21:15:08.251796+0000","mapping_epoch":504,"log_start":"0'0","ondisk_log_start":"0'0","created":479,"last_epoch_clean":505,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:10:24.268662+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:10:24.268662+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13418,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":11,"num_read_kb":21,"num_write":6,"num_write_kb":55,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":40252,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.14","version":"79'22","reported_seq":99,"reported_epoch":535,"state":"active+clean","last_fresh":"2023-09-23T21:15:13.391846+0000","last_change":"2023-09-23T21:11:27.770162+0000","last_active":"2023-09-23T21:15:13.391846+0000","last_peered":"2023-09-23T21:15:13.391846+0000","last_clean":"2023-09-23T21:15:13.391846+0000","last_became_active":"2023-09-23T21:11:27.752275+0000","last_became_peered":"2023-09-23T21:11:27.752275+0000","last_unstale":"2023-09-23T21:15:13.391846+0000","last_undegraded":"2023-09-23T21:15:13.391846+0000","last_fullsized":"2023-09-23T21:15:13.391846+0000","mapping_epoch":452,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":453,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:05:11.769428+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4734,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,2],"acting":[4,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":464,"reported_epoch":535,"state":"active+clean","last_fresh":"2023-09-23T21:15:13.391024+0000","last_change":"2023-09-23T21:12:10.839271+0000","last_active":"2023-09-23T21:15:13.391024+0000","last_peered":"2023-09-23T21:15:13.391024+0000","last_clean":"2023-09-23T21:15:13.391024+0000","last_became_active":"2023-09-23T21:12:10.839042+0000","last_became_peered":"2023-09-23T21:12:10.839042+0000","last_unstale":"2023-09-23T21:15:13.391024+0000","last_undegraded":"2023-09-23T21:15:13.391024+0000","last_fullsized":"2023-09-23T21:15:13.391024+0000","mapping_epoch":470,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:10:26.414737+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,2],"acting":[4,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.11","version":"79'161","reported_seq":458,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-09-23T21:15:08.250088+0000","last_change":"2023-09-23T21:13:33.767406+0000","last_active":"2023-09-23T21:15:08.250088+0000","last_peered":"2023-09-23T21:15:08.250088+0000","last_clean":"2023-09-23T21:15:08.250088+0000","last_became_active":"2023-09-23T21:13:33.767067+0000","last_became_peered":"2023-09-23T21:13:33.767067+0000","last_unstale":"2023-09-23T21:15:08.250088+0000","last_undegraded":"2023-09-23T21:15:08.250088+0000","last_fullsized":"2023-09-23T21:15:08.250088+0000","mapping_epoch":504,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":505,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:28.756634+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:28.756634+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":23,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":144,"reported_epoch":535,"state":"active+clean","last_fresh":"2023-09-23T21:15:13.405246+0000","last_change":"2023-09-23T21:11:27.777154+0000","last_active":"2023-09-23T21:15:13.405246+0000","last_peered":"2023-09-23T21:15:13.405246+0000","last_clean":"2023-09-23T21:15:13.405246+0000","last_became_active":"2023-09-23T21:11:27.754438+0000","last_became_peered":"2023-09-23T21:11:27.754438+0000","last_unstale":"2023-09-23T21:15:13.405246+0000","last_undegraded":"2023-09-23T21:15:13.405246+0000","last_fullsized":"2023-09-23T21:15:13.405246+0000","mapping_epoch":452,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":453,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:33:02.845882+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":76,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-09-23T21:15:08.244588+0000","last_change":"2023-09-23T21:12:10.764264+0000","last_active":"2023-09-23T21:15:08.244588+0000","last_peered":"2023-09-23T21:15:08.244588+0000","last_clean":"2023-09-23T21:15:08.244588+0000","last_became_active":"2023-09-23T21:12:10.763486+0000","last_became_peered":"2023-09-23T21:12:10.763486+0000","last_unstale":"2023-09-23T21:15:08.244588+0000","last_undegraded":"2023-09-23T21:15:08.244588+0000","last_fullsized":"2023-09-23T21:15:08.244588+0000","mapping_epoch":470,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:32:27.577036+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,7],"acting":[2,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.10","version":"79'161","reported_seq":670,"reported_epoch":535,"state":"active+clean","last_fresh":"2023-09-23T21:15:13.390260+0000","last_change":"2023-09-23T21:11:11.716703+0000","last_active":"2023-09-23T21:15:13.390260+0000","last_peered":"2023-09-23T21:15:13.390260+0000","last_clean":"2023-09-23T21:15:13.390260+0000","last_became_active":"2023-09-23T21:11:11.716385+0000","last_became_peered":"2023-09-23T21:11:11.716385+0000","last_unstale":"2023-09-23T21:15:13.390260+0000","last_undegraded":"2023-09-23T21:15:13.390260+0000","last_fullsized":"2023-09-23T21:15:13.390260+0000","mapping_epoch":446,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":447,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:06:50.336992+0000","last_deep_scrub":"79'161","last_deep_scrub_stamp":"2023-09-23T21:06:50.336992+0000","last_clean_scrub_stamp":"2023-09-23T21:06:50.336992+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:44:29.538890+0000","scrub_duration":0.0026150930000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,7],"acting":[4,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":475,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-09-23T21:15:08.250953+0000","last_change":"2023-09-23T21:13:12.270990+0000","last_active":"2023-09-23T21:15:08.250953+0000","last_peered":"2023-09-23T21:15:08.250953+0000","last_clean":"2023-09-23T21:15:08.250953+0000","last_became_active":"2023-09-23T21:13:12.237201+0000","last_became_peered":"2023-09-23T21:13:12.237201+0000","last_unstale":"2023-09-23T21:15:08.250953+0000","last_undegraded":"2023-09-23T21:15:08.250953+0000","last_fullsized":"2023-09-23T21:15:08.250953+0000","mapping_epoch":493,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:59:54.523877+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":287,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-09-23T21:15:08.245400+0000","last_change":"2023-09-23T21:10:26.758671+0000","last_active":"2023-09-23T21:15:08.245400+0000","last_peered":"2023-09-23T21:15:08.245400+0000","last_clean":"2023-09-23T21:15:08.245400+0000","last_became_active":"2023-09-23T21:10:26.758244+0000","last_became_peered":"2023-09-23T21:10:26.758244+0000","last_unstale":"2023-09-23T21:15:08.245400+0000","last_undegraded":"2023-09-23T21:15:08.245400+0000","last_fullsized":"2023-09-23T21:15:08.245400+0000","mapping_epoch":423,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":424,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:14:02.735609+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.18","version":"535'6375","reported_seq":7467,"reported_epoch":535,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T21:15:14.922763+0000","last_change":"2023-09-23T21:13:13.412097+0000","last_active":"2023-09-23T21:15:14.922763+0000","last_peered":"2023-09-23T21:15:14.922763+0000","last_clean":"2023-09-23T21:11:45.720015+0000","last_became_active":"2023-09-23T21:13:13.395632+0000","last_became_peered":"2023-09-23T21:13:13.395632+0000","last_unstale":"2023-09-23T21:15:14.922763+0000","last_undegraded":"2023-09-23T21:13:13.220971+0000","last_fullsized":"2023-09-23T21:13:13.183825+0000","mapping_epoch":494,"log_start":"434'1327","ondisk_log_start":"434'1327","created":438,"last_epoch_clean":439,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_clean_scrub_stamp":"2023-09-23T21:10:15.663179+0000","objects_scrubbed":0,"log_size":5048,"log_dups_size":0,"ondisk_log_size":5048,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":884736,"num_objects":14,"num_object_clones":0,"num_object_copies":42,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":661,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":14,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5026,"num_write_kb":161248,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":231,"num_bytes_recovered":15138816,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,7],"acting":[2,7],"avail_no_missing":["2","7"],"object_location_counts":[{"shards":"2,7","objects":-20}],"blocked_by":[],"up_primary":0,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.13","version":"79'161","reported_seq":612,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-09-23T21:15:08.244882+0000","last_change":"2023-09-23T21:13:14.933096+0000","last_active":"2023-09-23T21:15:08.244882+0000","last_peered":"2023-09-23T21:15:08.244882+0000","last_clean":"2023-09-23T21:15:08.244882+0000","last_became_active":"2023-09-23T21:13:14.932703+0000","last_became_peered":"2023-09-23T21:13:14.932703+0000","last_unstale":"2023-09-23T21:15:08.244882+0000","last_undegraded":"2023-09-23T21:15:08.244882+0000","last_fullsized":"2023-09-23T21:15:08.244882+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:36.420362+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:36.420362+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:29:42.163911+0000","scrub_duration":0.028838444000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":527,"reported_epoch":535,"state":"active+clean","last_fresh":"2023-09-23T21:15:13.390331+0000","last_change":"2023-09-23T21:13:12.216467+0000","last_active":"2023-09-23T21:15:13.390331+0000","last_peered":"2023-09-23T21:15:13.390331+0000","last_clean":"2023-09-23T21:15:13.390331+0000","last_became_active":"2023-09-23T21:13:12.215259+0000","last_became_peered":"2023-09-23T21:13:12.215259+0000","last_unstale":"2023-09-23T21:15:13.390331+0000","last_undegraded":"2023-09-23T21:15:13.390331+0000","last_fullsized":"2023-09-23T21:15:13.390331+0000","mapping_epoch":493,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:35:33.987839+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":44,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-09-23T21:15:08.251669+0000","last_change":"2023-09-23T21:13:30.804108+0000","last_active":"2023-09-23T21:15:08.251669+0000","last_peered":"2023-09-23T21:15:08.251669+0000","last_clean":"2023-09-23T21:15:08.251669+0000","last_became_active":"2023-09-23T21:13:30.796505+0000","last_became_peered":"2023-09-23T21:13:30.796505+0000","last_unstale":"2023-09-23T21:15:08.251669+0000","last_undegraded":"2023-09-23T21:15:08.251669+0000","last_fullsized":"2023-09-23T21:15:08.251669+0000","mapping_epoch":501,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:19:33.508496+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.19","version":"535'6516","reported_seq":7622,"reported_epoch":535,"state":"active+clean","last_fresh":"2023-09-23T21:15:14.915740+0000","last_change":"2023-09-23T21:10:58.790716+0000","last_active":"2023-09-23T21:15:14.915740+0000","last_peered":"2023-09-23T21:15:14.915740+0000","last_clean":"2023-09-23T21:15:14.915740+0000","last_became_active":"2023-09-23T21:10:58.787975+0000","last_became_peered":"2023-09-23T21:10:58.787975+0000","last_unstale":"2023-09-23T21:15:14.915740+0000","last_undegraded":"2023-09-23T21:15:14.915740+0000","last_fullsized":"2023-09-23T21:15:14.915740+0000","mapping_epoch":438,"log_start":"435'1616","ondisk_log_start":"435'1616","created":438,"last_epoch_clean":439,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_clean_scrub_stamp":"2023-09-23T21:10:15.663179+0000","objects_scrubbed":0,"log_size":4900,"log_dups_size":0,"ondisk_log_size":4900,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5129,"num_write_kb":162912,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":237,"num_bytes_recovered":15499264,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.12","version":"79'161","reported_seq":769,"reported_epoch":535,"state":"active+clean","last_fresh":"2023-09-23T21:15:13.396424+0000","last_change":"2023-09-23T21:13:15.206574+0000","last_active":"2023-09-23T21:15:13.396424+0000","last_peered":"2023-09-23T21:15:13.396424+0000","last_clean":"2023-09-23T21:15:13.396424+0000","last_became_active":"2023-09-23T21:13:15.206186+0000","last_became_peered":"2023-09-23T21:13:15.206186+0000","last_unstale":"2023-09-23T21:15:13.396424+0000","last_undegraded":"2023-09-23T21:15:13.396424+0000","last_fullsized":"2023-09-23T21:15:13.396424+0000","mapping_epoch":496,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":497,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:05:56.785173+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:05:56.785173+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:34:29.221809+0000","scrub_duration":0.0042114370000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,0],"acting":[5,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":54,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-09-23T21:15:08.250525+0000","last_change":"2023-09-23T21:13:38.048791+0000","last_active":"2023-09-23T21:15:08.250525+0000","last_peered":"2023-09-23T21:15:08.250525+0000","last_clean":"2023-09-23T21:15:08.250525+0000","last_became_active":"2023-09-23T21:13:30.801386+0000","last_became_peered":"2023-09-23T21:13:30.801386+0000","last_unstale":"2023-09-23T21:15:08.250525+0000","last_undegraded":"2023-09-23T21:15:08.250525+0000","last_fullsized":"2023-09-23T21:15:08.250525+0000","mapping_epoch":501,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:52:49.170414+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":372,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-09-23T21:15:08.246592+0000","last_change":"2023-09-23T21:12:10.765780+0000","last_active":"2023-09-23T21:15:08.246592+0000","last_peered":"2023-09-23T21:15:08.246592+0000","last_clean":"2023-09-23T21:15:08.246592+0000","last_became_active":"2023-09-23T21:12:10.765372+0000","last_became_peered":"2023-09-23T21:12:10.765372+0000","last_unstale":"2023-09-23T21:15:08.246592+0000","last_undegraded":"2023-09-23T21:15:08.246592+0000","last_fullsized":"2023-09-23T21:15:08.246592+0000","mapping_epoch":470,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:56:08.446099+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.6","version":"535'6493","reported_seq":9417,"reported_epoch":535,"state":"active+clean","last_fresh":"2023-09-23T21:15:14.835820+0000","last_change":"2023-09-23T21:13:39.390285+0000","last_active":"2023-09-23T21:15:14.835820+0000","last_peered":"2023-09-23T21:15:14.835820+0000","last_clean":"2023-09-23T21:15:14.835820+0000","last_became_active":"2023-09-23T21:13:39.378284+0000","last_became_peered":"2023-09-23T21:13:39.378284+0000","last_unstale":"2023-09-23T21:15:14.835820+0000","last_undegraded":"2023-09-23T21:15:14.835820+0000","last_fullsized":"2023-09-23T21:15:14.835820+0000","mapping_epoch":508,"log_start":"435'2176","ondisk_log_start":"435'2176","created":416,"last_epoch_clean":509,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_clean_scrub_stamp":"2023-09-23T21:10:15.663179+0000","objects_scrubbed":0,"log_size":4317,"log_dups_size":0,"ondisk_log_size":4317,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5087,"num_write_kb":162112,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2399,"num_bytes_recovered":113967104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,4],"acting":[0,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.d","version":"79'161","reported_seq":689,"reported_epoch":535,"state":"active+clean","last_fresh":"2023-09-23T21:15:13.404876+0000","last_change":"2023-09-23T21:12:30.444461+0000","last_active":"2023-09-23T21:15:13.404876+0000","last_peered":"2023-09-23T21:15:13.404876+0000","last_clean":"2023-09-23T21:15:13.404876+0000","last_became_active":"2023-09-23T21:12:30.443979+0000","last_became_peered":"2023-09-23T21:12:30.443979+0000","last_unstale":"2023-09-23T21:15:13.404876+0000","last_undegraded":"2023-09-23T21:15:13.404876+0000","last_fullsized":"2023-09-23T21:15:13.404876+0000","mapping_epoch":470,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:09:36.858077+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:09:36.858077+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0024987500000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":182,"reported_epoch":535,"state":"active+clean","last_fresh":"2023-09-23T21:15:13.406332+0000","last_change":"2023-09-23T21:13:59.872834+0000","last_active":"2023-09-23T21:15:13.406332+0000","last_peered":"2023-09-23T21:15:13.406332+0000","last_clean":"2023-09-23T21:15:13.406332+0000","last_became_active":"2023-09-23T21:13:30.770556+0000","last_became_peered":"2023-09-23T21:13:30.770556+0000","last_unstale":"2023-09-23T21:15:13.406332+0000","last_undegraded":"2023-09-23T21:15:13.406332+0000","last_fullsized":"2023-09-23T21:15:13.406332+0000","mapping_epoch":501,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":521,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:57:55.460052+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":132,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":243,"reported_epoch":535,"state":"active+clean","last_fresh":"2023-09-23T21:15:13.395913+0000","last_change":"2023-09-23T21:10:26.750725+0000","last_active":"2023-09-23T21:15:13.395913+0000","last_peered":"2023-09-23T21:15:13.395913+0000","last_clean":"2023-09-23T21:15:13.395913+0000","last_became_active":"2023-09-23T21:10:26.750307+0000","last_became_peered":"2023-09-23T21:10:26.750307+0000","last_unstale":"2023-09-23T21:15:13.395913+0000","last_undegraded":"2023-09-23T21:15:13.395913+0000","last_fullsized":"2023-09-23T21:15:13.395913+0000","mapping_epoch":423,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":424,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:06:22.040548+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,4],"acting":[5,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.7","version":"535'6322","reported_seq":6525,"reported_epoch":535,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T21:15:14.114047+0000","last_change":"2023-09-23T21:13:38.417178+0000","last_active":"2023-09-23T21:15:14.114047+0000","last_peered":"2023-09-23T21:15:14.114047+0000","last_clean":"2023-09-23T21:11:26.699871+0000","last_became_active":"2023-09-23T21:13:38.408523+0000","last_became_peered":"2023-09-23T21:13:38.408523+0000","last_unstale":"2023-09-23T21:15:14.114047+0000","last_undegraded":"2023-09-23T21:13:38.369833+0000","last_fullsized":"2023-09-23T21:13:38.358198+0000","mapping_epoch":507,"log_start":"436'2567","ondisk_log_start":"436'2567","created":416,"last_epoch_clean":439,"parent":"0.0","parent_split_bits":0,"last_scrub":"441'2973","last_scrub_stamp":"2023-09-23T21:11:06.079326+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_clean_scrub_stamp":"2023-09-23T21:11:06.079326+0000","objects_scrubbed":1647,"log_size":3755,"log_dups_size":0,"ondisk_log_size":3755,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:27:59.805526+0000","scrub_duration":0.44319714500000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":1376256,"num_objects":21,"num_object_clones":0,"num_object_copies":63,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1165,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":21,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3354,"num_write_kb":55296,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,2],"acting":[3,5],"avail_no_missing":["3","5"],"object_location_counts":[{"shards":"3,4,5","objects":21},{"shards":"3,5","objects":-277}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.c","version":"79'161","reported_seq":904,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-09-23T21:15:09.226353+0000","last_change":"2023-09-23T21:15:06.730883+0000","last_active":"2023-09-23T21:15:09.226353+0000","last_peered":"2023-09-23T21:15:09.226353+0000","last_clean":"2023-09-23T21:15:09.226353+0000","last_became_active":"2023-09-23T21:13:13.401526+0000","last_became_peered":"2023-09-23T21:13:13.401526+0000","last_unstale":"2023-09-23T21:15:09.226353+0000","last_undegraded":"2023-09-23T21:15:09.226353+0000","last_fullsized":"2023-09-23T21:15:09.226353+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":532,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:22.040907+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:22.040907+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.013021176000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":163840,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":491520,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":168,"reported_epoch":535,"state":"active+clean","last_fresh":"2023-09-23T21:15:13.405788+0000","last_change":"2023-09-23T21:13:59.884113+0000","last_active":"2023-09-23T21:15:13.405788+0000","last_peered":"2023-09-23T21:15:13.405788+0000","last_clean":"2023-09-23T21:15:13.405788+0000","last_became_active":"2023-09-23T21:13:30.806778+0000","last_became_peered":"2023-09-23T21:13:30.806778+0000","last_unstale":"2023-09-23T21:15:13.405788+0000","last_undegraded":"2023-09-23T21:15:13.405788+0000","last_fullsized":"2023-09-23T21:15:13.405788+0000","mapping_epoch":501,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":521,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:36:15.284465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":498,"reported_epoch":535,"state":"active+clean","last_fresh":"2023-09-23T21:15:13.390359+0000","last_change":"2023-09-23T21:13:15.152792+0000","last_active":"2023-09-23T21:15:13.390359+0000","last_peered":"2023-09-23T21:15:13.390359+0000","last_clean":"2023-09-23T21:15:13.390359+0000","last_became_active":"2023-09-23T21:13:14.515084+0000","last_became_peered":"2023-09-23T21:13:14.515084+0000","last_unstale":"2023-09-23T21:15:13.390359+0000","last_undegraded":"2023-09-23T21:15:13.390359+0000","last_fullsized":"2023-09-23T21:15:13.390359+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:08:16.702072+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.4","version":"535'6301","reported_seq":13253,"reported_epoch":535,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T21:15:14.120808+0000","last_change":"2023-09-23T21:13:30.847170+0000","last_active":"2023-09-23T21:15:14.120808+0000","last_peered":"2023-09-23T21:15:14.120808+0000","last_clean":"2023-09-23T21:11:06.743842+0000","last_became_active":"2023-09-23T21:13:30.834806+0000","last_became_peered":"2023-09-23T21:13:30.834806+0000","last_unstale":"2023-09-23T21:15:14.120808+0000","last_undegraded":"2023-09-23T21:13:30.760230+0000","last_fullsized":"2023-09-23T21:13:30.749974+0000","mapping_epoch":501,"log_start":"436'2545","ondisk_log_start":"436'2545","created":416,"last_epoch_clean":439,"parent":"0.0","parent_split_bits":0,"last_scrub":"440'2861","last_scrub_stamp":"2023-09-23T21:11:03.842926+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_clean_scrub_stamp":"2023-09-23T21:11:03.842926+0000","objects_scrubbed":1526,"log_size":3756,"log_dups_size":0,"ondisk_log_size":3756,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:32:29.379756+0000","scrub_duration":0.39378383900000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":1310720,"num_objects":20,"num_object_clones":0,"num_object_copies":60,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1130,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":20,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3446,"num_write_kb":62080,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2723,"num_bytes_recovered":163577856,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[3,4],"avail_no_missing":["3","4"],"object_location_counts":[{"shards":"1,2,3,4","objects":20},{"shards":"3,4","objects":-340}],"blocked_by":[],"up_primary":0,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.f","version":"79'161","reported_seq":823,"reported_epoch":535,"state":"active+clean","last_fresh":"2023-09-23T21:15:13.396963+0000","last_change":"2023-09-23T21:11:30.750338+0000","last_active":"2023-09-23T21:15:13.396963+0000","last_peered":"2023-09-23T21:15:13.396963+0000","last_clean":"2023-09-23T21:15:13.396963+0000","last_became_active":"2023-09-23T21:11:30.749953+0000","last_became_peered":"2023-09-23T21:11:30.749953+0000","last_unstale":"2023-09-23T21:15:13.396963+0000","last_undegraded":"2023-09-23T21:15:13.396963+0000","last_fullsized":"2023-09-23T21:15:13.396963+0000","mapping_epoch":455,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":456,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:27.880776+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:27.880776+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:07:10.405723+0000","scrub_duration":0.0014004899999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,3],"acting":[5,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":259,"reported_epoch":535,"state":"active+clean","last_fresh":"2023-09-23T21:15:13.396508+0000","last_change":"2023-09-23T21:10:26.765068+0000","last_active":"2023-09-23T21:15:13.396508+0000","last_peered":"2023-09-23T21:15:13.396508+0000","last_clean":"2023-09-23T21:15:13.396508+0000","last_became_active":"2023-09-23T21:10:26.758364+0000","last_became_peered":"2023-09-23T21:10:26.758364+0000","last_unstale":"2023-09-23T21:15:13.396508+0000","last_undegraded":"2023-09-23T21:15:13.396508+0000","last_fullsized":"2023-09-23T21:15:13.396508+0000","mapping_epoch":423,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":424,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:13:21.057529+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":280,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":475,"reported_epoch":535,"state":"active+clean","last_fresh":"2023-09-23T21:15:13.396013+0000","last_change":"2023-09-23T21:13:30.770746+0000","last_active":"2023-09-23T21:15:13.396013+0000","last_peered":"2023-09-23T21:15:13.396013+0000","last_clean":"2023-09-23T21:15:13.396013+0000","last_became_active":"2023-09-23T21:13:30.770437+0000","last_became_peered":"2023-09-23T21:13:30.770437+0000","last_unstale":"2023-09-23T21:15:13.396013+0000","last_undegraded":"2023-09-23T21:15:13.396013+0000","last_fullsized":"2023-09-23T21:15:13.396013+0000","mapping_epoch":501,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:33:48.325730+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,0],"acting":[5,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.5","version":"535'6363","reported_seq":6557,"reported_epoch":535,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T21:15:14.865152+0000","last_change":"2023-09-23T21:13:13.436794+0000","last_active":"2023-09-23T21:15:14.865152+0000","last_peered":"2023-09-23T21:15:14.865152+0000","last_clean":"2023-09-23T21:11:45.768408+0000","last_became_active":"2023-09-23T21:13:13.424939+0000","last_became_peered":"2023-09-23T21:13:13.424939+0000","last_unstale":"2023-09-23T21:15:14.865152+0000","last_undegraded":"2023-09-23T21:13:13.225067+0000","last_fullsized":"2023-09-23T21:13:13.198194+0000","mapping_epoch":494,"log_start":"434'1366","ondisk_log_start":"434'1366","created":416,"last_epoch_clean":439,"parent":"0.0","parent_split_bits":0,"last_scrub":"440'2840","last_scrub_stamp":"2023-09-23T21:11:03.208833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_clean_scrub_stamp":"2023-09-23T21:11:03.208833+0000","objects_scrubbed":1498,"log_size":4997,"log_dups_size":0,"ondisk_log_size":4997,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:56:40.574477+0000","scrub_duration":0.45608471699999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":644,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3532,"num_write_kb":65088,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,3],"acting":[2,3],"avail_no_missing":["2","3"],"object_location_counts":[{"shards":"2,3","objects":-46}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.e","version":"79'161","reported_seq":704,"reported_epoch":535,"state":"active+clean","last_fresh":"2023-09-23T21:15:13.390780+0000","last_change":"2023-09-23T21:12:30.440952+0000","last_active":"2023-09-23T21:15:13.390780+0000","last_peered":"2023-09-23T21:15:13.390780+0000","last_clean":"2023-09-23T21:15:13.390780+0000","last_became_active":"2023-09-23T21:12:30.440463+0000","last_became_peered":"2023-09-23T21:12:30.440463+0000","last_unstale":"2023-09-23T21:15:13.390780+0000","last_undegraded":"2023-09-23T21:15:13.390780+0000","last_fullsized":"2023-09-23T21:15:13.390780+0000","mapping_epoch":455,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:10:39.567232+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:10:39.567232+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.005229694,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":332,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-09-23T21:15:08.244772+0000","last_change":"2023-09-23T21:13:13.402418+0000","last_active":"2023-09-23T21:15:08.244772+0000","last_peered":"2023-09-23T21:15:08.244772+0000","last_clean":"2023-09-23T21:15:08.244772+0000","last_became_active":"2023-09-23T21:13:13.400984+0000","last_became_peered":"2023-09-23T21:13:13.400984+0000","last_unstale":"2023-09-23T21:15:08.244772+0000","last_undegraded":"2023-09-23T21:15:08.244772+0000","last_fullsized":"2023-09-23T21:15:08.244772+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":495,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:28:01.411191+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,0],"acting":[3,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":507,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-09-23T21:15:08.251304+0000","last_change":"2023-09-23T21:13:14.339670+0000","last_active":"2023-09-23T21:15:08.251304+0000","last_peered":"2023-09-23T21:15:08.251304+0000","last_clean":"2023-09-23T21:15:08.251304+0000","last_became_active":"2023-09-23T21:13:14.339341+0000","last_became_peered":"2023-09-23T21:13:14.339341+0000","last_unstale":"2023-09-23T21:15:08.251304+0000","last_undegraded":"2023-09-23T21:15:08.251304+0000","last_fullsized":"2023-09-23T21:15:08.251304+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:23:19.411368+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.2","version":"535'6384","reported_seq":10373,"reported_epoch":535,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T21:15:14.940254+0000","last_change":"2023-09-23T21:13:13.407366+0000","last_active":"2023-09-23T21:15:14.940254+0000","last_peered":"2023-09-23T21:15:14.940254+0000","last_clean":"2023-09-23T21:11:45.752822+0000","last_became_active":"2023-09-23T21:13:13.398616+0000","last_became_peered":"2023-09-23T21:13:13.398616+0000","last_unstale":"2023-09-23T21:15:14.940254+0000","last_undegraded":"2023-09-23T21:13:13.289815+0000","last_fullsized":"2023-09-23T21:13:13.190270+0000","mapping_epoch":494,"log_start":"435'2215","ondisk_log_start":"435'2215","created":416,"last_epoch_clean":439,"parent":"0.0","parent_split_bits":0,"last_scrub":"440'3007","last_scrub_stamp":"2023-09-23T21:11:05.553661+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_clean_scrub_stamp":"2023-09-23T21:11:05.553661+0000","objects_scrubbed":1622,"log_size":4169,"log_dups_size":0,"ondisk_log_size":4169,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:51:21.839818+0000","scrub_duration":0.39156952499999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1258,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3382,"num_write_kb":56320,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1873,"num_bytes_recovered":122748928,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,3],"avail_no_missing":["4","1"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.9","version":"79'161","reported_seq":689,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-09-23T21:15:08.247392+0000","last_change":"2023-09-23T21:12:30.436118+0000","last_active":"2023-09-23T21:15:08.247392+0000","last_peered":"2023-09-23T21:15:08.247392+0000","last_clean":"2023-09-23T21:15:08.247392+0000","last_became_active":"2023-09-23T21:12:30.435779+0000","last_became_peered":"2023-09-23T21:12:30.435779+0000","last_unstale":"2023-09-23T21:15:08.247392+0000","last_undegraded":"2023-09-23T21:15:08.247392+0000","last_fullsized":"2023-09-23T21:15:08.247392+0000","mapping_epoch":433,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:10:37.696951+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:10:37.696951+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.001386504,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,3],"acting":[2,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":93,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-09-23T21:15:08.251398+0000","last_change":"2023-09-23T21:13:30.793273+0000","last_active":"2023-09-23T21:15:08.251398+0000","last_peered":"2023-09-23T21:15:08.251398+0000","last_clean":"2023-09-23T21:15:08.251398+0000","last_became_active":"2023-09-23T21:13:30.791269+0000","last_became_peered":"2023-09-23T21:13:30.791269+0000","last_unstale":"2023-09-23T21:15:08.251398+0000","last_undegraded":"2023-09-23T21:15:08.251398+0000","last_fullsized":"2023-09-23T21:15:08.251398+0000","mapping_epoch":501,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:38:07.142290+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":109,"reported_epoch":535,"state":"active+clean","last_fresh":"2023-09-23T21:15:13.391486+0000","last_change":"2023-09-23T21:12:14.313466+0000","last_active":"2023-09-23T21:15:13.391486+0000","last_peered":"2023-09-23T21:15:13.391486+0000","last_clean":"2023-09-23T21:15:13.391486+0000","last_became_active":"2023-09-23T21:12:14.209837+0000","last_became_peered":"2023-09-23T21:12:14.209837+0000","last_unstale":"2023-09-23T21:15:13.391486+0000","last_undegraded":"2023-09-23T21:15:13.391486+0000","last_fullsized":"2023-09-23T21:15:13.391486+0000","mapping_epoch":470,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":474,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:09:40.375241+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,7],"acting":[4,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.3","version":"535'6435","reported_seq":8573,"reported_epoch":535,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T21:15:14.894207+0000","last_change":"2023-09-23T21:13:45.480705+0000","last_active":"2023-09-23T21:15:14.894207+0000","last_peered":"2023-09-23T21:15:14.894207+0000","last_clean":"2023-09-23T21:11:26.700990+0000","last_became_active":"2023-09-23T21:13:45.471702+0000","last_became_peered":"2023-09-23T21:13:45.471702+0000","last_unstale":"2023-09-23T21:15:14.894207+0000","last_undegraded":"2023-09-23T21:13:45.412179+0000","last_fullsized":"2023-09-23T21:13:45.401701+0000","mapping_epoch":513,"log_start":"435'1468","ondisk_log_start":"435'1468","created":416,"last_epoch_clean":439,"parent":"0.0","parent_split_bits":0,"last_scrub":"440'2873","last_scrub_stamp":"2023-09-23T21:11:02.114783+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_clean_scrub_stamp":"2023-09-23T21:11:02.114783+0000","objects_scrubbed":1532,"log_size":4967,"log_dups_size":0,"ondisk_log_size":4967,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:42:48.549370+0000","scrub_duration":0.32435022600000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1184,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3568,"num_write_kb":65152,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":865,"num_bytes_recovered":56688640,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5],"avail_no_missing":["2","5"],"object_location_counts":[{"shards":"2,5","objects":-319}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.8","version":"79'161","reported_seq":655,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-09-23T21:15:08.245592+0000","last_change":"2023-09-23T21:12:30.433561+0000","last_active":"2023-09-23T21:15:08.245592+0000","last_peered":"2023-09-23T21:15:08.245592+0000","last_clean":"2023-09-23T21:15:08.245592+0000","last_became_active":"2023-09-23T21:12:30.433217+0000","last_became_peered":"2023-09-23T21:12:30.433217+0000","last_unstale":"2023-09-23T21:15:08.245592+0000","last_undegraded":"2023-09-23T21:15:08.245592+0000","last_fullsized":"2023-09-23T21:15:08.245592+0000","mapping_epoch":359,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:06:52.098882+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:06:52.098882+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0092173299999999993,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":131344,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":112,"reported_epoch":535,"state":"active+clean","last_fresh":"2023-09-23T21:15:13.392083+0000","last_change":"2023-09-23T21:12:10.823380+0000","last_active":"2023-09-23T21:15:13.392083+0000","last_peered":"2023-09-23T21:15:13.392083+0000","last_clean":"2023-09-23T21:15:13.392083+0000","last_became_active":"2023-09-23T21:12:10.823044+0000","last_became_peered":"2023-09-23T21:12:10.823044+0000","last_unstale":"2023-09-23T21:15:13.392083+0000","last_undegraded":"2023-09-23T21:15:13.392083+0000","last_fullsized":"2023-09-23T21:15:13.392083+0000","mapping_epoch":470,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:53:29.846536+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,5],"acting":[4,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":113,"reported_epoch":535,"state":"active+clean","last_fresh":"2023-09-23T21:15:13.390599+0000","last_change":"2023-09-23T21:13:30.772451+0000","last_active":"2023-09-23T21:15:13.390599+0000","last_peered":"2023-09-23T21:15:13.390599+0000","last_clean":"2023-09-23T21:15:13.390599+0000","last_became_active":"2023-09-23T21:13:30.772040+0000","last_became_peered":"2023-09-23T21:13:30.772040+0000","last_unstale":"2023-09-23T21:15:13.390599+0000","last_undegraded":"2023-09-23T21:15:13.390599+0000","last_fullsized":"2023-09-23T21:15:13.390599+0000","mapping_epoch":501,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:24:49.246727+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,0],"acting":[4,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.0","version":"535'6354","reported_seq":14681,"reported_epoch":535,"state":"active+clean","last_fresh":"2023-09-23T21:15:14.933728+0000","last_change":"2023-09-23T21:15:08.273520+0000","last_active":"2023-09-23T21:15:14.933728+0000","last_peered":"2023-09-23T21:15:14.933728+0000","last_clean":"2023-09-23T21:15:14.933728+0000","last_became_active":"2023-09-23T21:15:08.252420+0000","last_became_peered":"2023-09-23T21:15:08.252420+0000","last_unstale":"2023-09-23T21:15:14.933728+0000","last_undegraded":"2023-09-23T21:15:14.933728+0000","last_fullsized":"2023-09-23T21:15:14.933728+0000","mapping_epoch":533,"log_start":"435'2206","ondisk_log_start":"435'2206","created":416,"last_epoch_clean":534,"parent":"0.0","parent_split_bits":0,"last_scrub":"440'2961","last_scrub_stamp":"2023-09-23T21:11:04.564028+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_clean_scrub_stamp":"2023-09-23T21:11:04.564028+0000","objects_scrubbed":1608,"log_size":4148,"log_dups_size":0,"ondisk_log_size":4148,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:28:18.331836+0000","scrub_duration":0.41858158299999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3396,"num_write_kb":57216,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3149,"num_bytes_recovered":153747456,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.b","version":"79'161","reported_seq":538,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-09-23T21:15:08.251112+0000","last_change":"2023-09-23T21:13:13.373313+0000","last_active":"2023-09-23T21:15:08.251112+0000","last_peered":"2023-09-23T21:15:08.251112+0000","last_clean":"2023-09-23T21:15:08.251112+0000","last_became_active":"2023-09-23T21:13:13.372328+0000","last_became_peered":"2023-09-23T21:13:13.372328+0000","last_unstale":"2023-09-23T21:15:08.251112+0000","last_undegraded":"2023-09-23T21:15:08.251112+0000","last_fullsized":"2023-09-23T21:15:08.251112+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":495,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:05:55.772264+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:05:55.772264+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.001862507,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":93,"reported_epoch":535,"state":"active+clean","last_fresh":"2023-09-23T21:15:13.390244+0000","last_change":"2023-09-23T21:11:27.754717+0000","last_active":"2023-09-23T21:15:13.390244+0000","last_peered":"2023-09-23T21:15:13.390244+0000","last_clean":"2023-09-23T21:15:13.390244+0000","last_became_active":"2023-09-23T21:11:27.754376+0000","last_became_peered":"2023-09-23T21:11:27.754376+0000","last_unstale":"2023-09-23T21:15:13.390244+0000","last_undegraded":"2023-09-23T21:15:13.390244+0000","last_fullsized":"2023-09-23T21:15:13.390244+0000","mapping_epoch":452,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":453,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:26:41.734541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":467,"reported_epoch":535,"state":"active+clean","last_fresh":"2023-09-23T21:15:13.391126+0000","last_change":"2023-09-23T21:12:10.811698+0000","last_active":"2023-09-23T21:15:13.391126+0000","last_peered":"2023-09-23T21:15:13.391126+0000","last_clean":"2023-09-23T21:15:13.391126+0000","last_became_active":"2023-09-23T21:12:10.789656+0000","last_became_peered":"2023-09-23T21:12:10.789656+0000","last_unstale":"2023-09-23T21:15:13.391126+0000","last_undegraded":"2023-09-23T21:15:13.391126+0000","last_fullsized":"2023-09-23T21:15:13.391126+0000","mapping_epoch":470,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:39:15.112091+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,5],"acting":[4,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.1","version":"535'6269","reported_seq":12691,"reported_epoch":535,"state":"active+clean","last_fresh":"2023-09-23T21:15:14.918095+0000","last_change":"2023-09-23T21:13:53.483736+0000","last_active":"2023-09-23T21:15:14.918095+0000","last_peered":"2023-09-23T21:15:14.918095+0000","last_clean":"2023-09-23T21:15:14.918095+0000","last_became_active":"2023-09-23T21:13:53.472685+0000","last_became_peered":"2023-09-23T21:13:53.472685+0000","last_unstale":"2023-09-23T21:15:14.918095+0000","last_undegraded":"2023-09-23T21:15:14.918095+0000","last_fullsized":"2023-09-23T21:15:14.918095+0000","mapping_epoch":519,"log_start":"435'2038","ondisk_log_start":"435'2038","created":416,"last_epoch_clean":520,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_clean_scrub_stamp":"2023-09-23T21:10:15.663179+0000","objects_scrubbed":0,"log_size":4231,"log_dups_size":0,"ondisk_log_size":4231,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":1507328,"num_objects":23,"num_object_clones":0,"num_object_copies":69,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":23,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4921,"num_write_kb":158208,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3005,"num_bytes_recovered":156499968,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.a","version":"79'161","reported_seq":681,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-09-23T21:15:08.247519+0000","last_change":"2023-09-23T21:12:30.437790+0000","last_active":"2023-09-23T21:15:08.247519+0000","last_peered":"2023-09-23T21:15:08.247519+0000","last_clean":"2023-09-23T21:15:08.247519+0000","last_became_active":"2023-09-23T21:12:30.437427+0000","last_became_peered":"2023-09-23T21:12:30.437427+0000","last_unstale":"2023-09-23T21:15:08.247519+0000","last_undegraded":"2023-09-23T21:15:08.247519+0000","last_fullsized":"2023-09-23T21:15:08.247519+0000","mapping_epoch":446,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:06:51.075448+0000","last_deep_scrub":"79'161","last_deep_scrub_stamp":"2023-09-23T21:05:32.630292+0000","last_clean_scrub_stamp":"2023-09-23T21:06:51.075448+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0054793890000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,4],"acting":[2,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":382,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-09-23T21:15:08.242219+0000","last_change":"2023-09-23T21:12:10.792918+0000","last_active":"2023-09-23T21:15:08.242219+0000","last_peered":"2023-09-23T21:15:08.242219+0000","last_clean":"2023-09-23T21:15:08.242219+0000","last_became_active":"2023-09-23T21:12:10.792679+0000","last_became_peered":"2023-09-23T21:12:10.792679+0000","last_unstale":"2023-09-23T21:15:08.242219+0000","last_undegraded":"2023-09-23T21:15:08.242219+0000","last_fullsized":"2023-09-23T21:15:08.242219+0000","mapping_epoch":470,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:30:48.008690+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,4],"acting":[3,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":513,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-09-23T21:15:08.246866+0000","last_change":"2023-09-23T21:12:10.790812+0000","last_active":"2023-09-23T21:15:08.246866+0000","last_peered":"2023-09-23T21:15:08.246866+0000","last_clean":"2023-09-23T21:15:08.246866+0000","last_became_active":"2023-09-23T21:12:10.790492+0000","last_became_peered":"2023-09-23T21:12:10.790492+0000","last_unstale":"2023-09-23T21:15:08.246866+0000","last_undegraded":"2023-09-23T21:15:08.246866+0000","last_fullsized":"2023-09-23T21:15:08.246866+0000","mapping_epoch":470,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:02:19.863118+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.c","version":"535'10038","reported_seq":13883,"reported_epoch":536,"state":"peering","last_fresh":"2023-09-23T21:15:16.787488+0000","last_change":"2023-09-23T21:15:16.788670+0000","last_active":"2023-09-23T21:15:16.787461+0000","last_peered":"2023-09-23T21:15:16.351370+0000","last_clean":"2023-09-23T21:11:45.750376+0000","last_became_active":"2023-09-23T21:13:13.404327+0000","last_became_peered":"2023-09-23T21:13:13.404327+0000","last_unstale":"2023-09-23T21:15:16.787488+0000","last_undegraded":"2023-09-23T21:15:16.787488+0000","last_fullsized":"2023-09-23T21:15:16.787488+0000","mapping_epoch":536,"log_start":"482'5651","ondisk_log_start":"482'5651","created":416,"last_epoch_clean":434,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_clean_scrub_stamp":"2023-09-23T21:10:15.663179+0000","objects_scrubbed":0,"log_size":4387,"log_dups_size":0,"ondisk_log_size":4387,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:29:23.899790+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10038,"num_write_kb":321216,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1837,"num_bytes_recovered":36175872,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[0,4],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.7","version":"79'161","reported_seq":627,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-09-23T21:15:08.246503+0000","last_change":"2023-09-23T21:13:44.416548+0000","last_active":"2023-09-23T21:15:08.246503+0000","last_peered":"2023-09-23T21:15:08.246503+0000","last_clean":"2023-09-23T21:15:08.246503+0000","last_became_active":"2023-09-23T21:13:44.415617+0000","last_became_peered":"2023-09-23T21:13:44.415617+0000","last_unstale":"2023-09-23T21:15:08.246503+0000","last_undegraded":"2023-09-23T21:15:08.246503+0000","last_fullsized":"2023-09-23T21:15:08.246503+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":3,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:25.916867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:25.916867+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.001665323,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":464,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-09-23T21:15:08.251592+0000","last_change":"2023-09-23T21:13:13.211959+0000","last_active":"2023-09-23T21:15:08.251592+0000","last_peered":"2023-09-23T21:15:08.251592+0000","last_clean":"2023-09-23T21:15:08.251592+0000","last_became_active":"2023-09-23T21:13:13.211712+0000","last_became_peered":"2023-09-23T21:13:13.211712+0000","last_unstale":"2023-09-23T21:15:08.251592+0000","last_undegraded":"2023-09-23T21:15:08.251592+0000","last_fullsized":"2023-09-23T21:15:08.251592+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":495,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:37:21.655775+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,2],"acting":[0,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":213,"reported_epoch":535,"state":"active+clean","last_fresh":"2023-09-23T21:15:13.396430+0000","last_change":"2023-09-23T21:05:04.326889+0000","last_active":"2023-09-23T21:15:13.396430+0000","last_peered":"2023-09-23T21:15:13.396430+0000","last_clean":"2023-09-23T21:15:13.396430+0000","last_became_active":"2023-09-23T21:05:04.326032+0000","last_became_peered":"2023-09-23T21:05:04.326032+0000","last_unstale":"2023-09-23T21:15:13.396430+0000","last_undegraded":"2023-09-23T21:15:13.396430+0000","last_fullsized":"2023-09-23T21:15:13.396430+0000","mapping_epoch":332,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:00:17.507889+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":545,"reported_epoch":535,"state":"active+clean","last_fresh":"2023-09-23T21:15:13.396165+0000","last_change":"2023-09-23T21:13:13.285608+0000","last_active":"2023-09-23T21:15:13.396165+0000","last_peered":"2023-09-23T21:15:13.396165+0000","last_clean":"2023-09-23T21:15:13.396165+0000","last_became_active":"2023-09-23T21:13:13.285255+0000","last_became_peered":"2023-09-23T21:13:13.285255+0000","last_unstale":"2023-09-23T21:15:13.396165+0000","last_undegraded":"2023-09-23T21:15:13.396165+0000","last_fullsized":"2023-09-23T21:15:13.396165+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":495,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:42:30.087599+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.a","version":"535'9872","reported_seq":18614,"reported_epoch":535,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T21:15:14.895870+0000","last_change":"2023-09-23T21:13:31.927282+0000","last_active":"2023-09-23T21:15:14.895870+0000","last_peered":"2023-09-23T21:15:14.895870+0000","last_clean":"2023-09-23T21:13:30.353360+0000","last_became_active":"2023-09-23T21:13:31.896923+0000","last_became_peered":"2023-09-23T21:13:31.896923+0000","last_unstale":"2023-09-23T21:15:14.895870+0000","last_undegraded":"2023-09-23T21:13:31.776904+0000","last_fullsized":"2023-09-23T21:13:31.751334+0000","mapping_epoch":502,"log_start":"480'5512","ondisk_log_start":"480'5512","created":416,"last_epoch_clean":499,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_clean_scrub_stamp":"2023-09-23T21:10:15.663179+0000","objects_scrubbed":0,"log_size":4360,"log_dups_size":0,"ondisk_log_size":4360,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:23:14.477602+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2243,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9872,"num_write_kb":315904,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4293,"num_bytes_recovered":281346048,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[2,5],"avail_no_missing":["2","3","5"],"object_location_counts":[{"shards":"2,5","objects":-643}],"blocked_by":[],"up_primary":0,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.1","version":"79'161","reported_seq":606,"reported_epoch":535,"state":"active+clean","last_fresh":"2023-09-23T21:15:13.409694+0000","last_change":"2023-09-23T21:13:12.204135+0000","last_active":"2023-09-23T21:15:13.409694+0000","last_peered":"2023-09-23T21:15:13.409694+0000","last_clean":"2023-09-23T21:15:13.409694+0000","last_became_active":"2023-09-23T21:13:12.203380+0000","last_became_peered":"2023-09-23T21:13:12.203380+0000","last_unstale":"2023-09-23T21:15:13.409694+0000","last_undegraded":"2023-09-23T21:15:13.409694+0000","last_fullsized":"2023-09-23T21:15:13.409694+0000","mapping_epoch":493,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:06:52.575900+0000","last_deep_scrub":"79'161","last_deep_scrub_stamp":"2023-09-23T21:05:30.652189+0000","last_clean_scrub_stamp":"2023-09-23T21:06:52.575900+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":201,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-09-23T21:15:08.250897+0000","last_change":"2023-09-23T21:13:13.238617+0000","last_active":"2023-09-23T21:15:08.250897+0000","last_peered":"2023-09-23T21:15:08.250897+0000","last_clean":"2023-09-23T21:15:08.250897+0000","last_became_active":"2023-09-23T21:13:13.232926+0000","last_became_peered":"2023-09-23T21:13:13.232926+0000","last_unstale":"2023-09-23T21:15:08.250897+0000","last_undegraded":"2023-09-23T21:15:08.250897+0000","last_fullsized":"2023-09-23T21:15:08.250897+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":495,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:17:21.823772+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":469,"reported_epoch":535,"state":"active+clean","last_fresh":"2023-09-23T21:15:13.391092+0000","last_change":"2023-09-23T21:13:14.144918+0000","last_active":"2023-09-23T21:15:13.391092+0000","last_peered":"2023-09-23T21:15:13.391092+0000","last_clean":"2023-09-23T21:15:13.391092+0000","last_became_active":"2023-09-23T21:13:14.132388+0000","last_became_peered":"2023-09-23T21:13:14.132388+0000","last_unstale":"2023-09-23T21:15:13.391092+0000","last_undegraded":"2023-09-23T21:15:13.391092+0000","last_fullsized":"2023-09-23T21:15:13.391092+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":495,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:56:02.619905+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,5],"acting":[4,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":355,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-09-23T21:15:08.244811+0000","last_change":"2023-09-23T21:12:10.764503+0000","last_active":"2023-09-23T21:15:08.244811+0000","last_peered":"2023-09-23T21:15:08.244811+0000","last_clean":"2023-09-23T21:15:08.244811+0000","last_became_active":"2023-09-23T21:12:10.763933+0000","last_became_peered":"2023-09-23T21:12:10.763933+0000","last_unstale":"2023-09-23T21:15:08.244811+0000","last_undegraded":"2023-09-23T21:15:08.244811+0000","last_fullsized":"2023-09-23T21:15:08.244811+0000","mapping_epoch":470,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:28:33.202825+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,7],"acting":[2,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"10.9","version":"535'6353","reported_seq":7504,"reported_epoch":535,"state":"active+clean","last_fresh":"2023-09-23T21:15:14.805420+0000","last_change":"2023-09-23T21:11:08.062606+0000","last_active":"2023-09-23T21:15:14.805420+0000","last_peered":"2023-09-23T21:15:14.805420+0000","last_clean":"2023-09-23T21:15:14.805420+0000","last_became_active":"2023-09-23T21:10:58.780961+0000","last_became_peered":"2023-09-23T21:10:58.780961+0000","last_unstale":"2023-09-23T21:15:14.805420+0000","last_undegraded":"2023-09-23T21:15:14.805420+0000","last_fullsized":"2023-09-23T21:15:14.805420+0000","mapping_epoch":428,"log_start":"434'1179","ondisk_log_start":"434'1179","created":416,"last_epoch_clean":439,"parent":"0.0","parent_split_bits":0,"last_scrub":"443'3049","last_scrub_stamp":"2023-09-23T21:11:08.062479+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_clean_scrub_stamp":"2023-09-23T21:11:08.062479+0000","objects_scrubbed":1623,"log_size":5174,"log_dups_size":0,"ondisk_log_size":5174,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:56:22.338088+0000","scrub_duration":0.28207626200000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3305,"num_write_kb":53824,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.2","version":"79'161","reported_seq":815,"reported_epoch":535,"state":"active+clean","last_fresh":"2023-09-23T21:15:13.415089+0000","last_change":"2023-09-23T21:13:12.209907+0000","last_active":"2023-09-23T21:15:13.415089+0000","last_peered":"2023-09-23T21:15:13.415089+0000","last_clean":"2023-09-23T21:15:13.415089+0000","last_became_active":"2023-09-23T21:13:12.205664+0000","last_became_peered":"2023-09-23T21:13:12.205664+0000","last_unstale":"2023-09-23T21:15:13.415089+0000","last_undegraded":"2023-09-23T21:15:13.415089+0000","last_fullsized":"2023-09-23T21:15:13.415089+0000","mapping_epoch":493,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:24.802451+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:24.802451+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:26:16.035788+0000","scrub_duration":0.0090861250000000005,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,4],"acting":[0,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":543,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-09-23T21:15:08.247706+0000","last_change":"2023-09-23T21:13:12.206980+0000","last_active":"2023-09-23T21:15:08.247706+0000","last_peered":"2023-09-23T21:15:08.247706+0000","last_clean":"2023-09-23T21:15:08.247706+0000","last_became_active":"2023-09-23T21:13:12.206650+0000","last_became_peered":"2023-09-23T21:13:12.206650+0000","last_unstale":"2023-09-23T21:15:08.247706+0000","last_undegraded":"2023-09-23T21:15:08.247706+0000","last_fullsized":"2023-09-23T21:15:08.247706+0000","mapping_epoch":493,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:10:01.892936+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":527,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-09-23T21:15:08.245688+0000","last_change":"2023-09-23T21:13:13.407073+0000","last_active":"2023-09-23T21:15:08.245688+0000","last_peered":"2023-09-23T21:15:08.245688+0000","last_clean":"2023-09-23T21:15:08.245688+0000","last_became_active":"2023-09-23T21:13:13.395047+0000","last_became_peered":"2023-09-23T21:13:13.395047+0000","last_unstale":"2023-09-23T21:15:08.245688+0000","last_undegraded":"2023-09-23T21:15:08.245688+0000","last_fullsized":"2023-09-23T21:15:08.245688+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":495,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:04:43.384190+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,5],"acting":[2,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":44,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-09-23T21:15:08.250786+0000","last_change":"2023-09-23T21:13:30.785054+0000","last_active":"2023-09-23T21:15:08.250786+0000","last_peered":"2023-09-23T21:15:08.250786+0000","last_clean":"2023-09-23T21:15:08.250786+0000","last_became_active":"2023-09-23T21:13:30.784701+0000","last_became_peered":"2023-09-23T21:13:30.784701+0000","last_unstale":"2023-09-23T21:15:08.250786+0000","last_undegraded":"2023-09-23T21:15:08.250786+0000","last_fullsized":"2023-09-23T21:15:08.250786+0000","mapping_epoch":501,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:19:20.152252+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.b","version":"535'10019","reported_seq":21056,"reported_epoch":535,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2023-09-23T21:15:13.698752+0000","last_change":"2023-09-23T21:13:59.891994+0000","last_active":"2023-09-23T21:15:13.698752+0000","last_peered":"2023-09-23T21:15:13.698752+0000","last_clean":"2023-09-23T21:11:26.732248+0000","last_became_active":"2023-09-23T21:13:14.937458+0000","last_became_peered":"2023-09-23T21:13:14.937458+0000","last_unstale":"2023-09-23T21:15:13.698752+0000","last_undegraded":"2023-09-23T21:13:59.884123+0000","last_fullsized":"2023-09-23T21:15:13.698752+0000","mapping_epoch":494,"log_start":"426'600","ondisk_log_start":"426'600","created":416,"last_epoch_clean":430,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_clean_scrub_stamp":"2023-09-23T21:10:15.663179+0000","objects_scrubbed":0,"log_size":9419,"log_dups_size":0,"ondisk_log_size":9419,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:36:15.760947+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4784128,"num_objects":73,"num_object_clones":0,"num_object_copies":219,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":8,"num_objects_misplaced":1337,"num_objects_unfound":0,"num_objects_dirty":73,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10019,"num_write_kb":322944,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5424,"num_bytes_recovered":355270656,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[1,4,7],"avail_no_missing":["1","7"],"object_location_counts":[{"shards":"1,4,7","objects":-2}],"blocked_by":[],"up_primary":0,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":320,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-09-23T21:15:08.246383+0000","last_change":"2023-09-23T21:10:26.769764+0000","last_active":"2023-09-23T21:15:08.246383+0000","last_peered":"2023-09-23T21:15:08.246383+0000","last_clean":"2023-09-23T21:15:08.246383+0000","last_became_active":"2023-09-23T21:10:26.769326+0000","last_became_peered":"2023-09-23T21:10:26.769326+0000","last_unstale":"2023-09-23T21:15:08.246383+0000","last_undegraded":"2023-09-23T21:15:08.246383+0000","last_fullsized":"2023-09-23T21:15:08.246383+0000","mapping_epoch":423,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":424,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:57:45.219837+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"79'161","reported_seq":717,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-09-23T21:15:08.250124+0000","last_change":"2023-09-23T21:13:33.766895+0000","last_active":"2023-09-23T21:15:08.250124+0000","last_peered":"2023-09-23T21:15:08.250124+0000","last_clean":"2023-09-23T21:15:08.250124+0000","last_became_active":"2023-09-23T21:13:33.766538+0000","last_became_peered":"2023-09-23T21:13:33.766538+0000","last_unstale":"2023-09-23T21:15:08.250124+0000","last_undegraded":"2023-09-23T21:15:08.250124+0000","last_fullsized":"2023-09-23T21:15:08.250124+0000","mapping_epoch":504,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":505,"parent":"0.0","parent_split_bits":0,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:20.738273+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:20.738273+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:35:35.855987+0000","scrub_duration":0.0088336339999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,2],"acting":[0,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":543,"reported_epoch":535,"state":"active+clean","last_fresh":"2023-09-23T21:15:13.396092+0000","last_change":"2023-09-23T21:11:08.760417+0000","last_active":"2023-09-23T21:15:13.396092+0000","last_peered":"2023-09-23T21:15:13.396092+0000","last_clean":"2023-09-23T21:15:13.396092+0000","last_became_active":"2023-09-23T21:11:08.760150+0000","last_became_peered":"2023-09-23T21:11:08.760150+0000","last_unstale":"2023-09-23T21:15:13.396092+0000","last_undegraded":"2023-09-23T21:15:13.396092+0000","last_fullsized":"2023-09-23T21:15:13.396092+0000","mapping_epoch":443,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":444,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:23:10.849768+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":110,"reported_epoch":535,"state":"active+clean","last_fresh":"2023-09-23T21:15:13.390769+0000","last_change":"2023-09-23T21:13:12.216020+0000","last_active":"2023-09-23T21:15:13.390769+0000","last_peered":"2023-09-23T21:15:13.390769+0000","last_clean":"2023-09-23T21:15:13.390769+0000","last_became_active":"2023-09-23T21:13:12.215180+0000","last_became_peered":"2023-09-23T21:13:12.215180+0000","last_unstale":"2023-09-23T21:15:13.390769+0000","last_undegraded":"2023-09-23T21:15:13.390769+0000","last_fullsized":"2023-09-23T21:15:13.390769+0000","mapping_epoch":493,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:26:18.704684+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.8","version":"535'6192","reported_seq":8593,"reported_epoch":535,"state":"active+clean","last_fresh":"2023-09-23T21:15:14.908231+0000","last_change":"2023-09-23T21:14:03.796930+0000","last_active":"2023-09-23T21:15:14.908231+0000","last_peered":"2023-09-23T21:15:14.908231+0000","last_clean":"2023-09-23T21:15:14.908231+0000","last_became_active":"2023-09-23T21:14:03.786177+0000","last_became_peered":"2023-09-23T21:14:03.786177+0000","last_unstale":"2023-09-23T21:15:14.908231+0000","last_undegraded":"2023-09-23T21:15:14.908231+0000","last_fullsized":"2023-09-23T21:15:14.908231+0000","mapping_epoch":524,"log_start":"435'1693","ondisk_log_start":"435'1693","created":416,"last_epoch_clean":525,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_clean_scrub_stamp":"2023-09-23T21:10:15.663179+0000","objects_scrubbed":0,"log_size":4499,"log_dups_size":0,"ondisk_log_size":4499,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4842,"num_write_kb":154528,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":880,"num_bytes_recovered":16908288,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,7],"acting":[0,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.3","version":"79'161","reported_seq":838,"reported_epoch":535,"state":"active+clean","last_fresh":"2023-09-23T21:15:13.390249+0000","last_change":"2023-09-23T21:13:33.771279+0000","last_active":"2023-09-23T21:15:13.390249+0000","last_peered":"2023-09-23T21:15:13.390249+0000","last_clean":"2023-09-23T21:15:13.390249+0000","last_became_active":"2023-09-23T21:13:33.770980+0000","last_became_peered":"2023-09-23T21:13:33.770980+0000","last_unstale":"2023-09-23T21:15:13.390249+0000","last_undegraded":"2023-09-23T21:15:13.390249+0000","last_fullsized":"2023-09-23T21:15:13.390249+0000","mapping_epoch":504,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":505,"parent":"0.0","parent_split_bits":3,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:05:57.832302+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:05:57.832302+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:19:42.309604+0000","scrub_duration":0.010206847999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":352,"reported_epoch":535,"state":"active+clean","last_fresh":"2023-09-23T21:15:13.414720+0000","last_change":"2023-09-23T21:13:13.212775+0000","last_active":"2023-09-23T21:15:13.414720+0000","last_peered":"2023-09-23T21:15:13.414720+0000","last_clean":"2023-09-23T21:15:13.414720+0000","last_became_active":"2023-09-23T21:13:13.212460+0000","last_became_peered":"2023-09-23T21:13:13.212460+0000","last_unstale":"2023-09-23T21:15:13.414720+0000","last_undegraded":"2023-09-23T21:15:13.414720+0000","last_fullsized":"2023-09-23T21:15:13.414720+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":495,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:27:10.853970+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":534,"reported_epoch":535,"state":"active+clean","last_fresh":"2023-09-23T21:15:13.409516+0000","last_change":"2023-09-23T21:13:44.409212+0000","last_active":"2023-09-23T21:15:13.409516+0000","last_peered":"2023-09-23T21:15:13.409516+0000","last_clean":"2023-09-23T21:15:13.409516+0000","last_became_active":"2023-09-23T21:13:44.408958+0000","last_became_peered":"2023-09-23T21:13:44.408958+0000","last_unstale":"2023-09-23T21:15:13.409516+0000","last_undegraded":"2023-09-23T21:15:13.409516+0000","last_fullsized":"2023-09-23T21:15:13.409516+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:03:52.749964+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":44,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-09-23T21:15:08.251753+0000","last_change":"2023-09-23T21:13:30.804379+0000","last_active":"2023-09-23T21:15:08.251753+0000","last_peered":"2023-09-23T21:15:08.251753+0000","last_clean":"2023-09-23T21:15:08.251753+0000","last_became_active":"2023-09-23T21:13:30.799907+0000","last_became_peered":"2023-09-23T21:13:30.799907+0000","last_unstale":"2023-09-23T21:15:08.251753+0000","last_undegraded":"2023-09-23T21:15:08.251753+0000","last_fullsized":"2023-09-23T21:15:08.251753+0000","mapping_epoch":501,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:49:42.859212+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,7],"acting":[0,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.d","version":"535'10002","reported_seq":10329,"reported_epoch":535,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-23T21:15:14.945918+0000","last_change":"2023-09-23T21:13:31.801486+0000","last_active":"2023-09-23T21:15:14.945918+0000","last_peered":"2023-09-23T21:15:14.945918+0000","last_clean":"2023-09-23T21:11:37.659929+0000","last_became_active":"2023-09-23T21:13:31.786626+0000","last_became_peered":"2023-09-23T21:13:31.786626+0000","last_unstale":"2023-09-23T21:15:14.945918+0000","last_undegraded":"2023-09-23T21:13:31.760908+0000","last_fullsized":"2023-09-23T21:15:14.945918+0000","mapping_epoch":502,"log_start":"489'6095","ondisk_log_start":"489'6095","created":416,"last_epoch_clean":417,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_clean_scrub_stamp":"2023-09-23T21:10:15.663179+0000","objects_scrubbed":0,"log_size":3907,"log_dups_size":0,"ondisk_log_size":3907,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:46:02.424493+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1186,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":10002,"num_write_kb":320001,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":79,"num_bytes_recovered":5111832,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,3],"avail_no_missing":["4","3","7"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.6","version":"79'161","reported_seq":453,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-09-23T21:15:08.251715+0000","last_change":"2023-09-23T21:13:33.770715+0000","last_active":"2023-09-23T21:15:08.251715+0000","last_peered":"2023-09-23T21:15:08.251715+0000","last_clean":"2023-09-23T21:15:08.251715+0000","last_became_active":"2023-09-23T21:13:33.769303+0000","last_became_peered":"2023-09-23T21:13:33.769303+0000","last_unstale":"2023-09-23T21:15:08.251715+0000","last_undegraded":"2023-09-23T21:15:08.251715+0000","last_fullsized":"2023-09-23T21:15:08.251715+0000","mapping_epoch":504,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":505,"parent":"0.0","parent_split_bits":3,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:10:24.268662+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:10:24.268662+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13417,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":11,"num_read_kb":20,"num_write":5,"num_write_kb":55,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":40252,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":162,"reported_epoch":535,"state":"active+clean","last_fresh":"2023-09-23T21:15:13.405054+0000","last_change":"2023-09-23T21:13:30.770826+0000","last_active":"2023-09-23T21:15:13.405054+0000","last_peered":"2023-09-23T21:15:13.405054+0000","last_clean":"2023-09-23T21:15:13.405054+0000","last_became_active":"2023-09-23T21:13:30.770512+0000","last_became_peered":"2023-09-23T21:13:30.770512+0000","last_unstale":"2023-09-23T21:15:13.405054+0000","last_undegraded":"2023-09-23T21:15:13.405054+0000","last_fullsized":"2023-09-23T21:15:13.405054+0000","mapping_epoch":501,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:44:23.470538+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":418,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-09-23T21:15:08.245059+0000","last_change":"2023-09-23T21:13:30.790590+0000","last_active":"2023-09-23T21:15:08.245059+0000","last_peered":"2023-09-23T21:15:08.245059+0000","last_clean":"2023-09-23T21:15:08.245059+0000","last_became_active":"2023-09-23T21:13:30.790253+0000","last_became_peered":"2023-09-23T21:13:30.790253+0000","last_unstale":"2023-09-23T21:15:08.245059+0000","last_undegraded":"2023-09-23T21:15:08.245059+0000","last_fullsized":"2023-09-23T21:15:08.245059+0000","mapping_epoch":501,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:23:41.843250+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,0],"acting":[2,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":481,"reported_epoch":535,"state":"active+clean","last_fresh":"2023-09-23T21:15:13.412094+0000","last_change":"2023-09-23T21:13:12.210546+0000","last_active":"2023-09-23T21:15:13.412094+0000","last_peered":"2023-09-23T21:15:13.412094+0000","last_clean":"2023-09-23T21:15:13.412094+0000","last_became_active":"2023-09-23T21:13:12.205934+0000","last_became_peered":"2023-09-23T21:13:12.205934+0000","last_unstale":"2023-09-23T21:15:13.412094+0000","last_undegraded":"2023-09-23T21:15:13.412094+0000","last_fullsized":"2023-09-23T21:15:13.412094+0000","mapping_epoch":493,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:21:18.299428+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.e","version":"535'10224","reported_seq":17202,"reported_epoch":535,"state":"active+clean","last_fresh":"2023-09-23T21:15:14.916260+0000","last_change":"2023-09-23T21:12:38.623611+0000","last_active":"2023-09-23T21:15:14.916260+0000","last_peered":"2023-09-23T21:15:14.916260+0000","last_clean":"2023-09-23T21:15:14.916260+0000","last_became_active":"2023-09-23T21:12:38.610155+0000","last_became_peered":"2023-09-23T21:12:38.610155+0000","last_unstale":"2023-09-23T21:15:14.916260+0000","last_undegraded":"2023-09-23T21:15:14.916260+0000","last_fullsized":"2023-09-23T21:15:14.916260+0000","mapping_epoch":483,"log_start":"489'6293","ondisk_log_start":"489'6293","created":416,"last_epoch_clean":484,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_clean_scrub_stamp":"2023-09-23T21:10:15.663179+0000","objects_scrubbed":0,"log_size":3931,"log_dups_size":0,"ondisk_log_size":3931,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:19:27.097170+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10224,"num_write_kb":327168,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3412,"num_bytes_recovered":223608832,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.5","version":"79'161","reported_seq":563,"reported_epoch":535,"state":"active+clean","last_fresh":"2023-09-23T21:15:13.390610+0000","last_change":"2023-09-23T21:13:38.373969+0000","last_active":"2023-09-23T21:15:13.390610+0000","last_peered":"2023-09-23T21:15:13.390610+0000","last_clean":"2023-09-23T21:15:13.390610+0000","last_became_active":"2023-09-23T21:13:38.373633+0000","last_became_peered":"2023-09-23T21:13:38.373633+0000","last_unstale":"2023-09-23T21:15:13.390610+0000","last_undegraded":"2023-09-23T21:15:13.390610+0000","last_fullsized":"2023-09-23T21:15:13.390610+0000","mapping_epoch":507,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":3,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:09:34.892947+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:09:34.892947+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13417,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":11,"num_read_kb":20,"num_write":5,"num_write_kb":55,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":40252,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":75,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-09-23T21:15:08.250982+0000","last_change":"2023-09-23T21:13:12.235743+0000","last_active":"2023-09-23T21:15:08.250982+0000","last_peered":"2023-09-23T21:15:08.250982+0000","last_clean":"2023-09-23T21:15:08.250982+0000","last_became_active":"2023-09-23T21:13:12.216241+0000","last_became_peered":"2023-09-23T21:13:12.216241+0000","last_unstale":"2023-09-23T21:15:08.250982+0000","last_undegraded":"2023-09-23T21:15:08.250982+0000","last_fullsized":"2023-09-23T21:15:08.250982+0000","mapping_epoch":493,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:12:19.090710+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":516,"reported_epoch":535,"state":"active+clean","last_fresh":"2023-09-23T21:15:13.390984+0000","last_change":"2023-09-23T21:13:30.807520+0000","last_active":"2023-09-23T21:15:13.390984+0000","last_peered":"2023-09-23T21:15:13.390984+0000","last_clean":"2023-09-23T21:15:13.390984+0000","last_became_active":"2023-09-23T21:13:30.807222+0000","last_became_peered":"2023-09-23T21:13:30.807222+0000","last_unstale":"2023-09-23T21:15:13.390984+0000","last_undegraded":"2023-09-23T21:15:13.390984+0000","last_fullsized":"2023-09-23T21:15:13.390984+0000","mapping_epoch":501,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:40:31.989749+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":376,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-09-23T21:15:08.242323+0000","last_change":"2023-09-23T21:13:30.794095+0000","last_active":"2023-09-23T21:15:08.242323+0000","last_peered":"2023-09-23T21:15:08.242323+0000","last_clean":"2023-09-23T21:15:08.242323+0000","last_became_active":"2023-09-23T21:13:30.793791+0000","last_became_peered":"2023-09-23T21:13:30.793791+0000","last_unstale":"2023-09-23T21:15:08.242323+0000","last_undegraded":"2023-09-23T21:15:08.242323+0000","last_fullsized":"2023-09-23T21:15:08.242323+0000","mapping_epoch":501,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:36:28.707362+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.f","version":"535'10080","reported_seq":10291,"reported_epoch":535,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T21:15:14.150516+0000","last_change":"2023-09-23T21:13:59.874192+0000","last_active":"2023-09-23T21:15:14.150516+0000","last_peered":"2023-09-23T21:15:14.150516+0000","last_clean":"2023-09-23T21:11:07.652290+0000","last_became_active":"2023-09-23T21:11:09.897463+0000","last_became_peered":"2023-09-23T21:11:09.897463+0000","last_unstale":"2023-09-23T21:15:14.150516+0000","last_undegraded":"2023-09-23T21:11:27.781677+0000","last_fullsized":"2023-09-23T21:11:09.761652+0000","mapping_epoch":444,"log_start":"497'6409","ondisk_log_start":"497'6409","created":416,"last_epoch_clean":417,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_clean_scrub_stamp":"2023-09-23T21:10:15.663179+0000","objects_scrubbed":0,"log_size":3671,"log_dups_size":0,"ondisk_log_size":3671,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:42:51.588441+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":2228224,"num_objects":34,"num_object_clones":0,"num_object_copies":102,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3295,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":34,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10080,"num_write_kb":323648,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":32,"num_bytes_recovered":2097152,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2],"avail_no_missing":["3","1","2"],"object_location_counts":[{"shards":"2,3","objects":-3261}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.4","version":"79'161","reported_seq":906,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-09-23T21:15:08.244525+0000","last_change":"2023-09-23T21:10:29.807785+0000","last_active":"2023-09-23T21:15:08.244525+0000","last_peered":"2023-09-23T21:15:08.244525+0000","last_clean":"2023-09-23T21:15:08.244525+0000","last_became_active":"2023-09-23T21:10:29.807366+0000","last_became_peered":"2023-09-23T21:10:29.807366+0000","last_unstale":"2023-09-23T21:15:08.244525+0000","last_undegraded":"2023-09-23T21:15:08.244525+0000","last_fullsized":"2023-09-23T21:15:08.244525+0000","mapping_epoch":426,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":427,"parent":"0.0","parent_split_bits":3,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:20.012764+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:20.012764+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:24:45.763834+0000","scrub_duration":0.0045251919999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":474,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-09-23T21:15:08.247007+0000","last_change":"2023-09-23T21:10:26.768364+0000","last_active":"2023-09-23T21:15:08.247007+0000","last_peered":"2023-09-23T21:15:08.247007+0000","last_clean":"2023-09-23T21:15:08.247007+0000","last_became_active":"2023-09-23T21:10:26.768052+0000","last_became_peered":"2023-09-23T21:10:26.768052+0000","last_unstale":"2023-09-23T21:15:08.247007+0000","last_undegraded":"2023-09-23T21:15:08.247007+0000","last_fullsized":"2023-09-23T21:15:08.247007+0000","mapping_epoch":423,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":424,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:22:42.201200+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,3],"acting":[2,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.6","version":"70'1","reported_seq":505,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-09-23T21:15:08.251887+0000","last_change":"2023-09-23T21:13:30.793290+0000","last_active":"2023-09-23T21:15:08.251887+0000","last_peered":"2023-09-23T21:15:08.251887+0000","last_clean":"2023-09-23T21:15:08.251887+0000","last_became_active":"2023-09-23T21:13:30.786079+0000","last_became_peered":"2023-09-23T21:13:30.786079+0000","last_unstale":"2023-09-23T21:15:08.251887+0000","last_undegraded":"2023-09-23T21:15:08.251887+0000","last_fullsized":"2023-09-23T21:15:08.251887+0000","mapping_epoch":501,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:34:43.281095+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":110,"reported_epoch":535,"state":"active+clean","last_fresh":"2023-09-23T21:15:13.390203+0000","last_change":"2023-09-23T21:13:12.216895+0000","last_active":"2023-09-23T21:15:13.390203+0000","last_peered":"2023-09-23T21:15:13.390203+0000","last_clean":"2023-09-23T21:15:13.390203+0000","last_became_active":"2023-09-23T21:13:12.216654+0000","last_became_peered":"2023-09-23T21:13:12.216654+0000","last_unstale":"2023-09-23T21:15:13.390203+0000","last_undegraded":"2023-09-23T21:15:13.390203+0000","last_fullsized":"2023-09-23T21:15:13.390203+0000","mapping_epoch":493,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:02:40.430096+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.19","version":"79'161","reported_seq":688,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-09-23T21:15:08.247199+0000","last_change":"2023-09-23T21:12:30.442914+0000","last_active":"2023-09-23T21:15:08.247199+0000","last_peered":"2023-09-23T21:15:08.247199+0000","last_clean":"2023-09-23T21:15:08.247199+0000","last_became_active":"2023-09-23T21:12:30.442578+0000","last_became_peered":"2023-09-23T21:12:30.442578+0000","last_unstale":"2023-09-23T21:15:08.247199+0000","last_undegraded":"2023-09-23T21:15:08.247199+0000","last_fullsized":"2023-09-23T21:15:08.247199+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":479,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:10:37.696951+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:10:37.696951+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.001386504,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,3],"acting":[2,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"10.12","version":"535'6316","reported_seq":10217,"reported_epoch":535,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T21:15:14.939202+0000","last_change":"2023-09-23T21:13:13.419184+0000","last_active":"2023-09-23T21:15:14.939202+0000","last_peered":"2023-09-23T21:15:14.939202+0000","last_clean":"2023-09-23T21:11:45.774771+0000","last_became_active":"2023-09-23T21:13:13.409281+0000","last_became_peered":"2023-09-23T21:13:13.409281+0000","last_unstale":"2023-09-23T21:15:14.939202+0000","last_undegraded":"2023-09-23T21:13:13.284063+0000","last_fullsized":"2023-09-23T21:13:13.191117+0000","mapping_epoch":494,"log_start":"435'2097","ondisk_log_start":"435'2097","created":438,"last_epoch_clean":439,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_clean_scrub_stamp":"2023-09-23T21:10:15.663179+0000","objects_scrubbed":0,"log_size":4219,"log_dups_size":0,"ondisk_log_size":4219,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1252,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4939,"num_write_kb":157952,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1845,"num_bytes_recovered":120913920,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,3],"avail_no_missing":["4","1"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":44,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-09-23T21:15:08.251457+0000","last_change":"2023-09-23T21:13:30.791297+0000","last_active":"2023-09-23T21:15:08.251457+0000","last_peered":"2023-09-23T21:15:08.251457+0000","last_clean":"2023-09-23T21:15:08.251457+0000","last_became_active":"2023-09-23T21:13:30.788721+0000","last_became_peered":"2023-09-23T21:13:30.788721+0000","last_unstale":"2023-09-23T21:15:08.251457+0000","last_undegraded":"2023-09-23T21:15:08.251457+0000","last_fullsized":"2023-09-23T21:15:08.251457+0000","mapping_epoch":501,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:31:18.147883+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":38,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-09-23T21:15:08.244975+0000","last_change":"2023-09-23T21:13:38.373814+0000","last_active":"2023-09-23T21:15:08.244975+0000","last_peered":"2023-09-23T21:15:08.244975+0000","last_clean":"2023-09-23T21:15:08.244975+0000","last_became_active":"2023-09-23T21:13:38.373458+0000","last_became_peered":"2023-09-23T21:13:38.373458+0000","last_unstale":"2023-09-23T21:15:08.244975+0000","last_undegraded":"2023-09-23T21:15:08.244975+0000","last_fullsized":"2023-09-23T21:15:08.244975+0000","mapping_epoch":507,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:23:08.288401+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.1e","version":"79'161","reported_seq":703,"reported_epoch":535,"state":"active+clean","last_fresh":"2023-09-23T21:15:13.390632+0000","last_change":"2023-09-23T21:12:30.448761+0000","last_active":"2023-09-23T21:15:13.390632+0000","last_peered":"2023-09-23T21:15:13.390632+0000","last_clean":"2023-09-23T21:15:13.390632+0000","last_became_active":"2023-09-23T21:12:30.448043+0000","last_became_peered":"2023-09-23T21:12:30.448043+0000","last_unstale":"2023-09-23T21:15:13.390632+0000","last_undegraded":"2023-09-23T21:15:13.390632+0000","last_fullsized":"2023-09-23T21:15:13.390632+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":479,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:10:39.567232+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:10:39.567232+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.005229694,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.15","version":"535'6281","reported_seq":6433,"reported_epoch":535,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T21:15:14.853904+0000","last_change":"2023-09-23T21:13:13.436807+0000","last_active":"2023-09-23T21:15:14.853904+0000","last_peered":"2023-09-23T21:15:14.853904+0000","last_clean":"2023-09-23T21:11:45.765730+0000","last_became_active":"2023-09-23T21:13:13.429206+0000","last_became_peered":"2023-09-23T21:13:13.429206+0000","last_unstale":"2023-09-23T21:15:14.853904+0000","last_undegraded":"2023-09-23T21:13:13.246273+0000","last_fullsized":"2023-09-23T21:13:13.203825+0000","mapping_epoch":494,"log_start":"434'1188","ondisk_log_start":"434'1188","created":438,"last_epoch_clean":439,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_clean_scrub_stamp":"2023-09-23T21:10:15.663179+0000","objects_scrubbed":0,"log_size":5093,"log_dups_size":0,"ondisk_log_size":5093,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":65536,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":643,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4947,"num_write_kb":158336,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,3],"acting":[2,3],"avail_no_missing":["2","3"],"object_location_counts":[{"shards":"2,3","objects":-35}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":117,"reported_epoch":535,"state":"active+clean","last_fresh":"2023-09-23T21:15:13.391573+0000","last_change":"2023-09-23T21:15:06.721854+0000","last_active":"2023-09-23T21:15:13.391573+0000","last_peered":"2023-09-23T21:15:13.391573+0000","last_clean":"2023-09-23T21:15:13.391573+0000","last_became_active":"2023-09-23T21:12:14.209408+0000","last_became_peered":"2023-09-23T21:12:14.209408+0000","last_unstale":"2023-09-23T21:15:13.391573+0000","last_undegraded":"2023-09-23T21:15:13.391573+0000","last_fullsized":"2023-09-23T21:15:13.391573+0000","mapping_epoch":470,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":532,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:45:40.142413+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,7],"acting":[4,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":100,"reported_epoch":535,"state":"active+clean","last_fresh":"2023-09-23T21:15:13.390482+0000","last_change":"2023-09-23T21:12:10.770474+0000","last_active":"2023-09-23T21:15:13.390482+0000","last_peered":"2023-09-23T21:15:13.390482+0000","last_clean":"2023-09-23T21:15:13.390482+0000","last_became_active":"2023-09-23T21:12:10.770184+0000","last_became_peered":"2023-09-23T21:12:10.770184+0000","last_unstale":"2023-09-23T21:15:13.390482+0000","last_undegraded":"2023-09-23T21:15:13.390482+0000","last_fullsized":"2023-09-23T21:15:13.390482+0000","mapping_epoch":470,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:05:09.407151+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.14","version":"535'6239","reported_seq":12346,"reported_epoch":535,"state":"active+clean","last_fresh":"2023-09-23T21:15:14.678674+0000","last_change":"2023-09-23T21:14:01.515957+0000","last_active":"2023-09-23T21:15:14.678674+0000","last_peered":"2023-09-23T21:15:14.678674+0000","last_clean":"2023-09-23T21:15:14.678674+0000","last_became_active":"2023-09-23T21:14:01.506428+0000","last_became_peered":"2023-09-23T21:14:01.506428+0000","last_unstale":"2023-09-23T21:15:14.678674+0000","last_undegraded":"2023-09-23T21:15:14.678674+0000","last_fullsized":"2023-09-23T21:15:14.678674+0000","mapping_epoch":522,"log_start":"435'1816","ondisk_log_start":"435'1816","created":438,"last_epoch_clean":523,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_clean_scrub_stamp":"2023-09-23T21:10:15.663179+0000","objects_scrubbed":0,"log_size":4423,"log_dups_size":0,"ondisk_log_size":4423,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":5,"num_object_clones":0,"num_object_copies":15,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4905,"num_write_kb":157120,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2660,"num_bytes_recovered":126156800,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":325,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-09-23T21:15:08.248589+0000","last_change":"2023-09-23T21:12:10.771160+0000","last_active":"2023-09-23T21:15:08.248589+0000","last_peered":"2023-09-23T21:15:08.248589+0000","last_clean":"2023-09-23T21:15:08.248589+0000","last_became_active":"2023-09-23T21:12:10.757808+0000","last_became_peered":"2023-09-23T21:12:10.757808+0000","last_unstale":"2023-09-23T21:15:08.248589+0000","last_undegraded":"2023-09-23T21:15:08.248589+0000","last_fullsized":"2023-09-23T21:15:08.248589+0000","mapping_epoch":470,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:30:50.987139+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":110,"reported_epoch":535,"state":"active+clean","last_fresh":"2023-09-23T21:15:13.391962+0000","last_change":"2023-09-23T21:13:12.216602+0000","last_active":"2023-09-23T21:15:13.391962+0000","last_peered":"2023-09-23T21:15:13.391962+0000","last_clean":"2023-09-23T21:15:13.391962+0000","last_became_active":"2023-09-23T21:13:12.216204+0000","last_became_peered":"2023-09-23T21:13:12.216204+0000","last_unstale":"2023-09-23T21:15:13.391962+0000","last_undegraded":"2023-09-23T21:15:13.391962+0000","last_fullsized":"2023-09-23T21:15:13.391962+0000","mapping_epoch":493,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:23:37.043390+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1c","version":"79'161","reported_seq":901,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-09-23T21:15:08.243548+0000","last_change":"2023-09-23T21:13:13.402773+0000","last_active":"2023-09-23T21:15:08.243548+0000","last_peered":"2023-09-23T21:15:08.243548+0000","last_clean":"2023-09-23T21:15:08.243548+0000","last_became_active":"2023-09-23T21:13:13.402025+0000","last_became_peered":"2023-09-23T21:13:13.402025+0000","last_unstale":"2023-09-23T21:15:08.243548+0000","last_undegraded":"2023-09-23T21:15:08.243548+0000","last_fullsized":"2023-09-23T21:15:08.243548+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":479,"last_epoch_clean":495,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:22.040907+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:22.040907+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.013021176000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":163840,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":163840,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.17","version":"535'6237","reported_seq":10629,"reported_epoch":535,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T21:15:14.150571+0000","last_change":"2023-09-23T21:13:39.447969+0000","last_active":"2023-09-23T21:15:14.150571+0000","last_peered":"2023-09-23T21:15:14.150571+0000","last_clean":"2023-09-23T21:13:37.065694+0000","last_became_active":"2023-09-23T21:13:39.439739+0000","last_became_peered":"2023-09-23T21:13:39.439739+0000","last_unstale":"2023-09-23T21:15:14.150571+0000","last_undegraded":"2023-09-23T21:13:39.411853+0000","last_fullsized":"2023-09-23T21:13:39.404657+0000","mapping_epoch":508,"log_start":"436'2440","ondisk_log_start":"436'2440","created":438,"last_epoch_clean":487,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_clean_scrub_stamp":"2023-09-23T21:10:15.663179+0000","objects_scrubbed":0,"log_size":3797,"log_dups_size":0,"ondisk_log_size":3797,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4161536,"num_objects":64,"num_object_clones":0,"num_object_copies":192,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1109,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":64,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4872,"num_write_kb":157920,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2112,"num_bytes_recovered":116719616,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,2],"acting":[3,5],"avail_no_missing":["3","4","5"],"object_location_counts":[{"shards":"3,4,5","objects":19},{"shards":"3,5","objects":-247}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":124,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-09-23T21:15:08.244541+0000","last_change":"2023-09-23T21:12:10.779066+0000","last_active":"2023-09-23T21:15:08.244541+0000","last_peered":"2023-09-23T21:15:08.244541+0000","last_clean":"2023-09-23T21:15:08.244541+0000","last_became_active":"2023-09-23T21:12:10.778691+0000","last_became_peered":"2023-09-23T21:12:10.778691+0000","last_unstale":"2023-09-23T21:15:08.244541+0000","last_undegraded":"2023-09-23T21:15:08.244541+0000","last_fullsized":"2023-09-23T21:15:08.244541+0000","mapping_epoch":470,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:07:22.043268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,7],"acting":[2,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":202,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-09-23T21:15:08.247780+0000","last_change":"2023-09-23T21:13:12.200767+0000","last_active":"2023-09-23T21:15:08.247780+0000","last_peered":"2023-09-23T21:15:08.247780+0000","last_clean":"2023-09-23T21:15:08.247780+0000","last_became_active":"2023-09-23T21:13:12.200435+0000","last_became_peered":"2023-09-23T21:13:12.200435+0000","last_unstale":"2023-09-23T21:15:08.247780+0000","last_undegraded":"2023-09-23T21:15:08.247780+0000","last_fullsized":"2023-09-23T21:15:08.247780+0000","mapping_epoch":493,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:15:44.937886+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.1d","version":"79'161","reported_seq":688,"reported_epoch":535,"state":"active+clean","last_fresh":"2023-09-23T21:15:13.404691+0000","last_change":"2023-09-23T21:12:30.444331+0000","last_active":"2023-09-23T21:15:13.404691+0000","last_peered":"2023-09-23T21:15:13.404691+0000","last_clean":"2023-09-23T21:15:13.404691+0000","last_became_active":"2023-09-23T21:12:30.444001+0000","last_became_peered":"2023-09-23T21:12:30.444001+0000","last_unstale":"2023-09-23T21:15:13.404691+0000","last_undegraded":"2023-09-23T21:15:13.404691+0000","last_fullsized":"2023-09-23T21:15:13.404691+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":479,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:09:36.858077+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:09:36.858077+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0024987500000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"10.16","version":"535'6391","reported_seq":9908,"reported_epoch":535,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T21:15:14.891900+0000","last_change":"2023-09-23T21:13:14.424843+0000","last_active":"2023-09-23T21:15:14.891900+0000","last_peered":"2023-09-23T21:15:14.891900+0000","last_clean":"2023-09-23T21:11:45.652255+0000","last_became_active":"2023-09-23T21:13:14.414710+0000","last_became_peered":"2023-09-23T21:13:14.414710+0000","last_unstale":"2023-09-23T21:15:14.891900+0000","last_undegraded":"2023-09-23T21:13:13.211957+0000","last_fullsized":"2023-09-23T21:13:13.188859+0000","mapping_epoch":494,"log_start":"435'2168","ondisk_log_start":"435'2168","created":438,"last_epoch_clean":460,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:10:15.663179+0000","last_clean_scrub_stamp":"2023-09-23T21:10:15.663179+0000","objects_scrubbed":0,"log_size":4223,"log_dups_size":0,"ondisk_log_size":4223,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":1376256,"num_objects":21,"num_object_clones":0,"num_object_copies":63,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":663,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":21,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4985,"num_write_kb":160192,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1670,"num_bytes_recovered":109445120,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,4],"acting":[4,5],"avail_no_missing":["4","5"],"object_location_counts":[{"shards":"4,5","objects":-12}],"blocked_by":[],"up_primary":0,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":61,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-09-23T21:15:08.250620+0000","last_change":"2023-09-23T21:13:38.380574+0000","last_active":"2023-09-23T21:15:08.250620+0000","last_peered":"2023-09-23T21:15:08.250620+0000","last_clean":"2023-09-23T21:15:08.250620+0000","last_became_active":"2023-09-23T21:13:38.373252+0000","last_became_peered":"2023-09-23T21:13:38.373252+0000","last_unstale":"2023-09-23T21:15:08.250620+0000","last_undegraded":"2023-09-23T21:15:08.250620+0000","last_fullsized":"2023-09-23T21:15:08.250620+0000","mapping_epoch":507,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:58:14.788031+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":11,"num_bytes_recovered":3324,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":110,"reported_epoch":535,"state":"active+clean","last_fresh":"2023-09-23T21:15:13.390169+0000","last_change":"2023-09-23T21:12:10.793610+0000","last_active":"2023-09-23T21:15:13.390169+0000","last_peered":"2023-09-23T21:15:13.390169+0000","last_clean":"2023-09-23T21:15:13.390169+0000","last_became_active":"2023-09-23T21:12:10.793081+0000","last_became_peered":"2023-09-23T21:12:10.793081+0000","last_unstale":"2023-09-23T21:15:13.390169+0000","last_undegraded":"2023-09-23T21:15:13.390169+0000","last_fullsized":"2023-09-23T21:15:13.390169+0000","mapping_epoch":470,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:34:36.291550+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":10,"num_pg":26,"stat_sum":{"num_bytes":21168128,"num_objects":325,"num_object_clones":0,"num_object_copies":975,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":17467,"num_objects_misplaced":2169,"num_objects_unfound":0,"num_objects_dirty":325,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":147124,"num_write_kb":4309313,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":47384,"num_bytes_recovered":2758967320,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2932084736,"data_stored":2932080664,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":118874,"ondisk_log_size":118874,"up":78,"acting":64,"num_store_stats":7},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":60,"num_bytes_recovered":8898,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":126976,"data_stored":22458,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":14322,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":31,"stat_sum":{"num_bytes":697708,"num_objects":3,"num_object_clones":0,"num_object_copies":9,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3,"num_whiteouts":0,"num_read":89,"num_read_kb":165,"num_write":44,"num_write_kb":440,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":977379,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2109440,"data_stored":2098784,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4991,"ondisk_log_size":4991,"up":93,"acting":93,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201698,"num_pgs":69,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":507768,"kb_used_data":5616,"kb_used_omap":0,"kb_used_meta":502144,"kb_avail":93237384,"statfs":{"total":95995035648,"available":95475081216,"internally_reserved":0,"allocated":5750784,"data_stored":4192288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":514195456},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:15:15 2023","interfaces":[{"interface":"back","average":{"1min":1.3440000000000001,"5min":2.2090000000000001,"15min":2.3530000000000002},"min":{"1min":0.25,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":4.3070000000000004,"5min":37.207999999999998,"15min":37.207999999999998},"last":0.308},{"interface":"front","average":{"1min":2.089,"5min":2.2050000000000001,"15min":2.2240000000000002},"min":{"1min":0.253,"5min":0.253,"15min":0.253},"max":{"1min":10.429,"5min":37.122,"15min":37.122},"last":0.50900000000000001}]},{"osd":1,"last update":"Sat Sep 23 21:15:09 2023","interfaces":[{"interface":"back","average":{"1min":2.0329999999999999,"5min":1.3320000000000001,"15min":2.0289999999999999},"min":{"1min":0.28000000000000003,"5min":0.27500000000000002,"15min":0.26700000000000002},"max":{"1min":6.0860000000000003,"5min":14.034000000000001,"15min":14.034000000000001},"last":0.49199999999999999},{"interface":"front","average":{"1min":1.554,"5min":1.4359999999999999,"15min":1.9079999999999999},"min":{"1min":0.33900000000000002,"5min":0.29899999999999999,"15min":0.26300000000000001},"max":{"1min":4.2690000000000001,"5min":11.226000000000001,"15min":11.226000000000001},"last":0.34899999999999998}]},{"osd":2,"last update":"Sat Sep 23 21:14:30 2023","interfaces":[{"interface":"back","average":{"1min":0.80900000000000005,"5min":1.0880000000000001,"15min":1.7170000000000001},"min":{"1min":0.29699999999999999,"5min":0.26400000000000001,"15min":0.26200000000000001},"max":{"1min":3.0190000000000001,"5min":6.4219999999999997,"15min":14.478999999999999},"last":0.36599999999999999},{"interface":"front","average":{"1min":1.101,"5min":1.3520000000000001,"15min":1.7330000000000001},"min":{"1min":0.30399999999999999,"5min":0.26700000000000002,"15min":0.26500000000000001},"max":{"1min":5.7640000000000002,"5min":11.198,"15min":14.135},"last":0.44900000000000001}]},{"osd":3,"last update":"Sat Sep 23 21:14:58 2023","interfaces":[{"interface":"back","average":{"1min":1.7270000000000001,"5min":1.3009999999999999,"15min":2},"min":{"1min":0.30599999999999999,"5min":0.30599999999999999,"15min":0.29099999999999998},"max":{"1min":4.1619999999999999,"5min":7.5190000000000001,"15min":31.786000000000001},"last":0.46000000000000002},{"interface":"front","average":{"1min":2.7410000000000001,"5min":1.4890000000000001,"15min":2.0390000000000001},"min":{"1min":0.35599999999999998,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":11.743,"5min":11.743,"15min":32.429000000000002},"last":0.47199999999999998}]},{"osd":4,"last update":"Sat Sep 23 21:14:58 2023","interfaces":[{"interface":"back","average":{"1min":1.1100000000000001,"5min":1.48,"15min":1.143},"min":{"1min":0.32500000000000001,"5min":0.26200000000000001,"15min":0.248},"max":{"1min":4.2130000000000001,"5min":11.297000000000001,"15min":11.297000000000001},"last":0.38100000000000001},{"interface":"front","average":{"1min":1.085,"5min":1.24,"15min":1.105},"min":{"1min":0.29699999999999999,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":3.7850000000000001,"5min":11.093999999999999,"15min":12.752000000000001},"last":0.42799999999999999}]},{"osd":5,"last update":"Sat Sep 23 21:14:58 2023","interfaces":[{"interface":"back","average":{"1min":1.359,"5min":1.1990000000000001,"15min":1.075},"min":{"1min":0.32300000000000001,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":6.133,"5min":9.4789999999999992,"15min":9.4789999999999992},"last":0.41299999999999998},{"interface":"front","average":{"1min":0.98699999999999999,"5min":1.3360000000000001,"15min":1.1279999999999999},"min":{"1min":0.33300000000000002,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":4.181,"5min":11.257,"15min":11.257},"last":0.47999999999999998}]},{"osd":6,"last update":"Sat Sep 23 21:15:09 2023","interfaces":[{"interface":"back","average":{"1min":1.325,"5min":1.3029999999999999,"15min":1.095},"min":{"1min":0.40200000000000002,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":7.5339999999999998,"5min":10.941000000000001,"15min":10.941000000000001},"last":0.55300000000000005},{"interface":"front","average":{"1min":1.298,"5min":1.2130000000000001,"15min":1.1799999999999999},"min":{"1min":0.39200000000000002,"5min":0.33100000000000002,"15min":0.32300000000000001},"max":{"1min":4.7720000000000002,"5min":8.923,"15min":12.715},"last":0.53200000000000003}]}]},{"osd":6,"up_from":0,"seq":704374636781,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":1,"up_from":0,"seq":1503238553713,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":0,"up_from":493,"seq":2117418876954,"num_pgs":75,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":972816,"kb_used_data":774344,"kb_used_omap":36,"kb_used_meta":198427,"kb_avail":92772336,"statfs":{"total":95995035648,"available":94998872064,"internally_reserved":0,"allocated":792928256,"data_stored":790421164,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":37336,"internal_metadata":203189800},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1,3],"upper_bound":16},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 21:14:12 2023","interfaces":[{"interface":"back","average":{"1min":0.75600000000000001,"5min":0.75600000000000001,"15min":0.75600000000000001},"min":{"1min":0.23000000000000001,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":4.9000000000000004,"5min":4.9000000000000004,"15min":4.9000000000000004},"last":3.6349999999999998},{"interface":"front","average":{"1min":0.77100000000000002,"5min":0.77100000000000002,"15min":0.77100000000000002},"min":{"1min":0.24299999999999999,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":3.444,"5min":3.444,"15min":3.444},"last":11.964}]},{"osd":2,"last update":"Sat Sep 23 21:14:12 2023","interfaces":[{"interface":"back","average":{"1min":0.65400000000000003,"5min":0.65400000000000003,"15min":0.65400000000000003},"min":{"1min":0.26900000000000002,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":4.8630000000000004,"5min":4.8630000000000004,"15min":4.8630000000000004},"last":7.133},{"interface":"front","average":{"1min":0.96799999999999997,"5min":0.96799999999999997,"15min":0.96799999999999997},"min":{"1min":0.29299999999999998,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":4.649,"5min":4.649,"15min":4.649},"last":1.5249999999999999}]},{"osd":3,"last update":"Sat Sep 23 21:14:12 2023","interfaces":[{"interface":"back","average":{"1min":0.86799999999999999,"5min":0.86799999999999999,"15min":0.86799999999999999},"min":{"1min":0.23400000000000001,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":4.5110000000000001,"5min":4.5110000000000001,"15min":4.5110000000000001},"last":6.2060000000000004},{"interface":"front","average":{"1min":1.1180000000000001,"5min":1.1180000000000001,"15min":1.1180000000000001},"min":{"1min":0.26200000000000001,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":3.7200000000000002,"5min":3.7200000000000002,"15min":3.7200000000000002},"last":4.1989999999999998}]},{"osd":4,"last update":"Sat Sep 23 21:14:12 2023","interfaces":[{"interface":"back","average":{"1min":1.093,"5min":1.093,"15min":1.093},"min":{"1min":0.30199999999999999,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":3.7799999999999998,"5min":3.7799999999999998,"15min":3.7799999999999998},"last":4.2800000000000002},{"interface":"front","average":{"1min":0.996,"5min":0.996,"15min":0.996},"min":{"1min":0.317,"5min":0.317,"15min":0.317},"max":{"1min":5.093,"5min":5.093,"15min":5.093},"last":3.6699999999999999}]},{"osd":5,"last update":"Sat Sep 23 21:14:12 2023","interfaces":[{"interface":"back","average":{"1min":1.236,"5min":1.236,"15min":1.236},"min":{"1min":0.33200000000000002,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":4.5640000000000001,"5min":4.5640000000000001,"15min":4.5640000000000001},"last":1.5529999999999999},{"interface":"front","average":{"1min":1.0640000000000001,"5min":1.0640000000000001,"15min":1.0640000000000001},"min":{"1min":0.33200000000000002,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":4.5599999999999996,"5min":4.5599999999999996,"15min":4.5599999999999996},"last":4.2569999999999997}]},{"osd":6,"last update":"Sat Sep 23 21:14:12 2023","interfaces":[{"interface":"back","average":{"1min":1.127,"5min":1.127,"15min":1.127},"min":{"1min":0.38100000000000001,"5min":0.38100000000000001,"15min":0.38100000000000001},"max":{"1min":6.2750000000000004,"5min":6.2750000000000004,"15min":6.2750000000000004},"last":3.7040000000000002},{"interface":"front","average":{"1min":1.3080000000000001,"5min":1.3080000000000001,"15min":1.3080000000000001},"min":{"1min":0.35599999999999998,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":5.0860000000000003,"5min":5.0860000000000003,"15min":5.0860000000000003},"last":1.5700000000000001}]},{"osd":7,"last update":"Sat Sep 23 21:14:12 2023","interfaces":[{"interface":"back","average":{"1min":1.2090000000000001,"5min":1.2090000000000001,"15min":1.2090000000000001},"min":{"1min":0.41099999999999998,"5min":0.41099999999999998,"15min":0.41099999999999998},"max":{"1min":5.1539999999999999,"5min":5.1539999999999999,"15min":5.1539999999999999},"last":4.2279999999999998},{"interface":"front","average":{"1min":1.1579999999999999,"5min":1.1579999999999999,"15min":1.1579999999999999},"min":{"1min":0.441,"5min":0.441,"15min":0.441},"max":{"1min":5.1289999999999996,"5min":5.1289999999999996,"15min":5.1289999999999996},"last":3.6869999999999998}]}]},{"osd":2,"up_from":93,"seq":399431958826,"num_pgs":69,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1046808,"kb_used_data":236944,"kb_used_omap":81,"kb_used_meta":809774,"kb_avail":92698344,"statfs":{"total":95995035648,"available":94923104256,"internally_reserved":0,"allocated":242630656,"data_stored":240114967,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":83750,"internal_metadata":829208794},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,0,7],"upper_bound":16},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:14:15 2023","interfaces":[{"interface":"back","average":{"1min":0.83299999999999996,"5min":0.83299999999999996,"15min":0.83299999999999996},"min":{"1min":0.20899999999999999,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":3.7290000000000001,"5min":3.7290000000000001,"15min":3.7290000000000001},"last":6.0149999999999997},{"interface":"front","average":{"1min":1.665,"5min":1.665,"15min":1.665},"min":{"1min":0.23499999999999999,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":18.111999999999998,"5min":18.111999999999998,"15min":18.111999999999998},"last":8.1059999999999999}]},{"osd":1,"last update":"Sat Sep 23 21:14:15 2023","interfaces":[{"interface":"back","average":{"1min":0.95099999999999996,"5min":1.474,"15min":2.5339999999999998},"min":{"1min":0.23999999999999999,"5min":0.188,"15min":0.188},"max":{"1min":6.0499999999999998,"5min":12.484999999999999,"15min":34.396999999999998},"last":5.8680000000000003},{"interface":"front","average":{"1min":1.597,"5min":1.3129999999999999,"15min":2.782},"min":{"1min":0.29899999999999999,"5min":0.217,"15min":0.217},"max":{"1min":18.081,"5min":18.081,"15min":34.462000000000003},"last":16.274000000000001}]},{"osd":3,"last update":"Sat Sep 23 21:14:58 2023","interfaces":[{"interface":"back","average":{"1min":2.246,"5min":1.274,"15min":2.4550000000000001},"min":{"1min":0.37,"5min":0.20599999999999999,"15min":0.20599999999999999},"max":{"1min":14.442,"5min":14.442,"15min":41.057000000000002},"last":0.32700000000000001},{"interface":"front","average":{"1min":2.633,"5min":1.4419999999999999,"15min":2.6970000000000001},"min":{"1min":0.28599999999999998,"5min":0.21299999999999999,"15min":0.21299999999999999},"max":{"1min":14.622,"5min":14.622,"15min":41.012},"last":16.047999999999998}]},{"osd":4,"last update":"Sat Sep 23 21:14:15 2023","interfaces":[{"interface":"back","average":{"1min":0.64800000000000002,"5min":1.256,"15min":1.4219999999999999},"min":{"1min":0.34100000000000003,"5min":0.249,"15min":0.249},"max":{"1min":1.7210000000000001,"5min":10.266999999999999,"15min":10.266999999999999},"last":0.36599999999999999},{"interface":"front","average":{"1min":1.0029999999999999,"5min":1.456,"15min":1.8759999999999999},"min":{"1min":0.38300000000000001,"5min":0.29399999999999998,"15min":0.26700000000000002},"max":{"1min":3.5819999999999999,"5min":8.6780000000000008,"15min":14.039999999999999},"last":5.556}]},{"osd":5,"last update":"Sat Sep 23 21:14:15 2023","interfaces":[{"interface":"back","average":{"1min":0.94399999999999995,"5min":1.276,"15min":1.5209999999999999},"min":{"1min":0.311,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":3.3759999999999999,"5min":9.5039999999999996,"15min":10.069000000000001},"last":0.38400000000000001},{"interface":"front","average":{"1min":0.83999999999999997,"5min":1.3149999999999999,"15min":1.8120000000000001},"min":{"1min":0.41499999999999998,"5min":0.29699999999999999,"15min":0.29399999999999998},"max":{"1min":3.629,"5min":8.7270000000000003,"15min":13.98},"last":5.8330000000000002}]},{"osd":6,"last update":"Sat Sep 23 21:14:15 2023","interfaces":[{"interface":"back","average":{"1min":0.94999999999999996,"5min":1.569,"15min":2.0529999999999999},"min":{"1min":0.436,"5min":0.373,"15min":0.35699999999999998},"max":{"1min":3.665,"5min":8.7680000000000007,"15min":14.016},"last":5.6150000000000002},{"interface":"front","average":{"1min":1.2450000000000001,"5min":1.7729999999999999,"15min":2.2330000000000001},"min":{"1min":0.45700000000000002,"5min":0.38100000000000001,"15min":0.35599999999999998},"max":{"1min":3.7610000000000001,"5min":9.3469999999999995,"15min":14.068},"last":5.649}]},{"osd":7,"last update":"Sat Sep 23 21:14:15 2023","interfaces":[{"interface":"back","average":{"1min":1.8120000000000001,"5min":1.4790000000000001,"15min":1.5840000000000001},"min":{"1min":0.433,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":18.021999999999998,"5min":18.021999999999998,"15min":18.021999999999998},"last":6.0510000000000002},{"interface":"front","average":{"1min":1.242,"5min":1.476,"15min":1.675},"min":{"1min":0.39400000000000002,"5min":0.34799999999999998,"15min":0.32200000000000001},"max":{"1min":9.6349999999999998,"5min":10.359,"15min":10.92},"last":0.44800000000000001}]}]},{"osd":3,"up_from":98,"seq":420906795299,"num_pgs":82,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1447928,"kb_used_data":565424,"kb_used_omap":7,"kb_used_meta":882488,"kb_avail":92297224,"statfs":{"total":95995035648,"available":94512357376,"internally_reserved":0,"allocated":578994176,"data_stored":576504287,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7576,"internal_metadata":903668328},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:15:13 2023","interfaces":[{"interface":"back","average":{"1min":10.904,"5min":2.4510000000000001,"15min":1.042},"min":{"1min":0.28499999999999998,"5min":0.20499999999999999,"15min":0.20499999999999999},"max":{"1min":147.38900000000001,"5min":147.38900000000001,"15min":147.38900000000001},"last":1.4390000000000001},{"interface":"front","average":{"1min":10.178000000000001,"5min":2.5699999999999998,"15min":1.302},"min":{"1min":0.23100000000000001,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":149.989,"5min":149.989,"15min":149.989},"last":0.23100000000000001}]},{"osd":1,"last update":"Sat Sep 23 21:15:13 2023","interfaces":[{"interface":"back","average":{"1min":1.5589999999999999,"5min":1.2,"15min":2.1880000000000002},"min":{"1min":0.23699999999999999,"5min":0.20000000000000001,"15min":0.20000000000000001},"max":{"1min":5.0700000000000003,"5min":22.577999999999999,"15min":22.577999999999999},"last":0.36499999999999999},{"interface":"front","average":{"1min":1.5600000000000001,"5min":1.345,"15min":2.1389999999999998},"min":{"1min":0.26100000000000001,"5min":0.216,"15min":0.216},"max":{"1min":5.7329999999999997,"5min":21.617999999999999,"15min":21.617999999999999},"last":1.29}]},{"osd":2,"last update":"Sat Sep 23 21:15:00 2023","interfaces":[{"interface":"back","average":{"1min":1.78,"5min":1.9419999999999999,"15min":2.468},"min":{"1min":0.23899999999999999,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":10.946,"5min":56.317999999999998,"15min":56.317999999999998},"last":0.80900000000000005},{"interface":"front","average":{"1min":1.554,"5min":1.978,"15min":2.5609999999999999},"min":{"1min":0.246,"5min":0.216,"15min":0.216},"max":{"1min":6.29,"5min":55.988,"15min":55.988},"last":2.8980000000000001}]},{"osd":4,"last update":"Sat Sep 23 21:15:00 2023","interfaces":[{"interface":"back","average":{"1min":2.2959999999999998,"5min":1.526,"15min":1.831},"min":{"1min":0.29899999999999999,"5min":0.29899999999999999,"15min":0.28399999999999997},"max":{"1min":10.807,"5min":21.478999999999999,"15min":21.478999999999999},"last":0.29399999999999998},{"interface":"front","average":{"1min":1.415,"5min":1.4830000000000001,"15min":1.9710000000000001},"min":{"1min":0.314,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":6.3499999999999996,"5min":22.402000000000001,"15min":22.402000000000001},"last":0.32300000000000001}]},{"osd":5,"last update":"Sat Sep 23 21:15:00 2023","interfaces":[{"interface":"back","average":{"1min":1.925,"5min":1.456,"15min":1.95},"min":{"1min":0.34599999999999997,"5min":0.311,"15min":0.311},"max":{"1min":10.882999999999999,"5min":21.562000000000001,"15min":21.562000000000001},"last":0.44},{"interface":"front","average":{"1min":1.429,"5min":1.302,"15min":2.141},"min":{"1min":0.34799999999999998,"5min":0.31,"15min":0.31},"max":{"1min":4.2779999999999996,"5min":8.8450000000000006,"15min":14.029},"last":1.3420000000000001}]},{"osd":6,"last update":"Sat Sep 23 21:14:15 2023","interfaces":[{"interface":"back","average":{"1min":1.254,"5min":1.591,"15min":2.0510000000000002},"min":{"1min":0.38900000000000001,"5min":0.34999999999999998,"15min":0.33700000000000002},"max":{"1min":4.2380000000000004,"5min":22.446999999999999,"15min":22.446999999999999},"last":0.41399999999999998},{"interface":"front","average":{"1min":1.2450000000000001,"5min":1.746,"15min":2.278},"min":{"1min":0.44,"5min":0.39000000000000001,"15min":0.373},"max":{"1min":9.3889999999999993,"5min":12.375999999999999,"15min":13.576000000000001},"last":1.3779999999999999}]},{"osd":7,"last update":"Sat Sep 23 21:15:00 2023","interfaces":[{"interface":"back","average":{"1min":2.4289999999999998,"5min":1.8480000000000001,"15min":2.1000000000000001},"min":{"1min":0.439,"5min":0.38800000000000001,"15min":0.33900000000000002},"max":{"1min":10.914999999999999,"5min":21.518999999999998,"15min":21.518999999999998},"last":0.47999999999999998},{"interface":"front","average":{"1min":2.4220000000000002,"5min":1.903,"15min":2.3380000000000001},"min":{"1min":0.42499999999999999,"5min":0.42499999999999999,"15min":0.38900000000000001},"max":{"1min":8.2590000000000003,"5min":9.9879999999999995,"15min":13.603999999999999},"last":1.4099999999999999}]}]},{"osd":4,"up_from":31,"seq":133143986606,"num_pgs":77,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":629872,"kb_used_data":149352,"kb_used_omap":0,"kb_used_meta":480512,"kb_avail":93115280,"statfs":{"total":95995035648,"available":95350046720,"internally_reserved":0,"allocated":152936448,"data_stored":151378407,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":492044288},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:15:14 2023","interfaces":[{"interface":"back","average":{"1min":1.944,"5min":1.1080000000000001,"15min":0.96899999999999997},"min":{"1min":0.23799999999999999,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":8.0079999999999991,"5min":8.0079999999999991,"15min":8.0079999999999991},"last":8.0079999999999991},{"interface":"front","average":{"1min":0.876,"5min":1.0820000000000001,"15min":1.1160000000000001},"min":{"1min":0.23799999999999999,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":2.4649999999999999,"5min":10.945,"15min":10.945},"last":0.39200000000000002}]},{"osd":1,"last update":"Sat Sep 23 21:15:14 2023","interfaces":[{"interface":"back","average":{"1min":1.345,"5min":1.0429999999999999,"15min":2.5840000000000001},"min":{"1min":0.32700000000000001,"5min":0.221,"15min":0.221},"max":{"1min":4.0410000000000004,"5min":7.8609999999999998,"15min":37.960000000000001},"last":0.32700000000000001},{"interface":"front","average":{"1min":2.246,"5min":1.319,"15min":2.516},"min":{"1min":0.30299999999999999,"5min":0.28499999999999998,"15min":0.27000000000000002},"max":{"1min":13.983000000000001,"5min":13.983000000000001,"15min":37.933},"last":0.93700000000000006}]},{"osd":2,"last update":"Sat Sep 23 21:15:14 2023","interfaces":[{"interface":"back","average":{"1min":2.3559999999999999,"5min":1.3660000000000001,"15min":1.734},"min":{"1min":0.26400000000000001,"5min":0.247,"15min":0.247},"max":{"1min":14.026,"5min":14.026,"15min":16.928000000000001},"last":1.032},{"interface":"front","average":{"1min":3.0419999999999998,"5min":1.6220000000000001,"15min":1.7949999999999999},"min":{"1min":0.33000000000000002,"5min":0.28599999999999998,"15min":0.27800000000000002},"max":{"1min":14.339,"5min":14.339,"15min":14.417999999999999},"last":2.4390000000000001}]},{"osd":3,"last update":"Sat Sep 23 21:14:47 2023","interfaces":[{"interface":"back","average":{"1min":1.0389999999999999,"5min":1.1990000000000001,"15min":2.056},"min":{"1min":0.27900000000000003,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":2.766,"5min":9.3279999999999994,"15min":15.01},"last":0.51100000000000001},{"interface":"front","average":{"1min":1.899,"5min":1.3879999999999999,"15min":2},"min":{"1min":0.32200000000000001,"5min":0.32200000000000001,"15min":0.30199999999999999},"max":{"1min":14.305,"5min":14.305,"15min":15.004},"last":1.093}]},{"osd":5,"last update":"Sat Sep 23 21:15:04 2023","interfaces":[{"interface":"back","average":{"1min":2.0310000000000001,"5min":1.337,"15min":1.0840000000000001},"min":{"1min":0.32000000000000001,"5min":0.222,"15min":0.222},"max":{"1min":14.398999999999999,"5min":14.398999999999999,"15min":14.398999999999999},"last":3.3210000000000002},{"interface":"front","average":{"1min":1.675,"5min":1.2050000000000001,"15min":1.121},"min":{"1min":0.29699999999999999,"5min":0.27200000000000002,"15min":0.255},"max":{"1min":8.016,"5min":8.016,"15min":9.8979999999999997},"last":0.42299999999999999}]},{"osd":6,"last update":"Sat Sep 23 21:15:14 2023","interfaces":[{"interface":"back","average":{"1min":1.403,"5min":1.248,"15min":1.2749999999999999},"min":{"1min":0.34300000000000003,"5min":0.28599999999999998,"15min":0.26800000000000002},"max":{"1min":8.7420000000000009,"5min":24.452999999999999,"15min":24.452999999999999},"last":0.36099999999999999},{"interface":"front","average":{"1min":2.472,"5min":1.425,"15min":1.0680000000000001},"min":{"1min":0.40300000000000002,"5min":0.33500000000000002,"15min":0.32500000000000001},"max":{"1min":14.374000000000001,"5min":14.374000000000001,"15min":14.387},"last":1.012}]},{"osd":7,"last update":"Sat Sep 23 21:14:47 2023","interfaces":[{"interface":"back","average":{"1min":1.9910000000000001,"5min":1.355,"15min":1.1299999999999999},"min":{"1min":0.38300000000000001,"5min":0.30099999999999999,"15min":0.29299999999999998},"max":{"1min":14.422000000000001,"5min":14.422000000000001,"15min":16.890999999999998},"last":0.98899999999999999},{"interface":"front","average":{"1min":1.536,"5min":1.234,"15min":1.2090000000000001},"min":{"1min":0.39200000000000002,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":7.7229999999999999,"5min":7.7229999999999999,"15min":11.170999999999999},"last":0.47499999999999998}]}]},{"osd":5,"up_from":39,"seq":167503724972,"num_pgs":59,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":631840,"kb_used_data":112856,"kb_used_omap":0,"kb_used_meta":518976,"kb_avail":93113312,"statfs":{"total":95995035648,"available":95348031488,"internally_reserved":0,"allocated":115564544,"data_stored":114018621,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":531431424},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:15:14 2023","interfaces":[{"interface":"back","average":{"1min":1.3720000000000001,"5min":0.84599999999999997,"15min":0.75800000000000001},"min":{"1min":0.23100000000000001,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":4.3680000000000003,"5min":4.3680000000000003,"15min":4.3680000000000003},"last":3.8220000000000001},{"interface":"front","average":{"1min":1.6839999999999999,"5min":0.872,"15min":0.73599999999999999},"min":{"1min":0.23899999999999999,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":6.577,"5min":6.577,"15min":6.577},"last":2.367}]},{"osd":1,"last update":"Sat Sep 23 21:15:11 2023","interfaces":[{"interface":"back","average":{"1min":1.1140000000000001,"5min":0.94099999999999995,"15min":1.6830000000000001},"min":{"1min":0.29399999999999998,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":7.5839999999999996,"5min":7.5839999999999996,"15min":25.239999999999998},"last":4.1200000000000001},{"interface":"front","average":{"1min":1.123,"5min":1.472,"15min":1.7549999999999999},"min":{"1min":0.28299999999999997,"5min":0.28299999999999997,"15min":0.252},"max":{"1min":4.2199999999999998,"5min":11.116,"15min":25.253},"last":4.1360000000000001}]},{"osd":2,"last update":"Sat Sep 23 21:14:26 2023","interfaces":[{"interface":"back","average":{"1min":0.73099999999999998,"5min":1.401,"15min":1.7110000000000001},"min":{"1min":0.27800000000000002,"5min":0.26700000000000002,"15min":0.23499999999999999},"max":{"1min":4.0490000000000004,"5min":9.2889999999999997,"15min":12.634},"last":1.601},{"interface":"front","average":{"1min":0.78600000000000003,"5min":1.222,"15min":1.762},"min":{"1min":0.30199999999999999,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":2.524,"5min":9.7170000000000005,"15min":10.333},"last":0.52800000000000002}]},{"osd":3,"last update":"Sat Sep 23 21:14:48 2023","interfaces":[{"interface":"back","average":{"1min":1.0109999999999999,"5min":1.6339999999999999,"15min":2.484},"min":{"1min":0.32400000000000001,"5min":0.311,"15min":0.29299999999999998},"max":{"1min":3.0390000000000001,"5min":14.242000000000001,"15min":61.667999999999999},"last":3.9700000000000002},{"interface":"front","average":{"1min":1.284,"5min":1.6559999999999999,"15min":2.5910000000000002},"min":{"1min":0.39100000000000001,"5min":0.34899999999999998,"15min":0.30599999999999999},"max":{"1min":6.508,"5min":12.44,"15min":62.276000000000003},"last":2.1840000000000002}]},{"osd":4,"last update":"Sat Sep 23 21:15:00 2023","interfaces":[{"interface":"back","average":{"1min":1.0920000000000001,"5min":1.5289999999999999,"15min":1.157},"min":{"1min":0.253,"5min":0.253,"15min":0.23400000000000001},"max":{"1min":4.7750000000000004,"5min":12.776,"15min":12.776},"last":0.28299999999999997},{"interface":"front","average":{"1min":1.163,"5min":1.3029999999999999,"15min":1.1539999999999999},"min":{"1min":0.35999999999999999,"5min":0.27700000000000002,"15min":0.26900000000000002},"max":{"1min":4.8200000000000003,"5min":9.7119999999999997,"15min":11.973000000000001},"last":1.5609999999999999}]},{"osd":6,"last update":"Sat Sep 23 21:15:11 2023","interfaces":[{"interface":"back","average":{"1min":1.0880000000000001,"5min":1.0449999999999999,"15min":1.071},"min":{"1min":0.36499999999999999,"5min":0.313,"15min":0.25900000000000001},"max":{"1min":5.2249999999999996,"5min":7.7030000000000003,"15min":11.128},"last":0.42799999999999999},{"interface":"front","average":{"1min":1.022,"5min":1.5489999999999999,"15min":1.2450000000000001},"min":{"1min":0.34300000000000003,"5min":0.34300000000000003,"15min":0.317},"max":{"1min":4.3550000000000004,"5min":10.999000000000001,"15min":11.112},"last":0.34300000000000003}]},{"osd":7,"last update":"Sat Sep 23 21:14:48 2023","interfaces":[{"interface":"back","average":{"1min":0.84999999999999998,"5min":0.98499999999999999,"15min":1.8129999999999999},"min":{"1min":0.34499999999999997,"5min":0.31,"15min":0.309},"max":{"1min":3.3039999999999998,"5min":5.5659999999999998,"15min":189.405},"last":0.38900000000000001},{"interface":"front","average":{"1min":0.92000000000000004,"5min":1.472,"15min":1.9259999999999999},"min":{"1min":0.40899999999999997,"5min":0.35399999999999998,"15min":0.32500000000000001},"max":{"1min":3.3239999999999998,"5min":11.183999999999999,"15min":189.685},"last":0.45000000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8488,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1259,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5322,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":714,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":5434,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":4838,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":783613952,"data_stored":783613952,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1089863680,"data_stored":1089863680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":235536384,"data_stored":235536384,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":566628352,"data_stored":566624280,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":146866176,"data_stored":146866176,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":109576192,"data_stored":109576192,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T21:15:19.906 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph status --format=json 2023-09-23T21:15:20.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:15:19 smithi070 ceph-mon[168282]: failed to encode map e538 with expected crc 2023-09-23T21:15:20.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:15:19 smithi070 ceph-mon[168282]: failed to encode map e538 with expected crc 2023-09-23T21:15:20.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:15:19 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_4", "pool2": "unique_pool_4", "yes_i_really_really_mean_it": true}]': finished 2023-09-23T21:15:20.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:15:19 smithi070 ceph-mon[168282]: osdmap e538: 8 total, 8 up, 6 in 2023-09-23T21:15:20.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:15:19 smithi070 ceph-mon[168282]: pgmap v1363: 103 pgs: 103 active+clean; 687 KiB data, 5.0 GiB used, 531 GiB / 536 GiB avail 2023-09-23T21:15:20.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:15:19 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "31"}]: dispatch 2023-09-23T21:15:20.131 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:15:19 smithi026 ceph-mon[175151]: failed to encode map e538 with expected crc 2023-09-23T21:15:20.131 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:15:19 smithi026 ceph-mon[175151]: failed to encode map e538 with expected crc 2023-09-23T21:15:20.131 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:15:19 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_4", "pool2": "unique_pool_4", "yes_i_really_really_mean_it": true}]': finished 2023-09-23T21:15:20.131 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:15:19 smithi026 ceph-mon[175151]: osdmap e538: 8 total, 8 up, 6 in 2023-09-23T21:15:20.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:15:19 smithi026 ceph-mon[175151]: pgmap v1363: 103 pgs: 103 active+clean; 687 KiB data, 5.0 GiB used, 531 GiB / 536 GiB avail 2023-09-23T21:15:20.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:15:19 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "31"}]: dispatch 2023-09-23T21:15:20.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:15:19 smithi026 ceph-mon[179688]: failed to encode map e538 with expected crc 2023-09-23T21:15:20.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:15:19 smithi026 ceph-mon[179688]: failed to encode map e538 with expected crc 2023-09-23T21:15:20.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:15:19 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_4", "pool2": "unique_pool_4", "yes_i_really_really_mean_it": true}]': finished 2023-09-23T21:15:20.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:15:19 smithi026 ceph-mon[179688]: osdmap e538: 8 total, 8 up, 6 in 2023-09-23T21:15:20.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:15:19 smithi026 ceph-mon[179688]: pgmap v1363: 103 pgs: 103 active+clean; 687 KiB data, 5.0 GiB used, 531 GiB / 536 GiB avail 2023-09-23T21:15:20.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:15:19 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "31"}]: dispatch 2023-09-23T21:15:20.271 INFO:teuthology.task.full_sequential:In full_sequential, running task radosbench... 2023-09-23T21:15:20.271 INFO:tasks.radosbench:Beginning radosbench... 2023-09-23T21:15:20.271 INFO:tasks.cephadm.ceph_manager.ceph:creating pool_name unique_pool_5 2023-09-23T21:15:20.271 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph osd pool create unique_pool_5 16 2023-09-23T21:15:20.447 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:15:20.451 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:15:20.452 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:15:20.553 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:15:20.589 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:15:20.799 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:15:20 smithi026 ceph-mon[175151]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 17467/1392 objects degraded (1254.813%), 16 pgs degraded, 15 pgs undersized) 2023-09-23T21:15:20.799 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:15:20 smithi026 ceph-mon[175151]: failed to encode map e539 with expected crc 2023-09-23T21:15:20.943 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:15:20.947 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:15:20.947 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:15:21.048 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:15:21.056 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:15:20 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "31"}]': finished 2023-09-23T21:15:21.056 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:15:20 smithi026 ceph-mon[175151]: osdmap e539: 8 total, 8 up, 6 in 2023-09-23T21:15:21.056 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:15:20 smithi026 ceph-mon[175151]: failed to encode map e539 with expected crc 2023-09-23T21:15:21.056 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:15:20 smithi026 ceph-mon[175151]: failed to encode map e539 with expected crc 2023-09-23T21:15:21.056 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:15:20 smithi026 ceph-mon[175151]: failed to encode map e539 with expected crc 2023-09-23T21:15:21.056 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:15:20 smithi026 ceph-mon[175151]: failed to encode map e539 with expected crc 2023-09-23T21:15:21.056 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:15:20 smithi026 ceph-mon[175151]: osdmap e540: 8 total, 8 up, 6 in 2023-09-23T21:15:21.057 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:15:20 smithi026 ceph-mon[179688]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 17467/1392 objects degraded (1254.813%), 16 pgs degraded, 15 pgs undersized) 2023-09-23T21:15:21.057 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:15:20 smithi026 ceph-mon[179688]: failed to encode map e539 with expected crc 2023-09-23T21:15:21.057 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:15:20 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "31"}]': finished 2023-09-23T21:15:21.057 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:15:20 smithi026 ceph-mon[179688]: osdmap e539: 8 total, 8 up, 6 in 2023-09-23T21:15:21.057 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:15:20 smithi026 ceph-mon[179688]: failed to encode map e539 with expected crc 2023-09-23T21:15:21.057 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:15:20 smithi026 ceph-mon[179688]: failed to encode map e539 with expected crc 2023-09-23T21:15:21.057 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:15:20 smithi026 ceph-mon[179688]: failed to encode map e539 with expected crc 2023-09-23T21:15:21.057 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:15:20 smithi026 ceph-mon[179688]: failed to encode map e539 with expected crc 2023-09-23T21:15:21.057 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:15:20 smithi026 ceph-mon[179688]: osdmap e540: 8 total, 8 up, 6 in 2023-09-23T21:15:21.077 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:15:20 smithi070 ceph-mon[168282]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 17467/1392 objects degraded (1254.813%), 16 pgs degraded, 15 pgs undersized) 2023-09-23T21:15:21.077 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:15:20 smithi070 ceph-mon[168282]: failed to encode map e539 with expected crc 2023-09-23T21:15:21.078 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:15:20 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "31"}]': finished 2023-09-23T21:15:21.078 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:15:20 smithi070 ceph-mon[168282]: osdmap e539: 8 total, 8 up, 6 in 2023-09-23T21:15:21.078 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:15:20 smithi070 ceph-mon[168282]: failed to encode map e539 with expected crc 2023-09-23T21:15:21.078 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:15:20 smithi070 ceph-mon[168282]: failed to encode map e539 with expected crc 2023-09-23T21:15:21.078 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:15:20 smithi070 ceph-mon[168282]: failed to encode map e539 with expected crc 2023-09-23T21:15:21.078 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:15:20 smithi070 ceph-mon[168282]: failed to encode map e539 with expected crc 2023-09-23T21:15:21.078 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:15:20 smithi070 ceph-mon[168282]: osdmap e540: 8 total, 8 up, 6 in 2023-09-23T21:15:21.080 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:15:21.116 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:15:21.119 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:15:21.120 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:15:21.220 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:15:21.250 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:15:21.286 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:15:21.289 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:15:21.289 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:15:21.315 INFO:teuthology.orchestra.run.smithi026.stdout:ERROR: (22) Invalid argument 2023-09-23T21:15:21.315 INFO:teuthology.orchestra.run.smithi026.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T21:15:21.390 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:15:21.423 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:15:21.428 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:15:21.509 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0: {} 2023-09-23T21:15:21.531 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1: {} 2023-09-23T21:15:21.560 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2: {} 2023-09-23T21:15:21.579 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3: {} 2023-09-23T21:15:21.599 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4: {} 2023-09-23T21:15:21.619 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T21:15:21.624 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:15:21.639 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6: {} 2023-09-23T21:15:21.671 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T21:15:21.672 INFO:teuthology.orchestra.run.smithi026.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T21:15:21.672 INFO:teuthology.orchestra.run.smithi026.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T21:15:21.672 INFO:teuthology.orchestra.run.smithi026.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T21:15:21.672 INFO:teuthology.orchestra.run.smithi026.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T21:15:21.672 INFO:teuthology.orchestra.run.smithi026.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T21:15:21.673 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T21:15:21.673 INFO:teuthology.orchestra.run.smithi026.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T21:15:21.673 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T21:15:21.805 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:15:21 smithi026 ceph-mon[175151]: failed to encode map e540 with expected crc 2023-09-23T21:15:21.805 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:15:21 smithi026 ceph-mon[175151]: failed to encode map e540 with expected crc 2023-09-23T21:15:21.806 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:15:21 smithi026 ceph-mon[175151]: failed to encode map e540 with expected crc 2023-09-23T21:15:21.806 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:15:21 smithi026 ceph-mon[175151]: failed to encode map e540 with expected crc 2023-09-23T21:15:21.806 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:15:21 smithi026 ceph-mon[175151]: failed to encode map e540 with expected crc 2023-09-23T21:15:21.806 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:15:21 smithi026 ceph-mon[175151]: failed to encode map e540 with expected crc 2023-09-23T21:15:21.806 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:15:21 smithi026 ceph-mon[175151]: pgmap v1366: 103 pgs: 2 unknown, 101 active+clean; 687 KiB data, 4.9 GiB used, 532 GiB / 536 GiB avail 2023-09-23T21:15:21.806 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:15:21 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/4224661889' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:15:21.806 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:15:21 smithi026 ceph-mon[175151]: osdmap e541: 8 total, 8 up, 6 in 2023-09-23T21:15:21.806 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:15:21 smithi026 ceph-mon[179688]: failed to encode map e540 with expected crc 2023-09-23T21:15:21.806 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:15:21 smithi026 ceph-mon[179688]: failed to encode map e540 with expected crc 2023-09-23T21:15:21.807 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:15:21 smithi026 ceph-mon[179688]: failed to encode map e540 with expected crc 2023-09-23T21:15:21.807 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:15:21 smithi026 ceph-mon[179688]: failed to encode map e540 with expected crc 2023-09-23T21:15:21.807 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:15:21 smithi026 ceph-mon[179688]: failed to encode map e540 with expected crc 2023-09-23T21:15:21.807 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:15:21 smithi026 ceph-mon[179688]: failed to encode map e540 with expected crc 2023-09-23T21:15:21.807 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:15:21 smithi026 ceph-mon[179688]: pgmap v1366: 103 pgs: 2 unknown, 101 active+clean; 687 KiB data, 4.9 GiB used, 532 GiB / 536 GiB avail 2023-09-23T21:15:21.807 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:15:21 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/4224661889' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:15:21.807 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:15:21 smithi026 ceph-mon[179688]: osdmap e541: 8 total, 8 up, 6 in 2023-09-23T21:15:22.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:15:21 smithi070 ceph-mon[168282]: failed to encode map e540 with expected crc 2023-09-23T21:15:22.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:15:21 smithi070 ceph-mon[168282]: failed to encode map e540 with expected crc 2023-09-23T21:15:22.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:15:21 smithi070 ceph-mon[168282]: failed to encode map e540 with expected crc 2023-09-23T21:15:22.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:15:21 smithi070 ceph-mon[168282]: failed to encode map e540 with expected crc 2023-09-23T21:15:22.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:15:21 smithi070 ceph-mon[168282]: failed to encode map e540 with expected crc 2023-09-23T21:15:22.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:15:21 smithi070 ceph-mon[168282]: failed to encode map e540 with expected crc 2023-09-23T21:15:22.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:15:21 smithi070 ceph-mon[168282]: pgmap v1366: 103 pgs: 2 unknown, 101 active+clean; 687 KiB data, 4.9 GiB used, 532 GiB / 536 GiB avail 2023-09-23T21:15:22.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:15:21 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/4224661889' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:15:22.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:15:21 smithi070 ceph-mon[168282]: osdmap e541: 8 total, 8 up, 6 in 2023-09-23T21:15:22.123 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T21:15:22.127 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:15:22.127 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:15:22.228 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:15:22.251 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_historic_ops 2023-09-23T21:15:22.260 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:15:22.297 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:15:22.300 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:15:22.300 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:15:22.401 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:15:22.429 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:15:22.463 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:15:22.466 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:15:22.466 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:15:22.567 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:15:22.594 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:15:22.628 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:15:22.631 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:15:22.631 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:15:22.732 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:15:22.762 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:15:22.801 INFO:teuthology.orchestra.run.smithi026.stderr:nodeep-scrub is set 2023-09-23T21:15:22.807 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:15:22.812 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:15:22.813 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:15:22.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:15:22 smithi070 ceph-mon[168282]: failed to encode map e541 with expected crc 2023-09-23T21:15:22.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:15:22 smithi070 ceph-mon[168282]: failed to encode map e541 with expected crc 2023-09-23T21:15:22.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:15:22 smithi070 ceph-mon[168282]: failed to encode map e541 with expected crc 2023-09-23T21:15:22.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:15:22 smithi070 ceph-mon[168282]: failed to encode map e541 with expected crc 2023-09-23T21:15:22.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:15:22 smithi070 ceph-mon[168282]: failed to encode map e541 with expected crc 2023-09-23T21:15:22.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:15:22 smithi070 ceph-mon[168282]: failed to encode map e541 with expected crc 2023-09-23T21:15:22.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:15:22 smithi070 ceph-mon[168282]: failed to encode map e541 with expected crc 2023-09-23T21:15:22.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:15:22 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/195945709' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:15:22.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:15:22 smithi070 ceph-mon[168282]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:15:22.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:15:22 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/195945709' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T21:15:22.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:15:22 smithi070 ceph-mon[168282]: osdmap e542: 8 total, 8 up, 6 in 2023-09-23T21:15:22.913 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:15:22.943 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:15:23.131 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:15:22 smithi026 ceph-mon[175151]: failed to encode map e541 with expected crc 2023-09-23T21:15:23.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:15:22 smithi026 ceph-mon[175151]: failed to encode map e541 with expected crc 2023-09-23T21:15:23.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:15:22 smithi026 ceph-mon[175151]: failed to encode map e541 with expected crc 2023-09-23T21:15:23.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:15:22 smithi026 ceph-mon[175151]: failed to encode map e541 with expected crc 2023-09-23T21:15:23.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:15:22 smithi026 ceph-mon[175151]: failed to encode map e541 with expected crc 2023-09-23T21:15:23.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:15:22 smithi026 ceph-mon[175151]: failed to encode map e541 with expected crc 2023-09-23T21:15:23.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:15:22 smithi026 ceph-mon[175151]: failed to encode map e541 with expected crc 2023-09-23T21:15:23.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:15:22 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/195945709' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:15:23.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:15:22 smithi026 ceph-mon[175151]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:15:23.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:15:22 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/195945709' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T21:15:23.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:15:22 smithi026 ceph-mon[175151]: osdmap e542: 8 total, 8 up, 6 in 2023-09-23T21:15:23.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:15:22 smithi026 ceph-mon[179688]: failed to encode map e541 with expected crc 2023-09-23T21:15:23.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:15:22 smithi026 ceph-mon[179688]: failed to encode map e541 with expected crc 2023-09-23T21:15:23.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:15:22 smithi026 ceph-mon[179688]: failed to encode map e541 with expected crc 2023-09-23T21:15:23.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:15:22 smithi026 ceph-mon[179688]: failed to encode map e541 with expected crc 2023-09-23T21:15:23.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:15:22 smithi026 ceph-mon[179688]: failed to encode map e541 with expected crc 2023-09-23T21:15:23.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:15:22 smithi026 ceph-mon[179688]: failed to encode map e541 with expected crc 2023-09-23T21:15:23.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:15:22 smithi026 ceph-mon[179688]: failed to encode map e541 with expected crc 2023-09-23T21:15:23.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:15:22 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/195945709' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:15:23.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:15:22 smithi026 ceph-mon[179688]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:15:23.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:15:22 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/195945709' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T21:15:23.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:15:22 smithi026 ceph-mon[179688]: osdmap e542: 8 total, 8 up, 6 in 2023-09-23T21:15:23.503 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:15:23.507 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:15:23.507 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:15:23.608 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:15:23.637 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:15:23.721 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:15:23.725 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:15:23.725 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:15:23.811 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:15:23 smithi026 ceph-mon[179688]: failed to encode map e542 with expected crc 2023-09-23T21:15:23.812 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:15:23 smithi026 ceph-mon[179688]: failed to encode map e542 with expected crc 2023-09-23T21:15:23.812 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:15:23 smithi026 ceph-mon[179688]: failed to encode map e542 with expected crc 2023-09-23T21:15:23.812 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:15:23 smithi026 ceph-mon[179688]: failed to encode map e542 with expected crc 2023-09-23T21:15:23.812 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:15:23 smithi026 ceph-mon[179688]: pgmap v1369: 103 pgs: 3 active+undersized+remapped, 2 remapped+peering, 2 unknown, 96 active+clean; 674 KiB data, 4.9 GiB used, 532 GiB / 536 GiB avail; 54 KiB/s, 0 objects/s recovering 2023-09-23T21:15:23.812 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:15:23 smithi026 ceph-mon[179688]: osdmap e543: 8 total, 8 up, 6 in 2023-09-23T21:15:23.826 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:15:23.854 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:15:23.893 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:15:23.898 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:15:23.898 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:15:23.999 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:15:24.030 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:15:24.067 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:15:24.069 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:15:23 smithi026 ceph-mon[175151]: failed to encode map e542 with expected crc 2023-09-23T21:15:24.069 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:15:23 smithi026 ceph-mon[175151]: failed to encode map e542 with expected crc 2023-09-23T21:15:24.069 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:15:23 smithi026 ceph-mon[175151]: failed to encode map e542 with expected crc 2023-09-23T21:15:24.069 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:15:23 smithi026 ceph-mon[175151]: failed to encode map e542 with expected crc 2023-09-23T21:15:24.070 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:15:23 smithi026 ceph-mon[175151]: pgmap v1369: 103 pgs: 3 active+undersized+remapped, 2 remapped+peering, 2 unknown, 96 active+clean; 674 KiB data, 4.9 GiB used, 532 GiB / 536 GiB avail; 54 KiB/s, 0 objects/s recovering 2023-09-23T21:15:24.070 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:15:23 smithi026 ceph-mon[175151]: osdmap e543: 8 total, 8 up, 6 in 2023-09-23T21:15:24.070 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:15:24.071 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:15:24.115 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:15:24.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:15:23 smithi070 ceph-mon[168282]: failed to encode map e542 with expected crc 2023-09-23T21:15:24.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:15:23 smithi070 ceph-mon[168282]: failed to encode map e542 with expected crc 2023-09-23T21:15:24.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:15:23 smithi070 ceph-mon[168282]: failed to encode map e542 with expected crc 2023-09-23T21:15:24.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:15:23 smithi070 ceph-mon[168282]: failed to encode map e542 with expected crc 2023-09-23T21:15:24.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:15:23 smithi070 ceph-mon[168282]: pgmap v1369: 103 pgs: 3 active+undersized+remapped, 2 remapped+peering, 2 unknown, 96 active+clean; 674 KiB data, 4.9 GiB used, 532 GiB / 536 GiB avail; 54 KiB/s, 0 objects/s recovering 2023-09-23T21:15:24.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:15:23 smithi070 ceph-mon[168282]: osdmap e543: 8 total, 8 up, 6 in 2023-09-23T21:15:24.172 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:15:24.203 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:15:24.312 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T21:15:24.436 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:15:24.439 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:15:24.440 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:15:24.540 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:15:24.573 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:15:24.610 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:15:24.613 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:15:24.613 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:15:24.714 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:15:24.747 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:15:24.748 INFO:teuthology.orchestra.run.smithi026.stdout: 2023-09-23T21:15:24.748 INFO:teuthology.orchestra.run.smithi026.stdout:{"fsid":"b3f8e4ec-5a50-11ee-8db4-212e2dc638e7","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"noscrub,nodeep-scrub flag(s) set","count":20},"muted":false},"PG_AVAILABILITY":{"severity":"HEALTH_WARN","summary":{"message":"Reduced data availability: 1 pg inactive, 2 pgs peering","count":3},"muted":false}},"mutes":[]},"election_epoch":30,"quorum":[0,1,2],"quorum_names":["a","b","c"],"quorum_age":1643,"monmap":{"epoch":4,"min_mon_release_name":"reef","num_mons":3},"osdmap":{"epoch":543,"num_osds":8,"num_up_osds":8,"osd_up_since":1695503591,"num_in_osds":6,"osd_in_since":1695503530,"num_remapped_pgs":0},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":96},{"state_name":"active+undersized+remapped","count":3},{"state_name":"unknown","count":2},{"state_name":"remapped+peering","count":2}],"num_pgs":103,"num_pools":4,"num_objects":25,"data_bytes":689794,"bytes_used":5207691264,"bytes_avail":570762522624,"bytes_total":575970213888,"unknown_pgs_ratio":0.019417475908994675,"inactive_pgs_ratio":0.019417475908994675,"recovering_objects_per_sec":0,"recovering_bytes_per_sec":55588,"recovering_keys_per_sec":0,"num_objects_recovered":1,"num_bytes_recovered":222435,"num_keys_recovered":0},"fsmap":{"epoch":8,"id":1,"up":1,"in":1,"max":1,"by_rank":[{"filesystem_id":1,"rank":0,"name":"foo.smithi026.znfpnl","status":"up:active","gid":14658}],"up:standby":1},"mgrmap":{"available":true,"num_standbys":1,"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"services":{"dashboard":"https://172.21.15.26:8443/","prometheus":"http://172.21.15.26:9283/"}},"servicemap":{"epoch":28,"modified":"2023-09-23T21:14:11.047835+0000","services":{}},"progress_events":{"0ed42496-3a31-41bd-a432-1bbd8c010433":{"message":"Global Recovery Event (3m)\n [==========================..] (remaining: 8s)","progress":0.96116507053375244,"add_to_ceph_s":true},"fc746e9d-3f76-434d-b6d2-8fc9e8a72a5e":{"message":"Upgrade to 18.2.0-460-gbb0daeff (4m)\n [============................] (remaining: 5m)","progress":0.44999998807907104,"add_to_ceph_s":true}}} 2023-09-23T21:15:24.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:15:24 smithi070 ceph-mon[168282]: failed to encode map e543 with expected crc 2023-09-23T21:15:24.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:15:24 smithi070 ceph-mon[168282]: failed to encode map e543 with expected crc 2023-09-23T21:15:24.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:15:24 smithi070 ceph-mon[168282]: failed to encode map e543 with expected crc 2023-09-23T21:15:24.871 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:15:24 smithi070 ceph-mon[168282]: failed to encode map e543 with expected crc 2023-09-23T21:15:24.871 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:15:24 smithi070 ceph-mon[168282]: failed to encode map e543 with expected crc 2023-09-23T21:15:24.871 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:15:24 smithi070 ceph-mon[168282]: failed to encode map e543 with expected crc 2023-09-23T21:15:24.871 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:15:24 smithi070 ceph-mon[168282]: Health check failed: Reduced data availability: 1 pg inactive, 2 pgs peering (PG_AVAILABILITY) 2023-09-23T21:15:24.875 INFO:teuthology.orchestra.run.smithi026.stderr:pool 'unique_pool_5' created 2023-09-23T21:15:24.881 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:15:24 smithi026 ceph-mon[179688]: failed to encode map e543 with expected crc 2023-09-23T21:15:24.881 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:15:24 smithi026 ceph-mon[179688]: failed to encode map e543 with expected crc 2023-09-23T21:15:24.881 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:15:24 smithi026 ceph-mon[179688]: failed to encode map e543 with expected crc 2023-09-23T21:15:24.881 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:15:24 smithi026 ceph-mon[179688]: failed to encode map e543 with expected crc 2023-09-23T21:15:24.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:15:24 smithi026 ceph-mon[179688]: failed to encode map e543 with expected crc 2023-09-23T21:15:24.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:15:24 smithi026 ceph-mon[179688]: failed to encode map e543 with expected crc 2023-09-23T21:15:24.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:15:24 smithi026 ceph-mon[179688]: Health check failed: Reduced data availability: 1 pg inactive, 2 pgs peering (PG_AVAILABILITY) 2023-09-23T21:15:24.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:15:24 smithi026 ceph-mon[175151]: failed to encode map e543 with expected crc 2023-09-23T21:15:24.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:15:24 smithi026 ceph-mon[175151]: failed to encode map e543 with expected crc 2023-09-23T21:15:24.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:15:24 smithi026 ceph-mon[175151]: failed to encode map e543 with expected crc 2023-09-23T21:15:24.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:15:24 smithi026 ceph-mon[175151]: failed to encode map e543 with expected crc 2023-09-23T21:15:24.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:15:24 smithi026 ceph-mon[175151]: failed to encode map e543 with expected crc 2023-09-23T21:15:24.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:15:24 smithi026 ceph-mon[175151]: failed to encode map e543 with expected crc 2023-09-23T21:15:24.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:15:24 smithi026 ceph-mon[175151]: Health check failed: Reduced data availability: 1 pg inactive, 2 pgs peering (PG_AVAILABILITY) 2023-09-23T21:15:25.608 INFO:tasks.cephadm.ceph_manager.ceph:making progress, resetting timeout 2023-09-23T21:15:25.608 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T21:15:25.609 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph osd pool application enable unique_pool_5 rados --yes-i-really-mean-it || true 2023-09-23T21:15:25.667 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:15:25.714 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:15:25.719 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:15:25.719 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:15:25.820 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:15:25.854 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:15:25.895 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T21:15:25.914 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:15:25.918 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:15:25.918 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:15:25.935 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:15:25 smithi026 ceph-mon[179688]: failed to encode map e543 with expected crc 2023-09-23T21:15:25.936 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:15:25 smithi026 ceph-mon[179688]: failed to encode map e543 with expected crc 2023-09-23T21:15:25.936 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:15:25 smithi026 ceph-mon[179688]: failed to encode map e543 with expected crc 2023-09-23T21:15:25.936 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:15:25 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/4112792893' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_5", "pg_num": 16}]: dispatch 2023-09-23T21:15:25.936 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:15:25 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_5", "pg_num": 16}]: dispatch 2023-09-23T21:15:25.936 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:15:25 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/3188106764' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-23T21:15:25.936 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:15:25 smithi026 ceph-mon[179688]: failed to encode map e544 with expected crc 2023-09-23T21:15:25.936 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:15:25 smithi026 ceph-mon[179688]: failed to encode map e544 with expected crc 2023-09-23T21:15:25.936 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:15:25 smithi026 ceph-mon[179688]: failed to encode map e544 with expected crc 2023-09-23T21:15:25.936 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:15:25 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_5", "pg_num": 16}]': finished 2023-09-23T21:15:25.936 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:15:25 smithi026 ceph-mon[179688]: osdmap e544: 8 total, 8 up, 6 in 2023-09-23T21:15:25.936 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:15:25 smithi026 ceph-mon[179688]: pgmap v1372: 119 pgs: 3 active+undersized+remapped, 1 remapped+peering, 18 unknown, 97 active+clean; 674 KiB data, 4.8 GiB used, 532 GiB / 536 GiB avail; 54 KiB/s, 0 objects/s recovering 2023-09-23T21:15:25.937 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:15:25 smithi026 ceph-mon[175151]: failed to encode map e543 with expected crc 2023-09-23T21:15:25.937 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:15:25 smithi026 ceph-mon[175151]: failed to encode map e543 with expected crc 2023-09-23T21:15:25.937 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:15:25 smithi026 ceph-mon[175151]: failed to encode map e543 with expected crc 2023-09-23T21:15:25.937 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:15:25 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/4112792893' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_5", "pg_num": 16}]: dispatch 2023-09-23T21:15:25.937 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:15:25 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_5", "pg_num": 16}]: dispatch 2023-09-23T21:15:25.937 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:15:25 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/3188106764' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-23T21:15:25.937 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:15:25 smithi026 ceph-mon[175151]: failed to encode map e544 with expected crc 2023-09-23T21:15:25.937 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:15:25 smithi026 ceph-mon[175151]: failed to encode map e544 with expected crc 2023-09-23T21:15:25.937 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:15:25 smithi026 ceph-mon[175151]: failed to encode map e544 with expected crc 2023-09-23T21:15:25.937 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:15:25 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_5", "pg_num": 16}]': finished 2023-09-23T21:15:25.937 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:15:25 smithi026 ceph-mon[175151]: osdmap e544: 8 total, 8 up, 6 in 2023-09-23T21:15:25.938 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:15:25 smithi026 ceph-mon[175151]: pgmap v1372: 119 pgs: 3 active+undersized+remapped, 1 remapped+peering, 18 unknown, 97 active+clean; 674 KiB data, 4.8 GiB used, 532 GiB / 536 GiB avail; 54 KiB/s, 0 objects/s recovering 2023-09-23T21:15:26.019 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:15:26.022 INFO:teuthology.orchestra.run.smithi026.stdout:{ 2023-09-23T21:15:26.022 INFO:teuthology.orchestra.run.smithi026.stdout: "size": 20, 2023-09-23T21:15:26.022 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 600, 2023-09-23T21:15:26.022 INFO:teuthology.orchestra.run.smithi026.stdout: "ops": [ 2023-09-23T21:15:26.022 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:26.022 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.46328.0:52880 10.f 10:fac33bdf:::benchmark_data_smithi026_379838_object52879:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e443)", 2023-09-23T21:15:26.023 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T21:11:07.890130+0000", 2023-09-23T21:15:26.023 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 258.12982441899999, 2023-09-23T21:15:26.023 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.0274747440000001, 2023-09-23T21:15:26.023 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:15:26.023 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:15:26.023 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T21:15:26.023 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.46328", 2023-09-23T21:15:26.023 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/3748861181", 2023-09-23T21:15:26.023 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 52880 2023-09-23T21:15:26.023 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:26.023 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:15:26.023 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:26.024 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:15:26.024 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:11:07.890130+0000", 2023-09-23T21:15:26.024 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:15:26.024 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:26.024 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:26.024 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:15:26.024 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:11:07.890130+0000", 2023-09-23T21:15:26.024 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:15:26.024 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:26.024 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:26.024 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:15:26.024 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:11:07.890128+0000", 2023-09-23T21:15:26.025 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T21:15:26.025 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:26.025 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:26.025 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:15:26.025 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:11:07.890152+0000", 2023-09-23T21:15:26.025 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.4278999999999999e-05 2023-09-23T21:15:26.025 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:26.025 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:26.025 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:15:26.025 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:11:07.890170+0000", 2023-09-23T21:15:26.025 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.7439e-05 2023-09-23T21:15:26.025 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:26.025 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:26.026 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:15:26.026 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:11:07.890178+0000", 2023-09-23T21:15:26.026 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.2090000000000005e-06 2023-09-23T21:15:26.026 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:26.026 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:26.026 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:15:26.026 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:11:07.890233+0000", 2023-09-23T21:15:26.026 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.4895000000000002e-05 2023-09-23T21:15:26.026 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:26.026 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:26.026 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T21:15:26.026 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:11:07.890244+0000", 2023-09-23T21:15:26.027 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.129e-05 2023-09-23T21:15:26.027 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:26.027 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:26.027 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:15:26.027 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:11:08.752714+0000", 2023-09-23T21:15:26.027 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.86246954600000003 2023-09-23T21:15:26.027 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:26.027 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:26.027 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T21:15:26.027 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:11:08.752730+0000", 2023-09-23T21:15:26.027 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.5832999999999999e-05 2023-09-23T21:15:26.027 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:26.028 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:26.028 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:15:26.028 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:11:09.903407+0000", 2023-09-23T21:15:26.028 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.150676941 2023-09-23T21:15:26.028 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:26.028 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:26.028 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:15:26.028 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:11:09.903612+0000", 2023-09-23T21:15:26.028 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00020550899999999999 2023-09-23T21:15:26.028 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:26.028 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:26.028 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 2,4", 2023-09-23T21:15:26.028 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:11:09.903848+0000", 2023-09-23T21:15:26.029 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000236357 2023-09-23T21:15:26.029 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:26.029 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:26.029 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:15:26.029 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:11:09.911070+0000", 2023-09-23T21:15:26.029 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.007221529 2023-09-23T21:15:26.029 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:26.029 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:26.029 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:15:26.029 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:11:09.911137+0000", 2023-09-23T21:15:26.029 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.6685000000000003e-05 2023-09-23T21:15:26.029 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:26.030 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:26.030 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T21:15:26.030 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:11:09.915716+0000", 2023-09-23T21:15:26.030 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0045796450000000002 2023-09-23T21:15:26.030 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:26.030 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:26.030 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T21:15:26.030 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:11:09.915773+0000", 2023-09-23T21:15:26.030 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.6789999999999997e-05 2023-09-23T21:15:26.030 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:26.030 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:26.030 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T21:15:26.030 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:11:09.917604+0000", 2023-09-23T21:15:26.031 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0018312980000000001 2023-09-23T21:15:26.031 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:15:26.031 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:15:26.031 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:15:26.031 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:26.031 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:26.031 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.46328.0:52887 10.f 10:fbdd74d4:::benchmark_data_smithi026_379838_object52886:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e443)", 2023-09-23T21:15:26.031 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T21:11:07.896648+0000", 2023-09-23T21:15:26.031 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 258.12330609200001, 2023-09-23T21:15:26.031 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.0227663140000001, 2023-09-23T21:15:26.031 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:15:26.031 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:15:26.032 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T21:15:26.032 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.46328", 2023-09-23T21:15:26.032 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/3748861181", 2023-09-23T21:15:26.032 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 52887 2023-09-23T21:15:26.032 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:26.032 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:15:26.032 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:26.032 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:15:26.032 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:11:07.896648+0000", 2023-09-23T21:15:26.032 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:15:26.032 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:26.032 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:26.032 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:15:26.033 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:11:07.896648+0000", 2023-09-23T21:15:26.033 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:15:26.033 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:26.033 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:26.033 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:15:26.033 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:11:07.896646+0000", 2023-09-23T21:15:26.033 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999981 2023-09-23T21:15:26.033 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:26.033 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:26.033 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:15:26.033 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:11:07.896690+0000", 2023-09-23T21:15:26.033 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.3930000000000001e-05 2023-09-23T21:15:26.034 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:26.034 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:26.034 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:15:26.034 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:11:07.896716+0000", 2023-09-23T21:15:26.034 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.5618000000000001e-05 2023-09-23T21:15:26.034 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:26.034 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:26.034 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:15:26.034 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:11:07.896725+0000", 2023-09-23T21:15:26.034 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.7490000000000004e-06 2023-09-23T21:15:26.034 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:26.034 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:26.035 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:15:26.035 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:11:07.896786+0000", 2023-09-23T21:15:26.035 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.1486000000000001e-05 2023-09-23T21:15:26.035 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:26.035 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:26.035 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T21:15:26.035 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:11:07.896801+0000", 2023-09-23T21:15:26.035 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.4833e-05 2023-09-23T21:15:26.035 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:26.035 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:26.035 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:15:26.035 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:11:08.752895+0000", 2023-09-23T21:15:26.036 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.856094621 2023-09-23T21:15:26.036 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:26.036 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:26.036 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T21:15:26.036 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:11:08.752907+0000", 2023-09-23T21:15:26.036 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1761e-05 2023-09-23T21:15:26.036 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:26.036 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:26.036 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:15:26.036 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:11:09.908371+0000", 2023-09-23T21:15:26.036 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.155464222 2023-09-23T21:15:26.036 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:26.037 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:26.037 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:15:26.037 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:11:09.908622+0000", 2023-09-23T21:15:26.037 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00025081700000000002 2023-09-23T21:15:26.037 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:26.037 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:26.037 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 2,4", 2023-09-23T21:15:26.037 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:11:09.908830+0000", 2023-09-23T21:15:26.037 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000207767 2023-09-23T21:15:26.037 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:26.037 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:26.037 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:15:26.038 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:11:09.917668+0000", 2023-09-23T21:15:26.038 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0088383479999999993 2023-09-23T21:15:26.038 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:26.038 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:26.038 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T21:15:26.038 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:11:09.917729+0000", 2023-09-23T21:15:26.038 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.0803e-05 2023-09-23T21:15:26.038 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:26.038 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:26.038 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:15:26.038 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:11:09.917804+0000", 2023-09-23T21:15:26.038 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.4768e-05 2023-09-23T21:15:26.038 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:26.039 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:26.039 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T21:15:26.039 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:11:09.917847+0000", 2023-09-23T21:15:26.039 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.2777000000000001e-05 2023-09-23T21:15:26.039 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:26.039 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:26.039 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T21:15:26.039 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:11:09.919414+0000", 2023-09-23T21:15:26.039 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0015675400000000001 2023-09-23T21:15:26.039 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:15:26.039 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:15:26.039 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:15:26.040 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:26.040 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:26.040 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.46328.0:52896 10.f 10:f2aef048:::benchmark_data_smithi026_379838_object52895:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e443)", 2023-09-23T21:15:26.040 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T21:11:07.901431+0000", 2023-09-23T21:15:26.040 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 258.11852317400002, 2023-09-23T21:15:26.040 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.019779099, 2023-09-23T21:15:26.040 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:15:26.040 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:15:26.040 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T21:15:26.040 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.46328", 2023-09-23T21:15:26.040 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/3748861181", 2023-09-23T21:15:26.040 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 52896 2023-09-23T21:15:26.041 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:26.041 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:15:26.041 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:26.041 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:15:26.041 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:11:07.901431+0000", 2023-09-23T21:15:26.041 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:15:26.041 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:26.041 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:26.041 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:15:26.041 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:11:07.901431+0000", 2023-09-23T21:15:26.041 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:15:26.041 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:26.042 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:26.042 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:15:26.042 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:11:07.901429+0000", 2023-09-23T21:15:26.042 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T21:15:26.042 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:26.042 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:26.042 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:15:26.042 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:11:07.901455+0000", 2023-09-23T21:15:26.042 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.5446e-05 2023-09-23T21:15:26.042 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:26.042 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:26.042 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:15:26.043 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:11:07.901471+0000", 2023-09-23T21:15:26.043 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.6279000000000001e-05 2023-09-23T21:15:26.043 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:26.043 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:26.043 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:15:26.043 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:11:07.901485+0000", 2023-09-23T21:15:26.043 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.4406999999999999e-05 2023-09-23T21:15:26.043 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:26.043 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:26.043 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:15:26.043 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:11:07.901571+0000", 2023-09-23T21:15:26.043 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.5341999999999997e-05 2023-09-23T21:15:26.044 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:26.044 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:26.044 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T21:15:26.044 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:11:07.901582+0000", 2023-09-23T21:15:26.044 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1362000000000001e-05 2023-09-23T21:15:26.044 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:26.044 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:26.044 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:15:26.044 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:11:08.752947+0000", 2023-09-23T21:15:26.044 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.85136492900000005 2023-09-23T21:15:26.044 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:26.044 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:26.045 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T21:15:26.045 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:11:08.752959+0000", 2023-09-23T21:15:26.045 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1695000000000001e-05 2023-09-23T21:15:26.045 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:26.045 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:26.045 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:15:26.045 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:11:09.911237+0000", 2023-09-23T21:15:26.045 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.158278355 2023-09-23T21:15:26.045 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:26.045 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:26.045 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:15:26.045 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:11:09.911413+0000", 2023-09-23T21:15:26.045 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000175766 2023-09-23T21:15:26.046 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:26.046 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:26.046 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 2,4", 2023-09-23T21:15:26.046 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:11:09.911527+0000", 2023-09-23T21:15:26.046 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000114356 2023-09-23T21:15:26.046 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:26.046 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:26.046 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:15:26.046 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:11:09.917715+0000", 2023-09-23T21:15:26.046 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0061880620000000002 2023-09-23T21:15:26.046 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:26.046 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:26.047 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T21:15:26.047 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:11:09.917741+0000", 2023-09-23T21:15:26.047 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.5473000000000002e-05 2023-09-23T21:15:26.047 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:26.047 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:26.047 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:15:26.047 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:11:09.919484+0000", 2023-09-23T21:15:26.047 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0017433349999999999 2023-09-23T21:15:26.047 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:26.047 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:26.047 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T21:15:26.047 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:11:09.919523+0000", 2023-09-23T21:15:26.048 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.8448999999999997e-05 2023-09-23T21:15:26.048 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:26.048 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:26.048 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T21:15:26.048 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:11:09.921210+0000", 2023-09-23T21:15:26.048 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0016874399999999999 2023-09-23T21:15:26.048 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:15:26.048 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:15:26.048 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:15:26.048 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:26.048 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:26.048 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.46328.0:52912 10.f 10:f8b90dae:::benchmark_data_smithi026_379838_object52911:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e443)", 2023-09-23T21:15:26.048 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T21:11:07.912529+0000", 2023-09-23T21:15:26.049 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 258.107424966, 2023-09-23T21:15:26.049 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.0110510220000002, 2023-09-23T21:15:26.049 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:15:26.049 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:15:26.049 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T21:15:26.049 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.46328", 2023-09-23T21:15:26.049 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/3748861181", 2023-09-23T21:15:26.049 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 52912 2023-09-23T21:15:26.049 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:26.049 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:15:26.049 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:26.049 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:15:26.050 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:11:07.912529+0000", 2023-09-23T21:15:26.050 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:15:26.050 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:26.050 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:26.050 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:15:26.050 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:11:07.912529+0000", 2023-09-23T21:15:26.050 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:15:26.050 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:26.050 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:26.050 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:15:26.050 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:11:07.912528+0000", 2023-09-23T21:15:26.051 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T21:15:26.051 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:26.051 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:26.051 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:15:26.051 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:11:07.912548+0000", 2023-09-23T21:15:26.051 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.0103999999999999e-05 2023-09-23T21:15:26.051 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:26.051 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:26.051 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:15:26.051 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:11:07.912564+0000", 2023-09-23T21:15:26.051 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.6240999999999999e-05 2023-09-23T21:15:26.051 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:26.052 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:26.052 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:15:26.052 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:11:07.912572+0000", 2023-09-23T21:15:26.052 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.8890000000000002e-06 2023-09-23T21:15:26.052 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:26.052 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:26.052 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:15:26.052 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:11:07.912628+0000", 2023-09-23T21:15:26.052 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.6709999999999997e-05 2023-09-23T21:15:26.052 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:26.052 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:26.052 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T21:15:26.052 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:11:07.912642+0000", 2023-09-23T21:15:26.053 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.3434e-05 2023-09-23T21:15:26.053 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:26.053 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:26.053 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:15:26.053 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:11:08.752996+0000", 2023-09-23T21:15:26.053 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.84035380400000004 2023-09-23T21:15:26.053 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:26.053 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:26.053 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T21:15:26.053 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:11:08.753009+0000", 2023-09-23T21:15:26.053 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.2962000000000001e-05 2023-09-23T21:15:26.054 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:26.054 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:26.054 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:15:26.054 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:11:09.913481+0000", 2023-09-23T21:15:26.054 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.160472811 2023-09-23T21:15:26.054 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:26.054 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:26.054 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:15:26.054 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:11:09.913640+0000", 2023-09-23T21:15:26.054 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00015848899999999999 2023-09-23T21:15:26.054 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:26.054 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:26.055 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 2,4", 2023-09-23T21:15:26.055 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:11:09.913837+0000", 2023-09-23T21:15:26.055 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00019724599999999999 2023-09-23T21:15:26.055 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:26.055 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:26.055 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T21:15:26.055 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:11:09.917751+0000", 2023-09-23T21:15:26.055 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0039136689999999998 2023-09-23T21:15:26.055 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:26.055 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:26.055 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:15:26.055 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:11:09.921273+0000", 2023-09-23T21:15:26.056 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0035223229999999999 2023-09-23T21:15:26.056 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:26.056 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:26.056 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:15:26.056 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:11:09.921382+0000", 2023-09-23T21:15:26.056 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000108409 2023-09-23T21:15:26.056 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:26.056 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:26.056 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T21:15:26.056 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:11:09.921436+0000", 2023-09-23T21:15:26.056 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.4129000000000003e-05 2023-09-23T21:15:26.056 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:26.057 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:26.057 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T21:15:26.057 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:11:09.923580+0000", 2023-09-23T21:15:26.057 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0021443460000000001 2023-09-23T21:15:26.057 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:15:26.057 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:15:26.057 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:15:26.057 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:26.057 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:26.057 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.46328.0:52921 10.f 10:f87e3b19:::benchmark_data_smithi026_379838_object52920:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e443)", 2023-09-23T21:15:26.057 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T21:11:07.919142+0000", 2023-09-23T21:15:26.057 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 258.10081199500002, 2023-09-23T21:15:26.058 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.019676655, 2023-09-23T21:15:26.058 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:15:26.058 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:15:26.058 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T21:15:26.058 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.46328", 2023-09-23T21:15:26.058 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/3748861181", 2023-09-23T21:15:26.058 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 52921 2023-09-23T21:15:26.058 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:26.058 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:15:26.058 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:26.058 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:15:26.058 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:11:07.919142+0000", 2023-09-23T21:15:26.059 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:15:26.059 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:26.059 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:26.059 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:15:26.059 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:11:07.919142+0000", 2023-09-23T21:15:26.059 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:15:26.059 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:26.059 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:26.059 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:15:26.059 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:11:07.919141+0000", 2023-09-23T21:15:26.059 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T21:15:26.059 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:26.060 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:26.060 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:15:26.060 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:11:07.919182+0000", 2023-09-23T21:15:26.060 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.1378999999999998e-05 2023-09-23T21:15:26.060 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:26.060 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:26.060 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:15:26.060 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:11:07.919197+0000", 2023-09-23T21:15:26.060 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.4661000000000001e-05 2023-09-23T21:15:26.060 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:26.060 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:26.060 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:15:26.061 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:11:07.919206+0000", 2023-09-23T21:15:26.061 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.9700000000000005e-06 2023-09-23T21:15:26.061 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:26.061 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:26.061 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:15:26.061 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:11:07.919267+0000", 2023-09-23T21:15:26.061 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.1587000000000005e-05 2023-09-23T21:15:26.061 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:26.061 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:26.061 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T21:15:26.061 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:11:07.919279+0000", 2023-09-23T21:15:26.061 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1235e-05 2023-09-23T21:15:26.062 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:26.062 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:26.062 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:15:26.062 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:11:08.753634+0000", 2023-09-23T21:15:26.062 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.83435543199999995 2023-09-23T21:15:26.062 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:26.062 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:26.062 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T21:15:26.062 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:11:08.753644+0000", 2023-09-23T21:15:26.062 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.6339999999999996e-06 2023-09-23T21:15:26.062 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:26.063 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:26.063 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:15:26.063 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:11:09.923642+0000", 2023-09-23T21:15:26.063 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1699981720000001 2023-09-23T21:15:26.063 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:26.063 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:26.063 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:15:26.063 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:11:09.923816+0000", 2023-09-23T21:15:26.063 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000173874 2023-09-23T21:15:26.063 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:26.063 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:26.063 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 2,4", 2023-09-23T21:15:26.064 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:11:09.923990+0000", 2023-09-23T21:15:26.064 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000174043 2023-09-23T21:15:26.064 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:26.064 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:26.064 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:15:26.064 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:11:09.930879+0000", 2023-09-23T21:15:26.064 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0068894730000000001 2023-09-23T21:15:26.064 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:26.064 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:26.064 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T21:15:26.064 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:11:09.933571+0000", 2023-09-23T21:15:26.064 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00269199 2023-09-23T21:15:26.065 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:26.065 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:26.065 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:15:26.065 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:11:09.936184+0000", 2023-09-23T21:15:26.065 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0026129320000000001 2023-09-23T21:15:26.065 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:26.065 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:26.065 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T21:15:26.065 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:11:09.936248+0000", 2023-09-23T21:15:26.065 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.4333999999999998e-05 2023-09-23T21:15:26.065 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:26.065 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:26.066 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T21:15:26.066 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:11:09.938819+0000", 2023-09-23T21:15:26.066 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.002570227 2023-09-23T21:15:26.066 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:15:26.066 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:15:26.066 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:15:26.066 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:26.066 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:26.066 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.46328.0:84527 10.4 10:22b37af2:::benchmark_data_smithi026_379838_object4570:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e471)", 2023-09-23T21:15:26.066 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T21:12:10.746775+0000", 2023-09-23T21:15:26.066 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 195.27317917400001, 2023-09-23T21:15:26.066 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.6027426719999998, 2023-09-23T21:15:26.067 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:15:26.067 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:15:26.067 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T21:15:26.067 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.46328", 2023-09-23T21:15:26.067 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/3748861181", 2023-09-23T21:15:26.067 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 84527 2023-09-23T21:15:26.067 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:26.067 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:15:26.067 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:26.067 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:15:26.067 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:12:10.746775+0000", 2023-09-23T21:15:26.067 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:15:26.068 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:26.068 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:26.068 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:15:26.068 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:12:10.746775+0000", 2023-09-23T21:15:26.068 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:15:26.068 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:26.068 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:26.068 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:15:26.068 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:12:10.746773+0000", 2023-09-23T21:15:26.068 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999976 2023-09-23T21:15:26.068 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:26.068 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:26.069 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:15:26.069 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:12:10.746781+0000", 2023-09-23T21:15:26.069 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.5960000000000004e-06 2023-09-23T21:15:26.069 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:26.069 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:26.069 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:15:26.069 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:12:10.746797+0000", 2023-09-23T21:15:26.069 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.5469e-05 2023-09-23T21:15:26.069 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:26.069 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:26.069 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:15:26.069 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:12:10.746814+0000", 2023-09-23T21:15:26.070 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.7703e-05 2023-09-23T21:15:26.070 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:26.070 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:26.070 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:15:26.070 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:12:10.747566+0000", 2023-09-23T21:15:26.070 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00075137800000000003 2023-09-23T21:15:26.070 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:26.070 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:26.070 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T21:15:26.070 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:12:10.747577+0000", 2023-09-23T21:15:26.070 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1302e-05 2023-09-23T21:15:26.070 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:26.071 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:26.071 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:15:26.071 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:12:14.322579+0000", 2023-09-23T21:15:26.071 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.575001528 2023-09-23T21:15:26.071 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:26.071 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:26.071 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:15:26.071 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:12:14.322698+0000", 2023-09-23T21:15:26.071 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000119119 2023-09-23T21:15:26.071 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:26.071 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:26.071 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 2,4", 2023-09-23T21:15:26.072 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:12:14.322866+0000", 2023-09-23T21:15:26.072 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00016831800000000001 2023-09-23T21:15:26.072 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:26.072 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:26.072 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T21:15:26.072 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:12:14.329391+0000", 2023-09-23T21:15:26.072 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.006524813 2023-09-23T21:15:26.072 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:26.072 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:26.072 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:15:26.072 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:12:14.335516+0000", 2023-09-23T21:15:26.072 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0061251830000000002 2023-09-23T21:15:26.073 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:26.073 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:26.073 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:15:26.073 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:12:14.348412+0000", 2023-09-23T21:15:26.073 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.012896151 2023-09-23T21:15:26.073 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:26.073 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:26.073 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T21:15:26.073 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:12:14.348499+0000", 2023-09-23T21:15:26.073 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.7356999999999997e-05 2023-09-23T21:15:26.073 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:26.073 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:26.074 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T21:15:26.074 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:12:14.349518+0000", 2023-09-23T21:15:26.074 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0010180339999999999 2023-09-23T21:15:26.074 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:15:26.074 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:15:26.074 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:15:26.074 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:26.074 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:26.074 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.46328.0:84529 10.4 10:21df4bc4:::benchmark_data_smithi026_379838_object4572:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e471)", 2023-09-23T21:15:26.074 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T21:12:10.747521+0000", 2023-09-23T21:15:26.074 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 195.27243287900001, 2023-09-23T21:15:26.074 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.6069034379999998, 2023-09-23T21:15:26.075 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:15:26.075 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:15:26.075 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T21:15:26.075 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.46328", 2023-09-23T21:15:26.075 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/3748861181", 2023-09-23T21:15:26.075 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 84529 2023-09-23T21:15:26.075 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:26.075 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:15:26.075 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:26.075 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:15:26.075 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:12:10.747521+0000", 2023-09-23T21:15:26.075 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:15:26.076 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:26.076 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:26.076 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:15:26.076 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:12:10.747521+0000", 2023-09-23T21:15:26.076 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:15:26.076 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:26.076 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:26.076 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:15:26.076 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:12:10.747520+0000", 2023-09-23T21:15:26.076 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.999999 2023-09-23T21:15:26.076 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:26.076 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:26.076 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:15:26.077 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:12:10.747531+0000", 2023-09-23T21:15:26.077 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0919e-05 2023-09-23T21:15:26.077 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:26.077 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:26.077 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:15:26.077 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:12:10.747544+0000", 2023-09-23T21:15:26.077 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.2306e-05 2023-09-23T21:15:26.077 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:26.077 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:26.077 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:15:26.077 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:12:10.747551+0000", 2023-09-23T21:15:26.077 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.7510000000000008e-06 2023-09-23T21:15:26.078 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:26.078 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:26.078 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:15:26.078 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:12:10.747652+0000", 2023-09-23T21:15:26.078 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00010068099999999999 2023-09-23T21:15:26.078 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:26.078 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:26.078 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T21:15:26.078 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:12:10.747658+0000", 2023-09-23T21:15:26.078 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.0190000000000003e-06 2023-09-23T21:15:26.078 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:26.078 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:26.079 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:15:26.079 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:12:14.324413+0000", 2023-09-23T21:15:26.079 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.5767554380000002 2023-09-23T21:15:26.079 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:26.079 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:26.079 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:15:26.079 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:12:14.324659+0000", 2023-09-23T21:15:26.079 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00024532200000000002 2023-09-23T21:15:26.079 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:26.079 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:26.079 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 2,4", 2023-09-23T21:15:26.079 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:12:14.324868+0000", 2023-09-23T21:15:26.079 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000209268 2023-09-23T21:15:26.080 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:26.080 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:26.080 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T21:15:26.080 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:12:14.329471+0000", 2023-09-23T21:15:26.080 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.004602994 2023-09-23T21:15:26.080 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:26.080 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:26.080 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:15:26.080 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:12:14.335551+0000", 2023-09-23T21:15:26.080 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0060798500000000004 2023-09-23T21:15:26.080 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:26.080 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:26.081 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:15:26.081 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:12:14.353065+0000", 2023-09-23T21:15:26.081 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.017514140000000001 2023-09-23T21:15:26.081 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:26.081 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:26.081 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T21:15:26.081 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:12:14.353136+0000", 2023-09-23T21:15:26.081 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.0988999999999994e-05 2023-09-23T21:15:26.081 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:26.081 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:26.081 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T21:15:26.081 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:12:14.354425+0000", 2023-09-23T21:15:26.082 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.001288635 2023-09-23T21:15:26.082 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:15:26.082 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:15:26.082 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:15:26.082 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:26.082 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:26.082 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.46328.0:84536 10.4 10:2156e66c:::benchmark_data_smithi026_379838_object4579:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e471)", 2023-09-23T21:15:26.082 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T21:12:10.747558+0000", 2023-09-23T21:15:26.082 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 195.27239562299999, 2023-09-23T21:15:26.082 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.616975386, 2023-09-23T21:15:26.082 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:15:26.082 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:15:26.082 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T21:15:26.083 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.46328", 2023-09-23T21:15:26.083 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/3748861181", 2023-09-23T21:15:26.083 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 84536 2023-09-23T21:15:26.083 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:26.083 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:15:26.083 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:26.083 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:15:26.083 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:12:10.747558+0000", 2023-09-23T21:15:26.083 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:15:26.083 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:26.083 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:26.083 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:15:26.084 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:12:10.747558+0000", 2023-09-23T21:15:26.084 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:15:26.084 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:26.084 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:26.084 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:15:26.084 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:12:10.747558+0000", 2023-09-23T21:15:26.084 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T21:15:26.084 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:26.084 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:26.084 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:15:26.084 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:12:10.747561+0000", 2023-09-23T21:15:26.084 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.7039999999999999e-06 2023-09-23T21:15:26.085 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:26.085 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:26.085 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:15:26.085 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:12:10.747567+0000", 2023-09-23T21:15:26.085 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.896e-06 2023-09-23T21:15:26.085 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:26.085 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:26.085 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:15:26.085 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:12:10.747572+0000", 2023-09-23T21:15:26.085 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.8840000000000002e-06 2023-09-23T21:15:26.085 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:26.085 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:26.086 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:15:26.086 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:12:10.747684+0000", 2023-09-23T21:15:26.086 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00011202899999999999 2023-09-23T21:15:26.086 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:26.086 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:26.086 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T21:15:26.086 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:12:10.747692+0000", 2023-09-23T21:15:26.086 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.3639999999999999e-06 2023-09-23T21:15:26.086 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:26.086 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:26.086 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:15:26.086 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:12:14.326153+0000", 2023-09-23T21:15:26.086 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.5784612500000001 2023-09-23T21:15:26.087 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:26.087 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:26.087 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:15:26.087 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:12:14.326337+0000", 2023-09-23T21:15:26.087 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000184098 2023-09-23T21:15:26.087 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:26.087 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:26.087 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 2,4", 2023-09-23T21:15:26.087 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:12:14.326530+0000", 2023-09-23T21:15:26.087 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00019268900000000001 2023-09-23T21:15:26.087 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:26.087 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:26.088 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T21:15:26.088 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:12:14.329485+0000", 2023-09-23T21:15:26.088 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0029546009999999998 2023-09-23T21:15:26.088 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:26.088 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:26.088 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:15:26.088 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:12:14.335606+0000", 2023-09-23T21:15:26.088 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0061211089999999996 2023-09-23T21:15:26.088 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:26.088 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:26.088 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:15:26.088 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:12:14.363402+0000", 2023-09-23T21:15:26.088 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.027796292 2023-09-23T21:15:26.089 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:26.089 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:26.089 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T21:15:26.089 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:12:14.363483+0000", 2023-09-23T21:15:26.089 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.0990999999999998e-05 2023-09-23T21:15:26.089 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:26.089 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:26.089 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T21:15:26.089 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:12:14.364534+0000", 2023-09-23T21:15:26.089 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.001050839 2023-09-23T21:15:26.089 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:15:26.090 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:15:26.090 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:15:26.090 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:26.090 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:26.090 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.46328.0:84594 10.4 10:208343da:::benchmark_data_smithi026_379838_object4637:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e472)", 2023-09-23T21:15:26.090 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T21:12:11.874722+0000", 2023-09-23T21:15:26.090 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 194.14523211100001, 2023-09-23T21:15:26.090 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.4972512490000001, 2023-09-23T21:15:26.090 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:15:26.090 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:15:26.090 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T21:15:26.090 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.46328", 2023-09-23T21:15:26.091 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/3748861181", 2023-09-23T21:15:26.091 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 84594 2023-09-23T21:15:26.091 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:26.091 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:15:26.091 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:26.091 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:15:26.091 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:12:11.874722+0000", 2023-09-23T21:15:26.091 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:15:26.091 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:26.091 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:26.091 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:15:26.091 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:12:11.874722+0000", 2023-09-23T21:15:26.091 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:15:26.092 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:26.092 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:26.092 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:15:26.092 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:12:11.874720+0000", 2023-09-23T21:15:26.092 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999981 2023-09-23T21:15:26.092 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:26.092 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:26.092 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:15:26.092 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:12:11.874729+0000", 2023-09-23T21:15:26.092 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.721e-06 2023-09-23T21:15:26.092 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:26.092 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:26.092 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:15:26.093 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:12:11.874747+0000", 2023-09-23T21:15:26.093 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.8465e-05 2023-09-23T21:15:26.093 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:26.093 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:26.093 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:15:26.093 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:12:11.874755+0000", 2023-09-23T21:15:26.093 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.9689999999999994e-06 2023-09-23T21:15:26.093 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:26.093 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:26.093 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:15:26.093 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:12:11.874808+0000", 2023-09-23T21:15:26.093 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.2398000000000002e-05 2023-09-23T21:15:26.094 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:26.094 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:26.094 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T21:15:26.094 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:12:11.874818+0000", 2023-09-23T21:15:26.094 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0376e-05 2023-09-23T21:15:26.094 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:26.094 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:26.094 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:15:26.094 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:12:14.327898+0000", 2023-09-23T21:15:26.094 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.45308057 2023-09-23T21:15:26.094 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:26.094 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:26.095 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:15:26.095 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:12:14.328090+0000", 2023-09-23T21:15:26.095 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00019147700000000001 2023-09-23T21:15:26.095 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:26.095 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:26.095 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 2,4", 2023-09-23T21:15:26.095 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:12:14.328276+0000", 2023-09-23T21:15:26.095 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00018571399999999999 2023-09-23T21:15:26.095 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:26.095 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:26.095 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T21:15:26.095 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:12:14.331270+0000", 2023-09-23T21:15:26.095 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0029942509999999999 2023-09-23T21:15:26.096 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:26.096 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:26.096 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:15:26.096 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:12:14.340311+0000", 2023-09-23T21:15:26.096 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0090410690000000005 2023-09-23T21:15:26.096 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:26.096 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:26.096 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:15:26.096 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:12:14.370513+0000", 2023-09-23T21:15:26.096 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.030202113999999999 2023-09-23T21:15:26.096 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:26.096 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:26.096 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T21:15:26.097 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:12:14.370574+0000", 2023-09-23T21:15:26.097 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.1033000000000003e-05 2023-09-23T21:15:26.097 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:26.097 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:26.097 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T21:15:26.097 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:12:14.371973+0000", 2023-09-23T21:15:26.097 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0013990230000000001 2023-09-23T21:15:26.097 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:15:26.097 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:15:26.097 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:15:26.097 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:26.097 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:26.098 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.46328.0:84621 10.4 10:20433a92:::benchmark_data_smithi026_379838_object4664:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e472)", 2023-09-23T21:15:26.098 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T21:12:11.963265+0000", 2023-09-23T21:15:26.098 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 194.05668867899999, 2023-09-23T21:15:26.098 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.4097880979999999, 2023-09-23T21:15:26.098 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:15:26.098 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:15:26.098 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T21:15:26.098 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.46328", 2023-09-23T21:15:26.098 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/3748861181", 2023-09-23T21:15:26.098 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 84621 2023-09-23T21:15:26.098 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:26.098 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:15:26.099 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:26.099 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:15:26.099 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:12:11.963265+0000", 2023-09-23T21:15:26.099 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:15:26.099 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:26.099 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:26.099 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:15:26.099 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:12:11.963265+0000", 2023-09-23T21:15:26.099 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:15:26.099 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:26.099 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:26.099 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:15:26.099 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:12:11.963264+0000", 2023-09-23T21:15:26.100 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.999999 2023-09-23T21:15:26.100 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:26.100 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:26.100 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:15:26.100 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:12:11.963269+0000", 2023-09-23T21:15:26.100 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.1930000000000001e-06 2023-09-23T21:15:26.100 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:26.100 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:26.100 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:15:26.100 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:12:11.963282+0000", 2023-09-23T21:15:26.100 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.2065999999999999e-05 2023-09-23T21:15:26.100 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:26.101 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:26.101 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:15:26.101 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:12:11.963288+0000", 2023-09-23T21:15:26.101 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.4520000000000002e-06 2023-09-23T21:15:26.101 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:26.101 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:26.101 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:15:26.101 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:12:11.963340+0000", 2023-09-23T21:15:26.101 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.2364999999999998e-05 2023-09-23T21:15:26.101 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:26.101 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:26.101 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T21:15:26.101 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:12:11.963347+0000", 2023-09-23T21:15:26.102 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.0199999999999997e-06 2023-09-23T21:15:26.102 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:26.102 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:26.102 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:15:26.102 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:12:14.329517+0000", 2023-09-23T21:15:26.102 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.3661698869999999 2023-09-23T21:15:26.102 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:26.102 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:26.102 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:15:26.102 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:12:14.329653+0000", 2023-09-23T21:15:26.102 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000136031 2023-09-23T21:15:26.102 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:26.103 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:26.103 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 2,4", 2023-09-23T21:15:26.103 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:12:14.329807+0000", 2023-09-23T21:15:26.103 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00015407499999999999 2023-09-23T21:15:26.103 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:26.103 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:26.103 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T21:15:26.103 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:12:14.334419+0000", 2023-09-23T21:15:26.103 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0046119159999999998 2023-09-23T21:15:26.103 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:26.103 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:26.103 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:15:26.103 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:12:14.342415+0000", 2023-09-23T21:15:26.104 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0079952889999999992 2023-09-23T21:15:26.104 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:26.104 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:26.104 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:15:26.104 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:12:14.372027+0000", 2023-09-23T21:15:26.104 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.029612402999999999 2023-09-23T21:15:26.104 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:26.104 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:26.104 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T21:15:26.104 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:12:14.372083+0000", 2023-09-23T21:15:26.104 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.5946000000000003e-05 2023-09-23T21:15:26.104 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:26.105 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:26.105 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T21:15:26.105 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:12:14.373053+0000", 2023-09-23T21:15:26.105 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00097052800000000002 2023-09-23T21:15:26.105 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:15:26.105 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:15:26.105 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:15:26.105 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:26.105 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:26.105 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.46328.0:84629 10.4 10:20e2ea72:::benchmark_data_smithi026_379838_object4672:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e472)", 2023-09-23T21:15:26.105 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T21:12:12.006722+0000", 2023-09-23T21:15:26.105 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 194.01323171999999, 2023-09-23T21:15:26.105 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.3675720459999998, 2023-09-23T21:15:26.106 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:15:26.106 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:15:26.106 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T21:15:26.106 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.46328", 2023-09-23T21:15:26.106 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/3748861181", 2023-09-23T21:15:26.106 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 84629 2023-09-23T21:15:26.106 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:26.106 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:15:26.106 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:26.106 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:15:26.106 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:12:12.006722+0000", 2023-09-23T21:15:26.106 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:15:26.107 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:26.107 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:26.107 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:15:26.107 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:12:12.006722+0000", 2023-09-23T21:15:26.107 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:15:26.107 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:26.107 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:26.107 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:15:26.107 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:12:12.006721+0000", 2023-09-23T21:15:26.107 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T21:15:26.107 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:26.107 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:26.108 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:15:26.108 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:12:12.006727+0000", 2023-09-23T21:15:26.108 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.3210000000000001e-06 2023-09-23T21:15:26.108 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:26.108 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:26.108 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:15:26.108 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:12:12.006743+0000", 2023-09-23T21:15:26.108 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.5254e-05 2023-09-23T21:15:26.108 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:26.108 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:26.108 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:15:26.108 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:12:12.006749+0000", 2023-09-23T21:15:26.108 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.3829999999999997e-06 2023-09-23T21:15:26.109 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:26.109 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:26.109 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:15:26.109 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:12:12.006800+0000", 2023-09-23T21:15:26.109 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.1360999999999998e-05 2023-09-23T21:15:26.109 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:26.109 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:26.109 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T21:15:26.109 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:12:12.006808+0000", 2023-09-23T21:15:26.109 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.9959999999999993e-06 2023-09-23T21:15:26.109 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:26.109 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:26.110 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:15:26.110 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:12:14.331324+0000", 2023-09-23T21:15:26.110 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.3245158749999999 2023-09-23T21:15:26.110 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:26.110 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:26.110 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:15:26.110 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:12:14.331511+0000", 2023-09-23T21:15:26.110 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000186861 2023-09-23T21:15:26.110 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:26.110 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:26.111 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 2,4", 2023-09-23T21:15:26.111 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:12:14.331694+0000", 2023-09-23T21:15:26.111 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000182826 2023-09-23T21:15:26.111 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:26.111 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:26.111 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T21:15:26.111 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:12:14.334424+0000", 2023-09-23T21:15:26.111 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0027297300000000001 2023-09-23T21:15:26.111 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:26.111 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:26.111 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:15:26.111 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:12:14.342510+0000", 2023-09-23T21:15:26.111 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0080862109999999994 2023-09-23T21:15:26.112 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:26.112 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:26.112 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:15:26.112 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:12:14.373596+0000", 2023-09-23T21:15:26.112 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.031086417000000002 2023-09-23T21:15:26.112 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:26.112 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:26.112 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T21:15:26.112 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:12:14.373662+0000", 2023-09-23T21:15:26.112 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.6118999999999995e-05 2023-09-23T21:15:26.112 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:26.112 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:26.113 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T21:15:26.113 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:12:14.374294+0000", 2023-09-23T21:15:26.113 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00063200400000000003 2023-09-23T21:15:26.113 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:15:26.113 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:15:26.113 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:15:26.113 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:26.113 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:26.113 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.46328.0:84648 10.4 10:2219cf7e:::benchmark_data_smithi026_379838_object4691:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e472)", 2023-09-23T21:15:26.113 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T21:12:12.178615+0000", 2023-09-23T21:15:26.113 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 193.84133877299999, 2023-09-23T21:15:26.113 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.196273095, 2023-09-23T21:15:26.113 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:15:26.114 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:15:26.114 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T21:15:26.114 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.46328", 2023-09-23T21:15:26.114 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/3748861181", 2023-09-23T21:15:26.114 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 84648 2023-09-23T21:15:26.114 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:26.114 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:15:26.114 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:26.114 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:15:26.114 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:12:12.178615+0000", 2023-09-23T21:15:26.114 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:15:26.114 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:26.115 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:26.115 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:15:26.115 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:12:12.178615+0000", 2023-09-23T21:15:26.115 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:15:26.115 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:26.115 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:26.115 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:15:26.115 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:12:12.178614+0000", 2023-09-23T21:15:26.115 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T21:15:26.115 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:26.115 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:26.115 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:15:26.116 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:12:12.178620+0000", 2023-09-23T21:15:26.116 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.4459999999999998e-06 2023-09-23T21:15:26.116 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:26.116 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:26.116 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:15:26.116 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:12:12.178634+0000", 2023-09-23T21:15:26.116 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.4124999999999999e-05 2023-09-23T21:15:26.116 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:26.116 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:26.116 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:15:26.116 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:12:12.178644+0000", 2023-09-23T21:15:26.116 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0020000000000001e-05 2023-09-23T21:15:26.117 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:26.117 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:26.117 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:15:26.117 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:12:12.178714+0000", 2023-09-23T21:15:26.117 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.9993999999999994e-05 2023-09-23T21:15:26.117 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:26.117 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:26.117 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T21:15:26.117 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:12:12.178726+0000", 2023-09-23T21:15:26.117 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1712000000000001e-05 2023-09-23T21:15:26.117 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:26.117 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:26.118 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:15:26.118 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:12:14.332951+0000", 2023-09-23T21:15:26.118 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.1542246010000001 2023-09-23T21:15:26.118 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:26.118 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:26.118 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:15:26.118 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:12:14.333138+0000", 2023-09-23T21:15:26.118 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00018694499999999999 2023-09-23T21:15:26.118 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:26.118 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:26.118 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 2,4", 2023-09-23T21:15:26.118 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:12:14.333323+0000", 2023-09-23T21:15:26.119 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00018518699999999999 2023-09-23T21:15:26.119 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:26.119 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:26.119 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T21:15:26.119 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:12:14.334556+0000", 2023-09-23T21:15:26.119 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0012332479999999999 2023-09-23T21:15:26.119 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:26.119 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:26.119 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:15:26.119 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:12:14.342542+0000", 2023-09-23T21:15:26.119 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0079859570000000001 2023-09-23T21:15:26.119 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:26.120 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:26.120 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:15:26.120 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:12:14.374342+0000", 2023-09-23T21:15:26.120 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.031800434000000002 2023-09-23T21:15:26.120 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:26.120 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:26.120 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T21:15:26.120 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:12:14.374382+0000", 2023-09-23T21:15:26.120 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.9786999999999997e-05 2023-09-23T21:15:26.120 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:26.120 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:26.120 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T21:15:26.121 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:12:14.374888+0000", 2023-09-23T21:15:26.121 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00050613000000000003 2023-09-23T21:15:26.121 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:15:26.121 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:15:26.121 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:15:26.121 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:26.121 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:26.121 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.46328.0:105641 10.14 10:2a689774:::benchmark_data_smithi026_379838_object25684:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e502)", 2023-09-23T21:15:26.121 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T21:13:30.753104+0000", 2023-09-23T21:15:26.121 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 115.266849847, 2023-09-23T21:15:26.121 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.170934886, 2023-09-23T21:15:26.121 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:15:26.122 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:15:26.122 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T21:15:26.122 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.46328", 2023-09-23T21:15:26.122 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/3748861181", 2023-09-23T21:15:26.122 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 105641 2023-09-23T21:15:26.122 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:26.122 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:15:26.122 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:26.122 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:15:26.122 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:30.753104+0000", 2023-09-23T21:15:26.122 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:15:26.122 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:26.123 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:26.123 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:15:26.123 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:30.753104+0000", 2023-09-23T21:15:26.123 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:15:26.123 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:26.123 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:26.123 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:15:26.123 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:30.753103+0000", 2023-09-23T21:15:26.123 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T21:15:26.123 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:26.123 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:26.123 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:15:26.124 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:30.753109+0000", 2023-09-23T21:15:26.124 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.8369999999999999e-06 2023-09-23T21:15:26.124 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:26.124 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:26.124 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:15:26.124 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:30.753123+0000", 2023-09-23T21:15:26.124 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.3400999999999999e-05 2023-09-23T21:15:26.124 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:26.124 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:26.124 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:15:26.124 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:30.753132+0000", 2023-09-23T21:15:26.124 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.1810000000000007e-06 2023-09-23T21:15:26.125 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:26.125 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:26.125 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:15:26.125 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:30.753304+0000", 2023-09-23T21:15:26.125 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000171559 2023-09-23T21:15:26.125 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:26.125 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:26.125 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T21:15:26.125 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:30.753312+0000", 2023-09-23T21:15:26.125 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.1280000000000008e-06 2023-09-23T21:15:26.125 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:26.125 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:26.126 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:15:26.126 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:31.913101+0000", 2023-09-23T21:15:26.126 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1597891279999999 2023-09-23T21:15:26.126 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:26.126 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:26.126 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:15:26.126 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:31.913255+0000", 2023-09-23T21:15:26.126 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00015409399999999999 2023-09-23T21:15:26.126 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:26.126 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:26.126 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 0,4", 2023-09-23T21:15:26.126 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:31.913688+0000", 2023-09-23T21:15:26.127 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00043271599999999998 2023-09-23T21:15:26.127 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:26.127 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:26.127 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T21:15:26.127 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:31.922493+0000", 2023-09-23T21:15:26.127 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0088049400000000007 2023-09-23T21:15:26.127 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:26.127 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:26.127 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:15:26.127 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:31.922622+0000", 2023-09-23T21:15:26.127 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00012902000000000001 2023-09-23T21:15:26.127 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:26.128 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:26.128 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:15:26.128 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:31.922708+0000", 2023-09-23T21:15:26.128 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.6774000000000006e-05 2023-09-23T21:15:26.128 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:26.128 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:26.128 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T21:15:26.128 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:31.922747+0000", 2023-09-23T21:15:26.128 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.8661999999999997e-05 2023-09-23T21:15:26.128 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:26.128 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:26.128 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T21:15:26.129 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:31.924039+0000", 2023-09-23T21:15:26.129 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.001292014 2023-09-23T21:15:26.129 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:15:26.129 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:15:26.129 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:15:26.129 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:26.129 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:26.129 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.46328.0:105645 10.14 10:280e6361:::benchmark_data_smithi026_379838_object25688:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e502)", 2023-09-23T21:15:26.129 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T21:13:30.753142+0000", 2023-09-23T21:15:26.129 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 115.266811636, 2023-09-23T21:15:26.129 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1739396420000001, 2023-09-23T21:15:26.129 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:15:26.130 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:15:26.130 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T21:15:26.130 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.46328", 2023-09-23T21:15:26.130 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/3748861181", 2023-09-23T21:15:26.130 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 105645 2023-09-23T21:15:26.130 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:26.130 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:15:26.130 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:26.130 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:15:26.130 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:30.753142+0000", 2023-09-23T21:15:26.130 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:15:26.130 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:26.131 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:26.131 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:15:26.131 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:30.753142+0000", 2023-09-23T21:15:26.131 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:15:26.131 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:26.131 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:26.131 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:15:26.131 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:30.753142+0000", 2023-09-23T21:15:26.131 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T21:15:26.131 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:26.131 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:26.131 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:15:26.132 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:30.753145+0000", 2023-09-23T21:15:26.132 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.6289999999999999e-06 2023-09-23T21:15:26.132 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:26.132 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:26.132 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:15:26.132 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:30.753153+0000", 2023-09-23T21:15:26.132 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.7060000000000004e-06 2023-09-23T21:15:26.132 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:26.132 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:26.132 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:15:26.132 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:30.753159+0000", 2023-09-23T21:15:26.132 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.2400000000000004e-06 2023-09-23T21:15:26.133 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:26.133 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:26.133 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:15:26.133 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:30.753326+0000", 2023-09-23T21:15:26.133 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00016653999999999999 2023-09-23T21:15:26.133 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:26.133 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:26.133 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T21:15:26.133 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:30.753331+0000", 2023-09-23T21:15:26.133 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.9359999999999998e-06 2023-09-23T21:15:26.133 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:26.133 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:26.134 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:15:26.134 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:31.917261+0000", 2023-09-23T21:15:26.134 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1639297260000001 2023-09-23T21:15:26.134 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:26.134 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:26.134 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:15:26.134 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:31.917419+0000", 2023-09-23T21:15:26.134 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00015840900000000001 2023-09-23T21:15:26.134 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:26.134 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:26.134 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 0,4", 2023-09-23T21:15:26.134 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:31.917841+0000", 2023-09-23T21:15:26.135 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00042226600000000003 2023-09-23T21:15:26.135 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:26.135 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:26.135 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T21:15:26.135 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:31.922503+0000", 2023-09-23T21:15:26.135 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0046611999999999999 2023-09-23T21:15:26.135 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:26.135 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:26.135 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:15:26.135 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:31.924185+0000", 2023-09-23T21:15:26.135 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.001682539 2023-09-23T21:15:26.135 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:26.136 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:26.136 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:15:26.136 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:31.925781+0000", 2023-09-23T21:15:26.136 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0015954739999999999 2023-09-23T21:15:26.136 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:26.136 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:26.136 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T21:15:26.136 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:31.925838+0000", 2023-09-23T21:15:26.136 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.7954000000000001e-05 2023-09-23T21:15:26.136 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:26.136 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:26.136 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T21:15:26.137 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:31.927082+0000", 2023-09-23T21:15:26.137 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.001243552 2023-09-23T21:15:26.137 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:15:26.137 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:15:26.137 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:15:26.137 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:26.137 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:26.137 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.46328.0:105646 10.14 10:29b02dab:::benchmark_data_smithi026_379838_object25689:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e502)", 2023-09-23T21:15:26.137 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T21:13:30.753165+0000", 2023-09-23T21:15:26.137 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 115.266789268, 2023-09-23T21:15:26.137 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1754176160000001, 2023-09-23T21:15:26.137 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:15:26.138 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:15:26.138 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T21:15:26.138 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.46328", 2023-09-23T21:15:26.138 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/3748861181", 2023-09-23T21:15:26.138 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 105646 2023-09-23T21:15:26.138 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:26.138 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:15:26.138 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:26.138 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:15:26.138 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:30.753165+0000", 2023-09-23T21:15:26.138 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:15:26.138 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:26.139 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:26.139 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:15:26.139 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:30.753165+0000", 2023-09-23T21:15:26.139 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:15:26.139 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:26.139 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:26.139 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:15:26.139 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:30.753164+0000", 2023-09-23T21:15:26.139 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T21:15:26.139 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:26.139 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:26.139 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:15:26.140 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:30.753167+0000", 2023-09-23T21:15:26.140 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.6769999999999999e-06 2023-09-23T21:15:26.140 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:26.140 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:26.140 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:15:26.140 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:30.753174+0000", 2023-09-23T21:15:26.140 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.5350000000000001e-06 2023-09-23T21:15:26.140 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:26.140 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:26.140 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:15:26.140 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:30.753179+0000", 2023-09-23T21:15:26.140 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.3499999999999996e-06 2023-09-23T21:15:26.141 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:26.141 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:26.141 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:15:26.141 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:30.753347+0000", 2023-09-23T21:15:26.141 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00016797000000000001 2023-09-23T21:15:26.141 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:26.141 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:26.141 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T21:15:26.141 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:30.753352+0000", 2023-09-23T21:15:26.141 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.3059999999999997e-06 2023-09-23T21:15:26.141 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:26.141 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:26.142 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:15:26.142 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:31.920180+0000", 2023-09-23T21:15:26.142 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1668273419999999 2023-09-23T21:15:26.142 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:26.142 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:26.142 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:15:26.142 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:31.920376+0000", 2023-09-23T21:15:26.142 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00019646600000000001 2023-09-23T21:15:26.142 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:26.142 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:26.142 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 0,4", 2023-09-23T21:15:26.142 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:31.920796+0000", 2023-09-23T21:15:26.143 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00042050099999999999 2023-09-23T21:15:26.143 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:26.143 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:26.143 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:15:26.143 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:31.925527+0000", 2023-09-23T21:15:26.143 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0047302710000000003 2023-09-23T21:15:26.143 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:26.143 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:26.143 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T21:15:26.143 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:31.925688+0000", 2023-09-23T21:15:26.143 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00016078199999999999 2023-09-23T21:15:26.143 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:26.143 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:26.144 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:15:26.144 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:31.927391+0000", 2023-09-23T21:15:26.144 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.001703212 2023-09-23T21:15:26.144 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:26.144 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:26.144 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T21:15:26.144 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:31.927433+0000", 2023-09-23T21:15:26.144 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.2213999999999999e-05 2023-09-23T21:15:26.144 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:15:26.144 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:15:26.144 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T21:15:26.144 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:31.928582+0000", 2023-09-23T21:15:26.145 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0011494210000000001 2023-09-23T21:15:26.145 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:15:26.145 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:15:26.145 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:15:26.145 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:15:26.145 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:15:26.145 INFO:teuthology.orchestra.run.smithi026.stdout:} 2023-09-23T21:15:26.146 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:15:26.146 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:15:26.147 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:15:25 smithi070 ceph-mon[168282]: failed to encode map e543 with expected crc 2023-09-23T21:15:26.147 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:15:25 smithi070 ceph-mon[168282]: failed to encode map e543 with expected crc 2023-09-23T21:15:26.147 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:15:25 smithi070 ceph-mon[168282]: failed to encode map e543 with expected crc 2023-09-23T21:15:26.147 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:15:25 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/4112792893' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_5", "pg_num": 16}]: dispatch 2023-09-23T21:15:26.147 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:15:25 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_5", "pg_num": 16}]: dispatch 2023-09-23T21:15:26.147 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:15:25 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/3188106764' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-23T21:15:26.147 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:15:25 smithi070 ceph-mon[168282]: failed to encode map e544 with expected crc 2023-09-23T21:15:26.147 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:15:25 smithi070 ceph-mon[168282]: failed to encode map e544 with expected crc 2023-09-23T21:15:26.147 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:15:25 smithi070 ceph-mon[168282]: failed to encode map e544 with expected crc 2023-09-23T21:15:26.148 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:15:25 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_5", "pg_num": 16}]': finished 2023-09-23T21:15:26.148 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:15:25 smithi070 ceph-mon[168282]: osdmap e544: 8 total, 8 up, 6 in 2023-09-23T21:15:26.148 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:15:25 smithi070 ceph-mon[168282]: pgmap v1372: 119 pgs: 3 active+undersized+remapped, 1 remapped+peering, 18 unknown, 97 active+clean; 674 KiB data, 4.8 GiB used, 532 GiB / 536 GiB avail; 54 KiB/s, 0 objects/s recovering 2023-09-23T21:15:26.148 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:15:26.148 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:15:26.249 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:15:26.279 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:15:26.314 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:15:26.317 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:15:26.317 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:15:26.418 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:15:26.447 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:15:26.743 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:15:26.746 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:15:26.747 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:15:26.796 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_ops_in_flight 2023-09-23T21:15:26.848 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:15:26.855 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:15:26.882 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:15:27.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:15:26 smithi070 ceph-mon[168282]: failed to encode map e545 with expected crc 2023-09-23T21:15:27.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:15:26 smithi070 ceph-mon[168282]: failed to encode map e545 with expected crc 2023-09-23T21:15:27.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:15:26 smithi070 ceph-mon[168282]: osdmap e545: 8 total, 8 up, 6 in 2023-09-23T21:15:27.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:15:26 smithi070 ceph-mon[168282]: failed to encode map e545 with expected crc 2023-09-23T21:15:27.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:15:26 smithi070 ceph-mon[168282]: failed to encode map e545 with expected crc 2023-09-23T21:15:27.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:15:26 smithi070 ceph-mon[168282]: failed to encode map e545 with expected crc 2023-09-23T21:15:27.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:15:26 smithi070 ceph-mon[168282]: failed to encode map e545 with expected crc 2023-09-23T21:15:27.131 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:15:26 smithi026 ceph-mon[175151]: failed to encode map e545 with expected crc 2023-09-23T21:15:27.131 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:15:26 smithi026 ceph-mon[175151]: failed to encode map e545 with expected crc 2023-09-23T21:15:27.131 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:15:26 smithi026 ceph-mon[175151]: osdmap e545: 8 total, 8 up, 6 in 2023-09-23T21:15:27.131 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:15:26 smithi026 ceph-mon[175151]: failed to encode map e545 with expected crc 2023-09-23T21:15:27.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:15:26 smithi026 ceph-mon[175151]: failed to encode map e545 with expected crc 2023-09-23T21:15:27.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:15:26 smithi026 ceph-mon[175151]: failed to encode map e545 with expected crc 2023-09-23T21:15:27.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:15:26 smithi026 ceph-mon[175151]: failed to encode map e545 with expected crc 2023-09-23T21:15:27.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:15:26 smithi026 ceph-mon[179688]: failed to encode map e545 with expected crc 2023-09-23T21:15:27.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:15:26 smithi026 ceph-mon[179688]: failed to encode map e545 with expected crc 2023-09-23T21:15:27.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:15:26 smithi026 ceph-mon[179688]: osdmap e545: 8 total, 8 up, 6 in 2023-09-23T21:15:27.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:15:26 smithi026 ceph-mon[179688]: failed to encode map e545 with expected crc 2023-09-23T21:15:27.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:15:26 smithi026 ceph-mon[179688]: failed to encode map e545 with expected crc 2023-09-23T21:15:27.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:15:26 smithi026 ceph-mon[179688]: failed to encode map e545 with expected crc 2023-09-23T21:15:27.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:15:26 smithi026 ceph-mon[179688]: failed to encode map e545 with expected crc 2023-09-23T21:15:27.140 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:15:27.161 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T21:15:27.166 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:15:27.167 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:15:27.173 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0: {} 2023-09-23T21:15:27.195 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1: {} 2023-09-23T21:15:27.216 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2: {} 2023-09-23T21:15:27.236 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3: {} 2023-09-23T21:15:27.255 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4: {} 2023-09-23T21:15:27.267 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:15:27.274 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T21:15:27.298 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6: {} 2023-09-23T21:15:27.300 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:15:27.318 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T21:15:27.320 INFO:teuthology.orchestra.run.smithi026.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T21:15:27.320 INFO:teuthology.orchestra.run.smithi026.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T21:15:27.320 INFO:teuthology.orchestra.run.smithi026.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T21:15:27.320 INFO:teuthology.orchestra.run.smithi026.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T21:15:27.320 INFO:teuthology.orchestra.run.smithi026.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T21:15:27.320 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T21:15:27.320 INFO:teuthology.orchestra.run.smithi026.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T21:15:27.320 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T21:15:27.406 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T21:15:27.438 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:15:27.442 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:15:27.442 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:15:27.543 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:15:27.569 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:15:27.604 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:15:27.606 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:15:27.607 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:15:27.708 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:15:27.736 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:15:27.907 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:15:27.924 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:15:27.928 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:15:27.929 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:15:28.029 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:15:28.054 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:15:27 smithi026 ceph-mon[175151]: failed to encode map e546 with expected crc 2023-09-23T21:15:28.054 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:15:27 smithi026 ceph-mon[175151]: failed to encode map e546 with expected crc 2023-09-23T21:15:28.055 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:15:27 smithi026 ceph-mon[175151]: failed to encode map e546 with expected crc 2023-09-23T21:15:28.055 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:15:27 smithi026 ceph-mon[175151]: osdmap e546: 8 total, 8 up, 6 in 2023-09-23T21:15:28.055 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:15:27 smithi026 ceph-mon[175151]: pgmap v1375: 119 pgs: 2 active+undersized+remapped, 1 remapped+peering, 2 creating+peering, 7 unknown, 107 active+clean; 674 KiB data, 4.6 GiB used, 532 GiB / 536 GiB avail 2023-09-23T21:15:28.055 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:15:27 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/1909031410' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:15:28.055 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:15:27 smithi026 ceph-mon[179688]: failed to encode map e546 with expected crc 2023-09-23T21:15:28.055 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:15:27 smithi026 ceph-mon[179688]: failed to encode map e546 with expected crc 2023-09-23T21:15:28.055 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:15:27 smithi026 ceph-mon[179688]: failed to encode map e546 with expected crc 2023-09-23T21:15:28.055 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:15:27 smithi026 ceph-mon[179688]: osdmap e546: 8 total, 8 up, 6 in 2023-09-23T21:15:28.055 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:15:27 smithi026 ceph-mon[179688]: pgmap v1375: 119 pgs: 2 active+undersized+remapped, 1 remapped+peering, 2 creating+peering, 7 unknown, 107 active+clean; 674 KiB data, 4.6 GiB used, 532 GiB / 536 GiB avail 2023-09-23T21:15:28.056 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:15:27 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/1909031410' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:15:28.058 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:15:28.189 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:15:27 smithi070 ceph-mon[168282]: failed to encode map e546 with expected crc 2023-09-23T21:15:28.190 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:15:27 smithi070 ceph-mon[168282]: failed to encode map e546 with expected crc 2023-09-23T21:15:28.190 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:15:27 smithi070 ceph-mon[168282]: failed to encode map e546 with expected crc 2023-09-23T21:15:28.190 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:15:27 smithi070 ceph-mon[168282]: osdmap e546: 8 total, 8 up, 6 in 2023-09-23T21:15:28.190 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:15:27 smithi070 ceph-mon[168282]: pgmap v1375: 119 pgs: 2 active+undersized+remapped, 1 remapped+peering, 2 creating+peering, 7 unknown, 107 active+clean; 674 KiB data, 4.6 GiB used, 532 GiB / 536 GiB avail 2023-09-23T21:15:28.190 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:15:27 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/1909031410' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:15:28.254 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T21:15:28.257 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:15:28.257 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:15:28.358 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:15:28.388 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:15:28.421 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:15:28.424 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:15:28.424 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:15:28.525 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:15:28.553 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:15:28.586 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:15:28.589 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:15:28.589 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:15:28.690 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:15:28.716 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:15:28.754 INFO:teuthology.orchestra.run.smithi070.stdout:ERROR: (22) Invalid argument 2023-09-23T21:15:28.754 INFO:teuthology.orchestra.run.smithi070.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T21:15:28.754 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:15:28.757 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:15:28.757 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:15:28.858 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:15:28.890 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:15:29.027 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T21:15:29.029 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:15:29.030 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:15:29.130 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:15:29.148 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_blocked_ops 2023-09-23T21:15:29.161 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:15:29.359 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T21:15:29.362 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:15:29 smithi026 ceph-mon[179688]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2023-09-23T21:15:29.362 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:15:29 smithi026 ceph-mon[175151]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2023-09-23T21:15:29.363 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:15:29.363 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:15:29.464 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:15:29.496 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:15:29.605 INFO:teuthology.orchestra.run.smithi026.stdout: 2023-09-23T21:15:29.607 INFO:teuthology.orchestra.run.smithi026.stderr:dumped all 2023-09-23T21:15:29.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:15:29 smithi070 ceph-mon[168282]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2023-09-23T21:15:29.631 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 21:15:29 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[164029]: ::ffff:172.21.15.70 - - [23/Sep/2023:21:15:29] "GET /metrics HTTP/1.1" 200 36422 "" "Prometheus/2.43.0" 2023-09-23T21:15:29.633 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T21:15:29.636 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:15:29.637 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:15:29.737 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:15:29.740 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T21:15:29.766 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:15:29.848 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T21:15:29.865 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:15:29.867 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:15:29.868 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:15:29.969 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:15:29.993 INFO:teuthology.orchestra.run.smithi026.stdout:{"pg_ready":true,"pg_map":{"version":1376,"stamp":"2023-09-23T21:15:29.098399+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":689794,"num_objects":25,"num_object_clones":0,"num_object_copies":75,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":25,"num_whiteouts":0,"num_read":78,"num_read_kb":144,"num_write":99,"num_write_kb":414,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":64,"num_bytes_recovered":1208712,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":5046,"ondisk_log_size":5046,"up":345,"acting":345,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":352,"num_osds":6,"num_per_pool_osds":6,"num_per_pool_omap_osds":6,"kb":562470912,"kb_used":4805276,"kb_used_data":1425004,"kb_used_omap":125,"kb_used_meta":3380098,"kb_avail":557665636,"statfs":{"total":575970213888,"available":571049611264,"internally_reserved":0,"allocated":1459204096,"data_stored":1446898578,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":128662,"internal_metadata":3461220714},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":5,"apply_latency_ms":5,"commit_latency_ns":5000000,"apply_latency_ns":5000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"5.310879"},"pg_stats":[{"pgid":"1.18","version":"79'161","reported_seq":512,"reported_epoch":545,"state":"active+clean","last_fresh":"2023-09-23T21:15:25.890384+0000","last_change":"2023-09-23T21:15:23.801613+0000","last_active":"2023-09-23T21:15:25.890384+0000","last_peered":"2023-09-23T21:15:25.890384+0000","last_clean":"2023-09-23T21:15:25.890384+0000","last_became_active":"2023-09-23T21:15:23.801118+0000","last_became_peered":"2023-09-23T21:15:23.801118+0000","last_unstale":"2023-09-23T21:15:25.890384+0000","last_undegraded":"2023-09-23T21:15:25.890384+0000","last_fullsized":"2023-09-23T21:15:25.890384+0000","mapping_epoch":542,"log_start":"0'0","ondisk_log_start":"0'0","created":479,"last_epoch_clean":543,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:06:52.098882+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:06:52.098882+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0082644970000000009,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":131344,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":262688,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1a","version":"68'1","reported_seq":136,"reported_epoch":545,"state":"active+clean","last_fresh":"2023-09-23T21:15:25.884792+0000","last_change":"2023-09-23T21:13:13.291403+0000","last_active":"2023-09-23T21:15:25.884792+0000","last_peered":"2023-09-23T21:15:25.884792+0000","last_clean":"2023-09-23T21:15:25.884792+0000","last_became_active":"2023-09-23T21:13:13.291123+0000","last_became_peered":"2023-09-23T21:13:13.291123+0000","last_unstale":"2023-09-23T21:15:25.884792+0000","last_undegraded":"2023-09-23T21:15:25.884792+0000","last_fullsized":"2023-09-23T21:15:25.884792+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":495,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:45:15.427430+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,0],"acting":[4,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":205,"reported_epoch":545,"state":"active+clean","last_fresh":"2023-09-23T21:15:25.884854+0000","last_change":"2023-09-23T21:12:10.767963+0000","last_active":"2023-09-23T21:15:25.884854+0000","last_peered":"2023-09-23T21:15:25.884854+0000","last_clean":"2023-09-23T21:15:25.884854+0000","last_became_active":"2023-09-23T21:12:10.767126+0000","last_became_peered":"2023-09-23T21:12:10.767126+0000","last_unstale":"2023-09-23T21:15:25.884854+0000","last_undegraded":"2023-09-23T21:15:25.884854+0000","last_fullsized":"2023-09-23T21:15:25.884854+0000","mapping_epoch":470,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:45:47.138277+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,5],"acting":[4,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1b","version":"79'161","reported_seq":723,"reported_epoch":545,"state":"active+clean","last_fresh":"2023-09-23T21:15:25.891805+0000","last_change":"2023-09-23T21:15:23.802986+0000","last_active":"2023-09-23T21:15:25.891805+0000","last_peered":"2023-09-23T21:15:25.891805+0000","last_clean":"2023-09-23T21:15:25.891805+0000","last_became_active":"2023-09-23T21:15:23.802580+0000","last_became_peered":"2023-09-23T21:15:23.802580+0000","last_unstale":"2023-09-23T21:15:25.891805+0000","last_undegraded":"2023-09-23T21:15:25.891805+0000","last_fullsized":"2023-09-23T21:15:25.891805+0000","mapping_epoch":542,"log_start":"0'0","ondisk_log_start":"0'0","created":479,"last_epoch_clean":543,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:05:55.772264+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:05:55.772264+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0043894679999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":539,"reported_epoch":542,"state":"active+clean","last_fresh":"2023-09-23T21:15:22.800026+0000","last_change":"2023-09-23T21:12:10.824116+0000","last_active":"2023-09-23T21:15:22.800026+0000","last_peered":"2023-09-23T21:15:22.800026+0000","last_clean":"2023-09-23T21:15:22.800026+0000","last_became_active":"2023-09-23T21:12:10.823849+0000","last_became_peered":"2023-09-23T21:12:10.823849+0000","last_unstale":"2023-09-23T21:15:22.800026+0000","last_undegraded":"2023-09-23T21:15:22.800026+0000","last_fullsized":"2023-09-23T21:15:22.800026+0000","mapping_epoch":470,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:54:13.777603+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":541,"reported_epoch":543,"state":"active+clean","last_fresh":"2023-09-23T21:15:23.792136+0000","last_change":"2023-09-23T21:13:44.413713+0000","last_active":"2023-09-23T21:15:23.792136+0000","last_peered":"2023-09-23T21:15:23.792136+0000","last_clean":"2023-09-23T21:15:23.792136+0000","last_became_active":"2023-09-23T21:13:44.413366+0000","last_became_peered":"2023-09-23T21:13:44.413366+0000","last_unstale":"2023-09-23T21:15:23.792136+0000","last_undegraded":"2023-09-23T21:15:23.792136+0000","last_fullsized":"2023-09-23T21:15:23.792136+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:31:56.349530+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1a","version":"79'161","reported_seq":593,"reported_epoch":545,"state":"active+clean","last_fresh":"2023-09-23T21:15:25.884149+0000","last_change":"2023-09-23T21:15:23.800891+0000","last_active":"2023-09-23T21:15:25.884149+0000","last_peered":"2023-09-23T21:15:25.884149+0000","last_clean":"2023-09-23T21:15:25.884149+0000","last_became_active":"2023-09-23T21:15:23.800519+0000","last_became_peered":"2023-09-23T21:15:23.800519+0000","last_unstale":"2023-09-23T21:15:25.884149+0000","last_undegraded":"2023-09-23T21:15:25.884149+0000","last_fullsized":"2023-09-23T21:15:25.884149+0000","mapping_epoch":542,"log_start":"0'0","ondisk_log_start":"0'0","created":479,"last_epoch_clean":543,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:06:51.075448+0000","last_deep_scrub":"79'161","last_deep_scrub_stamp":"2023-09-23T21:05:32.630292+0000","last_clean_scrub_stamp":"2023-09-23T21:06:51.075448+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0054793890000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,5],"acting":[4,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":53,"reported_epoch":543,"state":"active+clean","last_fresh":"2023-09-23T21:15:23.791045+0000","last_change":"2023-09-23T21:13:30.806084+0000","last_active":"2023-09-23T21:15:23.791045+0000","last_peered":"2023-09-23T21:15:23.791045+0000","last_clean":"2023-09-23T21:15:23.791045+0000","last_became_active":"2023-09-23T21:13:30.805743+0000","last_became_peered":"2023-09-23T21:13:30.805743+0000","last_unstale":"2023-09-23T21:15:23.791045+0000","last_undegraded":"2023-09-23T21:15:23.791045+0000","last_fullsized":"2023-09-23T21:15:23.791045+0000","mapping_epoch":501,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:48:10.849281+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":418,"reported_epoch":543,"state":"active+clean","last_fresh":"2023-09-23T21:15:23.792044+0000","last_change":"2023-09-23T21:13:44.410921+0000","last_active":"2023-09-23T21:15:23.792044+0000","last_peered":"2023-09-23T21:15:23.792044+0000","last_clean":"2023-09-23T21:15:23.792044+0000","last_became_active":"2023-09-23T21:13:44.410633+0000","last_became_peered":"2023-09-23T21:13:44.410633+0000","last_unstale":"2023-09-23T21:15:23.792044+0000","last_undegraded":"2023-09-23T21:15:23.792044+0000","last_fullsized":"2023-09-23T21:15:23.792044+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:57:10.116786+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.15","version":"79'161","reported_seq":512,"reported_epoch":545,"state":"active+clean","last_fresh":"2023-09-23T21:15:25.882761+0000","last_change":"2023-09-23T21:15:23.799841+0000","last_active":"2023-09-23T21:15:25.882761+0000","last_peered":"2023-09-23T21:15:25.882761+0000","last_clean":"2023-09-23T21:15:25.882761+0000","last_became_active":"2023-09-23T21:15:23.799409+0000","last_became_peered":"2023-09-23T21:15:23.799409+0000","last_unstale":"2023-09-23T21:15:25.882761+0000","last_undegraded":"2023-09-23T21:15:25.882761+0000","last_fullsized":"2023-09-23T21:15:25.882761+0000","mapping_epoch":542,"log_start":"0'0","ondisk_log_start":"0'0","created":479,"last_epoch_clean":543,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:09:34.892947+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:09:34.892947+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0081306369999999996,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,3],"acting":[7,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":518,"reported_epoch":543,"state":"active+clean","last_fresh":"2023-09-23T21:15:23.792186+0000","last_change":"2023-09-23T21:13:12.235728+0000","last_active":"2023-09-23T21:15:23.792186+0000","last_peered":"2023-09-23T21:15:23.792186+0000","last_clean":"2023-09-23T21:15:23.792186+0000","last_became_active":"2023-09-23T21:13:12.216549+0000","last_became_peered":"2023-09-23T21:13:12.216549+0000","last_unstale":"2023-09-23T21:15:23.792186+0000","last_undegraded":"2023-09-23T21:15:23.792186+0000","last_fullsized":"2023-09-23T21:15:23.792186+0000","mapping_epoch":493,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:39:33.324239+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":581,"reported_epoch":542,"state":"active+clean","last_fresh":"2023-09-23T21:15:22.799142+0000","last_change":"2023-09-23T21:12:10.790349+0000","last_active":"2023-09-23T21:15:22.799142+0000","last_peered":"2023-09-23T21:15:22.799142+0000","last_clean":"2023-09-23T21:15:22.799142+0000","last_became_active":"2023-09-23T21:12:10.789977+0000","last_became_peered":"2023-09-23T21:12:10.789977+0000","last_unstale":"2023-09-23T21:15:22.799142+0000","last_undegraded":"2023-09-23T21:15:22.799142+0000","last_fullsized":"2023-09-23T21:15:22.799142+0000","mapping_epoch":470,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:49:40.191147+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,5],"acting":[3,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"79'161","reported_seq":906,"reported_epoch":544,"state":"active+clean","last_fresh":"2023-09-23T21:15:24.872895+0000","last_change":"2023-09-23T21:11:32.760292+0000","last_active":"2023-09-23T21:15:24.872895+0000","last_peered":"2023-09-23T21:15:24.872895+0000","last_clean":"2023-09-23T21:15:24.872895+0000","last_became_active":"2023-09-23T21:11:32.759909+0000","last_became_peered":"2023-09-23T21:11:32.759909+0000","last_unstale":"2023-09-23T21:15:24.872895+0000","last_undegraded":"2023-09-23T21:15:24.872895+0000","last_fullsized":"2023-09-23T21:15:24.872895+0000","mapping_epoch":457,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":458,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:25.096401+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:25.096401+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:13:09.225685+0000","scrub_duration":0.002859393,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":270,"reported_epoch":545,"state":"active+clean","last_fresh":"2023-09-23T21:15:25.892813+0000","last_change":"2023-09-23T21:12:10.779468+0000","last_active":"2023-09-23T21:15:25.892813+0000","last_peered":"2023-09-23T21:15:25.892813+0000","last_clean":"2023-09-23T21:15:25.892813+0000","last_became_active":"2023-09-23T21:12:10.779282+0000","last_became_peered":"2023-09-23T21:12:10.779282+0000","last_unstale":"2023-09-23T21:15:25.892813+0000","last_undegraded":"2023-09-23T21:15:25.892813+0000","last_fullsized":"2023-09-23T21:15:25.892813+0000","mapping_epoch":470,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:36:35.417986+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":518,"reported_epoch":542,"state":"active+clean","last_fresh":"2023-09-23T21:15:22.798863+0000","last_change":"2023-09-23T21:10:26.758958+0000","last_active":"2023-09-23T21:15:22.798863+0000","last_peered":"2023-09-23T21:15:22.798863+0000","last_clean":"2023-09-23T21:15:22.798863+0000","last_became_active":"2023-09-23T21:10:26.758539+0000","last_became_peered":"2023-09-23T21:10:26.758539+0000","last_unstale":"2023-09-23T21:15:22.798863+0000","last_undegraded":"2023-09-23T21:15:22.798863+0000","last_fullsized":"2023-09-23T21:15:22.798863+0000","mapping_epoch":423,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":424,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:44:42.319189+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.17","version":"79'161","reported_seq":644,"reported_epoch":543,"state":"active+clean","last_fresh":"2023-09-23T21:15:23.799832+0000","last_change":"2023-09-23T21:15:23.799556+0000","last_active":"2023-09-23T21:15:23.799832+0000","last_peered":"2023-09-23T21:15:23.799832+0000","last_clean":"2023-09-23T21:15:23.799832+0000","last_became_active":"2023-09-23T21:15:23.799272+0000","last_became_peered":"2023-09-23T21:15:23.799272+0000","last_unstale":"2023-09-23T21:15:23.799832+0000","last_undegraded":"2023-09-23T21:15:23.799832+0000","last_fullsized":"2023-09-23T21:15:23.799832+0000","mapping_epoch":542,"log_start":"0'0","ondisk_log_start":"0'0","created":479,"last_epoch_clean":543,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:25.916867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:25.916867+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.001665323,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":427,"reported_epoch":542,"state":"active+clean","last_fresh":"2023-09-23T21:15:22.801847+0000","last_change":"2023-09-23T21:11:08.766045+0000","last_active":"2023-09-23T21:15:22.801847+0000","last_peered":"2023-09-23T21:15:22.801847+0000","last_clean":"2023-09-23T21:15:22.801847+0000","last_became_active":"2023-09-23T21:11:08.758504+0000","last_became_peered":"2023-09-23T21:11:08.758504+0000","last_unstale":"2023-09-23T21:15:22.801847+0000","last_undegraded":"2023-09-23T21:15:22.801847+0000","last_fullsized":"2023-09-23T21:15:22.801847+0000","mapping_epoch":443,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":444,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:02:19.482574+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":360,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,4],"acting":[3,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":232,"reported_epoch":544,"state":"active+clean","last_fresh":"2023-09-23T21:15:24.872787+0000","last_change":"2023-09-23T21:13:30.821489+0000","last_active":"2023-09-23T21:15:24.872787+0000","last_peered":"2023-09-23T21:15:24.872787+0000","last_clean":"2023-09-23T21:15:24.872787+0000","last_became_active":"2023-09-23T21:13:30.821125+0000","last_became_peered":"2023-09-23T21:13:30.821125+0000","last_unstale":"2023-09-23T21:15:24.872787+0000","last_undegraded":"2023-09-23T21:15:24.872787+0000","last_fullsized":"2023-09-23T21:15:24.872787+0000","mapping_epoch":501,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:47:18.553417+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,5],"acting":[2,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.16","version":"79'161","reported_seq":462,"reported_epoch":543,"state":"active+clean","last_fresh":"2023-09-23T21:15:23.792081+0000","last_change":"2023-09-23T21:13:33.770962+0000","last_active":"2023-09-23T21:15:23.792081+0000","last_peered":"2023-09-23T21:15:23.792081+0000","last_clean":"2023-09-23T21:15:23.792081+0000","last_became_active":"2023-09-23T21:13:33.770407+0000","last_became_peered":"2023-09-23T21:13:33.770407+0000","last_unstale":"2023-09-23T21:15:23.792081+0000","last_undegraded":"2023-09-23T21:15:23.792081+0000","last_fullsized":"2023-09-23T21:15:23.792081+0000","mapping_epoch":504,"log_start":"0'0","ondisk_log_start":"0'0","created":479,"last_epoch_clean":505,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:10:24.268662+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:10:24.268662+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13418,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":11,"num_read_kb":21,"num_write":6,"num_write_kb":55,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":40252,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.14","version":"79'22","reported_seq":109,"reported_epoch":545,"state":"active+clean","last_fresh":"2023-09-23T21:15:25.885015+0000","last_change":"2023-09-23T21:11:27.770162+0000","last_active":"2023-09-23T21:15:25.885015+0000","last_peered":"2023-09-23T21:15:25.885015+0000","last_clean":"2023-09-23T21:15:25.885015+0000","last_became_active":"2023-09-23T21:11:27.752275+0000","last_became_peered":"2023-09-23T21:11:27.752275+0000","last_unstale":"2023-09-23T21:15:25.885015+0000","last_undegraded":"2023-09-23T21:15:25.885015+0000","last_fullsized":"2023-09-23T21:15:25.885015+0000","mapping_epoch":452,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":453,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:05:11.769428+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4734,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,2],"acting":[4,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":474,"reported_epoch":545,"state":"active+clean","last_fresh":"2023-09-23T21:15:25.884491+0000","last_change":"2023-09-23T21:12:10.839271+0000","last_active":"2023-09-23T21:15:25.884491+0000","last_peered":"2023-09-23T21:15:25.884491+0000","last_clean":"2023-09-23T21:15:25.884491+0000","last_became_active":"2023-09-23T21:12:10.839042+0000","last_became_peered":"2023-09-23T21:12:10.839042+0000","last_unstale":"2023-09-23T21:15:25.884491+0000","last_undegraded":"2023-09-23T21:15:25.884491+0000","last_fullsized":"2023-09-23T21:15:25.884491+0000","mapping_epoch":470,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:10:26.414737+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,2],"acting":[4,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.11","version":"79'161","reported_seq":467,"reported_epoch":543,"state":"active+clean","last_fresh":"2023-09-23T21:15:23.791304+0000","last_change":"2023-09-23T21:13:33.767406+0000","last_active":"2023-09-23T21:15:23.791304+0000","last_peered":"2023-09-23T21:15:23.791304+0000","last_clean":"2023-09-23T21:15:23.791304+0000","last_became_active":"2023-09-23T21:13:33.767067+0000","last_became_peered":"2023-09-23T21:13:33.767067+0000","last_unstale":"2023-09-23T21:15:23.791304+0000","last_undegraded":"2023-09-23T21:15:23.791304+0000","last_fullsized":"2023-09-23T21:15:23.791304+0000","mapping_epoch":504,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":505,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:28.756634+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:28.756634+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":23,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":154,"reported_epoch":545,"state":"active+clean","last_fresh":"2023-09-23T21:15:25.882749+0000","last_change":"2023-09-23T21:11:27.777154+0000","last_active":"2023-09-23T21:15:25.882749+0000","last_peered":"2023-09-23T21:15:25.882749+0000","last_clean":"2023-09-23T21:15:25.882749+0000","last_became_active":"2023-09-23T21:11:27.754438+0000","last_became_peered":"2023-09-23T21:11:27.754438+0000","last_unstale":"2023-09-23T21:15:25.882749+0000","last_undegraded":"2023-09-23T21:15:25.882749+0000","last_fullsized":"2023-09-23T21:15:25.882749+0000","mapping_epoch":452,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":453,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:33:02.845882+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":86,"reported_epoch":544,"state":"active+clean","last_fresh":"2023-09-23T21:15:24.872524+0000","last_change":"2023-09-23T21:12:10.764264+0000","last_active":"2023-09-23T21:15:24.872524+0000","last_peered":"2023-09-23T21:15:24.872524+0000","last_clean":"2023-09-23T21:15:24.872524+0000","last_became_active":"2023-09-23T21:12:10.763486+0000","last_became_peered":"2023-09-23T21:12:10.763486+0000","last_unstale":"2023-09-23T21:15:24.872524+0000","last_undegraded":"2023-09-23T21:15:24.872524+0000","last_fullsized":"2023-09-23T21:15:24.872524+0000","mapping_epoch":470,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:32:27.577036+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,7],"acting":[2,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.10","version":"79'161","reported_seq":680,"reported_epoch":545,"state":"active+clean","last_fresh":"2023-09-23T21:15:25.884023+0000","last_change":"2023-09-23T21:11:11.716703+0000","last_active":"2023-09-23T21:15:25.884023+0000","last_peered":"2023-09-23T21:15:25.884023+0000","last_clean":"2023-09-23T21:15:25.884023+0000","last_became_active":"2023-09-23T21:11:11.716385+0000","last_became_peered":"2023-09-23T21:11:11.716385+0000","last_unstale":"2023-09-23T21:15:25.884023+0000","last_undegraded":"2023-09-23T21:15:25.884023+0000","last_fullsized":"2023-09-23T21:15:25.884023+0000","mapping_epoch":446,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":447,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:06:50.336992+0000","last_deep_scrub":"79'161","last_deep_scrub_stamp":"2023-09-23T21:06:50.336992+0000","last_clean_scrub_stamp":"2023-09-23T21:06:50.336992+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:44:29.538890+0000","scrub_duration":0.0026150930000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,7],"acting":[4,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":484,"reported_epoch":543,"state":"active+clean","last_fresh":"2023-09-23T21:15:23.791577+0000","last_change":"2023-09-23T21:13:12.270990+0000","last_active":"2023-09-23T21:15:23.791577+0000","last_peered":"2023-09-23T21:15:23.791577+0000","last_clean":"2023-09-23T21:15:23.791577+0000","last_became_active":"2023-09-23T21:13:12.237201+0000","last_became_peered":"2023-09-23T21:13:12.237201+0000","last_unstale":"2023-09-23T21:15:23.791577+0000","last_undegraded":"2023-09-23T21:15:23.791577+0000","last_fullsized":"2023-09-23T21:15:23.791577+0000","mapping_epoch":493,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:59:54.523877+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":295,"reported_epoch":542,"state":"active+clean","last_fresh":"2023-09-23T21:15:22.801748+0000","last_change":"2023-09-23T21:10:26.758671+0000","last_active":"2023-09-23T21:15:22.801748+0000","last_peered":"2023-09-23T21:15:22.801748+0000","last_clean":"2023-09-23T21:15:22.801748+0000","last_became_active":"2023-09-23T21:10:26.758244+0000","last_became_peered":"2023-09-23T21:10:26.758244+0000","last_unstale":"2023-09-23T21:15:22.801748+0000","last_undegraded":"2023-09-23T21:15:22.801748+0000","last_fullsized":"2023-09-23T21:15:22.801748+0000","mapping_epoch":423,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":424,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:14:02.735609+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.13","version":"79'161","reported_seq":620,"reported_epoch":542,"state":"active+clean","last_fresh":"2023-09-23T21:15:22.799201+0000","last_change":"2023-09-23T21:13:14.933096+0000","last_active":"2023-09-23T21:15:22.799201+0000","last_peered":"2023-09-23T21:15:22.799201+0000","last_clean":"2023-09-23T21:15:22.799201+0000","last_became_active":"2023-09-23T21:13:14.932703+0000","last_became_peered":"2023-09-23T21:13:14.932703+0000","last_unstale":"2023-09-23T21:15:22.799201+0000","last_undegraded":"2023-09-23T21:15:22.799201+0000","last_fullsized":"2023-09-23T21:15:22.799201+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:36.420362+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:36.420362+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:29:42.163911+0000","scrub_duration":0.028838444000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":537,"reported_epoch":545,"state":"active+clean","last_fresh":"2023-09-23T21:15:25.884200+0000","last_change":"2023-09-23T21:13:12.216467+0000","last_active":"2023-09-23T21:15:25.884200+0000","last_peered":"2023-09-23T21:15:25.884200+0000","last_clean":"2023-09-23T21:15:25.884200+0000","last_became_active":"2023-09-23T21:13:12.215259+0000","last_became_peered":"2023-09-23T21:13:12.215259+0000","last_unstale":"2023-09-23T21:15:25.884200+0000","last_undegraded":"2023-09-23T21:15:25.884200+0000","last_fullsized":"2023-09-23T21:15:25.884200+0000","mapping_epoch":493,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:35:33.987839+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":53,"reported_epoch":543,"state":"active+clean","last_fresh":"2023-09-23T21:15:23.791761+0000","last_change":"2023-09-23T21:13:30.804108+0000","last_active":"2023-09-23T21:15:23.791761+0000","last_peered":"2023-09-23T21:15:23.791761+0000","last_clean":"2023-09-23T21:15:23.791761+0000","last_became_active":"2023-09-23T21:13:30.796505+0000","last_became_peered":"2023-09-23T21:13:30.796505+0000","last_unstale":"2023-09-23T21:15:23.791761+0000","last_undegraded":"2023-09-23T21:15:23.791761+0000","last_fullsized":"2023-09-23T21:15:23.791761+0000","mapping_epoch":501,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:19:33.508496+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.12","version":"79'161","reported_seq":779,"reported_epoch":545,"state":"active+clean","last_fresh":"2023-09-23T21:15:25.891899+0000","last_change":"2023-09-23T21:13:15.206574+0000","last_active":"2023-09-23T21:15:25.891899+0000","last_peered":"2023-09-23T21:15:25.891899+0000","last_clean":"2023-09-23T21:15:25.891899+0000","last_became_active":"2023-09-23T21:13:15.206186+0000","last_became_peered":"2023-09-23T21:13:15.206186+0000","last_unstale":"2023-09-23T21:15:25.891899+0000","last_undegraded":"2023-09-23T21:15:25.891899+0000","last_fullsized":"2023-09-23T21:15:25.891899+0000","mapping_epoch":496,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":497,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:05:56.785173+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:05:56.785173+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:34:29.221809+0000","scrub_duration":0.0042114370000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,0],"acting":[5,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":63,"reported_epoch":543,"state":"active+clean","last_fresh":"2023-09-23T21:15:23.791090+0000","last_change":"2023-09-23T21:13:38.048791+0000","last_active":"2023-09-23T21:15:23.791090+0000","last_peered":"2023-09-23T21:15:23.791090+0000","last_clean":"2023-09-23T21:15:23.791090+0000","last_became_active":"2023-09-23T21:13:30.801386+0000","last_became_peered":"2023-09-23T21:13:30.801386+0000","last_unstale":"2023-09-23T21:15:23.791090+0000","last_undegraded":"2023-09-23T21:15:23.791090+0000","last_fullsized":"2023-09-23T21:15:23.791090+0000","mapping_epoch":501,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:52:49.170414+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":380,"reported_epoch":542,"state":"active+clean","last_fresh":"2023-09-23T21:15:22.800634+0000","last_change":"2023-09-23T21:12:10.765780+0000","last_active":"2023-09-23T21:15:22.800634+0000","last_peered":"2023-09-23T21:15:22.800634+0000","last_clean":"2023-09-23T21:15:22.800634+0000","last_became_active":"2023-09-23T21:12:10.765372+0000","last_became_peered":"2023-09-23T21:12:10.765372+0000","last_unstale":"2023-09-23T21:15:22.800634+0000","last_undegraded":"2023-09-23T21:15:22.800634+0000","last_fullsized":"2023-09-23T21:15:22.800634+0000","mapping_epoch":470,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:56:08.446099+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.7","version":"0'0","reported_seq":10,"reported_epoch":545,"state":"active+clean","last_fresh":"2023-09-23T21:15:25.903158+0000","last_change":"2023-09-23T21:15:25.902813+0000","last_active":"2023-09-23T21:15:25.903158+0000","last_peered":"2023-09-23T21:15:25.903158+0000","last_clean":"2023-09-23T21:15:25.903158+0000","last_became_active":"2023-09-23T21:15:25.902476+0000","last_became_peered":"2023-09-23T21:15:25.902476+0000","last_unstale":"2023-09-23T21:15:25.903158+0000","last_undegraded":"2023-09-23T21:15:25.903158+0000","last_fullsized":"2023-09-23T21:15:25.903158+0000","mapping_epoch":544,"log_start":"0'0","ondisk_log_start":"0'0","created":544,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_clean_scrub_stamp":"2023-09-23T21:15:24.860968+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:41:25.699152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.d","version":"79'161","reported_seq":699,"reported_epoch":545,"state":"active+clean","last_fresh":"2023-09-23T21:15:25.882957+0000","last_change":"2023-09-23T21:12:30.444461+0000","last_active":"2023-09-23T21:15:25.882957+0000","last_peered":"2023-09-23T21:15:25.882957+0000","last_clean":"2023-09-23T21:15:25.882957+0000","last_became_active":"2023-09-23T21:12:30.443979+0000","last_became_peered":"2023-09-23T21:12:30.443979+0000","last_unstale":"2023-09-23T21:15:25.882957+0000","last_undegraded":"2023-09-23T21:15:25.882957+0000","last_fullsized":"2023-09-23T21:15:25.882957+0000","mapping_epoch":470,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:09:36.858077+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:09:36.858077+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0024987500000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":192,"reported_epoch":545,"state":"active+clean","last_fresh":"2023-09-23T21:15:25.883279+0000","last_change":"2023-09-23T21:13:59.872834+0000","last_active":"2023-09-23T21:15:25.883279+0000","last_peered":"2023-09-23T21:15:25.883279+0000","last_clean":"2023-09-23T21:15:25.883279+0000","last_became_active":"2023-09-23T21:13:30.770556+0000","last_became_peered":"2023-09-23T21:13:30.770556+0000","last_unstale":"2023-09-23T21:15:25.883279+0000","last_undegraded":"2023-09-23T21:15:25.883279+0000","last_fullsized":"2023-09-23T21:15:25.883279+0000","mapping_epoch":501,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":521,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:57:55.460052+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":132,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":253,"reported_epoch":545,"state":"active+clean","last_fresh":"2023-09-23T21:15:25.890509+0000","last_change":"2023-09-23T21:10:26.750725+0000","last_active":"2023-09-23T21:15:25.890509+0000","last_peered":"2023-09-23T21:15:25.890509+0000","last_clean":"2023-09-23T21:15:25.890509+0000","last_became_active":"2023-09-23T21:10:26.750307+0000","last_became_peered":"2023-09-23T21:10:26.750307+0000","last_unstale":"2023-09-23T21:15:25.890509+0000","last_undegraded":"2023-09-23T21:15:25.890509+0000","last_fullsized":"2023-09-23T21:15:25.890509+0000","mapping_epoch":423,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":424,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:06:22.040548+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,4],"acting":[5,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.6","version":"0'0","reported_seq":1,"reported_epoch":544,"state":"creating+peering","last_fresh":"2023-09-23T21:15:24.868566+0000","last_change":"2023-09-23T21:15:24.868566+0000","last_active":"2023-09-23T21:15:24.860968+0000","last_peered":"2023-09-23T21:15:24.860968+0000","last_clean":"2023-09-23T21:15:24.860968+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-09-23T21:15:24.868566+0000","last_undegraded":"2023-09-23T21:15:24.868566+0000","last_fullsized":"2023-09-23T21:15:24.868566+0000","mapping_epoch":544,"log_start":"0'0","ondisk_log_start":"0'0","created":544,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_clean_scrub_stamp":"2023-09-23T21:15:24.860968+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:02:41.735858+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.c","version":"79'161","reported_seq":912,"reported_epoch":542,"state":"active+clean","last_fresh":"2023-09-23T21:15:22.800512+0000","last_change":"2023-09-23T21:15:06.730883+0000","last_active":"2023-09-23T21:15:22.800512+0000","last_peered":"2023-09-23T21:15:22.800512+0000","last_clean":"2023-09-23T21:15:22.800512+0000","last_became_active":"2023-09-23T21:13:13.401526+0000","last_became_peered":"2023-09-23T21:13:13.401526+0000","last_unstale":"2023-09-23T21:15:22.800512+0000","last_undegraded":"2023-09-23T21:15:22.800512+0000","last_fullsized":"2023-09-23T21:15:22.800512+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":532,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:22.040907+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:22.040907+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.013021176000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":163840,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":491520,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":178,"reported_epoch":545,"state":"active+clean","last_fresh":"2023-09-23T21:15:25.883479+0000","last_change":"2023-09-23T21:13:59.884113+0000","last_active":"2023-09-23T21:15:25.883479+0000","last_peered":"2023-09-23T21:15:25.883479+0000","last_clean":"2023-09-23T21:15:25.883479+0000","last_became_active":"2023-09-23T21:13:30.806778+0000","last_became_peered":"2023-09-23T21:13:30.806778+0000","last_unstale":"2023-09-23T21:15:25.883479+0000","last_undegraded":"2023-09-23T21:15:25.883479+0000","last_fullsized":"2023-09-23T21:15:25.883479+0000","mapping_epoch":501,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":521,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:36:15.284465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":508,"reported_epoch":545,"state":"active+clean","last_fresh":"2023-09-23T21:15:25.884316+0000","last_change":"2023-09-23T21:13:15.152792+0000","last_active":"2023-09-23T21:15:25.884316+0000","last_peered":"2023-09-23T21:15:25.884316+0000","last_clean":"2023-09-23T21:15:25.884316+0000","last_became_active":"2023-09-23T21:13:14.515084+0000","last_became_peered":"2023-09-23T21:13:14.515084+0000","last_unstale":"2023-09-23T21:15:25.884316+0000","last_undegraded":"2023-09-23T21:15:25.884316+0000","last_fullsized":"2023-09-23T21:15:25.884316+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:08:16.702072+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.5","version":"0'0","reported_seq":10,"reported_epoch":545,"state":"active+clean","last_fresh":"2023-09-23T21:15:25.893910+0000","last_change":"2023-09-23T21:15:25.893591+0000","last_active":"2023-09-23T21:15:25.893910+0000","last_peered":"2023-09-23T21:15:25.893910+0000","last_clean":"2023-09-23T21:15:25.893910+0000","last_became_active":"2023-09-23T21:15:25.893280+0000","last_became_peered":"2023-09-23T21:15:25.893280+0000","last_unstale":"2023-09-23T21:15:25.893910+0000","last_undegraded":"2023-09-23T21:15:25.893910+0000","last_fullsized":"2023-09-23T21:15:25.893910+0000","mapping_epoch":544,"log_start":"0'0","ondisk_log_start":"0'0","created":544,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_clean_scrub_stamp":"2023-09-23T21:15:24.860968+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:02:55.710366+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.f","version":"79'161","reported_seq":833,"reported_epoch":545,"state":"active+clean","last_fresh":"2023-09-23T21:15:25.892975+0000","last_change":"2023-09-23T21:11:30.750338+0000","last_active":"2023-09-23T21:15:25.892975+0000","last_peered":"2023-09-23T21:15:25.892975+0000","last_clean":"2023-09-23T21:15:25.892975+0000","last_became_active":"2023-09-23T21:11:30.749953+0000","last_became_peered":"2023-09-23T21:11:30.749953+0000","last_unstale":"2023-09-23T21:15:25.892975+0000","last_undegraded":"2023-09-23T21:15:25.892975+0000","last_fullsized":"2023-09-23T21:15:25.892975+0000","mapping_epoch":455,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":456,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:27.880776+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:27.880776+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:07:10.405723+0000","scrub_duration":0.0014004899999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,3],"acting":[5,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":269,"reported_epoch":545,"state":"active+clean","last_fresh":"2023-09-23T21:15:25.892664+0000","last_change":"2023-09-23T21:10:26.765068+0000","last_active":"2023-09-23T21:15:25.892664+0000","last_peered":"2023-09-23T21:15:25.892664+0000","last_clean":"2023-09-23T21:15:25.892664+0000","last_became_active":"2023-09-23T21:10:26.758364+0000","last_became_peered":"2023-09-23T21:10:26.758364+0000","last_unstale":"2023-09-23T21:15:25.892664+0000","last_undegraded":"2023-09-23T21:15:25.892664+0000","last_fullsized":"2023-09-23T21:15:25.892664+0000","mapping_epoch":423,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":424,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:13:21.057529+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":280,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":485,"reported_epoch":545,"state":"active+clean","last_fresh":"2023-09-23T21:15:25.891609+0000","last_change":"2023-09-23T21:13:30.770746+0000","last_active":"2023-09-23T21:15:25.891609+0000","last_peered":"2023-09-23T21:15:25.891609+0000","last_clean":"2023-09-23T21:15:25.891609+0000","last_became_active":"2023-09-23T21:13:30.770437+0000","last_became_peered":"2023-09-23T21:13:30.770437+0000","last_unstale":"2023-09-23T21:15:25.891609+0000","last_undegraded":"2023-09-23T21:15:25.891609+0000","last_fullsized":"2023-09-23T21:15:25.891609+0000","mapping_epoch":501,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:33:48.325730+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,0],"acting":[5,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.4","version":"0'0","reported_seq":10,"reported_epoch":545,"state":"active+clean","last_fresh":"2023-09-23T21:15:25.892857+0000","last_change":"2023-09-23T21:15:25.892580+0000","last_active":"2023-09-23T21:15:25.892857+0000","last_peered":"2023-09-23T21:15:25.892857+0000","last_clean":"2023-09-23T21:15:25.892857+0000","last_became_active":"2023-09-23T21:15:25.892251+0000","last_became_peered":"2023-09-23T21:15:25.892251+0000","last_unstale":"2023-09-23T21:15:25.892857+0000","last_undegraded":"2023-09-23T21:15:25.892857+0000","last_fullsized":"2023-09-23T21:15:25.892857+0000","mapping_epoch":544,"log_start":"0'0","ondisk_log_start":"0'0","created":544,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_clean_scrub_stamp":"2023-09-23T21:15:24.860968+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:21:43.221845+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,3],"acting":[7,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.e","version":"79'161","reported_seq":714,"reported_epoch":545,"state":"active+clean","last_fresh":"2023-09-23T21:15:25.885893+0000","last_change":"2023-09-23T21:12:30.440952+0000","last_active":"2023-09-23T21:15:25.885893+0000","last_peered":"2023-09-23T21:15:25.885893+0000","last_clean":"2023-09-23T21:15:25.885893+0000","last_became_active":"2023-09-23T21:12:30.440463+0000","last_became_peered":"2023-09-23T21:12:30.440463+0000","last_unstale":"2023-09-23T21:15:25.885893+0000","last_undegraded":"2023-09-23T21:15:25.885893+0000","last_fullsized":"2023-09-23T21:15:25.885893+0000","mapping_epoch":455,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:10:39.567232+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:10:39.567232+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.005229694,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":340,"reported_epoch":542,"state":"active+clean","last_fresh":"2023-09-23T21:15:22.800814+0000","last_change":"2023-09-23T21:13:13.402418+0000","last_active":"2023-09-23T21:15:22.800814+0000","last_peered":"2023-09-23T21:15:22.800814+0000","last_clean":"2023-09-23T21:15:22.800814+0000","last_became_active":"2023-09-23T21:13:13.400984+0000","last_became_peered":"2023-09-23T21:13:13.400984+0000","last_unstale":"2023-09-23T21:15:22.800814+0000","last_undegraded":"2023-09-23T21:15:22.800814+0000","last_fullsized":"2023-09-23T21:15:22.800814+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":495,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:28:01.411191+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,0],"acting":[3,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":517,"reported_epoch":544,"state":"active+clean","last_fresh":"2023-09-23T21:15:24.869775+0000","last_change":"2023-09-23T21:13:14.339670+0000","last_active":"2023-09-23T21:15:24.869775+0000","last_peered":"2023-09-23T21:15:24.869775+0000","last_clean":"2023-09-23T21:15:24.869775+0000","last_became_active":"2023-09-23T21:13:14.339341+0000","last_became_peered":"2023-09-23T21:13:14.339341+0000","last_unstale":"2023-09-23T21:15:24.869775+0000","last_undegraded":"2023-09-23T21:15:24.869775+0000","last_fullsized":"2023-09-23T21:15:24.869775+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:23:19.411368+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.3","version":"0'0","reported_seq":10,"reported_epoch":545,"state":"active+clean","last_fresh":"2023-09-23T21:15:25.892335+0000","last_change":"2023-09-23T21:15:25.891637+0000","last_active":"2023-09-23T21:15:25.892335+0000","last_peered":"2023-09-23T21:15:25.892335+0000","last_clean":"2023-09-23T21:15:25.892335+0000","last_became_active":"2023-09-23T21:15:25.891330+0000","last_became_peered":"2023-09-23T21:15:25.891330+0000","last_unstale":"2023-09-23T21:15:25.892335+0000","last_undegraded":"2023-09-23T21:15:25.892335+0000","last_fullsized":"2023-09-23T21:15:25.892335+0000","mapping_epoch":544,"log_start":"0'0","ondisk_log_start":"0'0","created":544,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_clean_scrub_stamp":"2023-09-23T21:15:24.860968+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:22:07.243528+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,2],"acting":[7,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.9","version":"79'161","reported_seq":699,"reported_epoch":544,"state":"active+clean","last_fresh":"2023-09-23T21:15:24.872875+0000","last_change":"2023-09-23T21:12:30.436118+0000","last_active":"2023-09-23T21:15:24.872875+0000","last_peered":"2023-09-23T21:15:24.872875+0000","last_clean":"2023-09-23T21:15:24.872875+0000","last_became_active":"2023-09-23T21:12:30.435779+0000","last_became_peered":"2023-09-23T21:12:30.435779+0000","last_unstale":"2023-09-23T21:15:24.872875+0000","last_undegraded":"2023-09-23T21:15:24.872875+0000","last_fullsized":"2023-09-23T21:15:24.872875+0000","mapping_epoch":433,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:10:37.696951+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:10:37.696951+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.001386504,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,3],"acting":[2,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":102,"reported_epoch":543,"state":"active+clean","last_fresh":"2023-09-23T21:15:23.791784+0000","last_change":"2023-09-23T21:13:30.793273+0000","last_active":"2023-09-23T21:15:23.791784+0000","last_peered":"2023-09-23T21:15:23.791784+0000","last_clean":"2023-09-23T21:15:23.791784+0000","last_became_active":"2023-09-23T21:13:30.791269+0000","last_became_peered":"2023-09-23T21:13:30.791269+0000","last_unstale":"2023-09-23T21:15:23.791784+0000","last_undegraded":"2023-09-23T21:15:23.791784+0000","last_fullsized":"2023-09-23T21:15:23.791784+0000","mapping_epoch":501,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:38:07.142290+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":119,"reported_epoch":545,"state":"active+clean","last_fresh":"2023-09-23T21:15:25.884666+0000","last_change":"2023-09-23T21:12:14.313466+0000","last_active":"2023-09-23T21:15:25.884666+0000","last_peered":"2023-09-23T21:15:25.884666+0000","last_clean":"2023-09-23T21:15:25.884666+0000","last_became_active":"2023-09-23T21:12:14.209837+0000","last_became_peered":"2023-09-23T21:12:14.209837+0000","last_unstale":"2023-09-23T21:15:25.884666+0000","last_undegraded":"2023-09-23T21:15:25.884666+0000","last_fullsized":"2023-09-23T21:15:25.884666+0000","mapping_epoch":470,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":474,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:09:40.375241+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,7],"acting":[4,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.2","version":"0'0","reported_seq":10,"reported_epoch":545,"state":"active+clean","last_fresh":"2023-09-23T21:15:25.901288+0000","last_change":"2023-09-23T21:15:25.900671+0000","last_active":"2023-09-23T21:15:25.901288+0000","last_peered":"2023-09-23T21:15:25.901288+0000","last_clean":"2023-09-23T21:15:25.901288+0000","last_became_active":"2023-09-23T21:15:25.900364+0000","last_became_peered":"2023-09-23T21:15:25.900364+0000","last_unstale":"2023-09-23T21:15:25.901288+0000","last_undegraded":"2023-09-23T21:15:25.901288+0000","last_fullsized":"2023-09-23T21:15:25.901288+0000","mapping_epoch":544,"log_start":"0'0","ondisk_log_start":"0'0","created":544,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_clean_scrub_stamp":"2023-09-23T21:15:24.860968+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:03:40.630119+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.8","version":"79'161","reported_seq":665,"reported_epoch":544,"state":"active+clean","last_fresh":"2023-09-23T21:15:24.872114+0000","last_change":"2023-09-23T21:12:30.433561+0000","last_active":"2023-09-23T21:15:24.872114+0000","last_peered":"2023-09-23T21:15:24.872114+0000","last_clean":"2023-09-23T21:15:24.872114+0000","last_became_active":"2023-09-23T21:12:30.433217+0000","last_became_peered":"2023-09-23T21:12:30.433217+0000","last_unstale":"2023-09-23T21:15:24.872114+0000","last_undegraded":"2023-09-23T21:15:24.872114+0000","last_fullsized":"2023-09-23T21:15:24.872114+0000","mapping_epoch":359,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:06:52.098882+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:06:52.098882+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0092173299999999993,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":131344,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":122,"reported_epoch":545,"state":"active+clean","last_fresh":"2023-09-23T21:15:25.884647+0000","last_change":"2023-09-23T21:12:10.823380+0000","last_active":"2023-09-23T21:15:25.884647+0000","last_peered":"2023-09-23T21:15:25.884647+0000","last_clean":"2023-09-23T21:15:25.884647+0000","last_became_active":"2023-09-23T21:12:10.823044+0000","last_became_peered":"2023-09-23T21:12:10.823044+0000","last_unstale":"2023-09-23T21:15:25.884647+0000","last_undegraded":"2023-09-23T21:15:25.884647+0000","last_fullsized":"2023-09-23T21:15:25.884647+0000","mapping_epoch":470,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:53:29.846536+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,5],"acting":[4,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":123,"reported_epoch":545,"state":"active+clean","last_fresh":"2023-09-23T21:15:25.884442+0000","last_change":"2023-09-23T21:13:30.772451+0000","last_active":"2023-09-23T21:15:25.884442+0000","last_peered":"2023-09-23T21:15:25.884442+0000","last_clean":"2023-09-23T21:15:25.884442+0000","last_became_active":"2023-09-23T21:13:30.772040+0000","last_became_peered":"2023-09-23T21:13:30.772040+0000","last_unstale":"2023-09-23T21:15:25.884442+0000","last_undegraded":"2023-09-23T21:15:25.884442+0000","last_fullsized":"2023-09-23T21:15:25.884442+0000","mapping_epoch":501,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:24:49.246727+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,0],"acting":[4,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.1","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-09-23T21:15:24.860968+0000","last_change":"2023-09-23T21:15:24.860968+0000","last_active":"2023-09-23T21:15:24.860968+0000","last_peered":"2023-09-23T21:15:24.860968+0000","last_clean":"2023-09-23T21:15:24.860968+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-09-23T21:15:24.860968+0000","last_undegraded":"2023-09-23T21:15:24.860968+0000","last_fullsized":"2023-09-23T21:15:24.860968+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_clean_scrub_stamp":"2023-09-23T21:15:24.860968+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"1.b","version":"79'161","reported_seq":547,"reported_epoch":543,"state":"active+clean","last_fresh":"2023-09-23T21:15:23.791385+0000","last_change":"2023-09-23T21:13:13.373313+0000","last_active":"2023-09-23T21:15:23.791385+0000","last_peered":"2023-09-23T21:15:23.791385+0000","last_clean":"2023-09-23T21:15:23.791385+0000","last_became_active":"2023-09-23T21:13:13.372328+0000","last_became_peered":"2023-09-23T21:13:13.372328+0000","last_unstale":"2023-09-23T21:15:23.791385+0000","last_undegraded":"2023-09-23T21:15:23.791385+0000","last_fullsized":"2023-09-23T21:15:23.791385+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":495,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:05:55.772264+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:05:55.772264+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.001862507,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":103,"reported_epoch":545,"state":"active+clean","last_fresh":"2023-09-23T21:15:25.884072+0000","last_change":"2023-09-23T21:11:27.754717+0000","last_active":"2023-09-23T21:15:25.884072+0000","last_peered":"2023-09-23T21:15:25.884072+0000","last_clean":"2023-09-23T21:15:25.884072+0000","last_became_active":"2023-09-23T21:11:27.754376+0000","last_became_peered":"2023-09-23T21:11:27.754376+0000","last_unstale":"2023-09-23T21:15:25.884072+0000","last_undegraded":"2023-09-23T21:15:25.884072+0000","last_fullsized":"2023-09-23T21:15:25.884072+0000","mapping_epoch":452,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":453,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:26:41.734541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":477,"reported_epoch":545,"state":"active+clean","last_fresh":"2023-09-23T21:15:25.886117+0000","last_change":"2023-09-23T21:12:10.811698+0000","last_active":"2023-09-23T21:15:25.886117+0000","last_peered":"2023-09-23T21:15:25.886117+0000","last_clean":"2023-09-23T21:15:25.886117+0000","last_became_active":"2023-09-23T21:12:10.789656+0000","last_became_peered":"2023-09-23T21:12:10.789656+0000","last_unstale":"2023-09-23T21:15:25.886117+0000","last_undegraded":"2023-09-23T21:15:25.886117+0000","last_fullsized":"2023-09-23T21:15:25.886117+0000","mapping_epoch":470,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:39:15.112091+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,5],"acting":[4,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.0","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-09-23T21:15:24.860968+0000","last_change":"2023-09-23T21:15:24.860968+0000","last_active":"2023-09-23T21:15:24.860968+0000","last_peered":"2023-09-23T21:15:24.860968+0000","last_clean":"2023-09-23T21:15:24.860968+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-09-23T21:15:24.860968+0000","last_undegraded":"2023-09-23T21:15:24.860968+0000","last_fullsized":"2023-09-23T21:15:24.860968+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_clean_scrub_stamp":"2023-09-23T21:15:24.860968+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"1.a","version":"79'161","reported_seq":691,"reported_epoch":544,"state":"active+clean","last_fresh":"2023-09-23T21:15:24.871948+0000","last_change":"2023-09-23T21:12:30.437790+0000","last_active":"2023-09-23T21:15:24.871948+0000","last_peered":"2023-09-23T21:15:24.871948+0000","last_clean":"2023-09-23T21:15:24.871948+0000","last_became_active":"2023-09-23T21:12:30.437427+0000","last_became_peered":"2023-09-23T21:12:30.437427+0000","last_unstale":"2023-09-23T21:15:24.871948+0000","last_undegraded":"2023-09-23T21:15:24.871948+0000","last_fullsized":"2023-09-23T21:15:24.871948+0000","mapping_epoch":446,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:06:51.075448+0000","last_deep_scrub":"79'161","last_deep_scrub_stamp":"2023-09-23T21:05:32.630292+0000","last_clean_scrub_stamp":"2023-09-23T21:06:51.075448+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0054793890000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,4],"acting":[2,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":390,"reported_epoch":542,"state":"active+clean","last_fresh":"2023-09-23T21:15:22.798856+0000","last_change":"2023-09-23T21:12:10.792918+0000","last_active":"2023-09-23T21:15:22.798856+0000","last_peered":"2023-09-23T21:15:22.798856+0000","last_clean":"2023-09-23T21:15:22.798856+0000","last_became_active":"2023-09-23T21:12:10.792679+0000","last_became_peered":"2023-09-23T21:12:10.792679+0000","last_unstale":"2023-09-23T21:15:22.798856+0000","last_undegraded":"2023-09-23T21:15:22.798856+0000","last_fullsized":"2023-09-23T21:15:22.798856+0000","mapping_epoch":470,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:30:48.008690+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,4],"acting":[3,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":521,"reported_epoch":542,"state":"active+clean","last_fresh":"2023-09-23T21:15:22.801263+0000","last_change":"2023-09-23T21:12:10.790812+0000","last_active":"2023-09-23T21:15:22.801263+0000","last_peered":"2023-09-23T21:15:22.801263+0000","last_clean":"2023-09-23T21:15:22.801263+0000","last_became_active":"2023-09-23T21:12:10.790492+0000","last_became_peered":"2023-09-23T21:12:10.790492+0000","last_unstale":"2023-09-23T21:15:22.801263+0000","last_undegraded":"2023-09-23T21:15:22.801263+0000","last_fullsized":"2023-09-23T21:15:22.801263+0000","mapping_epoch":470,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:02:19.863118+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.d","version":"0'0","reported_seq":10,"reported_epoch":545,"state":"active+clean","last_fresh":"2023-09-23T21:15:25.893348+0000","last_change":"2023-09-23T21:15:25.892834+0000","last_active":"2023-09-23T21:15:25.893348+0000","last_peered":"2023-09-23T21:15:25.893348+0000","last_clean":"2023-09-23T21:15:25.893348+0000","last_became_active":"2023-09-23T21:15:25.892519+0000","last_became_peered":"2023-09-23T21:15:25.892519+0000","last_unstale":"2023-09-23T21:15:25.893348+0000","last_undegraded":"2023-09-23T21:15:25.893348+0000","last_fullsized":"2023-09-23T21:15:25.893348+0000","mapping_epoch":544,"log_start":"0'0","ondisk_log_start":"0'0","created":544,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_clean_scrub_stamp":"2023-09-23T21:15:24.860968+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:46:28.375665+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,5],"acting":[2,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.7","version":"79'161","reported_seq":635,"reported_epoch":542,"state":"active+clean","last_fresh":"2023-09-23T21:15:22.800512+0000","last_change":"2023-09-23T21:13:44.416548+0000","last_active":"2023-09-23T21:15:22.800512+0000","last_peered":"2023-09-23T21:15:22.800512+0000","last_clean":"2023-09-23T21:15:22.800512+0000","last_became_active":"2023-09-23T21:13:44.415617+0000","last_became_peered":"2023-09-23T21:13:44.415617+0000","last_unstale":"2023-09-23T21:15:22.800512+0000","last_undegraded":"2023-09-23T21:15:22.800512+0000","last_fullsized":"2023-09-23T21:15:22.800512+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":3,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:25.916867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:25.916867+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.001665323,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":474,"reported_epoch":543,"state":"active+clean","last_fresh":"2023-09-23T21:15:23.792340+0000","last_change":"2023-09-23T21:13:13.211959+0000","last_active":"2023-09-23T21:15:23.792340+0000","last_peered":"2023-09-23T21:15:23.792340+0000","last_clean":"2023-09-23T21:15:23.792340+0000","last_became_active":"2023-09-23T21:13:13.211712+0000","last_became_peered":"2023-09-23T21:13:13.211712+0000","last_unstale":"2023-09-23T21:15:23.792340+0000","last_undegraded":"2023-09-23T21:15:23.792340+0000","last_fullsized":"2023-09-23T21:15:23.792340+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":495,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:37:21.655775+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,2],"acting":[0,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":223,"reported_epoch":545,"state":"active+clean","last_fresh":"2023-09-23T21:15:25.892486+0000","last_change":"2023-09-23T21:05:04.326889+0000","last_active":"2023-09-23T21:15:25.892486+0000","last_peered":"2023-09-23T21:15:25.892486+0000","last_clean":"2023-09-23T21:15:25.892486+0000","last_became_active":"2023-09-23T21:05:04.326032+0000","last_became_peered":"2023-09-23T21:05:04.326032+0000","last_unstale":"2023-09-23T21:15:25.892486+0000","last_undegraded":"2023-09-23T21:15:25.892486+0000","last_fullsized":"2023-09-23T21:15:25.892486+0000","mapping_epoch":332,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:00:17.507889+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":555,"reported_epoch":545,"state":"active+clean","last_fresh":"2023-09-23T21:15:25.891754+0000","last_change":"2023-09-23T21:13:13.285608+0000","last_active":"2023-09-23T21:15:25.891754+0000","last_peered":"2023-09-23T21:15:25.891754+0000","last_clean":"2023-09-23T21:15:25.891754+0000","last_became_active":"2023-09-23T21:13:13.285255+0000","last_became_peered":"2023-09-23T21:13:13.285255+0000","last_unstale":"2023-09-23T21:15:25.891754+0000","last_undegraded":"2023-09-23T21:15:25.891754+0000","last_fullsized":"2023-09-23T21:15:25.891754+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":495,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:42:30.087599+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.b","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-09-23T21:15:24.860968+0000","last_change":"2023-09-23T21:15:24.860968+0000","last_active":"2023-09-23T21:15:24.860968+0000","last_peered":"2023-09-23T21:15:24.860968+0000","last_clean":"2023-09-23T21:15:24.860968+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-09-23T21:15:24.860968+0000","last_undegraded":"2023-09-23T21:15:24.860968+0000","last_fullsized":"2023-09-23T21:15:24.860968+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_clean_scrub_stamp":"2023-09-23T21:15:24.860968+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"1.1","version":"79'161","reported_seq":615,"reported_epoch":543,"state":"active+clean","last_fresh":"2023-09-23T21:15:23.791926+0000","last_change":"2023-09-23T21:13:12.204135+0000","last_active":"2023-09-23T21:15:23.791926+0000","last_peered":"2023-09-23T21:15:23.791926+0000","last_clean":"2023-09-23T21:15:23.791926+0000","last_became_active":"2023-09-23T21:13:12.203380+0000","last_became_peered":"2023-09-23T21:13:12.203380+0000","last_unstale":"2023-09-23T21:15:23.791926+0000","last_undegraded":"2023-09-23T21:15:23.791926+0000","last_fullsized":"2023-09-23T21:15:23.791926+0000","mapping_epoch":493,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:06:52.575900+0000","last_deep_scrub":"79'161","last_deep_scrub_stamp":"2023-09-23T21:05:30.652189+0000","last_clean_scrub_stamp":"2023-09-23T21:06:52.575900+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":211,"reported_epoch":543,"state":"active+clean","last_fresh":"2023-09-23T21:15:23.791456+0000","last_change":"2023-09-23T21:13:13.238617+0000","last_active":"2023-09-23T21:15:23.791456+0000","last_peered":"2023-09-23T21:15:23.791456+0000","last_clean":"2023-09-23T21:15:23.791456+0000","last_became_active":"2023-09-23T21:13:13.232926+0000","last_became_peered":"2023-09-23T21:13:13.232926+0000","last_unstale":"2023-09-23T21:15:23.791456+0000","last_undegraded":"2023-09-23T21:15:23.791456+0000","last_fullsized":"2023-09-23T21:15:23.791456+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":495,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:17:21.823772+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":479,"reported_epoch":545,"state":"active+clean","last_fresh":"2023-09-23T21:15:25.884769+0000","last_change":"2023-09-23T21:13:14.144918+0000","last_active":"2023-09-23T21:15:25.884769+0000","last_peered":"2023-09-23T21:15:25.884769+0000","last_clean":"2023-09-23T21:15:25.884769+0000","last_became_active":"2023-09-23T21:13:14.132388+0000","last_became_peered":"2023-09-23T21:13:14.132388+0000","last_unstale":"2023-09-23T21:15:25.884769+0000","last_undegraded":"2023-09-23T21:15:25.884769+0000","last_fullsized":"2023-09-23T21:15:25.884769+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":495,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:56:02.619905+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,5],"acting":[4,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":365,"reported_epoch":544,"state":"active+clean","last_fresh":"2023-09-23T21:15:24.872831+0000","last_change":"2023-09-23T21:12:10.764503+0000","last_active":"2023-09-23T21:15:24.872831+0000","last_peered":"2023-09-23T21:15:24.872831+0000","last_clean":"2023-09-23T21:15:24.872831+0000","last_became_active":"2023-09-23T21:12:10.763933+0000","last_became_peered":"2023-09-23T21:12:10.763933+0000","last_unstale":"2023-09-23T21:15:24.872831+0000","last_undegraded":"2023-09-23T21:15:24.872831+0000","last_fullsized":"2023-09-23T21:15:24.872831+0000","mapping_epoch":470,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:28:33.202825+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,7],"acting":[2,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"11.8","version":"0'0","reported_seq":1,"reported_epoch":544,"state":"creating+peering","last_fresh":"2023-09-23T21:15:24.868503+0000","last_change":"2023-09-23T21:15:24.868503+0000","last_active":"2023-09-23T21:15:24.860968+0000","last_peered":"2023-09-23T21:15:24.860968+0000","last_clean":"2023-09-23T21:15:24.860968+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-09-23T21:15:24.868503+0000","last_undegraded":"2023-09-23T21:15:24.868503+0000","last_fullsized":"2023-09-23T21:15:24.868503+0000","mapping_epoch":544,"log_start":"0'0","ondisk_log_start":"0'0","created":544,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_clean_scrub_stamp":"2023-09-23T21:15:24.860968+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:16:01.535087+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,5],"acting":[0,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.2","version":"79'161","reported_seq":824,"reported_epoch":543,"state":"active+clean","last_fresh":"2023-09-23T21:15:23.791248+0000","last_change":"2023-09-23T21:13:12.209907+0000","last_active":"2023-09-23T21:15:23.791248+0000","last_peered":"2023-09-23T21:15:23.791248+0000","last_clean":"2023-09-23T21:15:23.791248+0000","last_became_active":"2023-09-23T21:13:12.205664+0000","last_became_peered":"2023-09-23T21:13:12.205664+0000","last_unstale":"2023-09-23T21:15:23.791248+0000","last_undegraded":"2023-09-23T21:15:23.791248+0000","last_fullsized":"2023-09-23T21:15:23.791248+0000","mapping_epoch":493,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:24.802451+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:24.802451+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:26:16.035788+0000","scrub_duration":0.0090861250000000005,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,4],"acting":[0,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":553,"reported_epoch":544,"state":"active+clean","last_fresh":"2023-09-23T21:15:24.873106+0000","last_change":"2023-09-23T21:13:12.206980+0000","last_active":"2023-09-23T21:15:24.873106+0000","last_peered":"2023-09-23T21:15:24.873106+0000","last_clean":"2023-09-23T21:15:24.873106+0000","last_became_active":"2023-09-23T21:13:12.206650+0000","last_became_peered":"2023-09-23T21:13:12.206650+0000","last_unstale":"2023-09-23T21:15:24.873106+0000","last_undegraded":"2023-09-23T21:15:24.873106+0000","last_fullsized":"2023-09-23T21:15:24.873106+0000","mapping_epoch":493,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:10:01.892936+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":537,"reported_epoch":544,"state":"active+clean","last_fresh":"2023-09-23T21:15:24.872258+0000","last_change":"2023-09-23T21:13:13.407073+0000","last_active":"2023-09-23T21:15:24.872258+0000","last_peered":"2023-09-23T21:15:24.872258+0000","last_clean":"2023-09-23T21:15:24.872258+0000","last_became_active":"2023-09-23T21:13:13.395047+0000","last_became_peered":"2023-09-23T21:13:13.395047+0000","last_unstale":"2023-09-23T21:15:24.872258+0000","last_undegraded":"2023-09-23T21:15:24.872258+0000","last_fullsized":"2023-09-23T21:15:24.872258+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":495,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:04:43.384190+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,5],"acting":[2,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":53,"reported_epoch":543,"state":"active+clean","last_fresh":"2023-09-23T21:15:23.792098+0000","last_change":"2023-09-23T21:13:30.785054+0000","last_active":"2023-09-23T21:15:23.792098+0000","last_peered":"2023-09-23T21:15:23.792098+0000","last_clean":"2023-09-23T21:15:23.792098+0000","last_became_active":"2023-09-23T21:13:30.784701+0000","last_became_peered":"2023-09-23T21:13:30.784701+0000","last_unstale":"2023-09-23T21:15:23.792098+0000","last_undegraded":"2023-09-23T21:15:23.792098+0000","last_fullsized":"2023-09-23T21:15:23.792098+0000","mapping_epoch":501,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:19:20.152252+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.a","version":"0'0","reported_seq":10,"reported_epoch":545,"state":"active+clean","last_fresh":"2023-09-23T21:15:25.894587+0000","last_change":"2023-09-23T21:15:25.893846+0000","last_active":"2023-09-23T21:15:25.894587+0000","last_peered":"2023-09-23T21:15:25.894587+0000","last_clean":"2023-09-23T21:15:25.894587+0000","last_became_active":"2023-09-23T21:15:25.893452+0000","last_became_peered":"2023-09-23T21:15:25.893452+0000","last_unstale":"2023-09-23T21:15:25.894587+0000","last_undegraded":"2023-09-23T21:15:25.894587+0000","last_fullsized":"2023-09-23T21:15:25.894587+0000","mapping_epoch":544,"log_start":"0'0","ondisk_log_start":"0'0","created":544,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_clean_scrub_stamp":"2023-09-23T21:15:24.860968+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:42:32.842297+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,5],"acting":[2,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":331,"reported_epoch":544,"state":"active+clean","last_fresh":"2023-09-23T21:15:24.872645+0000","last_change":"2023-09-23T21:10:26.769764+0000","last_active":"2023-09-23T21:15:24.872645+0000","last_peered":"2023-09-23T21:15:24.872645+0000","last_clean":"2023-09-23T21:15:24.872645+0000","last_became_active":"2023-09-23T21:10:26.769326+0000","last_became_peered":"2023-09-23T21:10:26.769326+0000","last_unstale":"2023-09-23T21:15:24.872645+0000","last_undegraded":"2023-09-23T21:15:24.872645+0000","last_fullsized":"2023-09-23T21:15:24.872645+0000","mapping_epoch":423,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":424,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:57:45.219837+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"79'161","reported_seq":726,"reported_epoch":543,"state":"active+clean","last_fresh":"2023-09-23T21:15:23.790877+0000","last_change":"2023-09-23T21:13:33.766895+0000","last_active":"2023-09-23T21:15:23.790877+0000","last_peered":"2023-09-23T21:15:23.790877+0000","last_clean":"2023-09-23T21:15:23.790877+0000","last_became_active":"2023-09-23T21:13:33.766538+0000","last_became_peered":"2023-09-23T21:13:33.766538+0000","last_unstale":"2023-09-23T21:15:23.790877+0000","last_undegraded":"2023-09-23T21:15:23.790877+0000","last_fullsized":"2023-09-23T21:15:23.790877+0000","mapping_epoch":504,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":505,"parent":"0.0","parent_split_bits":0,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:20.738273+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:20.738273+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:35:35.855987+0000","scrub_duration":0.0088336339999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,2],"acting":[0,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":553,"reported_epoch":545,"state":"active+clean","last_fresh":"2023-09-23T21:15:25.891701+0000","last_change":"2023-09-23T21:11:08.760417+0000","last_active":"2023-09-23T21:15:25.891701+0000","last_peered":"2023-09-23T21:15:25.891701+0000","last_clean":"2023-09-23T21:15:25.891701+0000","last_became_active":"2023-09-23T21:11:08.760150+0000","last_became_peered":"2023-09-23T21:11:08.760150+0000","last_unstale":"2023-09-23T21:15:25.891701+0000","last_undegraded":"2023-09-23T21:15:25.891701+0000","last_fullsized":"2023-09-23T21:15:25.891701+0000","mapping_epoch":443,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":444,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:23:10.849768+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":120,"reported_epoch":545,"state":"active+clean","last_fresh":"2023-09-23T21:15:25.884758+0000","last_change":"2023-09-23T21:13:12.216020+0000","last_active":"2023-09-23T21:15:25.884758+0000","last_peered":"2023-09-23T21:15:25.884758+0000","last_clean":"2023-09-23T21:15:25.884758+0000","last_became_active":"2023-09-23T21:13:12.215180+0000","last_became_peered":"2023-09-23T21:13:12.215180+0000","last_unstale":"2023-09-23T21:15:25.884758+0000","last_undegraded":"2023-09-23T21:15:25.884758+0000","last_fullsized":"2023-09-23T21:15:25.884758+0000","mapping_epoch":493,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:26:18.704684+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.9","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-09-23T21:15:24.860968+0000","last_change":"2023-09-23T21:15:24.860968+0000","last_active":"2023-09-23T21:15:24.860968+0000","last_peered":"2023-09-23T21:15:24.860968+0000","last_clean":"2023-09-23T21:15:24.860968+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-09-23T21:15:24.860968+0000","last_undegraded":"2023-09-23T21:15:24.860968+0000","last_fullsized":"2023-09-23T21:15:24.860968+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_clean_scrub_stamp":"2023-09-23T21:15:24.860968+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"1.3","version":"79'161","reported_seq":848,"reported_epoch":545,"state":"active+clean","last_fresh":"2023-09-23T21:15:25.884077+0000","last_change":"2023-09-23T21:13:33.771279+0000","last_active":"2023-09-23T21:15:25.884077+0000","last_peered":"2023-09-23T21:15:25.884077+0000","last_clean":"2023-09-23T21:15:25.884077+0000","last_became_active":"2023-09-23T21:13:33.770980+0000","last_became_peered":"2023-09-23T21:13:33.770980+0000","last_unstale":"2023-09-23T21:15:25.884077+0000","last_undegraded":"2023-09-23T21:15:25.884077+0000","last_fullsized":"2023-09-23T21:15:25.884077+0000","mapping_epoch":504,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":505,"parent":"0.0","parent_split_bits":3,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:05:57.832302+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:05:57.832302+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:19:42.309604+0000","scrub_duration":0.010206847999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":361,"reported_epoch":543,"state":"active+clean","last_fresh":"2023-09-23T21:15:23.791185+0000","last_change":"2023-09-23T21:13:13.212775+0000","last_active":"2023-09-23T21:15:23.791185+0000","last_peered":"2023-09-23T21:15:23.791185+0000","last_clean":"2023-09-23T21:15:23.791185+0000","last_became_active":"2023-09-23T21:13:13.212460+0000","last_became_peered":"2023-09-23T21:13:13.212460+0000","last_unstale":"2023-09-23T21:15:23.791185+0000","last_undegraded":"2023-09-23T21:15:23.791185+0000","last_fullsized":"2023-09-23T21:15:23.791185+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":495,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:27:10.853970+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":543,"reported_epoch":543,"state":"active+clean","last_fresh":"2023-09-23T21:15:23.791447+0000","last_change":"2023-09-23T21:13:44.409212+0000","last_active":"2023-09-23T21:15:23.791447+0000","last_peered":"2023-09-23T21:15:23.791447+0000","last_clean":"2023-09-23T21:15:23.791447+0000","last_became_active":"2023-09-23T21:13:44.408958+0000","last_became_peered":"2023-09-23T21:13:44.408958+0000","last_unstale":"2023-09-23T21:15:23.791447+0000","last_undegraded":"2023-09-23T21:15:23.791447+0000","last_fullsized":"2023-09-23T21:15:23.791447+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:03:52.749964+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":53,"reported_epoch":543,"state":"active+clean","last_fresh":"2023-09-23T21:15:23.791968+0000","last_change":"2023-09-23T21:13:30.804379+0000","last_active":"2023-09-23T21:15:23.791968+0000","last_peered":"2023-09-23T21:15:23.791968+0000","last_clean":"2023-09-23T21:15:23.791968+0000","last_became_active":"2023-09-23T21:13:30.799907+0000","last_became_peered":"2023-09-23T21:13:30.799907+0000","last_unstale":"2023-09-23T21:15:23.791968+0000","last_undegraded":"2023-09-23T21:15:23.791968+0000","last_fullsized":"2023-09-23T21:15:23.791968+0000","mapping_epoch":501,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:49:42.859212+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,7],"acting":[0,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.c","version":"0'0","reported_seq":10,"reported_epoch":545,"state":"active+clean","last_fresh":"2023-09-23T21:15:25.888036+0000","last_change":"2023-09-23T21:15:25.887291+0000","last_active":"2023-09-23T21:15:25.888036+0000","last_peered":"2023-09-23T21:15:25.888036+0000","last_clean":"2023-09-23T21:15:25.888036+0000","last_became_active":"2023-09-23T21:15:25.887008+0000","last_became_peered":"2023-09-23T21:15:25.887008+0000","last_unstale":"2023-09-23T21:15:25.888036+0000","last_undegraded":"2023-09-23T21:15:25.888036+0000","last_fullsized":"2023-09-23T21:15:25.888036+0000","mapping_epoch":544,"log_start":"0'0","ondisk_log_start":"0'0","created":544,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_clean_scrub_stamp":"2023-09-23T21:15:24.860968+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:18:00.855026+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,7],"acting":[2,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.6","version":"79'161","reported_seq":462,"reported_epoch":543,"state":"active+clean","last_fresh":"2023-09-23T21:15:23.791875+0000","last_change":"2023-09-23T21:13:33.770715+0000","last_active":"2023-09-23T21:15:23.791875+0000","last_peered":"2023-09-23T21:15:23.791875+0000","last_clean":"2023-09-23T21:15:23.791875+0000","last_became_active":"2023-09-23T21:13:33.769303+0000","last_became_peered":"2023-09-23T21:13:33.769303+0000","last_unstale":"2023-09-23T21:15:23.791875+0000","last_undegraded":"2023-09-23T21:15:23.791875+0000","last_fullsized":"2023-09-23T21:15:23.791875+0000","mapping_epoch":504,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":505,"parent":"0.0","parent_split_bits":3,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:10:24.268662+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:10:24.268662+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13417,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":11,"num_read_kb":20,"num_write":5,"num_write_kb":55,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":40252,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":172,"reported_epoch":545,"state":"active+clean","last_fresh":"2023-09-23T21:15:25.882999+0000","last_change":"2023-09-23T21:13:30.770826+0000","last_active":"2023-09-23T21:15:25.882999+0000","last_peered":"2023-09-23T21:15:25.882999+0000","last_clean":"2023-09-23T21:15:25.882999+0000","last_became_active":"2023-09-23T21:13:30.770512+0000","last_became_peered":"2023-09-23T21:13:30.770512+0000","last_unstale":"2023-09-23T21:15:25.882999+0000","last_undegraded":"2023-09-23T21:15:25.882999+0000","last_fullsized":"2023-09-23T21:15:25.882999+0000","mapping_epoch":501,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:44:23.470538+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":428,"reported_epoch":544,"state":"active+clean","last_fresh":"2023-09-23T21:15:24.873015+0000","last_change":"2023-09-23T21:13:30.790590+0000","last_active":"2023-09-23T21:15:24.873015+0000","last_peered":"2023-09-23T21:15:24.873015+0000","last_clean":"2023-09-23T21:15:24.873015+0000","last_became_active":"2023-09-23T21:13:30.790253+0000","last_became_peered":"2023-09-23T21:13:30.790253+0000","last_unstale":"2023-09-23T21:15:24.873015+0000","last_undegraded":"2023-09-23T21:15:24.873015+0000","last_fullsized":"2023-09-23T21:15:24.873015+0000","mapping_epoch":501,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:23:41.843250+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,0],"acting":[2,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":490,"reported_epoch":543,"state":"active+clean","last_fresh":"2023-09-23T21:15:23.792059+0000","last_change":"2023-09-23T21:13:12.210546+0000","last_active":"2023-09-23T21:15:23.792059+0000","last_peered":"2023-09-23T21:15:23.792059+0000","last_clean":"2023-09-23T21:15:23.792059+0000","last_became_active":"2023-09-23T21:13:12.205934+0000","last_became_peered":"2023-09-23T21:13:12.205934+0000","last_unstale":"2023-09-23T21:15:23.792059+0000","last_undegraded":"2023-09-23T21:15:23.792059+0000","last_fullsized":"2023-09-23T21:15:23.792059+0000","mapping_epoch":493,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:21:18.299428+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.f","version":"0'0","reported_seq":10,"reported_epoch":545,"state":"active+clean","last_fresh":"2023-09-23T21:15:25.904811+0000","last_change":"2023-09-23T21:15:25.902442+0000","last_active":"2023-09-23T21:15:25.904811+0000","last_peered":"2023-09-23T21:15:25.904811+0000","last_clean":"2023-09-23T21:15:25.904811+0000","last_became_active":"2023-09-23T21:15:25.902151+0000","last_became_peered":"2023-09-23T21:15:25.902151+0000","last_unstale":"2023-09-23T21:15:25.904811+0000","last_undegraded":"2023-09-23T21:15:25.904811+0000","last_fullsized":"2023-09-23T21:15:25.904811+0000","mapping_epoch":544,"log_start":"0'0","ondisk_log_start":"0'0","created":544,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_clean_scrub_stamp":"2023-09-23T21:15:24.860968+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:32:45.174211+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,0],"acting":[2,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.5","version":"79'161","reported_seq":573,"reported_epoch":545,"state":"active+clean","last_fresh":"2023-09-23T21:15:25.884579+0000","last_change":"2023-09-23T21:13:38.373969+0000","last_active":"2023-09-23T21:15:25.884579+0000","last_peered":"2023-09-23T21:15:25.884579+0000","last_clean":"2023-09-23T21:15:25.884579+0000","last_became_active":"2023-09-23T21:13:38.373633+0000","last_became_peered":"2023-09-23T21:13:38.373633+0000","last_unstale":"2023-09-23T21:15:25.884579+0000","last_undegraded":"2023-09-23T21:15:25.884579+0000","last_fullsized":"2023-09-23T21:15:25.884579+0000","mapping_epoch":507,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":3,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:09:34.892947+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:09:34.892947+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13417,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":11,"num_read_kb":20,"num_write":5,"num_write_kb":55,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":40252,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":85,"reported_epoch":543,"state":"active+clean","last_fresh":"2023-09-23T21:15:23.792190+0000","last_change":"2023-09-23T21:13:12.235743+0000","last_active":"2023-09-23T21:15:23.792190+0000","last_peered":"2023-09-23T21:15:23.792190+0000","last_clean":"2023-09-23T21:15:23.792190+0000","last_became_active":"2023-09-23T21:13:12.216241+0000","last_became_peered":"2023-09-23T21:13:12.216241+0000","last_unstale":"2023-09-23T21:15:23.792190+0000","last_undegraded":"2023-09-23T21:15:23.792190+0000","last_fullsized":"2023-09-23T21:15:23.792190+0000","mapping_epoch":493,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:12:19.090710+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":526,"reported_epoch":545,"state":"active+clean","last_fresh":"2023-09-23T21:15:25.885983+0000","last_change":"2023-09-23T21:13:30.807520+0000","last_active":"2023-09-23T21:15:25.885983+0000","last_peered":"2023-09-23T21:15:25.885983+0000","last_clean":"2023-09-23T21:15:25.885983+0000","last_became_active":"2023-09-23T21:13:30.807222+0000","last_became_peered":"2023-09-23T21:13:30.807222+0000","last_unstale":"2023-09-23T21:15:25.885983+0000","last_undegraded":"2023-09-23T21:15:25.885983+0000","last_fullsized":"2023-09-23T21:15:25.885983+0000","mapping_epoch":501,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:40:31.989749+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":384,"reported_epoch":542,"state":"active+clean","last_fresh":"2023-09-23T21:15:22.798944+0000","last_change":"2023-09-23T21:13:30.794095+0000","last_active":"2023-09-23T21:15:22.798944+0000","last_peered":"2023-09-23T21:15:22.798944+0000","last_clean":"2023-09-23T21:15:22.798944+0000","last_became_active":"2023-09-23T21:13:30.793791+0000","last_became_peered":"2023-09-23T21:13:30.793791+0000","last_unstale":"2023-09-23T21:15:22.798944+0000","last_undegraded":"2023-09-23T21:15:22.798944+0000","last_fullsized":"2023-09-23T21:15:22.798944+0000","mapping_epoch":501,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:36:28.707362+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.e","version":"0'0","reported_seq":10,"reported_epoch":545,"state":"active+clean","last_fresh":"2023-09-23T21:15:25.903201+0000","last_change":"2023-09-23T21:15:25.902844+0000","last_active":"2023-09-23T21:15:25.903201+0000","last_peered":"2023-09-23T21:15:25.903201+0000","last_clean":"2023-09-23T21:15:25.903201+0000","last_became_active":"2023-09-23T21:15:25.902523+0000","last_became_peered":"2023-09-23T21:15:25.902523+0000","last_unstale":"2023-09-23T21:15:25.903201+0000","last_undegraded":"2023-09-23T21:15:25.903201+0000","last_fullsized":"2023-09-23T21:15:25.903201+0000","mapping_epoch":544,"log_start":"0'0","ondisk_log_start":"0'0","created":544,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_clean_scrub_stamp":"2023-09-23T21:15:24.860968+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:34:03.916540+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.4","version":"79'161","reported_seq":914,"reported_epoch":542,"state":"active+clean","last_fresh":"2023-09-23T21:15:22.801045+0000","last_change":"2023-09-23T21:10:29.807785+0000","last_active":"2023-09-23T21:15:22.801045+0000","last_peered":"2023-09-23T21:15:22.801045+0000","last_clean":"2023-09-23T21:15:22.801045+0000","last_became_active":"2023-09-23T21:10:29.807366+0000","last_became_peered":"2023-09-23T21:10:29.807366+0000","last_unstale":"2023-09-23T21:15:22.801045+0000","last_undegraded":"2023-09-23T21:15:22.801045+0000","last_fullsized":"2023-09-23T21:15:22.801045+0000","mapping_epoch":426,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":427,"parent":"0.0","parent_split_bits":3,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:20.012764+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:20.012764+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:24:45.763834+0000","scrub_duration":0.0045251919999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":484,"reported_epoch":544,"state":"active+clean","last_fresh":"2023-09-23T21:15:24.873913+0000","last_change":"2023-09-23T21:10:26.768364+0000","last_active":"2023-09-23T21:15:24.873913+0000","last_peered":"2023-09-23T21:15:24.873913+0000","last_clean":"2023-09-23T21:15:24.873913+0000","last_became_active":"2023-09-23T21:10:26.768052+0000","last_became_peered":"2023-09-23T21:10:26.768052+0000","last_unstale":"2023-09-23T21:15:24.873913+0000","last_undegraded":"2023-09-23T21:15:24.873913+0000","last_fullsized":"2023-09-23T21:15:24.873913+0000","mapping_epoch":423,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":424,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:22:42.201200+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,3],"acting":[2,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.6","version":"70'1","reported_seq":514,"reported_epoch":543,"state":"active+clean","last_fresh":"2023-09-23T21:15:23.792189+0000","last_change":"2023-09-23T21:13:30.793290+0000","last_active":"2023-09-23T21:15:23.792189+0000","last_peered":"2023-09-23T21:15:23.792189+0000","last_clean":"2023-09-23T21:15:23.792189+0000","last_became_active":"2023-09-23T21:13:30.786079+0000","last_became_peered":"2023-09-23T21:13:30.786079+0000","last_unstale":"2023-09-23T21:15:23.792189+0000","last_undegraded":"2023-09-23T21:15:23.792189+0000","last_fullsized":"2023-09-23T21:15:23.792189+0000","mapping_epoch":501,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:34:43.281095+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":120,"reported_epoch":545,"state":"active+clean","last_fresh":"2023-09-23T21:15:25.883970+0000","last_change":"2023-09-23T21:13:12.216895+0000","last_active":"2023-09-23T21:15:25.883970+0000","last_peered":"2023-09-23T21:15:25.883970+0000","last_clean":"2023-09-23T21:15:25.883970+0000","last_became_active":"2023-09-23T21:13:12.216654+0000","last_became_peered":"2023-09-23T21:13:12.216654+0000","last_unstale":"2023-09-23T21:15:25.883970+0000","last_undegraded":"2023-09-23T21:15:25.883970+0000","last_fullsized":"2023-09-23T21:15:25.883970+0000","mapping_epoch":493,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:02:40.430096+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.19","version":"79'161","reported_seq":705,"reported_epoch":543,"state":"active+clean","last_fresh":"2023-09-23T21:15:23.796174+0000","last_change":"2023-09-23T21:15:23.795956+0000","last_active":"2023-09-23T21:15:23.796174+0000","last_peered":"2023-09-23T21:15:23.796174+0000","last_clean":"2023-09-23T21:15:23.796174+0000","last_became_active":"2023-09-23T21:15:23.795663+0000","last_became_peered":"2023-09-23T21:15:23.795663+0000","last_unstale":"2023-09-23T21:15:23.796174+0000","last_undegraded":"2023-09-23T21:15:23.796174+0000","last_fullsized":"2023-09-23T21:15:23.796174+0000","mapping_epoch":542,"log_start":"0'0","ondisk_log_start":"0'0","created":479,"last_epoch_clean":543,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:10:37.696951+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:10:37.696951+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.001386504,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":53,"reported_epoch":543,"state":"active+clean","last_fresh":"2023-09-23T21:15:23.791931+0000","last_change":"2023-09-23T21:13:30.791297+0000","last_active":"2023-09-23T21:15:23.791931+0000","last_peered":"2023-09-23T21:15:23.791931+0000","last_clean":"2023-09-23T21:15:23.791931+0000","last_became_active":"2023-09-23T21:13:30.788721+0000","last_became_peered":"2023-09-23T21:13:30.788721+0000","last_unstale":"2023-09-23T21:15:23.791931+0000","last_undegraded":"2023-09-23T21:15:23.791931+0000","last_fullsized":"2023-09-23T21:15:23.791931+0000","mapping_epoch":501,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:31:18.147883+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":48,"reported_epoch":544,"state":"active+clean","last_fresh":"2023-09-23T21:15:24.872980+0000","last_change":"2023-09-23T21:13:38.373814+0000","last_active":"2023-09-23T21:15:24.872980+0000","last_peered":"2023-09-23T21:15:24.872980+0000","last_clean":"2023-09-23T21:15:24.872980+0000","last_became_active":"2023-09-23T21:13:38.373458+0000","last_became_peered":"2023-09-23T21:13:38.373458+0000","last_unstale":"2023-09-23T21:15:24.872980+0000","last_undegraded":"2023-09-23T21:15:24.872980+0000","last_fullsized":"2023-09-23T21:15:24.872980+0000","mapping_epoch":507,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:23:08.288401+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.1e","version":"79'161","reported_seq":479,"reported_epoch":543,"state":"active+clean","last_fresh":"2023-09-23T21:15:23.794465+0000","last_change":"2023-09-23T21:15:20.791055+0000","last_active":"2023-09-23T21:15:23.794465+0000","last_peered":"2023-09-23T21:15:23.794465+0000","last_clean":"2023-09-23T21:15:23.794465+0000","last_became_active":"2023-09-23T21:15:20.790772+0000","last_became_peered":"2023-09-23T21:15:20.790772+0000","last_unstale":"2023-09-23T21:15:23.794465+0000","last_undegraded":"2023-09-23T21:15:23.794465+0000","last_fullsized":"2023-09-23T21:15:23.794465+0000","mapping_epoch":539,"log_start":"0'0","ondisk_log_start":"0'0","created":479,"last_epoch_clean":540,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:10:39.567232+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:10:39.567232+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.024789746000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,4],"acting":[3,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":127,"reported_epoch":545,"state":"active+clean","last_fresh":"2023-09-23T21:15:25.884845+0000","last_change":"2023-09-23T21:15:06.721854+0000","last_active":"2023-09-23T21:15:25.884845+0000","last_peered":"2023-09-23T21:15:25.884845+0000","last_clean":"2023-09-23T21:15:25.884845+0000","last_became_active":"2023-09-23T21:12:14.209408+0000","last_became_peered":"2023-09-23T21:12:14.209408+0000","last_unstale":"2023-09-23T21:15:25.884845+0000","last_undegraded":"2023-09-23T21:15:25.884845+0000","last_fullsized":"2023-09-23T21:15:25.884845+0000","mapping_epoch":470,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":532,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:45:40.142413+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,7],"acting":[4,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":110,"reported_epoch":545,"state":"active+clean","last_fresh":"2023-09-23T21:15:25.884377+0000","last_change":"2023-09-23T21:12:10.770474+0000","last_active":"2023-09-23T21:15:25.884377+0000","last_peered":"2023-09-23T21:15:25.884377+0000","last_clean":"2023-09-23T21:15:25.884377+0000","last_became_active":"2023-09-23T21:12:10.770184+0000","last_became_peered":"2023-09-23T21:12:10.770184+0000","last_unstale":"2023-09-23T21:15:25.884377+0000","last_undegraded":"2023-09-23T21:15:25.884377+0000","last_fullsized":"2023-09-23T21:15:25.884377+0000","mapping_epoch":470,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:05:09.407151+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":335,"reported_epoch":544,"state":"active+clean","last_fresh":"2023-09-23T21:15:24.873174+0000","last_change":"2023-09-23T21:12:10.771160+0000","last_active":"2023-09-23T21:15:24.873174+0000","last_peered":"2023-09-23T21:15:24.873174+0000","last_clean":"2023-09-23T21:15:24.873174+0000","last_became_active":"2023-09-23T21:12:10.757808+0000","last_became_peered":"2023-09-23T21:12:10.757808+0000","last_unstale":"2023-09-23T21:15:24.873174+0000","last_undegraded":"2023-09-23T21:15:24.873174+0000","last_fullsized":"2023-09-23T21:15:24.873174+0000","mapping_epoch":470,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:30:50.987139+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":120,"reported_epoch":545,"state":"active+clean","last_fresh":"2023-09-23T21:15:25.884404+0000","last_change":"2023-09-23T21:13:12.216602+0000","last_active":"2023-09-23T21:15:25.884404+0000","last_peered":"2023-09-23T21:15:25.884404+0000","last_clean":"2023-09-23T21:15:25.884404+0000","last_became_active":"2023-09-23T21:13:12.216204+0000","last_became_peered":"2023-09-23T21:13:12.216204+0000","last_unstale":"2023-09-23T21:15:25.884404+0000","last_undegraded":"2023-09-23T21:15:25.884404+0000","last_fullsized":"2023-09-23T21:15:25.884404+0000","mapping_epoch":493,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:23:37.043390+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1c","version":"79'161","reported_seq":926,"reported_epoch":543,"state":"active+clean","last_fresh":"2023-09-23T21:15:23.800688+0000","last_change":"2023-09-23T21:15:23.800688+0000","last_active":"2023-09-23T21:15:23.800688+0000","last_peered":"2023-09-23T21:15:23.800688+0000","last_clean":"2023-09-23T21:15:23.800688+0000","last_became_active":"2023-09-23T21:15:23.800361+0000","last_became_peered":"2023-09-23T21:15:23.800361+0000","last_unstale":"2023-09-23T21:15:23.800688+0000","last_undegraded":"2023-09-23T21:15:23.800688+0000","last_fullsized":"2023-09-23T21:15:23.800688+0000","mapping_epoch":542,"log_start":"0'0","ondisk_log_start":"0'0","created":479,"last_epoch_clean":543,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:22.040907+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:22.040907+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.013021176000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":163840,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":163840,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":134,"reported_epoch":544,"state":"active+clean","last_fresh":"2023-09-23T21:15:24.873015+0000","last_change":"2023-09-23T21:12:10.779066+0000","last_active":"2023-09-23T21:15:24.873015+0000","last_peered":"2023-09-23T21:15:24.873015+0000","last_clean":"2023-09-23T21:15:24.873015+0000","last_became_active":"2023-09-23T21:12:10.778691+0000","last_became_peered":"2023-09-23T21:12:10.778691+0000","last_unstale":"2023-09-23T21:15:24.873015+0000","last_undegraded":"2023-09-23T21:15:24.873015+0000","last_fullsized":"2023-09-23T21:15:24.873015+0000","mapping_epoch":470,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:07:22.043268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,7],"acting":[2,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":213,"reported_epoch":545,"state":"active+clean","last_fresh":"2023-09-23T21:15:25.883564+0000","last_change":"2023-09-23T21:13:12.200767+0000","last_active":"2023-09-23T21:15:25.883564+0000","last_peered":"2023-09-23T21:15:25.883564+0000","last_clean":"2023-09-23T21:15:25.883564+0000","last_became_active":"2023-09-23T21:13:12.200435+0000","last_became_peered":"2023-09-23T21:13:12.200435+0000","last_unstale":"2023-09-23T21:15:25.883564+0000","last_undegraded":"2023-09-23T21:15:25.883564+0000","last_fullsized":"2023-09-23T21:15:25.883564+0000","mapping_epoch":493,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:15:44.937886+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.1d","version":"79'161","reported_seq":674,"reported_epoch":543,"state":"active+clean","last_fresh":"2023-09-23T21:15:23.802388+0000","last_change":"2023-09-23T21:15:23.802125+0000","last_active":"2023-09-23T21:15:23.802388+0000","last_peered":"2023-09-23T21:15:23.802388+0000","last_clean":"2023-09-23T21:15:23.802388+0000","last_became_active":"2023-09-23T21:15:23.801704+0000","last_became_peered":"2023-09-23T21:15:23.801704+0000","last_unstale":"2023-09-23T21:15:23.802388+0000","last_undegraded":"2023-09-23T21:15:23.802388+0000","last_fullsized":"2023-09-23T21:15:23.802388+0000","mapping_epoch":542,"log_start":"0'0","ondisk_log_start":"0'0","created":479,"last_epoch_clean":543,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:09:36.858077+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:09:36.858077+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.010304610000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,2],"acting":[3,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":70,"reported_epoch":543,"state":"active+clean","last_fresh":"2023-09-23T21:15:23.791986+0000","last_change":"2023-09-23T21:13:38.380574+0000","last_active":"2023-09-23T21:15:23.791986+0000","last_peered":"2023-09-23T21:15:23.791986+0000","last_clean":"2023-09-23T21:15:23.791986+0000","last_became_active":"2023-09-23T21:13:38.373252+0000","last_became_peered":"2023-09-23T21:13:38.373252+0000","last_unstale":"2023-09-23T21:15:23.791986+0000","last_undegraded":"2023-09-23T21:15:23.791986+0000","last_fullsized":"2023-09-23T21:15:23.791986+0000","mapping_epoch":507,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:58:14.788031+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":11,"num_bytes_recovered":3324,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":120,"reported_epoch":545,"state":"active+clean","last_fresh":"2023-09-23T21:15:25.883906+0000","last_change":"2023-09-23T21:12:10.793610+0000","last_active":"2023-09-23T21:15:25.883906+0000","last_peered":"2023-09-23T21:15:25.883906+0000","last_clean":"2023-09-23T21:15:25.883906+0000","last_became_active":"2023-09-23T21:12:10.793081+0000","last_became_peered":"2023-09-23T21:12:10.793081+0000","last_unstale":"2023-09-23T21:15:25.883906+0000","last_undegraded":"2023-09-23T21:15:25.883906+0000","last_fullsized":"2023-09-23T21:15:25.883906+0000","mapping_epoch":470,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:34:36.291550+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":11,"num_pg":16,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":36,"acting":36,"num_store_stats":5},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":60,"num_bytes_recovered":8898,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":118784,"data_stored":21350,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":14322,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":31,"stat_sum":{"num_bytes":684290,"num_objects":3,"num_object_clones":0,"num_object_copies":9,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3,"num_whiteouts":0,"num_read":78,"num_read_kb":144,"num_write":38,"num_write_kb":385,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1199814,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2109440,"data_stored":2098784,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4991,"ondisk_log_size":4991,"up":93,"acting":93,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201700,"num_pgs":63,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":514572,"kb_used_data":5956,"kb_used_omap":0,"kb_used_meta":508608,"kb_avail":93230580,"statfs":{"total":95995035648,"available":95468113920,"internally_reserved":0,"allocated":6098944,"data_stored":4526745,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":520814592},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:15:15 2023","interfaces":[{"interface":"back","average":{"1min":1.3440000000000001,"5min":2.2090000000000001,"15min":2.3530000000000002},"min":{"1min":0.25,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":4.3070000000000004,"5min":37.207999999999998,"15min":37.207999999999998},"last":0.373},{"interface":"front","average":{"1min":2.089,"5min":2.2050000000000001,"15min":2.2240000000000002},"min":{"1min":0.253,"5min":0.253,"15min":0.253},"max":{"1min":10.429,"5min":37.122,"15min":37.122},"last":0.47199999999999998}]},{"osd":1,"last update":"Sat Sep 23 21:15:09 2023","interfaces":[{"interface":"back","average":{"1min":2.0329999999999999,"5min":1.3320000000000001,"15min":2.0289999999999999},"min":{"1min":0.28000000000000003,"5min":0.27500000000000002,"15min":0.26700000000000002},"max":{"1min":6.0860000000000003,"5min":14.034000000000001,"15min":14.034000000000001},"last":0.39700000000000002},{"interface":"front","average":{"1min":1.554,"5min":1.4359999999999999,"15min":1.9079999999999999},"min":{"1min":0.33900000000000002,"5min":0.29899999999999999,"15min":0.26300000000000001},"max":{"1min":4.2690000000000001,"5min":11.226000000000001,"15min":11.226000000000001},"last":0.60299999999999998}]},{"osd":2,"last update":"Sat Sep 23 21:14:30 2023","interfaces":[{"interface":"back","average":{"1min":0.80900000000000005,"5min":1.0880000000000001,"15min":1.7170000000000001},"min":{"1min":0.29699999999999999,"5min":0.26400000000000001,"15min":0.26200000000000001},"max":{"1min":3.0190000000000001,"5min":6.4219999999999997,"15min":14.478999999999999},"last":0.34799999999999998},{"interface":"front","average":{"1min":1.101,"5min":1.3520000000000001,"15min":1.7330000000000001},"min":{"1min":0.30399999999999999,"5min":0.26700000000000002,"15min":0.26500000000000001},"max":{"1min":5.7640000000000002,"5min":11.198,"15min":14.135},"last":0.53700000000000003}]},{"osd":3,"last update":"Sat Sep 23 21:14:58 2023","interfaces":[{"interface":"back","average":{"1min":1.7270000000000001,"5min":1.3009999999999999,"15min":2},"min":{"1min":0.30599999999999999,"5min":0.30599999999999999,"15min":0.29099999999999998},"max":{"1min":4.1619999999999999,"5min":7.5190000000000001,"15min":31.786000000000001},"last":0.441},{"interface":"front","average":{"1min":2.7410000000000001,"5min":1.4890000000000001,"15min":2.0390000000000001},"min":{"1min":0.35599999999999998,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":11.743,"5min":11.743,"15min":32.429000000000002},"last":0.56399999999999995}]},{"osd":4,"last update":"Sat Sep 23 21:14:58 2023","interfaces":[{"interface":"back","average":{"1min":1.1100000000000001,"5min":1.48,"15min":1.143},"min":{"1min":0.32500000000000001,"5min":0.26200000000000001,"15min":0.248},"max":{"1min":4.2130000000000001,"5min":11.297000000000001,"15min":11.297000000000001},"last":0.66600000000000004},{"interface":"front","average":{"1min":1.085,"5min":1.24,"15min":1.105},"min":{"1min":0.29699999999999999,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":3.7850000000000001,"5min":11.093999999999999,"15min":12.752000000000001},"last":0.41499999999999998}]},{"osd":5,"last update":"Sat Sep 23 21:14:58 2023","interfaces":[{"interface":"back","average":{"1min":1.359,"5min":1.1990000000000001,"15min":1.075},"min":{"1min":0.32300000000000001,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":6.133,"5min":9.4789999999999992,"15min":9.4789999999999992},"last":0.58799999999999997},{"interface":"front","average":{"1min":0.98699999999999999,"5min":1.3360000000000001,"15min":1.1279999999999999},"min":{"1min":0.33300000000000002,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":4.181,"5min":11.257,"15min":11.257},"last":0.63800000000000001}]},{"osd":6,"last update":"Sat Sep 23 21:15:09 2023","interfaces":[{"interface":"back","average":{"1min":1.325,"5min":1.3029999999999999,"15min":1.095},"min":{"1min":0.40200000000000002,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":7.5339999999999998,"5min":10.941000000000001,"15min":10.941000000000001},"last":0.61499999999999999},{"interface":"front","average":{"1min":1.298,"5min":1.2130000000000001,"15min":1.1799999999999999},"min":{"1min":0.39200000000000002,"5min":0.33100000000000002,"15min":0.32300000000000001},"max":{"1min":4.7720000000000002,"5min":8.923,"15min":12.715},"last":0.65100000000000002}]}]},{"osd":6,"up_from":0,"seq":704374636783,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":1,"up_from":0,"seq":1503238553716,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":0,"up_from":493,"seq":2117418876956,"num_pgs":71,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":707704,"kb_used_data":517424,"kb_used_omap":36,"kb_used_meta":190235,"kb_avail":93037448,"statfs":{"total":95995035648,"available":95270346752,"internally_reserved":0,"allocated":529842176,"data_stored":527303249,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":37336,"internal_metadata":194801192},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 21:15:17 2023","interfaces":[{"interface":"back","average":{"1min":1.6990000000000001,"5min":0.94399999999999995,"15min":0.81799999999999995},"min":{"1min":0.22500000000000001,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":4.665,"5min":4.9000000000000004,"15min":4.9000000000000004},"last":0.27000000000000002},{"interface":"front","average":{"1min":1.5620000000000001,"5min":0.92900000000000005,"15min":0.82299999999999995},"min":{"1min":0.20799999999999999,"5min":0.20799999999999999,"15min":0.20799999999999999},"max":{"1min":11.964,"5min":11.964,"15min":11.964},"last":0.37}]},{"osd":2,"last update":"Sat Sep 23 21:15:17 2023","interfaces":[{"interface":"back","average":{"1min":1.661,"5min":0.85499999999999998,"15min":0.72099999999999997},"min":{"1min":0.29399999999999998,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":7.133,"5min":7.133,"15min":7.133},"last":0.39900000000000002},{"interface":"front","average":{"1min":1.647,"5min":1.103,"15min":1.0129999999999999},"min":{"1min":0.27900000000000003,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":6.6470000000000002,"5min":6.6470000000000002,"15min":6.6470000000000002},"last":0.28599999999999998}]},{"osd":3,"last update":"Sat Sep 23 21:15:17 2023","interfaces":[{"interface":"back","average":{"1min":1.8939999999999999,"5min":1.073,"15min":0.93600000000000005},"min":{"1min":0.27300000000000002,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":6.2060000000000004,"5min":6.2060000000000004,"15min":6.2060000000000004},"last":0.32300000000000001},{"interface":"front","average":{"1min":1.4199999999999999,"5min":1.1779999999999999,"15min":1.1379999999999999},"min":{"1min":0.28299999999999997,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":4.8040000000000003,"5min":4.8040000000000003,"15min":4.8040000000000003},"last":0.35099999999999998}]},{"osd":4,"last update":"Sat Sep 23 21:15:17 2023","interfaces":[{"interface":"back","average":{"1min":1.2789999999999999,"5min":1.1299999999999999,"15min":1.105},"min":{"1min":0.312,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":4.4409999999999998,"5min":4.4409999999999998,"15min":4.4409999999999998},"last":0.53200000000000003},{"interface":"front","average":{"1min":1.4570000000000001,"5min":1.0880000000000001,"15min":1.026},"min":{"1min":0.41699999999999998,"5min":0.317,"15min":0.317},"max":{"1min":4.6929999999999996,"5min":5.093,"15min":5.093},"last":0.501}]},{"osd":5,"last update":"Sat Sep 23 21:15:17 2023","interfaces":[{"interface":"back","average":{"1min":1.413,"5min":1.2709999999999999,"15min":1.2470000000000001},"min":{"1min":0.29999999999999999,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":4.0869999999999997,"5min":4.5640000000000001,"15min":4.5640000000000001},"last":0.52400000000000002},{"interface":"front","average":{"1min":1.3959999999999999,"5min":1.1299999999999999,"15min":1.0860000000000001},"min":{"1min":0.313,"5min":0.313,"15min":0.313},"max":{"1min":7.4569999999999999,"5min":7.4569999999999999,"15min":7.4569999999999999},"last":0.434}]},{"osd":6,"last update":"Sat Sep 23 21:15:17 2023","interfaces":[{"interface":"back","average":{"1min":1.8420000000000001,"5min":1.27,"15min":1.1739999999999999},"min":{"1min":0.46600000000000003,"5min":0.38100000000000001,"15min":0.38100000000000001},"max":{"1min":7.2050000000000001,"5min":7.2050000000000001,"15min":7.2050000000000001},"last":0.45100000000000001},{"interface":"front","average":{"1min":1.5640000000000001,"5min":1.359,"15min":1.325},"min":{"1min":0.39600000000000002,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":4.7850000000000001,"5min":5.0860000000000003,"15min":5.0860000000000003},"last":0.55400000000000005}]},{"osd":7,"last update":"Sat Sep 23 21:15:17 2023","interfaces":[{"interface":"back","average":{"1min":2.1600000000000001,"5min":1.399,"15min":1.272},"min":{"1min":0.40899999999999997,"5min":0.40899999999999997,"15min":0.40899999999999997},"max":{"1min":7.5810000000000004,"5min":7.5810000000000004,"15min":7.5810000000000004},"last":0.48599999999999999},{"interface":"front","average":{"1min":1.6850000000000001,"5min":1.2629999999999999,"15min":1.1930000000000001},"min":{"1min":0.56000000000000005,"5min":0.441,"15min":0.441},"max":{"1min":4.6349999999999998,"5min":5.1289999999999996,"15min":5.1289999999999996},"last":0.47199999999999998}]}]},{"osd":2,"up_from":93,"seq":399431958828,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":984512,"kb_used_data":191032,"kb_used_omap":81,"kb_used_meta":793390,"kb_avail":92760640,"statfs":{"total":95995035648,"available":94986895360,"internally_reserved":0,"allocated":195616768,"data_stored":193068762,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":83750,"internal_metadata":812431578},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:15:17 2023","interfaces":[{"interface":"back","average":{"1min":2.2210000000000001,"5min":1.1100000000000001,"15min":0.92500000000000004},"min":{"1min":0.20899999999999999,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":9.9329999999999998,"5min":9.9329999999999998,"15min":9.9329999999999998},"last":0.442},{"interface":"front","average":{"1min":2.6659999999999999,"5min":1.865,"15min":1.7310000000000001},"min":{"1min":0.24399999999999999,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":11.269,"5min":18.111999999999998,"15min":18.111999999999998},"last":0.46800000000000003}]},{"osd":1,"last update":"Sat Sep 23 21:15:17 2023","interfaces":[{"interface":"back","average":{"1min":2.0539999999999998,"5min":1.2629999999999999,"15min":2.472},"min":{"1min":0.20599999999999999,"5min":0.188,"15min":0.188},"max":{"1min":9.9890000000000008,"5min":11.005000000000001,"15min":34.396999999999998},"last":0.49199999999999999},{"interface":"front","average":{"1min":2.2280000000000002,"5min":1.296,"15min":2.6859999999999999},"min":{"1min":0.26200000000000001,"5min":0.217,"15min":0.217},"max":{"1min":16.274000000000001,"5min":18.081,"15min":34.462000000000003},"last":0.42399999999999999}]},{"osd":3,"last update":"Sat Sep 23 21:14:58 2023","interfaces":[{"interface":"back","average":{"1min":2.246,"5min":1.274,"15min":2.4550000000000001},"min":{"1min":0.37,"5min":0.20599999999999999,"15min":0.20599999999999999},"max":{"1min":14.442,"5min":14.442,"15min":41.057000000000002},"last":0.45500000000000002},{"interface":"front","average":{"1min":2.633,"5min":1.4419999999999999,"15min":2.6970000000000001},"min":{"1min":0.28599999999999998,"5min":0.21299999999999999,"15min":0.21299999999999999},"max":{"1min":14.622,"5min":14.622,"15min":41.012},"last":0.626}]},{"osd":4,"last update":"Sat Sep 23 21:15:17 2023","interfaces":[{"interface":"back","average":{"1min":1.5780000000000001,"5min":1.2749999999999999,"15min":1.4219999999999999},"min":{"1min":0.32500000000000001,"5min":0.249,"15min":0.249},"max":{"1min":7.9770000000000003,"5min":10.266999999999999,"15min":10.266999999999999},"last":0.71599999999999997},{"interface":"front","average":{"1min":1.74,"5min":1.4419999999999999,"15min":1.8029999999999999},"min":{"1min":0.308,"5min":0.29399999999999998,"15min":0.26700000000000002},"max":{"1min":5.556,"5min":7.5960000000000001,"15min":14.039999999999999},"last":0.80300000000000005}]},{"osd":5,"last update":"Sat Sep 23 21:15:17 2023","interfaces":[{"interface":"back","average":{"1min":1.1850000000000001,"5min":1.218,"15min":1.5049999999999999},"min":{"1min":0.34699999999999998,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":3.7250000000000001,"5min":9.5039999999999996,"15min":10.069000000000001},"last":0.73699999999999999},{"interface":"front","average":{"1min":1.6779999999999999,"5min":1.2849999999999999,"15min":1.7529999999999999},"min":{"1min":0.33800000000000002,"5min":0.29699999999999999,"15min":0.29399999999999998},"max":{"1min":5.8330000000000002,"5min":7.5549999999999997,"15min":13.98},"last":0.90800000000000003}]},{"osd":6,"last update":"Sat Sep 23 21:15:17 2023","interfaces":[{"interface":"back","average":{"1min":1.9239999999999999,"5min":1.571,"15min":1.988},"min":{"1min":0.46300000000000002,"5min":0.373,"15min":0.35699999999999998},"max":{"1min":7.8559999999999999,"5min":7.8559999999999999,"15min":14.016},"last":0.85399999999999998},{"interface":"front","average":{"1min":1.7330000000000001,"5min":1.7330000000000001,"15min":2.157},"min":{"1min":0.377,"5min":0.377,"15min":0.35599999999999998},"max":{"1min":5.649,"5min":9.3469999999999995,"15min":14.068},"last":0.94499999999999995}]},{"osd":7,"last update":"Sat Sep 23 21:15:17 2023","interfaces":[{"interface":"back","average":{"1min":1.6839999999999999,"5min":1.462,"15min":1.6040000000000001},"min":{"1min":0.33700000000000002,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":6.0510000000000002,"5min":18.021999999999998,"15min":18.021999999999998},"last":0.871},{"interface":"front","average":{"1min":1.359,"5min":1.425,"15min":1.6910000000000001},"min":{"1min":0.41899999999999998,"5min":0.34799999999999998,"15min":0.32200000000000001},"max":{"1min":3.7549999999999999,"5min":10.359,"15min":10.92},"last":0.79300000000000004}]}]},{"osd":3,"up_from":98,"seq":420906795301,"num_pgs":65,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1391892,"kb_used_data":509388,"kb_used_omap":7,"kb_used_meta":882488,"kb_avail":92353260,"statfs":{"total":95995035648,"available":94569738240,"internally_reserved":0,"allocated":521613312,"data_stored":519095138,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7576,"internal_metadata":903668328},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":4,"apply_latency_ms":4,"commit_latency_ns":4000000,"apply_latency_ns":4000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:15:13 2023","interfaces":[{"interface":"back","average":{"1min":10.904,"5min":2.4510000000000001,"15min":1.042},"min":{"1min":0.28499999999999998,"5min":0.20499999999999999,"15min":0.20499999999999999},"max":{"1min":147.38900000000001,"5min":147.38900000000001,"15min":147.38900000000001},"last":0.45800000000000002},{"interface":"front","average":{"1min":10.178000000000001,"5min":2.5699999999999998,"15min":1.302},"min":{"1min":0.23100000000000001,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":149.989,"5min":149.989,"15min":149.989},"last":0.34799999999999998}]},{"osd":1,"last update":"Sat Sep 23 21:15:13 2023","interfaces":[{"interface":"back","average":{"1min":1.5589999999999999,"5min":1.2,"15min":2.1880000000000002},"min":{"1min":0.23699999999999999,"5min":0.20000000000000001,"15min":0.20000000000000001},"max":{"1min":5.0700000000000003,"5min":22.577999999999999,"15min":22.577999999999999},"last":0.32200000000000001},{"interface":"front","average":{"1min":1.5600000000000001,"5min":1.345,"15min":2.1389999999999998},"min":{"1min":0.26100000000000001,"5min":0.216,"15min":0.216},"max":{"1min":5.7329999999999997,"5min":21.617999999999999,"15min":21.617999999999999},"last":0.51500000000000001}]},{"osd":2,"last update":"Sat Sep 23 21:15:00 2023","interfaces":[{"interface":"back","average":{"1min":1.78,"5min":1.9419999999999999,"15min":2.468},"min":{"1min":0.23899999999999999,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":10.946,"5min":56.317999999999998,"15min":56.317999999999998},"last":0.70599999999999996},{"interface":"front","average":{"1min":1.554,"5min":1.978,"15min":2.5609999999999999},"min":{"1min":0.246,"5min":0.216,"15min":0.216},"max":{"1min":6.29,"5min":55.988,"15min":55.988},"last":0.64400000000000002}]},{"osd":4,"last update":"Sat Sep 23 21:15:00 2023","interfaces":[{"interface":"back","average":{"1min":2.2959999999999998,"5min":1.526,"15min":1.831},"min":{"1min":0.29899999999999999,"5min":0.29899999999999999,"15min":0.28399999999999997},"max":{"1min":10.807,"5min":21.478999999999999,"15min":21.478999999999999},"last":0.73599999999999999},{"interface":"front","average":{"1min":1.415,"5min":1.4830000000000001,"15min":1.9710000000000001},"min":{"1min":0.314,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":6.3499999999999996,"5min":22.402000000000001,"15min":22.402000000000001},"last":0.68799999999999994}]},{"osd":5,"last update":"Sat Sep 23 21:15:00 2023","interfaces":[{"interface":"back","average":{"1min":1.925,"5min":1.456,"15min":1.95},"min":{"1min":0.34599999999999997,"5min":0.311,"15min":0.311},"max":{"1min":10.882999999999999,"5min":21.562000000000001,"15min":21.562000000000001},"last":0.66700000000000004},{"interface":"front","average":{"1min":1.429,"5min":1.302,"15min":2.141},"min":{"1min":0.34799999999999998,"5min":0.31,"15min":0.31},"max":{"1min":4.2779999999999996,"5min":8.8450000000000006,"15min":14.029},"last":0.56000000000000005}]},{"osd":6,"last update":"Sat Sep 23 21:15:17 2023","interfaces":[{"interface":"back","average":{"1min":1.194,"5min":1.3919999999999999,"15min":1.9930000000000001},"min":{"1min":0.35899999999999999,"5min":0.34999999999999998,"15min":0.33900000000000002},"max":{"1min":3.536,"5min":22.446999999999999,"15min":22.446999999999999},"last":0.58199999999999996},{"interface":"front","average":{"1min":1.518,"5min":1.7969999999999999,"15min":2.1230000000000002},"min":{"1min":0.378,"5min":0.378,"15min":0.373},"max":{"1min":4.3760000000000003,"5min":12.375999999999999,"15min":13.576000000000001},"last":0.628}]},{"osd":7,"last update":"Sat Sep 23 21:15:00 2023","interfaces":[{"interface":"back","average":{"1min":2.4289999999999998,"5min":1.8480000000000001,"15min":2.1000000000000001},"min":{"1min":0.439,"5min":0.38800000000000001,"15min":0.33900000000000002},"max":{"1min":10.914999999999999,"5min":21.518999999999998,"15min":21.518999999999998},"last":0.61299999999999999},{"interface":"front","average":{"1min":2.4220000000000002,"5min":1.903,"15min":2.3380000000000001},"min":{"1min":0.42499999999999999,"5min":0.42499999999999999,"15min":0.38900000000000001},"max":{"1min":8.2590000000000003,"5min":9.9879999999999995,"15min":13.603999999999999},"last":0.59899999999999998}]}]},{"osd":4,"up_from":31,"seq":133143986608,"num_pgs":57,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":587784,"kb_used_data":103552,"kb_used_omap":0,"kb_used_meta":484224,"kb_avail":93157368,"statfs":{"total":95995035648,"available":95393144832,"internally_reserved":0,"allocated":106037248,"data_stored":104468854,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":495845376},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:15:14 2023","interfaces":[{"interface":"back","average":{"1min":1.944,"5min":1.1080000000000001,"15min":0.96899999999999997},"min":{"1min":0.23799999999999999,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":8.0079999999999991,"5min":8.0079999999999991,"15min":8.0079999999999991},"last":0.46899999999999997},{"interface":"front","average":{"1min":0.876,"5min":1.0820000000000001,"15min":1.1160000000000001},"min":{"1min":0.23799999999999999,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":2.4649999999999999,"5min":10.945,"15min":10.945},"last":0.71299999999999997}]},{"osd":1,"last update":"Sat Sep 23 21:15:14 2023","interfaces":[{"interface":"back","average":{"1min":1.345,"5min":1.0429999999999999,"15min":2.5840000000000001},"min":{"1min":0.32700000000000001,"5min":0.221,"15min":0.221},"max":{"1min":4.0410000000000004,"5min":7.8609999999999998,"15min":37.960000000000001},"last":0.64400000000000002},{"interface":"front","average":{"1min":2.246,"5min":1.319,"15min":2.516},"min":{"1min":0.30299999999999999,"5min":0.28499999999999998,"15min":0.27000000000000002},"max":{"1min":13.983000000000001,"5min":13.983000000000001,"15min":37.933},"last":0.54700000000000004}]},{"osd":2,"last update":"Sat Sep 23 21:15:14 2023","interfaces":[{"interface":"back","average":{"1min":2.3559999999999999,"5min":1.3660000000000001,"15min":1.734},"min":{"1min":0.26400000000000001,"5min":0.247,"15min":0.247},"max":{"1min":14.026,"5min":14.026,"15min":16.928000000000001},"last":0.5},{"interface":"front","average":{"1min":3.0419999999999998,"5min":1.6220000000000001,"15min":1.7949999999999999},"min":{"1min":0.33000000000000002,"5min":0.28599999999999998,"15min":0.27800000000000002},"max":{"1min":14.339,"5min":14.339,"15min":14.417999999999999},"last":0.69599999999999995}]},{"osd":3,"last update":"Sat Sep 23 21:14:47 2023","interfaces":[{"interface":"back","average":{"1min":1.0389999999999999,"5min":1.1990000000000001,"15min":2.056},"min":{"1min":0.27900000000000003,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":2.766,"5min":9.3279999999999994,"15min":15.01},"last":0.57699999999999996},{"interface":"front","average":{"1min":1.899,"5min":1.3879999999999999,"15min":2},"min":{"1min":0.32200000000000001,"5min":0.32200000000000001,"15min":0.30199999999999999},"max":{"1min":14.305,"5min":14.305,"15min":15.004},"last":0.60199999999999998}]},{"osd":5,"last update":"Sat Sep 23 21:15:04 2023","interfaces":[{"interface":"back","average":{"1min":2.0310000000000001,"5min":1.337,"15min":1.0840000000000001},"min":{"1min":0.32000000000000001,"5min":0.222,"15min":0.222},"max":{"1min":14.398999999999999,"5min":14.398999999999999,"15min":14.398999999999999},"last":0.442},{"interface":"front","average":{"1min":1.675,"5min":1.2050000000000001,"15min":1.121},"min":{"1min":0.29699999999999999,"5min":0.27200000000000002,"15min":0.255},"max":{"1min":8.016,"5min":8.016,"15min":9.8979999999999997},"last":0.67400000000000004}]},{"osd":6,"last update":"Sat Sep 23 21:15:14 2023","interfaces":[{"interface":"back","average":{"1min":1.403,"5min":1.248,"15min":1.2749999999999999},"min":{"1min":0.34300000000000003,"5min":0.28599999999999998,"15min":0.26800000000000002},"max":{"1min":8.7420000000000009,"5min":24.452999999999999,"15min":24.452999999999999},"last":0.76800000000000002},{"interface":"front","average":{"1min":2.472,"5min":1.425,"15min":1.0680000000000001},"min":{"1min":0.40300000000000002,"5min":0.33500000000000002,"15min":0.32500000000000001},"max":{"1min":14.374000000000001,"5min":14.374000000000001,"15min":14.387},"last":0.745}]},{"osd":7,"last update":"Sat Sep 23 21:14:47 2023","interfaces":[{"interface":"back","average":{"1min":1.9910000000000001,"5min":1.355,"15min":1.1299999999999999},"min":{"1min":0.38300000000000001,"5min":0.30099999999999999,"15min":0.29299999999999998},"max":{"1min":14.422000000000001,"5min":14.422000000000001,"15min":16.890999999999998},"last":0.66300000000000003},{"interface":"front","average":{"1min":1.536,"5min":1.234,"15min":1.2090000000000001},"min":{"1min":0.39200000000000002,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":7.7229999999999999,"5min":7.7229999999999999,"15min":11.170999999999999},"last":0.72799999999999998}]}]},{"osd":5,"up_from":39,"seq":167503724974,"num_pgs":55,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":618812,"kb_used_data":97652,"kb_used_omap":0,"kb_used_meta":521152,"kb_avail":93126340,"statfs":{"total":95995035648,"available":95361372160,"internally_reserved":0,"allocated":99995648,"data_stored":98435830,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":533659648},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:15:14 2023","interfaces":[{"interface":"back","average":{"1min":1.3720000000000001,"5min":0.84599999999999997,"15min":0.75800000000000001},"min":{"1min":0.23100000000000001,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":4.3680000000000003,"5min":4.3680000000000003,"15min":4.3680000000000003},"last":0.49199999999999999},{"interface":"front","average":{"1min":1.6839999999999999,"5min":0.872,"15min":0.73599999999999999},"min":{"1min":0.23899999999999999,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":6.577,"5min":6.577,"15min":6.577},"last":0.36699999999999999}]},{"osd":1,"last update":"Sat Sep 23 21:15:11 2023","interfaces":[{"interface":"back","average":{"1min":1.1140000000000001,"5min":0.94099999999999995,"15min":1.6830000000000001},"min":{"1min":0.29399999999999998,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":7.5839999999999996,"5min":7.5839999999999996,"15min":25.239999999999998},"last":0.42499999999999999},{"interface":"front","average":{"1min":1.123,"5min":1.472,"15min":1.7549999999999999},"min":{"1min":0.28299999999999997,"5min":0.28299999999999997,"15min":0.252},"max":{"1min":4.2199999999999998,"5min":11.116,"15min":25.253},"last":0.46400000000000002}]},{"osd":2,"last update":"Sat Sep 23 21:15:27 2023","interfaces":[{"interface":"back","average":{"1min":1.373,"5min":1.3320000000000001,"15min":1.6240000000000001},"min":{"1min":0.28399999999999997,"5min":0.27800000000000002,"15min":0.23499999999999999},"max":{"1min":3.8730000000000002,"5min":9.2889999999999997,"15min":12.634},"last":0.439},{"interface":"front","average":{"1min":1.472,"5min":1.1559999999999999,"15min":1.6319999999999999},"min":{"1min":0.29799999999999999,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":7.2720000000000002,"5min":9.7170000000000005,"15min":9.7650000000000006},"last":0.32500000000000001}]},{"osd":3,"last update":"Sat Sep 23 21:14:48 2023","interfaces":[{"interface":"back","average":{"1min":1.0109999999999999,"5min":1.6339999999999999,"15min":2.484},"min":{"1min":0.32400000000000001,"5min":0.311,"15min":0.29299999999999998},"max":{"1min":3.0390000000000001,"5min":14.242000000000001,"15min":61.667999999999999},"last":0.38100000000000001},{"interface":"front","average":{"1min":1.284,"5min":1.6559999999999999,"15min":2.5910000000000002},"min":{"1min":0.39100000000000001,"5min":0.34899999999999998,"15min":0.30599999999999999},"max":{"1min":6.508,"5min":12.44,"15min":62.276000000000003},"last":0.501}]},{"osd":4,"last update":"Sat Sep 23 21:15:00 2023","interfaces":[{"interface":"back","average":{"1min":1.0920000000000001,"5min":1.5289999999999999,"15min":1.157},"min":{"1min":0.253,"5min":0.253,"15min":0.23400000000000001},"max":{"1min":4.7750000000000004,"5min":12.776,"15min":12.776},"last":0.41299999999999998},{"interface":"front","average":{"1min":1.163,"5min":1.3029999999999999,"15min":1.1539999999999999},"min":{"1min":0.35999999999999999,"5min":0.27700000000000002,"15min":0.26900000000000002},"max":{"1min":4.8200000000000003,"5min":9.7119999999999997,"15min":11.973000000000001},"last":0.39800000000000002}]},{"osd":6,"last update":"Sat Sep 23 21:15:11 2023","interfaces":[{"interface":"back","average":{"1min":1.0880000000000001,"5min":1.0449999999999999,"15min":1.071},"min":{"1min":0.36499999999999999,"5min":0.313,"15min":0.25900000000000001},"max":{"1min":5.2249999999999996,"5min":7.7030000000000003,"15min":11.128},"last":0.52300000000000002},{"interface":"front","average":{"1min":1.022,"5min":1.5489999999999999,"15min":1.2450000000000001},"min":{"1min":0.34300000000000003,"5min":0.34300000000000003,"15min":0.317},"max":{"1min":4.3550000000000004,"5min":10.999000000000001,"15min":11.112},"last":0.53500000000000003}]},{"osd":7,"last update":"Sat Sep 23 21:14:48 2023","interfaces":[{"interface":"back","average":{"1min":0.84999999999999998,"5min":0.98499999999999999,"15min":1.8129999999999999},"min":{"1min":0.34499999999999997,"5min":0.31,"15min":0.309},"max":{"1min":3.3039999999999998,"5min":5.5659999999999998,"15min":189.405},"last":0.47399999999999998},{"interface":"front","average":{"1min":0.92000000000000004,"5min":1.472,"15min":1.9259999999999999},"min":{"1min":0.40899999999999997,"5min":0.35399999999999998,"15min":0.32500000000000001},"max":{"1min":3.3239999999999998,"5min":11.183999999999999,"15min":189.685},"last":0.51800000000000002}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8488,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1259,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5322,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":4838,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T21:15:30.001 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:15:30.035 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:15:30.038 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:15:30.039 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:15:30.140 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:15:30.169 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:15:30.214 INFO:teuthology.orchestra.run.smithi026.stderr:enabled application 'rados' on pool 'unique_pool_5' 2023-09-23T21:15:30.247 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:15:30 smithi026 ceph-mon[175151]: pgmap v1376: 119 pgs: 2 creating+peering, 4 unknown, 113 active+clean; 674 KiB data, 4.6 GiB used, 532 GiB / 536 GiB avail 2023-09-23T21:15:30.247 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:15:30 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/2427027992' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_5", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T21:15:30.247 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:15:30 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_5", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T21:15:30.247 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:15:30 smithi026 ceph-mon[175151]: from='client.36672 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T21:15:30.247 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:15:30 smithi026 ceph-mon[179688]: pgmap v1376: 119 pgs: 2 creating+peering, 4 unknown, 113 active+clean; 674 KiB data, 4.6 GiB used, 532 GiB / 536 GiB avail 2023-09-23T21:15:30.248 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:15:30 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/2427027992' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_5", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T21:15:30.248 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:15:30 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_5", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T21:15:30.248 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:15:30 smithi026 ceph-mon[179688]: from='client.36672 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T21:15:30.396 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:15:30.468 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:15:30.469 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:15:30 smithi070 ceph-mon[168282]: pgmap v1376: 119 pgs: 2 creating+peering, 4 unknown, 113 active+clean; 674 KiB data, 4.6 GiB used, 532 GiB / 536 GiB avail 2023-09-23T21:15:30.470 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:15:30 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/2427027992' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_5", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T21:15:30.470 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:15:30 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_5", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T21:15:30.470 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:15:30 smithi070 ceph-mon[168282]: from='client.36672 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T21:15:30.470 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:15:30.471 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:15:30.572 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:15:30.603 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:15:30.944 INFO:teuthology.orchestra.run.smithi070.stdout:ERROR: (22) Invalid argument 2023-09-23T21:15:30.944 INFO:teuthology.orchestra.run.smithi070.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T21:15:31.162 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T21:15:31.177 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:15:31.178 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:15:31.216 INFO:teuthology.orchestra.run.smithi026.stderr:noscrub is unset 2023-09-23T21:15:31.279 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:15:31.311 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:15:31.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:15:31 smithi070 ceph-mon[168282]: failed to encode map e547 with expected crc 2023-09-23T21:15:31.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:15:31 smithi070 ceph-mon[168282]: failed to encode map e547 with expected crc 2023-09-23T21:15:31.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:15:31 smithi070 ceph-mon[168282]: failed to encode map e547 with expected crc 2023-09-23T21:15:31.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:15:31 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_5", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-09-23T21:15:31.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:15:31 smithi070 ceph-mon[168282]: osdmap e547: 8 total, 8 up, 6 in 2023-09-23T21:15:31.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:15:31 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/3505931233' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T21:15:31.383 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_historic_ops 2023-09-23T21:15:31.554 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:15:31.554 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:15:31 smithi026 ceph-mon[175151]: failed to encode map e547 with expected crc 2023-09-23T21:15:31.554 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:15:31 smithi026 ceph-mon[175151]: failed to encode map e547 with expected crc 2023-09-23T21:15:31.554 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:15:31 smithi026 ceph-mon[175151]: failed to encode map e547 with expected crc 2023-09-23T21:15:31.554 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:15:31 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_5", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-09-23T21:15:31.555 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:15:31 smithi026 ceph-mon[175151]: osdmap e547: 8 total, 8 up, 6 in 2023-09-23T21:15:31.555 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:15:31 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/3505931233' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T21:15:31.555 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:15:31 smithi026 ceph-mon[179688]: failed to encode map e547 with expected crc 2023-09-23T21:15:31.555 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:15:31 smithi026 ceph-mon[179688]: failed to encode map e547 with expected crc 2023-09-23T21:15:31.555 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:15:31 smithi026 ceph-mon[179688]: failed to encode map e547 with expected crc 2023-09-23T21:15:31.555 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:15:31 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_5", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-09-23T21:15:31.555 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:15:31 smithi026 ceph-mon[179688]: osdmap e547: 8 total, 8 up, 6 in 2023-09-23T21:15:31.555 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:15:31 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/3505931233' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T21:15:31.556 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:15:31.556 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:15:31.657 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:15:31.685 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:15:31.721 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:15:31.723 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:15:31.724 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:15:31.825 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:15:31.852 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:15:31.889 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:15:31.891 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:15:31.892 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:15:31.954 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0: {} 2023-09-23T21:15:31.972 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1: {} 2023-09-23T21:15:31.980 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T21:15:31.992 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2: {} 2023-09-23T21:15:31.992 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:15:32.011 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3: {} 2023-09-23T21:15:32.021 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:15:32.031 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4: {} 2023-09-23T21:15:32.051 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T21:15:32.057 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:15:32.060 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:15:32.060 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:15:32.071 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6: {} 2023-09-23T21:15:32.090 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T21:15:32.092 INFO:teuthology.orchestra.run.smithi026.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T21:15:32.092 INFO:teuthology.orchestra.run.smithi026.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T21:15:32.092 INFO:teuthology.orchestra.run.smithi026.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T21:15:32.092 INFO:teuthology.orchestra.run.smithi026.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T21:15:32.092 INFO:teuthology.orchestra.run.smithi026.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T21:15:32.092 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T21:15:32.092 INFO:teuthology.orchestra.run.smithi026.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T21:15:32.092 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T21:15:32.162 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:15:32.193 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:15:32.362 DEBUG:teuthology.orchestra.run.smithi026:> /bin/sh -c 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados --no-log-to-stderr --name client.0 -b 65536 --object-size 65536 -p unique_pool_5 bench 90 write' 2023-09-23T21:15:32.365 INFO:tasks.radosbench:joining radosbench (timing out after 3000s) 2023-09-23T21:15:32.365 INFO:teuthology.orchestra.run:waiting for 3000 2023-09-23T21:15:32.381 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:15:32 smithi026 ceph-mon[179688]: failed to encode map e547 with expected crc 2023-09-23T21:15:32.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:15:32 smithi026 ceph-mon[179688]: pgmap v1378: 119 pgs: 119 active+clean; 674 KiB data, 4.4 GiB used, 532 GiB / 536 GiB avail 2023-09-23T21:15:32.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:15:32 smithi026 ceph-mon[179688]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:15:32.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:15:32 smithi026 ceph-mon[179688]: failed to encode map e548 with expected crc 2023-09-23T21:15:32.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:15:32 smithi026 ceph-mon[179688]: failed to encode map e548 with expected crc 2023-09-23T21:15:32.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:15:32 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/3505931233' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T21:15:32.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:15:32 smithi026 ceph-mon[179688]: osdmap e548: 8 total, 8 up, 6 in 2023-09-23T21:15:32.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:15:32 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:15:32.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:15:32 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:15:32.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:15:32 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/4265377152' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:15:32.383 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:15:32 smithi026 ceph-mon[175151]: failed to encode map e547 with expected crc 2023-09-23T21:15:32.383 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:15:32 smithi026 ceph-mon[175151]: pgmap v1378: 119 pgs: 119 active+clean; 674 KiB data, 4.4 GiB used, 532 GiB / 536 GiB avail 2023-09-23T21:15:32.383 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:15:32 smithi026 ceph-mon[175151]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:15:32.383 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:15:32 smithi026 ceph-mon[175151]: failed to encode map e548 with expected crc 2023-09-23T21:15:32.383 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:15:32 smithi026 ceph-mon[175151]: failed to encode map e548 with expected crc 2023-09-23T21:15:32.383 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:15:32 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/3505931233' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T21:15:32.383 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:15:32 smithi026 ceph-mon[175151]: osdmap e548: 8 total, 8 up, 6 in 2023-09-23T21:15:32.383 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:15:32 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:15:32.383 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:15:32 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:15:32.383 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:15:32 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/4265377152' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:15:32.440 INFO:tasks.radosbench.radosbench.0.smithi026.stdout:hints = 1 2023-09-23T21:15:32.441 INFO:tasks.radosbench.radosbench.0.smithi026.stdout:Maintaining 16 concurrent writes of 65536 bytes to objects of size 65536 for up to 90 seconds or 0 objects 2023-09-23T21:15:32.441 INFO:tasks.radosbench.radosbench.0.smithi026.stdout:Object prefix: benchmark_data_smithi026_424533 2023-09-23T21:15:32.441 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T21:15:32.441 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 0 0 0 0 0 0 - 0 2023-09-23T21:15:32.507 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:15:32.512 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:15:32.513 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:15:32.542 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:15:32 smithi070 ceph-mon[168282]: failed to encode map e547 with expected crc 2023-09-23T21:15:32.542 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:15:32 smithi070 ceph-mon[168282]: pgmap v1378: 119 pgs: 119 active+clean; 674 KiB data, 4.4 GiB used, 532 GiB / 536 GiB avail 2023-09-23T21:15:32.543 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:15:32 smithi070 ceph-mon[168282]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:15:32.543 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:15:32 smithi070 ceph-mon[168282]: failed to encode map e548 with expected crc 2023-09-23T21:15:32.543 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:15:32 smithi070 ceph-mon[168282]: failed to encode map e548 with expected crc 2023-09-23T21:15:32.543 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:15:32 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/3505931233' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T21:15:32.543 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:15:32 smithi070 ceph-mon[168282]: osdmap e548: 8 total, 8 up, 6 in 2023-09-23T21:15:32.543 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:15:32 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:15:32.543 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:15:32 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:15:32.543 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:15:32 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/4265377152' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:15:32.614 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:15:32.659 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:15:33.001 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T21:15:33.361 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:15:33.364 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:15:33.364 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:15:33.381 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:15:33 smithi026 ceph-mon[179688]: failed to encode map e548 with expected crc 2023-09-23T21:15:33.381 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:15:33 smithi026 ceph-mon[179688]: failed to encode map e548 with expected crc 2023-09-23T21:15:33.381 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:15:33 smithi026 ceph-mon[179688]: 1.c scrub starts 2023-09-23T21:15:33.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:15:33 smithi026 ceph-mon[179688]: 1.c scrub ok 2023-09-23T21:15:33.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:15:33 smithi026 ceph-mon[175151]: failed to encode map e548 with expected crc 2023-09-23T21:15:33.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:15:33 smithi026 ceph-mon[175151]: failed to encode map e548 with expected crc 2023-09-23T21:15:33.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:15:33 smithi026 ceph-mon[175151]: 1.c scrub starts 2023-09-23T21:15:33.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:15:33 smithi026 ceph-mon[175151]: 1.c scrub ok 2023-09-23T21:15:33.441 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 1 16 2855 2839 177.428 177.438 0.00564032 0.00561764 2023-09-23T21:15:33.465 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:15:33.525 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:15:33.590 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:15:33.595 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:15:33.595 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:15:33.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:15:33 smithi070 ceph-mon[168282]: failed to encode map e548 with expected crc 2023-09-23T21:15:33.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:15:33 smithi070 ceph-mon[168282]: failed to encode map e548 with expected crc 2023-09-23T21:15:33.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:15:33 smithi070 ceph-mon[168282]: 1.c scrub starts 2023-09-23T21:15:33.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:15:33 smithi070 ceph-mon[168282]: 1.c scrub ok 2023-09-23T21:15:33.667 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T21:15:33.696 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:15:33.774 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:15:33.803 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:15:33.846 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T21:15:33.851 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:15:33.852 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:15:33.953 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:15:34.013 INFO:teuthology.orchestra.run.smithi070.stdout:{ 2023-09-23T21:15:34.014 INFO:teuthology.orchestra.run.smithi070.stdout: "size": 20, 2023-09-23T21:15:34.014 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 600, 2023-09-23T21:15:34.014 INFO:teuthology.orchestra.run.smithi070.stdout: "ops": [ 2023-09-23T21:15:34.014 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.014 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.46328.0:74333 10.d 10:bc627dcb:::benchmark_data_smithi026_379838_object74332:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e460)", 2023-09-23T21:15:34.014 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:11:37.749827+0000", 2023-09-23T21:15:34.014 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 236.246487723, 2023-09-23T21:15:34.014 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.4276132540000002, 2023-09-23T21:15:34.014 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:15:34.015 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:15:34.015 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:15:34.015 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.46328", 2023-09-23T21:15:34.015 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/3748861181", 2023-09-23T21:15:34.015 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 74333 2023-09-23T21:15:34.015 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.015 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:15:34.015 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.015 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:15:34.015 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:37.749827+0000", 2023-09-23T21:15:34.015 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:15:34.016 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.016 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.016 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:15:34.016 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:37.749827+0000", 2023-09-23T21:15:34.016 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:15:34.016 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.016 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.016 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:15:34.016 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:37.749825+0000", 2023-09-23T21:15:34.016 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999981 2023-09-23T21:15:34.016 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.017 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.017 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:15:34.017 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:37.749879+0000", 2023-09-23T21:15:34.017 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.4143e-05 2023-09-23T21:15:34.017 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.017 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.017 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:15:34.017 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:37.749894+0000", 2023-09-23T21:15:34.017 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.4973e-05 2023-09-23T21:15:34.017 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.017 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.018 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:15:34.018 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:37.749903+0000", 2023-09-23T21:15:34.018 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.0059999999999998e-06 2023-09-23T21:15:34.018 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.018 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.018 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:15:34.018 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:37.749946+0000", 2023-09-23T21:15:34.018 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.2349999999999999e-05 2023-09-23T21:15:34.019 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.019 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.019 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:15:34.019 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:37.749958+0000", 2023-09-23T21:15:34.019 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.2622999999999999e-05 2023-09-23T21:15:34.019 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.019 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.019 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:15:34.019 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:38.737069+0000", 2023-09-23T21:15:34.019 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.98711088800000002 2023-09-23T21:15:34.020 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.020 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.020 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:15:34.020 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:38.737077+0000", 2023-09-23T21:15:34.020 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.0150000000000003e-06 2023-09-23T21:15:34.020 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.020 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.020 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:15:34.020 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:40.172893+0000", 2023-09-23T21:15:34.020 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.435816266 2023-09-23T21:15:34.020 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.020 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.021 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:15:34.021 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:40.173003+0000", 2023-09-23T21:15:34.021 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00010993 2023-09-23T21:15:34.021 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.021 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.021 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 0,3,7", 2023-09-23T21:15:34.021 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:40.173114+0000", 2023-09-23T21:15:34.021 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000110849 2023-09-23T21:15:34.021 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.021 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.022 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:15:34.022 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:40.175684+0000", 2023-09-23T21:15:34.022 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0025697369999999999 2023-09-23T21:15:34.022 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.022 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.022 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:15:34.022 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:40.177298+0000", 2023-09-23T21:15:34.022 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0016144969999999999 2023-09-23T21:15:34.022 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.023 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.023 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:15:34.023 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:40.177308+0000", 2023-09-23T21:15:34.023 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.8879999999999992e-06 2023-09-23T21:15:34.023 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.023 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.023 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:15:34.023 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:40.177333+0000", 2023-09-23T21:15:34.023 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.4905999999999998e-05 2023-09-23T21:15:34.023 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.023 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.024 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:15:34.024 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:40.177376+0000", 2023-09-23T21:15:34.024 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.2635000000000001e-05 2023-09-23T21:15:34.024 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.024 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.024 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:15:34.024 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:40.177440+0000", 2023-09-23T21:15:34.024 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.4252999999999996e-05 2023-09-23T21:15:34.024 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:15:34.024 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:15:34.024 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:15:34.025 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.025 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.025 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.46328.0:74356 10.d 10:ba33591e:::benchmark_data_smithi026_379838_object74355:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e460)", 2023-09-23T21:15:34.025 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:11:37.787894+0000", 2023-09-23T21:15:34.025 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 236.20842096800001, 2023-09-23T21:15:34.025 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.3971935719999999, 2023-09-23T21:15:34.025 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:15:34.025 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:15:34.025 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:15:34.025 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.46328", 2023-09-23T21:15:34.025 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/3748861181", 2023-09-23T21:15:34.026 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 74356 2023-09-23T21:15:34.026 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.026 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:15:34.026 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.026 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:15:34.026 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:37.787894+0000", 2023-09-23T21:15:34.026 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:15:34.026 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.026 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.027 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:15:34.027 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:37.787894+0000", 2023-09-23T21:15:34.027 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:15:34.027 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.027 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.027 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:15:34.027 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:37.787893+0000", 2023-09-23T21:15:34.027 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999995 2023-09-23T21:15:34.027 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.027 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.027 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:15:34.028 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:37.787916+0000", 2023-09-23T21:15:34.028 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.2857000000000002e-05 2023-09-23T21:15:34.028 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.028 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.028 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:15:34.028 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:37.787923+0000", 2023-09-23T21:15:34.028 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.6540000000000002e-06 2023-09-23T21:15:34.028 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.028 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.028 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:15:34.029 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:37.787927+0000", 2023-09-23T21:15:34.029 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.4399999999999998e-06 2023-09-23T21:15:34.029 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.029 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.029 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:15:34.029 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:37.787987+0000", 2023-09-23T21:15:34.029 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.9561000000000001e-05 2023-09-23T21:15:34.029 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.029 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.029 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:15:34.029 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:37.787997+0000", 2023-09-23T21:15:34.029 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.9189999999999994e-06 2023-09-23T21:15:34.029 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.030 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.030 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:15:34.030 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:38.737092+0000", 2023-09-23T21:15:34.030 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.94909543100000004 2023-09-23T21:15:34.030 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.030 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.030 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:15:34.030 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:38.737097+0000", 2023-09-23T21:15:34.030 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.9169999999999997e-06 2023-09-23T21:15:34.030 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.031 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.031 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:15:34.031 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:40.175712+0000", 2023-09-23T21:15:34.031 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.438615602 2023-09-23T21:15:34.031 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.031 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.031 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:15:34.031 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:40.175796+0000", 2023-09-23T21:15:34.031 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.3318000000000004e-05 2023-09-23T21:15:34.031 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.031 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.031 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 0,3,7", 2023-09-23T21:15:34.032 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:40.175911+0000", 2023-09-23T21:15:34.032 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000115275 2023-09-23T21:15:34.032 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.032 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.032 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:15:34.032 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:40.181314+0000", 2023-09-23T21:15:34.032 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.005402919 2023-09-23T21:15:34.032 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.032 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.032 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:15:34.032 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:40.181389+0000", 2023-09-23T21:15:34.032 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.4692999999999998e-05 2023-09-23T21:15:34.033 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.033 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.033 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:15:34.033 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:40.181395+0000", 2023-09-23T21:15:34.033 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.6200000000000001e-06 2023-09-23T21:15:34.033 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.033 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.033 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:15:34.033 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:40.184985+0000", 2023-09-23T21:15:34.033 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0035895900000000001 2023-09-23T21:15:34.033 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.034 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.034 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:15:34.034 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:40.185022+0000", 2023-09-23T21:15:34.034 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.7166000000000003e-05 2023-09-23T21:15:34.034 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.034 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.034 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:15:34.034 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:40.185087+0000", 2023-09-23T21:15:34.034 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.5100999999999999e-05 2023-09-23T21:15:34.034 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:15:34.034 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:15:34.034 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:15:34.035 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.035 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.035 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.46328.0:74367 10.d 10:b9926de2:::benchmark_data_smithi026_379838_object74366:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e460)", 2023-09-23T21:15:34.035 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:11:37.805093+0000", 2023-09-23T21:15:34.035 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 236.191221171, 2023-09-23T21:15:34.035 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.3801344150000001, 2023-09-23T21:15:34.035 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:15:34.035 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:15:34.035 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:15:34.035 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.46328", 2023-09-23T21:15:34.035 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/3748861181", 2023-09-23T21:15:34.035 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 74367 2023-09-23T21:15:34.036 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.036 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:15:34.036 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.036 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:15:34.036 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:37.805093+0000", 2023-09-23T21:15:34.036 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:15:34.036 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.036 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.036 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:15:34.036 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:37.805093+0000", 2023-09-23T21:15:34.036 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:15:34.036 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.037 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.037 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:15:34.037 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:37.805092+0000", 2023-09-23T21:15:34.037 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999981 2023-09-23T21:15:34.037 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.037 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.037 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:15:34.037 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:37.805182+0000", 2023-09-23T21:15:34.037 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.0195e-05 2023-09-23T21:15:34.037 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.037 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.037 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:15:34.037 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:37.805195+0000", 2023-09-23T21:15:34.038 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.2823000000000001e-05 2023-09-23T21:15:34.038 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.038 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.038 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:15:34.038 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:37.805201+0000", 2023-09-23T21:15:34.038 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.759e-06 2023-09-23T21:15:34.038 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.038 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.038 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:15:34.038 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:37.805275+0000", 2023-09-23T21:15:34.038 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.3203999999999999e-05 2023-09-23T21:15:34.038 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.038 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.039 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:15:34.039 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:37.805287+0000", 2023-09-23T21:15:34.039 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.2526e-05 2023-09-23T21:15:34.039 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.039 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.039 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:15:34.039 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:38.737108+0000", 2023-09-23T21:15:34.039 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.93182051399999999 2023-09-23T21:15:34.039 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.039 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.039 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:15:34.039 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:38.737112+0000", 2023-09-23T21:15:34.040 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.7900000000000001e-06 2023-09-23T21:15:34.040 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.040 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.040 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:15:34.040 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:40.177476+0000", 2023-09-23T21:15:34.040 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.440364829 2023-09-23T21:15:34.040 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.040 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.040 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:15:34.040 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:40.177607+0000", 2023-09-23T21:15:34.040 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00013115199999999999 2023-09-23T21:15:34.040 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.041 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.041 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 0,3,7", 2023-09-23T21:15:34.041 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:40.177728+0000", 2023-09-23T21:15:34.041 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00012074 2023-09-23T21:15:34.041 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.041 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.041 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:15:34.041 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:40.181346+0000", 2023-09-23T21:15:34.041 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0036178339999999999 2023-09-23T21:15:34.041 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.041 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.042 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:15:34.042 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:40.184839+0000", 2023-09-23T21:15:34.042 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.003493351 2023-09-23T21:15:34.042 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.042 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.042 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:15:34.042 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:40.184931+0000", 2023-09-23T21:15:34.042 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.1612000000000003e-05 2023-09-23T21:15:34.042 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.042 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.042 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:15:34.042 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:40.185147+0000", 2023-09-23T21:15:34.043 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00021584999999999999 2023-09-23T21:15:34.043 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.043 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.043 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:15:34.043 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:40.185173+0000", 2023-09-23T21:15:34.043 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.5884000000000002e-05 2023-09-23T21:15:34.043 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.043 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.043 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:15:34.043 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:40.185228+0000", 2023-09-23T21:15:34.043 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.5062000000000002e-05 2023-09-23T21:15:34.043 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:15:34.043 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:15:34.044 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:15:34.044 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.044 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.044 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.46328.0:74398 10.d 10:bdc342d2:::benchmark_data_smithi026_379838_object74397:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e460)", 2023-09-23T21:15:34.044 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:11:37.837382+0000", 2023-09-23T21:15:34.044 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 236.15893238699999, 2023-09-23T21:15:34.044 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.3480186879999998, 2023-09-23T21:15:34.044 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:15:34.044 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:15:34.044 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:15:34.045 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.46328", 2023-09-23T21:15:34.045 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/3748861181", 2023-09-23T21:15:34.045 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 74398 2023-09-23T21:15:34.045 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.045 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:15:34.045 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.045 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:15:34.045 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:37.837382+0000", 2023-09-23T21:15:34.045 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:15:34.045 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.045 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.045 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:15:34.045 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:37.837382+0000", 2023-09-23T21:15:34.046 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:15:34.046 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.046 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.046 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:15:34.046 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:37.837381+0000", 2023-09-23T21:15:34.046 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.999999 2023-09-23T21:15:34.046 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.047 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.047 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:15:34.047 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:37.837411+0000", 2023-09-23T21:15:34.047 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.0006e-05 2023-09-23T21:15:34.047 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.047 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.047 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:15:34.048 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:37.837421+0000", 2023-09-23T21:15:34.048 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.4029999999999996e-06 2023-09-23T21:15:34.048 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.048 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.049 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:15:34.049 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:37.837425+0000", 2023-09-23T21:15:34.049 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.7249999999999997e-06 2023-09-23T21:15:34.049 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.049 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.049 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:15:34.050 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:37.837477+0000", 2023-09-23T21:15:34.050 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.1430000000000001e-05 2023-09-23T21:15:34.050 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.050 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.050 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:15:34.051 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:37.837488+0000", 2023-09-23T21:15:34.051 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.1357e-05 2023-09-23T21:15:34.051 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.051 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.051 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:15:34.051 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:38.737135+0000", 2023-09-23T21:15:34.051 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.89964650899999998 2023-09-23T21:15:34.051 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.051 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.051 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:15:34.051 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:38.737138+0000", 2023-09-23T21:15:34.052 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.8709999999999999e-06 2023-09-23T21:15:34.052 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.052 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.052 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:15:34.052 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:40.179280+0000", 2023-09-23T21:15:34.052 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.442141605 2023-09-23T21:15:34.052 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.052 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.052 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:15:34.052 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:40.179376+0000", 2023-09-23T21:15:34.052 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.5909999999999995e-05 2023-09-23T21:15:34.053 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.053 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.053 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 0,3,7", 2023-09-23T21:15:34.053 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:40.179505+0000", 2023-09-23T21:15:34.053 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00012945900000000001 2023-09-23T21:15:34.053 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.053 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.053 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:15:34.053 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:40.184887+0000", 2023-09-23T21:15:34.053 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.005381884 2023-09-23T21:15:34.053 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.054 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.054 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:15:34.054 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:40.184944+0000", 2023-09-23T21:15:34.054 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.6521999999999999e-05 2023-09-23T21:15:34.054 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.054 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.054 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:15:34.054 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:40.185125+0000", 2023-09-23T21:15:34.054 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00018092 2023-09-23T21:15:34.054 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.054 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.055 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:15:34.055 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:40.185307+0000", 2023-09-23T21:15:34.055 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000181991 2023-09-23T21:15:34.055 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.055 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.055 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:15:34.055 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:40.185340+0000", 2023-09-23T21:15:34.055 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.3021000000000001e-05 2023-09-23T21:15:34.055 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.055 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.055 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:15:34.055 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:40.185401+0000", 2023-09-23T21:15:34.056 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.1158e-05 2023-09-23T21:15:34.056 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:15:34.056 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:15:34.056 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:15:34.056 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.056 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.056 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.46328.0:74400 10.d 10:b25fac0d:::benchmark_data_smithi026_379838_object74399:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e460)", 2023-09-23T21:15:34.056 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:11:37.840259+0000", 2023-09-23T21:15:34.056 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 236.15605532000001, 2023-09-23T21:15:34.056 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.345304139, 2023-09-23T21:15:34.056 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:15:34.057 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:15:34.057 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:15:34.057 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.46328", 2023-09-23T21:15:34.057 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/3748861181", 2023-09-23T21:15:34.057 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 74400 2023-09-23T21:15:34.057 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.057 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:15:34.057 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.057 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:15:34.057 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:37.840259+0000", 2023-09-23T21:15:34.057 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:15:34.057 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.058 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.058 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:15:34.058 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:37.840259+0000", 2023-09-23T21:15:34.058 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:15:34.058 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.058 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.058 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:15:34.058 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:37.840257+0000", 2023-09-23T21:15:34.058 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999981 2023-09-23T21:15:34.058 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.058 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.059 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:15:34.059 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:37.840311+0000", 2023-09-23T21:15:34.059 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.3449000000000003e-05 2023-09-23T21:15:34.059 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.059 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.059 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:15:34.059 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:37.840322+0000", 2023-09-23T21:15:34.059 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.1197e-05 2023-09-23T21:15:34.059 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.059 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.059 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:15:34.059 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:37.840328+0000", 2023-09-23T21:15:34.060 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.2129999999999996e-06 2023-09-23T21:15:34.060 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.060 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.060 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:15:34.060 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:37.840388+0000", 2023-09-23T21:15:34.060 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.0081000000000003e-05 2023-09-23T21:15:34.060 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.060 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.060 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:15:34.060 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:37.840399+0000", 2023-09-23T21:15:34.060 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.0465000000000001e-05 2023-09-23T21:15:34.060 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.061 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.061 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:15:34.061 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:38.737150+0000", 2023-09-23T21:15:34.061 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.89675132499999999 2023-09-23T21:15:34.061 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.061 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.061 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:15:34.061 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:38.737154+0000", 2023-09-23T21:15:34.061 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.6200000000000001e-06 2023-09-23T21:15:34.061 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.061 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.061 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:15:34.062 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:40.181411+0000", 2023-09-23T21:15:34.062 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.444257009 2023-09-23T21:15:34.062 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.062 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.062 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:15:34.062 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:40.181497+0000", 2023-09-23T21:15:34.062 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.6570000000000006e-05 2023-09-23T21:15:34.062 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.062 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.062 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 0,3,7", 2023-09-23T21:15:34.062 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:40.181626+0000", 2023-09-23T21:15:34.062 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00012901699999999999 2023-09-23T21:15:34.063 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.063 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.063 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:15:34.063 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:40.184950+0000", 2023-09-23T21:15:34.063 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.003323501 2023-09-23T21:15:34.063 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.063 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.063 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:15:34.063 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:40.185262+0000", 2023-09-23T21:15:34.063 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000312045 2023-09-23T21:15:34.063 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.063 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.064 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:15:34.064 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:40.185281+0000", 2023-09-23T21:15:34.064 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.9055999999999998e-05 2023-09-23T21:15:34.064 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.064 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.064 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:15:34.064 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:40.185481+0000", 2023-09-23T21:15:34.064 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00019994199999999999 2023-09-23T21:15:34.064 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.064 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.064 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:15:34.064 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:40.185508+0000", 2023-09-23T21:15:34.064 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.6801e-05 2023-09-23T21:15:34.065 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.065 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.065 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:15:34.065 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:40.185563+0000", 2023-09-23T21:15:34.065 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.5670000000000001e-05 2023-09-23T21:15:34.065 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:15:34.065 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:15:34.065 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:15:34.065 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.065 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.065 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.46328.0:74403 10.d 10:b5a2ad18:::benchmark_data_smithi026_379838_object74402:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e460)", 2023-09-23T21:15:34.065 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:11:37.844596+0000", 2023-09-23T21:15:34.066 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 236.151718816, 2023-09-23T21:15:34.066 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.3410775579999998, 2023-09-23T21:15:34.066 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:15:34.066 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:15:34.066 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:15:34.066 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.46328", 2023-09-23T21:15:34.066 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/3748861181", 2023-09-23T21:15:34.066 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 74403 2023-09-23T21:15:34.067 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.067 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:15:34.067 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.067 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:15:34.067 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:37.844596+0000", 2023-09-23T21:15:34.067 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:15:34.067 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.067 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.067 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:15:34.067 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:37.844596+0000", 2023-09-23T21:15:34.067 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:15:34.067 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.068 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.068 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:15:34.068 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:37.844594+0000", 2023-09-23T21:15:34.068 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T21:15:34.068 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.068 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.068 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:15:34.068 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:37.844658+0000", 2023-09-23T21:15:34.068 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.4053000000000005e-05 2023-09-23T21:15:34.068 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.068 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.068 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:15:34.068 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:37.844675+0000", 2023-09-23T21:15:34.069 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.6670000000000001e-05 2023-09-23T21:15:34.069 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.069 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.069 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:15:34.069 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:37.844682+0000", 2023-09-23T21:15:34.069 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.7839999999999998e-06 2023-09-23T21:15:34.069 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.069 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.069 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:15:34.069 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:37.844751+0000", 2023-09-23T21:15:34.069 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.9698999999999998e-05 2023-09-23T21:15:34.069 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.069 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.070 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:15:34.070 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:37.844762+0000", 2023-09-23T21:15:34.070 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.049e-05 2023-09-23T21:15:34.070 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.070 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.070 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:15:34.070 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:38.737168+0000", 2023-09-23T21:15:34.070 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.89240619799999998 2023-09-23T21:15:34.071 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.071 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.071 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:15:34.071 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:38.737174+0000", 2023-09-23T21:15:34.071 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.3319999999999997e-06 2023-09-23T21:15:34.071 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.071 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.071 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:15:34.071 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:40.183023+0000", 2023-09-23T21:15:34.072 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.445848958 2023-09-23T21:15:34.072 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.072 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.072 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:15:34.072 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:40.183163+0000", 2023-09-23T21:15:34.072 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00013908200000000001 2023-09-23T21:15:34.072 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.072 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.072 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 0,3,7", 2023-09-23T21:15:34.072 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:40.183342+0000", 2023-09-23T21:15:34.072 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000179218 2023-09-23T21:15:34.072 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.072 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.073 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:15:34.073 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:40.185290+0000", 2023-09-23T21:15:34.073 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0019481769999999999 2023-09-23T21:15:34.073 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.073 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.073 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:15:34.073 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:40.185432+0000", 2023-09-23T21:15:34.073 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000142435 2023-09-23T21:15:34.073 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.073 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.073 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:15:34.073 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:40.185457+0000", 2023-09-23T21:15:34.074 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.5054000000000001e-05 2023-09-23T21:15:34.074 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.074 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.074 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:15:34.074 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:40.185592+0000", 2023-09-23T21:15:34.074 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00013438499999999999 2023-09-23T21:15:34.074 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.074 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.074 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:15:34.074 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:40.185619+0000", 2023-09-23T21:15:34.074 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.6727e-05 2023-09-23T21:15:34.074 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.075 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.075 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:15:34.075 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:40.185673+0000", 2023-09-23T21:15:34.075 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.4796000000000002e-05 2023-09-23T21:15:34.075 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:15:34.075 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:15:34.075 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:15:34.075 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.075 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.075 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.46328.0:74404 10.d 10:bd0dcc09:::benchmark_data_smithi026_379838_object74403:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e460)", 2023-09-23T21:15:34.075 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:11:37.844695+0000", 2023-09-23T21:15:34.075 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 236.151619678, 2023-09-23T21:15:34.076 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.3484742669999998, 2023-09-23T21:15:34.076 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:15:34.076 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:15:34.076 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:15:34.076 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.46328", 2023-09-23T21:15:34.076 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/3748861181", 2023-09-23T21:15:34.076 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 74404 2023-09-23T21:15:34.076 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.076 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:15:34.076 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.076 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:15:34.076 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:37.844695+0000", 2023-09-23T21:15:34.076 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:15:34.077 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.077 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.077 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:15:34.077 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:37.844695+0000", 2023-09-23T21:15:34.077 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:15:34.077 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.077 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.077 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:15:34.077 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:37.844695+0000", 2023-09-23T21:15:34.077 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999995 2023-09-23T21:15:34.077 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.078 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.078 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:15:34.078 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:37.844719+0000", 2023-09-23T21:15:34.078 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.4070000000000002e-05 2023-09-23T21:15:34.078 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.078 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.079 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:15:34.079 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:37.844726+0000", 2023-09-23T21:15:34.079 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.3459999999999996e-06 2023-09-23T21:15:34.079 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.079 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.079 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:15:34.079 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:37.844730+0000", 2023-09-23T21:15:34.080 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.5920000000000001e-06 2023-09-23T21:15:34.080 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.080 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.080 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:15:34.080 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:37.844800+0000", 2023-09-23T21:15:34.080 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.0759000000000005e-05 2023-09-23T21:15:34.081 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.081 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.081 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:15:34.081 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:37.844812+0000", 2023-09-23T21:15:34.081 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.2130000000000001e-05 2023-09-23T21:15:34.081 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.082 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.082 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:15:34.082 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:38.737187+0000", 2023-09-23T21:15:34.082 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.89237498199999998 2023-09-23T21:15:34.082 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.082 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.083 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:15:34.083 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:38.737191+0000", 2023-09-23T21:15:34.083 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.7189999999999999e-06 2023-09-23T21:15:34.083 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.083 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.083 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:15:34.084 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:40.185696+0000", 2023-09-23T21:15:34.084 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.4485052139999999 2023-09-23T21:15:34.084 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.084 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.084 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:15:34.084 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:40.185770+0000", 2023-09-23T21:15:34.085 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.3560999999999996e-05 2023-09-23T21:15:34.085 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.085 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.085 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 0,3,7", 2023-09-23T21:15:34.085 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:40.185864+0000", 2023-09-23T21:15:34.085 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.4244000000000001e-05 2023-09-23T21:15:34.085 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.085 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.086 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:15:34.086 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:40.192926+0000", 2023-09-23T21:15:34.086 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0070613910000000002 2023-09-23T21:15:34.086 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.086 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.087 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:15:34.087 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:40.192930+0000", 2023-09-23T21:15:34.087 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.8080000000000004e-06 2023-09-23T21:15:34.087 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.087 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.087 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:15:34.088 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:40.192975+0000", 2023-09-23T21:15:34.088 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.4990000000000001e-05 2023-09-23T21:15:34.088 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.088 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.088 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:15:34.089 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:40.193012+0000", 2023-09-23T21:15:34.089 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.6443000000000003e-05 2023-09-23T21:15:34.089 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.089 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.089 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:15:34.089 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:40.193069+0000", 2023-09-23T21:15:34.089 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.7065000000000003e-05 2023-09-23T21:15:34.089 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.089 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.090 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:15:34.090 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:40.193169+0000", 2023-09-23T21:15:34.090 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000100315 2023-09-23T21:15:34.090 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:15:34.090 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:15:34.091 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:15:34.091 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.091 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.091 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_repop(client.46328.0:84573 10.c e471/465 10:3fce4c21:::benchmark_data_smithi026_379838_object4616:head v 471'5331, mlcod=470'5329)", 2023-09-23T21:15:34.091 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:12:10.780185+0000", 2023-09-23T21:15:34.091 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 203.21612994099999, 2023-09-23T21:15:34.092 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.5674745780000001, 2023-09-23T21:15:34.092 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:15:34.092 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:15:34.092 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:15:34.092 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.092 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:15:34.092 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:12:10.780185+0000", 2023-09-23T21:15:34.092 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:15:34.092 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.092 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.092 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:15:34.092 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:12:10.780185+0000", 2023-09-23T21:15:34.093 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:15:34.093 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.093 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.093 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:15:34.093 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:12:10.780184+0000", 2023-09-23T21:15:34.093 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999995 2023-09-23T21:15:34.093 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.094 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.094 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:15:34.094 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:12:10.780189+0000", 2023-09-23T21:15:34.094 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.7650000000000001e-06 2023-09-23T21:15:34.094 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.094 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.095 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:15:34.095 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:12:10.780196+0000", 2023-09-23T21:15:34.095 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.9040000000000003e-06 2023-09-23T21:15:34.095 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.095 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.095 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:15:34.095 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:12:10.780203+0000", 2023-09-23T21:15:34.095 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.3690000000000001e-06 2023-09-23T21:15:34.095 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.095 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.095 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:15:34.095 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:12:14.336808+0000", 2023-09-23T21:15:34.096 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.556604718 2023-09-23T21:15:34.096 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.096 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.096 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:15:34.096 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:12:14.336836+0000", 2023-09-23T21:15:34.096 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.7379e-05 2023-09-23T21:15:34.096 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.096 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.096 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:15:34.096 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:12:14.347611+0000", 2023-09-23T21:15:34.096 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.01077504 2023-09-23T21:15:34.096 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.096 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.097 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:15:34.097 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:12:14.347659+0000", 2023-09-23T21:15:34.097 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.8668000000000002e-05 2023-09-23T21:15:34.097 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:15:34.097 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:15:34.097 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:15:34.097 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.097 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.097 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_repop(client.46328.0:84576 10.c e471/465 10:3b2a76db:::benchmark_data_smithi026_379838_object4619:head v 471'5332, mlcod=470'5329)", 2023-09-23T21:15:34.097 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:12:10.849370+0000", 2023-09-23T21:15:34.097 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 203.146944461, 2023-09-23T21:15:34.097 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.503394407, 2023-09-23T21:15:34.098 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:15:34.098 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:15:34.098 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:15:34.098 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.098 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:15:34.098 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:12:10.849370+0000", 2023-09-23T21:15:34.098 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:15:34.098 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.098 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.098 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:15:34.098 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:12:10.849370+0000", 2023-09-23T21:15:34.098 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:15:34.099 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.099 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.099 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:15:34.099 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:12:10.849369+0000", 2023-09-23T21:15:34.099 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.999999 2023-09-23T21:15:34.099 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.099 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.099 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:15:34.099 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:12:10.849380+0000", 2023-09-23T21:15:34.099 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.0987e-05 2023-09-23T21:15:34.099 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.099 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.099 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:15:34.099 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:12:10.849392+0000", 2023-09-23T21:15:34.100 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.1843e-05 2023-09-23T21:15:34.100 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.100 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.100 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:15:34.100 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:12:10.849399+0000", 2023-09-23T21:15:34.100 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.8549999999999996e-06 2023-09-23T21:15:34.100 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.100 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.100 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:15:34.100 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:12:14.345110+0000", 2023-09-23T21:15:34.100 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.4957117150000001 2023-09-23T21:15:34.101 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.101 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.101 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:15:34.101 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:12:14.345139+0000", 2023-09-23T21:15:34.101 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.8362999999999999e-05 2023-09-23T21:15:34.101 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.101 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.101 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:15:34.101 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:12:14.352724+0000", 2023-09-23T21:15:34.101 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0075854609999999999 2023-09-23T21:15:34.101 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.101 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.101 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:15:34.102 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:12:14.352765+0000", 2023-09-23T21:15:34.102 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.0368999999999999e-05 2023-09-23T21:15:34.102 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:15:34.102 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:15:34.102 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:15:34.102 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.102 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.102 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.46328.0:84587 10.0 10:01ac773b:::benchmark_data_smithi026_379838_object4630:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e472)", 2023-09-23T21:15:34.102 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:12:11.856050+0000", 2023-09-23T21:15:34.102 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 202.14026500200001, 2023-09-23T21:15:34.102 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.9827590000000002, 2023-09-23T21:15:34.103 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:15:34.103 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:15:34.103 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:15:34.103 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.46328", 2023-09-23T21:15:34.103 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/3748861181", 2023-09-23T21:15:34.103 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 84587 2023-09-23T21:15:34.103 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.103 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:15:34.103 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.103 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:15:34.103 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:12:11.856050+0000", 2023-09-23T21:15:34.103 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:15:34.103 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.104 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.104 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:15:34.104 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:12:11.856050+0000", 2023-09-23T21:15:34.104 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:15:34.104 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.104 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.104 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:15:34.104 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:12:11.856048+0000", 2023-09-23T21:15:34.104 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.999999 2023-09-23T21:15:34.104 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.104 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.105 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:15:34.105 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:12:11.856055+0000", 2023-09-23T21:15:34.105 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.883e-06 2023-09-23T21:15:34.105 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.105 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.105 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:15:34.105 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:12:11.856067+0000", 2023-09-23T21:15:34.105 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.1541e-05 2023-09-23T21:15:34.105 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.105 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.105 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:15:34.106 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:12:11.856073+0000", 2023-09-23T21:15:34.106 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.348e-06 2023-09-23T21:15:34.106 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.106 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.106 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:15:34.106 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:12:14.482257+0000", 2023-09-23T21:15:34.106 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.62618397 2023-09-23T21:15:34.106 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.106 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.106 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for missing object", 2023-09-23T21:15:34.106 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:12:14.486854+0000", 2023-09-23T21:15:34.106 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.004597126 2023-09-23T21:15:34.107 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.107 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.107 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:15:34.107 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:12:14.521249+0000", 2023-09-23T21:15:34.107 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.034394451999999999 2023-09-23T21:15:34.107 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.107 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.107 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for rw locks", 2023-09-23T21:15:34.107 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:12:14.527211+0000", 2023-09-23T21:15:34.107 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0059623009999999997 2023-09-23T21:15:34.107 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.107 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.108 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:15:34.108 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:12:14.800609+0000", 2023-09-23T21:15:34.108 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.273397942 2023-09-23T21:15:34.108 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.108 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.108 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:15:34.108 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:12:14.803332+0000", 2023-09-23T21:15:34.108 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0027228970000000002 2023-09-23T21:15:34.108 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.108 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.108 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 5,7", 2023-09-23T21:15:34.108 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:12:14.808034+0000", 2023-09-23T21:15:34.108 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0047023059999999998 2023-09-23T21:15:34.109 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.109 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.109 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:15:34.109 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:12:14.814675+0000", 2023-09-23T21:15:34.109 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0066411070000000003 2023-09-23T21:15:34.109 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.109 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.109 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:15:34.109 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:12:14.827001+0000", 2023-09-23T21:15:34.109 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.012326043 2023-09-23T21:15:34.109 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.109 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.110 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:15:34.110 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:12:14.828326+0000", 2023-09-23T21:15:34.110 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.001324483 2023-09-23T21:15:34.110 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.110 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.110 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:15:34.110 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:12:14.830512+0000", 2023-09-23T21:15:34.110 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0021857640000000002 2023-09-23T21:15:34.110 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.110 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.110 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:15:34.110 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:12:14.838809+0000", 2023-09-23T21:15:34.111 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0082969350000000001 2023-09-23T21:15:34.111 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:15:34.111 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:15:34.111 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:15:34.111 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.111 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.111 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.46328.0:84588 10.0 10:04267038:::benchmark_data_smithi026_379838_object4631:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e472)", 2023-09-23T21:15:34.111 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:12:11.858701+0000", 2023-09-23T21:15:34.111 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 202.13761371999999, 2023-09-23T21:15:34.111 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.915373185, 2023-09-23T21:15:34.111 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:15:34.111 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:15:34.111 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:15:34.112 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.46328", 2023-09-23T21:15:34.112 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/3748861181", 2023-09-23T21:15:34.112 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 84588 2023-09-23T21:15:34.112 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.112 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:15:34.112 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.112 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:15:34.112 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:12:11.858701+0000", 2023-09-23T21:15:34.112 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:15:34.112 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.112 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.112 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:15:34.113 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:12:11.858701+0000", 2023-09-23T21:15:34.113 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:15:34.113 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.113 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.113 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:15:34.113 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:12:11.858700+0000", 2023-09-23T21:15:34.113 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T21:15:34.113 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.113 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.113 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:15:34.113 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:12:11.858704+0000", 2023-09-23T21:15:34.113 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.9200000000000003e-06 2023-09-23T21:15:34.114 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.114 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.114 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:15:34.114 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:12:11.858714+0000", 2023-09-23T21:15:34.114 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.5719999999999992e-06 2023-09-23T21:15:34.114 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.114 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.114 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:15:34.114 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:12:11.858719+0000", 2023-09-23T21:15:34.114 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.3510000000000001e-06 2023-09-23T21:15:34.114 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.114 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.115 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:15:34.115 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:12:14.494615+0000", 2023-09-23T21:15:34.115 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.6358956099999999 2023-09-23T21:15:34.115 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.115 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.115 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for missing object", 2023-09-23T21:15:34.115 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:12:14.499085+0000", 2023-09-23T21:15:34.115 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0044701410000000004 2023-09-23T21:15:34.115 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.115 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.115 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:15:34.115 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:12:14.544838+0000", 2023-09-23T21:15:34.115 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.045753291000000001 2023-09-23T21:15:34.116 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.116 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.116 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for rw locks", 2023-09-23T21:15:34.116 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:12:14.550086+0000", 2023-09-23T21:15:34.116 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00524771 2023-09-23T21:15:34.116 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.116 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.116 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:15:34.116 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:12:14.703126+0000", 2023-09-23T21:15:34.116 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.15303945499999999 2023-09-23T21:15:34.116 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.116 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.117 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:15:34.117 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:12:14.708052+0000", 2023-09-23T21:15:34.117 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0049264809999999999 2023-09-23T21:15:34.117 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.117 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.117 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 5,7", 2023-09-23T21:15:34.117 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:12:14.713464+0000", 2023-09-23T21:15:34.117 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0054116909999999997 2023-09-23T21:15:34.117 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.117 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.117 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:15:34.117 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:12:14.735488+0000", 2023-09-23T21:15:34.117 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.022024681000000001 2023-09-23T21:15:34.118 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.118 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.118 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:15:34.118 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:12:14.765728+0000", 2023-09-23T21:15:34.118 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.030239499999999999 2023-09-23T21:15:34.118 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.118 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.118 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:15:34.118 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:12:14.766660+0000", 2023-09-23T21:15:34.118 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00093233900000000002 2023-09-23T21:15:34.118 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.119 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.119 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:15:34.119 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:12:14.768085+0000", 2023-09-23T21:15:34.119 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.001425001 2023-09-23T21:15:34.119 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.119 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.119 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:15:34.119 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:12:14.774074+0000", 2023-09-23T21:15:34.119 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0059887650000000001 2023-09-23T21:15:34.119 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:15:34.119 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:15:34.119 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:15:34.119 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.120 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.120 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_repop(client.46328.0:84616 10.c e472/465 10:3d3dd8d1:::benchmark_data_smithi026_379838_object4659:head v 472'5333, mlcod=470'5329)", 2023-09-23T21:15:34.120 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:12:11.948252+0000", 2023-09-23T21:15:34.120 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 202.04806288099999, 2023-09-23T21:15:34.120 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.4124408110000002, 2023-09-23T21:15:34.120 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:15:34.120 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:15:34.120 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:15:34.120 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.120 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:15:34.120 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:12:11.948252+0000", 2023-09-23T21:15:34.120 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:15:34.121 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.121 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.121 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:15:34.121 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:12:11.948252+0000", 2023-09-23T21:15:34.121 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:15:34.121 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.121 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.121 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:15:34.121 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:12:11.948251+0000", 2023-09-23T21:15:34.121 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.999999 2023-09-23T21:15:34.121 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.121 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.121 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:15:34.122 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:12:11.948260+0000", 2023-09-23T21:15:34.122 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.5789999999999993e-06 2023-09-23T21:15:34.122 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.122 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.122 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:15:34.122 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:12:11.948271+0000", 2023-09-23T21:15:34.122 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.0705e-05 2023-09-23T21:15:34.122 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.122 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.122 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:15:34.122 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:12:11.948278+0000", 2023-09-23T21:15:34.122 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.584e-06 2023-09-23T21:15:34.123 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.123 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.123 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:15:34.123 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:12:14.350144+0000", 2023-09-23T21:15:34.123 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.4018661539999999 2023-09-23T21:15:34.123 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.123 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.123 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:15:34.123 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:12:14.350169+0000", 2023-09-23T21:15:34.123 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.5304999999999999e-05 2023-09-23T21:15:34.123 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.123 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.123 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:15:34.124 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:12:14.360651+0000", 2023-09-23T21:15:34.124 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.010481656000000001 2023-09-23T21:15:34.124 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.124 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.124 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:15:34.124 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:12:14.360693+0000", 2023-09-23T21:15:34.124 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.1706999999999999e-05 2023-09-23T21:15:34.124 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:15:34.124 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:15:34.124 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:15:34.124 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.124 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.125 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_repop(client.46328.0:84636 10.c e472/465 10:3153f2e1:::benchmark_data_smithi026_379838_object4679:head v 472'5334, mlcod=470'5329)", 2023-09-23T21:15:34.125 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:12:12.046281+0000", 2023-09-23T21:15:34.125 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 201.950033808, 2023-09-23T21:15:34.125 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.3167927970000002, 2023-09-23T21:15:34.125 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:15:34.125 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:15:34.125 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:15:34.125 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.125 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:15:34.125 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:12:12.046281+0000", 2023-09-23T21:15:34.125 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:15:34.125 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.126 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.126 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:15:34.126 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:12:12.046281+0000", 2023-09-23T21:15:34.126 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:15:34.126 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.126 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.126 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:15:34.126 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:12:12.046280+0000", 2023-09-23T21:15:34.126 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T21:15:34.126 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.126 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.127 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:15:34.127 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:12:12.046291+0000", 2023-09-23T21:15:34.127 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.1494e-05 2023-09-23T21:15:34.127 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.127 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.127 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:15:34.127 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:12:12.046307+0000", 2023-09-23T21:15:34.127 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.5900999999999999e-05 2023-09-23T21:15:34.127 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.127 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.127 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:15:34.127 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:12:12.046315+0000", 2023-09-23T21:15:34.127 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.9740000000000002e-06 2023-09-23T21:15:34.128 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.128 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.128 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:15:34.128 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:12:14.355294+0000", 2023-09-23T21:15:34.128 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.308979511 2023-09-23T21:15:34.128 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.128 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.128 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:15:34.128 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:12:14.355318+0000", 2023-09-23T21:15:34.128 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.3159999999999998e-05 2023-09-23T21:15:34.128 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.128 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.129 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:15:34.129 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:12:14.363038+0000", 2023-09-23T21:15:34.129 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0077202060000000003 2023-09-23T21:15:34.129 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.129 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.129 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:15:34.129 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:12:14.363074+0000", 2023-09-23T21:15:34.129 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.5750999999999997e-05 2023-09-23T21:15:34.129 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:15:34.129 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:15:34.129 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:15:34.129 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.129 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.130 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.46328.0:99949 10.13 10:c89660e9:::benchmark_data_smithi026_379838_object19992:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e492)", 2023-09-23T21:15:34.130 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:13:11.118271+0000", 2023-09-23T21:15:34.130 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 142.878043684, 2023-09-23T21:15:34.130 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.4608746269999999, 2023-09-23T21:15:34.130 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:15:34.130 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "reached pg", 2023-09-23T21:15:34.130 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:15:34.130 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.46328", 2023-09-23T21:15:34.130 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/3748861181", 2023-09-23T21:15:34.130 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 99949 2023-09-23T21:15:34.130 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.130 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:15:34.131 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.131 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:15:34.131 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:11.118271+0000", 2023-09-23T21:15:34.131 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:15:34.131 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.131 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.131 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:15:34.131 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:11.118271+0000", 2023-09-23T21:15:34.131 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:15:34.131 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.131 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.131 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:15:34.132 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:11.118270+0000", 2023-09-23T21:15:34.132 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T21:15:34.132 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.132 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.132 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:15:34.132 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:11.118275+0000", 2023-09-23T21:15:34.132 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.9799999999999998e-06 2023-09-23T21:15:34.132 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.132 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.132 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:15:34.132 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:11.118284+0000", 2023-09-23T21:15:34.132 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.0950000000000002e-06 2023-09-23T21:15:34.132 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.133 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.133 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:15:34.133 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:11.118288+0000", 2023-09-23T21:15:34.133 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.826e-06 2023-09-23T21:15:34.133 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.133 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.133 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:15:34.133 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:11.119293+0000", 2023-09-23T21:15:34.133 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.001004291 2023-09-23T21:15:34.133 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.133 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.133 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for missing object", 2023-09-23T21:15:34.134 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:11.122652+0000", 2023-09-23T21:15:34.134 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0033593080000000001 2023-09-23T21:15:34.134 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.134 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.134 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:15:34.134 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:11.143684+0000", 2023-09-23T21:15:34.134 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.021032241 2023-09-23T21:15:34.134 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.134 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.134 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for rw locks", 2023-09-23T21:15:34.134 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:11.147201+0000", 2023-09-23T21:15:34.135 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0035162399999999999 2023-09-23T21:15:34.135 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.135 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.135 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:15:34.135 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:11.162703+0000", 2023-09-23T21:15:34.135 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.015502653999999999 2023-09-23T21:15:34.135 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.135 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.135 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:15:34.135 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:11.165461+0000", 2023-09-23T21:15:34.135 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.002758 2023-09-23T21:15:34.135 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.136 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.136 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 2,5", 2023-09-23T21:15:34.136 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:11.172337+0000", 2023-09-23T21:15:34.136 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0068755500000000002 2023-09-23T21:15:34.136 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.136 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.136 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:15:34.136 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:11.195056+0000", 2023-09-23T21:15:34.136 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.022719261000000001 2023-09-23T21:15:34.136 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.136 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.136 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:15:34.137 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:11.214701+0000", 2023-09-23T21:15:34.137 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.019645477000000001 2023-09-23T21:15:34.137 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.137 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.137 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:15:34.137 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:11.214712+0000", 2023-09-23T21:15:34.137 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.0464e-05 2023-09-23T21:15:34.137 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.137 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.137 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:15:34.137 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:12.192752+0000", 2023-09-23T21:15:34.137 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.97804052900000005 2023-09-23T21:15:34.137 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.138 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.138 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:15:34.138 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:12.192762+0000", 2023-09-23T21:15:34.138 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.9469999999999998e-06 2023-09-23T21:15:34.138 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.138 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.138 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:15:34.138 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:14.334672+0000", 2023-09-23T21:15:34.138 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.1419095559999999 2023-09-23T21:15:34.138 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.138 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.139 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for missing object", 2023-09-23T21:15:34.139 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:14.338733+0000", 2023-09-23T21:15:34.139 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0040611099999999997 2023-09-23T21:15:34.139 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.139 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.139 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:15:34.139 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:15.576529+0000", 2023-09-23T21:15:34.139 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.237796049 2023-09-23T21:15:34.139 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.139 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.139 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:15:34.139 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:15.579146+0000", 2023-09-23T21:15:34.140 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0026163979999999998 2023-09-23T21:15:34.140 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:15:34.140 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:15:34.140 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:15:34.140 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.140 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.140 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.46328.0:99952 10.1 10:86c6d668:::benchmark_data_smithi026_379838_object19995:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e492)", 2023-09-23T21:15:34.140 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:13:11.126669+0000", 2023-09-23T21:15:34.140 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 142.86964549999999, 2023-09-23T21:15:34.140 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.242467735, 2023-09-23T21:15:34.140 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:15:34.140 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:15:34.141 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:15:34.141 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.46328", 2023-09-23T21:15:34.141 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/3748861181", 2023-09-23T21:15:34.141 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 99952 2023-09-23T21:15:34.141 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.141 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:15:34.141 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.142 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:15:34.142 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:11.126669+0000", 2023-09-23T21:15:34.142 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:15:34.142 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.142 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.142 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:15:34.142 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:11.126669+0000", 2023-09-23T21:15:34.142 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:15:34.143 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.143 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.143 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:15:34.143 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:11.126667+0000", 2023-09-23T21:15:34.143 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999981 2023-09-23T21:15:34.143 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.143 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.143 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:15:34.144 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:11.126674+0000", 2023-09-23T21:15:34.144 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.9650000000000002e-06 2023-09-23T21:15:34.144 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.144 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.144 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:15:34.144 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:11.126691+0000", 2023-09-23T21:15:34.144 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.7359000000000001e-05 2023-09-23T21:15:34.144 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.145 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.145 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:15:34.145 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:11.126698+0000", 2023-09-23T21:15:34.145 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.1029999999999996e-06 2023-09-23T21:15:34.145 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.145 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.145 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:15:34.146 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:11.136504+0000", 2023-09-23T21:15:34.146 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0098059319999999998 2023-09-23T21:15:34.146 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.146 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.146 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for missing object", 2023-09-23T21:15:34.146 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:11.139550+0000", 2023-09-23T21:15:34.146 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0030458830000000001 2023-09-23T21:15:34.146 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.146 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.146 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:15:34.146 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:11.160803+0000", 2023-09-23T21:15:34.147 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.02125318 2023-09-23T21:15:34.147 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.147 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.147 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for rw locks", 2023-09-23T21:15:34.147 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:11.165287+0000", 2023-09-23T21:15:34.147 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0044835869999999998 2023-09-23T21:15:34.147 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.147 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.147 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:15:34.147 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:11.202380+0000", 2023-09-23T21:15:34.147 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.037092768999999998 2023-09-23T21:15:34.147 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.147 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.148 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:15:34.148 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:11.202394+0000", 2023-09-23T21:15:34.148 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.3949e-05 2023-09-23T21:15:34.148 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.148 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.148 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:15:34.148 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:12.193673+0000", 2023-09-23T21:15:34.148 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.99127966099999998 2023-09-23T21:15:34.148 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.148 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.148 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:15:34.148 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:12.193684+0000", 2023-09-23T21:15:34.148 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.0499000000000001e-05 2023-09-23T21:15:34.149 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.149 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.149 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:15:34.149 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:15.335139+0000", 2023-09-23T21:15:34.149 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.1414547779999999 2023-09-23T21:15:34.149 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.149 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.149 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:15:34.149 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:15.338252+0000", 2023-09-23T21:15:34.149 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.003112821 2023-09-23T21:15:34.149 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.149 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.149 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 0,7", 2023-09-23T21:15:34.150 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:15.342990+0000", 2023-09-23T21:15:34.150 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0047385450000000003 2023-09-23T21:15:34.150 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.150 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.150 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:15:34.150 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:15.355654+0000", 2023-09-23T21:15:34.150 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.012663496 2023-09-23T21:15:34.150 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.150 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.150 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:15:34.150 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:15.361946+0000", 2023-09-23T21:15:34.150 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0062922300000000002 2023-09-23T21:15:34.151 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.151 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.151 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:15:34.151 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:15.362731+0000", 2023-09-23T21:15:34.151 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00078492199999999996 2023-09-23T21:15:34.151 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.151 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.151 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:15:34.151 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:15.363991+0000", 2023-09-23T21:15:34.151 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.001260082 2023-09-23T21:15:34.151 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.151 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.152 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:15:34.152 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:15.369137+0000", 2023-09-23T21:15:34.152 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.005146014 2023-09-23T21:15:34.152 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:15:34.152 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:15:34.152 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:15:34.152 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.152 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.152 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.46328.0:99963 10.13 10:c9506fe5:::benchmark_data_smithi026_379838_object20006:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e492)", 2023-09-23T21:15:34.152 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:13:11.156341+0000", 2023-09-23T21:15:34.152 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 142.839973926, 2023-09-23T21:15:34.152 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.6561101320000002, 2023-09-23T21:15:34.153 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:15:34.153 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:15:34.153 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:15:34.153 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.46328", 2023-09-23T21:15:34.153 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/3748861181", 2023-09-23T21:15:34.153 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 99963 2023-09-23T21:15:34.153 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.153 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:15:34.153 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.153 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:15:34.153 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:11.156341+0000", 2023-09-23T21:15:34.153 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:15:34.154 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.154 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.154 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:15:34.154 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:11.156341+0000", 2023-09-23T21:15:34.154 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:15:34.154 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.154 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.154 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:15:34.154 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:11.156339+0000", 2023-09-23T21:15:34.154 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999981 2023-09-23T21:15:34.154 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.154 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.155 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:15:34.155 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:11.156347+0000", 2023-09-23T21:15:34.155 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.0439999999999996e-06 2023-09-23T21:15:34.155 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.155 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.155 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:15:34.155 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:11.156363+0000", 2023-09-23T21:15:34.155 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.5798e-05 2023-09-23T21:15:34.155 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.155 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.155 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:15:34.155 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:11.156370+0000", 2023-09-23T21:15:34.155 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.6899999999999992e-06 2023-09-23T21:15:34.156 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.156 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.156 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:15:34.156 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:11.191653+0000", 2023-09-23T21:15:34.156 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.035282273000000003 2023-09-23T21:15:34.156 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.156 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.156 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for missing object", 2023-09-23T21:15:34.156 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:11.194637+0000", 2023-09-23T21:15:34.156 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.002984586 2023-09-23T21:15:34.156 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.157 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.157 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:15:34.157 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:11.214739+0000", 2023-09-23T21:15:34.157 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.020102346 2023-09-23T21:15:34.157 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.157 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.157 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:15:34.157 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:11.214746+0000", 2023-09-23T21:15:34.157 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.9419999999999998e-06 2023-09-23T21:15:34.157 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.157 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.157 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:15:34.157 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:12.192789+0000", 2023-09-23T21:15:34.158 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.97804239400000004 2023-09-23T21:15:34.158 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.158 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.158 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:15:34.158 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:12.192796+0000", 2023-09-23T21:15:34.158 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.6290000000000001e-06 2023-09-23T21:15:34.158 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.158 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.158 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:15:34.158 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:15.579652+0000", 2023-09-23T21:15:34.158 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.3868554130000001 2023-09-23T21:15:34.158 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.159 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.159 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for missing object", 2023-09-23T21:15:34.159 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:15.583515+0000", 2023-09-23T21:15:34.159 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0038636059999999999 2023-09-23T21:15:34.159 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.159 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.159 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:15:34.159 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:15.604488+0000", 2023-09-23T21:15:34.159 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.020972781999999999 2023-09-23T21:15:34.159 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.159 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.159 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for degraded object", 2023-09-23T21:15:34.160 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:15.607187+0000", 2023-09-23T21:15:34.160 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0026986549999999999 2023-09-23T21:15:34.160 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.160 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.160 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:15:34.160 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:15.742812+0000", 2023-09-23T21:15:34.160 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.13562508700000001 2023-09-23T21:15:34.160 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.160 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.160 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:15:34.160 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:15.746122+0000", 2023-09-23T21:15:34.160 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0033102119999999999 2023-09-23T21:15:34.161 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.161 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.161 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 0,5", 2023-09-23T21:15:34.161 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:15.753005+0000", 2023-09-23T21:15:34.161 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0068824690000000004 2023-09-23T21:15:34.161 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.161 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.161 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:15:34.161 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:15.781458+0000", 2023-09-23T21:15:34.161 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.028453556000000001 2023-09-23T21:15:34.161 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.161 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.161 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:15:34.162 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:15.792765+0000", 2023-09-23T21:15:34.162 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.011306800000000001 2023-09-23T21:15:34.162 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.162 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.162 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:15:34.162 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:15.805221+0000", 2023-09-23T21:15:34.162 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.012456211 2023-09-23T21:15:34.162 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.162 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.162 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:15:34.162 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:15.806627+0000", 2023-09-23T21:15:34.162 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0014054130000000001 2023-09-23T21:15:34.163 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.163 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.163 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:15:34.163 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:15.812451+0000", 2023-09-23T21:15:34.163 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00582418 2023-09-23T21:15:34.163 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:15:34.163 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:15:34.163 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:15:34.163 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.163 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.163 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.46328.0:99965 10.16 10:68a95ca6:::benchmark_data_smithi026_379838_object20008:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e492)", 2023-09-23T21:15:34.163 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:13:11.199392+0000", 2023-09-23T21:15:34.163 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 142.79692285100001, 2023-09-23T21:15:34.164 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.2292175890000001, 2023-09-23T21:15:34.164 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:15:34.164 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:15:34.164 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:15:34.164 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.46328", 2023-09-23T21:15:34.164 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/3748861181", 2023-09-23T21:15:34.164 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 99965 2023-09-23T21:15:34.164 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.164 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:15:34.164 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.164 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:15:34.164 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:11.199392+0000", 2023-09-23T21:15:34.165 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:15:34.165 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.165 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.165 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:15:34.165 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:11.199392+0000", 2023-09-23T21:15:34.165 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:15:34.165 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.165 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.165 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:15:34.165 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:11.199390+0000", 2023-09-23T21:15:34.165 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T21:15:34.165 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.166 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.166 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:15:34.166 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:11.199396+0000", 2023-09-23T21:15:34.166 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.1240000000000002e-06 2023-09-23T21:15:34.166 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.166 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.166 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:15:34.166 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:11.199407+0000", 2023-09-23T21:15:34.166 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.0682e-05 2023-09-23T21:15:34.166 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.166 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.167 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:15:34.167 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:11.199413+0000", 2023-09-23T21:15:34.167 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.2570000000000004e-06 2023-09-23T21:15:34.167 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.167 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.167 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:15:34.168 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:11.232630+0000", 2023-09-23T21:15:34.168 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.033216351999999998 2023-09-23T21:15:34.168 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.168 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.168 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:15:34.168 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:11.232640+0000", 2023-09-23T21:15:34.168 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.0414999999999999e-05 2023-09-23T21:15:34.168 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.168 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.168 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:15:34.168 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:12.191472+0000", 2023-09-23T21:15:34.168 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.95883214699999997 2023-09-23T21:15:34.169 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.169 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.169 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:15:34.169 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:12.191481+0000", 2023-09-23T21:15:34.169 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.6649999999999992e-06 2023-09-23T21:15:34.169 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.169 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.169 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:15:34.169 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:14.423279+0000", 2023-09-23T21:15:34.169 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.2317976210000001 2023-09-23T21:15:34.169 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.170 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.170 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:15:34.170 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:14.423386+0000", 2023-09-23T21:15:34.170 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00010726399999999999 2023-09-23T21:15:34.170 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.170 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.170 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 0,5", 2023-09-23T21:15:34.170 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:14.423617+0000", 2023-09-23T21:15:34.170 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000231597 2023-09-23T21:15:34.170 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.170 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.170 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:15:34.171 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:14.425276+0000", 2023-09-23T21:15:34.171 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.001658299 2023-09-23T21:15:34.171 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.171 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.171 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:15:34.171 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:14.425806+0000", 2023-09-23T21:15:34.171 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00053022900000000001 2023-09-23T21:15:34.171 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.171 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.171 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:15:34.171 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:14.428365+0000", 2023-09-23T21:15:34.171 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.002559253 2023-09-23T21:15:34.172 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.172 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.172 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:15:34.172 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:14.428423+0000", 2023-09-23T21:15:34.172 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.7596999999999998e-05 2023-09-23T21:15:34.172 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.172 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.172 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:15:34.172 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:14.428609+0000", 2023-09-23T21:15:34.172 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00018645399999999999 2023-09-23T21:15:34.172 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:15:34.173 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:15:34.173 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:15:34.173 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.173 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.173 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.46328.0:99979 10.1 10:8711b949:::benchmark_data_smithi026_379838_object20022:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e493)", 2023-09-23T21:15:34.173 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:13:11.860281+0000", 2023-09-23T21:15:34.173 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 142.13603309999999, 2023-09-23T21:15:34.173 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.5904489540000002, 2023-09-23T21:15:34.173 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:15:34.174 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:15:34.174 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:15:34.174 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.46328", 2023-09-23T21:15:34.174 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/3748861181", 2023-09-23T21:15:34.174 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 99979 2023-09-23T21:15:34.174 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.174 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:15:34.174 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.174 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:15:34.174 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:11.860281+0000", 2023-09-23T21:15:34.175 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:15:34.175 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.175 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.175 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:15:34.175 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:11.860281+0000", 2023-09-23T21:15:34.175 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:15:34.175 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.175 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.175 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:15:34.175 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:11.860280+0000", 2023-09-23T21:15:34.175 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T21:15:34.175 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.176 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.176 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:15:34.176 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:11.860286+0000", 2023-09-23T21:15:34.176 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.2720000000000003e-06 2023-09-23T21:15:34.176 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.176 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.176 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:15:34.176 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:11.860301+0000", 2023-09-23T21:15:34.176 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.4901e-05 2023-09-23T21:15:34.176 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.176 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.176 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:15:34.177 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:11.860308+0000", 2023-09-23T21:15:34.177 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.3779999999999998e-06 2023-09-23T21:15:34.177 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.177 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.177 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:15:34.177 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:11.860364+0000", 2023-09-23T21:15:34.177 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.5866999999999999e-05 2023-09-23T21:15:34.177 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.177 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.177 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:15:34.177 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:11.860373+0000", 2023-09-23T21:15:34.177 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.8500000000000006e-06 2023-09-23T21:15:34.178 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.178 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.178 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:15:34.178 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:12.196314+0000", 2023-09-23T21:15:34.178 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.33594100999999998 2023-09-23T21:15:34.178 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.178 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.178 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:15:34.178 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:12.196326+0000", 2023-09-23T21:15:34.178 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.1333e-05 2023-09-23T21:15:34.179 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.179 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.179 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:15:34.179 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:15.369429+0000", 2023-09-23T21:15:34.179 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.1731031180000002 2023-09-23T21:15:34.179 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.179 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.179 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for missing object", 2023-09-23T21:15:34.179 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:15.372799+0000", 2023-09-23T21:15:34.179 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0033701310000000002 2023-09-23T21:15:34.179 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.179 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.180 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:15:34.180 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:15.389753+0000", 2023-09-23T21:15:34.180 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.016954440000000001 2023-09-23T21:15:34.180 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.180 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.180 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for rw locks", 2023-09-23T21:15:34.180 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:15.392893+0000", 2023-09-23T21:15:34.180 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.003139293 2023-09-23T21:15:34.180 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.180 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.180 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:15:34.180 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:15.415439+0000", 2023-09-23T21:15:34.180 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.022546297999999999 2023-09-23T21:15:34.181 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.181 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.181 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:15:34.181 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:15.417698+0000", 2023-09-23T21:15:34.181 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0022593629999999999 2023-09-23T21:15:34.181 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.181 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.181 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 0,7", 2023-09-23T21:15:34.181 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:15.422787+0000", 2023-09-23T21:15:34.181 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0050888920000000002 2023-09-23T21:15:34.182 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.182 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.182 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:15:34.182 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:15.442882+0000", 2023-09-23T21:15:34.182 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.020094441000000001 2023-09-23T21:15:34.182 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.183 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.183 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:15:34.183 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:15.443740+0000", 2023-09-23T21:15:34.183 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00085842100000000005 2023-09-23T21:15:34.183 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.183 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.184 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:15:34.184 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:15.444499+0000", 2023-09-23T21:15:34.184 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00075862800000000001 2023-09-23T21:15:34.184 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.184 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.184 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:15:34.184 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:15.445752+0000", 2023-09-23T21:15:34.184 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0012533970000000001 2023-09-23T21:15:34.184 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.184 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.185 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:15:34.185 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:15.450730+0000", 2023-09-23T21:15:34.185 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0049782749999999999 2023-09-23T21:15:34.185 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:15:34.185 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:15:34.185 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:15:34.186 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.186 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.186 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.46328.0:99984 10.13 10:c97226d3:::benchmark_data_smithi026_379838_object20027:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e493)", 2023-09-23T21:15:34.186 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:13:11.898920+0000", 2023-09-23T21:15:34.186 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 142.097394651, 2023-09-23T21:15:34.186 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.923359048, 2023-09-23T21:15:34.186 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:15:34.186 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:15:34.186 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:15:34.186 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.46328", 2023-09-23T21:15:34.186 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/3748861181", 2023-09-23T21:15:34.187 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 99984 2023-09-23T21:15:34.187 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.187 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:15:34.187 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.187 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:15:34.187 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:11.898920+0000", 2023-09-23T21:15:34.187 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:15:34.187 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.187 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.187 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:15:34.187 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:11.898920+0000", 2023-09-23T21:15:34.188 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:15:34.188 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.188 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.188 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:15:34.188 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:11.898919+0000", 2023-09-23T21:15:34.188 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.999999 2023-09-23T21:15:34.188 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.188 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.189 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:15:34.189 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:11.898923+0000", 2023-09-23T21:15:34.189 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.4410000000000003e-06 2023-09-23T21:15:34.189 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.189 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.189 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:15:34.189 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:11.898939+0000", 2023-09-23T21:15:34.189 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.5588000000000001e-05 2023-09-23T21:15:34.189 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.189 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.190 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:15:34.190 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:11.898947+0000", 2023-09-23T21:15:34.190 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.7950000000000008e-06 2023-09-23T21:15:34.190 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.190 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.190 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:15:34.190 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:11.899000+0000", 2023-09-23T21:15:34.190 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.2905999999999998e-05 2023-09-23T21:15:34.190 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.190 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.190 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:15:34.190 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:11.899012+0000", 2023-09-23T21:15:34.191 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.2668e-05 2023-09-23T21:15:34.191 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.191 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.191 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:15:34.191 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:12.192819+0000", 2023-09-23T21:15:34.191 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.29380640800000002 2023-09-23T21:15:34.191 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.191 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.191 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:15:34.191 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:12.192826+0000", 2023-09-23T21:15:34.191 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.1439999999999997e-06 2023-09-23T21:15:34.191 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.191 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.192 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:15:34.192 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:15.584066+0000", 2023-09-23T21:15:34.192 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.3912398499999998 2023-09-23T21:15:34.192 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.192 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.192 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for missing object", 2023-09-23T21:15:34.192 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:15.587625+0000", 2023-09-23T21:15:34.192 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0035591070000000002 2023-09-23T21:15:34.192 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.192 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.192 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:15:34.193 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:15.621463+0000", 2023-09-23T21:15:34.193 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.033837837000000003 2023-09-23T21:15:34.193 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.193 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.193 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for degraded object", 2023-09-23T21:15:34.193 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:15.624790+0000", 2023-09-23T21:15:34.193 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0033272829999999999 2023-09-23T21:15:34.193 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.193 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.193 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:15:34.194 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:15.769569+0000", 2023-09-23T21:15:34.194 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.14477948299999999 2023-09-23T21:15:34.194 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.194 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.194 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:15:34.194 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:15.772068+0000", 2023-09-23T21:15:34.194 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0024988359999999999 2023-09-23T21:15:34.194 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.194 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.194 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 0,5", 2023-09-23T21:15:34.194 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:15.777050+0000", 2023-09-23T21:15:34.195 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0049818029999999999 2023-09-23T21:15:34.195 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.195 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.195 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:15:34.195 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:15.812721+0000", 2023-09-23T21:15:34.195 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.035671139999999997 2023-09-23T21:15:34.196 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.196 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.196 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:15:34.196 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:15.813568+0000", 2023-09-23T21:15:34.196 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00084671699999999998 2023-09-23T21:15:34.196 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.197 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.197 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:15:34.197 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:15.814693+0000", 2023-09-23T21:15:34.197 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0011254850000000001 2023-09-23T21:15:34.197 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.197 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.197 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:15:34.198 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:15.815984+0000", 2023-09-23T21:15:34.198 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.001290329 2023-09-23T21:15:34.198 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.198 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.198 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:15:34.198 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:15.822279+0000", 2023-09-23T21:15:34.199 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0062953109999999996 2023-09-23T21:15:34.199 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:15:34.199 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:15:34.199 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:15:34.199 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.199 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.199 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.46328.0:99987 10.13 10:c9fb87f0:::benchmark_data_smithi026_379838_object20030:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e493)", 2023-09-23T21:15:34.200 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:13:11.949115+0000", 2023-09-23T21:15:34.200 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 142.047199933, 2023-09-23T21:15:34.200 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.882972664, 2023-09-23T21:15:34.200 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:15:34.200 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:15:34.200 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:15:34.200 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.46328", 2023-09-23T21:15:34.201 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/3748861181", 2023-09-23T21:15:34.201 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 99987 2023-09-23T21:15:34.201 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.201 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:15:34.201 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.201 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:15:34.201 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:11.949115+0000", 2023-09-23T21:15:34.201 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:15:34.202 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.202 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.202 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:15:34.202 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:11.949115+0000", 2023-09-23T21:15:34.202 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:15:34.202 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.202 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.202 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:15:34.203 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:11.949113+0000", 2023-09-23T21:15:34.203 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.999999 2023-09-23T21:15:34.203 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.203 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.203 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:15:34.203 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:11.949119+0000", 2023-09-23T21:15:34.203 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.4190000000000001e-06 2023-09-23T21:15:34.203 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.204 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.204 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:15:34.204 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:11.949132+0000", 2023-09-23T21:15:34.204 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.2986000000000001e-05 2023-09-23T21:15:34.204 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.204 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.205 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:15:34.205 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:11.949137+0000", 2023-09-23T21:15:34.205 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.6019999999999999e-06 2023-09-23T21:15:34.205 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.205 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.205 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:15:34.205 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:11.949210+0000", 2023-09-23T21:15:34.205 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.2071999999999997e-05 2023-09-23T21:15:34.205 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.205 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.205 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:15:34.205 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:11.949219+0000", 2023-09-23T21:15:34.206 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.0410000000000003e-06 2023-09-23T21:15:34.206 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.206 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.206 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:15:34.206 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:12.192846+0000", 2023-09-23T21:15:34.206 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.243627597 2023-09-23T21:15:34.206 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.206 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.206 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:15:34.206 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:12.192853+0000", 2023-09-23T21:15:34.206 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.0169999999999999e-06 2023-09-23T21:15:34.206 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.207 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.207 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:15:34.207 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:15.588023+0000", 2023-09-23T21:15:34.207 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.3951699780000002 2023-09-23T21:15:34.207 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.207 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.207 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for missing object", 2023-09-23T21:15:34.207 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:15.591088+0000", 2023-09-23T21:15:34.207 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0030649900000000001 2023-09-23T21:15:34.207 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.207 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.207 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:15:34.207 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:15.640214+0000", 2023-09-23T21:15:34.208 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.049126213000000002 2023-09-23T21:15:34.208 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.208 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.208 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for degraded object", 2023-09-23T21:15:34.208 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:15.642230+0000", 2023-09-23T21:15:34.208 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0020155440000000002 2023-09-23T21:15:34.208 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.208 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.208 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:15:34.208 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:15.793072+0000", 2023-09-23T21:15:34.208 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.15084209300000001 2023-09-23T21:15:34.208 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.208 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.209 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:15:34.209 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:15.795467+0000", 2023-09-23T21:15:34.209 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.002394752 2023-09-23T21:15:34.209 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.209 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.209 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 0,5", 2023-09-23T21:15:34.209 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:15.800354+0000", 2023-09-23T21:15:34.209 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0048867590000000001 2023-09-23T21:15:34.209 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.209 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.209 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:15:34.210 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:15.813842+0000", 2023-09-23T21:15:34.210 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.013488557999999999 2023-09-23T21:15:34.210 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.210 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.210 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:15:34.210 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:15.823275+0000", 2023-09-23T21:15:34.210 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0094329649999999998 2023-09-23T21:15:34.210 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.210 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.211 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:15:34.211 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:15.824202+0000", 2023-09-23T21:15:34.211 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00092731499999999995 2023-09-23T21:15:34.211 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.211 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.211 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:15:34.211 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:15.825761+0000", 2023-09-23T21:15:34.211 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0015584139999999999 2023-09-23T21:15:34.211 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:34.211 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:34.211 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:15:34.211 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:15.832087+0000", 2023-09-23T21:15:34.211 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0063265379999999996 2023-09-23T21:15:34.212 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:15:34.212 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:15:34.212 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:15:34.212 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:15:34.212 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:15:34.212 INFO:teuthology.orchestra.run.smithi070.stdout:} 2023-09-23T21:15:34.256 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:15:34.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:15:34 smithi070 ceph-mon[168282]: 1.d scrub starts 2023-09-23T21:15:34.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:15:34 smithi070 ceph-mon[168282]: 1.d scrub ok 2023-09-23T21:15:34.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:15:34 smithi070 ceph-mon[168282]: pgmap v1380: 119 pgs: 119 active+clean; 674 KiB data, 4.4 GiB used, 532 GiB / 536 GiB avail 2023-09-23T21:15:34.381 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:15:34 smithi026 ceph-mon[179688]: 1.d scrub starts 2023-09-23T21:15:34.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:15:34 smithi026 ceph-mon[179688]: 1.d scrub ok 2023-09-23T21:15:34.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:15:34 smithi026 ceph-mon[179688]: pgmap v1380: 119 pgs: 119 active+clean; 674 KiB data, 4.4 GiB used, 532 GiB / 536 GiB avail 2023-09-23T21:15:34.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:15:34 smithi026 ceph-mon[175151]: 1.d scrub starts 2023-09-23T21:15:34.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:15:34 smithi026 ceph-mon[175151]: 1.d scrub ok 2023-09-23T21:15:34.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:15:34 smithi026 ceph-mon[175151]: pgmap v1380: 119 pgs: 119 active+clean; 674 KiB data, 4.4 GiB used, 532 GiB / 536 GiB avail 2023-09-23T21:15:34.382 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:15:34.385 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:15:34.386 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:15:34.444 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 2 16 5559 5543 172.986 169 0.00192728 0.00576832 2023-09-23T21:15:34.477 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_ops_in_flight 2023-09-23T21:15:34.486 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:15:34.552 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T21:15:34.576 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:15:34.636 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:15:34.637 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:15:34.638 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:15:34.738 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:15:34.765 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:15:34.777 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:15:34.820 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:15:34.825 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:15:34.825 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:15:34.926 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:15:34.988 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:15:35.205 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:15:35.209 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:15:35.210 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:15:35.211 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T21:15:35.310 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:15:35.345 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:15:35.445 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 3 16 8218 8202 170.716 166.188 0.00738585 0.0058451 2023-09-23T21:15:35.446 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:15:35.448 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:15:35.449 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:15:35.489 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:15:35 smithi026 ceph-mon[175151]: 1.1 scrub starts 2023-09-23T21:15:35.489 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:15:35 smithi026 ceph-mon[175151]: 1.1 scrub ok 2023-09-23T21:15:35.489 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:15:35 smithi026 ceph-mon[179688]: 1.1 scrub starts 2023-09-23T21:15:35.490 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:15:35 smithi026 ceph-mon[179688]: 1.1 scrub ok 2023-09-23T21:15:35.490 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:15:35 smithi070 ceph-mon[168282]: 1.1 scrub starts 2023-09-23T21:15:35.490 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:15:35 smithi070 ceph-mon[168282]: 1.1 scrub ok 2023-09-23T21:15:35.549 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:15:35.590 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:15:35.642 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:15:35.648 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:15:35.649 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:15:35.749 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:15:35.785 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:15:35.978 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:15:35.983 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:15:35.984 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:15:36.085 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:15:36.179 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:15:36.253 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:15:36.261 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:15:36.261 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:15:36.325 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:15:36.362 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:15:36.393 INFO:teuthology.orchestra.run.smithi026.stdout: 2023-09-23T21:15:36.406 INFO:teuthology.orchestra.run.smithi026.stderr:dumped all 2023-09-23T21:15:36.428 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:15:36 smithi026 ceph-mon[175151]: pgmap v1381: 119 pgs: 119 active+clean; 74 MiB data, 4.5 GiB used, 532 GiB / 536 GiB avail; 9.2 MiB/s wr, 146 op/s 2023-09-23T21:15:36.428 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:15:36 smithi026 ceph-mon[179688]: pgmap v1381: 119 pgs: 119 active+clean; 74 MiB data, 4.5 GiB used, 532 GiB / 536 GiB avail; 9.2 MiB/s wr, 146 op/s 2023-09-23T21:15:36.438 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:15:36.444 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 4 16 10543 10527 164.342 145.312 0.00495654 0.00606872 2023-09-23T21:15:36.541 INFO:teuthology.orchestra.run.smithi070.stdout:{ 2023-09-23T21:15:36.541 INFO:teuthology.orchestra.run.smithi070.stdout: "ops": [ 2023-09-23T21:15:36.541 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:36.541 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.46823.0:10781 11.5 11:a24573d8:::benchmark_data_smithi026_424533_object10780:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e548)", 2023-09-23T21:15:36.541 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:15:36.536645+0000", 2023-09-23T21:15:36.541 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 0.0031683219999999999, 2023-09-23T21:15:36.541 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0031923920000000001, 2023-09-23T21:15:36.541 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:15:36.542 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "waiting for sub ops", 2023-09-23T21:15:36.542 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:15:36.542 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.46823", 2023-09-23T21:15:36.542 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/2490075711", 2023-09-23T21:15:36.542 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 10781 2023-09-23T21:15:36.542 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:36.542 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:15:36.542 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:36.542 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:15:36.542 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:15:36.536645+0000", 2023-09-23T21:15:36.542 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:15:36.542 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:36.543 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:36.543 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:15:36.543 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:15:36.536645+0000", 2023-09-23T21:15:36.543 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:15:36.543 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:36.543 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:36.543 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:15:36.543 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:15:36.536644+0000", 2023-09-23T21:15:36.543 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T21:15:36.543 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:36.543 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:36.543 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:15:36.544 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:15:36.536695+0000", 2023-09-23T21:15:36.544 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.1521999999999999e-05 2023-09-23T21:15:36.544 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:36.544 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:36.544 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:15:36.544 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:15:36.536710+0000", 2023-09-23T21:15:36.544 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.4646e-05 2023-09-23T21:15:36.544 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:36.544 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:36.544 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:15:36.544 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:15:36.536716+0000", 2023-09-23T21:15:36.544 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.2620000000000004e-06 2023-09-23T21:15:36.545 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:36.545 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:36.545 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:15:36.545 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:15:36.536761+0000", 2023-09-23T21:15:36.545 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.4728000000000003e-05 2023-09-23T21:15:36.545 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:36.545 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:36.545 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:15:36.545 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:15:36.536888+0000", 2023-09-23T21:15:36.545 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000127257 2023-09-23T21:15:36.545 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:36.545 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:36.546 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 3,7", 2023-09-23T21:15:36.546 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:15:36.537028+0000", 2023-09-23T21:15:36.546 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00014000699999999999 2023-09-23T21:15:36.546 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:36.546 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:36.546 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:15:36.546 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:15:36.537710+0000", 2023-09-23T21:15:36.546 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00068247000000000002 2023-09-23T21:15:36.546 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:36.546 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:36.546 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:15:36.546 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:15:36.538004+0000", 2023-09-23T21:15:36.547 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00029401500000000001 2023-09-23T21:15:36.547 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:15:36.547 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:15:36.547 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:15:36.547 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:36.547 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:36.547 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_repop(client.46823.0:10780 11.a e548/544 11:5548522c:::benchmark_data_smithi026_424533_object10779:head v 548'680, mlcod=548'677)", 2023-09-23T21:15:36.547 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:15:36.539432+0000", 2023-09-23T21:15:36.547 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 0.00038145300000000002, 2023-09-23T21:15:36.547 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00046180599999999999, 2023-09-23T21:15:36.547 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:15:36.547 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "started", 2023-09-23T21:15:36.547 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:15:36.548 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:36.548 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:15:36.548 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:15:36.539432+0000", 2023-09-23T21:15:36.548 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:15:36.548 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:36.548 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:36.548 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:15:36.548 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:15:36.539432+0000", 2023-09-23T21:15:36.548 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:15:36.548 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:36.548 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:36.549 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:15:36.549 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:15:36.539431+0000", 2023-09-23T21:15:36.549 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999995 2023-09-23T21:15:36.549 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:36.549 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:36.549 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:15:36.549 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:15:36.539489+0000", 2023-09-23T21:15:36.549 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.7493e-05 2023-09-23T21:15:36.549 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:36.549 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:36.549 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:15:36.549 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:15:36.539507+0000", 2023-09-23T21:15:36.549 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.8085e-05 2023-09-23T21:15:36.550 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:36.550 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:36.550 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:15:36.550 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:15:36.539511+0000", 2023-09-23T21:15:36.550 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.155e-06 2023-09-23T21:15:36.550 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:36.550 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:36.550 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:15:36.550 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:15:36.539560+0000", 2023-09-23T21:15:36.550 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.8897999999999998e-05 2023-09-23T21:15:36.550 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:36.550 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:36.551 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:15:36.551 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:15:36.539576+0000", 2023-09-23T21:15:36.551 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.5981000000000002e-05 2023-09-23T21:15:36.551 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:15:36.551 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:15:36.551 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:15:36.551 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:36.551 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:36.551 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_repop(client.46823.0:10784 11.a e548/544 11:547443d3:::benchmark_data_smithi026_424533_object10783:head v 548'681, mlcod=548'677)", 2023-09-23T21:15:36.551 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:15:36.539529+0000", 2023-09-23T21:15:36.551 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 0.00028400499999999999, 2023-09-23T21:15:36.551 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000429243, 2023-09-23T21:15:36.552 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:15:36.552 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "started", 2023-09-23T21:15:36.552 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:15:36.552 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:36.552 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:15:36.552 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:15:36.539529+0000", 2023-09-23T21:15:36.552 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:15:36.552 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:36.552 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:36.552 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:15:36.552 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:15:36.539529+0000", 2023-09-23T21:15:36.552 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:15:36.553 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:36.553 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:36.553 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:15:36.553 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:15:36.539529+0000", 2023-09-23T21:15:36.553 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999995 2023-09-23T21:15:36.553 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:36.553 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:36.553 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:15:36.553 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:15:36.539556+0000", 2023-09-23T21:15:36.553 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.6976000000000001e-05 2023-09-23T21:15:36.553 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:36.553 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:36.553 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:15:36.554 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:15:36.539560+0000", 2023-09-23T21:15:36.554 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.3610000000000003e-06 2023-09-23T21:15:36.554 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:36.554 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:36.554 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:15:36.554 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:15:36.539563+0000", 2023-09-23T21:15:36.554 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.5189999999999999e-06 2023-09-23T21:15:36.554 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:36.554 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:36.554 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:15:36.554 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:15:36.539918+0000", 2023-09-23T21:15:36.555 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00035557500000000001 2023-09-23T21:15:36.555 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:36.555 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:36.555 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:15:36.555 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:15:36.539934+0000", 2023-09-23T21:15:36.555 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.5468000000000002e-05 2023-09-23T21:15:36.555 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:15:36.555 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:15:36.555 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:15:36.555 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:36.555 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:36.555 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_repop(client.46823.0:10787 11.d e548/544 11:b08ff84d:::benchmark_data_smithi026_424533_object10786:head v 548'726, mlcod=548'724)", 2023-09-23T21:15:36.556 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:15:36.539582+0000", 2023-09-23T21:15:36.556 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 0.00023148299999999999, 2023-09-23T21:15:36.556 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00050873300000000004, 2023-09-23T21:15:36.556 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:15:36.556 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "started", 2023-09-23T21:15:36.556 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:15:36.556 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:36.556 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:15:36.556 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:15:36.539582+0000", 2023-09-23T21:15:36.556 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:15:36.556 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:36.556 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:36.557 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:15:36.557 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:15:36.539582+0000", 2023-09-23T21:15:36.557 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:15:36.557 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:36.557 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:36.557 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:15:36.557 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:15:36.539581+0000", 2023-09-23T21:15:36.557 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967296 2023-09-23T21:15:36.557 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:36.557 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:36.557 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:15:36.557 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:15:36.539602+0000", 2023-09-23T21:15:36.557 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.0795e-05 2023-09-23T21:15:36.558 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:36.558 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:36.558 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:15:36.558 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:15:36.539606+0000", 2023-09-23T21:15:36.558 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.036e-06 2023-09-23T21:15:36.558 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:36.558 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:36.558 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:15:36.558 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:15:36.539609+0000", 2023-09-23T21:15:36.558 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.373e-06 2023-09-23T21:15:36.558 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:36.558 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:36.559 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:15:36.559 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:15:36.539638+0000", 2023-09-23T21:15:36.559 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.9445e-05 2023-09-23T21:15:36.559 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:36.559 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:36.559 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:15:36.559 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:15:36.539655+0000", 2023-09-23T21:15:36.559 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.7088999999999999e-05 2023-09-23T21:15:36.559 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:15:36.559 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:15:36.559 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:15:36.559 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:15:36.559 INFO:teuthology.orchestra.run.smithi070.stdout: ], 2023-09-23T21:15:36.560 INFO:teuthology.orchestra.run.smithi070.stdout: "num_ops": 4 2023-09-23T21:15:36.560 INFO:teuthology.orchestra.run.smithi070.stdout:} 2023-09-23T21:15:36.564 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:15:36 smithi070 ceph-mon[168282]: pgmap v1381: 119 pgs: 119 active+clean; 74 MiB data, 4.5 GiB used, 532 GiB / 536 GiB avail; 9.2 MiB/s wr, 146 op/s 2023-09-23T21:15:36.914 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_blocked_ops 2023-09-23T21:15:37.060 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:15:37.068 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:15:37.069 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:15:37.169 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:15:37.222 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:15:37.275 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:15:37.279 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:15:37.279 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:15:37.378 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:15:37 smithi026 ceph-mon[179688]: 1.b scrub starts 2023-09-23T21:15:37.378 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:15:37 smithi026 ceph-mon[179688]: 1.b scrub ok 2023-09-23T21:15:37.378 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:15:37 smithi026 ceph-mon[179688]: from='client.46829 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T21:15:37.379 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:15:37 smithi026 ceph-mon[175151]: 1.b scrub starts 2023-09-23T21:15:37.379 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:15:37 smithi026 ceph-mon[175151]: 1.b scrub ok 2023-09-23T21:15:37.379 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:15:37 smithi026 ceph-mon[175151]: from='client.46829 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T21:15:37.380 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:15:37.441 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:15:37.444 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 5 16 12962 12946 161.709 151.188 0.00802528 0.00617833 2023-09-23T21:15:37.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:15:37 smithi070 ceph-mon[168282]: 1.b scrub starts 2023-09-23T21:15:37.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:15:37 smithi070 ceph-mon[168282]: 1.b scrub ok 2023-09-23T21:15:37.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:15:37 smithi070 ceph-mon[168282]: from='client.46829 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T21:15:37.676 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T21:15:38.445 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 6 16 15611 15595 162.348 165.562 0.0103654 0.00615384 2023-09-23T21:15:38.469 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:15:38 smithi070 ceph-mon[168282]: 1.8 scrub starts 2023-09-23T21:15:38.469 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:15:38 smithi070 ceph-mon[168282]: 1.8 scrub ok 2023-09-23T21:15:38.469 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:15:38 smithi070 ceph-mon[168282]: 1.15 scrub starts 2023-09-23T21:15:38.469 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:15:38 smithi070 ceph-mon[168282]: 1.15 scrub ok 2023-09-23T21:15:38.469 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:15:38 smithi070 ceph-mon[168282]: pgmap v1382: 119 pgs: 119 active+clean; 436 MiB data, 5.1 GiB used, 531 GiB / 536 GiB avail; 54 MiB/s wr, 868 op/s 2023-09-23T21:15:38.469 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:15:38 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:15:38.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:15:38 smithi026 ceph-mon[179688]: 1.8 scrub starts 2023-09-23T21:15:38.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:15:38 smithi026 ceph-mon[179688]: 1.8 scrub ok 2023-09-23T21:15:38.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:15:38 smithi026 ceph-mon[179688]: 1.15 scrub starts 2023-09-23T21:15:38.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:15:38 smithi026 ceph-mon[179688]: 1.15 scrub ok 2023-09-23T21:15:38.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:15:38 smithi026 ceph-mon[179688]: pgmap v1382: 119 pgs: 119 active+clean; 436 MiB data, 5.1 GiB used, 531 GiB / 536 GiB avail; 54 MiB/s wr, 868 op/s 2023-09-23T21:15:38.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:15:38 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:15:38.633 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:15:38 smithi026 ceph-mon[175151]: 1.8 scrub starts 2023-09-23T21:15:38.633 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:15:38 smithi026 ceph-mon[175151]: 1.8 scrub ok 2023-09-23T21:15:38.633 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:15:38 smithi026 ceph-mon[175151]: 1.15 scrub starts 2023-09-23T21:15:38.634 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:15:38 smithi026 ceph-mon[175151]: 1.15 scrub ok 2023-09-23T21:15:38.634 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:15:38 smithi026 ceph-mon[175151]: pgmap v1382: 119 pgs: 119 active+clean; 436 MiB data, 5.1 GiB used, 531 GiB / 536 GiB avail; 54 MiB/s wr, 868 op/s 2023-09-23T21:15:38.634 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:15:38 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:15:39.138 INFO:teuthology.orchestra.run.smithi070.stdout:{ 2023-09-23T21:15:39.138 INFO:teuthology.orchestra.run.smithi070.stdout: "ops": [], 2023-09-23T21:15:39.138 INFO:teuthology.orchestra.run.smithi070.stdout: "complaint_time": 30, 2023-09-23T21:15:39.138 INFO:teuthology.orchestra.run.smithi070.stdout: "num_blocked_ops": 0 2023-09-23T21:15:39.138 INFO:teuthology.orchestra.run.smithi070.stdout:} 2023-09-23T21:15:39.311 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:15:39.316 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:15:39.316 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:15:39.417 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:15:39.444 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 7 16 17944 17928 159.985 145.812 0.00145795 0.00620025 2023-09-23T21:15:39.445 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:15:39.584 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_historic_ops 2023-09-23T21:15:39.821 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:15:39.831 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:15:39.831 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:15:39.843 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 21:15:39 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[164029]: ::ffff:172.21.15.70 - - [23/Sep/2023:21:15:39] "GET /metrics HTTP/1.1" 200 36305 "" "Prometheus/2.43.0" 2023-09-23T21:15:39.871 INFO:teuthology.orchestra.run.smithi026.stdout:{"pg_ready":true,"pg_map":{"version":1381,"stamp":"2023-09-23T21:15:35.100130+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":77727362,"num_objects":1199,"num_object_clones":0,"num_object_copies":3597,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1199,"num_whiteouts":0,"num_read":78,"num_read_kb":144,"num_write":1272,"num_write_kb":75486,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":63,"num_bytes_recovered":717192,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6219,"ondisk_log_size":6219,"up":357,"acting":357,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":378,"num_osds":6,"num_per_pool_osds":6,"num_per_pool_omap_osds":6,"kb":562470912,"kb_used":4731888,"kb_used_data":1341440,"kb_used_omap":125,"kb_used_meta":3390274,"kb_avail":557739024,"statfs":{"total":575970213888,"available":571124760576,"internally_reserved":0,"allocated":1373634560,"data_stored":1361308381,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":128662,"internal_metadata":3471640938},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":77037568,"num_objects":1174,"num_object_clones":0,"num_object_copies":3522,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1174,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1173,"num_write_kb":75072,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":-1,"num_bytes_recovered":-491520,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"8.002208"},"pg_stats":[{"pgid":"1.18","version":"79'161","reported_seq":514,"reported_epoch":547,"state":"active+clean","last_fresh":"2023-09-23T21:15:30.549345+0000","last_change":"2023-09-23T21:15:23.801613+0000","last_active":"2023-09-23T21:15:30.549345+0000","last_peered":"2023-09-23T21:15:30.549345+0000","last_clean":"2023-09-23T21:15:30.549345+0000","last_became_active":"2023-09-23T21:15:23.801118+0000","last_became_peered":"2023-09-23T21:15:23.801118+0000","last_unstale":"2023-09-23T21:15:30.549345+0000","last_undegraded":"2023-09-23T21:15:30.549345+0000","last_fullsized":"2023-09-23T21:15:30.549345+0000","mapping_epoch":542,"log_start":"0'0","ondisk_log_start":"0'0","created":479,"last_epoch_clean":543,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:06:52.098882+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:06:52.098882+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0082644970000000009,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":131344,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":262688,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1a","version":"68'1","reported_seq":138,"reported_epoch":547,"state":"active+clean","last_fresh":"2023-09-23T21:15:30.212230+0000","last_change":"2023-09-23T21:13:13.291403+0000","last_active":"2023-09-23T21:15:30.212230+0000","last_peered":"2023-09-23T21:15:30.212230+0000","last_clean":"2023-09-23T21:15:30.212230+0000","last_became_active":"2023-09-23T21:13:13.291123+0000","last_became_peered":"2023-09-23T21:13:13.291123+0000","last_unstale":"2023-09-23T21:15:30.212230+0000","last_undegraded":"2023-09-23T21:15:30.212230+0000","last_fullsized":"2023-09-23T21:15:30.212230+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":495,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:45:15.427430+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,0],"acting":[4,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":207,"reported_epoch":547,"state":"active+clean","last_fresh":"2023-09-23T21:15:30.212644+0000","last_change":"2023-09-23T21:12:10.767963+0000","last_active":"2023-09-23T21:15:30.212644+0000","last_peered":"2023-09-23T21:15:30.212644+0000","last_clean":"2023-09-23T21:15:30.212644+0000","last_became_active":"2023-09-23T21:12:10.767126+0000","last_became_peered":"2023-09-23T21:12:10.767126+0000","last_unstale":"2023-09-23T21:15:30.212644+0000","last_undegraded":"2023-09-23T21:15:30.212644+0000","last_fullsized":"2023-09-23T21:15:30.212644+0000","mapping_epoch":470,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:45:47.138277+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,5],"acting":[4,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1b","version":"79'161","reported_seq":725,"reported_epoch":547,"state":"active+clean","last_fresh":"2023-09-23T21:15:30.550277+0000","last_change":"2023-09-23T21:15:23.802986+0000","last_active":"2023-09-23T21:15:30.550277+0000","last_peered":"2023-09-23T21:15:30.550277+0000","last_clean":"2023-09-23T21:15:30.550277+0000","last_became_active":"2023-09-23T21:15:23.802580+0000","last_became_peered":"2023-09-23T21:15:23.802580+0000","last_unstale":"2023-09-23T21:15:30.550277+0000","last_undegraded":"2023-09-23T21:15:30.550277+0000","last_fullsized":"2023-09-23T21:15:30.550277+0000","mapping_epoch":542,"log_start":"0'0","ondisk_log_start":"0'0","created":479,"last_epoch_clean":543,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:05:55.772264+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:05:55.772264+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0043894679999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":544,"reported_epoch":547,"state":"active+clean","last_fresh":"2023-09-23T21:15:30.965605+0000","last_change":"2023-09-23T21:12:10.824116+0000","last_active":"2023-09-23T21:15:30.965605+0000","last_peered":"2023-09-23T21:15:30.965605+0000","last_clean":"2023-09-23T21:15:30.965605+0000","last_became_active":"2023-09-23T21:12:10.823849+0000","last_became_peered":"2023-09-23T21:12:10.823849+0000","last_unstale":"2023-09-23T21:15:30.965605+0000","last_undegraded":"2023-09-23T21:15:30.965605+0000","last_fullsized":"2023-09-23T21:15:30.965605+0000","mapping_epoch":470,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:54:13.777603+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":543,"reported_epoch":545,"state":"active+clean","last_fresh":"2023-09-23T21:15:25.899872+0000","last_change":"2023-09-23T21:13:44.413713+0000","last_active":"2023-09-23T21:15:25.899872+0000","last_peered":"2023-09-23T21:15:25.899872+0000","last_clean":"2023-09-23T21:15:25.899872+0000","last_became_active":"2023-09-23T21:13:44.413366+0000","last_became_peered":"2023-09-23T21:13:44.413366+0000","last_unstale":"2023-09-23T21:15:25.899872+0000","last_undegraded":"2023-09-23T21:15:25.899872+0000","last_fullsized":"2023-09-23T21:15:25.899872+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:31:56.349530+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1a","version":"79'161","reported_seq":595,"reported_epoch":547,"state":"active+clean","last_fresh":"2023-09-23T21:15:30.211786+0000","last_change":"2023-09-23T21:15:23.800891+0000","last_active":"2023-09-23T21:15:30.211786+0000","last_peered":"2023-09-23T21:15:30.211786+0000","last_clean":"2023-09-23T21:15:30.211786+0000","last_became_active":"2023-09-23T21:15:23.800519+0000","last_became_peered":"2023-09-23T21:15:23.800519+0000","last_unstale":"2023-09-23T21:15:30.211786+0000","last_undegraded":"2023-09-23T21:15:30.211786+0000","last_fullsized":"2023-09-23T21:15:30.211786+0000","mapping_epoch":542,"log_start":"0'0","ondisk_log_start":"0'0","created":479,"last_epoch_clean":543,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:06:51.075448+0000","last_deep_scrub":"79'161","last_deep_scrub_stamp":"2023-09-23T21:05:32.630292+0000","last_clean_scrub_stamp":"2023-09-23T21:06:51.075448+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0054793890000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,5],"acting":[4,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":55,"reported_epoch":545,"state":"active+clean","last_fresh":"2023-09-23T21:15:25.890039+0000","last_change":"2023-09-23T21:13:30.806084+0000","last_active":"2023-09-23T21:15:25.890039+0000","last_peered":"2023-09-23T21:15:25.890039+0000","last_clean":"2023-09-23T21:15:25.890039+0000","last_became_active":"2023-09-23T21:13:30.805743+0000","last_became_peered":"2023-09-23T21:13:30.805743+0000","last_unstale":"2023-09-23T21:15:25.890039+0000","last_undegraded":"2023-09-23T21:15:25.890039+0000","last_fullsized":"2023-09-23T21:15:25.890039+0000","mapping_epoch":501,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:48:10.849281+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":420,"reported_epoch":545,"state":"active+clean","last_fresh":"2023-09-23T21:15:25.892540+0000","last_change":"2023-09-23T21:13:44.410921+0000","last_active":"2023-09-23T21:15:25.892540+0000","last_peered":"2023-09-23T21:15:25.892540+0000","last_clean":"2023-09-23T21:15:25.892540+0000","last_became_active":"2023-09-23T21:13:44.410633+0000","last_became_peered":"2023-09-23T21:13:44.410633+0000","last_unstale":"2023-09-23T21:15:25.892540+0000","last_undegraded":"2023-09-23T21:15:25.892540+0000","last_fullsized":"2023-09-23T21:15:25.892540+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:57:10.116786+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.15","version":"79'161","reported_seq":514,"reported_epoch":547,"state":"active+clean","last_fresh":"2023-09-23T21:15:30.213685+0000","last_change":"2023-09-23T21:15:23.799841+0000","last_active":"2023-09-23T21:15:30.213685+0000","last_peered":"2023-09-23T21:15:30.213685+0000","last_clean":"2023-09-23T21:15:30.213685+0000","last_became_active":"2023-09-23T21:15:23.799409+0000","last_became_peered":"2023-09-23T21:15:23.799409+0000","last_unstale":"2023-09-23T21:15:30.213685+0000","last_undegraded":"2023-09-23T21:15:30.213685+0000","last_fullsized":"2023-09-23T21:15:30.213685+0000","mapping_epoch":542,"log_start":"0'0","ondisk_log_start":"0'0","created":479,"last_epoch_clean":543,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:09:34.892947+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:09:34.892947+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0081306369999999996,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,3],"acting":[7,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":520,"reported_epoch":545,"state":"active+clean","last_fresh":"2023-09-23T21:15:25.892696+0000","last_change":"2023-09-23T21:13:12.235728+0000","last_active":"2023-09-23T21:15:25.892696+0000","last_peered":"2023-09-23T21:15:25.892696+0000","last_clean":"2023-09-23T21:15:25.892696+0000","last_became_active":"2023-09-23T21:13:12.216549+0000","last_became_peered":"2023-09-23T21:13:12.216549+0000","last_unstale":"2023-09-23T21:15:25.892696+0000","last_undegraded":"2023-09-23T21:15:25.892696+0000","last_fullsized":"2023-09-23T21:15:25.892696+0000","mapping_epoch":493,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:39:33.324239+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":586,"reported_epoch":547,"state":"active+clean","last_fresh":"2023-09-23T21:15:30.965565+0000","last_change":"2023-09-23T21:12:10.790349+0000","last_active":"2023-09-23T21:15:30.965565+0000","last_peered":"2023-09-23T21:15:30.965565+0000","last_clean":"2023-09-23T21:15:30.965565+0000","last_became_active":"2023-09-23T21:12:10.789977+0000","last_became_peered":"2023-09-23T21:12:10.789977+0000","last_unstale":"2023-09-23T21:15:30.965565+0000","last_undegraded":"2023-09-23T21:15:30.965565+0000","last_fullsized":"2023-09-23T21:15:30.965565+0000","mapping_epoch":470,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:49:40.191147+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,5],"acting":[3,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"79'161","reported_seq":908,"reported_epoch":546,"state":"active+clean","last_fresh":"2023-09-23T21:15:27.423575+0000","last_change":"2023-09-23T21:11:32.760292+0000","last_active":"2023-09-23T21:15:27.423575+0000","last_peered":"2023-09-23T21:15:27.423575+0000","last_clean":"2023-09-23T21:15:27.423575+0000","last_became_active":"2023-09-23T21:11:32.759909+0000","last_became_peered":"2023-09-23T21:11:32.759909+0000","last_unstale":"2023-09-23T21:15:27.423575+0000","last_undegraded":"2023-09-23T21:15:27.423575+0000","last_fullsized":"2023-09-23T21:15:27.423575+0000","mapping_epoch":457,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":458,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:25.096401+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:25.096401+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:13:09.225685+0000","scrub_duration":0.002859393,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":272,"reported_epoch":547,"state":"active+clean","last_fresh":"2023-09-23T21:15:30.549979+0000","last_change":"2023-09-23T21:12:10.779468+0000","last_active":"2023-09-23T21:15:30.549979+0000","last_peered":"2023-09-23T21:15:30.549979+0000","last_clean":"2023-09-23T21:15:30.549979+0000","last_became_active":"2023-09-23T21:12:10.779282+0000","last_became_peered":"2023-09-23T21:12:10.779282+0000","last_unstale":"2023-09-23T21:15:30.549979+0000","last_undegraded":"2023-09-23T21:15:30.549979+0000","last_fullsized":"2023-09-23T21:15:30.549979+0000","mapping_epoch":470,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:36:35.417986+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":523,"reported_epoch":547,"state":"active+clean","last_fresh":"2023-09-23T21:15:30.965819+0000","last_change":"2023-09-23T21:10:26.758958+0000","last_active":"2023-09-23T21:15:30.965819+0000","last_peered":"2023-09-23T21:15:30.965819+0000","last_clean":"2023-09-23T21:15:30.965819+0000","last_became_active":"2023-09-23T21:10:26.758539+0000","last_became_peered":"2023-09-23T21:10:26.758539+0000","last_unstale":"2023-09-23T21:15:30.965819+0000","last_undegraded":"2023-09-23T21:15:30.965819+0000","last_fullsized":"2023-09-23T21:15:30.965819+0000","mapping_epoch":423,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":424,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:44:42.319189+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.17","version":"79'161","reported_seq":646,"reported_epoch":545,"state":"active+clean","last_fresh":"2023-09-23T21:15:25.892290+0000","last_change":"2023-09-23T21:15:23.799556+0000","last_active":"2023-09-23T21:15:25.892290+0000","last_peered":"2023-09-23T21:15:25.892290+0000","last_clean":"2023-09-23T21:15:25.892290+0000","last_became_active":"2023-09-23T21:15:23.799272+0000","last_became_peered":"2023-09-23T21:15:23.799272+0000","last_unstale":"2023-09-23T21:15:25.892290+0000","last_undegraded":"2023-09-23T21:15:25.892290+0000","last_fullsized":"2023-09-23T21:15:25.892290+0000","mapping_epoch":542,"log_start":"0'0","ondisk_log_start":"0'0","created":479,"last_epoch_clean":543,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:25.916867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:25.916867+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.001665323,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":432,"reported_epoch":547,"state":"active+clean","last_fresh":"2023-09-23T21:15:30.966825+0000","last_change":"2023-09-23T21:11:08.766045+0000","last_active":"2023-09-23T21:15:30.966825+0000","last_peered":"2023-09-23T21:15:30.966825+0000","last_clean":"2023-09-23T21:15:30.966825+0000","last_became_active":"2023-09-23T21:11:08.758504+0000","last_became_peered":"2023-09-23T21:11:08.758504+0000","last_unstale":"2023-09-23T21:15:30.966825+0000","last_undegraded":"2023-09-23T21:15:30.966825+0000","last_fullsized":"2023-09-23T21:15:30.966825+0000","mapping_epoch":443,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":444,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:02:19.482574+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":360,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,4],"acting":[3,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":234,"reported_epoch":546,"state":"active+clean","last_fresh":"2023-09-23T21:15:27.423265+0000","last_change":"2023-09-23T21:13:30.821489+0000","last_active":"2023-09-23T21:15:27.423265+0000","last_peered":"2023-09-23T21:15:27.423265+0000","last_clean":"2023-09-23T21:15:27.423265+0000","last_became_active":"2023-09-23T21:13:30.821125+0000","last_became_peered":"2023-09-23T21:13:30.821125+0000","last_unstale":"2023-09-23T21:15:27.423265+0000","last_undegraded":"2023-09-23T21:15:27.423265+0000","last_fullsized":"2023-09-23T21:15:27.423265+0000","mapping_epoch":501,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:47:18.553417+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,5],"acting":[2,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.16","version":"79'161","reported_seq":464,"reported_epoch":545,"state":"active+clean","last_fresh":"2023-09-23T21:15:25.899796+0000","last_change":"2023-09-23T21:13:33.770962+0000","last_active":"2023-09-23T21:15:25.899796+0000","last_peered":"2023-09-23T21:15:25.899796+0000","last_clean":"2023-09-23T21:15:25.899796+0000","last_became_active":"2023-09-23T21:13:33.770407+0000","last_became_peered":"2023-09-23T21:13:33.770407+0000","last_unstale":"2023-09-23T21:15:25.899796+0000","last_undegraded":"2023-09-23T21:15:25.899796+0000","last_fullsized":"2023-09-23T21:15:25.899796+0000","mapping_epoch":504,"log_start":"0'0","ondisk_log_start":"0'0","created":479,"last_epoch_clean":505,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:10:24.268662+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:10:24.268662+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13418,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":11,"num_read_kb":21,"num_write":6,"num_write_kb":55,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":40252,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.14","version":"79'22","reported_seq":111,"reported_epoch":547,"state":"active+clean","last_fresh":"2023-09-23T21:15:30.213412+0000","last_change":"2023-09-23T21:11:27.770162+0000","last_active":"2023-09-23T21:15:30.213412+0000","last_peered":"2023-09-23T21:15:30.213412+0000","last_clean":"2023-09-23T21:15:30.213412+0000","last_became_active":"2023-09-23T21:11:27.752275+0000","last_became_peered":"2023-09-23T21:11:27.752275+0000","last_unstale":"2023-09-23T21:15:30.213412+0000","last_undegraded":"2023-09-23T21:15:30.213412+0000","last_fullsized":"2023-09-23T21:15:30.213412+0000","mapping_epoch":452,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":453,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:05:11.769428+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4734,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,2],"acting":[4,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":476,"reported_epoch":547,"state":"active+clean","last_fresh":"2023-09-23T21:15:30.212177+0000","last_change":"2023-09-23T21:12:10.839271+0000","last_active":"2023-09-23T21:15:30.212177+0000","last_peered":"2023-09-23T21:15:30.212177+0000","last_clean":"2023-09-23T21:15:30.212177+0000","last_became_active":"2023-09-23T21:12:10.839042+0000","last_became_peered":"2023-09-23T21:12:10.839042+0000","last_unstale":"2023-09-23T21:15:30.212177+0000","last_undegraded":"2023-09-23T21:15:30.212177+0000","last_fullsized":"2023-09-23T21:15:30.212177+0000","mapping_epoch":470,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:10:26.414737+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,2],"acting":[4,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.11","version":"79'161","reported_seq":469,"reported_epoch":545,"state":"active+clean","last_fresh":"2023-09-23T21:15:25.893415+0000","last_change":"2023-09-23T21:13:33.767406+0000","last_active":"2023-09-23T21:15:25.893415+0000","last_peered":"2023-09-23T21:15:25.893415+0000","last_clean":"2023-09-23T21:15:25.893415+0000","last_became_active":"2023-09-23T21:13:33.767067+0000","last_became_peered":"2023-09-23T21:13:33.767067+0000","last_unstale":"2023-09-23T21:15:25.893415+0000","last_undegraded":"2023-09-23T21:15:25.893415+0000","last_fullsized":"2023-09-23T21:15:25.893415+0000","mapping_epoch":504,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":505,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:28.756634+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:28.756634+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":23,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":156,"reported_epoch":547,"state":"active+clean","last_fresh":"2023-09-23T21:15:30.213182+0000","last_change":"2023-09-23T21:11:27.777154+0000","last_active":"2023-09-23T21:15:30.213182+0000","last_peered":"2023-09-23T21:15:30.213182+0000","last_clean":"2023-09-23T21:15:30.213182+0000","last_became_active":"2023-09-23T21:11:27.754438+0000","last_became_peered":"2023-09-23T21:11:27.754438+0000","last_unstale":"2023-09-23T21:15:30.213182+0000","last_undegraded":"2023-09-23T21:15:30.213182+0000","last_fullsized":"2023-09-23T21:15:30.213182+0000","mapping_epoch":452,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":453,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:33:02.845882+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":88,"reported_epoch":546,"state":"active+clean","last_fresh":"2023-09-23T21:15:27.423342+0000","last_change":"2023-09-23T21:12:10.764264+0000","last_active":"2023-09-23T21:15:27.423342+0000","last_peered":"2023-09-23T21:15:27.423342+0000","last_clean":"2023-09-23T21:15:27.423342+0000","last_became_active":"2023-09-23T21:12:10.763486+0000","last_became_peered":"2023-09-23T21:12:10.763486+0000","last_unstale":"2023-09-23T21:15:27.423342+0000","last_undegraded":"2023-09-23T21:15:27.423342+0000","last_fullsized":"2023-09-23T21:15:27.423342+0000","mapping_epoch":470,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:32:27.577036+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,7],"acting":[2,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.10","version":"79'161","reported_seq":682,"reported_epoch":547,"state":"active+clean","last_fresh":"2023-09-23T21:15:30.211802+0000","last_change":"2023-09-23T21:11:11.716703+0000","last_active":"2023-09-23T21:15:30.211802+0000","last_peered":"2023-09-23T21:15:30.211802+0000","last_clean":"2023-09-23T21:15:30.211802+0000","last_became_active":"2023-09-23T21:11:11.716385+0000","last_became_peered":"2023-09-23T21:11:11.716385+0000","last_unstale":"2023-09-23T21:15:30.211802+0000","last_undegraded":"2023-09-23T21:15:30.211802+0000","last_fullsized":"2023-09-23T21:15:30.211802+0000","mapping_epoch":446,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":447,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:06:50.336992+0000","last_deep_scrub":"79'161","last_deep_scrub_stamp":"2023-09-23T21:06:50.336992+0000","last_clean_scrub_stamp":"2023-09-23T21:06:50.336992+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:44:29.538890+0000","scrub_duration":0.0026150930000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,7],"acting":[4,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":487,"reported_epoch":546,"state":"active+clean","last_fresh":"2023-09-23T21:15:27.494932+0000","last_change":"2023-09-23T21:13:12.270990+0000","last_active":"2023-09-23T21:15:27.494932+0000","last_peered":"2023-09-23T21:15:27.494932+0000","last_clean":"2023-09-23T21:15:27.494932+0000","last_became_active":"2023-09-23T21:13:12.237201+0000","last_became_peered":"2023-09-23T21:13:12.237201+0000","last_unstale":"2023-09-23T21:15:27.494932+0000","last_undegraded":"2023-09-23T21:15:27.494932+0000","last_fullsized":"2023-09-23T21:15:27.494932+0000","mapping_epoch":493,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:59:54.523877+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":300,"reported_epoch":547,"state":"active+clean","last_fresh":"2023-09-23T21:15:30.965664+0000","last_change":"2023-09-23T21:10:26.758671+0000","last_active":"2023-09-23T21:15:30.965664+0000","last_peered":"2023-09-23T21:15:30.965664+0000","last_clean":"2023-09-23T21:15:30.965664+0000","last_became_active":"2023-09-23T21:10:26.758244+0000","last_became_peered":"2023-09-23T21:10:26.758244+0000","last_unstale":"2023-09-23T21:15:30.965664+0000","last_undegraded":"2023-09-23T21:15:30.965664+0000","last_fullsized":"2023-09-23T21:15:30.965664+0000","mapping_epoch":423,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":424,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:14:02.735609+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.13","version":"79'161","reported_seq":625,"reported_epoch":547,"state":"active+clean","last_fresh":"2023-09-23T21:15:30.965621+0000","last_change":"2023-09-23T21:13:14.933096+0000","last_active":"2023-09-23T21:15:30.965621+0000","last_peered":"2023-09-23T21:15:30.965621+0000","last_clean":"2023-09-23T21:15:30.965621+0000","last_became_active":"2023-09-23T21:13:14.932703+0000","last_became_peered":"2023-09-23T21:13:14.932703+0000","last_unstale":"2023-09-23T21:15:30.965621+0000","last_undegraded":"2023-09-23T21:15:30.965621+0000","last_fullsized":"2023-09-23T21:15:30.965621+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:36.420362+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:36.420362+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:29:42.163911+0000","scrub_duration":0.028838444000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":539,"reported_epoch":547,"state":"active+clean","last_fresh":"2023-09-23T21:15:30.213130+0000","last_change":"2023-09-23T21:13:12.216467+0000","last_active":"2023-09-23T21:15:30.213130+0000","last_peered":"2023-09-23T21:15:30.213130+0000","last_clean":"2023-09-23T21:15:30.213130+0000","last_became_active":"2023-09-23T21:13:12.215259+0000","last_became_peered":"2023-09-23T21:13:12.215259+0000","last_unstale":"2023-09-23T21:15:30.213130+0000","last_undegraded":"2023-09-23T21:15:30.213130+0000","last_fullsized":"2023-09-23T21:15:30.213130+0000","mapping_epoch":493,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:35:33.987839+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":55,"reported_epoch":545,"state":"active+clean","last_fresh":"2023-09-23T21:15:25.899530+0000","last_change":"2023-09-23T21:13:30.804108+0000","last_active":"2023-09-23T21:15:25.899530+0000","last_peered":"2023-09-23T21:15:25.899530+0000","last_clean":"2023-09-23T21:15:25.899530+0000","last_became_active":"2023-09-23T21:13:30.796505+0000","last_became_peered":"2023-09-23T21:13:30.796505+0000","last_unstale":"2023-09-23T21:15:25.899530+0000","last_undegraded":"2023-09-23T21:15:25.899530+0000","last_fullsized":"2023-09-23T21:15:25.899530+0000","mapping_epoch":501,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:19:33.508496+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.12","version":"79'161","reported_seq":781,"reported_epoch":547,"state":"active+clean","last_fresh":"2023-09-23T21:15:30.550689+0000","last_change":"2023-09-23T21:13:15.206574+0000","last_active":"2023-09-23T21:15:30.550689+0000","last_peered":"2023-09-23T21:15:30.550689+0000","last_clean":"2023-09-23T21:15:30.550689+0000","last_became_active":"2023-09-23T21:13:15.206186+0000","last_became_peered":"2023-09-23T21:13:15.206186+0000","last_unstale":"2023-09-23T21:15:30.550689+0000","last_undegraded":"2023-09-23T21:15:30.550689+0000","last_fullsized":"2023-09-23T21:15:30.550689+0000","mapping_epoch":496,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":497,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:05:56.785173+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:05:56.785173+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:34:29.221809+0000","scrub_duration":0.0042114370000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,0],"acting":[5,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":65,"reported_epoch":545,"state":"active+clean","last_fresh":"2023-09-23T21:15:25.890129+0000","last_change":"2023-09-23T21:13:38.048791+0000","last_active":"2023-09-23T21:15:25.890129+0000","last_peered":"2023-09-23T21:15:25.890129+0000","last_clean":"2023-09-23T21:15:25.890129+0000","last_became_active":"2023-09-23T21:13:30.801386+0000","last_became_peered":"2023-09-23T21:13:30.801386+0000","last_unstale":"2023-09-23T21:15:25.890129+0000","last_undegraded":"2023-09-23T21:15:25.890129+0000","last_fullsized":"2023-09-23T21:15:25.890129+0000","mapping_epoch":501,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:52:49.170414+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":385,"reported_epoch":547,"state":"active+clean","last_fresh":"2023-09-23T21:15:30.965937+0000","last_change":"2023-09-23T21:12:10.765780+0000","last_active":"2023-09-23T21:15:30.965937+0000","last_peered":"2023-09-23T21:15:30.965937+0000","last_clean":"2023-09-23T21:15:30.965937+0000","last_became_active":"2023-09-23T21:12:10.765372+0000","last_became_peered":"2023-09-23T21:12:10.765372+0000","last_unstale":"2023-09-23T21:15:30.965937+0000","last_undegraded":"2023-09-23T21:15:30.965937+0000","last_fullsized":"2023-09-23T21:15:30.965937+0000","mapping_epoch":470,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:56:08.446099+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.7","version":"0'0","reported_seq":12,"reported_epoch":547,"state":"active+clean","last_fresh":"2023-09-23T21:15:30.213681+0000","last_change":"2023-09-23T21:15:25.902813+0000","last_active":"2023-09-23T21:15:30.213681+0000","last_peered":"2023-09-23T21:15:30.213681+0000","last_clean":"2023-09-23T21:15:30.213681+0000","last_became_active":"2023-09-23T21:15:25.902476+0000","last_became_peered":"2023-09-23T21:15:25.902476+0000","last_unstale":"2023-09-23T21:15:30.213681+0000","last_undegraded":"2023-09-23T21:15:30.213681+0000","last_fullsized":"2023-09-23T21:15:30.213681+0000","mapping_epoch":544,"log_start":"0'0","ondisk_log_start":"0'0","created":544,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_clean_scrub_stamp":"2023-09-23T21:15:24.860968+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:41:25.699152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.d","version":"79'161","reported_seq":701,"reported_epoch":547,"state":"active+clean","last_fresh":"2023-09-23T21:15:30.214405+0000","last_change":"2023-09-23T21:12:30.444461+0000","last_active":"2023-09-23T21:15:30.214405+0000","last_peered":"2023-09-23T21:15:30.214405+0000","last_clean":"2023-09-23T21:15:30.214405+0000","last_became_active":"2023-09-23T21:12:30.443979+0000","last_became_peered":"2023-09-23T21:12:30.443979+0000","last_unstale":"2023-09-23T21:15:30.214405+0000","last_undegraded":"2023-09-23T21:15:30.214405+0000","last_fullsized":"2023-09-23T21:15:30.214405+0000","mapping_epoch":470,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:09:36.858077+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:09:36.858077+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0024987500000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":194,"reported_epoch":547,"state":"active+clean","last_fresh":"2023-09-23T21:15:30.214823+0000","last_change":"2023-09-23T21:13:59.872834+0000","last_active":"2023-09-23T21:15:30.214823+0000","last_peered":"2023-09-23T21:15:30.214823+0000","last_clean":"2023-09-23T21:15:30.214823+0000","last_became_active":"2023-09-23T21:13:30.770556+0000","last_became_peered":"2023-09-23T21:13:30.770556+0000","last_unstale":"2023-09-23T21:15:30.214823+0000","last_undegraded":"2023-09-23T21:15:30.214823+0000","last_fullsized":"2023-09-23T21:15:30.214823+0000","mapping_epoch":501,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":521,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:57:55.460052+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":132,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":255,"reported_epoch":547,"state":"active+clean","last_fresh":"2023-09-23T21:15:30.549529+0000","last_change":"2023-09-23T21:10:26.750725+0000","last_active":"2023-09-23T21:15:30.549529+0000","last_peered":"2023-09-23T21:15:30.549529+0000","last_clean":"2023-09-23T21:15:30.549529+0000","last_became_active":"2023-09-23T21:10:26.750307+0000","last_became_peered":"2023-09-23T21:10:26.750307+0000","last_unstale":"2023-09-23T21:15:30.549529+0000","last_undegraded":"2023-09-23T21:15:30.549529+0000","last_fullsized":"2023-09-23T21:15:30.549529+0000","mapping_epoch":423,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":424,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:06:22.040548+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,4],"acting":[5,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.6","version":"0'0","reported_seq":11,"reported_epoch":545,"state":"active+clean","last_fresh":"2023-09-23T21:15:25.906906+0000","last_change":"2023-09-23T21:15:25.904499+0000","last_active":"2023-09-23T21:15:25.906906+0000","last_peered":"2023-09-23T21:15:25.906906+0000","last_clean":"2023-09-23T21:15:25.906906+0000","last_became_active":"2023-09-23T21:15:25.904185+0000","last_became_peered":"2023-09-23T21:15:25.904185+0000","last_unstale":"2023-09-23T21:15:25.906906+0000","last_undegraded":"2023-09-23T21:15:25.906906+0000","last_fullsized":"2023-09-23T21:15:25.906906+0000","mapping_epoch":544,"log_start":"0'0","ondisk_log_start":"0'0","created":544,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_clean_scrub_stamp":"2023-09-23T21:15:24.860968+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:02:41.735858+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.c","version":"79'161","reported_seq":929,"reported_epoch":548,"state":"active+clean","last_fresh":"2023-09-23T21:15:31.992633+0000","last_change":"2023-09-23T21:15:31.992756+0000","last_active":"2023-09-23T21:15:31.992633+0000","last_peered":"2023-09-23T21:15:31.992633+0000","last_clean":"2023-09-23T21:15:31.992633+0000","last_became_active":"2023-09-23T21:13:13.401526+0000","last_became_peered":"2023-09-23T21:13:13.401526+0000","last_unstale":"2023-09-23T21:15:31.992633+0000","last_undegraded":"2023-09-23T21:15:31.992633+0000","last_fullsized":"2023-09-23T21:15:31.992633+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":532,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:15:31.992598+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:15:31.992598+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:41:14.215543+0000","scrub_duration":0.002960678,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":180,"reported_epoch":547,"state":"active+clean","last_fresh":"2023-09-23T21:15:30.214632+0000","last_change":"2023-09-23T21:13:59.884113+0000","last_active":"2023-09-23T21:15:30.214632+0000","last_peered":"2023-09-23T21:15:30.214632+0000","last_clean":"2023-09-23T21:15:30.214632+0000","last_became_active":"2023-09-23T21:13:30.806778+0000","last_became_peered":"2023-09-23T21:13:30.806778+0000","last_unstale":"2023-09-23T21:15:30.214632+0000","last_undegraded":"2023-09-23T21:15:30.214632+0000","last_fullsized":"2023-09-23T21:15:30.214632+0000","mapping_epoch":501,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":521,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:36:15.284465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":510,"reported_epoch":547,"state":"active+clean","last_fresh":"2023-09-23T21:15:30.213176+0000","last_change":"2023-09-23T21:13:15.152792+0000","last_active":"2023-09-23T21:15:30.213176+0000","last_peered":"2023-09-23T21:15:30.213176+0000","last_clean":"2023-09-23T21:15:30.213176+0000","last_became_active":"2023-09-23T21:13:14.515084+0000","last_became_peered":"2023-09-23T21:13:14.515084+0000","last_unstale":"2023-09-23T21:15:30.213176+0000","last_undegraded":"2023-09-23T21:15:30.213176+0000","last_fullsized":"2023-09-23T21:15:30.213176+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:08:16.702072+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.5","version":"0'0","reported_seq":12,"reported_epoch":547,"state":"active+clean","last_fresh":"2023-09-23T21:15:30.550006+0000","last_change":"2023-09-23T21:15:25.893591+0000","last_active":"2023-09-23T21:15:30.550006+0000","last_peered":"2023-09-23T21:15:30.550006+0000","last_clean":"2023-09-23T21:15:30.550006+0000","last_became_active":"2023-09-23T21:15:25.893280+0000","last_became_peered":"2023-09-23T21:15:25.893280+0000","last_unstale":"2023-09-23T21:15:30.550006+0000","last_undegraded":"2023-09-23T21:15:30.550006+0000","last_fullsized":"2023-09-23T21:15:30.550006+0000","mapping_epoch":544,"log_start":"0'0","ondisk_log_start":"0'0","created":544,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_clean_scrub_stamp":"2023-09-23T21:15:24.860968+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:02:55.710366+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.f","version":"79'161","reported_seq":835,"reported_epoch":547,"state":"active+clean","last_fresh":"2023-09-23T21:15:30.550215+0000","last_change":"2023-09-23T21:11:30.750338+0000","last_active":"2023-09-23T21:15:30.550215+0000","last_peered":"2023-09-23T21:15:30.550215+0000","last_clean":"2023-09-23T21:15:30.550215+0000","last_became_active":"2023-09-23T21:11:30.749953+0000","last_became_peered":"2023-09-23T21:11:30.749953+0000","last_unstale":"2023-09-23T21:15:30.550215+0000","last_undegraded":"2023-09-23T21:15:30.550215+0000","last_fullsized":"2023-09-23T21:15:30.550215+0000","mapping_epoch":455,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":456,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:27.880776+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:27.880776+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:07:10.405723+0000","scrub_duration":0.0014004899999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,3],"acting":[5,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":271,"reported_epoch":547,"state":"active+clean","last_fresh":"2023-09-23T21:15:30.550024+0000","last_change":"2023-09-23T21:10:26.765068+0000","last_active":"2023-09-23T21:15:30.550024+0000","last_peered":"2023-09-23T21:15:30.550024+0000","last_clean":"2023-09-23T21:15:30.550024+0000","last_became_active":"2023-09-23T21:10:26.758364+0000","last_became_peered":"2023-09-23T21:10:26.758364+0000","last_unstale":"2023-09-23T21:15:30.550024+0000","last_undegraded":"2023-09-23T21:15:30.550024+0000","last_fullsized":"2023-09-23T21:15:30.550024+0000","mapping_epoch":423,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":424,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:13:21.057529+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":280,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":487,"reported_epoch":547,"state":"active+clean","last_fresh":"2023-09-23T21:15:30.550466+0000","last_change":"2023-09-23T21:13:30.770746+0000","last_active":"2023-09-23T21:15:30.550466+0000","last_peered":"2023-09-23T21:15:30.550466+0000","last_clean":"2023-09-23T21:15:30.550466+0000","last_became_active":"2023-09-23T21:13:30.770437+0000","last_became_peered":"2023-09-23T21:13:30.770437+0000","last_unstale":"2023-09-23T21:15:30.550466+0000","last_undegraded":"2023-09-23T21:15:30.550466+0000","last_fullsized":"2023-09-23T21:15:30.550466+0000","mapping_epoch":501,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:33:48.325730+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,0],"acting":[5,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.4","version":"0'0","reported_seq":12,"reported_epoch":547,"state":"active+clean","last_fresh":"2023-09-23T21:15:30.213642+0000","last_change":"2023-09-23T21:15:25.892580+0000","last_active":"2023-09-23T21:15:30.213642+0000","last_peered":"2023-09-23T21:15:30.213642+0000","last_clean":"2023-09-23T21:15:30.213642+0000","last_became_active":"2023-09-23T21:15:25.892251+0000","last_became_peered":"2023-09-23T21:15:25.892251+0000","last_unstale":"2023-09-23T21:15:30.213642+0000","last_undegraded":"2023-09-23T21:15:30.213642+0000","last_fullsized":"2023-09-23T21:15:30.213642+0000","mapping_epoch":544,"log_start":"0'0","ondisk_log_start":"0'0","created":544,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_clean_scrub_stamp":"2023-09-23T21:15:24.860968+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:21:43.221845+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,3],"acting":[7,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.e","version":"79'161","reported_seq":716,"reported_epoch":547,"state":"active+clean","last_fresh":"2023-09-23T21:15:30.212665+0000","last_change":"2023-09-23T21:12:30.440952+0000","last_active":"2023-09-23T21:15:30.212665+0000","last_peered":"2023-09-23T21:15:30.212665+0000","last_clean":"2023-09-23T21:15:30.212665+0000","last_became_active":"2023-09-23T21:12:30.440463+0000","last_became_peered":"2023-09-23T21:12:30.440463+0000","last_unstale":"2023-09-23T21:15:30.212665+0000","last_undegraded":"2023-09-23T21:15:30.212665+0000","last_fullsized":"2023-09-23T21:15:30.212665+0000","mapping_epoch":455,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:10:39.567232+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:10:39.567232+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.005229694,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":345,"reported_epoch":547,"state":"active+clean","last_fresh":"2023-09-23T21:15:30.967141+0000","last_change":"2023-09-23T21:13:13.402418+0000","last_active":"2023-09-23T21:15:30.967141+0000","last_peered":"2023-09-23T21:15:30.967141+0000","last_clean":"2023-09-23T21:15:30.967141+0000","last_became_active":"2023-09-23T21:13:13.400984+0000","last_became_peered":"2023-09-23T21:13:13.400984+0000","last_unstale":"2023-09-23T21:15:30.967141+0000","last_undegraded":"2023-09-23T21:15:30.967141+0000","last_fullsized":"2023-09-23T21:15:30.967141+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":495,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:28:01.411191+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,0],"acting":[3,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":519,"reported_epoch":545,"state":"active+clean","last_fresh":"2023-09-23T21:15:25.902011+0000","last_change":"2023-09-23T21:13:14.339670+0000","last_active":"2023-09-23T21:15:25.902011+0000","last_peered":"2023-09-23T21:15:25.902011+0000","last_clean":"2023-09-23T21:15:25.902011+0000","last_became_active":"2023-09-23T21:13:14.339341+0000","last_became_peered":"2023-09-23T21:13:14.339341+0000","last_unstale":"2023-09-23T21:15:25.902011+0000","last_undegraded":"2023-09-23T21:15:25.902011+0000","last_fullsized":"2023-09-23T21:15:25.902011+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:23:19.411368+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.3","version":"0'0","reported_seq":12,"reported_epoch":547,"state":"active+clean","last_fresh":"2023-09-23T21:15:30.212900+0000","last_change":"2023-09-23T21:15:25.891637+0000","last_active":"2023-09-23T21:15:30.212900+0000","last_peered":"2023-09-23T21:15:30.212900+0000","last_clean":"2023-09-23T21:15:30.212900+0000","last_became_active":"2023-09-23T21:15:25.891330+0000","last_became_peered":"2023-09-23T21:15:25.891330+0000","last_unstale":"2023-09-23T21:15:30.212900+0000","last_undegraded":"2023-09-23T21:15:30.212900+0000","last_fullsized":"2023-09-23T21:15:30.212900+0000","mapping_epoch":544,"log_start":"0'0","ondisk_log_start":"0'0","created":544,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_clean_scrub_stamp":"2023-09-23T21:15:24.860968+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:22:07.243528+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,2],"acting":[7,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.9","version":"79'161","reported_seq":701,"reported_epoch":546,"state":"active+clean","last_fresh":"2023-09-23T21:15:27.422883+0000","last_change":"2023-09-23T21:12:30.436118+0000","last_active":"2023-09-23T21:15:27.422883+0000","last_peered":"2023-09-23T21:15:27.422883+0000","last_clean":"2023-09-23T21:15:27.422883+0000","last_became_active":"2023-09-23T21:12:30.435779+0000","last_became_peered":"2023-09-23T21:12:30.435779+0000","last_unstale":"2023-09-23T21:15:27.422883+0000","last_undegraded":"2023-09-23T21:15:27.422883+0000","last_fullsized":"2023-09-23T21:15:27.422883+0000","mapping_epoch":433,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:10:37.696951+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:10:37.696951+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.001386504,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,3],"acting":[2,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":104,"reported_epoch":545,"state":"active+clean","last_fresh":"2023-09-23T21:15:25.902154+0000","last_change":"2023-09-23T21:13:30.793273+0000","last_active":"2023-09-23T21:15:25.902154+0000","last_peered":"2023-09-23T21:15:25.902154+0000","last_clean":"2023-09-23T21:15:25.902154+0000","last_became_active":"2023-09-23T21:13:30.791269+0000","last_became_peered":"2023-09-23T21:13:30.791269+0000","last_unstale":"2023-09-23T21:15:25.902154+0000","last_undegraded":"2023-09-23T21:15:25.902154+0000","last_fullsized":"2023-09-23T21:15:25.902154+0000","mapping_epoch":501,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:38:07.142290+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":121,"reported_epoch":547,"state":"active+clean","last_fresh":"2023-09-23T21:15:30.213336+0000","last_change":"2023-09-23T21:12:14.313466+0000","last_active":"2023-09-23T21:15:30.213336+0000","last_peered":"2023-09-23T21:15:30.213336+0000","last_clean":"2023-09-23T21:15:30.213336+0000","last_became_active":"2023-09-23T21:12:14.209837+0000","last_became_peered":"2023-09-23T21:12:14.209837+0000","last_unstale":"2023-09-23T21:15:30.213336+0000","last_undegraded":"2023-09-23T21:15:30.213336+0000","last_fullsized":"2023-09-23T21:15:30.213336+0000","mapping_epoch":470,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":474,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:09:40.375241+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,7],"acting":[4,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.2","version":"0'0","reported_seq":12,"reported_epoch":546,"state":"active+clean","last_fresh":"2023-09-23T21:15:27.423074+0000","last_change":"2023-09-23T21:15:25.900671+0000","last_active":"2023-09-23T21:15:27.423074+0000","last_peered":"2023-09-23T21:15:27.423074+0000","last_clean":"2023-09-23T21:15:27.423074+0000","last_became_active":"2023-09-23T21:15:25.900364+0000","last_became_peered":"2023-09-23T21:15:25.900364+0000","last_unstale":"2023-09-23T21:15:27.423074+0000","last_undegraded":"2023-09-23T21:15:27.423074+0000","last_fullsized":"2023-09-23T21:15:27.423074+0000","mapping_epoch":544,"log_start":"0'0","ondisk_log_start":"0'0","created":544,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_clean_scrub_stamp":"2023-09-23T21:15:24.860968+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:03:40.630119+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.8","version":"79'161","reported_seq":667,"reported_epoch":546,"state":"active+clean","last_fresh":"2023-09-23T21:15:27.422889+0000","last_change":"2023-09-23T21:12:30.433561+0000","last_active":"2023-09-23T21:15:27.422889+0000","last_peered":"2023-09-23T21:15:27.422889+0000","last_clean":"2023-09-23T21:15:27.422889+0000","last_became_active":"2023-09-23T21:12:30.433217+0000","last_became_peered":"2023-09-23T21:12:30.433217+0000","last_unstale":"2023-09-23T21:15:27.422889+0000","last_undegraded":"2023-09-23T21:15:27.422889+0000","last_fullsized":"2023-09-23T21:15:27.422889+0000","mapping_epoch":359,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:06:52.098882+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:06:52.098882+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0092173299999999993,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":131344,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":124,"reported_epoch":547,"state":"active+clean","last_fresh":"2023-09-23T21:15:30.212145+0000","last_change":"2023-09-23T21:12:10.823380+0000","last_active":"2023-09-23T21:15:30.212145+0000","last_peered":"2023-09-23T21:15:30.212145+0000","last_clean":"2023-09-23T21:15:30.212145+0000","last_became_active":"2023-09-23T21:12:10.823044+0000","last_became_peered":"2023-09-23T21:12:10.823044+0000","last_unstale":"2023-09-23T21:15:30.212145+0000","last_undegraded":"2023-09-23T21:15:30.212145+0000","last_fullsized":"2023-09-23T21:15:30.212145+0000","mapping_epoch":470,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:53:29.846536+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,5],"acting":[4,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":125,"reported_epoch":547,"state":"active+clean","last_fresh":"2023-09-23T21:15:30.212408+0000","last_change":"2023-09-23T21:13:30.772451+0000","last_active":"2023-09-23T21:15:30.212408+0000","last_peered":"2023-09-23T21:15:30.212408+0000","last_clean":"2023-09-23T21:15:30.212408+0000","last_became_active":"2023-09-23T21:13:30.772040+0000","last_became_peered":"2023-09-23T21:13:30.772040+0000","last_unstale":"2023-09-23T21:15:30.212408+0000","last_undegraded":"2023-09-23T21:15:30.212408+0000","last_fullsized":"2023-09-23T21:15:30.212408+0000","mapping_epoch":501,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:24:49.246727+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,0],"acting":[4,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.1","version":"548'299","reported_seq":312,"reported_epoch":548,"state":"active+clean","last_fresh":"2023-09-23T21:15:34.212942+0000","last_change":"2023-09-23T21:15:25.903355+0000","last_active":"2023-09-23T21:15:34.212942+0000","last_peered":"2023-09-23T21:15:34.212942+0000","last_clean":"2023-09-23T21:15:34.212942+0000","last_became_active":"2023-09-23T21:15:25.902988+0000","last_became_peered":"2023-09-23T21:15:25.902988+0000","last_unstale":"2023-09-23T21:15:34.212942+0000","last_undegraded":"2023-09-23T21:15:34.212942+0000","last_fullsized":"2023-09-23T21:15:34.212942+0000","mapping_epoch":544,"log_start":"0'0","ondisk_log_start":"0'0","created":544,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_clean_scrub_stamp":"2023-09-23T21:15:24.860968+0000","objects_scrubbed":0,"log_size":299,"log_dups_size":0,"ondisk_log_size":299,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:47:02.629642+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19595264,"num_objects":299,"num_object_clones":0,"num_object_copies":897,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":299,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":299,"num_write_kb":19136,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,2],"acting":[3,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.b","version":"79'161","reported_seq":550,"reported_epoch":546,"state":"active+clean","last_fresh":"2023-09-23T21:15:27.493300+0000","last_change":"2023-09-23T21:13:13.373313+0000","last_active":"2023-09-23T21:15:27.493300+0000","last_peered":"2023-09-23T21:15:27.493300+0000","last_clean":"2023-09-23T21:15:27.493300+0000","last_became_active":"2023-09-23T21:13:13.372328+0000","last_became_peered":"2023-09-23T21:13:13.372328+0000","last_unstale":"2023-09-23T21:15:27.493300+0000","last_undegraded":"2023-09-23T21:15:27.493300+0000","last_fullsized":"2023-09-23T21:15:27.493300+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":495,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:05:55.772264+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:05:55.772264+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.001862507,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":105,"reported_epoch":547,"state":"active+clean","last_fresh":"2023-09-23T21:15:30.213097+0000","last_change":"2023-09-23T21:11:27.754717+0000","last_active":"2023-09-23T21:15:30.213097+0000","last_peered":"2023-09-23T21:15:30.213097+0000","last_clean":"2023-09-23T21:15:30.213097+0000","last_became_active":"2023-09-23T21:11:27.754376+0000","last_became_peered":"2023-09-23T21:11:27.754376+0000","last_unstale":"2023-09-23T21:15:30.213097+0000","last_undegraded":"2023-09-23T21:15:30.213097+0000","last_fullsized":"2023-09-23T21:15:30.213097+0000","mapping_epoch":452,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":453,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:26:41.734541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":479,"reported_epoch":547,"state":"active+clean","last_fresh":"2023-09-23T21:15:30.212754+0000","last_change":"2023-09-23T21:12:10.811698+0000","last_active":"2023-09-23T21:15:30.212754+0000","last_peered":"2023-09-23T21:15:30.212754+0000","last_clean":"2023-09-23T21:15:30.212754+0000","last_became_active":"2023-09-23T21:12:10.789656+0000","last_became_peered":"2023-09-23T21:12:10.789656+0000","last_unstale":"2023-09-23T21:15:30.212754+0000","last_undegraded":"2023-09-23T21:15:30.212754+0000","last_fullsized":"2023-09-23T21:15:30.212754+0000","mapping_epoch":470,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:39:15.112091+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,5],"acting":[4,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.0","version":"548'317","reported_seq":330,"reported_epoch":548,"state":"active+clean","last_fresh":"2023-09-23T21:15:34.180036+0000","last_change":"2023-09-23T21:15:25.902910+0000","last_active":"2023-09-23T21:15:34.180036+0000","last_peered":"2023-09-23T21:15:34.180036+0000","last_clean":"2023-09-23T21:15:34.180036+0000","last_became_active":"2023-09-23T21:15:25.902656+0000","last_became_peered":"2023-09-23T21:15:25.902656+0000","last_unstale":"2023-09-23T21:15:34.180036+0000","last_undegraded":"2023-09-23T21:15:34.180036+0000","last_fullsized":"2023-09-23T21:15:34.180036+0000","mapping_epoch":544,"log_start":"0'0","ondisk_log_start":"0'0","created":544,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_clean_scrub_stamp":"2023-09-23T21:15:24.860968+0000","objects_scrubbed":0,"log_size":317,"log_dups_size":0,"ondisk_log_size":317,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:30:07.984391+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":20774912,"num_objects":317,"num_object_clones":0,"num_object_copies":951,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":317,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":317,"num_write_kb":20288,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.a","version":"79'161","reported_seq":693,"reported_epoch":546,"state":"active+clean","last_fresh":"2023-09-23T21:15:27.423168+0000","last_change":"2023-09-23T21:12:30.437790+0000","last_active":"2023-09-23T21:15:27.423168+0000","last_peered":"2023-09-23T21:15:27.423168+0000","last_clean":"2023-09-23T21:15:27.423168+0000","last_became_active":"2023-09-23T21:12:30.437427+0000","last_became_peered":"2023-09-23T21:12:30.437427+0000","last_unstale":"2023-09-23T21:15:27.423168+0000","last_undegraded":"2023-09-23T21:15:27.423168+0000","last_fullsized":"2023-09-23T21:15:27.423168+0000","mapping_epoch":446,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:06:51.075448+0000","last_deep_scrub":"79'161","last_deep_scrub_stamp":"2023-09-23T21:05:32.630292+0000","last_clean_scrub_stamp":"2023-09-23T21:06:51.075448+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0054793890000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,4],"acting":[2,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":395,"reported_epoch":547,"state":"active+clean","last_fresh":"2023-09-23T21:15:30.965794+0000","last_change":"2023-09-23T21:12:10.792918+0000","last_active":"2023-09-23T21:15:30.965794+0000","last_peered":"2023-09-23T21:15:30.965794+0000","last_clean":"2023-09-23T21:15:30.965794+0000","last_became_active":"2023-09-23T21:12:10.792679+0000","last_became_peered":"2023-09-23T21:12:10.792679+0000","last_unstale":"2023-09-23T21:15:30.965794+0000","last_undegraded":"2023-09-23T21:15:30.965794+0000","last_fullsized":"2023-09-23T21:15:30.965794+0000","mapping_epoch":470,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:30:48.008690+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,4],"acting":[3,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":526,"reported_epoch":547,"state":"active+clean","last_fresh":"2023-09-23T21:15:30.965753+0000","last_change":"2023-09-23T21:12:10.790812+0000","last_active":"2023-09-23T21:15:30.965753+0000","last_peered":"2023-09-23T21:15:30.965753+0000","last_clean":"2023-09-23T21:15:30.965753+0000","last_became_active":"2023-09-23T21:12:10.790492+0000","last_became_peered":"2023-09-23T21:12:10.790492+0000","last_unstale":"2023-09-23T21:15:30.965753+0000","last_undegraded":"2023-09-23T21:15:30.965753+0000","last_fullsized":"2023-09-23T21:15:30.965753+0000","mapping_epoch":470,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:02:19.863118+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.d","version":"0'0","reported_seq":12,"reported_epoch":546,"state":"active+clean","last_fresh":"2023-09-23T21:15:27.423433+0000","last_change":"2023-09-23T21:15:25.892834+0000","last_active":"2023-09-23T21:15:27.423433+0000","last_peered":"2023-09-23T21:15:27.423433+0000","last_clean":"2023-09-23T21:15:27.423433+0000","last_became_active":"2023-09-23T21:15:25.892519+0000","last_became_peered":"2023-09-23T21:15:25.892519+0000","last_unstale":"2023-09-23T21:15:27.423433+0000","last_undegraded":"2023-09-23T21:15:27.423433+0000","last_fullsized":"2023-09-23T21:15:27.423433+0000","mapping_epoch":544,"log_start":"0'0","ondisk_log_start":"0'0","created":544,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_clean_scrub_stamp":"2023-09-23T21:15:24.860968+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:46:28.375665+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,5],"acting":[2,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.7","version":"79'161","reported_seq":640,"reported_epoch":547,"state":"active+clean","last_fresh":"2023-09-23T21:15:30.967183+0000","last_change":"2023-09-23T21:13:44.416548+0000","last_active":"2023-09-23T21:15:30.967183+0000","last_peered":"2023-09-23T21:15:30.967183+0000","last_clean":"2023-09-23T21:15:30.967183+0000","last_became_active":"2023-09-23T21:13:44.415617+0000","last_became_peered":"2023-09-23T21:13:44.415617+0000","last_unstale":"2023-09-23T21:15:30.967183+0000","last_undegraded":"2023-09-23T21:15:30.967183+0000","last_fullsized":"2023-09-23T21:15:30.967183+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":3,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:25.916867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:25.916867+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.001665323,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":476,"reported_epoch":545,"state":"active+clean","last_fresh":"2023-09-23T21:15:25.890880+0000","last_change":"2023-09-23T21:13:13.211959+0000","last_active":"2023-09-23T21:15:25.890880+0000","last_peered":"2023-09-23T21:15:25.890880+0000","last_clean":"2023-09-23T21:15:25.890880+0000","last_became_active":"2023-09-23T21:13:13.211712+0000","last_became_peered":"2023-09-23T21:13:13.211712+0000","last_unstale":"2023-09-23T21:15:25.890880+0000","last_undegraded":"2023-09-23T21:15:25.890880+0000","last_fullsized":"2023-09-23T21:15:25.890880+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":495,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:37:21.655775+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,2],"acting":[0,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":225,"reported_epoch":547,"state":"active+clean","last_fresh":"2023-09-23T21:15:30.549938+0000","last_change":"2023-09-23T21:05:04.326889+0000","last_active":"2023-09-23T21:15:30.549938+0000","last_peered":"2023-09-23T21:15:30.549938+0000","last_clean":"2023-09-23T21:15:30.549938+0000","last_became_active":"2023-09-23T21:05:04.326032+0000","last_became_peered":"2023-09-23T21:05:04.326032+0000","last_unstale":"2023-09-23T21:15:30.549938+0000","last_undegraded":"2023-09-23T21:15:30.549938+0000","last_fullsized":"2023-09-23T21:15:30.549938+0000","mapping_epoch":332,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:00:17.507889+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":557,"reported_epoch":547,"state":"active+clean","last_fresh":"2023-09-23T21:15:30.550587+0000","last_change":"2023-09-23T21:13:13.285608+0000","last_active":"2023-09-23T21:15:30.550587+0000","last_peered":"2023-09-23T21:15:30.550587+0000","last_clean":"2023-09-23T21:15:30.550587+0000","last_became_active":"2023-09-23T21:13:13.285255+0000","last_became_peered":"2023-09-23T21:13:13.285255+0000","last_unstale":"2023-09-23T21:15:30.550587+0000","last_undegraded":"2023-09-23T21:15:30.550587+0000","last_fullsized":"2023-09-23T21:15:30.550587+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":495,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:42:30.087599+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.b","version":"548'277","reported_seq":290,"reported_epoch":548,"state":"active+clean","last_fresh":"2023-09-23T21:15:34.229750+0000","last_change":"2023-09-23T21:15:25.906791+0000","last_active":"2023-09-23T21:15:34.229750+0000","last_peered":"2023-09-23T21:15:34.229750+0000","last_clean":"2023-09-23T21:15:34.229750+0000","last_became_active":"2023-09-23T21:15:25.906512+0000","last_became_peered":"2023-09-23T21:15:25.906512+0000","last_unstale":"2023-09-23T21:15:34.229750+0000","last_undegraded":"2023-09-23T21:15:34.229750+0000","last_fullsized":"2023-09-23T21:15:34.229750+0000","mapping_epoch":544,"log_start":"0'0","ondisk_log_start":"0'0","created":544,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_clean_scrub_stamp":"2023-09-23T21:15:24.860968+0000","objects_scrubbed":0,"log_size":277,"log_dups_size":0,"ondisk_log_size":277,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:58:51.749775+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":18153472,"num_objects":277,"num_object_clones":0,"num_object_copies":831,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":277,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":277,"num_write_kb":17728,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1","version":"79'161","reported_seq":617,"reported_epoch":545,"state":"active+clean","last_fresh":"2023-09-23T21:15:25.893861+0000","last_change":"2023-09-23T21:13:12.204135+0000","last_active":"2023-09-23T21:15:25.893861+0000","last_peered":"2023-09-23T21:15:25.893861+0000","last_clean":"2023-09-23T21:15:25.893861+0000","last_became_active":"2023-09-23T21:13:12.203380+0000","last_became_peered":"2023-09-23T21:13:12.203380+0000","last_unstale":"2023-09-23T21:15:25.893861+0000","last_undegraded":"2023-09-23T21:15:25.893861+0000","last_fullsized":"2023-09-23T21:15:25.893861+0000","mapping_epoch":493,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:06:52.575900+0000","last_deep_scrub":"79'161","last_deep_scrub_stamp":"2023-09-23T21:05:30.652189+0000","last_clean_scrub_stamp":"2023-09-23T21:06:52.575900+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":213,"reported_epoch":545,"state":"active+clean","last_fresh":"2023-09-23T21:15:25.891693+0000","last_change":"2023-09-23T21:13:13.238617+0000","last_active":"2023-09-23T21:15:25.891693+0000","last_peered":"2023-09-23T21:15:25.891693+0000","last_clean":"2023-09-23T21:15:25.891693+0000","last_became_active":"2023-09-23T21:13:13.232926+0000","last_became_peered":"2023-09-23T21:13:13.232926+0000","last_unstale":"2023-09-23T21:15:25.891693+0000","last_undegraded":"2023-09-23T21:15:25.891693+0000","last_fullsized":"2023-09-23T21:15:25.891693+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":495,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:17:21.823772+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":481,"reported_epoch":547,"state":"active+clean","last_fresh":"2023-09-23T21:15:30.212308+0000","last_change":"2023-09-23T21:13:14.144918+0000","last_active":"2023-09-23T21:15:30.212308+0000","last_peered":"2023-09-23T21:15:30.212308+0000","last_clean":"2023-09-23T21:15:30.212308+0000","last_became_active":"2023-09-23T21:13:14.132388+0000","last_became_peered":"2023-09-23T21:13:14.132388+0000","last_unstale":"2023-09-23T21:15:30.212308+0000","last_undegraded":"2023-09-23T21:15:30.212308+0000","last_fullsized":"2023-09-23T21:15:30.212308+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":495,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:56:02.619905+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,5],"acting":[4,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":367,"reported_epoch":546,"state":"active+clean","last_fresh":"2023-09-23T21:15:27.423525+0000","last_change":"2023-09-23T21:12:10.764503+0000","last_active":"2023-09-23T21:15:27.423525+0000","last_peered":"2023-09-23T21:15:27.423525+0000","last_clean":"2023-09-23T21:15:27.423525+0000","last_became_active":"2023-09-23T21:12:10.763933+0000","last_became_peered":"2023-09-23T21:12:10.763933+0000","last_unstale":"2023-09-23T21:15:27.423525+0000","last_undegraded":"2023-09-23T21:15:27.423525+0000","last_fullsized":"2023-09-23T21:15:27.423525+0000","mapping_epoch":470,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:28:33.202825+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,7],"acting":[2,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"11.8","version":"0'0","reported_seq":11,"reported_epoch":545,"state":"active+clean","last_fresh":"2023-09-23T21:15:25.906942+0000","last_change":"2023-09-23T21:15:25.905508+0000","last_active":"2023-09-23T21:15:25.906942+0000","last_peered":"2023-09-23T21:15:25.906942+0000","last_clean":"2023-09-23T21:15:25.906942+0000","last_became_active":"2023-09-23T21:15:25.905240+0000","last_became_peered":"2023-09-23T21:15:25.905240+0000","last_unstale":"2023-09-23T21:15:25.906942+0000","last_undegraded":"2023-09-23T21:15:25.906942+0000","last_fullsized":"2023-09-23T21:15:25.906942+0000","mapping_epoch":544,"log_start":"0'0","ondisk_log_start":"0'0","created":544,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_clean_scrub_stamp":"2023-09-23T21:15:24.860968+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:16:01.535087+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,5],"acting":[0,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.2","version":"79'161","reported_seq":826,"reported_epoch":545,"state":"active+clean","last_fresh":"2023-09-23T21:15:25.891554+0000","last_change":"2023-09-23T21:13:12.209907+0000","last_active":"2023-09-23T21:15:25.891554+0000","last_peered":"2023-09-23T21:15:25.891554+0000","last_clean":"2023-09-23T21:15:25.891554+0000","last_became_active":"2023-09-23T21:13:12.205664+0000","last_became_peered":"2023-09-23T21:13:12.205664+0000","last_unstale":"2023-09-23T21:15:25.891554+0000","last_undegraded":"2023-09-23T21:15:25.891554+0000","last_fullsized":"2023-09-23T21:15:25.891554+0000","mapping_epoch":493,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:24.802451+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:24.802451+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:26:16.035788+0000","scrub_duration":0.0090861250000000005,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,4],"acting":[0,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":555,"reported_epoch":546,"state":"active+clean","last_fresh":"2023-09-23T21:15:27.423080+0000","last_change":"2023-09-23T21:13:12.206980+0000","last_active":"2023-09-23T21:15:27.423080+0000","last_peered":"2023-09-23T21:15:27.423080+0000","last_clean":"2023-09-23T21:15:27.423080+0000","last_became_active":"2023-09-23T21:13:12.206650+0000","last_became_peered":"2023-09-23T21:13:12.206650+0000","last_unstale":"2023-09-23T21:15:27.423080+0000","last_undegraded":"2023-09-23T21:15:27.423080+0000","last_fullsized":"2023-09-23T21:15:27.423080+0000","mapping_epoch":493,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:10:01.892936+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":539,"reported_epoch":546,"state":"active+clean","last_fresh":"2023-09-23T21:15:27.423068+0000","last_change":"2023-09-23T21:13:13.407073+0000","last_active":"2023-09-23T21:15:27.423068+0000","last_peered":"2023-09-23T21:15:27.423068+0000","last_clean":"2023-09-23T21:15:27.423068+0000","last_became_active":"2023-09-23T21:13:13.395047+0000","last_became_peered":"2023-09-23T21:13:13.395047+0000","last_unstale":"2023-09-23T21:15:27.423068+0000","last_undegraded":"2023-09-23T21:15:27.423068+0000","last_fullsized":"2023-09-23T21:15:27.423068+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":495,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:04:43.384190+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,5],"acting":[2,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":55,"reported_epoch":545,"state":"active+clean","last_fresh":"2023-09-23T21:15:25.892617+0000","last_change":"2023-09-23T21:13:30.785054+0000","last_active":"2023-09-23T21:15:25.892617+0000","last_peered":"2023-09-23T21:15:25.892617+0000","last_clean":"2023-09-23T21:15:25.892617+0000","last_became_active":"2023-09-23T21:13:30.784701+0000","last_became_peered":"2023-09-23T21:13:30.784701+0000","last_unstale":"2023-09-23T21:15:25.892617+0000","last_undegraded":"2023-09-23T21:15:25.892617+0000","last_fullsized":"2023-09-23T21:15:25.892617+0000","mapping_epoch":501,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:19:20.152252+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.a","version":"0'0","reported_seq":12,"reported_epoch":546,"state":"active+clean","last_fresh":"2023-09-23T21:15:27.422979+0000","last_change":"2023-09-23T21:15:25.893846+0000","last_active":"2023-09-23T21:15:27.422979+0000","last_peered":"2023-09-23T21:15:27.422979+0000","last_clean":"2023-09-23T21:15:27.422979+0000","last_became_active":"2023-09-23T21:15:25.893452+0000","last_became_peered":"2023-09-23T21:15:25.893452+0000","last_unstale":"2023-09-23T21:15:27.422979+0000","last_undegraded":"2023-09-23T21:15:27.422979+0000","last_fullsized":"2023-09-23T21:15:27.422979+0000","mapping_epoch":544,"log_start":"0'0","ondisk_log_start":"0'0","created":544,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_clean_scrub_stamp":"2023-09-23T21:15:24.860968+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:42:32.842297+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,5],"acting":[2,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":333,"reported_epoch":546,"state":"active+clean","last_fresh":"2023-09-23T21:15:27.423625+0000","last_change":"2023-09-23T21:10:26.769764+0000","last_active":"2023-09-23T21:15:27.423625+0000","last_peered":"2023-09-23T21:15:27.423625+0000","last_clean":"2023-09-23T21:15:27.423625+0000","last_became_active":"2023-09-23T21:10:26.769326+0000","last_became_peered":"2023-09-23T21:10:26.769326+0000","last_unstale":"2023-09-23T21:15:27.423625+0000","last_undegraded":"2023-09-23T21:15:27.423625+0000","last_fullsized":"2023-09-23T21:15:27.423625+0000","mapping_epoch":423,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":424,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:57:45.219837+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"79'161","reported_seq":728,"reported_epoch":545,"state":"active+clean","last_fresh":"2023-09-23T21:15:25.889771+0000","last_change":"2023-09-23T21:13:33.766895+0000","last_active":"2023-09-23T21:15:25.889771+0000","last_peered":"2023-09-23T21:15:25.889771+0000","last_clean":"2023-09-23T21:15:25.889771+0000","last_became_active":"2023-09-23T21:13:33.766538+0000","last_became_peered":"2023-09-23T21:13:33.766538+0000","last_unstale":"2023-09-23T21:15:25.889771+0000","last_undegraded":"2023-09-23T21:15:25.889771+0000","last_fullsized":"2023-09-23T21:15:25.889771+0000","mapping_epoch":504,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":505,"parent":"0.0","parent_split_bits":0,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:20.738273+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:20.738273+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:35:35.855987+0000","scrub_duration":0.0088336339999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,2],"acting":[0,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":555,"reported_epoch":547,"state":"active+clean","last_fresh":"2023-09-23T21:15:30.550547+0000","last_change":"2023-09-23T21:11:08.760417+0000","last_active":"2023-09-23T21:15:30.550547+0000","last_peered":"2023-09-23T21:15:30.550547+0000","last_clean":"2023-09-23T21:15:30.550547+0000","last_became_active":"2023-09-23T21:11:08.760150+0000","last_became_peered":"2023-09-23T21:11:08.760150+0000","last_unstale":"2023-09-23T21:15:30.550547+0000","last_undegraded":"2023-09-23T21:15:30.550547+0000","last_fullsized":"2023-09-23T21:15:30.550547+0000","mapping_epoch":443,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":444,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:23:10.849768+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":122,"reported_epoch":547,"state":"active+clean","last_fresh":"2023-09-23T21:15:30.212496+0000","last_change":"2023-09-23T21:13:12.216020+0000","last_active":"2023-09-23T21:15:30.212496+0000","last_peered":"2023-09-23T21:15:30.212496+0000","last_clean":"2023-09-23T21:15:30.212496+0000","last_became_active":"2023-09-23T21:13:12.215180+0000","last_became_peered":"2023-09-23T21:13:12.215180+0000","last_unstale":"2023-09-23T21:15:30.212496+0000","last_undegraded":"2023-09-23T21:15:30.212496+0000","last_fullsized":"2023-09-23T21:15:30.212496+0000","mapping_epoch":493,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:26:18.704684+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.9","version":"548'280","reported_seq":293,"reported_epoch":548,"state":"active+clean","last_fresh":"2023-09-23T21:15:34.220569+0000","last_change":"2023-09-23T21:15:25.903039+0000","last_active":"2023-09-23T21:15:34.220569+0000","last_peered":"2023-09-23T21:15:34.220569+0000","last_clean":"2023-09-23T21:15:34.220569+0000","last_became_active":"2023-09-23T21:15:25.902593+0000","last_became_peered":"2023-09-23T21:15:25.902593+0000","last_unstale":"2023-09-23T21:15:34.220569+0000","last_undegraded":"2023-09-23T21:15:34.220569+0000","last_fullsized":"2023-09-23T21:15:34.220569+0000","mapping_epoch":544,"log_start":"0'0","ondisk_log_start":"0'0","created":544,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_clean_scrub_stamp":"2023-09-23T21:15:24.860968+0000","objects_scrubbed":0,"log_size":280,"log_dups_size":0,"ondisk_log_size":280,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:16:11.481971+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":18350080,"num_objects":280,"num_object_clones":0,"num_object_copies":840,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":280,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":280,"num_write_kb":17920,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,7],"acting":[3,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.3","version":"79'161","reported_seq":850,"reported_epoch":547,"state":"active+clean","last_fresh":"2023-09-23T21:15:30.211906+0000","last_change":"2023-09-23T21:13:33.771279+0000","last_active":"2023-09-23T21:15:30.211906+0000","last_peered":"2023-09-23T21:15:30.211906+0000","last_clean":"2023-09-23T21:15:30.211906+0000","last_became_active":"2023-09-23T21:13:33.770980+0000","last_became_peered":"2023-09-23T21:13:33.770980+0000","last_unstale":"2023-09-23T21:15:30.211906+0000","last_undegraded":"2023-09-23T21:15:30.211906+0000","last_fullsized":"2023-09-23T21:15:30.211906+0000","mapping_epoch":504,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":505,"parent":"0.0","parent_split_bits":3,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:05:57.832302+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:05:57.832302+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:19:42.309604+0000","scrub_duration":0.010206847999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":363,"reported_epoch":545,"state":"active+clean","last_fresh":"2023-09-23T21:15:25.890212+0000","last_change":"2023-09-23T21:13:13.212775+0000","last_active":"2023-09-23T21:15:25.890212+0000","last_peered":"2023-09-23T21:15:25.890212+0000","last_clean":"2023-09-23T21:15:25.890212+0000","last_became_active":"2023-09-23T21:13:13.212460+0000","last_became_peered":"2023-09-23T21:13:13.212460+0000","last_unstale":"2023-09-23T21:15:25.890212+0000","last_undegraded":"2023-09-23T21:15:25.890212+0000","last_fullsized":"2023-09-23T21:15:25.890212+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":495,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:27:10.853970+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":545,"reported_epoch":545,"state":"active+clean","last_fresh":"2023-09-23T21:15:25.893624+0000","last_change":"2023-09-23T21:13:44.409212+0000","last_active":"2023-09-23T21:15:25.893624+0000","last_peered":"2023-09-23T21:15:25.893624+0000","last_clean":"2023-09-23T21:15:25.893624+0000","last_became_active":"2023-09-23T21:13:44.408958+0000","last_became_peered":"2023-09-23T21:13:44.408958+0000","last_unstale":"2023-09-23T21:15:25.893624+0000","last_undegraded":"2023-09-23T21:15:25.893624+0000","last_fullsized":"2023-09-23T21:15:25.893624+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:03:52.749964+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":55,"reported_epoch":545,"state":"active+clean","last_fresh":"2023-09-23T21:15:25.899712+0000","last_change":"2023-09-23T21:13:30.804379+0000","last_active":"2023-09-23T21:15:25.899712+0000","last_peered":"2023-09-23T21:15:25.899712+0000","last_clean":"2023-09-23T21:15:25.899712+0000","last_became_active":"2023-09-23T21:13:30.799907+0000","last_became_peered":"2023-09-23T21:13:30.799907+0000","last_unstale":"2023-09-23T21:15:25.899712+0000","last_undegraded":"2023-09-23T21:15:25.899712+0000","last_fullsized":"2023-09-23T21:15:25.899712+0000","mapping_epoch":501,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:49:42.859212+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,7],"acting":[0,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.c","version":"0'0","reported_seq":12,"reported_epoch":546,"state":"active+clean","last_fresh":"2023-09-23T21:15:27.423630+0000","last_change":"2023-09-23T21:15:25.887291+0000","last_active":"2023-09-23T21:15:27.423630+0000","last_peered":"2023-09-23T21:15:27.423630+0000","last_clean":"2023-09-23T21:15:27.423630+0000","last_became_active":"2023-09-23T21:15:25.887008+0000","last_became_peered":"2023-09-23T21:15:25.887008+0000","last_unstale":"2023-09-23T21:15:27.423630+0000","last_undegraded":"2023-09-23T21:15:27.423630+0000","last_fullsized":"2023-09-23T21:15:27.423630+0000","mapping_epoch":544,"log_start":"0'0","ondisk_log_start":"0'0","created":544,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_clean_scrub_stamp":"2023-09-23T21:15:24.860968+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:18:00.855026+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,7],"acting":[2,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.6","version":"79'161","reported_seq":464,"reported_epoch":545,"state":"active+clean","last_fresh":"2023-09-23T21:15:25.899626+0000","last_change":"2023-09-23T21:13:33.770715+0000","last_active":"2023-09-23T21:15:25.899626+0000","last_peered":"2023-09-23T21:15:25.899626+0000","last_clean":"2023-09-23T21:15:25.899626+0000","last_became_active":"2023-09-23T21:13:33.769303+0000","last_became_peered":"2023-09-23T21:13:33.769303+0000","last_unstale":"2023-09-23T21:15:25.899626+0000","last_undegraded":"2023-09-23T21:15:25.899626+0000","last_fullsized":"2023-09-23T21:15:25.899626+0000","mapping_epoch":504,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":505,"parent":"0.0","parent_split_bits":3,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:10:24.268662+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:10:24.268662+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13417,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":11,"num_read_kb":20,"num_write":5,"num_write_kb":55,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":40252,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":174,"reported_epoch":547,"state":"active+clean","last_fresh":"2023-09-23T21:15:30.214428+0000","last_change":"2023-09-23T21:13:30.770826+0000","last_active":"2023-09-23T21:15:30.214428+0000","last_peered":"2023-09-23T21:15:30.214428+0000","last_clean":"2023-09-23T21:15:30.214428+0000","last_became_active":"2023-09-23T21:13:30.770512+0000","last_became_peered":"2023-09-23T21:13:30.770512+0000","last_unstale":"2023-09-23T21:15:30.214428+0000","last_undegraded":"2023-09-23T21:15:30.214428+0000","last_fullsized":"2023-09-23T21:15:30.214428+0000","mapping_epoch":501,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:44:23.470538+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":430,"reported_epoch":546,"state":"active+clean","last_fresh":"2023-09-23T21:15:27.423706+0000","last_change":"2023-09-23T21:13:30.790590+0000","last_active":"2023-09-23T21:15:27.423706+0000","last_peered":"2023-09-23T21:15:27.423706+0000","last_clean":"2023-09-23T21:15:27.423706+0000","last_became_active":"2023-09-23T21:13:30.790253+0000","last_became_peered":"2023-09-23T21:13:30.790253+0000","last_unstale":"2023-09-23T21:15:27.423706+0000","last_undegraded":"2023-09-23T21:15:27.423706+0000","last_fullsized":"2023-09-23T21:15:27.423706+0000","mapping_epoch":501,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:23:41.843250+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,0],"acting":[2,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":492,"reported_epoch":545,"state":"active+clean","last_fresh":"2023-09-23T21:15:25.902328+0000","last_change":"2023-09-23T21:13:12.210546+0000","last_active":"2023-09-23T21:15:25.902328+0000","last_peered":"2023-09-23T21:15:25.902328+0000","last_clean":"2023-09-23T21:15:25.902328+0000","last_became_active":"2023-09-23T21:13:12.205934+0000","last_became_peered":"2023-09-23T21:13:12.205934+0000","last_unstale":"2023-09-23T21:15:25.902328+0000","last_undegraded":"2023-09-23T21:15:25.902328+0000","last_fullsized":"2023-09-23T21:15:25.902328+0000","mapping_epoch":493,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:21:18.299428+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.f","version":"0'0","reported_seq":12,"reported_epoch":546,"state":"active+clean","last_fresh":"2023-09-23T21:15:27.423198+0000","last_change":"2023-09-23T21:15:25.902442+0000","last_active":"2023-09-23T21:15:27.423198+0000","last_peered":"2023-09-23T21:15:27.423198+0000","last_clean":"2023-09-23T21:15:27.423198+0000","last_became_active":"2023-09-23T21:15:25.902151+0000","last_became_peered":"2023-09-23T21:15:25.902151+0000","last_unstale":"2023-09-23T21:15:27.423198+0000","last_undegraded":"2023-09-23T21:15:27.423198+0000","last_fullsized":"2023-09-23T21:15:27.423198+0000","mapping_epoch":544,"log_start":"0'0","ondisk_log_start":"0'0","created":544,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_clean_scrub_stamp":"2023-09-23T21:15:24.860968+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:32:45.174211+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,0],"acting":[2,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.5","version":"79'161","reported_seq":575,"reported_epoch":547,"state":"active+clean","last_fresh":"2023-09-23T21:15:30.212502+0000","last_change":"2023-09-23T21:13:38.373969+0000","last_active":"2023-09-23T21:15:30.212502+0000","last_peered":"2023-09-23T21:15:30.212502+0000","last_clean":"2023-09-23T21:15:30.212502+0000","last_became_active":"2023-09-23T21:13:38.373633+0000","last_became_peered":"2023-09-23T21:13:38.373633+0000","last_unstale":"2023-09-23T21:15:30.212502+0000","last_undegraded":"2023-09-23T21:15:30.212502+0000","last_fullsized":"2023-09-23T21:15:30.212502+0000","mapping_epoch":507,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":3,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:09:34.892947+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:09:34.892947+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13417,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":11,"num_read_kb":20,"num_write":5,"num_write_kb":55,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":40252,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":87,"reported_epoch":545,"state":"active+clean","last_fresh":"2023-09-23T21:15:25.892785+0000","last_change":"2023-09-23T21:13:12.235743+0000","last_active":"2023-09-23T21:15:25.892785+0000","last_peered":"2023-09-23T21:15:25.892785+0000","last_clean":"2023-09-23T21:15:25.892785+0000","last_became_active":"2023-09-23T21:13:12.216241+0000","last_became_peered":"2023-09-23T21:13:12.216241+0000","last_unstale":"2023-09-23T21:15:25.892785+0000","last_undegraded":"2023-09-23T21:15:25.892785+0000","last_fullsized":"2023-09-23T21:15:25.892785+0000","mapping_epoch":493,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:12:19.090710+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":528,"reported_epoch":547,"state":"active+clean","last_fresh":"2023-09-23T21:15:30.212724+0000","last_change":"2023-09-23T21:13:30.807520+0000","last_active":"2023-09-23T21:15:30.212724+0000","last_peered":"2023-09-23T21:15:30.212724+0000","last_clean":"2023-09-23T21:15:30.212724+0000","last_became_active":"2023-09-23T21:13:30.807222+0000","last_became_peered":"2023-09-23T21:13:30.807222+0000","last_unstale":"2023-09-23T21:15:30.212724+0000","last_undegraded":"2023-09-23T21:15:30.212724+0000","last_fullsized":"2023-09-23T21:15:30.212724+0000","mapping_epoch":501,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:40:31.989749+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":389,"reported_epoch":547,"state":"active+clean","last_fresh":"2023-09-23T21:15:30.965675+0000","last_change":"2023-09-23T21:13:30.794095+0000","last_active":"2023-09-23T21:15:30.965675+0000","last_peered":"2023-09-23T21:15:30.965675+0000","last_clean":"2023-09-23T21:15:30.965675+0000","last_became_active":"2023-09-23T21:13:30.793791+0000","last_became_peered":"2023-09-23T21:13:30.793791+0000","last_unstale":"2023-09-23T21:15:30.965675+0000","last_undegraded":"2023-09-23T21:15:30.965675+0000","last_fullsized":"2023-09-23T21:15:30.965675+0000","mapping_epoch":501,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:36:28.707362+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.e","version":"0'0","reported_seq":12,"reported_epoch":547,"state":"active+clean","last_fresh":"2023-09-23T21:15:30.212811+0000","last_change":"2023-09-23T21:15:25.902844+0000","last_active":"2023-09-23T21:15:30.212811+0000","last_peered":"2023-09-23T21:15:30.212811+0000","last_clean":"2023-09-23T21:15:30.212811+0000","last_became_active":"2023-09-23T21:15:25.902523+0000","last_became_peered":"2023-09-23T21:15:25.902523+0000","last_unstale":"2023-09-23T21:15:30.212811+0000","last_undegraded":"2023-09-23T21:15:30.212811+0000","last_fullsized":"2023-09-23T21:15:30.212811+0000","mapping_epoch":544,"log_start":"0'0","ondisk_log_start":"0'0","created":544,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_clean_scrub_stamp":"2023-09-23T21:15:24.860968+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:34:03.916540+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.4","version":"79'161","reported_seq":919,"reported_epoch":547,"state":"active+clean","last_fresh":"2023-09-23T21:15:30.966676+0000","last_change":"2023-09-23T21:10:29.807785+0000","last_active":"2023-09-23T21:15:30.966676+0000","last_peered":"2023-09-23T21:15:30.966676+0000","last_clean":"2023-09-23T21:15:30.966676+0000","last_became_active":"2023-09-23T21:10:29.807366+0000","last_became_peered":"2023-09-23T21:10:29.807366+0000","last_unstale":"2023-09-23T21:15:30.966676+0000","last_undegraded":"2023-09-23T21:15:30.966676+0000","last_fullsized":"2023-09-23T21:15:30.966676+0000","mapping_epoch":426,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":427,"parent":"0.0","parent_split_bits":3,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:20.012764+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:20.012764+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:24:45.763834+0000","scrub_duration":0.0045251919999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":486,"reported_epoch":546,"state":"active+clean","last_fresh":"2023-09-23T21:15:27.423552+0000","last_change":"2023-09-23T21:10:26.768364+0000","last_active":"2023-09-23T21:15:27.423552+0000","last_peered":"2023-09-23T21:15:27.423552+0000","last_clean":"2023-09-23T21:15:27.423552+0000","last_became_active":"2023-09-23T21:10:26.768052+0000","last_became_peered":"2023-09-23T21:10:26.768052+0000","last_unstale":"2023-09-23T21:15:27.423552+0000","last_undegraded":"2023-09-23T21:15:27.423552+0000","last_fullsized":"2023-09-23T21:15:27.423552+0000","mapping_epoch":423,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":424,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:22:42.201200+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,3],"acting":[2,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.6","version":"70'1","reported_seq":516,"reported_epoch":545,"state":"active+clean","last_fresh":"2023-09-23T21:15:25.899961+0000","last_change":"2023-09-23T21:13:30.793290+0000","last_active":"2023-09-23T21:15:25.899961+0000","last_peered":"2023-09-23T21:15:25.899961+0000","last_clean":"2023-09-23T21:15:25.899961+0000","last_became_active":"2023-09-23T21:13:30.786079+0000","last_became_peered":"2023-09-23T21:13:30.786079+0000","last_unstale":"2023-09-23T21:15:25.899961+0000","last_undegraded":"2023-09-23T21:15:25.899961+0000","last_fullsized":"2023-09-23T21:15:25.899961+0000","mapping_epoch":501,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:34:43.281095+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":122,"reported_epoch":547,"state":"active+clean","last_fresh":"2023-09-23T21:15:30.213062+0000","last_change":"2023-09-23T21:13:12.216895+0000","last_active":"2023-09-23T21:15:30.213062+0000","last_peered":"2023-09-23T21:15:30.213062+0000","last_clean":"2023-09-23T21:15:30.213062+0000","last_became_active":"2023-09-23T21:13:12.216654+0000","last_became_peered":"2023-09-23T21:13:12.216654+0000","last_unstale":"2023-09-23T21:15:30.213062+0000","last_undegraded":"2023-09-23T21:15:30.213062+0000","last_fullsized":"2023-09-23T21:15:30.213062+0000","mapping_epoch":493,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:02:40.430096+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.19","version":"79'161","reported_seq":707,"reported_epoch":545,"state":"active+clean","last_fresh":"2023-09-23T21:15:25.893321+0000","last_change":"2023-09-23T21:15:23.795956+0000","last_active":"2023-09-23T21:15:25.893321+0000","last_peered":"2023-09-23T21:15:25.893321+0000","last_clean":"2023-09-23T21:15:25.893321+0000","last_became_active":"2023-09-23T21:15:23.795663+0000","last_became_peered":"2023-09-23T21:15:23.795663+0000","last_unstale":"2023-09-23T21:15:25.893321+0000","last_undegraded":"2023-09-23T21:15:25.893321+0000","last_fullsized":"2023-09-23T21:15:25.893321+0000","mapping_epoch":542,"log_start":"0'0","ondisk_log_start":"0'0","created":479,"last_epoch_clean":543,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:10:37.696951+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:10:37.696951+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.001386504,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":55,"reported_epoch":545,"state":"active+clean","last_fresh":"2023-09-23T21:15:25.902239+0000","last_change":"2023-09-23T21:13:30.791297+0000","last_active":"2023-09-23T21:15:25.902239+0000","last_peered":"2023-09-23T21:15:25.902239+0000","last_clean":"2023-09-23T21:15:25.902239+0000","last_became_active":"2023-09-23T21:13:30.788721+0000","last_became_peered":"2023-09-23T21:13:30.788721+0000","last_unstale":"2023-09-23T21:15:25.902239+0000","last_undegraded":"2023-09-23T21:15:25.902239+0000","last_fullsized":"2023-09-23T21:15:25.902239+0000","mapping_epoch":501,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:31:18.147883+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":50,"reported_epoch":546,"state":"active+clean","last_fresh":"2023-09-23T21:15:27.423677+0000","last_change":"2023-09-23T21:13:38.373814+0000","last_active":"2023-09-23T21:15:27.423677+0000","last_peered":"2023-09-23T21:15:27.423677+0000","last_clean":"2023-09-23T21:15:27.423677+0000","last_became_active":"2023-09-23T21:13:38.373458+0000","last_became_peered":"2023-09-23T21:13:38.373458+0000","last_unstale":"2023-09-23T21:15:27.423677+0000","last_undegraded":"2023-09-23T21:15:27.423677+0000","last_fullsized":"2023-09-23T21:15:27.423677+0000","mapping_epoch":507,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:23:08.288401+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.1e","version":"79'161","reported_seq":484,"reported_epoch":547,"state":"active+clean","last_fresh":"2023-09-23T21:15:30.966637+0000","last_change":"2023-09-23T21:15:20.791055+0000","last_active":"2023-09-23T21:15:30.966637+0000","last_peered":"2023-09-23T21:15:30.966637+0000","last_clean":"2023-09-23T21:15:30.966637+0000","last_became_active":"2023-09-23T21:15:20.790772+0000","last_became_peered":"2023-09-23T21:15:20.790772+0000","last_unstale":"2023-09-23T21:15:30.966637+0000","last_undegraded":"2023-09-23T21:15:30.966637+0000","last_fullsized":"2023-09-23T21:15:30.966637+0000","mapping_epoch":539,"log_start":"0'0","ondisk_log_start":"0'0","created":479,"last_epoch_clean":540,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:10:39.567232+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:10:39.567232+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.024789746000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,4],"acting":[3,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":129,"reported_epoch":547,"state":"active+clean","last_fresh":"2023-09-23T21:15:30.213346+0000","last_change":"2023-09-23T21:15:06.721854+0000","last_active":"2023-09-23T21:15:30.213346+0000","last_peered":"2023-09-23T21:15:30.213346+0000","last_clean":"2023-09-23T21:15:30.213346+0000","last_became_active":"2023-09-23T21:12:14.209408+0000","last_became_peered":"2023-09-23T21:12:14.209408+0000","last_unstale":"2023-09-23T21:15:30.213346+0000","last_undegraded":"2023-09-23T21:15:30.213346+0000","last_fullsized":"2023-09-23T21:15:30.213346+0000","mapping_epoch":470,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":532,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:45:40.142413+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,7],"acting":[4,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":112,"reported_epoch":547,"state":"active+clean","last_fresh":"2023-09-23T21:15:30.212071+0000","last_change":"2023-09-23T21:12:10.770474+0000","last_active":"2023-09-23T21:15:30.212071+0000","last_peered":"2023-09-23T21:15:30.212071+0000","last_clean":"2023-09-23T21:15:30.212071+0000","last_became_active":"2023-09-23T21:12:10.770184+0000","last_became_peered":"2023-09-23T21:12:10.770184+0000","last_unstale":"2023-09-23T21:15:30.212071+0000","last_undegraded":"2023-09-23T21:15:30.212071+0000","last_fullsized":"2023-09-23T21:15:30.212071+0000","mapping_epoch":470,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:05:09.407151+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":337,"reported_epoch":546,"state":"active+clean","last_fresh":"2023-09-23T21:15:27.423679+0000","last_change":"2023-09-23T21:12:10.771160+0000","last_active":"2023-09-23T21:15:27.423679+0000","last_peered":"2023-09-23T21:15:27.423679+0000","last_clean":"2023-09-23T21:15:27.423679+0000","last_became_active":"2023-09-23T21:12:10.757808+0000","last_became_peered":"2023-09-23T21:12:10.757808+0000","last_unstale":"2023-09-23T21:15:27.423679+0000","last_undegraded":"2023-09-23T21:15:27.423679+0000","last_fullsized":"2023-09-23T21:15:27.423679+0000","mapping_epoch":470,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:30:50.987139+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":122,"reported_epoch":547,"state":"active+clean","last_fresh":"2023-09-23T21:15:30.211965+0000","last_change":"2023-09-23T21:13:12.216602+0000","last_active":"2023-09-23T21:15:30.211965+0000","last_peered":"2023-09-23T21:15:30.211965+0000","last_clean":"2023-09-23T21:15:30.211965+0000","last_became_active":"2023-09-23T21:13:12.216204+0000","last_became_peered":"2023-09-23T21:13:12.216204+0000","last_unstale":"2023-09-23T21:15:30.211965+0000","last_undegraded":"2023-09-23T21:15:30.211965+0000","last_fullsized":"2023-09-23T21:15:30.211965+0000","mapping_epoch":493,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:23:37.043390+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1c","version":"79'161","reported_seq":931,"reported_epoch":547,"state":"active+clean","last_fresh":"2023-09-23T21:15:30.966600+0000","last_change":"2023-09-23T21:15:23.800688+0000","last_active":"2023-09-23T21:15:30.966600+0000","last_peered":"2023-09-23T21:15:30.966600+0000","last_clean":"2023-09-23T21:15:30.966600+0000","last_became_active":"2023-09-23T21:15:23.800361+0000","last_became_peered":"2023-09-23T21:15:23.800361+0000","last_unstale":"2023-09-23T21:15:30.966600+0000","last_undegraded":"2023-09-23T21:15:30.966600+0000","last_fullsized":"2023-09-23T21:15:30.966600+0000","mapping_epoch":542,"log_start":"0'0","ondisk_log_start":"0'0","created":479,"last_epoch_clean":543,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:22.040907+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:22.040907+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.013021176000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":163840,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":163840,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":136,"reported_epoch":546,"state":"active+clean","last_fresh":"2023-09-23T21:15:27.423322+0000","last_change":"2023-09-23T21:12:10.779066+0000","last_active":"2023-09-23T21:15:27.423322+0000","last_peered":"2023-09-23T21:15:27.423322+0000","last_clean":"2023-09-23T21:15:27.423322+0000","last_became_active":"2023-09-23T21:12:10.778691+0000","last_became_peered":"2023-09-23T21:12:10.778691+0000","last_unstale":"2023-09-23T21:15:27.423322+0000","last_undegraded":"2023-09-23T21:15:27.423322+0000","last_fullsized":"2023-09-23T21:15:27.423322+0000","mapping_epoch":470,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:07:22.043268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,7],"acting":[2,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":215,"reported_epoch":546,"state":"active+clean","last_fresh":"2023-09-23T21:15:27.423090+0000","last_change":"2023-09-23T21:13:12.200767+0000","last_active":"2023-09-23T21:15:27.423090+0000","last_peered":"2023-09-23T21:15:27.423090+0000","last_clean":"2023-09-23T21:15:27.423090+0000","last_became_active":"2023-09-23T21:13:12.200435+0000","last_became_peered":"2023-09-23T21:13:12.200435+0000","last_unstale":"2023-09-23T21:15:27.423090+0000","last_undegraded":"2023-09-23T21:15:27.423090+0000","last_fullsized":"2023-09-23T21:15:27.423090+0000","mapping_epoch":493,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:15:44.937886+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.1d","version":"79'161","reported_seq":679,"reported_epoch":547,"state":"active+clean","last_fresh":"2023-09-23T21:15:30.966638+0000","last_change":"2023-09-23T21:15:23.802125+0000","last_active":"2023-09-23T21:15:30.966638+0000","last_peered":"2023-09-23T21:15:30.966638+0000","last_clean":"2023-09-23T21:15:30.966638+0000","last_became_active":"2023-09-23T21:15:23.801704+0000","last_became_peered":"2023-09-23T21:15:23.801704+0000","last_unstale":"2023-09-23T21:15:30.966638+0000","last_undegraded":"2023-09-23T21:15:30.966638+0000","last_fullsized":"2023-09-23T21:15:30.966638+0000","mapping_epoch":542,"log_start":"0'0","ondisk_log_start":"0'0","created":479,"last_epoch_clean":543,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:09:36.858077+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:09:36.858077+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.010304610000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,2],"acting":[3,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":72,"reported_epoch":545,"state":"active+clean","last_fresh":"2023-09-23T21:15:25.892449+0000","last_change":"2023-09-23T21:13:38.380574+0000","last_active":"2023-09-23T21:15:25.892449+0000","last_peered":"2023-09-23T21:15:25.892449+0000","last_clean":"2023-09-23T21:15:25.892449+0000","last_became_active":"2023-09-23T21:13:38.373252+0000","last_became_peered":"2023-09-23T21:13:38.373252+0000","last_unstale":"2023-09-23T21:15:25.892449+0000","last_undegraded":"2023-09-23T21:15:25.892449+0000","last_fullsized":"2023-09-23T21:15:25.892449+0000","mapping_epoch":507,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:58:14.788031+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":11,"num_bytes_recovered":3324,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":122,"reported_epoch":547,"state":"active+clean","last_fresh":"2023-09-23T21:15:30.211762+0000","last_change":"2023-09-23T21:12:10.793610+0000","last_active":"2023-09-23T21:15:30.211762+0000","last_peered":"2023-09-23T21:15:30.211762+0000","last_clean":"2023-09-23T21:15:30.211762+0000","last_became_active":"2023-09-23T21:12:10.793081+0000","last_became_peered":"2023-09-23T21:12:10.793081+0000","last_unstale":"2023-09-23T21:15:30.211762+0000","last_undegraded":"2023-09-23T21:15:30.211762+0000","last_fullsized":"2023-09-23T21:15:30.211762+0000","mapping_epoch":470,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:34:36.291550+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":11,"num_pg":16,"stat_sum":{"num_bytes":76873728,"num_objects":1173,"num_object_clones":0,"num_object_copies":3519,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1173,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1173,"num_write_kb":75072,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":179961856,"data_stored":179961856,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1173,"ondisk_log_size":1173,"up":48,"acting":48,"num_store_stats":6},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":60,"num_bytes_recovered":8898,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":118784,"data_stored":21350,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":14322,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":31,"stat_sum":{"num_bytes":848130,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":78,"num_read_kb":144,"num_write":38,"num_write_kb":385,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":708294,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2109440,"data_stored":2098784,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4991,"ondisk_log_size":4991,"up":93,"acting":93,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201701,"num_pgs":63,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":514588,"kb_used_data":5972,"kb_used_omap":0,"kb_used_meta":508608,"kb_avail":93230564,"statfs":{"total":95995035648,"available":95468097536,"internally_reserved":0,"allocated":6115328,"data_stored":4542876,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":520814592},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:15:15 2023","interfaces":[{"interface":"back","average":{"1min":1.3440000000000001,"5min":2.2090000000000001,"15min":2.3530000000000002},"min":{"1min":0.25,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":4.3070000000000004,"5min":37.207999999999998,"15min":37.207999999999998},"last":0.373},{"interface":"front","average":{"1min":2.089,"5min":2.2050000000000001,"15min":2.2240000000000002},"min":{"1min":0.253,"5min":0.253,"15min":0.253},"max":{"1min":10.429,"5min":37.122,"15min":37.122},"last":0.47199999999999998}]},{"osd":1,"last update":"Sat Sep 23 21:15:09 2023","interfaces":[{"interface":"back","average":{"1min":2.0329999999999999,"5min":1.3320000000000001,"15min":2.0289999999999999},"min":{"1min":0.28000000000000003,"5min":0.27500000000000002,"15min":0.26700000000000002},"max":{"1min":6.0860000000000003,"5min":14.034000000000001,"15min":14.034000000000001},"last":0.39700000000000002},{"interface":"front","average":{"1min":1.554,"5min":1.4359999999999999,"15min":1.9079999999999999},"min":{"1min":0.33900000000000002,"5min":0.29899999999999999,"15min":0.26300000000000001},"max":{"1min":4.2690000000000001,"5min":11.226000000000001,"15min":11.226000000000001},"last":0.60299999999999998}]},{"osd":2,"last update":"Sat Sep 23 21:14:30 2023","interfaces":[{"interface":"back","average":{"1min":0.80900000000000005,"5min":1.0880000000000001,"15min":1.7170000000000001},"min":{"1min":0.29699999999999999,"5min":0.26400000000000001,"15min":0.26200000000000001},"max":{"1min":3.0190000000000001,"5min":6.4219999999999997,"15min":14.478999999999999},"last":0.34799999999999998},{"interface":"front","average":{"1min":1.101,"5min":1.3520000000000001,"15min":1.7330000000000001},"min":{"1min":0.30399999999999999,"5min":0.26700000000000002,"15min":0.26500000000000001},"max":{"1min":5.7640000000000002,"5min":11.198,"15min":14.135},"last":0.53700000000000003}]},{"osd":3,"last update":"Sat Sep 23 21:14:58 2023","interfaces":[{"interface":"back","average":{"1min":1.7270000000000001,"5min":1.3009999999999999,"15min":2},"min":{"1min":0.30599999999999999,"5min":0.30599999999999999,"15min":0.29099999999999998},"max":{"1min":4.1619999999999999,"5min":7.5190000000000001,"15min":31.786000000000001},"last":0.441},{"interface":"front","average":{"1min":2.7410000000000001,"5min":1.4890000000000001,"15min":2.0390000000000001},"min":{"1min":0.35599999999999998,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":11.743,"5min":11.743,"15min":32.429000000000002},"last":0.56399999999999995}]},{"osd":4,"last update":"Sat Sep 23 21:14:58 2023","interfaces":[{"interface":"back","average":{"1min":1.1100000000000001,"5min":1.48,"15min":1.143},"min":{"1min":0.32500000000000001,"5min":0.26200000000000001,"15min":0.248},"max":{"1min":4.2130000000000001,"5min":11.297000000000001,"15min":11.297000000000001},"last":0.66600000000000004},{"interface":"front","average":{"1min":1.085,"5min":1.24,"15min":1.105},"min":{"1min":0.29699999999999999,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":3.7850000000000001,"5min":11.093999999999999,"15min":12.752000000000001},"last":0.41499999999999998}]},{"osd":5,"last update":"Sat Sep 23 21:14:58 2023","interfaces":[{"interface":"back","average":{"1min":1.359,"5min":1.1990000000000001,"15min":1.075},"min":{"1min":0.32300000000000001,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":6.133,"5min":9.4789999999999992,"15min":9.4789999999999992},"last":0.58799999999999997},{"interface":"front","average":{"1min":0.98699999999999999,"5min":1.3360000000000001,"15min":1.1279999999999999},"min":{"1min":0.33300000000000002,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":4.181,"5min":11.257,"15min":11.257},"last":0.63800000000000001}]},{"osd":6,"last update":"Sat Sep 23 21:15:09 2023","interfaces":[{"interface":"back","average":{"1min":1.325,"5min":1.3029999999999999,"15min":1.095},"min":{"1min":0.40200000000000002,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":7.5339999999999998,"5min":10.941000000000001,"15min":10.941000000000001},"last":0.61499999999999999},{"interface":"front","average":{"1min":1.298,"5min":1.2130000000000001,"15min":1.1799999999999999},"min":{"1min":0.39200000000000002,"5min":0.33100000000000002,"15min":0.32300000000000001},"max":{"1min":4.7720000000000002,"5min":8.923,"15min":12.715},"last":0.65100000000000002}]}]},{"osd":6,"up_from":0,"seq":704374636785,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":1,"up_from":0,"seq":1503238553717,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":0,"up_from":493,"seq":2117418876957,"num_pgs":78,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":605388,"kb_used_data":415108,"kb_used_omap":36,"kb_used_meta":190235,"kb_avail":93139764,"statfs":{"total":95995035648,"available":95375118336,"internally_reserved":0,"allocated":425070592,"data_stored":422527624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":37336,"internal_metadata":194801192},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 21:15:17 2023","interfaces":[{"interface":"back","average":{"1min":1.6990000000000001,"5min":0.94399999999999995,"15min":0.81799999999999995},"min":{"1min":0.22500000000000001,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":4.665,"5min":4.9000000000000004,"15min":4.9000000000000004},"last":0.375},{"interface":"front","average":{"1min":1.5620000000000001,"5min":0.92900000000000005,"15min":0.82299999999999995},"min":{"1min":0.20799999999999999,"5min":0.20799999999999999,"15min":0.20799999999999999},"max":{"1min":11.964,"5min":11.964,"15min":11.964},"last":0.46200000000000002}]},{"osd":2,"last update":"Sat Sep 23 21:15:17 2023","interfaces":[{"interface":"back","average":{"1min":1.661,"5min":0.85499999999999998,"15min":0.72099999999999997},"min":{"1min":0.29399999999999998,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":7.133,"5min":7.133,"15min":7.133},"last":0.32400000000000001},{"interface":"front","average":{"1min":1.647,"5min":1.103,"15min":1.0129999999999999},"min":{"1min":0.27900000000000003,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":6.6470000000000002,"5min":6.6470000000000002,"15min":6.6470000000000002},"last":0.52600000000000002}]},{"osd":3,"last update":"Sat Sep 23 21:15:17 2023","interfaces":[{"interface":"back","average":{"1min":1.8939999999999999,"5min":1.073,"15min":0.93600000000000005},"min":{"1min":0.27300000000000002,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":6.2060000000000004,"5min":6.2060000000000004,"15min":6.2060000000000004},"last":0.64900000000000002},{"interface":"front","average":{"1min":1.4199999999999999,"5min":1.1779999999999999,"15min":1.1379999999999999},"min":{"1min":0.28299999999999997,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":4.8040000000000003,"5min":4.8040000000000003,"15min":4.8040000000000003},"last":0.29599999999999999}]},{"osd":4,"last update":"Sat Sep 23 21:15:17 2023","interfaces":[{"interface":"back","average":{"1min":1.2789999999999999,"5min":1.1299999999999999,"15min":1.105},"min":{"1min":0.312,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":4.4409999999999998,"5min":4.4409999999999998,"15min":4.4409999999999998},"last":0.38600000000000001},{"interface":"front","average":{"1min":1.4570000000000001,"5min":1.0880000000000001,"15min":1.026},"min":{"1min":0.41699999999999998,"5min":0.317,"15min":0.317},"max":{"1min":4.6929999999999996,"5min":5.093,"15min":5.093},"last":0.48899999999999999}]},{"osd":5,"last update":"Sat Sep 23 21:15:17 2023","interfaces":[{"interface":"back","average":{"1min":1.413,"5min":1.2709999999999999,"15min":1.2470000000000001},"min":{"1min":0.29999999999999999,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":4.0869999999999997,"5min":4.5640000000000001,"15min":4.5640000000000001},"last":0.57299999999999995},{"interface":"front","average":{"1min":1.3959999999999999,"5min":1.1299999999999999,"15min":1.0860000000000001},"min":{"1min":0.313,"5min":0.313,"15min":0.313},"max":{"1min":7.4569999999999999,"5min":7.4569999999999999,"15min":7.4569999999999999},"last":0.503}]},{"osd":6,"last update":"Sat Sep 23 21:15:17 2023","interfaces":[{"interface":"back","average":{"1min":1.8420000000000001,"5min":1.27,"15min":1.1739999999999999},"min":{"1min":0.46600000000000003,"5min":0.38100000000000001,"15min":0.38100000000000001},"max":{"1min":7.2050000000000001,"5min":7.2050000000000001,"15min":7.2050000000000001},"last":0.54400000000000004},{"interface":"front","average":{"1min":1.5640000000000001,"5min":1.359,"15min":1.325},"min":{"1min":0.39600000000000002,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":4.7850000000000001,"5min":5.0860000000000003,"15min":5.0860000000000003},"last":0.61499999999999999}]},{"osd":7,"last update":"Sat Sep 23 21:15:17 2023","interfaces":[{"interface":"back","average":{"1min":2.1600000000000001,"5min":1.399,"15min":1.272},"min":{"1min":0.40899999999999997,"5min":0.40899999999999997,"15min":0.40899999999999997},"max":{"1min":7.5810000000000004,"5min":7.5810000000000004,"15min":7.5810000000000004},"last":0.441},{"interface":"front","average":{"1min":1.6850000000000001,"5min":1.2629999999999999,"15min":1.1930000000000001},"min":{"1min":0.56000000000000005,"5min":0.441,"15min":0.441},"max":{"1min":4.6349999999999998,"5min":5.1289999999999996,"15min":5.1289999999999996},"last":0.41999999999999998}]}]},{"osd":2,"up_from":93,"seq":399431958829,"num_pgs":49,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":948636,"kb_used_data":155156,"kb_used_omap":81,"kb_used_meta":793390,"kb_avail":92796516,"statfs":{"total":95995035648,"available":95023632384,"internally_reserved":0,"allocated":158879744,"data_stored":156327577,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":83750,"internal_metadata":812431578},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:15:17 2023","interfaces":[{"interface":"back","average":{"1min":2.2210000000000001,"5min":1.1100000000000001,"15min":0.92500000000000004},"min":{"1min":0.20899999999999999,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":9.9329999999999998,"5min":9.9329999999999998,"15min":9.9329999999999998},"last":0.39300000000000002},{"interface":"front","average":{"1min":2.6659999999999999,"5min":1.865,"15min":1.7310000000000001},"min":{"1min":0.24399999999999999,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":11.269,"5min":18.111999999999998,"15min":18.111999999999998},"last":0.35299999999999998}]},{"osd":1,"last update":"Sat Sep 23 21:15:17 2023","interfaces":[{"interface":"back","average":{"1min":2.0539999999999998,"5min":1.2629999999999999,"15min":2.472},"min":{"1min":0.20599999999999999,"5min":0.188,"15min":0.188},"max":{"1min":9.9890000000000008,"5min":11.005000000000001,"15min":34.396999999999998},"last":0.52100000000000002},{"interface":"front","average":{"1min":2.2280000000000002,"5min":1.296,"15min":2.6859999999999999},"min":{"1min":0.26200000000000001,"5min":0.217,"15min":0.217},"max":{"1min":16.274000000000001,"5min":18.081,"15min":34.462000000000003},"last":0.48699999999999999}]},{"osd":3,"last update":"Sat Sep 23 21:14:58 2023","interfaces":[{"interface":"back","average":{"1min":2.246,"5min":1.274,"15min":2.4550000000000001},"min":{"1min":0.37,"5min":0.20599999999999999,"15min":0.20599999999999999},"max":{"1min":14.442,"5min":14.442,"15min":41.057000000000002},"last":0.371},{"interface":"front","average":{"1min":2.633,"5min":1.4419999999999999,"15min":2.6970000000000001},"min":{"1min":0.28599999999999998,"5min":0.21299999999999999,"15min":0.21299999999999999},"max":{"1min":14.622,"5min":14.622,"15min":41.012},"last":0.56999999999999995}]},{"osd":4,"last update":"Sat Sep 23 21:15:17 2023","interfaces":[{"interface":"back","average":{"1min":1.5780000000000001,"5min":1.2749999999999999,"15min":1.4219999999999999},"min":{"1min":0.32500000000000001,"5min":0.249,"15min":0.249},"max":{"1min":7.9770000000000003,"5min":10.266999999999999,"15min":10.266999999999999},"last":0.504},{"interface":"front","average":{"1min":1.74,"5min":1.4419999999999999,"15min":1.8029999999999999},"min":{"1min":0.308,"5min":0.29399999999999998,"15min":0.26700000000000002},"max":{"1min":5.556,"5min":7.5960000000000001,"15min":14.039999999999999},"last":0.72199999999999998}]},{"osd":5,"last update":"Sat Sep 23 21:15:17 2023","interfaces":[{"interface":"back","average":{"1min":1.1850000000000001,"5min":1.218,"15min":1.5049999999999999},"min":{"1min":0.34699999999999998,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":3.7250000000000001,"5min":9.5039999999999996,"15min":10.069000000000001},"last":0.55400000000000005},{"interface":"front","average":{"1min":1.6779999999999999,"5min":1.2849999999999999,"15min":1.7529999999999999},"min":{"1min":0.33800000000000002,"5min":0.29699999999999999,"15min":0.29399999999999998},"max":{"1min":5.8330000000000002,"5min":7.5549999999999997,"15min":13.98},"last":0.76800000000000002}]},{"osd":6,"last update":"Sat Sep 23 21:15:17 2023","interfaces":[{"interface":"back","average":{"1min":1.9239999999999999,"5min":1.571,"15min":1.988},"min":{"1min":0.46300000000000002,"5min":0.373,"15min":0.35699999999999998},"max":{"1min":7.8559999999999999,"5min":7.8559999999999999,"15min":14.016},"last":0.84299999999999997},{"interface":"front","average":{"1min":1.7330000000000001,"5min":1.7330000000000001,"15min":2.157},"min":{"1min":0.377,"5min":0.377,"15min":0.35599999999999998},"max":{"1min":5.649,"5min":9.3469999999999995,"15min":14.068},"last":0.879}]},{"osd":7,"last update":"Sat Sep 23 21:15:17 2023","interfaces":[{"interface":"back","average":{"1min":1.6839999999999999,"5min":1.462,"15min":1.6040000000000001},"min":{"1min":0.33700000000000002,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":6.0510000000000002,"5min":18.021999999999998,"15min":18.021999999999998},"last":0.63100000000000001},{"interface":"front","average":{"1min":1.359,"5min":1.425,"15min":1.6910000000000001},"min":{"1min":0.41899999999999998,"5min":0.34799999999999998,"15min":0.32200000000000001},"max":{"1min":3.7549999999999999,"5min":10.359,"15min":10.92},"last":0.67500000000000004}]}]},{"osd":3,"up_from":98,"seq":420906795303,"num_pgs":73,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1484360,"kb_used_data":592000,"kb_used_omap":7,"kb_used_meta":892344,"kb_avail":92260792,"statfs":{"total":95995035648,"available":94475051008,"internally_reserved":0,"allocated":606208000,"data_stored":603678126,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7576,"internal_metadata":913760872},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:15:13 2023","interfaces":[{"interface":"back","average":{"1min":10.904,"5min":2.4510000000000001,"15min":1.042},"min":{"1min":0.28499999999999998,"5min":0.20499999999999999,"15min":0.20499999999999999},"max":{"1min":147.38900000000001,"5min":147.38900000000001,"15min":147.38900000000001},"last":0.29799999999999999},{"interface":"front","average":{"1min":10.178000000000001,"5min":2.5699999999999998,"15min":1.302},"min":{"1min":0.23100000000000001,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":149.989,"5min":149.989,"15min":149.989},"last":0.23599999999999999}]},{"osd":1,"last update":"Sat Sep 23 21:15:13 2023","interfaces":[{"interface":"back","average":{"1min":1.5589999999999999,"5min":1.2,"15min":2.1880000000000002},"min":{"1min":0.23699999999999999,"5min":0.20000000000000001,"15min":0.20000000000000001},"max":{"1min":5.0700000000000003,"5min":22.577999999999999,"15min":22.577999999999999},"last":0.26800000000000002},{"interface":"front","average":{"1min":1.5600000000000001,"5min":1.345,"15min":2.1389999999999998},"min":{"1min":0.26100000000000001,"5min":0.216,"15min":0.216},"max":{"1min":5.7329999999999997,"5min":21.617999999999999,"15min":21.617999999999999},"last":0.39700000000000002}]},{"osd":2,"last update":"Sat Sep 23 21:15:00 2023","interfaces":[{"interface":"back","average":{"1min":1.78,"5min":1.9419999999999999,"15min":2.468},"min":{"1min":0.23899999999999999,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":10.946,"5min":56.317999999999998,"15min":56.317999999999998},"last":0.35999999999999999},{"interface":"front","average":{"1min":1.554,"5min":1.978,"15min":2.5609999999999999},"min":{"1min":0.246,"5min":0.216,"15min":0.216},"max":{"1min":6.29,"5min":55.988,"15min":55.988},"last":0.318}]},{"osd":4,"last update":"Sat Sep 23 21:15:00 2023","interfaces":[{"interface":"back","average":{"1min":2.2959999999999998,"5min":1.526,"15min":1.831},"min":{"1min":0.29899999999999999,"5min":0.29899999999999999,"15min":0.28399999999999997},"max":{"1min":10.807,"5min":21.478999999999999,"15min":21.478999999999999},"last":0.40799999999999997},{"interface":"front","average":{"1min":1.415,"5min":1.4830000000000001,"15min":1.9710000000000001},"min":{"1min":0.314,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":6.3499999999999996,"5min":22.402000000000001,"15min":22.402000000000001},"last":0.41899999999999998}]},{"osd":5,"last update":"Sat Sep 23 21:15:00 2023","interfaces":[{"interface":"back","average":{"1min":1.925,"5min":1.456,"15min":1.95},"min":{"1min":0.34599999999999997,"5min":0.311,"15min":0.311},"max":{"1min":10.882999999999999,"5min":21.562000000000001,"15min":21.562000000000001},"last":0.434},{"interface":"front","average":{"1min":1.429,"5min":1.302,"15min":2.141},"min":{"1min":0.34799999999999998,"5min":0.31,"15min":0.31},"max":{"1min":4.2779999999999996,"5min":8.8450000000000006,"15min":14.029},"last":0.47699999999999998}]},{"osd":6,"last update":"Sat Sep 23 21:15:17 2023","interfaces":[{"interface":"back","average":{"1min":1.194,"5min":1.3919999999999999,"15min":1.9930000000000001},"min":{"1min":0.35899999999999999,"5min":0.34999999999999998,"15min":0.33900000000000002},"max":{"1min":3.536,"5min":22.446999999999999,"15min":22.446999999999999},"last":0.44900000000000001},{"interface":"front","average":{"1min":1.518,"5min":1.7969999999999999,"15min":2.1230000000000002},"min":{"1min":0.378,"5min":0.378,"15min":0.373},"max":{"1min":4.3760000000000003,"5min":12.375999999999999,"15min":13.576000000000001},"last":0.57199999999999995}]},{"osd":7,"last update":"Sat Sep 23 21:15:00 2023","interfaces":[{"interface":"back","average":{"1min":2.4289999999999998,"5min":1.8480000000000001,"15min":2.1000000000000001},"min":{"1min":0.439,"5min":0.38800000000000001,"15min":0.33900000000000002},"max":{"1min":10.914999999999999,"5min":21.518999999999998,"15min":21.518999999999998},"last":0.46700000000000003},{"interface":"front","average":{"1min":2.4220000000000002,"5min":1.903,"15min":2.3380000000000001},"min":{"1min":0.42499999999999999,"5min":0.42499999999999999,"15min":0.38900000000000001},"max":{"1min":8.2590000000000003,"5min":9.9879999999999995,"15min":13.603999999999999},"last":0.53400000000000003}]}]},{"osd":4,"up_from":31,"seq":133143986609,"num_pgs":60,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":569688,"kb_used_data":85136,"kb_used_omap":0,"kb_used_meta":484544,"kb_avail":93175464,"statfs":{"total":95995035648,"available":95411675136,"internally_reserved":0,"allocated":87179264,"data_stored":85610617,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":496173056},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:15:14 2023","interfaces":[{"interface":"back","average":{"1min":1.944,"5min":1.1080000000000001,"15min":0.96899999999999997},"min":{"1min":0.23799999999999999,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":8.0079999999999991,"5min":8.0079999999999991,"15min":8.0079999999999991},"last":0.54800000000000004},{"interface":"front","average":{"1min":0.876,"5min":1.0820000000000001,"15min":1.1160000000000001},"min":{"1min":0.23799999999999999,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":2.4649999999999999,"5min":10.945,"15min":10.945},"last":0.48799999999999999}]},{"osd":1,"last update":"Sat Sep 23 21:15:14 2023","interfaces":[{"interface":"back","average":{"1min":1.345,"5min":1.0429999999999999,"15min":2.5840000000000001},"min":{"1min":0.32700000000000001,"5min":0.221,"15min":0.221},"max":{"1min":4.0410000000000004,"5min":7.8609999999999998,"15min":37.960000000000001},"last":0.46400000000000002},{"interface":"front","average":{"1min":2.246,"5min":1.319,"15min":2.516},"min":{"1min":0.30299999999999999,"5min":0.28499999999999998,"15min":0.27000000000000002},"max":{"1min":13.983000000000001,"5min":13.983000000000001,"15min":37.933},"last":0.58199999999999996}]},{"osd":2,"last update":"Sat Sep 23 21:15:14 2023","interfaces":[{"interface":"back","average":{"1min":2.3559999999999999,"5min":1.3660000000000001,"15min":1.734},"min":{"1min":0.26400000000000001,"5min":0.247,"15min":0.247},"max":{"1min":14.026,"5min":14.026,"15min":16.928000000000001},"last":0.47599999999999998},{"interface":"front","average":{"1min":3.0419999999999998,"5min":1.6220000000000001,"15min":1.7949999999999999},"min":{"1min":0.33000000000000002,"5min":0.28599999999999998,"15min":0.27800000000000002},"max":{"1min":14.339,"5min":14.339,"15min":14.417999999999999},"last":0.629}]},{"osd":3,"last update":"Sat Sep 23 21:14:47 2023","interfaces":[{"interface":"back","average":{"1min":1.0389999999999999,"5min":1.1990000000000001,"15min":2.056},"min":{"1min":0.27900000000000003,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":2.766,"5min":9.3279999999999994,"15min":15.01},"last":0.56699999999999995},{"interface":"front","average":{"1min":1.899,"5min":1.3879999999999999,"15min":2},"min":{"1min":0.32200000000000001,"5min":0.32200000000000001,"15min":0.30199999999999999},"max":{"1min":14.305,"5min":14.305,"15min":15.004},"last":0.60899999999999999}]},{"osd":5,"last update":"Sat Sep 23 21:15:04 2023","interfaces":[{"interface":"back","average":{"1min":2.0310000000000001,"5min":1.337,"15min":1.0840000000000001},"min":{"1min":0.32000000000000001,"5min":0.222,"15min":0.222},"max":{"1min":14.398999999999999,"5min":14.398999999999999,"15min":14.398999999999999},"last":0.312},{"interface":"front","average":{"1min":1.675,"5min":1.2050000000000001,"15min":1.121},"min":{"1min":0.29699999999999999,"5min":0.27200000000000002,"15min":0.255},"max":{"1min":8.016,"5min":8.016,"15min":9.8979999999999997},"last":0.32700000000000001}]},{"osd":6,"last update":"Sat Sep 23 21:15:14 2023","interfaces":[{"interface":"back","average":{"1min":1.403,"5min":1.248,"15min":1.2749999999999999},"min":{"1min":0.34300000000000003,"5min":0.28599999999999998,"15min":0.26800000000000002},"max":{"1min":8.7420000000000009,"5min":24.452999999999999,"15min":24.452999999999999},"last":0.39900000000000002},{"interface":"front","average":{"1min":2.472,"5min":1.425,"15min":1.0680000000000001},"min":{"1min":0.40300000000000002,"5min":0.33500000000000002,"15min":0.32500000000000001},"max":{"1min":14.374000000000001,"5min":14.374000000000001,"15min":14.387},"last":0.41099999999999998}]},{"osd":7,"last update":"Sat Sep 23 21:14:47 2023","interfaces":[{"interface":"back","average":{"1min":1.9910000000000001,"5min":1.355,"15min":1.1299999999999999},"min":{"1min":0.38300000000000001,"5min":0.30099999999999999,"15min":0.29299999999999998},"max":{"1min":14.422000000000001,"5min":14.422000000000001,"15min":16.890999999999998},"last":0.35099999999999998},{"interface":"front","average":{"1min":1.536,"5min":1.234,"15min":1.2090000000000001},"min":{"1min":0.39200000000000002,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":7.7229999999999999,"5min":7.7229999999999999,"15min":11.170999999999999},"last":0.38700000000000001}]}]},{"osd":5,"up_from":39,"seq":167503724975,"num_pgs":55,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":609228,"kb_used_data":88068,"kb_used_omap":0,"kb_used_meta":521152,"kb_avail":93135924,"statfs":{"total":95995035648,"available":95371186176,"internally_reserved":0,"allocated":90181632,"data_stored":88621561,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":533659648},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:15:14 2023","interfaces":[{"interface":"back","average":{"1min":1.3720000000000001,"5min":0.84599999999999997,"15min":0.75800000000000001},"min":{"1min":0.23100000000000001,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":4.3680000000000003,"5min":4.3680000000000003,"15min":4.3680000000000003},"last":0.49199999999999999},{"interface":"front","average":{"1min":1.6839999999999999,"5min":0.872,"15min":0.73599999999999999},"min":{"1min":0.23899999999999999,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":6.577,"5min":6.577,"15min":6.577},"last":0.36699999999999999}]},{"osd":1,"last update":"Sat Sep 23 21:15:11 2023","interfaces":[{"interface":"back","average":{"1min":1.1140000000000001,"5min":0.94099999999999995,"15min":1.6830000000000001},"min":{"1min":0.29399999999999998,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":7.5839999999999996,"5min":7.5839999999999996,"15min":25.239999999999998},"last":0.42499999999999999},{"interface":"front","average":{"1min":1.123,"5min":1.472,"15min":1.7549999999999999},"min":{"1min":0.28299999999999997,"5min":0.28299999999999997,"15min":0.252},"max":{"1min":4.2199999999999998,"5min":11.116,"15min":25.253},"last":0.46400000000000002}]},{"osd":2,"last update":"Sat Sep 23 21:15:27 2023","interfaces":[{"interface":"back","average":{"1min":1.373,"5min":1.3320000000000001,"15min":1.6240000000000001},"min":{"1min":0.28399999999999997,"5min":0.27800000000000002,"15min":0.23499999999999999},"max":{"1min":3.8730000000000002,"5min":9.2889999999999997,"15min":12.634},"last":0.439},{"interface":"front","average":{"1min":1.472,"5min":1.1559999999999999,"15min":1.6319999999999999},"min":{"1min":0.29799999999999999,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":7.2720000000000002,"5min":9.7170000000000005,"15min":9.7650000000000006},"last":0.32500000000000001}]},{"osd":3,"last update":"Sat Sep 23 21:14:48 2023","interfaces":[{"interface":"back","average":{"1min":1.0109999999999999,"5min":1.6339999999999999,"15min":2.484},"min":{"1min":0.32400000000000001,"5min":0.311,"15min":0.29299999999999998},"max":{"1min":3.0390000000000001,"5min":14.242000000000001,"15min":61.667999999999999},"last":0.38100000000000001},{"interface":"front","average":{"1min":1.284,"5min":1.6559999999999999,"15min":2.5910000000000002},"min":{"1min":0.39100000000000001,"5min":0.34899999999999998,"15min":0.30599999999999999},"max":{"1min":6.508,"5min":12.44,"15min":62.276000000000003},"last":0.501}]},{"osd":4,"last update":"Sat Sep 23 21:15:00 2023","interfaces":[{"interface":"back","average":{"1min":1.0920000000000001,"5min":1.5289999999999999,"15min":1.157},"min":{"1min":0.253,"5min":0.253,"15min":0.23400000000000001},"max":{"1min":4.7750000000000004,"5min":12.776,"15min":12.776},"last":0.41299999999999998},{"interface":"front","average":{"1min":1.163,"5min":1.3029999999999999,"15min":1.1539999999999999},"min":{"1min":0.35999999999999999,"5min":0.27700000000000002,"15min":0.26900000000000002},"max":{"1min":4.8200000000000003,"5min":9.7119999999999997,"15min":11.973000000000001},"last":0.39800000000000002}]},{"osd":6,"last update":"Sat Sep 23 21:15:11 2023","interfaces":[{"interface":"back","average":{"1min":1.0880000000000001,"5min":1.0449999999999999,"15min":1.071},"min":{"1min":0.36499999999999999,"5min":0.313,"15min":0.25900000000000001},"max":{"1min":5.2249999999999996,"5min":7.7030000000000003,"15min":11.128},"last":0.52300000000000002},{"interface":"front","average":{"1min":1.022,"5min":1.5489999999999999,"15min":1.2450000000000001},"min":{"1min":0.34300000000000003,"5min":0.34300000000000003,"15min":0.317},"max":{"1min":4.3550000000000004,"5min":10.999000000000001,"15min":11.112},"last":0.53500000000000003}]},{"osd":7,"last update":"Sat Sep 23 21:14:48 2023","interfaces":[{"interface":"back","average":{"1min":0.84999999999999998,"5min":0.98499999999999999,"15min":1.8129999999999999},"min":{"1min":0.34499999999999997,"5min":0.31,"15min":0.309},"max":{"1min":3.3039999999999998,"5min":5.5659999999999998,"15min":189.405},"last":0.47399999999999998},{"interface":"front","average":{"1min":0.92000000000000004,"5min":1.472,"15min":1.9259999999999999},"min":{"1min":0.40899999999999997,"5min":0.35399999999999998,"15min":0.32500000000000001},"max":{"1min":3.3239999999999998,"5min":11.183999999999999,"15min":189.685},"last":0.51800000000000002}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8488,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1259,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5322,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":4838,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":179961856,"data_stored":179961856,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T21:15:39.880 INFO:tasks.cephadm.ceph_manager.ceph:recovered! 2023-09-23T21:15:39.880 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T21:15:39.880 INFO:tasks.thrashosds.thrasher:Growing pool .mgr 2023-09-23T21:15:39.880 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T21:15:39.931 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:15:39.966 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:15:40.010 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:15:40.015 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:15:40.016 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:15:40.116 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:15:40.196 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:15:40.272 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T21:15:40.279 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:15:40.280 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:15:40.301 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T21:15:40.366 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:15:40 smithi026 ceph-mon[175151]: 1.1b scrub starts 2023-09-23T21:15:40.366 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:15:40 smithi026 ceph-mon[175151]: 1.1b scrub ok 2023-09-23T21:15:40.366 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:15:40 smithi026 ceph-mon[175151]: 1.e scrub starts 2023-09-23T21:15:40.366 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:15:40 smithi026 ceph-mon[175151]: 1.e scrub ok 2023-09-23T21:15:40.367 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:15:40 smithi026 ceph-mon[175151]: pgmap v1383: 119 pgs: 119 active+clean; 583 MiB data, 5.7 GiB used, 531 GiB / 536 GiB avail; 65 MiB/s wr, 1.05k op/s 2023-09-23T21:15:40.367 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:15:40 smithi026 ceph-mon[175151]: 1.18 scrub starts 2023-09-23T21:15:40.369 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:15:40 smithi026 ceph-mon[179688]: 1.1b scrub starts 2023-09-23T21:15:40.369 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:15:40 smithi026 ceph-mon[179688]: 1.1b scrub ok 2023-09-23T21:15:40.369 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:15:40 smithi026 ceph-mon[179688]: 1.e scrub starts 2023-09-23T21:15:40.369 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:15:40 smithi026 ceph-mon[179688]: 1.e scrub ok 2023-09-23T21:15:40.369 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:15:40 smithi026 ceph-mon[179688]: pgmap v1383: 119 pgs: 119 active+clean; 583 MiB data, 5.7 GiB used, 531 GiB / 536 GiB avail; 65 MiB/s wr, 1.05k op/s 2023-09-23T21:15:40.369 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:15:40 smithi026 ceph-mon[179688]: 1.18 scrub starts 2023-09-23T21:15:40.381 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:15:40.446 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 8 16 19817 19801 154.62 117.062 0.00860411 0.00645596 2023-09-23T21:15:40.454 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:15:40.573 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:15:40.574 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:15:40 smithi070 ceph-mon[168282]: 1.1b scrub starts 2023-09-23T21:15:40.574 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:15:40 smithi070 ceph-mon[168282]: 1.1b scrub ok 2023-09-23T21:15:40.575 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:15:40 smithi070 ceph-mon[168282]: 1.e scrub starts 2023-09-23T21:15:40.575 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:15:40 smithi070 ceph-mon[168282]: 1.e scrub ok 2023-09-23T21:15:40.575 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:15:40 smithi070 ceph-mon[168282]: pgmap v1383: 119 pgs: 119 active+clean; 583 MiB data, 5.7 GiB used, 531 GiB / 536 GiB avail; 65 MiB/s wr, 1.05k op/s 2023-09-23T21:15:40.575 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:15:40 smithi070 ceph-mon[168282]: 1.18 scrub starts 2023-09-23T21:15:40.580 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:15:40.580 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:15:40.681 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:15:40.742 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:15:40.897 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:15:40.903 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:15:40.904 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:15:40.944 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:15:41.005 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:15:41.042 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:15:41.140 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:15:41.145 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:15:41.145 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:15:41.246 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:15:41.292 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:15:41.339 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:15:41.342 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:15:41.343 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:15:41.440 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:15:41 smithi070 ceph-mon[168282]: 1.18 scrub ok 2023-09-23T21:15:41.440 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:15:41 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/1469654044' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:15:41.440 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:15:41 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:15:41.444 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:15:41.445 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 9 16 21669 21653 150.302 115.75 0.0086567 0.00664866 2023-09-23T21:15:41.495 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:15:41.520 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:15:41 smithi026 ceph-mon[175151]: 1.18 scrub ok 2023-09-23T21:15:41.520 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:15:41 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/1469654044' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:15:41.521 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:15:41 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:15:41.525 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:15:41 smithi026 ceph-mon[179688]: 1.18 scrub ok 2023-09-23T21:15:41.525 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:15:41 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/1469654044' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:15:41.525 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:15:41 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:15:41.586 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:15:41.590 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:15:41.591 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:15:41.692 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:15:41.706 INFO:teuthology.orchestra.run.smithi070.stdout:{ 2023-09-23T21:15:41.706 INFO:teuthology.orchestra.run.smithi070.stdout: "size": 20, 2023-09-23T21:15:41.706 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 600, 2023-09-23T21:15:41.706 INFO:teuthology.orchestra.run.smithi070.stdout: "ops": [ 2023-09-23T21:15:41.707 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:41.707 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.46328.0:100001 10.10 10:0c7ac00a:::benchmark_data_smithi026_379838_object20044:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e495)", 2023-09-23T21:15:41.707 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:13:13.465471+0000", 2023-09-23T21:15:41.707 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 148.23848341799999, 2023-09-23T21:15:41.707 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.4773325069999999, 2023-09-23T21:15:41.707 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:15:41.707 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:15:41.707 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:15:41.707 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.46328", 2023-09-23T21:15:41.707 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/3748861181", 2023-09-23T21:15:41.707 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 100001 2023-09-23T21:15:41.707 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:41.708 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:15:41.708 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:41.708 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:15:41.708 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:13.465471+0000", 2023-09-23T21:15:41.708 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:15:41.708 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:41.708 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:41.708 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:15:41.708 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:13.465471+0000", 2023-09-23T21:15:41.708 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:15:41.708 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:41.708 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:41.708 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:15:41.709 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:13.465470+0000", 2023-09-23T21:15:41.709 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T21:15:41.709 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:41.709 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:41.709 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:15:41.709 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:13.465477+0000", 2023-09-23T21:15:41.709 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.8259999999999993e-06 2023-09-23T21:15:41.709 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:41.709 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:41.709 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:15:41.709 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:13.465496+0000", 2023-09-23T21:15:41.709 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.8465999999999999e-05 2023-09-23T21:15:41.710 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:41.710 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:41.710 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:15:41.710 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:13.465504+0000", 2023-09-23T21:15:41.710 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.1869999999999997e-06 2023-09-23T21:15:41.710 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:41.710 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:41.710 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:15:41.710 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:13.465570+0000", 2023-09-23T21:15:41.710 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.5831e-05 2023-09-23T21:15:41.710 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:41.710 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:41.711 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:15:41.711 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:13.465583+0000", 2023-09-23T21:15:41.711 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.2883e-05 2023-09-23T21:15:41.711 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:41.711 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:41.711 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:15:41.711 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:16.722777+0000", 2023-09-23T21:15:41.711 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.2571938070000002 2023-09-23T21:15:41.711 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:41.711 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:41.711 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for degraded object", 2023-09-23T21:15:41.711 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:16.723120+0000", 2023-09-23T21:15:41.711 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00034327099999999997 2023-09-23T21:15:41.712 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:41.712 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:41.712 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:15:41.712 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:16.875534+0000", 2023-09-23T21:15:41.712 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.152413828 2023-09-23T21:15:41.712 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:41.712 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:41.712 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:15:41.712 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:16.875609+0000", 2023-09-23T21:15:41.712 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.4875000000000004e-05 2023-09-23T21:15:41.712 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:41.712 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:41.712 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 4,7", 2023-09-23T21:15:41.713 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:16.875890+0000", 2023-09-23T21:15:41.713 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00028097300000000001 2023-09-23T21:15:41.713 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:41.713 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:41.713 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:15:41.713 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:16.881216+0000", 2023-09-23T21:15:41.713 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0053265589999999998 2023-09-23T21:15:41.713 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:41.713 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:41.713 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:15:41.713 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:16.893186+0000", 2023-09-23T21:15:41.713 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.011969762 2023-09-23T21:15:41.714 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:41.714 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:41.714 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:15:41.714 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:16.941662+0000", 2023-09-23T21:15:41.714 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.048476545000000003 2023-09-23T21:15:41.714 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:41.714 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:41.714 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:15:41.714 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:16.941724+0000", 2023-09-23T21:15:41.714 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.1719999999999999e-05 2023-09-23T21:15:41.715 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:41.715 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:41.715 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:15:41.715 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:16.942804+0000", 2023-09-23T21:15:41.715 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.001079483 2023-09-23T21:15:41.715 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:15:41.715 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:15:41.715 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:15:41.715 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:41.715 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:41.715 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.46328.0:100035 10.10 10:091f2b4d:::benchmark_data_smithi026_379838_object20078:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e496)", 2023-09-23T21:15:41.715 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:13:15.077410+0000", 2023-09-23T21:15:41.716 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 146.62654474199999, 2023-09-23T21:15:41.716 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.866327224, 2023-09-23T21:15:41.716 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:15:41.716 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:15:41.716 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:15:41.716 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.46328", 2023-09-23T21:15:41.716 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/3748861181", 2023-09-23T21:15:41.716 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 100035 2023-09-23T21:15:41.716 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:41.716 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:15:41.716 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:41.716 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:15:41.716 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:15.077410+0000", 2023-09-23T21:15:41.717 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:15:41.717 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:41.717 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:41.717 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:15:41.717 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:15.077410+0000", 2023-09-23T21:15:41.717 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:15:41.717 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:41.717 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:41.717 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:15:41.717 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:15.077408+0000", 2023-09-23T21:15:41.717 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T21:15:41.717 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:41.717 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:41.718 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:15:41.718 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:15.077416+0000", 2023-09-23T21:15:41.718 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.6799999999999993e-06 2023-09-23T21:15:41.718 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:41.718 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:41.718 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:15:41.718 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:15.077434+0000", 2023-09-23T21:15:41.718 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.7634000000000001e-05 2023-09-23T21:15:41.718 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:41.718 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:41.718 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:15:41.718 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:15.077441+0000", 2023-09-23T21:15:41.719 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.8329999999999994e-06 2023-09-23T21:15:41.719 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:41.719 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:41.719 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:15:41.719 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:15.077507+0000", 2023-09-23T21:15:41.719 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.5270000000000004e-05 2023-09-23T21:15:41.719 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:41.719 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:41.719 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:15:41.719 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:15.077520+0000", 2023-09-23T21:15:41.719 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.3301e-05 2023-09-23T21:15:41.719 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:41.720 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:41.720 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:15:41.720 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:16.723157+0000", 2023-09-23T21:15:41.720 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.6456375030000001 2023-09-23T21:15:41.720 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:41.720 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:41.720 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for degraded object", 2023-09-23T21:15:41.720 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:16.723547+0000", 2023-09-23T21:15:41.720 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000389126 2023-09-23T21:15:41.720 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:41.720 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:41.720 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:15:41.720 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:16.878482+0000", 2023-09-23T21:15:41.721 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.15493558900000001 2023-09-23T21:15:41.721 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:41.721 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:41.721 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:15:41.721 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:16.878543+0000", 2023-09-23T21:15:41.721 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.0544000000000002e-05 2023-09-23T21:15:41.721 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:41.721 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:41.721 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 4,7", 2023-09-23T21:15:41.721 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:16.878732+0000", 2023-09-23T21:15:41.721 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000189736 2023-09-23T21:15:41.721 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:41.722 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:41.722 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:15:41.722 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:16.883755+0000", 2023-09-23T21:15:41.722 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0050220610000000004 2023-09-23T21:15:41.722 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:41.722 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:41.722 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:15:41.722 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:16.895687+0000", 2023-09-23T21:15:41.722 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.011932720000000001 2023-09-23T21:15:41.722 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:41.722 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:41.722 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:15:41.723 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:16.942854+0000", 2023-09-23T21:15:41.723 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.047166300000000001 2023-09-23T21:15:41.723 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:41.723 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:41.723 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:15:41.723 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:16.942900+0000", 2023-09-23T21:15:41.723 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.6932999999999999e-05 2023-09-23T21:15:41.723 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:41.723 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:41.723 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:15:41.723 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:16.943737+0000", 2023-09-23T21:15:41.723 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00083654400000000005 2023-09-23T21:15:41.724 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:15:41.724 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:15:41.724 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:15:41.724 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:41.724 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:41.724 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.46328.0:100041 10.10 10:0d6c00f5:::benchmark_data_smithi026_379838_object20084:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e497)", 2023-09-23T21:15:41.724 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:13:15.390855+0000", 2023-09-23T21:15:41.724 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 146.31309976, 2023-09-23T21:15:41.724 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.5537436790000001, 2023-09-23T21:15:41.724 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:15:41.724 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:15:41.724 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:15:41.725 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.46328", 2023-09-23T21:15:41.725 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/3748861181", 2023-09-23T21:15:41.725 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 100041 2023-09-23T21:15:41.725 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:41.725 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:15:41.725 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:41.725 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:15:41.725 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:15.390855+0000", 2023-09-23T21:15:41.725 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:15:41.725 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:41.725 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:41.725 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:15:41.726 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:15.390855+0000", 2023-09-23T21:15:41.726 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:15:41.726 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:41.726 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:41.726 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:15:41.726 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:15.390853+0000", 2023-09-23T21:15:41.726 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T21:15:41.726 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:41.726 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:41.726 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:15:41.726 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:15.390861+0000", 2023-09-23T21:15:41.726 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.6559999999999992e-06 2023-09-23T21:15:41.727 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:41.727 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:41.727 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:15:41.727 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:15.390875+0000", 2023-09-23T21:15:41.727 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.4440000000000001e-05 2023-09-23T21:15:41.727 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:41.727 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:41.727 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:15:41.727 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:15.390882+0000", 2023-09-23T21:15:41.727 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.3249999999999996e-06 2023-09-23T21:15:41.727 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:41.727 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:41.728 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:15:41.728 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:15.390949+0000", 2023-09-23T21:15:41.728 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.7775999999999996e-05 2023-09-23T21:15:41.728 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:41.728 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:41.728 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:15:41.728 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:15.390963+0000", 2023-09-23T21:15:41.728 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.3139000000000001e-05 2023-09-23T21:15:41.728 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:41.728 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:41.728 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:15:41.728 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:16.723581+0000", 2023-09-23T21:15:41.729 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.332618455 2023-09-23T21:15:41.729 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:41.729 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:41.729 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for degraded object", 2023-09-23T21:15:41.729 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:16.723937+0000", 2023-09-23T21:15:41.729 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00035614099999999998 2023-09-23T21:15:41.729 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:41.729 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:41.729 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:15:41.729 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:16.881249+0000", 2023-09-23T21:15:41.729 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.15731209800000001 2023-09-23T21:15:41.729 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:41.729 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:41.730 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:15:41.730 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:16.881317+0000", 2023-09-23T21:15:41.730 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.7847000000000003e-05 2023-09-23T21:15:41.730 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:41.730 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:41.730 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 4,7", 2023-09-23T21:15:41.730 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:16.881502+0000", 2023-09-23T21:15:41.730 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00018458999999999999 2023-09-23T21:15:41.730 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:41.730 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:41.730 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:15:41.730 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:16.886249+0000", 2023-09-23T21:15:41.731 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0047471739999999998 2023-09-23T21:15:41.731 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:41.731 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:41.731 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:15:41.731 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:16.895740+0000", 2023-09-23T21:15:41.731 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0094913419999999998 2023-09-23T21:15:41.731 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:41.731 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:41.731 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:15:41.731 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:16.943800+0000", 2023-09-23T21:15:41.731 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.048059719000000001 2023-09-23T21:15:41.731 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:41.732 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:41.732 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:15:41.732 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:16.943851+0000", 2023-09-23T21:15:41.732 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.1156999999999998e-05 2023-09-23T21:15:41.732 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:41.732 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:41.732 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:15:41.732 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:16.944598+0000", 2023-09-23T21:15:41.732 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00074733099999999995 2023-09-23T21:15:41.732 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:15:41.732 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:15:41.732 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:15:41.733 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:41.733 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:41.733 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.46328.0:100102 10.10 10:0b658828:::benchmark_data_smithi026_379838_object20145:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e497)", 2023-09-23T21:15:41.733 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:13:15.673805+0000", 2023-09-23T21:15:41.733 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 146.030149373, 2023-09-23T21:15:41.733 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.2715812390000001, 2023-09-23T21:15:41.733 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:15:41.733 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:15:41.733 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:15:41.733 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.46328", 2023-09-23T21:15:41.733 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/3748861181", 2023-09-23T21:15:41.733 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 100102 2023-09-23T21:15:41.733 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:41.734 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:15:41.734 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:41.734 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:15:41.734 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:15.673805+0000", 2023-09-23T21:15:41.734 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:15:41.734 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:41.734 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:41.734 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:15:41.734 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:15.673805+0000", 2023-09-23T21:15:41.734 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:15:41.734 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:41.735 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:41.735 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:15:41.735 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:15.673804+0000", 2023-09-23T21:15:41.735 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T21:15:41.735 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:41.735 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:41.735 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:15:41.735 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:15.673810+0000", 2023-09-23T21:15:41.735 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.7379999999999997e-06 2023-09-23T21:15:41.735 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:41.735 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:41.735 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:15:41.736 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:15.673828+0000", 2023-09-23T21:15:41.736 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.7768999999999999e-05 2023-09-23T21:15:41.736 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:41.736 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:41.736 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:15:41.736 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:15.673836+0000", 2023-09-23T21:15:41.736 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.6399999999999997e-06 2023-09-23T21:15:41.736 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:41.736 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:41.736 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:15:41.736 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:15.673897+0000", 2023-09-23T21:15:41.736 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.1247000000000005e-05 2023-09-23T21:15:41.737 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:41.737 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:41.737 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:15:41.737 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:15.673909+0000", 2023-09-23T21:15:41.737 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.1698e-05 2023-09-23T21:15:41.737 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:41.737 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:41.737 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:15:41.737 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:16.723986+0000", 2023-09-23T21:15:41.737 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.0500774100000001 2023-09-23T21:15:41.737 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:41.737 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:41.738 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for degraded object", 2023-09-23T21:15:41.738 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:16.724288+0000", 2023-09-23T21:15:41.738 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000301472 2023-09-23T21:15:41.738 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:41.738 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:41.738 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:15:41.738 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:16.883809+0000", 2023-09-23T21:15:41.738 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.15952158999999999 2023-09-23T21:15:41.738 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:41.738 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:41.738 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:15:41.738 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:16.883893+0000", 2023-09-23T21:15:41.739 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.4110000000000006e-05 2023-09-23T21:15:41.739 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:41.739 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:41.739 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 4,7", 2023-09-23T21:15:41.739 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:16.884072+0000", 2023-09-23T21:15:41.739 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00017881400000000001 2023-09-23T21:15:41.739 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:41.739 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:41.739 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:15:41.739 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:16.890278+0000", 2023-09-23T21:15:41.739 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.006205398 2023-09-23T21:15:41.739 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:41.740 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:41.740 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:15:41.740 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:16.895778+0000", 2023-09-23T21:15:41.740 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0055007260000000001 2023-09-23T21:15:41.740 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:41.740 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:41.740 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:15:41.740 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:16.944655+0000", 2023-09-23T21:15:41.740 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.048876997999999998 2023-09-23T21:15:41.740 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:41.740 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:41.740 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:15:41.741 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:16.944704+0000", 2023-09-23T21:15:41.741 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.8625999999999998e-05 2023-09-23T21:15:41.741 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:41.741 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:41.741 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:15:41.741 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:16.945386+0000", 2023-09-23T21:15:41.741 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00068246099999999998 2023-09-23T21:15:41.741 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:15:41.741 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:15:41.741 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:15:41.741 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:41.741 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:41.742 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.46328.0:100156 10.10 10:0d90ee30:::benchmark_data_smithi026_379838_object20199:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e497)", 2023-09-23T21:15:41.742 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:13:15.866363+0000", 2023-09-23T21:15:41.742 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 145.837591984, 2023-09-23T21:15:41.742 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.0797975689999999, 2023-09-23T21:15:41.742 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:15:41.742 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:15:41.742 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:15:41.742 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.46328", 2023-09-23T21:15:41.742 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/3748861181", 2023-09-23T21:15:41.742 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 100156 2023-09-23T21:15:41.742 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:41.742 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:15:41.743 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:41.743 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:15:41.743 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:15.866363+0000", 2023-09-23T21:15:41.743 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:15:41.743 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:41.743 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:41.743 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:15:41.743 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:15.866363+0000", 2023-09-23T21:15:41.743 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:15:41.743 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:41.743 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:41.743 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:15:41.744 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:15.866361+0000", 2023-09-23T21:15:41.744 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T21:15:41.744 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:41.744 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:41.744 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:15:41.744 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:15.866368+0000", 2023-09-23T21:15:41.744 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.0759999999999997e-06 2023-09-23T21:15:41.744 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:41.744 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:41.744 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:15:41.744 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:15.866384+0000", 2023-09-23T21:15:41.744 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.6299000000000001e-05 2023-09-23T21:15:41.744 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:41.745 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:41.745 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:15:41.745 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:15.866391+0000", 2023-09-23T21:15:41.745 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.5579999999999997e-06 2023-09-23T21:15:41.745 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:41.745 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:41.745 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:15:41.745 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:15.866446+0000", 2023-09-23T21:15:41.745 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.5102999999999997e-05 2023-09-23T21:15:41.745 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:41.745 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:41.745 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:15:41.746 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:15.866456+0000", 2023-09-23T21:15:41.746 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.5179999999999993e-06 2023-09-23T21:15:41.746 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:41.746 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:41.746 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:15:41.746 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:16.724330+0000", 2023-09-23T21:15:41.746 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.85787401799999996 2023-09-23T21:15:41.746 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:41.746 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:41.746 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for degraded object", 2023-09-23T21:15:41.746 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:16.724694+0000", 2023-09-23T21:15:41.746 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00036395399999999999 2023-09-23T21:15:41.747 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:41.747 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:41.747 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:15:41.747 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:16.886296+0000", 2023-09-23T21:15:41.747 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.16160285299999999 2023-09-23T21:15:41.747 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:41.747 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:41.747 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:15:41.747 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:16.886397+0000", 2023-09-23T21:15:41.747 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000100832 2023-09-23T21:15:41.747 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:41.747 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:41.747 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 4,7", 2023-09-23T21:15:41.748 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:16.886638+0000", 2023-09-23T21:15:41.748 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000240444 2023-09-23T21:15:41.748 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:41.748 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:41.748 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:15:41.748 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:16.893114+0000", 2023-09-23T21:15:41.748 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0064762530000000004 2023-09-23T21:15:41.748 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:41.748 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:41.748 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:15:41.748 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:16.895818+0000", 2023-09-23T21:15:41.748 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0027042379999999999 2023-09-23T21:15:41.748 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:41.749 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:41.749 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:15:41.749 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:16.945430+0000", 2023-09-23T21:15:41.749 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.049611719999999998 2023-09-23T21:15:41.749 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:41.749 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:41.749 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:15:41.749 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:16.945478+0000", 2023-09-23T21:15:41.749 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.8433000000000002e-05 2023-09-23T21:15:41.749 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:41.749 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:41.749 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:15:41.750 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:16.946160+0000", 2023-09-23T21:15:41.750 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00068183899999999995 2023-09-23T21:15:41.750 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:15:41.750 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:15:41.750 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:15:41.751 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:41.751 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:41.751 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.46328.0:100162 10.10 10:0dbd73eb:::benchmark_data_smithi026_379838_object20205:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e497)", 2023-09-23T21:15:41.751 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:13:15.889020+0000", 2023-09-23T21:15:41.751 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 145.81493404700001, 2023-09-23T21:15:41.751 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.057916241, 2023-09-23T21:15:41.751 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:15:41.751 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:15:41.752 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:15:41.752 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.46328", 2023-09-23T21:15:41.752 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/3748861181", 2023-09-23T21:15:41.752 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 100162 2023-09-23T21:15:41.752 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:41.752 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:15:41.752 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:41.752 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:15:41.752 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:15.889020+0000", 2023-09-23T21:15:41.752 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:15:41.752 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:41.752 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:41.753 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:15:41.753 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:15.889020+0000", 2023-09-23T21:15:41.753 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:15:41.753 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:41.753 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:41.753 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:15:41.753 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:15.889019+0000", 2023-09-23T21:15:41.753 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.999999 2023-09-23T21:15:41.753 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:41.753 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:41.753 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:15:41.753 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:15.889025+0000", 2023-09-23T21:15:41.753 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.2950000000000001e-06 2023-09-23T21:15:41.754 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:41.754 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:41.754 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:15:41.754 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:15.889037+0000", 2023-09-23T21:15:41.754 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.2515e-05 2023-09-23T21:15:41.754 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:41.754 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:41.754 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:15:41.754 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:15.889042+0000", 2023-09-23T21:15:41.754 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.259e-06 2023-09-23T21:15:41.754 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:41.754 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:41.755 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:15:41.755 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:15.889091+0000", 2023-09-23T21:15:41.755 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.8362000000000002e-05 2023-09-23T21:15:41.755 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:41.755 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:41.755 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:15:41.755 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:15.889097+0000", 2023-09-23T21:15:41.755 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.5320000000000003e-06 2023-09-23T21:15:41.755 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:41.755 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:41.755 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:15:41.755 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:16.724728+0000", 2023-09-23T21:15:41.756 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.835630977 2023-09-23T21:15:41.756 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:41.756 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:41.756 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for degraded object", 2023-09-23T21:15:41.756 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:16.725083+0000", 2023-09-23T21:15:41.756 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00035435300000000001 2023-09-23T21:15:41.756 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:41.756 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:41.756 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:15:41.756 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:16.888992+0000", 2023-09-23T21:15:41.756 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.163909412 2023-09-23T21:15:41.756 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:41.756 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:41.757 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:15:41.757 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:16.889058+0000", 2023-09-23T21:15:41.757 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.6261999999999997e-05 2023-09-23T21:15:41.757 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:41.757 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:41.757 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 4,7", 2023-09-23T21:15:41.757 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:16.889271+0000", 2023-09-23T21:15:41.757 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00021296299999999999 2023-09-23T21:15:41.757 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:41.757 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:41.757 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:15:41.757 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:16.894373+0000", 2023-09-23T21:15:41.758 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0051016830000000001 2023-09-23T21:15:41.758 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:41.758 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:41.758 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:15:41.758 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:16.895865+0000", 2023-09-23T21:15:41.758 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0014915620000000001 2023-09-23T21:15:41.758 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:41.758 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:41.758 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:15:41.758 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:16.946201+0000", 2023-09-23T21:15:41.758 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.050336233000000001 2023-09-23T21:15:41.758 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:41.759 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:41.759 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:15:41.759 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:16.946250+0000", 2023-09-23T21:15:41.759 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.9400999999999997e-05 2023-09-23T21:15:41.759 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:41.759 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:41.759 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:15:41.759 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:16.946937+0000", 2023-09-23T21:15:41.759 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00068656700000000001 2023-09-23T21:15:41.759 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:15:41.759 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:15:41.759 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:15:41.760 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:41.760 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:41.760 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.46328.0:100182 10.10 10:0d89bf4d:::benchmark_data_smithi026_379838_object20225:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e497)", 2023-09-23T21:15:41.760 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:13:15.978413+0000", 2023-09-23T21:15:41.760 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 145.72554110199999, 2023-09-23T21:15:41.760 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.96927643900000005, 2023-09-23T21:15:41.760 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:15:41.760 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:15:41.760 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:15:41.760 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.46328", 2023-09-23T21:15:41.760 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/3748861181", 2023-09-23T21:15:41.760 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 100182 2023-09-23T21:15:41.761 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:41.761 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:15:41.761 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:41.761 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:15:41.761 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:15.978413+0000", 2023-09-23T21:15:41.761 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:15:41.761 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:41.761 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:41.761 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:15:41.761 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:15.978413+0000", 2023-09-23T21:15:41.761 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:15:41.761 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:41.762 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:41.762 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:15:41.762 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:15.978412+0000", 2023-09-23T21:15:41.762 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.999999 2023-09-23T21:15:41.762 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:41.762 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:41.762 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:15:41.762 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:15.978417+0000", 2023-09-23T21:15:41.762 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.0599999999999998e-06 2023-09-23T21:15:41.762 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:41.762 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:41.762 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:15:41.763 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:15.978429+0000", 2023-09-23T21:15:41.763 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.1386000000000001e-05 2023-09-23T21:15:41.763 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:41.763 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:41.763 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:15:41.763 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:15.978434+0000", 2023-09-23T21:15:41.763 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.2989999999999996e-06 2023-09-23T21:15:41.763 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:41.763 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:41.763 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:15:41.763 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:15.978481+0000", 2023-09-23T21:15:41.763 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.6622999999999997e-05 2023-09-23T21:15:41.764 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:41.764 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:41.764 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:15:41.764 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:15.978487+0000", 2023-09-23T21:15:41.764 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.5950000000000004e-06 2023-09-23T21:15:41.764 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:41.764 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:41.764 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:15:41.764 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:16.725115+0000", 2023-09-23T21:15:41.764 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.74662764199999998 2023-09-23T21:15:41.764 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:41.764 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:41.764 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for degraded object", 2023-09-23T21:15:41.765 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:16.725478+0000", 2023-09-23T21:15:41.765 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00036277599999999998 2023-09-23T21:15:41.765 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:41.765 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:41.765 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:15:41.765 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:16.891492+0000", 2023-09-23T21:15:41.765 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.166013891 2023-09-23T21:15:41.765 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:41.765 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:41.765 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:15:41.765 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:16.891595+0000", 2023-09-23T21:15:41.765 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000103737 2023-09-23T21:15:41.766 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:41.766 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:41.766 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 4,7", 2023-09-23T21:15:41.766 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:16.891852+0000", 2023-09-23T21:15:41.766 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00025655900000000001 2023-09-23T21:15:41.766 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:41.766 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:41.766 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:15:41.766 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:16.895701+0000", 2023-09-23T21:15:41.766 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0038490569999999999 2023-09-23T21:15:41.766 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:41.767 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:41.767 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:15:41.767 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:16.895891+0000", 2023-09-23T21:15:41.767 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00019018299999999999 2023-09-23T21:15:41.767 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:41.767 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:41.767 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:15:41.767 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:16.946983+0000", 2023-09-23T21:15:41.767 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.051091717000000002 2023-09-23T21:15:41.767 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:41.767 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:41.767 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:15:41.768 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:16.947035+0000", 2023-09-23T21:15:41.768 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.2031999999999999e-05 2023-09-23T21:15:41.768 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:41.768 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:41.768 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:15:41.768 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:16.947690+0000", 2023-09-23T21:15:41.768 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000655063 2023-09-23T21:15:41.768 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:15:41.768 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:15:41.768 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:15:41.768 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:41.768 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:41.768 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_repop(client.46328.0:157646 10.3 e534/513 10:c26ea38f:::benchmark_data_smithi026_379838_object77689:head v 534'6358, mlcod=451'3566)", 2023-09-23T21:15:41.769 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:15:12.342334+0000", 2023-09-23T21:15:41.769 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 29.361620822999999, 2023-09-23T21:15:41.769 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.013017730999999999, 2023-09-23T21:15:41.769 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:15:41.769 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:15:41.769 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:15:41.769 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:41.769 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:15:41.769 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:15:12.342334+0000", 2023-09-23T21:15:41.769 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:15:41.769 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:41.769 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:41.769 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:15:41.770 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:15:12.342334+0000", 2023-09-23T21:15:41.770 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:15:41.770 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:41.770 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:41.770 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:15:41.770 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:15:12.342333+0000", 2023-09-23T21:15:41.770 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.999999 2023-09-23T21:15:41.770 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:41.770 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:41.770 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:15:41.770 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:15:12.342342+0000", 2023-09-23T21:15:41.771 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.8550000000000008e-06 2023-09-23T21:15:41.771 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:41.771 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:41.771 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:15:41.771 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:15:12.342352+0000", 2023-09-23T21:15:41.771 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.0536999999999999e-05 2023-09-23T21:15:41.771 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:41.771 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:41.771 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:15:41.771 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:15:12.342361+0000", 2023-09-23T21:15:41.771 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.4360000000000002e-06 2023-09-23T21:15:41.771 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:41.771 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:41.772 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:15:41.772 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:15:12.342403+0000", 2023-09-23T21:15:41.772 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.2143e-05 2023-09-23T21:15:41.772 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:41.772 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:41.772 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:15:41.772 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:15:12.342430+0000", 2023-09-23T21:15:41.772 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.7121999999999999e-05 2023-09-23T21:15:41.772 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:41.772 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:41.772 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:15:41.772 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:15:12.355308+0000", 2023-09-23T21:15:41.773 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.012877911000000001 2023-09-23T21:15:41.773 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:41.773 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:41.773 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:15:41.773 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:15:12.355351+0000", 2023-09-23T21:15:41.773 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.3553999999999997e-05 2023-09-23T21:15:41.773 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:15:41.773 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:15:41.773 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:15:41.773 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:41.773 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:41.773 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_repop(client.46328.0:157659 10.16 e534/494 10:6d405d47:::benchmark_data_smithi026_379838_object77702:head v 534'6328, mlcod=464'3876)", 2023-09-23T21:15:41.773 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:15:12.343503+0000", 2023-09-23T21:15:41.774 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 29.360451733000001, 2023-09-23T21:15:41.774 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.013478775, 2023-09-23T21:15:41.774 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:15:41.774 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:15:41.774 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:15:41.774 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:41.774 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:15:41.774 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:15:12.343503+0000", 2023-09-23T21:15:41.774 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:15:41.774 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:41.774 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:41.774 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:15:41.775 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:15:12.343503+0000", 2023-09-23T21:15:41.775 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:15:41.775 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:41.775 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:41.775 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:15:41.775 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:15:12.343502+0000", 2023-09-23T21:15:41.775 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999995 2023-09-23T21:15:41.775 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:41.775 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:41.775 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:15:41.775 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:15:12.343509+0000", 2023-09-23T21:15:41.775 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.2899999999999997e-06 2023-09-23T21:15:41.776 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:41.776 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:41.776 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:15:41.776 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:15:12.343525+0000", 2023-09-23T21:15:41.776 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.5109000000000001e-05 2023-09-23T21:15:41.776 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:41.776 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:41.776 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:15:41.776 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:15:12.343531+0000", 2023-09-23T21:15:41.776 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.0140000000000004e-06 2023-09-23T21:15:41.776 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:41.776 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:41.777 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:15:41.777 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:15:12.343614+0000", 2023-09-23T21:15:41.777 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.3245999999999995e-05 2023-09-23T21:15:41.777 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:41.777 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:41.777 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:15:41.777 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:15:12.343637+0000", 2023-09-23T21:15:41.777 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.3051999999999999e-05 2023-09-23T21:15:41.777 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:41.777 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:41.777 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:15:41.777 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:15:12.356956+0000", 2023-09-23T21:15:41.777 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.013319078 2023-09-23T21:15:41.778 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:41.778 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:41.778 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:15:41.778 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:15:12.356982+0000", 2023-09-23T21:15:41.778 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.5687e-05 2023-09-23T21:15:41.778 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:15:41.778 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:15:41.778 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:15:41.778 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:41.778 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:41.778 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.46823.0:1317 11.5 11:aed35a5b:::benchmark_data_smithi026_424533_object1316:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e548)", 2023-09-23T21:15:41.778 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:15:32.801249+0000", 2023-09-23T21:15:41.779 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 8.9027052369999993, 2023-09-23T21:15:41.779 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.056119338999999997, 2023-09-23T21:15:41.779 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:15:41.779 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:15:41.779 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:15:41.779 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.46823", 2023-09-23T21:15:41.779 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/2490075711", 2023-09-23T21:15:41.779 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 1317 2023-09-23T21:15:41.779 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:41.779 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:15:41.779 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:41.779 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:15:41.779 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:15:32.801249+0000", 2023-09-23T21:15:41.780 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:15:41.780 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:41.780 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:41.780 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:15:41.780 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:15:32.801249+0000", 2023-09-23T21:15:41.780 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:15:41.780 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:41.780 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:41.780 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:15:41.780 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:15:32.801248+0000", 2023-09-23T21:15:41.780 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999981 2023-09-23T21:15:41.781 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:41.781 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:41.781 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:15:41.781 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:15:32.801305+0000", 2023-09-23T21:15:41.781 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.7207999999999998e-05 2023-09-23T21:15:41.781 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:41.781 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:41.781 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:15:41.781 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:15:32.801315+0000", 2023-09-23T21:15:41.781 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.0200000000000001e-05 2023-09-23T21:15:41.781 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:41.781 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:41.781 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:15:41.782 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:15:32.801319+0000", 2023-09-23T21:15:41.782 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.4299999999999999e-06 2023-09-23T21:15:41.782 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:41.782 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:41.782 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:15:41.782 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:15:32.801351+0000", 2023-09-23T21:15:41.782 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.1832000000000003e-05 2023-09-23T21:15:41.782 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:41.782 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:41.782 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:15:41.782 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:15:32.801428+0000", 2023-09-23T21:15:41.782 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.6815000000000002e-05 2023-09-23T21:15:41.783 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:41.783 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:41.783 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 3,7", 2023-09-23T21:15:41.783 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:15:32.801521+0000", 2023-09-23T21:15:41.783 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.3070000000000002e-05 2023-09-23T21:15:41.783 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:41.783 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:41.783 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:15:41.783 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:15:32.801962+0000", 2023-09-23T21:15:41.783 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000441225 2023-09-23T21:15:41.783 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:41.783 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:41.783 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:15:41.784 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:15:32.857215+0000", 2023-09-23T21:15:41.784 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.055252392999999997 2023-09-23T21:15:41.784 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:41.784 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:41.784 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:15:41.784 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:15:32.857310+0000", 2023-09-23T21:15:41.784 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.4765000000000005e-05 2023-09-23T21:15:41.784 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:41.784 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:41.784 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:15:41.784 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:15:32.857342+0000", 2023-09-23T21:15:41.784 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.2119999999999997e-05 2023-09-23T21:15:41.785 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:41.785 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:41.785 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:15:41.785 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:15:32.857369+0000", 2023-09-23T21:15:41.785 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.6981000000000001e-05 2023-09-23T21:15:41.785 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:15:41.785 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:15:41.785 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:15:41.785 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:41.785 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:41.785 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_repop(client.46823.0:1318 11.a e548/544 11:58813509:::benchmark_data_smithi026_424533_object1317:head v 548'80, mlcod=548'78)", 2023-09-23T21:15:41.786 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:15:32.802043+0000", 2023-09-23T21:15:41.786 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 8.9019119440000001, 2023-09-23T21:15:41.786 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.055060499999999998, 2023-09-23T21:15:41.786 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:15:41.786 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:15:41.786 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:15:41.786 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:41.786 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:15:41.786 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:15:32.802043+0000", 2023-09-23T21:15:41.786 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:15:41.786 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:41.786 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:41.787 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:15:41.787 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:15:32.802043+0000", 2023-09-23T21:15:41.787 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:15:41.787 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:41.787 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:41.787 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:15:41.787 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:15:32.802042+0000", 2023-09-23T21:15:41.787 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.999999 2023-09-23T21:15:41.787 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:41.787 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:41.787 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:15:41.787 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:15:32.802114+0000", 2023-09-23T21:15:41.788 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.1962e-05 2023-09-23T21:15:41.788 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:41.788 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:41.788 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:15:41.788 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:15:32.802123+0000", 2023-09-23T21:15:41.788 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.679e-06 2023-09-23T21:15:41.788 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:41.788 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:41.788 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:15:41.788 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:15:32.802129+0000", 2023-09-23T21:15:41.788 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.5799999999999999e-06 2023-09-23T21:15:41.788 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:41.788 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:41.789 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:15:41.789 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:15:32.802164+0000", 2023-09-23T21:15:41.789 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.4996999999999997e-05 2023-09-23T21:15:41.789 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:41.789 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:41.789 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:15:41.789 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:15:32.802183+0000", 2023-09-23T21:15:41.789 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.8851e-05 2023-09-23T21:15:41.789 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:41.789 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:41.789 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:15:41.789 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:15:32.802577+0000", 2023-09-23T21:15:41.789 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00039452999999999998 2023-09-23T21:15:41.790 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:41.790 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:41.790 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:15:41.790 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:15:32.857103+0000", 2023-09-23T21:15:41.790 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.054525637000000002 2023-09-23T21:15:41.790 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:15:41.790 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:15:41.790 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:15:41.790 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:41.790 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:41.790 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_repop(client.46823.0:1315 11.3 e548/544 11:c16f1130:::benchmark_data_smithi026_424533_object1314:head v 548'86, mlcod=548'84)", 2023-09-23T21:15:41.790 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:15:32.802789+0000", 2023-09-23T21:15:41.791 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 8.9011660100000007, 2023-09-23T21:15:41.791 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.054896684000000001, 2023-09-23T21:15:41.791 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:15:41.791 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:15:41.791 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:15:41.791 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:41.791 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:15:41.791 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:15:32.802789+0000", 2023-09-23T21:15:41.791 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:15:41.791 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:41.791 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:41.791 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:15:41.792 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:15:32.802789+0000", 2023-09-23T21:15:41.792 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:15:41.792 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:41.792 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:41.792 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:15:41.792 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:15:32.802788+0000", 2023-09-23T21:15:41.792 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.999999 2023-09-23T21:15:41.792 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:41.792 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:41.792 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:15:41.792 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:15:32.802813+0000", 2023-09-23T21:15:41.792 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.5128000000000001e-05 2023-09-23T21:15:41.792 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:41.793 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:41.793 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:15:41.793 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:15:32.857082+0000", 2023-09-23T21:15:41.793 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.054269046000000001 2023-09-23T21:15:41.793 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:41.793 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:41.793 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:15:41.793 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:15:32.857094+0000", 2023-09-23T21:15:41.793 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.1778e-05 2023-09-23T21:15:41.793 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:41.793 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:41.793 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:15:41.794 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:15:32.857145+0000", 2023-09-23T21:15:41.794 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.1396000000000001e-05 2023-09-23T21:15:41.794 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:41.794 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:41.794 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:15:41.794 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:15:32.857167+0000", 2023-09-23T21:15:41.794 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.1851000000000001e-05 2023-09-23T21:15:41.794 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:41.794 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:41.794 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:15:41.794 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:15:32.857662+0000", 2023-09-23T21:15:41.795 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00049472700000000004 2023-09-23T21:15:41.795 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:41.795 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:41.795 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:15:41.795 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:15:32.857685+0000", 2023-09-23T21:15:41.795 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.3544e-05 2023-09-23T21:15:41.795 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:15:41.795 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:15:41.795 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:15:41.795 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:41.795 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:41.795 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_repop(client.46823.0:1333 11.8 e548/544 11:1d2f860d:::benchmark_data_smithi026_424533_object1332:head v 548'85, mlcod=548'83)", 2023-09-23T21:15:41.795 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:15:32.806707+0000", 2023-09-23T21:15:41.796 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 8.8972474350000006, 2023-09-23T21:15:41.796 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.051647506000000003, 2023-09-23T21:15:41.796 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:15:41.796 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:15:41.796 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:15:41.796 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:41.796 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:15:41.796 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:15:32.806707+0000", 2023-09-23T21:15:41.796 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:15:41.796 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:41.796 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:41.796 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:15:41.797 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:15:32.806707+0000", 2023-09-23T21:15:41.797 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:15:41.797 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:41.797 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:41.797 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:15:41.797 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:15:32.806706+0000", 2023-09-23T21:15:41.797 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.999999 2023-09-23T21:15:41.797 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:41.797 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:41.797 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:15:41.797 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:15:32.806765+0000", 2023-09-23T21:15:41.797 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.8828e-05 2023-09-23T21:15:41.798 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:41.798 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:41.798 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:15:41.798 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:15:32.857086+0000", 2023-09-23T21:15:41.798 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.050320640999999999 2023-09-23T21:15:41.798 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:41.798 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:41.798 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:15:41.798 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:15:32.857094+0000", 2023-09-23T21:15:41.798 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.1729999999999995e-06 2023-09-23T21:15:41.798 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:41.798 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:41.799 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:15:41.799 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:15:32.857151+0000", 2023-09-23T21:15:41.799 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.6668e-05 2023-09-23T21:15:41.799 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:41.799 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:41.799 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:15:41.799 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:15:32.857167+0000", 2023-09-23T21:15:41.799 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.6739999999999999e-05 2023-09-23T21:15:41.799 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:41.799 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:41.799 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:15:41.799 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:15:32.858313+0000", 2023-09-23T21:15:41.800 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0011459980000000001 2023-09-23T21:15:41.800 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:41.800 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:41.800 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:15:41.800 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:15:32.858355+0000", 2023-09-23T21:15:41.800 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.1216e-05 2023-09-23T21:15:41.800 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:15:41.800 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:15:41.800 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:15:41.800 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:41.800 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:41.801 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.46823.0:2672 11.5 11:a165482c:::benchmark_data_smithi026_424533_object2671:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e548)", 2023-09-23T21:15:41.801 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:15:33.243737+0000", 2023-09-23T21:15:41.801 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 8.4602179839999998, 2023-09-23T21:15:41.801 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.14439406599999999, 2023-09-23T21:15:41.801 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:15:41.801 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:15:41.801 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:15:41.801 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.46823", 2023-09-23T21:15:41.801 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/2490075711", 2023-09-23T21:15:41.801 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 2672 2023-09-23T21:15:41.801 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:41.801 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:15:41.802 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:41.802 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:15:41.802 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:15:33.243737+0000", 2023-09-23T21:15:41.802 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:15:41.802 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:41.802 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:41.802 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:15:41.802 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:15:33.243737+0000", 2023-09-23T21:15:41.802 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:15:41.802 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:41.802 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:41.802 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:15:41.803 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:15:33.243735+0000", 2023-09-23T21:15:41.803 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.999999 2023-09-23T21:15:41.803 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:41.803 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:41.803 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:15:41.803 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:15:33.243790+0000", 2023-09-23T21:15:41.803 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.4113000000000002e-05 2023-09-23T21:15:41.803 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:41.803 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:41.803 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:15:41.803 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:15:33.243802+0000", 2023-09-23T21:15:41.803 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.2925000000000001e-05 2023-09-23T21:15:41.804 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:41.804 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:41.804 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:15:41.804 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:15:33.243808+0000", 2023-09-23T21:15:41.804 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.0270000000000001e-06 2023-09-23T21:15:41.804 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:41.804 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:41.804 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:15:41.804 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:15:33.243853+0000", 2023-09-23T21:15:41.804 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.4233000000000002e-05 2023-09-23T21:15:41.804 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:41.804 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:41.805 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:15:41.805 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:15:33.243983+0000", 2023-09-23T21:15:41.805 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000130772 2023-09-23T21:15:41.805 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:41.805 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:41.805 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 3,7", 2023-09-23T21:15:41.805 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:15:33.244130+0000", 2023-09-23T21:15:41.805 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00014616799999999999 2023-09-23T21:15:41.805 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:41.805 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:41.805 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:15:41.805 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:15:33.244567+0000", 2023-09-23T21:15:41.806 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00043769499999999999 2023-09-23T21:15:41.806 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:41.806 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:41.806 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:15:41.806 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:15:33.245190+0000", 2023-09-23T21:15:41.806 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00062226700000000002 2023-09-23T21:15:41.806 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:41.806 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:41.806 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:15:41.806 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:15:33.388036+0000", 2023-09-23T21:15:41.806 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.142846205 2023-09-23T21:15:41.806 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:41.807 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:41.807 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:15:41.807 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:15:33.388091+0000", 2023-09-23T21:15:41.807 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.5288999999999999e-05 2023-09-23T21:15:41.807 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:41.807 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:41.807 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:15:41.807 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:15:33.388131+0000", 2023-09-23T21:15:41.807 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.9498e-05 2023-09-23T21:15:41.807 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:15:41.807 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:15:41.807 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:15:41.808 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:41.808 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:41.808 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.46823.0:6621 11.5 11:a62d6623:::benchmark_data_smithi026_424533_object6620:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e548)", 2023-09-23T21:15:41.808 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:15:34.844187+0000", 2023-09-23T21:15:41.808 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 6.8597675589999998, 2023-09-23T21:15:41.808 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.015667704000000001, 2023-09-23T21:15:41.808 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:15:41.808 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:15:41.808 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:15:41.808 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.46823", 2023-09-23T21:15:41.808 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/2490075711", 2023-09-23T21:15:41.808 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 6621 2023-09-23T21:15:41.808 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:41.809 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:15:41.809 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:41.809 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:15:41.809 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:15:34.844187+0000", 2023-09-23T21:15:41.809 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:15:41.809 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:41.809 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:41.809 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:15:41.809 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:15:34.844187+0000", 2023-09-23T21:15:41.809 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:15:41.809 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:41.809 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:41.810 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:15:41.810 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:15:34.844186+0000", 2023-09-23T21:15:41.810 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.999999 2023-09-23T21:15:41.810 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:41.810 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:41.810 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:15:41.810 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:15:34.844300+0000", 2023-09-23T21:15:41.810 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000114641 2023-09-23T21:15:41.810 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:41.810 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:41.810 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:15:41.810 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:15:34.844316+0000", 2023-09-23T21:15:41.810 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.5242000000000001e-05 2023-09-23T21:15:41.811 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:41.811 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:41.811 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:15:41.811 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:15:34.844323+0000", 2023-09-23T21:15:41.811 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.8040000000000004e-06 2023-09-23T21:15:41.811 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:41.811 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:41.811 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:15:41.811 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:15:34.844368+0000", 2023-09-23T21:15:41.811 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.5887999999999998e-05 2023-09-23T21:15:41.811 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:41.811 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:41.812 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:15:41.812 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:15:34.844488+0000", 2023-09-23T21:15:41.812 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00011924199999999999 2023-09-23T21:15:41.812 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:41.812 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:41.812 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 3,7", 2023-09-23T21:15:41.812 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:15:34.844634+0000", 2023-09-23T21:15:41.812 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00014638499999999999 2023-09-23T21:15:41.812 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:41.812 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:41.812 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:15:41.812 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:15:34.845278+0000", 2023-09-23T21:15:41.813 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00064349199999999996 2023-09-23T21:15:41.813 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:41.813 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:41.813 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:15:41.813 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:15:34.845485+0000", 2023-09-23T21:15:41.813 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000206961 2023-09-23T21:15:41.813 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:41.813 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:41.813 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:15:41.813 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:15:34.859753+0000", 2023-09-23T21:15:41.813 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.014268416000000001 2023-09-23T21:15:41.813 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:41.813 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:41.814 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:15:41.814 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:15:34.859809+0000", 2023-09-23T21:15:41.814 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.6229e-05 2023-09-23T21:15:41.814 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:41.814 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:41.814 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:15:41.814 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:15:34.859855+0000", 2023-09-23T21:15:41.814 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.5529999999999999e-05 2023-09-23T21:15:41.814 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:15:41.814 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:15:41.814 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:15:41.814 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:41.815 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:41.815 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.46823.0:10897 11.5 11:ae65c900:::benchmark_data_smithi026_424533_object10896:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e548)", 2023-09-23T21:15:41.815 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:15:36.575706+0000", 2023-09-23T21:15:41.815 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 5.1282488979999998, 2023-09-23T21:15:41.815 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.029557157000000001, 2023-09-23T21:15:41.815 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:15:41.815 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:15:41.815 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:15:41.815 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.46823", 2023-09-23T21:15:41.815 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/2490075711", 2023-09-23T21:15:41.815 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 10897 2023-09-23T21:15:41.815 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:41.816 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:15:41.816 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:41.816 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:15:41.816 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:15:36.575706+0000", 2023-09-23T21:15:41.816 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:15:41.816 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:41.816 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:41.816 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:15:41.816 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:15:36.575706+0000", 2023-09-23T21:15:41.816 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:15:41.816 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:41.816 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:41.817 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:15:41.817 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:15:36.575704+0000", 2023-09-23T21:15:41.817 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T21:15:41.817 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:41.817 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:41.817 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:15:41.817 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:15:36.575771+0000", 2023-09-23T21:15:41.817 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.6204000000000005e-05 2023-09-23T21:15:41.817 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:41.817 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:41.817 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:15:41.817 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:15:36.575782+0000", 2023-09-23T21:15:41.817 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.173e-05 2023-09-23T21:15:41.818 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:41.818 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:41.818 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:15:41.818 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:15:36.575787+0000", 2023-09-23T21:15:41.818 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.4700000000000004e-06 2023-09-23T21:15:41.818 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:41.818 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:41.818 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:15:41.818 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:15:36.575828+0000", 2023-09-23T21:15:41.818 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.1610000000000003e-05 2023-09-23T21:15:41.818 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:41.818 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:41.819 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:15:41.819 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:15:36.575948+0000", 2023-09-23T21:15:41.819 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000119256 2023-09-23T21:15:41.819 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:41.819 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:41.819 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 3,7", 2023-09-23T21:15:41.819 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:15:36.576074+0000", 2023-09-23T21:15:41.819 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000126351 2023-09-23T21:15:41.819 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:41.819 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:41.819 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:15:41.819 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:15:36.577259+0000", 2023-09-23T21:15:41.820 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0011844729999999999 2023-09-23T21:15:41.820 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:41.820 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:41.820 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:15:41.820 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:15:36.577290+0000", 2023-09-23T21:15:41.820 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.1599e-05 2023-09-23T21:15:41.820 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:41.820 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:41.820 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:15:41.820 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:15:36.605178+0000", 2023-09-23T21:15:41.820 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.027887968999999999 2023-09-23T21:15:41.820 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:41.821 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:41.821 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:15:41.821 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:15:36.605228+0000", 2023-09-23T21:15:41.821 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.0157000000000001e-05 2023-09-23T21:15:41.821 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:41.821 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:41.821 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:15:41.821 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:15:36.605263+0000", 2023-09-23T21:15:41.821 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.4564000000000003e-05 2023-09-23T21:15:41.821 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:15:41.821 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:15:41.821 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:15:41.821 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:41.822 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:41.822 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.46823.0:10899 11.5 11:a9a9a2f5:::benchmark_data_smithi026_424533_object10898:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e548)", 2023-09-23T21:15:41.822 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:15:36.575797+0000", 2023-09-23T21:15:41.822 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 5.1281576109999998, 2023-09-23T21:15:41.822 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.029567940000000001, 2023-09-23T21:15:41.822 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:15:41.822 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:15:41.822 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:15:41.822 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.46823", 2023-09-23T21:15:41.822 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/2490075711", 2023-09-23T21:15:41.822 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 10899 2023-09-23T21:15:41.823 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:41.823 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:15:41.823 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:41.823 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:15:41.823 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:15:36.575797+0000", 2023-09-23T21:15:41.823 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:15:41.823 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:41.823 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:41.823 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:15:41.823 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:15:36.575797+0000", 2023-09-23T21:15:41.823 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:15:41.823 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:41.824 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:41.824 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:15:41.824 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:15:36.575797+0000", 2023-09-23T21:15:41.824 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999995 2023-09-23T21:15:41.824 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:41.824 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:41.824 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:15:41.824 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:15:36.575898+0000", 2023-09-23T21:15:41.824 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000101722 2023-09-23T21:15:41.824 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:41.824 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:41.824 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:15:41.825 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:15:36.575915+0000", 2023-09-23T21:15:41.825 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.6506000000000001e-05 2023-09-23T21:15:41.825 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:41.825 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:41.825 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:15:41.825 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:15:36.575923+0000", 2023-09-23T21:15:41.825 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.2020000000000004e-06 2023-09-23T21:15:41.825 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:41.825 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:41.825 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:15:41.825 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:15:36.576652+0000", 2023-09-23T21:15:41.825 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00072916599999999995 2023-09-23T21:15:41.826 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:41.826 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:41.826 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:15:41.826 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:15:36.576766+0000", 2023-09-23T21:15:41.826 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000113838 2023-09-23T21:15:41.826 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:41.826 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:41.826 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 3,7", 2023-09-23T21:15:41.826 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:15:36.576906+0000", 2023-09-23T21:15:41.826 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00013951 2023-09-23T21:15:41.826 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:41.826 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:41.827 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:15:41.827 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:15:36.577367+0000", 2023-09-23T21:15:41.827 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00046176700000000003 2023-09-23T21:15:41.827 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:41.827 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:41.827 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:15:41.827 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:15:36.577572+0000", 2023-09-23T21:15:41.827 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00020514200000000001 2023-09-23T21:15:41.827 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:41.827 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:41.827 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:15:41.827 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:15:36.605290+0000", 2023-09-23T21:15:41.828 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.027717469000000002 2023-09-23T21:15:41.828 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:41.828 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:41.828 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:15:41.828 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:15:36.605331+0000", 2023-09-23T21:15:41.828 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.0908000000000002e-05 2023-09-23T21:15:41.828 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:41.828 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:41.828 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:15:41.828 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:15:36.605365+0000", 2023-09-23T21:15:41.828 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.4068e-05 2023-09-23T21:15:41.828 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:15:41.828 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:15:41.829 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:15:41.829 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:41.829 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:41.829 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.46823.0:19457 11.5 11:a769eafd:::benchmark_data_smithi026_424533_object19456:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e548)", 2023-09-23T21:15:41.829 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:15:40.207936+0000", 2023-09-23T21:15:41.829 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 1.496018667, 2023-09-23T21:15:41.829 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.01600294, 2023-09-23T21:15:41.829 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:15:41.829 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:15:41.829 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:15:41.829 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.46823", 2023-09-23T21:15:41.829 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/2490075711", 2023-09-23T21:15:41.830 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 19457 2023-09-23T21:15:41.830 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:41.830 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:15:41.830 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:41.830 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:15:41.830 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:15:40.207936+0000", 2023-09-23T21:15:41.830 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:15:41.830 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:41.830 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:41.830 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:15:41.830 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:15:40.207936+0000", 2023-09-23T21:15:41.830 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:15:41.831 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:41.831 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:41.831 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:15:41.831 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:15:40.207934+0000", 2023-09-23T21:15:41.831 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T21:15:41.831 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:41.831 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:41.831 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:15:41.831 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:15:40.208038+0000", 2023-09-23T21:15:41.831 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000104202 2023-09-23T21:15:41.831 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:41.831 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:41.831 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:15:41.832 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:15:40.208083+0000", 2023-09-23T21:15:41.832 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.4747999999999999e-05 2023-09-23T21:15:41.832 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:41.832 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:41.832 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:15:41.832 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:15:40.208101+0000", 2023-09-23T21:15:41.832 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.7875000000000001e-05 2023-09-23T21:15:41.832 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:41.832 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:41.832 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:15:41.832 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:15:40.208148+0000", 2023-09-23T21:15:41.832 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.7063e-05 2023-09-23T21:15:41.833 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:41.833 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:41.833 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:15:41.833 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:15:40.208292+0000", 2023-09-23T21:15:41.833 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00014394200000000001 2023-09-23T21:15:41.833 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:41.833 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:41.833 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 3,7", 2023-09-23T21:15:41.833 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:15:40.208415+0000", 2023-09-23T21:15:41.833 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000122676 2023-09-23T21:15:41.833 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:41.833 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:41.833 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:15:41.834 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:15:40.209746+0000", 2023-09-23T21:15:41.834 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0013311239999999999 2023-09-23T21:15:41.834 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:41.834 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:41.834 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:15:41.834 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:15:40.210052+0000", 2023-09-23T21:15:41.834 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00030642000000000001 2023-09-23T21:15:41.834 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:41.834 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:41.834 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:15:41.834 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:15:40.223852+0000", 2023-09-23T21:15:41.834 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.013799485 2023-09-23T21:15:41.835 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:41.835 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:41.835 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:15:41.835 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:15:40.223900+0000", 2023-09-23T21:15:41.835 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.8025000000000002e-05 2023-09-23T21:15:41.835 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:41.835 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:41.835 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:15:41.835 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:15:40.223939+0000", 2023-09-23T21:15:41.835 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.9038999999999997e-05 2023-09-23T21:15:41.835 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:15:41.835 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:15:41.835 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:15:41.836 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:41.836 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:41.836 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.46823.0:19864 11.5 11:ad92b6bb:::benchmark_data_smithi026_424533_object19863:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e548)", 2023-09-23T21:15:41.836 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:15:40.476300+0000", 2023-09-23T21:15:41.836 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 1.2276548110000001, 2023-09-23T21:15:41.836 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.01751952, 2023-09-23T21:15:41.836 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:15:41.836 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:15:41.836 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:15:41.836 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.46823", 2023-09-23T21:15:41.836 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/2490075711", 2023-09-23T21:15:41.836 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 19864 2023-09-23T21:15:41.837 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:41.837 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:15:41.837 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:41.837 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:15:41.837 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:15:40.476300+0000", 2023-09-23T21:15:41.837 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:15:41.837 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:41.837 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:41.837 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:15:41.837 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:15:40.476300+0000", 2023-09-23T21:15:41.837 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:15:41.837 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:41.838 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:41.838 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:15:41.838 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:15:40.476299+0000", 2023-09-23T21:15:41.838 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999995 2023-09-23T21:15:41.838 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:41.838 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:41.838 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:15:41.838 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:15:40.476484+0000", 2023-09-23T21:15:41.838 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00018440000000000001 2023-09-23T21:15:41.838 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:41.838 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:41.838 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:15:41.839 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:15:40.476493+0000", 2023-09-23T21:15:41.839 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.5109999999999992e-06 2023-09-23T21:15:41.839 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:41.839 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:41.839 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:15:41.839 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:15:40.476499+0000", 2023-09-23T21:15:41.839 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.4739999999999996e-06 2023-09-23T21:15:41.839 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:41.839 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:41.839 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:15:41.839 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:15:40.480782+0000", 2023-09-23T21:15:41.839 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0042831129999999999 2023-09-23T21:15:41.840 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:41.840 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:41.840 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:15:41.840 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:15:40.480876+0000", 2023-09-23T21:15:41.840 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.4537999999999995e-05 2023-09-23T21:15:41.840 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:41.840 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:41.840 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 3,7", 2023-09-23T21:15:41.840 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:15:40.480965+0000", 2023-09-23T21:15:41.840 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.8263000000000005e-05 2023-09-23T21:15:41.840 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:41.840 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:41.840 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:15:41.841 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:15:40.482440+0000", 2023-09-23T21:15:41.841 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.001475552 2023-09-23T21:15:41.841 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:41.841 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:41.841 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:15:41.841 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:15:40.482805+0000", 2023-09-23T21:15:41.841 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00036468299999999998 2023-09-23T21:15:41.841 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:41.841 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:41.841 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:15:41.841 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:15:40.493756+0000", 2023-09-23T21:15:41.841 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.010950797 2023-09-23T21:15:41.842 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:41.842 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:41.842 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:15:41.842 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:15:40.493793+0000", 2023-09-23T21:15:41.842 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.7642000000000003e-05 2023-09-23T21:15:41.842 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:41.842 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:41.842 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:15:41.842 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:15:40.493819+0000", 2023-09-23T21:15:41.842 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.5848000000000001e-05 2023-09-23T21:15:41.842 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:15:41.842 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:15:41.843 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:15:41.843 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:41.843 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:41.843 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.46823.0:19865 11.5 11:aca53b65:::benchmark_data_smithi026_424533_object19864:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e548)", 2023-09-23T21:15:41.843 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:15:40.476505+0000", 2023-09-23T21:15:41.843 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 1.2274491190000001, 2023-09-23T21:15:41.843 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.017407484000000001, 2023-09-23T21:15:41.843 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:15:41.843 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:15:41.843 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:15:41.843 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.46823", 2023-09-23T21:15:41.843 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/2490075711", 2023-09-23T21:15:41.844 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 19865 2023-09-23T21:15:41.844 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:41.844 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:15:41.844 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:41.844 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:15:41.844 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:15:40.476505+0000", 2023-09-23T21:15:41.844 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:15:41.844 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:41.844 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:41.844 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:15:41.844 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:15:40.476505+0000", 2023-09-23T21:15:41.844 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:15:41.844 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:41.845 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:41.845 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:15:41.845 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:15:40.476505+0000", 2023-09-23T21:15:41.845 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999995 2023-09-23T21:15:41.845 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:41.845 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:41.845 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:15:41.845 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:15:40.476616+0000", 2023-09-23T21:15:41.845 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000111302 2023-09-23T21:15:41.845 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:41.845 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:41.845 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:15:41.846 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:15:40.476625+0000", 2023-09-23T21:15:41.846 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.2860000000000005e-06 2023-09-23T21:15:41.846 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:41.846 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:41.846 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:15:41.846 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:15:40.476631+0000", 2023-09-23T21:15:41.846 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.5570000000000003e-06 2023-09-23T21:15:41.846 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:41.846 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:41.846 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:15:41.846 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:15:40.481531+0000", 2023-09-23T21:15:41.846 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.004900448 2023-09-23T21:15:41.847 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:41.847 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:41.847 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:15:41.847 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:15:40.481660+0000", 2023-09-23T21:15:41.847 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00012887599999999999 2023-09-23T21:15:41.847 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:41.847 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:41.847 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 3,7", 2023-09-23T21:15:41.847 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:15:40.481803+0000", 2023-09-23T21:15:41.847 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00014279700000000001 2023-09-23T21:15:41.847 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:41.847 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:41.848 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:15:41.848 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:15:40.482533+0000", 2023-09-23T21:15:41.848 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00073016400000000005 2023-09-23T21:15:41.848 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:41.848 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:41.848 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:15:41.848 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:15:40.482960+0000", 2023-09-23T21:15:41.848 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00042701799999999998 2023-09-23T21:15:41.848 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:41.848 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:41.848 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:15:41.848 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:15:40.493853+0000", 2023-09-23T21:15:41.848 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.010892249 2023-09-23T21:15:41.849 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:41.849 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:41.849 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:15:41.849 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:15:40.493888+0000", 2023-09-23T21:15:41.849 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.5818999999999997e-05 2023-09-23T21:15:41.849 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:15:41.849 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:15:41.849 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:15:41.849 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:15:40.493913+0000", 2023-09-23T21:15:41.849 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.4530000000000001e-05 2023-09-23T21:15:41.849 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:15:41.849 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:15:41.850 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:15:41.850 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:15:41.850 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:15:41.850 INFO:teuthology.orchestra.run.smithi070.stdout:} 2023-09-23T21:15:41.850 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:15:41.851 INFO:teuthology.orchestra.run.smithi026.stderr:nodeep-scrub is unset 2023-09-23T21:15:41.894 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0: {} 2023-09-23T21:15:41.943 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1: {} 2023-09-23T21:15:41.983 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2: {} 2023-09-23T21:15:42.021 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3: {} 2023-09-23T21:15:42.059 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4: {} 2023-09-23T21:15:42.097 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_ops_in_flight 2023-09-23T21:15:42.099 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T21:15:42.138 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6: {} 2023-09-23T21:15:42.149 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:15:42.156 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:15:42.157 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:15:42.173 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T21:15:42.183 INFO:teuthology.orchestra.run.smithi026.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T21:15:42.183 INFO:teuthology.orchestra.run.smithi026.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T21:15:42.183 INFO:teuthology.orchestra.run.smithi026.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T21:15:42.183 INFO:teuthology.orchestra.run.smithi026.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T21:15:42.183 INFO:teuthology.orchestra.run.smithi026.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T21:15:42.183 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T21:15:42.183 INFO:teuthology.orchestra.run.smithi026.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T21:15:42.183 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T21:15:42.258 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:15:42.318 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:15:42.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:15:42 smithi070 ceph-mon[168282]: 1.5 scrub starts 2023-09-23T21:15:42.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:15:42 smithi070 ceph-mon[168282]: pgmap v1384: 119 pgs: 119 active+clean; 870 MiB data, 6.5 GiB used, 530 GiB / 536 GiB avail; 87 MiB/s wr, 1.39k op/s 2023-09-23T21:15:42.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:15:42 smithi070 ceph-mon[168282]: 1.5 scrub ok 2023-09-23T21:15:42.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:15:42 smithi070 ceph-mon[168282]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T21:15:42.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:15:42 smithi070 ceph-mon[168282]: Cluster is now healthy 2023-09-23T21:15:42.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:15:42 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T21:15:42.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:15:42 smithi070 ceph-mon[168282]: osdmap e549: 8 total, 8 up, 6 in 2023-09-23T21:15:42.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:15:42 smithi070 ceph-mon[168282]: failed to encode map e549 with expected crc 2023-09-23T21:15:42.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:15:42 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/3201700044' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:15:42.394 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T21:15:42.399 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:15:42.399 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:15:42.446 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 10 16 23430 23414 146.275 110.062 0.00653719 0.0068279 2023-09-23T21:15:42.500 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:15:42.557 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:15:42.599 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:15:42.604 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:15:42.604 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:15:42.633 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:15:42 smithi026 ceph-mon[175151]: 1.5 scrub starts 2023-09-23T21:15:42.634 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:15:42 smithi026 ceph-mon[175151]: pgmap v1384: 119 pgs: 119 active+clean; 870 MiB data, 6.5 GiB used, 530 GiB / 536 GiB avail; 87 MiB/s wr, 1.39k op/s 2023-09-23T21:15:42.634 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:15:42 smithi026 ceph-mon[175151]: 1.5 scrub ok 2023-09-23T21:15:42.634 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:15:42 smithi026 ceph-mon[175151]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T21:15:42.634 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:15:42 smithi026 ceph-mon[175151]: Cluster is now healthy 2023-09-23T21:15:42.634 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:15:42 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T21:15:42.634 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:15:42 smithi026 ceph-mon[175151]: osdmap e549: 8 total, 8 up, 6 in 2023-09-23T21:15:42.634 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:15:42 smithi026 ceph-mon[175151]: failed to encode map e549 with expected crc 2023-09-23T21:15:42.634 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:15:42 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/3201700044' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:15:42.635 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:15:42 smithi026 ceph-mon[179688]: 1.5 scrub starts 2023-09-23T21:15:42.635 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:15:42 smithi026 ceph-mon[179688]: pgmap v1384: 119 pgs: 119 active+clean; 870 MiB data, 6.5 GiB used, 530 GiB / 536 GiB avail; 87 MiB/s wr, 1.39k op/s 2023-09-23T21:15:42.635 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:15:42 smithi026 ceph-mon[179688]: 1.5 scrub ok 2023-09-23T21:15:42.635 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:15:42 smithi026 ceph-mon[179688]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T21:15:42.635 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:15:42 smithi026 ceph-mon[179688]: Cluster is now healthy 2023-09-23T21:15:42.635 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:15:42 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T21:15:42.636 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:15:42 smithi026 ceph-mon[179688]: osdmap e549: 8 total, 8 up, 6 in 2023-09-23T21:15:42.636 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:15:42 smithi026 ceph-mon[179688]: failed to encode map e549 with expected crc 2023-09-23T21:15:42.636 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:15:42 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/3201700044' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:15:42.705 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:15:42.745 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:15:42.789 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:15:42.793 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:15:42.793 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:15:42.807 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T21:15:42.894 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:15:42.927 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:15:42.986 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:15:42.993 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:15:42.993 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:15:43.094 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:15:43.139 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:15:43.189 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:15:43.193 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:15:43.193 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:15:43.294 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:15:43.329 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:15:43.330 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:15:43 smithi026 ceph-mon[175151]: failed to encode map e549 with expected crc 2023-09-23T21:15:43.447 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 11 16 25356 25340 143.917 120.375 0.0113655 0.00694201 2023-09-23T21:15:43.586 INFO:teuthology.orchestra.run.smithi026.stdout: 2023-09-23T21:15:43.602 INFO:teuthology.orchestra.run.smithi026.stderr:dumped all 2023-09-23T21:15:43.614 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:15:43.617 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:15:43.618 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:15:43.618 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:15:43 smithi070 ceph-mon[168282]: failed to encode map e549 with expected crc 2023-09-23T21:15:43.618 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:15:43 smithi070 ceph-mon[168282]: failed to encode map e549 with expected crc 2023-09-23T21:15:43.631 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:15:43 smithi026 ceph-mon[179688]: failed to encode map e549 with expected crc 2023-09-23T21:15:43.631 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:15:43 smithi026 ceph-mon[179688]: failed to encode map e549 with expected crc 2023-09-23T21:15:43.631 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:15:43 smithi026 ceph-mon[175151]: failed to encode map e549 with expected crc 2023-09-23T21:15:43.719 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:15:43.763 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:15:43.865 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:15:43.869 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:15:43.870 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:15:43.971 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:15:44.035 INFO:teuthology.orchestra.run.smithi026.stdout:{"pg_ready":true,"pg_map":{"version":1386,"stamp":"2023-09-23T21:15:43.107560+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":1357474118,"num_objects":20727,"num_object_clones":0,"num_object_copies":62181,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":20727,"num_whiteouts":0,"num_read":45,"num_read_kb":83,"num_write":20784,"num_write_kb":1325113,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":62,"num_bytes_recovered":333747,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":25747,"ondisk_log_size":25747,"up":357,"acting":357,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":378,"num_osds":6,"num_per_pool_osds":6,"num_per_pool_omap_osds":6,"kb":562470912,"kb_used":8109160,"kb_used_data":4595192,"kb_used_omap":125,"kb_used_meta":3513794,"kb_avail":554361752,"statfs":{"total":575970213888,"available":567666434048,"internally_reserved":0,"allocated":4705476608,"data_stored":4693142063,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":128662,"internal_metadata":3598125418},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[10,5,8,4],"upper_bound":16},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":1356784324,"num_objects":20702,"num_object_clones":0,"num_object_copies":62106,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":20702,"num_whiteouts":0,"num_read":-33,"num_read_kb":-61,"num_write":20685,"num_write_kb":1324699,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":-2,"num_bytes_recovered":-874965,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.008197"},"pg_stats":[{"pgid":"1.18","version":"79'161","reported_seq":525,"reported_epoch":548,"state":"active+clean","last_fresh":"2023-09-23T21:15:39.474980+0000","last_change":"2023-09-23T21:15:39.474890+0000","last_active":"2023-09-23T21:15:39.474980+0000","last_peered":"2023-09-23T21:15:39.474980+0000","last_clean":"2023-09-23T21:15:39.474980+0000","last_became_active":"2023-09-23T21:15:23.801118+0000","last_became_peered":"2023-09-23T21:15:23.801118+0000","last_unstale":"2023-09-23T21:15:39.474980+0000","last_undegraded":"2023-09-23T21:15:39.474980+0000","last_fullsized":"2023-09-23T21:15:39.474980+0000","mapping_epoch":542,"log_start":"0'0","ondisk_log_start":"0'0","created":479,"last_epoch_clean":543,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:15:39.474823+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:15:39.474823+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:17:03.560219+0000","scrub_duration":0.074664395999999994,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1a","version":"68'1","reported_seq":139,"reported_epoch":548,"state":"active+clean","last_fresh":"2023-09-23T21:15:31.213546+0000","last_change":"2023-09-23T21:13:13.291403+0000","last_active":"2023-09-23T21:15:31.213546+0000","last_peered":"2023-09-23T21:15:31.213546+0000","last_clean":"2023-09-23T21:15:31.213546+0000","last_became_active":"2023-09-23T21:13:13.291123+0000","last_became_peered":"2023-09-23T21:13:13.291123+0000","last_unstale":"2023-09-23T21:15:31.213546+0000","last_undegraded":"2023-09-23T21:15:31.213546+0000","last_fullsized":"2023-09-23T21:15:31.213546+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":495,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:45:15.427430+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,0],"acting":[4,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":208,"reported_epoch":548,"state":"active+clean","last_fresh":"2023-09-23T21:15:31.214125+0000","last_change":"2023-09-23T21:12:10.767963+0000","last_active":"2023-09-23T21:15:31.214125+0000","last_peered":"2023-09-23T21:15:31.214125+0000","last_clean":"2023-09-23T21:15:31.214125+0000","last_became_active":"2023-09-23T21:12:10.767126+0000","last_became_peered":"2023-09-23T21:12:10.767126+0000","last_unstale":"2023-09-23T21:15:31.214125+0000","last_undegraded":"2023-09-23T21:15:31.214125+0000","last_fullsized":"2023-09-23T21:15:31.214125+0000","mapping_epoch":470,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:45:47.138277+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,5],"acting":[4,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1b","version":"79'161","reported_seq":736,"reported_epoch":548,"state":"active+clean","last_fresh":"2023-09-23T21:15:38.423441+0000","last_change":"2023-09-23T21:15:38.423372+0000","last_active":"2023-09-23T21:15:38.423441+0000","last_peered":"2023-09-23T21:15:38.423441+0000","last_clean":"2023-09-23T21:15:38.423441+0000","last_became_active":"2023-09-23T21:15:23.802580+0000","last_became_peered":"2023-09-23T21:15:23.802580+0000","last_unstale":"2023-09-23T21:15:38.423441+0000","last_undegraded":"2023-09-23T21:15:38.423441+0000","last_fullsized":"2023-09-23T21:15:38.423441+0000","mapping_epoch":542,"log_start":"0'0","ondisk_log_start":"0'0","created":479,"last_epoch_clean":543,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:15:38.423271+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:15:38.423271+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:04:46.575186+0000","scrub_duration":0.00258465,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":544,"reported_epoch":547,"state":"active+clean","last_fresh":"2023-09-23T21:15:30.965605+0000","last_change":"2023-09-23T21:12:10.824116+0000","last_active":"2023-09-23T21:15:30.965605+0000","last_peered":"2023-09-23T21:15:30.965605+0000","last_clean":"2023-09-23T21:15:30.965605+0000","last_became_active":"2023-09-23T21:12:10.823849+0000","last_became_peered":"2023-09-23T21:12:10.823849+0000","last_unstale":"2023-09-23T21:15:30.965605+0000","last_undegraded":"2023-09-23T21:15:30.965605+0000","last_fullsized":"2023-09-23T21:15:30.965605+0000","mapping_epoch":470,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:54:13.777603+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":545,"reported_epoch":547,"state":"active+clean","last_fresh":"2023-09-23T21:15:30.994354+0000","last_change":"2023-09-23T21:13:44.413713+0000","last_active":"2023-09-23T21:15:30.994354+0000","last_peered":"2023-09-23T21:15:30.994354+0000","last_clean":"2023-09-23T21:15:30.994354+0000","last_became_active":"2023-09-23T21:13:44.413366+0000","last_became_peered":"2023-09-23T21:13:44.413366+0000","last_unstale":"2023-09-23T21:15:30.994354+0000","last_undegraded":"2023-09-23T21:15:30.994354+0000","last_fullsized":"2023-09-23T21:15:30.994354+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:31:56.349530+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1a","version":"79'161","reported_seq":596,"reported_epoch":548,"state":"active+clean","last_fresh":"2023-09-23T21:15:31.213323+0000","last_change":"2023-09-23T21:15:23.800891+0000","last_active":"2023-09-23T21:15:31.213323+0000","last_peered":"2023-09-23T21:15:31.213323+0000","last_clean":"2023-09-23T21:15:31.213323+0000","last_became_active":"2023-09-23T21:15:23.800519+0000","last_became_peered":"2023-09-23T21:15:23.800519+0000","last_unstale":"2023-09-23T21:15:31.213323+0000","last_undegraded":"2023-09-23T21:15:31.213323+0000","last_fullsized":"2023-09-23T21:15:31.213323+0000","mapping_epoch":542,"log_start":"0'0","ondisk_log_start":"0'0","created":479,"last_epoch_clean":543,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:06:51.075448+0000","last_deep_scrub":"79'161","last_deep_scrub_stamp":"2023-09-23T21:05:32.630292+0000","last_clean_scrub_stamp":"2023-09-23T21:06:51.075448+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0054793890000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,5],"acting":[4,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":57,"reported_epoch":547,"state":"active+clean","last_fresh":"2023-09-23T21:15:30.995643+0000","last_change":"2023-09-23T21:13:30.806084+0000","last_active":"2023-09-23T21:15:30.995643+0000","last_peered":"2023-09-23T21:15:30.995643+0000","last_clean":"2023-09-23T21:15:30.995643+0000","last_became_active":"2023-09-23T21:13:30.805743+0000","last_became_peered":"2023-09-23T21:13:30.805743+0000","last_unstale":"2023-09-23T21:15:30.995643+0000","last_undegraded":"2023-09-23T21:15:30.995643+0000","last_fullsized":"2023-09-23T21:15:30.995643+0000","mapping_epoch":501,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:48:10.849281+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":422,"reported_epoch":547,"state":"active+clean","last_fresh":"2023-09-23T21:15:30.994766+0000","last_change":"2023-09-23T21:13:44.410921+0000","last_active":"2023-09-23T21:15:30.994766+0000","last_peered":"2023-09-23T21:15:30.994766+0000","last_clean":"2023-09-23T21:15:30.994766+0000","last_became_active":"2023-09-23T21:13:44.410633+0000","last_became_peered":"2023-09-23T21:13:44.410633+0000","last_unstale":"2023-09-23T21:15:30.994766+0000","last_undegraded":"2023-09-23T21:15:30.994766+0000","last_fullsized":"2023-09-23T21:15:30.994766+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:57:10.116786+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.15","version":"79'161","reported_seq":525,"reported_epoch":548,"state":"active+clean","last_fresh":"2023-09-23T21:15:36.742405+0000","last_change":"2023-09-23T21:15:36.742339+0000","last_active":"2023-09-23T21:15:36.742405+0000","last_peered":"2023-09-23T21:15:36.742405+0000","last_clean":"2023-09-23T21:15:36.742405+0000","last_became_active":"2023-09-23T21:15:23.799409+0000","last_became_peered":"2023-09-23T21:15:23.799409+0000","last_unstale":"2023-09-23T21:15:36.742405+0000","last_undegraded":"2023-09-23T21:15:36.742405+0000","last_fullsized":"2023-09-23T21:15:36.742405+0000","mapping_epoch":542,"log_start":"0'0","ondisk_log_start":"0'0","created":479,"last_epoch_clean":543,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:15:36.742247+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:15:36.742247+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:03:02.689639+0000","scrub_duration":0.004253466,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,3],"acting":[7,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":523,"reported_epoch":548,"state":"active+clean","last_fresh":"2023-09-23T21:15:31.868280+0000","last_change":"2023-09-23T21:13:12.235728+0000","last_active":"2023-09-23T21:15:31.868280+0000","last_peered":"2023-09-23T21:15:31.868280+0000","last_clean":"2023-09-23T21:15:31.868280+0000","last_became_active":"2023-09-23T21:13:12.216549+0000","last_became_peered":"2023-09-23T21:13:12.216549+0000","last_unstale":"2023-09-23T21:15:31.868280+0000","last_undegraded":"2023-09-23T21:15:31.868280+0000","last_fullsized":"2023-09-23T21:15:31.868280+0000","mapping_epoch":493,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:39:33.324239+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":586,"reported_epoch":547,"state":"active+clean","last_fresh":"2023-09-23T21:15:30.965565+0000","last_change":"2023-09-23T21:12:10.790349+0000","last_active":"2023-09-23T21:15:30.965565+0000","last_peered":"2023-09-23T21:15:30.965565+0000","last_clean":"2023-09-23T21:15:30.965565+0000","last_became_active":"2023-09-23T21:12:10.789977+0000","last_became_peered":"2023-09-23T21:12:10.789977+0000","last_unstale":"2023-09-23T21:15:30.965565+0000","last_undegraded":"2023-09-23T21:15:30.965565+0000","last_fullsized":"2023-09-23T21:15:30.965565+0000","mapping_epoch":470,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:49:40.191147+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,5],"acting":[3,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"79'161","reported_seq":909,"reported_epoch":547,"state":"active+clean","last_fresh":"2023-09-23T21:15:30.995293+0000","last_change":"2023-09-23T21:11:32.760292+0000","last_active":"2023-09-23T21:15:30.995293+0000","last_peered":"2023-09-23T21:15:30.995293+0000","last_clean":"2023-09-23T21:15:30.995293+0000","last_became_active":"2023-09-23T21:11:32.759909+0000","last_became_peered":"2023-09-23T21:11:32.759909+0000","last_unstale":"2023-09-23T21:15:30.995293+0000","last_undegraded":"2023-09-23T21:15:30.995293+0000","last_fullsized":"2023-09-23T21:15:30.995293+0000","mapping_epoch":457,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":458,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:25.096401+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:25.096401+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:13:09.225685+0000","scrub_duration":0.002859393,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":273,"reported_epoch":548,"state":"active+clean","last_fresh":"2023-09-23T21:15:31.869519+0000","last_change":"2023-09-23T21:12:10.779468+0000","last_active":"2023-09-23T21:15:31.869519+0000","last_peered":"2023-09-23T21:15:31.869519+0000","last_clean":"2023-09-23T21:15:31.869519+0000","last_became_active":"2023-09-23T21:12:10.779282+0000","last_became_peered":"2023-09-23T21:12:10.779282+0000","last_unstale":"2023-09-23T21:15:31.869519+0000","last_undegraded":"2023-09-23T21:15:31.869519+0000","last_fullsized":"2023-09-23T21:15:31.869519+0000","mapping_epoch":470,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:36:35.417986+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":523,"reported_epoch":547,"state":"active+clean","last_fresh":"2023-09-23T21:15:30.965819+0000","last_change":"2023-09-23T21:10:26.758958+0000","last_active":"2023-09-23T21:15:30.965819+0000","last_peered":"2023-09-23T21:15:30.965819+0000","last_clean":"2023-09-23T21:15:30.965819+0000","last_became_active":"2023-09-23T21:10:26.758539+0000","last_became_peered":"2023-09-23T21:10:26.758539+0000","last_unstale":"2023-09-23T21:15:30.965819+0000","last_undegraded":"2023-09-23T21:15:30.965819+0000","last_fullsized":"2023-09-23T21:15:30.965819+0000","mapping_epoch":423,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":424,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:44:42.319189+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.17","version":"79'161","reported_seq":648,"reported_epoch":547,"state":"active+clean","last_fresh":"2023-09-23T21:15:30.994501+0000","last_change":"2023-09-23T21:15:23.799556+0000","last_active":"2023-09-23T21:15:30.994501+0000","last_peered":"2023-09-23T21:15:30.994501+0000","last_clean":"2023-09-23T21:15:30.994501+0000","last_became_active":"2023-09-23T21:15:23.799272+0000","last_became_peered":"2023-09-23T21:15:23.799272+0000","last_unstale":"2023-09-23T21:15:30.994501+0000","last_undegraded":"2023-09-23T21:15:30.994501+0000","last_fullsized":"2023-09-23T21:15:30.994501+0000","mapping_epoch":542,"log_start":"0'0","ondisk_log_start":"0'0","created":479,"last_epoch_clean":543,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:25.916867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:25.916867+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.001665323,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":432,"reported_epoch":547,"state":"active+clean","last_fresh":"2023-09-23T21:15:30.966825+0000","last_change":"2023-09-23T21:11:08.766045+0000","last_active":"2023-09-23T21:15:30.966825+0000","last_peered":"2023-09-23T21:15:30.966825+0000","last_clean":"2023-09-23T21:15:30.966825+0000","last_became_active":"2023-09-23T21:11:08.758504+0000","last_became_peered":"2023-09-23T21:11:08.758504+0000","last_unstale":"2023-09-23T21:15:30.966825+0000","last_undegraded":"2023-09-23T21:15:30.966825+0000","last_fullsized":"2023-09-23T21:15:30.966825+0000","mapping_epoch":443,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":444,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:02:19.482574+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":360,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,4],"acting":[3,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":235,"reported_epoch":547,"state":"active+clean","last_fresh":"2023-09-23T21:15:30.994253+0000","last_change":"2023-09-23T21:13:30.821489+0000","last_active":"2023-09-23T21:15:30.994253+0000","last_peered":"2023-09-23T21:15:30.994253+0000","last_clean":"2023-09-23T21:15:30.994253+0000","last_became_active":"2023-09-23T21:13:30.821125+0000","last_became_peered":"2023-09-23T21:13:30.821125+0000","last_unstale":"2023-09-23T21:15:30.994253+0000","last_undegraded":"2023-09-23T21:15:30.994253+0000","last_fullsized":"2023-09-23T21:15:30.994253+0000","mapping_epoch":501,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:47:18.553417+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,5],"acting":[2,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.16","version":"79'161","reported_seq":466,"reported_epoch":547,"state":"active+clean","last_fresh":"2023-09-23T21:15:30.994280+0000","last_change":"2023-09-23T21:13:33.770962+0000","last_active":"2023-09-23T21:15:30.994280+0000","last_peered":"2023-09-23T21:15:30.994280+0000","last_clean":"2023-09-23T21:15:30.994280+0000","last_became_active":"2023-09-23T21:13:33.770407+0000","last_became_peered":"2023-09-23T21:13:33.770407+0000","last_unstale":"2023-09-23T21:15:30.994280+0000","last_undegraded":"2023-09-23T21:15:30.994280+0000","last_fullsized":"2023-09-23T21:15:30.994280+0000","mapping_epoch":504,"log_start":"0'0","ondisk_log_start":"0'0","created":479,"last_epoch_clean":505,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:10:24.268662+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:10:24.268662+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13418,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":11,"num_read_kb":21,"num_write":6,"num_write_kb":55,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":40252,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.14","version":"79'22","reported_seq":112,"reported_epoch":548,"state":"active+clean","last_fresh":"2023-09-23T21:15:31.213899+0000","last_change":"2023-09-23T21:11:27.770162+0000","last_active":"2023-09-23T21:15:31.213899+0000","last_peered":"2023-09-23T21:15:31.213899+0000","last_clean":"2023-09-23T21:15:31.213899+0000","last_became_active":"2023-09-23T21:11:27.752275+0000","last_became_peered":"2023-09-23T21:11:27.752275+0000","last_unstale":"2023-09-23T21:15:31.213899+0000","last_undegraded":"2023-09-23T21:15:31.213899+0000","last_fullsized":"2023-09-23T21:15:31.213899+0000","mapping_epoch":452,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":453,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:05:11.769428+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4734,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,2],"acting":[4,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":477,"reported_epoch":548,"state":"active+clean","last_fresh":"2023-09-23T21:15:31.213709+0000","last_change":"2023-09-23T21:12:10.839271+0000","last_active":"2023-09-23T21:15:31.213709+0000","last_peered":"2023-09-23T21:15:31.213709+0000","last_clean":"2023-09-23T21:15:31.213709+0000","last_became_active":"2023-09-23T21:12:10.839042+0000","last_became_peered":"2023-09-23T21:12:10.839042+0000","last_unstale":"2023-09-23T21:15:31.213709+0000","last_undegraded":"2023-09-23T21:15:31.213709+0000","last_fullsized":"2023-09-23T21:15:31.213709+0000","mapping_epoch":470,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:10:26.414737+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,2],"acting":[4,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.11","version":"79'161","reported_seq":471,"reported_epoch":547,"state":"active+clean","last_fresh":"2023-09-23T21:15:30.995365+0000","last_change":"2023-09-23T21:13:33.767406+0000","last_active":"2023-09-23T21:15:30.995365+0000","last_peered":"2023-09-23T21:15:30.995365+0000","last_clean":"2023-09-23T21:15:30.995365+0000","last_became_active":"2023-09-23T21:13:33.767067+0000","last_became_peered":"2023-09-23T21:13:33.767067+0000","last_unstale":"2023-09-23T21:15:30.995365+0000","last_undegraded":"2023-09-23T21:15:30.995365+0000","last_fullsized":"2023-09-23T21:15:30.995365+0000","mapping_epoch":504,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":505,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:28.756634+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:28.756634+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":23,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":157,"reported_epoch":548,"state":"active+clean","last_fresh":"2023-09-23T21:15:31.871016+0000","last_change":"2023-09-23T21:11:27.777154+0000","last_active":"2023-09-23T21:15:31.871016+0000","last_peered":"2023-09-23T21:15:31.871016+0000","last_clean":"2023-09-23T21:15:31.871016+0000","last_became_active":"2023-09-23T21:11:27.754438+0000","last_became_peered":"2023-09-23T21:11:27.754438+0000","last_unstale":"2023-09-23T21:15:31.871016+0000","last_undegraded":"2023-09-23T21:15:31.871016+0000","last_fullsized":"2023-09-23T21:15:31.871016+0000","mapping_epoch":452,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":453,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:33:02.845882+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":89,"reported_epoch":547,"state":"active+clean","last_fresh":"2023-09-23T21:15:30.994910+0000","last_change":"2023-09-23T21:12:10.764264+0000","last_active":"2023-09-23T21:15:30.994910+0000","last_peered":"2023-09-23T21:15:30.994910+0000","last_clean":"2023-09-23T21:15:30.994910+0000","last_became_active":"2023-09-23T21:12:10.763486+0000","last_became_peered":"2023-09-23T21:12:10.763486+0000","last_unstale":"2023-09-23T21:15:30.994910+0000","last_undegraded":"2023-09-23T21:15:30.994910+0000","last_fullsized":"2023-09-23T21:15:30.994910+0000","mapping_epoch":470,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:32:27.577036+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,7],"acting":[2,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.10","version":"79'161","reported_seq":683,"reported_epoch":548,"state":"active+clean","last_fresh":"2023-09-23T21:15:31.213263+0000","last_change":"2023-09-23T21:11:11.716703+0000","last_active":"2023-09-23T21:15:31.213263+0000","last_peered":"2023-09-23T21:15:31.213263+0000","last_clean":"2023-09-23T21:15:31.213263+0000","last_became_active":"2023-09-23T21:11:11.716385+0000","last_became_peered":"2023-09-23T21:11:11.716385+0000","last_unstale":"2023-09-23T21:15:31.213263+0000","last_undegraded":"2023-09-23T21:15:31.213263+0000","last_fullsized":"2023-09-23T21:15:31.213263+0000","mapping_epoch":446,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":447,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:06:50.336992+0000","last_deep_scrub":"79'161","last_deep_scrub_stamp":"2023-09-23T21:06:50.336992+0000","last_clean_scrub_stamp":"2023-09-23T21:06:50.336992+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:44:29.538890+0000","scrub_duration":0.0026150930000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,7],"acting":[4,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":489,"reported_epoch":547,"state":"active+clean","last_fresh":"2023-09-23T21:15:30.996683+0000","last_change":"2023-09-23T21:13:12.270990+0000","last_active":"2023-09-23T21:15:30.996683+0000","last_peered":"2023-09-23T21:15:30.996683+0000","last_clean":"2023-09-23T21:15:30.996683+0000","last_became_active":"2023-09-23T21:13:12.237201+0000","last_became_peered":"2023-09-23T21:13:12.237201+0000","last_unstale":"2023-09-23T21:15:30.996683+0000","last_undegraded":"2023-09-23T21:15:30.996683+0000","last_fullsized":"2023-09-23T21:15:30.996683+0000","mapping_epoch":493,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:59:54.523877+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":300,"reported_epoch":547,"state":"active+clean","last_fresh":"2023-09-23T21:15:30.965664+0000","last_change":"2023-09-23T21:10:26.758671+0000","last_active":"2023-09-23T21:15:30.965664+0000","last_peered":"2023-09-23T21:15:30.965664+0000","last_clean":"2023-09-23T21:15:30.965664+0000","last_became_active":"2023-09-23T21:10:26.758244+0000","last_became_peered":"2023-09-23T21:10:26.758244+0000","last_unstale":"2023-09-23T21:15:30.965664+0000","last_undegraded":"2023-09-23T21:15:30.965664+0000","last_fullsized":"2023-09-23T21:15:30.965664+0000","mapping_epoch":423,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":424,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:14:02.735609+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.13","version":"79'161","reported_seq":625,"reported_epoch":547,"state":"active+clean","last_fresh":"2023-09-23T21:15:30.965621+0000","last_change":"2023-09-23T21:13:14.933096+0000","last_active":"2023-09-23T21:15:30.965621+0000","last_peered":"2023-09-23T21:15:30.965621+0000","last_clean":"2023-09-23T21:15:30.965621+0000","last_became_active":"2023-09-23T21:13:14.932703+0000","last_became_peered":"2023-09-23T21:13:14.932703+0000","last_unstale":"2023-09-23T21:15:30.965621+0000","last_undegraded":"2023-09-23T21:15:30.965621+0000","last_fullsized":"2023-09-23T21:15:30.965621+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:36.420362+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:36.420362+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:29:42.163911+0000","scrub_duration":0.028838444000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":540,"reported_epoch":548,"state":"active+clean","last_fresh":"2023-09-23T21:15:31.213408+0000","last_change":"2023-09-23T21:13:12.216467+0000","last_active":"2023-09-23T21:15:31.213408+0000","last_peered":"2023-09-23T21:15:31.213408+0000","last_clean":"2023-09-23T21:15:31.213408+0000","last_became_active":"2023-09-23T21:13:12.215259+0000","last_became_peered":"2023-09-23T21:13:12.215259+0000","last_unstale":"2023-09-23T21:15:31.213408+0000","last_undegraded":"2023-09-23T21:15:31.213408+0000","last_fullsized":"2023-09-23T21:15:31.213408+0000","mapping_epoch":493,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:35:33.987839+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":57,"reported_epoch":547,"state":"active+clean","last_fresh":"2023-09-23T21:15:30.993710+0000","last_change":"2023-09-23T21:13:30.804108+0000","last_active":"2023-09-23T21:15:30.993710+0000","last_peered":"2023-09-23T21:15:30.993710+0000","last_clean":"2023-09-23T21:15:30.993710+0000","last_became_active":"2023-09-23T21:13:30.796505+0000","last_became_peered":"2023-09-23T21:13:30.796505+0000","last_unstale":"2023-09-23T21:15:30.993710+0000","last_undegraded":"2023-09-23T21:15:30.993710+0000","last_fullsized":"2023-09-23T21:15:30.993710+0000","mapping_epoch":501,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:19:33.508496+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.12","version":"79'161","reported_seq":782,"reported_epoch":548,"state":"active+clean","last_fresh":"2023-09-23T21:15:31.869821+0000","last_change":"2023-09-23T21:13:15.206574+0000","last_active":"2023-09-23T21:15:31.869821+0000","last_peered":"2023-09-23T21:15:31.869821+0000","last_clean":"2023-09-23T21:15:31.869821+0000","last_became_active":"2023-09-23T21:13:15.206186+0000","last_became_peered":"2023-09-23T21:13:15.206186+0000","last_unstale":"2023-09-23T21:15:31.869821+0000","last_undegraded":"2023-09-23T21:15:31.869821+0000","last_fullsized":"2023-09-23T21:15:31.869821+0000","mapping_epoch":496,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":497,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:05:56.785173+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:05:56.785173+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:34:29.221809+0000","scrub_duration":0.0042114370000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,0],"acting":[5,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":67,"reported_epoch":547,"state":"active+clean","last_fresh":"2023-09-23T21:15:30.995069+0000","last_change":"2023-09-23T21:13:38.048791+0000","last_active":"2023-09-23T21:15:30.995069+0000","last_peered":"2023-09-23T21:15:30.995069+0000","last_clean":"2023-09-23T21:15:30.995069+0000","last_became_active":"2023-09-23T21:13:30.801386+0000","last_became_peered":"2023-09-23T21:13:30.801386+0000","last_unstale":"2023-09-23T21:15:30.995069+0000","last_undegraded":"2023-09-23T21:15:30.995069+0000","last_fullsized":"2023-09-23T21:15:30.995069+0000","mapping_epoch":501,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:52:49.170414+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":385,"reported_epoch":547,"state":"active+clean","last_fresh":"2023-09-23T21:15:30.965937+0000","last_change":"2023-09-23T21:12:10.765780+0000","last_active":"2023-09-23T21:15:30.965937+0000","last_peered":"2023-09-23T21:15:30.965937+0000","last_clean":"2023-09-23T21:15:30.965937+0000","last_became_active":"2023-09-23T21:12:10.765372+0000","last_became_peered":"2023-09-23T21:12:10.765372+0000","last_unstale":"2023-09-23T21:15:30.965937+0000","last_undegraded":"2023-09-23T21:15:30.965937+0000","last_fullsized":"2023-09-23T21:15:30.965937+0000","mapping_epoch":470,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:56:08.446099+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.7","version":"549'1457","reported_seq":1470,"reported_epoch":549,"state":"active+clean","last_fresh":"2023-09-23T21:15:42.010964+0000","last_change":"2023-09-23T21:15:25.902813+0000","last_active":"2023-09-23T21:15:42.010964+0000","last_peered":"2023-09-23T21:15:42.010964+0000","last_clean":"2023-09-23T21:15:42.010964+0000","last_became_active":"2023-09-23T21:15:25.902476+0000","last_became_peered":"2023-09-23T21:15:25.902476+0000","last_unstale":"2023-09-23T21:15:42.010964+0000","last_undegraded":"2023-09-23T21:15:42.010964+0000","last_fullsized":"2023-09-23T21:15:42.010964+0000","mapping_epoch":544,"log_start":"0'0","ondisk_log_start":"0'0","created":544,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_clean_scrub_stamp":"2023-09-23T21:15:24.860968+0000","objects_scrubbed":0,"log_size":1457,"log_dups_size":0,"ondisk_log_size":1457,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:41:25.699152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":95485952,"num_objects":1457,"num_object_clones":0,"num_object_copies":4371,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1457,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1457,"num_write_kb":93248,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.d","version":"79'161","reported_seq":712,"reported_epoch":548,"state":"active+clean","last_fresh":"2023-09-23T21:15:32.726824+0000","last_change":"2023-09-23T21:15:32.726755+0000","last_active":"2023-09-23T21:15:32.726824+0000","last_peered":"2023-09-23T21:15:32.726824+0000","last_clean":"2023-09-23T21:15:32.726824+0000","last_became_active":"2023-09-23T21:12:30.443979+0000","last_became_peered":"2023-09-23T21:12:30.443979+0000","last_unstale":"2023-09-23T21:15:32.726824+0000","last_undegraded":"2023-09-23T21:15:32.726824+0000","last_fullsized":"2023-09-23T21:15:32.726824+0000","mapping_epoch":470,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:15:32.726690+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:15:32.726690+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:15:47.173381+0000","scrub_duration":0.0027129250000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":195,"reported_epoch":548,"state":"active+clean","last_fresh":"2023-09-23T21:15:31.872180+0000","last_change":"2023-09-23T21:13:59.872834+0000","last_active":"2023-09-23T21:15:31.872180+0000","last_peered":"2023-09-23T21:15:31.872180+0000","last_clean":"2023-09-23T21:15:31.872180+0000","last_became_active":"2023-09-23T21:13:30.770556+0000","last_became_peered":"2023-09-23T21:13:30.770556+0000","last_unstale":"2023-09-23T21:15:31.872180+0000","last_undegraded":"2023-09-23T21:15:31.872180+0000","last_fullsized":"2023-09-23T21:15:31.872180+0000","mapping_epoch":501,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":521,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:57:55.460052+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":132,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":256,"reported_epoch":548,"state":"active+clean","last_fresh":"2023-09-23T21:15:31.869551+0000","last_change":"2023-09-23T21:10:26.750725+0000","last_active":"2023-09-23T21:15:31.869551+0000","last_peered":"2023-09-23T21:15:31.869551+0000","last_clean":"2023-09-23T21:15:31.869551+0000","last_became_active":"2023-09-23T21:10:26.750307+0000","last_became_peered":"2023-09-23T21:10:26.750307+0000","last_unstale":"2023-09-23T21:15:31.869551+0000","last_undegraded":"2023-09-23T21:15:31.869551+0000","last_fullsized":"2023-09-23T21:15:31.869551+0000","mapping_epoch":423,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":424,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:06:22.040548+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,4],"acting":[5,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.6","version":"548'1191","reported_seq":1204,"reported_epoch":548,"state":"active+clean","last_fresh":"2023-09-23T21:15:40.240063+0000","last_change":"2023-09-23T21:15:25.904499+0000","last_active":"2023-09-23T21:15:40.240063+0000","last_peered":"2023-09-23T21:15:40.240063+0000","last_clean":"2023-09-23T21:15:40.240063+0000","last_became_active":"2023-09-23T21:15:25.904185+0000","last_became_peered":"2023-09-23T21:15:25.904185+0000","last_unstale":"2023-09-23T21:15:40.240063+0000","last_undegraded":"2023-09-23T21:15:40.240063+0000","last_fullsized":"2023-09-23T21:15:40.240063+0000","mapping_epoch":544,"log_start":"0'0","ondisk_log_start":"0'0","created":544,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_clean_scrub_stamp":"2023-09-23T21:15:24.860968+0000","objects_scrubbed":0,"log_size":1191,"log_dups_size":0,"ondisk_log_size":1191,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:02:41.735858+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":78053376,"num_objects":1191,"num_object_clones":0,"num_object_copies":3573,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1191,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1191,"num_write_kb":76224,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.c","version":"79'161","reported_seq":929,"reported_epoch":548,"state":"active+clean","last_fresh":"2023-09-23T21:15:31.992633+0000","last_change":"2023-09-23T21:15:31.992756+0000","last_active":"2023-09-23T21:15:31.992633+0000","last_peered":"2023-09-23T21:15:31.992633+0000","last_clean":"2023-09-23T21:15:31.992633+0000","last_became_active":"2023-09-23T21:13:13.401526+0000","last_became_peered":"2023-09-23T21:13:13.401526+0000","last_unstale":"2023-09-23T21:15:31.992633+0000","last_undegraded":"2023-09-23T21:15:31.992633+0000","last_fullsized":"2023-09-23T21:15:31.992633+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":532,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:15:31.992598+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:15:31.992598+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:41:14.215543+0000","scrub_duration":0.002960678,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":181,"reported_epoch":548,"state":"active+clean","last_fresh":"2023-09-23T21:15:31.871978+0000","last_change":"2023-09-23T21:13:59.884113+0000","last_active":"2023-09-23T21:15:31.871978+0000","last_peered":"2023-09-23T21:15:31.871978+0000","last_clean":"2023-09-23T21:15:31.871978+0000","last_became_active":"2023-09-23T21:13:30.806778+0000","last_became_peered":"2023-09-23T21:13:30.806778+0000","last_unstale":"2023-09-23T21:15:31.871978+0000","last_undegraded":"2023-09-23T21:15:31.871978+0000","last_fullsized":"2023-09-23T21:15:31.871978+0000","mapping_epoch":501,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":521,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:36:15.284465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":511,"reported_epoch":548,"state":"active+clean","last_fresh":"2023-09-23T21:15:31.213413+0000","last_change":"2023-09-23T21:13:15.152792+0000","last_active":"2023-09-23T21:15:31.213413+0000","last_peered":"2023-09-23T21:15:31.213413+0000","last_clean":"2023-09-23T21:15:31.213413+0000","last_became_active":"2023-09-23T21:13:14.515084+0000","last_became_peered":"2023-09-23T21:13:14.515084+0000","last_unstale":"2023-09-23T21:15:31.213413+0000","last_undegraded":"2023-09-23T21:15:31.213413+0000","last_fullsized":"2023-09-23T21:15:31.213413+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:08:16.702072+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.5","version":"549'1473","reported_seq":1486,"reported_epoch":549,"state":"active+clean","last_fresh":"2023-09-23T21:15:42.318784+0000","last_change":"2023-09-23T21:15:25.893591+0000","last_active":"2023-09-23T21:15:42.318784+0000","last_peered":"2023-09-23T21:15:42.318784+0000","last_clean":"2023-09-23T21:15:42.318784+0000","last_became_active":"2023-09-23T21:15:25.893280+0000","last_became_peered":"2023-09-23T21:15:25.893280+0000","last_unstale":"2023-09-23T21:15:42.318784+0000","last_undegraded":"2023-09-23T21:15:42.318784+0000","last_fullsized":"2023-09-23T21:15:42.318784+0000","mapping_epoch":544,"log_start":"0'0","ondisk_log_start":"0'0","created":544,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_clean_scrub_stamp":"2023-09-23T21:15:24.860968+0000","objects_scrubbed":0,"log_size":1473,"log_dups_size":0,"ondisk_log_size":1473,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:02:55.710366+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":96534528,"num_objects":1473,"num_object_clones":0,"num_object_copies":4419,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1473,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1473,"num_write_kb":94272,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.f","version":"79'161","reported_seq":836,"reported_epoch":548,"state":"active+clean","last_fresh":"2023-09-23T21:15:31.869723+0000","last_change":"2023-09-23T21:11:30.750338+0000","last_active":"2023-09-23T21:15:31.869723+0000","last_peered":"2023-09-23T21:15:31.869723+0000","last_clean":"2023-09-23T21:15:31.869723+0000","last_became_active":"2023-09-23T21:11:30.749953+0000","last_became_peered":"2023-09-23T21:11:30.749953+0000","last_unstale":"2023-09-23T21:15:31.869723+0000","last_undegraded":"2023-09-23T21:15:31.869723+0000","last_fullsized":"2023-09-23T21:15:31.869723+0000","mapping_epoch":455,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":456,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:27.880776+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:27.880776+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:07:10.405723+0000","scrub_duration":0.0014004899999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,3],"acting":[5,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":272,"reported_epoch":548,"state":"active+clean","last_fresh":"2023-09-23T21:15:31.870057+0000","last_change":"2023-09-23T21:10:26.765068+0000","last_active":"2023-09-23T21:15:31.870057+0000","last_peered":"2023-09-23T21:15:31.870057+0000","last_clean":"2023-09-23T21:15:31.870057+0000","last_became_active":"2023-09-23T21:10:26.758364+0000","last_became_peered":"2023-09-23T21:10:26.758364+0000","last_unstale":"2023-09-23T21:15:31.870057+0000","last_undegraded":"2023-09-23T21:15:31.870057+0000","last_fullsized":"2023-09-23T21:15:31.870057+0000","mapping_epoch":423,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":424,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:13:21.057529+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":280,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":488,"reported_epoch":548,"state":"active+clean","last_fresh":"2023-09-23T21:15:31.869442+0000","last_change":"2023-09-23T21:13:30.770746+0000","last_active":"2023-09-23T21:15:31.869442+0000","last_peered":"2023-09-23T21:15:31.869442+0000","last_clean":"2023-09-23T21:15:31.869442+0000","last_became_active":"2023-09-23T21:13:30.770437+0000","last_became_peered":"2023-09-23T21:13:30.770437+0000","last_unstale":"2023-09-23T21:15:31.869442+0000","last_undegraded":"2023-09-23T21:15:31.869442+0000","last_fullsized":"2023-09-23T21:15:31.869442+0000","mapping_epoch":501,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:33:48.325730+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,0],"acting":[5,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.4","version":"549'1458","reported_seq":1471,"reported_epoch":549,"state":"active+clean","last_fresh":"2023-09-23T21:15:42.291008+0000","last_change":"2023-09-23T21:15:25.892580+0000","last_active":"2023-09-23T21:15:42.291008+0000","last_peered":"2023-09-23T21:15:42.291008+0000","last_clean":"2023-09-23T21:15:42.291008+0000","last_became_active":"2023-09-23T21:15:25.892251+0000","last_became_peered":"2023-09-23T21:15:25.892251+0000","last_unstale":"2023-09-23T21:15:42.291008+0000","last_undegraded":"2023-09-23T21:15:42.291008+0000","last_fullsized":"2023-09-23T21:15:42.291008+0000","mapping_epoch":544,"log_start":"0'0","ondisk_log_start":"0'0","created":544,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_clean_scrub_stamp":"2023-09-23T21:15:24.860968+0000","objects_scrubbed":0,"log_size":1458,"log_dups_size":0,"ondisk_log_size":1458,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:21:43.221845+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":95551488,"num_objects":1458,"num_object_clones":0,"num_object_copies":4374,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1458,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1458,"num_write_kb":93312,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,3],"acting":[7,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.e","version":"79'161","reported_seq":727,"reported_epoch":548,"state":"active+clean","last_fresh":"2023-09-23T21:15:39.047111+0000","last_change":"2023-09-23T21:15:39.047046+0000","last_active":"2023-09-23T21:15:39.047111+0000","last_peered":"2023-09-23T21:15:39.047111+0000","last_clean":"2023-09-23T21:15:39.047111+0000","last_became_active":"2023-09-23T21:12:30.440463+0000","last_became_peered":"2023-09-23T21:12:30.440463+0000","last_unstale":"2023-09-23T21:15:39.047111+0000","last_undegraded":"2023-09-23T21:15:39.047111+0000","last_fullsized":"2023-09-23T21:15:39.047111+0000","mapping_epoch":455,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:15:39.046962+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:15:39.046962+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:48:00.972293+0000","scrub_duration":0.0082638769999999993,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":345,"reported_epoch":547,"state":"active+clean","last_fresh":"2023-09-23T21:15:30.967141+0000","last_change":"2023-09-23T21:13:13.402418+0000","last_active":"2023-09-23T21:15:30.967141+0000","last_peered":"2023-09-23T21:15:30.967141+0000","last_clean":"2023-09-23T21:15:30.967141+0000","last_became_active":"2023-09-23T21:13:13.400984+0000","last_became_peered":"2023-09-23T21:13:13.400984+0000","last_unstale":"2023-09-23T21:15:30.967141+0000","last_undegraded":"2023-09-23T21:15:30.967141+0000","last_fullsized":"2023-09-23T21:15:30.967141+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":495,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:28:01.411191+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,0],"acting":[3,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":521,"reported_epoch":547,"state":"active+clean","last_fresh":"2023-09-23T21:15:30.995823+0000","last_change":"2023-09-23T21:13:14.339670+0000","last_active":"2023-09-23T21:15:30.995823+0000","last_peered":"2023-09-23T21:15:30.995823+0000","last_clean":"2023-09-23T21:15:30.995823+0000","last_became_active":"2023-09-23T21:13:14.339341+0000","last_became_peered":"2023-09-23T21:13:14.339341+0000","last_unstale":"2023-09-23T21:15:30.995823+0000","last_undegraded":"2023-09-23T21:15:30.995823+0000","last_fullsized":"2023-09-23T21:15:30.995823+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:23:19.411368+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.3","version":"549'1457","reported_seq":1470,"reported_epoch":549,"state":"active+clean","last_fresh":"2023-09-23T21:15:42.263971+0000","last_change":"2023-09-23T21:15:25.891637+0000","last_active":"2023-09-23T21:15:42.263971+0000","last_peered":"2023-09-23T21:15:42.263971+0000","last_clean":"2023-09-23T21:15:42.263971+0000","last_became_active":"2023-09-23T21:15:25.891330+0000","last_became_peered":"2023-09-23T21:15:25.891330+0000","last_unstale":"2023-09-23T21:15:42.263971+0000","last_undegraded":"2023-09-23T21:15:42.263971+0000","last_fullsized":"2023-09-23T21:15:42.263971+0000","mapping_epoch":544,"log_start":"0'0","ondisk_log_start":"0'0","created":544,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_clean_scrub_stamp":"2023-09-23T21:15:24.860968+0000","objects_scrubbed":0,"log_size":1457,"log_dups_size":0,"ondisk_log_size":1457,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:22:07.243528+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":95485952,"num_objects":1457,"num_object_clones":0,"num_object_copies":4371,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1457,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1457,"num_write_kb":93248,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,2],"acting":[7,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.9","version":"79'161","reported_seq":702,"reported_epoch":547,"state":"active+clean","last_fresh":"2023-09-23T21:15:30.994000+0000","last_change":"2023-09-23T21:12:30.436118+0000","last_active":"2023-09-23T21:15:30.994000+0000","last_peered":"2023-09-23T21:15:30.994000+0000","last_clean":"2023-09-23T21:15:30.994000+0000","last_became_active":"2023-09-23T21:12:30.435779+0000","last_became_peered":"2023-09-23T21:12:30.435779+0000","last_unstale":"2023-09-23T21:15:30.994000+0000","last_undegraded":"2023-09-23T21:15:30.994000+0000","last_fullsized":"2023-09-23T21:15:30.994000+0000","mapping_epoch":433,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:10:37.696951+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:10:37.696951+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.001386504,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,3],"acting":[2,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":106,"reported_epoch":547,"state":"active+clean","last_fresh":"2023-09-23T21:15:30.995748+0000","last_change":"2023-09-23T21:13:30.793273+0000","last_active":"2023-09-23T21:15:30.995748+0000","last_peered":"2023-09-23T21:15:30.995748+0000","last_clean":"2023-09-23T21:15:30.995748+0000","last_became_active":"2023-09-23T21:13:30.791269+0000","last_became_peered":"2023-09-23T21:13:30.791269+0000","last_unstale":"2023-09-23T21:15:30.995748+0000","last_undegraded":"2023-09-23T21:15:30.995748+0000","last_fullsized":"2023-09-23T21:15:30.995748+0000","mapping_epoch":501,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:38:07.142290+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":122,"reported_epoch":548,"state":"active+clean","last_fresh":"2023-09-23T21:15:31.213719+0000","last_change":"2023-09-23T21:12:14.313466+0000","last_active":"2023-09-23T21:15:31.213719+0000","last_peered":"2023-09-23T21:15:31.213719+0000","last_clean":"2023-09-23T21:15:31.213719+0000","last_became_active":"2023-09-23T21:12:14.209837+0000","last_became_peered":"2023-09-23T21:12:14.209837+0000","last_unstale":"2023-09-23T21:15:31.213719+0000","last_undegraded":"2023-09-23T21:15:31.213719+0000","last_fullsized":"2023-09-23T21:15:31.213719+0000","mapping_epoch":470,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":474,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:09:40.375241+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,7],"acting":[4,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.2","version":"548'1329","reported_seq":1340,"reported_epoch":548,"state":"active+clean","last_fresh":"2023-09-23T21:15:41.332527+0000","last_change":"2023-09-23T21:15:25.900671+0000","last_active":"2023-09-23T21:15:41.332527+0000","last_peered":"2023-09-23T21:15:41.332527+0000","last_clean":"2023-09-23T21:15:41.332527+0000","last_became_active":"2023-09-23T21:15:25.900364+0000","last_became_peered":"2023-09-23T21:15:25.900364+0000","last_unstale":"2023-09-23T21:15:41.332527+0000","last_undegraded":"2023-09-23T21:15:41.332527+0000","last_fullsized":"2023-09-23T21:15:41.332527+0000","mapping_epoch":544,"log_start":"0'0","ondisk_log_start":"0'0","created":544,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_clean_scrub_stamp":"2023-09-23T21:15:24.860968+0000","objects_scrubbed":0,"log_size":1329,"log_dups_size":0,"ondisk_log_size":1329,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:03:40.630119+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":87097344,"num_objects":1329,"num_object_clones":0,"num_object_copies":3987,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1329,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1329,"num_write_kb":85056,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.8","version":"79'161","reported_seq":680,"reported_epoch":548,"state":"active+clean","last_fresh":"2023-09-23T21:15:36.367680+0000","last_change":"2023-09-23T21:15:36.367819+0000","last_active":"2023-09-23T21:15:36.367680+0000","last_peered":"2023-09-23T21:15:36.367680+0000","last_clean":"2023-09-23T21:15:36.367680+0000","last_became_active":"2023-09-23T21:12:30.433217+0000","last_became_peered":"2023-09-23T21:12:30.433217+0000","last_unstale":"2023-09-23T21:15:36.367680+0000","last_undegraded":"2023-09-23T21:15:36.367680+0000","last_fullsized":"2023-09-23T21:15:36.367680+0000","mapping_epoch":359,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:15:36.367640+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:15:36.367640+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:49:35.243653+0000","scrub_duration":0.005826899,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":125,"reported_epoch":548,"state":"active+clean","last_fresh":"2023-09-23T21:15:31.213558+0000","last_change":"2023-09-23T21:12:10.823380+0000","last_active":"2023-09-23T21:15:31.213558+0000","last_peered":"2023-09-23T21:15:31.213558+0000","last_clean":"2023-09-23T21:15:31.213558+0000","last_became_active":"2023-09-23T21:12:10.823044+0000","last_became_peered":"2023-09-23T21:12:10.823044+0000","last_unstale":"2023-09-23T21:15:31.213558+0000","last_undegraded":"2023-09-23T21:15:31.213558+0000","last_fullsized":"2023-09-23T21:15:31.213558+0000","mapping_epoch":470,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:53:29.846536+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,5],"acting":[4,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":126,"reported_epoch":548,"state":"active+clean","last_fresh":"2023-09-23T21:15:31.213917+0000","last_change":"2023-09-23T21:13:30.772451+0000","last_active":"2023-09-23T21:15:31.213917+0000","last_peered":"2023-09-23T21:15:31.213917+0000","last_clean":"2023-09-23T21:15:31.213917+0000","last_became_active":"2023-09-23T21:13:30.772040+0000","last_became_peered":"2023-09-23T21:13:30.772040+0000","last_unstale":"2023-09-23T21:15:31.213917+0000","last_undegraded":"2023-09-23T21:15:31.213917+0000","last_fullsized":"2023-09-23T21:15:31.213917+0000","mapping_epoch":501,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:24:49.246727+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,0],"acting":[4,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.1","version":"548'1079","reported_seq":1092,"reported_epoch":548,"state":"active+clean","last_fresh":"2023-09-23T21:15:39.228781+0000","last_change":"2023-09-23T21:15:25.903355+0000","last_active":"2023-09-23T21:15:39.228781+0000","last_peered":"2023-09-23T21:15:39.228781+0000","last_clean":"2023-09-23T21:15:39.228781+0000","last_became_active":"2023-09-23T21:15:25.902988+0000","last_became_peered":"2023-09-23T21:15:25.902988+0000","last_unstale":"2023-09-23T21:15:39.228781+0000","last_undegraded":"2023-09-23T21:15:39.228781+0000","last_fullsized":"2023-09-23T21:15:39.228781+0000","mapping_epoch":544,"log_start":"0'0","ondisk_log_start":"0'0","created":544,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_clean_scrub_stamp":"2023-09-23T21:15:24.860968+0000","objects_scrubbed":0,"log_size":1079,"log_dups_size":0,"ondisk_log_size":1079,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:47:02.629642+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70713344,"num_objects":1079,"num_object_clones":0,"num_object_copies":3237,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1079,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1079,"num_write_kb":69056,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,2],"acting":[3,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.b","version":"79'161","reported_seq":564,"reported_epoch":548,"state":"active+clean","last_fresh":"2023-09-23T21:15:36.339904+0000","last_change":"2023-09-23T21:15:36.340009+0000","last_active":"2023-09-23T21:15:36.339904+0000","last_peered":"2023-09-23T21:15:36.339904+0000","last_clean":"2023-09-23T21:15:36.339904+0000","last_became_active":"2023-09-23T21:13:13.372328+0000","last_became_peered":"2023-09-23T21:13:13.372328+0000","last_unstale":"2023-09-23T21:15:36.339904+0000","last_undegraded":"2023-09-23T21:15:36.339904+0000","last_fullsized":"2023-09-23T21:15:36.339904+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":495,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:15:36.339885+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:15:36.339885+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:01:26.605006+0000","scrub_duration":0.0068473279999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":106,"reported_epoch":548,"state":"active+clean","last_fresh":"2023-09-23T21:15:31.213336+0000","last_change":"2023-09-23T21:11:27.754717+0000","last_active":"2023-09-23T21:15:31.213336+0000","last_peered":"2023-09-23T21:15:31.213336+0000","last_clean":"2023-09-23T21:15:31.213336+0000","last_became_active":"2023-09-23T21:11:27.754376+0000","last_became_peered":"2023-09-23T21:11:27.754376+0000","last_unstale":"2023-09-23T21:15:31.213336+0000","last_undegraded":"2023-09-23T21:15:31.213336+0000","last_fullsized":"2023-09-23T21:15:31.213336+0000","mapping_epoch":452,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":453,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:26:41.734541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":480,"reported_epoch":548,"state":"active+clean","last_fresh":"2023-09-23T21:15:31.214251+0000","last_change":"2023-09-23T21:12:10.811698+0000","last_active":"2023-09-23T21:15:31.214251+0000","last_peered":"2023-09-23T21:15:31.214251+0000","last_clean":"2023-09-23T21:15:31.214251+0000","last_became_active":"2023-09-23T21:12:10.789656+0000","last_became_peered":"2023-09-23T21:12:10.789656+0000","last_unstale":"2023-09-23T21:15:31.214251+0000","last_undegraded":"2023-09-23T21:15:31.214251+0000","last_fullsized":"2023-09-23T21:15:31.214251+0000","mapping_epoch":470,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:39:15.112091+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,5],"acting":[4,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.0","version":"548'1124","reported_seq":1137,"reported_epoch":548,"state":"active+clean","last_fresh":"2023-09-23T21:15:39.220534+0000","last_change":"2023-09-23T21:15:25.902910+0000","last_active":"2023-09-23T21:15:39.220534+0000","last_peered":"2023-09-23T21:15:39.220534+0000","last_clean":"2023-09-23T21:15:39.220534+0000","last_became_active":"2023-09-23T21:15:25.902656+0000","last_became_peered":"2023-09-23T21:15:25.902656+0000","last_unstale":"2023-09-23T21:15:39.220534+0000","last_undegraded":"2023-09-23T21:15:39.220534+0000","last_fullsized":"2023-09-23T21:15:39.220534+0000","mapping_epoch":544,"log_start":"0'0","ondisk_log_start":"0'0","created":544,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_clean_scrub_stamp":"2023-09-23T21:15:24.860968+0000","objects_scrubbed":0,"log_size":1124,"log_dups_size":0,"ondisk_log_size":1124,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:30:07.984391+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":73662464,"num_objects":1124,"num_object_clones":0,"num_object_copies":3372,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1124,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1124,"num_write_kb":71936,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.a","version":"79'161","reported_seq":694,"reported_epoch":547,"state":"active+clean","last_fresh":"2023-09-23T21:15:30.994539+0000","last_change":"2023-09-23T21:12:30.437790+0000","last_active":"2023-09-23T21:15:30.994539+0000","last_peered":"2023-09-23T21:15:30.994539+0000","last_clean":"2023-09-23T21:15:30.994539+0000","last_became_active":"2023-09-23T21:12:30.437427+0000","last_became_peered":"2023-09-23T21:12:30.437427+0000","last_unstale":"2023-09-23T21:15:30.994539+0000","last_undegraded":"2023-09-23T21:15:30.994539+0000","last_fullsized":"2023-09-23T21:15:30.994539+0000","mapping_epoch":446,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:06:51.075448+0000","last_deep_scrub":"79'161","last_deep_scrub_stamp":"2023-09-23T21:05:32.630292+0000","last_clean_scrub_stamp":"2023-09-23T21:06:51.075448+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0054793890000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,4],"acting":[2,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":395,"reported_epoch":547,"state":"active+clean","last_fresh":"2023-09-23T21:15:30.965794+0000","last_change":"2023-09-23T21:12:10.792918+0000","last_active":"2023-09-23T21:15:30.965794+0000","last_peered":"2023-09-23T21:15:30.965794+0000","last_clean":"2023-09-23T21:15:30.965794+0000","last_became_active":"2023-09-23T21:12:10.792679+0000","last_became_peered":"2023-09-23T21:12:10.792679+0000","last_unstale":"2023-09-23T21:15:30.965794+0000","last_undegraded":"2023-09-23T21:15:30.965794+0000","last_fullsized":"2023-09-23T21:15:30.965794+0000","mapping_epoch":470,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:30:48.008690+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,4],"acting":[3,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":526,"reported_epoch":547,"state":"active+clean","last_fresh":"2023-09-23T21:15:30.965753+0000","last_change":"2023-09-23T21:12:10.790812+0000","last_active":"2023-09-23T21:15:30.965753+0000","last_peered":"2023-09-23T21:15:30.965753+0000","last_clean":"2023-09-23T21:15:30.965753+0000","last_became_active":"2023-09-23T21:12:10.790492+0000","last_became_peered":"2023-09-23T21:12:10.790492+0000","last_unstale":"2023-09-23T21:15:30.965753+0000","last_undegraded":"2023-09-23T21:15:30.965753+0000","last_fullsized":"2023-09-23T21:15:30.965753+0000","mapping_epoch":470,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:02:19.863118+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.d","version":"548'1367","reported_seq":1380,"reported_epoch":548,"state":"active+clean","last_fresh":"2023-09-23T21:15:41.328715+0000","last_change":"2023-09-23T21:15:25.892834+0000","last_active":"2023-09-23T21:15:41.328715+0000","last_peered":"2023-09-23T21:15:41.328715+0000","last_clean":"2023-09-23T21:15:41.328715+0000","last_became_active":"2023-09-23T21:15:25.892519+0000","last_became_peered":"2023-09-23T21:15:25.892519+0000","last_unstale":"2023-09-23T21:15:41.328715+0000","last_undegraded":"2023-09-23T21:15:41.328715+0000","last_fullsized":"2023-09-23T21:15:41.328715+0000","mapping_epoch":544,"log_start":"0'0","ondisk_log_start":"0'0","created":544,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_clean_scrub_stamp":"2023-09-23T21:15:24.860968+0000","objects_scrubbed":0,"log_size":1367,"log_dups_size":0,"ondisk_log_size":1367,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:46:28.375665+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":89587712,"num_objects":1367,"num_object_clones":0,"num_object_copies":4101,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1367,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1367,"num_write_kb":87488,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,5],"acting":[2,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.7","version":"79'161","reported_seq":640,"reported_epoch":547,"state":"active+clean","last_fresh":"2023-09-23T21:15:30.967183+0000","last_change":"2023-09-23T21:13:44.416548+0000","last_active":"2023-09-23T21:15:30.967183+0000","last_peered":"2023-09-23T21:15:30.967183+0000","last_clean":"2023-09-23T21:15:30.967183+0000","last_became_active":"2023-09-23T21:13:44.415617+0000","last_became_peered":"2023-09-23T21:13:44.415617+0000","last_unstale":"2023-09-23T21:15:30.967183+0000","last_undegraded":"2023-09-23T21:15:30.967183+0000","last_fullsized":"2023-09-23T21:15:30.967183+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":3,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:25.916867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:25.916867+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.001665323,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":478,"reported_epoch":547,"state":"active+clean","last_fresh":"2023-09-23T21:15:30.993815+0000","last_change":"2023-09-23T21:13:13.211959+0000","last_active":"2023-09-23T21:15:30.993815+0000","last_peered":"2023-09-23T21:15:30.993815+0000","last_clean":"2023-09-23T21:15:30.993815+0000","last_became_active":"2023-09-23T21:13:13.211712+0000","last_became_peered":"2023-09-23T21:13:13.211712+0000","last_unstale":"2023-09-23T21:15:30.993815+0000","last_undegraded":"2023-09-23T21:15:30.993815+0000","last_fullsized":"2023-09-23T21:15:30.993815+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":495,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:37:21.655775+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,2],"acting":[0,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":226,"reported_epoch":548,"state":"active+clean","last_fresh":"2023-09-23T21:15:31.869971+0000","last_change":"2023-09-23T21:05:04.326889+0000","last_active":"2023-09-23T21:15:31.869971+0000","last_peered":"2023-09-23T21:15:31.869971+0000","last_clean":"2023-09-23T21:15:31.869971+0000","last_became_active":"2023-09-23T21:05:04.326032+0000","last_became_peered":"2023-09-23T21:05:04.326032+0000","last_unstale":"2023-09-23T21:15:31.869971+0000","last_undegraded":"2023-09-23T21:15:31.869971+0000","last_fullsized":"2023-09-23T21:15:31.869971+0000","mapping_epoch":332,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:00:17.507889+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":558,"reported_epoch":548,"state":"active+clean","last_fresh":"2023-09-23T21:15:31.869648+0000","last_change":"2023-09-23T21:13:13.285608+0000","last_active":"2023-09-23T21:15:31.869648+0000","last_peered":"2023-09-23T21:15:31.869648+0000","last_clean":"2023-09-23T21:15:31.869648+0000","last_became_active":"2023-09-23T21:13:13.285255+0000","last_became_peered":"2023-09-23T21:13:13.285255+0000","last_unstale":"2023-09-23T21:15:31.869648+0000","last_undegraded":"2023-09-23T21:15:31.869648+0000","last_fullsized":"2023-09-23T21:15:31.869648+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":495,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:42:30.087599+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.b","version":"548'1026","reported_seq":1039,"reported_epoch":548,"state":"active+clean","last_fresh":"2023-09-23T21:15:39.225001+0000","last_change":"2023-09-23T21:15:25.906791+0000","last_active":"2023-09-23T21:15:39.225001+0000","last_peered":"2023-09-23T21:15:39.225001+0000","last_clean":"2023-09-23T21:15:39.225001+0000","last_became_active":"2023-09-23T21:15:25.906512+0000","last_became_peered":"2023-09-23T21:15:25.906512+0000","last_unstale":"2023-09-23T21:15:39.225001+0000","last_undegraded":"2023-09-23T21:15:39.225001+0000","last_fullsized":"2023-09-23T21:15:39.225001+0000","mapping_epoch":544,"log_start":"0'0","ondisk_log_start":"0'0","created":544,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_clean_scrub_stamp":"2023-09-23T21:15:24.860968+0000","objects_scrubbed":0,"log_size":1026,"log_dups_size":0,"ondisk_log_size":1026,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:58:51.749775+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":67239936,"num_objects":1026,"num_object_clones":0,"num_object_copies":3078,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1026,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1026,"num_write_kb":65664,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1","version":"79'161","reported_seq":631,"reported_epoch":548,"state":"active+clean","last_fresh":"2023-09-23T21:15:34.345172+0000","last_change":"2023-09-23T21:15:34.345290+0000","last_active":"2023-09-23T21:15:34.345172+0000","last_peered":"2023-09-23T21:15:34.345172+0000","last_clean":"2023-09-23T21:15:34.345172+0000","last_became_active":"2023-09-23T21:13:12.203380+0000","last_became_peered":"2023-09-23T21:13:12.203380+0000","last_unstale":"2023-09-23T21:15:34.345172+0000","last_undegraded":"2023-09-23T21:15:34.345172+0000","last_fullsized":"2023-09-23T21:15:34.345172+0000","mapping_epoch":493,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:15:34.345138+0000","last_deep_scrub":"79'161","last_deep_scrub_stamp":"2023-09-23T21:05:30.652189+0000","last_clean_scrub_stamp":"2023-09-23T21:15:34.345138+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:47:50.349409+0000","scrub_duration":0.0057520000000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":215,"reported_epoch":547,"state":"active+clean","last_fresh":"2023-09-23T21:15:30.996554+0000","last_change":"2023-09-23T21:13:13.238617+0000","last_active":"2023-09-23T21:15:30.996554+0000","last_peered":"2023-09-23T21:15:30.996554+0000","last_clean":"2023-09-23T21:15:30.996554+0000","last_became_active":"2023-09-23T21:13:13.232926+0000","last_became_peered":"2023-09-23T21:13:13.232926+0000","last_unstale":"2023-09-23T21:15:30.996554+0000","last_undegraded":"2023-09-23T21:15:30.996554+0000","last_fullsized":"2023-09-23T21:15:30.996554+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":495,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:17:21.823772+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":482,"reported_epoch":548,"state":"active+clean","last_fresh":"2023-09-23T21:15:31.213844+0000","last_change":"2023-09-23T21:13:14.144918+0000","last_active":"2023-09-23T21:15:31.213844+0000","last_peered":"2023-09-23T21:15:31.213844+0000","last_clean":"2023-09-23T21:15:31.213844+0000","last_became_active":"2023-09-23T21:13:14.132388+0000","last_became_peered":"2023-09-23T21:13:14.132388+0000","last_unstale":"2023-09-23T21:15:31.213844+0000","last_undegraded":"2023-09-23T21:15:31.213844+0000","last_fullsized":"2023-09-23T21:15:31.213844+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":495,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:56:02.619905+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,5],"acting":[4,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":368,"reported_epoch":547,"state":"active+clean","last_fresh":"2023-09-23T21:15:30.994814+0000","last_change":"2023-09-23T21:12:10.764503+0000","last_active":"2023-09-23T21:15:30.994814+0000","last_peered":"2023-09-23T21:15:30.994814+0000","last_clean":"2023-09-23T21:15:30.994814+0000","last_became_active":"2023-09-23T21:12:10.763933+0000","last_became_peered":"2023-09-23T21:12:10.763933+0000","last_unstale":"2023-09-23T21:15:30.994814+0000","last_undegraded":"2023-09-23T21:15:30.994814+0000","last_fullsized":"2023-09-23T21:15:30.994814+0000","mapping_epoch":470,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:28:33.202825+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,7],"acting":[2,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"11.8","version":"548'1200","reported_seq":1213,"reported_epoch":548,"state":"active+clean","last_fresh":"2023-09-23T21:15:40.243738+0000","last_change":"2023-09-23T21:15:25.905508+0000","last_active":"2023-09-23T21:15:40.243738+0000","last_peered":"2023-09-23T21:15:40.243738+0000","last_clean":"2023-09-23T21:15:40.243738+0000","last_became_active":"2023-09-23T21:15:25.905240+0000","last_became_peered":"2023-09-23T21:15:25.905240+0000","last_unstale":"2023-09-23T21:15:40.243738+0000","last_undegraded":"2023-09-23T21:15:40.243738+0000","last_fullsized":"2023-09-23T21:15:40.243738+0000","mapping_epoch":544,"log_start":"0'0","ondisk_log_start":"0'0","created":544,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_clean_scrub_stamp":"2023-09-23T21:15:24.860968+0000","objects_scrubbed":0,"log_size":1200,"log_dups_size":0,"ondisk_log_size":1200,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:16:01.535087+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":78643200,"num_objects":1200,"num_object_clones":0,"num_object_copies":3600,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1200,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1200,"num_write_kb":76800,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,5],"acting":[0,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.2","version":"79'161","reported_seq":828,"reported_epoch":547,"state":"active+clean","last_fresh":"2023-09-23T21:15:30.996243+0000","last_change":"2023-09-23T21:13:12.209907+0000","last_active":"2023-09-23T21:15:30.996243+0000","last_peered":"2023-09-23T21:15:30.996243+0000","last_clean":"2023-09-23T21:15:30.996243+0000","last_became_active":"2023-09-23T21:13:12.205664+0000","last_became_peered":"2023-09-23T21:13:12.205664+0000","last_unstale":"2023-09-23T21:15:30.996243+0000","last_undegraded":"2023-09-23T21:15:30.996243+0000","last_fullsized":"2023-09-23T21:15:30.996243+0000","mapping_epoch":493,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:24.802451+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:24.802451+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:26:16.035788+0000","scrub_duration":0.0090861250000000005,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,4],"acting":[0,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":557,"reported_epoch":548,"state":"active+clean","last_fresh":"2023-09-23T21:15:31.523174+0000","last_change":"2023-09-23T21:13:12.206980+0000","last_active":"2023-09-23T21:15:31.523174+0000","last_peered":"2023-09-23T21:15:31.523174+0000","last_clean":"2023-09-23T21:15:31.523174+0000","last_became_active":"2023-09-23T21:13:12.206650+0000","last_became_peered":"2023-09-23T21:13:12.206650+0000","last_unstale":"2023-09-23T21:15:31.523174+0000","last_undegraded":"2023-09-23T21:15:31.523174+0000","last_fullsized":"2023-09-23T21:15:31.523174+0000","mapping_epoch":493,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:10:01.892936+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":540,"reported_epoch":547,"state":"active+clean","last_fresh":"2023-09-23T21:15:30.993896+0000","last_change":"2023-09-23T21:13:13.407073+0000","last_active":"2023-09-23T21:15:30.993896+0000","last_peered":"2023-09-23T21:15:30.993896+0000","last_clean":"2023-09-23T21:15:30.993896+0000","last_became_active":"2023-09-23T21:13:13.395047+0000","last_became_peered":"2023-09-23T21:13:13.395047+0000","last_unstale":"2023-09-23T21:15:30.993896+0000","last_undegraded":"2023-09-23T21:15:30.993896+0000","last_fullsized":"2023-09-23T21:15:30.993896+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":495,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:04:43.384190+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,5],"acting":[2,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":57,"reported_epoch":547,"state":"active+clean","last_fresh":"2023-09-23T21:15:30.994263+0000","last_change":"2023-09-23T21:13:30.785054+0000","last_active":"2023-09-23T21:15:30.994263+0000","last_peered":"2023-09-23T21:15:30.994263+0000","last_clean":"2023-09-23T21:15:30.994263+0000","last_became_active":"2023-09-23T21:13:30.784701+0000","last_became_peered":"2023-09-23T21:13:30.784701+0000","last_unstale":"2023-09-23T21:15:30.994263+0000","last_undegraded":"2023-09-23T21:15:30.994263+0000","last_fullsized":"2023-09-23T21:15:30.994263+0000","mapping_epoch":501,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:19:20.152252+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.a","version":"548'1362","reported_seq":1374,"reported_epoch":548,"state":"active+clean","last_fresh":"2023-09-23T21:15:41.329661+0000","last_change":"2023-09-23T21:15:25.893846+0000","last_active":"2023-09-23T21:15:41.329661+0000","last_peered":"2023-09-23T21:15:41.329661+0000","last_clean":"2023-09-23T21:15:41.329661+0000","last_became_active":"2023-09-23T21:15:25.893452+0000","last_became_peered":"2023-09-23T21:15:25.893452+0000","last_unstale":"2023-09-23T21:15:41.329661+0000","last_undegraded":"2023-09-23T21:15:41.329661+0000","last_fullsized":"2023-09-23T21:15:41.329661+0000","mapping_epoch":544,"log_start":"0'0","ondisk_log_start":"0'0","created":544,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_clean_scrub_stamp":"2023-09-23T21:15:24.860968+0000","objects_scrubbed":0,"log_size":1362,"log_dups_size":0,"ondisk_log_size":1362,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:42:32.842297+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":89260032,"num_objects":1362,"num_object_clones":0,"num_object_copies":4086,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1362,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1362,"num_write_kb":87168,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,5],"acting":[2,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":334,"reported_epoch":547,"state":"active+clean","last_fresh":"2023-09-23T21:15:30.995587+0000","last_change":"2023-09-23T21:10:26.769764+0000","last_active":"2023-09-23T21:15:30.995587+0000","last_peered":"2023-09-23T21:15:30.995587+0000","last_clean":"2023-09-23T21:15:30.995587+0000","last_became_active":"2023-09-23T21:10:26.769326+0000","last_became_peered":"2023-09-23T21:10:26.769326+0000","last_unstale":"2023-09-23T21:15:30.995587+0000","last_undegraded":"2023-09-23T21:15:30.995587+0000","last_fullsized":"2023-09-23T21:15:30.995587+0000","mapping_epoch":423,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":424,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:57:45.219837+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"79'161","reported_seq":731,"reported_epoch":548,"state":"active+clean","last_fresh":"2023-09-23T21:15:31.867419+0000","last_change":"2023-09-23T21:13:33.766895+0000","last_active":"2023-09-23T21:15:31.867419+0000","last_peered":"2023-09-23T21:15:31.867419+0000","last_clean":"2023-09-23T21:15:31.867419+0000","last_became_active":"2023-09-23T21:13:33.766538+0000","last_became_peered":"2023-09-23T21:13:33.766538+0000","last_unstale":"2023-09-23T21:15:31.867419+0000","last_undegraded":"2023-09-23T21:15:31.867419+0000","last_fullsized":"2023-09-23T21:15:31.867419+0000","mapping_epoch":504,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":505,"parent":"0.0","parent_split_bits":0,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:20.738273+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:20.738273+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:35:35.855987+0000","scrub_duration":0.0088336339999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,2],"acting":[0,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":556,"reported_epoch":548,"state":"active+clean","last_fresh":"2023-09-23T21:15:31.869577+0000","last_change":"2023-09-23T21:11:08.760417+0000","last_active":"2023-09-23T21:15:31.869577+0000","last_peered":"2023-09-23T21:15:31.869577+0000","last_clean":"2023-09-23T21:15:31.869577+0000","last_became_active":"2023-09-23T21:11:08.760150+0000","last_became_peered":"2023-09-23T21:11:08.760150+0000","last_unstale":"2023-09-23T21:15:31.869577+0000","last_undegraded":"2023-09-23T21:15:31.869577+0000","last_fullsized":"2023-09-23T21:15:31.869577+0000","mapping_epoch":443,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":444,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:23:10.849768+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":123,"reported_epoch":548,"state":"active+clean","last_fresh":"2023-09-23T21:15:31.214016+0000","last_change":"2023-09-23T21:13:12.216020+0000","last_active":"2023-09-23T21:15:31.214016+0000","last_peered":"2023-09-23T21:15:31.214016+0000","last_clean":"2023-09-23T21:15:31.214016+0000","last_became_active":"2023-09-23T21:13:12.215180+0000","last_became_peered":"2023-09-23T21:13:12.215180+0000","last_unstale":"2023-09-23T21:15:31.214016+0000","last_undegraded":"2023-09-23T21:15:31.214016+0000","last_fullsized":"2023-09-23T21:15:31.214016+0000","mapping_epoch":493,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:26:18.704684+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.9","version":"548'1084","reported_seq":1096,"reported_epoch":548,"state":"active+clean","last_fresh":"2023-09-23T21:15:39.227133+0000","last_change":"2023-09-23T21:15:25.903039+0000","last_active":"2023-09-23T21:15:39.227133+0000","last_peered":"2023-09-23T21:15:39.227133+0000","last_clean":"2023-09-23T21:15:39.227133+0000","last_became_active":"2023-09-23T21:15:25.902593+0000","last_became_peered":"2023-09-23T21:15:25.902593+0000","last_unstale":"2023-09-23T21:15:39.227133+0000","last_undegraded":"2023-09-23T21:15:39.227133+0000","last_fullsized":"2023-09-23T21:15:39.227133+0000","mapping_epoch":544,"log_start":"0'0","ondisk_log_start":"0'0","created":544,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_clean_scrub_stamp":"2023-09-23T21:15:24.860968+0000","objects_scrubbed":0,"log_size":1084,"log_dups_size":0,"ondisk_log_size":1084,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:16:11.481971+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":71041024,"num_objects":1084,"num_object_clones":0,"num_object_copies":3252,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1084,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1084,"num_write_kb":69376,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,7],"acting":[3,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.3","version":"79'161","reported_seq":851,"reported_epoch":548,"state":"active+clean","last_fresh":"2023-09-23T21:15:31.213480+0000","last_change":"2023-09-23T21:13:33.771279+0000","last_active":"2023-09-23T21:15:31.213480+0000","last_peered":"2023-09-23T21:15:31.213480+0000","last_clean":"2023-09-23T21:15:31.213480+0000","last_became_active":"2023-09-23T21:13:33.770980+0000","last_became_peered":"2023-09-23T21:13:33.770980+0000","last_unstale":"2023-09-23T21:15:31.213480+0000","last_undegraded":"2023-09-23T21:15:31.213480+0000","last_fullsized":"2023-09-23T21:15:31.213480+0000","mapping_epoch":504,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":505,"parent":"0.0","parent_split_bits":3,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:05:57.832302+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:05:57.832302+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:19:42.309604+0000","scrub_duration":0.010206847999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":365,"reported_epoch":547,"state":"active+clean","last_fresh":"2023-09-23T21:15:30.995734+0000","last_change":"2023-09-23T21:13:13.212775+0000","last_active":"2023-09-23T21:15:30.995734+0000","last_peered":"2023-09-23T21:15:30.995734+0000","last_clean":"2023-09-23T21:15:30.995734+0000","last_became_active":"2023-09-23T21:13:13.212460+0000","last_became_peered":"2023-09-23T21:13:13.212460+0000","last_unstale":"2023-09-23T21:15:30.995734+0000","last_undegraded":"2023-09-23T21:15:30.995734+0000","last_fullsized":"2023-09-23T21:15:30.995734+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":495,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:27:10.853970+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":547,"reported_epoch":547,"state":"active+clean","last_fresh":"2023-09-23T21:15:30.995667+0000","last_change":"2023-09-23T21:13:44.409212+0000","last_active":"2023-09-23T21:15:30.995667+0000","last_peered":"2023-09-23T21:15:30.995667+0000","last_clean":"2023-09-23T21:15:30.995667+0000","last_became_active":"2023-09-23T21:13:44.408958+0000","last_became_peered":"2023-09-23T21:13:44.408958+0000","last_unstale":"2023-09-23T21:15:30.995667+0000","last_undegraded":"2023-09-23T21:15:30.995667+0000","last_fullsized":"2023-09-23T21:15:30.995667+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:03:52.749964+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":57,"reported_epoch":547,"state":"active+clean","last_fresh":"2023-09-23T21:15:30.993619+0000","last_change":"2023-09-23T21:13:30.804379+0000","last_active":"2023-09-23T21:15:30.993619+0000","last_peered":"2023-09-23T21:15:30.993619+0000","last_clean":"2023-09-23T21:15:30.993619+0000","last_became_active":"2023-09-23T21:13:30.799907+0000","last_became_peered":"2023-09-23T21:13:30.799907+0000","last_unstale":"2023-09-23T21:15:30.993619+0000","last_undegraded":"2023-09-23T21:15:30.993619+0000","last_fullsized":"2023-09-23T21:15:30.993619+0000","mapping_epoch":501,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:49:42.859212+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,7],"acting":[0,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.c","version":"548'1374","reported_seq":1387,"reported_epoch":548,"state":"active+clean","last_fresh":"2023-09-23T21:15:41.316985+0000","last_change":"2023-09-23T21:15:25.887291+0000","last_active":"2023-09-23T21:15:41.316985+0000","last_peered":"2023-09-23T21:15:41.316985+0000","last_clean":"2023-09-23T21:15:41.316985+0000","last_became_active":"2023-09-23T21:15:25.887008+0000","last_became_peered":"2023-09-23T21:15:25.887008+0000","last_unstale":"2023-09-23T21:15:41.316985+0000","last_undegraded":"2023-09-23T21:15:41.316985+0000","last_fullsized":"2023-09-23T21:15:41.316985+0000","mapping_epoch":544,"log_start":"0'0","ondisk_log_start":"0'0","created":544,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_clean_scrub_stamp":"2023-09-23T21:15:24.860968+0000","objects_scrubbed":0,"log_size":1374,"log_dups_size":0,"ondisk_log_size":1374,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:18:00.855026+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90046464,"num_objects":1374,"num_object_clones":0,"num_object_copies":4122,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1374,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1374,"num_write_kb":87936,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,7],"acting":[2,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.6","version":"79'161","reported_seq":466,"reported_epoch":547,"state":"active+clean","last_fresh":"2023-09-23T21:15:30.994174+0000","last_change":"2023-09-23T21:13:33.770715+0000","last_active":"2023-09-23T21:15:30.994174+0000","last_peered":"2023-09-23T21:15:30.994174+0000","last_clean":"2023-09-23T21:15:30.994174+0000","last_became_active":"2023-09-23T21:13:33.769303+0000","last_became_peered":"2023-09-23T21:13:33.769303+0000","last_unstale":"2023-09-23T21:15:30.994174+0000","last_undegraded":"2023-09-23T21:15:30.994174+0000","last_fullsized":"2023-09-23T21:15:30.994174+0000","mapping_epoch":504,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":505,"parent":"0.0","parent_split_bits":3,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:10:24.268662+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:10:24.268662+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13417,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":11,"num_read_kb":20,"num_write":5,"num_write_kb":55,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":40252,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":175,"reported_epoch":548,"state":"active+clean","last_fresh":"2023-09-23T21:15:31.871526+0000","last_change":"2023-09-23T21:13:30.770826+0000","last_active":"2023-09-23T21:15:31.871526+0000","last_peered":"2023-09-23T21:15:31.871526+0000","last_clean":"2023-09-23T21:15:31.871526+0000","last_became_active":"2023-09-23T21:13:30.770512+0000","last_became_peered":"2023-09-23T21:13:30.770512+0000","last_unstale":"2023-09-23T21:15:31.871526+0000","last_undegraded":"2023-09-23T21:15:31.871526+0000","last_fullsized":"2023-09-23T21:15:31.871526+0000","mapping_epoch":501,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:44:23.470538+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":431,"reported_epoch":547,"state":"active+clean","last_fresh":"2023-09-23T21:15:30.994717+0000","last_change":"2023-09-23T21:13:30.790590+0000","last_active":"2023-09-23T21:15:30.994717+0000","last_peered":"2023-09-23T21:15:30.994717+0000","last_clean":"2023-09-23T21:15:30.994717+0000","last_became_active":"2023-09-23T21:13:30.790253+0000","last_became_peered":"2023-09-23T21:13:30.790253+0000","last_unstale":"2023-09-23T21:15:30.994717+0000","last_undegraded":"2023-09-23T21:15:30.994717+0000","last_fullsized":"2023-09-23T21:15:30.994717+0000","mapping_epoch":501,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:23:41.843250+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,0],"acting":[2,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":494,"reported_epoch":547,"state":"active+clean","last_fresh":"2023-09-23T21:15:30.996138+0000","last_change":"2023-09-23T21:13:12.210546+0000","last_active":"2023-09-23T21:15:30.996138+0000","last_peered":"2023-09-23T21:15:30.996138+0000","last_clean":"2023-09-23T21:15:30.996138+0000","last_became_active":"2023-09-23T21:13:12.205934+0000","last_became_peered":"2023-09-23T21:13:12.205934+0000","last_unstale":"2023-09-23T21:15:30.996138+0000","last_undegraded":"2023-09-23T21:15:30.996138+0000","last_fullsized":"2023-09-23T21:15:30.996138+0000","mapping_epoch":493,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:21:18.299428+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.f","version":"548'1332","reported_seq":1345,"reported_epoch":548,"state":"active+clean","last_fresh":"2023-09-23T21:15:41.328368+0000","last_change":"2023-09-23T21:15:25.902442+0000","last_active":"2023-09-23T21:15:41.328368+0000","last_peered":"2023-09-23T21:15:41.328368+0000","last_clean":"2023-09-23T21:15:41.328368+0000","last_became_active":"2023-09-23T21:15:25.902151+0000","last_became_peered":"2023-09-23T21:15:25.902151+0000","last_unstale":"2023-09-23T21:15:41.328368+0000","last_undegraded":"2023-09-23T21:15:41.328368+0000","last_fullsized":"2023-09-23T21:15:41.328368+0000","mapping_epoch":544,"log_start":"0'0","ondisk_log_start":"0'0","created":544,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_clean_scrub_stamp":"2023-09-23T21:15:24.860968+0000","objects_scrubbed":0,"log_size":1332,"log_dups_size":0,"ondisk_log_size":1332,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:32:45.174211+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":87293952,"num_objects":1332,"num_object_clones":0,"num_object_copies":3996,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1332,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1332,"num_write_kb":85248,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,0],"acting":[2,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.5","version":"79'161","reported_seq":586,"reported_epoch":548,"state":"active+clean","last_fresh":"2023-09-23T21:15:41.108695+0000","last_change":"2023-09-23T21:15:41.108638+0000","last_active":"2023-09-23T21:15:41.108695+0000","last_peered":"2023-09-23T21:15:41.108695+0000","last_clean":"2023-09-23T21:15:41.108695+0000","last_became_active":"2023-09-23T21:13:38.373633+0000","last_became_peered":"2023-09-23T21:13:38.373633+0000","last_unstale":"2023-09-23T21:15:41.108695+0000","last_undegraded":"2023-09-23T21:15:41.108695+0000","last_fullsized":"2023-09-23T21:15:41.108695+0000","mapping_epoch":507,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":3,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:15:41.108579+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:15:41.108579+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:21:58.075649+0000","scrub_duration":0.0061511960000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":89,"reported_epoch":547,"state":"active+clean","last_fresh":"2023-09-23T21:15:30.994969+0000","last_change":"2023-09-23T21:13:12.235743+0000","last_active":"2023-09-23T21:15:30.994969+0000","last_peered":"2023-09-23T21:15:30.994969+0000","last_clean":"2023-09-23T21:15:30.994969+0000","last_became_active":"2023-09-23T21:13:12.216241+0000","last_became_peered":"2023-09-23T21:13:12.216241+0000","last_unstale":"2023-09-23T21:15:30.994969+0000","last_undegraded":"2023-09-23T21:15:30.994969+0000","last_fullsized":"2023-09-23T21:15:30.994969+0000","mapping_epoch":493,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:12:19.090710+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":529,"reported_epoch":548,"state":"active+clean","last_fresh":"2023-09-23T21:15:31.214174+0000","last_change":"2023-09-23T21:13:30.807520+0000","last_active":"2023-09-23T21:15:31.214174+0000","last_peered":"2023-09-23T21:15:31.214174+0000","last_clean":"2023-09-23T21:15:31.214174+0000","last_became_active":"2023-09-23T21:13:30.807222+0000","last_became_peered":"2023-09-23T21:13:30.807222+0000","last_unstale":"2023-09-23T21:15:31.214174+0000","last_undegraded":"2023-09-23T21:15:31.214174+0000","last_fullsized":"2023-09-23T21:15:31.214174+0000","mapping_epoch":501,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:40:31.989749+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":389,"reported_epoch":547,"state":"active+clean","last_fresh":"2023-09-23T21:15:30.965675+0000","last_change":"2023-09-23T21:13:30.794095+0000","last_active":"2023-09-23T21:15:30.965675+0000","last_peered":"2023-09-23T21:15:30.965675+0000","last_clean":"2023-09-23T21:15:30.965675+0000","last_became_active":"2023-09-23T21:13:30.793791+0000","last_became_peered":"2023-09-23T21:13:30.793791+0000","last_unstale":"2023-09-23T21:15:30.965675+0000","last_undegraded":"2023-09-23T21:15:30.965675+0000","last_fullsized":"2023-09-23T21:15:30.965675+0000","mapping_epoch":501,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:36:28.707362+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.e","version":"549'1388","reported_seq":1401,"reported_epoch":549,"state":"active+clean","last_fresh":"2023-09-23T21:15:42.043769+0000","last_change":"2023-09-23T21:15:25.902844+0000","last_active":"2023-09-23T21:15:42.043769+0000","last_peered":"2023-09-23T21:15:42.043769+0000","last_clean":"2023-09-23T21:15:42.043769+0000","last_became_active":"2023-09-23T21:15:25.902523+0000","last_became_peered":"2023-09-23T21:15:25.902523+0000","last_unstale":"2023-09-23T21:15:42.043769+0000","last_undegraded":"2023-09-23T21:15:42.043769+0000","last_fullsized":"2023-09-23T21:15:42.043769+0000","mapping_epoch":544,"log_start":"0'0","ondisk_log_start":"0'0","created":544,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_clean_scrub_stamp":"2023-09-23T21:15:24.860968+0000","objects_scrubbed":0,"log_size":1388,"log_dups_size":0,"ondisk_log_size":1388,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:34:03.916540+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90963968,"num_objects":1388,"num_object_clones":0,"num_object_copies":4164,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1388,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1388,"num_write_kb":88832,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.4","version":"79'161","reported_seq":919,"reported_epoch":547,"state":"active+clean","last_fresh":"2023-09-23T21:15:30.966676+0000","last_change":"2023-09-23T21:10:29.807785+0000","last_active":"2023-09-23T21:15:30.966676+0000","last_peered":"2023-09-23T21:15:30.966676+0000","last_clean":"2023-09-23T21:15:30.966676+0000","last_became_active":"2023-09-23T21:10:29.807366+0000","last_became_peered":"2023-09-23T21:10:29.807366+0000","last_unstale":"2023-09-23T21:15:30.966676+0000","last_undegraded":"2023-09-23T21:15:30.966676+0000","last_fullsized":"2023-09-23T21:15:30.966676+0000","mapping_epoch":426,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":427,"parent":"0.0","parent_split_bits":3,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:20.012764+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:20.012764+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:24:45.763834+0000","scrub_duration":0.0045251919999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":487,"reported_epoch":547,"state":"active+clean","last_fresh":"2023-09-23T21:15:30.996124+0000","last_change":"2023-09-23T21:10:26.768364+0000","last_active":"2023-09-23T21:15:30.996124+0000","last_peered":"2023-09-23T21:15:30.996124+0000","last_clean":"2023-09-23T21:15:30.996124+0000","last_became_active":"2023-09-23T21:10:26.768052+0000","last_became_peered":"2023-09-23T21:10:26.768052+0000","last_unstale":"2023-09-23T21:15:30.996124+0000","last_undegraded":"2023-09-23T21:15:30.996124+0000","last_fullsized":"2023-09-23T21:15:30.996124+0000","mapping_epoch":423,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":424,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:22:42.201200+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,3],"acting":[2,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.6","version":"70'1","reported_seq":518,"reported_epoch":547,"state":"active+clean","last_fresh":"2023-09-23T21:15:30.993469+0000","last_change":"2023-09-23T21:13:30.793290+0000","last_active":"2023-09-23T21:15:30.993469+0000","last_peered":"2023-09-23T21:15:30.993469+0000","last_clean":"2023-09-23T21:15:30.993469+0000","last_became_active":"2023-09-23T21:13:30.786079+0000","last_became_peered":"2023-09-23T21:13:30.786079+0000","last_unstale":"2023-09-23T21:15:30.993469+0000","last_undegraded":"2023-09-23T21:15:30.993469+0000","last_fullsized":"2023-09-23T21:15:30.993469+0000","mapping_epoch":501,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:34:43.281095+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":123,"reported_epoch":548,"state":"active+clean","last_fresh":"2023-09-23T21:15:31.213331+0000","last_change":"2023-09-23T21:13:12.216895+0000","last_active":"2023-09-23T21:15:31.213331+0000","last_peered":"2023-09-23T21:15:31.213331+0000","last_clean":"2023-09-23T21:15:31.213331+0000","last_became_active":"2023-09-23T21:13:12.216654+0000","last_became_peered":"2023-09-23T21:13:12.216654+0000","last_unstale":"2023-09-23T21:15:31.213331+0000","last_undegraded":"2023-09-23T21:15:31.213331+0000","last_fullsized":"2023-09-23T21:15:31.213331+0000","mapping_epoch":493,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:02:40.430096+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.19","version":"79'161","reported_seq":709,"reported_epoch":547,"state":"active+clean","last_fresh":"2023-09-23T21:15:30.995279+0000","last_change":"2023-09-23T21:15:23.795956+0000","last_active":"2023-09-23T21:15:30.995279+0000","last_peered":"2023-09-23T21:15:30.995279+0000","last_clean":"2023-09-23T21:15:30.995279+0000","last_became_active":"2023-09-23T21:15:23.795663+0000","last_became_peered":"2023-09-23T21:15:23.795663+0000","last_unstale":"2023-09-23T21:15:30.995279+0000","last_undegraded":"2023-09-23T21:15:30.995279+0000","last_fullsized":"2023-09-23T21:15:30.995279+0000","mapping_epoch":542,"log_start":"0'0","ondisk_log_start":"0'0","created":479,"last_epoch_clean":543,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:10:37.696951+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:10:37.696951+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.001386504,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":57,"reported_epoch":547,"state":"active+clean","last_fresh":"2023-09-23T21:15:30.995591+0000","last_change":"2023-09-23T21:13:30.791297+0000","last_active":"2023-09-23T21:15:30.995591+0000","last_peered":"2023-09-23T21:15:30.995591+0000","last_clean":"2023-09-23T21:15:30.995591+0000","last_became_active":"2023-09-23T21:13:30.788721+0000","last_became_peered":"2023-09-23T21:13:30.788721+0000","last_unstale":"2023-09-23T21:15:30.995591+0000","last_undegraded":"2023-09-23T21:15:30.995591+0000","last_fullsized":"2023-09-23T21:15:30.995591+0000","mapping_epoch":501,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:31:18.147883+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":51,"reported_epoch":547,"state":"active+clean","last_fresh":"2023-09-23T21:15:30.995463+0000","last_change":"2023-09-23T21:13:38.373814+0000","last_active":"2023-09-23T21:15:30.995463+0000","last_peered":"2023-09-23T21:15:30.995463+0000","last_clean":"2023-09-23T21:15:30.995463+0000","last_became_active":"2023-09-23T21:13:38.373458+0000","last_became_peered":"2023-09-23T21:13:38.373458+0000","last_unstale":"2023-09-23T21:15:30.995463+0000","last_undegraded":"2023-09-23T21:15:30.995463+0000","last_fullsized":"2023-09-23T21:15:30.995463+0000","mapping_epoch":507,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:23:08.288401+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.1e","version":"79'161","reported_seq":484,"reported_epoch":547,"state":"active+clean","last_fresh":"2023-09-23T21:15:30.966637+0000","last_change":"2023-09-23T21:15:20.791055+0000","last_active":"2023-09-23T21:15:30.966637+0000","last_peered":"2023-09-23T21:15:30.966637+0000","last_clean":"2023-09-23T21:15:30.966637+0000","last_became_active":"2023-09-23T21:15:20.790772+0000","last_became_peered":"2023-09-23T21:15:20.790772+0000","last_unstale":"2023-09-23T21:15:30.966637+0000","last_undegraded":"2023-09-23T21:15:30.966637+0000","last_fullsized":"2023-09-23T21:15:30.966637+0000","mapping_epoch":539,"log_start":"0'0","ondisk_log_start":"0'0","created":479,"last_epoch_clean":540,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:10:39.567232+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:10:39.567232+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.024789746000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,4],"acting":[3,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":130,"reported_epoch":548,"state":"active+clean","last_fresh":"2023-09-23T21:15:31.213804+0000","last_change":"2023-09-23T21:15:06.721854+0000","last_active":"2023-09-23T21:15:31.213804+0000","last_peered":"2023-09-23T21:15:31.213804+0000","last_clean":"2023-09-23T21:15:31.213804+0000","last_became_active":"2023-09-23T21:12:14.209408+0000","last_became_peered":"2023-09-23T21:12:14.209408+0000","last_unstale":"2023-09-23T21:15:31.213804+0000","last_undegraded":"2023-09-23T21:15:31.213804+0000","last_fullsized":"2023-09-23T21:15:31.213804+0000","mapping_epoch":470,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":532,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:45:40.142413+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,7],"acting":[4,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":113,"reported_epoch":548,"state":"active+clean","last_fresh":"2023-09-23T21:15:31.213626+0000","last_change":"2023-09-23T21:12:10.770474+0000","last_active":"2023-09-23T21:15:31.213626+0000","last_peered":"2023-09-23T21:15:31.213626+0000","last_clean":"2023-09-23T21:15:31.213626+0000","last_became_active":"2023-09-23T21:12:10.770184+0000","last_became_peered":"2023-09-23T21:12:10.770184+0000","last_unstale":"2023-09-23T21:15:31.213626+0000","last_undegraded":"2023-09-23T21:15:31.213626+0000","last_fullsized":"2023-09-23T21:15:31.213626+0000","mapping_epoch":470,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:05:09.407151+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":338,"reported_epoch":547,"state":"active+clean","last_fresh":"2023-09-23T21:15:30.993695+0000","last_change":"2023-09-23T21:12:10.771160+0000","last_active":"2023-09-23T21:15:30.993695+0000","last_peered":"2023-09-23T21:15:30.993695+0000","last_clean":"2023-09-23T21:15:30.993695+0000","last_became_active":"2023-09-23T21:12:10.757808+0000","last_became_peered":"2023-09-23T21:12:10.757808+0000","last_unstale":"2023-09-23T21:15:30.993695+0000","last_undegraded":"2023-09-23T21:15:30.993695+0000","last_fullsized":"2023-09-23T21:15:30.993695+0000","mapping_epoch":470,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:30:50.987139+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":123,"reported_epoch":548,"state":"active+clean","last_fresh":"2023-09-23T21:15:31.213473+0000","last_change":"2023-09-23T21:13:12.216602+0000","last_active":"2023-09-23T21:15:31.213473+0000","last_peered":"2023-09-23T21:15:31.213473+0000","last_clean":"2023-09-23T21:15:31.213473+0000","last_became_active":"2023-09-23T21:13:12.216204+0000","last_became_peered":"2023-09-23T21:13:12.216204+0000","last_unstale":"2023-09-23T21:15:31.213473+0000","last_undegraded":"2023-09-23T21:15:31.213473+0000","last_fullsized":"2023-09-23T21:15:31.213473+0000","mapping_epoch":493,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:23:37.043390+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1c","version":"79'161","reported_seq":931,"reported_epoch":547,"state":"active+clean","last_fresh":"2023-09-23T21:15:30.966600+0000","last_change":"2023-09-23T21:15:23.800688+0000","last_active":"2023-09-23T21:15:30.966600+0000","last_peered":"2023-09-23T21:15:30.966600+0000","last_clean":"2023-09-23T21:15:30.966600+0000","last_became_active":"2023-09-23T21:15:23.800361+0000","last_became_peered":"2023-09-23T21:15:23.800361+0000","last_unstale":"2023-09-23T21:15:30.966600+0000","last_undegraded":"2023-09-23T21:15:30.966600+0000","last_fullsized":"2023-09-23T21:15:30.966600+0000","mapping_epoch":542,"log_start":"0'0","ondisk_log_start":"0'0","created":479,"last_epoch_clean":543,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:22.040907+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:22.040907+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.013021176000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":163840,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":163840,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":137,"reported_epoch":547,"state":"active+clean","last_fresh":"2023-09-23T21:15:30.993323+0000","last_change":"2023-09-23T21:12:10.779066+0000","last_active":"2023-09-23T21:15:30.993323+0000","last_peered":"2023-09-23T21:15:30.993323+0000","last_clean":"2023-09-23T21:15:30.993323+0000","last_became_active":"2023-09-23T21:12:10.778691+0000","last_became_peered":"2023-09-23T21:12:10.778691+0000","last_unstale":"2023-09-23T21:15:30.993323+0000","last_undegraded":"2023-09-23T21:15:30.993323+0000","last_fullsized":"2023-09-23T21:15:30.993323+0000","mapping_epoch":470,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:07:22.043268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,7],"acting":[2,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":216,"reported_epoch":547,"state":"active+clean","last_fresh":"2023-09-23T21:15:30.994358+0000","last_change":"2023-09-23T21:13:12.200767+0000","last_active":"2023-09-23T21:15:30.994358+0000","last_peered":"2023-09-23T21:15:30.994358+0000","last_clean":"2023-09-23T21:15:30.994358+0000","last_became_active":"2023-09-23T21:13:12.200435+0000","last_became_peered":"2023-09-23T21:13:12.200435+0000","last_unstale":"2023-09-23T21:15:30.994358+0000","last_undegraded":"2023-09-23T21:15:30.994358+0000","last_fullsized":"2023-09-23T21:15:30.994358+0000","mapping_epoch":493,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:15:44.937886+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.1d","version":"79'161","reported_seq":679,"reported_epoch":547,"state":"active+clean","last_fresh":"2023-09-23T21:15:30.966638+0000","last_change":"2023-09-23T21:15:23.802125+0000","last_active":"2023-09-23T21:15:30.966638+0000","last_peered":"2023-09-23T21:15:30.966638+0000","last_clean":"2023-09-23T21:15:30.966638+0000","last_became_active":"2023-09-23T21:15:23.801704+0000","last_became_peered":"2023-09-23T21:15:23.801704+0000","last_unstale":"2023-09-23T21:15:30.966638+0000","last_undegraded":"2023-09-23T21:15:30.966638+0000","last_fullsized":"2023-09-23T21:15:30.966638+0000","mapping_epoch":542,"log_start":"0'0","ondisk_log_start":"0'0","created":479,"last_epoch_clean":543,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:09:36.858077+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:09:36.858077+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.010304610000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,2],"acting":[3,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":75,"reported_epoch":548,"state":"active+clean","last_fresh":"2023-09-23T21:15:31.868094+0000","last_change":"2023-09-23T21:13:38.380574+0000","last_active":"2023-09-23T21:15:31.868094+0000","last_peered":"2023-09-23T21:15:31.868094+0000","last_clean":"2023-09-23T21:15:31.868094+0000","last_became_active":"2023-09-23T21:13:38.373252+0000","last_became_peered":"2023-09-23T21:13:38.373252+0000","last_unstale":"2023-09-23T21:15:31.868094+0000","last_undegraded":"2023-09-23T21:15:31.868094+0000","last_fullsized":"2023-09-23T21:15:31.868094+0000","mapping_epoch":507,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:58:14.788031+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":11,"num_bytes_recovered":3324,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":123,"reported_epoch":548,"state":"active+clean","last_fresh":"2023-09-23T21:15:31.213248+0000","last_change":"2023-09-23T21:12:10.793610+0000","last_active":"2023-09-23T21:15:31.213248+0000","last_peered":"2023-09-23T21:15:31.213248+0000","last_clean":"2023-09-23T21:15:31.213248+0000","last_became_active":"2023-09-23T21:12:10.793081+0000","last_became_peered":"2023-09-23T21:12:10.793081+0000","last_unstale":"2023-09-23T21:15:31.213248+0000","last_undegraded":"2023-09-23T21:15:31.213248+0000","last_fullsized":"2023-09-23T21:15:31.213248+0000","mapping_epoch":470,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:34:36.291550+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":11,"num_pg":16,"stat_sum":{"num_bytes":1356660736,"num_objects":20701,"num_object_clones":0,"num_object_copies":62103,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":20701,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":20701,"num_write_kb":1324864,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":4124180480,"data_stored":4124180480,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":20701,"ondisk_log_size":20701,"up":48,"acting":48,"num_store_stats":6},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":60,"num_bytes_recovered":8898,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":118784,"data_stored":21350,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":14322,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":31,"stat_sum":{"num_bytes":807878,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":45,"num_read_kb":83,"num_write":22,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":324849,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2109440,"data_stored":2098784,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4991,"ondisk_log_size":4991,"up":93,"acting":93,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201703,"num_pgs":63,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1242276,"kb_used_data":718556,"kb_used_omap":0,"kb_used_meta":523712,"kb_avail":92502876,"statfs":{"total":95995035648,"available":94722945024,"internally_reserved":0,"allocated":735801344,"data_stored":734228772,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":536281088},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,2,0,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:15:15 2023","interfaces":[{"interface":"back","average":{"1min":1.3440000000000001,"5min":2.2090000000000001,"15min":2.3530000000000002},"min":{"1min":0.25,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":4.3070000000000004,"5min":37.207999999999998,"15min":37.207999999999998},"last":1.256},{"interface":"front","average":{"1min":2.089,"5min":2.2050000000000001,"15min":2.2240000000000002},"min":{"1min":0.253,"5min":0.253,"15min":0.253},"max":{"1min":10.429,"5min":37.122,"15min":37.122},"last":2.7730000000000001}]},{"osd":1,"last update":"Sat Sep 23 21:15:09 2023","interfaces":[{"interface":"back","average":{"1min":2.0329999999999999,"5min":1.3320000000000001,"15min":2.0289999999999999},"min":{"1min":0.28000000000000003,"5min":0.27500000000000002,"15min":0.26700000000000002},"max":{"1min":6.0860000000000003,"5min":14.034000000000001,"15min":14.034000000000001},"last":1.4399999999999999},{"interface":"front","average":{"1min":1.554,"5min":1.4359999999999999,"15min":1.9079999999999999},"min":{"1min":0.33900000000000002,"5min":0.29899999999999999,"15min":0.26300000000000001},"max":{"1min":4.2690000000000001,"5min":11.226000000000001,"15min":11.226000000000001},"last":1.3149999999999999}]},{"osd":2,"last update":"Sat Sep 23 21:15:32 2023","interfaces":[{"interface":"back","average":{"1min":2.2799999999999998,"5min":1.361,"15min":1.6879999999999999},"min":{"1min":0.29799999999999999,"5min":0.26400000000000001,"15min":0.26200000000000001},"max":{"1min":12.042,"5min":12.042,"15min":14.478999999999999},"last":0.45500000000000002},{"interface":"front","average":{"1min":1.25,"5min":1.391,"15min":1.6599999999999999},"min":{"1min":0.36099999999999999,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":4.1349999999999998,"5min":11.198,"15min":14.135},"last":1.286}]},{"osd":3,"last update":"Sat Sep 23 21:14:58 2023","interfaces":[{"interface":"back","average":{"1min":1.7270000000000001,"5min":1.3009999999999999,"15min":2},"min":{"1min":0.30599999999999999,"5min":0.30599999999999999,"15min":0.29099999999999998},"max":{"1min":4.1619999999999999,"5min":7.5190000000000001,"15min":31.786000000000001},"last":0.374},{"interface":"front","average":{"1min":2.7410000000000001,"5min":1.4890000000000001,"15min":2.0390000000000001},"min":{"1min":0.35599999999999998,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":11.743,"5min":11.743,"15min":32.429000000000002},"last":0.84899999999999998}]},{"osd":4,"last update":"Sat Sep 23 21:14:58 2023","interfaces":[{"interface":"back","average":{"1min":1.1100000000000001,"5min":1.48,"15min":1.143},"min":{"1min":0.32500000000000001,"5min":0.26200000000000001,"15min":0.248},"max":{"1min":4.2130000000000001,"5min":11.297000000000001,"15min":11.297000000000001},"last":1.1919999999999999},{"interface":"front","average":{"1min":1.085,"5min":1.24,"15min":1.105},"min":{"1min":0.29699999999999999,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":3.7850000000000001,"5min":11.093999999999999,"15min":12.752000000000001},"last":0.44600000000000001}]},{"osd":5,"last update":"Sat Sep 23 21:14:58 2023","interfaces":[{"interface":"back","average":{"1min":1.359,"5min":1.1990000000000001,"15min":1.075},"min":{"1min":0.32300000000000001,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":6.133,"5min":9.4789999999999992,"15min":9.4789999999999992},"last":0.42699999999999999},{"interface":"front","average":{"1min":0.98699999999999999,"5min":1.3360000000000001,"15min":1.1279999999999999},"min":{"1min":0.33300000000000002,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":4.181,"5min":11.257,"15min":11.257},"last":1.226}]},{"osd":6,"last update":"Sat Sep 23 21:15:09 2023","interfaces":[{"interface":"back","average":{"1min":1.325,"5min":1.3029999999999999,"15min":1.095},"min":{"1min":0.40200000000000002,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":7.5339999999999998,"5min":10.941000000000001,"15min":10.941000000000001},"last":0.81499999999999995},{"interface":"front","average":{"1min":1.298,"5min":1.2130000000000001,"15min":1.1799999999999999},"min":{"1min":0.39200000000000002,"5min":0.33100000000000002,"15min":0.32300000000000001},"max":{"1min":4.7720000000000002,"5min":8.923,"15min":12.715},"last":0.78000000000000003}]}]},{"osd":6,"up_from":0,"seq":704374636786,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":1,"up_from":0,"seq":1503238553718,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":0,"up_from":493,"seq":2117418876959,"num_pgs":78,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1116448,"kb_used_data":894168,"kb_used_omap":36,"kb_used_meta":222235,"kb_avail":92628704,"statfs":{"total":95995035648,"available":94851792896,"internally_reserved":0,"allocated":915628032,"data_stored":913081001,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":37336,"internal_metadata":227569192},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[4,0,6,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 21:15:17 2023","interfaces":[{"interface":"back","average":{"1min":1.6990000000000001,"5min":0.94399999999999995,"15min":0.81799999999999995},"min":{"1min":0.22500000000000001,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":4.665,"5min":4.9000000000000004,"15min":4.9000000000000004},"last":1.0580000000000001},{"interface":"front","average":{"1min":1.5620000000000001,"5min":0.92900000000000005,"15min":0.82299999999999995},"min":{"1min":0.20799999999999999,"5min":0.20799999999999999,"15min":0.20799999999999999},"max":{"1min":11.964,"5min":11.964,"15min":11.964},"last":1.675}]},{"osd":2,"last update":"Sat Sep 23 21:15:17 2023","interfaces":[{"interface":"back","average":{"1min":1.661,"5min":0.85499999999999998,"15min":0.72099999999999997},"min":{"1min":0.29399999999999998,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":7.133,"5min":7.133,"15min":7.133},"last":1.099},{"interface":"front","average":{"1min":1.647,"5min":1.103,"15min":1.0129999999999999},"min":{"1min":0.27900000000000003,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":6.6470000000000002,"5min":6.6470000000000002,"15min":6.6470000000000002},"last":2.6429999999999998}]},{"osd":3,"last update":"Sat Sep 23 21:15:17 2023","interfaces":[{"interface":"back","average":{"1min":1.8939999999999999,"5min":1.073,"15min":0.93600000000000005},"min":{"1min":0.27300000000000002,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":6.2060000000000004,"5min":6.2060000000000004,"15min":6.2060000000000004},"last":1.7370000000000001},{"interface":"front","average":{"1min":1.4199999999999999,"5min":1.1779999999999999,"15min":1.1379999999999999},"min":{"1min":0.28299999999999997,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":4.8040000000000003,"5min":4.8040000000000003,"15min":4.8040000000000003},"last":0.56299999999999994}]},{"osd":4,"last update":"Sat Sep 23 21:15:17 2023","interfaces":[{"interface":"back","average":{"1min":1.2789999999999999,"5min":1.1299999999999999,"15min":1.105},"min":{"1min":0.312,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":4.4409999999999998,"5min":4.4409999999999998,"15min":4.4409999999999998},"last":0.65900000000000003},{"interface":"front","average":{"1min":1.4570000000000001,"5min":1.0880000000000001,"15min":1.026},"min":{"1min":0.41699999999999998,"5min":0.317,"15min":0.317},"max":{"1min":4.6929999999999996,"5min":5.093,"15min":5.093},"last":1.159}]},{"osd":5,"last update":"Sat Sep 23 21:15:17 2023","interfaces":[{"interface":"back","average":{"1min":1.413,"5min":1.2709999999999999,"15min":1.2470000000000001},"min":{"1min":0.29999999999999999,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":4.0869999999999997,"5min":4.5640000000000001,"15min":4.5640000000000001},"last":1.764},{"interface":"front","average":{"1min":1.3959999999999999,"5min":1.1299999999999999,"15min":1.0860000000000001},"min":{"1min":0.313,"5min":0.313,"15min":0.313},"max":{"1min":7.4569999999999999,"5min":7.4569999999999999,"15min":7.4569999999999999},"last":0.71499999999999997}]},{"osd":6,"last update":"Sat Sep 23 21:15:17 2023","interfaces":[{"interface":"back","average":{"1min":1.8420000000000001,"5min":1.27,"15min":1.1739999999999999},"min":{"1min":0.46600000000000003,"5min":0.38100000000000001,"15min":0.38100000000000001},"max":{"1min":7.2050000000000001,"5min":7.2050000000000001,"15min":7.2050000000000001},"last":1.1910000000000001},{"interface":"front","average":{"1min":1.5640000000000001,"5min":1.359,"15min":1.325},"min":{"1min":0.39600000000000002,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":4.7850000000000001,"5min":5.0860000000000003,"15min":5.0860000000000003},"last":1.71}]},{"osd":7,"last update":"Sat Sep 23 21:15:17 2023","interfaces":[{"interface":"back","average":{"1min":2.1600000000000001,"5min":1.399,"15min":1.272},"min":{"1min":0.40899999999999997,"5min":0.40899999999999997,"15min":0.40899999999999997},"max":{"1min":7.5810000000000004,"5min":7.5810000000000004,"15min":7.5810000000000004},"last":0.68899999999999995},{"interface":"front","average":{"1min":1.6850000000000001,"5min":1.2629999999999999,"15min":1.1930000000000001},"min":{"1min":0.56000000000000005,"5min":0.441,"15min":0.441},"max":{"1min":4.6349999999999998,"5min":5.1289999999999996,"15min":5.1289999999999996},"last":1.1299999999999999}]}]},{"osd":2,"up_from":93,"seq":399431958831,"num_pgs":49,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1662120,"kb_used_data":820256,"kb_used_omap":81,"kb_used_meta":841774,"kb_avail":92083032,"statfs":{"total":95995035648,"available":94293024768,"internally_reserved":0,"allocated":839942144,"data_stored":837386034,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":83750,"internal_metadata":861976794},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,1],"upper_bound":4},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:15:17 2023","interfaces":[{"interface":"back","average":{"1min":2.2210000000000001,"5min":1.1100000000000001,"15min":0.92500000000000004},"min":{"1min":0.20899999999999999,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":9.9329999999999998,"5min":9.9329999999999998,"15min":9.9329999999999998},"last":1.5920000000000001},{"interface":"front","average":{"1min":2.6659999999999999,"5min":1.865,"15min":1.7310000000000001},"min":{"1min":0.24399999999999999,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":11.269,"5min":18.111999999999998,"15min":18.111999999999998},"last":0.46500000000000002}]},{"osd":1,"last update":"Sat Sep 23 21:15:17 2023","interfaces":[{"interface":"back","average":{"1min":2.0539999999999998,"5min":1.2629999999999999,"15min":2.472},"min":{"1min":0.20599999999999999,"5min":0.188,"15min":0.188},"max":{"1min":9.9890000000000008,"5min":11.005000000000001,"15min":34.396999999999998},"last":2.2530000000000001},{"interface":"front","average":{"1min":2.2280000000000002,"5min":1.296,"15min":2.6859999999999999},"min":{"1min":0.26200000000000001,"5min":0.217,"15min":0.217},"max":{"1min":16.274000000000001,"5min":18.081,"15min":34.462000000000003},"last":0.52000000000000002}]},{"osd":3,"last update":"Sat Sep 23 21:14:58 2023","interfaces":[{"interface":"back","average":{"1min":2.246,"5min":1.274,"15min":2.4550000000000001},"min":{"1min":0.37,"5min":0.20599999999999999,"15min":0.20599999999999999},"max":{"1min":14.442,"5min":14.442,"15min":41.057000000000002},"last":0.35899999999999999},{"interface":"front","average":{"1min":2.633,"5min":1.4419999999999999,"15min":2.6970000000000001},"min":{"1min":0.28599999999999998,"5min":0.21299999999999999,"15min":0.21299999999999999},"max":{"1min":14.622,"5min":14.622,"15min":41.012},"last":1.6910000000000001}]},{"osd":4,"last update":"Sat Sep 23 21:15:17 2023","interfaces":[{"interface":"back","average":{"1min":1.5780000000000001,"5min":1.2749999999999999,"15min":1.4219999999999999},"min":{"1min":0.32500000000000001,"5min":0.249,"15min":0.249},"max":{"1min":7.9770000000000003,"5min":10.266999999999999,"15min":10.266999999999999},"last":0.41999999999999998},{"interface":"front","average":{"1min":1.74,"5min":1.4419999999999999,"15min":1.8029999999999999},"min":{"1min":0.308,"5min":0.29399999999999998,"15min":0.26700000000000002},"max":{"1min":5.556,"5min":7.5960000000000001,"15min":14.039999999999999},"last":1.425}]},{"osd":5,"last update":"Sat Sep 23 21:15:17 2023","interfaces":[{"interface":"back","average":{"1min":1.1850000000000001,"5min":1.218,"15min":1.5049999999999999},"min":{"1min":0.34699999999999998,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":3.7250000000000001,"5min":9.5039999999999996,"15min":10.069000000000001},"last":0.626},{"interface":"front","average":{"1min":1.6779999999999999,"5min":1.2849999999999999,"15min":1.7529999999999999},"min":{"1min":0.33800000000000002,"5min":0.29699999999999999,"15min":0.29399999999999998},"max":{"1min":5.8330000000000002,"5min":7.5549999999999997,"15min":13.98},"last":1.51}]},{"osd":6,"last update":"Sat Sep 23 21:15:17 2023","interfaces":[{"interface":"back","average":{"1min":1.9239999999999999,"5min":1.571,"15min":1.988},"min":{"1min":0.46300000000000002,"5min":0.373,"15min":0.35699999999999998},"max":{"1min":7.8559999999999999,"5min":7.8559999999999999,"15min":14.016},"last":2.7789999999999999},{"interface":"front","average":{"1min":1.7330000000000001,"5min":1.7330000000000001,"15min":2.157},"min":{"1min":0.377,"5min":0.377,"15min":0.35599999999999998},"max":{"1min":5.649,"5min":9.3469999999999995,"15min":14.068},"last":1.5609999999999999}]},{"osd":7,"last update":"Sat Sep 23 21:15:17 2023","interfaces":[{"interface":"back","average":{"1min":1.6839999999999999,"5min":1.462,"15min":1.6040000000000001},"min":{"1min":0.33700000000000002,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":6.0510000000000002,"5min":18.021999999999998,"15min":18.021999999999998},"last":0.64900000000000002},{"interface":"front","average":{"1min":1.359,"5min":1.425,"15min":1.6910000000000001},"min":{"1min":0.41899999999999998,"5min":0.34799999999999998,"15min":0.32200000000000001},"max":{"1min":3.7549999999999999,"5min":10.359,"15min":10.92},"last":1.139}]}]},{"osd":3,"up_from":98,"seq":420906795304,"num_pgs":73,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1900936,"kb_used_data":1004480,"kb_used_omap":7,"kb_used_meta":896440,"kb_avail":91844216,"statfs":{"total":95995035648,"available":94048477184,"internally_reserved":0,"allocated":1028587520,"data_stored":1026057646,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7576,"internal_metadata":917955176},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,1,2,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:15:13 2023","interfaces":[{"interface":"back","average":{"1min":10.904,"5min":2.4510000000000001,"15min":1.042},"min":{"1min":0.28499999999999998,"5min":0.20499999999999999,"15min":0.20499999999999999},"max":{"1min":147.38900000000001,"5min":147.38900000000001,"15min":147.38900000000001},"last":2.1230000000000002},{"interface":"front","average":{"1min":10.178000000000001,"5min":2.5699999999999998,"15min":1.302},"min":{"1min":0.23100000000000001,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":149.989,"5min":149.989,"15min":149.989},"last":1.4330000000000001}]},{"osd":1,"last update":"Sat Sep 23 21:15:13 2023","interfaces":[{"interface":"back","average":{"1min":1.5589999999999999,"5min":1.2,"15min":2.1880000000000002},"min":{"1min":0.23699999999999999,"5min":0.20000000000000001,"15min":0.20000000000000001},"max":{"1min":5.0700000000000003,"5min":22.577999999999999,"15min":22.577999999999999},"last":0.84499999999999997},{"interface":"front","average":{"1min":1.5600000000000001,"5min":1.345,"15min":2.1389999999999998},"min":{"1min":0.26100000000000001,"5min":0.216,"15min":0.216},"max":{"1min":5.7329999999999997,"5min":21.617999999999999,"15min":21.617999999999999},"last":1.101}]},{"osd":2,"last update":"Sat Sep 23 21:15:00 2023","interfaces":[{"interface":"back","average":{"1min":1.78,"5min":1.9419999999999999,"15min":2.468},"min":{"1min":0.23899999999999999,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":10.946,"5min":56.317999999999998,"15min":56.317999999999998},"last":0.497},{"interface":"front","average":{"1min":1.554,"5min":1.978,"15min":2.5609999999999999},"min":{"1min":0.246,"5min":0.216,"15min":0.216},"max":{"1min":6.29,"5min":55.988,"15min":55.988},"last":0.58099999999999996}]},{"osd":4,"last update":"Sat Sep 23 21:15:00 2023","interfaces":[{"interface":"back","average":{"1min":2.2959999999999998,"5min":1.526,"15min":1.831},"min":{"1min":0.29899999999999999,"5min":0.29899999999999999,"15min":0.28399999999999997},"max":{"1min":10.807,"5min":21.478999999999999,"15min":21.478999999999999},"last":0.67700000000000005},{"interface":"front","average":{"1min":1.415,"5min":1.4830000000000001,"15min":1.9710000000000001},"min":{"1min":0.314,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":6.3499999999999996,"5min":22.402000000000001,"15min":22.402000000000001},"last":0.68700000000000006}]},{"osd":5,"last update":"Sat Sep 23 21:15:00 2023","interfaces":[{"interface":"back","average":{"1min":1.925,"5min":1.456,"15min":1.95},"min":{"1min":0.34599999999999997,"5min":0.311,"15min":0.311},"max":{"1min":10.882999999999999,"5min":21.562000000000001,"15min":21.562000000000001},"last":1.4670000000000001},{"interface":"front","average":{"1min":1.429,"5min":1.302,"15min":2.141},"min":{"1min":0.34799999999999998,"5min":0.31,"15min":0.31},"max":{"1min":4.2779999999999996,"5min":8.8450000000000006,"15min":14.029},"last":1.069}]},{"osd":6,"last update":"Sat Sep 23 21:15:17 2023","interfaces":[{"interface":"back","average":{"1min":1.194,"5min":1.3919999999999999,"15min":1.9930000000000001},"min":{"1min":0.35899999999999999,"5min":0.34999999999999998,"15min":0.33900000000000002},"max":{"1min":3.536,"5min":22.446999999999999,"15min":22.446999999999999},"last":1.004},{"interface":"front","average":{"1min":1.518,"5min":1.7969999999999999,"15min":2.1230000000000002},"min":{"1min":0.378,"5min":0.378,"15min":0.373},"max":{"1min":4.3760000000000003,"5min":12.375999999999999,"15min":13.576000000000001},"last":1.177}]},{"osd":7,"last update":"Sat Sep 23 21:15:00 2023","interfaces":[{"interface":"back","average":{"1min":2.4289999999999998,"5min":1.8480000000000001,"15min":2.1000000000000001},"min":{"1min":0.439,"5min":0.38800000000000001,"15min":0.33900000000000002},"max":{"1min":10.914999999999999,"5min":21.518999999999998,"15min":21.518999999999998},"last":0.70699999999999996},{"interface":"front","average":{"1min":2.4220000000000002,"5min":1.903,"15min":2.3380000000000001},"min":{"1min":0.42499999999999999,"5min":0.42499999999999999,"15min":0.38900000000000001},"max":{"1min":8.2590000000000003,"5min":9.9879999999999995,"15min":13.603999999999999},"last":1.1399999999999999}]}]},{"osd":4,"up_from":31,"seq":133143986611,"num_pgs":60,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1069792,"kb_used_data":565272,"kb_used_omap":0,"kb_used_meta":504512,"kb_avail":92675360,"statfs":{"total":95995035648,"available":94899568640,"internally_reserved":0,"allocated":578838528,"data_stored":577269761,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":516620288},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,1],"upper_bound":4},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:15:14 2023","interfaces":[{"interface":"back","average":{"1min":1.944,"5min":1.1080000000000001,"15min":0.96899999999999997},"min":{"1min":0.23799999999999999,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":8.0079999999999991,"5min":8.0079999999999991,"15min":8.0079999999999991},"last":0.255},{"interface":"front","average":{"1min":0.876,"5min":1.0820000000000001,"15min":1.1160000000000001},"min":{"1min":0.23799999999999999,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":2.4649999999999999,"5min":10.945,"15min":10.945},"last":0.51400000000000001}]},{"osd":1,"last update":"Sat Sep 23 21:15:14 2023","interfaces":[{"interface":"back","average":{"1min":1.345,"5min":1.0429999999999999,"15min":2.5840000000000001},"min":{"1min":0.32700000000000001,"5min":0.221,"15min":0.221},"max":{"1min":4.0410000000000004,"5min":7.8609999999999998,"15min":37.960000000000001},"last":0.85499999999999998},{"interface":"front","average":{"1min":2.246,"5min":1.319,"15min":2.516},"min":{"1min":0.30299999999999999,"5min":0.28499999999999998,"15min":0.27000000000000002},"max":{"1min":13.983000000000001,"5min":13.983000000000001,"15min":37.933},"last":0.86399999999999999}]},{"osd":2,"last update":"Sat Sep 23 21:15:14 2023","interfaces":[{"interface":"back","average":{"1min":2.3559999999999999,"5min":1.3660000000000001,"15min":1.734},"min":{"1min":0.26400000000000001,"5min":0.247,"15min":0.247},"max":{"1min":14.026,"5min":14.026,"15min":16.928000000000001},"last":1.5189999999999999},{"interface":"front","average":{"1min":3.0419999999999998,"5min":1.6220000000000001,"15min":1.7949999999999999},"min":{"1min":0.33000000000000002,"5min":0.28599999999999998,"15min":0.27800000000000002},"max":{"1min":14.339,"5min":14.339,"15min":14.417999999999999},"last":2.2069999999999999}]},{"osd":3,"last update":"Sat Sep 23 21:14:47 2023","interfaces":[{"interface":"back","average":{"1min":1.0389999999999999,"5min":1.1990000000000001,"15min":2.056},"min":{"1min":0.27900000000000003,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":2.766,"5min":9.3279999999999994,"15min":15.01},"last":0.49099999999999999},{"interface":"front","average":{"1min":1.899,"5min":1.3879999999999999,"15min":2},"min":{"1min":0.32200000000000001,"5min":0.32200000000000001,"15min":0.30199999999999999},"max":{"1min":14.305,"5min":14.305,"15min":15.004},"last":0.65300000000000002}]},{"osd":5,"last update":"Sat Sep 23 21:15:04 2023","interfaces":[{"interface":"back","average":{"1min":2.0310000000000001,"5min":1.337,"15min":1.0840000000000001},"min":{"1min":0.32000000000000001,"5min":0.222,"15min":0.222},"max":{"1min":14.398999999999999,"5min":14.398999999999999,"15min":14.398999999999999},"last":0.34399999999999997},{"interface":"front","average":{"1min":1.675,"5min":1.2050000000000001,"15min":1.121},"min":{"1min":0.29699999999999999,"5min":0.27200000000000002,"15min":0.255},"max":{"1min":8.016,"5min":8.016,"15min":9.8979999999999997},"last":0.44500000000000001}]},{"osd":6,"last update":"Sat Sep 23 21:15:14 2023","interfaces":[{"interface":"back","average":{"1min":1.403,"5min":1.248,"15min":1.2749999999999999},"min":{"1min":0.34300000000000003,"5min":0.28599999999999998,"15min":0.26800000000000002},"max":{"1min":8.7420000000000009,"5min":24.452999999999999,"15min":24.452999999999999},"last":0.54400000000000004},{"interface":"front","average":{"1min":2.472,"5min":1.425,"15min":1.0680000000000001},"min":{"1min":0.40300000000000002,"5min":0.33500000000000002,"15min":0.32500000000000001},"max":{"1min":14.374000000000001,"5min":14.374000000000001,"15min":14.387},"last":0.68700000000000006}]},{"osd":7,"last update":"Sat Sep 23 21:14:47 2023","interfaces":[{"interface":"back","average":{"1min":1.9910000000000001,"5min":1.355,"15min":1.1299999999999999},"min":{"1min":0.38300000000000001,"5min":0.30099999999999999,"15min":0.29299999999999998},"max":{"1min":14.422000000000001,"5min":14.422000000000001,"15min":16.890999999999998},"last":0.95899999999999996},{"interface":"front","average":{"1min":1.536,"5min":1.234,"15min":1.2090000000000001},"min":{"1min":0.39200000000000002,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":7.7229999999999999,"5min":7.7229999999999999,"15min":11.170999999999999},"last":0.47599999999999998}]}]},{"osd":5,"up_from":39,"seq":167503724977,"num_pgs":55,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1117588,"kb_used_data":592460,"kb_used_omap":0,"kb_used_meta":525120,"kb_avail":92627564,"statfs":{"total":95995035648,"available":94850625536,"internally_reserved":0,"allocated":606679040,"data_stored":605118849,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":537722880},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:15:14 2023","interfaces":[{"interface":"back","average":{"1min":1.3720000000000001,"5min":0.84599999999999997,"15min":0.75800000000000001},"min":{"1min":0.23100000000000001,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":4.3680000000000003,"5min":4.3680000000000003,"15min":4.3680000000000003},"last":0.25600000000000001},{"interface":"front","average":{"1min":1.6839999999999999,"5min":0.872,"15min":0.73599999999999999},"min":{"1min":0.23899999999999999,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":6.577,"5min":6.577,"15min":6.577},"last":0.46500000000000002}]},{"osd":1,"last update":"Sat Sep 23 21:15:11 2023","interfaces":[{"interface":"back","average":{"1min":1.1140000000000001,"5min":0.94099999999999995,"15min":1.6830000000000001},"min":{"1min":0.29399999999999998,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":7.5839999999999996,"5min":7.5839999999999996,"15min":25.239999999999998},"last":0.41199999999999998},{"interface":"front","average":{"1min":1.123,"5min":1.472,"15min":1.7549999999999999},"min":{"1min":0.28299999999999997,"5min":0.28299999999999997,"15min":0.252},"max":{"1min":4.2199999999999998,"5min":11.116,"15min":25.253},"last":0.34000000000000002}]},{"osd":2,"last update":"Sat Sep 23 21:15:27 2023","interfaces":[{"interface":"back","average":{"1min":1.373,"5min":1.3320000000000001,"15min":1.6240000000000001},"min":{"1min":0.28399999999999997,"5min":0.27800000000000002,"15min":0.23499999999999999},"max":{"1min":3.8730000000000002,"5min":9.2889999999999997,"15min":12.634},"last":0.441},{"interface":"front","average":{"1min":1.472,"5min":1.1559999999999999,"15min":1.6319999999999999},"min":{"1min":0.29799999999999999,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":7.2720000000000002,"5min":9.7170000000000005,"15min":9.7650000000000006},"last":0.86799999999999999}]},{"osd":3,"last update":"Sat Sep 23 21:14:48 2023","interfaces":[{"interface":"back","average":{"1min":1.0109999999999999,"5min":1.6339999999999999,"15min":2.484},"min":{"1min":0.32400000000000001,"5min":0.311,"15min":0.29299999999999998},"max":{"1min":3.0390000000000001,"5min":14.242000000000001,"15min":61.667999999999999},"last":0.85499999999999998},{"interface":"front","average":{"1min":1.284,"5min":1.6559999999999999,"15min":2.5910000000000002},"min":{"1min":0.39100000000000001,"5min":0.34899999999999998,"15min":0.30599999999999999},"max":{"1min":6.508,"5min":12.44,"15min":62.276000000000003},"last":0.39900000000000002}]},{"osd":4,"last update":"Sat Sep 23 21:15:00 2023","interfaces":[{"interface":"back","average":{"1min":1.0920000000000001,"5min":1.5289999999999999,"15min":1.157},"min":{"1min":0.253,"5min":0.253,"15min":0.23400000000000001},"max":{"1min":4.7750000000000004,"5min":12.776,"15min":12.776},"last":0.29699999999999999},{"interface":"front","average":{"1min":1.163,"5min":1.3029999999999999,"15min":1.1539999999999999},"min":{"1min":0.35999999999999999,"5min":0.27700000000000002,"15min":0.26900000000000002},"max":{"1min":4.8200000000000003,"5min":9.7119999999999997,"15min":11.973000000000001},"last":0.311}]},{"osd":6,"last update":"Sat Sep 23 21:15:11 2023","interfaces":[{"interface":"back","average":{"1min":1.0880000000000001,"5min":1.0449999999999999,"15min":1.071},"min":{"1min":0.36499999999999999,"5min":0.313,"15min":0.25900000000000001},"max":{"1min":5.2249999999999996,"5min":7.7030000000000003,"15min":11.128},"last":0.28000000000000003},{"interface":"front","average":{"1min":1.022,"5min":1.5489999999999999,"15min":1.2450000000000001},"min":{"1min":0.34300000000000003,"5min":0.34300000000000003,"15min":0.317},"max":{"1min":4.3550000000000004,"5min":10.999000000000001,"15min":11.112},"last":0.45600000000000002}]},{"osd":7,"last update":"Sat Sep 23 21:14:48 2023","interfaces":[{"interface":"back","average":{"1min":0.84999999999999998,"5min":0.98499999999999999,"15min":1.8129999999999999},"min":{"1min":0.34499999999999997,"5min":0.31,"15min":0.309},"max":{"1min":3.3039999999999998,"5min":5.5659999999999998,"15min":189.405},"last":0.32600000000000001},{"interface":"front","average":{"1min":0.92000000000000004,"5min":1.472,"15min":1.9259999999999999},"min":{"1min":0.40899999999999997,"5min":0.35399999999999998,"15min":0.32500000000000001},"max":{"1min":3.3239999999999998,"5min":11.183999999999999,"15min":189.685},"last":0.42899999999999999}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8488,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1259,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5322,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":4838,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":792592384,"data_stored":792592384,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":793837568,"data_stored":793837568,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":642449408,"data_stored":642449408,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":556662784,"data_stored":556662784,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":577961984,"data_stored":577961984,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":760676352,"data_stored":760676352,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T21:15:44.044 INFO:tasks.cephadm.ceph_manager.ceph:increase pool size by 10 2023-09-23T21:15:44.044 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph osd pool set .mgr pg_num 41 2023-09-23T21:15:44.045 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:15:44.113 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:15:44.121 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:15:44.122 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:15:44.223 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:15:44.240 INFO:teuthology.orchestra.run.smithi070.stdout:ERROR: (22) Invalid argument 2023-09-23T21:15:44.240 INFO:teuthology.orchestra.run.smithi070.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T21:15:44.311 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:15:44.377 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T21:15:44.384 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:15:44.384 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:15:44.447 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 12 16 27167 27151 141.34 113.188 0.0060594 0.00706801 2023-09-23T21:15:44.485 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:15:44.518 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:15:44 smithi070 ceph-mon[168282]: pgmap v1386: 119 pgs: 119 active+clean; 1.3 GiB data, 7.7 GiB used, 529 GiB / 536 GiB avail; 129 MiB/s wr, 2.07k op/s 2023-09-23T21:15:44.518 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:15:44 smithi070 ceph-mon[168282]: 1.6 scrub starts 2023-09-23T21:15:44.518 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:15:44 smithi070 ceph-mon[168282]: 1.6 scrub ok 2023-09-23T21:15:44.518 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:15:44 smithi070 ceph-mon[168282]: 1.9 scrub starts 2023-09-23T21:15:44.518 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:15:44 smithi070 ceph-mon[168282]: 1.9 scrub ok 2023-09-23T21:15:44.518 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:15:44 smithi070 ceph-mon[168282]: from='client.36708 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T21:15:44.522 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:15:44.620 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:15:44.626 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:15:44.627 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:15:44.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:15:44 smithi026 ceph-mon[179688]: pgmap v1386: 119 pgs: 119 active+clean; 1.3 GiB data, 7.7 GiB used, 529 GiB / 536 GiB avail; 129 MiB/s wr, 2.07k op/s 2023-09-23T21:15:44.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:15:44 smithi026 ceph-mon[179688]: 1.6 scrub starts 2023-09-23T21:15:44.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:15:44 smithi026 ceph-mon[179688]: 1.6 scrub ok 2023-09-23T21:15:44.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:15:44 smithi026 ceph-mon[179688]: 1.9 scrub starts 2023-09-23T21:15:44.633 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:15:44 smithi026 ceph-mon[179688]: 1.9 scrub ok 2023-09-23T21:15:44.633 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:15:44 smithi026 ceph-mon[179688]: from='client.36708 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T21:15:44.633 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:15:44 smithi026 ceph-mon[175151]: pgmap v1386: 119 pgs: 119 active+clean; 1.3 GiB data, 7.7 GiB used, 529 GiB / 536 GiB avail; 129 MiB/s wr, 2.07k op/s 2023-09-23T21:15:44.633 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:15:44 smithi026 ceph-mon[175151]: 1.6 scrub starts 2023-09-23T21:15:44.633 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:15:44 smithi026 ceph-mon[175151]: 1.6 scrub ok 2023-09-23T21:15:44.634 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:15:44 smithi026 ceph-mon[175151]: 1.9 scrub starts 2023-09-23T21:15:44.634 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:15:44 smithi026 ceph-mon[175151]: 1.9 scrub ok 2023-09-23T21:15:44.634 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:15:44 smithi026 ceph-mon[175151]: from='client.36708 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T21:15:44.639 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T21:15:44.703 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_blocked_ops 2023-09-23T21:15:44.728 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:15:44.788 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:15:44.802 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T21:15:44.817 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:15:44.848 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:15:44.852 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:15:44.853 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:15:44.954 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:15:44.984 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:15:45.028 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:15:45.031 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:15:45.031 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:15:45.132 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:15:45.173 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:15:45.215 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:15:45.219 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:15:45.219 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:15:45.320 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:15:45.399 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:15:45.411 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T21:15:45.447 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 13 16 28774 28758 138.195 100.438 0.0116522 0.00722913 2023-09-23T21:15:45.532 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T21:15:45.536 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:15:45 smithi026 ceph-mon[179688]: 1.1a scrub starts 2023-09-23T21:15:45.536 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:15:45 smithi026 ceph-mon[179688]: 1.1a scrub ok 2023-09-23T21:15:45.537 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:15:45.537 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:15:45.538 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:15:45 smithi026 ceph-mon[175151]: 1.1a scrub starts 2023-09-23T21:15:45.538 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:15:45 smithi026 ceph-mon[175151]: 1.1a scrub ok 2023-09-23T21:15:45.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:15:45 smithi070 ceph-mon[168282]: 1.1a scrub starts 2023-09-23T21:15:45.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:15:45 smithi070 ceph-mon[168282]: 1.1a scrub ok 2023-09-23T21:15:45.638 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:15:45.712 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:15:45.946 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:15:45.950 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:15:45.962 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:15:45.963 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:15:46.063 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:15:46.104 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:15:46.115 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:15:46.207 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:15:46.211 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:15:46.212 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:15:46.312 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:15:46.361 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:15:46.447 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 14 16 30414 30398 135.645 102.5 0.00739744 0.00736719 2023-09-23T21:15:46.466 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:15:46.472 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:15:46.472 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:15:46.573 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:15:46.576 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:15:46 smithi026 ceph-mon[179688]: pgmap v1387: 119 pgs: 119 active+clean; 1.4 GiB data, 8.0 GiB used, 528 GiB / 536 GiB avail; 137 MiB/s wr, 2.19k op/s 2023-09-23T21:15:46.576 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:15:46 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:15:46.576 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:15:46 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:15:46.577 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:15:46 smithi026 ceph-mon[175151]: pgmap v1387: 119 pgs: 119 active+clean; 1.4 GiB data, 8.0 GiB used, 528 GiB / 536 GiB avail; 137 MiB/s wr, 2.19k op/s 2023-09-23T21:15:46.577 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:15:46 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:15:46.577 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:15:46 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:15:46.603 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:15:46 smithi070 ceph-mon[168282]: pgmap v1387: 119 pgs: 119 active+clean; 1.4 GiB data, 8.0 GiB used, 528 GiB / 536 GiB avail; 137 MiB/s wr, 2.19k op/s 2023-09-23T21:15:46.603 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:15:46 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:15:46.603 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:15:46 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:15:46.621 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:15:46.769 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:15:46.773 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:15:46.774 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:15:46.853 INFO:teuthology.orchestra.run.smithi070.stdout:ERROR: (22) Invalid argument 2023-09-23T21:15:46.853 INFO:teuthology.orchestra.run.smithi070.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T21:15:46.875 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:15:46.923 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:15:46.974 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:15:46.979 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:15:46.979 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:15:47.080 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:15:47.150 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:15:47.318 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_historic_ops 2023-09-23T21:15:47.364 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:15:47 smithi026 ceph-mon[179688]: 1.16 scrub starts 2023-09-23T21:15:47.364 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:15:47 smithi026 ceph-mon[179688]: 1.16 scrub ok 2023-09-23T21:15:47.364 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:15:47 smithi026 ceph-mon[179688]: 1.a scrub starts 2023-09-23T21:15:47.364 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:15:47 smithi026 ceph-mon[179688]: 1.a scrub ok 2023-09-23T21:15:47.364 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:15:47 smithi026 ceph-mon[175151]: 1.16 scrub starts 2023-09-23T21:15:47.364 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:15:47 smithi026 ceph-mon[175151]: 1.16 scrub ok 2023-09-23T21:15:47.364 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:15:47 smithi026 ceph-mon[175151]: 1.a scrub starts 2023-09-23T21:15:47.365 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:15:47 smithi026 ceph-mon[175151]: 1.a scrub ok 2023-09-23T21:15:47.450 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 15 16 32060 32044 133.435 102.875 0.0119616 0.00748845 2023-09-23T21:15:47.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:15:47 smithi070 ceph-mon[168282]: 1.16 scrub starts 2023-09-23T21:15:47.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:15:47 smithi070 ceph-mon[168282]: 1.16 scrub ok 2023-09-23T21:15:47.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:15:47 smithi070 ceph-mon[168282]: 1.a scrub starts 2023-09-23T21:15:47.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:15:47 smithi070 ceph-mon[168282]: 1.a scrub ok 2023-09-23T21:15:47.699 INFO:teuthology.orchestra.run.smithi026.stderr:Error EINVAL: specified pg_num 41 < pg_num_max 32 2023-09-23T21:15:48.024 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T21:15:48.450 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 16 16 33813 33797 131.943 109.562 0.00747661 0.00757239 2023-09-23T21:15:48.562 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:15:48 smithi070 ceph-mon[168282]: pgmap v1388: 119 pgs: 119 active+clean; 1.7 GiB data, 8.8 GiB used, 528 GiB / 536 GiB avail; 132 MiB/s wr, 2.10k op/s 2023-09-23T21:15:48.562 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:15:48 smithi070 ceph-mon[168282]: 1.11 scrub starts 2023-09-23T21:15:48.562 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:15:48 smithi070 ceph-mon[168282]: 1.11 scrub ok 2023-09-23T21:15:48.562 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:15:48 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/3686627752' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "41"}]: dispatch 2023-09-23T21:15:48.630 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:15:48 smithi026 ceph-mon[179688]: pgmap v1388: 119 pgs: 119 active+clean; 1.7 GiB data, 8.8 GiB used, 528 GiB / 536 GiB avail; 132 MiB/s wr, 2.10k op/s 2023-09-23T21:15:48.630 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:15:48 smithi026 ceph-mon[179688]: 1.11 scrub starts 2023-09-23T21:15:48.630 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:15:48 smithi026 ceph-mon[179688]: 1.11 scrub ok 2023-09-23T21:15:48.630 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:15:48 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/3686627752' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "41"}]: dispatch 2023-09-23T21:15:48.630 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:15:48 smithi026 ceph-mon[175151]: pgmap v1388: 119 pgs: 119 active+clean; 1.7 GiB data, 8.8 GiB used, 528 GiB / 536 GiB avail; 132 MiB/s wr, 2.10k op/s 2023-09-23T21:15:48.631 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:15:48 smithi026 ceph-mon[175151]: 1.11 scrub starts 2023-09-23T21:15:48.631 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:15:48 smithi026 ceph-mon[175151]: 1.11 scrub ok 2023-09-23T21:15:48.631 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:15:48 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/3686627752' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "41"}]: dispatch 2023-09-23T21:15:49.376 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:15:49 smithi026 ceph-mon[175151]: 1.19 scrub starts 2023-09-23T21:15:49.376 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:15:49 smithi026 ceph-mon[175151]: 1.19 scrub ok 2023-09-23T21:15:49.450 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 17 16 35601 35585 130.755 111.75 0.0161091 0.00764328 2023-09-23T21:15:49.531 INFO:teuthology.orchestra.run.smithi070.stdout:ERROR: (22) Invalid argument 2023-09-23T21:15:49.531 INFO:teuthology.orchestra.run.smithi070.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T21:15:49.551 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:15:49 smithi070 ceph-mon[168282]: 1.19 scrub starts 2023-09-23T21:15:49.552 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:15:49 smithi070 ceph-mon[168282]: 1.19 scrub ok 2023-09-23T21:15:49.631 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 21:15:49 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[164029]: ::ffff:172.21.15.70 - - [23/Sep/2023:21:15:49] "GET /metrics HTTP/1.1" 200 36366 "" "Prometheus/2.43.0" 2023-09-23T21:15:49.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:15:49 smithi026 ceph-mon[179688]: 1.19 scrub starts 2023-09-23T21:15:49.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:15:49 smithi026 ceph-mon[179688]: 1.19 scrub ok 2023-09-23T21:15:49.744 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:15:49.753 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:15:49.753 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:15:49.854 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:15:49.928 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_ops_in_flight 2023-09-23T21:15:49.933 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:15:50.450 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 18 16 36988 36972 128.308 86.6875 0.0108959 0.00778896 2023-09-23T21:15:50.497 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T21:15:50.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:15:50 smithi026 ceph-mon[175151]: pgmap v1389: 119 pgs: 119 active+clean; 1.8 GiB data, 9.3 GiB used, 527 GiB / 536 GiB avail; 127 MiB/s wr, 2.03k op/s 2023-09-23T21:15:50.633 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:15:50 smithi026 ceph-mon[179688]: pgmap v1389: 119 pgs: 119 active+clean; 1.8 GiB data, 9.3 GiB used, 527 GiB / 536 GiB avail; 127 MiB/s wr, 2.03k op/s 2023-09-23T21:15:50.721 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:15:50 smithi070 ceph-mon[168282]: pgmap v1389: 119 pgs: 119 active+clean; 1.8 GiB data, 9.3 GiB used, 527 GiB / 536 GiB avail; 127 MiB/s wr, 2.03k op/s 2023-09-23T21:15:51.200 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:15:51.206 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:15:51.207 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:15:51.307 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:15:51.388 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:15:51.450 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 19 16 38579 38563 126.788 99.4375 0.00633778 0.0078815 2023-09-23T21:15:51.816 INFO:teuthology.orchestra.run.smithi070.stdout:ERROR: (22) Invalid argument 2023-09-23T21:15:51.816 INFO:teuthology.orchestra.run.smithi070.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T21:15:52.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:15:51 smithi070 ceph-mon[168282]: 1.17 scrub starts 2023-09-23T21:15:52.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:15:51 smithi070 ceph-mon[168282]: 1.17 scrub ok 2023-09-23T21:15:52.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:15:51 smithi070 ceph-mon[168282]: 1.7 scrub starts 2023-09-23T21:15:52.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:15:51 smithi070 ceph-mon[168282]: 1.7 scrub ok 2023-09-23T21:15:52.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:15:51 smithi070 ceph-mon[168282]: pgmap v1390: 119 pgs: 119 active+clean; 2.0 GiB data, 9.8 GiB used, 527 GiB / 536 GiB avail; 117 MiB/s wr, 1.88k op/s 2023-09-23T21:15:52.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:15:51 smithi026 ceph-mon[179688]: 1.17 scrub starts 2023-09-23T21:15:52.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:15:51 smithi026 ceph-mon[179688]: 1.17 scrub ok 2023-09-23T21:15:52.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:15:51 smithi026 ceph-mon[179688]: 1.7 scrub starts 2023-09-23T21:15:52.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:15:51 smithi026 ceph-mon[179688]: 1.7 scrub ok 2023-09-23T21:15:52.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:15:51 smithi026 ceph-mon[179688]: pgmap v1390: 119 pgs: 119 active+clean; 2.0 GiB data, 9.8 GiB used, 527 GiB / 536 GiB avail; 117 MiB/s wr, 1.88k op/s 2023-09-23T21:15:52.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:15:51 smithi026 ceph-mon[175151]: 1.17 scrub starts 2023-09-23T21:15:52.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:15:51 smithi026 ceph-mon[175151]: 1.17 scrub ok 2023-09-23T21:15:52.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:15:51 smithi026 ceph-mon[175151]: 1.7 scrub starts 2023-09-23T21:15:52.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:15:51 smithi026 ceph-mon[175151]: 1.7 scrub ok 2023-09-23T21:15:52.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:15:51 smithi026 ceph-mon[175151]: pgmap v1390: 119 pgs: 119 active+clean; 2.0 GiB data, 9.8 GiB used, 527 GiB / 536 GiB avail; 117 MiB/s wr, 1.88k op/s 2023-09-23T21:15:52.195 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_blocked_ops 2023-09-23T21:15:52.298 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:15:52.305 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:15:52.305 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:15:52.323 DEBUG:teuthology.orchestra.run:got remote process result: 22 2023-09-23T21:15:52.406 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:15:52.450 INFO:tasks.radosbench.radosbench.0.smithi026.stdout:2023-09-23T21:15:52.451487+0000 min lat: 0.00111295 max lat: 0.154633 avg lat: 0.00796675 2023-09-23T21:15:52.451 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T21:15:52.451 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 20 16 40173 40157 125.431 99.625 0.00643226 0.00796675 2023-09-23T21:15:52.457 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:15:52.512 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:15:52.519 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:15:52.520 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:15:52.620 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:15:52.662 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:15:52.704 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:15:52.710 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:15:52.710 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:15:52.811 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:15:52.858 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:15:52.905 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:15:52.909 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:15:52.909 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:15:52.931 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T21:15:53.010 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:15:53.040 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:15:53.097 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:15:53.102 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:15:53.103 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:15:53.203 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:15:53.247 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:15:53.299 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:15:53.304 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:15:53.304 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:15:53.405 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:15:53.451 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:15:53.451 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 21 16 41605 41589 123.714 89.5 0.0144783 0.00807677 2023-09-23T21:15:53.556 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0: {} 2023-09-23T21:15:53.595 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1: {} 2023-09-23T21:15:53.638 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2: {} 2023-09-23T21:15:53.658 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:15:53.663 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:15:53.664 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:15:53.685 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3: {} 2023-09-23T21:15:53.720 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4: {} 2023-09-23T21:15:53.758 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T21:15:53.765 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:15:53.793 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6: {} 2023-09-23T21:15:53.811 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:15:53.844 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T21:15:53.855 INFO:teuthology.orchestra.run.smithi026.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T21:15:53.855 INFO:teuthology.orchestra.run.smithi026.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T21:15:53.855 INFO:teuthology.orchestra.run.smithi026.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T21:15:53.855 INFO:teuthology.orchestra.run.smithi026.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T21:15:53.855 INFO:teuthology.orchestra.run.smithi026.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T21:15:53.855 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T21:15:53.855 INFO:teuthology.orchestra.run.smithi026.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T21:15:53.856 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T21:15:53.860 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:15:53.863 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:15:53.864 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:15:53.965 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:15:54.016 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:15:54.061 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:15:54.064 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:15:54.065 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:15:54.166 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:15:54.202 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:15:54 smithi026 ceph-mon[175151]: 1.1e scrub starts 2023-09-23T21:15:54.202 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:15:54 smithi026 ceph-mon[175151]: 1.1e scrub ok 2023-09-23T21:15:54.202 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:15:54 smithi026 ceph-mon[175151]: pgmap v1391: 119 pgs: 119 active+clean; 2.3 GiB data, 11 GiB used, 526 GiB / 536 GiB avail; 113 MiB/s wr, 1.81k op/s 2023-09-23T21:15:54.202 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:15:54 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/4263043919' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:15:54.202 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:15:54 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/2445361532' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T21:15:54.202 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:15:54 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T21:15:54.202 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:15:54 smithi026 ceph-mon[179688]: 1.1e scrub starts 2023-09-23T21:15:54.203 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:15:54 smithi026 ceph-mon[179688]: 1.1e scrub ok 2023-09-23T21:15:54.203 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:15:54 smithi026 ceph-mon[179688]: pgmap v1391: 119 pgs: 119 active+clean; 2.3 GiB data, 11 GiB used, 526 GiB / 536 GiB avail; 113 MiB/s wr, 1.81k op/s 2023-09-23T21:15:54.203 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:15:54 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/4263043919' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:15:54.203 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:15:54 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/2445361532' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T21:15:54.203 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:15:54 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T21:15:54.227 INFO:teuthology.orchestra.run.smithi026.stderr:noscrub is set 2023-09-23T21:15:54.236 INFO:teuthology.orchestra.run.smithi070.stdout:{ 2023-09-23T21:15:54.237 INFO:teuthology.orchestra.run.smithi070.stdout: "ops": [], 2023-09-23T21:15:54.237 INFO:teuthology.orchestra.run.smithi070.stdout: "complaint_time": 30, 2023-09-23T21:15:54.237 INFO:teuthology.orchestra.run.smithi070.stdout: "num_blocked_ops": 0 2023-09-23T21:15:54.237 INFO:teuthology.orchestra.run.smithi070.stdout:} 2023-09-23T21:15:54.240 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:15:54.264 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:15:54 smithi070 ceph-mon[168282]: 1.1e scrub starts 2023-09-23T21:15:54.264 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:15:54 smithi070 ceph-mon[168282]: 1.1e scrub ok 2023-09-23T21:15:54.265 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:15:54 smithi070 ceph-mon[168282]: pgmap v1391: 119 pgs: 119 active+clean; 2.3 GiB data, 11 GiB used, 526 GiB / 536 GiB avail; 113 MiB/s wr, 1.81k op/s 2023-09-23T21:15:54.265 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:15:54 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/4263043919' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:15:54.265 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:15:54 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/2445361532' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T21:15:54.265 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:15:54 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T21:15:54.326 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:15:54.335 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:15:54.335 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:15:54.436 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:15:54.452 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 22 16 43029 43013 122.135 89 0.0153005 0.008181 2023-09-23T21:15:54.510 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:15:54.624 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:15:54.632 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:15:54.633 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:15:54.734 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:15:54.806 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:15:55.452 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 23 16 44582 44566 121.045 97.0625 0.00888797 0.00825581 2023-09-23T21:15:55.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:15:55 smithi070 ceph-mon[168282]: 1.1c scrub starts 2023-09-23T21:15:55.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:15:55 smithi070 ceph-mon[168282]: 1.1c scrub ok 2023-09-23T21:15:55.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:15:55 smithi070 ceph-mon[168282]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:15:55.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:15:55 smithi070 ceph-mon[168282]: failed to encode map e550 with expected crc 2023-09-23T21:15:55.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:15:55 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T21:15:55.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:15:55 smithi070 ceph-mon[168282]: failed to encode map e550 with expected crc 2023-09-23T21:15:55.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:15:55 smithi070 ceph-mon[168282]: osdmap e550: 8 total, 8 up, 6 in 2023-09-23T21:15:55.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:15:55 smithi070 ceph-mon[168282]: failed to encode map e550 with expected crc 2023-09-23T21:15:55.637 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:15:55 smithi026 ceph-mon[175151]: 1.1c scrub starts 2023-09-23T21:15:55.637 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:15:55 smithi026 ceph-mon[175151]: 1.1c scrub ok 2023-09-23T21:15:55.637 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:15:55 smithi026 ceph-mon[175151]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:15:55.637 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:15:55 smithi026 ceph-mon[175151]: failed to encode map e550 with expected crc 2023-09-23T21:15:55.637 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:15:55 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T21:15:55.637 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:15:55 smithi026 ceph-mon[175151]: failed to encode map e550 with expected crc 2023-09-23T21:15:55.637 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:15:55 smithi026 ceph-mon[175151]: osdmap e550: 8 total, 8 up, 6 in 2023-09-23T21:15:55.637 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:15:55 smithi026 ceph-mon[175151]: failed to encode map e550 with expected crc 2023-09-23T21:15:55.637 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:15:55 smithi026 ceph-mon[179688]: 1.1c scrub starts 2023-09-23T21:15:55.638 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:15:55 smithi026 ceph-mon[179688]: 1.1c scrub ok 2023-09-23T21:15:55.638 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:15:55 smithi026 ceph-mon[179688]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:15:55.638 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:15:55 smithi026 ceph-mon[179688]: failed to encode map e550 with expected crc 2023-09-23T21:15:55.638 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:15:55 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T21:15:55.638 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:15:55 smithi026 ceph-mon[179688]: failed to encode map e550 with expected crc 2023-09-23T21:15:55.638 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:15:55 smithi026 ceph-mon[179688]: osdmap e550: 8 total, 8 up, 6 in 2023-09-23T21:15:55.638 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:15:55 smithi026 ceph-mon[179688]: failed to encode map e550 with expected crc 2023-09-23T21:15:56.329 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T21:15:56.452 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 24 16 46128 46112 120.028 96.625 0.00782272 0.00832615 2023-09-23T21:15:56.460 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:15:56.465 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:15:56.465 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:15:56.470 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:15:56 smithi070 ceph-mon[168282]: pgmap v1393: 119 pgs: 119 active+clean; 2.4 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 104 MiB/s wr, 1.66k op/s 2023-09-23T21:15:56.531 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_historic_ops 2023-09-23T21:15:56.567 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:15:56.608 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:15:56.631 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:15:56 smithi026 ceph-mon[179688]: pgmap v1393: 119 pgs: 119 active+clean; 2.4 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 104 MiB/s wr, 1.66k op/s 2023-09-23T21:15:56.631 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:15:56 smithi026 ceph-mon[175151]: pgmap v1393: 119 pgs: 119 active+clean; 2.4 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 104 MiB/s wr, 1.66k op/s 2023-09-23T21:15:56.657 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T21:15:56.659 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:15:56.665 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:15:56.665 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:15:56.766 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:15:56.818 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:15:56.868 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:15:56.873 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:15:56.873 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:15:56.974 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:15:57.059 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:15:57.125 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:15:57.135 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:15:57.135 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:15:57.174 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T21:15:57.221 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:15:57.237 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:15:57.294 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:15:57.325 INFO:tasks.thrashosds.thrasher:in_osds: [0, 2, 3, 4, 5, 7] out_osds: [6, 1] dead_osds: [] live_osds: [2, 3, 4, 5, 7, 6, 1, 0] 2023-09-23T21:15:57.325 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T21:15:57.325 INFO:tasks.thrashosds.thrasher:Removing osd 7, in_osds are: [0, 2, 3, 4, 5, 7] 2023-09-23T21:15:57.325 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph osd out 7 2023-09-23T21:15:57.452 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 25 16 47336 47320 118.247 75.5 0.0270122 0.00845048 2023-09-23T21:15:57.503 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:15:57.508 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:15:57.508 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:15:57.511 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:15:57.609 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:15:57.694 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:15:57.833 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:15:57.838 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:15:57.839 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:15:57.940 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:15:57.976 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:15:58.227 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:15:58 smithi026 ceph-mon[179688]: pgmap v1394: 119 pgs: 119 active+clean; 2.7 GiB data, 12 GiB used, 525 GiB / 536 GiB avail; 99 MiB/s wr, 1.59k op/s 2023-09-23T21:15:58.227 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:15:58 smithi026 ceph-mon[175151]: pgmap v1394: 119 pgs: 119 active+clean; 2.7 GiB data, 12 GiB used, 525 GiB / 536 GiB avail; 99 MiB/s wr, 1.59k op/s 2023-09-23T21:15:58.307 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:15:58.310 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:15:58.310 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:15:58.348 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:15:58.411 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:15:58.441 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:15:58.452 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 26 16 48542 48526 116.598 75.375 0.0146208 0.00857163 2023-09-23T21:15:58.488 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:15:58.489 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:15:58 smithi070 ceph-mon[168282]: pgmap v1394: 119 pgs: 119 active+clean; 2.7 GiB data, 12 GiB used, 525 GiB / 536 GiB avail; 99 MiB/s wr, 1.59k op/s 2023-09-23T21:15:58.490 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:15:58.491 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:15:58.592 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:15:58.629 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:15:59.076 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:15:59.078 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:15:59.079 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:15:59.179 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:15:59.225 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:15:59.275 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:15:59.279 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:15:59.279 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:15:59.380 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:15:59.407 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:15:59.453 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 27 16 49743 49727 115.06 75.0625 0.0109737 0.00868485 2023-09-23T21:15:59.752 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:15:59.756 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:15:59.757 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:15:59.858 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:15:59.881 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 21:15:59 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[164029]: ::ffff:172.21.15.70 - - [23/Sep/2023:21:15:59] "GET /metrics HTTP/1.1" 200 36366 "" "Prometheus/2.43.0" 2023-09-23T21:15:59.895 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:15:59.944 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:15:59.948 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:15:59.949 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:16:00.050 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:16:00.155 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:16:00.283 INFO:teuthology.orchestra.run.smithi070.stdout:{ 2023-09-23T21:16:00.284 INFO:teuthology.orchestra.run.smithi070.stdout: "size": 20, 2023-09-23T21:16:00.284 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 600, 2023-09-23T21:16:00.284 INFO:teuthology.orchestra.run.smithi070.stdout: "ops": [] 2023-09-23T21:16:00.284 INFO:teuthology.orchestra.run.smithi070.stdout:} 2023-09-23T21:16:00.308 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:16:00 smithi070 ceph-mon[168282]: pgmap v1395: 119 pgs: 119 active+clean; 2.8 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 98 MiB/s wr, 1.56k op/s 2023-09-23T21:16:00.403 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:16:00.416 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:16:00.417 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:16:00.452 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 28 16 51012 50996 113.783 79.3125 0.0200915 0.00878177 2023-09-23T21:16:00.517 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:16:00.521 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:16:00 smithi026 ceph-mon[175151]: pgmap v1395: 119 pgs: 119 active+clean; 2.8 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 98 MiB/s wr, 1.56k op/s 2023-09-23T21:16:00.522 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:16:00 smithi026 ceph-mon[179688]: pgmap v1395: 119 pgs: 119 active+clean; 2.8 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 98 MiB/s wr, 1.56k op/s 2023-09-23T21:16:00.524 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0: {} 2023-09-23T21:16:00.575 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1: {} 2023-09-23T21:16:00.622 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2: {} 2023-09-23T21:16:00.626 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:16:00.673 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3: {} 2023-09-23T21:16:00.711 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:16:00.714 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:16:00.715 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:16:00.730 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4: {} 2023-09-23T21:16:00.771 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T21:16:00.776 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_ops_in_flight 2023-09-23T21:16:00.809 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6: {} 2023-09-23T21:16:00.816 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:16:00.848 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T21:16:00.852 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:16:00.858 INFO:teuthology.orchestra.run.smithi026.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T21:16:00.858 INFO:teuthology.orchestra.run.smithi026.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T21:16:00.858 INFO:teuthology.orchestra.run.smithi026.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T21:16:00.858 INFO:teuthology.orchestra.run.smithi026.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T21:16:00.859 INFO:teuthology.orchestra.run.smithi026.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T21:16:00.859 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T21:16:00.859 INFO:teuthology.orchestra.run.smithi026.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T21:16:00.859 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T21:16:00.892 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:16:00.895 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:16:00.896 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:16:00.997 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:16:01.030 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:16:01.069 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:16:01.071 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:16:01.072 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:16:01.173 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:16:01.203 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:16:01.244 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:16:01.247 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:16:01.247 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:16:01.348 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:16:01.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:16:01 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/96049379' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:16:01.385 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:16:01.427 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:16:01.430 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:16:01.430 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:16:01.453 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 29 16 51962 51946 111.907 59.375 0.0162531 0.00892931 2023-09-23T21:16:01.531 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:16:01.588 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:16:01 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/96049379' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:16:01.588 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:16:01 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/96049379' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:16:01.633 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:16:01.699 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T21:16:01.703 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:16:01.704 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:16:01.805 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:16:01.841 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:16:01.871 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:16:01.884 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:16:01.885 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:16:01 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mon-a[175127]: 2023-09-23T21:16:01.770+0000 7f1e2d487700 -1 mon.a@0(leader).osd e550 definitely_dead 0 2023-09-23T21:16:01.887 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:16:01.888 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:16:01.988 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:16:02.026 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:16:02.075 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:16:02.080 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:16:02.081 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:16:02.182 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:16:02.217 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:16:02.265 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:16:02.269 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:16:02.270 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:16:02.293 INFO:teuthology.orchestra.run.smithi026.stderr:marked out osd.7. 2023-09-23T21:16:02.299 INFO:teuthology.orchestra.run.smithi026.stderr:nodeep-scrub is set 2023-09-23T21:16:02.370 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:16:02.396 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:16:02.409 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:16:02 smithi026 ceph-mon[179688]: pgmap v1396: 119 pgs: 119 active+clean; 2.9 GiB data, 13 GiB used, 524 GiB / 536 GiB avail; 94 MiB/s wr, 1.51k op/s 2023-09-23T21:16:02.409 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:16:02 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:16:02.409 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:16:02 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:16:02.409 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:16:02 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/2983860888' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:16:02.410 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:16:02 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:16:02.410 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:16:02 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/1309610691' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["7"]}]: dispatch 2023-09-23T21:16:02.410 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:16:02 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["7"]}]: dispatch 2023-09-23T21:16:02.410 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:16:02 smithi026 ceph-mon[179688]: Client client.admin marked osd.7 out, while it was still marked up 2023-09-23T21:16:02.410 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:16:02 smithi026 ceph-mon[175151]: pgmap v1396: 119 pgs: 119 active+clean; 2.9 GiB data, 13 GiB used, 524 GiB / 536 GiB avail; 94 MiB/s wr, 1.51k op/s 2023-09-23T21:16:02.410 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:16:02 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:16:02.410 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:16:02 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:16:02.410 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:16:02 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/2983860888' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:16:02.411 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:16:02 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:16:02.411 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:16:02 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/1309610691' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["7"]}]: dispatch 2023-09-23T21:16:02.411 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:16:02 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["7"]}]: dispatch 2023-09-23T21:16:02.411 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:16:02 smithi026 ceph-mon[175151]: Client client.admin marked osd.7 out, while it was still marked up 2023-09-23T21:16:02.428 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:16:02.430 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:16:02.431 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:16:02.453 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 30 16 52992 52976 110.323 64.375 0.00251915 0.00902029 2023-09-23T21:16:02.532 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:16:02.563 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:16:02.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:16:02 smithi070 ceph-mon[168282]: pgmap v1396: 119 pgs: 119 active+clean; 2.9 GiB data, 13 GiB used, 524 GiB / 536 GiB avail; 94 MiB/s wr, 1.51k op/s 2023-09-23T21:16:02.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:16:02 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:16:02.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:16:02 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:16:02.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:16:02 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/2983860888' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:16:02.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:16:02 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:16:02.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:16:02 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/1309610691' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["7"]}]: dispatch 2023-09-23T21:16:02.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:16:02 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["7"]}]: dispatch 2023-09-23T21:16:02.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:16:02 smithi070 ceph-mon[168282]: Client client.admin marked osd.7 out, while it was still marked up 2023-09-23T21:16:02.815 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:16:02.818 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:16:02.818 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:16:02.921 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:16:02.950 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:16:02.983 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:16:02.986 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:16:02.986 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:16:03.087 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:16:03.116 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:16:03.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:16:03 smithi026 ceph-mon[179688]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:16:03.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:16:03 smithi026 ceph-mon[179688]: failed to encode map e551 with expected crc 2023-09-23T21:16:03.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:16:03 smithi026 ceph-mon[179688]: failed to encode map e551 with expected crc 2023-09-23T21:16:03.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:16:03 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T21:16:03.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:16:03 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["7"]}]': finished 2023-09-23T21:16:03.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:16:03 smithi026 ceph-mon[179688]: osdmap e551: 8 total, 8 up, 5 in 2023-09-23T21:16:03.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:16:03 smithi026 ceph-mon[179688]: failed to encode map e551 with expected crc 2023-09-23T21:16:03.383 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:16:03 smithi026 ceph-mon[175151]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:16:03.383 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:16:03 smithi026 ceph-mon[175151]: failed to encode map e551 with expected crc 2023-09-23T21:16:03.383 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:16:03 smithi026 ceph-mon[175151]: failed to encode map e551 with expected crc 2023-09-23T21:16:03.383 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:16:03 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T21:16:03.383 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:16:03 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["7"]}]': finished 2023-09-23T21:16:03.383 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:16:03 smithi026 ceph-mon[175151]: osdmap e551: 8 total, 8 up, 5 in 2023-09-23T21:16:03.383 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:16:03 smithi026 ceph-mon[175151]: failed to encode map e551 with expected crc 2023-09-23T21:16:03.403 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:16:03.406 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:16:03.406 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:16:03.453 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 31 16 52992 52976 106.765 0 - 0.00902029 2023-09-23T21:16:03.507 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:16:03.574 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:16:03.589 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T21:16:03.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:16:03 smithi070 ceph-mon[168282]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:16:03.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:16:03 smithi070 ceph-mon[168282]: failed to encode map e551 with expected crc 2023-09-23T21:16:03.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:16:03 smithi070 ceph-mon[168282]: failed to encode map e551 with expected crc 2023-09-23T21:16:03.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:16:03 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T21:16:03.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:16:03 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["7"]}]': finished 2023-09-23T21:16:03.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:16:03 smithi070 ceph-mon[168282]: osdmap e551: 8 total, 8 up, 5 in 2023-09-23T21:16:03.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:16:03 smithi070 ceph-mon[168282]: failed to encode map e551 with expected crc 2023-09-23T21:16:03.664 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T21:16:03.668 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:16:03.668 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:16:03.731 INFO:teuthology.orchestra.run.smithi026.stdout:ERROR: (22) Invalid argument 2023-09-23T21:16:03.732 INFO:teuthology.orchestra.run.smithi026.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T21:16:03.769 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:16:03.795 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:16:03.827 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:16:03.829 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:16:03.830 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:16:03.931 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:16:03.959 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:16:04.057 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T21:16:04.060 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:16:04.060 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:16:04.078 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_blocked_ops 2023-09-23T21:16:04.162 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:16:04.189 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:16:04.221 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:16:04.223 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:16:04.224 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:16:04.286 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:16:04 smithi026 ceph-mon[179688]: pgmap v1398: 119 pgs: 8 peering, 111 active+clean; 3.1 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 93 MiB/s wr, 1.48k op/s 2023-09-23T21:16:04.286 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:16:04 smithi026 ceph-mon[179688]: failed to encode map e552 with expected crc 2023-09-23T21:16:04.286 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:16:04 smithi026 ceph-mon[179688]: failed to encode map e552 with expected crc 2023-09-23T21:16:04.286 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:16:04 smithi026 ceph-mon[179688]: osdmap e552: 8 total, 8 up, 5 in 2023-09-23T21:16:04.286 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:16:04 smithi026 ceph-mon[179688]: failed to encode map e552 with expected crc 2023-09-23T21:16:04.286 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:16:04 smithi026 ceph-mon[179688]: failed to encode map e552 with expected crc 2023-09-23T21:16:04.286 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:16:04 smithi026 ceph-mon[179688]: failed to encode map e552 with expected crc 2023-09-23T21:16:04.286 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:16:04 smithi026 ceph-mon[179688]: failed to encode map e552 with expected crc 2023-09-23T21:16:04.286 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:16:04 smithi026 ceph-mon[179688]: failed to encode map e552 with expected crc 2023-09-23T21:16:04.286 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:16:04 smithi026 ceph-mon[179688]: failed to encode map e552 with expected crc 2023-09-23T21:16:04.287 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:16:04 smithi026 ceph-mon[179688]: failed to encode map e552 with expected crc 2023-09-23T21:16:04.287 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:16:04 smithi026 ceph-mon[175151]: pgmap v1398: 119 pgs: 8 peering, 111 active+clean; 3.1 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 93 MiB/s wr, 1.48k op/s 2023-09-23T21:16:04.287 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:16:04 smithi026 ceph-mon[175151]: failed to encode map e552 with expected crc 2023-09-23T21:16:04.287 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:16:04 smithi026 ceph-mon[175151]: failed to encode map e552 with expected crc 2023-09-23T21:16:04.287 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:16:04 smithi026 ceph-mon[175151]: osdmap e552: 8 total, 8 up, 5 in 2023-09-23T21:16:04.287 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:16:04 smithi026 ceph-mon[175151]: failed to encode map e552 with expected crc 2023-09-23T21:16:04.287 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:16:04 smithi026 ceph-mon[175151]: failed to encode map e552 with expected crc 2023-09-23T21:16:04.287 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:16:04 smithi026 ceph-mon[175151]: failed to encode map e552 with expected crc 2023-09-23T21:16:04.287 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:16:04 smithi026 ceph-mon[175151]: failed to encode map e552 with expected crc 2023-09-23T21:16:04.287 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:16:04 smithi026 ceph-mon[175151]: failed to encode map e552 with expected crc 2023-09-23T21:16:04.287 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:16:04 smithi026 ceph-mon[175151]: failed to encode map e552 with expected crc 2023-09-23T21:16:04.288 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:16:04 smithi026 ceph-mon[175151]: failed to encode map e552 with expected crc 2023-09-23T21:16:04.325 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:16:04.370 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:16:04.413 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:16:04.414 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:16:04.417 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:16:04.418 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:16:04.457 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 32 16 52992 52976 103.415 0 - 0.00902029 2023-09-23T21:16:04.519 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:16:04.546 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:16:04 smithi070 ceph-mon[168282]: pgmap v1398: 119 pgs: 8 peering, 111 active+clean; 3.1 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 93 MiB/s wr, 1.48k op/s 2023-09-23T21:16:04.547 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:16:04 smithi070 ceph-mon[168282]: failed to encode map e552 with expected crc 2023-09-23T21:16:04.547 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:16:04 smithi070 ceph-mon[168282]: failed to encode map e552 with expected crc 2023-09-23T21:16:04.547 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:16:04 smithi070 ceph-mon[168282]: osdmap e552: 8 total, 8 up, 5 in 2023-09-23T21:16:04.547 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:16:04 smithi070 ceph-mon[168282]: failed to encode map e552 with expected crc 2023-09-23T21:16:04.547 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:16:04 smithi070 ceph-mon[168282]: failed to encode map e552 with expected crc 2023-09-23T21:16:04.547 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:16:04 smithi070 ceph-mon[168282]: failed to encode map e552 with expected crc 2023-09-23T21:16:04.547 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:16:04 smithi070 ceph-mon[168282]: failed to encode map e552 with expected crc 2023-09-23T21:16:04.547 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:16:04 smithi070 ceph-mon[168282]: failed to encode map e552 with expected crc 2023-09-23T21:16:04.547 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:16:04 smithi070 ceph-mon[168282]: failed to encode map e552 with expected crc 2023-09-23T21:16:04.547 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:16:04 smithi070 ceph-mon[168282]: failed to encode map e552 with expected crc 2023-09-23T21:16:04.550 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:16:04.587 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:16:04.590 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:16:04.590 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:16:04.691 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:16:04.730 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:16:04.780 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:16:04.784 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:16:04.784 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:16:04.885 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:16:05.004 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:16:05.056 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:16:05.211 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:16:05.222 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:16:05.223 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:16:05.314 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:16:05 smithi026 ceph-mon[175151]: failed to encode map e553 with expected crc 2023-09-23T21:16:05.315 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:16:05 smithi026 ceph-mon[175151]: failed to encode map e553 with expected crc 2023-09-23T21:16:05.315 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:16:05 smithi026 ceph-mon[175151]: failed to encode map e553 with expected crc 2023-09-23T21:16:05.315 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:16:05 smithi026 ceph-mon[175151]: osdmap e553: 8 total, 8 up, 5 in 2023-09-23T21:16:05.315 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:16:05 smithi026 ceph-mon[175151]: failed to encode map e553 with expected crc 2023-09-23T21:16:05.315 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:16:05 smithi026 ceph-mon[175151]: failed to encode map e553 with expected crc 2023-09-23T21:16:05.323 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:16:05.392 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:16:05.457 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 33 16 53688 53672 101.6 14.5 0.0242457 0.00983426 2023-09-23T21:16:05.507 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T21:16:05.575 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T21:16:05.579 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:16:05 smithi026 ceph-mon[179688]: failed to encode map e553 with expected crc 2023-09-23T21:16:05.579 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:16:05 smithi026 ceph-mon[179688]: failed to encode map e553 with expected crc 2023-09-23T21:16:05.579 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:16:05 smithi026 ceph-mon[179688]: failed to encode map e553 with expected crc 2023-09-23T21:16:05.580 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:16:05 smithi026 ceph-mon[179688]: osdmap e553: 8 total, 8 up, 5 in 2023-09-23T21:16:05.581 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:16:05 smithi026 ceph-mon[179688]: failed to encode map e553 with expected crc 2023-09-23T21:16:05.581 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:16:05 smithi026 ceph-mon[179688]: failed to encode map e553 with expected crc 2023-09-23T21:16:05.582 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:16:05.582 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:16:05.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:16:05 smithi070 ceph-mon[168282]: failed to encode map e553 with expected crc 2023-09-23T21:16:05.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:16:05 smithi070 ceph-mon[168282]: failed to encode map e553 with expected crc 2023-09-23T21:16:05.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:16:05 smithi070 ceph-mon[168282]: failed to encode map e553 with expected crc 2023-09-23T21:16:05.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:16:05 smithi070 ceph-mon[168282]: osdmap e553: 8 total, 8 up, 5 in 2023-09-23T21:16:05.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:16:05 smithi070 ceph-mon[168282]: failed to encode map e553 with expected crc 2023-09-23T21:16:05.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:16:05 smithi070 ceph-mon[168282]: failed to encode map e553 with expected crc 2023-09-23T21:16:05.683 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:16:05.711 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:16:05.747 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:16:05.750 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:16:05.751 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:16:05.851 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:16:05.979 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:16:06.158 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:16:06.163 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:16:06.164 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:16:06.265 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:16:06.304 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:16:06.361 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:16:06.365 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:16:06.366 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:16:06.458 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 34 16 54424 54408 99.9654 46 0.0116071 0.0099975 2023-09-23T21:16:06.466 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:16:06.543 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:16:06.567 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:16:06 smithi026 ceph-mon[175151]: pgmap v1401: 119 pgs: 7 remapped+peering, 8 peering, 104 active+clean; 3.2 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 69 MiB/s wr, 1.11k op/s; 14 B/s, 0 objects/s recovering 2023-09-23T21:16:06.567 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:16:06 smithi026 ceph-mon[175151]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2023-09-23T21:16:06.567 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:16:06 smithi026 ceph-mon[175151]: failed to encode map e554 with expected crc 2023-09-23T21:16:06.567 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:16:06 smithi026 ceph-mon[175151]: failed to encode map e554 with expected crc 2023-09-23T21:16:06.567 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:16:06 smithi026 ceph-mon[175151]: osdmap e554: 8 total, 8 up, 5 in 2023-09-23T21:16:06.567 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:16:06 smithi026 ceph-mon[175151]: failed to encode map e554 with expected crc 2023-09-23T21:16:06.568 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:16:06 smithi026 ceph-mon[175151]: failed to encode map e554 with expected crc 2023-09-23T21:16:06.568 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:16:06 smithi026 ceph-mon[175151]: failed to encode map e554 with expected crc 2023-09-23T21:16:06.569 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:16:06 smithi026 ceph-mon[179688]: pgmap v1401: 119 pgs: 7 remapped+peering, 8 peering, 104 active+clean; 3.2 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 69 MiB/s wr, 1.11k op/s; 14 B/s, 0 objects/s recovering 2023-09-23T21:16:06.570 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:16:06 smithi026 ceph-mon[179688]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2023-09-23T21:16:06.570 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:16:06 smithi026 ceph-mon[179688]: failed to encode map e554 with expected crc 2023-09-23T21:16:06.570 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:16:06 smithi026 ceph-mon[179688]: failed to encode map e554 with expected crc 2023-09-23T21:16:06.570 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:16:06 smithi026 ceph-mon[179688]: osdmap e554: 8 total, 8 up, 5 in 2023-09-23T21:16:06.570 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:16:06 smithi026 ceph-mon[179688]: failed to encode map e554 with expected crc 2023-09-23T21:16:06.570 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:16:06 smithi026 ceph-mon[179688]: failed to encode map e554 with expected crc 2023-09-23T21:16:06.570 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:16:06 smithi026 ceph-mon[179688]: failed to encode map e554 with expected crc 2023-09-23T21:16:06.581 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0: {} 2023-09-23T21:16:06.600 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:16:06.603 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:16:06.603 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:16:06 smithi070 ceph-mon[168282]: pgmap v1401: 119 pgs: 7 remapped+peering, 8 peering, 104 active+clean; 3.2 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 69 MiB/s wr, 1.11k op/s; 14 B/s, 0 objects/s recovering 2023-09-23T21:16:06.603 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:16:06 smithi070 ceph-mon[168282]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2023-09-23T21:16:06.605 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:16:06 smithi070 ceph-mon[168282]: failed to encode map e554 with expected crc 2023-09-23T21:16:06.605 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:16:06 smithi070 ceph-mon[168282]: failed to encode map e554 with expected crc 2023-09-23T21:16:06.605 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:16:06 smithi070 ceph-mon[168282]: osdmap e554: 8 total, 8 up, 5 in 2023-09-23T21:16:06.605 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:16:06 smithi070 ceph-mon[168282]: failed to encode map e554 with expected crc 2023-09-23T21:16:06.605 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:16:06 smithi070 ceph-mon[168282]: failed to encode map e554 with expected crc 2023-09-23T21:16:06.606 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:16:06 smithi070 ceph-mon[168282]: failed to encode map e554 with expected crc 2023-09-23T21:16:06.606 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:16:06.624 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1: {} 2023-09-23T21:16:06.680 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2: {} 2023-09-23T21:16:06.707 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:16:06.728 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3: {} 2023-09-23T21:16:06.738 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:16:06.764 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4: {} 2023-09-23T21:16:06.770 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:16:06.782 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:16:06.784 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:16:06.784 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:16:06.810 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T21:16:06.861 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6: {} 2023-09-23T21:16:06.885 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:16:06.894 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T21:16:06.897 INFO:teuthology.orchestra.run.smithi026.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T21:16:06.898 INFO:teuthology.orchestra.run.smithi026.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T21:16:06.898 INFO:teuthology.orchestra.run.smithi026.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T21:16:06.898 INFO:teuthology.orchestra.run.smithi026.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T21:16:06.898 INFO:teuthology.orchestra.run.smithi026.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T21:16:06.898 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T21:16:06.898 INFO:teuthology.orchestra.run.smithi026.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T21:16:06.899 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T21:16:06.927 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:16:06.973 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:16:06.976 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:16:06.978 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:16:07.078 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:16:07.111 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:16:07.150 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:16:07.153 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:16:07.154 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:16:07.255 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:16:07.290 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:16:07.335 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:16:07.338 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:16:07.339 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:16:07.440 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:16:07.461 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 35 16 55176 55160 98.4525 47 0.0073086 0.0101492 2023-09-23T21:16:07.470 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:16:07.511 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:16:07.514 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:16:07.514 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:16:07.528 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:16:07 smithi026 ceph-mon[179688]: failed to encode map e555 with expected crc 2023-09-23T21:16:07.528 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:16:07 smithi026 ceph-mon[179688]: osdmap e555: 8 total, 8 up, 5 in 2023-09-23T21:16:07.529 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:16:07 smithi026 ceph-mon[179688]: failed to encode map e555 with expected crc 2023-09-23T21:16:07.529 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:16:07 smithi026 ceph-mon[179688]: failed to encode map e555 with expected crc 2023-09-23T21:16:07.529 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:16:07 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/3883496450' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:16:07.531 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:16:07 smithi026 ceph-mon[175151]: failed to encode map e555 with expected crc 2023-09-23T21:16:07.531 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:16:07 smithi026 ceph-mon[175151]: osdmap e555: 8 total, 8 up, 5 in 2023-09-23T21:16:07.532 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:16:07 smithi026 ceph-mon[175151]: failed to encode map e555 with expected crc 2023-09-23T21:16:07.532 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:16:07 smithi026 ceph-mon[175151]: failed to encode map e555 with expected crc 2023-09-23T21:16:07.532 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:16:07 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/3883496450' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:16:07.615 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:16:07.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:16:07 smithi070 ceph-mon[168282]: failed to encode map e555 with expected crc 2023-09-23T21:16:07.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:16:07 smithi070 ceph-mon[168282]: osdmap e555: 8 total, 8 up, 5 in 2023-09-23T21:16:07.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:16:07 smithi070 ceph-mon[168282]: failed to encode map e555 with expected crc 2023-09-23T21:16:07.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:16:07 smithi070 ceph-mon[168282]: failed to encode map e555 with expected crc 2023-09-23T21:16:07.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:16:07 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/3883496450' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:16:07.645 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:16:07.667 INFO:teuthology.orchestra.run.smithi026.stdout:{ 2023-09-23T21:16:07.667 INFO:teuthology.orchestra.run.smithi026.stdout: "ops": [], 2023-09-23T21:16:07.668 INFO:teuthology.orchestra.run.smithi026.stdout: "complaint_time": 30, 2023-09-23T21:16:07.668 INFO:teuthology.orchestra.run.smithi026.stdout: "num_blocked_ops": 0 2023-09-23T21:16:07.668 INFO:teuthology.orchestra.run.smithi026.stdout:} 2023-09-23T21:16:07.686 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:16:07.690 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:16:07.691 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:16:07.792 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:16:07.862 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:16:08.167 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:16:08.176 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:16:08.176 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:16:08.277 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:16:08.305 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:16:08.351 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:16:08.358 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:16:08.358 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:16:08.426 INFO:tasks.thrashosds.thrasher:in_osds: [0, 2, 3, 4, 5] out_osds: [6, 1, 7] dead_osds: [] live_osds: [2, 3, 4, 5, 7, 6, 1, 0] 2023-09-23T21:16:08.426 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T21:16:08.426 INFO:tasks.thrashosds.thrasher:Growing pool unique_pool_5 2023-09-23T21:16:08.426 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T21:16:08.434 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:16:08 smithi026 ceph-mon[175151]: failed to encode map e555 with expected crc 2023-09-23T21:16:08.435 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:16:08 smithi026 ceph-mon[175151]: failed to encode map e555 with expected crc 2023-09-23T21:16:08.435 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:16:08 smithi026 ceph-mon[175151]: pgmap v1404: 119 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 4 active+undersized+remapped, 7 remapped+peering, 8 peering, 96 active+clean; 3.3 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 43 MiB/s wr, 685 op/s; 13243/161472 objects degraded (8.201%); 1.2 KiB/s, 0 objects/s recovering 2023-09-23T21:16:08.435 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:16:08 smithi026 ceph-mon[175151]: Health check failed: Degraded data redundancy: 13243/161472 objects degraded (8.201%), 4 pgs degraded (PG_DEGRADED) 2023-09-23T21:16:08.436 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:16:08 smithi026 ceph-mon[179688]: failed to encode map e555 with expected crc 2023-09-23T21:16:08.436 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:16:08 smithi026 ceph-mon[179688]: failed to encode map e555 with expected crc 2023-09-23T21:16:08.436 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:16:08 smithi026 ceph-mon[179688]: pgmap v1404: 119 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 4 active+undersized+remapped, 7 remapped+peering, 8 peering, 96 active+clean; 3.3 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 43 MiB/s wr, 685 op/s; 13243/161472 objects degraded (8.201%); 1.2 KiB/s, 0 objects/s recovering 2023-09-23T21:16:08.436 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:16:08 smithi026 ceph-mon[179688]: Health check failed: Degraded data redundancy: 13243/161472 objects degraded (8.201%), 4 pgs degraded (PG_DEGRADED) 2023-09-23T21:16:08.437 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_historic_ops 2023-09-23T21:16:08.458 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:16:08.460 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 36 16 55969 55953 97.0904 49.5625 0.020318 0.0102926 2023-09-23T21:16:08.588 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:16:08.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:16:08 smithi070 ceph-mon[168282]: failed to encode map e555 with expected crc 2023-09-23T21:16:08.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:16:08 smithi070 ceph-mon[168282]: failed to encode map e555 with expected crc 2023-09-23T21:16:08.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:16:08 smithi070 ceph-mon[168282]: pgmap v1404: 119 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 4 active+undersized+remapped, 7 remapped+peering, 8 peering, 96 active+clean; 3.3 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 43 MiB/s wr, 685 op/s; 13243/161472 objects degraded (8.201%); 1.2 KiB/s, 0 objects/s recovering 2023-09-23T21:16:08.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:16:08 smithi070 ceph-mon[168282]: Health check failed: Degraded data redundancy: 13243/161472 objects degraded (8.201%), 4 pgs degraded (PG_DEGRADED) 2023-09-23T21:16:08.690 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T21:16:08.703 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:16:08.703 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:16:08.804 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:16:08.930 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:16:09.192 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:16:09.200 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:16:09.201 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:16:09.302 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:16:09.405 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:16:09.444 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:16:09.459 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 37 16 56527 56511 95.4095 34.875 0.0419014 0.0104724 2023-09-23T21:16:09.488 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:16:09.513 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:16:09.526 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:16:09 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T21:16:09.526 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:16:09.527 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:16:09.527 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:16:09 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T21:16:09.628 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:16:09.715 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:16:09.725 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T21:16:09.807 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 21:16:09 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[164029]: ::ffff:172.21.15.70 - - [23/Sep/2023:21:16:09] "GET /metrics HTTP/1.1" 200 36359 "" "Prometheus/2.43.0" 2023-09-23T21:16:09.845 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:16:09.863 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:16:09.864 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:16:09.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:16:09 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T21:16:09.964 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:16:10.061 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:16:10.460 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 38 16 57220 57204 94.039 43.3125 0.00881756 0.0106279 2023-09-23T21:16:10.467 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T21:16:10.477 INFO:teuthology.orchestra.run.smithi026.stderr:noscrub is unset 2023-09-23T21:16:10.479 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:16:10.479 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:16:10.580 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:16:10.684 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:16:10 smithi026 ceph-mon[179688]: pgmap v1405: 119 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 4 active+undersized+remapped, 7 remapped+peering, 102 active+clean; 3.3 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 32 MiB/s wr, 512 op/s; 16490/162225 objects degraded (10.165%); 3177/162225 objects misplaced (1.958%); 1.8 MiB/s, 29 objects/s recovering 2023-09-23T21:16:10.685 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:16:10 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/1280110883' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T21:16:10.685 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:16:10 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T21:16:10.690 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:16:10 smithi026 ceph-mon[175151]: pgmap v1405: 119 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 4 active+undersized+remapped, 7 remapped+peering, 102 active+clean; 3.3 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 32 MiB/s wr, 512 op/s; 16490/162225 objects degraded (10.165%); 3177/162225 objects misplaced (1.958%); 1.8 MiB/s, 29 objects/s recovering 2023-09-23T21:16:10.690 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:16:10 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/1280110883' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T21:16:10.690 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:16:10 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T21:16:10.704 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:16:10.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:16:10 smithi070 ceph-mon[168282]: pgmap v1405: 119 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 4 active+undersized+remapped, 7 remapped+peering, 102 active+clean; 3.3 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 32 MiB/s wr, 512 op/s; 16490/162225 objects degraded (10.165%); 3177/162225 objects misplaced (1.958%); 1.8 MiB/s, 29 objects/s recovering 2023-09-23T21:16:10.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:16:10 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/1280110883' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T21:16:10.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:16:10 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T21:16:11.258 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:16:11.270 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:16:11.271 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:16:11.372 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:16:11.467 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 39 16 57966 57950 92.8071 46.625 0.0330996 0.0107672 2023-09-23T21:16:11.481 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:16:11 smithi026 ceph-mon[175151]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:16:11.481 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:16:11 smithi026 ceph-mon[175151]: failed to encode map e556 with expected crc 2023-09-23T21:16:11.481 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:16:11 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T21:16:11.481 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:16:11 smithi026 ceph-mon[175151]: osdmap e556: 8 total, 8 up, 5 in 2023-09-23T21:16:11.481 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:16:11 smithi026 ceph-mon[175151]: failed to encode map e556 with expected crc 2023-09-23T21:16:11.481 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:16:11 smithi026 ceph-mon[175151]: failed to encode map e556 with expected crc 2023-09-23T21:16:11.482 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:16:11 smithi026 ceph-mon[179688]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:16:11.482 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:16:11 smithi026 ceph-mon[179688]: failed to encode map e556 with expected crc 2023-09-23T21:16:11.482 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:16:11 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T21:16:11.482 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:16:11 smithi026 ceph-mon[179688]: osdmap e556: 8 total, 8 up, 5 in 2023-09-23T21:16:11.482 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:16:11 smithi026 ceph-mon[179688]: failed to encode map e556 with expected crc 2023-09-23T21:16:11.482 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:16:11 smithi026 ceph-mon[179688]: failed to encode map e556 with expected crc 2023-09-23T21:16:11.507 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:16:11.672 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:16:11.679 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:16:11.679 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:16:11.780 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:16:11.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:16:11 smithi070 ceph-mon[168282]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:16:11.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:16:11 smithi070 ceph-mon[168282]: failed to encode map e556 with expected crc 2023-09-23T21:16:11.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:16:11 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T21:16:11.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:16:11 smithi070 ceph-mon[168282]: osdmap e556: 8 total, 8 up, 5 in 2023-09-23T21:16:11.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:16:11 smithi070 ceph-mon[168282]: failed to encode map e556 with expected crc 2023-09-23T21:16:11.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:16:11 smithi070 ceph-mon[168282]: failed to encode map e556 with expected crc 2023-09-23T21:16:11.881 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:16:12.188 INFO:teuthology.orchestra.run.smithi026.stdout:{ 2023-09-23T21:16:12.188 INFO:teuthology.orchestra.run.smithi026.stdout: "size": 20, 2023-09-23T21:16:12.188 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 600, 2023-09-23T21:16:12.188 INFO:teuthology.orchestra.run.smithi026.stdout: "ops": [ 2023-09-23T21:16:12.188 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.188 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35856.0:188867 9.e 9:7af985d6:::benchmark_data_smithi026_342732_object75591:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e396)", 2023-09-23T21:16:12.188 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T21:09:07.394276+0000", 2023-09-23T21:16:12.188 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 424.78816650800002, 2023-09-23T21:16:12.188 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.157474551, 2023-09-23T21:16:12.188 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:16:12.189 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:16:12.189 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T21:16:12.189 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35856", 2023-09-23T21:16:12.189 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/4180396258", 2023-09-23T21:16:12.189 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 188867 2023-09-23T21:16:12.189 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.189 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:16:12.189 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.189 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:16:12.189 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:07.394276+0000", 2023-09-23T21:16:12.189 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:16:12.190 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.190 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.190 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:16:12.190 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:07.394276+0000", 2023-09-23T21:16:12.190 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:16:12.190 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.190 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.190 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:16:12.190 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:07.394274+0000", 2023-09-23T21:16:12.190 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T21:16:12.190 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.190 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.191 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:16:12.191 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:07.394282+0000", 2023-09-23T21:16:12.191 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.5320000000000001e-06 2023-09-23T21:16:12.191 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.191 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.191 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:16:12.191 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:07.396944+0000", 2023-09-23T21:16:12.191 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0026620440000000001 2023-09-23T21:16:12.191 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.191 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.191 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:16:12.191 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:07.396961+0000", 2023-09-23T21:16:12.192 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.7005000000000001e-05 2023-09-23T21:16:12.192 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.192 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.192 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:16:12.192 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:07.402354+0000", 2023-09-23T21:16:12.192 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0053932590000000001 2023-09-23T21:16:12.192 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.192 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.192 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T21:16:12.192 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:07.402376+0000", 2023-09-23T21:16:12.192 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.2246e-05 2023-09-23T21:16:12.192 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.193 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.193 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:16:12.193 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:08.395392+0000", 2023-09-23T21:16:12.193 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.99301544900000005 2023-09-23T21:16:12.193 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.193 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.193 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T21:16:12.193 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:08.395405+0000", 2023-09-23T21:16:12.193 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.3638000000000001e-05 2023-09-23T21:16:12.193 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.193 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.193 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:16:12.194 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:09.522891+0000", 2023-09-23T21:16:12.194 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.127485616 2023-09-23T21:16:12.194 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.194 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.194 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:16:12.194 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:09.525704+0000", 2023-09-23T21:16:12.194 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0028126359999999999 2023-09-23T21:16:12.194 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.194 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.194 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 0,4,5", 2023-09-23T21:16:12.194 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:09.526082+0000", 2023-09-23T21:16:12.194 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00037785199999999997 2023-09-23T21:16:12.195 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.195 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.195 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:16:12.195 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:09.542468+0000", 2023-09-23T21:16:12.195 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.016386563999999999 2023-09-23T21:16:12.195 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.195 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.195 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:16:12.195 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:09.542529+0000", 2023-09-23T21:16:12.195 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.1137999999999996e-05 2023-09-23T21:16:12.195 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.195 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.196 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T21:16:12.196 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:09.548525+0000", 2023-09-23T21:16:12.196 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0059955060000000003 2023-09-23T21:16:12.196 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.196 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.196 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:16:12.196 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:09.551567+0000", 2023-09-23T21:16:12.196 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0030419209999999999 2023-09-23T21:16:12.196 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.196 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.196 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T21:16:12.196 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:09.551623+0000", 2023-09-23T21:16:12.196 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.5937000000000003e-05 2023-09-23T21:16:12.197 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.197 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.197 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T21:16:12.197 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:09.551750+0000", 2023-09-23T21:16:12.197 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00012785199999999999 2023-09-23T21:16:12.197 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:16:12.197 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:16:12.197 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:16:12.197 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.197 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.197 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35856.0:188878 9.e 9:74bb71f0:::benchmark_data_smithi026_342732_object75602:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e396)", 2023-09-23T21:16:12.197 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T21:09:07.396973+0000", 2023-09-23T21:16:12.198 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 424.78546972300001, 2023-09-23T21:16:12.198 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.1550696789999999, 2023-09-23T21:16:12.198 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:16:12.198 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:16:12.198 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T21:16:12.198 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35856", 2023-09-23T21:16:12.198 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/4180396258", 2023-09-23T21:16:12.198 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 188878 2023-09-23T21:16:12.198 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.198 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:16:12.198 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.199 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:16:12.199 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:07.396973+0000", 2023-09-23T21:16:12.199 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:16:12.199 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.199 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.199 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:16:12.199 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:07.396973+0000", 2023-09-23T21:16:12.199 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:16:12.199 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.199 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.199 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:16:12.199 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:07.396972+0000", 2023-09-23T21:16:12.199 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.999999 2023-09-23T21:16:12.200 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.200 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.200 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:16:12.200 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:07.396978+0000", 2023-09-23T21:16:12.200 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.3110000000000002e-06 2023-09-23T21:16:12.200 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.200 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.200 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:16:12.200 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:07.396988+0000", 2023-09-23T21:16:12.200 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.8320000000000001e-06 2023-09-23T21:16:12.200 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.200 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.201 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:16:12.201 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:07.396994+0000", 2023-09-23T21:16:12.201 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.3389999999999998e-06 2023-09-23T21:16:12.201 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.201 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.201 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:16:12.201 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:07.402401+0000", 2023-09-23T21:16:12.201 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00540647 2023-09-23T21:16:12.201 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.201 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.201 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T21:16:12.201 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:07.402408+0000", 2023-09-23T21:16:12.202 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.2690000000000002e-06 2023-09-23T21:16:12.202 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.202 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.202 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:16:12.202 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:08.395431+0000", 2023-09-23T21:16:12.202 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.99302307400000001 2023-09-23T21:16:12.202 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.202 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.202 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T21:16:12.202 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:08.395439+0000", 2023-09-23T21:16:12.202 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.2349999999999999e-06 2023-09-23T21:16:12.202 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.203 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.203 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:16:12.203 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:09.542558+0000", 2023-09-23T21:16:12.203 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1471185479999999 2023-09-23T21:16:12.203 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.203 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.203 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:16:12.203 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:09.542701+0000", 2023-09-23T21:16:12.203 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00014349299999999999 2023-09-23T21:16:12.203 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.203 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.203 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 0,4,5", 2023-09-23T21:16:12.204 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:09.542855+0000", 2023-09-23T21:16:12.204 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00015409899999999999 2023-09-23T21:16:12.204 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.204 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.204 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T21:16:12.204 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:09.551436+0000", 2023-09-23T21:16:12.204 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0085808550000000001 2023-09-23T21:16:12.204 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.204 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.204 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:16:12.204 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:09.551486+0000", 2023-09-23T21:16:12.204 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.9679000000000003e-05 2023-09-23T21:16:12.205 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.205 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.205 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:16:12.205 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:09.551531+0000", 2023-09-23T21:16:12.205 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.5185000000000002e-05 2023-09-23T21:16:12.205 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.205 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.205 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:16:12.205 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:09.551848+0000", 2023-09-23T21:16:12.205 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00031644000000000003 2023-09-23T21:16:12.205 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.205 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.205 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T21:16:12.206 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:09.551907+0000", 2023-09-23T21:16:12.206 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.9713000000000002e-05 2023-09-23T21:16:12.206 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.206 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.206 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T21:16:12.206 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:09.552042+0000", 2023-09-23T21:16:12.206 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00013509700000000001 2023-09-23T21:16:12.206 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:16:12.206 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:16:12.206 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:16:12.206 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.206 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.207 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35856.0:188898 9.e 9:7bd64526:::benchmark_data_smithi026_342732_object75622:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e396)", 2023-09-23T21:16:12.207 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T21:09:07.397000+0000", 2023-09-23T21:16:12.207 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 424.78544291100002, 2023-09-23T21:16:12.207 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.1683603769999999, 2023-09-23T21:16:12.207 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:16:12.207 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:16:12.207 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T21:16:12.207 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35856", 2023-09-23T21:16:12.207 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/4180396258", 2023-09-23T21:16:12.207 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 188898 2023-09-23T21:16:12.207 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.207 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:16:12.208 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.208 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:16:12.208 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:07.397000+0000", 2023-09-23T21:16:12.208 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:16:12.208 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.208 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.208 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:16:12.208 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:07.397000+0000", 2023-09-23T21:16:12.208 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:16:12.208 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.208 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.208 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:16:12.208 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:07.396999+0000", 2023-09-23T21:16:12.209 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T21:16:12.209 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.209 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.209 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:16:12.209 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:07.397002+0000", 2023-09-23T21:16:12.209 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.4020000000000002e-06 2023-09-23T21:16:12.209 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.209 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.209 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:16:12.209 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:07.397010+0000", 2023-09-23T21:16:12.209 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.2339999999999997e-06 2023-09-23T21:16:12.209 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.209 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.210 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:16:12.210 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:07.397015+0000", 2023-09-23T21:16:12.210 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.6749999999999999e-06 2023-09-23T21:16:12.210 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.210 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.210 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:16:12.210 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:07.402430+0000", 2023-09-23T21:16:12.210 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0054143790000000004 2023-09-23T21:16:12.210 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.210 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.210 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T21:16:12.210 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:07.402436+0000", 2023-09-23T21:16:12.211 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.5459999999999997e-06 2023-09-23T21:16:12.211 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.211 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.211 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:16:12.211 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:08.395464+0000", 2023-09-23T21:16:12.211 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.99302792500000003 2023-09-23T21:16:12.211 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.211 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.211 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T21:16:12.211 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:08.395480+0000", 2023-09-23T21:16:12.211 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.6220000000000001e-05 2023-09-23T21:16:12.211 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.212 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.212 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:16:12.212 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:09.545709+0000", 2023-09-23T21:16:12.212 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1502284659999999 2023-09-23T21:16:12.212 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.212 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.212 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:16:12.212 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:09.545860+0000", 2023-09-23T21:16:12.212 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00015087 2023-09-23T21:16:12.212 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.212 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.212 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 0,4,5", 2023-09-23T21:16:12.212 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:09.546018+0000", 2023-09-23T21:16:12.213 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000158324 2023-09-23T21:16:12.213 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.213 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.213 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T21:16:12.213 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:09.551495+0000", 2023-09-23T21:16:12.213 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0054766429999999998 2023-09-23T21:16:12.213 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.213 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.213 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:16:12.213 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:09.551808+0000", 2023-09-23T21:16:12.213 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00031368299999999998 2023-09-23T21:16:12.213 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.213 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.214 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:16:12.214 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:09.558471+0000", 2023-09-23T21:16:12.214 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0066625859999999999 2023-09-23T21:16:12.214 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.214 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.214 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:16:12.214 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:09.565161+0000", 2023-09-23T21:16:12.214 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0066899250000000002 2023-09-23T21:16:12.214 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.214 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.214 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T21:16:12.214 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:09.565222+0000", 2023-09-23T21:16:12.215 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.0640999999999998e-05 2023-09-23T21:16:12.215 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.215 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.215 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T21:16:12.215 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:09.565360+0000", 2023-09-23T21:16:12.215 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000138279 2023-09-23T21:16:12.215 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:16:12.215 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:16:12.215 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:16:12.215 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.215 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.215 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35856.0:188916 9.e 9:7db6608b:::benchmark_data_smithi026_342732_object75640:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e396)", 2023-09-23T21:16:12.215 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T21:09:07.397020+0000", 2023-09-23T21:16:12.216 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 424.78542266699998, 2023-09-23T21:16:12.216 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.1686653690000002, 2023-09-23T21:16:12.216 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:16:12.216 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:16:12.216 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T21:16:12.216 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35856", 2023-09-23T21:16:12.216 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/4180396258", 2023-09-23T21:16:12.216 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 188916 2023-09-23T21:16:12.216 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.216 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:16:12.216 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.216 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:16:12.216 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:07.397020+0000", 2023-09-23T21:16:12.217 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:16:12.217 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.217 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.217 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:16:12.217 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:07.397020+0000", 2023-09-23T21:16:12.217 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:16:12.217 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.217 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.217 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:16:12.217 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:07.397019+0000", 2023-09-23T21:16:12.217 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T21:16:12.217 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.218 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.218 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:16:12.218 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:07.397022+0000", 2023-09-23T21:16:12.218 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.2309999999999999e-06 2023-09-23T21:16:12.218 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.218 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.218 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:16:12.218 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:07.397027+0000", 2023-09-23T21:16:12.218 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.5690000000000004e-06 2023-09-23T21:16:12.218 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.218 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.218 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:16:12.219 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:07.397032+0000", 2023-09-23T21:16:12.219 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.3120000000000004e-06 2023-09-23T21:16:12.219 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.219 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.219 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:16:12.219 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:07.402469+0000", 2023-09-23T21:16:12.219 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0054371130000000004 2023-09-23T21:16:12.219 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.219 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.219 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T21:16:12.219 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:07.402488+0000", 2023-09-23T21:16:12.219 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.8858000000000001e-05 2023-09-23T21:16:12.220 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.220 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.220 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:16:12.220 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:08.395505+0000", 2023-09-23T21:16:12.220 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.99301743399999998 2023-09-23T21:16:12.220 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.220 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.220 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T21:16:12.220 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:08.395513+0000", 2023-09-23T21:16:12.220 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.8869999999999992e-06 2023-09-23T21:16:12.220 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.220 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.220 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:16:12.221 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:09.548556+0000", 2023-09-23T21:16:12.221 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.153043475 2023-09-23T21:16:12.221 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.221 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.221 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:16:12.221 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:09.548673+0000", 2023-09-23T21:16:12.221 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000116536 2023-09-23T21:16:12.221 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.221 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.221 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 0,4,5", 2023-09-23T21:16:12.221 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:09.548765+0000", 2023-09-23T21:16:12.221 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.2607000000000003e-05 2023-09-23T21:16:12.222 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.222 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.222 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T21:16:12.222 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:09.551762+0000", 2023-09-23T21:16:12.222 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0029969060000000001 2023-09-23T21:16:12.222 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.222 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.222 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:16:12.222 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:09.552102+0000", 2023-09-23T21:16:12.222 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00033987500000000001 2023-09-23T21:16:12.222 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.222 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.223 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:16:12.223 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:09.558781+0000", 2023-09-23T21:16:12.223 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0066787050000000001 2023-09-23T21:16:12.223 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.223 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.223 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:16:12.223 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:09.565510+0000", 2023-09-23T21:16:12.223 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0067290680000000004 2023-09-23T21:16:12.223 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.223 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.223 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T21:16:12.223 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:09.565562+0000", 2023-09-23T21:16:12.223 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.2034999999999999e-05 2023-09-23T21:16:12.224 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.224 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.224 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T21:16:12.224 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:09.565685+0000", 2023-09-23T21:16:12.224 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00012306799999999999 2023-09-23T21:16:12.224 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:16:12.224 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:16:12.224 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:16:12.224 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.224 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.224 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35856.0:188936 9.e 9:7020b174:::benchmark_data_smithi026_342732_object75660:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e396)", 2023-09-23T21:16:12.224 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T21:09:07.397035+0000", 2023-09-23T21:16:12.224 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 424.78540731499999, 2023-09-23T21:16:12.225 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.181647866, 2023-09-23T21:16:12.225 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:16:12.225 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:16:12.225 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T21:16:12.225 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35856", 2023-09-23T21:16:12.225 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/4180396258", 2023-09-23T21:16:12.225 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 188936 2023-09-23T21:16:12.225 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.225 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:16:12.225 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.225 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:16:12.225 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:07.397035+0000", 2023-09-23T21:16:12.225 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:16:12.226 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.226 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.226 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:16:12.226 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:07.397035+0000", 2023-09-23T21:16:12.226 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:16:12.226 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.226 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.226 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:16:12.226 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:07.397035+0000", 2023-09-23T21:16:12.226 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T21:16:12.226 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.226 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.227 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:16:12.227 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:07.397037+0000", 2023-09-23T21:16:12.227 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.4090000000000001e-06 2023-09-23T21:16:12.227 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.227 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.227 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:16:12.227 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:07.397043+0000", 2023-09-23T21:16:12.227 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.9880000000000001e-06 2023-09-23T21:16:12.227 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.227 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.227 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:16:12.227 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:07.397048+0000", 2023-09-23T21:16:12.227 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.0810000000000001e-06 2023-09-23T21:16:12.228 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.228 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.228 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:16:12.228 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:07.402510+0000", 2023-09-23T21:16:12.228 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.005461367 2023-09-23T21:16:12.228 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.228 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.228 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T21:16:12.228 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:07.402517+0000", 2023-09-23T21:16:12.228 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.9140000000000002e-06 2023-09-23T21:16:12.228 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.228 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.228 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:16:12.229 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:08.395537+0000", 2023-09-23T21:16:12.229 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.99302077200000005 2023-09-23T21:16:12.229 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.229 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.229 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T21:16:12.229 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:08.395545+0000", 2023-09-23T21:16:12.229 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.8120000000000008e-06 2023-09-23T21:16:12.229 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.229 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.229 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:16:12.229 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:09.552130+0000", 2023-09-23T21:16:12.229 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1565847460000001 2023-09-23T21:16:12.229 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.230 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.230 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:16:12.230 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:09.552284+0000", 2023-09-23T21:16:12.230 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00015413699999999999 2023-09-23T21:16:12.230 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.230 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.230 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 0,4,5", 2023-09-23T21:16:12.230 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:09.552445+0000", 2023-09-23T21:16:12.230 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000161049 2023-09-23T21:16:12.230 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.230 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.230 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T21:16:12.231 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:09.558498+0000", 2023-09-23T21:16:12.231 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.006053381 2023-09-23T21:16:12.231 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.231 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.231 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:16:12.231 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:09.558837+0000", 2023-09-23T21:16:12.231 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00033902500000000001 2023-09-23T21:16:12.231 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.231 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.231 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:16:12.231 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:09.565923+0000", 2023-09-23T21:16:12.231 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0070852090000000003 2023-09-23T21:16:12.231 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.232 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.232 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:16:12.232 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:09.578541+0000", 2023-09-23T21:16:12.232 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.012618369000000001 2023-09-23T21:16:12.232 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.232 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.232 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T21:16:12.232 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:09.578597+0000", 2023-09-23T21:16:12.232 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.5949999999999998e-05 2023-09-23T21:16:12.232 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.232 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.232 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T21:16:12.233 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:09.578683+0000", 2023-09-23T21:16:12.233 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.5983000000000007e-05 2023-09-23T21:16:12.233 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:16:12.233 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:16:12.233 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:16:12.233 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.233 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.233 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35856.0:188956 9.e 9:78624dbf:::benchmark_data_smithi026_342732_object75680:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e396)", 2023-09-23T21:16:12.233 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T21:09:07.397052+0000", 2023-09-23T21:16:12.233 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 424.78539016600001, 2023-09-23T21:16:12.233 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.1819009989999998, 2023-09-23T21:16:12.233 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:16:12.234 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:16:12.234 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T21:16:12.234 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35856", 2023-09-23T21:16:12.234 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/4180396258", 2023-09-23T21:16:12.234 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 188956 2023-09-23T21:16:12.234 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.234 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:16:12.234 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.234 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:16:12.234 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:07.397052+0000", 2023-09-23T21:16:12.234 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:16:12.234 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.235 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.235 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:16:12.235 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:07.397052+0000", 2023-09-23T21:16:12.235 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:16:12.235 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.235 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.235 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:16:12.235 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:07.397052+0000", 2023-09-23T21:16:12.235 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T21:16:12.235 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.235 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.235 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:16:12.235 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:07.397054+0000", 2023-09-23T21:16:12.236 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.3259999999999998e-06 2023-09-23T21:16:12.236 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.236 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.236 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:16:12.236 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:07.397060+0000", 2023-09-23T21:16:12.236 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.1299999999999998e-06 2023-09-23T21:16:12.236 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.236 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.236 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:16:12.236 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:07.397065+0000", 2023-09-23T21:16:12.236 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.8659999999999997e-06 2023-09-23T21:16:12.236 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.237 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.237 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:16:12.237 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:07.402545+0000", 2023-09-23T21:16:12.237 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.005479696 2023-09-23T21:16:12.237 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.237 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.237 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T21:16:12.237 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:07.402552+0000", 2023-09-23T21:16:12.237 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.9330000000000004e-06 2023-09-23T21:16:12.237 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.237 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.237 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:16:12.237 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:08.395568+0000", 2023-09-23T21:16:12.238 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.99301622000000001 2023-09-23T21:16:12.238 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.238 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.238 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T21:16:12.238 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:08.395576+0000", 2023-09-23T21:16:12.238 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.2889999999999998e-06 2023-09-23T21:16:12.238 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.238 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.238 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:16:12.238 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:09.555297+0000", 2023-09-23T21:16:12.238 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.159720909 2023-09-23T21:16:12.238 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.238 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.239 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:16:12.239 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:09.555464+0000", 2023-09-23T21:16:12.239 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000166756 2023-09-23T21:16:12.239 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.239 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.239 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 0,4,5", 2023-09-23T21:16:12.239 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:09.555627+0000", 2023-09-23T21:16:12.239 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000162697 2023-09-23T21:16:12.239 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.239 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.239 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:16:12.239 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:09.558888+0000", 2023-09-23T21:16:12.240 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0032610009999999999 2023-09-23T21:16:12.240 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.240 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.240 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T21:16:12.240 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:09.569432+0000", 2023-09-23T21:16:12.240 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.010544661 2023-09-23T21:16:12.240 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.240 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.240 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:16:12.240 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:09.572340+0000", 2023-09-23T21:16:12.240 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0029073549999999999 2023-09-23T21:16:12.240 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.240 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.241 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:16:12.241 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:09.578784+0000", 2023-09-23T21:16:12.241 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0064438840000000004 2023-09-23T21:16:12.241 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.241 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.241 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T21:16:12.241 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:09.578830+0000", 2023-09-23T21:16:12.241 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.6789000000000002e-05 2023-09-23T21:16:12.241 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.241 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.241 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T21:16:12.241 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:09.578953+0000", 2023-09-23T21:16:12.241 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000122774 2023-09-23T21:16:12.242 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:16:12.242 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:16:12.242 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:16:12.242 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.242 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.242 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35856.0:188959 9.e 9:7368e502:::benchmark_data_smithi026_342732_object75683:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e396)", 2023-09-23T21:16:12.242 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T21:09:07.397070+0000", 2023-09-23T21:16:12.242 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 424.78537276700001, 2023-09-23T21:16:12.242 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.182075142, 2023-09-23T21:16:12.242 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:16:12.242 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:16:12.242 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T21:16:12.242 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35856", 2023-09-23T21:16:12.243 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/4180396258", 2023-09-23T21:16:12.243 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 188959 2023-09-23T21:16:12.243 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.243 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:16:12.243 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.243 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:16:12.243 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:07.397070+0000", 2023-09-23T21:16:12.243 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:16:12.243 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.243 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.243 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:16:12.243 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:07.397070+0000", 2023-09-23T21:16:12.244 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:16:12.244 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.244 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.244 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:16:12.244 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:07.397069+0000", 2023-09-23T21:16:12.244 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T21:16:12.244 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.244 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.244 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:16:12.244 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:07.397072+0000", 2023-09-23T21:16:12.244 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.5409999999999999e-06 2023-09-23T21:16:12.244 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.244 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.245 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:16:12.245 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:07.397078+0000", 2023-09-23T21:16:12.245 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.7590000000000003e-06 2023-09-23T21:16:12.245 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.245 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.245 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:16:12.245 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:07.397083+0000", 2023-09-23T21:16:12.245 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.2469999999999999e-06 2023-09-23T21:16:12.245 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.245 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.245 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:16:12.245 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:07.402573+0000", 2023-09-23T21:16:12.245 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0054898539999999997 2023-09-23T21:16:12.246 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.246 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.246 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T21:16:12.246 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:07.402578+0000", 2023-09-23T21:16:12.246 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.6960000000000002e-06 2023-09-23T21:16:12.246 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.246 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.246 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:16:12.246 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:08.395600+0000", 2023-09-23T21:16:12.246 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.99302183499999996 2023-09-23T21:16:12.246 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.246 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.247 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T21:16:12.247 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:08.395608+0000", 2023-09-23T21:16:12.247 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.5630000000000003e-06 2023-09-23T21:16:12.247 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.247 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.247 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:16:12.247 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:09.558920+0000", 2023-09-23T21:16:12.247 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1633124770000001 2023-09-23T21:16:12.247 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.247 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.247 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:16:12.247 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:09.559075+0000", 2023-09-23T21:16:12.247 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000154878 2023-09-23T21:16:12.248 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.248 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.248 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 0,4,5", 2023-09-23T21:16:12.248 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:09.559251+0000", 2023-09-23T21:16:12.248 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00017559800000000001 2023-09-23T21:16:12.248 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.248 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.248 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:16:12.248 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:09.565431+0000", 2023-09-23T21:16:12.248 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0061802020000000001 2023-09-23T21:16:12.248 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.248 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.249 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T21:16:12.249 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:09.569439+0000", 2023-09-23T21:16:12.249 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.004008419 2023-09-23T21:16:12.249 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.249 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.249 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:16:12.249 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:09.572363+0000", 2023-09-23T21:16:12.249 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.002923967 2023-09-23T21:16:12.249 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.249 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.249 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:16:12.249 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:09.579035+0000", 2023-09-23T21:16:12.249 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0066717160000000003 2023-09-23T21:16:12.250 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.250 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.250 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T21:16:12.250 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:09.579080+0000", 2023-09-23T21:16:12.250 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.4631e-05 2023-09-23T21:16:12.250 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.250 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.250 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T21:16:12.250 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:09.579145+0000", 2023-09-23T21:16:12.250 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.5166000000000006e-05 2023-09-23T21:16:12.250 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:16:12.251 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:16:12.251 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:16:12.251 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.251 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.251 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35856.0:188967 9.e 9:7be9207b:::benchmark_data_smithi026_342732_object75691:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e396)", 2023-09-23T21:16:12.251 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T21:09:07.397087+0000", 2023-09-23T21:16:12.251 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 424.785355573, 2023-09-23T21:16:12.251 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.1822950090000002, 2023-09-23T21:16:12.251 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:16:12.251 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:16:12.251 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T21:16:12.251 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35856", 2023-09-23T21:16:12.252 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/4180396258", 2023-09-23T21:16:12.252 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 188967 2023-09-23T21:16:12.252 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.252 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:16:12.252 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.252 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:16:12.252 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:07.397087+0000", 2023-09-23T21:16:12.252 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:16:12.252 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.252 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.252 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:16:12.252 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:07.397087+0000", 2023-09-23T21:16:12.252 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:16:12.253 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.253 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.253 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:16:12.253 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:07.397086+0000", 2023-09-23T21:16:12.253 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T21:16:12.253 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.253 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.253 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:16:12.253 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:07.397089+0000", 2023-09-23T21:16:12.253 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.4990000000000001e-06 2023-09-23T21:16:12.253 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.253 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.253 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:16:12.254 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:07.397095+0000", 2023-09-23T21:16:12.254 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.947e-06 2023-09-23T21:16:12.254 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.254 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.254 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:16:12.254 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:07.397106+0000", 2023-09-23T21:16:12.254 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1267000000000001e-05 2023-09-23T21:16:12.254 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.254 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.254 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:16:12.254 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:07.402597+0000", 2023-09-23T21:16:12.254 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0054908120000000003 2023-09-23T21:16:12.255 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.255 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.255 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T21:16:12.255 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:07.402603+0000", 2023-09-23T21:16:12.255 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.4699999999999999e-06 2023-09-23T21:16:12.255 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.255 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.255 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:16:12.255 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:08.395631+0000", 2023-09-23T21:16:12.255 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.99302780300000004 2023-09-23T21:16:12.255 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.255 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.256 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T21:16:12.256 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:08.395639+0000", 2023-09-23T21:16:12.256 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.4780000000000002e-06 2023-09-23T21:16:12.256 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.256 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.256 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:16:12.256 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:09.562140+0000", 2023-09-23T21:16:12.256 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1665013959999999 2023-09-23T21:16:12.256 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.256 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.256 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:16:12.256 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:09.562309+0000", 2023-09-23T21:16:12.256 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00016833199999999999 2023-09-23T21:16:12.257 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.257 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.257 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 0,4,5", 2023-09-23T21:16:12.257 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:09.562455+0000", 2023-09-23T21:16:12.257 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00014627299999999999 2023-09-23T21:16:12.257 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.257 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.257 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:16:12.257 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:09.565732+0000", 2023-09-23T21:16:12.257 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0032774319999999998 2023-09-23T21:16:12.257 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.257 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.257 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T21:16:12.258 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:09.572296+0000", 2023-09-23T21:16:12.258 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0065640430000000003 2023-09-23T21:16:12.258 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.258 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.258 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:16:12.258 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:09.572382+0000", 2023-09-23T21:16:12.258 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.5730999999999996e-05 2023-09-23T21:16:12.258 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.258 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.258 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:16:12.258 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:09.579214+0000", 2023-09-23T21:16:12.258 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0068315529999999998 2023-09-23T21:16:12.259 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.259 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.259 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T21:16:12.259 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:09.579263+0000", 2023-09-23T21:16:12.259 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.9546000000000003e-05 2023-09-23T21:16:12.259 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.259 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.259 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T21:16:12.259 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:09.579382+0000", 2023-09-23T21:16:12.259 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000118781 2023-09-23T21:16:12.259 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:16:12.259 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:16:12.259 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:16:12.260 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.260 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.260 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35856.0:189010 9.e 9:754d4c4d:::benchmark_data_smithi026_342732_object75734:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e396)", 2023-09-23T21:16:12.260 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T21:09:07.397111+0000", 2023-09-23T21:16:12.260 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 424.78533160299997, 2023-09-23T21:16:12.260 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.188492697, 2023-09-23T21:16:12.260 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:16:12.260 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:16:12.260 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T21:16:12.260 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35856", 2023-09-23T21:16:12.260 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/4180396258", 2023-09-23T21:16:12.260 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 189010 2023-09-23T21:16:12.261 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.261 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:16:12.261 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.261 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:16:12.261 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:07.397111+0000", 2023-09-23T21:16:12.261 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:16:12.261 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.261 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.261 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:16:12.261 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:07.397111+0000", 2023-09-23T21:16:12.261 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:16:12.261 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.261 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.262 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:16:12.262 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:07.397110+0000", 2023-09-23T21:16:12.262 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T21:16:12.262 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.262 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.262 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:16:12.262 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:07.397113+0000", 2023-09-23T21:16:12.262 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.6919999999999998e-06 2023-09-23T21:16:12.262 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.262 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.262 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:16:12.262 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:07.397119+0000", 2023-09-23T21:16:12.263 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.1079999999999998e-06 2023-09-23T21:16:12.263 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.263 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.263 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:16:12.263 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:07.397125+0000", 2023-09-23T21:16:12.263 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.3809999999999998e-06 2023-09-23T21:16:12.263 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.263 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.263 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:16:12.263 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:07.402624+0000", 2023-09-23T21:16:12.263 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0054991320000000003 2023-09-23T21:16:12.263 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.263 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.264 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T21:16:12.264 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:07.402630+0000", 2023-09-23T21:16:12.264 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.0759999999999999e-06 2023-09-23T21:16:12.264 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.264 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.264 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:16:12.264 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:08.395662+0000", 2023-09-23T21:16:12.264 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.99303197700000001 2023-09-23T21:16:12.264 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.264 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.264 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T21:16:12.264 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:08.395669+0000", 2023-09-23T21:16:12.265 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.559e-06 2023-09-23T21:16:12.265 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.265 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.265 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:16:12.265 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:09.566338+0000", 2023-09-23T21:16:12.265 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.170668745 2023-09-23T21:16:12.265 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.265 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.265 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:16:12.265 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:09.566468+0000", 2023-09-23T21:16:12.265 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00012997799999999999 2023-09-23T21:16:12.265 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.266 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.266 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 0,4,5", 2023-09-23T21:16:12.266 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:09.566614+0000", 2023-09-23T21:16:12.266 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000145444 2023-09-23T21:16:12.266 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.266 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.266 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:16:12.266 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:09.572403+0000", 2023-09-23T21:16:12.266 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0057895969999999996 2023-09-23T21:16:12.266 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.266 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.266 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T21:16:12.266 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:09.578691+0000", 2023-09-23T21:16:12.267 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0062879920000000001 2023-09-23T21:16:12.267 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.267 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.267 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:16:12.267 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:09.578723+0000", 2023-09-23T21:16:12.267 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.1377999999999997e-05 2023-09-23T21:16:12.267 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.267 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.267 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:16:12.267 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:09.585401+0000", 2023-09-23T21:16:12.267 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0066785459999999996 2023-09-23T21:16:12.267 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.268 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.268 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T21:16:12.268 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:09.585454+0000", 2023-09-23T21:16:12.268 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.2787999999999997e-05 2023-09-23T21:16:12.268 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.268 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.268 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T21:16:12.268 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:09.585604+0000", 2023-09-23T21:16:12.268 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000149686 2023-09-23T21:16:12.268 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:16:12.268 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:16:12.268 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:16:12.268 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.269 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.269 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35856.0:189039 9.e 9:707fd1cc:::benchmark_data_smithi026_342732_object75763:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e396)", 2023-09-23T21:16:12.269 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T21:09:07.397129+0000", 2023-09-23T21:16:12.269 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 424.78531385500003, 2023-09-23T21:16:12.269 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.1886985710000002, 2023-09-23T21:16:12.269 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:16:12.269 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:16:12.269 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T21:16:12.269 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35856", 2023-09-23T21:16:12.269 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/4180396258", 2023-09-23T21:16:12.269 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 189039 2023-09-23T21:16:12.269 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.270 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:16:12.270 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.270 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:16:12.270 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:07.397129+0000", 2023-09-23T21:16:12.270 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:16:12.270 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.270 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.270 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:16:12.270 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:07.397129+0000", 2023-09-23T21:16:12.270 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:16:12.270 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.270 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.271 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:16:12.271 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:07.397128+0000", 2023-09-23T21:16:12.271 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T21:16:12.271 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.271 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.271 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:16:12.271 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:07.397131+0000", 2023-09-23T21:16:12.271 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.3910000000000001e-06 2023-09-23T21:16:12.271 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.271 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.271 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:16:12.271 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:07.397137+0000", 2023-09-23T21:16:12.271 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.3400000000000003e-06 2023-09-23T21:16:12.272 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.272 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.272 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:16:12.272 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:07.397142+0000", 2023-09-23T21:16:12.272 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.4410000000000001e-06 2023-09-23T21:16:12.272 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.272 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.272 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:16:12.272 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:07.402651+0000", 2023-09-23T21:16:12.272 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0055080709999999998 2023-09-23T21:16:12.272 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.272 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.272 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T21:16:12.273 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:07.402657+0000", 2023-09-23T21:16:12.273 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.3160000000000002e-06 2023-09-23T21:16:12.273 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.273 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.273 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:16:12.273 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:08.395693+0000", 2023-09-23T21:16:12.273 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.99303612500000005 2023-09-23T21:16:12.273 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.273 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.273 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T21:16:12.273 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:08.395700+0000", 2023-09-23T21:16:12.274 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.5170000000000002e-06 2023-09-23T21:16:12.274 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.274 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.274 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:16:12.274 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:09.569468+0000", 2023-09-23T21:16:12.274 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.173767797 2023-09-23T21:16:12.274 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.274 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.274 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:16:12.274 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:09.569598+0000", 2023-09-23T21:16:12.274 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00012940799999999999 2023-09-23T21:16:12.274 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.275 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.275 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 0,4,5", 2023-09-23T21:16:12.275 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:09.569730+0000", 2023-09-23T21:16:12.275 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00013209800000000001 2023-09-23T21:16:12.275 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.275 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.275 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T21:16:12.275 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:09.578695+0000", 2023-09-23T21:16:12.275 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0089652740000000005 2023-09-23T21:16:12.275 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.275 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.275 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:16:12.275 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:09.578748+0000", 2023-09-23T21:16:12.276 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.2713999999999997e-05 2023-09-23T21:16:12.276 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.276 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.276 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:16:12.276 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:09.578996+0000", 2023-09-23T21:16:12.276 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00024833199999999998 2023-09-23T21:16:12.276 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.276 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.276 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:16:12.276 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:09.585652+0000", 2023-09-23T21:16:12.276 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0066561609999999998 2023-09-23T21:16:12.276 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.277 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.277 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T21:16:12.277 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:09.585704+0000", 2023-09-23T21:16:12.277 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.1337e-05 2023-09-23T21:16:12.277 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.277 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.277 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T21:16:12.277 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:09.585827+0000", 2023-09-23T21:16:12.277 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00012356100000000001 2023-09-23T21:16:12.277 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:16:12.277 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:16:12.277 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:16:12.277 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.277 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.278 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35856.0:189079 9.e 9:736d7ce0:::benchmark_data_smithi026_342732_object75803:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e396)", 2023-09-23T21:16:12.278 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T21:09:07.397147+0000", 2023-09-23T21:16:12.278 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 424.78529581200002, 2023-09-23T21:16:12.278 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.1889064239999998, 2023-09-23T21:16:12.278 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:16:12.278 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:16:12.278 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T21:16:12.278 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35856", 2023-09-23T21:16:12.278 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/4180396258", 2023-09-23T21:16:12.278 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 189079 2023-09-23T21:16:12.278 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.278 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:16:12.279 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.279 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:16:12.279 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:07.397147+0000", 2023-09-23T21:16:12.279 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:16:12.279 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.279 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.279 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:16:12.279 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:07.397147+0000", 2023-09-23T21:16:12.279 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:16:12.279 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.279 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.280 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:16:12.280 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:07.397146+0000", 2023-09-23T21:16:12.280 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T21:16:12.280 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.280 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.280 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:16:12.280 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:07.397149+0000", 2023-09-23T21:16:12.280 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.446e-06 2023-09-23T21:16:12.280 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.280 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.280 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:16:12.280 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:07.397155+0000", 2023-09-23T21:16:12.280 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.3369999999999996e-06 2023-09-23T21:16:12.281 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.281 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.281 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:16:12.281 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:07.397160+0000", 2023-09-23T21:16:12.281 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.9749999999999998e-06 2023-09-23T21:16:12.281 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.281 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.281 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:16:12.281 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:07.402677+0000", 2023-09-23T21:16:12.281 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0055166549999999996 2023-09-23T21:16:12.281 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.281 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.281 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T21:16:12.282 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:07.402682+0000", 2023-09-23T21:16:12.282 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.5539999999999997e-06 2023-09-23T21:16:12.282 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.282 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.282 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:16:12.282 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:08.395724+0000", 2023-09-23T21:16:12.282 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.99304205300000004 2023-09-23T21:16:12.282 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.282 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.282 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T21:16:12.282 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:08.395732+0000", 2023-09-23T21:16:12.282 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.2009999999999999e-06 2023-09-23T21:16:12.283 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.283 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.283 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:16:12.283 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:09.572418+0000", 2023-09-23T21:16:12.283 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1766850739999999 2023-09-23T21:16:12.283 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.283 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.283 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:16:12.283 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:09.572528+0000", 2023-09-23T21:16:12.283 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000110073 2023-09-23T21:16:12.283 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.283 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.283 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 0,4,5", 2023-09-23T21:16:12.284 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:09.572664+0000", 2023-09-23T21:16:12.284 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00013662200000000001 2023-09-23T21:16:12.284 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.284 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.284 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T21:16:12.284 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:09.578699+0000", 2023-09-23T21:16:12.284 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0060349169999999999 2023-09-23T21:16:12.284 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.284 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.284 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:16:12.284 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:09.579176+0000", 2023-09-23T21:16:12.284 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000477042 2023-09-23T21:16:12.285 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.285 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.285 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:16:12.285 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:09.585344+0000", 2023-09-23T21:16:12.285 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0061675410000000003 2023-09-23T21:16:12.285 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.285 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.285 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:16:12.285 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:09.585882+0000", 2023-09-23T21:16:12.285 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00053863299999999995 2023-09-23T21:16:12.285 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.285 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.285 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T21:16:12.286 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:09.585929+0000", 2023-09-23T21:16:12.286 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.7030999999999997e-05 2023-09-23T21:16:12.286 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.286 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.286 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T21:16:12.286 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:09.586053+0000", 2023-09-23T21:16:12.286 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00012363000000000001 2023-09-23T21:16:12.286 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:16:12.286 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:16:12.286 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:16:12.286 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.286 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.287 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35856.0:189134 9.e 9:723254c9:::benchmark_data_smithi026_342732_object75858:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e396)", 2023-09-23T21:16:12.287 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T21:09:07.397164+0000", 2023-09-23T21:16:12.287 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 424.78527829900003, 2023-09-23T21:16:12.287 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.2140037029999999, 2023-09-23T21:16:12.287 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:16:12.287 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:16:12.287 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T21:16:12.287 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35856", 2023-09-23T21:16:12.287 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/4180396258", 2023-09-23T21:16:12.287 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 189134 2023-09-23T21:16:12.287 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.287 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:16:12.288 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.288 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:16:12.288 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:07.397164+0000", 2023-09-23T21:16:12.288 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:16:12.288 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.288 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.288 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:16:12.288 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:07.397164+0000", 2023-09-23T21:16:12.288 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:16:12.288 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.288 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.288 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:16:12.288 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:07.397164+0000", 2023-09-23T21:16:12.289 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T21:16:12.289 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.289 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.289 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:16:12.289 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:07.397172+0000", 2023-09-23T21:16:12.289 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.225e-06 2023-09-23T21:16:12.289 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.289 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.289 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:16:12.289 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:07.397179+0000", 2023-09-23T21:16:12.289 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.5320000000000003e-06 2023-09-23T21:16:12.289 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.289 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.290 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:16:12.290 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:07.397184+0000", 2023-09-23T21:16:12.290 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.2279999999999998e-06 2023-09-23T21:16:12.290 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.290 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.290 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:16:12.290 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:07.402701+0000", 2023-09-23T21:16:12.290 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0055169479999999998 2023-09-23T21:16:12.290 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.290 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.290 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T21:16:12.290 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:07.402707+0000", 2023-09-23T21:16:12.291 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.3500000000000002e-06 2023-09-23T21:16:12.291 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.291 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.291 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:16:12.291 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:08.395757+0000", 2023-09-23T21:16:12.291 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.99305015799999996 2023-09-23T21:16:12.291 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.291 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.291 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T21:16:12.291 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:08.395765+0000", 2023-09-23T21:16:12.291 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.5870000000000004e-06 2023-09-23T21:16:12.291 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.291 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.292 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:16:12.292 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:09.575078+0000", 2023-09-23T21:16:12.292 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.179313482 2023-09-23T21:16:12.292 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.292 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.292 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:16:12.292 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:09.575192+0000", 2023-09-23T21:16:12.292 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00011385199999999999 2023-09-23T21:16:12.292 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.292 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.292 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 0,4,5", 2023-09-23T21:16:12.292 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:09.575323+0000", 2023-09-23T21:16:12.293 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000130524 2023-09-23T21:16:12.293 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.293 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.293 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T21:16:12.293 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:09.578727+0000", 2023-09-23T21:16:12.293 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0034039579999999999 2023-09-23T21:16:12.293 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.293 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.293 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:16:12.293 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:09.579430+0000", 2023-09-23T21:16:12.293 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00070300600000000005 2023-09-23T21:16:12.293 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.293 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.294 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:16:12.294 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:09.586100+0000", 2023-09-23T21:16:12.294 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0066703910000000003 2023-09-23T21:16:12.294 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.294 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.294 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:16:12.294 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:09.610991+0000", 2023-09-23T21:16:12.294 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.024891309 2023-09-23T21:16:12.294 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.294 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.294 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T21:16:12.294 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:09.611043+0000", 2023-09-23T21:16:12.295 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.1882999999999998e-05 2023-09-23T21:16:12.295 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.295 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.295 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T21:16:12.295 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:09.611168+0000", 2023-09-23T21:16:12.295 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000124623 2023-09-23T21:16:12.295 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:16:12.295 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:16:12.295 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:16:12.295 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.295 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.295 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35856.0:189139 9.e 9:71d43f8f:::benchmark_data_smithi026_342732_object75863:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e396)", 2023-09-23T21:16:12.296 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T21:09:07.397188+0000", 2023-09-23T21:16:12.296 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 424.78525465400003, 2023-09-23T21:16:12.296 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.2142602409999999, 2023-09-23T21:16:12.296 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:16:12.296 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:16:12.296 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T21:16:12.296 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35856", 2023-09-23T21:16:12.296 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/4180396258", 2023-09-23T21:16:12.296 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 189139 2023-09-23T21:16:12.296 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.296 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:16:12.296 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.296 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:16:12.297 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:07.397188+0000", 2023-09-23T21:16:12.297 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:16:12.297 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.297 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.297 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:16:12.297 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:07.397188+0000", 2023-09-23T21:16:12.297 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:16:12.297 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.297 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.297 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:16:12.297 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:07.397187+0000", 2023-09-23T21:16:12.297 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T21:16:12.297 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.298 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.298 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:16:12.298 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:07.397190+0000", 2023-09-23T21:16:12.298 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.1119999999999999e-06 2023-09-23T21:16:12.298 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.298 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.298 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:16:12.298 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:07.397195+0000", 2023-09-23T21:16:12.298 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.8000000000000004e-06 2023-09-23T21:16:12.298 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.298 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.298 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:16:12.298 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:07.397200+0000", 2023-09-23T21:16:12.299 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.6639999999999997e-06 2023-09-23T21:16:12.299 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.299 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.299 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:16:12.299 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:07.402726+0000", 2023-09-23T21:16:12.299 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0055261060000000002 2023-09-23T21:16:12.299 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.299 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.299 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T21:16:12.299 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:07.402732+0000", 2023-09-23T21:16:12.299 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.2990000000000002e-06 2023-09-23T21:16:12.299 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.300 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.300 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:16:12.300 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:08.395789+0000", 2023-09-23T21:16:12.300 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.99305684100000002 2023-09-23T21:16:12.300 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.300 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.300 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T21:16:12.300 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:08.395797+0000", 2023-09-23T21:16:12.300 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.6930000000000007e-06 2023-09-23T21:16:12.300 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.300 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.300 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:16:12.301 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:09.579458+0000", 2023-09-23T21:16:12.301 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.183661047 2023-09-23T21:16:12.301 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.301 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.301 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:16:12.301 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:09.579599+0000", 2023-09-23T21:16:12.301 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000140731 2023-09-23T21:16:12.301 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.301 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.301 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 0,4,5", 2023-09-23T21:16:12.301 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:09.579718+0000", 2023-09-23T21:16:12.301 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000119183 2023-09-23T21:16:12.301 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.302 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.302 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T21:16:12.302 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:09.585354+0000", 2023-09-23T21:16:12.302 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0056359349999999999 2023-09-23T21:16:12.302 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.302 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.302 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:16:12.302 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:09.610947+0000", 2023-09-23T21:16:12.302 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.025592658000000001 2023-09-23T21:16:12.302 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.302 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.302 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:16:12.303 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:09.611210+0000", 2023-09-23T21:16:12.303 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00026378600000000001 2023-09-23T21:16:12.303 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.303 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.303 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:16:12.303 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:09.611283+0000", 2023-09-23T21:16:12.303 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.2791000000000002e-05 2023-09-23T21:16:12.303 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.303 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.303 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T21:16:12.303 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:09.611331+0000", 2023-09-23T21:16:12.303 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.8251000000000003e-05 2023-09-23T21:16:12.304 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.304 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.304 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T21:16:12.304 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:09.611448+0000", 2023-09-23T21:16:12.304 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000116659 2023-09-23T21:16:12.304 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:16:12.304 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:16:12.304 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:16:12.304 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.304 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.304 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35856.0:189144 9.e 9:7527167d:::benchmark_data_smithi026_342732_object75868:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e396)", 2023-09-23T21:16:12.304 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T21:09:07.397204+0000", 2023-09-23T21:16:12.305 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 424.78523828300001, 2023-09-23T21:16:12.305 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.214518773, 2023-09-23T21:16:12.305 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:16:12.305 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:16:12.305 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T21:16:12.305 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35856", 2023-09-23T21:16:12.305 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/4180396258", 2023-09-23T21:16:12.305 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 189144 2023-09-23T21:16:12.305 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.305 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:16:12.305 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.305 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:16:12.306 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:07.397204+0000", 2023-09-23T21:16:12.306 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:16:12.306 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.306 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.306 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:16:12.306 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:07.397204+0000", 2023-09-23T21:16:12.306 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:16:12.306 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.306 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.306 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:16:12.306 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:07.397204+0000", 2023-09-23T21:16:12.306 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T21:16:12.307 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.307 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.307 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:16:12.307 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:07.397206+0000", 2023-09-23T21:16:12.307 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.39e-06 2023-09-23T21:16:12.307 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.307 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.307 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:16:12.307 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:07.397212+0000", 2023-09-23T21:16:12.307 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.2009999999999996e-06 2023-09-23T21:16:12.307 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.307 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.308 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:16:12.308 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:07.397217+0000", 2023-09-23T21:16:12.308 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.6190000000000002e-06 2023-09-23T21:16:12.308 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.308 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.308 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:16:12.308 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:07.402753+0000", 2023-09-23T21:16:12.308 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0055363239999999996 2023-09-23T21:16:12.308 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.308 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.308 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T21:16:12.308 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:07.402759+0000", 2023-09-23T21:16:12.308 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.1249999999999998e-06 2023-09-23T21:16:12.309 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.309 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.309 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:16:12.309 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:08.395821+0000", 2023-09-23T21:16:12.309 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.99306115800000005 2023-09-23T21:16:12.309 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.309 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.309 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T21:16:12.309 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:08.395828+0000", 2023-09-23T21:16:12.310 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.7069999999999992e-06 2023-09-23T21:16:12.310 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.310 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.310 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:16:12.310 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:09.582488+0000", 2023-09-23T21:16:12.310 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1866593400000001 2023-09-23T21:16:12.311 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.311 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.311 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:16:12.311 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:09.582642+0000", 2023-09-23T21:16:12.311 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000154349 2023-09-23T21:16:12.311 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.311 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.311 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 0,4,5", 2023-09-23T21:16:12.311 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:09.582788+0000", 2023-09-23T21:16:12.311 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00014585600000000001 2023-09-23T21:16:12.311 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.311 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.312 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T21:16:12.312 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:09.585841+0000", 2023-09-23T21:16:12.312 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00305273 2023-09-23T21:16:12.312 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.312 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.312 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:16:12.312 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:09.611247+0000", 2023-09-23T21:16:12.312 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.025406245000000001 2023-09-23T21:16:12.312 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.312 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.312 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:16:12.312 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:09.611497+0000", 2023-09-23T21:16:12.313 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000250275 2023-09-23T21:16:12.313 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.313 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.313 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:16:12.313 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:09.611563+0000", 2023-09-23T21:16:12.313 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.6006000000000005e-05 2023-09-23T21:16:12.313 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.313 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.313 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T21:16:12.313 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:09.611604+0000", 2023-09-23T21:16:12.313 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.0479000000000003e-05 2023-09-23T21:16:12.313 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.313 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.314 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T21:16:12.314 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:09.611723+0000", 2023-09-23T21:16:12.314 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000119325 2023-09-23T21:16:12.314 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:16:12.314 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:16:12.314 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:16:12.314 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.314 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.314 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35856.0:189187 9.e 9:7021101b:::benchmark_data_smithi026_342732_object75911:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e396)", 2023-09-23T21:16:12.314 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T21:09:07.397221+0000", 2023-09-23T21:16:12.314 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 424.78522162299998, 2023-09-23T21:16:12.314 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.2150392590000001, 2023-09-23T21:16:12.315 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:16:12.315 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:16:12.315 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T21:16:12.315 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35856", 2023-09-23T21:16:12.315 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/4180396258", 2023-09-23T21:16:12.315 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 189187 2023-09-23T21:16:12.315 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.315 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:16:12.315 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.315 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:16:12.315 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:07.397221+0000", 2023-09-23T21:16:12.315 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:16:12.315 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.316 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.316 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:16:12.316 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:07.397221+0000", 2023-09-23T21:16:12.316 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:16:12.316 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.316 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.316 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:16:12.316 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:07.397220+0000", 2023-09-23T21:16:12.316 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T21:16:12.316 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.316 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.316 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:16:12.317 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:07.397223+0000", 2023-09-23T21:16:12.317 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.0949999999999998e-06 2023-09-23T21:16:12.317 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.317 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.317 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:16:12.317 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:07.397228+0000", 2023-09-23T21:16:12.317 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.1789999999999999e-06 2023-09-23T21:16:12.317 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.317 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.317 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:16:12.317 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:07.397233+0000", 2023-09-23T21:16:12.317 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.9740000000000001e-06 2023-09-23T21:16:12.317 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.318 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.318 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:16:12.318 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:07.402781+0000", 2023-09-23T21:16:12.318 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0055482609999999996 2023-09-23T21:16:12.318 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.318 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.318 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T21:16:12.318 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:07.402788+0000", 2023-09-23T21:16:12.318 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.5350000000000001e-06 2023-09-23T21:16:12.318 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.318 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.319 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:16:12.319 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:08.395853+0000", 2023-09-23T21:16:12.319 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.99306562899999995 2023-09-23T21:16:12.319 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.319 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.319 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T21:16:12.319 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:08.395861+0000", 2023-09-23T21:16:12.319 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.7680000000000008e-06 2023-09-23T21:16:12.319 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.319 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.319 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:16:12.319 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:09.586126+0000", 2023-09-23T21:16:12.319 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1902646290000001 2023-09-23T21:16:12.320 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.320 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.320 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:16:12.320 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:09.586267+0000", 2023-09-23T21:16:12.320 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00014103 2023-09-23T21:16:12.320 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.320 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.320 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 0,4,5", 2023-09-23T21:16:12.320 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:09.586415+0000", 2023-09-23T21:16:12.320 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000147971 2023-09-23T21:16:12.320 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.320 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.321 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T21:16:12.321 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:09.610957+0000", 2023-09-23T21:16:12.321 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.024542325 2023-09-23T21:16:12.321 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.321 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.321 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:16:12.321 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:09.611531+0000", 2023-09-23T21:16:12.321 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00057401399999999997 2023-09-23T21:16:12.321 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.321 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.321 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:16:12.321 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:09.611761+0000", 2023-09-23T21:16:12.321 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000230304 2023-09-23T21:16:12.322 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.322 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.322 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:16:12.322 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:09.612100+0000", 2023-09-23T21:16:12.322 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00033894699999999998 2023-09-23T21:16:12.322 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.322 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.322 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T21:16:12.322 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:09.612144+0000", 2023-09-23T21:16:12.322 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.3738e-05 2023-09-23T21:16:12.322 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.322 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.323 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T21:16:12.323 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:09.612260+0000", 2023-09-23T21:16:12.323 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000116191 2023-09-23T21:16:12.323 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:16:12.323 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:16:12.323 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:16:12.326 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.326 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.326 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35856.0:189207 9.e 9:77e2cdae:::benchmark_data_smithi026_342732_object75931:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e396)", 2023-09-23T21:16:12.326 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T21:09:07.397237+0000", 2023-09-23T21:16:12.326 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 424.78520576900002, 2023-09-23T21:16:12.326 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.2290301229999998, 2023-09-23T21:16:12.326 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:16:12.327 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:16:12.327 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T21:16:12.327 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35856", 2023-09-23T21:16:12.327 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/4180396258", 2023-09-23T21:16:12.327 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 189207 2023-09-23T21:16:12.327 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.327 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:16:12.327 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.327 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:16:12.327 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:07.397237+0000", 2023-09-23T21:16:12.327 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:16:12.327 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.327 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.328 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:16:12.328 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:07.397237+0000", 2023-09-23T21:16:12.328 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:16:12.328 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.328 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.328 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:16:12.328 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:07.397236+0000", 2023-09-23T21:16:12.328 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T21:16:12.328 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.328 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.328 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:16:12.328 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:07.397238+0000", 2023-09-23T21:16:12.329 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.1919999999999999e-06 2023-09-23T21:16:12.329 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.329 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.329 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:16:12.329 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:07.397244+0000", 2023-09-23T21:16:12.329 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.6969999999999998e-06 2023-09-23T21:16:12.329 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.329 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.329 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:16:12.329 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:07.397249+0000", 2023-09-23T21:16:12.329 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.3819999999999997e-06 2023-09-23T21:16:12.329 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.329 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.330 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:16:12.330 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:07.402809+0000", 2023-09-23T21:16:12.330 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0055603400000000004 2023-09-23T21:16:12.330 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.330 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.330 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T21:16:12.330 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:07.402815+0000", 2023-09-23T21:16:12.330 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.0290000000000002e-06 2023-09-23T21:16:12.330 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.330 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.330 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:16:12.330 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:08.395886+0000", 2023-09-23T21:16:12.331 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.99307133599999997 2023-09-23T21:16:12.331 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.331 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.331 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T21:16:12.331 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:08.395894+0000", 2023-09-23T21:16:12.331 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.2109999999999998e-06 2023-09-23T21:16:12.331 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.331 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.331 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:16:12.331 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:09.594412+0000", 2023-09-23T21:16:12.331 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.198517316 2023-09-23T21:16:12.331 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.331 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.332 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:16:12.332 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:09.594555+0000", 2023-09-23T21:16:12.332 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000143366 2023-09-23T21:16:12.332 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.332 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.332 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 0,4,5", 2023-09-23T21:16:12.332 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:09.608184+0000", 2023-09-23T21:16:12.332 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.013628707 2023-09-23T21:16:12.332 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.332 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.332 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:16:12.332 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:09.615278+0000", 2023-09-23T21:16:12.333 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0070938980000000004 2023-09-23T21:16:12.333 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.333 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.333 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:16:12.333 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:09.615309+0000", 2023-09-23T21:16:12.333 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.0870000000000001e-05 2023-09-23T21:16:12.333 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.333 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.333 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:16:12.333 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:09.618125+0000", 2023-09-23T21:16:12.333 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.002816416 2023-09-23T21:16:12.333 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.334 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.334 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T21:16:12.334 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:09.626083+0000", 2023-09-23T21:16:12.334 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0079576569999999999 2023-09-23T21:16:12.334 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.334 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.334 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T21:16:12.334 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:09.626140+0000", 2023-09-23T21:16:12.334 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.7599999999999997e-05 2023-09-23T21:16:12.334 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.334 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.334 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T21:16:12.335 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:09:09.626267+0000", 2023-09-23T21:16:12.335 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00012646599999999999 2023-09-23T21:16:12.335 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:16:12.335 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:16:12.335 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:16:12.335 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.335 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.335 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.46328.0:99961 10.b 10:d80f00d3:::benchmark_data_smithi026_379838_object20004:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e492)", 2023-09-23T21:16:12.335 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T21:13:11.149471+0000", 2023-09-23T21:16:12.335 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 181.032971457, 2023-09-23T21:16:12.335 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.811917964, 2023-09-23T21:16:12.335 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:16:12.336 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "reached pg", 2023-09-23T21:16:12.336 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T21:16:12.336 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.46328", 2023-09-23T21:16:12.336 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/3748861181", 2023-09-23T21:16:12.336 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 99961 2023-09-23T21:16:12.336 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.336 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:16:12.336 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.336 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:16:12.336 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:11.149471+0000", 2023-09-23T21:16:12.336 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:16:12.336 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.337 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.337 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:16:12.337 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:11.149471+0000", 2023-09-23T21:16:12.337 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:16:12.337 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.337 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.337 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:16:12.337 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:11.149469+0000", 2023-09-23T21:16:12.337 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T21:16:12.337 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.337 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.337 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:16:12.338 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:11.149484+0000", 2023-09-23T21:16:12.338 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.4637000000000001e-05 2023-09-23T21:16:12.338 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.338 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.338 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:16:12.338 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:11.149503+0000", 2023-09-23T21:16:12.338 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.8737999999999999e-05 2023-09-23T21:16:12.338 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.338 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.338 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:16:12.338 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:11.149512+0000", 2023-09-23T21:16:12.338 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.3370000000000006e-06 2023-09-23T21:16:12.339 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.339 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.339 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:16:12.339 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:11.150665+0000", 2023-09-23T21:16:12.339 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0011530850000000001 2023-09-23T21:16:12.339 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.339 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.339 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:16:12.339 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:11.150795+0000", 2023-09-23T21:16:12.339 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000129398 2023-09-23T21:16:12.339 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.339 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.340 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 4,7", 2023-09-23T21:16:12.340 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:11.150882+0000", 2023-09-23T21:16:12.340 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.6952000000000004e-05 2023-09-23T21:16:12.340 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.340 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.340 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T21:16:12.340 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:11.153502+0000", 2023-09-23T21:16:12.340 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0026203039999999999 2023-09-23T21:16:12.340 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.340 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.340 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:16:12.340 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:11.156965+0000", 2023-09-23T21:16:12.340 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0034635500000000001 2023-09-23T21:16:12.341 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.341 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.341 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:16:12.341 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:11.185381+0000", 2023-09-23T21:16:12.341 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.028415947 2023-09-23T21:16:12.341 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.341 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.341 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T21:16:12.341 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:11.185397+0000", 2023-09-23T21:16:12.341 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.5376e-05 2023-09-23T21:16:12.341 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.341 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.341 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:16:12.342 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:12.183496+0000", 2023-09-23T21:16:12.342 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.99809900100000004 2023-09-23T21:16:12.342 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.342 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.342 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T21:16:12.342 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:12.183513+0000", 2023-09-23T21:16:12.342 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.7025999999999999e-05 2023-09-23T21:16:12.342 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.342 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.342 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:16:12.342 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:14.950804+0000", 2023-09-23T21:16:12.342 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.7672911330000001 2023-09-23T21:16:12.343 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.343 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.343 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for missing object", 2023-09-23T21:16:12.343 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:14.950913+0000", 2023-09-23T21:16:12.343 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00010914 2023-09-23T21:16:12.343 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.343 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.343 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:16:12.343 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:14.961306+0000", 2023-09-23T21:16:12.343 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.010392409999999999 2023-09-23T21:16:12.343 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.343 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.343 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T21:16:12.344 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:14.961389+0000", 2023-09-23T21:16:12.344 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.3421e-05 2023-09-23T21:16:12.344 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:16:12.344 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:16:12.344 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:16:12.344 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.344 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.344 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.46328.0:99991 10.b 10:dccafc24:::benchmark_data_smithi026_379838_object20034:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e495)", 2023-09-23T21:16:12.344 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T21:13:13.432695+0000", 2023-09-23T21:16:12.344 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 178.74974766899999, 2023-09-23T21:16:12.344 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.1350699199999998, 2023-09-23T21:16:12.344 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:16:12.345 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:16:12.345 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T21:16:12.345 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.46328", 2023-09-23T21:16:12.345 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/3748861181", 2023-09-23T21:16:12.345 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 99991 2023-09-23T21:16:12.345 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.345 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:16:12.345 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.345 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:16:12.345 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:13.432695+0000", 2023-09-23T21:16:12.345 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:16:12.345 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.345 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.346 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:16:12.346 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:13.432695+0000", 2023-09-23T21:16:12.346 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:16:12.346 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.346 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.346 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:16:12.346 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:13.432692+0000", 2023-09-23T21:16:12.346 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999976 2023-09-23T21:16:12.346 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.346 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.346 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:16:12.347 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:13.432704+0000", 2023-09-23T21:16:12.347 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1975e-05 2023-09-23T21:16:12.347 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.347 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.347 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:16:12.347 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:13.432730+0000", 2023-09-23T21:16:12.347 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.5446e-05 2023-09-23T21:16:12.347 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.347 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.347 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:16:12.347 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:13.432741+0000", 2023-09-23T21:16:12.347 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1083e-05 2023-09-23T21:16:12.348 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.348 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.348 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:16:12.348 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:13.432838+0000", 2023-09-23T21:16:12.348 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.6885999999999994e-05 2023-09-23T21:16:12.348 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.348 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.348 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T21:16:12.348 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:13.432851+0000", 2023-09-23T21:16:12.348 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.2723e-05 2023-09-23T21:16:12.348 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.348 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.348 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:16:12.349 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:14.951924+0000", 2023-09-23T21:16:12.349 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.519073458 2023-09-23T21:16:12.349 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.349 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.349 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:16:12.349 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:14.952032+0000", 2023-09-23T21:16:12.349 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000107868 2023-09-23T21:16:12.349 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.349 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.349 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 0,4,7", 2023-09-23T21:16:12.349 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:14.952124+0000", 2023-09-23T21:16:12.349 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.2560999999999997e-05 2023-09-23T21:16:12.349 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.350 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.350 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T21:16:12.350 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:14.958867+0000", 2023-09-23T21:16:12.350 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0067425769999999996 2023-09-23T21:16:12.350 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.350 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.350 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:16:12.350 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:14.958916+0000", 2023-09-23T21:16:12.350 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.9147999999999998e-05 2023-09-23T21:16:12.350 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.350 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.350 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:16:12.351 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:14.961436+0000", 2023-09-23T21:16:12.351 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0025194560000000002 2023-09-23T21:16:12.351 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.351 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.351 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:16:12.351 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:15.565415+0000", 2023-09-23T21:16:12.351 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.60397895599999996 2023-09-23T21:16:12.351 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.351 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.351 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T21:16:12.351 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:15.565489+0000", 2023-09-23T21:16:12.351 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.4133000000000003e-05 2023-09-23T21:16:12.352 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.352 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.352 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T21:16:12.352 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:15.567765+0000", 2023-09-23T21:16:12.352 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0022759690000000001 2023-09-23T21:16:12.352 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:16:12.352 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:16:12.352 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:16:12.352 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.352 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.352 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.46328.0:99996 10.b 10:d8bf44a0:::benchmark_data_smithi026_379838_object20039:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e495)", 2023-09-23T21:16:12.352 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T21:13:13.440762+0000", 2023-09-23T21:16:12.352 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 178.741680387, 2023-09-23T21:16:12.353 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.1321380149999998, 2023-09-23T21:16:12.353 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:16:12.353 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:16:12.353 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T21:16:12.353 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.46328", 2023-09-23T21:16:12.353 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/3748861181", 2023-09-23T21:16:12.353 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 99996 2023-09-23T21:16:12.353 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.353 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:16:12.353 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.353 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:16:12.353 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:13.440762+0000", 2023-09-23T21:16:12.354 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:16:12.354 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.354 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.354 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:16:12.354 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:13.440762+0000", 2023-09-23T21:16:12.354 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:16:12.354 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.354 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.354 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:16:12.354 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:13.440760+0000", 2023-09-23T21:16:12.354 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999981 2023-09-23T21:16:12.354 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.354 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.355 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:16:12.355 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:13.440770+0000", 2023-09-23T21:16:12.355 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.9359999999999994e-06 2023-09-23T21:16:12.355 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.355 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.355 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:16:12.355 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:13.440793+0000", 2023-09-23T21:16:12.355 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.2983e-05 2023-09-23T21:16:12.355 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.355 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.355 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:16:12.355 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:13.440803+0000", 2023-09-23T21:16:12.356 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0256e-05 2023-09-23T21:16:12.356 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.356 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.356 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:16:12.356 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:13.440853+0000", 2023-09-23T21:16:12.356 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.9181999999999998e-05 2023-09-23T21:16:12.356 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.356 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.356 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T21:16:12.356 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:13.440861+0000", 2023-09-23T21:16:12.356 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.5320000000000006e-06 2023-09-23T21:16:12.357 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.357 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.357 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:16:12.357 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:14.954519+0000", 2023-09-23T21:16:12.357 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.5136580290000001 2023-09-23T21:16:12.357 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.357 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.357 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:16:12.357 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:14.954615+0000", 2023-09-23T21:16:12.357 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.5758e-05 2023-09-23T21:16:12.357 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.357 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.357 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 0,4,7", 2023-09-23T21:16:12.358 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:14.954702+0000", 2023-09-23T21:16:12.358 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.6988000000000001e-05 2023-09-23T21:16:12.358 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.358 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.358 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T21:16:12.358 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:14.961280+0000", 2023-09-23T21:16:12.358 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0065785490000000004 2023-09-23T21:16:12.358 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.358 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.358 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:16:12.358 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:14.961462+0000", 2023-09-23T21:16:12.358 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00018181800000000001 2023-09-23T21:16:12.359 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.359 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.359 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:16:12.359 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:14.961562+0000", 2023-09-23T21:16:12.359 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.9278999999999996e-05 2023-09-23T21:16:12.359 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.359 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.359 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:16:12.359 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:15.570540+0000", 2023-09-23T21:16:12.359 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.60897849199999998 2023-09-23T21:16:12.359 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.359 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.360 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T21:16:12.360 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:15.570591+0000", 2023-09-23T21:16:12.360 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.0818e-05 2023-09-23T21:16:12.360 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.360 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.360 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T21:16:12.360 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:15.572900+0000", 2023-09-23T21:16:12.360 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.002309195 2023-09-23T21:16:12.360 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:16:12.360 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:16:12.360 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:16:12.360 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.360 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.361 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.46328.0:100011 10.b 10:d7ee5e06:::benchmark_data_smithi026_379838_object20054:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e495)", 2023-09-23T21:16:12.361 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T21:13:13.547090+0000", 2023-09-23T21:16:12.361 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 178.635352681, 2023-09-23T21:16:12.361 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.0284788900000001, 2023-09-23T21:16:12.361 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:16:12.361 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:16:12.361 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T21:16:12.361 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.46328", 2023-09-23T21:16:12.361 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/3748861181", 2023-09-23T21:16:12.361 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 100011 2023-09-23T21:16:12.361 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.361 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:16:12.361 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.362 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:16:12.362 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:13.547090+0000", 2023-09-23T21:16:12.362 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:16:12.362 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.362 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.362 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:16:12.362 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:13.547090+0000", 2023-09-23T21:16:12.362 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:16:12.362 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.362 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.362 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:16:12.362 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:13.547088+0000", 2023-09-23T21:16:12.363 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999981 2023-09-23T21:16:12.363 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.363 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.363 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:16:12.363 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:13.547098+0000", 2023-09-23T21:16:12.363 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.5470000000000002e-06 2023-09-23T21:16:12.363 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.363 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.363 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:16:12.363 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:13.547118+0000", 2023-09-23T21:16:12.363 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.0462000000000002e-05 2023-09-23T21:16:12.363 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.364 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.364 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:16:12.364 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:13.547126+0000", 2023-09-23T21:16:12.364 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.0450000000000001e-06 2023-09-23T21:16:12.364 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.364 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.364 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:16:12.364 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:13.547187+0000", 2023-09-23T21:16:12.364 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.1297999999999995e-05 2023-09-23T21:16:12.364 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.364 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.364 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T21:16:12.365 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:13.547199+0000", 2023-09-23T21:16:12.365 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1885999999999999e-05 2023-09-23T21:16:12.365 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.365 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.365 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:16:12.365 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:14.956781+0000", 2023-09-23T21:16:12.365 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.4095821509999999 2023-09-23T21:16:12.365 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.365 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.365 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:16:12.365 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:14.956891+0000", 2023-09-23T21:16:12.365 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000109615 2023-09-23T21:16:12.366 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.366 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.366 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 0,4,7", 2023-09-23T21:16:12.366 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:14.956975+0000", 2023-09-23T21:16:12.366 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.3517999999999996e-05 2023-09-23T21:16:12.366 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.366 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.366 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T21:16:12.366 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:14.961386+0000", 2023-09-23T21:16:12.366 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0044118610000000004 2023-09-23T21:16:12.366 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.366 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.366 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:16:12.367 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:14.961517+0000", 2023-09-23T21:16:12.367 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000130619 2023-09-23T21:16:12.367 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.367 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.367 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:16:12.367 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:14.961609+0000", 2023-09-23T21:16:12.367 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.1634999999999999e-05 2023-09-23T21:16:12.367 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.367 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.367 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:16:12.367 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:15.573396+0000", 2023-09-23T21:16:12.367 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.611787839 2023-09-23T21:16:12.368 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.368 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.368 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T21:16:12.368 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:15.573454+0000", 2023-09-23T21:16:12.368 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.7442000000000003e-05 2023-09-23T21:16:12.368 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:12.368 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:12.368 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T21:16:12.368 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:15.575569+0000", 2023-09-23T21:16:12.368 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0021147100000000001 2023-09-23T21:16:12.368 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:16:12.368 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:16:12.368 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:16:12.369 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:16:12.369 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:16:12.369 INFO:teuthology.orchestra.run.smithi026.stdout:} 2023-09-23T21:16:12.369 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:16:12.425 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:16:12 smithi026 ceph-mon[179688]: pgmap v1407: 119 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 111 active+clean; 3.4 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 36 MiB/s wr, 575 op/s; 23080/166185 objects degraded (13.888%); 3177/166185 objects misplaced (1.912%); 1.8 MiB/s, 28 objects/s recovering 2023-09-23T21:16:12.425 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:16:12 smithi026 ceph-mon[179688]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2023-09-23T21:16:12.428 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:16:12.447 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:16:12.447 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:16:12.469 INFO:tasks.radosbench.radosbench.0.smithi026.stdout:2023-09-23T21:16:12.470178+0000 min lat: 0.00111295 max lat: 2.31793 avg lat: 0.0109253 2023-09-23T21:16:12.469 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T21:16:12.469 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 40 16 58589 58573 91.4557 38.9375 0.0351321 0.0109253 2023-09-23T21:16:12.549 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:16:12.624 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:16:12.764 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:16:12 smithi026 ceph-mon[175151]: pgmap v1407: 119 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 111 active+clean; 3.4 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 36 MiB/s wr, 575 op/s; 23080/166185 objects degraded (13.888%); 3177/166185 objects misplaced (1.912%); 1.8 MiB/s, 28 objects/s recovering 2023-09-23T21:16:12.764 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:16:12 smithi026 ceph-mon[175151]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2023-09-23T21:16:12.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:16:12 smithi070 ceph-mon[168282]: pgmap v1407: 119 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 111 active+clean; 3.4 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 36 MiB/s wr, 575 op/s; 23080/166185 objects degraded (13.888%); 3177/166185 objects misplaced (1.912%); 1.8 MiB/s, 28 objects/s recovering 2023-09-23T21:16:12.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:16:12 smithi070 ceph-mon[168282]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2023-09-23T21:16:12.956 INFO:teuthology.orchestra.run.smithi026.stdout: 2023-09-23T21:16:12.981 INFO:teuthology.orchestra.run.smithi026.stderr:dumped all 2023-09-23T21:16:13.040 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:16:13.046 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:16:13.047 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:16:13.148 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:16:13.152 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_ops_in_flight 2023-09-23T21:16:13.243 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:16:13.281 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T21:16:13.469 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 41 16 59240 59224 90.2179 40.6875 0.0130475 0.0110777 2023-09-23T21:16:13.508 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:16:13.529 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:16:13.530 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:16:13.569 INFO:teuthology.orchestra.run.smithi026.stdout:{"pg_ready":true,"pg_map":{"version":1407,"stamp":"2023-09-23T21:16:11.127056+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":3629389728,"num_objects":55395,"num_object_clones":0,"num_object_copies":166185,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":23080,"num_objects_misplaced":3177,"num_objects_unfound":0,"num_objects_dirty":55395,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":55432,"num_write_kb":3543773,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":235,"num_bytes_recovered":11351998,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":60417,"ondisk_log_size":60417,"up":357,"acting":349,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":364,"num_osds":5,"num_per_pool_osds":5,"num_per_pool_omap_osds":5,"kb":468725760,"kb_used":12390048,"kb_used_data":9248120,"kb_used_omap":125,"kb_used_meta":3141762,"kb_avail":456335712,"statfs":{"total":479975178240,"available":467287769088,"internally_reserved":0,"allocated":9470074880,"data_stored":9459178684,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":128662,"internal_metadata":3217164650},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,4,3,0,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":7,"apply_latency_ms":7,"commit_latency_ns":7000000,"apply_latency_ns":7000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":226754560,"num_objects":3460,"num_object_clones":0,"num_object_copies":10380,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":23080,"num_objects_misplaced":3177,"num_objects_unfound":0,"num_objects_dirty":3460,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3460,"num_write_kb":221440,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":173,"num_bytes_recovered":11343010,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"6.009653"},"pg_stats":[{"pgid":"1.18","version":"79'161","reported_seq":555,"reported_epoch":555,"state":"active+clean","last_fresh":"2023-09-23T21:16:06.355168+0000","last_change":"2023-09-23T21:16:06.355168+0000","last_active":"2023-09-23T21:16:06.355168+0000","last_peered":"2023-09-23T21:16:06.355168+0000","last_clean":"2023-09-23T21:16:06.355168+0000","last_became_active":"2023-09-23T21:16:06.354829+0000","last_became_peered":"2023-09-23T21:16:06.354829+0000","last_unstale":"2023-09-23T21:16:06.355168+0000","last_undegraded":"2023-09-23T21:16:06.355168+0000","last_fullsized":"2023-09-23T21:16:06.355168+0000","mapping_epoch":554,"log_start":"0'0","ondisk_log_start":"0'0","created":479,"last_epoch_clean":555,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:15:39.474823+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:15:39.474823+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:39:16.202426+0000","scrub_duration":0.074664395999999994,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":262688,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,0],"acting":[5,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1a","version":"68'1","reported_seq":145,"reported_epoch":554,"state":"active+clean","last_fresh":"2023-09-23T21:16:05.329417+0000","last_change":"2023-09-23T21:13:13.291403+0000","last_active":"2023-09-23T21:16:05.329417+0000","last_peered":"2023-09-23T21:16:05.329417+0000","last_clean":"2023-09-23T21:16:05.329417+0000","last_became_active":"2023-09-23T21:13:13.291123+0000","last_became_peered":"2023-09-23T21:13:13.291123+0000","last_unstale":"2023-09-23T21:16:05.329417+0000","last_undegraded":"2023-09-23T21:16:05.329417+0000","last_fullsized":"2023-09-23T21:16:05.329417+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":495,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:45:15.427430+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,0],"acting":[4,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":214,"reported_epoch":554,"state":"active+clean","last_fresh":"2023-09-23T21:16:05.327937+0000","last_change":"2023-09-23T21:12:10.767963+0000","last_active":"2023-09-23T21:16:05.327937+0000","last_peered":"2023-09-23T21:16:05.327937+0000","last_clean":"2023-09-23T21:16:05.327937+0000","last_became_active":"2023-09-23T21:12:10.767126+0000","last_became_peered":"2023-09-23T21:12:10.767126+0000","last_unstale":"2023-09-23T21:16:05.327937+0000","last_undegraded":"2023-09-23T21:16:05.327937+0000","last_fullsized":"2023-09-23T21:16:05.327937+0000","mapping_epoch":470,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:45:47.138277+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,5],"acting":[4,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1b","version":"79'161","reported_seq":742,"reported_epoch":554,"state":"active+clean","last_fresh":"2023-09-23T21:16:05.332927+0000","last_change":"2023-09-23T21:15:38.423372+0000","last_active":"2023-09-23T21:16:05.332927+0000","last_peered":"2023-09-23T21:16:05.332927+0000","last_clean":"2023-09-23T21:16:05.332927+0000","last_became_active":"2023-09-23T21:15:23.802580+0000","last_became_peered":"2023-09-23T21:15:23.802580+0000","last_unstale":"2023-09-23T21:16:05.332927+0000","last_undegraded":"2023-09-23T21:16:05.332927+0000","last_fullsized":"2023-09-23T21:16:05.332927+0000","mapping_epoch":542,"log_start":"0'0","ondisk_log_start":"0'0","created":479,"last_epoch_clean":543,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:15:38.423271+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:15:38.423271+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:04:46.575186+0000","scrub_duration":0.00258465,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":560,"reported_epoch":554,"state":"active+clean","last_fresh":"2023-09-23T21:16:05.329767+0000","last_change":"2023-09-23T21:16:03.305634+0000","last_active":"2023-09-23T21:16:05.329767+0000","last_peered":"2023-09-23T21:16:05.329767+0000","last_clean":"2023-09-23T21:16:05.329767+0000","last_became_active":"2023-09-23T21:16:03.305290+0000","last_became_peered":"2023-09-23T21:16:03.305290+0000","last_unstale":"2023-09-23T21:16:05.329767+0000","last_undegraded":"2023-09-23T21:16:05.329767+0000","last_fullsized":"2023-09-23T21:16:05.329767+0000","mapping_epoch":551,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":552,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:48:54.662945+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,2],"acting":[3,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":561,"reported_epoch":554,"state":"active+clean","last_fresh":"2023-09-23T21:16:05.320421+0000","last_change":"2023-09-23T21:16:03.298713+0000","last_active":"2023-09-23T21:16:05.320421+0000","last_peered":"2023-09-23T21:16:05.320421+0000","last_clean":"2023-09-23T21:16:05.320421+0000","last_became_active":"2023-09-23T21:16:03.297332+0000","last_became_peered":"2023-09-23T21:16:03.297332+0000","last_unstale":"2023-09-23T21:16:05.320421+0000","last_undegraded":"2023-09-23T21:16:05.320421+0000","last_fullsized":"2023-09-23T21:16:05.320421+0000","mapping_epoch":551,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":552,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:59:32.873371+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,5],"acting":[0,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1a","version":"79'161","reported_seq":632,"reported_epoch":555,"state":"active+clean","last_fresh":"2023-09-23T21:16:06.353183+0000","last_change":"2023-09-23T21:16:06.353183+0000","last_active":"2023-09-23T21:16:06.353183+0000","last_peered":"2023-09-23T21:16:06.353183+0000","last_clean":"2023-09-23T21:16:06.353183+0000","last_became_active":"2023-09-23T21:16:06.352826+0000","last_became_peered":"2023-09-23T21:16:06.352826+0000","last_unstale":"2023-09-23T21:16:06.353183+0000","last_undegraded":"2023-09-23T21:16:06.353183+0000","last_fullsized":"2023-09-23T21:16:06.353183+0000","mapping_epoch":554,"log_start":"0'0","ondisk_log_start":"0'0","created":479,"last_epoch_clean":555,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:15:45.158107+0000","last_deep_scrub":"79'161","last_deep_scrub_stamp":"2023-09-23T21:05:32.630292+0000","last_clean_scrub_stamp":"2023-09-23T21:15:45.158107+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:47:56.529197+0000","scrub_duration":0.001999373,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,2],"acting":[4,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":65,"reported_epoch":554,"state":"active+clean","last_fresh":"2023-09-23T21:16:05.318713+0000","last_change":"2023-09-23T21:13:30.806084+0000","last_active":"2023-09-23T21:16:05.318713+0000","last_peered":"2023-09-23T21:16:05.318713+0000","last_clean":"2023-09-23T21:16:05.318713+0000","last_became_active":"2023-09-23T21:13:30.805743+0000","last_became_peered":"2023-09-23T21:13:30.805743+0000","last_unstale":"2023-09-23T21:16:05.318713+0000","last_undegraded":"2023-09-23T21:16:05.318713+0000","last_fullsized":"2023-09-23T21:16:05.318713+0000","mapping_epoch":501,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:48:10.849281+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":429,"reported_epoch":554,"state":"active+clean","last_fresh":"2023-09-23T21:16:05.331781+0000","last_change":"2023-09-23T21:13:44.410921+0000","last_active":"2023-09-23T21:16:05.331781+0000","last_peered":"2023-09-23T21:16:05.331781+0000","last_clean":"2023-09-23T21:16:05.331781+0000","last_became_active":"2023-09-23T21:13:44.410633+0000","last_became_peered":"2023-09-23T21:13:44.410633+0000","last_unstale":"2023-09-23T21:16:05.331781+0000","last_undegraded":"2023-09-23T21:16:05.331781+0000","last_fullsized":"2023-09-23T21:16:05.331781+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:57:10.116786+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.15","version":"79'161","reported_seq":518,"reported_epoch":555,"state":"active+clean","last_fresh":"2023-09-23T21:16:06.360556+0000","last_change":"2023-09-23T21:16:06.360556+0000","last_active":"2023-09-23T21:16:06.360556+0000","last_peered":"2023-09-23T21:16:06.360556+0000","last_clean":"2023-09-23T21:16:06.360556+0000","last_became_active":"2023-09-23T21:16:06.360183+0000","last_became_peered":"2023-09-23T21:16:06.360183+0000","last_unstale":"2023-09-23T21:16:06.360556+0000","last_undegraded":"2023-09-23T21:16:06.360556+0000","last_fullsized":"2023-09-23T21:16:06.360556+0000","mapping_epoch":554,"log_start":"0'0","ondisk_log_start":"0'0","created":479,"last_epoch_clean":555,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:15:36.742247+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:15:36.742247+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0081306369999999996,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,0],"acting":[3,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":530,"reported_epoch":554,"state":"active+clean","last_fresh":"2023-09-23T21:16:05.331992+0000","last_change":"2023-09-23T21:13:12.235728+0000","last_active":"2023-09-23T21:16:05.331992+0000","last_peered":"2023-09-23T21:16:05.331992+0000","last_clean":"2023-09-23T21:16:05.331992+0000","last_became_active":"2023-09-23T21:13:12.216549+0000","last_became_peered":"2023-09-23T21:13:12.216549+0000","last_unstale":"2023-09-23T21:16:05.331992+0000","last_undegraded":"2023-09-23T21:16:05.331992+0000","last_fullsized":"2023-09-23T21:16:05.331992+0000","mapping_epoch":493,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:39:33.324239+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":593,"reported_epoch":554,"state":"active+clean","last_fresh":"2023-09-23T21:16:05.325010+0000","last_change":"2023-09-23T21:12:10.790349+0000","last_active":"2023-09-23T21:16:05.325010+0000","last_peered":"2023-09-23T21:16:05.325010+0000","last_clean":"2023-09-23T21:16:05.325010+0000","last_became_active":"2023-09-23T21:12:10.789977+0000","last_became_peered":"2023-09-23T21:12:10.789977+0000","last_unstale":"2023-09-23T21:16:05.325010+0000","last_undegraded":"2023-09-23T21:16:05.325010+0000","last_fullsized":"2023-09-23T21:16:05.325010+0000","mapping_epoch":470,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:49:40.191147+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,5],"acting":[3,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"79'161","reported_seq":933,"reported_epoch":555,"state":"active+clean","last_fresh":"2023-09-23T21:16:06.370039+0000","last_change":"2023-09-23T21:16:06.370155+0000","last_active":"2023-09-23T21:16:06.370039+0000","last_peered":"2023-09-23T21:16:06.370039+0000","last_clean":"2023-09-23T21:16:01.347521+0000","last_became_active":"2023-09-23T21:16:06.369755+0000","last_became_peered":"2023-09-23T21:16:06.369755+0000","last_unstale":"2023-09-23T21:16:06.370039+0000","last_undegraded":"2023-09-23T21:16:06.370039+0000","last_fullsized":"2023-09-23T21:16:06.370039+0000","mapping_epoch":554,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":555,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:25.096401+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:25.096401+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:00:50.633877+0000","scrub_duration":0.002859393,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,0],"acting":[2,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":289,"reported_epoch":554,"state":"active+clean","last_fresh":"2023-09-23T21:16:05.318592+0000","last_change":"2023-09-23T21:16:03.310585+0000","last_active":"2023-09-23T21:16:05.318592+0000","last_peered":"2023-09-23T21:16:05.318592+0000","last_clean":"2023-09-23T21:16:05.318592+0000","last_became_active":"2023-09-23T21:16:03.310214+0000","last_became_peered":"2023-09-23T21:16:03.310214+0000","last_unstale":"2023-09-23T21:16:05.318592+0000","last_undegraded":"2023-09-23T21:16:05.318592+0000","last_fullsized":"2023-09-23T21:16:05.318592+0000","mapping_epoch":551,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":552,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:48:03.265194+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":539,"reported_epoch":554,"state":"active+clean","last_fresh":"2023-09-23T21:16:05.329687+0000","last_change":"2023-09-23T21:16:03.305743+0000","last_active":"2023-09-23T21:16:05.329687+0000","last_peered":"2023-09-23T21:16:05.329687+0000","last_clean":"2023-09-23T21:16:05.329687+0000","last_became_active":"2023-09-23T21:16:03.302075+0000","last_became_peered":"2023-09-23T21:16:03.302075+0000","last_unstale":"2023-09-23T21:16:05.329687+0000","last_undegraded":"2023-09-23T21:16:05.329687+0000","last_fullsized":"2023-09-23T21:16:05.329687+0000","mapping_epoch":551,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":552,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:22:56.441676+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,0],"acting":[3,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.17","version":"79'161","reported_seq":684,"reported_epoch":555,"state":"active+clean","last_fresh":"2023-09-23T21:16:06.354354+0000","last_change":"2023-09-23T21:16:06.354354+0000","last_active":"2023-09-23T21:16:06.354354+0000","last_peered":"2023-09-23T21:16:06.354354+0000","last_clean":"2023-09-23T21:16:06.354354+0000","last_became_active":"2023-09-23T21:16:06.354027+0000","last_became_peered":"2023-09-23T21:16:06.354027+0000","last_unstale":"2023-09-23T21:16:06.354354+0000","last_undegraded":"2023-09-23T21:16:06.354354+0000","last_fullsized":"2023-09-23T21:16:06.354354+0000","mapping_epoch":554,"log_start":"0'0","ondisk_log_start":"0'0","created":479,"last_epoch_clean":555,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:15:50.325741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:15:50.325741+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:57:11.993539+0000","scrub_duration":0.0040500149999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":455,"reported_epoch":555,"state":"active+clean","last_fresh":"2023-09-23T21:16:06.340633+0000","last_change":"2023-09-23T21:16:03.318059+0000","last_active":"2023-09-23T21:16:06.340633+0000","last_peered":"2023-09-23T21:16:06.340633+0000","last_clean":"2023-09-23T21:16:06.340633+0000","last_became_active":"2023-09-23T21:16:03.305424+0000","last_became_peered":"2023-09-23T21:16:03.305424+0000","last_unstale":"2023-09-23T21:16:06.340633+0000","last_undegraded":"2023-09-23T21:16:06.340633+0000","last_fullsized":"2023-09-23T21:16:06.340633+0000","mapping_epoch":551,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":552,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:08:27.992736+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":450,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":242,"reported_epoch":554,"state":"active+clean","last_fresh":"2023-09-23T21:16:05.315763+0000","last_change":"2023-09-23T21:13:30.821489+0000","last_active":"2023-09-23T21:16:05.315763+0000","last_peered":"2023-09-23T21:16:05.315763+0000","last_clean":"2023-09-23T21:16:05.315763+0000","last_became_active":"2023-09-23T21:13:30.821125+0000","last_became_peered":"2023-09-23T21:13:30.821125+0000","last_unstale":"2023-09-23T21:16:05.315763+0000","last_undegraded":"2023-09-23T21:16:05.315763+0000","last_fullsized":"2023-09-23T21:16:05.315763+0000","mapping_epoch":501,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:47:18.553417+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,5],"acting":[2,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.16","version":"79'161","reported_seq":486,"reported_epoch":554,"state":"active+clean","last_fresh":"2023-09-23T21:16:05.320046+0000","last_change":"2023-09-23T21:15:46.326726+0000","last_active":"2023-09-23T21:16:05.320046+0000","last_peered":"2023-09-23T21:16:05.320046+0000","last_clean":"2023-09-23T21:16:05.320046+0000","last_became_active":"2023-09-23T21:13:33.770407+0000","last_became_peered":"2023-09-23T21:13:33.770407+0000","last_unstale":"2023-09-23T21:16:05.320046+0000","last_undegraded":"2023-09-23T21:16:05.320046+0000","last_fullsized":"2023-09-23T21:16:05.320046+0000","mapping_epoch":504,"log_start":"0'0","ondisk_log_start":"0'0","created":479,"last_epoch_clean":505,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:15:46.326589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:15:46.326589+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:50:23.810054+0000","scrub_duration":0.0062069059999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.14","version":"79'22","reported_seq":138,"reported_epoch":554,"state":"active+clean","last_fresh":"2023-09-23T21:16:05.323024+0000","last_change":"2023-09-23T21:16:03.324068+0000","last_active":"2023-09-23T21:16:05.323024+0000","last_peered":"2023-09-23T21:16:05.323024+0000","last_clean":"2023-09-23T21:16:05.323024+0000","last_became_active":"2023-09-23T21:16:03.304480+0000","last_became_peered":"2023-09-23T21:16:03.304480+0000","last_unstale":"2023-09-23T21:16:05.323024+0000","last_undegraded":"2023-09-23T21:16:05.323024+0000","last_fullsized":"2023-09-23T21:16:05.323024+0000","mapping_epoch":551,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":552,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:37:05.045329+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":9468,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,0],"acting":[4,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":483,"reported_epoch":554,"state":"active+clean","last_fresh":"2023-09-23T21:16:05.322807+0000","last_change":"2023-09-23T21:12:10.839271+0000","last_active":"2023-09-23T21:16:05.322807+0000","last_peered":"2023-09-23T21:16:05.322807+0000","last_clean":"2023-09-23T21:16:05.322807+0000","last_became_active":"2023-09-23T21:12:10.839042+0000","last_became_peered":"2023-09-23T21:12:10.839042+0000","last_unstale":"2023-09-23T21:16:05.322807+0000","last_undegraded":"2023-09-23T21:16:05.322807+0000","last_fullsized":"2023-09-23T21:16:05.322807+0000","mapping_epoch":470,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:10:26.414737+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,2],"acting":[4,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.11","version":"79'161","reported_seq":491,"reported_epoch":554,"state":"active+clean","last_fresh":"2023-09-23T21:16:05.318562+0000","last_change":"2023-09-23T21:15:47.300109+0000","last_active":"2023-09-23T21:16:05.318562+0000","last_peered":"2023-09-23T21:16:05.318562+0000","last_clean":"2023-09-23T21:16:05.318562+0000","last_became_active":"2023-09-23T21:13:33.767067+0000","last_became_peered":"2023-09-23T21:13:33.767067+0000","last_unstale":"2023-09-23T21:16:05.318562+0000","last_undegraded":"2023-09-23T21:16:05.318562+0000","last_fullsized":"2023-09-23T21:16:05.318562+0000","mapping_epoch":504,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":505,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:15:47.299947+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:15:47.299947+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:04:59.029359+0000","scrub_duration":0.0061353110000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":68,"reported_epoch":554,"state":"active+clean","last_fresh":"2023-09-23T21:16:05.322486+0000","last_change":"2023-09-23T21:16:03.315469+0000","last_active":"2023-09-23T21:16:05.322486+0000","last_peered":"2023-09-23T21:16:05.322486+0000","last_clean":"2023-09-23T21:16:05.322486+0000","last_became_active":"2023-09-23T21:16:03.304297+0000","last_became_peered":"2023-09-23T21:16:03.304297+0000","last_unstale":"2023-09-23T21:16:05.322486+0000","last_undegraded":"2023-09-23T21:16:05.322486+0000","last_fullsized":"2023-09-23T21:16:05.322486+0000","mapping_epoch":551,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":552,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:59:18.439120+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,0],"acting":[4,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":105,"reported_epoch":554,"state":"active+clean","last_fresh":"2023-09-23T21:16:05.320859+0000","last_change":"2023-09-23T21:16:03.304323+0000","last_active":"2023-09-23T21:16:05.320859+0000","last_peered":"2023-09-23T21:16:05.320859+0000","last_clean":"2023-09-23T21:16:05.320859+0000","last_became_active":"2023-09-23T21:16:03.303951+0000","last_became_peered":"2023-09-23T21:16:03.303951+0000","last_unstale":"2023-09-23T21:16:05.320859+0000","last_undegraded":"2023-09-23T21:16:05.320859+0000","last_fullsized":"2023-09-23T21:16:05.320859+0000","mapping_epoch":551,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":552,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:15:23.177639+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,5],"acting":[2,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.10","version":"79'161","reported_seq":709,"reported_epoch":555,"state":"active+clean","last_fresh":"2023-09-23T21:16:06.353481+0000","last_change":"2023-09-23T21:16:06.353481+0000","last_active":"2023-09-23T21:16:06.353481+0000","last_peered":"2023-09-23T21:16:06.353481+0000","last_clean":"2023-09-23T21:16:06.353481+0000","last_became_active":"2023-09-23T21:16:06.353116+0000","last_became_peered":"2023-09-23T21:16:06.353116+0000","last_unstale":"2023-09-23T21:16:06.353481+0000","last_undegraded":"2023-09-23T21:16:06.353481+0000","last_fullsized":"2023-09-23T21:16:06.353481+0000","mapping_epoch":554,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":555,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:06:50.336992+0000","last_deep_scrub":"79'161","last_deep_scrub_stamp":"2023-09-23T21:06:50.336992+0000","last_clean_scrub_stamp":"2023-09-23T21:06:50.336992+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:49:51.307900+0000","scrub_duration":0.0026150930000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":505,"reported_epoch":554,"state":"active+clean","last_fresh":"2023-09-23T21:16:05.320130+0000","last_change":"2023-09-23T21:16:03.295317+0000","last_active":"2023-09-23T21:16:05.320130+0000","last_peered":"2023-09-23T21:16:05.320130+0000","last_clean":"2023-09-23T21:16:05.320130+0000","last_became_active":"2023-09-23T21:16:03.295016+0000","last_became_peered":"2023-09-23T21:16:03.295016+0000","last_unstale":"2023-09-23T21:16:05.320130+0000","last_undegraded":"2023-09-23T21:16:05.320130+0000","last_fullsized":"2023-09-23T21:16:05.320130+0000","mapping_epoch":551,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":552,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:56:38.216050+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":316,"reported_epoch":554,"state":"active+clean","last_fresh":"2023-09-23T21:16:05.333152+0000","last_change":"2023-09-23T21:16:03.305745+0000","last_active":"2023-09-23T21:16:05.333152+0000","last_peered":"2023-09-23T21:16:05.333152+0000","last_clean":"2023-09-23T21:16:05.333152+0000","last_became_active":"2023-09-23T21:16:03.304996+0000","last_became_peered":"2023-09-23T21:16:03.304996+0000","last_unstale":"2023-09-23T21:16:05.333152+0000","last_undegraded":"2023-09-23T21:16:05.333152+0000","last_fullsized":"2023-09-23T21:16:05.333152+0000","mapping_epoch":551,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":552,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:11:36.450316+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,0],"acting":[3,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.13","version":"79'161","reported_seq":632,"reported_epoch":554,"state":"active+clean","last_fresh":"2023-09-23T21:16:05.333664+0000","last_change":"2023-09-23T21:13:14.933096+0000","last_active":"2023-09-23T21:16:05.333664+0000","last_peered":"2023-09-23T21:16:05.333664+0000","last_clean":"2023-09-23T21:16:05.333664+0000","last_became_active":"2023-09-23T21:13:14.932703+0000","last_became_peered":"2023-09-23T21:13:14.932703+0000","last_unstale":"2023-09-23T21:16:05.333664+0000","last_undegraded":"2023-09-23T21:16:05.333664+0000","last_fullsized":"2023-09-23T21:16:05.333664+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:36.420362+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:36.420362+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:29:42.163911+0000","scrub_duration":0.028838444000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":556,"reported_epoch":554,"state":"active+clean","last_fresh":"2023-09-23T21:16:05.322657+0000","last_change":"2023-09-23T21:16:03.304956+0000","last_active":"2023-09-23T21:16:05.322657+0000","last_peered":"2023-09-23T21:16:05.322657+0000","last_clean":"2023-09-23T21:16:05.322657+0000","last_became_active":"2023-09-23T21:16:03.304652+0000","last_became_peered":"2023-09-23T21:16:03.304652+0000","last_unstale":"2023-09-23T21:16:05.322657+0000","last_undegraded":"2023-09-23T21:16:05.322657+0000","last_fullsized":"2023-09-23T21:16:05.322657+0000","mapping_epoch":551,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":552,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:01:27.596582+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":74,"reported_epoch":554,"state":"active+clean","last_fresh":"2023-09-23T21:16:05.319798+0000","last_change":"2023-09-23T21:16:03.297659+0000","last_active":"2023-09-23T21:16:05.319798+0000","last_peered":"2023-09-23T21:16:05.319798+0000","last_clean":"2023-09-23T21:16:05.319798+0000","last_became_active":"2023-09-23T21:16:03.296258+0000","last_became_peered":"2023-09-23T21:16:03.296258+0000","last_unstale":"2023-09-23T21:16:05.319798+0000","last_undegraded":"2023-09-23T21:16:05.319798+0000","last_fullsized":"2023-09-23T21:16:05.319798+0000","mapping_epoch":551,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":552,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:02:28.322653+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.12","version":"79'161","reported_seq":808,"reported_epoch":555,"state":"active+clean","last_fresh":"2023-09-23T21:16:06.351150+0000","last_change":"2023-09-23T21:16:06.351150+0000","last_active":"2023-09-23T21:16:06.351150+0000","last_peered":"2023-09-23T21:16:06.351150+0000","last_clean":"2023-09-23T21:16:06.351150+0000","last_became_active":"2023-09-23T21:16:06.350834+0000","last_became_peered":"2023-09-23T21:16:06.350834+0000","last_unstale":"2023-09-23T21:16:06.351150+0000","last_undegraded":"2023-09-23T21:16:06.351150+0000","last_fullsized":"2023-09-23T21:16:06.351150+0000","mapping_epoch":554,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":555,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:05:56.785173+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:05:56.785173+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:06:52.904682+0000","scrub_duration":0.0042114370000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":75,"reported_epoch":554,"state":"active+clean","last_fresh":"2023-09-23T21:16:05.318816+0000","last_change":"2023-09-23T21:13:38.048791+0000","last_active":"2023-09-23T21:16:05.318816+0000","last_peered":"2023-09-23T21:16:05.318816+0000","last_clean":"2023-09-23T21:16:05.318816+0000","last_became_active":"2023-09-23T21:13:30.801386+0000","last_became_peered":"2023-09-23T21:13:30.801386+0000","last_unstale":"2023-09-23T21:16:05.318816+0000","last_undegraded":"2023-09-23T21:16:05.318816+0000","last_fullsized":"2023-09-23T21:16:05.318816+0000","mapping_epoch":501,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:52:49.170414+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":401,"reported_epoch":554,"state":"active+clean","last_fresh":"2023-09-23T21:16:05.335897+0000","last_change":"2023-09-23T21:16:03.306209+0000","last_active":"2023-09-23T21:16:05.335897+0000","last_peered":"2023-09-23T21:16:05.335897+0000","last_clean":"2023-09-23T21:16:05.335897+0000","last_became_active":"2023-09-23T21:16:03.305875+0000","last_became_peered":"2023-09-23T21:16:03.305875+0000","last_unstale":"2023-09-23T21:16:05.335897+0000","last_undegraded":"2023-09-23T21:16:05.335897+0000","last_fullsized":"2023-09-23T21:16:05.335897+0000","mapping_epoch":551,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":552,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:50:52.300194+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.7","version":"555'3483","reported_seq":3502,"reported_epoch":555,"state":"active+clean","last_fresh":"2023-09-23T21:16:07.023391+0000","last_change":"2023-09-23T21:15:25.902813+0000","last_active":"2023-09-23T21:16:07.023391+0000","last_peered":"2023-09-23T21:16:07.023391+0000","last_clean":"2023-09-23T21:16:07.023391+0000","last_became_active":"2023-09-23T21:15:25.902476+0000","last_became_peered":"2023-09-23T21:15:25.902476+0000","last_unstale":"2023-09-23T21:16:07.023391+0000","last_undegraded":"2023-09-23T21:16:07.023391+0000","last_fullsized":"2023-09-23T21:16:07.023391+0000","mapping_epoch":544,"log_start":"0'0","ondisk_log_start":"0'0","created":544,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_clean_scrub_stamp":"2023-09-23T21:15:24.860968+0000","objects_scrubbed":0,"log_size":3483,"log_dups_size":0,"ondisk_log_size":3483,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:41:25.699152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":228261888,"num_objects":3483,"num_object_clones":0,"num_object_copies":10449,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3483,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3483,"num_write_kb":222912,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.d","version":"79'161","reported_seq":596,"reported_epoch":555,"state":"active+clean","last_fresh":"2023-09-23T21:16:06.353398+0000","last_change":"2023-09-23T21:16:06.353398+0000","last_active":"2023-09-23T21:16:06.353398+0000","last_peered":"2023-09-23T21:16:06.353398+0000","last_clean":"2023-09-23T21:16:06.353398+0000","last_became_active":"2023-09-23T21:16:06.353084+0000","last_became_peered":"2023-09-23T21:16:06.353084+0000","last_unstale":"2023-09-23T21:16:06.353398+0000","last_undegraded":"2023-09-23T21:16:06.353398+0000","last_fullsized":"2023-09-23T21:16:06.353398+0000","mapping_epoch":554,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":555,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:15:32.726690+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:15:32.726690+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0024987500000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,0],"acting":[4,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":55,"reported_epoch":554,"state":"active+clean","last_fresh":"2023-09-23T21:16:05.322392+0000","last_change":"2023-09-23T21:16:03.326126+0000","last_active":"2023-09-23T21:16:05.322392+0000","last_peered":"2023-09-23T21:16:05.322392+0000","last_clean":"2023-09-23T21:16:05.322392+0000","last_became_active":"2023-09-23T21:16:03.304992+0000","last_became_peered":"2023-09-23T21:16:03.304992+0000","last_unstale":"2023-09-23T21:16:05.322392+0000","last_undegraded":"2023-09-23T21:16:05.322392+0000","last_fullsized":"2023-09-23T21:16:05.322392+0000","mapping_epoch":551,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":552,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:33:30.260026+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":66,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,5],"acting":[4,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":272,"reported_epoch":554,"state":"active+clean","last_fresh":"2023-09-23T21:16:05.322309+0000","last_change":"2023-09-23T21:16:03.310005+0000","last_active":"2023-09-23T21:16:05.322309+0000","last_peered":"2023-09-23T21:16:05.322309+0000","last_clean":"2023-09-23T21:16:05.322309+0000","last_became_active":"2023-09-23T21:16:03.309744+0000","last_became_peered":"2023-09-23T21:16:03.309744+0000","last_unstale":"2023-09-23T21:16:05.322309+0000","last_undegraded":"2023-09-23T21:16:05.322309+0000","last_fullsized":"2023-09-23T21:16:05.322309+0000","mapping_epoch":551,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":552,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:51:15.807355+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,3],"acting":[5,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.6","version":"555'3506","reported_seq":3526,"reported_epoch":555,"state":"active+clean","last_fresh":"2023-09-23T21:16:10.285671+0000","last_change":"2023-09-23T21:15:25.904499+0000","last_active":"2023-09-23T21:16:10.285671+0000","last_peered":"2023-09-23T21:16:10.285671+0000","last_clean":"2023-09-23T21:16:10.285671+0000","last_became_active":"2023-09-23T21:15:25.904185+0000","last_became_peered":"2023-09-23T21:15:25.904185+0000","last_unstale":"2023-09-23T21:16:10.285671+0000","last_undegraded":"2023-09-23T21:16:10.285671+0000","last_fullsized":"2023-09-23T21:16:10.285671+0000","mapping_epoch":544,"log_start":"0'0","ondisk_log_start":"0'0","created":544,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_clean_scrub_stamp":"2023-09-23T21:15:24.860968+0000","objects_scrubbed":0,"log_size":3506,"log_dups_size":0,"ondisk_log_size":3506,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:02:41.735858+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":229769216,"num_objects":3506,"num_object_clones":0,"num_object_copies":10518,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3506,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3506,"num_write_kb":224384,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.c","version":"79'161","reported_seq":936,"reported_epoch":554,"state":"active+clean","last_fresh":"2023-09-23T21:16:05.332783+0000","last_change":"2023-09-23T21:15:31.992756+0000","last_active":"2023-09-23T21:16:05.332783+0000","last_peered":"2023-09-23T21:16:05.332783+0000","last_clean":"2023-09-23T21:16:05.332783+0000","last_became_active":"2023-09-23T21:13:13.401526+0000","last_became_peered":"2023-09-23T21:13:13.401526+0000","last_unstale":"2023-09-23T21:16:05.332783+0000","last_undegraded":"2023-09-23T21:16:05.332783+0000","last_fullsized":"2023-09-23T21:16:05.332783+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":532,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:15:31.992598+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:15:31.992598+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:41:14.215543+0000","scrub_duration":0.002960678,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":64,"reported_epoch":554,"state":"active+clean","last_fresh":"2023-09-23T21:16:05.324910+0000","last_change":"2023-09-23T21:16:03.327982+0000","last_active":"2023-09-23T21:16:05.324910+0000","last_peered":"2023-09-23T21:16:05.324910+0000","last_clean":"2023-09-23T21:16:05.324910+0000","last_became_active":"2023-09-23T21:16:03.305384+0000","last_became_peered":"2023-09-23T21:16:03.305384+0000","last_unstale":"2023-09-23T21:16:05.324910+0000","last_undegraded":"2023-09-23T21:16:05.324910+0000","last_fullsized":"2023-09-23T21:16:05.324910+0000","mapping_epoch":551,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":552,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:00:49.175494+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,5],"acting":[4,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":517,"reported_epoch":554,"state":"active+clean","last_fresh":"2023-09-23T21:16:05.322740+0000","last_change":"2023-09-23T21:13:15.152792+0000","last_active":"2023-09-23T21:16:05.322740+0000","last_peered":"2023-09-23T21:16:05.322740+0000","last_clean":"2023-09-23T21:16:05.322740+0000","last_became_active":"2023-09-23T21:13:14.515084+0000","last_became_peered":"2023-09-23T21:13:14.515084+0000","last_unstale":"2023-09-23T21:16:05.322740+0000","last_undegraded":"2023-09-23T21:16:05.322740+0000","last_fullsized":"2023-09-23T21:16:05.322740+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:08:16.702072+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.5","version":"555'3458","reported_seq":3823,"reported_epoch":555,"state":"active+recovering+undersized+remapped","last_fresh":"2023-09-23T21:16:07.345967+0000","last_change":"2023-09-23T21:16:04.490109+0000","last_active":"2023-09-23T21:16:07.345967+0000","last_peered":"2023-09-23T21:16:07.345967+0000","last_clean":"2023-09-23T21:16:02.276271+0000","last_became_active":"2023-09-23T21:16:04.440153+0000","last_became_peered":"2023-09-23T21:16:04.440153+0000","last_unstale":"2023-09-23T21:16:07.345967+0000","last_undegraded":"2023-09-23T21:16:07.345967+0000","last_fullsized":"2023-09-23T21:16:04.283468+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":544,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_clean_scrub_stamp":"2023-09-23T21:15:24.860968+0000","objects_scrubbed":0,"log_size":3458,"log_dups_size":0,"ondisk_log_size":3458,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:54:44.876272+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":226623488,"num_objects":3458,"num_object_clones":0,"num_object_copies":10374,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":3177,"num_objects_unfound":0,"num_objects_dirty":3458,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3458,"num_write_kb":221312,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":168,"num_bytes_recovered":11010048,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,0],"acting":[5,3],"avail_no_missing":["5","3","7"],"object_location_counts":[{"shards":"3,5","objects":281},{"shards":"3,5,7","objects":3177}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.f","version":"79'161","reported_seq":842,"reported_epoch":554,"state":"active+clean","last_fresh":"2023-09-23T21:16:05.323559+0000","last_change":"2023-09-23T21:11:30.750338+0000","last_active":"2023-09-23T21:16:05.323559+0000","last_peered":"2023-09-23T21:16:05.323559+0000","last_clean":"2023-09-23T21:16:05.323559+0000","last_became_active":"2023-09-23T21:11:30.749953+0000","last_became_peered":"2023-09-23T21:11:30.749953+0000","last_unstale":"2023-09-23T21:16:05.323559+0000","last_undegraded":"2023-09-23T21:16:05.323559+0000","last_fullsized":"2023-09-23T21:16:05.323559+0000","mapping_epoch":455,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":456,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:27.880776+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:27.880776+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:07:10.405723+0000","scrub_duration":0.0014004899999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,3],"acting":[5,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":292,"reported_epoch":554,"state":"active+clean","last_fresh":"2023-09-23T21:16:05.328483+0000","last_change":"2023-09-23T21:16:03.320850+0000","last_active":"2023-09-23T21:16:05.328483+0000","last_peered":"2023-09-23T21:16:05.328483+0000","last_clean":"2023-09-23T21:16:05.328483+0000","last_became_active":"2023-09-23T21:16:03.309905+0000","last_became_peered":"2023-09-23T21:16:03.309905+0000","last_unstale":"2023-09-23T21:16:05.328483+0000","last_undegraded":"2023-09-23T21:16:05.328483+0000","last_fullsized":"2023-09-23T21:16:05.328483+0000","mapping_epoch":551,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":552,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:17:02.135251+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":350,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":504,"reported_epoch":554,"state":"active+clean","last_fresh":"2023-09-23T21:16:05.323350+0000","last_change":"2023-09-23T21:16:03.308165+0000","last_active":"2023-09-23T21:16:05.323350+0000","last_peered":"2023-09-23T21:16:05.323350+0000","last_clean":"2023-09-23T21:16:05.323350+0000","last_became_active":"2023-09-23T21:16:03.307845+0000","last_became_peered":"2023-09-23T21:16:03.307845+0000","last_unstale":"2023-09-23T21:16:05.323350+0000","last_undegraded":"2023-09-23T21:16:05.323350+0000","last_fullsized":"2023-09-23T21:16:05.323350+0000","mapping_epoch":551,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":552,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:34:11.207251+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.4","version":"555'3384","reported_seq":3409,"reported_epoch":555,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T21:16:06.366554+0000","last_change":"2023-09-23T21:16:04.497913+0000","last_active":"2023-09-23T21:16:06.366554+0000","last_peered":"2023-09-23T21:16:06.366554+0000","last_clean":"2023-09-23T21:16:02.247284+0000","last_became_active":"2023-09-23T21:16:04.489193+0000","last_became_peered":"2023-09-23T21:16:04.489193+0000","last_unstale":"2023-09-23T21:16:06.366554+0000","last_undegraded":"2023-09-23T21:16:04.358200+0000","last_fullsized":"2023-09-23T21:16:04.337355+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":544,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_clean_scrub_stamp":"2023-09-23T21:15:24.860968+0000","objects_scrubbed":0,"log_size":3384,"log_dups_size":0,"ondisk_log_size":3384,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:59:34.964975+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":221773824,"num_objects":3384,"num_object_clones":0,"num_object_copies":10152,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3293,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3384,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3384,"num_write_kb":216576,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3],"avail_no_missing":["2","3","7"],"object_location_counts":[{"shards":"2,3","objects":91},{"shards":"2,3,7","objects":3293}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.e","version":"79'161","reported_seq":753,"reported_epoch":555,"state":"active+clean","last_fresh":"2023-09-23T21:16:06.355733+0000","last_change":"2023-09-23T21:16:06.355733+0000","last_active":"2023-09-23T21:16:06.355733+0000","last_peered":"2023-09-23T21:16:06.355733+0000","last_clean":"2023-09-23T21:16:06.355733+0000","last_became_active":"2023-09-23T21:16:06.355367+0000","last_became_peered":"2023-09-23T21:16:06.355367+0000","last_unstale":"2023-09-23T21:16:06.355733+0000","last_undegraded":"2023-09-23T21:16:06.355733+0000","last_fullsized":"2023-09-23T21:16:06.355733+0000","mapping_epoch":554,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":555,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:15:39.046962+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:15:39.046962+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:28:27.348227+0000","scrub_duration":0.0082638769999999993,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":353,"reported_epoch":555,"state":"active+clean","last_fresh":"2023-09-23T21:16:06.342373+0000","last_change":"2023-09-23T21:13:13.402418+0000","last_active":"2023-09-23T21:16:06.342373+0000","last_peered":"2023-09-23T21:16:06.342373+0000","last_clean":"2023-09-23T21:16:06.342373+0000","last_became_active":"2023-09-23T21:13:13.400984+0000","last_became_peered":"2023-09-23T21:13:13.400984+0000","last_unstale":"2023-09-23T21:16:06.342373+0000","last_undegraded":"2023-09-23T21:16:06.342373+0000","last_fullsized":"2023-09-23T21:16:06.342373+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":495,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:28:01.411191+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,0],"acting":[3,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":528,"reported_epoch":554,"state":"active+clean","last_fresh":"2023-09-23T21:16:05.318608+0000","last_change":"2023-09-23T21:13:14.339670+0000","last_active":"2023-09-23T21:16:05.318608+0000","last_peered":"2023-09-23T21:16:05.318608+0000","last_clean":"2023-09-23T21:16:05.318608+0000","last_became_active":"2023-09-23T21:13:14.339341+0000","last_became_peered":"2023-09-23T21:13:14.339341+0000","last_unstale":"2023-09-23T21:16:05.318608+0000","last_undegraded":"2023-09-23T21:16:05.318608+0000","last_fullsized":"2023-09-23T21:16:05.318608+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:23:19.411368+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.3","version":"555'3385","reported_seq":3413,"reported_epoch":555,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T21:16:07.317228+0000","last_change":"2023-09-23T21:16:04.499532+0000","last_active":"2023-09-23T21:16:07.317228+0000","last_peered":"2023-09-23T21:16:07.317228+0000","last_clean":"2023-09-23T21:16:02.266319+0000","last_became_active":"2023-09-23T21:16:04.492294+0000","last_became_peered":"2023-09-23T21:16:04.492294+0000","last_unstale":"2023-09-23T21:16:07.317228+0000","last_undegraded":"2023-09-23T21:16:04.305551+0000","last_fullsized":"2023-09-23T21:16:04.282583+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":544,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_clean_scrub_stamp":"2023-09-23T21:15:24.860968+0000","objects_scrubbed":0,"log_size":3385,"log_dups_size":0,"ondisk_log_size":3385,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:54:42.087923+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":221839360,"num_objects":3385,"num_object_clones":0,"num_object_copies":10155,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3247,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3385,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3385,"num_write_kb":216640,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2],"avail_no_missing":["5","2","7"],"object_location_counts":[{"shards":"2,5","objects":138},{"shards":"2,5,7","objects":3247}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.9","version":"79'161","reported_seq":737,"reported_epoch":555,"state":"active+clean","last_fresh":"2023-09-23T21:16:06.358552+0000","last_change":"2023-09-23T21:16:06.358683+0000","last_active":"2023-09-23T21:16:06.358552+0000","last_peered":"2023-09-23T21:16:06.358552+0000","last_clean":"2023-09-23T21:15:54.212503+0000","last_became_active":"2023-09-23T21:16:06.358295+0000","last_became_peered":"2023-09-23T21:16:06.358295+0000","last_unstale":"2023-09-23T21:16:06.358552+0000","last_undegraded":"2023-09-23T21:16:06.358552+0000","last_fullsized":"2023-09-23T21:16:06.358552+0000","mapping_epoch":554,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":555,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:15:43.437369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:15:43.437369+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:27:43.169880+0000","scrub_duration":0.0024129429999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,5],"acting":[2,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":114,"reported_epoch":554,"state":"active+clean","last_fresh":"2023-09-23T21:16:05.318692+0000","last_change":"2023-09-23T21:13:30.793273+0000","last_active":"2023-09-23T21:16:05.318692+0000","last_peered":"2023-09-23T21:16:05.318692+0000","last_clean":"2023-09-23T21:16:05.318692+0000","last_became_active":"2023-09-23T21:13:30.791269+0000","last_became_peered":"2023-09-23T21:13:30.791269+0000","last_unstale":"2023-09-23T21:16:05.318692+0000","last_undegraded":"2023-09-23T21:16:05.318692+0000","last_fullsized":"2023-09-23T21:16:05.318692+0000","mapping_epoch":501,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:38:07.142290+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":138,"reported_epoch":554,"state":"active+clean","last_fresh":"2023-09-23T21:16:05.322749+0000","last_change":"2023-09-23T21:16:03.305294+0000","last_active":"2023-09-23T21:16:05.322749+0000","last_peered":"2023-09-23T21:16:05.322749+0000","last_clean":"2023-09-23T21:16:05.322749+0000","last_became_active":"2023-09-23T21:16:03.304965+0000","last_became_peered":"2023-09-23T21:16:03.304965+0000","last_unstale":"2023-09-23T21:16:05.322749+0000","last_undegraded":"2023-09-23T21:16:05.322749+0000","last_fullsized":"2023-09-23T21:16:05.322749+0000","mapping_epoch":551,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":552,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:57:37.154658+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.2","version":"554'3424","reported_seq":3444,"reported_epoch":555,"state":"active+clean","last_fresh":"2023-09-23T21:16:06.350684+0000","last_change":"2023-09-23T21:15:25.900671+0000","last_active":"2023-09-23T21:16:06.350684+0000","last_peered":"2023-09-23T21:16:06.350684+0000","last_clean":"2023-09-23T21:16:06.350684+0000","last_became_active":"2023-09-23T21:15:25.900364+0000","last_became_peered":"2023-09-23T21:15:25.900364+0000","last_unstale":"2023-09-23T21:16:06.350684+0000","last_undegraded":"2023-09-23T21:16:06.350684+0000","last_fullsized":"2023-09-23T21:16:06.350684+0000","mapping_epoch":544,"log_start":"0'0","ondisk_log_start":"0'0","created":544,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_clean_scrub_stamp":"2023-09-23T21:15:24.860968+0000","objects_scrubbed":0,"log_size":3424,"log_dups_size":0,"ondisk_log_size":3424,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:03:40.630119+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":224395264,"num_objects":3424,"num_object_clones":0,"num_object_copies":10272,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3424,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3424,"num_write_kb":219136,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.8","version":"79'161","reported_seq":688,"reported_epoch":554,"state":"active+clean","last_fresh":"2023-09-23T21:16:05.312357+0000","last_change":"2023-09-23T21:15:36.367819+0000","last_active":"2023-09-23T21:16:05.312357+0000","last_peered":"2023-09-23T21:16:05.312357+0000","last_clean":"2023-09-23T21:16:05.312357+0000","last_became_active":"2023-09-23T21:12:30.433217+0000","last_became_peered":"2023-09-23T21:12:30.433217+0000","last_unstale":"2023-09-23T21:16:05.312357+0000","last_undegraded":"2023-09-23T21:16:05.312357+0000","last_fullsized":"2023-09-23T21:16:05.312357+0000","mapping_epoch":359,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:15:36.367640+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:15:36.367640+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:49:35.243653+0000","scrub_duration":0.005826899,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":131,"reported_epoch":554,"state":"active+clean","last_fresh":"2023-09-23T21:16:05.329369+0000","last_change":"2023-09-23T21:12:10.823380+0000","last_active":"2023-09-23T21:16:05.329369+0000","last_peered":"2023-09-23T21:16:05.329369+0000","last_clean":"2023-09-23T21:16:05.329369+0000","last_became_active":"2023-09-23T21:12:10.823044+0000","last_became_peered":"2023-09-23T21:12:10.823044+0000","last_unstale":"2023-09-23T21:16:05.329369+0000","last_undegraded":"2023-09-23T21:16:05.329369+0000","last_fullsized":"2023-09-23T21:16:05.329369+0000","mapping_epoch":470,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:53:29.846536+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,5],"acting":[4,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":132,"reported_epoch":554,"state":"active+clean","last_fresh":"2023-09-23T21:16:05.327547+0000","last_change":"2023-09-23T21:13:30.772451+0000","last_active":"2023-09-23T21:16:05.327547+0000","last_peered":"2023-09-23T21:16:05.327547+0000","last_clean":"2023-09-23T21:16:05.327547+0000","last_became_active":"2023-09-23T21:13:30.772040+0000","last_became_peered":"2023-09-23T21:13:30.772040+0000","last_unstale":"2023-09-23T21:16:05.327547+0000","last_undegraded":"2023-09-23T21:16:05.327547+0000","last_fullsized":"2023-09-23T21:16:05.327547+0000","mapping_epoch":501,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:24:49.246727+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,0],"acting":[4,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.1","version":"555'3590","reported_seq":3609,"reported_epoch":555,"state":"active+clean","last_fresh":"2023-09-23T21:16:09.158673+0000","last_change":"2023-09-23T21:15:25.903355+0000","last_active":"2023-09-23T21:16:09.158673+0000","last_peered":"2023-09-23T21:16:09.158673+0000","last_clean":"2023-09-23T21:16:09.158673+0000","last_became_active":"2023-09-23T21:15:25.902988+0000","last_became_peered":"2023-09-23T21:15:25.902988+0000","last_unstale":"2023-09-23T21:16:09.158673+0000","last_undegraded":"2023-09-23T21:16:09.158673+0000","last_fullsized":"2023-09-23T21:16:09.158673+0000","mapping_epoch":544,"log_start":"0'0","ondisk_log_start":"0'0","created":544,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_clean_scrub_stamp":"2023-09-23T21:15:24.860968+0000","objects_scrubbed":0,"log_size":3590,"log_dups_size":0,"ondisk_log_size":3590,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:47:02.629642+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":235274240,"num_objects":3590,"num_object_clones":0,"num_object_copies":10770,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3590,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3590,"num_write_kb":229760,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,2],"acting":[3,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.b","version":"79'161","reported_seq":571,"reported_epoch":554,"state":"active+clean","last_fresh":"2023-09-23T21:16:05.318354+0000","last_change":"2023-09-23T21:15:36.340009+0000","last_active":"2023-09-23T21:16:05.318354+0000","last_peered":"2023-09-23T21:16:05.318354+0000","last_clean":"2023-09-23T21:16:05.318354+0000","last_became_active":"2023-09-23T21:13:13.372328+0000","last_became_peered":"2023-09-23T21:13:13.372328+0000","last_unstale":"2023-09-23T21:16:05.318354+0000","last_undegraded":"2023-09-23T21:16:05.318354+0000","last_fullsized":"2023-09-23T21:16:05.318354+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":495,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:15:36.339885+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:15:36.339885+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:01:26.605006+0000","scrub_duration":0.0068473279999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":122,"reported_epoch":554,"state":"active+clean","last_fresh":"2023-09-23T21:16:05.322565+0000","last_change":"2023-09-23T21:16:03.300524+0000","last_active":"2023-09-23T21:16:05.322565+0000","last_peered":"2023-09-23T21:16:05.322565+0000","last_clean":"2023-09-23T21:16:05.322565+0000","last_became_active":"2023-09-23T21:16:03.300190+0000","last_became_peered":"2023-09-23T21:16:03.300190+0000","last_unstale":"2023-09-23T21:16:05.322565+0000","last_undegraded":"2023-09-23T21:16:05.322565+0000","last_fullsized":"2023-09-23T21:16:05.322565+0000","mapping_epoch":551,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":552,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:24:11.525888+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":486,"reported_epoch":554,"state":"active+clean","last_fresh":"2023-09-23T21:16:05.325223+0000","last_change":"2023-09-23T21:12:10.811698+0000","last_active":"2023-09-23T21:16:05.325223+0000","last_peered":"2023-09-23T21:16:05.325223+0000","last_clean":"2023-09-23T21:16:05.325223+0000","last_became_active":"2023-09-23T21:12:10.789656+0000","last_became_peered":"2023-09-23T21:12:10.789656+0000","last_unstale":"2023-09-23T21:16:05.325223+0000","last_undegraded":"2023-09-23T21:16:05.325223+0000","last_fullsized":"2023-09-23T21:16:05.325223+0000","mapping_epoch":470,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:39:15.112091+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,5],"acting":[4,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.0","version":"555'3501","reported_seq":3529,"reported_epoch":555,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T21:16:09.216808+0000","last_change":"2023-09-23T21:16:04.496166+0000","last_active":"2023-09-23T21:16:09.216808+0000","last_peered":"2023-09-23T21:16:09.216808+0000","last_clean":"2023-09-23T21:16:02.267015+0000","last_became_active":"2023-09-23T21:16:04.480018+0000","last_became_peered":"2023-09-23T21:16:04.480018+0000","last_unstale":"2023-09-23T21:16:09.216808+0000","last_undegraded":"2023-09-23T21:16:04.317521+0000","last_fullsized":"2023-09-23T21:16:04.281984+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":544,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_clean_scrub_stamp":"2023-09-23T21:15:24.860968+0000","objects_scrubbed":0,"log_size":3501,"log_dups_size":0,"ondisk_log_size":3501,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:07:47.715637+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":229441536,"num_objects":3501,"num_object_clones":0,"num_object_copies":10503,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3297,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3501,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3501,"num_write_kb":224064,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,2],"acting":[3,0],"avail_no_missing":["3","0","7"],"object_location_counts":[{"shards":"0,3","objects":204},{"shards":"0,3,7","objects":3297}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.a","version":"79'161","reported_seq":714,"reported_epoch":554,"state":"active+clean","last_fresh":"2023-09-23T21:16:05.315594+0000","last_change":"2023-09-23T21:15:46.420777+0000","last_active":"2023-09-23T21:16:05.315594+0000","last_peered":"2023-09-23T21:16:05.315594+0000","last_clean":"2023-09-23T21:16:05.315594+0000","last_became_active":"2023-09-23T21:12:30.437427+0000","last_became_peered":"2023-09-23T21:12:30.437427+0000","last_unstale":"2023-09-23T21:16:05.315594+0000","last_undegraded":"2023-09-23T21:16:05.315594+0000","last_fullsized":"2023-09-23T21:16:05.315594+0000","mapping_epoch":446,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:15:46.420650+0000","last_deep_scrub":"79'161","last_deep_scrub_stamp":"2023-09-23T21:05:32.630292+0000","last_clean_scrub_stamp":"2023-09-23T21:15:46.420650+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:50:23.452475+0000","scrub_duration":0.0072898930000000004,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,4],"acting":[2,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":411,"reported_epoch":554,"state":"active+clean","last_fresh":"2023-09-23T21:16:05.324697+0000","last_change":"2023-09-23T21:16:03.302250+0000","last_active":"2023-09-23T21:16:05.324697+0000","last_peered":"2023-09-23T21:16:05.324697+0000","last_clean":"2023-09-23T21:16:05.324697+0000","last_became_active":"2023-09-23T21:16:03.301883+0000","last_became_peered":"2023-09-23T21:16:03.301883+0000","last_unstale":"2023-09-23T21:16:05.324697+0000","last_undegraded":"2023-09-23T21:16:05.324697+0000","last_fullsized":"2023-09-23T21:16:05.324697+0000","mapping_epoch":551,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":552,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:27:24.556174+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,5],"acting":[3,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":542,"reported_epoch":554,"state":"active+clean","last_fresh":"2023-09-23T21:16:05.336069+0000","last_change":"2023-09-23T21:16:03.305682+0000","last_active":"2023-09-23T21:16:05.336069+0000","last_peered":"2023-09-23T21:16:05.336069+0000","last_clean":"2023-09-23T21:16:05.336069+0000","last_became_active":"2023-09-23T21:16:03.305015+0000","last_became_peered":"2023-09-23T21:16:03.305015+0000","last_unstale":"2023-09-23T21:16:05.336069+0000","last_undegraded":"2023-09-23T21:16:05.336069+0000","last_fullsized":"2023-09-23T21:16:05.336069+0000","mapping_epoch":551,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":552,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:37:41.953441+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,5],"acting":[3,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.d","version":"555'3445","reported_seq":3465,"reported_epoch":555,"state":"active+clean","last_fresh":"2023-09-23T21:16:06.358722+0000","last_change":"2023-09-23T21:15:25.892834+0000","last_active":"2023-09-23T21:16:06.358722+0000","last_peered":"2023-09-23T21:16:06.358722+0000","last_clean":"2023-09-23T21:16:06.358722+0000","last_became_active":"2023-09-23T21:15:25.892519+0000","last_became_peered":"2023-09-23T21:15:25.892519+0000","last_unstale":"2023-09-23T21:16:06.358722+0000","last_undegraded":"2023-09-23T21:16:06.358722+0000","last_fullsized":"2023-09-23T21:16:06.358722+0000","mapping_epoch":544,"log_start":"0'0","ondisk_log_start":"0'0","created":544,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_clean_scrub_stamp":"2023-09-23T21:15:24.860968+0000","objects_scrubbed":0,"log_size":3445,"log_dups_size":0,"ondisk_log_size":3445,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:46:28.375665+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":225771520,"num_objects":3445,"num_object_clones":0,"num_object_copies":10335,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3445,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3445,"num_write_kb":220480,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,5],"acting":[2,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.7","version":"79'161","reported_seq":677,"reported_epoch":555,"state":"active+clean","last_fresh":"2023-09-23T21:16:06.371599+0000","last_change":"2023-09-23T21:16:06.371599+0000","last_active":"2023-09-23T21:16:06.371599+0000","last_peered":"2023-09-23T21:16:06.371599+0000","last_clean":"2023-09-23T21:16:06.371599+0000","last_became_active":"2023-09-23T21:16:06.371227+0000","last_became_peered":"2023-09-23T21:16:06.371227+0000","last_unstale":"2023-09-23T21:16:06.371599+0000","last_undegraded":"2023-09-23T21:16:06.371599+0000","last_fullsized":"2023-09-23T21:16:06.371599+0000","mapping_epoch":554,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":555,"parent":"0.0","parent_split_bits":3,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:15:51.016427+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:15:51.016427+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:52:11.956754+0000","scrub_duration":0.0017648640000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,2],"acting":[3,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":494,"reported_epoch":554,"state":"active+clean","last_fresh":"2023-09-23T21:16:05.320128+0000","last_change":"2023-09-23T21:16:03.295351+0000","last_active":"2023-09-23T21:16:05.320128+0000","last_peered":"2023-09-23T21:16:05.320128+0000","last_clean":"2023-09-23T21:16:05.320128+0000","last_became_active":"2023-09-23T21:16:03.295028+0000","last_became_peered":"2023-09-23T21:16:03.295028+0000","last_unstale":"2023-09-23T21:16:05.320128+0000","last_undegraded":"2023-09-23T21:16:05.320128+0000","last_fullsized":"2023-09-23T21:16:05.320128+0000","mapping_epoch":551,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":552,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:22:24.204534+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,3],"acting":[0,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":232,"reported_epoch":554,"state":"active+clean","last_fresh":"2023-09-23T21:16:05.326074+0000","last_change":"2023-09-23T21:05:04.326889+0000","last_active":"2023-09-23T21:16:05.326074+0000","last_peered":"2023-09-23T21:16:05.326074+0000","last_clean":"2023-09-23T21:16:05.326074+0000","last_became_active":"2023-09-23T21:05:04.326032+0000","last_became_peered":"2023-09-23T21:05:04.326032+0000","last_unstale":"2023-09-23T21:16:05.326074+0000","last_undegraded":"2023-09-23T21:16:05.326074+0000","last_fullsized":"2023-09-23T21:16:05.326074+0000","mapping_epoch":332,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:00:17.507889+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":564,"reported_epoch":554,"state":"active+clean","last_fresh":"2023-09-23T21:16:05.321463+0000","last_change":"2023-09-23T21:13:13.285608+0000","last_active":"2023-09-23T21:16:05.321463+0000","last_peered":"2023-09-23T21:16:05.321463+0000","last_clean":"2023-09-23T21:16:05.321463+0000","last_became_active":"2023-09-23T21:13:13.285255+0000","last_became_peered":"2023-09-23T21:13:13.285255+0000","last_unstale":"2023-09-23T21:16:05.321463+0000","last_undegraded":"2023-09-23T21:16:05.321463+0000","last_fullsized":"2023-09-23T21:16:05.321463+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":495,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:42:30.087599+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.b","version":"555'3435","reported_seq":3455,"reported_epoch":555,"state":"active+clean","last_fresh":"2023-09-23T21:16:09.225509+0000","last_change":"2023-09-23T21:15:25.906791+0000","last_active":"2023-09-23T21:16:09.225509+0000","last_peered":"2023-09-23T21:16:09.225509+0000","last_clean":"2023-09-23T21:16:09.225509+0000","last_became_active":"2023-09-23T21:15:25.906512+0000","last_became_peered":"2023-09-23T21:15:25.906512+0000","last_unstale":"2023-09-23T21:16:09.225509+0000","last_undegraded":"2023-09-23T21:16:09.225509+0000","last_fullsized":"2023-09-23T21:16:09.225509+0000","mapping_epoch":544,"log_start":"0'0","ondisk_log_start":"0'0","created":544,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_clean_scrub_stamp":"2023-09-23T21:15:24.860968+0000","objects_scrubbed":0,"log_size":3435,"log_dups_size":0,"ondisk_log_size":3435,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:58:51.749775+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":225116160,"num_objects":3435,"num_object_clones":0,"num_object_copies":10305,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3435,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3435,"num_write_kb":219840,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1","version":"79'161","reported_seq":638,"reported_epoch":554,"state":"active+clean","last_fresh":"2023-09-23T21:16:05.319116+0000","last_change":"2023-09-23T21:15:34.345290+0000","last_active":"2023-09-23T21:16:05.319116+0000","last_peered":"2023-09-23T21:16:05.319116+0000","last_clean":"2023-09-23T21:16:05.319116+0000","last_became_active":"2023-09-23T21:13:12.203380+0000","last_became_peered":"2023-09-23T21:13:12.203380+0000","last_unstale":"2023-09-23T21:16:05.319116+0000","last_undegraded":"2023-09-23T21:16:05.319116+0000","last_fullsized":"2023-09-23T21:16:05.319116+0000","mapping_epoch":493,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:15:34.345138+0000","last_deep_scrub":"79'161","last_deep_scrub_stamp":"2023-09-23T21:05:30.652189+0000","last_clean_scrub_stamp":"2023-09-23T21:15:34.345138+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:47:50.349409+0000","scrub_duration":0.0057520000000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":222,"reported_epoch":554,"state":"active+clean","last_fresh":"2023-09-23T21:16:05.319996+0000","last_change":"2023-09-23T21:13:13.238617+0000","last_active":"2023-09-23T21:16:05.319996+0000","last_peered":"2023-09-23T21:16:05.319996+0000","last_clean":"2023-09-23T21:16:05.319996+0000","last_became_active":"2023-09-23T21:13:13.232926+0000","last_became_peered":"2023-09-23T21:13:13.232926+0000","last_unstale":"2023-09-23T21:16:05.319996+0000","last_undegraded":"2023-09-23T21:16:05.319996+0000","last_fullsized":"2023-09-23T21:16:05.319996+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":495,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:17:21.823772+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":488,"reported_epoch":554,"state":"active+clean","last_fresh":"2023-09-23T21:16:05.323028+0000","last_change":"2023-09-23T21:13:14.144918+0000","last_active":"2023-09-23T21:16:05.323028+0000","last_peered":"2023-09-23T21:16:05.323028+0000","last_clean":"2023-09-23T21:16:05.323028+0000","last_became_active":"2023-09-23T21:13:14.132388+0000","last_became_peered":"2023-09-23T21:13:14.132388+0000","last_unstale":"2023-09-23T21:16:05.323028+0000","last_undegraded":"2023-09-23T21:16:05.323028+0000","last_fullsized":"2023-09-23T21:16:05.323028+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":495,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:56:02.619905+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,5],"acting":[4,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":384,"reported_epoch":554,"state":"active+clean","last_fresh":"2023-09-23T21:16:05.321011+0000","last_change":"2023-09-23T21:16:03.304400+0000","last_active":"2023-09-23T21:16:05.321011+0000","last_peered":"2023-09-23T21:16:05.321011+0000","last_clean":"2023-09-23T21:16:05.321011+0000","last_became_active":"2023-09-23T21:16:03.304050+0000","last_became_peered":"2023-09-23T21:16:03.304050+0000","last_unstale":"2023-09-23T21:16:05.321011+0000","last_undegraded":"2023-09-23T21:16:05.321011+0000","last_fullsized":"2023-09-23T21:16:05.321011+0000","mapping_epoch":551,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":552,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:16:56.774755+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,5],"acting":[2,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"11.8","version":"555'3508","reported_seq":3528,"reported_epoch":555,"state":"active+clean","last_fresh":"2023-09-23T21:16:10.284382+0000","last_change":"2023-09-23T21:15:25.905508+0000","last_active":"2023-09-23T21:16:10.284382+0000","last_peered":"2023-09-23T21:16:10.284382+0000","last_clean":"2023-09-23T21:16:10.284382+0000","last_became_active":"2023-09-23T21:15:25.905240+0000","last_became_peered":"2023-09-23T21:15:25.905240+0000","last_unstale":"2023-09-23T21:16:10.284382+0000","last_undegraded":"2023-09-23T21:16:10.284382+0000","last_fullsized":"2023-09-23T21:16:10.284382+0000","mapping_epoch":544,"log_start":"0'0","ondisk_log_start":"0'0","created":544,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_clean_scrub_stamp":"2023-09-23T21:15:24.860968+0000","objects_scrubbed":0,"log_size":3508,"log_dups_size":0,"ondisk_log_size":3508,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:16:01.535087+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":229900288,"num_objects":3508,"num_object_clones":0,"num_object_copies":10524,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3508,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3508,"num_write_kb":224512,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,5],"acting":[0,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.2","version":"79'161","reported_seq":852,"reported_epoch":555,"state":"active+clean","last_fresh":"2023-09-23T21:16:06.366702+0000","last_change":"2023-09-23T21:16:06.366702+0000","last_active":"2023-09-23T21:16:06.366702+0000","last_peered":"2023-09-23T21:16:06.366702+0000","last_clean":"2023-09-23T21:16:06.366702+0000","last_became_active":"2023-09-23T21:16:06.366321+0000","last_became_peered":"2023-09-23T21:16:06.366321+0000","last_unstale":"2023-09-23T21:16:06.366702+0000","last_undegraded":"2023-09-23T21:16:06.366702+0000","last_fullsized":"2023-09-23T21:16:06.366702+0000","mapping_epoch":554,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":555,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:24.802451+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:24.802451+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:17:28.526317+0000","scrub_duration":0.0090861250000000005,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,2],"acting":[0,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":564,"reported_epoch":554,"state":"active+clean","last_fresh":"2023-09-23T21:16:05.313019+0000","last_change":"2023-09-23T21:13:12.206980+0000","last_active":"2023-09-23T21:16:05.313019+0000","last_peered":"2023-09-23T21:16:05.313019+0000","last_clean":"2023-09-23T21:16:05.313019+0000","last_became_active":"2023-09-23T21:13:12.206650+0000","last_became_peered":"2023-09-23T21:13:12.206650+0000","last_unstale":"2023-09-23T21:16:05.313019+0000","last_undegraded":"2023-09-23T21:16:05.313019+0000","last_fullsized":"2023-09-23T21:16:05.313019+0000","mapping_epoch":493,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:10:01.892936+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":547,"reported_epoch":554,"state":"active+clean","last_fresh":"2023-09-23T21:16:05.312593+0000","last_change":"2023-09-23T21:13:13.407073+0000","last_active":"2023-09-23T21:16:05.312593+0000","last_peered":"2023-09-23T21:16:05.312593+0000","last_clean":"2023-09-23T21:16:05.312593+0000","last_became_active":"2023-09-23T21:13:13.395047+0000","last_became_peered":"2023-09-23T21:13:13.395047+0000","last_unstale":"2023-09-23T21:16:05.312593+0000","last_undegraded":"2023-09-23T21:16:05.312593+0000","last_fullsized":"2023-09-23T21:16:05.312593+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":495,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:04:43.384190+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,5],"acting":[2,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":74,"reported_epoch":554,"state":"active+clean","last_fresh":"2023-09-23T21:16:05.331899+0000","last_change":"2023-09-23T21:16:03.303074+0000","last_active":"2023-09-23T21:16:05.331899+0000","last_peered":"2023-09-23T21:16:05.331899+0000","last_clean":"2023-09-23T21:16:05.331899+0000","last_became_active":"2023-09-23T21:16:03.302783+0000","last_became_peered":"2023-09-23T21:16:03.302783+0000","last_unstale":"2023-09-23T21:16:05.331899+0000","last_undegraded":"2023-09-23T21:16:05.331899+0000","last_fullsized":"2023-09-23T21:16:05.331899+0000","mapping_epoch":551,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":552,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:06:44.994049+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.a","version":"555'3483","reported_seq":3502,"reported_epoch":555,"state":"active+clean","last_fresh":"2023-09-23T21:16:06.360600+0000","last_change":"2023-09-23T21:15:25.893846+0000","last_active":"2023-09-23T21:16:06.360600+0000","last_peered":"2023-09-23T21:16:06.360600+0000","last_clean":"2023-09-23T21:16:06.360600+0000","last_became_active":"2023-09-23T21:15:25.893452+0000","last_became_peered":"2023-09-23T21:15:25.893452+0000","last_unstale":"2023-09-23T21:16:06.360600+0000","last_undegraded":"2023-09-23T21:16:06.360600+0000","last_fullsized":"2023-09-23T21:16:06.360600+0000","mapping_epoch":544,"log_start":"0'0","ondisk_log_start":"0'0","created":544,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_clean_scrub_stamp":"2023-09-23T21:15:24.860968+0000","objects_scrubbed":0,"log_size":3483,"log_dups_size":0,"ondisk_log_size":3483,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:42:32.842297+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":228261888,"num_objects":3483,"num_object_clones":0,"num_object_copies":10449,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3483,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3483,"num_write_kb":222912,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,5],"acting":[2,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":350,"reported_epoch":554,"state":"active+clean","last_fresh":"2023-09-23T21:16:05.315178+0000","last_change":"2023-09-23T21:16:03.304554+0000","last_active":"2023-09-23T21:16:05.315178+0000","last_peered":"2023-09-23T21:16:05.315178+0000","last_clean":"2023-09-23T21:16:05.315178+0000","last_became_active":"2023-09-23T21:16:03.303981+0000","last_became_peered":"2023-09-23T21:16:03.303981+0000","last_unstale":"2023-09-23T21:16:05.315178+0000","last_undegraded":"2023-09-23T21:16:05.315178+0000","last_fullsized":"2023-09-23T21:16:05.315178+0000","mapping_epoch":551,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":552,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:48:16.955761+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"79'161","reported_seq":755,"reported_epoch":555,"state":"active+clean","last_fresh":"2023-09-23T21:16:06.358406+0000","last_change":"2023-09-23T21:16:06.358406+0000","last_active":"2023-09-23T21:16:06.358406+0000","last_peered":"2023-09-23T21:16:06.358406+0000","last_clean":"2023-09-23T21:16:06.358406+0000","last_became_active":"2023-09-23T21:16:06.357676+0000","last_became_peered":"2023-09-23T21:16:06.357676+0000","last_unstale":"2023-09-23T21:16:06.358406+0000","last_undegraded":"2023-09-23T21:16:06.358406+0000","last_fullsized":"2023-09-23T21:16:06.358406+0000","mapping_epoch":554,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":555,"parent":"0.0","parent_split_bits":0,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:20.738273+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:20.738273+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:57:34.656729+0000","scrub_duration":0.0088336339999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":562,"reported_epoch":554,"state":"active+clean","last_fresh":"2023-09-23T21:16:05.321386+0000","last_change":"2023-09-23T21:11:08.760417+0000","last_active":"2023-09-23T21:16:05.321386+0000","last_peered":"2023-09-23T21:16:05.321386+0000","last_clean":"2023-09-23T21:16:05.321386+0000","last_became_active":"2023-09-23T21:11:08.760150+0000","last_became_peered":"2023-09-23T21:11:08.760150+0000","last_unstale":"2023-09-23T21:16:05.321386+0000","last_undegraded":"2023-09-23T21:16:05.321386+0000","last_fullsized":"2023-09-23T21:16:05.321386+0000","mapping_epoch":443,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":444,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:23:10.849768+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":139,"reported_epoch":554,"state":"active+clean","last_fresh":"2023-09-23T21:16:05.327711+0000","last_change":"2023-09-23T21:16:03.306007+0000","last_active":"2023-09-23T21:16:05.327711+0000","last_peered":"2023-09-23T21:16:05.327711+0000","last_clean":"2023-09-23T21:16:05.327711+0000","last_became_active":"2023-09-23T21:16:03.305689+0000","last_became_peered":"2023-09-23T21:16:03.305689+0000","last_unstale":"2023-09-23T21:16:05.327711+0000","last_undegraded":"2023-09-23T21:16:05.327711+0000","last_fullsized":"2023-09-23T21:16:05.327711+0000","mapping_epoch":551,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":552,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:03:30.007328+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.9","version":"555'3533","reported_seq":3564,"reported_epoch":555,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T21:16:09.205182+0000","last_change":"2023-09-23T21:16:04.572211+0000","last_active":"2023-09-23T21:16:09.205182+0000","last_peered":"2023-09-23T21:16:09.205182+0000","last_clean":"2023-09-23T21:16:02.272109+0000","last_became_active":"2023-09-23T21:16:04.498018+0000","last_became_peered":"2023-09-23T21:16:04.498018+0000","last_unstale":"2023-09-23T21:16:09.205182+0000","last_undegraded":"2023-09-23T21:16:04.468499+0000","last_fullsized":"2023-09-23T21:16:04.282156+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":544,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_clean_scrub_stamp":"2023-09-23T21:15:24.860968+0000","objects_scrubbed":0,"log_size":3533,"log_dups_size":0,"ondisk_log_size":3533,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:05:31.855325+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":231538688,"num_objects":3533,"num_object_clones":0,"num_object_copies":10599,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3293,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3533,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3533,"num_write_kb":226112,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0],"avail_no_missing":["3","0","7"],"object_location_counts":[{"shards":"0,3","objects":240},{"shards":"0,3,7","objects":3293}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.3","version":"79'161","reported_seq":857,"reported_epoch":554,"state":"active+clean","last_fresh":"2023-09-23T21:16:05.321698+0000","last_change":"2023-09-23T21:13:33.771279+0000","last_active":"2023-09-23T21:16:05.321698+0000","last_peered":"2023-09-23T21:16:05.321698+0000","last_clean":"2023-09-23T21:16:05.321698+0000","last_became_active":"2023-09-23T21:13:33.770980+0000","last_became_peered":"2023-09-23T21:13:33.770980+0000","last_unstale":"2023-09-23T21:16:05.321698+0000","last_undegraded":"2023-09-23T21:16:05.321698+0000","last_fullsized":"2023-09-23T21:16:05.321698+0000","mapping_epoch":504,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":505,"parent":"0.0","parent_split_bits":3,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:05:57.832302+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:05:57.832302+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:19:42.309604+0000","scrub_duration":0.010206847999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":381,"reported_epoch":554,"state":"active+clean","last_fresh":"2023-09-23T21:16:05.318903+0000","last_change":"2023-09-23T21:16:03.300125+0000","last_active":"2023-09-23T21:16:05.318903+0000","last_peered":"2023-09-23T21:16:05.318903+0000","last_clean":"2023-09-23T21:16:05.318903+0000","last_became_active":"2023-09-23T21:16:03.299802+0000","last_became_peered":"2023-09-23T21:16:03.299802+0000","last_unstale":"2023-09-23T21:16:05.318903+0000","last_undegraded":"2023-09-23T21:16:05.318903+0000","last_fullsized":"2023-09-23T21:16:05.318903+0000","mapping_epoch":551,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":552,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:54:27.721872+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,5],"acting":[0,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":554,"reported_epoch":554,"state":"active+clean","last_fresh":"2023-09-23T21:16:05.318873+0000","last_change":"2023-09-23T21:13:44.409212+0000","last_active":"2023-09-23T21:16:05.318873+0000","last_peered":"2023-09-23T21:16:05.318873+0000","last_clean":"2023-09-23T21:16:05.318873+0000","last_became_active":"2023-09-23T21:13:44.408958+0000","last_became_peered":"2023-09-23T21:13:44.408958+0000","last_unstale":"2023-09-23T21:16:05.318873+0000","last_undegraded":"2023-09-23T21:16:05.318873+0000","last_fullsized":"2023-09-23T21:16:05.318873+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:03:52.749964+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":74,"reported_epoch":554,"state":"active+clean","last_fresh":"2023-09-23T21:16:05.319967+0000","last_change":"2023-09-23T21:16:03.298026+0000","last_active":"2023-09-23T21:16:05.319967+0000","last_peered":"2023-09-23T21:16:05.319967+0000","last_clean":"2023-09-23T21:16:05.319967+0000","last_became_active":"2023-09-23T21:16:03.296800+0000","last_became_peered":"2023-09-23T21:16:03.296800+0000","last_unstale":"2023-09-23T21:16:05.319967+0000","last_undegraded":"2023-09-23T21:16:05.319967+0000","last_fullsized":"2023-09-23T21:16:05.319967+0000","mapping_epoch":551,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":552,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:20:45.519914+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,5],"acting":[0,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.c","version":"554'3383","reported_seq":3411,"reported_epoch":555,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T21:16:06.366738+0000","last_change":"2023-09-23T21:16:04.468617+0000","last_active":"2023-09-23T21:16:06.366738+0000","last_peered":"2023-09-23T21:16:06.366738+0000","last_clean":"2023-09-23T21:16:02.275167+0000","last_became_active":"2023-09-23T21:16:04.457601+0000","last_became_peered":"2023-09-23T21:16:04.457601+0000","last_unstale":"2023-09-23T21:16:06.366738+0000","last_undegraded":"2023-09-23T21:16:04.334088+0000","last_fullsized":"2023-09-23T21:16:04.288950+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":544,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_clean_scrub_stamp":"2023-09-23T21:15:24.860968+0000","objects_scrubbed":0,"log_size":3383,"log_dups_size":0,"ondisk_log_size":3383,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:49:10.481384+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":221708288,"num_objects":3383,"num_object_clones":0,"num_object_copies":10149,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3299,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3383,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3383,"num_write_kb":216512,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,5],"acting":[2,4],"avail_no_missing":["2","4","7"],"object_location_counts":[{"shards":"2,4","objects":84},{"shards":"2,4,7","objects":3299}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.6","version":"79'161","reported_seq":486,"reported_epoch":554,"state":"active+clean","last_fresh":"2023-09-23T21:16:05.319887+0000","last_change":"2023-09-23T21:15:43.333919+0000","last_active":"2023-09-23T21:16:05.319887+0000","last_peered":"2023-09-23T21:16:05.319887+0000","last_clean":"2023-09-23T21:16:05.319887+0000","last_became_active":"2023-09-23T21:13:33.769303+0000","last_became_peered":"2023-09-23T21:13:33.769303+0000","last_unstale":"2023-09-23T21:16:05.319887+0000","last_undegraded":"2023-09-23T21:16:05.319887+0000","last_fullsized":"2023-09-23T21:16:05.319887+0000","mapping_epoch":504,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":505,"parent":"0.0","parent_split_bits":3,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:15:43.333733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:15:43.333733+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:42:29.787357+0000","scrub_duration":0.0092107530000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":134,"reported_epoch":554,"state":"active+clean","last_fresh":"2023-09-23T21:16:05.321512+0000","last_change":"2023-09-23T21:16:03.297700+0000","last_active":"2023-09-23T21:16:05.321512+0000","last_peered":"2023-09-23T21:16:05.321512+0000","last_clean":"2023-09-23T21:16:05.321512+0000","last_became_active":"2023-09-23T21:16:03.297392+0000","last_became_peered":"2023-09-23T21:16:03.297392+0000","last_unstale":"2023-09-23T21:16:05.321512+0000","last_undegraded":"2023-09-23T21:16:05.321512+0000","last_fullsized":"2023-09-23T21:16:05.321512+0000","mapping_epoch":551,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":552,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:52:19.462100+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,2],"acting":[0,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":438,"reported_epoch":554,"state":"active+clean","last_fresh":"2023-09-23T21:16:05.326007+0000","last_change":"2023-09-23T21:13:30.790590+0000","last_active":"2023-09-23T21:16:05.326007+0000","last_peered":"2023-09-23T21:16:05.326007+0000","last_clean":"2023-09-23T21:16:05.326007+0000","last_became_active":"2023-09-23T21:13:30.790253+0000","last_became_peered":"2023-09-23T21:13:30.790253+0000","last_unstale":"2023-09-23T21:16:05.326007+0000","last_undegraded":"2023-09-23T21:16:05.326007+0000","last_fullsized":"2023-09-23T21:16:05.326007+0000","mapping_epoch":501,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:23:41.843250+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,0],"acting":[2,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":510,"reported_epoch":554,"state":"active+clean","last_fresh":"2023-09-23T21:16:05.318867+0000","last_change":"2023-09-23T21:16:03.300123+0000","last_active":"2023-09-23T21:16:05.318867+0000","last_peered":"2023-09-23T21:16:05.318867+0000","last_clean":"2023-09-23T21:16:05.318867+0000","last_became_active":"2023-09-23T21:16:03.299787+0000","last_became_peered":"2023-09-23T21:16:03.299787+0000","last_unstale":"2023-09-23T21:16:05.318867+0000","last_undegraded":"2023-09-23T21:16:05.318867+0000","last_fullsized":"2023-09-23T21:16:05.318867+0000","mapping_epoch":551,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":552,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:26:54.345438+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.f","version":"554'3431","reported_seq":3459,"reported_epoch":555,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T21:16:06.354591+0000","last_change":"2023-09-23T21:16:04.443490+0000","last_active":"2023-09-23T21:16:06.354591+0000","last_peered":"2023-09-23T21:16:06.354591+0000","last_clean":"2023-09-23T21:16:02.283321+0000","last_became_active":"2023-09-23T21:16:04.426841+0000","last_became_peered":"2023-09-23T21:16:04.426841+0000","last_unstale":"2023-09-23T21:16:06.354591+0000","last_undegraded":"2023-09-23T21:16:04.318737+0000","last_fullsized":"2023-09-23T21:16:04.295008+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":544,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_clean_scrub_stamp":"2023-09-23T21:15:24.860968+0000","objects_scrubbed":0,"log_size":3431,"log_dups_size":0,"ondisk_log_size":3431,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:24:06.094046+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":224854016,"num_objects":3431,"num_object_clones":0,"num_object_copies":10293,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3357,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3431,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3431,"num_write_kb":219584,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,4],"acting":[2,0],"avail_no_missing":["2","0","7"],"object_location_counts":[{"shards":"0,2","objects":74},{"shards":"0,2,7","objects":3357}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.5","version":"79'161","reported_seq":592,"reported_epoch":554,"state":"active+clean","last_fresh":"2023-09-23T21:16:05.327631+0000","last_change":"2023-09-23T21:15:41.108638+0000","last_active":"2023-09-23T21:16:05.327631+0000","last_peered":"2023-09-23T21:16:05.327631+0000","last_clean":"2023-09-23T21:16:05.327631+0000","last_became_active":"2023-09-23T21:13:38.373633+0000","last_became_peered":"2023-09-23T21:13:38.373633+0000","last_unstale":"2023-09-23T21:16:05.327631+0000","last_undegraded":"2023-09-23T21:16:05.327631+0000","last_fullsized":"2023-09-23T21:16:05.327631+0000","mapping_epoch":507,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":3,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:15:41.108579+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:15:41.108579+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:21:58.075649+0000","scrub_duration":0.0061511960000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":109,"reported_epoch":554,"state":"active+clean","last_fresh":"2023-09-23T21:16:05.332156+0000","last_change":"2023-09-23T21:16:03.318211+0000","last_active":"2023-09-23T21:16:05.332156+0000","last_peered":"2023-09-23T21:16:05.332156+0000","last_clean":"2023-09-23T21:16:05.332156+0000","last_became_active":"2023-09-23T21:16:03.303015+0000","last_became_peered":"2023-09-23T21:16:03.303015+0000","last_unstale":"2023-09-23T21:16:05.332156+0000","last_undegraded":"2023-09-23T21:16:05.332156+0000","last_fullsized":"2023-09-23T21:16:05.332156+0000","mapping_epoch":551,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":552,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:40:08.913321+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,5],"acting":[0,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":535,"reported_epoch":554,"state":"active+clean","last_fresh":"2023-09-23T21:16:05.325090+0000","last_change":"2023-09-23T21:13:30.807520+0000","last_active":"2023-09-23T21:16:05.325090+0000","last_peered":"2023-09-23T21:16:05.325090+0000","last_clean":"2023-09-23T21:16:05.325090+0000","last_became_active":"2023-09-23T21:13:30.807222+0000","last_became_peered":"2023-09-23T21:13:30.807222+0000","last_unstale":"2023-09-23T21:16:05.325090+0000","last_undegraded":"2023-09-23T21:16:05.325090+0000","last_fullsized":"2023-09-23T21:16:05.325090+0000","mapping_epoch":501,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:40:31.989749+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":406,"reported_epoch":554,"state":"active+clean","last_fresh":"2023-09-23T21:16:05.324777+0000","last_change":"2023-09-23T21:16:03.302310+0000","last_active":"2023-09-23T21:16:05.324777+0000","last_peered":"2023-09-23T21:16:05.324777+0000","last_clean":"2023-09-23T21:16:05.324777+0000","last_became_active":"2023-09-23T21:16:03.301972+0000","last_became_peered":"2023-09-23T21:16:03.301972+0000","last_unstale":"2023-09-23T21:16:05.324777+0000","last_undegraded":"2023-09-23T21:16:05.324777+0000","last_fullsized":"2023-09-23T21:16:05.324777+0000","mapping_epoch":551,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":552,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:45:05.416579+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.e","version":"555'3422","reported_seq":3451,"reported_epoch":555,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T21:16:07.005112+0000","last_change":"2023-09-23T21:16:04.539515+0000","last_active":"2023-09-23T21:16:07.005112+0000","last_peered":"2023-09-23T21:16:07.005112+0000","last_clean":"2023-09-23T21:16:02.279814+0000","last_became_active":"2023-09-23T21:16:04.529483+0000","last_became_peered":"2023-09-23T21:16:04.529483+0000","last_unstale":"2023-09-23T21:16:07.005112+0000","last_undegraded":"2023-09-23T21:16:04.306110+0000","last_fullsized":"2023-09-23T21:16:04.286054+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":544,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_clean_scrub_stamp":"2023-09-23T21:15:24.860968+0000","objects_scrubbed":0,"log_size":3422,"log_dups_size":0,"ondisk_log_size":3422,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:13:07.891388+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":224264192,"num_objects":3422,"num_object_clones":0,"num_object_copies":10266,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3294,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3422,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3422,"num_write_kb":219008,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0],"avail_no_missing":["4","0","7"],"object_location_counts":[{"shards":"0,4","objects":128},{"shards":"0,4,7","objects":3294}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.4","version":"79'161","reported_seq":943,"reported_epoch":555,"state":"active+clean","last_fresh":"2023-09-23T21:16:06.355860+0000","last_change":"2023-09-23T21:16:06.355860+0000","last_active":"2023-09-23T21:16:06.355860+0000","last_peered":"2023-09-23T21:16:06.355860+0000","last_clean":"2023-09-23T21:16:06.355860+0000","last_became_active":"2023-09-23T21:16:06.355606+0000","last_became_peered":"2023-09-23T21:16:06.355606+0000","last_unstale":"2023-09-23T21:16:06.355860+0000","last_undegraded":"2023-09-23T21:16:06.355860+0000","last_fullsized":"2023-09-23T21:16:06.355860+0000","mapping_epoch":554,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":555,"parent":"0.0","parent_split_bits":3,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:20.012764+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:20.012764+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:26:44.975104+0000","scrub_duration":0.0045251919999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,4],"acting":[3,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":503,"reported_epoch":554,"state":"active+clean","last_fresh":"2023-09-23T21:16:05.315963+0000","last_change":"2023-09-23T21:16:03.297206+0000","last_active":"2023-09-23T21:16:05.315963+0000","last_peered":"2023-09-23T21:16:05.315963+0000","last_clean":"2023-09-23T21:16:05.315963+0000","last_became_active":"2023-09-23T21:16:03.296874+0000","last_became_peered":"2023-09-23T21:16:03.296874+0000","last_unstale":"2023-09-23T21:16:05.315963+0000","last_undegraded":"2023-09-23T21:16:05.315963+0000","last_fullsized":"2023-09-23T21:16:05.315963+0000","mapping_epoch":551,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":552,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:51:36.046712+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.6","version":"70'1","reported_seq":539,"reported_epoch":554,"state":"active+clean","last_fresh":"2023-09-23T21:16:05.320595+0000","last_change":"2023-09-23T21:16:03.309441+0000","last_active":"2023-09-23T21:16:05.320595+0000","last_peered":"2023-09-23T21:16:05.320595+0000","last_clean":"2023-09-23T21:16:05.320595+0000","last_became_active":"2023-09-23T21:16:03.298366+0000","last_became_peered":"2023-09-23T21:16:03.298366+0000","last_unstale":"2023-09-23T21:16:05.320595+0000","last_undegraded":"2023-09-23T21:16:05.320595+0000","last_fullsized":"2023-09-23T21:16:05.320595+0000","mapping_epoch":551,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":552,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:32:17.480958+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,5],"acting":[0,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":129,"reported_epoch":554,"state":"active+clean","last_fresh":"2023-09-23T21:16:05.322475+0000","last_change":"2023-09-23T21:13:12.216895+0000","last_active":"2023-09-23T21:16:05.322475+0000","last_peered":"2023-09-23T21:16:05.322475+0000","last_clean":"2023-09-23T21:16:05.322475+0000","last_became_active":"2023-09-23T21:13:12.216654+0000","last_became_peered":"2023-09-23T21:13:12.216654+0000","last_unstale":"2023-09-23T21:16:05.322475+0000","last_undegraded":"2023-09-23T21:16:05.322475+0000","last_fullsized":"2023-09-23T21:16:05.322475+0000","mapping_epoch":493,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:02:40.430096+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.19","version":"79'161","reported_seq":745,"reported_epoch":555,"state":"active+clean","last_fresh":"2023-09-23T21:16:06.355985+0000","last_change":"2023-09-23T21:16:06.355985+0000","last_active":"2023-09-23T21:16:06.355985+0000","last_peered":"2023-09-23T21:16:06.355985+0000","last_clean":"2023-09-23T21:16:06.355985+0000","last_became_active":"2023-09-23T21:16:06.355669+0000","last_became_peered":"2023-09-23T21:16:06.355669+0000","last_unstale":"2023-09-23T21:16:06.355985+0000","last_undegraded":"2023-09-23T21:16:06.355985+0000","last_fullsized":"2023-09-23T21:16:06.355985+0000","mapping_epoch":554,"log_start":"0'0","ondisk_log_start":"0'0","created":479,"last_epoch_clean":555,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:15:48.271791+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:15:48.271791+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:05:06.351635+0000","scrub_duration":0.0057149180000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":74,"reported_epoch":554,"state":"active+clean","last_fresh":"2023-09-23T21:16:05.318780+0000","last_change":"2023-09-23T21:16:03.298682+0000","last_active":"2023-09-23T21:16:05.318780+0000","last_peered":"2023-09-23T21:16:05.318780+0000","last_clean":"2023-09-23T21:16:05.318780+0000","last_became_active":"2023-09-23T21:16:03.297884+0000","last_became_peered":"2023-09-23T21:16:03.297884+0000","last_unstale":"2023-09-23T21:16:05.318780+0000","last_undegraded":"2023-09-23T21:16:05.318780+0000","last_fullsized":"2023-09-23T21:16:05.318780+0000","mapping_epoch":551,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":552,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:21:52.303581+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":58,"reported_epoch":554,"state":"active+clean","last_fresh":"2023-09-23T21:16:05.325910+0000","last_change":"2023-09-23T21:13:38.373814+0000","last_active":"2023-09-23T21:16:05.325910+0000","last_peered":"2023-09-23T21:16:05.325910+0000","last_clean":"2023-09-23T21:16:05.325910+0000","last_became_active":"2023-09-23T21:13:38.373458+0000","last_became_peered":"2023-09-23T21:13:38.373458+0000","last_unstale":"2023-09-23T21:16:05.325910+0000","last_undegraded":"2023-09-23T21:16:05.325910+0000","last_fullsized":"2023-09-23T21:16:05.325910+0000","mapping_epoch":507,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:23:08.288401+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.1e","version":"79'161","reported_seq":520,"reported_epoch":555,"state":"active+clean","last_fresh":"2023-09-23T21:16:06.358734+0000","last_change":"2023-09-23T21:16:06.358734+0000","last_active":"2023-09-23T21:16:06.358734+0000","last_peered":"2023-09-23T21:16:06.358734+0000","last_clean":"2023-09-23T21:16:06.358734+0000","last_became_active":"2023-09-23T21:16:06.358167+0000","last_became_peered":"2023-09-23T21:16:06.358167+0000","last_unstale":"2023-09-23T21:16:06.358734+0000","last_undegraded":"2023-09-23T21:16:06.358734+0000","last_fullsized":"2023-09-23T21:16:06.358734+0000","mapping_epoch":554,"log_start":"0'0","ondisk_log_start":"0'0","created":479,"last_epoch_clean":555,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:15:53.061319+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:15:53.061319+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:42:43.609298+0000","scrub_duration":0.007429878,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":150,"reported_epoch":554,"state":"active+clean","last_fresh":"2023-09-23T21:16:05.322859+0000","last_change":"2023-09-23T21:16:03.330054+0000","last_active":"2023-09-23T21:16:05.322859+0000","last_peered":"2023-09-23T21:16:05.322859+0000","last_clean":"2023-09-23T21:16:05.322859+0000","last_became_active":"2023-09-23T21:16:03.305266+0000","last_became_peered":"2023-09-23T21:16:03.305266+0000","last_unstale":"2023-09-23T21:16:05.322859+0000","last_undegraded":"2023-09-23T21:16:05.322859+0000","last_fullsized":"2023-09-23T21:16:05.322859+0000","mapping_epoch":551,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":552,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:43:19.578055+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,2],"acting":[4,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":119,"reported_epoch":554,"state":"active+clean","last_fresh":"2023-09-23T21:16:05.322653+0000","last_change":"2023-09-23T21:12:10.770474+0000","last_active":"2023-09-23T21:16:05.322653+0000","last_peered":"2023-09-23T21:16:05.322653+0000","last_clean":"2023-09-23T21:16:05.322653+0000","last_became_active":"2023-09-23T21:12:10.770184+0000","last_became_peered":"2023-09-23T21:12:10.770184+0000","last_unstale":"2023-09-23T21:16:05.322653+0000","last_undegraded":"2023-09-23T21:16:05.322653+0000","last_fullsized":"2023-09-23T21:16:05.322653+0000","mapping_epoch":470,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:05:09.407151+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":345,"reported_epoch":554,"state":"active+clean","last_fresh":"2023-09-23T21:16:05.316561+0000","last_change":"2023-09-23T21:12:10.771160+0000","last_active":"2023-09-23T21:16:05.316561+0000","last_peered":"2023-09-23T21:16:05.316561+0000","last_clean":"2023-09-23T21:16:05.316561+0000","last_became_active":"2023-09-23T21:12:10.757808+0000","last_became_peered":"2023-09-23T21:12:10.757808+0000","last_unstale":"2023-09-23T21:16:05.316561+0000","last_undegraded":"2023-09-23T21:16:05.316561+0000","last_fullsized":"2023-09-23T21:16:05.316561+0000","mapping_epoch":470,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:30:50.987139+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":129,"reported_epoch":554,"state":"active+clean","last_fresh":"2023-09-23T21:16:05.325181+0000","last_change":"2023-09-23T21:13:12.216602+0000","last_active":"2023-09-23T21:16:05.325181+0000","last_peered":"2023-09-23T21:16:05.325181+0000","last_clean":"2023-09-23T21:16:05.325181+0000","last_became_active":"2023-09-23T21:13:12.216204+0000","last_became_peered":"2023-09-23T21:13:12.216204+0000","last_unstale":"2023-09-23T21:16:05.325181+0000","last_undegraded":"2023-09-23T21:16:05.325181+0000","last_fullsized":"2023-09-23T21:16:05.325181+0000","mapping_epoch":493,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:23:37.043390+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1c","version":"79'161","reported_seq":967,"reported_epoch":555,"state":"active+clean","last_fresh":"2023-09-23T21:16:06.373705+0000","last_change":"2023-09-23T21:16:06.373705+0000","last_active":"2023-09-23T21:16:06.373705+0000","last_peered":"2023-09-23T21:16:06.373705+0000","last_clean":"2023-09-23T21:16:06.373705+0000","last_became_active":"2023-09-23T21:16:06.371307+0000","last_became_peered":"2023-09-23T21:16:06.371307+0000","last_unstale":"2023-09-23T21:16:06.373705+0000","last_undegraded":"2023-09-23T21:16:06.373705+0000","last_fullsized":"2023-09-23T21:16:06.373705+0000","mapping_epoch":554,"log_start":"0'0","ondisk_log_start":"0'0","created":479,"last_epoch_clean":555,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:15:54.019828+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:15:54.019828+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:29:19.635056+0000","scrub_duration":0.0020962530000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,2],"acting":[3,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":153,"reported_epoch":554,"state":"active+clean","last_fresh":"2023-09-23T21:16:05.317954+0000","last_change":"2023-09-23T21:16:03.304185+0000","last_active":"2023-09-23T21:16:05.317954+0000","last_peered":"2023-09-23T21:16:05.317954+0000","last_clean":"2023-09-23T21:16:05.317954+0000","last_became_active":"2023-09-23T21:16:03.303832+0000","last_became_peered":"2023-09-23T21:16:03.303832+0000","last_unstale":"2023-09-23T21:16:05.317954+0000","last_undegraded":"2023-09-23T21:16:05.317954+0000","last_fullsized":"2023-09-23T21:16:05.317954+0000","mapping_epoch":551,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":552,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:12:49.724922+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":223,"reported_epoch":554,"state":"active+clean","last_fresh":"2023-09-23T21:16:05.313125+0000","last_change":"2023-09-23T21:13:12.200767+0000","last_active":"2023-09-23T21:16:05.313125+0000","last_peered":"2023-09-23T21:16:05.313125+0000","last_clean":"2023-09-23T21:16:05.313125+0000","last_became_active":"2023-09-23T21:13:12.200435+0000","last_became_peered":"2023-09-23T21:13:12.200435+0000","last_unstale":"2023-09-23T21:16:05.313125+0000","last_undegraded":"2023-09-23T21:16:05.313125+0000","last_fullsized":"2023-09-23T21:16:05.313125+0000","mapping_epoch":493,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:15:44.937886+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.1d","version":"79'161","reported_seq":686,"reported_epoch":554,"state":"active+clean","last_fresh":"2023-09-23T21:16:05.332861+0000","last_change":"2023-09-23T21:15:23.802125+0000","last_active":"2023-09-23T21:16:05.332861+0000","last_peered":"2023-09-23T21:16:05.332861+0000","last_clean":"2023-09-23T21:16:05.332861+0000","last_became_active":"2023-09-23T21:15:23.801704+0000","last_became_peered":"2023-09-23T21:15:23.801704+0000","last_unstale":"2023-09-23T21:16:05.332861+0000","last_undegraded":"2023-09-23T21:16:05.332861+0000","last_fullsized":"2023-09-23T21:16:05.332861+0000","mapping_epoch":542,"log_start":"0'0","ondisk_log_start":"0'0","created":479,"last_epoch_clean":543,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:09:36.858077+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:09:36.858077+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.010304610000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,2],"acting":[3,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":82,"reported_epoch":554,"state":"active+clean","last_fresh":"2023-09-23T21:16:05.331696+0000","last_change":"2023-09-23T21:13:38.380574+0000","last_active":"2023-09-23T21:16:05.331696+0000","last_peered":"2023-09-23T21:16:05.331696+0000","last_clean":"2023-09-23T21:16:05.331696+0000","last_became_active":"2023-09-23T21:13:38.373252+0000","last_became_peered":"2023-09-23T21:13:38.373252+0000","last_unstale":"2023-09-23T21:16:05.331696+0000","last_undegraded":"2023-09-23T21:16:05.331696+0000","last_fullsized":"2023-09-23T21:16:05.331696+0000","mapping_epoch":507,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:58:14.788031+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":11,"num_bytes_recovered":3324,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":129,"reported_epoch":554,"state":"active+clean","last_fresh":"2023-09-23T21:16:05.323529+0000","last_change":"2023-09-23T21:12:10.793610+0000","last_active":"2023-09-23T21:16:05.323529+0000","last_peered":"2023-09-23T21:16:05.323529+0000","last_clean":"2023-09-23T21:16:05.323529+0000","last_became_active":"2023-09-23T21:12:10.793081+0000","last_became_peered":"2023-09-23T21:12:10.793081+0000","last_unstale":"2023-09-23T21:16:05.323529+0000","last_undegraded":"2023-09-23T21:16:05.323529+0000","last_fullsized":"2023-09-23T21:16:05.323529+0000","mapping_epoch":470,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:34:36.291550+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":11,"num_pg":16,"stat_sum":{"num_bytes":3628793856,"num_objects":55371,"num_object_clones":0,"num_object_copies":166113,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":23080,"num_objects_misplaced":3177,"num_objects_unfound":0,"num_objects_dirty":55371,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":55371,"num_write_kb":3543744,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":169,"num_bytes_recovered":11075584,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":10974920704,"data_stored":10974920704,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":55371,"ondisk_log_size":55371,"up":48,"acting":40,"num_store_stats":6},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":65,"num_bytes_recovered":13726,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":147456,"data_stored":26300,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":14322,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":31,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":262688,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2109440,"data_stored":2098784,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4991,"ondisk_log_size":4991,"up":93,"acting":93,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":0,"seq":236223201708,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":6,"up_from":0,"seq":704374636792,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":1,"up_from":0,"seq":1503238553724,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":0,"up_from":493,"seq":2117418876965,"num_pgs":83,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2577424,"kb_used_data":2318088,"kb_used_omap":36,"kb_used_meta":259291,"kb_avail":91167728,"statfs":{"total":95995035648,"available":93355753472,"internally_reserved":0,"allocated":2373722112,"data_stored":2371126547,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":37336,"internal_metadata":265514536},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 21:15:17 2023","interfaces":[{"interface":"back","average":{"1min":1.6990000000000001,"5min":0.94399999999999995,"15min":0.81799999999999995},"min":{"1min":0.22500000000000001,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":4.665,"5min":4.9000000000000004,"15min":4.9000000000000004},"last":1.258},{"interface":"front","average":{"1min":1.5620000000000001,"5min":0.92900000000000005,"15min":0.82299999999999995},"min":{"1min":0.20799999999999999,"5min":0.20799999999999999,"15min":0.20799999999999999},"max":{"1min":11.964,"5min":11.964,"15min":11.964},"last":3.113}]},{"osd":2,"last update":"Sat Sep 23 21:15:17 2023","interfaces":[{"interface":"back","average":{"1min":1.661,"5min":0.85499999999999998,"15min":0.72099999999999997},"min":{"1min":0.29399999999999998,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":7.133,"5min":7.133,"15min":7.133},"last":1.7410000000000001},{"interface":"front","average":{"1min":1.647,"5min":1.103,"15min":1.0129999999999999},"min":{"1min":0.27900000000000003,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":6.6470000000000002,"5min":6.6470000000000002,"15min":6.6470000000000002},"last":1.8520000000000001}]},{"osd":3,"last update":"Sat Sep 23 21:15:17 2023","interfaces":[{"interface":"back","average":{"1min":1.8939999999999999,"5min":1.073,"15min":0.93600000000000005},"min":{"1min":0.27300000000000002,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":6.2060000000000004,"5min":6.2060000000000004,"15min":6.2060000000000004},"last":3.46},{"interface":"front","average":{"1min":1.4199999999999999,"5min":1.1779999999999999,"15min":1.1379999999999999},"min":{"1min":0.28299999999999997,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":4.8040000000000003,"5min":4.8040000000000003,"15min":4.8040000000000003},"last":0.81200000000000006}]},{"osd":4,"last update":"Sat Sep 23 21:15:17 2023","interfaces":[{"interface":"back","average":{"1min":1.2789999999999999,"5min":1.1299999999999999,"15min":1.105},"min":{"1min":0.312,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":4.4409999999999998,"5min":4.4409999999999998,"15min":4.4409999999999998},"last":0.72299999999999998},{"interface":"front","average":{"1min":1.4570000000000001,"5min":1.0880000000000001,"15min":1.026},"min":{"1min":0.41699999999999998,"5min":0.317,"15min":0.317},"max":{"1min":4.6929999999999996,"5min":5.093,"15min":5.093},"last":0.42799999999999999}]},{"osd":5,"last update":"Sat Sep 23 21:15:17 2023","interfaces":[{"interface":"back","average":{"1min":1.413,"5min":1.2709999999999999,"15min":1.2470000000000001},"min":{"1min":0.29999999999999999,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":4.0869999999999997,"5min":4.5640000000000001,"15min":4.5640000000000001},"last":1.788},{"interface":"front","average":{"1min":1.3959999999999999,"5min":1.1299999999999999,"15min":1.0860000000000001},"min":{"1min":0.313,"5min":0.313,"15min":0.313},"max":{"1min":7.4569999999999999,"5min":7.4569999999999999,"15min":7.4569999999999999},"last":0.76100000000000001}]},{"osd":6,"last update":"Sat Sep 23 21:15:17 2023","interfaces":[{"interface":"back","average":{"1min":1.8420000000000001,"5min":1.27,"15min":1.1739999999999999},"min":{"1min":0.46600000000000003,"5min":0.38100000000000001,"15min":0.38100000000000001},"max":{"1min":7.2050000000000001,"5min":7.2050000000000001,"15min":7.2050000000000001},"last":0.5},{"interface":"front","average":{"1min":1.5640000000000001,"5min":1.359,"15min":1.325},"min":{"1min":0.39600000000000002,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":4.7850000000000001,"5min":5.0860000000000003,"15min":5.0860000000000003},"last":1.8220000000000001}]},{"osd":7,"last update":"Sat Sep 23 21:15:17 2023","interfaces":[{"interface":"back","average":{"1min":2.1600000000000001,"5min":1.399,"15min":1.272},"min":{"1min":0.40899999999999997,"5min":0.40899999999999997,"15min":0.40899999999999997},"max":{"1min":7.5810000000000004,"5min":7.5810000000000004,"15min":7.5810000000000004},"last":0.78600000000000003},{"interface":"front","average":{"1min":1.6850000000000001,"5min":1.2629999999999999,"15min":1.1930000000000001},"min":{"1min":0.56000000000000005,"5min":0.441,"15min":0.441},"max":{"1min":4.6349999999999998,"5min":5.1289999999999996,"15min":5.1289999999999996},"last":0.46999999999999997}]}]},{"osd":2,"up_from":93,"seq":399431958836,"num_pgs":62,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2855600,"kb_used_data":1971880,"kb_used_omap":81,"kb_used_meta":883630,"kb_avail":90889552,"statfs":{"total":95995035648,"available":93070901248,"internally_reserved":0,"allocated":2019205120,"data_stored":2016627417,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":83750,"internal_metadata":904837338},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:15:17 2023","interfaces":[{"interface":"back","average":{"1min":2.2210000000000001,"5min":1.1100000000000001,"15min":0.92500000000000004},"min":{"1min":0.20899999999999999,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":9.9329999999999998,"5min":9.9329999999999998,"15min":9.9329999999999998},"last":0.26000000000000001},{"interface":"front","average":{"1min":2.6659999999999999,"5min":1.865,"15min":1.7310000000000001},"min":{"1min":0.24399999999999999,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":11.269,"5min":18.111999999999998,"15min":18.111999999999998},"last":0.23999999999999999}]},{"osd":1,"last update":"Sat Sep 23 21:15:17 2023","interfaces":[{"interface":"back","average":{"1min":2.0539999999999998,"5min":1.2629999999999999,"15min":2.472},"min":{"1min":0.20599999999999999,"5min":0.188,"15min":0.188},"max":{"1min":9.9890000000000008,"5min":11.005000000000001,"15min":34.396999999999998},"last":0.38200000000000001},{"interface":"front","average":{"1min":2.2280000000000002,"5min":1.296,"15min":2.6859999999999999},"min":{"1min":0.26200000000000001,"5min":0.217,"15min":0.217},"max":{"1min":16.274000000000001,"5min":18.081,"15min":34.462000000000003},"last":0.40600000000000003}]},{"osd":3,"last update":"Sat Sep 23 21:15:58 2023","interfaces":[{"interface":"back","average":{"1min":1.6579999999999999,"5min":1.4950000000000001,"15min":2.2519999999999998},"min":{"1min":0.32700000000000001,"5min":0.20599999999999999,"15min":0.20599999999999999},"max":{"1min":7.5220000000000002,"5min":14.442,"15min":41.057000000000002},"last":7.085},{"interface":"front","average":{"1min":2.7149999999999999,"5min":1.8440000000000001,"15min":2.5910000000000002},"min":{"1min":0.40899999999999997,"5min":0.21299999999999999,"15min":0.21299999999999999},"max":{"1min":16.047999999999998,"5min":16.047999999999998,"15min":41.012},"last":1.405}]},{"osd":4,"last update":"Sat Sep 23 21:15:17 2023","interfaces":[{"interface":"back","average":{"1min":1.5780000000000001,"5min":1.2749999999999999,"15min":1.4219999999999999},"min":{"1min":0.32500000000000001,"5min":0.249,"15min":0.249},"max":{"1min":7.9770000000000003,"5min":10.266999999999999,"15min":10.266999999999999},"last":3.5590000000000002},{"interface":"front","average":{"1min":1.74,"5min":1.4419999999999999,"15min":1.8029999999999999},"min":{"1min":0.308,"5min":0.29399999999999998,"15min":0.26700000000000002},"max":{"1min":5.556,"5min":7.5960000000000001,"15min":14.039999999999999},"last":0.48499999999999999}]},{"osd":5,"last update":"Sat Sep 23 21:15:17 2023","interfaces":[{"interface":"back","average":{"1min":1.1850000000000001,"5min":1.218,"15min":1.5049999999999999},"min":{"1min":0.34699999999999998,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":3.7250000000000001,"5min":9.5039999999999996,"15min":10.069000000000001},"last":3.6030000000000002},{"interface":"front","average":{"1min":1.6779999999999999,"5min":1.2849999999999999,"15min":1.7529999999999999},"min":{"1min":0.33800000000000002,"5min":0.29699999999999999,"15min":0.29399999999999998},"max":{"1min":5.8330000000000002,"5min":7.5549999999999997,"15min":13.98},"last":0.54700000000000004}]},{"osd":6,"last update":"Sat Sep 23 21:15:17 2023","interfaces":[{"interface":"back","average":{"1min":1.9239999999999999,"5min":1.571,"15min":1.988},"min":{"1min":0.46300000000000002,"5min":0.373,"15min":0.35699999999999998},"max":{"1min":7.8559999999999999,"5min":7.8559999999999999,"15min":14.016},"last":0.59999999999999998},{"interface":"front","average":{"1min":1.7330000000000001,"5min":1.7330000000000001,"15min":2.157},"min":{"1min":0.377,"5min":0.377,"15min":0.35599999999999998},"max":{"1min":5.649,"5min":9.3469999999999995,"15min":14.068},"last":0.51800000000000002}]},{"osd":7,"last update":"Sat Sep 23 21:15:17 2023","interfaces":[{"interface":"back","average":{"1min":1.6839999999999999,"5min":1.462,"15min":1.6040000000000001},"min":{"1min":0.33700000000000002,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":6.0510000000000002,"5min":18.021999999999998,"15min":18.021999999999998},"last":0.441},{"interface":"front","average":{"1min":1.359,"5min":1.425,"15min":1.6910000000000001},"min":{"1min":0.41899999999999998,"5min":0.34799999999999998,"15min":0.32200000000000001},"max":{"1min":3.7549999999999999,"5min":10.359,"15min":10.92},"last":3.629}]}]},{"osd":3,"up_from":98,"seq":420906795310,"num_pgs":82,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3235488,"kb_used_data":2296920,"kb_used_omap":7,"kb_used_meta":938552,"kb_avail":90509664,"statfs":{"total":95995035648,"available":92681895936,"internally_reserved":0,"allocated":2352046080,"data_stored":2349486846,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7576,"internal_metadata":961077864},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,4,3,0,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:15:13 2023","interfaces":[{"interface":"back","average":{"1min":10.904,"5min":2.4510000000000001,"15min":1.042},"min":{"1min":0.28499999999999998,"5min":0.20499999999999999,"15min":0.20499999999999999},"max":{"1min":147.38900000000001,"5min":147.38900000000001,"15min":147.38900000000001},"last":0.54100000000000004},{"interface":"front","average":{"1min":10.178000000000001,"5min":2.5699999999999998,"15min":1.302},"min":{"1min":0.23100000000000001,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":149.989,"5min":149.989,"15min":149.989},"last":10.577999999999999}]},{"osd":1,"last update":"Sat Sep 23 21:15:13 2023","interfaces":[{"interface":"back","average":{"1min":1.5589999999999999,"5min":1.2,"15min":2.1880000000000002},"min":{"1min":0.23699999999999999,"5min":0.20000000000000001,"15min":0.20000000000000001},"max":{"1min":5.0700000000000003,"5min":22.577999999999999,"15min":22.577999999999999},"last":5.2549999999999999},{"interface":"front","average":{"1min":1.5600000000000001,"5min":1.345,"15min":2.1389999999999998},"min":{"1min":0.26100000000000001,"5min":0.216,"15min":0.216},"max":{"1min":5.7329999999999997,"5min":21.617999999999999,"15min":21.617999999999999},"last":2.1539999999999999}]},{"osd":2,"last update":"Sat Sep 23 21:16:00 2023","interfaces":[{"interface":"back","average":{"1min":1.4159999999999999,"5min":2.0699999999999998,"15min":2.1509999999999998},"min":{"1min":0.32400000000000001,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":4.1980000000000004,"5min":56.317999999999998,"15min":56.317999999999998},"last":0.45500000000000002},{"interface":"front","average":{"1min":1.3480000000000001,"5min":2.077,"15min":2.339},"min":{"1min":0.318,"5min":0.216,"15min":0.216},"max":{"1min":4.1630000000000003,"5min":55.988,"15min":55.988},"last":17.423999999999999}]},{"osd":4,"last update":"Sat Sep 23 21:16:00 2023","interfaces":[{"interface":"back","average":{"1min":1.2090000000000001,"5min":1.6160000000000001,"15min":1.694},"min":{"1min":0.29399999999999998,"5min":0.29399999999999998,"15min":0.28399999999999997},"max":{"1min":4.0609999999999999,"5min":21.478999999999999,"15min":21.478999999999999},"last":10.475},{"interface":"front","average":{"1min":0.96299999999999997,"5min":1.5229999999999999,"15min":1.8540000000000001},"min":{"1min":0.32300000000000001,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":4.2690000000000001,"5min":22.402000000000001,"15min":22.402000000000001},"last":5.3090000000000002}]},{"osd":5,"last update":"Sat Sep 23 21:16:00 2023","interfaces":[{"interface":"back","average":{"1min":1.3640000000000001,"5min":1.571,"15min":1.821},"min":{"1min":0.434,"5min":0.311,"15min":0.311},"max":{"1min":4.1449999999999996,"5min":21.562000000000001,"15min":21.562000000000001},"last":10.516},{"interface":"front","average":{"1min":1.089,"5min":1.3420000000000001,"15min":2.0070000000000001},"min":{"1min":0.47399999999999998,"5min":0.31,"15min":0.31},"max":{"1min":3.5790000000000002,"5min":8.8450000000000006,"15min":14.029},"last":0.5}]},{"osd":6,"last update":"Sat Sep 23 21:15:17 2023","interfaces":[{"interface":"back","average":{"1min":1.194,"5min":1.3919999999999999,"15min":1.9930000000000001},"min":{"1min":0.35899999999999999,"5min":0.34999999999999998,"15min":0.33900000000000002},"max":{"1min":3.536,"5min":22.446999999999999,"15min":22.446999999999999},"last":5.2889999999999997},{"interface":"front","average":{"1min":1.518,"5min":1.7969999999999999,"15min":2.1230000000000002},"min":{"1min":0.378,"5min":0.378,"15min":0.373},"max":{"1min":4.3760000000000003,"5min":12.375999999999999,"15min":13.576000000000001},"last":0.60199999999999998}]},{"osd":7,"last update":"Sat Sep 23 21:16:00 2023","interfaces":[{"interface":"back","average":{"1min":1.3120000000000001,"5min":1.9450000000000001,"15min":1.968},"min":{"1min":0.46700000000000003,"5min":0.38800000000000001,"15min":0.33900000000000002},"max":{"1min":4.1070000000000002,"5min":21.518999999999998,"15min":21.518999999999998},"last":10.547000000000001},{"interface":"front","average":{"1min":1.2829999999999999,"5min":1.974,"15min":2.2160000000000002},"min":{"1min":0.49299999999999999,"5min":0.42499999999999999,"15min":0.40300000000000002},"max":{"1min":3.8290000000000002,"5min":9.9879999999999995,"15min":13.603999999999999},"last":0.57399999999999995}]}]},{"osd":4,"up_from":31,"seq":133143986616,"num_pgs":66,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1837216,"kb_used_data":1318232,"kb_used_omap":0,"kb_used_meta":518976,"kb_avail":91907936,"statfs":{"total":95995035648,"available":94113726464,"internally_reserved":0,"allocated":1349869568,"data_stored":1348285449,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":531431424},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:15:14 2023","interfaces":[{"interface":"back","average":{"1min":1.944,"5min":1.1080000000000001,"15min":0.96899999999999997},"min":{"1min":0.23799999999999999,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":8.0079999999999991,"5min":8.0079999999999991,"15min":8.0079999999999991},"last":0.23799999999999999},{"interface":"front","average":{"1min":0.876,"5min":1.0820000000000001,"15min":1.1160000000000001},"min":{"1min":0.23799999999999999,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":2.4649999999999999,"5min":10.945,"15min":10.945},"last":0.44}]},{"osd":1,"last update":"Sat Sep 23 21:15:14 2023","interfaces":[{"interface":"back","average":{"1min":1.345,"5min":1.0429999999999999,"15min":2.5840000000000001},"min":{"1min":0.32700000000000001,"5min":0.221,"15min":0.221},"max":{"1min":4.0410000000000004,"5min":7.8609999999999998,"15min":37.960000000000001},"last":0.40100000000000002},{"interface":"front","average":{"1min":2.246,"5min":1.319,"15min":2.516},"min":{"1min":0.30299999999999999,"5min":0.28499999999999998,"15min":0.27000000000000002},"max":{"1min":13.983000000000001,"5min":13.983000000000001,"15min":37.933},"last":0.36299999999999999}]},{"osd":2,"last update":"Sat Sep 23 21:15:14 2023","interfaces":[{"interface":"back","average":{"1min":2.3559999999999999,"5min":1.3660000000000001,"15min":1.734},"min":{"1min":0.26400000000000001,"5min":0.247,"15min":0.247},"max":{"1min":14.026,"5min":14.026,"15min":16.928000000000001},"last":0.94099999999999995},{"interface":"front","average":{"1min":3.0419999999999998,"5min":1.6220000000000001,"15min":1.7949999999999999},"min":{"1min":0.33000000000000002,"5min":0.28599999999999998,"15min":0.27800000000000002},"max":{"1min":14.339,"5min":14.339,"15min":14.417999999999999},"last":1.631}]},{"osd":3,"last update":"Sat Sep 23 21:15:52 2023","interfaces":[{"interface":"back","average":{"1min":1.456,"5min":1.282,"15min":1.9159999999999999},"min":{"1min":0.44800000000000001,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":4.0030000000000001,"5min":9.3279999999999994,"15min":15.01},"last":0.95099999999999996},{"interface":"front","average":{"1min":1.454,"5min":1.4810000000000001,"15min":1.863},"min":{"1min":0.46800000000000003,"5min":0.32200000000000001,"15min":0.30199999999999999},"max":{"1min":3.7240000000000002,"5min":14.305,"15min":15.004},"last":0.96699999999999997}]},{"osd":5,"last update":"Sat Sep 23 21:16:06 2023","interfaces":[{"interface":"back","average":{"1min":0.68000000000000005,"5min":1.3720000000000001,"15min":1.107},"min":{"1min":0.307,"5min":0.222,"15min":0.222},"max":{"1min":3.3210000000000002,"5min":14.398999999999999,"15min":14.398999999999999},"last":0.315},{"interface":"front","average":{"1min":0.63100000000000001,"5min":1.2170000000000001,"15min":1.1259999999999999},"min":{"1min":0.27800000000000002,"5min":0.27200000000000002,"15min":0.255},"max":{"1min":1.9279999999999999,"5min":8.016,"15min":9.8979999999999997},"last":0.28999999999999998}]},{"osd":6,"last update":"Sat Sep 23 21:15:14 2023","interfaces":[{"interface":"back","average":{"1min":1.403,"5min":1.248,"15min":1.2749999999999999},"min":{"1min":0.34300000000000003,"5min":0.28599999999999998,"15min":0.26800000000000002},"max":{"1min":8.7420000000000009,"5min":24.452999999999999,"15min":24.452999999999999},"last":0.34300000000000003},{"interface":"front","average":{"1min":2.472,"5min":1.425,"15min":1.0680000000000001},"min":{"1min":0.40300000000000002,"5min":0.33500000000000002,"15min":0.32500000000000001},"max":{"1min":14.374000000000001,"5min":14.374000000000001,"15min":14.387},"last":0.41799999999999998}]},{"osd":7,"last update":"Sat Sep 23 21:15:52 2023","interfaces":[{"interface":"back","average":{"1min":0.83899999999999997,"5min":1.385,"15min":1.151},"min":{"1min":0.32000000000000001,"5min":0.30099999999999999,"15min":0.29299999999999998},"max":{"1min":2.7730000000000001,"5min":14.422000000000001,"15min":16.890999999999998},"last":0.39000000000000001},{"interface":"front","average":{"1min":0.95499999999999996,"5min":1.329,"15min":1.2350000000000001},"min":{"1min":0.38300000000000001,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":5.1150000000000002,"5min":7.7229999999999999,"15min":11.170999999999999},"last":0.376}]}]},{"osd":5,"up_from":39,"seq":167503724982,"num_pgs":71,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1884320,"kb_used_data":1343000,"kb_used_omap":0,"kb_used_meta":541312,"kb_avail":91860832,"statfs":{"total":95995035648,"available":94065491968,"internally_reserved":0,"allocated":1375232000,"data_stored":1373652425,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":554303488},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:15:14 2023","interfaces":[{"interface":"back","average":{"1min":1.3720000000000001,"5min":0.84599999999999997,"15min":0.75800000000000001},"min":{"1min":0.23100000000000001,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":4.3680000000000003,"5min":4.3680000000000003,"15min":4.3680000000000003},"last":0.29699999999999999},{"interface":"front","average":{"1min":1.6839999999999999,"5min":0.872,"15min":0.73599999999999999},"min":{"1min":0.23899999999999999,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":6.577,"5min":6.577,"15min":6.577},"last":0.39200000000000002}]},{"osd":1,"last update":"Sat Sep 23 21:15:11 2023","interfaces":[{"interface":"back","average":{"1min":1.1140000000000001,"5min":0.94099999999999995,"15min":1.6830000000000001},"min":{"1min":0.29399999999999998,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":7.5839999999999996,"5min":7.5839999999999996,"15min":25.239999999999998},"last":0.63},{"interface":"front","average":{"1min":1.123,"5min":1.472,"15min":1.7549999999999999},"min":{"1min":0.28299999999999997,"5min":0.28299999999999997,"15min":0.252},"max":{"1min":4.2199999999999998,"5min":11.116,"15min":25.253},"last":0.57999999999999996}]},{"osd":2,"last update":"Sat Sep 23 21:15:27 2023","interfaces":[{"interface":"back","average":{"1min":1.373,"5min":1.3320000000000001,"15min":1.6240000000000001},"min":{"1min":0.28399999999999997,"5min":0.27800000000000002,"15min":0.23499999999999999},"max":{"1min":3.8730000000000002,"5min":9.2889999999999997,"15min":12.634},"last":0.42399999999999999},{"interface":"front","average":{"1min":1.472,"5min":1.1559999999999999,"15min":1.6319999999999999},"min":{"1min":0.29799999999999999,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":7.2720000000000002,"5min":9.7170000000000005,"15min":9.7650000000000006},"last":1.2130000000000001}]},{"osd":3,"last update":"Sat Sep 23 21:15:49 2023","interfaces":[{"interface":"back","average":{"1min":1.7290000000000001,"5min":1.8160000000000001,"15min":2.27},"min":{"1min":0.38100000000000001,"5min":0.311,"15min":0.29299999999999998},"max":{"1min":6.5940000000000003,"5min":14.242000000000001,"15min":61.667999999999999},"last":0.437},{"interface":"front","average":{"1min":1.556,"5min":1.7130000000000001,"15min":2.4289999999999998},"min":{"1min":0.39900000000000002,"5min":0.34899999999999998,"15min":0.30599999999999999},"max":{"1min":6.6120000000000001,"5min":12.44,"15min":62.276000000000003},"last":1.0920000000000001}]},{"osd":4,"last update":"Sat Sep 23 21:16:04 2023","interfaces":[{"interface":"back","average":{"1min":0.46899999999999997,"5min":1.544,"15min":1.1419999999999999},"min":{"1min":0.28299999999999997,"5min":0.253,"15min":0.253},"max":{"1min":0.90900000000000003,"5min":12.776,"15min":12.776},"last":0.33200000000000002},{"interface":"front","average":{"1min":0.71699999999999997,"5min":1.3600000000000001,"15min":1.1479999999999999},"min":{"1min":0.311,"5min":0.28000000000000003,"15min":0.26900000000000002},"max":{"1min":1.681,"5min":9.7119999999999997,"15min":11.973000000000001},"last":0.35799999999999998}]},{"osd":6,"last update":"Sat Sep 23 21:15:11 2023","interfaces":[{"interface":"back","average":{"1min":1.0880000000000001,"5min":1.0449999999999999,"15min":1.071},"min":{"1min":0.36499999999999999,"5min":0.313,"15min":0.25900000000000001},"max":{"1min":5.2249999999999996,"5min":7.7030000000000003,"15min":11.128},"last":0.51100000000000001},{"interface":"front","average":{"1min":1.022,"5min":1.5489999999999999,"15min":1.2450000000000001},"min":{"1min":0.34300000000000003,"5min":0.34300000000000003,"15min":0.317},"max":{"1min":4.3550000000000004,"5min":10.999000000000001,"15min":11.112},"last":0.38}]},{"osd":7,"last update":"Sat Sep 23 21:15:49 2023","interfaces":[{"interface":"back","average":{"1min":0.97599999999999998,"5min":1.0740000000000001,"15min":1.8100000000000001},"min":{"1min":0.32400000000000001,"5min":0.31,"15min":0.309},"max":{"1min":5.1769999999999996,"5min":5.5659999999999998,"15min":189.405},"last":0.53500000000000003},{"interface":"front","average":{"1min":1.1259999999999999,"5min":1.597,"15min":1.9510000000000001},"min":{"1min":0.35199999999999998,"5min":0.35199999999999998,"15min":0.32500000000000001},"max":{"1min":4.5369999999999999,"5min":11.183999999999999,"15min":189.685},"last":0.40600000000000003}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":5434,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8488,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1259,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5322,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":646,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":4838,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":2390294528,"data_stored":2390294528,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":2016280576,"data_stored":2016280576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":2108686336,"data_stored":2108686336,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1351024640,"data_stored":1351024640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1376845824,"data_stored":1376845824,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1731788800,"data_stored":1731788800,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T21:16:13.577 INFO:tasks.cephadm.ceph_manager.ceph:increase pool size by 10 2023-09-23T21:16:13.578 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph osd pool set unique_pool_5 pg_num 26 2023-09-23T21:16:13.631 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:16:13.690 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:16:13 smithi026 ceph-mon[179688]: Health check update: Degraded data redundancy: 23080/172806 objects degraded (13.356%), 7 pgs degraded (PG_DEGRADED) 2023-09-23T21:16:13.690 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:16:13 smithi026 ceph-mon[175151]: Health check update: Degraded data redundancy: 23080/172806 objects degraded (13.356%), 7 pgs degraded (PG_DEGRADED) 2023-09-23T21:16:13.764 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:16:13.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:16:13 smithi070 ceph-mon[168282]: Health check update: Degraded data redundancy: 23080/172806 objects degraded (13.356%), 7 pgs degraded (PG_DEGRADED) 2023-09-23T21:16:13.969 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:16:13.982 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:16:13.982 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:16:14.083 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:16:14.206 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:16:14.267 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:16:14.283 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:16:14.299 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:16:14.300 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:16:14.328 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:16:14.401 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:16:14.438 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:16:14.472 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 42 16 59831 59815 88.9453 36.9375 0.0250081 0.0112361 2023-09-23T21:16:14.489 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:16:14.491 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:16:14.492 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:16:14.563 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:16:14 smithi026 ceph-mon[175151]: from='client.46868 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T21:16:14.564 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:16:14 smithi026 ceph-mon[175151]: pgmap v1408: 119 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 111 active+clean; 3.5 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 43 MiB/s wr, 681 op/s; 23080/172806 objects degraded (13.356%); 2901/172806 objects misplaced (1.679%); 3.6 MiB/s, 57 objects/s recovering 2023-09-23T21:16:14.564 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:16:14 smithi026 ceph-mon[179688]: from='client.46868 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T21:16:14.564 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:16:14 smithi026 ceph-mon[179688]: pgmap v1408: 119 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 111 active+clean; 3.5 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 43 MiB/s wr, 681 op/s; 23080/172806 objects degraded (13.356%); 2901/172806 objects misplaced (1.679%); 3.6 MiB/s, 57 objects/s recovering 2023-09-23T21:16:14.592 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:16:14.710 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:16:14.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:16:14 smithi070 ceph-mon[168282]: from='client.46868 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T21:16:14.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:16:14 smithi070 ceph-mon[168282]: pgmap v1408: 119 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 111 active+clean; 3.5 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 43 MiB/s wr, 681 op/s; 23080/172806 objects degraded (13.356%); 2901/172806 objects misplaced (1.679%); 3.6 MiB/s, 57 objects/s recovering 2023-09-23T21:16:14.893 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:16:14.955 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:16:14.969 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:16:14.970 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:16:15.018 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0: {} 2023-09-23T21:16:15.071 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:16:15.079 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1: {} 2023-09-23T21:16:15.108 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:16:15.126 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2: {} 2023-09-23T21:16:15.166 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3: {} 2023-09-23T21:16:15.207 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4: {} 2023-09-23T21:16:15.241 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T21:16:15.278 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6: {} 2023-09-23T21:16:15.319 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T21:16:15.338 INFO:teuthology.orchestra.run.smithi026.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T21:16:15.338 INFO:teuthology.orchestra.run.smithi026.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T21:16:15.339 INFO:teuthology.orchestra.run.smithi026.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T21:16:15.339 INFO:teuthology.orchestra.run.smithi026.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T21:16:15.339 INFO:teuthology.orchestra.run.smithi026.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T21:16:15.339 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T21:16:15.339 INFO:teuthology.orchestra.run.smithi026.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T21:16:15.339 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T21:16:15.402 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:16:15.406 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:16:15.410 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:16:15.472 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 43 16 60477 60461 87.8164 40.375 0.0219126 0.0113802 2023-09-23T21:16:15.510 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:16:15.542 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:16:15.584 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:16:15.595 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:16:15.595 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:16:15.696 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:16:15.726 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:16:15 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/3525774156' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:16:15.729 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:16:15.777 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:16:15.782 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:16:15.783 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:16:15.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:16:15 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/3525774156' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:16:15.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:16:15 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/3525774156' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:16:15.884 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:16:15.914 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:16:15.953 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:16:15.955 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:16:15.956 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:16:16.056 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:16:16.139 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:16:16.474 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 44 16 61226 61210 86.8826 46.8125 0.0117063 0.0115049 2023-09-23T21:16:16.526 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:16:16 smithi026 ceph-mon[175151]: pgmap v1409: 119 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 111 active+clean; 3.6 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 44 MiB/s wr, 698 op/s; 23080/175248 objects degraded (13.170%); 2901/175248 objects misplaced (1.655%); 3.2 MiB/s, 50 objects/s recovering 2023-09-23T21:16:16.526 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:16:16 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:16:16.526 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:16:16 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:16:16.526 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:16:16 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:16:16.527 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:16:16 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:16:16.803 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:16:16 smithi026 ceph-mon[179688]: pgmap v1409: 119 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 111 active+clean; 3.6 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 44 MiB/s wr, 698 op/s; 23080/175248 objects degraded (13.170%); 2901/175248 objects misplaced (1.655%); 3.2 MiB/s, 50 objects/s recovering 2023-09-23T21:16:16.803 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:16:16 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:16:16.803 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:16:16 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:16:16.803 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:16:16 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:16:16.803 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:16:16 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:16:16.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:16:16 smithi070 ceph-mon[168282]: pgmap v1409: 119 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 111 active+clean; 3.6 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 44 MiB/s wr, 698 op/s; 23080/175248 objects degraded (13.170%); 2901/175248 objects misplaced (1.655%); 3.2 MiB/s, 50 objects/s recovering 2023-09-23T21:16:16.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:16:16 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:16:16.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:16:16 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:16:16.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:16:16 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:16:16.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:16:16 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:16:17.426 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:16:17.434 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:16:17.435 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:16:17.473 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 45 16 62042 62026 86.0856 51 0.0115579 0.0116104 2023-09-23T21:16:17.535 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:16:17.566 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:16:17.607 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:16:17.613 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:16:17.613 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:16:17.714 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:16:17.821 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:16:18.473 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 46 16 62876 62860 85.3477 52.125 0.0171338 0.0117125 2023-09-23T21:16:18.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:16:18 smithi070 ceph-mon[168282]: pgmap v1410: 119 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 111 active+clean; 3.7 GiB data, 13 GiB used, 434 GiB / 447 GiB avail; 41 MiB/s wr, 659 op/s; 23080/181266 objects degraded (12.733%); 2901/181266 objects misplaced (1.600%); 2.8 MiB/s, 44 objects/s recovering 2023-09-23T21:16:18.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:16:18 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:16:18.619 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:16:18 smithi026 ceph-mon[175151]: pgmap v1410: 119 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 111 active+clean; 3.7 GiB data, 13 GiB used, 434 GiB / 447 GiB avail; 41 MiB/s wr, 659 op/s; 23080/181266 objects degraded (12.733%); 2901/181266 objects misplaced (1.600%); 2.8 MiB/s, 44 objects/s recovering 2023-09-23T21:16:18.619 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:16:18 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:16:18.620 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:16:18 smithi026 ceph-mon[179688]: pgmap v1410: 119 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 111 active+clean; 3.7 GiB data, 13 GiB used, 434 GiB / 447 GiB avail; 41 MiB/s wr, 659 op/s; 23080/181266 objects degraded (12.733%); 2901/181266 objects misplaced (1.600%); 2.8 MiB/s, 44 objects/s recovering 2023-09-23T21:16:18.620 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:16:18 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:16:18.805 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T21:16:19.473 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 47 16 63665 63649 84.5813 49.3125 0.0166824 0.0118165 2023-09-23T21:16:19.702 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:16:19 smithi026 ceph-mon[175151]: Health check update: Degraded data redundancy: 23080/182577 objects degraded (12.641%), 7 pgs degraded (PG_DEGRADED) 2023-09-23T21:16:19.703 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 21:16:19 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[164029]: ::ffff:172.21.15.70 - - [23/Sep/2023:21:16:19] "GET /metrics HTTP/1.1" 200 36395 "" "Prometheus/2.43.0" 2023-09-23T21:16:19.703 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:16:19 smithi026 ceph-mon[179688]: Health check update: Degraded data redundancy: 23080/182577 objects degraded (12.641%), 7 pgs degraded (PG_DEGRADED) 2023-09-23T21:16:19.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:16:19 smithi070 ceph-mon[168282]: Health check update: Degraded data redundancy: 23080/182577 objects degraded (12.641%), 7 pgs degraded (PG_DEGRADED) 2023-09-23T21:16:20.149 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:16:20.159 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:16:20.160 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:16:20.261 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:16:20.347 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:16:20.474 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 48 16 64444 64428 83.8339 48.6875 0.012455 0.0119231 2023-09-23T21:16:20.641 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:16:20 smithi026 ceph-mon[175151]: pgmap v1411: 119 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 111 active+clean; 3.7 GiB data, 13 GiB used, 434 GiB / 447 GiB avail; 42 MiB/s wr, 678 op/s; 23080/182577 objects degraded (12.641%); 2594/182577 objects misplaced (1.421%); 3.6 MiB/s, 58 objects/s recovering 2023-09-23T21:16:20.641 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:16:20 smithi026 ceph-mon[179688]: pgmap v1411: 119 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 111 active+clean; 3.7 GiB data, 13 GiB used, 434 GiB / 447 GiB avail; 42 MiB/s wr, 678 op/s; 23080/182577 objects degraded (12.641%); 2594/182577 objects misplaced (1.421%); 3.6 MiB/s, 58 objects/s recovering 2023-09-23T21:16:20.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:16:20 smithi070 ceph-mon[168282]: pgmap v1411: 119 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 111 active+clean; 3.7 GiB data, 13 GiB used, 434 GiB / 447 GiB avail; 42 MiB/s wr, 678 op/s; 23080/182577 objects degraded (12.641%); 2594/182577 objects misplaced (1.421%); 3.6 MiB/s, 58 objects/s recovering 2023-09-23T21:16:21.477 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 49 16 65260 65244 83.1641 51 0.020179 0.0120191 2023-09-23T21:16:22.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:16:21 smithi070 ceph-mon[168282]: pgmap v1412: 119 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 111 active+clean; 3.8 GiB data, 13 GiB used, 434 GiB / 447 GiB avail; 41 MiB/s wr, 648 op/s; 23080/187017 objects degraded (12.341%); 2594/187017 objects misplaced (1.387%); 3.4 MiB/s, 54 objects/s recovering 2023-09-23T21:16:22.131 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:16:21 smithi026 ceph-mon[175151]: pgmap v1412: 119 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 111 active+clean; 3.8 GiB data, 13 GiB used, 434 GiB / 447 GiB avail; 41 MiB/s wr, 648 op/s; 23080/187017 objects degraded (12.341%); 2594/187017 objects misplaced (1.387%); 3.4 MiB/s, 54 objects/s recovering 2023-09-23T21:16:22.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:16:21 smithi026 ceph-mon[179688]: pgmap v1412: 119 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 111 active+clean; 3.8 GiB data, 13 GiB used, 434 GiB / 447 GiB avail; 41 MiB/s wr, 648 op/s; 23080/187017 objects degraded (12.341%); 2594/187017 objects misplaced (1.387%); 3.4 MiB/s, 54 objects/s recovering 2023-09-23T21:16:22.473 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 50 16 66042 66026 82.4785 48.875 0.01439 0.0121173 2023-09-23T21:16:22.596 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:16:22.614 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:16:22.615 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:16:22.715 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:16:22.748 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:16:22.787 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:16:22.790 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:16:22.791 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:16:22.892 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:16:22.973 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:16:23.475 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 51 16 66856 66840 81.8576 50.875 0.00889713 0.0122117 2023-09-23T21:16:24.480 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 52 16 67601 67585 81.1748 46.5625 0.0188107 0.0123129 2023-09-23T21:16:24.514 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:16:24 smithi026 ceph-mon[179688]: pgmap v1413: 119 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 111 active+clean; 4.0 GiB data, 13 GiB used, 434 GiB / 447 GiB avail; 49 MiB/s wr, 786 op/s; 23080/194502 objects degraded (11.866%); 2254/194502 objects misplaced (1.159%); 4.8 MiB/s, 76 objects/s recovering 2023-09-23T21:16:24.516 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T21:16:24.520 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:16:24 smithi026 ceph-mon[175151]: pgmap v1413: 119 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 111 active+clean; 4.0 GiB data, 13 GiB used, 434 GiB / 447 GiB avail; 49 MiB/s wr, 786 op/s; 23080/194502 objects degraded (11.866%); 2254/194502 objects misplaced (1.159%); 4.8 MiB/s, 76 objects/s recovering 2023-09-23T21:16:24.530 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:16:24.530 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:16:24.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:16:24 smithi070 ceph-mon[168282]: pgmap v1413: 119 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 111 active+clean; 4.0 GiB data, 13 GiB used, 434 GiB / 447 GiB avail; 49 MiB/s wr, 786 op/s; 23080/194502 objects degraded (11.866%); 2254/194502 objects misplaced (1.159%); 4.8 MiB/s, 76 objects/s recovering 2023-09-23T21:16:24.631 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:16:24.660 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:16:24.698 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:16:24.700 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:16:24.700 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:16:24.801 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:16:24.888 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:16:25.478 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 53 16 68352 68336 80.5289 46.9375 0.0256686 0.0124117 2023-09-23T21:16:25.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:16:25 smithi070 ceph-mon[168282]: Health check update: Degraded data redundancy: 23080/197478 objects degraded (11.687%), 7 pgs degraded (PG_DEGRADED) 2023-09-23T21:16:25.634 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:16:25 smithi026 ceph-mon[175151]: Health check update: Degraded data redundancy: 23080/197478 objects degraded (11.687%), 7 pgs degraded (PG_DEGRADED) 2023-09-23T21:16:25.635 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:16:25 smithi026 ceph-mon[179688]: Health check update: Degraded data redundancy: 23080/197478 objects degraded (11.687%), 7 pgs degraded (PG_DEGRADED) 2023-09-23T21:16:25.668 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T21:16:25.675 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:16:25.676 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:16:25.777 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:16:25.805 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:16:26.478 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 54 16 69086 69070 79.8875 45.875 0.0101811 0.0125129 2023-09-23T21:16:26.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:16:26 smithi070 ceph-mon[168282]: pgmap v1414: 119 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 111 active+clean; 4.0 GiB data, 14 GiB used, 433 GiB / 447 GiB avail; 43 MiB/s wr, 685 op/s; 23080/197478 objects degraded (11.687%); 2254/197478 objects misplaced (1.141%); 3.4 MiB/s, 53 objects/s recovering 2023-09-23T21:16:26.631 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:16:26 smithi026 ceph-mon[175151]: pgmap v1414: 119 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 111 active+clean; 4.0 GiB data, 14 GiB used, 433 GiB / 447 GiB avail; 43 MiB/s wr, 685 op/s; 23080/197478 objects degraded (11.687%); 2254/197478 objects misplaced (1.141%); 3.4 MiB/s, 53 objects/s recovering 2023-09-23T21:16:26.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:16:26 smithi026 ceph-mon[179688]: pgmap v1414: 119 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 111 active+clean; 4.0 GiB data, 14 GiB used, 433 GiB / 447 GiB avail; 43 MiB/s wr, 685 op/s; 23080/197478 objects degraded (11.687%); 2254/197478 objects misplaced (1.141%); 3.4 MiB/s, 53 objects/s recovering 2023-09-23T21:16:27.483 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 55 16 69834 69818 79.2784 46.75 0.0125231 0.0126079 2023-09-23T21:16:28.310 INFO:teuthology.orchestra.run.smithi026.stderr:nodeep-scrub is unset 2023-09-23T21:16:28.441 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:16:28 smithi026 ceph-mon[175151]: pgmap v1415: 119 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 111 active+clean; 4.2 GiB data, 14 GiB used, 433 GiB / 447 GiB avail; 51 MiB/s wr, 818 op/s; 23080/204717 objects degraded (11.274%); 2254/204717 objects misplaced (1.101%); 3.4 MiB/s, 53 objects/s recovering 2023-09-23T21:16:28.441 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:16:28 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/2958998092' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:16:28.441 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:16:28 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:16:28.441 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:16:28 smithi026 ceph-mon[179688]: pgmap v1415: 119 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 111 active+clean; 4.2 GiB data, 14 GiB used, 433 GiB / 447 GiB avail; 51 MiB/s wr, 818 op/s; 23080/204717 objects degraded (11.274%); 2254/204717 objects misplaced (1.101%); 3.4 MiB/s, 53 objects/s recovering 2023-09-23T21:16:28.441 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:16:28 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/2958998092' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:16:28.441 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:16:28 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:16:28.483 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 56 16 70473 70457 78.5763 39.9375 0.0108745 0.0127201 2023-09-23T21:16:28.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:16:28 smithi070 ceph-mon[168282]: pgmap v1415: 119 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 111 active+clean; 4.2 GiB data, 14 GiB used, 433 GiB / 447 GiB avail; 51 MiB/s wr, 818 op/s; 23080/204717 objects degraded (11.274%); 2254/204717 objects misplaced (1.101%); 3.4 MiB/s, 53 objects/s recovering 2023-09-23T21:16:28.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:16:28 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/2958998092' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:16:28.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:16:28 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:16:28.640 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:16:28.648 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:16:28.649 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:16:28.727 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:16:28.750 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:16:28.784 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:16:28.840 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:16:28.844 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:16:28.844 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:16:28.945 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:16:29.054 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:16:29.242 INFO:teuthology.orchestra.run.smithi026.stdout:ERROR: (22) Invalid argument 2023-09-23T21:16:29.242 INFO:teuthology.orchestra.run.smithi026.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T21:16:29.246 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:16:29.256 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:16:29.257 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:16:29.351 INFO:teuthology.orchestra.run.smithi026.stderr:set pool 11 pg_num to 26 2023-09-23T21:16:29.351 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:16:29 smithi026 ceph-mon[175151]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T21:16:29.351 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:16:29 smithi026 ceph-mon[175151]: failed to encode map e557 with expected crc 2023-09-23T21:16:29.351 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:16:29 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T21:16:29.352 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:16:29 smithi026 ceph-mon[175151]: osdmap e557: 8 total, 8 up, 5 in 2023-09-23T21:16:29.352 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:16:29 smithi026 ceph-mon[175151]: failed to encode map e557 with expected crc 2023-09-23T21:16:29.352 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:16:29 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/1403395448' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pg_num", "val": "26"}]: dispatch 2023-09-23T21:16:29.352 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:16:29 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pg_num", "val": "26"}]: dispatch 2023-09-23T21:16:29.352 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:16:29 smithi026 ceph-mon[179688]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T21:16:29.352 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:16:29 smithi026 ceph-mon[179688]: failed to encode map e557 with expected crc 2023-09-23T21:16:29.352 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:16:29 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T21:16:29.352 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:16:29 smithi026 ceph-mon[179688]: osdmap e557: 8 total, 8 up, 5 in 2023-09-23T21:16:29.353 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:16:29 smithi026 ceph-mon[179688]: failed to encode map e557 with expected crc 2023-09-23T21:16:29.353 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:16:29 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/1403395448' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pg_num", "val": "26"}]: dispatch 2023-09-23T21:16:29.353 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:16:29 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pg_num", "val": "26"}]: dispatch 2023-09-23T21:16:29.358 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:16:29.388 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:16:29.425 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:16:29.428 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:16:29.428 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:16:29.483 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 57 16 71078 71062 77.8615 37.8125 0.0126152 0.0128378 2023-09-23T21:16:29.530 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:16:29.617 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 21:16:29 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[164029]: ::ffff:172.21.15.70 - - [23/Sep/2023:21:16:29] "GET /metrics HTTP/1.1" 200 36395 "" "Prometheus/2.43.0" 2023-09-23T21:16:29.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:16:29 smithi070 ceph-mon[168282]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T21:16:29.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:16:29 smithi070 ceph-mon[168282]: failed to encode map e557 with expected crc 2023-09-23T21:16:29.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:16:29 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T21:16:29.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:16:29 smithi070 ceph-mon[168282]: osdmap e557: 8 total, 8 up, 5 in 2023-09-23T21:16:29.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:16:29 smithi070 ceph-mon[168282]: failed to encode map e557 with expected crc 2023-09-23T21:16:29.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:16:29 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/1403395448' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pg_num", "val": "26"}]: dispatch 2023-09-23T21:16:29.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:16:29 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pg_num", "val": "26"}]: dispatch 2023-09-23T21:16:29.627 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:16:30.204 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_blocked_ops 2023-09-23T21:16:30.211 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:16:30.221 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:16:30.222 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:16:30.322 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:16:30.351 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:16:30.388 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:16:30 smithi026 ceph-mon[175151]: pgmap v1417: 119 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 111 active+clean; 4.2 GiB data, 14 GiB used, 433 GiB / 447 GiB avail; 49 MiB/s wr, 785 op/s; 23080/206163 objects degraded (11.195%); 1923/206163 objects misplaced (0.933%); 4.2 MiB/s, 67 objects/s recovering 2023-09-23T21:16:30.389 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:16:30 smithi026 ceph-mon[175151]: failed to encode map e558 with expected crc 2023-09-23T21:16:30.389 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:16:30 smithi026 ceph-mon[175151]: failed to encode map e558 with expected crc 2023-09-23T21:16:30.389 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:16:30 smithi026 ceph-mon[175151]: failed to encode map e558 with expected crc 2023-09-23T21:16:30.389 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:16:30 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pg_num", "val": "26"}]': finished 2023-09-23T21:16:30.389 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:16:30 smithi026 ceph-mon[175151]: osdmap e558: 8 total, 8 up, 5 in 2023-09-23T21:16:30.390 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:16:30 smithi026 ceph-mon[179688]: pgmap v1417: 119 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 111 active+clean; 4.2 GiB data, 14 GiB used, 433 GiB / 447 GiB avail; 49 MiB/s wr, 785 op/s; 23080/206163 objects degraded (11.195%); 1923/206163 objects misplaced (0.933%); 4.2 MiB/s, 67 objects/s recovering 2023-09-23T21:16:30.390 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:16:30 smithi026 ceph-mon[179688]: failed to encode map e558 with expected crc 2023-09-23T21:16:30.390 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:16:30 smithi026 ceph-mon[179688]: failed to encode map e558 with expected crc 2023-09-23T21:16:30.390 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:16:30 smithi026 ceph-mon[179688]: failed to encode map e558 with expected crc 2023-09-23T21:16:30.390 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:16:30 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pg_num", "val": "26"}]': finished 2023-09-23T21:16:30.390 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:16:30 smithi026 ceph-mon[179688]: osdmap e558: 8 total, 8 up, 5 in 2023-09-23T21:16:30.392 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:16:30.394 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:16:30.395 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:16:30.484 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 58 16 71746 71730 77.2372 41.75 0.022038 0.0129419 2023-09-23T21:16:30.496 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:16:30.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:16:30 smithi070 ceph-mon[168282]: pgmap v1417: 119 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 111 active+clean; 4.2 GiB data, 14 GiB used, 433 GiB / 447 GiB avail; 49 MiB/s wr, 785 op/s; 23080/206163 objects degraded (11.195%); 1923/206163 objects misplaced (0.933%); 4.2 MiB/s, 67 objects/s recovering 2023-09-23T21:16:30.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:16:30 smithi070 ceph-mon[168282]: failed to encode map e558 with expected crc 2023-09-23T21:16:30.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:16:30 smithi070 ceph-mon[168282]: failed to encode map e558 with expected crc 2023-09-23T21:16:30.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:16:30 smithi070 ceph-mon[168282]: failed to encode map e558 with expected crc 2023-09-23T21:16:30.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:16:30 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pg_num", "val": "26"}]': finished 2023-09-23T21:16:30.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:16:30 smithi070 ceph-mon[168282]: osdmap e558: 8 total, 8 up, 5 in 2023-09-23T21:16:30.623 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:16:30.813 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:16:30.819 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:16:30.819 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:16:30.920 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:16:30.953 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:16:30.993 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:16:30.996 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:16:30.997 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:16:31.097 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:16:31.197 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:16:31.261 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:16:31.355 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T21:16:31.424 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:16:31.437 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:16:31.437 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:16:31.485 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 59 16 72347 72331 76.5651 37.5625 0.029003 0.0130533 2023-09-23T21:16:31.538 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:16:31.565 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:16:31 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:16:31.565 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:16:31 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:16:31.565 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:16:31 smithi070 ceph-mon[168282]: Health check update: Degraded data redundancy: 23080/206163 objects degraded (11.195%), 7 pgs degraded (PG_DEGRADED) 2023-09-23T21:16:31.566 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:16:31 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pg_num_actual", "val": "26"}]: dispatch 2023-09-23T21:16:31.566 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:16:31 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:16:31.566 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:16:31 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:16:31.568 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:16:31.607 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:16:31.609 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:16:31.609 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:16:31.626 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:16:31 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:16:31.626 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:16:31 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:16:31.627 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:16:31 smithi026 ceph-mon[175151]: Health check update: Degraded data redundancy: 23080/206163 objects degraded (11.195%), 7 pgs degraded (PG_DEGRADED) 2023-09-23T21:16:31.627 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:16:31 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pg_num_actual", "val": "26"}]: dispatch 2023-09-23T21:16:31.627 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:16:31 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:16:31.627 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:16:31 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:16:31.629 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:16:31 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:16:31.630 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:16:31 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:16:31.630 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:16:31 smithi026 ceph-mon[179688]: Health check update: Degraded data redundancy: 23080/206163 objects degraded (11.195%), 7 pgs degraded (PG_DEGRADED) 2023-09-23T21:16:31.630 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:16:31 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pg_num_actual", "val": "26"}]: dispatch 2023-09-23T21:16:31.630 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:16:31 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:16:31.630 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:16:31 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:16:31.686 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0: {} 2023-09-23T21:16:31.710 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:16:31.734 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1: {} 2023-09-23T21:16:31.787 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2: {} 2023-09-23T21:16:31.838 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:16:31.845 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3: {} 2023-09-23T21:16:31.902 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4: {} 2023-09-23T21:16:32.017 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T21:16:32.078 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6: {} 2023-09-23T21:16:32.144 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T21:16:32.168 INFO:teuthology.orchestra.run.smithi026.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T21:16:32.169 INFO:teuthology.orchestra.run.smithi026.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T21:16:32.169 INFO:teuthology.orchestra.run.smithi026.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T21:16:32.169 INFO:teuthology.orchestra.run.smithi026.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T21:16:32.169 INFO:teuthology.orchestra.run.smithi026.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T21:16:32.169 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T21:16:32.169 INFO:teuthology.orchestra.run.smithi026.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T21:16:32.169 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T21:16:32.485 INFO:tasks.radosbench.radosbench.0.smithi026.stdout:2023-09-23T21:16:32.485684+0000 min lat: 0.00111295 max lat: 2.31793 avg lat: 0.0131158 2023-09-23T21:16:32.485 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T21:16:32.485 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 60 16 72575 72559 75.5272 14.25 0.318087 0.0131158 2023-09-23T21:16:32.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:16:32 smithi070 ceph-mon[168282]: pgmap v1419: 119 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 111 active+clean; 4.3 GiB data, 15 GiB used, 432 GiB / 447 GiB avail; 40 MiB/s wr, 647 op/s; 23080/210060 objects degraded (10.987%); 1923/210060 objects misplaced (0.915%); 2.6 MiB/s, 41 objects/s recovering 2023-09-23T21:16:32.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:16:32 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T21:16:32.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:16:32 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T21:16:32.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:16:32 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:16:32.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:16:32 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/3915966572' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:16:32.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:16:32 smithi070 ceph-mon[168282]: failed to encode map e559 with expected crc 2023-09-23T21:16:32.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:16:32 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pg_num_actual", "val": "26"}]': finished 2023-09-23T21:16:32.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:16:32 smithi070 ceph-mon[168282]: osdmap e559: 8 total, 8 up, 5 in 2023-09-23T21:16:32.631 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:16:32 smithi026 ceph-mon[175151]: pgmap v1419: 119 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 111 active+clean; 4.3 GiB data, 15 GiB used, 432 GiB / 447 GiB avail; 40 MiB/s wr, 647 op/s; 23080/210060 objects degraded (10.987%); 1923/210060 objects misplaced (0.915%); 2.6 MiB/s, 41 objects/s recovering 2023-09-23T21:16:32.631 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:16:32 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T21:16:32.631 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:16:32 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T21:16:32.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:16:32 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:16:32.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:16:32 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/3915966572' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:16:32.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:16:32 smithi026 ceph-mon[175151]: failed to encode map e559 with expected crc 2023-09-23T21:16:32.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:16:32 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pg_num_actual", "val": "26"}]': finished 2023-09-23T21:16:32.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:16:32 smithi026 ceph-mon[175151]: osdmap e559: 8 total, 8 up, 5 in 2023-09-23T21:16:32.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:16:32 smithi026 ceph-mon[179688]: pgmap v1419: 119 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 111 active+clean; 4.3 GiB data, 15 GiB used, 432 GiB / 447 GiB avail; 40 MiB/s wr, 647 op/s; 23080/210060 objects degraded (10.987%); 1923/210060 objects misplaced (0.915%); 2.6 MiB/s, 41 objects/s recovering 2023-09-23T21:16:32.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:16:32 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T21:16:32.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:16:32 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T21:16:32.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:16:32 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:16:32.633 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:16:32 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/3915966572' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:16:32.633 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:16:32 smithi026 ceph-mon[179688]: failed to encode map e559 with expected crc 2023-09-23T21:16:32.633 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:16:32 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pg_num_actual", "val": "26"}]': finished 2023-09-23T21:16:32.633 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:16:32 smithi026 ceph-mon[179688]: osdmap e559: 8 total, 8 up, 5 in 2023-09-23T21:16:33.485 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 61 16 72693 72677 74.4106 7.375 0.0040602 0.0132584 2023-09-23T21:16:33.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:16:33 smithi070 ceph-mon[168282]: failed to encode map e559 with expected crc 2023-09-23T21:16:33.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:16:33 smithi070 ceph-mon[168282]: failed to encode map e559 with expected crc 2023-09-23T21:16:33.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:16:33 smithi070 ceph-mon[168282]: failed to encode map e560 with expected crc 2023-09-23T21:16:33.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:16:33 smithi070 ceph-mon[168282]: osdmap e560: 8 total, 8 up, 5 in 2023-09-23T21:16:33.631 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:16:33 smithi026 ceph-mon[175151]: failed to encode map e559 with expected crc 2023-09-23T21:16:33.631 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:16:33 smithi026 ceph-mon[175151]: failed to encode map e559 with expected crc 2023-09-23T21:16:33.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:16:33 smithi026 ceph-mon[175151]: failed to encode map e560 with expected crc 2023-09-23T21:16:33.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:16:33 smithi026 ceph-mon[175151]: osdmap e560: 8 total, 8 up, 5 in 2023-09-23T21:16:33.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:16:33 smithi026 ceph-mon[179688]: failed to encode map e559 with expected crc 2023-09-23T21:16:33.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:16:33 smithi026 ceph-mon[179688]: failed to encode map e559 with expected crc 2023-09-23T21:16:33.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:16:33 smithi026 ceph-mon[179688]: failed to encode map e560 with expected crc 2023-09-23T21:16:33.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:16:33 smithi026 ceph-mon[179688]: osdmap e560: 8 total, 8 up, 5 in 2023-09-23T21:16:34.485 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 62 16 72956 72940 73.4761 16.4375 0.00240346 0.0134696 2023-09-23T21:16:34.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:16:34 smithi070 ceph-mon[168282]: failed to encode map e560 with expected crc 2023-09-23T21:16:34.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:16:34 smithi070 ceph-mon[168282]: failed to encode map e560 with expected crc 2023-09-23T21:16:34.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:16:34 smithi070 ceph-mon[168282]: failed to encode map e560 with expected crc 2023-09-23T21:16:34.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:16:34 smithi070 ceph-mon[168282]: failed to encode map e560 with expected crc 2023-09-23T21:16:34.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:16:34 smithi070 ceph-mon[168282]: pgmap v1422: 129 pgs: 1 peering, 2 remapped+peering, 10 unknown, 6 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 4.0 GiB data, 15 GiB used, 432 GiB / 447 GiB avail; 19833/195396 objects degraded (10.150%) 2023-09-23T21:16:34.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:16:34 smithi070 ceph-mon[168282]: failed to encode map e561 with expected crc 2023-09-23T21:16:34.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:16:34 smithi070 ceph-mon[168282]: osdmap e561: 8 total, 8 up, 5 in 2023-09-23T21:16:34.634 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:16:34 smithi026 ceph-mon[179688]: failed to encode map e560 with expected crc 2023-09-23T21:16:34.634 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:16:34 smithi026 ceph-mon[179688]: failed to encode map e560 with expected crc 2023-09-23T21:16:34.634 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:16:34 smithi026 ceph-mon[179688]: failed to encode map e560 with expected crc 2023-09-23T21:16:34.634 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:16:34 smithi026 ceph-mon[179688]: failed to encode map e560 with expected crc 2023-09-23T21:16:34.634 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:16:34 smithi026 ceph-mon[179688]: pgmap v1422: 129 pgs: 1 peering, 2 remapped+peering, 10 unknown, 6 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 4.0 GiB data, 15 GiB used, 432 GiB / 447 GiB avail; 19833/195396 objects degraded (10.150%) 2023-09-23T21:16:34.635 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:16:34 smithi026 ceph-mon[179688]: failed to encode map e561 with expected crc 2023-09-23T21:16:34.636 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:16:34 smithi026 ceph-mon[179688]: osdmap e561: 8 total, 8 up, 5 in 2023-09-23T21:16:34.636 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:16:34 smithi026 ceph-mon[175151]: failed to encode map e560 with expected crc 2023-09-23T21:16:34.636 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:16:34 smithi026 ceph-mon[175151]: failed to encode map e560 with expected crc 2023-09-23T21:16:34.636 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:16:34 smithi026 ceph-mon[175151]: failed to encode map e560 with expected crc 2023-09-23T21:16:34.637 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:16:34 smithi026 ceph-mon[175151]: failed to encode map e560 with expected crc 2023-09-23T21:16:34.637 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:16:34 smithi026 ceph-mon[175151]: pgmap v1422: 129 pgs: 1 peering, 2 remapped+peering, 10 unknown, 6 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 4.0 GiB data, 15 GiB used, 432 GiB / 447 GiB avail; 19833/195396 objects degraded (10.150%) 2023-09-23T21:16:34.637 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:16:34 smithi026 ceph-mon[175151]: failed to encode map e561 with expected crc 2023-09-23T21:16:34.637 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:16:34 smithi026 ceph-mon[175151]: osdmap e561: 8 total, 8 up, 5 in 2023-09-23T21:16:34.849 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:16:34.852 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:16:34.853 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:16:34.954 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:16:35.034 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:16:35.135 INFO:tasks.thrashosds.thrasher:in_osds: [0, 2, 3, 4, 5] out_osds: [6, 1, 7] dead_osds: [] live_osds: [2, 3, 4, 5, 7, 6, 1, 0] 2023-09-23T21:16:35.135 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T21:16:35.135 INFO:tasks.thrashosds.thrasher:Adding osd 7 2023-09-23T21:16:35.136 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph osd in 7 2023-09-23T21:16:35.485 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 63 16 73525 73509 72.8746 35.5625 0.0113407 0.013718 2023-09-23T21:16:35.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:16:35 smithi070 ceph-mon[168282]: failed to encode map e561 with expected crc 2023-09-23T21:16:35.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:16:35 smithi070 ceph-mon[168282]: failed to encode map e561 with expected crc 2023-09-23T21:16:35.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:16:35 smithi070 ceph-mon[168282]: failed to encode map e561 with expected crc 2023-09-23T21:16:35.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:16:35 smithi070 ceph-mon[168282]: osdmap e562: 8 total, 8 up, 5 in 2023-09-23T21:16:35.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:16:35 smithi026 ceph-mon[179688]: failed to encode map e561 with expected crc 2023-09-23T21:16:35.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:16:35 smithi026 ceph-mon[179688]: failed to encode map e561 with expected crc 2023-09-23T21:16:35.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:16:35 smithi026 ceph-mon[179688]: failed to encode map e561 with expected crc 2023-09-23T21:16:35.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:16:35 smithi026 ceph-mon[179688]: osdmap e562: 8 total, 8 up, 5 in 2023-09-23T21:16:35.636 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:16:35 smithi026 ceph-mon[175151]: failed to encode map e561 with expected crc 2023-09-23T21:16:35.636 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:16:35 smithi026 ceph-mon[175151]: failed to encode map e561 with expected crc 2023-09-23T21:16:35.636 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:16:35 smithi026 ceph-mon[175151]: failed to encode map e561 with expected crc 2023-09-23T21:16:35.637 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:16:35 smithi026 ceph-mon[175151]: osdmap e562: 8 total, 8 up, 5 in 2023-09-23T21:16:35.656 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:16:35.664 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:16:35.665 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:16:35.766 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:16:35.801 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:16:35.859 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:16:36.636 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:16:36.637 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 64 16 74523 74507 72.7106 62.375 0.00981065 0.0137477 2023-09-23T21:16:36.637 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:16:36.638 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:16:36 smithi070 ceph-mon[168282]: failed to encode map e562 with expected crc 2023-09-23T21:16:36.638 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:16:36 smithi070 ceph-mon[168282]: failed to encode map e562 with expected crc 2023-09-23T21:16:36.638 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:16:36 smithi070 ceph-mon[168282]: failed to encode map e562 with expected crc 2023-09-23T21:16:36.638 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:16:36 smithi070 ceph-mon[168282]: pgmap v1425: 129 pgs: 4 active+recovery_wait+degraded, 1 peering, 2 remapped+peering, 7 unknown, 4 active+recovery_wait+undersized+degraded+remapped, 111 active+clean; 4.0 GiB data, 15 GiB used, 432 GiB / 447 GiB avail; 19833/196989 objects degraded (10.068%) 2023-09-23T21:16:36.638 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:16:36 smithi070 ceph-mon[168282]: Health check update: Degraded data redundancy: 19833/195396 objects degraded (10.150%), 6 pgs degraded (PG_DEGRADED) 2023-09-23T21:16:36.638 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:16:36.639 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:16:36 smithi026 ceph-mon[175151]: failed to encode map e562 with expected crc 2023-09-23T21:16:36.639 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:16:36 smithi026 ceph-mon[175151]: failed to encode map e562 with expected crc 2023-09-23T21:16:36.639 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:16:36 smithi026 ceph-mon[175151]: failed to encode map e562 with expected crc 2023-09-23T21:16:36.639 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:16:36 smithi026 ceph-mon[175151]: pgmap v1425: 129 pgs: 4 active+recovery_wait+degraded, 1 peering, 2 remapped+peering, 7 unknown, 4 active+recovery_wait+undersized+degraded+remapped, 111 active+clean; 4.0 GiB data, 15 GiB used, 432 GiB / 447 GiB avail; 19833/196989 objects degraded (10.068%) 2023-09-23T21:16:36.639 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:16:36 smithi026 ceph-mon[175151]: Health check update: Degraded data redundancy: 19833/195396 objects degraded (10.150%), 6 pgs degraded (PG_DEGRADED) 2023-09-23T21:16:36.639 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:16:36 smithi026 ceph-mon[179688]: failed to encode map e562 with expected crc 2023-09-23T21:16:36.639 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:16:36 smithi026 ceph-mon[179688]: failed to encode map e562 with expected crc 2023-09-23T21:16:36.639 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:16:36 smithi026 ceph-mon[179688]: failed to encode map e562 with expected crc 2023-09-23T21:16:36.639 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:16:36 smithi026 ceph-mon[179688]: pgmap v1425: 129 pgs: 4 active+recovery_wait+degraded, 1 peering, 2 remapped+peering, 7 unknown, 4 active+recovery_wait+undersized+degraded+remapped, 111 active+clean; 4.0 GiB data, 15 GiB used, 432 GiB / 447 GiB avail; 19833/196989 objects degraded (10.068%) 2023-09-23T21:16:36.639 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:16:36 smithi026 ceph-mon[179688]: Health check update: Degraded data redundancy: 19833/195396 objects degraded (10.150%), 6 pgs degraded (PG_DEGRADED) 2023-09-23T21:16:36.738 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:16:36.831 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:16:36.921 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T21:16:36.928 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:16:36.929 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:16:37.029 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:16:37.081 INFO:teuthology.orchestra.run.smithi026.stdout:{ 2023-09-23T21:16:37.082 INFO:teuthology.orchestra.run.smithi026.stdout: "ops": [], 2023-09-23T21:16:37.082 INFO:teuthology.orchestra.run.smithi026.stdout: "complaint_time": 30, 2023-09-23T21:16:37.082 INFO:teuthology.orchestra.run.smithi026.stdout: "num_blocked_ops": 0 2023-09-23T21:16:37.082 INFO:teuthology.orchestra.run.smithi026.stdout:} 2023-09-23T21:16:37.103 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:16:37.485 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 65 16 75443 75427 72.4766 57.5 0.0161451 0.0137931 2023-09-23T21:16:37.748 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_historic_ops 2023-09-23T21:16:37.771 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:16:37.776 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:16:37.777 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:16:37.777 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:16:37.878 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:16:37.907 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:16:37.948 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:16:37.951 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:16:37.952 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:16:38.053 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:16:38.154 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:16:38.235 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:16:38.245 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:16:38.246 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:16:38.346 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:16:38.376 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:16:38.388 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:16:38 smithi026 ceph-mon[175151]: pgmap v1426: 129 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+degraded, 2 remapped+peering, 2 unknown, 2 active+recovery_wait+undersized+degraded+remapped, 116 active+clean; 4.2 GiB data, 15 GiB used, 432 GiB / 447 GiB avail; 16337/207918 objects degraded (7.857%); 3083/207918 objects misplaced (1.483%) 2023-09-23T21:16:38.389 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:16:38 smithi026 ceph-mon[179688]: pgmap v1426: 129 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+degraded, 2 remapped+peering, 2 unknown, 2 active+recovery_wait+undersized+degraded+remapped, 116 active+clean; 4.2 GiB data, 15 GiB used, 432 GiB / 447 GiB avail; 16337/207918 objects degraded (7.857%); 3083/207918 objects misplaced (1.483%) 2023-09-23T21:16:38.416 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:16:38.419 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:16:38.419 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:16:38.488 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 66 16 76330 76314 72.2149 55.4375 0.0111146 0.0138421 2023-09-23T21:16:38.520 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:16:38.606 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:16:38.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:16:38 smithi070 ceph-mon[168282]: pgmap v1426: 129 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+degraded, 2 remapped+peering, 2 unknown, 2 active+recovery_wait+undersized+degraded+remapped, 116 active+clean; 4.2 GiB data, 15 GiB used, 432 GiB / 447 GiB avail; 16337/207918 objects degraded (7.857%); 3083/207918 objects misplaced (1.483%) 2023-09-23T21:16:38.638 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T21:16:38.946 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:16:38.954 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:16:38.954 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:16:39.055 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:16:39.084 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:16:39.121 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:16:39.124 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:16:39.124 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:16:39.225 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:16:39.261 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:16:39.283 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:16:39.303 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:16:39.306 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:16:39.306 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:16:39.347 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:16:39 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/3285609271' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T21:16:39.381 INFO:teuthology.orchestra.run.smithi026.stderr:noscrub is set 2023-09-23T21:16:39.406 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:16:39.434 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:16:39.472 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:16:39.474 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:16:39.475 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:16:39.488 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 67 16 77139 77123 71.8919 50.5625 0.00927603 0.0139046 2023-09-23T21:16:39.576 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:16:39.608 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:16:39 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/3285609271' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T21:16:39.611 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:16:39.613 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:16:39 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/3285609271' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T21:16:39.654 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:16:39.659 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:16:39.660 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:16:39.761 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:16:39.792 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:16:39.832 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:16:39.838 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:16:39.839 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:16:39.881 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 21:16:39 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[164029]: ::ffff:172.21.15.70 - - [23/Sep/2023:21:16:39] "GET /metrics HTTP/1.1" 200 36376 "" "Prometheus/2.43.0" 2023-09-23T21:16:39.939 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:16:39.978 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:16:40.018 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:16:40.022 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:16:40.022 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:16:40.123 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:16:40.150 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:16:40.193 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:16:40.197 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:16:40.197 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:16:40.298 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:16:40.354 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:16:40 smithi026 ceph-mon[175151]: pgmap v1427: 129 pgs: 1 active+recovering+undersized+remapped, 8 active+recovery_wait+degraded, 4 active+recovery_wait+undersized+degraded+remapped, 116 active+clean; 4.5 GiB data, 15 GiB used, 432 GiB / 447 GiB avail; 90 MiB/s wr, 1.44k op/s; 21196/222507 objects degraded (9.526%); 3083/222507 objects misplaced (1.386%); 13 MiB/s, 204 objects/s recovering 2023-09-23T21:16:40.355 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:16:40 smithi026 ceph-mon[175151]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:16:40.355 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:16:40 smithi026 ceph-mon[175151]: failed to encode map e563 with expected crc 2023-09-23T21:16:40.355 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:16:40 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/3285609271' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T21:16:40.355 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:16:40 smithi026 ceph-mon[175151]: osdmap e563: 8 total, 8 up, 5 in 2023-09-23T21:16:40.426 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:16:40.493 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 68 16 78031 78015 71.6495 55.75 0.016842 0.0139503 2023-09-23T21:16:40.558 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:16:40.568 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:16:40.568 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:16:40.618 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:16:40 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mon-a[175127]: 2023-09-23T21:16:40.509+0000 7f1e2d487700 -1 mon.a@0(leader).osd e563 definitely_dead 0 2023-09-23T21:16:40.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:16:40 smithi070 ceph-mon[168282]: pgmap v1427: 129 pgs: 1 active+recovering+undersized+remapped, 8 active+recovery_wait+degraded, 4 active+recovery_wait+undersized+degraded+remapped, 116 active+clean; 4.5 GiB data, 15 GiB used, 432 GiB / 447 GiB avail; 90 MiB/s wr, 1.44k op/s; 21196/222507 objects degraded (9.526%); 3083/222507 objects misplaced (1.386%); 13 MiB/s, 204 objects/s recovering 2023-09-23T21:16:40.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:16:40 smithi070 ceph-mon[168282]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:16:40.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:16:40 smithi070 ceph-mon[168282]: failed to encode map e563 with expected crc 2023-09-23T21:16:40.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:16:40 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/3285609271' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T21:16:40.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:16:40 smithi070 ceph-mon[168282]: osdmap e563: 8 total, 8 up, 5 in 2023-09-23T21:16:40.620 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:16:40 smithi026 ceph-mon[179688]: pgmap v1427: 129 pgs: 1 active+recovering+undersized+remapped, 8 active+recovery_wait+degraded, 4 active+recovery_wait+undersized+degraded+remapped, 116 active+clean; 4.5 GiB data, 15 GiB used, 432 GiB / 447 GiB avail; 90 MiB/s wr, 1.44k op/s; 21196/222507 objects degraded (9.526%); 3083/222507 objects misplaced (1.386%); 13 MiB/s, 204 objects/s recovering 2023-09-23T21:16:40.620 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:16:40 smithi026 ceph-mon[179688]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:16:40.621 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:16:40 smithi026 ceph-mon[179688]: failed to encode map e563 with expected crc 2023-09-23T21:16:40.621 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:16:40 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/3285609271' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T21:16:40.621 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:16:40 smithi026 ceph-mon[179688]: osdmap e563: 8 total, 8 up, 5 in 2023-09-23T21:16:40.636 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:16:40.670 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:16:40.699 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:16:40.741 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:16:40.745 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:16:40.745 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:16:40.846 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:16:40.924 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:16:41.012 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:16:41.016 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:16:41.016 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:16:41.117 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:16:41.146 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:16:41.190 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:16:41.193 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:16:41.193 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:16:41.294 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:16:41.327 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:16:41.369 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:16:41.373 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:16:41.373 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:16:41.418 INFO:teuthology.orchestra.run.smithi026.stderr:marked in osd.7. 2023-09-23T21:16:41.446 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:16:41 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/3867827776' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["7"]}]: dispatch 2023-09-23T21:16:41.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:16:41 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["7"]}]: dispatch 2023-09-23T21:16:41.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:16:41 smithi026 ceph-mon[179688]: Health check update: Degraded data redundancy: 21196/222507 objects degraded (9.526%), 12 pgs degraded (PG_DEGRADED) 2023-09-23T21:16:41.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:16:41 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/3867827776' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["7"]}]: dispatch 2023-09-23T21:16:41.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:16:41 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["7"]}]: dispatch 2023-09-23T21:16:41.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:16:41 smithi026 ceph-mon[175151]: Health check update: Degraded data redundancy: 21196/222507 objects degraded (9.526%), 12 pgs degraded (PG_DEGRADED) 2023-09-23T21:16:41.474 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:16:41.493 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 69 16 78889 78873 71.3884 53.625 0.00359601 0.0139882 2023-09-23T21:16:41.497 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:16:41 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/3867827776' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["7"]}]: dispatch 2023-09-23T21:16:41.497 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:16:41 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["7"]}]: dispatch 2023-09-23T21:16:41.497 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:16:41 smithi070 ceph-mon[168282]: Health check update: Degraded data redundancy: 21196/222507 objects degraded (9.526%), 12 pgs degraded (PG_DEGRADED) 2023-09-23T21:16:41.499 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:16:41.531 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:16:41.533 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:16:41.534 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:16:41.585 INFO:teuthology.orchestra.run.smithi026.stdout:{ 2023-09-23T21:16:41.586 INFO:teuthology.orchestra.run.smithi026.stdout: "size": 20, 2023-09-23T21:16:41.586 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 600, 2023-09-23T21:16:41.586 INFO:teuthology.orchestra.run.smithi026.stdout: "ops": [ 2023-09-23T21:16:41.586 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:41.586 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.46328.0:99972 10.6 10:650fbbdc:::benchmark_data_smithi026_379838_object20015:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e494)", 2023-09-23T21:16:41.586 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T21:13:12.188907+0000", 2023-09-23T21:16:41.586 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 209.293582708, 2023-09-23T21:16:41.586 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.267026682, 2023-09-23T21:16:41.586 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:16:41.586 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:16:41.587 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T21:16:41.587 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.46328", 2023-09-23T21:16:41.587 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/3748861181", 2023-09-23T21:16:41.587 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 99972 2023-09-23T21:16:41.587 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:41.587 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:16:41.587 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:41.587 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:16:41.587 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:12.188907+0000", 2023-09-23T21:16:41.587 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:16:41.587 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:41.587 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:41.587 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:16:41.588 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:12.188907+0000", 2023-09-23T21:16:41.588 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:16:41.588 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:41.588 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:41.588 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:16:41.588 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:12.188906+0000", 2023-09-23T21:16:41.588 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.999999 2023-09-23T21:16:41.588 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:41.588 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:41.588 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:16:41.588 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:12.188911+0000", 2023-09-23T21:16:41.588 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.8790000000000002e-06 2023-09-23T21:16:41.589 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:41.589 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:41.589 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:16:41.589 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:12.188921+0000", 2023-09-23T21:16:41.589 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.8789999999999998e-06 2023-09-23T21:16:41.589 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:41.589 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:41.589 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:16:41.589 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:12.188931+0000", 2023-09-23T21:16:41.589 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0287e-05 2023-09-23T21:16:41.589 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:41.589 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:41.589 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:16:41.590 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:12.200174+0000", 2023-09-23T21:16:41.590 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.011242722 2023-09-23T21:16:41.590 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:41.590 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:41.590 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T21:16:41.590 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:12.200187+0000", 2023-09-23T21:16:41.590 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.3198e-05 2023-09-23T21:16:41.590 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:41.590 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:41.590 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:16:41.590 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:14.419319+0000", 2023-09-23T21:16:41.590 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.2191318880000002 2023-09-23T21:16:41.591 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:41.591 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:41.591 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for degraded object", 2023-09-23T21:16:41.591 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:14.422985+0000", 2023-09-23T21:16:41.591 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00366571 2023-09-23T21:16:41.591 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:41.591 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:41.591 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:16:41.591 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:14.433515+0000", 2023-09-23T21:16:41.591 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.010530688999999999 2023-09-23T21:16:41.591 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:41.591 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:41.591 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:16:41.592 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:14.434736+0000", 2023-09-23T21:16:41.592 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.001220383 2023-09-23T21:16:41.592 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:41.592 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:41.592 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 4,5", 2023-09-23T21:16:41.592 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:14.436852+0000", 2023-09-23T21:16:41.592 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.002116845 2023-09-23T21:16:41.592 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:41.592 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:41.592 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T21:16:41.592 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:14.451217+0000", 2023-09-23T21:16:41.592 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.014364965 2023-09-23T21:16:41.593 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:41.593 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:41.593 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:16:41.593 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:14.451640+0000", 2023-09-23T21:16:41.593 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00042237600000000002 2023-09-23T21:16:41.593 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:41.593 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:41.593 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:16:41.593 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:14.452035+0000", 2023-09-23T21:16:41.593 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00039501800000000002 2023-09-23T21:16:41.593 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:41.593 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:41.594 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T21:16:41.594 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:14.452663+0000", 2023-09-23T21:16:41.594 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00062800300000000002 2023-09-23T21:16:41.594 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:41.594 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:41.594 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T21:16:41.594 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:14.455934+0000", 2023-09-23T21:16:41.594 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.003270893 2023-09-23T21:16:41.594 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:16:41.594 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:16:41.594 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:16:41.594 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:41.595 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:41.595 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.46328.0:99999 10.6 10:641f61df:::benchmark_data_smithi026_379838_object20042:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e495)", 2023-09-23T21:16:41.595 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T21:13:13.453310+0000", 2023-09-23T21:16:41.595 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 208.029179736, 2023-09-23T21:16:41.595 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0071024850000001, 2023-09-23T21:16:41.595 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:16:41.595 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:16:41.595 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T21:16:41.595 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.46328", 2023-09-23T21:16:41.595 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/3748861181", 2023-09-23T21:16:41.595 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 99999 2023-09-23T21:16:41.595 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:41.596 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:16:41.596 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:41.596 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:16:41.596 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:13.453310+0000", 2023-09-23T21:16:41.596 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:16:41.596 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:41.596 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:41.596 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:16:41.596 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:13.453310+0000", 2023-09-23T21:16:41.596 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:16:41.596 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:41.596 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:41.596 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:16:41.597 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:13.453308+0000", 2023-09-23T21:16:41.597 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999981 2023-09-23T21:16:41.597 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:41.597 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:41.597 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:16:41.597 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:13.453318+0000", 2023-09-23T21:16:41.597 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0453e-05 2023-09-23T21:16:41.597 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:41.597 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:41.597 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:16:41.597 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:13.453333+0000", 2023-09-23T21:16:41.598 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.4979e-05 2023-09-23T21:16:41.598 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:41.598 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:41.598 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:16:41.598 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:13.453344+0000", 2023-09-23T21:16:41.598 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0560999999999999e-05 2023-09-23T21:16:41.598 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:41.598 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:41.598 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:16:41.598 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:14.423175+0000", 2023-09-23T21:16:41.598 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.96983093399999998 2023-09-23T21:16:41.598 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:41.599 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:41.599 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for degraded object", 2023-09-23T21:16:41.599 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:14.426649+0000", 2023-09-23T21:16:41.599 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0034738199999999999 2023-09-23T21:16:41.599 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:41.599 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:41.599 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:16:41.599 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:14.445239+0000", 2023-09-23T21:16:41.599 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.018589859 2023-09-23T21:16:41.599 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:41.599 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:41.599 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:16:41.600 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:14.446367+0000", 2023-09-23T21:16:41.601 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0011285240000000001 2023-09-23T21:16:41.602 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:41.602 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:41.602 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 4,5", 2023-09-23T21:16:41.602 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:14.448530+0000", 2023-09-23T21:16:41.602 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.002163257 2023-09-23T21:16:41.602 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:41.602 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:41.602 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T21:16:41.602 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:14.456072+0000", 2023-09-23T21:16:41.602 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0075418560000000004 2023-09-23T21:16:41.602 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:41.602 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:41.603 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:16:41.603 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:14.456530+0000", 2023-09-23T21:16:41.603 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00045766099999999999 2023-09-23T21:16:41.603 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:41.603 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:41.603 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:16:41.603 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:14.457026+0000", 2023-09-23T21:16:41.603 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00049629700000000004 2023-09-23T21:16:41.603 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:41.603 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:41.603 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T21:16:41.603 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:14.457752+0000", 2023-09-23T21:16:41.604 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00072537999999999997 2023-09-23T21:16:41.604 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:41.604 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:41.604 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T21:16:41.604 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:14.460413+0000", 2023-09-23T21:16:41.604 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.002660974 2023-09-23T21:16:41.604 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:16:41.604 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:16:41.604 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:16:41.604 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:41.604 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:41.604 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.46328.0:105700 10.a 10.d45319ea (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e501)", 2023-09-23T21:16:41.604 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T21:13:30.377651+0000", 2023-09-23T21:16:41.605 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 191.10483909000001, 2023-09-23T21:16:41.605 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.37983421699999997, 2023-09-23T21:16:41.605 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:16:41.605 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "reached pg", 2023-09-23T21:16:41.605 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T21:16:41.605 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.46328", 2023-09-23T21:16:41.605 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/3748861181", 2023-09-23T21:16:41.605 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 105700 2023-09-23T21:16:41.605 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:41.605 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:16:41.605 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:41.605 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:16:41.606 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:30.377651+0000", 2023-09-23T21:16:41.606 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:16:41.606 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:41.606 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:41.606 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:16:41.606 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:30.377651+0000", 2023-09-23T21:16:41.606 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:16:41.606 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:41.606 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:41.606 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:16:41.606 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:30.377649+0000", 2023-09-23T21:16:41.606 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T21:16:41.607 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:41.607 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:41.607 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:16:41.607 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:30.377656+0000", 2023-09-23T21:16:41.607 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.4030000000000004e-06 2023-09-23T21:16:41.607 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:41.607 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:41.607 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:16:41.607 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:30.377673+0000", 2023-09-23T21:16:41.607 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.7438000000000001e-05 2023-09-23T21:16:41.607 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:41.608 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:41.608 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:16:41.608 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:30.377683+0000", 2023-09-23T21:16:41.608 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.6889999999999999e-06 2023-09-23T21:16:41.608 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:41.608 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:41.608 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:16:41.608 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:30.379823+0000", 2023-09-23T21:16:41.608 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0021400099999999999 2023-09-23T21:16:41.608 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:41.608 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:41.608 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T21:16:41.609 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:30.379836+0000", 2023-09-23T21:16:41.609 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.3652000000000001e-05 2023-09-23T21:16:41.609 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:41.609 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:41.609 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:16:41.609 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:30.757463+0000", 2023-09-23T21:16:41.609 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.37762642200000002 2023-09-23T21:16:41.609 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:41.609 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:41.609 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T21:16:41.609 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:30.757485+0000", 2023-09-23T21:16:41.609 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.2050999999999999e-05 2023-09-23T21:16:41.609 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:16:41.610 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:16:41.610 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:16:41.610 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:41.610 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:41.610 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.46328.0:105702 10.a 10.26b86e5a (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e501)", 2023-09-23T21:16:41.610 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T21:13:30.388674+0000", 2023-09-23T21:16:41.610 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 191.093816097, 2023-09-23T21:16:41.610 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.36884504800000001, 2023-09-23T21:16:41.610 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:16:41.610 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "reached pg", 2023-09-23T21:16:41.610 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T21:16:41.610 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.46328", 2023-09-23T21:16:41.611 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/3748861181", 2023-09-23T21:16:41.611 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 105702 2023-09-23T21:16:41.611 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:41.611 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:16:41.611 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:41.611 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:16:41.611 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:30.388674+0000", 2023-09-23T21:16:41.611 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:16:41.611 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:41.611 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:41.611 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:16:41.611 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:30.388674+0000", 2023-09-23T21:16:41.611 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:16:41.612 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:41.612 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:41.612 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:16:41.612 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:30.388672+0000", 2023-09-23T21:16:41.612 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T21:16:41.612 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:41.612 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:41.612 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:16:41.612 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:30.388680+0000", 2023-09-23T21:16:41.612 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.4399999999999999e-06 2023-09-23T21:16:41.612 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:41.612 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:41.612 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:16:41.613 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:30.388690+0000", 2023-09-23T21:16:41.613 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0916e-05 2023-09-23T21:16:41.613 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:41.613 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:41.613 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:16:41.613 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:30.388701+0000", 2023-09-23T21:16:41.613 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0995000000000001e-05 2023-09-23T21:16:41.613 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:41.613 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:41.613 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:16:41.613 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:30.388769+0000", 2023-09-23T21:16:41.613 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.7804000000000003e-05 2023-09-23T21:16:41.613 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:41.614 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:41.614 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T21:16:41.614 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:30.388788+0000", 2023-09-23T21:16:41.614 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.8850000000000001e-05 2023-09-23T21:16:41.614 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:41.614 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:41.614 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:16:41.614 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:30.757509+0000", 2023-09-23T21:16:41.614 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.368720777 2023-09-23T21:16:41.614 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:41.614 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:41.614 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T21:16:41.615 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:30.757519+0000", 2023-09-23T21:16:41.615 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.8800000000000003e-06 2023-09-23T21:16:41.615 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:16:41.615 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:16:41.615 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:16:41.615 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:41.615 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:41.615 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_repop(client.46328.0:111231 10.0 e521/502 10:059cb734:::benchmark_data_smithi026_379838_object31274:head v 521'4830, mlcod=435'1946)", 2023-09-23T21:16:41.615 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T21:13:58.628366+0000", 2023-09-23T21:16:41.615 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 162.854123485, 2023-09-23T21:16:41.615 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.37221736, 2023-09-23T21:16:41.615 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:16:41.616 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:16:41.616 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:16:41.616 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:41.616 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:16:41.616 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:58.628366+0000", 2023-09-23T21:16:41.616 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:16:41.616 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:41.616 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:41.616 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:16:41.616 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:58.628366+0000", 2023-09-23T21:16:41.616 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:16:41.616 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:41.616 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:41.617 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:16:41.617 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:58.628366+0000", 2023-09-23T21:16:41.617 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T21:16:41.617 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:41.617 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:41.617 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:16:41.617 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:58.628376+0000", 2023-09-23T21:16:41.617 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0349e-05 2023-09-23T21:16:41.617 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:41.617 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:41.617 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:16:41.617 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:58.995558+0000", 2023-09-23T21:16:41.617 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.36718158400000001 2023-09-23T21:16:41.618 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:41.618 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:41.618 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:16:41.618 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:58.995569+0000", 2023-09-23T21:16:41.618 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1681e-05 2023-09-23T21:16:41.618 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:41.618 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:41.618 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:16:41.618 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:58.995616+0000", 2023-09-23T21:16:41.618 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.6307999999999998e-05 2023-09-23T21:16:41.618 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:41.618 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:41.619 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:16:41.619 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:58.995652+0000", 2023-09-23T21:16:41.619 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.5957999999999997e-05 2023-09-23T21:16:41.619 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:41.619 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:41.619 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T21:16:41.619 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:59.000554+0000", 2023-09-23T21:16:41.619 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0049026850000000004 2023-09-23T21:16:41.619 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:41.619 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:41.619 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T21:16:41.619 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:59.000584+0000", 2023-09-23T21:16:41.619 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.9453000000000001e-05 2023-09-23T21:16:41.620 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:16:41.620 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:16:41.620 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:16:41.620 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:41.620 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:41.620 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_repop(client.46328.0:111334 10.11 e521/502 10:8e9e8bfc:::benchmark_data_smithi026_379838_object31377:head v 521'4778, mlcod=520'4776)", 2023-09-23T21:16:41.620 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T21:13:58.629174+0000", 2023-09-23T21:16:41.620 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 162.85331534100001, 2023-09-23T21:16:41.620 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.36777505399999999, 2023-09-23T21:16:41.620 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:16:41.620 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:16:41.620 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:16:41.620 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:41.621 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:16:41.621 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:58.629174+0000", 2023-09-23T21:16:41.621 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:16:41.621 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:41.621 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:41.621 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:16:41.621 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:58.629174+0000", 2023-09-23T21:16:41.621 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:16:41.621 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:41.621 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:41.621 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:16:41.621 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:58.629174+0000", 2023-09-23T21:16:41.621 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.999999 2023-09-23T21:16:41.622 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:41.622 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:41.622 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:16:41.622 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:58.629181+0000", 2023-09-23T21:16:41.622 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.0110000000000003e-06 2023-09-23T21:16:41.622 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:41.622 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:41.622 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:16:41.622 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:58.995526+0000", 2023-09-23T21:16:41.622 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.36634555499999999 2023-09-23T21:16:41.622 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:41.622 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:41.623 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:16:41.623 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:58.995550+0000", 2023-09-23T21:16:41.623 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.3901999999999999e-05 2023-09-23T21:16:41.623 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:41.623 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:41.623 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:16:41.623 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:58.995617+0000", 2023-09-23T21:16:41.623 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.6772000000000004e-05 2023-09-23T21:16:41.623 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:41.623 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:41.623 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:16:41.623 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:58.995644+0000", 2023-09-23T21:16:41.623 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.6709000000000001e-05 2023-09-23T21:16:41.624 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:41.624 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:41.624 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T21:16:41.624 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:58.996910+0000", 2023-09-23T21:16:41.624 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0012662610000000001 2023-09-23T21:16:41.624 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:41.624 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:41.624 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T21:16:41.624 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:13:58.996949+0000", 2023-09-23T21:16:41.624 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.9594000000000001e-05 2023-09-23T21:16:41.624 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:16:41.624 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:16:41.625 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:16:41.625 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:41.625 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:41.625 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.46328.0:112004 10.14 10:2ab213e5:::benchmark_data_smithi026_379838_object32047:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e522)", 2023-09-23T21:16:41.625 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T21:14:00.500275+0000", 2023-09-23T21:16:41.625 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 160.98221493200001, 2023-09-23T21:16:41.625 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0387811549999999, 2023-09-23T21:16:41.625 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:16:41.625 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:16:41.625 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T21:16:41.625 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.46328", 2023-09-23T21:16:41.625 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/3748861181", 2023-09-23T21:16:41.625 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 112004 2023-09-23T21:16:41.626 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:41.626 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:16:41.626 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:41.626 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:16:41.626 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:14:00.500275+0000", 2023-09-23T21:16:41.626 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:16:41.626 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:41.626 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:41.626 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:16:41.626 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:14:00.500275+0000", 2023-09-23T21:16:41.626 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:16:41.626 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:41.627 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:41.627 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:16:41.627 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:14:00.500273+0000", 2023-09-23T21:16:41.627 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T21:16:41.627 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:41.627 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:41.627 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:16:41.627 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:14:00.500280+0000", 2023-09-23T21:16:41.627 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.3300000000000004e-06 2023-09-23T21:16:41.627 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:41.627 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:41.627 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:16:41.627 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:14:00.500292+0000", 2023-09-23T21:16:41.628 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1822e-05 2023-09-23T21:16:41.628 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:41.628 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:41.628 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:16:41.628 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:14:00.500299+0000", 2023-09-23T21:16:41.628 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.9079999999999995e-06 2023-09-23T21:16:41.628 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:41.628 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:41.628 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:16:41.628 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:14:00.500356+0000", 2023-09-23T21:16:41.628 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.7058000000000002e-05 2023-09-23T21:16:41.628 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:41.628 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:41.629 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T21:16:41.629 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:14:00.500366+0000", 2023-09-23T21:16:41.629 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.73e-06 2023-09-23T21:16:41.629 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:41.629 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:41.629 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:16:41.629 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:14:01.514070+0000", 2023-09-23T21:16:41.629 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.013704121 2023-09-23T21:16:41.629 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:41.629 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:41.629 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:16:41.629 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:14:01.514202+0000", 2023-09-23T21:16:41.629 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00013123599999999999 2023-09-23T21:16:41.630 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:41.630 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:41.630 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 3,4", 2023-09-23T21:16:41.630 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:14:01.514488+0000", 2023-09-23T21:16:41.630 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00028681699999999997 2023-09-23T21:16:41.630 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:41.630 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:41.630 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:16:41.630 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:14:01.519905+0000", 2023-09-23T21:16:41.630 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0054170959999999997 2023-09-23T21:16:41.630 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:41.631 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:41.631 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:16:41.631 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:14:01.525371+0000", 2023-09-23T21:16:41.631 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0054652729999999997 2023-09-23T21:16:41.631 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:41.631 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:41.631 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T21:16:41.631 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:14:01.538952+0000", 2023-09-23T21:16:41.631 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.013580938000000001 2023-09-23T21:16:41.631 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:41.631 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:41.631 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T21:16:41.631 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:14:01.539012+0000", 2023-09-23T21:16:41.632 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.0356000000000003e-05 2023-09-23T21:16:41.632 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:41.632 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:41.632 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T21:16:41.632 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:14:01.539056+0000", 2023-09-23T21:16:41.632 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.3961999999999997e-05 2023-09-23T21:16:41.632 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:16:41.632 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:16:41.632 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:16:41.632 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:41.632 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:41.632 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.46328.0:112018 10.14 10:2860496d:::benchmark_data_smithi026_379838_object32061:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e522)", 2023-09-23T21:16:41.633 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T21:14:00.531660+0000", 2023-09-23T21:16:41.633 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 160.950829505, 2023-09-23T21:16:41.633 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0075591260000001, 2023-09-23T21:16:41.633 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:16:41.633 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:16:41.633 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T21:16:41.633 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.46328", 2023-09-23T21:16:41.633 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/3748861181", 2023-09-23T21:16:41.633 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 112018 2023-09-23T21:16:41.633 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:41.633 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:16:41.633 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:41.633 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:16:41.634 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:14:00.531660+0000", 2023-09-23T21:16:41.634 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:16:41.634 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:41.634 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:41.634 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:16:41.634 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:14:00.531660+0000", 2023-09-23T21:16:41.634 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:16:41.634 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:41.634 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:41.634 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:16:41.634 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:14:00.531658+0000", 2023-09-23T21:16:41.634 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999981 2023-09-23T21:16:41.635 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:41.635 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:41.635 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:16:41.635 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:14:00.531666+0000", 2023-09-23T21:16:41.635 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.3610000000000003e-06 2023-09-23T21:16:41.635 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:41.635 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:41.635 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:16:41.635 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:14:00.531678+0000", 2023-09-23T21:16:41.635 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.2707999999999999e-05 2023-09-23T21:16:41.635 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:41.635 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:41.635 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:16:41.636 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:14:00.531687+0000", 2023-09-23T21:16:41.636 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.2360000000000004e-06 2023-09-23T21:16:41.636 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:41.636 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:41.636 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:16:41.636 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:14:00.531753+0000", 2023-09-23T21:16:41.636 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.6613999999999996e-05 2023-09-23T21:16:41.636 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:41.636 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:41.636 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T21:16:41.636 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:14:00.531766+0000", 2023-09-23T21:16:41.636 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.272e-05 2023-09-23T21:16:41.637 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:41.637 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:41.637 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:16:41.637 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:14:01.516057+0000", 2023-09-23T21:16:41.637 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.98429071400000001 2023-09-23T21:16:41.637 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:41.637 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:41.637 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:16:41.637 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:14:01.516195+0000", 2023-09-23T21:16:41.637 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00013835599999999999 2023-09-23T21:16:41.637 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:41.637 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:41.637 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 3,4", 2023-09-23T21:16:41.638 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:14:01.516467+0000", 2023-09-23T21:16:41.638 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00027230700000000001 2023-09-23T21:16:41.638 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:41.638 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:41.638 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:16:41.638 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:14:01.525339+0000", 2023-09-23T21:16:41.638 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.008871416 2023-09-23T21:16:41.638 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:41.638 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:41.638 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:16:41.638 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:14:01.525401+0000", 2023-09-23T21:16:41.638 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.2712000000000005e-05 2023-09-23T21:16:41.639 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:41.639 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:41.639 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T21:16:41.639 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:14:01.539151+0000", 2023-09-23T21:16:41.639 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.013749513 2023-09-23T21:16:41.639 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:41.639 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:41.639 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T21:16:41.639 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:14:01.539187+0000", 2023-09-23T21:16:41.639 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.6109000000000003e-05 2023-09-23T21:16:41.639 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:41.639 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:41.639 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T21:16:41.640 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:14:01.539219+0000", 2023-09-23T21:16:41.640 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.2370999999999999e-05 2023-09-23T21:16:41.640 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:16:41.640 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:16:41.640 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:16:41.640 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:41.640 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:41.640 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.46328.0:112020 10.14 10:2ad7942e:::benchmark_data_smithi026_379838_object32063:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e522)", 2023-09-23T21:16:41.640 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T21:14:00.532917+0000", 2023-09-23T21:16:41.640 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 160.94957245800001, 2023-09-23T21:16:41.640 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0063778370000001, 2023-09-23T21:16:41.640 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:16:41.640 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:16:41.641 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T21:16:41.641 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.46328", 2023-09-23T21:16:41.641 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/3748861181", 2023-09-23T21:16:41.641 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 112020 2023-09-23T21:16:41.641 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:41.641 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:16:41.641 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:41.641 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:16:41.641 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:14:00.532917+0000", 2023-09-23T21:16:41.641 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:16:41.641 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:41.641 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:41.641 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:16:41.642 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:14:00.532917+0000", 2023-09-23T21:16:41.642 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:16:41.642 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:41.642 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:41.642 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:16:41.642 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:14:00.532917+0000", 2023-09-23T21:16:41.642 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.999999 2023-09-23T21:16:41.642 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:41.642 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:41.642 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:16:41.642 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:14:00.532921+0000", 2023-09-23T21:16:41.642 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.3080000000000001e-06 2023-09-23T21:16:41.643 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:41.643 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:41.643 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:16:41.643 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:14:00.532932+0000", 2023-09-23T21:16:41.643 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1205e-05 2023-09-23T21:16:41.643 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:41.643 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:41.643 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:16:41.643 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:14:00.532937+0000", 2023-09-23T21:16:41.643 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.3059999999999997e-06 2023-09-23T21:16:41.643 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:41.643 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:41.643 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:16:41.644 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:14:00.532998+0000", 2023-09-23T21:16:41.644 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.0504000000000002e-05 2023-09-23T21:16:41.644 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:41.644 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:41.644 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T21:16:41.644 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:14:00.533009+0000", 2023-09-23T21:16:41.644 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1453e-05 2023-09-23T21:16:41.644 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:41.644 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:41.644 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:16:41.644 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:14:01.518082+0000", 2023-09-23T21:16:41.644 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.98507318399999999 2023-09-23T21:16:41.645 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:41.645 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:41.645 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:16:41.645 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:14:01.518190+0000", 2023-09-23T21:16:41.645 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000107796 2023-09-23T21:16:41.645 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:41.645 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:41.645 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 3,4", 2023-09-23T21:16:41.645 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:14:01.518375+0000", 2023-09-23T21:16:41.645 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00018488700000000001 2023-09-23T21:16:41.645 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:41.645 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:41.645 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:16:41.646 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:14:01.534073+0000", 2023-09-23T21:16:41.646 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.015698167999999998 2023-09-23T21:16:41.646 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:41.646 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:41.646 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:16:41.646 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:14:01.534230+0000", 2023-09-23T21:16:41.646 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00015691699999999999 2023-09-23T21:16:41.646 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:41.646 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:41.646 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T21:16:41.646 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:14:01.539226+0000", 2023-09-23T21:16:41.646 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00499564 2023-09-23T21:16:41.647 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:41.647 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:41.647 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T21:16:41.647 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:14:01.539263+0000", 2023-09-23T21:16:41.647 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.7036000000000003e-05 2023-09-23T21:16:41.647 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:41.647 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:41.647 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T21:16:41.647 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:14:01.539295+0000", 2023-09-23T21:16:41.647 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.2211e-05 2023-09-23T21:16:41.647 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:16:41.647 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:16:41.647 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:16:41.648 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:41.648 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:41.648 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.46328.0:112026 10.14 10:2d0ad96b:::benchmark_data_smithi026_379838_object32069:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e522)", 2023-09-23T21:16:41.648 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T21:14:00.587188+0000", 2023-09-23T21:16:41.648 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 160.89530140299999, 2023-09-23T21:16:41.648 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.95219126899999995, 2023-09-23T21:16:41.648 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:16:41.648 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:16:41.648 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T21:16:41.648 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.46328", 2023-09-23T21:16:41.648 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/3748861181", 2023-09-23T21:16:41.648 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 112026 2023-09-23T21:16:41.649 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:41.649 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:16:41.649 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:41.649 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:16:41.649 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:14:00.587188+0000", 2023-09-23T21:16:41.649 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:16:41.649 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:41.649 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:41.649 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:16:41.649 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:14:00.587188+0000", 2023-09-23T21:16:41.649 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:16:41.649 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:41.649 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:41.650 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:16:41.650 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:14:00.587187+0000", 2023-09-23T21:16:41.650 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999981 2023-09-23T21:16:41.650 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:41.650 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:41.650 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:16:41.650 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:14:00.587193+0000", 2023-09-23T21:16:41.650 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.3470000000000004e-06 2023-09-23T21:16:41.650 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:41.650 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:41.650 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:16:41.650 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:14:00.587206+0000", 2023-09-23T21:16:41.651 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.3137e-05 2023-09-23T21:16:41.651 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:41.651 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:41.651 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:16:41.651 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:14:00.587212+0000", 2023-09-23T21:16:41.651 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.003e-06 2023-09-23T21:16:41.651 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:41.651 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:41.651 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:16:41.651 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:14:00.587267+0000", 2023-09-23T21:16:41.651 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.4931e-05 2023-09-23T21:16:41.651 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:41.651 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:41.652 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T21:16:41.652 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:14:00.587274+0000", 2023-09-23T21:16:41.652 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.5299999999999999e-06 2023-09-23T21:16:41.652 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:41.652 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:41.652 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:16:41.652 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:14:01.519930+0000", 2023-09-23T21:16:41.652 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.93265571400000002 2023-09-23T21:16:41.652 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:41.652 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:41.652 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:16:41.653 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:14:01.520049+0000", 2023-09-23T21:16:41.653 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000118936 2023-09-23T21:16:41.653 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:41.653 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:41.653 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 3,4", 2023-09-23T21:16:41.653 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:14:01.520282+0000", 2023-09-23T21:16:41.653 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00023281300000000001 2023-09-23T21:16:41.653 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:41.653 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:41.653 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:16:41.653 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:14:01.534105+0000", 2023-09-23T21:16:41.653 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.013822872 2023-09-23T21:16:41.653 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:41.654 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:41.654 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:16:41.654 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:14:01.534317+0000", 2023-09-23T21:16:41.654 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00021244800000000001 2023-09-23T21:16:41.654 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:41.654 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:41.654 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T21:16:41.654 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:14:01.539301+0000", 2023-09-23T21:16:41.654 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0049840700000000002 2023-09-23T21:16:41.654 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:41.654 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:41.654 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T21:16:41.655 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:14:01.539344+0000", 2023-09-23T21:16:41.655 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.2635000000000001e-05 2023-09-23T21:16:41.655 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:41.655 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:41.655 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T21:16:41.655 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:14:01.539380+0000", 2023-09-23T21:16:41.655 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.5675e-05 2023-09-23T21:16:41.655 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:16:41.655 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:16:41.655 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:16:41.655 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:41.655 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:41.655 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.46328.0:112070 10.14 10:2bdb93b2:::benchmark_data_smithi026_379838_object32113:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e522)", 2023-09-23T21:16:41.656 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T21:14:00.670023+0000", 2023-09-23T21:16:41.656 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 160.81246722500001, 2023-09-23T21:16:41.656 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.86943998199999994, 2023-09-23T21:16:41.656 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:16:41.656 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:16:41.656 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T21:16:41.656 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.46328", 2023-09-23T21:16:41.656 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/3748861181", 2023-09-23T21:16:41.656 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 112070 2023-09-23T21:16:41.656 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:41.656 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:16:41.656 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:41.656 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:16:41.657 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:14:00.670023+0000", 2023-09-23T21:16:41.657 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:16:41.657 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:41.657 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:41.657 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:16:41.657 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:14:00.670023+0000", 2023-09-23T21:16:41.657 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:16:41.657 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:41.657 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:41.657 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:16:41.657 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:14:00.670020+0000", 2023-09-23T21:16:41.658 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999976 2023-09-23T21:16:41.658 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:41.658 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:41.658 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:16:41.658 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:14:00.670028+0000", 2023-09-23T21:16:41.658 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.3300000000000001e-06 2023-09-23T21:16:41.658 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:41.658 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:41.658 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:16:41.658 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:14:00.670040+0000", 2023-09-23T21:16:41.658 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.2071e-05 2023-09-23T21:16:41.658 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:41.658 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:41.659 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:16:41.659 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:14:00.670048+0000", 2023-09-23T21:16:41.659 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.9179999999999994e-06 2023-09-23T21:16:41.659 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:41.659 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:41.659 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:16:41.659 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:14:00.670116+0000", 2023-09-23T21:16:41.659 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.8277999999999999e-05 2023-09-23T21:16:41.659 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:41.659 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:41.659 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T21:16:41.659 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:14:00.670129+0000", 2023-09-23T21:16:41.660 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.3052e-05 2023-09-23T21:16:41.660 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:41.660 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:41.660 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:16:41.660 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:14:01.521719+0000", 2023-09-23T21:16:41.660 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.85158983099999996 2023-09-23T21:16:41.660 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:41.660 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:41.660 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:16:41.660 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:14:01.521848+0000", 2023-09-23T21:16:41.660 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000129558 2023-09-23T21:16:41.660 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:41.660 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:41.661 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 3,4", 2023-09-23T21:16:41.661 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:14:01.522134+0000", 2023-09-23T21:16:41.661 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00028569499999999998 2023-09-23T21:16:41.661 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:41.661 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:41.661 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:16:41.661 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:14:01.534134+0000", 2023-09-23T21:16:41.661 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.012000022000000001 2023-09-23T21:16:41.661 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:41.661 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:41.661 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:16:41.661 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:14:01.534348+0000", 2023-09-23T21:16:41.662 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00021344199999999999 2023-09-23T21:16:41.662 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:41.662 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:41.662 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T21:16:41.662 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:14:01.539386+0000", 2023-09-23T21:16:41.662 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0050384820000000004 2023-09-23T21:16:41.662 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:41.662 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:41.662 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T21:16:41.662 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:14:01.539428+0000", 2023-09-23T21:16:41.662 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.2148999999999999e-05 2023-09-23T21:16:41.662 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:41.663 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:41.663 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T21:16:41.663 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:14:01.539463+0000", 2023-09-23T21:16:41.663 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.4351999999999999e-05 2023-09-23T21:16:41.663 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:16:41.663 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:16:41.663 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:16:41.663 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:41.663 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:41.663 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.46328.0:112097 10.14 10:2eaf8bdc:::benchmark_data_smithi026_379838_object32140:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e522)", 2023-09-23T21:16:41.663 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T21:14:00.712118+0000", 2023-09-23T21:16:41.663 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 160.77037161999999, 2023-09-23T21:16:41.663 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.82743756300000004, 2023-09-23T21:16:41.664 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:16:41.664 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:16:41.664 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T21:16:41.664 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.46328", 2023-09-23T21:16:41.664 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/3748861181", 2023-09-23T21:16:41.664 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 112097 2023-09-23T21:16:41.664 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:41.664 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:16:41.664 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:41.664 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:16:41.664 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:14:00.712118+0000", 2023-09-23T21:16:41.664 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:16:41.665 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:41.665 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:41.665 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:16:41.665 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:14:00.712118+0000", 2023-09-23T21:16:41.665 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:16:41.665 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:41.665 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:41.665 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:16:41.665 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:14:00.712117+0000", 2023-09-23T21:16:41.665 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T21:16:41.665 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:41.665 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:41.665 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:16:41.666 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:14:00.712123+0000", 2023-09-23T21:16:41.666 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.9730000000000002e-06 2023-09-23T21:16:41.666 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:41.666 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:41.666 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:16:41.666 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:14:00.712138+0000", 2023-09-23T21:16:41.666 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.4958e-05 2023-09-23T21:16:41.666 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:41.666 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:41.666 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:16:41.666 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:14:00.712145+0000", 2023-09-23T21:16:41.666 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.5020000000000003e-06 2023-09-23T21:16:41.667 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:41.667 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:41.667 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:16:41.667 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:14:00.712199+0000", 2023-09-23T21:16:41.667 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.4141999999999998e-05 2023-09-23T21:16:41.667 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:41.667 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:41.667 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T21:16:41.667 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:14:00.712208+0000", 2023-09-23T21:16:41.667 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.5159999999999994e-06 2023-09-23T21:16:41.667 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:41.668 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:41.668 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:16:41.668 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:14:01.523550+0000", 2023-09-23T21:16:41.668 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.81134222600000006 2023-09-23T21:16:41.668 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:41.668 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:41.668 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:16:41.668 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:14:01.523654+0000", 2023-09-23T21:16:41.668 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000103562 2023-09-23T21:16:41.668 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:41.668 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:41.668 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 3,4", 2023-09-23T21:16:41.669 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:14:01.523893+0000", 2023-09-23T21:16:41.669 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00023928099999999999 2023-09-23T21:16:41.669 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:41.669 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:41.669 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:16:41.669 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:14:01.534164+0000", 2023-09-23T21:16:41.669 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.010271471000000001 2023-09-23T21:16:41.669 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:41.669 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:41.669 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:16:41.669 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:14:01.538942+0000", 2023-09-23T21:16:41.669 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0047775320000000001 2023-09-23T21:16:41.669 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:41.670 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:41.670 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T21:16:41.670 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:14:01.539469+0000", 2023-09-23T21:16:41.670 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00052666900000000001 2023-09-23T21:16:41.670 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:41.670 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:41.670 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T21:16:41.670 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:14:01.539521+0000", 2023-09-23T21:16:41.670 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.1941000000000003e-05 2023-09-23T21:16:41.670 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:41.670 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:41.671 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T21:16:41.671 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:14:01.539556+0000", 2023-09-23T21:16:41.671 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.5213000000000003e-05 2023-09-23T21:16:41.671 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:16:41.671 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:16:41.671 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:16:41.671 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:41.671 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:41.671 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.46328.0:112200 10.14 10:2a5c1230:::benchmark_data_smithi026_379838_object32243:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e522)", 2023-09-23T21:16:41.671 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T21:14:00.941399+0000", 2023-09-23T21:16:41.671 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 160.54109123699999, 2023-09-23T21:16:41.671 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.60795617000000002, 2023-09-23T21:16:41.672 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:16:41.672 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:16:41.672 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T21:16:41.672 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.46328", 2023-09-23T21:16:41.672 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/3748861181", 2023-09-23T21:16:41.672 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 112200 2023-09-23T21:16:41.672 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:41.672 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:16:41.672 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:41.672 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:16:41.672 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:14:00.941399+0000", 2023-09-23T21:16:41.672 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:16:41.673 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:41.673 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:41.673 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:16:41.673 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:14:00.941399+0000", 2023-09-23T21:16:41.673 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:16:41.673 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:41.673 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:41.673 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:16:41.673 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:14:00.941397+0000", 2023-09-23T21:16:41.673 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T21:16:41.673 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:41.673 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:41.674 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:16:41.674 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:14:00.941403+0000", 2023-09-23T21:16:41.674 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.3629999999999999e-06 2023-09-23T21:16:41.674 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:41.674 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:41.674 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:16:41.674 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:14:00.941418+0000", 2023-09-23T21:16:41.674 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.5143000000000001e-05 2023-09-23T21:16:41.674 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:41.674 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:41.674 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:16:41.674 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:14:00.941426+0000", 2023-09-23T21:16:41.675 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.4749999999999996e-06 2023-09-23T21:16:41.675 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:41.675 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:41.675 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:16:41.675 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:14:00.941491+0000", 2023-09-23T21:16:41.675 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.5562999999999996e-05 2023-09-23T21:16:41.675 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:41.675 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:41.675 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T21:16:41.675 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:14:00.941503+0000", 2023-09-23T21:16:41.675 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1919000000000001e-05 2023-09-23T21:16:41.675 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:41.676 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:41.676 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:16:41.676 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:14:01.525423+0000", 2023-09-23T21:16:41.676 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.58391958099999997 2023-09-23T21:16:41.676 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:41.676 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:41.676 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:16:41.676 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:14:01.525551+0000", 2023-09-23T21:16:41.676 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00012816499999999999 2023-09-23T21:16:41.676 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:41.676 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:41.676 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 3,4", 2023-09-23T21:16:41.677 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:14:01.525798+0000", 2023-09-23T21:16:41.677 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00024720699999999999 2023-09-23T21:16:41.677 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:41.677 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:41.677 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:16:41.677 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:14:01.534197+0000", 2023-09-23T21:16:41.677 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0083985180000000006 2023-09-23T21:16:41.677 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:41.677 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:41.677 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T21:16:41.677 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:14:01.539563+0000", 2023-09-23T21:16:41.677 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0053662090000000003 2023-09-23T21:16:41.678 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:41.678 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:41.678 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:16:41.678 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:14:01.549281+0000", 2023-09-23T21:16:41.678 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0097184379999999994 2023-09-23T21:16:41.678 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:41.678 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:41.678 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T21:16:41.678 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:14:01.549317+0000", 2023-09-23T21:16:41.678 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.5324000000000003e-05 2023-09-23T21:16:41.678 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:41.678 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:41.679 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T21:16:41.679 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:14:01.549355+0000", 2023-09-23T21:16:41.679 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.7929000000000002e-05 2023-09-23T21:16:41.679 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:16:41.679 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:16:41.679 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:16:41.679 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:41.679 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:41.679 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.46328.0:112293 10.14 10:2b83171c:::benchmark_data_smithi026_379838_object32336:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e522)", 2023-09-23T21:16:41.679 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T21:14:01.182239+0000", 2023-09-23T21:16:41.679 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 160.300251102, 2023-09-23T21:16:41.679 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.36771867699999999, 2023-09-23T21:16:41.680 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:16:41.680 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:16:41.680 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T21:16:41.680 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.46328", 2023-09-23T21:16:41.680 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/3748861181", 2023-09-23T21:16:41.680 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 112293 2023-09-23T21:16:41.680 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:41.680 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:16:41.680 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:41.680 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:16:41.680 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:14:01.182239+0000", 2023-09-23T21:16:41.680 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:16:41.681 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:41.681 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:41.681 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:16:41.681 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:14:01.182239+0000", 2023-09-23T21:16:41.681 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:16:41.681 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:41.681 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:41.681 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:16:41.681 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:14:01.182237+0000", 2023-09-23T21:16:41.681 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T21:16:41.681 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:41.681 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:41.681 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:16:41.682 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:14:01.182244+0000", 2023-09-23T21:16:41.682 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.6390000000000003e-06 2023-09-23T21:16:41.682 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:41.682 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:41.682 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:16:41.682 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:14:01.182254+0000", 2023-09-23T21:16:41.682 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0662999999999999e-05 2023-09-23T21:16:41.682 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:41.682 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:41.682 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:16:41.682 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:14:01.182262+0000", 2023-09-23T21:16:41.682 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.6090000000000003e-06 2023-09-23T21:16:41.683 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:41.683 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:41.683 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:16:41.683 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:14:01.182315+0000", 2023-09-23T21:16:41.683 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.3335999999999999e-05 2023-09-23T21:16:41.683 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:41.683 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:41.683 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T21:16:41.683 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:14:01.182326+0000", 2023-09-23T21:16:41.683 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0489000000000001e-05 2023-09-23T21:16:41.683 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:41.683 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:41.684 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:16:41.684 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:14:01.527190+0000", 2023-09-23T21:16:41.684 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.34486373300000001 2023-09-23T21:16:41.684 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:41.684 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:41.684 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:16:41.684 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:14:01.527315+0000", 2023-09-23T21:16:41.684 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00012556199999999999 2023-09-23T21:16:41.684 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:41.684 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:41.684 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 3,4", 2023-09-23T21:16:41.684 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:14:01.527576+0000", 2023-09-23T21:16:41.685 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000261086 2023-09-23T21:16:41.685 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:41.685 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:41.685 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:16:41.685 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:14:01.534259+0000", 2023-09-23T21:16:41.685 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0066831290000000003 2023-09-23T21:16:41.685 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:41.685 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:41.685 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T21:16:41.685 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:14:01.539568+0000", 2023-09-23T21:16:41.685 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0053088780000000004 2023-09-23T21:16:41.686 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:41.686 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:41.686 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:16:41.686 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:14:01.549878+0000", 2023-09-23T21:16:41.686 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.010309420999999999 2023-09-23T21:16:41.686 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:41.686 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:41.686 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T21:16:41.686 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:14:01.549920+0000", 2023-09-23T21:16:41.686 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.1817000000000003e-05 2023-09-23T21:16:41.686 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:41.686 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:41.687 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T21:16:41.687 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:14:01.549957+0000", 2023-09-23T21:16:41.687 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.7824000000000002e-05 2023-09-23T21:16:41.687 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:16:41.687 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:16:41.687 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:16:41.687 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:41.687 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:41.687 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.46328.0:112298 10.14 10:2a87c47f:::benchmark_data_smithi026_379838_object32341:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e522)", 2023-09-23T21:16:41.687 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T21:14:01.190720+0000", 2023-09-23T21:16:41.687 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 160.291769362, 2023-09-23T21:16:41.687 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.359408432, 2023-09-23T21:16:41.688 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:16:41.688 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:16:41.688 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T21:16:41.688 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.46328", 2023-09-23T21:16:41.688 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/3748861181", 2023-09-23T21:16:41.688 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 112298 2023-09-23T21:16:41.688 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:41.688 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:16:41.688 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:41.688 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:16:41.688 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:14:01.190720+0000", 2023-09-23T21:16:41.688 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:16:41.688 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:41.689 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:41.689 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:16:41.689 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:14:01.190720+0000", 2023-09-23T21:16:41.689 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:16:41.689 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:41.689 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:41.689 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:16:41.689 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:14:01.190719+0000", 2023-09-23T21:16:41.689 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999981 2023-09-23T21:16:41.689 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:41.689 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:41.689 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:16:41.690 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:14:01.190727+0000", 2023-09-23T21:16:41.690 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.3510000000000002e-06 2023-09-23T21:16:41.690 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:41.690 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:41.690 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:16:41.690 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:14:01.190746+0000", 2023-09-23T21:16:41.690 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.8822999999999999e-05 2023-09-23T21:16:41.690 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:41.690 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:41.690 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:16:41.690 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:14:01.190755+0000", 2023-09-23T21:16:41.690 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.5850000000000006e-06 2023-09-23T21:16:41.691 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:41.691 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:41.691 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:16:41.691 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:14:01.190830+0000", 2023-09-23T21:16:41.691 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.4629000000000007e-05 2023-09-23T21:16:41.691 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:41.691 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:41.691 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T21:16:41.691 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:14:01.190842+0000", 2023-09-23T21:16:41.691 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1792e-05 2023-09-23T21:16:41.691 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:41.692 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:41.692 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:16:41.692 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:14:01.528920+0000", 2023-09-23T21:16:41.692 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.33807835200000003 2023-09-23T21:16:41.692 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:41.692 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:41.692 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:16:41.692 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:14:01.529041+0000", 2023-09-23T21:16:41.692 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000120777 2023-09-23T21:16:41.692 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:41.692 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:41.692 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 3,4", 2023-09-23T21:16:41.693 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:14:01.529252+0000", 2023-09-23T21:16:41.693 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00021154400000000001 2023-09-23T21:16:41.693 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:41.693 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:41.693 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:16:41.693 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:14:01.534288+0000", 2023-09-23T21:16:41.693 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0050353739999999996 2023-09-23T21:16:41.693 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:41.693 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:41.693 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T21:16:41.693 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:14:01.539573+0000", 2023-09-23T21:16:41.693 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0052854779999999997 2023-09-23T21:16:41.694 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:41.694 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:41.694 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:16:41.694 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:14:01.550038+0000", 2023-09-23T21:16:41.694 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.010464865 2023-09-23T21:16:41.694 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:41.694 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:41.694 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T21:16:41.694 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:14:01.550087+0000", 2023-09-23T21:16:41.694 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.8566000000000002e-05 2023-09-23T21:16:41.694 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:41.694 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:41.695 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T21:16:41.695 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:14:01.550129+0000", 2023-09-23T21:16:41.695 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.2206000000000002e-05 2023-09-23T21:16:41.695 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:16:41.695 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:16:41.695 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:16:41.695 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:41.695 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:41.695 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.46328.0:112309 10.14 10:2f86a8f4:::benchmark_data_smithi026_379838_object32352:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e522)", 2023-09-23T21:16:41.695 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T21:14:01.216557+0000", 2023-09-23T21:16:41.695 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 160.26593301700001, 2023-09-23T21:16:41.695 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.34050878099999998, 2023-09-23T21:16:41.696 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:16:41.696 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:16:41.696 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T21:16:41.696 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.46328", 2023-09-23T21:16:41.696 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/3748861181", 2023-09-23T21:16:41.696 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 112309 2023-09-23T21:16:41.696 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:41.696 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:16:41.696 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:41.696 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:16:41.696 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:14:01.216557+0000", 2023-09-23T21:16:41.696 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:16:41.697 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:41.697 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:41.697 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:16:41.697 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:14:01.216557+0000", 2023-09-23T21:16:41.697 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:16:41.697 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:41.697 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:41.697 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:16:41.697 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:14:01.216555+0000", 2023-09-23T21:16:41.697 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T21:16:41.697 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:41.697 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:41.697 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:16:41.698 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:14:01.216563+0000", 2023-09-23T21:16:41.698 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.3239999999999997e-06 2023-09-23T21:16:41.698 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:41.698 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:41.698 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:16:41.698 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:14:01.216574+0000", 2023-09-23T21:16:41.698 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1693e-05 2023-09-23T21:16:41.698 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:41.698 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:41.698 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:16:41.698 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:14:01.216582+0000", 2023-09-23T21:16:41.698 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.089e-06 2023-09-23T21:16:41.699 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:41.699 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:41.699 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:16:41.699 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:14:01.216633+0000", 2023-09-23T21:16:41.699 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.0727999999999999e-05 2023-09-23T21:16:41.699 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:41.699 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:41.699 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T21:16:41.699 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:14:01.216643+0000", 2023-09-23T21:16:41.699 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0029e-05 2023-09-23T21:16:41.699 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:41.699 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:41.699 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:16:41.700 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:14:01.532419+0000", 2023-09-23T21:16:41.700 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.31577592300000001 2023-09-23T21:16:41.700 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:41.700 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:41.700 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:16:41.700 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:14:01.532568+0000", 2023-09-23T21:16:41.700 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00014903299999999999 2023-09-23T21:16:41.700 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:41.700 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:41.700 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 3,4", 2023-09-23T21:16:41.700 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:14:01.532883+0000", 2023-09-23T21:16:41.700 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00031542799999999998 2023-09-23T21:16:41.700 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:41.701 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:41.701 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:16:41.701 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:14:01.538892+0000", 2023-09-23T21:16:41.701 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.006008457 2023-09-23T21:16:41.701 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:41.701 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:41.701 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T21:16:41.701 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:14:01.539578+0000", 2023-09-23T21:16:41.701 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00068610700000000004 2023-09-23T21:16:41.701 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:41.701 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:41.701 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:16:41.702 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:14:01.556973+0000", 2023-09-23T21:16:41.702 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.017395458999999999 2023-09-23T21:16:41.702 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:41.702 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:41.702 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T21:16:41.702 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:14:01.557023+0000", 2023-09-23T21:16:41.702 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.9175999999999998e-05 2023-09-23T21:16:41.702 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:41.702 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:41.702 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T21:16:41.702 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:14:01.557066+0000", 2023-09-23T21:16:41.702 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.2904999999999997e-05 2023-09-23T21:16:41.703 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:16:41.703 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:16:41.703 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:16:41.703 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:41.703 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:41.703 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.46328.0:112317 10.14 10:2c28a9d8:::benchmark_data_smithi026_379838_object32360:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e522)", 2023-09-23T21:16:41.703 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T21:14:01.228376+0000", 2023-09-23T21:16:41.703 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 160.25411348399999, 2023-09-23T21:16:41.703 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.32879608300000002, 2023-09-23T21:16:41.703 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:16:41.703 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:16:41.703 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T21:16:41.703 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.46328", 2023-09-23T21:16:41.704 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/3748861181", 2023-09-23T21:16:41.704 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 112317 2023-09-23T21:16:41.704 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:41.704 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:16:41.704 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:41.704 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:16:41.704 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:14:01.228376+0000", 2023-09-23T21:16:41.704 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:16:41.704 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:41.704 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:41.704 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:16:41.704 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:14:01.228376+0000", 2023-09-23T21:16:41.704 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:16:41.705 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:41.705 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:41.705 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:16:41.705 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:14:01.228375+0000", 2023-09-23T21:16:41.705 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.999999 2023-09-23T21:16:41.705 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:41.705 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:41.705 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:16:41.705 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:14:01.228381+0000", 2023-09-23T21:16:41.705 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.003e-06 2023-09-23T21:16:41.705 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:41.705 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:41.706 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:16:41.706 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:14:01.228398+0000", 2023-09-23T21:16:41.706 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.7028e-05 2023-09-23T21:16:41.706 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:41.706 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:41.706 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:16:41.706 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:14:01.228408+0000", 2023-09-23T21:16:41.706 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.9639999999999998e-06 2023-09-23T21:16:41.706 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:41.706 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:41.706 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:16:41.706 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:14:01.228486+0000", 2023-09-23T21:16:41.707 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.7732999999999996e-05 2023-09-23T21:16:41.707 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:41.707 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:41.707 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T21:16:41.707 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:14:01.228499+0000", 2023-09-23T21:16:41.707 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.278e-05 2023-09-23T21:16:41.707 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:41.708 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:41.708 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:16:41.708 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:14:01.534371+0000", 2023-09-23T21:16:41.708 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.30587188599999998 2023-09-23T21:16:41.708 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:41.708 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:41.708 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:16:41.708 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:14:01.534466+0000", 2023-09-23T21:16:41.708 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.5261999999999997e-05 2023-09-23T21:16:41.708 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:41.708 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:41.708 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 3,4", 2023-09-23T21:16:41.709 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:14:01.534657+0000", 2023-09-23T21:16:41.709 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00019150300000000001 2023-09-23T21:16:41.709 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:41.709 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:41.709 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T21:16:41.709 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:14:01.539583+0000", 2023-09-23T21:16:41.709 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0049253930000000001 2023-09-23T21:16:41.709 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:41.709 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:41.709 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:16:41.709 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:14:01.546707+0000", 2023-09-23T21:16:41.709 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0071237269999999998 2023-09-23T21:16:41.710 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:41.710 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:41.710 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:16:41.710 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:14:01.557100+0000", 2023-09-23T21:16:41.710 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.010393062999999999 2023-09-23T21:16:41.710 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:41.710 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:41.710 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T21:16:41.710 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:14:01.557136+0000", 2023-09-23T21:16:41.710 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.6751000000000001e-05 2023-09-23T21:16:41.711 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:41.711 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:41.711 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T21:16:41.711 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:14:01.557172+0000", 2023-09-23T21:16:41.711 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.5939000000000003e-05 2023-09-23T21:16:41.711 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:16:41.711 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:16:41.711 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:16:41.711 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:41.711 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:41.711 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.46823.0:72557 11.16 11:6f3bd240:::benchmark_data_smithi026_424533_object72556:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e559)", 2023-09-23T21:16:41.712 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T21:16:31.939805+0000", 2023-09-23T21:16:41.712 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 9.5426847559999999, 2023-09-23T21:16:41.712 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.99811668799999997, 2023-09-23T21:16:41.712 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:16:41.712 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "reached pg", 2023-09-23T21:16:41.712 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T21:16:41.712 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.46823", 2023-09-23T21:16:41.712 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/2490075711", 2023-09-23T21:16:41.712 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 72557 2023-09-23T21:16:41.712 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:41.712 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:16:41.713 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:41.713 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:16:41.713 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:16:31.939805+0000", 2023-09-23T21:16:41.713 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:16:41.713 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:41.713 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:41.713 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:16:41.713 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:16:31.939805+0000", 2023-09-23T21:16:41.713 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:16:41.713 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:41.713 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:41.714 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:16:41.714 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:16:31.939803+0000", 2023-09-23T21:16:41.714 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999976 2023-09-23T21:16:41.714 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:41.714 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:41.714 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:16:41.714 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:16:31.939840+0000", 2023-09-23T21:16:41.714 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.6820999999999999e-05 2023-09-23T21:16:41.714 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:41.714 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:41.714 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:16:41.715 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:16:31.939857+0000", 2023-09-23T21:16:41.715 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.7529000000000001e-05 2023-09-23T21:16:41.715 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:41.715 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:41.715 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:16:41.715 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:16:31.939870+0000", 2023-09-23T21:16:41.715 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.235e-05 2023-09-23T21:16:41.715 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:41.715 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:41.715 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:16:41.715 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:16:32.093395+0000", 2023-09-23T21:16:41.715 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.153524991 2023-09-23T21:16:41.716 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:41.716 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:41.716 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T21:16:41.716 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:16:32.093401+0000", 2023-09-23T21:16:41.716 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.5139999999999998e-06 2023-09-23T21:16:41.716 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:41.716 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:41.716 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:16:41.716 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:16:32.927803+0000", 2023-09-23T21:16:41.716 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.83440154799999999 2023-09-23T21:16:41.716 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:41.716 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:41.717 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for degraded object", 2023-09-23T21:16:41.717 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:16:32.935466+0000", 2023-09-23T21:16:41.717 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0076630220000000002 2023-09-23T21:16:41.717 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:41.717 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:41.717 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:16:41.717 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:16:32.937791+0000", 2023-09-23T21:16:41.717 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0023257590000000002 2023-09-23T21:16:41.717 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:41.717 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:41.717 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T21:16:41.717 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:16:32.937922+0000", 2023-09-23T21:16:41.718 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00013033399999999999 2023-09-23T21:16:41.718 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:16:41.718 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:16:41.718 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:16:41.718 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:41.718 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:41.718 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.46823.0:72558 11.6 11:66c7c1af:::benchmark_data_smithi026_424533_object72557:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e559)", 2023-09-23T21:16:41.718 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T21:16:31.939882+0000", 2023-09-23T21:16:41.718 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 9.5426078679999993, 2023-09-23T21:16:41.718 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.97455942799999995, 2023-09-23T21:16:41.718 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:16:41.718 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "reached pg", 2023-09-23T21:16:41.719 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T21:16:41.719 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.46823", 2023-09-23T21:16:41.719 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/2490075711", 2023-09-23T21:16:41.719 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 72558 2023-09-23T21:16:41.719 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:41.719 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:16:41.719 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:41.719 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:16:41.719 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:16:31.939882+0000", 2023-09-23T21:16:41.719 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:16:41.719 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:41.720 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:41.720 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:16:41.720 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:16:31.939882+0000", 2023-09-23T21:16:41.720 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:16:41.720 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:41.720 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:41.720 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:16:41.720 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:16:31.939881+0000", 2023-09-23T21:16:41.720 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T21:16:41.720 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:41.720 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:41.720 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:16:41.721 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:16:31.939905+0000", 2023-09-23T21:16:41.721 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.3536999999999998e-05 2023-09-23T21:16:41.721 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:41.721 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:41.721 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:16:41.721 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:16:31.939912+0000", 2023-09-23T21:16:41.721 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.8990000000000004e-06 2023-09-23T21:16:41.721 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:41.721 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:41.721 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:16:41.721 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:16:31.939918+0000", 2023-09-23T21:16:41.721 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.7849999999999997e-06 2023-09-23T21:16:41.722 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:41.722 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:41.722 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:16:41.722 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:16:32.093187+0000", 2023-09-23T21:16:41.722 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.15326935899999999 2023-09-23T21:16:41.722 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:41.722 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:41.722 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T21:16:41.722 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:16:32.093227+0000", 2023-09-23T21:16:41.722 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.9799000000000003e-05 2023-09-23T21:16:41.722 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:41.722 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:41.723 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:16:41.723 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:16:32.898396+0000", 2023-09-23T21:16:41.723 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.80516965900000004 2023-09-23T21:16:41.723 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:41.723 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:41.723 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for degraded object", 2023-09-23T21:16:41.723 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:16:32.911610+0000", 2023-09-23T21:16:41.723 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.013213213999999999 2023-09-23T21:16:41.723 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:41.723 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:41.723 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:16:41.723 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:16:32.914334+0000", 2023-09-23T21:16:41.724 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0027240400000000001 2023-09-23T21:16:41.724 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:41.724 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:41.724 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T21:16:41.724 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:16:32.914441+0000", 2023-09-23T21:16:41.724 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000107657 2023-09-23T21:16:41.724 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:16:41.724 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:16:41.724 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:16:41.724 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:41.724 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:41.724 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.46823.0:72574 11.8 11:114bb195:::benchmark_data_smithi026_424533_object72573:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e559)", 2023-09-23T21:16:41.724 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T21:16:32.184679+0000", 2023-09-23T21:16:41.725 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 9.2978108610000003, 2023-09-23T21:16:41.725 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.74385123099999995, 2023-09-23T21:16:41.725 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T21:16:41.725 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:16:41.725 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T21:16:41.725 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.46823", 2023-09-23T21:16:41.725 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/2490075711", 2023-09-23T21:16:41.725 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 72574 2023-09-23T21:16:41.725 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:41.725 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T21:16:41.725 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:41.725 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T21:16:41.726 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:16:32.184679+0000", 2023-09-23T21:16:41.726 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:16:41.726 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:41.726 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:41.726 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T21:16:41.726 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:16:32.184679+0000", 2023-09-23T21:16:41.726 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T21:16:41.726 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:41.726 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:41.726 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T21:16:41.726 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:16:32.184677+0000", 2023-09-23T21:16:41.726 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999976 2023-09-23T21:16:41.726 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:41.727 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:41.727 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T21:16:41.727 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:16:32.184705+0000", 2023-09-23T21:16:41.727 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.8422999999999999e-05 2023-09-23T21:16:41.727 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:41.727 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:41.727 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T21:16:41.727 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:16:32.184726+0000", 2023-09-23T21:16:41.727 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.0746000000000001e-05 2023-09-23T21:16:41.727 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:41.727 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:41.727 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T21:16:41.728 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:16:32.184739+0000", 2023-09-23T21:16:41.728 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.2816000000000001e-05 2023-09-23T21:16:41.728 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:41.728 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:41.728 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:16:41.728 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:16:32.202881+0000", 2023-09-23T21:16:41.728 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.018142967999999999 2023-09-23T21:16:41.728 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:41.728 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:41.728 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T21:16:41.728 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:16:32.202893+0000", 2023-09-23T21:16:41.729 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1411e-05 2023-09-23T21:16:41.729 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:41.729 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:41.729 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T21:16:41.729 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:16:32.898343+0000", 2023-09-23T21:16:41.729 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.69545026300000001 2023-09-23T21:16:41.729 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:41.729 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:41.729 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T21:16:41.729 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:16:32.899214+0000", 2023-09-23T21:16:41.729 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00087053 2023-09-23T21:16:41.729 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:41.729 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:41.730 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 3,5", 2023-09-23T21:16:41.730 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:16:32.899544+0000", 2023-09-23T21:16:41.730 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00033066699999999998 2023-09-23T21:16:41.730 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:41.730 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:41.730 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:16:41.730 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:16:32.913456+0000", 2023-09-23T21:16:41.730 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.013911246 2023-09-23T21:16:41.730 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:41.730 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:41.730 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T21:16:41.730 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:16:32.926051+0000", 2023-09-23T21:16:41.731 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.012595554 2023-09-23T21:16:41.731 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:41.731 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:41.731 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:16:41.731 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:16:32.928422+0000", 2023-09-23T21:16:41.731 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0023706500000000002 2023-09-23T21:16:41.731 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:41.731 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:41.731 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T21:16:41.731 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:16:32.928489+0000", 2023-09-23T21:16:41.731 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.7073e-05 2023-09-23T21:16:41.731 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T21:16:41.732 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T21:16:41.732 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T21:16:41.732 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T21:16:32.928530+0000", 2023-09-23T21:16:41.732 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.1261999999999999e-05 2023-09-23T21:16:41.732 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:16:41.732 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:16:41.732 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:16:41.732 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T21:16:41.732 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:16:41.732 INFO:teuthology.orchestra.run.smithi026.stdout:} 2023-09-23T21:16:41.733 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:16:41.763 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:16:42.000 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:16:42.005 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:16:42.005 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:16:42.106 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:16:42.136 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:16:42.217 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T21:16:42.494 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 70 16 78889 78873 70.3693 0 - 0.0139882 2023-09-23T21:16:42.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:16:42 smithi070 ceph-mon[168282]: pgmap v1429: 129 pgs: 1 active+recovering+undersized+remapped, 8 active+recovery_wait+degraded, 4 active+recovery_wait+undersized+degraded+remapped, 116 active+clean; 4.6 GiB data, 16 GiB used, 431 GiB / 447 GiB avail; 91 MiB/s wr, 1.45k op/s; 21196/227175 objects degraded (9.330%); 3083/227175 objects misplaced (1.357%); 11 MiB/s, 176 objects/s recovering 2023-09-23T21:16:42.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:16:42 smithi070 ceph-mon[168282]: failed to encode map e564 with expected crc 2023-09-23T21:16:42.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:16:42 smithi070 ceph-mon[168282]: failed to encode map e564 with expected crc 2023-09-23T21:16:42.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:16:42 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["7"]}]': finished 2023-09-23T21:16:42.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:16:42 smithi070 ceph-mon[168282]: osdmap e564: 8 total, 8 up, 6 in 2023-09-23T21:16:42.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:16:42 smithi070 ceph-mon[168282]: failed to encode map e564 with expected crc 2023-09-23T21:16:42.881 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:16:42 smithi026 ceph-mon[175151]: pgmap v1429: 129 pgs: 1 active+recovering+undersized+remapped, 8 active+recovery_wait+degraded, 4 active+recovery_wait+undersized+degraded+remapped, 116 active+clean; 4.6 GiB data, 16 GiB used, 431 GiB / 447 GiB avail; 91 MiB/s wr, 1.45k op/s; 21196/227175 objects degraded (9.330%); 3083/227175 objects misplaced (1.357%); 11 MiB/s, 176 objects/s recovering 2023-09-23T21:16:42.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:16:42 smithi026 ceph-mon[175151]: failed to encode map e564 with expected crc 2023-09-23T21:16:42.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:16:42 smithi026 ceph-mon[175151]: failed to encode map e564 with expected crc 2023-09-23T21:16:42.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:16:42 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["7"]}]': finished 2023-09-23T21:16:42.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:16:42 smithi026 ceph-mon[175151]: osdmap e564: 8 total, 8 up, 6 in 2023-09-23T21:16:42.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:16:42 smithi026 ceph-mon[175151]: failed to encode map e564 with expected crc 2023-09-23T21:16:42.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:16:42 smithi026 ceph-mon[179688]: pgmap v1429: 129 pgs: 1 active+recovering+undersized+remapped, 8 active+recovery_wait+degraded, 4 active+recovery_wait+undersized+degraded+remapped, 116 active+clean; 4.6 GiB data, 16 GiB used, 431 GiB / 447 GiB avail; 91 MiB/s wr, 1.45k op/s; 21196/227175 objects degraded (9.330%); 3083/227175 objects misplaced (1.357%); 11 MiB/s, 176 objects/s recovering 2023-09-23T21:16:42.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:16:42 smithi026 ceph-mon[179688]: failed to encode map e564 with expected crc 2023-09-23T21:16:42.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:16:42 smithi026 ceph-mon[179688]: failed to encode map e564 with expected crc 2023-09-23T21:16:42.883 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:16:42 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["7"]}]': finished 2023-09-23T21:16:42.883 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:16:42 smithi026 ceph-mon[179688]: osdmap e564: 8 total, 8 up, 6 in 2023-09-23T21:16:42.883 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:16:42 smithi026 ceph-mon[179688]: failed to encode map e564 with expected crc 2023-09-23T21:16:43.494 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 71 16 78915 78899 69.4017 0.8125 0.00843151 0.014208 2023-09-23T21:16:43.631 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:16:43 smithi026 ceph-mon[175151]: failed to encode map e565 with expected crc 2023-09-23T21:16:43.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:16:43 smithi026 ceph-mon[175151]: osdmap e565: 8 total, 8 up, 6 in 2023-09-23T21:16:43.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:16:43 smithi026 ceph-mon[175151]: failed to encode map e565 with expected crc 2023-09-23T21:16:43.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:16:43 smithi026 ceph-mon[175151]: failed to encode map e565 with expected crc 2023-09-23T21:16:43.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:16:43 smithi026 ceph-mon[175151]: failed to encode map e565 with expected crc 2023-09-23T21:16:43.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:16:43 smithi026 ceph-mon[175151]: failed to encode map e565 with expected crc 2023-09-23T21:16:43.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:16:43 smithi026 ceph-mon[175151]: failed to encode map e565 with expected crc 2023-09-23T21:16:43.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:16:43 smithi026 ceph-mon[179688]: failed to encode map e565 with expected crc 2023-09-23T21:16:43.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:16:43 smithi026 ceph-mon[179688]: osdmap e565: 8 total, 8 up, 6 in 2023-09-23T21:16:43.633 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:16:43 smithi026 ceph-mon[179688]: failed to encode map e565 with expected crc 2023-09-23T21:16:43.633 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:16:43 smithi026 ceph-mon[179688]: failed to encode map e565 with expected crc 2023-09-23T21:16:43.633 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:16:43 smithi026 ceph-mon[179688]: failed to encode map e565 with expected crc 2023-09-23T21:16:43.633 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:16:43 smithi026 ceph-mon[179688]: failed to encode map e565 with expected crc 2023-09-23T21:16:43.633 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:16:43 smithi026 ceph-mon[179688]: failed to encode map e565 with expected crc 2023-09-23T21:16:43.647 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:16:43.664 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:16:43.665 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:16:43.765 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:16:43.804 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:16:43 smithi070 ceph-mon[168282]: failed to encode map e565 with expected crc 2023-09-23T21:16:43.805 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:16:43 smithi070 ceph-mon[168282]: osdmap e565: 8 total, 8 up, 6 in 2023-09-23T21:16:43.805 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:16:43 smithi070 ceph-mon[168282]: failed to encode map e565 with expected crc 2023-09-23T21:16:43.805 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:16:43 smithi070 ceph-mon[168282]: failed to encode map e565 with expected crc 2023-09-23T21:16:43.805 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:16:43 smithi070 ceph-mon[168282]: failed to encode map e565 with expected crc 2023-09-23T21:16:43.805 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:16:43 smithi070 ceph-mon[168282]: failed to encode map e565 with expected crc 2023-09-23T21:16:43.805 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:16:43 smithi070 ceph-mon[168282]: failed to encode map e565 with expected crc 2023-09-23T21:16:43.808 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:16:43.907 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:16:43.912 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:16:43.913 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:16:44.013 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:16:44.055 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:16:44.106 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:16:44.111 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:16:44.112 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:16:44.213 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:16:44.274 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:16:44.494 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 72 16 80011 79995 69.389 68.5 0.00251007 0.0143802 2023-09-23T21:16:44.631 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:16:44 smithi026 ceph-mon[175151]: pgmap v1432: 129 pgs: 5 remapped+peering, 30 peering, 4 active+recovery_wait+degraded, 90 active+clean; 4.6 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 61 MiB/s wr, 981 op/s; 6590/225594 objects degraded (2.921%); 11 MiB/s, 176 objects/s recovering 2023-09-23T21:16:44.631 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:16:44 smithi026 ceph-mon[175151]: failed to encode map e566 with expected crc 2023-09-23T21:16:44.631 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:16:44 smithi026 ceph-mon[175151]: failed to encode map e566 with expected crc 2023-09-23T21:16:44.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:16:44 smithi026 ceph-mon[175151]: failed to encode map e566 with expected crc 2023-09-23T21:16:44.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:16:44 smithi026 ceph-mon[175151]: osdmap e566: 8 total, 8 up, 6 in 2023-09-23T21:16:44.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:16:44 smithi026 ceph-mon[175151]: failed to encode map e566 with expected crc 2023-09-23T21:16:44.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:16:44 smithi026 ceph-mon[179688]: pgmap v1432: 129 pgs: 5 remapped+peering, 30 peering, 4 active+recovery_wait+degraded, 90 active+clean; 4.6 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 61 MiB/s wr, 981 op/s; 6590/225594 objects degraded (2.921%); 11 MiB/s, 176 objects/s recovering 2023-09-23T21:16:44.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:16:44 smithi026 ceph-mon[179688]: failed to encode map e566 with expected crc 2023-09-23T21:16:44.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:16:44 smithi026 ceph-mon[179688]: failed to encode map e566 with expected crc 2023-09-23T21:16:44.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:16:44 smithi026 ceph-mon[179688]: failed to encode map e566 with expected crc 2023-09-23T21:16:44.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:16:44 smithi026 ceph-mon[179688]: osdmap e566: 8 total, 8 up, 6 in 2023-09-23T21:16:44.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:16:44 smithi026 ceph-mon[179688]: failed to encode map e566 with expected crc 2023-09-23T21:16:44.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:16:44 smithi070 ceph-mon[168282]: pgmap v1432: 129 pgs: 5 remapped+peering, 30 peering, 4 active+recovery_wait+degraded, 90 active+clean; 4.6 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 61 MiB/s wr, 981 op/s; 6590/225594 objects degraded (2.921%); 11 MiB/s, 176 objects/s recovering 2023-09-23T21:16:44.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:16:44 smithi070 ceph-mon[168282]: failed to encode map e566 with expected crc 2023-09-23T21:16:44.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:16:44 smithi070 ceph-mon[168282]: failed to encode map e566 with expected crc 2023-09-23T21:16:44.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:16:44 smithi070 ceph-mon[168282]: failed to encode map e566 with expected crc 2023-09-23T21:16:44.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:16:44 smithi070 ceph-mon[168282]: osdmap e566: 8 total, 8 up, 6 in 2023-09-23T21:16:44.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:16:44 smithi070 ceph-mon[168282]: failed to encode map e566 with expected crc 2023-09-23T21:16:45.028 INFO:tasks.thrashosds.thrasher:Added osd 7 2023-09-23T21:16:45.495 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 73 16 80906 80890 69.2048 55.9375 0.00841896 0.014444 2023-09-23T21:16:45.581 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_ops_in_flight 2023-09-23T21:16:45.685 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:16:45.691 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:16:45 smithi026 ceph-mon[179688]: failed to encode map e567 with expected crc 2023-09-23T21:16:45.691 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:16:45 smithi026 ceph-mon[179688]: failed to encode map e567 with expected crc 2023-09-23T21:16:45.692 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:16:45 smithi026 ceph-mon[179688]: osdmap e567: 8 total, 8 up, 6 in 2023-09-23T21:16:45.692 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:16:45 smithi026 ceph-mon[179688]: failed to encode map e567 with expected crc 2023-09-23T21:16:45.692 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:16:45 smithi026 ceph-mon[179688]: failed to encode map e567 with expected crc 2023-09-23T21:16:45.692 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:16:45 smithi026 ceph-mon[179688]: failed to encode map e567 with expected crc 2023-09-23T21:16:45.692 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:16:45 smithi026 ceph-mon[179688]: failed to encode map e567 with expected crc 2023-09-23T21:16:45.692 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:16:45 smithi026 ceph-mon[179688]: failed to encode map e567 with expected crc 2023-09-23T21:16:45.692 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:16:45.693 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:16:45.693 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:16:45 smithi026 ceph-mon[175151]: failed to encode map e567 with expected crc 2023-09-23T21:16:45.693 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:16:45 smithi026 ceph-mon[175151]: failed to encode map e567 with expected crc 2023-09-23T21:16:45.693 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:16:45 smithi026 ceph-mon[175151]: osdmap e567: 8 total, 8 up, 6 in 2023-09-23T21:16:45.693 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:16:45 smithi026 ceph-mon[175151]: failed to encode map e567 with expected crc 2023-09-23T21:16:45.693 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:16:45 smithi026 ceph-mon[175151]: failed to encode map e567 with expected crc 2023-09-23T21:16:45.693 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:16:45 smithi026 ceph-mon[175151]: failed to encode map e567 with expected crc 2023-09-23T21:16:45.694 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:16:45 smithi026 ceph-mon[175151]: failed to encode map e567 with expected crc 2023-09-23T21:16:45.694 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:16:45 smithi026 ceph-mon[175151]: failed to encode map e567 with expected crc 2023-09-23T21:16:45.754 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0: {} 2023-09-23T21:16:45.793 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:16:45.794 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1: {} 2023-09-23T21:16:45.855 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:16:45.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:16:45 smithi070 ceph-mon[168282]: failed to encode map e567 with expected crc 2023-09-23T21:16:45.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:16:45 smithi070 ceph-mon[168282]: failed to encode map e567 with expected crc 2023-09-23T21:16:45.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:16:45 smithi070 ceph-mon[168282]: osdmap e567: 8 total, 8 up, 6 in 2023-09-23T21:16:45.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:16:45 smithi070 ceph-mon[168282]: failed to encode map e567 with expected crc 2023-09-23T21:16:45.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:16:45 smithi070 ceph-mon[168282]: failed to encode map e567 with expected crc 2023-09-23T21:16:45.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:16:45 smithi070 ceph-mon[168282]: failed to encode map e567 with expected crc 2023-09-23T21:16:45.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:16:45 smithi070 ceph-mon[168282]: failed to encode map e567 with expected crc 2023-09-23T21:16:45.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:16:45 smithi070 ceph-mon[168282]: failed to encode map e567 with expected crc 2023-09-23T21:16:45.876 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2: {} 2023-09-23T21:16:45.911 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3: {} 2023-09-23T21:16:45.917 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:16:45.925 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:16:45.925 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:16:45.928 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:16:45.950 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4: {} 2023-09-23T21:16:45.977 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T21:16:46.012 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6: {} 2023-09-23T21:16:46.027 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:16:46.044 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T21:16:46.054 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:16:46.055 INFO:teuthology.orchestra.run.smithi026.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T21:16:46.055 INFO:teuthology.orchestra.run.smithi026.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T21:16:46.055 INFO:teuthology.orchestra.run.smithi026.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T21:16:46.055 INFO:teuthology.orchestra.run.smithi026.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T21:16:46.055 INFO:teuthology.orchestra.run.smithi026.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T21:16:46.055 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T21:16:46.055 INFO:teuthology.orchestra.run.smithi026.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T21:16:46.056 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T21:16:46.101 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:16:46.103 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:16:46.106 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:16:46.206 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:16:46.246 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:16:46.291 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:16:46.295 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:16:46.296 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:16:46.396 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:16:46.437 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:16:46.494 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 74 16 81505 81489 68.7758 37.4375 0.0145417 0.0145354 2023-09-23T21:16:46.509 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T21:16:46.525 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:16:46.525 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:16:46.626 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:16:46.686 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:16:46 smithi026 ceph-mon[179688]: failed to encode map e567 with expected crc 2023-09-23T21:16:46.686 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:16:46 smithi026 ceph-mon[179688]: failed to encode map e567 with expected crc 2023-09-23T21:16:46.686 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:16:46 smithi026 ceph-mon[179688]: pgmap v1435: 129 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 4 active+undersized+remapped, 5 remapped+peering, 30 peering, 1 active+recovery_wait+degraded, 85 active+clean; 4.6 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 3.1 MiB/s wr, 48 op/s; 3314/227763 objects degraded (1.455%); 3.2 MiB/s, 48 objects/s recovering 2023-09-23T21:16:46.686 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:16:46 smithi026 ceph-mon[179688]: failed to encode map e568 with expected crc 2023-09-23T21:16:46.686 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:16:46 smithi026 ceph-mon[179688]: failed to encode map e568 with expected crc 2023-09-23T21:16:46.686 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:16:46 smithi026 ceph-mon[179688]: osdmap e568: 8 total, 8 up, 6 in 2023-09-23T21:16:46.686 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:16:46 smithi026 ceph-mon[179688]: failed to encode map e568 with expected crc 2023-09-23T21:16:46.686 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:16:46 smithi026 ceph-mon[179688]: failed to encode map e568 with expected crc 2023-09-23T21:16:46.687 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:16:46 smithi026 ceph-mon[179688]: failed to encode map e568 with expected crc 2023-09-23T21:16:46.687 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:16:46 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/3345981242' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:16:46.687 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:16:46 smithi026 ceph-mon[179688]: failed to encode map e568 with expected crc 2023-09-23T21:16:46.687 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:16:46 smithi026 ceph-mon[179688]: Health check update: Degraded data redundancy: 3314/227763 objects degraded (1.455%), 5 pgs degraded (PG_DEGRADED) 2023-09-23T21:16:46.687 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:16:46 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:16:46.687 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:16:46 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:16:46.687 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:16:46 smithi026 ceph-mon[179688]: osdmap e569: 8 total, 8 up, 6 in 2023-09-23T21:16:46.687 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:16:46 smithi026 ceph-mon[175151]: failed to encode map e567 with expected crc 2023-09-23T21:16:46.687 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:16:46 smithi026 ceph-mon[175151]: failed to encode map e567 with expected crc 2023-09-23T21:16:46.687 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:16:46 smithi026 ceph-mon[175151]: pgmap v1435: 129 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 4 active+undersized+remapped, 5 remapped+peering, 30 peering, 1 active+recovery_wait+degraded, 85 active+clean; 4.6 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 3.1 MiB/s wr, 48 op/s; 3314/227763 objects degraded (1.455%); 3.2 MiB/s, 48 objects/s recovering 2023-09-23T21:16:46.688 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:16:46 smithi026 ceph-mon[175151]: failed to encode map e568 with expected crc 2023-09-23T21:16:46.688 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:16:46 smithi026 ceph-mon[175151]: failed to encode map e568 with expected crc 2023-09-23T21:16:46.688 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:16:46 smithi026 ceph-mon[175151]: osdmap e568: 8 total, 8 up, 6 in 2023-09-23T21:16:46.688 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:16:46 smithi026 ceph-mon[175151]: failed to encode map e568 with expected crc 2023-09-23T21:16:46.688 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:16:46 smithi026 ceph-mon[175151]: failed to encode map e568 with expected crc 2023-09-23T21:16:46.688 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:16:46 smithi026 ceph-mon[175151]: failed to encode map e568 with expected crc 2023-09-23T21:16:46.688 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:16:46 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/3345981242' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:16:46.688 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:16:46 smithi026 ceph-mon[175151]: failed to encode map e568 with expected crc 2023-09-23T21:16:46.688 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:16:46 smithi026 ceph-mon[175151]: Health check update: Degraded data redundancy: 3314/227763 objects degraded (1.455%), 5 pgs degraded (PG_DEGRADED) 2023-09-23T21:16:46.688 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:16:46 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:16:46.688 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:16:46 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:16:46.688 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:16:46 smithi026 ceph-mon[175151]: osdmap e569: 8 total, 8 up, 6 in 2023-09-23T21:16:46.694 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:16:46.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:16:46 smithi070 ceph-mon[168282]: failed to encode map e567 with expected crc 2023-09-23T21:16:46.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:16:46 smithi070 ceph-mon[168282]: failed to encode map e567 with expected crc 2023-09-23T21:16:46.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:16:46 smithi070 ceph-mon[168282]: pgmap v1435: 129 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 4 active+undersized+remapped, 5 remapped+peering, 30 peering, 1 active+recovery_wait+degraded, 85 active+clean; 4.6 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 3.1 MiB/s wr, 48 op/s; 3314/227763 objects degraded (1.455%); 3.2 MiB/s, 48 objects/s recovering 2023-09-23T21:16:46.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:16:46 smithi070 ceph-mon[168282]: failed to encode map e568 with expected crc 2023-09-23T21:16:46.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:16:46 smithi070 ceph-mon[168282]: failed to encode map e568 with expected crc 2023-09-23T21:16:46.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:16:46 smithi070 ceph-mon[168282]: osdmap e568: 8 total, 8 up, 6 in 2023-09-23T21:16:46.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:16:46 smithi070 ceph-mon[168282]: failed to encode map e568 with expected crc 2023-09-23T21:16:46.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:16:46 smithi070 ceph-mon[168282]: failed to encode map e568 with expected crc 2023-09-23T21:16:46.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:16:46 smithi070 ceph-mon[168282]: failed to encode map e568 with expected crc 2023-09-23T21:16:46.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:16:46 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/3345981242' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:16:46.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:16:46 smithi070 ceph-mon[168282]: failed to encode map e568 with expected crc 2023-09-23T21:16:46.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:16:46 smithi070 ceph-mon[168282]: Health check update: Degraded data redundancy: 3314/227763 objects degraded (1.455%), 5 pgs degraded (PG_DEGRADED) 2023-09-23T21:16:46.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:16:46 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:16:46.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:16:46 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:16:46.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:16:46 smithi070 ceph-mon[168282]: osdmap e569: 8 total, 8 up, 6 in 2023-09-23T21:16:47.495 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 75 16 82329 82313 68.5452 51.5 0.0133122 0.014584 2023-09-23T21:16:47.599 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:16:47 smithi026 ceph-mon[179688]: failed to encode map e569 with expected crc 2023-09-23T21:16:47.599 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:16:47 smithi026 ceph-mon[179688]: failed to encode map e569 with expected crc 2023-09-23T21:16:47.599 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:16:47 smithi026 ceph-mon[179688]: failed to encode map e569 with expected crc 2023-09-23T21:16:47.600 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:16:47 smithi026 ceph-mon[179688]: failed to encode map e569 with expected crc 2023-09-23T21:16:47.600 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:16:47 smithi026 ceph-mon[175151]: failed to encode map e569 with expected crc 2023-09-23T21:16:47.600 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:16:47 smithi026 ceph-mon[175151]: failed to encode map e569 with expected crc 2023-09-23T21:16:47.600 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:16:47 smithi026 ceph-mon[175151]: failed to encode map e569 with expected crc 2023-09-23T21:16:47.600 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:16:47 smithi026 ceph-mon[175151]: failed to encode map e569 with expected crc 2023-09-23T21:16:47.851 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:16:47.859 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:16:47.859 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:16:47.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:16:47 smithi070 ceph-mon[168282]: failed to encode map e569 with expected crc 2023-09-23T21:16:47.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:16:47 smithi070 ceph-mon[168282]: failed to encode map e569 with expected crc 2023-09-23T21:16:47.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:16:47 smithi070 ceph-mon[168282]: failed to encode map e569 with expected crc 2023-09-23T21:16:47.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:16:47 smithi070 ceph-mon[168282]: failed to encode map e569 with expected crc 2023-09-23T21:16:47.957 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:16:47.960 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:16:48.049 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:16:48.136 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T21:16:48.154 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:16:48.155 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:16:48.256 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:16:48.322 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:16:48.409 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:16:48.416 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:16:48.417 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:16:48.494 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 76 16 83370 83354 68.4993 65.0625 0.0140411 0.0145933 2023-09-23T21:16:48.518 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:16:48.617 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:16:48.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:16:48 smithi070 ceph-mon[168282]: pgmap v1438: 129 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 10 active+recovery_wait+undersized+degraded+remapped, 4 active+undersized+remapped, 2 remapped+peering, 14 peering, 3 active+recovery_wait+degraded, 94 active+clean; 4.9 GiB data, 19 GiB used, 518 GiB / 536 GiB avail; 89 MiB/s wr, 1.42k op/s; 9615/242634 objects degraded (3.963%); 1427/242634 objects misplaced (0.588%); 2.2 MiB/s, 34 objects/s recovering 2023-09-23T21:16:48.885 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:16:48 smithi026 ceph-mon[179688]: pgmap v1438: 129 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 10 active+recovery_wait+undersized+degraded+remapped, 4 active+undersized+remapped, 2 remapped+peering, 14 peering, 3 active+recovery_wait+degraded, 94 active+clean; 4.9 GiB data, 19 GiB used, 518 GiB / 536 GiB avail; 89 MiB/s wr, 1.42k op/s; 9615/242634 objects degraded (3.963%); 1427/242634 objects misplaced (0.588%); 2.2 MiB/s, 34 objects/s recovering 2023-09-23T21:16:48.887 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:16:48 smithi026 ceph-mon[175151]: pgmap v1438: 129 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 10 active+recovery_wait+undersized+degraded+remapped, 4 active+undersized+remapped, 2 remapped+peering, 14 peering, 3 active+recovery_wait+degraded, 94 active+clean; 4.9 GiB data, 19 GiB used, 518 GiB / 536 GiB avail; 89 MiB/s wr, 1.42k op/s; 9615/242634 objects degraded (3.963%); 1427/242634 objects misplaced (0.588%); 2.2 MiB/s, 34 objects/s recovering 2023-09-23T21:16:49.340 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:16:49.344 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:16:49.345 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:16:49.446 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:16:49.477 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:16:49.478 INFO:teuthology.orchestra.run.smithi026.stderr:nodeep-scrub is set 2023-09-23T21:16:49.494 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 77 16 84368 84352 68.4197 62.375 0.0111777 0.0146089 2023-09-23T21:16:49.518 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:16:49.522 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:16:49.522 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:16:49.613 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:16:49 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/3677058596' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:16:49.613 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:16:49 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:16:49.618 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:16:49 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/3677058596' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:16:49.618 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:16:49 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:16:49.623 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:16:49.660 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:16:49.682 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T21:16:49.692 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:16:49.694 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:16:49.695 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:16:49.795 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:16:49.818 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:16:49 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/3677058596' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:16:49.819 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:16:49 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:16:49.821 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:16:49.852 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:16:49.855 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:16:49.855 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:16:49.881 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 21:16:49 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[164029]: ::ffff:172.21.15.70 - - [23/Sep/2023:21:16:49] "GET /metrics HTTP/1.1" 200 36372 "" "Prometheus/2.43.0" 2023-09-23T21:16:49.956 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:16:49.984 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:16:50.029 INFO:tasks.thrashosds.thrasher:in_osds: [0, 2, 3, 4, 5, 7] out_osds: [6, 1] dead_osds: [] live_osds: [2, 3, 4, 5, 7, 6, 1, 0] 2023-09-23T21:16:50.029 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T21:16:50.030 INFO:tasks.thrashosds.thrasher:inject_pause on osd.4 2023-09-23T21:16:50.030 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-09-23T21:16:50.030 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-09-23T21:16:50.030 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 config set filestore_inject_stall 3 2023-09-23T21:16:50.135 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:16:50.139 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:16:50.139 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:16:50.240 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:16:50.316 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:16:50.498 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 78 16 85091 85075 68.1202 45.1875 0.00916683 0.0146745 2023-09-23T21:16:50.499 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:16:50.504 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:16:50.505 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:16:50.589 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:16:50 smithi026 ceph-mon[175151]: pgmap v1439: 129 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 12 active+recovery_wait+undersized+degraded+remapped, 4 active+undersized+remapped, 4 peering, 6 active+recovery_wait+degraded, 101 active+clean; 4.9 GiB data, 19 GiB used, 518 GiB / 536 GiB avail; 64 MiB/s wr, 1.02k op/s; 10435/243279 objects degraded (4.289%); 1427/243279 objects misplaced (0.587%); 11 MiB/s, 3 keys/s, 171 objects/s recovering 2023-09-23T21:16:50.589 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:16:50 smithi026 ceph-mon[175151]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:16:50.589 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:16:50 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T21:16:50.589 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:16:50 smithi026 ceph-mon[175151]: osdmap e570: 8 total, 8 up, 6 in 2023-09-23T21:16:50.589 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:16:50 smithi026 ceph-mon[175151]: failed to encode map e570 with expected crc 2023-09-23T21:16:50.589 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:16:50 smithi026 ceph-mon[175151]: failed to encode map e570 with expected crc 2023-09-23T21:16:50.590 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:16:50 smithi026 ceph-mon[179688]: pgmap v1439: 129 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 12 active+recovery_wait+undersized+degraded+remapped, 4 active+undersized+remapped, 4 peering, 6 active+recovery_wait+degraded, 101 active+clean; 4.9 GiB data, 19 GiB used, 518 GiB / 536 GiB avail; 64 MiB/s wr, 1.02k op/s; 10435/243279 objects degraded (4.289%); 1427/243279 objects misplaced (0.587%); 11 MiB/s, 3 keys/s, 171 objects/s recovering 2023-09-23T21:16:50.590 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:16:50 smithi026 ceph-mon[179688]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:16:50.590 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:16:50 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T21:16:50.590 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:16:50 smithi026 ceph-mon[179688]: osdmap e570: 8 total, 8 up, 6 in 2023-09-23T21:16:50.590 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:16:50 smithi026 ceph-mon[179688]: failed to encode map e570 with expected crc 2023-09-23T21:16:50.590 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:16:50 smithi026 ceph-mon[179688]: failed to encode map e570 with expected crc 2023-09-23T21:16:50.606 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:16:50.645 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:16:50.692 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:16:50.696 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:16:50.696 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:16:50.750 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T21:16:50.797 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:16:50.821 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:16:50.838 INFO:teuthology.orchestra.run.smithi026.stdout:ERROR: (22) Invalid argument 2023-09-23T21:16:50.838 INFO:teuthology.orchestra.run.smithi026.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T21:16:50.838 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:16:50 smithi070 ceph-mon[168282]: pgmap v1439: 129 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 12 active+recovery_wait+undersized+degraded+remapped, 4 active+undersized+remapped, 4 peering, 6 active+recovery_wait+degraded, 101 active+clean; 4.9 GiB data, 19 GiB used, 518 GiB / 536 GiB avail; 64 MiB/s wr, 1.02k op/s; 10435/243279 objects degraded (4.289%); 1427/243279 objects misplaced (0.587%); 11 MiB/s, 3 keys/s, 171 objects/s recovering 2023-09-23T21:16:50.839 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:16:50 smithi070 ceph-mon[168282]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:16:50.839 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:16:50 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T21:16:50.839 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:16:50 smithi070 ceph-mon[168282]: osdmap e570: 8 total, 8 up, 6 in 2023-09-23T21:16:50.839 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:16:50 smithi070 ceph-mon[168282]: failed to encode map e570 with expected crc 2023-09-23T21:16:50.839 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:16:50 smithi070 ceph-mon[168282]: failed to encode map e570 with expected crc 2023-09-23T21:16:50.841 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:16:50.891 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:16:50.896 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:16:50.897 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:16:50.998 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:16:51.045 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:16:51.098 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:16:51.102 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:16:51.103 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:16:51.203 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:16:51.247 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:16:51.497 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 79 16 86208 86192 68.1415 69.8125 0.0154599 0.0146704 2023-09-23T21:16:51.518 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_blocked_ops 2023-09-23T21:16:51.546 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:16:51.553 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:16:51.553 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:16:51.654 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:16:51.761 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:16:51 smithi026 ceph-mon[175151]: failed to encode map e570 with expected crc 2023-09-23T21:16:51.761 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:16:51 smithi026 ceph-mon[175151]: Health check update: Degraded data redundancy: 10435/243279 objects degraded (4.289%), 18 pgs degraded (PG_DEGRADED) 2023-09-23T21:16:51.761 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:16:51 smithi026 ceph-mon[179688]: failed to encode map e570 with expected crc 2023-09-23T21:16:51.761 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:16:51 smithi026 ceph-mon[179688]: Health check update: Degraded data redundancy: 10435/243279 objects degraded (4.289%), 18 pgs degraded (PG_DEGRADED) 2023-09-23T21:16:51.769 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:16:51.856 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:16:51.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:16:51 smithi070 ceph-mon[168282]: failed to encode map e570 with expected crc 2023-09-23T21:16:51.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:16:51 smithi070 ceph-mon[168282]: Health check update: Degraded data redundancy: 10435/243279 objects degraded (4.289%), 18 pgs degraded (PG_DEGRADED) 2023-09-23T21:16:51.870 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:16:51.871 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:16:51.972 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:16:52.071 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:16:52.195 INFO:teuthology.orchestra.run.smithi070.stdout:{ 2023-09-23T21:16:52.195 INFO:teuthology.orchestra.run.smithi070.stdout: "success": "" 2023-09-23T21:16:52.195 INFO:teuthology.orchestra.run.smithi070.stdout:} 2023-09-23T21:16:52.254 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:16:52.259 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:16:52.259 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:16:52.360 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:16:52.407 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:16:52.497 INFO:tasks.radosbench.radosbench.0.smithi026.stdout:2023-09-23T21:16:52.498059+0000 min lat: 0.00111295 max lat: 2.31793 avg lat: 0.0146809 2023-09-23T21:16:52.497 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T21:16:52.497 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 80 16 87231 87215 68.0889 63.9375 0.0131639 0.0146809 2023-09-23T21:16:52.527 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:16:52.549 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:16:52.553 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:16:52.553 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:16:52.592 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T21:16:52.637 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0: {} 2023-09-23T21:16:52.654 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:16:52.691 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:16:52.695 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1: {} 2023-09-23T21:16:52.730 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:16:52.735 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2: {} 2023-09-23T21:16:52.735 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:16:52 smithi070 ceph-mon[168282]: pgmap v1441: 129 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 12 active+recovery_wait+undersized+degraded+remapped, 6 active+recovery_wait+degraded, 109 active+clean; 5.0 GiB data, 19 GiB used, 517 GiB / 536 GiB avail; 70 MiB/s wr, 1.13k op/s; 10435/248058 objects degraded (4.207%); 1427/248058 objects misplaced (0.575%); 10 MiB/s, 3 keys/s, 163 objects/s recovering 2023-09-23T21:16:52.736 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:16:52.736 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:16:52.776 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3: {} 2023-09-23T21:16:52.830 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4: {} 2023-09-23T21:16:52.836 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:16:52.862 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T21:16:52.889 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:16:52 smithi026 ceph-mon[175151]: pgmap v1441: 129 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 12 active+recovery_wait+undersized+degraded+remapped, 6 active+recovery_wait+degraded, 109 active+clean; 5.0 GiB data, 19 GiB used, 517 GiB / 536 GiB avail; 70 MiB/s wr, 1.13k op/s; 10435/248058 objects degraded (4.207%); 1427/248058 objects misplaced (0.575%); 10 MiB/s, 3 keys/s, 163 objects/s recovering 2023-09-23T21:16:52.890 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:16:52 smithi026 ceph-mon[179688]: pgmap v1441: 129 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 12 active+recovery_wait+undersized+degraded+remapped, 6 active+recovery_wait+degraded, 109 active+clean; 5.0 GiB data, 19 GiB used, 517 GiB / 536 GiB avail; 70 MiB/s wr, 1.13k op/s; 10435/248058 objects degraded (4.207%); 1427/248058 objects misplaced (0.575%); 10 MiB/s, 3 keys/s, 163 objects/s recovering 2023-09-23T21:16:52.905 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6: {} 2023-09-23T21:16:52.914 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:16:52.964 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T21:16:52.964 INFO:teuthology.orchestra.run.smithi026.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T21:16:52.964 INFO:teuthology.orchestra.run.smithi026.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T21:16:52.964 INFO:teuthology.orchestra.run.smithi026.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T21:16:52.964 INFO:teuthology.orchestra.run.smithi026.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T21:16:52.964 INFO:teuthology.orchestra.run.smithi026.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T21:16:52.964 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T21:16:52.964 INFO:teuthology.orchestra.run.smithi026.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T21:16:52.964 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T21:16:52.984 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:16:52.989 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:16:52.989 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:16:53.090 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:16:53.180 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:16:53.488 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:16:53 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/208542006' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:16:53.497 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 81 16 88271 88255 68.0505 65 0.0201374 0.0146899 2023-09-23T21:16:53.815 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:16:53 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/208542006' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:16:53.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:16:53 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/208542006' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:16:54.342 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:16:54.349 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:16:54.349 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:16:54.450 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:16:54.497 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 82 16 89437 89421 68.1092 72.875 0.0172006 0.0146778 2023-09-23T21:16:54.508 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:16:54 smithi026 ceph-mon[175151]: pgmap v1442: 129 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 12 active+recovery_wait+undersized+degraded+remapped, 6 active+recovery_wait+degraded, 109 active+clean; 5.2 GiB data, 20 GiB used, 517 GiB / 536 GiB avail; 76 MiB/s wr, 1.21k op/s; 10435/256941 objects degraded (4.061%); 1038/256941 objects misplaced (0.404%); 11 MiB/s, 2 keys/s, 176 objects/s recovering 2023-09-23T21:16:54.508 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:16:54 smithi026 ceph-mon[179688]: pgmap v1442: 129 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 12 active+recovery_wait+undersized+degraded+remapped, 6 active+recovery_wait+degraded, 109 active+clean; 5.2 GiB data, 20 GiB used, 517 GiB / 536 GiB avail; 76 MiB/s wr, 1.21k op/s; 10435/256941 objects degraded (4.061%); 1038/256941 objects misplaced (0.404%); 11 MiB/s, 2 keys/s, 176 objects/s recovering 2023-09-23T21:16:54.513 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:16:54.699 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:16:54.709 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:16:54.710 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:16:54.710 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:16:54.811 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:16:54.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:16:54 smithi070 ceph-mon[168282]: pgmap v1442: 129 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 12 active+recovery_wait+undersized+degraded+remapped, 6 active+recovery_wait+degraded, 109 active+clean; 5.2 GiB data, 20 GiB used, 517 GiB / 536 GiB avail; 76 MiB/s wr, 1.21k op/s; 10435/256941 objects degraded (4.061%); 1038/256941 objects misplaced (0.404%); 11 MiB/s, 2 keys/s, 176 objects/s recovering 2023-09-23T21:16:54.880 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:16:55.175 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:16:55.185 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:16:55.186 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:16:55.287 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:16:55.336 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:16:55.387 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:16:55.393 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:16:55.394 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:16:55.494 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:16:55.498 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 83 16 90495 90479 68.0853 66.125 0.0133177 0.014683 2023-09-23T21:16:55.594 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:16:55.897 INFO:teuthology.orchestra.run.smithi026.stdout:{ 2023-09-23T21:16:55.897 INFO:teuthology.orchestra.run.smithi026.stdout: "ops": [], 2023-09-23T21:16:55.898 INFO:teuthology.orchestra.run.smithi026.stdout: "complaint_time": 30, 2023-09-23T21:16:55.898 INFO:teuthology.orchestra.run.smithi026.stdout: "num_blocked_ops": 0 2023-09-23T21:16:55.898 INFO:teuthology.orchestra.run.smithi026.stdout:} 2023-09-23T21:16:56.281 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T21:16:56.409 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:16:56.429 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:16:56.430 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:16:56.498 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 84 16 91546 91530 68.0567 65.6875 0.00660445 0.0146882 2023-09-23T21:16:56.530 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:16:56.570 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:16:56.616 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:16:56.619 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:16:56.619 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:16:56.631 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:16:56 smithi026 ceph-mon[175151]: pgmap v1443: 129 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 12 active+recovery_wait+undersized+degraded+remapped, 6 active+recovery_wait+degraded, 109 active+clean; 5.3 GiB data, 20 GiB used, 517 GiB / 536 GiB avail; 76 MiB/s wr, 1.22k op/s; 10435/260829 objects degraded (4.001%); 1038/260829 objects misplaced (0.398%); 9.7 MiB/s, 2 keys/s, 156 objects/s recovering 2023-09-23T21:16:56.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:16:56 smithi026 ceph-mon[175151]: Health check update: Degraded data redundancy: 10435/260829 objects degraded (4.001%), 18 pgs degraded (PG_DEGRADED) 2023-09-23T21:16:56.633 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:16:56 smithi026 ceph-mon[179688]: pgmap v1443: 129 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 12 active+recovery_wait+undersized+degraded+remapped, 6 active+recovery_wait+degraded, 109 active+clean; 5.3 GiB data, 20 GiB used, 517 GiB / 536 GiB avail; 76 MiB/s wr, 1.22k op/s; 10435/260829 objects degraded (4.001%); 1038/260829 objects misplaced (0.398%); 9.7 MiB/s, 2 keys/s, 156 objects/s recovering 2023-09-23T21:16:56.634 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:16:56 smithi026 ceph-mon[179688]: Health check update: Degraded data redundancy: 10435/260829 objects degraded (4.001%), 18 pgs degraded (PG_DEGRADED) 2023-09-23T21:16:56.720 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:16:56.790 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:16:56.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:16:56 smithi070 ceph-mon[168282]: pgmap v1443: 129 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 12 active+recovery_wait+undersized+degraded+remapped, 6 active+recovery_wait+degraded, 109 active+clean; 5.3 GiB data, 20 GiB used, 517 GiB / 536 GiB avail; 76 MiB/s wr, 1.22k op/s; 10435/260829 objects degraded (4.001%); 1038/260829 objects misplaced (0.398%); 9.7 MiB/s, 2 keys/s, 156 objects/s recovering 2023-09-23T21:16:56.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:16:56 smithi070 ceph-mon[168282]: Health check update: Degraded data redundancy: 10435/260829 objects degraded (4.001%), 18 pgs degraded (PG_DEGRADED) 2023-09-23T21:16:57.501 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 85 16 92725 92709 68.1201 73.6875 0.0171816 0.0146751 2023-09-23T21:16:57.655 INFO:tasks.thrashosds.thrasher:in_osds: [0, 2, 3, 4, 5, 7] out_osds: [6, 1] dead_osds: [] live_osds: [2, 3, 4, 5, 7, 6, 1, 0] 2023-09-23T21:16:57.655 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T21:16:57.655 INFO:tasks.thrashosds.thrasher:inject_pause on osd.1 2023-09-23T21:16:57.655 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-09-23T21:16:57.656 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-09-23T21:16:57.656 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 config set filestore_inject_stall 3 2023-09-23T21:16:58.502 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 86 16 93922 93906 68.1978 74.8125 0.0183659 0.0146584 2023-09-23T21:16:58.620 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T21:16:58.632 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:16:58.633 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:16:58.676 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_historic_ops 2023-09-23T21:16:58.733 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:16:58.831 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:16:58 smithi026 ceph-mon[175151]: pgmap v1444: 129 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 12 active+recovery_wait+undersized+degraded+remapped, 6 active+recovery_wait+degraded, 109 active+clean; 5.5 GiB data, 20 GiB used, 516 GiB / 536 GiB avail; 61 MiB/s wr, 976 op/s; 10435/271935 objects degraded (3.837%); 614/271935 objects misplaced (0.226%); 10 MiB/s, 1 keys/s, 165 objects/s recovering 2023-09-23T21:16:58.831 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:16:58 smithi026 ceph-mon[179688]: pgmap v1444: 129 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 12 active+recovery_wait+undersized+degraded+remapped, 6 active+recovery_wait+degraded, 109 active+clean; 5.5 GiB data, 20 GiB used, 516 GiB / 536 GiB avail; 61 MiB/s wr, 976 op/s; 10435/271935 objects degraded (3.837%); 614/271935 objects misplaced (0.226%); 10 MiB/s, 1 keys/s, 165 objects/s recovering 2023-09-23T21:16:58.839 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:16:58.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:16:58 smithi070 ceph-mon[168282]: pgmap v1444: 129 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 12 active+recovery_wait+undersized+degraded+remapped, 6 active+recovery_wait+degraded, 109 active+clean; 5.5 GiB data, 20 GiB used, 516 GiB / 536 GiB avail; 61 MiB/s wr, 976 op/s; 10435/271935 objects degraded (3.837%); 614/271935 objects misplaced (0.226%); 10 MiB/s, 1 keys/s, 165 objects/s recovering 2023-09-23T21:16:58.933 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:16:58.937 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:16:58.938 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:16:59.039 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:16:59.108 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:16:59.184 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:16:59.187 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:16:59.188 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:16:59.261 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:16:59.288 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:16:59.324 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:16:59.369 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:16:59.487 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:16:59.500 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:16:59.501 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:16:59.501 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 87 16 94803 94787 68.0468 55.0625 0.0219733 0.0146904 2023-09-23T21:16:59.602 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:16:59.618 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 21:16:59 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[164029]: ::ffff:172.21.15.70 - - [23/Sep/2023:21:16:59] "GET /metrics HTTP/1.1" 200 36372 "" "Prometheus/2.43.0" 2023-09-23T21:16:59.666 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:16:59.808 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:16:59.812 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:16:59.813 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:16:59.914 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:16:59.959 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:17:00.016 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:17:00.020 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:17:00.021 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:17:00.121 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:17:00.199 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:17:00.503 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 88 16 95875 95859 68.0341 67 0.0076561 0.0146943 2023-09-23T21:17:00.521 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:17:00 smithi026 ceph-mon[175151]: pgmap v1445: 129 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 12 active+recovery_wait+undersized+degraded+remapped, 6 active+recovery_wait+degraded, 109 active+clean; 5.6 GiB data, 21 GiB used, 516 GiB / 536 GiB avail; 62 MiB/s wr, 988 op/s; 10435/272955 objects degraded (3.823%); 614/272955 objects misplaced (0.225%); 5.1 MiB/s, 81 objects/s recovering 2023-09-23T21:17:00.522 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:17:00 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/3795576824' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T21:17:00.522 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:17:00 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T21:17:00.522 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:17:00 smithi026 ceph-mon[179688]: pgmap v1445: 129 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 12 active+recovery_wait+undersized+degraded+remapped, 6 active+recovery_wait+degraded, 109 active+clean; 5.6 GiB data, 21 GiB used, 516 GiB / 536 GiB avail; 62 MiB/s wr, 988 op/s; 10435/272955 objects degraded (3.823%); 614/272955 objects misplaced (0.225%); 5.1 MiB/s, 81 objects/s recovering 2023-09-23T21:17:00.522 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:17:00 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/3795576824' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T21:17:00.522 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:17:00 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T21:17:00.555 INFO:teuthology.orchestra.run.smithi026.stderr:noscrub is unset 2023-09-23T21:17:00.864 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:17:00.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:17:00 smithi070 ceph-mon[168282]: pgmap v1445: 129 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 12 active+recovery_wait+undersized+degraded+remapped, 6 active+recovery_wait+degraded, 109 active+clean; 5.6 GiB data, 21 GiB used, 516 GiB / 536 GiB avail; 62 MiB/s wr, 988 op/s; 10435/272955 objects degraded (3.823%); 614/272955 objects misplaced (0.225%); 5.1 MiB/s, 81 objects/s recovering 2023-09-23T21:17:00.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:17:00 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/3795576824' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T21:17:00.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:17:00 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T21:17:00.885 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:17:00.885 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:17:00.986 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:17:01.059 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:17:01.200 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T21:17:01.204 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:17:01.204 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:17:01.305 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:17:01.359 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:17:01.478 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:17:01.481 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:17:01.481 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:17:01.504 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 89 16 96976 96960 68.0425 68.8125 0.0119133 0.014692 2023-09-23T21:17:01.581 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:17:01.611 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:17:01 smithi070 ceph-mon[168282]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:17:01.611 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:17:01 smithi070 ceph-mon[168282]: failed to encode map e571 with expected crc 2023-09-23T21:17:01.611 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:17:01 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T21:17:01.611 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:17:01 smithi070 ceph-mon[168282]: osdmap e571: 8 total, 8 up, 6 in 2023-09-23T21:17:01.611 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:17:01 smithi070 ceph-mon[168282]: failed to encode map e571 with expected crc 2023-09-23T21:17:01.611 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:17:01 smithi070 ceph-mon[168282]: failed to encode map e571 with expected crc 2023-09-23T21:17:01.611 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:17:01 smithi070 ceph-mon[168282]: failed to encode map e571 with expected crc 2023-09-23T21:17:01.611 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:17:01 smithi070 ceph-mon[168282]: failed to encode map e571 with expected crc 2023-09-23T21:17:01.612 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:17:01 smithi070 ceph-mon[168282]: Health check update: Degraded data redundancy: 10435/272955 objects degraded (3.823%), 18 pgs degraded (PG_DEGRADED) 2023-09-23T21:17:01.612 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:17:01 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:17:01.617 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:17:01.673 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:17:01.679 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:17:01.679 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:17:01.780 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:17:01.845 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:17:01.855 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:17:01 smithi026 ceph-mon[179688]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:17:01.855 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:17:01 smithi026 ceph-mon[179688]: failed to encode map e571 with expected crc 2023-09-23T21:17:01.855 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:17:01 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T21:17:01.855 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:17:01 smithi026 ceph-mon[179688]: osdmap e571: 8 total, 8 up, 6 in 2023-09-23T21:17:01.855 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:17:01 smithi026 ceph-mon[179688]: failed to encode map e571 with expected crc 2023-09-23T21:17:01.855 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:17:01 smithi026 ceph-mon[179688]: failed to encode map e571 with expected crc 2023-09-23T21:17:01.855 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:17:01 smithi026 ceph-mon[179688]: failed to encode map e571 with expected crc 2023-09-23T21:17:01.855 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:17:01 smithi026 ceph-mon[179688]: failed to encode map e571 with expected crc 2023-09-23T21:17:01.856 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:17:01 smithi026 ceph-mon[179688]: Health check update: Degraded data redundancy: 10435/272955 objects degraded (3.823%), 18 pgs degraded (PG_DEGRADED) 2023-09-23T21:17:01.856 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:17:01 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:17:01.856 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:17:01 smithi026 ceph-mon[175151]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:17:01.856 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:17:01 smithi026 ceph-mon[175151]: failed to encode map e571 with expected crc 2023-09-23T21:17:01.856 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:17:01 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T21:17:01.856 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:17:01 smithi026 ceph-mon[175151]: osdmap e571: 8 total, 8 up, 6 in 2023-09-23T21:17:01.856 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:17:01 smithi026 ceph-mon[175151]: failed to encode map e571 with expected crc 2023-09-23T21:17:01.856 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:17:01 smithi026 ceph-mon[175151]: failed to encode map e571 with expected crc 2023-09-23T21:17:01.856 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:17:01 smithi026 ceph-mon[175151]: failed to encode map e571 with expected crc 2023-09-23T21:17:01.856 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:17:01 smithi026 ceph-mon[175151]: failed to encode map e571 with expected crc 2023-09-23T21:17:01.857 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:17:01 smithi026 ceph-mon[175151]: Health check update: Degraded data redundancy: 10435/272955 objects degraded (3.823%), 18 pgs degraded (PG_DEGRADED) 2023-09-23T21:17:01.857 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:17:01 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:17:01.857 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:17:01.901 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:17:01.906 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:17:01.907 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:17:02.008 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:17:02.058 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:17:02.109 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:17:02.111 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:17:02.112 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:17:02.213 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:17:02.252 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:17:02.291 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:17:02.294 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:17:02.294 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:17:02.395 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:17:02.466 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:17:02.503 INFO:tasks.radosbench.radosbench.0.smithi026.stdout:Total time run: 90.011 2023-09-23T21:17:02.503 INFO:tasks.radosbench.radosbench.0.smithi026.stdout:Total writes made: 97967 2023-09-23T21:17:02.503 INFO:tasks.radosbench.radosbench.0.smithi026.stdout:Write size: 65536 2023-09-23T21:17:02.503 INFO:tasks.radosbench.radosbench.0.smithi026.stdout:Object size: 65536 2023-09-23T21:17:02.503 INFO:tasks.radosbench.radosbench.0.smithi026.stdout:Bandwidth (MB/sec): 68.0243 2023-09-23T21:17:02.504 INFO:tasks.radosbench.radosbench.0.smithi026.stdout:Stddev Bandwidth: 38.9023 2023-09-23T21:17:02.504 INFO:tasks.radosbench.radosbench.0.smithi026.stdout:Max bandwidth (MB/sec): 177.438 2023-09-23T21:17:02.504 INFO:tasks.radosbench.radosbench.0.smithi026.stdout:Min bandwidth (MB/sec): 0 2023-09-23T21:17:02.504 INFO:tasks.radosbench.radosbench.0.smithi026.stdout:Average IOPS: 1088 2023-09-23T21:17:02.504 INFO:tasks.radosbench.radosbench.0.smithi026.stdout:Stddev IOPS: 622.437 2023-09-23T21:17:02.504 INFO:tasks.radosbench.radosbench.0.smithi026.stdout:Max IOPS: 2839 2023-09-23T21:17:02.504 INFO:tasks.radosbench.radosbench.0.smithi026.stdout:Min IOPS: 0 2023-09-23T21:17:02.504 INFO:tasks.radosbench.radosbench.0.smithi026.stdout:Average Latency(s): 0.0146964 2023-09-23T21:17:02.504 INFO:tasks.radosbench.radosbench.0.smithi026.stdout:Stddev Latency(s): 0.0465605 2023-09-23T21:17:02.504 INFO:tasks.radosbench.radosbench.0.smithi026.stdout:Max latency(s): 2.31793 2023-09-23T21:17:02.504 INFO:tasks.radosbench.radosbench.0.smithi026.stdout:Min latency(s): 0.00111295 2023-09-23T21:17:02.508 INFO:tasks.radosbench.radosbench.0.smithi026.stdout:Cleaning up (deleting benchmark objects) 2023-09-23T21:17:02.710 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:17:02.717 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:17:02 smithi026 ceph-mon[179688]: pgmap v1447: 129 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 12 active+recovery_wait+undersized+degraded+remapped, 6 active+recovery_wait+degraded, 109 active+clean; 5.7 GiB data, 21 GiB used, 515 GiB / 536 GiB avail; 64 MiB/s wr, 1.03k op/s; 10435/278892 objects degraded (3.742%); 614/278892 objects misplaced (0.220%); 5.1 MiB/s, 81 objects/s recovering 2023-09-23T21:17:02.717 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:17:02.718 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:17:02.720 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:17:02 smithi026 ceph-mon[175151]: pgmap v1447: 129 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 12 active+recovery_wait+undersized+degraded+remapped, 6 active+recovery_wait+degraded, 109 active+clean; 5.7 GiB data, 21 GiB used, 515 GiB / 536 GiB avail; 64 MiB/s wr, 1.03k op/s; 10435/278892 objects degraded (3.742%); 614/278892 objects misplaced (0.220%); 5.1 MiB/s, 81 objects/s recovering 2023-09-23T21:17:02.819 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:17:02.848 INFO:teuthology.orchestra.run.smithi026.stdout:{ 2023-09-23T21:17:02.848 INFO:teuthology.orchestra.run.smithi026.stdout: "success": "filestore_inject_stall = '3' (not observed, change may require restart) " 2023-09-23T21:17:02.848 INFO:teuthology.orchestra.run.smithi026.stdout:} 2023-09-23T21:17:02.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:17:02 smithi070 ceph-mon[168282]: pgmap v1447: 129 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 12 active+recovery_wait+undersized+degraded+remapped, 6 active+recovery_wait+degraded, 109 active+clean; 5.7 GiB data, 21 GiB used, 515 GiB / 536 GiB avail; 64 MiB/s wr, 1.03k op/s; 10435/278892 objects degraded (3.742%); 614/278892 objects misplaced (0.220%); 5.1 MiB/s, 81 objects/s recovering 2023-09-23T21:17:02.890 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:17:03.392 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:17:03.395 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:17:03.395 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:17:03.496 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:17:03.523 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:17:03.555 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:17:03.557 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:17:03.558 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:17:03.571 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T21:17:03.658 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:17:03.689 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:17:03.773 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0: {} 2023-09-23T21:17:03.778 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:17:03.783 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:17:03.783 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:17:03.796 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1: {} 2023-09-23T21:17:03.818 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2: {} 2023-09-23T21:17:03.840 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3: {} 2023-09-23T21:17:03.860 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4: {} 2023-09-23T21:17:03.884 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:17:03.894 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T21:17:03.913 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:17:03.927 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6: {} 2023-09-23T21:17:03.961 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:17:03.967 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:17:03.968 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:17:03.969 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T21:17:03.975 INFO:teuthology.orchestra.run.smithi026.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T21:17:03.975 INFO:teuthology.orchestra.run.smithi026.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T21:17:03.975 INFO:teuthology.orchestra.run.smithi026.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T21:17:03.975 INFO:teuthology.orchestra.run.smithi026.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T21:17:03.975 INFO:teuthology.orchestra.run.smithi026.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T21:17:03.975 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T21:17:03.975 INFO:teuthology.orchestra.run.smithi026.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T21:17:03.976 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T21:17:04.005 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:17:03 smithi026 ceph-mon[179688]: failed to encode map e572 with expected crc 2023-09-23T21:17:04.005 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:17:03 smithi026 ceph-mon[179688]: failed to encode map e572 with expected crc 2023-09-23T21:17:04.005 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:17:03 smithi026 ceph-mon[179688]: osdmap e572: 8 total, 8 up, 6 in 2023-09-23T21:17:04.005 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:17:03 smithi026 ceph-mon[179688]: failed to encode map e572 with expected crc 2023-09-23T21:17:04.005 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:17:03 smithi026 ceph-mon[179688]: pgmap v1449: 129 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 12 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 112 active+clean; 5.9 GiB data, 22 GiB used, 515 GiB / 536 GiB avail; 74 MiB/s wr, 1.18k op/s; 10432/289185 objects degraded (3.607%); 78/289185 objects misplaced (0.027%); 7.5 MiB/s, 120 objects/s recovering 2023-09-23T21:17:04.005 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:17:03 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/2201487612' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:17:04.006 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:17:03 smithi026 ceph-mon[175151]: failed to encode map e572 with expected crc 2023-09-23T21:17:04.006 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:17:03 smithi026 ceph-mon[175151]: failed to encode map e572 with expected crc 2023-09-23T21:17:04.006 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:17:03 smithi026 ceph-mon[175151]: osdmap e572: 8 total, 8 up, 6 in 2023-09-23T21:17:04.006 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:17:03 smithi026 ceph-mon[175151]: failed to encode map e572 with expected crc 2023-09-23T21:17:04.006 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:17:03 smithi026 ceph-mon[175151]: pgmap v1449: 129 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 12 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 112 active+clean; 5.9 GiB data, 22 GiB used, 515 GiB / 536 GiB avail; 74 MiB/s wr, 1.18k op/s; 10432/289185 objects degraded (3.607%); 78/289185 objects misplaced (0.027%); 7.5 MiB/s, 120 objects/s recovering 2023-09-23T21:17:04.006 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:17:03 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/2201487612' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:17:04.069 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:17:04.097 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:17:04.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:17:03 smithi070 ceph-mon[168282]: failed to encode map e572 with expected crc 2023-09-23T21:17:04.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:17:03 smithi070 ceph-mon[168282]: failed to encode map e572 with expected crc 2023-09-23T21:17:04.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:17:03 smithi070 ceph-mon[168282]: osdmap e572: 8 total, 8 up, 6 in 2023-09-23T21:17:04.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:17:03 smithi070 ceph-mon[168282]: failed to encode map e572 with expected crc 2023-09-23T21:17:04.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:17:03 smithi070 ceph-mon[168282]: pgmap v1449: 129 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 12 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 112 active+clean; 5.9 GiB data, 22 GiB used, 515 GiB / 536 GiB avail; 74 MiB/s wr, 1.18k op/s; 10432/289185 objects degraded (3.607%); 78/289185 objects misplaced (0.027%); 7.5 MiB/s, 120 objects/s recovering 2023-09-23T21:17:04.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:17:03 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/2201487612' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:17:04.137 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:17:04.142 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:17:04.142 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:17:04.243 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:17:04.321 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:17:05.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:17:04 smithi026 ceph-mon[175151]: failed to encode map e573 with expected crc 2023-09-23T21:17:05.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:17:04 smithi026 ceph-mon[175151]: failed to encode map e573 with expected crc 2023-09-23T21:17:05.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:17:04 smithi026 ceph-mon[175151]: failed to encode map e573 with expected crc 2023-09-23T21:17:05.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:17:04 smithi026 ceph-mon[175151]: osdmap e573: 8 total, 8 up, 6 in 2023-09-23T21:17:05.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:17:04 smithi026 ceph-mon[175151]: failed to encode map e573 with expected crc 2023-09-23T21:17:05.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:17:04 smithi026 ceph-mon[179688]: failed to encode map e573 with expected crc 2023-09-23T21:17:05.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:17:04 smithi026 ceph-mon[179688]: failed to encode map e573 with expected crc 2023-09-23T21:17:05.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:17:04 smithi026 ceph-mon[179688]: failed to encode map e573 with expected crc 2023-09-23T21:17:05.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:17:04 smithi026 ceph-mon[179688]: osdmap e573: 8 total, 8 up, 6 in 2023-09-23T21:17:05.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:17:04 smithi026 ceph-mon[179688]: failed to encode map e573 with expected crc 2023-09-23T21:17:05.371 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:17:04 smithi070 ceph-mon[168282]: failed to encode map e573 with expected crc 2023-09-23T21:17:05.371 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:17:04 smithi070 ceph-mon[168282]: failed to encode map e573 with expected crc 2023-09-23T21:17:05.371 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:17:04 smithi070 ceph-mon[168282]: failed to encode map e573 with expected crc 2023-09-23T21:17:05.371 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:17:04 smithi070 ceph-mon[168282]: osdmap e573: 8 total, 8 up, 6 in 2023-09-23T21:17:05.371 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:17:04 smithi070 ceph-mon[168282]: failed to encode map e573 with expected crc 2023-09-23T21:17:05.703 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T21:17:05.713 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:17:05.714 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:17:05.814 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:17:05.850 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:17:05.897 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:17:05.899 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:17:05.900 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:17:05.920 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:17:06.001 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:17:06.072 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:17:06.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:17:05 smithi070 ceph-mon[168282]: pgmap v1451: 129 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 12 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 113 active+clean; 5.9 GiB data, 22 GiB used, 515 GiB / 536 GiB avail; 65 MiB/s wr, 1.08k op/s; 10429/290748 objects degraded (3.587%); 78/290748 objects misplaced (0.027%); 5.6 MiB/s, 90 objects/s recovering 2023-09-23T21:17:06.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:17:05 smithi070 ceph-mon[168282]: Health check update: Degraded data redundancy: 10432/289185 objects degraded (3.607%), 15 pgs degraded (PG_DEGRADED) 2023-09-23T21:17:06.132 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:17:06.140 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:17:06.141 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:17:06.141 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:17:05 smithi026 ceph-mon[179688]: pgmap v1451: 129 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 12 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 113 active+clean; 5.9 GiB data, 22 GiB used, 515 GiB / 536 GiB avail; 65 MiB/s wr, 1.08k op/s; 10429/290748 objects degraded (3.587%); 78/290748 objects misplaced (0.027%); 5.6 MiB/s, 90 objects/s recovering 2023-09-23T21:17:06.141 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:17:05 smithi026 ceph-mon[179688]: Health check update: Degraded data redundancy: 10432/289185 objects degraded (3.607%), 15 pgs degraded (PG_DEGRADED) 2023-09-23T21:17:06.142 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:17:05 smithi026 ceph-mon[175151]: pgmap v1451: 129 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 12 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 113 active+clean; 5.9 GiB data, 22 GiB used, 515 GiB / 536 GiB avail; 65 MiB/s wr, 1.08k op/s; 10429/290748 objects degraded (3.587%); 78/290748 objects misplaced (0.027%); 5.6 MiB/s, 90 objects/s recovering 2023-09-23T21:17:06.142 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:17:05 smithi026 ceph-mon[175151]: Health check update: Degraded data redundancy: 10432/289185 objects degraded (3.607%), 15 pgs degraded (PG_DEGRADED) 2023-09-23T21:17:06.239 INFO:teuthology.orchestra.run.smithi026.stdout:ERROR: (22) Invalid argument 2023-09-23T21:17:06.239 INFO:teuthology.orchestra.run.smithi026.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T21:17:06.242 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:17:06.280 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:17:06.326 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:17:06.330 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:17:06.330 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:17:06.431 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:17:06.487 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:17:07.579 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T21:17:07.863 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_ops_in_flight 2023-09-23T21:17:08.381 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:17:08.386 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:17:08.386 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:17:08.487 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:17:08.575 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:17:08 smithi026 ceph-mon[175151]: pgmap v1452: 129 pgs: 1 active+recovering+undersized+remapped, 11 active+recovery_wait+undersized+degraded+remapped, 117 active+clean; 5.8 GiB data, 22 GiB used, 515 GiB / 536 GiB avail; 154 B/s rd, 47 MiB/s wr, 1.18k op/s; 8886/285498 objects degraded (3.112%); 1157/285498 objects misplaced (0.405%); 9.4 MiB/s, 151 objects/s recovering 2023-09-23T21:17:08.575 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:17:08 smithi026 ceph-mon[179688]: pgmap v1452: 129 pgs: 1 active+recovering+undersized+remapped, 11 active+recovery_wait+undersized+degraded+remapped, 117 active+clean; 5.8 GiB data, 22 GiB used, 515 GiB / 536 GiB avail; 154 B/s rd, 47 MiB/s wr, 1.18k op/s; 8886/285498 objects degraded (3.112%); 1157/285498 objects misplaced (0.405%); 9.4 MiB/s, 151 objects/s recovering 2023-09-23T21:17:08.578 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:17:08.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:17:08 smithi070 ceph-mon[168282]: pgmap v1452: 129 pgs: 1 active+recovering+undersized+remapped, 11 active+recovery_wait+undersized+degraded+remapped, 117 active+clean; 5.8 GiB data, 22 GiB used, 515 GiB / 536 GiB avail; 154 B/s rd, 47 MiB/s wr, 1.18k op/s; 8886/285498 objects degraded (3.112%); 1157/285498 objects misplaced (0.405%); 9.4 MiB/s, 151 objects/s recovering 2023-09-23T21:17:08.673 INFO:tasks.thrashosds.thrasher:in_osds: [0, 2, 3, 4, 5, 7] out_osds: [6, 1] dead_osds: [] live_osds: [2, 3, 4, 5, 7, 6, 1, 0] 2023-09-23T21:17:08.673 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T21:17:08.673 INFO:tasks.thrashosds.thrasher:primary_affinity 2023-09-23T21:17:08.673 INFO:tasks.thrashosds.thrasher:Setting osd 2 primary_affinity to 1.000000 2023-09-23T21:17:08.674 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph osd primary-affinity 2 1 2023-09-23T21:17:09.195 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:17:09.202 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:17:09.202 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:17:09.303 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:17:09.337 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:17:09.381 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:17:09.383 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:17:09.384 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:17:09.485 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:17:09.553 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:17:09.819 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:17:09.883 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 21:17:09 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[164029]: ::ffff:172.21.15.70 - - [23/Sep/2023:21:17:09] "GET /metrics HTTP/1.1" 200 36386 "" "Prometheus/2.43.0" 2023-09-23T21:17:09.916 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:17:09.920 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:17:09.920 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:17:10.021 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:17:10.056 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:17:10.105 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:17:10.110 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:17:10.111 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:17:10.111 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:17:10.114 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:17:10.212 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:17:10.225 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:17:10 smithi026 ceph-mon[175151]: pgmap v1453: 129 pgs: 1 active+recovering+undersized+remapped, 11 active+recovery_wait+undersized+degraded+remapped, 117 active+clean; 5.8 GiB data, 22 GiB used, 515 GiB / 536 GiB avail; 127 B/s rd, 39 MiB/s wr, 1.01k op/s; 8886/284697 objects degraded (3.121%); 1157/284697 objects misplaced (0.406%); 7.8 MiB/s, 125 objects/s recovering 2023-09-23T21:17:10.225 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:17:10 smithi026 ceph-mon[179688]: pgmap v1453: 129 pgs: 1 active+recovering+undersized+remapped, 11 active+recovery_wait+undersized+degraded+remapped, 117 active+clean; 5.8 GiB data, 22 GiB used, 515 GiB / 536 GiB avail; 127 B/s rd, 39 MiB/s wr, 1.01k op/s; 8886/284697 objects degraded (3.121%); 1157/284697 objects misplaced (0.406%); 7.8 MiB/s, 125 objects/s recovering 2023-09-23T21:17:10.302 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:17:10.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:17:10 smithi070 ceph-mon[168282]: pgmap v1453: 129 pgs: 1 active+recovering+undersized+remapped, 11 active+recovery_wait+undersized+degraded+remapped, 117 active+clean; 5.8 GiB data, 22 GiB used, 515 GiB / 536 GiB avail; 127 B/s rd, 39 MiB/s wr, 1.01k op/s; 8886/284697 objects degraded (3.121%); 1157/284697 objects misplaced (0.406%); 7.8 MiB/s, 125 objects/s recovering 2023-09-23T21:17:10.546 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:17:10.552 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:17:10.552 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:17:10.653 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:17:10.721 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:17:10.864 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T21:17:10.880 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:17:10.881 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:17:10.982 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:17:11.015 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:17:11.060 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:17:11.066 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:17:11.067 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:17:11.167 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:17:11.231 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:17:11 smithi026 ceph-mon[175151]: Health check update: Degraded data redundancy: 8886/284697 objects degraded (3.121%), 11 pgs degraded (PG_DEGRADED) 2023-09-23T21:17:11.231 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:17:11 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/2337304306' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:17:11.231 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:17:11 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:17:11.262 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:17:11.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:17:11 smithi070 ceph-mon[168282]: Health check update: Degraded data redundancy: 8886/284697 objects degraded (3.121%), 11 pgs degraded (PG_DEGRADED) 2023-09-23T21:17:11.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:17:11 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/2337304306' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:17:11.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:17:11 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:17:11.631 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:17:11 smithi026 ceph-mon[179688]: Health check update: Degraded data redundancy: 8886/284697 objects degraded (3.121%), 11 pgs degraded (PG_DEGRADED) 2023-09-23T21:17:11.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:17:11 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/2337304306' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:17:11.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:17:11 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:17:11.850 INFO:teuthology.orchestra.run.smithi026.stderr:nodeep-scrub is unset 2023-09-23T21:17:12.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:17:12 smithi070 ceph-mon[168282]: pgmap v1454: 129 pgs: 1 active+recovering+undersized+remapped, 11 active+recovery_wait+undersized+degraded+remapped, 117 active+clean; 5.7 GiB data, 21 GiB used, 515 GiB / 536 GiB avail; 123 B/s rd, 12 MiB/s wr, 810 op/s; 8886/278604 objects degraded (3.189%); 1157/278604 objects misplaced (0.415%); 3.5 MiB/s, 56 objects/s recovering 2023-09-23T21:17:12.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:17:12 smithi070 ceph-mon[168282]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T21:17:12.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:17:12 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T21:17:12.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:17:12 smithi070 ceph-mon[168282]: osdmap e574: 8 total, 8 up, 6 in 2023-09-23T21:17:12.631 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:17:12 smithi026 ceph-mon[175151]: pgmap v1454: 129 pgs: 1 active+recovering+undersized+remapped, 11 active+recovery_wait+undersized+degraded+remapped, 117 active+clean; 5.7 GiB data, 21 GiB used, 515 GiB / 536 GiB avail; 123 B/s rd, 12 MiB/s wr, 810 op/s; 8886/278604 objects degraded (3.189%); 1157/278604 objects misplaced (0.415%); 3.5 MiB/s, 56 objects/s recovering 2023-09-23T21:17:12.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:17:12 smithi026 ceph-mon[175151]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T21:17:12.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:17:12 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T21:17:12.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:17:12 smithi026 ceph-mon[175151]: osdmap e574: 8 total, 8 up, 6 in 2023-09-23T21:17:12.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:17:12 smithi026 ceph-mon[179688]: pgmap v1454: 129 pgs: 1 active+recovering+undersized+remapped, 11 active+recovery_wait+undersized+degraded+remapped, 117 active+clean; 5.7 GiB data, 21 GiB used, 515 GiB / 536 GiB avail; 123 B/s rd, 12 MiB/s wr, 810 op/s; 8886/278604 objects degraded (3.189%); 1157/278604 objects misplaced (0.415%); 3.5 MiB/s, 56 objects/s recovering 2023-09-23T21:17:12.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:17:12 smithi026 ceph-mon[179688]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T21:17:12.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:17:12 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T21:17:12.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:17:12 smithi026 ceph-mon[179688]: osdmap e574: 8 total, 8 up, 6 in 2023-09-23T21:17:12.799 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:17:12.807 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:17:12.808 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:17:12.908 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:17:12.982 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:17:13.245 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:17:13 smithi026 ceph-mon[175151]: failed to encode map e574 with expected crc 2023-09-23T21:17:13.245 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:17:13 smithi026 ceph-mon[175151]: failed to encode map e574 with expected crc 2023-09-23T21:17:13.245 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:17:13 smithi026 ceph-mon[175151]: failed to encode map e574 with expected crc 2023-09-23T21:17:13.245 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:17:13 smithi026 ceph-mon[175151]: failed to encode map e574 with expected crc 2023-09-23T21:17:13.245 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:17:13 smithi026 ceph-mon[175151]: failed to encode map e574 with expected crc 2023-09-23T21:17:13.245 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:17:13 smithi026 ceph-mon[179688]: failed to encode map e574 with expected crc 2023-09-23T21:17:13.246 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:17:13 smithi026 ceph-mon[179688]: failed to encode map e574 with expected crc 2023-09-23T21:17:13.246 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:17:13 smithi026 ceph-mon[179688]: failed to encode map e574 with expected crc 2023-09-23T21:17:13.246 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:17:13 smithi026 ceph-mon[179688]: failed to encode map e574 with expected crc 2023-09-23T21:17:13.246 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:17:13 smithi026 ceph-mon[179688]: failed to encode map e574 with expected crc 2023-09-23T21:17:13.368 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:17:13.374 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:17:13.374 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:17:13.475 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:17:13.528 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:17:13 smithi070 ceph-mon[168282]: failed to encode map e574 with expected crc 2023-09-23T21:17:13.528 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:17:13 smithi070 ceph-mon[168282]: failed to encode map e574 with expected crc 2023-09-23T21:17:13.528 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:17:13 smithi070 ceph-mon[168282]: failed to encode map e574 with expected crc 2023-09-23T21:17:13.528 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:17:13 smithi070 ceph-mon[168282]: failed to encode map e574 with expected crc 2023-09-23T21:17:13.528 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:17:13 smithi070 ceph-mon[168282]: failed to encode map e574 with expected crc 2023-09-23T21:17:13.533 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:17:13.576 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:17:13.579 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:17:13.579 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:17:13.680 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:17:13.723 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:17:13.768 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:17:13.771 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:17:13.772 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:17:13.872 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:17:13.951 INFO:teuthology.orchestra.run.smithi026.stdout:ERROR: (22) Invalid argument 2023-09-23T21:17:13.951 INFO:teuthology.orchestra.run.smithi026.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T21:17:13.951 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:17:14.224 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:17:14.230 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:17:14.230 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:17:14.331 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:17:14.396 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:17:14.526 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:17:14 smithi026 ceph-mon[175151]: pgmap v1456: 129 pgs: 1 active+recovering+undersized+remapped, 11 active+recovery_wait+undersized+degraded+remapped, 117 active+clean; 5.5 GiB data, 21 GiB used, 516 GiB / 536 GiB avail; 110 B/s rd, 11 MiB/s wr, 1.10k op/s; 8886/268194 objects degraded (3.313%); 802/268194 objects misplaced (0.299%); 5.5 MiB/s, 88 objects/s recovering 2023-09-23T21:17:14.526 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:17:14 smithi026 ceph-mon[179688]: pgmap v1456: 129 pgs: 1 active+recovering+undersized+remapped, 11 active+recovery_wait+undersized+degraded+remapped, 117 active+clean; 5.5 GiB data, 21 GiB used, 516 GiB / 536 GiB avail; 110 B/s rd, 11 MiB/s wr, 1.10k op/s; 8886/268194 objects degraded (3.313%); 802/268194 objects misplaced (0.299%); 5.5 MiB/s, 88 objects/s recovering 2023-09-23T21:17:14.576 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:17:14.584 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:17:14.585 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:17:14.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:17:14 smithi070 ceph-mon[168282]: pgmap v1456: 129 pgs: 1 active+recovering+undersized+remapped, 11 active+recovery_wait+undersized+degraded+remapped, 117 active+clean; 5.5 GiB data, 21 GiB used, 516 GiB / 536 GiB avail; 110 B/s rd, 11 MiB/s wr, 1.10k op/s; 8886/268194 objects degraded (3.313%); 802/268194 objects misplaced (0.299%); 5.5 MiB/s, 88 objects/s recovering 2023-09-23T21:17:14.676 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_blocked_ops 2023-09-23T21:17:14.686 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:17:14.827 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:17:14.928 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0: {} 2023-09-23T21:17:14.949 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:17:14.953 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:17:14.953 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:17:14.972 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1: {} 2023-09-23T21:17:15.036 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2: {} 2023-09-23T21:17:15.054 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:17:15.083 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3: {} 2023-09-23T21:17:15.135 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:17:15.136 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4: {} 2023-09-23T21:17:15.181 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T21:17:15.201 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:17:15.217 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:17:15.217 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:17:15.222 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6: {} 2023-09-23T21:17:15.263 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T21:17:15.282 INFO:teuthology.orchestra.run.smithi026.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T21:17:15.282 INFO:teuthology.orchestra.run.smithi026.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T21:17:15.282 INFO:teuthology.orchestra.run.smithi026.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T21:17:15.282 INFO:teuthology.orchestra.run.smithi026.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T21:17:15.282 INFO:teuthology.orchestra.run.smithi026.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T21:17:15.282 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T21:17:15.283 INFO:teuthology.orchestra.run.smithi026.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T21:17:15.283 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T21:17:15.318 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:17:15.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:17:15 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/1947059487' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:17:15.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:17:15 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/1947059487' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:17:15.398 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T21:17:15.400 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:17:15.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:17:15 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/1947059487' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:17:16.326 INFO:teuthology.orchestra.run.smithi026.stderr:set osd.2 primary-affinity to 1 (8655362) 2023-09-23T21:17:16.397 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:17:16.401 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:17:16.401 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:17:16.502 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:17:16.566 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:17:16 smithi070 ceph-mon[168282]: pgmap v1457: 129 pgs: 1 active+recovering+undersized+remapped, 11 active+recovery_wait+undersized+degraded+remapped, 117 active+clean; 5.4 GiB data, 20 GiB used, 516 GiB / 536 GiB avail; 102 B/s rd, 4.9 MiB/s wr, 1.04k op/s; 8886/264174 objects degraded (3.364%); 802/264174 objects misplaced (0.304%); 5.1 MiB/s, 81 objects/s recovering 2023-09-23T21:17:16.566 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:17:16 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/3445653617' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T21:17:16.566 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:17:16 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T21:17:16.566 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:17:16 smithi070 ceph-mon[168282]: Health check update: Degraded data redundancy: 8886/264174 objects degraded (3.364%), 11 pgs degraded (PG_DEGRADED) 2023-09-23T21:17:16.566 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:17:16 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:17:16.566 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:17:16 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:17:16.575 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:17:16.624 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:17:16.628 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:17:16.628 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:17:16.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:17:16 smithi026 ceph-mon[175151]: pgmap v1457: 129 pgs: 1 active+recovering+undersized+remapped, 11 active+recovery_wait+undersized+degraded+remapped, 117 active+clean; 5.4 GiB data, 20 GiB used, 516 GiB / 536 GiB avail; 102 B/s rd, 4.9 MiB/s wr, 1.04k op/s; 8886/264174 objects degraded (3.364%); 802/264174 objects misplaced (0.304%); 5.1 MiB/s, 81 objects/s recovering 2023-09-23T21:17:16.633 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:17:16 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/3445653617' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T21:17:16.633 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:17:16 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T21:17:16.633 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:17:16 smithi026 ceph-mon[175151]: Health check update: Degraded data redundancy: 8886/264174 objects degraded (3.364%), 11 pgs degraded (PG_DEGRADED) 2023-09-23T21:17:16.633 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:17:16 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:17:16.633 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:17:16 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:17:16.633 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:17:16 smithi026 ceph-mon[179688]: pgmap v1457: 129 pgs: 1 active+recovering+undersized+remapped, 11 active+recovery_wait+undersized+degraded+remapped, 117 active+clean; 5.4 GiB data, 20 GiB used, 516 GiB / 536 GiB avail; 102 B/s rd, 4.9 MiB/s wr, 1.04k op/s; 8886/264174 objects degraded (3.364%); 802/264174 objects misplaced (0.304%); 5.1 MiB/s, 81 objects/s recovering 2023-09-23T21:17:16.633 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:17:16 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/3445653617' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T21:17:16.633 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:17:16 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T21:17:16.633 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:17:16 smithi026 ceph-mon[179688]: Health check update: Degraded data redundancy: 8886/264174 objects degraded (3.364%), 11 pgs degraded (PG_DEGRADED) 2023-09-23T21:17:16.633 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:17:16 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:17:16.634 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:17:16 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:17:16.729 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:17:16.808 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:17:17.154 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T21:17:17.165 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:17:17.165 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:17:17.266 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:17:17.361 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:17:17.566 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:17:17 smithi026 ceph-mon[175151]: failed to encode map e575 with expected crc 2023-09-23T21:17:17.566 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:17:17 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 2, "weight": 1.0}]': finished 2023-09-23T21:17:17.566 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:17:17 smithi026 ceph-mon[175151]: osdmap e575: 8 total, 8 up, 6 in 2023-09-23T21:17:17.566 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:17:17 smithi026 ceph-mon[175151]: failed to encode map e575 with expected crc 2023-09-23T21:17:17.566 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:17:17 smithi026 ceph-mon[179688]: failed to encode map e575 with expected crc 2023-09-23T21:17:17.567 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:17:17 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 2, "weight": 1.0}]': finished 2023-09-23T21:17:17.567 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:17:17 smithi026 ceph-mon[179688]: osdmap e575: 8 total, 8 up, 6 in 2023-09-23T21:17:17.567 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:17:17 smithi026 ceph-mon[179688]: failed to encode map e575 with expected crc 2023-09-23T21:17:17.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:17:17 smithi070 ceph-mon[168282]: failed to encode map e575 with expected crc 2023-09-23T21:17:17.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:17:17 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 2, "weight": 1.0}]': finished 2023-09-23T21:17:17.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:17:17 smithi070 ceph-mon[168282]: osdmap e575: 8 total, 8 up, 6 in 2023-09-23T21:17:17.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:17:17 smithi070 ceph-mon[168282]: failed to encode map e575 with expected crc 2023-09-23T21:17:18.070 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:17:18.077 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:17:18.078 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:17:18.179 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:17:18.183 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:17:18.260 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:17:18.261 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:17:18.364 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:17:18.375 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:17:18.376 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:17:18 smithi026 ceph-mon[179688]: pgmap v1459: 129 pgs: 1 active+recovering+undersized+remapped, 11 active+recovery_wait+undersized+degraded+remapped, 117 active+clean; 5.2 GiB data, 20 GiB used, 516 GiB / 536 GiB avail; 0 B/s wr, 1.30k op/s; 8886/253374 objects degraded (3.507%); 393/253374 objects misplaced (0.155%); 6.0 MiB/s, 95 objects/s recovering 2023-09-23T21:17:18.376 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:17:18.376 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:17:18 smithi026 ceph-mon[175151]: pgmap v1459: 129 pgs: 1 active+recovering+undersized+remapped, 11 active+recovery_wait+undersized+degraded+remapped, 117 active+clean; 5.2 GiB data, 20 GiB used, 516 GiB / 536 GiB avail; 0 B/s wr, 1.30k op/s; 8886/253374 objects degraded (3.507%); 393/253374 objects misplaced (0.155%); 6.0 MiB/s, 95 objects/s recovering 2023-09-23T21:17:18.477 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:17:18.550 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:17:18.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:17:18 smithi070 ceph-mon[168282]: pgmap v1459: 129 pgs: 1 active+recovering+undersized+remapped, 11 active+recovery_wait+undersized+degraded+remapped, 117 active+clean; 5.2 GiB data, 20 GiB used, 516 GiB / 536 GiB avail; 0 B/s wr, 1.30k op/s; 8886/253374 objects degraded (3.507%); 393/253374 objects misplaced (0.155%); 6.0 MiB/s, 95 objects/s recovering 2023-09-23T21:17:18.654 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T21:17:18.666 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:17:18.668 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:17:18.769 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:17:18.839 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:17:18.899 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T21:17:18.917 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:17:18.921 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:17:18.922 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:17:19.023 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:17:19.055 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:17:19.113 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:17:19.116 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:17:19.117 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:17:19.217 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:17:19.291 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:17:19.614 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 21:17:19 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[164029]: ::ffff:172.21.15.70 - - [23/Sep/2023:21:17:19] "GET /metrics HTTP/1.1" 200 36398 "" "Prometheus/2.43.0" 2023-09-23T21:17:19.625 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:17:19.630 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:17:19.631 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:17:19.732 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:17:19.848 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:17:19.941 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:17:19.952 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:17:19.952 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:17:20.053 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:17:20.084 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:17:20.130 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:17:20.136 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:17:20.137 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:17:20.237 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:17:20.322 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:17:20 smithi026 ceph-mon[179688]: pgmap v1460: 129 pgs: 1 active+recovering+undersized+remapped, 11 active+recovery_wait+undersized+degraded+remapped, 117 active+clean; 5.1 GiB data, 20 GiB used, 517 GiB / 536 GiB avail; 0 B/s wr, 1.09k op/s; 8886/252333 objects degraded (3.522%); 393/252333 objects misplaced (0.156%); 6.0 MiB/s, 95 objects/s recovering 2023-09-23T21:17:20.323 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:17:20 smithi026 ceph-mon[175151]: pgmap v1460: 129 pgs: 1 active+recovering+undersized+remapped, 11 active+recovery_wait+undersized+degraded+remapped, 117 active+clean; 5.1 GiB data, 20 GiB used, 517 GiB / 536 GiB avail; 0 B/s wr, 1.09k op/s; 8886/252333 objects degraded (3.522%); 393/252333 objects misplaced (0.156%); 6.0 MiB/s, 95 objects/s recovering 2023-09-23T21:17:20.341 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:17:20.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:17:20 smithi070 ceph-mon[168282]: pgmap v1460: 129 pgs: 1 active+recovering+undersized+remapped, 11 active+recovery_wait+undersized+degraded+remapped, 117 active+clean; 5.1 GiB data, 20 GiB used, 517 GiB / 536 GiB avail; 0 B/s wr, 1.09k op/s; 8886/252333 objects degraded (3.522%); 393/252333 objects misplaced (0.156%); 6.0 MiB/s, 95 objects/s recovering 2023-09-23T21:17:20.423 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:17:20.426 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:17:20.427 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:17:20.528 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:17:20.603 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:17:20.604 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:17:20.675 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:17:20.682 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:17:20.682 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:17:20.724 INFO:teuthology.orchestra.run.smithi026.stdout:{ 2023-09-23T21:17:20.724 INFO:teuthology.orchestra.run.smithi026.stdout: "ops": [], 2023-09-23T21:17:20.724 INFO:teuthology.orchestra.run.smithi026.stdout: "complaint_time": 30, 2023-09-23T21:17:20.724 INFO:teuthology.orchestra.run.smithi026.stdout: "num_blocked_ops": 0 2023-09-23T21:17:20.724 INFO:teuthology.orchestra.run.smithi026.stdout:} 2023-09-23T21:17:20.784 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:17:20.811 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:17:21.344 INFO:teuthology.orchestra.run.smithi026.stderr:noscrub is set 2023-09-23T21:17:21.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:17:21 smithi070 ceph-mon[168282]: failed to encode map e576 with expected crc 2023-09-23T21:17:21.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:17:21 smithi070 ceph-mon[168282]: failed to encode map e576 with expected crc 2023-09-23T21:17:21.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:17:21 smithi070 ceph-mon[168282]: osdmap e576: 8 total, 8 up, 6 in 2023-09-23T21:17:21.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:17:21 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/1365984447' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T21:17:21.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:17:21 smithi070 ceph-mon[168282]: Health check update: Degraded data redundancy: 8886/252333 objects degraded (3.522%), 11 pgs degraded (PG_DEGRADED) 2023-09-23T21:17:21.633 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:17:21 smithi026 ceph-mon[175151]: failed to encode map e576 with expected crc 2023-09-23T21:17:21.634 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:17:21 smithi026 ceph-mon[175151]: failed to encode map e576 with expected crc 2023-09-23T21:17:21.634 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:17:21 smithi026 ceph-mon[175151]: osdmap e576: 8 total, 8 up, 6 in 2023-09-23T21:17:21.634 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:17:21 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/1365984447' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T21:17:21.634 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:17:21 smithi026 ceph-mon[175151]: Health check update: Degraded data redundancy: 8886/252333 objects degraded (3.522%), 11 pgs degraded (PG_DEGRADED) 2023-09-23T21:17:21.634 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:17:21 smithi026 ceph-mon[179688]: failed to encode map e576 with expected crc 2023-09-23T21:17:21.634 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:17:21 smithi026 ceph-mon[179688]: failed to encode map e576 with expected crc 2023-09-23T21:17:21.634 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:17:21 smithi026 ceph-mon[179688]: osdmap e576: 8 total, 8 up, 6 in 2023-09-23T21:17:21.634 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:17:21 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/1365984447' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T21:17:21.635 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:17:21 smithi026 ceph-mon[179688]: Health check update: Degraded data redundancy: 8886/252333 objects degraded (3.522%), 11 pgs degraded (PG_DEGRADED) 2023-09-23T21:17:22.554 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:17:22.562 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:17:22.562 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:17:22.563 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:17:22 smithi026 ceph-mon[179688]: failed to encode map e576 with expected crc 2023-09-23T21:17:22.564 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:17:22 smithi026 ceph-mon[179688]: pgmap v1462: 129 pgs: 1 active+recovering+undersized+remapped, 11 active+recovery_wait+undersized+degraded+remapped, 117 active+clean; 5.0 GiB data, 19 GiB used, 517 GiB / 536 GiB avail; 0 B/s wr, 908 op/s; 8886/246375 objects degraded (3.607%); 393/246375 objects misplaced (0.160%); 3.2 MiB/s, 51 objects/s recovering 2023-09-23T21:17:22.564 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:17:22 smithi026 ceph-mon[179688]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:17:22.564 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:17:22 smithi026 ceph-mon[179688]: failed to encode map e577 with expected crc 2023-09-23T21:17:22.564 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:17:22 smithi026 ceph-mon[179688]: failed to encode map e577 with expected crc 2023-09-23T21:17:22.564 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:17:22 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/1365984447' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T21:17:22.564 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:17:22 smithi026 ceph-mon[179688]: osdmap e577: 8 total, 8 up, 6 in 2023-09-23T21:17:22.564 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:17:22 smithi026 ceph-mon[179688]: failed to encode map e577 with expected crc 2023-09-23T21:17:22.564 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:17:22 smithi026 ceph-mon[179688]: failed to encode map e577 with expected crc 2023-09-23T21:17:22.564 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:17:22 smithi026 ceph-mon[175151]: failed to encode map e576 with expected crc 2023-09-23T21:17:22.565 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:17:22 smithi026 ceph-mon[175151]: pgmap v1462: 129 pgs: 1 active+recovering+undersized+remapped, 11 active+recovery_wait+undersized+degraded+remapped, 117 active+clean; 5.0 GiB data, 19 GiB used, 517 GiB / 536 GiB avail; 0 B/s wr, 908 op/s; 8886/246375 objects degraded (3.607%); 393/246375 objects misplaced (0.160%); 3.2 MiB/s, 51 objects/s recovering 2023-09-23T21:17:22.565 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:17:22 smithi026 ceph-mon[175151]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:17:22.565 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:17:22 smithi026 ceph-mon[175151]: failed to encode map e577 with expected crc 2023-09-23T21:17:22.565 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:17:22 smithi026 ceph-mon[175151]: failed to encode map e577 with expected crc 2023-09-23T21:17:22.565 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:17:22 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/1365984447' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T21:17:22.565 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:17:22 smithi026 ceph-mon[175151]: osdmap e577: 8 total, 8 up, 6 in 2023-09-23T21:17:22.565 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:17:22 smithi026 ceph-mon[175151]: failed to encode map e577 with expected crc 2023-09-23T21:17:22.565 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:17:22 smithi026 ceph-mon[175151]: failed to encode map e577 with expected crc 2023-09-23T21:17:22.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:17:22 smithi070 ceph-mon[168282]: failed to encode map e576 with expected crc 2023-09-23T21:17:22.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:17:22 smithi070 ceph-mon[168282]: pgmap v1462: 129 pgs: 1 active+recovering+undersized+remapped, 11 active+recovery_wait+undersized+degraded+remapped, 117 active+clean; 5.0 GiB data, 19 GiB used, 517 GiB / 536 GiB avail; 0 B/s wr, 908 op/s; 8886/246375 objects degraded (3.607%); 393/246375 objects misplaced (0.160%); 3.2 MiB/s, 51 objects/s recovering 2023-09-23T21:17:22.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:17:22 smithi070 ceph-mon[168282]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:17:22.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:17:22 smithi070 ceph-mon[168282]: failed to encode map e577 with expected crc 2023-09-23T21:17:22.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:17:22 smithi070 ceph-mon[168282]: failed to encode map e577 with expected crc 2023-09-23T21:17:22.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:17:22 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/1365984447' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T21:17:22.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:17:22 smithi070 ceph-mon[168282]: osdmap e577: 8 total, 8 up, 6 in 2023-09-23T21:17:22.621 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:17:22 smithi070 ceph-mon[168282]: failed to encode map e577 with expected crc 2023-09-23T21:17:22.621 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:17:22 smithi070 ceph-mon[168282]: failed to encode map e577 with expected crc 2023-09-23T21:17:22.663 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:17:22.743 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:17:22.931 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:17:22.938 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:17:22.939 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:17:22.999 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_historic_ops 2023-09-23T21:17:23.039 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:17:23.074 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:17:23.089 INFO:tasks.thrashosds.thrasher:in_osds: [0, 2, 3, 4, 5, 7] out_osds: [6, 1] dead_osds: [] live_osds: [2, 3, 4, 5, 7, 6, 1, 0] 2023-09-23T21:17:23.090 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph osd reweight 0 1 2023-09-23T21:17:23.131 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:17:23.132 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:17:23.132 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:17:23.233 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:17:23.273 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:17:23.314 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:17:23.318 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:17:23.318 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:17:23.419 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:17:23.503 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:17:23.677 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:17:23.684 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:17:23.685 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:17:23.785 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:17:23.816 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:17:23.863 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:17:23.866 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:17:23.867 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:17:23.968 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:17:24.017 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:17:24.064 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T21:17:24.072 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:17:24.072 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:17:24.173 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:17:24.204 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:17:24.214 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:17:24.227 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:17:24.243 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:17:24.247 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:17:24.247 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:17:24.304 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0: {} 2023-09-23T21:17:24.325 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1: {} 2023-09-23T21:17:24.343 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2: {} 2023-09-23T21:17:24.348 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:17:24.363 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3: {} 2023-09-23T21:17:24.370 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:17:24 smithi026 ceph-mon[179688]: pgmap v1464: 129 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 118 active+clean; 4.8 GiB data, 19 GiB used, 518 GiB / 536 GiB avail; 0 B/s wr, 1.18k op/s; 8070/237927 objects degraded (3.392%); 208/237927 objects misplaced (0.087%); 13 MiB/s, 204 objects/s recovering 2023-09-23T21:17:24.370 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:17:24 smithi026 ceph-mon[179688]: osdmap e578: 8 total, 8 up, 6 in 2023-09-23T21:17:24.370 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:17:24 smithi026 ceph-mon[179688]: failed to encode map e578 with expected crc 2023-09-23T21:17:24.370 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:17:24 smithi026 ceph-mon[179688]: failed to encode map e578 with expected crc 2023-09-23T21:17:24.370 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:17:24 smithi026 ceph-mon[179688]: failed to encode map e578 with expected crc 2023-09-23T21:17:24.370 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:17:24 smithi026 ceph-mon[179688]: failed to encode map e578 with expected crc 2023-09-23T21:17:24.370 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:17:24 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/1102720615' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:17:24.370 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:17:24 smithi026 ceph-mon[175151]: pgmap v1464: 129 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 118 active+clean; 4.8 GiB data, 19 GiB used, 518 GiB / 536 GiB avail; 0 B/s wr, 1.18k op/s; 8070/237927 objects degraded (3.392%); 208/237927 objects misplaced (0.087%); 13 MiB/s, 204 objects/s recovering 2023-09-23T21:17:24.371 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:17:24 smithi026 ceph-mon[175151]: osdmap e578: 8 total, 8 up, 6 in 2023-09-23T21:17:24.371 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:17:24 smithi026 ceph-mon[175151]: failed to encode map e578 with expected crc 2023-09-23T21:17:24.371 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:17:24 smithi026 ceph-mon[175151]: failed to encode map e578 with expected crc 2023-09-23T21:17:24.371 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:17:24 smithi026 ceph-mon[175151]: failed to encode map e578 with expected crc 2023-09-23T21:17:24.371 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:17:24 smithi026 ceph-mon[175151]: failed to encode map e578 with expected crc 2023-09-23T21:17:24.371 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:17:24 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/1102720615' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:17:24.381 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:17:24.391 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4: {} 2023-09-23T21:17:24.418 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T21:17:24.458 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6: {} 2023-09-23T21:17:24.491 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:17:24.492 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T21:17:24.499 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:17:24.499 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:17:24.505 INFO:teuthology.orchestra.run.smithi026.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T21:17:24.505 INFO:teuthology.orchestra.run.smithi026.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T21:17:24.505 INFO:teuthology.orchestra.run.smithi026.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T21:17:24.506 INFO:teuthology.orchestra.run.smithi026.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T21:17:24.506 INFO:teuthology.orchestra.run.smithi026.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T21:17:24.506 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T21:17:24.506 INFO:teuthology.orchestra.run.smithi026.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T21:17:24.506 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T21:17:24.601 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:17:24.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:17:24 smithi070 ceph-mon[168282]: pgmap v1464: 129 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 118 active+clean; 4.8 GiB data, 19 GiB used, 518 GiB / 536 GiB avail; 0 B/s wr, 1.18k op/s; 8070/237927 objects degraded (3.392%); 208/237927 objects misplaced (0.087%); 13 MiB/s, 204 objects/s recovering 2023-09-23T21:17:24.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:17:24 smithi070 ceph-mon[168282]: osdmap e578: 8 total, 8 up, 6 in 2023-09-23T21:17:24.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:17:24 smithi070 ceph-mon[168282]: failed to encode map e578 with expected crc 2023-09-23T21:17:24.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:17:24 smithi070 ceph-mon[168282]: failed to encode map e578 with expected crc 2023-09-23T21:17:24.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:17:24 smithi070 ceph-mon[168282]: failed to encode map e578 with expected crc 2023-09-23T21:17:24.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:17:24 smithi070 ceph-mon[168282]: failed to encode map e578 with expected crc 2023-09-23T21:17:24.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:17:24 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/1102720615' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:17:24.632 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:17:24.701 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:17:24.705 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:17:24.705 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:17:24.806 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:17:24.838 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T21:17:24.839 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:17:24.894 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:17:24.902 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:17:24.903 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:17:25.004 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:17:25.042 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:17:25.089 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:17:25.094 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:17:25.095 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:17:25.195 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:17:25.235 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:17:25.284 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:17:25.288 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:17:25.289 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:17:25.390 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:17:25.439 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:17:25.481 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:17:25.484 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:17:25 smithi070 ceph-mon[168282]: failed to encode map e579 with expected crc 2023-09-23T21:17:25.485 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:17:25 smithi070 ceph-mon[168282]: failed to encode map e579 with expected crc 2023-09-23T21:17:25.485 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:17:25 smithi070 ceph-mon[168282]: osdmap e579: 8 total, 8 up, 6 in 2023-09-23T21:17:25.485 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:17:25 smithi070 ceph-mon[168282]: failed to encode map e579 with expected crc 2023-09-23T21:17:25.485 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:17:25 smithi070 ceph-mon[168282]: failed to encode map e579 with expected crc 2023-09-23T21:17:25.485 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:17:25.486 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:17:25.587 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:17:25.633 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:17:25 smithi026 ceph-mon[175151]: failed to encode map e579 with expected crc 2023-09-23T21:17:25.634 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:17:25 smithi026 ceph-mon[175151]: failed to encode map e579 with expected crc 2023-09-23T21:17:25.634 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:17:25 smithi026 ceph-mon[175151]: osdmap e579: 8 total, 8 up, 6 in 2023-09-23T21:17:25.634 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:17:25 smithi026 ceph-mon[175151]: failed to encode map e579 with expected crc 2023-09-23T21:17:25.634 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:17:25 smithi026 ceph-mon[175151]: failed to encode map e579 with expected crc 2023-09-23T21:17:25.634 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:17:25 smithi026 ceph-mon[179688]: failed to encode map e579 with expected crc 2023-09-23T21:17:25.634 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:17:25 smithi026 ceph-mon[179688]: failed to encode map e579 with expected crc 2023-09-23T21:17:25.634 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:17:25 smithi026 ceph-mon[179688]: osdmap e579: 8 total, 8 up, 6 in 2023-09-23T21:17:25.634 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:17:25 smithi026 ceph-mon[179688]: failed to encode map e579 with expected crc 2023-09-23T21:17:25.635 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:17:25 smithi026 ceph-mon[179688]: failed to encode map e579 with expected crc 2023-09-23T21:17:25.670 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:17:26.398 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:17:26.404 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:17:26.404 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:17:26.505 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:17:26.534 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:17:26.569 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:17:26.571 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:17:26.572 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:17:26.673 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:17:26.720 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:17:26 smithi026 ceph-mon[179688]: pgmap v1467: 129 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 118 active+clean; 4.8 GiB data, 19 GiB used, 518 GiB / 536 GiB avail; 0 B/s wr, 950 op/s; 8070/234615 objects degraded (3.440%); 208/234615 objects misplaced (0.089%); 13 MiB/s, 207 objects/s recovering 2023-09-23T21:17:26.721 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:17:26 smithi026 ceph-mon[179688]: Health check update: Degraded data redundancy: 8070/234615 objects degraded (3.440%), 10 pgs degraded (PG_DEGRADED) 2023-09-23T21:17:26.721 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:17:26 smithi026 ceph-mon[175151]: pgmap v1467: 129 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 118 active+clean; 4.8 GiB data, 19 GiB used, 518 GiB / 536 GiB avail; 0 B/s wr, 950 op/s; 8070/234615 objects degraded (3.440%); 208/234615 objects misplaced (0.089%); 13 MiB/s, 207 objects/s recovering 2023-09-23T21:17:26.721 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:17:26 smithi026 ceph-mon[175151]: Health check update: Degraded data redundancy: 8070/234615 objects degraded (3.440%), 10 pgs degraded (PG_DEGRADED) 2023-09-23T21:17:26.725 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:17:26.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:17:26 smithi070 ceph-mon[168282]: pgmap v1467: 129 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 118 active+clean; 4.8 GiB data, 19 GiB used, 518 GiB / 536 GiB avail; 0 B/s wr, 950 op/s; 8070/234615 objects degraded (3.440%); 208/234615 objects misplaced (0.089%); 13 MiB/s, 207 objects/s recovering 2023-09-23T21:17:26.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:17:26 smithi070 ceph-mon[168282]: Health check update: Degraded data redundancy: 8070/234615 objects degraded (3.440%), 10 pgs degraded (PG_DEGRADED) 2023-09-23T21:17:27.073 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T21:17:27.078 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:17:27.079 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:17:27.180 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:17:27.207 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:17:27.244 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:17:27.246 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:17:27.247 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:17:27.348 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:17:27.381 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T21:17:27.397 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:17:27.430 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:17:27.433 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:17:27.433 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:17:27.534 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:17:27.565 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:17:27.607 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:17:27.611 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:17:27.612 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:17:27.713 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:17:27.758 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:17:27.782 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:17:27.803 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:17:27.808 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:17:27.809 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:17:27.909 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:17:27.987 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:17:28.078 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:17:28.081 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:17:28.082 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:17:28.182 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:17:28.242 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:17:28.569 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:17:28.573 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:17:28.573 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:17:28.624 INFO:teuthology.orchestra.run.smithi026.stdout:ERROR: (22) Invalid argument 2023-09-23T21:17:28.624 INFO:teuthology.orchestra.run.smithi026.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T21:17:28.654 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:17:28 smithi026 ceph-mon[175151]: pgmap v1468: 129 pgs: 2 active+recovering+undersized+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 118 active+clean; 4.6 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 0 B/s wr, 1.20k op/s; 6434/224682 objects degraded (2.864%); 1346/224682 objects misplaced (0.599%); 16 MiB/s, 249 objects/s recovering 2023-09-23T21:17:28.654 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:17:28 smithi026 ceph-mon[179688]: pgmap v1468: 129 pgs: 2 active+recovering+undersized+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 118 active+clean; 4.6 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 0 B/s wr, 1.20k op/s; 6434/224682 objects degraded (2.864%); 1346/224682 objects misplaced (0.599%); 16 MiB/s, 249 objects/s recovering 2023-09-23T21:17:28.674 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:17:28.704 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:17:28.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:17:28 smithi070 ceph-mon[168282]: pgmap v1468: 129 pgs: 2 active+recovering+undersized+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 118 active+clean; 4.6 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 0 B/s wr, 1.20k op/s; 6434/224682 objects degraded (2.864%); 1346/224682 objects misplaced (0.599%); 16 MiB/s, 249 objects/s recovering 2023-09-23T21:17:29.010 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:17:29.022 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:17:29.022 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:17:29.123 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:17:29.150 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:17:29.187 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:17:29.190 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:17:29.190 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:17:29.291 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:17:29.336 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:17:29.383 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:17:29.390 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:17:29.391 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:17:29.406 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:17:29 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/893826448' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-09-23T21:17:29.406 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:17:29 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-09-23T21:17:29.448 INFO:teuthology.orchestra.run.smithi026.stderr:reweighted osd.0 to 1 (10000) 2023-09-23T21:17:29.492 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:17:29.554 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:17:29 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/893826448' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-09-23T21:17:29.555 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:17:29 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-09-23T21:17:29.555 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:17:29 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/893826448' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-09-23T21:17:29.555 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:17:29 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-09-23T21:17:29.556 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:17:29.882 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 21:17:29 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[164029]: ::ffff:172.21.15.70 - - [23/Sep/2023:21:17:29] "GET /metrics HTTP/1.1" 200 36398 "" "Prometheus/2.43.0" 2023-09-23T21:17:29.982 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:17:29.989 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:17:29.989 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:17:30.062 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_ops_in_flight 2023-09-23T21:17:30.074 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph osd reweight 2 1 2023-09-23T21:17:30.090 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:17:30.226 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:17:30.279 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:17:30.283 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:17:30.283 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:17:30.384 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:17:30.385 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:17:30.473 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:17:30.606 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:17:30.611 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:17:30.612 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:17:30.713 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:17:30.771 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:17:30 smithi026 ceph-mon[175151]: pgmap v1469: 129 pgs: 1 active+recovering+undersized+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 119 active+clean; 4.6 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 0 B/s wr, 960 op/s; 6434/223818 objects degraded (2.875%); 1138/223818 objects misplaced (0.508%); 14 MiB/s, 218 objects/s recovering 2023-09-23T21:17:30.772 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:17:30 smithi026 ceph-mon[175151]: failed to encode map e580 with expected crc 2023-09-23T21:17:30.772 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:17:30 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-09-23T21:17:30.772 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:17:30 smithi026 ceph-mon[175151]: osdmap e580: 8 total, 8 up, 6 in 2023-09-23T21:17:30.772 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:17:30 smithi026 ceph-mon[175151]: failed to encode map e580 with expected crc 2023-09-23T21:17:30.773 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:17:30 smithi026 ceph-mon[179688]: pgmap v1469: 129 pgs: 1 active+recovering+undersized+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 119 active+clean; 4.6 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 0 B/s wr, 960 op/s; 6434/223818 objects degraded (2.875%); 1138/223818 objects misplaced (0.508%); 14 MiB/s, 218 objects/s recovering 2023-09-23T21:17:30.773 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:17:30 smithi026 ceph-mon[179688]: failed to encode map e580 with expected crc 2023-09-23T21:17:30.773 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:17:30 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-09-23T21:17:30.773 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:17:30 smithi026 ceph-mon[179688]: osdmap e580: 8 total, 8 up, 6 in 2023-09-23T21:17:30.773 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:17:30 smithi026 ceph-mon[179688]: failed to encode map e580 with expected crc 2023-09-23T21:17:30.780 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T21:17:30.799 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:17:30.867 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T21:17:30.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:17:30 smithi070 ceph-mon[168282]: pgmap v1469: 129 pgs: 1 active+recovering+undersized+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 119 active+clean; 4.6 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 0 B/s wr, 960 op/s; 6434/223818 objects degraded (2.875%); 1138/223818 objects misplaced (0.508%); 14 MiB/s, 218 objects/s recovering 2023-09-23T21:17:30.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:17:30 smithi070 ceph-mon[168282]: failed to encode map e580 with expected crc 2023-09-23T21:17:30.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:17:30 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-09-23T21:17:30.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:17:30 smithi070 ceph-mon[168282]: osdmap e580: 8 total, 8 up, 6 in 2023-09-23T21:17:30.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:17:30 smithi070 ceph-mon[168282]: failed to encode map e580 with expected crc 2023-09-23T21:17:30.875 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:17:30.875 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:17:30.976 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:17:31.009 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:17:31.560 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:17:31.781 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:17:31.786 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:17:31.786 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:17:31.787 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:17:31 smithi070 ceph-mon[168282]: Health check update: Degraded data redundancy: 6434/223818 objects degraded (2.875%), 9 pgs degraded (PG_DEGRADED) 2023-09-23T21:17:31.787 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:17:31 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:17:31.787 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:17:31 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:17:31.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:17:31 smithi026 ceph-mon[175151]: Health check update: Degraded data redundancy: 6434/223818 objects degraded (2.875%), 9 pgs degraded (PG_DEGRADED) 2023-09-23T21:17:31.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:17:31 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:17:31.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:17:31 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:17:31.885 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:17:31 smithi026 ceph-mon[179688]: Health check update: Degraded data redundancy: 6434/223818 objects degraded (2.875%), 9 pgs degraded (PG_DEGRADED) 2023-09-23T21:17:31.886 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:17:31 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:17:31.886 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:17:31 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:17:31.887 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:17:31.995 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:17:32.449 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:17:32.452 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:17:32.453 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:17:32.453 INFO:teuthology.orchestra.run.smithi026.stderr:nodeep-scrub is set 2023-09-23T21:17:32.554 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:17:32.614 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:17:32.617 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:17:32 smithi026 ceph-mon[179688]: pgmap v1471: 129 pgs: 1 active+recovering+undersized+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 119 active+clean; 4.5 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 815 op/s; 6434/218844 objects degraded (2.940%); 1138/218844 objects misplaced (0.520%); 5.7 MiB/s, 90 objects/s recovering 2023-09-23T21:17:32.617 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:17:32 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T21:17:32.617 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:17:32 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/4106324908' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:17:32.617 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:17:32 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:17:32.618 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:17:32 smithi026 ceph-mon[175151]: pgmap v1471: 129 pgs: 1 active+recovering+undersized+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 119 active+clean; 4.5 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 815 op/s; 6434/218844 objects degraded (2.940%); 1138/218844 objects misplaced (0.520%); 5.7 MiB/s, 90 objects/s recovering 2023-09-23T21:17:32.618 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:17:32 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T21:17:32.618 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:17:32 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/4106324908' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:17:32.618 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:17:32 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:17:32.666 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:17:32.670 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:17:32.671 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:17:32.695 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:17:32 smithi070 ceph-mon[168282]: pgmap v1471: 129 pgs: 1 active+recovering+undersized+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 119 active+clean; 4.5 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 815 op/s; 6434/218844 objects degraded (2.940%); 1138/218844 objects misplaced (0.520%); 5.7 MiB/s, 90 objects/s recovering 2023-09-23T21:17:32.695 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:17:32 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T21:17:32.695 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:17:32 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/4106324908' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:17:32.696 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:17:32 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:17:32.771 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:17:32.852 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:17:33.581 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:17:33 smithi070 ceph-mon[168282]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:17:33.581 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:17:33 smithi070 ceph-mon[168282]: failed to encode map e581 with expected crc 2023-09-23T21:17:33.581 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:17:33 smithi070 ceph-mon[168282]: failed to encode map e581 with expected crc 2023-09-23T21:17:33.582 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:17:33 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T21:17:33.582 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:17:33 smithi070 ceph-mon[168282]: osdmap e581: 8 total, 8 up, 6 in 2023-09-23T21:17:33.582 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:17:33 smithi070 ceph-mon[168282]: failed to encode map e581 with expected crc 2023-09-23T21:17:33.582 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:17:33 smithi070 ceph-mon[168282]: failed to encode map e581 with expected crc 2023-09-23T21:17:33.814 INFO:teuthology.orchestra.run.smithi070.stdout:ERROR: (22) Invalid argument 2023-09-23T21:17:33.814 INFO:teuthology.orchestra.run.smithi070.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T21:17:33.881 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:17:33 smithi026 ceph-mon[175151]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:17:33.881 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:17:33 smithi026 ceph-mon[175151]: failed to encode map e581 with expected crc 2023-09-23T21:17:33.881 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:17:33 smithi026 ceph-mon[175151]: failed to encode map e581 with expected crc 2023-09-23T21:17:33.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:17:33 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T21:17:33.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:17:33 smithi026 ceph-mon[175151]: osdmap e581: 8 total, 8 up, 6 in 2023-09-23T21:17:33.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:17:33 smithi026 ceph-mon[175151]: failed to encode map e581 with expected crc 2023-09-23T21:17:33.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:17:33 smithi026 ceph-mon[175151]: failed to encode map e581 with expected crc 2023-09-23T21:17:33.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:17:33 smithi026 ceph-mon[179688]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:17:33.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:17:33 smithi026 ceph-mon[179688]: failed to encode map e581 with expected crc 2023-09-23T21:17:33.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:17:33 smithi026 ceph-mon[179688]: failed to encode map e581 with expected crc 2023-09-23T21:17:33.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:17:33 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T21:17:33.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:17:33 smithi026 ceph-mon[179688]: osdmap e581: 8 total, 8 up, 6 in 2023-09-23T21:17:33.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:17:33 smithi026 ceph-mon[179688]: failed to encode map e581 with expected crc 2023-09-23T21:17:33.883 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:17:33 smithi026 ceph-mon[179688]: failed to encode map e581 with expected crc 2023-09-23T21:17:34.218 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0: {} 2023-09-23T21:17:34.218 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1: {} 2023-09-23T21:17:34.219 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2: {} 2023-09-23T21:17:34.239 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3: {} 2023-09-23T21:17:34.259 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4: {} 2023-09-23T21:17:34.278 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T21:17:34.299 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6: {} 2023-09-23T21:17:34.318 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T21:17:34.320 INFO:teuthology.orchestra.run.smithi026.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T21:17:34.320 INFO:teuthology.orchestra.run.smithi026.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T21:17:34.320 INFO:teuthology.orchestra.run.smithi026.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T21:17:34.320 INFO:teuthology.orchestra.run.smithi026.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T21:17:34.320 INFO:teuthology.orchestra.run.smithi026.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T21:17:34.320 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T21:17:34.320 INFO:teuthology.orchestra.run.smithi026.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T21:17:34.320 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T21:17:34.512 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_blocked_ops 2023-09-23T21:17:34.598 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:17:34 smithi070 ceph-mon[168282]: pgmap v1473: 129 pgs: 1 active+recovering+undersized+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 119 active+clean; 4.3 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 1.01k op/s; 6434/210276 objects degraded (3.060%); 240/210276 objects misplaced (0.114%); 13 MiB/s, 200 objects/s recovering 2023-09-23T21:17:34.598 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:17:34 smithi070 ceph-mon[168282]: failed to encode map e582 with expected crc 2023-09-23T21:17:34.598 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:17:34 smithi070 ceph-mon[168282]: osdmap e582: 8 total, 8 up, 6 in 2023-09-23T21:17:34.598 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:17:34 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/2781949367' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:17:34.633 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:17:34 smithi026 ceph-mon[175151]: pgmap v1473: 129 pgs: 1 active+recovering+undersized+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 119 active+clean; 4.3 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 1.01k op/s; 6434/210276 objects degraded (3.060%); 240/210276 objects misplaced (0.114%); 13 MiB/s, 200 objects/s recovering 2023-09-23T21:17:34.633 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:17:34 smithi026 ceph-mon[175151]: failed to encode map e582 with expected crc 2023-09-23T21:17:34.633 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:17:34 smithi026 ceph-mon[175151]: osdmap e582: 8 total, 8 up, 6 in 2023-09-23T21:17:34.633 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:17:34 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/2781949367' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:17:34.637 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:17:34 smithi026 ceph-mon[179688]: pgmap v1473: 129 pgs: 1 active+recovering+undersized+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 119 active+clean; 4.3 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 1.01k op/s; 6434/210276 objects degraded (3.060%); 240/210276 objects misplaced (0.114%); 13 MiB/s, 200 objects/s recovering 2023-09-23T21:17:34.637 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:17:34 smithi026 ceph-mon[179688]: failed to encode map e582 with expected crc 2023-09-23T21:17:34.637 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:17:34 smithi026 ceph-mon[179688]: osdmap e582: 8 total, 8 up, 6 in 2023-09-23T21:17:34.637 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:17:34 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/2781949367' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:17:34.670 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:17:34.676 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:17:34.677 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:17:34.778 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:17:34.836 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:17:34.886 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:17:34.890 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:17:34.891 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:17:34.991 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:17:35.049 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:17:35.228 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:17:35.230 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:17:35.231 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:17:35.332 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:17:35.337 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T21:17:35.419 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:17:35.491 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:17:35.500 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:17:35.501 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:17:35.586 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:17:35 smithi070 ceph-mon[168282]: failed to encode map e583 with expected crc 2023-09-23T21:17:35.586 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:17:35 smithi070 ceph-mon[168282]: failed to encode map e583 with expected crc 2023-09-23T21:17:35.586 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:17:35 smithi070 ceph-mon[168282]: failed to encode map e582 with expected crc 2023-09-23T21:17:35.586 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:17:35 smithi070 ceph-mon[168282]: osdmap e583: 8 total, 8 up, 6 in 2023-09-23T21:17:35.586 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:17:35 smithi070 ceph-mon[168282]: failed to encode map e583 with expected crc 2023-09-23T21:17:35.602 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:17:35.674 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:17:35 smithi026 ceph-mon[175151]: failed to encode map e583 with expected crc 2023-09-23T21:17:35.674 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:17:35 smithi026 ceph-mon[175151]: failed to encode map e583 with expected crc 2023-09-23T21:17:35.674 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:17:35 smithi026 ceph-mon[175151]: failed to encode map e582 with expected crc 2023-09-23T21:17:35.674 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:17:35 smithi026 ceph-mon[175151]: osdmap e583: 8 total, 8 up, 6 in 2023-09-23T21:17:35.674 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:17:35 smithi026 ceph-mon[175151]: failed to encode map e583 with expected crc 2023-09-23T21:17:35.675 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:17:35 smithi026 ceph-mon[179688]: failed to encode map e583 with expected crc 2023-09-23T21:17:35.675 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:17:35 smithi026 ceph-mon[179688]: failed to encode map e583 with expected crc 2023-09-23T21:17:35.675 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:17:35 smithi026 ceph-mon[179688]: failed to encode map e582 with expected crc 2023-09-23T21:17:35.675 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:17:35 smithi026 ceph-mon[179688]: osdmap e583: 8 total, 8 up, 6 in 2023-09-23T21:17:35.675 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:17:35 smithi026 ceph-mon[179688]: failed to encode map e583 with expected crc 2023-09-23T21:17:35.679 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:17:35.861 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:17:35.870 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:17:35.871 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:17:35.972 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:17:36.012 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:17:36.212 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:17:36.215 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:17:36.215 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:17:36.316 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:17:36.395 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:17:36.462 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:17:36.467 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:17:36.467 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:17:36.490 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:17:36 smithi070 ceph-mon[168282]: pgmap v1476: 129 pgs: 1 active+recovering+undersized+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 119 active+clean; 4.2 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 771 op/s; 6434/207117 objects degraded (3.106%); 240/207117 objects misplaced (0.116%); 9.7 MiB/s, 155 objects/s recovering 2023-09-23T21:17:36.490 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:17:36 smithi070 ceph-mon[168282]: Health check update: Degraded data redundancy: 6434/207117 objects degraded (3.106%), 9 pgs degraded (PG_DEGRADED) 2023-09-23T21:17:36.490 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:17:36 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/1689820933' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T21:17:36.490 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:17:36 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T21:17:36.569 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:17:36.642 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:17:36.734 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:17:36.742 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:17:36.743 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:17:36.819 INFO:teuthology.orchestra.run.smithi070.stdout:{ 2023-09-23T21:17:36.819 INFO:teuthology.orchestra.run.smithi070.stdout: "ops": [], 2023-09-23T21:17:36.819 INFO:teuthology.orchestra.run.smithi070.stdout: "complaint_time": 30, 2023-09-23T21:17:36.819 INFO:teuthology.orchestra.run.smithi070.stdout: "num_blocked_ops": 0 2023-09-23T21:17:36.819 INFO:teuthology.orchestra.run.smithi070.stdout:} 2023-09-23T21:17:36.844 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:17:36.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:17:36 smithi026 ceph-mon[175151]: pgmap v1476: 129 pgs: 1 active+recovering+undersized+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 119 active+clean; 4.2 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 771 op/s; 6434/207117 objects degraded (3.106%); 240/207117 objects misplaced (0.116%); 9.7 MiB/s, 155 objects/s recovering 2023-09-23T21:17:36.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:17:36 smithi026 ceph-mon[175151]: Health check update: Degraded data redundancy: 6434/207117 objects degraded (3.106%), 9 pgs degraded (PG_DEGRADED) 2023-09-23T21:17:36.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:17:36 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/1689820933' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T21:17:36.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:17:36 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T21:17:36.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:17:36 smithi026 ceph-mon[179688]: pgmap v1476: 129 pgs: 1 active+recovering+undersized+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 119 active+clean; 4.2 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 771 op/s; 6434/207117 objects degraded (3.106%); 240/207117 objects misplaced (0.116%); 9.7 MiB/s, 155 objects/s recovering 2023-09-23T21:17:36.883 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:17:36 smithi026 ceph-mon[179688]: Health check update: Degraded data redundancy: 6434/207117 objects degraded (3.106%), 9 pgs degraded (PG_DEGRADED) 2023-09-23T21:17:36.883 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:17:36 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/1689820933' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T21:17:36.883 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:17:36 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T21:17:36.884 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:17:36.908 INFO:teuthology.orchestra.run.smithi026.stderr:reweighted osd.2 to 1 (10000) 2023-09-23T21:17:37.135 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T21:17:37.190 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T21:17:37.221 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:17:37.226 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:17:37.227 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:17:37.327 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:17:37.349 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph osd reweight 3 1 2023-09-23T21:17:37.357 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:17:38.042 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:17:38.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:17:37 smithi070 ceph-mon[168282]: failed to encode map e584 with expected crc 2023-09-23T21:17:38.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:17:37 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-09-23T21:17:38.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:17:37 smithi070 ceph-mon[168282]: osdmap e584: 8 total, 8 up, 6 in 2023-09-23T21:17:38.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:17:37 smithi070 ceph-mon[168282]: failed to encode map e584 with expected crc 2023-09-23T21:17:38.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:17:37 smithi070 ceph-mon[168282]: failed to encode map e584 with expected crc 2023-09-23T21:17:38.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:17:37 smithi070 ceph-mon[168282]: pgmap v1478: 129 pgs: 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 120 active+clean; 4.0 GiB data, 17 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 832 op/s; 5663/198414 objects degraded (2.854%); 209/198414 objects misplaced (0.105%); 11 MiB/s, 168 objects/s recovering 2023-09-23T21:17:38.383 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:17:37 smithi026 ceph-mon[179688]: failed to encode map e584 with expected crc 2023-09-23T21:17:38.384 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:17:37 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-09-23T21:17:38.384 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:17:37 smithi026 ceph-mon[179688]: osdmap e584: 8 total, 8 up, 6 in 2023-09-23T21:17:38.384 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:17:37 smithi026 ceph-mon[179688]: failed to encode map e584 with expected crc 2023-09-23T21:17:38.384 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:17:37 smithi026 ceph-mon[179688]: failed to encode map e584 with expected crc 2023-09-23T21:17:38.384 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:17:37 smithi026 ceph-mon[179688]: pgmap v1478: 129 pgs: 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 120 active+clean; 4.0 GiB data, 17 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 832 op/s; 5663/198414 objects degraded (2.854%); 209/198414 objects misplaced (0.105%); 11 MiB/s, 168 objects/s recovering 2023-09-23T21:17:38.386 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:17:37 smithi026 ceph-mon[175151]: failed to encode map e584 with expected crc 2023-09-23T21:17:38.386 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:17:37 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-09-23T21:17:38.386 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:17:37 smithi026 ceph-mon[175151]: osdmap e584: 8 total, 8 up, 6 in 2023-09-23T21:17:38.386 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:17:37 smithi026 ceph-mon[175151]: failed to encode map e584 with expected crc 2023-09-23T21:17:38.386 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:17:37 smithi026 ceph-mon[175151]: failed to encode map e584 with expected crc 2023-09-23T21:17:38.386 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:17:37 smithi026 ceph-mon[175151]: pgmap v1478: 129 pgs: 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 120 active+clean; 4.0 GiB data, 17 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 832 op/s; 5663/198414 objects degraded (2.854%); 209/198414 objects misplaced (0.105%); 11 MiB/s, 168 objects/s recovering 2023-09-23T21:17:38.489 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:17:38.859 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:17:39.611 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 21:17:39 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[164029]: ::ffff:172.21.15.70 - - [23/Sep/2023:21:17:39] "GET /metrics HTTP/1.1" 200 36383 "" "Prometheus/2.43.0" 2023-09-23T21:17:39.928 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:17:39.934 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:17:39.935 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:17:39.955 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_historic_ops 2023-09-23T21:17:40.035 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:17:40.118 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:17:40.162 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:17:39 smithi026 ceph-mon[175151]: failed to encode map e585 with expected crc 2023-09-23T21:17:40.162 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:17:39 smithi026 ceph-mon[175151]: osdmap e585: 8 total, 8 up, 6 in 2023-09-23T21:17:40.162 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:17:39 smithi026 ceph-mon[175151]: failed to encode map e585 with expected crc 2023-09-23T21:17:40.162 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:17:39 smithi026 ceph-mon[175151]: failed to encode map e585 with expected crc 2023-09-23T21:17:40.163 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:17:39 smithi026 ceph-mon[175151]: pgmap v1480: 129 pgs: 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 120 active+clean; 4.0 GiB data, 17 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 735 op/s; 5663/197610 objects degraded (2.866%); 209/197610 objects misplaced (0.106%); 8.7 MiB/s, 139 objects/s recovering 2023-09-23T21:17:40.163 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:17:40.165 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:17:39 smithi026 ceph-mon[179688]: failed to encode map e585 with expected crc 2023-09-23T21:17:40.165 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:17:39 smithi026 ceph-mon[179688]: osdmap e585: 8 total, 8 up, 6 in 2023-09-23T21:17:40.165 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:17:39 smithi026 ceph-mon[179688]: failed to encode map e585 with expected crc 2023-09-23T21:17:40.165 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:17:39 smithi026 ceph-mon[179688]: failed to encode map e585 with expected crc 2023-09-23T21:17:40.165 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:17:39 smithi026 ceph-mon[179688]: pgmap v1480: 129 pgs: 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 120 active+clean; 4.0 GiB data, 17 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 735 op/s; 5663/197610 objects degraded (2.866%); 209/197610 objects misplaced (0.106%); 8.7 MiB/s, 139 objects/s recovering 2023-09-23T21:17:40.166 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:17:39 smithi070 ceph-mon[168282]: failed to encode map e585 with expected crc 2023-09-23T21:17:40.166 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:17:39 smithi070 ceph-mon[168282]: osdmap e585: 8 total, 8 up, 6 in 2023-09-23T21:17:40.166 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:17:39 smithi070 ceph-mon[168282]: failed to encode map e585 with expected crc 2023-09-23T21:17:40.166 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:17:39 smithi070 ceph-mon[168282]: failed to encode map e585 with expected crc 2023-09-23T21:17:40.166 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:17:39 smithi070 ceph-mon[168282]: pgmap v1480: 129 pgs: 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 120 active+clean; 4.0 GiB data, 17 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 735 op/s; 5663/197610 objects degraded (2.866%); 209/197610 objects misplaced (0.106%); 8.7 MiB/s, 139 objects/s recovering 2023-09-23T21:17:40.167 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:17:40.169 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:17:40.270 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:17:40.359 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:17:40.435 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:17:40.450 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:17:40.450 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:17:40.551 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:17:40.588 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:17:40.752 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:17:40.759 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:17:40.759 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:17:40.860 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:17:40.898 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:17:40.931 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T21:17:40.951 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:17:40.956 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:17:40.956 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:17:41.057 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:17:41.125 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:17:41.196 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:17:40 smithi070 ceph-mon[168282]: failed to encode map e586 with expected crc 2023-09-23T21:17:41.196 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:17:40 smithi070 ceph-mon[168282]: failed to encode map e586 with expected crc 2023-09-23T21:17:41.197 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:17:40 smithi070 ceph-mon[168282]: osdmap e586: 8 total, 8 up, 6 in 2023-09-23T21:17:41.197 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:17:40 smithi070 ceph-mon[168282]: failed to encode map e586 with expected crc 2023-09-23T21:17:41.197 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:17:40 smithi070 ceph-mon[168282]: failed to encode map e586 with expected crc 2023-09-23T21:17:41.197 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:17:40 smithi070 ceph-mon[168282]: failed to encode map e586 with expected crc 2023-09-23T21:17:41.197 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:17:40 smithi070 ceph-mon[168282]: failed to encode map e586 with expected crc 2023-09-23T21:17:41.197 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:17:40 smithi070 ceph-mon[168282]: Health check update: Degraded data redundancy: 5663/197610 objects degraded (2.866%), 8 pgs degraded (PG_DEGRADED) 2023-09-23T21:17:41.228 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:17:40 smithi026 ceph-mon[179688]: failed to encode map e586 with expected crc 2023-09-23T21:17:41.228 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:17:40 smithi026 ceph-mon[179688]: failed to encode map e586 with expected crc 2023-09-23T21:17:41.228 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:17:40 smithi026 ceph-mon[179688]: osdmap e586: 8 total, 8 up, 6 in 2023-09-23T21:17:41.229 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:17:40 smithi026 ceph-mon[179688]: failed to encode map e586 with expected crc 2023-09-23T21:17:41.229 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:17:40 smithi026 ceph-mon[179688]: failed to encode map e586 with expected crc 2023-09-23T21:17:41.229 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:17:40 smithi026 ceph-mon[179688]: failed to encode map e586 with expected crc 2023-09-23T21:17:41.229 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:17:40 smithi026 ceph-mon[179688]: failed to encode map e586 with expected crc 2023-09-23T21:17:41.229 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:17:40 smithi026 ceph-mon[179688]: Health check update: Degraded data redundancy: 5663/197610 objects degraded (2.866%), 8 pgs degraded (PG_DEGRADED) 2023-09-23T21:17:41.229 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:17:40 smithi026 ceph-mon[175151]: failed to encode map e586 with expected crc 2023-09-23T21:17:41.229 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:17:40 smithi026 ceph-mon[175151]: failed to encode map e586 with expected crc 2023-09-23T21:17:41.229 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:17:40 smithi026 ceph-mon[175151]: osdmap e586: 8 total, 8 up, 6 in 2023-09-23T21:17:41.229 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:17:40 smithi026 ceph-mon[175151]: failed to encode map e586 with expected crc 2023-09-23T21:17:41.230 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:17:40 smithi026 ceph-mon[175151]: failed to encode map e586 with expected crc 2023-09-23T21:17:41.230 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:17:40 smithi026 ceph-mon[175151]: failed to encode map e586 with expected crc 2023-09-23T21:17:41.230 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:17:40 smithi026 ceph-mon[175151]: failed to encode map e586 with expected crc 2023-09-23T21:17:41.230 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:17:40 smithi026 ceph-mon[175151]: Health check update: Degraded data redundancy: 5663/197610 objects degraded (2.866%), 8 pgs degraded (PG_DEGRADED) 2023-09-23T21:17:41.240 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T21:17:41.253 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:17:41.254 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:17:41.355 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:17:41.398 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:17:41.459 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:17:41.465 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:17:41.465 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:17:41.566 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:17:41.642 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:17:41.900 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:17:41.908 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:17:41.908 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:17:42.009 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:17:42.065 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:17:42.124 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:17:42.128 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:17:42.129 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:17:42.226 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:17:41 smithi070 ceph-mon[168282]: pgmap v1482: 129 pgs: 2 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 120 active+clean; 3.9 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 804 op/s; 4827/192636 objects degraded (2.506%); 912/192636 objects misplaced (0.473%); 9.7 MiB/s, 155 objects/s recovering 2023-09-23T21:17:42.230 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:17:42.292 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:17:42.381 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:17:41 smithi026 ceph-mon[175151]: pgmap v1482: 129 pgs: 2 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 120 active+clean; 3.9 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 804 op/s; 4827/192636 objects degraded (2.506%); 912/192636 objects misplaced (0.473%); 9.7 MiB/s, 155 objects/s recovering 2023-09-23T21:17:42.381 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:17:41 smithi026 ceph-mon[179688]: pgmap v1482: 129 pgs: 2 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 120 active+clean; 3.9 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 804 op/s; 4827/192636 objects degraded (2.506%); 912/192636 objects misplaced (0.473%); 9.7 MiB/s, 155 objects/s recovering 2023-09-23T21:17:42.386 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:17:42.390 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:17:42.390 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:17:42.491 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:17:42.513 INFO:teuthology.orchestra.run.smithi070.stdout:{ 2023-09-23T21:17:42.513 INFO:teuthology.orchestra.run.smithi070.stdout: "size": 20, 2023-09-23T21:17:42.513 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 600, 2023-09-23T21:17:42.513 INFO:teuthology.orchestra.run.smithi070.stdout: "ops": [ 2023-09-23T21:17:42.513 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.514 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.46328.0:74333 10.d 10:bc627dcb:::benchmark_data_smithi026_379838_object74332:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e460)", 2023-09-23T21:17:42.514 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:11:37.749827+0000", 2023-09-23T21:17:42.514 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 364.75027587199997, 2023-09-23T21:17:42.514 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.4276132540000002, 2023-09-23T21:17:42.514 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:17:42.514 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:17:42.514 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:17:42.514 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.46328", 2023-09-23T21:17:42.514 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/3748861181", 2023-09-23T21:17:42.514 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 74333 2023-09-23T21:17:42.514 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.514 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:17:42.515 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.515 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:17:42.515 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:37.749827+0000", 2023-09-23T21:17:42.515 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:17:42.515 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.515 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.515 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:17:42.515 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:37.749827+0000", 2023-09-23T21:17:42.515 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:17:42.515 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.515 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.515 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:17:42.515 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:37.749825+0000", 2023-09-23T21:17:42.516 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999981 2023-09-23T21:17:42.516 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.516 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.516 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:17:42.516 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:37.749879+0000", 2023-09-23T21:17:42.516 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.4143e-05 2023-09-23T21:17:42.516 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.516 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.516 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:17:42.516 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:37.749894+0000", 2023-09-23T21:17:42.516 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.4973e-05 2023-09-23T21:17:42.516 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.516 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.517 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:17:42.517 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:37.749903+0000", 2023-09-23T21:17:42.517 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.0059999999999998e-06 2023-09-23T21:17:42.517 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.517 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.517 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:17:42.517 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:37.749946+0000", 2023-09-23T21:17:42.517 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.2349999999999999e-05 2023-09-23T21:17:42.517 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.517 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.517 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:17:42.517 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:37.749958+0000", 2023-09-23T21:17:42.517 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.2622999999999999e-05 2023-09-23T21:17:42.518 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.518 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.518 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:17:42.518 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:38.737069+0000", 2023-09-23T21:17:42.518 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.98711088800000002 2023-09-23T21:17:42.518 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.518 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.518 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:17:42.518 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:38.737077+0000", 2023-09-23T21:17:42.518 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.0150000000000003e-06 2023-09-23T21:17:42.519 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.519 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.519 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:17:42.519 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:40.172893+0000", 2023-09-23T21:17:42.519 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.435816266 2023-09-23T21:17:42.519 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.520 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.520 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:17:42.520 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:40.173003+0000", 2023-09-23T21:17:42.520 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00010993 2023-09-23T21:17:42.520 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.520 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.521 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 0,3,7", 2023-09-23T21:17:42.521 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:40.173114+0000", 2023-09-23T21:17:42.521 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000110849 2023-09-23T21:17:42.521 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.521 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.521 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:17:42.522 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:40.175684+0000", 2023-09-23T21:17:42.522 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0025697369999999999 2023-09-23T21:17:42.522 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.522 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.522 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:17:42.523 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:40.177298+0000", 2023-09-23T21:17:42.523 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0016144969999999999 2023-09-23T21:17:42.523 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.523 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.523 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:17:42.523 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:40.177308+0000", 2023-09-23T21:17:42.524 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.8879999999999992e-06 2023-09-23T21:17:42.524 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.524 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.524 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:17:42.524 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:40.177333+0000", 2023-09-23T21:17:42.524 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.4905999999999998e-05 2023-09-23T21:17:42.525 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.525 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.525 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:17:42.525 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:40.177376+0000", 2023-09-23T21:17:42.526 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.2635000000000001e-05 2023-09-23T21:17:42.526 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.527 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.527 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:17:42.527 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:40.177440+0000", 2023-09-23T21:17:42.527 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.4252999999999996e-05 2023-09-23T21:17:42.527 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:17:42.527 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:17:42.527 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:17:42.528 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.528 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.528 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.46328.0:74356 10.d 10:ba33591e:::benchmark_data_smithi026_379838_object74355:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e460)", 2023-09-23T21:17:42.528 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:11:37.787894+0000", 2023-09-23T21:17:42.528 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 364.71220911699999, 2023-09-23T21:17:42.528 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.3971935719999999, 2023-09-23T21:17:42.528 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:17:42.529 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:17:42.529 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:17:42.529 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.46328", 2023-09-23T21:17:42.529 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/3748861181", 2023-09-23T21:17:42.529 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 74356 2023-09-23T21:17:42.529 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.530 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:17:42.530 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.530 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:17:42.530 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:37.787894+0000", 2023-09-23T21:17:42.530 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:17:42.530 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.531 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.531 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:17:42.531 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:37.787894+0000", 2023-09-23T21:17:42.531 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:17:42.531 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.531 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.531 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:17:42.531 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:37.787893+0000", 2023-09-23T21:17:42.532 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999995 2023-09-23T21:17:42.532 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.532 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.532 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:17:42.532 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:37.787916+0000", 2023-09-23T21:17:42.532 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.2857000000000002e-05 2023-09-23T21:17:42.532 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.533 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.533 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:17:42.533 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:37.787923+0000", 2023-09-23T21:17:42.533 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.6540000000000002e-06 2023-09-23T21:17:42.533 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.533 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.534 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:17:42.534 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:37.787927+0000", 2023-09-23T21:17:42.534 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.4399999999999998e-06 2023-09-23T21:17:42.534 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.534 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.534 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:17:42.535 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:37.787987+0000", 2023-09-23T21:17:42.535 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.9561000000000001e-05 2023-09-23T21:17:42.535 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.535 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.535 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:17:42.535 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:37.787997+0000", 2023-09-23T21:17:42.536 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.9189999999999994e-06 2023-09-23T21:17:42.536 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.536 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.536 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:17:42.536 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:38.737092+0000", 2023-09-23T21:17:42.536 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.94909543100000004 2023-09-23T21:17:42.537 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.537 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.537 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:17:42.537 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:38.737097+0000", 2023-09-23T21:17:42.537 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.9169999999999997e-06 2023-09-23T21:17:42.537 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.537 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.537 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:17:42.537 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:40.175712+0000", 2023-09-23T21:17:42.537 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.438615602 2023-09-23T21:17:42.537 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.537 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.537 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:17:42.538 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:40.175796+0000", 2023-09-23T21:17:42.538 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.3318000000000004e-05 2023-09-23T21:17:42.538 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.538 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.538 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 0,3,7", 2023-09-23T21:17:42.538 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:40.175911+0000", 2023-09-23T21:17:42.538 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000115275 2023-09-23T21:17:42.538 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.539 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.539 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:17:42.539 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:40.181314+0000", 2023-09-23T21:17:42.539 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.005402919 2023-09-23T21:17:42.539 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.539 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.540 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:17:42.540 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:40.181389+0000", 2023-09-23T21:17:42.540 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.4692999999999998e-05 2023-09-23T21:17:42.540 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.540 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.540 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:17:42.540 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:40.181395+0000", 2023-09-23T21:17:42.540 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.6200000000000001e-06 2023-09-23T21:17:42.540 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.540 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.541 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:17:42.541 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:40.184985+0000", 2023-09-23T21:17:42.541 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0035895900000000001 2023-09-23T21:17:42.541 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.541 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.541 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:17:42.541 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:40.185022+0000", 2023-09-23T21:17:42.541 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.7166000000000003e-05 2023-09-23T21:17:42.541 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.541 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.541 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:17:42.541 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:40.185087+0000", 2023-09-23T21:17:42.542 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.5100999999999999e-05 2023-09-23T21:17:42.542 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:17:42.542 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:17:42.542 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:17:42.542 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.542 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.542 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.46328.0:74367 10.d 10:b9926de2:::benchmark_data_smithi026_379838_object74366:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e460)", 2023-09-23T21:17:42.542 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:11:37.805093+0000", 2023-09-23T21:17:42.542 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 364.69500932, 2023-09-23T21:17:42.543 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.3801344150000001, 2023-09-23T21:17:42.543 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:17:42.543 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:17:42.543 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:17:42.543 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.46328", 2023-09-23T21:17:42.543 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/3748861181", 2023-09-23T21:17:42.543 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 74367 2023-09-23T21:17:42.543 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.543 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:17:42.544 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.544 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:17:42.544 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:37.805093+0000", 2023-09-23T21:17:42.544 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:17:42.544 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.544 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.544 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:17:42.544 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:37.805093+0000", 2023-09-23T21:17:42.544 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:17:42.544 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.545 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.545 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:17:42.545 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:37.805092+0000", 2023-09-23T21:17:42.545 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999981 2023-09-23T21:17:42.545 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.545 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.545 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:17:42.546 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:37.805182+0000", 2023-09-23T21:17:42.546 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.0195e-05 2023-09-23T21:17:42.546 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.546 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.546 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:17:42.546 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:37.805195+0000", 2023-09-23T21:17:42.546 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.2823000000000001e-05 2023-09-23T21:17:42.546 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.546 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.546 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:17:42.546 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:37.805201+0000", 2023-09-23T21:17:42.547 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.759e-06 2023-09-23T21:17:42.547 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.547 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.547 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:17:42.547 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:37.805275+0000", 2023-09-23T21:17:42.547 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.3203999999999999e-05 2023-09-23T21:17:42.548 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.548 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.548 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:17:42.548 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:37.805287+0000", 2023-09-23T21:17:42.548 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.2526e-05 2023-09-23T21:17:42.549 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.549 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.549 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:17:42.549 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:38.737108+0000", 2023-09-23T21:17:42.549 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.93182051399999999 2023-09-23T21:17:42.549 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.550 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.550 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:17:42.550 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:38.737112+0000", 2023-09-23T21:17:42.550 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.7900000000000001e-06 2023-09-23T21:17:42.550 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.551 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.551 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:17:42.551 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:40.177476+0000", 2023-09-23T21:17:42.551 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.440364829 2023-09-23T21:17:42.551 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.551 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.551 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:17:42.551 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:40.177607+0000", 2023-09-23T21:17:42.551 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00013115199999999999 2023-09-23T21:17:42.552 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.552 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.552 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 0,3,7", 2023-09-23T21:17:42.552 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:40.177728+0000", 2023-09-23T21:17:42.552 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00012074 2023-09-23T21:17:42.552 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.552 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.552 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:17:42.552 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:40.181346+0000", 2023-09-23T21:17:42.552 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0036178339999999999 2023-09-23T21:17:42.552 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.552 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.553 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:17:42.553 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:40.184839+0000", 2023-09-23T21:17:42.553 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.003493351 2023-09-23T21:17:42.553 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.553 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.553 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:17:42.553 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:40.184931+0000", 2023-09-23T21:17:42.553 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.1612000000000003e-05 2023-09-23T21:17:42.553 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.553 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.553 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:17:42.553 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:40.185147+0000", 2023-09-23T21:17:42.554 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00021584999999999999 2023-09-23T21:17:42.554 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.554 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.554 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:17:42.554 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:40.185173+0000", 2023-09-23T21:17:42.554 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.5884000000000002e-05 2023-09-23T21:17:42.554 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.554 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.554 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:17:42.554 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:40.185228+0000", 2023-09-23T21:17:42.554 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.5062000000000002e-05 2023-09-23T21:17:42.554 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:17:42.554 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:17:42.555 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:17:42.555 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.555 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.555 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.46328.0:74398 10.d 10:bdc342d2:::benchmark_data_smithi026_379838_object74397:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e460)", 2023-09-23T21:17:42.555 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:11:37.837382+0000", 2023-09-23T21:17:42.555 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 364.66272053599999, 2023-09-23T21:17:42.555 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.3480186879999998, 2023-09-23T21:17:42.555 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:17:42.555 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:17:42.555 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:17:42.555 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.46328", 2023-09-23T21:17:42.555 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/3748861181", 2023-09-23T21:17:42.556 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 74398 2023-09-23T21:17:42.556 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.556 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:17:42.556 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.556 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:17:42.556 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:37.837382+0000", 2023-09-23T21:17:42.556 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:17:42.556 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.556 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.556 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:17:42.556 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:37.837382+0000", 2023-09-23T21:17:42.556 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:17:42.557 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.557 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.557 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:17:42.557 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:37.837381+0000", 2023-09-23T21:17:42.557 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.999999 2023-09-23T21:17:42.557 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.557 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.557 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:17:42.557 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:37.837411+0000", 2023-09-23T21:17:42.557 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.0006e-05 2023-09-23T21:17:42.557 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.557 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.558 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:17:42.558 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:37.837421+0000", 2023-09-23T21:17:42.558 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.4029999999999996e-06 2023-09-23T21:17:42.558 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.558 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.558 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:17:42.558 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:37.837425+0000", 2023-09-23T21:17:42.558 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.7249999999999997e-06 2023-09-23T21:17:42.558 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.558 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.558 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:17:42.558 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:37.837477+0000", 2023-09-23T21:17:42.559 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.1430000000000001e-05 2023-09-23T21:17:42.559 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.559 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.559 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:17:42.559 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:37.837488+0000", 2023-09-23T21:17:42.559 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.1357e-05 2023-09-23T21:17:42.559 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.559 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.559 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:17:42.559 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:38.737135+0000", 2023-09-23T21:17:42.559 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.89964650899999998 2023-09-23T21:17:42.559 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.559 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.560 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:17:42.560 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:38.737138+0000", 2023-09-23T21:17:42.560 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.8709999999999999e-06 2023-09-23T21:17:42.560 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.560 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.560 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:17:42.560 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:40.179280+0000", 2023-09-23T21:17:42.560 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.442141605 2023-09-23T21:17:42.560 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.560 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.560 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:17:42.560 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:40.179376+0000", 2023-09-23T21:17:42.561 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.5909999999999995e-05 2023-09-23T21:17:42.561 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.561 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.561 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 0,3,7", 2023-09-23T21:17:42.561 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:40.179505+0000", 2023-09-23T21:17:42.561 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00012945900000000001 2023-09-23T21:17:42.561 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.561 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.561 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:17:42.561 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:40.184887+0000", 2023-09-23T21:17:42.561 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.005381884 2023-09-23T21:17:42.561 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.562 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.562 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:17:42.562 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:40.184944+0000", 2023-09-23T21:17:42.562 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.6521999999999999e-05 2023-09-23T21:17:42.562 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.562 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.562 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:17:42.562 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:40.185125+0000", 2023-09-23T21:17:42.562 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00018092 2023-09-23T21:17:42.562 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.562 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.562 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:17:42.563 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:40.185307+0000", 2023-09-23T21:17:42.563 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000181991 2023-09-23T21:17:42.563 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.563 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.563 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:17:42.563 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:40.185340+0000", 2023-09-23T21:17:42.563 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.3021000000000001e-05 2023-09-23T21:17:42.563 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.563 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.563 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:17:42.563 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:40.185401+0000", 2023-09-23T21:17:42.563 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.1158e-05 2023-09-23T21:17:42.564 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:17:42.564 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:17:42.564 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:17:42.564 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.564 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.564 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.46328.0:74400 10.d 10:b25fac0d:::benchmark_data_smithi026_379838_object74399:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e460)", 2023-09-23T21:17:42.564 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:11:37.840259+0000", 2023-09-23T21:17:42.564 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 364.65984346900001, 2023-09-23T21:17:42.564 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.345304139, 2023-09-23T21:17:42.564 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:17:42.564 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:17:42.564 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:17:42.564 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.46328", 2023-09-23T21:17:42.565 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/3748861181", 2023-09-23T21:17:42.565 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 74400 2023-09-23T21:17:42.565 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.565 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:17:42.565 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.565 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:17:42.565 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:37.840259+0000", 2023-09-23T21:17:42.565 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:17:42.565 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.565 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.565 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:17:42.565 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:37.840259+0000", 2023-09-23T21:17:42.566 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:17:42.566 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.566 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.566 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:17:42.566 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:37.840257+0000", 2023-09-23T21:17:42.566 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999981 2023-09-23T21:17:42.566 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.566 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.566 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:17:42.566 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:37.840311+0000", 2023-09-23T21:17:42.566 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.3449000000000003e-05 2023-09-23T21:17:42.566 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.566 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.567 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:17:42.567 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:37.840322+0000", 2023-09-23T21:17:42.567 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.1197e-05 2023-09-23T21:17:42.567 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.567 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.567 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:17:42.567 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:37.840328+0000", 2023-09-23T21:17:42.567 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.2129999999999996e-06 2023-09-23T21:17:42.567 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.567 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.567 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:17:42.568 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:37.840388+0000", 2023-09-23T21:17:42.568 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.0081000000000003e-05 2023-09-23T21:17:42.568 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.568 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.568 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:17:42.568 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:37.840399+0000", 2023-09-23T21:17:42.568 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.0465000000000001e-05 2023-09-23T21:17:42.568 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.568 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.568 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:17:42.568 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:38.737150+0000", 2023-09-23T21:17:42.568 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.89675132499999999 2023-09-23T21:17:42.569 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.569 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.569 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:17:42.569 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:38.737154+0000", 2023-09-23T21:17:42.569 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.6200000000000001e-06 2023-09-23T21:17:42.569 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.569 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.569 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:17:42.569 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:40.181411+0000", 2023-09-23T21:17:42.569 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.444257009 2023-09-23T21:17:42.569 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.569 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.570 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:17:42.570 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:40.181497+0000", 2023-09-23T21:17:42.570 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.6570000000000006e-05 2023-09-23T21:17:42.570 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.570 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.570 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 0,3,7", 2023-09-23T21:17:42.570 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:40.181626+0000", 2023-09-23T21:17:42.570 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00012901699999999999 2023-09-23T21:17:42.570 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.570 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.570 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:17:42.570 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:40.184950+0000", 2023-09-23T21:17:42.571 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.003323501 2023-09-23T21:17:42.571 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.571 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.571 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:17:42.571 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:40.185262+0000", 2023-09-23T21:17:42.571 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000312045 2023-09-23T21:17:42.571 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.571 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.571 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:17:42.571 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:40.185281+0000", 2023-09-23T21:17:42.571 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.9055999999999998e-05 2023-09-23T21:17:42.571 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.572 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.572 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:17:42.572 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:40.185481+0000", 2023-09-23T21:17:42.572 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00019994199999999999 2023-09-23T21:17:42.572 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.572 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.572 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:17:42.572 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:40.185508+0000", 2023-09-23T21:17:42.572 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.6801e-05 2023-09-23T21:17:42.572 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.572 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.572 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:17:42.573 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:40.185563+0000", 2023-09-23T21:17:42.573 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.5670000000000001e-05 2023-09-23T21:17:42.573 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:17:42.573 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:17:42.573 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:17:42.573 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.573 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.573 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.46328.0:74403 10.d 10:b5a2ad18:::benchmark_data_smithi026_379838_object74402:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e460)", 2023-09-23T21:17:42.573 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:11:37.844596+0000", 2023-09-23T21:17:42.573 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 364.65550696499997, 2023-09-23T21:17:42.573 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.3410775579999998, 2023-09-23T21:17:42.574 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:17:42.574 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:17:42.574 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:17:42.574 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.46328", 2023-09-23T21:17:42.574 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/3748861181", 2023-09-23T21:17:42.574 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 74403 2023-09-23T21:17:42.574 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.574 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:17:42.574 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.574 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:17:42.574 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:37.844596+0000", 2023-09-23T21:17:42.574 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:17:42.575 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.575 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.575 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:17:42.575 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:37.844596+0000", 2023-09-23T21:17:42.575 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:17:42.575 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.575 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.575 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:17:42.575 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:37.844594+0000", 2023-09-23T21:17:42.575 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T21:17:42.575 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.575 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.576 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:17:42.576 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:37.844658+0000", 2023-09-23T21:17:42.576 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.4053000000000005e-05 2023-09-23T21:17:42.576 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.576 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.576 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:17:42.576 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:37.844675+0000", 2023-09-23T21:17:42.576 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.6670000000000001e-05 2023-09-23T21:17:42.576 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.576 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.576 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:17:42.576 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:37.844682+0000", 2023-09-23T21:17:42.576 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.7839999999999998e-06 2023-09-23T21:17:42.577 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.577 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.577 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:17:42.577 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:37.844751+0000", 2023-09-23T21:17:42.577 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.9698999999999998e-05 2023-09-23T21:17:42.577 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.577 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.577 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:17:42.577 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:37.844762+0000", 2023-09-23T21:17:42.577 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.049e-05 2023-09-23T21:17:42.577 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.577 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.578 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:17:42.578 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:38.737168+0000", 2023-09-23T21:17:42.578 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.89240619799999998 2023-09-23T21:17:42.578 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.578 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.578 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:17:42.578 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:38.737174+0000", 2023-09-23T21:17:42.578 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.3319999999999997e-06 2023-09-23T21:17:42.578 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.578 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.578 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:17:42.578 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:40.183023+0000", 2023-09-23T21:17:42.579 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.445848958 2023-09-23T21:17:42.579 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.579 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.579 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:17:42.579 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:40.183163+0000", 2023-09-23T21:17:42.579 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00013908200000000001 2023-09-23T21:17:42.579 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.579 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.579 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 0,3,7", 2023-09-23T21:17:42.579 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:40.183342+0000", 2023-09-23T21:17:42.579 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000179218 2023-09-23T21:17:42.579 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.579 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.580 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:17:42.580 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:40.185290+0000", 2023-09-23T21:17:42.580 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0019481769999999999 2023-09-23T21:17:42.580 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.580 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.580 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:17:42.580 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:40.185432+0000", 2023-09-23T21:17:42.580 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000142435 2023-09-23T21:17:42.580 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.580 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.580 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:17:42.580 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:40.185457+0000", 2023-09-23T21:17:42.580 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.5054000000000001e-05 2023-09-23T21:17:42.581 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.581 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.581 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:17:42.581 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:40.185592+0000", 2023-09-23T21:17:42.581 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00013438499999999999 2023-09-23T21:17:42.581 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.581 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.581 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:17:42.581 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:40.185619+0000", 2023-09-23T21:17:42.581 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.6727e-05 2023-09-23T21:17:42.581 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.581 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.582 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:17:42.582 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:40.185673+0000", 2023-09-23T21:17:42.582 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.4796000000000002e-05 2023-09-23T21:17:42.582 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:17:42.582 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:17:42.582 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:17:42.582 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.582 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.582 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.46328.0:74404 10.d 10:bd0dcc09:::benchmark_data_smithi026_379838_object74403:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e460)", 2023-09-23T21:17:42.582 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:11:37.844695+0000", 2023-09-23T21:17:42.582 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 364.65540782699998, 2023-09-23T21:17:42.582 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.3484742669999998, 2023-09-23T21:17:42.583 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:17:42.583 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:17:42.583 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:17:42.583 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.46328", 2023-09-23T21:17:42.583 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/3748861181", 2023-09-23T21:17:42.583 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 74404 2023-09-23T21:17:42.583 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.583 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:17:42.583 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.583 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:17:42.583 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:37.844695+0000", 2023-09-23T21:17:42.583 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:17:42.583 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.584 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.584 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:17:42.584 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:37.844695+0000", 2023-09-23T21:17:42.584 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:17:42.584 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.584 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.584 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:17:42.584 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:37.844695+0000", 2023-09-23T21:17:42.584 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999995 2023-09-23T21:17:42.584 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.584 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.584 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:17:42.585 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:37.844719+0000", 2023-09-23T21:17:42.585 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.4070000000000002e-05 2023-09-23T21:17:42.585 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.585 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.585 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:17:42.585 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:37.844726+0000", 2023-09-23T21:17:42.585 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.3459999999999996e-06 2023-09-23T21:17:42.585 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.585 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.585 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:17:42.585 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:37.844730+0000", 2023-09-23T21:17:42.585 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.5920000000000001e-06 2023-09-23T21:17:42.585 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.586 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.586 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:17:42.586 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:37.844800+0000", 2023-09-23T21:17:42.586 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.0759000000000005e-05 2023-09-23T21:17:42.586 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.586 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.586 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:17:42.586 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:37.844812+0000", 2023-09-23T21:17:42.586 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.2130000000000001e-05 2023-09-23T21:17:42.586 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.586 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.586 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:17:42.587 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:38.737187+0000", 2023-09-23T21:17:42.587 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.89237498199999998 2023-09-23T21:17:42.587 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.587 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.587 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:17:42.587 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:38.737191+0000", 2023-09-23T21:17:42.587 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.7189999999999999e-06 2023-09-23T21:17:42.587 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.587 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.587 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:17:42.587 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:40.185696+0000", 2023-09-23T21:17:42.587 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.4485052139999999 2023-09-23T21:17:42.588 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.588 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.588 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:17:42.588 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:40.185770+0000", 2023-09-23T21:17:42.588 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.3560999999999996e-05 2023-09-23T21:17:42.588 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.588 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.588 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 0,3,7", 2023-09-23T21:17:42.588 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:40.185864+0000", 2023-09-23T21:17:42.588 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.4244000000000001e-05 2023-09-23T21:17:42.588 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.588 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.588 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:17:42.589 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:40.192926+0000", 2023-09-23T21:17:42.589 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0070613910000000002 2023-09-23T21:17:42.589 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.589 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.589 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:17:42.589 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:40.192930+0000", 2023-09-23T21:17:42.589 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.8080000000000004e-06 2023-09-23T21:17:42.589 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.589 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.589 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:17:42.589 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:40.192975+0000", 2023-09-23T21:17:42.589 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.4990000000000001e-05 2023-09-23T21:17:42.590 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.590 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.590 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:17:42.590 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:40.193012+0000", 2023-09-23T21:17:42.590 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.6443000000000003e-05 2023-09-23T21:17:42.590 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.590 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.590 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:17:42.590 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:40.193069+0000", 2023-09-23T21:17:42.590 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.7065000000000003e-05 2023-09-23T21:17:42.590 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.590 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.591 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:17:42.591 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:40.193169+0000", 2023-09-23T21:17:42.591 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000100315 2023-09-23T21:17:42.591 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:17:42.591 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:17:42.591 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:17:42.591 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.591 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.591 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_repop(client.46328.0:84573 10.c e471/465 10:3fce4c21:::benchmark_data_smithi026_379838_object4616:head v 471'5331, mlcod=470'5329)", 2023-09-23T21:17:42.591 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:12:10.780185+0000", 2023-09-23T21:17:42.591 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 331.71991809000002, 2023-09-23T21:17:42.591 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.5674745780000001, 2023-09-23T21:17:42.592 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:17:42.592 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:17:42.592 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:17:42.592 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.592 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:17:42.592 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:12:10.780185+0000", 2023-09-23T21:17:42.592 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:17:42.592 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.592 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.592 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:17:42.592 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:12:10.780185+0000", 2023-09-23T21:17:42.592 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:17:42.593 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.593 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.593 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:17:42.593 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:12:10.780184+0000", 2023-09-23T21:17:42.593 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999995 2023-09-23T21:17:42.593 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.593 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.593 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:17:42.593 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:12:10.780189+0000", 2023-09-23T21:17:42.593 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.7650000000000001e-06 2023-09-23T21:17:42.593 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.593 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.594 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:17:42.594 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:12:10.780196+0000", 2023-09-23T21:17:42.594 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.9040000000000003e-06 2023-09-23T21:17:42.594 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.594 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.594 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:17:42.594 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:12:10.780203+0000", 2023-09-23T21:17:42.594 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.3690000000000001e-06 2023-09-23T21:17:42.594 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.594 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.594 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:17:42.594 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:12:14.336808+0000", 2023-09-23T21:17:42.594 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.556604718 2023-09-23T21:17:42.595 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.595 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.595 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:17:42.595 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:12:14.336836+0000", 2023-09-23T21:17:42.595 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.7379e-05 2023-09-23T21:17:42.595 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.595 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.595 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:17:42.595 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:12:14.347611+0000", 2023-09-23T21:17:42.595 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.01077504 2023-09-23T21:17:42.595 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.595 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.595 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:17:42.596 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:12:14.347659+0000", 2023-09-23T21:17:42.596 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.8668000000000002e-05 2023-09-23T21:17:42.596 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:17:42.596 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:17:42.596 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:17:42.596 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.596 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.596 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_repop(client.46328.0:84576 10.c e471/465 10:3b2a76db:::benchmark_data_smithi026_379838_object4619:head v 471'5332, mlcod=470'5329)", 2023-09-23T21:17:42.596 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:12:10.849370+0000", 2023-09-23T21:17:42.596 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 331.65073260999998, 2023-09-23T21:17:42.596 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.503394407, 2023-09-23T21:17:42.596 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:17:42.596 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:17:42.597 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:17:42.597 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.597 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:17:42.597 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:12:10.849370+0000", 2023-09-23T21:17:42.597 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:17:42.597 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.597 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.597 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:17:42.597 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:12:10.849370+0000", 2023-09-23T21:17:42.597 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:17:42.597 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.597 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.597 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:17:42.598 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:12:10.849369+0000", 2023-09-23T21:17:42.598 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.999999 2023-09-23T21:17:42.598 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.598 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.598 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:17:42.598 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:12:10.849380+0000", 2023-09-23T21:17:42.598 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.0987e-05 2023-09-23T21:17:42.598 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.598 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.598 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:17:42.598 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:12:10.849392+0000", 2023-09-23T21:17:42.598 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.1843e-05 2023-09-23T21:17:42.598 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.599 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.599 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:17:42.599 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:12:10.849399+0000", 2023-09-23T21:17:42.599 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.8549999999999996e-06 2023-09-23T21:17:42.599 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.599 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.599 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:17:42.599 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:12:14.345110+0000", 2023-09-23T21:17:42.599 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.4957117150000001 2023-09-23T21:17:42.599 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.599 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.599 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:17:42.600 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:12:14.345139+0000", 2023-09-23T21:17:42.600 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.8362999999999999e-05 2023-09-23T21:17:42.600 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.600 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.600 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:17:42.600 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:12:14.352724+0000", 2023-09-23T21:17:42.600 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0075854609999999999 2023-09-23T21:17:42.600 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.600 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.600 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:17:42.600 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:12:14.352765+0000", 2023-09-23T21:17:42.600 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.0368999999999999e-05 2023-09-23T21:17:42.600 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:17:42.601 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:17:42.601 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:17:42.601 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.601 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.601 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.46328.0:84587 10.0 10:01ac773b:::benchmark_data_smithi026_379838_object4630:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e472)", 2023-09-23T21:17:42.601 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:12:11.856050+0000", 2023-09-23T21:17:42.601 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 330.64405315099998, 2023-09-23T21:17:42.601 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.9827590000000002, 2023-09-23T21:17:42.601 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:17:42.601 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:17:42.601 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:17:42.601 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.46328", 2023-09-23T21:17:42.601 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/3748861181", 2023-09-23T21:17:42.602 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 84587 2023-09-23T21:17:42.602 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.602 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:17:42.602 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.602 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:17:42.602 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:12:11.856050+0000", 2023-09-23T21:17:42.602 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:17:42.602 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.602 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.602 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:17:42.602 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:12:11.856050+0000", 2023-09-23T21:17:42.602 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:17:42.603 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.603 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.603 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:17:42.603 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:12:11.856048+0000", 2023-09-23T21:17:42.603 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.999999 2023-09-23T21:17:42.603 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.603 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.603 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:17:42.603 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:12:11.856055+0000", 2023-09-23T21:17:42.603 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.883e-06 2023-09-23T21:17:42.603 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.603 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.604 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:17:42.604 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:12:11.856067+0000", 2023-09-23T21:17:42.604 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.1541e-05 2023-09-23T21:17:42.604 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.604 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.604 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:17:42.604 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:12:11.856073+0000", 2023-09-23T21:17:42.604 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.348e-06 2023-09-23T21:17:42.604 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.604 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.604 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:17:42.604 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:12:14.482257+0000", 2023-09-23T21:17:42.605 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.62618397 2023-09-23T21:17:42.605 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.605 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.605 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for missing object", 2023-09-23T21:17:42.605 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:12:14.486854+0000", 2023-09-23T21:17:42.605 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.004597126 2023-09-23T21:17:42.605 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.605 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.605 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:17:42.605 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:12:14.521249+0000", 2023-09-23T21:17:42.605 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.034394451999999999 2023-09-23T21:17:42.605 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.606 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.606 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for rw locks", 2023-09-23T21:17:42.606 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:12:14.527211+0000", 2023-09-23T21:17:42.606 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0059623009999999997 2023-09-23T21:17:42.606 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.606 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.606 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:17:42.606 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:12:14.800609+0000", 2023-09-23T21:17:42.606 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.273397942 2023-09-23T21:17:42.606 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.606 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.606 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:17:42.607 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:12:14.803332+0000", 2023-09-23T21:17:42.607 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0027228970000000002 2023-09-23T21:17:42.607 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.607 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.607 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 5,7", 2023-09-23T21:17:42.607 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:12:14.808034+0000", 2023-09-23T21:17:42.607 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0047023059999999998 2023-09-23T21:17:42.607 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.607 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.608 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:17:42.608 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:12:14.814675+0000", 2023-09-23T21:17:42.608 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0066411070000000003 2023-09-23T21:17:42.608 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.608 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.608 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:17:42.608 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:12:14.827001+0000", 2023-09-23T21:17:42.608 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.012326043 2023-09-23T21:17:42.608 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.608 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.608 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:17:42.608 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:12:14.828326+0000", 2023-09-23T21:17:42.608 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.001324483 2023-09-23T21:17:42.609 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.609 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.609 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:17:42.609 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:12:14.830512+0000", 2023-09-23T21:17:42.609 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0021857640000000002 2023-09-23T21:17:42.609 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.609 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.609 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:17:42.609 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:12:14.838809+0000", 2023-09-23T21:17:42.609 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0082969350000000001 2023-09-23T21:17:42.609 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:17:42.609 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:17:42.610 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:17:42.610 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.610 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.610 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.46328.0:84588 10.0 10:04267038:::benchmark_data_smithi026_379838_object4631:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e472)", 2023-09-23T21:17:42.610 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:12:11.858701+0000", 2023-09-23T21:17:42.610 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 330.64140186899999, 2023-09-23T21:17:42.610 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.915373185, 2023-09-23T21:17:42.610 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:17:42.610 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:17:42.610 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:17:42.610 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.46328", 2023-09-23T21:17:42.610 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/3748861181", 2023-09-23T21:17:42.611 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 84588 2023-09-23T21:17:42.611 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.611 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:17:42.611 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.611 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:17:42.611 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:12:11.858701+0000", 2023-09-23T21:17:42.611 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:17:42.611 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.611 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.611 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:17:42.611 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:12:11.858701+0000", 2023-09-23T21:17:42.611 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:17:42.611 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.612 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.612 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:17:42.612 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:12:11.858700+0000", 2023-09-23T21:17:42.612 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T21:17:42.612 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.612 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.612 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:17:42.612 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:12:11.858704+0000", 2023-09-23T21:17:42.612 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.9200000000000003e-06 2023-09-23T21:17:42.612 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.612 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.612 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:17:42.613 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:12:11.858714+0000", 2023-09-23T21:17:42.613 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.5719999999999992e-06 2023-09-23T21:17:42.613 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.613 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.613 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:17:42.613 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:12:11.858719+0000", 2023-09-23T21:17:42.613 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.3510000000000001e-06 2023-09-23T21:17:42.613 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.613 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.613 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:17:42.613 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:12:14.494615+0000", 2023-09-23T21:17:42.613 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.6358956099999999 2023-09-23T21:17:42.613 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.614 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.614 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for missing object", 2023-09-23T21:17:42.614 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:12:14.499085+0000", 2023-09-23T21:17:42.614 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0044701410000000004 2023-09-23T21:17:42.614 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.614 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.614 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:17:42.614 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:12:14.544838+0000", 2023-09-23T21:17:42.614 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.045753291000000001 2023-09-23T21:17:42.614 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.614 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.614 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for rw locks", 2023-09-23T21:17:42.615 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:12:14.550086+0000", 2023-09-23T21:17:42.615 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00524771 2023-09-23T21:17:42.615 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.615 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.615 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:17:42.615 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:12:14.703126+0000", 2023-09-23T21:17:42.615 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.15303945499999999 2023-09-23T21:17:42.615 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.615 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.615 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:17:42.615 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:12:14.708052+0000", 2023-09-23T21:17:42.615 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0049264809999999999 2023-09-23T21:17:42.615 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.616 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.616 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 5,7", 2023-09-23T21:17:42.616 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:12:14.713464+0000", 2023-09-23T21:17:42.616 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0054116909999999997 2023-09-23T21:17:42.616 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.616 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.616 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:17:42.616 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:12:14.735488+0000", 2023-09-23T21:17:42.616 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.022024681000000001 2023-09-23T21:17:42.616 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.616 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.616 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:17:42.616 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:12:14.765728+0000", 2023-09-23T21:17:42.617 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.030239499999999999 2023-09-23T21:17:42.617 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.617 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.617 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:17:42.617 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:12:14.766660+0000", 2023-09-23T21:17:42.617 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00093233900000000002 2023-09-23T21:17:42.617 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.617 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.617 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:17:42.617 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:12:14.768085+0000", 2023-09-23T21:17:42.617 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.001425001 2023-09-23T21:17:42.617 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.617 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.618 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:17:42.618 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:12:14.774074+0000", 2023-09-23T21:17:42.618 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0059887650000000001 2023-09-23T21:17:42.618 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:17:42.618 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:17:42.618 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:17:42.618 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.618 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.618 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_repop(client.46328.0:84616 10.c e472/465 10:3d3dd8d1:::benchmark_data_smithi026_379838_object4659:head v 472'5333, mlcod=470'5329)", 2023-09-23T21:17:42.618 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:12:11.948252+0000", 2023-09-23T21:17:42.618 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 330.55185103000002, 2023-09-23T21:17:42.618 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.4124408110000002, 2023-09-23T21:17:42.618 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:17:42.619 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:17:42.619 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:17:42.619 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.619 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:17:42.619 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:12:11.948252+0000", 2023-09-23T21:17:42.619 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:17:42.619 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.619 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.619 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:17:42.619 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:12:11.948252+0000", 2023-09-23T21:17:42.619 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:17:42.619 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.620 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.620 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:17:42.620 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:12:11.948251+0000", 2023-09-23T21:17:42.620 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.999999 2023-09-23T21:17:42.620 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.620 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.620 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:17:42.620 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:12:11.948260+0000", 2023-09-23T21:17:42.620 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.5789999999999993e-06 2023-09-23T21:17:42.620 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.620 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.620 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:17:42.620 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:12:11.948271+0000", 2023-09-23T21:17:42.621 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.0705e-05 2023-09-23T21:17:42.621 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.621 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.621 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:17:42.621 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:12:11.948278+0000", 2023-09-23T21:17:42.621 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.584e-06 2023-09-23T21:17:42.621 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.621 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.621 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:17:42.621 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:12:14.350144+0000", 2023-09-23T21:17:42.621 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.4018661539999999 2023-09-23T21:17:42.622 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.622 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.622 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:17:42.622 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:12:14.350169+0000", 2023-09-23T21:17:42.622 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.5304999999999999e-05 2023-09-23T21:17:42.622 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.622 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.622 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:17:42.622 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:12:14.360651+0000", 2023-09-23T21:17:42.622 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.010481656000000001 2023-09-23T21:17:42.622 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.622 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.623 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:17:42.623 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:12:14.360693+0000", 2023-09-23T21:17:42.623 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.1706999999999999e-05 2023-09-23T21:17:42.623 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:17:42.623 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:17:42.623 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:17:42.623 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.623 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.623 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_repop(client.46328.0:84636 10.c e472/465 10:3153f2e1:::benchmark_data_smithi026_379838_object4679:head v 472'5334, mlcod=470'5329)", 2023-09-23T21:17:42.623 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:12:12.046281+0000", 2023-09-23T21:17:42.623 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 330.453821957, 2023-09-23T21:17:42.623 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.3167927970000002, 2023-09-23T21:17:42.623 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:17:42.624 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:17:42.624 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:17:42.624 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.624 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:17:42.624 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:12:12.046281+0000", 2023-09-23T21:17:42.624 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:17:42.624 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.624 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.624 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:17:42.624 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:12:12.046281+0000", 2023-09-23T21:17:42.624 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:17:42.624 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.624 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.625 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:17:42.625 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:12:12.046280+0000", 2023-09-23T21:17:42.625 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T21:17:42.625 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.625 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.625 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:17:42.625 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:12:12.046291+0000", 2023-09-23T21:17:42.625 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.1494e-05 2023-09-23T21:17:42.625 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.625 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.625 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:17:42.625 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:12:12.046307+0000", 2023-09-23T21:17:42.625 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.5900999999999999e-05 2023-09-23T21:17:42.626 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.626 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.626 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:17:42.626 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:12:12.046315+0000", 2023-09-23T21:17:42.626 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.9740000000000002e-06 2023-09-23T21:17:42.626 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.626 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.626 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:17:42.626 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:12:14.355294+0000", 2023-09-23T21:17:42.626 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.308979511 2023-09-23T21:17:42.626 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.626 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.627 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:17:42.627 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:12:14.355318+0000", 2023-09-23T21:17:42.627 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.3159999999999998e-05 2023-09-23T21:17:42.627 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.627 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.627 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:17:42.627 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:12:14.363038+0000", 2023-09-23T21:17:42.627 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0077202060000000003 2023-09-23T21:17:42.627 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.627 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.627 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:17:42.627 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:12:14.363074+0000", 2023-09-23T21:17:42.627 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.5750999999999997e-05 2023-09-23T21:17:42.628 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:17:42.628 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:17:42.628 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:17:42.628 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.628 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.628 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.46328.0:99949 10.13 10:c89660e9:::benchmark_data_smithi026_379838_object19992:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e492)", 2023-09-23T21:17:42.628 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:13:11.118271+0000", 2023-09-23T21:17:42.628 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 271.38183183299998, 2023-09-23T21:17:42.628 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.4608746269999999, 2023-09-23T21:17:42.628 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:17:42.628 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "reached pg", 2023-09-23T21:17:42.628 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:17:42.629 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.46328", 2023-09-23T21:17:42.629 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/3748861181", 2023-09-23T21:17:42.629 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 99949 2023-09-23T21:17:42.629 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.629 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:17:42.629 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.629 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:17:42.630 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:11.118271+0000", 2023-09-23T21:17:42.630 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:17:42.630 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.631 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.631 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:17:42.631 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:11.118271+0000", 2023-09-23T21:17:42.631 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:17:42.631 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.631 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.631 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:17:42.631 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:11.118270+0000", 2023-09-23T21:17:42.631 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T21:17:42.633 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.633 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.633 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:17:42.633 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:11.118275+0000", 2023-09-23T21:17:42.633 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.9799999999999998e-06 2023-09-23T21:17:42.633 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.634 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.634 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:17:42.634 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:11.118284+0000", 2023-09-23T21:17:42.634 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.0950000000000002e-06 2023-09-23T21:17:42.634 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.634 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.634 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:17:42.634 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:11.118288+0000", 2023-09-23T21:17:42.634 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.826e-06 2023-09-23T21:17:42.634 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.634 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.634 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:17:42.635 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:11.119293+0000", 2023-09-23T21:17:42.635 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.001004291 2023-09-23T21:17:42.635 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.635 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.635 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for missing object", 2023-09-23T21:17:42.635 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:11.122652+0000", 2023-09-23T21:17:42.635 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0033593080000000001 2023-09-23T21:17:42.635 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.635 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.635 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:17:42.635 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:11.143684+0000", 2023-09-23T21:17:42.635 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.021032241 2023-09-23T21:17:42.636 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.636 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.636 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for rw locks", 2023-09-23T21:17:42.636 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:11.147201+0000", 2023-09-23T21:17:42.636 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0035162399999999999 2023-09-23T21:17:42.636 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.636 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.636 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:17:42.636 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:11.162703+0000", 2023-09-23T21:17:42.636 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.015502653999999999 2023-09-23T21:17:42.637 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.637 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.637 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:17:42.637 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:11.165461+0000", 2023-09-23T21:17:42.637 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.002758 2023-09-23T21:17:42.637 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.637 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.637 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 2,5", 2023-09-23T21:17:42.637 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:11.172337+0000", 2023-09-23T21:17:42.637 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0068755500000000002 2023-09-23T21:17:42.637 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.637 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.638 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:17:42.638 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:11.195056+0000", 2023-09-23T21:17:42.638 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.022719261000000001 2023-09-23T21:17:42.638 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.638 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.638 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:17:42.638 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:11.214701+0000", 2023-09-23T21:17:42.638 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.019645477000000001 2023-09-23T21:17:42.638 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.638 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.638 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:17:42.639 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:11.214712+0000", 2023-09-23T21:17:42.639 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.0464e-05 2023-09-23T21:17:42.639 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.639 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.639 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:17:42.639 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:12.192752+0000", 2023-09-23T21:17:42.639 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.97804052900000005 2023-09-23T21:17:42.639 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.639 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.639 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:17:42.639 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:12.192762+0000", 2023-09-23T21:17:42.639 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.9469999999999998e-06 2023-09-23T21:17:42.639 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.640 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.640 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:17:42.640 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:14.334672+0000", 2023-09-23T21:17:42.640 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.1419095559999999 2023-09-23T21:17:42.640 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.640 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.640 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for missing object", 2023-09-23T21:17:42.640 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:14.338733+0000", 2023-09-23T21:17:42.640 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0040611099999999997 2023-09-23T21:17:42.640 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.640 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.640 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:17:42.641 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:15.576529+0000", 2023-09-23T21:17:42.641 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.237796049 2023-09-23T21:17:42.641 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.641 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.641 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:17:42.641 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:15.579146+0000", 2023-09-23T21:17:42.641 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0026163979999999998 2023-09-23T21:17:42.641 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:17:42.641 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:17:42.641 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:17:42.641 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.641 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.642 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.46328.0:99952 10.1 10:86c6d668:::benchmark_data_smithi026_379838_object19995:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e492)", 2023-09-23T21:17:42.642 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:13:11.126669+0000", 2023-09-23T21:17:42.642 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 271.37343364899999, 2023-09-23T21:17:42.642 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.242467735, 2023-09-23T21:17:42.642 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:17:42.642 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:17:42.642 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:17:42.642 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.46328", 2023-09-23T21:17:42.642 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/3748861181", 2023-09-23T21:17:42.642 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 99952 2023-09-23T21:17:42.642 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.642 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:17:42.643 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.643 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:17:42.643 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:11.126669+0000", 2023-09-23T21:17:42.643 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:17:42.643 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.643 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.643 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:17:42.643 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:11.126669+0000", 2023-09-23T21:17:42.643 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:17:42.643 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.643 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.643 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:17:42.643 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:11.126667+0000", 2023-09-23T21:17:42.644 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999981 2023-09-23T21:17:42.644 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.644 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.644 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:17:42.644 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:11.126674+0000", 2023-09-23T21:17:42.644 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.9650000000000002e-06 2023-09-23T21:17:42.644 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.644 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.644 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:17:42.644 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:11.126691+0000", 2023-09-23T21:17:42.644 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.7359000000000001e-05 2023-09-23T21:17:42.644 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.645 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.645 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:17:42.645 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:11.126698+0000", 2023-09-23T21:17:42.645 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.1029999999999996e-06 2023-09-23T21:17:42.645 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.645 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.645 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:17:42.645 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:11.136504+0000", 2023-09-23T21:17:42.645 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0098059319999999998 2023-09-23T21:17:42.645 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.645 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.645 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for missing object", 2023-09-23T21:17:42.646 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:11.139550+0000", 2023-09-23T21:17:42.646 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0030458830000000001 2023-09-23T21:17:42.646 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.646 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.646 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:17:42.646 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:11.160803+0000", 2023-09-23T21:17:42.646 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.02125318 2023-09-23T21:17:42.646 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.646 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.646 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for rw locks", 2023-09-23T21:17:42.646 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:11.165287+0000", 2023-09-23T21:17:42.646 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0044835869999999998 2023-09-23T21:17:42.647 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.649 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.649 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:17:42.649 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:11.202380+0000", 2023-09-23T21:17:42.649 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.037092768999999998 2023-09-23T21:17:42.649 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.649 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.649 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:17:42.650 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:11.202394+0000", 2023-09-23T21:17:42.650 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.3949e-05 2023-09-23T21:17:42.650 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.650 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.650 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:17:42.650 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:12.193673+0000", 2023-09-23T21:17:42.650 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.99127966099999998 2023-09-23T21:17:42.650 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.650 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.650 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:17:42.650 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:12.193684+0000", 2023-09-23T21:17:42.650 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.0499000000000001e-05 2023-09-23T21:17:42.651 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.651 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.651 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:17:42.651 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:15.335139+0000", 2023-09-23T21:17:42.651 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.1414547779999999 2023-09-23T21:17:42.651 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.651 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.651 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:17:42.651 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:15.338252+0000", 2023-09-23T21:17:42.651 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.003112821 2023-09-23T21:17:42.651 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.651 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.651 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 0,7", 2023-09-23T21:17:42.652 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:15.342990+0000", 2023-09-23T21:17:42.652 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0047385450000000003 2023-09-23T21:17:42.652 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.652 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.652 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:17:42.652 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:15.355654+0000", 2023-09-23T21:17:42.652 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.012663496 2023-09-23T21:17:42.652 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.652 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.652 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:17:42.652 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:15.361946+0000", 2023-09-23T21:17:42.652 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0062922300000000002 2023-09-23T21:17:42.653 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.653 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.653 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:17:42.653 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:15.362731+0000", 2023-09-23T21:17:42.653 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00078492199999999996 2023-09-23T21:17:42.653 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.653 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.653 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:17:42.653 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:15.363991+0000", 2023-09-23T21:17:42.653 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.001260082 2023-09-23T21:17:42.653 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.653 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.653 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:17:42.654 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:15.369137+0000", 2023-09-23T21:17:42.654 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.005146014 2023-09-23T21:17:42.654 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:17:42.654 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:17:42.654 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:17:42.654 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.654 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.654 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.46328.0:99963 10.13 10:c9506fe5:::benchmark_data_smithi026_379838_object20006:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e492)", 2023-09-23T21:17:42.654 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:13:11.156341+0000", 2023-09-23T21:17:42.654 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 271.34376207499997, 2023-09-23T21:17:42.654 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.6561101320000002, 2023-09-23T21:17:42.654 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:17:42.655 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:17:42.655 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:17:42.655 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.46328", 2023-09-23T21:17:42.655 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/3748861181", 2023-09-23T21:17:42.655 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 99963 2023-09-23T21:17:42.655 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.655 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:17:42.655 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.655 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:17:42.655 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:11.156341+0000", 2023-09-23T21:17:42.655 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:17:42.655 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.655 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.656 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:17:42.656 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:11.156341+0000", 2023-09-23T21:17:42.656 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:17:42.656 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.656 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.656 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:17:42.656 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:11.156339+0000", 2023-09-23T21:17:42.657 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999981 2023-09-23T21:17:42.657 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.657 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.658 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:17:42.658 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:11.156347+0000", 2023-09-23T21:17:42.658 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.0439999999999996e-06 2023-09-23T21:17:42.658 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.658 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.658 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:17:42.658 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:11.156363+0000", 2023-09-23T21:17:42.658 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.5798e-05 2023-09-23T21:17:42.658 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.658 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.658 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:17:42.658 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:11.156370+0000", 2023-09-23T21:17:42.659 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.6899999999999992e-06 2023-09-23T21:17:42.659 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.659 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.659 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:17:42.659 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:11.191653+0000", 2023-09-23T21:17:42.659 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.035282273000000003 2023-09-23T21:17:42.659 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.659 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.659 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for missing object", 2023-09-23T21:17:42.659 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:11.194637+0000", 2023-09-23T21:17:42.659 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.002984586 2023-09-23T21:17:42.659 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.660 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.660 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:17:42.660 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:11.214739+0000", 2023-09-23T21:17:42.660 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.020102346 2023-09-23T21:17:42.660 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.660 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.660 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:17:42.660 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:11.214746+0000", 2023-09-23T21:17:42.660 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.9419999999999998e-06 2023-09-23T21:17:42.660 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.660 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.660 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:17:42.661 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:12.192789+0000", 2023-09-23T21:17:42.661 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.97804239400000004 2023-09-23T21:17:42.661 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.661 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.661 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:17:42.661 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:12.192796+0000", 2023-09-23T21:17:42.661 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.6290000000000001e-06 2023-09-23T21:17:42.661 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.661 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.661 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:17:42.661 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:15.579652+0000", 2023-09-23T21:17:42.661 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.3868554130000001 2023-09-23T21:17:42.662 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.662 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.662 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for missing object", 2023-09-23T21:17:42.662 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:15.583515+0000", 2023-09-23T21:17:42.662 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0038636059999999999 2023-09-23T21:17:42.662 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.662 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.662 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:17:42.662 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:15.604488+0000", 2023-09-23T21:17:42.662 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.020972781999999999 2023-09-23T21:17:42.662 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.662 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.663 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for degraded object", 2023-09-23T21:17:42.663 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:15.607187+0000", 2023-09-23T21:17:42.663 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0026986549999999999 2023-09-23T21:17:42.663 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.663 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.663 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:17:42.663 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:15.742812+0000", 2023-09-23T21:17:42.663 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.13562508700000001 2023-09-23T21:17:42.663 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.663 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.663 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:17:42.664 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:15.746122+0000", 2023-09-23T21:17:42.664 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0033102119999999999 2023-09-23T21:17:42.664 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.664 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.664 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 0,5", 2023-09-23T21:17:42.664 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:15.753005+0000", 2023-09-23T21:17:42.664 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0068824690000000004 2023-09-23T21:17:42.664 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.664 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.664 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:17:42.664 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:15.781458+0000", 2023-09-23T21:17:42.664 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.028453556000000001 2023-09-23T21:17:42.665 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.665 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.665 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:17:42.665 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:15.792765+0000", 2023-09-23T21:17:42.665 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.011306800000000001 2023-09-23T21:17:42.665 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.665 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.665 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:17:42.665 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:15.805221+0000", 2023-09-23T21:17:42.665 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.012456211 2023-09-23T21:17:42.665 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.665 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.666 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:17:42.666 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:15.806627+0000", 2023-09-23T21:17:42.666 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0014054130000000001 2023-09-23T21:17:42.666 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.666 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.666 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:17:42.666 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:15.812451+0000", 2023-09-23T21:17:42.666 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00582418 2023-09-23T21:17:42.666 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:17:42.666 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:17:42.666 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:17:42.666 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.666 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.667 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.46328.0:99965 10.16 10:68a95ca6:::benchmark_data_smithi026_379838_object20008:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e492)", 2023-09-23T21:17:42.667 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:13:11.199392+0000", 2023-09-23T21:17:42.667 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 271.30071099999998, 2023-09-23T21:17:42.667 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.2292175890000001, 2023-09-23T21:17:42.667 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:17:42.667 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:17:42.667 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:17:42.667 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.46328", 2023-09-23T21:17:42.667 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/3748861181", 2023-09-23T21:17:42.667 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 99965 2023-09-23T21:17:42.667 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.667 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:17:42.668 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.668 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:17:42.668 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:11.199392+0000", 2023-09-23T21:17:42.668 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:17:42.668 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.668 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.668 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:17:42.668 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:11.199392+0000", 2023-09-23T21:17:42.668 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:17:42.668 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.668 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.668 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:17:42.669 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:11.199390+0000", 2023-09-23T21:17:42.669 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T21:17:42.669 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.669 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.669 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:17:42.669 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:11.199396+0000", 2023-09-23T21:17:42.669 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.1240000000000002e-06 2023-09-23T21:17:42.669 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.669 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.669 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:17:42.669 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:11.199407+0000", 2023-09-23T21:17:42.669 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.0682e-05 2023-09-23T21:17:42.669 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.670 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.670 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:17:42.670 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:11.199413+0000", 2023-09-23T21:17:42.670 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.2570000000000004e-06 2023-09-23T21:17:42.670 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.670 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.670 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:17:42.670 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:11.232630+0000", 2023-09-23T21:17:42.670 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.033216351999999998 2023-09-23T21:17:42.670 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.670 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.670 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:17:42.671 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:11.232640+0000", 2023-09-23T21:17:42.671 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.0414999999999999e-05 2023-09-23T21:17:42.671 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.671 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.671 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:17:42.671 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:12.191472+0000", 2023-09-23T21:17:42.671 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.95883214699999997 2023-09-23T21:17:42.671 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.671 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.671 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:17:42.671 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:12.191481+0000", 2023-09-23T21:17:42.671 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.6649999999999992e-06 2023-09-23T21:17:42.672 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.672 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.672 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:17:42.672 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:14.423279+0000", 2023-09-23T21:17:42.672 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.2317976210000001 2023-09-23T21:17:42.672 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.672 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.672 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:17:42.672 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:14.423386+0000", 2023-09-23T21:17:42.672 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00010726399999999999 2023-09-23T21:17:42.672 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.672 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.672 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 0,5", 2023-09-23T21:17:42.673 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:14.423617+0000", 2023-09-23T21:17:42.673 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000231597 2023-09-23T21:17:42.673 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.673 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.673 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:17:42.673 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:14.425276+0000", 2023-09-23T21:17:42.673 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.001658299 2023-09-23T21:17:42.673 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.673 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.673 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:17:42.673 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:14.425806+0000", 2023-09-23T21:17:42.674 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00053022900000000001 2023-09-23T21:17:42.674 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.674 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.674 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:17:42.674 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:14.428365+0000", 2023-09-23T21:17:42.674 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.002559253 2023-09-23T21:17:42.674 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.674 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.674 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:17:42.674 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:14.428423+0000", 2023-09-23T21:17:42.674 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.7596999999999998e-05 2023-09-23T21:17:42.674 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.675 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.675 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:17:42.675 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:14.428609+0000", 2023-09-23T21:17:42.675 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00018645399999999999 2023-09-23T21:17:42.675 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:17:42.675 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:17:42.675 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:17:42.675 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.675 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.675 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.46328.0:99979 10.1 10:8711b949:::benchmark_data_smithi026_379838_object20022:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e493)", 2023-09-23T21:17:42.675 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:13:11.860281+0000", 2023-09-23T21:17:42.675 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 270.63982124900002, 2023-09-23T21:17:42.676 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.5904489540000002, 2023-09-23T21:17:42.676 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:17:42.676 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:17:42.676 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:17:42.676 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.46328", 2023-09-23T21:17:42.676 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/3748861181", 2023-09-23T21:17:42.676 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 99979 2023-09-23T21:17:42.676 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.676 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:17:42.676 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.676 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:17:42.676 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:11.860281+0000", 2023-09-23T21:17:42.677 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:17:42.677 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.677 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.677 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:17:42.677 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:11.860281+0000", 2023-09-23T21:17:42.677 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:17:42.677 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.677 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.677 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:17:42.677 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:11.860280+0000", 2023-09-23T21:17:42.677 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T21:17:42.677 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.678 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.678 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:17:42.678 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:11.860286+0000", 2023-09-23T21:17:42.678 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.2720000000000003e-06 2023-09-23T21:17:42.678 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.678 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.678 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:17:42.678 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:11.860301+0000", 2023-09-23T21:17:42.678 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.4901e-05 2023-09-23T21:17:42.678 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.678 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.679 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:17:42.679 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:11.860308+0000", 2023-09-23T21:17:42.679 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.3779999999999998e-06 2023-09-23T21:17:42.679 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.679 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.679 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:17:42.680 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:11.860364+0000", 2023-09-23T21:17:42.680 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.5866999999999999e-05 2023-09-23T21:17:42.680 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.680 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.680 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:17:42.680 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:11.860373+0000", 2023-09-23T21:17:42.680 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.8500000000000006e-06 2023-09-23T21:17:42.680 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.680 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.680 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:17:42.680 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:12.196314+0000", 2023-09-23T21:17:42.681 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.33594100999999998 2023-09-23T21:17:42.681 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.681 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.681 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:17:42.681 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:12.196326+0000", 2023-09-23T21:17:42.681 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.1333e-05 2023-09-23T21:17:42.681 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.681 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.681 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:17:42.681 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:15.369429+0000", 2023-09-23T21:17:42.681 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.1731031180000002 2023-09-23T21:17:42.681 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.682 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.682 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for missing object", 2023-09-23T21:17:42.682 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:15.372799+0000", 2023-09-23T21:17:42.682 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0033701310000000002 2023-09-23T21:17:42.682 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.682 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.683 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:17:42.683 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:15.389753+0000", 2023-09-23T21:17:42.684 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.016954440000000001 2023-09-23T21:17:42.684 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.684 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.684 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for rw locks", 2023-09-23T21:17:42.684 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:15.392893+0000", 2023-09-23T21:17:42.684 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.003139293 2023-09-23T21:17:42.684 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.685 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.685 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:17:42.685 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:15.415439+0000", 2023-09-23T21:17:42.685 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.022546297999999999 2023-09-23T21:17:42.685 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.685 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.685 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:17:42.686 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:15.417698+0000", 2023-09-23T21:17:42.686 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0022593629999999999 2023-09-23T21:17:42.686 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.686 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.686 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 0,7", 2023-09-23T21:17:42.686 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:15.422787+0000", 2023-09-23T21:17:42.687 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0050888920000000002 2023-09-23T21:17:42.687 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.687 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.687 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:17:42.687 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:15.442882+0000", 2023-09-23T21:17:42.687 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.020094441000000001 2023-09-23T21:17:42.687 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.688 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.688 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:17:42.688 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:15.443740+0000", 2023-09-23T21:17:42.688 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00085842100000000005 2023-09-23T21:17:42.688 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.688 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.688 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:17:42.689 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:15.444499+0000", 2023-09-23T21:17:42.689 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00075862800000000001 2023-09-23T21:17:42.689 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.689 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.689 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:17:42.689 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:15.445752+0000", 2023-09-23T21:17:42.690 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0012533970000000001 2023-09-23T21:17:42.690 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.690 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.690 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:17:42.690 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:15.450730+0000", 2023-09-23T21:17:42.690 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0049782749999999999 2023-09-23T21:17:42.690 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:17:42.691 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:17:42.691 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:17:42.691 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.691 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.691 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.46328.0:99984 10.13 10:c97226d3:::benchmark_data_smithi026_379838_object20027:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e493)", 2023-09-23T21:17:42.691 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:13:11.898920+0000", 2023-09-23T21:17:42.691 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 270.6011828, 2023-09-23T21:17:42.692 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.923359048, 2023-09-23T21:17:42.692 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:17:42.692 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:17:42.692 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:17:42.692 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.46328", 2023-09-23T21:17:42.692 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/3748861181", 2023-09-23T21:17:42.693 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 99984 2023-09-23T21:17:42.693 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.693 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:17:42.693 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.693 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:17:42.693 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:11.898920+0000", 2023-09-23T21:17:42.693 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:17:42.694 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.694 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.694 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:17:42.694 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:11.898920+0000", 2023-09-23T21:17:42.694 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:17:42.694 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.695 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.695 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:17:42.695 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:11.898919+0000", 2023-09-23T21:17:42.695 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.999999 2023-09-23T21:17:42.695 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.695 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.695 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:17:42.695 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:11.898923+0000", 2023-09-23T21:17:42.695 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.4410000000000003e-06 2023-09-23T21:17:42.696 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.696 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.696 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:17:42.696 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:11.898939+0000", 2023-09-23T21:17:42.696 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.5588000000000001e-05 2023-09-23T21:17:42.696 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.696 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.697 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:17:42.697 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:11.898947+0000", 2023-09-23T21:17:42.697 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.7950000000000008e-06 2023-09-23T21:17:42.697 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.697 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.698 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:17:42.698 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:11.899000+0000", 2023-09-23T21:17:42.698 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.2905999999999998e-05 2023-09-23T21:17:42.698 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.698 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.698 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:17:42.698 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:11.899012+0000", 2023-09-23T21:17:42.698 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.2668e-05 2023-09-23T21:17:42.699 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.699 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.699 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:17:42.699 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:12.192819+0000", 2023-09-23T21:17:42.699 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.29380640800000002 2023-09-23T21:17:42.699 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.699 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.699 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:17:42.699 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:12.192826+0000", 2023-09-23T21:17:42.699 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.1439999999999997e-06 2023-09-23T21:17:42.700 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.700 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.700 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:17:42.700 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:15.584066+0000", 2023-09-23T21:17:42.700 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.3912398499999998 2023-09-23T21:17:42.700 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.701 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.701 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for missing object", 2023-09-23T21:17:42.701 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:15.587625+0000", 2023-09-23T21:17:42.701 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0035591070000000002 2023-09-23T21:17:42.701 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.701 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.701 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:17:42.701 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:15.621463+0000", 2023-09-23T21:17:42.701 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.033837837000000003 2023-09-23T21:17:42.701 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.702 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.702 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for degraded object", 2023-09-23T21:17:42.702 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:15.624790+0000", 2023-09-23T21:17:42.702 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0033272829999999999 2023-09-23T21:17:42.702 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.702 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.702 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:17:42.702 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:15.769569+0000", 2023-09-23T21:17:42.703 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.14477948299999999 2023-09-23T21:17:42.703 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.703 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.703 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:17:42.703 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:15.772068+0000", 2023-09-23T21:17:42.703 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0024988359999999999 2023-09-23T21:17:42.703 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.703 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.703 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 0,5", 2023-09-23T21:17:42.703 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:15.777050+0000", 2023-09-23T21:17:42.703 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0049818029999999999 2023-09-23T21:17:42.703 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.704 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.704 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:17:42.704 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:15.812721+0000", 2023-09-23T21:17:42.704 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.035671139999999997 2023-09-23T21:17:42.704 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.704 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.704 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:17:42.704 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:15.813568+0000", 2023-09-23T21:17:42.704 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00084671699999999998 2023-09-23T21:17:42.704 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.704 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.704 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:17:42.705 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:15.814693+0000", 2023-09-23T21:17:42.705 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0011254850000000001 2023-09-23T21:17:42.705 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.705 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.705 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:17:42.705 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:15.815984+0000", 2023-09-23T21:17:42.705 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.001290329 2023-09-23T21:17:42.705 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.705 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.705 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:17:42.705 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:15.822279+0000", 2023-09-23T21:17:42.705 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0062953109999999996 2023-09-23T21:17:42.706 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:17:42.706 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:17:42.706 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:17:42.706 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.706 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.706 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.46328.0:99987 10.13 10:c9fb87f0:::benchmark_data_smithi026_379838_object20030:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e493)", 2023-09-23T21:17:42.706 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:13:11.949115+0000", 2023-09-23T21:17:42.706 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 270.550988082, 2023-09-23T21:17:42.706 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.882972664, 2023-09-23T21:17:42.706 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:17:42.706 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:17:42.706 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:17:42.707 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.46328", 2023-09-23T21:17:42.707 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/3748861181", 2023-09-23T21:17:42.707 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 99987 2023-09-23T21:17:42.707 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.707 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:17:42.707 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.707 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:17:42.707 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:11.949115+0000", 2023-09-23T21:17:42.707 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:17:42.707 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.707 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.707 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:17:42.707 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:11.949115+0000", 2023-09-23T21:17:42.708 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:17:42.708 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.708 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.708 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:17:42.708 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:11.949113+0000", 2023-09-23T21:17:42.708 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.999999 2023-09-23T21:17:42.708 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.708 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.708 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:17:42.708 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:11.949119+0000", 2023-09-23T21:17:42.708 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.4190000000000001e-06 2023-09-23T21:17:42.708 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.709 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.709 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:17:42.709 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:11.949132+0000", 2023-09-23T21:17:42.709 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.2986000000000001e-05 2023-09-23T21:17:42.709 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.709 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.709 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:17:42.709 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:11.949137+0000", 2023-09-23T21:17:42.709 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.6019999999999999e-06 2023-09-23T21:17:42.709 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.709 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.709 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:17:42.709 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:11.949210+0000", 2023-09-23T21:17:42.710 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.2071999999999997e-05 2023-09-23T21:17:42.710 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.710 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.710 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:17:42.710 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:11.949219+0000", 2023-09-23T21:17:42.710 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.0410000000000003e-06 2023-09-23T21:17:42.710 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.710 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.710 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:17:42.710 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:12.192846+0000", 2023-09-23T21:17:42.710 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.243627597 2023-09-23T21:17:42.710 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.711 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.711 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:17:42.711 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:12.192853+0000", 2023-09-23T21:17:42.711 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.0169999999999999e-06 2023-09-23T21:17:42.711 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.711 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.711 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:17:42.711 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:15.588023+0000", 2023-09-23T21:17:42.711 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.3951699780000002 2023-09-23T21:17:42.711 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.711 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.711 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for missing object", 2023-09-23T21:17:42.712 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:15.591088+0000", 2023-09-23T21:17:42.712 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0030649900000000001 2023-09-23T21:17:42.712 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.712 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.712 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:17:42.712 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:15.640214+0000", 2023-09-23T21:17:42.712 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.049126213000000002 2023-09-23T21:17:42.712 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.712 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.712 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for degraded object", 2023-09-23T21:17:42.712 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:15.642230+0000", 2023-09-23T21:17:42.712 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0020155440000000002 2023-09-23T21:17:42.713 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.713 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.713 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:17:42.713 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:15.793072+0000", 2023-09-23T21:17:42.713 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.15084209300000001 2023-09-23T21:17:42.713 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.713 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.713 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:17:42.713 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:15.795467+0000", 2023-09-23T21:17:42.713 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.002394752 2023-09-23T21:17:42.714 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.714 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.714 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 0,5", 2023-09-23T21:17:42.714 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:15.800354+0000", 2023-09-23T21:17:42.714 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0048867590000000001 2023-09-23T21:17:42.714 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.714 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.714 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:17:42.714 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:15.813842+0000", 2023-09-23T21:17:42.714 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.013488557999999999 2023-09-23T21:17:42.714 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.715 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.715 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:17:42.715 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:15.823275+0000", 2023-09-23T21:17:42.715 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0094329649999999998 2023-09-23T21:17:42.715 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.715 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.715 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:17:42.715 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:15.824202+0000", 2023-09-23T21:17:42.715 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00092731499999999995 2023-09-23T21:17:42.715 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.715 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.715 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:17:42.715 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:15.825761+0000", 2023-09-23T21:17:42.716 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0015584139999999999 2023-09-23T21:17:42.716 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:42.716 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:42.716 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:17:42.716 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:15.832087+0000", 2023-09-23T21:17:42.716 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0063265379999999996 2023-09-23T21:17:42.716 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:17:42.716 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:17:42.716 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:17:42.716 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:17:42.716 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:17:42.716 INFO:teuthology.orchestra.run.smithi070.stdout:} 2023-09-23T21:17:42.717 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:17:42.718 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:17:42.718 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:17:42.718 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:17:42.819 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:17:42.866 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:17:42.971 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:17:42.977 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:17:42.978 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:17:43.074 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_ops_in_flight 2023-09-23T21:17:43.078 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:17:43.167 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:17:43.218 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:17:43.224 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:17:43.225 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:17:43.326 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:17:43.386 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:17:43 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:17:43.386 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:17:43 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:17:43.387 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:17:43 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:17:43.387 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:17:43 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:17:43.398 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:17:43.581 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:17:43.597 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:17:43.598 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:17:43.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:17:43 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:17:43.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:17:43 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:17:43.655 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0: {} 2023-09-23T21:17:43.693 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1: {} 2023-09-23T21:17:43.699 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:17:43.728 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2: {} 2023-09-23T21:17:43.730 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:17:43.760 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T21:17:43.775 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:17:43.780 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:17:43.780 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:17:43.796 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3: {} 2023-09-23T21:17:43.830 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4: {} 2023-09-23T21:17:43.877 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T21:17:43.881 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:17:43.929 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6: {} 2023-09-23T21:17:43.978 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T21:17:43.992 INFO:teuthology.orchestra.run.smithi026.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T21:17:43.993 INFO:teuthology.orchestra.run.smithi026.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T21:17:43.993 INFO:teuthology.orchestra.run.smithi026.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T21:17:43.993 INFO:teuthology.orchestra.run.smithi026.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T21:17:43.993 INFO:teuthology.orchestra.run.smithi026.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T21:17:43.993 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T21:17:43.993 INFO:teuthology.orchestra.run.smithi026.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T21:17:43.993 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T21:17:44.007 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:17:44.270 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:17:44 smithi026 ceph-mon[179688]: pgmap v1483: 129 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 6 active+recovery_wait+undersized+degraded+remapped, 121 active+clean; 3.8 GiB data, 16 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 813 op/s; 4826/184548 objects degraded (2.615%); 703/184548 objects misplaced (0.381%); 5.8 MiB/s, 92 objects/s recovering 2023-09-23T21:17:44.270 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:17:44 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/2631229424' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:17:44.270 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:17:44 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/1537383771' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-09-23T21:17:44.286 INFO:teuthology.orchestra.run.smithi026.stderr:reweighted osd.3 to 1 (10000) 2023-09-23T21:17:44.539 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:17:44 smithi026 ceph-mon[175151]: pgmap v1483: 129 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 6 active+recovery_wait+undersized+degraded+remapped, 121 active+clean; 3.8 GiB data, 16 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 813 op/s; 4826/184548 objects degraded (2.615%); 703/184548 objects misplaced (0.381%); 5.8 MiB/s, 92 objects/s recovering 2023-09-23T21:17:44.540 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:17:44 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/2631229424' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:17:44.540 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:17:44 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/1537383771' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-09-23T21:17:44.594 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:17:44 smithi070 ceph-mon[168282]: pgmap v1483: 129 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 6 active+recovery_wait+undersized+degraded+remapped, 121 active+clean; 3.8 GiB data, 16 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 813 op/s; 4826/184548 objects degraded (2.615%); 703/184548 objects misplaced (0.381%); 5.8 MiB/s, 92 objects/s recovering 2023-09-23T21:17:44.594 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:17:44 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/2631229424' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:17:44.595 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:17:44 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/1537383771' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-09-23T21:17:44.739 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:17:44.746 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:17:44.747 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:17:44.847 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:17:44.891 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:17:44.929 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph osd reweight 4 1 2023-09-23T21:17:44.988 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:17:44.991 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:17:44.992 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:17:45.069 INFO:teuthology.orchestra.run.smithi070.stdout:ERROR: (22) Invalid argument 2023-09-23T21:17:45.070 INFO:teuthology.orchestra.run.smithi070.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T21:17:45.093 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:17:45.262 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:17:45.373 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:17:45.386 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:17:45.386 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:17:45.487 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:17:45.539 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:17:45 smithi026 ceph-mon[175151]: failed to encode map e587 with expected crc 2023-09-23T21:17:45.539 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:17:45 smithi026 ceph-mon[175151]: failed to encode map e587 with expected crc 2023-09-23T21:17:45.539 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:17:45 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/1537383771' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-09-23T21:17:45.539 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:17:45 smithi026 ceph-mon[175151]: osdmap e587: 8 total, 8 up, 6 in 2023-09-23T21:17:45.539 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:17:45 smithi026 ceph-mon[175151]: failed to encode map e587 with expected crc 2023-09-23T21:17:45.540 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:17:45 smithi026 ceph-mon[179688]: failed to encode map e587 with expected crc 2023-09-23T21:17:45.541 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:17:45 smithi026 ceph-mon[179688]: failed to encode map e587 with expected crc 2023-09-23T21:17:45.541 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:17:45 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/1537383771' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-09-23T21:17:45.541 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:17:45 smithi026 ceph-mon[179688]: osdmap e587: 8 total, 8 up, 6 in 2023-09-23T21:17:45.541 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:17:45 smithi026 ceph-mon[179688]: failed to encode map e587 with expected crc 2023-09-23T21:17:45.542 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:17:45.550 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:17:45 smithi070 ceph-mon[168282]: failed to encode map e587 with expected crc 2023-09-23T21:17:45.550 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:17:45 smithi070 ceph-mon[168282]: failed to encode map e587 with expected crc 2023-09-23T21:17:45.550 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:17:45 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/1537383771' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-09-23T21:17:45.550 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:17:45 smithi070 ceph-mon[168282]: osdmap e587: 8 total, 8 up, 6 in 2023-09-23T21:17:45.551 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:17:45 smithi070 ceph-mon[168282]: failed to encode map e587 with expected crc 2023-09-23T21:17:45.552 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_blocked_ops 2023-09-23T21:17:45.603 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:17:45.608 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:17:45.609 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:17:45.710 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:17:45.745 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:17:45.767 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:17:45.785 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:17:45.789 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:17:45.789 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:17:45.890 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:17:45.920 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:17:45.959 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:17:45.962 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:17:45.963 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:17:46.063 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:17:46.094 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:17:46.208 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T21:17:46.234 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T21:17:46.236 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:17:46.237 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:17:46.290 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:17:46 smithi026 ceph-mon[175151]: pgmap v1485: 129 pgs: 1 active+undersized+remapped, 1 active+recovery_wait, 6 active+recovery_wait+undersized+degraded+remapped, 121 active+clean; 3.7 GiB data, 16 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 874 op/s; 4826/181113 objects degraded (2.665%); 10 MiB/s, 166 objects/s recovering 2023-09-23T21:17:46.290 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:17:46 smithi026 ceph-mon[175151]: failed to encode map e588 with expected crc 2023-09-23T21:17:46.290 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:17:46 smithi026 ceph-mon[175151]: failed to encode map e588 with expected crc 2023-09-23T21:17:46.291 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:17:46 smithi026 ceph-mon[175151]: failed to encode map e588 with expected crc 2023-09-23T21:17:46.291 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:17:46 smithi026 ceph-mon[175151]: osdmap e588: 8 total, 8 up, 6 in 2023-09-23T21:17:46.291 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:17:46 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:17:46.291 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:17:46 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/1785242261' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T21:17:46.291 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:17:46 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T21:17:46.291 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:17:46 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:17:46.291 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:17:46 smithi026 ceph-mon[175151]: Health check update: Degraded data redundancy: 4826/181113 objects degraded (2.665%), 6 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T21:17:46.291 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:17:46 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T21:17:46.291 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:17:46 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T21:17:46.291 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:17:46 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:17:46.292 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:17:46 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:17:46.292 INFO:teuthology.orchestra.run.smithi026.stderr:noscrub is unset 2023-09-23T21:17:46.338 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:17:46.413 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:17:46.576 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:17:46 smithi026 ceph-mon[179688]: pgmap v1485: 129 pgs: 1 active+undersized+remapped, 1 active+recovery_wait, 6 active+recovery_wait+undersized+degraded+remapped, 121 active+clean; 3.7 GiB data, 16 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 874 op/s; 4826/181113 objects degraded (2.665%); 10 MiB/s, 166 objects/s recovering 2023-09-23T21:17:46.576 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:17:46 smithi026 ceph-mon[179688]: failed to encode map e588 with expected crc 2023-09-23T21:17:46.576 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:17:46 smithi026 ceph-mon[179688]: failed to encode map e588 with expected crc 2023-09-23T21:17:46.576 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:17:46 smithi026 ceph-mon[179688]: failed to encode map e588 with expected crc 2023-09-23T21:17:46.576 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:17:46 smithi026 ceph-mon[179688]: osdmap e588: 8 total, 8 up, 6 in 2023-09-23T21:17:46.577 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:17:46 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:17:46.577 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:17:46 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/1785242261' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T21:17:46.577 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:17:46 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T21:17:46.577 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:17:46 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:17:46.577 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:17:46 smithi026 ceph-mon[179688]: Health check update: Degraded data redundancy: 4826/181113 objects degraded (2.665%), 6 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T21:17:46.577 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:17:46 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T21:17:46.577 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:17:46 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T21:17:46.577 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:17:46 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:17:46.577 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:17:46 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:17:46.621 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:17:46 smithi070 ceph-mon[168282]: pgmap v1485: 129 pgs: 1 active+undersized+remapped, 1 active+recovery_wait, 6 active+recovery_wait+undersized+degraded+remapped, 121 active+clean; 3.7 GiB data, 16 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 874 op/s; 4826/181113 objects degraded (2.665%); 10 MiB/s, 166 objects/s recovering 2023-09-23T21:17:46.621 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:17:46 smithi070 ceph-mon[168282]: failed to encode map e588 with expected crc 2023-09-23T21:17:46.622 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:17:46 smithi070 ceph-mon[168282]: failed to encode map e588 with expected crc 2023-09-23T21:17:46.622 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:17:46 smithi070 ceph-mon[168282]: failed to encode map e588 with expected crc 2023-09-23T21:17:46.622 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:17:46 smithi070 ceph-mon[168282]: osdmap e588: 8 total, 8 up, 6 in 2023-09-23T21:17:46.622 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:17:46 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:17:46.622 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:17:46 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/1785242261' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T21:17:46.622 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:17:46 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T21:17:46.622 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:17:46 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:17:46.622 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:17:46 smithi070 ceph-mon[168282]: Health check update: Degraded data redundancy: 4826/181113 objects degraded (2.665%), 6 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T21:17:46.622 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:17:46 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T21:17:46.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:17:46 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T21:17:46.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:17:46 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:17:46.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:17:46 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:17:46.819 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:17:46.826 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:17:46.827 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:17:46.928 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:17:47.033 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:17:47.101 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:17:47.106 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:17:47.106 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:17:47.207 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:17:47.245 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:17:47.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:17:47 smithi070 ceph-mon[168282]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:17:47.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:17:47 smithi070 ceph-mon[168282]: failed to encode map e589 with expected crc 2023-09-23T21:17:47.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:17:47 smithi070 ceph-mon[168282]: failed to encode map e589 with expected crc 2023-09-23T21:17:47.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:17:47 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T21:17:47.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:17:47 smithi070 ceph-mon[168282]: osdmap e589: 8 total, 8 up, 6 in 2023-09-23T21:17:47.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:17:47 smithi070 ceph-mon[168282]: failed to encode map e589 with expected crc 2023-09-23T21:17:47.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:17:47 smithi070 ceph-mon[168282]: failed to encode map e589 with expected crc 2023-09-23T21:17:47.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:17:47 smithi026 ceph-mon[179688]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:17:47.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:17:47 smithi026 ceph-mon[179688]: failed to encode map e589 with expected crc 2023-09-23T21:17:47.383 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:17:47 smithi026 ceph-mon[179688]: failed to encode map e589 with expected crc 2023-09-23T21:17:47.383 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:17:47 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T21:17:47.383 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:17:47 smithi026 ceph-mon[179688]: osdmap e589: 8 total, 8 up, 6 in 2023-09-23T21:17:47.383 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:17:47 smithi026 ceph-mon[179688]: failed to encode map e589 with expected crc 2023-09-23T21:17:47.383 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:17:47 smithi026 ceph-mon[179688]: failed to encode map e589 with expected crc 2023-09-23T21:17:47.383 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:17:47 smithi026 ceph-mon[175151]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:17:47.384 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:17:47 smithi026 ceph-mon[175151]: failed to encode map e589 with expected crc 2023-09-23T21:17:47.384 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:17:47 smithi026 ceph-mon[175151]: failed to encode map e589 with expected crc 2023-09-23T21:17:47.384 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:17:47 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T21:17:47.384 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:17:47 smithi026 ceph-mon[175151]: osdmap e589: 8 total, 8 up, 6 in 2023-09-23T21:17:47.384 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:17:47 smithi026 ceph-mon[175151]: failed to encode map e589 with expected crc 2023-09-23T21:17:47.384 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:17:47 smithi026 ceph-mon[175151]: failed to encode map e589 with expected crc 2023-09-23T21:17:47.387 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T21:17:47.399 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:17:47.405 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:17:47.405 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:17:47.506 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:17:47.609 INFO:teuthology.orchestra.run.smithi070.stdout:ERROR: (22) Invalid argument 2023-09-23T21:17:47.609 INFO:teuthology.orchestra.run.smithi070.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T21:17:47.664 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:17:47.790 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:17:47.803 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:17:47.803 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:17:47.904 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:17:47.996 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:17:48.013 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_historic_ops 2023-09-23T21:17:48.061 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T21:17:48.071 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:17:48.072 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:17:48.172 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:17:48.212 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:17:48.269 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:17:48.281 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:17:48.281 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:17:48.296 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:17:48.339 INFO:teuthology.orchestra.run.smithi026.stderr:reweighted osd.4 to 1 (10000) 2023-09-23T21:17:48.366 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:17:48 smithi026 ceph-mon[179688]: pgmap v1488: 129 pgs: 1 active+recovering+undersized+remapped, 1 active+undersized+remapped, 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 121 active+clean; 3.5 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 1.12k op/s; 4042/172461 objects degraded (2.344%); 464/172461 objects misplaced (0.269%); 13 MiB/s, 205 objects/s recovering 2023-09-23T21:17:48.367 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:17:48 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/322265940' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-09-23T21:17:48.367 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:17:48 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-09-23T21:17:48.367 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:17:48 smithi026 ceph-mon[175151]: pgmap v1488: 129 pgs: 1 active+recovering+undersized+remapped, 1 active+undersized+remapped, 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 121 active+clean; 3.5 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 1.12k op/s; 4042/172461 objects degraded (2.344%); 464/172461 objects misplaced (0.269%); 13 MiB/s, 205 objects/s recovering 2023-09-23T21:17:48.367 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:17:48 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/322265940' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-09-23T21:17:48.367 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:17:48 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-09-23T21:17:48.382 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:17:48.466 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:17:48.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:17:48 smithi070 ceph-mon[168282]: pgmap v1488: 129 pgs: 1 active+recovering+undersized+remapped, 1 active+undersized+remapped, 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 121 active+clean; 3.5 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 1.12k op/s; 4042/172461 objects degraded (2.344%); 464/172461 objects misplaced (0.269%); 13 MiB/s, 205 objects/s recovering 2023-09-23T21:17:48.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:17:48 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/322265940' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-09-23T21:17:48.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:17:48 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-09-23T21:17:48.731 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T21:17:48.785 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:17:48.790 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:17:48.790 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:17:48.860 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph osd reweight 5 1 2023-09-23T21:17:48.887 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T21:17:48.891 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:17:48.976 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:17:49.072 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:17:49.087 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:17:49.087 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:17:49.188 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:17:49.305 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:17:49.381 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:17:49 smithi026 ceph-mon[175151]: failed to encode map e590 with expected crc 2023-09-23T21:17:49.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:17:49 smithi026 ceph-mon[175151]: failed to encode map e590 with expected crc 2023-09-23T21:17:49.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:17:49 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-09-23T21:17:49.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:17:49 smithi026 ceph-mon[175151]: osdmap e590: 8 total, 8 up, 6 in 2023-09-23T21:17:49.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:17:49 smithi026 ceph-mon[175151]: failed to encode map e590 with expected crc 2023-09-23T21:17:49.383 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:17:49 smithi026 ceph-mon[179688]: failed to encode map e590 with expected crc 2023-09-23T21:17:49.383 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:17:49 smithi026 ceph-mon[179688]: failed to encode map e590 with expected crc 2023-09-23T21:17:49.383 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:17:49 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-09-23T21:17:49.383 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:17:49 smithi026 ceph-mon[179688]: osdmap e590: 8 total, 8 up, 6 in 2023-09-23T21:17:49.383 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:17:49 smithi026 ceph-mon[179688]: failed to encode map e590 with expected crc 2023-09-23T21:17:49.557 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:17:49 smithi070 ceph-mon[168282]: failed to encode map e590 with expected crc 2023-09-23T21:17:49.557 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:17:49 smithi070 ceph-mon[168282]: failed to encode map e590 with expected crc 2023-09-23T21:17:49.557 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:17:49 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-09-23T21:17:49.558 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:17:49 smithi070 ceph-mon[168282]: osdmap e590: 8 total, 8 up, 6 in 2023-09-23T21:17:49.558 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:17:49 smithi070 ceph-mon[168282]: failed to encode map e590 with expected crc 2023-09-23T21:17:49.884 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 21:17:49 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[164029]: ::ffff:172.21.15.70 - - [23/Sep/2023:21:17:49] "GET /metrics HTTP/1.1" 200 36391 "" "Prometheus/2.43.0" 2023-09-23T21:17:50.168 INFO:teuthology.orchestra.run.smithi070.stdout:ERROR: (22) Invalid argument 2023-09-23T21:17:50.168 INFO:teuthology.orchestra.run.smithi070.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T21:17:50.548 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_ops_in_flight 2023-09-23T21:17:50.553 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:17:50 smithi070 ceph-mon[168282]: pgmap v1490: 129 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 1 active+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 121 active+clean; 3.5 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 903 op/s; 4042/171192 objects degraded (2.361%); 464/171192 objects misplaced (0.271%); 13 MiB/s, 207 objects/s recovering 2023-09-23T21:17:50.631 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:17:50 smithi026 ceph-mon[179688]: pgmap v1490: 129 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 1 active+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 121 active+clean; 3.5 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 903 op/s; 4042/171192 objects degraded (2.361%); 464/171192 objects misplaced (0.271%); 13 MiB/s, 207 objects/s recovering 2023-09-23T21:17:50.634 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:17:50 smithi026 ceph-mon[175151]: pgmap v1490: 129 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 1 active+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 121 active+clean; 3.5 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 903 op/s; 4042/171192 objects degraded (2.361%); 464/171192 objects misplaced (0.271%); 13 MiB/s, 207 objects/s recovering 2023-09-23T21:17:51.073 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T21:17:51.085 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:17:51.085 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:17:51.186 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:17:51.249 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T21:17:51.255 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:17:51.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:17:51 smithi070 ceph-mon[168282]: 11.7 scrub starts 2023-09-23T21:17:51.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:17:51 smithi070 ceph-mon[168282]: Health check update: Degraded data redundancy: 4042/171192 objects degraded (2.361%), 5 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T21:17:51.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:17:51 smithi070 ceph-mon[168282]: failed to encode map e591 with expected crc 2023-09-23T21:17:51.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:17:51 smithi070 ceph-mon[168282]: osdmap e591: 8 total, 8 up, 6 in 2023-09-23T21:17:51.631 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:17:51 smithi026 ceph-mon[175151]: 11.7 scrub starts 2023-09-23T21:17:51.631 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:17:51 smithi026 ceph-mon[175151]: Health check update: Degraded data redundancy: 4042/171192 objects degraded (2.361%), 5 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T21:17:51.631 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:17:51 smithi026 ceph-mon[175151]: failed to encode map e591 with expected crc 2023-09-23T21:17:51.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:17:51 smithi026 ceph-mon[175151]: osdmap e591: 8 total, 8 up, 6 in 2023-09-23T21:17:51.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:17:51 smithi026 ceph-mon[179688]: 11.7 scrub starts 2023-09-23T21:17:51.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:17:51 smithi026 ceph-mon[179688]: Health check update: Degraded data redundancy: 4042/171192 objects degraded (2.361%), 5 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T21:17:51.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:17:51 smithi026 ceph-mon[179688]: failed to encode map e591 with expected crc 2023-09-23T21:17:51.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:17:51 smithi026 ceph-mon[179688]: osdmap e591: 8 total, 8 up, 6 in 2023-09-23T21:17:52.273 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:17:52.280 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:17:52.281 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:17:52.281 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:17:52.382 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:17:52.438 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:17:52.479 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:17:52.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:17:52 smithi070 ceph-mon[168282]: 11.17 scrub starts 2023-09-23T21:17:52.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:17:52 smithi070 ceph-mon[168282]: failed to encode map e591 with expected crc 2023-09-23T21:17:52.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:17:52 smithi070 ceph-mon[168282]: failed to encode map e591 with expected crc 2023-09-23T21:17:52.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:17:52 smithi070 ceph-mon[168282]: pgmap v1492: 129 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 121 active+clean; 3.4 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 826 op/s; 4042/166497 objects degraded (2.428%); 464/166497 objects misplaced (0.279%); 3.4 MiB/s, 54 objects/s recovering 2023-09-23T21:17:52.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:17:52 smithi070 ceph-mon[168282]: failed to encode map e592 with expected crc 2023-09-23T21:17:52.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:17:52 smithi070 ceph-mon[168282]: osdmap e592: 8 total, 8 up, 6 in 2023-09-23T21:17:52.631 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:17:52 smithi026 ceph-mon[175151]: 11.17 scrub starts 2023-09-23T21:17:52.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:17:52 smithi026 ceph-mon[175151]: failed to encode map e591 with expected crc 2023-09-23T21:17:52.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:17:52 smithi026 ceph-mon[175151]: failed to encode map e591 with expected crc 2023-09-23T21:17:52.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:17:52 smithi026 ceph-mon[175151]: pgmap v1492: 129 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 121 active+clean; 3.4 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 826 op/s; 4042/166497 objects degraded (2.428%); 464/166497 objects misplaced (0.279%); 3.4 MiB/s, 54 objects/s recovering 2023-09-23T21:17:52.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:17:52 smithi026 ceph-mon[175151]: failed to encode map e592 with expected crc 2023-09-23T21:17:52.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:17:52 smithi026 ceph-mon[175151]: osdmap e592: 8 total, 8 up, 6 in 2023-09-23T21:17:52.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:17:52 smithi026 ceph-mon[179688]: 11.17 scrub starts 2023-09-23T21:17:52.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:17:52 smithi026 ceph-mon[179688]: failed to encode map e591 with expected crc 2023-09-23T21:17:52.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:17:52 smithi026 ceph-mon[179688]: failed to encode map e591 with expected crc 2023-09-23T21:17:52.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:17:52 smithi026 ceph-mon[179688]: pgmap v1492: 129 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 121 active+clean; 3.4 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 826 op/s; 4042/166497 objects degraded (2.428%); 464/166497 objects misplaced (0.279%); 3.4 MiB/s, 54 objects/s recovering 2023-09-23T21:17:52.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:17:52 smithi026 ceph-mon[179688]: failed to encode map e592 with expected crc 2023-09-23T21:17:52.633 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:17:52 smithi026 ceph-mon[179688]: osdmap e592: 8 total, 8 up, 6 in 2023-09-23T21:17:52.669 INFO:teuthology.orchestra.run.smithi070.stdout:ERROR: (22) Invalid argument 2023-09-23T21:17:52.669 INFO:teuthology.orchestra.run.smithi070.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T21:17:53.044 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_blocked_ops 2023-09-23T21:17:53.365 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:17:53 smithi070 ceph-mon[168282]: 11.7 scrub starts 2023-09-23T21:17:53.365 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:17:53 smithi070 ceph-mon[168282]: failed to encode map e592 with expected crc 2023-09-23T21:17:53.365 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:17:53 smithi070 ceph-mon[168282]: failed to encode map e592 with expected crc 2023-09-23T21:17:53.365 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:17:53 smithi070 ceph-mon[168282]: failed to encode map e592 with expected crc 2023-09-23T21:17:53.494 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:17:53.507 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:17:53.507 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:17:53.608 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:17:53.634 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:17:53 smithi026 ceph-mon[175151]: 11.7 scrub starts 2023-09-23T21:17:53.634 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:17:53 smithi026 ceph-mon[175151]: failed to encode map e592 with expected crc 2023-09-23T21:17:53.634 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:17:53 smithi026 ceph-mon[175151]: failed to encode map e592 with expected crc 2023-09-23T21:17:53.634 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:17:53 smithi026 ceph-mon[175151]: failed to encode map e592 with expected crc 2023-09-23T21:17:53.634 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:17:53 smithi026 ceph-mon[179688]: 11.7 scrub starts 2023-09-23T21:17:53.635 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:17:53 smithi026 ceph-mon[179688]: failed to encode map e592 with expected crc 2023-09-23T21:17:53.635 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:17:53 smithi026 ceph-mon[179688]: failed to encode map e592 with expected crc 2023-09-23T21:17:53.635 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:17:53 smithi026 ceph-mon[179688]: failed to encode map e592 with expected crc 2023-09-23T21:17:53.651 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:17:53.652 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T21:17:53.722 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:17:53.726 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:17:53.727 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:17:53.808 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0: {} 2023-09-23T21:17:53.828 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:17:53.854 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1: {} 2023-09-23T21:17:53.903 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2: {} 2023-09-23T21:17:53.910 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:17:53.948 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3: {} 2023-09-23T21:17:54.005 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4: {} 2023-09-23T21:17:54.039 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T21:17:54.081 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6: {} 2023-09-23T21:17:54.121 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T21:17:54.131 INFO:teuthology.orchestra.run.smithi026.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T21:17:54.131 INFO:teuthology.orchestra.run.smithi026.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T21:17:54.131 INFO:teuthology.orchestra.run.smithi026.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T21:17:54.131 INFO:teuthology.orchestra.run.smithi026.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T21:17:54.131 INFO:teuthology.orchestra.run.smithi026.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T21:17:54.131 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T21:17:54.131 INFO:teuthology.orchestra.run.smithi026.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T21:17:54.131 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T21:17:54.214 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T21:17:54.221 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:17:54.222 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:17:54.323 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:17:54.368 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:17:54.428 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:17:54.431 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:17:54.432 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:17:54.533 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:17:54.600 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:17:54 smithi026 ceph-mon[179688]: 11.15 scrub starts 2023-09-23T21:17:54.601 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:17:54 smithi026 ceph-mon[179688]: pgmap v1494: 129 pgs: 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 123 active+clean; 3.2 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 829 op/s; 4042/157521 objects degraded (2.566%); 4.8 MiB/s, 77 objects/s recovering 2023-09-23T21:17:54.601 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:17:54 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/3072947764' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:17:54.601 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:17:54 smithi026 ceph-mon[175151]: 11.15 scrub starts 2023-09-23T21:17:54.601 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:17:54 smithi026 ceph-mon[175151]: pgmap v1494: 129 pgs: 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 123 active+clean; 3.2 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 829 op/s; 4042/157521 objects degraded (2.566%); 4.8 MiB/s, 77 objects/s recovering 2023-09-23T21:17:54.601 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:17:54 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/3072947764' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:17:54.621 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:17:54 smithi070 ceph-mon[168282]: 11.15 scrub starts 2023-09-23T21:17:54.621 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:17:54 smithi070 ceph-mon[168282]: pgmap v1494: 129 pgs: 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 123 active+clean; 3.2 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 829 op/s; 4042/157521 objects degraded (2.566%); 4.8 MiB/s, 77 objects/s recovering 2023-09-23T21:17:54.621 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:17:54 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/3072947764' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:17:54.622 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:17:54.787 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:17:54.794 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:17:54.794 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:17:54.895 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:17:54.963 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:17:55.030 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:17:55.038 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:17:55.039 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:17:55.060 INFO:teuthology.orchestra.run.smithi070.stdout:{ 2023-09-23T21:17:55.060 INFO:teuthology.orchestra.run.smithi070.stdout: "ops": [], 2023-09-23T21:17:55.061 INFO:teuthology.orchestra.run.smithi070.stdout: "complaint_time": 30, 2023-09-23T21:17:55.061 INFO:teuthology.orchestra.run.smithi070.stdout: "num_blocked_ops": 0 2023-09-23T21:17:55.061 INFO:teuthology.orchestra.run.smithi070.stdout:} 2023-09-23T21:17:55.139 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:17:55.217 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:17:55.484 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_historic_ops 2023-09-23T21:17:56.246 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T21:17:56.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:17:56 smithi026 ceph-mon[179688]: pgmap v1495: 129 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 123 active+clean; 3.1 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 845 op/s; 3182/153744 objects degraded (2.070%); 358/153744 objects misplaced (0.233%); 8.8 MiB/s, 140 objects/s recovering 2023-09-23T21:17:56.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:17:56 smithi026 ceph-mon[179688]: Health check update: Degraded data redundancy: 3182/153744 objects degraded (2.070%), 4 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T21:17:56.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:17:56 smithi026 ceph-mon[175151]: pgmap v1495: 129 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 123 active+clean; 3.1 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 845 op/s; 3182/153744 objects degraded (2.070%); 358/153744 objects misplaced (0.233%); 8.8 MiB/s, 140 objects/s recovering 2023-09-23T21:17:56.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:17:56 smithi026 ceph-mon[175151]: Health check update: Degraded data redundancy: 3182/153744 objects degraded (2.070%), 4 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T21:17:56.667 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:17:56 smithi070 ceph-mon[168282]: pgmap v1495: 129 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 123 active+clean; 3.1 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 845 op/s; 3182/153744 objects degraded (2.070%); 358/153744 objects misplaced (0.233%); 8.8 MiB/s, 140 objects/s recovering 2023-09-23T21:17:56.667 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:17:56 smithi070 ceph-mon[168282]: Health check update: Degraded data redundancy: 3182/153744 objects degraded (2.070%), 4 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T21:17:56.812 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:17:56.819 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:17:56.819 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:17:56.920 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:17:56.961 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:17:57.065 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:17:57.067 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:17:57.068 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:17:57.169 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:17:57.225 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:17:57.325 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:17:57.328 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:17:57.328 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:17:57.429 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:17:57.486 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:17:57.489 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:17:57 smithi070 ceph-mon[168282]: 11.17 scrub starts 2023-09-23T21:17:57.569 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:17:57.575 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:17:57.575 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:17:57.590 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:17:57 smithi026 ceph-mon[175151]: 11.17 scrub starts 2023-09-23T21:17:57.590 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:17:57 smithi026 ceph-mon[179688]: 11.17 scrub starts 2023-09-23T21:17:57.677 INFO:teuthology.orchestra.run.smithi070.stdout:{ 2023-09-23T21:17:57.677 INFO:teuthology.orchestra.run.smithi070.stdout: "size": 20, 2023-09-23T21:17:57.677 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 600, 2023-09-23T21:17:57.677 INFO:teuthology.orchestra.run.smithi070.stdout: "ops": [ 2023-09-23T21:17:57.677 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:57.677 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.46328.0:12430 10.1 10:8e1b46c6:::benchmark_data_smithi026_379838_object12429:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e430)", 2023-09-23T21:17:57.677 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:10:32.729016+0000", 2023-09-23T21:17:57.677 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 444.945718752, 2023-09-23T21:17:57.677 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.96804327999999995, 2023-09-23T21:17:57.677 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:17:57.678 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:17:57.678 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:17:57.678 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.46328", 2023-09-23T21:17:57.678 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/3748861181", 2023-09-23T21:17:57.678 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 12430 2023-09-23T21:17:57.678 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:57.678 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:17:57.678 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:57.678 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:17:57.678 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:10:32.729016+0000", 2023-09-23T21:17:57.678 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:17:57.678 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:57.679 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:57.679 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:17:57.679 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:10:32.729016+0000", 2023-09-23T21:17:57.679 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:17:57.679 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:57.679 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:57.679 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:17:57.679 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:10:32.729015+0000", 2023-09-23T21:17:57.679 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T21:17:57.679 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:57.679 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:57.679 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:17:57.680 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:10:32.729127+0000", 2023-09-23T21:17:57.680 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000111895 2023-09-23T21:17:57.680 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:57.680 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:57.680 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:17:57.680 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:10:32.729138+0000", 2023-09-23T21:17:57.680 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.0900999999999999e-05 2023-09-23T21:17:57.680 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:57.680 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:57.680 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:17:57.680 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:10:32.729143+0000", 2023-09-23T21:17:57.680 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.3329999999999996e-06 2023-09-23T21:17:57.680 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:57.681 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:57.681 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:17:57.681 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:10:32.729222+0000", 2023-09-23T21:17:57.681 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.9169e-05 2023-09-23T21:17:57.681 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:57.681 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:57.681 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:17:57.681 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:10:32.729245+0000", 2023-09-23T21:17:57.681 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.3125e-05 2023-09-23T21:17:57.681 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:57.681 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:57.681 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:17:57.682 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:10:33.686993+0000", 2023-09-23T21:17:57.682 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.95774722400000001 2023-09-23T21:17:57.682 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:57.682 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:57.682 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:17:57.682 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:10:33.687143+0000", 2023-09-23T21:17:57.682 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00015004499999999999 2023-09-23T21:17:57.682 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:57.682 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:57.682 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 0,1", 2023-09-23T21:17:57.682 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:10:33.687248+0000", 2023-09-23T21:17:57.682 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000105204 2023-09-23T21:17:57.682 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:57.683 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:57.683 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:17:57.683 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:10:33.689705+0000", 2023-09-23T21:17:57.683 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0024572420000000001 2023-09-23T21:17:57.683 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:57.683 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:57.683 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:17:57.683 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:10:33.696947+0000", 2023-09-23T21:17:57.683 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0072415550000000002 2023-09-23T21:17:57.683 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:57.683 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:57.683 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:17:57.684 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:10:33.696974+0000", 2023-09-23T21:17:57.684 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.7412000000000001e-05 2023-09-23T21:17:57.684 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:57.684 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:57.684 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:17:57.684 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:10:33.697024+0000", 2023-09-23T21:17:57.684 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.9570000000000001e-05 2023-09-23T21:17:57.684 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:57.684 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:57.684 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:17:57.684 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:10:33.697060+0000", 2023-09-23T21:17:57.684 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.6023999999999999e-05 2023-09-23T21:17:57.684 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:17:57.685 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:17:57.685 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:17:57.685 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:57.685 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:57.685 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.46328.0:12477 10.1 10:8cecffa5:::benchmark_data_smithi026_379838_object12476:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e430)", 2023-09-23T21:17:57.685 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:10:32.755815+0000", 2023-09-23T21:17:57.685 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 444.91891965000002, 2023-09-23T21:17:57.685 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.94137344599999995, 2023-09-23T21:17:57.685 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:17:57.685 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:17:57.685 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:17:57.685 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.46328", 2023-09-23T21:17:57.685 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/3748861181", 2023-09-23T21:17:57.686 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 12477 2023-09-23T21:17:57.686 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:57.686 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:17:57.686 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:57.686 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:17:57.686 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:10:32.755815+0000", 2023-09-23T21:17:57.686 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:17:57.686 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:57.686 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:57.686 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:17:57.686 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:10:32.755815+0000", 2023-09-23T21:17:57.686 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:17:57.686 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:57.687 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:57.687 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:17:57.687 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:10:32.755814+0000", 2023-09-23T21:17:57.687 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T21:17:57.687 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:57.687 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:57.687 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:17:57.687 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:10:32.755861+0000", 2023-09-23T21:17:57.687 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.672e-05 2023-09-23T21:17:57.687 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:57.687 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:57.687 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:17:57.688 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:10:32.755871+0000", 2023-09-23T21:17:57.688 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.9550000000000004e-06 2023-09-23T21:17:57.688 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:57.688 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:57.688 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:17:57.688 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:10:32.755876+0000", 2023-09-23T21:17:57.688 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.763e-06 2023-09-23T21:17:57.688 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:57.688 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:57.688 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:17:57.688 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:10:32.755945+0000", 2023-09-23T21:17:57.688 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.9622000000000005e-05 2023-09-23T21:17:57.688 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:57.689 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:57.689 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:17:57.689 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:10:32.755959+0000", 2023-09-23T21:17:57.689 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.3378e-05 2023-09-23T21:17:57.689 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:57.689 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:57.689 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:17:57.689 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:10:33.687722+0000", 2023-09-23T21:17:57.689 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.93176342400000001 2023-09-23T21:17:57.689 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:57.689 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:57.689 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:17:57.689 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:10:33.687805+0000", 2023-09-23T21:17:57.690 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.2688999999999994e-05 2023-09-23T21:17:57.690 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:57.690 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:57.690 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 0,1", 2023-09-23T21:17:57.690 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:10:33.687896+0000", 2023-09-23T21:17:57.690 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.1217000000000004e-05 2023-09-23T21:17:57.690 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:57.690 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:57.690 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:17:57.690 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:10:33.690957+0000", 2023-09-23T21:17:57.690 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0030608549999999999 2023-09-23T21:17:57.690 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:57.690 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:57.691 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:17:57.691 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:10:33.697095+0000", 2023-09-23T21:17:57.691 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0061379900000000003 2023-09-23T21:17:57.691 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:57.691 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:57.691 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:17:57.691 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:10:33.697119+0000", 2023-09-23T21:17:57.691 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.4345000000000002e-05 2023-09-23T21:17:57.691 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:57.691 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:57.691 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:17:57.691 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:10:33.697156+0000", 2023-09-23T21:17:57.691 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.6891000000000003e-05 2023-09-23T21:17:57.692 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:57.692 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:57.692 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:17:57.692 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:10:33.697189+0000", 2023-09-23T21:17:57.692 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.2793000000000003e-05 2023-09-23T21:17:57.692 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:17:57.692 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:17:57.692 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:17:57.692 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:57.692 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:57.693 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.46328.0:84576 10.c 10:3b2a76db:::benchmark_data_smithi026_379838_object4619:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e471)", 2023-09-23T21:17:57.693 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:12:10.844583+0000", 2023-09-23T21:17:57.693 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 346.830151844, 2023-09-23T21:17:57.694 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.511043044, 2023-09-23T21:17:57.694 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:17:57.694 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:17:57.694 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:17:57.694 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.46328", 2023-09-23T21:17:57.694 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/3748861181", 2023-09-23T21:17:57.694 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 84576 2023-09-23T21:17:57.694 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:57.695 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:17:57.695 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:57.695 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:17:57.695 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:12:10.844583+0000", 2023-09-23T21:17:57.695 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:17:57.695 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:57.696 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:57.696 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:17:57.696 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:12:10.844583+0000", 2023-09-23T21:17:57.696 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:17:57.696 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:57.696 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:57.696 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:17:57.696 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:12:10.844581+0000", 2023-09-23T21:17:57.697 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999981 2023-09-23T21:17:57.697 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:57.697 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:57.697 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:17:57.697 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:12:10.844591+0000", 2023-09-23T21:17:57.697 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.098e-06 2023-09-23T21:17:57.697 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:57.698 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:57.698 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:17:57.698 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:12:10.844609+0000", 2023-09-23T21:17:57.698 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.823e-05 2023-09-23T21:17:57.698 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:57.698 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:57.698 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:17:57.699 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:12:10.844617+0000", 2023-09-23T21:17:57.699 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.2509999999999994e-06 2023-09-23T21:17:57.699 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:57.699 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:57.699 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:17:57.699 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:12:10.844680+0000", 2023-09-23T21:17:57.699 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.3287999999999994e-05 2023-09-23T21:17:57.700 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:57.700 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:57.700 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:17:57.700 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:12:10.844854+0000", 2023-09-23T21:17:57.700 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000173163 2023-09-23T21:17:57.700 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:57.700 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:57.701 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 4", 2023-09-23T21:17:57.701 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:12:10.844964+0000", 2023-09-23T21:17:57.701 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000110653 2023-09-23T21:17:57.701 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:57.701 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:57.701 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:17:57.701 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:12:10.847302+0000", 2023-09-23T21:17:57.702 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0023382170000000001 2023-09-23T21:17:57.702 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:57.702 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:57.702 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:17:57.702 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:12:14.355481+0000", 2023-09-23T21:17:57.703 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.5081785220000001 2023-09-23T21:17:57.703 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:57.703 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:57.703 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:17:57.703 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:12:14.355532+0000", 2023-09-23T21:17:57.703 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.1251000000000001e-05 2023-09-23T21:17:57.703 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:57.703 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:57.703 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:17:57.703 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:12:14.355626+0000", 2023-09-23T21:17:57.704 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.4170000000000001e-05 2023-09-23T21:17:57.704 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:17:57.704 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:17:57.704 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:17:57.704 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:57.704 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:57.704 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.46328.0:84616 10.c 10:3d3dd8d1:::benchmark_data_smithi026_379838_object4659:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e472)", 2023-09-23T21:17:57.704 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:12:11.942844+0000", 2023-09-23T21:17:57.704 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 345.73189067700002, 2023-09-23T21:17:57.704 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.418419783, 2023-09-23T21:17:57.704 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:17:57.704 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:17:57.705 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:17:57.705 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.46328", 2023-09-23T21:17:57.705 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/3748861181", 2023-09-23T21:17:57.705 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 84616 2023-09-23T21:17:57.705 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:57.705 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:17:57.705 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:57.705 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:17:57.705 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:12:11.942844+0000", 2023-09-23T21:17:57.705 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:17:57.705 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:57.705 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:57.706 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:17:57.706 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:12:11.942844+0000", 2023-09-23T21:17:57.706 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:17:57.706 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:57.706 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:57.706 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:17:57.706 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:12:11.942843+0000", 2023-09-23T21:17:57.706 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T21:17:57.706 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:57.706 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:57.706 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:17:57.706 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:12:11.942853+0000", 2023-09-23T21:17:57.707 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.8509999999999994e-06 2023-09-23T21:17:57.707 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:57.707 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:57.707 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:17:57.707 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:12:11.942864+0000", 2023-09-23T21:17:57.707 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.1440999999999999e-05 2023-09-23T21:17:57.707 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:57.707 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:57.707 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:17:57.707 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:12:11.942872+0000", 2023-09-23T21:17:57.707 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.5669999999999997e-06 2023-09-23T21:17:57.707 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:57.707 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:57.708 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:17:57.708 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:12:11.942924+0000", 2023-09-23T21:17:57.708 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.1711e-05 2023-09-23T21:17:57.708 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:57.708 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:57.708 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:17:57.708 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:12:11.943052+0000", 2023-09-23T21:17:57.708 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00012839399999999999 2023-09-23T21:17:57.708 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:57.708 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:57.708 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 4", 2023-09-23T21:17:57.708 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:12:11.943174+0000", 2023-09-23T21:17:57.709 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000122368 2023-09-23T21:17:57.709 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:57.709 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:57.709 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:17:57.709 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:12:11.945587+0000", 2023-09-23T21:17:57.709 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0024131119999999998 2023-09-23T21:17:57.709 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:57.709 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:57.709 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:17:57.709 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:12:14.361092+0000", 2023-09-23T21:17:57.709 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.4155044399999999 2023-09-23T21:17:57.709 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:57.709 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:57.710 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:17:57.710 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:12:14.361151+0000", 2023-09-23T21:17:57.710 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.8795999999999997e-05 2023-09-23T21:17:57.710 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:57.710 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:57.710 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:17:57.710 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:12:14.361264+0000", 2023-09-23T21:17:57.710 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00011353600000000001 2023-09-23T21:17:57.710 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:17:57.710 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:17:57.710 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:17:57.711 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:57.711 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:57.711 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.46328.0:84636 10.c 10:3153f2e1:::benchmark_data_smithi026_379838_object4679:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e472)", 2023-09-23T21:17:57.711 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:12:12.042148+0000", 2023-09-23T21:17:57.711 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 345.63258741300001, 2023-09-23T21:17:57.711 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.3219429410000001, 2023-09-23T21:17:57.711 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:17:57.711 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:17:57.711 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:17:57.711 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.46328", 2023-09-23T21:17:57.711 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/3748861181", 2023-09-23T21:17:57.711 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 84636 2023-09-23T21:17:57.711 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:57.712 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:17:57.712 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:57.712 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:17:57.712 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:12:12.042148+0000", 2023-09-23T21:17:57.712 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:17:57.712 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:57.712 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:57.712 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:17:57.712 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:12:12.042148+0000", 2023-09-23T21:17:57.712 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:17:57.712 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:57.713 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:57.713 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:17:57.713 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:12:12.042146+0000", 2023-09-23T21:17:57.713 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T21:17:57.713 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:57.713 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:57.713 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:17:57.713 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:12:12.042155+0000", 2023-09-23T21:17:57.713 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.6279999999999994e-06 2023-09-23T21:17:57.713 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:57.713 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:57.713 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:17:57.714 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:12:12.042173+0000", 2023-09-23T21:17:57.714 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.7943000000000001e-05 2023-09-23T21:17:57.714 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:57.714 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:57.714 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:17:57.714 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:12:12.042181+0000", 2023-09-23T21:17:57.714 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.7399999999999993e-06 2023-09-23T21:17:57.714 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:57.714 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:57.714 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:17:57.714 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:12:12.042261+0000", 2023-09-23T21:17:57.714 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.9375999999999994e-05 2023-09-23T21:17:57.715 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:57.715 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:57.715 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:17:57.715 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:12:12.042414+0000", 2023-09-23T21:17:57.715 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00015368399999999999 2023-09-23T21:17:57.715 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:57.715 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:57.715 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 4", 2023-09-23T21:17:57.715 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:12:12.042558+0000", 2023-09-23T21:17:57.715 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000143747 2023-09-23T21:17:57.715 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:57.715 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:57.716 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:17:57.716 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:12:12.045442+0000", 2023-09-23T21:17:57.716 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.002883891 2023-09-23T21:17:57.716 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:57.716 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:57.716 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:17:57.716 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:12:14.363936+0000", 2023-09-23T21:17:57.716 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.3184934959999999 2023-09-23T21:17:57.716 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:57.716 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:57.716 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:17:57.717 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:12:14.363990+0000", 2023-09-23T21:17:57.717 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.4308000000000002e-05 2023-09-23T21:17:57.717 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:57.717 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:57.717 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:17:57.717 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:12:14.364091+0000", 2023-09-23T21:17:57.717 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00010072 2023-09-23T21:17:57.717 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:17:57.717 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:17:57.717 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:17:57.718 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:57.718 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:57.718 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.46328.0:84650 10.c 10:3f92ba16:::benchmark_data_smithi026_379838_object4693:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e472)", 2023-09-23T21:17:57.718 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:12:12.183251+0000", 2023-09-23T21:17:57.718 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 345.49148375499999, 2023-09-23T21:17:57.718 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.1852957819999999, 2023-09-23T21:17:57.718 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:17:57.718 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:17:57.718 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:17:57.718 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.46328", 2023-09-23T21:17:57.719 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/3748861181", 2023-09-23T21:17:57.719 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 84650 2023-09-23T21:17:57.719 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:57.719 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:17:57.719 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:57.719 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:17:57.719 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:12:12.183251+0000", 2023-09-23T21:17:57.719 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:17:57.719 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:57.720 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:57.720 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:17:57.720 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:12:12.183251+0000", 2023-09-23T21:17:57.720 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:17:57.720 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:57.720 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:57.720 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:17:57.720 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:12:12.183249+0000", 2023-09-23T21:17:57.720 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999981 2023-09-23T21:17:57.720 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:57.720 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:57.720 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:17:57.721 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:12:12.183260+0000", 2023-09-23T21:17:57.721 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.0908e-05 2023-09-23T21:17:57.721 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:57.721 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:57.721 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:17:57.721 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:12:12.183283+0000", 2023-09-23T21:17:57.721 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.3317000000000001e-05 2023-09-23T21:17:57.721 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:57.721 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:57.721 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:17:57.721 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:12:12.183294+0000", 2023-09-23T21:17:57.721 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.0893000000000001e-05 2023-09-23T21:17:57.721 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:57.722 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:57.722 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:17:57.722 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:12:12.183355+0000", 2023-09-23T21:17:57.722 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.0557000000000003e-05 2023-09-23T21:17:57.722 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:57.722 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:57.722 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:17:57.722 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:12:12.183467+0000", 2023-09-23T21:17:57.722 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000111694 2023-09-23T21:17:57.722 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:57.722 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:57.722 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 4", 2023-09-23T21:17:57.723 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:12:12.183562+0000", 2023-09-23T21:17:57.723 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.5743999999999997e-05 2023-09-23T21:17:57.723 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:57.723 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:57.723 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:17:57.723 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:12:12.186356+0000", 2023-09-23T21:17:57.723 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.002794 2023-09-23T21:17:57.723 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:57.723 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:57.723 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:17:57.723 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:12:14.368394+0000", 2023-09-23T21:17:57.723 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.1820373540000002 2023-09-23T21:17:57.724 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:57.724 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:57.724 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:17:57.724 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:12:14.368450+0000", 2023-09-23T21:17:57.724 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.6008000000000003e-05 2023-09-23T21:17:57.724 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:57.724 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:57.724 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:17:57.724 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:12:14.368547+0000", 2023-09-23T21:17:57.724 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.7404000000000005e-05 2023-09-23T21:17:57.724 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:17:57.725 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:17:57.725 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:17:57.725 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:57.725 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:57.725 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.46328.0:99131 10.11 10:8be3a531:::benchmark_data_smithi026_379838_object19174:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e490)", 2023-09-23T21:17:57.725 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:13:08.165674+0000", 2023-09-23T21:17:57.725 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 289.50906124199997, 2023-09-23T21:17:57.725 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.0267941519999999, 2023-09-23T21:17:57.725 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:17:57.725 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:17:57.725 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:17:57.725 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.46328", 2023-09-23T21:17:57.726 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/3748861181", 2023-09-23T21:17:57.726 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 99131 2023-09-23T21:17:57.726 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:57.726 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:17:57.726 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:57.726 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:17:57.726 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:08.165674+0000", 2023-09-23T21:17:57.726 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:17:57.726 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:57.726 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:57.726 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:17:57.726 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:08.165674+0000", 2023-09-23T21:17:57.726 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:17:57.727 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:57.727 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:57.727 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:17:57.727 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:08.165672+0000", 2023-09-23T21:17:57.727 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999976 2023-09-23T21:17:57.727 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:57.727 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:57.727 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:17:57.727 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:08.165680+0000", 2023-09-23T21:17:57.727 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.8590000000000005e-06 2023-09-23T21:17:57.727 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:57.727 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:57.727 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:17:57.728 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:08.165697+0000", 2023-09-23T21:17:57.728 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.7915999999999999e-05 2023-09-23T21:17:57.728 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:57.728 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:57.728 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:17:57.728 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:08.165706+0000", 2023-09-23T21:17:57.728 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.2709999999999992e-06 2023-09-23T21:17:57.728 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:57.728 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:57.728 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:17:57.728 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:08.165764+0000", 2023-09-23T21:17:57.728 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.8168000000000003e-05 2023-09-23T21:17:57.729 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:57.729 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:57.729 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:17:57.729 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:08.165773+0000", 2023-09-23T21:17:57.729 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.8000000000000004e-06 2023-09-23T21:17:57.729 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:57.729 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:57.729 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:17:57.729 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:09.168140+0000", 2023-09-23T21:17:57.729 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.002367622 2023-09-23T21:17:57.729 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:57.729 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:57.730 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:17:57.730 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:09.168266+0000", 2023-09-23T21:17:57.730 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000125475 2023-09-23T21:17:57.730 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:57.730 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:57.730 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 2,5", 2023-09-23T21:17:57.730 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:09.168387+0000", 2023-09-23T21:17:57.730 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000120924 2023-09-23T21:17:57.730 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:57.730 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:57.730 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:17:57.730 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:09.182822+0000", 2023-09-23T21:17:57.731 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.014435694000000001 2023-09-23T21:17:57.731 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:57.731 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:57.731 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:17:57.731 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:09.192311+0000", 2023-09-23T21:17:57.731 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0094883820000000001 2023-09-23T21:17:57.731 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:57.731 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:57.731 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:17:57.731 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:09.192385+0000", 2023-09-23T21:17:57.731 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.4046000000000002e-05 2023-09-23T21:17:57.731 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:57.732 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:57.732 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:17:57.732 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:09.192432+0000", 2023-09-23T21:17:57.732 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.6811999999999998e-05 2023-09-23T21:17:57.732 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:57.732 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:57.732 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:17:57.732 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:09.192468+0000", 2023-09-23T21:17:57.732 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.6359000000000002e-05 2023-09-23T21:17:57.732 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:17:57.732 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:17:57.732 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:17:57.733 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:57.733 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:57.733 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.46328.0:99137 10.11 10:8f77d61b:::benchmark_data_smithi026_379838_object19180:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e490)", 2023-09-23T21:17:57.733 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:13:08.177453+0000", 2023-09-23T21:17:57.733 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 289.49728170600002, 2023-09-23T21:17:57.733 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.015184732, 2023-09-23T21:17:57.733 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:17:57.733 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:17:57.733 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:17:57.733 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.46328", 2023-09-23T21:17:57.733 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/3748861181", 2023-09-23T21:17:57.733 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 99137 2023-09-23T21:17:57.733 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:57.734 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:17:57.734 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:57.734 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:17:57.734 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:08.177453+0000", 2023-09-23T21:17:57.734 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:17:57.734 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:57.734 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:57.734 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:17:57.734 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:08.177453+0000", 2023-09-23T21:17:57.734 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:17:57.734 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:57.735 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:57.735 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:17:57.735 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:08.177452+0000", 2023-09-23T21:17:57.735 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T21:17:57.735 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:57.735 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:57.735 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:17:57.735 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:08.177459+0000", 2023-09-23T21:17:57.735 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.1640000000000004e-06 2023-09-23T21:17:57.735 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:57.735 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:57.735 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:17:57.735 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:08.177471+0000", 2023-09-23T21:17:57.736 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.1389999999999999e-05 2023-09-23T21:17:57.736 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:57.736 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:57.736 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:17:57.736 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:08.177476+0000", 2023-09-23T21:17:57.736 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.6559999999999997e-06 2023-09-23T21:17:57.736 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:57.736 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:57.736 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:17:57.736 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:08.177530+0000", 2023-09-23T21:17:57.736 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.3658e-05 2023-09-23T21:17:57.736 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:57.737 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:57.737 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:17:57.737 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:08.177539+0000", 2023-09-23T21:17:57.737 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.8689999999999993e-06 2023-09-23T21:17:57.737 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:57.737 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:57.737 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:17:57.737 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:09.170152+0000", 2023-09-23T21:17:57.737 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.99261289500000005 2023-09-23T21:17:57.737 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:57.737 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:57.737 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:17:57.738 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:09.170272+0000", 2023-09-23T21:17:57.738 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00012026199999999999 2023-09-23T21:17:57.738 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:57.738 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:57.738 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 2,5", 2023-09-23T21:17:57.738 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:09.170397+0000", 2023-09-23T21:17:57.738 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000124797 2023-09-23T21:17:57.738 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:57.738 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:57.738 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:17:57.738 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:09.182836+0000", 2023-09-23T21:17:57.738 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.012439019000000001 2023-09-23T21:17:57.739 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:57.739 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:57.739 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:17:57.739 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:09.192337+0000", 2023-09-23T21:17:57.739 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0095009229999999997 2023-09-23T21:17:57.739 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:57.739 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:57.739 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:17:57.739 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:09.192568+0000", 2023-09-23T21:17:57.739 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00023114500000000001 2023-09-23T21:17:57.739 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:57.739 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:57.740 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:17:57.740 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:09.192606+0000", 2023-09-23T21:17:57.740 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.8201000000000002e-05 2023-09-23T21:17:57.740 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:57.740 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:57.740 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:17:57.740 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:09.192638+0000", 2023-09-23T21:17:57.740 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.2156999999999997e-05 2023-09-23T21:17:57.740 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:17:57.740 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:17:57.740 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:17:57.740 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:57.741 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:57.741 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.46328.0:99138 10.11 10:8adc2e9e:::benchmark_data_smithi026_379838_object19181:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e490)", 2023-09-23T21:17:57.741 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:13:08.182950+0000", 2023-09-23T21:17:57.741 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 289.491785515, 2023-09-23T21:17:57.741 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.0124994359999999, 2023-09-23T21:17:57.741 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:17:57.741 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:17:57.741 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:17:57.741 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.46328", 2023-09-23T21:17:57.741 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/3748861181", 2023-09-23T21:17:57.741 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 99138 2023-09-23T21:17:57.741 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:57.742 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:17:57.742 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:57.742 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:17:57.742 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:08.182950+0000", 2023-09-23T21:17:57.742 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:17:57.742 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:57.742 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:57.742 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:17:57.742 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:08.182950+0000", 2023-09-23T21:17:57.742 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:17:57.742 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:57.742 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:57.743 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:17:57.743 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:08.182948+0000", 2023-09-23T21:17:57.743 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T21:17:57.743 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:57.743 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:57.743 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:17:57.743 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:08.182956+0000", 2023-09-23T21:17:57.743 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.1669999999999999e-06 2023-09-23T21:17:57.743 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:57.743 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:57.743 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:17:57.743 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:08.182976+0000", 2023-09-23T21:17:57.743 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.9758e-05 2023-09-23T21:17:57.744 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:57.744 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:57.744 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:17:57.744 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:08.182983+0000", 2023-09-23T21:17:57.744 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.2799999999999998e-06 2023-09-23T21:17:57.744 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:57.744 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:57.744 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:17:57.744 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:08.183043+0000", 2023-09-23T21:17:57.744 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.0022999999999998e-05 2023-09-23T21:17:57.744 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:57.744 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:57.745 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:17:57.745 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:08.183053+0000", 2023-09-23T21:17:57.745 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.005e-05 2023-09-23T21:17:57.745 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:57.745 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:57.745 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:17:57.745 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:09.172086+0000", 2023-09-23T21:17:57.745 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.98903244300000004 2023-09-23T21:17:57.745 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:57.745 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:57.745 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:17:57.745 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:09.172158+0000", 2023-09-23T21:17:57.745 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.2646999999999997e-05 2023-09-23T21:17:57.746 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:57.746 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:57.746 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 2,5", 2023-09-23T21:17:57.746 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:09.174279+0000", 2023-09-23T21:17:57.746 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0021205820000000002 2023-09-23T21:17:57.746 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:57.746 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:57.746 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:17:57.746 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:09.182841+0000", 2023-09-23T21:17:57.746 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0085623030000000003 2023-09-23T21:17:57.746 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:57.746 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:57.747 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:17:57.747 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:09.192361+0000", 2023-09-23T21:17:57.747 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0095194019999999997 2023-09-23T21:17:57.747 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:57.747 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:57.747 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:17:57.747 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:09.195355+0000", 2023-09-23T21:17:57.747 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0029945000000000002 2023-09-23T21:17:57.747 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:57.747 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:57.747 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:17:57.747 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:09.195404+0000", 2023-09-23T21:17:57.747 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.9138999999999998e-05 2023-09-23T21:17:57.748 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:57.748 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:57.748 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:17:57.748 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:09.195449+0000", 2023-09-23T21:17:57.748 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.4781999999999999e-05 2023-09-23T21:17:57.748 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:17:57.748 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:17:57.748 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:17:57.748 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:57.748 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:57.748 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.46328.0:99977 10.c 10:3f71b81a:::benchmark_data_smithi026_379838_object20020:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e493)", 2023-09-23T21:17:57.748 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:13:11.836139+0000", 2023-09-23T21:17:57.748 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 285.83859588399997, 2023-09-23T21:17:57.749 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.594515414, 2023-09-23T21:17:57.749 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:17:57.749 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:17:57.749 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:17:57.749 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.46328", 2023-09-23T21:17:57.749 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/3748861181", 2023-09-23T21:17:57.749 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 99977 2023-09-23T21:17:57.749 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:57.749 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:17:57.749 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:57.749 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:17:57.749 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:11.836139+0000", 2023-09-23T21:17:57.750 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:17:57.750 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:57.750 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:57.750 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:17:57.750 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:11.836139+0000", 2023-09-23T21:17:57.750 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:17:57.750 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:57.750 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:57.750 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:17:57.750 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:11.836138+0000", 2023-09-23T21:17:57.750 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T21:17:57.750 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:57.750 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:57.751 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:17:57.751 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:11.836143+0000", 2023-09-23T21:17:57.751 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.293e-06 2023-09-23T21:17:57.751 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:57.751 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:57.751 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:17:57.751 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:11.836157+0000", 2023-09-23T21:17:57.751 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.3258000000000001e-05 2023-09-23T21:17:57.751 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:57.751 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:57.751 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:17:57.751 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:11.836162+0000", 2023-09-23T21:17:57.752 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.4979999999999997e-06 2023-09-23T21:17:57.752 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:57.752 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:57.752 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:17:57.752 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:11.836212+0000", 2023-09-23T21:17:57.752 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.0225000000000001e-05 2023-09-23T21:17:57.752 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:57.752 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:57.752 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:17:57.752 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:11.836222+0000", 2023-09-23T21:17:57.752 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.8959999999999998e-06 2023-09-23T21:17:57.752 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:57.753 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:57.753 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:17:57.753 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:12.185368+0000", 2023-09-23T21:17:57.753 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.34914541399999999 2023-09-23T21:17:57.753 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:57.753 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:57.753 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:17:57.753 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:12.185381+0000", 2023-09-23T21:17:57.753 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.3546e-05 2023-09-23T21:17:57.753 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:57.753 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:57.753 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:17:57.754 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:13.420174+0000", 2023-09-23T21:17:57.754 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.2347927400000001 2023-09-23T21:17:57.754 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:57.754 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:57.754 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:17:57.754 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:13.420332+0000", 2023-09-23T21:17:57.754 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00015844899999999999 2023-09-23T21:17:57.754 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:57.754 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:57.754 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 0,4", 2023-09-23T21:17:57.754 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:13.420503+0000", 2023-09-23T21:17:57.754 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000170373 2023-09-23T21:17:57.755 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:57.755 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:57.755 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:17:57.755 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:13.425495+0000", 2023-09-23T21:17:57.755 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0049922029999999997 2023-09-23T21:17:57.755 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:57.755 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:57.755 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:17:57.755 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:13.425508+0000", 2023-09-23T21:17:57.755 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.2761e-05 2023-09-23T21:17:57.755 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:57.755 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:57.755 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:17:57.756 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:13.430310+0000", 2023-09-23T21:17:57.756 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0048026220000000003 2023-09-23T21:17:57.756 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:57.756 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:57.756 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:17:57.756 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:13.430370+0000", 2023-09-23T21:17:57.756 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.012e-05 2023-09-23T21:17:57.756 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:57.756 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:57.756 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:17:57.756 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:13.430655+0000", 2023-09-23T21:17:57.756 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00028428600000000002 2023-09-23T21:17:57.757 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:17:57.757 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:17:57.757 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:17:57.757 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:57.757 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:57.757 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.46328.0:99978 10.c 10:38231695:::benchmark_data_smithi026_379838_object20021:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e493)", 2023-09-23T21:17:57.757 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:13:11.847037+0000", 2023-09-23T21:17:57.757 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 285.82769781299999, 2023-09-23T21:17:57.757 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.583935267, 2023-09-23T21:17:57.757 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:17:57.757 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:17:57.757 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:17:57.758 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.46328", 2023-09-23T21:17:57.758 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/3748861181", 2023-09-23T21:17:57.758 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 99978 2023-09-23T21:17:57.758 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:57.758 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:17:57.758 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:57.758 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:17:57.758 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:11.847037+0000", 2023-09-23T21:17:57.758 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:17:57.758 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:57.758 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:57.758 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:17:57.759 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:11.847037+0000", 2023-09-23T21:17:57.759 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:17:57.759 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:57.759 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:57.759 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:17:57.759 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:11.847036+0000", 2023-09-23T21:17:57.759 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.999999 2023-09-23T21:17:57.759 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:57.759 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:57.759 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:17:57.759 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:11.847042+0000", 2023-09-23T21:17:57.759 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.4060000000000004e-06 2023-09-23T21:17:57.760 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:57.760 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:57.760 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:17:57.760 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:11.847057+0000", 2023-09-23T21:17:57.760 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.4637999999999999e-05 2023-09-23T21:17:57.760 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:57.760 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:57.760 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:17:57.760 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:11.847062+0000", 2023-09-23T21:17:57.760 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.0370000000000002e-06 2023-09-23T21:17:57.760 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:57.760 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:57.761 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:17:57.761 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:11.847110+0000", 2023-09-23T21:17:57.761 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.8687999999999999e-05 2023-09-23T21:17:57.761 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:57.761 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:57.761 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:17:57.761 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:11.847121+0000", 2023-09-23T21:17:57.761 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.0318e-05 2023-09-23T21:17:57.761 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:57.761 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:57.761 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:17:57.761 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:12.185400+0000", 2023-09-23T21:17:57.761 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.33827918099999998 2023-09-23T21:17:57.762 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:57.762 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:57.762 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:17:57.762 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:12.185404+0000", 2023-09-23T21:17:57.762 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.1219999999999997e-06 2023-09-23T21:17:57.762 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:57.762 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:57.762 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:17:57.762 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:13.422861+0000", 2023-09-23T21:17:57.762 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.2374567430000001 2023-09-23T21:17:57.762 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:57.762 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:57.763 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:17:57.763 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:13.423010+0000", 2023-09-23T21:17:57.763 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00014951799999999999 2023-09-23T21:17:57.763 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:57.763 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:57.763 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 0,4", 2023-09-23T21:17:57.763 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:13.423186+0000", 2023-09-23T21:17:57.763 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00017603200000000001 2023-09-23T21:17:57.763 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:57.763 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:57.763 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:17:57.763 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:13.425518+0000", 2023-09-23T21:17:57.764 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0023320609999999999 2023-09-23T21:17:57.764 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:57.764 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:57.764 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:17:57.764 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:13.425561+0000", 2023-09-23T21:17:57.764 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.2370000000000003e-05 2023-09-23T21:17:57.764 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:57.764 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:57.764 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:17:57.764 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:13.430729+0000", 2023-09-23T21:17:57.764 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0051681549999999998 2023-09-23T21:17:57.764 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:57.764 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:57.765 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:17:57.765 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:13.430788+0000", 2023-09-23T21:17:57.765 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.8826999999999998e-05 2023-09-23T21:17:57.765 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:57.765 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:57.765 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:17:57.765 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:13.430973+0000", 2023-09-23T21:17:57.765 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000184997 2023-09-23T21:17:57.765 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:17:57.765 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:17:57.765 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:17:57.765 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:57.766 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:57.766 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.46823.0:78878 11.13 11.673fe373 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e564)", 2023-09-23T21:17:57.766 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:16:41.442423+0000", 2023-09-23T21:17:57.766 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 76.232312582999995, 2023-09-23T21:17:57.766 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.96127208099999994, 2023-09-23T21:17:57.766 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:17:57.766 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "reached pg", 2023-09-23T21:17:57.766 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:17:57.766 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.46823", 2023-09-23T21:17:57.766 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/2490075711", 2023-09-23T21:17:57.766 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 78878 2023-09-23T21:17:57.766 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:57.766 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:17:57.767 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:57.767 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:17:57.767 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:16:41.442423+0000", 2023-09-23T21:17:57.767 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:17:57.767 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:57.767 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:57.767 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:17:57.767 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:16:41.442423+0000", 2023-09-23T21:17:57.767 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:17:57.767 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:57.767 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:57.767 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:17:57.768 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:16:41.442420+0000", 2023-09-23T21:17:57.768 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999976 2023-09-23T21:17:57.768 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:57.768 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:57.768 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:17:57.768 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:16:41.442544+0000", 2023-09-23T21:17:57.768 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00012382100000000001 2023-09-23T21:17:57.768 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:57.768 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:57.768 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:17:57.768 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:16:41.442884+0000", 2023-09-23T21:17:57.768 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00033983100000000002 2023-09-23T21:17:57.768 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:57.769 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:57.769 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:17:57.769 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:16:41.442895+0000", 2023-09-23T21:17:57.769 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.1404e-05 2023-09-23T21:17:57.769 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:57.769 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:57.769 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:17:57.769 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:16:41.442968+0000", 2023-09-23T21:17:57.769 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.2801999999999999e-05 2023-09-23T21:17:57.769 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:57.769 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:57.769 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:17:57.769 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:16:41.442982+0000", 2023-09-23T21:17:57.770 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.3477e-05 2023-09-23T21:17:57.770 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:57.770 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:57.770 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:17:57.770 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:16:42.403685+0000", 2023-09-23T21:17:57.770 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.96070322100000005 2023-09-23T21:17:57.770 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:57.770 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:57.770 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:17:57.770 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:16:42.403695+0000", 2023-09-23T21:17:57.770 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.8179999999999999e-06 2023-09-23T21:17:57.770 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:17:57.771 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:17:57.771 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:17:57.771 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:57.771 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:57.771 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.46823.0:78879 11.4 11.ec4574c4 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e564)", 2023-09-23T21:17:57.771 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:16:41.445700+0000", 2023-09-23T21:17:57.771 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 76.229035506000002, 2023-09-23T21:17:57.771 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.95796339399999997, 2023-09-23T21:17:57.771 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:17:57.771 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "reached pg", 2023-09-23T21:17:57.771 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:17:57.771 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.46823", 2023-09-23T21:17:57.772 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/2490075711", 2023-09-23T21:17:57.772 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 78879 2023-09-23T21:17:57.772 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:57.772 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:17:57.772 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:57.772 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:17:57.772 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:16:41.445700+0000", 2023-09-23T21:17:57.772 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:17:57.772 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:57.772 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:57.772 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:17:57.772 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:16:41.445700+0000", 2023-09-23T21:17:57.773 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:17:57.773 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:57.773 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:57.773 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:17:57.773 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:16:41.445698+0000", 2023-09-23T21:17:57.773 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T21:17:57.773 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:57.773 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:57.773 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:17:57.773 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:16:41.445787+0000", 2023-09-23T21:17:57.773 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.8627000000000004e-05 2023-09-23T21:17:57.773 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:57.773 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:57.774 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:17:57.774 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:16:41.445800+0000", 2023-09-23T21:17:57.774 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.324e-05 2023-09-23T21:17:57.774 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:57.774 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:57.774 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:17:57.774 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:16:41.445808+0000", 2023-09-23T21:17:57.774 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.909e-06 2023-09-23T21:17:57.774 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:57.774 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:57.774 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:17:57.774 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:16:41.447324+0000", 2023-09-23T21:17:57.775 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0015165879999999999 2023-09-23T21:17:57.775 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:57.775 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:57.775 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:17:57.775 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:16:41.447335+0000", 2023-09-23T21:17:57.775 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.0489000000000001e-05 2023-09-23T21:17:57.775 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:57.775 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:57.775 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:17:57.775 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:16:42.403651+0000", 2023-09-23T21:17:57.775 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.95631578299999997 2023-09-23T21:17:57.775 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:57.775 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:57.776 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:17:57.776 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:16:42.403663+0000", 2023-09-23T21:17:57.776 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.2391e-05 2023-09-23T21:17:57.776 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:17:57.776 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:17:57.776 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:17:57.776 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:57.776 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:57.776 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.46823.0:78884 11.14 11.89e78494 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e564)", 2023-09-23T21:17:57.776 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:16:41.456798+0000", 2023-09-23T21:17:57.776 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 76.217936961999996, 2023-09-23T21:17:57.776 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.94574142900000002, 2023-09-23T21:17:57.777 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:17:57.777 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "reached pg", 2023-09-23T21:17:57.777 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:17:57.777 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.46823", 2023-09-23T21:17:57.777 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/2490075711", 2023-09-23T21:17:57.777 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 78884 2023-09-23T21:17:57.777 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:57.777 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:17:57.777 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:57.777 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:17:57.777 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:16:41.456798+0000", 2023-09-23T21:17:57.777 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:17:57.777 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:57.778 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:57.778 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:17:57.778 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:16:41.456798+0000", 2023-09-23T21:17:57.778 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:17:57.778 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:57.778 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:57.778 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:17:57.778 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:16:41.456797+0000", 2023-09-23T21:17:57.778 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T21:17:57.778 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:57.778 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:57.778 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:17:57.779 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:16:41.456882+0000", 2023-09-23T21:17:57.779 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.4870000000000006e-05 2023-09-23T21:17:57.779 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:57.779 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:57.779 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:17:57.779 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:16:41.456897+0000", 2023-09-23T21:17:57.779 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.5135e-05 2023-09-23T21:17:57.779 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:57.779 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:57.779 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:17:57.779 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:16:41.456904+0000", 2023-09-23T21:17:57.779 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.7920000000000004e-06 2023-09-23T21:17:57.779 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:57.780 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:57.780 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:17:57.780 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:16:41.456954+0000", 2023-09-23T21:17:57.780 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.0022999999999999e-05 2023-09-23T21:17:57.780 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:57.780 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:57.780 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:17:57.780 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:16:41.456963+0000", 2023-09-23T21:17:57.780 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.0480000000000004e-06 2023-09-23T21:17:57.780 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:57.780 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:57.780 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:17:57.781 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:16:42.402524+0000", 2023-09-23T21:17:57.781 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.94556156099999999 2023-09-23T21:17:57.781 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:57.781 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:57.781 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:17:57.781 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:16:42.402540+0000", 2023-09-23T21:17:57.781 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.524e-05 2023-09-23T21:17:57.781 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:17:57.781 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:17:57.781 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:17:57.781 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:57.781 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:57.781 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.46823.0:78889 11.13 11.75b12073 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e564)", 2023-09-23T21:17:57.782 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:16:41.472025+0000", 2023-09-23T21:17:57.782 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 76.202710091, 2023-09-23T21:17:57.782 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.931725519, 2023-09-23T21:17:57.782 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:17:57.782 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "reached pg", 2023-09-23T21:17:57.782 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:17:57.782 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.46823", 2023-09-23T21:17:57.782 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/2490075711", 2023-09-23T21:17:57.782 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 78889 2023-09-23T21:17:57.782 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:57.782 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:17:57.782 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:57.783 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:17:57.783 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:16:41.472025+0000", 2023-09-23T21:17:57.783 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:17:57.783 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:57.783 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:57.783 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:17:57.783 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:16:41.472025+0000", 2023-09-23T21:17:57.783 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:17:57.783 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:57.783 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:57.783 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:17:57.783 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:16:41.472024+0000", 2023-09-23T21:17:57.784 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T21:17:57.784 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:57.784 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:57.784 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:17:57.784 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:16:41.472093+0000", 2023-09-23T21:17:57.784 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.9156e-05 2023-09-23T21:17:57.784 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:57.784 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:57.784 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:17:57.784 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:16:41.472109+0000", 2023-09-23T21:17:57.784 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.6322000000000001e-05 2023-09-23T21:17:57.784 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:57.785 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:57.785 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:17:57.785 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:16:41.472116+0000", 2023-09-23T21:17:57.785 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.3980000000000004e-06 2023-09-23T21:17:57.785 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:57.785 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:57.785 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:17:57.785 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:16:41.472174+0000", 2023-09-23T21:17:57.785 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.8882999999999998e-05 2023-09-23T21:17:57.785 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:57.785 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:57.785 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:17:57.785 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:16:41.472183+0000", 2023-09-23T21:17:57.786 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.3180000000000007e-06 2023-09-23T21:17:57.786 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:57.786 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:57.786 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:17:57.786 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:16:42.403731+0000", 2023-09-23T21:17:57.786 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.93154813700000005 2023-09-23T21:17:57.786 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:57.786 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:57.786 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:17:57.786 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:16:42.403751+0000", 2023-09-23T21:17:57.786 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.9650999999999999e-05 2023-09-23T21:17:57.787 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:17:57.787 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:17:57.787 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:17:57.787 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:57.787 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:57.787 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.46823.0:132646 11.13 11:ca339c49:::benchmark_data_smithi026_424533_object34676:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e585)", 2023-09-23T21:17:57.787 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:17:38.907633+0000", 2023-09-23T21:17:57.787 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 18.767101956000001, 2023-09-23T21:17:57.787 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.0115563889999999, 2023-09-23T21:17:57.787 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:17:57.787 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "reached pg", 2023-09-23T21:17:57.787 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:17:57.787 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.46823", 2023-09-23T21:17:57.788 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/2490075711", 2023-09-23T21:17:57.788 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 132646 2023-09-23T21:17:57.788 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:57.788 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:17:57.788 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:57.788 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:17:57.788 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:17:38.907633+0000", 2023-09-23T21:17:57.788 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:17:57.788 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:57.788 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:57.788 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:17:57.788 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:17:38.907633+0000", 2023-09-23T21:17:57.789 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:17:57.789 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:57.789 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:57.789 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:17:57.789 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:17:38.907632+0000", 2023-09-23T21:17:57.789 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T21:17:57.789 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:57.789 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:57.789 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:17:57.789 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:17:38.907638+0000", 2023-09-23T21:17:57.789 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.4930000000000003e-06 2023-09-23T21:17:57.789 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:57.790 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:57.790 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:17:57.790 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:17:38.907651+0000", 2023-09-23T21:17:57.790 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.261e-05 2023-09-23T21:17:57.790 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:57.790 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:57.790 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:17:57.790 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:17:38.907658+0000", 2023-09-23T21:17:57.790 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.5979999999999999e-06 2023-09-23T21:17:57.790 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:57.790 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:57.790 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:17:57.791 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:17:38.908926+0000", 2023-09-23T21:17:57.791 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.001268087 2023-09-23T21:17:57.791 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:57.791 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:57.791 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:17:57.791 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:17:38.908935+0000", 2023-09-23T21:17:57.791 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.8699999999999998e-06 2023-09-23T21:17:57.791 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:57.791 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:57.791 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:17:57.791 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:17:39.919098+0000", 2023-09-23T21:17:57.792 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.0101625329999999 2023-09-23T21:17:57.792 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:57.792 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:57.792 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:17:57.792 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:17:39.919190+0000", 2023-09-23T21:17:57.792 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.1686000000000003e-05 2023-09-23T21:17:57.792 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:17:57.792 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:17:57.792 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:17:57.792 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:57.792 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:57.792 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.46823.0:132654 11.13 11:cdfb262e:::benchmark_data_smithi026_424533_object34684:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e585)", 2023-09-23T21:17:57.793 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:17:38.907670+0000", 2023-09-23T21:17:57.793 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 18.767065321, 2023-09-23T21:17:57.793 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.049192116, 2023-09-23T21:17:57.793 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:17:57.793 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "reached pg", 2023-09-23T21:17:57.793 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:17:57.793 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.46823", 2023-09-23T21:17:57.793 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/2490075711", 2023-09-23T21:17:57.793 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 132654 2023-09-23T21:17:57.793 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:57.793 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:17:57.793 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:57.793 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:17:57.794 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:17:38.907670+0000", 2023-09-23T21:17:57.794 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:17:57.794 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:57.794 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:57.794 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:17:57.794 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:17:38.907670+0000", 2023-09-23T21:17:57.794 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:17:57.794 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:57.794 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:57.794 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:17:57.794 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:17:38.907669+0000", 2023-09-23T21:17:57.794 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999995 2023-09-23T21:17:57.795 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:57.795 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:57.795 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:17:57.795 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:17:38.907672+0000", 2023-09-23T21:17:57.795 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.7389999999999999e-06 2023-09-23T21:17:57.795 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:57.795 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:57.795 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:17:57.795 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:17:38.907678+0000", 2023-09-23T21:17:57.795 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.3010000000000003e-06 2023-09-23T21:17:57.795 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:57.795 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:57.796 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:17:57.796 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:17:38.907683+0000", 2023-09-23T21:17:57.796 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.5990000000000004e-06 2023-09-23T21:17:57.796 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:57.796 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:57.796 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:17:57.796 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:17:38.908961+0000", 2023-09-23T21:17:57.796 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00127783 2023-09-23T21:17:57.796 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:57.796 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:57.796 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:17:57.796 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:17:38.908968+0000", 2023-09-23T21:17:57.796 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.7440000000000001e-06 2023-09-23T21:17:57.797 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:57.797 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:57.797 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:17:57.797 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:17:39.919215+0000", 2023-09-23T21:17:57.797 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.0102470349999999 2023-09-23T21:17:57.797 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:57.797 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:57.797 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for missing object", 2023-09-23T21:17:57.797 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:17:39.919299+0000", 2023-09-23T21:17:57.797 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.4629000000000006e-05 2023-09-23T21:17:57.797 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:57.797 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:57.798 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:17:57.798 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:17:39.956770+0000", 2023-09-23T21:17:57.798 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.037470891999999999 2023-09-23T21:17:57.798 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:57.798 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:57.798 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:17:57.798 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:17:39.956862+0000", 2023-09-23T21:17:57.798 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.1791000000000003e-05 2023-09-23T21:17:57.798 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:17:57.798 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:17:57.798 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:17:57.798 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:57.799 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:57.799 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.46823.0:132697 11.13 11:cff8e3f6:::benchmark_data_smithi026_424533_object34727:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e585)", 2023-09-23T21:17:57.799 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:17:38.957240+0000", 2023-09-23T21:17:57.799 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 18.717495502999999, 2023-09-23T21:17:57.799 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.99985294400000002, 2023-09-23T21:17:57.799 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:17:57.799 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:17:57.799 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:17:57.799 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.46823", 2023-09-23T21:17:57.799 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/2490075711", 2023-09-23T21:17:57.799 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 132697 2023-09-23T21:17:57.799 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:57.799 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:17:57.800 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:57.800 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:17:57.800 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:17:38.957240+0000", 2023-09-23T21:17:57.800 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:17:57.800 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:57.800 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:57.800 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:17:57.800 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:17:38.957240+0000", 2023-09-23T21:17:57.800 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:17:57.800 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:57.800 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:57.800 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:17:57.800 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:17:38.957238+0000", 2023-09-23T21:17:57.801 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T21:17:57.801 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:57.801 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:57.801 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:17:57.801 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:17:38.957246+0000", 2023-09-23T21:17:57.801 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.3819999999999998e-06 2023-09-23T21:17:57.801 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:57.801 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:57.801 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:17:57.801 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:17:38.957261+0000", 2023-09-23T21:17:57.801 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.5299999999999999e-05 2023-09-23T21:17:57.801 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:57.801 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:57.802 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:17:57.802 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:17:38.957268+0000", 2023-09-23T21:17:57.802 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.0350000000000004e-06 2023-09-23T21:17:57.802 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:57.802 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:57.802 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:17:57.802 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:17:38.957336+0000", 2023-09-23T21:17:57.802 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.7711000000000003e-05 2023-09-23T21:17:57.802 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:57.802 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:57.802 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:17:57.802 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:17:38.957346+0000", 2023-09-23T21:17:57.803 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.0536e-05 2023-09-23T21:17:57.803 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:57.803 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:57.803 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:17:57.803 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:17:39.919336+0000", 2023-09-23T21:17:57.803 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.96198997799999997 2023-09-23T21:17:57.803 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:57.803 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:57.803 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:17:57.803 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:17:39.919447+0000", 2023-09-23T21:17:57.803 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000111399 2023-09-23T21:17:57.803 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:57.803 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:57.804 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 2,5", 2023-09-23T21:17:57.805 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:17:39.919789+0000", 2023-09-23T21:17:57.805 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00034173100000000001 2023-09-23T21:17:57.805 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:57.805 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:57.805 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:17:57.805 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:17:39.930825+0000", 2023-09-23T21:17:57.805 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.011035771 2023-09-23T21:17:57.806 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:57.806 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:57.806 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:17:57.806 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:17:39.956895+0000", 2023-09-23T21:17:57.806 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.026069858000000001 2023-09-23T21:17:57.806 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:57.806 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:57.806 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:17:57.807 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:17:39.956998+0000", 2023-09-23T21:17:57.807 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000102779 2023-09-23T21:17:57.807 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:57.807 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:57.807 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:17:57.807 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:17:39.957033+0000", 2023-09-23T21:17:57.807 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.5037999999999999e-05 2023-09-23T21:17:57.807 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:57.807 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:57.807 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:17:57.808 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:17:39.957093+0000", 2023-09-23T21:17:57.808 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.9908999999999998e-05 2023-09-23T21:17:57.808 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:17:57.808 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:17:57.808 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:17:57.808 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:57.808 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:57.808 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.46823.0:132706 11.13 11:ccd1f15a:::benchmark_data_smithi026_424533_object34736:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e585)", 2023-09-23T21:17:57.808 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:17:38.962140+0000", 2023-09-23T21:17:57.808 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 18.712595179000001, 2023-09-23T21:17:57.808 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.99511751500000001, 2023-09-23T21:17:57.809 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:17:57.809 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:17:57.809 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:17:57.809 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.46823", 2023-09-23T21:17:57.809 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/2490075711", 2023-09-23T21:17:57.809 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 132706 2023-09-23T21:17:57.809 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:57.809 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:17:57.809 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:57.810 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:17:57.810 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:17:38.962140+0000", 2023-09-23T21:17:57.810 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:17:57.810 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:57.810 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:57.810 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:17:57.810 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:17:38.962140+0000", 2023-09-23T21:17:57.810 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:17:57.810 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:57.810 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:57.811 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:17:57.811 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:17:38.962139+0000", 2023-09-23T21:17:57.811 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.999999 2023-09-23T21:17:57.811 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:57.811 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:57.811 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:17:57.811 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:17:38.962144+0000", 2023-09-23T21:17:57.811 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.0200000000000002e-06 2023-09-23T21:17:57.811 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:57.811 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:57.811 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:17:57.812 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:17:38.962154+0000", 2023-09-23T21:17:57.812 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.0553e-05 2023-09-23T21:17:57.812 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:57.812 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:57.812 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:17:57.812 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:17:38.962163+0000", 2023-09-23T21:17:57.812 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.5180000000000004e-06 2023-09-23T21:17:57.812 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:57.812 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:57.813 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:17:57.813 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:17:38.962206+0000", 2023-09-23T21:17:57.813 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.3259e-05 2023-09-23T21:17:57.813 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:57.813 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:57.813 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:17:57.813 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:17:38.962213+0000", 2023-09-23T21:17:57.813 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.9600000000000003e-06 2023-09-23T21:17:57.813 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:57.813 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:57.813 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:17:57.814 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:17:39.921342+0000", 2023-09-23T21:17:57.814 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.959129024 2023-09-23T21:17:57.814 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:57.814 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:57.814 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:17:57.814 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:17:39.921455+0000", 2023-09-23T21:17:57.814 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000112425 2023-09-23T21:17:57.814 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:57.814 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:57.814 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 2,5", 2023-09-23T21:17:57.815 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:17:39.921670+0000", 2023-09-23T21:17:57.815 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00021546799999999999 2023-09-23T21:17:57.815 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:57.815 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:57.815 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:17:57.815 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:17:39.930835+0000", 2023-09-23T21:17:57.815 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0091648760000000006 2023-09-23T21:17:57.815 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:57.815 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:57.815 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:17:57.816 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:17:39.956923+0000", 2023-09-23T21:17:57.816 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.026087909999999999 2023-09-23T21:17:57.816 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:57.816 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:57.816 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:17:57.816 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:17:39.957133+0000", 2023-09-23T21:17:57.816 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000209915 2023-09-23T21:17:57.816 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:57.817 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:57.817 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:17:57.817 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:17:39.957164+0000", 2023-09-23T21:17:57.817 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.1217000000000003e-05 2023-09-23T21:17:57.817 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:57.817 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:57.817 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:17:57.817 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:17:39.957257+0000", 2023-09-23T21:17:57.817 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.3411000000000004e-05 2023-09-23T21:17:57.817 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:17:57.817 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:17:57.818 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:17:57.818 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:57.818 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:57.818 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.46823.0:132782 11.13 11:c97129e2:::benchmark_data_smithi026_424533_object34812:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e585)", 2023-09-23T21:17:57.818 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:17:39.028998+0000", 2023-09-23T21:17:57.818 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 18.645736915000001, 2023-09-23T21:17:57.818 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.928436127, 2023-09-23T21:17:57.818 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:17:57.818 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:17:57.818 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:17:57.819 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.46823", 2023-09-23T21:17:57.819 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/2490075711", 2023-09-23T21:17:57.819 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 132782 2023-09-23T21:17:57.819 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:57.819 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:17:57.819 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:57.819 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:17:57.819 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:17:39.028998+0000", 2023-09-23T21:17:57.819 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:17:57.819 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:57.820 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:57.820 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:17:57.820 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:17:39.028998+0000", 2023-09-23T21:17:57.820 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:17:57.820 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:57.820 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:57.820 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:17:57.820 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:17:39.028997+0000", 2023-09-23T21:17:57.820 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T21:17:57.820 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:57.820 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:57.821 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:17:57.821 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:17:39.029003+0000", 2023-09-23T21:17:57.821 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.1129999999999997e-06 2023-09-23T21:17:57.821 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:57.821 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:57.821 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:17:57.821 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:17:39.029019+0000", 2023-09-23T21:17:57.821 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.6348999999999999e-05 2023-09-23T21:17:57.821 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:57.821 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:57.821 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:17:57.822 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:17:39.029028+0000", 2023-09-23T21:17:57.822 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.3010000000000007e-06 2023-09-23T21:17:57.822 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:57.822 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:57.822 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:17:57.822 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:17:39.029081+0000", 2023-09-23T21:17:57.822 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.3412999999999998e-05 2023-09-23T21:17:57.822 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:57.822 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:57.822 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:17:57.823 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:17:39.029088+0000", 2023-09-23T21:17:57.823 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.8210000000000004e-06 2023-09-23T21:17:57.823 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:57.823 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:57.823 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:17:57.823 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:17:39.922875+0000", 2023-09-23T21:17:57.823 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.89378761600000001 2023-09-23T21:17:57.823 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:57.823 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:57.823 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:17:57.824 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:17:39.922948+0000", 2023-09-23T21:17:57.824 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.2738999999999996e-05 2023-09-23T21:17:57.824 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:57.824 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:57.824 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 2,5", 2023-09-23T21:17:57.824 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:17:39.923097+0000", 2023-09-23T21:17:57.824 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00014898099999999999 2023-09-23T21:17:57.824 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:57.824 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:57.824 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:17:57.825 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:17:39.930843+0000", 2023-09-23T21:17:57.825 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0077458470000000001 2023-09-23T21:17:57.825 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:57.825 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:57.825 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:17:57.825 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:17:39.956950+0000", 2023-09-23T21:17:57.825 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.026106937 2023-09-23T21:17:57.825 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:57.825 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:57.825 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:17:57.826 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:17:39.957310+0000", 2023-09-23T21:17:57.826 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000359809 2023-09-23T21:17:57.826 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:57.826 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:57.826 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:17:57.826 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:17:39.957342+0000", 2023-09-23T21:17:57.826 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.1992000000000001e-05 2023-09-23T21:17:57.826 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:17:57.826 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:17:57.827 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:17:57.827 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:17:39.957434+0000", 2023-09-23T21:17:57.827 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.2558000000000004e-05 2023-09-23T21:17:57.827 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:17:57.827 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:17:57.827 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:17:57.827 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:17:57.827 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:17:57.827 INFO:teuthology.orchestra.run.smithi070.stdout:} 2023-09-23T21:17:57.828 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:17:57.866 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:17:58.053 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:17:58.057 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:17:58.058 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:17:58.072 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_ops_in_flight 2023-09-23T21:17:58.159 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:17:58.185 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:17:58.222 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:17:58.225 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:17:58.225 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:17:58.326 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:17:58.354 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:17:58.390 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:17:58.393 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:17:58.394 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:17:58.398 INFO:teuthology.orchestra.run.smithi026.stderr:reweighted osd.5 to 1 (10000) 2023-09-23T21:17:58.399 INFO:teuthology.orchestra.run.smithi026.stderr:nodeep-scrub is unset 2023-09-23T21:17:58.495 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:17:58.592 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:17:58.622 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:17:58 smithi070 ceph-mon[168282]: pgmap v1496: 129 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 123 active+clean; 2.9 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 1.13k op/s; 3182/144108 objects degraded (2.208%); 358/144108 objects misplaced (0.248%); 7.5 MiB/s, 120 objects/s recovering 2023-09-23T21:17:58.622 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:17:58 smithi070 ceph-mon[168282]: failed to encode map e593 with expected crc 2023-09-23T21:17:58.622 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:17:58 smithi070 ceph-mon[168282]: failed to encode map e593 with expected crc 2023-09-23T21:17:58.622 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:17:58 smithi070 ceph-mon[168282]: failed to encode map e593 with expected crc 2023-09-23T21:17:58.622 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:17:58 smithi070 ceph-mon[168282]: failed to encode map e593 with expected crc 2023-09-23T21:17:58.622 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:17:58 smithi070 ceph-mon[168282]: osdmap e593: 8 total, 8 up, 6 in 2023-09-23T21:17:58.622 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:17:58 smithi070 ceph-mon[168282]: failed to encode map e593 with expected crc 2023-09-23T21:17:58.622 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:17:58 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/1494435112' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:17:58.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:17:58 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:17:58.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:17:58 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/2410745806' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-09-23T21:17:58.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:17:58 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-09-23T21:17:58.757 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T21:17:58.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:17:58 smithi026 ceph-mon[179688]: pgmap v1496: 129 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 123 active+clean; 2.9 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 1.13k op/s; 3182/144108 objects degraded (2.208%); 358/144108 objects misplaced (0.248%); 7.5 MiB/s, 120 objects/s recovering 2023-09-23T21:17:58.883 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:17:58 smithi026 ceph-mon[179688]: failed to encode map e593 with expected crc 2023-09-23T21:17:58.883 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:17:58 smithi026 ceph-mon[179688]: failed to encode map e593 with expected crc 2023-09-23T21:17:58.883 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:17:58 smithi026 ceph-mon[179688]: failed to encode map e593 with expected crc 2023-09-23T21:17:58.883 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:17:58 smithi026 ceph-mon[179688]: failed to encode map e593 with expected crc 2023-09-23T21:17:58.883 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:17:58 smithi026 ceph-mon[179688]: osdmap e593: 8 total, 8 up, 6 in 2023-09-23T21:17:58.883 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:17:58 smithi026 ceph-mon[179688]: failed to encode map e593 with expected crc 2023-09-23T21:17:58.883 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:17:58 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/1494435112' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:17:58.883 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:17:58 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:17:58.883 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:17:58 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/2410745806' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-09-23T21:17:58.883 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:17:58 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-09-23T21:17:58.884 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:17:58 smithi026 ceph-mon[175151]: pgmap v1496: 129 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 123 active+clean; 2.9 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 1.13k op/s; 3182/144108 objects degraded (2.208%); 358/144108 objects misplaced (0.248%); 7.5 MiB/s, 120 objects/s recovering 2023-09-23T21:17:58.884 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:17:58 smithi026 ceph-mon[175151]: failed to encode map e593 with expected crc 2023-09-23T21:17:58.884 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:17:58 smithi026 ceph-mon[175151]: failed to encode map e593 with expected crc 2023-09-23T21:17:58.884 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:17:58 smithi026 ceph-mon[175151]: failed to encode map e593 with expected crc 2023-09-23T21:17:58.884 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:17:58 smithi026 ceph-mon[175151]: failed to encode map e593 with expected crc 2023-09-23T21:17:58.884 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:17:58 smithi026 ceph-mon[175151]: osdmap e593: 8 total, 8 up, 6 in 2023-09-23T21:17:58.884 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:17:58 smithi026 ceph-mon[175151]: failed to encode map e593 with expected crc 2023-09-23T21:17:58.885 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:17:58 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/1494435112' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:17:58.885 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:17:58 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:17:58.885 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:17:58 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/2410745806' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-09-23T21:17:58.886 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:17:58 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-09-23T21:17:59.024 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph osd reweight 7 1 2023-09-23T21:17:59.082 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T21:17:59.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:17:59 smithi070 ceph-mon[168282]: 11.5 scrub starts 2023-09-23T21:17:59.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:17:59 smithi070 ceph-mon[168282]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T21:17:59.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:17:59 smithi070 ceph-mon[168282]: failed to encode map e594 with expected crc 2023-09-23T21:17:59.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:17:59 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T21:17:59.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:17:59 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-09-23T21:17:59.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:17:59 smithi070 ceph-mon[168282]: osdmap e594: 8 total, 8 up, 6 in 2023-09-23T21:17:59.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:17:59 smithi070 ceph-mon[168282]: failed to encode map e594 with expected crc 2023-09-23T21:17:59.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:17:59 smithi070 ceph-mon[168282]: failed to encode map e594 with expected crc 2023-09-23T21:17:59.621 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:17:59 smithi070 ceph-mon[168282]: failed to encode map e594 with expected crc 2023-09-23T21:17:59.884 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:17:59 smithi026 ceph-mon[175151]: 11.5 scrub starts 2023-09-23T21:17:59.885 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:17:59 smithi026 ceph-mon[175151]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T21:17:59.885 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:17:59 smithi026 ceph-mon[175151]: failed to encode map e594 with expected crc 2023-09-23T21:17:59.885 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:17:59 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T21:17:59.885 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:17:59 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-09-23T21:17:59.885 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:17:59 smithi026 ceph-mon[175151]: osdmap e594: 8 total, 8 up, 6 in 2023-09-23T21:17:59.885 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:17:59 smithi026 ceph-mon[175151]: failed to encode map e594 with expected crc 2023-09-23T21:17:59.885 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:17:59 smithi026 ceph-mon[175151]: failed to encode map e594 with expected crc 2023-09-23T21:17:59.885 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:17:59 smithi026 ceph-mon[175151]: failed to encode map e594 with expected crc 2023-09-23T21:17:59.886 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 21:17:59 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[164029]: ::ffff:172.21.15.70 - - [23/Sep/2023:21:17:59] "GET /metrics HTTP/1.1" 200 36391 "" "Prometheus/2.43.0" 2023-09-23T21:17:59.886 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:17:59 smithi026 ceph-mon[179688]: 11.5 scrub starts 2023-09-23T21:17:59.886 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:17:59 smithi026 ceph-mon[179688]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T21:17:59.886 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:17:59 smithi026 ceph-mon[179688]: failed to encode map e594 with expected crc 2023-09-23T21:17:59.886 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:17:59 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T21:17:59.886 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:17:59 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-09-23T21:17:59.886 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:17:59 smithi026 ceph-mon[179688]: osdmap e594: 8 total, 8 up, 6 in 2023-09-23T21:17:59.886 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:17:59 smithi026 ceph-mon[179688]: failed to encode map e594 with expected crc 2023-09-23T21:17:59.886 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:17:59 smithi026 ceph-mon[179688]: failed to encode map e594 with expected crc 2023-09-23T21:17:59.886 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:17:59 smithi026 ceph-mon[179688]: failed to encode map e594 with expected crc 2023-09-23T21:18:00.077 INFO:teuthology.orchestra.run.smithi070.stdout:{ 2023-09-23T21:18:00.078 INFO:teuthology.orchestra.run.smithi070.stdout: "ops": [], 2023-09-23T21:18:00.078 INFO:teuthology.orchestra.run.smithi070.stdout: "num_ops": 0 2023-09-23T21:18:00.078 INFO:teuthology.orchestra.run.smithi070.stdout:} 2023-09-23T21:18:00.420 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T21:18:00.427 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:18:00.428 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:18:00.529 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:18:00.561 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_blocked_ops 2023-09-23T21:18:00.624 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:18:00.668 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:18:00 smithi026 ceph-mon[175151]: pgmap v1499: 129 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 123 active+clean; 2.9 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 859 op/s; 3182/142095 objects degraded (2.239%); 358/142095 objects misplaced (0.252%); 8.2 MiB/s, 131 objects/s recovering 2023-09-23T21:18:00.669 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:18:00 smithi026 ceph-mon[179688]: pgmap v1499: 129 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 123 active+clean; 2.9 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 859 op/s; 3182/142095 objects degraded (2.239%); 358/142095 objects misplaced (0.252%); 8.2 MiB/s, 131 objects/s recovering 2023-09-23T21:18:00.726 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:18:00.734 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:18:00.734 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:18:00.835 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:18:00.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:18:00 smithi070 ceph-mon[168282]: pgmap v1499: 129 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 123 active+clean; 2.9 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 859 op/s; 3182/142095 objects degraded (2.239%); 358/142095 objects misplaced (0.252%); 8.2 MiB/s, 131 objects/s recovering 2023-09-23T21:18:00.888 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:18:00.934 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:18:00.939 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:18:00.940 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:18:00.959 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:18:01.041 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:18:01.074 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:18:01.096 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:18:01.124 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:18:01.129 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:18:01.129 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:18:01.230 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:18:01.280 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T21:18:01.317 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:18:01.446 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:18:01.452 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:18:01.453 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:18:01.488 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:18:01 smithi026 ceph-mon[175151]: Health check update: Degraded data redundancy: 3182/142095 objects degraded (2.239%), 4 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T21:18:01.488 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:18:01 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:18:01.488 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:18:01 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:18:01.488 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:18:01 smithi026 ceph-mon[179688]: Health check update: Degraded data redundancy: 3182/142095 objects degraded (2.239%), 4 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T21:18:01.488 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:18:01 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:18:01.488 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:18:01 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:18:01.554 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:18:01.640 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:18:01.721 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:18:01 smithi070 ceph-mon[168282]: Health check update: Degraded data redundancy: 3182/142095 objects degraded (2.239%), 4 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T21:18:01.721 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:18:01 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:18:01.721 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:18:01 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:18:02.024 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T21:18:02.028 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:18:02.029 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:18:02.130 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:18:02.170 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:18:02.224 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:18:02.229 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:18:02.229 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:18:02.330 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:18:02.410 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:18:02.485 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:18:02.501 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:18:02.502 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:18:02.602 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:18:02.670 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T21:18:02.678 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:18:02 smithi026 ceph-mon[175151]: 11.15 scrub starts 2023-09-23T21:18:02.678 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:18:02 smithi026 ceph-mon[175151]: pgmap v1500: 129 pgs: 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 124 active+clean; 2.8 GiB data, 13 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 864 op/s; 3182/136767 objects degraded (2.327%); 6.7 MiB/s, 107 objects/s recovering 2023-09-23T21:18:02.678 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:18:02 smithi026 ceph-mon[179688]: 11.15 scrub starts 2023-09-23T21:18:02.678 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:18:02 smithi026 ceph-mon[179688]: pgmap v1500: 129 pgs: 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 124 active+clean; 2.8 GiB data, 13 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 864 op/s; 3182/136767 objects degraded (2.327%); 6.7 MiB/s, 107 objects/s recovering 2023-09-23T21:18:02.687 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:18:02.710 INFO:teuthology.orchestra.run.smithi070.stdout:{ 2023-09-23T21:18:02.711 INFO:teuthology.orchestra.run.smithi070.stdout: "ops": [], 2023-09-23T21:18:02.711 INFO:teuthology.orchestra.run.smithi070.stdout: "complaint_time": 30, 2023-09-23T21:18:02.711 INFO:teuthology.orchestra.run.smithi070.stdout: "num_blocked_ops": 0 2023-09-23T21:18:02.711 INFO:teuthology.orchestra.run.smithi070.stdout:} 2023-09-23T21:18:02.735 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:18:02 smithi070 ceph-mon[168282]: 11.15 scrub starts 2023-09-23T21:18:02.735 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:18:02 smithi070 ceph-mon[168282]: pgmap v1500: 129 pgs: 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 124 active+clean; 2.8 GiB data, 13 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 864 op/s; 3182/136767 objects degraded (2.327%); 6.7 MiB/s, 107 objects/s recovering 2023-09-23T21:18:02.776 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:18:02.783 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:18:02.783 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:18:02.884 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:18:03.011 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:18:03.200 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:18:03.213 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:18:03.214 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:18:03.314 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:18:03.358 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:18:03.568 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0: {} 2023-09-23T21:18:03.616 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1: {} 2023-09-23T21:18:03.648 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2: {} 2023-09-23T21:18:03.702 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3: {} 2023-09-23T21:18:03.738 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4: {} 2023-09-23T21:18:03.771 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T21:18:03.791 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6: {} 2023-09-23T21:18:03.795 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:18:03.813 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T21:18:03.815 INFO:teuthology.orchestra.run.smithi026.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T21:18:03.815 INFO:teuthology.orchestra.run.smithi026.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T21:18:03.815 INFO:teuthology.orchestra.run.smithi026.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T21:18:03.815 INFO:teuthology.orchestra.run.smithi026.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T21:18:03.815 INFO:teuthology.orchestra.run.smithi026.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T21:18:03.815 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T21:18:03.815 INFO:teuthology.orchestra.run.smithi026.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T21:18:03.815 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T21:18:03.851 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:18:03 smithi026 ceph-mon[179688]: 11.7 scrub starts 2023-09-23T21:18:03.852 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:18:03 smithi026 ceph-mon[175151]: 11.7 scrub starts 2023-09-23T21:18:03.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:18:03 smithi070 ceph-mon[168282]: 11.7 scrub starts 2023-09-23T21:18:04.480 INFO:teuthology.orchestra.run.smithi026.stderr:reweighted osd.7 to 1 (10000) 2023-09-23T21:18:04.713 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:18:04 smithi026 ceph-mon[179688]: pgmap v1501: 129 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 124 active+clean; 2.6 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 1.07k op/s; 2418/127995 objects degraded (1.889%); 252/127995 objects misplaced (0.197%); 6.8 MiB/s, 108 objects/s recovering 2023-09-23T21:18:04.713 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:18:04 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/1212457550' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:18:04.713 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:18:04 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/2280985090' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-09-23T21:18:04.713 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:18:04 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-09-23T21:18:04.717 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:18:04 smithi026 ceph-mon[175151]: pgmap v1501: 129 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 124 active+clean; 2.6 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 1.07k op/s; 2418/127995 objects degraded (1.889%); 252/127995 objects misplaced (0.197%); 6.8 MiB/s, 108 objects/s recovering 2023-09-23T21:18:04.717 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:18:04 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/1212457550' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:18:04.717 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:18:04 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/2280985090' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-09-23T21:18:04.717 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:18:04 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-09-23T21:18:04.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:18:04 smithi070 ceph-mon[168282]: pgmap v1501: 129 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 124 active+clean; 2.6 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 1.07k op/s; 2418/127995 objects degraded (1.889%); 252/127995 objects misplaced (0.197%); 6.8 MiB/s, 108 objects/s recovering 2023-09-23T21:18:04.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:18:04 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/1212457550' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:18:04.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:18:04 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/2280985090' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-09-23T21:18:04.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:18:04 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-09-23T21:18:05.118 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:18:05.121 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:18:05.121 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:18:05.193 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_historic_ops 2023-09-23T21:18:05.222 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:18:05.268 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:18:05.309 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:18:05.311 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:18:05.312 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:18:05.412 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:18:05.443 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:18:05.506 INFO:tasks.cephadm.ceph_manager.ceph:waiting for recovery to complete 2023-09-23T21:18:05.507 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T21:18:05.792 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:18:05 smithi026 ceph-mon[179688]: failed to encode map e595 with expected crc 2023-09-23T21:18:05.792 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:18:05 smithi026 ceph-mon[179688]: failed to encode map e595 with expected crc 2023-09-23T21:18:05.792 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:18:05 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-09-23T21:18:05.792 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:18:05 smithi026 ceph-mon[179688]: osdmap e595: 8 total, 8 up, 6 in 2023-09-23T21:18:05.792 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:18:05 smithi026 ceph-mon[179688]: 11.5 scrub starts 2023-09-23T21:18:05.793 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:18:05 smithi026 ceph-mon[175151]: failed to encode map e595 with expected crc 2023-09-23T21:18:05.793 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:18:05 smithi026 ceph-mon[175151]: failed to encode map e595 with expected crc 2023-09-23T21:18:05.793 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:18:05 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-09-23T21:18:05.794 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:18:05 smithi026 ceph-mon[175151]: osdmap e595: 8 total, 8 up, 6 in 2023-09-23T21:18:05.794 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:18:05 smithi026 ceph-mon[175151]: 11.5 scrub starts 2023-09-23T21:18:05.827 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:18:05.840 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:18:05.840 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:18:05.845 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T21:18:05.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:18:05 smithi070 ceph-mon[168282]: failed to encode map e595 with expected crc 2023-09-23T21:18:05.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:18:05 smithi070 ceph-mon[168282]: failed to encode map e595 with expected crc 2023-09-23T21:18:05.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:18:05 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-09-23T21:18:05.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:18:05 smithi070 ceph-mon[168282]: osdmap e595: 8 total, 8 up, 6 in 2023-09-23T21:18:05.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:18:05 smithi070 ceph-mon[168282]: 11.5 scrub starts 2023-09-23T21:18:05.941 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:18:06.009 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:18:06.354 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:18:06.363 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:18:06.363 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:18:06.464 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:18:06.496 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:18:06.539 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:18:06.634 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:18:06 smithi026 ceph-mon[175151]: pgmap v1503: 129 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 124 active+clean; 2.5 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 765 op/s; 1772/124179 objects degraded (1.427%); 839/124179 objects misplaced (0.676%); 7.4 MiB/s, 119 objects/s recovering 2023-09-23T21:18:06.634 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:18:06 smithi026 ceph-mon[175151]: failed to encode map e596 with expected crc 2023-09-23T21:18:06.635 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:18:06 smithi026 ceph-mon[175151]: failed to encode map e596 with expected crc 2023-09-23T21:18:06.635 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:18:06 smithi026 ceph-mon[175151]: osdmap e596: 8 total, 8 up, 6 in 2023-09-23T21:18:06.635 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:18:06 smithi026 ceph-mon[175151]: failed to encode map e596 with expected crc 2023-09-23T21:18:06.635 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:18:06 smithi026 ceph-mon[175151]: Health check update: Degraded data redundancy: 1772/124179 objects degraded (1.427%), 3 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-23T21:18:06.635 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:18:06 smithi026 ceph-mon[179688]: pgmap v1503: 129 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 124 active+clean; 2.5 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 765 op/s; 1772/124179 objects degraded (1.427%); 839/124179 objects misplaced (0.676%); 7.4 MiB/s, 119 objects/s recovering 2023-09-23T21:18:06.635 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:18:06 smithi026 ceph-mon[179688]: failed to encode map e596 with expected crc 2023-09-23T21:18:06.635 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:18:06 smithi026 ceph-mon[179688]: failed to encode map e596 with expected crc 2023-09-23T21:18:06.635 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:18:06 smithi026 ceph-mon[179688]: osdmap e596: 8 total, 8 up, 6 in 2023-09-23T21:18:06.635 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:18:06 smithi026 ceph-mon[179688]: failed to encode map e596 with expected crc 2023-09-23T21:18:06.635 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:18:06 smithi026 ceph-mon[179688]: Health check update: Degraded data redundancy: 1772/124179 objects degraded (1.427%), 3 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-23T21:18:06.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:18:06 smithi070 ceph-mon[168282]: pgmap v1503: 129 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 124 active+clean; 2.5 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 765 op/s; 1772/124179 objects degraded (1.427%); 839/124179 objects misplaced (0.676%); 7.4 MiB/s, 119 objects/s recovering 2023-09-23T21:18:06.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:18:06 smithi070 ceph-mon[168282]: failed to encode map e596 with expected crc 2023-09-23T21:18:06.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:18:06 smithi070 ceph-mon[168282]: failed to encode map e596 with expected crc 2023-09-23T21:18:06.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:18:06 smithi070 ceph-mon[168282]: osdmap e596: 8 total, 8 up, 6 in 2023-09-23T21:18:06.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:18:06 smithi070 ceph-mon[168282]: failed to encode map e596 with expected crc 2023-09-23T21:18:06.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:18:06 smithi070 ceph-mon[168282]: Health check update: Degraded data redundancy: 1772/124179 objects degraded (1.427%), 3 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-23T21:18:07.030 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:18:07.034 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:18:07.034 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:18:07.135 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:18:07.179 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:18:07.226 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:18:07.231 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:18:07.231 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:18:07.332 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:18:07.376 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:18:07.429 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:18:07.438 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:18:07.439 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:18:07.540 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:18:07.578 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:18:07.622 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:18:07 smithi070 ceph-mon[168282]: 11.17 deep-scrub starts 2023-09-23T21:18:07.622 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:18:07 smithi070 ceph-mon[168282]: 11.15 scrub starts 2023-09-23T21:18:07.751 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:18:07.758 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:18:07.759 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:18:07.850 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T21:18:07.860 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:18:07.881 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:18:07 smithi026 ceph-mon[179688]: 11.17 deep-scrub starts 2023-09-23T21:18:07.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:18:07 smithi026 ceph-mon[179688]: 11.15 scrub starts 2023-09-23T21:18:07.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:18:07 smithi026 ceph-mon[175151]: 11.17 deep-scrub starts 2023-09-23T21:18:07.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:18:07 smithi026 ceph-mon[175151]: 11.15 scrub starts 2023-09-23T21:18:07.913 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:18:08.018 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:18:08.023 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:18:08.024 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:18:08.124 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:18:08.174 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:18:08.271 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:18:08.286 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:18:08.287 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:18:08.388 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:18:08.407 INFO:teuthology.orchestra.run.smithi070.stdout:ERROR: (22) Invalid argument 2023-09-23T21:18:08.407 INFO:teuthology.orchestra.run.smithi070.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T21:18:08.438 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:18:08.496 INFO:teuthology.orchestra.run.smithi026.stderr:noscrub is set 2023-09-23T21:18:08.607 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:18:08 smithi070 ceph-mon[168282]: pgmap v1505: 129 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 124 active+clean; 2.4 GiB data, 12 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 1.07k op/s; 1772/116433 objects degraded (1.522%); 839/116433 objects misplaced (0.721%); 7.3 MiB/s, 116 objects/s recovering 2023-09-23T21:18:08.607 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:18:08 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/1129640535' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T21:18:08.794 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:18:08 smithi026 ceph-mon[175151]: pgmap v1505: 129 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 124 active+clean; 2.4 GiB data, 12 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 1.07k op/s; 1772/116433 objects degraded (1.522%); 839/116433 objects misplaced (0.721%); 7.3 MiB/s, 116 objects/s recovering 2023-09-23T21:18:08.794 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:18:08 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/1129640535' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T21:18:08.795 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:18:08 smithi026 ceph-mon[179688]: pgmap v1505: 129 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 124 active+clean; 2.4 GiB data, 12 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 1.07k op/s; 1772/116433 objects degraded (1.522%); 839/116433 objects misplaced (0.721%); 7.3 MiB/s, 116 objects/s recovering 2023-09-23T21:18:08.795 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:18:08 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/1129640535' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T21:18:08.871 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_ops_in_flight 2023-09-23T21:18:08.932 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:18:08.940 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:18:08.941 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:18:09.042 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:18:09.076 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:18:09.129 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:18:09.133 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:18:09.133 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:18:09.234 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:18:09.268 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:18:09.311 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:18:09.315 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:18:09.315 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:18:09.416 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:18:09.455 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:18:09.499 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:18:09.503 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:18:09.504 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:18:09.509 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:18:09 smithi026 ceph-mon[175151]: 11.7 scrub starts 2023-09-23T21:18:09.510 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:18:09 smithi026 ceph-mon[175151]: 11.5 scrub starts 2023-09-23T21:18:09.510 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:18:09 smithi026 ceph-mon[175151]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:18:09.510 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:18:09 smithi026 ceph-mon[175151]: failed to encode map e597 with expected crc 2023-09-23T21:18:09.510 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:18:09 smithi026 ceph-mon[175151]: failed to encode map e597 with expected crc 2023-09-23T21:18:09.510 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:18:09 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/1129640535' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T21:18:09.510 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:18:09 smithi026 ceph-mon[175151]: osdmap e597: 8 total, 8 up, 6 in 2023-09-23T21:18:09.510 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:18:09 smithi026 ceph-mon[175151]: failed to encode map e597 with expected crc 2023-09-23T21:18:09.510 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:18:09 smithi026 ceph-mon[175151]: failed to encode map e597 with expected crc 2023-09-23T21:18:09.511 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:18:09 smithi026 ceph-mon[179688]: 11.7 scrub starts 2023-09-23T21:18:09.511 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:18:09 smithi026 ceph-mon[179688]: 11.5 scrub starts 2023-09-23T21:18:09.511 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:18:09 smithi026 ceph-mon[179688]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:18:09.511 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:18:09 smithi026 ceph-mon[179688]: failed to encode map e597 with expected crc 2023-09-23T21:18:09.511 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:18:09 smithi026 ceph-mon[179688]: failed to encode map e597 with expected crc 2023-09-23T21:18:09.511 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:18:09 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/1129640535' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T21:18:09.511 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:18:09 smithi026 ceph-mon[179688]: osdmap e597: 8 total, 8 up, 6 in 2023-09-23T21:18:09.511 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:18:09 smithi026 ceph-mon[179688]: failed to encode map e597 with expected crc 2023-09-23T21:18:09.511 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:18:09 smithi026 ceph-mon[179688]: failed to encode map e597 with expected crc 2023-09-23T21:18:09.599 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:18:09.605 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:18:09.642 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:18:09.682 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:18:09.685 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:18:09.686 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:18:09.787 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:18:09.814 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:18:09 smithi070 ceph-mon[168282]: 11.7 scrub starts 2023-09-23T21:18:09.815 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:18:09 smithi070 ceph-mon[168282]: 11.5 scrub starts 2023-09-23T21:18:09.815 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:18:09 smithi070 ceph-mon[168282]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:18:09.815 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:18:09 smithi070 ceph-mon[168282]: failed to encode map e597 with expected crc 2023-09-23T21:18:09.815 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:18:09 smithi070 ceph-mon[168282]: failed to encode map e597 with expected crc 2023-09-23T21:18:09.815 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:18:09 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/1129640535' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T21:18:09.815 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:18:09 smithi070 ceph-mon[168282]: osdmap e597: 8 total, 8 up, 6 in 2023-09-23T21:18:09.815 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:18:09 smithi070 ceph-mon[168282]: failed to encode map e597 with expected crc 2023-09-23T21:18:09.816 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:18:09 smithi070 ceph-mon[168282]: failed to encode map e597 with expected crc 2023-09-23T21:18:09.818 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:18:09.853 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:18:09.857 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:18:09.857 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:18:09.864 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 21:18:09 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[164029]: ::ffff:172.21.15.70 - - [23/Sep/2023:21:18:09] "GET /metrics HTTP/1.1" 200 36383 "" "Prometheus/2.43.0" 2023-09-23T21:18:09.958 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:18:09.991 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:18:10.039 INFO:teuthology.orchestra.run.smithi026.stdout: 2023-09-23T21:18:10.039 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:18:10.044 INFO:teuthology.orchestra.run.smithi026.stderr:dumped all 2023-09-23T21:18:10.045 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:18:10.045 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:18:10.146 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:18:10.176 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:18:10.213 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:18:10.217 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:18:10.217 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:18:10.319 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:18:10.350 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:18:10.396 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:18:10.399 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:18:10.399 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:18:10.500 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:18:10.534 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:18:10.829 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:18:10 smithi026 ceph-mon[179688]: pgmap v1507: 129 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 125 active+clean; 2.3 GiB data, 12 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 748 op/s; 1772/114507 objects degraded (1.548%); 587/114507 objects misplaced (0.513%); 3.2 MiB/s, 51 objects/s recovering 2023-09-23T21:18:10.830 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:18:10 smithi026 ceph-mon[179688]: failed to encode map e598 with expected crc 2023-09-23T21:18:10.830 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:18:10 smithi026 ceph-mon[179688]: osdmap e598: 8 total, 8 up, 6 in 2023-09-23T21:18:10.830 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:18:10 smithi026 ceph-mon[179688]: failed to encode map e598 with expected crc 2023-09-23T21:18:10.830 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:18:10 smithi026 ceph-mon[179688]: failed to encode map e598 with expected crc 2023-09-23T21:18:10.830 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:18:10 smithi026 ceph-mon[179688]: failed to encode map e598 with expected crc 2023-09-23T21:18:10.830 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:18:10 smithi026 ceph-mon[175151]: pgmap v1507: 129 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 125 active+clean; 2.3 GiB data, 12 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 748 op/s; 1772/114507 objects degraded (1.548%); 587/114507 objects misplaced (0.513%); 3.2 MiB/s, 51 objects/s recovering 2023-09-23T21:18:10.830 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:18:10 smithi026 ceph-mon[175151]: failed to encode map e598 with expected crc 2023-09-23T21:18:10.830 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:18:10 smithi026 ceph-mon[175151]: osdmap e598: 8 total, 8 up, 6 in 2023-09-23T21:18:10.830 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:18:10 smithi026 ceph-mon[175151]: failed to encode map e598 with expected crc 2023-09-23T21:18:10.831 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:18:10 smithi026 ceph-mon[175151]: failed to encode map e598 with expected crc 2023-09-23T21:18:10.831 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:18:10 smithi026 ceph-mon[175151]: failed to encode map e598 with expected crc 2023-09-23T21:18:10.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:18:10 smithi070 ceph-mon[168282]: pgmap v1507: 129 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 125 active+clean; 2.3 GiB data, 12 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 748 op/s; 1772/114507 objects degraded (1.548%); 587/114507 objects misplaced (0.513%); 3.2 MiB/s, 51 objects/s recovering 2023-09-23T21:18:10.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:18:10 smithi070 ceph-mon[168282]: failed to encode map e598 with expected crc 2023-09-23T21:18:10.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:18:10 smithi070 ceph-mon[168282]: osdmap e598: 8 total, 8 up, 6 in 2023-09-23T21:18:10.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:18:10 smithi070 ceph-mon[168282]: failed to encode map e598 with expected crc 2023-09-23T21:18:10.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:18:10 smithi070 ceph-mon[168282]: failed to encode map e598 with expected crc 2023-09-23T21:18:10.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:18:10 smithi070 ceph-mon[168282]: failed to encode map e598 with expected crc 2023-09-23T21:18:11.020 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:18:11.031 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:18:11.031 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:18:11.132 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:18:11.185 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T21:18:11.221 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:18:11.525 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:18:11.533 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:18:11.534 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:18:11.542 INFO:teuthology.orchestra.run.smithi026.stdout:{"pg_ready":true,"pg_map":{"version":1508,"stamp":"2023-09-23T21:18:09.509215+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":2430080952,"num_objects":37096,"num_object_clones":0,"num_object_copies":111288,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1615,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":37096,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":158925,"num_write_kb":6269918,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":13642,"num_bytes_recovered":878791998,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":148872,"ondisk_log_size":148872,"up":387,"acting":384,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":412,"num_osds":6,"num_per_pool_osds":6,"num_per_pool_omap_osds":6,"kb":562470912,"kb_used":11996052,"kb_used_data":7840356,"kb_used_omap":125,"kb_used_meta":4155394,"kb_avail":550474860,"statfs":{"total":575970213888,"available":563686256640,"internally_reserved":0,"allocated":8028524544,"data_stored":8014971673,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":128662,"internal_metadata":4255123818},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":4,"apply_latency_ms":4,"commit_latency_ns":4000000,"apply_latency_ns":4000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-281608192,"num_objects":-4297,"num_object_clones":0,"num_object_copies":-12891,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":-157,"num_objects_misplaced":-839,"num_objects_unfound":0,"num_objects_dirty":-4297,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4297,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":996,"num_bytes_recovered":54984704,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"5.059257"},"pg_stats":[{"pgid":"11.12","version":"596'7306","reported_seq":7377,"reported_epoch":596,"state":"active+clean","last_fresh":"2023-09-23T21:18:06.635201+0000","last_change":"2023-09-23T21:16:32.944306+0000","last_active":"2023-09-23T21:18:06.635201+0000","last_peered":"2023-09-23T21:18:06.635201+0000","last_clean":"2023-09-23T21:18:06.635201+0000","last_became_active":"2023-09-23T21:16:32.940670+0000","last_became_peered":"2023-09-23T21:16:32.940670+0000","last_unstale":"2023-09-23T21:18:06.635201+0000","last_undegraded":"2023-09-23T21:18:06.635201+0000","last_fullsized":"2023-09-23T21:18:06.635201+0000","mapping_epoch":559,"log_start":"548'98","ondisk_log_start":"548'98","created":559,"last_epoch_clean":560,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_clean_scrub_stamp":"2023-09-23T21:15:24.860968+0000","objects_scrubbed":0,"log_size":7208,"log_dups_size":0,"ondisk_log_size":7208,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":80871424,"num_objects":1234,"num_object_clones":0,"num_object_copies":3702,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1234,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5022,"num_write_kb":200192,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.18","version":"79'161","reported_seq":620,"reported_epoch":595,"state":"active+clean","last_fresh":"2023-09-23T21:18:04.456599+0000","last_change":"2023-09-23T21:17:39.906535+0000","last_active":"2023-09-23T21:18:04.456599+0000","last_peered":"2023-09-23T21:18:04.456599+0000","last_clean":"2023-09-23T21:18:04.456599+0000","last_became_active":"2023-09-23T21:17:39.906154+0000","last_became_peered":"2023-09-23T21:17:39.906154+0000","last_unstale":"2023-09-23T21:18:04.456599+0000","last_undegraded":"2023-09-23T21:18:04.456599+0000","last_fullsized":"2023-09-23T21:18:04.456599+0000","mapping_epoch":585,"log_start":"0'0","ondisk_log_start":"0'0","created":479,"last_epoch_clean":586,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:15:39.474823+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:15:39.474823+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:48:39.784037+0000","scrub_duration":0.074664395999999994,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":525376,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1a","version":"68'1","reported_seq":186,"reported_epoch":595,"state":"active+clean","last_fresh":"2023-09-23T21:18:04.450782+0000","last_change":"2023-09-23T21:13:13.291403+0000","last_active":"2023-09-23T21:18:04.450782+0000","last_peered":"2023-09-23T21:18:04.450782+0000","last_clean":"2023-09-23T21:18:04.450782+0000","last_became_active":"2023-09-23T21:13:13.291123+0000","last_became_peered":"2023-09-23T21:13:13.291123+0000","last_unstale":"2023-09-23T21:18:04.450782+0000","last_undegraded":"2023-09-23T21:18:04.450782+0000","last_fullsized":"2023-09-23T21:18:04.450782+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":495,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:45:15.427430+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,0],"acting":[4,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":255,"reported_epoch":595,"state":"active+clean","last_fresh":"2023-09-23T21:18:04.457091+0000","last_change":"2023-09-23T21:12:10.767963+0000","last_active":"2023-09-23T21:18:04.457091+0000","last_peered":"2023-09-23T21:18:04.457091+0000","last_clean":"2023-09-23T21:18:04.457091+0000","last_became_active":"2023-09-23T21:12:10.767126+0000","last_became_peered":"2023-09-23T21:12:10.767126+0000","last_unstale":"2023-09-23T21:18:04.457091+0000","last_undegraded":"2023-09-23T21:18:04.457091+0000","last_fullsized":"2023-09-23T21:18:04.457091+0000","mapping_epoch":470,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:45:47.138277+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,5],"acting":[4,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.11","version":"597'7385","reported_seq":7455,"reported_epoch":597,"state":"active+clean","last_fresh":"2023-09-23T21:18:09.374826+0000","last_change":"2023-09-23T21:16:32.928552+0000","last_active":"2023-09-23T21:18:09.374826+0000","last_peered":"2023-09-23T21:18:09.374826+0000","last_clean":"2023-09-23T21:18:09.374826+0000","last_became_active":"2023-09-23T21:16:32.926639+0000","last_became_peered":"2023-09-23T21:16:32.926639+0000","last_unstale":"2023-09-23T21:18:09.374826+0000","last_undegraded":"2023-09-23T21:18:09.374826+0000","last_fullsized":"2023-09-23T21:18:09.374826+0000","mapping_epoch":559,"log_start":"550'3039","ondisk_log_start":"550'3039","created":559,"last_epoch_clean":560,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_clean_scrub_stamp":"2023-09-23T21:15:24.860968+0000","objects_scrubbed":0,"log_size":4346,"log_dups_size":0,"ondisk_log_size":4346,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":73924608,"num_objects":1128,"num_object_clones":0,"num_object_copies":3384,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1128,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5068,"num_write_kb":198272,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,2],"acting":[3,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1b","version":"79'161","reported_seq":783,"reported_epoch":595,"state":"active+clean","last_fresh":"2023-09-23T21:18:04.457124+0000","last_change":"2023-09-23T21:15:38.423372+0000","last_active":"2023-09-23T21:18:04.457124+0000","last_peered":"2023-09-23T21:18:04.457124+0000","last_clean":"2023-09-23T21:18:04.457124+0000","last_became_active":"2023-09-23T21:15:23.802580+0000","last_became_peered":"2023-09-23T21:15:23.802580+0000","last_unstale":"2023-09-23T21:18:04.457124+0000","last_undegraded":"2023-09-23T21:18:04.457124+0000","last_fullsized":"2023-09-23T21:18:04.457124+0000","mapping_epoch":542,"log_start":"0'0","ondisk_log_start":"0'0","created":479,"last_epoch_clean":543,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:15:38.423271+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:15:38.423271+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:04:46.575186+0000","scrub_duration":0.00258465,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":612,"reported_epoch":596,"state":"active+clean","last_fresh":"2023-09-23T21:18:05.472311+0000","last_change":"2023-09-23T21:16:42.448529+0000","last_active":"2023-09-23T21:18:05.472311+0000","last_peered":"2023-09-23T21:18:05.472311+0000","last_clean":"2023-09-23T21:18:05.472311+0000","last_became_active":"2023-09-23T21:16:42.447803+0000","last_became_peered":"2023-09-23T21:16:42.447803+0000","last_unstale":"2023-09-23T21:18:05.472311+0000","last_undegraded":"2023-09-23T21:18:05.472311+0000","last_fullsized":"2023-09-23T21:18:05.472311+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:41:33.121784+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":611,"reported_epoch":594,"state":"active+clean","last_fresh":"2023-09-23T21:17:58.397401+0000","last_change":"2023-09-23T21:16:42.441896+0000","last_active":"2023-09-23T21:17:58.397401+0000","last_peered":"2023-09-23T21:17:58.397401+0000","last_clean":"2023-09-23T21:17:58.397401+0000","last_became_active":"2023-09-23T21:16:42.441526+0000","last_became_peered":"2023-09-23T21:16:42.441526+0000","last_unstale":"2023-09-23T21:17:58.397401+0000","last_undegraded":"2023-09-23T21:17:58.397401+0000","last_fullsized":"2023-09-23T21:17:58.397401+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:27:36.326225+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.10","version":"597'7287","reported_seq":7379,"reported_epoch":597,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T21:18:09.261064+0000","last_change":"2023-09-23T21:17:02.209680+0000","last_active":"2023-09-23T21:18:09.261064+0000","last_peered":"2023-09-23T21:18:09.261064+0000","last_clean":"2023-09-23T21:16:02.267015+0000","last_became_active":"2023-09-23T21:16:43.501214+0000","last_became_peered":"2023-09-23T21:16:43.501214+0000","last_unstale":"2023-09-23T21:18:09.261064+0000","last_undegraded":"2023-09-23T21:17:02.208847+0000","last_fullsized":"2023-09-23T21:16:43.401617+0000","mapping_epoch":565,"log_start":"550'3023","ondisk_log_start":"550'3023","created":559,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_clean_scrub_stamp":"2023-09-23T21:15:24.860968+0000","objects_scrubbed":0,"log_size":4264,"log_dups_size":0,"ondisk_log_size":4264,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":73760768,"num_objects":1126,"num_object_clones":0,"num_object_copies":3378,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":812,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1126,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5030,"num_write_kb":196960,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":131072,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,0],"avail_no_missing":["3","0"],"object_location_counts":[{"shards":"0,2,3","objects":518},{"shards":"0,3","objects":314}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1a","version":"79'161","reported_seq":693,"reported_epoch":595,"state":"active+clean","last_fresh":"2023-09-23T21:18:04.450499+0000","last_change":"2023-09-23T21:16:45.419221+0000","last_active":"2023-09-23T21:18:04.450499+0000","last_peered":"2023-09-23T21:18:04.450499+0000","last_clean":"2023-09-23T21:18:04.450499+0000","last_became_active":"2023-09-23T21:16:45.418994+0000","last_became_peered":"2023-09-23T21:16:45.418994+0000","last_unstale":"2023-09-23T21:18:04.450499+0000","last_undegraded":"2023-09-23T21:18:04.450499+0000","last_fullsized":"2023-09-23T21:18:04.450499+0000","mapping_epoch":567,"log_start":"0'0","ondisk_log_start":"0'0","created":479,"last_epoch_clean":568,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:15:45.158107+0000","last_deep_scrub":"79'161","last_deep_scrub_stamp":"2023-09-23T21:05:32.630292+0000","last_clean_scrub_stamp":"2023-09-23T21:15:45.158107+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:08:08.342444+0000","scrub_duration":0.001999373,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,5],"acting":[4,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":105,"reported_epoch":594,"state":"active+clean","last_fresh":"2023-09-23T21:17:58.396265+0000","last_change":"2023-09-23T21:13:30.806084+0000","last_active":"2023-09-23T21:17:58.396265+0000","last_peered":"2023-09-23T21:17:58.396265+0000","last_clean":"2023-09-23T21:17:58.396265+0000","last_became_active":"2023-09-23T21:13:30.805743+0000","last_became_peered":"2023-09-23T21:13:30.805743+0000","last_unstale":"2023-09-23T21:17:58.396265+0000","last_undegraded":"2023-09-23T21:17:58.396265+0000","last_fullsized":"2023-09-23T21:17:58.396265+0000","mapping_epoch":501,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:48:10.849281+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":470,"reported_epoch":594,"state":"active+clean","last_fresh":"2023-09-23T21:17:58.396721+0000","last_change":"2023-09-23T21:13:44.410921+0000","last_active":"2023-09-23T21:17:58.396721+0000","last_peered":"2023-09-23T21:17:58.396721+0000","last_clean":"2023-09-23T21:17:58.396721+0000","last_became_active":"2023-09-23T21:13:44.410633+0000","last_became_peered":"2023-09-23T21:13:44.410633+0000","last_unstale":"2023-09-23T21:17:58.396721+0000","last_undegraded":"2023-09-23T21:17:58.396721+0000","last_fullsized":"2023-09-23T21:17:58.396721+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:57:10.116786+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.15","version":"79'161","reported_seq":544,"reported_epoch":595,"state":"active+clean","last_fresh":"2023-09-23T21:18:04.458604+0000","last_change":"2023-09-23T21:16:45.426006+0000","last_active":"2023-09-23T21:18:04.458604+0000","last_peered":"2023-09-23T21:18:04.458604+0000","last_clean":"2023-09-23T21:18:04.458604+0000","last_became_active":"2023-09-23T21:16:45.425654+0000","last_became_peered":"2023-09-23T21:16:45.425654+0000","last_unstale":"2023-09-23T21:18:04.458604+0000","last_undegraded":"2023-09-23T21:18:04.458604+0000","last_fullsized":"2023-09-23T21:18:04.458604+0000","mapping_epoch":567,"log_start":"0'0","ondisk_log_start":"0'0","created":479,"last_epoch_clean":568,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:15:36.742247+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:15:36.742247+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0081306369999999996,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,3],"acting":[7,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":571,"reported_epoch":594,"state":"active+clean","last_fresh":"2023-09-23T21:18:00.449403+0000","last_change":"2023-09-23T21:13:12.235728+0000","last_active":"2023-09-23T21:18:00.449403+0000","last_peered":"2023-09-23T21:18:00.449403+0000","last_clean":"2023-09-23T21:18:00.449403+0000","last_became_active":"2023-09-23T21:13:12.216549+0000","last_became_peered":"2023-09-23T21:13:12.216549+0000","last_unstale":"2023-09-23T21:18:00.449403+0000","last_undegraded":"2023-09-23T21:18:00.449403+0000","last_fullsized":"2023-09-23T21:18:00.449403+0000","mapping_epoch":493,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:39:33.324239+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":636,"reported_epoch":596,"state":"active+clean","last_fresh":"2023-09-23T21:18:05.471195+0000","last_change":"2023-09-23T21:12:10.790349+0000","last_active":"2023-09-23T21:18:05.471195+0000","last_peered":"2023-09-23T21:18:05.471195+0000","last_clean":"2023-09-23T21:18:05.471195+0000","last_became_active":"2023-09-23T21:12:10.789977+0000","last_became_peered":"2023-09-23T21:12:10.789977+0000","last_unstale":"2023-09-23T21:18:05.471195+0000","last_undegraded":"2023-09-23T21:18:05.471195+0000","last_fullsized":"2023-09-23T21:18:05.471195+0000","mapping_epoch":470,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:49:40.191147+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,5],"acting":[3,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"79'161","reported_seq":992,"reported_epoch":595,"state":"active+clean","last_fresh":"2023-09-23T21:18:04.463776+0000","last_change":"2023-09-23T21:16:45.424773+0000","last_active":"2023-09-23T21:18:04.463776+0000","last_peered":"2023-09-23T21:18:04.463776+0000","last_clean":"2023-09-23T21:18:04.463776+0000","last_became_active":"2023-09-23T21:16:45.424458+0000","last_became_peered":"2023-09-23T21:16:45.424458+0000","last_unstale":"2023-09-23T21:18:04.463776+0000","last_undegraded":"2023-09-23T21:18:04.463776+0000","last_fullsized":"2023-09-23T21:18:04.463776+0000","mapping_epoch":567,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":568,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:25.096401+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:25.096401+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:40:17.871561+0000","scrub_duration":0.002859393,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":340,"reported_epoch":595,"state":"active+clean","last_fresh":"2023-09-23T21:18:04.455898+0000","last_change":"2023-09-23T21:16:42.427856+0000","last_active":"2023-09-23T21:18:04.455898+0000","last_peered":"2023-09-23T21:18:04.455898+0000","last_clean":"2023-09-23T21:18:04.455898+0000","last_became_active":"2023-09-23T21:16:42.427529+0000","last_became_peered":"2023-09-23T21:16:42.427529+0000","last_unstale":"2023-09-23T21:18:04.455898+0000","last_undegraded":"2023-09-23T21:18:04.455898+0000","last_fullsized":"2023-09-23T21:18:04.455898+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:24:03.818632+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":591,"reported_epoch":596,"state":"active+clean","last_fresh":"2023-09-23T21:18:05.470262+0000","last_change":"2023-09-23T21:16:42.448719+0000","last_active":"2023-09-23T21:18:05.470262+0000","last_peered":"2023-09-23T21:18:05.470262+0000","last_clean":"2023-09-23T21:18:05.470262+0000","last_became_active":"2023-09-23T21:16:42.448219+0000","last_became_peered":"2023-09-23T21:16:42.448219+0000","last_unstale":"2023-09-23T21:18:05.470262+0000","last_undegraded":"2023-09-23T21:18:05.470262+0000","last_fullsized":"2023-09-23T21:18:05.470262+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:02:28.864357+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.17","version":"79'161","reported_seq":742,"reported_epoch":594,"state":"active+clean","last_fresh":"2023-09-23T21:17:58.396478+0000","last_change":"2023-09-23T21:16:45.413663+0000","last_active":"2023-09-23T21:17:58.396478+0000","last_peered":"2023-09-23T21:17:58.396478+0000","last_clean":"2023-09-23T21:17:58.396478+0000","last_became_active":"2023-09-23T21:16:45.413311+0000","last_became_peered":"2023-09-23T21:16:45.413311+0000","last_unstale":"2023-09-23T21:17:58.396478+0000","last_undegraded":"2023-09-23T21:17:58.396478+0000","last_fullsized":"2023-09-23T21:17:58.396478+0000","mapping_epoch":567,"log_start":"0'0","ondisk_log_start":"0'0","created":479,"last_epoch_clean":568,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:15:50.325741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:15:50.325741+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:07:43.285354+0000","scrub_duration":0.0040500149999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":512,"reported_epoch":596,"state":"active+clean","last_fresh":"2023-09-23T21:18:05.471995+0000","last_change":"2023-09-23T21:17:02.182606+0000","last_active":"2023-09-23T21:18:05.471995+0000","last_peered":"2023-09-23T21:18:05.471995+0000","last_clean":"2023-09-23T21:18:05.471995+0000","last_became_active":"2023-09-23T21:16:42.446517+0000","last_became_peered":"2023-09-23T21:16:42.446517+0000","last_unstale":"2023-09-23T21:18:05.471995+0000","last_undegraded":"2023-09-23T21:18:05.471995+0000","last_fullsized":"2023-09-23T21:18:05.471995+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:46:08.337878+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":540,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,4],"acting":[3,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":284,"reported_epoch":595,"state":"active+clean","last_fresh":"2023-09-23T21:18:04.452094+0000","last_change":"2023-09-23T21:13:30.821489+0000","last_active":"2023-09-23T21:18:04.452094+0000","last_peered":"2023-09-23T21:18:04.452094+0000","last_clean":"2023-09-23T21:18:04.452094+0000","last_became_active":"2023-09-23T21:13:30.821125+0000","last_became_peered":"2023-09-23T21:13:30.821125+0000","last_unstale":"2023-09-23T21:18:04.452094+0000","last_undegraded":"2023-09-23T21:18:04.452094+0000","last_fullsized":"2023-09-23T21:18:04.452094+0000","mapping_epoch":501,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:47:18.553417+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,5],"acting":[2,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.16","version":"79'161","reported_seq":526,"reported_epoch":594,"state":"active+clean","last_fresh":"2023-09-23T21:17:58.397328+0000","last_change":"2023-09-23T21:15:46.326726+0000","last_active":"2023-09-23T21:17:58.397328+0000","last_peered":"2023-09-23T21:17:58.397328+0000","last_clean":"2023-09-23T21:17:58.397328+0000","last_became_active":"2023-09-23T21:13:33.770407+0000","last_became_peered":"2023-09-23T21:13:33.770407+0000","last_unstale":"2023-09-23T21:17:58.397328+0000","last_undegraded":"2023-09-23T21:17:58.397328+0000","last_fullsized":"2023-09-23T21:17:58.397328+0000","mapping_epoch":504,"log_start":"0'0","ondisk_log_start":"0'0","created":479,"last_epoch_clean":505,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:15:46.326589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:15:46.326589+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:50:23.810054+0000","scrub_duration":0.0062069059999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.14","version":"79'22","reported_seq":199,"reported_epoch":595,"state":"active+clean","last_fresh":"2023-09-23T21:18:04.450894+0000","last_change":"2023-09-23T21:17:02.198735+0000","last_active":"2023-09-23T21:18:04.450894+0000","last_peered":"2023-09-23T21:18:04.450894+0000","last_clean":"2023-09-23T21:18:04.450894+0000","last_became_active":"2023-09-23T21:16:42.492861+0000","last_became_peered":"2023-09-23T21:16:42.492861+0000","last_unstale":"2023-09-23T21:18:04.450894+0000","last_undegraded":"2023-09-23T21:18:04.450894+0000","last_fullsized":"2023-09-23T21:18:04.450894+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:00:09.949965+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":14202,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,2],"acting":[4,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":524,"reported_epoch":595,"state":"active+clean","last_fresh":"2023-09-23T21:18:04.451009+0000","last_change":"2023-09-23T21:12:10.839271+0000","last_active":"2023-09-23T21:18:04.451009+0000","last_peered":"2023-09-23T21:18:04.451009+0000","last_clean":"2023-09-23T21:18:04.451009+0000","last_became_active":"2023-09-23T21:12:10.839042+0000","last_became_peered":"2023-09-23T21:12:10.839042+0000","last_unstale":"2023-09-23T21:18:04.451009+0000","last_undegraded":"2023-09-23T21:18:04.451009+0000","last_fullsized":"2023-09-23T21:18:04.451009+0000","mapping_epoch":470,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:10:26.414737+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,2],"acting":[4,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.11","version":"79'161","reported_seq":531,"reported_epoch":594,"state":"active+clean","last_fresh":"2023-09-23T21:17:58.396264+0000","last_change":"2023-09-23T21:15:47.300109+0000","last_active":"2023-09-23T21:17:58.396264+0000","last_peered":"2023-09-23T21:17:58.396264+0000","last_clean":"2023-09-23T21:17:58.396264+0000","last_became_active":"2023-09-23T21:13:33.767067+0000","last_became_peered":"2023-09-23T21:13:33.767067+0000","last_unstale":"2023-09-23T21:17:58.396264+0000","last_undegraded":"2023-09-23T21:17:58.396264+0000","last_fullsized":"2023-09-23T21:17:58.396264+0000","mapping_epoch":504,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":505,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:15:47.299947+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:15:47.299947+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:04:59.029359+0000","scrub_duration":0.0061353110000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":47,"reported_epoch":595,"state":"active+clean","last_fresh":"2023-09-23T21:18:04.457861+0000","last_change":"2023-09-23T21:16:42.531060+0000","last_active":"2023-09-23T21:18:04.457861+0000","last_peered":"2023-09-23T21:18:04.457861+0000","last_clean":"2023-09-23T21:18:04.457861+0000","last_became_active":"2023-09-23T21:16:42.449198+0000","last_became_peered":"2023-09-23T21:16:42.449198+0000","last_unstale":"2023-09-23T21:18:04.457861+0000","last_undegraded":"2023-09-23T21:18:04.457861+0000","last_fullsized":"2023-09-23T21:18:04.457861+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:30:23.428886+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":156,"reported_epoch":595,"state":"active+clean","last_fresh":"2023-09-23T21:18:04.463455+0000","last_change":"2023-09-23T21:16:42.491023+0000","last_active":"2023-09-23T21:18:04.463455+0000","last_peered":"2023-09-23T21:18:04.463455+0000","last_clean":"2023-09-23T21:18:04.463455+0000","last_became_active":"2023-09-23T21:16:42.489896+0000","last_became_peered":"2023-09-23T21:16:42.489896+0000","last_unstale":"2023-09-23T21:18:04.463455+0000","last_undegraded":"2023-09-23T21:18:04.463455+0000","last_fullsized":"2023-09-23T21:18:04.463455+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:58:54.036098+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,7],"acting":[2,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.10","version":"79'161","reported_seq":770,"reported_epoch":595,"state":"active+clean","last_fresh":"2023-09-23T21:18:04.450449+0000","last_change":"2023-09-23T21:16:45.416227+0000","last_active":"2023-09-23T21:18:04.450449+0000","last_peered":"2023-09-23T21:18:04.450449+0000","last_clean":"2023-09-23T21:18:04.450449+0000","last_became_active":"2023-09-23T21:16:45.415841+0000","last_became_peered":"2023-09-23T21:16:45.415841+0000","last_unstale":"2023-09-23T21:18:04.450449+0000","last_undegraded":"2023-09-23T21:18:04.450449+0000","last_fullsized":"2023-09-23T21:18:04.450449+0000","mapping_epoch":567,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":568,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:06:50.336992+0000","last_deep_scrub":"79'161","last_deep_scrub_stamp":"2023-09-23T21:06:50.336992+0000","last_clean_scrub_stamp":"2023-09-23T21:06:50.336992+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:37:30.094658+0000","scrub_duration":0.0026150930000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,7],"acting":[4,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":555,"reported_epoch":594,"state":"active+clean","last_fresh":"2023-09-23T21:17:58.397544+0000","last_change":"2023-09-23T21:16:42.446324+0000","last_active":"2023-09-23T21:17:58.397544+0000","last_peered":"2023-09-23T21:17:58.397544+0000","last_clean":"2023-09-23T21:17:58.397544+0000","last_became_active":"2023-09-23T21:16:42.445997+0000","last_became_peered":"2023-09-23T21:16:42.445997+0000","last_unstale":"2023-09-23T21:17:58.397544+0000","last_undegraded":"2023-09-23T21:17:58.397544+0000","last_fullsized":"2023-09-23T21:17:58.397544+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:46:35.028292+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":368,"reported_epoch":596,"state":"active+clean","last_fresh":"2023-09-23T21:18:05.471850+0000","last_change":"2023-09-23T21:16:42.473872+0000","last_active":"2023-09-23T21:18:05.471850+0000","last_peered":"2023-09-23T21:18:05.471850+0000","last_clean":"2023-09-23T21:18:05.471850+0000","last_became_active":"2023-09-23T21:16:42.473595+0000","last_became_peered":"2023-09-23T21:16:42.473595+0000","last_unstale":"2023-09-23T21:18:05.471850+0000","last_undegraded":"2023-09-23T21:18:05.471850+0000","last_fullsized":"2023-09-23T21:18:05.471850+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:15:19.186428+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.19","version":"597'7297","reported_seq":9069,"reported_epoch":597,"state":"active+recovery_wait","last_fresh":"2023-09-23T21:18:09.369098+0000","last_change":"2023-09-23T21:17:46.325067+0000","last_active":"2023-09-23T21:18:09.369098+0000","last_peered":"2023-09-23T21:18:09.369098+0000","last_clean":"2023-09-23T21:16:02.272109+0000","last_became_active":"2023-09-23T21:17:46.300166+0000","last_became_peered":"2023-09-23T21:17:46.300166+0000","last_unstale":"2023-09-23T21:18:09.369098+0000","last_undegraded":"2023-09-23T21:18:09.369098+0000","last_fullsized":"2023-09-23T21:18:09.369098+0000","mapping_epoch":588,"log_start":"550'2986","ondisk_log_start":"550'2986","created":559,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_clean_scrub_stamp":"2023-09-23T21:15:24.860968+0000","objects_scrubbed":0,"log_size":4311,"log_dups_size":0,"ondisk_log_size":4311,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":69959680,"num_objects":1068,"num_object_clones":0,"num_object_copies":3204,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1068,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5014,"num_write_kb":194592,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":838,"num_bytes_recovered":54820864,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,7],"acting":[3,0,7],"avail_no_missing":["3","0","7"],"object_location_counts":[{"shards":"0,3,7","objects":1068}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.13","version":"79'161","reported_seq":675,"reported_epoch":596,"state":"active+clean","last_fresh":"2023-09-23T21:18:05.470355+0000","last_change":"2023-09-23T21:13:14.933096+0000","last_active":"2023-09-23T21:18:05.470355+0000","last_peered":"2023-09-23T21:18:05.470355+0000","last_clean":"2023-09-23T21:18:05.470355+0000","last_became_active":"2023-09-23T21:13:14.932703+0000","last_became_peered":"2023-09-23T21:13:14.932703+0000","last_unstale":"2023-09-23T21:18:05.470355+0000","last_undegraded":"2023-09-23T21:18:05.470355+0000","last_fullsized":"2023-09-23T21:18:05.470355+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:36.420362+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:36.420362+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:29:42.163911+0000","scrub_duration":0.028838444000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":607,"reported_epoch":595,"state":"active+clean","last_fresh":"2023-09-23T21:18:04.450583+0000","last_change":"2023-09-23T21:16:42.420467+0000","last_active":"2023-09-23T21:18:04.450583+0000","last_peered":"2023-09-23T21:18:04.450583+0000","last_clean":"2023-09-23T21:18:04.450583+0000","last_became_active":"2023-09-23T21:16:42.420149+0000","last_became_peered":"2023-09-23T21:16:42.420149+0000","last_unstale":"2023-09-23T21:18:04.450583+0000","last_undegraded":"2023-09-23T21:18:04.450583+0000","last_fullsized":"2023-09-23T21:18:04.450583+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:52:07.355000+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":123,"reported_epoch":594,"state":"active+clean","last_fresh":"2023-09-23T21:17:58.397119+0000","last_change":"2023-09-23T21:16:42.427663+0000","last_active":"2023-09-23T21:17:58.397119+0000","last_peered":"2023-09-23T21:17:58.397119+0000","last_clean":"2023-09-23T21:17:58.397119+0000","last_became_active":"2023-09-23T21:16:42.426961+0000","last_became_peered":"2023-09-23T21:16:42.426961+0000","last_unstale":"2023-09-23T21:17:58.397119+0000","last_undegraded":"2023-09-23T21:17:58.397119+0000","last_fullsized":"2023-09-23T21:17:58.397119+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:44:27.422007+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.18","version":"595'6993","reported_seq":7063,"reported_epoch":595,"state":"active+clean","last_fresh":"2023-09-23T21:18:04.837628+0000","last_change":"2023-09-23T21:16:32.928345+0000","last_active":"2023-09-23T21:18:04.837628+0000","last_peered":"2023-09-23T21:18:04.837628+0000","last_clean":"2023-09-23T21:18:04.837628+0000","last_became_active":"2023-09-23T21:16:32.926828+0000","last_became_peered":"2023-09-23T21:16:32.926828+0000","last_unstale":"2023-09-23T21:18:04.837628+0000","last_undegraded":"2023-09-23T21:18:04.837628+0000","last_fullsized":"2023-09-23T21:18:04.837628+0000","mapping_epoch":559,"log_start":"549'1603","ondisk_log_start":"549'1603","created":559,"last_epoch_clean":560,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_clean_scrub_stamp":"2023-09-23T21:15:24.860968+0000","objects_scrubbed":0,"log_size":5390,"log_dups_size":0,"ondisk_log_size":5390,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":78675968,"num_objects":1201,"num_object_clones":0,"num_object_copies":3603,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1201,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4769,"num_write_kb":191008,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,5],"acting":[0,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.12","version":"79'161","reported_seq":869,"reported_epoch":595,"state":"active+clean","last_fresh":"2023-09-23T21:18:04.456897+0000","last_change":"2023-09-23T21:16:45.412440+0000","last_active":"2023-09-23T21:18:04.456897+0000","last_peered":"2023-09-23T21:18:04.456897+0000","last_clean":"2023-09-23T21:18:04.456897+0000","last_became_active":"2023-09-23T21:16:45.412029+0000","last_became_peered":"2023-09-23T21:16:45.412029+0000","last_unstale":"2023-09-23T21:18:04.456897+0000","last_undegraded":"2023-09-23T21:18:04.456897+0000","last_fullsized":"2023-09-23T21:18:04.456897+0000","mapping_epoch":567,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":568,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:05:56.785173+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:05:56.785173+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:39:43.432998+0000","scrub_duration":0.0042114370000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,0],"acting":[5,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":115,"reported_epoch":594,"state":"active+clean","last_fresh":"2023-09-23T21:17:58.396364+0000","last_change":"2023-09-23T21:13:38.048791+0000","last_active":"2023-09-23T21:17:58.396364+0000","last_peered":"2023-09-23T21:17:58.396364+0000","last_clean":"2023-09-23T21:17:58.396364+0000","last_became_active":"2023-09-23T21:13:30.801386+0000","last_became_peered":"2023-09-23T21:13:30.801386+0000","last_unstale":"2023-09-23T21:17:58.396364+0000","last_undegraded":"2023-09-23T21:17:58.396364+0000","last_fullsized":"2023-09-23T21:17:58.396364+0000","mapping_epoch":501,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:52:49.170414+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":453,"reported_epoch":596,"state":"active+clean","last_fresh":"2023-09-23T21:18:05.471124+0000","last_change":"2023-09-23T21:16:42.518512+0000","last_active":"2023-09-23T21:18:05.471124+0000","last_peered":"2023-09-23T21:18:05.471124+0000","last_clean":"2023-09-23T21:18:05.471124+0000","last_became_active":"2023-09-23T21:16:42.517464+0000","last_became_peered":"2023-09-23T21:16:42.517464+0000","last_unstale":"2023-09-23T21:18:05.471124+0000","last_undegraded":"2023-09-23T21:18:05.471124+0000","last_fullsized":"2023-09-23T21:18:05.471124+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:47:52.391167+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.7","version":"596'7310","reported_seq":7390,"reported_epoch":596,"state":"active+clean","last_fresh":"2023-09-23T21:18:07.046831+0000","last_change":"2023-09-23T21:18:03.311929+0000","last_active":"2023-09-23T21:18:07.046831+0000","last_peered":"2023-09-23T21:18:07.046831+0000","last_clean":"2023-09-23T21:18:07.046831+0000","last_became_active":"2023-09-23T21:16:32.864890+0000","last_became_peered":"2023-09-23T21:16:32.864890+0000","last_unstale":"2023-09-23T21:18:07.046831+0000","last_undegraded":"2023-09-23T21:18:07.046831+0000","last_fullsized":"2023-09-23T21:18:07.046831+0000","mapping_epoch":544,"log_start":"548'360","ondisk_log_start":"548'360","created":544,"last_epoch_clean":560,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_clean_scrub_stamp":"2023-09-23T21:15:24.860968+0000","objects_scrubbed":0,"log_size":6950,"log_dups_size":0,"ondisk_log_size":6950,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":72974336,"num_objects":1113,"num_object_clones":0,"num_object_copies":3339,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1113,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5005,"num_write_kb":195808,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.d","version":"79'161","reported_seq":621,"reported_epoch":595,"state":"active+clean","last_fresh":"2023-09-23T21:18:04.458663+0000","last_change":"2023-09-23T21:16:45.416685+0000","last_active":"2023-09-23T21:18:04.458663+0000","last_peered":"2023-09-23T21:18:04.458663+0000","last_clean":"2023-09-23T21:18:04.458663+0000","last_became_active":"2023-09-23T21:16:45.416373+0000","last_became_peered":"2023-09-23T21:16:45.416373+0000","last_unstale":"2023-09-23T21:18:04.458663+0000","last_undegraded":"2023-09-23T21:18:04.458663+0000","last_fullsized":"2023-09-23T21:18:04.458663+0000","mapping_epoch":567,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":568,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:15:32.726690+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:15:32.726690+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0024987500000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":45,"reported_epoch":595,"state":"active+clean","last_fresh":"2023-09-23T21:18:04.459492+0000","last_change":"2023-09-23T21:17:02.190320+0000","last_active":"2023-09-23T21:18:04.459492+0000","last_peered":"2023-09-23T21:18:04.459492+0000","last_clean":"2023-09-23T21:18:04.459492+0000","last_became_active":"2023-09-23T21:16:42.451380+0000","last_became_peered":"2023-09-23T21:16:42.451380+0000","last_unstale":"2023-09-23T21:18:04.459492+0000","last_undegraded":"2023-09-23T21:18:04.459492+0000","last_fullsized":"2023-09-23T21:18:04.459492+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:48:58.425821+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":132,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":323,"reported_epoch":595,"state":"active+clean","last_fresh":"2023-09-23T21:18:04.456927+0000","last_change":"2023-09-23T21:16:42.428491+0000","last_active":"2023-09-23T21:18:04.456927+0000","last_peered":"2023-09-23T21:18:04.456927+0000","last_clean":"2023-09-23T21:18:04.456927+0000","last_became_active":"2023-09-23T21:16:42.428166+0000","last_became_peered":"2023-09-23T21:16:42.428166+0000","last_unstale":"2023-09-23T21:18:04.456927+0000","last_undegraded":"2023-09-23T21:18:04.456927+0000","last_fullsized":"2023-09-23T21:18:04.456927+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:23:01.430714+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,4],"acting":[5,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.6","version":"595'7056","reported_seq":7129,"reported_epoch":595,"state":"active+clean","last_fresh":"2023-09-23T21:18:04.951231+0000","last_change":"2023-09-23T21:16:33.949061+0000","last_active":"2023-09-23T21:18:04.951231+0000","last_peered":"2023-09-23T21:18:04.951231+0000","last_clean":"2023-09-23T21:18:04.951231+0000","last_became_active":"2023-09-23T21:16:32.896510+0000","last_became_peered":"2023-09-23T21:16:32.896510+0000","last_unstale":"2023-09-23T21:18:04.951231+0000","last_undegraded":"2023-09-23T21:18:04.951231+0000","last_fullsized":"2023-09-23T21:18:04.951231+0000","mapping_epoch":544,"log_start":"549'1825","ondisk_log_start":"549'1825","created":544,"last_epoch_clean":561,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_clean_scrub_stamp":"2023-09-23T21:15:24.860968+0000","objects_scrubbed":0,"log_size":5231,"log_dups_size":0,"ondisk_log_size":5231,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":80379904,"num_objects":1226,"num_object_clones":0,"num_object_copies":3678,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1226,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4822,"num_write_kb":193568,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.c","version":"79'161","reported_seq":979,"reported_epoch":596,"state":"active+clean","last_fresh":"2023-09-23T21:18:05.470393+0000","last_change":"2023-09-23T21:15:31.992756+0000","last_active":"2023-09-23T21:18:05.470393+0000","last_peered":"2023-09-23T21:18:05.470393+0000","last_clean":"2023-09-23T21:18:05.470393+0000","last_became_active":"2023-09-23T21:13:13.401526+0000","last_became_peered":"2023-09-23T21:13:13.401526+0000","last_unstale":"2023-09-23T21:18:05.470393+0000","last_undegraded":"2023-09-23T21:18:05.470393+0000","last_fullsized":"2023-09-23T21:18:05.470393+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":532,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:15:31.992598+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:15:31.992598+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:41:14.215543+0000","scrub_duration":0.002960678,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":45,"reported_epoch":595,"state":"active+clean","last_fresh":"2023-09-23T21:18:04.458974+0000","last_change":"2023-09-23T21:17:02.215996+0000","last_active":"2023-09-23T21:18:04.458974+0000","last_peered":"2023-09-23T21:18:04.458974+0000","last_clean":"2023-09-23T21:18:04.458974+0000","last_became_active":"2023-09-23T21:16:42.451205+0000","last_became_peered":"2023-09-23T21:16:42.451205+0000","last_unstale":"2023-09-23T21:18:04.458974+0000","last_undegraded":"2023-09-23T21:18:04.458974+0000","last_fullsized":"2023-09-23T21:18:04.458974+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:51:46.876281+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":558,"reported_epoch":595,"state":"active+clean","last_fresh":"2023-09-23T21:18:04.450960+0000","last_change":"2023-09-23T21:13:15.152792+0000","last_active":"2023-09-23T21:18:04.450960+0000","last_peered":"2023-09-23T21:18:04.450960+0000","last_clean":"2023-09-23T21:18:04.450960+0000","last_became_active":"2023-09-23T21:13:14.515084+0000","last_became_peered":"2023-09-23T21:13:14.515084+0000","last_unstale":"2023-09-23T21:18:04.450960+0000","last_undegraded":"2023-09-23T21:18:04.450960+0000","last_fullsized":"2023-09-23T21:18:04.450960+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:08:16.702072+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.5","version":"596'7280","reported_seq":12482,"reported_epoch":596,"state":"active+clean","last_fresh":"2023-09-23T21:18:07.355747+0000","last_change":"2023-09-23T21:18:04.534733+0000","last_active":"2023-09-23T21:18:07.355747+0000","last_peered":"2023-09-23T21:18:07.355747+0000","last_clean":"2023-09-23T21:18:07.355747+0000","last_became_active":"2023-09-23T21:17:24.389052+0000","last_became_peered":"2023-09-23T21:17:24.389052+0000","last_unstale":"2023-09-23T21:18:07.355747+0000","last_undegraded":"2023-09-23T21:18:07.355747+0000","last_fullsized":"2023-09-23T21:18:07.355747+0000","mapping_epoch":578,"log_start":"548'300","ondisk_log_start":"548'300","created":544,"last_epoch_clean":579,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_clean_scrub_stamp":"2023-09-23T21:15:24.860968+0000","objects_scrubbed":0,"log_size":6980,"log_dups_size":0,"ondisk_log_size":6980,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":79986688,"num_objects":1220,"num_object_clones":0,"num_object_copies":3660,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1220,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4984,"num_write_kb":198560,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1682,"num_bytes_recovered":110231552,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.f","version":"79'161","reported_seq":883,"reported_epoch":595,"state":"active+clean","last_fresh":"2023-09-23T21:18:04.455362+0000","last_change":"2023-09-23T21:11:30.750338+0000","last_active":"2023-09-23T21:18:04.455362+0000","last_peered":"2023-09-23T21:18:04.455362+0000","last_clean":"2023-09-23T21:18:04.455362+0000","last_became_active":"2023-09-23T21:11:30.749953+0000","last_became_peered":"2023-09-23T21:11:30.749953+0000","last_unstale":"2023-09-23T21:18:04.455362+0000","last_undegraded":"2023-09-23T21:18:04.455362+0000","last_fullsized":"2023-09-23T21:18:04.455362+0000","mapping_epoch":455,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":456,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:27.880776+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:27.880776+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:07:10.405723+0000","scrub_duration":0.0014004899999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,3],"acting":[5,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":347,"reported_epoch":595,"state":"active+clean","last_fresh":"2023-09-23T21:18:04.456326+0000","last_change":"2023-09-23T21:17:02.175012+0000","last_active":"2023-09-23T21:18:04.456326+0000","last_peered":"2023-09-23T21:18:04.456326+0000","last_clean":"2023-09-23T21:18:04.456326+0000","last_became_active":"2023-09-23T21:16:42.451114+0000","last_became_peered":"2023-09-23T21:16:42.451114+0000","last_unstale":"2023-09-23T21:18:04.456326+0000","last_undegraded":"2023-09-23T21:18:04.456326+0000","last_fullsized":"2023-09-23T21:18:04.456326+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:00:53.868525+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":420,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":555,"reported_epoch":595,"state":"active+clean","last_fresh":"2023-09-23T21:18:04.456536+0000","last_change":"2023-09-23T21:16:42.432943+0000","last_active":"2023-09-23T21:18:04.456536+0000","last_peered":"2023-09-23T21:18:04.456536+0000","last_clean":"2023-09-23T21:18:04.456536+0000","last_became_active":"2023-09-23T21:16:42.427440+0000","last_became_peered":"2023-09-23T21:16:42.427440+0000","last_unstale":"2023-09-23T21:18:04.456536+0000","last_undegraded":"2023-09-23T21:18:04.456536+0000","last_fullsized":"2023-09-23T21:18:04.456536+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:59:32.044609+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,0],"acting":[5,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.4","version":"596'7136","reported_seq":8761,"reported_epoch":596,"state":"active+clean","last_fresh":"2023-09-23T21:18:07.297399+0000","last_change":"2023-09-23T21:18:05.512415+0000","last_active":"2023-09-23T21:18:07.297399+0000","last_peered":"2023-09-23T21:18:07.297399+0000","last_clean":"2023-09-23T21:18:07.297399+0000","last_became_active":"2023-09-23T21:18:05.477101+0000","last_became_peered":"2023-09-23T21:18:05.477101+0000","last_unstale":"2023-09-23T21:18:07.297399+0000","last_undegraded":"2023-09-23T21:18:07.297399+0000","last_fullsized":"2023-09-23T21:18:07.297399+0000","mapping_epoch":595,"log_start":"548'883","ondisk_log_start":"548'883","created":544,"last_epoch_clean":596,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_clean_scrub_stamp":"2023-09-23T21:15:24.860968+0000","objects_scrubbed":0,"log_size":6253,"log_dups_size":0,"ondisk_log_size":6253,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":82640896,"num_objects":1261,"num_object_clones":0,"num_object_copies":3783,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1261,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4899,"num_write_kb":197120,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":765,"num_bytes_recovered":50135040,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,3],"acting":[7,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.e","version":"79'161","reported_seq":814,"reported_epoch":595,"state":"active+clean","last_fresh":"2023-09-23T21:18:04.457040+0000","last_change":"2023-09-23T21:16:45.433204+0000","last_active":"2023-09-23T21:18:04.457040+0000","last_peered":"2023-09-23T21:18:04.457040+0000","last_clean":"2023-09-23T21:18:04.457040+0000","last_became_active":"2023-09-23T21:16:45.432836+0000","last_became_peered":"2023-09-23T21:16:45.432836+0000","last_unstale":"2023-09-23T21:18:04.457040+0000","last_undegraded":"2023-09-23T21:18:04.457040+0000","last_fullsized":"2023-09-23T21:18:04.457040+0000","mapping_epoch":567,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":568,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:15:39.046962+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:15:39.046962+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:11:20.720731+0000","scrub_duration":0.0082638769999999993,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":395,"reported_epoch":596,"state":"active+clean","last_fresh":"2023-09-23T21:18:05.470670+0000","last_change":"2023-09-23T21:13:13.402418+0000","last_active":"2023-09-23T21:18:05.470670+0000","last_peered":"2023-09-23T21:18:05.470670+0000","last_clean":"2023-09-23T21:18:05.470670+0000","last_became_active":"2023-09-23T21:13:13.400984+0000","last_became_peered":"2023-09-23T21:13:13.400984+0000","last_unstale":"2023-09-23T21:18:05.470670+0000","last_undegraded":"2023-09-23T21:18:05.470670+0000","last_fullsized":"2023-09-23T21:18:05.470670+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":495,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:28:01.411191+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,0],"acting":[3,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":569,"reported_epoch":594,"state":"active+clean","last_fresh":"2023-09-23T21:17:58.396985+0000","last_change":"2023-09-23T21:13:14.339670+0000","last_active":"2023-09-23T21:17:58.396985+0000","last_peered":"2023-09-23T21:17:58.396985+0000","last_clean":"2023-09-23T21:17:58.396985+0000","last_became_active":"2023-09-23T21:13:14.339341+0000","last_became_peered":"2023-09-23T21:13:14.339341+0000","last_unstale":"2023-09-23T21:17:58.396985+0000","last_undegraded":"2023-09-23T21:17:58.396985+0000","last_fullsized":"2023-09-23T21:17:58.396985+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:23:19.411368+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.3","version":"596'7070","reported_seq":8739,"reported_epoch":596,"state":"active+clean","last_fresh":"2023-09-23T21:18:07.308347+0000","last_change":"2023-09-23T21:17:51.904100+0000","last_active":"2023-09-23T21:18:07.308347+0000","last_peered":"2023-09-23T21:18:07.308347+0000","last_clean":"2023-09-23T21:18:07.308347+0000","last_became_active":"2023-09-23T21:17:51.872537+0000","last_became_peered":"2023-09-23T21:17:51.872537+0000","last_unstale":"2023-09-23T21:18:07.308347+0000","last_undegraded":"2023-09-23T21:18:07.308347+0000","last_fullsized":"2023-09-23T21:18:07.308347+0000","mapping_epoch":591,"log_start":"548'863","ondisk_log_start":"548'863","created":544,"last_epoch_clean":592,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_clean_scrub_stamp":"2023-09-23T21:15:24.860968+0000","objects_scrubbed":0,"log_size":6207,"log_dups_size":0,"ondisk_log_size":6207,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":74579968,"num_objects":1138,"num_object_clones":0,"num_object_copies":3414,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1138,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4854,"num_write_kb":191744,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":786,"num_bytes_recovered":51511296,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,2],"acting":[7,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.9","version":"79'161","reported_seq":797,"reported_epoch":595,"state":"active+clean","last_fresh":"2023-09-23T21:18:04.450819+0000","last_change":"2023-09-23T21:16:45.433968+0000","last_active":"2023-09-23T21:18:04.450819+0000","last_peered":"2023-09-23T21:18:04.450819+0000","last_clean":"2023-09-23T21:18:04.450819+0000","last_became_active":"2023-09-23T21:16:45.433602+0000","last_became_peered":"2023-09-23T21:16:45.433602+0000","last_unstale":"2023-09-23T21:18:04.450819+0000","last_undegraded":"2023-09-23T21:18:04.450819+0000","last_fullsized":"2023-09-23T21:18:04.450819+0000","mapping_epoch":567,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":568,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:15:43.437369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:15:43.437369+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:16:44.538102+0000","scrub_duration":0.0024129429999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,3],"acting":[2,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":155,"reported_epoch":595,"state":"active+clean","last_fresh":"2023-09-23T21:18:04.459924+0000","last_change":"2023-09-23T21:13:30.793273+0000","last_active":"2023-09-23T21:18:04.459924+0000","last_peered":"2023-09-23T21:18:04.459924+0000","last_clean":"2023-09-23T21:18:04.459924+0000","last_became_active":"2023-09-23T21:13:30.791269+0000","last_became_peered":"2023-09-23T21:13:30.791269+0000","last_unstale":"2023-09-23T21:18:04.459924+0000","last_undegraded":"2023-09-23T21:18:04.459924+0000","last_fullsized":"2023-09-23T21:18:04.459924+0000","mapping_epoch":501,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:38:07.142290+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":189,"reported_epoch":595,"state":"active+clean","last_fresh":"2023-09-23T21:18:04.450723+0000","last_change":"2023-09-23T21:16:42.417057+0000","last_active":"2023-09-23T21:18:04.450723+0000","last_peered":"2023-09-23T21:18:04.450723+0000","last_clean":"2023-09-23T21:18:04.450723+0000","last_became_active":"2023-09-23T21:16:42.416601+0000","last_became_peered":"2023-09-23T21:16:42.416601+0000","last_unstale":"2023-09-23T21:18:04.450723+0000","last_undegraded":"2023-09-23T21:18:04.450723+0000","last_fullsized":"2023-09-23T21:18:04.450723+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:56:52.167562+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,7],"acting":[4,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.2","version":"596'7311","reported_seq":7383,"reported_epoch":596,"state":"active+clean","last_fresh":"2023-09-23T21:18:06.629488+0000","last_change":"2023-09-23T21:16:32.870210+0000","last_active":"2023-09-23T21:18:06.629488+0000","last_peered":"2023-09-23T21:18:06.629488+0000","last_clean":"2023-09-23T21:18:06.629488+0000","last_became_active":"2023-09-23T21:16:32.867780+0000","last_became_peered":"2023-09-23T21:16:32.867780+0000","last_unstale":"2023-09-23T21:18:06.629488+0000","last_undegraded":"2023-09-23T21:18:06.629488+0000","last_fullsized":"2023-09-23T21:18:06.629488+0000","mapping_epoch":544,"log_start":"548'2","ondisk_log_start":"548'2","created":544,"last_epoch_clean":560,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_clean_scrub_stamp":"2023-09-23T21:15:24.860968+0000","objects_scrubbed":0,"log_size":7309,"log_dups_size":0,"ondisk_log_size":7309,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":78446592,"num_objects":1197,"num_object_clones":0,"num_object_copies":3591,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1197,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5027,"num_write_kb":199168,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.8","version":"79'161","reported_seq":730,"reported_epoch":595,"state":"active+clean","last_fresh":"2023-09-23T21:18:04.448332+0000","last_change":"2023-09-23T21:15:36.367819+0000","last_active":"2023-09-23T21:18:04.448332+0000","last_peered":"2023-09-23T21:18:04.448332+0000","last_clean":"2023-09-23T21:18:04.448332+0000","last_became_active":"2023-09-23T21:12:30.433217+0000","last_became_peered":"2023-09-23T21:12:30.433217+0000","last_unstale":"2023-09-23T21:18:04.448332+0000","last_undegraded":"2023-09-23T21:18:04.448332+0000","last_fullsized":"2023-09-23T21:18:04.448332+0000","mapping_epoch":359,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:15:36.367640+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:15:36.367640+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:49:35.243653+0000","scrub_duration":0.005826899,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":172,"reported_epoch":595,"state":"active+clean","last_fresh":"2023-09-23T21:18:04.450710+0000","last_change":"2023-09-23T21:12:10.823380+0000","last_active":"2023-09-23T21:18:04.450710+0000","last_peered":"2023-09-23T21:18:04.450710+0000","last_clean":"2023-09-23T21:18:04.450710+0000","last_became_active":"2023-09-23T21:12:10.823044+0000","last_became_peered":"2023-09-23T21:12:10.823044+0000","last_unstale":"2023-09-23T21:18:04.450710+0000","last_undegraded":"2023-09-23T21:18:04.450710+0000","last_fullsized":"2023-09-23T21:18:04.450710+0000","mapping_epoch":470,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:53:29.846536+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,5],"acting":[4,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":173,"reported_epoch":595,"state":"active+clean","last_fresh":"2023-09-23T21:18:04.456719+0000","last_change":"2023-09-23T21:13:30.772451+0000","last_active":"2023-09-23T21:18:04.456719+0000","last_peered":"2023-09-23T21:18:04.456719+0000","last_clean":"2023-09-23T21:18:04.456719+0000","last_became_active":"2023-09-23T21:13:30.772040+0000","last_became_peered":"2023-09-23T21:13:30.772040+0000","last_unstale":"2023-09-23T21:18:04.456719+0000","last_undegraded":"2023-09-23T21:18:04.456719+0000","last_fullsized":"2023-09-23T21:18:04.456719+0000","mapping_epoch":501,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:24:49.246727+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,0],"acting":[4,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.1","version":"597'7493","reported_seq":7564,"reported_epoch":597,"state":"active+clean","last_fresh":"2023-09-23T21:18:09.299655+0000","last_change":"2023-09-23T21:16:32.911457+0000","last_active":"2023-09-23T21:18:09.299655+0000","last_peered":"2023-09-23T21:18:09.299655+0000","last_clean":"2023-09-23T21:18:09.299655+0000","last_became_active":"2023-09-23T21:16:32.905672+0000","last_became_peered":"2023-09-23T21:16:32.905672+0000","last_unstale":"2023-09-23T21:18:09.299655+0000","last_undegraded":"2023-09-23T21:18:09.299655+0000","last_fullsized":"2023-09-23T21:18:09.299655+0000","mapping_epoch":544,"log_start":"550'3330","ondisk_log_start":"550'3330","created":544,"last_epoch_clean":560,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_clean_scrub_stamp":"2023-09-23T21:15:24.860968+0000","objects_scrubbed":0,"log_size":4163,"log_dups_size":0,"ondisk_log_size":4163,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":71565312,"num_objects":1092,"num_object_clones":0,"num_object_copies":3276,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1092,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5176,"num_write_kb":200576,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,2],"acting":[3,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.b","version":"79'161","reported_seq":612,"reported_epoch":594,"state":"active+clean","last_fresh":"2023-09-23T21:17:58.396663+0000","last_change":"2023-09-23T21:15:36.340009+0000","last_active":"2023-09-23T21:17:58.396663+0000","last_peered":"2023-09-23T21:17:58.396663+0000","last_clean":"2023-09-23T21:17:58.396663+0000","last_became_active":"2023-09-23T21:13:13.372328+0000","last_became_peered":"2023-09-23T21:13:13.372328+0000","last_unstale":"2023-09-23T21:17:58.396663+0000","last_undegraded":"2023-09-23T21:17:58.396663+0000","last_fullsized":"2023-09-23T21:17:58.396663+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":495,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:15:36.339885+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:15:36.339885+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:01:26.605006+0000","scrub_duration":0.0068473279999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":173,"reported_epoch":595,"state":"active+clean","last_fresh":"2023-09-23T21:18:04.450548+0000","last_change":"2023-09-23T21:16:42.425103+0000","last_active":"2023-09-23T21:18:04.450548+0000","last_peered":"2023-09-23T21:18:04.450548+0000","last_clean":"2023-09-23T21:18:04.450548+0000","last_became_active":"2023-09-23T21:16:42.424543+0000","last_became_peered":"2023-09-23T21:16:42.424543+0000","last_unstale":"2023-09-23T21:18:04.450548+0000","last_undegraded":"2023-09-23T21:18:04.450548+0000","last_fullsized":"2023-09-23T21:18:04.450548+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:11:28.885246+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":527,"reported_epoch":595,"state":"active+clean","last_fresh":"2023-09-23T21:18:04.457279+0000","last_change":"2023-09-23T21:12:10.811698+0000","last_active":"2023-09-23T21:18:04.457279+0000","last_peered":"2023-09-23T21:18:04.457279+0000","last_clean":"2023-09-23T21:18:04.457279+0000","last_became_active":"2023-09-23T21:12:10.789656+0000","last_became_peered":"2023-09-23T21:12:10.789656+0000","last_unstale":"2023-09-23T21:18:04.457279+0000","last_undegraded":"2023-09-23T21:18:04.457279+0000","last_fullsized":"2023-09-23T21:18:04.457279+0000","mapping_epoch":470,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:39:15.112091+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,5],"acting":[4,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.0","version":"597'7221","reported_seq":8916,"reported_epoch":597,"state":"active+undersized+remapped","last_fresh":"2023-09-23T21:18:09.328502+0000","last_change":"2023-09-23T21:18:09.328502+0000","last_active":"2023-09-23T21:18:09.328502+0000","last_peered":"2023-09-23T21:18:09.328502+0000","last_clean":"2023-09-23T21:16:02.267015+0000","last_became_active":"2023-09-23T21:16:43.480595+0000","last_became_peered":"2023-09-23T21:16:43.480595+0000","last_unstale":"2023-09-23T21:18:09.328502+0000","last_undegraded":"2023-09-23T21:18:09.328502+0000","last_fullsized":"2023-09-23T21:16:43.405455+0000","mapping_epoch":565,"log_start":"550'2842","ondisk_log_start":"550'2842","created":544,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_clean_scrub_stamp":"2023-09-23T21:15:24.860968+0000","objects_scrubbed":0,"log_size":4379,"log_dups_size":0,"ondisk_log_size":4379,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70483968,"num_objects":1075,"num_object_clones":0,"num_object_copies":3225,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1075,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4963,"num_write_kb":193248,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":803,"num_bytes_recovered":42336256,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,0],"avail_no_missing":["3","0","7"],"object_location_counts":[{"shards":"0,3","objects":1075}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.a","version":"79'161","reported_seq":756,"reported_epoch":595,"state":"active+clean","last_fresh":"2023-09-23T21:18:04.451868+0000","last_change":"2023-09-23T21:15:46.420777+0000","last_active":"2023-09-23T21:18:04.451868+0000","last_peered":"2023-09-23T21:18:04.451868+0000","last_clean":"2023-09-23T21:18:04.451868+0000","last_became_active":"2023-09-23T21:12:30.437427+0000","last_became_peered":"2023-09-23T21:12:30.437427+0000","last_unstale":"2023-09-23T21:18:04.451868+0000","last_undegraded":"2023-09-23T21:18:04.451868+0000","last_fullsized":"2023-09-23T21:18:04.451868+0000","mapping_epoch":446,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:15:46.420650+0000","last_deep_scrub":"79'161","last_deep_scrub_stamp":"2023-09-23T21:05:32.630292+0000","last_clean_scrub_stamp":"2023-09-23T21:15:46.420650+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:50:23.452475+0000","scrub_duration":0.0072898930000000004,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,4],"acting":[2,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":463,"reported_epoch":596,"state":"active+clean","last_fresh":"2023-09-23T21:18:05.470340+0000","last_change":"2023-09-23T21:16:42.447403+0000","last_active":"2023-09-23T21:18:05.470340+0000","last_peered":"2023-09-23T21:18:05.470340+0000","last_clean":"2023-09-23T21:18:05.470340+0000","last_became_active":"2023-09-23T21:16:42.447080+0000","last_became_peered":"2023-09-23T21:16:42.447080+0000","last_unstale":"2023-09-23T21:18:05.470340+0000","last_undegraded":"2023-09-23T21:18:05.470340+0000","last_fullsized":"2023-09-23T21:18:05.470340+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:15:19.446334+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,4],"acting":[3,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":594,"reported_epoch":596,"state":"active+clean","last_fresh":"2023-09-23T21:18:05.471326+0000","last_change":"2023-09-23T21:16:42.518739+0000","last_active":"2023-09-23T21:18:05.471326+0000","last_peered":"2023-09-23T21:18:05.471326+0000","last_clean":"2023-09-23T21:18:05.471326+0000","last_became_active":"2023-09-23T21:16:42.517945+0000","last_became_peered":"2023-09-23T21:16:42.517945+0000","last_unstale":"2023-09-23T21:18:05.471326+0000","last_undegraded":"2023-09-23T21:18:05.471326+0000","last_fullsized":"2023-09-23T21:18:05.471326+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:47:26.236933+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.d","version":"596'10105","reported_seq":10168,"reported_epoch":596,"state":"active+clean","last_fresh":"2023-09-23T21:18:06.646769+0000","last_change":"2023-09-23T21:15:25.892834+0000","last_active":"2023-09-23T21:18:06.646769+0000","last_peered":"2023-09-23T21:18:06.646769+0000","last_clean":"2023-09-23T21:18:06.646769+0000","last_became_active":"2023-09-23T21:15:25.892519+0000","last_became_peered":"2023-09-23T21:15:25.892519+0000","last_unstale":"2023-09-23T21:18:06.646769+0000","last_undegraded":"2023-09-23T21:18:06.646769+0000","last_fullsized":"2023-09-23T21:18:06.646769+0000","mapping_epoch":544,"log_start":"563'5072","ondisk_log_start":"563'5072","created":544,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_clean_scrub_stamp":"2023-09-23T21:15:24.860968+0000","objects_scrubbed":0,"log_size":5033,"log_dups_size":0,"ondisk_log_size":5033,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:46:28.375665+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":162922520,"num_objects":2487,"num_object_clones":0,"num_object_copies":7461,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2487,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":10105,"num_write_kb":402881,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,5],"acting":[2,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.7","version":"79'161","reported_seq":737,"reported_epoch":596,"state":"active+clean","last_fresh":"2023-09-23T21:18:05.471046+0000","last_change":"2023-09-23T21:16:45.425645+0000","last_active":"2023-09-23T21:18:05.471046+0000","last_peered":"2023-09-23T21:18:05.471046+0000","last_clean":"2023-09-23T21:18:05.471046+0000","last_became_active":"2023-09-23T21:16:45.425241+0000","last_became_peered":"2023-09-23T21:16:45.425241+0000","last_unstale":"2023-09-23T21:18:05.471046+0000","last_undegraded":"2023-09-23T21:18:05.471046+0000","last_fullsized":"2023-09-23T21:18:05.471046+0000","mapping_epoch":567,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":568,"parent":"0.0","parent_split_bits":3,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:15:51.016427+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:15:51.016427+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:49:31.176462+0000","scrub_duration":0.0017648640000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":544,"reported_epoch":594,"state":"active+clean","last_fresh":"2023-09-23T21:17:58.396999+0000","last_change":"2023-09-23T21:16:42.518826+0000","last_active":"2023-09-23T21:17:58.396999+0000","last_peered":"2023-09-23T21:17:58.396999+0000","last_clean":"2023-09-23T21:17:58.396999+0000","last_became_active":"2023-09-23T21:16:42.518488+0000","last_became_peered":"2023-09-23T21:16:42.518488+0000","last_unstale":"2023-09-23T21:17:58.396999+0000","last_undegraded":"2023-09-23T21:17:58.396999+0000","last_fullsized":"2023-09-23T21:17:58.396999+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:42:44.144383+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,2],"acting":[0,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":273,"reported_epoch":595,"state":"active+clean","last_fresh":"2023-09-23T21:18:04.456171+0000","last_change":"2023-09-23T21:05:04.326889+0000","last_active":"2023-09-23T21:18:04.456171+0000","last_peered":"2023-09-23T21:18:04.456171+0000","last_clean":"2023-09-23T21:18:04.456171+0000","last_became_active":"2023-09-23T21:05:04.326032+0000","last_became_peered":"2023-09-23T21:05:04.326032+0000","last_unstale":"2023-09-23T21:18:04.456171+0000","last_undegraded":"2023-09-23T21:18:04.456171+0000","last_fullsized":"2023-09-23T21:18:04.456171+0000","mapping_epoch":332,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:00:17.507889+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":605,"reported_epoch":595,"state":"active+clean","last_fresh":"2023-09-23T21:18:04.456692+0000","last_change":"2023-09-23T21:13:13.285608+0000","last_active":"2023-09-23T21:18:04.456692+0000","last_peered":"2023-09-23T21:18:04.456692+0000","last_clean":"2023-09-23T21:18:04.456692+0000","last_became_active":"2023-09-23T21:13:13.285255+0000","last_became_peered":"2023-09-23T21:13:13.285255+0000","last_unstale":"2023-09-23T21:18:04.456692+0000","last_undegraded":"2023-09-23T21:18:04.456692+0000","last_fullsized":"2023-09-23T21:18:04.456692+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":495,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:42:30.087599+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.b","version":"597'9815","reported_seq":9878,"reported_epoch":597,"state":"active+clean","last_fresh":"2023-09-23T21:18:09.352237+0000","last_change":"2023-09-23T21:15:25.906791+0000","last_active":"2023-09-23T21:18:09.352237+0000","last_peered":"2023-09-23T21:18:09.352237+0000","last_clean":"2023-09-23T21:18:09.352237+0000","last_became_active":"2023-09-23T21:15:25.906512+0000","last_became_peered":"2023-09-23T21:15:25.906512+0000","last_unstale":"2023-09-23T21:18:09.352237+0000","last_undegraded":"2023-09-23T21:18:09.352237+0000","last_fullsized":"2023-09-23T21:18:09.352237+0000","mapping_epoch":544,"log_start":"573'6245","ondisk_log_start":"573'6245","created":544,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_clean_scrub_stamp":"2023-09-23T21:15:24.860968+0000","objects_scrubbed":0,"log_size":3570,"log_dups_size":0,"ondisk_log_size":3570,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:58:51.749775+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":139919360,"num_objects":2135,"num_object_clones":0,"num_object_copies":6405,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2135,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9815,"num_write_kb":382400,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1","version":"79'161","reported_seq":679,"reported_epoch":594,"state":"active+clean","last_fresh":"2023-09-23T21:17:58.396827+0000","last_change":"2023-09-23T21:15:34.345290+0000","last_active":"2023-09-23T21:17:58.396827+0000","last_peered":"2023-09-23T21:17:58.396827+0000","last_clean":"2023-09-23T21:17:58.396827+0000","last_became_active":"2023-09-23T21:13:12.203380+0000","last_became_peered":"2023-09-23T21:13:12.203380+0000","last_unstale":"2023-09-23T21:17:58.396827+0000","last_undegraded":"2023-09-23T21:17:58.396827+0000","last_fullsized":"2023-09-23T21:17:58.396827+0000","mapping_epoch":493,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:15:34.345138+0000","last_deep_scrub":"79'161","last_deep_scrub_stamp":"2023-09-23T21:05:30.652189+0000","last_clean_scrub_stamp":"2023-09-23T21:15:34.345138+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:47:50.349409+0000","scrub_duration":0.0057520000000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":263,"reported_epoch":594,"state":"active+clean","last_fresh":"2023-09-23T21:17:58.397487+0000","last_change":"2023-09-23T21:13:13.238617+0000","last_active":"2023-09-23T21:17:58.397487+0000","last_peered":"2023-09-23T21:17:58.397487+0000","last_clean":"2023-09-23T21:17:58.397487+0000","last_became_active":"2023-09-23T21:13:13.232926+0000","last_became_peered":"2023-09-23T21:13:13.232926+0000","last_unstale":"2023-09-23T21:17:58.397487+0000","last_undegraded":"2023-09-23T21:17:58.397487+0000","last_fullsized":"2023-09-23T21:17:58.397487+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":495,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:17:21.823772+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":529,"reported_epoch":595,"state":"active+clean","last_fresh":"2023-09-23T21:18:04.451092+0000","last_change":"2023-09-23T21:13:14.144918+0000","last_active":"2023-09-23T21:18:04.451092+0000","last_peered":"2023-09-23T21:18:04.451092+0000","last_clean":"2023-09-23T21:18:04.451092+0000","last_became_active":"2023-09-23T21:13:14.132388+0000","last_became_peered":"2023-09-23T21:13:14.132388+0000","last_unstale":"2023-09-23T21:18:04.451092+0000","last_undegraded":"2023-09-23T21:18:04.451092+0000","last_fullsized":"2023-09-23T21:18:04.451092+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":495,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:56:02.619905+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,5],"acting":[4,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":435,"reported_epoch":595,"state":"active+clean","last_fresh":"2023-09-23T21:18:04.463688+0000","last_change":"2023-09-23T21:16:42.491111+0000","last_active":"2023-09-23T21:18:04.463688+0000","last_peered":"2023-09-23T21:18:04.463688+0000","last_clean":"2023-09-23T21:18:04.463688+0000","last_became_active":"2023-09-23T21:16:42.490275+0000","last_became_peered":"2023-09-23T21:16:42.490275+0000","last_unstale":"2023-09-23T21:18:04.463688+0000","last_undegraded":"2023-09-23T21:18:04.463688+0000","last_fullsized":"2023-09-23T21:18:04.463688+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:39:11.652564+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,7],"acting":[2,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"11.8","version":"595'7136","reported_seq":7207,"reported_epoch":595,"state":"active+clean","last_fresh":"2023-09-23T21:18:04.915739+0000","last_change":"2023-09-23T21:16:32.898234+0000","last_active":"2023-09-23T21:18:04.915739+0000","last_peered":"2023-09-23T21:18:04.915739+0000","last_clean":"2023-09-23T21:18:04.915739+0000","last_became_active":"2023-09-23T21:16:32.894560+0000","last_became_peered":"2023-09-23T21:16:32.894560+0000","last_unstale":"2023-09-23T21:18:04.915739+0000","last_undegraded":"2023-09-23T21:18:04.915739+0000","last_fullsized":"2023-09-23T21:18:04.915739+0000","mapping_epoch":544,"log_start":"549'1980","ondisk_log_start":"549'1980","created":544,"last_epoch_clean":560,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_clean_scrub_stamp":"2023-09-23T21:15:24.860968+0000","objects_scrubbed":0,"log_size":5156,"log_dups_size":0,"ondisk_log_size":5156,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":78348288,"num_objects":1195,"num_object_clones":0,"num_object_copies":3585,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1195,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4911,"num_write_kb":195424,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,5],"acting":[0,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.2","version":"79'161","reported_seq":910,"reported_epoch":594,"state":"active+clean","last_fresh":"2023-09-23T21:17:58.397327+0000","last_change":"2023-09-23T21:16:45.417075+0000","last_active":"2023-09-23T21:17:58.397327+0000","last_peered":"2023-09-23T21:17:58.397327+0000","last_clean":"2023-09-23T21:17:58.397327+0000","last_became_active":"2023-09-23T21:16:45.416752+0000","last_became_peered":"2023-09-23T21:16:45.416752+0000","last_unstale":"2023-09-23T21:17:58.397327+0000","last_undegraded":"2023-09-23T21:17:58.397327+0000","last_fullsized":"2023-09-23T21:17:58.397327+0000","mapping_epoch":567,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":568,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:24.802451+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:24.802451+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:01:51.129779+0000","scrub_duration":0.0090861250000000005,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,4],"acting":[0,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":606,"reported_epoch":595,"state":"active+clean","last_fresh":"2023-09-23T21:18:04.456526+0000","last_change":"2023-09-23T21:13:12.206980+0000","last_active":"2023-09-23T21:18:04.456526+0000","last_peered":"2023-09-23T21:18:04.456526+0000","last_clean":"2023-09-23T21:18:04.456526+0000","last_became_active":"2023-09-23T21:13:12.206650+0000","last_became_peered":"2023-09-23T21:13:12.206650+0000","last_unstale":"2023-09-23T21:18:04.456526+0000","last_undegraded":"2023-09-23T21:18:04.456526+0000","last_fullsized":"2023-09-23T21:18:04.456526+0000","mapping_epoch":493,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:10:01.892936+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":589,"reported_epoch":595,"state":"active+clean","last_fresh":"2023-09-23T21:18:04.448539+0000","last_change":"2023-09-23T21:13:13.407073+0000","last_active":"2023-09-23T21:18:04.448539+0000","last_peered":"2023-09-23T21:18:04.448539+0000","last_clean":"2023-09-23T21:18:04.448539+0000","last_became_active":"2023-09-23T21:13:13.395047+0000","last_became_peered":"2023-09-23T21:13:13.395047+0000","last_unstale":"2023-09-23T21:18:04.448539+0000","last_undegraded":"2023-09-23T21:18:04.448539+0000","last_fullsized":"2023-09-23T21:18:04.448539+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":495,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:04:43.384190+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,5],"acting":[2,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":124,"reported_epoch":594,"state":"active+clean","last_fresh":"2023-09-23T21:17:58.396803+0000","last_change":"2023-09-23T21:16:42.517729+0000","last_active":"2023-09-23T21:17:58.396803+0000","last_peered":"2023-09-23T21:17:58.396803+0000","last_clean":"2023-09-23T21:17:58.396803+0000","last_became_active":"2023-09-23T21:16:42.517373+0000","last_became_peered":"2023-09-23T21:16:42.517373+0000","last_unstale":"2023-09-23T21:17:58.396803+0000","last_undegraded":"2023-09-23T21:17:58.396803+0000","last_fullsized":"2023-09-23T21:17:58.396803+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:05:57.370550+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.a","version":"596'10003","reported_seq":10065,"reported_epoch":596,"state":"active+clean","last_fresh":"2023-09-23T21:18:06.634398+0000","last_change":"2023-09-23T21:15:25.893846+0000","last_active":"2023-09-23T21:18:06.634398+0000","last_peered":"2023-09-23T21:18:06.634398+0000","last_clean":"2023-09-23T21:18:06.634398+0000","last_became_active":"2023-09-23T21:15:25.893452+0000","last_became_peered":"2023-09-23T21:15:25.893452+0000","last_unstale":"2023-09-23T21:18:06.634398+0000","last_undegraded":"2023-09-23T21:18:06.634398+0000","last_fullsized":"2023-09-23T21:18:06.634398+0000","mapping_epoch":544,"log_start":"563'4925","ondisk_log_start":"563'4925","created":544,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_clean_scrub_stamp":"2023-09-23T21:15:24.860968+0000","objects_scrubbed":0,"log_size":5078,"log_dups_size":0,"ondisk_log_size":5078,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:42:32.842297+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":154075136,"num_objects":2351,"num_object_clones":0,"num_object_copies":7053,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2351,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10003,"num_write_kb":395328,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,5],"acting":[2,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":401,"reported_epoch":595,"state":"active+clean","last_fresh":"2023-09-23T21:18:04.451995+0000","last_change":"2023-09-23T21:16:42.423216+0000","last_active":"2023-09-23T21:18:04.451995+0000","last_peered":"2023-09-23T21:18:04.451995+0000","last_clean":"2023-09-23T21:18:04.451995+0000","last_became_active":"2023-09-23T21:16:42.422856+0000","last_became_peered":"2023-09-23T21:16:42.422856+0000","last_unstale":"2023-09-23T21:18:04.451995+0000","last_undegraded":"2023-09-23T21:18:04.451995+0000","last_fullsized":"2023-09-23T21:18:04.451995+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:06:54.105798+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"79'161","reported_seq":813,"reported_epoch":594,"state":"active+clean","last_fresh":"2023-09-23T21:17:58.396086+0000","last_change":"2023-09-23T21:16:45.428032+0000","last_active":"2023-09-23T21:17:58.396086+0000","last_peered":"2023-09-23T21:17:58.396086+0000","last_clean":"2023-09-23T21:17:58.396086+0000","last_became_active":"2023-09-23T21:16:45.427674+0000","last_became_peered":"2023-09-23T21:16:45.427674+0000","last_unstale":"2023-09-23T21:17:58.396086+0000","last_undegraded":"2023-09-23T21:17:58.396086+0000","last_fullsized":"2023-09-23T21:17:58.396086+0000","mapping_epoch":567,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":568,"parent":"0.0","parent_split_bits":0,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:20.738273+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:20.738273+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:39:42.441711+0000","scrub_duration":0.0088336339999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,2],"acting":[0,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":603,"reported_epoch":595,"state":"active+clean","last_fresh":"2023-09-23T21:18:04.456618+0000","last_change":"2023-09-23T21:11:08.760417+0000","last_active":"2023-09-23T21:18:04.456618+0000","last_peered":"2023-09-23T21:18:04.456618+0000","last_clean":"2023-09-23T21:18:04.456618+0000","last_became_active":"2023-09-23T21:11:08.760150+0000","last_became_peered":"2023-09-23T21:11:08.760150+0000","last_unstale":"2023-09-23T21:18:04.456618+0000","last_undegraded":"2023-09-23T21:18:04.456618+0000","last_fullsized":"2023-09-23T21:18:04.456618+0000","mapping_epoch":443,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":444,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:23:10.849768+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":190,"reported_epoch":595,"state":"active+clean","last_fresh":"2023-09-23T21:18:04.456891+0000","last_change":"2023-09-23T21:16:42.424717+0000","last_active":"2023-09-23T21:18:04.456891+0000","last_peered":"2023-09-23T21:18:04.456891+0000","last_clean":"2023-09-23T21:18:04.456891+0000","last_became_active":"2023-09-23T21:16:42.424376+0000","last_became_peered":"2023-09-23T21:16:42.424376+0000","last_unstale":"2023-09-23T21:18:04.456891+0000","last_undegraded":"2023-09-23T21:18:04.456891+0000","last_fullsized":"2023-09-23T21:18:04.456891+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:25:14.162321+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.9","version":"597'7382","reported_seq":9204,"reported_epoch":597,"state":"active+clean","last_fresh":"2023-09-23T21:18:09.364402+0000","last_change":"2023-09-23T21:17:58.411362+0000","last_active":"2023-09-23T21:18:09.364402+0000","last_peered":"2023-09-23T21:18:09.364402+0000","last_clean":"2023-09-23T21:18:09.364402+0000","last_became_active":"2023-09-23T21:17:58.399421+0000","last_became_peered":"2023-09-23T21:17:58.399421+0000","last_unstale":"2023-09-23T21:18:09.364402+0000","last_undegraded":"2023-09-23T21:18:09.364402+0000","last_fullsized":"2023-09-23T21:18:09.364402+0000","mapping_epoch":593,"log_start":"550'3112","ondisk_log_start":"550'3112","created":544,"last_epoch_clean":594,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_clean_scrub_stamp":"2023-09-23T21:15:24.860968+0000","objects_scrubbed":0,"log_size":4270,"log_dups_size":0,"ondisk_log_size":4270,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":77496320,"num_objects":1182,"num_object_clones":0,"num_object_copies":3546,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1182,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5098,"num_write_kb":200992,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":860,"num_bytes_recovered":56393728,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,7],"acting":[3,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.3","version":"79'161","reported_seq":898,"reported_epoch":595,"state":"active+clean","last_fresh":"2023-09-23T21:18:04.450679+0000","last_change":"2023-09-23T21:13:33.771279+0000","last_active":"2023-09-23T21:18:04.450679+0000","last_peered":"2023-09-23T21:18:04.450679+0000","last_clean":"2023-09-23T21:18:04.450679+0000","last_became_active":"2023-09-23T21:13:33.770980+0000","last_became_peered":"2023-09-23T21:13:33.770980+0000","last_unstale":"2023-09-23T21:18:04.450679+0000","last_undegraded":"2023-09-23T21:18:04.450679+0000","last_fullsized":"2023-09-23T21:18:04.450679+0000","mapping_epoch":504,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":505,"parent":"0.0","parent_split_bits":3,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:05:57.832302+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:05:57.832302+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:19:42.309604+0000","scrub_duration":0.010206847999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":431,"reported_epoch":594,"state":"active+clean","last_fresh":"2023-09-23T21:17:58.396411+0000","last_change":"2023-09-23T21:16:42.455365+0000","last_active":"2023-09-23T21:17:58.396411+0000","last_peered":"2023-09-23T21:17:58.396411+0000","last_clean":"2023-09-23T21:17:58.396411+0000","last_became_active":"2023-09-23T21:16:42.452829+0000","last_became_peered":"2023-09-23T21:16:42.452829+0000","last_unstale":"2023-09-23T21:17:58.396411+0000","last_undegraded":"2023-09-23T21:17:58.396411+0000","last_fullsized":"2023-09-23T21:17:58.396411+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:49:26.303389+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":595,"reported_epoch":594,"state":"active+clean","last_fresh":"2023-09-23T21:17:58.396456+0000","last_change":"2023-09-23T21:13:44.409212+0000","last_active":"2023-09-23T21:17:58.396456+0000","last_peered":"2023-09-23T21:17:58.396456+0000","last_clean":"2023-09-23T21:17:58.396456+0000","last_became_active":"2023-09-23T21:13:44.408958+0000","last_became_peered":"2023-09-23T21:13:44.408958+0000","last_unstale":"2023-09-23T21:17:58.396456+0000","last_undegraded":"2023-09-23T21:17:58.396456+0000","last_fullsized":"2023-09-23T21:17:58.396456+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:03:52.749964+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":124,"reported_epoch":594,"state":"active+clean","last_fresh":"2023-09-23T21:17:58.397261+0000","last_change":"2023-09-23T21:16:42.427490+0000","last_active":"2023-09-23T21:17:58.397261+0000","last_peered":"2023-09-23T21:17:58.397261+0000","last_clean":"2023-09-23T21:17:58.397261+0000","last_became_active":"2023-09-23T21:16:42.426566+0000","last_became_peered":"2023-09-23T21:16:42.426566+0000","last_unstale":"2023-09-23T21:17:58.397261+0000","last_undegraded":"2023-09-23T21:17:58.397261+0000","last_fullsized":"2023-09-23T21:17:58.397261+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:20:26.396005+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,7],"acting":[0,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.c","version":"596'9852","reported_seq":13019,"reported_epoch":596,"state":"active+clean","last_fresh":"2023-09-23T21:18:06.548937+0000","last_change":"2023-09-23T21:17:21.358806+0000","last_active":"2023-09-23T21:18:06.548937+0000","last_peered":"2023-09-23T21:18:06.548937+0000","last_clean":"2023-09-23T21:18:06.548937+0000","last_became_active":"2023-09-23T21:17:21.346629+0000","last_became_peered":"2023-09-23T21:17:21.346629+0000","last_unstale":"2023-09-23T21:18:06.548937+0000","last_undegraded":"2023-09-23T21:18:06.548937+0000","last_fullsized":"2023-09-23T21:18:06.548937+0000","mapping_epoch":576,"log_start":"563'4800","ondisk_log_start":"563'4800","created":544,"last_epoch_clean":577,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_clean_scrub_stamp":"2023-09-23T21:15:24.860968+0000","objects_scrubbed":0,"log_size":5052,"log_dups_size":0,"ondisk_log_size":5052,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:59:48.173244+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":152305664,"num_objects":2324,"num_object_clones":0,"num_object_copies":6972,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2324,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9852,"num_write_kb":389632,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1539,"num_bytes_recovered":100859904,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,7],"acting":[2,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.6","version":"79'161","reported_seq":527,"reported_epoch":594,"state":"active+clean","last_fresh":"2023-09-23T21:17:58.397307+0000","last_change":"2023-09-23T21:15:43.333919+0000","last_active":"2023-09-23T21:17:58.397307+0000","last_peered":"2023-09-23T21:17:58.397307+0000","last_clean":"2023-09-23T21:17:58.397307+0000","last_became_active":"2023-09-23T21:13:33.769303+0000","last_became_peered":"2023-09-23T21:13:33.769303+0000","last_unstale":"2023-09-23T21:17:58.397307+0000","last_undegraded":"2023-09-23T21:17:58.397307+0000","last_fullsized":"2023-09-23T21:17:58.397307+0000","mapping_epoch":504,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":505,"parent":"0.0","parent_split_bits":3,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:15:43.333733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:15:43.333733+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:42:29.787357+0000","scrub_duration":0.0092107530000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":41,"reported_epoch":595,"state":"active+clean","last_fresh":"2023-09-23T21:18:04.458517+0000","last_change":"2023-09-23T21:16:42.483813+0000","last_active":"2023-09-23T21:18:04.458517+0000","last_peered":"2023-09-23T21:18:04.458517+0000","last_clean":"2023-09-23T21:18:04.458517+0000","last_became_active":"2023-09-23T21:16:42.482847+0000","last_became_peered":"2023-09-23T21:16:42.482847+0000","last_unstale":"2023-09-23T21:18:04.458517+0000","last_undegraded":"2023-09-23T21:18:04.458517+0000","last_fullsized":"2023-09-23T21:18:04.458517+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:38:57.905679+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":480,"reported_epoch":595,"state":"active+clean","last_fresh":"2023-09-23T21:18:04.464004+0000","last_change":"2023-09-23T21:13:30.790590+0000","last_active":"2023-09-23T21:18:04.464004+0000","last_peered":"2023-09-23T21:18:04.464004+0000","last_clean":"2023-09-23T21:18:04.464004+0000","last_became_active":"2023-09-23T21:13:30.790253+0000","last_became_peered":"2023-09-23T21:13:30.790253+0000","last_unstale":"2023-09-23T21:18:04.464004+0000","last_undegraded":"2023-09-23T21:18:04.464004+0000","last_fullsized":"2023-09-23T21:18:04.464004+0000","mapping_epoch":501,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:23:41.843250+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,0],"acting":[2,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":560,"reported_epoch":594,"state":"active+clean","last_fresh":"2023-09-23T21:17:58.397226+0000","last_change":"2023-09-23T21:16:42.428535+0000","last_active":"2023-09-23T21:17:58.397226+0000","last_peered":"2023-09-23T21:17:58.397226+0000","last_clean":"2023-09-23T21:17:58.397226+0000","last_became_active":"2023-09-23T21:16:42.428149+0000","last_became_peered":"2023-09-23T21:16:42.428149+0000","last_unstale":"2023-09-23T21:17:58.397226+0000","last_undegraded":"2023-09-23T21:17:58.397226+0000","last_fullsized":"2023-09-23T21:17:58.397226+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:38:59.979788+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.f","version":"596'9870","reported_seq":13486,"reported_epoch":596,"state":"active+clean","last_fresh":"2023-09-23T21:18:06.632173+0000","last_change":"2023-09-23T21:17:03.878062+0000","last_active":"2023-09-23T21:18:06.632173+0000","last_peered":"2023-09-23T21:18:06.632173+0000","last_clean":"2023-09-23T21:18:06.632173+0000","last_became_active":"2023-09-23T21:17:03.870998+0000","last_became_peered":"2023-09-23T21:17:03.870998+0000","last_unstale":"2023-09-23T21:18:06.632173+0000","last_undegraded":"2023-09-23T21:18:06.632173+0000","last_fullsized":"2023-09-23T21:18:06.632173+0000","mapping_epoch":572,"log_start":"562'4800","ondisk_log_start":"562'4800","created":544,"last_epoch_clean":573,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_clean_scrub_stamp":"2023-09-23T21:15:24.860968+0000","objects_scrubbed":0,"log_size":5070,"log_dups_size":0,"ondisk_log_size":5070,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:19:31.942955+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":150601728,"num_objects":2298,"num_object_clones":0,"num_object_copies":6894,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2298,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9870,"num_write_kb":389376,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1762,"num_bytes_recovered":115474432,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,0],"acting":[2,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.5","version":"79'161","reported_seq":633,"reported_epoch":595,"state":"active+clean","last_fresh":"2023-09-23T21:18:04.456808+0000","last_change":"2023-09-23T21:15:41.108638+0000","last_active":"2023-09-23T21:18:04.456808+0000","last_peered":"2023-09-23T21:18:04.456808+0000","last_clean":"2023-09-23T21:18:04.456808+0000","last_became_active":"2023-09-23T21:13:38.373633+0000","last_became_peered":"2023-09-23T21:13:38.373633+0000","last_unstale":"2023-09-23T21:18:04.456808+0000","last_undegraded":"2023-09-23T21:18:04.456808+0000","last_fullsized":"2023-09-23T21:18:04.456808+0000","mapping_epoch":507,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":3,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:15:41.108579+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:15:41.108579+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:21:58.075649+0000","scrub_duration":0.0061511960000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":163,"reported_epoch":594,"state":"active+clean","last_fresh":"2023-09-23T21:17:58.397262+0000","last_change":"2023-09-23T21:17:02.204086+0000","last_active":"2023-09-23T21:17:58.397262+0000","last_peered":"2023-09-23T21:17:58.397262+0000","last_clean":"2023-09-23T21:17:58.397262+0000","last_became_active":"2023-09-23T21:16:42.517549+0000","last_became_peered":"2023-09-23T21:16:42.517549+0000","last_unstale":"2023-09-23T21:17:58.397262+0000","last_undegraded":"2023-09-23T21:17:58.397262+0000","last_fullsized":"2023-09-23T21:17:58.397262+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:43:07.991844+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":576,"reported_epoch":595,"state":"active+clean","last_fresh":"2023-09-23T21:18:04.457121+0000","last_change":"2023-09-23T21:13:30.807520+0000","last_active":"2023-09-23T21:18:04.457121+0000","last_peered":"2023-09-23T21:18:04.457121+0000","last_clean":"2023-09-23T21:18:04.457121+0000","last_became_active":"2023-09-23T21:13:30.807222+0000","last_became_peered":"2023-09-23T21:13:30.807222+0000","last_unstale":"2023-09-23T21:18:04.457121+0000","last_undegraded":"2023-09-23T21:18:04.457121+0000","last_fullsized":"2023-09-23T21:18:04.457121+0000","mapping_epoch":501,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:40:31.989749+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":458,"reported_epoch":596,"state":"active+clean","last_fresh":"2023-09-23T21:18:05.470432+0000","last_change":"2023-09-23T21:16:42.463411+0000","last_active":"2023-09-23T21:18:05.470432+0000","last_peered":"2023-09-23T21:18:05.470432+0000","last_clean":"2023-09-23T21:18:05.470432+0000","last_became_active":"2023-09-23T21:16:42.463020+0000","last_became_peered":"2023-09-23T21:16:42.463020+0000","last_unstale":"2023-09-23T21:18:05.470432+0000","last_undegraded":"2023-09-23T21:18:05.470432+0000","last_fullsized":"2023-09-23T21:18:05.470432+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:07:34.346129+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.e","version":"596'9929","reported_seq":14123,"reported_epoch":596,"state":"active+clean","last_fresh":"2023-09-23T21:18:07.064973+0000","last_change":"2023-09-23T21:17:34.456174+0000","last_active":"2023-09-23T21:18:07.064973+0000","last_peered":"2023-09-23T21:18:07.064973+0000","last_clean":"2023-09-23T21:18:07.064973+0000","last_became_active":"2023-09-23T21:17:34.444637+0000","last_became_peered":"2023-09-23T21:17:34.444637+0000","last_unstale":"2023-09-23T21:18:07.064973+0000","last_undegraded":"2023-09-23T21:18:07.064973+0000","last_fullsized":"2023-09-23T21:18:07.064973+0000","mapping_epoch":582,"log_start":"567'5000","ondisk_log_start":"567'5000","created":544,"last_epoch_clean":583,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_clean_scrub_stamp":"2023-09-23T21:15:24.860968+0000","objects_scrubbed":0,"log_size":4929,"log_dups_size":0,"ondisk_log_size":4929,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:17:47.772535+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":152502272,"num_objects":2327,"num_object_clones":0,"num_object_copies":6981,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2327,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9929,"num_write_kb":392192,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2052,"num_bytes_recovered":134479872,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.4","version":"79'161","reported_seq":1003,"reported_epoch":596,"state":"active+clean","last_fresh":"2023-09-23T21:18:05.470491+0000","last_change":"2023-09-23T21:16:45.426136+0000","last_active":"2023-09-23T21:18:05.470491+0000","last_peered":"2023-09-23T21:18:05.470491+0000","last_clean":"2023-09-23T21:18:05.470491+0000","last_became_active":"2023-09-23T21:16:45.425669+0000","last_became_peered":"2023-09-23T21:16:45.425669+0000","last_unstale":"2023-09-23T21:18:05.470491+0000","last_undegraded":"2023-09-23T21:18:05.470491+0000","last_fullsized":"2023-09-23T21:18:05.470491+0000","mapping_epoch":567,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":568,"parent":"0.0","parent_split_bits":3,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:20.012764+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:20.012764+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:52:39.142509+0000","scrub_duration":0.0045251919999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":554,"reported_epoch":595,"state":"active+clean","last_fresh":"2023-09-23T21:18:04.456438+0000","last_change":"2023-09-23T21:16:42.459732+0000","last_active":"2023-09-23T21:18:04.456438+0000","last_peered":"2023-09-23T21:18:04.456438+0000","last_clean":"2023-09-23T21:18:04.456438+0000","last_became_active":"2023-09-23T21:16:42.459413+0000","last_became_peered":"2023-09-23T21:16:42.459413+0000","last_unstale":"2023-09-23T21:18:04.456438+0000","last_undegraded":"2023-09-23T21:18:04.456438+0000","last_fullsized":"2023-09-23T21:18:04.456438+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:37:44.548863+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,3],"acting":[2,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.6","version":"70'1","reported_seq":596,"reported_epoch":594,"state":"active+clean","last_fresh":"2023-09-23T21:17:58.397503+0000","last_change":"2023-09-23T21:17:02.208077+0000","last_active":"2023-09-23T21:17:58.397503+0000","last_peered":"2023-09-23T21:17:58.397503+0000","last_clean":"2023-09-23T21:17:58.397503+0000","last_became_active":"2023-09-23T21:16:42.431308+0000","last_became_peered":"2023-09-23T21:16:42.431308+0000","last_unstale":"2023-09-23T21:17:58.397503+0000","last_undegraded":"2023-09-23T21:17:58.397503+0000","last_fullsized":"2023-09-23T21:17:58.397503+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:16:49.009013+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":170,"reported_epoch":595,"state":"active+clean","last_fresh":"2023-09-23T21:18:04.450497+0000","last_change":"2023-09-23T21:13:12.216895+0000","last_active":"2023-09-23T21:18:04.450497+0000","last_peered":"2023-09-23T21:18:04.450497+0000","last_clean":"2023-09-23T21:18:04.450497+0000","last_became_active":"2023-09-23T21:13:12.216654+0000","last_became_peered":"2023-09-23T21:13:12.216654+0000","last_unstale":"2023-09-23T21:18:04.450497+0000","last_undegraded":"2023-09-23T21:18:04.450497+0000","last_fullsized":"2023-09-23T21:18:04.450497+0000","mapping_epoch":493,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:02:40.430096+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.13","version":"596'7076","reported_seq":8716,"reported_epoch":596,"state":"active+clean","last_fresh":"2023-09-23T21:18:07.209174+0000","last_change":"2023-09-23T21:17:50.364498+0000","last_active":"2023-09-23T21:18:07.209174+0000","last_peered":"2023-09-23T21:18:07.209174+0000","last_clean":"2023-09-23T21:18:07.209174+0000","last_became_active":"2023-09-23T21:17:39.911716+0000","last_became_peered":"2023-09-23T21:17:39.911716+0000","last_unstale":"2023-09-23T21:18:07.209174+0000","last_undegraded":"2023-09-23T21:18:07.209174+0000","last_fullsized":"2023-09-23T21:18:07.209174+0000","mapping_epoch":585,"log_start":"548'914","ondisk_log_start":"548'914","created":559,"last_epoch_clean":590,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_clean_scrub_stamp":"2023-09-23T21:15:24.860968+0000","objects_scrubbed":0,"log_size":6162,"log_dups_size":0,"ondisk_log_size":6162,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":73400320,"num_objects":1120,"num_object_clones":0,"num_object_copies":3360,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1120,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4860,"num_write_kb":191360,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":772,"num_bytes_recovered":50528256,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,2],"acting":[7,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.19","version":"79'161","reported_seq":803,"reported_epoch":594,"state":"active+clean","last_fresh":"2023-09-23T21:17:58.396196+0000","last_change":"2023-09-23T21:16:45.428879+0000","last_active":"2023-09-23T21:17:58.396196+0000","last_peered":"2023-09-23T21:17:58.396196+0000","last_clean":"2023-09-23T21:17:58.396196+0000","last_became_active":"2023-09-23T21:16:45.428660+0000","last_became_peered":"2023-09-23T21:16:45.428660+0000","last_unstale":"2023-09-23T21:17:58.396196+0000","last_undegraded":"2023-09-23T21:17:58.396196+0000","last_fullsized":"2023-09-23T21:17:58.396196+0000","mapping_epoch":567,"log_start":"0'0","ondisk_log_start":"0'0","created":479,"last_epoch_clean":568,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:15:48.271791+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:15:48.271791+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:25:26.361045+0000","scrub_duration":0.0057149180000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":123,"reported_epoch":594,"state":"active+clean","last_fresh":"2023-09-23T21:17:58.397182+0000","last_change":"2023-09-23T21:16:42.434830+0000","last_active":"2023-09-23T21:17:58.397182+0000","last_peered":"2023-09-23T21:17:58.397182+0000","last_clean":"2023-09-23T21:17:58.397182+0000","last_became_active":"2023-09-23T21:16:42.430289+0000","last_became_peered":"2023-09-23T21:16:42.430289+0000","last_unstale":"2023-09-23T21:17:58.397182+0000","last_undegraded":"2023-09-23T21:17:58.397182+0000","last_fullsized":"2023-09-23T21:17:58.397182+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:38:48.053766+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":100,"reported_epoch":595,"state":"active+clean","last_fresh":"2023-09-23T21:18:04.463929+0000","last_change":"2023-09-23T21:13:38.373814+0000","last_active":"2023-09-23T21:18:04.463929+0000","last_peered":"2023-09-23T21:18:04.463929+0000","last_clean":"2023-09-23T21:18:04.463929+0000","last_became_active":"2023-09-23T21:13:38.373458+0000","last_became_peered":"2023-09-23T21:13:38.373458+0000","last_unstale":"2023-09-23T21:18:04.463929+0000","last_undegraded":"2023-09-23T21:18:04.463929+0000","last_fullsized":"2023-09-23T21:18:04.463929+0000","mapping_epoch":507,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:23:08.288401+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"11.14","version":"596'7149","reported_seq":7229,"reported_epoch":596,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T21:18:06.651770+0000","last_change":"2023-09-23T21:16:43.483955+0000","last_active":"2023-09-23T21:18:06.651770+0000","last_peered":"2023-09-23T21:18:06.651770+0000","last_clean":"2023-09-23T21:16:02.247284+0000","last_became_active":"2023-09-23T21:16:43.479075+0000","last_became_peered":"2023-09-23T21:16:43.479075+0000","last_unstale":"2023-09-23T21:18:06.651770+0000","last_undegraded":"2023-09-23T21:16:43.418318+0000","last_fullsized":"2023-09-23T21:16:43.409626+0000","mapping_epoch":565,"log_start":"0'0","ondisk_log_start":"0'0","created":559,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_clean_scrub_stamp":"2023-09-23T21:15:24.860968+0000","objects_scrubbed":0,"log_size":7149,"log_dups_size":0,"ondisk_log_size":7149,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":71958528,"num_objects":1098,"num_object_clones":0,"num_object_copies":3294,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":803,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1098,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4912,"num_write_kb":192320,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,3],"acting":[2,3],"avail_no_missing":["2","3"],"object_location_counts":[{"shards":"0,2,3","objects":580},{"shards":"2,3","objects":295}],"blocked_by":[],"up_primary":7,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.1e","version":"79'161","reported_seq":580,"reported_epoch":596,"state":"active+clean","last_fresh":"2023-09-23T21:18:05.471601+0000","last_change":"2023-09-23T21:16:45.420788+0000","last_active":"2023-09-23T21:18:05.471601+0000","last_peered":"2023-09-23T21:18:05.471601+0000","last_clean":"2023-09-23T21:18:05.471601+0000","last_became_active":"2023-09-23T21:16:45.420448+0000","last_became_peered":"2023-09-23T21:16:45.420448+0000","last_unstale":"2023-09-23T21:18:05.471601+0000","last_undegraded":"2023-09-23T21:18:05.471601+0000","last_fullsized":"2023-09-23T21:18:05.471601+0000","mapping_epoch":567,"log_start":"0'0","ondisk_log_start":"0'0","created":479,"last_epoch_clean":568,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:15:53.061319+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:15:53.061319+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:46:31.982439+0000","scrub_duration":0.007429878,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,4],"acting":[3,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":205,"reported_epoch":595,"state":"active+clean","last_fresh":"2023-09-23T21:18:04.450805+0000","last_change":"2023-09-23T21:16:42.518584+0000","last_active":"2023-09-23T21:18:04.450805+0000","last_peered":"2023-09-23T21:18:04.450805+0000","last_clean":"2023-09-23T21:18:04.450805+0000","last_became_active":"2023-09-23T21:16:42.416632+0000","last_became_peered":"2023-09-23T21:16:42.416632+0000","last_unstale":"2023-09-23T21:18:04.450805+0000","last_undegraded":"2023-09-23T21:18:04.450805+0000","last_fullsized":"2023-09-23T21:18:04.450805+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:02:18.972080+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,7],"acting":[4,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":160,"reported_epoch":595,"state":"active+clean","last_fresh":"2023-09-23T21:18:04.450930+0000","last_change":"2023-09-23T21:12:10.770474+0000","last_active":"2023-09-23T21:18:04.450930+0000","last_peered":"2023-09-23T21:18:04.450930+0000","last_clean":"2023-09-23T21:18:04.450930+0000","last_became_active":"2023-09-23T21:12:10.770184+0000","last_became_peered":"2023-09-23T21:12:10.770184+0000","last_unstale":"2023-09-23T21:18:04.450930+0000","last_undegraded":"2023-09-23T21:18:04.450930+0000","last_fullsized":"2023-09-23T21:18:04.450930+0000","mapping_epoch":470,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:05:09.407151+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.15","version":"596'7331","reported_seq":12568,"reported_epoch":596,"state":"active+clean","last_fresh":"2023-09-23T21:18:07.369723+0000","last_change":"2023-09-23T21:18:06.522430+0000","last_active":"2023-09-23T21:18:07.369723+0000","last_peered":"2023-09-23T21:18:07.369723+0000","last_clean":"2023-09-23T21:18:07.369723+0000","last_became_active":"2023-09-23T21:16:46.418280+0000","last_became_peered":"2023-09-23T21:16:46.418280+0000","last_unstale":"2023-09-23T21:18:07.369723+0000","last_undegraded":"2023-09-23T21:18:07.369723+0000","last_fullsized":"2023-09-23T21:18:07.369723+0000","mapping_epoch":568,"log_start":"548'353","ondisk_log_start":"548'353","created":559,"last_epoch_clean":569,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_clean_scrub_stamp":"2023-09-23T21:15:24.860968+0000","objects_scrubbed":0,"log_size":6978,"log_dups_size":0,"ondisk_log_size":6978,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":74678272,"num_objects":1140,"num_object_clones":0,"num_object_copies":3420,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1140,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5036,"num_write_kb":197600,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1697,"num_bytes_recovered":111214592,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":387,"reported_epoch":595,"state":"active+clean","last_fresh":"2023-09-23T21:18:04.466550+0000","last_change":"2023-09-23T21:12:10.771160+0000","last_active":"2023-09-23T21:18:04.466550+0000","last_peered":"2023-09-23T21:18:04.466550+0000","last_clean":"2023-09-23T21:18:04.466550+0000","last_became_active":"2023-09-23T21:12:10.757808+0000","last_became_peered":"2023-09-23T21:12:10.757808+0000","last_unstale":"2023-09-23T21:18:04.466550+0000","last_undegraded":"2023-09-23T21:18:04.466550+0000","last_fullsized":"2023-09-23T21:18:04.466550+0000","mapping_epoch":470,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:30:50.987139+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":170,"reported_epoch":595,"state":"active+clean","last_fresh":"2023-09-23T21:18:04.450616+0000","last_change":"2023-09-23T21:13:12.216602+0000","last_active":"2023-09-23T21:18:04.450616+0000","last_peered":"2023-09-23T21:18:04.450616+0000","last_clean":"2023-09-23T21:18:04.450616+0000","last_became_active":"2023-09-23T21:13:12.216204+0000","last_became_peered":"2023-09-23T21:13:12.216204+0000","last_unstale":"2023-09-23T21:18:04.450616+0000","last_undegraded":"2023-09-23T21:18:04.450616+0000","last_fullsized":"2023-09-23T21:18:04.450616+0000","mapping_epoch":493,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:23:37.043390+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.16","version":"595'7025","reported_seq":7097,"reported_epoch":595,"state":"active+clean","last_fresh":"2023-09-23T21:18:04.754778+0000","last_change":"2023-09-23T21:16:33.955304+0000","last_active":"2023-09-23T21:18:04.754778+0000","last_peered":"2023-09-23T21:18:04.754778+0000","last_clean":"2023-09-23T21:18:04.754778+0000","last_became_active":"2023-09-23T21:16:32.926127+0000","last_became_peered":"2023-09-23T21:16:32.926127+0000","last_unstale":"2023-09-23T21:18:04.754778+0000","last_undegraded":"2023-09-23T21:18:04.754778+0000","last_fullsized":"2023-09-23T21:18:04.754778+0000","mapping_epoch":559,"log_start":"549'1600","ondisk_log_start":"549'1600","created":559,"last_epoch_clean":561,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_clean_scrub_stamp":"2023-09-23T21:15:24.860968+0000","objects_scrubbed":0,"log_size":5425,"log_dups_size":0,"ondisk_log_size":5425,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":74022912,"num_objects":1130,"num_object_clones":0,"num_object_copies":3390,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1130,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4792,"num_write_kb":189472,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1c","version":"79'161","reported_seq":1027,"reported_epoch":596,"state":"active+clean","last_fresh":"2023-09-23T21:18:05.470288+0000","last_change":"2023-09-23T21:16:45.419779+0000","last_active":"2023-09-23T21:18:05.470288+0000","last_peered":"2023-09-23T21:18:05.470288+0000","last_clean":"2023-09-23T21:18:05.470288+0000","last_became_active":"2023-09-23T21:16:45.419394+0000","last_became_peered":"2023-09-23T21:16:45.419394+0000","last_unstale":"2023-09-23T21:18:05.470288+0000","last_undegraded":"2023-09-23T21:18:05.470288+0000","last_fullsized":"2023-09-23T21:18:05.470288+0000","mapping_epoch":567,"log_start":"0'0","ondisk_log_start":"0'0","created":479,"last_epoch_clean":568,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:15:54.019828+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:15:54.019828+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:59:05.422258+0000","scrub_duration":0.0020962530000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":204,"reported_epoch":595,"state":"active+clean","last_fresh":"2023-09-23T21:18:04.453288+0000","last_change":"2023-09-23T21:16:42.424187+0000","last_active":"2023-09-23T21:18:04.453288+0000","last_peered":"2023-09-23T21:18:04.453288+0000","last_clean":"2023-09-23T21:18:04.453288+0000","last_became_active":"2023-09-23T21:16:42.423805+0000","last_became_peered":"2023-09-23T21:16:42.423805+0000","last_unstale":"2023-09-23T21:18:04.453288+0000","last_undegraded":"2023-09-23T21:18:04.453288+0000","last_fullsized":"2023-09-23T21:18:04.453288+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:55:46.051199+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,7],"acting":[2,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":265,"reported_epoch":595,"state":"active+clean","last_fresh":"2023-09-23T21:18:04.456625+0000","last_change":"2023-09-23T21:13:12.200767+0000","last_active":"2023-09-23T21:18:04.456625+0000","last_peered":"2023-09-23T21:18:04.456625+0000","last_clean":"2023-09-23T21:18:04.456625+0000","last_became_active":"2023-09-23T21:13:12.200435+0000","last_became_peered":"2023-09-23T21:13:12.200435+0000","last_unstale":"2023-09-23T21:18:04.456625+0000","last_undegraded":"2023-09-23T21:18:04.456625+0000","last_fullsized":"2023-09-23T21:18:04.456625+0000","mapping_epoch":493,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:15:44.937886+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"11.17","version":"596'7352","reported_seq":7431,"reported_epoch":596,"state":"active+clean","last_fresh":"2023-09-23T21:18:07.046819+0000","last_change":"2023-09-23T21:18:06.404241+0000","last_active":"2023-09-23T21:18:07.046819+0000","last_peered":"2023-09-23T21:18:07.046819+0000","last_clean":"2023-09-23T21:18:07.046819+0000","last_became_active":"2023-09-23T21:16:32.890013+0000","last_became_peered":"2023-09-23T21:16:32.890013+0000","last_unstale":"2023-09-23T21:18:07.046819+0000","last_undegraded":"2023-09-23T21:18:07.046819+0000","last_fullsized":"2023-09-23T21:18:07.046819+0000","mapping_epoch":559,"log_start":"548'389","ondisk_log_start":"548'389","created":559,"last_epoch_clean":560,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_clean_scrub_stamp":"2023-09-23T21:15:24.860968+0000","objects_scrubbed":0,"log_size":6963,"log_dups_size":0,"ondisk_log_size":6963,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":79003648,"num_objects":1206,"num_object_clones":0,"num_object_copies":3618,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1206,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5048,"num_write_kb":200096,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1d","version":"79'161","reported_seq":729,"reported_epoch":596,"state":"active+clean","last_fresh":"2023-09-23T21:18:05.471507+0000","last_change":"2023-09-23T21:15:23.802125+0000","last_active":"2023-09-23T21:18:05.471507+0000","last_peered":"2023-09-23T21:18:05.471507+0000","last_clean":"2023-09-23T21:18:05.471507+0000","last_became_active":"2023-09-23T21:15:23.801704+0000","last_became_peered":"2023-09-23T21:15:23.801704+0000","last_unstale":"2023-09-23T21:18:05.471507+0000","last_undegraded":"2023-09-23T21:18:05.471507+0000","last_fullsized":"2023-09-23T21:18:05.471507+0000","mapping_epoch":542,"log_start":"0'0","ondisk_log_start":"0'0","created":479,"last_epoch_clean":543,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:09:36.858077+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:09:36.858077+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.010304610000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,2],"acting":[3,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":123,"reported_epoch":594,"state":"active+clean","last_fresh":"2023-09-23T21:17:58.396716+0000","last_change":"2023-09-23T21:13:38.380574+0000","last_active":"2023-09-23T21:17:58.396716+0000","last_peered":"2023-09-23T21:17:58.396716+0000","last_clean":"2023-09-23T21:17:58.396716+0000","last_became_active":"2023-09-23T21:13:38.373252+0000","last_became_peered":"2023-09-23T21:13:38.373252+0000","last_unstale":"2023-09-23T21:17:58.396716+0000","last_undegraded":"2023-09-23T21:17:58.396716+0000","last_fullsized":"2023-09-23T21:17:58.396716+0000","mapping_epoch":507,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:58:14.788031+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":11,"num_bytes_recovered":3324,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":170,"reported_epoch":595,"state":"active+clean","last_fresh":"2023-09-23T21:18:04.450387+0000","last_change":"2023-09-23T21:12:10.793610+0000","last_active":"2023-09-23T21:18:04.450387+0000","last_peered":"2023-09-23T21:18:04.450387+0000","last_clean":"2023-09-23T21:18:04.450387+0000","last_became_active":"2023-09-23T21:12:10.793081+0000","last_became_peered":"2023-09-23T21:12:10.793081+0000","last_unstale":"2023-09-23T21:18:04.450387+0000","last_undegraded":"2023-09-23T21:18:04.450387+0000","last_fullsized":"2023-09-23T21:18:04.450387+0000","mapping_epoch":470,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:34:36.291550+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":11,"num_pg":26,"stat_sum":{"num_bytes":2429485080,"num_objects":37072,"num_object_clones":0,"num_object_copies":111216,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1615,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":37072,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":158864,"num_write_kb":6269889,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":13560,"num_bytes_recovered":878247936,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":7908896768,"data_stored":7908884552,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":143826,"ondisk_log_size":143826,"up":78,"acting":75,"num_store_stats":6},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":80,"num_bytes_recovered":18686,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":118784,"data_stored":21350,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":14322,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":31,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":525376,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2109440,"data_stored":2098784,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4991,"ondisk_log_size":4991,"up":93,"acting":93,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201732,"num_pgs":68,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1728836,"kb_used_data":1123132,"kb_used_omap":0,"kb_used_meta":605696,"kb_avail":92016316,"statfs":{"total":95995035648,"available":94224707584,"internally_reserved":0,"allocated":1150087168,"data_stored":1148358012,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":620232704},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:17:19 2023","interfaces":[{"interface":"back","average":{"1min":1.002,"5min":1.655,"15min":2.169},"min":{"1min":0.22800000000000001,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":3.4689999999999999,"5min":37.207999999999998,"15min":37.207999999999998},"last":0.42199999999999999},{"interface":"front","average":{"1min":1.147,"5min":1.849,"15min":2.105},"min":{"1min":0.32200000000000001,"5min":0.253,"15min":0.253},"max":{"1min":3.702,"5min":37.122,"15min":37.122},"last":0.318}]},{"osd":1,"last update":"Sat Sep 23 21:17:13 2023","interfaces":[{"interface":"back","average":{"1min":1.0149999999999999,"5min":1.2030000000000001,"15min":1.8600000000000001},"min":{"1min":0.308,"5min":0.27500000000000002,"15min":0.26700000000000002},"max":{"1min":3.4279999999999999,"5min":7.5369999999999999,"15min":14.034000000000001},"last":0.39700000000000002},{"interface":"front","average":{"1min":1.2909999999999999,"5min":1.4850000000000001,"15min":1.8200000000000001},"min":{"1min":0.29999999999999999,"5min":0.29999999999999999,"15min":0.26300000000000001},"max":{"1min":5.4039999999999999,"5min":11.002000000000001,"15min":11.226000000000001},"last":0.88100000000000001}]},{"osd":2,"last update":"Sat Sep 23 21:17:37 2023","interfaces":[{"interface":"back","average":{"1min":2.9670000000000001,"5min":1.9039999999999999,"15min":1.825},"min":{"1min":0.372,"5min":0.26400000000000001,"15min":0.26200000000000001},"max":{"1min":25.471,"5min":25.471,"15min":25.471},"last":1.484},{"interface":"front","average":{"1min":3.073,"5min":1.73,"15min":1.7430000000000001},"min":{"1min":0.372,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":27.187999999999999,"5min":27.187999999999999,"15min":27.187999999999999},"last":0.78000000000000003}]},{"osd":3,"last update":"Sat Sep 23 21:18:04 2023","interfaces":[{"interface":"back","average":{"1min":1.7070000000000001,"5min":1.484,"15min":1.8280000000000001},"min":{"1min":0.33200000000000002,"5min":0.30599999999999999,"15min":0.29099999999999998},"max":{"1min":6.8739999999999997,"5min":8.6259999999999994,"15min":12.359999999999999},"last":1.5409999999999999},{"interface":"front","average":{"1min":1.1180000000000001,"5min":1.732,"15min":1.831},"min":{"1min":0.28799999999999998,"5min":0.28799999999999998,"15min":0.28699999999999998},"max":{"1min":3.8300000000000001,"5min":15.82,"15min":15.82},"last":0.95399999999999996}]},{"osd":4,"last update":"Sat Sep 23 21:18:04 2023","interfaces":[{"interface":"back","average":{"1min":0.93400000000000005,"5min":1.02,"15min":1.1799999999999999},"min":{"1min":0.251,"5min":0.251,"15min":0.248},"max":{"1min":3.9990000000000001,"5min":7.6609999999999996,"15min":11.297000000000001},"last":0.47299999999999998},{"interface":"front","average":{"1min":1.0089999999999999,"5min":1.073,"15min":1.137},"min":{"1min":0.28599999999999998,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":5.6349999999999998,"5min":8.6059999999999999,"15min":12.752000000000001},"last":0.28599999999999998}]},{"osd":5,"last update":"Sat Sep 23 21:18:04 2023","interfaces":[{"interface":"back","average":{"1min":1.286,"5min":1.095,"15min":1.089},"min":{"1min":0.28999999999999998,"5min":0.28999999999999998,"15min":0.26000000000000001},"max":{"1min":6.9740000000000002,"5min":7.5910000000000002,"15min":9.4789999999999992},"last":0.69699999999999995},{"interface":"front","average":{"1min":1.381,"5min":1.171,"15min":1.2130000000000001},"min":{"1min":0.35599999999999998,"5min":0.32000000000000001,"15min":0.26200000000000001},"max":{"1min":6.7439999999999998,"5min":7.6929999999999996,"15min":11.257},"last":0.503}]},{"osd":6,"last update":"Sat Sep 23 21:17:13 2023","interfaces":[{"interface":"back","average":{"1min":0.56200000000000006,"5min":1.236,"15min":1.081},"min":{"1min":0.32000000000000001,"5min":0.32000000000000001,"15min":0.30099999999999999},"max":{"1min":1.226,"5min":10.941000000000001,"15min":10.941000000000001},"last":0.44},{"interface":"front","average":{"1min":0.621,"5min":1.1719999999999999,"15min":1.161},"min":{"1min":0.36299999999999999,"5min":0.36299999999999999,"15min":0.32300000000000001},"max":{"1min":1.512,"5min":8.923,"15min":12.715},"last":0.41199999999999998}]}]},{"osd":6,"up_from":0,"seq":704374636816,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":1,"up_from":0,"seq":1503238553748,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":0,"up_from":493,"seq":2117418876988,"num_pgs":76,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2042572,"kb_used_data":1687236,"kb_used_omap":36,"kb_used_meta":355291,"kb_avail":91702580,"statfs":{"total":95995035648,"available":93903441920,"internally_reserved":0,"allocated":1727729664,"data_stored":1724948595,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":37336,"internal_metadata":363818536},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 21:17:23 2023","interfaces":[{"interface":"back","average":{"1min":2.0419999999999998,"5min":1.3200000000000001,"15min":0.94399999999999995},"min":{"1min":0.214,"5min":0.214,"15min":0.214},"max":{"1min":13.779999999999999,"5min":13.779999999999999,"15min":13.779999999999999},"last":6.7779999999999996},{"interface":"front","average":{"1min":2.0209999999999999,"5min":1.401,"15min":0.98099999999999998},"min":{"1min":0.32300000000000001,"5min":0.20799999999999999,"15min":0.20799999999999999},"max":{"1min":6.8810000000000002,"5min":11.964,"15min":11.964},"last":4.7300000000000004}]},{"osd":2,"last update":"Sat Sep 23 21:17:23 2023","interfaces":[{"interface":"back","average":{"1min":3.0510000000000002,"5min":1.52,"15min":0.94199999999999995},"min":{"1min":0.27500000000000002,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":13.637,"5min":13.637,"15min":13.637},"last":6.8140000000000001},{"interface":"front","average":{"1min":3.6240000000000001,"5min":2.0019999999999998,"15min":1.3120000000000001},"min":{"1min":0.40799999999999997,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":26.571000000000002,"5min":26.571000000000002,"15min":26.571000000000002},"last":4.8710000000000004}]},{"osd":3,"last update":"Sat Sep 23 21:17:23 2023","interfaces":[{"interface":"back","average":{"1min":4.0890000000000004,"5min":1.974,"15min":1.236},"min":{"1min":0.26900000000000002,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":33.634,"5min":33.634,"15min":33.634},"last":5.306},{"interface":"front","average":{"1min":2.5600000000000001,"5min":1.736,"15min":1.3240000000000001},"min":{"1min":0.30299999999999999,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":13.851000000000001,"5min":15.382999999999999,"15min":15.382999999999999},"last":7.2359999999999998}]},{"osd":4,"last update":"Sat Sep 23 21:17:23 2023","interfaces":[{"interface":"back","average":{"1min":2.7320000000000002,"5min":1.6479999999999999,"15min":1.278},"min":{"1min":0.34599999999999997,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":22.780000000000001,"5min":22.780000000000001,"15min":22.780000000000001},"last":4.7880000000000003},{"interface":"front","average":{"1min":2.2320000000000002,"5min":1.3100000000000001,"15min":1.1000000000000001},"min":{"1min":0.28499999999999998,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":22.652000000000001,"5min":22.652000000000001,"15min":22.652000000000001},"last":6.8390000000000004}]},{"osd":5,"last update":"Sat Sep 23 21:17:23 2023","interfaces":[{"interface":"back","average":{"1min":1.958,"5min":1.504,"15min":1.325},"min":{"1min":0.312,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":7.8630000000000004,"5min":8.9329999999999998,"15min":8.9329999999999998},"last":4.694},{"interface":"front","average":{"1min":1.825,"5min":1.528,"15min":1.218},"min":{"1min":0.40300000000000002,"5min":0.313,"15min":0.313},"max":{"1min":6.577,"5min":15.321999999999999,"15min":15.321999999999999},"last":4.9459999999999997}]},{"osd":6,"last update":"Sat Sep 23 21:17:23 2023","interfaces":[{"interface":"back","average":{"1min":1.4430000000000001,"5min":1.3919999999999999,"15min":1.2150000000000001},"min":{"1min":0.36599999999999999,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":6.6449999999999996,"5min":7.2050000000000001,"15min":7.2050000000000001},"last":6.9409999999999998},{"interface":"front","average":{"1min":1.6990000000000001,"5min":1.5269999999999999,"15min":1.381},"min":{"1min":0.38900000000000001,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":5.5,"5min":8.8889999999999993,"15min":8.8889999999999993},"last":4.8170000000000002}]},{"osd":7,"last update":"Sat Sep 23 21:17:23 2023","interfaces":[{"interface":"back","average":{"1min":1.9950000000000001,"5min":1.7470000000000001,"15min":1.3879999999999999},"min":{"1min":0.434,"5min":0.40899999999999997,"15min":0.40899999999999997},"max":{"1min":6.649,"5min":15.273999999999999,"15min":15.273999999999999},"last":4.8360000000000003},{"interface":"front","average":{"1min":2.996,"5min":1.6519999999999999,"15min":1.3220000000000001},"min":{"1min":0.36699999999999999,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":33.779000000000003,"5min":33.779000000000003,"15min":33.779000000000003},"last":6.9109999999999996}]}]},{"osd":2,"up_from":93,"seq":399431958860,"num_pgs":60,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2489528,"kb_used_data":1497840,"kb_used_omap":81,"kb_used_meta":991470,"kb_avail":91255624,"statfs":{"total":95995035648,"available":93445758976,"internally_reserved":0,"allocated":1533788160,"data_stored":1530971170,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":83750,"internal_metadata":1015265498},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:17:23 2023","interfaces":[{"interface":"back","average":{"1min":1.952,"5min":1.577,"15min":1.081},"min":{"1min":0.187,"5min":0.187,"15min":0.187},"max":{"1min":13.25,"5min":13.25,"15min":13.25},"last":6.2530000000000001},{"interface":"front","average":{"1min":2.266,"5min":1.972,"15min":1.7669999999999999},"min":{"1min":0.28199999999999997,"5min":0.20200000000000001,"15min":0.20200000000000001},"max":{"1min":9.4169999999999998,"5min":18.111999999999998,"15min":18.111999999999998},"last":0.82199999999999995}]},{"osd":1,"last update":"Sat Sep 23 21:17:23 2023","interfaces":[{"interface":"back","average":{"1min":2.7349999999999999,"5min":1.48,"15min":2.351},"min":{"1min":0.33300000000000002,"5min":0.188,"15min":0.188},"max":{"1min":8.9000000000000004,"5min":9.9890000000000008,"15min":34.396999999999998},"last":0.53700000000000003},{"interface":"front","average":{"1min":2.117,"5min":1.5669999999999999,"15min":2.4430000000000001},"min":{"1min":0.27500000000000002,"5min":0.217,"15min":0.217},"max":{"1min":8.5510000000000002,"5min":18.081,"15min":34.462000000000003},"last":0.52300000000000002}]},{"osd":3,"last update":"Sat Sep 23 21:18:01 2023","interfaces":[{"interface":"back","average":{"1min":2.0880000000000001,"5min":2.04,"15min":2.2389999999999999},"min":{"1min":0.248,"5min":0.20599999999999999,"15min":0.20599999999999999},"max":{"1min":7.3719999999999999,"5min":20.106999999999999,"15min":35.549999999999997},"last":0.48399999999999999},{"interface":"front","average":{"1min":2.6659999999999999,"5min":2.556,"15min":2.6099999999999999},"min":{"1min":0.35799999999999998,"5min":0.28499999999999998,"15min":0.21299999999999999},"max":{"1min":10.162000000000001,"5min":20.239000000000001,"15min":36.951999999999998},"last":0.59999999999999998}]},{"osd":4,"last update":"Sat Sep 23 21:17:23 2023","interfaces":[{"interface":"back","average":{"1min":1.627,"5min":1.409,"15min":1.448},"min":{"1min":0.29599999999999999,"5min":0.29599999999999999,"15min":0.249},"max":{"1min":5.6369999999999996,"5min":10.266999999999999,"15min":10.266999999999999},"last":0.41699999999999998},{"interface":"front","average":{"1min":2.379,"5min":1.587,"15min":1.7949999999999999},"min":{"1min":0.30299999999999999,"5min":0.29399999999999998,"15min":0.26700000000000002},"max":{"1min":10.459,"5min":10.459,"15min":12.49},"last":0.57399999999999995}]},{"osd":5,"last update":"Sat Sep 23 21:17:23 2023","interfaces":[{"interface":"back","average":{"1min":1.915,"5min":1.3460000000000001,"15min":1.5569999999999999},"min":{"1min":0.41199999999999998,"5min":0.29099999999999998,"15min":0.28000000000000003},"max":{"1min":5.7089999999999996,"5min":7.6180000000000003,"15min":9.5039999999999996},"last":0.93500000000000005},{"interface":"front","average":{"1min":2.3809999999999998,"5min":1.417,"15min":1.746},"min":{"1min":0.378,"5min":0.29699999999999999,"15min":0.29399999999999998},"max":{"1min":10.425000000000001,"5min":10.425000000000001,"15min":12.57},"last":0.63600000000000001}]},{"osd":6,"last update":"Sat Sep 23 21:17:23 2023","interfaces":[{"interface":"back","average":{"1min":2.3069999999999999,"5min":1.6839999999999999,"15min":1.9570000000000001},"min":{"1min":0.42899999999999999,"5min":0.373,"15min":0.35699999999999998},"max":{"1min":10.335000000000001,"5min":10.335000000000001,"15min":12.518000000000001},"last":1.036},{"interface":"front","average":{"1min":2.4169999999999998,"5min":1.831,"15min":2.1019999999999999},"min":{"1min":0.44500000000000001,"5min":0.377,"15min":0.35599999999999998},"max":{"1min":10.381,"5min":10.381,"15min":12.544},"last":0.97799999999999998}]},{"osd":7,"last update":"Sat Sep 23 21:17:23 2023","interfaces":[{"interface":"back","average":{"1min":1.6220000000000001,"5min":1.6319999999999999,"15min":1.613},"min":{"1min":0.39000000000000001,"5min":0.33700000000000002,"15min":0.29799999999999999},"max":{"1min":7.266,"5min":18.021999999999998,"15min":18.021999999999998},"last":0.69499999999999995},{"interface":"front","average":{"1min":1.702,"5min":1.615,"15min":1.714},"min":{"1min":0.434,"5min":0.34799999999999998,"15min":0.32200000000000001},"max":{"1min":5.7329999999999997,"5min":10.359,"15min":10.92},"last":1.4139999999999999}]}]},{"osd":3,"up_from":98,"seq":420906795334,"num_pgs":85,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2576784,"kb_used_data":1520072,"kb_used_omap":7,"kb_used_meta":1056696,"kb_avail":91168368,"statfs":{"total":95995035648,"available":93356408832,"internally_reserved":0,"allocated":1556553728,"data_stored":1553778342,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7576,"internal_metadata":1082057320},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:17:17 2023","interfaces":[{"interface":"back","average":{"1min":5.2619999999999996,"5min":3.7010000000000001,"15min":1.4590000000000001},"min":{"1min":0.18099999999999999,"5min":0.18099999999999999,"15min":0.18099999999999999},"max":{"1min":81.408000000000001,"5min":147.38900000000001,"15min":147.38900000000001},"last":0.22500000000000001},{"interface":"front","average":{"1min":4.4119999999999999,"5min":3.4809999999999999,"15min":1.605},"min":{"1min":0.223,"5min":0.223,"15min":0.223},"max":{"1min":82.397000000000006,"5min":149.989,"15min":149.989},"last":0.38500000000000001}]},{"osd":1,"last update":"Sat Sep 23 21:17:17 2023","interfaces":[{"interface":"back","average":{"1min":1.7030000000000001,"5min":1.234,"15min":2.1120000000000001},"min":{"1min":0.29399999999999998,"5min":0.20000000000000001,"15min":0.20000000000000001},"max":{"1min":7.2759999999999998,"5min":7.2759999999999998,"15min":22.577999999999999},"last":4.4340000000000002},{"interface":"front","average":{"1min":1.1000000000000001,"5min":0.998,"15min":2.0099999999999998},"min":{"1min":0.255,"5min":0.216,"15min":0.216},"max":{"1min":3.5339999999999998,"5min":7.6829999999999998,"15min":21.617999999999999},"last":5.109}]},{"osd":2,"last update":"Sat Sep 23 21:18:02 2023","interfaces":[{"interface":"back","average":{"1min":1.4750000000000001,"5min":1.958,"15min":2.004},"min":{"1min":0.27400000000000002,"5min":0.221,"15min":0.20899999999999999},"max":{"1min":11.321,"5min":56.317999999999998,"15min":56.317999999999998},"last":0.55800000000000005},{"interface":"front","average":{"1min":1.9910000000000001,"5min":2.4780000000000002,"15min":2.355},"min":{"1min":0.23400000000000001,"5min":0.23400000000000001,"15min":0.216},"max":{"1min":11.371,"5min":55.988,"15min":55.988},"last":10.398999999999999}]},{"osd":4,"last update":"Sat Sep 23 21:18:02 2023","interfaces":[{"interface":"back","average":{"1min":0.72899999999999998,"5min":1.2010000000000001,"15min":1.5509999999999999},"min":{"1min":0.29199999999999998,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":2.5409999999999999,"5min":10.807,"15min":21.478999999999999},"last":3.0750000000000002},{"interface":"front","average":{"1min":1.091,"5min":1.3180000000000001,"15min":1.7829999999999999},"min":{"1min":0.35899999999999999,"5min":0.311,"15min":0.28499999999999998},"max":{"1min":3.129,"5min":7.3140000000000001,"15min":22.402000000000001},"last":1.52}]},{"osd":5,"last update":"Sat Sep 23 21:18:02 2023","interfaces":[{"interface":"back","average":{"1min":0.83199999999999996,"5min":1.3160000000000001,"15min":1.6930000000000001},"min":{"1min":0.318,"5min":0.313,"15min":0.311},"max":{"1min":4.1379999999999999,"5min":10.882999999999999,"15min":21.562000000000001},"last":3.109},{"interface":"front","average":{"1min":1.571,"5min":1.234,"15min":1.8},"min":{"1min":0.42699999999999999,"5min":0.34300000000000003,"15min":0.31},"max":{"1min":5.7300000000000004,"5min":5.7300000000000004,"15min":14.029},"last":0.33600000000000002}]},{"osd":6,"last update":"Sat Sep 23 21:17:18 2023","interfaces":[{"interface":"back","average":{"1min":1.48,"5min":1.4359999999999999,"15min":1.9059999999999999},"min":{"1min":0.42899999999999999,"5min":0.34999999999999998,"15min":0.33900000000000002},"max":{"1min":7.3410000000000002,"5min":10.102,"15min":22.446999999999999},"last":1.476},{"interface":"front","average":{"1min":1.2549999999999999,"5min":1.4319999999999999,"15min":1.9019999999999999},"min":{"1min":0.45800000000000002,"5min":0.378,"15min":0.373},"max":{"1min":5.5510000000000002,"5min":10.148,"15min":12.375999999999999},"last":0.373}]},{"osd":7,"last update":"Sat Sep 23 21:18:02 2023","interfaces":[{"interface":"back","average":{"1min":1.0249999999999999,"5min":1.48,"15min":1.847},"min":{"1min":0.41299999999999998,"5min":0.41199999999999998,"15min":0.33900000000000002},"max":{"1min":4.242,"5min":10.914999999999999,"15min":21.518999999999998},"last":1.0569999999999999},{"interface":"front","average":{"1min":1.5409999999999999,"5min":1.528,"15min":1.9850000000000001},"min":{"1min":0.51800000000000002,"5min":0.42499999999999999,"15min":0.40300000000000002},"max":{"1min":4.6920000000000002,"5min":8.2590000000000003,"15min":12.624000000000001},"last":0.92000000000000004}]}]},{"osd":4,"up_from":31,"seq":133143986640,"num_pgs":62,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1514756,"kb_used_data":960764,"kb_used_omap":0,"kb_used_meta":553984,"kb_avail":92230396,"statfs":{"total":95995035648,"available":94443925504,"internally_reserved":0,"allocated":983822336,"data_stored":982092913,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":567279616},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:17:21 2023","interfaces":[{"interface":"back","average":{"1min":2.508,"5min":1.399,"15min":1.0660000000000001},"min":{"1min":0.252,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":28.683,"5min":28.683,"15min":28.683},"last":4.2649999999999997},{"interface":"front","average":{"1min":2.839,"5min":1.411,"15min":1.226},"min":{"1min":0.27600000000000002,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":29.155000000000001,"5min":29.155000000000001,"15min":29.155000000000001},"last":4.1840000000000002}]},{"osd":1,"last update":"Sat Sep 23 21:17:21 2023","interfaces":[{"interface":"back","average":{"1min":1.821,"5min":1.214,"15min":2.246},"min":{"1min":0.312,"5min":0.251,"15min":0.221},"max":{"1min":5.6319999999999997,"5min":7.8609999999999998,"15min":37.960000000000001},"last":0.39000000000000001},{"interface":"front","average":{"1min":1.637,"5min":1.3049999999999999,"15min":2.1930000000000001},"min":{"1min":0.374,"5min":0.28499999999999998,"15min":0.27000000000000002},"max":{"1min":5.6130000000000004,"5min":13.983000000000001,"15min":37.933},"last":0.40200000000000002}]},{"osd":2,"last update":"Sat Sep 23 21:17:21 2023","interfaces":[{"interface":"back","average":{"1min":2.6659999999999999,"5min":1.6419999999999999,"15min":1.7529999999999999},"min":{"1min":0.33300000000000002,"5min":0.26000000000000001,"15min":0.247},"max":{"1min":17.602,"5min":17.602,"15min":17.602},"last":5.8879999999999999},{"interface":"front","average":{"1min":1.79,"5min":1.669,"15min":1.806},"min":{"1min":0.39400000000000002,"5min":0.28599999999999998,"15min":0.27800000000000002},"max":{"1min":6.0720000000000001,"5min":14.339,"15min":14.417999999999999},"last":0.83399999999999996}]},{"osd":3,"last update":"Sat Sep 23 21:17:52 2023","interfaces":[{"interface":"back","average":{"1min":5.2910000000000004,"5min":2.1000000000000001,"15min":2.1299999999999999},"min":{"1min":0.312,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":67.105999999999995,"5min":67.105999999999995,"15min":67.105999999999995},"last":0.60999999999999999},{"interface":"front","average":{"1min":4.9290000000000003,"5min":2.1930000000000001,"15min":2.056},"min":{"1min":0.33600000000000002,"5min":0.32200000000000001,"15min":0.30199999999999999},"max":{"1min":67.046999999999997,"5min":67.046999999999997,"15min":67.046999999999997},"last":0.60499999999999998}]},{"osd":5,"last update":"Sat Sep 23 21:17:08 2023","interfaces":[{"interface":"back","average":{"1min":0.77100000000000002,"5min":1.3520000000000001,"15min":1.127},"min":{"1min":0.27700000000000002,"5min":0.222,"15min":0.222},"max":{"1min":2.637,"5min":14.398999999999999,"15min":14.398999999999999},"last":0.308},{"interface":"front","average":{"1min":0.58099999999999996,"5min":1.1639999999999999,"15min":1.1080000000000001},"min":{"1min":0.25800000000000001,"5min":0.25800000000000001,"15min":0.255},"max":{"1min":3.1600000000000001,"5min":8.016,"15min":9.8979999999999997},"last":0.28899999999999998}]},{"osd":6,"last update":"Sat Sep 23 21:17:21 2023","interfaces":[{"interface":"back","average":{"1min":0.91000000000000003,"5min":1.335,"15min":1.3069999999999999},"min":{"1min":0.27200000000000002,"5min":0.27200000000000002,"15min":0.26800000000000002},"max":{"1min":4.7270000000000003,"5min":24.452999999999999,"15min":24.452999999999999},"last":0.34100000000000003},{"interface":"front","average":{"1min":0.80800000000000005,"5min":1.3200000000000001,"15min":1.095},"min":{"1min":0.36399999999999999,"5min":0.36399999999999999,"15min":0.32500000000000001},"max":{"1min":2.7069999999999999,"5min":14.374000000000001,"15min":14.387},"last":0.38}]},{"osd":7,"last update":"Sat Sep 23 21:17:52 2023","interfaces":[{"interface":"back","average":{"1min":1.091,"5min":1.194,"15min":1.1919999999999999},"min":{"1min":0.33200000000000002,"5min":0.32000000000000001,"15min":0.30099999999999999},"max":{"1min":5.7789999999999999,"5min":14.422000000000001,"15min":16.890999999999998},"last":0.35699999999999998},{"interface":"front","average":{"1min":0.73399999999999999,"5min":1.034,"15min":1.2230000000000001},"min":{"1min":0.29199999999999998,"5min":0.29199999999999998,"15min":0.27400000000000002},"max":{"1min":3.681,"5min":7.7229999999999999,"15min":11.170999999999999},"last":0.32900000000000001}]}]},{"osd":5,"up_from":39,"seq":167503725006,"num_pgs":61,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1643576,"kb_used_data":1051312,"kb_used_omap":0,"kb_used_meta":592256,"kb_avail":92101576,"statfs":{"total":95995035648,"available":94312013824,"internally_reserved":0,"allocated":1076543488,"data_stored":1074822641,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":606470144},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:17:14 2023","interfaces":[{"interface":"back","average":{"1min":1.482,"5min":1.0860000000000001,"15min":0.83799999999999997},"min":{"1min":0.23100000000000001,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":5.9569999999999999,"5min":8.673,"15min":8.673},"last":0.312},{"interface":"front","average":{"1min":1.3460000000000001,"5min":1.177,"15min":0.83799999999999997},"min":{"1min":0.30299999999999999,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":3.339,"5min":9.7279999999999998,"15min":9.7279999999999998},"last":2.516}]},{"osd":1,"last update":"Sat Sep 23 21:17:14 2023","interfaces":[{"interface":"back","average":{"1min":1.9079999999999999,"5min":1.0840000000000001,"15min":1.6479999999999999},"min":{"1min":0.33200000000000002,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":4.8200000000000003,"5min":7.5839999999999996,"15min":25.239999999999998},"last":2.4329999999999998},{"interface":"front","average":{"1min":1.4410000000000001,"5min":1.351,"15min":1.7190000000000001},"min":{"1min":0.34699999999999998,"5min":0.28299999999999997,"15min":0.252},"max":{"1min":4.8049999999999997,"5min":11.116,"15min":25.253},"last":2.4249999999999998}]},{"osd":2,"last update":"Sat Sep 23 21:17:32 2023","interfaces":[{"interface":"back","average":{"1min":1.615,"5min":1.506,"15min":1.6439999999999999},"min":{"1min":0.27300000000000002,"5min":0.27300000000000002,"15min":0.25800000000000001},"max":{"1min":7.3550000000000004,"5min":9.6560000000000006,"15min":12.634},"last":0.60199999999999998},{"interface":"front","average":{"1min":1.9410000000000001,"5min":1.327,"15min":1.6579999999999999},"min":{"1min":0.314,"5min":0.27600000000000002,"15min":0.25700000000000001},"max":{"1min":9.1630000000000003,"5min":9.1630000000000003,"15min":9.7650000000000006},"last":2.5750000000000002}]},{"osd":3,"last update":"Sat Sep 23 21:17:52 2023","interfaces":[{"interface":"back","average":{"1min":1.4259999999999999,"5min":1.4570000000000001,"15min":1.931},"min":{"1min":0.314,"5min":0.314,"15min":0.29299999999999998},"max":{"1min":3.6030000000000002,"5min":6.5940000000000003,"15min":25.503},"last":2.3959999999999999},{"interface":"front","average":{"1min":1.7310000000000001,"5min":1.748,"15min":2.157},"min":{"1min":0.311,"5min":0.311,"15min":0.30599999999999999},"max":{"1min":9.0619999999999994,"5min":12.382,"15min":25.466000000000001},"last":2.3330000000000002}]},{"osd":4,"last update":"Sat Sep 23 21:17:08 2023","interfaces":[{"interface":"back","average":{"1min":0.73699999999999999,"5min":1.4890000000000001,"15min":1.1639999999999999},"min":{"1min":0.28799999999999998,"5min":0.253,"15min":0.253},"max":{"1min":3.2770000000000001,"5min":12.776,"15min":12.776},"last":0.38300000000000001},{"interface":"front","average":{"1min":0.91800000000000004,"5min":1.405,"15min":1.1779999999999999},"min":{"1min":0.246,"5min":0.246,"15min":0.246},"max":{"1min":5.149,"5min":9.7119999999999997,"15min":11.973000000000001},"last":0.56399999999999995}]},{"osd":6,"last update":"Sat Sep 23 21:17:14 2023","interfaces":[{"interface":"back","average":{"1min":1.556,"5min":1.147,"15min":1.1160000000000001},"min":{"1min":0.28199999999999997,"5min":0.28000000000000003,"15min":0.25900000000000001},"max":{"1min":4.3490000000000002,"5min":7.7030000000000003,"15min":11.128},"last":0.495},{"interface":"front","average":{"1min":1.208,"5min":1.452,"15min":1.268},"min":{"1min":0.39400000000000002,"5min":0.34300000000000003,"15min":0.317},"max":{"1min":4.367,"5min":10.999000000000001,"15min":11.112},"last":0.54100000000000004}]},{"osd":7,"last update":"Sat Sep 23 21:17:52 2023","interfaces":[{"interface":"back","average":{"1min":1.109,"5min":1.012,"15min":1.819},"min":{"1min":0.36199999999999999,"5min":0.32400000000000001,"15min":0.309},"max":{"1min":4.2629999999999999,"5min":5.5659999999999998,"15min":189.405},"last":0.45700000000000002},{"interface":"front","average":{"1min":1.0649999999999999,"5min":1.1719999999999999,"15min":1.9670000000000001},"min":{"1min":0.39900000000000002,"5min":0.35199999999999998,"15min":0.32500000000000001},"max":{"1min":2.7879999999999998,"5min":11.183999999999999,"15min":189.685},"last":0.47799999999999998}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8488,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1259,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5322,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":4838,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1719795712,"data_stored":1719795712,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1497436160,"data_stored":1497432088,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1541799936,"data_stored":1541799936,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":963514368,"data_stored":963510296,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1052315648,"data_stored":1052311576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1134034944,"data_stored":1134034944,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T21:18:11.554 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T21:18:11.568 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:18:11.635 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:18:11.671 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:18:11.714 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:18:11.717 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:18:11.718 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:18:11.792 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:18:11 smithi026 ceph-mon[179688]: from='client.56764 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T21:18:11.793 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:18:11 smithi026 ceph-mon[179688]: failed to encode map e599 with expected crc 2023-09-23T21:18:11.793 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:18:11 smithi026 ceph-mon[179688]: osdmap e599: 8 total, 8 up, 6 in 2023-09-23T21:18:11.793 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:18:11 smithi026 ceph-mon[179688]: failed to encode map e599 with expected crc 2023-09-23T21:18:11.793 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:18:11 smithi026 ceph-mon[179688]: failed to encode map e599 with expected crc 2023-09-23T21:18:11.793 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:18:11 smithi026 ceph-mon[179688]: Health check update: Degraded data redundancy: 1772/114507 objects degraded (1.548%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T21:18:11.793 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:18:11 smithi026 ceph-mon[175151]: from='client.56764 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T21:18:11.793 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:18:11 smithi026 ceph-mon[175151]: failed to encode map e599 with expected crc 2023-09-23T21:18:11.793 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:18:11 smithi026 ceph-mon[175151]: osdmap e599: 8 total, 8 up, 6 in 2023-09-23T21:18:11.793 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:18:11 smithi026 ceph-mon[175151]: failed to encode map e599 with expected crc 2023-09-23T21:18:11.794 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:18:11 smithi026 ceph-mon[175151]: failed to encode map e599 with expected crc 2023-09-23T21:18:11.794 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:18:11 smithi026 ceph-mon[175151]: Health check update: Degraded data redundancy: 1772/114507 objects degraded (1.548%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T21:18:11.819 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:18:11.847 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:18:11 smithi070 ceph-mon[168282]: from='client.56764 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T21:18:11.848 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:18:11 smithi070 ceph-mon[168282]: failed to encode map e599 with expected crc 2023-09-23T21:18:11.848 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:18:11 smithi070 ceph-mon[168282]: osdmap e599: 8 total, 8 up, 6 in 2023-09-23T21:18:11.848 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:18:11 smithi070 ceph-mon[168282]: failed to encode map e599 with expected crc 2023-09-23T21:18:11.848 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:18:11 smithi070 ceph-mon[168282]: failed to encode map e599 with expected crc 2023-09-23T21:18:11.848 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:18:11 smithi070 ceph-mon[168282]: Health check update: Degraded data redundancy: 1772/114507 objects degraded (1.548%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T21:18:11.850 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:18:11.893 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:18:11.897 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:18:11.898 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:18:11.998 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:18:12.033 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:18:12.077 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:18:12.079 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:18:12.079 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:18:12.180 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:18:12.258 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:18:12.344 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T21:18:12.354 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:18:12.354 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:18:12.455 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:18:12.495 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:18:12.539 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:18:12.547 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:18:12.547 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:18:12.648 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:18:12.732 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:18:12.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:18:12 smithi070 ceph-mon[168282]: pgmap v1510: 129 pgs: 1 active+recovery_wait, 1 active+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 125 active+clean; 2.2 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 742 op/s; 1615/109698 objects degraded (1.472%); 9.2 MiB/s, 173 objects/s recovering 2023-09-23T21:18:12.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:18:12 smithi070 ceph-mon[168282]: failed to encode map e599 with expected crc 2023-09-23T21:18:12.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:18:12 smithi026 ceph-mon[179688]: pgmap v1510: 129 pgs: 1 active+recovery_wait, 1 active+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 125 active+clean; 2.2 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 742 op/s; 1615/109698 objects degraded (1.472%); 9.2 MiB/s, 173 objects/s recovering 2023-09-23T21:18:12.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:18:12 smithi026 ceph-mon[179688]: failed to encode map e599 with expected crc 2023-09-23T21:18:12.883 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:18:12 smithi026 ceph-mon[175151]: pgmap v1510: 129 pgs: 1 active+recovery_wait, 1 active+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 125 active+clean; 2.2 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 742 op/s; 1615/109698 objects degraded (1.472%); 9.2 MiB/s, 173 objects/s recovering 2023-09-23T21:18:12.883 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:18:12 smithi026 ceph-mon[175151]: failed to encode map e599 with expected crc 2023-09-23T21:18:12.960 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T21:18:12.973 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:18:12.974 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:18:13.075 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:18:13.170 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:18:13.212 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0: {} 2023-09-23T21:18:13.251 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1: {} 2023-09-23T21:18:13.252 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:18:13.260 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:18:13.261 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:18:13.270 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:18:13.291 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2: {} 2023-09-23T21:18:13.336 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:18:13.340 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3: {} 2023-09-23T21:18:13.362 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:18:13.374 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4: {} 2023-09-23T21:18:13.422 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T21:18:13.448 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:18:13.469 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6: {} 2023-09-23T21:18:13.511 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T21:18:13.513 INFO:teuthology.orchestra.run.smithi026.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T21:18:13.514 INFO:teuthology.orchestra.run.smithi026.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T21:18:13.514 INFO:teuthology.orchestra.run.smithi026.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T21:18:13.514 INFO:teuthology.orchestra.run.smithi026.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T21:18:13.514 INFO:teuthology.orchestra.run.smithi026.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T21:18:13.514 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T21:18:13.514 INFO:teuthology.orchestra.run.smithi026.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T21:18:13.514 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T21:18:13.552 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:18:13 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/431517964' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:18:13.650 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:18:13.656 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:18:13.657 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:18:13.758 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:18:13.826 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:18:13 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/431517964' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:18:13.834 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:18:13.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:18:13 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/431517964' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:18:14.136 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T21:18:14.140 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:18:14.141 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:18:14.241 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:18:14.278 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:18:14.323 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:18:14.326 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:18:14.326 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:18:14.427 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:18:14.502 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:18:14.816 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:18:14 smithi026 ceph-mon[175151]: pgmap v1511: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 1 active+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 125 active+clean; 2.1 GiB data, 11 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 799 op/s; 1119/102033 objects degraded (1.097%); 170/102033 objects misplaced (0.167%); 12 MiB/s, 220 objects/s recovering 2023-09-23T21:18:14.816 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:18:14 smithi026 ceph-mon[179688]: pgmap v1511: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 1 active+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 125 active+clean; 2.1 GiB data, 11 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 799 op/s; 1119/102033 objects degraded (1.097%); 170/102033 objects misplaced (0.167%); 12 MiB/s, 220 objects/s recovering 2023-09-23T21:18:14.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:18:14 smithi070 ceph-mon[168282]: pgmap v1511: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 1 active+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 125 active+clean; 2.1 GiB data, 11 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 799 op/s; 1119/102033 objects degraded (1.097%); 170/102033 objects misplaced (0.167%); 12 MiB/s, 220 objects/s recovering 2023-09-23T21:18:15.015 INFO:teuthology.orchestra.run.smithi026.stdout:{ 2023-09-23T21:18:15.015 INFO:teuthology.orchestra.run.smithi026.stdout: "ops": [], 2023-09-23T21:18:15.015 INFO:teuthology.orchestra.run.smithi026.stdout: "num_ops": 0 2023-09-23T21:18:15.015 INFO:teuthology.orchestra.run.smithi026.stdout:} 2023-09-23T21:18:15.045 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T21:18:15.053 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:18:15.053 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:18:15.154 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:18:15.217 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:18:15.292 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:18:15.297 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:18:15.297 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:18:15.398 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:18:15.482 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:18:15.588 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:18:15.601 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:18:15.602 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:18:15.633 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_blocked_ops 2023-09-23T21:18:15.703 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:18:15.736 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:18:15.794 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:18:15.797 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:18:15.797 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:18:15.898 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:18:15.930 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:18:15.972 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:18:15.976 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:18:15.977 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:18:16.078 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:18:16.162 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:18:16.181 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T21:18:16.256 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:18:16.264 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:18:16.265 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:18:16.366 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:18:16.394 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:18:16.436 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:18:16.441 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:18:16.441 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:18:16.542 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:18:16.563 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:18:16 smithi026 ceph-mon[179688]: pgmap v1512: 129 pgs: 1 active+recovery_wait, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 126 active+clean; 2.0 GiB data, 11 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 782 op/s; 1119/98772 objects degraded (1.133%); 170/98772 objects misplaced (0.172%); 8.5 MiB/s, 159 objects/s recovering 2023-09-23T21:18:16.564 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:18:16 smithi026 ceph-mon[179688]: Health check update: Degraded data redundancy: 1119/98772 objects degraded (1.133%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T21:18:16.564 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:18:16 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:18:16.564 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:18:16 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:18:16.564 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:18:16 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/2861733547' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:18:16.564 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:18:16 smithi026 ceph-mon[175151]: pgmap v1512: 129 pgs: 1 active+recovery_wait, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 126 active+clean; 2.0 GiB data, 11 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 782 op/s; 1119/98772 objects degraded (1.133%); 170/98772 objects misplaced (0.172%); 8.5 MiB/s, 159 objects/s recovering 2023-09-23T21:18:16.564 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:18:16 smithi026 ceph-mon[175151]: Health check update: Degraded data redundancy: 1119/98772 objects degraded (1.133%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T21:18:16.564 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:18:16 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:18:16.564 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:18:16 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:18:16.565 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:18:16 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/2861733547' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:18:16.573 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:18:16.613 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:18:16.616 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:18:16.617 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:18:16.686 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:18:16.717 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:18:16.735 INFO:teuthology.orchestra.run.smithi026.stdout: 2023-09-23T21:18:16.779 INFO:teuthology.orchestra.run.smithi026.stderr:dumped all 2023-09-23T21:18:16.790 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:18:16.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:18:16 smithi070 ceph-mon[168282]: pgmap v1512: 129 pgs: 1 active+recovery_wait, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 126 active+clean; 2.0 GiB data, 11 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 782 op/s; 1119/98772 objects degraded (1.133%); 170/98772 objects misplaced (0.172%); 8.5 MiB/s, 159 objects/s recovering 2023-09-23T21:18:16.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:18:16 smithi070 ceph-mon[168282]: Health check update: Degraded data redundancy: 1119/98772 objects degraded (1.133%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T21:18:16.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:18:16 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:18:16.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:18:16 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:18:16.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:18:16 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/2861733547' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:18:16.914 INFO:teuthology.orchestra.run.smithi026.stderr:nodeep-scrub is set 2023-09-23T21:18:16.946 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T21:18:16.956 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:18:16.956 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:18:17.057 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:18:17.086 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:18:17.128 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:18:17.131 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:18:17.131 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:18:17.232 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:18:17.262 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:18:17.305 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:18:17.309 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:18:17.309 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:18:17.410 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:18:17.441 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:18:17.479 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:18:17.482 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:18:17.483 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:18:17.583 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:18:17.610 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:18:17.646 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:18:17.649 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:18:17.649 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:18:17.750 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:18:17.782 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:18:17.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:18:17 smithi070 ceph-mon[168282]: from='client.56773 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T21:18:17.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:18:17 smithi070 ceph-mon[168282]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:18:17.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:18:17 smithi070 ceph-mon[168282]: failed to encode map e600 with expected crc 2023-09-23T21:18:17.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:18:17 smithi070 ceph-mon[168282]: failed to encode map e600 with expected crc 2023-09-23T21:18:17.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:18:17 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/2861733547' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T21:18:17.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:18:17 smithi070 ceph-mon[168282]: osdmap e600: 8 total, 8 up, 6 in 2023-09-23T21:18:17.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:18:17 smithi070 ceph-mon[168282]: failed to encode map e600 with expected crc 2023-09-23T21:18:17.881 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:18:17 smithi026 ceph-mon[175151]: from='client.56773 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T21:18:17.881 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:18:17 smithi026 ceph-mon[175151]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:18:17.881 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:18:17 smithi026 ceph-mon[175151]: failed to encode map e600 with expected crc 2023-09-23T21:18:17.881 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:18:17 smithi026 ceph-mon[175151]: failed to encode map e600 with expected crc 2023-09-23T21:18:17.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:18:17 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/2861733547' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T21:18:17.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:18:17 smithi026 ceph-mon[175151]: osdmap e600: 8 total, 8 up, 6 in 2023-09-23T21:18:17.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:18:17 smithi026 ceph-mon[175151]: failed to encode map e600 with expected crc 2023-09-23T21:18:17.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:18:17 smithi026 ceph-mon[179688]: from='client.56773 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T21:18:17.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:18:17 smithi026 ceph-mon[179688]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:18:17.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:18:17 smithi026 ceph-mon[179688]: failed to encode map e600 with expected crc 2023-09-23T21:18:17.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:18:17 smithi026 ceph-mon[179688]: failed to encode map e600 with expected crc 2023-09-23T21:18:17.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:18:17 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/2861733547' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T21:18:17.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:18:17 smithi026 ceph-mon[179688]: osdmap e600: 8 total, 8 up, 6 in 2023-09-23T21:18:17.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:18:17 smithi026 ceph-mon[179688]: failed to encode map e600 with expected crc 2023-09-23T21:18:18.830 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:18:18.833 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:18:18 smithi026 ceph-mon[175151]: pgmap v1514: 129 pgs: 1 active+undersized+remapped, 1 active+recovery_wait, 1 active+recovery_wait+undersized+degraded+remapped, 126 active+clean; 1.8 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 908 op/s; 812/90348 objects degraded (0.899%); 4.0 MiB/s, 104 objects/s recovering 2023-09-23T21:18:18.833 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:18:18 smithi026 ceph-mon[175151]: failed to encode map e601 with expected crc 2023-09-23T21:18:18.834 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:18:18 smithi026 ceph-mon[175151]: failed to encode map e601 with expected crc 2023-09-23T21:18:18.834 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:18:18 smithi026 ceph-mon[175151]: failed to encode map e601 with expected crc 2023-09-23T21:18:18.834 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:18:18 smithi026 ceph-mon[175151]: osdmap e601: 8 total, 8 up, 6 in 2023-09-23T21:18:18.834 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:18:18 smithi026 ceph-mon[175151]: failed to encode map e601 with expected crc 2023-09-23T21:18:18.834 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:18:18 smithi026 ceph-mon[175151]: failed to encode map e601 with expected crc 2023-09-23T21:18:18.834 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:18:18.835 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:18:18.837 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:18:18 smithi026 ceph-mon[179688]: pgmap v1514: 129 pgs: 1 active+undersized+remapped, 1 active+recovery_wait, 1 active+recovery_wait+undersized+degraded+remapped, 126 active+clean; 1.8 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 908 op/s; 812/90348 objects degraded (0.899%); 4.0 MiB/s, 104 objects/s recovering 2023-09-23T21:18:18.838 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:18:18 smithi026 ceph-mon[179688]: failed to encode map e601 with expected crc 2023-09-23T21:18:18.838 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:18:18 smithi026 ceph-mon[179688]: failed to encode map e601 with expected crc 2023-09-23T21:18:18.838 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:18:18 smithi026 ceph-mon[179688]: failed to encode map e601 with expected crc 2023-09-23T21:18:18.838 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:18:18 smithi026 ceph-mon[179688]: osdmap e601: 8 total, 8 up, 6 in 2023-09-23T21:18:18.838 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:18:18 smithi026 ceph-mon[179688]: failed to encode map e601 with expected crc 2023-09-23T21:18:18.838 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:18:18 smithi026 ceph-mon[179688]: failed to encode map e601 with expected crc 2023-09-23T21:18:18.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:18:18 smithi070 ceph-mon[168282]: pgmap v1514: 129 pgs: 1 active+undersized+remapped, 1 active+recovery_wait, 1 active+recovery_wait+undersized+degraded+remapped, 126 active+clean; 1.8 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 908 op/s; 812/90348 objects degraded (0.899%); 4.0 MiB/s, 104 objects/s recovering 2023-09-23T21:18:18.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:18:18 smithi070 ceph-mon[168282]: failed to encode map e601 with expected crc 2023-09-23T21:18:18.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:18:18 smithi070 ceph-mon[168282]: failed to encode map e601 with expected crc 2023-09-23T21:18:18.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:18:18 smithi070 ceph-mon[168282]: failed to encode map e601 with expected crc 2023-09-23T21:18:18.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:18:18 smithi070 ceph-mon[168282]: osdmap e601: 8 total, 8 up, 6 in 2023-09-23T21:18:18.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:18:18 smithi070 ceph-mon[168282]: failed to encode map e601 with expected crc 2023-09-23T21:18:18.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:18:18 smithi070 ceph-mon[168282]: failed to encode map e601 with expected crc 2023-09-23T21:18:18.936 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:18:18.981 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:18:19.027 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:18:19.032 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:18:19.032 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:18:19.133 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:18:19.164 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:18:19.222 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:18:19.225 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:18:19.226 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:18:19.327 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:18:19.396 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:18:19.882 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 21:18:19 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[164029]: ::ffff:172.21.15.70 - - [23/Sep/2023:21:18:19] "GET /metrics HTTP/1.1" 200 36378 "" "Prometheus/2.43.0" 2023-09-23T21:18:20.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:18:20 smithi070 ceph-mon[168282]: pgmap v1516: 129 pgs: 1 peering, 1 active+recovery_wait, 1 active+recovery_wait+undersized+degraded+remapped, 126 active+clean; 1.8 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 899 op/s; 812/88089 objects degraded (0.922%); 3.9 MiB/s, 100 objects/s recovering 2023-09-23T21:18:20.884 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:18:20 smithi026 ceph-mon[175151]: pgmap v1516: 129 pgs: 1 peering, 1 active+recovery_wait, 1 active+recovery_wait+undersized+degraded+remapped, 126 active+clean; 1.8 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 899 op/s; 812/88089 objects degraded (0.922%); 3.9 MiB/s, 100 objects/s recovering 2023-09-23T21:18:20.885 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:18:20 smithi026 ceph-mon[179688]: pgmap v1516: 129 pgs: 1 peering, 1 active+recovery_wait, 1 active+recovery_wait+undersized+degraded+remapped, 126 active+clean; 1.8 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 899 op/s; 812/88089 objects degraded (0.922%); 3.9 MiB/s, 100 objects/s recovering 2023-09-23T21:18:20.918 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:18:20.921 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:18:20.922 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:18:20.961 INFO:teuthology.orchestra.run.smithi026.stdout:{"pg_ready":true,"pg_map":{"version":1512,"stamp":"2023-09-23T21:18:15.191978+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":2156664760,"num_objects":32924,"num_object_clones":0,"num_object_copies":98772,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1119,"num_objects_misplaced":170,"num_objects_unfound":0,"num_objects_dirty":32924,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":163097,"num_write_kb":6269918,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":13968,"num_bytes_recovered":900156734,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":146495,"ondisk_log_size":146495,"up":387,"acting":385,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":412,"num_osds":6,"num_per_pool_osds":6,"num_per_pool_omap_osds":6,"kb":562470912,"kb_used":11261616,"kb_used_data":7031872,"kb_used_omap":125,"kb_used_meta":4229314,"kb_avail":551209296,"statfs":{"total":575970213888,"available":564438319104,"internally_reserved":0,"allocated":7200636928,"data_stored":7187064107,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":128662,"internal_metadata":4330817898},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,6,1,4],"upper_bound":16},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-343736320,"num_objects":-5245,"num_object_clones":0,"num_object_copies":-15735,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":-653,"num_objects_misplaced":-417,"num_objects_unfound":0,"num_objects_dirty":-5245,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5245,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1070,"num_bytes_recovered":59834368,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"6.702103"},"pg_stats":[{"pgid":"11.12","version":"599'7448","reported_seq":7522,"reported_epoch":599,"state":"active+clean","last_fresh":"2023-09-23T21:18:11.602498+0000","last_change":"2023-09-23T21:16:32.944306+0000","last_active":"2023-09-23T21:18:11.602498+0000","last_peered":"2023-09-23T21:18:11.602498+0000","last_clean":"2023-09-23T21:18:11.602498+0000","last_became_active":"2023-09-23T21:16:32.940670+0000","last_became_peered":"2023-09-23T21:16:32.940670+0000","last_unstale":"2023-09-23T21:18:11.602498+0000","last_undegraded":"2023-09-23T21:18:11.602498+0000","last_fullsized":"2023-09-23T21:18:11.602498+0000","mapping_epoch":559,"log_start":"548'374","ondisk_log_start":"548'374","created":559,"last_epoch_clean":560,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_clean_scrub_stamp":"2023-09-23T21:15:24.860968+0000","objects_scrubbed":0,"log_size":7074,"log_dups_size":0,"ondisk_log_size":7074,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":71565312,"num_objects":1092,"num_object_clones":0,"num_object_copies":3276,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1092,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5164,"num_write_kb":200192,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.18","version":"79'161","reported_seq":623,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-09-23T21:18:09.531700+0000","last_change":"2023-09-23T21:17:39.906535+0000","last_active":"2023-09-23T21:18:09.531700+0000","last_peered":"2023-09-23T21:18:09.531700+0000","last_clean":"2023-09-23T21:18:09.531700+0000","last_became_active":"2023-09-23T21:17:39.906154+0000","last_became_peered":"2023-09-23T21:17:39.906154+0000","last_unstale":"2023-09-23T21:18:09.531700+0000","last_undegraded":"2023-09-23T21:18:09.531700+0000","last_fullsized":"2023-09-23T21:18:09.531700+0000","mapping_epoch":585,"log_start":"0'0","ondisk_log_start":"0'0","created":479,"last_epoch_clean":586,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:15:39.474823+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:15:39.474823+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:48:39.784037+0000","scrub_duration":0.074664395999999994,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":525376,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1a","version":"68'1","reported_seq":189,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-09-23T21:18:09.518975+0000","last_change":"2023-09-23T21:13:13.291403+0000","last_active":"2023-09-23T21:18:09.518975+0000","last_peered":"2023-09-23T21:18:09.518975+0000","last_clean":"2023-09-23T21:18:09.518975+0000","last_became_active":"2023-09-23T21:13:13.291123+0000","last_became_peered":"2023-09-23T21:13:13.291123+0000","last_unstale":"2023-09-23T21:18:09.518975+0000","last_undegraded":"2023-09-23T21:18:09.518975+0000","last_fullsized":"2023-09-23T21:18:09.518975+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":495,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:45:15.427430+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,0],"acting":[4,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":258,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-09-23T21:18:09.519293+0000","last_change":"2023-09-23T21:12:10.767963+0000","last_active":"2023-09-23T21:18:09.519293+0000","last_peered":"2023-09-23T21:18:09.519293+0000","last_clean":"2023-09-23T21:18:09.519293+0000","last_became_active":"2023-09-23T21:12:10.767126+0000","last_became_peered":"2023-09-23T21:12:10.767126+0000","last_unstale":"2023-09-23T21:18:09.519293+0000","last_undegraded":"2023-09-23T21:18:09.519293+0000","last_fullsized":"2023-09-23T21:18:09.519293+0000","mapping_epoch":470,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:45:47.138277+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,5],"acting":[4,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.11","version":"599'7515","reported_seq":7587,"reported_epoch":599,"state":"active+clean","last_fresh":"2023-09-23T21:18:14.379790+0000","last_change":"2023-09-23T21:16:32.928552+0000","last_active":"2023-09-23T21:18:14.379790+0000","last_peered":"2023-09-23T21:18:14.379790+0000","last_clean":"2023-09-23T21:18:14.379790+0000","last_became_active":"2023-09-23T21:16:32.926639+0000","last_became_peered":"2023-09-23T21:16:32.926639+0000","last_unstale":"2023-09-23T21:18:14.379790+0000","last_undegraded":"2023-09-23T21:18:14.379790+0000","last_fullsized":"2023-09-23T21:18:14.379790+0000","mapping_epoch":559,"log_start":"550'3292","ondisk_log_start":"550'3292","created":559,"last_epoch_clean":560,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_clean_scrub_stamp":"2023-09-23T21:15:24.860968+0000","objects_scrubbed":0,"log_size":4223,"log_dups_size":0,"ondisk_log_size":4223,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":65404928,"num_objects":998,"num_object_clones":0,"num_object_copies":2994,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":998,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5198,"num_write_kb":198272,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,2],"acting":[3,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1b","version":"79'161","reported_seq":786,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-09-23T21:18:09.532214+0000","last_change":"2023-09-23T21:15:38.423372+0000","last_active":"2023-09-23T21:18:09.532214+0000","last_peered":"2023-09-23T21:18:09.532214+0000","last_clean":"2023-09-23T21:18:09.532214+0000","last_became_active":"2023-09-23T21:15:23.802580+0000","last_became_peered":"2023-09-23T21:15:23.802580+0000","last_unstale":"2023-09-23T21:18:09.532214+0000","last_undegraded":"2023-09-23T21:18:09.532214+0000","last_fullsized":"2023-09-23T21:18:09.532214+0000","mapping_epoch":542,"log_start":"0'0","ondisk_log_start":"0'0","created":479,"last_epoch_clean":543,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:15:38.423271+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:15:38.423271+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:04:46.575186+0000","scrub_duration":0.00258465,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":614,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-09-23T21:18:09.520990+0000","last_change":"2023-09-23T21:16:42.448529+0000","last_active":"2023-09-23T21:18:09.520990+0000","last_peered":"2023-09-23T21:18:09.520990+0000","last_clean":"2023-09-23T21:18:09.520990+0000","last_became_active":"2023-09-23T21:16:42.447803+0000","last_became_peered":"2023-09-23T21:16:42.447803+0000","last_unstale":"2023-09-23T21:18:09.520990+0000","last_undegraded":"2023-09-23T21:18:09.520990+0000","last_fullsized":"2023-09-23T21:18:09.520990+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:41:33.121784+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":614,"reported_epoch":597,"state":"active+clean","last_fresh":"2023-09-23T21:18:08.522531+0000","last_change":"2023-09-23T21:16:42.441896+0000","last_active":"2023-09-23T21:18:08.522531+0000","last_peered":"2023-09-23T21:18:08.522531+0000","last_clean":"2023-09-23T21:18:08.522531+0000","last_became_active":"2023-09-23T21:16:42.441526+0000","last_became_peered":"2023-09-23T21:16:42.441526+0000","last_unstale":"2023-09-23T21:18:08.522531+0000","last_undegraded":"2023-09-23T21:18:08.522531+0000","last_fullsized":"2023-09-23T21:18:08.522531+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:27:36.326225+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.10","version":"599'7412","reported_seq":7506,"reported_epoch":599,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T21:18:14.268250+0000","last_change":"2023-09-23T21:17:02.209680+0000","last_active":"2023-09-23T21:18:14.268250+0000","last_peered":"2023-09-23T21:18:14.268250+0000","last_clean":"2023-09-23T21:16:02.267015+0000","last_became_active":"2023-09-23T21:16:43.501214+0000","last_became_peered":"2023-09-23T21:16:43.501214+0000","last_unstale":"2023-09-23T21:18:14.268250+0000","last_undegraded":"2023-09-23T21:17:02.208847+0000","last_fullsized":"2023-09-23T21:16:43.401617+0000","mapping_epoch":565,"log_start":"550'3245","ondisk_log_start":"550'3245","created":559,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_clean_scrub_stamp":"2023-09-23T21:15:24.860968+0000","objects_scrubbed":0,"log_size":4167,"log_dups_size":0,"ondisk_log_size":4167,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":65568768,"num_objects":1001,"num_object_clones":0,"num_object_copies":3003,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":812,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1001,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5155,"num_write_kb":196960,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":131072,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,0],"avail_no_missing":["3","0"],"object_location_counts":[{"shards":"0,2,3","objects":393},{"shards":"0,3","objects":189}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1a","version":"79'161","reported_seq":696,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-09-23T21:18:09.518517+0000","last_change":"2023-09-23T21:16:45.419221+0000","last_active":"2023-09-23T21:18:09.518517+0000","last_peered":"2023-09-23T21:18:09.518517+0000","last_clean":"2023-09-23T21:18:09.518517+0000","last_became_active":"2023-09-23T21:16:45.418994+0000","last_became_peered":"2023-09-23T21:16:45.418994+0000","last_unstale":"2023-09-23T21:18:09.518517+0000","last_undegraded":"2023-09-23T21:18:09.518517+0000","last_fullsized":"2023-09-23T21:18:09.518517+0000","mapping_epoch":567,"log_start":"0'0","ondisk_log_start":"0'0","created":479,"last_epoch_clean":568,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:15:45.158107+0000","last_deep_scrub":"79'161","last_deep_scrub_stamp":"2023-09-23T21:05:32.630292+0000","last_clean_scrub_stamp":"2023-09-23T21:15:45.158107+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:08:08.342444+0000","scrub_duration":0.001999373,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,5],"acting":[4,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":108,"reported_epoch":597,"state":"active+clean","last_fresh":"2023-09-23T21:18:08.516339+0000","last_change":"2023-09-23T21:13:30.806084+0000","last_active":"2023-09-23T21:18:08.516339+0000","last_peered":"2023-09-23T21:18:08.516339+0000","last_clean":"2023-09-23T21:18:08.516339+0000","last_became_active":"2023-09-23T21:13:30.805743+0000","last_became_peered":"2023-09-23T21:13:30.805743+0000","last_unstale":"2023-09-23T21:18:08.516339+0000","last_undegraded":"2023-09-23T21:18:08.516339+0000","last_fullsized":"2023-09-23T21:18:08.516339+0000","mapping_epoch":501,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:48:10.849281+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":473,"reported_epoch":597,"state":"active+clean","last_fresh":"2023-09-23T21:18:08.524713+0000","last_change":"2023-09-23T21:13:44.410921+0000","last_active":"2023-09-23T21:18:08.524713+0000","last_peered":"2023-09-23T21:18:08.524713+0000","last_clean":"2023-09-23T21:18:08.524713+0000","last_became_active":"2023-09-23T21:13:44.410633+0000","last_became_peered":"2023-09-23T21:13:44.410633+0000","last_unstale":"2023-09-23T21:18:08.524713+0000","last_undegraded":"2023-09-23T21:18:08.524713+0000","last_fullsized":"2023-09-23T21:18:08.524713+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:57:10.116786+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.15","version":"79'161","reported_seq":547,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-09-23T21:18:09.519008+0000","last_change":"2023-09-23T21:16:45.426006+0000","last_active":"2023-09-23T21:18:09.519008+0000","last_peered":"2023-09-23T21:18:09.519008+0000","last_clean":"2023-09-23T21:18:09.519008+0000","last_became_active":"2023-09-23T21:16:45.425654+0000","last_became_peered":"2023-09-23T21:16:45.425654+0000","last_unstale":"2023-09-23T21:18:09.519008+0000","last_undegraded":"2023-09-23T21:18:09.519008+0000","last_fullsized":"2023-09-23T21:18:09.519008+0000","mapping_epoch":567,"log_start":"0'0","ondisk_log_start":"0'0","created":479,"last_epoch_clean":568,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:15:36.742247+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:15:36.742247+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0081306369999999996,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,3],"acting":[7,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":574,"reported_epoch":597,"state":"active+clean","last_fresh":"2023-09-23T21:18:08.524860+0000","last_change":"2023-09-23T21:13:12.235728+0000","last_active":"2023-09-23T21:18:08.524860+0000","last_peered":"2023-09-23T21:18:08.524860+0000","last_clean":"2023-09-23T21:18:08.524860+0000","last_became_active":"2023-09-23T21:13:12.216549+0000","last_became_peered":"2023-09-23T21:13:12.216549+0000","last_unstale":"2023-09-23T21:18:08.524860+0000","last_undegraded":"2023-09-23T21:18:08.524860+0000","last_fullsized":"2023-09-23T21:18:08.524860+0000","mapping_epoch":493,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:39:33.324239+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":638,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-09-23T21:18:09.516838+0000","last_change":"2023-09-23T21:12:10.790349+0000","last_active":"2023-09-23T21:18:09.516838+0000","last_peered":"2023-09-23T21:18:09.516838+0000","last_clean":"2023-09-23T21:18:09.516838+0000","last_became_active":"2023-09-23T21:12:10.789977+0000","last_became_peered":"2023-09-23T21:12:10.789977+0000","last_unstale":"2023-09-23T21:18:09.516838+0000","last_undegraded":"2023-09-23T21:18:09.516838+0000","last_fullsized":"2023-09-23T21:18:09.516838+0000","mapping_epoch":470,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:49:40.191147+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,5],"acting":[3,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"79'161","reported_seq":995,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-09-23T21:18:09.537001+0000","last_change":"2023-09-23T21:16:45.424773+0000","last_active":"2023-09-23T21:18:09.537001+0000","last_peered":"2023-09-23T21:18:09.537001+0000","last_clean":"2023-09-23T21:18:09.537001+0000","last_became_active":"2023-09-23T21:16:45.424458+0000","last_became_peered":"2023-09-23T21:16:45.424458+0000","last_unstale":"2023-09-23T21:18:09.537001+0000","last_undegraded":"2023-09-23T21:18:09.537001+0000","last_fullsized":"2023-09-23T21:18:09.537001+0000","mapping_epoch":567,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":568,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:25.096401+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:25.096401+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:40:17.871561+0000","scrub_duration":0.002859393,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":343,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-09-23T21:18:09.532326+0000","last_change":"2023-09-23T21:16:42.427856+0000","last_active":"2023-09-23T21:18:09.532326+0000","last_peered":"2023-09-23T21:18:09.532326+0000","last_clean":"2023-09-23T21:18:09.532326+0000","last_became_active":"2023-09-23T21:16:42.427529+0000","last_became_peered":"2023-09-23T21:16:42.427529+0000","last_unstale":"2023-09-23T21:18:09.532326+0000","last_undegraded":"2023-09-23T21:18:09.532326+0000","last_fullsized":"2023-09-23T21:18:09.532326+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:24:03.818632+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":593,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-09-23T21:18:09.520579+0000","last_change":"2023-09-23T21:16:42.448719+0000","last_active":"2023-09-23T21:18:09.520579+0000","last_peered":"2023-09-23T21:18:09.520579+0000","last_clean":"2023-09-23T21:18:09.520579+0000","last_became_active":"2023-09-23T21:16:42.448219+0000","last_became_peered":"2023-09-23T21:16:42.448219+0000","last_unstale":"2023-09-23T21:18:09.520579+0000","last_undegraded":"2023-09-23T21:18:09.520579+0000","last_fullsized":"2023-09-23T21:18:09.520579+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:02:28.864357+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.17","version":"79'161","reported_seq":745,"reported_epoch":597,"state":"active+clean","last_fresh":"2023-09-23T21:18:08.524332+0000","last_change":"2023-09-23T21:16:45.413663+0000","last_active":"2023-09-23T21:18:08.524332+0000","last_peered":"2023-09-23T21:18:08.524332+0000","last_clean":"2023-09-23T21:18:08.524332+0000","last_became_active":"2023-09-23T21:16:45.413311+0000","last_became_peered":"2023-09-23T21:16:45.413311+0000","last_unstale":"2023-09-23T21:18:08.524332+0000","last_undegraded":"2023-09-23T21:18:08.524332+0000","last_fullsized":"2023-09-23T21:18:08.524332+0000","mapping_epoch":567,"log_start":"0'0","ondisk_log_start":"0'0","created":479,"last_epoch_clean":568,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:15:50.325741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:15:50.325741+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:07:43.285354+0000","scrub_duration":0.0040500149999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":514,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-09-23T21:18:09.516547+0000","last_change":"2023-09-23T21:17:02.182606+0000","last_active":"2023-09-23T21:18:09.516547+0000","last_peered":"2023-09-23T21:18:09.516547+0000","last_clean":"2023-09-23T21:18:09.516547+0000","last_became_active":"2023-09-23T21:16:42.446517+0000","last_became_peered":"2023-09-23T21:16:42.446517+0000","last_unstale":"2023-09-23T21:18:09.516547+0000","last_undegraded":"2023-09-23T21:18:09.516547+0000","last_fullsized":"2023-09-23T21:18:09.516547+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:46:08.337878+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":540,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,4],"acting":[3,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":287,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-09-23T21:18:09.533679+0000","last_change":"2023-09-23T21:13:30.821489+0000","last_active":"2023-09-23T21:18:09.533679+0000","last_peered":"2023-09-23T21:18:09.533679+0000","last_clean":"2023-09-23T21:18:09.533679+0000","last_became_active":"2023-09-23T21:13:30.821125+0000","last_became_peered":"2023-09-23T21:13:30.821125+0000","last_unstale":"2023-09-23T21:18:09.533679+0000","last_undegraded":"2023-09-23T21:18:09.533679+0000","last_fullsized":"2023-09-23T21:18:09.533679+0000","mapping_epoch":501,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:47:18.553417+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,5],"acting":[2,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.16","version":"79'161","reported_seq":529,"reported_epoch":597,"state":"active+clean","last_fresh":"2023-09-23T21:18:08.522442+0000","last_change":"2023-09-23T21:15:46.326726+0000","last_active":"2023-09-23T21:18:08.522442+0000","last_peered":"2023-09-23T21:18:08.522442+0000","last_clean":"2023-09-23T21:18:08.522442+0000","last_became_active":"2023-09-23T21:13:33.770407+0000","last_became_peered":"2023-09-23T21:13:33.770407+0000","last_unstale":"2023-09-23T21:18:08.522442+0000","last_undegraded":"2023-09-23T21:18:08.522442+0000","last_fullsized":"2023-09-23T21:18:08.522442+0000","mapping_epoch":504,"log_start":"0'0","ondisk_log_start":"0'0","created":479,"last_epoch_clean":505,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:15:46.326589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:15:46.326589+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:50:23.810054+0000","scrub_duration":0.0062069059999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.14","version":"79'22","reported_seq":202,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-09-23T21:18:09.519730+0000","last_change":"2023-09-23T21:17:02.198735+0000","last_active":"2023-09-23T21:18:09.519730+0000","last_peered":"2023-09-23T21:18:09.519730+0000","last_clean":"2023-09-23T21:18:09.519730+0000","last_became_active":"2023-09-23T21:16:42.492861+0000","last_became_peered":"2023-09-23T21:16:42.492861+0000","last_unstale":"2023-09-23T21:18:09.519730+0000","last_undegraded":"2023-09-23T21:18:09.519730+0000","last_fullsized":"2023-09-23T21:18:09.519730+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:00:09.949965+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":14202,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,2],"acting":[4,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":527,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-09-23T21:18:09.519441+0000","last_change":"2023-09-23T21:12:10.839271+0000","last_active":"2023-09-23T21:18:09.519441+0000","last_peered":"2023-09-23T21:18:09.519441+0000","last_clean":"2023-09-23T21:18:09.519441+0000","last_became_active":"2023-09-23T21:12:10.839042+0000","last_became_peered":"2023-09-23T21:12:10.839042+0000","last_unstale":"2023-09-23T21:18:09.519441+0000","last_undegraded":"2023-09-23T21:18:09.519441+0000","last_fullsized":"2023-09-23T21:18:09.519441+0000","mapping_epoch":470,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:10:26.414737+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,2],"acting":[4,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.11","version":"79'161","reported_seq":534,"reported_epoch":597,"state":"active+clean","last_fresh":"2023-09-23T21:18:08.515384+0000","last_change":"2023-09-23T21:15:47.300109+0000","last_active":"2023-09-23T21:18:08.515384+0000","last_peered":"2023-09-23T21:18:08.515384+0000","last_clean":"2023-09-23T21:18:08.515384+0000","last_became_active":"2023-09-23T21:13:33.767067+0000","last_became_peered":"2023-09-23T21:13:33.767067+0000","last_unstale":"2023-09-23T21:18:08.515384+0000","last_undegraded":"2023-09-23T21:18:08.515384+0000","last_fullsized":"2023-09-23T21:18:08.515384+0000","mapping_epoch":504,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":505,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:15:47.299947+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:15:47.299947+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:04:59.029359+0000","scrub_duration":0.0061353110000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":50,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-09-23T21:18:09.518524+0000","last_change":"2023-09-23T21:16:42.531060+0000","last_active":"2023-09-23T21:18:09.518524+0000","last_peered":"2023-09-23T21:18:09.518524+0000","last_clean":"2023-09-23T21:18:09.518524+0000","last_became_active":"2023-09-23T21:16:42.449198+0000","last_became_peered":"2023-09-23T21:16:42.449198+0000","last_unstale":"2023-09-23T21:18:09.518524+0000","last_undegraded":"2023-09-23T21:18:09.518524+0000","last_fullsized":"2023-09-23T21:18:09.518524+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:30:23.428886+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":159,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-09-23T21:18:09.536761+0000","last_change":"2023-09-23T21:16:42.491023+0000","last_active":"2023-09-23T21:18:09.536761+0000","last_peered":"2023-09-23T21:18:09.536761+0000","last_clean":"2023-09-23T21:18:09.536761+0000","last_became_active":"2023-09-23T21:16:42.489896+0000","last_became_peered":"2023-09-23T21:16:42.489896+0000","last_unstale":"2023-09-23T21:18:09.536761+0000","last_undegraded":"2023-09-23T21:18:09.536761+0000","last_fullsized":"2023-09-23T21:18:09.536761+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:58:54.036098+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,7],"acting":[2,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.10","version":"79'161","reported_seq":773,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-09-23T21:18:09.518080+0000","last_change":"2023-09-23T21:16:45.416227+0000","last_active":"2023-09-23T21:18:09.518080+0000","last_peered":"2023-09-23T21:18:09.518080+0000","last_clean":"2023-09-23T21:18:09.518080+0000","last_became_active":"2023-09-23T21:16:45.415841+0000","last_became_peered":"2023-09-23T21:16:45.415841+0000","last_unstale":"2023-09-23T21:18:09.518080+0000","last_undegraded":"2023-09-23T21:18:09.518080+0000","last_fullsized":"2023-09-23T21:18:09.518080+0000","mapping_epoch":567,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":568,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:06:50.336992+0000","last_deep_scrub":"79'161","last_deep_scrub_stamp":"2023-09-23T21:06:50.336992+0000","last_clean_scrub_stamp":"2023-09-23T21:06:50.336992+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:37:30.094658+0000","scrub_duration":0.0026150930000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,7],"acting":[4,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":558,"reported_epoch":597,"state":"active+clean","last_fresh":"2023-09-23T21:18:08.523796+0000","last_change":"2023-09-23T21:16:42.446324+0000","last_active":"2023-09-23T21:18:08.523796+0000","last_peered":"2023-09-23T21:18:08.523796+0000","last_clean":"2023-09-23T21:18:08.523796+0000","last_became_active":"2023-09-23T21:16:42.445997+0000","last_became_peered":"2023-09-23T21:16:42.445997+0000","last_unstale":"2023-09-23T21:18:08.523796+0000","last_undegraded":"2023-09-23T21:18:08.523796+0000","last_fullsized":"2023-09-23T21:18:08.523796+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:46:35.028292+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":370,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-09-23T21:18:09.516383+0000","last_change":"2023-09-23T21:16:42.473872+0000","last_active":"2023-09-23T21:18:09.516383+0000","last_peered":"2023-09-23T21:18:09.516383+0000","last_clean":"2023-09-23T21:18:09.516383+0000","last_became_active":"2023-09-23T21:16:42.473595+0000","last_became_peered":"2023-09-23T21:16:42.473595+0000","last_unstale":"2023-09-23T21:18:09.516383+0000","last_undegraded":"2023-09-23T21:18:09.516383+0000","last_fullsized":"2023-09-23T21:18:09.516383+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:15:19.186428+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.19","version":"599'7431","reported_seq":9205,"reported_epoch":599,"state":"active+recovery_wait","last_fresh":"2023-09-23T21:18:14.386772+0000","last_change":"2023-09-23T21:17:46.325067+0000","last_active":"2023-09-23T21:18:14.386772+0000","last_peered":"2023-09-23T21:18:14.386772+0000","last_clean":"2023-09-23T21:16:02.272109+0000","last_became_active":"2023-09-23T21:17:46.300166+0000","last_became_peered":"2023-09-23T21:17:46.300166+0000","last_unstale":"2023-09-23T21:18:14.386772+0000","last_undegraded":"2023-09-23T21:18:14.386772+0000","last_fullsized":"2023-09-23T21:18:14.386772+0000","mapping_epoch":588,"log_start":"550'3246","ondisk_log_start":"550'3246","created":559,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_clean_scrub_stamp":"2023-09-23T21:15:24.860968+0000","objects_scrubbed":0,"log_size":4185,"log_dups_size":0,"ondisk_log_size":4185,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":61177856,"num_objects":934,"num_object_clones":0,"num_object_copies":2802,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":934,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5148,"num_write_kb":194592,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":838,"num_bytes_recovered":54820864,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,7],"acting":[3,0,7],"avail_no_missing":["3","0","7"],"object_location_counts":[{"shards":"0,3,7","objects":934}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.13","version":"79'161","reported_seq":677,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-09-23T21:18:09.517452+0000","last_change":"2023-09-23T21:13:14.933096+0000","last_active":"2023-09-23T21:18:09.517452+0000","last_peered":"2023-09-23T21:18:09.517452+0000","last_clean":"2023-09-23T21:18:09.517452+0000","last_became_active":"2023-09-23T21:13:14.932703+0000","last_became_peered":"2023-09-23T21:13:14.932703+0000","last_unstale":"2023-09-23T21:18:09.517452+0000","last_undegraded":"2023-09-23T21:18:09.517452+0000","last_fullsized":"2023-09-23T21:18:09.517452+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:36.420362+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:36.420362+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:29:42.163911+0000","scrub_duration":0.028838444000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":610,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-09-23T21:18:09.518524+0000","last_change":"2023-09-23T21:16:42.420467+0000","last_active":"2023-09-23T21:18:09.518524+0000","last_peered":"2023-09-23T21:18:09.518524+0000","last_clean":"2023-09-23T21:18:09.518524+0000","last_became_active":"2023-09-23T21:16:42.420149+0000","last_became_peered":"2023-09-23T21:16:42.420149+0000","last_unstale":"2023-09-23T21:18:09.518524+0000","last_undegraded":"2023-09-23T21:18:09.518524+0000","last_fullsized":"2023-09-23T21:18:09.518524+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:52:07.355000+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":126,"reported_epoch":597,"state":"active+clean","last_fresh":"2023-09-23T21:18:08.521908+0000","last_change":"2023-09-23T21:16:42.427663+0000","last_active":"2023-09-23T21:18:08.521908+0000","last_peered":"2023-09-23T21:18:08.521908+0000","last_clean":"2023-09-23T21:18:08.521908+0000","last_became_active":"2023-09-23T21:16:42.426961+0000","last_became_peered":"2023-09-23T21:16:42.426961+0000","last_unstale":"2023-09-23T21:18:08.521908+0000","last_undegraded":"2023-09-23T21:18:08.521908+0000","last_fullsized":"2023-09-23T21:18:08.521908+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:44:27.422007+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.18","version":"598'7128","reported_seq":7201,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-09-23T21:18:09.874469+0000","last_change":"2023-09-23T21:16:32.928345+0000","last_active":"2023-09-23T21:18:09.874469+0000","last_peered":"2023-09-23T21:18:09.874469+0000","last_clean":"2023-09-23T21:18:09.874469+0000","last_became_active":"2023-09-23T21:16:32.926828+0000","last_became_peered":"2023-09-23T21:16:32.926828+0000","last_unstale":"2023-09-23T21:18:09.874469+0000","last_undegraded":"2023-09-23T21:18:09.874469+0000","last_fullsized":"2023-09-23T21:18:09.874469+0000","mapping_epoch":559,"log_start":"549'1866","ondisk_log_start":"549'1866","created":559,"last_epoch_clean":560,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_clean_scrub_stamp":"2023-09-23T21:15:24.860968+0000","objects_scrubbed":0,"log_size":5262,"log_dups_size":0,"ondisk_log_size":5262,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":69828608,"num_objects":1066,"num_object_clones":0,"num_object_copies":3198,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1066,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4904,"num_write_kb":191008,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,5],"acting":[0,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.12","version":"79'161","reported_seq":872,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-09-23T21:18:09.532357+0000","last_change":"2023-09-23T21:16:45.412440+0000","last_active":"2023-09-23T21:18:09.532357+0000","last_peered":"2023-09-23T21:18:09.532357+0000","last_clean":"2023-09-23T21:18:09.532357+0000","last_became_active":"2023-09-23T21:16:45.412029+0000","last_became_peered":"2023-09-23T21:16:45.412029+0000","last_unstale":"2023-09-23T21:18:09.532357+0000","last_undegraded":"2023-09-23T21:18:09.532357+0000","last_fullsized":"2023-09-23T21:18:09.532357+0000","mapping_epoch":567,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":568,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:05:56.785173+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:05:56.785173+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:39:43.432998+0000","scrub_duration":0.0042114370000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,0],"acting":[5,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":118,"reported_epoch":597,"state":"active+clean","last_fresh":"2023-09-23T21:18:08.516516+0000","last_change":"2023-09-23T21:13:38.048791+0000","last_active":"2023-09-23T21:18:08.516516+0000","last_peered":"2023-09-23T21:18:08.516516+0000","last_clean":"2023-09-23T21:18:08.516516+0000","last_became_active":"2023-09-23T21:13:30.801386+0000","last_became_peered":"2023-09-23T21:13:30.801386+0000","last_unstale":"2023-09-23T21:18:08.516516+0000","last_undegraded":"2023-09-23T21:18:08.516516+0000","last_fullsized":"2023-09-23T21:18:08.516516+0000","mapping_epoch":501,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:52:49.170414+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":455,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-09-23T21:18:09.517902+0000","last_change":"2023-09-23T21:16:42.518512+0000","last_active":"2023-09-23T21:18:09.517902+0000","last_peered":"2023-09-23T21:18:09.517902+0000","last_clean":"2023-09-23T21:18:09.517902+0000","last_became_active":"2023-09-23T21:16:42.517464+0000","last_became_peered":"2023-09-23T21:16:42.517464+0000","last_unstale":"2023-09-23T21:18:09.517902+0000","last_undegraded":"2023-09-23T21:18:09.517902+0000","last_fullsized":"2023-09-23T21:18:09.517902+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:47:52.391167+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.7","version":"599'7443","reported_seq":7529,"reported_epoch":599,"state":"active+clean","last_fresh":"2023-09-23T21:18:12.006532+0000","last_change":"2023-09-23T21:18:08.418283+0000","last_active":"2023-09-23T21:18:12.006532+0000","last_peered":"2023-09-23T21:18:12.006532+0000","last_clean":"2023-09-23T21:18:12.006532+0000","last_became_active":"2023-09-23T21:16:32.864890+0000","last_became_peered":"2023-09-23T21:16:32.864890+0000","last_unstale":"2023-09-23T21:18:12.006532+0000","last_undegraded":"2023-09-23T21:18:12.006532+0000","last_fullsized":"2023-09-23T21:18:12.006532+0000","mapping_epoch":544,"log_start":"548'640","ondisk_log_start":"548'640","created":544,"last_epoch_clean":560,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_clean_scrub_stamp":"2023-09-23T21:15:24.860968+0000","objects_scrubbed":0,"log_size":6803,"log_dups_size":0,"ondisk_log_size":6803,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":64258048,"num_objects":980,"num_object_clones":0,"num_object_copies":2940,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":980,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5138,"num_write_kb":195808,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.d","version":"79'161","reported_seq":624,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-09-23T21:18:09.519288+0000","last_change":"2023-09-23T21:16:45.416685+0000","last_active":"2023-09-23T21:18:09.519288+0000","last_peered":"2023-09-23T21:18:09.519288+0000","last_clean":"2023-09-23T21:18:09.519288+0000","last_became_active":"2023-09-23T21:16:45.416373+0000","last_became_peered":"2023-09-23T21:16:45.416373+0000","last_unstale":"2023-09-23T21:18:09.519288+0000","last_undegraded":"2023-09-23T21:18:09.519288+0000","last_fullsized":"2023-09-23T21:18:09.519288+0000","mapping_epoch":567,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":568,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:15:32.726690+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:15:32.726690+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0024987500000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":48,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-09-23T21:18:09.519245+0000","last_change":"2023-09-23T21:17:02.190320+0000","last_active":"2023-09-23T21:18:09.519245+0000","last_peered":"2023-09-23T21:18:09.519245+0000","last_clean":"2023-09-23T21:18:09.519245+0000","last_became_active":"2023-09-23T21:16:42.451380+0000","last_became_peered":"2023-09-23T21:16:42.451380+0000","last_unstale":"2023-09-23T21:18:09.519245+0000","last_undegraded":"2023-09-23T21:18:09.519245+0000","last_fullsized":"2023-09-23T21:18:09.519245+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:48:58.425821+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":132,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":326,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-09-23T21:18:09.531910+0000","last_change":"2023-09-23T21:16:42.428491+0000","last_active":"2023-09-23T21:18:09.531910+0000","last_peered":"2023-09-23T21:18:09.531910+0000","last_clean":"2023-09-23T21:18:09.531910+0000","last_became_active":"2023-09-23T21:16:42.428166+0000","last_became_peered":"2023-09-23T21:16:42.428166+0000","last_unstale":"2023-09-23T21:18:09.531910+0000","last_undegraded":"2023-09-23T21:18:09.531910+0000","last_fullsized":"2023-09-23T21:18:09.531910+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:23:01.430714+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,4],"acting":[5,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.6","version":"598'7218","reported_seq":7294,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-09-23T21:18:09.919775+0000","last_change":"2023-09-23T21:16:33.949061+0000","last_active":"2023-09-23T21:18:09.919775+0000","last_peered":"2023-09-23T21:18:09.919775+0000","last_clean":"2023-09-23T21:18:09.919775+0000","last_became_active":"2023-09-23T21:16:32.896510+0000","last_became_peered":"2023-09-23T21:16:32.896510+0000","last_unstale":"2023-09-23T21:18:09.919775+0000","last_undegraded":"2023-09-23T21:18:09.919775+0000","last_fullsized":"2023-09-23T21:18:09.919775+0000","mapping_epoch":544,"log_start":"549'2133","ondisk_log_start":"549'2133","created":544,"last_epoch_clean":561,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_clean_scrub_stamp":"2023-09-23T21:15:24.860968+0000","objects_scrubbed":0,"log_size":5085,"log_dups_size":0,"ondisk_log_size":5085,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":69763072,"num_objects":1064,"num_object_clones":0,"num_object_copies":3192,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1064,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4984,"num_write_kb":193568,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.c","version":"79'161","reported_seq":981,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-09-23T21:18:09.516095+0000","last_change":"2023-09-23T21:15:31.992756+0000","last_active":"2023-09-23T21:18:09.516095+0000","last_peered":"2023-09-23T21:18:09.516095+0000","last_clean":"2023-09-23T21:18:09.516095+0000","last_became_active":"2023-09-23T21:13:13.401526+0000","last_became_peered":"2023-09-23T21:13:13.401526+0000","last_unstale":"2023-09-23T21:18:09.516095+0000","last_undegraded":"2023-09-23T21:18:09.516095+0000","last_fullsized":"2023-09-23T21:18:09.516095+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":532,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:15:31.992598+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:15:31.992598+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:41:14.215543+0000","scrub_duration":0.002960678,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":48,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-09-23T21:18:09.520041+0000","last_change":"2023-09-23T21:17:02.215996+0000","last_active":"2023-09-23T21:18:09.520041+0000","last_peered":"2023-09-23T21:18:09.520041+0000","last_clean":"2023-09-23T21:18:09.520041+0000","last_became_active":"2023-09-23T21:16:42.451205+0000","last_became_peered":"2023-09-23T21:16:42.451205+0000","last_unstale":"2023-09-23T21:18:09.520041+0000","last_undegraded":"2023-09-23T21:18:09.520041+0000","last_fullsized":"2023-09-23T21:18:09.520041+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:51:46.876281+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":561,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-09-23T21:18:09.518606+0000","last_change":"2023-09-23T21:13:15.152792+0000","last_active":"2023-09-23T21:18:09.518606+0000","last_peered":"2023-09-23T21:18:09.518606+0000","last_clean":"2023-09-23T21:18:09.518606+0000","last_became_active":"2023-09-23T21:13:14.515084+0000","last_became_peered":"2023-09-23T21:13:14.515084+0000","last_unstale":"2023-09-23T21:18:09.518606+0000","last_undegraded":"2023-09-23T21:18:09.518606+0000","last_fullsized":"2023-09-23T21:18:09.518606+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:08:16.702072+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.5","version":"599'7418","reported_seq":12626,"reported_epoch":599,"state":"active+clean","last_fresh":"2023-09-23T21:18:12.288019+0000","last_change":"2023-09-23T21:18:08.488193+0000","last_active":"2023-09-23T21:18:12.288019+0000","last_peered":"2023-09-23T21:18:12.288019+0000","last_clean":"2023-09-23T21:18:12.288019+0000","last_became_active":"2023-09-23T21:17:24.389052+0000","last_became_peered":"2023-09-23T21:17:24.389052+0000","last_unstale":"2023-09-23T21:18:12.288019+0000","last_undegraded":"2023-09-23T21:18:12.288019+0000","last_fullsized":"2023-09-23T21:18:12.288019+0000","mapping_epoch":578,"log_start":"548'300","ondisk_log_start":"548'300","created":544,"last_epoch_clean":579,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_clean_scrub_stamp":"2023-09-23T21:15:24.860968+0000","objects_scrubbed":0,"log_size":7118,"log_dups_size":0,"ondisk_log_size":7118,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70942720,"num_objects":1082,"num_object_clones":0,"num_object_copies":3246,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1082,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5122,"num_write_kb":198560,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1682,"num_bytes_recovered":110231552,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.f","version":"79'161","reported_seq":886,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-09-23T21:18:09.532531+0000","last_change":"2023-09-23T21:11:30.750338+0000","last_active":"2023-09-23T21:18:09.532531+0000","last_peered":"2023-09-23T21:18:09.532531+0000","last_clean":"2023-09-23T21:18:09.532531+0000","last_became_active":"2023-09-23T21:11:30.749953+0000","last_became_peered":"2023-09-23T21:11:30.749953+0000","last_unstale":"2023-09-23T21:18:09.532531+0000","last_undegraded":"2023-09-23T21:18:09.532531+0000","last_fullsized":"2023-09-23T21:18:09.532531+0000","mapping_epoch":455,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":456,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:27.880776+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:27.880776+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:07:10.405723+0000","scrub_duration":0.0014004899999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,3],"acting":[5,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":350,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-09-23T21:18:09.532384+0000","last_change":"2023-09-23T21:17:02.175012+0000","last_active":"2023-09-23T21:18:09.532384+0000","last_peered":"2023-09-23T21:18:09.532384+0000","last_clean":"2023-09-23T21:18:09.532384+0000","last_became_active":"2023-09-23T21:16:42.451114+0000","last_became_peered":"2023-09-23T21:16:42.451114+0000","last_unstale":"2023-09-23T21:18:09.532384+0000","last_undegraded":"2023-09-23T21:18:09.532384+0000","last_fullsized":"2023-09-23T21:18:09.532384+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:00:53.868525+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":420,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":558,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-09-23T21:18:09.531913+0000","last_change":"2023-09-23T21:16:42.432943+0000","last_active":"2023-09-23T21:18:09.531913+0000","last_peered":"2023-09-23T21:18:09.531913+0000","last_clean":"2023-09-23T21:18:09.531913+0000","last_became_active":"2023-09-23T21:16:42.427440+0000","last_became_peered":"2023-09-23T21:16:42.427440+0000","last_unstale":"2023-09-23T21:18:09.531913+0000","last_undegraded":"2023-09-23T21:18:09.531913+0000","last_fullsized":"2023-09-23T21:18:09.531913+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:59:32.044609+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,0],"acting":[5,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.4","version":"599'7244","reported_seq":8872,"reported_epoch":599,"state":"active+clean","last_fresh":"2023-09-23T21:18:12.271729+0000","last_change":"2023-09-23T21:18:05.512415+0000","last_active":"2023-09-23T21:18:12.271729+0000","last_peered":"2023-09-23T21:18:12.271729+0000","last_clean":"2023-09-23T21:18:12.271729+0000","last_became_active":"2023-09-23T21:18:05.477101+0000","last_became_peered":"2023-09-23T21:18:05.477101+0000","last_unstale":"2023-09-23T21:18:12.271729+0000","last_undegraded":"2023-09-23T21:18:12.271729+0000","last_fullsized":"2023-09-23T21:18:12.271729+0000","mapping_epoch":595,"log_start":"548'1118","ondisk_log_start":"548'1118","created":544,"last_epoch_clean":596,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_clean_scrub_stamp":"2023-09-23T21:15:24.860968+0000","objects_scrubbed":0,"log_size":6126,"log_dups_size":0,"ondisk_log_size":6126,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":75563008,"num_objects":1153,"num_object_clones":0,"num_object_copies":3459,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1153,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5007,"num_write_kb":197120,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":765,"num_bytes_recovered":50135040,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,3],"acting":[7,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.e","version":"79'161","reported_seq":817,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-09-23T21:18:09.519902+0000","last_change":"2023-09-23T21:16:45.433204+0000","last_active":"2023-09-23T21:18:09.519902+0000","last_peered":"2023-09-23T21:18:09.519902+0000","last_clean":"2023-09-23T21:18:09.519902+0000","last_became_active":"2023-09-23T21:16:45.432836+0000","last_became_peered":"2023-09-23T21:16:45.432836+0000","last_unstale":"2023-09-23T21:18:09.519902+0000","last_undegraded":"2023-09-23T21:18:09.519902+0000","last_fullsized":"2023-09-23T21:18:09.519902+0000","mapping_epoch":567,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":568,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:15:39.046962+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:15:39.046962+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:11:20.720731+0000","scrub_duration":0.0082638769999999993,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":397,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-09-23T21:18:09.516498+0000","last_change":"2023-09-23T21:13:13.402418+0000","last_active":"2023-09-23T21:18:09.516498+0000","last_peered":"2023-09-23T21:18:09.516498+0000","last_clean":"2023-09-23T21:18:09.516498+0000","last_became_active":"2023-09-23T21:13:13.400984+0000","last_became_peered":"2023-09-23T21:13:13.400984+0000","last_unstale":"2023-09-23T21:18:09.516498+0000","last_undegraded":"2023-09-23T21:18:09.516498+0000","last_fullsized":"2023-09-23T21:18:09.516498+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":495,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:28:01.411191+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,0],"acting":[3,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":572,"reported_epoch":597,"state":"active+clean","last_fresh":"2023-09-23T21:18:08.521279+0000","last_change":"2023-09-23T21:13:14.339670+0000","last_active":"2023-09-23T21:18:08.521279+0000","last_peered":"2023-09-23T21:18:08.521279+0000","last_clean":"2023-09-23T21:18:08.521279+0000","last_became_active":"2023-09-23T21:13:14.339341+0000","last_became_peered":"2023-09-23T21:13:14.339341+0000","last_unstale":"2023-09-23T21:18:08.521279+0000","last_undegraded":"2023-09-23T21:18:08.521279+0000","last_fullsized":"2023-09-23T21:18:08.521279+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:23:19.411368+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.3","version":"599'7202","reported_seq":8874,"reported_epoch":599,"state":"active+clean","last_fresh":"2023-09-23T21:18:12.257158+0000","last_change":"2023-09-23T21:17:51.904100+0000","last_active":"2023-09-23T21:18:12.257158+0000","last_peered":"2023-09-23T21:18:12.257158+0000","last_clean":"2023-09-23T21:18:12.257158+0000","last_became_active":"2023-09-23T21:17:51.872537+0000","last_became_peered":"2023-09-23T21:17:51.872537+0000","last_unstale":"2023-09-23T21:18:12.257158+0000","last_undegraded":"2023-09-23T21:18:12.257158+0000","last_fullsized":"2023-09-23T21:18:12.257158+0000","mapping_epoch":591,"log_start":"548'1141","ondisk_log_start":"548'1141","created":544,"last_epoch_clean":592,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_clean_scrub_stamp":"2023-09-23T21:15:24.860968+0000","objects_scrubbed":0,"log_size":6061,"log_dups_size":0,"ondisk_log_size":6061,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":65929216,"num_objects":1006,"num_object_clones":0,"num_object_copies":3018,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1006,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4986,"num_write_kb":191744,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":786,"num_bytes_recovered":51511296,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,2],"acting":[7,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.9","version":"79'161","reported_seq":800,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-09-23T21:18:09.532049+0000","last_change":"2023-09-23T21:16:45.433968+0000","last_active":"2023-09-23T21:18:09.532049+0000","last_peered":"2023-09-23T21:18:09.532049+0000","last_clean":"2023-09-23T21:18:09.532049+0000","last_became_active":"2023-09-23T21:16:45.433602+0000","last_became_peered":"2023-09-23T21:16:45.433602+0000","last_unstale":"2023-09-23T21:18:09.532049+0000","last_undegraded":"2023-09-23T21:18:09.532049+0000","last_fullsized":"2023-09-23T21:18:09.532049+0000","mapping_epoch":567,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":568,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:15:43.437369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:15:43.437369+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:16:44.538102+0000","scrub_duration":0.0024129429999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,3],"acting":[2,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":158,"reported_epoch":597,"state":"active+clean","last_fresh":"2023-09-23T21:18:08.521356+0000","last_change":"2023-09-23T21:13:30.793273+0000","last_active":"2023-09-23T21:18:08.521356+0000","last_peered":"2023-09-23T21:18:08.521356+0000","last_clean":"2023-09-23T21:18:08.521356+0000","last_became_active":"2023-09-23T21:13:30.791269+0000","last_became_peered":"2023-09-23T21:13:30.791269+0000","last_unstale":"2023-09-23T21:18:08.521356+0000","last_undegraded":"2023-09-23T21:18:08.521356+0000","last_fullsized":"2023-09-23T21:18:08.521356+0000","mapping_epoch":501,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:38:07.142290+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":192,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-09-23T21:18:09.519594+0000","last_change":"2023-09-23T21:16:42.417057+0000","last_active":"2023-09-23T21:18:09.519594+0000","last_peered":"2023-09-23T21:18:09.519594+0000","last_clean":"2023-09-23T21:18:09.519594+0000","last_became_active":"2023-09-23T21:16:42.416601+0000","last_became_peered":"2023-09-23T21:16:42.416601+0000","last_unstale":"2023-09-23T21:18:09.519594+0000","last_undegraded":"2023-09-23T21:18:09.519594+0000","last_fullsized":"2023-09-23T21:18:09.519594+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:56:52.167562+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,7],"acting":[4,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.2","version":"599'7412","reported_seq":7487,"reported_epoch":599,"state":"active+clean","last_fresh":"2023-09-23T21:18:11.553872+0000","last_change":"2023-09-23T21:16:32.870210+0000","last_active":"2023-09-23T21:18:11.553872+0000","last_peered":"2023-09-23T21:18:11.553872+0000","last_clean":"2023-09-23T21:18:11.553872+0000","last_became_active":"2023-09-23T21:16:32.867780+0000","last_became_peered":"2023-09-23T21:16:32.867780+0000","last_unstale":"2023-09-23T21:18:11.553872+0000","last_undegraded":"2023-09-23T21:18:11.553872+0000","last_fullsized":"2023-09-23T21:18:11.553872+0000","mapping_epoch":544,"log_start":"548'190","ondisk_log_start":"548'190","created":544,"last_epoch_clean":560,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_clean_scrub_stamp":"2023-09-23T21:15:24.860968+0000","objects_scrubbed":0,"log_size":7222,"log_dups_size":0,"ondisk_log_size":7222,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":71827456,"num_objects":1096,"num_object_clones":0,"num_object_copies":3288,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1096,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5128,"num_write_kb":199168,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.8","version":"79'161","reported_seq":733,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-09-23T21:18:09.532176+0000","last_change":"2023-09-23T21:15:36.367819+0000","last_active":"2023-09-23T21:18:09.532176+0000","last_peered":"2023-09-23T21:18:09.532176+0000","last_clean":"2023-09-23T21:18:09.532176+0000","last_became_active":"2023-09-23T21:12:30.433217+0000","last_became_peered":"2023-09-23T21:12:30.433217+0000","last_unstale":"2023-09-23T21:18:09.532176+0000","last_undegraded":"2023-09-23T21:18:09.532176+0000","last_fullsized":"2023-09-23T21:18:09.532176+0000","mapping_epoch":359,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:15:36.367640+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:15:36.367640+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:49:35.243653+0000","scrub_duration":0.005826899,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":175,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-09-23T21:18:09.518893+0000","last_change":"2023-09-23T21:12:10.823380+0000","last_active":"2023-09-23T21:18:09.518893+0000","last_peered":"2023-09-23T21:18:09.518893+0000","last_clean":"2023-09-23T21:18:09.518893+0000","last_became_active":"2023-09-23T21:12:10.823044+0000","last_became_peered":"2023-09-23T21:12:10.823044+0000","last_unstale":"2023-09-23T21:18:09.518893+0000","last_undegraded":"2023-09-23T21:18:09.518893+0000","last_fullsized":"2023-09-23T21:18:09.518893+0000","mapping_epoch":470,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:53:29.846536+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,5],"acting":[4,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":176,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-09-23T21:18:09.518923+0000","last_change":"2023-09-23T21:13:30.772451+0000","last_active":"2023-09-23T21:18:09.518923+0000","last_peered":"2023-09-23T21:18:09.518923+0000","last_clean":"2023-09-23T21:18:09.518923+0000","last_became_active":"2023-09-23T21:13:30.772040+0000","last_became_peered":"2023-09-23T21:13:30.772040+0000","last_unstale":"2023-09-23T21:18:09.518923+0000","last_undegraded":"2023-09-23T21:18:09.518923+0000","last_fullsized":"2023-09-23T21:18:09.518923+0000","mapping_epoch":501,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:24:49.246727+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,0],"acting":[4,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.1","version":"599'7627","reported_seq":7700,"reported_epoch":599,"state":"active+clean","last_fresh":"2023-09-23T21:18:14.284892+0000","last_change":"2023-09-23T21:16:32.911457+0000","last_active":"2023-09-23T21:18:14.284892+0000","last_peered":"2023-09-23T21:18:14.284892+0000","last_clean":"2023-09-23T21:18:14.284892+0000","last_became_active":"2023-09-23T21:16:32.905672+0000","last_became_peered":"2023-09-23T21:16:32.905672+0000","last_unstale":"2023-09-23T21:18:14.284892+0000","last_undegraded":"2023-09-23T21:18:14.284892+0000","last_fullsized":"2023-09-23T21:18:14.284892+0000","mapping_epoch":544,"log_start":"555'3596","ondisk_log_start":"555'3596","created":544,"last_epoch_clean":560,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_clean_scrub_stamp":"2023-09-23T21:15:24.860968+0000","objects_scrubbed":0,"log_size":4031,"log_dups_size":0,"ondisk_log_size":4031,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":62783488,"num_objects":958,"num_object_clones":0,"num_object_copies":2874,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":958,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5310,"num_write_kb":200576,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,2],"acting":[3,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.b","version":"79'161","reported_seq":615,"reported_epoch":597,"state":"active+clean","last_fresh":"2023-09-23T21:18:08.520982+0000","last_change":"2023-09-23T21:15:36.340009+0000","last_active":"2023-09-23T21:18:08.520982+0000","last_peered":"2023-09-23T21:18:08.520982+0000","last_clean":"2023-09-23T21:18:08.520982+0000","last_became_active":"2023-09-23T21:13:13.372328+0000","last_became_peered":"2023-09-23T21:13:13.372328+0000","last_unstale":"2023-09-23T21:18:08.520982+0000","last_undegraded":"2023-09-23T21:18:08.520982+0000","last_fullsized":"2023-09-23T21:18:08.520982+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":495,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:15:36.339885+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:15:36.339885+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:01:26.605006+0000","scrub_duration":0.0068473279999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":176,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-09-23T21:18:09.519046+0000","last_change":"2023-09-23T21:16:42.425103+0000","last_active":"2023-09-23T21:18:09.519046+0000","last_peered":"2023-09-23T21:18:09.519046+0000","last_clean":"2023-09-23T21:18:09.519046+0000","last_became_active":"2023-09-23T21:16:42.424543+0000","last_became_peered":"2023-09-23T21:16:42.424543+0000","last_unstale":"2023-09-23T21:18:09.519046+0000","last_undegraded":"2023-09-23T21:18:09.519046+0000","last_fullsized":"2023-09-23T21:18:09.519046+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:11:28.885246+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":530,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-09-23T21:18:09.520089+0000","last_change":"2023-09-23T21:12:10.811698+0000","last_active":"2023-09-23T21:18:09.520089+0000","last_peered":"2023-09-23T21:18:09.520089+0000","last_clean":"2023-09-23T21:18:09.520089+0000","last_became_active":"2023-09-23T21:12:10.789656+0000","last_became_peered":"2023-09-23T21:12:10.789656+0000","last_unstale":"2023-09-23T21:18:09.520089+0000","last_undegraded":"2023-09-23T21:18:09.520089+0000","last_fullsized":"2023-09-23T21:18:09.520089+0000","mapping_epoch":470,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:39:15.112091+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,5],"acting":[4,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.0","version":"599'7356","reported_seq":9064,"reported_epoch":599,"state":"active+clean","last_fresh":"2023-09-23T21:18:14.393469+0000","last_change":"2023-09-23T21:18:10.533031+0000","last_active":"2023-09-23T21:18:14.393469+0000","last_peered":"2023-09-23T21:18:14.393469+0000","last_clean":"2023-09-23T21:18:14.393469+0000","last_became_active":"2023-09-23T21:18:10.519762+0000","last_became_peered":"2023-09-23T21:18:10.519762+0000","last_unstale":"2023-09-23T21:18:14.393469+0000","last_undegraded":"2023-09-23T21:18:14.393469+0000","last_fullsized":"2023-09-23T21:18:14.393469+0000","mapping_epoch":598,"log_start":"550'3122","ondisk_log_start":"550'3122","created":544,"last_epoch_clean":599,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_clean_scrub_stamp":"2023-09-23T21:15:24.860968+0000","objects_scrubbed":0,"log_size":4234,"log_dups_size":0,"ondisk_log_size":4234,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":61636608,"num_objects":940,"num_object_clones":0,"num_object_copies":2820,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":940,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5098,"num_write_kb":193248,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":803,"num_bytes_recovered":42336256,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.a","version":"79'161","reported_seq":759,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-09-23T21:18:09.533425+0000","last_change":"2023-09-23T21:15:46.420777+0000","last_active":"2023-09-23T21:18:09.533425+0000","last_peered":"2023-09-23T21:18:09.533425+0000","last_clean":"2023-09-23T21:18:09.533425+0000","last_became_active":"2023-09-23T21:12:30.437427+0000","last_became_peered":"2023-09-23T21:12:30.437427+0000","last_unstale":"2023-09-23T21:18:09.533425+0000","last_undegraded":"2023-09-23T21:18:09.533425+0000","last_fullsized":"2023-09-23T21:18:09.533425+0000","mapping_epoch":446,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:15:46.420650+0000","last_deep_scrub":"79'161","last_deep_scrub_stamp":"2023-09-23T21:05:32.630292+0000","last_clean_scrub_stamp":"2023-09-23T21:15:46.420650+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:50:23.452475+0000","scrub_duration":0.0072898930000000004,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,4],"acting":[2,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":465,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-09-23T21:18:09.515510+0000","last_change":"2023-09-23T21:16:42.447403+0000","last_active":"2023-09-23T21:18:09.515510+0000","last_peered":"2023-09-23T21:18:09.515510+0000","last_clean":"2023-09-23T21:18:09.515510+0000","last_became_active":"2023-09-23T21:16:42.447080+0000","last_became_peered":"2023-09-23T21:16:42.447080+0000","last_unstale":"2023-09-23T21:18:09.515510+0000","last_undegraded":"2023-09-23T21:18:09.515510+0000","last_fullsized":"2023-09-23T21:18:09.515510+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:15:19.446334+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,4],"acting":[3,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":596,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-09-23T21:18:09.518112+0000","last_change":"2023-09-23T21:16:42.518739+0000","last_active":"2023-09-23T21:18:09.518112+0000","last_peered":"2023-09-23T21:18:09.518112+0000","last_clean":"2023-09-23T21:18:09.518112+0000","last_became_active":"2023-09-23T21:16:42.517945+0000","last_became_peered":"2023-09-23T21:16:42.517945+0000","last_unstale":"2023-09-23T21:18:09.518112+0000","last_undegraded":"2023-09-23T21:18:09.518112+0000","last_fullsized":"2023-09-23T21:18:09.518112+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:47:26.236933+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.d","version":"599'10378","reported_seq":10444,"reported_epoch":599,"state":"active+clean","last_fresh":"2023-09-23T21:18:11.653926+0000","last_change":"2023-09-23T21:15:25.892834+0000","last_active":"2023-09-23T21:18:11.653926+0000","last_peered":"2023-09-23T21:18:11.653926+0000","last_clean":"2023-09-23T21:18:11.653926+0000","last_became_active":"2023-09-23T21:15:25.892519+0000","last_became_peered":"2023-09-23T21:15:25.892519+0000","last_unstale":"2023-09-23T21:18:11.653926+0000","last_undegraded":"2023-09-23T21:18:11.653926+0000","last_fullsized":"2023-09-23T21:18:11.653926+0000","mapping_epoch":544,"log_start":"569'5372","ondisk_log_start":"569'5372","created":544,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_clean_scrub_stamp":"2023-09-23T21:15:24.860968+0000","objects_scrubbed":0,"log_size":5006,"log_dups_size":0,"ondisk_log_size":5006,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:46:28.375665+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":145031192,"num_objects":2214,"num_object_clones":0,"num_object_copies":6642,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2214,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":10378,"num_write_kb":402881,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,5],"acting":[2,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.7","version":"79'161","reported_seq":739,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-09-23T21:18:09.517818+0000","last_change":"2023-09-23T21:16:45.425645+0000","last_active":"2023-09-23T21:18:09.517818+0000","last_peered":"2023-09-23T21:18:09.517818+0000","last_clean":"2023-09-23T21:18:09.517818+0000","last_became_active":"2023-09-23T21:16:45.425241+0000","last_became_peered":"2023-09-23T21:16:45.425241+0000","last_unstale":"2023-09-23T21:18:09.517818+0000","last_undegraded":"2023-09-23T21:18:09.517818+0000","last_fullsized":"2023-09-23T21:18:09.517818+0000","mapping_epoch":567,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":568,"parent":"0.0","parent_split_bits":3,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:15:51.016427+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:15:51.016427+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:49:31.176462+0000","scrub_duration":0.0017648640000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":547,"reported_epoch":597,"state":"active+clean","last_fresh":"2023-09-23T21:18:08.521443+0000","last_change":"2023-09-23T21:16:42.518826+0000","last_active":"2023-09-23T21:18:08.521443+0000","last_peered":"2023-09-23T21:18:08.521443+0000","last_clean":"2023-09-23T21:18:08.521443+0000","last_became_active":"2023-09-23T21:16:42.518488+0000","last_became_peered":"2023-09-23T21:16:42.518488+0000","last_unstale":"2023-09-23T21:18:08.521443+0000","last_undegraded":"2023-09-23T21:18:08.521443+0000","last_fullsized":"2023-09-23T21:18:08.521443+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:42:44.144383+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,2],"acting":[0,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":276,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-09-23T21:18:09.532285+0000","last_change":"2023-09-23T21:05:04.326889+0000","last_active":"2023-09-23T21:18:09.532285+0000","last_peered":"2023-09-23T21:18:09.532285+0000","last_clean":"2023-09-23T21:18:09.532285+0000","last_became_active":"2023-09-23T21:05:04.326032+0000","last_became_peered":"2023-09-23T21:05:04.326032+0000","last_unstale":"2023-09-23T21:18:09.532285+0000","last_undegraded":"2023-09-23T21:18:09.532285+0000","last_fullsized":"2023-09-23T21:18:09.532285+0000","mapping_epoch":332,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:00:17.507889+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":608,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-09-23T21:18:09.532144+0000","last_change":"2023-09-23T21:13:13.285608+0000","last_active":"2023-09-23T21:18:09.532144+0000","last_peered":"2023-09-23T21:18:09.532144+0000","last_clean":"2023-09-23T21:18:09.532144+0000","last_became_active":"2023-09-23T21:13:13.285255+0000","last_became_peered":"2023-09-23T21:13:13.285255+0000","last_unstale":"2023-09-23T21:18:09.532144+0000","last_undegraded":"2023-09-23T21:18:09.532144+0000","last_fullsized":"2023-09-23T21:18:09.532144+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":495,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:42:30.087599+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.b","version":"599'10098","reported_seq":10163,"reported_epoch":599,"state":"active+clean","last_fresh":"2023-09-23T21:18:14.397302+0000","last_change":"2023-09-23T21:15:25.906791+0000","last_active":"2023-09-23T21:18:14.397302+0000","last_peered":"2023-09-23T21:18:14.397302+0000","last_clean":"2023-09-23T21:18:14.397302+0000","last_became_active":"2023-09-23T21:15:25.906512+0000","last_became_peered":"2023-09-23T21:15:25.906512+0000","last_unstale":"2023-09-23T21:18:14.397302+0000","last_undegraded":"2023-09-23T21:18:14.397302+0000","last_fullsized":"2023-09-23T21:18:14.397302+0000","mapping_epoch":544,"log_start":"574'6545","ondisk_log_start":"574'6545","created":544,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_clean_scrub_stamp":"2023-09-23T21:15:24.860968+0000","objects_scrubbed":0,"log_size":3553,"log_dups_size":0,"ondisk_log_size":3553,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:58:51.749775+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":121372672,"num_objects":1852,"num_object_clones":0,"num_object_copies":5556,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1852,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10098,"num_write_kb":382400,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1","version":"79'161","reported_seq":682,"reported_epoch":597,"state":"active+clean","last_fresh":"2023-09-23T21:18:08.521936+0000","last_change":"2023-09-23T21:15:34.345290+0000","last_active":"2023-09-23T21:18:08.521936+0000","last_peered":"2023-09-23T21:18:08.521936+0000","last_clean":"2023-09-23T21:18:08.521936+0000","last_became_active":"2023-09-23T21:13:12.203380+0000","last_became_peered":"2023-09-23T21:13:12.203380+0000","last_unstale":"2023-09-23T21:18:08.521936+0000","last_undegraded":"2023-09-23T21:18:08.521936+0000","last_fullsized":"2023-09-23T21:18:08.521936+0000","mapping_epoch":493,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:15:34.345138+0000","last_deep_scrub":"79'161","last_deep_scrub_stamp":"2023-09-23T21:05:30.652189+0000","last_clean_scrub_stamp":"2023-09-23T21:15:34.345138+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:47:50.349409+0000","scrub_duration":0.0057520000000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":266,"reported_epoch":597,"state":"active+clean","last_fresh":"2023-09-23T21:18:08.523719+0000","last_change":"2023-09-23T21:13:13.238617+0000","last_active":"2023-09-23T21:18:08.523719+0000","last_peered":"2023-09-23T21:18:08.523719+0000","last_clean":"2023-09-23T21:18:08.523719+0000","last_became_active":"2023-09-23T21:13:13.232926+0000","last_became_peered":"2023-09-23T21:13:13.232926+0000","last_unstale":"2023-09-23T21:18:08.523719+0000","last_undegraded":"2023-09-23T21:18:08.523719+0000","last_fullsized":"2023-09-23T21:18:08.523719+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":495,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:17:21.823772+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":532,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-09-23T21:18:09.519642+0000","last_change":"2023-09-23T21:13:14.144918+0000","last_active":"2023-09-23T21:18:09.519642+0000","last_peered":"2023-09-23T21:18:09.519642+0000","last_clean":"2023-09-23T21:18:09.519642+0000","last_became_active":"2023-09-23T21:13:14.132388+0000","last_became_peered":"2023-09-23T21:13:14.132388+0000","last_unstale":"2023-09-23T21:18:09.519642+0000","last_undegraded":"2023-09-23T21:18:09.519642+0000","last_fullsized":"2023-09-23T21:18:09.519642+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":495,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:56:02.619905+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,5],"acting":[4,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":438,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-09-23T21:18:09.536929+0000","last_change":"2023-09-23T21:16:42.491111+0000","last_active":"2023-09-23T21:18:09.536929+0000","last_peered":"2023-09-23T21:18:09.536929+0000","last_clean":"2023-09-23T21:18:09.536929+0000","last_became_active":"2023-09-23T21:16:42.490275+0000","last_became_peered":"2023-09-23T21:16:42.490275+0000","last_unstale":"2023-09-23T21:18:09.536929+0000","last_undegraded":"2023-09-23T21:18:09.536929+0000","last_fullsized":"2023-09-23T21:18:09.536929+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:39:11.652564+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,7],"acting":[2,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"11.8","version":"598'7262","reported_seq":7336,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-09-23T21:18:09.840533+0000","last_change":"2023-09-23T21:16:32.898234+0000","last_active":"2023-09-23T21:18:09.840533+0000","last_peered":"2023-09-23T21:18:09.840533+0000","last_clean":"2023-09-23T21:18:09.840533+0000","last_became_active":"2023-09-23T21:16:32.894560+0000","last_became_peered":"2023-09-23T21:16:32.894560+0000","last_unstale":"2023-09-23T21:18:09.840533+0000","last_undegraded":"2023-09-23T21:18:09.840533+0000","last_fullsized":"2023-09-23T21:18:09.840533+0000","mapping_epoch":544,"log_start":"549'2263","ondisk_log_start":"549'2263","created":544,"last_epoch_clean":560,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_clean_scrub_stamp":"2023-09-23T21:15:24.860968+0000","objects_scrubbed":0,"log_size":4999,"log_dups_size":0,"ondisk_log_size":4999,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70090752,"num_objects":1069,"num_object_clones":0,"num_object_copies":3207,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1069,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5037,"num_write_kb":195424,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,5],"acting":[0,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.2","version":"79'161","reported_seq":913,"reported_epoch":597,"state":"active+clean","last_fresh":"2023-09-23T21:18:08.521641+0000","last_change":"2023-09-23T21:16:45.417075+0000","last_active":"2023-09-23T21:18:08.521641+0000","last_peered":"2023-09-23T21:18:08.521641+0000","last_clean":"2023-09-23T21:18:08.521641+0000","last_became_active":"2023-09-23T21:16:45.416752+0000","last_became_peered":"2023-09-23T21:16:45.416752+0000","last_unstale":"2023-09-23T21:18:08.521641+0000","last_undegraded":"2023-09-23T21:18:08.521641+0000","last_fullsized":"2023-09-23T21:18:08.521641+0000","mapping_epoch":567,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":568,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:24.802451+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:24.802451+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:01:51.129779+0000","scrub_duration":0.0090861250000000005,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,4],"acting":[0,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":609,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-09-23T21:18:09.532308+0000","last_change":"2023-09-23T21:13:12.206980+0000","last_active":"2023-09-23T21:18:09.532308+0000","last_peered":"2023-09-23T21:18:09.532308+0000","last_clean":"2023-09-23T21:18:09.532308+0000","last_became_active":"2023-09-23T21:13:12.206650+0000","last_became_peered":"2023-09-23T21:13:12.206650+0000","last_unstale":"2023-09-23T21:18:09.532308+0000","last_undegraded":"2023-09-23T21:18:09.532308+0000","last_fullsized":"2023-09-23T21:18:09.532308+0000","mapping_epoch":493,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:10:01.892936+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":593,"reported_epoch":599,"state":"active+clean","last_fresh":"2023-09-23T21:18:10.560284+0000","last_change":"2023-09-23T21:13:13.407073+0000","last_active":"2023-09-23T21:18:10.560284+0000","last_peered":"2023-09-23T21:18:10.560284+0000","last_clean":"2023-09-23T21:18:10.560284+0000","last_became_active":"2023-09-23T21:13:13.395047+0000","last_became_peered":"2023-09-23T21:13:13.395047+0000","last_unstale":"2023-09-23T21:18:10.560284+0000","last_undegraded":"2023-09-23T21:18:10.560284+0000","last_fullsized":"2023-09-23T21:18:10.560284+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":495,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:04:43.384190+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,5],"acting":[2,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":127,"reported_epoch":597,"state":"active+clean","last_fresh":"2023-09-23T21:18:08.524784+0000","last_change":"2023-09-23T21:16:42.517729+0000","last_active":"2023-09-23T21:18:08.524784+0000","last_peered":"2023-09-23T21:18:08.524784+0000","last_clean":"2023-09-23T21:18:08.524784+0000","last_became_active":"2023-09-23T21:16:42.517373+0000","last_became_peered":"2023-09-23T21:16:42.517373+0000","last_unstale":"2023-09-23T21:18:08.524784+0000","last_undegraded":"2023-09-23T21:18:08.524784+0000","last_fullsized":"2023-09-23T21:18:08.524784+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:05:57.370550+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.a","version":"599'10248","reported_seq":10313,"reported_epoch":599,"state":"active+clean","last_fresh":"2023-09-23T21:18:11.600502+0000","last_change":"2023-09-23T21:15:25.893846+0000","last_active":"2023-09-23T21:18:11.600502+0000","last_peered":"2023-09-23T21:18:11.600502+0000","last_clean":"2023-09-23T21:18:11.600502+0000","last_became_active":"2023-09-23T21:15:25.893452+0000","last_became_peered":"2023-09-23T21:15:25.893452+0000","last_unstale":"2023-09-23T21:18:11.600502+0000","last_undegraded":"2023-09-23T21:18:11.600502+0000","last_fullsized":"2023-09-23T21:18:11.600502+0000","mapping_epoch":544,"log_start":"569'5225","ondisk_log_start":"569'5225","created":544,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_clean_scrub_stamp":"2023-09-23T21:15:24.860968+0000","objects_scrubbed":0,"log_size":5023,"log_dups_size":0,"ondisk_log_size":5023,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:42:32.842297+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":138018816,"num_objects":2106,"num_object_clones":0,"num_object_copies":6318,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2106,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10248,"num_write_kb":395328,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,5],"acting":[2,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":404,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-09-23T21:18:09.533836+0000","last_change":"2023-09-23T21:16:42.423216+0000","last_active":"2023-09-23T21:18:09.533836+0000","last_peered":"2023-09-23T21:18:09.533836+0000","last_clean":"2023-09-23T21:18:09.533836+0000","last_became_active":"2023-09-23T21:16:42.422856+0000","last_became_peered":"2023-09-23T21:16:42.422856+0000","last_unstale":"2023-09-23T21:18:09.533836+0000","last_undegraded":"2023-09-23T21:18:09.533836+0000","last_fullsized":"2023-09-23T21:18:09.533836+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:06:54.105798+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"79'161","reported_seq":816,"reported_epoch":597,"state":"active+clean","last_fresh":"2023-09-23T21:18:08.516037+0000","last_change":"2023-09-23T21:16:45.428032+0000","last_active":"2023-09-23T21:18:08.516037+0000","last_peered":"2023-09-23T21:18:08.516037+0000","last_clean":"2023-09-23T21:18:08.516037+0000","last_became_active":"2023-09-23T21:16:45.427674+0000","last_became_peered":"2023-09-23T21:16:45.427674+0000","last_unstale":"2023-09-23T21:18:08.516037+0000","last_undegraded":"2023-09-23T21:18:08.516037+0000","last_fullsized":"2023-09-23T21:18:08.516037+0000","mapping_epoch":567,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":568,"parent":"0.0","parent_split_bits":0,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:20.738273+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:20.738273+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:39:42.441711+0000","scrub_duration":0.0088336339999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,2],"acting":[0,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":606,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-09-23T21:18:09.532051+0000","last_change":"2023-09-23T21:11:08.760417+0000","last_active":"2023-09-23T21:18:09.532051+0000","last_peered":"2023-09-23T21:18:09.532051+0000","last_clean":"2023-09-23T21:18:09.532051+0000","last_became_active":"2023-09-23T21:11:08.760150+0000","last_became_peered":"2023-09-23T21:11:08.760150+0000","last_unstale":"2023-09-23T21:18:09.532051+0000","last_undegraded":"2023-09-23T21:18:09.532051+0000","last_fullsized":"2023-09-23T21:18:09.532051+0000","mapping_epoch":443,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":444,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:23:10.849768+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":193,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-09-23T21:18:09.519099+0000","last_change":"2023-09-23T21:16:42.424717+0000","last_active":"2023-09-23T21:18:09.519099+0000","last_peered":"2023-09-23T21:18:09.519099+0000","last_clean":"2023-09-23T21:18:09.519099+0000","last_became_active":"2023-09-23T21:16:42.424376+0000","last_became_peered":"2023-09-23T21:16:42.424376+0000","last_unstale":"2023-09-23T21:18:09.519099+0000","last_undegraded":"2023-09-23T21:18:09.519099+0000","last_fullsized":"2023-09-23T21:18:09.519099+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:25:14.162321+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.9","version":"599'7528","reported_seq":9352,"reported_epoch":599,"state":"active+clean","last_fresh":"2023-09-23T21:18:14.397590+0000","last_change":"2023-09-23T21:17:58.411362+0000","last_active":"2023-09-23T21:18:14.397590+0000","last_peered":"2023-09-23T21:18:14.397590+0000","last_clean":"2023-09-23T21:18:14.397590+0000","last_became_active":"2023-09-23T21:17:58.399421+0000","last_became_peered":"2023-09-23T21:17:58.399421+0000","last_unstale":"2023-09-23T21:18:14.397590+0000","last_undegraded":"2023-09-23T21:18:14.397590+0000","last_fullsized":"2023-09-23T21:18:14.397590+0000","mapping_epoch":593,"log_start":"555'3409","ondisk_log_start":"555'3409","created":544,"last_epoch_clean":594,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_clean_scrub_stamp":"2023-09-23T21:15:24.860968+0000","objects_scrubbed":0,"log_size":4119,"log_dups_size":0,"ondisk_log_size":4119,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":67928064,"num_objects":1036,"num_object_clones":0,"num_object_copies":3108,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1036,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5244,"num_write_kb":200992,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":860,"num_bytes_recovered":56393728,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,7],"acting":[3,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.3","version":"79'161","reported_seq":901,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-09-23T21:18:09.519122+0000","last_change":"2023-09-23T21:13:33.771279+0000","last_active":"2023-09-23T21:18:09.519122+0000","last_peered":"2023-09-23T21:18:09.519122+0000","last_clean":"2023-09-23T21:18:09.519122+0000","last_became_active":"2023-09-23T21:13:33.770980+0000","last_became_peered":"2023-09-23T21:13:33.770980+0000","last_unstale":"2023-09-23T21:18:09.519122+0000","last_undegraded":"2023-09-23T21:18:09.519122+0000","last_fullsized":"2023-09-23T21:18:09.519122+0000","mapping_epoch":504,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":505,"parent":"0.0","parent_split_bits":3,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:05:57.832302+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:05:57.832302+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:19:42.309604+0000","scrub_duration":0.010206847999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":434,"reported_epoch":597,"state":"active+clean","last_fresh":"2023-09-23T21:18:08.516592+0000","last_change":"2023-09-23T21:16:42.455365+0000","last_active":"2023-09-23T21:18:08.516592+0000","last_peered":"2023-09-23T21:18:08.516592+0000","last_clean":"2023-09-23T21:18:08.516592+0000","last_became_active":"2023-09-23T21:16:42.452829+0000","last_became_peered":"2023-09-23T21:16:42.452829+0000","last_unstale":"2023-09-23T21:18:08.516592+0000","last_undegraded":"2023-09-23T21:18:08.516592+0000","last_fullsized":"2023-09-23T21:18:08.516592+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:49:26.303389+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":598,"reported_epoch":597,"state":"active+clean","last_fresh":"2023-09-23T21:18:08.521763+0000","last_change":"2023-09-23T21:13:44.409212+0000","last_active":"2023-09-23T21:18:08.521763+0000","last_peered":"2023-09-23T21:18:08.521763+0000","last_clean":"2023-09-23T21:18:08.521763+0000","last_became_active":"2023-09-23T21:13:44.408958+0000","last_became_peered":"2023-09-23T21:13:44.408958+0000","last_unstale":"2023-09-23T21:18:08.521763+0000","last_undegraded":"2023-09-23T21:18:08.521763+0000","last_fullsized":"2023-09-23T21:18:08.521763+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:03:52.749964+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":127,"reported_epoch":597,"state":"active+clean","last_fresh":"2023-09-23T21:18:08.522248+0000","last_change":"2023-09-23T21:16:42.427490+0000","last_active":"2023-09-23T21:18:08.522248+0000","last_peered":"2023-09-23T21:18:08.522248+0000","last_clean":"2023-09-23T21:18:08.522248+0000","last_became_active":"2023-09-23T21:16:42.426566+0000","last_became_peered":"2023-09-23T21:16:42.426566+0000","last_unstale":"2023-09-23T21:18:08.522248+0000","last_undegraded":"2023-09-23T21:18:08.522248+0000","last_fullsized":"2023-09-23T21:18:08.522248+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:20:26.396005+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,7],"acting":[0,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.c","version":"599'10100","reported_seq":13270,"reported_epoch":599,"state":"active+clean","last_fresh":"2023-09-23T21:18:11.613847+0000","last_change":"2023-09-23T21:17:21.358806+0000","last_active":"2023-09-23T21:18:11.613847+0000","last_peered":"2023-09-23T21:18:11.613847+0000","last_clean":"2023-09-23T21:18:11.613847+0000","last_became_active":"2023-09-23T21:17:21.346629+0000","last_became_peered":"2023-09-23T21:17:21.346629+0000","last_unstale":"2023-09-23T21:18:11.613847+0000","last_undegraded":"2023-09-23T21:18:11.613847+0000","last_fullsized":"2023-09-23T21:18:11.613847+0000","mapping_epoch":576,"log_start":"568'5000","ondisk_log_start":"568'5000","created":544,"last_epoch_clean":577,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_clean_scrub_stamp":"2023-09-23T21:15:24.860968+0000","objects_scrubbed":0,"log_size":5100,"log_dups_size":0,"ondisk_log_size":5100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:59:48.173244+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":136052736,"num_objects":2076,"num_object_clones":0,"num_object_copies":6228,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2076,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10100,"num_write_kb":389632,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1539,"num_bytes_recovered":100859904,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,7],"acting":[2,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.6","version":"79'161","reported_seq":530,"reported_epoch":597,"state":"active+clean","last_fresh":"2023-09-23T21:18:08.522768+0000","last_change":"2023-09-23T21:15:43.333919+0000","last_active":"2023-09-23T21:18:08.522768+0000","last_peered":"2023-09-23T21:18:08.522768+0000","last_clean":"2023-09-23T21:18:08.522768+0000","last_became_active":"2023-09-23T21:13:33.769303+0000","last_became_peered":"2023-09-23T21:13:33.769303+0000","last_unstale":"2023-09-23T21:18:08.522768+0000","last_undegraded":"2023-09-23T21:18:08.522768+0000","last_fullsized":"2023-09-23T21:18:08.522768+0000","mapping_epoch":504,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":505,"parent":"0.0","parent_split_bits":3,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:15:43.333733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:15:43.333733+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:42:29.787357+0000","scrub_duration":0.0092107530000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":44,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-09-23T21:18:09.518925+0000","last_change":"2023-09-23T21:16:42.483813+0000","last_active":"2023-09-23T21:18:09.518925+0000","last_peered":"2023-09-23T21:18:09.518925+0000","last_clean":"2023-09-23T21:18:09.518925+0000","last_became_active":"2023-09-23T21:16:42.482847+0000","last_became_peered":"2023-09-23T21:16:42.482847+0000","last_unstale":"2023-09-23T21:18:09.518925+0000","last_undegraded":"2023-09-23T21:18:09.518925+0000","last_fullsized":"2023-09-23T21:18:09.518925+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:38:57.905679+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":483,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-09-23T21:18:09.537233+0000","last_change":"2023-09-23T21:13:30.790590+0000","last_active":"2023-09-23T21:18:09.537233+0000","last_peered":"2023-09-23T21:18:09.537233+0000","last_clean":"2023-09-23T21:18:09.537233+0000","last_became_active":"2023-09-23T21:13:30.790253+0000","last_became_peered":"2023-09-23T21:13:30.790253+0000","last_unstale":"2023-09-23T21:18:09.537233+0000","last_undegraded":"2023-09-23T21:18:09.537233+0000","last_fullsized":"2023-09-23T21:18:09.537233+0000","mapping_epoch":501,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:23:41.843250+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,0],"acting":[2,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":563,"reported_epoch":597,"state":"active+clean","last_fresh":"2023-09-23T21:18:08.521454+0000","last_change":"2023-09-23T21:16:42.428535+0000","last_active":"2023-09-23T21:18:08.521454+0000","last_peered":"2023-09-23T21:18:08.521454+0000","last_clean":"2023-09-23T21:18:08.521454+0000","last_became_active":"2023-09-23T21:16:42.428149+0000","last_became_peered":"2023-09-23T21:16:42.428149+0000","last_unstale":"2023-09-23T21:18:08.521454+0000","last_undegraded":"2023-09-23T21:18:08.521454+0000","last_fullsized":"2023-09-23T21:18:08.521454+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:38:59.979788+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.f","version":"599'10123","reported_seq":13742,"reported_epoch":599,"state":"active+clean","last_fresh":"2023-09-23T21:18:11.636653+0000","last_change":"2023-09-23T21:17:03.878062+0000","last_active":"2023-09-23T21:18:11.636653+0000","last_peered":"2023-09-23T21:18:11.636653+0000","last_clean":"2023-09-23T21:18:11.636653+0000","last_became_active":"2023-09-23T21:17:03.870998+0000","last_became_peered":"2023-09-23T21:17:03.870998+0000","last_unstale":"2023-09-23T21:18:11.636653+0000","last_undegraded":"2023-09-23T21:18:11.636653+0000","last_fullsized":"2023-09-23T21:18:11.636653+0000","mapping_epoch":572,"log_start":"569'5100","ondisk_log_start":"569'5100","created":544,"last_epoch_clean":573,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_clean_scrub_stamp":"2023-09-23T21:15:24.860968+0000","objects_scrubbed":0,"log_size":5023,"log_dups_size":0,"ondisk_log_size":5023,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:19:31.942955+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":134021120,"num_objects":2045,"num_object_clones":0,"num_object_copies":6135,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2045,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10123,"num_write_kb":389376,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1762,"num_bytes_recovered":115474432,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,0],"acting":[2,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.5","version":"79'161","reported_seq":636,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-09-23T21:18:09.519013+0000","last_change":"2023-09-23T21:15:41.108638+0000","last_active":"2023-09-23T21:18:09.519013+0000","last_peered":"2023-09-23T21:18:09.519013+0000","last_clean":"2023-09-23T21:18:09.519013+0000","last_became_active":"2023-09-23T21:13:38.373633+0000","last_became_peered":"2023-09-23T21:13:38.373633+0000","last_unstale":"2023-09-23T21:18:09.519013+0000","last_undegraded":"2023-09-23T21:18:09.519013+0000","last_fullsized":"2023-09-23T21:18:09.519013+0000","mapping_epoch":507,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":3,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:15:41.108579+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:15:41.108579+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:21:58.075649+0000","scrub_duration":0.0061511960000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":166,"reported_epoch":597,"state":"active+clean","last_fresh":"2023-09-23T21:18:08.527587+0000","last_change":"2023-09-23T21:17:02.204086+0000","last_active":"2023-09-23T21:18:08.527587+0000","last_peered":"2023-09-23T21:18:08.527587+0000","last_clean":"2023-09-23T21:18:08.527587+0000","last_became_active":"2023-09-23T21:16:42.517549+0000","last_became_peered":"2023-09-23T21:16:42.517549+0000","last_unstale":"2023-09-23T21:18:08.527587+0000","last_undegraded":"2023-09-23T21:18:08.527587+0000","last_fullsized":"2023-09-23T21:18:08.527587+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:43:07.991844+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":579,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-09-23T21:18:09.519972+0000","last_change":"2023-09-23T21:13:30.807520+0000","last_active":"2023-09-23T21:18:09.519972+0000","last_peered":"2023-09-23T21:18:09.519972+0000","last_clean":"2023-09-23T21:18:09.519972+0000","last_became_active":"2023-09-23T21:13:30.807222+0000","last_became_peered":"2023-09-23T21:13:30.807222+0000","last_unstale":"2023-09-23T21:18:09.519972+0000","last_undegraded":"2023-09-23T21:18:09.519972+0000","last_fullsized":"2023-09-23T21:18:09.519972+0000","mapping_epoch":501,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:40:31.989749+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":460,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-09-23T21:18:09.515590+0000","last_change":"2023-09-23T21:16:42.463411+0000","last_active":"2023-09-23T21:18:09.515590+0000","last_peered":"2023-09-23T21:18:09.515590+0000","last_clean":"2023-09-23T21:18:09.515590+0000","last_became_active":"2023-09-23T21:16:42.463020+0000","last_became_peered":"2023-09-23T21:16:42.463020+0000","last_unstale":"2023-09-23T21:18:09.515590+0000","last_undegraded":"2023-09-23T21:18:09.515590+0000","last_fullsized":"2023-09-23T21:18:09.515590+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:07:34.346129+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.e","version":"599'10171","reported_seq":14368,"reported_epoch":599,"state":"active+clean","last_fresh":"2023-09-23T21:18:12.049915+0000","last_change":"2023-09-23T21:17:34.456174+0000","last_active":"2023-09-23T21:18:12.049915+0000","last_peered":"2023-09-23T21:18:12.049915+0000","last_clean":"2023-09-23T21:18:12.049915+0000","last_became_active":"2023-09-23T21:17:34.444637+0000","last_became_peered":"2023-09-23T21:17:34.444637+0000","last_unstale":"2023-09-23T21:18:12.049915+0000","last_undegraded":"2023-09-23T21:18:12.049915+0000","last_fullsized":"2023-09-23T21:18:12.049915+0000","mapping_epoch":582,"log_start":"570'5300","ondisk_log_start":"570'5300","created":544,"last_epoch_clean":583,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_clean_scrub_stamp":"2023-09-23T21:15:24.860968+0000","objects_scrubbed":0,"log_size":4871,"log_dups_size":0,"ondisk_log_size":4871,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:17:47.772535+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":136642560,"num_objects":2085,"num_object_clones":0,"num_object_copies":6255,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2085,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10171,"num_write_kb":392192,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2052,"num_bytes_recovered":134479872,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.4","version":"79'161","reported_seq":1005,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-09-23T21:18:09.516178+0000","last_change":"2023-09-23T21:16:45.426136+0000","last_active":"2023-09-23T21:18:09.516178+0000","last_peered":"2023-09-23T21:18:09.516178+0000","last_clean":"2023-09-23T21:18:09.516178+0000","last_became_active":"2023-09-23T21:16:45.425669+0000","last_became_peered":"2023-09-23T21:16:45.425669+0000","last_unstale":"2023-09-23T21:18:09.516178+0000","last_undegraded":"2023-09-23T21:18:09.516178+0000","last_fullsized":"2023-09-23T21:18:09.516178+0000","mapping_epoch":567,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":568,"parent":"0.0","parent_split_bits":3,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:20.012764+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:20.012764+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:52:39.142509+0000","scrub_duration":0.0045251919999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":557,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-09-23T21:18:09.536073+0000","last_change":"2023-09-23T21:16:42.459732+0000","last_active":"2023-09-23T21:18:09.536073+0000","last_peered":"2023-09-23T21:18:09.536073+0000","last_clean":"2023-09-23T21:18:09.536073+0000","last_became_active":"2023-09-23T21:16:42.459413+0000","last_became_peered":"2023-09-23T21:16:42.459413+0000","last_unstale":"2023-09-23T21:18:09.536073+0000","last_undegraded":"2023-09-23T21:18:09.536073+0000","last_fullsized":"2023-09-23T21:18:09.536073+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:37:44.548863+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,3],"acting":[2,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.6","version":"70'1","reported_seq":599,"reported_epoch":597,"state":"active+clean","last_fresh":"2023-09-23T21:18:08.522688+0000","last_change":"2023-09-23T21:17:02.208077+0000","last_active":"2023-09-23T21:18:08.522688+0000","last_peered":"2023-09-23T21:18:08.522688+0000","last_clean":"2023-09-23T21:18:08.522688+0000","last_became_active":"2023-09-23T21:16:42.431308+0000","last_became_peered":"2023-09-23T21:16:42.431308+0000","last_unstale":"2023-09-23T21:18:08.522688+0000","last_undegraded":"2023-09-23T21:18:08.522688+0000","last_fullsized":"2023-09-23T21:18:08.522688+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:16:49.009013+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":173,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-09-23T21:18:09.518407+0000","last_change":"2023-09-23T21:13:12.216895+0000","last_active":"2023-09-23T21:18:09.518407+0000","last_peered":"2023-09-23T21:18:09.518407+0000","last_clean":"2023-09-23T21:18:09.518407+0000","last_became_active":"2023-09-23T21:13:12.216654+0000","last_became_peered":"2023-09-23T21:13:12.216654+0000","last_unstale":"2023-09-23T21:18:09.518407+0000","last_undegraded":"2023-09-23T21:18:09.518407+0000","last_fullsized":"2023-09-23T21:18:09.518407+0000","mapping_epoch":493,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:02:40.430096+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.13","version":"599'7196","reported_seq":8839,"reported_epoch":599,"state":"active+clean","last_fresh":"2023-09-23T21:18:12.218779+0000","last_change":"2023-09-23T21:17:50.364498+0000","last_active":"2023-09-23T21:18:12.218779+0000","last_peered":"2023-09-23T21:18:12.218779+0000","last_clean":"2023-09-23T21:18:12.218779+0000","last_became_active":"2023-09-23T21:17:39.911716+0000","last_became_peered":"2023-09-23T21:17:39.911716+0000","last_unstale":"2023-09-23T21:18:12.218779+0000","last_undegraded":"2023-09-23T21:18:12.218779+0000","last_fullsized":"2023-09-23T21:18:12.218779+0000","mapping_epoch":585,"log_start":"548'1145","ondisk_log_start":"548'1145","created":559,"last_epoch_clean":590,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_clean_scrub_stamp":"2023-09-23T21:15:24.860968+0000","objects_scrubbed":0,"log_size":6051,"log_dups_size":0,"ondisk_log_size":6051,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":65536000,"num_objects":1000,"num_object_clones":0,"num_object_copies":3000,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1000,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4980,"num_write_kb":191360,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":772,"num_bytes_recovered":50528256,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,2],"acting":[7,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.19","version":"79'161","reported_seq":806,"reported_epoch":597,"state":"active+clean","last_fresh":"2023-09-23T21:18:08.515305+0000","last_change":"2023-09-23T21:16:45.428879+0000","last_active":"2023-09-23T21:18:08.515305+0000","last_peered":"2023-09-23T21:18:08.515305+0000","last_clean":"2023-09-23T21:18:08.515305+0000","last_became_active":"2023-09-23T21:16:45.428660+0000","last_became_peered":"2023-09-23T21:16:45.428660+0000","last_unstale":"2023-09-23T21:18:08.515305+0000","last_undegraded":"2023-09-23T21:18:08.515305+0000","last_fullsized":"2023-09-23T21:18:08.515305+0000","mapping_epoch":567,"log_start":"0'0","ondisk_log_start":"0'0","created":479,"last_epoch_clean":568,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:15:48.271791+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:15:48.271791+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:25:26.361045+0000","scrub_duration":0.0057149180000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":126,"reported_epoch":597,"state":"active+clean","last_fresh":"2023-09-23T21:18:08.521410+0000","last_change":"2023-09-23T21:16:42.434830+0000","last_active":"2023-09-23T21:18:08.521410+0000","last_peered":"2023-09-23T21:18:08.521410+0000","last_clean":"2023-09-23T21:18:08.521410+0000","last_became_active":"2023-09-23T21:16:42.430289+0000","last_became_peered":"2023-09-23T21:16:42.430289+0000","last_unstale":"2023-09-23T21:18:08.521410+0000","last_undegraded":"2023-09-23T21:18:08.521410+0000","last_fullsized":"2023-09-23T21:18:08.521410+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:38:48.053766+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":103,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-09-23T21:18:09.537162+0000","last_change":"2023-09-23T21:13:38.373814+0000","last_active":"2023-09-23T21:18:09.537162+0000","last_peered":"2023-09-23T21:18:09.537162+0000","last_clean":"2023-09-23T21:18:09.537162+0000","last_became_active":"2023-09-23T21:13:38.373458+0000","last_became_peered":"2023-09-23T21:13:38.373458+0000","last_unstale":"2023-09-23T21:18:09.537162+0000","last_undegraded":"2023-09-23T21:18:09.537162+0000","last_fullsized":"2023-09-23T21:18:09.537162+0000","mapping_epoch":507,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:23:08.288401+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"11.14","version":"599'7282","reported_seq":8018,"reported_epoch":599,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-09-23T21:18:11.663238+0000","last_change":"2023-09-23T21:18:09.330540+0000","last_active":"2023-09-23T21:18:11.663238+0000","last_peered":"2023-09-23T21:18:11.663238+0000","last_clean":"2023-09-23T21:16:02.247284+0000","last_became_active":"2023-09-23T21:16:43.479075+0000","last_became_peered":"2023-09-23T21:16:43.479075+0000","last_unstale":"2023-09-23T21:18:11.663238+0000","last_undegraded":"2023-09-23T21:16:43.418318+0000","last_fullsized":"2023-09-23T21:16:43.409626+0000","mapping_epoch":565,"log_start":"548'168","ondisk_log_start":"548'168","created":559,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_clean_scrub_stamp":"2023-09-23T21:15:24.860968+0000","objects_scrubbed":0,"log_size":7114,"log_dups_size":0,"ondisk_log_size":7114,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":63242240,"num_objects":965,"num_object_clones":0,"num_object_copies":2895,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":307,"num_objects_misplaced":170,"num_objects_unfound":0,"num_objects_dirty":965,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5045,"num_write_kb":192320,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":326,"num_bytes_recovered":21364736,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,3],"acting":[2,3],"avail_no_missing":["2","3"],"object_location_counts":[{"shards":"0,2,3","objects":170},{"shards":"2,3","objects":488}],"blocked_by":[],"up_primary":7,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.1e","version":"79'161","reported_seq":582,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-09-23T21:18:09.520054+0000","last_change":"2023-09-23T21:16:45.420788+0000","last_active":"2023-09-23T21:18:09.520054+0000","last_peered":"2023-09-23T21:18:09.520054+0000","last_clean":"2023-09-23T21:18:09.520054+0000","last_became_active":"2023-09-23T21:16:45.420448+0000","last_became_peered":"2023-09-23T21:16:45.420448+0000","last_unstale":"2023-09-23T21:18:09.520054+0000","last_undegraded":"2023-09-23T21:18:09.520054+0000","last_fullsized":"2023-09-23T21:18:09.520054+0000","mapping_epoch":567,"log_start":"0'0","ondisk_log_start":"0'0","created":479,"last_epoch_clean":568,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:15:53.061319+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:15:53.061319+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:46:31.982439+0000","scrub_duration":0.007429878,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,4],"acting":[3,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":208,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-09-23T21:18:09.519647+0000","last_change":"2023-09-23T21:16:42.518584+0000","last_active":"2023-09-23T21:18:09.519647+0000","last_peered":"2023-09-23T21:18:09.519647+0000","last_clean":"2023-09-23T21:18:09.519647+0000","last_became_active":"2023-09-23T21:16:42.416632+0000","last_became_peered":"2023-09-23T21:16:42.416632+0000","last_unstale":"2023-09-23T21:18:09.519647+0000","last_undegraded":"2023-09-23T21:18:09.519647+0000","last_fullsized":"2023-09-23T21:18:09.519647+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:02:18.972080+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,7],"acting":[4,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":163,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-09-23T21:18:09.519305+0000","last_change":"2023-09-23T21:12:10.770474+0000","last_active":"2023-09-23T21:18:09.519305+0000","last_peered":"2023-09-23T21:18:09.519305+0000","last_clean":"2023-09-23T21:18:09.519305+0000","last_became_active":"2023-09-23T21:12:10.770184+0000","last_became_peered":"2023-09-23T21:12:10.770184+0000","last_unstale":"2023-09-23T21:18:09.519305+0000","last_undegraded":"2023-09-23T21:18:09.519305+0000","last_fullsized":"2023-09-23T21:18:09.519305+0000","mapping_epoch":470,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:05:09.407151+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.15","version":"599'7475","reported_seq":12715,"reported_epoch":599,"state":"active+clean","last_fresh":"2023-09-23T21:18:12.315352+0000","last_change":"2023-09-23T21:18:06.522430+0000","last_active":"2023-09-23T21:18:12.315352+0000","last_peered":"2023-09-23T21:18:12.315352+0000","last_clean":"2023-09-23T21:18:12.315352+0000","last_became_active":"2023-09-23T21:16:46.418280+0000","last_became_peered":"2023-09-23T21:16:46.418280+0000","last_unstale":"2023-09-23T21:18:12.315352+0000","last_undegraded":"2023-09-23T21:18:12.315352+0000","last_fullsized":"2023-09-23T21:18:12.315352+0000","mapping_epoch":568,"log_start":"548'621","ondisk_log_start":"548'621","created":559,"last_epoch_clean":569,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_clean_scrub_stamp":"2023-09-23T21:15:24.860968+0000","objects_scrubbed":0,"log_size":6854,"log_dups_size":0,"ondisk_log_size":6854,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":65241088,"num_objects":996,"num_object_clones":0,"num_object_copies":2988,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":996,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5180,"num_write_kb":197600,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1697,"num_bytes_recovered":111214592,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":390,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-09-23T21:18:09.533439+0000","last_change":"2023-09-23T21:12:10.771160+0000","last_active":"2023-09-23T21:18:09.533439+0000","last_peered":"2023-09-23T21:18:09.533439+0000","last_clean":"2023-09-23T21:18:09.533439+0000","last_became_active":"2023-09-23T21:12:10.757808+0000","last_became_peered":"2023-09-23T21:12:10.757808+0000","last_unstale":"2023-09-23T21:18:09.533439+0000","last_undegraded":"2023-09-23T21:18:09.533439+0000","last_fullsized":"2023-09-23T21:18:09.533439+0000","mapping_epoch":470,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:30:50.987139+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":173,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-09-23T21:18:09.518709+0000","last_change":"2023-09-23T21:13:12.216602+0000","last_active":"2023-09-23T21:18:09.518709+0000","last_peered":"2023-09-23T21:18:09.518709+0000","last_clean":"2023-09-23T21:18:09.518709+0000","last_became_active":"2023-09-23T21:13:12.216204+0000","last_became_peered":"2023-09-23T21:13:12.216204+0000","last_unstale":"2023-09-23T21:18:09.518709+0000","last_undegraded":"2023-09-23T21:18:09.518709+0000","last_fullsized":"2023-09-23T21:18:09.518709+0000","mapping_epoch":493,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:23:37.043390+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.16","version":"598'7132","reported_seq":7207,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-09-23T21:18:09.854765+0000","last_change":"2023-09-23T21:16:33.955304+0000","last_active":"2023-09-23T21:18:09.854765+0000","last_peered":"2023-09-23T21:18:09.854765+0000","last_clean":"2023-09-23T21:18:09.854765+0000","last_became_active":"2023-09-23T21:16:32.926127+0000","last_became_peered":"2023-09-23T21:16:32.926127+0000","last_unstale":"2023-09-23T21:18:09.854765+0000","last_undegraded":"2023-09-23T21:18:09.854765+0000","last_fullsized":"2023-09-23T21:18:09.854765+0000","mapping_epoch":559,"log_start":"549'1822","ondisk_log_start":"549'1822","created":559,"last_epoch_clean":561,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_clean_scrub_stamp":"2023-09-23T21:15:24.860968+0000","objects_scrubbed":0,"log_size":5310,"log_dups_size":0,"ondisk_log_size":5310,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":67010560,"num_objects":1023,"num_object_clones":0,"num_object_copies":3069,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1023,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4899,"num_write_kb":189472,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1c","version":"79'161","reported_seq":1029,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-09-23T21:18:09.515997+0000","last_change":"2023-09-23T21:16:45.419779+0000","last_active":"2023-09-23T21:18:09.515997+0000","last_peered":"2023-09-23T21:18:09.515997+0000","last_clean":"2023-09-23T21:18:09.515997+0000","last_became_active":"2023-09-23T21:16:45.419394+0000","last_became_peered":"2023-09-23T21:16:45.419394+0000","last_unstale":"2023-09-23T21:18:09.515997+0000","last_undegraded":"2023-09-23T21:18:09.515997+0000","last_fullsized":"2023-09-23T21:18:09.515997+0000","mapping_epoch":567,"log_start":"0'0","ondisk_log_start":"0'0","created":479,"last_epoch_clean":568,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:15:54.019828+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:15:54.019828+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:59:05.422258+0000","scrub_duration":0.0020962530000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":207,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-09-23T21:18:09.533415+0000","last_change":"2023-09-23T21:16:42.424187+0000","last_active":"2023-09-23T21:18:09.533415+0000","last_peered":"2023-09-23T21:18:09.533415+0000","last_clean":"2023-09-23T21:18:09.533415+0000","last_became_active":"2023-09-23T21:16:42.423805+0000","last_became_peered":"2023-09-23T21:16:42.423805+0000","last_unstale":"2023-09-23T21:18:09.533415+0000","last_undegraded":"2023-09-23T21:18:09.533415+0000","last_fullsized":"2023-09-23T21:18:09.533415+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:55:46.051199+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,7],"acting":[2,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":268,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-09-23T21:18:09.532333+0000","last_change":"2023-09-23T21:13:12.200767+0000","last_active":"2023-09-23T21:18:09.532333+0000","last_peered":"2023-09-23T21:18:09.532333+0000","last_clean":"2023-09-23T21:18:09.532333+0000","last_became_active":"2023-09-23T21:13:12.200435+0000","last_became_peered":"2023-09-23T21:13:12.200435+0000","last_unstale":"2023-09-23T21:18:09.532333+0000","last_undegraded":"2023-09-23T21:18:09.532333+0000","last_fullsized":"2023-09-23T21:18:09.532333+0000","mapping_epoch":493,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:15:44.937886+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"11.17","version":"599'7495","reported_seq":7577,"reported_epoch":599,"state":"active+clean","last_fresh":"2023-09-23T21:18:12.058183+0000","last_change":"2023-09-23T21:18:06.404241+0000","last_active":"2023-09-23T21:18:12.058183+0000","last_peered":"2023-09-23T21:18:12.058183+0000","last_clean":"2023-09-23T21:18:12.058183+0000","last_became_active":"2023-09-23T21:16:32.890013+0000","last_became_peered":"2023-09-23T21:16:32.890013+0000","last_unstale":"2023-09-23T21:18:12.058183+0000","last_undegraded":"2023-09-23T21:18:12.058183+0000","last_fullsized":"2023-09-23T21:18:12.058183+0000","mapping_epoch":559,"log_start":"548'660","ondisk_log_start":"548'660","created":559,"last_epoch_clean":560,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_clean_scrub_stamp":"2023-09-23T21:15:24.860968+0000","objects_scrubbed":0,"log_size":6835,"log_dups_size":0,"ondisk_log_size":6835,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":69632000,"num_objects":1063,"num_object_clones":0,"num_object_copies":3189,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1063,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5191,"num_write_kb":200096,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1d","version":"79'161","reported_seq":731,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-09-23T21:18:09.516039+0000","last_change":"2023-09-23T21:15:23.802125+0000","last_active":"2023-09-23T21:18:09.516039+0000","last_peered":"2023-09-23T21:18:09.516039+0000","last_clean":"2023-09-23T21:18:09.516039+0000","last_became_active":"2023-09-23T21:15:23.801704+0000","last_became_peered":"2023-09-23T21:15:23.801704+0000","last_unstale":"2023-09-23T21:18:09.516039+0000","last_undegraded":"2023-09-23T21:18:09.516039+0000","last_fullsized":"2023-09-23T21:18:09.516039+0000","mapping_epoch":542,"log_start":"0'0","ondisk_log_start":"0'0","created":479,"last_epoch_clean":543,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:09:36.858077+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:09:36.858077+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.010304610000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,2],"acting":[3,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":126,"reported_epoch":597,"state":"active+clean","last_fresh":"2023-09-23T21:18:08.524635+0000","last_change":"2023-09-23T21:13:38.380574+0000","last_active":"2023-09-23T21:18:08.524635+0000","last_peered":"2023-09-23T21:18:08.524635+0000","last_clean":"2023-09-23T21:18:08.524635+0000","last_became_active":"2023-09-23T21:13:38.373252+0000","last_became_peered":"2023-09-23T21:13:38.373252+0000","last_unstale":"2023-09-23T21:18:08.524635+0000","last_undegraded":"2023-09-23T21:18:08.524635+0000","last_fullsized":"2023-09-23T21:18:08.524635+0000","mapping_epoch":507,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:58:14.788031+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":11,"num_bytes_recovered":3324,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":173,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-09-23T21:18:09.518047+0000","last_change":"2023-09-23T21:12:10.793610+0000","last_active":"2023-09-23T21:18:09.518047+0000","last_peered":"2023-09-23T21:18:09.518047+0000","last_clean":"2023-09-23T21:18:09.518047+0000","last_became_active":"2023-09-23T21:12:10.793081+0000","last_became_peered":"2023-09-23T21:12:10.793081+0000","last_unstale":"2023-09-23T21:18:09.518047+0000","last_undegraded":"2023-09-23T21:18:09.518047+0000","last_fullsized":"2023-09-23T21:18:09.518047+0000","mapping_epoch":470,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:34:36.291550+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":11,"num_pg":26,"stat_sum":{"num_bytes":2156068888,"num_objects":32900,"num_object_clones":0,"num_object_copies":98700,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1119,"num_objects_misplaced":170,"num_objects_unfound":0,"num_objects_dirty":32900,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":163036,"num_write_kb":6269889,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":13886,"num_bytes_recovered":899612672,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":7075672064,"data_stored":7075659848,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":141449,"ondisk_log_size":141449,"up":78,"acting":76,"num_store_stats":6},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":80,"num_bytes_recovered":18686,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":118784,"data_stored":21350,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":14322,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":31,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":525376,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2109440,"data_stored":2098784,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4991,"ondisk_log_size":4991,"up":93,"acting":93,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201733,"num_pgs":68,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1654620,"kb_used_data":1037332,"kb_used_omap":0,"kb_used_meta":617280,"kb_avail":92090532,"statfs":{"total":95995035648,"available":94300704768,"internally_reserved":0,"allocated":1062227968,"data_stored":1060498655,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":632094720},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:17:19 2023","interfaces":[{"interface":"back","average":{"1min":1.002,"5min":1.655,"15min":2.169},"min":{"1min":0.22800000000000001,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":3.4689999999999999,"5min":37.207999999999998,"15min":37.207999999999998},"last":0.24299999999999999},{"interface":"front","average":{"1min":1.147,"5min":1.849,"15min":2.105},"min":{"1min":0.32200000000000001,"5min":0.253,"15min":0.253},"max":{"1min":3.702,"5min":37.122,"15min":37.122},"last":0.35899999999999999}]},{"osd":1,"last update":"Sat Sep 23 21:17:13 2023","interfaces":[{"interface":"back","average":{"1min":1.0149999999999999,"5min":1.2030000000000001,"15min":1.8600000000000001},"min":{"1min":0.308,"5min":0.27500000000000002,"15min":0.26700000000000002},"max":{"1min":3.4279999999999999,"5min":7.5369999999999999,"15min":14.034000000000001},"last":0.308},{"interface":"front","average":{"1min":1.2909999999999999,"5min":1.4850000000000001,"15min":1.8200000000000001},"min":{"1min":0.29999999999999999,"5min":0.29999999999999999,"15min":0.26300000000000001},"max":{"1min":5.4039999999999999,"5min":11.002000000000001,"15min":11.226000000000001},"last":1.381}]},{"osd":2,"last update":"Sat Sep 23 21:17:37 2023","interfaces":[{"interface":"back","average":{"1min":2.9670000000000001,"5min":1.9039999999999999,"15min":1.825},"min":{"1min":0.372,"5min":0.26400000000000001,"15min":0.26200000000000001},"max":{"1min":25.471,"5min":25.471,"15min":25.471},"last":2.7949999999999999},{"interface":"front","average":{"1min":3.073,"5min":1.73,"15min":1.7430000000000001},"min":{"1min":0.372,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":27.187999999999999,"5min":27.187999999999999,"15min":27.187999999999999},"last":0.46600000000000003}]},{"osd":3,"last update":"Sat Sep 23 21:18:04 2023","interfaces":[{"interface":"back","average":{"1min":1.7070000000000001,"5min":1.484,"15min":1.8280000000000001},"min":{"1min":0.33200000000000002,"5min":0.30599999999999999,"15min":0.29099999999999998},"max":{"1min":6.8739999999999997,"5min":8.6259999999999994,"15min":12.359999999999999},"last":3.9009999999999998},{"interface":"front","average":{"1min":1.1180000000000001,"5min":1.732,"15min":1.831},"min":{"1min":0.28799999999999998,"5min":0.28799999999999998,"15min":0.28699999999999998},"max":{"1min":3.8300000000000001,"5min":15.82,"15min":15.82},"last":0.48499999999999999}]},{"osd":4,"last update":"Sat Sep 23 21:18:04 2023","interfaces":[{"interface":"back","average":{"1min":0.93400000000000005,"5min":1.02,"15min":1.1799999999999999},"min":{"1min":0.251,"5min":0.251,"15min":0.248},"max":{"1min":3.9990000000000001,"5min":7.6609999999999996,"15min":11.297000000000001},"last":0.29799999999999999},{"interface":"front","average":{"1min":1.0089999999999999,"5min":1.073,"15min":1.137},"min":{"1min":0.28599999999999998,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":5.6349999999999998,"5min":8.6059999999999999,"15min":12.752000000000001},"last":0.41399999999999998}]},{"osd":5,"last update":"Sat Sep 23 21:18:04 2023","interfaces":[{"interface":"back","average":{"1min":1.286,"5min":1.095,"15min":1.089},"min":{"1min":0.28999999999999998,"5min":0.28999999999999998,"15min":0.26000000000000001},"max":{"1min":6.9740000000000002,"5min":7.5910000000000002,"15min":9.4789999999999992},"last":0.42899999999999999},{"interface":"front","average":{"1min":1.381,"5min":1.171,"15min":1.2130000000000001},"min":{"1min":0.35599999999999998,"5min":0.32000000000000001,"15min":0.26200000000000001},"max":{"1min":6.7439999999999998,"5min":7.6929999999999996,"15min":11.257},"last":0.44}]},{"osd":6,"last update":"Sat Sep 23 21:17:13 2023","interfaces":[{"interface":"back","average":{"1min":0.56200000000000006,"5min":1.236,"15min":1.081},"min":{"1min":0.32000000000000001,"5min":0.32000000000000001,"15min":0.30099999999999999},"max":{"1min":1.226,"5min":10.941000000000001,"15min":10.941000000000001},"last":0.55300000000000005},{"interface":"front","average":{"1min":0.621,"5min":1.1719999999999999,"15min":1.161},"min":{"1min":0.36299999999999999,"5min":0.36299999999999999,"15min":0.32300000000000001},"max":{"1min":1.512,"5min":8.923,"15min":12.715},"last":2.734}]}]},{"osd":6,"up_from":0,"seq":704374636817,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":1,"up_from":0,"seq":1503238553749,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":0,"up_from":493,"seq":2117418876989,"num_pgs":76,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1881064,"kb_used_data":1499488,"kb_used_omap":36,"kb_used_meta":381531,"kb_avail":91864088,"statfs":{"total":95995035648,"available":94068826112,"internally_reserved":0,"allocated":1535475712,"data_stored":1532690692,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":37336,"internal_metadata":390688296},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 21:17:23 2023","interfaces":[{"interface":"back","average":{"1min":2.0419999999999998,"5min":1.3200000000000001,"15min":0.94399999999999995},"min":{"1min":0.214,"5min":0.214,"15min":0.214},"max":{"1min":13.779999999999999,"5min":13.779999999999999,"15min":13.779999999999999},"last":0.433},{"interface":"front","average":{"1min":2.0209999999999999,"5min":1.401,"15min":0.98099999999999998},"min":{"1min":0.32300000000000001,"5min":0.20799999999999999,"15min":0.20799999999999999},"max":{"1min":6.8810000000000002,"5min":11.964,"15min":11.964},"last":3.4220000000000002}]},{"osd":2,"last update":"Sat Sep 23 21:17:23 2023","interfaces":[{"interface":"back","average":{"1min":3.0510000000000002,"5min":1.52,"15min":0.94199999999999995},"min":{"1min":0.27500000000000002,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":13.637,"5min":13.637,"15min":13.637},"last":0.46800000000000003},{"interface":"front","average":{"1min":3.6240000000000001,"5min":2.0019999999999998,"15min":1.3120000000000001},"min":{"1min":0.40799999999999997,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":26.571000000000002,"5min":26.571000000000002,"15min":26.571000000000002},"last":3.5019999999999998}]},{"osd":3,"last update":"Sat Sep 23 21:17:23 2023","interfaces":[{"interface":"back","average":{"1min":4.0890000000000004,"5min":1.974,"15min":1.236},"min":{"1min":0.26900000000000002,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":33.634,"5min":33.634,"15min":33.634},"last":5.093},{"interface":"front","average":{"1min":2.5600000000000001,"5min":1.736,"15min":1.3240000000000001},"min":{"1min":0.30299999999999999,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":13.851000000000001,"5min":15.382999999999999,"15min":15.382999999999999},"last":1.252}]},{"osd":4,"last update":"Sat Sep 23 21:17:23 2023","interfaces":[{"interface":"back","average":{"1min":2.7320000000000002,"5min":1.6479999999999999,"15min":1.278},"min":{"1min":0.34599999999999997,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":22.780000000000001,"5min":22.780000000000001,"15min":22.780000000000001},"last":1.224},{"interface":"front","average":{"1min":2.2320000000000002,"5min":1.3100000000000001,"15min":1.1000000000000001},"min":{"1min":0.28499999999999998,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":22.652000000000001,"5min":22.652000000000001,"15min":22.652000000000001},"last":6.9119999999999999}]},{"osd":5,"last update":"Sat Sep 23 21:17:23 2023","interfaces":[{"interface":"back","average":{"1min":1.958,"5min":1.504,"15min":1.325},"min":{"1min":0.312,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":7.8630000000000004,"5min":8.9329999999999998,"15min":8.9329999999999998},"last":0.71999999999999997},{"interface":"front","average":{"1min":1.825,"5min":1.528,"15min":1.218},"min":{"1min":0.40300000000000002,"5min":0.313,"15min":0.313},"max":{"1min":6.577,"5min":15.321999999999999,"15min":15.321999999999999},"last":1.1910000000000001}]},{"osd":6,"last update":"Sat Sep 23 21:17:23 2023","interfaces":[{"interface":"back","average":{"1min":1.4430000000000001,"5min":1.3919999999999999,"15min":1.2150000000000001},"min":{"1min":0.36599999999999999,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":6.6449999999999996,"5min":7.2050000000000001,"15min":7.2050000000000001},"last":0.53100000000000003},{"interface":"front","average":{"1min":1.6990000000000001,"5min":1.5269999999999999,"15min":1.381},"min":{"1min":0.38900000000000001,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":5.5,"5min":8.8889999999999993,"15min":8.8889999999999993},"last":0.68799999999999994}]},{"osd":7,"last update":"Sat Sep 23 21:17:23 2023","interfaces":[{"interface":"back","average":{"1min":1.9950000000000001,"5min":1.7470000000000001,"15min":1.3879999999999999},"min":{"1min":0.434,"5min":0.40899999999999997,"15min":0.40899999999999997},"max":{"1min":6.649,"5min":15.273999999999999,"15min":15.273999999999999},"last":1.7410000000000001},{"interface":"front","average":{"1min":2.996,"5min":1.6519999999999999,"15min":1.3220000000000001},"min":{"1min":0.36699999999999999,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":33.779000000000003,"5min":33.779000000000003,"15min":33.779000000000003},"last":0.5}]}]},{"osd":2,"up_from":93,"seq":399431958861,"num_pgs":60,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2347608,"kb_used_data":1347728,"kb_used_omap":81,"kb_used_meta":999662,"kb_avail":91397544,"statfs":{"total":95995035648,"available":93591085056,"internally_reserved":0,"allocated":1380073472,"data_stored":1377248687,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":83750,"internal_metadata":1023654106},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:17:23 2023","interfaces":[{"interface":"back","average":{"1min":1.952,"5min":1.577,"15min":1.081},"min":{"1min":0.187,"5min":0.187,"15min":0.187},"max":{"1min":13.25,"5min":13.25,"15min":13.25},"last":2.9620000000000002},{"interface":"front","average":{"1min":2.266,"5min":1.972,"15min":1.7669999999999999},"min":{"1min":0.28199999999999997,"5min":0.20200000000000001,"15min":0.20200000000000001},"max":{"1min":9.4169999999999998,"5min":18.111999999999998,"15min":18.111999999999998},"last":2.5720000000000001}]},{"osd":1,"last update":"Sat Sep 23 21:17:23 2023","interfaces":[{"interface":"back","average":{"1min":2.7349999999999999,"5min":1.48,"15min":2.351},"min":{"1min":0.33300000000000002,"5min":0.188,"15min":0.188},"max":{"1min":8.9000000000000004,"5min":9.9890000000000008,"15min":34.396999999999998},"last":0.55600000000000005},{"interface":"front","average":{"1min":2.117,"5min":1.5669999999999999,"15min":2.4430000000000001},"min":{"1min":0.27500000000000002,"5min":0.217,"15min":0.217},"max":{"1min":8.5510000000000002,"5min":18.081,"15min":34.462000000000003},"last":2.2829999999999999}]},{"osd":3,"last update":"Sat Sep 23 21:18:01 2023","interfaces":[{"interface":"back","average":{"1min":2.0880000000000001,"5min":2.04,"15min":2.2389999999999999},"min":{"1min":0.248,"5min":0.20599999999999999,"15min":0.20599999999999999},"max":{"1min":7.3719999999999999,"5min":20.106999999999999,"15min":35.549999999999997},"last":8.141},{"interface":"front","average":{"1min":2.6659999999999999,"5min":2.556,"15min":2.6099999999999999},"min":{"1min":0.35799999999999998,"5min":0.28499999999999998,"15min":0.21299999999999999},"max":{"1min":10.162000000000001,"5min":20.239000000000001,"15min":36.951999999999998},"last":3.0510000000000002}]},{"osd":4,"last update":"Sat Sep 23 21:17:23 2023","interfaces":[{"interface":"back","average":{"1min":1.627,"5min":1.409,"15min":1.448},"min":{"1min":0.29599999999999999,"5min":0.29599999999999999,"15min":0.249},"max":{"1min":5.6369999999999996,"5min":10.266999999999999,"15min":10.266999999999999},"last":1.909},{"interface":"front","average":{"1min":2.379,"5min":1.587,"15min":1.7949999999999999},"min":{"1min":0.30299999999999999,"5min":0.29399999999999998,"15min":0.26700000000000002},"max":{"1min":10.459,"5min":10.459,"15min":12.49},"last":0.53200000000000003}]},{"osd":5,"last update":"Sat Sep 23 21:17:23 2023","interfaces":[{"interface":"back","average":{"1min":1.915,"5min":1.3460000000000001,"15min":1.5569999999999999},"min":{"1min":0.41199999999999998,"5min":0.29099999999999998,"15min":0.28000000000000003},"max":{"1min":5.7089999999999996,"5min":7.6180000000000003,"15min":9.5039999999999996},"last":1.9630000000000001},{"interface":"front","average":{"1min":2.3809999999999998,"5min":1.417,"15min":1.746},"min":{"1min":0.378,"5min":0.29699999999999999,"15min":0.29399999999999998},"max":{"1min":10.425000000000001,"5min":10.425000000000001,"15min":12.57},"last":0.499}]},{"osd":6,"last update":"Sat Sep 23 21:17:23 2023","interfaces":[{"interface":"back","average":{"1min":2.3069999999999999,"5min":1.6839999999999999,"15min":1.9570000000000001},"min":{"1min":0.42899999999999999,"5min":0.373,"15min":0.35699999999999998},"max":{"1min":10.335000000000001,"5min":10.335000000000001,"15min":12.518000000000001},"last":0.57899999999999996},{"interface":"front","average":{"1min":2.4169999999999998,"5min":1.831,"15min":2.1019999999999999},"min":{"1min":0.44500000000000001,"5min":0.377,"15min":0.35599999999999998},"max":{"1min":10.381,"5min":10.381,"15min":12.544},"last":0.59999999999999998}]},{"osd":7,"last update":"Sat Sep 23 21:17:23 2023","interfaces":[{"interface":"back","average":{"1min":1.6220000000000001,"5min":1.6319999999999999,"15min":1.613},"min":{"1min":0.39000000000000001,"5min":0.33700000000000002,"15min":0.29799999999999999},"max":{"1min":7.266,"5min":18.021999999999998,"15min":18.021999999999998},"last":2.3199999999999998},{"interface":"front","average":{"1min":1.702,"5min":1.615,"15min":1.714},"min":{"1min":0.434,"5min":0.34799999999999998,"15min":0.32200000000000001},"max":{"1min":5.7329999999999997,"5min":10.359,"15min":10.92},"last":1.9890000000000001}]}]},{"osd":3,"up_from":98,"seq":420906795335,"num_pgs":85,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2432744,"kb_used_data":1367712,"kb_used_omap":7,"kb_used_meta":1064888,"kb_avail":91312408,"statfs":{"total":95995035648,"available":93503905792,"internally_reserved":0,"allocated":1400537088,"data_stored":1397753969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7576,"internal_metadata":1090445928},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,6,1,4],"upper_bound":16},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:17:17 2023","interfaces":[{"interface":"back","average":{"1min":5.2619999999999996,"5min":3.7010000000000001,"15min":1.4590000000000001},"min":{"1min":0.18099999999999999,"5min":0.18099999999999999,"15min":0.18099999999999999},"max":{"1min":81.408000000000001,"5min":147.38900000000001,"15min":147.38900000000001},"last":1.006},{"interface":"front","average":{"1min":4.4119999999999999,"5min":3.4809999999999999,"15min":1.605},"min":{"1min":0.223,"5min":0.223,"15min":0.223},"max":{"1min":82.397000000000006,"5min":149.989,"15min":149.989},"last":2.4990000000000001}]},{"osd":1,"last update":"Sat Sep 23 21:17:17 2023","interfaces":[{"interface":"back","average":{"1min":1.7030000000000001,"5min":1.234,"15min":2.1120000000000001},"min":{"1min":0.29399999999999998,"5min":0.20000000000000001,"15min":0.20000000000000001},"max":{"1min":7.2759999999999998,"5min":7.2759999999999998,"15min":22.577999999999999},"last":4.6280000000000001},{"interface":"front","average":{"1min":1.1000000000000001,"5min":0.998,"15min":2.0099999999999998},"min":{"1min":0.255,"5min":0.216,"15min":0.216},"max":{"1min":3.5339999999999998,"5min":7.6829999999999998,"15min":21.617999999999999},"last":1.04}]},{"osd":2,"last update":"Sat Sep 23 21:18:02 2023","interfaces":[{"interface":"back","average":{"1min":1.4750000000000001,"5min":1.958,"15min":2.004},"min":{"1min":0.27400000000000002,"5min":0.221,"15min":0.20899999999999999},"max":{"1min":11.321,"5min":56.317999999999998,"15min":56.317999999999998},"last":3.1000000000000001},{"interface":"front","average":{"1min":1.9910000000000001,"5min":2.4780000000000002,"15min":2.355},"min":{"1min":0.23400000000000001,"5min":0.23400000000000001,"15min":0.216},"max":{"1min":11.371,"5min":55.988,"15min":55.988},"last":3.802}]},{"osd":4,"last update":"Sat Sep 23 21:18:02 2023","interfaces":[{"interface":"back","average":{"1min":0.72899999999999998,"5min":1.2010000000000001,"15min":1.5509999999999999},"min":{"1min":0.29199999999999998,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":2.5409999999999999,"5min":10.807,"15min":21.478999999999999},"last":1.627},{"interface":"front","average":{"1min":1.091,"5min":1.3180000000000001,"15min":1.7829999999999999},"min":{"1min":0.35899999999999999,"5min":0.311,"15min":0.28499999999999998},"max":{"1min":3.129,"5min":7.3140000000000001,"15min":22.402000000000001},"last":1.173}]},{"osd":5,"last update":"Sat Sep 23 21:18:02 2023","interfaces":[{"interface":"back","average":{"1min":0.83199999999999996,"5min":1.3160000000000001,"15min":1.6930000000000001},"min":{"1min":0.318,"5min":0.313,"15min":0.311},"max":{"1min":4.1379999999999999,"5min":10.882999999999999,"15min":21.562000000000001},"last":1.5920000000000001},{"interface":"front","average":{"1min":1.571,"5min":1.234,"15min":1.8},"min":{"1min":0.42699999999999999,"5min":0.34300000000000003,"15min":0.31},"max":{"1min":5.7300000000000004,"5min":5.7300000000000004,"15min":14.029},"last":1.075}]},{"osd":6,"last update":"Sat Sep 23 21:17:18 2023","interfaces":[{"interface":"back","average":{"1min":1.48,"5min":1.4359999999999999,"15min":1.9059999999999999},"min":{"1min":0.42899999999999999,"5min":0.34999999999999998,"15min":0.33900000000000002},"max":{"1min":7.3410000000000002,"5min":10.102,"15min":22.446999999999999},"last":1.1970000000000001},{"interface":"front","average":{"1min":1.2549999999999999,"5min":1.4319999999999999,"15min":1.9019999999999999},"min":{"1min":0.45800000000000002,"5min":0.378,"15min":0.373},"max":{"1min":5.5510000000000002,"5min":10.148,"15min":12.375999999999999},"last":1.127}]},{"osd":7,"last update":"Sat Sep 23 21:18:02 2023","interfaces":[{"interface":"back","average":{"1min":1.0249999999999999,"5min":1.48,"15min":1.847},"min":{"1min":0.41299999999999998,"5min":0.41199999999999998,"15min":0.33900000000000002},"max":{"1min":4.242,"5min":10.914999999999999,"15min":21.518999999999998},"last":1.655},{"interface":"front","average":{"1min":1.5409999999999999,"5min":1.528,"15min":1.9850000000000001},"min":{"1min":0.51800000000000002,"5min":0.42499999999999999,"15min":0.40300000000000002},"max":{"1min":4.6920000000000002,"5min":8.2590000000000003,"15min":12.624000000000001},"last":1.1020000000000001}]}]},{"osd":4,"up_from":31,"seq":133143986641,"num_pgs":62,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1411996,"kb_used_data":848276,"kb_used_omap":0,"kb_used_meta":563712,"kb_avail":92333156,"statfs":{"total":95995035648,"available":94549151744,"internally_reserved":0,"allocated":868634624,"data_stored":866905044,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":577241088},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:17:21 2023","interfaces":[{"interface":"back","average":{"1min":2.508,"5min":1.399,"15min":1.0660000000000001},"min":{"1min":0.252,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":28.683,"5min":28.683,"15min":28.683},"last":2.2730000000000001},{"interface":"front","average":{"1min":2.839,"5min":1.411,"15min":1.226},"min":{"1min":0.27600000000000002,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":29.155000000000001,"5min":29.155000000000001,"15min":29.155000000000001},"last":0.47799999999999998}]},{"osd":1,"last update":"Sat Sep 23 21:17:21 2023","interfaces":[{"interface":"back","average":{"1min":1.821,"5min":1.214,"15min":2.246},"min":{"1min":0.312,"5min":0.251,"15min":0.221},"max":{"1min":5.6319999999999997,"5min":7.8609999999999998,"15min":37.960000000000001},"last":0.499},{"interface":"front","average":{"1min":1.637,"5min":1.3049999999999999,"15min":2.1930000000000001},"min":{"1min":0.374,"5min":0.28499999999999998,"15min":0.27000000000000002},"max":{"1min":5.6130000000000004,"5min":13.983000000000001,"15min":37.933},"last":1.883}]},{"osd":2,"last update":"Sat Sep 23 21:17:21 2023","interfaces":[{"interface":"back","average":{"1min":2.6659999999999999,"5min":1.6419999999999999,"15min":1.7529999999999999},"min":{"1min":0.33300000000000002,"5min":0.26000000000000001,"15min":0.247},"max":{"1min":17.602,"5min":17.602,"15min":17.602},"last":1.9259999999999999},{"interface":"front","average":{"1min":1.79,"5min":1.669,"15min":1.806},"min":{"1min":0.39400000000000002,"5min":0.28599999999999998,"15min":0.27800000000000002},"max":{"1min":6.0720000000000001,"5min":14.339,"15min":14.417999999999999},"last":1.9790000000000001}]},{"osd":3,"last update":"Sat Sep 23 21:17:52 2023","interfaces":[{"interface":"back","average":{"1min":5.2910000000000004,"5min":2.1000000000000001,"15min":2.1299999999999999},"min":{"1min":0.312,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":67.105999999999995,"5min":67.105999999999995,"15min":67.105999999999995},"last":0.42099999999999999},{"interface":"front","average":{"1min":4.9290000000000003,"5min":2.1930000000000001,"15min":2.056},"min":{"1min":0.33600000000000002,"5min":0.32200000000000001,"15min":0.30199999999999999},"max":{"1min":67.046999999999997,"5min":67.046999999999997,"15min":67.046999999999997},"last":1.952}]},{"osd":5,"last update":"Sat Sep 23 21:17:08 2023","interfaces":[{"interface":"back","average":{"1min":0.77100000000000002,"5min":1.3520000000000001,"15min":1.127},"min":{"1min":0.27700000000000002,"5min":0.222,"15min":0.222},"max":{"1min":2.637,"5min":14.398999999999999,"15min":14.398999999999999},"last":2.0019999999999998},{"interface":"front","average":{"1min":0.58099999999999996,"5min":1.1639999999999999,"15min":1.1080000000000001},"min":{"1min":0.25800000000000001,"5min":0.25800000000000001,"15min":0.255},"max":{"1min":3.1600000000000001,"5min":8.016,"15min":9.8979999999999997},"last":2.4390000000000001}]},{"osd":6,"last update":"Sat Sep 23 21:17:21 2023","interfaces":[{"interface":"back","average":{"1min":0.91000000000000003,"5min":1.335,"15min":1.3069999999999999},"min":{"1min":0.27200000000000002,"5min":0.27200000000000002,"15min":0.26800000000000002},"max":{"1min":4.7270000000000003,"5min":24.452999999999999,"15min":24.452999999999999},"last":0.45300000000000001},{"interface":"front","average":{"1min":0.80800000000000005,"5min":1.3200000000000001,"15min":1.095},"min":{"1min":0.36399999999999999,"5min":0.36399999999999999,"15min":0.32500000000000001},"max":{"1min":2.7069999999999999,"5min":14.374000000000001,"15min":14.387},"last":2.0859999999999999}]},{"osd":7,"last update":"Sat Sep 23 21:17:52 2023","interfaces":[{"interface":"back","average":{"1min":1.091,"5min":1.194,"15min":1.1919999999999999},"min":{"1min":0.33200000000000002,"5min":0.32000000000000001,"15min":0.30099999999999999},"max":{"1min":5.7789999999999999,"5min":14.422000000000001,"15min":16.890999999999998},"last":2.1739999999999999},{"interface":"front","average":{"1min":0.73399999999999999,"5min":1.034,"15min":1.2230000000000001},"min":{"1min":0.29199999999999998,"5min":0.29199999999999998,"15min":0.27400000000000002},"max":{"1min":3.681,"5min":7.7229999999999999,"15min":11.170999999999999},"last":2.4710000000000001}]}]},{"osd":5,"up_from":39,"seq":167503725007,"num_pgs":61,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1533584,"kb_used_data":931336,"kb_used_omap":0,"kb_used_meta":602240,"kb_avail":92211568,"statfs":{"total":95995035648,"available":94424645632,"internally_reserved":0,"allocated":953688064,"data_stored":951967060,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":616693760},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:17:14 2023","interfaces":[{"interface":"back","average":{"1min":1.482,"5min":1.0860000000000001,"15min":0.83799999999999997},"min":{"1min":0.23100000000000001,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":5.9569999999999999,"5min":8.673,"15min":8.673},"last":0.55300000000000005},{"interface":"front","average":{"1min":1.3460000000000001,"5min":1.177,"15min":0.83799999999999997},"min":{"1min":0.30299999999999999,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":3.339,"5min":9.7279999999999998,"15min":9.7279999999999998},"last":0.498}]},{"osd":1,"last update":"Sat Sep 23 21:17:14 2023","interfaces":[{"interface":"back","average":{"1min":1.9079999999999999,"5min":1.0840000000000001,"15min":1.6479999999999999},"min":{"1min":0.33200000000000002,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":4.8200000000000003,"5min":7.5839999999999996,"15min":25.239999999999998},"last":0.44900000000000001},{"interface":"front","average":{"1min":1.4410000000000001,"5min":1.351,"15min":1.7190000000000001},"min":{"1min":0.34699999999999998,"5min":0.28299999999999997,"15min":0.252},"max":{"1min":4.8049999999999997,"5min":11.116,"15min":25.253},"last":0.58499999999999996}]},{"osd":2,"last update":"Sat Sep 23 21:17:32 2023","interfaces":[{"interface":"back","average":{"1min":1.615,"5min":1.506,"15min":1.6439999999999999},"min":{"1min":0.27300000000000002,"5min":0.27300000000000002,"15min":0.25800000000000001},"max":{"1min":7.3550000000000004,"5min":9.6560000000000006,"15min":12.634},"last":0.52700000000000002},{"interface":"front","average":{"1min":1.9410000000000001,"5min":1.327,"15min":1.6579999999999999},"min":{"1min":0.314,"5min":0.27600000000000002,"15min":0.25700000000000001},"max":{"1min":9.1630000000000003,"5min":9.1630000000000003,"15min":9.7650000000000006},"last":1.51}]},{"osd":3,"last update":"Sat Sep 23 21:17:52 2023","interfaces":[{"interface":"back","average":{"1min":1.4259999999999999,"5min":1.4570000000000001,"15min":1.931},"min":{"1min":0.314,"5min":0.314,"15min":0.29299999999999998},"max":{"1min":3.6030000000000002,"5min":6.5940000000000003,"15min":25.503},"last":2.0390000000000001},{"interface":"front","average":{"1min":1.7310000000000001,"5min":1.748,"15min":2.157},"min":{"1min":0.311,"5min":0.311,"15min":0.30599999999999999},"max":{"1min":9.0619999999999994,"5min":12.382,"15min":25.466000000000001},"last":1.1180000000000001}]},{"osd":4,"last update":"Sat Sep 23 21:18:08 2023","interfaces":[{"interface":"back","average":{"1min":0.82399999999999995,"5min":0.91400000000000003,"15min":1.155},"min":{"1min":0.29999999999999999,"5min":0.253,"15min":0.253},"max":{"1min":5.0279999999999996,"5min":11.156000000000001,"15min":12.776},"last":0.80800000000000005},{"interface":"front","average":{"1min":0.73399999999999999,"5min":1.008,"15min":1.1419999999999999},"min":{"1min":0.28599999999999998,"5min":0.246,"15min":0.246},"max":{"1min":3.415,"5min":9.7119999999999997,"15min":11.973000000000001},"last":0.46100000000000002}]},{"osd":6,"last update":"Sat Sep 23 21:17:14 2023","interfaces":[{"interface":"back","average":{"1min":1.556,"5min":1.147,"15min":1.1160000000000001},"min":{"1min":0.28199999999999997,"5min":0.28000000000000003,"15min":0.25900000000000001},"max":{"1min":4.3490000000000002,"5min":7.7030000000000003,"15min":11.128},"last":0.63400000000000001},{"interface":"front","average":{"1min":1.208,"5min":1.452,"15min":1.268},"min":{"1min":0.39400000000000002,"5min":0.34300000000000003,"15min":0.317},"max":{"1min":4.367,"5min":10.999000000000001,"15min":11.112},"last":0.83999999999999997}]},{"osd":7,"last update":"Sat Sep 23 21:17:52 2023","interfaces":[{"interface":"back","average":{"1min":1.109,"5min":1.012,"15min":1.819},"min":{"1min":0.36199999999999999,"5min":0.32400000000000001,"15min":0.309},"max":{"1min":4.2629999999999999,"5min":5.5659999999999998,"15min":189.405},"last":0.66000000000000003},{"interface":"front","average":{"1min":1.0649999999999999,"5min":1.1719999999999999,"15min":1.9670000000000001},"min":{"1min":0.39900000000000002,"5min":0.35199999999999998,"15min":0.32500000000000001},"max":{"1min":2.7879999999999998,"5min":11.183999999999999,"15min":189.685},"last":0.97799999999999998}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8488,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1259,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5322,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":4838,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1527513088,"data_stored":1527513088,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1346375680,"data_stored":1346371608,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1383006208,"data_stored":1383006208,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":848498688,"data_stored":848494616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":928911360,"data_stored":928907288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1041367040,"data_stored":1041367040,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T21:18:20.969 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph status --format=json 2023-09-23T21:18:21.022 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:18:21.082 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:18:21.129 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:18:21.131 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:18:21.132 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:18:21.222 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:18:21.233 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:18:21.247 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T21:18:21.259 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:18:21.292 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:18:21.294 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:18:21.295 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:18:21.395 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:18:21.426 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:18:21.535 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:18:21.539 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:18:21.540 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:18:21.641 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:18:21.697 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:18:21.716 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:18:21 smithi026 ceph-mon[179688]: failed to encode map e602 with expected crc 2023-09-23T21:18:21.716 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:18:21 smithi026 ceph-mon[179688]: osdmap e602: 8 total, 8 up, 6 in 2023-09-23T21:18:21.716 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:18:21 smithi026 ceph-mon[179688]: failed to encode map e602 with expected crc 2023-09-23T21:18:21.716 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:18:21 smithi026 ceph-mon[179688]: failed to encode map e602 with expected crc 2023-09-23T21:18:21.716 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:18:21 smithi026 ceph-mon[179688]: failed to encode map e602 with expected crc 2023-09-23T21:18:21.717 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:18:21 smithi026 ceph-mon[179688]: failed to encode map e602 with expected crc 2023-09-23T21:18:21.717 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:18:21 smithi026 ceph-mon[179688]: Health check update: Degraded data redundancy: 812/88089 objects degraded (0.922%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T21:18:21.717 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:18:21 smithi026 ceph-mon[175151]: failed to encode map e602 with expected crc 2023-09-23T21:18:21.717 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:18:21 smithi026 ceph-mon[175151]: osdmap e602: 8 total, 8 up, 6 in 2023-09-23T21:18:21.717 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:18:21 smithi026 ceph-mon[175151]: failed to encode map e602 with expected crc 2023-09-23T21:18:21.717 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:18:21 smithi026 ceph-mon[175151]: failed to encode map e602 with expected crc 2023-09-23T21:18:21.717 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:18:21 smithi026 ceph-mon[175151]: failed to encode map e602 with expected crc 2023-09-23T21:18:21.717 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:18:21 smithi026 ceph-mon[175151]: failed to encode map e602 with expected crc 2023-09-23T21:18:21.717 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:18:21 smithi026 ceph-mon[175151]: Health check update: Degraded data redundancy: 812/88089 objects degraded (0.922%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T21:18:21.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:18:21 smithi070 ceph-mon[168282]: failed to encode map e602 with expected crc 2023-09-23T21:18:21.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:18:21 smithi070 ceph-mon[168282]: osdmap e602: 8 total, 8 up, 6 in 2023-09-23T21:18:21.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:18:21 smithi070 ceph-mon[168282]: failed to encode map e602 with expected crc 2023-09-23T21:18:21.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:18:21 smithi070 ceph-mon[168282]: failed to encode map e602 with expected crc 2023-09-23T21:18:21.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:18:21 smithi070 ceph-mon[168282]: failed to encode map e602 with expected crc 2023-09-23T21:18:21.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:18:21 smithi070 ceph-mon[168282]: failed to encode map e602 with expected crc 2023-09-23T21:18:21.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:18:21 smithi070 ceph-mon[168282]: Health check update: Degraded data redundancy: 812/88089 objects degraded (0.922%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T21:18:21.974 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T21:18:21.987 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:18:21.988 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:18:22.089 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:18:22.109 INFO:teuthology.orchestra.run.smithi026.stdout:{ 2023-09-23T21:18:22.109 INFO:teuthology.orchestra.run.smithi026.stdout: "ops": [], 2023-09-23T21:18:22.109 INFO:teuthology.orchestra.run.smithi026.stdout: "complaint_time": 30, 2023-09-23T21:18:22.109 INFO:teuthology.orchestra.run.smithi026.stdout: "num_blocked_ops": 0 2023-09-23T21:18:22.110 INFO:teuthology.orchestra.run.smithi026.stdout:} 2023-09-23T21:18:22.188 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:18:22.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:18:22 smithi070 ceph-mon[168282]: pgmap v1518: 129 pgs: 1 peering, 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 126 active+clean; 1.7 GiB data, 9.6 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 868 op/s; 207/83136 objects degraded (0.249%); 5.3 MiB/s, 180 objects/s recovering 2023-09-23T21:18:22.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:18:22 smithi070 ceph-mon[168282]: failed to encode map e603 with expected crc 2023-09-23T21:18:22.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:18:22 smithi070 ceph-mon[168282]: failed to encode map e603 with expected crc 2023-09-23T21:18:22.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:18:22 smithi070 ceph-mon[168282]: osdmap e603: 8 total, 8 up, 6 in 2023-09-23T21:18:22.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:18:22 smithi070 ceph-mon[168282]: failed to encode map e603 with expected crc 2023-09-23T21:18:22.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:18:22 smithi070 ceph-mon[168282]: failed to encode map e603 with expected crc 2023-09-23T21:18:22.881 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:18:22 smithi026 ceph-mon[179688]: pgmap v1518: 129 pgs: 1 peering, 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 126 active+clean; 1.7 GiB data, 9.6 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 868 op/s; 207/83136 objects degraded (0.249%); 5.3 MiB/s, 180 objects/s recovering 2023-09-23T21:18:22.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:18:22 smithi026 ceph-mon[179688]: failed to encode map e603 with expected crc 2023-09-23T21:18:22.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:18:22 smithi026 ceph-mon[179688]: failed to encode map e603 with expected crc 2023-09-23T21:18:22.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:18:22 smithi026 ceph-mon[179688]: osdmap e603: 8 total, 8 up, 6 in 2023-09-23T21:18:22.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:18:22 smithi026 ceph-mon[179688]: failed to encode map e603 with expected crc 2023-09-23T21:18:22.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:18:22 smithi026 ceph-mon[179688]: failed to encode map e603 with expected crc 2023-09-23T21:18:22.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:18:22 smithi026 ceph-mon[175151]: pgmap v1518: 129 pgs: 1 peering, 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 126 active+clean; 1.7 GiB data, 9.6 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 868 op/s; 207/83136 objects degraded (0.249%); 5.3 MiB/s, 180 objects/s recovering 2023-09-23T21:18:22.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:18:22 smithi026 ceph-mon[175151]: failed to encode map e603 with expected crc 2023-09-23T21:18:22.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:18:22 smithi026 ceph-mon[175151]: failed to encode map e603 with expected crc 2023-09-23T21:18:22.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:18:22 smithi026 ceph-mon[175151]: osdmap e603: 8 total, 8 up, 6 in 2023-09-23T21:18:22.883 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:18:22 smithi026 ceph-mon[175151]: failed to encode map e603 with expected crc 2023-09-23T21:18:22.883 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:18:22 smithi026 ceph-mon[175151]: failed to encode map e603 with expected crc 2023-09-23T21:18:23.248 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:18:23.254 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:18:23.255 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:18:23.275 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:18:23.327 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:18:23.356 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:18:23.451 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:18:23.627 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:18:23.634 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:18:23.635 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:18:23.648 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_historic_ops 2023-09-23T21:18:23.736 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:18:23.844 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:18:24.174 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T21:18:24.179 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:18:24.179 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:18:24.280 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:18:24.314 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:18:24.372 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:18:24.377 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:18:24.377 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:18:24.478 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:18:24.556 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:18:24.611 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:18:24 smithi070 ceph-mon[168282]: pgmap v1520: 129 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 127 active+clean; 1.5 GiB data, 9.2 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 835 op/s; 207/75300 objects degraded (0.275%); 3.6 MiB/s, 100 objects/s recovering 2023-09-23T21:18:24.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:18:24 smithi026 ceph-mon[175151]: pgmap v1520: 129 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 127 active+clean; 1.5 GiB data, 9.2 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 835 op/s; 207/75300 objects degraded (0.275%); 3.6 MiB/s, 100 objects/s recovering 2023-09-23T21:18:24.883 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:18:24 smithi026 ceph-mon[179688]: pgmap v1520: 129 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 127 active+clean; 1.5 GiB data, 9.2 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 835 op/s; 207/75300 objects degraded (0.275%); 3.6 MiB/s, 100 objects/s recovering 2023-09-23T21:18:25.568 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:18:25.587 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:18:25.587 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:18:25.633 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:18:25 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "17"}]: dispatch 2023-09-23T21:18:25.633 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:18:25 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "17"}]: dispatch 2023-09-23T21:18:25.689 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:18:25.718 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:18:25.754 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:18:25.759 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:18:25.759 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:18:25.860 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:18:25.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:18:25 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "17"}]: dispatch 2023-09-23T21:18:25.921 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:18:25.969 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0: {} 2023-09-23T21:18:26.004 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1: {} 2023-09-23T21:18:26.038 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2: {} 2023-09-23T21:18:26.081 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3: {} 2023-09-23T21:18:26.113 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4: {} 2023-09-23T21:18:26.146 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T21:18:26.178 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6: {} 2023-09-23T21:18:26.210 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T21:18:26.215 INFO:teuthology.orchestra.run.smithi026.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T21:18:26.215 INFO:teuthology.orchestra.run.smithi026.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T21:18:26.215 INFO:teuthology.orchestra.run.smithi026.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T21:18:26.215 INFO:teuthology.orchestra.run.smithi026.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T21:18:26.215 INFO:teuthology.orchestra.run.smithi026.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T21:18:26.215 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T21:18:26.215 INFO:teuthology.orchestra.run.smithi026.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T21:18:26.216 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T21:18:26.241 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:18:26.247 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:18:26.248 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:18:26.249 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:18:26.349 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:18:26.378 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:18:26.411 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:18:26.414 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:18:26.414 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:18:26.515 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:18:26.548 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:18:26.618 INFO:teuthology.orchestra.run.smithi026.stdout: 2023-09-23T21:18:26.618 INFO:teuthology.orchestra.run.smithi026.stdout:{"fsid":"b3f8e4ec-5a50-11ee-8db4-212e2dc638e7","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"noscrub,nodeep-scrub flag(s) set","count":20},"muted":false}},"mutes":[]},"election_epoch":30,"quorum":[0,1,2],"quorum_names":["a","b","c"],"quorum_age":1825,"monmap":{"epoch":4,"min_mon_release_name":"reef","num_mons":3},"osdmap":{"epoch":605,"num_osds":8,"num_up_osds":8,"osd_up_since":1695503591,"num_in_osds":6,"osd_in_since":1695503801,"num_remapped_pgs":1},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":129}],"num_pgs":129,"num_pools":5,"num_objects":23941,"data_bytes":1567954872,"bytes_used":9646731264,"bytes_avail":566323482624,"bytes_total":575970213888,"recovering_objects_per_sec":111,"recovering_bytes_per_sec":3065555,"recovering_keys_per_sec":0,"num_objects_recovered":812,"num_bytes_recovered":22413312,"num_keys_recovered":0,"write_bytes_sec":0,"read_op_per_sec":0,"write_op_per_sec":741},"fsmap":{"epoch":8,"id":1,"up":1,"in":1,"max":1,"by_rank":[{"filesystem_id":1,"rank":0,"name":"foo.smithi026.znfpnl","status":"up:active","gid":14658}],"up:standby":1},"mgrmap":{"available":true,"num_standbys":1,"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"services":{"dashboard":"https://172.21.15.26:8443/","prometheus":"http://172.21.15.26:9283/"}},"servicemap":{"epoch":30,"modified":"2023-09-23T21:17:37.170873+0000","services":{}},"progress_events":{"65997a66-5819-4593-bf43-3f88b3de0d61":{"message":"Global Recovery Event (2m)\n [===========================.] (remaining: 2s)","progress":0.97674417495727539,"add_to_ceph_s":true},"fc746e9d-3f76-434d-b6d2-8fc9e8a72a5e":{"message":"Upgrade to 18.2.0-460-gbb0daeff (4m)\n [============................] (remaining: 5m)","progress":0.44999998807907104,"add_to_ceph_s":true}}} 2023-09-23T21:18:26.843 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T21:18:26.847 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:18:26 smithi026 ceph-mon[175151]: pgmap v1521: 129 pgs: 129 active+clean; 1.5 GiB data, 9.0 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 741 op/s; 2.9 MiB/s, 111 objects/s recovering 2023-09-23T21:18:26.847 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:18:26 smithi026 ceph-mon[175151]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 207/75300 objects degraded (0.275%), 1 pg degraded, 1 pg undersized) 2023-09-23T21:18:26.847 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:18:26 smithi026 ceph-mon[175151]: failed to encode map e604 with expected crc 2023-09-23T21:18:26.847 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:18:26 smithi026 ceph-mon[175151]: failed to encode map e604 with expected crc 2023-09-23T21:18:26.847 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:18:26 smithi026 ceph-mon[175151]: failed to encode map e604 with expected crc 2023-09-23T21:18:26.847 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:18:26 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "17"}]': finished 2023-09-23T21:18:26.847 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:18:26 smithi026 ceph-mon[175151]: osdmap e604: 8 total, 8 up, 6 in 2023-09-23T21:18:26.848 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:18:26 smithi026 ceph-mon[175151]: failed to encode map e605 with expected crc 2023-09-23T21:18:26.848 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:18:26 smithi026 ceph-mon[175151]: osdmap e605: 8 total, 8 up, 6 in 2023-09-23T21:18:26.848 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:18:26 smithi026 ceph-mon[175151]: failed to encode map e605 with expected crc 2023-09-23T21:18:26.848 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:18:26 smithi026 ceph-mon[175151]: failed to encode map e605 with expected crc 2023-09-23T21:18:26.848 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:18:26 smithi026 ceph-mon[175151]: failed to encode map e605 with expected crc 2023-09-23T21:18:26.848 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:18:26 smithi026 ceph-mon[175151]: failed to encode map e605 with expected crc 2023-09-23T21:18:26.848 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:18:26 smithi026 ceph-mon[175151]: failed to encode map e605 with expected crc 2023-09-23T21:18:26.848 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:18:26 smithi026 ceph-mon[175151]: failed to encode map e605 with expected crc 2023-09-23T21:18:26.848 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:18:26 smithi026 ceph-mon[175151]: failed to encode map e605 with expected crc 2023-09-23T21:18:26.848 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:18:26 smithi026 ceph-mon[175151]: failed to encode map e605 with expected crc 2023-09-23T21:18:26.848 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:18:26 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/4009625920' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:18:26.849 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:18:26 smithi026 ceph-mon[179688]: pgmap v1521: 129 pgs: 129 active+clean; 1.5 GiB data, 9.0 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 741 op/s; 2.9 MiB/s, 111 objects/s recovering 2023-09-23T21:18:26.849 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:18:26 smithi026 ceph-mon[179688]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 207/75300 objects degraded (0.275%), 1 pg degraded, 1 pg undersized) 2023-09-23T21:18:26.849 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:18:26 smithi026 ceph-mon[179688]: failed to encode map e604 with expected crc 2023-09-23T21:18:26.849 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:18:26 smithi026 ceph-mon[179688]: failed to encode map e604 with expected crc 2023-09-23T21:18:26.849 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:18:26 smithi026 ceph-mon[179688]: failed to encode map e604 with expected crc 2023-09-23T21:18:26.849 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:18:26 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "17"}]': finished 2023-09-23T21:18:26.849 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:18:26 smithi026 ceph-mon[179688]: osdmap e604: 8 total, 8 up, 6 in 2023-09-23T21:18:26.849 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:18:26 smithi026 ceph-mon[179688]: failed to encode map e605 with expected crc 2023-09-23T21:18:26.849 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:18:26 smithi026 ceph-mon[179688]: osdmap e605: 8 total, 8 up, 6 in 2023-09-23T21:18:26.849 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:18:26 smithi026 ceph-mon[179688]: failed to encode map e605 with expected crc 2023-09-23T21:18:26.849 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:18:26 smithi026 ceph-mon[179688]: failed to encode map e605 with expected crc 2023-09-23T21:18:26.850 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:18:26 smithi026 ceph-mon[179688]: failed to encode map e605 with expected crc 2023-09-23T21:18:26.850 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:18:26 smithi026 ceph-mon[179688]: failed to encode map e605 with expected crc 2023-09-23T21:18:26.850 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:18:26 smithi026 ceph-mon[179688]: failed to encode map e605 with expected crc 2023-09-23T21:18:26.850 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:18:26 smithi026 ceph-mon[179688]: failed to encode map e605 with expected crc 2023-09-23T21:18:26.850 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:18:26 smithi026 ceph-mon[179688]: failed to encode map e605 with expected crc 2023-09-23T21:18:26.850 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:18:26 smithi026 ceph-mon[179688]: failed to encode map e605 with expected crc 2023-09-23T21:18:26.850 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:18:26 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/4009625920' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:18:26.850 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:18:26.851 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:18:26.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:18:26 smithi070 ceph-mon[168282]: pgmap v1521: 129 pgs: 129 active+clean; 1.5 GiB data, 9.0 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 741 op/s; 2.9 MiB/s, 111 objects/s recovering 2023-09-23T21:18:26.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:18:26 smithi070 ceph-mon[168282]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 207/75300 objects degraded (0.275%), 1 pg degraded, 1 pg undersized) 2023-09-23T21:18:26.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:18:26 smithi070 ceph-mon[168282]: failed to encode map e604 with expected crc 2023-09-23T21:18:26.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:18:26 smithi070 ceph-mon[168282]: failed to encode map e604 with expected crc 2023-09-23T21:18:26.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:18:26 smithi070 ceph-mon[168282]: failed to encode map e604 with expected crc 2023-09-23T21:18:26.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:18:26 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "17"}]': finished 2023-09-23T21:18:26.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:18:26 smithi070 ceph-mon[168282]: osdmap e604: 8 total, 8 up, 6 in 2023-09-23T21:18:26.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:18:26 smithi070 ceph-mon[168282]: failed to encode map e605 with expected crc 2023-09-23T21:18:26.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:18:26 smithi070 ceph-mon[168282]: osdmap e605: 8 total, 8 up, 6 in 2023-09-23T21:18:26.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:18:26 smithi070 ceph-mon[168282]: failed to encode map e605 with expected crc 2023-09-23T21:18:26.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:18:26 smithi070 ceph-mon[168282]: failed to encode map e605 with expected crc 2023-09-23T21:18:26.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:18:26 smithi070 ceph-mon[168282]: failed to encode map e605 with expected crc 2023-09-23T21:18:26.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:18:26 smithi070 ceph-mon[168282]: failed to encode map e605 with expected crc 2023-09-23T21:18:26.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:18:26 smithi070 ceph-mon[168282]: failed to encode map e605 with expected crc 2023-09-23T21:18:26.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:18:26 smithi070 ceph-mon[168282]: failed to encode map e605 with expected crc 2023-09-23T21:18:26.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:18:26 smithi070 ceph-mon[168282]: failed to encode map e605 with expected crc 2023-09-23T21:18:26.871 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:18:26 smithi070 ceph-mon[168282]: failed to encode map e605 with expected crc 2023-09-23T21:18:26.871 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:18:26 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/4009625920' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:18:26.873 INFO:teuthology.orchestra.run.smithi026.stderr:noscrub is unset 2023-09-23T21:18:26.952 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:18:26.979 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:18:27.010 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:18:27.013 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:18:27.013 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:18:27.114 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:18:27.140 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:18:27.171 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:18:27.174 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:18:27.174 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:18:27.275 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:18:27.304 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:18:27.853 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:18:27 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/1405930359' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-23T21:18:27.853 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:18:27 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/3796611933' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T21:18:27.853 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:18:27 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T21:18:27.853 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:18:27 smithi026 ceph-mon[179688]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:18:27.853 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:18:27 smithi026 ceph-mon[179688]: failed to encode map e606 with expected crc 2023-09-23T21:18:27.854 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:18:27 smithi026 ceph-mon[179688]: failed to encode map e606 with expected crc 2023-09-23T21:18:27.854 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:18:27 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T21:18:27.854 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:18:27 smithi026 ceph-mon[179688]: osdmap e606: 8 total, 8 up, 6 in 2023-09-23T21:18:27.854 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:18:27 smithi026 ceph-mon[179688]: 11.10 starting backfill to osd.2 from (548'800,563'4722] MIN to 603'7750 2023-09-23T21:18:27.854 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:18:27 smithi026 ceph-mon[179688]: 11.10 starting backfill to osd.5 from (0'0,0'0] MAX to 603'7750 2023-09-23T21:18:27.854 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:18:27 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "18"}]: dispatch 2023-09-23T21:18:27.854 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:18:27 smithi026 ceph-mon[179688]: failed to encode map e606 with expected crc 2023-09-23T21:18:27.854 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:18:27 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/1405930359' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-23T21:18:27.854 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:18:27 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/3796611933' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T21:18:27.855 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:18:27 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T21:18:27.855 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:18:27 smithi026 ceph-mon[175151]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:18:27.855 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:18:27 smithi026 ceph-mon[175151]: failed to encode map e606 with expected crc 2023-09-23T21:18:27.855 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:18:27 smithi026 ceph-mon[175151]: failed to encode map e606 with expected crc 2023-09-23T21:18:27.855 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:18:27 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T21:18:27.855 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:18:27 smithi026 ceph-mon[175151]: osdmap e606: 8 total, 8 up, 6 in 2023-09-23T21:18:27.855 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:18:27 smithi026 ceph-mon[175151]: 11.10 starting backfill to osd.2 from (548'800,563'4722] MIN to 603'7750 2023-09-23T21:18:27.855 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:18:27 smithi026 ceph-mon[175151]: 11.10 starting backfill to osd.5 from (0'0,0'0] MAX to 603'7750 2023-09-23T21:18:27.855 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:18:27 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "18"}]: dispatch 2023-09-23T21:18:27.855 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:18:27 smithi026 ceph-mon[175151]: failed to encode map e606 with expected crc 2023-09-23T21:18:27.894 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T21:18:27.905 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:18:27.905 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:18:28.007 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:18:28.076 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:18:28.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:18:27 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/1405930359' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-23T21:18:28.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:18:27 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/3796611933' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T21:18:28.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:18:27 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T21:18:28.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:18:27 smithi070 ceph-mon[168282]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:18:28.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:18:27 smithi070 ceph-mon[168282]: failed to encode map e606 with expected crc 2023-09-23T21:18:28.121 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:18:27 smithi070 ceph-mon[168282]: failed to encode map e606 with expected crc 2023-09-23T21:18:28.121 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:18:27 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T21:18:28.121 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:18:27 smithi070 ceph-mon[168282]: osdmap e606: 8 total, 8 up, 6 in 2023-09-23T21:18:28.121 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:18:27 smithi070 ceph-mon[168282]: 11.10 starting backfill to osd.2 from (548'800,563'4722] MIN to 603'7750 2023-09-23T21:18:28.121 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:18:27 smithi070 ceph-mon[168282]: 11.10 starting backfill to osd.5 from (0'0,0'0] MAX to 603'7750 2023-09-23T21:18:28.121 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:18:27 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "18"}]: dispatch 2023-09-23T21:18:28.121 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:18:27 smithi070 ceph-mon[168282]: failed to encode map e606 with expected crc 2023-09-23T21:18:28.585 INFO:tasks.cephadm.ceph_manager.ceph:making progress, resetting timeout 2023-09-23T21:18:28.585 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T21:18:28.639 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:18:28.642 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:18:28.643 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:18:28.744 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:18:28.770 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:18:28.802 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:18:28.805 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:18:28.805 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:18:28.881 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:18:28 smithi026 ceph-mon[175151]: pgmap v1525: 129 pgs: 129 active+clean; 1.3 GiB data, 8.6 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 1.13k op/s; 0 B/s, 36 objects/s recovering 2023-09-23T21:18:28.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:18:28 smithi026 ceph-mon[175151]: failed to encode map e606 with expected crc 2023-09-23T21:18:28.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:18:28 smithi026 ceph-mon[175151]: failed to encode map e607 with expected crc 2023-09-23T21:18:28.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:18:28 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "18"}]': finished 2023-09-23T21:18:28.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:18:28 smithi026 ceph-mon[175151]: osdmap e607: 8 total, 8 up, 6 in 2023-09-23T21:18:28.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:18:28 smithi026 ceph-mon[175151]: failed to encode map e607 with expected crc 2023-09-23T21:18:28.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:18:28 smithi026 ceph-mon[179688]: pgmap v1525: 129 pgs: 129 active+clean; 1.3 GiB data, 8.6 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 1.13k op/s; 0 B/s, 36 objects/s recovering 2023-09-23T21:18:28.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:18:28 smithi026 ceph-mon[179688]: failed to encode map e606 with expected crc 2023-09-23T21:18:28.883 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:18:28 smithi026 ceph-mon[179688]: failed to encode map e607 with expected crc 2023-09-23T21:18:28.883 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:18:28 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "18"}]': finished 2023-09-23T21:18:28.883 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:18:28 smithi026 ceph-mon[179688]: osdmap e607: 8 total, 8 up, 6 in 2023-09-23T21:18:28.883 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:18:28 smithi026 ceph-mon[179688]: failed to encode map e607 with expected crc 2023-09-23T21:18:28.906 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:18:28.929 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:18:28 smithi070 ceph-mon[168282]: pgmap v1525: 129 pgs: 129 active+clean; 1.3 GiB data, 8.6 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 1.13k op/s; 0 B/s, 36 objects/s recovering 2023-09-23T21:18:28.929 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:18:28 smithi070 ceph-mon[168282]: failed to encode map e606 with expected crc 2023-09-23T21:18:28.929 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:18:28 smithi070 ceph-mon[168282]: failed to encode map e607 with expected crc 2023-09-23T21:18:28.929 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:18:28 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "18"}]': finished 2023-09-23T21:18:28.930 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:18:28 smithi070 ceph-mon[168282]: osdmap e607: 8 total, 8 up, 6 in 2023-09-23T21:18:28.930 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:18:28 smithi070 ceph-mon[168282]: failed to encode map e607 with expected crc 2023-09-23T21:18:28.932 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:18:28.966 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:18:28.969 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:18:28.970 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:18:29.070 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:18:29.096 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:18:29.129 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:18:29.132 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:18:29.132 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:18:29.233 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:18:29.277 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:18:29.288 INFO:teuthology.orchestra.run.smithi026.stdout:ERROR: (22) Invalid argument 2023-09-23T21:18:29.288 INFO:teuthology.orchestra.run.smithi026.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T21:18:29.321 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:18:29.323 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:18:29.324 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:18:29.378 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:18:29.425 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:18:29.451 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:18:29.485 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:18:29.488 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:18:29.488 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:18:29.589 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:18:29.615 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:18:29.648 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:18:29.651 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:18:29.651 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:18:29.752 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:18:29.784 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:18:29.818 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_ops_in_flight 2023-09-23T21:18:29.827 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T21:18:29.829 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:18:29.830 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:18:29.864 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 21:18:29 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[164029]: ::ffff:172.21.15.70 - - [23/Sep/2023:21:18:29] "GET /metrics HTTP/1.1" 200 36378 "" "Prometheus/2.43.0" 2023-09-23T21:18:29.884 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T21:18:29.930 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:18:29.956 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:18:29.973 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T21:18:29.989 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:18:29.992 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:18:29.992 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:18:30.093 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:18:30.124 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:18:30.163 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:18:30.166 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:18:29 smithi070 ceph-mon[168282]: 1.d scrub starts 2023-09-23T21:18:30.166 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:18:29 smithi070 ceph-mon[168282]: failed to encode map e608 with expected crc 2023-09-23T21:18:30.166 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:18:29 smithi070 ceph-mon[168282]: failed to encode map e608 with expected crc 2023-09-23T21:18:30.166 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:18:29 smithi070 ceph-mon[168282]: failed to encode map e608 with expected crc 2023-09-23T21:18:30.166 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:18:29 smithi070 ceph-mon[168282]: osdmap e608: 8 total, 8 up, 6 in 2023-09-23T21:18:30.166 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:18:29 smithi070 ceph-mon[168282]: failed to encode map e608 with expected crc 2023-09-23T21:18:30.167 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:18:29 smithi070 ceph-mon[168282]: pgmap v1528: 129 pgs: 129 active+clean; 1.3 GiB data, 8.3 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 760 op/s 2023-09-23T21:18:30.167 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:18:29 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "19"}]: dispatch 2023-09-23T21:18:30.167 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:18:30.168 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:18:30.268 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:18:30.306 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:18:30.344 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:18:30.346 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:18:30.346 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:18:30.353 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:18:29 smithi026 ceph-mon[175151]: 1.d scrub starts 2023-09-23T21:18:30.353 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:18:29 smithi026 ceph-mon[175151]: failed to encode map e608 with expected crc 2023-09-23T21:18:30.353 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:18:29 smithi026 ceph-mon[175151]: failed to encode map e608 with expected crc 2023-09-23T21:18:30.353 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:18:29 smithi026 ceph-mon[175151]: failed to encode map e608 with expected crc 2023-09-23T21:18:30.353 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:18:29 smithi026 ceph-mon[175151]: osdmap e608: 8 total, 8 up, 6 in 2023-09-23T21:18:30.353 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:18:29 smithi026 ceph-mon[175151]: failed to encode map e608 with expected crc 2023-09-23T21:18:30.353 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:18:29 smithi026 ceph-mon[175151]: pgmap v1528: 129 pgs: 129 active+clean; 1.3 GiB data, 8.3 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 760 op/s 2023-09-23T21:18:30.354 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:18:29 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "19"}]: dispatch 2023-09-23T21:18:30.354 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:18:29 smithi026 ceph-mon[179688]: 1.d scrub starts 2023-09-23T21:18:30.354 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:18:29 smithi026 ceph-mon[179688]: failed to encode map e608 with expected crc 2023-09-23T21:18:30.355 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:18:29 smithi026 ceph-mon[179688]: failed to encode map e608 with expected crc 2023-09-23T21:18:30.355 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:18:29 smithi026 ceph-mon[179688]: failed to encode map e608 with expected crc 2023-09-23T21:18:30.355 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:18:29 smithi026 ceph-mon[179688]: osdmap e608: 8 total, 8 up, 6 in 2023-09-23T21:18:30.355 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:18:29 smithi026 ceph-mon[179688]: failed to encode map e608 with expected crc 2023-09-23T21:18:30.355 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:18:29 smithi026 ceph-mon[179688]: pgmap v1528: 129 pgs: 129 active+clean; 1.3 GiB data, 8.3 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 760 op/s 2023-09-23T21:18:30.355 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:18:29 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "19"}]: dispatch 2023-09-23T21:18:30.447 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:18:30.476 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:18:30.510 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:18:30.513 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:18:30.513 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:18:30.614 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:18:30.616 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:18:30.671 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:18:30.932 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T21:18:30.936 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:18:30 smithi026 ceph-mon[179688]: 11.6 scrub starts 2023-09-23T21:18:30.936 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:18:30 smithi026 ceph-mon[179688]: 11.6 scrub ok 2023-09-23T21:18:30.936 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:18:30 smithi026 ceph-mon[179688]: failed to encode map e609 with expected crc 2023-09-23T21:18:30.936 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:18:30 smithi026 ceph-mon[179688]: failed to encode map e609 with expected crc 2023-09-23T21:18:30.936 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:18:30 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "19"}]': finished 2023-09-23T21:18:30.936 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:18:30 smithi026 ceph-mon[179688]: osdmap e609: 8 total, 8 up, 6 in 2023-09-23T21:18:30.936 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:18:30 smithi026 ceph-mon[179688]: failed to encode map e609 with expected crc 2023-09-23T21:18:30.937 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:18:30 smithi026 ceph-mon[179688]: failed to encode map e609 with expected crc 2023-09-23T21:18:30.937 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:18:30 smithi026 ceph-mon[179688]: 11.11 starting backfill to osd.7 from (0'0,0'0] MAX to 606'7849 2023-09-23T21:18:30.937 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:18:30 smithi026 ceph-mon[179688]: 11.7 scrub starts 2023-09-23T21:18:30.937 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:18:30 smithi026 ceph-mon[179688]: 11.7 scrub ok 2023-09-23T21:18:30.937 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:18:30 smithi026 ceph-mon[179688]: osdmap e610: 8 total, 8 up, 6 in 2023-09-23T21:18:30.938 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:18:30 smithi026 ceph-mon[175151]: 11.6 scrub starts 2023-09-23T21:18:30.938 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:18:30 smithi026 ceph-mon[175151]: 11.6 scrub ok 2023-09-23T21:18:30.939 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:18:30 smithi026 ceph-mon[175151]: failed to encode map e609 with expected crc 2023-09-23T21:18:30.939 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:18:30 smithi026 ceph-mon[175151]: failed to encode map e609 with expected crc 2023-09-23T21:18:30.939 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:18:30 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "19"}]': finished 2023-09-23T21:18:30.939 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:18:30 smithi026 ceph-mon[175151]: osdmap e609: 8 total, 8 up, 6 in 2023-09-23T21:18:30.939 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:18:30 smithi026 ceph-mon[175151]: failed to encode map e609 with expected crc 2023-09-23T21:18:30.939 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:18:30 smithi026 ceph-mon[175151]: failed to encode map e609 with expected crc 2023-09-23T21:18:30.939 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:18:30 smithi026 ceph-mon[175151]: 11.11 starting backfill to osd.7 from (0'0,0'0] MAX to 606'7849 2023-09-23T21:18:30.939 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:18:30 smithi026 ceph-mon[175151]: 11.7 scrub starts 2023-09-23T21:18:30.939 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:18:30 smithi026 ceph-mon[175151]: 11.7 scrub ok 2023-09-23T21:18:30.939 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:18:30 smithi026 ceph-mon[175151]: osdmap e610: 8 total, 8 up, 6 in 2023-09-23T21:18:30.940 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:18:30.940 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:18:31.041 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:18:31.092 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:18:31.097 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:18:31.107 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:18:31.274 INFO:teuthology.orchestra.run.smithi026.stdout: 2023-09-23T21:18:31.276 INFO:teuthology.orchestra.run.smithi026.stderr:dumped all 2023-09-23T21:18:31.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:18:30 smithi070 ceph-mon[168282]: 11.6 scrub starts 2023-09-23T21:18:31.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:18:30 smithi070 ceph-mon[168282]: 11.6 scrub ok 2023-09-23T21:18:31.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:18:30 smithi070 ceph-mon[168282]: failed to encode map e609 with expected crc 2023-09-23T21:18:31.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:18:30 smithi070 ceph-mon[168282]: failed to encode map e609 with expected crc 2023-09-23T21:18:31.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:18:30 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "19"}]': finished 2023-09-23T21:18:31.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:18:30 smithi070 ceph-mon[168282]: osdmap e609: 8 total, 8 up, 6 in 2023-09-23T21:18:31.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:18:30 smithi070 ceph-mon[168282]: failed to encode map e609 with expected crc 2023-09-23T21:18:31.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:18:30 smithi070 ceph-mon[168282]: failed to encode map e609 with expected crc 2023-09-23T21:18:31.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:18:30 smithi070 ceph-mon[168282]: 11.11 starting backfill to osd.7 from (0'0,0'0] MAX to 606'7849 2023-09-23T21:18:31.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:18:30 smithi070 ceph-mon[168282]: 11.7 scrub starts 2023-09-23T21:18:31.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:18:30 smithi070 ceph-mon[168282]: 11.7 scrub ok 2023-09-23T21:18:31.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:18:30 smithi070 ceph-mon[168282]: osdmap e610: 8 total, 8 up, 6 in 2023-09-23T21:18:32.104 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:18:32.110 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:18:32.110 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:18:32.211 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:18:32.216 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:18:31 smithi026 ceph-mon[175151]: 11.8 scrub starts 2023-09-23T21:18:32.216 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:18:31 smithi026 ceph-mon[175151]: 11.8 scrub ok 2023-09-23T21:18:32.216 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:18:31 smithi026 ceph-mon[175151]: failed to encode map e610 with expected crc 2023-09-23T21:18:32.216 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:18:31 smithi026 ceph-mon[175151]: failed to encode map e610 with expected crc 2023-09-23T21:18:32.216 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:18:31 smithi026 ceph-mon[175151]: failed to encode map e610 with expected crc 2023-09-23T21:18:32.216 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:18:31 smithi026 ceph-mon[175151]: failed to encode map e610 with expected crc 2023-09-23T21:18:32.216 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:18:31 smithi026 ceph-mon[175151]: pgmap v1531: 129 pgs: 1 peering, 128 active+clean; 1.2 GiB data, 8.1 GiB used, 528 GiB / 536 GiB avail; 9.8 MiB/s, 156 objects/s recovering 2023-09-23T21:18:32.216 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:18:31 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:18:32.216 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:18:31 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:18:32.216 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:18:31 smithi026 ceph-mon[175151]: from='client.56791 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T21:18:32.216 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:18:31 smithi026 ceph-mon[175151]: 11.15 scrub starts 2023-09-23T21:18:32.217 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:18:31 smithi026 ceph-mon[175151]: 11.15 scrub ok 2023-09-23T21:18:32.217 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:18:31 smithi026 ceph-mon[175151]: osdmap e611: 8 total, 8 up, 6 in 2023-09-23T21:18:32.217 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:18:31 smithi026 ceph-mon[179688]: 11.8 scrub starts 2023-09-23T21:18:32.217 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:18:31 smithi026 ceph-mon[179688]: 11.8 scrub ok 2023-09-23T21:18:32.217 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:18:31 smithi026 ceph-mon[179688]: failed to encode map e610 with expected crc 2023-09-23T21:18:32.217 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:18:31 smithi026 ceph-mon[179688]: failed to encode map e610 with expected crc 2023-09-23T21:18:32.217 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:18:31 smithi026 ceph-mon[179688]: failed to encode map e610 with expected crc 2023-09-23T21:18:32.217 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:18:31 smithi026 ceph-mon[179688]: failed to encode map e610 with expected crc 2023-09-23T21:18:32.217 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:18:31 smithi026 ceph-mon[179688]: pgmap v1531: 129 pgs: 1 peering, 128 active+clean; 1.2 GiB data, 8.1 GiB used, 528 GiB / 536 GiB avail; 9.8 MiB/s, 156 objects/s recovering 2023-09-23T21:18:32.217 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:18:31 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:18:32.217 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:18:31 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:18:32.218 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:18:31 smithi026 ceph-mon[179688]: from='client.56791 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T21:18:32.218 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:18:31 smithi026 ceph-mon[179688]: 11.15 scrub starts 2023-09-23T21:18:32.218 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:18:31 smithi026 ceph-mon[179688]: 11.15 scrub ok 2023-09-23T21:18:32.218 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:18:31 smithi026 ceph-mon[179688]: osdmap e611: 8 total, 8 up, 6 in 2023-09-23T21:18:32.321 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:18:32.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:18:31 smithi070 ceph-mon[168282]: 11.8 scrub starts 2023-09-23T21:18:32.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:18:31 smithi070 ceph-mon[168282]: 11.8 scrub ok 2023-09-23T21:18:32.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:18:31 smithi070 ceph-mon[168282]: failed to encode map e610 with expected crc 2023-09-23T21:18:32.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:18:31 smithi070 ceph-mon[168282]: failed to encode map e610 with expected crc 2023-09-23T21:18:32.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:18:31 smithi070 ceph-mon[168282]: failed to encode map e610 with expected crc 2023-09-23T21:18:32.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:18:31 smithi070 ceph-mon[168282]: failed to encode map e610 with expected crc 2023-09-23T21:18:32.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:18:31 smithi070 ceph-mon[168282]: pgmap v1531: 129 pgs: 1 peering, 128 active+clean; 1.2 GiB data, 8.1 GiB used, 528 GiB / 536 GiB avail; 9.8 MiB/s, 156 objects/s recovering 2023-09-23T21:18:32.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:18:31 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:18:32.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:18:31 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:18:32.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:18:31 smithi070 ceph-mon[168282]: from='client.56791 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T21:18:32.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:18:31 smithi070 ceph-mon[168282]: 11.15 scrub starts 2023-09-23T21:18:32.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:18:31 smithi070 ceph-mon[168282]: 11.15 scrub ok 2023-09-23T21:18:32.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:18:31 smithi070 ceph-mon[168282]: osdmap e611: 8 total, 8 up, 6 in 2023-09-23T21:18:33.134 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:18:32 smithi026 ceph-mon[179688]: 11.16 scrub starts 2023-09-23T21:18:33.134 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:18:32 smithi026 ceph-mon[179688]: 11.16 scrub ok 2023-09-23T21:18:33.134 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:18:32 smithi026 ceph-mon[179688]: failed to encode map e611 with expected crc 2023-09-23T21:18:33.134 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:18:32 smithi026 ceph-mon[179688]: failed to encode map e611 with expected crc 2023-09-23T21:18:33.134 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:18:32 smithi026 ceph-mon[179688]: 11.12 starting backfill to osd.7 from (0'0,0'0] MAX to 607'7944 2023-09-23T21:18:33.134 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:18:32 smithi026 ceph-mon[179688]: failed to encode map e611 with expected crc 2023-09-23T21:18:33.134 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:18:32 smithi026 ceph-mon[179688]: 1.1d scrub starts 2023-09-23T21:18:33.135 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:18:32 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:18:33.135 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:18:32 smithi026 ceph-mon[179688]: osdmap e612: 8 total, 8 up, 6 in 2023-09-23T21:18:33.135 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:18:32 smithi026 ceph-mon[175151]: 11.16 scrub starts 2023-09-23T21:18:33.135 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:18:32 smithi026 ceph-mon[175151]: 11.16 scrub ok 2023-09-23T21:18:33.135 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:18:32 smithi026 ceph-mon[175151]: failed to encode map e611 with expected crc 2023-09-23T21:18:33.135 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:18:32 smithi026 ceph-mon[175151]: failed to encode map e611 with expected crc 2023-09-23T21:18:33.135 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:18:32 smithi026 ceph-mon[175151]: 11.12 starting backfill to osd.7 from (0'0,0'0] MAX to 607'7944 2023-09-23T21:18:33.135 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:18:32 smithi026 ceph-mon[175151]: failed to encode map e611 with expected crc 2023-09-23T21:18:33.135 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:18:32 smithi026 ceph-mon[175151]: 1.1d scrub starts 2023-09-23T21:18:33.135 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:18:32 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:18:33.136 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:18:32 smithi026 ceph-mon[175151]: osdmap e612: 8 total, 8 up, 6 in 2023-09-23T21:18:33.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:18:32 smithi070 ceph-mon[168282]: 11.16 scrub starts 2023-09-23T21:18:33.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:18:32 smithi070 ceph-mon[168282]: 11.16 scrub ok 2023-09-23T21:18:33.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:18:32 smithi070 ceph-mon[168282]: failed to encode map e611 with expected crc 2023-09-23T21:18:33.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:18:32 smithi070 ceph-mon[168282]: failed to encode map e611 with expected crc 2023-09-23T21:18:33.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:18:32 smithi070 ceph-mon[168282]: 11.12 starting backfill to osd.7 from (0'0,0'0] MAX to 607'7944 2023-09-23T21:18:33.371 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:18:32 smithi070 ceph-mon[168282]: failed to encode map e611 with expected crc 2023-09-23T21:18:33.371 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:18:32 smithi070 ceph-mon[168282]: 1.1d scrub starts 2023-09-23T21:18:33.371 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:18:32 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:18:33.371 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:18:32 smithi070 ceph-mon[168282]: osdmap e612: 8 total, 8 up, 6 in 2023-09-23T21:18:33.518 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:18:33.530 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:18:33.530 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:18:33.604 INFO:teuthology.orchestra.run.smithi026.stdout:{"pg_ready":true,"pg_map":{"version":1531,"stamp":"2023-09-23T21:18:31.196358+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":1303746488,"num_objects":19910,"num_object_clones":0,"num_object_copies":59730,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":19910,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":170681,"num_write_kb":6076350,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":15882,"num_bytes_recovered":974671166,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":143830,"ondisk_log_size":143830,"up":387,"acting":387,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":398,"num_osds":6,"num_per_pool_osds":6,"num_per_pool_omap_osds":6,"kb":562470912,"kb_used":8457544,"kb_used_data":4197272,"kb_used_omap":125,"kb_used_meta":4260098,"kb_avail":554013368,"statfs":{"total":575970213888,"available":567309688832,"internally_reserved":0,"allocated":4298006528,"data_stored":4284368956,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":128662,"internal_metadata":4362340714},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,4,0,2],"upper_bound":64},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-96174080,"num_objects":-1467,"num_object_clones":0,"num_object_copies":-4401,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":-1467,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":-3963,"num_write_kb":-193568,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":625,"num_bytes_recovered":40960000,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"4.000989"},"pg_stats":[{"pgid":"11.12","version":"605'7909","reported_seq":7989,"reported_epoch":605,"state":"active+clean","last_fresh":"2023-09-23T21:18:26.291680+0000","last_change":"2023-09-23T21:16:32.944306+0000","last_active":"2023-09-23T21:18:26.291680+0000","last_peered":"2023-09-23T21:18:26.291680+0000","last_clean":"2023-09-23T21:18:26.291680+0000","last_became_active":"2023-09-23T21:16:32.940670+0000","last_became_peered":"2023-09-23T21:16:32.940670+0000","last_unstale":"2023-09-23T21:18:26.291680+0000","last_undegraded":"2023-09-23T21:18:26.291680+0000","last_fullsized":"2023-09-23T21:18:26.291680+0000","mapping_epoch":559,"log_start":"548'632","ondisk_log_start":"548'632","created":559,"last_epoch_clean":560,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_clean_scrub_stamp":"2023-09-23T21:15:24.860968+0000","objects_scrubbed":0,"log_size":7277,"log_dups_size":0,"ondisk_log_size":7277,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":41353216,"num_objects":631,"num_object_clones":0,"num_object_copies":1893,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":631,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5625,"num_write_kb":200192,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.18","version":"79'161","reported_seq":629,"reported_epoch":604,"state":"active+clean","last_fresh":"2023-09-23T21:18:25.630683+0000","last_change":"2023-09-23T21:17:39.906535+0000","last_active":"2023-09-23T21:18:25.630683+0000","last_peered":"2023-09-23T21:18:25.630683+0000","last_clean":"2023-09-23T21:18:25.630683+0000","last_became_active":"2023-09-23T21:17:39.906154+0000","last_became_peered":"2023-09-23T21:17:39.906154+0000","last_unstale":"2023-09-23T21:18:25.630683+0000","last_undegraded":"2023-09-23T21:18:25.630683+0000","last_fullsized":"2023-09-23T21:18:25.630683+0000","mapping_epoch":585,"log_start":"0'0","ondisk_log_start":"0'0","created":479,"last_epoch_clean":586,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:15:39.474823+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:15:39.474823+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:48:39.784037+0000","scrub_duration":0.074664395999999994,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":525376,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1a","version":"68'1","reported_seq":196,"reported_epoch":605,"state":"active+clean","last_fresh":"2023-09-23T21:18:25.885212+0000","last_change":"2023-09-23T21:13:13.291403+0000","last_active":"2023-09-23T21:18:25.885212+0000","last_peered":"2023-09-23T21:18:25.885212+0000","last_clean":"2023-09-23T21:18:25.885212+0000","last_became_active":"2023-09-23T21:13:13.291123+0000","last_became_peered":"2023-09-23T21:13:13.291123+0000","last_unstale":"2023-09-23T21:18:25.885212+0000","last_undegraded":"2023-09-23T21:18:25.885212+0000","last_fullsized":"2023-09-23T21:18:25.885212+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":495,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:45:15.427430+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,0],"acting":[4,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":265,"reported_epoch":605,"state":"active+clean","last_fresh":"2023-09-23T21:18:25.885854+0000","last_change":"2023-09-23T21:12:10.767963+0000","last_active":"2023-09-23T21:18:25.885854+0000","last_peered":"2023-09-23T21:18:25.885854+0000","last_clean":"2023-09-23T21:18:25.885854+0000","last_became_active":"2023-09-23T21:12:10.767126+0000","last_became_peered":"2023-09-23T21:12:10.767126+0000","last_unstale":"2023-09-23T21:18:25.885854+0000","last_undegraded":"2023-09-23T21:18:25.885854+0000","last_fullsized":"2023-09-23T21:18:25.885854+0000","mapping_epoch":470,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:45:47.138277+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,5],"acting":[4,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.11","version":"603'7786","reported_seq":7862,"reported_epoch":603,"state":"active+clean","last_fresh":"2023-09-23T21:18:24.346500+0000","last_change":"2023-09-23T21:16:32.928552+0000","last_active":"2023-09-23T21:18:24.346500+0000","last_peered":"2023-09-23T21:18:24.346500+0000","last_clean":"2023-09-23T21:18:24.346500+0000","last_became_active":"2023-09-23T21:16:32.926639+0000","last_became_peered":"2023-09-23T21:16:32.926639+0000","last_unstale":"2023-09-23T21:18:24.346500+0000","last_undegraded":"2023-09-23T21:18:24.346500+0000","last_fullsized":"2023-09-23T21:18:24.346500+0000","mapping_epoch":559,"log_start":"555'3628","ondisk_log_start":"555'3628","created":559,"last_epoch_clean":560,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_clean_scrub_stamp":"2023-09-23T21:15:24.860968+0000","objects_scrubbed":0,"log_size":4158,"log_dups_size":0,"ondisk_log_size":4158,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47644672,"num_objects":727,"num_object_clones":0,"num_object_copies":2181,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":727,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5469,"num_write_kb":198272,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,2],"acting":[3,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1b","version":"79'161","reported_seq":792,"reported_epoch":604,"state":"active+clean","last_fresh":"2023-09-23T21:18:25.634461+0000","last_change":"2023-09-23T21:15:38.423372+0000","last_active":"2023-09-23T21:18:25.634461+0000","last_peered":"2023-09-23T21:18:25.634461+0000","last_clean":"2023-09-23T21:18:25.634461+0000","last_became_active":"2023-09-23T21:15:23.802580+0000","last_became_peered":"2023-09-23T21:15:23.802580+0000","last_unstale":"2023-09-23T21:18:25.634461+0000","last_undegraded":"2023-09-23T21:18:25.634461+0000","last_fullsized":"2023-09-23T21:18:25.634461+0000","mapping_epoch":542,"log_start":"0'0","ondisk_log_start":"0'0","created":479,"last_epoch_clean":543,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:15:38.423271+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:15:38.423271+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:04:46.575186+0000","scrub_duration":0.00258465,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":623,"reported_epoch":607,"state":"active+clean","last_fresh":"2023-09-23T21:18:27.905142+0000","last_change":"2023-09-23T21:16:42.448529+0000","last_active":"2023-09-23T21:18:27.905142+0000","last_peered":"2023-09-23T21:18:27.905142+0000","last_clean":"2023-09-23T21:18:27.905142+0000","last_became_active":"2023-09-23T21:16:42.447803+0000","last_became_peered":"2023-09-23T21:16:42.447803+0000","last_unstale":"2023-09-23T21:18:27.905142+0000","last_undegraded":"2023-09-23T21:18:27.905142+0000","last_fullsized":"2023-09-23T21:18:27.905142+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:41:33.121784+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":625,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-23T21:18:28.897638+0000","last_change":"2023-09-23T21:16:42.441896+0000","last_active":"2023-09-23T21:18:28.897638+0000","last_peered":"2023-09-23T21:18:28.897638+0000","last_clean":"2023-09-23T21:18:28.897638+0000","last_became_active":"2023-09-23T21:16:42.441526+0000","last_became_peered":"2023-09-23T21:16:42.441526+0000","last_unstale":"2023-09-23T21:18:28.897638+0000","last_undegraded":"2023-09-23T21:18:28.897638+0000","last_fullsized":"2023-09-23T21:18:28.897638+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:27:36.326225+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.10","version":"607'7793","reported_seq":11416,"reported_epoch":609,"state":"peering","last_fresh":"2023-09-23T21:18:29.896524+0000","last_change":"2023-09-23T21:18:29.896524+0000","last_active":"2023-09-23T21:18:29.896339+0000","last_peered":"2023-09-23T21:18:29.525136+0000","last_clean":"2023-09-23T21:18:29.525136+0000","last_became_active":"2023-09-23T21:18:27.468628+0000","last_became_peered":"2023-09-23T21:18:27.468628+0000","last_unstale":"2023-09-23T21:18:29.896524+0000","last_undegraded":"2023-09-23T21:18:29.896524+0000","last_fullsized":"2023-09-23T21:18:29.896524+0000","mapping_epoch":609,"log_start":"556'3618","ondisk_log_start":"556'3618","created":559,"last_epoch_clean":608,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_clean_scrub_stamp":"2023-09-23T21:15:24.860968+0000","objects_scrubbed":0,"log_size":4175,"log_dups_size":0,"ondisk_log_size":4175,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":40599552,"num_objects":620,"num_object_clones":0,"num_object_copies":1860,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":620,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5536,"num_write_kb":196960,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1440,"num_bytes_recovered":63569920,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,5],"acting":[0,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1a","version":"79'161","reported_seq":703,"reported_epoch":605,"state":"active+clean","last_fresh":"2023-09-23T21:18:25.884936+0000","last_change":"2023-09-23T21:16:45.419221+0000","last_active":"2023-09-23T21:18:25.884936+0000","last_peered":"2023-09-23T21:18:25.884936+0000","last_clean":"2023-09-23T21:18:25.884936+0000","last_became_active":"2023-09-23T21:16:45.418994+0000","last_became_peered":"2023-09-23T21:16:45.418994+0000","last_unstale":"2023-09-23T21:18:25.884936+0000","last_undegraded":"2023-09-23T21:18:25.884936+0000","last_fullsized":"2023-09-23T21:18:25.884936+0000","mapping_epoch":567,"log_start":"0'0","ondisk_log_start":"0'0","created":479,"last_epoch_clean":568,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:15:45.158107+0000","last_deep_scrub":"79'161","last_deep_scrub_stamp":"2023-09-23T21:05:32.630292+0000","last_clean_scrub_stamp":"2023-09-23T21:15:45.158107+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:08:08.342444+0000","scrub_duration":0.001999373,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,5],"acting":[4,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":120,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-23T21:18:28.896756+0000","last_change":"2023-09-23T21:13:30.806084+0000","last_active":"2023-09-23T21:18:28.896756+0000","last_peered":"2023-09-23T21:18:28.896756+0000","last_clean":"2023-09-23T21:18:28.896756+0000","last_became_active":"2023-09-23T21:13:30.805743+0000","last_became_peered":"2023-09-23T21:13:30.805743+0000","last_unstale":"2023-09-23T21:18:28.896756+0000","last_undegraded":"2023-09-23T21:18:28.896756+0000","last_fullsized":"2023-09-23T21:18:28.896756+0000","mapping_epoch":501,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:48:10.849281+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":484,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-23T21:18:28.897756+0000","last_change":"2023-09-23T21:13:44.410921+0000","last_active":"2023-09-23T21:18:28.897756+0000","last_peered":"2023-09-23T21:18:28.897756+0000","last_clean":"2023-09-23T21:18:28.897756+0000","last_became_active":"2023-09-23T21:13:44.410633+0000","last_became_peered":"2023-09-23T21:13:44.410633+0000","last_unstale":"2023-09-23T21:18:28.897756+0000","last_undegraded":"2023-09-23T21:18:28.897756+0000","last_fullsized":"2023-09-23T21:18:28.897756+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:57:10.116786+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.15","version":"79'161","reported_seq":554,"reported_epoch":605,"state":"active+clean","last_fresh":"2023-09-23T21:18:25.878947+0000","last_change":"2023-09-23T21:16:45.426006+0000","last_active":"2023-09-23T21:18:25.878947+0000","last_peered":"2023-09-23T21:18:25.878947+0000","last_clean":"2023-09-23T21:18:25.878947+0000","last_became_active":"2023-09-23T21:16:45.425654+0000","last_became_peered":"2023-09-23T21:16:45.425654+0000","last_unstale":"2023-09-23T21:18:25.878947+0000","last_undegraded":"2023-09-23T21:18:25.878947+0000","last_fullsized":"2023-09-23T21:18:25.878947+0000","mapping_epoch":567,"log_start":"0'0","ondisk_log_start":"0'0","created":479,"last_epoch_clean":568,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:15:36.742247+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:15:36.742247+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0081306369999999996,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,3],"acting":[7,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":585,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-23T21:18:28.897879+0000","last_change":"2023-09-23T21:13:12.235728+0000","last_active":"2023-09-23T21:18:28.897879+0000","last_peered":"2023-09-23T21:18:28.897879+0000","last_clean":"2023-09-23T21:18:28.897879+0000","last_became_active":"2023-09-23T21:13:12.216549+0000","last_became_peered":"2023-09-23T21:13:12.216549+0000","last_unstale":"2023-09-23T21:18:28.897879+0000","last_undegraded":"2023-09-23T21:18:28.897879+0000","last_fullsized":"2023-09-23T21:18:28.897879+0000","mapping_epoch":493,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:39:33.324239+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":647,"reported_epoch":607,"state":"active+clean","last_fresh":"2023-09-23T21:18:27.897457+0000","last_change":"2023-09-23T21:12:10.790349+0000","last_active":"2023-09-23T21:18:27.897457+0000","last_peered":"2023-09-23T21:18:27.897457+0000","last_clean":"2023-09-23T21:18:27.897457+0000","last_became_active":"2023-09-23T21:12:10.789977+0000","last_became_peered":"2023-09-23T21:12:10.789977+0000","last_unstale":"2023-09-23T21:18:27.897457+0000","last_undegraded":"2023-09-23T21:18:27.897457+0000","last_fullsized":"2023-09-23T21:18:27.897457+0000","mapping_epoch":470,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:49:40.191147+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,5],"acting":[3,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"79'161","reported_seq":1002,"reported_epoch":604,"state":"active+clean","last_fresh":"2023-09-23T21:18:25.633502+0000","last_change":"2023-09-23T21:16:45.424773+0000","last_active":"2023-09-23T21:18:25.633502+0000","last_peered":"2023-09-23T21:18:25.633502+0000","last_clean":"2023-09-23T21:18:25.633502+0000","last_became_active":"2023-09-23T21:16:45.424458+0000","last_became_peered":"2023-09-23T21:16:45.424458+0000","last_unstale":"2023-09-23T21:18:25.633502+0000","last_undegraded":"2023-09-23T21:18:25.633502+0000","last_fullsized":"2023-09-23T21:18:25.633502+0000","mapping_epoch":567,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":568,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:25.096401+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:25.096401+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:40:17.871561+0000","scrub_duration":0.002859393,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":349,"reported_epoch":604,"state":"active+clean","last_fresh":"2023-09-23T21:18:25.634227+0000","last_change":"2023-09-23T21:16:42.427856+0000","last_active":"2023-09-23T21:18:25.634227+0000","last_peered":"2023-09-23T21:18:25.634227+0000","last_clean":"2023-09-23T21:18:25.634227+0000","last_became_active":"2023-09-23T21:16:42.427529+0000","last_became_peered":"2023-09-23T21:16:42.427529+0000","last_unstale":"2023-09-23T21:18:25.634227+0000","last_undegraded":"2023-09-23T21:18:25.634227+0000","last_fullsized":"2023-09-23T21:18:25.634227+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:24:03.818632+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":602,"reported_epoch":607,"state":"active+clean","last_fresh":"2023-09-23T21:18:27.901244+0000","last_change":"2023-09-23T21:16:42.448719+0000","last_active":"2023-09-23T21:18:27.901244+0000","last_peered":"2023-09-23T21:18:27.901244+0000","last_clean":"2023-09-23T21:18:27.901244+0000","last_became_active":"2023-09-23T21:16:42.448219+0000","last_became_peered":"2023-09-23T21:16:42.448219+0000","last_unstale":"2023-09-23T21:18:27.901244+0000","last_undegraded":"2023-09-23T21:18:27.901244+0000","last_fullsized":"2023-09-23T21:18:27.901244+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:02:28.864357+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.17","version":"79'161","reported_seq":756,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-23T21:18:28.896980+0000","last_change":"2023-09-23T21:16:45.413663+0000","last_active":"2023-09-23T21:18:28.896980+0000","last_peered":"2023-09-23T21:18:28.896980+0000","last_clean":"2023-09-23T21:18:28.896980+0000","last_became_active":"2023-09-23T21:16:45.413311+0000","last_became_peered":"2023-09-23T21:16:45.413311+0000","last_unstale":"2023-09-23T21:18:28.896980+0000","last_undegraded":"2023-09-23T21:18:28.896980+0000","last_fullsized":"2023-09-23T21:18:28.896980+0000","mapping_epoch":567,"log_start":"0'0","ondisk_log_start":"0'0","created":479,"last_epoch_clean":568,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:15:50.325741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:15:50.325741+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:07:43.285354+0000","scrub_duration":0.0040500149999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":523,"reported_epoch":607,"state":"active+clean","last_fresh":"2023-09-23T21:18:27.902101+0000","last_change":"2023-09-23T21:17:02.182606+0000","last_active":"2023-09-23T21:18:27.902101+0000","last_peered":"2023-09-23T21:18:27.902101+0000","last_clean":"2023-09-23T21:18:27.902101+0000","last_became_active":"2023-09-23T21:16:42.446517+0000","last_became_peered":"2023-09-23T21:16:42.446517+0000","last_unstale":"2023-09-23T21:18:27.902101+0000","last_undegraded":"2023-09-23T21:18:27.902101+0000","last_fullsized":"2023-09-23T21:18:27.902101+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:46:08.337878+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":540,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,4],"acting":[3,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":293,"reported_epoch":604,"state":"active+clean","last_fresh":"2023-09-23T21:18:25.634323+0000","last_change":"2023-09-23T21:13:30.821489+0000","last_active":"2023-09-23T21:18:25.634323+0000","last_peered":"2023-09-23T21:18:25.634323+0000","last_clean":"2023-09-23T21:18:25.634323+0000","last_became_active":"2023-09-23T21:13:30.821125+0000","last_became_peered":"2023-09-23T21:13:30.821125+0000","last_unstale":"2023-09-23T21:18:25.634323+0000","last_undegraded":"2023-09-23T21:18:25.634323+0000","last_fullsized":"2023-09-23T21:18:25.634323+0000","mapping_epoch":501,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:47:18.553417+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,5],"acting":[2,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.16","version":"79'161","reported_seq":541,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-23T21:18:28.897563+0000","last_change":"2023-09-23T21:15:46.326726+0000","last_active":"2023-09-23T21:18:28.897563+0000","last_peered":"2023-09-23T21:18:28.897563+0000","last_clean":"2023-09-23T21:18:28.897563+0000","last_became_active":"2023-09-23T21:13:33.770407+0000","last_became_peered":"2023-09-23T21:13:33.770407+0000","last_unstale":"2023-09-23T21:18:28.897563+0000","last_undegraded":"2023-09-23T21:18:28.897563+0000","last_fullsized":"2023-09-23T21:18:28.897563+0000","mapping_epoch":504,"log_start":"0'0","ondisk_log_start":"0'0","created":479,"last_epoch_clean":505,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:15:46.326589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:15:46.326589+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:50:23.810054+0000","scrub_duration":0.0062069059999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.14","version":"79'22","reported_seq":209,"reported_epoch":605,"state":"active+clean","last_fresh":"2023-09-23T21:18:25.885452+0000","last_change":"2023-09-23T21:17:02.198735+0000","last_active":"2023-09-23T21:18:25.885452+0000","last_peered":"2023-09-23T21:18:25.885452+0000","last_clean":"2023-09-23T21:18:25.885452+0000","last_became_active":"2023-09-23T21:16:42.492861+0000","last_became_peered":"2023-09-23T21:16:42.492861+0000","last_unstale":"2023-09-23T21:18:25.885452+0000","last_undegraded":"2023-09-23T21:18:25.885452+0000","last_fullsized":"2023-09-23T21:18:25.885452+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:00:09.949965+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":14202,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,2],"acting":[4,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":534,"reported_epoch":605,"state":"active+clean","last_fresh":"2023-09-23T21:18:25.885117+0000","last_change":"2023-09-23T21:12:10.839271+0000","last_active":"2023-09-23T21:18:25.885117+0000","last_peered":"2023-09-23T21:18:25.885117+0000","last_clean":"2023-09-23T21:18:25.885117+0000","last_became_active":"2023-09-23T21:12:10.839042+0000","last_became_peered":"2023-09-23T21:12:10.839042+0000","last_unstale":"2023-09-23T21:18:25.885117+0000","last_undegraded":"2023-09-23T21:18:25.885117+0000","last_fullsized":"2023-09-23T21:18:25.885117+0000","mapping_epoch":470,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:10:26.414737+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,2],"acting":[4,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.11","version":"79'161","reported_seq":546,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-23T21:18:28.896569+0000","last_change":"2023-09-23T21:15:47.300109+0000","last_active":"2023-09-23T21:18:28.896569+0000","last_peered":"2023-09-23T21:18:28.896569+0000","last_clean":"2023-09-23T21:18:28.896569+0000","last_became_active":"2023-09-23T21:13:33.767067+0000","last_became_peered":"2023-09-23T21:13:33.767067+0000","last_unstale":"2023-09-23T21:18:28.896569+0000","last_undegraded":"2023-09-23T21:18:28.896569+0000","last_fullsized":"2023-09-23T21:18:28.896569+0000","mapping_epoch":504,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":505,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:15:47.299947+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:15:47.299947+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:04:59.029359+0000","scrub_duration":0.0061353110000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":57,"reported_epoch":605,"state":"active+clean","last_fresh":"2023-09-23T21:18:25.878742+0000","last_change":"2023-09-23T21:16:42.531060+0000","last_active":"2023-09-23T21:18:25.878742+0000","last_peered":"2023-09-23T21:18:25.878742+0000","last_clean":"2023-09-23T21:18:25.878742+0000","last_became_active":"2023-09-23T21:16:42.449198+0000","last_became_peered":"2023-09-23T21:16:42.449198+0000","last_unstale":"2023-09-23T21:18:25.878742+0000","last_undegraded":"2023-09-23T21:18:25.878742+0000","last_fullsized":"2023-09-23T21:18:25.878742+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:30:23.428886+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":165,"reported_epoch":604,"state":"active+clean","last_fresh":"2023-09-23T21:18:25.633189+0000","last_change":"2023-09-23T21:16:42.491023+0000","last_active":"2023-09-23T21:18:25.633189+0000","last_peered":"2023-09-23T21:18:25.633189+0000","last_clean":"2023-09-23T21:18:25.633189+0000","last_became_active":"2023-09-23T21:16:42.489896+0000","last_became_peered":"2023-09-23T21:16:42.489896+0000","last_unstale":"2023-09-23T21:18:25.633189+0000","last_undegraded":"2023-09-23T21:18:25.633189+0000","last_fullsized":"2023-09-23T21:18:25.633189+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:58:54.036098+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,7],"acting":[2,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.10","version":"79'161","reported_seq":780,"reported_epoch":605,"state":"active+clean","last_fresh":"2023-09-23T21:18:25.884760+0000","last_change":"2023-09-23T21:16:45.416227+0000","last_active":"2023-09-23T21:18:25.884760+0000","last_peered":"2023-09-23T21:18:25.884760+0000","last_clean":"2023-09-23T21:18:25.884760+0000","last_became_active":"2023-09-23T21:16:45.415841+0000","last_became_peered":"2023-09-23T21:16:45.415841+0000","last_unstale":"2023-09-23T21:18:25.884760+0000","last_undegraded":"2023-09-23T21:18:25.884760+0000","last_fullsized":"2023-09-23T21:18:25.884760+0000","mapping_epoch":567,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":568,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:06:50.336992+0000","last_deep_scrub":"79'161","last_deep_scrub_stamp":"2023-09-23T21:06:50.336992+0000","last_clean_scrub_stamp":"2023-09-23T21:06:50.336992+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:37:30.094658+0000","scrub_duration":0.0026150930000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,7],"acting":[4,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":569,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-23T21:18:28.896401+0000","last_change":"2023-09-23T21:16:42.446324+0000","last_active":"2023-09-23T21:18:28.896401+0000","last_peered":"2023-09-23T21:18:28.896401+0000","last_clean":"2023-09-23T21:18:28.896401+0000","last_became_active":"2023-09-23T21:16:42.445997+0000","last_became_peered":"2023-09-23T21:16:42.445997+0000","last_unstale":"2023-09-23T21:18:28.896401+0000","last_undegraded":"2023-09-23T21:18:28.896401+0000","last_fullsized":"2023-09-23T21:18:28.896401+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:46:35.028292+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":379,"reported_epoch":607,"state":"active+clean","last_fresh":"2023-09-23T21:18:27.901960+0000","last_change":"2023-09-23T21:16:42.473872+0000","last_active":"2023-09-23T21:18:27.901960+0000","last_peered":"2023-09-23T21:18:27.901960+0000","last_clean":"2023-09-23T21:18:27.901960+0000","last_became_active":"2023-09-23T21:16:42.473595+0000","last_became_peered":"2023-09-23T21:16:42.473595+0000","last_unstale":"2023-09-23T21:18:27.901960+0000","last_undegraded":"2023-09-23T21:18:27.901960+0000","last_fullsized":"2023-09-23T21:18:27.901960+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:15:19.186428+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.19","version":"607'7804","reported_seq":9591,"reported_epoch":607,"state":"active+clean","last_fresh":"2023-09-23T21:18:28.352569+0000","last_change":"2023-09-23T21:18:20.363419+0000","last_active":"2023-09-23T21:18:28.352569+0000","last_peered":"2023-09-23T21:18:28.352569+0000","last_clean":"2023-09-23T21:18:28.352569+0000","last_became_active":"2023-09-23T21:17:46.300166+0000","last_became_peered":"2023-09-23T21:17:46.300166+0000","last_unstale":"2023-09-23T21:18:28.352569+0000","last_undegraded":"2023-09-23T21:18:28.352569+0000","last_fullsized":"2023-09-23T21:18:28.352569+0000","mapping_epoch":588,"log_start":"556'3643","ondisk_log_start":"556'3643","created":559,"last_epoch_clean":601,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_clean_scrub_stamp":"2023-09-23T21:15:24.860968+0000","objects_scrubbed":0,"log_size":4161,"log_dups_size":0,"ondisk_log_size":4161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":36732928,"num_objects":561,"num_object_clones":0,"num_object_copies":1683,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":561,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5521,"num_write_kb":194592,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":838,"num_bytes_recovered":54820864,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,7],"acting":[3,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.13","version":"79'161","reported_seq":686,"reported_epoch":607,"state":"active+clean","last_fresh":"2023-09-23T21:18:27.902370+0000","last_change":"2023-09-23T21:13:14.933096+0000","last_active":"2023-09-23T21:18:27.902370+0000","last_peered":"2023-09-23T21:18:27.902370+0000","last_clean":"2023-09-23T21:18:27.902370+0000","last_became_active":"2023-09-23T21:13:14.932703+0000","last_became_peered":"2023-09-23T21:13:14.932703+0000","last_unstale":"2023-09-23T21:18:27.902370+0000","last_undegraded":"2023-09-23T21:18:27.902370+0000","last_fullsized":"2023-09-23T21:18:27.902370+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:36.420362+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:36.420362+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:29:42.163911+0000","scrub_duration":0.028838444000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":617,"reported_epoch":605,"state":"active+clean","last_fresh":"2023-09-23T21:18:25.884843+0000","last_change":"2023-09-23T21:16:42.420467+0000","last_active":"2023-09-23T21:18:25.884843+0000","last_peered":"2023-09-23T21:18:25.884843+0000","last_clean":"2023-09-23T21:18:25.884843+0000","last_became_active":"2023-09-23T21:16:42.420149+0000","last_became_peered":"2023-09-23T21:16:42.420149+0000","last_unstale":"2023-09-23T21:18:25.884843+0000","last_undegraded":"2023-09-23T21:18:25.884843+0000","last_fullsized":"2023-09-23T21:18:25.884843+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:52:07.355000+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":138,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-23T21:18:28.897289+0000","last_change":"2023-09-23T21:16:42.427663+0000","last_active":"2023-09-23T21:18:28.897289+0000","last_peered":"2023-09-23T21:18:28.897289+0000","last_clean":"2023-09-23T21:18:28.897289+0000","last_became_active":"2023-09-23T21:16:42.426961+0000","last_became_peered":"2023-09-23T21:16:42.426961+0000","last_unstale":"2023-09-23T21:18:28.897289+0000","last_undegraded":"2023-09-23T21:18:28.897289+0000","last_fullsized":"2023-09-23T21:18:28.897289+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:44:27.422007+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.18","version":"609'7583","reported_seq":7667,"reported_epoch":609,"state":"active+clean","last_fresh":"2023-09-23T21:18:30.129759+0000","last_change":"2023-09-23T21:16:32.928345+0000","last_active":"2023-09-23T21:18:30.129759+0000","last_peered":"2023-09-23T21:18:30.129759+0000","last_clean":"2023-09-23T21:18:30.129759+0000","last_became_active":"2023-09-23T21:16:32.926828+0000","last_became_peered":"2023-09-23T21:16:32.926828+0000","last_unstale":"2023-09-23T21:18:30.129759+0000","last_undegraded":"2023-09-23T21:18:30.129759+0000","last_fullsized":"2023-09-23T21:18:30.129759+0000","mapping_epoch":559,"log_start":"550'2665","ondisk_log_start":"550'2665","created":559,"last_epoch_clean":560,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_clean_scrub_stamp":"2023-09-23T21:15:24.860968+0000","objects_scrubbed":0,"log_size":4918,"log_dups_size":0,"ondisk_log_size":4918,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":40009728,"num_objects":611,"num_object_clones":0,"num_object_copies":1833,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":611,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5359,"num_write_kb":191008,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,5],"acting":[0,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.12","version":"79'161","reported_seq":878,"reported_epoch":604,"state":"active+clean","last_fresh":"2023-09-23T21:18:25.634540+0000","last_change":"2023-09-23T21:16:45.412440+0000","last_active":"2023-09-23T21:18:25.634540+0000","last_peered":"2023-09-23T21:18:25.634540+0000","last_clean":"2023-09-23T21:18:25.634540+0000","last_became_active":"2023-09-23T21:16:45.412029+0000","last_became_peered":"2023-09-23T21:16:45.412029+0000","last_unstale":"2023-09-23T21:18:25.634540+0000","last_undegraded":"2023-09-23T21:18:25.634540+0000","last_fullsized":"2023-09-23T21:18:25.634540+0000","mapping_epoch":567,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":568,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:05:56.785173+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:05:56.785173+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:39:43.432998+0000","scrub_duration":0.0042114370000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,0],"acting":[5,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":130,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-23T21:18:28.896850+0000","last_change":"2023-09-23T21:13:38.048791+0000","last_active":"2023-09-23T21:18:28.896850+0000","last_peered":"2023-09-23T21:18:28.896850+0000","last_clean":"2023-09-23T21:18:28.896850+0000","last_became_active":"2023-09-23T21:13:30.801386+0000","last_became_peered":"2023-09-23T21:13:30.801386+0000","last_unstale":"2023-09-23T21:18:28.896850+0000","last_undegraded":"2023-09-23T21:18:28.896850+0000","last_fullsized":"2023-09-23T21:18:28.896850+0000","mapping_epoch":501,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:52:49.170414+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":464,"reported_epoch":607,"state":"active+clean","last_fresh":"2023-09-23T21:18:27.901185+0000","last_change":"2023-09-23T21:16:42.518512+0000","last_active":"2023-09-23T21:18:27.901185+0000","last_peered":"2023-09-23T21:18:27.901185+0000","last_clean":"2023-09-23T21:18:27.901185+0000","last_became_active":"2023-09-23T21:16:42.517464+0000","last_became_peered":"2023-09-23T21:16:42.517464+0000","last_unstale":"2023-09-23T21:18:27.901185+0000","last_undegraded":"2023-09-23T21:18:27.901185+0000","last_fullsized":"2023-09-23T21:18:27.901185+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:47:52.391167+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.7","version":"605'7836","reported_seq":7929,"reported_epoch":605,"state":"active+clean","last_fresh":"2023-09-23T21:18:26.301108+0000","last_change":"2023-09-23T21:18:08.418283+0000","last_active":"2023-09-23T21:18:26.301108+0000","last_peered":"2023-09-23T21:18:26.301108+0000","last_clean":"2023-09-23T21:18:26.301108+0000","last_became_active":"2023-09-23T21:16:32.864890+0000","last_became_peered":"2023-09-23T21:16:32.864890+0000","last_unstale":"2023-09-23T21:18:26.301108+0000","last_undegraded":"2023-09-23T21:18:26.301108+0000","last_fullsized":"2023-09-23T21:18:26.301108+0000","mapping_epoch":544,"log_start":"549'1435","ondisk_log_start":"549'1435","created":544,"last_epoch_clean":560,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_clean_scrub_stamp":"2023-09-23T21:15:24.860968+0000","objects_scrubbed":0,"log_size":6401,"log_dups_size":0,"ondisk_log_size":6401,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":38502400,"num_objects":587,"num_object_clones":0,"num_object_copies":1761,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":587,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5531,"num_write_kb":195808,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.d","version":"79'161","reported_seq":631,"reported_epoch":605,"state":"active+clean","last_fresh":"2023-09-23T21:18:25.879049+0000","last_change":"2023-09-23T21:16:45.416685+0000","last_active":"2023-09-23T21:18:25.879049+0000","last_peered":"2023-09-23T21:18:25.879049+0000","last_clean":"2023-09-23T21:18:25.879049+0000","last_became_active":"2023-09-23T21:16:45.416373+0000","last_became_peered":"2023-09-23T21:16:45.416373+0000","last_unstale":"2023-09-23T21:18:25.879049+0000","last_undegraded":"2023-09-23T21:18:25.879049+0000","last_fullsized":"2023-09-23T21:18:25.879049+0000","mapping_epoch":567,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":568,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:15:32.726690+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:15:32.726690+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0024987500000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":55,"reported_epoch":605,"state":"active+clean","last_fresh":"2023-09-23T21:18:25.879376+0000","last_change":"2023-09-23T21:17:02.190320+0000","last_active":"2023-09-23T21:18:25.879376+0000","last_peered":"2023-09-23T21:18:25.879376+0000","last_clean":"2023-09-23T21:18:25.879376+0000","last_became_active":"2023-09-23T21:16:42.451380+0000","last_became_peered":"2023-09-23T21:16:42.451380+0000","last_unstale":"2023-09-23T21:18:25.879376+0000","last_undegraded":"2023-09-23T21:18:25.879376+0000","last_fullsized":"2023-09-23T21:18:25.879376+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:48:58.425821+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":132,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":332,"reported_epoch":604,"state":"active+clean","last_fresh":"2023-09-23T21:18:25.633783+0000","last_change":"2023-09-23T21:16:42.428491+0000","last_active":"2023-09-23T21:18:25.633783+0000","last_peered":"2023-09-23T21:18:25.633783+0000","last_clean":"2023-09-23T21:18:25.633783+0000","last_became_active":"2023-09-23T21:16:42.428166+0000","last_became_peered":"2023-09-23T21:16:42.428166+0000","last_unstale":"2023-09-23T21:18:25.633783+0000","last_undegraded":"2023-09-23T21:18:25.633783+0000","last_fullsized":"2023-09-23T21:18:25.633783+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:23:01.430714+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,4],"acting":[5,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.6","version":"609'7680","reported_seq":7791,"reported_epoch":609,"state":"active+clean","last_fresh":"2023-09-23T21:18:30.042623+0000","last_change":"2023-09-23T21:18:29.652253+0000","last_active":"2023-09-23T21:18:30.042623+0000","last_peered":"2023-09-23T21:18:30.042623+0000","last_clean":"2023-09-23T21:18:30.042623+0000","last_became_active":"2023-09-23T21:16:32.896510+0000","last_became_peered":"2023-09-23T21:16:32.896510+0000","last_unstale":"2023-09-23T21:18:30.042623+0000","last_undegraded":"2023-09-23T21:18:30.042623+0000","last_fullsized":"2023-09-23T21:18:30.042623+0000","mapping_epoch":544,"log_start":"550'2936","ondisk_log_start":"550'2936","created":544,"last_epoch_clean":561,"parent":"0.0","parent_split_bits":0,"last_scrub":"607'7678","last_scrub_stamp":"2023-09-23T21:18:29.652124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_clean_scrub_stamp":"2023-09-23T21:18:29.652124+0000","objects_scrubbed":618,"log_size":4744,"log_dups_size":0,"ondisk_log_size":4744,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:05:14.537109+0000","scrub_duration":0.077048132000000005,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":40370176,"num_objects":616,"num_object_clones":0,"num_object_copies":1848,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":616,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.c","version":"79'161","reported_seq":990,"reported_epoch":607,"state":"active+clean","last_fresh":"2023-09-23T21:18:27.900051+0000","last_change":"2023-09-23T21:15:31.992756+0000","last_active":"2023-09-23T21:18:27.900051+0000","last_peered":"2023-09-23T21:18:27.900051+0000","last_clean":"2023-09-23T21:18:27.900051+0000","last_became_active":"2023-09-23T21:13:13.401526+0000","last_became_peered":"2023-09-23T21:13:13.401526+0000","last_unstale":"2023-09-23T21:18:27.900051+0000","last_undegraded":"2023-09-23T21:18:27.900051+0000","last_fullsized":"2023-09-23T21:18:27.900051+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":532,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:15:31.992598+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:15:31.992598+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:41:14.215543+0000","scrub_duration":0.002960678,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":55,"reported_epoch":605,"state":"active+clean","last_fresh":"2023-09-23T21:18:25.879126+0000","last_change":"2023-09-23T21:17:02.215996+0000","last_active":"2023-09-23T21:18:25.879126+0000","last_peered":"2023-09-23T21:18:25.879126+0000","last_clean":"2023-09-23T21:18:25.879126+0000","last_became_active":"2023-09-23T21:16:42.451205+0000","last_became_peered":"2023-09-23T21:16:42.451205+0000","last_unstale":"2023-09-23T21:18:25.879126+0000","last_undegraded":"2023-09-23T21:18:25.879126+0000","last_fullsized":"2023-09-23T21:18:25.879126+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:51:46.876281+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":568,"reported_epoch":605,"state":"active+clean","last_fresh":"2023-09-23T21:18:25.884888+0000","last_change":"2023-09-23T21:13:15.152792+0000","last_active":"2023-09-23T21:18:25.884888+0000","last_peered":"2023-09-23T21:18:25.884888+0000","last_clean":"2023-09-23T21:18:25.884888+0000","last_became_active":"2023-09-23T21:13:14.515084+0000","last_became_peered":"2023-09-23T21:13:14.515084+0000","last_unstale":"2023-09-23T21:18:25.884888+0000","last_undegraded":"2023-09-23T21:18:25.884888+0000","last_fullsized":"2023-09-23T21:18:25.884888+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:08:16.702072+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.5","version":"605'7838","reported_seq":13052,"reported_epoch":605,"state":"active+clean","last_fresh":"2023-09-23T21:18:26.214847+0000","last_change":"2023-09-23T21:18:08.488193+0000","last_active":"2023-09-23T21:18:26.214847+0000","last_peered":"2023-09-23T21:18:26.214847+0000","last_clean":"2023-09-23T21:18:26.214847+0000","last_became_active":"2023-09-23T21:17:24.389052+0000","last_became_peered":"2023-09-23T21:17:24.389052+0000","last_unstale":"2023-09-23T21:18:26.214847+0000","last_undegraded":"2023-09-23T21:18:26.214847+0000","last_fullsized":"2023-09-23T21:18:26.214847+0000","mapping_epoch":578,"log_start":"548'1153","ondisk_log_start":"548'1153","created":544,"last_epoch_clean":579,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_clean_scrub_stamp":"2023-09-23T21:15:24.860968+0000","objects_scrubbed":0,"log_size":6685,"log_dups_size":0,"ondisk_log_size":6685,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":43417600,"num_objects":662,"num_object_clones":0,"num_object_copies":1986,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":662,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5542,"num_write_kb":198560,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1682,"num_bytes_recovered":110231552,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.f","version":"79'161","reported_seq":892,"reported_epoch":604,"state":"active+clean","last_fresh":"2023-09-23T21:18:25.634576+0000","last_change":"2023-09-23T21:11:30.750338+0000","last_active":"2023-09-23T21:18:25.634576+0000","last_peered":"2023-09-23T21:18:25.634576+0000","last_clean":"2023-09-23T21:18:25.634576+0000","last_became_active":"2023-09-23T21:11:30.749953+0000","last_became_peered":"2023-09-23T21:11:30.749953+0000","last_unstale":"2023-09-23T21:18:25.634576+0000","last_undegraded":"2023-09-23T21:18:25.634576+0000","last_fullsized":"2023-09-23T21:18:25.634576+0000","mapping_epoch":455,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":456,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:27.880776+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:27.880776+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:07:10.405723+0000","scrub_duration":0.0014004899999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,3],"acting":[5,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":356,"reported_epoch":604,"state":"active+clean","last_fresh":"2023-09-23T21:18:25.634516+0000","last_change":"2023-09-23T21:17:02.175012+0000","last_active":"2023-09-23T21:18:25.634516+0000","last_peered":"2023-09-23T21:18:25.634516+0000","last_clean":"2023-09-23T21:18:25.634516+0000","last_became_active":"2023-09-23T21:16:42.451114+0000","last_became_peered":"2023-09-23T21:16:42.451114+0000","last_unstale":"2023-09-23T21:18:25.634516+0000","last_undegraded":"2023-09-23T21:18:25.634516+0000","last_fullsized":"2023-09-23T21:18:25.634516+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:00:53.868525+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":420,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":564,"reported_epoch":604,"state":"active+clean","last_fresh":"2023-09-23T21:18:25.633996+0000","last_change":"2023-09-23T21:16:42.432943+0000","last_active":"2023-09-23T21:18:25.633996+0000","last_peered":"2023-09-23T21:18:25.633996+0000","last_clean":"2023-09-23T21:18:25.633996+0000","last_became_active":"2023-09-23T21:16:42.427440+0000","last_became_peered":"2023-09-23T21:16:42.427440+0000","last_unstale":"2023-09-23T21:18:25.633996+0000","last_undegraded":"2023-09-23T21:18:25.633996+0000","last_fullsized":"2023-09-23T21:18:25.633996+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:59:32.044609+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,0],"acting":[5,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.4","version":"605'7666","reported_seq":9301,"reported_epoch":605,"state":"active+clean","last_fresh":"2023-09-23T21:18:26.354754+0000","last_change":"2023-09-23T21:18:05.512415+0000","last_active":"2023-09-23T21:18:26.354754+0000","last_peered":"2023-09-23T21:18:26.354754+0000","last_clean":"2023-09-23T21:18:26.354754+0000","last_became_active":"2023-09-23T21:18:05.477101+0000","last_became_peered":"2023-09-23T21:18:05.477101+0000","last_unstale":"2023-09-23T21:18:26.354754+0000","last_undegraded":"2023-09-23T21:18:26.354754+0000","last_fullsized":"2023-09-23T21:18:26.354754+0000","mapping_epoch":595,"log_start":"549'1962","ondisk_log_start":"549'1962","created":544,"last_epoch_clean":596,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_clean_scrub_stamp":"2023-09-23T21:15:24.860968+0000","objects_scrubbed":0,"log_size":5704,"log_dups_size":0,"ondisk_log_size":5704,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47906816,"num_objects":731,"num_object_clones":0,"num_object_copies":2193,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":731,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5429,"num_write_kb":197120,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":765,"num_bytes_recovered":50135040,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,3],"acting":[7,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.e","version":"79'161","reported_seq":824,"reported_epoch":605,"state":"active+clean","last_fresh":"2023-09-23T21:18:25.885724+0000","last_change":"2023-09-23T21:16:45.433204+0000","last_active":"2023-09-23T21:18:25.885724+0000","last_peered":"2023-09-23T21:18:25.885724+0000","last_clean":"2023-09-23T21:18:25.885724+0000","last_became_active":"2023-09-23T21:16:45.432836+0000","last_became_peered":"2023-09-23T21:16:45.432836+0000","last_unstale":"2023-09-23T21:18:25.885724+0000","last_undegraded":"2023-09-23T21:18:25.885724+0000","last_fullsized":"2023-09-23T21:18:25.885724+0000","mapping_epoch":567,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":568,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:15:39.046962+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:15:39.046962+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:11:20.720731+0000","scrub_duration":0.0082638769999999993,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":407,"reported_epoch":607,"state":"active+clean","last_fresh":"2023-09-23T21:18:27.900408+0000","last_change":"2023-09-23T21:13:13.402418+0000","last_active":"2023-09-23T21:18:27.900408+0000","last_peered":"2023-09-23T21:18:27.900408+0000","last_clean":"2023-09-23T21:18:27.900408+0000","last_became_active":"2023-09-23T21:13:13.400984+0000","last_became_peered":"2023-09-23T21:13:13.400984+0000","last_unstale":"2023-09-23T21:18:27.900408+0000","last_undegraded":"2023-09-23T21:18:27.900408+0000","last_fullsized":"2023-09-23T21:18:27.900408+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":495,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:28:01.411191+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,0],"acting":[3,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":583,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-23T21:18:28.896565+0000","last_change":"2023-09-23T21:13:14.339670+0000","last_active":"2023-09-23T21:18:28.896565+0000","last_peered":"2023-09-23T21:18:28.896565+0000","last_clean":"2023-09-23T21:18:28.896565+0000","last_became_active":"2023-09-23T21:13:14.339341+0000","last_became_peered":"2023-09-23T21:13:14.339341+0000","last_unstale":"2023-09-23T21:18:28.896565+0000","last_undegraded":"2023-09-23T21:18:28.896565+0000","last_fullsized":"2023-09-23T21:18:28.896565+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:23:19.411368+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.3","version":"605'7586","reported_seq":9265,"reported_epoch":605,"state":"active+clean","last_fresh":"2023-09-23T21:18:26.192015+0000","last_change":"2023-09-23T21:17:51.904100+0000","last_active":"2023-09-23T21:18:26.192015+0000","last_peered":"2023-09-23T21:18:26.192015+0000","last_clean":"2023-09-23T21:18:26.192015+0000","last_became_active":"2023-09-23T21:17:51.872537+0000","last_became_peered":"2023-09-23T21:17:51.872537+0000","last_unstale":"2023-09-23T21:18:26.192015+0000","last_undegraded":"2023-09-23T21:18:26.192015+0000","last_fullsized":"2023-09-23T21:18:26.192015+0000","mapping_epoch":591,"log_start":"549'1923","ondisk_log_start":"549'1923","created":544,"last_epoch_clean":592,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_clean_scrub_stamp":"2023-09-23T21:15:24.860968+0000","objects_scrubbed":0,"log_size":5663,"log_dups_size":0,"ondisk_log_size":5663,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":40763392,"num_objects":622,"num_object_clones":0,"num_object_copies":1866,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":622,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5370,"num_write_kb":191744,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":786,"num_bytes_recovered":51511296,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,2],"acting":[7,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.9","version":"79'161","reported_seq":806,"reported_epoch":604,"state":"active+clean","last_fresh":"2023-09-23T21:18:25.632420+0000","last_change":"2023-09-23T21:16:45.433968+0000","last_active":"2023-09-23T21:18:25.632420+0000","last_peered":"2023-09-23T21:18:25.632420+0000","last_clean":"2023-09-23T21:18:25.632420+0000","last_became_active":"2023-09-23T21:16:45.433602+0000","last_became_peered":"2023-09-23T21:16:45.433602+0000","last_unstale":"2023-09-23T21:18:25.632420+0000","last_undegraded":"2023-09-23T21:18:25.632420+0000","last_fullsized":"2023-09-23T21:18:25.632420+0000","mapping_epoch":567,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":568,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:15:43.437369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:15:43.437369+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:16:44.538102+0000","scrub_duration":0.0024129429999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,3],"acting":[2,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":169,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-23T21:18:28.896668+0000","last_change":"2023-09-23T21:13:30.793273+0000","last_active":"2023-09-23T21:18:28.896668+0000","last_peered":"2023-09-23T21:18:28.896668+0000","last_clean":"2023-09-23T21:18:28.896668+0000","last_became_active":"2023-09-23T21:13:30.791269+0000","last_became_peered":"2023-09-23T21:13:30.791269+0000","last_unstale":"2023-09-23T21:18:28.896668+0000","last_undegraded":"2023-09-23T21:18:28.896668+0000","last_fullsized":"2023-09-23T21:18:28.896668+0000","mapping_epoch":501,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:38:07.142290+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":199,"reported_epoch":605,"state":"active+clean","last_fresh":"2023-09-23T21:18:25.885375+0000","last_change":"2023-09-23T21:16:42.417057+0000","last_active":"2023-09-23T21:18:25.885375+0000","last_peered":"2023-09-23T21:18:25.885375+0000","last_clean":"2023-09-23T21:18:25.885375+0000","last_became_active":"2023-09-23T21:16:42.416601+0000","last_became_peered":"2023-09-23T21:16:42.416601+0000","last_unstale":"2023-09-23T21:18:25.885375+0000","last_undegraded":"2023-09-23T21:18:25.885375+0000","last_fullsized":"2023-09-23T21:18:25.885375+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:56:52.167562+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,7],"acting":[4,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.2","version":"605'7820","reported_seq":7901,"reported_epoch":605,"state":"active+clean","last_fresh":"2023-09-23T21:18:26.324009+0000","last_change":"2023-09-23T21:16:32.870210+0000","last_active":"2023-09-23T21:18:26.324009+0000","last_peered":"2023-09-23T21:18:26.324009+0000","last_clean":"2023-09-23T21:18:26.324009+0000","last_became_active":"2023-09-23T21:16:32.867780+0000","last_became_peered":"2023-09-23T21:16:32.867780+0000","last_unstale":"2023-09-23T21:18:26.324009+0000","last_undegraded":"2023-09-23T21:18:26.324009+0000","last_fullsized":"2023-09-23T21:18:26.324009+0000","mapping_epoch":544,"log_start":"548'448","ondisk_log_start":"548'448","created":544,"last_epoch_clean":560,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_clean_scrub_stamp":"2023-09-23T21:15:24.860968+0000","objects_scrubbed":0,"log_size":7372,"log_dups_size":0,"ondisk_log_size":7372,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":45088768,"num_objects":688,"num_object_clones":0,"num_object_copies":2064,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":688,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5536,"num_write_kb":199168,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.8","version":"79'161","reported_seq":739,"reported_epoch":604,"state":"active+clean","last_fresh":"2023-09-23T21:18:25.632902+0000","last_change":"2023-09-23T21:15:36.367819+0000","last_active":"2023-09-23T21:18:25.632902+0000","last_peered":"2023-09-23T21:18:25.632902+0000","last_clean":"2023-09-23T21:18:25.632902+0000","last_became_active":"2023-09-23T21:12:30.433217+0000","last_became_peered":"2023-09-23T21:12:30.433217+0000","last_unstale":"2023-09-23T21:18:25.632902+0000","last_undegraded":"2023-09-23T21:18:25.632902+0000","last_fullsized":"2023-09-23T21:18:25.632902+0000","mapping_epoch":359,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:15:36.367640+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:15:36.367640+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:49:35.243653+0000","scrub_duration":0.005826899,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":182,"reported_epoch":605,"state":"active+clean","last_fresh":"2023-09-23T21:18:25.885164+0000","last_change":"2023-09-23T21:12:10.823380+0000","last_active":"2023-09-23T21:18:25.885164+0000","last_peered":"2023-09-23T21:18:25.885164+0000","last_clean":"2023-09-23T21:18:25.885164+0000","last_became_active":"2023-09-23T21:12:10.823044+0000","last_became_peered":"2023-09-23T21:12:10.823044+0000","last_unstale":"2023-09-23T21:18:25.885164+0000","last_undegraded":"2023-09-23T21:18:25.885164+0000","last_fullsized":"2023-09-23T21:18:25.885164+0000","mapping_epoch":470,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:53:29.846536+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,5],"acting":[4,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":183,"reported_epoch":605,"state":"active+clean","last_fresh":"2023-09-23T21:18:25.885592+0000","last_change":"2023-09-23T21:13:30.772451+0000","last_active":"2023-09-23T21:18:25.885592+0000","last_peered":"2023-09-23T21:18:25.885592+0000","last_clean":"2023-09-23T21:18:25.885592+0000","last_became_active":"2023-09-23T21:13:30.772040+0000","last_became_peered":"2023-09-23T21:13:30.772040+0000","last_unstale":"2023-09-23T21:18:25.885592+0000","last_undegraded":"2023-09-23T21:18:25.885592+0000","last_fullsized":"2023-09-23T21:18:25.885592+0000","mapping_epoch":501,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:24:49.246727+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,0],"acting":[4,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.1","version":"607'7996","reported_seq":8078,"reported_epoch":607,"state":"active+clean","last_fresh":"2023-09-23T21:18:28.303581+0000","last_change":"2023-09-23T21:16:32.911457+0000","last_active":"2023-09-23T21:18:28.303581+0000","last_peered":"2023-09-23T21:18:28.303581+0000","last_clean":"2023-09-23T21:18:28.303581+0000","last_became_active":"2023-09-23T21:16:32.905672+0000","last_became_peered":"2023-09-23T21:16:32.905672+0000","last_unstale":"2023-09-23T21:18:28.303581+0000","last_undegraded":"2023-09-23T21:18:28.303581+0000","last_fullsized":"2023-09-23T21:18:28.303581+0000","mapping_epoch":544,"log_start":"556'3963","ondisk_log_start":"556'3963","created":544,"last_epoch_clean":560,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_clean_scrub_stamp":"2023-09-23T21:15:24.860968+0000","objects_scrubbed":0,"log_size":4033,"log_dups_size":0,"ondisk_log_size":4033,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":38600704,"num_objects":589,"num_object_clones":0,"num_object_copies":1767,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":589,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5679,"num_write_kb":200576,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,2],"acting":[3,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.b","version":"79'161","reported_seq":626,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-23T21:18:28.896216+0000","last_change":"2023-09-23T21:15:36.340009+0000","last_active":"2023-09-23T21:18:28.896216+0000","last_peered":"2023-09-23T21:18:28.896216+0000","last_clean":"2023-09-23T21:18:28.896216+0000","last_became_active":"2023-09-23T21:13:13.372328+0000","last_became_peered":"2023-09-23T21:13:13.372328+0000","last_unstale":"2023-09-23T21:18:28.896216+0000","last_undegraded":"2023-09-23T21:18:28.896216+0000","last_fullsized":"2023-09-23T21:18:28.896216+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":495,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:15:36.339885+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:15:36.339885+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:01:26.605006+0000","scrub_duration":0.0068473279999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":183,"reported_epoch":605,"state":"active+clean","last_fresh":"2023-09-23T21:18:25.884807+0000","last_change":"2023-09-23T21:16:42.425103+0000","last_active":"2023-09-23T21:18:25.884807+0000","last_peered":"2023-09-23T21:18:25.884807+0000","last_clean":"2023-09-23T21:18:25.884807+0000","last_became_active":"2023-09-23T21:16:42.424543+0000","last_became_peered":"2023-09-23T21:16:42.424543+0000","last_unstale":"2023-09-23T21:18:25.884807+0000","last_undegraded":"2023-09-23T21:18:25.884807+0000","last_fullsized":"2023-09-23T21:18:25.884807+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:11:28.885246+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":537,"reported_epoch":605,"state":"active+clean","last_fresh":"2023-09-23T21:18:25.885895+0000","last_change":"2023-09-23T21:12:10.811698+0000","last_active":"2023-09-23T21:18:25.885895+0000","last_peered":"2023-09-23T21:18:25.885895+0000","last_clean":"2023-09-23T21:18:25.885895+0000","last_became_active":"2023-09-23T21:12:10.789656+0000","last_became_peered":"2023-09-23T21:12:10.789656+0000","last_unstale":"2023-09-23T21:18:25.885895+0000","last_undegraded":"2023-09-23T21:18:25.885895+0000","last_fullsized":"2023-09-23T21:18:25.885895+0000","mapping_epoch":470,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:39:15.112091+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,5],"acting":[4,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.0","version":"607'7718","reported_seq":9435,"reported_epoch":607,"state":"active+clean","last_fresh":"2023-09-23T21:18:28.420235+0000","last_change":"2023-09-23T21:18:10.533031+0000","last_active":"2023-09-23T21:18:28.420235+0000","last_peered":"2023-09-23T21:18:28.420235+0000","last_clean":"2023-09-23T21:18:28.420235+0000","last_became_active":"2023-09-23T21:18:10.519762+0000","last_became_peered":"2023-09-23T21:18:10.519762+0000","last_unstale":"2023-09-23T21:18:28.420235+0000","last_undegraded":"2023-09-23T21:18:28.420235+0000","last_fullsized":"2023-09-23T21:18:28.420235+0000","mapping_epoch":598,"log_start":"555'3471","ondisk_log_start":"555'3471","created":544,"last_epoch_clean":599,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_clean_scrub_stamp":"2023-09-23T21:15:24.860968+0000","objects_scrubbed":0,"log_size":4247,"log_dups_size":0,"ondisk_log_size":4247,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":37912576,"num_objects":578,"num_object_clones":0,"num_object_copies":1734,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":578,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5460,"num_write_kb":193248,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":803,"num_bytes_recovered":42336256,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.a","version":"79'161","reported_seq":765,"reported_epoch":604,"state":"active+clean","last_fresh":"2023-09-23T21:18:25.634119+0000","last_change":"2023-09-23T21:15:46.420777+0000","last_active":"2023-09-23T21:18:25.634119+0000","last_peered":"2023-09-23T21:18:25.634119+0000","last_clean":"2023-09-23T21:18:25.634119+0000","last_became_active":"2023-09-23T21:12:30.437427+0000","last_became_peered":"2023-09-23T21:12:30.437427+0000","last_unstale":"2023-09-23T21:18:25.634119+0000","last_undegraded":"2023-09-23T21:18:25.634119+0000","last_fullsized":"2023-09-23T21:18:25.634119+0000","mapping_epoch":446,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:15:46.420650+0000","last_deep_scrub":"79'161","last_deep_scrub_stamp":"2023-09-23T21:05:32.630292+0000","last_clean_scrub_stamp":"2023-09-23T21:15:46.420650+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:50:23.452475+0000","scrub_duration":0.0072898930000000004,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,4],"acting":[2,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":474,"reported_epoch":607,"state":"active+clean","last_fresh":"2023-09-23T21:18:27.897072+0000","last_change":"2023-09-23T21:16:42.447403+0000","last_active":"2023-09-23T21:18:27.897072+0000","last_peered":"2023-09-23T21:18:27.897072+0000","last_clean":"2023-09-23T21:18:27.897072+0000","last_became_active":"2023-09-23T21:16:42.447080+0000","last_became_peered":"2023-09-23T21:16:42.447080+0000","last_unstale":"2023-09-23T21:18:27.897072+0000","last_undegraded":"2023-09-23T21:18:27.897072+0000","last_fullsized":"2023-09-23T21:18:27.897072+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:15:19.446334+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,4],"acting":[3,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":605,"reported_epoch":607,"state":"active+clean","last_fresh":"2023-09-23T21:18:27.901330+0000","last_change":"2023-09-23T21:16:42.518739+0000","last_active":"2023-09-23T21:18:27.901330+0000","last_peered":"2023-09-23T21:18:27.901330+0000","last_clean":"2023-09-23T21:18:27.901330+0000","last_became_active":"2023-09-23T21:16:42.517945+0000","last_became_peered":"2023-09-23T21:16:42.517945+0000","last_unstale":"2023-09-23T21:18:27.901330+0000","last_undegraded":"2023-09-23T21:18:27.901330+0000","last_fullsized":"2023-09-23T21:18:27.901330+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:47:26.236933+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.d","version":"605'11313","reported_seq":11385,"reported_epoch":605,"state":"active+clean","last_fresh":"2023-09-23T21:18:26.330203+0000","last_change":"2023-09-23T21:15:25.892834+0000","last_active":"2023-09-23T21:18:26.330203+0000","last_peered":"2023-09-23T21:18:26.330203+0000","last_clean":"2023-09-23T21:18:26.330203+0000","last_became_active":"2023-09-23T21:15:25.892519+0000","last_became_peered":"2023-09-23T21:15:25.892519+0000","last_unstale":"2023-09-23T21:18:26.330203+0000","last_undegraded":"2023-09-23T21:18:26.330203+0000","last_fullsized":"2023-09-23T21:18:26.330203+0000","mapping_epoch":544,"log_start":"570'5572","ondisk_log_start":"570'5572","created":544,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_clean_scrub_stamp":"2023-09-23T21:15:24.860968+0000","objects_scrubbed":0,"log_size":5741,"log_dups_size":0,"ondisk_log_size":5741,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:46:28.375665+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":83755032,"num_objects":1279,"num_object_clones":0,"num_object_copies":3837,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1279,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":11313,"num_write_kb":402881,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,5],"acting":[2,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.7","version":"79'161","reported_seq":748,"reported_epoch":607,"state":"active+clean","last_fresh":"2023-09-23T21:18:27.901091+0000","last_change":"2023-09-23T21:16:45.425645+0000","last_active":"2023-09-23T21:18:27.901091+0000","last_peered":"2023-09-23T21:18:27.901091+0000","last_clean":"2023-09-23T21:18:27.901091+0000","last_became_active":"2023-09-23T21:16:45.425241+0000","last_became_peered":"2023-09-23T21:16:45.425241+0000","last_unstale":"2023-09-23T21:18:27.901091+0000","last_undegraded":"2023-09-23T21:18:27.901091+0000","last_fullsized":"2023-09-23T21:18:27.901091+0000","mapping_epoch":567,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":568,"parent":"0.0","parent_split_bits":3,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:15:51.016427+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:15:51.016427+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:49:31.176462+0000","scrub_duration":0.0017648640000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":558,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-23T21:18:28.896487+0000","last_change":"2023-09-23T21:16:42.518826+0000","last_active":"2023-09-23T21:18:28.896487+0000","last_peered":"2023-09-23T21:18:28.896487+0000","last_clean":"2023-09-23T21:18:28.896487+0000","last_became_active":"2023-09-23T21:16:42.518488+0000","last_became_peered":"2023-09-23T21:16:42.518488+0000","last_unstale":"2023-09-23T21:18:28.896487+0000","last_undegraded":"2023-09-23T21:18:28.896487+0000","last_fullsized":"2023-09-23T21:18:28.896487+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:42:44.144383+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,2],"acting":[0,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":282,"reported_epoch":604,"state":"active+clean","last_fresh":"2023-09-23T21:18:25.634424+0000","last_change":"2023-09-23T21:05:04.326889+0000","last_active":"2023-09-23T21:18:25.634424+0000","last_peered":"2023-09-23T21:18:25.634424+0000","last_clean":"2023-09-23T21:18:25.634424+0000","last_became_active":"2023-09-23T21:05:04.326032+0000","last_became_peered":"2023-09-23T21:05:04.326032+0000","last_unstale":"2023-09-23T21:18:25.634424+0000","last_undegraded":"2023-09-23T21:18:25.634424+0000","last_fullsized":"2023-09-23T21:18:25.634424+0000","mapping_epoch":332,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:00:17.507889+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":614,"reported_epoch":604,"state":"active+clean","last_fresh":"2023-09-23T21:18:25.634328+0000","last_change":"2023-09-23T21:13:13.285608+0000","last_active":"2023-09-23T21:18:25.634328+0000","last_peered":"2023-09-23T21:18:25.634328+0000","last_clean":"2023-09-23T21:18:25.634328+0000","last_became_active":"2023-09-23T21:13:13.285255+0000","last_became_peered":"2023-09-23T21:13:13.285255+0000","last_unstale":"2023-09-23T21:18:25.634328+0000","last_undegraded":"2023-09-23T21:18:25.634328+0000","last_fullsized":"2023-09-23T21:18:25.634328+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":495,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:42:30.087599+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.b","version":"607'10780","reported_seq":10854,"reported_epoch":607,"state":"active+clean","last_fresh":"2023-09-23T21:18:28.355133+0000","last_change":"2023-09-23T21:15:25.906791+0000","last_active":"2023-09-23T21:18:28.355133+0000","last_peered":"2023-09-23T21:18:28.355133+0000","last_clean":"2023-09-23T21:18:28.355133+0000","last_became_active":"2023-09-23T21:15:25.906512+0000","last_became_peered":"2023-09-23T21:15:25.906512+0000","last_unstale":"2023-09-23T21:18:28.355133+0000","last_undegraded":"2023-09-23T21:18:28.355133+0000","last_fullsized":"2023-09-23T21:18:28.355133+0000","mapping_epoch":544,"log_start":"575'6945","ondisk_log_start":"575'6945","created":544,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_clean_scrub_stamp":"2023-09-23T21:15:24.860968+0000","objects_scrubbed":0,"log_size":3835,"log_dups_size":0,"ondisk_log_size":3835,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:58:51.749775+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":76677120,"num_objects":1170,"num_object_clones":0,"num_object_copies":3510,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1170,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10780,"num_write_kb":382400,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1","version":"79'161","reported_seq":693,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-23T21:18:28.896917+0000","last_change":"2023-09-23T21:15:34.345290+0000","last_active":"2023-09-23T21:18:28.896917+0000","last_peered":"2023-09-23T21:18:28.896917+0000","last_clean":"2023-09-23T21:18:28.896917+0000","last_became_active":"2023-09-23T21:13:12.203380+0000","last_became_peered":"2023-09-23T21:13:12.203380+0000","last_unstale":"2023-09-23T21:18:28.896917+0000","last_undegraded":"2023-09-23T21:18:28.896917+0000","last_fullsized":"2023-09-23T21:18:28.896917+0000","mapping_epoch":493,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:15:34.345138+0000","last_deep_scrub":"79'161","last_deep_scrub_stamp":"2023-09-23T21:05:30.652189+0000","last_clean_scrub_stamp":"2023-09-23T21:15:34.345138+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:47:50.349409+0000","scrub_duration":0.0057520000000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":277,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-23T21:18:28.896313+0000","last_change":"2023-09-23T21:13:13.238617+0000","last_active":"2023-09-23T21:18:28.896313+0000","last_peered":"2023-09-23T21:18:28.896313+0000","last_clean":"2023-09-23T21:18:28.896313+0000","last_became_active":"2023-09-23T21:13:13.232926+0000","last_became_peered":"2023-09-23T21:13:13.232926+0000","last_unstale":"2023-09-23T21:18:28.896313+0000","last_undegraded":"2023-09-23T21:18:28.896313+0000","last_fullsized":"2023-09-23T21:18:28.896313+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":495,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:17:21.823772+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":539,"reported_epoch":605,"state":"active+clean","last_fresh":"2023-09-23T21:18:25.885211+0000","last_change":"2023-09-23T21:13:14.144918+0000","last_active":"2023-09-23T21:18:25.885211+0000","last_peered":"2023-09-23T21:18:25.885211+0000","last_clean":"2023-09-23T21:18:25.885211+0000","last_became_active":"2023-09-23T21:13:14.132388+0000","last_became_peered":"2023-09-23T21:13:14.132388+0000","last_unstale":"2023-09-23T21:18:25.885211+0000","last_undegraded":"2023-09-23T21:18:25.885211+0000","last_fullsized":"2023-09-23T21:18:25.885211+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":495,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:56:02.619905+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,5],"acting":[4,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":445,"reported_epoch":605,"state":"active+clean","last_fresh":"2023-09-23T21:18:25.872337+0000","last_change":"2023-09-23T21:16:42.491111+0000","last_active":"2023-09-23T21:18:25.872337+0000","last_peered":"2023-09-23T21:18:25.872337+0000","last_clean":"2023-09-23T21:18:25.872337+0000","last_became_active":"2023-09-23T21:16:42.490275+0000","last_became_peered":"2023-09-23T21:16:42.490275+0000","last_unstale":"2023-09-23T21:18:25.872337+0000","last_undegraded":"2023-09-23T21:18:25.872337+0000","last_fullsized":"2023-09-23T21:18:25.872337+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:39:11.652564+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,7],"acting":[2,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"11.8","version":"609'7757","reported_seq":7842,"reported_epoch":609,"state":"active+clean","last_fresh":"2023-09-23T21:18:30.112277+0000","last_change":"2023-09-23T21:16:32.898234+0000","last_active":"2023-09-23T21:18:30.112277+0000","last_peered":"2023-09-23T21:18:30.112277+0000","last_clean":"2023-09-23T21:18:30.112277+0000","last_became_active":"2023-09-23T21:16:32.894560+0000","last_became_peered":"2023-09-23T21:16:32.894560+0000","last_unstale":"2023-09-23T21:18:30.112277+0000","last_undegraded":"2023-09-23T21:18:30.112277+0000","last_fullsized":"2023-09-23T21:18:30.112277+0000","mapping_epoch":544,"log_start":"550'3087","ondisk_log_start":"550'3087","created":544,"last_epoch_clean":560,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_clean_scrub_stamp":"2023-09-23T21:15:24.860968+0000","objects_scrubbed":0,"log_size":4670,"log_dups_size":0,"ondisk_log_size":4670,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":37650432,"num_objects":574,"num_object_clones":0,"num_object_copies":1722,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":574,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5532,"num_write_kb":195424,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,5],"acting":[0,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.2","version":"79'161","reported_seq":924,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-23T21:18:28.895986+0000","last_change":"2023-09-23T21:16:45.417075+0000","last_active":"2023-09-23T21:18:28.895986+0000","last_peered":"2023-09-23T21:18:28.895986+0000","last_clean":"2023-09-23T21:18:28.895986+0000","last_became_active":"2023-09-23T21:16:45.416752+0000","last_became_peered":"2023-09-23T21:16:45.416752+0000","last_unstale":"2023-09-23T21:18:28.895986+0000","last_undegraded":"2023-09-23T21:18:28.895986+0000","last_fullsized":"2023-09-23T21:18:28.895986+0000","mapping_epoch":567,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":568,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:24.802451+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:24.802451+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:01:51.129779+0000","scrub_duration":0.0090861250000000005,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,4],"acting":[0,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":615,"reported_epoch":604,"state":"active+clean","last_fresh":"2023-09-23T21:18:25.633448+0000","last_change":"2023-09-23T21:13:12.206980+0000","last_active":"2023-09-23T21:18:25.633448+0000","last_peered":"2023-09-23T21:18:25.633448+0000","last_clean":"2023-09-23T21:18:25.633448+0000","last_became_active":"2023-09-23T21:13:12.206650+0000","last_became_peered":"2023-09-23T21:13:12.206650+0000","last_unstale":"2023-09-23T21:18:25.633448+0000","last_undegraded":"2023-09-23T21:18:25.633448+0000","last_fullsized":"2023-09-23T21:18:25.633448+0000","mapping_epoch":493,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:10:01.892936+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":599,"reported_epoch":604,"state":"active+clean","last_fresh":"2023-09-23T21:18:25.633397+0000","last_change":"2023-09-23T21:13:13.407073+0000","last_active":"2023-09-23T21:18:25.633397+0000","last_peered":"2023-09-23T21:18:25.633397+0000","last_clean":"2023-09-23T21:18:25.633397+0000","last_became_active":"2023-09-23T21:13:13.395047+0000","last_became_peered":"2023-09-23T21:13:13.395047+0000","last_unstale":"2023-09-23T21:18:25.633397+0000","last_undegraded":"2023-09-23T21:18:25.633397+0000","last_fullsized":"2023-09-23T21:18:25.633397+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":495,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:04:43.384190+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,5],"acting":[2,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":138,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-23T21:18:28.897833+0000","last_change":"2023-09-23T21:16:42.517729+0000","last_active":"2023-09-23T21:18:28.897833+0000","last_peered":"2023-09-23T21:18:28.897833+0000","last_clean":"2023-09-23T21:18:28.897833+0000","last_became_active":"2023-09-23T21:16:42.517373+0000","last_became_peered":"2023-09-23T21:16:42.517373+0000","last_unstale":"2023-09-23T21:18:28.897833+0000","last_undegraded":"2023-09-23T21:18:28.897833+0000","last_fullsized":"2023-09-23T21:18:28.897833+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:05:57.370550+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.a","version":"605'11132","reported_seq":11203,"reported_epoch":605,"state":"active+clean","last_fresh":"2023-09-23T21:18:26.371226+0000","last_change":"2023-09-23T21:15:25.893846+0000","last_active":"2023-09-23T21:18:26.371226+0000","last_peered":"2023-09-23T21:18:26.371226+0000","last_clean":"2023-09-23T21:18:26.371226+0000","last_became_active":"2023-09-23T21:15:25.893452+0000","last_became_peered":"2023-09-23T21:15:25.893452+0000","last_unstale":"2023-09-23T21:18:26.371226+0000","last_undegraded":"2023-09-23T21:18:26.371226+0000","last_fullsized":"2023-09-23T21:18:26.371226+0000","mapping_epoch":544,"log_start":"570'5525","ondisk_log_start":"570'5525","created":544,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_clean_scrub_stamp":"2023-09-23T21:15:24.860968+0000","objects_scrubbed":0,"log_size":5607,"log_dups_size":0,"ondisk_log_size":5607,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:42:32.842297+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":80084992,"num_objects":1222,"num_object_clones":0,"num_object_copies":3666,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1222,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11132,"num_write_kb":395328,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,5],"acting":[2,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":410,"reported_epoch":604,"state":"active+clean","last_fresh":"2023-09-23T21:18:25.633450+0000","last_change":"2023-09-23T21:16:42.423216+0000","last_active":"2023-09-23T21:18:25.633450+0000","last_peered":"2023-09-23T21:18:25.633450+0000","last_clean":"2023-09-23T21:18:25.633450+0000","last_became_active":"2023-09-23T21:16:42.422856+0000","last_became_peered":"2023-09-23T21:16:42.422856+0000","last_unstale":"2023-09-23T21:18:25.633450+0000","last_undegraded":"2023-09-23T21:18:25.633450+0000","last_fullsized":"2023-09-23T21:18:25.633450+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:06:54.105798+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"79'161","reported_seq":827,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-23T21:18:28.896555+0000","last_change":"2023-09-23T21:16:45.428032+0000","last_active":"2023-09-23T21:18:28.896555+0000","last_peered":"2023-09-23T21:18:28.896555+0000","last_clean":"2023-09-23T21:18:28.896555+0000","last_became_active":"2023-09-23T21:16:45.427674+0000","last_became_peered":"2023-09-23T21:16:45.427674+0000","last_unstale":"2023-09-23T21:18:28.896555+0000","last_undegraded":"2023-09-23T21:18:28.896555+0000","last_fullsized":"2023-09-23T21:18:28.896555+0000","mapping_epoch":567,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":568,"parent":"0.0","parent_split_bits":0,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:20.738273+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:20.738273+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:39:42.441711+0000","scrub_duration":0.0088336339999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,2],"acting":[0,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":612,"reported_epoch":604,"state":"active+clean","last_fresh":"2023-09-23T21:18:25.634156+0000","last_change":"2023-09-23T21:11:08.760417+0000","last_active":"2023-09-23T21:18:25.634156+0000","last_peered":"2023-09-23T21:18:25.634156+0000","last_clean":"2023-09-23T21:18:25.634156+0000","last_became_active":"2023-09-23T21:11:08.760150+0000","last_became_peered":"2023-09-23T21:11:08.760150+0000","last_unstale":"2023-09-23T21:18:25.634156+0000","last_undegraded":"2023-09-23T21:18:25.634156+0000","last_fullsized":"2023-09-23T21:18:25.634156+0000","mapping_epoch":443,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":444,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:23:10.849768+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":200,"reported_epoch":605,"state":"active+clean","last_fresh":"2023-09-23T21:18:25.885665+0000","last_change":"2023-09-23T21:16:42.424717+0000","last_active":"2023-09-23T21:18:25.885665+0000","last_peered":"2023-09-23T21:18:25.885665+0000","last_clean":"2023-09-23T21:18:25.885665+0000","last_became_active":"2023-09-23T21:16:42.424376+0000","last_became_peered":"2023-09-23T21:16:42.424376+0000","last_unstale":"2023-09-23T21:18:25.885665+0000","last_undegraded":"2023-09-23T21:18:25.885665+0000","last_fullsized":"2023-09-23T21:18:25.885665+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:25:14.162321+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.9","version":"607'7924","reported_seq":9757,"reported_epoch":607,"state":"active+clean","last_fresh":"2023-09-23T21:18:28.262071+0000","last_change":"2023-09-23T21:17:58.411362+0000","last_active":"2023-09-23T21:18:28.262071+0000","last_peered":"2023-09-23T21:18:28.262071+0000","last_clean":"2023-09-23T21:18:28.262071+0000","last_became_active":"2023-09-23T21:17:58.399421+0000","last_became_peered":"2023-09-23T21:17:58.399421+0000","last_unstale":"2023-09-23T21:18:28.262071+0000","last_undegraded":"2023-09-23T21:18:28.262071+0000","last_fullsized":"2023-09-23T21:18:28.262071+0000","mapping_epoch":593,"log_start":"556'3732","ondisk_log_start":"556'3732","created":544,"last_epoch_clean":594,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_clean_scrub_stamp":"2023-09-23T21:15:24.860968+0000","objects_scrubbed":0,"log_size":4192,"log_dups_size":0,"ondisk_log_size":4192,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":41975808,"num_objects":640,"num_object_clones":0,"num_object_copies":1920,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":640,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5640,"num_write_kb":200992,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":860,"num_bytes_recovered":56393728,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,7],"acting":[3,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.3","version":"79'161","reported_seq":908,"reported_epoch":605,"state":"active+clean","last_fresh":"2023-09-23T21:18:25.884898+0000","last_change":"2023-09-23T21:13:33.771279+0000","last_active":"2023-09-23T21:18:25.884898+0000","last_peered":"2023-09-23T21:18:25.884898+0000","last_clean":"2023-09-23T21:18:25.884898+0000","last_became_active":"2023-09-23T21:13:33.770980+0000","last_became_peered":"2023-09-23T21:13:33.770980+0000","last_unstale":"2023-09-23T21:18:25.884898+0000","last_undegraded":"2023-09-23T21:18:25.884898+0000","last_fullsized":"2023-09-23T21:18:25.884898+0000","mapping_epoch":504,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":505,"parent":"0.0","parent_split_bits":3,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:05:57.832302+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:05:57.832302+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:19:42.309604+0000","scrub_duration":0.010206847999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":445,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-23T21:18:28.896868+0000","last_change":"2023-09-23T21:16:42.455365+0000","last_active":"2023-09-23T21:18:28.896868+0000","last_peered":"2023-09-23T21:18:28.896868+0000","last_clean":"2023-09-23T21:18:28.896868+0000","last_became_active":"2023-09-23T21:16:42.452829+0000","last_became_peered":"2023-09-23T21:16:42.452829+0000","last_unstale":"2023-09-23T21:18:28.896868+0000","last_undegraded":"2023-09-23T21:18:28.896868+0000","last_fullsized":"2023-09-23T21:18:28.896868+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:49:26.303389+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":609,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-23T21:18:28.896733+0000","last_change":"2023-09-23T21:13:44.409212+0000","last_active":"2023-09-23T21:18:28.896733+0000","last_peered":"2023-09-23T21:18:28.896733+0000","last_clean":"2023-09-23T21:18:28.896733+0000","last_became_active":"2023-09-23T21:13:44.408958+0000","last_became_peered":"2023-09-23T21:13:44.408958+0000","last_unstale":"2023-09-23T21:18:28.896733+0000","last_undegraded":"2023-09-23T21:18:28.896733+0000","last_fullsized":"2023-09-23T21:18:28.896733+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:03:52.749964+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":138,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-23T21:18:28.897470+0000","last_change":"2023-09-23T21:16:42.427490+0000","last_active":"2023-09-23T21:18:28.897470+0000","last_peered":"2023-09-23T21:18:28.897470+0000","last_clean":"2023-09-23T21:18:28.897470+0000","last_became_active":"2023-09-23T21:16:42.426566+0000","last_became_peered":"2023-09-23T21:16:42.426566+0000","last_unstale":"2023-09-23T21:18:28.897470+0000","last_undegraded":"2023-09-23T21:18:28.897470+0000","last_fullsized":"2023-09-23T21:18:28.897470+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:20:26.396005+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,7],"acting":[0,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.c","version":"605'10865","reported_seq":14041,"reported_epoch":605,"state":"active+clean","last_fresh":"2023-09-23T21:18:26.232889+0000","last_change":"2023-09-23T21:17:21.358806+0000","last_active":"2023-09-23T21:18:26.232889+0000","last_peered":"2023-09-23T21:18:26.232889+0000","last_clean":"2023-09-23T21:18:26.232889+0000","last_became_active":"2023-09-23T21:17:21.346629+0000","last_became_peered":"2023-09-23T21:17:21.346629+0000","last_unstale":"2023-09-23T21:18:26.232889+0000","last_undegraded":"2023-09-23T21:18:26.232889+0000","last_fullsized":"2023-09-23T21:18:26.232889+0000","mapping_epoch":576,"log_start":"570'5300","ondisk_log_start":"570'5300","created":544,"last_epoch_clean":577,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_clean_scrub_stamp":"2023-09-23T21:15:24.860968+0000","objects_scrubbed":0,"log_size":5565,"log_dups_size":0,"ondisk_log_size":5565,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:59:48.173244+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":85917696,"num_objects":1311,"num_object_clones":0,"num_object_copies":3933,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1311,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10865,"num_write_kb":389632,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1539,"num_bytes_recovered":100859904,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,7],"acting":[2,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.6","version":"79'161","reported_seq":541,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-23T21:18:28.897387+0000","last_change":"2023-09-23T21:15:43.333919+0000","last_active":"2023-09-23T21:18:28.897387+0000","last_peered":"2023-09-23T21:18:28.897387+0000","last_clean":"2023-09-23T21:18:28.897387+0000","last_became_active":"2023-09-23T21:13:33.769303+0000","last_became_peered":"2023-09-23T21:13:33.769303+0000","last_unstale":"2023-09-23T21:18:28.897387+0000","last_undegraded":"2023-09-23T21:18:28.897387+0000","last_fullsized":"2023-09-23T21:18:28.897387+0000","mapping_epoch":504,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":505,"parent":"0.0","parent_split_bits":3,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:15:43.333733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:15:43.333733+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:42:29.787357+0000","scrub_duration":0.0092107530000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":51,"reported_epoch":605,"state":"active+clean","last_fresh":"2023-09-23T21:18:25.878847+0000","last_change":"2023-09-23T21:16:42.483813+0000","last_active":"2023-09-23T21:18:25.878847+0000","last_peered":"2023-09-23T21:18:25.878847+0000","last_clean":"2023-09-23T21:18:25.878847+0000","last_became_active":"2023-09-23T21:16:42.482847+0000","last_became_peered":"2023-09-23T21:16:42.482847+0000","last_unstale":"2023-09-23T21:18:25.878847+0000","last_undegraded":"2023-09-23T21:18:25.878847+0000","last_fullsized":"2023-09-23T21:18:25.878847+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:38:57.905679+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":490,"reported_epoch":605,"state":"active+clean","last_fresh":"2023-09-23T21:18:25.872708+0000","last_change":"2023-09-23T21:13:30.790590+0000","last_active":"2023-09-23T21:18:25.872708+0000","last_peered":"2023-09-23T21:18:25.872708+0000","last_clean":"2023-09-23T21:18:25.872708+0000","last_became_active":"2023-09-23T21:13:30.790253+0000","last_became_peered":"2023-09-23T21:13:30.790253+0000","last_unstale":"2023-09-23T21:18:25.872708+0000","last_undegraded":"2023-09-23T21:18:25.872708+0000","last_fullsized":"2023-09-23T21:18:25.872708+0000","mapping_epoch":501,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:23:41.843250+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,0],"acting":[2,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":574,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-23T21:18:28.896772+0000","last_change":"2023-09-23T21:16:42.428535+0000","last_active":"2023-09-23T21:18:28.896772+0000","last_peered":"2023-09-23T21:18:28.896772+0000","last_clean":"2023-09-23T21:18:28.896772+0000","last_became_active":"2023-09-23T21:16:42.428149+0000","last_became_peered":"2023-09-23T21:16:42.428149+0000","last_unstale":"2023-09-23T21:18:28.896772+0000","last_undegraded":"2023-09-23T21:18:28.896772+0000","last_fullsized":"2023-09-23T21:18:28.896772+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:38:59.979788+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.f","version":"605'10946","reported_seq":14571,"reported_epoch":605,"state":"active+clean","last_fresh":"2023-09-23T21:18:26.342680+0000","last_change":"2023-09-23T21:17:03.878062+0000","last_active":"2023-09-23T21:18:26.342680+0000","last_peered":"2023-09-23T21:18:26.342680+0000","last_clean":"2023-09-23T21:18:26.342680+0000","last_became_active":"2023-09-23T21:17:03.870998+0000","last_became_peered":"2023-09-23T21:17:03.870998+0000","last_unstale":"2023-09-23T21:18:26.342680+0000","last_undegraded":"2023-09-23T21:18:26.342680+0000","last_fullsized":"2023-09-23T21:18:26.342680+0000","mapping_epoch":572,"log_start":"570'5300","ondisk_log_start":"570'5300","created":544,"last_epoch_clean":573,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_clean_scrub_stamp":"2023-09-23T21:15:24.860968+0000","objects_scrubbed":0,"log_size":5646,"log_dups_size":0,"ondisk_log_size":5646,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:19:31.942955+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":80084992,"num_objects":1222,"num_object_clones":0,"num_object_copies":3666,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1222,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10946,"num_write_kb":389376,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1762,"num_bytes_recovered":115474432,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,0],"acting":[2,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.5","version":"79'161","reported_seq":643,"reported_epoch":605,"state":"active+clean","last_fresh":"2023-09-23T21:18:25.885634+0000","last_change":"2023-09-23T21:15:41.108638+0000","last_active":"2023-09-23T21:18:25.885634+0000","last_peered":"2023-09-23T21:18:25.885634+0000","last_clean":"2023-09-23T21:18:25.885634+0000","last_became_active":"2023-09-23T21:13:38.373633+0000","last_became_peered":"2023-09-23T21:13:38.373633+0000","last_unstale":"2023-09-23T21:18:25.885634+0000","last_undegraded":"2023-09-23T21:18:25.885634+0000","last_fullsized":"2023-09-23T21:18:25.885634+0000","mapping_epoch":507,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":3,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:15:41.108579+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:15:41.108579+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:21:58.075649+0000","scrub_duration":0.0061511960000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":177,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-23T21:18:28.897967+0000","last_change":"2023-09-23T21:17:02.204086+0000","last_active":"2023-09-23T21:18:28.897967+0000","last_peered":"2023-09-23T21:18:28.897967+0000","last_clean":"2023-09-23T21:18:28.897967+0000","last_became_active":"2023-09-23T21:16:42.517549+0000","last_became_peered":"2023-09-23T21:16:42.517549+0000","last_unstale":"2023-09-23T21:18:28.897967+0000","last_undegraded":"2023-09-23T21:18:28.897967+0000","last_fullsized":"2023-09-23T21:18:28.897967+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:43:07.991844+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":586,"reported_epoch":605,"state":"active+clean","last_fresh":"2023-09-23T21:18:25.885790+0000","last_change":"2023-09-23T21:13:30.807520+0000","last_active":"2023-09-23T21:18:25.885790+0000","last_peered":"2023-09-23T21:18:25.885790+0000","last_clean":"2023-09-23T21:18:25.885790+0000","last_became_active":"2023-09-23T21:13:30.807222+0000","last_became_peered":"2023-09-23T21:13:30.807222+0000","last_unstale":"2023-09-23T21:18:25.885790+0000","last_undegraded":"2023-09-23T21:18:25.885790+0000","last_fullsized":"2023-09-23T21:18:25.885790+0000","mapping_epoch":501,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:40:31.989749+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":469,"reported_epoch":607,"state":"active+clean","last_fresh":"2023-09-23T21:18:27.897139+0000","last_change":"2023-09-23T21:16:42.463411+0000","last_active":"2023-09-23T21:18:27.897139+0000","last_peered":"2023-09-23T21:18:27.897139+0000","last_clean":"2023-09-23T21:18:27.897139+0000","last_became_active":"2023-09-23T21:16:42.463020+0000","last_became_peered":"2023-09-23T21:16:42.463020+0000","last_unstale":"2023-09-23T21:18:27.897139+0000","last_undegraded":"2023-09-23T21:18:27.897139+0000","last_fullsized":"2023-09-23T21:18:27.897139+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:07:34.346129+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.e","version":"605'10990","reported_seq":15194,"reported_epoch":605,"state":"active+clean","last_fresh":"2023-09-23T21:18:26.288057+0000","last_change":"2023-09-23T21:17:34.456174+0000","last_active":"2023-09-23T21:18:26.288057+0000","last_peered":"2023-09-23T21:18:26.288057+0000","last_clean":"2023-09-23T21:18:26.288057+0000","last_became_active":"2023-09-23T21:17:34.444637+0000","last_became_peered":"2023-09-23T21:17:34.444637+0000","last_unstale":"2023-09-23T21:18:26.288057+0000","last_undegraded":"2023-09-23T21:18:26.288057+0000","last_fullsized":"2023-09-23T21:18:26.288057+0000","mapping_epoch":582,"log_start":"571'6100","ondisk_log_start":"571'6100","created":544,"last_epoch_clean":583,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_clean_scrub_stamp":"2023-09-23T21:15:24.860968+0000","objects_scrubbed":0,"log_size":4890,"log_dups_size":0,"ondisk_log_size":4890,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:17:47.772535+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":82968576,"num_objects":1266,"num_object_clones":0,"num_object_copies":3798,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1266,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10990,"num_write_kb":392192,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2052,"num_bytes_recovered":134479872,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.4","version":"79'161","reported_seq":1014,"reported_epoch":607,"state":"active+clean","last_fresh":"2023-09-23T21:18:27.900125+0000","last_change":"2023-09-23T21:16:45.426136+0000","last_active":"2023-09-23T21:18:27.900125+0000","last_peered":"2023-09-23T21:18:27.900125+0000","last_clean":"2023-09-23T21:18:27.900125+0000","last_became_active":"2023-09-23T21:16:45.425669+0000","last_became_peered":"2023-09-23T21:16:45.425669+0000","last_unstale":"2023-09-23T21:18:27.900125+0000","last_undegraded":"2023-09-23T21:18:27.900125+0000","last_fullsized":"2023-09-23T21:18:27.900125+0000","mapping_epoch":567,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":568,"parent":"0.0","parent_split_bits":3,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:20.012764+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:20.012764+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:52:39.142509+0000","scrub_duration":0.0045251919999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":563,"reported_epoch":604,"state":"active+clean","last_fresh":"2023-09-23T21:18:25.633277+0000","last_change":"2023-09-23T21:16:42.459732+0000","last_active":"2023-09-23T21:18:25.633277+0000","last_peered":"2023-09-23T21:18:25.633277+0000","last_clean":"2023-09-23T21:18:25.633277+0000","last_became_active":"2023-09-23T21:16:42.459413+0000","last_became_peered":"2023-09-23T21:16:42.459413+0000","last_unstale":"2023-09-23T21:18:25.633277+0000","last_undegraded":"2023-09-23T21:18:25.633277+0000","last_fullsized":"2023-09-23T21:18:25.633277+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:37:44.548863+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,3],"acting":[2,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.6","version":"70'1","reported_seq":610,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-23T21:18:28.897793+0000","last_change":"2023-09-23T21:17:02.208077+0000","last_active":"2023-09-23T21:18:28.897793+0000","last_peered":"2023-09-23T21:18:28.897793+0000","last_clean":"2023-09-23T21:18:28.897793+0000","last_became_active":"2023-09-23T21:16:42.431308+0000","last_became_peered":"2023-09-23T21:16:42.431308+0000","last_unstale":"2023-09-23T21:18:28.897793+0000","last_undegraded":"2023-09-23T21:18:28.897793+0000","last_fullsized":"2023-09-23T21:18:28.897793+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:16:49.009013+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":180,"reported_epoch":605,"state":"active+clean","last_fresh":"2023-09-23T21:18:25.884779+0000","last_change":"2023-09-23T21:13:12.216895+0000","last_active":"2023-09-23T21:18:25.884779+0000","last_peered":"2023-09-23T21:18:25.884779+0000","last_clean":"2023-09-23T21:18:25.884779+0000","last_became_active":"2023-09-23T21:13:12.216654+0000","last_became_peered":"2023-09-23T21:13:12.216654+0000","last_unstale":"2023-09-23T21:18:25.884779+0000","last_undegraded":"2023-09-23T21:18:25.884779+0000","last_fullsized":"2023-09-23T21:18:25.884779+0000","mapping_epoch":493,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:02:40.430096+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.13","version":"605'7574","reported_seq":9224,"reported_epoch":605,"state":"active+clean","last_fresh":"2023-09-23T21:18:26.221966+0000","last_change":"2023-09-23T21:17:50.364498+0000","last_active":"2023-09-23T21:18:26.221966+0000","last_peered":"2023-09-23T21:18:26.221966+0000","last_clean":"2023-09-23T21:18:26.221966+0000","last_became_active":"2023-09-23T21:17:39.911716+0000","last_became_peered":"2023-09-23T21:17:39.911716+0000","last_unstale":"2023-09-23T21:18:26.221966+0000","last_undegraded":"2023-09-23T21:18:26.221966+0000","last_fullsized":"2023-09-23T21:18:26.221966+0000","mapping_epoch":585,"log_start":"549'1881","ondisk_log_start":"549'1881","created":559,"last_epoch_clean":590,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_clean_scrub_stamp":"2023-09-23T21:15:24.860968+0000","objects_scrubbed":0,"log_size":5693,"log_dups_size":0,"ondisk_log_size":5693,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":40763392,"num_objects":622,"num_object_clones":0,"num_object_copies":1866,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":622,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5358,"num_write_kb":191360,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":772,"num_bytes_recovered":50528256,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,2],"acting":[7,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.19","version":"79'161","reported_seq":817,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-23T21:18:28.896526+0000","last_change":"2023-09-23T21:16:45.428879+0000","last_active":"2023-09-23T21:18:28.896526+0000","last_peered":"2023-09-23T21:18:28.896526+0000","last_clean":"2023-09-23T21:18:28.896526+0000","last_became_active":"2023-09-23T21:16:45.428660+0000","last_became_peered":"2023-09-23T21:16:45.428660+0000","last_unstale":"2023-09-23T21:18:28.896526+0000","last_undegraded":"2023-09-23T21:18:28.896526+0000","last_fullsized":"2023-09-23T21:18:28.896526+0000","mapping_epoch":567,"log_start":"0'0","ondisk_log_start":"0'0","created":479,"last_epoch_clean":568,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:15:48.271791+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:15:48.271791+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:25:26.361045+0000","scrub_duration":0.0057149180000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":137,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-23T21:18:28.898019+0000","last_change":"2023-09-23T21:16:42.434830+0000","last_active":"2023-09-23T21:18:28.898019+0000","last_peered":"2023-09-23T21:18:28.898019+0000","last_clean":"2023-09-23T21:18:28.898019+0000","last_became_active":"2023-09-23T21:16:42.430289+0000","last_became_peered":"2023-09-23T21:16:42.430289+0000","last_unstale":"2023-09-23T21:18:28.898019+0000","last_undegraded":"2023-09-23T21:18:28.898019+0000","last_fullsized":"2023-09-23T21:18:28.898019+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:38:48.053766+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":109,"reported_epoch":604,"state":"active+clean","last_fresh":"2023-09-23T21:18:25.633602+0000","last_change":"2023-09-23T21:13:38.373814+0000","last_active":"2023-09-23T21:18:25.633602+0000","last_peered":"2023-09-23T21:18:25.633602+0000","last_clean":"2023-09-23T21:18:25.633602+0000","last_became_active":"2023-09-23T21:13:38.373458+0000","last_became_peered":"2023-09-23T21:13:38.373458+0000","last_unstale":"2023-09-23T21:18:25.633602+0000","last_undegraded":"2023-09-23T21:18:25.633602+0000","last_fullsized":"2023-09-23T21:18:25.633602+0000","mapping_epoch":507,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:23:08.288401+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"11.14","version":"605'7704","reported_seq":9413,"reported_epoch":605,"state":"active+clean","last_fresh":"2023-09-23T21:18:26.339338+0000","last_change":"2023-09-23T21:18:17.930003+0000","last_active":"2023-09-23T21:18:26.339338+0000","last_peered":"2023-09-23T21:18:26.339338+0000","last_clean":"2023-09-23T21:18:26.339338+0000","last_became_active":"2023-09-23T21:18:17.894915+0000","last_became_peered":"2023-09-23T21:18:17.894915+0000","last_unstale":"2023-09-23T21:18:26.339338+0000","last_undegraded":"2023-09-23T21:18:26.339338+0000","last_fullsized":"2023-09-23T21:18:26.339338+0000","mapping_epoch":600,"log_start":"549'2163","ondisk_log_start":"549'2163","created":559,"last_epoch_clean":601,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_clean_scrub_stamp":"2023-09-23T21:15:24.860968+0000","objects_scrubbed":0,"log_size":5541,"log_dups_size":0,"ondisk_log_size":5541,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":35586048,"num_objects":543,"num_object_clones":0,"num_object_copies":1629,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":543,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5467,"num_write_kb":192320,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":803,"num_bytes_recovered":32505856,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,3],"acting":[7,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.1e","version":"79'161","reported_seq":591,"reported_epoch":607,"state":"active+clean","last_fresh":"2023-09-23T21:18:27.900701+0000","last_change":"2023-09-23T21:16:45.420788+0000","last_active":"2023-09-23T21:18:27.900701+0000","last_peered":"2023-09-23T21:18:27.900701+0000","last_clean":"2023-09-23T21:18:27.900701+0000","last_became_active":"2023-09-23T21:16:45.420448+0000","last_became_peered":"2023-09-23T21:16:45.420448+0000","last_unstale":"2023-09-23T21:18:27.900701+0000","last_undegraded":"2023-09-23T21:18:27.900701+0000","last_fullsized":"2023-09-23T21:18:27.900701+0000","mapping_epoch":567,"log_start":"0'0","ondisk_log_start":"0'0","created":479,"last_epoch_clean":568,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:15:53.061319+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:15:53.061319+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:46:31.982439+0000","scrub_duration":0.007429878,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,4],"acting":[3,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":215,"reported_epoch":605,"state":"active+clean","last_fresh":"2023-09-23T21:18:25.885398+0000","last_change":"2023-09-23T21:16:42.518584+0000","last_active":"2023-09-23T21:18:25.885398+0000","last_peered":"2023-09-23T21:18:25.885398+0000","last_clean":"2023-09-23T21:18:25.885398+0000","last_became_active":"2023-09-23T21:16:42.416632+0000","last_became_peered":"2023-09-23T21:16:42.416632+0000","last_unstale":"2023-09-23T21:18:25.885398+0000","last_undegraded":"2023-09-23T21:18:25.885398+0000","last_fullsized":"2023-09-23T21:18:25.885398+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:02:18.972080+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,7],"acting":[4,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":170,"reported_epoch":605,"state":"active+clean","last_fresh":"2023-09-23T21:18:25.885039+0000","last_change":"2023-09-23T21:12:10.770474+0000","last_active":"2023-09-23T21:18:25.885039+0000","last_peered":"2023-09-23T21:18:25.885039+0000","last_clean":"2023-09-23T21:18:25.885039+0000","last_became_active":"2023-09-23T21:12:10.770184+0000","last_became_peered":"2023-09-23T21:12:10.770184+0000","last_unstale":"2023-09-23T21:18:25.885039+0000","last_undegraded":"2023-09-23T21:18:25.885039+0000","last_fullsized":"2023-09-23T21:18:25.885039+0000","mapping_epoch":470,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:05:09.407151+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.15","version":"605'7871","reported_seq":13117,"reported_epoch":605,"state":"active+clean","last_fresh":"2023-09-23T21:18:26.220606+0000","last_change":"2023-09-23T21:18:06.522430+0000","last_active":"2023-09-23T21:18:26.220606+0000","last_peered":"2023-09-23T21:18:26.220606+0000","last_clean":"2023-09-23T21:18:26.220606+0000","last_became_active":"2023-09-23T21:16:46.418280+0000","last_became_peered":"2023-09-23T21:16:46.418280+0000","last_unstale":"2023-09-23T21:18:26.220606+0000","last_undegraded":"2023-09-23T21:18:26.220606+0000","last_fullsized":"2023-09-23T21:18:26.220606+0000","mapping_epoch":568,"log_start":"548'1377","ondisk_log_start":"548'1377","created":559,"last_epoch_clean":569,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_clean_scrub_stamp":"2023-09-23T21:15:24.860968+0000","objects_scrubbed":0,"log_size":6494,"log_dups_size":0,"ondisk_log_size":6494,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":39288832,"num_objects":600,"num_object_clones":0,"num_object_copies":1800,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":600,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5576,"num_write_kb":197600,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1697,"num_bytes_recovered":111214592,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":396,"reported_epoch":604,"state":"active+clean","last_fresh":"2023-09-23T21:18:25.633296+0000","last_change":"2023-09-23T21:12:10.771160+0000","last_active":"2023-09-23T21:18:25.633296+0000","last_peered":"2023-09-23T21:18:25.633296+0000","last_clean":"2023-09-23T21:18:25.633296+0000","last_became_active":"2023-09-23T21:12:10.757808+0000","last_became_peered":"2023-09-23T21:12:10.757808+0000","last_unstale":"2023-09-23T21:18:25.633296+0000","last_undegraded":"2023-09-23T21:18:25.633296+0000","last_fullsized":"2023-09-23T21:18:25.633296+0000","mapping_epoch":470,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:30:50.987139+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":180,"reported_epoch":605,"state":"active+clean","last_fresh":"2023-09-23T21:18:25.885051+0000","last_change":"2023-09-23T21:13:12.216602+0000","last_active":"2023-09-23T21:18:25.885051+0000","last_peered":"2023-09-23T21:18:25.885051+0000","last_clean":"2023-09-23T21:18:25.885051+0000","last_became_active":"2023-09-23T21:13:12.216204+0000","last_became_peered":"2023-09-23T21:13:12.216204+0000","last_unstale":"2023-09-23T21:18:25.885051+0000","last_undegraded":"2023-09-23T21:18:25.885051+0000","last_fullsized":"2023-09-23T21:18:25.885051+0000","mapping_epoch":493,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:23:37.043390+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.16","version":"609'7603","reported_seq":7689,"reported_epoch":609,"state":"active+clean","last_fresh":"2023-09-23T21:18:30.117217+0000","last_change":"2023-09-23T21:16:33.955304+0000","last_active":"2023-09-23T21:18:30.117217+0000","last_peered":"2023-09-23T21:18:30.117217+0000","last_clean":"2023-09-23T21:18:30.117217+0000","last_became_active":"2023-09-23T21:16:32.926127+0000","last_became_peered":"2023-09-23T21:16:32.926127+0000","last_unstale":"2023-09-23T21:18:30.117217+0000","last_undegraded":"2023-09-23T21:18:30.117217+0000","last_fullsized":"2023-09-23T21:18:30.117217+0000","mapping_epoch":559,"log_start":"550'2669","ondisk_log_start":"550'2669","created":559,"last_epoch_clean":561,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_clean_scrub_stamp":"2023-09-23T21:15:24.860968+0000","objects_scrubbed":0,"log_size":4934,"log_dups_size":0,"ondisk_log_size":4934,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":36143104,"num_objects":552,"num_object_clones":0,"num_object_copies":1656,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":552,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5370,"num_write_kb":189472,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1c","version":"79'161","reported_seq":1038,"reported_epoch":607,"state":"active+clean","last_fresh":"2023-09-23T21:18:27.899972+0000","last_change":"2023-09-23T21:16:45.419779+0000","last_active":"2023-09-23T21:18:27.899972+0000","last_peered":"2023-09-23T21:18:27.899972+0000","last_clean":"2023-09-23T21:18:27.899972+0000","last_became_active":"2023-09-23T21:16:45.419394+0000","last_became_peered":"2023-09-23T21:16:45.419394+0000","last_unstale":"2023-09-23T21:18:27.899972+0000","last_undegraded":"2023-09-23T21:18:27.899972+0000","last_fullsized":"2023-09-23T21:18:27.899972+0000","mapping_epoch":567,"log_start":"0'0","ondisk_log_start":"0'0","created":479,"last_epoch_clean":568,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:15:54.019828+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:15:54.019828+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:59:05.422258+0000","scrub_duration":0.0020962530000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":213,"reported_epoch":604,"state":"active+clean","last_fresh":"2023-09-23T21:18:25.632506+0000","last_change":"2023-09-23T21:16:42.424187+0000","last_active":"2023-09-23T21:18:25.632506+0000","last_peered":"2023-09-23T21:18:25.632506+0000","last_clean":"2023-09-23T21:18:25.632506+0000","last_became_active":"2023-09-23T21:16:42.423805+0000","last_became_peered":"2023-09-23T21:16:42.423805+0000","last_unstale":"2023-09-23T21:18:25.632506+0000","last_undegraded":"2023-09-23T21:18:25.632506+0000","last_fullsized":"2023-09-23T21:18:25.632506+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:55:46.051199+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,7],"acting":[2,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":274,"reported_epoch":604,"state":"active+clean","last_fresh":"2023-09-23T21:18:25.633585+0000","last_change":"2023-09-23T21:13:12.200767+0000","last_active":"2023-09-23T21:18:25.633585+0000","last_peered":"2023-09-23T21:18:25.633585+0000","last_clean":"2023-09-23T21:18:25.633585+0000","last_became_active":"2023-09-23T21:13:12.200435+0000","last_became_peered":"2023-09-23T21:13:12.200435+0000","last_unstale":"2023-09-23T21:18:25.633585+0000","last_undegraded":"2023-09-23T21:18:25.633585+0000","last_fullsized":"2023-09-23T21:18:25.633585+0000","mapping_epoch":493,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:15:44.937886+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"11.17","version":"605'7896","reported_seq":7985,"reported_epoch":605,"state":"active+clean","last_fresh":"2023-09-23T21:18:26.362471+0000","last_change":"2023-09-23T21:18:06.404241+0000","last_active":"2023-09-23T21:18:26.362471+0000","last_peered":"2023-09-23T21:18:26.362471+0000","last_clean":"2023-09-23T21:18:26.362471+0000","last_became_active":"2023-09-23T21:16:32.890013+0000","last_became_peered":"2023-09-23T21:16:32.890013+0000","last_unstale":"2023-09-23T21:18:26.362471+0000","last_undegraded":"2023-09-23T21:18:26.362471+0000","last_fullsized":"2023-09-23T21:18:26.362471+0000","mapping_epoch":559,"log_start":"549'1458","ondisk_log_start":"549'1458","created":559,"last_epoch_clean":560,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_clean_scrub_stamp":"2023-09-23T21:15:24.860968+0000","objects_scrubbed":0,"log_size":6438,"log_dups_size":0,"ondisk_log_size":6438,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":43352064,"num_objects":662,"num_object_clones":0,"num_object_copies":1986,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":662,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5592,"num_write_kb":200096,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1d","version":"79'161","reported_seq":740,"reported_epoch":607,"state":"active+clean","last_fresh":"2023-09-23T21:18:27.901589+0000","last_change":"2023-09-23T21:15:23.802125+0000","last_active":"2023-09-23T21:18:27.901589+0000","last_peered":"2023-09-23T21:18:27.901589+0000","last_clean":"2023-09-23T21:18:27.901589+0000","last_became_active":"2023-09-23T21:15:23.801704+0000","last_became_peered":"2023-09-23T21:15:23.801704+0000","last_unstale":"2023-09-23T21:18:27.901589+0000","last_undegraded":"2023-09-23T21:18:27.901589+0000","last_fullsized":"2023-09-23T21:18:27.901589+0000","mapping_epoch":542,"log_start":"0'0","ondisk_log_start":"0'0","created":479,"last_epoch_clean":543,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:09:36.858077+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:09:36.858077+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.010304610000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,2],"acting":[3,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":137,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-23T21:18:28.897675+0000","last_change":"2023-09-23T21:13:38.380574+0000","last_active":"2023-09-23T21:18:28.897675+0000","last_peered":"2023-09-23T21:18:28.897675+0000","last_clean":"2023-09-23T21:18:28.897675+0000","last_became_active":"2023-09-23T21:13:38.373252+0000","last_became_peered":"2023-09-23T21:13:38.373252+0000","last_unstale":"2023-09-23T21:18:28.897675+0000","last_undegraded":"2023-09-23T21:18:28.897675+0000","last_fullsized":"2023-09-23T21:18:28.897675+0000","mapping_epoch":507,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:58:14.788031+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":11,"num_bytes_recovered":3324,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":180,"reported_epoch":605,"state":"active+clean","last_fresh":"2023-09-23T21:18:25.884709+0000","last_change":"2023-09-23T21:12:10.793610+0000","last_active":"2023-09-23T21:18:25.884709+0000","last_peered":"2023-09-23T21:18:25.884709+0000","last_clean":"2023-09-23T21:18:25.884709+0000","last_became_active":"2023-09-23T21:12:10.793081+0000","last_became_peered":"2023-09-23T21:12:10.793081+0000","last_unstale":"2023-09-23T21:18:25.884709+0000","last_undegraded":"2023-09-23T21:18:25.884709+0000","last_fullsized":"2023-09-23T21:18:25.884709+0000","mapping_epoch":470,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:34:36.291550+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":11,"num_pg":26,"stat_sum":{"num_bytes":1303150616,"num_objects":19886,"num_object_clones":0,"num_object_copies":59658,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":19886,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":170620,"num_write_kb":6076321,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":15800,"num_bytes_recovered":974127104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":4221186048,"data_stored":4221173832,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":138784,"ondisk_log_size":138784,"up":78,"acting":78,"num_store_stats":6},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":80,"num_bytes_recovered":18686,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":118784,"data_stored":21350,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":14322,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":31,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":525376,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2109440,"data_stored":2098784,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4991,"ondisk_log_size":4991,"up":93,"acting":93,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201736,"num_pgs":68,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1279884,"kb_used_data":653700,"kb_used_omap":0,"kb_used_meta":626176,"kb_avail":92465268,"statfs":{"total":95995035648,"available":94684434432,"internally_reserved":0,"allocated":669388800,"data_stored":667658858,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":641204224},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:18:22 2023","interfaces":[{"interface":"back","average":{"1min":1.5209999999999999,"5min":1.474,"15min":2.1080000000000001},"min":{"1min":0.22900000000000001,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":3.968,"5min":37.207999999999998,"15min":37.207999999999998},"last":0.376},{"interface":"front","average":{"1min":1.73,"5min":1.748,"15min":2.0720000000000001},"min":{"1min":0.318,"5min":0.253,"15min":0.253},"max":{"1min":5.2910000000000004,"5min":37.122,"15min":37.122},"last":0.41099999999999998}]},{"osd":1,"last update":"Sat Sep 23 21:18:16 2023","interfaces":[{"interface":"back","average":{"1min":1.2609999999999999,"5min":1.2210000000000001,"15min":1.7969999999999999},"min":{"1min":0.27600000000000002,"5min":0.27600000000000002,"15min":0.26700000000000002},"max":{"1min":6.9130000000000003,"5min":7.5369999999999999,"15min":14.034000000000001},"last":0.436},{"interface":"front","average":{"1min":1.4370000000000001,"5min":1.379,"15min":1.776},"min":{"1min":0.34300000000000003,"5min":0.29999999999999999,"15min":0.26300000000000001},"max":{"1min":3.9300000000000002,"5min":11.002000000000001,"15min":11.226000000000001},"last":0.56200000000000006}]},{"osd":2,"last update":"Sat Sep 23 21:17:37 2023","interfaces":[{"interface":"back","average":{"1min":2.9670000000000001,"5min":1.9039999999999999,"15min":1.825},"min":{"1min":0.372,"5min":0.26400000000000001,"15min":0.26200000000000001},"max":{"1min":25.471,"5min":25.471,"15min":25.471},"last":0.39400000000000002},{"interface":"front","average":{"1min":3.073,"5min":1.73,"15min":1.7430000000000001},"min":{"1min":0.372,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":27.187999999999999,"5min":27.187999999999999,"15min":27.187999999999999},"last":0.52200000000000002}]},{"osd":3,"last update":"Sat Sep 23 21:18:04 2023","interfaces":[{"interface":"back","average":{"1min":1.7070000000000001,"5min":1.484,"15min":1.8280000000000001},"min":{"1min":0.33200000000000002,"5min":0.30599999999999999,"15min":0.29099999999999998},"max":{"1min":6.8739999999999997,"5min":8.6259999999999994,"15min":12.359999999999999},"last":0.50900000000000001},{"interface":"front","average":{"1min":1.1180000000000001,"5min":1.732,"15min":1.831},"min":{"1min":0.28799999999999998,"5min":0.28799999999999998,"15min":0.28699999999999998},"max":{"1min":3.8300000000000001,"5min":15.82,"15min":15.82},"last":0.495}]},{"osd":4,"last update":"Sat Sep 23 21:18:04 2023","interfaces":[{"interface":"back","average":{"1min":0.93400000000000005,"5min":1.02,"15min":1.1799999999999999},"min":{"1min":0.251,"5min":0.251,"15min":0.248},"max":{"1min":3.9990000000000001,"5min":7.6609999999999996,"15min":11.297000000000001},"last":0.46500000000000002},{"interface":"front","average":{"1min":1.0089999999999999,"5min":1.073,"15min":1.137},"min":{"1min":0.28599999999999998,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":5.6349999999999998,"5min":8.6059999999999999,"15min":12.752000000000001},"last":0.53800000000000003}]},{"osd":5,"last update":"Sat Sep 23 21:18:04 2023","interfaces":[{"interface":"back","average":{"1min":1.286,"5min":1.095,"15min":1.089},"min":{"1min":0.28999999999999998,"5min":0.28999999999999998,"15min":0.26000000000000001},"max":{"1min":6.9740000000000002,"5min":7.5910000000000002,"15min":9.4789999999999992},"last":0.48399999999999999},{"interface":"front","average":{"1min":1.381,"5min":1.171,"15min":1.2130000000000001},"min":{"1min":0.35599999999999998,"5min":0.32000000000000001,"15min":0.26200000000000001},"max":{"1min":6.7439999999999998,"5min":7.6929999999999996,"15min":11.257},"last":0.59099999999999997}]},{"osd":6,"last update":"Sat Sep 23 21:18:16 2023","interfaces":[{"interface":"back","average":{"1min":0.86099999999999999,"5min":0.88300000000000001,"15min":1.0620000000000001},"min":{"1min":0.34599999999999997,"5min":0.32000000000000001,"15min":0.30099999999999999},"max":{"1min":5.7069999999999999,"5min":7.5339999999999998,"15min":10.941000000000001},"last":0.60299999999999998},{"interface":"front","average":{"1min":1.379,"5min":1.0429999999999999,"15min":1.181},"min":{"1min":0.41099999999999998,"5min":0.36299999999999999,"15min":0.33100000000000002},"max":{"1min":5.6760000000000002,"5min":5.6760000000000002,"15min":12.715},"last":0.57499999999999996}]}]},{"osd":6,"up_from":0,"seq":704374636820,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":1,"up_from":0,"seq":1503238553752,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":0,"up_from":493,"seq":2117418876993,"num_pgs":75,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1184916,"kb_used_data":807436,"kb_used_omap":36,"kb_used_meta":377435,"kb_avail":92560236,"statfs":{"total":95995035648,"available":94781681664,"internally_reserved":0,"allocated":826814464,"data_stored":824002239,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":37336,"internal_metadata":386493992},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 21:18:27 2023","interfaces":[{"interface":"back","average":{"1min":1.3620000000000001,"5min":1.4410000000000001,"15min":0.98399999999999999},"min":{"1min":0.25900000000000001,"5min":0.214,"15min":0.214},"max":{"1min":6.7779999999999996,"5min":13.779999999999999,"15min":13.779999999999999},"last":0.79900000000000004},{"interface":"front","average":{"1min":2.5499999999999998,"5min":1.7569999999999999,"15min":1.099},"min":{"1min":0.35099999999999998,"5min":0.20799999999999999,"15min":0.20799999999999999},"max":{"1min":10.551,"5min":11.964,"15min":11.964},"last":0.40799999999999997}]},{"osd":2,"last update":"Sat Sep 23 21:18:27 2023","interfaces":[{"interface":"back","average":{"1min":2.0579999999999998,"5min":1.8,"15min":1.036},"min":{"1min":0.36599999999999999,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":7.3449999999999998,"5min":13.637,"15min":13.637},"last":0.91600000000000004},{"interface":"front","average":{"1min":3.258,"5min":2.46,"15min":1.4650000000000001},"min":{"1min":0.30199999999999999,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":10.401999999999999,"5min":26.571000000000002,"15min":26.571000000000002},"last":6.2569999999999997}]},{"osd":3,"last update":"Sat Sep 23 21:18:27 2023","interfaces":[{"interface":"back","average":{"1min":3.9950000000000001,"5min":2.6000000000000001,"15min":1.4450000000000001},"min":{"1min":0.253,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":37.079999999999998,"5min":37.079999999999998,"15min":37.079999999999998},"last":1.097},{"interface":"front","average":{"1min":3.4550000000000001,"5min":2.2029999999999998,"15min":1.4790000000000001},"min":{"1min":0.34699999999999998,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":32.935000000000002,"5min":32.935000000000002,"15min":32.935000000000002},"last":3.4430000000000001}]},{"osd":4,"last update":"Sat Sep 23 21:18:27 2023","interfaces":[{"interface":"back","average":{"1min":1.546,"5min":1.7390000000000001,"15min":1.3080000000000001},"min":{"1min":0.34499999999999997,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":4.7880000000000003,"5min":22.780000000000001,"15min":22.780000000000001},"last":3.4710000000000001},{"interface":"front","average":{"1min":1.9390000000000001,"5min":1.4990000000000001,"15min":1.163},"min":{"1min":0.32500000000000001,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":6.9119999999999999,"5min":22.652000000000001,"15min":22.652000000000001},"last":3.4049999999999998}]},{"osd":5,"last update":"Sat Sep 23 21:18:27 2023","interfaces":[{"interface":"back","average":{"1min":1.5549999999999999,"5min":1.5680000000000001,"15min":1.3460000000000001},"min":{"1min":0.312,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":6.1139999999999999,"5min":8.9329999999999998,"15min":8.9329999999999998},"last":1.2270000000000001},{"interface":"front","average":{"1min":1.9039999999999999,"5min":1.696,"15min":1.274},"min":{"1min":0.40100000000000002,"5min":0.313,"15min":0.313},"max":{"1min":6.5990000000000002,"5min":15.321999999999999,"15min":15.321999999999999},"last":3.5190000000000001}]},{"osd":6,"last update":"Sat Sep 23 21:18:27 2023","interfaces":[{"interface":"back","average":{"1min":1.5720000000000001,"5min":1.4810000000000001,"15min":1.2450000000000001},"min":{"1min":0.372,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":6.9409999999999998,"5min":7.2050000000000001,"15min":7.2050000000000001},"last":3.3279999999999998},{"interface":"front","average":{"1min":1.647,"5min":1.595,"15min":1.403},"min":{"1min":0.38400000000000001,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":6.266,"5min":8.8889999999999993,"15min":8.8889999999999993},"last":1.0609999999999999}]},{"osd":7,"last update":"Sat Sep 23 21:18:27 2023","interfaces":[{"interface":"back","average":{"1min":1.595,"5min":1.8240000000000001,"15min":1.4139999999999999},"min":{"1min":0.34699999999999998,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":4.8360000000000003,"5min":15.273999999999999,"15min":15.273999999999999},"last":3.4940000000000002},{"interface":"front","average":{"1min":1.75,"5min":1.7709999999999999,"15min":1.3620000000000001},"min":{"1min":0.36199999999999999,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":6.9109999999999996,"5min":33.779000000000003,"15min":33.779000000000003},"last":3.3820000000000001}]}]},{"osd":2,"up_from":93,"seq":399431958864,"num_pgs":53,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1830616,"kb_used_data":821008,"kb_used_omap":81,"kb_used_meta":1009518,"kb_avail":91914536,"statfs":{"total":95995035648,"available":94120484864,"internally_reserved":0,"allocated":840712192,"data_stored":837871593,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":83750,"internal_metadata":1033746650},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,3,0,2],"upper_bound":64},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:18:23 2023","interfaces":[{"interface":"back","average":{"1min":2.6339999999999999,"5min":1.9370000000000001,"15min":1.2010000000000001},"min":{"1min":0.221,"5min":0.187,"15min":0.187},"max":{"1min":9.9499999999999993,"5min":13.25,"15min":13.25},"last":0.221},{"interface":"front","average":{"1min":2.2509999999999999,"5min":2.089,"15min":1.806},"min":{"1min":0.26800000000000002,"5min":0.20200000000000001,"15min":0.20200000000000001},"max":{"1min":9.8849999999999998,"5min":18.111999999999998,"15min":18.111999999999998},"last":5.2439999999999998}]},{"osd":1,"last update":"Sat Sep 23 21:18:23 2023","interfaces":[{"interface":"back","average":{"1min":1.645,"5min":1.756,"15min":2.2629999999999999},"min":{"1min":0.34000000000000002,"5min":0.20599999999999999,"15min":0.188},"max":{"1min":10.535,"5min":10.535,"15min":34.396999999999998},"last":2.577},{"interface":"front","average":{"1min":2.2240000000000002,"5min":1.95,"15min":2.3460000000000001},"min":{"1min":0.24099999999999999,"5min":0.24099999999999999,"15min":0.217},"max":{"1min":6.0990000000000002,"5min":18.081,"15min":34.462000000000003},"last":4.9740000000000002}]},{"osd":3,"last update":"Sat Sep 23 21:18:01 2023","interfaces":[{"interface":"back","average":{"1min":2.0880000000000001,"5min":2.04,"15min":2.2389999999999999},"min":{"1min":0.248,"5min":0.20599999999999999,"15min":0.20599999999999999},"max":{"1min":7.3719999999999999,"5min":20.106999999999999,"15min":35.549999999999997},"last":4.8739999999999997},{"interface":"front","average":{"1min":2.6659999999999999,"5min":2.556,"15min":2.6099999999999999},"min":{"1min":0.35799999999999998,"5min":0.28499999999999998,"15min":0.21299999999999999},"max":{"1min":10.162000000000001,"5min":20.239000000000001,"15min":36.951999999999998},"last":2.9689999999999999}]},{"osd":4,"last update":"Sat Sep 23 21:18:23 2023","interfaces":[{"interface":"back","average":{"1min":1.2749999999999999,"5min":1.3460000000000001,"15min":1.4450000000000001},"min":{"1min":0.27000000000000002,"5min":0.27000000000000002,"15min":0.249},"max":{"1min":3.8530000000000002,"5min":7.9770000000000003,"15min":10.266999999999999},"last":0.99399999999999999},{"interface":"front","average":{"1min":1.363,"5min":1.5640000000000001,"15min":1.7110000000000001},"min":{"1min":0.38900000000000001,"5min":0.30299999999999999,"15min":0.26700000000000002},"max":{"1min":4.2210000000000001,"5min":10.459,"15min":12.49},"last":2.4710000000000001}]},{"osd":5,"last update":"Sat Sep 23 21:18:23 2023","interfaces":[{"interface":"back","average":{"1min":1.6160000000000001,"5min":1.456,"15min":1.5349999999999999},"min":{"1min":0.32300000000000001,"5min":0.29099999999999998,"15min":0.28000000000000003},"max":{"1min":5.4139999999999997,"5min":7.6180000000000003,"15min":9.5039999999999996},"last":1.016},{"interface":"front","average":{"1min":1.474,"5min":1.5169999999999999,"15min":1.6890000000000001},"min":{"1min":0.34899999999999998,"5min":0.33800000000000002,"15min":0.29399999999999998},"max":{"1min":4.2649999999999997,"5min":10.425000000000001,"15min":12.57},"last":2.548}]},{"osd":6,"last update":"Sat Sep 23 21:18:23 2023","interfaces":[{"interface":"back","average":{"1min":1.542,"5min":1.633,"15min":1.8979999999999999},"min":{"1min":0.54100000000000004,"5min":0.377,"15min":0.35699999999999998},"max":{"1min":4.2990000000000004,"5min":10.335000000000001,"15min":12.518000000000001},"last":2.5960000000000001},{"interface":"front","average":{"1min":1.7190000000000001,"5min":1.696,"15min":2.0510000000000002},"min":{"1min":0.501,"5min":0.377,"15min":0.35599999999999998},"max":{"1min":4.7969999999999997,"5min":10.381,"15min":12.544},"last":2.5129999999999999}]},{"osd":7,"last update":"Sat Sep 23 21:18:23 2023","interfaces":[{"interface":"back","average":{"1min":1.6339999999999999,"5min":1.637,"15min":1.6519999999999999},"min":{"1min":0.31,"5min":0.31,"15min":0.29799999999999999},"max":{"1min":5.7480000000000002,"5min":18.021999999999998,"15min":18.021999999999998},"last":1.0589999999999999},{"interface":"front","average":{"1min":2.145,"5min":1.6339999999999999,"15min":1.73},"min":{"1min":0.35999999999999999,"5min":0.34999999999999998,"15min":0.32200000000000001},"max":{"1min":6.694,"5min":9.6349999999999998,"15min":10.92},"last":0.96299999999999997}]}]},{"osd":3,"up_from":98,"seq":420906795338,"num_pgs":79,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1970116,"kb_used_data":883068,"kb_used_omap":7,"kb_used_meta":1087032,"kb_avail":91775036,"statfs":{"total":95995035648,"available":93977636864,"internally_reserved":0,"allocated":904261632,"data_stored":901458668,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7576,"internal_metadata":1113121384},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:18:18 2023","interfaces":[{"interface":"back","average":{"1min":2.2069999999999999,"5min":4.0750000000000002,"15min":1.583},"min":{"1min":0.22500000000000001,"5min":0.18099999999999999,"15min":0.18099999999999999},"max":{"1min":10.439,"5min":147.38900000000001,"15min":147.38900000000001},"last":0.20399999999999999},{"interface":"front","average":{"1min":1.1899999999999999,"5min":3.5859999999999999,"15min":1.6399999999999999},"min":{"1min":0.252,"5min":0.223,"15min":0.223},"max":{"1min":5.7279999999999998,"5min":149.989,"15min":149.989},"last":0.27300000000000002}]},{"osd":1,"last update":"Sat Sep 23 21:18:18 2023","interfaces":[{"interface":"back","average":{"1min":2.2029999999999998,"5min":1.5940000000000001,"15min":2.1160000000000001},"min":{"1min":0.27000000000000002,"5min":0.218,"15min":0.20000000000000001},"max":{"1min":6.1150000000000002,"5min":7.2759999999999998,"15min":22.577999999999999},"last":0.313},{"interface":"front","average":{"1min":1.827,"5min":1.2969999999999999,"15min":1.984},"min":{"1min":0.26100000000000001,"5min":0.249,"15min":0.216},"max":{"1min":6.1219999999999999,"5min":7.6829999999999998,"15min":21.617999999999999},"last":0.29399999999999998}]},{"osd":2,"last update":"Sat Sep 23 21:18:02 2023","interfaces":[{"interface":"back","average":{"1min":1.4750000000000001,"5min":1.958,"15min":2.004},"min":{"1min":0.27400000000000002,"5min":0.221,"15min":0.20899999999999999},"max":{"1min":11.321,"5min":56.317999999999998,"15min":56.317999999999998},"last":0.38400000000000001},{"interface":"front","average":{"1min":1.9910000000000001,"5min":2.4780000000000002,"15min":2.355},"min":{"1min":0.23400000000000001,"5min":0.23400000000000001,"15min":0.216},"max":{"1min":11.371,"5min":55.988,"15min":55.988},"last":0.39700000000000002}]},{"osd":4,"last update":"Sat Sep 23 21:18:02 2023","interfaces":[{"interface":"back","average":{"1min":0.72899999999999998,"5min":1.2010000000000001,"15min":1.5509999999999999},"min":{"1min":0.29199999999999998,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":2.5409999999999999,"5min":10.807,"15min":21.478999999999999},"last":0.47999999999999998},{"interface":"front","average":{"1min":1.091,"5min":1.3180000000000001,"15min":1.7829999999999999},"min":{"1min":0.35899999999999999,"5min":0.311,"15min":0.28499999999999998},"max":{"1min":3.129,"5min":7.3140000000000001,"15min":22.402000000000001},"last":0.43099999999999999}]},{"osd":5,"last update":"Sat Sep 23 21:18:02 2023","interfaces":[{"interface":"back","average":{"1min":0.83199999999999996,"5min":1.3160000000000001,"15min":1.6930000000000001},"min":{"1min":0.318,"5min":0.313,"15min":0.311},"max":{"1min":4.1379999999999999,"5min":10.882999999999999,"15min":21.562000000000001},"last":0.52600000000000002},{"interface":"front","average":{"1min":1.571,"5min":1.234,"15min":1.8},"min":{"1min":0.42699999999999999,"5min":0.34300000000000003,"15min":0.31},"max":{"1min":5.7300000000000004,"5min":5.7300000000000004,"15min":14.029},"last":0.438}]},{"osd":6,"last update":"Sat Sep 23 21:18:18 2023","interfaces":[{"interface":"back","average":{"1min":1.6180000000000001,"5min":1.464,"15min":1.839},"min":{"1min":0.39800000000000002,"5min":0.35899999999999999,"15min":0.33900000000000002},"max":{"1min":4.9749999999999996,"5min":7.3410000000000002,"15min":22.446999999999999},"last":0.45600000000000002},{"interface":"front","average":{"1min":1.657,"5min":1.3480000000000001,"15min":1.8280000000000001},"min":{"1min":0.373,"5min":0.373,"15min":0.373},"max":{"1min":7.5949999999999998,"5min":9.3889999999999993,"15min":12.375999999999999},"last":0.53800000000000003}]},{"osd":7,"last update":"Sat Sep 23 21:18:02 2023","interfaces":[{"interface":"back","average":{"1min":1.0249999999999999,"5min":1.48,"15min":1.847},"min":{"1min":0.41299999999999998,"5min":0.41199999999999998,"15min":0.33900000000000002},"max":{"1min":4.242,"5min":10.914999999999999,"15min":21.518999999999998},"last":0.55400000000000005},{"interface":"front","average":{"1min":1.5409999999999999,"5min":1.528,"15min":1.9850000000000001},"min":{"1min":0.51800000000000002,"5min":0.42499999999999999,"15min":0.40300000000000002},"max":{"1min":4.6920000000000002,"5min":8.2590000000000003,"15min":12.624000000000001},"last":0.48999999999999999}]}]},{"osd":4,"up_from":31,"seq":133143986644,"num_pgs":62,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1065868,"kb_used_data":497732,"kb_used_omap":0,"kb_used_meta":568128,"kb_avail":92679284,"statfs":{"total":95995035648,"available":94903586816,"internally_reserved":0,"allocated":509677568,"data_stored":507947359,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":581763072},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:18:24 2023","interfaces":[{"interface":"back","average":{"1min":2.2690000000000001,"5min":1.673,"15min":1.157},"min":{"1min":0.245,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":20.515000000000001,"5min":28.683,"15min":28.683},"last":0.245},{"interface":"front","average":{"1min":3.085,"5min":1.8009999999999999,"15min":1.3560000000000001},"min":{"1min":0.249,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":20.792000000000002,"5min":29.155000000000001,"15min":29.155000000000001},"last":7.6020000000000003}]},{"osd":1,"last update":"Sat Sep 23 21:18:24 2023","interfaces":[{"interface":"back","average":{"1min":1.7330000000000001,"5min":1.2829999999999999,"15min":2.1070000000000002},"min":{"1min":0.28499999999999998,"5min":0.26900000000000002,"15min":0.221},"max":{"1min":8.1020000000000003,"5min":8.1020000000000003,"15min":37.960000000000001},"last":2.5790000000000002},{"interface":"front","average":{"1min":1.55,"5min":1.399,"15min":2.0550000000000002},"min":{"1min":0.33100000000000002,"5min":0.28499999999999998,"15min":0.27000000000000002},"max":{"1min":8.1509999999999998,"5min":13.983000000000001,"15min":37.933},"last":2.5630000000000002}]},{"osd":2,"last update":"Sat Sep 23 21:18:24 2023","interfaces":[{"interface":"back","average":{"1min":2.177,"5min":1.8500000000000001,"15min":1.754},"min":{"1min":0.28399999999999997,"5min":0.26400000000000001,"15min":0.247},"max":{"1min":9.8399999999999999,"5min":17.602,"15min":17.602},"last":0.34399999999999997},{"interface":"front","average":{"1min":1.8839999999999999,"5min":1.7849999999999999,"15min":1.831},"min":{"1min":0.35399999999999998,"5min":0.311,"15min":0.27800000000000002},"max":{"1min":5.5250000000000004,"5min":14.339,"15min":14.417999999999999},"last":3.4910000000000001}]},{"osd":3,"last update":"Sat Sep 23 21:17:52 2023","interfaces":[{"interface":"back","average":{"1min":5.2910000000000004,"5min":2.1000000000000001,"15min":2.1299999999999999},"min":{"1min":0.312,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":67.105999999999995,"5min":67.105999999999995,"15min":67.105999999999995},"last":3.0209999999999999},{"interface":"front","average":{"1min":4.9290000000000003,"5min":2.1930000000000001,"15min":2.056},"min":{"1min":0.33600000000000002,"5min":0.32200000000000001,"15min":0.30199999999999999},"max":{"1min":67.046999999999997,"5min":67.046999999999997,"15min":67.046999999999997},"last":3.0419999999999998}]},{"osd":5,"last update":"Sat Sep 23 21:18:13 2023","interfaces":[{"interface":"back","average":{"1min":0.89300000000000002,"5min":1.1080000000000001,"15min":1.123},"min":{"1min":0.254,"5min":0.254,"15min":0.222},"max":{"1min":2.923,"5min":14.398999999999999,"15min":14.398999999999999},"last":0.26900000000000002},{"interface":"front","average":{"1min":1.0349999999999999,"5min":0.98799999999999999,"15min":1.1080000000000001},"min":{"1min":0.28100000000000003,"5min":0.25800000000000001,"15min":0.255},"max":{"1min":3.6000000000000001,"5min":8.016,"15min":9.8979999999999997},"last":0.40300000000000002}]},{"osd":6,"last update":"Sat Sep 23 21:18:24 2023","interfaces":[{"interface":"back","average":{"1min":1.077,"5min":0.96599999999999997,"15min":1.2649999999999999},"min":{"1min":0.308,"5min":0.27200000000000002,"15min":0.26800000000000002},"max":{"1min":3.2050000000000001,"5min":8.7420000000000009,"15min":24.452999999999999},"last":0.32400000000000001},{"interface":"front","average":{"1min":0.83799999999999997,"5min":1.1499999999999999,"15min":1.0820000000000001},"min":{"1min":0.38,"5min":0.36399999999999999,"15min":0.32500000000000001},"max":{"1min":2.5409999999999999,"5min":14.374000000000001,"15min":14.387},"last":0.443}]},{"osd":7,"last update":"Sat Sep 23 21:17:52 2023","interfaces":[{"interface":"back","average":{"1min":1.091,"5min":1.194,"15min":1.1919999999999999},"min":{"1min":0.33200000000000002,"5min":0.32000000000000001,"15min":0.30099999999999999},"max":{"1min":5.7789999999999999,"5min":14.422000000000001,"15min":16.890999999999998},"last":0.38700000000000001},{"interface":"front","average":{"1min":0.73399999999999999,"5min":1.034,"15min":1.2230000000000001},"min":{"1min":0.29199999999999998,"5min":0.29199999999999998,"15min":0.27400000000000002},"max":{"1min":3.681,"5min":7.7229999999999999,"15min":11.170999999999999},"last":0.36299999999999999}]}]},{"osd":5,"up_from":39,"seq":167503725010,"num_pgs":61,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1126144,"kb_used_data":534328,"kb_used_omap":0,"kb_used_meta":591808,"kb_avail":92619008,"statfs":{"total":95995035648,"available":94841864192,"internally_reserved":0,"allocated":547151872,"data_stored":545430239,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":606011392},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:18:16 2023","interfaces":[{"interface":"back","average":{"1min":1.452,"5min":1.234,"15min":0.88800000000000001},"min":{"1min":0.22500000000000001,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":5.9829999999999997,"5min":8.673,"15min":8.673},"last":0.45300000000000001},{"interface":"front","average":{"1min":2.5219999999999998,"5min":1.548,"15min":0.96199999999999997},"min":{"1min":0.215,"5min":0.215,"15min":0.215},"max":{"1min":12.259,"5min":12.259,"15min":12.259},"last":0.29999999999999999}]},{"osd":1,"last update":"Sat Sep 23 21:18:16 2023","interfaces":[{"interface":"back","average":{"1min":1.4490000000000001,"5min":1.254,"15min":1.625},"min":{"1min":0.32200000000000001,"5min":0.248,"15min":0.23499999999999999},"max":{"1min":4.3220000000000001,"5min":7.5839999999999996,"15min":25.239999999999998},"last":0.36399999999999999},{"interface":"front","average":{"1min":1.3700000000000001,"5min":1.304,"15min":1.7070000000000001},"min":{"1min":0.36899999999999999,"5min":0.28299999999999997,"15min":0.252},"max":{"1min":4.9500000000000002,"5min":11.116,"15min":25.253},"last":0.34399999999999997}]},{"osd":2,"last update":"Sat Sep 23 21:17:32 2023","interfaces":[{"interface":"back","average":{"1min":1.615,"5min":1.506,"15min":1.6439999999999999},"min":{"1min":0.27300000000000002,"5min":0.27300000000000002,"15min":0.25800000000000001},"max":{"1min":7.3550000000000004,"5min":9.6560000000000006,"15min":12.634},"last":0.32800000000000001},{"interface":"front","average":{"1min":1.9410000000000001,"5min":1.327,"15min":1.6579999999999999},"min":{"1min":0.314,"5min":0.27600000000000002,"15min":0.25700000000000001},"max":{"1min":9.1630000000000003,"5min":9.1630000000000003,"15min":9.7650000000000006},"last":1.2629999999999999}]},{"osd":3,"last update":"Sat Sep 23 21:17:52 2023","interfaces":[{"interface":"back","average":{"1min":1.4259999999999999,"5min":1.4570000000000001,"15min":1.931},"min":{"1min":0.314,"5min":0.314,"15min":0.29299999999999998},"max":{"1min":3.6030000000000002,"5min":6.5940000000000003,"15min":25.503},"last":1.3759999999999999},{"interface":"front","average":{"1min":1.7310000000000001,"5min":1.748,"15min":2.157},"min":{"1min":0.311,"5min":0.311,"15min":0.30599999999999999},"max":{"1min":9.0619999999999994,"5min":12.382,"15min":25.466000000000001},"last":0.41599999999999998}]},{"osd":4,"last update":"Sat Sep 23 21:18:08 2023","interfaces":[{"interface":"back","average":{"1min":0.82399999999999995,"5min":0.91400000000000003,"15min":1.155},"min":{"1min":0.29999999999999999,"5min":0.253,"15min":0.253},"max":{"1min":5.0279999999999996,"5min":11.156000000000001,"15min":12.776},"last":0.51600000000000001},{"interface":"front","average":{"1min":0.73399999999999999,"5min":1.008,"15min":1.1419999999999999},"min":{"1min":0.28599999999999998,"5min":0.246,"15min":0.246},"max":{"1min":3.415,"5min":9.7119999999999997,"15min":11.973000000000001},"last":0.46999999999999997}]},{"osd":6,"last update":"Sat Sep 23 21:18:16 2023","interfaces":[{"interface":"back","average":{"1min":1.0649999999999999,"5min":0.99299999999999999,"15min":1.087},"min":{"1min":0.32200000000000001,"5min":0.28000000000000003,"15min":0.25900000000000001},"max":{"1min":5.8520000000000003,"5min":5.8520000000000003,"15min":8.0909999999999993},"last":0.57399999999999995},{"interface":"front","average":{"1min":1.095,"5min":1.0700000000000001,"15min":1.2330000000000001},"min":{"1min":0.39700000000000002,"5min":0.34300000000000003,"15min":0.317},"max":{"1min":7.1470000000000002,"5min":10.999000000000001,"15min":10.999000000000001},"last":0.59399999999999997}]},{"osd":7,"last update":"Sat Sep 23 21:17:52 2023","interfaces":[{"interface":"back","average":{"1min":1.109,"5min":1.012,"15min":1.819},"min":{"1min":0.36199999999999999,"5min":0.32400000000000001,"15min":0.309},"max":{"1min":4.2629999999999999,"5min":5.5659999999999998,"15min":189.405},"last":0.40400000000000003},{"interface":"front","average":{"1min":1.0649999999999999,"5min":1.1719999999999999,"15min":1.9670000000000001},"min":{"1min":0.39900000000000002,"5min":0.35199999999999998,"15min":0.32500000000000001},"max":{"1min":2.7879999999999998,"5min":11.183999999999999,"15min":189.685},"last":0.38100000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8488,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1259,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5322,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":4838,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":808583168,"data_stored":808583168,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":821694464,"data_stored":821690392,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":890306560,"data_stored":890306560,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":502861824,"data_stored":502857752,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":535105536,"data_stored":535101464,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":662634496,"data_stored":662634496,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T21:18:33.631 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:18:33.694 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:18:33.962 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:18:33 smithi026 ceph-mon[175151]: 11.18 scrub starts 2023-09-23T21:18:33.962 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:18:33 smithi026 ceph-mon[175151]: 11.18 scrub ok 2023-09-23T21:18:33.962 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:18:33 smithi026 ceph-mon[175151]: failed to encode map e612 with expected crc 2023-09-23T21:18:33.962 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:18:33 smithi026 ceph-mon[175151]: failed to encode map e612 with expected crc 2023-09-23T21:18:33.963 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:18:33 smithi026 ceph-mon[175151]: failed to encode map e612 with expected crc 2023-09-23T21:18:33.963 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:18:33 smithi026 ceph-mon[175151]: failed to encode map e612 with expected crc 2023-09-23T21:18:33.963 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:18:33 smithi026 ceph-mon[175151]: pgmap v1534: 129 pgs: 1 remapped+peering, 1 active+clean+remapped, 1 peering, 126 active+clean; 1.2 GiB data, 8.0 GiB used, 528 GiB / 536 GiB avail 2023-09-23T21:18:33.963 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:18:33 smithi026 ceph-mon[175151]: 11.5 scrub starts 2023-09-23T21:18:33.963 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:18:33 smithi026 ceph-mon[175151]: 11.17 scrub starts 2023-09-23T21:18:33.964 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:18:33 smithi026 ceph-mon[179688]: 11.18 scrub starts 2023-09-23T21:18:33.965 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:18:33 smithi026 ceph-mon[179688]: 11.18 scrub ok 2023-09-23T21:18:33.965 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:18:33 smithi026 ceph-mon[179688]: failed to encode map e612 with expected crc 2023-09-23T21:18:33.965 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:18:33 smithi026 ceph-mon[179688]: failed to encode map e612 with expected crc 2023-09-23T21:18:33.965 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:18:33 smithi026 ceph-mon[179688]: failed to encode map e612 with expected crc 2023-09-23T21:18:33.965 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:18:33 smithi026 ceph-mon[179688]: failed to encode map e612 with expected crc 2023-09-23T21:18:33.965 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:18:33 smithi026 ceph-mon[179688]: pgmap v1534: 129 pgs: 1 remapped+peering, 1 active+clean+remapped, 1 peering, 126 active+clean; 1.2 GiB data, 8.0 GiB used, 528 GiB / 536 GiB avail 2023-09-23T21:18:33.965 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:18:33 smithi026 ceph-mon[179688]: 11.5 scrub starts 2023-09-23T21:18:33.965 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:18:33 smithi026 ceph-mon[179688]: 11.17 scrub starts 2023-09-23T21:18:34.140 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:18:34.141 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:18:34.142 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:18:34.242 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:18:34.292 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:18:34.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:18:33 smithi070 ceph-mon[168282]: 11.18 scrub starts 2023-09-23T21:18:34.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:18:33 smithi070 ceph-mon[168282]: 11.18 scrub ok 2023-09-23T21:18:34.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:18:33 smithi070 ceph-mon[168282]: failed to encode map e612 with expected crc 2023-09-23T21:18:34.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:18:33 smithi070 ceph-mon[168282]: failed to encode map e612 with expected crc 2023-09-23T21:18:34.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:18:33 smithi070 ceph-mon[168282]: failed to encode map e612 with expected crc 2023-09-23T21:18:34.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:18:33 smithi070 ceph-mon[168282]: failed to encode map e612 with expected crc 2023-09-23T21:18:34.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:18:33 smithi070 ceph-mon[168282]: pgmap v1534: 129 pgs: 1 remapped+peering, 1 active+clean+remapped, 1 peering, 126 active+clean; 1.2 GiB data, 8.0 GiB used, 528 GiB / 536 GiB avail 2023-09-23T21:18:34.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:18:33 smithi070 ceph-mon[168282]: 11.5 scrub starts 2023-09-23T21:18:34.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:18:33 smithi070 ceph-mon[168282]: 11.17 scrub starts 2023-09-23T21:18:34.502 INFO:teuthology.orchestra.run.smithi026.stdout:ERROR: (22) Invalid argument 2023-09-23T21:18:34.502 INFO:teuthology.orchestra.run.smithi026.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T21:18:34.607 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:18:34.610 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:18:34.611 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:18:34.712 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:18:34.743 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:18:34.996 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T21:18:35.003 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:18:35.003 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:18:35.010 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_blocked_ops 2023-09-23T21:18:35.104 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:18:35.183 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:18:34 smithi026 ceph-mon[175151]: 11.17 scrub ok 2023-09-23T21:18:35.184 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:18:34 smithi026 ceph-mon[175151]: failed to encode map e613 with expected crc 2023-09-23T21:18:35.184 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:18:34 smithi026 ceph-mon[175151]: failed to encode map e613 with expected crc 2023-09-23T21:18:35.184 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:18:34 smithi026 ceph-mon[175151]: failed to encode map e613 with expected crc 2023-09-23T21:18:35.184 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:18:34 smithi026 ceph-mon[175151]: osdmap e613: 8 total, 8 up, 6 in 2023-09-23T21:18:35.184 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:18:34 smithi026 ceph-mon[175151]: failed to encode map e613 with expected crc 2023-09-23T21:18:35.184 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:18:34 smithi026 ceph-mon[175151]: 11.2 scrub starts 2023-09-23T21:18:35.184 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:18:34 smithi026 ceph-mon[175151]: 11.2 scrub ok 2023-09-23T21:18:35.184 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:18:34 smithi026 ceph-mon[179688]: 11.17 scrub ok 2023-09-23T21:18:35.184 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:18:34 smithi026 ceph-mon[179688]: failed to encode map e613 with expected crc 2023-09-23T21:18:35.184 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:18:34 smithi026 ceph-mon[179688]: failed to encode map e613 with expected crc 2023-09-23T21:18:35.185 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:18:34 smithi026 ceph-mon[179688]: failed to encode map e613 with expected crc 2023-09-23T21:18:35.185 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:18:34 smithi026 ceph-mon[179688]: osdmap e613: 8 total, 8 up, 6 in 2023-09-23T21:18:35.185 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:18:34 smithi026 ceph-mon[179688]: failed to encode map e613 with expected crc 2023-09-23T21:18:35.185 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:18:34 smithi026 ceph-mon[179688]: 11.2 scrub starts 2023-09-23T21:18:35.185 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:18:34 smithi026 ceph-mon[179688]: 11.2 scrub ok 2023-09-23T21:18:35.210 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:18:35.305 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:18:35.325 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:18:35.325 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:18:35.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:18:34 smithi070 ceph-mon[168282]: 11.17 scrub ok 2023-09-23T21:18:35.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:18:34 smithi070 ceph-mon[168282]: failed to encode map e613 with expected crc 2023-09-23T21:18:35.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:18:34 smithi070 ceph-mon[168282]: failed to encode map e613 with expected crc 2023-09-23T21:18:35.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:18:34 smithi070 ceph-mon[168282]: failed to encode map e613 with expected crc 2023-09-23T21:18:35.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:18:34 smithi070 ceph-mon[168282]: osdmap e613: 8 total, 8 up, 6 in 2023-09-23T21:18:35.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:18:34 smithi070 ceph-mon[168282]: failed to encode map e613 with expected crc 2023-09-23T21:18:35.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:18:34 smithi070 ceph-mon[168282]: 11.2 scrub starts 2023-09-23T21:18:35.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:18:34 smithi070 ceph-mon[168282]: 11.2 scrub ok 2023-09-23T21:18:35.427 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:18:35.447 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0: {} 2023-09-23T21:18:35.485 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1: {} 2023-09-23T21:18:35.535 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2: {} 2023-09-23T21:18:35.573 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3: {} 2023-09-23T21:18:35.594 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4: {} 2023-09-23T21:18:35.640 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:18:35.645 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T21:18:35.682 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:18:35.683 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6: {} 2023-09-23T21:18:35.685 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:18:35.685 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:18:35.732 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T21:18:35.761 INFO:teuthology.orchestra.run.smithi026.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T21:18:35.762 INFO:teuthology.orchestra.run.smithi026.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T21:18:35.762 INFO:teuthology.orchestra.run.smithi026.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T21:18:35.762 INFO:teuthology.orchestra.run.smithi026.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T21:18:35.762 INFO:teuthology.orchestra.run.smithi026.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T21:18:35.762 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T21:18:35.762 INFO:teuthology.orchestra.run.smithi026.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T21:18:35.762 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T21:18:35.786 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:18:35.817 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:18:35.849 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:18:35.852 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:18:35.856 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:18:35.857 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:18:35.957 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:18:36.027 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:18:36.131 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:18:35 smithi026 ceph-mon[175151]: failed to encode map e614 with expected crc 2023-09-23T21:18:36.131 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:18:35 smithi026 ceph-mon[175151]: failed to encode map e614 with expected crc 2023-09-23T21:18:36.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:18:35 smithi026 ceph-mon[175151]: failed to encode map e614 with expected crc 2023-09-23T21:18:36.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:18:35 smithi026 ceph-mon[175151]: osdmap e614: 8 total, 8 up, 6 in 2023-09-23T21:18:36.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:18:35 smithi026 ceph-mon[175151]: failed to encode map e614 with expected crc 2023-09-23T21:18:36.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:18:35 smithi026 ceph-mon[175151]: pgmap v1537: 129 pgs: 1 remapped+peering, 1 active+clean+remapped, 1 peering, 126 active+clean; 1.1 GiB data, 7.8 GiB used, 529 GiB / 536 GiB avail 2023-09-23T21:18:36.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:18:35 smithi026 ceph-mon[175151]: 11.1 scrub starts 2023-09-23T21:18:36.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:18:35 smithi026 ceph-mon[175151]: 11.5 scrub starts 2023-09-23T21:18:36.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:18:35 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/1620398232' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:18:36.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:18:35 smithi026 ceph-mon[175151]: 11.1 scrub ok 2023-09-23T21:18:36.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:18:35 smithi026 ceph-mon[179688]: failed to encode map e614 with expected crc 2023-09-23T21:18:36.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:18:35 smithi026 ceph-mon[179688]: failed to encode map e614 with expected crc 2023-09-23T21:18:36.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:18:35 smithi026 ceph-mon[179688]: failed to encode map e614 with expected crc 2023-09-23T21:18:36.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:18:35 smithi026 ceph-mon[179688]: osdmap e614: 8 total, 8 up, 6 in 2023-09-23T21:18:36.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:18:35 smithi026 ceph-mon[179688]: failed to encode map e614 with expected crc 2023-09-23T21:18:36.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:18:35 smithi026 ceph-mon[179688]: pgmap v1537: 129 pgs: 1 remapped+peering, 1 active+clean+remapped, 1 peering, 126 active+clean; 1.1 GiB data, 7.8 GiB used, 529 GiB / 536 GiB avail 2023-09-23T21:18:36.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:18:35 smithi026 ceph-mon[179688]: 11.1 scrub starts 2023-09-23T21:18:36.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:18:35 smithi026 ceph-mon[179688]: 11.5 scrub starts 2023-09-23T21:18:36.134 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:18:35 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/1620398232' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:18:36.134 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:18:35 smithi026 ceph-mon[179688]: 11.1 scrub ok 2023-09-23T21:18:36.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:18:35 smithi070 ceph-mon[168282]: failed to encode map e614 with expected crc 2023-09-23T21:18:36.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:18:35 smithi070 ceph-mon[168282]: failed to encode map e614 with expected crc 2023-09-23T21:18:36.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:18:35 smithi070 ceph-mon[168282]: failed to encode map e614 with expected crc 2023-09-23T21:18:36.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:18:35 smithi070 ceph-mon[168282]: osdmap e614: 8 total, 8 up, 6 in 2023-09-23T21:18:36.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:18:35 smithi070 ceph-mon[168282]: failed to encode map e614 with expected crc 2023-09-23T21:18:36.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:18:35 smithi070 ceph-mon[168282]: pgmap v1537: 129 pgs: 1 remapped+peering, 1 active+clean+remapped, 1 peering, 126 active+clean; 1.1 GiB data, 7.8 GiB used, 529 GiB / 536 GiB avail 2023-09-23T21:18:36.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:18:35 smithi070 ceph-mon[168282]: 11.1 scrub starts 2023-09-23T21:18:36.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:18:35 smithi070 ceph-mon[168282]: 11.5 scrub starts 2023-09-23T21:18:36.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:18:35 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/1620398232' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:18:36.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:18:35 smithi070 ceph-mon[168282]: 11.1 scrub ok 2023-09-23T21:18:36.614 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T21:18:36.823 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T21:18:36.837 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:18:36.837 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:18:36.938 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:18:36.955 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:18:36 smithi026 ceph-mon[175151]: 1.15 scrub starts 2023-09-23T21:18:36.955 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:18:36 smithi026 ceph-mon[175151]: 1.15 scrub ok 2023-09-23T21:18:36.955 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:18:36 smithi026 ceph-mon[175151]: 11.11 scrub starts 2023-09-23T21:18:36.955 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:18:36 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/2454927323' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:18:36.955 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:18:36 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:18:36.956 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:18:36 smithi026 ceph-mon[175151]: 11.11 scrub ok 2023-09-23T21:18:36.956 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:18:36 smithi026 ceph-mon[175151]: 11.5 scrub starts 2023-09-23T21:18:36.976 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:18:36.976 INFO:teuthology.orchestra.run.smithi026.stderr:nodeep-scrub is unset 2023-09-23T21:18:37.026 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:18:37.029 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:18:37.029 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:18:37.130 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:18:37.187 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:18:37.240 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:18:37.244 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:18:36 smithi070 ceph-mon[168282]: 1.15 scrub starts 2023-09-23T21:18:37.245 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:18:36 smithi070 ceph-mon[168282]: 1.15 scrub ok 2023-09-23T21:18:37.245 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:18:36 smithi070 ceph-mon[168282]: 11.11 scrub starts 2023-09-23T21:18:37.245 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:18:36 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/2454927323' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:18:37.245 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:18:36 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:18:37.245 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:18:36 smithi070 ceph-mon[168282]: 11.11 scrub ok 2023-09-23T21:18:37.245 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:18:36 smithi070 ceph-mon[168282]: 11.5 scrub starts 2023-09-23T21:18:37.246 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:18:37.246 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:18:37.347 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:18:37.381 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:18:36 smithi026 ceph-mon[179688]: 1.15 scrub starts 2023-09-23T21:18:37.381 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:18:36 smithi026 ceph-mon[179688]: 1.15 scrub ok 2023-09-23T21:18:37.381 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:18:36 smithi026 ceph-mon[179688]: 11.11 scrub starts 2023-09-23T21:18:37.381 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:18:36 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/2454927323' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:18:37.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:18:36 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:18:37.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:18:36 smithi026 ceph-mon[179688]: 11.11 scrub ok 2023-09-23T21:18:37.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:18:36 smithi026 ceph-mon[179688]: 11.5 scrub starts 2023-09-23T21:18:37.386 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:18:37.824 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:18:37.834 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:18:37.835 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:18:37.936 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:18:37.997 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:18:38.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:18:37 smithi026 ceph-mon[179688]: 11.5 scrub ok 2023-09-23T21:18:38.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:18:37 smithi026 ceph-mon[179688]: 11.10 scrub starts 2023-09-23T21:18:38.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:18:37 smithi026 ceph-mon[179688]: 11.10 scrub ok 2023-09-23T21:18:38.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:18:37 smithi026 ceph-mon[179688]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T21:18:38.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:18:37 smithi026 ceph-mon[179688]: Cluster is now healthy 2023-09-23T21:18:38.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:18:37 smithi026 ceph-mon[179688]: failed to encode map e615 with expected crc 2023-09-23T21:18:38.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:18:37 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T21:18:38.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:18:37 smithi026 ceph-mon[179688]: osdmap e615: 8 total, 8 up, 6 in 2023-09-23T21:18:38.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:18:37 smithi026 ceph-mon[179688]: failed to encode map e615 with expected crc 2023-09-23T21:18:38.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:18:37 smithi026 ceph-mon[179688]: failed to encode map e615 with expected crc 2023-09-23T21:18:38.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:18:37 smithi026 ceph-mon[179688]: failed to encode map e615 with expected crc 2023-09-23T21:18:38.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:18:37 smithi026 ceph-mon[179688]: pgmap v1539: 129 pgs: 129 active+clean; 1.0 GiB data, 7.6 GiB used, 529 GiB / 536 GiB avail 2023-09-23T21:18:38.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:18:37 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "20"}]: dispatch 2023-09-23T21:18:38.137 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:18:37 smithi026 ceph-mon[175151]: 11.5 scrub ok 2023-09-23T21:18:38.137 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:18:37 smithi026 ceph-mon[175151]: 11.10 scrub starts 2023-09-23T21:18:38.137 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:18:37 smithi026 ceph-mon[175151]: 11.10 scrub ok 2023-09-23T21:18:38.137 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:18:37 smithi026 ceph-mon[175151]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T21:18:38.137 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:18:37 smithi026 ceph-mon[175151]: Cluster is now healthy 2023-09-23T21:18:38.138 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:18:37 smithi026 ceph-mon[175151]: failed to encode map e615 with expected crc 2023-09-23T21:18:38.138 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:18:37 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T21:18:38.138 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:18:37 smithi026 ceph-mon[175151]: osdmap e615: 8 total, 8 up, 6 in 2023-09-23T21:18:38.138 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:18:37 smithi026 ceph-mon[175151]: failed to encode map e615 with expected crc 2023-09-23T21:18:38.138 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:18:37 smithi026 ceph-mon[175151]: failed to encode map e615 with expected crc 2023-09-23T21:18:38.138 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:18:37 smithi026 ceph-mon[175151]: failed to encode map e615 with expected crc 2023-09-23T21:18:38.138 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:18:37 smithi026 ceph-mon[175151]: pgmap v1539: 129 pgs: 129 active+clean; 1.0 GiB data, 7.6 GiB used, 529 GiB / 536 GiB avail 2023-09-23T21:18:38.138 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:18:37 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "20"}]: dispatch 2023-09-23T21:18:38.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:18:37 smithi070 ceph-mon[168282]: 11.5 scrub ok 2023-09-23T21:18:38.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:18:37 smithi070 ceph-mon[168282]: 11.10 scrub starts 2023-09-23T21:18:38.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:18:37 smithi070 ceph-mon[168282]: 11.10 scrub ok 2023-09-23T21:18:38.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:18:37 smithi070 ceph-mon[168282]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T21:18:38.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:18:37 smithi070 ceph-mon[168282]: Cluster is now healthy 2023-09-23T21:18:38.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:18:37 smithi070 ceph-mon[168282]: failed to encode map e615 with expected crc 2023-09-23T21:18:38.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:18:37 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T21:18:38.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:18:37 smithi070 ceph-mon[168282]: osdmap e615: 8 total, 8 up, 6 in 2023-09-23T21:18:38.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:18:37 smithi070 ceph-mon[168282]: failed to encode map e615 with expected crc 2023-09-23T21:18:38.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:18:37 smithi070 ceph-mon[168282]: failed to encode map e615 with expected crc 2023-09-23T21:18:38.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:18:37 smithi070 ceph-mon[168282]: failed to encode map e615 with expected crc 2023-09-23T21:18:38.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:18:37 smithi070 ceph-mon[168282]: pgmap v1539: 129 pgs: 129 active+clean; 1.0 GiB data, 7.6 GiB used, 529 GiB / 536 GiB avail 2023-09-23T21:18:38.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:18:37 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "20"}]: dispatch 2023-09-23T21:18:38.824 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:18:38.827 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:18:38.827 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:18:38.879 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T21:18:38.929 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:18:38.955 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:18:38.992 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:18:38.994 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:18:38.995 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:18:39.089 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:18:38 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "20"}]': finished 2023-09-23T21:18:39.089 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:18:38 smithi026 ceph-mon[179688]: osdmap e616: 8 total, 8 up, 6 in 2023-09-23T21:18:39.089 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:18:38 smithi026 ceph-mon[179688]: failed to encode map e616 with expected crc 2023-09-23T21:18:39.089 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:18:38 smithi026 ceph-mon[179688]: failed to encode map e616 with expected crc 2023-09-23T21:18:39.089 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:18:38 smithi026 ceph-mon[179688]: failed to encode map e616 with expected crc 2023-09-23T21:18:39.089 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:18:38 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "20"}]': finished 2023-09-23T21:18:39.090 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:18:38 smithi026 ceph-mon[175151]: osdmap e616: 8 total, 8 up, 6 in 2023-09-23T21:18:39.090 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:18:38 smithi026 ceph-mon[175151]: failed to encode map e616 with expected crc 2023-09-23T21:18:39.090 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:18:38 smithi026 ceph-mon[175151]: failed to encode map e616 with expected crc 2023-09-23T21:18:39.090 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:18:38 smithi026 ceph-mon[175151]: failed to encode map e616 with expected crc 2023-09-23T21:18:39.095 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:18:39.132 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:18:39.307 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:18:39.310 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:18:39.310 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:18:39.323 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:18:39.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:18:38 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "20"}]': finished 2023-09-23T21:18:39.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:18:38 smithi070 ceph-mon[168282]: osdmap e616: 8 total, 8 up, 6 in 2023-09-23T21:18:39.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:18:38 smithi070 ceph-mon[168282]: failed to encode map e616 with expected crc 2023-09-23T21:18:39.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:18:38 smithi070 ceph-mon[168282]: failed to encode map e616 with expected crc 2023-09-23T21:18:39.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:18:38 smithi070 ceph-mon[168282]: failed to encode map e616 with expected crc 2023-09-23T21:18:39.412 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:18:39.437 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:18:39.471 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:18:39.474 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:18:39.474 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:18:39.575 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:18:39.602 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:18:39.636 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:18:39.639 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:18:39.639 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:18:39.740 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:18:39.766 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:18:39.800 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:18:39.802 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:18:39.802 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:18:39.822 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 21:18:39 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[164029]: ::ffff:172.21.15.70 - - [23/Sep/2023:21:18:39] "GET /metrics HTTP/1.1" 200 36375 "" "Prometheus/2.43.0" 2023-09-23T21:18:39.903 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:18:39.930 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:18:39.964 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:18:39.967 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:18:39.967 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:18:40.068 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:18:40.089 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:18:39 smithi026 ceph-mon[175151]: failed to encode map e616 with expected crc 2023-09-23T21:18:40.089 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:18:39 smithi026 ceph-mon[175151]: failed to encode map e617 with expected crc 2023-09-23T21:18:40.089 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:18:39 smithi026 ceph-mon[175151]: failed to encode map e617 with expected crc 2023-09-23T21:18:40.089 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:18:39 smithi026 ceph-mon[175151]: osdmap e617: 8 total, 8 up, 6 in 2023-09-23T21:18:40.089 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:18:39 smithi026 ceph-mon[175151]: 11.12 deep-scrub starts 2023-09-23T21:18:40.089 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:18:39 smithi026 ceph-mon[175151]: 11.19 deep-scrub starts 2023-09-23T21:18:40.089 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:18:39 smithi026 ceph-mon[175151]: pgmap v1542: 129 pgs: 129 active+clean; 977 MiB data, 7.4 GiB used, 529 GiB / 536 GiB avail 2023-09-23T21:18:40.089 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:18:39 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "21"}]: dispatch 2023-09-23T21:18:40.090 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:18:39 smithi026 ceph-mon[179688]: failed to encode map e616 with expected crc 2023-09-23T21:18:40.090 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:18:39 smithi026 ceph-mon[179688]: failed to encode map e617 with expected crc 2023-09-23T21:18:40.090 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:18:39 smithi026 ceph-mon[179688]: failed to encode map e617 with expected crc 2023-09-23T21:18:40.090 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:18:39 smithi026 ceph-mon[179688]: osdmap e617: 8 total, 8 up, 6 in 2023-09-23T21:18:40.090 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:18:39 smithi026 ceph-mon[179688]: 11.12 deep-scrub starts 2023-09-23T21:18:40.090 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:18:39 smithi026 ceph-mon[179688]: 11.19 deep-scrub starts 2023-09-23T21:18:40.090 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:18:39 smithi026 ceph-mon[179688]: pgmap v1542: 129 pgs: 129 active+clean; 977 MiB data, 7.4 GiB used, 529 GiB / 536 GiB avail 2023-09-23T21:18:40.090 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:18:39 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "21"}]: dispatch 2023-09-23T21:18:40.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:18:39 smithi070 ceph-mon[168282]: failed to encode map e616 with expected crc 2023-09-23T21:18:40.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:18:39 smithi070 ceph-mon[168282]: failed to encode map e617 with expected crc 2023-09-23T21:18:40.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:18:39 smithi070 ceph-mon[168282]: failed to encode map e617 with expected crc 2023-09-23T21:18:40.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:18:39 smithi070 ceph-mon[168282]: osdmap e617: 8 total, 8 up, 6 in 2023-09-23T21:18:40.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:18:39 smithi070 ceph-mon[168282]: 11.12 deep-scrub starts 2023-09-23T21:18:40.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:18:39 smithi070 ceph-mon[168282]: 11.19 deep-scrub starts 2023-09-23T21:18:40.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:18:39 smithi070 ceph-mon[168282]: pgmap v1542: 129 pgs: 129 active+clean; 977 MiB data, 7.4 GiB used, 529 GiB / 536 GiB avail 2023-09-23T21:18:40.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:18:39 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "21"}]: dispatch 2023-09-23T21:18:40.239 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:18:40.911 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T21:18:41.133 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:18:41.166 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T21:18:41.170 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:18:41.170 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:18:41.271 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:18:41.297 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:18:40 smithi026 ceph-mon[179688]: failed to encode map e618 with expected crc 2023-09-23T21:18:41.298 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:18:40 smithi026 ceph-mon[179688]: failed to encode map e618 with expected crc 2023-09-23T21:18:41.298 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:18:40 smithi026 ceph-mon[179688]: failed to encode map e618 with expected crc 2023-09-23T21:18:41.298 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:18:40 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "21"}]': finished 2023-09-23T21:18:41.298 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:18:40 smithi026 ceph-mon[179688]: osdmap e618: 8 total, 8 up, 6 in 2023-09-23T21:18:41.298 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:18:40 smithi026 ceph-mon[179688]: 11.13 starting backfill to osd.0 from (0'0,0'0] MAX to 615'7764 2023-09-23T21:18:41.298 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:18:40 smithi026 ceph-mon[179688]: 11.13 starting backfill to osd.4 from (0'0,0'0] MAX to 615'7764 2023-09-23T21:18:41.299 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:18:40 smithi026 ceph-mon[179688]: 11.12 deep-scrub ok 2023-09-23T21:18:41.299 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:18:40 smithi026 ceph-mon[179688]: osdmap e619: 8 total, 8 up, 6 in 2023-09-23T21:18:41.299 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:18:40 smithi026 ceph-mon[175151]: failed to encode map e618 with expected crc 2023-09-23T21:18:41.299 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:18:40 smithi026 ceph-mon[175151]: failed to encode map e618 with expected crc 2023-09-23T21:18:41.299 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:18:40 smithi026 ceph-mon[175151]: failed to encode map e618 with expected crc 2023-09-23T21:18:41.299 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:18:40 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "21"}]': finished 2023-09-23T21:18:41.300 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:18:40 smithi026 ceph-mon[175151]: osdmap e618: 8 total, 8 up, 6 in 2023-09-23T21:18:41.300 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:18:40 smithi026 ceph-mon[175151]: 11.13 starting backfill to osd.0 from (0'0,0'0] MAX to 615'7764 2023-09-23T21:18:41.300 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:18:40 smithi026 ceph-mon[175151]: 11.13 starting backfill to osd.4 from (0'0,0'0] MAX to 615'7764 2023-09-23T21:18:41.300 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:18:40 smithi026 ceph-mon[175151]: 11.12 deep-scrub ok 2023-09-23T21:18:41.300 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:18:40 smithi026 ceph-mon[175151]: osdmap e619: 8 total, 8 up, 6 in 2023-09-23T21:18:41.300 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:18:41.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:18:40 smithi070 ceph-mon[168282]: failed to encode map e618 with expected crc 2023-09-23T21:18:41.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:18:40 smithi070 ceph-mon[168282]: failed to encode map e618 with expected crc 2023-09-23T21:18:41.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:18:40 smithi070 ceph-mon[168282]: failed to encode map e618 with expected crc 2023-09-23T21:18:41.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:18:40 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "21"}]': finished 2023-09-23T21:18:41.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:18:40 smithi070 ceph-mon[168282]: osdmap e618: 8 total, 8 up, 6 in 2023-09-23T21:18:41.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:18:40 smithi070 ceph-mon[168282]: 11.13 starting backfill to osd.0 from (0'0,0'0] MAX to 615'7764 2023-09-23T21:18:41.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:18:40 smithi070 ceph-mon[168282]: 11.13 starting backfill to osd.4 from (0'0,0'0] MAX to 615'7764 2023-09-23T21:18:41.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:18:40 smithi070 ceph-mon[168282]: 11.12 deep-scrub ok 2023-09-23T21:18:41.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:18:40 smithi070 ceph-mon[168282]: osdmap e619: 8 total, 8 up, 6 in 2023-09-23T21:18:41.430 INFO:teuthology.orchestra.run.smithi026.stdout:{ 2023-09-23T21:18:41.430 INFO:teuthology.orchestra.run.smithi026.stdout: "ops": [], 2023-09-23T21:18:41.430 INFO:teuthology.orchestra.run.smithi026.stdout: "complaint_time": 30, 2023-09-23T21:18:41.430 INFO:teuthology.orchestra.run.smithi026.stdout: "num_blocked_ops": 0 2023-09-23T21:18:41.430 INFO:teuthology.orchestra.run.smithi026.stdout:} 2023-09-23T21:18:42.221 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:18:42.224 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:18:42.224 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:18:42.325 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:18:42.351 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:18:41 smithi070 ceph-mon[168282]: failed to encode map e619 with expected crc 2023-09-23T21:18:42.351 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:18:41 smithi070 ceph-mon[168282]: failed to encode map e619 with expected crc 2023-09-23T21:18:42.351 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:18:41 smithi070 ceph-mon[168282]: failed to encode map e619 with expected crc 2023-09-23T21:18:42.351 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:18:41 smithi070 ceph-mon[168282]: failed to encode map e619 with expected crc 2023-09-23T21:18:42.351 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:18:41 smithi070 ceph-mon[168282]: failed to encode map e619 with expected crc 2023-09-23T21:18:42.352 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:18:41 smithi070 ceph-mon[168282]: pgmap v1545: 129 pgs: 1 active+clean+scrubbing+deep, 128 active+clean; 904 MiB data, 7.1 GiB used, 529 GiB / 536 GiB avail 2023-09-23T21:18:42.352 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:18:41 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "22"}]: dispatch 2023-09-23T21:18:42.352 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:18:41 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "22"}]': finished 2023-09-23T21:18:42.352 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:18:41 smithi070 ceph-mon[168282]: osdmap e620: 8 total, 8 up, 6 in 2023-09-23T21:18:42.353 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:18:42.366 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:18:41 smithi026 ceph-mon[175151]: failed to encode map e619 with expected crc 2023-09-23T21:18:42.366 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:18:41 smithi026 ceph-mon[175151]: failed to encode map e619 with expected crc 2023-09-23T21:18:42.366 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:18:41 smithi026 ceph-mon[175151]: failed to encode map e619 with expected crc 2023-09-23T21:18:42.366 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:18:41 smithi026 ceph-mon[175151]: failed to encode map e619 with expected crc 2023-09-23T21:18:42.366 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:18:41 smithi026 ceph-mon[175151]: failed to encode map e619 with expected crc 2023-09-23T21:18:42.366 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:18:41 smithi026 ceph-mon[175151]: pgmap v1545: 129 pgs: 1 active+clean+scrubbing+deep, 128 active+clean; 904 MiB data, 7.1 GiB used, 529 GiB / 536 GiB avail 2023-09-23T21:18:42.366 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:18:41 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "22"}]: dispatch 2023-09-23T21:18:42.366 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:18:41 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "22"}]': finished 2023-09-23T21:18:42.367 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:18:41 smithi026 ceph-mon[175151]: osdmap e620: 8 total, 8 up, 6 in 2023-09-23T21:18:42.367 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:18:41 smithi026 ceph-mon[179688]: failed to encode map e619 with expected crc 2023-09-23T21:18:42.367 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:18:41 smithi026 ceph-mon[179688]: failed to encode map e619 with expected crc 2023-09-23T21:18:42.367 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:18:41 smithi026 ceph-mon[179688]: failed to encode map e619 with expected crc 2023-09-23T21:18:42.367 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:18:41 smithi026 ceph-mon[179688]: failed to encode map e619 with expected crc 2023-09-23T21:18:42.367 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:18:41 smithi026 ceph-mon[179688]: failed to encode map e619 with expected crc 2023-09-23T21:18:42.367 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:18:41 smithi026 ceph-mon[179688]: pgmap v1545: 129 pgs: 1 active+clean+scrubbing+deep, 128 active+clean; 904 MiB data, 7.1 GiB used, 529 GiB / 536 GiB avail 2023-09-23T21:18:42.367 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:18:41 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "22"}]: dispatch 2023-09-23T21:18:42.367 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:18:41 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "22"}]': finished 2023-09-23T21:18:42.368 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:18:41 smithi026 ceph-mon[179688]: osdmap e620: 8 total, 8 up, 6 in 2023-09-23T21:18:42.390 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:18:42.393 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:18:42.393 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:18:42.494 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:18:42.521 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:18:42.556 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:18:42.559 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:18:42.560 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:18:42.660 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:18:42.695 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:18:42.715 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_historic_ops 2023-09-23T21:18:42.788 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:18:42.791 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:18:42.791 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:18:42.892 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:18:42.921 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:18:43.094 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:18:43.098 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:18:43.098 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:18:43.199 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:18:43.227 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:18:43.261 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:18:43.263 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:18:42 smithi070 ceph-mon[168282]: failed to encode map e620 with expected crc 2023-09-23T21:18:43.263 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:18:42 smithi070 ceph-mon[168282]: failed to encode map e620 with expected crc 2023-09-23T21:18:43.263 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:18:42 smithi070 ceph-mon[168282]: 11.14 starting backfill to osd.4 from (0'0,0'0] MAX to 616'7891 2023-09-23T21:18:43.263 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:18:42 smithi070 ceph-mon[168282]: failed to encode map e620 with expected crc 2023-09-23T21:18:43.263 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:18:42 smithi070 ceph-mon[168282]: 11.14 starting backfill to osd.5 from (0'0,0'0] MAX to 616'7891 2023-09-23T21:18:43.263 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:18:42 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:18:43.263 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:18:42 smithi070 ceph-mon[168282]: osdmap e621: 8 total, 8 up, 6 in 2023-09-23T21:18:43.264 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:18:43.264 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:18:43.338 INFO:teuthology.orchestra.run.smithi026.stdout: 2023-09-23T21:18:43.340 INFO:teuthology.orchestra.run.smithi026.stderr:dumped all 2023-09-23T21:18:43.362 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:18:42 smithi026 ceph-mon[179688]: failed to encode map e620 with expected crc 2023-09-23T21:18:43.362 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:18:42 smithi026 ceph-mon[179688]: failed to encode map e620 with expected crc 2023-09-23T21:18:43.362 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:18:42 smithi026 ceph-mon[179688]: 11.14 starting backfill to osd.4 from (0'0,0'0] MAX to 616'7891 2023-09-23T21:18:43.362 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:18:42 smithi026 ceph-mon[179688]: failed to encode map e620 with expected crc 2023-09-23T21:18:43.362 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:18:42 smithi026 ceph-mon[179688]: 11.14 starting backfill to osd.5 from (0'0,0'0] MAX to 616'7891 2023-09-23T21:18:43.362 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:18:42 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:18:43.362 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:18:42 smithi026 ceph-mon[179688]: osdmap e621: 8 total, 8 up, 6 in 2023-09-23T21:18:43.362 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:18:42 smithi026 ceph-mon[175151]: failed to encode map e620 with expected crc 2023-09-23T21:18:43.363 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:18:42 smithi026 ceph-mon[175151]: failed to encode map e620 with expected crc 2023-09-23T21:18:43.363 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:18:42 smithi026 ceph-mon[175151]: 11.14 starting backfill to osd.4 from (0'0,0'0] MAX to 616'7891 2023-09-23T21:18:43.363 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:18:42 smithi026 ceph-mon[175151]: failed to encode map e620 with expected crc 2023-09-23T21:18:43.363 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:18:42 smithi026 ceph-mon[175151]: 11.14 starting backfill to osd.5 from (0'0,0'0] MAX to 616'7891 2023-09-23T21:18:43.363 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:18:42 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:18:43.363 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:18:42 smithi026 ceph-mon[175151]: osdmap e621: 8 total, 8 up, 6 in 2023-09-23T21:18:43.365 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:18:43.391 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:18:43.423 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:18:43.425 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:18:43.426 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:18:43.527 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:18:43.560 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:18:43.750 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T21:18:43.753 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:18:43.754 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:18:43.815 INFO:teuthology.orchestra.run.smithi026.stdout:{"pg_ready":true,"pg_map":{"version":1548,"stamp":"2023-09-23T21:18:43.201916+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":857380792,"num_objects":13098,"num_object_clones":0,"num_object_copies":39294,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":13098,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":110083,"num_write_kb":3713214,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":11462,"num_bytes_recovered":715803966,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":140979,"ondisk_log_size":140979,"up":387,"acting":387,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":399,"num_osds":6,"num_per_pool_osds":6,"num_per_pool_omap_osds":6,"kb":562470912,"kb_used":7102168,"kb_used_data":2819368,"kb_used_omap":125,"kb_used_meta":4282626,"kb_avail":555368744,"statfs":{"total":575970213888,"available":568697593856,"internally_reserved":0,"allocated":2887032832,"data_stored":2873277054,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":128662,"internal_metadata":4385409386},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,2,1,0,0,0,2,2,7,5],"upper_bound":2048},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-167280640,"num_objects":-2553,"num_object_clones":0,"num_object_copies":-7659,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":-2553,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":-14647,"num_write_kb":-597728,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":-1624,"num_bytes_recovered":-75628544,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"4.000919"},"pg_stats":[{"pgid":"11.12","version":"618'8574","reported_seq":9903,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-09-23T21:18:40.699795+0000","last_change":"2023-09-23T21:18:40.100577+0000","last_active":"2023-09-23T21:18:40.699795+0000","last_peered":"2023-09-23T21:18:40.699795+0000","last_clean":"2023-09-23T21:18:40.699795+0000","last_became_active":"2023-09-23T21:18:34.930698+0000","last_became_peered":"2023-09-23T21:18:34.930698+0000","last_unstale":"2023-09-23T21:18:40.699795+0000","last_undegraded":"2023-09-23T21:18:40.699795+0000","last_fullsized":"2023-09-23T21:18:40.699795+0000","mapping_epoch":613,"log_start":"548'1325","ondisk_log_start":"548'1325","created":559,"last_epoch_clean":614,"parent":"0.0","parent_split_bits":5,"last_scrub":"618'8566","last_scrub_stamp":"2023-09-23T21:18:40.100396+0000","last_deep_scrub":"618'8566","last_deep_scrub_stamp":"2023-09-23T21:18:40.100396+0000","last_clean_scrub_stamp":"2023-09-23T21:18:40.100396+0000","objects_scrubbed":431,"log_size":7249,"log_dups_size":0,"ondisk_log_size":7249,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":2,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:16:14.066996+0000","scrub_duration":1.094034102,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":27656192,"num_objects":422,"num_object_clones":0,"num_object_copies":1266,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":422,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,0],"acting":[2,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.18","version":"79'161","reported_seq":644,"reported_epoch":619,"state":"active+clean","last_fresh":"2023-09-23T21:18:40.890684+0000","last_change":"2023-09-23T21:17:39.906535+0000","last_active":"2023-09-23T21:18:40.890684+0000","last_peered":"2023-09-23T21:18:40.890684+0000","last_clean":"2023-09-23T21:18:40.890684+0000","last_became_active":"2023-09-23T21:17:39.906154+0000","last_became_peered":"2023-09-23T21:17:39.906154+0000","last_unstale":"2023-09-23T21:18:40.890684+0000","last_undegraded":"2023-09-23T21:18:40.890684+0000","last_fullsized":"2023-09-23T21:18:40.890684+0000","mapping_epoch":585,"log_start":"0'0","ondisk_log_start":"0'0","created":479,"last_epoch_clean":586,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:15:39.474823+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:15:39.474823+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:48:39.784037+0000","scrub_duration":0.074664395999999994,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":525376,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1a","version":"68'1","reported_seq":210,"reported_epoch":619,"state":"active+clean","last_fresh":"2023-09-23T21:18:40.882195+0000","last_change":"2023-09-23T21:13:13.291403+0000","last_active":"2023-09-23T21:18:40.882195+0000","last_peered":"2023-09-23T21:18:40.882195+0000","last_clean":"2023-09-23T21:18:40.882195+0000","last_became_active":"2023-09-23T21:13:13.291123+0000","last_became_peered":"2023-09-23T21:13:13.291123+0000","last_unstale":"2023-09-23T21:18:40.882195+0000","last_undegraded":"2023-09-23T21:18:40.882195+0000","last_fullsized":"2023-09-23T21:18:40.882195+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":495,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:45:15.427430+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,0],"acting":[4,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":279,"reported_epoch":619,"state":"active+clean","last_fresh":"2023-09-23T21:18:40.882549+0000","last_change":"2023-09-23T21:12:10.767963+0000","last_active":"2023-09-23T21:18:40.882549+0000","last_peered":"2023-09-23T21:18:40.882549+0000","last_clean":"2023-09-23T21:18:40.882549+0000","last_became_active":"2023-09-23T21:12:10.767126+0000","last_became_peered":"2023-09-23T21:12:10.767126+0000","last_unstale":"2023-09-23T21:18:40.882549+0000","last_undegraded":"2023-09-23T21:18:40.882549+0000","last_fullsized":"2023-09-23T21:18:40.882549+0000","mapping_epoch":470,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:45:47.138277+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,5],"acting":[4,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.11","version":"618'8074","reported_seq":9391,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-09-23T21:18:40.685054+0000","last_change":"2023-09-23T21:18:36.195191+0000","last_active":"2023-09-23T21:18:40.685054+0000","last_peered":"2023-09-23T21:18:40.685054+0000","last_clean":"2023-09-23T21:18:40.685054+0000","last_became_active":"2023-09-23T21:18:32.876101+0000","last_became_peered":"2023-09-23T21:18:32.876101+0000","last_unstale":"2023-09-23T21:18:40.685054+0000","last_undegraded":"2023-09-23T21:18:40.685054+0000","last_fullsized":"2023-09-23T21:18:40.685054+0000","mapping_epoch":611,"log_start":"555'3628","ondisk_log_start":"555'3628","created":559,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":5,"last_scrub":"614'7979","last_scrub_stamp":"2023-09-23T21:18:36.195023+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_clean_scrub_stamp":"2023-09-23T21:18:36.195023+0000","objects_scrubbed":487,"log_size":4446,"log_dups_size":0,"ondisk_log_size":4446,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:18:09.227629+0000","scrub_duration":0.17685009800000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":25559040,"num_objects":390,"num_object_clones":0,"num_object_copies":1170,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":390,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":97,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,7],"acting":[2,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.1b","version":"79'161","reported_seq":807,"reported_epoch":619,"state":"active+clean","last_fresh":"2023-09-23T21:18:40.891247+0000","last_change":"2023-09-23T21:15:38.423372+0000","last_active":"2023-09-23T21:18:40.891247+0000","last_peered":"2023-09-23T21:18:40.891247+0000","last_clean":"2023-09-23T21:18:40.891247+0000","last_became_active":"2023-09-23T21:15:23.802580+0000","last_became_peered":"2023-09-23T21:15:23.802580+0000","last_unstale":"2023-09-23T21:18:40.891247+0000","last_undegraded":"2023-09-23T21:18:40.891247+0000","last_fullsized":"2023-09-23T21:18:40.891247+0000","mapping_epoch":542,"log_start":"0'0","ondisk_log_start":"0'0","created":479,"last_epoch_clean":543,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:15:38.423271+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:15:38.423271+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:04:46.575186+0000","scrub_duration":0.00258465,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":632,"reported_epoch":616,"state":"active+clean","last_fresh":"2023-09-23T21:18:37.975535+0000","last_change":"2023-09-23T21:16:42.448529+0000","last_active":"2023-09-23T21:18:37.975535+0000","last_peered":"2023-09-23T21:18:37.975535+0000","last_clean":"2023-09-23T21:18:37.975535+0000","last_became_active":"2023-09-23T21:16:42.447803+0000","last_became_peered":"2023-09-23T21:16:42.447803+0000","last_unstale":"2023-09-23T21:18:37.975535+0000","last_undegraded":"2023-09-23T21:18:37.975535+0000","last_fullsized":"2023-09-23T21:18:37.975535+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:41:33.121784+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":634,"reported_epoch":617,"state":"active+clean","last_fresh":"2023-09-23T21:18:38.971903+0000","last_change":"2023-09-23T21:16:42.441896+0000","last_active":"2023-09-23T21:18:38.971903+0000","last_peered":"2023-09-23T21:18:38.971903+0000","last_clean":"2023-09-23T21:18:38.971903+0000","last_became_active":"2023-09-23T21:16:42.441526+0000","last_became_peered":"2023-09-23T21:16:42.441526+0000","last_unstale":"2023-09-23T21:18:38.971903+0000","last_undegraded":"2023-09-23T21:18:38.971903+0000","last_fullsized":"2023-09-23T21:18:38.971903+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:27:36.326225+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.10","version":"618'8007","reported_seq":11667,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-09-23T21:18:40.230756+0000","last_change":"2023-09-23T21:18:36.698828+0000","last_active":"2023-09-23T21:18:40.230756+0000","last_peered":"2023-09-23T21:18:40.230756+0000","last_clean":"2023-09-23T21:18:40.230756+0000","last_became_active":"2023-09-23T21:18:30.880043+0000","last_became_peered":"2023-09-23T21:18:30.880043+0000","last_unstale":"2023-09-23T21:18:40.230756+0000","last_undegraded":"2023-09-23T21:18:40.230756+0000","last_fullsized":"2023-09-23T21:18:40.230756+0000","mapping_epoch":609,"log_start":"556'3618","ondisk_log_start":"556'3618","created":559,"last_epoch_clean":610,"parent":"0.0","parent_split_bits":5,"last_scrub":"614'7936","last_scrub_stamp":"2023-09-23T21:18:36.698511+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_clean_scrub_stamp":"2023-09-23T21:18:36.698511+0000","objects_scrubbed":486,"log_size":4389,"log_dups_size":0,"ondisk_log_size":4389,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:26:33.228381+0000","scrub_duration":0.115560511,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":27000832,"num_objects":412,"num_object_clones":0,"num_object_copies":1236,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":412,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":74,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,5],"acting":[0,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1a","version":"79'161","reported_seq":717,"reported_epoch":619,"state":"active+clean","last_fresh":"2023-09-23T21:18:40.881945+0000","last_change":"2023-09-23T21:16:45.419221+0000","last_active":"2023-09-23T21:18:40.881945+0000","last_peered":"2023-09-23T21:18:40.881945+0000","last_clean":"2023-09-23T21:18:40.881945+0000","last_became_active":"2023-09-23T21:16:45.418994+0000","last_became_peered":"2023-09-23T21:16:45.418994+0000","last_unstale":"2023-09-23T21:18:40.881945+0000","last_undegraded":"2023-09-23T21:18:40.881945+0000","last_fullsized":"2023-09-23T21:18:40.881945+0000","mapping_epoch":567,"log_start":"0'0","ondisk_log_start":"0'0","created":479,"last_epoch_clean":568,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:15:45.158107+0000","last_deep_scrub":"79'161","last_deep_scrub_stamp":"2023-09-23T21:05:32.630292+0000","last_clean_scrub_stamp":"2023-09-23T21:15:45.158107+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:08:08.342444+0000","scrub_duration":0.001999373,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,5],"acting":[4,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":129,"reported_epoch":617,"state":"active+clean","last_fresh":"2023-09-23T21:18:38.970808+0000","last_change":"2023-09-23T21:13:30.806084+0000","last_active":"2023-09-23T21:18:38.970808+0000","last_peered":"2023-09-23T21:18:38.970808+0000","last_clean":"2023-09-23T21:18:38.970808+0000","last_became_active":"2023-09-23T21:13:30.805743+0000","last_became_peered":"2023-09-23T21:13:30.805743+0000","last_unstale":"2023-09-23T21:18:38.970808+0000","last_undegraded":"2023-09-23T21:18:38.970808+0000","last_fullsized":"2023-09-23T21:18:38.970808+0000","mapping_epoch":501,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:48:10.849281+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":493,"reported_epoch":617,"state":"active+clean","last_fresh":"2023-09-23T21:18:38.971960+0000","last_change":"2023-09-23T21:13:44.410921+0000","last_active":"2023-09-23T21:18:38.971960+0000","last_peered":"2023-09-23T21:18:38.971960+0000","last_clean":"2023-09-23T21:18:38.971960+0000","last_became_active":"2023-09-23T21:13:44.410633+0000","last_became_peered":"2023-09-23T21:13:44.410633+0000","last_unstale":"2023-09-23T21:18:38.971960+0000","last_undegraded":"2023-09-23T21:18:38.971960+0000","last_fullsized":"2023-09-23T21:18:38.971960+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:57:10.116786+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.15","version":"79'161","reported_seq":578,"reported_epoch":619,"state":"active+clean","last_fresh":"2023-09-23T21:18:40.889452+0000","last_change":"2023-09-23T21:18:35.622513+0000","last_active":"2023-09-23T21:18:40.889452+0000","last_peered":"2023-09-23T21:18:40.889452+0000","last_clean":"2023-09-23T21:18:40.889452+0000","last_became_active":"2023-09-23T21:16:45.425654+0000","last_became_peered":"2023-09-23T21:16:45.425654+0000","last_unstale":"2023-09-23T21:18:40.889452+0000","last_undegraded":"2023-09-23T21:18:40.889452+0000","last_fullsized":"2023-09-23T21:18:40.889452+0000","mapping_epoch":567,"log_start":"0'0","ondisk_log_start":"0'0","created":479,"last_epoch_clean":568,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:18:35.622422+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:18:35.622422+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:21:48.421117+0000","scrub_duration":0.083031242000000005,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,3],"acting":[7,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":594,"reported_epoch":617,"state":"active+clean","last_fresh":"2023-09-23T21:18:38.972124+0000","last_change":"2023-09-23T21:13:12.235728+0000","last_active":"2023-09-23T21:18:38.972124+0000","last_peered":"2023-09-23T21:18:38.972124+0000","last_clean":"2023-09-23T21:18:38.972124+0000","last_became_active":"2023-09-23T21:13:12.216549+0000","last_became_peered":"2023-09-23T21:13:12.216549+0000","last_unstale":"2023-09-23T21:18:38.972124+0000","last_undegraded":"2023-09-23T21:18:38.972124+0000","last_fullsized":"2023-09-23T21:18:38.972124+0000","mapping_epoch":493,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:39:33.324239+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":656,"reported_epoch":616,"state":"active+clean","last_fresh":"2023-09-23T21:18:37.973989+0000","last_change":"2023-09-23T21:12:10.790349+0000","last_active":"2023-09-23T21:18:37.973989+0000","last_peered":"2023-09-23T21:18:37.973989+0000","last_clean":"2023-09-23T21:18:37.973989+0000","last_became_active":"2023-09-23T21:12:10.789977+0000","last_became_peered":"2023-09-23T21:12:10.789977+0000","last_unstale":"2023-09-23T21:18:37.973989+0000","last_undegraded":"2023-09-23T21:18:37.973989+0000","last_fullsized":"2023-09-23T21:18:37.973989+0000","mapping_epoch":470,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:49:40.191147+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,5],"acting":[3,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"79'161","reported_seq":1016,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-09-23T21:18:39.984513+0000","last_change":"2023-09-23T21:16:45.424773+0000","last_active":"2023-09-23T21:18:39.984513+0000","last_peered":"2023-09-23T21:18:39.984513+0000","last_clean":"2023-09-23T21:18:39.984513+0000","last_became_active":"2023-09-23T21:16:45.424458+0000","last_became_peered":"2023-09-23T21:16:45.424458+0000","last_unstale":"2023-09-23T21:18:39.984513+0000","last_undegraded":"2023-09-23T21:18:39.984513+0000","last_fullsized":"2023-09-23T21:18:39.984513+0000","mapping_epoch":567,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":568,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:25.096401+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:25.096401+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:40:17.871561+0000","scrub_duration":0.002859393,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":364,"reported_epoch":619,"state":"active+clean","last_fresh":"2023-09-23T21:18:40.891413+0000","last_change":"2023-09-23T21:16:42.427856+0000","last_active":"2023-09-23T21:18:40.891413+0000","last_peered":"2023-09-23T21:18:40.891413+0000","last_clean":"2023-09-23T21:18:40.891413+0000","last_became_active":"2023-09-23T21:16:42.427529+0000","last_became_peered":"2023-09-23T21:16:42.427529+0000","last_unstale":"2023-09-23T21:18:40.891413+0000","last_undegraded":"2023-09-23T21:18:40.891413+0000","last_fullsized":"2023-09-23T21:18:40.891413+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:24:03.818632+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":611,"reported_epoch":616,"state":"active+clean","last_fresh":"2023-09-23T21:18:37.975377+0000","last_change":"2023-09-23T21:16:42.448719+0000","last_active":"2023-09-23T21:18:37.975377+0000","last_peered":"2023-09-23T21:18:37.975377+0000","last_clean":"2023-09-23T21:18:37.975377+0000","last_became_active":"2023-09-23T21:16:42.448219+0000","last_became_peered":"2023-09-23T21:16:42.448219+0000","last_unstale":"2023-09-23T21:18:37.975377+0000","last_undegraded":"2023-09-23T21:18:37.975377+0000","last_fullsized":"2023-09-23T21:18:37.975377+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:02:28.864357+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.17","version":"79'161","reported_seq":765,"reported_epoch":617,"state":"active+clean","last_fresh":"2023-09-23T21:18:38.971574+0000","last_change":"2023-09-23T21:16:45.413663+0000","last_active":"2023-09-23T21:18:38.971574+0000","last_peered":"2023-09-23T21:18:38.971574+0000","last_clean":"2023-09-23T21:18:38.971574+0000","last_became_active":"2023-09-23T21:16:45.413311+0000","last_became_peered":"2023-09-23T21:16:45.413311+0000","last_unstale":"2023-09-23T21:18:38.971574+0000","last_undegraded":"2023-09-23T21:18:38.971574+0000","last_fullsized":"2023-09-23T21:18:38.971574+0000","mapping_epoch":567,"log_start":"0'0","ondisk_log_start":"0'0","created":479,"last_epoch_clean":568,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:15:50.325741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:15:50.325741+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:07:43.285354+0000","scrub_duration":0.0040500149999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":533,"reported_epoch":616,"state":"active+clean","last_fresh":"2023-09-23T21:18:37.974529+0000","last_change":"2023-09-23T21:17:02.182606+0000","last_active":"2023-09-23T21:18:37.974529+0000","last_peered":"2023-09-23T21:18:37.974529+0000","last_clean":"2023-09-23T21:18:37.974529+0000","last_became_active":"2023-09-23T21:16:42.446517+0000","last_became_peered":"2023-09-23T21:16:42.446517+0000","last_unstale":"2023-09-23T21:18:37.974529+0000","last_undegraded":"2023-09-23T21:18:37.974529+0000","last_fullsized":"2023-09-23T21:18:37.974529+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:46:08.337878+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":540,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,4],"acting":[3,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":307,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-09-23T21:18:40.005403+0000","last_change":"2023-09-23T21:13:30.821489+0000","last_active":"2023-09-23T21:18:40.005403+0000","last_peered":"2023-09-23T21:18:40.005403+0000","last_clean":"2023-09-23T21:18:40.005403+0000","last_became_active":"2023-09-23T21:13:30.821125+0000","last_became_peered":"2023-09-23T21:13:30.821125+0000","last_unstale":"2023-09-23T21:18:40.005403+0000","last_undegraded":"2023-09-23T21:18:40.005403+0000","last_fullsized":"2023-09-23T21:18:40.005403+0000","mapping_epoch":501,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:47:18.553417+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,5],"acting":[2,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.16","version":"79'161","reported_seq":550,"reported_epoch":617,"state":"active+clean","last_fresh":"2023-09-23T21:18:38.971816+0000","last_change":"2023-09-23T21:15:46.326726+0000","last_active":"2023-09-23T21:18:38.971816+0000","last_peered":"2023-09-23T21:18:38.971816+0000","last_clean":"2023-09-23T21:18:38.971816+0000","last_became_active":"2023-09-23T21:13:33.770407+0000","last_became_peered":"2023-09-23T21:13:33.770407+0000","last_unstale":"2023-09-23T21:18:38.971816+0000","last_undegraded":"2023-09-23T21:18:38.971816+0000","last_fullsized":"2023-09-23T21:18:38.971816+0000","mapping_epoch":504,"log_start":"0'0","ondisk_log_start":"0'0","created":479,"last_epoch_clean":505,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:15:46.326589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:15:46.326589+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:50:23.810054+0000","scrub_duration":0.0062069059999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.14","version":"79'22","reported_seq":223,"reported_epoch":619,"state":"active+clean","last_fresh":"2023-09-23T21:18:40.882508+0000","last_change":"2023-09-23T21:17:02.198735+0000","last_active":"2023-09-23T21:18:40.882508+0000","last_peered":"2023-09-23T21:18:40.882508+0000","last_clean":"2023-09-23T21:18:40.882508+0000","last_became_active":"2023-09-23T21:16:42.492861+0000","last_became_peered":"2023-09-23T21:16:42.492861+0000","last_unstale":"2023-09-23T21:18:40.882508+0000","last_undegraded":"2023-09-23T21:18:40.882508+0000","last_fullsized":"2023-09-23T21:18:40.882508+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:00:09.949965+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":14202,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,2],"acting":[4,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":548,"reported_epoch":619,"state":"active+clean","last_fresh":"2023-09-23T21:18:40.882389+0000","last_change":"2023-09-23T21:12:10.839271+0000","last_active":"2023-09-23T21:18:40.882389+0000","last_peered":"2023-09-23T21:18:40.882389+0000","last_clean":"2023-09-23T21:18:40.882389+0000","last_became_active":"2023-09-23T21:12:10.839042+0000","last_became_peered":"2023-09-23T21:12:10.839042+0000","last_unstale":"2023-09-23T21:18:40.882389+0000","last_undegraded":"2023-09-23T21:18:40.882389+0000","last_fullsized":"2023-09-23T21:18:40.882389+0000","mapping_epoch":470,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:10:26.414737+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,2],"acting":[4,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.11","version":"79'161","reported_seq":555,"reported_epoch":617,"state":"active+clean","last_fresh":"2023-09-23T21:18:38.970902+0000","last_change":"2023-09-23T21:15:47.300109+0000","last_active":"2023-09-23T21:18:38.970902+0000","last_peered":"2023-09-23T21:18:38.970902+0000","last_clean":"2023-09-23T21:18:38.970902+0000","last_became_active":"2023-09-23T21:13:33.767067+0000","last_became_peered":"2023-09-23T21:13:33.767067+0000","last_unstale":"2023-09-23T21:18:38.970902+0000","last_undegraded":"2023-09-23T21:18:38.970902+0000","last_fullsized":"2023-09-23T21:18:38.970902+0000","mapping_epoch":504,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":505,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:15:47.299947+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:15:47.299947+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:04:59.029359+0000","scrub_duration":0.0061353110000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":71,"reported_epoch":619,"state":"active+clean","last_fresh":"2023-09-23T21:18:40.889072+0000","last_change":"2023-09-23T21:16:42.531060+0000","last_active":"2023-09-23T21:18:40.889072+0000","last_peered":"2023-09-23T21:18:40.889072+0000","last_clean":"2023-09-23T21:18:40.889072+0000","last_became_active":"2023-09-23T21:16:42.449198+0000","last_became_peered":"2023-09-23T21:16:42.449198+0000","last_unstale":"2023-09-23T21:18:40.889072+0000","last_undegraded":"2023-09-23T21:18:40.889072+0000","last_fullsized":"2023-09-23T21:18:40.889072+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:30:23.428886+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":179,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-09-23T21:18:39.984187+0000","last_change":"2023-09-23T21:16:42.491023+0000","last_active":"2023-09-23T21:18:39.984187+0000","last_peered":"2023-09-23T21:18:39.984187+0000","last_clean":"2023-09-23T21:18:39.984187+0000","last_became_active":"2023-09-23T21:16:42.489896+0000","last_became_peered":"2023-09-23T21:16:42.489896+0000","last_unstale":"2023-09-23T21:18:39.984187+0000","last_undegraded":"2023-09-23T21:18:39.984187+0000","last_fullsized":"2023-09-23T21:18:39.984187+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:58:54.036098+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,7],"acting":[2,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.10","version":"79'161","reported_seq":794,"reported_epoch":619,"state":"active+clean","last_fresh":"2023-09-23T21:18:40.881869+0000","last_change":"2023-09-23T21:16:45.416227+0000","last_active":"2023-09-23T21:18:40.881869+0000","last_peered":"2023-09-23T21:18:40.881869+0000","last_clean":"2023-09-23T21:18:40.881869+0000","last_became_active":"2023-09-23T21:16:45.415841+0000","last_became_peered":"2023-09-23T21:16:45.415841+0000","last_unstale":"2023-09-23T21:18:40.881869+0000","last_undegraded":"2023-09-23T21:18:40.881869+0000","last_fullsized":"2023-09-23T21:18:40.881869+0000","mapping_epoch":567,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":568,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:06:50.336992+0000","last_deep_scrub":"79'161","last_deep_scrub_stamp":"2023-09-23T21:06:50.336992+0000","last_clean_scrub_stamp":"2023-09-23T21:06:50.336992+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:37:30.094658+0000","scrub_duration":0.0026150930000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,7],"acting":[4,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":578,"reported_epoch":617,"state":"active+clean","last_fresh":"2023-09-23T21:18:38.972494+0000","last_change":"2023-09-23T21:16:42.446324+0000","last_active":"2023-09-23T21:18:38.972494+0000","last_peered":"2023-09-23T21:18:38.972494+0000","last_clean":"2023-09-23T21:18:38.972494+0000","last_became_active":"2023-09-23T21:16:42.445997+0000","last_became_peered":"2023-09-23T21:16:42.445997+0000","last_unstale":"2023-09-23T21:18:38.972494+0000","last_undegraded":"2023-09-23T21:18:38.972494+0000","last_fullsized":"2023-09-23T21:18:38.972494+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:46:35.028292+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":388,"reported_epoch":616,"state":"active+clean","last_fresh":"2023-09-23T21:18:37.974377+0000","last_change":"2023-09-23T21:16:42.473872+0000","last_active":"2023-09-23T21:18:37.974377+0000","last_peered":"2023-09-23T21:18:37.974377+0000","last_clean":"2023-09-23T21:18:37.974377+0000","last_became_active":"2023-09-23T21:16:42.473595+0000","last_became_peered":"2023-09-23T21:16:42.473595+0000","last_unstale":"2023-09-23T21:18:37.974377+0000","last_undegraded":"2023-09-23T21:18:37.974377+0000","last_fullsized":"2023-09-23T21:18:37.974377+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:15:19.186428+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.19","version":"616'7975","reported_seq":9772,"reported_epoch":617,"state":"active+clean+scrubbing+deep","last_fresh":"2023-09-23T21:18:39.018094+0000","last_change":"2023-09-23T21:18:39.192555+0000","last_active":"2023-09-23T21:18:39.018094+0000","last_peered":"2023-09-23T21:18:39.018094+0000","last_clean":"2023-09-23T21:18:39.018094+0000","last_became_active":"2023-09-23T21:17:46.300166+0000","last_became_peered":"2023-09-23T21:17:46.300166+0000","last_unstale":"2023-09-23T21:18:39.018094+0000","last_undegraded":"2023-09-23T21:18:39.018094+0000","last_fullsized":"2023-09-23T21:18:39.018094+0000","mapping_epoch":588,"log_start":"556'3819","ondisk_log_start":"556'3819","created":559,"last_epoch_clean":601,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_clean_scrub_stamp":"2023-09-23T21:15:24.860968+0000","objects_scrubbed":0,"log_size":4156,"log_dups_size":0,"ondisk_log_size":4156,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":25526272,"num_objects":390,"num_object_clones":0,"num_object_copies":1170,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":390,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5692,"num_write_kb":194592,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":838,"num_bytes_recovered":54820864,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,7],"acting":[3,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.13","version":"79'161","reported_seq":695,"reported_epoch":616,"state":"active+clean","last_fresh":"2023-09-23T21:18:37.974457+0000","last_change":"2023-09-23T21:13:14.933096+0000","last_active":"2023-09-23T21:18:37.974457+0000","last_peered":"2023-09-23T21:18:37.974457+0000","last_clean":"2023-09-23T21:18:37.974457+0000","last_became_active":"2023-09-23T21:13:14.932703+0000","last_became_peered":"2023-09-23T21:13:14.932703+0000","last_unstale":"2023-09-23T21:18:37.974457+0000","last_undegraded":"2023-09-23T21:18:37.974457+0000","last_fullsized":"2023-09-23T21:18:37.974457+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:36.420362+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:36.420362+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:29:42.163911+0000","scrub_duration":0.028838444000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":631,"reported_epoch":619,"state":"active+clean","last_fresh":"2023-09-23T21:18:40.882028+0000","last_change":"2023-09-23T21:16:42.420467+0000","last_active":"2023-09-23T21:18:40.882028+0000","last_peered":"2023-09-23T21:18:40.882028+0000","last_clean":"2023-09-23T21:18:40.882028+0000","last_became_active":"2023-09-23T21:16:42.420149+0000","last_became_peered":"2023-09-23T21:16:42.420149+0000","last_unstale":"2023-09-23T21:18:40.882028+0000","last_undegraded":"2023-09-23T21:18:40.882028+0000","last_fullsized":"2023-09-23T21:18:40.882028+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:52:07.355000+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":147,"reported_epoch":617,"state":"active+clean","last_fresh":"2023-09-23T21:18:38.971566+0000","last_change":"2023-09-23T21:16:42.427663+0000","last_active":"2023-09-23T21:18:38.971566+0000","last_peered":"2023-09-23T21:18:38.971566+0000","last_clean":"2023-09-23T21:18:38.971566+0000","last_became_active":"2023-09-23T21:16:42.426961+0000","last_became_peered":"2023-09-23T21:16:42.426961+0000","last_unstale":"2023-09-23T21:18:38.971566+0000","last_undegraded":"2023-09-23T21:18:38.971566+0000","last_fullsized":"2023-09-23T21:18:38.971566+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:44:27.422007+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.18","version":"618'7775","reported_seq":7890,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-09-23T21:18:40.455845+0000","last_change":"2023-09-23T21:18:32.625291+0000","last_active":"2023-09-23T21:18:40.455845+0000","last_peered":"2023-09-23T21:18:40.455845+0000","last_clean":"2023-09-23T21:18:40.455845+0000","last_became_active":"2023-09-23T21:16:32.926828+0000","last_became_peered":"2023-09-23T21:16:32.926828+0000","last_unstale":"2023-09-23T21:18:40.455845+0000","last_undegraded":"2023-09-23T21:18:40.455845+0000","last_fullsized":"2023-09-23T21:18:40.455845+0000","mapping_epoch":559,"log_start":"550'3074","ondisk_log_start":"550'3074","created":559,"last_epoch_clean":560,"parent":"0.0","parent_split_bits":5,"last_scrub":"611'7608","last_scrub_stamp":"2023-09-23T21:18:32.625076+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_clean_scrub_stamp":"2023-09-23T21:18:32.625076+0000","objects_scrubbed":541,"log_size":4701,"log_dups_size":0,"ondisk_log_size":4701,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:29:49.621612+0000","scrub_duration":0.078956060999999994,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":24510464,"num_objects":374,"num_object_clones":0,"num_object_copies":1122,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":374,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":167,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,5],"acting":[0,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.12","version":"79'161","reported_seq":893,"reported_epoch":619,"state":"active+clean","last_fresh":"2023-09-23T21:18:40.891057+0000","last_change":"2023-09-23T21:16:45.412440+0000","last_active":"2023-09-23T21:18:40.891057+0000","last_peered":"2023-09-23T21:18:40.891057+0000","last_clean":"2023-09-23T21:18:40.891057+0000","last_became_active":"2023-09-23T21:16:45.412029+0000","last_became_peered":"2023-09-23T21:16:45.412029+0000","last_unstale":"2023-09-23T21:18:40.891057+0000","last_undegraded":"2023-09-23T21:18:40.891057+0000","last_fullsized":"2023-09-23T21:18:40.891057+0000","mapping_epoch":567,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":568,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:05:56.785173+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:05:56.785173+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:39:43.432998+0000","scrub_duration":0.0042114370000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,0],"acting":[5,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":139,"reported_epoch":617,"state":"active+clean","last_fresh":"2023-09-23T21:18:38.970904+0000","last_change":"2023-09-23T21:13:38.048791+0000","last_active":"2023-09-23T21:18:38.970904+0000","last_peered":"2023-09-23T21:18:38.970904+0000","last_clean":"2023-09-23T21:18:38.970904+0000","last_became_active":"2023-09-23T21:13:30.801386+0000","last_became_peered":"2023-09-23T21:13:30.801386+0000","last_unstale":"2023-09-23T21:18:38.970904+0000","last_undegraded":"2023-09-23T21:18:38.970904+0000","last_fullsized":"2023-09-23T21:18:38.970904+0000","mapping_epoch":501,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:52:49.170414+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":473,"reported_epoch":616,"state":"active+clean","last_fresh":"2023-09-23T21:18:37.974125+0000","last_change":"2023-09-23T21:16:42.518512+0000","last_active":"2023-09-23T21:18:37.974125+0000","last_peered":"2023-09-23T21:18:37.974125+0000","last_clean":"2023-09-23T21:18:37.974125+0000","last_became_active":"2023-09-23T21:16:42.517464+0000","last_became_peered":"2023-09-23T21:16:42.517464+0000","last_unstale":"2023-09-23T21:18:37.974125+0000","last_undegraded":"2023-09-23T21:18:37.974125+0000","last_fullsized":"2023-09-23T21:18:37.974125+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:47:52.391167+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.7","version":"620'8059","reported_seq":8198,"reported_epoch":620,"state":"active+clean","last_fresh":"2023-09-23T21:18:42.056407+0000","last_change":"2023-09-23T21:18:30.328388+0000","last_active":"2023-09-23T21:18:42.056407+0000","last_peered":"2023-09-23T21:18:42.056407+0000","last_clean":"2023-09-23T21:18:42.056407+0000","last_became_active":"2023-09-23T21:16:32.864890+0000","last_became_peered":"2023-09-23T21:16:32.864890+0000","last_unstale":"2023-09-23T21:18:42.056407+0000","last_undegraded":"2023-09-23T21:18:42.056407+0000","last_fullsized":"2023-09-23T21:18:42.056407+0000","mapping_epoch":544,"log_start":"549'2003","ondisk_log_start":"549'2003","created":544,"last_epoch_clean":560,"parent":"0.0","parent_split_bits":0,"last_scrub":"609'7868","last_scrub_stamp":"2023-09-23T21:18:30.328316+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_clean_scrub_stamp":"2023-09-23T21:18:30.328316+0000","objects_scrubbed":571,"log_size":6056,"log_dups_size":0,"ondisk_log_size":6056,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:34:57.557430+0000","scrub_duration":0.064969499999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":24903680,"num_objects":380,"num_object_clones":0,"num_object_copies":1140,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":380,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":191,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.d","version":"79'161","reported_seq":648,"reported_epoch":619,"state":"active+clean","last_fresh":"2023-09-23T21:18:40.889478+0000","last_change":"2023-09-23T21:18:28.543985+0000","last_active":"2023-09-23T21:18:40.889478+0000","last_peered":"2023-09-23T21:18:40.889478+0000","last_clean":"2023-09-23T21:18:40.889478+0000","last_became_active":"2023-09-23T21:16:45.416373+0000","last_became_peered":"2023-09-23T21:16:45.416373+0000","last_unstale":"2023-09-23T21:18:40.889478+0000","last_undegraded":"2023-09-23T21:18:40.889478+0000","last_fullsized":"2023-09-23T21:18:40.889478+0000","mapping_epoch":567,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":568,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:15:32.726690+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:15:32.726690+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0024987500000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":69,"reported_epoch":619,"state":"active+clean","last_fresh":"2023-09-23T21:18:40.889875+0000","last_change":"2023-09-23T21:17:02.190320+0000","last_active":"2023-09-23T21:18:40.889875+0000","last_peered":"2023-09-23T21:18:40.889875+0000","last_clean":"2023-09-23T21:18:40.889875+0000","last_became_active":"2023-09-23T21:16:42.451380+0000","last_became_peered":"2023-09-23T21:16:42.451380+0000","last_unstale":"2023-09-23T21:18:40.889875+0000","last_undegraded":"2023-09-23T21:18:40.889875+0000","last_fullsized":"2023-09-23T21:18:40.889875+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:48:58.425821+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":132,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":347,"reported_epoch":619,"state":"active+clean","last_fresh":"2023-09-23T21:18:40.890870+0000","last_change":"2023-09-23T21:16:42.428491+0000","last_active":"2023-09-23T21:18:40.890870+0000","last_peered":"2023-09-23T21:18:40.890870+0000","last_clean":"2023-09-23T21:18:40.890870+0000","last_became_active":"2023-09-23T21:16:42.428166+0000","last_became_peered":"2023-09-23T21:16:42.428166+0000","last_unstale":"2023-09-23T21:18:40.890870+0000","last_undegraded":"2023-09-23T21:18:40.890870+0000","last_fullsized":"2023-09-23T21:18:40.890870+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:23:01.430714+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,4],"acting":[5,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.6","version":"618'7859","reported_seq":7979,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-09-23T21:18:40.248652+0000","last_change":"2023-09-23T21:18:29.652253+0000","last_active":"2023-09-23T21:18:40.248652+0000","last_peered":"2023-09-23T21:18:40.248652+0000","last_clean":"2023-09-23T21:18:40.248652+0000","last_became_active":"2023-09-23T21:16:32.896510+0000","last_became_peered":"2023-09-23T21:16:32.896510+0000","last_unstale":"2023-09-23T21:18:40.248652+0000","last_undegraded":"2023-09-23T21:18:40.248652+0000","last_fullsized":"2023-09-23T21:18:40.248652+0000","mapping_epoch":544,"log_start":"554'3314","ondisk_log_start":"554'3314","created":544,"last_epoch_clean":561,"parent":"0.0","parent_split_bits":0,"last_scrub":"607'7678","last_scrub_stamp":"2023-09-23T21:18:29.652124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_clean_scrub_stamp":"2023-09-23T21:18:29.652124+0000","objects_scrubbed":618,"log_size":4545,"log_dups_size":0,"ondisk_log_size":4545,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:05:14.537109+0000","scrub_duration":0.077048132000000005,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":28639232,"num_objects":437,"num_object_clones":0,"num_object_copies":1311,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":437,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":181,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.c","version":"79'161","reported_seq":999,"reported_epoch":616,"state":"active+clean","last_fresh":"2023-09-23T21:18:37.974767+0000","last_change":"2023-09-23T21:15:31.992756+0000","last_active":"2023-09-23T21:18:37.974767+0000","last_peered":"2023-09-23T21:18:37.974767+0000","last_clean":"2023-09-23T21:18:37.974767+0000","last_became_active":"2023-09-23T21:13:13.401526+0000","last_became_peered":"2023-09-23T21:13:13.401526+0000","last_unstale":"2023-09-23T21:18:37.974767+0000","last_undegraded":"2023-09-23T21:18:37.974767+0000","last_fullsized":"2023-09-23T21:18:37.974767+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":532,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:15:31.992598+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:15:31.992598+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:41:14.215543+0000","scrub_duration":0.002960678,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":69,"reported_epoch":619,"state":"active+clean","last_fresh":"2023-09-23T21:18:40.889654+0000","last_change":"2023-09-23T21:17:02.215996+0000","last_active":"2023-09-23T21:18:40.889654+0000","last_peered":"2023-09-23T21:18:40.889654+0000","last_clean":"2023-09-23T21:18:40.889654+0000","last_became_active":"2023-09-23T21:16:42.451205+0000","last_became_peered":"2023-09-23T21:16:42.451205+0000","last_unstale":"2023-09-23T21:18:40.889654+0000","last_undegraded":"2023-09-23T21:18:40.889654+0000","last_fullsized":"2023-09-23T21:18:40.889654+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:51:46.876281+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":582,"reported_epoch":619,"state":"active+clean","last_fresh":"2023-09-23T21:18:40.882092+0000","last_change":"2023-09-23T21:13:15.152792+0000","last_active":"2023-09-23T21:18:40.882092+0000","last_peered":"2023-09-23T21:18:40.882092+0000","last_clean":"2023-09-23T21:18:40.882092+0000","last_became_active":"2023-09-23T21:13:14.515084+0000","last_became_peered":"2023-09-23T21:13:14.515084+0000","last_unstale":"2023-09-23T21:18:40.882092+0000","last_undegraded":"2023-09-23T21:18:40.882092+0000","last_fullsized":"2023-09-23T21:18:40.882092+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:08:16.702072+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.5","version":"620'8081","reported_seq":13345,"reported_epoch":620,"state":"active+clean","last_fresh":"2023-09-23T21:18:42.104223+0000","last_change":"2023-09-23T21:18:36.457465+0000","last_active":"2023-09-23T21:18:42.104223+0000","last_peered":"2023-09-23T21:18:42.104223+0000","last_clean":"2023-09-23T21:18:42.104223+0000","last_became_active":"2023-09-23T21:17:24.389052+0000","last_became_peered":"2023-09-23T21:17:24.389052+0000","last_unstale":"2023-09-23T21:18:42.104223+0000","last_undegraded":"2023-09-23T21:18:42.104223+0000","last_fullsized":"2023-09-23T21:18:42.104223+0000","mapping_epoch":578,"log_start":"549'1652","ondisk_log_start":"549'1652","created":544,"last_epoch_clean":579,"parent":"0.0","parent_split_bits":0,"last_scrub":"614'7983","last_scrub_stamp":"2023-09-23T21:18:36.457393+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_clean_scrub_stamp":"2023-09-23T21:18:36.457393+0000","objects_scrubbed":481,"log_size":6429,"log_dups_size":0,"ondisk_log_size":6429,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:07:29.014321+0000","scrub_duration":0.14039997700000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":24838144,"num_objects":379,"num_object_clones":0,"num_object_copies":1137,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":379,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":102,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.f","version":"79'161","reported_seq":907,"reported_epoch":619,"state":"active+clean","last_fresh":"2023-09-23T21:18:40.891503+0000","last_change":"2023-09-23T21:11:30.750338+0000","last_active":"2023-09-23T21:18:40.891503+0000","last_peered":"2023-09-23T21:18:40.891503+0000","last_clean":"2023-09-23T21:18:40.891503+0000","last_became_active":"2023-09-23T21:11:30.749953+0000","last_became_peered":"2023-09-23T21:11:30.749953+0000","last_unstale":"2023-09-23T21:18:40.891503+0000","last_undegraded":"2023-09-23T21:18:40.891503+0000","last_fullsized":"2023-09-23T21:18:40.891503+0000","mapping_epoch":455,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":456,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:27.880776+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:27.880776+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:07:10.405723+0000","scrub_duration":0.0014004899999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,3],"acting":[5,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":371,"reported_epoch":619,"state":"active+clean","last_fresh":"2023-09-23T21:18:40.891460+0000","last_change":"2023-09-23T21:17:02.175012+0000","last_active":"2023-09-23T21:18:40.891460+0000","last_peered":"2023-09-23T21:18:40.891460+0000","last_clean":"2023-09-23T21:18:40.891460+0000","last_became_active":"2023-09-23T21:16:42.451114+0000","last_became_peered":"2023-09-23T21:16:42.451114+0000","last_unstale":"2023-09-23T21:18:40.891460+0000","last_undegraded":"2023-09-23T21:18:40.891460+0000","last_fullsized":"2023-09-23T21:18:40.891460+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:00:53.868525+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":420,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":579,"reported_epoch":619,"state":"active+clean","last_fresh":"2023-09-23T21:18:40.890936+0000","last_change":"2023-09-23T21:16:42.432943+0000","last_active":"2023-09-23T21:18:40.890936+0000","last_peered":"2023-09-23T21:18:40.890936+0000","last_clean":"2023-09-23T21:18:40.890936+0000","last_became_active":"2023-09-23T21:16:42.427440+0000","last_became_peered":"2023-09-23T21:16:42.427440+0000","last_unstale":"2023-09-23T21:18:40.890936+0000","last_undegraded":"2023-09-23T21:18:40.890936+0000","last_fullsized":"2023-09-23T21:18:40.890936+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:59:32.044609+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,0],"acting":[5,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.4","version":"620'7880","reported_seq":9529,"reported_epoch":620,"state":"active+clean","last_fresh":"2023-09-23T21:18:42.070734+0000","last_change":"2023-09-23T21:18:05.512415+0000","last_active":"2023-09-23T21:18:42.070734+0000","last_peered":"2023-09-23T21:18:42.070734+0000","last_clean":"2023-09-23T21:18:42.070734+0000","last_became_active":"2023-09-23T21:18:05.477101+0000","last_became_peered":"2023-09-23T21:18:05.477101+0000","last_unstale":"2023-09-23T21:18:42.070734+0000","last_undegraded":"2023-09-23T21:18:42.070734+0000","last_fullsized":"2023-09-23T21:18:42.070734+0000","mapping_epoch":595,"log_start":"549'2655","ondisk_log_start":"549'2655","created":544,"last_epoch_clean":596,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_clean_scrub_stamp":"2023-09-23T21:15:24.860968+0000","objects_scrubbed":0,"log_size":5225,"log_dups_size":0,"ondisk_log_size":5225,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":33882112,"num_objects":517,"num_object_clones":0,"num_object_copies":1551,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":517,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5643,"num_write_kb":197120,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":765,"num_bytes_recovered":50135040,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,3],"acting":[7,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.e","version":"79'161","reported_seq":838,"reported_epoch":619,"state":"active+clean","last_fresh":"2023-09-23T21:18:40.882614+0000","last_change":"2023-09-23T21:16:45.433204+0000","last_active":"2023-09-23T21:18:40.882614+0000","last_peered":"2023-09-23T21:18:40.882614+0000","last_clean":"2023-09-23T21:18:40.882614+0000","last_became_active":"2023-09-23T21:16:45.432836+0000","last_became_peered":"2023-09-23T21:16:45.432836+0000","last_unstale":"2023-09-23T21:18:40.882614+0000","last_undegraded":"2023-09-23T21:18:40.882614+0000","last_fullsized":"2023-09-23T21:18:40.882614+0000","mapping_epoch":567,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":568,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:15:39.046962+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:15:39.046962+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:11:20.720731+0000","scrub_duration":0.0082638769999999993,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":416,"reported_epoch":616,"state":"active+clean","last_fresh":"2023-09-23T21:18:37.975113+0000","last_change":"2023-09-23T21:13:13.402418+0000","last_active":"2023-09-23T21:18:37.975113+0000","last_peered":"2023-09-23T21:18:37.975113+0000","last_clean":"2023-09-23T21:18:37.975113+0000","last_became_active":"2023-09-23T21:13:13.400984+0000","last_became_peered":"2023-09-23T21:13:13.400984+0000","last_unstale":"2023-09-23T21:18:37.975113+0000","last_undegraded":"2023-09-23T21:18:37.975113+0000","last_fullsized":"2023-09-23T21:18:37.975113+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":495,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:28:01.411191+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,0],"acting":[3,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":592,"reported_epoch":617,"state":"active+clean","last_fresh":"2023-09-23T21:18:38.971356+0000","last_change":"2023-09-23T21:13:14.339670+0000","last_active":"2023-09-23T21:18:38.971356+0000","last_peered":"2023-09-23T21:18:38.971356+0000","last_clean":"2023-09-23T21:18:38.971356+0000","last_became_active":"2023-09-23T21:13:14.339341+0000","last_became_peered":"2023-09-23T21:13:14.339341+0000","last_unstale":"2023-09-23T21:18:38.971356+0000","last_undegraded":"2023-09-23T21:18:38.971356+0000","last_fullsized":"2023-09-23T21:18:38.971356+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:23:19.411368+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.3","version":"620'7817","reported_seq":9510,"reported_epoch":620,"state":"active+clean","last_fresh":"2023-09-23T21:18:42.122418+0000","last_change":"2023-09-23T21:17:51.904100+0000","last_active":"2023-09-23T21:18:42.122418+0000","last_peered":"2023-09-23T21:18:42.122418+0000","last_clean":"2023-09-23T21:18:42.122418+0000","last_became_active":"2023-09-23T21:17:51.872537+0000","last_became_peered":"2023-09-23T21:17:51.872537+0000","last_unstale":"2023-09-23T21:18:42.122418+0000","last_undegraded":"2023-09-23T21:18:42.122418+0000","last_fullsized":"2023-09-23T21:18:42.122418+0000","mapping_epoch":591,"log_start":"549'2622","ondisk_log_start":"549'2622","created":544,"last_epoch_clean":592,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_clean_scrub_stamp":"2023-09-23T21:15:24.860968+0000","objects_scrubbed":0,"log_size":5195,"log_dups_size":0,"ondisk_log_size":5195,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":25624576,"num_objects":391,"num_object_clones":0,"num_object_copies":1173,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":391,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5601,"num_write_kb":191744,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":786,"num_bytes_recovered":51511296,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,2],"acting":[7,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.9","version":"79'161","reported_seq":820,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-09-23T21:18:39.983553+0000","last_change":"2023-09-23T21:16:45.433968+0000","last_active":"2023-09-23T21:18:39.983553+0000","last_peered":"2023-09-23T21:18:39.983553+0000","last_clean":"2023-09-23T21:18:39.983553+0000","last_became_active":"2023-09-23T21:16:45.433602+0000","last_became_peered":"2023-09-23T21:16:45.433602+0000","last_unstale":"2023-09-23T21:18:39.983553+0000","last_undegraded":"2023-09-23T21:18:39.983553+0000","last_fullsized":"2023-09-23T21:18:39.983553+0000","mapping_epoch":567,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":568,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:15:43.437369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:15:43.437369+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:16:44.538102+0000","scrub_duration":0.0024129429999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,3],"acting":[2,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":178,"reported_epoch":617,"state":"active+clean","last_fresh":"2023-09-23T21:18:38.971434+0000","last_change":"2023-09-23T21:13:30.793273+0000","last_active":"2023-09-23T21:18:38.971434+0000","last_peered":"2023-09-23T21:18:38.971434+0000","last_clean":"2023-09-23T21:18:38.971434+0000","last_became_active":"2023-09-23T21:13:30.791269+0000","last_became_peered":"2023-09-23T21:13:30.791269+0000","last_unstale":"2023-09-23T21:18:38.971434+0000","last_undegraded":"2023-09-23T21:18:38.971434+0000","last_fullsized":"2023-09-23T21:18:38.971434+0000","mapping_epoch":501,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:38:07.142290+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":213,"reported_epoch":619,"state":"active+clean","last_fresh":"2023-09-23T21:18:40.882225+0000","last_change":"2023-09-23T21:16:42.417057+0000","last_active":"2023-09-23T21:18:40.882225+0000","last_peered":"2023-09-23T21:18:40.882225+0000","last_clean":"2023-09-23T21:18:40.882225+0000","last_became_active":"2023-09-23T21:16:42.416601+0000","last_became_peered":"2023-09-23T21:16:42.416601+0000","last_unstale":"2023-09-23T21:18:40.882225+0000","last_undegraded":"2023-09-23T21:18:40.882225+0000","last_fullsized":"2023-09-23T21:18:40.882225+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:56:52.167562+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,7],"acting":[4,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.2","version":"618'8056","reported_seq":8173,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-09-23T21:18:40.681816+0000","last_change":"2023-09-23T21:18:34.137284+0000","last_active":"2023-09-23T21:18:40.681816+0000","last_peered":"2023-09-23T21:18:40.681816+0000","last_clean":"2023-09-23T21:18:40.681816+0000","last_became_active":"2023-09-23T21:16:32.867780+0000","last_became_peered":"2023-09-23T21:16:32.867780+0000","last_unstale":"2023-09-23T21:18:40.681816+0000","last_undegraded":"2023-09-23T21:18:40.681816+0000","last_fullsized":"2023-09-23T21:18:40.681816+0000","mapping_epoch":544,"log_start":"548'448","ondisk_log_start":"548'448","created":544,"last_epoch_clean":560,"parent":"0.0","parent_split_bits":0,"last_scrub":"613'7927","last_scrub_stamp":"2023-09-23T21:18:34.136961+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_clean_scrub_stamp":"2023-09-23T21:18:34.136961+0000","objects_scrubbed":557,"log_size":7608,"log_dups_size":0,"ondisk_log_size":7608,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:53:14.227837+0000","scrub_duration":0.135147037,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":27983872,"num_objects":427,"num_object_clones":0,"num_object_copies":1281,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":427,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":130,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.8","version":"79'161","reported_seq":753,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-09-23T21:18:39.983600+0000","last_change":"2023-09-23T21:15:36.367819+0000","last_active":"2023-09-23T21:18:39.983600+0000","last_peered":"2023-09-23T21:18:39.983600+0000","last_clean":"2023-09-23T21:18:39.983600+0000","last_became_active":"2023-09-23T21:12:30.433217+0000","last_became_peered":"2023-09-23T21:12:30.433217+0000","last_unstale":"2023-09-23T21:18:39.983600+0000","last_undegraded":"2023-09-23T21:18:39.983600+0000","last_fullsized":"2023-09-23T21:18:39.983600+0000","mapping_epoch":359,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:15:36.367640+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:15:36.367640+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:49:35.243653+0000","scrub_duration":0.005826899,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":196,"reported_epoch":619,"state":"active+clean","last_fresh":"2023-09-23T21:18:40.882149+0000","last_change":"2023-09-23T21:12:10.823380+0000","last_active":"2023-09-23T21:18:40.882149+0000","last_peered":"2023-09-23T21:18:40.882149+0000","last_clean":"2023-09-23T21:18:40.882149+0000","last_became_active":"2023-09-23T21:12:10.823044+0000","last_became_peered":"2023-09-23T21:12:10.823044+0000","last_unstale":"2023-09-23T21:18:40.882149+0000","last_undegraded":"2023-09-23T21:18:40.882149+0000","last_fullsized":"2023-09-23T21:18:40.882149+0000","mapping_epoch":470,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:53:29.846536+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,5],"acting":[4,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":197,"reported_epoch":619,"state":"active+clean","last_fresh":"2023-09-23T21:18:40.882356+0000","last_change":"2023-09-23T21:13:30.772451+0000","last_active":"2023-09-23T21:18:40.882356+0000","last_peered":"2023-09-23T21:18:40.882356+0000","last_clean":"2023-09-23T21:18:40.882356+0000","last_became_active":"2023-09-23T21:13:30.772040+0000","last_became_peered":"2023-09-23T21:13:30.772040+0000","last_unstale":"2023-09-23T21:18:40.882356+0000","last_undegraded":"2023-09-23T21:18:40.882356+0000","last_fullsized":"2023-09-23T21:18:40.882356+0000","mapping_epoch":501,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:24:49.246727+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,0],"acting":[4,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.1","version":"616'8177","reported_seq":8290,"reported_epoch":616,"state":"active+clean","last_fresh":"2023-09-23T21:18:38.801327+0000","last_change":"2023-09-23T21:18:35.452919+0000","last_active":"2023-09-23T21:18:38.801327+0000","last_peered":"2023-09-23T21:18:38.801327+0000","last_clean":"2023-09-23T21:18:38.801327+0000","last_became_active":"2023-09-23T21:16:32.905672+0000","last_became_peered":"2023-09-23T21:16:32.905672+0000","last_unstale":"2023-09-23T21:18:38.801327+0000","last_undegraded":"2023-09-23T21:18:38.801327+0000","last_fullsized":"2023-09-23T21:18:38.801327+0000","mapping_epoch":544,"log_start":"556'4141","ondisk_log_start":"556'4141","created":544,"last_epoch_clean":560,"parent":"0.0","parent_split_bits":0,"last_scrub":"614'8079","last_scrub_stamp":"2023-09-23T21:18:35.452658+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_clean_scrub_stamp":"2023-09-23T21:18:35.452658+0000","objects_scrubbed":563,"log_size":4036,"log_dups_size":0,"ondisk_log_size":4036,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:19:47.092568+0000","scrub_duration":0.23111627100000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":29949952,"num_objects":457,"num_object_clones":0,"num_object_copies":1371,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":457,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":106,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,2],"acting":[3,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.b","version":"79'161","reported_seq":635,"reported_epoch":617,"state":"active+clean","last_fresh":"2023-09-23T21:18:38.971103+0000","last_change":"2023-09-23T21:15:36.340009+0000","last_active":"2023-09-23T21:18:38.971103+0000","last_peered":"2023-09-23T21:18:38.971103+0000","last_clean":"2023-09-23T21:18:38.971103+0000","last_became_active":"2023-09-23T21:13:13.372328+0000","last_became_peered":"2023-09-23T21:13:13.372328+0000","last_unstale":"2023-09-23T21:18:38.971103+0000","last_undegraded":"2023-09-23T21:18:38.971103+0000","last_fullsized":"2023-09-23T21:18:38.971103+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":495,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:15:36.339885+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:15:36.339885+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:01:26.605006+0000","scrub_duration":0.0068473279999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":197,"reported_epoch":619,"state":"active+clean","last_fresh":"2023-09-23T21:18:40.881945+0000","last_change":"2023-09-23T21:16:42.425103+0000","last_active":"2023-09-23T21:18:40.881945+0000","last_peered":"2023-09-23T21:18:40.881945+0000","last_clean":"2023-09-23T21:18:40.881945+0000","last_became_active":"2023-09-23T21:16:42.424543+0000","last_became_peered":"2023-09-23T21:16:42.424543+0000","last_unstale":"2023-09-23T21:18:40.881945+0000","last_undegraded":"2023-09-23T21:18:40.881945+0000","last_fullsized":"2023-09-23T21:18:40.881945+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:11:28.885246+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":551,"reported_epoch":619,"state":"active+clean","last_fresh":"2023-09-23T21:18:40.882764+0000","last_change":"2023-09-23T21:12:10.811698+0000","last_active":"2023-09-23T21:18:40.882764+0000","last_peered":"2023-09-23T21:18:40.882764+0000","last_clean":"2023-09-23T21:18:40.882764+0000","last_became_active":"2023-09-23T21:12:10.789656+0000","last_became_peered":"2023-09-23T21:12:10.789656+0000","last_unstale":"2023-09-23T21:18:40.882764+0000","last_undegraded":"2023-09-23T21:18:40.882764+0000","last_fullsized":"2023-09-23T21:18:40.882764+0000","mapping_epoch":470,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:39:15.112091+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,5],"acting":[4,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.0","version":"616'7905","reported_seq":9631,"reported_epoch":616,"state":"active+clean","last_fresh":"2023-09-23T21:18:38.781584+0000","last_change":"2023-09-23T21:18:10.533031+0000","last_active":"2023-09-23T21:18:38.781584+0000","last_peered":"2023-09-23T21:18:38.781584+0000","last_clean":"2023-09-23T21:18:38.781584+0000","last_became_active":"2023-09-23T21:18:10.519762+0000","last_became_peered":"2023-09-23T21:18:10.519762+0000","last_unstale":"2023-09-23T21:18:38.781584+0000","last_undegraded":"2023-09-23T21:18:38.781584+0000","last_fullsized":"2023-09-23T21:18:38.781584+0000","mapping_epoch":598,"log_start":"556'3690","ondisk_log_start":"556'3690","created":544,"last_epoch_clean":599,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_clean_scrub_stamp":"2023-09-23T21:15:24.860968+0000","objects_scrubbed":0,"log_size":4215,"log_dups_size":0,"ondisk_log_size":4215,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":25657344,"num_objects":391,"num_object_clones":0,"num_object_copies":1173,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":391,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5647,"num_write_kb":193248,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":803,"num_bytes_recovered":42336256,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.a","version":"79'161","reported_seq":779,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-09-23T21:18:40.005325+0000","last_change":"2023-09-23T21:15:46.420777+0000","last_active":"2023-09-23T21:18:40.005325+0000","last_peered":"2023-09-23T21:18:40.005325+0000","last_clean":"2023-09-23T21:18:40.005325+0000","last_became_active":"2023-09-23T21:12:30.437427+0000","last_became_peered":"2023-09-23T21:12:30.437427+0000","last_unstale":"2023-09-23T21:18:40.005325+0000","last_undegraded":"2023-09-23T21:18:40.005325+0000","last_fullsized":"2023-09-23T21:18:40.005325+0000","mapping_epoch":446,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:15:46.420650+0000","last_deep_scrub":"79'161","last_deep_scrub_stamp":"2023-09-23T21:05:32.630292+0000","last_clean_scrub_stamp":"2023-09-23T21:15:46.420650+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:50:23.452475+0000","scrub_duration":0.0072898930000000004,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,4],"acting":[2,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":483,"reported_epoch":616,"state":"active+clean","last_fresh":"2023-09-23T21:18:37.973587+0000","last_change":"2023-09-23T21:16:42.447403+0000","last_active":"2023-09-23T21:18:37.973587+0000","last_peered":"2023-09-23T21:18:37.973587+0000","last_clean":"2023-09-23T21:18:37.973587+0000","last_became_active":"2023-09-23T21:16:42.447080+0000","last_became_peered":"2023-09-23T21:16:42.447080+0000","last_unstale":"2023-09-23T21:18:37.973587+0000","last_undegraded":"2023-09-23T21:18:37.973587+0000","last_fullsized":"2023-09-23T21:18:37.973587+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:15:19.446334+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,4],"acting":[3,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":614,"reported_epoch":616,"state":"active+clean","last_fresh":"2023-09-23T21:18:37.974289+0000","last_change":"2023-09-23T21:16:42.518739+0000","last_active":"2023-09-23T21:18:37.974289+0000","last_peered":"2023-09-23T21:18:37.974289+0000","last_clean":"2023-09-23T21:18:37.974289+0000","last_became_active":"2023-09-23T21:16:42.517945+0000","last_became_peered":"2023-09-23T21:16:42.517945+0000","last_unstale":"2023-09-23T21:18:37.974289+0000","last_undegraded":"2023-09-23T21:18:37.974289+0000","last_fullsized":"2023-09-23T21:18:37.974289+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:47:26.236933+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.d","version":"618'11790","reported_seq":11876,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-09-23T21:18:40.723176+0000","last_change":"2023-09-23T21:15:25.892834+0000","last_active":"2023-09-23T21:18:40.723176+0000","last_peered":"2023-09-23T21:18:40.723176+0000","last_clean":"2023-09-23T21:18:40.723176+0000","last_became_active":"2023-09-23T21:15:25.892519+0000","last_became_peered":"2023-09-23T21:15:25.892519+0000","last_unstale":"2023-09-23T21:18:40.723176+0000","last_undegraded":"2023-09-23T21:18:40.723176+0000","last_fullsized":"2023-09-23T21:18:40.723176+0000","mapping_epoch":544,"log_start":"570'6072","ondisk_log_start":"570'6072","created":544,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_clean_scrub_stamp":"2023-09-23T21:15:24.860968+0000","objects_scrubbed":0,"log_size":5718,"log_dups_size":0,"ondisk_log_size":5718,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:46:28.375665+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":52494360,"num_objects":802,"num_object_clones":0,"num_object_copies":2406,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":802,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":11790,"num_write_kb":402881,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,5],"acting":[2,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.7","version":"79'161","reported_seq":757,"reported_epoch":616,"state":"active+clean","last_fresh":"2023-09-23T21:18:37.974039+0000","last_change":"2023-09-23T21:16:45.425645+0000","last_active":"2023-09-23T21:18:37.974039+0000","last_peered":"2023-09-23T21:18:37.974039+0000","last_clean":"2023-09-23T21:18:37.974039+0000","last_became_active":"2023-09-23T21:16:45.425241+0000","last_became_peered":"2023-09-23T21:16:45.425241+0000","last_unstale":"2023-09-23T21:18:37.974039+0000","last_undegraded":"2023-09-23T21:18:37.974039+0000","last_fullsized":"2023-09-23T21:18:37.974039+0000","mapping_epoch":567,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":568,"parent":"0.0","parent_split_bits":3,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:15:51.016427+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:15:51.016427+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:49:31.176462+0000","scrub_duration":0.0017648640000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":567,"reported_epoch":617,"state":"active+clean","last_fresh":"2023-09-23T21:18:38.971604+0000","last_change":"2023-09-23T21:16:42.518826+0000","last_active":"2023-09-23T21:18:38.971604+0000","last_peered":"2023-09-23T21:18:38.971604+0000","last_clean":"2023-09-23T21:18:38.971604+0000","last_became_active":"2023-09-23T21:16:42.518488+0000","last_became_peered":"2023-09-23T21:16:42.518488+0000","last_unstale":"2023-09-23T21:18:38.971604+0000","last_undegraded":"2023-09-23T21:18:38.971604+0000","last_fullsized":"2023-09-23T21:18:38.971604+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:42:44.144383+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,2],"acting":[0,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":297,"reported_epoch":619,"state":"active+clean","last_fresh":"2023-09-23T21:18:40.891365+0000","last_change":"2023-09-23T21:05:04.326889+0000","last_active":"2023-09-23T21:18:40.891365+0000","last_peered":"2023-09-23T21:18:40.891365+0000","last_clean":"2023-09-23T21:18:40.891365+0000","last_became_active":"2023-09-23T21:05:04.326032+0000","last_became_peered":"2023-09-23T21:05:04.326032+0000","last_unstale":"2023-09-23T21:18:40.891365+0000","last_undegraded":"2023-09-23T21:18:40.891365+0000","last_fullsized":"2023-09-23T21:18:40.891365+0000","mapping_epoch":332,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:00:17.507889+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":629,"reported_epoch":619,"state":"active+clean","last_fresh":"2023-09-23T21:18:40.890998+0000","last_change":"2023-09-23T21:13:13.285608+0000","last_active":"2023-09-23T21:18:40.890998+0000","last_peered":"2023-09-23T21:18:40.890998+0000","last_clean":"2023-09-23T21:18:40.890998+0000","last_became_active":"2023-09-23T21:13:13.285255+0000","last_became_peered":"2023-09-23T21:13:13.285255+0000","last_unstale":"2023-09-23T21:18:40.890998+0000","last_undegraded":"2023-09-23T21:18:40.890998+0000","last_fullsized":"2023-09-23T21:18:40.890998+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":495,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:42:30.087599+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.b","version":"616'11150","reported_seq":11233,"reported_epoch":616,"state":"active+clean","last_fresh":"2023-09-23T21:18:38.744635+0000","last_change":"2023-09-23T21:15:25.906791+0000","last_active":"2023-09-23T21:18:38.744635+0000","last_peered":"2023-09-23T21:18:38.744635+0000","last_clean":"2023-09-23T21:18:38.744635+0000","last_became_active":"2023-09-23T21:15:25.906512+0000","last_became_peered":"2023-09-23T21:15:25.906512+0000","last_unstale":"2023-09-23T21:18:38.744635+0000","last_undegraded":"2023-09-23T21:18:38.744635+0000","last_fullsized":"2023-09-23T21:18:38.744635+0000","mapping_epoch":544,"log_start":"579'7345","ondisk_log_start":"579'7345","created":544,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_clean_scrub_stamp":"2023-09-23T21:15:24.860968+0000","objects_scrubbed":0,"log_size":3805,"log_dups_size":0,"ondisk_log_size":3805,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:58:51.749775+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":52428800,"num_objects":800,"num_object_clones":0,"num_object_copies":2400,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":800,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11150,"num_write_kb":382400,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1","version":"79'161","reported_seq":702,"reported_epoch":617,"state":"active+clean","last_fresh":"2023-09-23T21:18:38.971608+0000","last_change":"2023-09-23T21:15:34.345290+0000","last_active":"2023-09-23T21:18:38.971608+0000","last_peered":"2023-09-23T21:18:38.971608+0000","last_clean":"2023-09-23T21:18:38.971608+0000","last_became_active":"2023-09-23T21:13:12.203380+0000","last_became_peered":"2023-09-23T21:13:12.203380+0000","last_unstale":"2023-09-23T21:18:38.971608+0000","last_undegraded":"2023-09-23T21:18:38.971608+0000","last_fullsized":"2023-09-23T21:18:38.971608+0000","mapping_epoch":493,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:15:34.345138+0000","last_deep_scrub":"79'161","last_deep_scrub_stamp":"2023-09-23T21:05:30.652189+0000","last_clean_scrub_stamp":"2023-09-23T21:15:34.345138+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:47:50.349409+0000","scrub_duration":0.0057520000000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":286,"reported_epoch":617,"state":"active+clean","last_fresh":"2023-09-23T21:18:38.972402+0000","last_change":"2023-09-23T21:13:13.238617+0000","last_active":"2023-09-23T21:18:38.972402+0000","last_peered":"2023-09-23T21:18:38.972402+0000","last_clean":"2023-09-23T21:18:38.972402+0000","last_became_active":"2023-09-23T21:13:13.232926+0000","last_became_peered":"2023-09-23T21:13:13.232926+0000","last_unstale":"2023-09-23T21:18:38.972402+0000","last_undegraded":"2023-09-23T21:18:38.972402+0000","last_fullsized":"2023-09-23T21:18:38.972402+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":495,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:17:21.823772+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":553,"reported_epoch":619,"state":"active+clean","last_fresh":"2023-09-23T21:18:40.882614+0000","last_change":"2023-09-23T21:13:14.144918+0000","last_active":"2023-09-23T21:18:40.882614+0000","last_peered":"2023-09-23T21:18:40.882614+0000","last_clean":"2023-09-23T21:18:40.882614+0000","last_became_active":"2023-09-23T21:13:14.132388+0000","last_became_peered":"2023-09-23T21:13:14.132388+0000","last_unstale":"2023-09-23T21:18:40.882614+0000","last_undegraded":"2023-09-23T21:18:40.882614+0000","last_fullsized":"2023-09-23T21:18:40.882614+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":495,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:56:02.619905+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,5],"acting":[4,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":459,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-09-23T21:18:39.984408+0000","last_change":"2023-09-23T21:16:42.491111+0000","last_active":"2023-09-23T21:18:39.984408+0000","last_peered":"2023-09-23T21:18:39.984408+0000","last_clean":"2023-09-23T21:18:39.984408+0000","last_became_active":"2023-09-23T21:16:42.490275+0000","last_became_peered":"2023-09-23T21:16:42.490275+0000","last_unstale":"2023-09-23T21:18:39.984408+0000","last_undegraded":"2023-09-23T21:18:39.984408+0000","last_fullsized":"2023-09-23T21:18:39.984408+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:39:11.652564+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,7],"acting":[2,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"11.8","version":"618'7953","reported_seq":8071,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-09-23T21:18:40.433340+0000","last_change":"2023-09-23T21:18:30.671221+0000","last_active":"2023-09-23T21:18:40.433340+0000","last_peered":"2023-09-23T21:18:40.433340+0000","last_clean":"2023-09-23T21:18:40.433340+0000","last_became_active":"2023-09-23T21:16:32.894560+0000","last_became_peered":"2023-09-23T21:16:32.894560+0000","last_unstale":"2023-09-23T21:18:40.433340+0000","last_undegraded":"2023-09-23T21:18:40.433340+0000","last_fullsized":"2023-09-23T21:18:40.433340+0000","mapping_epoch":544,"log_start":"555'3479","ondisk_log_start":"555'3479","created":544,"last_epoch_clean":560,"parent":"0.0","parent_split_bits":0,"last_scrub":"609'7757","last_scrub_stamp":"2023-09-23T21:18:30.670962+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_clean_scrub_stamp":"2023-09-23T21:18:30.670962+0000","objects_scrubbed":619,"log_size":4474,"log_dups_size":0,"ondisk_log_size":4474,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:13:50.200821+0000","scrub_duration":0.101488842,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":27721728,"num_objects":423,"num_object_clones":0,"num_object_copies":1269,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":423,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":196,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,5],"acting":[0,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.2","version":"79'161","reported_seq":933,"reported_epoch":617,"state":"active+clean","last_fresh":"2023-09-23T21:18:38.972183+0000","last_change":"2023-09-23T21:16:45.417075+0000","last_active":"2023-09-23T21:18:38.972183+0000","last_peered":"2023-09-23T21:18:38.972183+0000","last_clean":"2023-09-23T21:18:38.972183+0000","last_became_active":"2023-09-23T21:16:45.416752+0000","last_became_peered":"2023-09-23T21:16:45.416752+0000","last_unstale":"2023-09-23T21:18:38.972183+0000","last_undegraded":"2023-09-23T21:18:38.972183+0000","last_fullsized":"2023-09-23T21:18:38.972183+0000","mapping_epoch":567,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":568,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:24.802451+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:24.802451+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:01:51.129779+0000","scrub_duration":0.0090861250000000005,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,4],"acting":[0,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":629,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-09-23T21:18:39.983754+0000","last_change":"2023-09-23T21:13:12.206980+0000","last_active":"2023-09-23T21:18:39.983754+0000","last_peered":"2023-09-23T21:18:39.983754+0000","last_clean":"2023-09-23T21:18:39.983754+0000","last_became_active":"2023-09-23T21:13:12.206650+0000","last_became_peered":"2023-09-23T21:13:12.206650+0000","last_unstale":"2023-09-23T21:18:39.983754+0000","last_undegraded":"2023-09-23T21:18:39.983754+0000","last_fullsized":"2023-09-23T21:18:39.983754+0000","mapping_epoch":493,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:10:01.892936+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":613,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-09-23T21:18:39.984130+0000","last_change":"2023-09-23T21:13:13.407073+0000","last_active":"2023-09-23T21:18:39.984130+0000","last_peered":"2023-09-23T21:18:39.984130+0000","last_clean":"2023-09-23T21:18:39.984130+0000","last_became_active":"2023-09-23T21:13:13.395047+0000","last_became_peered":"2023-09-23T21:13:13.395047+0000","last_unstale":"2023-09-23T21:18:39.984130+0000","last_undegraded":"2023-09-23T21:18:39.984130+0000","last_fullsized":"2023-09-23T21:18:39.984130+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":495,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:04:43.384190+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,5],"acting":[2,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":147,"reported_epoch":617,"state":"active+clean","last_fresh":"2023-09-23T21:18:38.972039+0000","last_change":"2023-09-23T21:16:42.517729+0000","last_active":"2023-09-23T21:18:38.972039+0000","last_peered":"2023-09-23T21:18:38.972039+0000","last_clean":"2023-09-23T21:18:38.972039+0000","last_became_active":"2023-09-23T21:16:42.517373+0000","last_became_peered":"2023-09-23T21:16:42.517373+0000","last_unstale":"2023-09-23T21:18:38.972039+0000","last_undegraded":"2023-09-23T21:18:38.972039+0000","last_fullsized":"2023-09-23T21:18:38.972039+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:05:57.370550+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.a","version":"618'11583","reported_seq":11668,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-09-23T21:18:40.730270+0000","last_change":"2023-09-23T21:15:25.893846+0000","last_active":"2023-09-23T21:18:40.730270+0000","last_peered":"2023-09-23T21:18:40.730270+0000","last_clean":"2023-09-23T21:18:40.730270+0000","last_became_active":"2023-09-23T21:15:25.893452+0000","last_became_peered":"2023-09-23T21:15:25.893452+0000","last_unstale":"2023-09-23T21:18:40.730270+0000","last_undegraded":"2023-09-23T21:18:40.730270+0000","last_fullsized":"2023-09-23T21:18:40.730270+0000","mapping_epoch":544,"log_start":"570'5825","ondisk_log_start":"570'5825","created":544,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_clean_scrub_stamp":"2023-09-23T21:15:24.860968+0000","objects_scrubbed":0,"log_size":5758,"log_dups_size":0,"ondisk_log_size":5758,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:42:32.842297+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":50528256,"num_objects":771,"num_object_clones":0,"num_object_copies":2313,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":771,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11583,"num_write_kb":395328,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,5],"acting":[2,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":424,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-09-23T21:18:39.984279+0000","last_change":"2023-09-23T21:16:42.423216+0000","last_active":"2023-09-23T21:18:39.984279+0000","last_peered":"2023-09-23T21:18:39.984279+0000","last_clean":"2023-09-23T21:18:39.984279+0000","last_became_active":"2023-09-23T21:16:42.422856+0000","last_became_peered":"2023-09-23T21:16:42.422856+0000","last_unstale":"2023-09-23T21:18:39.984279+0000","last_undegraded":"2023-09-23T21:18:39.984279+0000","last_fullsized":"2023-09-23T21:18:39.984279+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:06:54.105798+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"79'161","reported_seq":836,"reported_epoch":617,"state":"active+clean","last_fresh":"2023-09-23T21:18:38.970606+0000","last_change":"2023-09-23T21:16:45.428032+0000","last_active":"2023-09-23T21:18:38.970606+0000","last_peered":"2023-09-23T21:18:38.970606+0000","last_clean":"2023-09-23T21:18:38.970606+0000","last_became_active":"2023-09-23T21:16:45.427674+0000","last_became_peered":"2023-09-23T21:16:45.427674+0000","last_unstale":"2023-09-23T21:18:38.970606+0000","last_undegraded":"2023-09-23T21:18:38.970606+0000","last_fullsized":"2023-09-23T21:18:38.970606+0000","mapping_epoch":567,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":568,"parent":"0.0","parent_split_bits":0,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:20.738273+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:20.738273+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:39:42.441711+0000","scrub_duration":0.0088336339999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,2],"acting":[0,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":627,"reported_epoch":619,"state":"active+clean","last_fresh":"2023-09-23T21:18:40.890969+0000","last_change":"2023-09-23T21:11:08.760417+0000","last_active":"2023-09-23T21:18:40.890969+0000","last_peered":"2023-09-23T21:18:40.890969+0000","last_clean":"2023-09-23T21:18:40.890969+0000","last_became_active":"2023-09-23T21:11:08.760150+0000","last_became_peered":"2023-09-23T21:11:08.760150+0000","last_unstale":"2023-09-23T21:18:40.890969+0000","last_undegraded":"2023-09-23T21:18:40.890969+0000","last_fullsized":"2023-09-23T21:18:40.890969+0000","mapping_epoch":443,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":444,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:23:10.849768+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":214,"reported_epoch":619,"state":"active+clean","last_fresh":"2023-09-23T21:18:40.882436+0000","last_change":"2023-09-23T21:16:42.424717+0000","last_active":"2023-09-23T21:18:40.882436+0000","last_peered":"2023-09-23T21:18:40.882436+0000","last_clean":"2023-09-23T21:18:40.882436+0000","last_became_active":"2023-09-23T21:16:42.424376+0000","last_became_peered":"2023-09-23T21:16:42.424376+0000","last_unstale":"2023-09-23T21:18:40.882436+0000","last_undegraded":"2023-09-23T21:18:40.882436+0000","last_fullsized":"2023-09-23T21:18:40.882436+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:25:14.162321+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.9","version":"616'8102","reported_seq":9944,"reported_epoch":616,"state":"active+clean","last_fresh":"2023-09-23T21:18:38.687783+0000","last_change":"2023-09-23T21:17:58.411362+0000","last_active":"2023-09-23T21:18:38.687783+0000","last_peered":"2023-09-23T21:18:38.687783+0000","last_clean":"2023-09-23T21:18:38.687783+0000","last_became_active":"2023-09-23T21:17:58.399421+0000","last_became_peered":"2023-09-23T21:17:58.399421+0000","last_unstale":"2023-09-23T21:18:38.687783+0000","last_undegraded":"2023-09-23T21:18:38.687783+0000","last_fullsized":"2023-09-23T21:18:38.687783+0000","mapping_epoch":593,"log_start":"556'4013","ondisk_log_start":"556'4013","created":544,"last_epoch_clean":594,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_clean_scrub_stamp":"2023-09-23T21:15:24.860968+0000","objects_scrubbed":0,"log_size":4089,"log_dups_size":0,"ondisk_log_size":4089,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":30310400,"num_objects":462,"num_object_clones":0,"num_object_copies":1386,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":462,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5818,"num_write_kb":200992,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":860,"num_bytes_recovered":56393728,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,7],"acting":[3,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.3","version":"79'161","reported_seq":922,"reported_epoch":619,"state":"active+clean","last_fresh":"2023-09-23T21:18:40.882021+0000","last_change":"2023-09-23T21:13:33.771279+0000","last_active":"2023-09-23T21:18:40.882021+0000","last_peered":"2023-09-23T21:18:40.882021+0000","last_clean":"2023-09-23T21:18:40.882021+0000","last_became_active":"2023-09-23T21:13:33.770980+0000","last_became_peered":"2023-09-23T21:13:33.770980+0000","last_unstale":"2023-09-23T21:18:40.882021+0000","last_undegraded":"2023-09-23T21:18:40.882021+0000","last_fullsized":"2023-09-23T21:18:40.882021+0000","mapping_epoch":504,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":505,"parent":"0.0","parent_split_bits":3,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:05:57.832302+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:05:57.832302+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:19:42.309604+0000","scrub_duration":0.010206847999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":454,"reported_epoch":617,"state":"active+clean","last_fresh":"2023-09-23T21:18:38.970937+0000","last_change":"2023-09-23T21:16:42.455365+0000","last_active":"2023-09-23T21:18:38.970937+0000","last_peered":"2023-09-23T21:18:38.970937+0000","last_clean":"2023-09-23T21:18:38.970937+0000","last_became_active":"2023-09-23T21:16:42.452829+0000","last_became_peered":"2023-09-23T21:16:42.452829+0000","last_unstale":"2023-09-23T21:18:38.970937+0000","last_undegraded":"2023-09-23T21:18:38.970937+0000","last_fullsized":"2023-09-23T21:18:38.970937+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:49:26.303389+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":618,"reported_epoch":617,"state":"active+clean","last_fresh":"2023-09-23T21:18:38.971179+0000","last_change":"2023-09-23T21:13:44.409212+0000","last_active":"2023-09-23T21:18:38.971179+0000","last_peered":"2023-09-23T21:18:38.971179+0000","last_clean":"2023-09-23T21:18:38.971179+0000","last_became_active":"2023-09-23T21:13:44.408958+0000","last_became_peered":"2023-09-23T21:13:44.408958+0000","last_unstale":"2023-09-23T21:18:38.971179+0000","last_undegraded":"2023-09-23T21:18:38.971179+0000","last_fullsized":"2023-09-23T21:18:38.971179+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:03:52.749964+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":147,"reported_epoch":617,"state":"active+clean","last_fresh":"2023-09-23T21:18:38.971730+0000","last_change":"2023-09-23T21:16:42.427490+0000","last_active":"2023-09-23T21:18:38.971730+0000","last_peered":"2023-09-23T21:18:38.971730+0000","last_clean":"2023-09-23T21:18:38.971730+0000","last_became_active":"2023-09-23T21:16:42.426566+0000","last_became_peered":"2023-09-23T21:16:42.426566+0000","last_unstale":"2023-09-23T21:18:38.971730+0000","last_undegraded":"2023-09-23T21:18:38.971730+0000","last_fullsized":"2023-09-23T21:18:38.971730+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:20:26.396005+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,7],"acting":[0,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.c","version":"618'11322","reported_seq":14512,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-09-23T21:18:40.715983+0000","last_change":"2023-09-23T21:17:21.358806+0000","last_active":"2023-09-23T21:18:40.715983+0000","last_peered":"2023-09-23T21:18:40.715983+0000","last_clean":"2023-09-23T21:18:40.715983+0000","last_became_active":"2023-09-23T21:17:21.346629+0000","last_became_peered":"2023-09-23T21:17:21.346629+0000","last_unstale":"2023-09-23T21:18:40.715983+0000","last_undegraded":"2023-09-23T21:18:40.715983+0000","last_fullsized":"2023-09-23T21:18:40.715983+0000","mapping_epoch":576,"log_start":"570'5600","ondisk_log_start":"570'5600","created":544,"last_epoch_clean":577,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_clean_scrub_stamp":"2023-09-23T21:15:24.860968+0000","objects_scrubbed":0,"log_size":5722,"log_dups_size":0,"ondisk_log_size":5722,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:59:48.173244+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":55967744,"num_objects":854,"num_object_clones":0,"num_object_copies":2562,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":854,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11322,"num_write_kb":389632,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1539,"num_bytes_recovered":100859904,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,7],"acting":[2,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.6","version":"79'161","reported_seq":550,"reported_epoch":617,"state":"active+clean","last_fresh":"2023-09-23T21:18:38.971651+0000","last_change":"2023-09-23T21:15:43.333919+0000","last_active":"2023-09-23T21:18:38.971651+0000","last_peered":"2023-09-23T21:18:38.971651+0000","last_clean":"2023-09-23T21:18:38.971651+0000","last_became_active":"2023-09-23T21:13:33.769303+0000","last_became_peered":"2023-09-23T21:13:33.769303+0000","last_unstale":"2023-09-23T21:18:38.971651+0000","last_undegraded":"2023-09-23T21:18:38.971651+0000","last_fullsized":"2023-09-23T21:18:38.971651+0000","mapping_epoch":504,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":505,"parent":"0.0","parent_split_bits":3,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:15:43.333733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:15:43.333733+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:42:29.787357+0000","scrub_duration":0.0092107530000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":65,"reported_epoch":619,"state":"active+clean","last_fresh":"2023-09-23T21:18:40.889394+0000","last_change":"2023-09-23T21:16:42.483813+0000","last_active":"2023-09-23T21:18:40.889394+0000","last_peered":"2023-09-23T21:18:40.889394+0000","last_clean":"2023-09-23T21:18:40.889394+0000","last_became_active":"2023-09-23T21:16:42.482847+0000","last_became_peered":"2023-09-23T21:16:42.482847+0000","last_unstale":"2023-09-23T21:18:40.889394+0000","last_undegraded":"2023-09-23T21:18:40.889394+0000","last_fullsized":"2023-09-23T21:18:40.889394+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:38:57.905679+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":504,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-09-23T21:18:39.984734+0000","last_change":"2023-09-23T21:13:30.790590+0000","last_active":"2023-09-23T21:18:39.984734+0000","last_peered":"2023-09-23T21:18:39.984734+0000","last_clean":"2023-09-23T21:18:39.984734+0000","last_became_active":"2023-09-23T21:13:30.790253+0000","last_became_peered":"2023-09-23T21:13:30.790253+0000","last_unstale":"2023-09-23T21:18:39.984734+0000","last_undegraded":"2023-09-23T21:18:39.984734+0000","last_fullsized":"2023-09-23T21:18:39.984734+0000","mapping_epoch":501,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:23:41.843250+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,0],"acting":[2,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":583,"reported_epoch":617,"state":"active+clean","last_fresh":"2023-09-23T21:18:38.971973+0000","last_change":"2023-09-23T21:16:42.428535+0000","last_active":"2023-09-23T21:18:38.971973+0000","last_peered":"2023-09-23T21:18:38.971973+0000","last_clean":"2023-09-23T21:18:38.971973+0000","last_became_active":"2023-09-23T21:16:42.428149+0000","last_became_peered":"2023-09-23T21:16:42.428149+0000","last_unstale":"2023-09-23T21:18:38.971973+0000","last_undegraded":"2023-09-23T21:18:38.971973+0000","last_fullsized":"2023-09-23T21:18:38.971973+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:38:59.979788+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.f","version":"618'11399","reported_seq":15038,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-09-23T21:18:40.685237+0000","last_change":"2023-09-23T21:17:03.878062+0000","last_active":"2023-09-23T21:18:40.685237+0000","last_peered":"2023-09-23T21:18:40.685237+0000","last_clean":"2023-09-23T21:18:40.685237+0000","last_became_active":"2023-09-23T21:17:03.870998+0000","last_became_peered":"2023-09-23T21:17:03.870998+0000","last_unstale":"2023-09-23T21:18:40.685237+0000","last_undegraded":"2023-09-23T21:18:40.685237+0000","last_fullsized":"2023-09-23T21:18:40.685237+0000","mapping_epoch":572,"log_start":"570'5700","ondisk_log_start":"570'5700","created":544,"last_epoch_clean":573,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_clean_scrub_stamp":"2023-09-23T21:15:24.860968+0000","objects_scrubbed":0,"log_size":5699,"log_dups_size":0,"ondisk_log_size":5699,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:19:31.942955+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":50397184,"num_objects":769,"num_object_clones":0,"num_object_copies":2307,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":769,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11399,"num_write_kb":389376,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1762,"num_bytes_recovered":115474432,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,0],"acting":[2,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.5","version":"79'161","reported_seq":657,"reported_epoch":619,"state":"active+clean","last_fresh":"2023-09-23T21:18:40.882385+0000","last_change":"2023-09-23T21:15:41.108638+0000","last_active":"2023-09-23T21:18:40.882385+0000","last_peered":"2023-09-23T21:18:40.882385+0000","last_clean":"2023-09-23T21:18:40.882385+0000","last_became_active":"2023-09-23T21:13:38.373633+0000","last_became_peered":"2023-09-23T21:13:38.373633+0000","last_unstale":"2023-09-23T21:18:40.882385+0000","last_undegraded":"2023-09-23T21:18:40.882385+0000","last_fullsized":"2023-09-23T21:18:40.882385+0000","mapping_epoch":507,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":3,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:15:41.108579+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:15:41.108579+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:21:58.075649+0000","scrub_duration":0.0061511960000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":186,"reported_epoch":617,"state":"active+clean","last_fresh":"2023-09-23T21:18:38.972330+0000","last_change":"2023-09-23T21:17:02.204086+0000","last_active":"2023-09-23T21:18:38.972330+0000","last_peered":"2023-09-23T21:18:38.972330+0000","last_clean":"2023-09-23T21:18:38.972330+0000","last_became_active":"2023-09-23T21:16:42.517549+0000","last_became_peered":"2023-09-23T21:16:42.517549+0000","last_unstale":"2023-09-23T21:18:38.972330+0000","last_undegraded":"2023-09-23T21:18:38.972330+0000","last_fullsized":"2023-09-23T21:18:38.972330+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:43:07.991844+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":600,"reported_epoch":619,"state":"active+clean","last_fresh":"2023-09-23T21:18:40.882684+0000","last_change":"2023-09-23T21:13:30.807520+0000","last_active":"2023-09-23T21:18:40.882684+0000","last_peered":"2023-09-23T21:18:40.882684+0000","last_clean":"2023-09-23T21:18:40.882684+0000","last_became_active":"2023-09-23T21:13:30.807222+0000","last_became_peered":"2023-09-23T21:13:30.807222+0000","last_unstale":"2023-09-23T21:18:40.882684+0000","last_undegraded":"2023-09-23T21:18:40.882684+0000","last_fullsized":"2023-09-23T21:18:40.882684+0000","mapping_epoch":501,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:40:31.989749+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":478,"reported_epoch":616,"state":"active+clean","last_fresh":"2023-09-23T21:18:37.973657+0000","last_change":"2023-09-23T21:16:42.463411+0000","last_active":"2023-09-23T21:18:37.973657+0000","last_peered":"2023-09-23T21:18:37.973657+0000","last_clean":"2023-09-23T21:18:37.973657+0000","last_became_active":"2023-09-23T21:16:42.463020+0000","last_became_peered":"2023-09-23T21:16:42.463020+0000","last_unstale":"2023-09-23T21:18:37.973657+0000","last_undegraded":"2023-09-23T21:18:37.973657+0000","last_fullsized":"2023-09-23T21:18:37.973657+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:07:34.346129+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.e","version":"620'11440","reported_seq":15658,"reported_epoch":620,"state":"active+clean","last_fresh":"2023-09-23T21:18:42.091676+0000","last_change":"2023-09-23T21:17:34.456174+0000","last_active":"2023-09-23T21:18:42.091676+0000","last_peered":"2023-09-23T21:18:42.091676+0000","last_clean":"2023-09-23T21:18:42.091676+0000","last_became_active":"2023-09-23T21:17:34.444637+0000","last_became_peered":"2023-09-23T21:17:34.444637+0000","last_unstale":"2023-09-23T21:18:42.091676+0000","last_undegraded":"2023-09-23T21:18:42.091676+0000","last_fullsized":"2023-09-23T21:18:42.091676+0000","mapping_epoch":582,"log_start":"573'6647","ondisk_log_start":"573'6647","created":544,"last_epoch_clean":583,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_clean_scrub_stamp":"2023-09-23T21:15:24.860968+0000","objects_scrubbed":0,"log_size":4793,"log_dups_size":0,"ondisk_log_size":4793,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:17:47.772535+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53477376,"num_objects":816,"num_object_clones":0,"num_object_copies":2448,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":816,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11440,"num_write_kb":392192,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2052,"num_bytes_recovered":134479872,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.4","version":"79'161","reported_seq":1023,"reported_epoch":616,"state":"active+clean","last_fresh":"2023-09-23T21:18:37.974838+0000","last_change":"2023-09-23T21:16:45.426136+0000","last_active":"2023-09-23T21:18:37.974838+0000","last_peered":"2023-09-23T21:18:37.974838+0000","last_clean":"2023-09-23T21:18:37.974838+0000","last_became_active":"2023-09-23T21:16:45.425669+0000","last_became_peered":"2023-09-23T21:16:45.425669+0000","last_unstale":"2023-09-23T21:18:37.974838+0000","last_undegraded":"2023-09-23T21:18:37.974838+0000","last_fullsized":"2023-09-23T21:18:37.974838+0000","mapping_epoch":567,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":568,"parent":"0.0","parent_split_bits":3,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:20.012764+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:20.012764+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:52:39.142509+0000","scrub_duration":0.0045251919999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":577,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-09-23T21:18:39.984232+0000","last_change":"2023-09-23T21:16:42.459732+0000","last_active":"2023-09-23T21:18:39.984232+0000","last_peered":"2023-09-23T21:18:39.984232+0000","last_clean":"2023-09-23T21:18:39.984232+0000","last_became_active":"2023-09-23T21:16:42.459413+0000","last_became_peered":"2023-09-23T21:16:42.459413+0000","last_unstale":"2023-09-23T21:18:39.984232+0000","last_undegraded":"2023-09-23T21:18:39.984232+0000","last_fullsized":"2023-09-23T21:18:39.984232+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:37:44.548863+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,3],"acting":[2,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.6","version":"70'1","reported_seq":619,"reported_epoch":617,"state":"active+clean","last_fresh":"2023-09-23T21:18:38.972072+0000","last_change":"2023-09-23T21:17:02.208077+0000","last_active":"2023-09-23T21:18:38.972072+0000","last_peered":"2023-09-23T21:18:38.972072+0000","last_clean":"2023-09-23T21:18:38.972072+0000","last_became_active":"2023-09-23T21:16:42.431308+0000","last_became_peered":"2023-09-23T21:16:42.431308+0000","last_unstale":"2023-09-23T21:18:38.972072+0000","last_undegraded":"2023-09-23T21:18:38.972072+0000","last_fullsized":"2023-09-23T21:18:38.972072+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:16:49.009013+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":194,"reported_epoch":619,"state":"active+clean","last_fresh":"2023-09-23T21:18:40.881959+0000","last_change":"2023-09-23T21:13:12.216895+0000","last_active":"2023-09-23T21:18:40.881959+0000","last_peered":"2023-09-23T21:18:40.881959+0000","last_clean":"2023-09-23T21:18:40.881959+0000","last_became_active":"2023-09-23T21:13:12.216654+0000","last_became_peered":"2023-09-23T21:13:12.216654+0000","last_unstale":"2023-09-23T21:18:40.881959+0000","last_undegraded":"2023-09-23T21:18:40.881959+0000","last_fullsized":"2023-09-23T21:18:40.881959+0000","mapping_epoch":493,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:02:40.430096+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.13","version":"618'7790","reported_seq":10664,"reported_epoch":620,"state":"peering","last_fresh":"2023-09-23T21:18:41.885763+0000","last_change":"2023-09-23T21:18:41.886040+0000","last_active":"2023-09-23T21:18:41.885728+0000","last_peered":"2023-09-23T21:18:41.012863+0000","last_clean":"2023-09-23T21:18:37.973115+0000","last_became_active":"2023-09-23T21:18:40.055201+0000","last_became_peered":"2023-09-23T21:18:40.055201+0000","last_unstale":"2023-09-23T21:18:41.885763+0000","last_undegraded":"2023-09-23T21:18:41.885763+0000","last_fullsized":"2023-09-23T21:18:41.885763+0000","mapping_epoch":620,"log_start":"549'2514","ondisk_log_start":"549'2514","created":559,"last_epoch_clean":619,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_clean_scrub_stamp":"2023-09-23T21:15:24.860968+0000","objects_scrubbed":0,"log_size":5276,"log_dups_size":0,"ondisk_log_size":5276,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26607616,"num_objects":406,"num_object_clones":0,"num_object_copies":1218,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":406,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5574,"num_write_kb":191360,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1172,"num_bytes_recovered":76742656,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[0,2,5,7],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.19","version":"79'161","reported_seq":826,"reported_epoch":617,"state":"active+clean","last_fresh":"2023-09-23T21:18:38.970808+0000","last_change":"2023-09-23T21:16:45.428879+0000","last_active":"2023-09-23T21:18:38.970808+0000","last_peered":"2023-09-23T21:18:38.970808+0000","last_clean":"2023-09-23T21:18:38.970808+0000","last_became_active":"2023-09-23T21:16:45.428660+0000","last_became_peered":"2023-09-23T21:16:45.428660+0000","last_unstale":"2023-09-23T21:18:38.970808+0000","last_undegraded":"2023-09-23T21:18:38.970808+0000","last_fullsized":"2023-09-23T21:18:38.970808+0000","mapping_epoch":567,"log_start":"0'0","ondisk_log_start":"0'0","created":479,"last_epoch_clean":568,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:15:48.271791+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:15:48.271791+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:25:26.361045+0000","scrub_duration":0.0057149180000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":147,"reported_epoch":617,"state":"active+clean","last_fresh":"2023-09-23T21:18:38.971890+0000","last_change":"2023-09-23T21:16:42.434830+0000","last_active":"2023-09-23T21:18:38.971890+0000","last_peered":"2023-09-23T21:18:38.971890+0000","last_clean":"2023-09-23T21:18:38.971890+0000","last_became_active":"2023-09-23T21:16:42.430289+0000","last_became_peered":"2023-09-23T21:16:42.430289+0000","last_unstale":"2023-09-23T21:18:38.971890+0000","last_undegraded":"2023-09-23T21:18:38.971890+0000","last_fullsized":"2023-09-23T21:18:38.971890+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:38:48.053766+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":124,"reported_epoch":619,"state":"active+clean","last_fresh":"2023-09-23T21:18:40.883243+0000","last_change":"2023-09-23T21:13:38.373814+0000","last_active":"2023-09-23T21:18:40.883243+0000","last_peered":"2023-09-23T21:18:40.883243+0000","last_clean":"2023-09-23T21:18:40.883243+0000","last_became_active":"2023-09-23T21:13:38.373458+0000","last_became_peered":"2023-09-23T21:13:38.373458+0000","last_unstale":"2023-09-23T21:18:40.883243+0000","last_undegraded":"2023-09-23T21:18:40.883243+0000","last_fullsized":"2023-09-23T21:18:40.883243+0000","mapping_epoch":507,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:23:08.288401+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"11.14","version":"616'7891","reported_seq":9610,"reported_epoch":616,"state":"remapped+peering","last_fresh":"2023-09-23T21:18:38.602203+0000","last_change":"2023-09-23T21:18:40.882769+0000","last_active":"2023-09-23T21:18:40.882769+0000","last_peered":"2023-09-23T21:18:38.602203+0000","last_clean":"2023-09-23T21:18:38.602203+0000","last_became_active":"2023-09-23T21:18:17.894915+0000","last_became_peered":"2023-09-23T21:18:17.894915+0000","last_unstale":"2023-09-23T21:18:38.602203+0000","last_undegraded":"2023-09-23T21:18:38.602203+0000","last_fullsized":"2023-09-23T21:18:38.602203+0000","mapping_epoch":619,"log_start":"550'2789","ondisk_log_start":"550'2789","created":559,"last_epoch_clean":601,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_clean_scrub_stamp":"2023-09-23T21:15:24.860968+0000","objects_scrubbed":0,"log_size":5102,"log_dups_size":0,"ondisk_log_size":5102,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":23330816,"num_objects":356,"num_object_clones":0,"num_object_copies":1068,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":356,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5654,"num_write_kb":192320,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":803,"num_bytes_recovered":32505856,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[2,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[3,4,5,7],"up_primary":4,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.1e","version":"79'161","reported_seq":600,"reported_epoch":616,"state":"active+clean","last_fresh":"2023-09-23T21:18:37.973224+0000","last_change":"2023-09-23T21:16:45.420788+0000","last_active":"2023-09-23T21:18:37.973224+0000","last_peered":"2023-09-23T21:18:37.973224+0000","last_clean":"2023-09-23T21:18:37.973224+0000","last_became_active":"2023-09-23T21:16:45.420448+0000","last_became_peered":"2023-09-23T21:16:45.420448+0000","last_unstale":"2023-09-23T21:18:37.973224+0000","last_undegraded":"2023-09-23T21:18:37.973224+0000","last_fullsized":"2023-09-23T21:18:37.973224+0000","mapping_epoch":567,"log_start":"0'0","ondisk_log_start":"0'0","created":479,"last_epoch_clean":568,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:15:53.061319+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:15:53.061319+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:46:31.982439+0000","scrub_duration":0.007429878,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,4],"acting":[3,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":229,"reported_epoch":619,"state":"active+clean","last_fresh":"2023-09-23T21:18:40.882345+0000","last_change":"2023-09-23T21:16:42.518584+0000","last_active":"2023-09-23T21:18:40.882345+0000","last_peered":"2023-09-23T21:18:40.882345+0000","last_clean":"2023-09-23T21:18:40.882345+0000","last_became_active":"2023-09-23T21:16:42.416632+0000","last_became_peered":"2023-09-23T21:16:42.416632+0000","last_unstale":"2023-09-23T21:18:40.882345+0000","last_undegraded":"2023-09-23T21:18:40.882345+0000","last_fullsized":"2023-09-23T21:18:40.882345+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:02:18.972080+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,7],"acting":[4,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":184,"reported_epoch":619,"state":"active+clean","last_fresh":"2023-09-23T21:18:40.882201+0000","last_change":"2023-09-23T21:12:10.770474+0000","last_active":"2023-09-23T21:18:40.882201+0000","last_peered":"2023-09-23T21:18:40.882201+0000","last_clean":"2023-09-23T21:18:40.882201+0000","last_became_active":"2023-09-23T21:12:10.770184+0000","last_became_peered":"2023-09-23T21:12:10.770184+0000","last_unstale":"2023-09-23T21:18:40.882201+0000","last_undegraded":"2023-09-23T21:18:40.882201+0000","last_fullsized":"2023-09-23T21:18:40.882201+0000","mapping_epoch":470,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:05:09.407151+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.15","version":"615'8046","reported_seq":13336,"reported_epoch":615,"state":"active+clean","last_fresh":"2023-09-23T21:18:37.312238+0000","last_change":"2023-09-23T21:18:31.392646+0000","last_active":"2023-09-23T21:18:37.312238+0000","last_peered":"2023-09-23T21:18:37.312238+0000","last_clean":"2023-09-23T21:18:37.312238+0000","last_became_active":"2023-09-23T21:16:46.418280+0000","last_became_peered":"2023-09-23T21:16:46.418280+0000","last_unstale":"2023-09-23T21:18:37.312238+0000","last_undegraded":"2023-09-23T21:18:37.312238+0000","last_fullsized":"2023-09-23T21:18:37.312238+0000","mapping_epoch":568,"log_start":"549'1575","ondisk_log_start":"549'1575","created":559,"last_epoch_clean":569,"parent":"0.0","parent_split_bits":5,"last_scrub":"610'7908","last_scrub_stamp":"2023-09-23T21:18:31.392582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_clean_scrub_stamp":"2023-09-23T21:18:31.392582+0000","objects_scrubbed":603,"log_size":6471,"log_dups_size":0,"ondisk_log_size":6471,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:27:26.319214+0000","scrub_duration":0.067034226000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":30474240,"num_objects":465,"num_object_clones":0,"num_object_copies":1395,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":465,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":138,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":410,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-09-23T21:18:39.984281+0000","last_change":"2023-09-23T21:12:10.771160+0000","last_active":"2023-09-23T21:18:39.984281+0000","last_peered":"2023-09-23T21:18:39.984281+0000","last_clean":"2023-09-23T21:18:39.984281+0000","last_became_active":"2023-09-23T21:12:10.757808+0000","last_became_peered":"2023-09-23T21:12:10.757808+0000","last_unstale":"2023-09-23T21:18:39.984281+0000","last_undegraded":"2023-09-23T21:18:39.984281+0000","last_fullsized":"2023-09-23T21:18:39.984281+0000","mapping_epoch":470,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:30:50.987139+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":194,"reported_epoch":619,"state":"active+clean","last_fresh":"2023-09-23T21:18:40.882038+0000","last_change":"2023-09-23T21:13:12.216602+0000","last_active":"2023-09-23T21:18:40.882038+0000","last_peered":"2023-09-23T21:18:40.882038+0000","last_clean":"2023-09-23T21:18:40.882038+0000","last_became_active":"2023-09-23T21:13:12.216204+0000","last_became_peered":"2023-09-23T21:13:12.216204+0000","last_unstale":"2023-09-23T21:18:40.882038+0000","last_undegraded":"2023-09-23T21:18:40.882038+0000","last_fullsized":"2023-09-23T21:18:40.882038+0000","mapping_epoch":493,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:23:37.043390+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.16","version":"618'7776","reported_seq":7893,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-09-23T21:18:40.218283+0000","last_change":"2023-09-23T21:18:31.602663+0000","last_active":"2023-09-23T21:18:40.218283+0000","last_peered":"2023-09-23T21:18:40.218283+0000","last_clean":"2023-09-23T21:18:40.218283+0000","last_became_active":"2023-09-23T21:16:32.926127+0000","last_became_peered":"2023-09-23T21:16:32.926127+0000","last_unstale":"2023-09-23T21:18:40.218283+0000","last_undegraded":"2023-09-23T21:18:40.218283+0000","last_fullsized":"2023-09-23T21:18:40.218283+0000","mapping_epoch":559,"log_start":"550'3033","ondisk_log_start":"550'3033","created":559,"last_epoch_clean":561,"parent":"0.0","parent_split_bits":5,"last_scrub":"610'7609","last_scrub_stamp":"2023-09-23T21:18:31.602481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_clean_scrub_stamp":"2023-09-23T21:18:31.602481+0000","objects_scrubbed":532,"log_size":4743,"log_dups_size":0,"ondisk_log_size":4743,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:58:12.261619+0000","scrub_duration":0.058905792999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":23920640,"num_objects":365,"num_object_clones":0,"num_object_copies":1095,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":365,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":167,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1c","version":"79'161","reported_seq":1047,"reported_epoch":616,"state":"active+clean","last_fresh":"2023-09-23T21:18:37.974685+0000","last_change":"2023-09-23T21:16:45.419779+0000","last_active":"2023-09-23T21:18:37.974685+0000","last_peered":"2023-09-23T21:18:37.974685+0000","last_clean":"2023-09-23T21:18:37.974685+0000","last_became_active":"2023-09-23T21:16:45.419394+0000","last_became_peered":"2023-09-23T21:16:45.419394+0000","last_unstale":"2023-09-23T21:18:37.974685+0000","last_undegraded":"2023-09-23T21:18:37.974685+0000","last_fullsized":"2023-09-23T21:18:37.974685+0000","mapping_epoch":567,"log_start":"0'0","ondisk_log_start":"0'0","created":479,"last_epoch_clean":568,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:15:54.019828+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:15:54.019828+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:59:05.422258+0000","scrub_duration":0.0020962530000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":228,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-09-23T21:18:39.984088+0000","last_change":"2023-09-23T21:16:42.424187+0000","last_active":"2023-09-23T21:18:39.984088+0000","last_peered":"2023-09-23T21:18:39.984088+0000","last_clean":"2023-09-23T21:18:39.984088+0000","last_became_active":"2023-09-23T21:16:42.423805+0000","last_became_peered":"2023-09-23T21:16:42.423805+0000","last_unstale":"2023-09-23T21:18:39.984088+0000","last_undegraded":"2023-09-23T21:18:39.984088+0000","last_fullsized":"2023-09-23T21:18:39.984088+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:55:46.051199+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,7],"acting":[2,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":288,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-09-23T21:18:39.983775+0000","last_change":"2023-09-23T21:13:12.200767+0000","last_active":"2023-09-23T21:18:39.983775+0000","last_peered":"2023-09-23T21:18:39.983775+0000","last_clean":"2023-09-23T21:18:39.983775+0000","last_became_active":"2023-09-23T21:13:12.200435+0000","last_became_peered":"2023-09-23T21:13:12.200435+0000","last_unstale":"2023-09-23T21:18:39.983775+0000","last_undegraded":"2023-09-23T21:18:39.983775+0000","last_fullsized":"2023-09-23T21:18:39.983775+0000","mapping_epoch":493,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:15:44.937886+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"11.17","version":"620'8124","reported_seq":8259,"reported_epoch":620,"state":"active+clean","last_fresh":"2023-09-23T21:18:42.047419+0000","last_change":"2023-09-23T21:18:33.499324+0000","last_active":"2023-09-23T21:18:42.047419+0000","last_peered":"2023-09-23T21:18:42.047419+0000","last_clean":"2023-09-23T21:18:42.047419+0000","last_became_active":"2023-09-23T21:16:32.890013+0000","last_became_peered":"2023-09-23T21:16:32.890013+0000","last_unstale":"2023-09-23T21:18:42.047419+0000","last_undegraded":"2023-09-23T21:18:42.047419+0000","last_fullsized":"2023-09-23T21:18:42.047419+0000","mapping_epoch":559,"log_start":"549'2091","ondisk_log_start":"549'2091","created":559,"last_epoch_clean":560,"parent":"0.0","parent_split_bits":5,"last_scrub":"612'7976","last_scrub_stamp":"2023-09-23T21:18:33.499229+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_clean_scrub_stamp":"2023-09-23T21:18:33.499229+0000","objects_scrubbed":569,"log_size":6033,"log_dups_size":0,"ondisk_log_size":6033,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:33:17.311461+0000","scrub_duration":0.14642005499999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":27394048,"num_objects":418,"num_object_clones":0,"num_object_copies":1254,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":418,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":151,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1d","version":"79'161","reported_seq":750,"reported_epoch":616,"state":"active+clean","last_fresh":"2023-09-23T21:18:37.973990+0000","last_change":"2023-09-23T21:18:32.873842+0000","last_active":"2023-09-23T21:18:37.973990+0000","last_peered":"2023-09-23T21:18:37.973990+0000","last_clean":"2023-09-23T21:18:37.973990+0000","last_became_active":"2023-09-23T21:15:23.801704+0000","last_became_peered":"2023-09-23T21:15:23.801704+0000","last_unstale":"2023-09-23T21:18:37.973990+0000","last_undegraded":"2023-09-23T21:18:37.973990+0000","last_fullsized":"2023-09-23T21:18:37.973990+0000","mapping_epoch":542,"log_start":"0'0","ondisk_log_start":"0'0","created":479,"last_epoch_clean":543,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:09:36.858077+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:09:36.858077+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.010304610000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,2],"acting":[3,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":146,"reported_epoch":617,"state":"active+clean","last_fresh":"2023-09-23T21:18:38.971876+0000","last_change":"2023-09-23T21:13:38.380574+0000","last_active":"2023-09-23T21:18:38.971876+0000","last_peered":"2023-09-23T21:18:38.971876+0000","last_clean":"2023-09-23T21:18:38.971876+0000","last_became_active":"2023-09-23T21:13:38.373252+0000","last_became_peered":"2023-09-23T21:13:38.373252+0000","last_unstale":"2023-09-23T21:18:38.971876+0000","last_undegraded":"2023-09-23T21:18:38.971876+0000","last_fullsized":"2023-09-23T21:18:38.971876+0000","mapping_epoch":507,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:58:14.788031+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":11,"num_bytes_recovered":3324,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":194,"reported_epoch":619,"state":"active+clean","last_fresh":"2023-09-23T21:18:40.881869+0000","last_change":"2023-09-23T21:12:10.793610+0000","last_active":"2023-09-23T21:18:40.881869+0000","last_peered":"2023-09-23T21:18:40.881869+0000","last_clean":"2023-09-23T21:18:40.881869+0000","last_became_active":"2023-09-23T21:12:10.793081+0000","last_became_peered":"2023-09-23T21:12:10.793081+0000","last_unstale":"2023-09-23T21:18:40.881869+0000","last_undegraded":"2023-09-23T21:18:40.881869+0000","last_fullsized":"2023-09-23T21:18:40.881869+0000","mapping_epoch":470,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:34:36.291550+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":11,"num_pg":26,"stat_sum":{"num_bytes":856784920,"num_objects":13074,"num_object_clones":0,"num_object_copies":39222,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":13074,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":110022,"num_write_kb":3713185,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":11380,"num_bytes_recovered":715259904,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2827038720,"data_stored":2827026504,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":135933,"ondisk_log_size":135933,"up":78,"acting":78,"num_store_stats":6},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":80,"num_bytes_recovered":18686,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":118784,"data_stored":21350,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":14322,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":31,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":525376,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2109440,"data_stored":2098784,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4991,"ondisk_log_size":4991,"up":93,"acting":93,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201739,"num_pgs":70,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1101756,"kb_used_data":469620,"kb_used_omap":0,"kb_used_meta":632128,"kb_avail":92643396,"statfs":{"total":95995035648,"available":94866837504,"internally_reserved":0,"allocated":480890880,"data_stored":479159687,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":647299072},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1],"upper_bound":2},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:18:22 2023","interfaces":[{"interface":"back","average":{"1min":1.5209999999999999,"5min":1.474,"15min":2.1080000000000001},"min":{"1min":0.22900000000000001,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":3.968,"5min":37.207999999999998,"15min":37.207999999999998},"last":1.585},{"interface":"front","average":{"1min":1.73,"5min":1.748,"15min":2.0720000000000001},"min":{"1min":0.318,"5min":0.253,"15min":0.253},"max":{"1min":5.2910000000000004,"5min":37.122,"15min":37.122},"last":0.24099999999999999}]},{"osd":1,"last update":"Sat Sep 23 21:18:16 2023","interfaces":[{"interface":"back","average":{"1min":1.2609999999999999,"5min":1.2210000000000001,"15min":1.7969999999999999},"min":{"1min":0.27600000000000002,"5min":0.27600000000000002,"15min":0.26700000000000002},"max":{"1min":6.9130000000000003,"5min":7.5369999999999999,"15min":14.034000000000001},"last":0.47899999999999998},{"interface":"front","average":{"1min":1.4370000000000001,"5min":1.379,"15min":1.776},"min":{"1min":0.34300000000000003,"5min":0.29999999999999999,"15min":0.26300000000000001},"max":{"1min":3.9300000000000002,"5min":11.002000000000001,"15min":11.226000000000001},"last":0.60499999999999998}]},{"osd":2,"last update":"Sat Sep 23 21:18:38 2023","interfaces":[{"interface":"back","average":{"1min":2.105,"5min":2.0739999999999998,"15min":1.8069999999999999},"min":{"1min":0.27200000000000002,"5min":0.27200000000000002,"15min":0.26200000000000001},"max":{"1min":18.488,"5min":25.471,"15min":25.471},"last":0.51900000000000002},{"interface":"front","average":{"1min":1.9330000000000001,"5min":1.857,"15min":1.752},"min":{"1min":0.28799999999999998,"5min":0.28799999999999998,"15min":0.26700000000000002},"max":{"1min":17.536000000000001,"5min":27.187999999999999,"15min":27.187999999999999},"last":0.68200000000000005}]},{"osd":3,"last update":"Sat Sep 23 21:18:04 2023","interfaces":[{"interface":"back","average":{"1min":1.7070000000000001,"5min":1.484,"15min":1.8280000000000001},"min":{"1min":0.33200000000000002,"5min":0.30599999999999999,"15min":0.29099999999999998},"max":{"1min":6.8739999999999997,"5min":8.6259999999999994,"15min":12.359999999999999},"last":2.2050000000000001},{"interface":"front","average":{"1min":1.1180000000000001,"5min":1.732,"15min":1.831},"min":{"1min":0.28799999999999998,"5min":0.28799999999999998,"15min":0.28699999999999998},"max":{"1min":3.8300000000000001,"5min":15.82,"15min":15.82},"last":1.4570000000000001}]},{"osd":4,"last update":"Sat Sep 23 21:18:04 2023","interfaces":[{"interface":"back","average":{"1min":0.93400000000000005,"5min":1.02,"15min":1.1799999999999999},"min":{"1min":0.251,"5min":0.251,"15min":0.248},"max":{"1min":3.9990000000000001,"5min":7.6609999999999996,"15min":11.297000000000001},"last":0.40300000000000002},{"interface":"front","average":{"1min":1.0089999999999999,"5min":1.073,"15min":1.137},"min":{"1min":0.28599999999999998,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":5.6349999999999998,"5min":8.6059999999999999,"15min":12.752000000000001},"last":0.33300000000000002}]},{"osd":5,"last update":"Sat Sep 23 21:18:04 2023","interfaces":[{"interface":"back","average":{"1min":1.286,"5min":1.095,"15min":1.089},"min":{"1min":0.28999999999999998,"5min":0.28999999999999998,"15min":0.26000000000000001},"max":{"1min":6.9740000000000002,"5min":7.5910000000000002,"15min":9.4789999999999992},"last":0.53800000000000003},{"interface":"front","average":{"1min":1.381,"5min":1.171,"15min":1.2130000000000001},"min":{"1min":0.35599999999999998,"5min":0.32000000000000001,"15min":0.26200000000000001},"max":{"1min":6.7439999999999998,"5min":7.6929999999999996,"15min":11.257},"last":0.373}]},{"osd":6,"last update":"Sat Sep 23 21:18:16 2023","interfaces":[{"interface":"back","average":{"1min":0.86099999999999999,"5min":0.88300000000000001,"15min":1.0620000000000001},"min":{"1min":0.34599999999999997,"5min":0.32000000000000001,"15min":0.30099999999999999},"max":{"1min":5.7069999999999999,"5min":7.5339999999999998,"15min":10.941000000000001},"last":0.42299999999999999},{"interface":"front","average":{"1min":1.379,"5min":1.0429999999999999,"15min":1.181},"min":{"1min":0.41099999999999998,"5min":0.36299999999999999,"15min":0.33100000000000002},"max":{"1min":5.6760000000000002,"5min":5.6760000000000002,"15min":12.715},"last":0.501}]}]},{"osd":6,"up_from":0,"seq":704374636822,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":1,"up_from":0,"seq":1503238553754,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":0,"up_from":493,"seq":2117418876995,"num_pgs":74,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":921856,"kb_used_data":544376,"kb_used_omap":36,"kb_used_meta":377435,"kb_avail":92823296,"statfs":{"total":95995035648,"available":95051055104,"internally_reserved":0,"allocated":557441024,"data_stored":554594391,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":37336,"internal_metadata":386493992},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 21:18:27 2023","interfaces":[{"interface":"back","average":{"1min":1.3620000000000001,"5min":1.4410000000000001,"15min":0.98399999999999999},"min":{"1min":0.25900000000000001,"5min":0.214,"15min":0.214},"max":{"1min":6.7779999999999996,"5min":13.779999999999999,"15min":13.779999999999999},"last":0.23000000000000001},{"interface":"front","average":{"1min":2.5499999999999998,"5min":1.7569999999999999,"15min":1.099},"min":{"1min":0.35099999999999998,"5min":0.20799999999999999,"15min":0.20799999999999999},"max":{"1min":10.551,"5min":11.964,"15min":11.964},"last":0.316}]},{"osd":2,"last update":"Sat Sep 23 21:18:27 2023","interfaces":[{"interface":"back","average":{"1min":2.0579999999999998,"5min":1.8,"15min":1.036},"min":{"1min":0.36599999999999999,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":7.3449999999999998,"5min":13.637,"15min":13.637},"last":0.27500000000000002},{"interface":"front","average":{"1min":3.258,"5min":2.46,"15min":1.4650000000000001},"min":{"1min":0.30199999999999999,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":10.401999999999999,"5min":26.571000000000002,"15min":26.571000000000002},"last":1.9470000000000001}]},{"osd":3,"last update":"Sat Sep 23 21:18:27 2023","interfaces":[{"interface":"back","average":{"1min":3.9950000000000001,"5min":2.6000000000000001,"15min":1.4450000000000001},"min":{"1min":0.253,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":37.079999999999998,"5min":37.079999999999998,"15min":37.079999999999998},"last":0.255},{"interface":"front","average":{"1min":3.4550000000000001,"5min":2.2029999999999998,"15min":1.4790000000000001},"min":{"1min":0.34699999999999998,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":32.935000000000002,"5min":32.935000000000002,"15min":32.935000000000002},"last":0.29299999999999998}]},{"osd":4,"last update":"Sat Sep 23 21:18:27 2023","interfaces":[{"interface":"back","average":{"1min":1.546,"5min":1.7390000000000001,"15min":1.3080000000000001},"min":{"1min":0.34499999999999997,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":4.7880000000000003,"5min":22.780000000000001,"15min":22.780000000000001},"last":0.34200000000000003},{"interface":"front","average":{"1min":1.9390000000000001,"5min":1.4990000000000001,"15min":1.163},"min":{"1min":0.32500000000000001,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":6.9119999999999999,"5min":22.652000000000001,"15min":22.652000000000001},"last":0.75900000000000001}]},{"osd":5,"last update":"Sat Sep 23 21:18:27 2023","interfaces":[{"interface":"back","average":{"1min":1.5549999999999999,"5min":1.5680000000000001,"15min":1.3460000000000001},"min":{"1min":0.312,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":6.1139999999999999,"5min":8.9329999999999998,"15min":8.9329999999999998},"last":0.38800000000000001},{"interface":"front","average":{"1min":1.9039999999999999,"5min":1.696,"15min":1.274},"min":{"1min":0.40100000000000002,"5min":0.313,"15min":0.313},"max":{"1min":6.5990000000000002,"5min":15.321999999999999,"15min":15.321999999999999},"last":0.377}]},{"osd":6,"last update":"Sat Sep 23 21:18:27 2023","interfaces":[{"interface":"back","average":{"1min":1.5720000000000001,"5min":1.4810000000000001,"15min":1.2450000000000001},"min":{"1min":0.372,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":6.9409999999999998,"5min":7.2050000000000001,"15min":7.2050000000000001},"last":0.81999999999999995},{"interface":"front","average":{"1min":1.647,"5min":1.595,"15min":1.403},"min":{"1min":0.38400000000000001,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":6.266,"5min":8.8889999999999993,"15min":8.8889999999999993},"last":0.56799999999999995}]},{"osd":7,"last update":"Sat Sep 23 21:18:27 2023","interfaces":[{"interface":"back","average":{"1min":1.595,"5min":1.8240000000000001,"15min":1.4139999999999999},"min":{"1min":0.34699999999999998,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":4.8360000000000003,"5min":15.273999999999999,"15min":15.273999999999999},"last":0.40899999999999997},{"interface":"front","average":{"1min":1.75,"5min":1.7709999999999999,"15min":1.3620000000000001},"min":{"1min":0.36199999999999999,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":6.9109999999999996,"5min":33.779000000000003,"15min":33.779000000000003},"last":0.79200000000000004}]}]},{"osd":2,"up_from":93,"seq":399431958867,"num_pgs":53,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1521532,"kb_used_data":507828,"kb_used_omap":81,"kb_used_meta":1013614,"kb_avail":92223620,"statfs":{"total":95995035648,"available":94436986880,"internally_reserved":0,"allocated":520015872,"data_stored":517125941,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":83750,"internal_metadata":1037940954},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:18:23 2023","interfaces":[{"interface":"back","average":{"1min":2.6339999999999999,"5min":1.9370000000000001,"15min":1.2010000000000001},"min":{"1min":0.221,"5min":0.187,"15min":0.187},"max":{"1min":9.9499999999999993,"5min":13.25,"15min":13.25},"last":2.5739999999999998},{"interface":"front","average":{"1min":2.2509999999999999,"5min":2.089,"15min":1.806},"min":{"1min":0.26800000000000002,"5min":0.20200000000000001,"15min":0.20200000000000001},"max":{"1min":9.8849999999999998,"5min":18.111999999999998,"15min":18.111999999999998},"last":3.367}]},{"osd":1,"last update":"Sat Sep 23 21:18:23 2023","interfaces":[{"interface":"back","average":{"1min":1.645,"5min":1.756,"15min":2.2629999999999999},"min":{"1min":0.34000000000000002,"5min":0.20599999999999999,"15min":0.188},"max":{"1min":10.535,"5min":10.535,"15min":34.396999999999998},"last":5.7069999999999999},{"interface":"front","average":{"1min":2.2240000000000002,"5min":1.95,"15min":2.3460000000000001},"min":{"1min":0.24099999999999999,"5min":0.24099999999999999,"15min":0.217},"max":{"1min":6.0990000000000002,"5min":18.081,"15min":34.462000000000003},"last":3.4980000000000002}]},{"osd":3,"last update":"Sat Sep 23 21:18:01 2023","interfaces":[{"interface":"back","average":{"1min":2.0880000000000001,"5min":2.04,"15min":2.2389999999999999},"min":{"1min":0.248,"5min":0.20599999999999999,"15min":0.20599999999999999},"max":{"1min":7.3719999999999999,"5min":20.106999999999999,"15min":35.549999999999997},"last":0.40100000000000002},{"interface":"front","average":{"1min":2.6659999999999999,"5min":2.556,"15min":2.6099999999999999},"min":{"1min":0.35799999999999998,"5min":0.28499999999999998,"15min":0.21299999999999999},"max":{"1min":10.162000000000001,"5min":20.239000000000001,"15min":36.951999999999998},"last":5.6660000000000004}]},{"osd":4,"last update":"Sat Sep 23 21:18:23 2023","interfaces":[{"interface":"back","average":{"1min":1.2749999999999999,"5min":1.3460000000000001,"15min":1.4450000000000001},"min":{"1min":0.27000000000000002,"5min":0.27000000000000002,"15min":0.249},"max":{"1min":3.8530000000000002,"5min":7.9770000000000003,"15min":10.266999999999999},"last":0.30599999999999999},{"interface":"front","average":{"1min":1.363,"5min":1.5640000000000001,"15min":1.7110000000000001},"min":{"1min":0.38900000000000001,"5min":0.30299999999999999,"15min":0.26700000000000002},"max":{"1min":4.2210000000000001,"5min":10.459,"15min":12.49},"last":2.508}]},{"osd":5,"last update":"Sat Sep 23 21:18:23 2023","interfaces":[{"interface":"back","average":{"1min":1.6160000000000001,"5min":1.456,"15min":1.5349999999999999},"min":{"1min":0.32300000000000001,"5min":0.29099999999999998,"15min":0.28000000000000003},"max":{"1min":5.4139999999999997,"5min":7.6180000000000003,"15min":9.5039999999999996},"last":1.071},{"interface":"front","average":{"1min":1.474,"5min":1.5169999999999999,"15min":1.6890000000000001},"min":{"1min":0.34899999999999998,"5min":0.33800000000000002,"15min":0.29399999999999998},"max":{"1min":4.2649999999999997,"5min":10.425000000000001,"15min":12.57},"last":2.544}]},{"osd":6,"last update":"Sat Sep 23 21:18:23 2023","interfaces":[{"interface":"back","average":{"1min":1.542,"5min":1.633,"15min":1.8979999999999999},"min":{"1min":0.54100000000000004,"5min":0.377,"15min":0.35699999999999998},"max":{"1min":4.2990000000000004,"5min":10.335000000000001,"15min":12.518000000000001},"last":2.601},{"interface":"front","average":{"1min":1.7190000000000001,"5min":1.696,"15min":2.0510000000000002},"min":{"1min":0.501,"5min":0.377,"15min":0.35599999999999998},"max":{"1min":4.7969999999999997,"5min":10.381,"15min":12.544},"last":2.6280000000000001}]},{"osd":7,"last update":"Sat Sep 23 21:18:23 2023","interfaces":[{"interface":"back","average":{"1min":1.6339999999999999,"5min":1.637,"15min":1.6519999999999999},"min":{"1min":0.31,"5min":0.31,"15min":0.29799999999999999},"max":{"1min":5.7480000000000002,"5min":18.021999999999998,"15min":18.021999999999998},"last":2.4199999999999999},{"interface":"front","average":{"1min":2.145,"5min":1.6339999999999999,"15min":1.73},"min":{"1min":0.35999999999999999,"5min":0.34999999999999998,"15min":0.32200000000000001},"max":{"1min":6.694,"5min":9.6349999999999998,"15min":10.92},"last":0.372}]}]},{"osd":3,"up_from":98,"seq":420906795340,"num_pgs":79,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1688684,"kb_used_data":605732,"kb_used_omap":7,"kb_used_meta":1082936,"kb_avail":92056468,"statfs":{"total":95995035648,"available":94265823232,"internally_reserved":0,"allocated":620269568,"data_stored":617435978,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7576,"internal_metadata":1108927080},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:18:18 2023","interfaces":[{"interface":"back","average":{"1min":2.2069999999999999,"5min":4.0750000000000002,"15min":1.583},"min":{"1min":0.22500000000000001,"5min":0.18099999999999999,"15min":0.18099999999999999},"max":{"1min":10.439,"5min":147.38900000000001,"15min":147.38900000000001},"last":2.7250000000000001},{"interface":"front","average":{"1min":1.1899999999999999,"5min":3.5859999999999999,"15min":1.6399999999999999},"min":{"1min":0.252,"5min":0.223,"15min":0.223},"max":{"1min":5.7279999999999998,"5min":149.989,"15min":149.989},"last":0.214}]},{"osd":1,"last update":"Sat Sep 23 21:18:18 2023","interfaces":[{"interface":"back","average":{"1min":2.2029999999999998,"5min":1.5940000000000001,"15min":2.1160000000000001},"min":{"1min":0.27000000000000002,"5min":0.218,"15min":0.20000000000000001},"max":{"1min":6.1150000000000002,"5min":7.2759999999999998,"15min":22.577999999999999},"last":6.5350000000000001},{"interface":"front","average":{"1min":1.827,"5min":1.2969999999999999,"15min":1.984},"min":{"1min":0.26100000000000001,"5min":0.249,"15min":0.216},"max":{"1min":6.1219999999999999,"5min":7.6829999999999998,"15min":21.617999999999999},"last":2.7799999999999998}]},{"osd":2,"last update":"Sat Sep 23 21:18:02 2023","interfaces":[{"interface":"back","average":{"1min":1.4750000000000001,"5min":1.958,"15min":2.004},"min":{"1min":0.27400000000000002,"5min":0.221,"15min":0.20899999999999999},"max":{"1min":11.321,"5min":56.317999999999998,"15min":56.317999999999998},"last":0.34999999999999998},{"interface":"front","average":{"1min":1.9910000000000001,"5min":2.4780000000000002,"15min":2.355},"min":{"1min":0.23400000000000001,"5min":0.23400000000000001,"15min":0.216},"max":{"1min":11.371,"5min":55.988,"15min":55.988},"last":6.4240000000000004}]},{"osd":4,"last update":"Sat Sep 23 21:18:02 2023","interfaces":[{"interface":"back","average":{"1min":0.72899999999999998,"5min":1.2010000000000001,"15min":1.5509999999999999},"min":{"1min":0.29199999999999998,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":2.5409999999999999,"5min":10.807,"15min":21.478999999999999},"last":0.41699999999999998},{"interface":"front","average":{"1min":1.091,"5min":1.3180000000000001,"15min":1.7829999999999999},"min":{"1min":0.35899999999999999,"5min":0.311,"15min":0.28499999999999998},"max":{"1min":3.129,"5min":7.3140000000000001,"15min":22.402000000000001},"last":6.4699999999999998}]},{"osd":5,"last update":"Sat Sep 23 21:18:02 2023","interfaces":[{"interface":"back","average":{"1min":0.83199999999999996,"5min":1.3160000000000001,"15min":1.6930000000000001},"min":{"1min":0.318,"5min":0.313,"15min":0.311},"max":{"1min":4.1379999999999999,"5min":10.882999999999999,"15min":21.562000000000001},"last":0.438},{"interface":"front","average":{"1min":1.571,"5min":1.234,"15min":1.8},"min":{"1min":0.42699999999999999,"5min":0.34300000000000003,"15min":0.31},"max":{"1min":5.7300000000000004,"5min":5.7300000000000004,"15min":14.029},"last":2.8010000000000002}]},{"osd":6,"last update":"Sat Sep 23 21:18:18 2023","interfaces":[{"interface":"back","average":{"1min":1.6180000000000001,"5min":1.464,"15min":1.839},"min":{"1min":0.39800000000000002,"5min":0.35899999999999999,"15min":0.33900000000000002},"max":{"1min":4.9749999999999996,"5min":7.3410000000000002,"15min":22.446999999999999},"last":6.5049999999999999},{"interface":"front","average":{"1min":1.657,"5min":1.3480000000000001,"15min":1.8280000000000001},"min":{"1min":0.373,"5min":0.373,"15min":0.373},"max":{"1min":7.5949999999999998,"5min":9.3889999999999993,"15min":12.375999999999999},"last":2.839}]},{"osd":7,"last update":"Sat Sep 23 21:18:02 2023","interfaces":[{"interface":"back","average":{"1min":1.0249999999999999,"5min":1.48,"15min":1.847},"min":{"1min":0.41299999999999998,"5min":0.41199999999999998,"15min":0.33900000000000002},"max":{"1min":4.242,"5min":10.914999999999999,"15min":21.518999999999998},"last":0.45400000000000001},{"interface":"front","average":{"1min":1.5409999999999999,"5min":1.528,"15min":1.9850000000000001},"min":{"1min":0.51800000000000002,"5min":0.42499999999999999,"15min":0.40300000000000002},"max":{"1min":4.6920000000000002,"5min":8.2590000000000003,"15min":12.624000000000001},"last":2.8210000000000002}]}]},{"osd":4,"up_from":31,"seq":133143986647,"num_pgs":63,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":915516,"kb_used_data":342516,"kb_used_omap":0,"kb_used_meta":572992,"kb_avail":92829636,"statfs":{"total":95995035648,"available":95057547264,"internally_reserved":0,"allocated":350736384,"data_stored":349004924,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":586743808},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,0,0,0,2,2,7,5],"upper_bound":2048},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:18:24 2023","interfaces":[{"interface":"back","average":{"1min":2.2690000000000001,"5min":1.673,"15min":1.157},"min":{"1min":0.245,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":20.515000000000001,"5min":28.683,"15min":28.683},"last":0.45700000000000002},{"interface":"front","average":{"1min":3.085,"5min":1.8009999999999999,"15min":1.3560000000000001},"min":{"1min":0.249,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":20.792000000000002,"5min":29.155000000000001,"15min":29.155000000000001},"last":0.27500000000000002}]},{"osd":1,"last update":"Sat Sep 23 21:18:24 2023","interfaces":[{"interface":"back","average":{"1min":1.7330000000000001,"5min":1.2829999999999999,"15min":2.1070000000000002},"min":{"1min":0.28499999999999998,"5min":0.26900000000000002,"15min":0.221},"max":{"1min":8.1020000000000003,"5min":8.1020000000000003,"15min":37.960000000000001},"last":0.378},{"interface":"front","average":{"1min":1.55,"5min":1.399,"15min":2.0550000000000002},"min":{"1min":0.33100000000000002,"5min":0.28499999999999998,"15min":0.27000000000000002},"max":{"1min":8.1509999999999998,"5min":13.983000000000001,"15min":37.933},"last":0.443}]},{"osd":2,"last update":"Sat Sep 23 21:18:24 2023","interfaces":[{"interface":"back","average":{"1min":2.177,"5min":1.8500000000000001,"15min":1.754},"min":{"1min":0.28399999999999997,"5min":0.26400000000000001,"15min":0.247},"max":{"1min":9.8399999999999999,"5min":17.602,"15min":17.602},"last":0.33500000000000002},{"interface":"front","average":{"1min":1.8839999999999999,"5min":1.7849999999999999,"15min":1.831},"min":{"1min":0.35399999999999998,"5min":0.311,"15min":0.27800000000000002},"max":{"1min":5.5250000000000004,"5min":14.339,"15min":14.417999999999999},"last":0.36699999999999999}]},{"osd":3,"last update":"Sat Sep 23 21:17:52 2023","interfaces":[{"interface":"back","average":{"1min":5.2910000000000004,"5min":2.1000000000000001,"15min":2.1299999999999999},"min":{"1min":0.312,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":67.105999999999995,"5min":67.105999999999995,"15min":67.105999999999995},"last":0.34999999999999998},{"interface":"front","average":{"1min":4.9290000000000003,"5min":2.1930000000000001,"15min":2.056},"min":{"1min":0.33600000000000002,"5min":0.32200000000000001,"15min":0.30199999999999999},"max":{"1min":67.046999999999997,"5min":67.046999999999997,"15min":67.046999999999997},"last":0.46999999999999997}]},{"osd":5,"last update":"Sat Sep 23 21:18:13 2023","interfaces":[{"interface":"back","average":{"1min":0.89300000000000002,"5min":1.1080000000000001,"15min":1.123},"min":{"1min":0.254,"5min":0.254,"15min":0.222},"max":{"1min":2.923,"5min":14.398999999999999,"15min":14.398999999999999},"last":0.30099999999999999},{"interface":"front","average":{"1min":1.0349999999999999,"5min":0.98799999999999999,"15min":1.1080000000000001},"min":{"1min":0.28100000000000003,"5min":0.25800000000000001,"15min":0.255},"max":{"1min":3.6000000000000001,"5min":8.016,"15min":9.8979999999999997},"last":0.40500000000000003}]},{"osd":6,"last update":"Sat Sep 23 21:18:24 2023","interfaces":[{"interface":"back","average":{"1min":1.077,"5min":0.96599999999999997,"15min":1.2649999999999999},"min":{"1min":0.308,"5min":0.27200000000000002,"15min":0.26800000000000002},"max":{"1min":3.2050000000000001,"5min":8.7420000000000009,"15min":24.452999999999999},"last":0.311},{"interface":"front","average":{"1min":0.83799999999999997,"5min":1.1499999999999999,"15min":1.0820000000000001},"min":{"1min":0.38,"5min":0.36399999999999999,"15min":0.32500000000000001},"max":{"1min":2.5409999999999999,"5min":14.374000000000001,"15min":14.387},"last":0.42199999999999999}]},{"osd":7,"last update":"Sat Sep 23 21:17:52 2023","interfaces":[{"interface":"back","average":{"1min":1.091,"5min":1.194,"15min":1.1919999999999999},"min":{"1min":0.33200000000000002,"5min":0.32000000000000001,"15min":0.30099999999999999},"max":{"1min":5.7789999999999999,"5min":14.422000000000001,"15min":16.890999999999998},"last":0.39300000000000002},{"interface":"front","average":{"1min":0.73399999999999999,"5min":1.034,"15min":1.2230000000000001},"min":{"1min":0.29199999999999998,"5min":0.29199999999999998,"15min":0.27400000000000002},"max":{"1min":3.681,"5min":7.7229999999999999,"15min":11.170999999999999},"last":0.48599999999999999}]}]},{"osd":5,"up_from":39,"seq":167503725013,"num_pgs":60,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":952824,"kb_used_data":349296,"kb_used_omap":0,"kb_used_meta":603520,"kb_avail":92792328,"statfs":{"total":95995035648,"available":95019343872,"internally_reserved":0,"allocated":357679104,"data_stored":355956133,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":618004480},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:18:16 2023","interfaces":[{"interface":"back","average":{"1min":1.452,"5min":1.234,"15min":0.88800000000000001},"min":{"1min":0.22500000000000001,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":5.9829999999999997,"5min":8.673,"15min":8.673},"last":0.73099999999999998},{"interface":"front","average":{"1min":2.5219999999999998,"5min":1.548,"15min":0.96199999999999997},"min":{"1min":0.215,"5min":0.215,"15min":0.215},"max":{"1min":12.259,"5min":12.259,"15min":12.259},"last":0.56299999999999994}]},{"osd":1,"last update":"Sat Sep 23 21:18:16 2023","interfaces":[{"interface":"back","average":{"1min":1.4490000000000001,"5min":1.254,"15min":1.625},"min":{"1min":0.32200000000000001,"5min":0.248,"15min":0.23499999999999999},"max":{"1min":4.3220000000000001,"5min":7.5839999999999996,"15min":25.239999999999998},"last":0.34300000000000003},{"interface":"front","average":{"1min":1.3700000000000001,"5min":1.304,"15min":1.7070000000000001},"min":{"1min":0.36899999999999999,"5min":0.28299999999999997,"15min":0.252},"max":{"1min":4.9500000000000002,"5min":11.116,"15min":25.253},"last":0.443}]},{"osd":2,"last update":"Sat Sep 23 21:18:35 2023","interfaces":[{"interface":"back","average":{"1min":1.895,"5min":1.5109999999999999,"15min":1.6419999999999999},"min":{"1min":0.27600000000000002,"5min":0.27300000000000002,"15min":0.25800000000000001},"max":{"1min":9.2479999999999993,"5min":9.6560000000000006,"15min":12.634},"last":0.80500000000000005},{"interface":"front","average":{"1min":2.0840000000000001,"5min":1.6000000000000001,"15min":1.663},"min":{"1min":0.29099999999999998,"5min":0.29099999999999998,"15min":0.25700000000000001},"max":{"1min":9.4060000000000006,"5min":9.4060000000000006,"15min":9.7170000000000005},"last":1.0449999999999999}]},{"osd":3,"last update":"Sat Sep 23 21:17:52 2023","interfaces":[{"interface":"back","average":{"1min":1.4259999999999999,"5min":1.4570000000000001,"15min":1.931},"min":{"1min":0.314,"5min":0.314,"15min":0.29299999999999998},"max":{"1min":3.6030000000000002,"5min":6.5940000000000003,"15min":25.503},"last":0.66600000000000004},{"interface":"front","average":{"1min":1.7310000000000001,"5min":1.748,"15min":2.157},"min":{"1min":0.311,"5min":0.311,"15min":0.30599999999999999},"max":{"1min":9.0619999999999994,"5min":12.382,"15min":25.466000000000001},"last":0.46899999999999997}]},{"osd":4,"last update":"Sat Sep 23 21:18:08 2023","interfaces":[{"interface":"back","average":{"1min":0.82399999999999995,"5min":0.91400000000000003,"15min":1.155},"min":{"1min":0.29999999999999999,"5min":0.253,"15min":0.253},"max":{"1min":5.0279999999999996,"5min":11.156000000000001,"15min":12.776},"last":0.377},{"interface":"front","average":{"1min":0.73399999999999999,"5min":1.008,"15min":1.1419999999999999},"min":{"1min":0.28599999999999998,"5min":0.246,"15min":0.246},"max":{"1min":3.415,"5min":9.7119999999999997,"15min":11.973000000000001},"last":0.505}]},{"osd":6,"last update":"Sat Sep 23 21:18:16 2023","interfaces":[{"interface":"back","average":{"1min":1.0649999999999999,"5min":0.99299999999999999,"15min":1.087},"min":{"1min":0.32200000000000001,"5min":0.28000000000000003,"15min":0.25900000000000001},"max":{"1min":5.8520000000000003,"5min":5.8520000000000003,"15min":8.0909999999999993},"last":0.54800000000000004},{"interface":"front","average":{"1min":1.095,"5min":1.0700000000000001,"15min":1.2330000000000001},"min":{"1min":0.39700000000000002,"5min":0.34300000000000003,"15min":0.317},"max":{"1min":7.1470000000000002,"5min":10.999000000000001,"15min":10.999000000000001},"last":0.57799999999999996}]},{"osd":7,"last update":"Sat Sep 23 21:17:52 2023","interfaces":[{"interface":"back","average":{"1min":1.109,"5min":1.012,"15min":1.819},"min":{"1min":0.36199999999999999,"5min":0.32400000000000001,"15min":0.309},"max":{"1min":4.2629999999999999,"5min":5.5659999999999998,"15min":189.405},"last":0.63500000000000001},{"interface":"front","average":{"1min":1.0649999999999999,"5min":1.1719999999999999,"15min":1.9670000000000001},"min":{"1min":0.39900000000000002,"5min":0.35199999999999998,"15min":0.32500000000000001},"max":{"1min":2.7879999999999998,"5min":11.183999999999999,"15min":189.685},"last":0.52200000000000002}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8488,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1259,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5322,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":4838,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":548601856,"data_stored":548601856,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":509874176,"data_stored":509870104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":612106240,"data_stored":612106240,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":341118976,"data_stored":341114904,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":349900800,"data_stored":349896728,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":465436672,"data_stored":465436672,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T21:18:43.824 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph status --format=json 2023-09-23T21:18:43.854 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:18:43.880 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:18:43.916 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:18:43.919 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:18:43.919 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:18:43.938 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:18:44.020 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:18:44.037 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:18:44.063 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:18:43 smithi026 ceph-mon[175151]: 11.3 scrub starts 2023-09-23T21:18:44.063 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:18:43 smithi026 ceph-mon[175151]: failed to encode map e621 with expected crc 2023-09-23T21:18:44.063 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:18:43 smithi026 ceph-mon[175151]: failed to encode map e621 with expected crc 2023-09-23T21:18:44.063 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:18:43 smithi026 ceph-mon[175151]: failed to encode map e621 with expected crc 2023-09-23T21:18:44.063 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:18:43 smithi026 ceph-mon[175151]: failed to encode map e621 with expected crc 2023-09-23T21:18:44.063 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:18:43 smithi026 ceph-mon[175151]: failed to encode map e621 with expected crc 2023-09-23T21:18:44.063 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:18:43 smithi026 ceph-mon[175151]: 11.14 scrub starts 2023-09-23T21:18:44.064 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:18:43 smithi026 ceph-mon[175151]: 11.14 scrub ok 2023-09-23T21:18:44.064 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:18:43 smithi026 ceph-mon[175151]: pgmap v1548: 129 pgs: 1 peering, 1 remapped+peering, 1 active+clean+scrubbing+deep, 126 active+clean; 818 MiB data, 6.8 GiB used, 530 GiB / 536 GiB avail 2023-09-23T21:18:44.064 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:18:43 smithi026 ceph-mon[175151]: from='client.36924 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T21:18:44.064 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:18:43 smithi026 ceph-mon[175151]: osdmap e622: 8 total, 8 up, 6 in 2023-09-23T21:18:44.064 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:18:43 smithi026 ceph-mon[179688]: 11.3 scrub starts 2023-09-23T21:18:44.064 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:18:43 smithi026 ceph-mon[179688]: failed to encode map e621 with expected crc 2023-09-23T21:18:44.064 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:18:43 smithi026 ceph-mon[179688]: failed to encode map e621 with expected crc 2023-09-23T21:18:44.065 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:18:43 smithi026 ceph-mon[179688]: failed to encode map e621 with expected crc 2023-09-23T21:18:44.065 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:18:43 smithi026 ceph-mon[179688]: failed to encode map e621 with expected crc 2023-09-23T21:18:44.065 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:18:43 smithi026 ceph-mon[179688]: failed to encode map e621 with expected crc 2023-09-23T21:18:44.065 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:18:43 smithi026 ceph-mon[179688]: 11.14 scrub starts 2023-09-23T21:18:44.065 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:18:43 smithi026 ceph-mon[179688]: 11.14 scrub ok 2023-09-23T21:18:44.065 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:18:43 smithi026 ceph-mon[179688]: pgmap v1548: 129 pgs: 1 peering, 1 remapped+peering, 1 active+clean+scrubbing+deep, 126 active+clean; 818 MiB data, 6.8 GiB used, 530 GiB / 536 GiB avail 2023-09-23T21:18:44.065 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:18:43 smithi026 ceph-mon[179688]: from='client.36924 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T21:18:44.065 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:18:43 smithi026 ceph-mon[179688]: osdmap e622: 8 total, 8 up, 6 in 2023-09-23T21:18:44.083 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:18:44.200 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0: {} 2023-09-23T21:18:44.242 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1: {} 2023-09-23T21:18:44.278 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:18:44.278 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2: {} 2023-09-23T21:18:44.285 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:18:44.286 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:18:44.326 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3: {} 2023-09-23T21:18:44.366 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4: {} 2023-09-23T21:18:44.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:18:43 smithi070 ceph-mon[168282]: 11.3 scrub starts 2023-09-23T21:18:44.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:18:43 smithi070 ceph-mon[168282]: failed to encode map e621 with expected crc 2023-09-23T21:18:44.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:18:43 smithi070 ceph-mon[168282]: failed to encode map e621 with expected crc 2023-09-23T21:18:44.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:18:43 smithi070 ceph-mon[168282]: failed to encode map e621 with expected crc 2023-09-23T21:18:44.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:18:43 smithi070 ceph-mon[168282]: failed to encode map e621 with expected crc 2023-09-23T21:18:44.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:18:43 smithi070 ceph-mon[168282]: failed to encode map e621 with expected crc 2023-09-23T21:18:44.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:18:43 smithi070 ceph-mon[168282]: 11.14 scrub starts 2023-09-23T21:18:44.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:18:43 smithi070 ceph-mon[168282]: 11.14 scrub ok 2023-09-23T21:18:44.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:18:43 smithi070 ceph-mon[168282]: pgmap v1548: 129 pgs: 1 peering, 1 remapped+peering, 1 active+clean+scrubbing+deep, 126 active+clean; 818 MiB data, 6.8 GiB used, 530 GiB / 536 GiB avail 2023-09-23T21:18:44.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:18:43 smithi070 ceph-mon[168282]: from='client.36924 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T21:18:44.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:18:43 smithi070 ceph-mon[168282]: osdmap e622: 8 total, 8 up, 6 in 2023-09-23T21:18:44.387 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:18:44.399 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T21:18:44.418 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:18:44.432 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6: {} 2023-09-23T21:18:44.456 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:18:44.459 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:18:44.460 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:18:44.472 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T21:18:44.482 INFO:teuthology.orchestra.run.smithi026.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T21:18:44.483 INFO:teuthology.orchestra.run.smithi026.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T21:18:44.483 INFO:teuthology.orchestra.run.smithi026.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T21:18:44.483 INFO:teuthology.orchestra.run.smithi026.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T21:18:44.483 INFO:teuthology.orchestra.run.smithi026.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T21:18:44.483 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T21:18:44.483 INFO:teuthology.orchestra.run.smithi026.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T21:18:44.483 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T21:18:44.561 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:18:44.604 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:18:44.920 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T21:18:44.925 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:18:44.925 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:18:45.026 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:18:45.062 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:18:45.106 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:18:45.112 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:18:45.112 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:18:45.213 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:18:45.254 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:18:45 smithi070 ceph-mon[168282]: 11.4 scrub starts 2023-09-23T21:18:45.254 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:18:45 smithi070 ceph-mon[168282]: 11.4 scrub ok 2023-09-23T21:18:45.254 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:18:45 smithi070 ceph-mon[168282]: failed to encode map e622 with expected crc 2023-09-23T21:18:45.254 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:18:45 smithi070 ceph-mon[168282]: failed to encode map e622 with expected crc 2023-09-23T21:18:45.255 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:18:45 smithi070 ceph-mon[168282]: 11.15 starting backfill to osd.2 from (0'0,0'0] MAX to 618'8100 2023-09-23T21:18:45.255 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:18:45 smithi070 ceph-mon[168282]: 11.15 starting backfill to osd.4 from (0'0,0'0] MAX to 618'8100 2023-09-23T21:18:45.255 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:18:45 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/3105162009' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:18:45.255 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:18:45 smithi070 ceph-mon[168282]: osdmap e623: 8 total, 8 up, 6 in 2023-09-23T21:18:45.255 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:18:45.315 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:18:45.319 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:18:45.320 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:18:45.384 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:18:45 smithi026 ceph-mon[175151]: 11.4 scrub starts 2023-09-23T21:18:45.385 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:18:45 smithi026 ceph-mon[175151]: 11.4 scrub ok 2023-09-23T21:18:45.385 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:18:45 smithi026 ceph-mon[175151]: failed to encode map e622 with expected crc 2023-09-23T21:18:45.385 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:18:45 smithi026 ceph-mon[175151]: failed to encode map e622 with expected crc 2023-09-23T21:18:45.385 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:18:45 smithi026 ceph-mon[175151]: 11.15 starting backfill to osd.2 from (0'0,0'0] MAX to 618'8100 2023-09-23T21:18:45.385 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:18:45 smithi026 ceph-mon[175151]: 11.15 starting backfill to osd.4 from (0'0,0'0] MAX to 618'8100 2023-09-23T21:18:45.385 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:18:45 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/3105162009' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:18:45.385 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:18:45 smithi026 ceph-mon[175151]: osdmap e623: 8 total, 8 up, 6 in 2023-09-23T21:18:45.385 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:18:45 smithi026 ceph-mon[179688]: 11.4 scrub starts 2023-09-23T21:18:45.386 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:18:45 smithi026 ceph-mon[179688]: 11.4 scrub ok 2023-09-23T21:18:45.386 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:18:45 smithi026 ceph-mon[179688]: failed to encode map e622 with expected crc 2023-09-23T21:18:45.386 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:18:45 smithi026 ceph-mon[179688]: failed to encode map e622 with expected crc 2023-09-23T21:18:45.386 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:18:45 smithi026 ceph-mon[179688]: 11.15 starting backfill to osd.2 from (0'0,0'0] MAX to 618'8100 2023-09-23T21:18:45.386 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:18:45 smithi026 ceph-mon[179688]: 11.15 starting backfill to osd.4 from (0'0,0'0] MAX to 618'8100 2023-09-23T21:18:45.386 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:18:45 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/3105162009' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:18:45.386 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:18:45 smithi026 ceph-mon[179688]: osdmap e623: 8 total, 8 up, 6 in 2023-09-23T21:18:45.421 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:18:45.496 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:18:46.027 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:18:46.032 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:18:46.033 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:18:46.133 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:18:46.168 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:18:46.215 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:18:46.220 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:18:46.221 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:18:46.322 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:18:46.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:18:46 smithi070 ceph-mon[168282]: 1.d scrub starts 2023-09-23T21:18:46.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:18:46 smithi070 ceph-mon[168282]: failed to encode map e623 with expected crc 2023-09-23T21:18:46.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:18:46 smithi070 ceph-mon[168282]: failed to encode map e623 with expected crc 2023-09-23T21:18:46.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:18:46 smithi070 ceph-mon[168282]: failed to encode map e623 with expected crc 2023-09-23T21:18:46.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:18:46 smithi070 ceph-mon[168282]: failed to encode map e623 with expected crc 2023-09-23T21:18:46.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:18:46 smithi070 ceph-mon[168282]: pgmap v1551: 129 pgs: 1 active+remapped+backfilling, 1 peering, 1 remapped+peering, 126 active+clean; 801 MiB data, 6.7 GiB used, 530 GiB / 536 GiB avail; 690/38481 objects misplaced (1.793%) 2023-09-23T21:18:46.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:18:46 smithi026 ceph-mon[175151]: 1.d scrub starts 2023-09-23T21:18:46.383 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:18:46 smithi026 ceph-mon[175151]: failed to encode map e623 with expected crc 2023-09-23T21:18:46.383 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:18:46 smithi026 ceph-mon[175151]: failed to encode map e623 with expected crc 2023-09-23T21:18:46.383 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:18:46 smithi026 ceph-mon[175151]: failed to encode map e623 with expected crc 2023-09-23T21:18:46.383 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:18:46 smithi026 ceph-mon[175151]: failed to encode map e623 with expected crc 2023-09-23T21:18:46.383 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:18:46 smithi026 ceph-mon[175151]: pgmap v1551: 129 pgs: 1 active+remapped+backfilling, 1 peering, 1 remapped+peering, 126 active+clean; 801 MiB data, 6.7 GiB used, 530 GiB / 536 GiB avail; 690/38481 objects misplaced (1.793%) 2023-09-23T21:18:46.383 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:18:46 smithi026 ceph-mon[179688]: 1.d scrub starts 2023-09-23T21:18:46.383 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:18:46 smithi026 ceph-mon[179688]: failed to encode map e623 with expected crc 2023-09-23T21:18:46.383 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:18:46 smithi026 ceph-mon[179688]: failed to encode map e623 with expected crc 2023-09-23T21:18:46.384 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:18:46 smithi026 ceph-mon[179688]: failed to encode map e623 with expected crc 2023-09-23T21:18:46.384 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:18:46 smithi026 ceph-mon[179688]: failed to encode map e623 with expected crc 2023-09-23T21:18:46.384 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:18:46 smithi026 ceph-mon[179688]: pgmap v1551: 129 pgs: 1 active+remapped+backfilling, 1 peering, 1 remapped+peering, 126 active+clean; 801 MiB data, 6.7 GiB used, 530 GiB / 536 GiB avail; 690/38481 objects misplaced (1.793%) 2023-09-23T21:18:46.406 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:18:46.638 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:18:46.645 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:18:46.645 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:18:46.746 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:18:46.818 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:18:46.950 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:18:46.955 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:18:46.955 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:18:47.056 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:18:47.093 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:18:47.098 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:18:47.130 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:18:47.135 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:18:47.135 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:18:47.236 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:18:47.290 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:18:47.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:18:47 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:18:47.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:18:47 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:18:47.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:18:47 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T21:18:47.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:18:47 smithi070 ceph-mon[168282]: failed to encode map e624 with expected crc 2023-09-23T21:18:47.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:18:47 smithi070 ceph-mon[168282]: osdmap e624: 8 total, 8 up, 6 in 2023-09-23T21:18:47.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:18:47 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:18:47.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:18:47 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:18:47.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:18:47 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "23"}]: dispatch 2023-09-23T21:18:47.447 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:18:47.452 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:18:47.452 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:18:47.520 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:18:47 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:18:47.520 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:18:47 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:18:47.520 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:18:47 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T21:18:47.520 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:18:47 smithi026 ceph-mon[179688]: failed to encode map e624 with expected crc 2023-09-23T21:18:47.520 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:18:47 smithi026 ceph-mon[179688]: osdmap e624: 8 total, 8 up, 6 in 2023-09-23T21:18:47.520 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:18:47 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:18:47.520 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:18:47 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:18:47.520 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:18:47 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "23"}]: dispatch 2023-09-23T21:18:47.521 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:18:47 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:18:47.521 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:18:47 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:18:47.521 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:18:47 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T21:18:47.521 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:18:47 smithi026 ceph-mon[175151]: failed to encode map e624 with expected crc 2023-09-23T21:18:47.521 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:18:47 smithi026 ceph-mon[175151]: osdmap e624: 8 total, 8 up, 6 in 2023-09-23T21:18:47.521 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:18:47 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:18:47.521 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:18:47 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:18:47.521 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:18:47 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "23"}]: dispatch 2023-09-23T21:18:47.530 INFO:teuthology.orchestra.run.smithi026.stdout:ERROR: (22) Invalid argument 2023-09-23T21:18:47.530 INFO:teuthology.orchestra.run.smithi026.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T21:18:47.553 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:18:47.584 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:18:47.723 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:18:47.729 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:18:47.729 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:18:47.831 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:18:47.862 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:18:48.010 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:18:48.016 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:18:48.017 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:18:48.118 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:18:48.197 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:18:48.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:18:48 smithi070 ceph-mon[168282]: 11.3 scrub starts 2023-09-23T21:18:48.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:18:48 smithi070 ceph-mon[168282]: 11.3 scrub ok 2023-09-23T21:18:48.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:18:48 smithi070 ceph-mon[168282]: failed to encode map e624 with expected crc 2023-09-23T21:18:48.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:18:48 smithi070 ceph-mon[168282]: failed to encode map e624 with expected crc 2023-09-23T21:18:48.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:18:48 smithi070 ceph-mon[168282]: pgmap v1553: 129 pgs: 1 active+remapped+backfilling, 128 active+clean; 695 MiB data, 6.6 GiB used, 530 GiB / 536 GiB avail; 690/33420 objects misplaced (2.065%) 2023-09-23T21:18:48.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:18:48 smithi070 ceph-mon[168282]: 11.13 scrub starts 2023-09-23T21:18:48.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:18:48 smithi070 ceph-mon[168282]: 11.13 scrub ok 2023-09-23T21:18:48.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:18:48 smithi070 ceph-mon[168282]: failed to encode map e625 with expected crc 2023-09-23T21:18:48.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:18:48 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "23"}]': finished 2023-09-23T21:18:48.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:18:48 smithi070 ceph-mon[168282]: osdmap e625: 8 total, 8 up, 6 in 2023-09-23T21:18:48.631 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:18:48 smithi026 ceph-mon[179688]: 11.3 scrub starts 2023-09-23T21:18:48.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:18:48 smithi026 ceph-mon[179688]: 11.3 scrub ok 2023-09-23T21:18:48.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:18:48 smithi026 ceph-mon[179688]: failed to encode map e624 with expected crc 2023-09-23T21:18:48.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:18:48 smithi026 ceph-mon[179688]: failed to encode map e624 with expected crc 2023-09-23T21:18:48.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:18:48 smithi026 ceph-mon[179688]: pgmap v1553: 129 pgs: 1 active+remapped+backfilling, 128 active+clean; 695 MiB data, 6.6 GiB used, 530 GiB / 536 GiB avail; 690/33420 objects misplaced (2.065%) 2023-09-23T21:18:48.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:18:48 smithi026 ceph-mon[179688]: 11.13 scrub starts 2023-09-23T21:18:48.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:18:48 smithi026 ceph-mon[179688]: 11.13 scrub ok 2023-09-23T21:18:48.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:18:48 smithi026 ceph-mon[179688]: failed to encode map e625 with expected crc 2023-09-23T21:18:48.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:18:48 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "23"}]': finished 2023-09-23T21:18:48.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:18:48 smithi026 ceph-mon[179688]: osdmap e625: 8 total, 8 up, 6 in 2023-09-23T21:18:48.633 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:18:48 smithi026 ceph-mon[175151]: 11.3 scrub starts 2023-09-23T21:18:48.633 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:18:48 smithi026 ceph-mon[175151]: 11.3 scrub ok 2023-09-23T21:18:48.633 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:18:48 smithi026 ceph-mon[175151]: failed to encode map e624 with expected crc 2023-09-23T21:18:48.633 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:18:48 smithi026 ceph-mon[175151]: failed to encode map e624 with expected crc 2023-09-23T21:18:48.633 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:18:48 smithi026 ceph-mon[175151]: pgmap v1553: 129 pgs: 1 active+remapped+backfilling, 128 active+clean; 695 MiB data, 6.6 GiB used, 530 GiB / 536 GiB avail; 690/33420 objects misplaced (2.065%) 2023-09-23T21:18:48.633 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:18:48 smithi026 ceph-mon[175151]: 11.13 scrub starts 2023-09-23T21:18:48.633 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:18:48 smithi026 ceph-mon[175151]: 11.13 scrub ok 2023-09-23T21:18:48.633 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:18:48 smithi026 ceph-mon[175151]: failed to encode map e625 with expected crc 2023-09-23T21:18:48.633 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:18:48 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "23"}]': finished 2023-09-23T21:18:48.633 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:18:48 smithi026 ceph-mon[175151]: osdmap e625: 8 total, 8 up, 6 in 2023-09-23T21:18:48.640 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T21:18:48.645 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:18:48.645 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:18:48.746 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:18:48.776 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:18:48.898 INFO:teuthology.orchestra.run.smithi026.stderr:noscrub is set 2023-09-23T21:18:48.976 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T21:18:49.262 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:18:49 smithi026 ceph-mon[175151]: 1.d scrub starts 2023-09-23T21:18:49.538 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:18:49 smithi026 ceph-mon[175151]: 1.d scrub ok 2023-09-23T21:18:49.538 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:18:49 smithi026 ceph-mon[175151]: failed to encode map e625 with expected crc 2023-09-23T21:18:49.538 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:18:49 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/2235428831' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T21:18:49.538 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:18:49 smithi026 ceph-mon[175151]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:18:49.538 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:18:49 smithi026 ceph-mon[175151]: failed to encode map e626 with expected crc 2023-09-23T21:18:49.538 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:18:49 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/2235428831' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T21:18:49.538 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:18:49 smithi026 ceph-mon[175151]: osdmap e626: 8 total, 8 up, 6 in 2023-09-23T21:18:49.538 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:18:49 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "24"}]: dispatch 2023-09-23T21:18:49.539 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:18:49 smithi026 ceph-mon[179688]: 1.d scrub starts 2023-09-23T21:18:49.539 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:18:49 smithi026 ceph-mon[179688]: 1.d scrub ok 2023-09-23T21:18:49.539 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:18:49 smithi026 ceph-mon[179688]: failed to encode map e625 with expected crc 2023-09-23T21:18:49.539 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:18:49 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/2235428831' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T21:18:49.539 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:18:49 smithi026 ceph-mon[179688]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:18:49.539 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:18:49 smithi026 ceph-mon[179688]: failed to encode map e626 with expected crc 2023-09-23T21:18:49.539 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:18:49 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/2235428831' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T21:18:49.539 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:18:49 smithi026 ceph-mon[179688]: osdmap e626: 8 total, 8 up, 6 in 2023-09-23T21:18:49.539 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:18:49 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "24"}]: dispatch 2023-09-23T21:18:49.543 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_ops_in_flight 2023-09-23T21:18:49.567 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:18:49.572 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:18:49.572 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:18:49.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:18:49 smithi070 ceph-mon[168282]: 1.d scrub starts 2023-09-23T21:18:49.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:18:49 smithi070 ceph-mon[168282]: 1.d scrub ok 2023-09-23T21:18:49.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:18:49 smithi070 ceph-mon[168282]: failed to encode map e625 with expected crc 2023-09-23T21:18:49.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:18:49 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/2235428831' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T21:18:49.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:18:49 smithi070 ceph-mon[168282]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:18:49.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:18:49 smithi070 ceph-mon[168282]: failed to encode map e626 with expected crc 2023-09-23T21:18:49.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:18:49 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/2235428831' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T21:18:49.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:18:49 smithi070 ceph-mon[168282]: osdmap e626: 8 total, 8 up, 6 in 2023-09-23T21:18:49.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:18:49 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "24"}]: dispatch 2023-09-23T21:18:49.673 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:18:49.702 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:18:49.735 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:18:49.737 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:18:49.738 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:18:49.839 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:18:49.866 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:18:49.881 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 21:18:49 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[164029]: ::ffff:172.21.15.70 - - [23/Sep/2023:21:18:49] "GET /metrics HTTP/1.1" 200 36360 "" "Prometheus/2.43.0" 2023-09-23T21:18:49.900 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:18:49.906 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:18:49.906 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:18:50.007 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:18:50.039 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:18:50.081 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:18:50.085 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:18:50.085 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:18:50.186 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:18:50.237 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:18:50.291 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:18:50.292 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:18:50 smithi070 ceph-mon[168282]: failed to encode map e626 with expected crc 2023-09-23T21:18:50.292 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:18:50 smithi070 ceph-mon[168282]: failed to encode map e626 with expected crc 2023-09-23T21:18:50.293 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:18:50 smithi070 ceph-mon[168282]: failed to encode map e626 with expected crc 2023-09-23T21:18:50.293 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:18:50 smithi070 ceph-mon[168282]: failed to encode map e626 with expected crc 2023-09-23T21:18:50.293 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:18:50 smithi070 ceph-mon[168282]: pgmap v1556: 129 pgs: 1 active+remapped+backfilling, 128 active+clean; 673 MiB data, 6.5 GiB used, 530 GiB / 536 GiB avail; 690/32349 objects misplaced (2.133%) 2023-09-23T21:18:50.293 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:18:50 smithi070 ceph-mon[168282]: failed to encode map e627 with expected crc 2023-09-23T21:18:50.293 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:18:50 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "24"}]': finished 2023-09-23T21:18:50.293 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:18:50 smithi070 ceph-mon[168282]: osdmap e627: 8 total, 8 up, 6 in 2023-09-23T21:18:50.293 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:18:50 smithi070 ceph-mon[168282]: 11.16 starting backfill to osd.3 from (0'0,0'0] MAX to 624'7888 2023-09-23T21:18:50.293 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:18:50 smithi070 ceph-mon[168282]: 11.16 starting backfill to osd.5 from (0'0,0'0] MAX to 624'7888 2023-09-23T21:18:50.293 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:18:50 smithi070 ceph-mon[168282]: 11.16 starting backfill to osd.7 from (0'0,0'0] MAX to 624'7888 2023-09-23T21:18:50.294 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:18:50.294 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:18:50.367 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:18:50.382 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:18:50.395 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:18:50.477 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:18:50.542 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:18:50.549 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:18:50.550 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:18:50.550 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:18:50 smithi026 ceph-mon[179688]: failed to encode map e626 with expected crc 2023-09-23T21:18:50.550 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:18:50 smithi026 ceph-mon[179688]: failed to encode map e626 with expected crc 2023-09-23T21:18:50.550 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:18:50 smithi026 ceph-mon[179688]: failed to encode map e626 with expected crc 2023-09-23T21:18:50.551 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:18:50 smithi026 ceph-mon[179688]: failed to encode map e626 with expected crc 2023-09-23T21:18:50.551 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:18:50 smithi026 ceph-mon[179688]: pgmap v1556: 129 pgs: 1 active+remapped+backfilling, 128 active+clean; 673 MiB data, 6.5 GiB used, 530 GiB / 536 GiB avail; 690/32349 objects misplaced (2.133%) 2023-09-23T21:18:50.551 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:18:50 smithi026 ceph-mon[179688]: failed to encode map e627 with expected crc 2023-09-23T21:18:50.551 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:18:50 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "24"}]': finished 2023-09-23T21:18:50.551 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:18:50 smithi026 ceph-mon[179688]: osdmap e627: 8 total, 8 up, 6 in 2023-09-23T21:18:50.551 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:18:50 smithi026 ceph-mon[179688]: 11.16 starting backfill to osd.3 from (0'0,0'0] MAX to 624'7888 2023-09-23T21:18:50.552 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:18:50 smithi026 ceph-mon[179688]: 11.16 starting backfill to osd.5 from (0'0,0'0] MAX to 624'7888 2023-09-23T21:18:50.552 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:18:50 smithi026 ceph-mon[179688]: 11.16 starting backfill to osd.7 from (0'0,0'0] MAX to 624'7888 2023-09-23T21:18:50.552 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:18:50 smithi026 ceph-mon[175151]: failed to encode map e626 with expected crc 2023-09-23T21:18:50.552 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:18:50 smithi026 ceph-mon[175151]: failed to encode map e626 with expected crc 2023-09-23T21:18:50.552 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:18:50 smithi026 ceph-mon[175151]: failed to encode map e626 with expected crc 2023-09-23T21:18:50.552 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:18:50 smithi026 ceph-mon[175151]: failed to encode map e626 with expected crc 2023-09-23T21:18:50.553 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:18:50 smithi026 ceph-mon[175151]: pgmap v1556: 129 pgs: 1 active+remapped+backfilling, 128 active+clean; 673 MiB data, 6.5 GiB used, 530 GiB / 536 GiB avail; 690/32349 objects misplaced (2.133%) 2023-09-23T21:18:50.553 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:18:50 smithi026 ceph-mon[175151]: failed to encode map e627 with expected crc 2023-09-23T21:18:50.553 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:18:50 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "24"}]': finished 2023-09-23T21:18:50.553 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:18:50 smithi026 ceph-mon[175151]: osdmap e627: 8 total, 8 up, 6 in 2023-09-23T21:18:50.553 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:18:50 smithi026 ceph-mon[175151]: 11.16 starting backfill to osd.3 from (0'0,0'0] MAX to 624'7888 2023-09-23T21:18:50.553 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:18:50 smithi026 ceph-mon[175151]: 11.16 starting backfill to osd.5 from (0'0,0'0] MAX to 624'7888 2023-09-23T21:18:50.554 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:18:50 smithi026 ceph-mon[175151]: 11.16 starting backfill to osd.7 from (0'0,0'0] MAX to 624'7888 2023-09-23T21:18:50.653 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:18:50.679 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:18:50.714 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:18:50.717 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:18:50.717 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:18:50.818 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:18:50.845 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:18:50.883 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:18:50.886 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:18:50.887 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:18:50.898 INFO:teuthology.orchestra.run.smithi026.stdout: 2023-09-23T21:18:50.898 INFO:teuthology.orchestra.run.smithi026.stdout:{"fsid":"b3f8e4ec-5a50-11ee-8db4-212e2dc638e7","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"noscrub flag(s) set","count":7},"muted":false}},"mutes":[]},"election_epoch":30,"quorum":[0,1,2],"quorum_names":["a","b","c"],"quorum_age":1849,"monmap":{"epoch":4,"min_mon_release_name":"reef","num_mons":3},"osdmap":{"epoch":628,"num_osds":8,"num_up_osds":8,"osd_up_since":1695503591,"num_in_osds":6,"osd_in_since":1695503801,"num_remapped_pgs":2},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":128},{"state_name":"active+remapped+backfilling","count":1}],"num_pgs":129,"num_pools":5,"num_objects":10783,"data_bytes":705664952,"bytes_used":6971015168,"bytes_avail":568999198720,"bytes_total":575970213888,"misplaced_objects":690,"misplaced_total":32349,"misplaced_ratio":0.021329871093387739},"fsmap":{"epoch":8,"id":1,"up":1,"in":1,"max":1,"by_rank":[{"filesystem_id":1,"rank":0,"name":"foo.smithi026.znfpnl","status":"up:active","gid":14658}],"up:standby":1},"mgrmap":{"available":true,"num_standbys":1,"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"services":{"dashboard":"https://172.21.15.26:8443/","prometheus":"http://172.21.15.26:9283/"}},"servicemap":{"epoch":30,"modified":"2023-09-23T21:17:37.170873+0000","services":{}},"progress_events":{"175d6d9d-570c-488d-becb-7b226eb562e1":{"message":"Global Recovery Event (0s)\n [............................] ","progress":0,"add_to_ceph_s":true},"fc746e9d-3f76-434d-b6d2-8fc9e8a72a5e":{"message":"Upgrade to 18.2.0-460-gbb0daeff (4m)\n [============................] (remaining: 5m)","progress":0.44999998807907104,"add_to_ceph_s":true}}} 2023-09-23T21:18:50.987 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:18:51.014 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:18:51.059 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:18:51.061 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:18:51.062 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:18:51.163 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:18:51.191 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:18:51.378 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:18:51.380 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:18:51.380 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:18:51.481 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:18:51.512 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:18:51.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:18:51 smithi070 ceph-mon[168282]: failed to encode map e627 with expected crc 2023-09-23T21:18:51.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:18:51 smithi070 ceph-mon[168282]: failed to encode map e627 with expected crc 2023-09-23T21:18:51.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:18:51 smithi070 ceph-mon[168282]: failed to encode map e628 with expected crc 2023-09-23T21:18:51.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:18:51 smithi070 ceph-mon[168282]: osdmap e628: 8 total, 8 up, 6 in 2023-09-23T21:18:51.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:18:51 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/2363396168' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-23T21:18:51.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:18:51 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "25"}]: dispatch 2023-09-23T21:18:51.631 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:18:51 smithi026 ceph-mon[179688]: failed to encode map e627 with expected crc 2023-09-23T21:18:51.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:18:51 smithi026 ceph-mon[179688]: failed to encode map e627 with expected crc 2023-09-23T21:18:51.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:18:51 smithi026 ceph-mon[179688]: failed to encode map e628 with expected crc 2023-09-23T21:18:51.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:18:51 smithi026 ceph-mon[179688]: osdmap e628: 8 total, 8 up, 6 in 2023-09-23T21:18:51.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:18:51 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/2363396168' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-23T21:18:51.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:18:51 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "25"}]: dispatch 2023-09-23T21:18:51.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:18:51 smithi026 ceph-mon[175151]: failed to encode map e627 with expected crc 2023-09-23T21:18:51.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:18:51 smithi026 ceph-mon[175151]: failed to encode map e627 with expected crc 2023-09-23T21:18:51.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:18:51 smithi026 ceph-mon[175151]: failed to encode map e628 with expected crc 2023-09-23T21:18:51.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:18:51 smithi026 ceph-mon[175151]: osdmap e628: 8 total, 8 up, 6 in 2023-09-23T21:18:51.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:18:51 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/2363396168' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-23T21:18:51.633 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:18:51 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "25"}]: dispatch 2023-09-23T21:18:51.998 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T21:18:52.029 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:18:52.046 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:18:52.047 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:18:52.148 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:18:52.269 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:18:52.283 INFO:tasks.cephadm.ceph_manager.ceph:no progress seen, keeping timeout for now 2023-09-23T21:18:52.283 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T21:18:52.298 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:18:52 smithi026 ceph-mon[175151]: failed to encode map e628 with expected crc 2023-09-23T21:18:52.298 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:18:52 smithi026 ceph-mon[175151]: failed to encode map e628 with expected crc 2023-09-23T21:18:52.298 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:18:52 smithi026 ceph-mon[175151]: failed to encode map e628 with expected crc 2023-09-23T21:18:52.298 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:18:52 smithi026 ceph-mon[175151]: pgmap v1559: 129 pgs: 2 active+remapped+backfilling, 127 active+clean; 601 MiB data, 6.2 GiB used, 530 GiB / 536 GiB avail; 1248/28878 objects misplaced (4.322%) 2023-09-23T21:18:52.298 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:18:52 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:18:52.298 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:18:52 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:18:52.298 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:18:52 smithi026 ceph-mon[175151]: failed to encode map e629 with expected crc 2023-09-23T21:18:52.298 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:18:52 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "25"}]': finished 2023-09-23T21:18:52.299 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:18:52 smithi026 ceph-mon[175151]: osdmap e629: 8 total, 8 up, 6 in 2023-09-23T21:18:52.299 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:18:52 smithi026 ceph-mon[175151]: 11.17 starting backfill to osd.3 from (0'0,0'0] MAX to 626'8261 2023-09-23T21:18:52.299 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:18:52 smithi026 ceph-mon[175151]: 11.17 starting backfill to osd.7 from (0'0,0'0] MAX to 626'8261 2023-09-23T21:18:52.299 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:18:52 smithi026 ceph-mon[179688]: failed to encode map e628 with expected crc 2023-09-23T21:18:52.299 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:18:52 smithi026 ceph-mon[179688]: failed to encode map e628 with expected crc 2023-09-23T21:18:52.299 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:18:52 smithi026 ceph-mon[179688]: failed to encode map e628 with expected crc 2023-09-23T21:18:52.299 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:18:52 smithi026 ceph-mon[179688]: pgmap v1559: 129 pgs: 2 active+remapped+backfilling, 127 active+clean; 601 MiB data, 6.2 GiB used, 530 GiB / 536 GiB avail; 1248/28878 objects misplaced (4.322%) 2023-09-23T21:18:52.299 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:18:52 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:18:52.300 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:18:52 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:18:52.300 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:18:52 smithi026 ceph-mon[179688]: failed to encode map e629 with expected crc 2023-09-23T21:18:52.300 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:18:52 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "25"}]': finished 2023-09-23T21:18:52.300 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:18:52 smithi026 ceph-mon[179688]: osdmap e629: 8 total, 8 up, 6 in 2023-09-23T21:18:52.300 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:18:52 smithi026 ceph-mon[179688]: 11.17 starting backfill to osd.3 from (0'0,0'0] MAX to 626'8261 2023-09-23T21:18:52.300 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:18:52 smithi026 ceph-mon[179688]: 11.17 starting backfill to osd.7 from (0'0,0'0] MAX to 626'8261 2023-09-23T21:18:52.371 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T21:18:52.376 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:18:52.377 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:18:52.478 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:18:52.509 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:18:52.546 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:18:52.548 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:18:52 smithi070 ceph-mon[168282]: failed to encode map e628 with expected crc 2023-09-23T21:18:52.548 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:18:52 smithi070 ceph-mon[168282]: failed to encode map e628 with expected crc 2023-09-23T21:18:52.548 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:18:52 smithi070 ceph-mon[168282]: failed to encode map e628 with expected crc 2023-09-23T21:18:52.548 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:18:52 smithi070 ceph-mon[168282]: pgmap v1559: 129 pgs: 2 active+remapped+backfilling, 127 active+clean; 601 MiB data, 6.2 GiB used, 530 GiB / 536 GiB avail; 1248/28878 objects misplaced (4.322%) 2023-09-23T21:18:52.548 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:18:52 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:18:52.548 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:18:52 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:18:52.548 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:18:52 smithi070 ceph-mon[168282]: failed to encode map e629 with expected crc 2023-09-23T21:18:52.549 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:18:52 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "25"}]': finished 2023-09-23T21:18:52.549 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:18:52 smithi070 ceph-mon[168282]: osdmap e629: 8 total, 8 up, 6 in 2023-09-23T21:18:52.549 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:18:52 smithi070 ceph-mon[168282]: 11.17 starting backfill to osd.3 from (0'0,0'0] MAX to 626'8261 2023-09-23T21:18:52.549 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:18:52 smithi070 ceph-mon[168282]: 11.17 starting backfill to osd.7 from (0'0,0'0] MAX to 626'8261 2023-09-23T21:18:52.549 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:18:52.550 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:18:52.631 INFO:teuthology.orchestra.run.smithi026.stdout:ERROR: (22) Invalid argument 2023-09-23T21:18:52.631 INFO:teuthology.orchestra.run.smithi026.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T21:18:52.650 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:18:52.677 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:18:52.718 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:18:52.720 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:18:52.721 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:18:52.821 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:18:52.858 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:18:53.205 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:18:53.210 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:18:53.210 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:18:53.211 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0: {} 2023-09-23T21:18:53.236 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_blocked_ops 2023-09-23T21:18:53.238 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1: {} 2023-09-23T21:18:53.248 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:18:53.263 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2: {} 2023-09-23T21:18:53.287 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3: {} 2023-09-23T21:18:53.311 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:18:53.321 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4: {} 2023-09-23T21:18:53.330 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T21:18:53.339 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:18:53.360 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6: {} 2023-09-23T21:18:53.380 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T21:18:53.381 INFO:teuthology.orchestra.run.smithi026.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T21:18:53.381 INFO:teuthology.orchestra.run.smithi026.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T21:18:53.381 INFO:teuthology.orchestra.run.smithi026.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T21:18:53.381 INFO:teuthology.orchestra.run.smithi026.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T21:18:53.381 INFO:teuthology.orchestra.run.smithi026.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T21:18:53.382 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T21:18:53.382 INFO:teuthology.orchestra.run.smithi026.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T21:18:53.382 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T21:18:53.407 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:18:53.409 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:18:53.410 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:18:53.511 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:18:53.538 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:18:53.543 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:18:53 smithi070 ceph-mon[168282]: failed to encode map e629 with expected crc 2023-09-23T21:18:53.543 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:18:53 smithi070 ceph-mon[168282]: failed to encode map e629 with expected crc 2023-09-23T21:18:53.543 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:18:53 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:18:53.543 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:18:53 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:18:53.543 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:18:53 smithi070 ceph-mon[168282]: failed to encode map e630 with expected crc 2023-09-23T21:18:53.544 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:18:53 smithi070 ceph-mon[168282]: osdmap e630: 8 total, 8 up, 6 in 2023-09-23T21:18:53.544 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:18:53 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/2682887991' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:18:53.573 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:18:53.576 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:18:53.577 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:18:53.631 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:18:53 smithi026 ceph-mon[175151]: failed to encode map e629 with expected crc 2023-09-23T21:18:53.631 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:18:53 smithi026 ceph-mon[175151]: failed to encode map e629 with expected crc 2023-09-23T21:18:53.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:18:53 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:18:53.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:18:53 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:18:53.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:18:53 smithi026 ceph-mon[175151]: failed to encode map e630 with expected crc 2023-09-23T21:18:53.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:18:53 smithi026 ceph-mon[175151]: osdmap e630: 8 total, 8 up, 6 in 2023-09-23T21:18:53.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:18:53 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/2682887991' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:18:53.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:18:53 smithi026 ceph-mon[179688]: failed to encode map e629 with expected crc 2023-09-23T21:18:53.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:18:53 smithi026 ceph-mon[179688]: failed to encode map e629 with expected crc 2023-09-23T21:18:53.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:18:53 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:18:53.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:18:53 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:18:53.633 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:18:53 smithi026 ceph-mon[179688]: failed to encode map e630 with expected crc 2023-09-23T21:18:53.633 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:18:53 smithi026 ceph-mon[179688]: osdmap e630: 8 total, 8 up, 6 in 2023-09-23T21:18:53.633 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:18:53 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/2682887991' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:18:53.677 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:18:53.704 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:18:54.069 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:18:54.071 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:18:54.072 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:18:54.172 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:18:54.200 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:18:54.253 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:18:54.256 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:18:54.256 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:18:54.357 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:18:54.411 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:18:54.537 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:18:54.541 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:18:54 smithi026 ceph-mon[179688]: failed to encode map e630 with expected crc 2023-09-23T21:18:54.541 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:18:54 smithi026 ceph-mon[179688]: failed to encode map e630 with expected crc 2023-09-23T21:18:54.542 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:18:54 smithi026 ceph-mon[179688]: pgmap v1562: 129 pgs: 1 peering, 128 active+clean; 530 MiB data, 5.9 GiB used, 531 GiB / 536 GiB avail 2023-09-23T21:18:54.542 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:18:54 smithi026 ceph-mon[179688]: failed to encode map e631 with expected crc 2023-09-23T21:18:54.542 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:18:54 smithi026 ceph-mon[179688]: osdmap e631: 8 total, 8 up, 6 in 2023-09-23T21:18:54.542 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:18:54 smithi026 ceph-mon[179688]: 11.18 starting backfill to osd.2 from (0'0,0'0] MAX to 627'7900 2023-09-23T21:18:54.542 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:18:54 smithi026 ceph-mon[179688]: 11.18 starting backfill to osd.4 from (0'0,0'0] MAX to 627'7900 2023-09-23T21:18:54.542 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:18:54.543 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:18:54.543 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:18:54 smithi026 ceph-mon[175151]: failed to encode map e630 with expected crc 2023-09-23T21:18:54.543 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:18:54 smithi026 ceph-mon[175151]: failed to encode map e630 with expected crc 2023-09-23T21:18:54.543 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:18:54 smithi026 ceph-mon[175151]: pgmap v1562: 129 pgs: 1 peering, 128 active+clean; 530 MiB data, 5.9 GiB used, 531 GiB / 536 GiB avail 2023-09-23T21:18:54.543 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:18:54 smithi026 ceph-mon[175151]: failed to encode map e631 with expected crc 2023-09-23T21:18:54.544 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:18:54 smithi026 ceph-mon[175151]: osdmap e631: 8 total, 8 up, 6 in 2023-09-23T21:18:54.544 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:18:54 smithi026 ceph-mon[175151]: 11.18 starting backfill to osd.2 from (0'0,0'0] MAX to 627'7900 2023-09-23T21:18:54.544 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:18:54 smithi026 ceph-mon[175151]: 11.18 starting backfill to osd.4 from (0'0,0'0] MAX to 627'7900 2023-09-23T21:18:54.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:18:54 smithi070 ceph-mon[168282]: failed to encode map e630 with expected crc 2023-09-23T21:18:54.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:18:54 smithi070 ceph-mon[168282]: failed to encode map e630 with expected crc 2023-09-23T21:18:54.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:18:54 smithi070 ceph-mon[168282]: pgmap v1562: 129 pgs: 1 peering, 128 active+clean; 530 MiB data, 5.9 GiB used, 531 GiB / 536 GiB avail 2023-09-23T21:18:54.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:18:54 smithi070 ceph-mon[168282]: failed to encode map e631 with expected crc 2023-09-23T21:18:54.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:18:54 smithi070 ceph-mon[168282]: osdmap e631: 8 total, 8 up, 6 in 2023-09-23T21:18:54.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:18:54 smithi070 ceph-mon[168282]: 11.18 starting backfill to osd.2 from (0'0,0'0] MAX to 627'7900 2023-09-23T21:18:54.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:18:54 smithi070 ceph-mon[168282]: 11.18 starting backfill to osd.4 from (0'0,0'0] MAX to 627'7900 2023-09-23T21:18:54.644 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:18:54.676 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:18:54.721 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:18:54.725 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:18:54.725 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:18:54.826 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:18:54.859 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:18:54.900 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:18:54.904 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:18:54.905 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:18:54.916 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:18:55.005 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:18:55.058 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:18:55.112 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:18:55.116 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:18:55.116 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:18:55.217 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:18:55.255 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:18:55.310 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:18:55.314 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:18:55 smithi070 ceph-mon[168282]: failed to encode map e631 with expected crc 2023-09-23T21:18:55.315 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:18:55 smithi070 ceph-mon[168282]: failed to encode map e632 with expected crc 2023-09-23T21:18:55.315 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:18:55 smithi070 ceph-mon[168282]: osdmap e632: 8 total, 8 up, 6 in 2023-09-23T21:18:55.315 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:18:55.316 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:18:55.416 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:18:55.464 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:18:55.495 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:18:55 smithi026 ceph-mon[179688]: failed to encode map e631 with expected crc 2023-09-23T21:18:55.495 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:18:55 smithi026 ceph-mon[179688]: failed to encode map e632 with expected crc 2023-09-23T21:18:55.495 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:18:55 smithi026 ceph-mon[179688]: osdmap e632: 8 total, 8 up, 6 in 2023-09-23T21:18:55.496 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:18:55 smithi026 ceph-mon[175151]: failed to encode map e631 with expected crc 2023-09-23T21:18:55.496 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:18:55 smithi026 ceph-mon[175151]: failed to encode map e632 with expected crc 2023-09-23T21:18:55.496 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:18:55 smithi026 ceph-mon[175151]: osdmap e632: 8 total, 8 up, 6 in 2023-09-23T21:18:55.513 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:18:55.516 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:18:55.517 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:18:55.545 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:18:55.617 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:18:55.653 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:18:55.714 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:18:55.720 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:18:55.720 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:18:55.821 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:18:55.928 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:18:56.343 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:18:56.351 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:18:56.352 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:18:56.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:18:56 smithi070 ceph-mon[168282]: failed to encode map e632 with expected crc 2023-09-23T21:18:56.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:18:56 smithi070 ceph-mon[168282]: failed to encode map e632 with expected crc 2023-09-23T21:18:56.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:18:56 smithi070 ceph-mon[168282]: failed to encode map e632 with expected crc 2023-09-23T21:18:56.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:18:56 smithi070 ceph-mon[168282]: pgmap v1565: 129 pgs: 1 peering, 128 active+clean; 515 MiB data, 5.9 GiB used, 531 GiB / 536 GiB avail 2023-09-23T21:18:56.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:18:56 smithi070 ceph-mon[168282]: osdmap e633: 8 total, 8 up, 6 in 2023-09-23T21:18:56.453 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:18:56.485 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:18:56.525 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:18:56.527 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:18:56.528 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:18:56.561 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T21:18:56.600 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:18:56 smithi026 ceph-mon[179688]: failed to encode map e632 with expected crc 2023-09-23T21:18:56.600 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:18:56 smithi026 ceph-mon[179688]: failed to encode map e632 with expected crc 2023-09-23T21:18:56.600 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:18:56 smithi026 ceph-mon[179688]: failed to encode map e632 with expected crc 2023-09-23T21:18:56.600 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:18:56 smithi026 ceph-mon[179688]: pgmap v1565: 129 pgs: 1 peering, 128 active+clean; 515 MiB data, 5.9 GiB used, 531 GiB / 536 GiB avail 2023-09-23T21:18:56.600 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:18:56 smithi026 ceph-mon[179688]: osdmap e633: 8 total, 8 up, 6 in 2023-09-23T21:18:56.601 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:18:56 smithi026 ceph-mon[175151]: failed to encode map e632 with expected crc 2023-09-23T21:18:56.601 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:18:56 smithi026 ceph-mon[175151]: failed to encode map e632 with expected crc 2023-09-23T21:18:56.601 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:18:56 smithi026 ceph-mon[175151]: failed to encode map e632 with expected crc 2023-09-23T21:18:56.602 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:18:56 smithi026 ceph-mon[175151]: pgmap v1565: 129 pgs: 1 peering, 128 active+clean; 515 MiB data, 5.9 GiB used, 531 GiB / 536 GiB avail 2023-09-23T21:18:56.602 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:18:56 smithi026 ceph-mon[175151]: osdmap e633: 8 total, 8 up, 6 in 2023-09-23T21:18:56.628 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:18:56.659 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:18:56.750 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T21:18:56.757 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:18:56.757 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:18:56.858 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:18:56.886 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:18:56.921 INFO:teuthology.orchestra.run.smithi026.stderr:nodeep-scrub is set 2023-09-23T21:18:57.311 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:18:57 smithi026 ceph-mon[179688]: failed to encode map e633 with expected crc 2023-09-23T21:18:57.311 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:18:57 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:18:57.312 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:18:57 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:18:57.312 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:18:57 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/2760205170' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:18:57.312 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:18:57 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:18:57.312 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:18:57 smithi026 ceph-mon[179688]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:18:57.312 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:18:57 smithi026 ceph-mon[179688]: failed to encode map e634 with expected crc 2023-09-23T21:18:57.312 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:18:57 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T21:18:57.312 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:18:57 smithi026 ceph-mon[179688]: osdmap e634: 8 total, 8 up, 6 in 2023-09-23T21:18:57.547 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:18:57.560 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:18:57.560 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:18:57 smithi026 ceph-mon[175151]: failed to encode map e633 with expected crc 2023-09-23T21:18:57.560 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:18:57 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:18:57.560 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:18:57 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:18:57.560 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:18:57 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/2760205170' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:18:57.560 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:18:57 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:18:57.561 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:18:57 smithi026 ceph-mon[175151]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:18:57.561 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:18:57 smithi026 ceph-mon[175151]: failed to encode map e634 with expected crc 2023-09-23T21:18:57.561 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:18:57 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T21:18:57.561 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:18:57 smithi026 ceph-mon[175151]: osdmap e634: 8 total, 8 up, 6 in 2023-09-23T21:18:57.561 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:18:57.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:18:57 smithi070 ceph-mon[168282]: failed to encode map e633 with expected crc 2023-09-23T21:18:57.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:18:57 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:18:57.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:18:57 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:18:57.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:18:57 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/2760205170' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:18:57.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:18:57 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:18:57.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:18:57 smithi070 ceph-mon[168282]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:18:57.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:18:57 smithi070 ceph-mon[168282]: failed to encode map e634 with expected crc 2023-09-23T21:18:57.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:18:57 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T21:18:57.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:18:57 smithi070 ceph-mon[168282]: osdmap e634: 8 total, 8 up, 6 in 2023-09-23T21:18:57.662 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:18:57.690 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:18:58.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:18:58 smithi070 ceph-mon[168282]: failed to encode map e634 with expected crc 2023-09-23T21:18:58.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:18:58 smithi070 ceph-mon[168282]: failed to encode map e634 with expected crc 2023-09-23T21:18:58.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:18:58 smithi070 ceph-mon[168282]: failed to encode map e634 with expected crc 2023-09-23T21:18:58.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:18:58 smithi070 ceph-mon[168282]: pgmap v1568: 129 pgs: 2 peering, 127 active+clean; 414 MiB data, 5.7 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 463 op/s; 3.5 MiB/s, 56 objects/s recovering 2023-09-23T21:18:58.634 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:18:58 smithi026 ceph-mon[175151]: failed to encode map e634 with expected crc 2023-09-23T21:18:58.634 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:18:58 smithi026 ceph-mon[175151]: failed to encode map e634 with expected crc 2023-09-23T21:18:58.634 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:18:58 smithi026 ceph-mon[175151]: failed to encode map e634 with expected crc 2023-09-23T21:18:58.634 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:18:58 smithi026 ceph-mon[175151]: pgmap v1568: 129 pgs: 2 peering, 127 active+clean; 414 MiB data, 5.7 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 463 op/s; 3.5 MiB/s, 56 objects/s recovering 2023-09-23T21:18:58.634 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:18:58 smithi026 ceph-mon[179688]: failed to encode map e634 with expected crc 2023-09-23T21:18:58.634 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:18:58 smithi026 ceph-mon[179688]: failed to encode map e634 with expected crc 2023-09-23T21:18:58.634 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:18:58 smithi026 ceph-mon[179688]: failed to encode map e634 with expected crc 2023-09-23T21:18:58.634 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:18:58 smithi026 ceph-mon[179688]: pgmap v1568: 129 pgs: 2 peering, 127 active+clean; 414 MiB data, 5.7 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 463 op/s; 3.5 MiB/s, 56 objects/s recovering 2023-09-23T21:18:59.004 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T21:18:59.012 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:18:59.013 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:18:59.114 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:18:59.159 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:18:59.197 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:18:59.201 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:18:59.201 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:18:59.302 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:18:59.418 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:18:59.501 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T21:18:59.512 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:18:59.513 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:18:59.568 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:18:59.614 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:18:59.615 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 21:18:59 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[164029]: ::ffff:172.21.15.70 - - [23/Sep/2023:21:18:59] "GET /metrics HTTP/1.1" 200 36360 "" "Prometheus/2.43.0" 2023-09-23T21:18:59.644 INFO:teuthology.orchestra.run.smithi026.stdout: 2023-09-23T21:18:59.646 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:18:59.662 INFO:teuthology.orchestra.run.smithi026.stderr:dumped all 2023-09-23T21:18:59.685 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:18:59.691 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:18:59.692 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:18:59.792 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:18:59.824 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:18:59.868 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:18:59.871 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:18:59.871 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:18:59.972 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:18:59.989 INFO:teuthology.orchestra.run.smithi026.stdout:{ 2023-09-23T21:18:59.990 INFO:teuthology.orchestra.run.smithi026.stdout: "ops": [], 2023-09-23T21:18:59.990 INFO:teuthology.orchestra.run.smithi026.stdout: "complaint_time": 30, 2023-09-23T21:18:59.990 INFO:teuthology.orchestra.run.smithi026.stdout: "num_blocked_ops": 0 2023-09-23T21:18:59.990 INFO:teuthology.orchestra.run.smithi026.stdout:} 2023-09-23T21:19:00.011 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:00.061 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:19:00.064 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:00.065 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:19:00.166 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:19:00.239 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:00.257 INFO:teuthology.orchestra.run.smithi026.stdout:{"pg_ready":true,"pg_map":{"version":1569,"stamp":"2023-09-23T21:18:59.205622+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":403609528,"num_objects":6174,"num_object_clones":0,"num_object_copies":18522,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6174,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":94334,"num_write_kb":2940670,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":9019,"num_bytes_recovered":575884606,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":134824,"ondisk_log_size":134824,"up":387,"acting":387,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":403,"num_osds":6,"num_per_pool_osds":6,"num_per_pool_omap_osds":6,"kb":562470912,"kb_used":5781272,"kb_used_data":1458408,"kb_used_omap":125,"kb_used_meta":4322370,"kb_avail":556689640,"statfs":{"total":575970213888,"available":570050191360,"internally_reserved":0,"allocated":1493409792,"data_stored":1479558868,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":128662,"internal_metadata":4426107242},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,4,4,0,5,6],"upper_bound":128},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-152240128,"num_objects":-2323,"num_object_clones":0,"num_object_copies":-6969,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":-2323,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2323,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":483,"num_bytes_recovered":31653888,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"5.314349"},"pg_stats":[{"pgid":"11.12","version":"633'8814","reported_seq":10158,"reported_epoch":633,"state":"active+clean","last_fresh":"2023-09-23T21:18:56.588766+0000","last_change":"2023-09-23T21:18:40.100577+0000","last_active":"2023-09-23T21:18:56.588766+0000","last_peered":"2023-09-23T21:18:56.588766+0000","last_clean":"2023-09-23T21:18:56.588766+0000","last_became_active":"2023-09-23T21:18:34.930698+0000","last_became_peered":"2023-09-23T21:18:34.930698+0000","last_unstale":"2023-09-23T21:18:56.588766+0000","last_undegraded":"2023-09-23T21:18:56.588766+0000","last_fullsized":"2023-09-23T21:18:56.588766+0000","mapping_epoch":613,"log_start":"549'2020","ondisk_log_start":"549'2020","created":559,"last_epoch_clean":614,"parent":"0.0","parent_split_bits":5,"last_scrub":"618'8566","last_scrub_stamp":"2023-09-23T21:18:40.100396+0000","last_deep_scrub":"618'8566","last_deep_scrub_stamp":"2023-09-23T21:18:40.100396+0000","last_clean_scrub_stamp":"2023-09-23T21:18:40.100396+0000","objects_scrubbed":431,"log_size":6794,"log_dups_size":0,"ondisk_log_size":6794,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":2,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:16:14.066996+0000","scrub_duration":1.094034102,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":11927552,"num_objects":182,"num_object_clones":0,"num_object_copies":546,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":182,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":249,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,0],"acting":[2,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.18","version":"79'161","reported_seq":658,"reported_epoch":633,"state":"active+clean","last_fresh":"2023-09-23T21:18:55.913205+0000","last_change":"2023-09-23T21:17:39.906535+0000","last_active":"2023-09-23T21:18:55.913205+0000","last_peered":"2023-09-23T21:18:55.913205+0000","last_clean":"2023-09-23T21:18:55.913205+0000","last_became_active":"2023-09-23T21:17:39.906154+0000","last_became_peered":"2023-09-23T21:17:39.906154+0000","last_unstale":"2023-09-23T21:18:55.913205+0000","last_undegraded":"2023-09-23T21:18:55.913205+0000","last_fullsized":"2023-09-23T21:18:55.913205+0000","mapping_epoch":585,"log_start":"0'0","ondisk_log_start":"0'0","created":479,"last_epoch_clean":586,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:15:39.474823+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:15:39.474823+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:48:39.784037+0000","scrub_duration":0.074664395999999994,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":525376,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1a","version":"68'1","reported_seq":224,"reported_epoch":633,"state":"active+clean","last_fresh":"2023-09-23T21:18:55.908045+0000","last_change":"2023-09-23T21:13:13.291403+0000","last_active":"2023-09-23T21:18:55.908045+0000","last_peered":"2023-09-23T21:18:55.908045+0000","last_clean":"2023-09-23T21:18:55.908045+0000","last_became_active":"2023-09-23T21:13:13.291123+0000","last_became_peered":"2023-09-23T21:13:13.291123+0000","last_unstale":"2023-09-23T21:18:55.908045+0000","last_undegraded":"2023-09-23T21:18:55.908045+0000","last_fullsized":"2023-09-23T21:18:55.908045+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":495,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:45:15.427430+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,0],"acting":[4,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":293,"reported_epoch":633,"state":"active+clean","last_fresh":"2023-09-23T21:18:55.908849+0000","last_change":"2023-09-23T21:12:10.767963+0000","last_active":"2023-09-23T21:18:55.908849+0000","last_peered":"2023-09-23T21:18:55.908849+0000","last_clean":"2023-09-23T21:18:55.908849+0000","last_became_active":"2023-09-23T21:12:10.767126+0000","last_became_peered":"2023-09-23T21:12:10.767126+0000","last_unstale":"2023-09-23T21:18:55.908849+0000","last_undegraded":"2023-09-23T21:18:55.908849+0000","last_fullsized":"2023-09-23T21:18:55.908849+0000","mapping_epoch":470,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:45:47.138277+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,5],"acting":[4,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.11","version":"633'8282","reported_seq":9613,"reported_epoch":633,"state":"active+clean","last_fresh":"2023-09-23T21:18:56.491084+0000","last_change":"2023-09-23T21:18:36.195191+0000","last_active":"2023-09-23T21:18:56.491084+0000","last_peered":"2023-09-23T21:18:56.491084+0000","last_clean":"2023-09-23T21:18:56.491084+0000","last_became_active":"2023-09-23T21:18:32.876101+0000","last_became_peered":"2023-09-23T21:18:32.876101+0000","last_unstale":"2023-09-23T21:18:56.491084+0000","last_undegraded":"2023-09-23T21:18:56.491084+0000","last_fullsized":"2023-09-23T21:18:56.491084+0000","mapping_epoch":611,"log_start":"555'3628","ondisk_log_start":"555'3628","created":559,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":5,"last_scrub":"614'7979","last_scrub_stamp":"2023-09-23T21:18:36.195023+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_clean_scrub_stamp":"2023-09-23T21:18:36.195023+0000","objects_scrubbed":487,"log_size":4654,"log_dups_size":0,"ondisk_log_size":4654,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:18:09.227629+0000","scrub_duration":0.17685009800000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":11927552,"num_objects":182,"num_object_clones":0,"num_object_copies":546,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":182,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":305,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,7],"acting":[2,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.1b","version":"79'161","reported_seq":821,"reported_epoch":633,"state":"active+clean","last_fresh":"2023-09-23T21:18:55.915164+0000","last_change":"2023-09-23T21:15:38.423372+0000","last_active":"2023-09-23T21:18:55.915164+0000","last_peered":"2023-09-23T21:18:55.915164+0000","last_clean":"2023-09-23T21:18:55.915164+0000","last_became_active":"2023-09-23T21:15:23.802580+0000","last_became_peered":"2023-09-23T21:15:23.802580+0000","last_unstale":"2023-09-23T21:18:55.915164+0000","last_undegraded":"2023-09-23T21:18:55.915164+0000","last_fullsized":"2023-09-23T21:18:55.915164+0000","mapping_epoch":542,"log_start":"0'0","ondisk_log_start":"0'0","created":479,"last_epoch_clean":543,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:15:38.423271+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:15:38.423271+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:04:46.575186+0000","scrub_duration":0.00258465,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":646,"reported_epoch":630,"state":"active+clean","last_fresh":"2023-09-23T21:18:52.897822+0000","last_change":"2023-09-23T21:16:42.448529+0000","last_active":"2023-09-23T21:18:52.897822+0000","last_peered":"2023-09-23T21:18:52.897822+0000","last_clean":"2023-09-23T21:18:52.897822+0000","last_became_active":"2023-09-23T21:16:42.447803+0000","last_became_peered":"2023-09-23T21:16:42.447803+0000","last_unstale":"2023-09-23T21:18:52.897822+0000","last_undegraded":"2023-09-23T21:18:52.897822+0000","last_fullsized":"2023-09-23T21:18:52.897822+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:41:33.121784+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":648,"reported_epoch":631,"state":"active+clean","last_fresh":"2023-09-23T21:18:53.895735+0000","last_change":"2023-09-23T21:16:42.441896+0000","last_active":"2023-09-23T21:18:53.895735+0000","last_peered":"2023-09-23T21:18:53.895735+0000","last_clean":"2023-09-23T21:18:53.895735+0000","last_became_active":"2023-09-23T21:16:42.441526+0000","last_became_peered":"2023-09-23T21:16:42.441526+0000","last_unstale":"2023-09-23T21:18:53.895735+0000","last_undegraded":"2023-09-23T21:18:53.895735+0000","last_fullsized":"2023-09-23T21:18:53.895735+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:27:36.326225+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.10","version":"632'8209","reported_seq":11883,"reported_epoch":632,"state":"active+clean","last_fresh":"2023-09-23T21:18:55.439435+0000","last_change":"2023-09-23T21:18:36.698828+0000","last_active":"2023-09-23T21:18:55.439435+0000","last_peered":"2023-09-23T21:18:55.439435+0000","last_clean":"2023-09-23T21:18:55.439435+0000","last_became_active":"2023-09-23T21:18:30.880043+0000","last_became_peered":"2023-09-23T21:18:30.880043+0000","last_unstale":"2023-09-23T21:18:55.439435+0000","last_undegraded":"2023-09-23T21:18:55.439435+0000","last_fullsized":"2023-09-23T21:18:55.439435+0000","mapping_epoch":609,"log_start":"556'3752","ondisk_log_start":"556'3752","created":559,"last_epoch_clean":610,"parent":"0.0","parent_split_bits":5,"last_scrub":"614'7936","last_scrub_stamp":"2023-09-23T21:18:36.698511+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_clean_scrub_stamp":"2023-09-23T21:18:36.698511+0000","objects_scrubbed":486,"log_size":4457,"log_dups_size":0,"ondisk_log_size":4457,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:26:33.228381+0000","scrub_duration":0.115560511,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13762560,"num_objects":210,"num_object_clones":0,"num_object_copies":630,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":210,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":276,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,5],"acting":[0,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1a","version":"79'161","reported_seq":731,"reported_epoch":633,"state":"active+clean","last_fresh":"2023-09-23T21:18:55.907529+0000","last_change":"2023-09-23T21:16:45.419221+0000","last_active":"2023-09-23T21:18:55.907529+0000","last_peered":"2023-09-23T21:18:55.907529+0000","last_clean":"2023-09-23T21:18:55.907529+0000","last_became_active":"2023-09-23T21:16:45.418994+0000","last_became_peered":"2023-09-23T21:16:45.418994+0000","last_unstale":"2023-09-23T21:18:55.907529+0000","last_undegraded":"2023-09-23T21:18:55.907529+0000","last_fullsized":"2023-09-23T21:18:55.907529+0000","mapping_epoch":567,"log_start":"0'0","ondisk_log_start":"0'0","created":479,"last_epoch_clean":568,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:15:45.158107+0000","last_deep_scrub":"79'161","last_deep_scrub_stamp":"2023-09-23T21:05:32.630292+0000","last_clean_scrub_stamp":"2023-09-23T21:15:45.158107+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:08:08.342444+0000","scrub_duration":0.001999373,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,5],"acting":[4,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":143,"reported_epoch":631,"state":"active+clean","last_fresh":"2023-09-23T21:18:53.895037+0000","last_change":"2023-09-23T21:13:30.806084+0000","last_active":"2023-09-23T21:18:53.895037+0000","last_peered":"2023-09-23T21:18:53.895037+0000","last_clean":"2023-09-23T21:18:53.895037+0000","last_became_active":"2023-09-23T21:13:30.805743+0000","last_became_peered":"2023-09-23T21:13:30.805743+0000","last_unstale":"2023-09-23T21:18:53.895037+0000","last_undegraded":"2023-09-23T21:18:53.895037+0000","last_fullsized":"2023-09-23T21:18:53.895037+0000","mapping_epoch":501,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:48:10.849281+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":507,"reported_epoch":631,"state":"active+clean","last_fresh":"2023-09-23T21:18:53.895918+0000","last_change":"2023-09-23T21:13:44.410921+0000","last_active":"2023-09-23T21:18:53.895918+0000","last_peered":"2023-09-23T21:18:53.895918+0000","last_clean":"2023-09-23T21:18:53.895918+0000","last_became_active":"2023-09-23T21:13:44.410633+0000","last_became_peered":"2023-09-23T21:13:44.410633+0000","last_unstale":"2023-09-23T21:18:53.895918+0000","last_undegraded":"2023-09-23T21:18:53.895918+0000","last_fullsized":"2023-09-23T21:18:53.895918+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:57:10.116786+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.15","version":"79'161","reported_seq":592,"reported_epoch":633,"state":"active+clean","last_fresh":"2023-09-23T21:18:55.913785+0000","last_change":"2023-09-23T21:18:35.622513+0000","last_active":"2023-09-23T21:18:55.913785+0000","last_peered":"2023-09-23T21:18:55.913785+0000","last_clean":"2023-09-23T21:18:55.913785+0000","last_became_active":"2023-09-23T21:16:45.425654+0000","last_became_peered":"2023-09-23T21:16:45.425654+0000","last_unstale":"2023-09-23T21:18:55.913785+0000","last_undegraded":"2023-09-23T21:18:55.913785+0000","last_fullsized":"2023-09-23T21:18:55.913785+0000","mapping_epoch":567,"log_start":"0'0","ondisk_log_start":"0'0","created":479,"last_epoch_clean":568,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:18:35.622422+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:18:35.622422+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:21:48.421117+0000","scrub_duration":0.083031242000000005,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,3],"acting":[7,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":608,"reported_epoch":631,"state":"active+clean","last_fresh":"2023-09-23T21:18:53.896104+0000","last_change":"2023-09-23T21:13:12.235728+0000","last_active":"2023-09-23T21:18:53.896104+0000","last_peered":"2023-09-23T21:18:53.896104+0000","last_clean":"2023-09-23T21:18:53.896104+0000","last_became_active":"2023-09-23T21:13:12.216549+0000","last_became_peered":"2023-09-23T21:13:12.216549+0000","last_unstale":"2023-09-23T21:18:53.896104+0000","last_undegraded":"2023-09-23T21:18:53.896104+0000","last_fullsized":"2023-09-23T21:18:53.896104+0000","mapping_epoch":493,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:39:33.324239+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":670,"reported_epoch":630,"state":"active+clean","last_fresh":"2023-09-23T21:18:52.898760+0000","last_change":"2023-09-23T21:12:10.790349+0000","last_active":"2023-09-23T21:18:52.898760+0000","last_peered":"2023-09-23T21:18:52.898760+0000","last_clean":"2023-09-23T21:18:52.898760+0000","last_became_active":"2023-09-23T21:12:10.789977+0000","last_became_peered":"2023-09-23T21:12:10.789977+0000","last_unstale":"2023-09-23T21:18:52.898760+0000","last_undegraded":"2023-09-23T21:18:52.898760+0000","last_fullsized":"2023-09-23T21:18:52.898760+0000","mapping_epoch":470,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:49:40.191147+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,5],"acting":[3,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"79'161","reported_seq":1030,"reported_epoch":632,"state":"active+clean","last_fresh":"2023-09-23T21:18:54.904617+0000","last_change":"2023-09-23T21:16:45.424773+0000","last_active":"2023-09-23T21:18:54.904617+0000","last_peered":"2023-09-23T21:18:54.904617+0000","last_clean":"2023-09-23T21:18:54.904617+0000","last_became_active":"2023-09-23T21:16:45.424458+0000","last_became_peered":"2023-09-23T21:16:45.424458+0000","last_unstale":"2023-09-23T21:18:54.904617+0000","last_undegraded":"2023-09-23T21:18:54.904617+0000","last_fullsized":"2023-09-23T21:18:54.904617+0000","mapping_epoch":567,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":568,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:25.096401+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:25.096401+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:40:17.871561+0000","scrub_duration":0.002859393,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":378,"reported_epoch":633,"state":"active+clean","last_fresh":"2023-09-23T21:18:55.913885+0000","last_change":"2023-09-23T21:16:42.427856+0000","last_active":"2023-09-23T21:18:55.913885+0000","last_peered":"2023-09-23T21:18:55.913885+0000","last_clean":"2023-09-23T21:18:55.913885+0000","last_became_active":"2023-09-23T21:16:42.427529+0000","last_became_peered":"2023-09-23T21:16:42.427529+0000","last_unstale":"2023-09-23T21:18:55.913885+0000","last_undegraded":"2023-09-23T21:18:55.913885+0000","last_fullsized":"2023-09-23T21:18:55.913885+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:24:03.818632+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":625,"reported_epoch":630,"state":"active+clean","last_fresh":"2023-09-23T21:18:52.897727+0000","last_change":"2023-09-23T21:16:42.448719+0000","last_active":"2023-09-23T21:18:52.897727+0000","last_peered":"2023-09-23T21:18:52.897727+0000","last_clean":"2023-09-23T21:18:52.897727+0000","last_became_active":"2023-09-23T21:16:42.448219+0000","last_became_peered":"2023-09-23T21:16:42.448219+0000","last_unstale":"2023-09-23T21:18:52.897727+0000","last_undegraded":"2023-09-23T21:18:52.897727+0000","last_fullsized":"2023-09-23T21:18:52.897727+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:02:28.864357+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.17","version":"79'161","reported_seq":779,"reported_epoch":631,"state":"active+clean","last_fresh":"2023-09-23T21:18:53.895662+0000","last_change":"2023-09-23T21:16:45.413663+0000","last_active":"2023-09-23T21:18:53.895662+0000","last_peered":"2023-09-23T21:18:53.895662+0000","last_clean":"2023-09-23T21:18:53.895662+0000","last_became_active":"2023-09-23T21:16:45.413311+0000","last_became_peered":"2023-09-23T21:16:45.413311+0000","last_unstale":"2023-09-23T21:18:53.895662+0000","last_undegraded":"2023-09-23T21:18:53.895662+0000","last_fullsized":"2023-09-23T21:18:53.895662+0000","mapping_epoch":567,"log_start":"0'0","ondisk_log_start":"0'0","created":479,"last_epoch_clean":568,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:15:50.325741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:15:50.325741+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:07:43.285354+0000","scrub_duration":0.0040500149999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":547,"reported_epoch":630,"state":"active+clean","last_fresh":"2023-09-23T21:18:52.898852+0000","last_change":"2023-09-23T21:17:02.182606+0000","last_active":"2023-09-23T21:18:52.898852+0000","last_peered":"2023-09-23T21:18:52.898852+0000","last_clean":"2023-09-23T21:18:52.898852+0000","last_became_active":"2023-09-23T21:16:42.446517+0000","last_became_peered":"2023-09-23T21:16:42.446517+0000","last_unstale":"2023-09-23T21:18:52.898852+0000","last_undegraded":"2023-09-23T21:18:52.898852+0000","last_fullsized":"2023-09-23T21:18:52.898852+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:46:08.337878+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":540,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,4],"acting":[3,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":321,"reported_epoch":632,"state":"active+clean","last_fresh":"2023-09-23T21:18:54.902390+0000","last_change":"2023-09-23T21:13:30.821489+0000","last_active":"2023-09-23T21:18:54.902390+0000","last_peered":"2023-09-23T21:18:54.902390+0000","last_clean":"2023-09-23T21:18:54.902390+0000","last_became_active":"2023-09-23T21:13:30.821125+0000","last_became_peered":"2023-09-23T21:13:30.821125+0000","last_unstale":"2023-09-23T21:18:54.902390+0000","last_undegraded":"2023-09-23T21:18:54.902390+0000","last_fullsized":"2023-09-23T21:18:54.902390+0000","mapping_epoch":501,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:47:18.553417+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,5],"acting":[2,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.16","version":"79'161","reported_seq":564,"reported_epoch":631,"state":"active+clean","last_fresh":"2023-09-23T21:18:53.895719+0000","last_change":"2023-09-23T21:15:46.326726+0000","last_active":"2023-09-23T21:18:53.895719+0000","last_peered":"2023-09-23T21:18:53.895719+0000","last_clean":"2023-09-23T21:18:53.895719+0000","last_became_active":"2023-09-23T21:13:33.770407+0000","last_became_peered":"2023-09-23T21:13:33.770407+0000","last_unstale":"2023-09-23T21:18:53.895719+0000","last_undegraded":"2023-09-23T21:18:53.895719+0000","last_fullsized":"2023-09-23T21:18:53.895719+0000","mapping_epoch":504,"log_start":"0'0","ondisk_log_start":"0'0","created":479,"last_epoch_clean":505,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:15:46.326589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:15:46.326589+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:50:23.810054+0000","scrub_duration":0.0062069059999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.14","version":"79'22","reported_seq":237,"reported_epoch":633,"state":"active+clean","last_fresh":"2023-09-23T21:18:55.908597+0000","last_change":"2023-09-23T21:17:02.198735+0000","last_active":"2023-09-23T21:18:55.908597+0000","last_peered":"2023-09-23T21:18:55.908597+0000","last_clean":"2023-09-23T21:18:55.908597+0000","last_became_active":"2023-09-23T21:16:42.492861+0000","last_became_peered":"2023-09-23T21:16:42.492861+0000","last_unstale":"2023-09-23T21:18:55.908597+0000","last_undegraded":"2023-09-23T21:18:55.908597+0000","last_fullsized":"2023-09-23T21:18:55.908597+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:00:09.949965+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":14202,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,2],"acting":[4,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":562,"reported_epoch":633,"state":"active+clean","last_fresh":"2023-09-23T21:18:55.909167+0000","last_change":"2023-09-23T21:12:10.839271+0000","last_active":"2023-09-23T21:18:55.909167+0000","last_peered":"2023-09-23T21:18:55.909167+0000","last_clean":"2023-09-23T21:18:55.909167+0000","last_became_active":"2023-09-23T21:12:10.839042+0000","last_became_peered":"2023-09-23T21:12:10.839042+0000","last_unstale":"2023-09-23T21:18:55.909167+0000","last_undegraded":"2023-09-23T21:18:55.909167+0000","last_fullsized":"2023-09-23T21:18:55.909167+0000","mapping_epoch":470,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:10:26.414737+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,2],"acting":[4,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.11","version":"79'161","reported_seq":569,"reported_epoch":631,"state":"active+clean","last_fresh":"2023-09-23T21:18:53.894727+0000","last_change":"2023-09-23T21:15:47.300109+0000","last_active":"2023-09-23T21:18:53.894727+0000","last_peered":"2023-09-23T21:18:53.894727+0000","last_clean":"2023-09-23T21:18:53.894727+0000","last_became_active":"2023-09-23T21:13:33.767067+0000","last_became_peered":"2023-09-23T21:13:33.767067+0000","last_unstale":"2023-09-23T21:18:53.894727+0000","last_undegraded":"2023-09-23T21:18:53.894727+0000","last_fullsized":"2023-09-23T21:18:53.894727+0000","mapping_epoch":504,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":505,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:15:47.299947+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:15:47.299947+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:04:59.029359+0000","scrub_duration":0.0061353110000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":85,"reported_epoch":633,"state":"active+clean","last_fresh":"2023-09-23T21:18:55.914122+0000","last_change":"2023-09-23T21:16:42.531060+0000","last_active":"2023-09-23T21:18:55.914122+0000","last_peered":"2023-09-23T21:18:55.914122+0000","last_clean":"2023-09-23T21:18:55.914122+0000","last_became_active":"2023-09-23T21:16:42.449198+0000","last_became_peered":"2023-09-23T21:16:42.449198+0000","last_unstale":"2023-09-23T21:18:55.914122+0000","last_undegraded":"2023-09-23T21:18:55.914122+0000","last_fullsized":"2023-09-23T21:18:55.914122+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:30:23.428886+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":193,"reported_epoch":632,"state":"active+clean","last_fresh":"2023-09-23T21:18:54.904396+0000","last_change":"2023-09-23T21:16:42.491023+0000","last_active":"2023-09-23T21:18:54.904396+0000","last_peered":"2023-09-23T21:18:54.904396+0000","last_clean":"2023-09-23T21:18:54.904396+0000","last_became_active":"2023-09-23T21:16:42.489896+0000","last_became_peered":"2023-09-23T21:16:42.489896+0000","last_unstale":"2023-09-23T21:18:54.904396+0000","last_undegraded":"2023-09-23T21:18:54.904396+0000","last_fullsized":"2023-09-23T21:18:54.904396+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:58:54.036098+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,7],"acting":[2,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.10","version":"79'161","reported_seq":808,"reported_epoch":633,"state":"active+clean","last_fresh":"2023-09-23T21:18:55.908110+0000","last_change":"2023-09-23T21:16:45.416227+0000","last_active":"2023-09-23T21:18:55.908110+0000","last_peered":"2023-09-23T21:18:55.908110+0000","last_clean":"2023-09-23T21:18:55.908110+0000","last_became_active":"2023-09-23T21:16:45.415841+0000","last_became_peered":"2023-09-23T21:16:45.415841+0000","last_unstale":"2023-09-23T21:18:55.908110+0000","last_undegraded":"2023-09-23T21:18:55.908110+0000","last_fullsized":"2023-09-23T21:18:55.908110+0000","mapping_epoch":567,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":568,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:06:50.336992+0000","last_deep_scrub":"79'161","last_deep_scrub_stamp":"2023-09-23T21:06:50.336992+0000","last_clean_scrub_stamp":"2023-09-23T21:06:50.336992+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:37:30.094658+0000","scrub_duration":0.0026150930000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,7],"acting":[4,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":592,"reported_epoch":631,"state":"active+clean","last_fresh":"2023-09-23T21:18:53.895053+0000","last_change":"2023-09-23T21:16:42.446324+0000","last_active":"2023-09-23T21:18:53.895053+0000","last_peered":"2023-09-23T21:18:53.895053+0000","last_clean":"2023-09-23T21:18:53.895053+0000","last_became_active":"2023-09-23T21:16:42.445997+0000","last_became_peered":"2023-09-23T21:16:42.445997+0000","last_unstale":"2023-09-23T21:18:53.895053+0000","last_undegraded":"2023-09-23T21:18:53.895053+0000","last_fullsized":"2023-09-23T21:18:53.895053+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:46:35.028292+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":402,"reported_epoch":630,"state":"active+clean","last_fresh":"2023-09-23T21:18:52.898729+0000","last_change":"2023-09-23T21:16:42.473872+0000","last_active":"2023-09-23T21:18:52.898729+0000","last_peered":"2023-09-23T21:18:52.898729+0000","last_clean":"2023-09-23T21:18:52.898729+0000","last_became_active":"2023-09-23T21:16:42.473595+0000","last_became_peered":"2023-09-23T21:16:42.473595+0000","last_unstale":"2023-09-23T21:18:52.898729+0000","last_undegraded":"2023-09-23T21:18:52.898729+0000","last_fullsized":"2023-09-23T21:18:52.898729+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:15:19.186428+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.19","version":"631'8146","reported_seq":9957,"reported_epoch":631,"state":"active+clean","last_fresh":"2023-09-23T21:18:54.292236+0000","last_change":"2023-09-23T21:18:39.990368+0000","last_active":"2023-09-23T21:18:54.292236+0000","last_peered":"2023-09-23T21:18:54.292236+0000","last_clean":"2023-09-23T21:18:54.292236+0000","last_became_active":"2023-09-23T21:17:46.300166+0000","last_became_peered":"2023-09-23T21:17:46.300166+0000","last_unstale":"2023-09-23T21:18:54.292236+0000","last_undegraded":"2023-09-23T21:18:54.292236+0000","last_fullsized":"2023-09-23T21:18:54.292236+0000","mapping_epoch":588,"log_start":"556'4049","ondisk_log_start":"556'4049","created":559,"last_epoch_clean":601,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_clean_scrub_stamp":"2023-09-23T21:15:24.860968+0000","objects_scrubbed":0,"log_size":4097,"log_dups_size":0,"ondisk_log_size":4097,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":14319616,"num_objects":219,"num_object_clones":0,"num_object_copies":657,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":219,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5863,"num_write_kb":194592,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":838,"num_bytes_recovered":54820864,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,7],"acting":[3,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.13","version":"79'161","reported_seq":709,"reported_epoch":630,"state":"active+clean","last_fresh":"2023-09-23T21:18:52.898023+0000","last_change":"2023-09-23T21:13:14.933096+0000","last_active":"2023-09-23T21:18:52.898023+0000","last_peered":"2023-09-23T21:18:52.898023+0000","last_clean":"2023-09-23T21:18:52.898023+0000","last_became_active":"2023-09-23T21:13:14.932703+0000","last_became_peered":"2023-09-23T21:13:14.932703+0000","last_unstale":"2023-09-23T21:18:52.898023+0000","last_undegraded":"2023-09-23T21:18:52.898023+0000","last_fullsized":"2023-09-23T21:18:52.898023+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:36.420362+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:36.420362+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:29:42.163911+0000","scrub_duration":0.028838444000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":645,"reported_epoch":633,"state":"active+clean","last_fresh":"2023-09-23T21:18:55.907661+0000","last_change":"2023-09-23T21:16:42.420467+0000","last_active":"2023-09-23T21:18:55.907661+0000","last_peered":"2023-09-23T21:18:55.907661+0000","last_clean":"2023-09-23T21:18:55.907661+0000","last_became_active":"2023-09-23T21:16:42.420149+0000","last_became_peered":"2023-09-23T21:16:42.420149+0000","last_unstale":"2023-09-23T21:18:55.907661+0000","last_undegraded":"2023-09-23T21:18:55.907661+0000","last_fullsized":"2023-09-23T21:18:55.907661+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:52:07.355000+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":161,"reported_epoch":631,"state":"active+clean","last_fresh":"2023-09-23T21:18:53.895548+0000","last_change":"2023-09-23T21:16:42.427663+0000","last_active":"2023-09-23T21:18:53.895548+0000","last_peered":"2023-09-23T21:18:53.895548+0000","last_clean":"2023-09-23T21:18:53.895548+0000","last_became_active":"2023-09-23T21:16:42.426961+0000","last_became_peered":"2023-09-23T21:16:42.426961+0000","last_unstale":"2023-09-23T21:18:53.895548+0000","last_undegraded":"2023-09-23T21:18:53.895548+0000","last_fullsized":"2023-09-23T21:18:53.895548+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:44:27.422007+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.18","version":"632'7955","reported_seq":8776,"reported_epoch":633,"state":"peering","last_fresh":"2023-09-23T21:18:55.894729+0000","last_change":"2023-09-23T21:18:55.894729+0000","last_active":"2023-09-23T21:18:55.894729+0000","last_peered":"2023-09-23T21:18:55.835951+0000","last_clean":"2023-09-23T21:18:55.835951+0000","last_became_active":"2023-09-23T21:18:53.953424+0000","last_became_peered":"2023-09-23T21:18:53.953424+0000","last_unstale":"2023-09-23T21:18:55.894729+0000","last_undegraded":"2023-09-23T21:18:55.894729+0000","last_fullsized":"2023-09-23T21:18:55.894729+0000","mapping_epoch":633,"log_start":"554'3311","ondisk_log_start":"554'3311","created":559,"last_epoch_clean":632,"parent":"0.0","parent_split_bits":5,"last_scrub":"611'7608","last_scrub_stamp":"2023-09-23T21:18:32.625076+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_clean_scrub_stamp":"2023-09-23T21:18:32.625076+0000","objects_scrubbed":541,"log_size":4644,"log_dups_size":0,"ondisk_log_size":4644,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:29:11.075466+0000","scrub_duration":0.078956060999999994,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":12713984,"num_objects":194,"num_object_clones":0,"num_object_copies":582,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":194,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":347,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":225,"num_bytes_recovered":14745600,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,5],"acting":[2,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.12","version":"79'161","reported_seq":907,"reported_epoch":633,"state":"active+clean","last_fresh":"2023-09-23T21:18:55.915516+0000","last_change":"2023-09-23T21:16:45.412440+0000","last_active":"2023-09-23T21:18:55.915516+0000","last_peered":"2023-09-23T21:18:55.915516+0000","last_clean":"2023-09-23T21:18:55.915516+0000","last_became_active":"2023-09-23T21:16:45.412029+0000","last_became_peered":"2023-09-23T21:16:45.412029+0000","last_unstale":"2023-09-23T21:18:55.915516+0000","last_undegraded":"2023-09-23T21:18:55.915516+0000","last_fullsized":"2023-09-23T21:18:55.915516+0000","mapping_epoch":567,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":568,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:05:56.785173+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:05:56.785173+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:39:43.432998+0000","scrub_duration":0.0042114370000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,0],"acting":[5,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":153,"reported_epoch":631,"state":"active+clean","last_fresh":"2023-09-23T21:18:53.911537+0000","last_change":"2023-09-23T21:13:38.048791+0000","last_active":"2023-09-23T21:18:53.911537+0000","last_peered":"2023-09-23T21:18:53.911537+0000","last_clean":"2023-09-23T21:18:53.911537+0000","last_became_active":"2023-09-23T21:13:30.801386+0000","last_became_peered":"2023-09-23T21:13:30.801386+0000","last_unstale":"2023-09-23T21:18:53.911537+0000","last_undegraded":"2023-09-23T21:18:53.911537+0000","last_fullsized":"2023-09-23T21:18:53.911537+0000","mapping_epoch":501,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:52:49.170414+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":487,"reported_epoch":630,"state":"active+clean","last_fresh":"2023-09-23T21:18:52.900120+0000","last_change":"2023-09-23T21:16:42.518512+0000","last_active":"2023-09-23T21:18:52.900120+0000","last_peered":"2023-09-23T21:18:52.900120+0000","last_clean":"2023-09-23T21:18:52.900120+0000","last_became_active":"2023-09-23T21:16:42.517464+0000","last_became_peered":"2023-09-23T21:16:42.517464+0000","last_unstale":"2023-09-23T21:18:52.900120+0000","last_undegraded":"2023-09-23T21:18:52.900120+0000","last_fullsized":"2023-09-23T21:18:52.900120+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:47:52.391167+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.7","version":"634'8257","reported_seq":8410,"reported_epoch":634,"state":"active+clean","last_fresh":"2023-09-23T21:18:57.062406+0000","last_change":"2023-09-23T21:18:30.328388+0000","last_active":"2023-09-23T21:18:57.062406+0000","last_peered":"2023-09-23T21:18:57.062406+0000","last_clean":"2023-09-23T21:18:57.062406+0000","last_became_active":"2023-09-23T21:16:32.864890+0000","last_became_peered":"2023-09-23T21:16:32.864890+0000","last_unstale":"2023-09-23T21:18:57.062406+0000","last_undegraded":"2023-09-23T21:18:57.062406+0000","last_fullsized":"2023-09-23T21:18:57.062406+0000","mapping_epoch":544,"log_start":"549'2695","ondisk_log_start":"549'2695","created":544,"last_epoch_clean":560,"parent":"0.0","parent_split_bits":0,"last_scrub":"609'7868","last_scrub_stamp":"2023-09-23T21:18:30.328316+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_clean_scrub_stamp":"2023-09-23T21:18:30.328316+0000","objects_scrubbed":571,"log_size":5562,"log_dups_size":0,"ondisk_log_size":5562,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:34:57.557430+0000","scrub_duration":0.064969499999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":11927552,"num_objects":182,"num_object_clones":0,"num_object_copies":546,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":182,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":389,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.d","version":"79'161","reported_seq":675,"reported_epoch":633,"state":"active+clean","last_fresh":"2023-09-23T21:18:55.913845+0000","last_change":"2023-09-23T21:18:47.440374+0000","last_active":"2023-09-23T21:18:55.913845+0000","last_peered":"2023-09-23T21:18:55.913845+0000","last_clean":"2023-09-23T21:18:55.913845+0000","last_became_active":"2023-09-23T21:16:45.416373+0000","last_became_peered":"2023-09-23T21:16:45.416373+0000","last_unstale":"2023-09-23T21:18:55.913845+0000","last_undegraded":"2023-09-23T21:18:55.913845+0000","last_fullsized":"2023-09-23T21:18:55.913845+0000","mapping_epoch":567,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":568,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:18:47.440263+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:18:47.440263+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:11:36.676449+0000","scrub_duration":0.0023849959999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":83,"reported_epoch":633,"state":"active+clean","last_fresh":"2023-09-23T21:18:55.916321+0000","last_change":"2023-09-23T21:17:02.190320+0000","last_active":"2023-09-23T21:18:55.916321+0000","last_peered":"2023-09-23T21:18:55.916321+0000","last_clean":"2023-09-23T21:18:55.916321+0000","last_became_active":"2023-09-23T21:16:42.451380+0000","last_became_peered":"2023-09-23T21:16:42.451380+0000","last_unstale":"2023-09-23T21:18:55.916321+0000","last_undegraded":"2023-09-23T21:18:55.916321+0000","last_fullsized":"2023-09-23T21:18:55.916321+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:48:58.425821+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":132,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":361,"reported_epoch":633,"state":"active+clean","last_fresh":"2023-09-23T21:18:55.914338+0000","last_change":"2023-09-23T21:16:42.428491+0000","last_active":"2023-09-23T21:18:55.914338+0000","last_peered":"2023-09-23T21:18:55.914338+0000","last_clean":"2023-09-23T21:18:55.914338+0000","last_became_active":"2023-09-23T21:16:42.428166+0000","last_became_peered":"2023-09-23T21:16:42.428166+0000","last_unstale":"2023-09-23T21:18:55.914338+0000","last_undegraded":"2023-09-23T21:18:55.914338+0000","last_fullsized":"2023-09-23T21:18:55.914338+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:23:01.430714+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,4],"acting":[5,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.6","version":"632'8075","reported_seq":8209,"reported_epoch":632,"state":"active+clean","last_fresh":"2023-09-23T21:18:55.119692+0000","last_change":"2023-09-23T21:18:29.652253+0000","last_active":"2023-09-23T21:18:55.119692+0000","last_peered":"2023-09-23T21:18:55.119692+0000","last_clean":"2023-09-23T21:18:55.119692+0000","last_became_active":"2023-09-23T21:16:32.896510+0000","last_became_peered":"2023-09-23T21:16:32.896510+0000","last_unstale":"2023-09-23T21:18:55.119692+0000","last_undegraded":"2023-09-23T21:18:55.119692+0000","last_fullsized":"2023-09-23T21:18:55.119692+0000","mapping_epoch":544,"log_start":"556'3624","ondisk_log_start":"556'3624","created":544,"last_epoch_clean":561,"parent":"0.0","parent_split_bits":0,"last_scrub":"607'7678","last_scrub_stamp":"2023-09-23T21:18:29.652124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_clean_scrub_stamp":"2023-09-23T21:18:29.652124+0000","objects_scrubbed":618,"log_size":4451,"log_dups_size":0,"ondisk_log_size":4451,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:05:14.537109+0000","scrub_duration":0.077048132000000005,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":14483456,"num_objects":221,"num_object_clones":0,"num_object_copies":663,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":221,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":397,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.c","version":"79'161","reported_seq":1013,"reported_epoch":630,"state":"active+clean","last_fresh":"2023-09-23T21:18:52.898993+0000","last_change":"2023-09-23T21:15:31.992756+0000","last_active":"2023-09-23T21:18:52.898993+0000","last_peered":"2023-09-23T21:18:52.898993+0000","last_clean":"2023-09-23T21:18:52.898993+0000","last_became_active":"2023-09-23T21:13:13.401526+0000","last_became_peered":"2023-09-23T21:13:13.401526+0000","last_unstale":"2023-09-23T21:18:52.898993+0000","last_undegraded":"2023-09-23T21:18:52.898993+0000","last_fullsized":"2023-09-23T21:18:52.898993+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":532,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:15:31.992598+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:15:31.992598+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:41:14.215543+0000","scrub_duration":0.002960678,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":83,"reported_epoch":633,"state":"active+clean","last_fresh":"2023-09-23T21:18:55.914395+0000","last_change":"2023-09-23T21:17:02.215996+0000","last_active":"2023-09-23T21:18:55.914395+0000","last_peered":"2023-09-23T21:18:55.914395+0000","last_clean":"2023-09-23T21:18:55.914395+0000","last_became_active":"2023-09-23T21:16:42.451205+0000","last_became_peered":"2023-09-23T21:16:42.451205+0000","last_unstale":"2023-09-23T21:18:55.914395+0000","last_undegraded":"2023-09-23T21:18:55.914395+0000","last_fullsized":"2023-09-23T21:18:55.914395+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:51:46.876281+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":596,"reported_epoch":633,"state":"active+clean","last_fresh":"2023-09-23T21:18:55.907667+0000","last_change":"2023-09-23T21:13:15.152792+0000","last_active":"2023-09-23T21:18:55.907667+0000","last_peered":"2023-09-23T21:18:55.907667+0000","last_clean":"2023-09-23T21:18:55.907667+0000","last_became_active":"2023-09-23T21:13:14.515084+0000","last_became_peered":"2023-09-23T21:13:14.515084+0000","last_unstale":"2023-09-23T21:18:55.907667+0000","last_undegraded":"2023-09-23T21:18:55.907667+0000","last_fullsized":"2023-09-23T21:18:55.907667+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:08:16.702072+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.5","version":"634'8298","reported_seq":13576,"reported_epoch":634,"state":"active+clean","last_fresh":"2023-09-23T21:18:57.292741+0000","last_change":"2023-09-23T21:18:36.457465+0000","last_active":"2023-09-23T21:18:57.292741+0000","last_peered":"2023-09-23T21:18:57.292741+0000","last_clean":"2023-09-23T21:18:57.292741+0000","last_became_active":"2023-09-23T21:17:24.389052+0000","last_became_peered":"2023-09-23T21:17:24.389052+0000","last_unstale":"2023-09-23T21:18:57.292741+0000","last_undegraded":"2023-09-23T21:18:57.292741+0000","last_fullsized":"2023-09-23T21:18:57.292741+0000","mapping_epoch":578,"log_start":"549'2087","ondisk_log_start":"549'2087","created":544,"last_epoch_clean":579,"parent":"0.0","parent_split_bits":0,"last_scrub":"614'7983","last_scrub_stamp":"2023-09-23T21:18:36.457393+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_clean_scrub_stamp":"2023-09-23T21:18:36.457393+0000","objects_scrubbed":481,"log_size":6211,"log_dups_size":0,"ondisk_log_size":6211,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:07:29.014321+0000","scrub_duration":0.14039997700000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":10616832,"num_objects":162,"num_object_clones":0,"num_object_copies":486,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":162,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":319,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.f","version":"79'161","reported_seq":921,"reported_epoch":633,"state":"active+clean","last_fresh":"2023-09-23T21:18:55.914842+0000","last_change":"2023-09-23T21:11:30.750338+0000","last_active":"2023-09-23T21:18:55.914842+0000","last_peered":"2023-09-23T21:18:55.914842+0000","last_clean":"2023-09-23T21:18:55.914842+0000","last_became_active":"2023-09-23T21:11:30.749953+0000","last_became_peered":"2023-09-23T21:11:30.749953+0000","last_unstale":"2023-09-23T21:18:55.914842+0000","last_undegraded":"2023-09-23T21:18:55.914842+0000","last_fullsized":"2023-09-23T21:18:55.914842+0000","mapping_epoch":455,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":456,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:27.880776+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:27.880776+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:07:10.405723+0000","scrub_duration":0.0014004899999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,3],"acting":[5,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":385,"reported_epoch":633,"state":"active+clean","last_fresh":"2023-09-23T21:18:55.914549+0000","last_change":"2023-09-23T21:17:02.175012+0000","last_active":"2023-09-23T21:18:55.914549+0000","last_peered":"2023-09-23T21:18:55.914549+0000","last_clean":"2023-09-23T21:18:55.914549+0000","last_became_active":"2023-09-23T21:16:42.451114+0000","last_became_peered":"2023-09-23T21:16:42.451114+0000","last_unstale":"2023-09-23T21:18:55.914549+0000","last_undegraded":"2023-09-23T21:18:55.914549+0000","last_fullsized":"2023-09-23T21:18:55.914549+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:00:53.868525+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":420,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":593,"reported_epoch":633,"state":"active+clean","last_fresh":"2023-09-23T21:18:55.915240+0000","last_change":"2023-09-23T21:16:42.432943+0000","last_active":"2023-09-23T21:18:55.915240+0000","last_peered":"2023-09-23T21:18:55.915240+0000","last_clean":"2023-09-23T21:18:55.915240+0000","last_became_active":"2023-09-23T21:16:42.427440+0000","last_became_peered":"2023-09-23T21:16:42.427440+0000","last_unstale":"2023-09-23T21:18:55.915240+0000","last_undegraded":"2023-09-23T21:18:55.915240+0000","last_fullsized":"2023-09-23T21:18:55.915240+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:59:32.044609+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,0],"acting":[5,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.4","version":"634'8154","reported_seq":9845,"reported_epoch":634,"state":"active+clean","last_fresh":"2023-09-23T21:18:57.262897+0000","last_change":"2023-09-23T21:18:43.538320+0000","last_active":"2023-09-23T21:18:57.262897+0000","last_peered":"2023-09-23T21:18:57.262897+0000","last_clean":"2023-09-23T21:18:57.262897+0000","last_became_active":"2023-09-23T21:18:05.477101+0000","last_became_peered":"2023-09-23T21:18:05.477101+0000","last_unstale":"2023-09-23T21:18:57.262897+0000","last_undegraded":"2023-09-23T21:18:57.262897+0000","last_fullsized":"2023-09-23T21:18:57.262897+0000","mapping_epoch":595,"log_start":"553'3321","ondisk_log_start":"553'3321","created":544,"last_epoch_clean":596,"parent":"0.0","parent_split_bits":0,"last_scrub":"621'7884","last_scrub_stamp":"2023-09-23T21:18:43.538247+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_clean_scrub_stamp":"2023-09-23T21:18:43.538247+0000","objects_scrubbed":456,"log_size":4833,"log_dups_size":0,"ondisk_log_size":4833,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:09:58.379074+0000","scrub_duration":0.050975133999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":12189696,"num_objects":186,"num_object_clones":0,"num_object_copies":558,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":186,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":270,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,3],"acting":[7,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.e","version":"79'161","reported_seq":852,"reported_epoch":633,"state":"active+clean","last_fresh":"2023-09-23T21:18:55.908832+0000","last_change":"2023-09-23T21:16:45.433204+0000","last_active":"2023-09-23T21:18:55.908832+0000","last_peered":"2023-09-23T21:18:55.908832+0000","last_clean":"2023-09-23T21:18:55.908832+0000","last_became_active":"2023-09-23T21:16:45.432836+0000","last_became_peered":"2023-09-23T21:16:45.432836+0000","last_unstale":"2023-09-23T21:18:55.908832+0000","last_undegraded":"2023-09-23T21:18:55.908832+0000","last_fullsized":"2023-09-23T21:18:55.908832+0000","mapping_epoch":567,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":568,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:15:39.046962+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:15:39.046962+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:11:20.720731+0000","scrub_duration":0.0082638769999999993,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":430,"reported_epoch":630,"state":"active+clean","last_fresh":"2023-09-23T21:18:52.899286+0000","last_change":"2023-09-23T21:13:13.402418+0000","last_active":"2023-09-23T21:18:52.899286+0000","last_peered":"2023-09-23T21:18:52.899286+0000","last_clean":"2023-09-23T21:18:52.899286+0000","last_became_active":"2023-09-23T21:13:13.400984+0000","last_became_peered":"2023-09-23T21:13:13.400984+0000","last_unstale":"2023-09-23T21:18:52.899286+0000","last_undegraded":"2023-09-23T21:18:52.899286+0000","last_fullsized":"2023-09-23T21:18:52.899286+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":495,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:28:01.411191+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,0],"acting":[3,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":606,"reported_epoch":631,"state":"active+clean","last_fresh":"2023-09-23T21:18:53.895498+0000","last_change":"2023-09-23T21:13:14.339670+0000","last_active":"2023-09-23T21:18:53.895498+0000","last_peered":"2023-09-23T21:18:53.895498+0000","last_clean":"2023-09-23T21:18:53.895498+0000","last_became_active":"2023-09-23T21:13:14.339341+0000","last_became_peered":"2023-09-23T21:13:14.339341+0000","last_unstale":"2023-09-23T21:18:53.895498+0000","last_undegraded":"2023-09-23T21:18:53.895498+0000","last_fullsized":"2023-09-23T21:18:53.895498+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:23:19.411368+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.3","version":"634'8048","reported_seq":9781,"reported_epoch":634,"state":"active+clean","last_fresh":"2023-09-23T21:18:57.253064+0000","last_change":"2023-09-23T21:18:46.551584+0000","last_active":"2023-09-23T21:18:57.253064+0000","last_peered":"2023-09-23T21:18:57.253064+0000","last_clean":"2023-09-23T21:18:57.253064+0000","last_became_active":"2023-09-23T21:17:51.872537+0000","last_became_peered":"2023-09-23T21:17:51.872537+0000","last_unstale":"2023-09-23T21:18:57.253064+0000","last_undegraded":"2023-09-23T21:18:57.253064+0000","last_fullsized":"2023-09-23T21:18:57.253064+0000","mapping_epoch":591,"log_start":"550'3191","ondisk_log_start":"550'3191","created":544,"last_epoch_clean":592,"parent":"0.0","parent_split_bits":0,"last_scrub":"623'7891","last_scrub_stamp":"2023-09-23T21:18:46.551470+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_clean_scrub_stamp":"2023-09-23T21:18:46.551470+0000","objects_scrubbed":325,"log_size":4857,"log_dups_size":0,"ondisk_log_size":4857,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:22:45.095160+0000","scrub_duration":0.104744095,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":11010048,"num_objects":168,"num_object_clones":0,"num_object_copies":504,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":168,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":157,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,2],"acting":[7,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.9","version":"79'161","reported_seq":834,"reported_epoch":632,"state":"active+clean","last_fresh":"2023-09-23T21:18:54.902363+0000","last_change":"2023-09-23T21:16:45.433968+0000","last_active":"2023-09-23T21:18:54.902363+0000","last_peered":"2023-09-23T21:18:54.902363+0000","last_clean":"2023-09-23T21:18:54.902363+0000","last_became_active":"2023-09-23T21:16:45.433602+0000","last_became_peered":"2023-09-23T21:16:45.433602+0000","last_unstale":"2023-09-23T21:18:54.902363+0000","last_undegraded":"2023-09-23T21:18:54.902363+0000","last_fullsized":"2023-09-23T21:18:54.902363+0000","mapping_epoch":567,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":568,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:15:43.437369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:15:43.437369+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:16:44.538102+0000","scrub_duration":0.0024129429999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,3],"acting":[2,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":192,"reported_epoch":631,"state":"active+clean","last_fresh":"2023-09-23T21:18:53.895623+0000","last_change":"2023-09-23T21:13:30.793273+0000","last_active":"2023-09-23T21:18:53.895623+0000","last_peered":"2023-09-23T21:18:53.895623+0000","last_clean":"2023-09-23T21:18:53.895623+0000","last_became_active":"2023-09-23T21:13:30.791269+0000","last_became_peered":"2023-09-23T21:13:30.791269+0000","last_unstale":"2023-09-23T21:18:53.895623+0000","last_undegraded":"2023-09-23T21:18:53.895623+0000","last_fullsized":"2023-09-23T21:18:53.895623+0000","mapping_epoch":501,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:38:07.142290+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":227,"reported_epoch":633,"state":"active+clean","last_fresh":"2023-09-23T21:18:55.908413+0000","last_change":"2023-09-23T21:16:42.417057+0000","last_active":"2023-09-23T21:18:55.908413+0000","last_peered":"2023-09-23T21:18:55.908413+0000","last_clean":"2023-09-23T21:18:55.908413+0000","last_became_active":"2023-09-23T21:16:42.416601+0000","last_became_peered":"2023-09-23T21:16:42.416601+0000","last_unstale":"2023-09-23T21:18:55.908413+0000","last_undegraded":"2023-09-23T21:18:55.908413+0000","last_fullsized":"2023-09-23T21:18:55.908413+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:56:52.167562+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,7],"acting":[4,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.2","version":"633'8278","reported_seq":8409,"reported_epoch":633,"state":"active+clean","last_fresh":"2023-09-23T21:18:56.248757+0000","last_change":"2023-09-23T21:18:34.137284+0000","last_active":"2023-09-23T21:18:56.248757+0000","last_peered":"2023-09-23T21:18:56.248757+0000","last_clean":"2023-09-23T21:18:56.248757+0000","last_became_active":"2023-09-23T21:16:32.867780+0000","last_became_peered":"2023-09-23T21:16:32.867780+0000","last_unstale":"2023-09-23T21:18:56.248757+0000","last_undegraded":"2023-09-23T21:18:56.248757+0000","last_fullsized":"2023-09-23T21:18:56.248757+0000","mapping_epoch":544,"log_start":"548'808","ondisk_log_start":"548'808","created":544,"last_epoch_clean":560,"parent":"0.0","parent_split_bits":0,"last_scrub":"613'7927","last_scrub_stamp":"2023-09-23T21:18:34.136961+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_clean_scrub_stamp":"2023-09-23T21:18:34.136961+0000","objects_scrubbed":557,"log_size":7470,"log_dups_size":0,"ondisk_log_size":7470,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:53:14.227837+0000","scrub_duration":0.135147037,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13434880,"num_objects":205,"num_object_clones":0,"num_object_copies":615,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":205,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":352,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.8","version":"79'161","reported_seq":767,"reported_epoch":632,"state":"active+clean","last_fresh":"2023-09-23T21:18:54.902825+0000","last_change":"2023-09-23T21:15:36.367819+0000","last_active":"2023-09-23T21:18:54.902825+0000","last_peered":"2023-09-23T21:18:54.902825+0000","last_clean":"2023-09-23T21:18:54.902825+0000","last_became_active":"2023-09-23T21:12:30.433217+0000","last_became_peered":"2023-09-23T21:12:30.433217+0000","last_unstale":"2023-09-23T21:18:54.902825+0000","last_undegraded":"2023-09-23T21:18:54.902825+0000","last_fullsized":"2023-09-23T21:18:54.902825+0000","mapping_epoch":359,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:15:36.367640+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:15:36.367640+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:49:35.243653+0000","scrub_duration":0.005826899,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":210,"reported_epoch":633,"state":"active+clean","last_fresh":"2023-09-23T21:18:55.908016+0000","last_change":"2023-09-23T21:12:10.823380+0000","last_active":"2023-09-23T21:18:55.908016+0000","last_peered":"2023-09-23T21:18:55.908016+0000","last_clean":"2023-09-23T21:18:55.908016+0000","last_became_active":"2023-09-23T21:12:10.823044+0000","last_became_peered":"2023-09-23T21:12:10.823044+0000","last_unstale":"2023-09-23T21:18:55.908016+0000","last_undegraded":"2023-09-23T21:18:55.908016+0000","last_fullsized":"2023-09-23T21:18:55.908016+0000","mapping_epoch":470,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:53:29.846536+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,5],"acting":[4,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":211,"reported_epoch":633,"state":"active+clean","last_fresh":"2023-09-23T21:18:55.908673+0000","last_change":"2023-09-23T21:13:30.772451+0000","last_active":"2023-09-23T21:18:55.908673+0000","last_peered":"2023-09-23T21:18:55.908673+0000","last_clean":"2023-09-23T21:18:55.908673+0000","last_became_active":"2023-09-23T21:13:30.772040+0000","last_became_peered":"2023-09-23T21:13:30.772040+0000","last_unstale":"2023-09-23T21:18:55.908673+0000","last_undegraded":"2023-09-23T21:18:55.908673+0000","last_fullsized":"2023-09-23T21:18:55.908673+0000","mapping_epoch":501,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:24:49.246727+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,0],"acting":[4,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.1","version":"631'8386","reported_seq":8513,"reported_epoch":631,"state":"active+clean","last_fresh":"2023-09-23T21:18:54.366121+0000","last_change":"2023-09-23T21:18:35.452919+0000","last_active":"2023-09-23T21:18:54.366121+0000","last_peered":"2023-09-23T21:18:54.366121+0000","last_clean":"2023-09-23T21:18:54.366121+0000","last_became_active":"2023-09-23T21:16:32.905672+0000","last_became_peered":"2023-09-23T21:16:32.905672+0000","last_unstale":"2023-09-23T21:18:54.366121+0000","last_undegraded":"2023-09-23T21:18:54.366121+0000","last_fullsized":"2023-09-23T21:18:54.366121+0000","mapping_epoch":544,"log_start":"556'4438","ondisk_log_start":"556'4438","created":544,"last_epoch_clean":560,"parent":"0.0","parent_split_bits":0,"last_scrub":"614'8079","last_scrub_stamp":"2023-09-23T21:18:35.452658+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_clean_scrub_stamp":"2023-09-23T21:18:35.452658+0000","objects_scrubbed":563,"log_size":3948,"log_dups_size":0,"ondisk_log_size":3948,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:19:47.092568+0000","scrub_duration":0.23111627100000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":16252928,"num_objects":248,"num_object_clones":0,"num_object_copies":744,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":248,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":315,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,2],"acting":[3,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.b","version":"79'161","reported_seq":649,"reported_epoch":631,"state":"active+clean","last_fresh":"2023-09-23T21:18:53.895210+0000","last_change":"2023-09-23T21:15:36.340009+0000","last_active":"2023-09-23T21:18:53.895210+0000","last_peered":"2023-09-23T21:18:53.895210+0000","last_clean":"2023-09-23T21:18:53.895210+0000","last_became_active":"2023-09-23T21:13:13.372328+0000","last_became_peered":"2023-09-23T21:13:13.372328+0000","last_unstale":"2023-09-23T21:18:53.895210+0000","last_undegraded":"2023-09-23T21:18:53.895210+0000","last_fullsized":"2023-09-23T21:18:53.895210+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":495,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:15:36.339885+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:15:36.339885+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:01:26.605006+0000","scrub_duration":0.0068473279999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":211,"reported_epoch":633,"state":"active+clean","last_fresh":"2023-09-23T21:18:55.907577+0000","last_change":"2023-09-23T21:16:42.425103+0000","last_active":"2023-09-23T21:18:55.907577+0000","last_peered":"2023-09-23T21:18:55.907577+0000","last_clean":"2023-09-23T21:18:55.907577+0000","last_became_active":"2023-09-23T21:16:42.424543+0000","last_became_peered":"2023-09-23T21:16:42.424543+0000","last_unstale":"2023-09-23T21:18:55.907577+0000","last_undegraded":"2023-09-23T21:18:55.907577+0000","last_fullsized":"2023-09-23T21:18:55.907577+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:11:28.885246+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":565,"reported_epoch":633,"state":"active+clean","last_fresh":"2023-09-23T21:18:55.908976+0000","last_change":"2023-09-23T21:12:10.811698+0000","last_active":"2023-09-23T21:18:55.908976+0000","last_peered":"2023-09-23T21:18:55.908976+0000","last_clean":"2023-09-23T21:18:55.908976+0000","last_became_active":"2023-09-23T21:12:10.789656+0000","last_became_peered":"2023-09-23T21:12:10.789656+0000","last_unstale":"2023-09-23T21:18:55.908976+0000","last_undegraded":"2023-09-23T21:18:55.908976+0000","last_fullsized":"2023-09-23T21:18:55.908976+0000","mapping_epoch":470,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:39:15.112091+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,5],"acting":[4,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.0","version":"631'8079","reported_seq":9819,"reported_epoch":631,"state":"active+clean","last_fresh":"2023-09-23T21:18:54.311733+0000","last_change":"2023-09-23T21:18:10.533031+0000","last_active":"2023-09-23T21:18:54.311733+0000","last_peered":"2023-09-23T21:18:54.311733+0000","last_clean":"2023-09-23T21:18:54.311733+0000","last_became_active":"2023-09-23T21:18:10.519762+0000","last_became_peered":"2023-09-23T21:18:10.519762+0000","last_unstale":"2023-09-23T21:18:54.311733+0000","last_undegraded":"2023-09-23T21:18:54.311733+0000","last_fullsized":"2023-09-23T21:18:54.311733+0000","mapping_epoch":598,"log_start":"556'3927","ondisk_log_start":"556'3927","created":544,"last_epoch_clean":599,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_clean_scrub_stamp":"2023-09-23T21:15:24.860968+0000","objects_scrubbed":0,"log_size":4152,"log_dups_size":0,"ondisk_log_size":4152,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":14254080,"num_objects":217,"num_object_clones":0,"num_object_copies":651,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":217,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5821,"num_write_kb":193248,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":803,"num_bytes_recovered":42336256,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.a","version":"79'161","reported_seq":793,"reported_epoch":632,"state":"active+clean","last_fresh":"2023-09-23T21:18:54.902154+0000","last_change":"2023-09-23T21:15:46.420777+0000","last_active":"2023-09-23T21:18:54.902154+0000","last_peered":"2023-09-23T21:18:54.902154+0000","last_clean":"2023-09-23T21:18:54.902154+0000","last_became_active":"2023-09-23T21:12:30.437427+0000","last_became_peered":"2023-09-23T21:12:30.437427+0000","last_unstale":"2023-09-23T21:18:54.902154+0000","last_undegraded":"2023-09-23T21:18:54.902154+0000","last_fullsized":"2023-09-23T21:18:54.902154+0000","mapping_epoch":446,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:15:46.420650+0000","last_deep_scrub":"79'161","last_deep_scrub_stamp":"2023-09-23T21:05:32.630292+0000","last_clean_scrub_stamp":"2023-09-23T21:15:46.420650+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:50:23.452475+0000","scrub_duration":0.0072898930000000004,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,4],"acting":[2,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":497,"reported_epoch":630,"state":"active+clean","last_fresh":"2023-09-23T21:18:52.898573+0000","last_change":"2023-09-23T21:16:42.447403+0000","last_active":"2023-09-23T21:18:52.898573+0000","last_peered":"2023-09-23T21:18:52.898573+0000","last_clean":"2023-09-23T21:18:52.898573+0000","last_became_active":"2023-09-23T21:16:42.447080+0000","last_became_peered":"2023-09-23T21:16:42.447080+0000","last_unstale":"2023-09-23T21:18:52.898573+0000","last_undegraded":"2023-09-23T21:18:52.898573+0000","last_fullsized":"2023-09-23T21:18:52.898573+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:15:19.446334+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,4],"acting":[3,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":628,"reported_epoch":630,"state":"active+clean","last_fresh":"2023-09-23T21:18:52.900220+0000","last_change":"2023-09-23T21:16:42.518739+0000","last_active":"2023-09-23T21:18:52.900220+0000","last_peered":"2023-09-23T21:18:52.900220+0000","last_clean":"2023-09-23T21:18:52.900220+0000","last_became_active":"2023-09-23T21:16:42.517945+0000","last_became_peered":"2023-09-23T21:16:42.517945+0000","last_unstale":"2023-09-23T21:18:52.900220+0000","last_undegraded":"2023-09-23T21:18:52.900220+0000","last_fullsized":"2023-09-23T21:18:52.900220+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:47:26.236933+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.d","version":"633'12217","reported_seq":12317,"reported_epoch":633,"state":"active+clean","last_fresh":"2023-09-23T21:18:56.509640+0000","last_change":"2023-09-23T21:15:25.892834+0000","last_active":"2023-09-23T21:18:56.509640+0000","last_peered":"2023-09-23T21:18:56.509640+0000","last_clean":"2023-09-23T21:18:56.509640+0000","last_became_active":"2023-09-23T21:15:25.892519+0000","last_became_peered":"2023-09-23T21:15:25.892519+0000","last_unstale":"2023-09-23T21:18:56.509640+0000","last_undegraded":"2023-09-23T21:18:56.509640+0000","last_fullsized":"2023-09-23T21:18:56.509640+0000","mapping_epoch":544,"log_start":"573'6590","ondisk_log_start":"573'6590","created":544,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_clean_scrub_stamp":"2023-09-23T21:15:24.860968+0000","objects_scrubbed":0,"log_size":5627,"log_dups_size":0,"ondisk_log_size":5627,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:46:28.375665+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":24510488,"num_objects":375,"num_object_clones":0,"num_object_copies":1125,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":375,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":12217,"num_write_kb":402881,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,5],"acting":[2,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.7","version":"79'161","reported_seq":771,"reported_epoch":630,"state":"active+clean","last_fresh":"2023-09-23T21:18:52.900109+0000","last_change":"2023-09-23T21:16:45.425645+0000","last_active":"2023-09-23T21:18:52.900109+0000","last_peered":"2023-09-23T21:18:52.900109+0000","last_clean":"2023-09-23T21:18:52.900109+0000","last_became_active":"2023-09-23T21:16:45.425241+0000","last_became_peered":"2023-09-23T21:16:45.425241+0000","last_unstale":"2023-09-23T21:18:52.900109+0000","last_undegraded":"2023-09-23T21:18:52.900109+0000","last_fullsized":"2023-09-23T21:18:52.900109+0000","mapping_epoch":567,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":568,"parent":"0.0","parent_split_bits":3,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:15:51.016427+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:15:51.016427+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:49:31.176462+0000","scrub_duration":0.0017648640000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":581,"reported_epoch":631,"state":"active+clean","last_fresh":"2023-09-23T21:18:53.895263+0000","last_change":"2023-09-23T21:16:42.518826+0000","last_active":"2023-09-23T21:18:53.895263+0000","last_peered":"2023-09-23T21:18:53.895263+0000","last_clean":"2023-09-23T21:18:53.895263+0000","last_became_active":"2023-09-23T21:16:42.518488+0000","last_became_peered":"2023-09-23T21:16:42.518488+0000","last_unstale":"2023-09-23T21:18:53.895263+0000","last_undegraded":"2023-09-23T21:18:53.895263+0000","last_fullsized":"2023-09-23T21:18:53.895263+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:42:44.144383+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,2],"acting":[0,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":311,"reported_epoch":633,"state":"active+clean","last_fresh":"2023-09-23T21:18:55.914372+0000","last_change":"2023-09-23T21:05:04.326889+0000","last_active":"2023-09-23T21:18:55.914372+0000","last_peered":"2023-09-23T21:18:55.914372+0000","last_clean":"2023-09-23T21:18:55.914372+0000","last_became_active":"2023-09-23T21:05:04.326032+0000","last_became_peered":"2023-09-23T21:05:04.326032+0000","last_unstale":"2023-09-23T21:18:55.914372+0000","last_undegraded":"2023-09-23T21:18:55.914372+0000","last_fullsized":"2023-09-23T21:18:55.914372+0000","mapping_epoch":332,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:00:17.507889+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":643,"reported_epoch":633,"state":"active+clean","last_fresh":"2023-09-23T21:18:55.915410+0000","last_change":"2023-09-23T21:13:13.285608+0000","last_active":"2023-09-23T21:18:55.915410+0000","last_peered":"2023-09-23T21:18:55.915410+0000","last_clean":"2023-09-23T21:18:55.915410+0000","last_became_active":"2023-09-23T21:13:13.285255+0000","last_became_peered":"2023-09-23T21:13:13.285255+0000","last_unstale":"2023-09-23T21:18:55.915410+0000","last_undegraded":"2023-09-23T21:18:55.915410+0000","last_fullsized":"2023-09-23T21:18:55.915410+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":495,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:42:30.087599+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.b","version":"631'11514","reported_seq":11611,"reported_epoch":631,"state":"active+clean","last_fresh":"2023-09-23T21:18:54.407119+0000","last_change":"2023-09-23T21:15:25.906791+0000","last_active":"2023-09-23T21:18:54.407119+0000","last_peered":"2023-09-23T21:18:54.407119+0000","last_clean":"2023-09-23T21:18:54.407119+0000","last_became_active":"2023-09-23T21:15:25.906512+0000","last_became_peered":"2023-09-23T21:15:25.906512+0000","last_unstale":"2023-09-23T21:18:54.407119+0000","last_undegraded":"2023-09-23T21:18:54.407119+0000","last_fullsized":"2023-09-23T21:18:54.407119+0000","mapping_epoch":544,"log_start":"580'7645","ondisk_log_start":"580'7645","created":544,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_clean_scrub_stamp":"2023-09-23T21:15:24.860968+0000","objects_scrubbed":0,"log_size":3869,"log_dups_size":0,"ondisk_log_size":3869,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:58:51.749775+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":28573696,"num_objects":436,"num_object_clones":0,"num_object_copies":1308,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":436,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11514,"num_write_kb":382400,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1","version":"79'161","reported_seq":716,"reported_epoch":631,"state":"active+clean","last_fresh":"2023-09-23T21:18:53.895374+0000","last_change":"2023-09-23T21:15:34.345290+0000","last_active":"2023-09-23T21:18:53.895374+0000","last_peered":"2023-09-23T21:18:53.895374+0000","last_clean":"2023-09-23T21:18:53.895374+0000","last_became_active":"2023-09-23T21:13:12.203380+0000","last_became_peered":"2023-09-23T21:13:12.203380+0000","last_unstale":"2023-09-23T21:18:53.895374+0000","last_undegraded":"2023-09-23T21:18:53.895374+0000","last_fullsized":"2023-09-23T21:18:53.895374+0000","mapping_epoch":493,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:15:34.345138+0000","last_deep_scrub":"79'161","last_deep_scrub_stamp":"2023-09-23T21:05:30.652189+0000","last_clean_scrub_stamp":"2023-09-23T21:15:34.345138+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:47:50.349409+0000","scrub_duration":0.0057520000000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":300,"reported_epoch":631,"state":"active+clean","last_fresh":"2023-09-23T21:18:53.895013+0000","last_change":"2023-09-23T21:13:13.238617+0000","last_active":"2023-09-23T21:18:53.895013+0000","last_peered":"2023-09-23T21:18:53.895013+0000","last_clean":"2023-09-23T21:18:53.895013+0000","last_became_active":"2023-09-23T21:13:13.232926+0000","last_became_peered":"2023-09-23T21:13:13.232926+0000","last_unstale":"2023-09-23T21:18:53.895013+0000","last_undegraded":"2023-09-23T21:18:53.895013+0000","last_fullsized":"2023-09-23T21:18:53.895013+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":495,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:17:21.823772+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":567,"reported_epoch":633,"state":"active+clean","last_fresh":"2023-09-23T21:18:55.909394+0000","last_change":"2023-09-23T21:13:14.144918+0000","last_active":"2023-09-23T21:18:55.909394+0000","last_peered":"2023-09-23T21:18:55.909394+0000","last_clean":"2023-09-23T21:18:55.909394+0000","last_became_active":"2023-09-23T21:13:14.132388+0000","last_became_peered":"2023-09-23T21:13:14.132388+0000","last_unstale":"2023-09-23T21:18:55.909394+0000","last_undegraded":"2023-09-23T21:18:55.909394+0000","last_fullsized":"2023-09-23T21:18:55.909394+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":495,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:56:02.619905+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,5],"acting":[4,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":473,"reported_epoch":632,"state":"active+clean","last_fresh":"2023-09-23T21:18:54.904639+0000","last_change":"2023-09-23T21:16:42.491111+0000","last_active":"2023-09-23T21:18:54.904639+0000","last_peered":"2023-09-23T21:18:54.904639+0000","last_clean":"2023-09-23T21:18:54.904639+0000","last_became_active":"2023-09-23T21:16:42.490275+0000","last_became_peered":"2023-09-23T21:16:42.490275+0000","last_unstale":"2023-09-23T21:18:54.904639+0000","last_undegraded":"2023-09-23T21:18:54.904639+0000","last_fullsized":"2023-09-23T21:18:54.904639+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:39:11.652564+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,7],"acting":[2,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"11.8","version":"632'8155","reported_seq":8287,"reported_epoch":632,"state":"active+clean","last_fresh":"2023-09-23T21:18:55.413488+0000","last_change":"2023-09-23T21:18:30.671221+0000","last_active":"2023-09-23T21:18:55.413488+0000","last_peered":"2023-09-23T21:18:55.413488+0000","last_clean":"2023-09-23T21:18:55.413488+0000","last_became_active":"2023-09-23T21:16:32.894560+0000","last_became_peered":"2023-09-23T21:16:32.894560+0000","last_unstale":"2023-09-23T21:18:55.413488+0000","last_undegraded":"2023-09-23T21:18:55.413488+0000","last_fullsized":"2023-09-23T21:18:55.413488+0000","mapping_epoch":544,"log_start":"556'3779","ondisk_log_start":"556'3779","created":544,"last_epoch_clean":560,"parent":"0.0","parent_split_bits":0,"last_scrub":"609'7757","last_scrub_stamp":"2023-09-23T21:18:30.670962+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_clean_scrub_stamp":"2023-09-23T21:18:30.670962+0000","objects_scrubbed":619,"log_size":4376,"log_dups_size":0,"ondisk_log_size":4376,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:13:50.200821+0000","scrub_duration":0.101488842,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":14483456,"num_objects":221,"num_object_clones":0,"num_object_copies":663,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":221,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":398,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,5],"acting":[0,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.2","version":"79'161","reported_seq":947,"reported_epoch":631,"state":"active+clean","last_fresh":"2023-09-23T21:18:53.894794+0000","last_change":"2023-09-23T21:16:45.417075+0000","last_active":"2023-09-23T21:18:53.894794+0000","last_peered":"2023-09-23T21:18:53.894794+0000","last_clean":"2023-09-23T21:18:53.894794+0000","last_became_active":"2023-09-23T21:16:45.416752+0000","last_became_peered":"2023-09-23T21:16:45.416752+0000","last_unstale":"2023-09-23T21:18:53.894794+0000","last_undegraded":"2023-09-23T21:18:53.894794+0000","last_fullsized":"2023-09-23T21:18:53.894794+0000","mapping_epoch":567,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":568,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:24.802451+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:24.802451+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:01:51.129779+0000","scrub_duration":0.0090861250000000005,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,4],"acting":[0,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":643,"reported_epoch":632,"state":"active+clean","last_fresh":"2023-09-23T21:18:54.902003+0000","last_change":"2023-09-23T21:13:12.206980+0000","last_active":"2023-09-23T21:18:54.902003+0000","last_peered":"2023-09-23T21:18:54.902003+0000","last_clean":"2023-09-23T21:18:54.902003+0000","last_became_active":"2023-09-23T21:13:12.206650+0000","last_became_peered":"2023-09-23T21:13:12.206650+0000","last_unstale":"2023-09-23T21:18:54.902003+0000","last_undegraded":"2023-09-23T21:18:54.902003+0000","last_fullsized":"2023-09-23T21:18:54.902003+0000","mapping_epoch":493,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:10:01.892936+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":627,"reported_epoch":632,"state":"active+clean","last_fresh":"2023-09-23T21:18:54.902901+0000","last_change":"2023-09-23T21:13:13.407073+0000","last_active":"2023-09-23T21:18:54.902901+0000","last_peered":"2023-09-23T21:18:54.902901+0000","last_clean":"2023-09-23T21:18:54.902901+0000","last_became_active":"2023-09-23T21:13:13.395047+0000","last_became_peered":"2023-09-23T21:13:13.395047+0000","last_unstale":"2023-09-23T21:18:54.902901+0000","last_undegraded":"2023-09-23T21:18:54.902901+0000","last_fullsized":"2023-09-23T21:18:54.902901+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":495,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:04:43.384190+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,5],"acting":[2,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":161,"reported_epoch":631,"state":"active+clean","last_fresh":"2023-09-23T21:18:53.896010+0000","last_change":"2023-09-23T21:16:42.517729+0000","last_active":"2023-09-23T21:18:53.896010+0000","last_peered":"2023-09-23T21:18:53.896010+0000","last_clean":"2023-09-23T21:18:53.896010+0000","last_became_active":"2023-09-23T21:16:42.517373+0000","last_became_peered":"2023-09-23T21:16:42.517373+0000","last_unstale":"2023-09-23T21:18:53.896010+0000","last_undegraded":"2023-09-23T21:18:53.896010+0000","last_fullsized":"2023-09-23T21:18:53.896010+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:05:57.370550+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.a","version":"633'12017","reported_seq":12116,"reported_epoch":633,"state":"active+clean","last_fresh":"2023-09-23T21:18:56.571213+0000","last_change":"2023-09-23T21:15:25.893846+0000","last_active":"2023-09-23T21:18:56.571213+0000","last_peered":"2023-09-23T21:18:56.571213+0000","last_clean":"2023-09-23T21:18:56.571213+0000","last_became_active":"2023-09-23T21:15:25.893452+0000","last_became_peered":"2023-09-23T21:15:25.893452+0000","last_unstale":"2023-09-23T21:18:56.571213+0000","last_undegraded":"2023-09-23T21:18:56.571213+0000","last_fullsized":"2023-09-23T21:18:56.571213+0000","mapping_epoch":544,"log_start":"573'6394","ondisk_log_start":"573'6394","created":544,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_clean_scrub_stamp":"2023-09-23T21:15:24.860968+0000","objects_scrubbed":0,"log_size":5623,"log_dups_size":0,"ondisk_log_size":5623,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:42:32.842297+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22085632,"num_objects":337,"num_object_clones":0,"num_object_copies":1011,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":337,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12017,"num_write_kb":395328,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,5],"acting":[2,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":438,"reported_epoch":632,"state":"active+clean","last_fresh":"2023-09-23T21:18:54.902102+0000","last_change":"2023-09-23T21:16:42.423216+0000","last_active":"2023-09-23T21:18:54.902102+0000","last_peered":"2023-09-23T21:18:54.902102+0000","last_clean":"2023-09-23T21:18:54.902102+0000","last_became_active":"2023-09-23T21:16:42.422856+0000","last_became_peered":"2023-09-23T21:16:42.422856+0000","last_unstale":"2023-09-23T21:18:54.902102+0000","last_undegraded":"2023-09-23T21:18:54.902102+0000","last_fullsized":"2023-09-23T21:18:54.902102+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:06:54.105798+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"79'161","reported_seq":850,"reported_epoch":631,"state":"active+clean","last_fresh":"2023-09-23T21:18:53.894605+0000","last_change":"2023-09-23T21:16:45.428032+0000","last_active":"2023-09-23T21:18:53.894605+0000","last_peered":"2023-09-23T21:18:53.894605+0000","last_clean":"2023-09-23T21:18:53.894605+0000","last_became_active":"2023-09-23T21:16:45.427674+0000","last_became_peered":"2023-09-23T21:16:45.427674+0000","last_unstale":"2023-09-23T21:18:53.894605+0000","last_undegraded":"2023-09-23T21:18:53.894605+0000","last_fullsized":"2023-09-23T21:18:53.894605+0000","mapping_epoch":567,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":568,"parent":"0.0","parent_split_bits":0,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:20.738273+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:20.738273+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:39:42.441711+0000","scrub_duration":0.0088336339999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,2],"acting":[0,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":641,"reported_epoch":633,"state":"active+clean","last_fresh":"2023-09-23T21:18:55.915368+0000","last_change":"2023-09-23T21:11:08.760417+0000","last_active":"2023-09-23T21:18:55.915368+0000","last_peered":"2023-09-23T21:18:55.915368+0000","last_clean":"2023-09-23T21:18:55.915368+0000","last_became_active":"2023-09-23T21:11:08.760150+0000","last_became_peered":"2023-09-23T21:11:08.760150+0000","last_unstale":"2023-09-23T21:18:55.915368+0000","last_undegraded":"2023-09-23T21:18:55.915368+0000","last_fullsized":"2023-09-23T21:18:55.915368+0000","mapping_epoch":443,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":444,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:23:10.849768+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":228,"reported_epoch":633,"state":"active+clean","last_fresh":"2023-09-23T21:18:55.908766+0000","last_change":"2023-09-23T21:16:42.424717+0000","last_active":"2023-09-23T21:18:55.908766+0000","last_peered":"2023-09-23T21:18:55.908766+0000","last_clean":"2023-09-23T21:18:55.908766+0000","last_became_active":"2023-09-23T21:16:42.424376+0000","last_became_peered":"2023-09-23T21:16:42.424376+0000","last_unstale":"2023-09-23T21:18:55.908766+0000","last_undegraded":"2023-09-23T21:18:55.908766+0000","last_fullsized":"2023-09-23T21:18:55.908766+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:25:14.162321+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.9","version":"631'8312","reported_seq":10168,"reported_epoch":631,"state":"active+clean","last_fresh":"2023-09-23T21:18:54.358758+0000","last_change":"2023-09-23T21:17:58.411362+0000","last_active":"2023-09-23T21:18:54.358758+0000","last_peered":"2023-09-23T21:18:54.358758+0000","last_clean":"2023-09-23T21:18:54.358758+0000","last_became_active":"2023-09-23T21:17:58.399421+0000","last_became_peered":"2023-09-23T21:17:58.399421+0000","last_unstale":"2023-09-23T21:18:54.358758+0000","last_undegraded":"2023-09-23T21:18:54.358758+0000","last_fullsized":"2023-09-23T21:18:54.358758+0000","mapping_epoch":593,"log_start":"556'4337","ondisk_log_start":"556'4337","created":544,"last_epoch_clean":594,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_clean_scrub_stamp":"2023-09-23T21:15:24.860968+0000","objects_scrubbed":0,"log_size":3975,"log_dups_size":0,"ondisk_log_size":3975,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":16547840,"num_objects":252,"num_object_clones":0,"num_object_copies":756,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":252,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6028,"num_write_kb":200992,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":860,"num_bytes_recovered":56393728,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,7],"acting":[3,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.3","version":"79'161","reported_seq":936,"reported_epoch":633,"state":"active+clean","last_fresh":"2023-09-23T21:18:55.908672+0000","last_change":"2023-09-23T21:13:33.771279+0000","last_active":"2023-09-23T21:18:55.908672+0000","last_peered":"2023-09-23T21:18:55.908672+0000","last_clean":"2023-09-23T21:18:55.908672+0000","last_became_active":"2023-09-23T21:13:33.770980+0000","last_became_peered":"2023-09-23T21:13:33.770980+0000","last_unstale":"2023-09-23T21:18:55.908672+0000","last_undegraded":"2023-09-23T21:18:55.908672+0000","last_fullsized":"2023-09-23T21:18:55.908672+0000","mapping_epoch":504,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":505,"parent":"0.0","parent_split_bits":3,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:05:57.832302+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:05:57.832302+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:19:42.309604+0000","scrub_duration":0.010206847999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":468,"reported_epoch":631,"state":"active+clean","last_fresh":"2023-09-23T21:18:53.911513+0000","last_change":"2023-09-23T21:16:42.455365+0000","last_active":"2023-09-23T21:18:53.911513+0000","last_peered":"2023-09-23T21:18:53.911513+0000","last_clean":"2023-09-23T21:18:53.911513+0000","last_became_active":"2023-09-23T21:16:42.452829+0000","last_became_peered":"2023-09-23T21:16:42.452829+0000","last_unstale":"2023-09-23T21:18:53.911513+0000","last_undegraded":"2023-09-23T21:18:53.911513+0000","last_fullsized":"2023-09-23T21:18:53.911513+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:49:26.303389+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":632,"reported_epoch":631,"state":"active+clean","last_fresh":"2023-09-23T21:18:53.895089+0000","last_change":"2023-09-23T21:13:44.409212+0000","last_active":"2023-09-23T21:18:53.895089+0000","last_peered":"2023-09-23T21:18:53.895089+0000","last_clean":"2023-09-23T21:18:53.895089+0000","last_became_active":"2023-09-23T21:13:44.408958+0000","last_became_peered":"2023-09-23T21:13:44.408958+0000","last_unstale":"2023-09-23T21:18:53.895089+0000","last_undegraded":"2023-09-23T21:18:53.895089+0000","last_fullsized":"2023-09-23T21:18:53.895089+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:03:52.749964+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":161,"reported_epoch":631,"state":"active+clean","last_fresh":"2023-09-23T21:18:53.895645+0000","last_change":"2023-09-23T21:16:42.427490+0000","last_active":"2023-09-23T21:18:53.895645+0000","last_peered":"2023-09-23T21:18:53.895645+0000","last_clean":"2023-09-23T21:18:53.895645+0000","last_became_active":"2023-09-23T21:16:42.426566+0000","last_became_peered":"2023-09-23T21:16:42.426566+0000","last_unstale":"2023-09-23T21:18:53.895645+0000","last_undegraded":"2023-09-23T21:18:53.895645+0000","last_fullsized":"2023-09-23T21:18:53.895645+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:20:26.396005+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,7],"acting":[0,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.c","version":"633'11779","reported_seq":14983,"reported_epoch":633,"state":"active+clean","last_fresh":"2023-09-23T21:18:56.523465+0000","last_change":"2023-09-23T21:17:21.358806+0000","last_active":"2023-09-23T21:18:56.523465+0000","last_peered":"2023-09-23T21:18:56.523465+0000","last_clean":"2023-09-23T21:18:56.523465+0000","last_became_active":"2023-09-23T21:17:21.346629+0000","last_became_peered":"2023-09-23T21:17:21.346629+0000","last_unstale":"2023-09-23T21:18:56.523465+0000","last_undegraded":"2023-09-23T21:18:56.523465+0000","last_fullsized":"2023-09-23T21:18:56.523465+0000","mapping_epoch":576,"log_start":"573'6154","ondisk_log_start":"573'6154","created":544,"last_epoch_clean":577,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_clean_scrub_stamp":"2023-09-23T21:15:24.860968+0000","objects_scrubbed":0,"log_size":5625,"log_dups_size":0,"ondisk_log_size":5625,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:59:48.173244+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26017792,"num_objects":397,"num_object_clones":0,"num_object_copies":1191,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":397,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11779,"num_write_kb":389632,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1539,"num_bytes_recovered":100859904,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,7],"acting":[2,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.6","version":"79'161","reported_seq":564,"reported_epoch":631,"state":"active+clean","last_fresh":"2023-09-23T21:18:53.895564+0000","last_change":"2023-09-23T21:15:43.333919+0000","last_active":"2023-09-23T21:18:53.895564+0000","last_peered":"2023-09-23T21:18:53.895564+0000","last_clean":"2023-09-23T21:18:53.895564+0000","last_became_active":"2023-09-23T21:13:33.769303+0000","last_became_peered":"2023-09-23T21:13:33.769303+0000","last_unstale":"2023-09-23T21:18:53.895564+0000","last_undegraded":"2023-09-23T21:18:53.895564+0000","last_fullsized":"2023-09-23T21:18:53.895564+0000","mapping_epoch":504,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":505,"parent":"0.0","parent_split_bits":3,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:15:43.333733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:15:43.333733+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:42:29.787357+0000","scrub_duration":0.0092107530000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":79,"reported_epoch":633,"state":"active+clean","last_fresh":"2023-09-23T21:18:55.913698+0000","last_change":"2023-09-23T21:16:42.483813+0000","last_active":"2023-09-23T21:18:55.913698+0000","last_peered":"2023-09-23T21:18:55.913698+0000","last_clean":"2023-09-23T21:18:55.913698+0000","last_became_active":"2023-09-23T21:16:42.482847+0000","last_became_peered":"2023-09-23T21:16:42.482847+0000","last_unstale":"2023-09-23T21:18:55.913698+0000","last_undegraded":"2023-09-23T21:18:55.913698+0000","last_fullsized":"2023-09-23T21:18:55.913698+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:38:57.905679+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":518,"reported_epoch":632,"state":"active+clean","last_fresh":"2023-09-23T21:18:54.904834+0000","last_change":"2023-09-23T21:13:30.790590+0000","last_active":"2023-09-23T21:18:54.904834+0000","last_peered":"2023-09-23T21:18:54.904834+0000","last_clean":"2023-09-23T21:18:54.904834+0000","last_became_active":"2023-09-23T21:13:30.790253+0000","last_became_peered":"2023-09-23T21:13:30.790253+0000","last_unstale":"2023-09-23T21:18:54.904834+0000","last_undegraded":"2023-09-23T21:18:54.904834+0000","last_fullsized":"2023-09-23T21:18:54.904834+0000","mapping_epoch":501,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:23:41.843250+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,0],"acting":[2,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":597,"reported_epoch":631,"state":"active+clean","last_fresh":"2023-09-23T21:18:53.895721+0000","last_change":"2023-09-23T21:16:42.428535+0000","last_active":"2023-09-23T21:18:53.895721+0000","last_peered":"2023-09-23T21:18:53.895721+0000","last_clean":"2023-09-23T21:18:53.895721+0000","last_became_active":"2023-09-23T21:16:42.428149+0000","last_became_peered":"2023-09-23T21:16:42.428149+0000","last_unstale":"2023-09-23T21:18:53.895721+0000","last_undegraded":"2023-09-23T21:18:53.895721+0000","last_fullsized":"2023-09-23T21:18:53.895721+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:38:59.979788+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.f","version":"633'11833","reported_seq":15486,"reported_epoch":633,"state":"active+clean","last_fresh":"2023-09-23T21:18:56.547002+0000","last_change":"2023-09-23T21:17:03.878062+0000","last_active":"2023-09-23T21:18:56.547002+0000","last_peered":"2023-09-23T21:18:56.547002+0000","last_clean":"2023-09-23T21:18:56.547002+0000","last_became_active":"2023-09-23T21:17:03.870998+0000","last_became_peered":"2023-09-23T21:17:03.870998+0000","last_unstale":"2023-09-23T21:18:56.547002+0000","last_undegraded":"2023-09-23T21:18:56.547002+0000","last_fullsized":"2023-09-23T21:18:56.547002+0000","mapping_epoch":572,"log_start":"573'6221","ondisk_log_start":"573'6221","created":544,"last_epoch_clean":573,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_clean_scrub_stamp":"2023-09-23T21:15:24.860968+0000","objects_scrubbed":0,"log_size":5612,"log_dups_size":0,"ondisk_log_size":5612,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:19:31.942955+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":21954560,"num_objects":335,"num_object_clones":0,"num_object_copies":1005,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":335,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11833,"num_write_kb":389376,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1762,"num_bytes_recovered":115474432,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,0],"acting":[2,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.5","version":"79'161","reported_seq":671,"reported_epoch":633,"state":"active+clean","last_fresh":"2023-09-23T21:18:55.908673+0000","last_change":"2023-09-23T21:15:41.108638+0000","last_active":"2023-09-23T21:18:55.908673+0000","last_peered":"2023-09-23T21:18:55.908673+0000","last_clean":"2023-09-23T21:18:55.908673+0000","last_became_active":"2023-09-23T21:13:38.373633+0000","last_became_peered":"2023-09-23T21:13:38.373633+0000","last_unstale":"2023-09-23T21:18:55.908673+0000","last_undegraded":"2023-09-23T21:18:55.908673+0000","last_fullsized":"2023-09-23T21:18:55.908673+0000","mapping_epoch":507,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":3,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:15:41.108579+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:15:41.108579+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:21:58.075649+0000","scrub_duration":0.0061511960000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":200,"reported_epoch":631,"state":"active+clean","last_fresh":"2023-09-23T21:18:53.896260+0000","last_change":"2023-09-23T21:17:02.204086+0000","last_active":"2023-09-23T21:18:53.896260+0000","last_peered":"2023-09-23T21:18:53.896260+0000","last_clean":"2023-09-23T21:18:53.896260+0000","last_became_active":"2023-09-23T21:16:42.517549+0000","last_became_peered":"2023-09-23T21:16:42.517549+0000","last_unstale":"2023-09-23T21:18:53.896260+0000","last_undegraded":"2023-09-23T21:18:53.896260+0000","last_fullsized":"2023-09-23T21:18:53.896260+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:43:07.991844+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":614,"reported_epoch":633,"state":"active+clean","last_fresh":"2023-09-23T21:18:55.908902+0000","last_change":"2023-09-23T21:13:30.807520+0000","last_active":"2023-09-23T21:18:55.908902+0000","last_peered":"2023-09-23T21:18:55.908902+0000","last_clean":"2023-09-23T21:18:55.908902+0000","last_became_active":"2023-09-23T21:13:30.807222+0000","last_became_peered":"2023-09-23T21:13:30.807222+0000","last_unstale":"2023-09-23T21:18:55.908902+0000","last_undegraded":"2023-09-23T21:18:55.908902+0000","last_fullsized":"2023-09-23T21:18:55.908902+0000","mapping_epoch":501,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:40:31.989749+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":492,"reported_epoch":630,"state":"active+clean","last_fresh":"2023-09-23T21:18:52.898573+0000","last_change":"2023-09-23T21:16:42.463411+0000","last_active":"2023-09-23T21:18:52.898573+0000","last_peered":"2023-09-23T21:18:52.898573+0000","last_clean":"2023-09-23T21:18:52.898573+0000","last_became_active":"2023-09-23T21:16:42.463020+0000","last_became_peered":"2023-09-23T21:16:42.463020+0000","last_unstale":"2023-09-23T21:18:52.898573+0000","last_undegraded":"2023-09-23T21:18:52.898573+0000","last_fullsized":"2023-09-23T21:18:52.898573+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:07:34.346129+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.e","version":"634'11891","reported_seq":16123,"reported_epoch":634,"state":"active+clean","last_fresh":"2023-09-23T21:18:57.106767+0000","last_change":"2023-09-23T21:17:34.456174+0000","last_active":"2023-09-23T21:18:57.106767+0000","last_peered":"2023-09-23T21:18:57.106767+0000","last_clean":"2023-09-23T21:18:57.106767+0000","last_became_active":"2023-09-23T21:17:34.444637+0000","last_became_peered":"2023-09-23T21:17:34.444637+0000","last_unstale":"2023-09-23T21:18:57.106767+0000","last_undegraded":"2023-09-23T21:18:57.106767+0000","last_fullsized":"2023-09-23T21:18:57.106767+0000","mapping_epoch":582,"log_start":"575'7223","ondisk_log_start":"575'7223","created":544,"last_epoch_clean":583,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_clean_scrub_stamp":"2023-09-23T21:15:24.860968+0000","objects_scrubbed":0,"log_size":4668,"log_dups_size":0,"ondisk_log_size":4668,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:17:47.772535+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":23920640,"num_objects":365,"num_object_clones":0,"num_object_copies":1095,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":365,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11891,"num_write_kb":392192,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2052,"num_bytes_recovered":134479872,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.4","version":"79'161","reported_seq":1037,"reported_epoch":630,"state":"active+clean","last_fresh":"2023-09-23T21:18:52.899095+0000","last_change":"2023-09-23T21:16:45.426136+0000","last_active":"2023-09-23T21:18:52.899095+0000","last_peered":"2023-09-23T21:18:52.899095+0000","last_clean":"2023-09-23T21:18:52.899095+0000","last_became_active":"2023-09-23T21:16:45.425669+0000","last_became_peered":"2023-09-23T21:16:45.425669+0000","last_unstale":"2023-09-23T21:18:52.899095+0000","last_undegraded":"2023-09-23T21:18:52.899095+0000","last_fullsized":"2023-09-23T21:18:52.899095+0000","mapping_epoch":567,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":568,"parent":"0.0","parent_split_bits":3,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:20.012764+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:20.012764+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:52:39.142509+0000","scrub_duration":0.0045251919999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":592,"reported_epoch":633,"state":"active+clean","last_fresh":"2023-09-23T21:18:55.933439+0000","last_change":"2023-09-23T21:16:42.459732+0000","last_active":"2023-09-23T21:18:55.933439+0000","last_peered":"2023-09-23T21:18:55.933439+0000","last_clean":"2023-09-23T21:18:55.933439+0000","last_became_active":"2023-09-23T21:16:42.459413+0000","last_became_peered":"2023-09-23T21:16:42.459413+0000","last_unstale":"2023-09-23T21:18:55.933439+0000","last_undegraded":"2023-09-23T21:18:55.933439+0000","last_fullsized":"2023-09-23T21:18:55.933439+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:37:44.548863+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,3],"acting":[2,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.6","version":"70'1","reported_seq":633,"reported_epoch":631,"state":"active+clean","last_fresh":"2023-09-23T21:18:53.895861+0000","last_change":"2023-09-23T21:17:02.208077+0000","last_active":"2023-09-23T21:18:53.895861+0000","last_peered":"2023-09-23T21:18:53.895861+0000","last_clean":"2023-09-23T21:18:53.895861+0000","last_became_active":"2023-09-23T21:16:42.431308+0000","last_became_peered":"2023-09-23T21:16:42.431308+0000","last_unstale":"2023-09-23T21:18:53.895861+0000","last_undegraded":"2023-09-23T21:18:53.895861+0000","last_fullsized":"2023-09-23T21:18:53.895861+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:16:49.009013+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":208,"reported_epoch":633,"state":"active+clean","last_fresh":"2023-09-23T21:18:55.907484+0000","last_change":"2023-09-23T21:13:12.216895+0000","last_active":"2023-09-23T21:18:55.907484+0000","last_peered":"2023-09-23T21:18:55.907484+0000","last_clean":"2023-09-23T21:18:55.907484+0000","last_became_active":"2023-09-23T21:13:12.216654+0000","last_became_peered":"2023-09-23T21:13:12.216654+0000","last_unstale":"2023-09-23T21:18:55.907484+0000","last_undegraded":"2023-09-23T21:18:55.907484+0000","last_fullsized":"2023-09-23T21:18:55.907484+0000","mapping_epoch":493,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:02:40.430096+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.13","version":"633'8013","reported_seq":10931,"reported_epoch":633,"state":"active+clean","last_fresh":"2023-09-23T21:18:56.519688+0000","last_change":"2023-09-23T21:18:47.282127+0000","last_active":"2023-09-23T21:18:56.519688+0000","last_peered":"2023-09-23T21:18:56.519688+0000","last_clean":"2023-09-23T21:18:56.519688+0000","last_became_active":"2023-09-23T21:18:42.892705+0000","last_became_peered":"2023-09-23T21:18:42.892705+0000","last_unstale":"2023-09-23T21:18:56.519688+0000","last_undegraded":"2023-09-23T21:18:56.519688+0000","last_fullsized":"2023-09-23T21:18:56.519688+0000","mapping_epoch":620,"log_start":"549'2514","ondisk_log_start":"549'2514","created":559,"last_epoch_clean":621,"parent":"0.0","parent_split_bits":5,"last_scrub":"624'7886","last_scrub_stamp":"2023-09-23T21:18:47.282029+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_clean_scrub_stamp":"2023-09-23T21:18:47.282029+0000","objects_scrubbed":302,"log_size":5499,"log_dups_size":0,"ondisk_log_size":5499,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:49:10.428430+0000","scrub_duration":0.049862798999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":11468800,"num_objects":175,"num_object_clones":0,"num_object_copies":525,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":175,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":127,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.19","version":"79'161","reported_seq":840,"reported_epoch":631,"state":"active+clean","last_fresh":"2023-09-23T21:18:53.894608+0000","last_change":"2023-09-23T21:16:45.428879+0000","last_active":"2023-09-23T21:18:53.894608+0000","last_peered":"2023-09-23T21:18:53.894608+0000","last_clean":"2023-09-23T21:18:53.894608+0000","last_became_active":"2023-09-23T21:16:45.428660+0000","last_became_peered":"2023-09-23T21:16:45.428660+0000","last_unstale":"2023-09-23T21:18:53.894608+0000","last_undegraded":"2023-09-23T21:18:53.894608+0000","last_fullsized":"2023-09-23T21:18:53.894608+0000","mapping_epoch":567,"log_start":"0'0","ondisk_log_start":"0'0","created":479,"last_epoch_clean":568,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:15:48.271791+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:15:48.271791+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:25:26.361045+0000","scrub_duration":0.0057149180000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":161,"reported_epoch":631,"state":"active+clean","last_fresh":"2023-09-23T21:18:53.895696+0000","last_change":"2023-09-23T21:16:42.434830+0000","last_active":"2023-09-23T21:18:53.895696+0000","last_peered":"2023-09-23T21:18:53.895696+0000","last_clean":"2023-09-23T21:18:53.895696+0000","last_became_active":"2023-09-23T21:16:42.430289+0000","last_became_peered":"2023-09-23T21:16:42.430289+0000","last_unstale":"2023-09-23T21:18:53.895696+0000","last_undegraded":"2023-09-23T21:18:53.895696+0000","last_fullsized":"2023-09-23T21:18:53.895696+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:38:48.053766+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":138,"reported_epoch":632,"state":"active+clean","last_fresh":"2023-09-23T21:18:54.904729+0000","last_change":"2023-09-23T21:13:38.373814+0000","last_active":"2023-09-23T21:18:54.904729+0000","last_peered":"2023-09-23T21:18:54.904729+0000","last_clean":"2023-09-23T21:18:54.904729+0000","last_became_active":"2023-09-23T21:13:38.373458+0000","last_became_peered":"2023-09-23T21:13:38.373458+0000","last_unstale":"2023-09-23T21:18:54.904729+0000","last_undegraded":"2023-09-23T21:18:54.904729+0000","last_fullsized":"2023-09-23T21:18:54.904729+0000","mapping_epoch":507,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:23:08.288401+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"11.14","version":"634'8140","reported_seq":11089,"reported_epoch":634,"state":"active+clean","last_fresh":"2023-09-23T21:18:57.032861+0000","last_change":"2023-09-23T21:18:44.928597+0000","last_active":"2023-09-23T21:18:57.032861+0000","last_peered":"2023-09-23T21:18:57.032861+0000","last_clean":"2023-09-23T21:18:57.032861+0000","last_became_active":"2023-09-23T21:18:44.896863+0000","last_became_peered":"2023-09-23T21:18:44.896863+0000","last_unstale":"2023-09-23T21:18:57.032861+0000","last_undegraded":"2023-09-23T21:18:57.032861+0000","last_fullsized":"2023-09-23T21:18:57.032861+0000","mapping_epoch":622,"log_start":"550'2789","ondisk_log_start":"550'2789","created":559,"last_epoch_clean":623,"parent":"0.0","parent_split_bits":5,"last_scrub":"621'7919","last_scrub_stamp":"2023-09-23T21:18:43.079722+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_clean_scrub_stamp":"2023-09-23T21:18:43.079722+0000","objects_scrubbed":386,"log_size":5351,"log_dups_size":0,"ondisk_log_size":5351,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:44:42.637399+0000","scrub_duration":0.092516535999999996,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":10747904,"num_objects":164,"num_object_clones":0,"num_object_copies":492,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":164,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":222,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1e","version":"79'161","reported_seq":614,"reported_epoch":630,"state":"active+clean","last_fresh":"2023-09-23T21:18:52.899825+0000","last_change":"2023-09-23T21:16:45.420788+0000","last_active":"2023-09-23T21:18:52.899825+0000","last_peered":"2023-09-23T21:18:52.899825+0000","last_clean":"2023-09-23T21:18:52.899825+0000","last_became_active":"2023-09-23T21:16:45.420448+0000","last_became_peered":"2023-09-23T21:16:45.420448+0000","last_unstale":"2023-09-23T21:18:52.899825+0000","last_undegraded":"2023-09-23T21:18:52.899825+0000","last_fullsized":"2023-09-23T21:18:52.899825+0000","mapping_epoch":567,"log_start":"0'0","ondisk_log_start":"0'0","created":479,"last_epoch_clean":568,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:15:53.061319+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:15:53.061319+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:46:31.982439+0000","scrub_duration":0.007429878,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,4],"acting":[3,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":243,"reported_epoch":633,"state":"active+clean","last_fresh":"2023-09-23T21:18:55.908515+0000","last_change":"2023-09-23T21:16:42.518584+0000","last_active":"2023-09-23T21:18:55.908515+0000","last_peered":"2023-09-23T21:18:55.908515+0000","last_clean":"2023-09-23T21:18:55.908515+0000","last_became_active":"2023-09-23T21:16:42.416632+0000","last_became_peered":"2023-09-23T21:16:42.416632+0000","last_unstale":"2023-09-23T21:18:55.908515+0000","last_undegraded":"2023-09-23T21:18:55.908515+0000","last_fullsized":"2023-09-23T21:18:55.908515+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:02:18.972080+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,7],"acting":[4,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":198,"reported_epoch":633,"state":"active+clean","last_fresh":"2023-09-23T21:18:55.909044+0000","last_change":"2023-09-23T21:12:10.770474+0000","last_active":"2023-09-23T21:18:55.909044+0000","last_peered":"2023-09-23T21:18:55.909044+0000","last_clean":"2023-09-23T21:18:55.909044+0000","last_became_active":"2023-09-23T21:12:10.770184+0000","last_became_peered":"2023-09-23T21:12:10.770184+0000","last_unstale":"2023-09-23T21:18:55.909044+0000","last_undegraded":"2023-09-23T21:18:55.909044+0000","last_fullsized":"2023-09-23T21:18:55.909044+0000","mapping_epoch":470,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:05:09.407151+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.15","version":"633'8328","reported_seq":14766,"reported_epoch":633,"state":"active+clean","last_fresh":"2023-09-23T21:18:56.313358+0000","last_change":"2023-09-23T21:18:47.931483+0000","last_active":"2023-09-23T21:18:56.313358+0000","last_peered":"2023-09-23T21:18:56.313358+0000","last_clean":"2023-09-23T21:18:56.313358+0000","last_became_active":"2023-09-23T21:18:47.898051+0000","last_became_peered":"2023-09-23T21:18:47.898051+0000","last_unstale":"2023-09-23T21:18:56.313358+0000","last_undegraded":"2023-09-23T21:18:56.313358+0000","last_fullsized":"2023-09-23T21:18:56.313358+0000","mapping_epoch":624,"log_start":"556'4217","ondisk_log_start":"556'4217","created":559,"last_epoch_clean":625,"parent":"0.0","parent_split_bits":5,"last_scrub":"610'7908","last_scrub_stamp":"2023-09-23T21:18:31.392582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_clean_scrub_stamp":"2023-09-23T21:18:31.392582+0000","objects_scrubbed":603,"log_size":4111,"log_dups_size":0,"ondisk_log_size":4111,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:31:46.220106+0000","scrub_duration":0.067034226000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":11993088,"num_objects":183,"num_object_clones":0,"num_object_copies":549,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":183,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":420,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":369,"num_bytes_recovered":24182784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,5],"acting":[2,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":424,"reported_epoch":632,"state":"active+clean","last_fresh":"2023-09-23T21:18:54.902221+0000","last_change":"2023-09-23T21:12:10.771160+0000","last_active":"2023-09-23T21:18:54.902221+0000","last_peered":"2023-09-23T21:18:54.902221+0000","last_clean":"2023-09-23T21:18:54.902221+0000","last_became_active":"2023-09-23T21:12:10.757808+0000","last_became_peered":"2023-09-23T21:12:10.757808+0000","last_unstale":"2023-09-23T21:18:54.902221+0000","last_undegraded":"2023-09-23T21:18:54.902221+0000","last_fullsized":"2023-09-23T21:18:54.902221+0000","mapping_epoch":470,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:30:50.987139+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":208,"reported_epoch":633,"state":"active+clean","last_fresh":"2023-09-23T21:18:55.907894+0000","last_change":"2023-09-23T21:13:12.216602+0000","last_active":"2023-09-23T21:18:55.907894+0000","last_peered":"2023-09-23T21:18:55.907894+0000","last_clean":"2023-09-23T21:18:55.907894+0000","last_became_active":"2023-09-23T21:13:12.216204+0000","last_became_peered":"2023-09-23T21:13:12.216204+0000","last_unstale":"2023-09-23T21:18:55.907894+0000","last_undegraded":"2023-09-23T21:18:55.907894+0000","last_fullsized":"2023-09-23T21:18:55.907894+0000","mapping_epoch":493,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:23:37.043390+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.16","version":"634'7984","reported_seq":9061,"reported_epoch":634,"state":"active+clean","last_fresh":"2023-09-23T21:18:57.253802+0000","last_change":"2023-09-23T21:18:52.925245+0000","last_active":"2023-09-23T21:18:57.253802+0000","last_peered":"2023-09-23T21:18:57.253802+0000","last_clean":"2023-09-23T21:18:57.253802+0000","last_became_active":"2023-09-23T21:18:52.900539+0000","last_became_peered":"2023-09-23T21:18:52.900539+0000","last_unstale":"2023-09-23T21:18:57.253802+0000","last_undegraded":"2023-09-23T21:18:57.253802+0000","last_fullsized":"2023-09-23T21:18:57.253802+0000","mapping_epoch":629,"log_start":"550'3239","ondisk_log_start":"550'3239","created":559,"last_epoch_clean":630,"parent":"0.0","parent_split_bits":5,"last_scrub":"610'7609","last_scrub_stamp":"2023-09-23T21:18:31.602481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_clean_scrub_stamp":"2023-09-23T21:18:31.602481+0000","objects_scrubbed":532,"log_size":4745,"log_dups_size":0,"ondisk_log_size":4745,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:45:18.428511+0000","scrub_duration":0.058905792999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":10289152,"num_objects":157,"num_object_clones":0,"num_object_copies":471,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":157,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":375,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":231,"num_bytes_recovered":15138816,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.1c","version":"79'161","reported_seq":1061,"reported_epoch":630,"state":"active+clean","last_fresh":"2023-09-23T21:18:52.898947+0000","last_change":"2023-09-23T21:16:45.419779+0000","last_active":"2023-09-23T21:18:52.898947+0000","last_peered":"2023-09-23T21:18:52.898947+0000","last_clean":"2023-09-23T21:18:52.898947+0000","last_became_active":"2023-09-23T21:16:45.419394+0000","last_became_peered":"2023-09-23T21:16:45.419394+0000","last_unstale":"2023-09-23T21:18:52.898947+0000","last_undegraded":"2023-09-23T21:18:52.898947+0000","last_fullsized":"2023-09-23T21:18:52.898947+0000","mapping_epoch":567,"log_start":"0'0","ondisk_log_start":"0'0","created":479,"last_epoch_clean":568,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:15:54.019828+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:15:54.019828+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:59:05.422258+0000","scrub_duration":0.0020962530000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":242,"reported_epoch":632,"state":"active+clean","last_fresh":"2023-09-23T21:18:54.902279+0000","last_change":"2023-09-23T21:16:42.424187+0000","last_active":"2023-09-23T21:18:54.902279+0000","last_peered":"2023-09-23T21:18:54.902279+0000","last_clean":"2023-09-23T21:18:54.902279+0000","last_became_active":"2023-09-23T21:16:42.423805+0000","last_became_peered":"2023-09-23T21:16:42.423805+0000","last_unstale":"2023-09-23T21:18:54.902279+0000","last_undegraded":"2023-09-23T21:18:54.902279+0000","last_fullsized":"2023-09-23T21:18:54.902279+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:55:46.051199+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,7],"acting":[2,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":302,"reported_epoch":632,"state":"active+clean","last_fresh":"2023-09-23T21:18:54.902084+0000","last_change":"2023-09-23T21:13:12.200767+0000","last_active":"2023-09-23T21:18:54.902084+0000","last_peered":"2023-09-23T21:18:54.902084+0000","last_clean":"2023-09-23T21:18:54.902084+0000","last_became_active":"2023-09-23T21:13:12.200435+0000","last_became_peered":"2023-09-23T21:13:12.200435+0000","last_unstale":"2023-09-23T21:18:54.902084+0000","last_undegraded":"2023-09-23T21:18:54.902084+0000","last_fullsized":"2023-09-23T21:18:54.902084+0000","mapping_epoch":493,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:15:44.937886+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"11.17","version":"634'8365","reported_seq":9312,"reported_epoch":634,"state":"active+clean","last_fresh":"2023-09-23T21:18:57.241589+0000","last_change":"2023-09-23T21:18:54.940090+0000","last_active":"2023-09-23T21:18:57.241589+0000","last_peered":"2023-09-23T21:18:57.241589+0000","last_clean":"2023-09-23T21:18:57.241589+0000","last_became_active":"2023-09-23T21:18:54.903043+0000","last_became_peered":"2023-09-23T21:18:54.903043+0000","last_unstale":"2023-09-23T21:18:57.241589+0000","last_undegraded":"2023-09-23T21:18:57.241589+0000","last_fullsized":"2023-09-23T21:18:57.241589+0000","mapping_epoch":631,"log_start":"556'3798","ondisk_log_start":"556'3798","created":559,"last_epoch_clean":632,"parent":"0.0","parent_split_bits":5,"last_scrub":"612'7976","last_scrub_stamp":"2023-09-23T21:18:33.499229+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_clean_scrub_stamp":"2023-09-23T21:18:33.499229+0000","objects_scrubbed":569,"log_size":4567,"log_dups_size":0,"ondisk_log_size":4567,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:02:07.039752+0000","scrub_duration":0.14642005499999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":11599872,"num_objects":177,"num_object_clones":0,"num_object_copies":531,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":177,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":392,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":258,"num_bytes_recovered":16908288,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,3],"acting":[7,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.1d","version":"79'161","reported_seq":764,"reported_epoch":630,"state":"active+clean","last_fresh":"2023-09-23T21:18:52.898369+0000","last_change":"2023-09-23T21:18:32.873842+0000","last_active":"2023-09-23T21:18:52.898369+0000","last_peered":"2023-09-23T21:18:52.898369+0000","last_clean":"2023-09-23T21:18:52.898369+0000","last_became_active":"2023-09-23T21:15:23.801704+0000","last_became_peered":"2023-09-23T21:15:23.801704+0000","last_unstale":"2023-09-23T21:18:52.898369+0000","last_undegraded":"2023-09-23T21:18:52.898369+0000","last_fullsized":"2023-09-23T21:18:52.898369+0000","mapping_epoch":542,"log_start":"0'0","ondisk_log_start":"0'0","created":479,"last_epoch_clean":543,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:09:36.858077+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:09:36.858077+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.010304610000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,2],"acting":[3,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":160,"reported_epoch":631,"state":"active+clean","last_fresh":"2023-09-23T21:18:53.895774+0000","last_change":"2023-09-23T21:13:38.380574+0000","last_active":"2023-09-23T21:18:53.895774+0000","last_peered":"2023-09-23T21:18:53.895774+0000","last_clean":"2023-09-23T21:18:53.895774+0000","last_became_active":"2023-09-23T21:13:38.373252+0000","last_became_peered":"2023-09-23T21:13:38.373252+0000","last_unstale":"2023-09-23T21:18:53.895774+0000","last_undegraded":"2023-09-23T21:18:53.895774+0000","last_fullsized":"2023-09-23T21:18:53.895774+0000","mapping_epoch":507,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:58:14.788031+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":11,"num_bytes_recovered":3324,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":208,"reported_epoch":633,"state":"active+clean","last_fresh":"2023-09-23T21:18:55.908031+0000","last_change":"2023-09-23T21:12:10.793610+0000","last_active":"2023-09-23T21:18:55.908031+0000","last_peered":"2023-09-23T21:18:55.908031+0000","last_clean":"2023-09-23T21:18:55.908031+0000","last_became_active":"2023-09-23T21:12:10.793081+0000","last_became_peered":"2023-09-23T21:12:10.793081+0000","last_unstale":"2023-09-23T21:18:55.908031+0000","last_undegraded":"2023-09-23T21:18:55.908031+0000","last_fullsized":"2023-09-23T21:18:55.908031+0000","mapping_epoch":470,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:34:36.291550+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":11,"num_pg":26,"stat_sum":{"num_bytes":403013656,"num_objects":6150,"num_object_clones":0,"num_object_copies":18450,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6150,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":94273,"num_write_kb":2940641,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8937,"num_bytes_recovered":575340544,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1354051584,"data_stored":1354039368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":129778,"ondisk_log_size":129778,"up":78,"acting":78,"num_store_stats":6},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":80,"num_bytes_recovered":18686,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":118784,"data_stored":21350,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":14322,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":31,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":525376,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2109440,"data_stored":2098784,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4991,"ondisk_log_size":4991,"up":93,"acting":93,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201742,"num_pgs":72,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":837292,"kb_used_data":197860,"kb_used_omap":0,"kb_used_meta":639424,"kb_avail":92907860,"statfs":{"total":95995035648,"available":95137648640,"internally_reserved":0,"allocated":202608640,"data_stored":200876229,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":654770176},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:18:22 2023","interfaces":[{"interface":"back","average":{"1min":1.5209999999999999,"5min":1.474,"15min":2.1080000000000001},"min":{"1min":0.22900000000000001,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":3.968,"5min":37.207999999999998,"15min":37.207999999999998},"last":0.55700000000000005},{"interface":"front","average":{"1min":1.73,"5min":1.748,"15min":2.0720000000000001},"min":{"1min":0.318,"5min":0.253,"15min":0.253},"max":{"1min":5.2910000000000004,"5min":37.122,"15min":37.122},"last":0.47599999999999998}]},{"osd":1,"last update":"Sat Sep 23 21:18:16 2023","interfaces":[{"interface":"back","average":{"1min":1.2609999999999999,"5min":1.2210000000000001,"15min":1.7969999999999999},"min":{"1min":0.27600000000000002,"5min":0.27600000000000002,"15min":0.26700000000000002},"max":{"1min":6.9130000000000003,"5min":7.5369999999999999,"15min":14.034000000000001},"last":0.50800000000000001},{"interface":"front","average":{"1min":1.4370000000000001,"5min":1.379,"15min":1.776},"min":{"1min":0.34300000000000003,"5min":0.29999999999999999,"15min":0.26300000000000001},"max":{"1min":3.9300000000000002,"5min":11.002000000000001,"15min":11.226000000000001},"last":0.59099999999999997}]},{"osd":2,"last update":"Sat Sep 23 21:18:38 2023","interfaces":[{"interface":"back","average":{"1min":2.105,"5min":2.0739999999999998,"15min":1.8069999999999999},"min":{"1min":0.27200000000000002,"5min":0.27200000000000002,"15min":0.26200000000000001},"max":{"1min":18.488,"5min":25.471,"15min":25.471},"last":0.56899999999999995},{"interface":"front","average":{"1min":1.9330000000000001,"5min":1.857,"15min":1.752},"min":{"1min":0.28799999999999998,"5min":0.28799999999999998,"15min":0.26700000000000002},"max":{"1min":17.536000000000001,"5min":27.187999999999999,"15min":27.187999999999999},"last":0.48999999999999999}]},{"osd":3,"last update":"Sat Sep 23 21:18:04 2023","interfaces":[{"interface":"back","average":{"1min":1.7070000000000001,"5min":1.484,"15min":1.8280000000000001},"min":{"1min":0.33200000000000002,"5min":0.30599999999999999,"15min":0.29099999999999998},"max":{"1min":6.8739999999999997,"5min":8.6259999999999994,"15min":12.359999999999999},"last":0.54600000000000004},{"interface":"front","average":{"1min":1.1180000000000001,"5min":1.732,"15min":1.831},"min":{"1min":0.28799999999999998,"5min":0.28799999999999998,"15min":0.28699999999999998},"max":{"1min":3.8300000000000001,"5min":15.82,"15min":15.82},"last":0.57599999999999996}]},{"osd":4,"last update":"Sat Sep 23 21:18:04 2023","interfaces":[{"interface":"back","average":{"1min":0.93400000000000005,"5min":1.02,"15min":1.1799999999999999},"min":{"1min":0.251,"5min":0.251,"15min":0.248},"max":{"1min":3.9990000000000001,"5min":7.6609999999999996,"15min":11.297000000000001},"last":0.40799999999999997},{"interface":"front","average":{"1min":1.0089999999999999,"5min":1.073,"15min":1.137},"min":{"1min":0.28599999999999998,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":5.6349999999999998,"5min":8.6059999999999999,"15min":12.752000000000001},"last":0.34499999999999997}]},{"osd":5,"last update":"Sat Sep 23 21:18:04 2023","interfaces":[{"interface":"back","average":{"1min":1.286,"5min":1.095,"15min":1.089},"min":{"1min":0.28999999999999998,"5min":0.28999999999999998,"15min":0.26000000000000001},"max":{"1min":6.9740000000000002,"5min":7.5910000000000002,"15min":9.4789999999999992},"last":0.46000000000000002},{"interface":"front","average":{"1min":1.381,"5min":1.171,"15min":1.2130000000000001},"min":{"1min":0.35599999999999998,"5min":0.32000000000000001,"15min":0.26200000000000001},"max":{"1min":6.7439999999999998,"5min":7.6929999999999996,"15min":11.257},"last":0.45100000000000001}]},{"osd":6,"last update":"Sat Sep 23 21:18:16 2023","interfaces":[{"interface":"back","average":{"1min":0.86099999999999999,"5min":0.88300000000000001,"15min":1.0620000000000001},"min":{"1min":0.34599999999999997,"5min":0.32000000000000001,"15min":0.30099999999999999},"max":{"1min":5.7069999999999999,"5min":7.5339999999999998,"15min":10.941000000000001},"last":0.42899999999999999},{"interface":"front","average":{"1min":1.379,"5min":1.0429999999999999,"15min":1.181},"min":{"1min":0.41099999999999998,"5min":0.36299999999999999,"15min":0.33100000000000002},"max":{"1min":5.6760000000000002,"5min":5.6760000000000002,"15min":12.715},"last":0.52000000000000002}]}]},{"osd":6,"up_from":0,"seq":704374636825,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":1,"up_from":0,"seq":1503238553758,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":0,"up_from":493,"seq":2117418876998,"num_pgs":73,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":698640,"kb_used_data":299400,"kb_used_omap":36,"kb_used_meta":399131,"kb_avail":93046512,"statfs":{"total":95995035648,"available":95279628288,"internally_reserved":0,"allocated":306585600,"data_stored":303708493,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":37336,"internal_metadata":408710696},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,4,4],"upper_bound":16},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 21:18:27 2023","interfaces":[{"interface":"back","average":{"1min":1.3620000000000001,"5min":1.4410000000000001,"15min":0.98399999999999999},"min":{"1min":0.25900000000000001,"5min":0.214,"15min":0.214},"max":{"1min":6.7779999999999996,"5min":13.779999999999999,"15min":13.779999999999999},"last":0.24399999999999999},{"interface":"front","average":{"1min":2.5499999999999998,"5min":1.7569999999999999,"15min":1.099},"min":{"1min":0.35099999999999998,"5min":0.20799999999999999,"15min":0.20799999999999999},"max":{"1min":10.551,"5min":11.964,"15min":11.964},"last":1.0289999999999999}]},{"osd":2,"last update":"Sat Sep 23 21:18:27 2023","interfaces":[{"interface":"back","average":{"1min":2.0579999999999998,"5min":1.8,"15min":1.036},"min":{"1min":0.36599999999999999,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":7.3449999999999998,"5min":13.637,"15min":13.637},"last":1.3340000000000001},{"interface":"front","average":{"1min":3.258,"5min":2.46,"15min":1.4650000000000001},"min":{"1min":0.30199999999999999,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":10.401999999999999,"5min":26.571000000000002,"15min":26.571000000000002},"last":0.89200000000000002}]},{"osd":3,"last update":"Sat Sep 23 21:18:27 2023","interfaces":[{"interface":"back","average":{"1min":3.9950000000000001,"5min":2.6000000000000001,"15min":1.4450000000000001},"min":{"1min":0.253,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":37.079999999999998,"5min":37.079999999999998,"15min":37.079999999999998},"last":3.3650000000000002},{"interface":"front","average":{"1min":3.4550000000000001,"5min":2.2029999999999998,"15min":1.4790000000000001},"min":{"1min":0.34699999999999998,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":32.935000000000002,"5min":32.935000000000002,"15min":32.935000000000002},"last":0.46000000000000002}]},{"osd":4,"last update":"Sat Sep 23 21:18:27 2023","interfaces":[{"interface":"back","average":{"1min":1.546,"5min":1.7390000000000001,"15min":1.3080000000000001},"min":{"1min":0.34499999999999997,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":4.7880000000000003,"5min":22.780000000000001,"15min":22.780000000000001},"last":0.499},{"interface":"front","average":{"1min":1.9390000000000001,"5min":1.4990000000000001,"15min":1.163},"min":{"1min":0.32500000000000001,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":6.9119999999999999,"5min":22.652000000000001,"15min":22.652000000000001},"last":0.878}]},{"osd":5,"last update":"Sat Sep 23 21:18:27 2023","interfaces":[{"interface":"back","average":{"1min":1.5549999999999999,"5min":1.5680000000000001,"15min":1.3460000000000001},"min":{"1min":0.312,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":6.1139999999999999,"5min":8.9329999999999998,"15min":8.9329999999999998},"last":0.97199999999999998},{"interface":"front","average":{"1min":1.9039999999999999,"5min":1.696,"15min":1.274},"min":{"1min":0.40100000000000002,"5min":0.313,"15min":0.313},"max":{"1min":6.5990000000000002,"5min":15.321999999999999,"15min":15.321999999999999},"last":0.52300000000000002}]},{"osd":6,"last update":"Sat Sep 23 21:18:27 2023","interfaces":[{"interface":"back","average":{"1min":1.5720000000000001,"5min":1.4810000000000001,"15min":1.2450000000000001},"min":{"1min":0.372,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":6.9409999999999998,"5min":7.2050000000000001,"15min":7.2050000000000001},"last":0.53500000000000003},{"interface":"front","average":{"1min":1.647,"5min":1.595,"15min":1.403},"min":{"1min":0.38400000000000001,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":6.266,"5min":8.8889999999999993,"15min":8.8889999999999993},"last":1.002}]},{"osd":7,"last update":"Sat Sep 23 21:18:27 2023","interfaces":[{"interface":"back","average":{"1min":1.595,"5min":1.8240000000000001,"15min":1.4139999999999999},"min":{"1min":0.34699999999999998,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":4.8360000000000003,"5min":15.273999999999999,"15min":15.273999999999999},"last":0.94599999999999995},{"interface":"front","average":{"1min":1.75,"5min":1.7709999999999999,"15min":1.3620000000000001},"min":{"1min":0.36199999999999999,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":6.9109999999999996,"5min":33.779000000000003,"15min":33.779000000000003},"last":0.59199999999999997}]}]},{"osd":2,"up_from":93,"seq":399431958870,"num_pgs":54,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1276308,"kb_used_data":262348,"kb_used_omap":81,"kb_used_meta":1013614,"kb_avail":92468844,"statfs":{"total":95995035648,"available":94688096256,"internally_reserved":0,"allocated":268644352,"data_stored":265716641,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":83750,"internal_metadata":1037940954},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,0,4,6],"upper_bound":128},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:18:23 2023","interfaces":[{"interface":"back","average":{"1min":2.6339999999999999,"5min":1.9370000000000001,"15min":1.2010000000000001},"min":{"1min":0.221,"5min":0.187,"15min":0.187},"max":{"1min":9.9499999999999993,"5min":13.25,"15min":13.25},"last":7.7249999999999996},{"interface":"front","average":{"1min":2.2509999999999999,"5min":2.089,"15min":1.806},"min":{"1min":0.26800000000000002,"5min":0.20200000000000001,"15min":0.20200000000000001},"max":{"1min":9.8849999999999998,"5min":18.111999999999998,"15min":18.111999999999998},"last":3.0030000000000001}]},{"osd":1,"last update":"Sat Sep 23 21:18:23 2023","interfaces":[{"interface":"back","average":{"1min":1.645,"5min":1.756,"15min":2.2629999999999999},"min":{"1min":0.34000000000000002,"5min":0.20599999999999999,"15min":0.188},"max":{"1min":10.535,"5min":10.535,"15min":34.396999999999998},"last":7.4939999999999998},{"interface":"front","average":{"1min":2.2240000000000002,"5min":1.95,"15min":2.3460000000000001},"min":{"1min":0.24099999999999999,"5min":0.24099999999999999,"15min":0.217},"max":{"1min":6.0990000000000002,"5min":18.081,"15min":34.462000000000003},"last":2.9550000000000001}]},{"osd":3,"last update":"Sat Sep 23 21:18:01 2023","interfaces":[{"interface":"back","average":{"1min":2.0880000000000001,"5min":2.04,"15min":2.2389999999999999},"min":{"1min":0.248,"5min":0.20599999999999999,"15min":0.20599999999999999},"max":{"1min":7.3719999999999999,"5min":20.106999999999999,"15min":35.549999999999997},"last":3.552},{"interface":"front","average":{"1min":2.6659999999999999,"5min":2.556,"15min":2.6099999999999999},"min":{"1min":0.35799999999999998,"5min":0.28499999999999998,"15min":0.21299999999999999},"max":{"1min":10.162000000000001,"5min":20.239000000000001,"15min":36.951999999999998},"last":8.2469999999999999}]},{"osd":4,"last update":"Sat Sep 23 21:18:23 2023","interfaces":[{"interface":"back","average":{"1min":1.2749999999999999,"5min":1.3460000000000001,"15min":1.4450000000000001},"min":{"1min":0.27000000000000002,"5min":0.27000000000000002,"15min":0.249},"max":{"1min":3.8530000000000002,"5min":7.9770000000000003,"15min":10.266999999999999},"last":3.1880000000000002},{"interface":"front","average":{"1min":1.363,"5min":1.5640000000000001,"15min":1.7110000000000001},"min":{"1min":0.38900000000000001,"5min":0.30299999999999999,"15min":0.26700000000000002},"max":{"1min":4.2210000000000001,"5min":10.459,"15min":12.49},"last":7.601}]},{"osd":5,"last update":"Sat Sep 23 21:18:23 2023","interfaces":[{"interface":"back","average":{"1min":1.6160000000000001,"5min":1.456,"15min":1.5349999999999999},"min":{"1min":0.32300000000000001,"5min":0.29099999999999998,"15min":0.28000000000000003},"max":{"1min":5.4139999999999997,"5min":7.6180000000000003,"15min":9.5039999999999996},"last":3.2690000000000001},{"interface":"front","average":{"1min":1.474,"5min":1.5169999999999999,"15min":1.6890000000000001},"min":{"1min":0.34899999999999998,"5min":0.33800000000000002,"15min":0.29399999999999998},"max":{"1min":4.2649999999999997,"5min":10.425000000000001,"15min":12.57},"last":8.3409999999999993}]},{"osd":6,"last update":"Sat Sep 23 21:18:23 2023","interfaces":[{"interface":"back","average":{"1min":1.542,"5min":1.633,"15min":1.8979999999999999},"min":{"1min":0.54100000000000004,"5min":0.377,"15min":0.35699999999999998},"max":{"1min":4.2990000000000004,"5min":10.335000000000001,"15min":12.518000000000001},"last":8.3729999999999993},{"interface":"front","average":{"1min":1.7190000000000001,"5min":1.696,"15min":2.0510000000000002},"min":{"1min":0.501,"5min":0.377,"15min":0.35599999999999998},"max":{"1min":4.7969999999999997,"5min":10.381,"15min":12.544},"last":8.4149999999999991}]},{"osd":7,"last update":"Sat Sep 23 21:18:23 2023","interfaces":[{"interface":"back","average":{"1min":1.6339999999999999,"5min":1.637,"15min":1.6519999999999999},"min":{"1min":0.31,"5min":0.31,"15min":0.29799999999999999},"max":{"1min":5.7480000000000002,"5min":18.021999999999998,"15min":18.021999999999998},"last":3.1190000000000002},{"interface":"front","average":{"1min":2.145,"5min":1.6339999999999999,"15min":1.73},"min":{"1min":0.35999999999999999,"5min":0.34999999999999998,"15min":0.32200000000000001},"max":{"1min":6.694,"5min":9.6349999999999998,"15min":10.92},"last":3.222}]}]},{"osd":3,"up_from":98,"seq":420906795343,"num_pgs":78,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1410036,"kb_used_data":331180,"kb_used_omap":7,"kb_used_meta":1078840,"kb_avail":92335116,"statfs":{"total":95995035648,"available":94551158784,"internally_reserved":0,"allocated":339128320,"data_stored":336271525,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7576,"internal_metadata":1104732776},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:18:18 2023","interfaces":[{"interface":"back","average":{"1min":2.2069999999999999,"5min":4.0750000000000002,"15min":1.583},"min":{"1min":0.22500000000000001,"5min":0.18099999999999999,"15min":0.18099999999999999},"max":{"1min":10.439,"5min":147.38900000000001,"15min":147.38900000000001},"last":0.314},{"interface":"front","average":{"1min":1.1899999999999999,"5min":3.5859999999999999,"15min":1.6399999999999999},"min":{"1min":0.252,"5min":0.223,"15min":0.223},"max":{"1min":5.7279999999999998,"5min":149.989,"15min":149.989},"last":0.66600000000000004}]},{"osd":1,"last update":"Sat Sep 23 21:18:18 2023","interfaces":[{"interface":"back","average":{"1min":2.2029999999999998,"5min":1.5940000000000001,"15min":2.1160000000000001},"min":{"1min":0.27000000000000002,"5min":0.218,"15min":0.20000000000000001},"max":{"1min":6.1150000000000002,"5min":7.2759999999999998,"15min":22.577999999999999},"last":0.372},{"interface":"front","average":{"1min":1.827,"5min":1.2969999999999999,"15min":1.984},"min":{"1min":0.26100000000000001,"5min":0.249,"15min":0.216},"max":{"1min":6.1219999999999999,"5min":7.6829999999999998,"15min":21.617999999999999},"last":0.35099999999999998}]},{"osd":2,"last update":"Sat Sep 23 21:18:02 2023","interfaces":[{"interface":"back","average":{"1min":1.4750000000000001,"5min":1.958,"15min":2.004},"min":{"1min":0.27400000000000002,"5min":0.221,"15min":0.20899999999999999},"max":{"1min":11.321,"5min":56.317999999999998,"15min":56.317999999999998},"last":0.72899999999999998},{"interface":"front","average":{"1min":1.9910000000000001,"5min":2.4780000000000002,"15min":2.355},"min":{"1min":0.23400000000000001,"5min":0.23400000000000001,"15min":0.216},"max":{"1min":11.371,"5min":55.988,"15min":55.988},"last":0.42399999999999999}]},{"osd":4,"last update":"Sat Sep 23 21:18:02 2023","interfaces":[{"interface":"back","average":{"1min":0.72899999999999998,"5min":1.2010000000000001,"15min":1.5509999999999999},"min":{"1min":0.29199999999999998,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":2.5409999999999999,"5min":10.807,"15min":21.478999999999999},"last":0.76400000000000001},{"interface":"front","average":{"1min":1.091,"5min":1.3180000000000001,"15min":1.7829999999999999},"min":{"1min":0.35899999999999999,"5min":0.311,"15min":0.28499999999999998},"max":{"1min":3.129,"5min":7.3140000000000001,"15min":22.402000000000001},"last":0.45300000000000001}]},{"osd":5,"last update":"Sat Sep 23 21:18:02 2023","interfaces":[{"interface":"back","average":{"1min":0.83199999999999996,"5min":1.3160000000000001,"15min":1.6930000000000001},"min":{"1min":0.318,"5min":0.313,"15min":0.311},"max":{"1min":4.1379999999999999,"5min":10.882999999999999,"15min":21.562000000000001},"last":0.82199999999999995},{"interface":"front","average":{"1min":1.571,"5min":1.234,"15min":1.8},"min":{"1min":0.42699999999999999,"5min":0.34300000000000003,"15min":0.31},"max":{"1min":5.7300000000000004,"5min":5.7300000000000004,"15min":14.029},"last":0.58999999999999997}]},{"osd":6,"last update":"Sat Sep 23 21:18:18 2023","interfaces":[{"interface":"back","average":{"1min":1.6180000000000001,"5min":1.464,"15min":1.839},"min":{"1min":0.39800000000000002,"5min":0.35899999999999999,"15min":0.33900000000000002},"max":{"1min":4.9749999999999996,"5min":7.3410000000000002,"15min":22.446999999999999},"last":0.47799999999999998},{"interface":"front","average":{"1min":1.657,"5min":1.3480000000000001,"15min":1.8280000000000001},"min":{"1min":0.373,"5min":0.373,"15min":0.373},"max":{"1min":7.5949999999999998,"5min":9.3889999999999993,"15min":12.375999999999999},"last":0.55500000000000005}]},{"osd":7,"last update":"Sat Sep 23 21:18:02 2023","interfaces":[{"interface":"back","average":{"1min":1.0249999999999999,"5min":1.48,"15min":1.847},"min":{"1min":0.41299999999999998,"5min":0.41199999999999998,"15min":0.33900000000000002},"max":{"1min":4.242,"5min":10.914999999999999,"15min":21.518999999999998},"last":0.79800000000000004},{"interface":"front","average":{"1min":1.5409999999999999,"5min":1.528,"15min":1.9850000000000001},"min":{"1min":0.51800000000000002,"5min":0.42499999999999999,"15min":0.40300000000000002},"max":{"1min":4.6920000000000002,"5min":8.2590000000000003,"15min":12.624000000000001},"last":0.62}]}]},{"osd":4,"up_from":31,"seq":133143986650,"num_pgs":65,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":781420,"kb_used_data":199396,"kb_used_omap":0,"kb_used_meta":582016,"kb_avail":92963732,"statfs":{"total":95995035648,"available":95194861568,"internally_reserved":0,"allocated":204181504,"data_stored":202448826,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":595984384},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,0,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:18:24 2023","interfaces":[{"interface":"back","average":{"1min":2.2690000000000001,"5min":1.673,"15min":1.157},"min":{"1min":0.245,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":20.515000000000001,"5min":28.683,"15min":28.683},"last":4.4429999999999996},{"interface":"front","average":{"1min":3.085,"5min":1.8009999999999999,"15min":1.3560000000000001},"min":{"1min":0.249,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":20.792000000000002,"5min":29.155000000000001,"15min":29.155000000000001},"last":6.5490000000000004}]},{"osd":1,"last update":"Sat Sep 23 21:18:24 2023","interfaces":[{"interface":"back","average":{"1min":1.7330000000000001,"5min":1.2829999999999999,"15min":2.1070000000000002},"min":{"1min":0.28499999999999998,"5min":0.26900000000000002,"15min":0.221},"max":{"1min":8.1020000000000003,"5min":8.1020000000000003,"15min":37.960000000000001},"last":0.312},{"interface":"front","average":{"1min":1.55,"5min":1.399,"15min":2.0550000000000002},"min":{"1min":0.33100000000000002,"5min":0.28499999999999998,"15min":0.27000000000000002},"max":{"1min":8.1509999999999998,"5min":13.983000000000001,"15min":37.933},"last":0.498}]},{"osd":2,"last update":"Sat Sep 23 21:18:24 2023","interfaces":[{"interface":"back","average":{"1min":2.177,"5min":1.8500000000000001,"15min":1.754},"min":{"1min":0.28399999999999997,"5min":0.26400000000000001,"15min":0.247},"max":{"1min":9.8399999999999999,"5min":17.602,"15min":17.602},"last":13.791},{"interface":"front","average":{"1min":1.8839999999999999,"5min":1.7849999999999999,"15min":1.831},"min":{"1min":0.35399999999999998,"5min":0.311,"15min":0.27800000000000002},"max":{"1min":5.5250000000000004,"5min":14.339,"15min":14.417999999999999},"last":13.439}]},{"osd":3,"last update":"Sat Sep 23 21:18:55 2023","interfaces":[{"interface":"back","average":{"1min":1.3129999999999999,"5min":2.1579999999999999,"15min":1.9790000000000001},"min":{"1min":0.29499999999999998,"5min":0.27900000000000003,"15min":0.25600000000000001},"max":{"1min":6.1230000000000002,"5min":67.105999999999995,"15min":67.105999999999995},"last":0.38400000000000001},{"interface":"front","average":{"1min":1.5289999999999999,"5min":2.2719999999999998,"15min":1.966},"min":{"1min":0.40400000000000003,"5min":0.32200000000000001,"15min":0.30199999999999999},"max":{"1min":6.1319999999999997,"5min":67.046999999999997,"15min":67.046999999999997},"last":0.64400000000000002}]},{"osd":5,"last update":"Sat Sep 23 21:18:13 2023","interfaces":[{"interface":"back","average":{"1min":0.89300000000000002,"5min":1.1080000000000001,"15min":1.123},"min":{"1min":0.254,"5min":0.254,"15min":0.222},"max":{"1min":2.923,"5min":14.398999999999999,"15min":14.398999999999999},"last":0.60199999999999998},{"interface":"front","average":{"1min":1.0349999999999999,"5min":0.98799999999999999,"15min":1.1080000000000001},"min":{"1min":0.28100000000000003,"5min":0.25800000000000001,"15min":0.255},"max":{"1min":3.6000000000000001,"5min":8.016,"15min":9.8979999999999997},"last":0.46000000000000002}]},{"osd":6,"last update":"Sat Sep 23 21:18:24 2023","interfaces":[{"interface":"back","average":{"1min":1.077,"5min":0.96599999999999997,"15min":1.2649999999999999},"min":{"1min":0.308,"5min":0.27200000000000002,"15min":0.26800000000000002},"max":{"1min":3.2050000000000001,"5min":8.7420000000000009,"15min":24.452999999999999},"last":0.52300000000000002},{"interface":"front","average":{"1min":0.83799999999999997,"5min":1.1499999999999999,"15min":1.0820000000000001},"min":{"1min":0.38,"5min":0.36399999999999999,"15min":0.32500000000000001},"max":{"1min":2.5409999999999999,"5min":14.374000000000001,"15min":14.387},"last":0.79700000000000004}]},{"osd":7,"last update":"Sat Sep 23 21:18:55 2023","interfaces":[{"interface":"back","average":{"1min":0.70599999999999996,"5min":1.077,"15min":1.1659999999999999},"min":{"1min":0.33800000000000002,"5min":0.32000000000000001,"15min":0.30099999999999999},"max":{"1min":2.6419999999999999,"5min":14.422000000000001,"15min":16.890999999999998},"last":0.82799999999999996},{"interface":"front","average":{"1min":0.70999999999999996,"5min":0.89800000000000002,"15min":1.179},"min":{"1min":0.32900000000000001,"5min":0.29199999999999998,"15min":0.27400000000000002},"max":{"1min":2.6579999999999999,"5min":7.7229999999999999,"15min":11.170999999999999},"last":0.56000000000000005}]}]},{"osd":5,"up_from":39,"seq":167503725016,"num_pgs":61,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":777576,"kb_used_data":168224,"kb_used_omap":0,"kb_used_meta":609344,"kb_avail":92967576,"statfs":{"total":95995035648,"available":95198797824,"internally_reserved":0,"allocated":172261376,"data_stored":170537154,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":623968256},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:18:16 2023","interfaces":[{"interface":"back","average":{"1min":1.452,"5min":1.234,"15min":0.88800000000000001},"min":{"1min":0.22500000000000001,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":5.9829999999999997,"5min":8.673,"15min":8.673},"last":88.272999999999996},{"interface":"front","average":{"1min":2.5219999999999998,"5min":1.548,"15min":0.96199999999999997},"min":{"1min":0.215,"5min":0.215,"15min":0.215},"max":{"1min":12.259,"5min":12.259,"15min":12.259},"last":88.296999999999997}]},{"osd":1,"last update":"Sat Sep 23 21:18:16 2023","interfaces":[{"interface":"back","average":{"1min":1.4490000000000001,"5min":1.254,"15min":1.625},"min":{"1min":0.32200000000000001,"5min":0.248,"15min":0.23499999999999999},"max":{"1min":4.3220000000000001,"5min":7.5839999999999996,"15min":25.239999999999998},"last":0.46100000000000002},{"interface":"front","average":{"1min":1.3700000000000001,"5min":1.304,"15min":1.7070000000000001},"min":{"1min":0.36899999999999999,"5min":0.28299999999999997,"15min":0.252},"max":{"1min":4.9500000000000002,"5min":11.116,"15min":25.253},"last":0.51200000000000001}]},{"osd":2,"last update":"Sat Sep 23 21:18:35 2023","interfaces":[{"interface":"back","average":{"1min":1.895,"5min":1.5109999999999999,"15min":1.6419999999999999},"min":{"1min":0.27600000000000002,"5min":0.27300000000000002,"15min":0.25800000000000001},"max":{"1min":9.2479999999999993,"5min":9.6560000000000006,"15min":12.634},"last":0.35399999999999998},{"interface":"front","average":{"1min":2.0840000000000001,"5min":1.6000000000000001,"15min":1.663},"min":{"1min":0.29099999999999998,"5min":0.29099999999999998,"15min":0.25700000000000001},"max":{"1min":9.4060000000000006,"5min":9.4060000000000006,"15min":9.7170000000000005},"last":0.44}]},{"osd":3,"last update":"Sat Sep 23 21:18:54 2023","interfaces":[{"interface":"back","average":{"1min":1.6870000000000001,"5min":1.5089999999999999,"15min":1.883},"min":{"1min":0.32000000000000001,"5min":0.314,"15min":0.29299999999999998},"max":{"1min":7.3700000000000001,"5min":7.3700000000000001,"15min":25.503},"last":0.40999999999999998},{"interface":"front","average":{"1min":1.6100000000000001,"5min":1.782,"15min":2.105},"min":{"1min":0.33200000000000002,"5min":0.311,"15min":0.30599999999999999},"max":{"1min":6.5880000000000001,"5min":12.382,"15min":25.466000000000001},"last":0.42999999999999999}]},{"osd":4,"last update":"Sat Sep 23 21:18:08 2023","interfaces":[{"interface":"back","average":{"1min":0.82399999999999995,"5min":0.91400000000000003,"15min":1.155},"min":{"1min":0.29999999999999999,"5min":0.253,"15min":0.253},"max":{"1min":5.0279999999999996,"5min":11.156000000000001,"15min":12.776},"last":0.33100000000000002},{"interface":"front","average":{"1min":0.73399999999999999,"5min":1.008,"15min":1.1419999999999999},"min":{"1min":0.28599999999999998,"5min":0.246,"15min":0.246},"max":{"1min":3.415,"5min":9.7119999999999997,"15min":11.973000000000001},"last":0.39700000000000002}]},{"osd":6,"last update":"Sat Sep 23 21:18:16 2023","interfaces":[{"interface":"back","average":{"1min":1.0649999999999999,"5min":0.99299999999999999,"15min":1.087},"min":{"1min":0.32200000000000001,"5min":0.28000000000000003,"15min":0.25900000000000001},"max":{"1min":5.8520000000000003,"5min":5.8520000000000003,"15min":8.0909999999999993},"last":0.377},{"interface":"front","average":{"1min":1.095,"5min":1.0700000000000001,"15min":1.2330000000000001},"min":{"1min":0.39700000000000002,"5min":0.34300000000000003,"15min":0.317},"max":{"1min":7.1470000000000002,"5min":10.999000000000001,"15min":10.999000000000001},"last":0.45000000000000001}]},{"osd":7,"last update":"Sat Sep 23 21:18:54 2023","interfaces":[{"interface":"back","average":{"1min":0.95499999999999996,"5min":0.94299999999999995,"15min":1.821},"min":{"1min":0.33600000000000002,"5min":0.32400000000000001,"15min":0.31},"max":{"1min":4.0519999999999996,"5min":5.1769999999999996,"15min":189.405},"last":0.42099999999999999},{"interface":"front","average":{"1min":0.97799999999999998,"5min":0.96699999999999997,"15min":1.9490000000000001},"min":{"1min":0.34999999999999998,"5min":0.34999999999999998,"15min":0.32500000000000001},"max":{"1min":5.0910000000000002,"5min":5.0910000000000002,"15min":189.685},"last":0.53500000000000003}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8488,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1259,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5322,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":4838,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":271253504,"data_stored":271253504,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":237244416,"data_stored":237240344,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":325451776,"data_stored":325451776,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":178589696,"data_stored":178585624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":163385344,"data_stored":163381272,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":178126848,"data_stored":178126848,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T21:19:00.314 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:19:00 smithi070 ceph-mon[168282]: pgmap v1569: 129 pgs: 1 peering, 128 active+clean; 385 MiB data, 5.5 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 437 op/s; 5.7 MiB/s, 90 objects/s recovering 2023-09-23T21:19:00.314 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:19:00 smithi070 ceph-mon[168282]: from='client.56827 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T21:19:00.573 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:19:00 smithi026 ceph-mon[179688]: pgmap v1569: 129 pgs: 1 peering, 128 active+clean; 385 MiB data, 5.5 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 437 op/s; 5.7 MiB/s, 90 objects/s recovering 2023-09-23T21:19:00.573 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:19:00 smithi026 ceph-mon[179688]: from='client.56827 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T21:19:00.575 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_historic_ops 2023-09-23T21:19:00.578 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:19:00 smithi026 ceph-mon[175151]: pgmap v1569: 129 pgs: 1 peering, 128 active+clean; 385 MiB data, 5.5 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 437 op/s; 5.7 MiB/s, 90 objects/s recovering 2023-09-23T21:19:00.578 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:19:00 smithi026 ceph-mon[175151]: from='client.56827 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T21:19:00.734 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T21:19:00.742 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:00.742 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:19:00.843 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:19:00.932 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:01.282 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T21:19:01.290 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:01.291 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:19:01.392 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:19:01.496 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:01.518 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T21:19:01.572 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:19:01.576 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:01.576 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:19:01.580 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:19:01.677 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:19:01.821 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:01.969 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:19:01.980 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:01.981 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:19:02.082 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:19:02.104 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:19:01 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:19:02.105 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:19:01 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:19:02.105 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:19:01 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:19:02.105 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:19:01 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:19:02.105 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:19:01 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T21:19:02.105 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:19:01 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T21:19:02.105 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:19:01 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:19:02.105 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:19:01 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:19:02.106 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:19:01 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:19:02.106 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:19:01 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:19:02.106 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:19:01 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:19:02.106 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:19:01 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T21:19:02.106 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:19:01 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T21:19:02.106 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:19:01 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:19:02.108 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:19:01 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:19:02.108 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:19:01 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:19:02.109 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:19:01 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:19:02.109 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:19:01 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:19:02.109 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:19:01 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T21:19:02.109 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:19:01 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T21:19:02.109 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:19:01 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:19:02.113 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:02.122 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0: {} 2023-09-23T21:19:02.152 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:19:02.155 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:02.156 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:19:02.166 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1: {} 2023-09-23T21:19:02.205 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2: {} 2023-09-23T21:19:02.244 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3: {} 2023-09-23T21:19:02.257 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:19:02.304 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4: {} 2023-09-23T21:19:02.341 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:02.354 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T21:19:02.390 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6: {} 2023-09-23T21:19:02.426 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T21:19:02.435 INFO:teuthology.orchestra.run.smithi026.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T21:19:02.435 INFO:teuthology.orchestra.run.smithi026.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T21:19:02.435 INFO:teuthology.orchestra.run.smithi026.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T21:19:02.435 INFO:teuthology.orchestra.run.smithi026.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T21:19:02.435 INFO:teuthology.orchestra.run.smithi026.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T21:19:02.435 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T21:19:02.435 INFO:teuthology.orchestra.run.smithi026.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T21:19:02.435 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T21:19:02.998 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:19:03.006 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:03.007 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:19:03.048 INFO:tasks.radosbench.radosbench.0.smithi026.stdout:Removed 97967 objects 2023-09-23T21:19:03.048 INFO:tasks.radosbench.radosbench.0.smithi026.stdout:Clean up completed and total clean up time :120.54 2023-09-23T21:19:03.104 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:19:02 smithi026 ceph-mon[175151]: pgmap v1570: 129 pgs: 1 peering, 128 active+clean; 312 MiB data, 5.2 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 514 op/s; 4.8 MiB/s, 76 objects/s recovering 2023-09-23T21:19:03.105 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:19:02 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/1297760476' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:19:03.105 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:19:02 smithi026 ceph-mon[179688]: pgmap v1570: 129 pgs: 1 peering, 128 active+clean; 312 MiB data, 5.2 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 514 op/s; 4.8 MiB/s, 76 objects/s recovering 2023-09-23T21:19:03.105 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:19:02 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/1297760476' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:19:03.108 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:19:03.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:19:02 smithi070 ceph-mon[168282]: pgmap v1570: 129 pgs: 1 peering, 128 active+clean; 312 MiB data, 5.2 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 514 op/s; 4.8 MiB/s, 76 objects/s recovering 2023-09-23T21:19:03.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:19:02 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/1297760476' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:19:03.138 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:03.165 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:19:03.173 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:19:03.176 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:03.177 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:19:03.267 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T21:19:03.278 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:19:03.306 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:03.339 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:19:03.342 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:03.342 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:19:03.443 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:19:03.473 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:03.516 INFO:teuthology.orchestra.run.smithi026.stdout:ERROR: (22) Invalid argument 2023-09-23T21:19:03.516 INFO:teuthology.orchestra.run.smithi026.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T21:19:03.518 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:19:03.521 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:03.521 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:19:03.622 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:19:03.650 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:03.684 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:19:03.686 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:03.687 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:19:03.788 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:19:03.814 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:03.846 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:19:03.849 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:03.849 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:19:03.857 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:19:03 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "26"}]: dispatch 2023-09-23T21:19:03.950 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:19:03.974 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:19:03 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "26"}]: dispatch 2023-09-23T21:19:03.976 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:04.008 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:19:04.011 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:04.011 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:19:04.112 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:19:04.131 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:19:03 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "26"}]: dispatch 2023-09-23T21:19:04.141 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:04.253 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:19:04.257 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:04.258 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_ops_in_flight 2023-09-23T21:19:04.258 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:19:04.359 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:19:04.385 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:04.418 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:19:04.420 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:04.421 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:19:04.521 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:19:04.547 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:04.548 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:19:04.578 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:19:04.581 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:04.581 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:19:04.682 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:19:04.710 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:04.793 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T21:19:04.797 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:04.797 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:19:04.898 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:19:04.927 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:04.966 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:19:04.969 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:19:04 smithi026 ceph-mon[179688]: pgmap v1571: 129 pgs: 129 active+clean; 109 MiB data, 4.7 GiB used, 532 GiB / 536 GiB avail; 0 B/s wr, 812 op/s; 3.8 MiB/s, 60 objects/s recovering 2023-09-23T21:19:04.969 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:19:04 smithi026 ceph-mon[179688]: failed to encode map e635 with expected crc 2023-09-23T21:19:04.969 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:19:04 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "26"}]': finished 2023-09-23T21:19:04.969 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:19:04 smithi026 ceph-mon[179688]: osdmap e635: 8 total, 8 up, 6 in 2023-09-23T21:19:04.969 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:19:04 smithi026 ceph-mon[179688]: failed to encode map e635 with expected crc 2023-09-23T21:19:04.969 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:19:04 smithi026 ceph-mon[179688]: failed to encode map e635 with expected crc 2023-09-23T21:19:04.969 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:19:04 smithi026 ceph-mon[175151]: pgmap v1571: 129 pgs: 129 active+clean; 109 MiB data, 4.7 GiB used, 532 GiB / 536 GiB avail; 0 B/s wr, 812 op/s; 3.8 MiB/s, 60 objects/s recovering 2023-09-23T21:19:04.970 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:19:04 smithi026 ceph-mon[175151]: failed to encode map e635 with expected crc 2023-09-23T21:19:04.970 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:19:04 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "26"}]': finished 2023-09-23T21:19:04.970 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:19:04 smithi026 ceph-mon[175151]: osdmap e635: 8 total, 8 up, 6 in 2023-09-23T21:19:04.970 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:19:04 smithi026 ceph-mon[175151]: failed to encode map e635 with expected crc 2023-09-23T21:19:04.970 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:19:04 smithi026 ceph-mon[175151]: failed to encode map e635 with expected crc 2023-09-23T21:19:04.970 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:04.971 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:19:05.072 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:19:05.104 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:05.107 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T21:19:05.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:19:04 smithi070 ceph-mon[168282]: pgmap v1571: 129 pgs: 129 active+clean; 109 MiB data, 4.7 GiB used, 532 GiB / 536 GiB avail; 0 B/s wr, 812 op/s; 3.8 MiB/s, 60 objects/s recovering 2023-09-23T21:19:05.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:19:04 smithi070 ceph-mon[168282]: failed to encode map e635 with expected crc 2023-09-23T21:19:05.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:19:04 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "26"}]': finished 2023-09-23T21:19:05.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:19:04 smithi070 ceph-mon[168282]: osdmap e635: 8 total, 8 up, 6 in 2023-09-23T21:19:05.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:19:04 smithi070 ceph-mon[168282]: failed to encode map e635 with expected crc 2023-09-23T21:19:05.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:19:04 smithi070 ceph-mon[168282]: failed to encode map e635 with expected crc 2023-09-23T21:19:05.179 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:19:05.196 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:19:05.200 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:05.200 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:19:05.301 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:19:05.330 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:05.466 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T21:19:05.469 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:05.469 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:19:05.570 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:19:05.601 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:05.875 INFO:teuthology.orchestra.run.smithi026.stderr:noscrub is unset 2023-09-23T21:19:05.911 INFO:tasks.cephadm.ceph_manager.ceph:removing pool_name unique_pool_5 2023-09-23T21:19:05.912 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph osd pool rm unique_pool_5 unique_pool_5 --yes-i-really-really-mean-it 2023-09-23T21:19:06.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:19:05 smithi070 ceph-mon[168282]: failed to encode map e635 with expected crc 2023-09-23T21:19:06.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:19:05 smithi070 ceph-mon[168282]: failed to encode map e636 with expected crc 2023-09-23T21:19:06.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:19:05 smithi070 ceph-mon[168282]: failed to encode map e636 with expected crc 2023-09-23T21:19:06.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:19:05 smithi070 ceph-mon[168282]: failed to encode map e636 with expected crc 2023-09-23T21:19:06.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:19:05 smithi070 ceph-mon[168282]: osdmap e636: 8 total, 8 up, 6 in 2023-09-23T21:19:06.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:19:05 smithi070 ceph-mon[168282]: failed to encode map e636 with expected crc 2023-09-23T21:19:06.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:19:05 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/1470673942' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T21:19:06.131 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:19:05 smithi026 ceph-mon[179688]: failed to encode map e635 with expected crc 2023-09-23T21:19:06.131 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:19:05 smithi026 ceph-mon[179688]: failed to encode map e636 with expected crc 2023-09-23T21:19:06.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:19:05 smithi026 ceph-mon[179688]: failed to encode map e636 with expected crc 2023-09-23T21:19:06.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:19:05 smithi026 ceph-mon[179688]: failed to encode map e636 with expected crc 2023-09-23T21:19:06.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:19:05 smithi026 ceph-mon[179688]: osdmap e636: 8 total, 8 up, 6 in 2023-09-23T21:19:06.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:19:05 smithi026 ceph-mon[179688]: failed to encode map e636 with expected crc 2023-09-23T21:19:06.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:19:05 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/1470673942' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T21:19:06.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:19:05 smithi026 ceph-mon[175151]: failed to encode map e635 with expected crc 2023-09-23T21:19:06.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:19:05 smithi026 ceph-mon[175151]: failed to encode map e636 with expected crc 2023-09-23T21:19:06.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:19:05 smithi026 ceph-mon[175151]: failed to encode map e636 with expected crc 2023-09-23T21:19:06.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:19:05 smithi026 ceph-mon[175151]: failed to encode map e636 with expected crc 2023-09-23T21:19:06.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:19:05 smithi026 ceph-mon[175151]: osdmap e636: 8 total, 8 up, 6 in 2023-09-23T21:19:06.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:19:05 smithi026 ceph-mon[175151]: failed to encode map e636 with expected crc 2023-09-23T21:19:06.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:19:05 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/1470673942' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T21:19:06.191 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:19:06.194 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:06.194 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:19:06.295 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:19:06.328 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:06.364 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:19:06.367 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:06.368 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:19:06.468 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:19:06.498 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:06.533 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:19:06.535 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:06.536 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:19:06.637 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:19:06.683 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:07.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:19:06 smithi070 ceph-mon[168282]: pgmap v1574: 129 pgs: 129 active+clean; 76 MiB data, 4.6 GiB used, 532 GiB / 536 GiB avail; 0 B/s wr, 675 op/s; 2.0 MiB/s, 32 objects/s recovering 2023-09-23T21:19:07.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:19:06 smithi070 ceph-mon[168282]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:19:07.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:19:06 smithi070 ceph-mon[168282]: failed to encode map e637 with expected crc 2023-09-23T21:19:07.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:19:06 smithi070 ceph-mon[168282]: failed to encode map e637 with expected crc 2023-09-23T21:19:07.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:19:06 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/1470673942' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T21:19:07.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:19:06 smithi070 ceph-mon[168282]: osdmap e637: 8 total, 8 up, 6 in 2023-09-23T21:19:07.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:19:06 smithi070 ceph-mon[168282]: 11.19 starting backfill to osd.2 from (0'0,0'0] MAX to 634'8378 2023-09-23T21:19:07.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:19:06 smithi070 ceph-mon[168282]: failed to encode map e637 with expected crc 2023-09-23T21:19:07.120 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:19:06 smithi026 ceph-mon[175151]: pgmap v1574: 129 pgs: 129 active+clean; 76 MiB data, 4.6 GiB used, 532 GiB / 536 GiB avail; 0 B/s wr, 675 op/s; 2.0 MiB/s, 32 objects/s recovering 2023-09-23T21:19:07.120 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:19:06 smithi026 ceph-mon[175151]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:19:07.120 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:19:06 smithi026 ceph-mon[175151]: failed to encode map e637 with expected crc 2023-09-23T21:19:07.120 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:19:06 smithi026 ceph-mon[175151]: failed to encode map e637 with expected crc 2023-09-23T21:19:07.121 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:19:06 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/1470673942' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T21:19:07.121 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:19:06 smithi026 ceph-mon[175151]: osdmap e637: 8 total, 8 up, 6 in 2023-09-23T21:19:07.121 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:19:06 smithi026 ceph-mon[175151]: 11.19 starting backfill to osd.2 from (0'0,0'0] MAX to 634'8378 2023-09-23T21:19:07.121 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:19:06 smithi026 ceph-mon[175151]: failed to encode map e637 with expected crc 2023-09-23T21:19:07.121 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:19:06 smithi026 ceph-mon[179688]: pgmap v1574: 129 pgs: 129 active+clean; 76 MiB data, 4.6 GiB used, 532 GiB / 536 GiB avail; 0 B/s wr, 675 op/s; 2.0 MiB/s, 32 objects/s recovering 2023-09-23T21:19:07.121 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:19:06 smithi026 ceph-mon[179688]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:19:07.121 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:19:06 smithi026 ceph-mon[179688]: failed to encode map e637 with expected crc 2023-09-23T21:19:07.121 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:19:06 smithi026 ceph-mon[179688]: failed to encode map e637 with expected crc 2023-09-23T21:19:07.121 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:19:06 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/1470673942' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T21:19:07.122 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:19:06 smithi026 ceph-mon[179688]: osdmap e637: 8 total, 8 up, 6 in 2023-09-23T21:19:07.122 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:19:06 smithi026 ceph-mon[179688]: 11.19 starting backfill to osd.2 from (0'0,0'0] MAX to 634'8378 2023-09-23T21:19:07.122 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:19:06 smithi026 ceph-mon[179688]: failed to encode map e637 with expected crc 2023-09-23T21:19:07.123 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:19:07.126 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:07.126 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:19:07.227 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:19:07.258 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:07.294 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:19:07.296 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:07.297 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:19:07.398 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:19:07.425 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:07.458 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:19:07.461 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:07.461 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:19:07.562 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:19:07.588 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:07.620 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:19:07.623 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:07.623 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:19:07.724 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:19:07.755 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:07.976 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:19:07.978 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:07.979 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:19:08.023 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:19:07 smithi026 ceph-mon[175151]: failed to encode map e638 with expected crc 2023-09-23T21:19:08.023 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:19:07 smithi026 ceph-mon[175151]: failed to encode map e638 with expected crc 2023-09-23T21:19:08.023 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:19:07 smithi026 ceph-mon[175151]: osdmap e638: 8 total, 8 up, 6 in 2023-09-23T21:19:08.023 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:19:07 smithi026 ceph-mon[175151]: failed to encode map e638 with expected crc 2023-09-23T21:19:08.023 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:19:07 smithi026 ceph-mon[175151]: pgmap v1577: 129 pgs: 1 remapped+peering, 128 active+clean; 9.4 MiB data, 4.3 GiB used, 532 GiB / 536 GiB avail; 0 B/s wr, 400 op/s 2023-09-23T21:19:08.023 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:19:07 smithi026 ceph-mon[175151]: failed to encode map e638 with expected crc 2023-09-23T21:19:08.024 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:19:07 smithi026 ceph-mon[179688]: failed to encode map e638 with expected crc 2023-09-23T21:19:08.024 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:19:07 smithi026 ceph-mon[179688]: failed to encode map e638 with expected crc 2023-09-23T21:19:08.024 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:19:07 smithi026 ceph-mon[179688]: osdmap e638: 8 total, 8 up, 6 in 2023-09-23T21:19:08.024 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:19:07 smithi026 ceph-mon[179688]: failed to encode map e638 with expected crc 2023-09-23T21:19:08.024 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:19:07 smithi026 ceph-mon[179688]: pgmap v1577: 129 pgs: 1 remapped+peering, 128 active+clean; 9.4 MiB data, 4.3 GiB used, 532 GiB / 536 GiB avail; 0 B/s wr, 400 op/s 2023-09-23T21:19:08.024 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:19:07 smithi026 ceph-mon[179688]: failed to encode map e638 with expected crc 2023-09-23T21:19:08.052 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:19:08.080 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:19:08.120 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:08.177 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:19:08.180 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:08.180 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:19:08.281 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:19:08.307 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:19:07 smithi070 ceph-mon[168282]: failed to encode map e638 with expected crc 2023-09-23T21:19:08.307 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:19:07 smithi070 ceph-mon[168282]: failed to encode map e638 with expected crc 2023-09-23T21:19:08.307 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:19:07 smithi070 ceph-mon[168282]: osdmap e638: 8 total, 8 up, 6 in 2023-09-23T21:19:08.307 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:19:07 smithi070 ceph-mon[168282]: failed to encode map e638 with expected crc 2023-09-23T21:19:08.307 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:19:07 smithi070 ceph-mon[168282]: pgmap v1577: 129 pgs: 1 remapped+peering, 128 active+clean; 9.4 MiB data, 4.3 GiB used, 532 GiB / 536 GiB avail; 0 B/s wr, 400 op/s 2023-09-23T21:19:08.307 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:19:07 smithi070 ceph-mon[168282]: failed to encode map e638 with expected crc 2023-09-23T21:19:08.309 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:08.341 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:19:08.343 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:08.344 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:19:08.445 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:19:08.479 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:08.664 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:19:08.668 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:08.668 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:19:08.769 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:19:08.799 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:08.803 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:19:08.942 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:19:08.946 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:08.946 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:19:09.047 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:19:09.072 INFO:teuthology.orchestra.run.smithi026.stdout: 2023-09-23T21:19:09.074 INFO:teuthology.orchestra.run.smithi026.stderr:dumped all 2023-09-23T21:19:09.075 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:09.106 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:19:09.109 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:09.109 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:19:09.176 INFO:teuthology.orchestra.run.smithi026.stdout:ERROR: (22) Invalid argument 2023-09-23T21:19:09.176 INFO:teuthology.orchestra.run.smithi026.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T21:19:09.183 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:19:08 smithi026 ceph-mon[179688]: failed to encode map e639 with expected crc 2023-09-23T21:19:09.184 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:19:08 smithi026 ceph-mon[179688]: failed to encode map e639 with expected crc 2023-09-23T21:19:09.184 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:19:08 smithi026 ceph-mon[179688]: osdmap e639: 8 total, 8 up, 6 in 2023-09-23T21:19:09.184 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:19:08 smithi026 ceph-mon[179688]: failed to encode map e639 with expected crc 2023-09-23T21:19:09.184 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:19:08 smithi026 ceph-mon[175151]: failed to encode map e639 with expected crc 2023-09-23T21:19:09.184 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:19:08 smithi026 ceph-mon[175151]: failed to encode map e639 with expected crc 2023-09-23T21:19:09.184 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:19:08 smithi026 ceph-mon[175151]: osdmap e639: 8 total, 8 up, 6 in 2023-09-23T21:19:09.184 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:19:08 smithi026 ceph-mon[175151]: failed to encode map e639 with expected crc 2023-09-23T21:19:09.210 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:19:09.234 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:19:08 smithi070 ceph-mon[168282]: failed to encode map e639 with expected crc 2023-09-23T21:19:09.234 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:19:08 smithi070 ceph-mon[168282]: failed to encode map e639 with expected crc 2023-09-23T21:19:09.234 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:19:08 smithi070 ceph-mon[168282]: osdmap e639: 8 total, 8 up, 6 in 2023-09-23T21:19:09.234 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:19:08 smithi070 ceph-mon[168282]: failed to encode map e639 with expected crc 2023-09-23T21:19:09.236 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:09.269 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:19:09.271 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:09.271 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:19:09.372 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:19:09.398 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:09.430 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:19:09.432 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:09.433 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:19:09.534 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:19:09.564 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:09.777 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:19:09.780 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:09.781 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:19:09.830 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 21:19:09 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[164029]: ::ffff:172.21.15.70 - - [23/Sep/2023:21:19:09] "GET /metrics HTTP/1.1" 200 36371 "" "Prometheus/2.43.0" 2023-09-23T21:19:09.831 INFO:teuthology.orchestra.run.smithi026.stdout:{"pg_ready":true,"pg_map":{"version":1578,"stamp":"2023-09-23T21:19:07.888548+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":1906592,"num_objects":43,"num_object_clones":0,"num_object_copies":129,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":43,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":100478,"num_write_kb":2940670,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":9019,"num_bytes_recovered":575884606,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":137827,"ondisk_log_size":137827,"up":387,"acting":387,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":392,"num_osds":6,"num_per_pool_osds":6,"num_per_pool_omap_osds":6,"kb":562470912,"kb_used":4439376,"kb_used_data":92000,"kb_used_omap":125,"kb_used_meta":4347202,"kb_avail":558031536,"statfs":{"total":575970213888,"available":571424292864,"internally_reserved":0,"allocated":94208000,"data_stored":80341082,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":128662,"internal_metadata":4451535210},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":5,"apply_latency_ms":5,"commit_latency_ns":5000000,"apply_latency_ns":5000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-112099352,"num_objects":-1712,"num_object_clones":0,"num_object_copies":-5136,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":-1712,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1725,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"4.031857"},"pg_stats":[{"pgid":"11.12","version":"634'8996","reported_seq":10344,"reported_epoch":636,"state":"active+clean","last_fresh":"2023-09-23T21:19:04.869079+0000","last_change":"2023-09-23T21:18:40.100577+0000","last_active":"2023-09-23T21:19:04.869079+0000","last_peered":"2023-09-23T21:19:04.869079+0000","last_clean":"2023-09-23T21:19:04.869079+0000","last_became_active":"2023-09-23T21:18:34.930698+0000","last_became_peered":"2023-09-23T21:18:34.930698+0000","last_unstale":"2023-09-23T21:19:04.869079+0000","last_undegraded":"2023-09-23T21:19:04.869079+0000","last_fullsized":"2023-09-23T21:19:04.869079+0000","mapping_epoch":613,"log_start":"549'2054","ondisk_log_start":"549'2054","created":559,"last_epoch_clean":614,"parent":"0.0","parent_split_bits":5,"last_scrub":"618'8566","last_scrub_stamp":"2023-09-23T21:18:40.100396+0000","last_deep_scrub":"618'8566","last_deep_scrub_stamp":"2023-09-23T21:18:40.100396+0000","last_clean_scrub_stamp":"2023-09-23T21:18:40.100396+0000","objects_scrubbed":431,"log_size":6942,"log_dups_size":0,"ondisk_log_size":6942,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":2,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:16:14.066996+0000","scrub_duration":1.094034102,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":431,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,0],"acting":[2,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.18","version":"79'161","reported_seq":662,"reported_epoch":637,"state":"active+clean","last_fresh":"2023-09-23T21:19:06.194492+0000","last_change":"2023-09-23T21:17:39.906535+0000","last_active":"2023-09-23T21:19:06.194492+0000","last_peered":"2023-09-23T21:19:06.194492+0000","last_clean":"2023-09-23T21:19:06.194492+0000","last_became_active":"2023-09-23T21:17:39.906154+0000","last_became_peered":"2023-09-23T21:17:39.906154+0000","last_unstale":"2023-09-23T21:19:06.194492+0000","last_undegraded":"2023-09-23T21:19:06.194492+0000","last_fullsized":"2023-09-23T21:19:06.194492+0000","mapping_epoch":585,"log_start":"0'0","ondisk_log_start":"0'0","created":479,"last_epoch_clean":586,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:15:39.474823+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:15:39.474823+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:48:39.784037+0000","scrub_duration":0.074664395999999994,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":525376,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1a","version":"68'1","reported_seq":228,"reported_epoch":637,"state":"active+clean","last_fresh":"2023-09-23T21:19:05.872823+0000","last_change":"2023-09-23T21:13:13.291403+0000","last_active":"2023-09-23T21:19:05.872823+0000","last_peered":"2023-09-23T21:19:05.872823+0000","last_clean":"2023-09-23T21:19:05.872823+0000","last_became_active":"2023-09-23T21:13:13.291123+0000","last_became_peered":"2023-09-23T21:13:13.291123+0000","last_unstale":"2023-09-23T21:19:05.872823+0000","last_undegraded":"2023-09-23T21:19:05.872823+0000","last_fullsized":"2023-09-23T21:19:05.872823+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":495,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:45:15.427430+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,0],"acting":[4,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":297,"reported_epoch":637,"state":"active+clean","last_fresh":"2023-09-23T21:19:05.873310+0000","last_change":"2023-09-23T21:12:10.767963+0000","last_active":"2023-09-23T21:19:05.873310+0000","last_peered":"2023-09-23T21:19:05.873310+0000","last_clean":"2023-09-23T21:19:05.873310+0000","last_became_active":"2023-09-23T21:12:10.767126+0000","last_became_peered":"2023-09-23T21:12:10.767126+0000","last_unstale":"2023-09-23T21:19:05.873310+0000","last_undegraded":"2023-09-23T21:19:05.873310+0000","last_fullsized":"2023-09-23T21:19:05.873310+0000","mapping_epoch":470,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:45:47.138277+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,5],"acting":[4,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.11","version":"634'8464","reported_seq":9799,"reported_epoch":636,"state":"active+clean","last_fresh":"2023-09-23T21:19:04.868725+0000","last_change":"2023-09-23T21:18:36.195191+0000","last_active":"2023-09-23T21:19:04.868725+0000","last_peered":"2023-09-23T21:19:04.868725+0000","last_clean":"2023-09-23T21:19:04.868725+0000","last_became_active":"2023-09-23T21:18:32.876101+0000","last_became_peered":"2023-09-23T21:18:32.876101+0000","last_unstale":"2023-09-23T21:19:04.868725+0000","last_undegraded":"2023-09-23T21:19:04.868725+0000","last_fullsized":"2023-09-23T21:19:04.868725+0000","mapping_epoch":611,"log_start":"555'3628","ondisk_log_start":"555'3628","created":559,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":5,"last_scrub":"614'7979","last_scrub_stamp":"2023-09-23T21:18:36.195023+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_clean_scrub_stamp":"2023-09-23T21:18:36.195023+0000","objects_scrubbed":487,"log_size":4836,"log_dups_size":0,"ondisk_log_size":4836,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:18:09.227629+0000","scrub_duration":0.17685009800000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":487,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,7],"acting":[2,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.1b","version":"79'161","reported_seq":825,"reported_epoch":637,"state":"active+clean","last_fresh":"2023-09-23T21:19:06.195444+0000","last_change":"2023-09-23T21:15:38.423372+0000","last_active":"2023-09-23T21:19:06.195444+0000","last_peered":"2023-09-23T21:19:06.195444+0000","last_clean":"2023-09-23T21:19:06.195444+0000","last_became_active":"2023-09-23T21:15:23.802580+0000","last_became_peered":"2023-09-23T21:15:23.802580+0000","last_unstale":"2023-09-23T21:19:06.195444+0000","last_undegraded":"2023-09-23T21:19:06.195444+0000","last_fullsized":"2023-09-23T21:19:06.195444+0000","mapping_epoch":542,"log_start":"0'0","ondisk_log_start":"0'0","created":479,"last_epoch_clean":543,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:15:38.423271+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:15:38.423271+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:04:46.575186+0000","scrub_duration":0.00258465,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":650,"reported_epoch":634,"state":"active+clean","last_fresh":"2023-09-23T21:18:56.915714+0000","last_change":"2023-09-23T21:16:42.448529+0000","last_active":"2023-09-23T21:18:56.915714+0000","last_peered":"2023-09-23T21:18:56.915714+0000","last_clean":"2023-09-23T21:18:56.915714+0000","last_became_active":"2023-09-23T21:16:42.447803+0000","last_became_peered":"2023-09-23T21:16:42.447803+0000","last_unstale":"2023-09-23T21:18:56.915714+0000","last_undegraded":"2023-09-23T21:18:56.915714+0000","last_fullsized":"2023-09-23T21:18:56.915714+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:41:33.121784+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":652,"reported_epoch":635,"state":"active+clean","last_fresh":"2023-09-23T21:19:04.067341+0000","last_change":"2023-09-23T21:16:42.441896+0000","last_active":"2023-09-23T21:19:04.067341+0000","last_peered":"2023-09-23T21:19:04.067341+0000","last_clean":"2023-09-23T21:19:04.067341+0000","last_became_active":"2023-09-23T21:16:42.441526+0000","last_became_peered":"2023-09-23T21:16:42.441526+0000","last_unstale":"2023-09-23T21:19:04.067341+0000","last_undegraded":"2023-09-23T21:19:04.067341+0000","last_fullsized":"2023-09-23T21:19:04.067341+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:27:36.326225+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.10","version":"634'8419","reported_seq":12097,"reported_epoch":635,"state":"active+clean","last_fresh":"2023-09-23T21:19:04.066353+0000","last_change":"2023-09-23T21:18:36.698828+0000","last_active":"2023-09-23T21:19:04.066353+0000","last_peered":"2023-09-23T21:19:04.066353+0000","last_clean":"2023-09-23T21:19:04.066353+0000","last_became_active":"2023-09-23T21:18:30.880043+0000","last_became_peered":"2023-09-23T21:18:30.880043+0000","last_unstale":"2023-09-23T21:19:04.066353+0000","last_undegraded":"2023-09-23T21:19:04.066353+0000","last_fullsized":"2023-09-23T21:19:04.066353+0000","mapping_epoch":609,"log_start":"556'3988","ondisk_log_start":"556'3988","created":559,"last_epoch_clean":610,"parent":"0.0","parent_split_bits":5,"last_scrub":"614'7936","last_scrub_stamp":"2023-09-23T21:18:36.698511+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_clean_scrub_stamp":"2023-09-23T21:18:36.698511+0000","objects_scrubbed":486,"log_size":4431,"log_dups_size":0,"ondisk_log_size":4431,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:26:33.228381+0000","scrub_duration":0.115560511,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":486,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,5],"acting":[0,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1a","version":"79'161","reported_seq":735,"reported_epoch":637,"state":"active+clean","last_fresh":"2023-09-23T21:19:05.872334+0000","last_change":"2023-09-23T21:16:45.419221+0000","last_active":"2023-09-23T21:19:05.872334+0000","last_peered":"2023-09-23T21:19:05.872334+0000","last_clean":"2023-09-23T21:19:05.872334+0000","last_became_active":"2023-09-23T21:16:45.418994+0000","last_became_peered":"2023-09-23T21:16:45.418994+0000","last_unstale":"2023-09-23T21:19:05.872334+0000","last_undegraded":"2023-09-23T21:19:05.872334+0000","last_fullsized":"2023-09-23T21:19:05.872334+0000","mapping_epoch":567,"log_start":"0'0","ondisk_log_start":"0'0","created":479,"last_epoch_clean":568,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:15:45.158107+0000","last_deep_scrub":"79'161","last_deep_scrub_stamp":"2023-09-23T21:05:32.630292+0000","last_clean_scrub_stamp":"2023-09-23T21:15:45.158107+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:08:08.342444+0000","scrub_duration":0.001999373,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,5],"acting":[4,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":147,"reported_epoch":635,"state":"active+clean","last_fresh":"2023-09-23T21:19:04.066407+0000","last_change":"2023-09-23T21:13:30.806084+0000","last_active":"2023-09-23T21:19:04.066407+0000","last_peered":"2023-09-23T21:19:04.066407+0000","last_clean":"2023-09-23T21:19:04.066407+0000","last_became_active":"2023-09-23T21:13:30.805743+0000","last_became_peered":"2023-09-23T21:13:30.805743+0000","last_unstale":"2023-09-23T21:19:04.066407+0000","last_undegraded":"2023-09-23T21:19:04.066407+0000","last_fullsized":"2023-09-23T21:19:04.066407+0000","mapping_epoch":501,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:48:10.849281+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":511,"reported_epoch":635,"state":"active+clean","last_fresh":"2023-09-23T21:19:04.067131+0000","last_change":"2023-09-23T21:13:44.410921+0000","last_active":"2023-09-23T21:19:04.067131+0000","last_peered":"2023-09-23T21:19:04.067131+0000","last_clean":"2023-09-23T21:19:04.067131+0000","last_became_active":"2023-09-23T21:13:44.410633+0000","last_became_peered":"2023-09-23T21:13:44.410633+0000","last_unstale":"2023-09-23T21:19:04.067131+0000","last_undegraded":"2023-09-23T21:19:04.067131+0000","last_fullsized":"2023-09-23T21:19:04.067131+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:57:10.116786+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.15","version":"79'161","reported_seq":596,"reported_epoch":637,"state":"active+clean","last_fresh":"2023-09-23T21:19:05.887531+0000","last_change":"2023-09-23T21:18:35.622513+0000","last_active":"2023-09-23T21:19:05.887531+0000","last_peered":"2023-09-23T21:19:05.887531+0000","last_clean":"2023-09-23T21:19:05.887531+0000","last_became_active":"2023-09-23T21:16:45.425654+0000","last_became_peered":"2023-09-23T21:16:45.425654+0000","last_unstale":"2023-09-23T21:19:05.887531+0000","last_undegraded":"2023-09-23T21:19:05.887531+0000","last_fullsized":"2023-09-23T21:19:05.887531+0000","mapping_epoch":567,"log_start":"0'0","ondisk_log_start":"0'0","created":479,"last_epoch_clean":568,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:18:35.622422+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:18:35.622422+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:21:48.421117+0000","scrub_duration":0.083031242000000005,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,3],"acting":[7,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":612,"reported_epoch":635,"state":"active+clean","last_fresh":"2023-09-23T21:19:04.067173+0000","last_change":"2023-09-23T21:13:12.235728+0000","last_active":"2023-09-23T21:19:04.067173+0000","last_peered":"2023-09-23T21:19:04.067173+0000","last_clean":"2023-09-23T21:19:04.067173+0000","last_became_active":"2023-09-23T21:13:12.216549+0000","last_became_peered":"2023-09-23T21:13:12.216549+0000","last_unstale":"2023-09-23T21:19:04.067173+0000","last_undegraded":"2023-09-23T21:19:04.067173+0000","last_fullsized":"2023-09-23T21:19:04.067173+0000","mapping_epoch":493,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:39:33.324239+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":674,"reported_epoch":634,"state":"active+clean","last_fresh":"2023-09-23T21:18:56.915588+0000","last_change":"2023-09-23T21:12:10.790349+0000","last_active":"2023-09-23T21:18:56.915588+0000","last_peered":"2023-09-23T21:18:56.915588+0000","last_clean":"2023-09-23T21:18:56.915588+0000","last_became_active":"2023-09-23T21:12:10.789977+0000","last_became_peered":"2023-09-23T21:12:10.789977+0000","last_unstale":"2023-09-23T21:18:56.915588+0000","last_undegraded":"2023-09-23T21:18:56.915588+0000","last_fullsized":"2023-09-23T21:18:56.915588+0000","mapping_epoch":470,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:49:40.191147+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,5],"acting":[3,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"79'161","reported_seq":1034,"reported_epoch":636,"state":"active+clean","last_fresh":"2023-09-23T21:19:04.869518+0000","last_change":"2023-09-23T21:16:45.424773+0000","last_active":"2023-09-23T21:19:04.869518+0000","last_peered":"2023-09-23T21:19:04.869518+0000","last_clean":"2023-09-23T21:19:04.869518+0000","last_became_active":"2023-09-23T21:16:45.424458+0000","last_became_peered":"2023-09-23T21:16:45.424458+0000","last_unstale":"2023-09-23T21:19:04.869518+0000","last_undegraded":"2023-09-23T21:19:04.869518+0000","last_fullsized":"2023-09-23T21:19:04.869518+0000","mapping_epoch":567,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":568,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:25.096401+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:25.096401+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:40:17.871561+0000","scrub_duration":0.002859393,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":382,"reported_epoch":637,"state":"active+clean","last_fresh":"2023-09-23T21:19:06.195349+0000","last_change":"2023-09-23T21:16:42.427856+0000","last_active":"2023-09-23T21:19:06.195349+0000","last_peered":"2023-09-23T21:19:06.195349+0000","last_clean":"2023-09-23T21:19:06.195349+0000","last_became_active":"2023-09-23T21:16:42.427529+0000","last_became_peered":"2023-09-23T21:16:42.427529+0000","last_unstale":"2023-09-23T21:19:06.195349+0000","last_undegraded":"2023-09-23T21:19:06.195349+0000","last_fullsized":"2023-09-23T21:19:06.195349+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:24:03.818632+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":629,"reported_epoch":634,"state":"active+clean","last_fresh":"2023-09-23T21:18:56.915577+0000","last_change":"2023-09-23T21:16:42.448719+0000","last_active":"2023-09-23T21:18:56.915577+0000","last_peered":"2023-09-23T21:18:56.915577+0000","last_clean":"2023-09-23T21:18:56.915577+0000","last_became_active":"2023-09-23T21:16:42.448219+0000","last_became_peered":"2023-09-23T21:16:42.448219+0000","last_unstale":"2023-09-23T21:18:56.915577+0000","last_undegraded":"2023-09-23T21:18:56.915577+0000","last_fullsized":"2023-09-23T21:18:56.915577+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:02:28.864357+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.17","version":"79'161","reported_seq":783,"reported_epoch":635,"state":"active+clean","last_fresh":"2023-09-23T21:19:04.066927+0000","last_change":"2023-09-23T21:16:45.413663+0000","last_active":"2023-09-23T21:19:04.066927+0000","last_peered":"2023-09-23T21:19:04.066927+0000","last_clean":"2023-09-23T21:19:04.066927+0000","last_became_active":"2023-09-23T21:16:45.413311+0000","last_became_peered":"2023-09-23T21:16:45.413311+0000","last_unstale":"2023-09-23T21:19:04.066927+0000","last_undegraded":"2023-09-23T21:19:04.066927+0000","last_fullsized":"2023-09-23T21:19:04.066927+0000","mapping_epoch":567,"log_start":"0'0","ondisk_log_start":"0'0","created":479,"last_epoch_clean":568,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:15:50.325741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:15:50.325741+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:07:43.285354+0000","scrub_duration":0.0040500149999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":551,"reported_epoch":634,"state":"active+clean","last_fresh":"2023-09-23T21:18:56.916567+0000","last_change":"2023-09-23T21:17:02.182606+0000","last_active":"2023-09-23T21:18:56.916567+0000","last_peered":"2023-09-23T21:18:56.916567+0000","last_clean":"2023-09-23T21:18:56.916567+0000","last_became_active":"2023-09-23T21:16:42.446517+0000","last_became_peered":"2023-09-23T21:16:42.446517+0000","last_unstale":"2023-09-23T21:18:56.916567+0000","last_undegraded":"2023-09-23T21:18:56.916567+0000","last_fullsized":"2023-09-23T21:18:56.916567+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:46:08.337878+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":540,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,4],"acting":[3,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":325,"reported_epoch":636,"state":"active+clean","last_fresh":"2023-09-23T21:19:04.869201+0000","last_change":"2023-09-23T21:13:30.821489+0000","last_active":"2023-09-23T21:19:04.869201+0000","last_peered":"2023-09-23T21:19:04.869201+0000","last_clean":"2023-09-23T21:19:04.869201+0000","last_became_active":"2023-09-23T21:13:30.821125+0000","last_became_peered":"2023-09-23T21:13:30.821125+0000","last_unstale":"2023-09-23T21:19:04.869201+0000","last_undegraded":"2023-09-23T21:19:04.869201+0000","last_fullsized":"2023-09-23T21:19:04.869201+0000","mapping_epoch":501,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:47:18.553417+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,5],"acting":[2,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.16","version":"79'161","reported_seq":568,"reported_epoch":635,"state":"active+clean","last_fresh":"2023-09-23T21:19:04.067294+0000","last_change":"2023-09-23T21:15:46.326726+0000","last_active":"2023-09-23T21:19:04.067294+0000","last_peered":"2023-09-23T21:19:04.067294+0000","last_clean":"2023-09-23T21:19:04.067294+0000","last_became_active":"2023-09-23T21:13:33.770407+0000","last_became_peered":"2023-09-23T21:13:33.770407+0000","last_unstale":"2023-09-23T21:19:04.067294+0000","last_undegraded":"2023-09-23T21:19:04.067294+0000","last_fullsized":"2023-09-23T21:19:04.067294+0000","mapping_epoch":504,"log_start":"0'0","ondisk_log_start":"0'0","created":479,"last_epoch_clean":505,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:15:46.326589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:15:46.326589+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:50:23.810054+0000","scrub_duration":0.0062069059999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.14","version":"79'22","reported_seq":241,"reported_epoch":637,"state":"active+clean","last_fresh":"2023-09-23T21:19:05.873024+0000","last_change":"2023-09-23T21:17:02.198735+0000","last_active":"2023-09-23T21:19:05.873024+0000","last_peered":"2023-09-23T21:19:05.873024+0000","last_clean":"2023-09-23T21:19:05.873024+0000","last_became_active":"2023-09-23T21:16:42.492861+0000","last_became_peered":"2023-09-23T21:16:42.492861+0000","last_unstale":"2023-09-23T21:19:05.873024+0000","last_undegraded":"2023-09-23T21:19:05.873024+0000","last_fullsized":"2023-09-23T21:19:05.873024+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:00:09.949965+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":14202,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,2],"acting":[4,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":566,"reported_epoch":637,"state":"active+clean","last_fresh":"2023-09-23T21:19:05.872764+0000","last_change":"2023-09-23T21:12:10.839271+0000","last_active":"2023-09-23T21:19:05.872764+0000","last_peered":"2023-09-23T21:19:05.872764+0000","last_clean":"2023-09-23T21:19:05.872764+0000","last_became_active":"2023-09-23T21:12:10.839042+0000","last_became_peered":"2023-09-23T21:12:10.839042+0000","last_unstale":"2023-09-23T21:19:05.872764+0000","last_undegraded":"2023-09-23T21:19:05.872764+0000","last_fullsized":"2023-09-23T21:19:05.872764+0000","mapping_epoch":470,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:10:26.414737+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,2],"acting":[4,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.11","version":"79'161","reported_seq":573,"reported_epoch":635,"state":"active+clean","last_fresh":"2023-09-23T21:19:04.066236+0000","last_change":"2023-09-23T21:15:47.300109+0000","last_active":"2023-09-23T21:19:04.066236+0000","last_peered":"2023-09-23T21:19:04.066236+0000","last_clean":"2023-09-23T21:19:04.066236+0000","last_became_active":"2023-09-23T21:13:33.767067+0000","last_became_peered":"2023-09-23T21:13:33.767067+0000","last_unstale":"2023-09-23T21:19:04.066236+0000","last_undegraded":"2023-09-23T21:19:04.066236+0000","last_fullsized":"2023-09-23T21:19:04.066236+0000","mapping_epoch":504,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":505,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:15:47.299947+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:15:47.299947+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:04:59.029359+0000","scrub_duration":0.0061353110000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":89,"reported_epoch":637,"state":"active+clean","last_fresh":"2023-09-23T21:19:05.887042+0000","last_change":"2023-09-23T21:16:42.531060+0000","last_active":"2023-09-23T21:19:05.887042+0000","last_peered":"2023-09-23T21:19:05.887042+0000","last_clean":"2023-09-23T21:19:05.887042+0000","last_became_active":"2023-09-23T21:16:42.449198+0000","last_became_peered":"2023-09-23T21:16:42.449198+0000","last_unstale":"2023-09-23T21:19:05.887042+0000","last_undegraded":"2023-09-23T21:19:05.887042+0000","last_fullsized":"2023-09-23T21:19:05.887042+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:30:23.428886+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":197,"reported_epoch":636,"state":"active+clean","last_fresh":"2023-09-23T21:19:04.869326+0000","last_change":"2023-09-23T21:16:42.491023+0000","last_active":"2023-09-23T21:19:04.869326+0000","last_peered":"2023-09-23T21:19:04.869326+0000","last_clean":"2023-09-23T21:19:04.869326+0000","last_became_active":"2023-09-23T21:16:42.489896+0000","last_became_peered":"2023-09-23T21:16:42.489896+0000","last_unstale":"2023-09-23T21:19:04.869326+0000","last_undegraded":"2023-09-23T21:19:04.869326+0000","last_fullsized":"2023-09-23T21:19:04.869326+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:58:54.036098+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,7],"acting":[2,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.10","version":"79'161","reported_seq":812,"reported_epoch":637,"state":"active+clean","last_fresh":"2023-09-23T21:19:05.872337+0000","last_change":"2023-09-23T21:16:45.416227+0000","last_active":"2023-09-23T21:19:05.872337+0000","last_peered":"2023-09-23T21:19:05.872337+0000","last_clean":"2023-09-23T21:19:05.872337+0000","last_became_active":"2023-09-23T21:16:45.415841+0000","last_became_peered":"2023-09-23T21:16:45.415841+0000","last_unstale":"2023-09-23T21:19:05.872337+0000","last_undegraded":"2023-09-23T21:19:05.872337+0000","last_fullsized":"2023-09-23T21:19:05.872337+0000","mapping_epoch":567,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":568,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:06:50.336992+0000","last_deep_scrub":"79'161","last_deep_scrub_stamp":"2023-09-23T21:06:50.336992+0000","last_clean_scrub_stamp":"2023-09-23T21:06:50.336992+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:37:30.094658+0000","scrub_duration":0.0026150930000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,7],"acting":[4,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":596,"reported_epoch":635,"state":"active+clean","last_fresh":"2023-09-23T21:19:04.066816+0000","last_change":"2023-09-23T21:16:42.446324+0000","last_active":"2023-09-23T21:19:04.066816+0000","last_peered":"2023-09-23T21:19:04.066816+0000","last_clean":"2023-09-23T21:19:04.066816+0000","last_became_active":"2023-09-23T21:16:42.445997+0000","last_became_peered":"2023-09-23T21:16:42.445997+0000","last_unstale":"2023-09-23T21:19:04.066816+0000","last_undegraded":"2023-09-23T21:19:04.066816+0000","last_fullsized":"2023-09-23T21:19:04.066816+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:46:35.028292+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":406,"reported_epoch":634,"state":"active+clean","last_fresh":"2023-09-23T21:18:56.916403+0000","last_change":"2023-09-23T21:16:42.473872+0000","last_active":"2023-09-23T21:18:56.916403+0000","last_peered":"2023-09-23T21:18:56.916403+0000","last_clean":"2023-09-23T21:18:56.916403+0000","last_became_active":"2023-09-23T21:16:42.473595+0000","last_became_peered":"2023-09-23T21:16:42.473595+0000","last_unstale":"2023-09-23T21:18:56.916403+0000","last_undegraded":"2023-09-23T21:18:56.916403+0000","last_fullsized":"2023-09-23T21:18:56.916403+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:15:19.186428+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.19","version":"634'8378","reported_seq":10196,"reported_epoch":636,"state":"remapped+peering","last_fresh":"2023-09-23T21:19:04.865614+0000","last_change":"2023-09-23T21:19:04.865614+0000","last_active":"2023-09-23T21:19:04.067193+0000","last_peered":"2023-09-23T21:19:02.983696+0000","last_clean":"2023-09-23T21:19:02.983696+0000","last_became_active":"2023-09-23T21:17:46.300166+0000","last_became_peered":"2023-09-23T21:17:46.300166+0000","last_unstale":"2023-09-23T21:19:04.865614+0000","last_undegraded":"2023-09-23T21:19:04.865614+0000","last_fullsized":"2023-09-23T21:19:04.865614+0000","mapping_epoch":636,"log_start":"556'4209","ondisk_log_start":"556'4209","created":559,"last_epoch_clean":601,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_clean_scrub_stamp":"2023-09-23T21:15:24.860968+0000","objects_scrubbed":0,"log_size":4169,"log_dups_size":0,"ondisk_log_size":4169,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6095,"num_write_kb":194592,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":838,"num_bytes_recovered":54820864,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,2],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.13","version":"79'161","reported_seq":713,"reported_epoch":634,"state":"active+clean","last_fresh":"2023-09-23T21:18:56.915515+0000","last_change":"2023-09-23T21:13:14.933096+0000","last_active":"2023-09-23T21:18:56.915515+0000","last_peered":"2023-09-23T21:18:56.915515+0000","last_clean":"2023-09-23T21:18:56.915515+0000","last_became_active":"2023-09-23T21:13:14.932703+0000","last_became_peered":"2023-09-23T21:13:14.932703+0000","last_unstale":"2023-09-23T21:18:56.915515+0000","last_undegraded":"2023-09-23T21:18:56.915515+0000","last_fullsized":"2023-09-23T21:18:56.915515+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:36.420362+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:36.420362+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:29:42.163911+0000","scrub_duration":0.028838444000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":649,"reported_epoch":637,"state":"active+clean","last_fresh":"2023-09-23T21:19:05.872418+0000","last_change":"2023-09-23T21:16:42.420467+0000","last_active":"2023-09-23T21:19:05.872418+0000","last_peered":"2023-09-23T21:19:05.872418+0000","last_clean":"2023-09-23T21:19:05.872418+0000","last_became_active":"2023-09-23T21:16:42.420149+0000","last_became_peered":"2023-09-23T21:16:42.420149+0000","last_unstale":"2023-09-23T21:19:05.872418+0000","last_undegraded":"2023-09-23T21:19:05.872418+0000","last_fullsized":"2023-09-23T21:19:05.872418+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:52:07.355000+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":165,"reported_epoch":635,"state":"active+clean","last_fresh":"2023-09-23T21:19:04.067090+0000","last_change":"2023-09-23T21:16:42.427663+0000","last_active":"2023-09-23T21:19:04.067090+0000","last_peered":"2023-09-23T21:19:04.067090+0000","last_clean":"2023-09-23T21:19:04.067090+0000","last_became_active":"2023-09-23T21:16:42.426961+0000","last_became_peered":"2023-09-23T21:16:42.426961+0000","last_unstale":"2023-09-23T21:19:04.067090+0000","last_undegraded":"2023-09-23T21:19:04.067090+0000","last_fullsized":"2023-09-23T21:19:04.067090+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:44:27.422007+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.18","version":"634'8149","reported_seq":8982,"reported_epoch":636,"state":"active+clean","last_fresh":"2023-09-23T21:19:04.866989+0000","last_change":"2023-09-23T21:18:56.949632+0000","last_active":"2023-09-23T21:19:04.866989+0000","last_peered":"2023-09-23T21:19:04.866989+0000","last_clean":"2023-09-23T21:19:04.866989+0000","last_became_active":"2023-09-23T21:18:56.922771+0000","last_became_peered":"2023-09-23T21:18:56.922771+0000","last_unstale":"2023-09-23T21:19:04.866989+0000","last_undegraded":"2023-09-23T21:19:04.866989+0000","last_fullsized":"2023-09-23T21:19:04.866989+0000","mapping_epoch":633,"log_start":"554'3311","ondisk_log_start":"554'3311","created":559,"last_epoch_clean":634,"parent":"0.0","parent_split_bits":5,"last_scrub":"611'7608","last_scrub_stamp":"2023-09-23T21:18:32.625076+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_clean_scrub_stamp":"2023-09-23T21:18:32.625076+0000","objects_scrubbed":541,"log_size":4838,"log_dups_size":0,"ondisk_log_size":4838,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:29:11.075466+0000","scrub_duration":0.078956060999999994,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":541,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":225,"num_bytes_recovered":14745600,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,5],"acting":[2,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.12","version":"79'161","reported_seq":911,"reported_epoch":637,"state":"active+clean","last_fresh":"2023-09-23T21:19:06.195177+0000","last_change":"2023-09-23T21:16:45.412440+0000","last_active":"2023-09-23T21:19:06.195177+0000","last_peered":"2023-09-23T21:19:06.195177+0000","last_clean":"2023-09-23T21:19:06.195177+0000","last_became_active":"2023-09-23T21:16:45.412029+0000","last_became_peered":"2023-09-23T21:16:45.412029+0000","last_unstale":"2023-09-23T21:19:06.195177+0000","last_undegraded":"2023-09-23T21:19:06.195177+0000","last_fullsized":"2023-09-23T21:19:06.195177+0000","mapping_epoch":567,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":568,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:05:56.785173+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:05:56.785173+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:39:43.432998+0000","scrub_duration":0.0042114370000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,0],"acting":[5,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":157,"reported_epoch":635,"state":"active+clean","last_fresh":"2023-09-23T21:19:04.066453+0000","last_change":"2023-09-23T21:13:38.048791+0000","last_active":"2023-09-23T21:19:04.066453+0000","last_peered":"2023-09-23T21:19:04.066453+0000","last_clean":"2023-09-23T21:19:04.066453+0000","last_became_active":"2023-09-23T21:13:30.801386+0000","last_became_peered":"2023-09-23T21:13:30.801386+0000","last_unstale":"2023-09-23T21:19:04.066453+0000","last_undegraded":"2023-09-23T21:19:04.066453+0000","last_fullsized":"2023-09-23T21:19:04.066453+0000","mapping_epoch":501,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:52:49.170414+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":491,"reported_epoch":634,"state":"active+clean","last_fresh":"2023-09-23T21:18:56.916336+0000","last_change":"2023-09-23T21:16:42.518512+0000","last_active":"2023-09-23T21:18:56.916336+0000","last_peered":"2023-09-23T21:18:56.916336+0000","last_clean":"2023-09-23T21:18:56.916336+0000","last_became_active":"2023-09-23T21:16:42.517464+0000","last_became_peered":"2023-09-23T21:16:42.517464+0000","last_unstale":"2023-09-23T21:18:56.916336+0000","last_undegraded":"2023-09-23T21:18:56.916336+0000","last_fullsized":"2023-09-23T21:18:56.916336+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:47:52.391167+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.7","version":"634'8439","reported_seq":8596,"reported_epoch":637,"state":"active+clean","last_fresh":"2023-09-23T21:19:05.873034+0000","last_change":"2023-09-23T21:18:30.328388+0000","last_active":"2023-09-23T21:19:05.873034+0000","last_peered":"2023-09-23T21:19:05.873034+0000","last_clean":"2023-09-23T21:19:05.873034+0000","last_became_active":"2023-09-23T21:16:32.864890+0000","last_became_peered":"2023-09-23T21:16:32.864890+0000","last_unstale":"2023-09-23T21:19:05.873034+0000","last_undegraded":"2023-09-23T21:19:05.873034+0000","last_fullsized":"2023-09-23T21:19:05.873034+0000","mapping_epoch":544,"log_start":"550'2927","ondisk_log_start":"550'2927","created":544,"last_epoch_clean":560,"parent":"0.0","parent_split_bits":0,"last_scrub":"609'7868","last_scrub_stamp":"2023-09-23T21:18:30.328316+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_clean_scrub_stamp":"2023-09-23T21:18:30.328316+0000","objects_scrubbed":571,"log_size":5512,"log_dups_size":0,"ondisk_log_size":5512,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:34:57.557430+0000","scrub_duration":0.064969499999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":571,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.d","version":"79'161","reported_seq":679,"reported_epoch":637,"state":"active+clean","last_fresh":"2023-09-23T21:19:05.887629+0000","last_change":"2023-09-23T21:18:47.440374+0000","last_active":"2023-09-23T21:19:05.887629+0000","last_peered":"2023-09-23T21:19:05.887629+0000","last_clean":"2023-09-23T21:19:05.887629+0000","last_became_active":"2023-09-23T21:16:45.416373+0000","last_became_peered":"2023-09-23T21:16:45.416373+0000","last_unstale":"2023-09-23T21:19:05.887629+0000","last_undegraded":"2023-09-23T21:19:05.887629+0000","last_fullsized":"2023-09-23T21:19:05.887629+0000","mapping_epoch":567,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":568,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:18:47.440263+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:18:47.440263+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:11:36.676449+0000","scrub_duration":0.0023849959999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":87,"reported_epoch":637,"state":"active+clean","last_fresh":"2023-09-23T21:19:05.887817+0000","last_change":"2023-09-23T21:17:02.190320+0000","last_active":"2023-09-23T21:19:05.887817+0000","last_peered":"2023-09-23T21:19:05.887817+0000","last_clean":"2023-09-23T21:19:05.887817+0000","last_became_active":"2023-09-23T21:16:42.451380+0000","last_became_peered":"2023-09-23T21:16:42.451380+0000","last_unstale":"2023-09-23T21:19:05.887817+0000","last_undegraded":"2023-09-23T21:19:05.887817+0000","last_fullsized":"2023-09-23T21:19:05.887817+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:48:58.425821+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":132,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":365,"reported_epoch":637,"state":"active+clean","last_fresh":"2023-09-23T21:19:06.194701+0000","last_change":"2023-09-23T21:16:42.428491+0000","last_active":"2023-09-23T21:19:06.194701+0000","last_peered":"2023-09-23T21:19:06.194701+0000","last_clean":"2023-09-23T21:19:06.194701+0000","last_became_active":"2023-09-23T21:16:42.428166+0000","last_became_peered":"2023-09-23T21:16:42.428166+0000","last_unstale":"2023-09-23T21:19:06.194701+0000","last_undegraded":"2023-09-23T21:19:06.194701+0000","last_fullsized":"2023-09-23T21:19:06.194701+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:23:01.430714+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,4],"acting":[5,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.6","version":"634'8296","reported_seq":8434,"reported_epoch":635,"state":"active+clean","last_fresh":"2023-09-23T21:19:04.066835+0000","last_change":"2023-09-23T21:18:29.652253+0000","last_active":"2023-09-23T21:19:04.066835+0000","last_peered":"2023-09-23T21:19:04.066835+0000","last_clean":"2023-09-23T21:19:04.066835+0000","last_became_active":"2023-09-23T21:16:32.896510+0000","last_became_peered":"2023-09-23T21:16:32.896510+0000","last_unstale":"2023-09-23T21:19:04.066835+0000","last_undegraded":"2023-09-23T21:19:04.066835+0000","last_fullsized":"2023-09-23T21:19:04.066835+0000","mapping_epoch":544,"log_start":"556'3823","ondisk_log_start":"556'3823","created":544,"last_epoch_clean":561,"parent":"0.0","parent_split_bits":0,"last_scrub":"607'7678","last_scrub_stamp":"2023-09-23T21:18:29.652124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_clean_scrub_stamp":"2023-09-23T21:18:29.652124+0000","objects_scrubbed":618,"log_size":4473,"log_dups_size":0,"ondisk_log_size":4473,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:05:14.537109+0000","scrub_duration":0.077048132000000005,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":618,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.c","version":"79'161","reported_seq":1017,"reported_epoch":634,"state":"active+clean","last_fresh":"2023-09-23T21:18:56.915979+0000","last_change":"2023-09-23T21:15:31.992756+0000","last_active":"2023-09-23T21:18:56.915979+0000","last_peered":"2023-09-23T21:18:56.915979+0000","last_clean":"2023-09-23T21:18:56.915979+0000","last_became_active":"2023-09-23T21:13:13.401526+0000","last_became_peered":"2023-09-23T21:13:13.401526+0000","last_unstale":"2023-09-23T21:18:56.915979+0000","last_undegraded":"2023-09-23T21:18:56.915979+0000","last_fullsized":"2023-09-23T21:18:56.915979+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":532,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:15:31.992598+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:15:31.992598+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:41:14.215543+0000","scrub_duration":0.002960678,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":87,"reported_epoch":637,"state":"active+clean","last_fresh":"2023-09-23T21:19:05.887627+0000","last_change":"2023-09-23T21:17:02.215996+0000","last_active":"2023-09-23T21:19:05.887627+0000","last_peered":"2023-09-23T21:19:05.887627+0000","last_clean":"2023-09-23T21:19:05.887627+0000","last_became_active":"2023-09-23T21:16:42.451205+0000","last_became_peered":"2023-09-23T21:16:42.451205+0000","last_unstale":"2023-09-23T21:19:05.887627+0000","last_undegraded":"2023-09-23T21:19:05.887627+0000","last_fullsized":"2023-09-23T21:19:05.887627+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:51:46.876281+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":600,"reported_epoch":637,"state":"active+clean","last_fresh":"2023-09-23T21:19:05.872479+0000","last_change":"2023-09-23T21:13:15.152792+0000","last_active":"2023-09-23T21:19:05.872479+0000","last_peered":"2023-09-23T21:19:05.872479+0000","last_clean":"2023-09-23T21:19:05.872479+0000","last_became_active":"2023-09-23T21:13:14.515084+0000","last_became_peered":"2023-09-23T21:13:14.515084+0000","last_unstale":"2023-09-23T21:19:05.872479+0000","last_undegraded":"2023-09-23T21:19:05.872479+0000","last_fullsized":"2023-09-23T21:19:05.872479+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:08:16.702072+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.5","version":"634'8460","reported_seq":13742,"reported_epoch":637,"state":"active+clean","last_fresh":"2023-09-23T21:19:06.195541+0000","last_change":"2023-09-23T21:18:36.457465+0000","last_active":"2023-09-23T21:19:06.195541+0000","last_peered":"2023-09-23T21:19:06.195541+0000","last_clean":"2023-09-23T21:19:06.195541+0000","last_became_active":"2023-09-23T21:17:24.389052+0000","last_became_peered":"2023-09-23T21:17:24.389052+0000","last_unstale":"2023-09-23T21:19:06.195541+0000","last_undegraded":"2023-09-23T21:19:06.195541+0000","last_fullsized":"2023-09-23T21:19:06.195541+0000","mapping_epoch":578,"log_start":"549'2411","ondisk_log_start":"549'2411","created":544,"last_epoch_clean":579,"parent":"0.0","parent_split_bits":0,"last_scrub":"614'7983","last_scrub_stamp":"2023-09-23T21:18:36.457393+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_clean_scrub_stamp":"2023-09-23T21:18:36.457393+0000","objects_scrubbed":481,"log_size":6049,"log_dups_size":0,"ondisk_log_size":6049,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:07:29.014321+0000","scrub_duration":0.14039997700000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":481,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.f","version":"79'161","reported_seq":925,"reported_epoch":637,"state":"active+clean","last_fresh":"2023-09-23T21:19:06.195468+0000","last_change":"2023-09-23T21:11:30.750338+0000","last_active":"2023-09-23T21:19:06.195468+0000","last_peered":"2023-09-23T21:19:06.195468+0000","last_clean":"2023-09-23T21:19:06.195468+0000","last_became_active":"2023-09-23T21:11:30.749953+0000","last_became_peered":"2023-09-23T21:11:30.749953+0000","last_unstale":"2023-09-23T21:19:06.195468+0000","last_undegraded":"2023-09-23T21:19:06.195468+0000","last_fullsized":"2023-09-23T21:19:06.195468+0000","mapping_epoch":455,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":456,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:27.880776+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:27.880776+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:07:10.405723+0000","scrub_duration":0.0014004899999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,3],"acting":[5,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":389,"reported_epoch":637,"state":"active+clean","last_fresh":"2023-09-23T21:19:06.195603+0000","last_change":"2023-09-23T21:17:02.175012+0000","last_active":"2023-09-23T21:19:06.195603+0000","last_peered":"2023-09-23T21:19:06.195603+0000","last_clean":"2023-09-23T21:19:06.195603+0000","last_became_active":"2023-09-23T21:16:42.451114+0000","last_became_peered":"2023-09-23T21:16:42.451114+0000","last_unstale":"2023-09-23T21:19:06.195603+0000","last_undegraded":"2023-09-23T21:19:06.195603+0000","last_fullsized":"2023-09-23T21:19:06.195603+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:00:53.868525+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":420,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":597,"reported_epoch":637,"state":"active+clean","last_fresh":"2023-09-23T21:19:06.194851+0000","last_change":"2023-09-23T21:16:42.432943+0000","last_active":"2023-09-23T21:19:06.194851+0000","last_peered":"2023-09-23T21:19:06.194851+0000","last_clean":"2023-09-23T21:19:06.194851+0000","last_became_active":"2023-09-23T21:16:42.427440+0000","last_became_peered":"2023-09-23T21:16:42.427440+0000","last_unstale":"2023-09-23T21:19:06.194851+0000","last_undegraded":"2023-09-23T21:19:06.194851+0000","last_fullsized":"2023-09-23T21:19:06.194851+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:59:32.044609+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,0],"acting":[5,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.4","version":"634'8340","reported_seq":10035,"reported_epoch":637,"state":"active+clean","last_fresh":"2023-09-23T21:19:05.887502+0000","last_change":"2023-09-23T21:18:43.538320+0000","last_active":"2023-09-23T21:19:05.887502+0000","last_peered":"2023-09-23T21:19:05.887502+0000","last_clean":"2023-09-23T21:19:05.887502+0000","last_became_active":"2023-09-23T21:18:05.477101+0000","last_became_peered":"2023-09-23T21:18:05.477101+0000","last_unstale":"2023-09-23T21:19:05.887502+0000","last_undegraded":"2023-09-23T21:19:05.887502+0000","last_fullsized":"2023-09-23T21:19:05.887502+0000","mapping_epoch":595,"log_start":"554'3358","ondisk_log_start":"554'3358","created":544,"last_epoch_clean":596,"parent":"0.0","parent_split_bits":0,"last_scrub":"621'7884","last_scrub_stamp":"2023-09-23T21:18:43.538247+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_clean_scrub_stamp":"2023-09-23T21:18:43.538247+0000","objects_scrubbed":456,"log_size":4982,"log_dups_size":0,"ondisk_log_size":4982,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:09:58.379074+0000","scrub_duration":0.050975133999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":456,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,3],"acting":[7,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.e","version":"79'161","reported_seq":856,"reported_epoch":637,"state":"active+clean","last_fresh":"2023-09-23T21:19:05.873122+0000","last_change":"2023-09-23T21:16:45.433204+0000","last_active":"2023-09-23T21:19:05.873122+0000","last_peered":"2023-09-23T21:19:05.873122+0000","last_clean":"2023-09-23T21:19:05.873122+0000","last_became_active":"2023-09-23T21:16:45.432836+0000","last_became_peered":"2023-09-23T21:16:45.432836+0000","last_unstale":"2023-09-23T21:19:05.873122+0000","last_undegraded":"2023-09-23T21:19:05.873122+0000","last_fullsized":"2023-09-23T21:19:05.873122+0000","mapping_epoch":567,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":568,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:15:39.046962+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:15:39.046962+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:11:20.720731+0000","scrub_duration":0.0082638769999999993,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":434,"reported_epoch":634,"state":"active+clean","last_fresh":"2023-09-23T21:18:56.915764+0000","last_change":"2023-09-23T21:13:13.402418+0000","last_active":"2023-09-23T21:18:56.915764+0000","last_peered":"2023-09-23T21:18:56.915764+0000","last_clean":"2023-09-23T21:18:56.915764+0000","last_became_active":"2023-09-23T21:13:13.400984+0000","last_became_peered":"2023-09-23T21:13:13.400984+0000","last_unstale":"2023-09-23T21:18:56.915764+0000","last_undegraded":"2023-09-23T21:18:56.915764+0000","last_fullsized":"2023-09-23T21:18:56.915764+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":495,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:28:01.411191+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,0],"acting":[3,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":610,"reported_epoch":635,"state":"active+clean","last_fresh":"2023-09-23T21:19:04.066922+0000","last_change":"2023-09-23T21:13:14.339670+0000","last_active":"2023-09-23T21:19:04.066922+0000","last_peered":"2023-09-23T21:19:04.066922+0000","last_clean":"2023-09-23T21:19:04.066922+0000","last_became_active":"2023-09-23T21:13:14.339341+0000","last_became_peered":"2023-09-23T21:13:14.339341+0000","last_unstale":"2023-09-23T21:19:04.066922+0000","last_undegraded":"2023-09-23T21:19:04.066922+0000","last_fullsized":"2023-09-23T21:19:04.066922+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:23:19.411368+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.3","version":"634'8216","reported_seq":9953,"reported_epoch":637,"state":"active+clean","last_fresh":"2023-09-23T21:19:05.887589+0000","last_change":"2023-09-23T21:18:46.551584+0000","last_active":"2023-09-23T21:19:05.887589+0000","last_peered":"2023-09-23T21:19:05.887589+0000","last_clean":"2023-09-23T21:19:05.887589+0000","last_became_active":"2023-09-23T21:17:51.872537+0000","last_became_peered":"2023-09-23T21:17:51.872537+0000","last_unstale":"2023-09-23T21:19:05.887589+0000","last_undegraded":"2023-09-23T21:19:05.887589+0000","last_fullsized":"2023-09-23T21:19:05.887589+0000","mapping_epoch":591,"log_start":"550'3191","ondisk_log_start":"550'3191","created":544,"last_epoch_clean":592,"parent":"0.0","parent_split_bits":0,"last_scrub":"623'7891","last_scrub_stamp":"2023-09-23T21:18:46.551470+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_clean_scrub_stamp":"2023-09-23T21:18:46.551470+0000","objects_scrubbed":325,"log_size":5025,"log_dups_size":0,"ondisk_log_size":5025,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:22:45.095160+0000","scrub_duration":0.104744095,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":325,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,2],"acting":[7,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.9","version":"79'161","reported_seq":838,"reported_epoch":636,"state":"active+clean","last_fresh":"2023-09-23T21:19:04.868729+0000","last_change":"2023-09-23T21:16:45.433968+0000","last_active":"2023-09-23T21:19:04.868729+0000","last_peered":"2023-09-23T21:19:04.868729+0000","last_clean":"2023-09-23T21:19:04.868729+0000","last_became_active":"2023-09-23T21:16:45.433602+0000","last_became_peered":"2023-09-23T21:16:45.433602+0000","last_unstale":"2023-09-23T21:19:04.868729+0000","last_undegraded":"2023-09-23T21:19:04.868729+0000","last_fullsized":"2023-09-23T21:19:04.868729+0000","mapping_epoch":567,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":568,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:15:43.437369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:15:43.437369+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:16:44.538102+0000","scrub_duration":0.0024129429999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,3],"acting":[2,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":196,"reported_epoch":635,"state":"active+clean","last_fresh":"2023-09-23T21:19:04.067042+0000","last_change":"2023-09-23T21:13:30.793273+0000","last_active":"2023-09-23T21:19:04.067042+0000","last_peered":"2023-09-23T21:19:04.067042+0000","last_clean":"2023-09-23T21:19:04.067042+0000","last_became_active":"2023-09-23T21:13:30.791269+0000","last_became_peered":"2023-09-23T21:13:30.791269+0000","last_unstale":"2023-09-23T21:19:04.067042+0000","last_undegraded":"2023-09-23T21:19:04.067042+0000","last_fullsized":"2023-09-23T21:19:04.067042+0000","mapping_epoch":501,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:38:07.142290+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":231,"reported_epoch":637,"state":"active+clean","last_fresh":"2023-09-23T21:19:05.872695+0000","last_change":"2023-09-23T21:16:42.417057+0000","last_active":"2023-09-23T21:19:05.872695+0000","last_peered":"2023-09-23T21:19:05.872695+0000","last_clean":"2023-09-23T21:19:05.872695+0000","last_became_active":"2023-09-23T21:16:42.416601+0000","last_became_peered":"2023-09-23T21:16:42.416601+0000","last_unstale":"2023-09-23T21:19:05.872695+0000","last_undegraded":"2023-09-23T21:19:05.872695+0000","last_fullsized":"2023-09-23T21:19:05.872695+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:56:52.167562+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,7],"acting":[4,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.2","version":"634'8483","reported_seq":8618,"reported_epoch":636,"state":"active+clean","last_fresh":"2023-09-23T21:19:04.868899+0000","last_change":"2023-09-23T21:18:34.137284+0000","last_active":"2023-09-23T21:19:04.868899+0000","last_peered":"2023-09-23T21:19:04.868899+0000","last_clean":"2023-09-23T21:19:04.868899+0000","last_became_active":"2023-09-23T21:16:32.867780+0000","last_became_peered":"2023-09-23T21:16:32.867780+0000","last_unstale":"2023-09-23T21:19:04.868899+0000","last_undegraded":"2023-09-23T21:19:04.868899+0000","last_fullsized":"2023-09-23T21:19:04.868899+0000","mapping_epoch":544,"log_start":"548'859","ondisk_log_start":"548'859","created":544,"last_epoch_clean":560,"parent":"0.0","parent_split_bits":0,"last_scrub":"613'7927","last_scrub_stamp":"2023-09-23T21:18:34.136961+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_clean_scrub_stamp":"2023-09-23T21:18:34.136961+0000","objects_scrubbed":557,"log_size":7624,"log_dups_size":0,"ondisk_log_size":7624,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:53:14.227837+0000","scrub_duration":0.135147037,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":557,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.8","version":"79'161","reported_seq":771,"reported_epoch":636,"state":"active+clean","last_fresh":"2023-09-23T21:19:04.868776+0000","last_change":"2023-09-23T21:15:36.367819+0000","last_active":"2023-09-23T21:19:04.868776+0000","last_peered":"2023-09-23T21:19:04.868776+0000","last_clean":"2023-09-23T21:19:04.868776+0000","last_became_active":"2023-09-23T21:12:30.433217+0000","last_became_peered":"2023-09-23T21:12:30.433217+0000","last_unstale":"2023-09-23T21:19:04.868776+0000","last_undegraded":"2023-09-23T21:19:04.868776+0000","last_fullsized":"2023-09-23T21:19:04.868776+0000","mapping_epoch":359,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:15:36.367640+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:15:36.367640+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:49:35.243653+0000","scrub_duration":0.005826899,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":214,"reported_epoch":637,"state":"active+clean","last_fresh":"2023-09-23T21:19:05.872600+0000","last_change":"2023-09-23T21:12:10.823380+0000","last_active":"2023-09-23T21:19:05.872600+0000","last_peered":"2023-09-23T21:19:05.872600+0000","last_clean":"2023-09-23T21:19:05.872600+0000","last_became_active":"2023-09-23T21:12:10.823044+0000","last_became_peered":"2023-09-23T21:12:10.823044+0000","last_unstale":"2023-09-23T21:19:05.872600+0000","last_undegraded":"2023-09-23T21:19:05.872600+0000","last_fullsized":"2023-09-23T21:19:05.872600+0000","mapping_epoch":470,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:53:29.846536+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,5],"acting":[4,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":215,"reported_epoch":637,"state":"active+clean","last_fresh":"2023-09-23T21:19:05.873022+0000","last_change":"2023-09-23T21:13:30.772451+0000","last_active":"2023-09-23T21:19:05.873022+0000","last_peered":"2023-09-23T21:19:05.873022+0000","last_clean":"2023-09-23T21:19:05.873022+0000","last_became_active":"2023-09-23T21:13:30.772040+0000","last_became_peered":"2023-09-23T21:13:30.772040+0000","last_unstale":"2023-09-23T21:19:05.873022+0000","last_undegraded":"2023-09-23T21:19:05.873022+0000","last_fullsized":"2023-09-23T21:19:05.873022+0000","mapping_epoch":501,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:24:49.246727+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,0],"acting":[4,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.1","version":"634'8634","reported_seq":8765,"reported_epoch":634,"state":"active+clean","last_fresh":"2023-09-23T21:19:03.047382+0000","last_change":"2023-09-23T21:18:35.452919+0000","last_active":"2023-09-23T21:19:03.047382+0000","last_peered":"2023-09-23T21:19:03.047382+0000","last_clean":"2023-09-23T21:19:03.047382+0000","last_became_active":"2023-09-23T21:16:32.905672+0000","last_became_peered":"2023-09-23T21:16:32.905672+0000","last_unstale":"2023-09-23T21:19:03.047382+0000","last_undegraded":"2023-09-23T21:19:03.047382+0000","last_fullsized":"2023-09-23T21:19:03.047382+0000","mapping_epoch":544,"log_start":"558'4551","ondisk_log_start":"558'4551","created":544,"last_epoch_clean":560,"parent":"0.0","parent_split_bits":0,"last_scrub":"614'8079","last_scrub_stamp":"2023-09-23T21:18:35.452658+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_clean_scrub_stamp":"2023-09-23T21:18:35.452658+0000","objects_scrubbed":563,"log_size":4083,"log_dups_size":0,"ondisk_log_size":4083,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:19:47.092568+0000","scrub_duration":0.23111627100000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":563,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,2],"acting":[3,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.b","version":"79'161","reported_seq":653,"reported_epoch":635,"state":"active+clean","last_fresh":"2023-09-23T21:19:04.066622+0000","last_change":"2023-09-23T21:15:36.340009+0000","last_active":"2023-09-23T21:19:04.066622+0000","last_peered":"2023-09-23T21:19:04.066622+0000","last_clean":"2023-09-23T21:19:04.066622+0000","last_became_active":"2023-09-23T21:13:13.372328+0000","last_became_peered":"2023-09-23T21:13:13.372328+0000","last_unstale":"2023-09-23T21:19:04.066622+0000","last_undegraded":"2023-09-23T21:19:04.066622+0000","last_fullsized":"2023-09-23T21:19:04.066622+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":495,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:15:36.339885+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:15:36.339885+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:01:26.605006+0000","scrub_duration":0.0068473279999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":215,"reported_epoch":637,"state":"active+clean","last_fresh":"2023-09-23T21:19:05.872354+0000","last_change":"2023-09-23T21:16:42.425103+0000","last_active":"2023-09-23T21:19:05.872354+0000","last_peered":"2023-09-23T21:19:05.872354+0000","last_clean":"2023-09-23T21:19:05.872354+0000","last_became_active":"2023-09-23T21:16:42.424543+0000","last_became_peered":"2023-09-23T21:16:42.424543+0000","last_unstale":"2023-09-23T21:19:05.872354+0000","last_undegraded":"2023-09-23T21:19:05.872354+0000","last_fullsized":"2023-09-23T21:19:05.872354+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:11:28.885246+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":569,"reported_epoch":637,"state":"active+clean","last_fresh":"2023-09-23T21:19:05.873271+0000","last_change":"2023-09-23T21:12:10.811698+0000","last_active":"2023-09-23T21:19:05.873271+0000","last_peered":"2023-09-23T21:19:05.873271+0000","last_clean":"2023-09-23T21:19:05.873271+0000","last_became_active":"2023-09-23T21:12:10.789656+0000","last_became_peered":"2023-09-23T21:12:10.789656+0000","last_unstale":"2023-09-23T21:19:05.873271+0000","last_undegraded":"2023-09-23T21:19:05.873271+0000","last_fullsized":"2023-09-23T21:19:05.873271+0000","mapping_epoch":470,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:39:15.112091+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,5],"acting":[4,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.0","version":"634'8290","reported_seq":10034,"reported_epoch":634,"state":"active+clean","last_fresh":"2023-09-23T21:19:03.046613+0000","last_change":"2023-09-23T21:18:10.533031+0000","last_active":"2023-09-23T21:19:03.046613+0000","last_peered":"2023-09-23T21:19:03.046613+0000","last_clean":"2023-09-23T21:19:03.046613+0000","last_became_active":"2023-09-23T21:18:10.519762+0000","last_became_peered":"2023-09-23T21:18:10.519762+0000","last_unstale":"2023-09-23T21:19:03.046613+0000","last_undegraded":"2023-09-23T21:19:03.046613+0000","last_fullsized":"2023-09-23T21:19:03.046613+0000","mapping_epoch":598,"log_start":"556'4055","ondisk_log_start":"556'4055","created":544,"last_epoch_clean":599,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_clean_scrub_stamp":"2023-09-23T21:15:24.860968+0000","objects_scrubbed":0,"log_size":4235,"log_dups_size":0,"ondisk_log_size":4235,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":425984,"num_objects":6,"num_object_clones":0,"num_object_copies":18,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6032,"num_write_kb":193248,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":803,"num_bytes_recovered":42336256,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.a","version":"79'161","reported_seq":797,"reported_epoch":636,"state":"active+clean","last_fresh":"2023-09-23T21:19:04.868991+0000","last_change":"2023-09-23T21:15:46.420777+0000","last_active":"2023-09-23T21:19:04.868991+0000","last_peered":"2023-09-23T21:19:04.868991+0000","last_clean":"2023-09-23T21:19:04.868991+0000","last_became_active":"2023-09-23T21:12:30.437427+0000","last_became_peered":"2023-09-23T21:12:30.437427+0000","last_unstale":"2023-09-23T21:19:04.868991+0000","last_undegraded":"2023-09-23T21:19:04.868991+0000","last_fullsized":"2023-09-23T21:19:04.868991+0000","mapping_epoch":446,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:15:46.420650+0000","last_deep_scrub":"79'161","last_deep_scrub_stamp":"2023-09-23T21:05:32.630292+0000","last_clean_scrub_stamp":"2023-09-23T21:15:46.420650+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:50:23.452475+0000","scrub_duration":0.0072898930000000004,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,4],"acting":[2,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":501,"reported_epoch":634,"state":"active+clean","last_fresh":"2023-09-23T21:18:56.915657+0000","last_change":"2023-09-23T21:16:42.447403+0000","last_active":"2023-09-23T21:18:56.915657+0000","last_peered":"2023-09-23T21:18:56.915657+0000","last_clean":"2023-09-23T21:18:56.915657+0000","last_became_active":"2023-09-23T21:16:42.447080+0000","last_became_peered":"2023-09-23T21:16:42.447080+0000","last_unstale":"2023-09-23T21:18:56.915657+0000","last_undegraded":"2023-09-23T21:18:56.915657+0000","last_fullsized":"2023-09-23T21:18:56.915657+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:15:19.446334+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,4],"acting":[3,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":633,"reported_epoch":634,"state":"active+clean","last_fresh":"2023-09-23T21:18:59.426256+0000","last_change":"2023-09-23T21:16:42.518739+0000","last_active":"2023-09-23T21:18:59.426256+0000","last_peered":"2023-09-23T21:18:59.426256+0000","last_clean":"2023-09-23T21:18:59.426256+0000","last_became_active":"2023-09-23T21:16:42.517945+0000","last_became_peered":"2023-09-23T21:16:42.517945+0000","last_unstale":"2023-09-23T21:18:59.426256+0000","last_undegraded":"2023-09-23T21:18:59.426256+0000","last_fullsized":"2023-09-23T21:18:59.426256+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:47:26.236933+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.d","version":"634'12592","reported_seq":12696,"reported_epoch":636,"state":"active+clean","last_fresh":"2023-09-23T21:19:04.869438+0000","last_change":"2023-09-23T21:15:25.892834+0000","last_active":"2023-09-23T21:19:04.869438+0000","last_peered":"2023-09-23T21:19:04.869438+0000","last_clean":"2023-09-23T21:19:04.869438+0000","last_became_active":"2023-09-23T21:15:25.892519+0000","last_became_peered":"2023-09-23T21:15:25.892519+0000","last_unstale":"2023-09-23T21:19:04.869438+0000","last_undegraded":"2023-09-23T21:19:04.869438+0000","last_fullsized":"2023-09-23T21:19:04.869438+0000","mapping_epoch":544,"log_start":"573'6790","ondisk_log_start":"573'6790","created":544,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_clean_scrub_stamp":"2023-09-23T21:15:24.860968+0000","objects_scrubbed":0,"log_size":5802,"log_dups_size":0,"ondisk_log_size":5802,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:46:28.375665+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":12592,"num_write_kb":402881,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,5],"acting":[2,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.7","version":"79'161","reported_seq":775,"reported_epoch":634,"state":"active+clean","last_fresh":"2023-09-23T21:18:56.916274+0000","last_change":"2023-09-23T21:16:45.425645+0000","last_active":"2023-09-23T21:18:56.916274+0000","last_peered":"2023-09-23T21:18:56.916274+0000","last_clean":"2023-09-23T21:18:56.916274+0000","last_became_active":"2023-09-23T21:16:45.425241+0000","last_became_peered":"2023-09-23T21:16:45.425241+0000","last_unstale":"2023-09-23T21:18:56.916274+0000","last_undegraded":"2023-09-23T21:18:56.916274+0000","last_fullsized":"2023-09-23T21:18:56.916274+0000","mapping_epoch":567,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":568,"parent":"0.0","parent_split_bits":3,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:15:51.016427+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:15:51.016427+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:49:31.176462+0000","scrub_duration":0.0017648640000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":585,"reported_epoch":635,"state":"active+clean","last_fresh":"2023-09-23T21:19:04.066793+0000","last_change":"2023-09-23T21:16:42.518826+0000","last_active":"2023-09-23T21:19:04.066793+0000","last_peered":"2023-09-23T21:19:04.066793+0000","last_clean":"2023-09-23T21:19:04.066793+0000","last_became_active":"2023-09-23T21:16:42.518488+0000","last_became_peered":"2023-09-23T21:16:42.518488+0000","last_unstale":"2023-09-23T21:19:04.066793+0000","last_undegraded":"2023-09-23T21:19:04.066793+0000","last_fullsized":"2023-09-23T21:19:04.066793+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:42:44.144383+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,2],"acting":[0,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":315,"reported_epoch":637,"state":"active+clean","last_fresh":"2023-09-23T21:19:06.195478+0000","last_change":"2023-09-23T21:05:04.326889+0000","last_active":"2023-09-23T21:19:06.195478+0000","last_peered":"2023-09-23T21:19:06.195478+0000","last_clean":"2023-09-23T21:19:06.195478+0000","last_became_active":"2023-09-23T21:05:04.326032+0000","last_became_peered":"2023-09-23T21:05:04.326032+0000","last_unstale":"2023-09-23T21:19:06.195478+0000","last_undegraded":"2023-09-23T21:19:06.195478+0000","last_fullsized":"2023-09-23T21:19:06.195478+0000","mapping_epoch":332,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:00:17.507889+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":647,"reported_epoch":637,"state":"active+clean","last_fresh":"2023-09-23T21:19:06.194987+0000","last_change":"2023-09-23T21:13:13.285608+0000","last_active":"2023-09-23T21:19:06.194987+0000","last_peered":"2023-09-23T21:19:06.194987+0000","last_clean":"2023-09-23T21:19:06.194987+0000","last_became_active":"2023-09-23T21:13:13.285255+0000","last_became_peered":"2023-09-23T21:13:13.285255+0000","last_unstale":"2023-09-23T21:19:06.194987+0000","last_undegraded":"2023-09-23T21:19:06.194987+0000","last_fullsized":"2023-09-23T21:19:06.194987+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":495,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:42:30.087599+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.b","version":"634'11950","reported_seq":12051,"reported_epoch":634,"state":"active+clean","last_fresh":"2023-09-23T21:19:03.034153+0000","last_change":"2023-09-23T21:15:25.906791+0000","last_active":"2023-09-23T21:19:03.034153+0000","last_peered":"2023-09-23T21:19:03.034153+0000","last_clean":"2023-09-23T21:19:03.034153+0000","last_became_active":"2023-09-23T21:15:25.906512+0000","last_became_peered":"2023-09-23T21:15:25.906512+0000","last_unstale":"2023-09-23T21:19:03.034153+0000","last_undegraded":"2023-09-23T21:19:03.034153+0000","last_fullsized":"2023-09-23T21:19:03.034153+0000","mapping_epoch":544,"log_start":"583'7945","ondisk_log_start":"583'7945","created":544,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_clean_scrub_stamp":"2023-09-23T21:15:24.860968+0000","objects_scrubbed":0,"log_size":4005,"log_dups_size":0,"ondisk_log_size":4005,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:58:51.749775+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11950,"num_write_kb":382400,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1","version":"79'161","reported_seq":720,"reported_epoch":635,"state":"active+clean","last_fresh":"2023-09-23T21:19:04.066770+0000","last_change":"2023-09-23T21:15:34.345290+0000","last_active":"2023-09-23T21:19:04.066770+0000","last_peered":"2023-09-23T21:19:04.066770+0000","last_clean":"2023-09-23T21:19:04.066770+0000","last_became_active":"2023-09-23T21:13:12.203380+0000","last_became_peered":"2023-09-23T21:13:12.203380+0000","last_unstale":"2023-09-23T21:19:04.066770+0000","last_undegraded":"2023-09-23T21:19:04.066770+0000","last_fullsized":"2023-09-23T21:19:04.066770+0000","mapping_epoch":493,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:15:34.345138+0000","last_deep_scrub":"79'161","last_deep_scrub_stamp":"2023-09-23T21:05:30.652189+0000","last_clean_scrub_stamp":"2023-09-23T21:15:34.345138+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:47:50.349409+0000","scrub_duration":0.0057520000000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":304,"reported_epoch":635,"state":"active+clean","last_fresh":"2023-09-23T21:19:04.066719+0000","last_change":"2023-09-23T21:13:13.238617+0000","last_active":"2023-09-23T21:19:04.066719+0000","last_peered":"2023-09-23T21:19:04.066719+0000","last_clean":"2023-09-23T21:19:04.066719+0000","last_became_active":"2023-09-23T21:13:13.232926+0000","last_became_peered":"2023-09-23T21:13:13.232926+0000","last_unstale":"2023-09-23T21:19:04.066719+0000","last_undegraded":"2023-09-23T21:19:04.066719+0000","last_fullsized":"2023-09-23T21:19:04.066719+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":495,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:17:21.823772+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":571,"reported_epoch":637,"state":"active+clean","last_fresh":"2023-09-23T21:19:05.872999+0000","last_change":"2023-09-23T21:13:14.144918+0000","last_active":"2023-09-23T21:19:05.872999+0000","last_peered":"2023-09-23T21:19:05.872999+0000","last_clean":"2023-09-23T21:19:05.872999+0000","last_became_active":"2023-09-23T21:13:14.132388+0000","last_became_peered":"2023-09-23T21:13:14.132388+0000","last_unstale":"2023-09-23T21:19:05.872999+0000","last_undegraded":"2023-09-23T21:19:05.872999+0000","last_fullsized":"2023-09-23T21:19:05.872999+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":495,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:56:02.619905+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,5],"acting":[4,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":477,"reported_epoch":636,"state":"active+clean","last_fresh":"2023-09-23T21:19:04.869435+0000","last_change":"2023-09-23T21:16:42.491111+0000","last_active":"2023-09-23T21:19:04.869435+0000","last_peered":"2023-09-23T21:19:04.869435+0000","last_clean":"2023-09-23T21:19:04.869435+0000","last_became_active":"2023-09-23T21:16:42.490275+0000","last_became_peered":"2023-09-23T21:16:42.490275+0000","last_unstale":"2023-09-23T21:19:04.869435+0000","last_undegraded":"2023-09-23T21:19:04.869435+0000","last_fullsized":"2023-09-23T21:19:04.869435+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:39:11.652564+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,7],"acting":[2,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"11.8","version":"634'8376","reported_seq":8512,"reported_epoch":635,"state":"active+clean","last_fresh":"2023-09-23T21:19:04.066111+0000","last_change":"2023-09-23T21:18:30.671221+0000","last_active":"2023-09-23T21:19:04.066111+0000","last_peered":"2023-09-23T21:19:04.066111+0000","last_clean":"2023-09-23T21:19:04.066111+0000","last_became_active":"2023-09-23T21:16:32.894560+0000","last_became_peered":"2023-09-23T21:16:32.894560+0000","last_unstale":"2023-09-23T21:19:04.066111+0000","last_undegraded":"2023-09-23T21:19:04.066111+0000","last_fullsized":"2023-09-23T21:19:04.066111+0000","mapping_epoch":544,"log_start":"556'3995","ondisk_log_start":"556'3995","created":544,"last_epoch_clean":560,"parent":"0.0","parent_split_bits":0,"last_scrub":"609'7757","last_scrub_stamp":"2023-09-23T21:18:30.670962+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_clean_scrub_stamp":"2023-09-23T21:18:30.670962+0000","objects_scrubbed":619,"log_size":4381,"log_dups_size":0,"ondisk_log_size":4381,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:13:50.200821+0000","scrub_duration":0.101488842,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":619,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,5],"acting":[0,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.2","version":"79'161","reported_seq":951,"reported_epoch":635,"state":"active+clean","last_fresh":"2023-09-23T21:19:04.066421+0000","last_change":"2023-09-23T21:16:45.417075+0000","last_active":"2023-09-23T21:19:04.066421+0000","last_peered":"2023-09-23T21:19:04.066421+0000","last_clean":"2023-09-23T21:19:04.066421+0000","last_became_active":"2023-09-23T21:16:45.416752+0000","last_became_peered":"2023-09-23T21:16:45.416752+0000","last_unstale":"2023-09-23T21:19:04.066421+0000","last_undegraded":"2023-09-23T21:19:04.066421+0000","last_fullsized":"2023-09-23T21:19:04.066421+0000","mapping_epoch":567,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":568,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:24.802451+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:24.802451+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:01:51.129779+0000","scrub_duration":0.0090861250000000005,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,4],"acting":[0,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":647,"reported_epoch":636,"state":"active+clean","last_fresh":"2023-09-23T21:19:04.868916+0000","last_change":"2023-09-23T21:13:12.206980+0000","last_active":"2023-09-23T21:19:04.868916+0000","last_peered":"2023-09-23T21:19:04.868916+0000","last_clean":"2023-09-23T21:19:04.868916+0000","last_became_active":"2023-09-23T21:13:12.206650+0000","last_became_peered":"2023-09-23T21:13:12.206650+0000","last_unstale":"2023-09-23T21:19:04.868916+0000","last_undegraded":"2023-09-23T21:19:04.868916+0000","last_fullsized":"2023-09-23T21:19:04.868916+0000","mapping_epoch":493,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:10:01.892936+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":631,"reported_epoch":636,"state":"active+clean","last_fresh":"2023-09-23T21:19:04.868843+0000","last_change":"2023-09-23T21:13:13.407073+0000","last_active":"2023-09-23T21:19:04.868843+0000","last_peered":"2023-09-23T21:19:04.868843+0000","last_clean":"2023-09-23T21:19:04.868843+0000","last_became_active":"2023-09-23T21:13:13.395047+0000","last_became_peered":"2023-09-23T21:13:13.395047+0000","last_unstale":"2023-09-23T21:19:04.868843+0000","last_undegraded":"2023-09-23T21:19:04.868843+0000","last_fullsized":"2023-09-23T21:19:04.868843+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":495,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:04:43.384190+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,5],"acting":[2,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":165,"reported_epoch":635,"state":"active+clean","last_fresh":"2023-09-23T21:19:04.067117+0000","last_change":"2023-09-23T21:16:42.517729+0000","last_active":"2023-09-23T21:19:04.067117+0000","last_peered":"2023-09-23T21:19:04.067117+0000","last_clean":"2023-09-23T21:19:04.067117+0000","last_became_active":"2023-09-23T21:16:42.517373+0000","last_became_peered":"2023-09-23T21:16:42.517373+0000","last_unstale":"2023-09-23T21:19:04.067117+0000","last_undegraded":"2023-09-23T21:19:04.067117+0000","last_fullsized":"2023-09-23T21:19:04.067117+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:05:57.370550+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.a","version":"634'12354","reported_seq":12457,"reported_epoch":636,"state":"active+clean","last_fresh":"2023-09-23T21:19:04.868794+0000","last_change":"2023-09-23T21:15:25.893846+0000","last_active":"2023-09-23T21:19:04.868794+0000","last_peered":"2023-09-23T21:19:04.868794+0000","last_clean":"2023-09-23T21:19:04.868794+0000","last_became_active":"2023-09-23T21:15:25.893452+0000","last_became_peered":"2023-09-23T21:15:25.893452+0000","last_unstale":"2023-09-23T21:19:04.868794+0000","last_undegraded":"2023-09-23T21:19:04.868794+0000","last_fullsized":"2023-09-23T21:19:04.868794+0000","mapping_epoch":544,"log_start":"573'6494","ondisk_log_start":"573'6494","created":544,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_clean_scrub_stamp":"2023-09-23T21:15:24.860968+0000","objects_scrubbed":0,"log_size":5860,"log_dups_size":0,"ondisk_log_size":5860,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:42:32.842297+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12354,"num_write_kb":395328,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,5],"acting":[2,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":442,"reported_epoch":636,"state":"active+clean","last_fresh":"2023-09-23T21:19:04.869102+0000","last_change":"2023-09-23T21:16:42.423216+0000","last_active":"2023-09-23T21:19:04.869102+0000","last_peered":"2023-09-23T21:19:04.869102+0000","last_clean":"2023-09-23T21:19:04.869102+0000","last_became_active":"2023-09-23T21:16:42.422856+0000","last_became_peered":"2023-09-23T21:16:42.422856+0000","last_unstale":"2023-09-23T21:19:04.869102+0000","last_undegraded":"2023-09-23T21:19:04.869102+0000","last_fullsized":"2023-09-23T21:19:04.869102+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:06:54.105798+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"79'161","reported_seq":854,"reported_epoch":635,"state":"active+clean","last_fresh":"2023-09-23T21:19:04.066235+0000","last_change":"2023-09-23T21:16:45.428032+0000","last_active":"2023-09-23T21:19:04.066235+0000","last_peered":"2023-09-23T21:19:04.066235+0000","last_clean":"2023-09-23T21:19:04.066235+0000","last_became_active":"2023-09-23T21:16:45.427674+0000","last_became_peered":"2023-09-23T21:16:45.427674+0000","last_unstale":"2023-09-23T21:19:04.066235+0000","last_undegraded":"2023-09-23T21:19:04.066235+0000","last_fullsized":"2023-09-23T21:19:04.066235+0000","mapping_epoch":567,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":568,"parent":"0.0","parent_split_bits":0,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:20.738273+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:20.738273+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:39:42.441711+0000","scrub_duration":0.0088336339999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,2],"acting":[0,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":645,"reported_epoch":637,"state":"active+clean","last_fresh":"2023-09-23T21:19:06.194973+0000","last_change":"2023-09-23T21:11:08.760417+0000","last_active":"2023-09-23T21:19:06.194973+0000","last_peered":"2023-09-23T21:19:06.194973+0000","last_clean":"2023-09-23T21:19:06.194973+0000","last_became_active":"2023-09-23T21:11:08.760150+0000","last_became_peered":"2023-09-23T21:11:08.760150+0000","last_unstale":"2023-09-23T21:19:06.194973+0000","last_undegraded":"2023-09-23T21:19:06.194973+0000","last_fullsized":"2023-09-23T21:19:06.194973+0000","mapping_epoch":443,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":444,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:23:10.849768+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":232,"reported_epoch":637,"state":"active+clean","last_fresh":"2023-09-23T21:19:05.873145+0000","last_change":"2023-09-23T21:16:42.424717+0000","last_active":"2023-09-23T21:19:05.873145+0000","last_peered":"2023-09-23T21:19:05.873145+0000","last_clean":"2023-09-23T21:19:05.873145+0000","last_became_active":"2023-09-23T21:16:42.424376+0000","last_became_peered":"2023-09-23T21:16:42.424376+0000","last_unstale":"2023-09-23T21:19:05.873145+0000","last_undegraded":"2023-09-23T21:19:05.873145+0000","last_fullsized":"2023-09-23T21:19:05.873145+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:25:14.162321+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.9","version":"634'8551","reported_seq":10411,"reported_epoch":634,"state":"active+clean","last_fresh":"2023-09-23T21:19:02.962621+0000","last_change":"2023-09-23T21:17:58.411362+0000","last_active":"2023-09-23T21:19:02.962621+0000","last_peered":"2023-09-23T21:19:02.962621+0000","last_clean":"2023-09-23T21:19:02.962621+0000","last_became_active":"2023-09-23T21:17:58.399421+0000","last_became_peered":"2023-09-23T21:17:58.399421+0000","last_unstale":"2023-09-23T21:19:02.962621+0000","last_undegraded":"2023-09-23T21:19:02.962621+0000","last_fullsized":"2023-09-23T21:19:02.962621+0000","mapping_epoch":593,"log_start":"558'4462","ondisk_log_start":"558'4462","created":544,"last_epoch_clean":594,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_clean_scrub_stamp":"2023-09-23T21:15:24.860968+0000","objects_scrubbed":0,"log_size":4089,"log_dups_size":0,"ondisk_log_size":4089,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":884736,"num_objects":13,"num_object_clones":0,"num_object_copies":39,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":13,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6267,"num_write_kb":200992,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":860,"num_bytes_recovered":56393728,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,7],"acting":[3,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.3","version":"79'161","reported_seq":940,"reported_epoch":637,"state":"active+clean","last_fresh":"2023-09-23T21:19:05.872456+0000","last_change":"2023-09-23T21:13:33.771279+0000","last_active":"2023-09-23T21:19:05.872456+0000","last_peered":"2023-09-23T21:19:05.872456+0000","last_clean":"2023-09-23T21:19:05.872456+0000","last_became_active":"2023-09-23T21:13:33.770980+0000","last_became_peered":"2023-09-23T21:13:33.770980+0000","last_unstale":"2023-09-23T21:19:05.872456+0000","last_undegraded":"2023-09-23T21:19:05.872456+0000","last_fullsized":"2023-09-23T21:19:05.872456+0000","mapping_epoch":504,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":505,"parent":"0.0","parent_split_bits":3,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:05:57.832302+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:05:57.832302+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:19:42.309604+0000","scrub_duration":0.010206847999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":472,"reported_epoch":635,"state":"active+clean","last_fresh":"2023-09-23T21:19:04.066521+0000","last_change":"2023-09-23T21:16:42.455365+0000","last_active":"2023-09-23T21:19:04.066521+0000","last_peered":"2023-09-23T21:19:04.066521+0000","last_clean":"2023-09-23T21:19:04.066521+0000","last_became_active":"2023-09-23T21:16:42.452829+0000","last_became_peered":"2023-09-23T21:16:42.452829+0000","last_unstale":"2023-09-23T21:19:04.066521+0000","last_undegraded":"2023-09-23T21:19:04.066521+0000","last_fullsized":"2023-09-23T21:19:04.066521+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:49:26.303389+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":636,"reported_epoch":635,"state":"active+clean","last_fresh":"2023-09-23T21:19:04.066495+0000","last_change":"2023-09-23T21:13:44.409212+0000","last_active":"2023-09-23T21:19:04.066495+0000","last_peered":"2023-09-23T21:19:04.066495+0000","last_clean":"2023-09-23T21:19:04.066495+0000","last_became_active":"2023-09-23T21:13:44.408958+0000","last_became_peered":"2023-09-23T21:13:44.408958+0000","last_unstale":"2023-09-23T21:19:04.066495+0000","last_undegraded":"2023-09-23T21:19:04.066495+0000","last_fullsized":"2023-09-23T21:19:04.066495+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:03:52.749964+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":165,"reported_epoch":635,"state":"active+clean","last_fresh":"2023-09-23T21:19:04.067265+0000","last_change":"2023-09-23T21:16:42.427490+0000","last_active":"2023-09-23T21:19:04.067265+0000","last_peered":"2023-09-23T21:19:04.067265+0000","last_clean":"2023-09-23T21:19:04.067265+0000","last_became_active":"2023-09-23T21:16:42.426566+0000","last_became_peered":"2023-09-23T21:16:42.426566+0000","last_unstale":"2023-09-23T21:19:04.067265+0000","last_undegraded":"2023-09-23T21:19:04.067265+0000","last_fullsized":"2023-09-23T21:19:04.067265+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:20:26.396005+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,7],"acting":[0,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.c","version":"634'12176","reported_seq":15384,"reported_epoch":636,"state":"active+clean","last_fresh":"2023-09-23T21:19:04.869539+0000","last_change":"2023-09-23T21:17:21.358806+0000","last_active":"2023-09-23T21:19:04.869539+0000","last_peered":"2023-09-23T21:19:04.869539+0000","last_clean":"2023-09-23T21:19:04.869539+0000","last_became_active":"2023-09-23T21:17:21.346629+0000","last_became_peered":"2023-09-23T21:17:21.346629+0000","last_unstale":"2023-09-23T21:19:04.869539+0000","last_undegraded":"2023-09-23T21:19:04.869539+0000","last_fullsized":"2023-09-23T21:19:04.869539+0000","mapping_epoch":576,"log_start":"573'6354","ondisk_log_start":"573'6354","created":544,"last_epoch_clean":577,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_clean_scrub_stamp":"2023-09-23T21:15:24.860968+0000","objects_scrubbed":0,"log_size":5822,"log_dups_size":0,"ondisk_log_size":5822,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:59:48.173244+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12176,"num_write_kb":389632,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1539,"num_bytes_recovered":100859904,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,7],"acting":[2,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.6","version":"79'161","reported_seq":568,"reported_epoch":635,"state":"active+clean","last_fresh":"2023-09-23T21:19:04.067192+0000","last_change":"2023-09-23T21:15:43.333919+0000","last_active":"2023-09-23T21:19:04.067192+0000","last_peered":"2023-09-23T21:19:04.067192+0000","last_clean":"2023-09-23T21:19:04.067192+0000","last_became_active":"2023-09-23T21:13:33.769303+0000","last_became_peered":"2023-09-23T21:13:33.769303+0000","last_unstale":"2023-09-23T21:19:04.067192+0000","last_undegraded":"2023-09-23T21:19:04.067192+0000","last_fullsized":"2023-09-23T21:19:04.067192+0000","mapping_epoch":504,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":505,"parent":"0.0","parent_split_bits":3,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:15:43.333733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:15:43.333733+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:42:29.787357+0000","scrub_duration":0.0092107530000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":83,"reported_epoch":637,"state":"active+clean","last_fresh":"2023-09-23T21:19:05.887379+0000","last_change":"2023-09-23T21:16:42.483813+0000","last_active":"2023-09-23T21:19:05.887379+0000","last_peered":"2023-09-23T21:19:05.887379+0000","last_clean":"2023-09-23T21:19:05.887379+0000","last_became_active":"2023-09-23T21:16:42.482847+0000","last_became_peered":"2023-09-23T21:16:42.482847+0000","last_unstale":"2023-09-23T21:19:05.887379+0000","last_undegraded":"2023-09-23T21:19:05.887379+0000","last_fullsized":"2023-09-23T21:19:05.887379+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:38:57.905679+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":522,"reported_epoch":636,"state":"active+clean","last_fresh":"2023-09-23T21:19:04.869650+0000","last_change":"2023-09-23T21:13:30.790590+0000","last_active":"2023-09-23T21:19:04.869650+0000","last_peered":"2023-09-23T21:19:04.869650+0000","last_clean":"2023-09-23T21:19:04.869650+0000","last_became_active":"2023-09-23T21:13:30.790253+0000","last_became_peered":"2023-09-23T21:13:30.790253+0000","last_unstale":"2023-09-23T21:19:04.869650+0000","last_undegraded":"2023-09-23T21:19:04.869650+0000","last_fullsized":"2023-09-23T21:19:04.869650+0000","mapping_epoch":501,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:23:41.843250+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,0],"acting":[2,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":601,"reported_epoch":635,"state":"active+clean","last_fresh":"2023-09-23T21:19:04.067220+0000","last_change":"2023-09-23T21:16:42.428535+0000","last_active":"2023-09-23T21:19:04.067220+0000","last_peered":"2023-09-23T21:19:04.067220+0000","last_clean":"2023-09-23T21:19:04.067220+0000","last_became_active":"2023-09-23T21:16:42.428149+0000","last_became_peered":"2023-09-23T21:16:42.428149+0000","last_unstale":"2023-09-23T21:19:04.067220+0000","last_undegraded":"2023-09-23T21:19:04.067220+0000","last_fullsized":"2023-09-23T21:19:04.067220+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:38:59.979788+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.f","version":"634'12168","reported_seq":15825,"reported_epoch":636,"state":"active+clean","last_fresh":"2023-09-23T21:19:04.869572+0000","last_change":"2023-09-23T21:17:03.878062+0000","last_active":"2023-09-23T21:19:04.869572+0000","last_peered":"2023-09-23T21:19:04.869572+0000","last_clean":"2023-09-23T21:19:04.869572+0000","last_became_active":"2023-09-23T21:17:03.870998+0000","last_became_peered":"2023-09-23T21:17:03.870998+0000","last_unstale":"2023-09-23T21:19:04.869572+0000","last_undegraded":"2023-09-23T21:19:04.869572+0000","last_fullsized":"2023-09-23T21:19:04.869572+0000","mapping_epoch":572,"log_start":"573'6321","ondisk_log_start":"573'6321","created":544,"last_epoch_clean":573,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_clean_scrub_stamp":"2023-09-23T21:15:24.860968+0000","objects_scrubbed":0,"log_size":5847,"log_dups_size":0,"ondisk_log_size":5847,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:19:31.942955+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12168,"num_write_kb":389376,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1762,"num_bytes_recovered":115474432,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,0],"acting":[2,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.5","version":"79'161","reported_seq":675,"reported_epoch":637,"state":"active+clean","last_fresh":"2023-09-23T21:19:05.873142+0000","last_change":"2023-09-23T21:15:41.108638+0000","last_active":"2023-09-23T21:19:05.873142+0000","last_peered":"2023-09-23T21:19:05.873142+0000","last_clean":"2023-09-23T21:19:05.873142+0000","last_became_active":"2023-09-23T21:13:38.373633+0000","last_became_peered":"2023-09-23T21:13:38.373633+0000","last_unstale":"2023-09-23T21:19:05.873142+0000","last_undegraded":"2023-09-23T21:19:05.873142+0000","last_fullsized":"2023-09-23T21:19:05.873142+0000","mapping_epoch":507,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":3,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:15:41.108579+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:15:41.108579+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:21:58.075649+0000","scrub_duration":0.0061511960000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":204,"reported_epoch":635,"state":"active+clean","last_fresh":"2023-09-23T21:19:04.067252+0000","last_change":"2023-09-23T21:17:02.204086+0000","last_active":"2023-09-23T21:19:04.067252+0000","last_peered":"2023-09-23T21:19:04.067252+0000","last_clean":"2023-09-23T21:19:04.067252+0000","last_became_active":"2023-09-23T21:16:42.517549+0000","last_became_peered":"2023-09-23T21:16:42.517549+0000","last_unstale":"2023-09-23T21:19:04.067252+0000","last_undegraded":"2023-09-23T21:19:04.067252+0000","last_fullsized":"2023-09-23T21:19:04.067252+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:43:07.991844+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":618,"reported_epoch":637,"state":"active+clean","last_fresh":"2023-09-23T21:19:05.873183+0000","last_change":"2023-09-23T21:13:30.807520+0000","last_active":"2023-09-23T21:19:05.873183+0000","last_peered":"2023-09-23T21:19:05.873183+0000","last_clean":"2023-09-23T21:19:05.873183+0000","last_became_active":"2023-09-23T21:13:30.807222+0000","last_became_peered":"2023-09-23T21:13:30.807222+0000","last_unstale":"2023-09-23T21:19:05.873183+0000","last_undegraded":"2023-09-23T21:19:05.873183+0000","last_fullsized":"2023-09-23T21:19:05.873183+0000","mapping_epoch":501,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:40:31.989749+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":496,"reported_epoch":634,"state":"active+clean","last_fresh":"2023-09-23T21:18:56.915274+0000","last_change":"2023-09-23T21:16:42.463411+0000","last_active":"2023-09-23T21:18:56.915274+0000","last_peered":"2023-09-23T21:18:56.915274+0000","last_clean":"2023-09-23T21:18:56.915274+0000","last_became_active":"2023-09-23T21:16:42.463020+0000","last_became_peered":"2023-09-23T21:16:42.463020+0000","last_unstale":"2023-09-23T21:18:56.915274+0000","last_undegraded":"2023-09-23T21:18:56.915274+0000","last_fullsized":"2023-09-23T21:18:56.915274+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:07:34.346129+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.e","version":"634'12256","reported_seq":16492,"reported_epoch":637,"state":"active+clean","last_fresh":"2023-09-23T21:19:05.873284+0000","last_change":"2023-09-23T21:17:34.456174+0000","last_active":"2023-09-23T21:19:05.873284+0000","last_peered":"2023-09-23T21:19:05.873284+0000","last_clean":"2023-09-23T21:19:05.873284+0000","last_became_active":"2023-09-23T21:17:34.444637+0000","last_became_peered":"2023-09-23T21:17:34.444637+0000","last_unstale":"2023-09-23T21:19:05.873284+0000","last_undegraded":"2023-09-23T21:19:05.873284+0000","last_fullsized":"2023-09-23T21:19:05.873284+0000","mapping_epoch":582,"log_start":"579'7523","ondisk_log_start":"579'7523","created":544,"last_epoch_clean":583,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_clean_scrub_stamp":"2023-09-23T21:15:24.860968+0000","objects_scrubbed":0,"log_size":4733,"log_dups_size":0,"ondisk_log_size":4733,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:17:47.772535+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12256,"num_write_kb":392192,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2052,"num_bytes_recovered":134479872,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.4","version":"79'161","reported_seq":1042,"reported_epoch":635,"state":"active+clean","last_fresh":"2023-09-23T21:19:04.068330+0000","last_change":"2023-09-23T21:16:45.426136+0000","last_active":"2023-09-23T21:19:04.068330+0000","last_peered":"2023-09-23T21:19:04.068330+0000","last_clean":"2023-09-23T21:19:04.068330+0000","last_became_active":"2023-09-23T21:16:45.425669+0000","last_became_peered":"2023-09-23T21:16:45.425669+0000","last_unstale":"2023-09-23T21:19:04.068330+0000","last_undegraded":"2023-09-23T21:19:04.068330+0000","last_fullsized":"2023-09-23T21:19:04.068330+0000","mapping_epoch":567,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":568,"parent":"0.0","parent_split_bits":3,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:20.012764+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:20.012764+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:52:39.142509+0000","scrub_duration":0.0045251919999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":596,"reported_epoch":636,"state":"active+clean","last_fresh":"2023-09-23T21:19:04.869765+0000","last_change":"2023-09-23T21:16:42.459732+0000","last_active":"2023-09-23T21:19:04.869765+0000","last_peered":"2023-09-23T21:19:04.869765+0000","last_clean":"2023-09-23T21:19:04.869765+0000","last_became_active":"2023-09-23T21:16:42.459413+0000","last_became_peered":"2023-09-23T21:16:42.459413+0000","last_unstale":"2023-09-23T21:19:04.869765+0000","last_undegraded":"2023-09-23T21:19:04.869765+0000","last_fullsized":"2023-09-23T21:19:04.869765+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:37:44.548863+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,3],"acting":[2,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.6","version":"70'1","reported_seq":637,"reported_epoch":635,"state":"active+clean","last_fresh":"2023-09-23T21:19:04.067428+0000","last_change":"2023-09-23T21:17:02.208077+0000","last_active":"2023-09-23T21:19:04.067428+0000","last_peered":"2023-09-23T21:19:04.067428+0000","last_clean":"2023-09-23T21:19:04.067428+0000","last_became_active":"2023-09-23T21:16:42.431308+0000","last_became_peered":"2023-09-23T21:16:42.431308+0000","last_unstale":"2023-09-23T21:19:04.067428+0000","last_undegraded":"2023-09-23T21:19:04.067428+0000","last_fullsized":"2023-09-23T21:19:04.067428+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:16:49.009013+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":212,"reported_epoch":637,"state":"active+clean","last_fresh":"2023-09-23T21:19:05.872292+0000","last_change":"2023-09-23T21:13:12.216895+0000","last_active":"2023-09-23T21:19:05.872292+0000","last_peered":"2023-09-23T21:19:05.872292+0000","last_clean":"2023-09-23T21:19:05.872292+0000","last_became_active":"2023-09-23T21:13:12.216654+0000","last_became_peered":"2023-09-23T21:13:12.216654+0000","last_unstale":"2023-09-23T21:19:05.872292+0000","last_undegraded":"2023-09-23T21:19:05.872292+0000","last_fullsized":"2023-09-23T21:19:05.872292+0000","mapping_epoch":493,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:02:40.430096+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.13","version":"634'8188","reported_seq":11110,"reported_epoch":637,"state":"active+clean","last_fresh":"2023-09-23T21:19:05.872780+0000","last_change":"2023-09-23T21:18:47.282127+0000","last_active":"2023-09-23T21:19:05.872780+0000","last_peered":"2023-09-23T21:19:05.872780+0000","last_clean":"2023-09-23T21:19:05.872780+0000","last_became_active":"2023-09-23T21:18:42.892705+0000","last_became_peered":"2023-09-23T21:18:42.892705+0000","last_unstale":"2023-09-23T21:19:05.872780+0000","last_undegraded":"2023-09-23T21:19:05.872780+0000","last_fullsized":"2023-09-23T21:19:05.872780+0000","mapping_epoch":620,"log_start":"549'2572","ondisk_log_start":"549'2572","created":559,"last_epoch_clean":621,"parent":"0.0","parent_split_bits":5,"last_scrub":"624'7886","last_scrub_stamp":"2023-09-23T21:18:47.282029+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_clean_scrub_stamp":"2023-09-23T21:18:47.282029+0000","objects_scrubbed":302,"log_size":5616,"log_dups_size":0,"ondisk_log_size":5616,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:49:10.428430+0000","scrub_duration":0.049862798999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":302,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.19","version":"79'161","reported_seq":844,"reported_epoch":635,"state":"active+clean","last_fresh":"2023-09-23T21:19:04.066111+0000","last_change":"2023-09-23T21:16:45.428879+0000","last_active":"2023-09-23T21:19:04.066111+0000","last_peered":"2023-09-23T21:19:04.066111+0000","last_clean":"2023-09-23T21:19:04.066111+0000","last_became_active":"2023-09-23T21:16:45.428660+0000","last_became_peered":"2023-09-23T21:16:45.428660+0000","last_unstale":"2023-09-23T21:19:04.066111+0000","last_undegraded":"2023-09-23T21:19:04.066111+0000","last_fullsized":"2023-09-23T21:19:04.066111+0000","mapping_epoch":567,"log_start":"0'0","ondisk_log_start":"0'0","created":479,"last_epoch_clean":568,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:15:48.271791+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:15:48.271791+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:25:26.361045+0000","scrub_duration":0.0057149180000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":165,"reported_epoch":635,"state":"active+clean","last_fresh":"2023-09-23T21:19:04.067153+0000","last_change":"2023-09-23T21:16:42.434830+0000","last_active":"2023-09-23T21:19:04.067153+0000","last_peered":"2023-09-23T21:19:04.067153+0000","last_clean":"2023-09-23T21:19:04.067153+0000","last_became_active":"2023-09-23T21:16:42.430289+0000","last_became_peered":"2023-09-23T21:16:42.430289+0000","last_unstale":"2023-09-23T21:19:04.067153+0000","last_undegraded":"2023-09-23T21:19:04.067153+0000","last_fullsized":"2023-09-23T21:19:04.067153+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:38:48.053766+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":142,"reported_epoch":636,"state":"active+clean","last_fresh":"2023-09-23T21:19:04.869614+0000","last_change":"2023-09-23T21:13:38.373814+0000","last_active":"2023-09-23T21:19:04.869614+0000","last_peered":"2023-09-23T21:19:04.869614+0000","last_clean":"2023-09-23T21:19:04.869614+0000","last_became_active":"2023-09-23T21:13:38.373458+0000","last_became_peered":"2023-09-23T21:13:38.373458+0000","last_unstale":"2023-09-23T21:19:04.869614+0000","last_undegraded":"2023-09-23T21:19:04.869614+0000","last_fullsized":"2023-09-23T21:19:04.869614+0000","mapping_epoch":507,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:23:08.288401+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"11.14","version":"634'8304","reported_seq":11257,"reported_epoch":637,"state":"active+clean","last_fresh":"2023-09-23T21:19:05.872887+0000","last_change":"2023-09-23T21:18:44.928597+0000","last_active":"2023-09-23T21:19:05.872887+0000","last_peered":"2023-09-23T21:19:05.872887+0000","last_clean":"2023-09-23T21:19:05.872887+0000","last_became_active":"2023-09-23T21:18:44.896863+0000","last_became_peered":"2023-09-23T21:18:44.896863+0000","last_unstale":"2023-09-23T21:19:05.872887+0000","last_undegraded":"2023-09-23T21:19:05.872887+0000","last_fullsized":"2023-09-23T21:19:05.872887+0000","mapping_epoch":622,"log_start":"550'2817","ondisk_log_start":"550'2817","created":559,"last_epoch_clean":623,"parent":"0.0","parent_split_bits":5,"last_scrub":"621'7919","last_scrub_stamp":"2023-09-23T21:18:43.079722+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_clean_scrub_stamp":"2023-09-23T21:18:43.079722+0000","objects_scrubbed":386,"log_size":5487,"log_dups_size":0,"ondisk_log_size":5487,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:44:42.637399+0000","scrub_duration":0.092516535999999996,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":386,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1e","version":"79'161","reported_seq":618,"reported_epoch":634,"state":"active+clean","last_fresh":"2023-09-23T21:18:56.916165+0000","last_change":"2023-09-23T21:16:45.420788+0000","last_active":"2023-09-23T21:18:56.916165+0000","last_peered":"2023-09-23T21:18:56.916165+0000","last_clean":"2023-09-23T21:18:56.916165+0000","last_became_active":"2023-09-23T21:16:45.420448+0000","last_became_peered":"2023-09-23T21:16:45.420448+0000","last_unstale":"2023-09-23T21:18:56.916165+0000","last_undegraded":"2023-09-23T21:18:56.916165+0000","last_fullsized":"2023-09-23T21:18:56.916165+0000","mapping_epoch":567,"log_start":"0'0","ondisk_log_start":"0'0","created":479,"last_epoch_clean":568,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:15:53.061319+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:15:53.061319+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:46:31.982439+0000","scrub_duration":0.007429878,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,4],"acting":[3,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":247,"reported_epoch":637,"state":"active+clean","last_fresh":"2023-09-23T21:19:05.872815+0000","last_change":"2023-09-23T21:16:42.518584+0000","last_active":"2023-09-23T21:19:05.872815+0000","last_peered":"2023-09-23T21:19:05.872815+0000","last_clean":"2023-09-23T21:19:05.872815+0000","last_became_active":"2023-09-23T21:16:42.416632+0000","last_became_peered":"2023-09-23T21:16:42.416632+0000","last_unstale":"2023-09-23T21:19:05.872815+0000","last_undegraded":"2023-09-23T21:19:05.872815+0000","last_fullsized":"2023-09-23T21:19:05.872815+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:02:18.972080+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,7],"acting":[4,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":202,"reported_epoch":637,"state":"active+clean","last_fresh":"2023-09-23T21:19:05.872663+0000","last_change":"2023-09-23T21:12:10.770474+0000","last_active":"2023-09-23T21:19:05.872663+0000","last_peered":"2023-09-23T21:19:05.872663+0000","last_clean":"2023-09-23T21:19:05.872663+0000","last_became_active":"2023-09-23T21:12:10.770184+0000","last_became_peered":"2023-09-23T21:12:10.770184+0000","last_unstale":"2023-09-23T21:19:05.872663+0000","last_undegraded":"2023-09-23T21:19:05.872663+0000","last_fullsized":"2023-09-23T21:19:05.872663+0000","mapping_epoch":470,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:05:09.407151+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.15","version":"634'8511","reported_seq":14953,"reported_epoch":636,"state":"active+clean","last_fresh":"2023-09-23T21:19:04.869590+0000","last_change":"2023-09-23T21:18:47.931483+0000","last_active":"2023-09-23T21:19:04.869590+0000","last_peered":"2023-09-23T21:19:04.869590+0000","last_clean":"2023-09-23T21:19:04.869590+0000","last_became_active":"2023-09-23T21:18:47.898051+0000","last_became_peered":"2023-09-23T21:18:47.898051+0000","last_unstale":"2023-09-23T21:19:04.869590+0000","last_undegraded":"2023-09-23T21:19:04.869590+0000","last_fullsized":"2023-09-23T21:19:04.869590+0000","mapping_epoch":624,"log_start":"556'4217","ondisk_log_start":"556'4217","created":559,"last_epoch_clean":625,"parent":"0.0","parent_split_bits":5,"last_scrub":"610'7908","last_scrub_stamp":"2023-09-23T21:18:31.392582+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_clean_scrub_stamp":"2023-09-23T21:18:31.392582+0000","objects_scrubbed":603,"log_size":4294,"log_dups_size":0,"ondisk_log_size":4294,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:31:46.220106+0000","scrub_duration":0.067034226000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":603,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":369,"num_bytes_recovered":24182784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,5],"acting":[2,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":428,"reported_epoch":636,"state":"active+clean","last_fresh":"2023-09-23T21:19:04.869626+0000","last_change":"2023-09-23T21:12:10.771160+0000","last_active":"2023-09-23T21:19:04.869626+0000","last_peered":"2023-09-23T21:19:04.869626+0000","last_clean":"2023-09-23T21:19:04.869626+0000","last_became_active":"2023-09-23T21:12:10.757808+0000","last_became_peered":"2023-09-23T21:12:10.757808+0000","last_unstale":"2023-09-23T21:19:04.869626+0000","last_undegraded":"2023-09-23T21:19:04.869626+0000","last_fullsized":"2023-09-23T21:19:04.869626+0000","mapping_epoch":470,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:30:50.987139+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":212,"reported_epoch":637,"state":"active+clean","last_fresh":"2023-09-23T21:19:05.872469+0000","last_change":"2023-09-23T21:13:12.216602+0000","last_active":"2023-09-23T21:19:05.872469+0000","last_peered":"2023-09-23T21:19:05.872469+0000","last_clean":"2023-09-23T21:19:05.872469+0000","last_became_active":"2023-09-23T21:13:12.216204+0000","last_became_peered":"2023-09-23T21:13:12.216204+0000","last_unstale":"2023-09-23T21:19:05.872469+0000","last_undegraded":"2023-09-23T21:19:05.872469+0000","last_fullsized":"2023-09-23T21:19:05.872469+0000","mapping_epoch":493,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:23:37.043390+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.16","version":"634'8141","reported_seq":9222,"reported_epoch":637,"state":"active+clean","last_fresh":"2023-09-23T21:19:06.195227+0000","last_change":"2023-09-23T21:18:52.925245+0000","last_active":"2023-09-23T21:19:06.195227+0000","last_peered":"2023-09-23T21:19:06.195227+0000","last_clean":"2023-09-23T21:19:06.195227+0000","last_became_active":"2023-09-23T21:18:52.900539+0000","last_became_peered":"2023-09-23T21:18:52.900539+0000","last_unstale":"2023-09-23T21:19:06.195227+0000","last_undegraded":"2023-09-23T21:19:06.195227+0000","last_fullsized":"2023-09-23T21:19:06.195227+0000","mapping_epoch":629,"log_start":"550'3239","ondisk_log_start":"550'3239","created":559,"last_epoch_clean":630,"parent":"0.0","parent_split_bits":5,"last_scrub":"610'7609","last_scrub_stamp":"2023-09-23T21:18:31.602481+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_clean_scrub_stamp":"2023-09-23T21:18:31.602481+0000","objects_scrubbed":532,"log_size":4902,"log_dups_size":0,"ondisk_log_size":4902,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:45:18.428511+0000","scrub_duration":0.058905792999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":532,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":231,"num_bytes_recovered":15138816,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.1c","version":"79'161","reported_seq":1065,"reported_epoch":634,"state":"active+clean","last_fresh":"2023-09-23T21:18:56.915340+0000","last_change":"2023-09-23T21:16:45.419779+0000","last_active":"2023-09-23T21:18:56.915340+0000","last_peered":"2023-09-23T21:18:56.915340+0000","last_clean":"2023-09-23T21:18:56.915340+0000","last_became_active":"2023-09-23T21:16:45.419394+0000","last_became_peered":"2023-09-23T21:16:45.419394+0000","last_unstale":"2023-09-23T21:18:56.915340+0000","last_undegraded":"2023-09-23T21:18:56.915340+0000","last_fullsized":"2023-09-23T21:18:56.915340+0000","mapping_epoch":567,"log_start":"0'0","ondisk_log_start":"0'0","created":479,"last_epoch_clean":568,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:15:54.019828+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:15:54.019828+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:59:05.422258+0000","scrub_duration":0.0020962530000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":246,"reported_epoch":636,"state":"active+clean","last_fresh":"2023-09-23T21:19:04.869670+0000","last_change":"2023-09-23T21:16:42.424187+0000","last_active":"2023-09-23T21:19:04.869670+0000","last_peered":"2023-09-23T21:19:04.869670+0000","last_clean":"2023-09-23T21:19:04.869670+0000","last_became_active":"2023-09-23T21:16:42.423805+0000","last_became_peered":"2023-09-23T21:16:42.423805+0000","last_unstale":"2023-09-23T21:19:04.869670+0000","last_undegraded":"2023-09-23T21:19:04.869670+0000","last_fullsized":"2023-09-23T21:19:04.869670+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:55:46.051199+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,7],"acting":[2,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":306,"reported_epoch":636,"state":"active+clean","last_fresh":"2023-09-23T21:19:04.868975+0000","last_change":"2023-09-23T21:13:12.200767+0000","last_active":"2023-09-23T21:19:04.868975+0000","last_peered":"2023-09-23T21:19:04.868975+0000","last_clean":"2023-09-23T21:19:04.868975+0000","last_became_active":"2023-09-23T21:13:12.200435+0000","last_became_peered":"2023-09-23T21:13:12.200435+0000","last_unstale":"2023-09-23T21:19:04.868975+0000","last_undegraded":"2023-09-23T21:19:04.868975+0000","last_fullsized":"2023-09-23T21:19:04.868975+0000","mapping_epoch":493,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:15:44.937886+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"11.17","version":"634'8542","reported_seq":9493,"reported_epoch":637,"state":"active+clean","last_fresh":"2023-09-23T21:19:05.887615+0000","last_change":"2023-09-23T21:18:54.940090+0000","last_active":"2023-09-23T21:19:05.887615+0000","last_peered":"2023-09-23T21:19:05.887615+0000","last_clean":"2023-09-23T21:19:05.887615+0000","last_became_active":"2023-09-23T21:18:54.903043+0000","last_became_peered":"2023-09-23T21:18:54.903043+0000","last_unstale":"2023-09-23T21:19:05.887615+0000","last_undegraded":"2023-09-23T21:19:05.887615+0000","last_fullsized":"2023-09-23T21:19:05.887615+0000","mapping_epoch":631,"log_start":"556'3798","ondisk_log_start":"556'3798","created":559,"last_epoch_clean":632,"parent":"0.0","parent_split_bits":5,"last_scrub":"612'7976","last_scrub_stamp":"2023-09-23T21:18:33.499229+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:15:24.860968+0000","last_clean_scrub_stamp":"2023-09-23T21:18:33.499229+0000","objects_scrubbed":569,"log_size":4744,"log_dups_size":0,"ondisk_log_size":4744,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:02:07.039752+0000","scrub_duration":0.14642005499999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":569,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":258,"num_bytes_recovered":16908288,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,3],"acting":[7,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.1d","version":"79'161","reported_seq":768,"reported_epoch":634,"state":"active+clean","last_fresh":"2023-09-23T21:18:56.916059+0000","last_change":"2023-09-23T21:18:32.873842+0000","last_active":"2023-09-23T21:18:56.916059+0000","last_peered":"2023-09-23T21:18:56.916059+0000","last_clean":"2023-09-23T21:18:56.916059+0000","last_became_active":"2023-09-23T21:15:23.801704+0000","last_became_peered":"2023-09-23T21:15:23.801704+0000","last_unstale":"2023-09-23T21:18:56.916059+0000","last_undegraded":"2023-09-23T21:18:56.916059+0000","last_fullsized":"2023-09-23T21:18:56.916059+0000","mapping_epoch":542,"log_start":"0'0","ondisk_log_start":"0'0","created":479,"last_epoch_clean":543,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:09:36.858077+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:09:36.858077+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.010304610000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,2],"acting":[3,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":164,"reported_epoch":635,"state":"active+clean","last_fresh":"2023-09-23T21:19:04.067048+0000","last_change":"2023-09-23T21:13:38.380574+0000","last_active":"2023-09-23T21:19:04.067048+0000","last_peered":"2023-09-23T21:19:04.067048+0000","last_clean":"2023-09-23T21:19:04.067048+0000","last_became_active":"2023-09-23T21:13:38.373252+0000","last_became_peered":"2023-09-23T21:13:38.373252+0000","last_unstale":"2023-09-23T21:19:04.067048+0000","last_undegraded":"2023-09-23T21:19:04.067048+0000","last_fullsized":"2023-09-23T21:19:04.067048+0000","mapping_epoch":507,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:58:14.788031+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":11,"num_bytes_recovered":3324,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":212,"reported_epoch":637,"state":"active+clean","last_fresh":"2023-09-23T21:19:05.872275+0000","last_change":"2023-09-23T21:12:10.793610+0000","last_active":"2023-09-23T21:19:05.872275+0000","last_peered":"2023-09-23T21:19:05.872275+0000","last_clean":"2023-09-23T21:19:05.872275+0000","last_became_active":"2023-09-23T21:12:10.793081+0000","last_became_peered":"2023-09-23T21:12:10.793081+0000","last_unstale":"2023-09-23T21:19:05.872275+0000","last_undegraded":"2023-09-23T21:19:05.872275+0000","last_fullsized":"2023-09-23T21:19:05.872275+0000","mapping_epoch":470,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:34:36.291550+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":11,"num_pg":26,"stat_sum":{"num_bytes":1310720,"num_objects":19,"num_object_clones":0,"num_object_copies":57,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":19,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":100417,"num_write_kb":2940641,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8937,"num_bytes_recovered":575340544,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":48037888,"data_stored":48037888,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":132781,"ondisk_log_size":132781,"up":78,"acting":78,"num_store_stats":6},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":80,"num_bytes_recovered":18686,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":118784,"data_stored":21350,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":14322,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":31,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":525376,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2109440,"data_stored":2098784,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4991,"ondisk_log_size":4991,"up":93,"acting":93,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201744,"num_pgs":68,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":657036,"kb_used_data":6852,"kb_used_omap":0,"kb_used_meta":650176,"kb_avail":93088116,"statfs":{"total":95995035648,"available":95322230784,"internally_reserved":0,"allocated":7016448,"data_stored":5283623,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":665780224},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:18:22 2023","interfaces":[{"interface":"back","average":{"1min":1.5209999999999999,"5min":1.474,"15min":2.1080000000000001},"min":{"1min":0.22900000000000001,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":3.968,"5min":37.207999999999998,"15min":37.207999999999998},"last":0.46800000000000003},{"interface":"front","average":{"1min":1.73,"5min":1.748,"15min":2.0720000000000001},"min":{"1min":0.318,"5min":0.253,"15min":0.253},"max":{"1min":5.2910000000000004,"5min":37.122,"15min":37.122},"last":0.433}]},{"osd":1,"last update":"Sat Sep 23 21:18:16 2023","interfaces":[{"interface":"back","average":{"1min":1.2609999999999999,"5min":1.2210000000000001,"15min":1.7969999999999999},"min":{"1min":0.27600000000000002,"5min":0.27600000000000002,"15min":0.26700000000000002},"max":{"1min":6.9130000000000003,"5min":7.5369999999999999,"15min":14.034000000000001},"last":0.55400000000000005},{"interface":"front","average":{"1min":1.4370000000000001,"5min":1.379,"15min":1.776},"min":{"1min":0.34300000000000003,"5min":0.29999999999999999,"15min":0.26300000000000001},"max":{"1min":3.9300000000000002,"5min":11.002000000000001,"15min":11.226000000000001},"last":0.622}]},{"osd":2,"last update":"Sat Sep 23 21:18:38 2023","interfaces":[{"interface":"back","average":{"1min":2.105,"5min":2.0739999999999998,"15min":1.8069999999999999},"min":{"1min":0.27200000000000002,"5min":0.27200000000000002,"15min":0.26200000000000001},"max":{"1min":18.488,"5min":25.471,"15min":25.471},"last":0.65100000000000002},{"interface":"front","average":{"1min":1.9330000000000001,"5min":1.857,"15min":1.752},"min":{"1min":0.28799999999999998,"5min":0.28799999999999998,"15min":0.26700000000000002},"max":{"1min":17.536000000000001,"5min":27.187999999999999,"15min":27.187999999999999},"last":0.53700000000000003}]},{"osd":3,"last update":"Sat Sep 23 21:19:06 2023","interfaces":[{"interface":"back","average":{"1min":1.6220000000000001,"5min":1.6040000000000001,"15min":1.788},"min":{"1min":0.317,"5min":0.30599999999999999,"15min":0.30099999999999999},"max":{"1min":5.7830000000000004,"5min":8.6259999999999994,"15min":12.359999999999999},"last":0.505},{"interface":"front","average":{"1min":1.504,"5min":1.8100000000000001,"15min":1.79},"min":{"1min":0.33400000000000002,"5min":0.28799999999999998,"15min":0.28699999999999998},"max":{"1min":4.6769999999999996,"5min":15.82,"15min":15.82},"last":0.67200000000000004}]},{"osd":4,"last update":"Sat Sep 23 21:19:06 2023","interfaces":[{"interface":"back","average":{"1min":0.42199999999999999,"5min":0.78500000000000003,"15min":1.1160000000000001},"min":{"1min":0.27000000000000002,"5min":0.251,"15min":0.248},"max":{"1min":0.92700000000000005,"5min":4.2130000000000001,"15min":11.297000000000001},"last":0.746},{"interface":"front","average":{"1min":0.61299999999999999,"5min":0.91800000000000004,"15min":1.0800000000000001},"min":{"1min":0.26800000000000002,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":3.73,"5min":7.1870000000000003,"15min":12.752000000000001},"last":0.81799999999999995}]},{"osd":5,"last update":"Sat Sep 23 21:19:06 2023","interfaces":[{"interface":"back","average":{"1min":0.57199999999999995,"5min":0.91600000000000004,"15min":1.0640000000000001},"min":{"1min":0.28199999999999997,"5min":0.28199999999999997,"15min":0.26000000000000001},"max":{"1min":1.7150000000000001,"5min":6.9740000000000002,"15min":9.4789999999999992},"last":0.69799999999999995},{"interface":"front","average":{"1min":0.65900000000000003,"5min":1.004,"15min":1.157},"min":{"1min":0.32300000000000001,"5min":0.32000000000000001,"15min":0.26200000000000001},"max":{"1min":3.1349999999999998,"5min":6.7439999999999998,"15min":11.257},"last":0.85499999999999998}]},{"osd":6,"last update":"Sat Sep 23 21:18:16 2023","interfaces":[{"interface":"back","average":{"1min":0.86099999999999999,"5min":0.88300000000000001,"15min":1.0620000000000001},"min":{"1min":0.34599999999999997,"5min":0.32000000000000001,"15min":0.30099999999999999},"max":{"1min":5.7069999999999999,"5min":7.5339999999999998,"15min":10.941000000000001},"last":0.76800000000000002},{"interface":"front","average":{"1min":1.379,"5min":1.0429999999999999,"15min":1.181},"min":{"1min":0.41099999999999998,"5min":0.36299999999999999,"15min":0.33100000000000002},"max":{"1min":5.6760000000000002,"5min":5.6760000000000002,"15min":12.715},"last":0.878}]}]},{"osd":6,"up_from":0,"seq":704374636827,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":1,"up_from":0,"seq":1503238553759,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":0,"up_from":493,"seq":2117418877000,"num_pgs":72,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":403260,"kb_used_data":8180,"kb_used_omap":36,"kb_used_meta":395035,"kb_avail":93341892,"statfs":{"total":95995035648,"available":95582097408,"internally_reserved":0,"allocated":8376320,"data_stored":5487674,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":37336,"internal_metadata":404516392},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 21:18:27 2023","interfaces":[{"interface":"back","average":{"1min":1.3620000000000001,"5min":1.4410000000000001,"15min":0.98399999999999999},"min":{"1min":0.25900000000000001,"5min":0.214,"15min":0.214},"max":{"1min":6.7779999999999996,"5min":13.779999999999999,"15min":13.779999999999999},"last":0.30299999999999999},{"interface":"front","average":{"1min":2.5499999999999998,"5min":1.7569999999999999,"15min":1.099},"min":{"1min":0.35099999999999998,"5min":0.20799999999999999,"15min":0.20799999999999999},"max":{"1min":10.551,"5min":11.964,"15min":11.964},"last":0.27700000000000002}]},{"osd":2,"last update":"Sat Sep 23 21:18:27 2023","interfaces":[{"interface":"back","average":{"1min":2.0579999999999998,"5min":1.8,"15min":1.036},"min":{"1min":0.36599999999999999,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":7.3449999999999998,"5min":13.637,"15min":13.637},"last":0.39400000000000002},{"interface":"front","average":{"1min":3.258,"5min":2.46,"15min":1.4650000000000001},"min":{"1min":0.30199999999999999,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":10.401999999999999,"5min":26.571000000000002,"15min":26.571000000000002},"last":0.32300000000000001}]},{"osd":3,"last update":"Sat Sep 23 21:18:27 2023","interfaces":[{"interface":"back","average":{"1min":3.9950000000000001,"5min":2.6000000000000001,"15min":1.4450000000000001},"min":{"1min":0.253,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":37.079999999999998,"5min":37.079999999999998,"15min":37.079999999999998},"last":0.34799999999999998},{"interface":"front","average":{"1min":3.4550000000000001,"5min":2.2029999999999998,"15min":1.4790000000000001},"min":{"1min":0.34699999999999998,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":32.935000000000002,"5min":32.935000000000002,"15min":32.935000000000002},"last":0.28899999999999998}]},{"osd":4,"last update":"Sat Sep 23 21:18:27 2023","interfaces":[{"interface":"back","average":{"1min":1.546,"5min":1.7390000000000001,"15min":1.3080000000000001},"min":{"1min":0.34499999999999997,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":4.7880000000000003,"5min":22.780000000000001,"15min":22.780000000000001},"last":0.38300000000000001},{"interface":"front","average":{"1min":1.9390000000000001,"5min":1.4990000000000001,"15min":1.163},"min":{"1min":0.32500000000000001,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":6.9119999999999999,"5min":22.652000000000001,"15min":22.652000000000001},"last":0.44500000000000001}]},{"osd":5,"last update":"Sat Sep 23 21:18:27 2023","interfaces":[{"interface":"back","average":{"1min":1.5549999999999999,"5min":1.5680000000000001,"15min":1.3460000000000001},"min":{"1min":0.312,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":6.1139999999999999,"5min":8.9329999999999998,"15min":8.9329999999999998},"last":0.433},{"interface":"front","average":{"1min":1.9039999999999999,"5min":1.696,"15min":1.274},"min":{"1min":0.40100000000000002,"5min":0.313,"15min":0.313},"max":{"1min":6.5990000000000002,"5min":15.321999999999999,"15min":15.321999999999999},"last":0.52900000000000003}]},{"osd":6,"last update":"Sat Sep 23 21:18:27 2023","interfaces":[{"interface":"back","average":{"1min":1.5720000000000001,"5min":1.4810000000000001,"15min":1.2450000000000001},"min":{"1min":0.372,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":6.9409999999999998,"5min":7.2050000000000001,"15min":7.2050000000000001},"last":0.55800000000000005},{"interface":"front","average":{"1min":1.647,"5min":1.595,"15min":1.403},"min":{"1min":0.38400000000000001,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":6.266,"5min":8.8889999999999993,"15min":8.8889999999999993},"last":0.40999999999999998}]},{"osd":7,"last update":"Sat Sep 23 21:18:27 2023","interfaces":[{"interface":"back","average":{"1min":1.595,"5min":1.8240000000000001,"15min":1.4139999999999999},"min":{"1min":0.34699999999999998,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":4.8360000000000003,"5min":15.273999999999999,"15min":15.273999999999999},"last":0.505},{"interface":"front","average":{"1min":1.75,"5min":1.7709999999999999,"15min":1.3620000000000001},"min":{"1min":0.36199999999999999,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":6.9109999999999996,"5min":33.779000000000003,"15min":33.779000000000003},"last":0.57799999999999996}]}]},{"osd":2,"up_from":93,"seq":399431958872,"num_pgs":52,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1035828,"kb_used_data":8172,"kb_used_omap":81,"kb_used_meta":1027566,"kb_avail":92709324,"statfs":{"total":95995035648,"available":94934347776,"internally_reserved":0,"allocated":8368128,"data_stored":5433082,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":83750,"internal_metadata":1052227802},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:18:23 2023","interfaces":[{"interface":"back","average":{"1min":2.6339999999999999,"5min":1.9370000000000001,"15min":1.2010000000000001},"min":{"1min":0.221,"5min":0.187,"15min":0.187},"max":{"1min":9.9499999999999993,"5min":13.25,"15min":13.25},"last":0.26500000000000001},{"interface":"front","average":{"1min":2.2509999999999999,"5min":2.089,"15min":1.806},"min":{"1min":0.26800000000000002,"5min":0.20200000000000001,"15min":0.20200000000000001},"max":{"1min":9.8849999999999998,"5min":18.111999999999998,"15min":18.111999999999998},"last":0.32900000000000001}]},{"osd":1,"last update":"Sat Sep 23 21:18:23 2023","interfaces":[{"interface":"back","average":{"1min":1.645,"5min":1.756,"15min":2.2629999999999999},"min":{"1min":0.34000000000000002,"5min":0.20599999999999999,"15min":0.188},"max":{"1min":10.535,"5min":10.535,"15min":34.396999999999998},"last":0.35699999999999998},{"interface":"front","average":{"1min":2.2240000000000002,"5min":1.95,"15min":2.3460000000000001},"min":{"1min":0.24099999999999999,"5min":0.24099999999999999,"15min":0.217},"max":{"1min":6.0990000000000002,"5min":18.081,"15min":34.462000000000003},"last":0.36799999999999999}]},{"osd":3,"last update":"Sat Sep 23 21:19:02 2023","interfaces":[{"interface":"back","average":{"1min":2.2949999999999999,"5min":2.3999999999999999,"15min":2.149},"min":{"1min":0.27300000000000002,"5min":0.248,"15min":0.20599999999999999},"max":{"1min":8.141,"5min":20.106999999999999,"15min":35.549999999999997},"last":0.34200000000000003},{"interface":"front","average":{"1min":2.593,"5min":2.9249999999999998,"15min":2.4609999999999999},"min":{"1min":0.34899999999999998,"5min":0.28599999999999998,"15min":0.21299999999999999},"max":{"1min":8.2469999999999999,"5min":20.239000000000001,"15min":36.951999999999998},"last":0.39100000000000001}]},{"osd":4,"last update":"Sat Sep 23 21:18:23 2023","interfaces":[{"interface":"back","average":{"1min":1.2749999999999999,"5min":1.3460000000000001,"15min":1.4450000000000001},"min":{"1min":0.27000000000000002,"5min":0.27000000000000002,"15min":0.249},"max":{"1min":3.8530000000000002,"5min":7.9770000000000003,"15min":10.266999999999999},"last":0.496},{"interface":"front","average":{"1min":1.363,"5min":1.5640000000000001,"15min":1.7110000000000001},"min":{"1min":0.38900000000000001,"5min":0.30299999999999999,"15min":0.26700000000000002},"max":{"1min":4.2210000000000001,"5min":10.459,"15min":12.49},"last":0.53600000000000003}]},{"osd":5,"last update":"Sat Sep 23 21:18:23 2023","interfaces":[{"interface":"back","average":{"1min":1.6160000000000001,"5min":1.456,"15min":1.5349999999999999},"min":{"1min":0.32300000000000001,"5min":0.29099999999999998,"15min":0.28000000000000003},"max":{"1min":5.4139999999999997,"5min":7.6180000000000003,"15min":9.5039999999999996},"last":0.47299999999999998},{"interface":"front","average":{"1min":1.474,"5min":1.5169999999999999,"15min":1.6890000000000001},"min":{"1min":0.34899999999999998,"5min":0.33800000000000002,"15min":0.29399999999999998},"max":{"1min":4.2649999999999997,"5min":10.425000000000001,"15min":12.57},"last":0.51400000000000001}]},{"osd":6,"last update":"Sat Sep 23 21:18:23 2023","interfaces":[{"interface":"back","average":{"1min":1.542,"5min":1.633,"15min":1.8979999999999999},"min":{"1min":0.54100000000000004,"5min":0.377,"15min":0.35699999999999998},"max":{"1min":4.2990000000000004,"5min":10.335000000000001,"15min":12.518000000000001},"last":0.57499999999999996},{"interface":"front","average":{"1min":1.7190000000000001,"5min":1.696,"15min":2.0510000000000002},"min":{"1min":0.501,"5min":0.377,"15min":0.35599999999999998},"max":{"1min":4.7969999999999997,"5min":10.381,"15min":12.544},"last":0.55500000000000005}]},{"osd":7,"last update":"Sat Sep 23 21:18:23 2023","interfaces":[{"interface":"back","average":{"1min":1.6339999999999999,"5min":1.637,"15min":1.6519999999999999},"min":{"1min":0.31,"5min":0.31,"15min":0.29799999999999999},"max":{"1min":5.7480000000000002,"5min":18.021999999999998,"15min":18.021999999999998},"last":0.505},{"interface":"front","average":{"1min":2.145,"5min":1.6339999999999999,"15min":1.73},"min":{"1min":0.35999999999999999,"5min":0.34999999999999998,"15min":0.32200000000000001},"max":{"1min":6.694,"5min":9.6349999999999998,"15min":10.92},"last":0.58799999999999997}]}]},{"osd":3,"up_from":98,"seq":420906795345,"num_pgs":75,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1129808,"kb_used_data":55048,"kb_used_omap":7,"kb_used_meta":1074744,"kb_avail":92615344,"statfs":{"total":95995035648,"available":94838112256,"internally_reserved":0,"allocated":56369152,"data_stored":53508215,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7576,"internal_metadata":1100538472},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:18:18 2023","interfaces":[{"interface":"back","average":{"1min":2.2069999999999999,"5min":4.0750000000000002,"15min":1.583},"min":{"1min":0.22500000000000001,"5min":0.18099999999999999,"15min":0.18099999999999999},"max":{"1min":10.439,"5min":147.38900000000001,"15min":147.38900000000001},"last":0.22900000000000001},{"interface":"front","average":{"1min":1.1899999999999999,"5min":3.5859999999999999,"15min":1.6399999999999999},"min":{"1min":0.252,"5min":0.223,"15min":0.223},"max":{"1min":5.7279999999999998,"5min":149.989,"15min":149.989},"last":0.376}]},{"osd":1,"last update":"Sat Sep 23 21:18:18 2023","interfaces":[{"interface":"back","average":{"1min":2.2029999999999998,"5min":1.5940000000000001,"15min":2.1160000000000001},"min":{"1min":0.27000000000000002,"5min":0.218,"15min":0.20000000000000001},"max":{"1min":6.1150000000000002,"5min":7.2759999999999998,"15min":22.577999999999999},"last":0.248},{"interface":"front","average":{"1min":1.827,"5min":1.2969999999999999,"15min":1.984},"min":{"1min":0.26100000000000001,"5min":0.249,"15min":0.216},"max":{"1min":6.1219999999999999,"5min":7.6829999999999998,"15min":21.617999999999999},"last":0.318}]},{"osd":2,"last update":"Sat Sep 23 21:19:03 2023","interfaces":[{"interface":"back","average":{"1min":0.84999999999999998,"5min":1.395,"15min":1.893},"min":{"1min":0.23799999999999999,"5min":0.23799999999999999,"15min":0.20899999999999999},"max":{"1min":3.1000000000000001,"5min":11.321,"15min":56.317999999999998},"last":0.27900000000000003},{"interface":"front","average":{"1min":3.1120000000000001,"5min":2.331,"15min":2.343},"min":{"1min":0.34899999999999998,"5min":0.23400000000000001,"15min":0.216},"max":{"1min":11.177,"5min":17.423999999999999,"15min":55.988},"last":0.39300000000000002}]},{"osd":4,"last update":"Sat Sep 23 21:19:03 2023","interfaces":[{"interface":"back","average":{"1min":1.0609999999999999,"5min":1.27,"15min":1.472},"min":{"1min":0.36899999999999999,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":5.1589999999999998,"5min":10.807,"15min":21.478999999999999},"last":0.45800000000000002},{"interface":"front","average":{"1min":2.145,"5min":1.4870000000000001,"15min":1.7549999999999999},"min":{"1min":0.39600000000000002,"5min":0.311,"15min":0.28499999999999998},"max":{"1min":11.202,"5min":11.202,"15min":22.402000000000001},"last":0.48099999999999998}]},{"osd":5,"last update":"Sat Sep 23 21:19:03 2023","interfaces":[{"interface":"back","average":{"1min":0.91400000000000003,"5min":1.3109999999999999,"15min":1.6040000000000001},"min":{"1min":0.34999999999999998,"5min":0.313,"15min":0.311},"max":{"1min":3.109,"5min":10.882999999999999,"15min":21.562000000000001},"last":0.49399999999999999},{"interface":"front","average":{"1min":0.90300000000000002,"5min":1.2470000000000001,"15min":1.679},"min":{"1min":0.30199999999999999,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":2.8010000000000002,"5min":5.7300000000000004,"15min":14.029},"last":0.43099999999999999}]},{"osd":6,"last update":"Sat Sep 23 21:18:18 2023","interfaces":[{"interface":"back","average":{"1min":1.6180000000000001,"5min":1.464,"15min":1.839},"min":{"1min":0.39800000000000002,"5min":0.35899999999999999,"15min":0.33900000000000002},"max":{"1min":4.9749999999999996,"5min":7.3410000000000002,"15min":22.446999999999999},"last":0.44500000000000001},{"interface":"front","average":{"1min":1.657,"5min":1.3480000000000001,"15min":1.8280000000000001},"min":{"1min":0.373,"5min":0.373,"15min":0.373},"max":{"1min":7.5949999999999998,"5min":9.3889999999999993,"15min":12.375999999999999},"last":0.46700000000000003}]},{"osd":7,"last update":"Sat Sep 23 21:19:03 2023","interfaces":[{"interface":"back","average":{"1min":0.91100000000000003,"5min":1.456,"15min":1.74},"min":{"1min":0.41099999999999998,"5min":0.41099999999999998,"15min":0.33900000000000002},"max":{"1min":2.4279999999999999,"5min":10.914999999999999,"15min":21.518999999999998},"last":0.53200000000000003},{"interface":"front","average":{"1min":1.2370000000000001,"5min":1.589,"15min":1.9059999999999999},"min":{"1min":0.38800000000000001,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":3.8130000000000002,"5min":8.2590000000000003,"15min":12.624000000000001},"last":0.51000000000000001}]}]},{"osd":4,"up_from":31,"seq":133143986652,"num_pgs":64,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":587080,"kb_used_data":6912,"kb_used_omap":0,"kb_used_meta":580160,"kb_avail":93158072,"statfs":{"total":95995035648,"available":95393865728,"internally_reserved":0,"allocated":7077888,"data_stored":5348868,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":594083840},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:18:24 2023","interfaces":[{"interface":"back","average":{"1min":2.2690000000000001,"5min":1.673,"15min":1.157},"min":{"1min":0.245,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":20.515000000000001,"5min":28.683,"15min":28.683},"last":0.48299999999999998},{"interface":"front","average":{"1min":3.085,"5min":1.8009999999999999,"15min":1.3560000000000001},"min":{"1min":0.249,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":20.792000000000002,"5min":29.155000000000001,"15min":29.155000000000001},"last":0.39400000000000002}]},{"osd":1,"last update":"Sat Sep 23 21:18:24 2023","interfaces":[{"interface":"back","average":{"1min":1.7330000000000001,"5min":1.2829999999999999,"15min":2.1070000000000002},"min":{"1min":0.28499999999999998,"5min":0.26900000000000002,"15min":0.221},"max":{"1min":8.1020000000000003,"5min":8.1020000000000003,"15min":37.960000000000001},"last":0.46800000000000003},{"interface":"front","average":{"1min":1.55,"5min":1.399,"15min":2.0550000000000002},"min":{"1min":0.33100000000000002,"5min":0.28499999999999998,"15min":0.27000000000000002},"max":{"1min":8.1509999999999998,"5min":13.983000000000001,"15min":37.933},"last":0.45300000000000001}]},{"osd":2,"last update":"Sat Sep 23 21:18:24 2023","interfaces":[{"interface":"back","average":{"1min":2.177,"5min":1.8500000000000001,"15min":1.754},"min":{"1min":0.28399999999999997,"5min":0.26400000000000001,"15min":0.247},"max":{"1min":9.8399999999999999,"5min":17.602,"15min":17.602},"last":0.501},{"interface":"front","average":{"1min":1.8839999999999999,"5min":1.7849999999999999,"15min":1.831},"min":{"1min":0.35399999999999998,"5min":0.311,"15min":0.27800000000000002},"max":{"1min":5.5250000000000004,"5min":14.339,"15min":14.417999999999999},"last":0.42799999999999999}]},{"osd":3,"last update":"Sat Sep 23 21:18:55 2023","interfaces":[{"interface":"back","average":{"1min":1.3129999999999999,"5min":2.1579999999999999,"15min":1.9790000000000001},"min":{"1min":0.29499999999999998,"5min":0.27900000000000003,"15min":0.25600000000000001},"max":{"1min":6.1230000000000002,"5min":67.105999999999995,"15min":67.105999999999995},"last":0.438},{"interface":"front","average":{"1min":1.5289999999999999,"5min":2.2719999999999998,"15min":1.966},"min":{"1min":0.40400000000000003,"5min":0.32200000000000001,"15min":0.30199999999999999},"max":{"1min":6.1319999999999997,"5min":67.046999999999997,"15min":67.046999999999997},"last":0.54100000000000004}]},{"osd":5,"last update":"Sat Sep 23 21:18:13 2023","interfaces":[{"interface":"back","average":{"1min":0.89300000000000002,"5min":1.1080000000000001,"15min":1.123},"min":{"1min":0.254,"5min":0.254,"15min":0.222},"max":{"1min":2.923,"5min":14.398999999999999,"15min":14.398999999999999},"last":0.371},{"interface":"front","average":{"1min":1.0349999999999999,"5min":0.98799999999999999,"15min":1.1080000000000001},"min":{"1min":0.28100000000000003,"5min":0.25800000000000001,"15min":0.255},"max":{"1min":3.6000000000000001,"5min":8.016,"15min":9.8979999999999997},"last":0.33800000000000002}]},{"osd":6,"last update":"Sat Sep 23 21:18:24 2023","interfaces":[{"interface":"back","average":{"1min":1.077,"5min":0.96599999999999997,"15min":1.2649999999999999},"min":{"1min":0.308,"5min":0.27200000000000002,"15min":0.26800000000000002},"max":{"1min":3.2050000000000001,"5min":8.7420000000000009,"15min":24.452999999999999},"last":0.51300000000000001},{"interface":"front","average":{"1min":0.83799999999999997,"5min":1.1499999999999999,"15min":1.0820000000000001},"min":{"1min":0.38,"5min":0.36399999999999999,"15min":0.32500000000000001},"max":{"1min":2.5409999999999999,"5min":14.374000000000001,"15min":14.387},"last":0.52400000000000002}]},{"osd":7,"last update":"Sat Sep 23 21:18:55 2023","interfaces":[{"interface":"back","average":{"1min":0.70599999999999996,"5min":1.077,"15min":1.1659999999999999},"min":{"1min":0.33800000000000002,"5min":0.32000000000000001,"15min":0.30099999999999999},"max":{"1min":2.6419999999999999,"5min":14.422000000000001,"15min":16.890999999999998},"last":0.56699999999999995},{"interface":"front","average":{"1min":0.70999999999999996,"5min":0.89800000000000002,"15min":1.179},"min":{"1min":0.32900000000000001,"5min":0.29199999999999998,"15min":0.27400000000000002},"max":{"1min":2.6579999999999999,"5min":7.7229999999999999,"15min":11.170999999999999},"last":0.41299999999999998}]}]},{"osd":5,"up_from":39,"seq":167503725018,"num_pgs":61,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":626364,"kb_used_data":6836,"kb_used_omap":0,"kb_used_meta":619520,"kb_avail":93118788,"statfs":{"total":95995035648,"available":95353638912,"internally_reserved":0,"allocated":7000064,"data_stored":5279620,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":634388480},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:18:16 2023","interfaces":[{"interface":"back","average":{"1min":1.452,"5min":1.234,"15min":0.88800000000000001},"min":{"1min":0.22500000000000001,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":5.9829999999999997,"5min":8.673,"15min":8.673},"last":0.65500000000000003},{"interface":"front","average":{"1min":2.5219999999999998,"5min":1.548,"15min":0.96199999999999997},"min":{"1min":0.215,"5min":0.215,"15min":0.215},"max":{"1min":12.259,"5min":12.259,"15min":12.259},"last":0.39100000000000001}]},{"osd":1,"last update":"Sat Sep 23 21:18:16 2023","interfaces":[{"interface":"back","average":{"1min":1.4490000000000001,"5min":1.254,"15min":1.625},"min":{"1min":0.32200000000000001,"5min":0.248,"15min":0.23499999999999999},"max":{"1min":4.3220000000000001,"5min":7.5839999999999996,"15min":25.239999999999998},"last":0.52600000000000002},{"interface":"front","average":{"1min":1.3700000000000001,"5min":1.304,"15min":1.7070000000000001},"min":{"1min":0.36899999999999999,"5min":0.28299999999999997,"15min":0.252},"max":{"1min":4.9500000000000002,"5min":11.116,"15min":25.253},"last":0.60799999999999998}]},{"osd":2,"last update":"Sat Sep 23 21:18:35 2023","interfaces":[{"interface":"back","average":{"1min":1.895,"5min":1.5109999999999999,"15min":1.6419999999999999},"min":{"1min":0.27600000000000002,"5min":0.27300000000000002,"15min":0.25800000000000001},"max":{"1min":9.2479999999999993,"5min":9.6560000000000006,"15min":12.634},"last":0.442},{"interface":"front","average":{"1min":2.0840000000000001,"5min":1.6000000000000001,"15min":1.663},"min":{"1min":0.29099999999999998,"5min":0.29099999999999998,"15min":0.25700000000000001},"max":{"1min":9.4060000000000006,"5min":9.4060000000000006,"15min":9.7170000000000005},"last":0.58099999999999996}]},{"osd":3,"last update":"Sat Sep 23 21:18:54 2023","interfaces":[{"interface":"back","average":{"1min":1.6870000000000001,"5min":1.5089999999999999,"15min":1.883},"min":{"1min":0.32000000000000001,"5min":0.314,"15min":0.29299999999999998},"max":{"1min":7.3700000000000001,"5min":7.3700000000000001,"15min":25.503},"last":0.73699999999999999},{"interface":"front","average":{"1min":1.6100000000000001,"5min":1.782,"15min":2.105},"min":{"1min":0.33200000000000002,"5min":0.311,"15min":0.30599999999999999},"max":{"1min":6.5880000000000001,"5min":12.382,"15min":25.466000000000001},"last":0.48799999999999999}]},{"osd":4,"last update":"Sat Sep 23 21:18:08 2023","interfaces":[{"interface":"back","average":{"1min":0.82399999999999995,"5min":0.91400000000000003,"15min":1.155},"min":{"1min":0.29999999999999999,"5min":0.253,"15min":0.253},"max":{"1min":5.0279999999999996,"5min":11.156000000000001,"15min":12.776},"last":0.76600000000000001},{"interface":"front","average":{"1min":0.73399999999999999,"5min":1.008,"15min":1.1419999999999999},"min":{"1min":0.28599999999999998,"5min":0.246,"15min":0.246},"max":{"1min":3.415,"5min":9.7119999999999997,"15min":11.973000000000001},"last":0.55500000000000005}]},{"osd":6,"last update":"Sat Sep 23 21:18:16 2023","interfaces":[{"interface":"back","average":{"1min":1.0649999999999999,"5min":0.99299999999999999,"15min":1.087},"min":{"1min":0.32200000000000001,"5min":0.28000000000000003,"15min":0.25900000000000001},"max":{"1min":5.8520000000000003,"5min":5.8520000000000003,"15min":8.0909999999999993},"last":0.63200000000000001},{"interface":"front","average":{"1min":1.095,"5min":1.0700000000000001,"15min":1.2330000000000001},"min":{"1min":0.39700000000000002,"5min":0.34300000000000003,"15min":0.317},"max":{"1min":7.1470000000000002,"5min":10.999000000000001,"15min":10.999000000000001},"last":0.79100000000000004}]},{"osd":7,"last update":"Sat Sep 23 21:18:54 2023","interfaces":[{"interface":"back","average":{"1min":0.95499999999999996,"5min":0.94299999999999995,"15min":1.821},"min":{"1min":0.33600000000000002,"5min":0.32400000000000001,"15min":0.31},"max":{"1min":4.0519999999999996,"5min":5.1769999999999996,"15min":189.405},"last":0.67300000000000004},{"interface":"front","average":{"1min":0.97799999999999998,"5min":0.96699999999999997,"15min":1.9490000000000001},"min":{"1min":0.34999999999999998,"5min":0.34999999999999998,"15min":0.32500000000000001},"max":{"1min":5.0910000000000002,"5min":5.0910000000000002,"15min":189.685},"last":0.81699999999999995}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8488,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1259,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5322,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":4838,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":48037888,"data_stored":48037888,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T21:19:09.840 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph status --format=json 2023-09-23T21:19:09.882 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:19:09.914 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:10.108 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T21:19:10.113 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:19:09 smithi026 ceph-mon[179688]: failed to encode map e639 with expected crc 2023-09-23T21:19:10.113 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:19:09 smithi026 ceph-mon[179688]: 11.19 scrub starts 2023-09-23T21:19:10.114 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:19:09 smithi026 ceph-mon[179688]: 11.19 scrub ok 2023-09-23T21:19:10.114 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:19:09 smithi026 ceph-mon[179688]: from='client.47048 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T21:19:10.114 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:19:09 smithi026 ceph-mon[179688]: pgmap v1579: 129 pgs: 1 remapped+peering, 128 active+clean; 1.8 MiB data, 4.2 GiB used, 532 GiB / 536 GiB avail; 0 B/s wr, 277 op/s 2023-09-23T21:19:10.114 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:19:09 smithi026 ceph-mon[175151]: failed to encode map e639 with expected crc 2023-09-23T21:19:10.114 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:19:09 smithi026 ceph-mon[175151]: 11.19 scrub starts 2023-09-23T21:19:10.114 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:19:09 smithi026 ceph-mon[175151]: 11.19 scrub ok 2023-09-23T21:19:10.114 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:19:09 smithi026 ceph-mon[175151]: from='client.47048 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T21:19:10.114 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:19:09 smithi026 ceph-mon[175151]: pgmap v1579: 129 pgs: 1 remapped+peering, 128 active+clean; 1.8 MiB data, 4.2 GiB used, 532 GiB / 536 GiB avail; 0 B/s wr, 277 op/s 2023-09-23T21:19:10.115 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:19:10.120 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:10.120 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:19:10.212 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_blocked_ops 2023-09-23T21:19:10.221 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:19:10.250 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:19:09 smithi070 ceph-mon[168282]: failed to encode map e639 with expected crc 2023-09-23T21:19:10.250 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:19:09 smithi070 ceph-mon[168282]: 11.19 scrub starts 2023-09-23T21:19:10.250 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:19:09 smithi070 ceph-mon[168282]: 11.19 scrub ok 2023-09-23T21:19:10.250 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:19:09 smithi070 ceph-mon[168282]: from='client.47048 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T21:19:10.250 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:19:09 smithi070 ceph-mon[168282]: pgmap v1579: 129 pgs: 1 remapped+peering, 128 active+clean; 1.8 MiB data, 4.2 GiB used, 532 GiB / 536 GiB avail; 0 B/s wr, 277 op/s 2023-09-23T21:19:10.253 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:10.289 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:19:10.291 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:10.292 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:19:10.393 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:19:10.421 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:10.607 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:19:10.610 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:10.610 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:19:10.711 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:19:10.756 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:10.793 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:19:10.797 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:10.798 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:19:10.818 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0: {} 2023-09-23T21:19:10.846 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1: {} 2023-09-23T21:19:10.866 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2: {} 2023-09-23T21:19:10.891 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3: {} 2023-09-23T21:19:10.899 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:19:10.913 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4: {} 2023-09-23T21:19:10.926 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:19:10.936 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T21:19:10.938 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:10.966 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6: {} 2023-09-23T21:19:10.986 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T21:19:11.072 INFO:teuthology.orchestra.run.smithi026.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T21:19:11.073 INFO:teuthology.orchestra.run.smithi026.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T21:19:11.073 INFO:teuthology.orchestra.run.smithi026.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T21:19:11.073 INFO:teuthology.orchestra.run.smithi026.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T21:19:11.073 INFO:teuthology.orchestra.run.smithi026.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T21:19:11.073 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T21:19:11.073 INFO:teuthology.orchestra.run.smithi026.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T21:19:11.073 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T21:19:11.073 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:19:11.074 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:19:10 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/2547520794' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:19:11.074 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:19:10 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/2547520794' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:19:11.074 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:19:11.075 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:11.075 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:19:11.076 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:19:11.176 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:19:11.205 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:19:10 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/2547520794' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:19:11.208 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:11.243 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:19:11.246 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:11.247 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:19:11.347 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:19:11.375 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:11.408 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:19:11.411 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:11.412 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:19:11.512 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:19:11.539 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:11.571 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:19:11.573 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:11.573 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:19:11.674 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:19:11.700 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:11.732 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:19:11.735 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:11.735 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:19:11.836 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:19:11.867 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:11.904 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:19:11.913 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:11.914 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:19:11.923 INFO:teuthology.orchestra.run.smithi026.stderr:pool 'unique_pool_5' removed 2023-09-23T21:19:11.945 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:19:11 smithi026 ceph-mon[179688]: 1.1d scrub starts 2023-09-23T21:19:11.945 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:19:11 smithi026 ceph-mon[179688]: 1.1d scrub ok 2023-09-23T21:19:11.945 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:19:11 smithi026 ceph-mon[179688]: pgmap v1580: 129 pgs: 129 active+clean; 1.8 MiB data, 4.2 GiB used, 532 GiB / 536 GiB avail 2023-09-23T21:19:11.945 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:19:11 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/1272222431' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_5", "pool2": "unique_pool_5", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-23T21:19:11.945 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:19:11 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_5", "pool2": "unique_pool_5", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-23T21:19:11.946 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:19:11 smithi026 ceph-mon[175151]: 1.1d scrub starts 2023-09-23T21:19:11.946 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:19:11 smithi026 ceph-mon[175151]: 1.1d scrub ok 2023-09-23T21:19:11.946 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:19:11 smithi026 ceph-mon[175151]: pgmap v1580: 129 pgs: 129 active+clean; 1.8 MiB data, 4.2 GiB used, 532 GiB / 536 GiB avail 2023-09-23T21:19:11.946 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:19:11 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/1272222431' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_5", "pool2": "unique_pool_5", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-23T21:19:11.946 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:19:11 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_5", "pool2": "unique_pool_5", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-23T21:19:12.015 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:19:12.040 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:12.071 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:19:12.074 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:12.074 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:19:12.175 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:19:12.199 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:19:11 smithi070 ceph-mon[168282]: 1.1d scrub starts 2023-09-23T21:19:12.199 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:19:11 smithi070 ceph-mon[168282]: 1.1d scrub ok 2023-09-23T21:19:12.199 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:19:11 smithi070 ceph-mon[168282]: pgmap v1580: 129 pgs: 129 active+clean; 1.8 MiB data, 4.2 GiB used, 532 GiB / 536 GiB avail 2023-09-23T21:19:12.199 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:19:11 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/1272222431' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_5", "pool2": "unique_pool_5", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-23T21:19:12.199 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:19:11 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_5", "pool2": "unique_pool_5", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-23T21:19:12.201 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:12.246 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:19:12.250 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:12.250 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:19:12.351 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:19:12.377 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:12.392 INFO:teuthology.task.sequential:In sequential, running task print... 2023-09-23T21:19:12.392 INFO:teuthology.task.print:**** done end radosbench.yaml 2023-09-23T21:19:12.393 INFO:teuthology.task.sequential:In sequential, running task workunit... 2023-09-23T21:19:12.393 INFO:tasks.workunit:Pulling workunits from ref quincy 2023-09-23T21:19:12.393 INFO:tasks.workunit:Making a separate scratch dir for every client... 2023-09-23T21:19:12.393 DEBUG:teuthology.orchestra.run.smithi026:> stat -- /home/ubuntu/cephtest/mnt.0 2023-09-23T21:19:12.408 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:19:12.410 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:12.411 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:19:12.475 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T21:19:12.476 INFO:teuthology.orchestra.run.smithi026.stderr:stat: cannot statx '/home/ubuntu/cephtest/mnt.0': No such file or directory 2023-09-23T21:19:12.476 DEBUG:teuthology.orchestra.run.smithi026:> mkdir -- /home/ubuntu/cephtest/mnt.0 2023-09-23T21:19:12.512 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:19:12.594 INFO:tasks.workunit:Created dir /home/ubuntu/cephtest/mnt.0 2023-09-23T21:19:12.594 DEBUG:teuthology.orchestra.run.smithi026:> cd -- /home/ubuntu/cephtest/mnt.0 && mkdir -- client.0 2023-09-23T21:19:12.600 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:12.619 INFO:tasks.workunit:timeout=3h 2023-09-23T21:19:12.619 INFO:tasks.workunit:cleanup=True 2023-09-23T21:19:12.620 DEBUG:teuthology.orchestra.run.smithi026:> rm -rf /home/ubuntu/cephtest/clone.client.0 && git clone --depth 1 --branch quincy https://github.com/ceph/ceph.git /home/ubuntu/cephtest/clone.client.0 && cd /home/ubuntu/cephtest/clone.client.0 2023-09-23T21:19:12.707 INFO:tasks.workunit.client.0.smithi026.stderr:Cloning into '/home/ubuntu/cephtest/clone.client.0'... 2023-09-23T21:19:12.844 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T21:19:12.846 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:12.847 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:19:12.948 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:19:12.974 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:13.005 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:19:13.007 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:13.008 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:19:13.108 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:19:13.134 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:13.165 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:19:13.167 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:19:12 smithi070 ceph-mon[168282]: failed to encode map e640 with expected crc 2023-09-23T21:19:13.167 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:19:12 smithi070 ceph-mon[168282]: failed to encode map e640 with expected crc 2023-09-23T21:19:13.168 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:19:12 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_5", "pool2": "unique_pool_5", "yes_i_really_really_mean_it": true}]': finished 2023-09-23T21:19:13.168 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:19:12 smithi070 ceph-mon[168282]: osdmap e640: 8 total, 8 up, 6 in 2023-09-23T21:19:13.168 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:13.169 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:19:13.260 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:19:12 smithi026 ceph-mon[179688]: failed to encode map e640 with expected crc 2023-09-23T21:19:13.260 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:19:12 smithi026 ceph-mon[179688]: failed to encode map e640 with expected crc 2023-09-23T21:19:13.260 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:19:12 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_5", "pool2": "unique_pool_5", "yes_i_really_really_mean_it": true}]': finished 2023-09-23T21:19:13.260 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:19:12 smithi026 ceph-mon[179688]: osdmap e640: 8 total, 8 up, 6 in 2023-09-23T21:19:13.261 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:19:12 smithi026 ceph-mon[175151]: failed to encode map e640 with expected crc 2023-09-23T21:19:13.261 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:19:12 smithi026 ceph-mon[175151]: failed to encode map e640 with expected crc 2023-09-23T21:19:13.261 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:19:12 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_5", "pool2": "unique_pool_5", "yes_i_really_really_mean_it": true}]': finished 2023-09-23T21:19:13.261 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:19:12 smithi026 ceph-mon[175151]: osdmap e640: 8 total, 8 up, 6 in 2023-09-23T21:19:13.269 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:19:13.295 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:13.326 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:19:13.329 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:13.329 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:19:13.430 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:19:13.460 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:13.768 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T21:19:13.825 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:19:13.828 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:13.829 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:19:13.930 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:19:13.960 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:13.995 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:19:13.998 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:13.999 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:19:14.049 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:19:13 smithi026 ceph-mon[179688]: failed to encode map e640 with expected crc 2023-09-23T21:19:14.049 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:19:13 smithi026 ceph-mon[179688]: pgmap v1582: 103 pgs: 103 active+clean; 582 KiB data, 4.2 GiB used, 532 GiB / 536 GiB avail 2023-09-23T21:19:14.049 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:19:13 smithi026 ceph-mon[175151]: failed to encode map e640 with expected crc 2023-09-23T21:19:14.050 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:19:13 smithi026 ceph-mon[175151]: pgmap v1582: 103 pgs: 103 active+clean; 582 KiB data, 4.2 GiB used, 532 GiB / 536 GiB avail 2023-09-23T21:19:14.099 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:19:14.129 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:14.215 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:19:14.218 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:14.218 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:19:14.319 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:19:14.349 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:19:13 smithi070 ceph-mon[168282]: failed to encode map e640 with expected crc 2023-09-23T21:19:14.349 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:19:13 smithi070 ceph-mon[168282]: pgmap v1582: 103 pgs: 103 active+clean; 582 KiB data, 4.2 GiB used, 532 GiB / 536 GiB avail 2023-09-23T21:19:14.352 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:14.389 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:19:14.392 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:14.393 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:19:14.493 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:19:14.523 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:14.558 INFO:teuthology.orchestra.run.smithi026.stdout:{ 2023-09-23T21:19:14.559 INFO:teuthology.orchestra.run.smithi026.stdout: "ops": [], 2023-09-23T21:19:14.559 INFO:teuthology.orchestra.run.smithi026.stdout: "complaint_time": 30, 2023-09-23T21:19:14.559 INFO:teuthology.orchestra.run.smithi026.stdout: "num_blocked_ops": 0 2023-09-23T21:19:14.559 INFO:teuthology.orchestra.run.smithi026.stdout:} 2023-09-23T21:19:14.563 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:19:14.567 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:14.567 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:19:14.592 INFO:teuthology.orchestra.run.smithi026.stdout: 2023-09-23T21:19:14.592 INFO:teuthology.orchestra.run.smithi026.stdout:{"fsid":"b3f8e4ec-5a50-11ee-8db4-212e2dc638e7","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"nodeep-scrub flag(s) set","count":12},"muted":false}},"mutes":[]},"election_epoch":30,"quorum":[0,1,2],"quorum_names":["a","b","c"],"quorum_age":1873,"monmap":{"epoch":4,"min_mon_release_name":"reef","num_mons":3},"osdmap":{"epoch":640,"num_osds":8,"num_up_osds":8,"osd_up_since":1695503591,"num_in_osds":6,"osd_in_since":1695503801,"num_remapped_pgs":0},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":103}],"num_pgs":103,"num_pools":4,"num_objects":24,"data_bytes":595872,"bytes_used":4539117568,"bytes_avail":571431096320,"bytes_total":575970213888},"fsmap":{"epoch":8,"id":1,"up":1,"in":1,"max":1,"by_rank":[{"filesystem_id":1,"rank":0,"name":"foo.smithi026.znfpnl","status":"up:active","gid":14658}],"up:standby":1},"mgrmap":{"available":true,"num_standbys":1,"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"services":{"dashboard":"https://172.21.15.26:8443/","prometheus":"http://172.21.15.26:9283/"}},"servicemap":{"epoch":30,"modified":"2023-09-23T21:17:37.170873+0000","services":{}},"progress_events":{"fc746e9d-3f76-434d-b6d2-8fc9e8a72a5e":{"message":"Upgrade to 18.2.0-460-gbb0daeff (4m)\n [============................] (remaining: 5m)","progress":0.44999998807907104,"add_to_ceph_s":true}}} 2023-09-23T21:19:14.601 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:19:14.669 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:19:14.704 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:14.922 INFO:teuthology.orchestra.run.smithi026.stderr:nodeep-scrub is unset 2023-09-23T21:19:15.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:19:14 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/1975881269' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:19:15.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:19:14 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:19:15.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:19:14 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/2385890925' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-23T21:19:15.381 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:19:14 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/1975881269' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:19:15.381 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:19:14 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:19:15.381 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:19:14 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/2385890925' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-23T21:19:15.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:19:14 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/1975881269' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:19:15.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:19:14 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:19:15.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:19:14 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/2385890925' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-23T21:19:15.728 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_historic_ops 2023-09-23T21:19:15.781 INFO:tasks.cephadm.ceph_manager.ceph:no progress seen, keeping timeout for now 2023-09-23T21:19:15.781 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T21:19:16.032 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:19:15 smithi026 ceph-mon[179688]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T21:19:16.032 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:19:15 smithi026 ceph-mon[179688]: Cluster is now healthy 2023-09-23T21:19:16.032 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:19:15 smithi026 ceph-mon[179688]: failed to encode map e641 with expected crc 2023-09-23T21:19:16.032 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:19:15 smithi026 ceph-mon[179688]: failed to encode map e641 with expected crc 2023-09-23T21:19:16.032 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:19:15 smithi026 ceph-mon[179688]: failed to encode map e641 with expected crc 2023-09-23T21:19:16.032 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:19:15 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T21:19:16.033 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:19:15 smithi026 ceph-mon[179688]: osdmap e641: 8 total, 8 up, 6 in 2023-09-23T21:19:16.033 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:19:15 smithi026 ceph-mon[179688]: pgmap v1584: 103 pgs: 103 active+clean; 582 KiB data, 4.2 GiB used, 532 GiB / 536 GiB avail 2023-09-23T21:19:16.033 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:19:15 smithi026 ceph-mon[175151]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T21:19:16.033 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:19:15 smithi026 ceph-mon[175151]: Cluster is now healthy 2023-09-23T21:19:16.033 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:19:15 smithi026 ceph-mon[175151]: failed to encode map e641 with expected crc 2023-09-23T21:19:16.033 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:19:15 smithi026 ceph-mon[175151]: failed to encode map e641 with expected crc 2023-09-23T21:19:16.033 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:19:15 smithi026 ceph-mon[175151]: failed to encode map e641 with expected crc 2023-09-23T21:19:16.033 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:19:15 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T21:19:16.033 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:19:15 smithi026 ceph-mon[175151]: osdmap e641: 8 total, 8 up, 6 in 2023-09-23T21:19:16.033 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:19:15 smithi026 ceph-mon[175151]: pgmap v1584: 103 pgs: 103 active+clean; 582 KiB data, 4.2 GiB used, 532 GiB / 536 GiB avail 2023-09-23T21:19:16.103 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T21:19:16.107 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:16.108 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:19:16.209 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:19:16.238 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:16.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:19:15 smithi070 ceph-mon[168282]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T21:19:16.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:19:15 smithi070 ceph-mon[168282]: Cluster is now healthy 2023-09-23T21:19:16.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:19:15 smithi070 ceph-mon[168282]: failed to encode map e641 with expected crc 2023-09-23T21:19:16.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:19:15 smithi070 ceph-mon[168282]: failed to encode map e641 with expected crc 2023-09-23T21:19:16.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:19:15 smithi070 ceph-mon[168282]: failed to encode map e641 with expected crc 2023-09-23T21:19:16.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:19:15 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T21:19:16.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:19:15 smithi070 ceph-mon[168282]: osdmap e641: 8 total, 8 up, 6 in 2023-09-23T21:19:16.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:19:15 smithi070 ceph-mon[168282]: pgmap v1584: 103 pgs: 103 active+clean; 582 KiB data, 4.2 GiB used, 532 GiB / 536 GiB avail 2023-09-23T21:19:16.644 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:19:16.647 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:16.648 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:19:16.748 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:19:16.781 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:16.817 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:19:16.820 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:16.820 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:19:16.921 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:19:16.959 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:17.028 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:19:17.047 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T21:19:17.053 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:17.053 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:19:17.154 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:19:17.185 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:17.252 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T21:19:17.257 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:17.257 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:19:17.324 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:19:17.358 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:19:17.390 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:17.427 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:19:17.430 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:17.431 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:19:17.433 DEBUG:teuthology.orchestra.run.smithi026:> cd -- /home/ubuntu/cephtest/clone.client.0/qa/workunits && if test -e Makefile ; then make ; fi && find -executable -type f -printf '%P\0' >/home/ubuntu/cephtest/workunits.list.client.0 2023-09-23T21:19:17.446 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0: {} 2023-09-23T21:19:17.461 INFO:tasks.workunit.client.0.smithi026.stdout:for d in direct_io fs ; do ( cd $d ; make all ) ; done 2023-09-23T21:19:17.463 INFO:tasks.workunit.client.0.smithi026.stdout:make[1]: Entering directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/direct_io' 2023-09-23T21:19:17.463 INFO:tasks.workunit.client.0.smithi026.stdout:cc -Wall -Wextra -D_GNU_SOURCE direct_io_test.c -o direct_io_test 2023-09-23T21:19:17.467 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1: {} 2023-09-23T21:19:17.486 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2: {} 2023-09-23T21:19:17.510 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3: {} 2023-09-23T21:19:17.526 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4: {} 2023-09-23T21:19:17.532 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:19:17.545 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T21:19:17.557 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:19:17 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:19:17.557 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:19:17 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:19:17.560 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:17.571 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6: {} 2023-09-23T21:19:17.584 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T21:19:17.586 INFO:teuthology.orchestra.run.smithi026.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T21:19:17.586 INFO:teuthology.orchestra.run.smithi026.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T21:19:17.586 INFO:teuthology.orchestra.run.smithi026.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T21:19:17.586 INFO:teuthology.orchestra.run.smithi026.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T21:19:17.586 INFO:teuthology.orchestra.run.smithi026.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T21:19:17.586 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T21:19:17.586 INFO:teuthology.orchestra.run.smithi026.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T21:19:17.586 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T21:19:17.593 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:19:17 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:19:17.594 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:19:17 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:19:17.594 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:19:17 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:19:17.594 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:19:17 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:19:17.594 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:19:17.597 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:17.598 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:19:17.637 INFO:tasks.workunit.client.0.smithi026.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_sync_io.c -o test_sync_io 2023-09-23T21:19:17.698 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:19:17.705 INFO:tasks.workunit.client.0.smithi026.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_short_dio_read.c -o test_short_dio_read 2023-09-23T21:19:17.725 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:17.750 INFO:tasks.workunit.client.0.smithi026.stdout:make[1]: Leaving directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/direct_io' 2023-09-23T21:19:17.752 INFO:tasks.workunit.client.0.smithi026.stdout:make[1]: Entering directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/fs' 2023-09-23T21:19:17.752 INFO:tasks.workunit.client.0.smithi026.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_o_trunc.c -o test_o_trunc 2023-09-23T21:19:17.758 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:19:17.760 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:17.761 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:19:17.791 INFO:tasks.workunit.client.0.smithi026.stdout:make[1]: Leaving directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/fs' 2023-09-23T21:19:17.831 DEBUG:teuthology.orchestra.run.smithi026:> set -ex 2023-09-23T21:19:17.831 DEBUG:teuthology.orchestra.run.smithi026:> dd if=/home/ubuntu/cephtest/workunits.list.client.0 of=/dev/stdout 2023-09-23T21:19:17.862 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:19:17.888 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:17.897 INFO:tasks.workunit:Running workunits matching cls/test_cls_rbd.sh on client.0... 2023-09-23T21:19:17.898 INFO:tasks.workunit:Running workunit cls/test_cls_rbd.sh... 2023-09-23T21:19:17.898 DEBUG:teuthology.orchestra.run.smithi026:workunit test cls/test_cls_rbd.sh> mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=quincy TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/cls/test_cls_rbd.sh 2023-09-23T21:19:17.921 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:19:17.924 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:17.924 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:19:18.025 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:19:18.045 INFO:tasks.workunit.client.0.smithi026.stdout:Running main() from gmock_main.cc 2023-09-23T21:19:18.045 INFO:tasks.workunit.client.0.smithi026.stdout:[==========] Running 70 tests from 1 test suite. 2023-09-23T21:19:18.045 INFO:tasks.workunit.client.0.smithi026.stdout:[----------] Global test environment set-up. 2023-09-23T21:19:18.045 INFO:tasks.workunit.client.0.smithi026.stdout:[----------] 70 tests from TestClsRbd 2023-09-23T21:19:18.058 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:18.197 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T21:19:18.614 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:19:18.616 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:18.617 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:19:18.718 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:19:18.750 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:18.787 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:19:18.789 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:19:18 smithi070 ceph-mon[168282]: pgmap v1585: 103 pgs: 103 active+clean; 582 KiB data, 4.2 GiB used, 532 GiB / 536 GiB avail 2023-09-23T21:19:18.789 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:19:18 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:19:18.789 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:19:18 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/1945935921' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:19:18.789 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:18.790 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:19:18.804 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:19:18 smithi026 ceph-mon[175151]: pgmap v1585: 103 pgs: 103 active+clean; 582 KiB data, 4.2 GiB used, 532 GiB / 536 GiB avail 2023-09-23T21:19:18.804 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:19:18 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:19:18.804 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:19:18 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/1945935921' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:19:18.804 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:19:18 smithi026 ceph-mon[179688]: pgmap v1585: 103 pgs: 103 active+clean; 582 KiB data, 4.2 GiB used, 532 GiB / 536 GiB avail 2023-09-23T21:19:18.804 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:19:18 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:19:18.804 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:19:18 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/1945935921' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:19:18.891 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:19:18.919 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:18.954 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:19:18.957 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:18.957 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:19:19.058 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:19:19.085 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:19.117 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:19:19.120 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:19.120 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:19:19.206 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:19:19.221 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:19:19.257 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:19.295 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:19:19.299 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:19.299 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:19:19.400 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:19:19.429 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:19.457 INFO:teuthology.orchestra.run.smithi026.stdout: 2023-09-23T21:19:19.463 INFO:teuthology.orchestra.run.smithi026.stderr:dumped all 2023-09-23T21:19:19.520 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:19:19.523 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:19.523 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:19:19.553 INFO:teuthology.orchestra.run.smithi026.stdout:ERROR: (22) Invalid argument 2023-09-23T21:19:19.553 INFO:teuthology.orchestra.run.smithi026.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T21:19:19.578 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] TestClsRbd.get_all_features 2023-09-23T21:19:19.591 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] TestClsRbd.get_all_features (14 ms) 2023-09-23T21:19:19.592 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] TestClsRbd.copyup 2023-09-23T21:19:19.624 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:19:19.650 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:19.683 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:19:19.686 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:19.686 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:19:19.775 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] TestClsRbd.copyup (183 ms) 2023-09-23T21:19:19.776 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] TestClsRbd.sparse_copyup 2023-09-23T21:19:19.786 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] TestClsRbd.sparse_copyup (11 ms) 2023-09-23T21:19:19.786 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] TestClsRbd.get_and_set_id 2023-09-23T21:19:19.787 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:19:19.792 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] TestClsRbd.get_and_set_id (7 ms) 2023-09-23T21:19:19.792 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] TestClsRbd.add_remove_child 2023-09-23T21:19:19.818 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:19:19 smithi026 ceph-mon[175151]: failed to encode map e642 with expected crc 2023-09-23T21:19:19.819 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:19:19 smithi026 ceph-mon[175151]: failed to encode map e642 with expected crc 2023-09-23T21:19:19.819 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:19:19 smithi026 ceph-mon[175151]: osdmap e642: 8 total, 8 up, 6 in 2023-09-23T21:19:19.819 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:19:19 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/4261198288' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-453899-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2023-09-23T21:19:19.819 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 21:19:19 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[164029]: ::ffff:172.21.15.70 - - [23/Sep/2023:21:19:19] "GET /metrics HTTP/1.1" 200 33950 "" "Prometheus/2.43.0" 2023-09-23T21:19:19.819 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:19:19 smithi026 ceph-mon[179688]: failed to encode map e642 with expected crc 2023-09-23T21:19:19.820 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:19:19 smithi026 ceph-mon[179688]: failed to encode map e642 with expected crc 2023-09-23T21:19:19.820 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:19:19 smithi026 ceph-mon[179688]: osdmap e642: 8 total, 8 up, 6 in 2023-09-23T21:19:19.820 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:19:19 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/4261198288' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-453899-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2023-09-23T21:19:19.820 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] TestClsRbd.add_remove_child (27 ms) 2023-09-23T21:19:19.820 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] TestClsRbd.directory_methods 2023-09-23T21:19:19.822 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:19.845 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] TestClsRbd.directory_methods (25 ms) 2023-09-23T21:19:19.845 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] TestClsRbd.create 2023-09-23T21:19:19.860 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] TestClsRbd.create (15 ms) 2023-09-23T21:19:19.860 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] TestClsRbd.get_features 2023-09-23T21:19:19.861 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] TestClsRbd.get_features (2 ms) 2023-09-23T21:19:19.861 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] TestClsRbd.get_object_prefix 2023-09-23T21:19:19.863 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] TestClsRbd.get_object_prefix (2 ms) 2023-09-23T21:19:19.863 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] TestClsRbd.get_create_timestamp 2023-09-23T21:19:19.865 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] TestClsRbd.get_create_timestamp (1 ms) 2023-09-23T21:19:19.865 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] TestClsRbd.get_access_timestamp 2023-09-23T21:19:19.865 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] TestClsRbd.get_access_timestamp (1 ms) 2023-09-23T21:19:19.866 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] TestClsRbd.get_modify_timestamp 2023-09-23T21:19:19.867 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] TestClsRbd.get_modify_timestamp (2 ms) 2023-09-23T21:19:19.867 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] TestClsRbd.get_data_pool 2023-09-23T21:19:19.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:19:19 smithi070 ceph-mon[168282]: failed to encode map e642 with expected crc 2023-09-23T21:19:19.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:19:19 smithi070 ceph-mon[168282]: failed to encode map e642 with expected crc 2023-09-23T21:19:19.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:19:19 smithi070 ceph-mon[168282]: osdmap e642: 8 total, 8 up, 6 in 2023-09-23T21:19:19.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:19:19 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/4261198288' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-453899-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2023-09-23T21:19:19.870 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] TestClsRbd.get_data_pool (3 ms) 2023-09-23T21:19:19.870 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] TestClsRbd.get_size 2023-09-23T21:19:19.876 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] TestClsRbd.get_size (6 ms) 2023-09-23T21:19:19.876 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] TestClsRbd.set_size 2023-09-23T21:19:19.881 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] TestClsRbd.set_size (4 ms) 2023-09-23T21:19:19.881 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] TestClsRbd.protection_status 2023-09-23T21:19:19.900 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] TestClsRbd.protection_status (19 ms) 2023-09-23T21:19:19.900 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] TestClsRbd.snapshot_limits 2023-09-23T21:19:19.906 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] TestClsRbd.snapshot_limits (7 ms) 2023-09-23T21:19:19.906 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] TestClsRbd.parents_v1 2023-09-23T21:19:19.947 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] TestClsRbd.parents_v1 (41 ms) 2023-09-23T21:19:19.947 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] TestClsRbd.parents_v2 2023-09-23T21:19:19.977 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] TestClsRbd.parents_v2 (30 ms) 2023-09-23T21:19:19.977 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] TestClsRbd.snapshots 2023-09-23T21:19:19.999 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] TestClsRbd.snapshots (22 ms) 2023-09-23T21:19:20.000 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] TestClsRbd.snapid_race 2023-09-23T21:19:20.003 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] TestClsRbd.snapid_race (3 ms) 2023-09-23T21:19:20.003 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] TestClsRbd.stripingv2 2023-09-23T21:19:20.012 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] TestClsRbd.stripingv2 (10 ms) 2023-09-23T21:19:20.012 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] TestClsRbd.object_map_save 2023-09-23T21:19:20.014 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] TestClsRbd.object_map_save (1 ms) 2023-09-23T21:19:20.014 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] TestClsRbd.object_map_resize 2023-09-23T21:19:20.018 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] TestClsRbd.object_map_resize (5 ms) 2023-09-23T21:19:20.018 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] TestClsRbd.object_map_update 2023-09-23T21:19:20.022 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] TestClsRbd.object_map_update (4 ms) 2023-09-23T21:19:20.022 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] TestClsRbd.object_map_load_enoent 2023-09-23T21:19:20.022 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] TestClsRbd.object_map_load_enoent (0 ms) 2023-09-23T21:19:20.022 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] TestClsRbd.object_map_snap_add 2023-09-23T21:19:20.026 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] TestClsRbd.object_map_snap_add (4 ms) 2023-09-23T21:19:20.026 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] TestClsRbd.object_map_snap_remove 2023-09-23T21:19:20.030 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] TestClsRbd.object_map_snap_remove (4 ms) 2023-09-23T21:19:20.030 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] TestClsRbd.flags 2023-09-23T21:19:20.035 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] TestClsRbd.flags (5 ms) 2023-09-23T21:19:20.035 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] TestClsRbd.metadata 2023-09-23T21:19:20.058 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] TestClsRbd.metadata (23 ms) 2023-09-23T21:19:20.059 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] TestClsRbd.set_features 2023-09-23T21:19:20.065 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] TestClsRbd.set_features (7 ms) 2023-09-23T21:19:20.065 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] TestClsRbd.mirror 2023-09-23T21:19:20.099 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] TestClsRbd.mirror (33 ms) 2023-09-23T21:19:20.099 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] TestClsRbd.mirror_image 2023-09-23T21:19:20.114 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] TestClsRbd.mirror_image (16 ms) 2023-09-23T21:19:20.115 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] TestClsRbd.mirror_image_status 2023-09-23T21:19:20.308 INFO:teuthology.orchestra.run.smithi026.stdout:{"pg_ready":true,"pg_map":{"version":1587,"stamp":"2023-09-23T21:19:19.211317+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":595872,"num_objects":24,"num_object_clones":0,"num_object_copies":72,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":24,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":82,"num_bytes_recovered":544062,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":5046,"ondisk_log_size":5046,"up":309,"acting":309,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":310,"num_osds":6,"num_per_pool_osds":6,"num_per_pool_omap_osds":6,"kb":562470912,"kb_used":4454460,"kb_used_data":75084,"kb_used_omap":125,"kb_used_meta":4379202,"kb_avail":558016452,"statfs":{"total":575970213888,"available":571408846848,"internally_reserved":0,"allocated":76886016,"data_stored":62933588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":128662,"internal_metadata":4484303210},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":68,"apply_latency_ms":68,"commit_latency_ns":68000000,"apply_latency_ns":68000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"7.303933"},"pg_stats":[{"pgid":"12.15","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-09-23T21:19:18.561631+0000","last_change":"2023-09-23T21:19:18.561631+0000","last_active":"2023-09-23T21:19:18.561631+0000","last_peered":"2023-09-23T21:19:18.561631+0000","last_clean":"2023-09-23T21:19:18.561631+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-09-23T21:19:18.561631+0000","last_undegraded":"2023-09-23T21:19:18.561631+0000","last_fullsized":"2023-09-23T21:19:18.561631+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:19:18.561631+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:19:18.561631+0000","last_clean_scrub_stamp":"2023-09-23T21:19:18.561631+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"1.18","version":"79'161","reported_seq":665,"reported_epoch":640,"state":"active+clean","last_fresh":"2023-09-23T21:19:12.219934+0000","last_change":"2023-09-23T21:17:39.906535+0000","last_active":"2023-09-23T21:19:12.219934+0000","last_peered":"2023-09-23T21:19:12.219934+0000","last_clean":"2023-09-23T21:19:12.219934+0000","last_became_active":"2023-09-23T21:17:39.906154+0000","last_became_peered":"2023-09-23T21:17:39.906154+0000","last_unstale":"2023-09-23T21:19:12.219934+0000","last_undegraded":"2023-09-23T21:19:12.219934+0000","last_fullsized":"2023-09-23T21:19:12.219934+0000","mapping_epoch":585,"log_start":"0'0","ondisk_log_start":"0'0","created":479,"last_epoch_clean":586,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:15:39.474823+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:15:39.474823+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:48:39.784037+0000","scrub_duration":0.074664395999999994,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":525376,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1a","version":"68'1","reported_seq":231,"reported_epoch":640,"state":"active+clean","last_fresh":"2023-09-23T21:19:11.909152+0000","last_change":"2023-09-23T21:13:13.291403+0000","last_active":"2023-09-23T21:19:11.909152+0000","last_peered":"2023-09-23T21:19:11.909152+0000","last_clean":"2023-09-23T21:19:11.909152+0000","last_became_active":"2023-09-23T21:13:13.291123+0000","last_became_peered":"2023-09-23T21:13:13.291123+0000","last_unstale":"2023-09-23T21:19:11.909152+0000","last_undegraded":"2023-09-23T21:19:11.909152+0000","last_fullsized":"2023-09-23T21:19:11.909152+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":495,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:45:15.427430+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,0],"acting":[4,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":300,"reported_epoch":640,"state":"active+clean","last_fresh":"2023-09-23T21:19:11.910413+0000","last_change":"2023-09-23T21:12:10.767963+0000","last_active":"2023-09-23T21:19:11.910413+0000","last_peered":"2023-09-23T21:19:11.910413+0000","last_clean":"2023-09-23T21:19:11.910413+0000","last_became_active":"2023-09-23T21:12:10.767126+0000","last_became_peered":"2023-09-23T21:12:10.767126+0000","last_unstale":"2023-09-23T21:19:11.910413+0000","last_undegraded":"2023-09-23T21:19:11.910413+0000","last_fullsized":"2023-09-23T21:19:11.910413+0000","mapping_epoch":470,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:45:47.138277+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,5],"acting":[4,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.16","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-09-23T21:19:18.561631+0000","last_change":"2023-09-23T21:19:18.561631+0000","last_active":"2023-09-23T21:19:18.561631+0000","last_peered":"2023-09-23T21:19:18.561631+0000","last_clean":"2023-09-23T21:19:18.561631+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-09-23T21:19:18.561631+0000","last_undegraded":"2023-09-23T21:19:18.561631+0000","last_fullsized":"2023-09-23T21:19:18.561631+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:19:18.561631+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:19:18.561631+0000","last_clean_scrub_stamp":"2023-09-23T21:19:18.561631+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"1.1b","version":"79'161","reported_seq":828,"reported_epoch":640,"state":"active+clean","last_fresh":"2023-09-23T21:19:12.220985+0000","last_change":"2023-09-23T21:15:38.423372+0000","last_active":"2023-09-23T21:19:12.220985+0000","last_peered":"2023-09-23T21:19:12.220985+0000","last_clean":"2023-09-23T21:19:12.220985+0000","last_became_active":"2023-09-23T21:15:23.802580+0000","last_became_peered":"2023-09-23T21:15:23.802580+0000","last_unstale":"2023-09-23T21:19:12.220985+0000","last_undegraded":"2023-09-23T21:19:12.220985+0000","last_fullsized":"2023-09-23T21:19:12.220985+0000","mapping_epoch":542,"log_start":"0'0","ondisk_log_start":"0'0","created":479,"last_epoch_clean":543,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:15:38.423271+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:15:38.423271+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:04:46.575186+0000","scrub_duration":0.00258465,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":655,"reported_epoch":639,"state":"active+clean","last_fresh":"2023-09-23T21:19:07.984363+0000","last_change":"2023-09-23T21:16:42.448529+0000","last_active":"2023-09-23T21:19:07.984363+0000","last_peered":"2023-09-23T21:19:07.984363+0000","last_clean":"2023-09-23T21:19:07.984363+0000","last_became_active":"2023-09-23T21:16:42.447803+0000","last_became_peered":"2023-09-23T21:16:42.447803+0000","last_unstale":"2023-09-23T21:19:07.984363+0000","last_undegraded":"2023-09-23T21:19:07.984363+0000","last_fullsized":"2023-09-23T21:19:07.984363+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:41:33.121784+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":658,"reported_epoch":640,"state":"active+clean","last_fresh":"2023-09-23T21:19:12.492269+0000","last_change":"2023-09-23T21:16:42.441896+0000","last_active":"2023-09-23T21:19:12.492269+0000","last_peered":"2023-09-23T21:19:12.492269+0000","last_clean":"2023-09-23T21:19:12.492269+0000","last_became_active":"2023-09-23T21:16:42.441526+0000","last_became_peered":"2023-09-23T21:16:42.441526+0000","last_unstale":"2023-09-23T21:19:12.492269+0000","last_undegraded":"2023-09-23T21:19:12.492269+0000","last_fullsized":"2023-09-23T21:19:12.492269+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:27:36.326225+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"12.17","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-09-23T21:19:18.561631+0000","last_change":"2023-09-23T21:19:18.561631+0000","last_active":"2023-09-23T21:19:18.561631+0000","last_peered":"2023-09-23T21:19:18.561631+0000","last_clean":"2023-09-23T21:19:18.561631+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-09-23T21:19:18.561631+0000","last_undegraded":"2023-09-23T21:19:18.561631+0000","last_fullsized":"2023-09-23T21:19:18.561631+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:19:18.561631+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:19:18.561631+0000","last_clean_scrub_stamp":"2023-09-23T21:19:18.561631+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"1.1a","version":"79'161","reported_seq":738,"reported_epoch":640,"state":"active+clean","last_fresh":"2023-09-23T21:19:11.908896+0000","last_change":"2023-09-23T21:16:45.419221+0000","last_active":"2023-09-23T21:19:11.908896+0000","last_peered":"2023-09-23T21:19:11.908896+0000","last_clean":"2023-09-23T21:19:11.908896+0000","last_became_active":"2023-09-23T21:16:45.418994+0000","last_became_peered":"2023-09-23T21:16:45.418994+0000","last_unstale":"2023-09-23T21:19:11.908896+0000","last_undegraded":"2023-09-23T21:19:11.908896+0000","last_fullsized":"2023-09-23T21:19:11.908896+0000","mapping_epoch":567,"log_start":"0'0","ondisk_log_start":"0'0","created":479,"last_epoch_clean":568,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:15:45.158107+0000","last_deep_scrub":"79'161","last_deep_scrub_stamp":"2023-09-23T21:05:32.630292+0000","last_clean_scrub_stamp":"2023-09-23T21:15:45.158107+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:08:08.342444+0000","scrub_duration":0.001999373,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,5],"acting":[4,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":152,"reported_epoch":640,"state":"active+clean","last_fresh":"2023-09-23T21:19:12.490169+0000","last_change":"2023-09-23T21:13:30.806084+0000","last_active":"2023-09-23T21:19:12.490169+0000","last_peered":"2023-09-23T21:19:12.490169+0000","last_clean":"2023-09-23T21:19:12.490169+0000","last_became_active":"2023-09-23T21:13:30.805743+0000","last_became_peered":"2023-09-23T21:13:30.805743+0000","last_unstale":"2023-09-23T21:19:12.490169+0000","last_undegraded":"2023-09-23T21:19:12.490169+0000","last_fullsized":"2023-09-23T21:19:12.490169+0000","mapping_epoch":501,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:48:10.849281+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":516,"reported_epoch":640,"state":"active+clean","last_fresh":"2023-09-23T21:19:12.488191+0000","last_change":"2023-09-23T21:13:44.410921+0000","last_active":"2023-09-23T21:19:12.488191+0000","last_peered":"2023-09-23T21:19:12.488191+0000","last_clean":"2023-09-23T21:19:12.488191+0000","last_became_active":"2023-09-23T21:13:44.410633+0000","last_became_peered":"2023-09-23T21:13:44.410633+0000","last_unstale":"2023-09-23T21:19:12.488191+0000","last_undegraded":"2023-09-23T21:19:12.488191+0000","last_fullsized":"2023-09-23T21:19:12.488191+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:57:10.116786+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"12.18","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-09-23T21:19:18.561631+0000","last_change":"2023-09-23T21:19:18.561631+0000","last_active":"2023-09-23T21:19:18.561631+0000","last_peered":"2023-09-23T21:19:18.561631+0000","last_clean":"2023-09-23T21:19:18.561631+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-09-23T21:19:18.561631+0000","last_undegraded":"2023-09-23T21:19:18.561631+0000","last_fullsized":"2023-09-23T21:19:18.561631+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:19:18.561631+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:19:18.561631+0000","last_clean_scrub_stamp":"2023-09-23T21:19:18.561631+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"1.15","version":"79'161","reported_seq":599,"reported_epoch":640,"state":"active+clean","last_fresh":"2023-09-23T21:19:12.492646+0000","last_change":"2023-09-23T21:18:35.622513+0000","last_active":"2023-09-23T21:19:12.492646+0000","last_peered":"2023-09-23T21:19:12.492646+0000","last_clean":"2023-09-23T21:19:12.492646+0000","last_became_active":"2023-09-23T21:16:45.425654+0000","last_became_peered":"2023-09-23T21:16:45.425654+0000","last_unstale":"2023-09-23T21:19:12.492646+0000","last_undegraded":"2023-09-23T21:19:12.492646+0000","last_fullsized":"2023-09-23T21:19:12.492646+0000","mapping_epoch":567,"log_start":"0'0","ondisk_log_start":"0'0","created":479,"last_epoch_clean":568,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:18:35.622422+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:18:35.622422+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:21:48.421117+0000","scrub_duration":0.083031242000000005,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,3],"acting":[7,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":617,"reported_epoch":640,"state":"active+clean","last_fresh":"2023-09-23T21:19:12.488079+0000","last_change":"2023-09-23T21:13:12.235728+0000","last_active":"2023-09-23T21:19:12.488079+0000","last_peered":"2023-09-23T21:19:12.488079+0000","last_clean":"2023-09-23T21:19:12.488079+0000","last_became_active":"2023-09-23T21:13:12.216549+0000","last_became_peered":"2023-09-23T21:13:12.216549+0000","last_unstale":"2023-09-23T21:19:12.488079+0000","last_undegraded":"2023-09-23T21:19:12.488079+0000","last_fullsized":"2023-09-23T21:19:12.488079+0000","mapping_epoch":493,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:39:33.324239+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":680,"reported_epoch":639,"state":"active+clean","last_fresh":"2023-09-23T21:19:09.427273+0000","last_change":"2023-09-23T21:12:10.790349+0000","last_active":"2023-09-23T21:19:09.427273+0000","last_peered":"2023-09-23T21:19:09.427273+0000","last_clean":"2023-09-23T21:19:09.427273+0000","last_became_active":"2023-09-23T21:12:10.789977+0000","last_became_peered":"2023-09-23T21:12:10.789977+0000","last_unstale":"2023-09-23T21:19:09.427273+0000","last_undegraded":"2023-09-23T21:19:09.427273+0000","last_fullsized":"2023-09-23T21:19:09.427273+0000","mapping_epoch":470,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:49:40.191147+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,5],"acting":[3,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"12.19","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-09-23T21:19:18.561631+0000","last_change":"2023-09-23T21:19:18.561631+0000","last_active":"2023-09-23T21:19:18.561631+0000","last_peered":"2023-09-23T21:19:18.561631+0000","last_clean":"2023-09-23T21:19:18.561631+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-09-23T21:19:18.561631+0000","last_undegraded":"2023-09-23T21:19:18.561631+0000","last_fullsized":"2023-09-23T21:19:18.561631+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:19:18.561631+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:19:18.561631+0000","last_clean_scrub_stamp":"2023-09-23T21:19:18.561631+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"1.14","version":"79'161","reported_seq":1038,"reported_epoch":640,"state":"active+clean","last_fresh":"2023-09-23T21:19:11.918211+0000","last_change":"2023-09-23T21:16:45.424773+0000","last_active":"2023-09-23T21:19:11.918211+0000","last_peered":"2023-09-23T21:19:11.918211+0000","last_clean":"2023-09-23T21:19:11.918211+0000","last_became_active":"2023-09-23T21:16:45.424458+0000","last_became_peered":"2023-09-23T21:16:45.424458+0000","last_unstale":"2023-09-23T21:19:11.918211+0000","last_undegraded":"2023-09-23T21:19:11.918211+0000","last_fullsized":"2023-09-23T21:19:11.918211+0000","mapping_epoch":567,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":568,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:25.096401+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:25.096401+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:40:17.871561+0000","scrub_duration":0.002859393,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":385,"reported_epoch":640,"state":"active+clean","last_fresh":"2023-09-23T21:19:12.223376+0000","last_change":"2023-09-23T21:16:42.427856+0000","last_active":"2023-09-23T21:19:12.223376+0000","last_peered":"2023-09-23T21:19:12.223376+0000","last_clean":"2023-09-23T21:19:12.223376+0000","last_became_active":"2023-09-23T21:16:42.427529+0000","last_became_peered":"2023-09-23T21:16:42.427529+0000","last_unstale":"2023-09-23T21:19:12.223376+0000","last_undegraded":"2023-09-23T21:19:12.223376+0000","last_fullsized":"2023-09-23T21:19:12.223376+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:24:03.818632+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":635,"reported_epoch":639,"state":"active+clean","last_fresh":"2023-09-23T21:19:09.427302+0000","last_change":"2023-09-23T21:16:42.448719+0000","last_active":"2023-09-23T21:19:09.427302+0000","last_peered":"2023-09-23T21:19:09.427302+0000","last_clean":"2023-09-23T21:19:09.427302+0000","last_became_active":"2023-09-23T21:16:42.448219+0000","last_became_peered":"2023-09-23T21:16:42.448219+0000","last_unstale":"2023-09-23T21:19:09.427302+0000","last_undegraded":"2023-09-23T21:19:09.427302+0000","last_fullsized":"2023-09-23T21:19:09.427302+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:02:28.864357+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"12.1a","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-09-23T21:19:18.561631+0000","last_change":"2023-09-23T21:19:18.561631+0000","last_active":"2023-09-23T21:19:18.561631+0000","last_peered":"2023-09-23T21:19:18.561631+0000","last_clean":"2023-09-23T21:19:18.561631+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-09-23T21:19:18.561631+0000","last_undegraded":"2023-09-23T21:19:18.561631+0000","last_fullsized":"2023-09-23T21:19:18.561631+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:19:18.561631+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:19:18.561631+0000","last_clean_scrub_stamp":"2023-09-23T21:19:18.561631+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"1.17","version":"79'161","reported_seq":788,"reported_epoch":640,"state":"active+clean","last_fresh":"2023-09-23T21:19:12.491953+0000","last_change":"2023-09-23T21:16:45.413663+0000","last_active":"2023-09-23T21:19:12.491953+0000","last_peered":"2023-09-23T21:19:12.491953+0000","last_clean":"2023-09-23T21:19:12.491953+0000","last_became_active":"2023-09-23T21:16:45.413311+0000","last_became_peered":"2023-09-23T21:16:45.413311+0000","last_unstale":"2023-09-23T21:19:12.491953+0000","last_undegraded":"2023-09-23T21:19:12.491953+0000","last_fullsized":"2023-09-23T21:19:12.491953+0000","mapping_epoch":567,"log_start":"0'0","ondisk_log_start":"0'0","created":479,"last_epoch_clean":568,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:15:50.325741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:15:50.325741+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:07:43.285354+0000","scrub_duration":0.0040500149999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":556,"reported_epoch":639,"state":"active+clean","last_fresh":"2023-09-23T21:19:07.984222+0000","last_change":"2023-09-23T21:17:02.182606+0000","last_active":"2023-09-23T21:19:07.984222+0000","last_peered":"2023-09-23T21:19:07.984222+0000","last_clean":"2023-09-23T21:19:07.984222+0000","last_became_active":"2023-09-23T21:16:42.446517+0000","last_became_peered":"2023-09-23T21:16:42.446517+0000","last_unstale":"2023-09-23T21:19:07.984222+0000","last_undegraded":"2023-09-23T21:19:07.984222+0000","last_fullsized":"2023-09-23T21:19:07.984222+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:46:08.337878+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":540,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,4],"acting":[3,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":330,"reported_epoch":640,"state":"active+clean","last_fresh":"2023-09-23T21:19:11.944135+0000","last_change":"2023-09-23T21:13:30.821489+0000","last_active":"2023-09-23T21:19:11.944135+0000","last_peered":"2023-09-23T21:19:11.944135+0000","last_clean":"2023-09-23T21:19:11.944135+0000","last_became_active":"2023-09-23T21:13:30.821125+0000","last_became_peered":"2023-09-23T21:13:30.821125+0000","last_unstale":"2023-09-23T21:19:11.944135+0000","last_undegraded":"2023-09-23T21:19:11.944135+0000","last_fullsized":"2023-09-23T21:19:11.944135+0000","mapping_epoch":501,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:47:18.553417+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,5],"acting":[2,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"12.1b","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-09-23T21:19:18.561631+0000","last_change":"2023-09-23T21:19:18.561631+0000","last_active":"2023-09-23T21:19:18.561631+0000","last_peered":"2023-09-23T21:19:18.561631+0000","last_clean":"2023-09-23T21:19:18.561631+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-09-23T21:19:18.561631+0000","last_undegraded":"2023-09-23T21:19:18.561631+0000","last_fullsized":"2023-09-23T21:19:18.561631+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:19:18.561631+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:19:18.561631+0000","last_clean_scrub_stamp":"2023-09-23T21:19:18.561631+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"1.16","version":"79'161","reported_seq":573,"reported_epoch":640,"state":"active+clean","last_fresh":"2023-09-23T21:19:12.492208+0000","last_change":"2023-09-23T21:15:46.326726+0000","last_active":"2023-09-23T21:19:12.492208+0000","last_peered":"2023-09-23T21:19:12.492208+0000","last_clean":"2023-09-23T21:19:12.492208+0000","last_became_active":"2023-09-23T21:13:33.770407+0000","last_became_peered":"2023-09-23T21:13:33.770407+0000","last_unstale":"2023-09-23T21:19:12.492208+0000","last_undegraded":"2023-09-23T21:19:12.492208+0000","last_fullsized":"2023-09-23T21:19:12.492208+0000","mapping_epoch":504,"log_start":"0'0","ondisk_log_start":"0'0","created":479,"last_epoch_clean":505,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:15:46.326589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:15:46.326589+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:50:23.810054+0000","scrub_duration":0.0062069059999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.14","version":"79'22","reported_seq":244,"reported_epoch":640,"state":"active+clean","last_fresh":"2023-09-23T21:19:11.909825+0000","last_change":"2023-09-23T21:17:02.198735+0000","last_active":"2023-09-23T21:19:11.909825+0000","last_peered":"2023-09-23T21:19:11.909825+0000","last_clean":"2023-09-23T21:19:11.909825+0000","last_became_active":"2023-09-23T21:16:42.492861+0000","last_became_peered":"2023-09-23T21:16:42.492861+0000","last_unstale":"2023-09-23T21:19:11.909825+0000","last_undegraded":"2023-09-23T21:19:11.909825+0000","last_fullsized":"2023-09-23T21:19:11.909825+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:00:09.949965+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":14202,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,2],"acting":[4,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":569,"reported_epoch":640,"state":"active+clean","last_fresh":"2023-09-23T21:19:11.909682+0000","last_change":"2023-09-23T21:12:10.839271+0000","last_active":"2023-09-23T21:19:11.909682+0000","last_peered":"2023-09-23T21:19:11.909682+0000","last_clean":"2023-09-23T21:19:11.909682+0000","last_became_active":"2023-09-23T21:12:10.839042+0000","last_became_peered":"2023-09-23T21:12:10.839042+0000","last_unstale":"2023-09-23T21:19:11.909682+0000","last_undegraded":"2023-09-23T21:19:11.909682+0000","last_fullsized":"2023-09-23T21:19:11.909682+0000","mapping_epoch":470,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:10:26.414737+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,2],"acting":[4,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.1c","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-09-23T21:19:18.561631+0000","last_change":"2023-09-23T21:19:18.561631+0000","last_active":"2023-09-23T21:19:18.561631+0000","last_peered":"2023-09-23T21:19:18.561631+0000","last_clean":"2023-09-23T21:19:18.561631+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-09-23T21:19:18.561631+0000","last_undegraded":"2023-09-23T21:19:18.561631+0000","last_fullsized":"2023-09-23T21:19:18.561631+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:19:18.561631+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:19:18.561631+0000","last_clean_scrub_stamp":"2023-09-23T21:19:18.561631+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"1.11","version":"79'161","reported_seq":578,"reported_epoch":640,"state":"active+clean","last_fresh":"2023-09-23T21:19:12.488404+0000","last_change":"2023-09-23T21:15:47.300109+0000","last_active":"2023-09-23T21:19:12.488404+0000","last_peered":"2023-09-23T21:19:12.488404+0000","last_clean":"2023-09-23T21:19:12.488404+0000","last_became_active":"2023-09-23T21:13:33.767067+0000","last_became_peered":"2023-09-23T21:13:33.767067+0000","last_unstale":"2023-09-23T21:19:12.488404+0000","last_undegraded":"2023-09-23T21:19:12.488404+0000","last_fullsized":"2023-09-23T21:19:12.488404+0000","mapping_epoch":504,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":505,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:15:47.299947+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:15:47.299947+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:04:59.029359+0000","scrub_duration":0.0061353110000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":92,"reported_epoch":640,"state":"active+clean","last_fresh":"2023-09-23T21:19:12.491854+0000","last_change":"2023-09-23T21:16:42.531060+0000","last_active":"2023-09-23T21:19:12.491854+0000","last_peered":"2023-09-23T21:19:12.491854+0000","last_clean":"2023-09-23T21:19:12.491854+0000","last_became_active":"2023-09-23T21:16:42.449198+0000","last_became_peered":"2023-09-23T21:16:42.449198+0000","last_unstale":"2023-09-23T21:19:12.491854+0000","last_undegraded":"2023-09-23T21:19:12.491854+0000","last_fullsized":"2023-09-23T21:19:12.491854+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:30:23.428886+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":202,"reported_epoch":640,"state":"active+clean","last_fresh":"2023-09-23T21:19:11.917930+0000","last_change":"2023-09-23T21:16:42.491023+0000","last_active":"2023-09-23T21:19:11.917930+0000","last_peered":"2023-09-23T21:19:11.917930+0000","last_clean":"2023-09-23T21:19:11.917930+0000","last_became_active":"2023-09-23T21:16:42.489896+0000","last_became_peered":"2023-09-23T21:16:42.489896+0000","last_unstale":"2023-09-23T21:19:11.917930+0000","last_undegraded":"2023-09-23T21:19:11.917930+0000","last_fullsized":"2023-09-23T21:19:11.917930+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:58:54.036098+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,7],"acting":[2,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"12.1d","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-09-23T21:19:18.561631+0000","last_change":"2023-09-23T21:19:18.561631+0000","last_active":"2023-09-23T21:19:18.561631+0000","last_peered":"2023-09-23T21:19:18.561631+0000","last_clean":"2023-09-23T21:19:18.561631+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-09-23T21:19:18.561631+0000","last_undegraded":"2023-09-23T21:19:18.561631+0000","last_fullsized":"2023-09-23T21:19:18.561631+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:19:18.561631+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:19:18.561631+0000","last_clean_scrub_stamp":"2023-09-23T21:19:18.561631+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"1.10","version":"79'161","reported_seq":815,"reported_epoch":640,"state":"active+clean","last_fresh":"2023-09-23T21:19:11.909070+0000","last_change":"2023-09-23T21:16:45.416227+0000","last_active":"2023-09-23T21:19:11.909070+0000","last_peered":"2023-09-23T21:19:11.909070+0000","last_clean":"2023-09-23T21:19:11.909070+0000","last_became_active":"2023-09-23T21:16:45.415841+0000","last_became_peered":"2023-09-23T21:16:45.415841+0000","last_unstale":"2023-09-23T21:19:11.909070+0000","last_undegraded":"2023-09-23T21:19:11.909070+0000","last_fullsized":"2023-09-23T21:19:11.909070+0000","mapping_epoch":567,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":568,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:06:50.336992+0000","last_deep_scrub":"79'161","last_deep_scrub_stamp":"2023-09-23T21:06:50.336992+0000","last_clean_scrub_stamp":"2023-09-23T21:06:50.336992+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:37:30.094658+0000","scrub_duration":0.0026150930000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,7],"acting":[4,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":601,"reported_epoch":640,"state":"active+clean","last_fresh":"2023-09-23T21:19:12.490972+0000","last_change":"2023-09-23T21:16:42.446324+0000","last_active":"2023-09-23T21:19:12.490972+0000","last_peered":"2023-09-23T21:19:12.490972+0000","last_clean":"2023-09-23T21:19:12.490972+0000","last_became_active":"2023-09-23T21:16:42.445997+0000","last_became_peered":"2023-09-23T21:16:42.445997+0000","last_unstale":"2023-09-23T21:19:12.490972+0000","last_undegraded":"2023-09-23T21:19:12.490972+0000","last_fullsized":"2023-09-23T21:19:12.490972+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:46:35.028292+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":411,"reported_epoch":639,"state":"active+clean","last_fresh":"2023-09-23T21:19:07.984108+0000","last_change":"2023-09-23T21:16:42.473872+0000","last_active":"2023-09-23T21:19:07.984108+0000","last_peered":"2023-09-23T21:19:07.984108+0000","last_clean":"2023-09-23T21:19:07.984108+0000","last_became_active":"2023-09-23T21:16:42.473595+0000","last_became_peered":"2023-09-23T21:16:42.473595+0000","last_unstale":"2023-09-23T21:19:07.984108+0000","last_undegraded":"2023-09-23T21:19:07.984108+0000","last_fullsized":"2023-09-23T21:19:07.984108+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:15:19.186428+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"12.1e","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-09-23T21:19:18.561631+0000","last_change":"2023-09-23T21:19:18.561631+0000","last_active":"2023-09-23T21:19:18.561631+0000","last_peered":"2023-09-23T21:19:18.561631+0000","last_clean":"2023-09-23T21:19:18.561631+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-09-23T21:19:18.561631+0000","last_undegraded":"2023-09-23T21:19:18.561631+0000","last_fullsized":"2023-09-23T21:19:18.561631+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:19:18.561631+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:19:18.561631+0000","last_clean_scrub_stamp":"2023-09-23T21:19:18.561631+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"1.13","version":"79'161","reported_seq":719,"reported_epoch":640,"state":"active+clean","last_fresh":"2023-09-23T21:19:12.466977+0000","last_change":"2023-09-23T21:13:14.933096+0000","last_active":"2023-09-23T21:19:12.466977+0000","last_peered":"2023-09-23T21:19:12.466977+0000","last_clean":"2023-09-23T21:19:12.466977+0000","last_became_active":"2023-09-23T21:13:14.932703+0000","last_became_peered":"2023-09-23T21:13:14.932703+0000","last_unstale":"2023-09-23T21:19:12.466977+0000","last_undegraded":"2023-09-23T21:19:12.466977+0000","last_fullsized":"2023-09-23T21:19:12.466977+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:36.420362+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:36.420362+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:29:42.163911+0000","scrub_duration":0.028838444000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":652,"reported_epoch":640,"state":"active+clean","last_fresh":"2023-09-23T21:19:11.908978+0000","last_change":"2023-09-23T21:16:42.420467+0000","last_active":"2023-09-23T21:19:11.908978+0000","last_peered":"2023-09-23T21:19:11.908978+0000","last_clean":"2023-09-23T21:19:11.908978+0000","last_became_active":"2023-09-23T21:16:42.420149+0000","last_became_peered":"2023-09-23T21:16:42.420149+0000","last_unstale":"2023-09-23T21:19:11.908978+0000","last_undegraded":"2023-09-23T21:19:11.908978+0000","last_fullsized":"2023-09-23T21:19:11.908978+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:52:07.355000+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":170,"reported_epoch":640,"state":"active+clean","last_fresh":"2023-09-23T21:19:12.492058+0000","last_change":"2023-09-23T21:16:42.427663+0000","last_active":"2023-09-23T21:19:12.492058+0000","last_peered":"2023-09-23T21:19:12.492058+0000","last_clean":"2023-09-23T21:19:12.492058+0000","last_became_active":"2023-09-23T21:16:42.426961+0000","last_became_peered":"2023-09-23T21:16:42.426961+0000","last_unstale":"2023-09-23T21:19:12.492058+0000","last_undegraded":"2023-09-23T21:19:12.492058+0000","last_fullsized":"2023-09-23T21:19:12.492058+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:44:27.422007+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"12.1f","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-09-23T21:19:18.561631+0000","last_change":"2023-09-23T21:19:18.561631+0000","last_active":"2023-09-23T21:19:18.561631+0000","last_peered":"2023-09-23T21:19:18.561631+0000","last_clean":"2023-09-23T21:19:18.561631+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-09-23T21:19:18.561631+0000","last_undegraded":"2023-09-23T21:19:18.561631+0000","last_fullsized":"2023-09-23T21:19:18.561631+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:19:18.561631+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:19:18.561631+0000","last_clean_scrub_stamp":"2023-09-23T21:19:18.561631+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"1.12","version":"79'161","reported_seq":914,"reported_epoch":640,"state":"active+clean","last_fresh":"2023-09-23T21:19:12.221190+0000","last_change":"2023-09-23T21:16:45.412440+0000","last_active":"2023-09-23T21:19:12.221190+0000","last_peered":"2023-09-23T21:19:12.221190+0000","last_clean":"2023-09-23T21:19:12.221190+0000","last_became_active":"2023-09-23T21:16:45.412029+0000","last_became_peered":"2023-09-23T21:16:45.412029+0000","last_unstale":"2023-09-23T21:19:12.221190+0000","last_undegraded":"2023-09-23T21:19:12.221190+0000","last_fullsized":"2023-09-23T21:19:12.221190+0000","mapping_epoch":567,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":568,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:05:56.785173+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:05:56.785173+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:39:43.432998+0000","scrub_duration":0.0042114370000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,0],"acting":[5,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":162,"reported_epoch":640,"state":"active+clean","last_fresh":"2023-09-23T21:19:12.490259+0000","last_change":"2023-09-23T21:13:38.048791+0000","last_active":"2023-09-23T21:19:12.490259+0000","last_peered":"2023-09-23T21:19:12.490259+0000","last_clean":"2023-09-23T21:19:12.490259+0000","last_became_active":"2023-09-23T21:13:30.801386+0000","last_became_peered":"2023-09-23T21:13:30.801386+0000","last_unstale":"2023-09-23T21:19:12.490259+0000","last_undegraded":"2023-09-23T21:19:12.490259+0000","last_fullsized":"2023-09-23T21:19:12.490259+0000","mapping_epoch":501,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:52:49.170414+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":496,"reported_epoch":639,"state":"active+clean","last_fresh":"2023-09-23T21:19:07.984498+0000","last_change":"2023-09-23T21:16:42.518512+0000","last_active":"2023-09-23T21:19:07.984498+0000","last_peered":"2023-09-23T21:19:07.984498+0000","last_clean":"2023-09-23T21:19:07.984498+0000","last_became_active":"2023-09-23T21:16:42.517464+0000","last_became_peered":"2023-09-23T21:16:42.517464+0000","last_unstale":"2023-09-23T21:19:07.984498+0000","last_undegraded":"2023-09-23T21:19:07.984498+0000","last_fullsized":"2023-09-23T21:19:07.984498+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:47:52.391167+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"12.0","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-09-23T21:19:18.561631+0000","last_change":"2023-09-23T21:19:18.561631+0000","last_active":"2023-09-23T21:19:18.561631+0000","last_peered":"2023-09-23T21:19:18.561631+0000","last_clean":"2023-09-23T21:19:18.561631+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-09-23T21:19:18.561631+0000","last_undegraded":"2023-09-23T21:19:18.561631+0000","last_fullsized":"2023-09-23T21:19:18.561631+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:19:18.561631+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:19:18.561631+0000","last_clean_scrub_stamp":"2023-09-23T21:19:18.561631+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"1.d","version":"79'161","reported_seq":682,"reported_epoch":640,"state":"active+clean","last_fresh":"2023-09-23T21:19:12.492736+0000","last_change":"2023-09-23T21:18:47.440374+0000","last_active":"2023-09-23T21:19:12.492736+0000","last_peered":"2023-09-23T21:19:12.492736+0000","last_clean":"2023-09-23T21:19:12.492736+0000","last_became_active":"2023-09-23T21:16:45.416373+0000","last_became_peered":"2023-09-23T21:16:45.416373+0000","last_unstale":"2023-09-23T21:19:12.492736+0000","last_undegraded":"2023-09-23T21:19:12.492736+0000","last_fullsized":"2023-09-23T21:19:12.492736+0000","mapping_epoch":567,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":568,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:18:47.440263+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:18:47.440263+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:11:36.676449+0000","scrub_duration":0.0023849959999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":90,"reported_epoch":640,"state":"active+clean","last_fresh":"2023-09-23T21:19:12.492372+0000","last_change":"2023-09-23T21:17:02.190320+0000","last_active":"2023-09-23T21:19:12.492372+0000","last_peered":"2023-09-23T21:19:12.492372+0000","last_clean":"2023-09-23T21:19:12.492372+0000","last_became_active":"2023-09-23T21:16:42.451380+0000","last_became_peered":"2023-09-23T21:16:42.451380+0000","last_unstale":"2023-09-23T21:19:12.492372+0000","last_undegraded":"2023-09-23T21:19:12.492372+0000","last_fullsized":"2023-09-23T21:19:12.492372+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:48:58.425821+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":132,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":368,"reported_epoch":640,"state":"active+clean","last_fresh":"2023-09-23T21:19:12.220673+0000","last_change":"2023-09-23T21:16:42.428491+0000","last_active":"2023-09-23T21:19:12.220673+0000","last_peered":"2023-09-23T21:19:12.220673+0000","last_clean":"2023-09-23T21:19:12.220673+0000","last_became_active":"2023-09-23T21:16:42.428166+0000","last_became_peered":"2023-09-23T21:16:42.428166+0000","last_unstale":"2023-09-23T21:19:12.220673+0000","last_undegraded":"2023-09-23T21:19:12.220673+0000","last_fullsized":"2023-09-23T21:19:12.220673+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:23:01.430714+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,4],"acting":[5,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"12.1","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-09-23T21:19:18.561631+0000","last_change":"2023-09-23T21:19:18.561631+0000","last_active":"2023-09-23T21:19:18.561631+0000","last_peered":"2023-09-23T21:19:18.561631+0000","last_clean":"2023-09-23T21:19:18.561631+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-09-23T21:19:18.561631+0000","last_undegraded":"2023-09-23T21:19:18.561631+0000","last_fullsized":"2023-09-23T21:19:18.561631+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:19:18.561631+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:19:18.561631+0000","last_clean_scrub_stamp":"2023-09-23T21:19:18.561631+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"1.c","version":"79'161","reported_seq":1022,"reported_epoch":639,"state":"active+clean","last_fresh":"2023-09-23T21:19:07.983813+0000","last_change":"2023-09-23T21:15:31.992756+0000","last_active":"2023-09-23T21:19:07.983813+0000","last_peered":"2023-09-23T21:19:07.983813+0000","last_clean":"2023-09-23T21:19:07.983813+0000","last_became_active":"2023-09-23T21:13:13.401526+0000","last_became_peered":"2023-09-23T21:13:13.401526+0000","last_unstale":"2023-09-23T21:19:07.983813+0000","last_undegraded":"2023-09-23T21:19:07.983813+0000","last_fullsized":"2023-09-23T21:19:07.983813+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":532,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:15:31.992598+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:15:31.992598+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:41:14.215543+0000","scrub_duration":0.002960678,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":90,"reported_epoch":640,"state":"active+clean","last_fresh":"2023-09-23T21:19:12.492910+0000","last_change":"2023-09-23T21:17:02.215996+0000","last_active":"2023-09-23T21:19:12.492910+0000","last_peered":"2023-09-23T21:19:12.492910+0000","last_clean":"2023-09-23T21:19:12.492910+0000","last_became_active":"2023-09-23T21:16:42.451205+0000","last_became_peered":"2023-09-23T21:16:42.451205+0000","last_unstale":"2023-09-23T21:19:12.492910+0000","last_undegraded":"2023-09-23T21:19:12.492910+0000","last_fullsized":"2023-09-23T21:19:12.492910+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:51:46.876281+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":603,"reported_epoch":640,"state":"active+clean","last_fresh":"2023-09-23T21:19:11.909055+0000","last_change":"2023-09-23T21:13:15.152792+0000","last_active":"2023-09-23T21:19:11.909055+0000","last_peered":"2023-09-23T21:19:11.909055+0000","last_clean":"2023-09-23T21:19:11.909055+0000","last_became_active":"2023-09-23T21:13:14.515084+0000","last_became_peered":"2023-09-23T21:13:14.515084+0000","last_unstale":"2023-09-23T21:19:11.909055+0000","last_undegraded":"2023-09-23T21:19:11.909055+0000","last_fullsized":"2023-09-23T21:19:11.909055+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:08:16.702072+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.2","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-09-23T21:19:18.561631+0000","last_change":"2023-09-23T21:19:18.561631+0000","last_active":"2023-09-23T21:19:18.561631+0000","last_peered":"2023-09-23T21:19:18.561631+0000","last_clean":"2023-09-23T21:19:18.561631+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-09-23T21:19:18.561631+0000","last_undegraded":"2023-09-23T21:19:18.561631+0000","last_fullsized":"2023-09-23T21:19:18.561631+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:19:18.561631+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:19:18.561631+0000","last_clean_scrub_stamp":"2023-09-23T21:19:18.561631+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"1.f","version":"79'161","reported_seq":928,"reported_epoch":640,"state":"active+clean","last_fresh":"2023-09-23T21:19:12.224539+0000","last_change":"2023-09-23T21:11:30.750338+0000","last_active":"2023-09-23T21:19:12.224539+0000","last_peered":"2023-09-23T21:19:12.224539+0000","last_clean":"2023-09-23T21:19:12.224539+0000","last_became_active":"2023-09-23T21:11:30.749953+0000","last_became_peered":"2023-09-23T21:11:30.749953+0000","last_unstale":"2023-09-23T21:19:12.224539+0000","last_undegraded":"2023-09-23T21:19:12.224539+0000","last_fullsized":"2023-09-23T21:19:12.224539+0000","mapping_epoch":455,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":456,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:27.880776+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:27.880776+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:07:10.405723+0000","scrub_duration":0.0014004899999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,3],"acting":[5,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":392,"reported_epoch":640,"state":"active+clean","last_fresh":"2023-09-23T21:19:12.224696+0000","last_change":"2023-09-23T21:17:02.175012+0000","last_active":"2023-09-23T21:19:12.224696+0000","last_peered":"2023-09-23T21:19:12.224696+0000","last_clean":"2023-09-23T21:19:12.224696+0000","last_became_active":"2023-09-23T21:16:42.451114+0000","last_became_peered":"2023-09-23T21:16:42.451114+0000","last_unstale":"2023-09-23T21:19:12.224696+0000","last_undegraded":"2023-09-23T21:19:12.224696+0000","last_fullsized":"2023-09-23T21:19:12.224696+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:00:53.868525+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":420,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":600,"reported_epoch":640,"state":"active+clean","last_fresh":"2023-09-23T21:19:12.220347+0000","last_change":"2023-09-23T21:16:42.432943+0000","last_active":"2023-09-23T21:19:12.220347+0000","last_peered":"2023-09-23T21:19:12.220347+0000","last_clean":"2023-09-23T21:19:12.220347+0000","last_became_active":"2023-09-23T21:16:42.427440+0000","last_became_peered":"2023-09-23T21:16:42.427440+0000","last_unstale":"2023-09-23T21:19:12.220347+0000","last_undegraded":"2023-09-23T21:19:12.220347+0000","last_fullsized":"2023-09-23T21:19:12.220347+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:59:32.044609+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,0],"acting":[5,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"12.3","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-09-23T21:19:18.561631+0000","last_change":"2023-09-23T21:19:18.561631+0000","last_active":"2023-09-23T21:19:18.561631+0000","last_peered":"2023-09-23T21:19:18.561631+0000","last_clean":"2023-09-23T21:19:18.561631+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-09-23T21:19:18.561631+0000","last_undegraded":"2023-09-23T21:19:18.561631+0000","last_fullsized":"2023-09-23T21:19:18.561631+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:19:18.561631+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:19:18.561631+0000","last_clean_scrub_stamp":"2023-09-23T21:19:18.561631+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"1.e","version":"79'161","reported_seq":859,"reported_epoch":640,"state":"active+clean","last_fresh":"2023-09-23T21:19:11.910451+0000","last_change":"2023-09-23T21:16:45.433204+0000","last_active":"2023-09-23T21:19:11.910451+0000","last_peered":"2023-09-23T21:19:11.910451+0000","last_clean":"2023-09-23T21:19:11.910451+0000","last_became_active":"2023-09-23T21:16:45.432836+0000","last_became_peered":"2023-09-23T21:16:45.432836+0000","last_unstale":"2023-09-23T21:19:11.910451+0000","last_undegraded":"2023-09-23T21:19:11.910451+0000","last_fullsized":"2023-09-23T21:19:11.910451+0000","mapping_epoch":567,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":568,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:15:39.046962+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:15:39.046962+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:11:20.720731+0000","scrub_duration":0.0082638769999999993,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":439,"reported_epoch":639,"state":"active+clean","last_fresh":"2023-09-23T21:19:07.984154+0000","last_change":"2023-09-23T21:13:13.402418+0000","last_active":"2023-09-23T21:19:07.984154+0000","last_peered":"2023-09-23T21:19:07.984154+0000","last_clean":"2023-09-23T21:19:07.984154+0000","last_became_active":"2023-09-23T21:13:13.400984+0000","last_became_peered":"2023-09-23T21:13:13.400984+0000","last_unstale":"2023-09-23T21:19:07.984154+0000","last_undegraded":"2023-09-23T21:19:07.984154+0000","last_fullsized":"2023-09-23T21:19:07.984154+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":495,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:28:01.411191+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,0],"acting":[3,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":615,"reported_epoch":640,"state":"active+clean","last_fresh":"2023-09-23T21:19:12.491418+0000","last_change":"2023-09-23T21:13:14.339670+0000","last_active":"2023-09-23T21:19:12.491418+0000","last_peered":"2023-09-23T21:19:12.491418+0000","last_clean":"2023-09-23T21:19:12.491418+0000","last_became_active":"2023-09-23T21:13:14.339341+0000","last_became_peered":"2023-09-23T21:13:14.339341+0000","last_unstale":"2023-09-23T21:19:12.491418+0000","last_undegraded":"2023-09-23T21:19:12.491418+0000","last_fullsized":"2023-09-23T21:19:12.491418+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:23:19.411368+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"12.4","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-09-23T21:19:18.561631+0000","last_change":"2023-09-23T21:19:18.561631+0000","last_active":"2023-09-23T21:19:18.561631+0000","last_peered":"2023-09-23T21:19:18.561631+0000","last_clean":"2023-09-23T21:19:18.561631+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-09-23T21:19:18.561631+0000","last_undegraded":"2023-09-23T21:19:18.561631+0000","last_fullsized":"2023-09-23T21:19:18.561631+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:19:18.561631+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:19:18.561631+0000","last_clean_scrub_stamp":"2023-09-23T21:19:18.561631+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"1.9","version":"79'161","reported_seq":843,"reported_epoch":641,"state":"active+clean","last_fresh":"2023-09-23T21:19:15.861466+0000","last_change":"2023-09-23T21:16:45.433968+0000","last_active":"2023-09-23T21:19:15.861466+0000","last_peered":"2023-09-23T21:19:15.861466+0000","last_clean":"2023-09-23T21:19:15.861466+0000","last_became_active":"2023-09-23T21:16:45.433602+0000","last_became_peered":"2023-09-23T21:16:45.433602+0000","last_unstale":"2023-09-23T21:19:15.861466+0000","last_undegraded":"2023-09-23T21:19:15.861466+0000","last_fullsized":"2023-09-23T21:19:15.861466+0000","mapping_epoch":567,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":568,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:15:43.437369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:15:43.437369+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:16:44.538102+0000","scrub_duration":0.0024129429999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,3],"acting":[2,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":201,"reported_epoch":640,"state":"active+clean","last_fresh":"2023-09-23T21:19:12.491525+0000","last_change":"2023-09-23T21:13:30.793273+0000","last_active":"2023-09-23T21:19:12.491525+0000","last_peered":"2023-09-23T21:19:12.491525+0000","last_clean":"2023-09-23T21:19:12.491525+0000","last_became_active":"2023-09-23T21:13:30.791269+0000","last_became_peered":"2023-09-23T21:13:30.791269+0000","last_unstale":"2023-09-23T21:19:12.491525+0000","last_undegraded":"2023-09-23T21:19:12.491525+0000","last_fullsized":"2023-09-23T21:19:12.491525+0000","mapping_epoch":501,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:38:07.142290+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":234,"reported_epoch":640,"state":"active+clean","last_fresh":"2023-09-23T21:19:11.909301+0000","last_change":"2023-09-23T21:16:42.417057+0000","last_active":"2023-09-23T21:19:11.909301+0000","last_peered":"2023-09-23T21:19:11.909301+0000","last_clean":"2023-09-23T21:19:11.909301+0000","last_became_active":"2023-09-23T21:16:42.416601+0000","last_became_peered":"2023-09-23T21:16:42.416601+0000","last_unstale":"2023-09-23T21:19:11.909301+0000","last_undegraded":"2023-09-23T21:19:11.909301+0000","last_fullsized":"2023-09-23T21:19:11.909301+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:56:52.167562+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,7],"acting":[4,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.5","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-09-23T21:19:18.561631+0000","last_change":"2023-09-23T21:19:18.561631+0000","last_active":"2023-09-23T21:19:18.561631+0000","last_peered":"2023-09-23T21:19:18.561631+0000","last_clean":"2023-09-23T21:19:18.561631+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-09-23T21:19:18.561631+0000","last_undegraded":"2023-09-23T21:19:18.561631+0000","last_fullsized":"2023-09-23T21:19:18.561631+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:19:18.561631+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:19:18.561631+0000","last_clean_scrub_stamp":"2023-09-23T21:19:18.561631+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"1.8","version":"79'161","reported_seq":775,"reported_epoch":640,"state":"active+clean","last_fresh":"2023-09-23T21:19:11.906647+0000","last_change":"2023-09-23T21:15:36.367819+0000","last_active":"2023-09-23T21:19:11.906647+0000","last_peered":"2023-09-23T21:19:11.906647+0000","last_clean":"2023-09-23T21:19:11.906647+0000","last_became_active":"2023-09-23T21:12:30.433217+0000","last_became_peered":"2023-09-23T21:12:30.433217+0000","last_unstale":"2023-09-23T21:19:11.906647+0000","last_undegraded":"2023-09-23T21:19:11.906647+0000","last_fullsized":"2023-09-23T21:19:11.906647+0000","mapping_epoch":359,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:15:36.367640+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:15:36.367640+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:49:35.243653+0000","scrub_duration":0.005826899,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":217,"reported_epoch":640,"state":"active+clean","last_fresh":"2023-09-23T21:19:11.909118+0000","last_change":"2023-09-23T21:12:10.823380+0000","last_active":"2023-09-23T21:19:11.909118+0000","last_peered":"2023-09-23T21:19:11.909118+0000","last_clean":"2023-09-23T21:19:11.909118+0000","last_became_active":"2023-09-23T21:12:10.823044+0000","last_became_peered":"2023-09-23T21:12:10.823044+0000","last_unstale":"2023-09-23T21:19:11.909118+0000","last_undegraded":"2023-09-23T21:19:11.909118+0000","last_fullsized":"2023-09-23T21:19:11.909118+0000","mapping_epoch":470,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:53:29.846536+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,5],"acting":[4,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":218,"reported_epoch":640,"state":"active+clean","last_fresh":"2023-09-23T21:19:11.910195+0000","last_change":"2023-09-23T21:13:30.772451+0000","last_active":"2023-09-23T21:19:11.910195+0000","last_peered":"2023-09-23T21:19:11.910195+0000","last_clean":"2023-09-23T21:19:11.910195+0000","last_became_active":"2023-09-23T21:13:30.772040+0000","last_became_peered":"2023-09-23T21:13:30.772040+0000","last_unstale":"2023-09-23T21:19:11.910195+0000","last_undegraded":"2023-09-23T21:19:11.910195+0000","last_fullsized":"2023-09-23T21:19:11.910195+0000","mapping_epoch":501,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:24:49.246727+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,0],"acting":[4,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.6","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-09-23T21:19:18.561631+0000","last_change":"2023-09-23T21:19:18.561631+0000","last_active":"2023-09-23T21:19:18.561631+0000","last_peered":"2023-09-23T21:19:18.561631+0000","last_clean":"2023-09-23T21:19:18.561631+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-09-23T21:19:18.561631+0000","last_undegraded":"2023-09-23T21:19:18.561631+0000","last_fullsized":"2023-09-23T21:19:18.561631+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:19:18.561631+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:19:18.561631+0000","last_clean_scrub_stamp":"2023-09-23T21:19:18.561631+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"1.b","version":"79'161","reported_seq":658,"reported_epoch":640,"state":"active+clean","last_fresh":"2023-09-23T21:19:12.491273+0000","last_change":"2023-09-23T21:15:36.340009+0000","last_active":"2023-09-23T21:19:12.491273+0000","last_peered":"2023-09-23T21:19:12.491273+0000","last_clean":"2023-09-23T21:19:12.491273+0000","last_became_active":"2023-09-23T21:13:13.372328+0000","last_became_peered":"2023-09-23T21:13:13.372328+0000","last_unstale":"2023-09-23T21:19:12.491273+0000","last_undegraded":"2023-09-23T21:19:12.491273+0000","last_fullsized":"2023-09-23T21:19:12.491273+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":495,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:15:36.339885+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:15:36.339885+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:01:26.605006+0000","scrub_duration":0.0068473279999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":218,"reported_epoch":640,"state":"active+clean","last_fresh":"2023-09-23T21:19:11.908880+0000","last_change":"2023-09-23T21:16:42.425103+0000","last_active":"2023-09-23T21:19:11.908880+0000","last_peered":"2023-09-23T21:19:11.908880+0000","last_clean":"2023-09-23T21:19:11.908880+0000","last_became_active":"2023-09-23T21:16:42.424543+0000","last_became_peered":"2023-09-23T21:16:42.424543+0000","last_unstale":"2023-09-23T21:19:11.908880+0000","last_undegraded":"2023-09-23T21:19:11.908880+0000","last_fullsized":"2023-09-23T21:19:11.908880+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:11:28.885246+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":572,"reported_epoch":640,"state":"active+clean","last_fresh":"2023-09-23T21:19:11.910654+0000","last_change":"2023-09-23T21:12:10.811698+0000","last_active":"2023-09-23T21:19:11.910654+0000","last_peered":"2023-09-23T21:19:11.910654+0000","last_clean":"2023-09-23T21:19:11.910654+0000","last_became_active":"2023-09-23T21:12:10.789656+0000","last_became_peered":"2023-09-23T21:12:10.789656+0000","last_unstale":"2023-09-23T21:19:11.910654+0000","last_undegraded":"2023-09-23T21:19:11.910654+0000","last_fullsized":"2023-09-23T21:19:11.910654+0000","mapping_epoch":470,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:39:15.112091+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,5],"acting":[4,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.7","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-09-23T21:19:18.561631+0000","last_change":"2023-09-23T21:19:18.561631+0000","last_active":"2023-09-23T21:19:18.561631+0000","last_peered":"2023-09-23T21:19:18.561631+0000","last_clean":"2023-09-23T21:19:18.561631+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-09-23T21:19:18.561631+0000","last_undegraded":"2023-09-23T21:19:18.561631+0000","last_fullsized":"2023-09-23T21:19:18.561631+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:19:18.561631+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:19:18.561631+0000","last_clean_scrub_stamp":"2023-09-23T21:19:18.561631+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"1.a","version":"79'161","reported_seq":801,"reported_epoch":640,"state":"active+clean","last_fresh":"2023-09-23T21:19:11.909801+0000","last_change":"2023-09-23T21:15:46.420777+0000","last_active":"2023-09-23T21:19:11.909801+0000","last_peered":"2023-09-23T21:19:11.909801+0000","last_clean":"2023-09-23T21:19:11.909801+0000","last_became_active":"2023-09-23T21:12:30.437427+0000","last_became_peered":"2023-09-23T21:12:30.437427+0000","last_unstale":"2023-09-23T21:19:11.909801+0000","last_undegraded":"2023-09-23T21:19:11.909801+0000","last_fullsized":"2023-09-23T21:19:11.909801+0000","mapping_epoch":446,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:15:46.420650+0000","last_deep_scrub":"79'161","last_deep_scrub_stamp":"2023-09-23T21:05:32.630292+0000","last_clean_scrub_stamp":"2023-09-23T21:15:46.420650+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:50:23.452475+0000","scrub_duration":0.0072898930000000004,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,4],"acting":[2,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":506,"reported_epoch":639,"state":"active+clean","last_fresh":"2023-09-23T21:19:07.983436+0000","last_change":"2023-09-23T21:16:42.447403+0000","last_active":"2023-09-23T21:19:07.983436+0000","last_peered":"2023-09-23T21:19:07.983436+0000","last_clean":"2023-09-23T21:19:07.983436+0000","last_became_active":"2023-09-23T21:16:42.447080+0000","last_became_peered":"2023-09-23T21:16:42.447080+0000","last_unstale":"2023-09-23T21:19:07.983436+0000","last_undegraded":"2023-09-23T21:19:07.983436+0000","last_fullsized":"2023-09-23T21:19:07.983436+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:15:19.446334+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,4],"acting":[3,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":638,"reported_epoch":639,"state":"active+clean","last_fresh":"2023-09-23T21:19:07.984648+0000","last_change":"2023-09-23T21:16:42.518739+0000","last_active":"2023-09-23T21:19:07.984648+0000","last_peered":"2023-09-23T21:19:07.984648+0000","last_clean":"2023-09-23T21:19:07.984648+0000","last_became_active":"2023-09-23T21:16:42.517945+0000","last_became_peered":"2023-09-23T21:16:42.517945+0000","last_unstale":"2023-09-23T21:19:07.984648+0000","last_undegraded":"2023-09-23T21:19:07.984648+0000","last_fullsized":"2023-09-23T21:19:07.984648+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:47:26.236933+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"12.a","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-09-23T21:19:18.561631+0000","last_change":"2023-09-23T21:19:18.561631+0000","last_active":"2023-09-23T21:19:18.561631+0000","last_peered":"2023-09-23T21:19:18.561631+0000","last_clean":"2023-09-23T21:19:18.561631+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-09-23T21:19:18.561631+0000","last_undegraded":"2023-09-23T21:19:18.561631+0000","last_fullsized":"2023-09-23T21:19:18.561631+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:19:18.561631+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:19:18.561631+0000","last_clean_scrub_stamp":"2023-09-23T21:19:18.561631+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"1.7","version":"79'161","reported_seq":780,"reported_epoch":639,"state":"active+clean","last_fresh":"2023-09-23T21:19:07.984498+0000","last_change":"2023-09-23T21:16:45.425645+0000","last_active":"2023-09-23T21:19:07.984498+0000","last_peered":"2023-09-23T21:19:07.984498+0000","last_clean":"2023-09-23T21:19:07.984498+0000","last_became_active":"2023-09-23T21:16:45.425241+0000","last_became_peered":"2023-09-23T21:16:45.425241+0000","last_unstale":"2023-09-23T21:19:07.984498+0000","last_undegraded":"2023-09-23T21:19:07.984498+0000","last_fullsized":"2023-09-23T21:19:07.984498+0000","mapping_epoch":567,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":568,"parent":"0.0","parent_split_bits":3,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:15:51.016427+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:15:51.016427+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:49:31.176462+0000","scrub_duration":0.0017648640000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":590,"reported_epoch":640,"state":"active+clean","last_fresh":"2023-09-23T21:19:12.491773+0000","last_change":"2023-09-23T21:16:42.518826+0000","last_active":"2023-09-23T21:19:12.491773+0000","last_peered":"2023-09-23T21:19:12.491773+0000","last_clean":"2023-09-23T21:19:12.491773+0000","last_became_active":"2023-09-23T21:16:42.518488+0000","last_became_peered":"2023-09-23T21:16:42.518488+0000","last_unstale":"2023-09-23T21:19:12.491773+0000","last_undegraded":"2023-09-23T21:19:12.491773+0000","last_fullsized":"2023-09-23T21:19:12.491773+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:42:44.144383+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,2],"acting":[0,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":318,"reported_epoch":640,"state":"active+clean","last_fresh":"2023-09-23T21:19:12.222218+0000","last_change":"2023-09-23T21:05:04.326889+0000","last_active":"2023-09-23T21:19:12.222218+0000","last_peered":"2023-09-23T21:19:12.222218+0000","last_clean":"2023-09-23T21:19:12.222218+0000","last_became_active":"2023-09-23T21:05:04.326032+0000","last_became_peered":"2023-09-23T21:05:04.326032+0000","last_unstale":"2023-09-23T21:19:12.222218+0000","last_undegraded":"2023-09-23T21:19:12.222218+0000","last_fullsized":"2023-09-23T21:19:12.222218+0000","mapping_epoch":332,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:00:17.507889+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":650,"reported_epoch":640,"state":"active+clean","last_fresh":"2023-09-23T21:19:12.220993+0000","last_change":"2023-09-23T21:13:13.285608+0000","last_active":"2023-09-23T21:19:12.220993+0000","last_peered":"2023-09-23T21:19:12.220993+0000","last_clean":"2023-09-23T21:19:12.220993+0000","last_became_active":"2023-09-23T21:13:13.285255+0000","last_became_peered":"2023-09-23T21:13:13.285255+0000","last_unstale":"2023-09-23T21:19:12.220993+0000","last_undegraded":"2023-09-23T21:19:12.220993+0000","last_fullsized":"2023-09-23T21:19:12.220993+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":495,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:42:30.087599+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"12.c","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-09-23T21:19:18.561631+0000","last_change":"2023-09-23T21:19:18.561631+0000","last_active":"2023-09-23T21:19:18.561631+0000","last_peered":"2023-09-23T21:19:18.561631+0000","last_clean":"2023-09-23T21:19:18.561631+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-09-23T21:19:18.561631+0000","last_undegraded":"2023-09-23T21:19:18.561631+0000","last_fullsized":"2023-09-23T21:19:18.561631+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:19:18.561631+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:19:18.561631+0000","last_clean_scrub_stamp":"2023-09-23T21:19:18.561631+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"1.1","version":"79'161","reported_seq":725,"reported_epoch":640,"state":"active+clean","last_fresh":"2023-09-23T21:19:12.488036+0000","last_change":"2023-09-23T21:15:34.345290+0000","last_active":"2023-09-23T21:19:12.488036+0000","last_peered":"2023-09-23T21:19:12.488036+0000","last_clean":"2023-09-23T21:19:12.488036+0000","last_became_active":"2023-09-23T21:13:12.203380+0000","last_became_peered":"2023-09-23T21:13:12.203380+0000","last_unstale":"2023-09-23T21:19:12.488036+0000","last_undegraded":"2023-09-23T21:19:12.488036+0000","last_fullsized":"2023-09-23T21:19:12.488036+0000","mapping_epoch":493,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:15:34.345138+0000","last_deep_scrub":"79'161","last_deep_scrub_stamp":"2023-09-23T21:05:30.652189+0000","last_clean_scrub_stamp":"2023-09-23T21:15:34.345138+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:47:50.349409+0000","scrub_duration":0.0057520000000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":309,"reported_epoch":640,"state":"active+clean","last_fresh":"2023-09-23T21:19:12.490903+0000","last_change":"2023-09-23T21:13:13.238617+0000","last_active":"2023-09-23T21:19:12.490903+0000","last_peered":"2023-09-23T21:19:12.490903+0000","last_clean":"2023-09-23T21:19:12.490903+0000","last_became_active":"2023-09-23T21:13:13.232926+0000","last_became_peered":"2023-09-23T21:13:13.232926+0000","last_unstale":"2023-09-23T21:19:12.490903+0000","last_undegraded":"2023-09-23T21:19:12.490903+0000","last_fullsized":"2023-09-23T21:19:12.490903+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":495,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:17:21.823772+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":574,"reported_epoch":640,"state":"active+clean","last_fresh":"2023-09-23T21:19:11.910844+0000","last_change":"2023-09-23T21:13:14.144918+0000","last_active":"2023-09-23T21:19:11.910844+0000","last_peered":"2023-09-23T21:19:11.910844+0000","last_clean":"2023-09-23T21:19:11.910844+0000","last_became_active":"2023-09-23T21:13:14.132388+0000","last_became_peered":"2023-09-23T21:13:14.132388+0000","last_unstale":"2023-09-23T21:19:11.910844+0000","last_undegraded":"2023-09-23T21:19:11.910844+0000","last_fullsized":"2023-09-23T21:19:11.910844+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":495,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:56:02.619905+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,5],"acting":[4,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":481,"reported_epoch":640,"state":"active+clean","last_fresh":"2023-09-23T21:19:11.918128+0000","last_change":"2023-09-23T21:16:42.491111+0000","last_active":"2023-09-23T21:19:11.918128+0000","last_peered":"2023-09-23T21:19:11.918128+0000","last_clean":"2023-09-23T21:19:11.918128+0000","last_became_active":"2023-09-23T21:16:42.490275+0000","last_became_peered":"2023-09-23T21:16:42.490275+0000","last_unstale":"2023-09-23T21:19:11.918128+0000","last_undegraded":"2023-09-23T21:19:11.918128+0000","last_fullsized":"2023-09-23T21:19:11.918128+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:39:11.652564+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,7],"acting":[2,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"12.f","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-09-23T21:19:18.561631+0000","last_change":"2023-09-23T21:19:18.561631+0000","last_active":"2023-09-23T21:19:18.561631+0000","last_peered":"2023-09-23T21:19:18.561631+0000","last_clean":"2023-09-23T21:19:18.561631+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-09-23T21:19:18.561631+0000","last_undegraded":"2023-09-23T21:19:18.561631+0000","last_fullsized":"2023-09-23T21:19:18.561631+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:19:18.561631+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:19:18.561631+0000","last_clean_scrub_stamp":"2023-09-23T21:19:18.561631+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"1.2","version":"79'161","reported_seq":956,"reported_epoch":640,"state":"active+clean","last_fresh":"2023-09-23T21:19:12.490727+0000","last_change":"2023-09-23T21:16:45.417075+0000","last_active":"2023-09-23T21:19:12.490727+0000","last_peered":"2023-09-23T21:19:12.490727+0000","last_clean":"2023-09-23T21:19:12.490727+0000","last_became_active":"2023-09-23T21:16:45.416752+0000","last_became_peered":"2023-09-23T21:16:45.416752+0000","last_unstale":"2023-09-23T21:19:12.490727+0000","last_undegraded":"2023-09-23T21:19:12.490727+0000","last_fullsized":"2023-09-23T21:19:12.490727+0000","mapping_epoch":567,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":568,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:24.802451+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:24.802451+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:01:51.129779+0000","scrub_duration":0.0090861250000000005,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,4],"acting":[0,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":651,"reported_epoch":640,"state":"active+clean","last_fresh":"2023-09-23T21:19:11.906915+0000","last_change":"2023-09-23T21:13:12.206980+0000","last_active":"2023-09-23T21:19:11.906915+0000","last_peered":"2023-09-23T21:19:11.906915+0000","last_clean":"2023-09-23T21:19:11.906915+0000","last_became_active":"2023-09-23T21:13:12.206650+0000","last_became_peered":"2023-09-23T21:13:12.206650+0000","last_unstale":"2023-09-23T21:19:11.906915+0000","last_undegraded":"2023-09-23T21:19:11.906915+0000","last_fullsized":"2023-09-23T21:19:11.906915+0000","mapping_epoch":493,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:10:01.892936+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":635,"reported_epoch":640,"state":"active+clean","last_fresh":"2023-09-23T21:19:11.906775+0000","last_change":"2023-09-23T21:13:13.407073+0000","last_active":"2023-09-23T21:19:11.906775+0000","last_peered":"2023-09-23T21:19:11.906775+0000","last_clean":"2023-09-23T21:19:11.906775+0000","last_became_active":"2023-09-23T21:13:13.395047+0000","last_became_peered":"2023-09-23T21:13:13.395047+0000","last_unstale":"2023-09-23T21:19:11.906775+0000","last_undegraded":"2023-09-23T21:19:11.906775+0000","last_fullsized":"2023-09-23T21:19:11.906775+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":495,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:04:43.384190+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,5],"acting":[2,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":170,"reported_epoch":640,"state":"active+clean","last_fresh":"2023-09-23T21:19:12.494815+0000","last_change":"2023-09-23T21:16:42.517729+0000","last_active":"2023-09-23T21:19:12.494815+0000","last_peered":"2023-09-23T21:19:12.494815+0000","last_clean":"2023-09-23T21:19:12.494815+0000","last_became_active":"2023-09-23T21:16:42.517373+0000","last_became_peered":"2023-09-23T21:16:42.517373+0000","last_unstale":"2023-09-23T21:19:12.494815+0000","last_undegraded":"2023-09-23T21:19:12.494815+0000","last_fullsized":"2023-09-23T21:19:12.494815+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:05:57.370550+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"12.d","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-09-23T21:19:18.561631+0000","last_change":"2023-09-23T21:19:18.561631+0000","last_active":"2023-09-23T21:19:18.561631+0000","last_peered":"2023-09-23T21:19:18.561631+0000","last_clean":"2023-09-23T21:19:18.561631+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-09-23T21:19:18.561631+0000","last_undegraded":"2023-09-23T21:19:18.561631+0000","last_fullsized":"2023-09-23T21:19:18.561631+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:19:18.561631+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:19:18.561631+0000","last_clean_scrub_stamp":"2023-09-23T21:19:18.561631+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":446,"reported_epoch":640,"state":"active+clean","last_fresh":"2023-09-23T21:19:11.916947+0000","last_change":"2023-09-23T21:16:42.423216+0000","last_active":"2023-09-23T21:19:11.916947+0000","last_peered":"2023-09-23T21:19:11.916947+0000","last_clean":"2023-09-23T21:19:11.916947+0000","last_became_active":"2023-09-23T21:16:42.422856+0000","last_became_peered":"2023-09-23T21:16:42.422856+0000","last_unstale":"2023-09-23T21:19:11.916947+0000","last_undegraded":"2023-09-23T21:19:11.916947+0000","last_fullsized":"2023-09-23T21:19:11.916947+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:06:54.105798+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"79'161","reported_seq":859,"reported_epoch":640,"state":"active+clean","last_fresh":"2023-09-23T21:19:12.488874+0000","last_change":"2023-09-23T21:16:45.428032+0000","last_active":"2023-09-23T21:19:12.488874+0000","last_peered":"2023-09-23T21:19:12.488874+0000","last_clean":"2023-09-23T21:19:12.488874+0000","last_became_active":"2023-09-23T21:16:45.427674+0000","last_became_peered":"2023-09-23T21:16:45.427674+0000","last_unstale":"2023-09-23T21:19:12.488874+0000","last_undegraded":"2023-09-23T21:19:12.488874+0000","last_fullsized":"2023-09-23T21:19:12.488874+0000","mapping_epoch":567,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":568,"parent":"0.0","parent_split_bits":0,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:20.738273+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:20.738273+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:39:42.441711+0000","scrub_duration":0.0088336339999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,2],"acting":[0,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":648,"reported_epoch":640,"state":"active+clean","last_fresh":"2023-09-23T21:19:12.220911+0000","last_change":"2023-09-23T21:11:08.760417+0000","last_active":"2023-09-23T21:19:12.220911+0000","last_peered":"2023-09-23T21:19:12.220911+0000","last_clean":"2023-09-23T21:19:12.220911+0000","last_became_active":"2023-09-23T21:11:08.760150+0000","last_became_peered":"2023-09-23T21:11:08.760150+0000","last_unstale":"2023-09-23T21:19:12.220911+0000","last_undegraded":"2023-09-23T21:19:12.220911+0000","last_fullsized":"2023-09-23T21:19:12.220911+0000","mapping_epoch":443,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":444,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:23:10.849768+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":235,"reported_epoch":640,"state":"active+clean","last_fresh":"2023-09-23T21:19:11.910291+0000","last_change":"2023-09-23T21:16:42.424717+0000","last_active":"2023-09-23T21:19:11.910291+0000","last_peered":"2023-09-23T21:19:11.910291+0000","last_clean":"2023-09-23T21:19:11.910291+0000","last_became_active":"2023-09-23T21:16:42.424376+0000","last_became_peered":"2023-09-23T21:16:42.424376+0000","last_unstale":"2023-09-23T21:19:11.910291+0000","last_undegraded":"2023-09-23T21:19:11.910291+0000","last_fullsized":"2023-09-23T21:19:11.910291+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:25:14.162321+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.e","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-09-23T21:19:18.561631+0000","last_change":"2023-09-23T21:19:18.561631+0000","last_active":"2023-09-23T21:19:18.561631+0000","last_peered":"2023-09-23T21:19:18.561631+0000","last_clean":"2023-09-23T21:19:18.561631+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-09-23T21:19:18.561631+0000","last_undegraded":"2023-09-23T21:19:18.561631+0000","last_fullsized":"2023-09-23T21:19:18.561631+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:19:18.561631+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:19:18.561631+0000","last_clean_scrub_stamp":"2023-09-23T21:19:18.561631+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"1.3","version":"79'161","reported_seq":943,"reported_epoch":640,"state":"active+clean","last_fresh":"2023-09-23T21:19:11.909020+0000","last_change":"2023-09-23T21:13:33.771279+0000","last_active":"2023-09-23T21:19:11.909020+0000","last_peered":"2023-09-23T21:19:11.909020+0000","last_clean":"2023-09-23T21:19:11.909020+0000","last_became_active":"2023-09-23T21:13:33.770980+0000","last_became_peered":"2023-09-23T21:13:33.770980+0000","last_unstale":"2023-09-23T21:19:11.909020+0000","last_undegraded":"2023-09-23T21:19:11.909020+0000","last_fullsized":"2023-09-23T21:19:11.909020+0000","mapping_epoch":504,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":505,"parent":"0.0","parent_split_bits":3,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:05:57.832302+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:05:57.832302+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:19:42.309604+0000","scrub_duration":0.010206847999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":478,"reported_epoch":641,"state":"active+clean","last_fresh":"2023-09-23T21:19:15.388388+0000","last_change":"2023-09-23T21:16:42.455365+0000","last_active":"2023-09-23T21:19:15.388388+0000","last_peered":"2023-09-23T21:19:15.388388+0000","last_clean":"2023-09-23T21:19:15.388388+0000","last_became_active":"2023-09-23T21:16:42.452829+0000","last_became_peered":"2023-09-23T21:16:42.452829+0000","last_unstale":"2023-09-23T21:19:15.388388+0000","last_undegraded":"2023-09-23T21:19:15.388388+0000","last_fullsized":"2023-09-23T21:19:15.388388+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:49:26.303389+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":641,"reported_epoch":640,"state":"active+clean","last_fresh":"2023-09-23T21:19:12.488275+0000","last_change":"2023-09-23T21:13:44.409212+0000","last_active":"2023-09-23T21:19:12.488275+0000","last_peered":"2023-09-23T21:19:12.488275+0000","last_clean":"2023-09-23T21:19:12.488275+0000","last_became_active":"2023-09-23T21:13:44.408958+0000","last_became_peered":"2023-09-23T21:13:44.408958+0000","last_unstale":"2023-09-23T21:19:12.488275+0000","last_undegraded":"2023-09-23T21:19:12.488275+0000","last_fullsized":"2023-09-23T21:19:12.488275+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:03:52.749964+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":170,"reported_epoch":640,"state":"active+clean","last_fresh":"2023-09-23T21:19:12.492156+0000","last_change":"2023-09-23T21:16:42.427490+0000","last_active":"2023-09-23T21:19:12.492156+0000","last_peered":"2023-09-23T21:19:12.492156+0000","last_clean":"2023-09-23T21:19:12.492156+0000","last_became_active":"2023-09-23T21:16:42.426566+0000","last_became_peered":"2023-09-23T21:16:42.426566+0000","last_unstale":"2023-09-23T21:19:12.492156+0000","last_undegraded":"2023-09-23T21:19:12.492156+0000","last_fullsized":"2023-09-23T21:19:12.492156+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:20:26.396005+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,7],"acting":[0,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"12.b","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-09-23T21:19:18.561631+0000","last_change":"2023-09-23T21:19:18.561631+0000","last_active":"2023-09-23T21:19:18.561631+0000","last_peered":"2023-09-23T21:19:18.561631+0000","last_clean":"2023-09-23T21:19:18.561631+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-09-23T21:19:18.561631+0000","last_undegraded":"2023-09-23T21:19:18.561631+0000","last_fullsized":"2023-09-23T21:19:18.561631+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:19:18.561631+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:19:18.561631+0000","last_clean_scrub_stamp":"2023-09-23T21:19:18.561631+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"1.6","version":"79'161","reported_seq":573,"reported_epoch":640,"state":"active+clean","last_fresh":"2023-09-23T21:19:12.492111+0000","last_change":"2023-09-23T21:15:43.333919+0000","last_active":"2023-09-23T21:19:12.492111+0000","last_peered":"2023-09-23T21:19:12.492111+0000","last_clean":"2023-09-23T21:19:12.492111+0000","last_became_active":"2023-09-23T21:13:33.769303+0000","last_became_peered":"2023-09-23T21:13:33.769303+0000","last_unstale":"2023-09-23T21:19:12.492111+0000","last_undegraded":"2023-09-23T21:19:12.492111+0000","last_fullsized":"2023-09-23T21:19:12.492111+0000","mapping_epoch":504,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":505,"parent":"0.0","parent_split_bits":3,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:15:43.333733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:15:43.333733+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:42:29.787357+0000","scrub_duration":0.0092107530000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":86,"reported_epoch":640,"state":"active+clean","last_fresh":"2023-09-23T21:19:12.492562+0000","last_change":"2023-09-23T21:16:42.483813+0000","last_active":"2023-09-23T21:19:12.492562+0000","last_peered":"2023-09-23T21:19:12.492562+0000","last_clean":"2023-09-23T21:19:12.492562+0000","last_became_active":"2023-09-23T21:16:42.482847+0000","last_became_peered":"2023-09-23T21:16:42.482847+0000","last_unstale":"2023-09-23T21:19:12.492562+0000","last_undegraded":"2023-09-23T21:19:12.492562+0000","last_fullsized":"2023-09-23T21:19:12.492562+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:38:57.905679+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":526,"reported_epoch":640,"state":"active+clean","last_fresh":"2023-09-23T21:19:11.922297+0000","last_change":"2023-09-23T21:13:30.790590+0000","last_active":"2023-09-23T21:19:11.922297+0000","last_peered":"2023-09-23T21:19:11.922297+0000","last_clean":"2023-09-23T21:19:11.922297+0000","last_became_active":"2023-09-23T21:13:30.790253+0000","last_became_peered":"2023-09-23T21:13:30.790253+0000","last_unstale":"2023-09-23T21:19:11.922297+0000","last_undegraded":"2023-09-23T21:19:11.922297+0000","last_fullsized":"2023-09-23T21:19:11.922297+0000","mapping_epoch":501,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:23:41.843250+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,0],"acting":[2,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":606,"reported_epoch":640,"state":"active+clean","last_fresh":"2023-09-23T21:19:12.491862+0000","last_change":"2023-09-23T21:16:42.428535+0000","last_active":"2023-09-23T21:19:12.491862+0000","last_peered":"2023-09-23T21:19:12.491862+0000","last_clean":"2023-09-23T21:19:12.491862+0000","last_became_active":"2023-09-23T21:16:42.428149+0000","last_became_peered":"2023-09-23T21:16:42.428149+0000","last_unstale":"2023-09-23T21:19:12.491862+0000","last_undegraded":"2023-09-23T21:19:12.491862+0000","last_fullsized":"2023-09-23T21:19:12.491862+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:38:59.979788+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"12.8","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-09-23T21:19:18.561631+0000","last_change":"2023-09-23T21:19:18.561631+0000","last_active":"2023-09-23T21:19:18.561631+0000","last_peered":"2023-09-23T21:19:18.561631+0000","last_clean":"2023-09-23T21:19:18.561631+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-09-23T21:19:18.561631+0000","last_undegraded":"2023-09-23T21:19:18.561631+0000","last_fullsized":"2023-09-23T21:19:18.561631+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:19:18.561631+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:19:18.561631+0000","last_clean_scrub_stamp":"2023-09-23T21:19:18.561631+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"1.5","version":"79'161","reported_seq":678,"reported_epoch":640,"state":"active+clean","last_fresh":"2023-09-23T21:19:11.910228+0000","last_change":"2023-09-23T21:15:41.108638+0000","last_active":"2023-09-23T21:19:11.910228+0000","last_peered":"2023-09-23T21:19:11.910228+0000","last_clean":"2023-09-23T21:19:11.910228+0000","last_became_active":"2023-09-23T21:13:38.373633+0000","last_became_peered":"2023-09-23T21:13:38.373633+0000","last_unstale":"2023-09-23T21:19:11.910228+0000","last_undegraded":"2023-09-23T21:19:11.910228+0000","last_fullsized":"2023-09-23T21:19:11.910228+0000","mapping_epoch":507,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":3,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:15:41.108579+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:15:41.108579+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:21:58.075649+0000","scrub_duration":0.0061511960000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":209,"reported_epoch":640,"state":"active+clean","last_fresh":"2023-09-23T21:19:12.494885+0000","last_change":"2023-09-23T21:17:02.204086+0000","last_active":"2023-09-23T21:19:12.494885+0000","last_peered":"2023-09-23T21:19:12.494885+0000","last_clean":"2023-09-23T21:19:12.494885+0000","last_became_active":"2023-09-23T21:16:42.517549+0000","last_became_peered":"2023-09-23T21:16:42.517549+0000","last_unstale":"2023-09-23T21:19:12.494885+0000","last_undegraded":"2023-09-23T21:19:12.494885+0000","last_fullsized":"2023-09-23T21:19:12.494885+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:43:07.991844+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":621,"reported_epoch":640,"state":"active+clean","last_fresh":"2023-09-23T21:19:11.910527+0000","last_change":"2023-09-23T21:13:30.807520+0000","last_active":"2023-09-23T21:19:11.910527+0000","last_peered":"2023-09-23T21:19:11.910527+0000","last_clean":"2023-09-23T21:19:11.910527+0000","last_became_active":"2023-09-23T21:13:30.807222+0000","last_became_peered":"2023-09-23T21:13:30.807222+0000","last_unstale":"2023-09-23T21:19:11.910527+0000","last_undegraded":"2023-09-23T21:19:11.910527+0000","last_fullsized":"2023-09-23T21:19:11.910527+0000","mapping_epoch":501,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:40:31.989749+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":501,"reported_epoch":639,"state":"active+clean","last_fresh":"2023-09-23T21:19:07.983498+0000","last_change":"2023-09-23T21:16:42.463411+0000","last_active":"2023-09-23T21:19:07.983498+0000","last_peered":"2023-09-23T21:19:07.983498+0000","last_clean":"2023-09-23T21:19:07.983498+0000","last_became_active":"2023-09-23T21:16:42.463020+0000","last_became_peered":"2023-09-23T21:16:42.463020+0000","last_unstale":"2023-09-23T21:19:07.983498+0000","last_undegraded":"2023-09-23T21:19:07.983498+0000","last_fullsized":"2023-09-23T21:19:07.983498+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:07:34.346129+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"12.9","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-09-23T21:19:18.561631+0000","last_change":"2023-09-23T21:19:18.561631+0000","last_active":"2023-09-23T21:19:18.561631+0000","last_peered":"2023-09-23T21:19:18.561631+0000","last_clean":"2023-09-23T21:19:18.561631+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-09-23T21:19:18.561631+0000","last_undegraded":"2023-09-23T21:19:18.561631+0000","last_fullsized":"2023-09-23T21:19:18.561631+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:19:18.561631+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:19:18.561631+0000","last_clean_scrub_stamp":"2023-09-23T21:19:18.561631+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"1.4","version":"79'161","reported_seq":1047,"reported_epoch":639,"state":"active+clean","last_fresh":"2023-09-23T21:19:07.983836+0000","last_change":"2023-09-23T21:16:45.426136+0000","last_active":"2023-09-23T21:19:07.983836+0000","last_peered":"2023-09-23T21:19:07.983836+0000","last_clean":"2023-09-23T21:19:07.983836+0000","last_became_active":"2023-09-23T21:16:45.425669+0000","last_became_peered":"2023-09-23T21:16:45.425669+0000","last_unstale":"2023-09-23T21:19:07.983836+0000","last_undegraded":"2023-09-23T21:19:07.983836+0000","last_fullsized":"2023-09-23T21:19:07.983836+0000","mapping_epoch":567,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":568,"parent":"0.0","parent_split_bits":3,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:20.012764+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:20.012764+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:52:39.142509+0000","scrub_duration":0.0045251919999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":600,"reported_epoch":640,"state":"active+clean","last_fresh":"2023-09-23T21:19:11.918203+0000","last_change":"2023-09-23T21:16:42.459732+0000","last_active":"2023-09-23T21:19:11.918203+0000","last_peered":"2023-09-23T21:19:11.918203+0000","last_clean":"2023-09-23T21:19:11.918203+0000","last_became_active":"2023-09-23T21:16:42.459413+0000","last_became_peered":"2023-09-23T21:16:42.459413+0000","last_unstale":"2023-09-23T21:19:11.918203+0000","last_undegraded":"2023-09-23T21:19:11.918203+0000","last_fullsized":"2023-09-23T21:19:11.918203+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:37:44.548863+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,3],"acting":[2,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.6","version":"70'1","reported_seq":642,"reported_epoch":640,"state":"active+clean","last_fresh":"2023-09-23T21:19:12.492703+0000","last_change":"2023-09-23T21:17:02.208077+0000","last_active":"2023-09-23T21:19:12.492703+0000","last_peered":"2023-09-23T21:19:12.492703+0000","last_clean":"2023-09-23T21:19:12.492703+0000","last_became_active":"2023-09-23T21:16:42.431308+0000","last_became_peered":"2023-09-23T21:16:42.431308+0000","last_unstale":"2023-09-23T21:19:12.492703+0000","last_undegraded":"2023-09-23T21:19:12.492703+0000","last_fullsized":"2023-09-23T21:19:12.492703+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:16:49.009013+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":215,"reported_epoch":640,"state":"active+clean","last_fresh":"2023-09-23T21:19:11.908879+0000","last_change":"2023-09-23T21:13:12.216895+0000","last_active":"2023-09-23T21:19:11.908879+0000","last_peered":"2023-09-23T21:19:11.908879+0000","last_clean":"2023-09-23T21:19:11.908879+0000","last_became_active":"2023-09-23T21:13:12.216654+0000","last_became_peered":"2023-09-23T21:13:12.216654+0000","last_unstale":"2023-09-23T21:19:11.908879+0000","last_undegraded":"2023-09-23T21:19:11.908879+0000","last_fullsized":"2023-09-23T21:19:11.908879+0000","mapping_epoch":493,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:02:40.430096+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.14","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-09-23T21:19:18.561631+0000","last_change":"2023-09-23T21:19:18.561631+0000","last_active":"2023-09-23T21:19:18.561631+0000","last_peered":"2023-09-23T21:19:18.561631+0000","last_clean":"2023-09-23T21:19:18.561631+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-09-23T21:19:18.561631+0000","last_undegraded":"2023-09-23T21:19:18.561631+0000","last_fullsized":"2023-09-23T21:19:18.561631+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:19:18.561631+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:19:18.561631+0000","last_clean_scrub_stamp":"2023-09-23T21:19:18.561631+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"1.19","version":"79'161","reported_seq":849,"reported_epoch":640,"state":"active+clean","last_fresh":"2023-09-23T21:19:12.488357+0000","last_change":"2023-09-23T21:16:45.428879+0000","last_active":"2023-09-23T21:19:12.488357+0000","last_peered":"2023-09-23T21:19:12.488357+0000","last_clean":"2023-09-23T21:19:12.488357+0000","last_became_active":"2023-09-23T21:16:45.428660+0000","last_became_peered":"2023-09-23T21:16:45.428660+0000","last_unstale":"2023-09-23T21:19:12.488357+0000","last_undegraded":"2023-09-23T21:19:12.488357+0000","last_fullsized":"2023-09-23T21:19:12.488357+0000","mapping_epoch":567,"log_start":"0'0","ondisk_log_start":"0'0","created":479,"last_epoch_clean":568,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:15:48.271791+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:15:48.271791+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:25:26.361045+0000","scrub_duration":0.0057149180000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":170,"reported_epoch":640,"state":"active+clean","last_fresh":"2023-09-23T21:19:12.491789+0000","last_change":"2023-09-23T21:16:42.434830+0000","last_active":"2023-09-23T21:19:12.491789+0000","last_peered":"2023-09-23T21:19:12.491789+0000","last_clean":"2023-09-23T21:19:12.491789+0000","last_became_active":"2023-09-23T21:16:42.430289+0000","last_became_peered":"2023-09-23T21:16:42.430289+0000","last_unstale":"2023-09-23T21:19:12.491789+0000","last_undegraded":"2023-09-23T21:19:12.491789+0000","last_fullsized":"2023-09-23T21:19:12.491789+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:38:48.053766+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":146,"reported_epoch":640,"state":"active+clean","last_fresh":"2023-09-23T21:19:11.922197+0000","last_change":"2023-09-23T21:13:38.373814+0000","last_active":"2023-09-23T21:19:11.922197+0000","last_peered":"2023-09-23T21:19:11.922197+0000","last_clean":"2023-09-23T21:19:11.922197+0000","last_became_active":"2023-09-23T21:13:38.373458+0000","last_became_peered":"2023-09-23T21:13:38.373458+0000","last_unstale":"2023-09-23T21:19:11.922197+0000","last_undegraded":"2023-09-23T21:19:11.922197+0000","last_fullsized":"2023-09-23T21:19:11.922197+0000","mapping_epoch":507,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:23:08.288401+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"12.13","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-09-23T21:19:18.561631+0000","last_change":"2023-09-23T21:19:18.561631+0000","last_active":"2023-09-23T21:19:18.561631+0000","last_peered":"2023-09-23T21:19:18.561631+0000","last_clean":"2023-09-23T21:19:18.561631+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-09-23T21:19:18.561631+0000","last_undegraded":"2023-09-23T21:19:18.561631+0000","last_fullsized":"2023-09-23T21:19:18.561631+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:19:18.561631+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:19:18.561631+0000","last_clean_scrub_stamp":"2023-09-23T21:19:18.561631+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"1.1e","version":"79'161","reported_seq":623,"reported_epoch":639,"state":"active+clean","last_fresh":"2023-09-23T21:19:07.984346+0000","last_change":"2023-09-23T21:16:45.420788+0000","last_active":"2023-09-23T21:19:07.984346+0000","last_peered":"2023-09-23T21:19:07.984346+0000","last_clean":"2023-09-23T21:19:07.984346+0000","last_became_active":"2023-09-23T21:16:45.420448+0000","last_became_peered":"2023-09-23T21:16:45.420448+0000","last_unstale":"2023-09-23T21:19:07.984346+0000","last_undegraded":"2023-09-23T21:19:07.984346+0000","last_fullsized":"2023-09-23T21:19:07.984346+0000","mapping_epoch":567,"log_start":"0'0","ondisk_log_start":"0'0","created":479,"last_epoch_clean":568,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:15:53.061319+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:15:53.061319+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:46:31.982439+0000","scrub_duration":0.007429878,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,4],"acting":[3,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":250,"reported_epoch":640,"state":"active+clean","last_fresh":"2023-09-23T21:19:11.909690+0000","last_change":"2023-09-23T21:16:42.518584+0000","last_active":"2023-09-23T21:19:11.909690+0000","last_peered":"2023-09-23T21:19:11.909690+0000","last_clean":"2023-09-23T21:19:11.909690+0000","last_became_active":"2023-09-23T21:16:42.416632+0000","last_became_peered":"2023-09-23T21:16:42.416632+0000","last_unstale":"2023-09-23T21:19:11.909690+0000","last_undegraded":"2023-09-23T21:19:11.909690+0000","last_fullsized":"2023-09-23T21:19:11.909690+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:02:18.972080+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,7],"acting":[4,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":205,"reported_epoch":640,"state":"active+clean","last_fresh":"2023-09-23T21:19:11.909415+0000","last_change":"2023-09-23T21:12:10.770474+0000","last_active":"2023-09-23T21:19:11.909415+0000","last_peered":"2023-09-23T21:19:11.909415+0000","last_clean":"2023-09-23T21:19:11.909415+0000","last_became_active":"2023-09-23T21:12:10.770184+0000","last_became_peered":"2023-09-23T21:12:10.770184+0000","last_unstale":"2023-09-23T21:19:11.909415+0000","last_undegraded":"2023-09-23T21:19:11.909415+0000","last_fullsized":"2023-09-23T21:19:11.909415+0000","mapping_epoch":470,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:05:09.407151+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.12","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-09-23T21:19:18.561631+0000","last_change":"2023-09-23T21:19:18.561631+0000","last_active":"2023-09-23T21:19:18.561631+0000","last_peered":"2023-09-23T21:19:18.561631+0000","last_clean":"2023-09-23T21:19:18.561631+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-09-23T21:19:18.561631+0000","last_undegraded":"2023-09-23T21:19:18.561631+0000","last_fullsized":"2023-09-23T21:19:18.561631+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:19:18.561631+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:19:18.561631+0000","last_clean_scrub_stamp":"2023-09-23T21:19:18.561631+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":432,"reported_epoch":640,"state":"active+clean","last_fresh":"2023-09-23T21:19:11.920868+0000","last_change":"2023-09-23T21:12:10.771160+0000","last_active":"2023-09-23T21:19:11.920868+0000","last_peered":"2023-09-23T21:19:11.920868+0000","last_clean":"2023-09-23T21:19:11.920868+0000","last_became_active":"2023-09-23T21:12:10.757808+0000","last_became_peered":"2023-09-23T21:12:10.757808+0000","last_unstale":"2023-09-23T21:19:11.920868+0000","last_undegraded":"2023-09-23T21:19:11.920868+0000","last_fullsized":"2023-09-23T21:19:11.920868+0000","mapping_epoch":470,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:30:50.987139+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":215,"reported_epoch":640,"state":"active+clean","last_fresh":"2023-09-23T21:19:11.909020+0000","last_change":"2023-09-23T21:13:12.216602+0000","last_active":"2023-09-23T21:19:11.909020+0000","last_peered":"2023-09-23T21:19:11.909020+0000","last_clean":"2023-09-23T21:19:11.909020+0000","last_became_active":"2023-09-23T21:13:12.216204+0000","last_became_peered":"2023-09-23T21:13:12.216204+0000","last_unstale":"2023-09-23T21:19:11.909020+0000","last_undegraded":"2023-09-23T21:19:11.909020+0000","last_fullsized":"2023-09-23T21:19:11.909020+0000","mapping_epoch":493,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:23:37.043390+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.11","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-09-23T21:19:18.561631+0000","last_change":"2023-09-23T21:19:18.561631+0000","last_active":"2023-09-23T21:19:18.561631+0000","last_peered":"2023-09-23T21:19:18.561631+0000","last_clean":"2023-09-23T21:19:18.561631+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-09-23T21:19:18.561631+0000","last_undegraded":"2023-09-23T21:19:18.561631+0000","last_fullsized":"2023-09-23T21:19:18.561631+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:19:18.561631+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:19:18.561631+0000","last_clean_scrub_stamp":"2023-09-23T21:19:18.561631+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"1.1c","version":"79'161","reported_seq":1070,"reported_epoch":639,"state":"active+clean","last_fresh":"2023-09-23T21:19:07.983721+0000","last_change":"2023-09-23T21:16:45.419779+0000","last_active":"2023-09-23T21:19:07.983721+0000","last_peered":"2023-09-23T21:19:07.983721+0000","last_clean":"2023-09-23T21:19:07.983721+0000","last_became_active":"2023-09-23T21:16:45.419394+0000","last_became_peered":"2023-09-23T21:16:45.419394+0000","last_unstale":"2023-09-23T21:19:07.983721+0000","last_undegraded":"2023-09-23T21:19:07.983721+0000","last_fullsized":"2023-09-23T21:19:07.983721+0000","mapping_epoch":567,"log_start":"0'0","ondisk_log_start":"0'0","created":479,"last_epoch_clean":568,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:15:54.019828+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:15:54.019828+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:59:05.422258+0000","scrub_duration":0.0020962530000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":250,"reported_epoch":640,"state":"active+clean","last_fresh":"2023-09-23T21:19:11.917907+0000","last_change":"2023-09-23T21:16:42.424187+0000","last_active":"2023-09-23T21:19:11.917907+0000","last_peered":"2023-09-23T21:19:11.917907+0000","last_clean":"2023-09-23T21:19:11.917907+0000","last_became_active":"2023-09-23T21:16:42.423805+0000","last_became_peered":"2023-09-23T21:16:42.423805+0000","last_unstale":"2023-09-23T21:19:11.917907+0000","last_undegraded":"2023-09-23T21:19:11.917907+0000","last_fullsized":"2023-09-23T21:19:11.917907+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:55:46.051199+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,7],"acting":[2,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":310,"reported_epoch":640,"state":"active+clean","last_fresh":"2023-09-23T21:19:11.906997+0000","last_change":"2023-09-23T21:13:12.200767+0000","last_active":"2023-09-23T21:19:11.906997+0000","last_peered":"2023-09-23T21:19:11.906997+0000","last_clean":"2023-09-23T21:19:11.906997+0000","last_became_active":"2023-09-23T21:13:12.200435+0000","last_became_peered":"2023-09-23T21:13:12.200435+0000","last_unstale":"2023-09-23T21:19:11.906997+0000","last_undegraded":"2023-09-23T21:19:11.906997+0000","last_fullsized":"2023-09-23T21:19:11.906997+0000","mapping_epoch":493,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:15:44.937886+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"12.10","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-09-23T21:19:18.561631+0000","last_change":"2023-09-23T21:19:18.561631+0000","last_active":"2023-09-23T21:19:18.561631+0000","last_peered":"2023-09-23T21:19:18.561631+0000","last_clean":"2023-09-23T21:19:18.561631+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-09-23T21:19:18.561631+0000","last_undegraded":"2023-09-23T21:19:18.561631+0000","last_fullsized":"2023-09-23T21:19:18.561631+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:19:18.561631+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:19:18.561631+0000","last_clean_scrub_stamp":"2023-09-23T21:19:18.561631+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"1.1d","version":"79'161","reported_seq":785,"reported_epoch":639,"state":"active+clean","last_fresh":"2023-09-23T21:19:10.941440+0000","last_change":"2023-09-23T21:19:10.941440+0000","last_active":"2023-09-23T21:19:10.941440+0000","last_peered":"2023-09-23T21:19:10.941440+0000","last_clean":"2023-09-23T21:19:10.941440+0000","last_became_active":"2023-09-23T21:15:23.801704+0000","last_became_peered":"2023-09-23T21:15:23.801704+0000","last_unstale":"2023-09-23T21:19:10.941440+0000","last_undegraded":"2023-09-23T21:19:10.941440+0000","last_fullsized":"2023-09-23T21:19:10.941440+0000","mapping_epoch":542,"log_start":"0'0","ondisk_log_start":"0'0","created":479,"last_epoch_clean":543,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:19:10.941299+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:19:10.941299+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:31:22.166514+0000","scrub_duration":0.0018618599999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,2],"acting":[3,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":169,"reported_epoch":640,"state":"active+clean","last_fresh":"2023-09-23T21:19:12.494734+0000","last_change":"2023-09-23T21:13:38.380574+0000","last_active":"2023-09-23T21:19:12.494734+0000","last_peered":"2023-09-23T21:19:12.494734+0000","last_clean":"2023-09-23T21:19:12.494734+0000","last_became_active":"2023-09-23T21:13:38.373252+0000","last_became_peered":"2023-09-23T21:13:38.373252+0000","last_unstale":"2023-09-23T21:19:12.494734+0000","last_undegraded":"2023-09-23T21:19:12.494734+0000","last_fullsized":"2023-09-23T21:19:12.494734+0000","mapping_epoch":507,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:58:14.788031+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":11,"num_bytes_recovered":3324,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":215,"reported_epoch":640,"state":"active+clean","last_fresh":"2023-09-23T21:19:11.908880+0000","last_change":"2023-09-23T21:12:10.793610+0000","last_active":"2023-09-23T21:19:11.908880+0000","last_peered":"2023-09-23T21:19:11.908880+0000","last_clean":"2023-09-23T21:19:11.908880+0000","last_became_active":"2023-09-23T21:12:10.793081+0000","last_became_peered":"2023-09-23T21:12:10.793081+0000","last_unstale":"2023-09-23T21:19:11.908880+0000","last_undegraded":"2023-09-23T21:19:11.908880+0000","last_fullsized":"2023-09-23T21:19:11.908880+0000","mapping_epoch":470,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:34:36.291550+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":12,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":80,"num_bytes_recovered":18686,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":118784,"data_stored":21350,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":14322,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":31,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":525376,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2109440,"data_stored":2098784,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4991,"ondisk_log_size":4991,"up":93,"acting":93,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201746,"num_pgs":55,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":659696,"kb_used_data":6888,"kb_used_omap":0,"kb_used_meta":652800,"kb_avail":93085456,"statfs":{"total":95995035648,"available":95319506944,"internally_reserved":0,"allocated":7053312,"data_stored":5315546,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":668467200},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":11,"apply_latency_ms":11,"commit_latency_ns":11000000,"apply_latency_ns":11000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:18:22 2023","interfaces":[{"interface":"back","average":{"1min":1.5209999999999999,"5min":1.474,"15min":2.1080000000000001},"min":{"1min":0.22900000000000001,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":3.968,"5min":37.207999999999998,"15min":37.207999999999998},"last":0.33000000000000002},{"interface":"front","average":{"1min":1.73,"5min":1.748,"15min":2.0720000000000001},"min":{"1min":0.318,"5min":0.253,"15min":0.253},"max":{"1min":5.2910000000000004,"5min":37.122,"15min":37.122},"last":0.376}]},{"osd":1,"last update":"Sat Sep 23 21:18:16 2023","interfaces":[{"interface":"back","average":{"1min":1.2609999999999999,"5min":1.2210000000000001,"15min":1.7969999999999999},"min":{"1min":0.27600000000000002,"5min":0.27600000000000002,"15min":0.26700000000000002},"max":{"1min":6.9130000000000003,"5min":7.5369999999999999,"15min":14.034000000000001},"last":0.504},{"interface":"front","average":{"1min":1.4370000000000001,"5min":1.379,"15min":1.776},"min":{"1min":0.34300000000000003,"5min":0.29999999999999999,"15min":0.26300000000000001},"max":{"1min":3.9300000000000002,"5min":11.002000000000001,"15min":11.226000000000001},"last":0.40400000000000003}]},{"osd":2,"last update":"Sat Sep 23 21:18:38 2023","interfaces":[{"interface":"back","average":{"1min":2.105,"5min":2.0739999999999998,"15min":1.8069999999999999},"min":{"1min":0.27200000000000002,"5min":0.27200000000000002,"15min":0.26200000000000001},"max":{"1min":18.488,"5min":25.471,"15min":25.471},"last":0.36299999999999999},{"interface":"front","average":{"1min":1.9330000000000001,"5min":1.857,"15min":1.752},"min":{"1min":0.28799999999999998,"5min":0.28799999999999998,"15min":0.26700000000000002},"max":{"1min":17.536000000000001,"5min":27.187999999999999,"15min":27.187999999999999},"last":0.48499999999999999}]},{"osd":3,"last update":"Sat Sep 23 21:19:06 2023","interfaces":[{"interface":"back","average":{"1min":1.6220000000000001,"5min":1.6040000000000001,"15min":1.788},"min":{"1min":0.317,"5min":0.30599999999999999,"15min":0.30099999999999999},"max":{"1min":5.7830000000000004,"5min":8.6259999999999994,"15min":12.359999999999999},"last":0.58099999999999996},{"interface":"front","average":{"1min":1.504,"5min":1.8100000000000001,"15min":1.79},"min":{"1min":0.33400000000000002,"5min":0.28799999999999998,"15min":0.28699999999999998},"max":{"1min":4.6769999999999996,"5min":15.82,"15min":15.82},"last":0.57099999999999995}]},{"osd":4,"last update":"Sat Sep 23 21:19:06 2023","interfaces":[{"interface":"back","average":{"1min":0.42199999999999999,"5min":0.78500000000000003,"15min":1.1160000000000001},"min":{"1min":0.27000000000000002,"5min":0.251,"15min":0.248},"max":{"1min":0.92700000000000005,"5min":4.2130000000000001,"15min":11.297000000000001},"last":0.44600000000000001},{"interface":"front","average":{"1min":0.61299999999999999,"5min":0.91800000000000004,"15min":1.0800000000000001},"min":{"1min":0.26800000000000002,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":3.73,"5min":7.1870000000000003,"15min":12.752000000000001},"last":0.45900000000000002}]},{"osd":5,"last update":"Sat Sep 23 21:19:06 2023","interfaces":[{"interface":"back","average":{"1min":0.57199999999999995,"5min":0.91600000000000004,"15min":1.0640000000000001},"min":{"1min":0.28199999999999997,"5min":0.28199999999999997,"15min":0.26000000000000001},"max":{"1min":1.7150000000000001,"5min":6.9740000000000002,"15min":9.4789999999999992},"last":0.55700000000000005},{"interface":"front","average":{"1min":0.65900000000000003,"5min":1.004,"15min":1.157},"min":{"1min":0.32300000000000001,"5min":0.32000000000000001,"15min":0.26200000000000001},"max":{"1min":3.1349999999999998,"5min":6.7439999999999998,"15min":11.257},"last":0.52400000000000002}]},{"osd":6,"last update":"Sat Sep 23 21:18:16 2023","interfaces":[{"interface":"back","average":{"1min":0.86099999999999999,"5min":0.88300000000000001,"15min":1.0620000000000001},"min":{"1min":0.34599999999999997,"5min":0.32000000000000001,"15min":0.30099999999999999},"max":{"1min":5.7069999999999999,"5min":7.5339999999999998,"15min":10.941000000000001},"last":0.54100000000000004},{"interface":"front","average":{"1min":1.379,"5min":1.0429999999999999,"15min":1.181},"min":{"1min":0.41099999999999998,"5min":0.36299999999999999,"15min":0.33100000000000002},"max":{"1min":5.6760000000000002,"5min":5.6760000000000002,"15min":12.715},"last":0.59099999999999997}]}]},{"osd":6,"up_from":0,"seq":704374636829,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":1,"up_from":0,"seq":1503238553762,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":0,"up_from":493,"seq":2117418877002,"num_pgs":56,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":403324,"kb_used_data":8244,"kb_used_omap":36,"kb_used_meta":395035,"kb_avail":93341828,"statfs":{"total":95995035648,"available":95582031872,"internally_reserved":0,"allocated":8441856,"data_stored":5529559,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":37336,"internal_metadata":404516392},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":31,"apply_latency_ms":31,"commit_latency_ns":31000000,"apply_latency_ns":31000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 21:18:27 2023","interfaces":[{"interface":"back","average":{"1min":1.3620000000000001,"5min":1.4410000000000001,"15min":0.98399999999999999},"min":{"1min":0.25900000000000001,"5min":0.214,"15min":0.214},"max":{"1min":6.7779999999999996,"5min":13.779999999999999,"15min":13.779999999999999},"last":0.39000000000000001},{"interface":"front","average":{"1min":2.5499999999999998,"5min":1.7569999999999999,"15min":1.099},"min":{"1min":0.35099999999999998,"5min":0.20799999999999999,"15min":0.20799999999999999},"max":{"1min":10.551,"5min":11.964,"15min":11.964},"last":0.27900000000000003}]},{"osd":2,"last update":"Sat Sep 23 21:18:27 2023","interfaces":[{"interface":"back","average":{"1min":2.0579999999999998,"5min":1.8,"15min":1.036},"min":{"1min":0.36599999999999999,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":7.3449999999999998,"5min":13.637,"15min":13.637},"last":0.25800000000000001},{"interface":"front","average":{"1min":3.258,"5min":2.46,"15min":1.4650000000000001},"min":{"1min":0.30199999999999999,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":10.401999999999999,"5min":26.571000000000002,"15min":26.571000000000002},"last":0.40500000000000003}]},{"osd":3,"last update":"Sat Sep 23 21:18:27 2023","interfaces":[{"interface":"back","average":{"1min":3.9950000000000001,"5min":2.6000000000000001,"15min":1.4450000000000001},"min":{"1min":0.253,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":37.079999999999998,"5min":37.079999999999998,"15min":37.079999999999998},"last":0.373},{"interface":"front","average":{"1min":3.4550000000000001,"5min":2.2029999999999998,"15min":1.4790000000000001},"min":{"1min":0.34699999999999998,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":32.935000000000002,"5min":32.935000000000002,"15min":32.935000000000002},"last":0.35099999999999998}]},{"osd":4,"last update":"Sat Sep 23 21:18:27 2023","interfaces":[{"interface":"back","average":{"1min":1.546,"5min":1.7390000000000001,"15min":1.3080000000000001},"min":{"1min":0.34499999999999997,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":4.7880000000000003,"5min":22.780000000000001,"15min":22.780000000000001},"last":0.56499999999999995},{"interface":"front","average":{"1min":1.9390000000000001,"5min":1.4990000000000001,"15min":1.163},"min":{"1min":0.32500000000000001,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":6.9119999999999999,"5min":22.652000000000001,"15min":22.652000000000001},"last":0.66100000000000003}]},{"osd":5,"last update":"Sat Sep 23 21:18:27 2023","interfaces":[{"interface":"back","average":{"1min":1.5549999999999999,"5min":1.5680000000000001,"15min":1.3460000000000001},"min":{"1min":0.312,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":6.1139999999999999,"5min":8.9329999999999998,"15min":8.9329999999999998},"last":0.73199999999999998},{"interface":"front","average":{"1min":1.9039999999999999,"5min":1.696,"15min":1.274},"min":{"1min":0.40100000000000002,"5min":0.313,"15min":0.313},"max":{"1min":6.5990000000000002,"5min":15.321999999999999,"15min":15.321999999999999},"last":0.81999999999999995}]},{"osd":6,"last update":"Sat Sep 23 21:18:27 2023","interfaces":[{"interface":"back","average":{"1min":1.5720000000000001,"5min":1.4810000000000001,"15min":1.2450000000000001},"min":{"1min":0.372,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":6.9409999999999998,"5min":7.2050000000000001,"15min":7.2050000000000001},"last":0.623},{"interface":"front","average":{"1min":1.647,"5min":1.595,"15min":1.403},"min":{"1min":0.38400000000000001,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":6.266,"5min":8.8889999999999993,"15min":8.8889999999999993},"last":0.70299999999999996}]},{"osd":7,"last update":"Sat Sep 23 21:18:27 2023","interfaces":[{"interface":"back","average":{"1min":1.595,"5min":1.8240000000000001,"15min":1.4139999999999999},"min":{"1min":0.34699999999999998,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":4.8360000000000003,"5min":15.273999999999999,"15min":15.273999999999999},"last":0.64500000000000002},{"interface":"front","average":{"1min":1.75,"5min":1.7709999999999999,"15min":1.3620000000000001},"min":{"1min":0.36199999999999999,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":6.9109999999999996,"5min":33.779000000000003,"15min":33.779000000000003},"last":0.76200000000000001}]}]},{"osd":2,"up_from":93,"seq":399431958874,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1035880,"kb_used_data":8224,"kb_used_omap":81,"kb_used_meta":1027566,"kb_avail":92709272,"statfs":{"total":95995035648,"available":94934294528,"internally_reserved":0,"allocated":8421376,"data_stored":5466548,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":83750,"internal_metadata":1052227802},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":14,"apply_latency_ms":14,"commit_latency_ns":14000000,"apply_latency_ns":14000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:18:23 2023","interfaces":[{"interface":"back","average":{"1min":2.6339999999999999,"5min":1.9370000000000001,"15min":1.2010000000000001},"min":{"1min":0.221,"5min":0.187,"15min":0.187},"max":{"1min":9.9499999999999993,"5min":13.25,"15min":13.25},"last":1.3799999999999999},{"interface":"front","average":{"1min":2.2509999999999999,"5min":2.089,"15min":1.806},"min":{"1min":0.26800000000000002,"5min":0.20200000000000001,"15min":0.20200000000000001},"max":{"1min":9.8849999999999998,"5min":18.111999999999998,"15min":18.111999999999998},"last":2.0230000000000001}]},{"osd":1,"last update":"Sat Sep 23 21:18:23 2023","interfaces":[{"interface":"back","average":{"1min":1.645,"5min":1.756,"15min":2.2629999999999999},"min":{"1min":0.34000000000000002,"5min":0.20599999999999999,"15min":0.188},"max":{"1min":10.535,"5min":10.535,"15min":34.396999999999998},"last":2.1520000000000001},{"interface":"front","average":{"1min":2.2240000000000002,"5min":1.95,"15min":2.3460000000000001},"min":{"1min":0.24099999999999999,"5min":0.24099999999999999,"15min":0.217},"max":{"1min":6.0990000000000002,"5min":18.081,"15min":34.462000000000003},"last":0.64400000000000002}]},{"osd":3,"last update":"Sat Sep 23 21:19:02 2023","interfaces":[{"interface":"back","average":{"1min":2.2949999999999999,"5min":2.3999999999999999,"15min":2.149},"min":{"1min":0.27300000000000002,"5min":0.248,"15min":0.20599999999999999},"max":{"1min":8.141,"5min":20.106999999999999,"15min":35.549999999999997},"last":0.41799999999999998},{"interface":"front","average":{"1min":2.593,"5min":2.9249999999999998,"15min":2.4609999999999999},"min":{"1min":0.34899999999999998,"5min":0.28599999999999998,"15min":0.21299999999999999},"max":{"1min":8.2469999999999999,"5min":20.239000000000001,"15min":36.951999999999998},"last":0.33700000000000002}]},{"osd":4,"last update":"Sat Sep 23 21:18:23 2023","interfaces":[{"interface":"back","average":{"1min":1.2749999999999999,"5min":1.3460000000000001,"15min":1.4450000000000001},"min":{"1min":0.27000000000000002,"5min":0.27000000000000002,"15min":0.249},"max":{"1min":3.8530000000000002,"5min":7.9770000000000003,"15min":10.266999999999999},"last":0.53900000000000003},{"interface":"front","average":{"1min":1.363,"5min":1.5640000000000001,"15min":1.7110000000000001},"min":{"1min":0.38900000000000001,"5min":0.30299999999999999,"15min":0.26700000000000002},"max":{"1min":4.2210000000000001,"5min":10.459,"15min":12.49},"last":0.52800000000000002}]},{"osd":5,"last update":"Sat Sep 23 21:18:23 2023","interfaces":[{"interface":"back","average":{"1min":1.6160000000000001,"5min":1.456,"15min":1.5349999999999999},"min":{"1min":0.32300000000000001,"5min":0.29099999999999998,"15min":0.28000000000000003},"max":{"1min":5.4139999999999997,"5min":7.6180000000000003,"15min":9.5039999999999996},"last":0.46300000000000002},{"interface":"front","average":{"1min":1.474,"5min":1.5169999999999999,"15min":1.6890000000000001},"min":{"1min":0.34899999999999998,"5min":0.33800000000000002,"15min":0.29399999999999998},"max":{"1min":4.2649999999999997,"5min":10.425000000000001,"15min":12.57},"last":0.48699999999999999}]},{"osd":6,"last update":"Sat Sep 23 21:18:23 2023","interfaces":[{"interface":"back","average":{"1min":1.542,"5min":1.633,"15min":1.8979999999999999},"min":{"1min":0.54100000000000004,"5min":0.377,"15min":0.35699999999999998},"max":{"1min":4.2990000000000004,"5min":10.335000000000001,"15min":12.518000000000001},"last":0.44},{"interface":"front","average":{"1min":1.7190000000000001,"5min":1.696,"15min":2.0510000000000002},"min":{"1min":0.501,"5min":0.377,"15min":0.35599999999999998},"max":{"1min":4.7969999999999997,"5min":10.381,"15min":12.544},"last":0.56200000000000006}]},{"osd":7,"last update":"Sat Sep 23 21:18:23 2023","interfaces":[{"interface":"back","average":{"1min":1.6339999999999999,"5min":1.637,"15min":1.6519999999999999},"min":{"1min":0.31,"5min":0.31,"15min":0.29799999999999999},"max":{"1min":5.7480000000000002,"5min":18.021999999999998,"15min":18.021999999999998},"last":0.61199999999999999},{"interface":"front","average":{"1min":2.145,"5min":1.6339999999999999,"15min":1.73},"min":{"1min":0.35999999999999999,"5min":0.34999999999999998,"15min":0.32200000000000001},"max":{"1min":6.694,"5min":9.6349999999999998,"15min":10.92},"last":0.51100000000000001}]}]},{"osd":3,"up_from":98,"seq":420906795347,"num_pgs":61,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1137372,"kb_used_data":37908,"kb_used_omap":7,"kb_used_meta":1099448,"kb_avail":92607780,"statfs":{"total":95995035648,"available":94830366720,"internally_reserved":0,"allocated":38817792,"data_stored":35929601,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7576,"internal_metadata":1125835368},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":12,"apply_latency_ms":12,"commit_latency_ns":12000000,"apply_latency_ns":12000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:18:18 2023","interfaces":[{"interface":"back","average":{"1min":2.2069999999999999,"5min":4.0750000000000002,"15min":1.583},"min":{"1min":0.22500000000000001,"5min":0.18099999999999999,"15min":0.18099999999999999},"max":{"1min":10.439,"5min":147.38900000000001,"15min":147.38900000000001},"last":0.33400000000000002},{"interface":"front","average":{"1min":1.1899999999999999,"5min":3.5859999999999999,"15min":1.6399999999999999},"min":{"1min":0.252,"5min":0.223,"15min":0.223},"max":{"1min":5.7279999999999998,"5min":149.989,"15min":149.989},"last":0.39800000000000002}]},{"osd":1,"last update":"Sat Sep 23 21:18:18 2023","interfaces":[{"interface":"back","average":{"1min":2.2029999999999998,"5min":1.5940000000000001,"15min":2.1160000000000001},"min":{"1min":0.27000000000000002,"5min":0.218,"15min":0.20000000000000001},"max":{"1min":6.1150000000000002,"5min":7.2759999999999998,"15min":22.577999999999999},"last":0.42899999999999999},{"interface":"front","average":{"1min":1.827,"5min":1.2969999999999999,"15min":1.984},"min":{"1min":0.26100000000000001,"5min":0.249,"15min":0.216},"max":{"1min":6.1219999999999999,"5min":7.6829999999999998,"15min":21.617999999999999},"last":0.44400000000000001}]},{"osd":2,"last update":"Sat Sep 23 21:19:03 2023","interfaces":[{"interface":"back","average":{"1min":0.84999999999999998,"5min":1.395,"15min":1.893},"min":{"1min":0.23799999999999999,"5min":0.23799999999999999,"15min":0.20899999999999999},"max":{"1min":3.1000000000000001,"5min":11.321,"15min":56.317999999999998},"last":0.46400000000000002},{"interface":"front","average":{"1min":3.1120000000000001,"5min":2.331,"15min":2.343},"min":{"1min":0.34899999999999998,"5min":0.23400000000000001,"15min":0.216},"max":{"1min":11.177,"5min":17.423999999999999,"15min":55.988},"last":0.375}]},{"osd":4,"last update":"Sat Sep 23 21:19:03 2023","interfaces":[{"interface":"back","average":{"1min":1.0609999999999999,"5min":1.27,"15min":1.472},"min":{"1min":0.36899999999999999,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":5.1589999999999998,"5min":10.807,"15min":21.478999999999999},"last":0.50900000000000001},{"interface":"front","average":{"1min":2.145,"5min":1.4870000000000001,"15min":1.7549999999999999},"min":{"1min":0.39600000000000002,"5min":0.311,"15min":0.28499999999999998},"max":{"1min":11.202,"5min":11.202,"15min":22.402000000000001},"last":0.51600000000000001}]},{"osd":5,"last update":"Sat Sep 23 21:19:03 2023","interfaces":[{"interface":"back","average":{"1min":0.91400000000000003,"5min":1.3109999999999999,"15min":1.6040000000000001},"min":{"1min":0.34999999999999998,"5min":0.313,"15min":0.311},"max":{"1min":3.109,"5min":10.882999999999999,"15min":21.562000000000001},"last":0.58899999999999997},{"interface":"front","average":{"1min":0.90300000000000002,"5min":1.2470000000000001,"15min":1.679},"min":{"1min":0.30199999999999999,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":2.8010000000000002,"5min":5.7300000000000004,"15min":14.029},"last":0.57499999999999996}]},{"osd":6,"last update":"Sat Sep 23 21:18:18 2023","interfaces":[{"interface":"back","average":{"1min":1.6180000000000001,"5min":1.464,"15min":1.839},"min":{"1min":0.39800000000000002,"5min":0.35899999999999999,"15min":0.33900000000000002},"max":{"1min":4.9749999999999996,"5min":7.3410000000000002,"15min":22.446999999999999},"last":0.61599999999999999},{"interface":"front","average":{"1min":1.657,"5min":1.3480000000000001,"15min":1.8280000000000001},"min":{"1min":0.373,"5min":0.373,"15min":0.373},"max":{"1min":7.5949999999999998,"5min":9.3889999999999993,"15min":12.375999999999999},"last":0.626}]},{"osd":7,"last update":"Sat Sep 23 21:19:03 2023","interfaces":[{"interface":"back","average":{"1min":0.91100000000000003,"5min":1.456,"15min":1.74},"min":{"1min":0.41099999999999998,"5min":0.41099999999999998,"15min":0.33900000000000002},"max":{"1min":2.4279999999999999,"5min":10.914999999999999,"15min":21.518999999999998},"last":0.68400000000000005},{"interface":"front","average":{"1min":1.2370000000000001,"5min":1.589,"15min":1.9059999999999999},"min":{"1min":0.38800000000000001,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":3.8130000000000002,"5min":8.2590000000000003,"15min":12.624000000000001},"last":0.66700000000000004}]}]},{"osd":4,"up_from":31,"seq":133143986654,"num_pgs":53,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":589164,"kb_used_data":6948,"kb_used_omap":0,"kb_used_meta":582208,"kb_avail":93155988,"statfs":{"total":95995035648,"available":95391731712,"internally_reserved":0,"allocated":7114752,"data_stored":5380791,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":596180992},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:18:24 2023","interfaces":[{"interface":"back","average":{"1min":2.2690000000000001,"5min":1.673,"15min":1.157},"min":{"1min":0.245,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":20.515000000000001,"5min":28.683,"15min":28.683},"last":0.65100000000000002},{"interface":"front","average":{"1min":3.085,"5min":1.8009999999999999,"15min":1.3560000000000001},"min":{"1min":0.249,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":20.792000000000002,"5min":29.155000000000001,"15min":29.155000000000001},"last":0.41099999999999998}]},{"osd":1,"last update":"Sat Sep 23 21:18:24 2023","interfaces":[{"interface":"back","average":{"1min":1.7330000000000001,"5min":1.2829999999999999,"15min":2.1070000000000002},"min":{"1min":0.28499999999999998,"5min":0.26900000000000002,"15min":0.221},"max":{"1min":8.1020000000000003,"5min":8.1020000000000003,"15min":37.960000000000001},"last":0.66600000000000004},{"interface":"front","average":{"1min":1.55,"5min":1.399,"15min":2.0550000000000002},"min":{"1min":0.33100000000000002,"5min":0.28499999999999998,"15min":0.27000000000000002},"max":{"1min":8.1509999999999998,"5min":13.983000000000001,"15min":37.933},"last":0.56000000000000005}]},{"osd":2,"last update":"Sat Sep 23 21:18:24 2023","interfaces":[{"interface":"back","average":{"1min":2.177,"5min":1.8500000000000001,"15min":1.754},"min":{"1min":0.28399999999999997,"5min":0.26400000000000001,"15min":0.247},"max":{"1min":9.8399999999999999,"5min":17.602,"15min":17.602},"last":0.46400000000000002},{"interface":"front","average":{"1min":1.8839999999999999,"5min":1.7849999999999999,"15min":1.831},"min":{"1min":0.35399999999999998,"5min":0.311,"15min":0.27800000000000002},"max":{"1min":5.5250000000000004,"5min":14.339,"15min":14.417999999999999},"last":0.60199999999999998}]},{"osd":3,"last update":"Sat Sep 23 21:18:55 2023","interfaces":[{"interface":"back","average":{"1min":1.3129999999999999,"5min":2.1579999999999999,"15min":1.9790000000000001},"min":{"1min":0.29499999999999998,"5min":0.27900000000000003,"15min":0.25600000000000001},"max":{"1min":6.1230000000000002,"5min":67.105999999999995,"15min":67.105999999999995},"last":0.52800000000000002},{"interface":"front","average":{"1min":1.5289999999999999,"5min":2.2719999999999998,"15min":1.966},"min":{"1min":0.40400000000000003,"5min":0.32200000000000001,"15min":0.30199999999999999},"max":{"1min":6.1319999999999997,"5min":67.046999999999997,"15min":67.046999999999997},"last":0.63500000000000001}]},{"osd":5,"last update":"Sat Sep 23 21:19:17 2023","interfaces":[{"interface":"back","average":{"1min":0.39400000000000002,"5min":0.95299999999999996,"15min":1.0700000000000001},"min":{"1min":0.26900000000000002,"5min":0.254,"15min":0.222},"max":{"1min":0.73999999999999999,"5min":14.398999999999999,"15min":14.398999999999999},"last":0.73999999999999999},{"interface":"front","average":{"1min":0.41699999999999998,"5min":0.86699999999999999,"15min":1.0589999999999999},"min":{"1min":0.29699999999999999,"5min":0.25800000000000001,"15min":0.255},"max":{"1min":0.57699999999999996,"5min":8.016,"15min":9.8979999999999997},"last":0.57699999999999996}]},{"osd":6,"last update":"Sat Sep 23 21:18:24 2023","interfaces":[{"interface":"back","average":{"1min":1.077,"5min":0.96599999999999997,"15min":1.2649999999999999},"min":{"1min":0.308,"5min":0.27200000000000002,"15min":0.26800000000000002},"max":{"1min":3.2050000000000001,"5min":8.7420000000000009,"15min":24.452999999999999},"last":0.62},{"interface":"front","average":{"1min":0.83799999999999997,"5min":1.1499999999999999,"15min":1.0820000000000001},"min":{"1min":0.38,"5min":0.36399999999999999,"15min":0.32500000000000001},"max":{"1min":2.5409999999999999,"5min":14.374000000000001,"15min":14.387},"last":0.71399999999999997}]},{"osd":7,"last update":"Sat Sep 23 21:18:55 2023","interfaces":[{"interface":"back","average":{"1min":0.70599999999999996,"5min":1.077,"15min":1.1659999999999999},"min":{"1min":0.33800000000000002,"5min":0.32000000000000001,"15min":0.30099999999999999},"max":{"1min":2.6419999999999999,"5min":14.422000000000001,"15min":16.890999999999998},"last":0.68300000000000005},{"interface":"front","average":{"1min":0.70999999999999996,"5min":0.89800000000000002,"15min":1.179},"min":{"1min":0.32900000000000001,"5min":0.29199999999999998,"15min":0.27400000000000002},"max":{"1min":2.6579999999999999,"5min":7.7229999999999999,"15min":11.170999999999999},"last":0.69699999999999995}]}]},{"osd":5,"up_from":39,"seq":167503725020,"num_pgs":48,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":629024,"kb_used_data":6872,"kb_used_omap":0,"kb_used_meta":622144,"kb_avail":93116128,"statfs":{"total":95995035648,"available":95350915072,"internally_reserved":0,"allocated":7036928,"data_stored":5311543,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":637075456},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:18:16 2023","interfaces":[{"interface":"back","average":{"1min":1.452,"5min":1.234,"15min":0.88800000000000001},"min":{"1min":0.22500000000000001,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":5.9829999999999997,"5min":8.673,"15min":8.673},"last":1.1870000000000001},{"interface":"front","average":{"1min":2.5219999999999998,"5min":1.548,"15min":0.96199999999999997},"min":{"1min":0.215,"5min":0.215,"15min":0.215},"max":{"1min":12.259,"5min":12.259,"15min":12.259},"last":0.94299999999999995}]},{"osd":1,"last update":"Sat Sep 23 21:18:16 2023","interfaces":[{"interface":"back","average":{"1min":1.4490000000000001,"5min":1.254,"15min":1.625},"min":{"1min":0.32200000000000001,"5min":0.248,"15min":0.23499999999999999},"max":{"1min":4.3220000000000001,"5min":7.5839999999999996,"15min":25.239999999999998},"last":1.139},{"interface":"front","average":{"1min":1.3700000000000001,"5min":1.304,"15min":1.7070000000000001},"min":{"1min":0.36899999999999999,"5min":0.28299999999999997,"15min":0.252},"max":{"1min":4.9500000000000002,"5min":11.116,"15min":25.253},"last":1.151}]},{"osd":2,"last update":"Sat Sep 23 21:18:35 2023","interfaces":[{"interface":"back","average":{"1min":1.895,"5min":1.5109999999999999,"15min":1.6419999999999999},"min":{"1min":0.27600000000000002,"5min":0.27300000000000002,"15min":0.25800000000000001},"max":{"1min":9.2479999999999993,"5min":9.6560000000000006,"15min":12.634},"last":0.78000000000000003},{"interface":"front","average":{"1min":2.0840000000000001,"5min":1.6000000000000001,"15min":1.663},"min":{"1min":0.29099999999999998,"5min":0.29099999999999998,"15min":0.25700000000000001},"max":{"1min":9.4060000000000006,"5min":9.4060000000000006,"15min":9.7170000000000005},"last":0.92300000000000004}]},{"osd":3,"last update":"Sat Sep 23 21:18:54 2023","interfaces":[{"interface":"back","average":{"1min":1.6870000000000001,"5min":1.5089999999999999,"15min":1.883},"min":{"1min":0.32000000000000001,"5min":0.314,"15min":0.29299999999999998},"max":{"1min":7.3700000000000001,"5min":7.3700000000000001,"15min":25.503},"last":1.044},{"interface":"front","average":{"1min":1.6100000000000001,"5min":1.782,"15min":2.105},"min":{"1min":0.33200000000000002,"5min":0.311,"15min":0.30599999999999999},"max":{"1min":6.5880000000000001,"5min":12.382,"15min":25.466000000000001},"last":0.97399999999999998}]},{"osd":4,"last update":"Sat Sep 23 21:19:12 2023","interfaces":[{"interface":"back","average":{"1min":0.71099999999999997,"5min":0.76600000000000001,"15min":1.1279999999999999},"min":{"1min":0.28699999999999998,"5min":0.253,"15min":0.253},"max":{"1min":2.8599999999999999,"5min":5.0279999999999996,"15min":12.776},"last":0.496},{"interface":"front","average":{"1min":0.65400000000000003,"5min":0.83699999999999997,"15min":1.1359999999999999},"min":{"1min":0.27500000000000002,"5min":0.246,"15min":0.246},"max":{"1min":2.4399999999999999,"5min":5.149,"15min":11.973000000000001},"last":0.59999999999999998}]},{"osd":6,"last update":"Sat Sep 23 21:18:16 2023","interfaces":[{"interface":"back","average":{"1min":1.0649999999999999,"5min":0.99299999999999999,"15min":1.087},"min":{"1min":0.32200000000000001,"5min":0.28000000000000003,"15min":0.25900000000000001},"max":{"1min":5.8520000000000003,"5min":5.8520000000000003,"15min":8.0909999999999993},"last":0.627},{"interface":"front","average":{"1min":1.095,"5min":1.0700000000000001,"15min":1.2330000000000001},"min":{"1min":0.39700000000000002,"5min":0.34300000000000003,"15min":0.317},"max":{"1min":7.1470000000000002,"5min":10.999000000000001,"15min":10.999000000000001},"last":0.64700000000000002}]},{"osd":7,"last update":"Sat Sep 23 21:18:54 2023","interfaces":[{"interface":"back","average":{"1min":0.95499999999999996,"5min":0.94299999999999995,"15min":1.821},"min":{"1min":0.33600000000000002,"5min":0.32400000000000001,"15min":0.31},"max":{"1min":4.0519999999999996,"5min":5.1769999999999996,"15min":189.405},"last":0.57799999999999996},{"interface":"front","average":{"1min":0.97799999999999998,"5min":0.96699999999999997,"15min":1.9490000000000001},"min":{"1min":0.34999999999999998,"5min":0.34999999999999998,"15min":0.32500000000000001},"max":{"1min":5.0910000000000002,"5min":5.0910000000000002,"15min":189.685},"last":0.69499999999999995}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8488,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1259,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5322,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":4838,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T21:19:20.328 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_ops_in_flight 2023-09-23T21:19:20.514 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:19:20.518 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:20.518 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:19:20.592 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T21:19:20.618 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:19:20.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:19:20 smithi070 ceph-mon[168282]: pgmap v1587: 135 pgs: 32 unknown, 103 active+clean; 582 KiB data, 4.2 GiB used, 532 GiB / 536 GiB avail 2023-09-23T21:19:20.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:19:20 smithi070 ceph-mon[168282]: from='client.47087 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T21:19:20.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:19:20 smithi070 ceph-mon[168282]: failed to encode map e643 with expected crc 2023-09-23T21:19:20.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:19:20 smithi070 ceph-mon[168282]: failed to encode map e643 with expected crc 2023-09-23T21:19:20.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:19:20 smithi070 ceph-mon[168282]: failed to encode map e643 with expected crc 2023-09-23T21:19:20.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:19:20 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/4261198288' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-453899-1","app": "rados","yes_i_really_mean_it": true}]': finished 2023-09-23T21:19:20.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:19:20 smithi070 ceph-mon[168282]: osdmap e643: 8 total, 8 up, 6 in 2023-09-23T21:19:20.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:19:20 smithi070 ceph-mon[168282]: failed to encode map e643 with expected crc 2023-09-23T21:19:20.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:19:20 smithi070 ceph-mon[168282]: failed to encode map e643 with expected crc 2023-09-23T21:19:20.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:19:20 smithi070 ceph-mon[168282]: failed to encode map e643 with expected crc 2023-09-23T21:19:20.673 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:20.723 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:19:20.726 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:20.727 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:19:20.828 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:19:20.856 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:20.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:19:20 smithi026 ceph-mon[175151]: pgmap v1587: 135 pgs: 32 unknown, 103 active+clean; 582 KiB data, 4.2 GiB used, 532 GiB / 536 GiB avail 2023-09-23T21:19:20.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:19:20 smithi026 ceph-mon[175151]: from='client.47087 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T21:19:20.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:19:20 smithi026 ceph-mon[175151]: failed to encode map e643 with expected crc 2023-09-23T21:19:20.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:19:20 smithi026 ceph-mon[175151]: failed to encode map e643 with expected crc 2023-09-23T21:19:20.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:19:20 smithi026 ceph-mon[175151]: failed to encode map e643 with expected crc 2023-09-23T21:19:20.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:19:20 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/4261198288' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-453899-1","app": "rados","yes_i_really_mean_it": true}]': finished 2023-09-23T21:19:20.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:19:20 smithi026 ceph-mon[175151]: osdmap e643: 8 total, 8 up, 6 in 2023-09-23T21:19:20.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:19:20 smithi026 ceph-mon[175151]: failed to encode map e643 with expected crc 2023-09-23T21:19:20.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:19:20 smithi026 ceph-mon[175151]: failed to encode map e643 with expected crc 2023-09-23T21:19:20.883 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:19:20 smithi026 ceph-mon[175151]: failed to encode map e643 with expected crc 2023-09-23T21:19:20.885 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:19:20 smithi026 ceph-mon[179688]: pgmap v1587: 135 pgs: 32 unknown, 103 active+clean; 582 KiB data, 4.2 GiB used, 532 GiB / 536 GiB avail 2023-09-23T21:19:20.885 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:19:20 smithi026 ceph-mon[179688]: from='client.47087 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T21:19:20.885 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:19:20 smithi026 ceph-mon[179688]: failed to encode map e643 with expected crc 2023-09-23T21:19:20.885 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:19:20 smithi026 ceph-mon[179688]: failed to encode map e643 with expected crc 2023-09-23T21:19:20.885 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:19:20 smithi026 ceph-mon[179688]: failed to encode map e643 with expected crc 2023-09-23T21:19:20.886 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:19:20 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/4261198288' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-453899-1","app": "rados","yes_i_really_mean_it": true}]': finished 2023-09-23T21:19:20.886 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:19:20 smithi026 ceph-mon[179688]: osdmap e643: 8 total, 8 up, 6 in 2023-09-23T21:19:20.886 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:19:20 smithi026 ceph-mon[179688]: failed to encode map e643 with expected crc 2023-09-23T21:19:20.886 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:19:20 smithi026 ceph-mon[179688]: failed to encode map e643 with expected crc 2023-09-23T21:19:20.886 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:19:20 smithi026 ceph-mon[179688]: failed to encode map e643 with expected crc 2023-09-23T21:19:20.890 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:19:20.893 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:20.894 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:19:20.943 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T21:19:20.995 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:19:21.021 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:21.065 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:19:21.068 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:21.069 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:19:21.170 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:19:21.209 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:21.259 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:19:21.264 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:21.264 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:19:21.365 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:19:21.393 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:21.398 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:19:21.479 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:19:21.482 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:21.483 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:19:21.584 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:19:21.616 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:21.663 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:19:21.666 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:21.667 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:19:21.751 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:19:21 smithi070 ceph-mon[168282]: failed to encode map e644 with expected crc 2023-09-23T21:19:21.752 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:19:21 smithi070 ceph-mon[168282]: failed to encode map e644 with expected crc 2023-09-23T21:19:21.752 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:19:21 smithi070 ceph-mon[168282]: osdmap e644: 8 total, 8 up, 6 in 2023-09-23T21:19:21.752 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:19:21 smithi070 ceph-mon[168282]: failed to encode map e644 with expected crc 2023-09-23T21:19:21.752 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:19:21 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/2741865729' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T21:19:21.752 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:19:21 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T21:19:21.767 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:19:21.796 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:21.834 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:19:21.836 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:21.837 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:19:21.881 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:19:21 smithi026 ceph-mon[175151]: failed to encode map e644 with expected crc 2023-09-23T21:19:21.881 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:19:21 smithi026 ceph-mon[175151]: failed to encode map e644 with expected crc 2023-09-23T21:19:21.881 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:19:21 smithi026 ceph-mon[175151]: osdmap e644: 8 total, 8 up, 6 in 2023-09-23T21:19:21.881 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:19:21 smithi026 ceph-mon[175151]: failed to encode map e644 with expected crc 2023-09-23T21:19:21.881 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:19:21 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/2741865729' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T21:19:21.881 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:19:21 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T21:19:21.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:19:21 smithi026 ceph-mon[179688]: failed to encode map e644 with expected crc 2023-09-23T21:19:21.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:19:21 smithi026 ceph-mon[179688]: failed to encode map e644 with expected crc 2023-09-23T21:19:21.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:19:21 smithi026 ceph-mon[179688]: osdmap e644: 8 total, 8 up, 6 in 2023-09-23T21:19:21.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:19:21 smithi026 ceph-mon[179688]: failed to encode map e644 with expected crc 2023-09-23T21:19:21.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:19:21 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/2741865729' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T21:19:21.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:19:21 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T21:19:21.898 INFO:teuthology.orchestra.run.smithi026.stderr:noscrub is set 2023-09-23T21:19:21.938 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:19:21.965 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:22.049 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:19:22.056 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:22.057 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:19:22.158 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:19:22.193 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:22.256 INFO:teuthology.orchestra.run.smithi070.stdout:ERROR: (22) Invalid argument 2023-09-23T21:19:22.256 INFO:teuthology.orchestra.run.smithi070.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T21:19:22.405 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:19:22.408 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:22.408 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:19:22.509 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:19:22.544 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:22.641 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:19:22.645 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:22.646 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:19:22.648 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:19:22 smithi026 ceph-mon[179688]: pgmap v1590: 135 pgs: 5 creating+peering, 18 unknown, 112 active+clean; 582 KiB data, 4.2 GiB used, 532 GiB / 536 GiB avail; 176 KiB/s rd, 108 KiB/s wr, 288 op/s 2023-09-23T21:19:22.649 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:19:22 smithi026 ceph-mon[179688]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:19:22.649 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:19:22 smithi026 ceph-mon[179688]: failed to encode map e645 with expected crc 2023-09-23T21:19:22.649 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:19:22 smithi026 ceph-mon[179688]: failed to encode map e645 with expected crc 2023-09-23T21:19:22.649 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:19:22 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T21:19:22.649 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:19:22 smithi026 ceph-mon[179688]: osdmap e645: 8 total, 8 up, 6 in 2023-09-23T21:19:22.649 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:19:22 smithi026 ceph-mon[175151]: pgmap v1590: 135 pgs: 5 creating+peering, 18 unknown, 112 active+clean; 582 KiB data, 4.2 GiB used, 532 GiB / 536 GiB avail; 176 KiB/s rd, 108 KiB/s wr, 288 op/s 2023-09-23T21:19:22.649 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:19:22 smithi026 ceph-mon[175151]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:19:22.650 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:19:22 smithi026 ceph-mon[175151]: failed to encode map e645 with expected crc 2023-09-23T21:19:22.650 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:19:22 smithi026 ceph-mon[175151]: failed to encode map e645 with expected crc 2023-09-23T21:19:22.650 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:19:22 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T21:19:22.650 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:19:22 smithi026 ceph-mon[175151]: osdmap e645: 8 total, 8 up, 6 in 2023-09-23T21:19:22.711 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:19:22 smithi070 ceph-mon[168282]: pgmap v1590: 135 pgs: 5 creating+peering, 18 unknown, 112 active+clean; 582 KiB data, 4.2 GiB used, 532 GiB / 536 GiB avail; 176 KiB/s rd, 108 KiB/s wr, 288 op/s 2023-09-23T21:19:22.712 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:19:22 smithi070 ceph-mon[168282]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:19:22.712 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:19:22 smithi070 ceph-mon[168282]: failed to encode map e645 with expected crc 2023-09-23T21:19:22.712 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:19:22 smithi070 ceph-mon[168282]: failed to encode map e645 with expected crc 2023-09-23T21:19:22.712 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:19:22 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T21:19:22.712 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:19:22 smithi070 ceph-mon[168282]: osdmap e645: 8 total, 8 up, 6 in 2023-09-23T21:19:22.713 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_blocked_ops 2023-09-23T21:19:22.746 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:19:22.782 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:22.803 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] TestClsRbd.mirror_image_status (2689 ms) 2023-09-23T21:19:22.803 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] TestClsRbd.mirror_image_map 2023-09-23T21:19:22.817 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:19:22.819 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:22.820 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:19:22.876 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0: {} 2023-09-23T21:19:22.899 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] TestClsRbd.mirror_image_map (96 ms) 2023-09-23T21:19:22.899 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] TestClsRbd.mirror_instances 2023-09-23T21:19:22.903 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1: {} 2023-09-23T21:19:22.907 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] TestClsRbd.mirror_instances (8 ms) 2023-09-23T21:19:22.908 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] TestClsRbd.mirror_snapshot 2023-09-23T21:19:22.921 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:19:22.922 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] TestClsRbd.mirror_snapshot (13 ms) 2023-09-23T21:19:22.922 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] TestClsRbd.group_dir_list 2023-09-23T21:19:22.924 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] TestClsRbd.group_dir_list (2 ms) 2023-09-23T21:19:22.924 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] TestClsRbd.group_dir_add 2023-09-23T21:19:22.924 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2: {} 2023-09-23T21:19:22.925 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] TestClsRbd.group_dir_add (3 ms) 2023-09-23T21:19:22.925 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] TestClsRbd.dir_add_already_existing 2023-09-23T21:19:22.929 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] TestClsRbd.dir_add_already_existing (3 ms) 2023-09-23T21:19:22.929 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] TestClsRbd.group_dir_rename 2023-09-23T21:19:22.936 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] TestClsRbd.group_dir_rename (7 ms) 2023-09-23T21:19:22.936 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] TestClsRbd.group_dir_remove 2023-09-23T21:19:22.939 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] TestClsRbd.group_dir_remove (3 ms) 2023-09-23T21:19:22.940 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] TestClsRbd.group_dir_remove_missing 2023-09-23T21:19:22.944 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] TestClsRbd.group_dir_remove_missing (4 ms) 2023-09-23T21:19:22.944 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] TestClsRbd.group_image_add 2023-09-23T21:19:22.946 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] TestClsRbd.group_image_add (3 ms) 2023-09-23T21:19:22.946 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] TestClsRbd.group_image_remove 2023-09-23T21:19:22.947 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3: {} 2023-09-23T21:19:22.949 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] TestClsRbd.group_image_remove (3 ms) 2023-09-23T21:19:22.949 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] TestClsRbd.group_image_list 2023-09-23T21:19:22.953 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] TestClsRbd.group_image_list (4 ms) 2023-09-23T21:19:22.953 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] TestClsRbd.group_image_clean 2023-09-23T21:19:22.953 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:22.958 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] TestClsRbd.group_image_clean (4 ms) 2023-09-23T21:19:22.958 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] TestClsRbd.image_group_add 2023-09-23T21:19:22.960 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] TestClsRbd.image_group_add (3 ms) 2023-09-23T21:19:22.961 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] TestClsRbd.image_group_remove 2023-09-23T21:19:22.964 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] TestClsRbd.image_group_remove (4 ms) 2023-09-23T21:19:22.964 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] TestClsRbd.image_group_get 2023-09-23T21:19:22.967 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] TestClsRbd.image_group_get (3 ms) 2023-09-23T21:19:22.967 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] TestClsRbd.group_snap_set_empty_name 2023-09-23T21:19:22.969 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] TestClsRbd.group_snap_set_empty_name (2 ms) 2023-09-23T21:19:22.969 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] TestClsRbd.group_snap_set_empty_id 2023-09-23T21:19:22.971 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] TestClsRbd.group_snap_set_empty_id (2 ms) 2023-09-23T21:19:22.971 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] TestClsRbd.group_snap_set_duplicate_id 2023-09-23T21:19:22.973 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4: {} 2023-09-23T21:19:22.974 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] TestClsRbd.group_snap_set_duplicate_id (3 ms) 2023-09-23T21:19:22.975 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] TestClsRbd.group_snap_set_duplicate_name 2023-09-23T21:19:22.977 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] TestClsRbd.group_snap_set_duplicate_name (3 ms) 2023-09-23T21:19:22.977 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] TestClsRbd.group_snap_set 2023-09-23T21:19:22.985 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] TestClsRbd.group_snap_set (8 ms) 2023-09-23T21:19:22.985 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] TestClsRbd.group_snap_list 2023-09-23T21:19:22.988 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] TestClsRbd.group_snap_list (3 ms) 2023-09-23T21:19:22.988 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] TestClsRbd.group_snap_list_max_return 2023-09-23T21:19:22.998 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:19:22.999 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T21:19:23.001 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:23.001 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:19:23.004 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] TestClsRbd.group_snap_list_max_return (16 ms) 2023-09-23T21:19:23.004 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] TestClsRbd.group_snap_list_max_read 2023-09-23T21:19:23.021 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6: {} 2023-09-23T21:19:23.048 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T21:19:23.049 INFO:teuthology.orchestra.run.smithi026.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T21:19:23.050 INFO:teuthology.orchestra.run.smithi026.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T21:19:23.050 INFO:teuthology.orchestra.run.smithi026.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T21:19:23.050 INFO:teuthology.orchestra.run.smithi026.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T21:19:23.050 INFO:teuthology.orchestra.run.smithi026.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T21:19:23.050 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T21:19:23.050 INFO:teuthology.orchestra.run.smithi026.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T21:19:23.050 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T21:19:23.103 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:19:23.129 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:23.162 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:19:23.165 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:23.165 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:19:23.218 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] TestClsRbd.group_snap_list_max_read (213 ms) 2023-09-23T21:19:23.218 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] TestClsRbd.group_snap_remove 2023-09-23T21:19:23.221 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] TestClsRbd.group_snap_remove (3 ms) 2023-09-23T21:19:23.221 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] TestClsRbd.group_snap_get_by_id 2023-09-23T21:19:23.222 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] TestClsRbd.group_snap_get_by_id (2 ms) 2023-09-23T21:19:23.223 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] TestClsRbd.trash_methods 2023-09-23T21:19:23.229 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] TestClsRbd.trash_methods (7 ms) 2023-09-23T21:19:23.229 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] TestClsRbd.op_features 2023-09-23T21:19:23.236 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] TestClsRbd.op_features (7 ms) 2023-09-23T21:19:23.236 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] TestClsRbd.clone_parent 2023-09-23T21:19:23.255 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] TestClsRbd.clone_parent (19 ms) 2023-09-23T21:19:23.255 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] TestClsRbd.clone_parent_ns 2023-09-23T21:19:23.266 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:19:23.266 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] TestClsRbd.clone_parent_ns (11 ms) 2023-09-23T21:19:23.266 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] TestClsRbd.clone_child 2023-09-23T21:19:23.279 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] TestClsRbd.clone_child (13 ms) 2023-09-23T21:19:23.279 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] TestClsRbd.namespace_methods 2023-09-23T21:19:23.286 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] TestClsRbd.namespace_methods (7 ms) 2023-09-23T21:19:23.287 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] TestClsRbd.migration 2023-09-23T21:19:23.296 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] TestClsRbd.migration (10 ms) 2023-09-23T21:19:23.296 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] TestClsRbd.migration_v1 2023-09-23T21:19:23.299 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:23.306 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] TestClsRbd.migration_v1 (10 ms) 2023-09-23T21:19:23.306 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] TestClsRbd.assert_snapc_seq 2023-09-23T21:19:23.317 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T21:19:23.324 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T21:19:23.445 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:19:23.446 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:23.447 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:19:23.547 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:19:23.575 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:23.613 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:19:23.615 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:23.615 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:19:23.716 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:19:23.744 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:23.777 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:19:23.779 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:23.780 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:19:23.862 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:19:23 smithi070 ceph-mon[168282]: failed to encode map e645 with expected crc 2023-09-23T21:19:23.863 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:19:23 smithi070 ceph-mon[168282]: failed to encode map e645 with expected crc 2023-09-23T21:19:23.863 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:19:23 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/928307476' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:19:23.881 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:19:23.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:19:23 smithi026 ceph-mon[179688]: failed to encode map e645 with expected crc 2023-09-23T21:19:23.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:19:23 smithi026 ceph-mon[179688]: failed to encode map e645 with expected crc 2023-09-23T21:19:23.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:19:23 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/928307476' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:19:23.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:19:23 smithi026 ceph-mon[175151]: failed to encode map e645 with expected crc 2023-09-23T21:19:23.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:19:23 smithi026 ceph-mon[175151]: failed to encode map e645 with expected crc 2023-09-23T21:19:23.883 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:19:23 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/928307476' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:19:23.910 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:23.925 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:19:24.102 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:19:24.105 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:24.105 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:19:24.206 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:19:24.234 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:24.267 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:19:24.270 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:24.271 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:19:24.371 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:19:24.402 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:24.439 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:19:24.442 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:24.442 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:19:24.543 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:19:24.566 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T21:19:24.571 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:24.595 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] TestClsRbd.assert_snapc_seq (1289 ms) 2023-09-23T21:19:24.596 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] TestClsRbd.sparsify 2023-09-23T21:19:24.611 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:19:24.615 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:24.616 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:19:24.631 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] TestClsRbd.sparsify (36 ms) 2023-09-23T21:19:24.668 INFO:teuthology.orchestra.run.smithi070.stdout:{ 2023-09-23T21:19:24.668 INFO:teuthology.orchestra.run.smithi070.stdout: "ops": [], 2023-09-23T21:19:24.668 INFO:teuthology.orchestra.run.smithi070.stdout: "complaint_time": 30, 2023-09-23T21:19:24.668 INFO:teuthology.orchestra.run.smithi070.stdout: "num_blocked_ops": 0 2023-09-23T21:19:24.668 INFO:teuthology.orchestra.run.smithi070.stdout:} 2023-09-23T21:19:24.693 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:19:24 smithi070 ceph-mon[168282]: pgmap v1592: 135 pgs: 5 creating+peering, 130 active+clean; 582 KiB data, 4.2 GiB used, 532 GiB / 536 GiB avail; 2.0 MiB/s rd, 1.0 MiB/s wr, 530 op/s 2023-09-23T21:19:24.693 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:19:24 smithi070 ceph-mon[168282]: failed to encode map e646 with expected crc 2023-09-23T21:19:24.693 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:19:24 smithi070 ceph-mon[168282]: failed to encode map e646 with expected crc 2023-09-23T21:19:24.693 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:19:24 smithi070 ceph-mon[168282]: failed to encode map e646 with expected crc 2023-09-23T21:19:24.694 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:19:24 smithi070 ceph-mon[168282]: osdmap e646: 8 total, 8 up, 6 in 2023-09-23T21:19:24.717 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:19:24.746 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:19:24 smithi026 ceph-mon[175151]: pgmap v1592: 135 pgs: 5 creating+peering, 130 active+clean; 582 KiB data, 4.2 GiB used, 532 GiB / 536 GiB avail; 2.0 MiB/s rd, 1.0 MiB/s wr, 530 op/s 2023-09-23T21:19:24.746 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:19:24 smithi026 ceph-mon[175151]: failed to encode map e646 with expected crc 2023-09-23T21:19:24.746 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:19:24 smithi026 ceph-mon[175151]: failed to encode map e646 with expected crc 2023-09-23T21:19:24.746 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:19:24 smithi026 ceph-mon[175151]: failed to encode map e646 with expected crc 2023-09-23T21:19:24.746 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:19:24 smithi026 ceph-mon[175151]: osdmap e646: 8 total, 8 up, 6 in 2023-09-23T21:19:24.747 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:19:24 smithi026 ceph-mon[179688]: pgmap v1592: 135 pgs: 5 creating+peering, 130 active+clean; 582 KiB data, 4.2 GiB used, 532 GiB / 536 GiB avail; 2.0 MiB/s rd, 1.0 MiB/s wr, 530 op/s 2023-09-23T21:19:24.747 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:19:24 smithi026 ceph-mon[179688]: failed to encode map e646 with expected crc 2023-09-23T21:19:24.747 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:19:24 smithi026 ceph-mon[179688]: failed to encode map e646 with expected crc 2023-09-23T21:19:24.747 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:19:24 smithi026 ceph-mon[179688]: failed to encode map e646 with expected crc 2023-09-23T21:19:24.747 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:19:24 smithi026 ceph-mon[179688]: osdmap e646: 8 total, 8 up, 6 in 2023-09-23T21:19:24.749 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:24.849 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T21:19:24.851 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:24.852 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:19:24.952 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:19:24.987 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:25.021 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:19:25.024 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:25.025 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:19:25.126 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:19:25.153 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:25.169 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_historic_ops 2023-09-23T21:19:25.188 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:19:25.190 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:25.191 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:19:25.291 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:19:25.319 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:25.333 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:19:25.364 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:19:25.369 INFO:teuthology.orchestra.run.smithi026.stdout: 2023-09-23T21:19:25.371 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:25.372 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:19:25.372 INFO:teuthology.orchestra.run.smithi026.stderr:dumped all 2023-09-23T21:19:25.473 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:19:25.484 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T21:19:25.503 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:25.607 INFO:tasks.workunit.client.0.smithi026.stdout:[----------] 70 tests from TestClsRbd (5054 ms total) 2023-09-23T21:19:25.607 INFO:tasks.workunit.client.0.smithi026.stdout: 2023-09-23T21:19:25.607 INFO:tasks.workunit.client.0.smithi026.stdout:[----------] Global test environment tear-down 2023-09-23T21:19:25.607 INFO:tasks.workunit.client.0.smithi026.stdout:[==========] 70 tests from 1 test suite ran. (7561 ms total) 2023-09-23T21:19:25.607 INFO:tasks.workunit.client.0.smithi026.stdout:[ PASSED ] 70 tests. 2023-09-23T21:19:25.609 INFO:teuthology.orchestra.run:Running command with timeout 3600 2023-09-23T21:19:25.609 DEBUG:teuthology.orchestra.run.smithi026:> sudo rm -rf -- /home/ubuntu/cephtest/mnt.0/client.0/tmp 2023-09-23T21:19:25.640 INFO:tasks.workunit:Stopping ['cls/test_cls_rbd.sh'] on client.0... 2023-09-23T21:19:25.641 DEBUG:teuthology.orchestra.run.smithi026:> sudo rm -rf -- /home/ubuntu/cephtest/workunits.list.client.0 /home/ubuntu/cephtest/clone.client.0 2023-09-23T21:19:25.773 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T21:19:25.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:19:25 smithi070 ceph-mon[168282]: failed to encode map e647 with expected crc 2023-09-23T21:19:25.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:19:25 smithi070 ceph-mon[168282]: failed to encode map e647 with expected crc 2023-09-23T21:19:25.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:19:25 smithi070 ceph-mon[168282]: osdmap e647: 8 total, 8 up, 6 in 2023-09-23T21:19:25.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:19:25 smithi070 ceph-mon[168282]: failed to encode map e647 with expected crc 2023-09-23T21:19:25.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:19:25 smithi070 ceph-mon[168282]: failed to encode map e647 with expected crc 2023-09-23T21:19:25.881 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:19:25 smithi026 ceph-mon[175151]: failed to encode map e647 with expected crc 2023-09-23T21:19:25.881 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:19:25 smithi026 ceph-mon[175151]: failed to encode map e647 with expected crc 2023-09-23T21:19:25.881 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:19:25 smithi026 ceph-mon[175151]: osdmap e647: 8 total, 8 up, 6 in 2023-09-23T21:19:25.881 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:19:25 smithi026 ceph-mon[175151]: failed to encode map e647 with expected crc 2023-09-23T21:19:25.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:19:25 smithi026 ceph-mon[175151]: failed to encode map e647 with expected crc 2023-09-23T21:19:25.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:19:25 smithi026 ceph-mon[179688]: failed to encode map e647 with expected crc 2023-09-23T21:19:25.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:19:25 smithi026 ceph-mon[179688]: failed to encode map e647 with expected crc 2023-09-23T21:19:25.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:19:25 smithi026 ceph-mon[179688]: osdmap e647: 8 total, 8 up, 6 in 2023-09-23T21:19:25.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:19:25 smithi026 ceph-mon[179688]: failed to encode map e647 with expected crc 2023-09-23T21:19:25.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:19:25 smithi026 ceph-mon[179688]: failed to encode map e647 with expected crc 2023-09-23T21:19:25.918 DEBUG:teuthology.parallel:result is None 2023-09-23T21:19:25.918 DEBUG:teuthology.orchestra.run.smithi026:> sudo rm -rf -- /home/ubuntu/cephtest/mnt.0/client.0 2023-09-23T21:19:25.947 INFO:tasks.workunit:Deleted dir /home/ubuntu/cephtest/mnt.0/client.0 2023-09-23T21:19:25.947 DEBUG:teuthology.orchestra.run.smithi026:> rmdir -- /home/ubuntu/cephtest/mnt.0 2023-09-23T21:19:26.007 INFO:tasks.workunit:Deleted artificial mount point /home/ubuntu/cephtest/mnt.0/client.0 2023-09-23T21:19:26.007 INFO:teuthology.task.sequential:In sequential, running task print... 2023-09-23T21:19:26.008 INFO:teuthology.task.print:**** done cls/test_cls_rbd.sh 5-workload 2023-09-23T21:19:26.008 INFO:teuthology.task.sequential:In sequential, running task workunit... 2023-09-23T21:19:26.008 INFO:tasks.workunit:Pulling workunits from ref quincy 2023-09-23T21:19:26.008 INFO:tasks.workunit:Making a separate scratch dir for every client... 2023-09-23T21:19:26.008 DEBUG:teuthology.orchestra.run.smithi026:> stat -- /home/ubuntu/cephtest/mnt.0 2023-09-23T21:19:26.014 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:19:26.017 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:26.018 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:19:26.034 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T21:19:26.034 INFO:teuthology.orchestra.run.smithi026.stderr:stat: cannot statx '/home/ubuntu/cephtest/mnt.0': No such file or directory 2023-09-23T21:19:26.035 DEBUG:teuthology.orchestra.run.smithi026:> mkdir -- /home/ubuntu/cephtest/mnt.0 2023-09-23T21:19:26.079 INFO:teuthology.orchestra.run.smithi026.stdout:{"pg_ready":true,"pg_map":{"version":1595,"stamp":"2023-09-23T21:19:25.212727+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":596246,"num_objects":61,"num_object_clones":0,"num_object_copies":183,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":61,"num_whiteouts":0,"num_read":1758,"num_read_kb":9847,"num_write":906,"num_write_kb":4928,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":82,"num_bytes_recovered":544062,"num_keys_recovered":20,"num_objects_omap":43,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":5742,"ondisk_log_size":5742,"up":405,"acting":405,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":376,"num_osds":6,"num_per_pool_osds":6,"num_per_pool_omap_osds":6,"kb":562470912,"kb_used":4463936,"kb_used_data":56176,"kb_used_omap":125,"kb_used_meta":4407554,"kb_avail":558006976,"statfs":{"total":575970213888,"available":571399143424,"internally_reserved":0,"allocated":57524224,"data_stored":43455503,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":128662,"internal_metadata":4513335658},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":218,"num_objects":30,"num_object_clones":0,"num_object_copies":90,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":30,"num_whiteouts":0,"num_read":684,"num_read_kb":8789,"num_write":190,"num_write_kb":4250,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":23,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"4.634097"},"pg_stats":[{"pgid":"12.15","version":"643'1","reported_seq":13,"reported_epoch":644,"state":"active+clean","last_fresh":"2023-09-23T21:19:20.625806+0000","last_change":"2023-09-23T21:19:19.592642+0000","last_active":"2023-09-23T21:19:20.625806+0000","last_peered":"2023-09-23T21:19:20.625806+0000","last_clean":"2023-09-23T21:19:20.625806+0000","last_became_active":"2023-09-23T21:19:19.590218+0000","last_became_peered":"2023-09-23T21:19:19.590218+0000","last_unstale":"2023-09-23T21:19:20.625806+0000","last_undegraded":"2023-09-23T21:19:20.625806+0000","last_fullsized":"2023-09-23T21:19:20.625806+0000","mapping_epoch":642,"log_start":"0'0","ondisk_log_start":"0'0","created":642,"last_epoch_clean":643,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:19:18.561631+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:19:18.561631+0000","last_clean_scrub_stamp":"2023-09-23T21:19:18.561631+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:31:33.380449+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":42,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":2,"num_read_kb":1,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,7],"acting":[2,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.18","version":"79'161","reported_seq":668,"reported_epoch":643,"state":"active+clean","last_fresh":"2023-09-23T21:19:19.571530+0000","last_change":"2023-09-23T21:17:39.906535+0000","last_active":"2023-09-23T21:19:19.571530+0000","last_peered":"2023-09-23T21:19:19.571530+0000","last_clean":"2023-09-23T21:19:19.571530+0000","last_became_active":"2023-09-23T21:17:39.906154+0000","last_became_peered":"2023-09-23T21:17:39.906154+0000","last_unstale":"2023-09-23T21:19:19.571530+0000","last_undegraded":"2023-09-23T21:19:19.571530+0000","last_fullsized":"2023-09-23T21:19:19.571530+0000","mapping_epoch":585,"log_start":"0'0","ondisk_log_start":"0'0","created":479,"last_epoch_clean":586,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:15:39.474823+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:15:39.474823+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:48:39.784037+0000","scrub_duration":0.074664395999999994,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":525376,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1a","version":"68'1","reported_seq":235,"reported_epoch":644,"state":"active+clean","last_fresh":"2023-09-23T21:19:20.590933+0000","last_change":"2023-09-23T21:13:13.291403+0000","last_active":"2023-09-23T21:19:20.590933+0000","last_peered":"2023-09-23T21:19:20.590933+0000","last_clean":"2023-09-23T21:19:20.590933+0000","last_became_active":"2023-09-23T21:13:13.291123+0000","last_became_peered":"2023-09-23T21:13:13.291123+0000","last_unstale":"2023-09-23T21:19:20.590933+0000","last_undegraded":"2023-09-23T21:19:20.590933+0000","last_fullsized":"2023-09-23T21:19:20.590933+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":495,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:45:15.427430+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,0],"acting":[4,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":304,"reported_epoch":644,"state":"active+clean","last_fresh":"2023-09-23T21:19:20.590852+0000","last_change":"2023-09-23T21:12:10.767963+0000","last_active":"2023-09-23T21:19:20.590852+0000","last_peered":"2023-09-23T21:19:20.590852+0000","last_clean":"2023-09-23T21:19:20.590852+0000","last_became_active":"2023-09-23T21:12:10.767126+0000","last_became_peered":"2023-09-23T21:12:10.767126+0000","last_unstale":"2023-09-23T21:19:20.590852+0000","last_undegraded":"2023-09-23T21:19:20.590852+0000","last_fullsized":"2023-09-23T21:19:20.590852+0000","mapping_epoch":470,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:45:47.138277+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,5],"acting":[4,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.16","version":"643'22","reported_seq":43,"reported_epoch":644,"state":"active+clean","last_fresh":"2023-09-23T21:19:20.590840+0000","last_change":"2023-09-23T21:19:19.580007+0000","last_active":"2023-09-23T21:19:20.590840+0000","last_peered":"2023-09-23T21:19:20.590840+0000","last_clean":"2023-09-23T21:19:20.590840+0000","last_became_active":"2023-09-23T21:19:19.579831+0000","last_became_peered":"2023-09-23T21:19:19.579831+0000","last_unstale":"2023-09-23T21:19:20.590840+0000","last_undegraded":"2023-09-23T21:19:20.590840+0000","last_fullsized":"2023-09-23T21:19:20.590840+0000","mapping_epoch":642,"log_start":"0'0","ondisk_log_start":"0'0","created":642,"last_epoch_clean":643,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:19:18.561631+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:19:18.561631+0000","last_clean_scrub_stamp":"2023-09-23T21:19:18.561631+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:47:58.316196+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":23,"num_read_kb":44,"num_write":14,"num_write_kb":17,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1b","version":"79'161","reported_seq":831,"reported_epoch":643,"state":"active+clean","last_fresh":"2023-09-23T21:19:19.572123+0000","last_change":"2023-09-23T21:15:38.423372+0000","last_active":"2023-09-23T21:19:19.572123+0000","last_peered":"2023-09-23T21:19:19.572123+0000","last_clean":"2023-09-23T21:19:19.572123+0000","last_became_active":"2023-09-23T21:15:23.802580+0000","last_became_peered":"2023-09-23T21:15:23.802580+0000","last_unstale":"2023-09-23T21:19:19.572123+0000","last_undegraded":"2023-09-23T21:19:19.572123+0000","last_fullsized":"2023-09-23T21:19:19.572123+0000","mapping_epoch":542,"log_start":"0'0","ondisk_log_start":"0'0","created":479,"last_epoch_clean":543,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:15:38.423271+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:15:38.423271+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:04:46.575186+0000","scrub_duration":0.00258465,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":661,"reported_epoch":645,"state":"active+clean","last_fresh":"2023-09-23T21:19:22.696909+0000","last_change":"2023-09-23T21:16:42.448529+0000","last_active":"2023-09-23T21:19:22.696909+0000","last_peered":"2023-09-23T21:19:22.696909+0000","last_clean":"2023-09-23T21:19:22.696909+0000","last_became_active":"2023-09-23T21:16:42.447803+0000","last_became_peered":"2023-09-23T21:16:42.447803+0000","last_unstale":"2023-09-23T21:19:22.696909+0000","last_undegraded":"2023-09-23T21:19:22.696909+0000","last_fullsized":"2023-09-23T21:19:22.696909+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:41:33.121784+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":660,"reported_epoch":642,"state":"active+clean","last_fresh":"2023-09-23T21:19:18.573809+0000","last_change":"2023-09-23T21:16:42.441896+0000","last_active":"2023-09-23T21:19:18.573809+0000","last_peered":"2023-09-23T21:19:18.573809+0000","last_clean":"2023-09-23T21:19:18.573809+0000","last_became_active":"2023-09-23T21:16:42.441526+0000","last_became_peered":"2023-09-23T21:16:42.441526+0000","last_unstale":"2023-09-23T21:19:18.573809+0000","last_undegraded":"2023-09-23T21:19:18.573809+0000","last_fullsized":"2023-09-23T21:19:18.573809+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:27:36.326225+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"12.17","version":"643'2","reported_seq":15,"reported_epoch":645,"state":"active+clean","last_fresh":"2023-09-23T21:19:22.698559+0000","last_change":"2023-09-23T21:19:19.589872+0000","last_active":"2023-09-23T21:19:22.698559+0000","last_peered":"2023-09-23T21:19:22.698559+0000","last_clean":"2023-09-23T21:19:22.698559+0000","last_became_active":"2023-09-23T21:19:19.589563+0000","last_became_peered":"2023-09-23T21:19:19.589563+0000","last_unstale":"2023-09-23T21:19:22.698559+0000","last_undegraded":"2023-09-23T21:19:22.698559+0000","last_fullsized":"2023-09-23T21:19:22.698559+0000","mapping_epoch":642,"log_start":"0'0","ondisk_log_start":"0'0","created":642,"last_epoch_clean":643,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:19:18.561631+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:19:18.561631+0000","last_clean_scrub_stamp":"2023-09-23T21:19:18.561631+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:49:56.441001+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1a","version":"79'161","reported_seq":742,"reported_epoch":644,"state":"active+clean","last_fresh":"2023-09-23T21:19:20.590390+0000","last_change":"2023-09-23T21:16:45.419221+0000","last_active":"2023-09-23T21:19:20.590390+0000","last_peered":"2023-09-23T21:19:20.590390+0000","last_clean":"2023-09-23T21:19:20.590390+0000","last_became_active":"2023-09-23T21:16:45.418994+0000","last_became_peered":"2023-09-23T21:16:45.418994+0000","last_unstale":"2023-09-23T21:19:20.590390+0000","last_undegraded":"2023-09-23T21:19:20.590390+0000","last_fullsized":"2023-09-23T21:19:20.590390+0000","mapping_epoch":567,"log_start":"0'0","ondisk_log_start":"0'0","created":479,"last_epoch_clean":568,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:15:45.158107+0000","last_deep_scrub":"79'161","last_deep_scrub_stamp":"2023-09-23T21:05:32.630292+0000","last_clean_scrub_stamp":"2023-09-23T21:15:45.158107+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:08:08.342444+0000","scrub_duration":0.001999373,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,5],"acting":[4,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":154,"reported_epoch":642,"state":"active+clean","last_fresh":"2023-09-23T21:19:18.571516+0000","last_change":"2023-09-23T21:13:30.806084+0000","last_active":"2023-09-23T21:19:18.571516+0000","last_peered":"2023-09-23T21:19:18.571516+0000","last_clean":"2023-09-23T21:19:18.571516+0000","last_became_active":"2023-09-23T21:13:30.805743+0000","last_became_peered":"2023-09-23T21:13:30.805743+0000","last_unstale":"2023-09-23T21:19:18.571516+0000","last_undegraded":"2023-09-23T21:19:18.571516+0000","last_fullsized":"2023-09-23T21:19:18.571516+0000","mapping_epoch":501,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:48:10.849281+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":518,"reported_epoch":642,"state":"active+clean","last_fresh":"2023-09-23T21:19:18.573263+0000","last_change":"2023-09-23T21:13:44.410921+0000","last_active":"2023-09-23T21:19:18.573263+0000","last_peered":"2023-09-23T21:19:18.573263+0000","last_clean":"2023-09-23T21:19:18.573263+0000","last_became_active":"2023-09-23T21:13:44.410633+0000","last_became_peered":"2023-09-23T21:13:44.410633+0000","last_unstale":"2023-09-23T21:19:18.573263+0000","last_undegraded":"2023-09-23T21:19:18.573263+0000","last_fullsized":"2023-09-23T21:19:18.573263+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:57:10.116786+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"12.18","version":"643'15","reported_seq":55,"reported_epoch":644,"state":"active+clean","last_fresh":"2023-09-23T21:19:20.590225+0000","last_change":"2023-09-23T21:19:19.583680+0000","last_active":"2023-09-23T21:19:20.590225+0000","last_peered":"2023-09-23T21:19:20.590225+0000","last_clean":"2023-09-23T21:19:20.590225+0000","last_became_active":"2023-09-23T21:19:19.583286+0000","last_became_peered":"2023-09-23T21:19:19.583286+0000","last_unstale":"2023-09-23T21:19:20.590225+0000","last_undegraded":"2023-09-23T21:19:20.590225+0000","last_fullsized":"2023-09-23T21:19:20.590225+0000","mapping_epoch":642,"log_start":"0'0","ondisk_log_start":"0'0","created":642,"last_epoch_clean":643,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:19:18.561631+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:19:18.561631+0000","last_clean_scrub_stamp":"2023-09-23T21:19:18.561631+0000","objects_scrubbed":0,"log_size":15,"log_dups_size":0,"ondisk_log_size":15,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:30:48.479238+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":86,"num_read_kb":82,"num_write":12,"num_write_kb":9,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.15","version":"79'161","reported_seq":602,"reported_epoch":643,"state":"active+clean","last_fresh":"2023-09-23T21:19:19.582652+0000","last_change":"2023-09-23T21:18:35.622513+0000","last_active":"2023-09-23T21:19:19.582652+0000","last_peered":"2023-09-23T21:19:19.582652+0000","last_clean":"2023-09-23T21:19:19.582652+0000","last_became_active":"2023-09-23T21:16:45.425654+0000","last_became_peered":"2023-09-23T21:16:45.425654+0000","last_unstale":"2023-09-23T21:19:19.582652+0000","last_undegraded":"2023-09-23T21:19:19.582652+0000","last_fullsized":"2023-09-23T21:19:19.582652+0000","mapping_epoch":567,"log_start":"0'0","ondisk_log_start":"0'0","created":479,"last_epoch_clean":568,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:18:35.622422+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:18:35.622422+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:21:48.421117+0000","scrub_duration":0.083031242000000005,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,3],"acting":[7,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":619,"reported_epoch":642,"state":"active+clean","last_fresh":"2023-09-23T21:19:18.573399+0000","last_change":"2023-09-23T21:13:12.235728+0000","last_active":"2023-09-23T21:19:18.573399+0000","last_peered":"2023-09-23T21:19:18.573399+0000","last_clean":"2023-09-23T21:19:18.573399+0000","last_became_active":"2023-09-23T21:13:12.216549+0000","last_became_peered":"2023-09-23T21:13:12.216549+0000","last_unstale":"2023-09-23T21:19:18.573399+0000","last_undegraded":"2023-09-23T21:19:18.573399+0000","last_fullsized":"2023-09-23T21:19:18.573399+0000","mapping_epoch":493,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:39:33.324239+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":686,"reported_epoch":645,"state":"active+clean","last_fresh":"2023-09-23T21:19:22.697836+0000","last_change":"2023-09-23T21:12:10.790349+0000","last_active":"2023-09-23T21:19:22.697836+0000","last_peered":"2023-09-23T21:19:22.697836+0000","last_clean":"2023-09-23T21:19:22.697836+0000","last_became_active":"2023-09-23T21:12:10.789977+0000","last_became_peered":"2023-09-23T21:12:10.789977+0000","last_unstale":"2023-09-23T21:19:22.697836+0000","last_undegraded":"2023-09-23T21:19:22.697836+0000","last_fullsized":"2023-09-23T21:19:22.697836+0000","mapping_epoch":470,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:49:40.191147+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,5],"acting":[3,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"12.19","version":"643'7","reported_seq":19,"reported_epoch":643,"state":"active+clean","last_fresh":"2023-09-23T21:19:20.013239+0000","last_change":"2023-09-23T21:19:19.585677+0000","last_active":"2023-09-23T21:19:20.013239+0000","last_peered":"2023-09-23T21:19:20.013239+0000","last_clean":"2023-09-23T21:19:20.013239+0000","last_became_active":"2023-09-23T21:19:19.585395+0000","last_became_peered":"2023-09-23T21:19:19.585395+0000","last_unstale":"2023-09-23T21:19:20.013239+0000","last_undegraded":"2023-09-23T21:19:20.013239+0000","last_fullsized":"2023-09-23T21:19:20.013239+0000","mapping_epoch":642,"log_start":"0'0","ondisk_log_start":"0'0","created":642,"last_epoch_clean":643,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:19:18.561631+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:19:18.561631+0000","last_clean_scrub_stamp":"2023-09-23T21:19:18.561631+0000","objects_scrubbed":0,"log_size":7,"log_dups_size":0,"ondisk_log_size":7,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:24:04.345920+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":13,"num_read_kb":10,"num_write":3,"num_write_kb":3,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.14","version":"79'161","reported_seq":1041,"reported_epoch":643,"state":"active+clean","last_fresh":"2023-09-23T21:19:19.579663+0000","last_change":"2023-09-23T21:16:45.424773+0000","last_active":"2023-09-23T21:19:19.579663+0000","last_peered":"2023-09-23T21:19:19.579663+0000","last_clean":"2023-09-23T21:19:19.579663+0000","last_became_active":"2023-09-23T21:16:45.424458+0000","last_became_peered":"2023-09-23T21:16:45.424458+0000","last_unstale":"2023-09-23T21:19:19.579663+0000","last_undegraded":"2023-09-23T21:19:19.579663+0000","last_fullsized":"2023-09-23T21:19:19.579663+0000","mapping_epoch":567,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":568,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:25.096401+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:25.096401+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:40:17.871561+0000","scrub_duration":0.002859393,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":388,"reported_epoch":643,"state":"active+clean","last_fresh":"2023-09-23T21:19:19.571858+0000","last_change":"2023-09-23T21:16:42.427856+0000","last_active":"2023-09-23T21:19:19.571858+0000","last_peered":"2023-09-23T21:19:19.571858+0000","last_clean":"2023-09-23T21:19:19.571858+0000","last_became_active":"2023-09-23T21:16:42.427529+0000","last_became_peered":"2023-09-23T21:16:42.427529+0000","last_unstale":"2023-09-23T21:19:19.571858+0000","last_undegraded":"2023-09-23T21:19:19.571858+0000","last_fullsized":"2023-09-23T21:19:19.571858+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:24:03.818632+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":641,"reported_epoch":645,"state":"active+clean","last_fresh":"2023-09-23T21:19:22.697819+0000","last_change":"2023-09-23T21:16:42.448719+0000","last_active":"2023-09-23T21:19:22.697819+0000","last_peered":"2023-09-23T21:19:22.697819+0000","last_clean":"2023-09-23T21:19:22.697819+0000","last_became_active":"2023-09-23T21:16:42.448219+0000","last_became_peered":"2023-09-23T21:16:42.448219+0000","last_unstale":"2023-09-23T21:19:22.697819+0000","last_undegraded":"2023-09-23T21:19:22.697819+0000","last_fullsized":"2023-09-23T21:19:22.697819+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:02:28.864357+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"12.1a","version":"0'0","reported_seq":11,"reported_epoch":644,"state":"active+clean","last_fresh":"2023-09-23T21:19:20.590260+0000","last_change":"2023-09-23T21:19:19.585537+0000","last_active":"2023-09-23T21:19:20.590260+0000","last_peered":"2023-09-23T21:19:20.590260+0000","last_clean":"2023-09-23T21:19:20.590260+0000","last_became_active":"2023-09-23T21:19:19.583660+0000","last_became_peered":"2023-09-23T21:19:19.583660+0000","last_unstale":"2023-09-23T21:19:20.590260+0000","last_undegraded":"2023-09-23T21:19:20.590260+0000","last_fullsized":"2023-09-23T21:19:20.590260+0000","mapping_epoch":642,"log_start":"0'0","ondisk_log_start":"0'0","created":642,"last_epoch_clean":643,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:19:18.561631+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:19:18.561631+0000","last_clean_scrub_stamp":"2023-09-23T21:19:18.561631+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:12:31.258899+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,5],"acting":[4,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.17","version":"79'161","reported_seq":790,"reported_epoch":642,"state":"active+clean","last_fresh":"2023-09-23T21:19:18.573022+0000","last_change":"2023-09-23T21:16:45.413663+0000","last_active":"2023-09-23T21:19:18.573022+0000","last_peered":"2023-09-23T21:19:18.573022+0000","last_clean":"2023-09-23T21:19:18.573022+0000","last_became_active":"2023-09-23T21:16:45.413311+0000","last_became_peered":"2023-09-23T21:16:45.413311+0000","last_unstale":"2023-09-23T21:19:18.573022+0000","last_undegraded":"2023-09-23T21:19:18.573022+0000","last_fullsized":"2023-09-23T21:19:18.573022+0000","mapping_epoch":567,"log_start":"0'0","ondisk_log_start":"0'0","created":479,"last_epoch_clean":568,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:15:50.325741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:15:50.325741+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:07:43.285354+0000","scrub_duration":0.0040500149999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":562,"reported_epoch":645,"state":"active+clean","last_fresh":"2023-09-23T21:19:22.697114+0000","last_change":"2023-09-23T21:17:02.182606+0000","last_active":"2023-09-23T21:19:22.697114+0000","last_peered":"2023-09-23T21:19:22.697114+0000","last_clean":"2023-09-23T21:19:22.697114+0000","last_became_active":"2023-09-23T21:16:42.446517+0000","last_became_peered":"2023-09-23T21:16:42.446517+0000","last_unstale":"2023-09-23T21:19:22.697114+0000","last_undegraded":"2023-09-23T21:19:22.697114+0000","last_fullsized":"2023-09-23T21:19:22.697114+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:46:08.337878+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":540,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,4],"acting":[3,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":333,"reported_epoch":643,"state":"active+clean","last_fresh":"2023-09-23T21:19:19.580291+0000","last_change":"2023-09-23T21:13:30.821489+0000","last_active":"2023-09-23T21:19:19.580291+0000","last_peered":"2023-09-23T21:19:19.580291+0000","last_clean":"2023-09-23T21:19:19.580291+0000","last_became_active":"2023-09-23T21:13:30.821125+0000","last_became_peered":"2023-09-23T21:13:30.821125+0000","last_unstale":"2023-09-23T21:19:19.580291+0000","last_undegraded":"2023-09-23T21:19:19.580291+0000","last_fullsized":"2023-09-23T21:19:19.580291+0000","mapping_epoch":501,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:47:18.553417+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,5],"acting":[2,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"12.1b","version":"0'0","reported_seq":11,"reported_epoch":644,"state":"active+clean","last_fresh":"2023-09-23T21:19:20.590496+0000","last_change":"2023-09-23T21:19:19.586141+0000","last_active":"2023-09-23T21:19:20.590496+0000","last_peered":"2023-09-23T21:19:20.590496+0000","last_clean":"2023-09-23T21:19:20.590496+0000","last_became_active":"2023-09-23T21:19:19.585844+0000","last_became_peered":"2023-09-23T21:19:19.585844+0000","last_unstale":"2023-09-23T21:19:20.590496+0000","last_undegraded":"2023-09-23T21:19:20.590496+0000","last_fullsized":"2023-09-23T21:19:20.590496+0000","mapping_epoch":642,"log_start":"0'0","ondisk_log_start":"0'0","created":642,"last_epoch_clean":643,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:19:18.561631+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:19:18.561631+0000","last_clean_scrub_stamp":"2023-09-23T21:19:18.561631+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:18:15.605754+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,5],"acting":[4,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.16","version":"79'161","reported_seq":575,"reported_epoch":642,"state":"active+clean","last_fresh":"2023-09-23T21:19:18.573766+0000","last_change":"2023-09-23T21:15:46.326726+0000","last_active":"2023-09-23T21:19:18.573766+0000","last_peered":"2023-09-23T21:19:18.573766+0000","last_clean":"2023-09-23T21:19:18.573766+0000","last_became_active":"2023-09-23T21:13:33.770407+0000","last_became_peered":"2023-09-23T21:13:33.770407+0000","last_unstale":"2023-09-23T21:19:18.573766+0000","last_undegraded":"2023-09-23T21:19:18.573766+0000","last_fullsized":"2023-09-23T21:19:18.573766+0000","mapping_epoch":504,"log_start":"0'0","ondisk_log_start":"0'0","created":479,"last_epoch_clean":505,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:15:46.326589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:15:46.326589+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:50:23.810054+0000","scrub_duration":0.0062069059999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.14","version":"79'22","reported_seq":248,"reported_epoch":644,"state":"active+clean","last_fresh":"2023-09-23T21:19:20.590703+0000","last_change":"2023-09-23T21:17:02.198735+0000","last_active":"2023-09-23T21:19:20.590703+0000","last_peered":"2023-09-23T21:19:20.590703+0000","last_clean":"2023-09-23T21:19:20.590703+0000","last_became_active":"2023-09-23T21:16:42.492861+0000","last_became_peered":"2023-09-23T21:16:42.492861+0000","last_unstale":"2023-09-23T21:19:20.590703+0000","last_undegraded":"2023-09-23T21:19:20.590703+0000","last_fullsized":"2023-09-23T21:19:20.590703+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:00:09.949965+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":14202,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,2],"acting":[4,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":573,"reported_epoch":644,"state":"active+clean","last_fresh":"2023-09-23T21:19:20.590574+0000","last_change":"2023-09-23T21:12:10.839271+0000","last_active":"2023-09-23T21:19:20.590574+0000","last_peered":"2023-09-23T21:19:20.590574+0000","last_clean":"2023-09-23T21:19:20.590574+0000","last_became_active":"2023-09-23T21:12:10.839042+0000","last_became_peered":"2023-09-23T21:12:10.839042+0000","last_unstale":"2023-09-23T21:19:20.590574+0000","last_undegraded":"2023-09-23T21:19:20.590574+0000","last_fullsized":"2023-09-23T21:19:20.590574+0000","mapping_epoch":470,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:10:26.414737+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,2],"acting":[4,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.1c","version":"0'0","reported_seq":10,"reported_epoch":643,"state":"active+clean","last_fresh":"2023-09-23T21:19:19.586608+0000","last_change":"2023-09-23T21:19:19.584651+0000","last_active":"2023-09-23T21:19:19.586608+0000","last_peered":"2023-09-23T21:19:19.586608+0000","last_clean":"2023-09-23T21:19:19.586608+0000","last_became_active":"2023-09-23T21:19:19.583703+0000","last_became_peered":"2023-09-23T21:19:19.583703+0000","last_unstale":"2023-09-23T21:19:19.586608+0000","last_undegraded":"2023-09-23T21:19:19.586608+0000","last_fullsized":"2023-09-23T21:19:19.586608+0000","mapping_epoch":642,"log_start":"0'0","ondisk_log_start":"0'0","created":642,"last_epoch_clean":643,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:19:18.561631+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:19:18.561631+0000","last_clean_scrub_stamp":"2023-09-23T21:19:18.561631+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:11:28.961197+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.11","version":"79'161","reported_seq":580,"reported_epoch":642,"state":"active+clean","last_fresh":"2023-09-23T21:19:18.573214+0000","last_change":"2023-09-23T21:15:47.300109+0000","last_active":"2023-09-23T21:19:18.573214+0000","last_peered":"2023-09-23T21:19:18.573214+0000","last_clean":"2023-09-23T21:19:18.573214+0000","last_became_active":"2023-09-23T21:13:33.767067+0000","last_became_peered":"2023-09-23T21:13:33.767067+0000","last_unstale":"2023-09-23T21:19:18.573214+0000","last_undegraded":"2023-09-23T21:19:18.573214+0000","last_fullsized":"2023-09-23T21:19:18.573214+0000","mapping_epoch":504,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":505,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:15:47.299947+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:15:47.299947+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:04:59.029359+0000","scrub_duration":0.0061353110000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":95,"reported_epoch":643,"state":"active+clean","last_fresh":"2023-09-23T21:19:19.580656+0000","last_change":"2023-09-23T21:16:42.531060+0000","last_active":"2023-09-23T21:19:19.580656+0000","last_peered":"2023-09-23T21:19:19.580656+0000","last_clean":"2023-09-23T21:19:19.580656+0000","last_became_active":"2023-09-23T21:16:42.449198+0000","last_became_peered":"2023-09-23T21:16:42.449198+0000","last_unstale":"2023-09-23T21:19:19.580656+0000","last_undegraded":"2023-09-23T21:19:19.580656+0000","last_fullsized":"2023-09-23T21:19:19.580656+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:30:23.428886+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":205,"reported_epoch":643,"state":"active+clean","last_fresh":"2023-09-23T21:19:19.579377+0000","last_change":"2023-09-23T21:16:42.491023+0000","last_active":"2023-09-23T21:19:19.579377+0000","last_peered":"2023-09-23T21:19:19.579377+0000","last_clean":"2023-09-23T21:19:19.579377+0000","last_became_active":"2023-09-23T21:16:42.489896+0000","last_became_peered":"2023-09-23T21:16:42.489896+0000","last_unstale":"2023-09-23T21:19:19.579377+0000","last_undegraded":"2023-09-23T21:19:19.579377+0000","last_fullsized":"2023-09-23T21:19:19.579377+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:58:54.036098+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,7],"acting":[2,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"12.1d","version":"643'2","reported_seq":12,"reported_epoch":643,"state":"active+clean","last_fresh":"2023-09-23T21:19:19.950097+0000","last_change":"2023-09-23T21:19:19.593211+0000","last_active":"2023-09-23T21:19:19.950097+0000","last_peered":"2023-09-23T21:19:19.950097+0000","last_clean":"2023-09-23T21:19:19.950097+0000","last_became_active":"2023-09-23T21:19:19.592849+0000","last_became_peered":"2023-09-23T21:19:19.592849+0000","last_unstale":"2023-09-23T21:19:19.950097+0000","last_undegraded":"2023-09-23T21:19:19.950097+0000","last_fullsized":"2023-09-23T21:19:19.950097+0000","mapping_epoch":642,"log_start":"0'0","ondisk_log_start":"0'0","created":642,"last_epoch_clean":643,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:19:18.561631+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:19:18.561631+0000","last_clean_scrub_stamp":"2023-09-23T21:19:18.561631+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:24:28.781771+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,7],"acting":[0,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.10","version":"79'161","reported_seq":819,"reported_epoch":644,"state":"active+clean","last_fresh":"2023-09-23T21:19:20.590123+0000","last_change":"2023-09-23T21:16:45.416227+0000","last_active":"2023-09-23T21:19:20.590123+0000","last_peered":"2023-09-23T21:19:20.590123+0000","last_clean":"2023-09-23T21:19:20.590123+0000","last_became_active":"2023-09-23T21:16:45.415841+0000","last_became_peered":"2023-09-23T21:16:45.415841+0000","last_unstale":"2023-09-23T21:19:20.590123+0000","last_undegraded":"2023-09-23T21:19:20.590123+0000","last_fullsized":"2023-09-23T21:19:20.590123+0000","mapping_epoch":567,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":568,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:06:50.336992+0000","last_deep_scrub":"79'161","last_deep_scrub_stamp":"2023-09-23T21:06:50.336992+0000","last_clean_scrub_stamp":"2023-09-23T21:06:50.336992+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:37:30.094658+0000","scrub_duration":0.0026150930000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,7],"acting":[4,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":603,"reported_epoch":642,"state":"active+clean","last_fresh":"2023-09-23T21:19:18.571821+0000","last_change":"2023-09-23T21:16:42.446324+0000","last_active":"2023-09-23T21:19:18.571821+0000","last_peered":"2023-09-23T21:19:18.571821+0000","last_clean":"2023-09-23T21:19:18.571821+0000","last_became_active":"2023-09-23T21:16:42.445997+0000","last_became_peered":"2023-09-23T21:16:42.445997+0000","last_unstale":"2023-09-23T21:19:18.571821+0000","last_undegraded":"2023-09-23T21:19:18.571821+0000","last_fullsized":"2023-09-23T21:19:18.571821+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:46:35.028292+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":418,"reported_epoch":646,"state":"active+clean","last_fresh":"2023-09-23T21:19:23.951715+0000","last_change":"2023-09-23T21:16:42.473872+0000","last_active":"2023-09-23T21:19:23.951715+0000","last_peered":"2023-09-23T21:19:23.951715+0000","last_clean":"2023-09-23T21:19:23.951715+0000","last_became_active":"2023-09-23T21:16:42.473595+0000","last_became_peered":"2023-09-23T21:16:42.473595+0000","last_unstale":"2023-09-23T21:19:23.951715+0000","last_undegraded":"2023-09-23T21:19:23.951715+0000","last_fullsized":"2023-09-23T21:19:23.951715+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:15:19.186428+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"12.1e","version":"643'7","reported_seq":19,"reported_epoch":644,"state":"active+clean","last_fresh":"2023-09-23T21:19:20.590787+0000","last_change":"2023-09-23T21:19:19.583873+0000","last_active":"2023-09-23T21:19:20.590787+0000","last_peered":"2023-09-23T21:19:20.590787+0000","last_clean":"2023-09-23T21:19:20.590787+0000","last_became_active":"2023-09-23T21:19:19.583569+0000","last_became_peered":"2023-09-23T21:19:19.583569+0000","last_unstale":"2023-09-23T21:19:20.590787+0000","last_undegraded":"2023-09-23T21:19:20.590787+0000","last_fullsized":"2023-09-23T21:19:20.590787+0000","mapping_epoch":642,"log_start":"0'0","ondisk_log_start":"0'0","created":642,"last_epoch_clean":643,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:19:18.561631+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:19:18.561631+0000","last_clean_scrub_stamp":"2023-09-23T21:19:18.561631+0000","objects_scrubbed":0,"log_size":7,"log_dups_size":0,"ondisk_log_size":7,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:53:34.367495+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":20,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":4,"num_read_kb":1,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,5],"acting":[4,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.13","version":"79'161","reported_seq":725,"reported_epoch":645,"state":"active+clean","last_fresh":"2023-09-23T21:19:22.698210+0000","last_change":"2023-09-23T21:13:14.933096+0000","last_active":"2023-09-23T21:19:22.698210+0000","last_peered":"2023-09-23T21:19:22.698210+0000","last_clean":"2023-09-23T21:19:22.698210+0000","last_became_active":"2023-09-23T21:13:14.932703+0000","last_became_peered":"2023-09-23T21:13:14.932703+0000","last_unstale":"2023-09-23T21:19:22.698210+0000","last_undegraded":"2023-09-23T21:19:22.698210+0000","last_fullsized":"2023-09-23T21:19:22.698210+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:36.420362+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:36.420362+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:29:42.163911+0000","scrub_duration":0.028838444000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":656,"reported_epoch":644,"state":"active+clean","last_fresh":"2023-09-23T21:19:20.590281+0000","last_change":"2023-09-23T21:16:42.420467+0000","last_active":"2023-09-23T21:19:20.590281+0000","last_peered":"2023-09-23T21:19:20.590281+0000","last_clean":"2023-09-23T21:19:20.590281+0000","last_became_active":"2023-09-23T21:16:42.420149+0000","last_became_peered":"2023-09-23T21:16:42.420149+0000","last_unstale":"2023-09-23T21:19:20.590281+0000","last_undegraded":"2023-09-23T21:19:20.590281+0000","last_fullsized":"2023-09-23T21:19:20.590281+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:52:07.355000+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":172,"reported_epoch":642,"state":"active+clean","last_fresh":"2023-09-23T21:19:18.573673+0000","last_change":"2023-09-23T21:16:42.427663+0000","last_active":"2023-09-23T21:19:18.573673+0000","last_peered":"2023-09-23T21:19:18.573673+0000","last_clean":"2023-09-23T21:19:18.573673+0000","last_became_active":"2023-09-23T21:16:42.426961+0000","last_became_peered":"2023-09-23T21:16:42.426961+0000","last_unstale":"2023-09-23T21:19:18.573673+0000","last_undegraded":"2023-09-23T21:19:18.573673+0000","last_fullsized":"2023-09-23T21:19:18.573673+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:44:27.422007+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"12.1f","version":"0'0","reported_seq":11,"reported_epoch":644,"state":"active+clean","last_fresh":"2023-09-23T21:19:20.625409+0000","last_change":"2023-09-23T21:19:19.591727+0000","last_active":"2023-09-23T21:19:20.625409+0000","last_peered":"2023-09-23T21:19:20.625409+0000","last_clean":"2023-09-23T21:19:20.625409+0000","last_became_active":"2023-09-23T21:19:19.590133+0000","last_became_peered":"2023-09-23T21:19:19.590133+0000","last_unstale":"2023-09-23T21:19:20.625409+0000","last_undegraded":"2023-09-23T21:19:20.625409+0000","last_fullsized":"2023-09-23T21:19:20.625409+0000","mapping_epoch":642,"log_start":"0'0","ondisk_log_start":"0'0","created":642,"last_epoch_clean":643,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:19:18.561631+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:19:18.561631+0000","last_clean_scrub_stamp":"2023-09-23T21:19:18.561631+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:28:21.401768+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,7],"acting":[2,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.12","version":"79'161","reported_seq":917,"reported_epoch":643,"state":"active+clean","last_fresh":"2023-09-23T21:19:19.572443+0000","last_change":"2023-09-23T21:16:45.412440+0000","last_active":"2023-09-23T21:19:19.572443+0000","last_peered":"2023-09-23T21:19:19.572443+0000","last_clean":"2023-09-23T21:19:19.572443+0000","last_became_active":"2023-09-23T21:16:45.412029+0000","last_became_peered":"2023-09-23T21:16:45.412029+0000","last_unstale":"2023-09-23T21:19:19.572443+0000","last_undegraded":"2023-09-23T21:19:19.572443+0000","last_fullsized":"2023-09-23T21:19:19.572443+0000","mapping_epoch":567,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":568,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:05:56.785173+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:05:56.785173+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:39:43.432998+0000","scrub_duration":0.0042114370000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,0],"acting":[5,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":164,"reported_epoch":642,"state":"active+clean","last_fresh":"2023-09-23T21:19:18.571517+0000","last_change":"2023-09-23T21:13:38.048791+0000","last_active":"2023-09-23T21:19:18.571517+0000","last_peered":"2023-09-23T21:19:18.571517+0000","last_clean":"2023-09-23T21:19:18.571517+0000","last_became_active":"2023-09-23T21:13:30.801386+0000","last_became_peered":"2023-09-23T21:13:30.801386+0000","last_unstale":"2023-09-23T21:19:18.571517+0000","last_undegraded":"2023-09-23T21:19:18.571517+0000","last_fullsized":"2023-09-23T21:19:18.571517+0000","mapping_epoch":501,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:52:49.170414+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":502,"reported_epoch":645,"state":"active+clean","last_fresh":"2023-09-23T21:19:22.697134+0000","last_change":"2023-09-23T21:16:42.518512+0000","last_active":"2023-09-23T21:19:22.697134+0000","last_peered":"2023-09-23T21:19:22.697134+0000","last_clean":"2023-09-23T21:19:22.697134+0000","last_became_active":"2023-09-23T21:16:42.517464+0000","last_became_peered":"2023-09-23T21:16:42.517464+0000","last_unstale":"2023-09-23T21:19:22.697134+0000","last_undegraded":"2023-09-23T21:19:22.697134+0000","last_fullsized":"2023-09-23T21:19:22.697134+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:47:52.391167+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"12.0","version":"643'3","reported_seq":16,"reported_epoch":644,"state":"active+clean","last_fresh":"2023-09-23T21:19:20.590013+0000","last_change":"2023-09-23T21:19:19.589735+0000","last_active":"2023-09-23T21:19:20.590013+0000","last_peered":"2023-09-23T21:19:20.590013+0000","last_clean":"2023-09-23T21:19:20.590013+0000","last_became_active":"2023-09-23T21:19:19.589464+0000","last_became_peered":"2023-09-23T21:19:19.589464+0000","last_unstale":"2023-09-23T21:19:20.590013+0000","last_undegraded":"2023-09-23T21:19:20.590013+0000","last_fullsized":"2023-09-23T21:19:20.590013+0000","mapping_epoch":642,"log_start":"0'0","ondisk_log_start":"0'0","created":642,"last_epoch_clean":643,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:19:18.561631+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:19:18.561631+0000","last_clean_scrub_stamp":"2023-09-23T21:19:18.561631+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:30:38.604932+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":5,"num_read_kb":3,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.d","version":"79'161","reported_seq":685,"reported_epoch":643,"state":"active+clean","last_fresh":"2023-09-23T21:19:19.582708+0000","last_change":"2023-09-23T21:18:47.440374+0000","last_active":"2023-09-23T21:19:19.582708+0000","last_peered":"2023-09-23T21:19:19.582708+0000","last_clean":"2023-09-23T21:19:19.582708+0000","last_became_active":"2023-09-23T21:16:45.416373+0000","last_became_peered":"2023-09-23T21:16:45.416373+0000","last_unstale":"2023-09-23T21:19:19.582708+0000","last_undegraded":"2023-09-23T21:19:19.582708+0000","last_fullsized":"2023-09-23T21:19:19.582708+0000","mapping_epoch":567,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":568,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:18:47.440263+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:18:47.440263+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:11:36.676449+0000","scrub_duration":0.0023849959999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":93,"reported_epoch":643,"state":"active+clean","last_fresh":"2023-09-23T21:19:19.582458+0000","last_change":"2023-09-23T21:17:02.190320+0000","last_active":"2023-09-23T21:19:19.582458+0000","last_peered":"2023-09-23T21:19:19.582458+0000","last_clean":"2023-09-23T21:19:19.582458+0000","last_became_active":"2023-09-23T21:16:42.451380+0000","last_became_peered":"2023-09-23T21:16:42.451380+0000","last_unstale":"2023-09-23T21:19:19.582458+0000","last_undegraded":"2023-09-23T21:19:19.582458+0000","last_fullsized":"2023-09-23T21:19:19.582458+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:48:58.425821+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":132,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":371,"reported_epoch":643,"state":"active+clean","last_fresh":"2023-09-23T21:19:19.571770+0000","last_change":"2023-09-23T21:16:42.428491+0000","last_active":"2023-09-23T21:19:19.571770+0000","last_peered":"2023-09-23T21:19:19.571770+0000","last_clean":"2023-09-23T21:19:19.571770+0000","last_became_active":"2023-09-23T21:16:42.428166+0000","last_became_peered":"2023-09-23T21:16:42.428166+0000","last_unstale":"2023-09-23T21:19:19.571770+0000","last_undegraded":"2023-09-23T21:19:19.571770+0000","last_fullsized":"2023-09-23T21:19:19.571770+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:23:01.430714+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,4],"acting":[5,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"12.1","version":"645'23","reported_seq":38,"reported_epoch":645,"state":"active+clean","last_fresh":"2023-09-23T21:19:23.267014+0000","last_change":"2023-09-23T21:19:19.585813+0000","last_active":"2023-09-23T21:19:23.267014+0000","last_peered":"2023-09-23T21:19:23.267014+0000","last_clean":"2023-09-23T21:19:23.267014+0000","last_became_active":"2023-09-23T21:19:19.585521+0000","last_became_peered":"2023-09-23T21:19:19.585521+0000","last_unstale":"2023-09-23T21:19:23.267014+0000","last_undegraded":"2023-09-23T21:19:23.267014+0000","last_fullsized":"2023-09-23T21:19:23.267014+0000","mapping_epoch":642,"log_start":"0'0","ondisk_log_start":"0'0","created":642,"last_epoch_clean":643,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:19:18.561631+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:19:18.561631+0000","last_clean_scrub_stamp":"2023-09-23T21:19:18.561631+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:44:54.969852+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":34,"num_read_kb":34,"num_write":22,"num_write_kb":16,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.c","version":"79'161","reported_seq":1028,"reported_epoch":645,"state":"active+clean","last_fresh":"2023-09-23T21:19:22.698258+0000","last_change":"2023-09-23T21:15:31.992756+0000","last_active":"2023-09-23T21:19:22.698258+0000","last_peered":"2023-09-23T21:19:22.698258+0000","last_clean":"2023-09-23T21:19:22.698258+0000","last_became_active":"2023-09-23T21:13:13.401526+0000","last_became_peered":"2023-09-23T21:13:13.401526+0000","last_unstale":"2023-09-23T21:19:22.698258+0000","last_undegraded":"2023-09-23T21:19:22.698258+0000","last_fullsized":"2023-09-23T21:19:22.698258+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":532,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:15:31.992598+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:15:31.992598+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:41:14.215543+0000","scrub_duration":0.002960678,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":93,"reported_epoch":643,"state":"active+clean","last_fresh":"2023-09-23T21:19:19.581114+0000","last_change":"2023-09-23T21:17:02.215996+0000","last_active":"2023-09-23T21:19:19.581114+0000","last_peered":"2023-09-23T21:19:19.581114+0000","last_clean":"2023-09-23T21:19:19.581114+0000","last_became_active":"2023-09-23T21:16:42.451205+0000","last_became_peered":"2023-09-23T21:16:42.451205+0000","last_unstale":"2023-09-23T21:19:19.581114+0000","last_undegraded":"2023-09-23T21:19:19.581114+0000","last_fullsized":"2023-09-23T21:19:19.581114+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:51:46.876281+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":607,"reported_epoch":644,"state":"active+clean","last_fresh":"2023-09-23T21:19:20.590346+0000","last_change":"2023-09-23T21:13:15.152792+0000","last_active":"2023-09-23T21:19:20.590346+0000","last_peered":"2023-09-23T21:19:20.590346+0000","last_clean":"2023-09-23T21:19:20.590346+0000","last_became_active":"2023-09-23T21:13:14.515084+0000","last_became_peered":"2023-09-23T21:13:14.515084+0000","last_unstale":"2023-09-23T21:19:20.590346+0000","last_undegraded":"2023-09-23T21:19:20.590346+0000","last_fullsized":"2023-09-23T21:19:20.590346+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:08:16.702072+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.2","version":"643'1","reported_seq":13,"reported_epoch":644,"state":"active+clean","last_fresh":"2023-09-23T21:19:20.590361+0000","last_change":"2023-09-23T21:19:19.585592+0000","last_active":"2023-09-23T21:19:20.590361+0000","last_peered":"2023-09-23T21:19:20.590361+0000","last_clean":"2023-09-23T21:19:20.590361+0000","last_became_active":"2023-09-23T21:19:19.583620+0000","last_became_peered":"2023-09-23T21:19:19.583620+0000","last_unstale":"2023-09-23T21:19:20.590361+0000","last_undegraded":"2023-09-23T21:19:20.590361+0000","last_fullsized":"2023-09-23T21:19:20.590361+0000","mapping_epoch":642,"log_start":"0'0","ondisk_log_start":"0'0","created":642,"last_epoch_clean":643,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:19:18.561631+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:19:18.561631+0000","last_clean_scrub_stamp":"2023-09-23T21:19:18.561631+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:15:46.459866+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,7],"acting":[4,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.f","version":"79'161","reported_seq":931,"reported_epoch":643,"state":"active+clean","last_fresh":"2023-09-23T21:19:19.572258+0000","last_change":"2023-09-23T21:11:30.750338+0000","last_active":"2023-09-23T21:19:19.572258+0000","last_peered":"2023-09-23T21:19:19.572258+0000","last_clean":"2023-09-23T21:19:19.572258+0000","last_became_active":"2023-09-23T21:11:30.749953+0000","last_became_peered":"2023-09-23T21:11:30.749953+0000","last_unstale":"2023-09-23T21:19:19.572258+0000","last_undegraded":"2023-09-23T21:19:19.572258+0000","last_fullsized":"2023-09-23T21:19:19.572258+0000","mapping_epoch":455,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":456,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:27.880776+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:27.880776+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:07:10.405723+0000","scrub_duration":0.0014004899999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,3],"acting":[5,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":395,"reported_epoch":643,"state":"active+clean","last_fresh":"2023-09-23T21:19:19.572023+0000","last_change":"2023-09-23T21:17:02.175012+0000","last_active":"2023-09-23T21:19:19.572023+0000","last_peered":"2023-09-23T21:19:19.572023+0000","last_clean":"2023-09-23T21:19:19.572023+0000","last_became_active":"2023-09-23T21:16:42.451114+0000","last_became_peered":"2023-09-23T21:16:42.451114+0000","last_unstale":"2023-09-23T21:19:19.572023+0000","last_undegraded":"2023-09-23T21:19:19.572023+0000","last_fullsized":"2023-09-23T21:19:19.572023+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:00:53.868525+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":420,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":603,"reported_epoch":643,"state":"active+clean","last_fresh":"2023-09-23T21:19:19.572223+0000","last_change":"2023-09-23T21:16:42.432943+0000","last_active":"2023-09-23T21:19:19.572223+0000","last_peered":"2023-09-23T21:19:19.572223+0000","last_clean":"2023-09-23T21:19:19.572223+0000","last_became_active":"2023-09-23T21:16:42.427440+0000","last_became_peered":"2023-09-23T21:16:42.427440+0000","last_unstale":"2023-09-23T21:19:19.572223+0000","last_undegraded":"2023-09-23T21:19:19.572223+0000","last_fullsized":"2023-09-23T21:19:19.572223+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:59:32.044609+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,0],"acting":[5,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"12.3","version":"0'0","reported_seq":10,"reported_epoch":643,"state":"active+clean","last_fresh":"2023-09-23T21:19:19.592607+0000","last_change":"2023-09-23T21:19:19.588468+0000","last_active":"2023-09-23T21:19:19.592607+0000","last_peered":"2023-09-23T21:19:19.592607+0000","last_clean":"2023-09-23T21:19:19.592607+0000","last_became_active":"2023-09-23T21:19:19.588124+0000","last_became_peered":"2023-09-23T21:19:19.588124+0000","last_unstale":"2023-09-23T21:19:19.592607+0000","last_undegraded":"2023-09-23T21:19:19.592607+0000","last_fullsized":"2023-09-23T21:19:19.592607+0000","mapping_epoch":642,"log_start":"0'0","ondisk_log_start":"0'0","created":642,"last_epoch_clean":643,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:19:18.561631+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:19:18.561631+0000","last_clean_scrub_stamp":"2023-09-23T21:19:18.561631+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:18:36.805559+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.e","version":"79'161","reported_seq":863,"reported_epoch":644,"state":"active+clean","last_fresh":"2023-09-23T21:19:20.590988+0000","last_change":"2023-09-23T21:16:45.433204+0000","last_active":"2023-09-23T21:19:20.590988+0000","last_peered":"2023-09-23T21:19:20.590988+0000","last_clean":"2023-09-23T21:19:20.590988+0000","last_became_active":"2023-09-23T21:16:45.432836+0000","last_became_peered":"2023-09-23T21:16:45.432836+0000","last_unstale":"2023-09-23T21:19:20.590988+0000","last_undegraded":"2023-09-23T21:19:20.590988+0000","last_fullsized":"2023-09-23T21:19:20.590988+0000","mapping_epoch":567,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":568,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:15:39.046962+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:15:39.046962+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:11:20.720731+0000","scrub_duration":0.0082638769999999993,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":445,"reported_epoch":645,"state":"active+clean","last_fresh":"2023-09-23T21:19:22.698405+0000","last_change":"2023-09-23T21:13:13.402418+0000","last_active":"2023-09-23T21:19:22.698405+0000","last_peered":"2023-09-23T21:19:22.698405+0000","last_clean":"2023-09-23T21:19:22.698405+0000","last_became_active":"2023-09-23T21:13:13.400984+0000","last_became_peered":"2023-09-23T21:13:13.400984+0000","last_unstale":"2023-09-23T21:19:22.698405+0000","last_undegraded":"2023-09-23T21:19:22.698405+0000","last_fullsized":"2023-09-23T21:19:22.698405+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":495,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:28:01.411191+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,0],"acting":[3,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":617,"reported_epoch":642,"state":"active+clean","last_fresh":"2023-09-23T21:19:18.571928+0000","last_change":"2023-09-23T21:13:14.339670+0000","last_active":"2023-09-23T21:19:18.571928+0000","last_peered":"2023-09-23T21:19:18.571928+0000","last_clean":"2023-09-23T21:19:18.571928+0000","last_became_active":"2023-09-23T21:13:14.339341+0000","last_became_peered":"2023-09-23T21:13:14.339341+0000","last_unstale":"2023-09-23T21:19:18.571928+0000","last_undegraded":"2023-09-23T21:19:18.571928+0000","last_fullsized":"2023-09-23T21:19:18.571928+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:23:19.411368+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"12.4","version":"0'0","reported_seq":11,"reported_epoch":644,"state":"active+clean","last_fresh":"2023-09-23T21:19:20.590509+0000","last_change":"2023-09-23T21:19:19.586272+0000","last_active":"2023-09-23T21:19:20.590509+0000","last_peered":"2023-09-23T21:19:20.590509+0000","last_clean":"2023-09-23T21:19:20.590509+0000","last_became_active":"2023-09-23T21:19:19.585958+0000","last_became_peered":"2023-09-23T21:19:19.585958+0000","last_unstale":"2023-09-23T21:19:20.590509+0000","last_undegraded":"2023-09-23T21:19:20.590509+0000","last_fullsized":"2023-09-23T21:19:20.590509+0000","mapping_epoch":642,"log_start":"0'0","ondisk_log_start":"0'0","created":642,"last_epoch_clean":643,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:19:18.561631+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:19:18.561631+0000","last_clean_scrub_stamp":"2023-09-23T21:19:18.561631+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:51:59.835282+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,7],"acting":[4,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.9","version":"79'161","reported_seq":846,"reported_epoch":643,"state":"active+clean","last_fresh":"2023-09-23T21:19:19.578625+0000","last_change":"2023-09-23T21:16:45.433968+0000","last_active":"2023-09-23T21:19:19.578625+0000","last_peered":"2023-09-23T21:19:19.578625+0000","last_clean":"2023-09-23T21:19:19.578625+0000","last_became_active":"2023-09-23T21:16:45.433602+0000","last_became_peered":"2023-09-23T21:16:45.433602+0000","last_unstale":"2023-09-23T21:19:19.578625+0000","last_undegraded":"2023-09-23T21:19:19.578625+0000","last_fullsized":"2023-09-23T21:19:19.578625+0000","mapping_epoch":567,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":568,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:15:43.437369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:15:43.437369+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:16:44.538102+0000","scrub_duration":0.0024129429999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,3],"acting":[2,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":203,"reported_epoch":642,"state":"active+clean","last_fresh":"2023-09-23T21:19:18.572019+0000","last_change":"2023-09-23T21:13:30.793273+0000","last_active":"2023-09-23T21:19:18.572019+0000","last_peered":"2023-09-23T21:19:18.572019+0000","last_clean":"2023-09-23T21:19:18.572019+0000","last_became_active":"2023-09-23T21:13:30.791269+0000","last_became_peered":"2023-09-23T21:13:30.791269+0000","last_unstale":"2023-09-23T21:19:18.572019+0000","last_undegraded":"2023-09-23T21:19:18.572019+0000","last_fullsized":"2023-09-23T21:19:18.572019+0000","mapping_epoch":501,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:38:07.142290+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":238,"reported_epoch":644,"state":"active+clean","last_fresh":"2023-09-23T21:19:20.590534+0000","last_change":"2023-09-23T21:16:42.417057+0000","last_active":"2023-09-23T21:19:20.590534+0000","last_peered":"2023-09-23T21:19:20.590534+0000","last_clean":"2023-09-23T21:19:20.590534+0000","last_became_active":"2023-09-23T21:16:42.416601+0000","last_became_peered":"2023-09-23T21:16:42.416601+0000","last_unstale":"2023-09-23T21:19:20.590534+0000","last_undegraded":"2023-09-23T21:19:20.590534+0000","last_fullsized":"2023-09-23T21:19:20.590534+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:56:52.167562+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,7],"acting":[4,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.5","version":"0'0","reported_seq":11,"reported_epoch":644,"state":"active+clean","last_fresh":"2023-09-23T21:19:20.626097+0000","last_change":"2023-09-23T21:19:19.591733+0000","last_active":"2023-09-23T21:19:20.626097+0000","last_peered":"2023-09-23T21:19:20.626097+0000","last_clean":"2023-09-23T21:19:20.626097+0000","last_became_active":"2023-09-23T21:19:19.589795+0000","last_became_peered":"2023-09-23T21:19:19.589795+0000","last_unstale":"2023-09-23T21:19:20.626097+0000","last_undegraded":"2023-09-23T21:19:20.626097+0000","last_fullsized":"2023-09-23T21:19:20.626097+0000","mapping_epoch":642,"log_start":"0'0","ondisk_log_start":"0'0","created":642,"last_epoch_clean":643,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:19:18.561631+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:19:18.561631+0000","last_clean_scrub_stamp":"2023-09-23T21:19:18.561631+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:39:41.719504+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.8","version":"79'161","reported_seq":778,"reported_epoch":643,"state":"active+clean","last_fresh":"2023-09-23T21:19:19.578168+0000","last_change":"2023-09-23T21:15:36.367819+0000","last_active":"2023-09-23T21:19:19.578168+0000","last_peered":"2023-09-23T21:19:19.578168+0000","last_clean":"2023-09-23T21:19:19.578168+0000","last_became_active":"2023-09-23T21:12:30.433217+0000","last_became_peered":"2023-09-23T21:12:30.433217+0000","last_unstale":"2023-09-23T21:19:19.578168+0000","last_undegraded":"2023-09-23T21:19:19.578168+0000","last_fullsized":"2023-09-23T21:19:19.578168+0000","mapping_epoch":359,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:15:36.367640+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:15:36.367640+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:49:35.243653+0000","scrub_duration":0.005826899,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":221,"reported_epoch":644,"state":"active+clean","last_fresh":"2023-09-23T21:19:20.590662+0000","last_change":"2023-09-23T21:12:10.823380+0000","last_active":"2023-09-23T21:19:20.590662+0000","last_peered":"2023-09-23T21:19:20.590662+0000","last_clean":"2023-09-23T21:19:20.590662+0000","last_became_active":"2023-09-23T21:12:10.823044+0000","last_became_peered":"2023-09-23T21:12:10.823044+0000","last_unstale":"2023-09-23T21:19:20.590662+0000","last_undegraded":"2023-09-23T21:19:20.590662+0000","last_fullsized":"2023-09-23T21:19:20.590662+0000","mapping_epoch":470,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:53:29.846536+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,5],"acting":[4,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":222,"reported_epoch":644,"state":"active+clean","last_fresh":"2023-09-23T21:19:20.590621+0000","last_change":"2023-09-23T21:13:30.772451+0000","last_active":"2023-09-23T21:19:20.590621+0000","last_peered":"2023-09-23T21:19:20.590621+0000","last_clean":"2023-09-23T21:19:20.590621+0000","last_became_active":"2023-09-23T21:13:30.772040+0000","last_became_peered":"2023-09-23T21:13:30.772040+0000","last_unstale":"2023-09-23T21:19:20.590621+0000","last_undegraded":"2023-09-23T21:19:20.590621+0000","last_fullsized":"2023-09-23T21:19:20.590621+0000","mapping_epoch":501,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:24:49.246727+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,0],"acting":[4,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.6","version":"643'450","reported_seq":509,"reported_epoch":643,"state":"active+clean","last_fresh":"2023-09-23T21:19:20.501465+0000","last_change":"2023-09-23T21:19:19.588515+0000","last_active":"2023-09-23T21:19:20.501465+0000","last_peered":"2023-09-23T21:19:20.501465+0000","last_clean":"2023-09-23T21:19:20.501465+0000","last_became_active":"2023-09-23T21:19:19.588233+0000","last_became_peered":"2023-09-23T21:19:19.588233+0000","last_unstale":"2023-09-23T21:19:20.501465+0000","last_undegraded":"2023-09-23T21:19:20.501465+0000","last_fullsized":"2023-09-23T21:19:20.501465+0000","mapping_epoch":642,"log_start":"0'0","ondisk_log_start":"0'0","created":642,"last_epoch_clean":643,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:19:18.561631+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:19:18.561631+0000","last_clean_scrub_stamp":"2023-09-23T21:19:18.561631+0000","objects_scrubbed":0,"log_size":450,"log_dups_size":0,"ondisk_log_size":450,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:01:24.240052+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":1024,"num_read_kb":991,"num_write":642,"num_write_kb":605,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.b","version":"79'161","reported_seq":660,"reported_epoch":642,"state":"active+clean","last_fresh":"2023-09-23T21:19:18.572758+0000","last_change":"2023-09-23T21:15:36.340009+0000","last_active":"2023-09-23T21:19:18.572758+0000","last_peered":"2023-09-23T21:19:18.572758+0000","last_clean":"2023-09-23T21:19:18.572758+0000","last_became_active":"2023-09-23T21:13:13.372328+0000","last_became_peered":"2023-09-23T21:13:13.372328+0000","last_unstale":"2023-09-23T21:19:18.572758+0000","last_undegraded":"2023-09-23T21:19:18.572758+0000","last_fullsized":"2023-09-23T21:19:18.572758+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":495,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:15:36.339885+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:15:36.339885+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:01:26.605006+0000","scrub_duration":0.0068473279999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":222,"reported_epoch":644,"state":"active+clean","last_fresh":"2023-09-23T21:19:20.590250+0000","last_change":"2023-09-23T21:16:42.425103+0000","last_active":"2023-09-23T21:19:20.590250+0000","last_peered":"2023-09-23T21:19:20.590250+0000","last_clean":"2023-09-23T21:19:20.590250+0000","last_became_active":"2023-09-23T21:16:42.424543+0000","last_became_peered":"2023-09-23T21:16:42.424543+0000","last_unstale":"2023-09-23T21:19:20.590250+0000","last_undegraded":"2023-09-23T21:19:20.590250+0000","last_fullsized":"2023-09-23T21:19:20.590250+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:11:28.885246+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":576,"reported_epoch":644,"state":"active+clean","last_fresh":"2023-09-23T21:19:20.591084+0000","last_change":"2023-09-23T21:12:10.811698+0000","last_active":"2023-09-23T21:19:20.591084+0000","last_peered":"2023-09-23T21:19:20.591084+0000","last_clean":"2023-09-23T21:19:20.591084+0000","last_became_active":"2023-09-23T21:12:10.789656+0000","last_became_peered":"2023-09-23T21:12:10.789656+0000","last_unstale":"2023-09-23T21:19:20.591084+0000","last_undegraded":"2023-09-23T21:19:20.591084+0000","last_fullsized":"2023-09-23T21:19:20.591084+0000","mapping_epoch":470,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:39:15.112091+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,5],"acting":[4,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.7","version":"643'23","reported_seq":54,"reported_epoch":643,"state":"active+clean","last_fresh":"2023-09-23T21:19:19.845770+0000","last_change":"2023-09-23T21:19:19.592993+0000","last_active":"2023-09-23T21:19:19.845770+0000","last_peered":"2023-09-23T21:19:19.845770+0000","last_clean":"2023-09-23T21:19:19.845770+0000","last_became_active":"2023-09-23T21:19:19.592719+0000","last_became_peered":"2023-09-23T21:19:19.592719+0000","last_unstale":"2023-09-23T21:19:19.845770+0000","last_undegraded":"2023-09-23T21:19:19.845770+0000","last_fullsized":"2023-09-23T21:19:19.845770+0000","mapping_epoch":642,"log_start":"0'0","ondisk_log_start":"0'0","created":642,"last_epoch_clean":643,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:19:18.561631+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:19:18.561631+0000","last_clean_scrub_stamp":"2023-09-23T21:19:18.561631+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:04:41.365907+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":40,"num_read_kb":39,"num_write":17,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,4],"acting":[7,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.a","version":"79'161","reported_seq":804,"reported_epoch":643,"state":"active+clean","last_fresh":"2023-09-23T21:19:19.580137+0000","last_change":"2023-09-23T21:15:46.420777+0000","last_active":"2023-09-23T21:19:19.580137+0000","last_peered":"2023-09-23T21:19:19.580137+0000","last_clean":"2023-09-23T21:19:19.580137+0000","last_became_active":"2023-09-23T21:12:30.437427+0000","last_became_peered":"2023-09-23T21:12:30.437427+0000","last_unstale":"2023-09-23T21:19:19.580137+0000","last_undegraded":"2023-09-23T21:19:19.580137+0000","last_fullsized":"2023-09-23T21:19:19.580137+0000","mapping_epoch":446,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:15:46.420650+0000","last_deep_scrub":"79'161","last_deep_scrub_stamp":"2023-09-23T21:05:32.630292+0000","last_clean_scrub_stamp":"2023-09-23T21:15:46.420650+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:50:23.452475+0000","scrub_duration":0.0072898930000000004,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,4],"acting":[2,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":513,"reported_epoch":646,"state":"active+clean","last_fresh":"2023-09-23T21:19:23.950883+0000","last_change":"2023-09-23T21:16:42.447403+0000","last_active":"2023-09-23T21:19:23.950883+0000","last_peered":"2023-09-23T21:19:23.950883+0000","last_clean":"2023-09-23T21:19:23.950883+0000","last_became_active":"2023-09-23T21:16:42.447080+0000","last_became_peered":"2023-09-23T21:16:42.447080+0000","last_unstale":"2023-09-23T21:19:23.950883+0000","last_undegraded":"2023-09-23T21:19:23.950883+0000","last_fullsized":"2023-09-23T21:19:23.950883+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:15:19.446334+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,4],"acting":[3,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":644,"reported_epoch":645,"state":"active+clean","last_fresh":"2023-09-23T21:19:22.697115+0000","last_change":"2023-09-23T21:16:42.518739+0000","last_active":"2023-09-23T21:19:22.697115+0000","last_peered":"2023-09-23T21:19:22.697115+0000","last_clean":"2023-09-23T21:19:22.697115+0000","last_became_active":"2023-09-23T21:16:42.517945+0000","last_became_peered":"2023-09-23T21:16:42.517945+0000","last_unstale":"2023-09-23T21:19:22.697115+0000","last_undegraded":"2023-09-23T21:19:22.697115+0000","last_fullsized":"2023-09-23T21:19:22.697115+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:47:26.236933+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"12.a","version":"643'14","reported_seq":31,"reported_epoch":644,"state":"active+clean","last_fresh":"2023-09-23T21:19:20.625264+0000","last_change":"2023-09-23T21:19:19.590235+0000","last_active":"2023-09-23T21:19:20.625264+0000","last_peered":"2023-09-23T21:19:20.625264+0000","last_clean":"2023-09-23T21:19:20.625264+0000","last_became_active":"2023-09-23T21:19:19.589367+0000","last_became_peered":"2023-09-23T21:19:19.589367+0000","last_unstale":"2023-09-23T21:19:20.625264+0000","last_undegraded":"2023-09-23T21:19:20.625264+0000","last_fullsized":"2023-09-23T21:19:20.625264+0000","mapping_epoch":642,"log_start":"0'0","ondisk_log_start":"0'0","created":642,"last_epoch_clean":643,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:19:18.561631+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:19:18.561631+0000","last_clean_scrub_stamp":"2023-09-23T21:19:18.561631+0000","objects_scrubbed":0,"log_size":14,"log_dups_size":0,"ondisk_log_size":14,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:17:45.343012+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":44,"num_read_kb":35,"num_write":10,"num_write_kb":8,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,3],"acting":[2,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.7","version":"79'161","reported_seq":786,"reported_epoch":645,"state":"active+clean","last_fresh":"2023-09-23T21:19:22.698735+0000","last_change":"2023-09-23T21:16:45.425645+0000","last_active":"2023-09-23T21:19:22.698735+0000","last_peered":"2023-09-23T21:19:22.698735+0000","last_clean":"2023-09-23T21:19:22.698735+0000","last_became_active":"2023-09-23T21:16:45.425241+0000","last_became_peered":"2023-09-23T21:16:45.425241+0000","last_unstale":"2023-09-23T21:19:22.698735+0000","last_undegraded":"2023-09-23T21:19:22.698735+0000","last_fullsized":"2023-09-23T21:19:22.698735+0000","mapping_epoch":567,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":568,"parent":"0.0","parent_split_bits":3,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:15:51.016427+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:15:51.016427+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:49:31.176462+0000","scrub_duration":0.0017648640000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":592,"reported_epoch":642,"state":"active+clean","last_fresh":"2023-09-23T21:19:18.571875+0000","last_change":"2023-09-23T21:16:42.518826+0000","last_active":"2023-09-23T21:19:18.571875+0000","last_peered":"2023-09-23T21:19:18.571875+0000","last_clean":"2023-09-23T21:19:18.571875+0000","last_became_active":"2023-09-23T21:16:42.518488+0000","last_became_peered":"2023-09-23T21:16:42.518488+0000","last_unstale":"2023-09-23T21:19:18.571875+0000","last_undegraded":"2023-09-23T21:19:18.571875+0000","last_fullsized":"2023-09-23T21:19:18.571875+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:42:44.144383+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,2],"acting":[0,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":321,"reported_epoch":643,"state":"active+clean","last_fresh":"2023-09-23T21:19:19.571961+0000","last_change":"2023-09-23T21:05:04.326889+0000","last_active":"2023-09-23T21:19:19.571961+0000","last_peered":"2023-09-23T21:19:19.571961+0000","last_clean":"2023-09-23T21:19:19.571961+0000","last_became_active":"2023-09-23T21:05:04.326032+0000","last_became_peered":"2023-09-23T21:05:04.326032+0000","last_unstale":"2023-09-23T21:19:19.571961+0000","last_undegraded":"2023-09-23T21:19:19.571961+0000","last_fullsized":"2023-09-23T21:19:19.571961+0000","mapping_epoch":332,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:00:17.507889+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":653,"reported_epoch":643,"state":"active+clean","last_fresh":"2023-09-23T21:19:19.572306+0000","last_change":"2023-09-23T21:13:13.285608+0000","last_active":"2023-09-23T21:19:19.572306+0000","last_peered":"2023-09-23T21:19:19.572306+0000","last_clean":"2023-09-23T21:19:19.572306+0000","last_became_active":"2023-09-23T21:13:13.285255+0000","last_became_peered":"2023-09-23T21:13:13.285255+0000","last_unstale":"2023-09-23T21:19:19.572306+0000","last_undegraded":"2023-09-23T21:19:19.572306+0000","last_fullsized":"2023-09-23T21:19:19.572306+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":495,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:42:30.087599+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"12.c","version":"0'0","reported_seq":10,"reported_epoch":643,"state":"active+clean","last_fresh":"2023-09-23T21:19:19.585219+0000","last_change":"2023-09-23T21:19:19.583924+0000","last_active":"2023-09-23T21:19:19.585219+0000","last_peered":"2023-09-23T21:19:19.585219+0000","last_clean":"2023-09-23T21:19:19.585219+0000","last_became_active":"2023-09-23T21:19:19.583391+0000","last_became_peered":"2023-09-23T21:19:19.583391+0000","last_unstale":"2023-09-23T21:19:19.585219+0000","last_undegraded":"2023-09-23T21:19:19.585219+0000","last_fullsized":"2023-09-23T21:19:19.585219+0000","mapping_epoch":642,"log_start":"0'0","ondisk_log_start":"0'0","created":642,"last_epoch_clean":643,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:19:18.561631+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:19:18.561631+0000","last_clean_scrub_stamp":"2023-09-23T21:19:18.561631+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:52:43.635932+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1","version":"79'161","reported_seq":727,"reported_epoch":642,"state":"active+clean","last_fresh":"2023-09-23T21:19:18.573679+0000","last_change":"2023-09-23T21:15:34.345290+0000","last_active":"2023-09-23T21:19:18.573679+0000","last_peered":"2023-09-23T21:19:18.573679+0000","last_clean":"2023-09-23T21:19:18.573679+0000","last_became_active":"2023-09-23T21:13:12.203380+0000","last_became_peered":"2023-09-23T21:13:12.203380+0000","last_unstale":"2023-09-23T21:19:18.573679+0000","last_undegraded":"2023-09-23T21:19:18.573679+0000","last_fullsized":"2023-09-23T21:19:18.573679+0000","mapping_epoch":493,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:15:34.345138+0000","last_deep_scrub":"79'161","last_deep_scrub_stamp":"2023-09-23T21:05:30.652189+0000","last_clean_scrub_stamp":"2023-09-23T21:15:34.345138+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:47:50.349409+0000","scrub_duration":0.0057520000000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":311,"reported_epoch":642,"state":"active+clean","last_fresh":"2023-09-23T21:19:18.571800+0000","last_change":"2023-09-23T21:13:13.238617+0000","last_active":"2023-09-23T21:19:18.571800+0000","last_peered":"2023-09-23T21:19:18.571800+0000","last_clean":"2023-09-23T21:19:18.571800+0000","last_became_active":"2023-09-23T21:13:13.232926+0000","last_became_peered":"2023-09-23T21:13:13.232926+0000","last_unstale":"2023-09-23T21:19:18.571800+0000","last_undegraded":"2023-09-23T21:19:18.571800+0000","last_fullsized":"2023-09-23T21:19:18.571800+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":495,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:17:21.823772+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":578,"reported_epoch":644,"state":"active+clean","last_fresh":"2023-09-23T21:19:20.590841+0000","last_change":"2023-09-23T21:13:14.144918+0000","last_active":"2023-09-23T21:19:20.590841+0000","last_peered":"2023-09-23T21:19:20.590841+0000","last_clean":"2023-09-23T21:19:20.590841+0000","last_became_active":"2023-09-23T21:13:14.132388+0000","last_became_peered":"2023-09-23T21:13:14.132388+0000","last_unstale":"2023-09-23T21:19:20.590841+0000","last_undegraded":"2023-09-23T21:19:20.590841+0000","last_fullsized":"2023-09-23T21:19:20.590841+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":495,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:56:02.619905+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,5],"acting":[4,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":484,"reported_epoch":643,"state":"active+clean","last_fresh":"2023-09-23T21:19:19.579568+0000","last_change":"2023-09-23T21:16:42.491111+0000","last_active":"2023-09-23T21:19:19.579568+0000","last_peered":"2023-09-23T21:19:19.579568+0000","last_clean":"2023-09-23T21:19:19.579568+0000","last_became_active":"2023-09-23T21:16:42.490275+0000","last_became_peered":"2023-09-23T21:16:42.490275+0000","last_unstale":"2023-09-23T21:19:19.579568+0000","last_undegraded":"2023-09-23T21:19:19.579568+0000","last_fullsized":"2023-09-23T21:19:19.579568+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:39:11.652564+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,7],"acting":[2,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"12.f","version":"643'10","reported_seq":25,"reported_epoch":643,"state":"active+clean","last_fresh":"2023-09-23T21:19:20.031017+0000","last_change":"2023-09-23T21:19:19.585791+0000","last_active":"2023-09-23T21:19:20.031017+0000","last_peered":"2023-09-23T21:19:20.031017+0000","last_clean":"2023-09-23T21:19:20.031017+0000","last_became_active":"2023-09-23T21:19:19.585499+0000","last_became_peered":"2023-09-23T21:19:19.585499+0000","last_unstale":"2023-09-23T21:19:20.031017+0000","last_undegraded":"2023-09-23T21:19:20.031017+0000","last_fullsized":"2023-09-23T21:19:20.031017+0000","mapping_epoch":642,"log_start":"0'0","ondisk_log_start":"0'0","created":642,"last_epoch_clean":643,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:19:18.561631+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:19:18.561631+0000","last_clean_scrub_stamp":"2023-09-23T21:19:18.561631+0000","objects_scrubbed":0,"log_size":10,"log_dups_size":0,"ondisk_log_size":10,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:54:24.666695+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":38,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":18,"num_read_kb":8199,"num_write":8,"num_write_kb":4100,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,4],"acting":[5,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.2","version":"79'161","reported_seq":958,"reported_epoch":642,"state":"active+clean","last_fresh":"2023-09-23T21:19:18.571644+0000","last_change":"2023-09-23T21:16:45.417075+0000","last_active":"2023-09-23T21:19:18.571644+0000","last_peered":"2023-09-23T21:19:18.571644+0000","last_clean":"2023-09-23T21:19:18.571644+0000","last_became_active":"2023-09-23T21:16:45.416752+0000","last_became_peered":"2023-09-23T21:16:45.416752+0000","last_unstale":"2023-09-23T21:19:18.571644+0000","last_undegraded":"2023-09-23T21:19:18.571644+0000","last_fullsized":"2023-09-23T21:19:18.571644+0000","mapping_epoch":567,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":568,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:24.802451+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:24.802451+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:01:51.129779+0000","scrub_duration":0.0090861250000000005,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,4],"acting":[0,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":654,"reported_epoch":643,"state":"active+clean","last_fresh":"2023-09-23T21:19:19.578825+0000","last_change":"2023-09-23T21:13:12.206980+0000","last_active":"2023-09-23T21:19:19.578825+0000","last_peered":"2023-09-23T21:19:19.578825+0000","last_clean":"2023-09-23T21:19:19.578825+0000","last_became_active":"2023-09-23T21:13:12.206650+0000","last_became_peered":"2023-09-23T21:13:12.206650+0000","last_unstale":"2023-09-23T21:19:19.578825+0000","last_undegraded":"2023-09-23T21:19:19.578825+0000","last_fullsized":"2023-09-23T21:19:19.578825+0000","mapping_epoch":493,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:10:01.892936+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":638,"reported_epoch":643,"state":"active+clean","last_fresh":"2023-09-23T21:19:19.579961+0000","last_change":"2023-09-23T21:13:13.407073+0000","last_active":"2023-09-23T21:19:19.579961+0000","last_peered":"2023-09-23T21:19:19.579961+0000","last_clean":"2023-09-23T21:19:19.579961+0000","last_became_active":"2023-09-23T21:13:13.395047+0000","last_became_peered":"2023-09-23T21:13:13.395047+0000","last_unstale":"2023-09-23T21:19:19.579961+0000","last_undegraded":"2023-09-23T21:19:19.579961+0000","last_fullsized":"2023-09-23T21:19:19.579961+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":495,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:04:43.384190+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,5],"acting":[2,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":172,"reported_epoch":642,"state":"active+clean","last_fresh":"2023-09-23T21:19:18.573346+0000","last_change":"2023-09-23T21:16:42.517729+0000","last_active":"2023-09-23T21:19:18.573346+0000","last_peered":"2023-09-23T21:19:18.573346+0000","last_clean":"2023-09-23T21:19:18.573346+0000","last_became_active":"2023-09-23T21:16:42.517373+0000","last_became_peered":"2023-09-23T21:16:42.517373+0000","last_unstale":"2023-09-23T21:19:18.573346+0000","last_undegraded":"2023-09-23T21:19:18.573346+0000","last_fullsized":"2023-09-23T21:19:18.573346+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:05:57.370550+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"12.d","version":"643'30","reported_seq":65,"reported_epoch":643,"state":"active+clean","last_fresh":"2023-09-23T21:19:20.027275+0000","last_change":"2023-09-23T21:19:19.594405+0000","last_active":"2023-09-23T21:19:20.027275+0000","last_peered":"2023-09-23T21:19:20.027275+0000","last_clean":"2023-09-23T21:19:20.027275+0000","last_became_active":"2023-09-23T21:19:19.594116+0000","last_became_peered":"2023-09-23T21:19:19.594116+0000","last_unstale":"2023-09-23T21:19:20.027275+0000","last_undegraded":"2023-09-23T21:19:20.027275+0000","last_fullsized":"2023-09-23T21:19:20.027275+0000","mapping_epoch":642,"log_start":"0'0","ondisk_log_start":"0'0","created":642,"last_epoch_clean":643,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:19:18.561631+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:19:18.561631+0000","last_clean_scrub_stamp":"2023-09-23T21:19:18.561631+0000","objects_scrubbed":0,"log_size":30,"log_dups_size":0,"ondisk_log_size":30,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:29:16.006493+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":80,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":139,"num_read_kb":104,"num_write":29,"num_write_kb":24,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,3],"acting":[7,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":449,"reported_epoch":643,"state":"active+clean","last_fresh":"2023-09-23T21:19:19.579295+0000","last_change":"2023-09-23T21:16:42.423216+0000","last_active":"2023-09-23T21:19:19.579295+0000","last_peered":"2023-09-23T21:19:19.579295+0000","last_clean":"2023-09-23T21:19:19.579295+0000","last_became_active":"2023-09-23T21:16:42.422856+0000","last_became_peered":"2023-09-23T21:16:42.422856+0000","last_unstale":"2023-09-23T21:19:19.579295+0000","last_undegraded":"2023-09-23T21:19:19.579295+0000","last_fullsized":"2023-09-23T21:19:19.579295+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:06:54.105798+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"79'161","reported_seq":861,"reported_epoch":642,"state":"active+clean","last_fresh":"2023-09-23T21:19:18.571381+0000","last_change":"2023-09-23T21:16:45.428032+0000","last_active":"2023-09-23T21:19:18.571381+0000","last_peered":"2023-09-23T21:19:18.571381+0000","last_clean":"2023-09-23T21:19:18.571381+0000","last_became_active":"2023-09-23T21:16:45.427674+0000","last_became_peered":"2023-09-23T21:16:45.427674+0000","last_unstale":"2023-09-23T21:19:18.571381+0000","last_undegraded":"2023-09-23T21:19:18.571381+0000","last_fullsized":"2023-09-23T21:19:18.571381+0000","mapping_epoch":567,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":568,"parent":"0.0","parent_split_bits":0,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:20.738273+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:20.738273+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:39:42.441711+0000","scrub_duration":0.0088336339999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,2],"acting":[0,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":651,"reported_epoch":643,"state":"active+clean","last_fresh":"2023-09-23T21:19:19.572281+0000","last_change":"2023-09-23T21:11:08.760417+0000","last_active":"2023-09-23T21:19:19.572281+0000","last_peered":"2023-09-23T21:19:19.572281+0000","last_clean":"2023-09-23T21:19:19.572281+0000","last_became_active":"2023-09-23T21:11:08.760150+0000","last_became_peered":"2023-09-23T21:11:08.760150+0000","last_unstale":"2023-09-23T21:19:19.572281+0000","last_undegraded":"2023-09-23T21:19:19.572281+0000","last_fullsized":"2023-09-23T21:19:19.572281+0000","mapping_epoch":443,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":444,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:23:10.849768+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":239,"reported_epoch":644,"state":"active+clean","last_fresh":"2023-09-23T21:19:20.590735+0000","last_change":"2023-09-23T21:16:42.424717+0000","last_active":"2023-09-23T21:19:20.590735+0000","last_peered":"2023-09-23T21:19:20.590735+0000","last_clean":"2023-09-23T21:19:20.590735+0000","last_became_active":"2023-09-23T21:16:42.424376+0000","last_became_peered":"2023-09-23T21:16:42.424376+0000","last_unstale":"2023-09-23T21:19:20.590735+0000","last_undegraded":"2023-09-23T21:19:20.590735+0000","last_fullsized":"2023-09-23T21:19:20.590735+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:25:14.162321+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.e","version":"643'9","reported_seq":47,"reported_epoch":643,"state":"active+clean","last_fresh":"2023-09-23T21:19:20.059227+0000","last_change":"2023-09-23T21:19:19.584448+0000","last_active":"2023-09-23T21:19:20.059227+0000","last_peered":"2023-09-23T21:19:20.059227+0000","last_clean":"2023-09-23T21:19:20.059227+0000","last_became_active":"2023-09-23T21:19:19.584170+0000","last_became_peered":"2023-09-23T21:19:19.584170+0000","last_unstale":"2023-09-23T21:19:20.059227+0000","last_undegraded":"2023-09-23T21:19:20.059227+0000","last_fullsized":"2023-09-23T21:19:20.059227+0000","mapping_epoch":642,"log_start":"0'0","ondisk_log_start":"0'0","created":642,"last_epoch_clean":643,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:19:18.561631+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:19:18.561631+0000","last_clean_scrub_stamp":"2023-09-23T21:19:18.561631+0000","objects_scrubbed":0,"log_size":9,"log_dups_size":0,"ondisk_log_size":9,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:54:53.628466+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":3,"num_object_clones":0,"num_object_copies":9,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3,"num_whiteouts":0,"num_read":32,"num_read_kb":52,"num_write":7,"num_write_kb":35,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":3,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.3","version":"79'161","reported_seq":947,"reported_epoch":644,"state":"active+clean","last_fresh":"2023-09-23T21:19:20.590214+0000","last_change":"2023-09-23T21:13:33.771279+0000","last_active":"2023-09-23T21:19:20.590214+0000","last_peered":"2023-09-23T21:19:20.590214+0000","last_clean":"2023-09-23T21:19:20.590214+0000","last_became_active":"2023-09-23T21:13:33.770980+0000","last_became_peered":"2023-09-23T21:13:33.770980+0000","last_unstale":"2023-09-23T21:19:20.590214+0000","last_undegraded":"2023-09-23T21:19:20.590214+0000","last_fullsized":"2023-09-23T21:19:20.590214+0000","mapping_epoch":504,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":505,"parent":"0.0","parent_split_bits":3,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:05:57.832302+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:05:57.832302+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:19:42.309604+0000","scrub_duration":0.010206847999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":480,"reported_epoch":642,"state":"active+clean","last_fresh":"2023-09-23T21:19:18.571687+0000","last_change":"2023-09-23T21:16:42.455365+0000","last_active":"2023-09-23T21:19:18.571687+0000","last_peered":"2023-09-23T21:19:18.571687+0000","last_clean":"2023-09-23T21:19:18.571687+0000","last_became_active":"2023-09-23T21:16:42.452829+0000","last_became_peered":"2023-09-23T21:16:42.452829+0000","last_unstale":"2023-09-23T21:19:18.571687+0000","last_undegraded":"2023-09-23T21:19:18.571687+0000","last_fullsized":"2023-09-23T21:19:18.571687+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:49:26.303389+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":643,"reported_epoch":642,"state":"active+clean","last_fresh":"2023-09-23T21:19:18.573385+0000","last_change":"2023-09-23T21:13:44.409212+0000","last_active":"2023-09-23T21:19:18.573385+0000","last_peered":"2023-09-23T21:19:18.573385+0000","last_clean":"2023-09-23T21:19:18.573385+0000","last_became_active":"2023-09-23T21:13:44.408958+0000","last_became_peered":"2023-09-23T21:13:44.408958+0000","last_unstale":"2023-09-23T21:19:18.573385+0000","last_undegraded":"2023-09-23T21:19:18.573385+0000","last_fullsized":"2023-09-23T21:19:18.573385+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:03:52.749964+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":172,"reported_epoch":642,"state":"active+clean","last_fresh":"2023-09-23T21:19:18.573778+0000","last_change":"2023-09-23T21:16:42.427490+0000","last_active":"2023-09-23T21:19:18.573778+0000","last_peered":"2023-09-23T21:19:18.573778+0000","last_clean":"2023-09-23T21:19:18.573778+0000","last_became_active":"2023-09-23T21:16:42.426566+0000","last_became_peered":"2023-09-23T21:16:42.426566+0000","last_unstale":"2023-09-23T21:19:18.573778+0000","last_undegraded":"2023-09-23T21:19:18.573778+0000","last_fullsized":"2023-09-23T21:19:18.573778+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:20:26.396005+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,7],"acting":[0,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"12.b","version":"643'37","reported_seq":73,"reported_epoch":644,"state":"active+clean","last_fresh":"2023-09-23T21:19:20.624741+0000","last_change":"2023-09-23T21:19:19.589572+0000","last_active":"2023-09-23T21:19:20.624741+0000","last_peered":"2023-09-23T21:19:20.624741+0000","last_clean":"2023-09-23T21:19:20.624741+0000","last_became_active":"2023-09-23T21:19:19.589224+0000","last_became_peered":"2023-09-23T21:19:19.589224+0000","last_unstale":"2023-09-23T21:19:20.624741+0000","last_undegraded":"2023-09-23T21:19:20.624741+0000","last_fullsized":"2023-09-23T21:19:20.624741+0000","mapping_epoch":642,"log_start":"0'0","ondisk_log_start":"0'0","created":642,"last_epoch_clean":643,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:19:18.561631+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:19:18.561631+0000","last_clean_scrub_stamp":"2023-09-23T21:19:18.561631+0000","objects_scrubbed":0,"log_size":37,"log_dups_size":0,"ondisk_log_size":37,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:58:57.310979+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":38,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":215,"num_read_kb":174,"num_write":44,"num_write_kb":41,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.6","version":"79'161","reported_seq":575,"reported_epoch":642,"state":"active+clean","last_fresh":"2023-09-23T21:19:18.573716+0000","last_change":"2023-09-23T21:15:43.333919+0000","last_active":"2023-09-23T21:19:18.573716+0000","last_peered":"2023-09-23T21:19:18.573716+0000","last_clean":"2023-09-23T21:19:18.573716+0000","last_became_active":"2023-09-23T21:13:33.769303+0000","last_became_peered":"2023-09-23T21:13:33.769303+0000","last_unstale":"2023-09-23T21:19:18.573716+0000","last_undegraded":"2023-09-23T21:19:18.573716+0000","last_fullsized":"2023-09-23T21:19:18.573716+0000","mapping_epoch":504,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":505,"parent":"0.0","parent_split_bits":3,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:15:43.333733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:15:43.333733+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:42:29.787357+0000","scrub_duration":0.0092107530000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":89,"reported_epoch":643,"state":"active+clean","last_fresh":"2023-09-23T21:19:19.582605+0000","last_change":"2023-09-23T21:16:42.483813+0000","last_active":"2023-09-23T21:19:19.582605+0000","last_peered":"2023-09-23T21:19:19.582605+0000","last_clean":"2023-09-23T21:19:19.582605+0000","last_became_active":"2023-09-23T21:16:42.482847+0000","last_became_peered":"2023-09-23T21:16:42.482847+0000","last_unstale":"2023-09-23T21:19:19.582605+0000","last_undegraded":"2023-09-23T21:19:19.582605+0000","last_fullsized":"2023-09-23T21:19:19.582605+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:38:57.905679+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":529,"reported_epoch":643,"state":"active+clean","last_fresh":"2023-09-23T21:19:19.579834+0000","last_change":"2023-09-23T21:13:30.790590+0000","last_active":"2023-09-23T21:19:19.579834+0000","last_peered":"2023-09-23T21:19:19.579834+0000","last_clean":"2023-09-23T21:19:19.579834+0000","last_became_active":"2023-09-23T21:13:30.790253+0000","last_became_peered":"2023-09-23T21:13:30.790253+0000","last_unstale":"2023-09-23T21:19:19.579834+0000","last_undegraded":"2023-09-23T21:19:19.579834+0000","last_fullsized":"2023-09-23T21:19:19.579834+0000","mapping_epoch":501,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:23:41.843250+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,0],"acting":[2,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":608,"reported_epoch":642,"state":"active+clean","last_fresh":"2023-09-23T21:19:18.572176+0000","last_change":"2023-09-23T21:16:42.428535+0000","last_active":"2023-09-23T21:19:18.572176+0000","last_peered":"2023-09-23T21:19:18.572176+0000","last_clean":"2023-09-23T21:19:18.572176+0000","last_became_active":"2023-09-23T21:16:42.428149+0000","last_became_peered":"2023-09-23T21:16:42.428149+0000","last_unstale":"2023-09-23T21:19:18.572176+0000","last_undegraded":"2023-09-23T21:19:18.572176+0000","last_fullsized":"2023-09-23T21:19:18.572176+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:38:59.979788+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"12.8","version":"645'14","reported_seq":33,"reported_epoch":645,"state":"active+clean","last_fresh":"2023-09-23T21:19:22.986145+0000","last_change":"2023-09-23T21:19:19.590020+0000","last_active":"2023-09-23T21:19:22.986145+0000","last_peered":"2023-09-23T21:19:22.986145+0000","last_clean":"2023-09-23T21:19:22.986145+0000","last_became_active":"2023-09-23T21:19:19.589738+0000","last_became_peered":"2023-09-23T21:19:19.589738+0000","last_unstale":"2023-09-23T21:19:22.986145+0000","last_undegraded":"2023-09-23T21:19:22.986145+0000","last_fullsized":"2023-09-23T21:19:22.986145+0000","mapping_epoch":642,"log_start":"0'0","ondisk_log_start":"0'0","created":642,"last_epoch_clean":643,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:19:18.561631+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:19:18.561631+0000","last_clean_scrub_stamp":"2023-09-23T21:19:18.561631+0000","objects_scrubbed":0,"log_size":14,"log_dups_size":0,"ondisk_log_size":14,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:06:35.311480+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":5,"num_object_clones":0,"num_object_copies":15,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5,"num_whiteouts":0,"num_read":21,"num_read_kb":21,"num_write":10,"num_write_kb":9,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":5,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.5","version":"79'161","reported_seq":682,"reported_epoch":644,"state":"active+clean","last_fresh":"2023-09-23T21:19:20.590707+0000","last_change":"2023-09-23T21:15:41.108638+0000","last_active":"2023-09-23T21:19:20.590707+0000","last_peered":"2023-09-23T21:19:20.590707+0000","last_clean":"2023-09-23T21:19:20.590707+0000","last_became_active":"2023-09-23T21:13:38.373633+0000","last_became_peered":"2023-09-23T21:13:38.373633+0000","last_unstale":"2023-09-23T21:19:20.590707+0000","last_undegraded":"2023-09-23T21:19:20.590707+0000","last_fullsized":"2023-09-23T21:19:20.590707+0000","mapping_epoch":507,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":3,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:15:41.108579+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:15:41.108579+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:21:58.075649+0000","scrub_duration":0.0061511960000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":211,"reported_epoch":642,"state":"active+clean","last_fresh":"2023-09-23T21:19:18.573457+0000","last_change":"2023-09-23T21:17:02.204086+0000","last_active":"2023-09-23T21:19:18.573457+0000","last_peered":"2023-09-23T21:19:18.573457+0000","last_clean":"2023-09-23T21:19:18.573457+0000","last_became_active":"2023-09-23T21:16:42.517549+0000","last_became_peered":"2023-09-23T21:16:42.517549+0000","last_unstale":"2023-09-23T21:19:18.573457+0000","last_undegraded":"2023-09-23T21:19:18.573457+0000","last_fullsized":"2023-09-23T21:19:18.573457+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:43:07.991844+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":625,"reported_epoch":644,"state":"active+clean","last_fresh":"2023-09-23T21:19:20.591010+0000","last_change":"2023-09-23T21:13:30.807520+0000","last_active":"2023-09-23T21:19:20.591010+0000","last_peered":"2023-09-23T21:19:20.591010+0000","last_clean":"2023-09-23T21:19:20.591010+0000","last_became_active":"2023-09-23T21:13:30.807222+0000","last_became_peered":"2023-09-23T21:13:30.807222+0000","last_unstale":"2023-09-23T21:19:20.591010+0000","last_undegraded":"2023-09-23T21:19:20.591010+0000","last_fullsized":"2023-09-23T21:19:20.591010+0000","mapping_epoch":501,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:40:31.989749+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":507,"reported_epoch":645,"state":"active+clean","last_fresh":"2023-09-23T21:19:22.697551+0000","last_change":"2023-09-23T21:16:42.463411+0000","last_active":"2023-09-23T21:19:22.697551+0000","last_peered":"2023-09-23T21:19:22.697551+0000","last_clean":"2023-09-23T21:19:22.697551+0000","last_became_active":"2023-09-23T21:16:42.463020+0000","last_became_peered":"2023-09-23T21:16:42.463020+0000","last_unstale":"2023-09-23T21:19:22.697551+0000","last_undegraded":"2023-09-23T21:19:22.697551+0000","last_fullsized":"2023-09-23T21:19:22.697551+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:07:34.346129+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"12.9","version":"643'4","reported_seq":18,"reported_epoch":644,"state":"active+clean","last_fresh":"2023-09-23T21:19:20.624620+0000","last_change":"2023-09-23T21:19:19.590095+0000","last_active":"2023-09-23T21:19:20.624620+0000","last_peered":"2023-09-23T21:19:20.624620+0000","last_clean":"2023-09-23T21:19:20.624620+0000","last_became_active":"2023-09-23T21:19:19.589744+0000","last_became_peered":"2023-09-23T21:19:19.589744+0000","last_unstale":"2023-09-23T21:19:20.624620+0000","last_undegraded":"2023-09-23T21:19:20.624620+0000","last_fullsized":"2023-09-23T21:19:20.624620+0000","mapping_epoch":642,"log_start":"0'0","ondisk_log_start":"0'0","created":642,"last_epoch_clean":643,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:19:18.561631+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:19:18.561631+0000","last_clean_scrub_stamp":"2023-09-23T21:19:18.561631+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:33:01.656585+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":12,"num_read_kb":12,"num_write":5,"num_write_kb":5,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.4","version":"79'161","reported_seq":1053,"reported_epoch":645,"state":"active+clean","last_fresh":"2023-09-23T21:19:22.698276+0000","last_change":"2023-09-23T21:16:45.426136+0000","last_active":"2023-09-23T21:19:22.698276+0000","last_peered":"2023-09-23T21:19:22.698276+0000","last_clean":"2023-09-23T21:19:22.698276+0000","last_became_active":"2023-09-23T21:16:45.425669+0000","last_became_peered":"2023-09-23T21:16:45.425669+0000","last_unstale":"2023-09-23T21:19:22.698276+0000","last_undegraded":"2023-09-23T21:19:22.698276+0000","last_fullsized":"2023-09-23T21:19:22.698276+0000","mapping_epoch":567,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":568,"parent":"0.0","parent_split_bits":3,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:20.012764+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:20.012764+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:52:39.142509+0000","scrub_duration":0.0045251919999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":603,"reported_epoch":643,"state":"active+clean","last_fresh":"2023-09-23T21:19:19.586095+0000","last_change":"2023-09-23T21:16:42.459732+0000","last_active":"2023-09-23T21:19:19.586095+0000","last_peered":"2023-09-23T21:19:19.586095+0000","last_clean":"2023-09-23T21:19:19.586095+0000","last_became_active":"2023-09-23T21:16:42.459413+0000","last_became_peered":"2023-09-23T21:16:42.459413+0000","last_unstale":"2023-09-23T21:19:19.586095+0000","last_undegraded":"2023-09-23T21:19:19.586095+0000","last_fullsized":"2023-09-23T21:19:19.586095+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:37:44.548863+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,3],"acting":[2,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.6","version":"70'1","reported_seq":644,"reported_epoch":642,"state":"active+clean","last_fresh":"2023-09-23T21:19:18.573864+0000","last_change":"2023-09-23T21:17:02.208077+0000","last_active":"2023-09-23T21:19:18.573864+0000","last_peered":"2023-09-23T21:19:18.573864+0000","last_clean":"2023-09-23T21:19:18.573864+0000","last_became_active":"2023-09-23T21:16:42.431308+0000","last_became_peered":"2023-09-23T21:16:42.431308+0000","last_unstale":"2023-09-23T21:19:18.573864+0000","last_undegraded":"2023-09-23T21:19:18.573864+0000","last_fullsized":"2023-09-23T21:19:18.573864+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:16:49.009013+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":219,"reported_epoch":644,"state":"active+clean","last_fresh":"2023-09-23T21:19:20.590188+0000","last_change":"2023-09-23T21:13:12.216895+0000","last_active":"2023-09-23T21:19:20.590188+0000","last_peered":"2023-09-23T21:19:20.590188+0000","last_clean":"2023-09-23T21:19:20.590188+0000","last_became_active":"2023-09-23T21:13:12.216654+0000","last_became_peered":"2023-09-23T21:13:12.216654+0000","last_unstale":"2023-09-23T21:19:20.590188+0000","last_undegraded":"2023-09-23T21:19:20.590188+0000","last_fullsized":"2023-09-23T21:19:20.590188+0000","mapping_epoch":493,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:02:40.430096+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.14","version":"645'12","reported_seq":30,"reported_epoch":645,"state":"active+clean","last_fresh":"2023-09-23T21:19:23.223825+0000","last_change":"2023-09-23T21:19:19.582856+0000","last_active":"2023-09-23T21:19:23.223825+0000","last_peered":"2023-09-23T21:19:23.223825+0000","last_clean":"2023-09-23T21:19:23.223825+0000","last_became_active":"2023-09-23T21:19:19.582581+0000","last_became_peered":"2023-09-23T21:19:19.582581+0000","last_unstale":"2023-09-23T21:19:23.223825+0000","last_undegraded":"2023-09-23T21:19:23.223825+0000","last_fullsized":"2023-09-23T21:19:23.223825+0000","mapping_epoch":642,"log_start":"0'0","ondisk_log_start":"0'0","created":642,"last_epoch_clean":643,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:19:18.561631+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:19:18.561631+0000","last_clean_scrub_stamp":"2023-09-23T21:19:18.561631+0000","objects_scrubbed":0,"log_size":12,"log_dups_size":0,"ondisk_log_size":12,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:21:09.713120+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":3,"num_object_clones":0,"num_object_copies":9,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3,"num_whiteouts":0,"num_read":34,"num_read_kb":33,"num_write":11,"num_write_kb":8,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":3,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.19","version":"79'161","reported_seq":851,"reported_epoch":642,"state":"active+clean","last_fresh":"2023-09-23T21:19:18.572151+0000","last_change":"2023-09-23T21:16:45.428879+0000","last_active":"2023-09-23T21:19:18.572151+0000","last_peered":"2023-09-23T21:19:18.572151+0000","last_clean":"2023-09-23T21:19:18.572151+0000","last_became_active":"2023-09-23T21:16:45.428660+0000","last_became_peered":"2023-09-23T21:16:45.428660+0000","last_unstale":"2023-09-23T21:19:18.572151+0000","last_undegraded":"2023-09-23T21:19:18.572151+0000","last_fullsized":"2023-09-23T21:19:18.572151+0000","mapping_epoch":567,"log_start":"0'0","ondisk_log_start":"0'0","created":479,"last_epoch_clean":568,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:15:48.271791+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:15:48.271791+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:25:26.361045+0000","scrub_duration":0.0057149180000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":172,"reported_epoch":642,"state":"active+clean","last_fresh":"2023-09-23T21:19:18.572101+0000","last_change":"2023-09-23T21:16:42.434830+0000","last_active":"2023-09-23T21:19:18.572101+0000","last_peered":"2023-09-23T21:19:18.572101+0000","last_clean":"2023-09-23T21:19:18.572101+0000","last_became_active":"2023-09-23T21:16:42.430289+0000","last_became_peered":"2023-09-23T21:16:42.430289+0000","last_unstale":"2023-09-23T21:19:18.572101+0000","last_undegraded":"2023-09-23T21:19:18.572101+0000","last_fullsized":"2023-09-23T21:19:18.572101+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:38:48.053766+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":149,"reported_epoch":643,"state":"active+clean","last_fresh":"2023-09-23T21:19:19.579747+0000","last_change":"2023-09-23T21:13:38.373814+0000","last_active":"2023-09-23T21:19:19.579747+0000","last_peered":"2023-09-23T21:19:19.579747+0000","last_clean":"2023-09-23T21:19:19.579747+0000","last_became_active":"2023-09-23T21:13:38.373458+0000","last_became_peered":"2023-09-23T21:13:38.373458+0000","last_unstale":"2023-09-23T21:19:19.579747+0000","last_undegraded":"2023-09-23T21:19:19.579747+0000","last_fullsized":"2023-09-23T21:19:19.579747+0000","mapping_epoch":507,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:23:08.288401+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"12.13","version":"0'0","reported_seq":10,"reported_epoch":643,"state":"active+clean","last_fresh":"2023-09-23T21:19:19.583584+0000","last_change":"2023-09-23T21:19:19.583179+0000","last_active":"2023-09-23T21:19:19.583584+0000","last_peered":"2023-09-23T21:19:19.583584+0000","last_clean":"2023-09-23T21:19:19.583584+0000","last_became_active":"2023-09-23T21:19:19.582918+0000","last_became_peered":"2023-09-23T21:19:19.582918+0000","last_unstale":"2023-09-23T21:19:19.583584+0000","last_undegraded":"2023-09-23T21:19:19.583584+0000","last_fullsized":"2023-09-23T21:19:19.583584+0000","mapping_epoch":642,"log_start":"0'0","ondisk_log_start":"0'0","created":642,"last_epoch_clean":643,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:19:18.561631+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:19:18.561631+0000","last_clean_scrub_stamp":"2023-09-23T21:19:18.561631+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:15:34.640697+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,2],"acting":[0,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1e","version":"79'161","reported_seq":630,"reported_epoch":646,"state":"active+clean","last_fresh":"2023-09-23T21:19:23.951777+0000","last_change":"2023-09-23T21:16:45.420788+0000","last_active":"2023-09-23T21:19:23.951777+0000","last_peered":"2023-09-23T21:19:23.951777+0000","last_clean":"2023-09-23T21:19:23.951777+0000","last_became_active":"2023-09-23T21:16:45.420448+0000","last_became_peered":"2023-09-23T21:16:45.420448+0000","last_unstale":"2023-09-23T21:19:23.951777+0000","last_undegraded":"2023-09-23T21:19:23.951777+0000","last_fullsized":"2023-09-23T21:19:23.951777+0000","mapping_epoch":567,"log_start":"0'0","ondisk_log_start":"0'0","created":479,"last_epoch_clean":568,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:15:53.061319+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:15:53.061319+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:46:31.982439+0000","scrub_duration":0.007429878,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,4],"acting":[3,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":254,"reported_epoch":644,"state":"active+clean","last_fresh":"2023-09-23T21:19:20.590607+0000","last_change":"2023-09-23T21:16:42.518584+0000","last_active":"2023-09-23T21:19:20.590607+0000","last_peered":"2023-09-23T21:19:20.590607+0000","last_clean":"2023-09-23T21:19:20.590607+0000","last_became_active":"2023-09-23T21:16:42.416632+0000","last_became_peered":"2023-09-23T21:16:42.416632+0000","last_unstale":"2023-09-23T21:19:20.590607+0000","last_undegraded":"2023-09-23T21:19:20.590607+0000","last_fullsized":"2023-09-23T21:19:20.590607+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:02:18.972080+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,7],"acting":[4,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":209,"reported_epoch":644,"state":"active+clean","last_fresh":"2023-09-23T21:19:20.590760+0000","last_change":"2023-09-23T21:12:10.770474+0000","last_active":"2023-09-23T21:19:20.590760+0000","last_peered":"2023-09-23T21:19:20.590760+0000","last_clean":"2023-09-23T21:19:20.590760+0000","last_became_active":"2023-09-23T21:12:10.770184+0000","last_became_peered":"2023-09-23T21:12:10.770184+0000","last_unstale":"2023-09-23T21:19:20.590760+0000","last_undegraded":"2023-09-23T21:19:20.590760+0000","last_fullsized":"2023-09-23T21:19:20.590760+0000","mapping_epoch":470,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:05:09.407151+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.12","version":"643'1","reported_seq":15,"reported_epoch":645,"state":"active+clean","last_fresh":"2023-09-23T21:19:22.697796+0000","last_change":"2023-09-23T21:19:19.577398+0000","last_active":"2023-09-23T21:19:22.697796+0000","last_peered":"2023-09-23T21:19:22.697796+0000","last_clean":"2023-09-23T21:19:22.697796+0000","last_became_active":"2023-09-23T21:19:19.577168+0000","last_became_peered":"2023-09-23T21:19:19.577168+0000","last_unstale":"2023-09-23T21:19:22.697796+0000","last_undegraded":"2023-09-23T21:19:22.697796+0000","last_fullsized":"2023-09-23T21:19:22.697796+0000","mapping_epoch":642,"log_start":"0'0","ondisk_log_start":"0'0","created":642,"last_epoch_clean":643,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:19:18.561631+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:19:18.561631+0000","last_clean_scrub_stamp":"2023-09-23T21:19:18.561631+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:33:59.116585+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":2,"num_read_kb":2,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,2],"acting":[3,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":435,"reported_epoch":643,"state":"active+clean","last_fresh":"2023-09-23T21:19:19.585838+0000","last_change":"2023-09-23T21:12:10.771160+0000","last_active":"2023-09-23T21:19:19.585838+0000","last_peered":"2023-09-23T21:19:19.585838+0000","last_clean":"2023-09-23T21:19:19.585838+0000","last_became_active":"2023-09-23T21:12:10.757808+0000","last_became_peered":"2023-09-23T21:12:10.757808+0000","last_unstale":"2023-09-23T21:19:19.585838+0000","last_undegraded":"2023-09-23T21:19:19.585838+0000","last_fullsized":"2023-09-23T21:19:19.585838+0000","mapping_epoch":470,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:30:50.987139+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":219,"reported_epoch":644,"state":"active+clean","last_fresh":"2023-09-23T21:19:20.590513+0000","last_change":"2023-09-23T21:13:12.216602+0000","last_active":"2023-09-23T21:19:20.590513+0000","last_peered":"2023-09-23T21:19:20.590513+0000","last_clean":"2023-09-23T21:19:20.590513+0000","last_became_active":"2023-09-23T21:13:12.216204+0000","last_became_peered":"2023-09-23T21:13:12.216204+0000","last_unstale":"2023-09-23T21:19:20.590513+0000","last_undegraded":"2023-09-23T21:19:20.590513+0000","last_fullsized":"2023-09-23T21:19:20.590513+0000","mapping_epoch":493,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:23:37.043390+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.11","version":"643'5","reported_seq":17,"reported_epoch":643,"state":"active+clean","last_fresh":"2023-09-23T21:19:20.006039+0000","last_change":"2023-09-23T21:19:19.587283+0000","last_active":"2023-09-23T21:19:20.006039+0000","last_peered":"2023-09-23T21:19:20.006039+0000","last_clean":"2023-09-23T21:19:20.006039+0000","last_became_active":"2023-09-23T21:19:19.586975+0000","last_became_peered":"2023-09-23T21:19:19.586975+0000","last_unstale":"2023-09-23T21:19:20.006039+0000","last_undegraded":"2023-09-23T21:19:20.006039+0000","last_fullsized":"2023-09-23T21:19:20.006039+0000","mapping_epoch":642,"log_start":"0'0","ondisk_log_start":"0'0","created":642,"last_epoch_clean":643,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:19:18.561631+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:19:18.561631+0000","last_clean_scrub_stamp":"2023-09-23T21:19:18.561631+0000","objects_scrubbed":0,"log_size":5,"log_dups_size":0,"ondisk_log_size":5,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:29:56.843616+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":4,"num_read_kb":4,"num_write":3,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,3],"acting":[5,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.1c","version":"79'161","reported_seq":1076,"reported_epoch":645,"state":"active+clean","last_fresh":"2023-09-23T21:19:22.698213+0000","last_change":"2023-09-23T21:16:45.419779+0000","last_active":"2023-09-23T21:19:22.698213+0000","last_peered":"2023-09-23T21:19:22.698213+0000","last_clean":"2023-09-23T21:19:22.698213+0000","last_became_active":"2023-09-23T21:16:45.419394+0000","last_became_peered":"2023-09-23T21:16:45.419394+0000","last_unstale":"2023-09-23T21:19:22.698213+0000","last_undegraded":"2023-09-23T21:19:22.698213+0000","last_fullsized":"2023-09-23T21:19:22.698213+0000","mapping_epoch":567,"log_start":"0'0","ondisk_log_start":"0'0","created":479,"last_epoch_clean":568,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:15:54.019828+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:15:54.019828+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:59:05.422258+0000","scrub_duration":0.0020962530000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":253,"reported_epoch":643,"state":"active+clean","last_fresh":"2023-09-23T21:19:19.586013+0000","last_change":"2023-09-23T21:16:42.424187+0000","last_active":"2023-09-23T21:19:19.586013+0000","last_peered":"2023-09-23T21:19:19.586013+0000","last_clean":"2023-09-23T21:19:19.586013+0000","last_became_active":"2023-09-23T21:16:42.423805+0000","last_became_peered":"2023-09-23T21:16:42.423805+0000","last_unstale":"2023-09-23T21:19:19.586013+0000","last_undegraded":"2023-09-23T21:19:19.586013+0000","last_fullsized":"2023-09-23T21:19:19.586013+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:55:46.051199+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,7],"acting":[2,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":313,"reported_epoch":643,"state":"active+clean","last_fresh":"2023-09-23T21:19:19.578904+0000","last_change":"2023-09-23T21:13:12.200767+0000","last_active":"2023-09-23T21:19:19.578904+0000","last_peered":"2023-09-23T21:19:19.578904+0000","last_clean":"2023-09-23T21:19:19.578904+0000","last_became_active":"2023-09-23T21:13:12.200435+0000","last_became_peered":"2023-09-23T21:13:12.200435+0000","last_unstale":"2023-09-23T21:19:19.578904+0000","last_undegraded":"2023-09-23T21:19:19.578904+0000","last_fullsized":"2023-09-23T21:19:19.578904+0000","mapping_epoch":493,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:15:44.937886+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"12.10","version":"643'4","reported_seq":14,"reported_epoch":643,"state":"active+clean","last_fresh":"2023-09-23T21:19:20.003874+0000","last_change":"2023-09-23T21:19:19.582571+0000","last_active":"2023-09-23T21:19:20.003874+0000","last_peered":"2023-09-23T21:19:20.003874+0000","last_clean":"2023-09-23T21:19:20.003874+0000","last_became_active":"2023-09-23T21:19:19.582245+0000","last_became_peered":"2023-09-23T21:19:19.582245+0000","last_unstale":"2023-09-23T21:19:20.003874+0000","last_undegraded":"2023-09-23T21:19:20.003874+0000","last_fullsized":"2023-09-23T21:19:20.003874+0000","mapping_epoch":642,"log_start":"0'0","ondisk_log_start":"0'0","created":642,"last_epoch_clean":643,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:19:18.561631+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:19:18.561631+0000","last_clean_scrub_stamp":"2023-09-23T21:19:18.561631+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:31:56.882635+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":156,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":5,"num_read_kb":5,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,5],"acting":[0,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1d","version":"79'161","reported_seq":791,"reported_epoch":645,"state":"active+clean","last_fresh":"2023-09-23T21:19:22.698426+0000","last_change":"2023-09-23T21:19:10.941440+0000","last_active":"2023-09-23T21:19:22.698426+0000","last_peered":"2023-09-23T21:19:22.698426+0000","last_clean":"2023-09-23T21:19:22.698426+0000","last_became_active":"2023-09-23T21:15:23.801704+0000","last_became_peered":"2023-09-23T21:15:23.801704+0000","last_unstale":"2023-09-23T21:19:22.698426+0000","last_undegraded":"2023-09-23T21:19:22.698426+0000","last_fullsized":"2023-09-23T21:19:22.698426+0000","mapping_epoch":542,"log_start":"0'0","ondisk_log_start":"0'0","created":479,"last_epoch_clean":543,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:19:10.941299+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:19:10.941299+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:31:22.166514+0000","scrub_duration":0.0018618599999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,2],"acting":[3,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":171,"reported_epoch":642,"state":"active+clean","last_fresh":"2023-09-23T21:19:18.573174+0000","last_change":"2023-09-23T21:13:38.380574+0000","last_active":"2023-09-23T21:19:18.573174+0000","last_peered":"2023-09-23T21:19:18.573174+0000","last_clean":"2023-09-23T21:19:18.573174+0000","last_became_active":"2023-09-23T21:13:38.373252+0000","last_became_peered":"2023-09-23T21:13:38.373252+0000","last_unstale":"2023-09-23T21:19:18.573174+0000","last_undegraded":"2023-09-23T21:19:18.573174+0000","last_fullsized":"2023-09-23T21:19:18.573174+0000","mapping_epoch":507,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:58:14.788031+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":11,"num_bytes_recovered":3324,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":219,"reported_epoch":644,"state":"active+clean","last_fresh":"2023-09-23T21:19:20.590105+0000","last_change":"2023-09-23T21:12:10.793610+0000","last_active":"2023-09-23T21:19:20.590105+0000","last_peered":"2023-09-23T21:19:20.590105+0000","last_clean":"2023-09-23T21:19:20.590105+0000","last_became_active":"2023-09-23T21:12:10.793081+0000","last_became_peered":"2023-09-23T21:12:10.793081+0000","last_unstale":"2023-09-23T21:19:20.590105+0000","last_undegraded":"2023-09-23T21:19:20.590105+0000","last_fullsized":"2023-09-23T21:19:20.590105+0000","mapping_epoch":470,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:34:36.291550+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":12,"num_pg":32,"stat_sum":{"num_bytes":374,"num_objects":37,"num_object_clones":0,"num_object_copies":111,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":37,"num_whiteouts":0,"num_read":1758,"num_read_kb":9847,"num_write":845,"num_write_kb":4899,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":29,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":90112,"data_stored":1156,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":696,"ondisk_log_size":696,"up":96,"acting":96,"num_store_stats":6},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":80,"num_bytes_recovered":18686,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":118784,"data_stored":21350,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":14322,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":31,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":525376,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2109440,"data_stored":2098784,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4991,"ondisk_log_size":4991,"up":93,"acting":93,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201747,"num_pgs":55,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":653224,"kb_used_data":6928,"kb_used_omap":0,"kb_used_meta":646272,"kb_avail":93091928,"statfs":{"total":95995035648,"available":95326134272,"internally_reserved":0,"allocated":7094272,"data_stored":5340107,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":661782528},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:18:22 2023","interfaces":[{"interface":"back","average":{"1min":1.5209999999999999,"5min":1.474,"15min":2.1080000000000001},"min":{"1min":0.22900000000000001,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":3.968,"5min":37.207999999999998,"15min":37.207999999999998},"last":0.46700000000000003},{"interface":"front","average":{"1min":1.73,"5min":1.748,"15min":2.0720000000000001},"min":{"1min":0.318,"5min":0.253,"15min":0.253},"max":{"1min":5.2910000000000004,"5min":37.122,"15min":37.122},"last":0.44}]},{"osd":1,"last update":"Sat Sep 23 21:19:18 2023","interfaces":[{"interface":"back","average":{"1min":0.80200000000000005,"5min":1.1930000000000001,"15min":1.704},"min":{"1min":0.29399999999999998,"5min":0.27600000000000002,"15min":0.26700000000000002},"max":{"1min":5.2619999999999996,"5min":6.9130000000000003,"15min":14.034000000000001},"last":0.45400000000000001},{"interface":"front","average":{"1min":0.84199999999999997,"5min":1.3380000000000001,"15min":1.6930000000000001},"min":{"1min":0.36599999999999999,"5min":0.29999999999999999,"15min":0.26300000000000001},"max":{"1min":2.512,"5min":11.002000000000001,"15min":11.226000000000001},"last":0.57099999999999995}]},{"osd":2,"last update":"Sat Sep 23 21:18:38 2023","interfaces":[{"interface":"back","average":{"1min":2.105,"5min":2.0739999999999998,"15min":1.8069999999999999},"min":{"1min":0.27200000000000002,"5min":0.27200000000000002,"15min":0.26200000000000001},"max":{"1min":18.488,"5min":25.471,"15min":25.471},"last":0.496},{"interface":"front","average":{"1min":1.9330000000000001,"5min":1.857,"15min":1.752},"min":{"1min":0.28799999999999998,"5min":0.28799999999999998,"15min":0.26700000000000002},"max":{"1min":17.536000000000001,"5min":27.187999999999999,"15min":27.187999999999999},"last":0.41899999999999998}]},{"osd":3,"last update":"Sat Sep 23 21:19:06 2023","interfaces":[{"interface":"back","average":{"1min":1.6220000000000001,"5min":1.6040000000000001,"15min":1.788},"min":{"1min":0.317,"5min":0.30599999999999999,"15min":0.30099999999999999},"max":{"1min":5.7830000000000004,"5min":8.6259999999999994,"15min":12.359999999999999},"last":0.38200000000000001},{"interface":"front","average":{"1min":1.504,"5min":1.8100000000000001,"15min":1.79},"min":{"1min":0.33400000000000002,"5min":0.28799999999999998,"15min":0.28699999999999998},"max":{"1min":4.6769999999999996,"5min":15.82,"15min":15.82},"last":0.55500000000000005}]},{"osd":4,"last update":"Sat Sep 23 21:19:06 2023","interfaces":[{"interface":"back","average":{"1min":0.42199999999999999,"5min":0.78500000000000003,"15min":1.1160000000000001},"min":{"1min":0.27000000000000002,"5min":0.251,"15min":0.248},"max":{"1min":0.92700000000000005,"5min":4.2130000000000001,"15min":11.297000000000001},"last":0.54500000000000004},{"interface":"front","average":{"1min":0.61299999999999999,"5min":0.91800000000000004,"15min":1.0800000000000001},"min":{"1min":0.26800000000000002,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":3.73,"5min":7.1870000000000003,"15min":12.752000000000001},"last":0.48399999999999999}]},{"osd":5,"last update":"Sat Sep 23 21:19:06 2023","interfaces":[{"interface":"back","average":{"1min":0.57199999999999995,"5min":0.91600000000000004,"15min":1.0640000000000001},"min":{"1min":0.28199999999999997,"5min":0.28199999999999997,"15min":0.26000000000000001},"max":{"1min":1.7150000000000001,"5min":6.9740000000000002,"15min":9.4789999999999992},"last":0.53300000000000003},{"interface":"front","average":{"1min":0.65900000000000003,"5min":1.004,"15min":1.157},"min":{"1min":0.32300000000000001,"5min":0.32000000000000001,"15min":0.26200000000000001},"max":{"1min":3.1349999999999998,"5min":6.7439999999999998,"15min":11.257},"last":0.52200000000000002}]},{"osd":6,"last update":"Sat Sep 23 21:19:18 2023","interfaces":[{"interface":"back","average":{"1min":0.72299999999999998,"5min":0.86899999999999999,"15min":1.0269999999999999},"min":{"1min":0.312,"5min":0.312,"15min":0.30099999999999999},"max":{"1min":3.8170000000000002,"5min":7.5339999999999998,"15min":10.941000000000001},"last":0.51000000000000001},{"interface":"front","average":{"1min":0.80800000000000005,"5min":1.006,"15min":1.145},"min":{"1min":0.35099999999999998,"5min":0.35099999999999998,"15min":0.33100000000000002},"max":{"1min":3.7559999999999998,"5min":5.6760000000000002,"15min":12.715},"last":0.58999999999999997}]}]},{"osd":6,"up_from":0,"seq":704374636831,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":1,"up_from":0,"seq":1503238553763,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":0,"up_from":493,"seq":2117418877003,"num_pgs":56,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":403356,"kb_used_data":8276,"kb_used_omap":36,"kb_used_meta":395035,"kb_avail":93341796,"statfs":{"total":95995035648,"available":95581999104,"internally_reserved":0,"allocated":8474624,"data_stored":5554763,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":37336,"internal_metadata":404516392},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 21:18:27 2023","interfaces":[{"interface":"back","average":{"1min":1.3620000000000001,"5min":1.4410000000000001,"15min":0.98399999999999999},"min":{"1min":0.25900000000000001,"5min":0.214,"15min":0.214},"max":{"1min":6.7779999999999996,"5min":13.779999999999999,"15min":13.779999999999999},"last":0.438},{"interface":"front","average":{"1min":2.5499999999999998,"5min":1.7569999999999999,"15min":1.099},"min":{"1min":0.35099999999999998,"5min":0.20799999999999999,"15min":0.20799999999999999},"max":{"1min":10.551,"5min":11.964,"15min":11.964},"last":0.55000000000000004}]},{"osd":2,"last update":"Sat Sep 23 21:18:27 2023","interfaces":[{"interface":"back","average":{"1min":2.0579999999999998,"5min":1.8,"15min":1.036},"min":{"1min":0.36599999999999999,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":7.3449999999999998,"5min":13.637,"15min":13.637},"last":0.64600000000000002},{"interface":"front","average":{"1min":3.258,"5min":2.46,"15min":1.4650000000000001},"min":{"1min":0.30199999999999999,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":10.401999999999999,"5min":26.571000000000002,"15min":26.571000000000002},"last":1.3480000000000001}]},{"osd":3,"last update":"Sat Sep 23 21:18:27 2023","interfaces":[{"interface":"back","average":{"1min":3.9950000000000001,"5min":2.6000000000000001,"15min":1.4450000000000001},"min":{"1min":0.253,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":37.079999999999998,"5min":37.079999999999998,"15min":37.079999999999998},"last":1.304},{"interface":"front","average":{"1min":3.4550000000000001,"5min":2.2029999999999998,"15min":1.4790000000000001},"min":{"1min":0.34699999999999998,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":32.935000000000002,"5min":32.935000000000002,"15min":32.935000000000002},"last":1.0489999999999999}]},{"osd":4,"last update":"Sat Sep 23 21:18:27 2023","interfaces":[{"interface":"back","average":{"1min":1.546,"5min":1.7390000000000001,"15min":1.3080000000000001},"min":{"1min":0.34499999999999997,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":4.7880000000000003,"5min":22.780000000000001,"15min":22.780000000000001},"last":1.1399999999999999},{"interface":"front","average":{"1min":1.9390000000000001,"5min":1.4990000000000001,"15min":1.163},"min":{"1min":0.32500000000000001,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":6.9119999999999999,"5min":22.652000000000001,"15min":22.652000000000001},"last":1.2390000000000001}]},{"osd":5,"last update":"Sat Sep 23 21:18:27 2023","interfaces":[{"interface":"back","average":{"1min":1.5549999999999999,"5min":1.5680000000000001,"15min":1.3460000000000001},"min":{"1min":0.312,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":6.1139999999999999,"5min":8.9329999999999998,"15min":8.9329999999999998},"last":1.04},{"interface":"front","average":{"1min":1.9039999999999999,"5min":1.696,"15min":1.274},"min":{"1min":0.40100000000000002,"5min":0.313,"15min":0.313},"max":{"1min":6.5990000000000002,"5min":15.321999999999999,"15min":15.321999999999999},"last":1.214}]},{"osd":6,"last update":"Sat Sep 23 21:18:27 2023","interfaces":[{"interface":"back","average":{"1min":1.5720000000000001,"5min":1.4810000000000001,"15min":1.2450000000000001},"min":{"1min":0.372,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":6.9409999999999998,"5min":7.2050000000000001,"15min":7.2050000000000001},"last":1.268},{"interface":"front","average":{"1min":1.647,"5min":1.595,"15min":1.403},"min":{"1min":0.38400000000000001,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":6.266,"5min":8.8889999999999993,"15min":8.8889999999999993},"last":0.99299999999999999}]},{"osd":7,"last update":"Sat Sep 23 21:18:27 2023","interfaces":[{"interface":"back","average":{"1min":1.595,"5min":1.8240000000000001,"15min":1.4139999999999999},"min":{"1min":0.34699999999999998,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":4.8360000000000003,"5min":15.273999999999999,"15min":15.273999999999999},"last":1.171},{"interface":"front","average":{"1min":1.75,"5min":1.7709999999999999,"15min":1.3620000000000001},"min":{"1min":0.36199999999999999,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":6.9109999999999996,"5min":33.779000000000003,"15min":33.779000000000003},"last":1.1890000000000001}]}]},{"osd":2,"up_from":93,"seq":399431958875,"num_pgs":56,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1040040,"kb_used_data":8288,"kb_used_omap":81,"kb_used_meta":1031662,"kb_avail":92705112,"statfs":{"total":95995035648,"available":94930034688,"internally_reserved":0,"allocated":8486912,"data_stored":5500315,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":83750,"internal_metadata":1056422106},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:18:23 2023","interfaces":[{"interface":"back","average":{"1min":2.6339999999999999,"5min":1.9370000000000001,"15min":1.2010000000000001},"min":{"1min":0.221,"5min":0.187,"15min":0.187},"max":{"1min":9.9499999999999993,"5min":13.25,"15min":13.25},"last":0.26600000000000001},{"interface":"front","average":{"1min":2.2509999999999999,"5min":2.089,"15min":1.806},"min":{"1min":0.26800000000000002,"5min":0.20200000000000001,"15min":0.20200000000000001},"max":{"1min":9.8849999999999998,"5min":18.111999999999998,"15min":18.111999999999998},"last":0.32300000000000001}]},{"osd":1,"last update":"Sat Sep 23 21:18:23 2023","interfaces":[{"interface":"back","average":{"1min":1.645,"5min":1.756,"15min":2.2629999999999999},"min":{"1min":0.34000000000000002,"5min":0.20599999999999999,"15min":0.188},"max":{"1min":10.535,"5min":10.535,"15min":34.396999999999998},"last":0.39700000000000002},{"interface":"front","average":{"1min":2.2240000000000002,"5min":1.95,"15min":2.3460000000000001},"min":{"1min":0.24099999999999999,"5min":0.24099999999999999,"15min":0.217},"max":{"1min":6.0990000000000002,"5min":18.081,"15min":34.462000000000003},"last":0.48199999999999998}]},{"osd":3,"last update":"Sat Sep 23 21:19:02 2023","interfaces":[{"interface":"back","average":{"1min":2.2949999999999999,"5min":2.3999999999999999,"15min":2.149},"min":{"1min":0.27300000000000002,"5min":0.248,"15min":0.20599999999999999},"max":{"1min":8.141,"5min":20.106999999999999,"15min":35.549999999999997},"last":0.30299999999999999},{"interface":"front","average":{"1min":2.593,"5min":2.9249999999999998,"15min":2.4609999999999999},"min":{"1min":0.34899999999999998,"5min":0.28599999999999998,"15min":0.21299999999999999},"max":{"1min":8.2469999999999999,"5min":20.239000000000001,"15min":36.951999999999998},"last":0.44400000000000001}]},{"osd":4,"last update":"Sat Sep 23 21:18:23 2023","interfaces":[{"interface":"back","average":{"1min":1.2749999999999999,"5min":1.3460000000000001,"15min":1.4450000000000001},"min":{"1min":0.27000000000000002,"5min":0.27000000000000002,"15min":0.249},"max":{"1min":3.8530000000000002,"5min":7.9770000000000003,"15min":10.266999999999999},"last":0.38300000000000001},{"interface":"front","average":{"1min":1.363,"5min":1.5640000000000001,"15min":1.7110000000000001},"min":{"1min":0.38900000000000001,"5min":0.30299999999999999,"15min":0.26700000000000002},"max":{"1min":4.2210000000000001,"5min":10.459,"15min":12.49},"last":0.51100000000000001}]},{"osd":5,"last update":"Sat Sep 23 21:18:23 2023","interfaces":[{"interface":"back","average":{"1min":1.6160000000000001,"5min":1.456,"15min":1.5349999999999999},"min":{"1min":0.32300000000000001,"5min":0.29099999999999998,"15min":0.28000000000000003},"max":{"1min":5.4139999999999997,"5min":7.6180000000000003,"15min":9.5039999999999996},"last":0.42299999999999999},{"interface":"front","average":{"1min":1.474,"5min":1.5169999999999999,"15min":1.6890000000000001},"min":{"1min":0.34899999999999998,"5min":0.33800000000000002,"15min":0.29399999999999998},"max":{"1min":4.2649999999999997,"5min":10.425000000000001,"15min":12.57},"last":0.59299999999999997}]},{"osd":6,"last update":"Sat Sep 23 21:18:23 2023","interfaces":[{"interface":"back","average":{"1min":1.542,"5min":1.633,"15min":1.8979999999999999},"min":{"1min":0.54100000000000004,"5min":0.377,"15min":0.35699999999999998},"max":{"1min":4.2990000000000004,"5min":10.335000000000001,"15min":12.518000000000001},"last":0.55400000000000005},{"interface":"front","average":{"1min":1.7190000000000001,"5min":1.696,"15min":2.0510000000000002},"min":{"1min":0.501,"5min":0.377,"15min":0.35599999999999998},"max":{"1min":4.7969999999999997,"5min":10.381,"15min":12.544},"last":0.64100000000000001}]},{"osd":7,"last update":"Sat Sep 23 21:18:23 2023","interfaces":[{"interface":"back","average":{"1min":1.6339999999999999,"5min":1.637,"15min":1.6519999999999999},"min":{"1min":0.31,"5min":0.31,"15min":0.29799999999999999},"max":{"1min":5.7480000000000002,"5min":18.021999999999998,"15min":18.021999999999998},"last":0.53500000000000003},{"interface":"front","average":{"1min":2.145,"5min":1.6339999999999999,"15min":1.73},"min":{"1min":0.35999999999999999,"5min":0.34999999999999998,"15min":0.32200000000000001},"max":{"1min":6.694,"5min":9.6349999999999998,"15min":10.92},"last":0.46899999999999997}]}]},{"osd":3,"up_from":98,"seq":420906795349,"num_pgs":81,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1148584,"kb_used_data":18784,"kb_used_omap":7,"kb_used_meta":1129784,"kb_avail":92596568,"statfs":{"total":95995035648,"available":94818885632,"internally_reserved":0,"allocated":19234816,"data_stored":16311016,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7576,"internal_metadata":1156899432},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:19:18 2023","interfaces":[{"interface":"back","average":{"1min":1.3009999999999999,"5min":4.2670000000000003,"15min":1.647},"min":{"1min":0.20399999999999999,"5min":0.18099999999999999,"15min":0.18099999999999999},"max":{"1min":9.3819999999999997,"5min":147.38900000000001,"15min":147.38900000000001},"last":0.34999999999999998},{"interface":"front","average":{"1min":0.93899999999999995,"5min":3.6400000000000001,"15min":1.6579999999999999},"min":{"1min":0.214,"5min":0.214,"15min":0.214},"max":{"1min":5.2110000000000003,"5min":149.989,"15min":149.989},"last":0.29799999999999999}]},{"osd":1,"last update":"Sat Sep 23 21:19:18 2023","interfaces":[{"interface":"back","average":{"1min":1.4710000000000001,"5min":1.6779999999999999,"15min":2.0720000000000001},"min":{"1min":0.248,"5min":0.23699999999999999,"15min":0.20000000000000001},"max":{"1min":11.146000000000001,"5min":11.146000000000001,"15min":22.577999999999999},"last":0.23400000000000001},{"interface":"front","average":{"1min":0.79100000000000004,"5min":1.335,"15min":1.8879999999999999},"min":{"1min":0.28899999999999998,"5min":0.255,"15min":0.216},"max":{"1min":2.7799999999999998,"5min":7.6829999999999998,"15min":21.617999999999999},"last":0.497}]},{"osd":2,"last update":"Sat Sep 23 21:19:03 2023","interfaces":[{"interface":"back","average":{"1min":0.84999999999999998,"5min":1.395,"15min":1.893},"min":{"1min":0.23799999999999999,"5min":0.23799999999999999,"15min":0.20899999999999999},"max":{"1min":3.1000000000000001,"5min":11.321,"15min":56.317999999999998},"last":1.135},{"interface":"front","average":{"1min":3.1120000000000001,"5min":2.331,"15min":2.343},"min":{"1min":0.34899999999999998,"5min":0.23400000000000001,"15min":0.216},"max":{"1min":11.177,"5min":17.423999999999999,"15min":55.988},"last":1.1080000000000001}]},{"osd":4,"last update":"Sat Sep 23 21:19:03 2023","interfaces":[{"interface":"back","average":{"1min":1.0609999999999999,"5min":1.27,"15min":1.472},"min":{"1min":0.36899999999999999,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":5.1589999999999998,"5min":10.807,"15min":21.478999999999999},"last":0.58999999999999997},{"interface":"front","average":{"1min":2.145,"5min":1.4870000000000001,"15min":1.7549999999999999},"min":{"1min":0.39600000000000002,"5min":0.311,"15min":0.28499999999999998},"max":{"1min":11.202,"5min":11.202,"15min":22.402000000000001},"last":0.32300000000000001}]},{"osd":5,"last update":"Sat Sep 23 21:19:03 2023","interfaces":[{"interface":"back","average":{"1min":0.91400000000000003,"5min":1.3109999999999999,"15min":1.6040000000000001},"min":{"1min":0.34999999999999998,"5min":0.313,"15min":0.311},"max":{"1min":3.109,"5min":10.882999999999999,"15min":21.562000000000001},"last":0.63},{"interface":"front","average":{"1min":0.90300000000000002,"5min":1.2470000000000001,"15min":1.679},"min":{"1min":0.30199999999999999,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":2.8010000000000002,"5min":5.7300000000000004,"15min":14.029},"last":0.42499999999999999}]},{"osd":6,"last update":"Sat Sep 23 21:19:18 2023","interfaces":[{"interface":"back","average":{"1min":1.605,"5min":1.534,"15min":1.794},"min":{"1min":0.44500000000000001,"5min":0.35899999999999999,"15min":0.34599999999999997},"max":{"1min":11.092000000000001,"5min":11.092000000000001,"15min":22.446999999999999},"last":0.38100000000000001},{"interface":"front","average":{"1min":0.80000000000000004,"5min":1.2589999999999999,"15min":1.7350000000000001},"min":{"1min":0.34300000000000003,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":2.839,"5min":7.5949999999999998,"15min":12.375999999999999},"last":0.46000000000000002}]},{"osd":7,"last update":"Sat Sep 23 21:19:03 2023","interfaces":[{"interface":"back","average":{"1min":0.91100000000000003,"5min":1.456,"15min":1.74},"min":{"1min":0.41099999999999998,"5min":0.41099999999999998,"15min":0.33900000000000002},"max":{"1min":2.4279999999999999,"5min":10.914999999999999,"15min":21.518999999999998},"last":0.54300000000000004},{"interface":"front","average":{"1min":1.2370000000000001,"5min":1.589,"15min":1.9059999999999999},"min":{"1min":0.38800000000000001,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":3.8130000000000002,"5min":8.2590000000000003,"15min":12.624000000000001},"last":0.56699999999999995}]}]},{"osd":4,"up_from":31,"seq":133143986655,"num_pgs":69,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":589544,"kb_used_data":6992,"kb_used_omap":0,"kb_used_meta":582528,"kb_avail":93155608,"statfs":{"total":95995035648,"available":95391342592,"internally_reserved":0,"allocated":7159808,"data_stored":5413262,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":596508672},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:18:24 2023","interfaces":[{"interface":"back","average":{"1min":2.2690000000000001,"5min":1.673,"15min":1.157},"min":{"1min":0.245,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":20.515000000000001,"5min":28.683,"15min":28.683},"last":0.252},{"interface":"front","average":{"1min":3.085,"5min":1.8009999999999999,"15min":1.3560000000000001},"min":{"1min":0.249,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":20.792000000000002,"5min":29.155000000000001,"15min":29.155000000000001},"last":0.35799999999999998}]},{"osd":1,"last update":"Sat Sep 23 21:18:24 2023","interfaces":[{"interface":"back","average":{"1min":1.7330000000000001,"5min":1.2829999999999999,"15min":2.1070000000000002},"min":{"1min":0.28499999999999998,"5min":0.26900000000000002,"15min":0.221},"max":{"1min":8.1020000000000003,"5min":8.1020000000000003,"15min":37.960000000000001},"last":0.31},{"interface":"front","average":{"1min":1.55,"5min":1.399,"15min":2.0550000000000002},"min":{"1min":0.33100000000000002,"5min":0.28499999999999998,"15min":0.27000000000000002},"max":{"1min":8.1509999999999998,"5min":13.983000000000001,"15min":37.933},"last":0.497}]},{"osd":2,"last update":"Sat Sep 23 21:18:24 2023","interfaces":[{"interface":"back","average":{"1min":2.177,"5min":1.8500000000000001,"15min":1.754},"min":{"1min":0.28399999999999997,"5min":0.26400000000000001,"15min":0.247},"max":{"1min":9.8399999999999999,"5min":17.602,"15min":17.602},"last":0.42899999999999999},{"interface":"front","average":{"1min":1.8839999999999999,"5min":1.7849999999999999,"15min":1.831},"min":{"1min":0.35399999999999998,"5min":0.311,"15min":0.27800000000000002},"max":{"1min":5.5250000000000004,"5min":14.339,"15min":14.417999999999999},"last":0.54400000000000004}]},{"osd":3,"last update":"Sat Sep 23 21:18:55 2023","interfaces":[{"interface":"back","average":{"1min":1.3129999999999999,"5min":2.1579999999999999,"15min":1.9790000000000001},"min":{"1min":0.29499999999999998,"5min":0.27900000000000003,"15min":0.25600000000000001},"max":{"1min":6.1230000000000002,"5min":67.105999999999995,"15min":67.105999999999995},"last":0.48599999999999999},{"interface":"front","average":{"1min":1.5289999999999999,"5min":2.2719999999999998,"15min":1.966},"min":{"1min":0.40400000000000003,"5min":0.32200000000000001,"15min":0.30199999999999999},"max":{"1min":6.1319999999999997,"5min":67.046999999999997,"15min":67.046999999999997},"last":0.58099999999999996}]},{"osd":5,"last update":"Sat Sep 23 21:19:17 2023","interfaces":[{"interface":"back","average":{"1min":0.39400000000000002,"5min":0.95299999999999996,"15min":1.0700000000000001},"min":{"1min":0.26900000000000002,"5min":0.254,"15min":0.222},"max":{"1min":0.73999999999999999,"5min":14.398999999999999,"15min":14.398999999999999},"last":0.378},{"interface":"front","average":{"1min":0.41699999999999998,"5min":0.86699999999999999,"15min":1.0589999999999999},"min":{"1min":0.29699999999999999,"5min":0.25800000000000001,"15min":0.255},"max":{"1min":0.57699999999999996,"5min":8.016,"15min":9.8979999999999997},"last":0.40999999999999998}]},{"osd":6,"last update":"Sat Sep 23 21:18:24 2023","interfaces":[{"interface":"back","average":{"1min":1.077,"5min":0.96599999999999997,"15min":1.2649999999999999},"min":{"1min":0.308,"5min":0.27200000000000002,"15min":0.26800000000000002},"max":{"1min":3.2050000000000001,"5min":8.7420000000000009,"15min":24.452999999999999},"last":0.39800000000000002},{"interface":"front","average":{"1min":0.83799999999999997,"5min":1.1499999999999999,"15min":1.0820000000000001},"min":{"1min":0.38,"5min":0.36399999999999999,"15min":0.32500000000000001},"max":{"1min":2.5409999999999999,"5min":14.374000000000001,"15min":14.387},"last":0.56399999999999995}]},{"osd":7,"last update":"Sat Sep 23 21:18:55 2023","interfaces":[{"interface":"back","average":{"1min":0.70599999999999996,"5min":1.077,"15min":1.1659999999999999},"min":{"1min":0.33800000000000002,"5min":0.32000000000000001,"15min":0.30099999999999999},"max":{"1min":2.6419999999999999,"5min":14.422000000000001,"15min":16.890999999999998},"last":0.52600000000000002},{"interface":"front","average":{"1min":0.70999999999999996,"5min":0.89800000000000002,"15min":1.179},"min":{"1min":0.32900000000000001,"5min":0.29199999999999998,"15min":0.27400000000000002},"max":{"1min":2.6579999999999999,"5min":7.7229999999999999,"15min":11.170999999999999},"last":0.44800000000000001}]}]},{"osd":5,"up_from":39,"seq":167503725021,"num_pgs":59,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":629188,"kb_used_data":6908,"kb_used_omap":0,"kb_used_meta":622272,"kb_avail":93115964,"statfs":{"total":95995035648,"available":95350747136,"internally_reserved":0,"allocated":7073792,"data_stored":5336040,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":637206528},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:19:20 2023","interfaces":[{"interface":"back","average":{"1min":4.3369999999999997,"5min":1.958,"15min":1.129},"min":{"1min":0.23599999999999999,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":88.272999999999996,"5min":88.272999999999996,"15min":88.272999999999996},"last":0.40300000000000002},{"interface":"front","average":{"1min":4.242,"5min":2.262,"15min":1.2},"min":{"1min":0.247,"5min":0.215,"15min":0.215},"max":{"1min":88.296999999999997,"5min":88.296999999999997,"15min":88.296999999999997},"last":0.42799999999999999}]},{"osd":1,"last update":"Sat Sep 23 21:19:20 2023","interfaces":[{"interface":"back","average":{"1min":1.0940000000000001,"5min":1.3420000000000001,"15min":1.579},"min":{"1min":0.27900000000000003,"5min":0.27900000000000003,"15min":0.23499999999999999},"max":{"1min":6.4160000000000004,"5min":7.5839999999999996,"15min":25.239999999999998},"last":0.45300000000000001},{"interface":"front","average":{"1min":1.056,"5min":1.222,"15min":1.6739999999999999},"min":{"1min":0.32900000000000001,"5min":0.28299999999999997,"15min":0.252},"max":{"1min":6.3579999999999997,"5min":6.3579999999999997,"15min":25.253},"last":0.53200000000000003}]},{"osd":2,"last update":"Sat Sep 23 21:18:35 2023","interfaces":[{"interface":"back","average":{"1min":1.895,"5min":1.5109999999999999,"15min":1.6419999999999999},"min":{"1min":0.27600000000000002,"5min":0.27300000000000002,"15min":0.25800000000000001},"max":{"1min":9.2479999999999993,"5min":9.6560000000000006,"15min":12.634},"last":0.51700000000000002},{"interface":"front","average":{"1min":2.0840000000000001,"5min":1.6000000000000001,"15min":1.663},"min":{"1min":0.29099999999999998,"5min":0.29099999999999998,"15min":0.25700000000000001},"max":{"1min":9.4060000000000006,"5min":9.4060000000000006,"15min":9.7170000000000005},"last":0.59899999999999998}]},{"osd":3,"last update":"Sat Sep 23 21:18:54 2023","interfaces":[{"interface":"back","average":{"1min":1.6870000000000001,"5min":1.5089999999999999,"15min":1.883},"min":{"1min":0.32000000000000001,"5min":0.314,"15min":0.29299999999999998},"max":{"1min":7.3700000000000001,"5min":7.3700000000000001,"15min":25.503},"last":0.69199999999999995},{"interface":"front","average":{"1min":1.6100000000000001,"5min":1.782,"15min":2.105},"min":{"1min":0.33200000000000002,"5min":0.311,"15min":0.30599999999999999},"max":{"1min":6.5880000000000001,"5min":12.382,"15min":25.466000000000001},"last":0.63}]},{"osd":4,"last update":"Sat Sep 23 21:19:12 2023","interfaces":[{"interface":"back","average":{"1min":0.71099999999999997,"5min":0.76600000000000001,"15min":1.1279999999999999},"min":{"1min":0.28699999999999998,"5min":0.253,"15min":0.253},"max":{"1min":2.8599999999999999,"5min":5.0279999999999996,"15min":12.776},"last":0.73099999999999998},{"interface":"front","average":{"1min":0.65400000000000003,"5min":0.83699999999999997,"15min":1.1359999999999999},"min":{"1min":0.27500000000000002,"5min":0.246,"15min":0.246},"max":{"1min":2.4399999999999999,"5min":5.149,"15min":11.973000000000001},"last":0.66100000000000003}]},{"osd":6,"last update":"Sat Sep 23 21:19:20 2023","interfaces":[{"interface":"back","average":{"1min":0.51600000000000001,"5min":0.97099999999999997,"15min":1.046},"min":{"1min":0.27400000000000002,"5min":0.27400000000000002,"15min":0.25900000000000001},"max":{"1min":1.3420000000000001,"5min":5.8520000000000003,"15min":8.0909999999999993},"last":0.74199999999999999},{"interface":"front","average":{"1min":0.66500000000000004,"5min":0.91600000000000004,"15min":1.196},"min":{"1min":0.377,"5min":0.34300000000000003,"15min":0.317},"max":{"1min":1.5329999999999999,"5min":7.1470000000000002,"15min":10.999000000000001},"last":0.77300000000000002}]},{"osd":7,"last update":"Sat Sep 23 21:18:54 2023","interfaces":[{"interface":"back","average":{"1min":0.95499999999999996,"5min":0.94299999999999995,"15min":1.821},"min":{"1min":0.33600000000000002,"5min":0.32400000000000001,"15min":0.31},"max":{"1min":4.0519999999999996,"5min":5.1769999999999996,"15min":189.405},"last":0.67300000000000004},{"interface":"front","average":{"1min":0.97799999999999998,"5min":0.96699999999999997,"15min":1.9490000000000001},"min":{"1min":0.34999999999999998,"5min":0.34999999999999998,"15min":0.32500000000000001},"max":{"1min":5.0910000000000002,"5min":5.0910000000000002,"15min":189.685},"last":0.755}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8488,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1259,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5322,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":4838,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":156,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":198,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":214,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":96,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":214,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":278,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T21:19:26.090 INFO:tasks.cephadm.ceph_manager.ceph:recovered! 2023-09-23T21:19:26.090 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T21:19:26.090 INFO:tasks.thrashosds.thrasher:Removing osd 5, in_osds are: [0, 2, 3, 4, 5, 7] 2023-09-23T21:19:26.090 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph osd out 5 2023-09-23T21:19:26.092 INFO:tasks.workunit:Created dir /home/ubuntu/cephtest/mnt.0 2023-09-23T21:19:26.092 DEBUG:teuthology.orchestra.run.smithi026:> cd -- /home/ubuntu/cephtest/mnt.0 && mkdir -- client.0 2023-09-23T21:19:26.119 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:19:26.151 INFO:tasks.workunit:timeout=3h 2023-09-23T21:19:26.152 INFO:tasks.workunit:cleanup=True 2023-09-23T21:19:26.152 DEBUG:teuthology.orchestra.run.smithi026:> rm -rf /home/ubuntu/cephtest/clone.client.0 && git clone --depth 1 --branch quincy https://github.com/ceph/ceph.git /home/ubuntu/cephtest/clone.client.0 && cd /home/ubuntu/cephtest/clone.client.0 2023-09-23T21:19:26.163 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:26.182 INFO:tasks.workunit.client.0.smithi026.stderr:Cloning into '/home/ubuntu/cephtest/clone.client.0'... 2023-09-23T21:19:26.470 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T21:19:26.472 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:26.473 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:19:26.573 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:19:26.597 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:19:26.604 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:26.639 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:19:26.642 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:26.643 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:19:26.743 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:19:26.770 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:19:26 smithi026 ceph-mon[175151]: pgmap v1595: 135 pgs: 135 active+clean; 582 KiB data, 4.3 GiB used, 532 GiB / 536 GiB avail; 1.9 MiB/s rd, 917 KiB/s wr, 188 op/s 2023-09-23T21:19:26.771 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:19:26 smithi026 ceph-mon[175151]: from='client.47105 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T21:19:26.771 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:19:26 smithi026 ceph-mon[175151]: failed to encode map e648 with expected crc 2023-09-23T21:19:26.771 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:19:26 smithi026 ceph-mon[175151]: failed to encode map e648 with expected crc 2023-09-23T21:19:26.771 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:19:26 smithi026 ceph-mon[175151]: osdmap e648: 8 total, 8 up, 6 in 2023-09-23T21:19:26.771 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:19:26 smithi026 ceph-mon[175151]: failed to encode map e648 with expected crc 2023-09-23T21:19:26.771 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:19:26 smithi026 ceph-mon[175151]: failed to encode map e648 with expected crc 2023-09-23T21:19:26.771 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:19:26 smithi026 ceph-mon[179688]: pgmap v1595: 135 pgs: 135 active+clean; 582 KiB data, 4.3 GiB used, 532 GiB / 536 GiB avail; 1.9 MiB/s rd, 917 KiB/s wr, 188 op/s 2023-09-23T21:19:26.771 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:19:26 smithi026 ceph-mon[179688]: from='client.47105 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T21:19:26.772 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:19:26 smithi026 ceph-mon[179688]: failed to encode map e648 with expected crc 2023-09-23T21:19:26.772 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:19:26 smithi026 ceph-mon[179688]: failed to encode map e648 with expected crc 2023-09-23T21:19:26.772 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:19:26 smithi026 ceph-mon[179688]: osdmap e648: 8 total, 8 up, 6 in 2023-09-23T21:19:26.772 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:19:26 smithi026 ceph-mon[179688]: failed to encode map e648 with expected crc 2023-09-23T21:19:26.772 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:19:26 smithi026 ceph-mon[179688]: failed to encode map e648 with expected crc 2023-09-23T21:19:26.774 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:26.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:19:26 smithi070 ceph-mon[168282]: pgmap v1595: 135 pgs: 135 active+clean; 582 KiB data, 4.3 GiB used, 532 GiB / 536 GiB avail; 1.9 MiB/s rd, 917 KiB/s wr, 188 op/s 2023-09-23T21:19:26.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:19:26 smithi070 ceph-mon[168282]: from='client.47105 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T21:19:26.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:19:26 smithi070 ceph-mon[168282]: failed to encode map e648 with expected crc 2023-09-23T21:19:26.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:19:26 smithi070 ceph-mon[168282]: failed to encode map e648 with expected crc 2023-09-23T21:19:26.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:19:26 smithi070 ceph-mon[168282]: osdmap e648: 8 total, 8 up, 6 in 2023-09-23T21:19:26.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:19:26 smithi070 ceph-mon[168282]: failed to encode map e648 with expected crc 2023-09-23T21:19:26.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:19:26 smithi070 ceph-mon[168282]: failed to encode map e648 with expected crc 2023-09-23T21:19:27.041 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T21:19:27.049 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:27.049 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:19:27.093 INFO:teuthology.orchestra.run.smithi070.stdout:{ 2023-09-23T21:19:27.094 INFO:teuthology.orchestra.run.smithi070.stdout: "size": 20, 2023-09-23T21:19:27.094 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 600, 2023-09-23T21:19:27.094 INFO:teuthology.orchestra.run.smithi070.stdout: "ops": [ 2023-09-23T21:19:27.094 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.094 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.46328.0:74333 10.d 10:bc627dcb:::benchmark_data_smithi026_379838_object74332:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e460)", 2023-09-23T21:19:27.094 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:11:37.749827+0000", 2023-09-23T21:19:27.094 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 469.34131973900003, 2023-09-23T21:19:27.094 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.4276132540000002, 2023-09-23T21:19:27.094 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:19:27.094 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:19:27.095 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:19:27.095 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.46328", 2023-09-23T21:19:27.095 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/3748861181", 2023-09-23T21:19:27.095 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 74333 2023-09-23T21:19:27.095 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.095 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:19:27.095 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.095 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:19:27.095 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:37.749827+0000", 2023-09-23T21:19:27.095 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:19:27.095 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.095 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.096 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:19:27.096 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:37.749827+0000", 2023-09-23T21:19:27.096 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:19:27.096 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.096 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.096 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:19:27.096 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:37.749825+0000", 2023-09-23T21:19:27.096 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999981 2023-09-23T21:19:27.096 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.096 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.096 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:19:27.096 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:37.749879+0000", 2023-09-23T21:19:27.097 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.4143e-05 2023-09-23T21:19:27.097 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.097 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.097 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:19:27.097 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:37.749894+0000", 2023-09-23T21:19:27.097 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.4973e-05 2023-09-23T21:19:27.097 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.097 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.097 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:19:27.097 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:37.749903+0000", 2023-09-23T21:19:27.097 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.0059999999999998e-06 2023-09-23T21:19:27.097 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.098 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.098 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:19:27.098 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:37.749946+0000", 2023-09-23T21:19:27.098 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.2349999999999999e-05 2023-09-23T21:19:27.098 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.098 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.098 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:19:27.098 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:37.749958+0000", 2023-09-23T21:19:27.098 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.2622999999999999e-05 2023-09-23T21:19:27.098 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.098 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.098 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:19:27.099 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:38.737069+0000", 2023-09-23T21:19:27.099 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.98711088800000002 2023-09-23T21:19:27.099 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.099 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.099 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:19:27.099 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:38.737077+0000", 2023-09-23T21:19:27.099 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.0150000000000003e-06 2023-09-23T21:19:27.099 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.099 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.099 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:19:27.099 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:40.172893+0000", 2023-09-23T21:19:27.099 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.435816266 2023-09-23T21:19:27.099 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.100 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.100 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:19:27.100 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:40.173003+0000", 2023-09-23T21:19:27.100 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00010993 2023-09-23T21:19:27.100 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.100 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.100 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 0,3,7", 2023-09-23T21:19:27.100 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:40.173114+0000", 2023-09-23T21:19:27.100 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000110849 2023-09-23T21:19:27.100 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.100 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.100 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:19:27.101 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:40.175684+0000", 2023-09-23T21:19:27.101 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0025697369999999999 2023-09-23T21:19:27.101 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.101 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.101 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:19:27.101 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:40.177298+0000", 2023-09-23T21:19:27.101 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0016144969999999999 2023-09-23T21:19:27.101 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.101 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.101 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:19:27.101 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:40.177308+0000", 2023-09-23T21:19:27.101 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.8879999999999992e-06 2023-09-23T21:19:27.102 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.102 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.102 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:19:27.102 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:40.177333+0000", 2023-09-23T21:19:27.102 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.4905999999999998e-05 2023-09-23T21:19:27.102 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.102 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.102 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:19:27.102 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:40.177376+0000", 2023-09-23T21:19:27.102 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.2635000000000001e-05 2023-09-23T21:19:27.102 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.102 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.103 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:19:27.103 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:40.177440+0000", 2023-09-23T21:19:27.103 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.4252999999999996e-05 2023-09-23T21:19:27.103 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:19:27.103 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:19:27.103 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:19:27.103 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.103 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.103 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.46328.0:74356 10.d 10:ba33591e:::benchmark_data_smithi026_379838_object74355:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e460)", 2023-09-23T21:19:27.103 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:11:37.787894+0000", 2023-09-23T21:19:27.103 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 469.30325298399998, 2023-09-23T21:19:27.103 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.3971935719999999, 2023-09-23T21:19:27.104 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:19:27.104 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:19:27.104 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:19:27.104 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.46328", 2023-09-23T21:19:27.104 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/3748861181", 2023-09-23T21:19:27.104 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 74356 2023-09-23T21:19:27.104 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.104 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:19:27.104 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.104 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:19:27.104 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:37.787894+0000", 2023-09-23T21:19:27.104 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:19:27.105 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.105 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.105 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:19:27.105 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:37.787894+0000", 2023-09-23T21:19:27.105 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:19:27.105 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.105 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.105 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:19:27.105 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:37.787893+0000", 2023-09-23T21:19:27.105 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999995 2023-09-23T21:19:27.105 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.105 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.106 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:19:27.106 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:37.787916+0000", 2023-09-23T21:19:27.106 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.2857000000000002e-05 2023-09-23T21:19:27.106 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.106 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.106 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:19:27.106 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:37.787923+0000", 2023-09-23T21:19:27.106 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.6540000000000002e-06 2023-09-23T21:19:27.106 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.106 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.106 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:19:27.106 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:37.787927+0000", 2023-09-23T21:19:27.107 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.4399999999999998e-06 2023-09-23T21:19:27.107 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.107 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.107 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:19:27.107 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:37.787987+0000", 2023-09-23T21:19:27.107 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.9561000000000001e-05 2023-09-23T21:19:27.107 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.107 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.107 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:19:27.107 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:37.787997+0000", 2023-09-23T21:19:27.107 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.9189999999999994e-06 2023-09-23T21:19:27.107 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.108 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.108 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:19:27.108 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:38.737092+0000", 2023-09-23T21:19:27.108 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.94909543100000004 2023-09-23T21:19:27.108 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.108 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.108 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:19:27.108 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:38.737097+0000", 2023-09-23T21:19:27.108 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.9169999999999997e-06 2023-09-23T21:19:27.108 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.108 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.108 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:19:27.109 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:40.175712+0000", 2023-09-23T21:19:27.109 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.438615602 2023-09-23T21:19:27.109 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.109 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.109 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:19:27.109 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:40.175796+0000", 2023-09-23T21:19:27.109 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.3318000000000004e-05 2023-09-23T21:19:27.109 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.109 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.109 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 0,3,7", 2023-09-23T21:19:27.109 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:40.175911+0000", 2023-09-23T21:19:27.109 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000115275 2023-09-23T21:19:27.110 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.110 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.110 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:19:27.110 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:40.181314+0000", 2023-09-23T21:19:27.110 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.005402919 2023-09-23T21:19:27.110 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.110 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.110 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:19:27.110 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:40.181389+0000", 2023-09-23T21:19:27.110 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.4692999999999998e-05 2023-09-23T21:19:27.110 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.110 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.111 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:19:27.111 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:40.181395+0000", 2023-09-23T21:19:27.111 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.6200000000000001e-06 2023-09-23T21:19:27.111 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.111 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.111 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:19:27.111 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:40.184985+0000", 2023-09-23T21:19:27.111 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0035895900000000001 2023-09-23T21:19:27.111 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.111 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.111 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:19:27.111 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:40.185022+0000", 2023-09-23T21:19:27.112 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.7166000000000003e-05 2023-09-23T21:19:27.112 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.112 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.112 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:19:27.112 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:40.185087+0000", 2023-09-23T21:19:27.112 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.5100999999999999e-05 2023-09-23T21:19:27.112 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:19:27.112 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:19:27.112 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:19:27.112 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.112 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.112 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.46328.0:74367 10.d 10:b9926de2:::benchmark_data_smithi026_379838_object74366:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e460)", 2023-09-23T21:19:27.113 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:11:37.805093+0000", 2023-09-23T21:19:27.113 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 469.28605318699999, 2023-09-23T21:19:27.113 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.3801344150000001, 2023-09-23T21:19:27.113 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:19:27.113 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:19:27.113 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:19:27.113 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.46328", 2023-09-23T21:19:27.113 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/3748861181", 2023-09-23T21:19:27.113 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 74367 2023-09-23T21:19:27.113 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.113 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:19:27.113 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.114 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:19:27.114 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:37.805093+0000", 2023-09-23T21:19:27.114 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:19:27.114 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.114 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.114 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:19:27.114 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:37.805093+0000", 2023-09-23T21:19:27.114 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:19:27.114 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.114 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.114 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:19:27.115 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:37.805092+0000", 2023-09-23T21:19:27.115 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999981 2023-09-23T21:19:27.115 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.115 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.115 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:19:27.115 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:37.805182+0000", 2023-09-23T21:19:27.115 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.0195e-05 2023-09-23T21:19:27.115 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.115 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.115 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:19:27.115 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:37.805195+0000", 2023-09-23T21:19:27.115 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.2823000000000001e-05 2023-09-23T21:19:27.116 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.116 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.116 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:19:27.116 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:37.805201+0000", 2023-09-23T21:19:27.116 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.759e-06 2023-09-23T21:19:27.116 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.116 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.116 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:19:27.116 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:37.805275+0000", 2023-09-23T21:19:27.116 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.3203999999999999e-05 2023-09-23T21:19:27.116 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.116 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.117 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:19:27.117 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:37.805287+0000", 2023-09-23T21:19:27.117 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.2526e-05 2023-09-23T21:19:27.117 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.117 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.117 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:19:27.117 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:38.737108+0000", 2023-09-23T21:19:27.117 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.93182051399999999 2023-09-23T21:19:27.117 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.117 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.117 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:19:27.117 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:38.737112+0000", 2023-09-23T21:19:27.117 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.7900000000000001e-06 2023-09-23T21:19:27.118 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.118 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.118 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:19:27.118 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:40.177476+0000", 2023-09-23T21:19:27.118 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.440364829 2023-09-23T21:19:27.118 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.118 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.118 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:19:27.118 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:40.177607+0000", 2023-09-23T21:19:27.118 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00013115199999999999 2023-09-23T21:19:27.118 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.118 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.119 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 0,3,7", 2023-09-23T21:19:27.119 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:40.177728+0000", 2023-09-23T21:19:27.119 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00012074 2023-09-23T21:19:27.119 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.119 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.119 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:19:27.119 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:40.181346+0000", 2023-09-23T21:19:27.119 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0036178339999999999 2023-09-23T21:19:27.119 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.119 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.119 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:19:27.119 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:40.184839+0000", 2023-09-23T21:19:27.120 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.003493351 2023-09-23T21:19:27.120 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.120 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.120 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:19:27.120 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:40.184931+0000", 2023-09-23T21:19:27.120 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.1612000000000003e-05 2023-09-23T21:19:27.120 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.120 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.120 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:19:27.120 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:40.185147+0000", 2023-09-23T21:19:27.120 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00021584999999999999 2023-09-23T21:19:27.120 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.120 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.121 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:19:27.121 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:40.185173+0000", 2023-09-23T21:19:27.121 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.5884000000000002e-05 2023-09-23T21:19:27.121 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.121 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.121 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:19:27.121 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:40.185228+0000", 2023-09-23T21:19:27.121 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.5062000000000002e-05 2023-09-23T21:19:27.121 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:19:27.121 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:19:27.121 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:19:27.121 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.122 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.122 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.46328.0:74398 10.d 10:bdc342d2:::benchmark_data_smithi026_379838_object74397:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e460)", 2023-09-23T21:19:27.122 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:11:37.837382+0000", 2023-09-23T21:19:27.122 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 469.25376440299999, 2023-09-23T21:19:27.122 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.3480186879999998, 2023-09-23T21:19:27.122 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:19:27.122 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:19:27.122 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:19:27.122 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.46328", 2023-09-23T21:19:27.122 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/3748861181", 2023-09-23T21:19:27.122 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 74398 2023-09-23T21:19:27.122 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.123 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:19:27.123 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.123 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:19:27.123 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:37.837382+0000", 2023-09-23T21:19:27.123 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:19:27.123 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.123 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.123 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:19:27.123 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:37.837382+0000", 2023-09-23T21:19:27.123 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:19:27.123 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.123 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.123 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:19:27.124 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:37.837381+0000", 2023-09-23T21:19:27.124 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.999999 2023-09-23T21:19:27.124 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.124 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.124 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:19:27.124 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:37.837411+0000", 2023-09-23T21:19:27.124 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.0006e-05 2023-09-23T21:19:27.124 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.124 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.124 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:19:27.124 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:37.837421+0000", 2023-09-23T21:19:27.124 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.4029999999999996e-06 2023-09-23T21:19:27.125 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.125 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.125 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:19:27.125 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:37.837425+0000", 2023-09-23T21:19:27.125 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.7249999999999997e-06 2023-09-23T21:19:27.125 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.125 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.125 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:19:27.125 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:37.837477+0000", 2023-09-23T21:19:27.125 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.1430000000000001e-05 2023-09-23T21:19:27.125 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.125 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.126 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:19:27.126 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:37.837488+0000", 2023-09-23T21:19:27.126 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.1357e-05 2023-09-23T21:19:27.126 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.126 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.126 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:19:27.126 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:38.737135+0000", 2023-09-23T21:19:27.126 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.89964650899999998 2023-09-23T21:19:27.126 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.126 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.126 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:19:27.126 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:38.737138+0000", 2023-09-23T21:19:27.127 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.8709999999999999e-06 2023-09-23T21:19:27.127 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.127 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.127 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:19:27.127 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:40.179280+0000", 2023-09-23T21:19:27.127 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.442141605 2023-09-23T21:19:27.127 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.127 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.127 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:19:27.127 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:40.179376+0000", 2023-09-23T21:19:27.127 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.5909999999999995e-05 2023-09-23T21:19:27.127 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.128 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.128 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 0,3,7", 2023-09-23T21:19:27.128 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:40.179505+0000", 2023-09-23T21:19:27.128 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00012945900000000001 2023-09-23T21:19:27.128 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.128 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.128 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:19:27.128 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:40.184887+0000", 2023-09-23T21:19:27.128 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.005381884 2023-09-23T21:19:27.128 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.128 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.128 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:19:27.129 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:40.184944+0000", 2023-09-23T21:19:27.129 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.6521999999999999e-05 2023-09-23T21:19:27.129 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.129 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.129 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:19:27.129 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:40.185125+0000", 2023-09-23T21:19:27.129 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00018092 2023-09-23T21:19:27.129 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.129 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.129 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:19:27.129 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:40.185307+0000", 2023-09-23T21:19:27.129 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000181991 2023-09-23T21:19:27.130 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.130 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.130 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:19:27.130 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:40.185340+0000", 2023-09-23T21:19:27.130 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.3021000000000001e-05 2023-09-23T21:19:27.130 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.130 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.130 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:19:27.130 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:40.185401+0000", 2023-09-23T21:19:27.130 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.1158e-05 2023-09-23T21:19:27.130 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:19:27.130 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:19:27.131 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:19:27.131 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.131 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.131 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.46328.0:74400 10.d 10:b25fac0d:::benchmark_data_smithi026_379838_object74399:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e460)", 2023-09-23T21:19:27.131 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:11:37.840259+0000", 2023-09-23T21:19:27.131 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 469.25088733600001, 2023-09-23T21:19:27.131 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.345304139, 2023-09-23T21:19:27.131 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:19:27.131 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:19:27.131 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:19:27.131 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.46328", 2023-09-23T21:19:27.131 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/3748861181", 2023-09-23T21:19:27.131 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 74400 2023-09-23T21:19:27.132 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.132 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:19:27.132 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.132 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:19:27.132 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:37.840259+0000", 2023-09-23T21:19:27.132 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:19:27.132 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.132 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.132 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:19:27.132 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:37.840259+0000", 2023-09-23T21:19:27.132 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:19:27.132 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.133 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.133 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:19:27.133 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:37.840257+0000", 2023-09-23T21:19:27.133 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999981 2023-09-23T21:19:27.133 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.133 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.133 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:19:27.133 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:37.840311+0000", 2023-09-23T21:19:27.133 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.3449000000000003e-05 2023-09-23T21:19:27.133 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.133 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.133 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:19:27.134 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:37.840322+0000", 2023-09-23T21:19:27.134 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.1197e-05 2023-09-23T21:19:27.134 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.134 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.134 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:19:27.134 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:37.840328+0000", 2023-09-23T21:19:27.134 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.2129999999999996e-06 2023-09-23T21:19:27.134 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.134 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.134 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:19:27.134 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:37.840388+0000", 2023-09-23T21:19:27.134 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.0081000000000003e-05 2023-09-23T21:19:27.135 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.135 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.135 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:19:27.135 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:37.840399+0000", 2023-09-23T21:19:27.135 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.0465000000000001e-05 2023-09-23T21:19:27.135 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.135 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.135 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:19:27.135 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:38.737150+0000", 2023-09-23T21:19:27.135 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.89675132499999999 2023-09-23T21:19:27.135 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.135 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.135 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:19:27.136 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:38.737154+0000", 2023-09-23T21:19:27.136 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.6200000000000001e-06 2023-09-23T21:19:27.136 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.136 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.136 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:19:27.136 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:40.181411+0000", 2023-09-23T21:19:27.136 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.444257009 2023-09-23T21:19:27.136 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.136 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.136 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:19:27.136 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:40.181497+0000", 2023-09-23T21:19:27.136 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.6570000000000006e-05 2023-09-23T21:19:27.137 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.137 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.137 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 0,3,7", 2023-09-23T21:19:27.137 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:40.181626+0000", 2023-09-23T21:19:27.137 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00012901699999999999 2023-09-23T21:19:27.137 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.137 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.137 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:19:27.137 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:40.184950+0000", 2023-09-23T21:19:27.137 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.003323501 2023-09-23T21:19:27.137 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.137 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.138 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:19:27.138 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:40.185262+0000", 2023-09-23T21:19:27.138 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000312045 2023-09-23T21:19:27.138 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.138 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.138 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:19:27.138 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:40.185281+0000", 2023-09-23T21:19:27.138 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.9055999999999998e-05 2023-09-23T21:19:27.138 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.138 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.138 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:19:27.138 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:40.185481+0000", 2023-09-23T21:19:27.139 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00019994199999999999 2023-09-23T21:19:27.139 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.139 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.139 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:19:27.139 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:40.185508+0000", 2023-09-23T21:19:27.139 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.6801e-05 2023-09-23T21:19:27.139 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.139 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.139 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:19:27.139 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:40.185563+0000", 2023-09-23T21:19:27.139 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.5670000000000001e-05 2023-09-23T21:19:27.139 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:19:27.139 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:19:27.140 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:19:27.140 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.140 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.140 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.46328.0:74403 10.d 10:b5a2ad18:::benchmark_data_smithi026_379838_object74402:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e460)", 2023-09-23T21:19:27.140 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:11:37.844596+0000", 2023-09-23T21:19:27.140 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 469.24655083200003, 2023-09-23T21:19:27.140 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.3410775579999998, 2023-09-23T21:19:27.140 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:19:27.140 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:19:27.140 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:19:27.140 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.46328", 2023-09-23T21:19:27.140 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/3748861181", 2023-09-23T21:19:27.140 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 74403 2023-09-23T21:19:27.141 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.141 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:19:27.141 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.141 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:19:27.141 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:37.844596+0000", 2023-09-23T21:19:27.141 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:19:27.141 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.141 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.141 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:19:27.141 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:37.844596+0000", 2023-09-23T21:19:27.141 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:19:27.141 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.141 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.142 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:19:27.142 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:37.844594+0000", 2023-09-23T21:19:27.142 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T21:19:27.142 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.142 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.142 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:19:27.142 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:37.844658+0000", 2023-09-23T21:19:27.142 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.4053000000000005e-05 2023-09-23T21:19:27.142 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.142 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.142 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:19:27.142 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:37.844675+0000", 2023-09-23T21:19:27.146 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.6670000000000001e-05 2023-09-23T21:19:27.146 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.146 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.146 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:19:27.146 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:37.844682+0000", 2023-09-23T21:19:27.146 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.7839999999999998e-06 2023-09-23T21:19:27.146 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.146 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.146 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:19:27.147 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:37.844751+0000", 2023-09-23T21:19:27.147 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.9698999999999998e-05 2023-09-23T21:19:27.147 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.147 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.147 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:19:27.147 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:37.844762+0000", 2023-09-23T21:19:27.147 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.049e-05 2023-09-23T21:19:27.147 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.147 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.147 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:19:27.147 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:38.737168+0000", 2023-09-23T21:19:27.147 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.89240619799999998 2023-09-23T21:19:27.147 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.148 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.148 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:19:27.148 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:38.737174+0000", 2023-09-23T21:19:27.148 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.3319999999999997e-06 2023-09-23T21:19:27.148 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.148 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.148 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:19:27.148 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:40.183023+0000", 2023-09-23T21:19:27.148 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.445848958 2023-09-23T21:19:27.148 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.148 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.148 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:19:27.148 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:40.183163+0000", 2023-09-23T21:19:27.149 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00013908200000000001 2023-09-23T21:19:27.149 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.149 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.149 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 0,3,7", 2023-09-23T21:19:27.149 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:40.183342+0000", 2023-09-23T21:19:27.149 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000179218 2023-09-23T21:19:27.149 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.149 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.149 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:19:27.149 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:40.185290+0000", 2023-09-23T21:19:27.149 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0019481769999999999 2023-09-23T21:19:27.150 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.150 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.150 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:19:27.150 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:40.185432+0000", 2023-09-23T21:19:27.150 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000142435 2023-09-23T21:19:27.150 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.150 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.150 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:19:27.150 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:40.185457+0000", 2023-09-23T21:19:27.150 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.5054000000000001e-05 2023-09-23T21:19:27.150 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.150 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.151 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:19:27.151 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:40.185592+0000", 2023-09-23T21:19:27.151 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00013438499999999999 2023-09-23T21:19:27.151 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.151 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.151 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:19:27.151 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:40.185619+0000", 2023-09-23T21:19:27.151 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.6727e-05 2023-09-23T21:19:27.151 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.151 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.151 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:19:27.151 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:40.185673+0000", 2023-09-23T21:19:27.151 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.4796000000000002e-05 2023-09-23T21:19:27.152 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:19:27.152 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:19:27.152 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:19:27.152 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.152 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.152 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.46328.0:74404 10.d 10:bd0dcc09:::benchmark_data_smithi026_379838_object74403:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e460)", 2023-09-23T21:19:27.152 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:11:37.844695+0000", 2023-09-23T21:19:27.152 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 469.24645169399997, 2023-09-23T21:19:27.152 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.3484742669999998, 2023-09-23T21:19:27.152 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:19:27.152 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:19:27.152 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:19:27.153 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.46328", 2023-09-23T21:19:27.153 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/3748861181", 2023-09-23T21:19:27.153 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 74404 2023-09-23T21:19:27.153 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.153 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:19:27.153 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.153 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:19:27.153 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:37.844695+0000", 2023-09-23T21:19:27.153 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:19:27.153 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.153 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.153 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:19:27.154 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:37.844695+0000", 2023-09-23T21:19:27.154 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:19:27.154 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.154 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.154 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:19:27.154 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:37.844695+0000", 2023-09-23T21:19:27.154 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999995 2023-09-23T21:19:27.154 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.154 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.154 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:19:27.154 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:37.844719+0000", 2023-09-23T21:19:27.154 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.4070000000000002e-05 2023-09-23T21:19:27.155 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.155 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.155 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:19:27.155 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:37.844726+0000", 2023-09-23T21:19:27.155 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.3459999999999996e-06 2023-09-23T21:19:27.155 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.155 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.155 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:19:27.155 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:37.844730+0000", 2023-09-23T21:19:27.155 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.5920000000000001e-06 2023-09-23T21:19:27.155 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.155 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.156 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:19:27.156 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:37.844800+0000", 2023-09-23T21:19:27.156 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.0759000000000005e-05 2023-09-23T21:19:27.156 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.156 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.156 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:19:27.156 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:37.844812+0000", 2023-09-23T21:19:27.156 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.2130000000000001e-05 2023-09-23T21:19:27.156 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.156 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.156 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:19:27.156 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:38.737187+0000", 2023-09-23T21:19:27.157 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.89237498199999998 2023-09-23T21:19:27.157 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.157 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.157 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:19:27.157 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:38.737191+0000", 2023-09-23T21:19:27.157 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.7189999999999999e-06 2023-09-23T21:19:27.157 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.157 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.157 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:19:27.157 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:40.185696+0000", 2023-09-23T21:19:27.157 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.4485052139999999 2023-09-23T21:19:27.157 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.157 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.158 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:19:27.158 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:40.185770+0000", 2023-09-23T21:19:27.158 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.3560999999999996e-05 2023-09-23T21:19:27.158 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.158 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.158 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 0,3,7", 2023-09-23T21:19:27.158 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:40.185864+0000", 2023-09-23T21:19:27.158 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.4244000000000001e-05 2023-09-23T21:19:27.158 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.158 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.158 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:19:27.158 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:40.192926+0000", 2023-09-23T21:19:27.159 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0070613910000000002 2023-09-23T21:19:27.159 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.159 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.159 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:19:27.159 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:40.192930+0000", 2023-09-23T21:19:27.159 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.8080000000000004e-06 2023-09-23T21:19:27.159 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.159 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.159 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:19:27.159 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:40.192975+0000", 2023-09-23T21:19:27.159 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.4990000000000001e-05 2023-09-23T21:19:27.159 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.160 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.160 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:19:27.160 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:40.193012+0000", 2023-09-23T21:19:27.160 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.6443000000000003e-05 2023-09-23T21:19:27.160 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.160 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.160 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:19:27.160 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:40.193069+0000", 2023-09-23T21:19:27.160 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.7065000000000003e-05 2023-09-23T21:19:27.160 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.160 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.160 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:19:27.161 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:11:40.193169+0000", 2023-09-23T21:19:27.161 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000100315 2023-09-23T21:19:27.161 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:19:27.161 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:19:27.161 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:19:27.161 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.161 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.161 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_repop(client.46328.0:84573 10.c e471/465 10:3fce4c21:::benchmark_data_smithi026_379838_object4616:head v 471'5331, mlcod=470'5329)", 2023-09-23T21:19:27.161 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:12:10.780185+0000", 2023-09-23T21:19:27.161 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 436.31096195700002, 2023-09-23T21:19:27.161 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.5674745780000001, 2023-09-23T21:19:27.161 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:19:27.162 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:19:27.162 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:19:27.162 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.162 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:19:27.162 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:12:10.780185+0000", 2023-09-23T21:19:27.162 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:19:27.162 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.162 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.162 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:19:27.162 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:12:10.780185+0000", 2023-09-23T21:19:27.162 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:19:27.162 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.163 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.163 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:19:27.163 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:12:10.780184+0000", 2023-09-23T21:19:27.163 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999995 2023-09-23T21:19:27.163 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.163 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.163 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:19:27.163 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:12:10.780189+0000", 2023-09-23T21:19:27.163 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.7650000000000001e-06 2023-09-23T21:19:27.163 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.163 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.163 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:19:27.163 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:12:10.780196+0000", 2023-09-23T21:19:27.164 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.9040000000000003e-06 2023-09-23T21:19:27.164 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.164 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.164 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:19:27.164 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:12:10.780203+0000", 2023-09-23T21:19:27.164 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.3690000000000001e-06 2023-09-23T21:19:27.164 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.164 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.164 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:19:27.164 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:12:14.336808+0000", 2023-09-23T21:19:27.164 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.556604718 2023-09-23T21:19:27.164 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.165 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.165 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:19:27.165 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:12:14.336836+0000", 2023-09-23T21:19:27.165 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.7379e-05 2023-09-23T21:19:27.165 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.165 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.165 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:19:27.165 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:12:14.347611+0000", 2023-09-23T21:19:27.165 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.01077504 2023-09-23T21:19:27.165 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.165 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.165 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:19:27.165 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:12:14.347659+0000", 2023-09-23T21:19:27.166 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.8668000000000002e-05 2023-09-23T21:19:27.166 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:19:27.166 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:19:27.166 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:19:27.166 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.166 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.166 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_repop(client.46328.0:84576 10.c e471/465 10:3b2a76db:::benchmark_data_smithi026_379838_object4619:head v 471'5332, mlcod=470'5329)", 2023-09-23T21:19:27.166 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:12:10.849370+0000", 2023-09-23T21:19:27.166 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 436.24177647699997, 2023-09-23T21:19:27.166 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.503394407, 2023-09-23T21:19:27.166 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:19:27.166 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:19:27.167 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:19:27.167 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.167 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:19:27.167 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:12:10.849370+0000", 2023-09-23T21:19:27.167 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:19:27.167 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.167 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.167 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:19:27.167 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:12:10.849370+0000", 2023-09-23T21:19:27.167 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:19:27.167 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.167 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.167 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:19:27.168 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:12:10.849369+0000", 2023-09-23T21:19:27.168 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.999999 2023-09-23T21:19:27.168 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.168 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.168 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:19:27.168 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:12:10.849380+0000", 2023-09-23T21:19:27.168 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.0987e-05 2023-09-23T21:19:27.168 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.168 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.168 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:19:27.168 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:12:10.849392+0000", 2023-09-23T21:19:27.168 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.1843e-05 2023-09-23T21:19:27.169 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.169 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.169 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:19:27.169 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:12:10.849399+0000", 2023-09-23T21:19:27.169 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.8549999999999996e-06 2023-09-23T21:19:27.169 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.169 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.169 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:19:27.169 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:12:14.345110+0000", 2023-09-23T21:19:27.169 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.4957117150000001 2023-09-23T21:19:27.169 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.169 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.169 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:19:27.170 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:12:14.345139+0000", 2023-09-23T21:19:27.170 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.8362999999999999e-05 2023-09-23T21:19:27.170 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.170 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.170 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:19:27.170 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:12:14.352724+0000", 2023-09-23T21:19:27.170 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0075854609999999999 2023-09-23T21:19:27.170 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.170 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.170 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:19:27.170 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:12:14.352765+0000", 2023-09-23T21:19:27.170 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.0368999999999999e-05 2023-09-23T21:19:27.171 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:19:27.171 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:19:27.171 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:19:27.171 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.171 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.171 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.46328.0:84587 10.0 10:01ac773b:::benchmark_data_smithi026_379838_object4630:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e472)", 2023-09-23T21:19:27.171 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:12:11.856050+0000", 2023-09-23T21:19:27.171 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 435.23509701799998, 2023-09-23T21:19:27.171 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.9827590000000002, 2023-09-23T21:19:27.171 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:19:27.171 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:19:27.171 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:19:27.172 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.46328", 2023-09-23T21:19:27.172 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/3748861181", 2023-09-23T21:19:27.172 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 84587 2023-09-23T21:19:27.172 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.172 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:19:27.172 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.172 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:19:27.172 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:12:11.856050+0000", 2023-09-23T21:19:27.172 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:19:27.172 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.172 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.172 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:19:27.172 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:12:11.856050+0000", 2023-09-23T21:19:27.173 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:19:27.173 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.173 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.173 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:19:27.173 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:12:11.856048+0000", 2023-09-23T21:19:27.173 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.999999 2023-09-23T21:19:27.173 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.173 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.173 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:19:27.173 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:12:11.856055+0000", 2023-09-23T21:19:27.173 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.883e-06 2023-09-23T21:19:27.173 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.173 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.174 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:19:27.174 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:12:11.856067+0000", 2023-09-23T21:19:27.174 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.1541e-05 2023-09-23T21:19:27.174 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.174 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.174 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:19:27.174 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:12:11.856073+0000", 2023-09-23T21:19:27.174 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.348e-06 2023-09-23T21:19:27.174 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.174 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.174 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:19:27.175 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:12:14.482257+0000", 2023-09-23T21:19:27.175 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.62618397 2023-09-23T21:19:27.175 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.175 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.175 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for missing object", 2023-09-23T21:19:27.175 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:12:14.486854+0000", 2023-09-23T21:19:27.175 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.004597126 2023-09-23T21:19:27.175 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.175 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.175 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:19:27.175 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:12:14.521249+0000", 2023-09-23T21:19:27.175 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.034394451999999999 2023-09-23T21:19:27.176 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.176 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.176 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for rw locks", 2023-09-23T21:19:27.176 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:12:14.527211+0000", 2023-09-23T21:19:27.176 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0059623009999999997 2023-09-23T21:19:27.176 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.176 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.176 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:19:27.176 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:12:14.800609+0000", 2023-09-23T21:19:27.176 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.273397942 2023-09-23T21:19:27.176 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.176 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.176 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:19:27.176 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:12:14.803332+0000", 2023-09-23T21:19:27.177 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0027228970000000002 2023-09-23T21:19:27.177 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.177 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.177 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 5,7", 2023-09-23T21:19:27.177 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:12:14.808034+0000", 2023-09-23T21:19:27.177 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0047023059999999998 2023-09-23T21:19:27.177 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.177 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.177 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:19:27.177 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:12:14.814675+0000", 2023-09-23T21:19:27.177 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0066411070000000003 2023-09-23T21:19:27.177 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.177 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.178 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:19:27.178 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:12:14.827001+0000", 2023-09-23T21:19:27.178 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.012326043 2023-09-23T21:19:27.178 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.178 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.178 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:19:27.178 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:12:14.828326+0000", 2023-09-23T21:19:27.178 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.001324483 2023-09-23T21:19:27.178 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.178 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.178 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:19:27.178 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:12:14.830512+0000", 2023-09-23T21:19:27.179 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0021857640000000002 2023-09-23T21:19:27.179 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.179 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.179 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:19:27.179 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:12:14.838809+0000", 2023-09-23T21:19:27.179 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0082969350000000001 2023-09-23T21:19:27.179 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:19:27.179 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:19:27.179 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:19:27.179 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.179 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.179 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.46328.0:84588 10.0 10:04267038:::benchmark_data_smithi026_379838_object4631:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e472)", 2023-09-23T21:19:27.180 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:12:11.858701+0000", 2023-09-23T21:19:27.180 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 435.23244573599999, 2023-09-23T21:19:27.180 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.915373185, 2023-09-23T21:19:27.180 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:19:27.180 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:19:27.180 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:19:27.180 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.46328", 2023-09-23T21:19:27.180 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/3748861181", 2023-09-23T21:19:27.180 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 84588 2023-09-23T21:19:27.180 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.180 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:19:27.180 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.181 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:19:27.181 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:12:11.858701+0000", 2023-09-23T21:19:27.181 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:19:27.181 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.181 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.181 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:19:27.181 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:12:11.858701+0000", 2023-09-23T21:19:27.181 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:19:27.181 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.181 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.181 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:19:27.182 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:12:11.858700+0000", 2023-09-23T21:19:27.182 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T21:19:27.182 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.182 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.182 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:19:27.182 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:12:11.858704+0000", 2023-09-23T21:19:27.182 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.9200000000000003e-06 2023-09-23T21:19:27.182 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.182 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.182 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:19:27.182 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:12:11.858714+0000", 2023-09-23T21:19:27.182 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.5719999999999992e-06 2023-09-23T21:19:27.183 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.183 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.183 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:19:27.183 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:12:11.858719+0000", 2023-09-23T21:19:27.183 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.3510000000000001e-06 2023-09-23T21:19:27.183 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.183 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.183 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:19:27.183 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:12:14.494615+0000", 2023-09-23T21:19:27.183 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.6358956099999999 2023-09-23T21:19:27.183 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.184 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.184 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for missing object", 2023-09-23T21:19:27.184 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:12:14.499085+0000", 2023-09-23T21:19:27.184 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0044701410000000004 2023-09-23T21:19:27.184 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.184 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.184 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:19:27.184 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:12:14.544838+0000", 2023-09-23T21:19:27.184 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.045753291000000001 2023-09-23T21:19:27.184 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.184 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.184 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for rw locks", 2023-09-23T21:19:27.185 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:12:14.550086+0000", 2023-09-23T21:19:27.185 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00524771 2023-09-23T21:19:27.185 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.185 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.185 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:19:27.185 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:12:14.703126+0000", 2023-09-23T21:19:27.185 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.15303945499999999 2023-09-23T21:19:27.185 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.185 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.185 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:19:27.185 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:12:14.708052+0000", 2023-09-23T21:19:27.185 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0049264809999999999 2023-09-23T21:19:27.186 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.186 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.186 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 5,7", 2023-09-23T21:19:27.186 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:12:14.713464+0000", 2023-09-23T21:19:27.186 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0054116909999999997 2023-09-23T21:19:27.186 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.186 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.186 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:19:27.186 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:12:14.735488+0000", 2023-09-23T21:19:27.186 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.022024681000000001 2023-09-23T21:19:27.186 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.186 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.187 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:19:27.187 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:12:14.765728+0000", 2023-09-23T21:19:27.187 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.030239499999999999 2023-09-23T21:19:27.187 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.187 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.187 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:19:27.187 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:12:14.766660+0000", 2023-09-23T21:19:27.187 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00093233900000000002 2023-09-23T21:19:27.187 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.187 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.187 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:19:27.187 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:12:14.768085+0000", 2023-09-23T21:19:27.187 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.001425001 2023-09-23T21:19:27.188 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.188 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.188 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:19:27.188 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:12:14.774074+0000", 2023-09-23T21:19:27.188 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0059887650000000001 2023-09-23T21:19:27.188 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:19:27.188 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:19:27.188 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:19:27.188 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.188 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.188 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_repop(client.46328.0:84616 10.c e472/465 10:3d3dd8d1:::benchmark_data_smithi026_379838_object4659:head v 472'5333, mlcod=470'5329)", 2023-09-23T21:19:27.188 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:12:11.948252+0000", 2023-09-23T21:19:27.189 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 435.14289489700002, 2023-09-23T21:19:27.189 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.4124408110000002, 2023-09-23T21:19:27.189 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:19:27.189 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:19:27.189 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:19:27.189 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.189 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:19:27.189 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:12:11.948252+0000", 2023-09-23T21:19:27.189 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:19:27.189 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.189 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.189 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:19:27.189 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:12:11.948252+0000", 2023-09-23T21:19:27.190 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:19:27.190 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.190 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.190 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:19:27.190 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:12:11.948251+0000", 2023-09-23T21:19:27.190 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.999999 2023-09-23T21:19:27.190 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.190 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.190 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:19:27.190 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:12:11.948260+0000", 2023-09-23T21:19:27.190 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.5789999999999993e-06 2023-09-23T21:19:27.190 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.190 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.191 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:19:27.191 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:12:11.948271+0000", 2023-09-23T21:19:27.191 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.0705e-05 2023-09-23T21:19:27.191 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.191 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.191 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:19:27.191 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:12:11.948278+0000", 2023-09-23T21:19:27.191 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.584e-06 2023-09-23T21:19:27.191 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.191 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.191 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:19:27.191 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:12:14.350144+0000", 2023-09-23T21:19:27.191 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.4018661539999999 2023-09-23T21:19:27.192 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.192 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.192 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:19:27.192 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:12:14.350169+0000", 2023-09-23T21:19:27.192 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.5304999999999999e-05 2023-09-23T21:19:27.192 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.192 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.192 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:19:27.192 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:12:14.360651+0000", 2023-09-23T21:19:27.192 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.010481656000000001 2023-09-23T21:19:27.192 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.192 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.193 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:19:27.193 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:12:14.360693+0000", 2023-09-23T21:19:27.193 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.1706999999999999e-05 2023-09-23T21:19:27.193 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:19:27.193 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:19:27.193 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:19:27.193 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.193 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.193 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_repop(client.46328.0:84636 10.c e472/465 10:3153f2e1:::benchmark_data_smithi026_379838_object4679:head v 472'5334, mlcod=470'5329)", 2023-09-23T21:19:27.193 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:12:12.046281+0000", 2023-09-23T21:19:27.193 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 435.044865824, 2023-09-23T21:19:27.194 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.3167927970000002, 2023-09-23T21:19:27.194 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:19:27.194 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:19:27.194 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:19:27.194 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.194 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:19:27.194 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:12:12.046281+0000", 2023-09-23T21:19:27.194 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:19:27.194 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.194 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.194 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:19:27.194 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:12:12.046281+0000", 2023-09-23T21:19:27.195 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:19:27.195 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.195 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.195 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:19:27.195 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:12:12.046280+0000", 2023-09-23T21:19:27.195 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T21:19:27.195 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.195 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.195 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:19:27.195 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:12:12.046291+0000", 2023-09-23T21:19:27.195 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.1494e-05 2023-09-23T21:19:27.196 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.196 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.196 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:19:27.196 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:12:12.046307+0000", 2023-09-23T21:19:27.196 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.5900999999999999e-05 2023-09-23T21:19:27.196 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.196 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.196 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:19:27.196 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:12:12.046315+0000", 2023-09-23T21:19:27.196 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.9740000000000002e-06 2023-09-23T21:19:27.196 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.196 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.196 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:19:27.197 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:12:14.355294+0000", 2023-09-23T21:19:27.197 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.308979511 2023-09-23T21:19:27.197 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.197 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.197 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:19:27.197 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:12:14.355318+0000", 2023-09-23T21:19:27.197 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.3159999999999998e-05 2023-09-23T21:19:27.197 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.197 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.197 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:19:27.197 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:12:14.363038+0000", 2023-09-23T21:19:27.197 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0077202060000000003 2023-09-23T21:19:27.197 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.198 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.198 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:19:27.198 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:12:14.363074+0000", 2023-09-23T21:19:27.198 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.5750999999999997e-05 2023-09-23T21:19:27.198 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:19:27.198 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:19:27.198 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:19:27.198 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.198 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.198 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.46328.0:99949 10.13 10:c89660e9:::benchmark_data_smithi026_379838_object19992:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e492)", 2023-09-23T21:19:27.198 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:13:11.118271+0000", 2023-09-23T21:19:27.198 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 375.97287569999997, 2023-09-23T21:19:27.199 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.4608746269999999, 2023-09-23T21:19:27.199 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:19:27.199 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "reached pg", 2023-09-23T21:19:27.199 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:19:27.199 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.46328", 2023-09-23T21:19:27.199 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/3748861181", 2023-09-23T21:19:27.199 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 99949 2023-09-23T21:19:27.199 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.199 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:19:27.199 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.199 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:19:27.199 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:11.118271+0000", 2023-09-23T21:19:27.200 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:19:27.200 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.200 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.200 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:19:27.200 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:11.118271+0000", 2023-09-23T21:19:27.200 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:19:27.200 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.200 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.200 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:19:27.200 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:11.118270+0000", 2023-09-23T21:19:27.200 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T21:19:27.200 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.201 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.201 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:19:27.201 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:11.118275+0000", 2023-09-23T21:19:27.201 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.9799999999999998e-06 2023-09-23T21:19:27.201 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.201 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.201 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:19:27.201 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:11.118284+0000", 2023-09-23T21:19:27.201 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.0950000000000002e-06 2023-09-23T21:19:27.201 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.201 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.201 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:19:27.201 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:11.118288+0000", 2023-09-23T21:19:27.202 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.826e-06 2023-09-23T21:19:27.202 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.202 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.202 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:19:27.202 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:11.119293+0000", 2023-09-23T21:19:27.202 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.001004291 2023-09-23T21:19:27.202 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.202 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.202 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for missing object", 2023-09-23T21:19:27.202 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:11.122652+0000", 2023-09-23T21:19:27.202 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0033593080000000001 2023-09-23T21:19:27.203 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.203 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.203 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:19:27.203 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:11.143684+0000", 2023-09-23T21:19:27.203 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.021032241 2023-09-23T21:19:27.203 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.203 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.203 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for rw locks", 2023-09-23T21:19:27.203 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:11.147201+0000", 2023-09-23T21:19:27.203 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0035162399999999999 2023-09-23T21:19:27.203 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.203 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.204 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:19:27.204 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:11.162703+0000", 2023-09-23T21:19:27.204 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.015502653999999999 2023-09-23T21:19:27.204 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.204 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.204 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:19:27.204 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:11.165461+0000", 2023-09-23T21:19:27.204 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.002758 2023-09-23T21:19:27.204 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.204 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.204 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 2,5", 2023-09-23T21:19:27.204 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:11.172337+0000", 2023-09-23T21:19:27.204 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0068755500000000002 2023-09-23T21:19:27.205 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.205 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.205 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:19:27.205 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:11.195056+0000", 2023-09-23T21:19:27.205 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.022719261000000001 2023-09-23T21:19:27.205 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.205 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.205 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:19:27.205 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:11.214701+0000", 2023-09-23T21:19:27.205 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.019645477000000001 2023-09-23T21:19:27.205 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.205 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.206 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:19:27.206 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:11.214712+0000", 2023-09-23T21:19:27.206 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.0464e-05 2023-09-23T21:19:27.206 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.206 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.206 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:19:27.206 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:12.192752+0000", 2023-09-23T21:19:27.206 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.97804052900000005 2023-09-23T21:19:27.206 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.206 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.206 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:19:27.206 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:12.192762+0000", 2023-09-23T21:19:27.207 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.9469999999999998e-06 2023-09-23T21:19:27.207 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.207 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.207 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:19:27.207 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:14.334672+0000", 2023-09-23T21:19:27.207 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.1419095559999999 2023-09-23T21:19:27.207 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.207 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.207 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for missing object", 2023-09-23T21:19:27.207 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:14.338733+0000", 2023-09-23T21:19:27.207 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0040611099999999997 2023-09-23T21:19:27.207 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.208 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.208 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:19:27.208 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:15.576529+0000", 2023-09-23T21:19:27.208 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.237796049 2023-09-23T21:19:27.208 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.208 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.208 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:19:27.208 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:15.579146+0000", 2023-09-23T21:19:27.208 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0026163979999999998 2023-09-23T21:19:27.208 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:19:27.208 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:19:27.208 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:19:27.208 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.209 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.209 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.46328.0:99952 10.1 10:86c6d668:::benchmark_data_smithi026_379838_object19995:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e492)", 2023-09-23T21:19:27.209 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:13:11.126669+0000", 2023-09-23T21:19:27.209 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 375.96447751599999, 2023-09-23T21:19:27.209 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.242467735, 2023-09-23T21:19:27.209 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:19:27.209 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:19:27.209 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:19:27.209 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.46328", 2023-09-23T21:19:27.209 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/3748861181", 2023-09-23T21:19:27.209 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 99952 2023-09-23T21:19:27.209 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.210 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:19:27.210 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.210 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:19:27.210 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:11.126669+0000", 2023-09-23T21:19:27.210 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:19:27.210 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.210 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.210 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:19:27.210 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:11.126669+0000", 2023-09-23T21:19:27.210 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:19:27.210 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.210 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.211 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:19:27.211 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:11.126667+0000", 2023-09-23T21:19:27.211 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999981 2023-09-23T21:19:27.211 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.211 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.211 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:19:27.211 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:11.126674+0000", 2023-09-23T21:19:27.211 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.9650000000000002e-06 2023-09-23T21:19:27.211 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.211 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.211 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:19:27.211 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:11.126691+0000", 2023-09-23T21:19:27.212 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.7359000000000001e-05 2023-09-23T21:19:27.212 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.212 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.212 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:19:27.212 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:11.126698+0000", 2023-09-23T21:19:27.212 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.1029999999999996e-06 2023-09-23T21:19:27.212 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.212 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.212 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:19:27.212 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:11.136504+0000", 2023-09-23T21:19:27.212 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0098059319999999998 2023-09-23T21:19:27.212 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.213 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.213 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for missing object", 2023-09-23T21:19:27.213 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:11.139550+0000", 2023-09-23T21:19:27.213 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0030458830000000001 2023-09-23T21:19:27.213 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.213 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.213 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:19:27.213 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:11.160803+0000", 2023-09-23T21:19:27.213 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.02125318 2023-09-23T21:19:27.213 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.213 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.213 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for rw locks", 2023-09-23T21:19:27.214 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:11.165287+0000", 2023-09-23T21:19:27.214 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0044835869999999998 2023-09-23T21:19:27.214 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.214 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.214 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:19:27.214 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:11.202380+0000", 2023-09-23T21:19:27.214 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.037092768999999998 2023-09-23T21:19:27.214 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.214 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.214 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:19:27.214 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:11.202394+0000", 2023-09-23T21:19:27.215 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.3949e-05 2023-09-23T21:19:27.215 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.215 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.215 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:19:27.215 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:12.193673+0000", 2023-09-23T21:19:27.215 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.99127966099999998 2023-09-23T21:19:27.215 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.215 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.215 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:19:27.215 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:12.193684+0000", 2023-09-23T21:19:27.215 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.0499000000000001e-05 2023-09-23T21:19:27.215 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.216 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.216 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:19:27.216 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:15.335139+0000", 2023-09-23T21:19:27.216 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.1414547779999999 2023-09-23T21:19:27.216 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.216 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.216 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:19:27.216 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:15.338252+0000", 2023-09-23T21:19:27.216 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.003112821 2023-09-23T21:19:27.216 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.216 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.216 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 0,7", 2023-09-23T21:19:27.217 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:15.342990+0000", 2023-09-23T21:19:27.217 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0047385450000000003 2023-09-23T21:19:27.217 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.217 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.217 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:19:27.217 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:15.355654+0000", 2023-09-23T21:19:27.217 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.012663496 2023-09-23T21:19:27.217 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.217 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.217 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:19:27.217 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:15.361946+0000", 2023-09-23T21:19:27.217 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0062922300000000002 2023-09-23T21:19:27.218 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.218 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.218 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:19:27.218 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:15.362731+0000", 2023-09-23T21:19:27.218 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00078492199999999996 2023-09-23T21:19:27.218 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.218 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.218 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:19:27.218 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:15.363991+0000", 2023-09-23T21:19:27.218 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.001260082 2023-09-23T21:19:27.218 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.218 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.219 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:19:27.219 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:15.369137+0000", 2023-09-23T21:19:27.219 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.005146014 2023-09-23T21:19:27.219 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:19:27.219 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:19:27.219 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:19:27.219 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.219 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.219 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.46328.0:99963 10.13 10:c9506fe5:::benchmark_data_smithi026_379838_object20006:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e492)", 2023-09-23T21:19:27.219 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:13:11.156341+0000", 2023-09-23T21:19:27.219 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 375.93480594200003, 2023-09-23T21:19:27.219 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.6561101320000002, 2023-09-23T21:19:27.220 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:19:27.220 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:19:27.220 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:19:27.220 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.46328", 2023-09-23T21:19:27.220 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/3748861181", 2023-09-23T21:19:27.220 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 99963 2023-09-23T21:19:27.220 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.220 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:19:27.220 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.220 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:19:27.220 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:11.156341+0000", 2023-09-23T21:19:27.220 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:19:27.221 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.221 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.221 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:19:27.221 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:11.156341+0000", 2023-09-23T21:19:27.221 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:19:27.221 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.221 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.221 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:19:27.221 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:11.156339+0000", 2023-09-23T21:19:27.221 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999981 2023-09-23T21:19:27.221 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.222 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.222 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:19:27.222 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:11.156347+0000", 2023-09-23T21:19:27.222 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.0439999999999996e-06 2023-09-23T21:19:27.222 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.222 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.222 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:19:27.222 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:11.156363+0000", 2023-09-23T21:19:27.222 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.5798e-05 2023-09-23T21:19:27.222 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.222 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.222 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:19:27.223 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:11.156370+0000", 2023-09-23T21:19:27.223 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.6899999999999992e-06 2023-09-23T21:19:27.223 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.223 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.223 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:19:27.223 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:11.191653+0000", 2023-09-23T21:19:27.223 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.035282273000000003 2023-09-23T21:19:27.223 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.223 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.223 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for missing object", 2023-09-23T21:19:27.223 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:11.194637+0000", 2023-09-23T21:19:27.223 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.002984586 2023-09-23T21:19:27.224 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.224 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.224 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:19:27.224 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:11.214739+0000", 2023-09-23T21:19:27.224 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.020102346 2023-09-23T21:19:27.224 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.224 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.224 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:19:27.224 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:11.214746+0000", 2023-09-23T21:19:27.224 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.9419999999999998e-06 2023-09-23T21:19:27.224 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.224 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.224 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:19:27.225 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:12.192789+0000", 2023-09-23T21:19:27.225 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.97804239400000004 2023-09-23T21:19:27.225 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.225 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.225 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:19:27.225 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:12.192796+0000", 2023-09-23T21:19:27.225 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.6290000000000001e-06 2023-09-23T21:19:27.225 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.225 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.225 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:19:27.225 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:15.579652+0000", 2023-09-23T21:19:27.225 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.3868554130000001 2023-09-23T21:19:27.226 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.226 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.226 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for missing object", 2023-09-23T21:19:27.226 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:15.583515+0000", 2023-09-23T21:19:27.226 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0038636059999999999 2023-09-23T21:19:27.226 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.226 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.226 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:19:27.226 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:15.604488+0000", 2023-09-23T21:19:27.226 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.020972781999999999 2023-09-23T21:19:27.226 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.226 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.227 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for degraded object", 2023-09-23T21:19:27.227 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:15.607187+0000", 2023-09-23T21:19:27.227 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0026986549999999999 2023-09-23T21:19:27.227 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.227 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.227 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:19:27.227 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:15.742812+0000", 2023-09-23T21:19:27.227 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.13562508700000001 2023-09-23T21:19:27.227 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.227 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.227 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:19:27.227 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:15.746122+0000", 2023-09-23T21:19:27.227 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0033102119999999999 2023-09-23T21:19:27.228 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.228 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.228 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 0,5", 2023-09-23T21:19:27.228 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:15.753005+0000", 2023-09-23T21:19:27.228 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0068824690000000004 2023-09-23T21:19:27.228 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.228 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.228 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:19:27.228 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:15.781458+0000", 2023-09-23T21:19:27.228 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.028453556000000001 2023-09-23T21:19:27.228 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.228 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.229 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:19:27.229 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:15.792765+0000", 2023-09-23T21:19:27.229 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.011306800000000001 2023-09-23T21:19:27.229 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.229 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.229 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:19:27.229 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:15.805221+0000", 2023-09-23T21:19:27.229 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.012456211 2023-09-23T21:19:27.229 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.229 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.229 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:19:27.229 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:15.806627+0000", 2023-09-23T21:19:27.230 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0014054130000000001 2023-09-23T21:19:27.230 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.230 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.230 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:19:27.230 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:15.812451+0000", 2023-09-23T21:19:27.230 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00582418 2023-09-23T21:19:27.230 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:19:27.230 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:19:27.230 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:19:27.230 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.230 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.230 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.46328.0:99965 10.16 10:68a95ca6:::benchmark_data_smithi026_379838_object20008:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e492)", 2023-09-23T21:19:27.231 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:13:11.199392+0000", 2023-09-23T21:19:27.231 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 375.89175486699997, 2023-09-23T21:19:27.231 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.2292175890000001, 2023-09-23T21:19:27.231 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:19:27.231 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:19:27.231 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:19:27.231 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.46328", 2023-09-23T21:19:27.231 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/3748861181", 2023-09-23T21:19:27.231 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 99965 2023-09-23T21:19:27.231 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.231 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:19:27.231 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.232 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:19:27.232 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:11.199392+0000", 2023-09-23T21:19:27.232 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:19:27.232 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.232 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.232 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:19:27.232 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:11.199392+0000", 2023-09-23T21:19:27.232 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:19:27.232 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.232 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.232 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:19:27.232 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:11.199390+0000", 2023-09-23T21:19:27.232 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T21:19:27.233 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.233 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.233 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:19:27.233 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:11.199396+0000", 2023-09-23T21:19:27.233 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.1240000000000002e-06 2023-09-23T21:19:27.233 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.233 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.233 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:19:27.233 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:11.199407+0000", 2023-09-23T21:19:27.233 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.0682e-05 2023-09-23T21:19:27.233 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.233 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.234 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:19:27.234 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:11.199413+0000", 2023-09-23T21:19:27.234 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.2570000000000004e-06 2023-09-23T21:19:27.234 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.234 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.234 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:19:27.234 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:11.232630+0000", 2023-09-23T21:19:27.234 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.033216351999999998 2023-09-23T21:19:27.234 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.234 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.234 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:19:27.234 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:11.232640+0000", 2023-09-23T21:19:27.235 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.0414999999999999e-05 2023-09-23T21:19:27.235 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.235 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.235 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:19:27.235 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:12.191472+0000", 2023-09-23T21:19:27.235 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.95883214699999997 2023-09-23T21:19:27.235 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.235 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.235 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:19:27.235 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:12.191481+0000", 2023-09-23T21:19:27.235 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.6649999999999992e-06 2023-09-23T21:19:27.235 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.235 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.236 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:19:27.236 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:14.423279+0000", 2023-09-23T21:19:27.236 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.2317976210000001 2023-09-23T21:19:27.236 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.236 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.236 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:19:27.236 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:14.423386+0000", 2023-09-23T21:19:27.236 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00010726399999999999 2023-09-23T21:19:27.236 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.236 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.236 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 0,5", 2023-09-23T21:19:27.236 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:14.423617+0000", 2023-09-23T21:19:27.237 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000231597 2023-09-23T21:19:27.237 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.237 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.237 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:19:27.237 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:14.425276+0000", 2023-09-23T21:19:27.237 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.001658299 2023-09-23T21:19:27.237 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.237 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.237 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:19:27.237 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:14.425806+0000", 2023-09-23T21:19:27.237 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00053022900000000001 2023-09-23T21:19:27.238 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.238 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.238 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:19:27.238 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:14.428365+0000", 2023-09-23T21:19:27.238 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.002559253 2023-09-23T21:19:27.238 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.238 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.238 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:19:27.238 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:14.428423+0000", 2023-09-23T21:19:27.238 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.7596999999999998e-05 2023-09-23T21:19:27.238 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.238 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.239 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:19:27.239 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:14.428609+0000", 2023-09-23T21:19:27.239 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00018645399999999999 2023-09-23T21:19:27.239 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:19:27.239 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:19:27.239 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:19:27.239 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.239 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.239 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.46328.0:99979 10.1 10:8711b949:::benchmark_data_smithi026_379838_object20022:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e493)", 2023-09-23T21:19:27.239 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:13:11.860281+0000", 2023-09-23T21:19:27.239 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 375.23086511600002, 2023-09-23T21:19:27.240 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.5904489540000002, 2023-09-23T21:19:27.240 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:19:27.240 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:19:27.240 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:19:27.240 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.46328", 2023-09-23T21:19:27.240 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/3748861181", 2023-09-23T21:19:27.240 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 99979 2023-09-23T21:19:27.240 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.240 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:19:27.240 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.240 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:19:27.240 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:11.860281+0000", 2023-09-23T21:19:27.241 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:19:27.241 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.241 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.241 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:19:27.241 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:11.860281+0000", 2023-09-23T21:19:27.241 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:19:27.241 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.241 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.241 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:19:27.241 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:11.860280+0000", 2023-09-23T21:19:27.241 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T21:19:27.241 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.241 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.242 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:19:27.242 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:11.860286+0000", 2023-09-23T21:19:27.242 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.2720000000000003e-06 2023-09-23T21:19:27.242 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.242 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.242 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:19:27.242 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:11.860301+0000", 2023-09-23T21:19:27.242 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.4901e-05 2023-09-23T21:19:27.242 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.242 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.242 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:19:27.242 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:11.860308+0000", 2023-09-23T21:19:27.243 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.3779999999999998e-06 2023-09-23T21:19:27.243 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.243 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.243 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:19:27.243 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:11.860364+0000", 2023-09-23T21:19:27.243 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.5866999999999999e-05 2023-09-23T21:19:27.243 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.243 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.243 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:19:27.243 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:11.860373+0000", 2023-09-23T21:19:27.243 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.8500000000000006e-06 2023-09-23T21:19:27.244 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.244 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.244 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:19:27.244 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:12.196314+0000", 2023-09-23T21:19:27.244 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.33594100999999998 2023-09-23T21:19:27.244 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.244 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.244 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:19:27.244 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:12.196326+0000", 2023-09-23T21:19:27.244 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.1333e-05 2023-09-23T21:19:27.244 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.244 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.244 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:19:27.245 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:15.369429+0000", 2023-09-23T21:19:27.245 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.1731031180000002 2023-09-23T21:19:27.245 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.245 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.245 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for missing object", 2023-09-23T21:19:27.245 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:15.372799+0000", 2023-09-23T21:19:27.245 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0033701310000000002 2023-09-23T21:19:27.245 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.245 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.245 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:19:27.245 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:15.389753+0000", 2023-09-23T21:19:27.245 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.016954440000000001 2023-09-23T21:19:27.246 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.246 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.246 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for rw locks", 2023-09-23T21:19:27.246 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:15.392893+0000", 2023-09-23T21:19:27.246 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.003139293 2023-09-23T21:19:27.246 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.246 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.246 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:19:27.246 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:15.415439+0000", 2023-09-23T21:19:27.246 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.022546297999999999 2023-09-23T21:19:27.246 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.246 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.247 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:19:27.247 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:15.417698+0000", 2023-09-23T21:19:27.247 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0022593629999999999 2023-09-23T21:19:27.247 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.247 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.247 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 0,7", 2023-09-23T21:19:27.247 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:15.422787+0000", 2023-09-23T21:19:27.247 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0050888920000000002 2023-09-23T21:19:27.247 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.247 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.247 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:19:27.247 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:15.442882+0000", 2023-09-23T21:19:27.248 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.020094441000000001 2023-09-23T21:19:27.248 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.248 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.248 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:19:27.248 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:15.443740+0000", 2023-09-23T21:19:27.248 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00085842100000000005 2023-09-23T21:19:27.248 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.248 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.248 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:19:27.248 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:15.444499+0000", 2023-09-23T21:19:27.248 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00075862800000000001 2023-09-23T21:19:27.248 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.248 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.249 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:19:27.249 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:15.445752+0000", 2023-09-23T21:19:27.249 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0012533970000000001 2023-09-23T21:19:27.249 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.249 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.249 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:19:27.249 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:15.450730+0000", 2023-09-23T21:19:27.249 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0049782749999999999 2023-09-23T21:19:27.249 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:19:27.249 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:19:27.249 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:19:27.249 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.250 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.250 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.46328.0:99984 10.13 10:c97226d3:::benchmark_data_smithi026_379838_object20027:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e493)", 2023-09-23T21:19:27.250 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:13:11.898920+0000", 2023-09-23T21:19:27.250 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 375.192226667, 2023-09-23T21:19:27.250 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.923359048, 2023-09-23T21:19:27.250 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:19:27.250 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:19:27.250 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:19:27.250 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.46328", 2023-09-23T21:19:27.250 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/3748861181", 2023-09-23T21:19:27.250 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 99984 2023-09-23T21:19:27.250 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.251 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:19:27.251 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.251 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:19:27.251 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:11.898920+0000", 2023-09-23T21:19:27.251 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:19:27.251 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.251 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.251 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:19:27.251 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:11.898920+0000", 2023-09-23T21:19:27.251 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:19:27.251 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.251 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.252 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:19:27.252 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:11.898919+0000", 2023-09-23T21:19:27.252 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.999999 2023-09-23T21:19:27.252 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.252 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.252 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:19:27.252 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:11.898923+0000", 2023-09-23T21:19:27.252 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.4410000000000003e-06 2023-09-23T21:19:27.252 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.252 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.252 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:19:27.252 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:11.898939+0000", 2023-09-23T21:19:27.253 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.5588000000000001e-05 2023-09-23T21:19:27.253 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.253 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.253 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:19:27.253 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:11.898947+0000", 2023-09-23T21:19:27.253 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.7950000000000008e-06 2023-09-23T21:19:27.253 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.253 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.253 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:19:27.253 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:11.899000+0000", 2023-09-23T21:19:27.253 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.2905999999999998e-05 2023-09-23T21:19:27.253 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.254 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.254 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:19:27.254 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:11.899012+0000", 2023-09-23T21:19:27.254 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.2668e-05 2023-09-23T21:19:27.254 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.254 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.254 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:19:27.254 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:12.192819+0000", 2023-09-23T21:19:27.254 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.29380640800000002 2023-09-23T21:19:27.254 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.254 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.254 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:19:27.255 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:12.192826+0000", 2023-09-23T21:19:27.255 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.1439999999999997e-06 2023-09-23T21:19:27.255 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.255 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.255 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:19:27.255 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:15.584066+0000", 2023-09-23T21:19:27.255 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.3912398499999998 2023-09-23T21:19:27.255 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.255 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.255 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for missing object", 2023-09-23T21:19:27.255 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:15.587625+0000", 2023-09-23T21:19:27.255 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0035591070000000002 2023-09-23T21:19:27.255 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.256 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.256 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:19:27.256 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:15.621463+0000", 2023-09-23T21:19:27.256 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.033837837000000003 2023-09-23T21:19:27.256 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.256 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.256 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for degraded object", 2023-09-23T21:19:27.256 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:15.624790+0000", 2023-09-23T21:19:27.256 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0033272829999999999 2023-09-23T21:19:27.256 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.256 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.256 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:19:27.257 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:15.769569+0000", 2023-09-23T21:19:27.257 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.14477948299999999 2023-09-23T21:19:27.257 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.257 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.257 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:19:27.257 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:15.772068+0000", 2023-09-23T21:19:27.257 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0024988359999999999 2023-09-23T21:19:27.257 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.257 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.257 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 0,5", 2023-09-23T21:19:27.257 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:15.777050+0000", 2023-09-23T21:19:27.258 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0049818029999999999 2023-09-23T21:19:27.258 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.258 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.258 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:19:27.258 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:15.812721+0000", 2023-09-23T21:19:27.258 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.035671139999999997 2023-09-23T21:19:27.258 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.258 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.258 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:19:27.258 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:15.813568+0000", 2023-09-23T21:19:27.258 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00084671699999999998 2023-09-23T21:19:27.258 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.259 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.259 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:19:27.259 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:15.814693+0000", 2023-09-23T21:19:27.259 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0011254850000000001 2023-09-23T21:19:27.259 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.259 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.259 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:19:27.259 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:15.815984+0000", 2023-09-23T21:19:27.259 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.001290329 2023-09-23T21:19:27.259 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.259 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.259 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:19:27.260 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:15.822279+0000", 2023-09-23T21:19:27.260 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0062953109999999996 2023-09-23T21:19:27.260 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:19:27.260 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:19:27.260 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:19:27.260 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.260 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.260 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.46328.0:99987 10.13 10:c9fb87f0:::benchmark_data_smithi026_379838_object20030:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e493)", 2023-09-23T21:19:27.260 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:13:11.949115+0000", 2023-09-23T21:19:27.260 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 375.142031949, 2023-09-23T21:19:27.260 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.882972664, 2023-09-23T21:19:27.260 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:19:27.261 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:19:27.261 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:19:27.261 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.46328", 2023-09-23T21:19:27.261 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/3748861181", 2023-09-23T21:19:27.261 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 99987 2023-09-23T21:19:27.261 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.261 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:19:27.261 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.261 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:19:27.261 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:11.949115+0000", 2023-09-23T21:19:27.261 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:19:27.261 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.262 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.262 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:19:27.262 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:11.949115+0000", 2023-09-23T21:19:27.262 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:19:27.262 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.262 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.262 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:19:27.262 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:11.949113+0000", 2023-09-23T21:19:27.262 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.999999 2023-09-23T21:19:27.262 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.262 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.262 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:19:27.262 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:11.949119+0000", 2023-09-23T21:19:27.263 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.4190000000000001e-06 2023-09-23T21:19:27.263 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.263 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.263 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:19:27.263 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:11.949132+0000", 2023-09-23T21:19:27.263 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.2986000000000001e-05 2023-09-23T21:19:27.263 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.263 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.263 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:19:27.263 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:11.949137+0000", 2023-09-23T21:19:27.263 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.6019999999999999e-06 2023-09-23T21:19:27.264 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.264 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.264 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:19:27.264 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:11.949210+0000", 2023-09-23T21:19:27.264 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.2071999999999997e-05 2023-09-23T21:19:27.264 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.264 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.264 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:19:27.264 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:11.949219+0000", 2023-09-23T21:19:27.264 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.0410000000000003e-06 2023-09-23T21:19:27.264 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.264 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.265 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:19:27.265 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:12.192846+0000", 2023-09-23T21:19:27.265 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.243627597 2023-09-23T21:19:27.265 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.265 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.265 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:19:27.265 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:12.192853+0000", 2023-09-23T21:19:27.265 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.0169999999999999e-06 2023-09-23T21:19:27.265 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.265 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.265 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:19:27.265 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:15.588023+0000", 2023-09-23T21:19:27.265 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.3951699780000002 2023-09-23T21:19:27.266 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.266 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.266 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for missing object", 2023-09-23T21:19:27.266 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:15.591088+0000", 2023-09-23T21:19:27.266 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0030649900000000001 2023-09-23T21:19:27.266 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.266 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.266 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:19:27.266 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:15.640214+0000", 2023-09-23T21:19:27.266 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.049126213000000002 2023-09-23T21:19:27.266 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.266 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.267 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for degraded object", 2023-09-23T21:19:27.267 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:15.642230+0000", 2023-09-23T21:19:27.267 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0020155440000000002 2023-09-23T21:19:27.267 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.267 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.267 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:19:27.267 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:15.793072+0000", 2023-09-23T21:19:27.267 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.15084209300000001 2023-09-23T21:19:27.267 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.267 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.267 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:19:27.267 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:15.795467+0000", 2023-09-23T21:19:27.268 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.002394752 2023-09-23T21:19:27.268 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.268 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.268 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 0,5", 2023-09-23T21:19:27.268 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:15.800354+0000", 2023-09-23T21:19:27.268 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0048867590000000001 2023-09-23T21:19:27.268 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.268 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.268 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:19:27.268 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:15.813842+0000", 2023-09-23T21:19:27.268 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.013488557999999999 2023-09-23T21:19:27.268 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.268 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.269 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:19:27.269 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:15.823275+0000", 2023-09-23T21:19:27.269 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0094329649999999998 2023-09-23T21:19:27.269 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.269 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.269 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:19:27.269 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:15.824202+0000", 2023-09-23T21:19:27.269 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00092731499999999995 2023-09-23T21:19:27.269 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.269 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.269 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:19:27.269 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:15.825761+0000", 2023-09-23T21:19:27.270 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0015584139999999999 2023-09-23T21:19:27.270 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:27.270 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:27.270 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:19:27.270 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:15.832087+0000", 2023-09-23T21:19:27.270 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0063265379999999996 2023-09-23T21:19:27.270 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:19:27.270 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:19:27.270 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:19:27.270 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:19:27.270 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:19:27.270 INFO:teuthology.orchestra.run.smithi070.stdout:} 2023-09-23T21:19:27.271 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:19:27.296 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:19:27.300 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:27.530 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_ops_in_flight 2023-09-23T21:19:27.532 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:19:27.534 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:27.535 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:19:27.610 INFO:teuthology.orchestra.run.smithi026.stderr:nodeep-scrub is set 2023-09-23T21:19:27.635 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:19:27.662 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:27.694 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:19:27.696 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:27.697 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:19:27.765 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:19:27 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/4068016900' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:19:27.765 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:19:27 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/4068016900' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:19:27.798 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:19:27.822 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:19:27 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/4068016900' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:19:27.825 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:27.858 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:19:27.860 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:27.861 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:19:27.961 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:19:27.994 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:28.131 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T21:19:28.198 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:19:28.202 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:28.203 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:19:28.303 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:19:28.332 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:28.526 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T21:19:28.530 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:28.530 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:19:28.631 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:19:28.663 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:28.702 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T21:19:28.705 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:28.706 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:19:28.785 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0: {} 2023-09-23T21:19:28.804 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1: {} 2023-09-23T21:19:28.807 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:19:28.829 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2: {} 2023-09-23T21:19:28.836 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:19:28 smithi026 ceph-mon[179688]: pgmap v1597: 103 pgs: 103 active+clean; 582 KiB data, 4.3 GiB used, 532 GiB / 536 GiB avail 2023-09-23T21:19:28.837 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:19:28 smithi026 ceph-mon[179688]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:19:28.837 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:19:28 smithi026 ceph-mon[179688]: failed to encode map e649 with expected crc 2023-09-23T21:19:28.837 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:19:28 smithi026 ceph-mon[179688]: failed to encode map e649 with expected crc 2023-09-23T21:19:28.837 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:19:28 smithi026 ceph-mon[179688]: failed to encode map e649 with expected crc 2023-09-23T21:19:28.837 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:19:28 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/4068016900' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T21:19:28.837 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:19:28 smithi026 ceph-mon[179688]: osdmap e649: 8 total, 8 up, 6 in 2023-09-23T21:19:28.837 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:19:28 smithi026 ceph-mon[175151]: pgmap v1597: 103 pgs: 103 active+clean; 582 KiB data, 4.3 GiB used, 532 GiB / 536 GiB avail 2023-09-23T21:19:28.837 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:19:28 smithi026 ceph-mon[175151]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:19:28.838 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:19:28 smithi026 ceph-mon[175151]: failed to encode map e649 with expected crc 2023-09-23T21:19:28.838 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:19:28 smithi026 ceph-mon[175151]: failed to encode map e649 with expected crc 2023-09-23T21:19:28.838 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:19:28 smithi026 ceph-mon[175151]: failed to encode map e649 with expected crc 2023-09-23T21:19:28.838 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:19:28 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/4068016900' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T21:19:28.838 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:19:28 smithi026 ceph-mon[175151]: osdmap e649: 8 total, 8 up, 6 in 2023-09-23T21:19:28.839 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:28.867 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3: {} 2023-09-23T21:19:28.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:19:28 smithi070 ceph-mon[168282]: pgmap v1597: 103 pgs: 103 active+clean; 582 KiB data, 4.3 GiB used, 532 GiB / 536 GiB avail 2023-09-23T21:19:28.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:19:28 smithi070 ceph-mon[168282]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:19:28.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:19:28 smithi070 ceph-mon[168282]: failed to encode map e649 with expected crc 2023-09-23T21:19:28.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:19:28 smithi070 ceph-mon[168282]: failed to encode map e649 with expected crc 2023-09-23T21:19:28.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:19:28 smithi070 ceph-mon[168282]: failed to encode map e649 with expected crc 2023-09-23T21:19:28.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:19:28 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/4068016900' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T21:19:28.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:19:28 smithi070 ceph-mon[168282]: osdmap e649: 8 total, 8 up, 6 in 2023-09-23T21:19:28.887 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4: {} 2023-09-23T21:19:28.908 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T21:19:28.929 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:19:28.932 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:28.933 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:19:28.933 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6: {} 2023-09-23T21:19:28.954 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T21:19:28.956 INFO:teuthology.orchestra.run.smithi026.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T21:19:28.956 INFO:teuthology.orchestra.run.smithi026.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T21:19:28.956 INFO:teuthology.orchestra.run.smithi026.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T21:19:28.956 INFO:teuthology.orchestra.run.smithi026.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T21:19:28.956 INFO:teuthology.orchestra.run.smithi026.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T21:19:28.956 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T21:19:28.956 INFO:teuthology.orchestra.run.smithi026.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T21:19:28.957 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T21:19:29.034 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:19:29.064 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:29.152 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:19:29.154 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:29.155 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:19:29.255 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:19:29.284 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:29.318 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:19:29.321 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:29.321 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:19:29.346 INFO:teuthology.orchestra.run.smithi070.stdout:ERROR: (22) Invalid argument 2023-09-23T21:19:29.346 INFO:teuthology.orchestra.run.smithi070.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T21:19:29.381 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:19:29 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mon-a[175127]: 2023-09-23T21:19:29.318+0000 7f1e2d487700 -1 mon.a@0(leader).osd e649 definitely_dead 0 2023-09-23T21:19:29.423 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:19:29.451 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:29.489 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:19:29.492 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:29.492 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:19:29.593 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:19:29.633 INFO:teuthology.orchestra.run.smithi026.stderr:marked out osd.5. 2023-09-23T21:19:29.635 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:29.672 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:19:29.674 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:29.675 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:19:29.772 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:19:29 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/393793602' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:19:29.772 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:19:29 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/2811652418' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["5"]}]: dispatch 2023-09-23T21:19:29.772 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:19:29 smithi070 ceph-mon[168282]: Client client.admin marked osd.5 out, while it was still marked up 2023-09-23T21:19:29.773 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_blocked_ops 2023-09-23T21:19:29.776 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:19:29.810 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:19:29 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/393793602' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:19:29.810 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:19:29 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/2811652418' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["5"]}]: dispatch 2023-09-23T21:19:29.810 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:19:29 smithi026 ceph-mon[179688]: Client client.admin marked osd.5 out, while it was still marked up 2023-09-23T21:19:29.810 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:29.812 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:19:29 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/393793602' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:19:29.812 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:19:29 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/2811652418' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["5"]}]: dispatch 2023-09-23T21:19:29.812 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:19:29 smithi026 ceph-mon[175151]: Client client.admin marked osd.5 out, while it was still marked up 2023-09-23T21:19:29.813 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 21:19:29 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[164029]: ::ffff:172.21.15.70 - - [23/Sep/2023:21:19:29] "GET /metrics HTTP/1.1" 200 33950 "" "Prometheus/2.43.0" 2023-09-23T21:19:29.855 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:19:29.857 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:29.858 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:19:29.959 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:19:29.986 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:30.021 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:19:30.024 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:30.025 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:19:30.126 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:19:30.153 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:30.185 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:19:30.188 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:30.188 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:19:30.258 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T21:19:30.289 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:19:30.332 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:30.374 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:19:30.377 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:30.378 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:19:30.410 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T21:19:30.479 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:19:30.505 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:30.537 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:19:30.539 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:30.540 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:19:30.634 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:19:30 smithi026 ceph-mon[175151]: pgmap v1599: 103 pgs: 103 active+clean; 582 KiB data, 4.3 GiB used, 532 GiB / 536 GiB avail 2023-09-23T21:19:30.634 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:19:30 smithi026 ceph-mon[175151]: failed to encode map e650 with expected crc 2023-09-23T21:19:30.634 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:19:30 smithi026 ceph-mon[175151]: failed to encode map e650 with expected crc 2023-09-23T21:19:30.634 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:19:30 smithi026 ceph-mon[175151]: failed to encode map e650 with expected crc 2023-09-23T21:19:30.635 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:19:30 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/2811652418' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["5"]}]': finished 2023-09-23T21:19:30.635 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:19:30 smithi026 ceph-mon[175151]: osdmap e650: 8 total, 8 up, 5 in 2023-09-23T21:19:30.635 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:19:30 smithi026 ceph-mon[179688]: pgmap v1599: 103 pgs: 103 active+clean; 582 KiB data, 4.3 GiB used, 532 GiB / 536 GiB avail 2023-09-23T21:19:30.635 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:19:30 smithi026 ceph-mon[179688]: failed to encode map e650 with expected crc 2023-09-23T21:19:30.635 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:19:30 smithi026 ceph-mon[179688]: failed to encode map e650 with expected crc 2023-09-23T21:19:30.635 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:19:30 smithi026 ceph-mon[179688]: failed to encode map e650 with expected crc 2023-09-23T21:19:30.635 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:19:30 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/2811652418' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["5"]}]': finished 2023-09-23T21:19:30.635 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:19:30 smithi026 ceph-mon[179688]: osdmap e650: 8 total, 8 up, 5 in 2023-09-23T21:19:30.641 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:19:30.677 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:30.691 DEBUG:teuthology.orchestra.run.smithi026:> cd -- /home/ubuntu/cephtest/clone.client.0/qa/workunits && if test -e Makefile ; then make ; fi && find -executable -type f -printf '%P\0' >/home/ubuntu/cephtest/workunits.list.client.0 2023-09-23T21:19:30.711 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T21:19:30.713 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:30.714 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:19:30.727 INFO:tasks.workunit.client.0.smithi026.stdout:for d in direct_io fs ; do ( cd $d ; make all ) ; done 2023-09-23T21:19:30.729 INFO:tasks.workunit.client.0.smithi026.stdout:make[1]: Entering directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/direct_io' 2023-09-23T21:19:30.729 INFO:tasks.workunit.client.0.smithi026.stdout:cc -Wall -Wextra -D_GNU_SOURCE direct_io_test.c -o direct_io_test 2023-09-23T21:19:30.779 INFO:tasks.workunit.client.0.smithi026.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_sync_io.c -o test_sync_io 2023-09-23T21:19:30.814 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:19:30.832 INFO:tasks.workunit.client.0.smithi026.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_short_dio_read.c -o test_short_dio_read 2023-09-23T21:19:30.833 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:19:30 smithi070 ceph-mon[168282]: pgmap v1599: 103 pgs: 103 active+clean; 582 KiB data, 4.3 GiB used, 532 GiB / 536 GiB avail 2023-09-23T21:19:30.834 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:19:30 smithi070 ceph-mon[168282]: failed to encode map e650 with expected crc 2023-09-23T21:19:30.834 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:19:30 smithi070 ceph-mon[168282]: failed to encode map e650 with expected crc 2023-09-23T21:19:30.834 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:19:30 smithi070 ceph-mon[168282]: failed to encode map e650 with expected crc 2023-09-23T21:19:30.834 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:19:30 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/2811652418' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["5"]}]': finished 2023-09-23T21:19:30.834 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:19:30 smithi070 ceph-mon[168282]: osdmap e650: 8 total, 8 up, 5 in 2023-09-23T21:19:30.841 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:30.874 INFO:tasks.workunit.client.0.smithi026.stdout:make[1]: Leaving directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/direct_io' 2023-09-23T21:19:30.875 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:19:30.875 INFO:tasks.workunit.client.0.smithi026.stdout:make[1]: Entering directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/fs' 2023-09-23T21:19:30.876 INFO:tasks.workunit.client.0.smithi026.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_o_trunc.c -o test_o_trunc 2023-09-23T21:19:30.878 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:30.878 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:19:30.907 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:19:30.969 INFO:tasks.workunit.client.0.smithi026.stdout:make[1]: Leaving directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/fs' 2023-09-23T21:19:30.972 DEBUG:teuthology.orchestra.run.smithi026:> set -ex 2023-09-23T21:19:30.972 DEBUG:teuthology.orchestra.run.smithi026:> dd if=/home/ubuntu/cephtest/workunits.list.client.0 of=/dev/stdout 2023-09-23T21:19:30.980 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:19:31.007 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:31.030 INFO:tasks.workunit:Running workunits matching rbd/import_export.sh on client.0... 2023-09-23T21:19:31.031 INFO:tasks.workunit:Running workunit rbd/import_export.sh... 2023-09-23T21:19:31.031 DEBUG:teuthology.orchestra.run.smithi026:workunit test rbd/import_export.sh> mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=quincy TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 RBD_CREATE_ARGS=--new-format adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/rbd/import_export.sh 2023-09-23T21:19:31.091 INFO:tasks.workunit.client.0.smithi026.stderr:+ export RBD_FORCE_ALLOW_V1=1 2023-09-23T21:19:31.092 INFO:tasks.workunit.client.0.smithi026.stderr:+ RBD_FORCE_ALLOW_V1=1 2023-09-23T21:19:31.092 INFO:tasks.workunit.client.0.smithi026.stderr:+ TMPDIR=/tmp/rbd_import_export_456745 2023-09-23T21:19:31.092 INFO:tasks.workunit.client.0.smithi026.stderr:+ rm -rf /tmp/rbd_import_export_456745 2023-09-23T21:19:31.092 INFO:tasks.workunit.client.0.smithi026.stderr:+ mkdir /tmp/rbd_import_export_456745 2023-09-23T21:19:31.093 INFO:tasks.workunit.client.0.smithi026.stderr:+ trap 'rm -rf /tmp/rbd_import_export_456745' INT TERM EXIT 2023-09-23T21:19:31.093 INFO:tasks.workunit.client.0.smithi026.stderr:+ mkdir foo.456745 2023-09-23T21:19:31.094 INFO:tasks.workunit.client.0.smithi026.stderr:+ rbd import foo.456745 foo.dir 2023-09-23T21:19:31.170 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:19:31.173 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:31.173 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:19:31.274 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:19:31.303 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:31.312 INFO:tasks.workunit.client.0.smithi026.stderr:rbd: cannot import a directory 2023-09-23T21:19:31.312 INFO:tasks.workunit.client.0.smithi026.stderr: Importing image: 0% complete...failed. 2023-09-23T21:19:31.312 INFO:tasks.workunit.client.0.smithi026.stderr:rbd: import failed: (21) Is a directory 2023-09-23T21:19:31.315 INFO:tasks.workunit.client.0.smithi026.stderr:+ true 2023-09-23T21:19:31.315 INFO:tasks.workunit.client.0.smithi026.stderr:+ rmdir foo.456745 2023-09-23T21:19:31.316 INFO:tasks.workunit.client.0.smithi026.stderr:+ dd if=/bin/sh of=/tmp/rbd_import_export_456745/img bs=1k count=1 seek=10 2023-09-23T21:19:31.317 INFO:tasks.workunit.client.0.smithi026.stderr:1+0 records in 2023-09-23T21:19:31.317 INFO:tasks.workunit.client.0.smithi026.stderr:1+0 records out 2023-09-23T21:19:31.317 INFO:tasks.workunit.client.0.smithi026.stderr:1024 bytes (1.0 kB, 1.0 KiB) copied, 5.3389e-05 s, 19.2 MB/s 2023-09-23T21:19:31.317 INFO:tasks.workunit.client.0.smithi026.stderr:+ dd if=/bin/dd of=/tmp/rbd_import_export_456745/img bs=1k count=10 seek=100 2023-09-23T21:19:31.318 INFO:tasks.workunit.client.0.smithi026.stderr:10+0 records in 2023-09-23T21:19:31.318 INFO:tasks.workunit.client.0.smithi026.stderr:10+0 records out 2023-09-23T21:19:31.318 INFO:tasks.workunit.client.0.smithi026.stderr:10240 bytes (10 kB, 10 KiB) copied, 5.6684e-05 s, 181 MB/s 2023-09-23T21:19:31.318 INFO:tasks.workunit.client.0.smithi026.stderr:+ dd if=/bin/rm of=/tmp/rbd_import_export_456745/img bs=1k count=100 seek=1000 2023-09-23T21:19:31.319 INFO:tasks.workunit.client.0.smithi026.stderr:70+1 records in 2023-09-23T21:19:31.319 INFO:tasks.workunit.client.0.smithi026.stderr:70+1 records out 2023-09-23T21:19:31.319 INFO:tasks.workunit.client.0.smithi026.stderr:72048 bytes (72 kB, 70 KiB) copied, 0.000201385 s, 358 MB/s 2023-09-23T21:19:31.319 INFO:tasks.workunit.client.0.smithi026.stderr:+ dd if=/bin/ls of=/tmp/rbd_import_export_456745/img bs=1k seek=10000 2023-09-23T21:19:31.320 INFO:tasks.workunit.client.0.smithi026.stderr:139+1 records in 2023-09-23T21:19:31.320 INFO:tasks.workunit.client.0.smithi026.stderr:139+1 records out 2023-09-23T21:19:31.320 INFO:tasks.workunit.client.0.smithi026.stderr:143224 bytes (143 kB, 140 KiB) copied, 0.000375323 s, 382 MB/s 2023-09-23T21:19:31.321 INFO:tasks.workunit.client.0.smithi026.stderr:+ dd if=/bin/ln of=/tmp/rbd_import_export_456745/img bs=1k seek=100000 2023-09-23T21:19:31.336 INFO:tasks.workunit.client.0.smithi026.stderr:70+1 records in 2023-09-23T21:19:31.337 INFO:tasks.workunit.client.0.smithi026.stderr:70+1 records out 2023-09-23T21:19:31.337 INFO:tasks.workunit.client.0.smithi026.stderr:72168 bytes (72 kB, 70 KiB) copied, 0.0156204 s, 4.6 MB/s 2023-09-23T21:19:31.337 INFO:tasks.workunit.client.0.smithi026.stderr:+ dd if=/bin/grep of=/tmp/rbd_import_export_456745/img bs=1k seek=1000000 2023-09-23T21:19:31.338 INFO:tasks.workunit.client.0.smithi026.stderr:189+1 records in 2023-09-23T21:19:31.338 INFO:tasks.workunit.client.0.smithi026.stderr:189+1 records out 2023-09-23T21:19:31.338 INFO:tasks.workunit.client.0.smithi026.stderr:193768 bytes (194 kB, 189 KiB) copied, 0.000555947 s, 349 MB/s 2023-09-23T21:19:31.338 INFO:tasks.workunit.client.0.smithi026.stderr:+ rbd rm testimg 2023-09-23T21:19:31.339 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:19:31.342 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:31.342 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:19:31.347 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T21:19:31.374 INFO:tasks.workunit.client.0.smithi026.stderr: Removing image: 0% complete...failed. 2023-09-23T21:19:31.374 INFO:tasks.workunit.client.0.smithi026.stderr:rbd: delete error: (2) No such file or directory 2023-09-23T21:19:31.377 INFO:tasks.workunit.client.0.smithi026.stderr:+ true 2023-09-23T21:19:31.377 INFO:tasks.workunit.client.0.smithi026.stderr:+ rbd import --new-format /tmp/rbd_import_export_456745/img testimg 2023-09-23T21:19:31.443 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:19:31.471 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:31.508 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T21:19:31.511 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:31.511 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:19:31.612 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:19:31.633 INFO:teuthology.orchestra.run.smithi070.stdout:ERROR: (22) Invalid argument 2023-09-23T21:19:31.633 INFO:teuthology.orchestra.run.smithi070.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T21:19:31.650 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:31.734 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:19:31.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:19:31 smithi026 ceph-mon[175151]: failed to encode map e651 with expected crc 2023-09-23T21:19:31.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:19:31 smithi026 ceph-mon[175151]: failed to encode map e651 with expected crc 2023-09-23T21:19:31.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:19:31 smithi026 ceph-mon[175151]: osdmap e651: 8 total, 8 up, 5 in 2023-09-23T21:19:31.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:19:31 smithi026 ceph-mon[175151]: failed to encode map e651 with expected crc 2023-09-23T21:19:31.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:19:31 smithi026 ceph-mon[175151]: failed to encode map e651 with expected crc 2023-09-23T21:19:31.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:19:31 smithi026 ceph-mon[175151]: failed to encode map e651 with expected crc 2023-09-23T21:19:31.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:19:31 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:19:31.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:19:31 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:19:31.737 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:19:31 smithi026 ceph-mon[179688]: failed to encode map e651 with expected crc 2023-09-23T21:19:31.737 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:19:31 smithi026 ceph-mon[179688]: failed to encode map e651 with expected crc 2023-09-23T21:19:31.737 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:19:31 smithi026 ceph-mon[179688]: osdmap e651: 8 total, 8 up, 5 in 2023-09-23T21:19:31.737 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:19:31 smithi026 ceph-mon[179688]: failed to encode map e651 with expected crc 2023-09-23T21:19:31.737 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:19:31 smithi026 ceph-mon[179688]: failed to encode map e651 with expected crc 2023-09-23T21:19:31.737 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:19:31 smithi026 ceph-mon[179688]: failed to encode map e651 with expected crc 2023-09-23T21:19:31.737 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:19:31 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:19:31.737 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:19:31 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:19:31.738 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:31.738 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:19:31.839 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:19:31.866 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:19:31 smithi070 ceph-mon[168282]: failed to encode map e651 with expected crc 2023-09-23T21:19:31.866 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:19:31 smithi070 ceph-mon[168282]: failed to encode map e651 with expected crc 2023-09-23T21:19:31.866 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:19:31 smithi070 ceph-mon[168282]: osdmap e651: 8 total, 8 up, 5 in 2023-09-23T21:19:31.866 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:19:31 smithi070 ceph-mon[168282]: failed to encode map e651 with expected crc 2023-09-23T21:19:31.866 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:19:31 smithi070 ceph-mon[168282]: failed to encode map e651 with expected crc 2023-09-23T21:19:31.866 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:19:31 smithi070 ceph-mon[168282]: failed to encode map e651 with expected crc 2023-09-23T21:19:31.867 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:19:31 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:19:31.867 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:19:31 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:19:31.867 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:31.984 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:19:31.987 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:31.987 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:19:32.057 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_historic_ops 2023-09-23T21:19:32.089 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:19:32.119 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:32.204 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:19:32.207 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:32.207 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:19:32.267 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:19:32.308 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:19:32.336 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:32.394 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T21:19:32.397 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:32.397 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:19:32.498 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:19:32.528 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:32.617 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:19:32.619 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:32.619 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:19:32.645 INFO:teuthology.orchestra.run.smithi026.stderr:noscrub is unset 2023-09-23T21:19:32.666 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T21:19:32.721 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:19:32.746 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:32.809 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:19:32.811 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:19:32 smithi070 ceph-mon[168282]: pgmap v1602: 103 pgs: 12 peering, 91 active+clean; 582 KiB data, 3.7 GiB used, 443 GiB / 447 GiB avail 2023-09-23T21:19:32.811 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:19:32 smithi070 ceph-mon[168282]: failed to encode map e652 with expected crc 2023-09-23T21:19:32.812 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:19:32 smithi070 ceph-mon[168282]: failed to encode map e652 with expected crc 2023-09-23T21:19:32.812 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:19:32 smithi070 ceph-mon[168282]: failed to encode map e652 with expected crc 2023-09-23T21:19:32.812 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:19:32 smithi070 ceph-mon[168282]: failed to encode map e652 with expected crc 2023-09-23T21:19:32.812 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:19:32 smithi070 ceph-mon[168282]: osdmap e652: 8 total, 8 up, 5 in 2023-09-23T21:19:32.812 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:19:32 smithi070 ceph-mon[168282]: failed to encode map e652 with expected crc 2023-09-23T21:19:32.812 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:19:32 smithi070 ceph-mon[168282]: failed to encode map e652 with expected crc 2023-09-23T21:19:32.812 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:19:32 smithi070 ceph-mon[168282]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2023-09-23T21:19:32.812 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:19:32 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:19:32.812 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:19:32 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/3044284034' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T21:19:32.812 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:19:32 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T21:19:32.813 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:32.814 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:19:32.915 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:19:33.131 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:19:32 smithi026 ceph-mon[175151]: pgmap v1602: 103 pgs: 12 peering, 91 active+clean; 582 KiB data, 3.7 GiB used, 443 GiB / 447 GiB avail 2023-09-23T21:19:33.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:19:32 smithi026 ceph-mon[175151]: failed to encode map e652 with expected crc 2023-09-23T21:19:33.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:19:32 smithi026 ceph-mon[175151]: failed to encode map e652 with expected crc 2023-09-23T21:19:33.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:19:32 smithi026 ceph-mon[175151]: failed to encode map e652 with expected crc 2023-09-23T21:19:33.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:19:32 smithi026 ceph-mon[175151]: failed to encode map e652 with expected crc 2023-09-23T21:19:33.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:19:32 smithi026 ceph-mon[175151]: osdmap e652: 8 total, 8 up, 5 in 2023-09-23T21:19:33.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:19:32 smithi026 ceph-mon[175151]: failed to encode map e652 with expected crc 2023-09-23T21:19:33.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:19:32 smithi026 ceph-mon[175151]: failed to encode map e652 with expected crc 2023-09-23T21:19:33.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:19:32 smithi026 ceph-mon[175151]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2023-09-23T21:19:33.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:19:32 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:19:33.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:19:32 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/3044284034' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T21:19:33.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:19:32 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T21:19:33.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:19:32 smithi026 ceph-mon[179688]: pgmap v1602: 103 pgs: 12 peering, 91 active+clean; 582 KiB data, 3.7 GiB used, 443 GiB / 447 GiB avail 2023-09-23T21:19:33.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:19:32 smithi026 ceph-mon[179688]: failed to encode map e652 with expected crc 2023-09-23T21:19:33.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:19:32 smithi026 ceph-mon[179688]: failed to encode map e652 with expected crc 2023-09-23T21:19:33.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:19:32 smithi026 ceph-mon[179688]: failed to encode map e652 with expected crc 2023-09-23T21:19:33.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:19:32 smithi026 ceph-mon[179688]: failed to encode map e652 with expected crc 2023-09-23T21:19:33.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:19:32 smithi026 ceph-mon[179688]: osdmap e652: 8 total, 8 up, 5 in 2023-09-23T21:19:33.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:19:32 smithi026 ceph-mon[179688]: failed to encode map e652 with expected crc 2023-09-23T21:19:33.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:19:32 smithi026 ceph-mon[179688]: failed to encode map e652 with expected crc 2023-09-23T21:19:33.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:19:32 smithi026 ceph-mon[179688]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2023-09-23T21:19:33.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:19:32 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:19:33.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:19:32 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/3044284034' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T21:19:33.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:19:32 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T21:19:33.140 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:33.196 INFO:tasks.workunit.client.0.smithi026.stderr: Importing image: 1% complete... Importing image: 2% complete... Importing image: 3% complete... Importing image: 4% complete... Importing image: 5% complete... Importing image: 6% complete... Importing image: 7% complete... Importing image: 8% complete... Importing image: 9% complete... Importing image: 10% complete... Importing image: 11% complete... Importing image: 12% complete... Importing image: 13% complete... Importing image: 14% complete... Importing image: 15% complete... Importing image: 16% complete... Importing image: 17% complete... Importing image: 18% complete... Importing image: 19% complete... Importing image: 20% complete... Importing image: 21% complete... Importing image: 22% complete... Importing image: 23% complete... Importing image: 24% complete... Importing image: 25% complete... Importing image: 26% complete... Importing image: 27% complete... Importing image: 28% complete... Importing image: 29% complete... Importing image: 30% complete... Importing image: 31% complete... Importing image: 32% complete... Importing image: 33% complete... Importing image: 34% complete... Importing image: 35% complete... Importing image: 36% complete... Importing image: 37% complete... Importing image: 38% complete... Importing image: 39% complete... Importing image: 40% complete... Importing image: 41% complete... Importing image: 42% complete... Importing image: 43% complete... Importing image: 44% complete... Importing image: 45% complete... Importing image: 46% complete... Importing image: 47% complete... Importing image: 48% complete... Importing image: 49% complete... Importing image: 50% complete... Importing image: 51% complete... Importing image: 52% complete... Importing image: 53% complete... Importing image: 54% complete... Importing image: 55% complete... Importing image: 56% complete... Importing image: 57% complete... Importing image: 58% complete... Importing image: 59% complete... Importing image: 60% complete... Importing image: 61% complete... Importing image: 62% complete... Importing image: 63% complete... Importing image: 64% complete... Importing image: 65% complete... Importing image: 66% complete... Importing image: 67% complete... Importing image: 68% complete... Importing image: 69% complete... Importing image: 70% complete... Importing image: 71% complete... Importing image: 72% complete... Importing image: 73% complete... Importing image: 74% complete... Importing image: 75% complete... Importing image: 76% complete... Importing image: 77% complete... Importing image: 78% complete... Importing image: 79% complete... Importing image: 80% complete... Importing image: 81% complete... Importing image: 82% complete... Importing image: 83% complete... Importing image: 84% complete... Importing image: 85% complete... Importing image: 86% complete... Importing image: 87% complete... Importing image: 88% complete... Importing image: 89% complete... Importing image: 90% complete... Importing image: 91% complete... Importing image: 92% complete... Importing image: 93% complete... Importing image: 94% complete... Importing image: 95% complete... Importing image: 96% complete... Importing image: 97% complete... Importing image: 98% complete... Importing image: 99% complete... Importing image: 100% complete...done. 2023-09-23T21:19:33.197 INFO:tasks.workunit.client.0.smithi026.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-09-23T21:19:33.199 INFO:tasks.workunit.client.0.smithi026.stderr:+ rbd export testimg /tmp/rbd_import_export_456745/img2 2023-09-23T21:19:33.496 INFO:tasks.workunit.client.0.smithi026.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2023-09-23T21:19:33.499 INFO:tasks.workunit.client.0.smithi026.stderr:+ rbd export testimg - 2023-09-23T21:19:33.775 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:19:33 smithi070 ceph-mon[168282]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:19:33.775 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:19:33 smithi070 ceph-mon[168282]: failed to encode map e653 with expected crc 2023-09-23T21:19:33.775 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:19:33 smithi070 ceph-mon[168282]: failed to encode map e653 with expected crc 2023-09-23T21:19:33.775 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:19:33 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T21:19:33.775 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:19:33 smithi070 ceph-mon[168282]: osdmap e653: 8 total, 8 up, 5 in 2023-09-23T21:19:33.776 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:19:33 smithi070 ceph-mon[168282]: failed to encode map e653 with expected crc 2023-09-23T21:19:33.776 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:19:33 smithi070 ceph-mon[168282]: osdmap e654: 8 total, 8 up, 5 in 2023-09-23T21:19:33.881 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:19:33 smithi026 ceph-mon[179688]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:19:33.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:19:33 smithi026 ceph-mon[179688]: failed to encode map e653 with expected crc 2023-09-23T21:19:33.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:19:33 smithi026 ceph-mon[179688]: failed to encode map e653 with expected crc 2023-09-23T21:19:33.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:19:33 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T21:19:33.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:19:33 smithi026 ceph-mon[179688]: osdmap e653: 8 total, 8 up, 5 in 2023-09-23T21:19:33.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:19:33 smithi026 ceph-mon[179688]: failed to encode map e653 with expected crc 2023-09-23T21:19:33.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:19:33 smithi026 ceph-mon[179688]: osdmap e654: 8 total, 8 up, 5 in 2023-09-23T21:19:33.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:19:33 smithi026 ceph-mon[175151]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:19:33.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:19:33 smithi026 ceph-mon[175151]: failed to encode map e653 with expected crc 2023-09-23T21:19:33.883 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:19:33 smithi026 ceph-mon[175151]: failed to encode map e653 with expected crc 2023-09-23T21:19:33.883 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:19:33 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T21:19:33.883 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:19:33 smithi026 ceph-mon[175151]: osdmap e653: 8 total, 8 up, 5 in 2023-09-23T21:19:33.883 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:19:33 smithi026 ceph-mon[175151]: failed to encode map e653 with expected crc 2023-09-23T21:19:33.883 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:19:33 smithi026 ceph-mon[175151]: osdmap e654: 8 total, 8 up, 5 in 2023-09-23T21:19:33.988 INFO:teuthology.orchestra.run.smithi070.stdout:ERROR: (22) Invalid argument 2023-09-23T21:19:33.988 INFO:teuthology.orchestra.run.smithi070.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T21:19:34.245 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:19:34.249 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:34.249 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:19:34.252 INFO:tasks.workunit.client.0.smithi026.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2023-09-23T21:19:34.256 INFO:tasks.workunit.client.0.smithi026.stderr:+ rbd rm testimg 2023-09-23T21:19:34.344 INFO:tasks.workunit.client.0.smithi026.stderr: Removing image: 1% complete... Removing image: 2% complete... Removing image: 3% complete... Removing image: 4% complete... Removing image: 5% complete... Removing image: 6% complete... Removing image: 7% complete... Removing image: 8% complete... Removing image: 9% complete... Removing image: 10% complete... Removing image: 11% complete... Removing image: 12% complete... Removing image: 13% complete... Removing image: 14% complete... Removing image: 15% complete... Removing image: 16% complete... Removing image: 17% complete... Removing image: 18% complete... Removing image: 19% complete... Removing image: 20% complete... Removing image: 21% complete... Removing image: 22% complete... Removing image: 23% complete... Removing image: 24% complete... Removing image: 25% complete... Removing image: 26% complete... Removing image: 27% complete... Removing image: 28% complete... Removing image: 29% complete... Removing image: 30% complete... Removing image: 31% complete... Removing image: 32% complete... Removing image: 33% complete... Removing image: 34% complete... Removing image: 35% complete... Removing image: 36% complete... Removing image: 37% complete... Removing image: 38% complete... Removing image: 39% complete... Removing image: 40% complete... Removing image: 41% complete... Removing image: 42% complete... Removing image: 43% complete... Removing image: 44% complete... Removing image: 45% complete... Removing image: 46% complete... Removing image: 47% complete... Removing image: 48% complete... Removing image: 49% complete... Removing image: 50% complete... Removing image: 51% complete... Removing image: 52% complete... Removing image: 53% complete... Removing image: 54% complete... Removing image: 55% complete... Removing image: 56% complete... Removing image: 57% complete... Removing image: 58% complete... Removing image: 59% complete... Removing image: 60% complete... Removing image: 61% complete... Removing image: 62% complete... Removing image: 63% complete... Removing image: 64% complete... Removing image: 65% complete... Removing image: 66% complete... Removing image: 67% complete... Removing image: 68% complete... Removing image: 69% complete... Removing image: 70% complete... Removing image: 71% complete... Removing image: 72% complete... Removing image: 73% complete... Removing image: 74% complete... Removing image: 75% complete... Removing image: 76% complete... Removing image: 77% complete... Removing image: 78% complete... Removing image: 79% complete... Removing image: 80% complete... Removing image: 81% complete... Removing image: 82% complete... Removing image: 83% complete... Removing image: 84% complete... Removing image: 85% complete... Removing image: 86% complete... Removing image: 87% complete... Removing image: 88% complete... Removing image: 89% complete... Removing image: 90% complete... Removing image: 91% complete... Removing image: 92% complete... Removing image: 93% complete... Removing image: 94% complete... Removing image: 95% complete... Removing image: 96% complete... Removing image: 97% complete... Removing image: 98% complete... Removing image: 99% complete... Removing image: 100% complete...done. 2023-09-23T21:19:34.347 INFO:tasks.workunit.client.0.smithi026.stderr:+ cmp /tmp/rbd_import_export_456745/img /tmp/rbd_import_export_456745/img2 2023-09-23T21:19:34.350 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:19:34.423 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_ops_in_flight 2023-09-23T21:19:34.458 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:34.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:19:34 smithi070 ceph-mon[168282]: failed to encode map e653 with expected crc 2023-09-23T21:19:34.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:19:34 smithi070 ceph-mon[168282]: pgmap v1605: 103 pgs: 11 active+undersized+remapped, 12 peering, 80 active+clean; 454 KiB data, 3.7 GiB used, 443 GiB / 447 GiB avail 2023-09-23T21:19:34.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:19:34 smithi070 ceph-mon[168282]: failed to encode map e654 with expected crc 2023-09-23T21:19:34.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:19:34 smithi070 ceph-mon[168282]: failed to encode map e654 with expected crc 2023-09-23T21:19:34.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:19:34 smithi070 ceph-mon[168282]: failed to encode map e654 with expected crc 2023-09-23T21:19:34.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:19:34 smithi070 ceph-mon[168282]: failed to encode map e654 with expected crc 2023-09-23T21:19:34.881 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:19:34 smithi026 ceph-mon[175151]: failed to encode map e653 with expected crc 2023-09-23T21:19:34.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:19:34 smithi026 ceph-mon[175151]: pgmap v1605: 103 pgs: 11 active+undersized+remapped, 12 peering, 80 active+clean; 454 KiB data, 3.7 GiB used, 443 GiB / 447 GiB avail 2023-09-23T21:19:34.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:19:34 smithi026 ceph-mon[175151]: failed to encode map e654 with expected crc 2023-09-23T21:19:34.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:19:34 smithi026 ceph-mon[175151]: failed to encode map e654 with expected crc 2023-09-23T21:19:34.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:19:34 smithi026 ceph-mon[175151]: failed to encode map e654 with expected crc 2023-09-23T21:19:34.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:19:34 smithi026 ceph-mon[175151]: failed to encode map e654 with expected crc 2023-09-23T21:19:34.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:19:34 smithi026 ceph-mon[179688]: failed to encode map e653 with expected crc 2023-09-23T21:19:34.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:19:34 smithi026 ceph-mon[179688]: pgmap v1605: 103 pgs: 11 active+undersized+remapped, 12 peering, 80 active+clean; 454 KiB data, 3.7 GiB used, 443 GiB / 447 GiB avail 2023-09-23T21:19:34.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:19:34 smithi026 ceph-mon[179688]: failed to encode map e654 with expected crc 2023-09-23T21:19:34.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:19:34 smithi026 ceph-mon[179688]: failed to encode map e654 with expected crc 2023-09-23T21:19:34.883 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:19:34 smithi026 ceph-mon[179688]: failed to encode map e654 with expected crc 2023-09-23T21:19:34.883 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:19:34 smithi026 ceph-mon[179688]: failed to encode map e654 with expected crc 2023-09-23T21:19:35.031 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T21:19:35.145 INFO:tasks.thrashosds.thrasher:in_osds: [0, 2, 3, 4, 7] out_osds: [6, 1, 5] dead_osds: [] live_osds: [2, 3, 4, 5, 7, 6, 1, 0] 2023-09-23T21:19:35.145 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T21:19:35.146 INFO:tasks.thrashosds.thrasher:Growing pool .mgr 2023-09-23T21:19:35.146 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T21:19:35.147 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:19:35.149 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:35.150 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:19:35.237 INFO:tasks.workunit.client.0.smithi026.stderr:+ cmp /tmp/rbd_import_export_456745/img /tmp/rbd_import_export_456745/img3 2023-09-23T21:19:35.251 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:19:35.278 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:35.760 INFO:tasks.workunit.client.0.smithi026.stderr:+ rm /tmp/rbd_import_export_456745/img2 /tmp/rbd_import_export_456745/img3 2023-09-23T21:19:35.826 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:19:35.829 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:35.830 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:19:35.916 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:19:35 smithi070 ceph-mon[168282]: 1.1b scrub starts 2023-09-23T21:19:35.916 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:19:35 smithi070 ceph-mon[168282]: 1.1b scrub ok 2023-09-23T21:19:35.930 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:19:35.959 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:35.968 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:19:35 smithi026 ceph-mon[175151]: 1.1b scrub starts 2023-09-23T21:19:35.968 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:19:35 smithi026 ceph-mon[175151]: 1.1b scrub ok 2023-09-23T21:19:35.969 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:19:35 smithi026 ceph-mon[179688]: 1.1b scrub starts 2023-09-23T21:19:35.969 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:19:35 smithi026 ceph-mon[179688]: 1.1b scrub ok 2023-09-23T21:19:35.994 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:19:35.996 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:35.997 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:19:35.998 INFO:tasks.workunit.client.0.smithi026.stderr:+ rbd import --new-format - testimg 2023-09-23T21:19:36.098 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:19:36.134 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:36.326 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:19:36.329 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:36.330 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:19:36.405 INFO:tasks.workunit.client.0.smithi026.stderr: Importing image: 100% complete...done. 2023-09-23T21:19:36.405 INFO:tasks.workunit.client.0.smithi026.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-09-23T21:19:36.409 INFO:tasks.workunit.client.0.smithi026.stderr:+ rbd export testimg /tmp/rbd_import_export_456745/img2 2023-09-23T21:19:36.431 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:19:36.458 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:36.492 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:19:36.495 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:36.495 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:19:36.557 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:19:36.596 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:19:36.625 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:36.680 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:19:36.682 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:36.683 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:19:36.696 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0: {} 2023-09-23T21:19:36.722 INFO:tasks.workunit.client.0.smithi026.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2023-09-23T21:19:36.724 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1: {} 2023-09-23T21:19:36.727 INFO:tasks.workunit.client.0.smithi026.stderr:+ rbd export testimg - 2023-09-23T21:19:36.745 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2: {} 2023-09-23T21:19:36.767 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:19:36 smithi070 ceph-mon[168282]: 1.18 scrub starts 2023-09-23T21:19:36.767 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:19:36 smithi070 ceph-mon[168282]: 1.18 scrub ok 2023-09-23T21:19:36.767 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:19:36 smithi070 ceph-mon[168282]: pgmap v1607: 103 pgs: 11 active+undersized+remapped, 12 peering, 80 active+clean; 454 KiB data, 3.7 GiB used, 443 GiB / 447 GiB avail 2023-09-23T21:19:36.770 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3: {} 2023-09-23T21:19:36.784 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:19:36.794 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4: {} 2023-09-23T21:19:36.812 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:36.815 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T21:19:36.835 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6: {} 2023-09-23T21:19:36.846 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:19:36.850 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:36.850 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:19:36.864 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T21:19:36.867 INFO:teuthology.orchestra.run.smithi026.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T21:19:36.867 INFO:teuthology.orchestra.run.smithi026.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T21:19:36.867 INFO:teuthology.orchestra.run.smithi026.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T21:19:36.867 INFO:teuthology.orchestra.run.smithi026.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T21:19:36.867 INFO:teuthology.orchestra.run.smithi026.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T21:19:36.867 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T21:19:36.867 INFO:teuthology.orchestra.run.smithi026.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T21:19:36.867 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T21:19:36.908 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:19:36 smithi026 ceph-mon[175151]: 1.18 scrub starts 2023-09-23T21:19:36.908 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:19:36 smithi026 ceph-mon[175151]: 1.18 scrub ok 2023-09-23T21:19:36.908 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:19:36 smithi026 ceph-mon[175151]: pgmap v1607: 103 pgs: 11 active+undersized+remapped, 12 peering, 80 active+clean; 454 KiB data, 3.7 GiB used, 443 GiB / 447 GiB avail 2023-09-23T21:19:36.909 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:19:36 smithi026 ceph-mon[179688]: 1.18 scrub starts 2023-09-23T21:19:36.909 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:19:36 smithi026 ceph-mon[179688]: 1.18 scrub ok 2023-09-23T21:19:36.909 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:19:36 smithi026 ceph-mon[179688]: pgmap v1607: 103 pgs: 11 active+undersized+remapped, 12 peering, 80 active+clean; 454 KiB data, 3.7 GiB used, 443 GiB / 447 GiB avail 2023-09-23T21:19:36.951 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:19:36.992 INFO:teuthology.orchestra.run.smithi070.stdout:{ 2023-09-23T21:19:36.992 INFO:teuthology.orchestra.run.smithi070.stdout: "ops": [], 2023-09-23T21:19:36.993 INFO:teuthology.orchestra.run.smithi070.stdout: "num_ops": 0 2023-09-23T21:19:36.993 INFO:teuthology.orchestra.run.smithi070.stdout:} 2023-09-23T21:19:36.993 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:37.083 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:19:37.086 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:37.087 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:19:37.185 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T21:19:37.188 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:19:37.215 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:37.351 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:19:37.353 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:37.354 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:19:37.386 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_blocked_ops 2023-09-23T21:19:37.454 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:19:37.492 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:37.528 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T21:19:37.530 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:37.531 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:19:37.570 INFO:tasks.workunit.client.0.smithi026.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2023-09-23T21:19:37.574 INFO:tasks.workunit.client.0.smithi026.stderr:+ rbd rm testimg 2023-09-23T21:19:37.632 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:19:37.660 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:37.667 INFO:tasks.workunit.client.0.smithi026.stderr: Removing image: 1% complete... Removing image: 2% complete... Removing image: 3% complete... Removing image: 4% complete... Removing image: 5% complete... Removing image: 6% complete... Removing image: 7% complete... Removing image: 8% complete... Removing image: 9% complete... Removing image: 10% complete... Removing image: 11% complete... Removing image: 12% complete... Removing image: 13% complete... Removing image: 14% complete... Removing image: 15% complete... Removing image: 16% complete... Removing image: 17% complete... Removing image: 18% complete... Removing image: 19% complete... Removing image: 20% complete... Removing image: 21% complete... Removing image: 22% complete... Removing image: 23% complete... Removing image: 24% complete... Removing image: 25% complete... Removing image: 26% complete... Removing image: 27% complete... Removing image: 28% complete... Removing image: 29% complete... Removing image: 30% complete... Removing image: 31% complete... Removing image: 32% complete... Removing image: 33% complete... Removing image: 34% complete... Removing image: 35% complete... Removing image: 36% complete... Removing image: 37% complete... Removing image: 38% complete... Removing image: 39% complete... Removing image: 40% complete... Removing image: 41% complete... Removing image: 42% complete... Removing image: 43% complete... Removing image: 44% complete... Removing image: 45% complete... Removing image: 46% complete... Removing image: 47% complete... Removing image: 48% complete... Removing image: 49% complete... Removing image: 50% complete... Removing image: 51% complete... Removing image: 52% complete... Removing image: 53% complete... Removing image: 54% complete... Removing image: 55% complete... Removing image: 56% complete... Removing image: 57% complete... Removing image: 58% complete... Removing image: 59% complete... Removing image: 60% complete... Removing image: 61% complete... Removing image: 62% complete... Removing image: 63% complete... Removing image: 64% complete... Removing image: 65% complete... Removing image: 66% complete... Removing image: 67% complete... Removing image: 68% complete... Removing image: 69% complete... Removing image: 70% complete... Removing image: 71% complete... Removing image: 72% complete... Removing image: 73% complete... Removing image: 74% complete... Removing image: 75% complete... Removing image: 76% complete... Removing image: 77% complete... Removing image: 78% complete... Removing image: 79% complete... Removing image: 80% complete... Removing image: 81% complete... Removing image: 82% complete... Removing image: 83% complete... Removing image: 84% complete... Removing image: 85% complete... Removing image: 86% complete... Removing image: 87% complete... Removing image: 88% complete... Removing image: 89% complete... Removing image: 90% complete... Removing image: 91% complete... Removing image: 92% complete... Removing image: 93% complete... Removing image: 94% complete... Removing image: 95% complete... Removing image: 96% complete... Removing image: 97% complete... Removing image: 98% complete... Removing image: 99% complete... Removing image: 100% complete...done. 2023-09-23T21:19:37.670 INFO:tasks.workunit.client.0.smithi026.stderr:+ cmp /tmp/rbd_import_export_456745/img /tmp/rbd_import_export_456745/img2 2023-09-23T21:19:37.692 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:19:37.695 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:37.695 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:19:37.741 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:19:37 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/1276512028' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:19:37.741 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:19:37 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/1276512028' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:19:37.796 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:19:37.829 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:37.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:19:37 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/1276512028' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:19:37.997 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:19:38.000 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:19:38.002 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:38.002 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:19:38.018 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T21:19:38.103 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:19:38.129 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:38.163 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:19:38.166 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:38.166 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:19:38.267 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:19:38.302 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:38.401 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:19:38.405 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:38.405 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:19:38.408 INFO:tasks.workunit.client.0.smithi026.stderr:+ cmp /tmp/rbd_import_export_456745/img /tmp/rbd_import_export_456745/img3 2023-09-23T21:19:38.506 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:19:38.540 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:38.629 INFO:teuthology.orchestra.run.smithi026.stdout: 2023-09-23T21:19:38.631 INFO:teuthology.orchestra.run.smithi026.stderr:dumped all 2023-09-23T21:19:38.768 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:19:38 smithi070 ceph-mon[168282]: pgmap v1608: 103 pgs: 2 active+undersized+remapped, 101 active+clean; 1.3 MiB data, 3.7 GiB used, 443 GiB / 447 GiB avail; 237 KiB/s rd, 120 KiB/s wr, 65 op/s; 0 B/s, 0 objects/s recovering 2023-09-23T21:19:38.768 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:19:38 smithi070 ceph-mon[168282]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2023-09-23T21:19:38.958 INFO:tasks.workunit.client.0.smithi026.stderr:+ rm /tmp/rbd_import_export_456745/img /tmp/rbd_import_export_456745/img2 /tmp/rbd_import_export_456745/img3 2023-09-23T21:19:39.085 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:19:39.087 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:19:38 smithi026 ceph-mon[179688]: pgmap v1608: 103 pgs: 2 active+undersized+remapped, 101 active+clean; 1.3 MiB data, 3.7 GiB used, 443 GiB / 447 GiB avail; 237 KiB/s rd, 120 KiB/s wr, 65 op/s; 0 B/s, 0 objects/s recovering 2023-09-23T21:19:39.087 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:19:38 smithi026 ceph-mon[179688]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2023-09-23T21:19:39.088 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:19:38 smithi026 ceph-mon[175151]: pgmap v1608: 103 pgs: 2 active+undersized+remapped, 101 active+clean; 1.3 MiB data, 3.7 GiB used, 443 GiB / 447 GiB avail; 237 KiB/s rd, 120 KiB/s wr, 65 op/s; 0 B/s, 0 objects/s recovering 2023-09-23T21:19:39.088 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:19:38 smithi026 ceph-mon[175151]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2023-09-23T21:19:39.088 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:39.089 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:19:39.190 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:19:39.221 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:39.253 INFO:teuthology.orchestra.run.smithi070.stdout:{ 2023-09-23T21:19:39.253 INFO:teuthology.orchestra.run.smithi070.stdout: "ops": [], 2023-09-23T21:19:39.253 INFO:teuthology.orchestra.run.smithi070.stdout: "complaint_time": 30, 2023-09-23T21:19:39.253 INFO:teuthology.orchestra.run.smithi070.stdout: "num_blocked_ops": 0 2023-09-23T21:19:39.253 INFO:teuthology.orchestra.run.smithi070.stdout:} 2023-09-23T21:19:39.256 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:19:39.256 INFO:tasks.workunit.client.0.smithi026.stderr:+ rbd help export 2023-09-23T21:19:39.256 INFO:tasks.workunit.client.0.smithi026.stderr:+ grep -q export-format 2023-09-23T21:19:39.258 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:39.259 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:19:39.284 INFO:tasks.workunit.client.0.smithi026.stderr:+ dd if=/bin/dd of=/tmp/rbd_import_export_456745/img bs=1k count=10 seek=100 2023-09-23T21:19:39.285 INFO:tasks.workunit.client.0.smithi026.stderr:10+0 records in 2023-09-23T21:19:39.285 INFO:tasks.workunit.client.0.smithi026.stderr:10+0 records out 2023-09-23T21:19:39.285 INFO:tasks.workunit.client.0.smithi026.stderr:10240 bytes (10 kB, 10 KiB) copied, 6.7185e-05 s, 152 MB/s 2023-09-23T21:19:39.285 INFO:tasks.workunit.client.0.smithi026.stderr:+ rbd import --new-format /tmp/rbd_import_export_456745/img testimg 2023-09-23T21:19:39.339 INFO:tasks.workunit.client.0.smithi026.stderr: Importing image: 100% complete...done. 2023-09-23T21:19:39.339 INFO:tasks.workunit.client.0.smithi026.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-09-23T21:19:39.342 INFO:tasks.workunit.client.0.smithi026.stderr:+ rbd snap create testimg@snap 2023-09-23T21:19:39.359 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:19:39.394 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:39.395 INFO:teuthology.orchestra.run.smithi026.stdout:{"pg_ready":true,"pg_map":{"version":1608,"stamp":"2023-09-23T21:19:37.215544+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":1379595,"num_objects":30,"num_object_clones":0,"num_object_copies":90,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":30,"num_whiteouts":0,"num_read":318,"num_read_kb":1420,"num_write":137,"num_write_kb":749,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":84,"num_bytes_recovered":18960,"num_keys_recovered":20,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":5127,"ondisk_log_size":5127,"up":309,"acting":307,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":291,"num_osds":5,"num_per_pool_osds":5,"num_per_pool_omap_osds":5,"kb":468725760,"kb_used":3862184,"kb_used_data":39932,"kb_used_omap":125,"kb_used_meta":3822082,"kb_avail":464863576,"statfs":{"total":479975178240,"available":476020301824,"internally_reserved":0,"allocated":40890368,"data_stored":28450494,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":128662,"internal_metadata":3913812330},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":783723,"num_objects":6,"num_object_clones":0,"num_object_copies":18,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6,"num_whiteouts":0,"num_read":318,"num_read_kb":1420,"num_write":76,"num_write_kb":720,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":-525102,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"6.001456"},"pg_stats":[{"pgid":"1.18","version":"79'161","reported_seq":697,"reported_epoch":654,"state":"active+clean","last_fresh":"2023-09-23T21:19:34.961775+0000","last_change":"2023-09-23T21:19:34.961897+0000","last_active":"2023-09-23T21:19:34.961775+0000","last_peered":"2023-09-23T21:19:34.961775+0000","last_clean":"2023-09-23T21:19:34.961775+0000","last_became_active":"2023-09-23T21:19:33.641493+0000","last_became_peered":"2023-09-23T21:19:33.641493+0000","last_unstale":"2023-09-23T21:19:34.961775+0000","last_undegraded":"2023-09-23T21:19:34.961775+0000","last_fullsized":"2023-09-23T21:19:34.961775+0000","mapping_epoch":653,"log_start":"0'0","ondisk_log_start":"0'0","created":479,"last_epoch_clean":654,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:19:34.961756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:19:34.961756+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:08:23.522573+0000","scrub_duration":0.002022102,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,0],"acting":[2,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1a","version":"68'1","reported_seq":244,"reported_epoch":653,"state":"active+clean","last_fresh":"2023-09-23T21:19:32.626750+0000","last_change":"2023-09-23T21:13:13.291403+0000","last_active":"2023-09-23T21:19:32.626750+0000","last_peered":"2023-09-23T21:19:32.626750+0000","last_clean":"2023-09-23T21:19:32.626750+0000","last_became_active":"2023-09-23T21:13:13.291123+0000","last_became_peered":"2023-09-23T21:13:13.291123+0000","last_unstale":"2023-09-23T21:19:32.626750+0000","last_undegraded":"2023-09-23T21:19:32.626750+0000","last_fullsized":"2023-09-23T21:19:32.626750+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":495,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:45:15.427430+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,0],"acting":[4,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":323,"reported_epoch":653,"state":"active+clean","last_fresh":"2023-09-23T21:19:32.627594+0000","last_change":"2023-09-23T21:19:30.642278+0000","last_active":"2023-09-23T21:19:32.627594+0000","last_peered":"2023-09-23T21:19:32.627594+0000","last_clean":"2023-09-23T21:19:32.627594+0000","last_became_active":"2023-09-23T21:19:30.637935+0000","last_became_peered":"2023-09-23T21:19:30.637935+0000","last_unstale":"2023-09-23T21:19:32.627594+0000","last_undegraded":"2023-09-23T21:19:32.627594+0000","last_fullsized":"2023-09-23T21:19:32.627594+0000","mapping_epoch":650,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":651,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:21:57.900400+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1b","version":"79'161","reported_seq":737,"reported_epoch":654,"state":"active+clean","last_fresh":"2023-09-23T21:19:34.862932+0000","last_change":"2023-09-23T21:19:34.862883+0000","last_active":"2023-09-23T21:19:34.862932+0000","last_peered":"2023-09-23T21:19:34.862932+0000","last_clean":"2023-09-23T21:19:34.862932+0000","last_became_active":"2023-09-23T21:19:33.639640+0000","last_became_peered":"2023-09-23T21:19:33.639640+0000","last_unstale":"2023-09-23T21:19:34.862932+0000","last_undegraded":"2023-09-23T21:19:34.862932+0000","last_fullsized":"2023-09-23T21:19:34.862932+0000","mapping_epoch":653,"log_start":"0'0","ondisk_log_start":"0'0","created":479,"last_epoch_clean":654,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:19:34.862788+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:19:34.862788+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:23:17.102526+0000","scrub_duration":0.001907228,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,2],"acting":[4,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":670,"reported_epoch":653,"state":"active+clean","last_fresh":"2023-09-23T21:19:32.629368+0000","last_change":"2023-09-23T21:16:42.448529+0000","last_active":"2023-09-23T21:19:32.629368+0000","last_peered":"2023-09-23T21:19:32.629368+0000","last_clean":"2023-09-23T21:19:32.629368+0000","last_became_active":"2023-09-23T21:16:42.447803+0000","last_became_peered":"2023-09-23T21:16:42.447803+0000","last_unstale":"2023-09-23T21:19:32.629368+0000","last_undegraded":"2023-09-23T21:19:32.629368+0000","last_fullsized":"2023-09-23T21:19:32.629368+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:41:33.121784+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":671,"reported_epoch":653,"state":"active+clean","last_fresh":"2023-09-23T21:19:32.629299+0000","last_change":"2023-09-23T21:16:42.441896+0000","last_active":"2023-09-23T21:19:32.629299+0000","last_peered":"2023-09-23T21:19:32.629299+0000","last_clean":"2023-09-23T21:19:32.629299+0000","last_became_active":"2023-09-23T21:16:42.441526+0000","last_became_peered":"2023-09-23T21:16:42.441526+0000","last_unstale":"2023-09-23T21:19:32.629299+0000","last_undegraded":"2023-09-23T21:19:32.629299+0000","last_fullsized":"2023-09-23T21:19:32.629299+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:27:36.326225+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1a","version":"79'161","reported_seq":771,"reported_epoch":654,"state":"active+clean","last_fresh":"2023-09-23T21:19:33.632804+0000","last_change":"2023-09-23T21:19:33.632804+0000","last_active":"2023-09-23T21:19:33.632804+0000","last_peered":"2023-09-23T21:19:33.632804+0000","last_clean":"2023-09-23T21:19:33.632804+0000","last_became_active":"2023-09-23T21:19:33.632406+0000","last_became_peered":"2023-09-23T21:19:33.632406+0000","last_unstale":"2023-09-23T21:19:33.632804+0000","last_undegraded":"2023-09-23T21:19:33.632804+0000","last_fullsized":"2023-09-23T21:19:33.632804+0000","mapping_epoch":653,"log_start":"0'0","ondisk_log_start":"0'0","created":479,"last_epoch_clean":654,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:15:45.158107+0000","last_deep_scrub":"79'161","last_deep_scrub_stamp":"2023-09-23T21:05:32.630292+0000","last_clean_scrub_stamp":"2023-09-23T21:15:45.158107+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:40:42.578254+0000","scrub_duration":0.001999373,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,2],"acting":[4,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":165,"reported_epoch":653,"state":"active+clean","last_fresh":"2023-09-23T21:19:32.627666+0000","last_change":"2023-09-23T21:13:30.806084+0000","last_active":"2023-09-23T21:19:32.627666+0000","last_peered":"2023-09-23T21:19:32.627666+0000","last_clean":"2023-09-23T21:19:32.627666+0000","last_became_active":"2023-09-23T21:13:30.805743+0000","last_became_peered":"2023-09-23T21:13:30.805743+0000","last_unstale":"2023-09-23T21:19:32.627666+0000","last_undegraded":"2023-09-23T21:19:32.627666+0000","last_fullsized":"2023-09-23T21:19:32.627666+0000","mapping_epoch":501,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:48:10.849281+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":539,"reported_epoch":653,"state":"active+clean","last_fresh":"2023-09-23T21:19:32.631084+0000","last_change":"2023-09-23T21:19:30.639145+0000","last_active":"2023-09-23T21:19:32.631084+0000","last_peered":"2023-09-23T21:19:32.631084+0000","last_clean":"2023-09-23T21:19:32.631084+0000","last_became_active":"2023-09-23T21:19:30.638805+0000","last_became_peered":"2023-09-23T21:19:30.638805+0000","last_unstale":"2023-09-23T21:19:32.631084+0000","last_undegraded":"2023-09-23T21:19:32.631084+0000","last_fullsized":"2023-09-23T21:19:32.631084+0000","mapping_epoch":650,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":651,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:23:01.782834+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.15","version":"79'161","reported_seq":620,"reported_epoch":652,"state":"active+undersized+remapped","last_fresh":"2023-09-23T21:19:31.637112+0000","last_change":"2023-09-23T21:19:31.637027+0000","last_active":"2023-09-23T21:19:31.637112+0000","last_peered":"2023-09-23T21:19:31.637112+0000","last_clean":"2023-09-23T21:19:28.232250+0000","last_became_active":"2023-09-23T21:19:31.637027+0000","last_became_peered":"2023-09-23T21:19:31.637027+0000","last_unstale":"2023-09-23T21:19:31.637112+0000","last_undegraded":"2023-09-23T21:19:31.637112+0000","last_fullsized":"2023-09-23T21:19:31.630161+0000","mapping_epoch":651,"log_start":"0'0","ondisk_log_start":"0'0","created":479,"last_epoch_clean":568,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:18:35.622422+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:18:35.622422+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:24:02.996855+0000","scrub_duration":0.083031242000000005,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3],"avail_no_missing":["7","0","3","5"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":639,"reported_epoch":653,"state":"active+clean","last_fresh":"2023-09-23T21:19:32.631261+0000","last_change":"2023-09-23T21:19:30.636889+0000","last_active":"2023-09-23T21:19:32.631261+0000","last_peered":"2023-09-23T21:19:32.631261+0000","last_clean":"2023-09-23T21:19:32.631261+0000","last_became_active":"2023-09-23T21:19:30.636520+0000","last_became_peered":"2023-09-23T21:19:30.636520+0000","last_unstale":"2023-09-23T21:19:32.631261+0000","last_undegraded":"2023-09-23T21:19:32.631261+0000","last_fullsized":"2023-09-23T21:19:32.631261+0000","mapping_epoch":650,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":651,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:36:06.289223+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":703,"reported_epoch":653,"state":"active+clean","last_fresh":"2023-09-23T21:19:32.628894+0000","last_change":"2023-09-23T21:19:30.648517+0000","last_active":"2023-09-23T21:19:32.628894+0000","last_peered":"2023-09-23T21:19:32.628894+0000","last_clean":"2023-09-23T21:19:32.628894+0000","last_became_active":"2023-09-23T21:19:30.648184+0000","last_became_peered":"2023-09-23T21:19:30.648184+0000","last_unstale":"2023-09-23T21:19:32.628894+0000","last_undegraded":"2023-09-23T21:19:32.628894+0000","last_fullsized":"2023-09-23T21:19:32.628894+0000","mapping_epoch":650,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":651,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:45:44.022190+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,2],"acting":[3,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"79'161","reported_seq":1051,"reported_epoch":653,"state":"active+clean","last_fresh":"2023-09-23T21:19:32.629110+0000","last_change":"2023-09-23T21:16:45.424773+0000","last_active":"2023-09-23T21:19:32.629110+0000","last_peered":"2023-09-23T21:19:32.629110+0000","last_clean":"2023-09-23T21:19:32.629110+0000","last_became_active":"2023-09-23T21:16:45.424458+0000","last_became_peered":"2023-09-23T21:16:45.424458+0000","last_unstale":"2023-09-23T21:19:32.629110+0000","last_undegraded":"2023-09-23T21:19:32.629110+0000","last_fullsized":"2023-09-23T21:19:32.629110+0000","mapping_epoch":567,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":568,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:25.096401+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:25.096401+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:40:17.871561+0000","scrub_duration":0.002859393,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":315,"reported_epoch":654,"state":"active+clean","last_fresh":"2023-09-23T21:19:33.633274+0000","last_change":"2023-09-23T21:19:30.642380+0000","last_active":"2023-09-23T21:19:33.633274+0000","last_peered":"2023-09-23T21:19:33.633274+0000","last_clean":"2023-09-23T21:19:33.633274+0000","last_became_active":"2023-09-23T21:19:30.642050+0000","last_became_peered":"2023-09-23T21:19:30.642050+0000","last_unstale":"2023-09-23T21:19:33.633274+0000","last_undegraded":"2023-09-23T21:19:33.633274+0000","last_fullsized":"2023-09-23T21:19:33.633274+0000","mapping_epoch":650,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":651,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:31:40.650598+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":658,"reported_epoch":653,"state":"active+clean","last_fresh":"2023-09-23T21:19:32.629267+0000","last_change":"2023-09-23T21:19:30.643841+0000","last_active":"2023-09-23T21:19:32.629267+0000","last_peered":"2023-09-23T21:19:32.629267+0000","last_clean":"2023-09-23T21:19:32.629267+0000","last_became_active":"2023-09-23T21:19:30.641826+0000","last_became_peered":"2023-09-23T21:19:30.641826+0000","last_unstale":"2023-09-23T21:19:32.629267+0000","last_undegraded":"2023-09-23T21:19:32.629267+0000","last_fullsized":"2023-09-23T21:19:32.629267+0000","mapping_epoch":650,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":651,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:36:08.403135+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.17","version":"79'161","reported_seq":818,"reported_epoch":654,"state":"active+clean","last_fresh":"2023-09-23T21:19:33.644079+0000","last_change":"2023-09-23T21:19:33.644079+0000","last_active":"2023-09-23T21:19:33.644079+0000","last_peered":"2023-09-23T21:19:33.644079+0000","last_clean":"2023-09-23T21:19:33.644079+0000","last_became_active":"2023-09-23T21:19:33.643672+0000","last_became_peered":"2023-09-23T21:19:33.643672+0000","last_unstale":"2023-09-23T21:19:33.644079+0000","last_undegraded":"2023-09-23T21:19:33.644079+0000","last_fullsized":"2023-09-23T21:19:33.644079+0000","mapping_epoch":653,"log_start":"0'0","ondisk_log_start":"0'0","created":479,"last_epoch_clean":654,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:15:50.325741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:15:50.325741+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:14:03.837900+0000","scrub_duration":0.0040500149999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":570,"reported_epoch":653,"state":"active+clean","last_fresh":"2023-09-23T21:19:32.631951+0000","last_change":"2023-09-23T21:17:02.182606+0000","last_active":"2023-09-23T21:19:32.631951+0000","last_peered":"2023-09-23T21:19:32.631951+0000","last_clean":"2023-09-23T21:19:32.631951+0000","last_became_active":"2023-09-23T21:16:42.446517+0000","last_became_peered":"2023-09-23T21:16:42.446517+0000","last_unstale":"2023-09-23T21:19:32.631951+0000","last_undegraded":"2023-09-23T21:19:32.631951+0000","last_fullsized":"2023-09-23T21:19:32.631951+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:46:08.337878+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":540,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,4],"acting":[3,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":352,"reported_epoch":653,"state":"active+clean","last_fresh":"2023-09-23T21:19:32.631950+0000","last_change":"2023-09-23T21:19:30.637412+0000","last_active":"2023-09-23T21:19:32.631950+0000","last_peered":"2023-09-23T21:19:32.631950+0000","last_clean":"2023-09-23T21:19:32.631950+0000","last_became_active":"2023-09-23T21:19:30.637099+0000","last_became_peered":"2023-09-23T21:19:30.637099+0000","last_unstale":"2023-09-23T21:19:32.631950+0000","last_undegraded":"2023-09-23T21:19:32.631950+0000","last_fullsized":"2023-09-23T21:19:32.631950+0000","mapping_epoch":650,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":651,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:59:58.577998+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,7],"acting":[2,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.16","version":"79'161","reported_seq":586,"reported_epoch":653,"state":"active+clean","last_fresh":"2023-09-23T21:19:32.629212+0000","last_change":"2023-09-23T21:15:46.326726+0000","last_active":"2023-09-23T21:19:32.629212+0000","last_peered":"2023-09-23T21:19:32.629212+0000","last_clean":"2023-09-23T21:19:32.629212+0000","last_became_active":"2023-09-23T21:13:33.770407+0000","last_became_peered":"2023-09-23T21:13:33.770407+0000","last_unstale":"2023-09-23T21:19:32.629212+0000","last_undegraded":"2023-09-23T21:19:32.629212+0000","last_fullsized":"2023-09-23T21:19:32.629212+0000","mapping_epoch":504,"log_start":"0'0","ondisk_log_start":"0'0","created":479,"last_epoch_clean":505,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:15:46.326589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:15:46.326589+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:50:23.810054+0000","scrub_duration":0.0062069059999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.14","version":"79'22","reported_seq":257,"reported_epoch":653,"state":"active+clean","last_fresh":"2023-09-23T21:19:32.626915+0000","last_change":"2023-09-23T21:17:02.198735+0000","last_active":"2023-09-23T21:19:32.626915+0000","last_peered":"2023-09-23T21:19:32.626915+0000","last_clean":"2023-09-23T21:19:32.626915+0000","last_became_active":"2023-09-23T21:16:42.492861+0000","last_became_peered":"2023-09-23T21:16:42.492861+0000","last_unstale":"2023-09-23T21:19:32.626915+0000","last_undegraded":"2023-09-23T21:19:32.626915+0000","last_fullsized":"2023-09-23T21:19:32.626915+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:00:09.949965+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":14202,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,2],"acting":[4,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":592,"reported_epoch":653,"state":"active+clean","last_fresh":"2023-09-23T21:19:32.627284+0000","last_change":"2023-09-23T21:19:30.638011+0000","last_active":"2023-09-23T21:19:32.627284+0000","last_peered":"2023-09-23T21:19:32.627284+0000","last_clean":"2023-09-23T21:19:32.627284+0000","last_became_active":"2023-09-23T21:19:30.636676+0000","last_became_peered":"2023-09-23T21:19:30.636676+0000","last_unstale":"2023-09-23T21:19:32.627284+0000","last_undegraded":"2023-09-23T21:19:32.627284+0000","last_fullsized":"2023-09-23T21:19:32.627284+0000","mapping_epoch":650,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":651,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:26:22.982323+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.11","version":"79'161","reported_seq":608,"reported_epoch":654,"state":"active+clean","last_fresh":"2023-09-23T21:19:33.646270+0000","last_change":"2023-09-23T21:19:33.646270+0000","last_active":"2023-09-23T21:19:33.646270+0000","last_peered":"2023-09-23T21:19:33.646270+0000","last_clean":"2023-09-23T21:19:33.646270+0000","last_became_active":"2023-09-23T21:19:33.645912+0000","last_became_peered":"2023-09-23T21:19:33.645912+0000","last_unstale":"2023-09-23T21:19:33.646270+0000","last_undegraded":"2023-09-23T21:19:33.646270+0000","last_fullsized":"2023-09-23T21:19:33.646270+0000","mapping_epoch":653,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":654,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:15:47.299947+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:15:47.299947+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:23:30.166866+0000","scrub_duration":0.0061353110000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":103,"reported_epoch":651,"state":"active+clean","last_fresh":"2023-09-23T21:19:30.627123+0000","last_change":"2023-09-23T21:16:42.531060+0000","last_active":"2023-09-23T21:19:30.627123+0000","last_peered":"2023-09-23T21:19:30.627123+0000","last_clean":"2023-09-23T21:19:30.627123+0000","last_became_active":"2023-09-23T21:16:42.449198+0000","last_became_peered":"2023-09-23T21:16:42.449198+0000","last_unstale":"2023-09-23T21:19:30.627123+0000","last_undegraded":"2023-09-23T21:19:30.627123+0000","last_fullsized":"2023-09-23T21:19:30.627123+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:30:23.428886+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":215,"reported_epoch":653,"state":"active+clean","last_fresh":"2023-09-23T21:19:32.628776+0000","last_change":"2023-09-23T21:16:42.491023+0000","last_active":"2023-09-23T21:19:32.628776+0000","last_peered":"2023-09-23T21:19:32.628776+0000","last_clean":"2023-09-23T21:19:32.628776+0000","last_became_active":"2023-09-23T21:16:42.489896+0000","last_became_peered":"2023-09-23T21:16:42.489896+0000","last_unstale":"2023-09-23T21:19:32.628776+0000","last_undegraded":"2023-09-23T21:19:32.628776+0000","last_fullsized":"2023-09-23T21:19:32.628776+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:58:54.036098+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,7],"acting":[2,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.10","version":"79'161","reported_seq":848,"reported_epoch":654,"state":"active+clean","last_fresh":"2023-09-23T21:19:33.639783+0000","last_change":"2023-09-23T21:19:33.639783+0000","last_active":"2023-09-23T21:19:33.639783+0000","last_peered":"2023-09-23T21:19:33.639783+0000","last_clean":"2023-09-23T21:19:33.639783+0000","last_became_active":"2023-09-23T21:19:33.639497+0000","last_became_peered":"2023-09-23T21:19:33.639497+0000","last_unstale":"2023-09-23T21:19:33.639783+0000","last_undegraded":"2023-09-23T21:19:33.639783+0000","last_fullsized":"2023-09-23T21:19:33.639783+0000","mapping_epoch":653,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":654,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:06:50.336992+0000","last_deep_scrub":"79'161","last_deep_scrub_stamp":"2023-09-23T21:06:50.336992+0000","last_clean_scrub_stamp":"2023-09-23T21:06:50.336992+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:39:29.427376+0000","scrub_duration":0.0026150930000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":614,"reported_epoch":653,"state":"active+clean","last_fresh":"2023-09-23T21:19:32.629625+0000","last_change":"2023-09-23T21:16:42.446324+0000","last_active":"2023-09-23T21:19:32.629625+0000","last_peered":"2023-09-23T21:19:32.629625+0000","last_clean":"2023-09-23T21:19:32.629625+0000","last_became_active":"2023-09-23T21:16:42.445997+0000","last_became_peered":"2023-09-23T21:16:42.445997+0000","last_unstale":"2023-09-23T21:19:32.629625+0000","last_undegraded":"2023-09-23T21:19:32.629625+0000","last_fullsized":"2023-09-23T21:19:32.629625+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:46:35.028292+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":435,"reported_epoch":653,"state":"active+clean","last_fresh":"2023-09-23T21:19:32.631803+0000","last_change":"2023-09-23T21:19:30.642090+0000","last_active":"2023-09-23T21:19:32.631803+0000","last_peered":"2023-09-23T21:19:32.631803+0000","last_clean":"2023-09-23T21:19:32.631803+0000","last_became_active":"2023-09-23T21:19:30.641761+0000","last_became_peered":"2023-09-23T21:19:30.641761+0000","last_unstale":"2023-09-23T21:19:32.631803+0000","last_undegraded":"2023-09-23T21:19:32.631803+0000","last_fullsized":"2023-09-23T21:19:32.631803+0000","mapping_epoch":650,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":651,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:20:06.738087+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.13","version":"79'161","reported_seq":733,"reported_epoch":653,"state":"active+clean","last_fresh":"2023-09-23T21:19:32.627590+0000","last_change":"2023-09-23T21:13:14.933096+0000","last_active":"2023-09-23T21:19:32.627590+0000","last_peered":"2023-09-23T21:19:32.627590+0000","last_clean":"2023-09-23T21:19:32.627590+0000","last_became_active":"2023-09-23T21:13:14.932703+0000","last_became_peered":"2023-09-23T21:13:14.932703+0000","last_unstale":"2023-09-23T21:19:32.627590+0000","last_undegraded":"2023-09-23T21:19:32.627590+0000","last_fullsized":"2023-09-23T21:19:32.627590+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:36.420362+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:36.420362+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:29:42.163911+0000","scrub_duration":0.028838444000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":665,"reported_epoch":653,"state":"active+clean","last_fresh":"2023-09-23T21:19:32.625707+0000","last_change":"2023-09-23T21:16:42.420467+0000","last_active":"2023-09-23T21:19:32.625707+0000","last_peered":"2023-09-23T21:19:32.625707+0000","last_clean":"2023-09-23T21:19:32.625707+0000","last_became_active":"2023-09-23T21:16:42.420149+0000","last_became_peered":"2023-09-23T21:16:42.420149+0000","last_unstale":"2023-09-23T21:19:32.625707+0000","last_undegraded":"2023-09-23T21:19:32.625707+0000","last_fullsized":"2023-09-23T21:19:32.625707+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:52:07.355000+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":183,"reported_epoch":653,"state":"active+clean","last_fresh":"2023-09-23T21:19:32.628901+0000","last_change":"2023-09-23T21:16:42.427663+0000","last_active":"2023-09-23T21:19:32.628901+0000","last_peered":"2023-09-23T21:19:32.628901+0000","last_clean":"2023-09-23T21:19:32.628901+0000","last_became_active":"2023-09-23T21:16:42.426961+0000","last_became_peered":"2023-09-23T21:16:42.426961+0000","last_unstale":"2023-09-23T21:19:32.628901+0000","last_undegraded":"2023-09-23T21:19:32.628901+0000","last_fullsized":"2023-09-23T21:19:32.628901+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:44:27.422007+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.12","version":"79'161","reported_seq":836,"reported_epoch":652,"state":"active+undersized+remapped","last_fresh":"2023-09-23T21:19:31.636489+0000","last_change":"2023-09-23T21:19:31.636390+0000","last_active":"2023-09-23T21:19:31.636489+0000","last_peered":"2023-09-23T21:19:31.636489+0000","last_clean":"2023-09-23T21:16:39.359385+0000","last_became_active":"2023-09-23T21:19:31.636390+0000","last_became_peered":"2023-09-23T21:19:31.636390+0000","last_unstale":"2023-09-23T21:19:31.636489+0000","last_undegraded":"2023-09-23T21:19:31.636489+0000","last_fullsized":"2023-09-23T21:19:31.629363+0000","mapping_epoch":651,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":568,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:05:56.785173+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:05:56.785173+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:31:09.814401+0000","scrub_duration":0.0042114370000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,2],"acting":[7,0],"avail_no_missing":["7","0","2","5"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":194,"reported_epoch":653,"state":"active+clean","last_fresh":"2023-09-23T21:19:32.627733+0000","last_change":"2023-09-23T21:19:30.658056+0000","last_active":"2023-09-23T21:19:32.627733+0000","last_peered":"2023-09-23T21:19:32.627733+0000","last_clean":"2023-09-23T21:19:32.627733+0000","last_became_active":"2023-09-23T21:19:30.636349+0000","last_became_peered":"2023-09-23T21:19:30.636349+0000","last_unstale":"2023-09-23T21:19:32.627733+0000","last_undegraded":"2023-09-23T21:19:32.627733+0000","last_fullsized":"2023-09-23T21:19:32.627733+0000","mapping_epoch":650,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":651,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:59:18.953067+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":511,"reported_epoch":653,"state":"active+clean","last_fresh":"2023-09-23T21:19:32.633033+0000","last_change":"2023-09-23T21:16:42.518512+0000","last_active":"2023-09-23T21:19:32.633033+0000","last_peered":"2023-09-23T21:19:32.633033+0000","last_clean":"2023-09-23T21:19:32.633033+0000","last_became_active":"2023-09-23T21:16:42.517464+0000","last_became_peered":"2023-09-23T21:16:42.517464+0000","last_unstale":"2023-09-23T21:19:32.633033+0000","last_undegraded":"2023-09-23T21:19:32.633033+0000","last_fullsized":"2023-09-23T21:19:32.633033+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:47:52.391167+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.d","version":"79'161","reported_seq":693,"reported_epoch":651,"state":"active+clean","last_fresh":"2023-09-23T21:19:30.626938+0000","last_change":"2023-09-23T21:18:47.440374+0000","last_active":"2023-09-23T21:19:30.626938+0000","last_peered":"2023-09-23T21:19:30.626938+0000","last_clean":"2023-09-23T21:19:30.626938+0000","last_became_active":"2023-09-23T21:16:45.416373+0000","last_became_peered":"2023-09-23T21:16:45.416373+0000","last_unstale":"2023-09-23T21:19:30.626938+0000","last_undegraded":"2023-09-23T21:19:30.626938+0000","last_fullsized":"2023-09-23T21:19:30.626938+0000","mapping_epoch":567,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":568,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:18:47.440263+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:18:47.440263+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:11:36.676449+0000","scrub_duration":0.0023849959999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":101,"reported_epoch":651,"state":"active+clean","last_fresh":"2023-09-23T21:19:30.631797+0000","last_change":"2023-09-23T21:17:02.190320+0000","last_active":"2023-09-23T21:19:30.631797+0000","last_peered":"2023-09-23T21:19:30.631797+0000","last_clean":"2023-09-23T21:19:30.631797+0000","last_became_active":"2023-09-23T21:16:42.451380+0000","last_became_peered":"2023-09-23T21:16:42.451380+0000","last_unstale":"2023-09-23T21:19:30.631797+0000","last_undegraded":"2023-09-23T21:19:30.631797+0000","last_fullsized":"2023-09-23T21:19:30.631797+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:48:58.425821+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":132,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":299,"reported_epoch":653,"state":"active+clean","last_fresh":"2023-09-23T21:19:32.625304+0000","last_change":"2023-09-23T21:19:30.635500+0000","last_active":"2023-09-23T21:19:32.625304+0000","last_peered":"2023-09-23T21:19:32.625304+0000","last_clean":"2023-09-23T21:19:32.625304+0000","last_became_active":"2023-09-23T21:19:30.635193+0000","last_became_peered":"2023-09-23T21:19:30.635193+0000","last_unstale":"2023-09-23T21:19:32.625304+0000","last_undegraded":"2023-09-23T21:19:32.625304+0000","last_fullsized":"2023-09-23T21:19:32.625304+0000","mapping_epoch":650,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":651,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:21:34.438775+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.c","version":"79'161","reported_seq":1036,"reported_epoch":653,"state":"active+clean","last_fresh":"2023-09-23T21:19:32.630309+0000","last_change":"2023-09-23T21:15:31.992756+0000","last_active":"2023-09-23T21:19:32.630309+0000","last_peered":"2023-09-23T21:19:32.630309+0000","last_clean":"2023-09-23T21:19:32.630309+0000","last_became_active":"2023-09-23T21:13:13.401526+0000","last_became_peered":"2023-09-23T21:13:13.401526+0000","last_unstale":"2023-09-23T21:19:32.630309+0000","last_undegraded":"2023-09-23T21:19:32.630309+0000","last_fullsized":"2023-09-23T21:19:32.630309+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":532,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:15:31.992598+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:15:31.992598+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:41:14.215543+0000","scrub_duration":0.002960678,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":101,"reported_epoch":651,"state":"active+clean","last_fresh":"2023-09-23T21:19:30.630474+0000","last_change":"2023-09-23T21:17:02.215996+0000","last_active":"2023-09-23T21:19:30.630474+0000","last_peered":"2023-09-23T21:19:30.630474+0000","last_clean":"2023-09-23T21:19:30.630474+0000","last_became_active":"2023-09-23T21:16:42.451205+0000","last_became_peered":"2023-09-23T21:16:42.451205+0000","last_unstale":"2023-09-23T21:19:30.630474+0000","last_undegraded":"2023-09-23T21:19:30.630474+0000","last_fullsized":"2023-09-23T21:19:30.630474+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:51:46.876281+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":616,"reported_epoch":653,"state":"active+clean","last_fresh":"2023-09-23T21:19:32.625787+0000","last_change":"2023-09-23T21:13:15.152792+0000","last_active":"2023-09-23T21:19:32.625787+0000","last_peered":"2023-09-23T21:19:32.625787+0000","last_clean":"2023-09-23T21:19:32.625787+0000","last_became_active":"2023-09-23T21:13:14.515084+0000","last_became_peered":"2023-09-23T21:13:14.515084+0000","last_unstale":"2023-09-23T21:19:32.625787+0000","last_undegraded":"2023-09-23T21:19:32.625787+0000","last_fullsized":"2023-09-23T21:19:32.625787+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:08:16.702072+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.f","version":"79'161","reported_seq":750,"reported_epoch":654,"state":"active+clean","last_fresh":"2023-09-23T21:19:33.640767+0000","last_change":"2023-09-23T21:19:33.640767+0000","last_active":"2023-09-23T21:19:33.640767+0000","last_peered":"2023-09-23T21:19:33.640767+0000","last_clean":"2023-09-23T21:19:33.640767+0000","last_became_active":"2023-09-23T21:19:33.640477+0000","last_became_peered":"2023-09-23T21:19:33.640477+0000","last_unstale":"2023-09-23T21:19:33.640767+0000","last_undegraded":"2023-09-23T21:19:33.640767+0000","last_fullsized":"2023-09-23T21:19:33.640767+0000","mapping_epoch":653,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":654,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:27.880776+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:27.880776+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:40:30.960460+0000","scrub_duration":0.0014004899999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":17,"reported_epoch":653,"state":"active+clean","last_fresh":"2023-09-23T21:19:32.626272+0000","last_change":"2023-09-23T21:19:30.662735+0000","last_active":"2023-09-23T21:19:32.626272+0000","last_peered":"2023-09-23T21:19:32.626272+0000","last_clean":"2023-09-23T21:19:32.626272+0000","last_became_active":"2023-09-23T21:19:30.635656+0000","last_became_peered":"2023-09-23T21:19:30.635656+0000","last_unstale":"2023-09-23T21:19:32.626272+0000","last_undegraded":"2023-09-23T21:19:32.626272+0000","last_fullsized":"2023-09-23T21:19:32.626272+0000","mapping_epoch":650,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":651,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:51:48.290452+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":140,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":530,"reported_epoch":653,"state":"active+clean","last_fresh":"2023-09-23T21:19:32.627668+0000","last_change":"2023-09-23T21:19:30.639010+0000","last_active":"2023-09-23T21:19:32.627668+0000","last_peered":"2023-09-23T21:19:32.627668+0000","last_clean":"2023-09-23T21:19:32.627668+0000","last_became_active":"2023-09-23T21:19:30.638768+0000","last_became_peered":"2023-09-23T21:19:30.638768+0000","last_unstale":"2023-09-23T21:19:32.627668+0000","last_undegraded":"2023-09-23T21:19:32.627668+0000","last_fullsized":"2023-09-23T21:19:32.627668+0000","mapping_epoch":650,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":651,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:48:45.915425+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.e","version":"79'161","reported_seq":872,"reported_epoch":653,"state":"active+clean","last_fresh":"2023-09-23T21:19:32.627482+0000","last_change":"2023-09-23T21:16:45.433204+0000","last_active":"2023-09-23T21:19:32.627482+0000","last_peered":"2023-09-23T21:19:32.627482+0000","last_clean":"2023-09-23T21:19:32.627482+0000","last_became_active":"2023-09-23T21:16:45.432836+0000","last_became_peered":"2023-09-23T21:16:45.432836+0000","last_unstale":"2023-09-23T21:19:32.627482+0000","last_undegraded":"2023-09-23T21:19:32.627482+0000","last_fullsized":"2023-09-23T21:19:32.627482+0000","mapping_epoch":567,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":568,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:15:39.046962+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:15:39.046962+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:11:20.720731+0000","scrub_duration":0.0082638769999999993,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":462,"reported_epoch":653,"state":"active+clean","last_fresh":"2023-09-23T21:19:32.630959+0000","last_change":"2023-09-23T21:19:30.634873+0000","last_active":"2023-09-23T21:19:32.630959+0000","last_peered":"2023-09-23T21:19:32.630959+0000","last_clean":"2023-09-23T21:19:32.630959+0000","last_became_active":"2023-09-23T21:19:30.634549+0000","last_became_peered":"2023-09-23T21:19:30.634549+0000","last_unstale":"2023-09-23T21:19:32.630959+0000","last_undegraded":"2023-09-23T21:19:32.630959+0000","last_fullsized":"2023-09-23T21:19:32.630959+0000","mapping_epoch":650,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":651,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:51:20.192644+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,2],"acting":[3,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":628,"reported_epoch":653,"state":"active+clean","last_fresh":"2023-09-23T21:19:32.632856+0000","last_change":"2023-09-23T21:13:14.339670+0000","last_active":"2023-09-23T21:19:32.632856+0000","last_peered":"2023-09-23T21:19:32.632856+0000","last_clean":"2023-09-23T21:19:32.632856+0000","last_became_active":"2023-09-23T21:13:14.339341+0000","last_became_peered":"2023-09-23T21:13:14.339341+0000","last_unstale":"2023-09-23T21:19:32.632856+0000","last_undegraded":"2023-09-23T21:19:32.632856+0000","last_fullsized":"2023-09-23T21:19:32.632856+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:23:19.411368+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.9","version":"79'161","reported_seq":856,"reported_epoch":653,"state":"active+clean","last_fresh":"2023-09-23T21:19:32.627845+0000","last_change":"2023-09-23T21:16:45.433968+0000","last_active":"2023-09-23T21:19:32.627845+0000","last_peered":"2023-09-23T21:19:32.627845+0000","last_clean":"2023-09-23T21:19:32.627845+0000","last_became_active":"2023-09-23T21:16:45.433602+0000","last_became_peered":"2023-09-23T21:16:45.433602+0000","last_unstale":"2023-09-23T21:19:32.627845+0000","last_undegraded":"2023-09-23T21:19:32.627845+0000","last_fullsized":"2023-09-23T21:19:32.627845+0000","mapping_epoch":567,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":568,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:15:43.437369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:15:43.437369+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:16:44.538102+0000","scrub_duration":0.0024129429999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,3],"acting":[2,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":214,"reported_epoch":653,"state":"active+clean","last_fresh":"2023-09-23T21:19:32.633063+0000","last_change":"2023-09-23T21:13:30.793273+0000","last_active":"2023-09-23T21:19:32.633063+0000","last_peered":"2023-09-23T21:19:32.633063+0000","last_clean":"2023-09-23T21:19:32.633063+0000","last_became_active":"2023-09-23T21:13:30.791269+0000","last_became_peered":"2023-09-23T21:13:30.791269+0000","last_unstale":"2023-09-23T21:19:32.633063+0000","last_undegraded":"2023-09-23T21:19:32.633063+0000","last_fullsized":"2023-09-23T21:19:32.633063+0000","mapping_epoch":501,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:38:07.142290+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":247,"reported_epoch":653,"state":"active+clean","last_fresh":"2023-09-23T21:19:32.626651+0000","last_change":"2023-09-23T21:16:42.417057+0000","last_active":"2023-09-23T21:19:32.626651+0000","last_peered":"2023-09-23T21:19:32.626651+0000","last_clean":"2023-09-23T21:19:32.626651+0000","last_became_active":"2023-09-23T21:16:42.416601+0000","last_became_peered":"2023-09-23T21:16:42.416601+0000","last_unstale":"2023-09-23T21:19:32.626651+0000","last_undegraded":"2023-09-23T21:19:32.626651+0000","last_fullsized":"2023-09-23T21:19:32.626651+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:56:52.167562+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,7],"acting":[4,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.8","version":"79'161","reported_seq":805,"reported_epoch":654,"state":"active+clean","last_fresh":"2023-09-23T21:19:33.642569+0000","last_change":"2023-09-23T21:19:33.642569+0000","last_active":"2023-09-23T21:19:33.642569+0000","last_peered":"2023-09-23T21:19:33.642569+0000","last_clean":"2023-09-23T21:19:33.642569+0000","last_became_active":"2023-09-23T21:19:33.641967+0000","last_became_peered":"2023-09-23T21:19:33.641967+0000","last_unstale":"2023-09-23T21:19:33.642569+0000","last_undegraded":"2023-09-23T21:19:33.642569+0000","last_fullsized":"2023-09-23T21:19:33.642569+0000","mapping_epoch":653,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":654,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:15:36.367640+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:15:36.367640+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:52:22.742871+0000","scrub_duration":0.005826899,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":240,"reported_epoch":653,"state":"active+clean","last_fresh":"2023-09-23T21:19:32.626541+0000","last_change":"2023-09-23T21:19:30.637441+0000","last_active":"2023-09-23T21:19:32.626541+0000","last_peered":"2023-09-23T21:19:32.626541+0000","last_clean":"2023-09-23T21:19:32.626541+0000","last_became_active":"2023-09-23T21:19:30.635390+0000","last_became_peered":"2023-09-23T21:19:30.635390+0000","last_unstale":"2023-09-23T21:19:32.626541+0000","last_undegraded":"2023-09-23T21:19:32.626541+0000","last_fullsized":"2023-09-23T21:19:32.626541+0000","mapping_epoch":650,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":651,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:36:18.999002+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,7],"acting":[4,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":231,"reported_epoch":653,"state":"active+clean","last_fresh":"2023-09-23T21:19:32.627355+0000","last_change":"2023-09-23T21:13:30.772451+0000","last_active":"2023-09-23T21:19:32.627355+0000","last_peered":"2023-09-23T21:19:32.627355+0000","last_clean":"2023-09-23T21:19:32.627355+0000","last_became_active":"2023-09-23T21:13:30.772040+0000","last_became_peered":"2023-09-23T21:13:30.772040+0000","last_unstale":"2023-09-23T21:19:32.627355+0000","last_undegraded":"2023-09-23T21:19:32.627355+0000","last_fullsized":"2023-09-23T21:19:32.627355+0000","mapping_epoch":501,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:24:49.246727+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,0],"acting":[4,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.b","version":"79'161","reported_seq":688,"reported_epoch":654,"state":"active+clean","last_fresh":"2023-09-23T21:19:33.645366+0000","last_change":"2023-09-23T21:19:33.645366+0000","last_active":"2023-09-23T21:19:33.645366+0000","last_peered":"2023-09-23T21:19:33.645366+0000","last_clean":"2023-09-23T21:19:33.645366+0000","last_became_active":"2023-09-23T21:19:33.644992+0000","last_became_peered":"2023-09-23T21:19:33.644992+0000","last_unstale":"2023-09-23T21:19:33.645366+0000","last_undegraded":"2023-09-23T21:19:33.645366+0000","last_fullsized":"2023-09-23T21:19:33.645366+0000","mapping_epoch":653,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":654,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:15:36.339885+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:15:36.339885+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:23:18.575298+0000","scrub_duration":0.0068473279999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,3],"acting":[0,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":231,"reported_epoch":653,"state":"active+clean","last_fresh":"2023-09-23T21:19:32.625621+0000","last_change":"2023-09-23T21:16:42.425103+0000","last_active":"2023-09-23T21:19:32.625621+0000","last_peered":"2023-09-23T21:19:32.625621+0000","last_clean":"2023-09-23T21:19:32.625621+0000","last_became_active":"2023-09-23T21:16:42.424543+0000","last_became_peered":"2023-09-23T21:16:42.424543+0000","last_unstale":"2023-09-23T21:19:32.625621+0000","last_undegraded":"2023-09-23T21:19:32.625621+0000","last_fullsized":"2023-09-23T21:19:32.625621+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:11:28.885246+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":595,"reported_epoch":653,"state":"active+clean","last_fresh":"2023-09-23T21:19:32.627011+0000","last_change":"2023-09-23T21:19:30.635633+0000","last_active":"2023-09-23T21:19:32.627011+0000","last_peered":"2023-09-23T21:19:32.627011+0000","last_clean":"2023-09-23T21:19:32.627011+0000","last_became_active":"2023-09-23T21:19:30.635295+0000","last_became_peered":"2023-09-23T21:19:30.635295+0000","last_unstale":"2023-09-23T21:19:32.627011+0000","last_undegraded":"2023-09-23T21:19:32.627011+0000","last_fullsized":"2023-09-23T21:19:32.627011+0000","mapping_epoch":650,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":651,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:17:02.473642+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,7],"acting":[4,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.a","version":"79'161","reported_seq":831,"reported_epoch":654,"state":"active+clean","last_fresh":"2023-09-23T21:19:33.639230+0000","last_change":"2023-09-23T21:19:33.639230+0000","last_active":"2023-09-23T21:19:33.639230+0000","last_peered":"2023-09-23T21:19:33.639230+0000","last_clean":"2023-09-23T21:19:33.639230+0000","last_became_active":"2023-09-23T21:19:33.638876+0000","last_became_peered":"2023-09-23T21:19:33.638876+0000","last_unstale":"2023-09-23T21:19:33.639230+0000","last_undegraded":"2023-09-23T21:19:33.639230+0000","last_fullsized":"2023-09-23T21:19:33.639230+0000","mapping_epoch":653,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":654,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:15:46.420650+0000","last_deep_scrub":"79'161","last_deep_scrub_stamp":"2023-09-23T21:05:32.630292+0000","last_clean_scrub_stamp":"2023-09-23T21:15:46.420650+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:55:09.974123+0000","scrub_duration":0.0072898930000000004,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,0],"acting":[2,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":521,"reported_epoch":653,"state":"active+clean","last_fresh":"2023-09-23T21:19:32.628102+0000","last_change":"2023-09-23T21:16:42.447403+0000","last_active":"2023-09-23T21:19:32.628102+0000","last_peered":"2023-09-23T21:19:32.628102+0000","last_clean":"2023-09-23T21:19:32.628102+0000","last_became_active":"2023-09-23T21:16:42.447080+0000","last_became_peered":"2023-09-23T21:16:42.447080+0000","last_unstale":"2023-09-23T21:19:32.628102+0000","last_undegraded":"2023-09-23T21:19:32.628102+0000","last_fullsized":"2023-09-23T21:19:32.628102+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:15:19.446334+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,4],"acting":[3,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":652,"reported_epoch":653,"state":"active+clean","last_fresh":"2023-09-23T21:19:32.633247+0000","last_change":"2023-09-23T21:16:42.518739+0000","last_active":"2023-09-23T21:19:32.633247+0000","last_peered":"2023-09-23T21:19:32.633247+0000","last_clean":"2023-09-23T21:19:32.633247+0000","last_became_active":"2023-09-23T21:16:42.517945+0000","last_became_peered":"2023-09-23T21:16:42.517945+0000","last_unstale":"2023-09-23T21:19:32.633247+0000","last_undegraded":"2023-09-23T21:19:32.633247+0000","last_fullsized":"2023-09-23T21:19:32.633247+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:47:26.236933+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.7","version":"79'161","reported_seq":811,"reported_epoch":654,"state":"active+clean","last_fresh":"2023-09-23T21:19:33.645253+0000","last_change":"2023-09-23T21:19:33.645253+0000","last_active":"2023-09-23T21:19:33.645253+0000","last_peered":"2023-09-23T21:19:33.645253+0000","last_clean":"2023-09-23T21:19:33.645253+0000","last_became_active":"2023-09-23T21:19:33.644706+0000","last_became_peered":"2023-09-23T21:19:33.644706+0000","last_unstale":"2023-09-23T21:19:33.645253+0000","last_undegraded":"2023-09-23T21:19:33.645253+0000","last_fullsized":"2023-09-23T21:19:33.645253+0000","mapping_epoch":653,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":654,"parent":"0.0","parent_split_bits":3,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:15:51.016427+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:15:51.016427+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:01:01.623826+0000","scrub_duration":0.0017648640000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.4","version":"654'25","reported_seq":711,"reported_epoch":654,"state":"active+clean","last_fresh":"2023-09-23T21:19:34.340226+0000","last_change":"2023-09-23T21:19:32.633792+0000","last_active":"2023-09-23T21:19:34.340226+0000","last_peered":"2023-09-23T21:19:34.340226+0000","last_clean":"2023-09-23T21:19:34.340226+0000","last_became_active":"2023-09-23T21:16:42.518488+0000","last_became_peered":"2023-09-23T21:16:42.518488+0000","last_unstale":"2023-09-23T21:19:34.340226+0000","last_undegraded":"2023-09-23T21:19:34.340226+0000","last_fullsized":"2023-09-23T21:19:34.340226+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":25,"log_dups_size":0,"ondisk_log_size":25,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:42:44.144383+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00076299600000000005,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":170,"num_read_kb":295,"num_write":15,"num_write_kb":74,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,2],"acting":[0,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"3.5","version":"0'0","reported_seq":145,"reported_epoch":654,"state":"active+clean","last_fresh":"2023-09-23T21:19:33.634953+0000","last_change":"2023-09-23T21:19:30.643393+0000","last_active":"2023-09-23T21:19:33.634953+0000","last_peered":"2023-09-23T21:19:33.634953+0000","last_clean":"2023-09-23T21:19:33.634953+0000","last_became_active":"2023-09-23T21:19:30.643100+0000","last_became_peered":"2023-09-23T21:19:30.643100+0000","last_unstale":"2023-09-23T21:19:33.634953+0000","last_undegraded":"2023-09-23T21:19:33.634953+0000","last_fullsized":"2023-09-23T21:19:33.634953+0000","mapping_epoch":650,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":651,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:38:20.831056+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,7],"acting":[3,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":511,"reported_epoch":653,"state":"active+clean","last_fresh":"2023-09-23T21:19:32.625542+0000","last_change":"2023-09-23T21:19:30.636897+0000","last_active":"2023-09-23T21:19:32.625542+0000","last_peered":"2023-09-23T21:19:32.625542+0000","last_clean":"2023-09-23T21:19:32.625542+0000","last_became_active":"2023-09-23T21:19:30.635812+0000","last_became_peered":"2023-09-23T21:19:30.635812+0000","last_unstale":"2023-09-23T21:19:32.625542+0000","last_undegraded":"2023-09-23T21:19:32.625542+0000","last_fullsized":"2023-09-23T21:19:32.625542+0000","mapping_epoch":650,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":651,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:17:40.371897+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,2],"acting":[4,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1","version":"79'161","reported_seq":756,"reported_epoch":654,"state":"active+clean","last_fresh":"2023-09-23T21:19:33.638597+0000","last_change":"2023-09-23T21:19:33.638597+0000","last_active":"2023-09-23T21:19:33.638597+0000","last_peered":"2023-09-23T21:19:33.638597+0000","last_clean":"2023-09-23T21:19:33.638597+0000","last_became_active":"2023-09-23T21:19:33.638177+0000","last_became_peered":"2023-09-23T21:19:33.638177+0000","last_unstale":"2023-09-23T21:19:33.638597+0000","last_undegraded":"2023-09-23T21:19:33.638597+0000","last_fullsized":"2023-09-23T21:19:33.638597+0000","mapping_epoch":653,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":654,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:15:34.345138+0000","last_deep_scrub":"79'161","last_deep_scrub_stamp":"2023-09-23T21:05:30.652189+0000","last_clean_scrub_stamp":"2023-09-23T21:15:34.345138+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:02:45.462934+0000","scrub_duration":0.0057520000000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"653'2","reported_seq":345,"reported_epoch":654,"state":"active+clean","last_fresh":"2023-09-23T21:19:34.305726+0000","last_change":"2023-09-23T21:19:32.630097+0000","last_active":"2023-09-23T21:19:34.305726+0000","last_peered":"2023-09-23T21:19:34.305726+0000","last_clean":"2023-09-23T21:19:34.305726+0000","last_became_active":"2023-09-23T21:19:30.638661+0000","last_became_peered":"2023-09-23T21:19:30.638661+0000","last_unstale":"2023-09-23T21:19:34.305726+0000","last_undegraded":"2023-09-23T21:19:34.305726+0000","last_fullsized":"2023-09-23T21:19:34.305726+0000","mapping_epoch":650,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":651,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:36:27.438599+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000311835,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":7,"num_read_kb":7,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"3.3","version":"0'0","reported_seq":597,"reported_epoch":653,"state":"active+clean","last_fresh":"2023-09-23T21:19:32.627443+0000","last_change":"2023-09-23T21:19:30.638123+0000","last_active":"2023-09-23T21:19:32.627443+0000","last_peered":"2023-09-23T21:19:32.627443+0000","last_clean":"2023-09-23T21:19:32.627443+0000","last_became_active":"2023-09-23T21:19:30.637749+0000","last_became_peered":"2023-09-23T21:19:30.637749+0000","last_unstale":"2023-09-23T21:19:32.627443+0000","last_undegraded":"2023-09-23T21:19:32.627443+0000","last_fullsized":"2023-09-23T21:19:32.627443+0000","mapping_epoch":650,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":651,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:35:54.298068+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":494,"reported_epoch":653,"state":"active+clean","last_fresh":"2023-09-23T21:19:32.629024+0000","last_change":"2023-09-23T21:16:42.491111+0000","last_active":"2023-09-23T21:19:32.629024+0000","last_peered":"2023-09-23T21:19:32.629024+0000","last_clean":"2023-09-23T21:19:32.629024+0000","last_became_active":"2023-09-23T21:16:42.490275+0000","last_became_peered":"2023-09-23T21:16:42.490275+0000","last_unstale":"2023-09-23T21:19:32.629024+0000","last_undegraded":"2023-09-23T21:19:32.629024+0000","last_fullsized":"2023-09-23T21:19:32.629024+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:39:11.652564+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,7],"acting":[2,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.2","version":"79'161","reported_seq":970,"reported_epoch":653,"state":"active+clean","last_fresh":"2023-09-23T21:19:32.628376+0000","last_change":"2023-09-23T21:16:45.417075+0000","last_active":"2023-09-23T21:19:32.628376+0000","last_peered":"2023-09-23T21:19:32.628376+0000","last_clean":"2023-09-23T21:19:32.628376+0000","last_became_active":"2023-09-23T21:16:45.416752+0000","last_became_peered":"2023-09-23T21:16:45.416752+0000","last_unstale":"2023-09-23T21:19:32.628376+0000","last_undegraded":"2023-09-23T21:19:32.628376+0000","last_fullsized":"2023-09-23T21:19:32.628376+0000","mapping_epoch":567,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":568,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:24.802451+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:24.802451+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:01:51.129779+0000","scrub_duration":0.0090861250000000005,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,4],"acting":[0,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"654'6","reported_seq":685,"reported_epoch":654,"state":"active+clean","last_fresh":"2023-09-23T21:19:36.453014+0000","last_change":"2023-09-23T21:19:32.632050+0000","last_active":"2023-09-23T21:19:36.453014+0000","last_peered":"2023-09-23T21:19:36.453014+0000","last_clean":"2023-09-23T21:19:36.453014+0000","last_became_active":"2023-09-23T21:13:12.206650+0000","last_became_peered":"2023-09-23T21:13:12.206650+0000","last_unstale":"2023-09-23T21:19:36.453014+0000","last_undegraded":"2023-09-23T21:19:36.453014+0000","last_fullsized":"2023-09-23T21:19:36.453014+0000","mapping_epoch":493,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":0,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:10:01.892936+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0034996430000000002,"stat_sum":{"num_bytes":783608,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":20,"num_read_kb":195,"num_write":5,"num_write_kb":192,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"3.0","version":"0'0","reported_seq":657,"reported_epoch":653,"state":"active+clean","last_fresh":"2023-09-23T21:19:32.627692+0000","last_change":"2023-09-23T21:19:30.634282+0000","last_active":"2023-09-23T21:19:32.627692+0000","last_peered":"2023-09-23T21:19:32.627692+0000","last_clean":"2023-09-23T21:19:32.627692+0000","last_became_active":"2023-09-23T21:19:30.633882+0000","last_became_peered":"2023-09-23T21:19:30.633882+0000","last_unstale":"2023-09-23T21:19:32.627692+0000","last_undegraded":"2023-09-23T21:19:32.627692+0000","last_fullsized":"2023-09-23T21:19:32.627692+0000","mapping_epoch":650,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":651,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:59:27.972139+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,3],"acting":[2,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":192,"reported_epoch":653,"state":"active+clean","last_fresh":"2023-09-23T21:19:32.631171+0000","last_change":"2023-09-23T21:19:30.639918+0000","last_active":"2023-09-23T21:19:32.631171+0000","last_peered":"2023-09-23T21:19:32.631171+0000","last_clean":"2023-09-23T21:19:32.631171+0000","last_became_active":"2023-09-23T21:19:30.639573+0000","last_became_peered":"2023-09-23T21:19:30.639573+0000","last_unstale":"2023-09-23T21:19:32.631171+0000","last_undegraded":"2023-09-23T21:19:32.631171+0000","last_fullsized":"2023-09-23T21:19:32.631171+0000","mapping_epoch":650,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":651,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:11:50.254704+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.3","version":"654'21","reported_seq":502,"reported_epoch":654,"state":"active+clean","last_fresh":"2023-09-23T21:19:34.344819+0000","last_change":"2023-09-23T21:19:32.632379+0000","last_active":"2023-09-23T21:19:34.344819+0000","last_peered":"2023-09-23T21:19:34.344819+0000","last_clean":"2023-09-23T21:19:34.344819+0000","last_became_active":"2023-09-23T21:19:30.634381+0000","last_became_peered":"2023-09-23T21:19:30.634381+0000","last_unstale":"2023-09-23T21:19:34.344819+0000","last_undegraded":"2023-09-23T21:19:34.344819+0000","last_fullsized":"2023-09-23T21:19:34.344819+0000","mapping_epoch":650,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":651,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":21,"log_dups_size":0,"ondisk_log_size":21,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:13:45.243362+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.002661049,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":59,"num_read_kb":47,"num_write":26,"num_write_kb":16,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,3],"acting":[2,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"1.0","version":"79'161","reported_seq":873,"reported_epoch":654,"state":"active+clean","last_fresh":"2023-09-23T21:19:33.632878+0000","last_change":"2023-09-23T21:16:45.428032+0000","last_active":"2023-09-23T21:19:33.632878+0000","last_peered":"2023-09-23T21:19:33.632878+0000","last_clean":"2023-09-23T21:19:33.632878+0000","last_became_active":"2023-09-23T21:16:45.427674+0000","last_became_peered":"2023-09-23T21:16:45.427674+0000","last_unstale":"2023-09-23T21:19:33.632878+0000","last_undegraded":"2023-09-23T21:19:33.632878+0000","last_fullsized":"2023-09-23T21:19:33.632878+0000","mapping_epoch":567,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":568,"parent":"0.0","parent_split_bits":0,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:20.738273+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:20.738273+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:39:42.441711+0000","scrub_duration":0.0088336339999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,2],"acting":[0,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":458,"reported_epoch":653,"state":"active+clean","last_fresh":"2023-09-23T21:19:32.626667+0000","last_change":"2023-09-23T21:19:30.635491+0000","last_active":"2023-09-23T21:19:32.626667+0000","last_peered":"2023-09-23T21:19:32.626667+0000","last_clean":"2023-09-23T21:19:32.626667+0000","last_became_active":"2023-09-23T21:19:30.635158+0000","last_became_peered":"2023-09-23T21:19:30.635158+0000","last_unstale":"2023-09-23T21:19:32.626667+0000","last_undegraded":"2023-09-23T21:19:32.626667+0000","last_fullsized":"2023-09-23T21:19:32.626667+0000","mapping_epoch":650,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":651,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:13:44.654497+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,7],"acting":[4,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":248,"reported_epoch":653,"state":"active+clean","last_fresh":"2023-09-23T21:19:32.627444+0000","last_change":"2023-09-23T21:16:42.424717+0000","last_active":"2023-09-23T21:19:32.627444+0000","last_peered":"2023-09-23T21:19:32.627444+0000","last_clean":"2023-09-23T21:19:32.627444+0000","last_became_active":"2023-09-23T21:16:42.424376+0000","last_became_peered":"2023-09-23T21:16:42.424376+0000","last_unstale":"2023-09-23T21:19:32.627444+0000","last_undegraded":"2023-09-23T21:19:32.627444+0000","last_fullsized":"2023-09-23T21:19:32.627444+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:25:14.162321+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.3","version":"79'161","reported_seq":976,"reported_epoch":654,"state":"active+clean","last_fresh":"2023-09-23T21:19:33.643576+0000","last_change":"2023-09-23T21:19:33.643576+0000","last_active":"2023-09-23T21:19:33.643576+0000","last_peered":"2023-09-23T21:19:33.643576+0000","last_clean":"2023-09-23T21:19:33.643576+0000","last_became_active":"2023-09-23T21:19:33.643287+0000","last_became_peered":"2023-09-23T21:19:33.643287+0000","last_unstale":"2023-09-23T21:19:33.643576+0000","last_undegraded":"2023-09-23T21:19:33.643576+0000","last_fullsized":"2023-09-23T21:19:33.643576+0000","mapping_epoch":653,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":654,"parent":"0.0","parent_split_bits":3,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:05:57.832302+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:05:57.832302+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:16:10.374830+0000","scrub_duration":0.010206847999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.0","version":"651'1","reported_seq":496,"reported_epoch":653,"state":"active+clean","last_fresh":"2023-09-23T21:19:32.628167+0000","last_change":"2023-09-23T21:19:32.628167+0000","last_active":"2023-09-23T21:19:32.628167+0000","last_peered":"2023-09-23T21:19:32.628167+0000","last_clean":"2023-09-23T21:19:32.628167+0000","last_became_active":"2023-09-23T21:16:42.452829+0000","last_became_peered":"2023-09-23T21:16:42.452829+0000","last_unstale":"2023-09-23T21:19:32.628167+0000","last_undegraded":"2023-09-23T21:19:32.628167+0000","last_fullsized":"2023-09-23T21:19:32.628167+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:49:26.303389+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000356566,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"3.1","version":"0'0","reported_seq":655,"reported_epoch":653,"state":"active+clean","last_fresh":"2023-09-23T21:19:32.628986+0000","last_change":"2023-09-23T21:13:44.409212+0000","last_active":"2023-09-23T21:19:32.628986+0000","last_peered":"2023-09-23T21:19:32.628986+0000","last_clean":"2023-09-23T21:19:32.628986+0000","last_became_active":"2023-09-23T21:13:44.408958+0000","last_became_peered":"2023-09-23T21:13:44.408958+0000","last_unstale":"2023-09-23T21:19:32.628986+0000","last_undegraded":"2023-09-23T21:19:32.628986+0000","last_fullsized":"2023-09-23T21:19:32.628986+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:03:52.749964+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":183,"reported_epoch":653,"state":"active+clean","last_fresh":"2023-09-23T21:19:32.629113+0000","last_change":"2023-09-23T21:16:42.427490+0000","last_active":"2023-09-23T21:19:32.629113+0000","last_peered":"2023-09-23T21:19:32.629113+0000","last_clean":"2023-09-23T21:19:32.629113+0000","last_became_active":"2023-09-23T21:16:42.426566+0000","last_became_peered":"2023-09-23T21:16:42.426566+0000","last_unstale":"2023-09-23T21:19:32.629113+0000","last_undegraded":"2023-09-23T21:19:32.629113+0000","last_fullsized":"2023-09-23T21:19:32.629113+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:20:26.396005+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,7],"acting":[0,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.6","version":"79'161","reported_seq":586,"reported_epoch":653,"state":"active+clean","last_fresh":"2023-09-23T21:19:32.629022+0000","last_change":"2023-09-23T21:15:43.333919+0000","last_active":"2023-09-23T21:19:32.629022+0000","last_peered":"2023-09-23T21:19:32.629022+0000","last_clean":"2023-09-23T21:19:32.629022+0000","last_became_active":"2023-09-23T21:13:33.769303+0000","last_became_peered":"2023-09-23T21:13:33.769303+0000","last_unstale":"2023-09-23T21:19:32.629022+0000","last_undegraded":"2023-09-23T21:19:32.629022+0000","last_fullsized":"2023-09-23T21:19:32.629022+0000","mapping_epoch":504,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":505,"parent":"0.0","parent_split_bits":3,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:15:43.333733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:15:43.333733+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:42:29.787357+0000","scrub_duration":0.0092107530000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":97,"reported_epoch":651,"state":"active+clean","last_fresh":"2023-09-23T21:19:30.626839+0000","last_change":"2023-09-23T21:16:42.483813+0000","last_active":"2023-09-23T21:19:30.626839+0000","last_peered":"2023-09-23T21:19:30.626839+0000","last_clean":"2023-09-23T21:19:30.626839+0000","last_became_active":"2023-09-23T21:16:42.482847+0000","last_became_peered":"2023-09-23T21:16:42.482847+0000","last_unstale":"2023-09-23T21:19:30.626839+0000","last_undegraded":"2023-09-23T21:19:30.626839+0000","last_fullsized":"2023-09-23T21:19:30.626839+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:38:57.905679+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":548,"reported_epoch":653,"state":"active+clean","last_fresh":"2023-09-23T21:19:32.629280+0000","last_change":"2023-09-23T21:19:30.642771+0000","last_active":"2023-09-23T21:19:32.629280+0000","last_peered":"2023-09-23T21:19:32.629280+0000","last_clean":"2023-09-23T21:19:32.629280+0000","last_became_active":"2023-09-23T21:19:30.642356+0000","last_became_peered":"2023-09-23T21:19:30.642356+0000","last_unstale":"2023-09-23T21:19:32.629280+0000","last_undegraded":"2023-09-23T21:19:32.629280+0000","last_fullsized":"2023-09-23T21:19:32.629280+0000","mapping_epoch":650,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":651,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:54:42.234332+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,4],"acting":[2,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":629,"reported_epoch":653,"state":"active+clean","last_fresh":"2023-09-23T21:19:32.633078+0000","last_change":"2023-09-23T21:19:30.639797+0000","last_active":"2023-09-23T21:19:32.633078+0000","last_peered":"2023-09-23T21:19:32.633078+0000","last_clean":"2023-09-23T21:19:32.633078+0000","last_became_active":"2023-09-23T21:19:30.639468+0000","last_became_peered":"2023-09-23T21:19:30.639468+0000","last_unstale":"2023-09-23T21:19:32.633078+0000","last_undegraded":"2023-09-23T21:19:32.633078+0000","last_fullsized":"2023-09-23T21:19:32.633078+0000","mapping_epoch":650,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":651,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:50:42.845762+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,2],"acting":[0,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.5","version":"79'161","reported_seq":711,"reported_epoch":654,"state":"active+clean","last_fresh":"2023-09-23T21:19:33.639639+0000","last_change":"2023-09-23T21:19:33.639639+0000","last_active":"2023-09-23T21:19:33.639639+0000","last_peered":"2023-09-23T21:19:33.639639+0000","last_clean":"2023-09-23T21:19:33.639639+0000","last_became_active":"2023-09-23T21:19:33.639275+0000","last_became_peered":"2023-09-23T21:19:33.639275+0000","last_unstale":"2023-09-23T21:19:33.639639+0000","last_undegraded":"2023-09-23T21:19:33.639639+0000","last_fullsized":"2023-09-23T21:19:33.639639+0000","mapping_epoch":653,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":654,"parent":"0.0","parent_split_bits":3,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:15:41.108579+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:15:41.108579+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:15:48.675079+0000","scrub_duration":0.0061511960000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,0],"acting":[4,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":222,"reported_epoch":653,"state":"active+clean","last_fresh":"2023-09-23T21:19:32.633212+0000","last_change":"2023-09-23T21:17:02.204086+0000","last_active":"2023-09-23T21:19:32.633212+0000","last_peered":"2023-09-23T21:19:32.633212+0000","last_clean":"2023-09-23T21:19:32.633212+0000","last_became_active":"2023-09-23T21:16:42.517549+0000","last_became_peered":"2023-09-23T21:16:42.517549+0000","last_unstale":"2023-09-23T21:19:32.633212+0000","last_undegraded":"2023-09-23T21:19:32.633212+0000","last_fullsized":"2023-09-23T21:19:32.633212+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:43:07.991844+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.6","version":"654'20","reported_seq":661,"reported_epoch":654,"state":"active+clean","last_fresh":"2023-09-23T21:19:36.401731+0000","last_change":"2023-09-23T21:19:32.627299+0000","last_active":"2023-09-23T21:19:36.401731+0000","last_peered":"2023-09-23T21:19:36.401731+0000","last_clean":"2023-09-23T21:19:36.401731+0000","last_became_active":"2023-09-23T21:13:30.807222+0000","last_became_peered":"2023-09-23T21:13:30.807222+0000","last_unstale":"2023-09-23T21:19:36.401731+0000","last_undegraded":"2023-09-23T21:19:36.401731+0000","last_fullsized":"2023-09-23T21:19:36.401731+0000","mapping_epoch":501,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":20,"log_dups_size":0,"ondisk_log_size":20,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:40:31.989749+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00048754499999999998,"stat_sum":{"num_bytes":96,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":58,"num_read_kb":320,"num_write":22,"num_write_kb":158,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"4.0","version":"0'0","reported_seq":515,"reported_epoch":653,"state":"active+clean","last_fresh":"2023-09-23T21:19:32.628178+0000","last_change":"2023-09-23T21:16:42.463411+0000","last_active":"2023-09-23T21:19:32.628178+0000","last_peered":"2023-09-23T21:19:32.628178+0000","last_clean":"2023-09-23T21:19:32.628178+0000","last_became_active":"2023-09-23T21:16:42.463020+0000","last_became_peered":"2023-09-23T21:16:42.463020+0000","last_unstale":"2023-09-23T21:19:32.628178+0000","last_undegraded":"2023-09-23T21:19:32.628178+0000","last_fullsized":"2023-09-23T21:19:32.628178+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:07:34.346129+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.4","version":"79'161","reported_seq":1078,"reported_epoch":654,"state":"active+clean","last_fresh":"2023-09-23T21:19:33.644983+0000","last_change":"2023-09-23T21:19:33.644983+0000","last_active":"2023-09-23T21:19:33.644983+0000","last_peered":"2023-09-23T21:19:33.644983+0000","last_clean":"2023-09-23T21:19:33.644983+0000","last_became_active":"2023-09-23T21:19:33.644547+0000","last_became_peered":"2023-09-23T21:19:33.644547+0000","last_unstale":"2023-09-23T21:19:33.644983+0000","last_undegraded":"2023-09-23T21:19:33.644983+0000","last_fullsized":"2023-09-23T21:19:33.644983+0000","mapping_epoch":653,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":654,"parent":"0.0","parent_split_bits":3,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:20.012764+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:20.012764+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:45:31.294744+0000","scrub_duration":0.0045251919999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,4],"acting":[3,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"654'6","reported_seq":627,"reported_epoch":654,"state":"active+clean","last_fresh":"2023-09-23T21:19:34.333521+0000","last_change":"2023-09-23T21:19:32.631956+0000","last_active":"2023-09-23T21:19:34.333521+0000","last_peered":"2023-09-23T21:19:34.333521+0000","last_clean":"2023-09-23T21:19:34.333521+0000","last_became_active":"2023-09-23T21:16:42.459413+0000","last_became_peered":"2023-09-23T21:16:42.459413+0000","last_unstale":"2023-09-23T21:19:34.333521+0000","last_undegraded":"2023-09-23T21:19:34.333521+0000","last_fullsized":"2023-09-23T21:19:34.333521+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":0,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:37:44.548863+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0039027469999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":4,"num_read_kb":556,"num_write":6,"num_write_kb":278,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,3],"acting":[2,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"3.6","version":"70'1","reported_seq":655,"reported_epoch":653,"state":"active+clean","last_fresh":"2023-09-23T21:19:32.629411+0000","last_change":"2023-09-23T21:17:02.208077+0000","last_active":"2023-09-23T21:19:32.629411+0000","last_peered":"2023-09-23T21:19:32.629411+0000","last_clean":"2023-09-23T21:19:32.629411+0000","last_became_active":"2023-09-23T21:16:42.431308+0000","last_became_peered":"2023-09-23T21:16:42.431308+0000","last_unstale":"2023-09-23T21:19:32.629411+0000","last_undegraded":"2023-09-23T21:19:32.629411+0000","last_fullsized":"2023-09-23T21:19:32.629411+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:16:49.009013+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":238,"reported_epoch":653,"state":"active+clean","last_fresh":"2023-09-23T21:19:32.625525+0000","last_change":"2023-09-23T21:19:30.636392+0000","last_active":"2023-09-23T21:19:32.625525+0000","last_peered":"2023-09-23T21:19:32.625525+0000","last_clean":"2023-09-23T21:19:32.625525+0000","last_became_active":"2023-09-23T21:19:30.635999+0000","last_became_peered":"2023-09-23T21:19:30.635999+0000","last_unstale":"2023-09-23T21:19:32.625525+0000","last_undegraded":"2023-09-23T21:19:32.625525+0000","last_fullsized":"2023-09-23T21:19:32.625525+0000","mapping_epoch":650,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":651,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:33:20.969444+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,7],"acting":[4,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.19","version":"79'161","reported_seq":862,"reported_epoch":653,"state":"active+clean","last_fresh":"2023-09-23T21:19:32.627844+0000","last_change":"2023-09-23T21:16:45.428879+0000","last_active":"2023-09-23T21:19:32.627844+0000","last_peered":"2023-09-23T21:19:32.627844+0000","last_clean":"2023-09-23T21:19:32.627844+0000","last_became_active":"2023-09-23T21:16:45.428660+0000","last_became_peered":"2023-09-23T21:16:45.428660+0000","last_unstale":"2023-09-23T21:19:32.627844+0000","last_undegraded":"2023-09-23T21:19:32.627844+0000","last_fullsized":"2023-09-23T21:19:32.627844+0000","mapping_epoch":567,"log_start":"0'0","ondisk_log_start":"0'0","created":479,"last_epoch_clean":568,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:15:48.271791+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:15:48.271791+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:25:26.361045+0000","scrub_duration":0.0057149180000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":183,"reported_epoch":653,"state":"active+clean","last_fresh":"2023-09-23T21:19:32.632993+0000","last_change":"2023-09-23T21:16:42.434830+0000","last_active":"2023-09-23T21:19:32.632993+0000","last_peered":"2023-09-23T21:19:32.632993+0000","last_clean":"2023-09-23T21:19:32.632993+0000","last_became_active":"2023-09-23T21:16:42.430289+0000","last_became_peered":"2023-09-23T21:16:42.430289+0000","last_unstale":"2023-09-23T21:19:32.632993+0000","last_undegraded":"2023-09-23T21:19:32.632993+0000","last_fullsized":"2023-09-23T21:19:32.632993+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:38:48.053766+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":159,"reported_epoch":653,"state":"active+clean","last_fresh":"2023-09-23T21:19:32.629191+0000","last_change":"2023-09-23T21:13:38.373814+0000","last_active":"2023-09-23T21:19:32.629191+0000","last_peered":"2023-09-23T21:19:32.629191+0000","last_clean":"2023-09-23T21:19:32.629191+0000","last_became_active":"2023-09-23T21:13:38.373458+0000","last_became_peered":"2023-09-23T21:13:38.373458+0000","last_unstale":"2023-09-23T21:19:32.629191+0000","last_undegraded":"2023-09-23T21:19:32.629191+0000","last_fullsized":"2023-09-23T21:19:32.629191+0000","mapping_epoch":507,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:23:08.288401+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.1e","version":"79'161","reported_seq":638,"reported_epoch":653,"state":"active+clean","last_fresh":"2023-09-23T21:19:32.633508+0000","last_change":"2023-09-23T21:16:45.420788+0000","last_active":"2023-09-23T21:19:32.633508+0000","last_peered":"2023-09-23T21:19:32.633508+0000","last_clean":"2023-09-23T21:19:32.633508+0000","last_became_active":"2023-09-23T21:16:45.420448+0000","last_became_peered":"2023-09-23T21:16:45.420448+0000","last_unstale":"2023-09-23T21:19:32.633508+0000","last_undegraded":"2023-09-23T21:19:32.633508+0000","last_fullsized":"2023-09-23T21:19:32.633508+0000","mapping_epoch":567,"log_start":"0'0","ondisk_log_start":"0'0","created":479,"last_epoch_clean":568,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:15:53.061319+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:15:53.061319+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:46:31.982439+0000","scrub_duration":0.007429878,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,4],"acting":[3,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":277,"reported_epoch":653,"state":"active+clean","last_fresh":"2023-09-23T21:19:32.626746+0000","last_change":"2023-09-23T21:19:30.660286+0000","last_active":"2023-09-23T21:19:32.626746+0000","last_peered":"2023-09-23T21:19:32.626746+0000","last_clean":"2023-09-23T21:19:32.626746+0000","last_became_active":"2023-09-23T21:19:30.631029+0000","last_became_peered":"2023-09-23T21:19:30.631029+0000","last_unstale":"2023-09-23T21:19:32.626746+0000","last_undegraded":"2023-09-23T21:19:32.626746+0000","last_fullsized":"2023-09-23T21:19:32.626746+0000","mapping_epoch":650,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":651,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:13:01.408389+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,2],"acting":[4,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":218,"reported_epoch":653,"state":"active+clean","last_fresh":"2023-09-23T21:19:32.627060+0000","last_change":"2023-09-23T21:12:10.770474+0000","last_active":"2023-09-23T21:19:32.627060+0000","last_peered":"2023-09-23T21:19:32.627060+0000","last_clean":"2023-09-23T21:19:32.627060+0000","last_became_active":"2023-09-23T21:12:10.770184+0000","last_became_peered":"2023-09-23T21:12:10.770184+0000","last_unstale":"2023-09-23T21:19:32.627060+0000","last_undegraded":"2023-09-23T21:19:32.627060+0000","last_fullsized":"2023-09-23T21:19:32.627060+0000","mapping_epoch":470,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:05:09.407151+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":458,"reported_epoch":653,"state":"active+clean","last_fresh":"2023-09-23T21:19:32.632465+0000","last_change":"2023-09-23T21:19:30.651034+0000","last_active":"2023-09-23T21:19:32.632465+0000","last_peered":"2023-09-23T21:19:32.632465+0000","last_clean":"2023-09-23T21:19:32.632465+0000","last_became_active":"2023-09-23T21:19:30.634668+0000","last_became_peered":"2023-09-23T21:19:30.634668+0000","last_unstale":"2023-09-23T21:19:32.632465+0000","last_undegraded":"2023-09-23T21:19:32.632465+0000","last_fullsized":"2023-09-23T21:19:32.632465+0000","mapping_epoch":650,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":651,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:41:32.520798+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,7],"acting":[2,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":228,"reported_epoch":653,"state":"active+clean","last_fresh":"2023-09-23T21:19:32.625651+0000","last_change":"2023-09-23T21:13:12.216602+0000","last_active":"2023-09-23T21:19:32.625651+0000","last_peered":"2023-09-23T21:19:32.625651+0000","last_clean":"2023-09-23T21:19:32.625651+0000","last_became_active":"2023-09-23T21:13:12.216204+0000","last_became_peered":"2023-09-23T21:13:12.216204+0000","last_unstale":"2023-09-23T21:19:32.625651+0000","last_undegraded":"2023-09-23T21:19:32.625651+0000","last_fullsized":"2023-09-23T21:19:32.625651+0000","mapping_epoch":493,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:23:37.043390+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1c","version":"79'161","reported_seq":1101,"reported_epoch":654,"state":"active+clean","last_fresh":"2023-09-23T21:19:33.645432+0000","last_change":"2023-09-23T21:19:33.645432+0000","last_active":"2023-09-23T21:19:33.645432+0000","last_peered":"2023-09-23T21:19:33.645432+0000","last_clean":"2023-09-23T21:19:33.645432+0000","last_became_active":"2023-09-23T21:19:33.645042+0000","last_became_peered":"2023-09-23T21:19:33.645042+0000","last_unstale":"2023-09-23T21:19:33.645432+0000","last_undegraded":"2023-09-23T21:19:33.645432+0000","last_fullsized":"2023-09-23T21:19:33.645432+0000","mapping_epoch":653,"log_start":"0'0","ondisk_log_start":"0'0","created":479,"last_epoch_clean":654,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:15:54.019828+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:15:54.019828+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:45:29.799601+0000","scrub_duration":0.0020962530000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":263,"reported_epoch":653,"state":"active+clean","last_fresh":"2023-09-23T21:19:32.626766+0000","last_change":"2023-09-23T21:16:42.424187+0000","last_active":"2023-09-23T21:19:32.626766+0000","last_peered":"2023-09-23T21:19:32.626766+0000","last_clean":"2023-09-23T21:19:32.626766+0000","last_became_active":"2023-09-23T21:16:42.423805+0000","last_became_peered":"2023-09-23T21:16:42.423805+0000","last_unstale":"2023-09-23T21:19:32.626766+0000","last_undegraded":"2023-09-23T21:19:32.626766+0000","last_fullsized":"2023-09-23T21:19:32.626766+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:55:46.051199+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,7],"acting":[2,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":323,"reported_epoch":653,"state":"active+clean","last_fresh":"2023-09-23T21:19:32.628663+0000","last_change":"2023-09-23T21:13:12.200767+0000","last_active":"2023-09-23T21:19:32.628663+0000","last_peered":"2023-09-23T21:19:32.628663+0000","last_clean":"2023-09-23T21:19:32.628663+0000","last_became_active":"2023-09-23T21:13:12.200435+0000","last_became_peered":"2023-09-23T21:13:12.200435+0000","last_unstale":"2023-09-23T21:19:32.628663+0000","last_undegraded":"2023-09-23T21:19:32.628663+0000","last_fullsized":"2023-09-23T21:19:32.628663+0000","mapping_epoch":493,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:15:44.937886+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.1d","version":"79'161","reported_seq":799,"reported_epoch":653,"state":"active+clean","last_fresh":"2023-09-23T21:19:32.631586+0000","last_change":"2023-09-23T21:19:10.941440+0000","last_active":"2023-09-23T21:19:32.631586+0000","last_peered":"2023-09-23T21:19:32.631586+0000","last_clean":"2023-09-23T21:19:32.631586+0000","last_became_active":"2023-09-23T21:15:23.801704+0000","last_became_peered":"2023-09-23T21:15:23.801704+0000","last_unstale":"2023-09-23T21:19:32.631586+0000","last_undegraded":"2023-09-23T21:19:32.631586+0000","last_fullsized":"2023-09-23T21:19:32.631586+0000","mapping_epoch":542,"log_start":"0'0","ondisk_log_start":"0'0","created":479,"last_epoch_clean":543,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:19:10.941299+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:19:10.941299+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:31:22.166514+0000","scrub_duration":0.0018618599999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,2],"acting":[3,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":197,"reported_epoch":653,"state":"active+clean","last_fresh":"2023-09-23T21:19:32.630995+0000","last_change":"2023-09-23T21:19:30.664321+0000","last_active":"2023-09-23T21:19:32.630995+0000","last_peered":"2023-09-23T21:19:32.630995+0000","last_clean":"2023-09-23T21:19:32.630995+0000","last_became_active":"2023-09-23T21:19:30.646140+0000","last_became_peered":"2023-09-23T21:19:30.646140+0000","last_unstale":"2023-09-23T21:19:32.630995+0000","last_undegraded":"2023-09-23T21:19:32.630995+0000","last_fullsized":"2023-09-23T21:19:32.630995+0000","mapping_epoch":650,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":651,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:42:27.475154+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":13,"num_bytes_recovered":3878,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":228,"reported_epoch":653,"state":"active+clean","last_fresh":"2023-09-23T21:19:32.625432+0000","last_change":"2023-09-23T21:12:10.793610+0000","last_active":"2023-09-23T21:19:32.625432+0000","last_peered":"2023-09-23T21:19:32.625432+0000","last_clean":"2023-09-23T21:19:32.625432+0000","last_became_active":"2023-09-23T21:12:10.793081+0000","last_became_peered":"2023-09-23T21:12:10.793081+0000","last_unstale":"2023-09-23T21:19:32.625432+0000","last_undegraded":"2023-09-23T21:19:32.625432+0000","last_fullsized":"2023-09-23T21:19:32.625432+0000","mapping_epoch":470,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:34:36.291550+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":84,"num_bytes_recovered":18960,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":126976,"data_stored":21974,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":14322,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":31,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2375680,"data_stored":2361472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4991,"ondisk_log_size":4991,"up":93,"acting":91,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":783723,"num_objects":6,"num_object_clones":0,"num_object_copies":18,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6,"num_whiteouts":0,"num_read":318,"num_read_kb":1420,"num_write":76,"num_write_kb":720,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":380928,"data_stored":357766,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":81,"ondisk_log_size":81,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201749,"num_pgs":55,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":658968,"kb_used_data":6992,"kb_used_omap":0,"kb_used_meta":651968,"kb_avail":93086184,"statfs":{"total":95995035648,"available":95320252416,"internally_reserved":0,"allocated":7159808,"data_stored":5404273,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":667615232},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:19:23 2023","interfaces":[{"interface":"back","average":{"1min":0.77800000000000002,"5min":1.145,"15min":1.9990000000000001},"min":{"1min":0.23599999999999999,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":3.448,"5min":5.1790000000000003,"15min":37.207999999999998},"last":0.40899999999999997},{"interface":"front","average":{"1min":0.86699999999999999,"5min":1.474,"15min":1.98},"min":{"1min":0.222,"5min":0.222,"15min":0.222},"max":{"1min":3.6930000000000001,"5min":10.429,"15min":37.122},"last":0.34000000000000002}]},{"osd":1,"last update":"Sat Sep 23 21:19:18 2023","interfaces":[{"interface":"back","average":{"1min":0.80200000000000005,"5min":1.1930000000000001,"15min":1.704},"min":{"1min":0.29399999999999998,"5min":0.27600000000000002,"15min":0.26700000000000002},"max":{"1min":5.2619999999999996,"5min":6.9130000000000003,"15min":14.034000000000001},"last":0.377},{"interface":"front","average":{"1min":0.84199999999999997,"5min":1.3380000000000001,"15min":1.6930000000000001},"min":{"1min":0.36599999999999999,"5min":0.29999999999999999,"15min":0.26300000000000001},"max":{"1min":2.512,"5min":11.002000000000001,"15min":11.226000000000001},"last":0.46500000000000002}]},{"osd":2,"last update":"Sat Sep 23 21:18:38 2023","interfaces":[{"interface":"back","average":{"1min":2.105,"5min":2.0739999999999998,"15min":1.8069999999999999},"min":{"1min":0.27200000000000002,"5min":0.27200000000000002,"15min":0.26200000000000001},"max":{"1min":18.488,"5min":25.471,"15min":25.471},"last":3.4060000000000001},{"interface":"front","average":{"1min":1.9330000000000001,"5min":1.857,"15min":1.752},"min":{"1min":0.28799999999999998,"5min":0.28799999999999998,"15min":0.26700000000000002},"max":{"1min":17.536000000000001,"5min":27.187999999999999,"15min":27.187999999999999},"last":0.316}]},{"osd":3,"last update":"Sat Sep 23 21:19:06 2023","interfaces":[{"interface":"back","average":{"1min":1.6220000000000001,"5min":1.6040000000000001,"15min":1.788},"min":{"1min":0.317,"5min":0.30599999999999999,"15min":0.30099999999999999},"max":{"1min":5.7830000000000004,"5min":8.6259999999999994,"15min":12.359999999999999},"last":0.44400000000000001},{"interface":"front","average":{"1min":1.504,"5min":1.8100000000000001,"15min":1.79},"min":{"1min":0.33400000000000002,"5min":0.28799999999999998,"15min":0.28699999999999998},"max":{"1min":4.6769999999999996,"5min":15.82,"15min":15.82},"last":4.0179999999999998}]},{"osd":4,"last update":"Sat Sep 23 21:19:06 2023","interfaces":[{"interface":"back","average":{"1min":0.42199999999999999,"5min":0.78500000000000003,"15min":1.1160000000000001},"min":{"1min":0.27000000000000002,"5min":0.251,"15min":0.248},"max":{"1min":0.92700000000000005,"5min":4.2130000000000001,"15min":11.297000000000001},"last":0.504},{"interface":"front","average":{"1min":0.61299999999999999,"5min":0.91800000000000004,"15min":1.0800000000000001},"min":{"1min":0.26800000000000002,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":3.73,"5min":7.1870000000000003,"15min":12.752000000000001},"last":0.39900000000000002}]},{"osd":5,"last update":"Sat Sep 23 21:19:06 2023","interfaces":[{"interface":"back","average":{"1min":0.57199999999999995,"5min":0.91600000000000004,"15min":1.0640000000000001},"min":{"1min":0.28199999999999997,"5min":0.28199999999999997,"15min":0.26000000000000001},"max":{"1min":1.7150000000000001,"5min":6.9740000000000002,"15min":9.4789999999999992},"last":0.48899999999999999},{"interface":"front","average":{"1min":0.65900000000000003,"5min":1.004,"15min":1.157},"min":{"1min":0.32300000000000001,"5min":0.32000000000000001,"15min":0.26200000000000001},"max":{"1min":3.1349999999999998,"5min":6.7439999999999998,"15min":11.257},"last":0.35799999999999998}]},{"osd":6,"last update":"Sat Sep 23 21:19:18 2023","interfaces":[{"interface":"back","average":{"1min":0.72299999999999998,"5min":0.86899999999999999,"15min":1.0269999999999999},"min":{"1min":0.312,"5min":0.312,"15min":0.30099999999999999},"max":{"1min":3.8170000000000002,"5min":7.5339999999999998,"15min":10.941000000000001},"last":0.42999999999999999},{"interface":"front","average":{"1min":0.80800000000000005,"5min":1.006,"15min":1.145},"min":{"1min":0.35099999999999998,"5min":0.35099999999999998,"15min":0.33100000000000002},"max":{"1min":3.7559999999999998,"5min":5.6760000000000002,"15min":12.715},"last":0.47499999999999998}]}]},{"osd":6,"up_from":0,"seq":704374636833,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":1,"up_from":0,"seq":1503238553765,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":0,"up_from":493,"seq":2117418877006,"num_pgs":63,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":429988,"kb_used_data":8668,"kb_used_omap":36,"kb_used_meta":421275,"kb_avail":93315164,"statfs":{"total":95995035648,"available":95554727936,"internally_reserved":0,"allocated":8876032,"data_stored":5909721,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":37336,"internal_metadata":431386152},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 21:19:30 2023","interfaces":[{"interface":"back","average":{"1min":0.73799999999999999,"5min":1.4370000000000001,"15min":0.98299999999999998},"min":{"1min":0.23000000000000001,"5min":0.214,"15min":0.214},"max":{"1min":3.8610000000000002,"5min":13.779999999999999,"15min":13.779999999999999},"last":0.497},{"interface":"front","average":{"1min":0.98099999999999998,"5min":1.7989999999999999,"15min":1.113},"min":{"1min":0.23300000000000001,"5min":0.20799999999999999,"15min":0.20799999999999999},"max":{"1min":3.0539999999999998,"5min":11.964,"15min":11.964},"last":0.33800000000000002}]},{"osd":2,"last update":"Sat Sep 23 21:19:30 2023","interfaces":[{"interface":"back","average":{"1min":2.8300000000000001,"5min":2.2360000000000002,"15min":1.181},"min":{"1min":0.25800000000000001,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":35.68,"5min":35.68,"15min":35.68},"last":0.44},{"interface":"front","average":{"1min":2.8889999999999998,"5min":2.8450000000000002,"15min":1.593},"min":{"1min":0.28499999999999998,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":35.832999999999998,"5min":35.832999999999998,"15min":35.832999999999998},"last":0.45800000000000002}]},{"osd":3,"last update":"Sat Sep 23 21:19:30 2023","interfaces":[{"interface":"back","average":{"1min":1.891,"5min":2.8039999999999998,"15min":1.5129999999999999},"min":{"1min":0.247,"5min":0.247,"15min":0.23400000000000001},"max":{"1min":8.3089999999999993,"5min":37.079999999999998,"15min":37.079999999999998},"last":0.38400000000000001},{"interface":"front","average":{"1min":0.98299999999999998,"5min":2.1760000000000002,"15min":1.47},"min":{"1min":0.28899999999999998,"5min":0.28299999999999997,"15min":0.26200000000000001},"max":{"1min":3.3069999999999999,"5min":32.935000000000002,"15min":32.935000000000002},"last":0.54500000000000004}]},{"osd":4,"last update":"Sat Sep 23 21:19:30 2023","interfaces":[{"interface":"back","average":{"1min":0.89300000000000002,"5min":1.6990000000000001,"15min":1.2949999999999999},"min":{"1min":0.34200000000000003,"5min":0.312,"15min":0.30199999999999999},"max":{"1min":2.4209999999999998,"5min":22.780000000000001,"15min":22.780000000000001},"last":0.73099999999999998},{"interface":"front","average":{"1min":0.98499999999999999,"5min":1.496,"15min":1.1619999999999999},"min":{"1min":0.33700000000000002,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":3.8199999999999998,"5min":22.652000000000001,"15min":22.652000000000001},"last":0.71699999999999997}]},{"osd":5,"last update":"Sat Sep 23 21:19:30 2023","interfaces":[{"interface":"back","average":{"1min":0.875,"5min":1.4950000000000001,"15min":1.3220000000000001},"min":{"1min":0.33200000000000002,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":3.0950000000000002,"5min":8.9329999999999998,"15min":8.9329999999999998},"last":0.68400000000000005},{"interface":"front","average":{"1min":0.91200000000000003,"5min":1.6659999999999999,"15min":1.264},"min":{"1min":0.377,"5min":0.313,"15min":0.313},"max":{"1min":2.4750000000000001,"5min":15.321999999999999,"15min":15.321999999999999},"last":0.67400000000000004}]},{"osd":6,"last update":"Sat Sep 23 21:19:30 2023","interfaces":[{"interface":"back","average":{"1min":1.042,"5min":1.464,"15min":1.2390000000000001},"min":{"1min":0.432,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":3.8999999999999999,"5min":7.2050000000000001,"15min":7.2050000000000001},"last":0.79300000000000004},{"interface":"front","average":{"1min":0.872,"5min":1.508,"15min":1.3740000000000001},"min":{"1min":0.35099999999999998,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":3.125,"5min":8.8889999999999993,"15min":8.8889999999999993},"last":0.56799999999999995}]},{"osd":7,"last update":"Sat Sep 23 21:19:30 2023","interfaces":[{"interface":"back","average":{"1min":0.93799999999999994,"5min":1.77,"15min":1.3959999999999999},"min":{"1min":0.39700000000000002,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":2.4449999999999998,"5min":15.273999999999999,"15min":15.273999999999999},"last":0.70599999999999996},{"interface":"front","average":{"1min":1.0489999999999999,"5min":1.7490000000000001,"15min":1.355},"min":{"1min":0.40300000000000002,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":3.9390000000000001,"5min":33.779000000000003,"15min":33.779000000000003},"last":0.755}]}]},{"osd":2,"up_from":93,"seq":399431958878,"num_pgs":53,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1040144,"kb_used_data":8388,"kb_used_omap":81,"kb_used_meta":1031662,"kb_avail":92705008,"statfs":{"total":95995035648,"available":94929928192,"internally_reserved":0,"allocated":8589312,"data_stored":5584158,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":83750,"internal_metadata":1056422106},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:19:26 2023","interfaces":[{"interface":"back","average":{"1min":1.7470000000000001,"5min":2.1200000000000001,"15min":1.262},"min":{"1min":0.20399999999999999,"5min":0.187,"15min":0.187},"max":{"1min":9.2799999999999994,"5min":13.25,"15min":13.25},"last":0.57299999999999995},{"interface":"front","average":{"1min":1.4650000000000001,"5min":2.0489999999999999,"15min":1.7929999999999999},"min":{"1min":0.29599999999999999,"5min":0.20200000000000001,"15min":0.20200000000000001},"max":{"1min":5.649,"5min":11.269,"15min":18.111999999999998},"last":0.33400000000000002}]},{"osd":1,"last update":"Sat Sep 23 21:19:26 2023","interfaces":[{"interface":"back","average":{"1min":2.0419999999999998,"5min":1.9750000000000001,"15min":2.2000000000000002},"min":{"1min":0.25800000000000001,"5min":0.20599999999999999,"15min":0.188},"max":{"1min":7.4939999999999998,"5min":10.535,"15min":34.396999999999998},"last":0.29899999999999999},{"interface":"front","average":{"1min":1.1439999999999999,"5min":1.8600000000000001,"15min":2.1779999999999999},"min":{"1min":0.34999999999999998,"5min":0.24099999999999999,"15min":0.217},"max":{"1min":4.9560000000000004,"5min":16.274000000000001,"15min":34.462000000000003},"last":0.40000000000000002}]},{"osd":3,"last update":"Sat Sep 23 21:19:02 2023","interfaces":[{"interface":"back","average":{"1min":2.2949999999999999,"5min":2.3999999999999999,"15min":2.149},"min":{"1min":0.27300000000000002,"5min":0.248,"15min":0.20599999999999999},"max":{"1min":8.141,"5min":20.106999999999999,"15min":35.549999999999997},"last":0.35499999999999998},{"interface":"front","average":{"1min":2.593,"5min":2.9249999999999998,"15min":2.4609999999999999},"min":{"1min":0.34899999999999998,"5min":0.28599999999999998,"15min":0.21299999999999999},"max":{"1min":8.2469999999999999,"5min":20.239000000000001,"15min":36.951999999999998},"last":0.63}]},{"osd":4,"last update":"Sat Sep 23 21:19:26 2023","interfaces":[{"interface":"back","average":{"1min":0.79200000000000004,"5min":1.375,"15min":1.4039999999999999},"min":{"1min":0.30599999999999999,"5min":0.27000000000000002,"15min":0.249},"max":{"1min":3.1880000000000002,"5min":7.9770000000000003,"15min":10.266999999999999},"last":0.58799999999999997},{"interface":"front","average":{"1min":1.3149999999999999,"5min":1.6259999999999999,"15min":1.645},"min":{"1min":0.375,"5min":0.30299999999999999,"15min":0.26700000000000002},"max":{"1min":7.601,"5min":10.459,"15min":12.49},"last":0.53400000000000003}]},{"osd":5,"last update":"Sat Sep 23 21:19:26 2023","interfaces":[{"interface":"back","average":{"1min":5.6959999999999997,"5min":2.407,"15min":1.827},"min":{"1min":0.36699999999999999,"5min":0.29099999999999998,"15min":0.28000000000000003},"max":{"1min":102.56100000000001,"5min":102.56100000000001,"15min":102.56100000000001},"last":0.51300000000000001},{"interface":"front","average":{"1min":6.4020000000000001,"5min":2.6299999999999999,"15min":1.954},"min":{"1min":0.46000000000000002,"5min":0.33800000000000002,"15min":0.29399999999999998},"max":{"1min":102.28700000000001,"5min":102.28700000000001,"15min":102.28700000000001},"last":0.442}]},{"osd":6,"last update":"Sat Sep 23 21:19:26 2023","interfaces":[{"interface":"back","average":{"1min":1.9430000000000001,"5min":1.831,"15min":1.8640000000000001},"min":{"1min":0.44,"5min":0.377,"15min":0.35699999999999998},"max":{"1min":9.8680000000000003,"5min":10.335000000000001,"15min":12.518000000000001},"last":0.48999999999999999},{"interface":"front","average":{"1min":1.5429999999999999,"5min":1.756,"15min":1.9930000000000001},"min":{"1min":0.443,"5min":0.377,"15min":0.35599999999999998},"max":{"1min":8.4149999999999991,"5min":10.381,"15min":12.544},"last":0.60499999999999998}]},{"osd":7,"last update":"Sat Sep 23 21:19:26 2023","interfaces":[{"interface":"back","average":{"1min":1.0900000000000001,"5min":1.4930000000000001,"15min":1.6359999999999999},"min":{"1min":0.42199999999999999,"5min":0.31,"15min":0.29799999999999999},"max":{"1min":3.1190000000000002,"5min":7.5819999999999999,"15min":18.021999999999998},"last":0.45700000000000002},{"interface":"front","average":{"1min":0.83099999999999996,"5min":1.5509999999999999,"15min":1.6879999999999999},"min":{"1min":0.372,"5min":0.34999999999999998,"15min":0.32200000000000001},"max":{"1min":3.222,"5min":7.46,"15min":10.92},"last":0.46899999999999997}]}]},{"osd":3,"up_from":98,"seq":420906795351,"num_pgs":61,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1138608,"kb_used_data":8808,"kb_used_omap":7,"kb_used_meta":1129784,"kb_avail":92606544,"statfs":{"total":95995035648,"available":94829101056,"internally_reserved":0,"allocated":9019392,"data_stored":6066558,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7576,"internal_metadata":1156899432},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:19:18 2023","interfaces":[{"interface":"back","average":{"1min":1.3009999999999999,"5min":4.2670000000000003,"15min":1.647},"min":{"1min":0.20399999999999999,"5min":0.18099999999999999,"15min":0.18099999999999999},"max":{"1min":9.3819999999999997,"5min":147.38900000000001,"15min":147.38900000000001},"last":0.38400000000000001},{"interface":"front","average":{"1min":0.93899999999999995,"5min":3.6400000000000001,"15min":1.6579999999999999},"min":{"1min":0.214,"5min":0.214,"15min":0.214},"max":{"1min":5.2110000000000003,"5min":149.989,"15min":149.989},"last":0.41699999999999998}]},{"osd":1,"last update":"Sat Sep 23 21:19:18 2023","interfaces":[{"interface":"back","average":{"1min":1.4710000000000001,"5min":1.6779999999999999,"15min":2.0720000000000001},"min":{"1min":0.248,"5min":0.23699999999999999,"15min":0.20000000000000001},"max":{"1min":11.146000000000001,"5min":11.146000000000001,"15min":22.577999999999999},"last":0.40500000000000003},{"interface":"front","average":{"1min":0.79100000000000004,"5min":1.335,"15min":1.8879999999999999},"min":{"1min":0.28899999999999998,"5min":0.255,"15min":0.216},"max":{"1min":2.7799999999999998,"5min":7.6829999999999998,"15min":21.617999999999999},"last":0.46700000000000003}]},{"osd":2,"last update":"Sat Sep 23 21:19:03 2023","interfaces":[{"interface":"back","average":{"1min":0.84999999999999998,"5min":1.395,"15min":1.893},"min":{"1min":0.23799999999999999,"5min":0.23799999999999999,"15min":0.20899999999999999},"max":{"1min":3.1000000000000001,"5min":11.321,"15min":56.317999999999998},"last":0.496},{"interface":"front","average":{"1min":3.1120000000000001,"5min":2.331,"15min":2.343},"min":{"1min":0.34899999999999998,"5min":0.23400000000000001,"15min":0.216},"max":{"1min":11.177,"5min":17.423999999999999,"15min":55.988},"last":0.48499999999999999}]},{"osd":4,"last update":"Sat Sep 23 21:19:03 2023","interfaces":[{"interface":"back","average":{"1min":1.0609999999999999,"5min":1.27,"15min":1.472},"min":{"1min":0.36899999999999999,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":5.1589999999999998,"5min":10.807,"15min":21.478999999999999},"last":0.56000000000000005},{"interface":"front","average":{"1min":2.145,"5min":1.4870000000000001,"15min":1.7549999999999999},"min":{"1min":0.39600000000000002,"5min":0.311,"15min":0.28499999999999998},"max":{"1min":11.202,"5min":11.202,"15min":22.402000000000001},"last":0.72999999999999998}]},{"osd":5,"last update":"Sat Sep 23 21:19:03 2023","interfaces":[{"interface":"back","average":{"1min":0.91400000000000003,"5min":1.3109999999999999,"15min":1.6040000000000001},"min":{"1min":0.34999999999999998,"5min":0.313,"15min":0.311},"max":{"1min":3.109,"5min":10.882999999999999,"15min":21.562000000000001},"last":0.54100000000000004},{"interface":"front","average":{"1min":0.90300000000000002,"5min":1.2470000000000001,"15min":1.679},"min":{"1min":0.30199999999999999,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":2.8010000000000002,"5min":5.7300000000000004,"15min":14.029},"last":0.63100000000000001}]},{"osd":6,"last update":"Sat Sep 23 21:19:18 2023","interfaces":[{"interface":"back","average":{"1min":1.605,"5min":1.534,"15min":1.794},"min":{"1min":0.44500000000000001,"5min":0.35899999999999999,"15min":0.34599999999999997},"max":{"1min":11.092000000000001,"5min":11.092000000000001,"15min":22.446999999999999},"last":0.69099999999999995},{"interface":"front","average":{"1min":0.80000000000000004,"5min":1.2589999999999999,"15min":1.7350000000000001},"min":{"1min":0.34300000000000003,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":2.839,"5min":7.5949999999999998,"15min":12.375999999999999},"last":0.65500000000000003}]},{"osd":7,"last update":"Sat Sep 23 21:19:03 2023","interfaces":[{"interface":"back","average":{"1min":0.91100000000000003,"5min":1.456,"15min":1.74},"min":{"1min":0.41099999999999998,"5min":0.41099999999999998,"15min":0.33900000000000002},"max":{"1min":2.4279999999999999,"5min":10.914999999999999,"15min":21.518999999999998},"last":0.70299999999999996},{"interface":"front","average":{"1min":1.2370000000000001,"5min":1.589,"15min":1.9059999999999999},"min":{"1min":0.38800000000000001,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":3.8130000000000002,"5min":8.2590000000000003,"15min":12.624000000000001},"last":0.61099999999999999}]}]},{"osd":4,"up_from":31,"seq":133143986658,"num_pgs":59,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":594476,"kb_used_data":7076,"kb_used_omap":0,"kb_used_meta":587392,"kb_avail":93150676,"statfs":{"total":95995035648,"available":95386292224,"internally_reserved":0,"allocated":7245824,"data_stored":5485784,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":601489408},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:19:24 2023","interfaces":[{"interface":"back","average":{"1min":0.73799999999999999,"5min":1.641,"15min":1.147},"min":{"1min":0.24299999999999999,"5min":0.22800000000000001,"15min":0.22500000000000001},"max":{"1min":4.4429999999999996,"5min":28.683,"15min":28.683},"last":0.55200000000000005},{"interface":"front","average":{"1min":1.038,"5min":1.782,"15min":1.3500000000000001},"min":{"1min":0.26000000000000001,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":6.5490000000000004,"5min":29.155000000000001,"15min":29.155000000000001},"last":0.501}]},{"osd":1,"last update":"Sat Sep 23 21:19:24 2023","interfaces":[{"interface":"back","average":{"1min":0.70099999999999996,"5min":1.2689999999999999,"15min":1.899},"min":{"1min":0.31,"5min":0.26900000000000002,"15min":0.221},"max":{"1min":2.964,"5min":8.1020000000000003,"15min":37.960000000000001},"last":0.53700000000000003},{"interface":"front","average":{"1min":0.75800000000000001,"5min":1.3899999999999999,"15min":1.8640000000000001},"min":{"1min":0.35799999999999998,"5min":0.30299999999999999,"15min":0.27000000000000002},"max":{"1min":2.976,"5min":13.983000000000001,"15min":37.933},"last":0.57599999999999996}]},{"osd":2,"last update":"Sat Sep 23 21:19:24 2023","interfaces":[{"interface":"back","average":{"1min":1.556,"5min":1.9970000000000001,"15min":1.6890000000000001},"min":{"1min":0.29099999999999998,"5min":0.26400000000000001,"15min":0.247},"max":{"1min":13.791,"5min":17.602,"15min":17.602},"last":0.56100000000000005},{"interface":"front","average":{"1min":1.6359999999999999,"5min":1.9399999999999999,"15min":1.7649999999999999},"min":{"1min":0.28799999999999998,"5min":0.28799999999999998,"15min":0.27800000000000002},"max":{"1min":13.439,"5min":14.339,"15min":14.417999999999999},"last":0.51800000000000002}]},{"osd":3,"last update":"Sat Sep 23 21:18:55 2023","interfaces":[{"interface":"back","average":{"1min":1.3129999999999999,"5min":2.1579999999999999,"15min":1.9790000000000001},"min":{"1min":0.29499999999999998,"5min":0.27900000000000003,"15min":0.25600000000000001},"max":{"1min":6.1230000000000002,"5min":67.105999999999995,"15min":67.105999999999995},"last":0.47599999999999998},{"interface":"front","average":{"1min":1.5289999999999999,"5min":2.2719999999999998,"15min":1.966},"min":{"1min":0.40400000000000003,"5min":0.32200000000000001,"15min":0.30199999999999999},"max":{"1min":6.1319999999999997,"5min":67.046999999999997,"15min":67.046999999999997},"last":0.59499999999999997}]},{"osd":5,"last update":"Sat Sep 23 21:19:17 2023","interfaces":[{"interface":"back","average":{"1min":0.39400000000000002,"5min":0.95299999999999996,"15min":1.0700000000000001},"min":{"1min":0.26900000000000002,"5min":0.254,"15min":0.222},"max":{"1min":0.73999999999999999,"5min":14.398999999999999,"15min":14.398999999999999},"last":0.49099999999999999},{"interface":"front","average":{"1min":0.41699999999999998,"5min":0.86699999999999999,"15min":1.0589999999999999},"min":{"1min":0.29699999999999999,"5min":0.25800000000000001,"15min":0.255},"max":{"1min":0.57699999999999996,"5min":8.016,"15min":9.8979999999999997},"last":0.52500000000000002}]},{"osd":6,"last update":"Sat Sep 23 21:19:24 2023","interfaces":[{"interface":"back","average":{"1min":0.439,"5min":0.90400000000000003,"15min":1.167},"min":{"1min":0.311,"5min":0.27200000000000002,"15min":0.26800000000000002},"max":{"1min":0.62,"5min":8.7420000000000009,"15min":24.452999999999999},"last":0.41699999999999998},{"interface":"front","average":{"1min":0.55800000000000005,"5min":1.071,"15min":1.0600000000000001},"min":{"1min":0.42199999999999999,"5min":0.36399999999999999,"15min":0.32500000000000001},"max":{"1min":0.79700000000000004,"5min":14.374000000000001,"15min":14.387},"last":0.54100000000000004}]},{"osd":7,"last update":"Sat Sep 23 21:18:55 2023","interfaces":[{"interface":"back","average":{"1min":0.70599999999999996,"5min":1.077,"15min":1.1659999999999999},"min":{"1min":0.33800000000000002,"5min":0.32000000000000001,"15min":0.30099999999999999},"max":{"1min":2.6419999999999999,"5min":14.422000000000001,"15min":16.890999999999998},"last":0.44700000000000001},{"interface":"front","average":{"1min":0.70999999999999996,"5min":0.89800000000000002,"15min":1.179},"min":{"1min":0.32900000000000001,"5min":0.29199999999999998,"15min":0.27400000000000002},"max":{"1min":2.6579999999999999,"5min":7.7229999999999999,"15min":11.170999999999999},"last":0.46100000000000002}]}]},{"osd":5,"up_from":0,"seq":167503725023,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":364544,"data_stored":357624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":96,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":8,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8488,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1259,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5322,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":32768,"data_stored":5504,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":4838,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T21:19:39.402 INFO:tasks.cephadm.ceph_manager.ceph:increase pool size by 10 2023-09-23T21:19:39.402 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph osd pool set .mgr pg_num 51 2023-09-23T21:19:39.479 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:19:39.483 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:39.484 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:19:39.493 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T21:19:39.585 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:19:39.618 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:39.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:19:39 smithi026 ceph-mon[175151]: from='client.37125 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T21:19:39.667 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:19:39 smithi026 ceph-mon[179688]: from='client.37125 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T21:19:39.667 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 21:19:39 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[164029]: ::ffff:172.21.15.70 - - [23/Sep/2023:21:19:39] "GET /metrics HTTP/1.1" 200 33944 "" "Prometheus/2.43.0" 2023-09-23T21:19:39.672 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T21:19:39.673 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_historic_ops 2023-09-23T21:19:39.677 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:39.677 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:19:39.692 INFO:tasks.workunit.client.0.smithi026.stderr: Creating snap: 10% complete... Creating snap: 100% complete...done. 2023-09-23T21:19:39.705 INFO:tasks.workunit.client.0.smithi026.stderr:+ rbd image-meta set testimg key1 value1 2023-09-23T21:19:39.758 INFO:tasks.workunit.client.0.smithi026.stderr:++ rbd image-meta list testimg 2023-09-23T21:19:39.778 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:19:39.806 INFO:tasks.workunit.client.0.smithi026.stderr:+ IMAGEMETA_BEFORE='There is 1 metadatum on this image: 2023-09-23T21:19:39.806 INFO:tasks.workunit.client.0.smithi026.stderr: 2023-09-23T21:19:39.806 INFO:tasks.workunit.client.0.smithi026.stderr:Key Value 2023-09-23T21:19:39.806 INFO:tasks.workunit.client.0.smithi026.stderr:key1 value1' 2023-09-23T21:19:39.807 INFO:tasks.workunit.client.0.smithi026.stderr:+ rbd export --export-format 2 testimg /tmp/rbd_import_export_456745/img_v2 2023-09-23T21:19:39.810 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:39.851 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:19:39.854 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:39.858 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:19:39.858 INFO:tasks.workunit.client.0.smithi026.stderr: Exporting image: 100% complete... Exporting image: 100% complete...done. 2023-09-23T21:19:39.861 INFO:tasks.workunit.client.0.smithi026.stderr:+ rbd import --export-format 2 /tmp/rbd_import_export_456745/img_v2 testimg_import 2023-09-23T21:19:39.959 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:19:39.984 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:19:39 smithi070 ceph-mon[168282]: from='client.37125 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T21:19:39.987 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:40.022 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:19:40.025 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:40.025 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:19:40.100 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:19:40.126 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:19:40.155 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:40.188 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:19:40.191 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:40.191 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:19:40.195 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:19:40.274 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T21:19:40.292 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:19:40.319 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:40.393 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:19:40.395 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:40.396 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:19:40.497 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:19:40.527 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:40.677 INFO:teuthology.orchestra.run.smithi026.stderr:nodeep-scrub is unset 2023-09-23T21:19:40.688 INFO:tasks.workunit.client.0.smithi026.stderr: Importing image: 50% complete... Importing image: 100% complete... Importing image: 100% complete...done. 2023-09-23T21:19:40.691 INFO:tasks.workunit.client.0.smithi026.stderr:+ rbd info testimg_import 2023-09-23T21:19:40.730 INFO:tasks.workunit.client.0.smithi026.stdout:rbd image 'testimg_import': 2023-09-23T21:19:40.730 INFO:tasks.workunit.client.0.smithi026.stdout: size 110 KiB in 1 objects 2023-09-23T21:19:40.730 INFO:tasks.workunit.client.0.smithi026.stdout: order 22 (4 MiB objects) 2023-09-23T21:19:40.730 INFO:tasks.workunit.client.0.smithi026.stdout: snapshot_count: 1 2023-09-23T21:19:40.731 INFO:tasks.workunit.client.0.smithi026.stdout: id: 912c71b3c13a 2023-09-23T21:19:40.731 INFO:tasks.workunit.client.0.smithi026.stdout: block_name_prefix: rbd_data.912c71b3c13a 2023-09-23T21:19:40.731 INFO:tasks.workunit.client.0.smithi026.stdout: format: 2 2023-09-23T21:19:40.731 INFO:tasks.workunit.client.0.smithi026.stdout: features: layering, exclusive-lock, object-map, fast-diff, deep-flatten 2023-09-23T21:19:40.731 INFO:tasks.workunit.client.0.smithi026.stdout: op_features: 2023-09-23T21:19:40.731 INFO:tasks.workunit.client.0.smithi026.stdout: flags: 2023-09-23T21:19:40.731 INFO:tasks.workunit.client.0.smithi026.stdout: create_timestamp: Sat Sep 23 21:19:39 2023 2023-09-23T21:19:40.731 INFO:tasks.workunit.client.0.smithi026.stdout: access_timestamp: Sat Sep 23 21:19:39 2023 2023-09-23T21:19:40.732 INFO:tasks.workunit.client.0.smithi026.stdout: modify_timestamp: Sat Sep 23 21:19:39 2023 2023-09-23T21:19:40.733 INFO:tasks.workunit.client.0.smithi026.stderr:+ rbd info testimg_import@snap 2023-09-23T21:19:40.770 INFO:tasks.workunit.client.0.smithi026.stdout:rbd image 'testimg_import': 2023-09-23T21:19:40.771 INFO:tasks.workunit.client.0.smithi026.stdout: size 110 KiB in 1 objects 2023-09-23T21:19:40.771 INFO:tasks.workunit.client.0.smithi026.stdout: order 22 (4 MiB objects) 2023-09-23T21:19:40.771 INFO:tasks.workunit.client.0.smithi026.stdout: snapshot_count: 1 2023-09-23T21:19:40.771 INFO:tasks.workunit.client.0.smithi026.stdout: id: 912c71b3c13a 2023-09-23T21:19:40.771 INFO:tasks.workunit.client.0.smithi026.stdout: block_name_prefix: rbd_data.912c71b3c13a 2023-09-23T21:19:40.771 INFO:tasks.workunit.client.0.smithi026.stdout: format: 2 2023-09-23T21:19:40.771 INFO:tasks.workunit.client.0.smithi026.stdout: features: layering, exclusive-lock, object-map, fast-diff, deep-flatten 2023-09-23T21:19:40.771 INFO:tasks.workunit.client.0.smithi026.stdout: op_features: 2023-09-23T21:19:40.771 INFO:tasks.workunit.client.0.smithi026.stdout: flags: 2023-09-23T21:19:40.771 INFO:tasks.workunit.client.0.smithi026.stdout: create_timestamp: Sat Sep 23 21:19:39 2023 2023-09-23T21:19:40.772 INFO:tasks.workunit.client.0.smithi026.stdout: access_timestamp: Sat Sep 23 21:19:39 2023 2023-09-23T21:19:40.772 INFO:tasks.workunit.client.0.smithi026.stdout: modify_timestamp: Sat Sep 23 21:19:39 2023 2023-09-23T21:19:40.772 INFO:tasks.workunit.client.0.smithi026.stdout: protected: False 2023-09-23T21:19:40.774 INFO:tasks.workunit.client.0.smithi026.stderr:++ rbd image-meta list testimg_import 2023-09-23T21:19:40.813 INFO:tasks.workunit.client.0.smithi026.stderr:+ IMAGEMETA_AFTER='There is 1 metadatum on this image: 2023-09-23T21:19:40.813 INFO:tasks.workunit.client.0.smithi026.stderr: 2023-09-23T21:19:40.814 INFO:tasks.workunit.client.0.smithi026.stderr:Key Value 2023-09-23T21:19:40.814 INFO:tasks.workunit.client.0.smithi026.stderr:key1 value1' 2023-09-23T21:19:40.814 INFO:tasks.workunit.client.0.smithi026.stderr:+ '[' 'There is 1 metadatum on this image: 2023-09-23T21:19:40.814 INFO:tasks.workunit.client.0.smithi026.stderr: 2023-09-23T21:19:40.814 INFO:tasks.workunit.client.0.smithi026.stderr:Key Value 2023-09-23T21:19:40.814 INFO:tasks.workunit.client.0.smithi026.stderr:key1 value1' = 'There is 1 metadatum on this image: 2023-09-23T21:19:40.814 INFO:tasks.workunit.client.0.smithi026.stderr: 2023-09-23T21:19:40.814 INFO:tasks.workunit.client.0.smithi026.stderr:Key Value 2023-09-23T21:19:40.815 INFO:tasks.workunit.client.0.smithi026.stderr:key1 value1' ']' 2023-09-23T21:19:40.815 INFO:tasks.workunit.client.0.smithi026.stderr:+ rbd export testimg_import /tmp/rbd_import_export_456745/img_import 2023-09-23T21:19:40.848 INFO:tasks.workunit.client.0.smithi026.stderr: Exporting image: 100% complete...done. 2023-09-23T21:19:40.851 INFO:tasks.workunit.client.0.smithi026.stderr:+ compare_files_and_ondisk_sizes /tmp/rbd_import_export_456745/img /tmp/rbd_import_export_456745/img_import 2023-09-23T21:19:40.851 INFO:tasks.workunit.client.0.smithi026.stderr:+ cmp -l /tmp/rbd_import_export_456745/img /tmp/rbd_import_export_456745/img_import 2023-09-23T21:19:40.852 INFO:tasks.workunit.client.0.smithi026.stderr:++ stat /tmp/rbd_import_export_456745/img --format %b 2023-09-23T21:19:40.853 INFO:tasks.workunit.client.0.smithi026.stderr:+ origsize=24 2023-09-23T21:19:40.853 INFO:tasks.workunit.client.0.smithi026.stderr:++ stat /tmp/rbd_import_export_456745/img_import --format %b 2023-09-23T21:19:40.854 INFO:tasks.workunit.client.0.smithi026.stderr:+ exportsize=224 2023-09-23T21:19:40.854 INFO:tasks.workunit.client.0.smithi026.stderr:+ difference=200 2023-09-23T21:19:40.855 INFO:tasks.workunit.client.0.smithi026.stderr:+ difference=200 2023-09-23T21:19:40.855 INFO:tasks.workunit.client.0.smithi026.stderr:+ test 200 -ge 0 -a 200 -lt 4096 2023-09-23T21:19:40.855 INFO:tasks.workunit.client.0.smithi026.stderr:+ rbd export testimg@snap /tmp/rbd_import_export_456745/img_snap 2023-09-23T21:19:40.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:19:40 smithi026 ceph-mon[179688]: pgmap v1609: 103 pgs: 103 active+clean; 1.3 MiB data, 3.7 GiB used, 443 GiB / 447 GiB avail; 187 KiB/s rd, 95 KiB/s wr, 51 op/s; 0 B/s, 0 objects/s recovering 2023-09-23T21:19:40.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:19:40 smithi026 ceph-mon[179688]: failed to encode map e655 with expected crc 2023-09-23T21:19:40.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:19:40 smithi026 ceph-mon[179688]: failed to encode map e655 with expected crc 2023-09-23T21:19:40.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:19:40 smithi026 ceph-mon[179688]: failed to encode map e655 with expected crc 2023-09-23T21:19:40.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:19:40 smithi026 ceph-mon[179688]: osdmap e655: 8 total, 8 up, 5 in 2023-09-23T21:19:40.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:19:40 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/475010201' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:19:40.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:19:40 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:19:40.883 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:19:40 smithi026 ceph-mon[175151]: pgmap v1609: 103 pgs: 103 active+clean; 1.3 MiB data, 3.7 GiB used, 443 GiB / 447 GiB avail; 187 KiB/s rd, 95 KiB/s wr, 51 op/s; 0 B/s, 0 objects/s recovering 2023-09-23T21:19:40.883 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:19:40 smithi026 ceph-mon[175151]: failed to encode map e655 with expected crc 2023-09-23T21:19:40.883 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:19:40 smithi026 ceph-mon[175151]: failed to encode map e655 with expected crc 2023-09-23T21:19:40.883 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:19:40 smithi026 ceph-mon[175151]: failed to encode map e655 with expected crc 2023-09-23T21:19:40.883 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:19:40 smithi026 ceph-mon[175151]: osdmap e655: 8 total, 8 up, 5 in 2023-09-23T21:19:40.883 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:19:40 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/475010201' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:19:40.883 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:19:40 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:19:40.891 INFO:tasks.workunit.client.0.smithi026.stderr: Exporting image: 100% complete...done. 2023-09-23T21:19:40.896 INFO:tasks.workunit.client.0.smithi026.stderr:+ rbd export testimg_import@snap /tmp/rbd_import_export_456745/img_snap_import 2023-09-23T21:19:40.937 INFO:tasks.workunit.client.0.smithi026.stderr: Exporting image: 100% complete...done. 2023-09-23T21:19:40.941 INFO:tasks.workunit.client.0.smithi026.stderr:+ compare_files_and_ondisk_sizes /tmp/rbd_import_export_456745/img_snap /tmp/rbd_import_export_456745/img_snap_import 2023-09-23T21:19:40.941 INFO:tasks.workunit.client.0.smithi026.stderr:+ cmp -l /tmp/rbd_import_export_456745/img_snap /tmp/rbd_import_export_456745/img_snap_import 2023-09-23T21:19:40.942 INFO:tasks.workunit.client.0.smithi026.stderr:++ stat /tmp/rbd_import_export_456745/img_snap --format %b 2023-09-23T21:19:40.943 INFO:tasks.workunit.client.0.smithi026.stderr:+ origsize=224 2023-09-23T21:19:40.943 INFO:tasks.workunit.client.0.smithi026.stderr:++ stat /tmp/rbd_import_export_456745/img_snap_import --format %b 2023-09-23T21:19:40.944 INFO:tasks.workunit.client.0.smithi026.stderr:+ exportsize=224 2023-09-23T21:19:40.944 INFO:tasks.workunit.client.0.smithi026.stderr:+ difference=0 2023-09-23T21:19:40.944 INFO:tasks.workunit.client.0.smithi026.stderr:+ difference=0 2023-09-23T21:19:40.944 INFO:tasks.workunit.client.0.smithi026.stderr:+ test 0 -ge 0 -a 0 -lt 4096 2023-09-23T21:19:40.944 INFO:tasks.workunit.client.0.smithi026.stderr:+ rm /tmp/rbd_import_export_456745/img_v2 2023-09-23T21:19:40.945 INFO:tasks.workunit.client.0.smithi026.stderr:+ rm /tmp/rbd_import_export_456745/img_import 2023-09-23T21:19:40.945 INFO:tasks.workunit.client.0.smithi026.stderr:+ rm /tmp/rbd_import_export_456745/img_snap 2023-09-23T21:19:40.946 INFO:tasks.workunit.client.0.smithi026.stderr:+ rm /tmp/rbd_import_export_456745/img_snap_import 2023-09-23T21:19:40.947 INFO:tasks.workunit.client.0.smithi026.stderr:+ rbd snap rm testimg_import@snap 2023-09-23T21:19:41.034 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:19:40 smithi070 ceph-mon[168282]: pgmap v1609: 103 pgs: 103 active+clean; 1.3 MiB data, 3.7 GiB used, 443 GiB / 447 GiB avail; 187 KiB/s rd, 95 KiB/s wr, 51 op/s; 0 B/s, 0 objects/s recovering 2023-09-23T21:19:41.035 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:19:40 smithi070 ceph-mon[168282]: failed to encode map e655 with expected crc 2023-09-23T21:19:41.035 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:19:40 smithi070 ceph-mon[168282]: failed to encode map e655 with expected crc 2023-09-23T21:19:41.035 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:19:40 smithi070 ceph-mon[168282]: failed to encode map e655 with expected crc 2023-09-23T21:19:41.035 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:19:40 smithi070 ceph-mon[168282]: osdmap e655: 8 total, 8 up, 5 in 2023-09-23T21:19:41.035 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:19:40 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/475010201' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:19:41.035 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:19:40 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:19:41.408 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:19:41.412 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:41.412 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:19:41.447 INFO:teuthology.orchestra.run.smithi070.stdout:{ 2023-09-23T21:19:41.448 INFO:teuthology.orchestra.run.smithi070.stdout: "size": 20, 2023-09-23T21:19:41.448 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 600, 2023-09-23T21:19:41.448 INFO:teuthology.orchestra.run.smithi070.stdout: "ops": [ 2023-09-23T21:19:41.448 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:41.448 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.46328.0:12430 10.1 10:8e1b46c6:::benchmark_data_smithi026_379838_object12429:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e430)", 2023-09-23T21:19:41.448 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:10:32.729016+0000", 2023-09-23T21:19:41.448 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 548.71650365999994, 2023-09-23T21:19:41.448 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.96804327999999995, 2023-09-23T21:19:41.448 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:19:41.448 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:19:41.448 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:19:41.449 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.46328", 2023-09-23T21:19:41.449 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/3748861181", 2023-09-23T21:19:41.449 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 12430 2023-09-23T21:19:41.449 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:41.449 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:19:41.449 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:41.449 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:19:41.449 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:10:32.729016+0000", 2023-09-23T21:19:41.449 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:19:41.449 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:41.449 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:41.449 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:19:41.450 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:10:32.729016+0000", 2023-09-23T21:19:41.450 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:19:41.450 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:41.450 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:41.450 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:19:41.450 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:10:32.729015+0000", 2023-09-23T21:19:41.450 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T21:19:41.451 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:41.451 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:41.451 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:19:41.451 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:10:32.729127+0000", 2023-09-23T21:19:41.451 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000111895 2023-09-23T21:19:41.451 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:41.451 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:41.451 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:19:41.451 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:10:32.729138+0000", 2023-09-23T21:19:41.451 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.0900999999999999e-05 2023-09-23T21:19:41.451 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:41.452 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:41.452 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:19:41.452 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:10:32.729143+0000", 2023-09-23T21:19:41.452 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.3329999999999996e-06 2023-09-23T21:19:41.452 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:41.452 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:41.452 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:19:41.452 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:10:32.729222+0000", 2023-09-23T21:19:41.452 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.9169e-05 2023-09-23T21:19:41.452 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:41.452 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:41.452 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:19:41.453 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:10:32.729245+0000", 2023-09-23T21:19:41.453 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.3125e-05 2023-09-23T21:19:41.453 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:41.453 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:41.453 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:19:41.453 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:10:33.686993+0000", 2023-09-23T21:19:41.453 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.95774722400000001 2023-09-23T21:19:41.453 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:41.453 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:41.453 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:19:41.453 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:10:33.687143+0000", 2023-09-23T21:19:41.453 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00015004499999999999 2023-09-23T21:19:41.453 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:41.454 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:41.454 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 0,1", 2023-09-23T21:19:41.454 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:10:33.687248+0000", 2023-09-23T21:19:41.454 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000105204 2023-09-23T21:19:41.454 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:41.454 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:41.454 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:19:41.454 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:10:33.689705+0000", 2023-09-23T21:19:41.454 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0024572420000000001 2023-09-23T21:19:41.454 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:41.454 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:41.454 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:19:41.455 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:10:33.696947+0000", 2023-09-23T21:19:41.455 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0072415550000000002 2023-09-23T21:19:41.455 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:41.455 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:41.455 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:19:41.455 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:10:33.696974+0000", 2023-09-23T21:19:41.455 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.7412000000000001e-05 2023-09-23T21:19:41.455 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:41.455 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:41.455 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:19:41.455 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:10:33.697024+0000", 2023-09-23T21:19:41.455 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.9570000000000001e-05 2023-09-23T21:19:41.456 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:41.456 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:41.456 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:19:41.456 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:10:33.697060+0000", 2023-09-23T21:19:41.456 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.6023999999999999e-05 2023-09-23T21:19:41.456 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:19:41.456 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:19:41.456 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:19:41.456 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:41.456 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:41.456 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.46328.0:84576 10.c 10:3b2a76db:::benchmark_data_smithi026_379838_object4619:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e471)", 2023-09-23T21:19:41.456 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:12:10.844583+0000", 2023-09-23T21:19:41.456 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 450.600936752, 2023-09-23T21:19:41.457 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.511043044, 2023-09-23T21:19:41.457 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:19:41.457 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:19:41.457 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:19:41.457 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.46328", 2023-09-23T21:19:41.457 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/3748861181", 2023-09-23T21:19:41.457 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 84576 2023-09-23T21:19:41.457 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:41.457 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:19:41.457 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:41.457 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:19:41.457 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:12:10.844583+0000", 2023-09-23T21:19:41.458 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:19:41.458 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:41.458 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:41.458 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:19:41.458 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:12:10.844583+0000", 2023-09-23T21:19:41.458 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:19:41.458 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:41.458 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:41.458 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:19:41.458 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:12:10.844581+0000", 2023-09-23T21:19:41.459 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999981 2023-09-23T21:19:41.459 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:41.459 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:41.459 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:19:41.459 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:12:10.844591+0000", 2023-09-23T21:19:41.459 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.098e-06 2023-09-23T21:19:41.459 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:41.459 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:41.459 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:19:41.459 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:12:10.844609+0000", 2023-09-23T21:19:41.459 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.823e-05 2023-09-23T21:19:41.459 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:41.460 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:41.460 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:19:41.460 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:12:10.844617+0000", 2023-09-23T21:19:41.460 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.2509999999999994e-06 2023-09-23T21:19:41.460 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:41.460 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:41.460 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:19:41.460 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:12:10.844680+0000", 2023-09-23T21:19:41.460 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.3287999999999994e-05 2023-09-23T21:19:41.460 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:41.460 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:41.460 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:19:41.460 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:12:10.844854+0000", 2023-09-23T21:19:41.461 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000173163 2023-09-23T21:19:41.461 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:41.461 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:41.461 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 4", 2023-09-23T21:19:41.461 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:12:10.844964+0000", 2023-09-23T21:19:41.461 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000110653 2023-09-23T21:19:41.461 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:41.461 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:41.461 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:19:41.461 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:12:10.847302+0000", 2023-09-23T21:19:41.461 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0023382170000000001 2023-09-23T21:19:41.462 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:41.462 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:41.462 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:19:41.462 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:12:14.355481+0000", 2023-09-23T21:19:41.462 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.5081785220000001 2023-09-23T21:19:41.462 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:41.462 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:41.462 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:19:41.462 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:12:14.355532+0000", 2023-09-23T21:19:41.463 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.1251000000000001e-05 2023-09-23T21:19:41.463 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:41.463 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:41.463 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:19:41.463 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:12:14.355626+0000", 2023-09-23T21:19:41.463 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.4170000000000001e-05 2023-09-23T21:19:41.463 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:19:41.463 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:19:41.463 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:19:41.463 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:41.463 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:41.464 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.46328.0:84616 10.c 10:3d3dd8d1:::benchmark_data_smithi026_379838_object4659:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e472)", 2023-09-23T21:19:41.464 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:12:11.942844+0000", 2023-09-23T21:19:41.464 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 449.50267558500002, 2023-09-23T21:19:41.464 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.418419783, 2023-09-23T21:19:41.464 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:19:41.464 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:19:41.464 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:19:41.464 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.46328", 2023-09-23T21:19:41.464 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/3748861181", 2023-09-23T21:19:41.464 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 84616 2023-09-23T21:19:41.464 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:41.465 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:19:41.465 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:41.465 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:19:41.465 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:12:11.942844+0000", 2023-09-23T21:19:41.465 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:19:41.465 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:41.465 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:41.465 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:19:41.465 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:12:11.942844+0000", 2023-09-23T21:19:41.465 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:19:41.465 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:41.465 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:41.466 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:19:41.466 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:12:11.942843+0000", 2023-09-23T21:19:41.466 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T21:19:41.466 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:41.466 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:41.466 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:19:41.466 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:12:11.942853+0000", 2023-09-23T21:19:41.466 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.8509999999999994e-06 2023-09-23T21:19:41.466 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:41.466 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:41.466 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:19:41.466 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:12:11.942864+0000", 2023-09-23T21:19:41.467 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.1440999999999999e-05 2023-09-23T21:19:41.467 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:41.467 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:41.467 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:19:41.467 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:12:11.942872+0000", 2023-09-23T21:19:41.467 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.5669999999999997e-06 2023-09-23T21:19:41.467 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:41.467 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:41.467 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:19:41.467 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:12:11.942924+0000", 2023-09-23T21:19:41.467 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.1711e-05 2023-09-23T21:19:41.467 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:41.468 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:41.468 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:19:41.468 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:12:11.943052+0000", 2023-09-23T21:19:41.468 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00012839399999999999 2023-09-23T21:19:41.468 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:41.468 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:41.468 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 4", 2023-09-23T21:19:41.468 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:12:11.943174+0000", 2023-09-23T21:19:41.468 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000122368 2023-09-23T21:19:41.468 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:41.468 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:41.468 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:19:41.469 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:12:11.945587+0000", 2023-09-23T21:19:41.469 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0024131119999999998 2023-09-23T21:19:41.469 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:41.469 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:41.469 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:19:41.469 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:12:14.361092+0000", 2023-09-23T21:19:41.469 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.4155044399999999 2023-09-23T21:19:41.469 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:41.469 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:41.469 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:19:41.469 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:12:14.361151+0000", 2023-09-23T21:19:41.469 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.8795999999999997e-05 2023-09-23T21:19:41.470 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:41.470 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:41.470 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:19:41.470 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:12:14.361264+0000", 2023-09-23T21:19:41.470 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00011353600000000001 2023-09-23T21:19:41.470 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:19:41.470 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:19:41.470 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:19:41.470 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:41.470 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:41.470 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.46328.0:84636 10.c 10:3153f2e1:::benchmark_data_smithi026_379838_object4679:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e472)", 2023-09-23T21:19:41.470 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:12:12.042148+0000", 2023-09-23T21:19:41.471 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 449.40337232100001, 2023-09-23T21:19:41.471 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.3219429410000001, 2023-09-23T21:19:41.471 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:19:41.471 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:19:41.471 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:19:41.471 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.46328", 2023-09-23T21:19:41.471 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/3748861181", 2023-09-23T21:19:41.471 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 84636 2023-09-23T21:19:41.471 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:41.471 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:19:41.471 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:41.471 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:19:41.471 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:12:12.042148+0000", 2023-09-23T21:19:41.472 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:19:41.472 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:41.472 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:41.472 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:19:41.472 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:12:12.042148+0000", 2023-09-23T21:19:41.472 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:19:41.472 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:41.472 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:41.472 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:19:41.472 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:12:12.042146+0000", 2023-09-23T21:19:41.472 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T21:19:41.472 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:41.473 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:41.473 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:19:41.473 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:12:12.042155+0000", 2023-09-23T21:19:41.473 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.6279999999999994e-06 2023-09-23T21:19:41.473 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:41.473 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:41.473 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:19:41.473 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:12:12.042173+0000", 2023-09-23T21:19:41.473 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.7943000000000001e-05 2023-09-23T21:19:41.473 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:41.473 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:41.473 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:19:41.474 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:12:12.042181+0000", 2023-09-23T21:19:41.474 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.7399999999999993e-06 2023-09-23T21:19:41.474 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:41.474 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:41.474 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:19:41.474 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:12:12.042261+0000", 2023-09-23T21:19:41.474 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.9375999999999994e-05 2023-09-23T21:19:41.474 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:41.474 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:41.474 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:19:41.474 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:12:12.042414+0000", 2023-09-23T21:19:41.474 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00015368399999999999 2023-09-23T21:19:41.474 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:41.475 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:41.475 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 4", 2023-09-23T21:19:41.475 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:12:12.042558+0000", 2023-09-23T21:19:41.475 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000143747 2023-09-23T21:19:41.475 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:41.475 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:41.475 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:19:41.475 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:12:12.045442+0000", 2023-09-23T21:19:41.475 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.002883891 2023-09-23T21:19:41.475 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:41.475 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:41.475 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:19:41.476 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:12:14.363936+0000", 2023-09-23T21:19:41.476 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.3184934959999999 2023-09-23T21:19:41.476 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:41.476 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:41.476 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:19:41.476 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:12:14.363990+0000", 2023-09-23T21:19:41.476 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.4308000000000002e-05 2023-09-23T21:19:41.476 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:41.476 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:41.476 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:19:41.476 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:12:14.364091+0000", 2023-09-23T21:19:41.476 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00010072 2023-09-23T21:19:41.477 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:19:41.477 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:19:41.477 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:19:41.477 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:41.477 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:41.477 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.46328.0:84650 10.c 10:3f92ba16:::benchmark_data_smithi026_379838_object4693:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e472)", 2023-09-23T21:19:41.477 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:12:12.183251+0000", 2023-09-23T21:19:41.477 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 449.26226866299999, 2023-09-23T21:19:41.477 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.1852957819999999, 2023-09-23T21:19:41.477 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:19:41.477 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:19:41.477 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:19:41.477 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.46328", 2023-09-23T21:19:41.478 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/3748861181", 2023-09-23T21:19:41.478 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 84650 2023-09-23T21:19:41.478 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:41.478 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:19:41.478 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:41.478 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:19:41.478 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:12:12.183251+0000", 2023-09-23T21:19:41.478 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:19:41.478 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:41.478 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:41.478 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:19:41.478 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:12:12.183251+0000", 2023-09-23T21:19:41.479 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:19:41.479 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:41.479 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:41.479 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:19:41.479 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:12:12.183249+0000", 2023-09-23T21:19:41.479 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999981 2023-09-23T21:19:41.479 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:41.479 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:41.479 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:19:41.479 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:12:12.183260+0000", 2023-09-23T21:19:41.479 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.0908e-05 2023-09-23T21:19:41.479 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:41.479 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:41.480 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:19:41.480 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:12:12.183283+0000", 2023-09-23T21:19:41.480 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.3317000000000001e-05 2023-09-23T21:19:41.480 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:41.480 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:41.480 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:19:41.480 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:12:12.183294+0000", 2023-09-23T21:19:41.480 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.0893000000000001e-05 2023-09-23T21:19:41.480 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:41.480 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:41.480 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:19:41.480 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:12:12.183355+0000", 2023-09-23T21:19:41.481 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.0557000000000003e-05 2023-09-23T21:19:41.481 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:41.481 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:41.481 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:19:41.481 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:12:12.183467+0000", 2023-09-23T21:19:41.481 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000111694 2023-09-23T21:19:41.481 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:41.481 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:41.481 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 4", 2023-09-23T21:19:41.481 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:12:12.183562+0000", 2023-09-23T21:19:41.481 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.5743999999999997e-05 2023-09-23T21:19:41.481 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:41.481 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:41.482 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:19:41.482 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:12:12.186356+0000", 2023-09-23T21:19:41.482 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.002794 2023-09-23T21:19:41.482 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:41.482 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:41.482 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:19:41.482 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:12:14.368394+0000", 2023-09-23T21:19:41.482 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.1820373540000002 2023-09-23T21:19:41.482 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:41.482 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:41.482 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:19:41.482 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:12:14.368450+0000", 2023-09-23T21:19:41.483 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.6008000000000003e-05 2023-09-23T21:19:41.483 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:41.483 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:41.483 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:19:41.483 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:12:14.368547+0000", 2023-09-23T21:19:41.483 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.7404000000000005e-05 2023-09-23T21:19:41.483 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:19:41.483 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:19:41.483 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:19:41.483 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:41.483 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:41.483 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.46328.0:99131 10.11 10:8be3a531:::benchmark_data_smithi026_379838_object19174:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e490)", 2023-09-23T21:19:41.484 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:13:08.165674+0000", 2023-09-23T21:19:41.484 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 393.27984615000003, 2023-09-23T21:19:41.484 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.0267941519999999, 2023-09-23T21:19:41.484 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:19:41.484 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:19:41.484 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:19:41.484 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.46328", 2023-09-23T21:19:41.484 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/3748861181", 2023-09-23T21:19:41.484 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 99131 2023-09-23T21:19:41.484 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:41.484 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:19:41.484 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:41.484 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:19:41.485 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:08.165674+0000", 2023-09-23T21:19:41.485 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:19:41.485 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:41.485 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:41.485 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:19:41.485 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:08.165674+0000", 2023-09-23T21:19:41.485 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:19:41.485 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:41.485 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:41.485 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:19:41.485 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:08.165672+0000", 2023-09-23T21:19:41.485 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999976 2023-09-23T21:19:41.486 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:41.486 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:41.486 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:19:41.486 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:08.165680+0000", 2023-09-23T21:19:41.486 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.8590000000000005e-06 2023-09-23T21:19:41.486 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:41.486 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:41.486 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:19:41.486 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:08.165697+0000", 2023-09-23T21:19:41.486 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.7915999999999999e-05 2023-09-23T21:19:41.486 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:41.486 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:41.487 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:19:41.487 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:08.165706+0000", 2023-09-23T21:19:41.487 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.2709999999999992e-06 2023-09-23T21:19:41.487 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:41.487 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:41.487 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:19:41.487 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:08.165764+0000", 2023-09-23T21:19:41.487 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.8168000000000003e-05 2023-09-23T21:19:41.487 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:41.487 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:41.489 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:19:41.489 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:08.165773+0000", 2023-09-23T21:19:41.489 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.8000000000000004e-06 2023-09-23T21:19:41.489 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:41.489 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:41.489 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:19:41.489 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:09.168140+0000", 2023-09-23T21:19:41.489 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.002367622 2023-09-23T21:19:41.489 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:41.489 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:41.490 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:19:41.490 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:09.168266+0000", 2023-09-23T21:19:41.490 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000125475 2023-09-23T21:19:41.490 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:41.490 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:41.490 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 2,5", 2023-09-23T21:19:41.490 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:09.168387+0000", 2023-09-23T21:19:41.490 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000120924 2023-09-23T21:19:41.490 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:41.490 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:41.490 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:19:41.490 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:09.182822+0000", 2023-09-23T21:19:41.491 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.014435694000000001 2023-09-23T21:19:41.491 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:41.491 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:41.491 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:19:41.491 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:09.192311+0000", 2023-09-23T21:19:41.491 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0094883820000000001 2023-09-23T21:19:41.491 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:41.491 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:41.491 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:19:41.491 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:09.192385+0000", 2023-09-23T21:19:41.491 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.4046000000000002e-05 2023-09-23T21:19:41.491 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:41.491 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:41.492 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:19:41.492 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:09.192432+0000", 2023-09-23T21:19:41.492 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.6811999999999998e-05 2023-09-23T21:19:41.492 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:41.492 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:41.492 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:19:41.492 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:09.192468+0000", 2023-09-23T21:19:41.492 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.6359000000000002e-05 2023-09-23T21:19:41.492 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:19:41.492 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:19:41.492 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:19:41.493 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:41.493 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:41.493 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.46328.0:99137 10.11 10:8f77d61b:::benchmark_data_smithi026_379838_object19180:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e490)", 2023-09-23T21:19:41.493 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:13:08.177453+0000", 2023-09-23T21:19:41.493 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 393.26806661400002, 2023-09-23T21:19:41.493 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.015184732, 2023-09-23T21:19:41.493 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:19:41.493 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:19:41.493 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:19:41.493 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.46328", 2023-09-23T21:19:41.493 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/3748861181", 2023-09-23T21:19:41.493 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 99137 2023-09-23T21:19:41.493 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:41.494 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:19:41.494 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:41.494 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:19:41.494 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:08.177453+0000", 2023-09-23T21:19:41.494 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:19:41.494 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:41.494 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:41.494 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:19:41.494 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:08.177453+0000", 2023-09-23T21:19:41.494 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:19:41.494 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:41.494 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:41.495 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:19:41.495 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:08.177452+0000", 2023-09-23T21:19:41.495 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T21:19:41.495 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:41.495 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:41.495 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:19:41.495 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:08.177459+0000", 2023-09-23T21:19:41.495 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.1640000000000004e-06 2023-09-23T21:19:41.495 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:41.495 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:41.495 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:19:41.495 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:08.177471+0000", 2023-09-23T21:19:41.495 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.1389999999999999e-05 2023-09-23T21:19:41.496 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:41.496 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:41.496 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:19:41.496 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:08.177476+0000", 2023-09-23T21:19:41.496 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.6559999999999997e-06 2023-09-23T21:19:41.496 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:41.496 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:41.496 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:19:41.496 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:08.177530+0000", 2023-09-23T21:19:41.496 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.3658e-05 2023-09-23T21:19:41.496 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:41.496 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:41.497 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:19:41.497 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:08.177539+0000", 2023-09-23T21:19:41.497 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.8689999999999993e-06 2023-09-23T21:19:41.497 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:41.497 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:41.497 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:19:41.497 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:09.170152+0000", 2023-09-23T21:19:41.497 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.99261289500000005 2023-09-23T21:19:41.497 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:41.497 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:41.497 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:19:41.497 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:09.170272+0000", 2023-09-23T21:19:41.497 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00012026199999999999 2023-09-23T21:19:41.498 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:41.498 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:41.498 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 2,5", 2023-09-23T21:19:41.498 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:09.170397+0000", 2023-09-23T21:19:41.498 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000124797 2023-09-23T21:19:41.498 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:41.498 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:41.498 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:19:41.498 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:09.182836+0000", 2023-09-23T21:19:41.498 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.012439019000000001 2023-09-23T21:19:41.498 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:41.498 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:41.499 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:19:41.499 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:09.192337+0000", 2023-09-23T21:19:41.499 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0095009229999999997 2023-09-23T21:19:41.499 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:41.499 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:41.499 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:19:41.499 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:09.192568+0000", 2023-09-23T21:19:41.499 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00023114500000000001 2023-09-23T21:19:41.499 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:41.499 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:41.499 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:19:41.499 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:09.192606+0000", 2023-09-23T21:19:41.500 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.8201000000000002e-05 2023-09-23T21:19:41.500 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:41.500 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:41.500 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:19:41.500 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:09.192638+0000", 2023-09-23T21:19:41.500 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.2156999999999997e-05 2023-09-23T21:19:41.500 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:19:41.500 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:19:41.500 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:19:41.500 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:41.500 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:41.500 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.46328.0:99138 10.11 10:8adc2e9e:::benchmark_data_smithi026_379838_object19181:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e490)", 2023-09-23T21:19:41.500 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:13:08.182950+0000", 2023-09-23T21:19:41.501 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 393.262570423, 2023-09-23T21:19:41.501 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.0124994359999999, 2023-09-23T21:19:41.501 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:19:41.501 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:19:41.501 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:19:41.501 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.46328", 2023-09-23T21:19:41.501 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/3748861181", 2023-09-23T21:19:41.501 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 99138 2023-09-23T21:19:41.501 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:41.501 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:19:41.501 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:41.501 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:19:41.502 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:08.182950+0000", 2023-09-23T21:19:41.502 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:19:41.502 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:41.502 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:41.502 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:19:41.502 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:08.182950+0000", 2023-09-23T21:19:41.502 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:19:41.502 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:41.502 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:41.502 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:19:41.502 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:08.182948+0000", 2023-09-23T21:19:41.502 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T21:19:41.503 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:41.503 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:41.503 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:19:41.503 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:08.182956+0000", 2023-09-23T21:19:41.503 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.1669999999999999e-06 2023-09-23T21:19:41.503 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:41.503 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:41.503 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:19:41.503 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:08.182976+0000", 2023-09-23T21:19:41.503 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.9758e-05 2023-09-23T21:19:41.503 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:41.503 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:41.503 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:19:41.504 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:08.182983+0000", 2023-09-23T21:19:41.504 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.2799999999999998e-06 2023-09-23T21:19:41.504 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:41.504 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:41.504 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:19:41.504 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:08.183043+0000", 2023-09-23T21:19:41.504 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.0022999999999998e-05 2023-09-23T21:19:41.504 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:41.504 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:41.504 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:19:41.504 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:08.183053+0000", 2023-09-23T21:19:41.504 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.005e-05 2023-09-23T21:19:41.504 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:41.505 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:41.505 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:19:41.505 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:09.172086+0000", 2023-09-23T21:19:41.505 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.98903244300000004 2023-09-23T21:19:41.505 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:41.505 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:41.505 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:19:41.505 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:09.172158+0000", 2023-09-23T21:19:41.505 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.2646999999999997e-05 2023-09-23T21:19:41.505 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:41.505 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:41.505 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 2,5", 2023-09-23T21:19:41.505 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:09.174279+0000", 2023-09-23T21:19:41.506 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0021205820000000002 2023-09-23T21:19:41.506 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:41.506 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:41.506 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:19:41.506 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:09.182841+0000", 2023-09-23T21:19:41.506 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0085623030000000003 2023-09-23T21:19:41.506 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:41.506 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:41.506 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:19:41.506 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:09.192361+0000", 2023-09-23T21:19:41.506 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0095194019999999997 2023-09-23T21:19:41.507 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:41.507 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:41.507 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:19:41.507 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:09.195355+0000", 2023-09-23T21:19:41.507 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0029945000000000002 2023-09-23T21:19:41.507 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:41.507 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:41.507 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:19:41.507 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:09.195404+0000", 2023-09-23T21:19:41.507 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.9138999999999998e-05 2023-09-23T21:19:41.507 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:41.507 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:41.508 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:19:41.508 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:09.195449+0000", 2023-09-23T21:19:41.508 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.4781999999999999e-05 2023-09-23T21:19:41.508 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:19:41.508 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:19:41.508 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:19:41.508 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:41.508 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:41.508 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.46328.0:99977 10.c 10:3f71b81a:::benchmark_data_smithi026_379838_object20020:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e493)", 2023-09-23T21:19:41.508 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:13:11.836139+0000", 2023-09-23T21:19:41.508 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 389.60938079200002, 2023-09-23T21:19:41.508 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.594515414, 2023-09-23T21:19:41.508 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:19:41.509 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:19:41.509 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:19:41.509 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.46328", 2023-09-23T21:19:41.509 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/3748861181", 2023-09-23T21:19:41.509 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 99977 2023-09-23T21:19:41.509 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:41.509 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:19:41.509 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:41.509 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:19:41.509 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:11.836139+0000", 2023-09-23T21:19:41.509 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:19:41.509 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:41.510 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:41.510 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:19:41.510 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:11.836139+0000", 2023-09-23T21:19:41.510 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:19:41.510 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:41.510 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:41.510 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:19:41.510 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:11.836138+0000", 2023-09-23T21:19:41.510 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T21:19:41.510 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:41.510 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:41.511 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:19:41.511 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:11.836143+0000", 2023-09-23T21:19:41.511 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.293e-06 2023-09-23T21:19:41.511 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:41.511 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:41.511 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:19:41.511 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:11.836157+0000", 2023-09-23T21:19:41.511 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.3258000000000001e-05 2023-09-23T21:19:41.511 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:41.511 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:41.511 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:19:41.511 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:11.836162+0000", 2023-09-23T21:19:41.512 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.4979999999999997e-06 2023-09-23T21:19:41.512 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:41.512 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:41.512 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:19:41.512 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:11.836212+0000", 2023-09-23T21:19:41.512 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.0225000000000001e-05 2023-09-23T21:19:41.512 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:41.512 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:41.512 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:19:41.512 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:11.836222+0000", 2023-09-23T21:19:41.512 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.8959999999999998e-06 2023-09-23T21:19:41.512 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:41.513 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:41.513 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:19:41.513 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:12.185368+0000", 2023-09-23T21:19:41.513 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.34914541399999999 2023-09-23T21:19:41.513 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:41.513 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:41.513 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:19:41.513 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:12.185381+0000", 2023-09-23T21:19:41.513 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.3546e-05 2023-09-23T21:19:41.513 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:41.513 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:41.513 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:19:41.514 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:13.420174+0000", 2023-09-23T21:19:41.514 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.2347927400000001 2023-09-23T21:19:41.514 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:41.514 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:41.514 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:19:41.514 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:13.420332+0000", 2023-09-23T21:19:41.514 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00015844899999999999 2023-09-23T21:19:41.514 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:41.514 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:41.514 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 0,4", 2023-09-23T21:19:41.514 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:13.420503+0000", 2023-09-23T21:19:41.514 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000170373 2023-09-23T21:19:41.515 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:41.515 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:41.515 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:19:41.515 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:13.425495+0000", 2023-09-23T21:19:41.515 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0049922029999999997 2023-09-23T21:19:41.515 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:41.515 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:41.515 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:19:41.515 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:13.425508+0000", 2023-09-23T21:19:41.515 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.2761e-05 2023-09-23T21:19:41.515 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:41.515 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:41.515 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:19:41.516 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:13.430310+0000", 2023-09-23T21:19:41.516 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0048026220000000003 2023-09-23T21:19:41.516 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:41.516 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:41.516 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:19:41.516 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:13.430370+0000", 2023-09-23T21:19:41.516 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.012e-05 2023-09-23T21:19:41.516 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:41.516 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:41.516 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:19:41.516 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:13.430655+0000", 2023-09-23T21:19:41.516 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00028428600000000002 2023-09-23T21:19:41.517 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:19:41.517 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:19:41.517 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:19:41.517 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:41.517 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:41.517 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.46328.0:99978 10.c 10:38231695:::benchmark_data_smithi026_379838_object20021:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e493)", 2023-09-23T21:19:41.517 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:13:11.847037+0000", 2023-09-23T21:19:41.517 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 389.59848272099998, 2023-09-23T21:19:41.517 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.583935267, 2023-09-23T21:19:41.517 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:19:41.517 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:19:41.517 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:19:41.518 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.46328", 2023-09-23T21:19:41.518 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/3748861181", 2023-09-23T21:19:41.518 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 99978 2023-09-23T21:19:41.518 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:41.518 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:19:41.518 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:41.518 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:19:41.518 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:11.847037+0000", 2023-09-23T21:19:41.518 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:19:41.518 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:41.518 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:41.518 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:19:41.519 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:11.847037+0000", 2023-09-23T21:19:41.519 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:19:41.519 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:41.519 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:41.519 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:19:41.519 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:11.847036+0000", 2023-09-23T21:19:41.519 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.999999 2023-09-23T21:19:41.519 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:41.519 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:41.519 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:19:41.519 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:11.847042+0000", 2023-09-23T21:19:41.519 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.4060000000000004e-06 2023-09-23T21:19:41.519 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:41.520 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:41.520 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:19:41.520 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:11.847057+0000", 2023-09-23T21:19:41.520 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.4637999999999999e-05 2023-09-23T21:19:41.520 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:41.520 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:41.520 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:19:41.520 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:11.847062+0000", 2023-09-23T21:19:41.520 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.0370000000000002e-06 2023-09-23T21:19:41.520 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:41.520 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:41.520 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:19:41.521 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:11.847110+0000", 2023-09-23T21:19:41.521 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.8687999999999999e-05 2023-09-23T21:19:41.521 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:41.521 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:41.521 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:19:41.521 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:11.847121+0000", 2023-09-23T21:19:41.521 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.0318e-05 2023-09-23T21:19:41.521 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:41.521 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:41.521 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:19:41.521 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:12.185400+0000", 2023-09-23T21:19:41.521 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.33827918099999998 2023-09-23T21:19:41.521 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:41.522 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:41.522 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:19:41.522 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:12.185404+0000", 2023-09-23T21:19:41.522 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.1219999999999997e-06 2023-09-23T21:19:41.522 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:41.522 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:41.522 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:19:41.522 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:13.422861+0000", 2023-09-23T21:19:41.522 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.2374567430000001 2023-09-23T21:19:41.522 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:41.522 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:41.522 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:19:41.523 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:13.423010+0000", 2023-09-23T21:19:41.523 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00014951799999999999 2023-09-23T21:19:41.523 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:41.523 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:41.523 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 0,4", 2023-09-23T21:19:41.523 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:13.423186+0000", 2023-09-23T21:19:41.523 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00017603200000000001 2023-09-23T21:19:41.523 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:41.523 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:41.523 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:19:41.523 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:13.425518+0000", 2023-09-23T21:19:41.523 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0023320609999999999 2023-09-23T21:19:41.524 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:41.524 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:41.524 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:19:41.524 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:13.425561+0000", 2023-09-23T21:19:41.524 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.2370000000000003e-05 2023-09-23T21:19:41.524 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:41.524 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:41.524 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:19:41.524 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:13.430729+0000", 2023-09-23T21:19:41.524 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0051681549999999998 2023-09-23T21:19:41.524 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:41.524 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:41.524 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:19:41.525 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:13.430788+0000", 2023-09-23T21:19:41.525 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.8826999999999998e-05 2023-09-23T21:19:41.525 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:41.525 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:41.525 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:19:41.525 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:13:13.430973+0000", 2023-09-23T21:19:41.525 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000184997 2023-09-23T21:19:41.525 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:19:41.525 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:19:41.525 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:19:41.525 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:41.525 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:41.526 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.46823.0:78878 11.13 11.673fe373 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e564)", 2023-09-23T21:19:41.526 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:16:41.442423+0000", 2023-09-23T21:19:41.526 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 180.00309749100001, 2023-09-23T21:19:41.526 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.96127208099999994, 2023-09-23T21:19:41.526 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:19:41.526 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "reached pg", 2023-09-23T21:19:41.526 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:19:41.526 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.46823", 2023-09-23T21:19:41.526 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/2490075711", 2023-09-23T21:19:41.526 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 78878 2023-09-23T21:19:41.526 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:41.526 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:19:41.527 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:41.527 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:19:41.527 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:16:41.442423+0000", 2023-09-23T21:19:41.527 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:19:41.527 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:41.527 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:41.527 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:19:41.527 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:16:41.442423+0000", 2023-09-23T21:19:41.527 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:19:41.527 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:41.527 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:41.527 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:19:41.528 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:16:41.442420+0000", 2023-09-23T21:19:41.528 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999976 2023-09-23T21:19:41.528 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:41.528 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:41.528 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:19:41.528 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:16:41.442544+0000", 2023-09-23T21:19:41.528 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00012382100000000001 2023-09-23T21:19:41.528 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:41.528 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:41.528 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:19:41.528 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:16:41.442884+0000", 2023-09-23T21:19:41.528 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00033983100000000002 2023-09-23T21:19:41.529 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:41.529 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:41.529 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:19:41.529 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:16:41.442895+0000", 2023-09-23T21:19:41.529 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.1404e-05 2023-09-23T21:19:41.529 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:41.529 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:41.529 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:19:41.529 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:16:41.442968+0000", 2023-09-23T21:19:41.529 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.2801999999999999e-05 2023-09-23T21:19:41.529 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:41.529 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:41.530 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:19:41.530 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:16:41.442982+0000", 2023-09-23T21:19:41.530 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.3477e-05 2023-09-23T21:19:41.530 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:41.530 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:41.530 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:19:41.530 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:16:42.403685+0000", 2023-09-23T21:19:41.530 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.96070322100000005 2023-09-23T21:19:41.530 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:41.530 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:41.530 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:19:41.530 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:16:42.403695+0000", 2023-09-23T21:19:41.531 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.8179999999999999e-06 2023-09-23T21:19:41.531 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:19:41.531 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:19:41.531 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:19:41.531 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:41.531 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:41.531 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.46823.0:78879 11.4 11.ec4574c4 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e564)", 2023-09-23T21:19:41.531 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:16:41.445700+0000", 2023-09-23T21:19:41.531 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 179.999820414, 2023-09-23T21:19:41.531 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.95796339399999997, 2023-09-23T21:19:41.531 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:19:41.531 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "reached pg", 2023-09-23T21:19:41.532 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:19:41.532 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.46823", 2023-09-23T21:19:41.532 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/2490075711", 2023-09-23T21:19:41.532 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 78879 2023-09-23T21:19:41.532 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:41.532 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:19:41.532 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:41.532 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:19:41.532 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:16:41.445700+0000", 2023-09-23T21:19:41.532 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:19:41.532 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:41.532 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:41.532 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:19:41.533 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:16:41.445700+0000", 2023-09-23T21:19:41.533 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:19:41.533 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:41.533 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:41.533 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:19:41.533 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:16:41.445698+0000", 2023-09-23T21:19:41.533 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T21:19:41.533 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:41.533 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:41.533 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:19:41.533 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:16:41.445787+0000", 2023-09-23T21:19:41.534 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.8627000000000004e-05 2023-09-23T21:19:41.534 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:41.534 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:41.534 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:19:41.534 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:16:41.445800+0000", 2023-09-23T21:19:41.534 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.324e-05 2023-09-23T21:19:41.534 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:41.534 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:41.534 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:19:41.534 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:16:41.445808+0000", 2023-09-23T21:19:41.534 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.909e-06 2023-09-23T21:19:41.534 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:41.535 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:41.535 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:19:41.535 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:16:41.447324+0000", 2023-09-23T21:19:41.535 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0015165879999999999 2023-09-23T21:19:41.535 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:41.535 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:41.535 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:19:41.535 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:16:41.447335+0000", 2023-09-23T21:19:41.535 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.0489000000000001e-05 2023-09-23T21:19:41.535 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:41.535 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:41.535 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:19:41.535 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:16:42.403651+0000", 2023-09-23T21:19:41.536 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.95631578299999997 2023-09-23T21:19:41.536 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:41.536 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:41.536 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:19:41.536 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:16:42.403663+0000", 2023-09-23T21:19:41.536 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.2391e-05 2023-09-23T21:19:41.536 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:19:41.536 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:19:41.536 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:19:41.536 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:41.536 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:41.536 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.46823.0:78884 11.14 11.89e78494 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e564)", 2023-09-23T21:19:41.537 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:16:41.456798+0000", 2023-09-23T21:19:41.537 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 179.98872187000001, 2023-09-23T21:19:41.537 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.94574142900000002, 2023-09-23T21:19:41.537 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:19:41.537 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "reached pg", 2023-09-23T21:19:41.537 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:19:41.537 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.46823", 2023-09-23T21:19:41.537 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/2490075711", 2023-09-23T21:19:41.537 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 78884 2023-09-23T21:19:41.537 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:41.537 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:19:41.537 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:41.537 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:19:41.538 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:16:41.456798+0000", 2023-09-23T21:19:41.538 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:19:41.538 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:41.538 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:41.538 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:19:41.538 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:16:41.456798+0000", 2023-09-23T21:19:41.538 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:19:41.538 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:41.538 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:41.538 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:19:41.538 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:16:41.456797+0000", 2023-09-23T21:19:41.538 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T21:19:41.539 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:41.539 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:41.539 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:19:41.539 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:16:41.456882+0000", 2023-09-23T21:19:41.539 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.4870000000000006e-05 2023-09-23T21:19:41.539 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:41.539 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:41.539 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:19:41.539 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:16:41.456897+0000", 2023-09-23T21:19:41.539 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.5135e-05 2023-09-23T21:19:41.539 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:41.539 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:41.540 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:19:41.540 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:16:41.456904+0000", 2023-09-23T21:19:41.540 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.7920000000000004e-06 2023-09-23T21:19:41.540 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:41.540 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:41.540 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:19:41.540 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:16:41.456954+0000", 2023-09-23T21:19:41.540 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.0022999999999999e-05 2023-09-23T21:19:41.540 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:41.540 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:41.540 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:19:41.540 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:16:41.456963+0000", 2023-09-23T21:19:41.541 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.0480000000000004e-06 2023-09-23T21:19:41.541 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:41.541 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:41.541 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:19:41.541 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:16:42.402524+0000", 2023-09-23T21:19:41.541 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.94556156099999999 2023-09-23T21:19:41.541 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:41.541 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:41.541 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:19:41.541 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:16:42.402540+0000", 2023-09-23T21:19:41.541 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.524e-05 2023-09-23T21:19:41.541 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:19:41.542 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:19:41.542 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:19:41.542 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:41.542 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:41.542 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.46823.0:132646 11.13 11:ca339c49:::benchmark_data_smithi026_424533_object34676:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e585)", 2023-09-23T21:19:41.542 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:17:38.907633+0000", 2023-09-23T21:19:41.542 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 122.537886864, 2023-09-23T21:19:41.542 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.0115563889999999, 2023-09-23T21:19:41.542 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:19:41.542 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "reached pg", 2023-09-23T21:19:41.542 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:19:41.542 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.46823", 2023-09-23T21:19:41.543 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/2490075711", 2023-09-23T21:19:41.543 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 132646 2023-09-23T21:19:41.543 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:41.543 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:19:41.543 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:41.543 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:19:41.543 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:17:38.907633+0000", 2023-09-23T21:19:41.543 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:19:41.543 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:41.543 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:41.543 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:19:41.543 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:17:38.907633+0000", 2023-09-23T21:19:41.544 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:19:41.544 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:41.544 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:41.544 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:19:41.544 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:17:38.907632+0000", 2023-09-23T21:19:41.544 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T21:19:41.544 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:41.544 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:41.544 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:19:41.544 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:17:38.907638+0000", 2023-09-23T21:19:41.544 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.4930000000000003e-06 2023-09-23T21:19:41.544 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:41.545 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:41.545 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:19:41.545 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:17:38.907651+0000", 2023-09-23T21:19:41.545 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.261e-05 2023-09-23T21:19:41.545 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:41.545 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:41.545 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:19:41.545 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:17:38.907658+0000", 2023-09-23T21:19:41.545 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.5979999999999999e-06 2023-09-23T21:19:41.545 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:41.545 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:41.545 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:19:41.546 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:17:38.908926+0000", 2023-09-23T21:19:41.546 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.001268087 2023-09-23T21:19:41.546 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:41.546 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:41.546 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:19:41.546 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:17:38.908935+0000", 2023-09-23T21:19:41.546 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.8699999999999998e-06 2023-09-23T21:19:41.546 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:41.546 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:41.546 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:19:41.546 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:17:39.919098+0000", 2023-09-23T21:19:41.546 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.0101625329999999 2023-09-23T21:19:41.547 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:41.547 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:41.547 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:19:41.547 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:17:39.919190+0000", 2023-09-23T21:19:41.547 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.1686000000000003e-05 2023-09-23T21:19:41.547 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:19:41.547 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:19:41.547 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:19:41.547 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:41.547 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:41.547 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.46823.0:132654 11.13 11:cdfb262e:::benchmark_data_smithi026_424533_object34684:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e585)", 2023-09-23T21:19:41.547 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:17:38.907670+0000", 2023-09-23T21:19:41.547 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 122.537850229, 2023-09-23T21:19:41.548 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.049192116, 2023-09-23T21:19:41.548 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:19:41.548 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "reached pg", 2023-09-23T21:19:41.548 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:19:41.548 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.46823", 2023-09-23T21:19:41.548 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/2490075711", 2023-09-23T21:19:41.548 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 132654 2023-09-23T21:19:41.548 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:41.548 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:19:41.548 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:41.548 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:19:41.548 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:17:38.907670+0000", 2023-09-23T21:19:41.549 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:19:41.549 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:41.549 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:41.549 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:19:41.549 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:17:38.907670+0000", 2023-09-23T21:19:41.549 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:19:41.549 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:41.549 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:41.549 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:19:41.549 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:17:38.907669+0000", 2023-09-23T21:19:41.549 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999995 2023-09-23T21:19:41.549 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:41.550 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:41.550 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:19:41.550 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:17:38.907672+0000", 2023-09-23T21:19:41.550 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.7389999999999999e-06 2023-09-23T21:19:41.550 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:41.550 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:41.550 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:19:41.550 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:17:38.907678+0000", 2023-09-23T21:19:41.550 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.3010000000000003e-06 2023-09-23T21:19:41.550 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:41.550 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:41.550 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:19:41.550 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:17:38.907683+0000", 2023-09-23T21:19:41.551 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.5990000000000004e-06 2023-09-23T21:19:41.551 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:41.551 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:41.551 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:19:41.551 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:17:38.908961+0000", 2023-09-23T21:19:41.551 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00127783 2023-09-23T21:19:41.551 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:41.551 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:41.551 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:19:41.551 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:17:38.908968+0000", 2023-09-23T21:19:41.551 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.7440000000000001e-06 2023-09-23T21:19:41.551 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:41.552 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:41.552 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:19:41.552 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:17:39.919215+0000", 2023-09-23T21:19:41.552 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.0102470349999999 2023-09-23T21:19:41.552 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:41.552 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:41.552 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for missing object", 2023-09-23T21:19:41.552 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:17:39.919299+0000", 2023-09-23T21:19:41.552 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.4629000000000006e-05 2023-09-23T21:19:41.552 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:41.552 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:41.552 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:19:41.553 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:17:39.956770+0000", 2023-09-23T21:19:41.553 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.037470891999999999 2023-09-23T21:19:41.553 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:41.553 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:41.553 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:19:41.553 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:17:39.956862+0000", 2023-09-23T21:19:41.553 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.1791000000000003e-05 2023-09-23T21:19:41.553 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:19:41.553 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:19:41.553 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:19:41.553 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:41.553 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:41.553 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.46823.0:132697 11.13 11:cff8e3f6:::benchmark_data_smithi026_424533_object34727:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e585)", 2023-09-23T21:19:41.553 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:17:38.957240+0000", 2023-09-23T21:19:41.554 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 122.48828041100001, 2023-09-23T21:19:41.554 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.99985294400000002, 2023-09-23T21:19:41.554 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:19:41.554 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:19:41.554 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:19:41.554 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.46823", 2023-09-23T21:19:41.554 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/2490075711", 2023-09-23T21:19:41.554 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 132697 2023-09-23T21:19:41.554 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:41.554 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:19:41.554 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:41.554 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:19:41.555 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:17:38.957240+0000", 2023-09-23T21:19:41.555 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:19:41.555 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:41.555 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:41.555 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:19:41.555 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:17:38.957240+0000", 2023-09-23T21:19:41.555 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:19:41.555 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:41.555 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:41.555 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:19:41.555 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:17:38.957238+0000", 2023-09-23T21:19:41.555 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T21:19:41.556 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:41.556 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:41.556 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:19:41.556 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:17:38.957246+0000", 2023-09-23T21:19:41.556 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.3819999999999998e-06 2023-09-23T21:19:41.556 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:41.556 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:41.556 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:19:41.556 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:17:38.957261+0000", 2023-09-23T21:19:41.556 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.5299999999999999e-05 2023-09-23T21:19:41.556 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:41.557 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:41.557 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:19:41.557 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:17:38.957268+0000", 2023-09-23T21:19:41.557 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.0350000000000004e-06 2023-09-23T21:19:41.557 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:41.557 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:41.557 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:19:41.557 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:17:38.957336+0000", 2023-09-23T21:19:41.557 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.7711000000000003e-05 2023-09-23T21:19:41.557 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:41.557 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:41.557 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:19:41.558 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:17:38.957346+0000", 2023-09-23T21:19:41.558 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.0536e-05 2023-09-23T21:19:41.558 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:41.558 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:41.558 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:19:41.558 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:17:39.919336+0000", 2023-09-23T21:19:41.558 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.96198997799999997 2023-09-23T21:19:41.558 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:41.558 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:41.558 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:19:41.558 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:17:39.919447+0000", 2023-09-23T21:19:41.558 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000111399 2023-09-23T21:19:41.558 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:41.559 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:41.559 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 2,5", 2023-09-23T21:19:41.559 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:17:39.919789+0000", 2023-09-23T21:19:41.559 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00034173100000000001 2023-09-23T21:19:41.559 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:41.559 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:41.559 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:19:41.559 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:17:39.930825+0000", 2023-09-23T21:19:41.559 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.011035771 2023-09-23T21:19:41.559 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:41.559 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:41.559 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:19:41.560 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:17:39.956895+0000", 2023-09-23T21:19:41.560 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.026069858000000001 2023-09-23T21:19:41.560 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:41.560 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:41.560 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:19:41.560 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:17:39.956998+0000", 2023-09-23T21:19:41.560 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000102779 2023-09-23T21:19:41.560 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:41.560 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:41.560 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:19:41.560 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:17:39.957033+0000", 2023-09-23T21:19:41.560 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.5037999999999999e-05 2023-09-23T21:19:41.561 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:41.561 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:41.561 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:19:41.561 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:17:39.957093+0000", 2023-09-23T21:19:41.561 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.9908999999999998e-05 2023-09-23T21:19:41.561 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:19:41.561 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:19:41.561 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:19:41.561 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:41.561 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:41.561 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.46823.0:132706 11.13 11:ccd1f15a:::benchmark_data_smithi026_424533_object34736:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e585)", 2023-09-23T21:19:41.561 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:17:38.962140+0000", 2023-09-23T21:19:41.561 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 122.483380087, 2023-09-23T21:19:41.562 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.99511751500000001, 2023-09-23T21:19:41.562 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:19:41.562 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:19:41.562 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:19:41.562 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.46823", 2023-09-23T21:19:41.562 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/2490075711", 2023-09-23T21:19:41.562 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 132706 2023-09-23T21:19:41.562 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:41.562 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:19:41.562 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:41.562 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:19:41.562 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:17:38.962140+0000", 2023-09-23T21:19:41.563 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:19:41.563 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:41.563 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:41.563 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:19:41.563 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:17:38.962140+0000", 2023-09-23T21:19:41.563 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:19:41.563 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:41.563 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:41.563 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:19:41.563 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:17:38.962139+0000", 2023-09-23T21:19:41.563 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.999999 2023-09-23T21:19:41.563 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:41.564 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:41.564 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:19:41.564 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:17:38.962144+0000", 2023-09-23T21:19:41.564 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.0200000000000002e-06 2023-09-23T21:19:41.564 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:41.564 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:41.564 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:19:41.564 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:17:38.962154+0000", 2023-09-23T21:19:41.564 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.0553e-05 2023-09-23T21:19:41.564 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:41.564 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:41.564 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:19:41.564 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:17:38.962163+0000", 2023-09-23T21:19:41.565 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.5180000000000004e-06 2023-09-23T21:19:41.565 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:41.565 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:41.565 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:19:41.565 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:17:38.962206+0000", 2023-09-23T21:19:41.565 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.3259e-05 2023-09-23T21:19:41.565 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:41.565 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:41.565 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:19:41.565 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:17:38.962213+0000", 2023-09-23T21:19:41.565 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.9600000000000003e-06 2023-09-23T21:19:41.565 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:41.566 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:41.566 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:19:41.566 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:17:39.921342+0000", 2023-09-23T21:19:41.566 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.959129024 2023-09-23T21:19:41.566 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:41.566 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:41.566 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:19:41.566 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:17:39.921455+0000", 2023-09-23T21:19:41.566 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000112425 2023-09-23T21:19:41.566 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:41.566 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:41.566 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 2,5", 2023-09-23T21:19:41.567 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:17:39.921670+0000", 2023-09-23T21:19:41.567 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00021546799999999999 2023-09-23T21:19:41.567 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:41.567 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:41.567 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:19:41.567 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:17:39.930835+0000", 2023-09-23T21:19:41.567 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0091648760000000006 2023-09-23T21:19:41.567 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:41.567 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:41.567 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:19:41.567 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:17:39.956923+0000", 2023-09-23T21:19:41.567 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.026087909999999999 2023-09-23T21:19:41.567 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:41.568 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:41.568 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:19:41.568 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:17:39.957133+0000", 2023-09-23T21:19:41.568 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000209915 2023-09-23T21:19:41.568 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:41.568 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:41.568 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:19:41.568 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:17:39.957164+0000", 2023-09-23T21:19:41.568 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.1217000000000003e-05 2023-09-23T21:19:41.568 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:41.568 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:41.568 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:19:41.569 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:17:39.957257+0000", 2023-09-23T21:19:41.569 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.3411000000000004e-05 2023-09-23T21:19:41.569 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:19:41.569 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:19:41.569 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:19:41.569 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:41.569 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:41.569 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.46823.0:167303 11.14 11:2842df3b:::benchmark_data_smithi026_424533_object69333:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e600)", 2023-09-23T21:19:41.569 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:18:16.958822+0000", 2023-09-23T21:19:41.569 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 84.486698261000001, 2023-09-23T21:19:41.569 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.97134605200000002, 2023-09-23T21:19:41.569 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:19:41.569 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:19:41.570 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:19:41.570 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.46823", 2023-09-23T21:19:41.570 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/2490075711", 2023-09-23T21:19:41.570 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 167303 2023-09-23T21:19:41.570 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:41.570 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:19:41.570 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:41.570 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:19:41.570 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:18:16.958822+0000", 2023-09-23T21:19:41.570 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:19:41.570 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:41.570 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:41.571 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:19:41.571 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:18:16.958822+0000", 2023-09-23T21:19:41.571 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:19:41.571 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:41.571 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:41.571 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:19:41.571 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:18:16.958821+0000", 2023-09-23T21:19:41.571 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.999999 2023-09-23T21:19:41.571 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:41.571 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:41.571 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:19:41.571 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:18:16.958825+0000", 2023-09-23T21:19:41.572 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.5600000000000004e-06 2023-09-23T21:19:41.572 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:41.572 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:41.572 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:19:41.572 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:18:16.958836+0000", 2023-09-23T21:19:41.572 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.0961000000000001e-05 2023-09-23T21:19:41.572 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:41.572 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:41.572 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:19:41.572 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:18:16.958842+0000", 2023-09-23T21:19:41.572 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.5990000000000001e-06 2023-09-23T21:19:41.572 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:41.573 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:41.573 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:19:41.573 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:18:16.958907+0000", 2023-09-23T21:19:41.573 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.4577000000000002e-05 2023-09-23T21:19:41.573 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:41.573 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:41.573 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:19:41.573 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:18:16.958915+0000", 2023-09-23T21:19:41.573 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.3960000000000006e-06 2023-09-23T21:19:41.573 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:41.573 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:41.573 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:19:41.574 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:18:17.907381+0000", 2023-09-23T21:19:41.574 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.94846557899999995 2023-09-23T21:19:41.574 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:41.574 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:41.574 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:19:41.574 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:18:17.907467+0000", 2023-09-23T21:19:41.574 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.6240000000000001e-05 2023-09-23T21:19:41.574 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:41.574 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:41.574 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 2,3", 2023-09-23T21:19:41.574 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:18:17.907772+0000", 2023-09-23T21:19:41.574 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00030535700000000002 2023-09-23T21:19:41.575 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:41.575 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:41.575 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:19:41.575 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:18:17.916538+0000", 2023-09-23T21:19:41.575 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0087654789999999996 2023-09-23T21:19:41.575 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:41.575 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:41.575 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:19:41.575 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:18:17.930069+0000", 2023-09-23T21:19:41.575 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.013531395999999999 2023-09-23T21:19:41.575 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:41.576 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:41.576 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:19:41.576 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:18:17.930106+0000", 2023-09-23T21:19:41.576 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.6674999999999997e-05 2023-09-23T21:19:41.576 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:41.576 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:41.576 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:19:41.576 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:18:17.930144+0000", 2023-09-23T21:19:41.576 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.8398000000000001e-05 2023-09-23T21:19:41.576 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:41.576 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:41.576 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:19:41.576 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:18:17.930168+0000", 2023-09-23T21:19:41.577 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.3762999999999999e-05 2023-09-23T21:19:41.577 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:19:41.577 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:19:41.577 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:19:41.577 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:41.577 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:41.577 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.46823.0:167309 11.14 11:2c3953b6:::benchmark_data_smithi026_424533_object69339:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e600)", 2023-09-23T21:19:41.577 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:18:16.964680+0000", 2023-09-23T21:19:41.577 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 84.480839634000006, 2023-09-23T21:19:41.577 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.96564490199999997, 2023-09-23T21:19:41.577 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:19:41.577 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:19:41.577 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:19:41.578 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.46823", 2023-09-23T21:19:41.578 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/2490075711", 2023-09-23T21:19:41.578 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 167309 2023-09-23T21:19:41.578 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:41.578 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:19:41.578 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:41.578 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:19:41.578 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:18:16.964680+0000", 2023-09-23T21:19:41.578 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:19:41.578 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:41.578 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:41.578 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:19:41.579 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:18:16.964680+0000", 2023-09-23T21:19:41.579 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:19:41.579 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:41.579 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:41.579 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:19:41.579 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:18:16.964679+0000", 2023-09-23T21:19:41.579 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T21:19:41.579 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:41.579 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:41.579 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:19:41.579 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:18:16.964686+0000", 2023-09-23T21:19:41.579 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.4850000000000003e-06 2023-09-23T21:19:41.580 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:41.580 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:41.580 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:19:41.580 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:18:16.964701+0000", 2023-09-23T21:19:41.580 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.437e-05 2023-09-23T21:19:41.580 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:41.580 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:41.580 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:19:41.580 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:18:16.964707+0000", 2023-09-23T21:19:41.580 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.0990000000000004e-06 2023-09-23T21:19:41.580 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:41.580 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:41.580 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:19:41.581 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:18:16.964765+0000", 2023-09-23T21:19:41.581 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.7673000000000002e-05 2023-09-23T21:19:41.581 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:41.581 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:41.581 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:19:41.581 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:18:16.964776+0000", 2023-09-23T21:19:41.581 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.1008e-05 2023-09-23T21:19:41.581 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:41.581 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:41.581 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:19:41.581 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:18:17.911560+0000", 2023-09-23T21:19:41.581 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.946784352 2023-09-23T21:19:41.582 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:41.582 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:41.582 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:19:41.582 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:18:17.911633+0000", 2023-09-23T21:19:41.582 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.3133999999999995e-05 2023-09-23T21:19:41.582 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:41.582 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:41.582 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 2,3", 2023-09-23T21:19:41.582 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:18:17.911820+0000", 2023-09-23T21:19:41.582 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00018671900000000001 2023-09-23T21:19:41.582 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:41.582 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:41.582 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:19:41.583 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:18:17.916545+0000", 2023-09-23T21:19:41.583 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0047253800000000004 2023-09-23T21:19:41.583 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:41.583 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:41.583 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:19:41.583 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:18:17.930202+0000", 2023-09-23T21:19:41.583 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.013656817999999999 2023-09-23T21:19:41.583 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:41.583 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:41.583 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:19:41.583 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:18:17.930261+0000", 2023-09-23T21:19:41.583 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.9049000000000003e-05 2023-09-23T21:19:41.584 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:41.584 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:41.584 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:19:41.584 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:18:17.930293+0000", 2023-09-23T21:19:41.584 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.1868e-05 2023-09-23T21:19:41.584 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:41.584 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:41.584 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:19:41.584 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:18:17.930325+0000", 2023-09-23T21:19:41.584 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.2360000000000002e-05 2023-09-23T21:19:41.584 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:19:41.584 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:19:41.584 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:19:41.585 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:41.585 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:41.585 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.46823.0:167316 11.14 11:2f935898:::benchmark_data_smithi026_424533_object69346:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e600)", 2023-09-23T21:19:41.585 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T21:18:16.974624+0000", 2023-09-23T21:19:41.585 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 84.470896510000003, 2023-09-23T21:19:41.585 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.95591005200000001, 2023-09-23T21:19:41.585 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T21:19:41.585 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:19:41.585 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T21:19:41.585 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.46823", 2023-09-23T21:19:41.585 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/2490075711", 2023-09-23T21:19:41.585 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 167316 2023-09-23T21:19:41.586 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:41.586 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T21:19:41.586 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:41.586 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T21:19:41.586 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:18:16.974624+0000", 2023-09-23T21:19:41.586 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:19:41.586 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:41.586 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:41.586 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T21:19:41.586 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:18:16.974624+0000", 2023-09-23T21:19:41.586 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T21:19:41.586 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:41.587 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:41.587 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T21:19:41.587 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:18:16.974622+0000", 2023-09-23T21:19:41.587 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.999999 2023-09-23T21:19:41.587 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:41.587 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:41.587 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T21:19:41.587 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:18:16.974628+0000", 2023-09-23T21:19:41.587 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.2610000000000001e-06 2023-09-23T21:19:41.587 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:41.587 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:41.587 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T21:19:41.587 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:18:16.974638+0000", 2023-09-23T21:19:41.588 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.0266e-05 2023-09-23T21:19:41.588 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:41.588 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:41.588 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T21:19:41.588 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:18:16.974643+0000", 2023-09-23T21:19:41.588 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.7350000000000004e-06 2023-09-23T21:19:41.588 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:41.588 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:41.588 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:19:41.588 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:18:16.974684+0000", 2023-09-23T21:19:41.588 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.1006e-05 2023-09-23T21:19:41.588 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:41.589 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:41.589 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T21:19:41.589 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:18:16.974691+0000", 2023-09-23T21:19:41.589 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.2309999999999999e-06 2023-09-23T21:19:41.589 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:41.589 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:41.589 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T21:19:41.589 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:18:17.912933+0000", 2023-09-23T21:19:41.589 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.93824234200000001 2023-09-23T21:19:41.589 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:41.589 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:41.589 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T21:19:41.589 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:18:17.913016+0000", 2023-09-23T21:19:41.590 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.2603999999999997e-05 2023-09-23T21:19:41.590 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:41.590 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:41.590 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 2,3", 2023-09-23T21:19:41.590 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:18:17.913198+0000", 2023-09-23T21:19:41.590 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00018191099999999999 2023-09-23T21:19:41.590 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:41.590 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:41.590 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T21:19:41.590 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:18:17.918968+0000", 2023-09-23T21:19:41.590 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0057699719999999999 2023-09-23T21:19:41.590 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:41.591 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:41.591 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:19:41.591 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:18:17.930228+0000", 2023-09-23T21:19:41.591 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.011259781 2023-09-23T21:19:41.591 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:41.591 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:41.591 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:19:41.591 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:18:17.930451+0000", 2023-09-23T21:19:41.591 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00022378599999999999 2023-09-23T21:19:41.591 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:41.591 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:41.591 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T21:19:41.592 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:18:17.930505+0000", 2023-09-23T21:19:41.592 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.4032e-05 2023-09-23T21:19:41.592 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T21:19:41.592 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T21:19:41.592 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T21:19:41.592 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T21:18:17.930534+0000", 2023-09-23T21:19:41.592 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.8198e-05 2023-09-23T21:19:41.592 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:19:41.592 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:19:41.592 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:19:41.592 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T21:19:41.592 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:19:41.593 INFO:teuthology.orchestra.run.smithi070.stdout:} 2023-09-23T21:19:41.593 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:19:41.665 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:41.750 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:19:41.753 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:41.753 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:19:41.800 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:19:41 smithi026 ceph-mon[175151]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T21:19:41.800 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:19:41 smithi026 ceph-mon[175151]: Cluster is now healthy 2023-09-23T21:19:41.800 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:19:41 smithi026 ceph-mon[175151]: failed to encode map e656 with expected crc 2023-09-23T21:19:41.800 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:19:41 smithi026 ceph-mon[175151]: failed to encode map e656 with expected crc 2023-09-23T21:19:41.800 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:19:41 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T21:19:41.800 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:19:41 smithi026 ceph-mon[175151]: osdmap e656: 8 total, 8 up, 5 in 2023-09-23T21:19:41.801 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:19:41 smithi026 ceph-mon[175151]: failed to encode map e656 with expected crc 2023-09-23T21:19:41.801 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:19:41 smithi026 ceph-mon[175151]: failed to encode map e657 with expected crc 2023-09-23T21:19:41.801 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:19:41 smithi026 ceph-mon[175151]: failed to encode map e657 with expected crc 2023-09-23T21:19:41.801 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:19:41 smithi026 ceph-mon[175151]: osdmap e657: 8 total, 8 up, 5 in 2023-09-23T21:19:41.801 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:19:41 smithi026 ceph-mon[175151]: failed to encode map e656 with expected crc 2023-09-23T21:19:41.801 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:19:41 smithi026 ceph-mon[175151]: failed to encode map e657 with expected crc 2023-09-23T21:19:41.801 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:19:41 smithi026 ceph-mon[179688]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T21:19:41.801 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:19:41 smithi026 ceph-mon[179688]: Cluster is now healthy 2023-09-23T21:19:41.801 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:19:41 smithi026 ceph-mon[179688]: failed to encode map e656 with expected crc 2023-09-23T21:19:41.801 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:19:41 smithi026 ceph-mon[179688]: failed to encode map e656 with expected crc 2023-09-23T21:19:41.802 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:19:41 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T21:19:41.802 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:19:41 smithi026 ceph-mon[179688]: osdmap e656: 8 total, 8 up, 5 in 2023-09-23T21:19:41.802 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:19:41 smithi026 ceph-mon[179688]: failed to encode map e656 with expected crc 2023-09-23T21:19:41.802 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:19:41 smithi026 ceph-mon[179688]: failed to encode map e657 with expected crc 2023-09-23T21:19:41.802 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:19:41 smithi026 ceph-mon[179688]: failed to encode map e657 with expected crc 2023-09-23T21:19:41.802 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:19:41 smithi026 ceph-mon[179688]: osdmap e657: 8 total, 8 up, 5 in 2023-09-23T21:19:41.802 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:19:41 smithi026 ceph-mon[179688]: failed to encode map e656 with expected crc 2023-09-23T21:19:41.802 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:19:41 smithi026 ceph-mon[179688]: failed to encode map e657 with expected crc 2023-09-23T21:19:41.813 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:19:41 smithi070 ceph-mon[168282]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T21:19:41.813 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:19:41 smithi070 ceph-mon[168282]: Cluster is now healthy 2023-09-23T21:19:41.813 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:19:41 smithi070 ceph-mon[168282]: failed to encode map e656 with expected crc 2023-09-23T21:19:41.813 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:19:41 smithi070 ceph-mon[168282]: failed to encode map e656 with expected crc 2023-09-23T21:19:41.813 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:19:41 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T21:19:41.813 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:19:41 smithi070 ceph-mon[168282]: osdmap e656: 8 total, 8 up, 5 in 2023-09-23T21:19:41.813 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:19:41 smithi070 ceph-mon[168282]: failed to encode map e656 with expected crc 2023-09-23T21:19:41.813 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:19:41 smithi070 ceph-mon[168282]: failed to encode map e657 with expected crc 2023-09-23T21:19:41.814 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:19:41 smithi070 ceph-mon[168282]: failed to encode map e657 with expected crc 2023-09-23T21:19:41.814 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:19:41 smithi070 ceph-mon[168282]: osdmap e657: 8 total, 8 up, 5 in 2023-09-23T21:19:41.814 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:19:41 smithi070 ceph-mon[168282]: failed to encode map e656 with expected crc 2023-09-23T21:19:41.814 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:19:41 smithi070 ceph-mon[168282]: failed to encode map e657 with expected crc 2023-09-23T21:19:41.815 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_ops_in_flight 2023-09-23T21:19:41.854 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:19:41.883 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:41.908 INFO:tasks.workunit.client.0.smithi026.stderr: Removing snap: 100% complete...done. 2023-09-23T21:19:41.926 INFO:tasks.workunit.client.0.smithi026.stderr:+ rbd remove testimg_import 2023-09-23T21:19:41.930 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:19:41.933 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:41.934 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:19:42.008 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0: {} 2023-09-23T21:19:42.017 INFO:tasks.workunit.client.0.smithi026.stderr: Removing image: 100% complete...done. 2023-09-23T21:19:42.020 INFO:tasks.workunit.client.0.smithi026.stderr:+ rbd snap rm testimg@snap 2023-09-23T21:19:42.032 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1: {} 2023-09-23T21:19:42.035 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:19:42.057 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2: {} 2023-09-23T21:19:42.072 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:42.086 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3: {} 2023-09-23T21:19:42.108 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4: {} 2023-09-23T21:19:42.127 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T21:19:42.147 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6: {} 2023-09-23T21:19:42.166 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T21:19:42.167 INFO:teuthology.orchestra.run.smithi026.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T21:19:42.168 INFO:teuthology.orchestra.run.smithi026.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T21:19:42.168 INFO:teuthology.orchestra.run.smithi026.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T21:19:42.168 INFO:teuthology.orchestra.run.smithi026.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T21:19:42.168 INFO:teuthology.orchestra.run.smithi026.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T21:19:42.168 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T21:19:42.168 INFO:teuthology.orchestra.run.smithi026.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T21:19:42.168 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T21:19:42.210 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:19:42.213 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:42.214 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:19:42.315 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:19:42.343 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:42.375 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:19:42.377 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:42.378 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:19:42.429 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T21:19:42.471 INFO:teuthology.orchestra.run.smithi026.stderr:Error EINVAL: specified pg_num 51 < pg_num_max 32 2023-09-23T21:19:42.478 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:19:42.507 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:42.869 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:19:42.872 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:19:42 smithi026 ceph-mon[175151]: pgmap v1613: 103 pgs: 103 active+clean; 1.3 MiB data, 3.7 GiB used, 443 GiB / 447 GiB avail; 411 KiB/s rd, 173 KiB/s wr, 156 op/s; 0 B/s, 0 objects/s recovering 2023-09-23T21:19:42.872 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:19:42 smithi026 ceph-mon[175151]: osdmap e658: 8 total, 8 up, 5 in 2023-09-23T21:19:42.872 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:19:42 smithi026 ceph-mon[175151]: failed to encode map e658 with expected crc 2023-09-23T21:19:42.872 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:19:42 smithi026 ceph-mon[175151]: failed to encode map e658 with expected crc 2023-09-23T21:19:42.873 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:19:42 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/251156299' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:19:42.873 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:19:42 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:19:42.873 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:19:42 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/4123745278' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "51"}]: dispatch 2023-09-23T21:19:42.873 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:19:42 smithi026 ceph-mon[179688]: pgmap v1613: 103 pgs: 103 active+clean; 1.3 MiB data, 3.7 GiB used, 443 GiB / 447 GiB avail; 411 KiB/s rd, 173 KiB/s wr, 156 op/s; 0 B/s, 0 objects/s recovering 2023-09-23T21:19:42.873 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:19:42 smithi026 ceph-mon[179688]: osdmap e658: 8 total, 8 up, 5 in 2023-09-23T21:19:42.873 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:19:42 smithi026 ceph-mon[179688]: failed to encode map e658 with expected crc 2023-09-23T21:19:42.873 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:19:42 smithi026 ceph-mon[179688]: failed to encode map e658 with expected crc 2023-09-23T21:19:42.873 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:19:42 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/251156299' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:19:42.873 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:19:42 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:19:42.874 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:19:42 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/4123745278' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "51"}]: dispatch 2023-09-23T21:19:42.874 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:42.874 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:19:42.880 DEBUG:teuthology.orchestra.run:got remote process result: 22 2023-09-23T21:19:42.908 INFO:tasks.workunit.client.0.smithi026.stderr: Removing snap: 100% complete...done. 2023-09-23T21:19:42.918 INFO:tasks.workunit.client.0.smithi026.stderr:+ rbd rm testimg 2023-09-23T21:19:42.967 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:19:42 smithi070 ceph-mon[168282]: pgmap v1613: 103 pgs: 103 active+clean; 1.3 MiB data, 3.7 GiB used, 443 GiB / 447 GiB avail; 411 KiB/s rd, 173 KiB/s wr, 156 op/s; 0 B/s, 0 objects/s recovering 2023-09-23T21:19:42.967 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:19:42 smithi070 ceph-mon[168282]: osdmap e658: 8 total, 8 up, 5 in 2023-09-23T21:19:42.968 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:19:42 smithi070 ceph-mon[168282]: failed to encode map e658 with expected crc 2023-09-23T21:19:42.968 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:19:42 smithi070 ceph-mon[168282]: failed to encode map e658 with expected crc 2023-09-23T21:19:42.968 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:19:42 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/251156299' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:19:42.968 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:19:42 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:19:42.968 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:19:42 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/4123745278' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "51"}]: dispatch 2023-09-23T21:19:42.975 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:19:42.999 INFO:tasks.workunit.client.0.smithi026.stderr: Removing image: 100% complete...done. 2023-09-23T21:19:43.003 INFO:tasks.workunit.client.0.smithi026.stderr:+ rbd import --order 20 /tmp/rbd_import_export_456745/img testimg 2023-09-23T21:19:43.004 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:43.055 INFO:tasks.workunit.client.0.smithi026.stderr: Importing image: 100% complete...done. 2023-09-23T21:19:43.056 INFO:tasks.workunit.client.0.smithi026.stderr:rbd: --order is deprecated, use --object-size 2023-09-23T21:19:43.058 INFO:tasks.workunit.client.0.smithi026.stderr:+ rbd export --export-format 2 testimg /tmp/rbd_import_export_456745/img_v2 2023-09-23T21:19:43.095 INFO:tasks.workunit.client.0.smithi026.stderr: Exporting image: 100% complete... Exporting image: 100% complete...done. 2023-09-23T21:19:43.098 INFO:tasks.workunit.client.0.smithi026.stderr:+ rbd import --export-format 2 /tmp/rbd_import_export_456745/img_v2 testimg_import 2023-09-23T21:19:43.154 INFO:tasks.workunit.client.0.smithi026.stderr: Importing image: 100% complete... Importing image: 100% complete...done. 2023-09-23T21:19:43.158 INFO:tasks.workunit.client.0.smithi026.stderr:+ rbd info testimg_import 2023-09-23T21:19:43.158 INFO:tasks.workunit.client.0.smithi026.stderr:+ grep order 2023-09-23T21:19:43.158 INFO:tasks.workunit.client.0.smithi026.stderr:+ awk '{print $2}' 2023-09-23T21:19:43.158 INFO:tasks.workunit.client.0.smithi026.stderr:+ grep 20 2023-09-23T21:19:43.195 INFO:tasks.workunit.client.0.smithi026.stdout:20 2023-09-23T21:19:43.195 INFO:tasks.workunit.client.0.smithi026.stderr:+ rm /tmp/rbd_import_export_456745/img_v2 2023-09-23T21:19:43.195 INFO:tasks.workunit.client.0.smithi026.stderr:+ rbd remove testimg_import 2023-09-23T21:19:43.203 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:19:43.206 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:43.207 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:19:43.280 INFO:tasks.workunit.client.0.smithi026.stderr: Removing image: 100% complete...done. 2023-09-23T21:19:43.283 INFO:tasks.workunit.client.0.smithi026.stderr:+ rbd remove testimg 2023-09-23T21:19:43.307 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:19:43.338 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:43.375 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T21:19:43.376 INFO:tasks.workunit.client.0.smithi026.stderr: Removing image: 100% complete...done. 2023-09-23T21:19:43.379 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:43.379 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:19:43.380 INFO:tasks.workunit.client.0.smithi026.stderr:+ rbd import --image-feature layering /tmp/rbd_import_export_456745/img testimg 2023-09-23T21:19:43.427 INFO:tasks.workunit.client.0.smithi026.stderr: Importing image: 100% complete...done. 2023-09-23T21:19:43.430 INFO:tasks.workunit.client.0.smithi026.stderr:++ rbd info testimg 2023-09-23T21:19:43.430 INFO:tasks.workunit.client.0.smithi026.stderr:++ grep features 2023-09-23T21:19:43.467 INFO:tasks.workunit.client.0.smithi026.stderr:+ FEATURES_BEFORE=' features: layering 2023-09-23T21:19:43.467 INFO:tasks.workunit.client.0.smithi026.stderr: op_features: ' 2023-09-23T21:19:43.467 INFO:tasks.workunit.client.0.smithi026.stderr:+ rbd export --export-format 2 testimg /tmp/rbd_import_export_456745/img_v2 2023-09-23T21:19:43.480 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:19:43.503 INFO:tasks.workunit.client.0.smithi026.stderr: Exporting image: 100% complete... Exporting image: 100% complete...done. 2023-09-23T21:19:43.507 INFO:tasks.workunit.client.0.smithi026.stderr:+ rbd import --export-format 2 /tmp/rbd_import_export_456745/img_v2 testimg_import 2023-09-23T21:19:43.507 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:43.556 INFO:tasks.workunit.client.0.smithi026.stderr: Importing image: 100% complete... Importing image: 100% complete...done. 2023-09-23T21:19:43.560 INFO:tasks.workunit.client.0.smithi026.stderr:++ rbd info testimg_import 2023-09-23T21:19:43.560 INFO:tasks.workunit.client.0.smithi026.stderr:++ grep features 2023-09-23T21:19:43.592 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:19:43.595 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T21:19:43.596 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:43.596 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:19:43.597 INFO:tasks.workunit.client.0.smithi026.stderr:+ FEATURES_AFTER=' features: layering 2023-09-23T21:19:43.598 INFO:tasks.workunit.client.0.smithi026.stderr: op_features: ' 2023-09-23T21:19:43.598 INFO:tasks.workunit.client.0.smithi026.stderr:+ '[' ' features: layering 2023-09-23T21:19:43.598 INFO:tasks.workunit.client.0.smithi026.stderr: op_features: ' '!=' ' features: layering 2023-09-23T21:19:43.598 INFO:tasks.workunit.client.0.smithi026.stderr: op_features: ' ']' 2023-09-23T21:19:43.598 INFO:tasks.workunit.client.0.smithi026.stderr:+ rm /tmp/rbd_import_export_456745/img_v2 2023-09-23T21:19:43.598 INFO:tasks.workunit.client.0.smithi026.stderr:+ rbd remove testimg_import 2023-09-23T21:19:43.666 INFO:tasks.workunit.client.0.smithi026.stderr: Removing image: 100% complete...done. 2023-09-23T21:19:43.669 INFO:tasks.workunit.client.0.smithi026.stderr:+ rbd remove testimg 2023-09-23T21:19:43.697 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:19:43.727 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:43.745 INFO:tasks.workunit.client.0.smithi026.stderr: Removing image: 100% complete...done. 2023-09-23T21:19:43.747 INFO:tasks.workunit.client.0.smithi026.stderr:+ rbd import --stripe-count 1000 --stripe-unit 4096 /tmp/rbd_import_export_456745/img testimg 2023-09-23T21:19:43.762 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:19:43.764 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:43.765 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:19:43.794 INFO:teuthology.orchestra.run.smithi070.stdout:ERROR: (22) Invalid argument 2023-09-23T21:19:43.794 INFO:teuthology.orchestra.run.smithi070.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T21:19:43.808 INFO:tasks.workunit.client.0.smithi026.stderr: Importing image: 100% complete...done. 2023-09-23T21:19:43.811 INFO:tasks.workunit.client.0.smithi026.stderr:+ rbd export --export-format 2 testimg /tmp/rbd_import_export_456745/img_v2 2023-09-23T21:19:43.852 INFO:tasks.workunit.client.0.smithi026.stderr: Exporting image: 100% complete... Exporting image: 100% complete...done. 2023-09-23T21:19:43.855 INFO:tasks.workunit.client.0.smithi026.stderr:+ rbd import --export-format 2 /tmp/rbd_import_export_456745/img_v2 testimg_import 2023-09-23T21:19:43.866 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:19:43.894 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:43.920 INFO:tasks.workunit.client.0.smithi026.stderr: Importing image: 100% complete... Importing image: 100% complete...done. 2023-09-23T21:19:43.923 INFO:tasks.workunit.client.0.smithi026.stderr:+ rbd info testimg_import 2023-09-23T21:19:43.924 INFO:tasks.workunit.client.0.smithi026.stderr:+ grep 'stripe unit' 2023-09-23T21:19:43.924 INFO:tasks.workunit.client.0.smithi026.stderr:+ grep -Ei '(4 KiB|4096)' 2023-09-23T21:19:43.961 INFO:tasks.workunit.client.0.smithi026.stdout: stripe unit: 4 KiB 2023-09-23T21:19:43.962 INFO:tasks.workunit.client.0.smithi026.stderr:+ rbd info testimg_import 2023-09-23T21:19:43.962 INFO:tasks.workunit.client.0.smithi026.stderr:+ grep 'stripe count' 2023-09-23T21:19:43.962 INFO:tasks.workunit.client.0.smithi026.stderr:+ awk '{print $3}' 2023-09-23T21:19:43.962 INFO:tasks.workunit.client.0.smithi026.stderr:+ grep 1000 2023-09-23T21:19:44.002 INFO:tasks.workunit.client.0.smithi026.stdout:1000 2023-09-23T21:19:44.002 INFO:tasks.workunit.client.0.smithi026.stderr:+ rm /tmp/rbd_import_export_456745/img_v2 2023-09-23T21:19:44.003 INFO:tasks.workunit.client.0.smithi026.stderr:+ rbd remove testimg_import 2023-09-23T21:19:44.085 INFO:tasks.workunit.client.0.smithi026.stderr: Removing image: 3% complete... Removing image: 7% complete... Removing image: 10% complete... Removing image: 14% complete... Removing image: 17% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 32% complete... Removing image: 35% complete... Removing image: 39% complete... Removing image: 42% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 57% complete... Removing image: 60% complete... Removing image: 64% complete... Removing image: 67% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 82% complete... Removing image: 85% complete... Removing image: 89% complete... Removing image: 92% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2023-09-23T21:19:44.088 INFO:tasks.workunit.client.0.smithi026.stderr:+ rbd remove testimg 2023-09-23T21:19:44.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:19:43 smithi070 ceph-mon[168282]: failed to encode map e659 with expected crc 2023-09-23T21:19:44.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:19:43 smithi070 ceph-mon[168282]: failed to encode map e659 with expected crc 2023-09-23T21:19:44.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:19:43 smithi070 ceph-mon[168282]: failed to encode map e659 with expected crc 2023-09-23T21:19:44.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:19:43 smithi070 ceph-mon[168282]: osdmap e659: 8 total, 8 up, 5 in 2023-09-23T21:19:44.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:19:43 smithi070 ceph-mon[168282]: failed to encode map e659 with expected crc 2023-09-23T21:19:44.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:19:43 smithi070 ceph-mon[168282]: pgmap v1616: 103 pgs: 103 active+clean; 692 KiB data, 3.7 GiB used, 443 GiB / 447 GiB avail; 479 KiB/s rd, 104 KiB/s wr, 327 op/s 2023-09-23T21:19:44.158 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:19:44.161 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:44.161 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:19:44.180 INFO:tasks.workunit.client.0.smithi026.stderr: Removing image: 3% complete... Removing image: 7% complete... Removing image: 10% complete... Removing image: 14% complete... Removing image: 17% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 32% complete... Removing image: 35% complete... Removing image: 39% complete... Removing image: 42% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 57% complete... Removing image: 60% complete... Removing image: 64% complete... Removing image: 67% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 82% complete... Removing image: 85% complete... Removing image: 89% complete... Removing image: 92% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2023-09-23T21:19:44.183 INFO:tasks.workunit.client.0.smithi026.stderr:+ rbd import --image-format=2 /tmp/rbd_import_export_456745/img testimg 2023-09-23T21:19:44.188 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_blocked_ops 2023-09-23T21:19:44.235 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:19:44.236 INFO:tasks.workunit.client.0.smithi026.stderr: Importing image: 100% complete...done. 2023-09-23T21:19:44.239 INFO:tasks.workunit.client.0.smithi026.stderr:+ rbd snap create testimg@snap1 2023-09-23T21:19:44.263 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:19:44.297 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:44.330 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:19:44.333 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:44.333 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:19:44.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:19:43 smithi026 ceph-mon[175151]: failed to encode map e659 with expected crc 2023-09-23T21:19:44.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:19:43 smithi026 ceph-mon[175151]: failed to encode map e659 with expected crc 2023-09-23T21:19:44.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:19:43 smithi026 ceph-mon[175151]: failed to encode map e659 with expected crc 2023-09-23T21:19:44.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:19:43 smithi026 ceph-mon[175151]: osdmap e659: 8 total, 8 up, 5 in 2023-09-23T21:19:44.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:19:43 smithi026 ceph-mon[175151]: failed to encode map e659 with expected crc 2023-09-23T21:19:44.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:19:43 smithi026 ceph-mon[175151]: pgmap v1616: 103 pgs: 103 active+clean; 692 KiB data, 3.7 GiB used, 443 GiB / 447 GiB avail; 479 KiB/s rd, 104 KiB/s wr, 327 op/s 2023-09-23T21:19:44.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:19:43 smithi026 ceph-mon[179688]: failed to encode map e659 with expected crc 2023-09-23T21:19:44.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:19:43 smithi026 ceph-mon[179688]: failed to encode map e659 with expected crc 2023-09-23T21:19:44.383 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:19:43 smithi026 ceph-mon[179688]: failed to encode map e659 with expected crc 2023-09-23T21:19:44.383 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:19:43 smithi026 ceph-mon[179688]: osdmap e659: 8 total, 8 up, 5 in 2023-09-23T21:19:44.383 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:19:43 smithi026 ceph-mon[179688]: failed to encode map e659 with expected crc 2023-09-23T21:19:44.383 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:19:43 smithi026 ceph-mon[179688]: pgmap v1616: 103 pgs: 103 active+clean; 692 KiB data, 3.7 GiB used, 443 GiB / 447 GiB avail; 479 KiB/s rd, 104 KiB/s wr, 327 op/s 2023-09-23T21:19:44.434 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:19:44.461 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:44.493 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:19:44.495 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:44.496 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:19:44.597 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:19:44.608 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T21:19:44.625 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:44.830 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T21:19:44.913 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:19:44.918 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:44.918 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:19:44.921 INFO:tasks.workunit.client.0.smithi026.stderr: Creating snap: 10% complete... Creating snap: 100% complete...done. 2023-09-23T21:19:44.927 INFO:tasks.workunit.client.0.smithi026.stderr:+ rbd snap create testimg@snap2 2023-09-23T21:19:45.019 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:19:45.045 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:45.080 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:19:45.087 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:45.087 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:19:45.188 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:19:45.222 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:45.258 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T21:19:45.262 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:45.262 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:19:45.363 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:19:45.391 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:45.494 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:19:45.646 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:19:45.649 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:45.649 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:19:45.750 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:19:45.778 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:45.813 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:19:45.816 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:45.816 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:19:45.917 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:19:45.920 INFO:teuthology.orchestra.run.smithi026.stderr:noscrub is set 2023-09-23T21:19:45.926 INFO:tasks.workunit.client.0.smithi026.stderr: Creating snap: 10% complete... Creating snap: 100% complete...done. 2023-09-23T21:19:45.934 INFO:tasks.workunit.client.0.smithi026.stderr:+ rbd snap protect testimg@snap2 2023-09-23T21:19:45.950 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:19:45 smithi026 ceph-mon[179688]: failed to encode map e660 with expected crc 2023-09-23T21:19:45.950 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:19:45 smithi026 ceph-mon[179688]: failed to encode map e660 with expected crc 2023-09-23T21:19:45.951 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:19:45 smithi026 ceph-mon[179688]: osdmap e660: 8 total, 8 up, 5 in 2023-09-23T21:19:45.951 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:19:45 smithi026 ceph-mon[179688]: failed to encode map e660 with expected crc 2023-09-23T21:19:45.951 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:19:45 smithi026 ceph-mon[179688]: pgmap v1618: 103 pgs: 103 active+clean; 692 KiB data, 3.7 GiB used, 443 GiB / 447 GiB avail; 152 KiB/s rd, 12 KiB/s wr, 142 op/s 2023-09-23T21:19:45.951 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:19:45 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/2399377669' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T21:19:45.951 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:19:45 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T21:19:45.951 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:19:45 smithi026 ceph-mon[175151]: failed to encode map e660 with expected crc 2023-09-23T21:19:45.952 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:19:45 smithi026 ceph-mon[175151]: failed to encode map e660 with expected crc 2023-09-23T21:19:45.952 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:19:45 smithi026 ceph-mon[175151]: osdmap e660: 8 total, 8 up, 5 in 2023-09-23T21:19:45.952 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:19:45 smithi026 ceph-mon[175151]: failed to encode map e660 with expected crc 2023-09-23T21:19:45.952 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:19:45 smithi026 ceph-mon[175151]: pgmap v1618: 103 pgs: 103 active+clean; 692 KiB data, 3.7 GiB used, 443 GiB / 447 GiB avail; 152 KiB/s rd, 12 KiB/s wr, 142 op/s 2023-09-23T21:19:45.952 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:19:45 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/2399377669' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T21:19:45.952 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:19:45 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T21:19:45.952 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:45.991 INFO:tasks.workunit.client.0.smithi026.stderr:+ rbd export --export-format 2 testimg /tmp/rbd_import_export_456745/snap_protect 2023-09-23T21:19:46.034 INFO:tasks.workunit.client.0.smithi026.stderr: Exporting image: 33% complete... Exporting image: 100% complete... Exporting image: 100% complete...done. 2023-09-23T21:19:46.037 INFO:tasks.workunit.client.0.smithi026.stderr:+ rbd import --export-format 2 /tmp/rbd_import_export_456745/snap_protect testimg_import 2023-09-23T21:19:46.044 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T21:19:46.047 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:46.047 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:19:46.148 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:19:46.176 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:46.188 INFO:teuthology.orchestra.run.smithi070.stdout:ERROR: (22) Invalid argument 2023-09-23T21:19:46.188 INFO:teuthology.orchestra.run.smithi070.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T21:19:46.208 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:19:45 smithi070 ceph-mon[168282]: failed to encode map e660 with expected crc 2023-09-23T21:19:46.208 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:19:45 smithi070 ceph-mon[168282]: failed to encode map e660 with expected crc 2023-09-23T21:19:46.208 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:19:45 smithi070 ceph-mon[168282]: osdmap e660: 8 total, 8 up, 5 in 2023-09-23T21:19:46.208 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:19:45 smithi070 ceph-mon[168282]: failed to encode map e660 with expected crc 2023-09-23T21:19:46.208 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:19:45 smithi070 ceph-mon[168282]: pgmap v1618: 103 pgs: 103 active+clean; 692 KiB data, 3.7 GiB used, 443 GiB / 447 GiB avail; 152 KiB/s rd, 12 KiB/s wr, 142 op/s 2023-09-23T21:19:46.208 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:19:45 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/2399377669' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T21:19:46.209 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:19:45 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T21:19:46.261 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:19:46.264 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:46.264 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:19:46.365 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:19:46.394 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:46.573 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_historic_ops 2023-09-23T21:19:46.633 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:19:46.635 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:46.635 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:19:46.736 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:19:46.764 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:46.850 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:19:46.854 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:46.854 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:19:46.955 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:19:46.981 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:47.013 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:19:47.016 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:47.016 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:19:47.033 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0: {} 2023-09-23T21:19:47.051 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1: {} 2023-09-23T21:19:47.069 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2: {} 2023-09-23T21:19:47.089 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3: {} 2023-09-23T21:19:47.108 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4: {} 2023-09-23T21:19:47.117 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:19:47.126 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T21:19:47.143 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:47.145 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6: {} 2023-09-23T21:19:47.164 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T21:19:47.165 INFO:teuthology.orchestra.run.smithi026.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T21:19:47.166 INFO:teuthology.orchestra.run.smithi026.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T21:19:47.166 INFO:teuthology.orchestra.run.smithi026.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T21:19:47.166 INFO:teuthology.orchestra.run.smithi026.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T21:19:47.166 INFO:teuthology.orchestra.run.smithi026.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T21:19:47.166 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T21:19:47.166 INFO:teuthology.orchestra.run.smithi026.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T21:19:47.166 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T21:19:47.172 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T21:19:47.176 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:19:47.178 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:19:46 smithi070 ceph-mon[168282]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:19:47.178 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:19:46 smithi070 ceph-mon[168282]: failed to encode map e661 with expected crc 2023-09-23T21:19:47.178 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:19:46 smithi070 ceph-mon[168282]: failed to encode map e661 with expected crc 2023-09-23T21:19:47.179 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:19:46 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T21:19:47.179 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:19:46 smithi070 ceph-mon[168282]: osdmap e661: 8 total, 8 up, 5 in 2023-09-23T21:19:47.179 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:19:46 smithi070 ceph-mon[168282]: failed to encode map e661 with expected crc 2023-09-23T21:19:47.179 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:19:46 smithi070 ceph-mon[168282]: failed to encode map e661 with expected crc 2023-09-23T21:19:47.179 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:19:46 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:19:47.179 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:47.180 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:19:47.187 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:19:46 smithi026 ceph-mon[175151]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:19:47.188 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:19:46 smithi026 ceph-mon[175151]: failed to encode map e661 with expected crc 2023-09-23T21:19:47.188 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:19:46 smithi026 ceph-mon[175151]: failed to encode map e661 with expected crc 2023-09-23T21:19:47.188 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:19:46 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T21:19:47.188 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:19:46 smithi026 ceph-mon[175151]: osdmap e661: 8 total, 8 up, 5 in 2023-09-23T21:19:47.188 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:19:46 smithi026 ceph-mon[175151]: failed to encode map e661 with expected crc 2023-09-23T21:19:47.188 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:19:46 smithi026 ceph-mon[175151]: failed to encode map e661 with expected crc 2023-09-23T21:19:47.188 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:19:46 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:19:47.188 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:19:46 smithi026 ceph-mon[179688]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:19:47.188 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:19:46 smithi026 ceph-mon[179688]: failed to encode map e661 with expected crc 2023-09-23T21:19:47.188 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:19:46 smithi026 ceph-mon[179688]: failed to encode map e661 with expected crc 2023-09-23T21:19:47.189 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:19:46 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T21:19:47.189 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:19:46 smithi026 ceph-mon[179688]: osdmap e661: 8 total, 8 up, 5 in 2023-09-23T21:19:47.189 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:19:46 smithi026 ceph-mon[179688]: failed to encode map e661 with expected crc 2023-09-23T21:19:47.189 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:19:46 smithi026 ceph-mon[179688]: failed to encode map e661 with expected crc 2023-09-23T21:19:47.189 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:19:46 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:19:47.280 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:19:47.313 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:47.349 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T21:19:47.351 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:47.352 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:19:47.453 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:19:47.480 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:47.514 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T21:19:47.516 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:47.517 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:19:47.617 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:19:47.645 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:47.678 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:19:47.681 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:47.682 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:19:47.782 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:19:47.810 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:47.881 INFO:tasks.thrashosds.thrasher:in_osds: [0, 2, 3, 4, 7] out_osds: [6, 1, 5] dead_osds: [] live_osds: [2, 3, 4, 5, 7, 6, 1, 0] 2023-09-23T21:19:47.881 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T21:19:47.881 INFO:tasks.thrashosds.thrasher:fixing pg num pool .mgr 2023-09-23T21:19:47.881 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T21:19:47.944 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:19:47.948 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:47.948 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:19:47.949 INFO:tasks.workunit.client.0.smithi026.stderr: Importing image: 33% complete... Importing image: 66% complete... Importing image: 100% complete... Importing image: 100% complete...done. 2023-09-23T21:19:47.952 INFO:tasks.workunit.client.0.smithi026.stderr:+ rbd info testimg_import@snap1 2023-09-23T21:19:47.952 INFO:tasks.workunit.client.0.smithi026.stderr:+ grep 'protected: False' 2023-09-23T21:19:47.992 INFO:tasks.workunit.client.0.smithi026.stdout: protected: False 2023-09-23T21:19:47.992 INFO:tasks.workunit.client.0.smithi026.stderr:+ rbd info testimg_import@snap2 2023-09-23T21:19:47.992 INFO:tasks.workunit.client.0.smithi026.stderr:+ grep 'protected: True' 2023-09-23T21:19:48.031 INFO:tasks.workunit.client.0.smithi026.stdout: protected: True 2023-09-23T21:19:48.031 INFO:tasks.workunit.client.0.smithi026.stderr:+ rm /tmp/rbd_import_export_456745/snap_protect 2023-09-23T21:19:48.032 INFO:tasks.workunit.client.0.smithi026.stderr:+ rbd snap unprotect testimg@snap2 2023-09-23T21:19:48.050 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:19:48.074 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:19:47 smithi070 ceph-mon[168282]: failed to encode map e662 with expected crc 2023-09-23T21:19:48.074 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:19:47 smithi070 ceph-mon[168282]: failed to encode map e662 with expected crc 2023-09-23T21:19:48.074 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:19:47 smithi070 ceph-mon[168282]: failed to encode map e662 with expected crc 2023-09-23T21:19:48.074 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:19:47 smithi070 ceph-mon[168282]: osdmap e662: 8 total, 8 up, 5 in 2023-09-23T21:19:48.074 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:19:47 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/422197540' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:19:48.074 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:19:47 smithi070 ceph-mon[168282]: pgmap v1621: 103 pgs: 103 active+clean; 802 KiB data, 3.7 GiB used, 443 GiB / 447 GiB avail; 361 KiB/s rd, 45 KiB/s wr, 484 op/s 2023-09-23T21:19:48.077 INFO:tasks.workunit.client.0.smithi026.stderr:+ rbd snap unprotect testimg_import@snap2 2023-09-23T21:19:48.077 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:48.126 INFO:tasks.workunit.client.0.smithi026.stderr:+ rbd snap purge testimg 2023-09-23T21:19:48.180 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:19:48.183 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:48.183 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:19:48.284 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:19:48.309 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:19:47 smithi026 ceph-mon[175151]: failed to encode map e662 with expected crc 2023-09-23T21:19:48.309 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:19:47 smithi026 ceph-mon[175151]: failed to encode map e662 with expected crc 2023-09-23T21:19:48.309 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:19:47 smithi026 ceph-mon[175151]: failed to encode map e662 with expected crc 2023-09-23T21:19:48.309 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:19:47 smithi026 ceph-mon[175151]: osdmap e662: 8 total, 8 up, 5 in 2023-09-23T21:19:48.309 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:19:47 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/422197540' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:19:48.309 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:19:47 smithi026 ceph-mon[175151]: pgmap v1621: 103 pgs: 103 active+clean; 802 KiB data, 3.7 GiB used, 443 GiB / 447 GiB avail; 361 KiB/s rd, 45 KiB/s wr, 484 op/s 2023-09-23T21:19:48.309 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:19:47 smithi026 ceph-mon[179688]: failed to encode map e662 with expected crc 2023-09-23T21:19:48.310 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:19:47 smithi026 ceph-mon[179688]: failed to encode map e662 with expected crc 2023-09-23T21:19:48.310 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:19:47 smithi026 ceph-mon[179688]: failed to encode map e662 with expected crc 2023-09-23T21:19:48.310 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:19:47 smithi026 ceph-mon[179688]: osdmap e662: 8 total, 8 up, 5 in 2023-09-23T21:19:48.310 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:19:47 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/422197540' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:19:48.310 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:19:47 smithi026 ceph-mon[179688]: pgmap v1621: 103 pgs: 103 active+clean; 802 KiB data, 3.7 GiB used, 443 GiB / 447 GiB avail; 361 KiB/s rd, 45 KiB/s wr, 484 op/s 2023-09-23T21:19:48.311 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:48.344 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:19:48.346 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:48.347 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:19:48.447 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:19:48.476 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:48.482 INFO:teuthology.orchestra.run.smithi070.stdout:{ 2023-09-23T21:19:48.482 INFO:teuthology.orchestra.run.smithi070.stdout: "size": 20, 2023-09-23T21:19:48.482 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 600, 2023-09-23T21:19:48.483 INFO:teuthology.orchestra.run.smithi070.stdout: "ops": [] 2023-09-23T21:19:48.483 INFO:teuthology.orchestra.run.smithi070.stdout:} 2023-09-23T21:19:48.506 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:19:48.510 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:19:48.512 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:48.513 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:19:48.614 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:19:48.641 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:48.747 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T21:19:48.866 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:19:48.869 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:48.869 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:19:48.915 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_ops_in_flight 2023-09-23T21:19:48.970 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:19:49.000 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:19:48 smithi026 ceph-mon[179688]: failed to encode map e663 with expected crc 2023-09-23T21:19:49.000 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:19:48 smithi026 ceph-mon[179688]: failed to encode map e663 with expected crc 2023-09-23T21:19:49.000 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:19:48 smithi026 ceph-mon[179688]: osdmap e663: 8 total, 8 up, 5 in 2023-09-23T21:19:49.000 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:19:48 smithi026 ceph-mon[179688]: failed to encode map e663 with expected crc 2023-09-23T21:19:49.001 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:19:48 smithi026 ceph-mon[175151]: failed to encode map e663 with expected crc 2023-09-23T21:19:49.001 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:19:48 smithi026 ceph-mon[175151]: failed to encode map e663 with expected crc 2023-09-23T21:19:49.001 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:19:48 smithi026 ceph-mon[175151]: osdmap e663: 8 total, 8 up, 5 in 2023-09-23T21:19:49.001 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:19:48 smithi026 ceph-mon[175151]: failed to encode map e663 with expected crc 2023-09-23T21:19:49.003 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:49.043 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:19:49.045 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:49.046 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:19:49.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:19:48 smithi070 ceph-mon[168282]: failed to encode map e663 with expected crc 2023-09-23T21:19:49.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:19:48 smithi070 ceph-mon[168282]: failed to encode map e663 with expected crc 2023-09-23T21:19:49.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:19:48 smithi070 ceph-mon[168282]: osdmap e663: 8 total, 8 up, 5 in 2023-09-23T21:19:49.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:19:48 smithi070 ceph-mon[168282]: failed to encode map e663 with expected crc 2023-09-23T21:19:49.147 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:19:49.174 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:49.208 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:19:49.210 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:49.211 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:19:49.312 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:19:49.342 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:49.492 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:19:49.494 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:49.495 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:19:49.571 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:19:49.596 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:19:49.611 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T21:19:49.627 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:49.715 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:19:49.718 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:49.719 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:19:49.820 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:19:49.852 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:49.882 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 21:19:49 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[164029]: ::ffff:172.21.15.70 - - [23/Sep/2023:21:19:49] "GET /metrics HTTP/1.1" 200 33991 "" "Prometheus/2.43.0" 2023-09-23T21:19:49.894 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:19:49.953 INFO:tasks.workunit.client.0.smithi026.stderr: Removing all snapshots: 50% complete... Removing all snapshots: 100% complete... Removing all snapshots: 100% complete...done. 2023-09-23T21:19:49.965 INFO:tasks.workunit.client.0.smithi026.stderr:+ rbd snap purge testimg_import 2023-09-23T21:19:49.994 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T21:19:50.000 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:50.000 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:19:50.101 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:19:50.129 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:50.154 INFO:teuthology.orchestra.run.smithi026.stdout: 2023-09-23T21:19:50.155 INFO:teuthology.orchestra.run.smithi026.stderr:dumped all 2023-09-23T21:19:50.163 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:19:50.165 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:50.166 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:19:50.190 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:19:49 smithi026 ceph-mon[179688]: failed to encode map e664 with expected crc 2023-09-23T21:19:50.190 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:19:49 smithi026 ceph-mon[179688]: failed to encode map e664 with expected crc 2023-09-23T21:19:50.190 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:19:49 smithi026 ceph-mon[179688]: failed to encode map e664 with expected crc 2023-09-23T21:19:50.191 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:19:49 smithi026 ceph-mon[179688]: osdmap e664: 8 total, 8 up, 5 in 2023-09-23T21:19:50.191 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:19:49 smithi026 ceph-mon[179688]: failed to encode map e664 with expected crc 2023-09-23T21:19:50.191 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:19:49 smithi026 ceph-mon[179688]: pgmap v1624: 103 pgs: 103 active+clean; 802 KiB data, 3.7 GiB used, 443 GiB / 447 GiB avail; 405 KiB/s rd, 50 KiB/s wr, 542 op/s 2023-09-23T21:19:50.192 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:19:49 smithi026 ceph-mon[175151]: failed to encode map e664 with expected crc 2023-09-23T21:19:50.192 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:19:49 smithi026 ceph-mon[175151]: failed to encode map e664 with expected crc 2023-09-23T21:19:50.192 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:19:49 smithi026 ceph-mon[175151]: failed to encode map e664 with expected crc 2023-09-23T21:19:50.192 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:19:49 smithi026 ceph-mon[175151]: osdmap e664: 8 total, 8 up, 5 in 2023-09-23T21:19:50.192 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:19:49 smithi026 ceph-mon[175151]: failed to encode map e664 with expected crc 2023-09-23T21:19:50.192 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:19:49 smithi026 ceph-mon[175151]: pgmap v1624: 103 pgs: 103 active+clean; 802 KiB data, 3.7 GiB used, 443 GiB / 447 GiB avail; 405 KiB/s rd, 50 KiB/s wr, 542 op/s 2023-09-23T21:19:50.267 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:19:50.300 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:50.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:19:49 smithi070 ceph-mon[168282]: failed to encode map e664 with expected crc 2023-09-23T21:19:50.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:19:49 smithi070 ceph-mon[168282]: failed to encode map e664 with expected crc 2023-09-23T21:19:50.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:19:49 smithi070 ceph-mon[168282]: failed to encode map e664 with expected crc 2023-09-23T21:19:50.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:19:49 smithi070 ceph-mon[168282]: osdmap e664: 8 total, 8 up, 5 in 2023-09-23T21:19:50.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:19:49 smithi070 ceph-mon[168282]: failed to encode map e664 with expected crc 2023-09-23T21:19:50.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:19:49 smithi070 ceph-mon[168282]: pgmap v1624: 103 pgs: 103 active+clean; 802 KiB data, 3.7 GiB used, 443 GiB / 447 GiB avail; 405 KiB/s rd, 50 KiB/s wr, 542 op/s 2023-09-23T21:19:50.781 INFO:teuthology.orchestra.run.smithi026.stdout:{"pg_ready":true,"pg_map":{"version":1625,"stamp":"2023-09-23T21:19:49.938319+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":821308,"num_objects":36,"num_object_clones":0,"num_object_copies":108,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":36,"num_whiteouts":0,"num_read":3249,"num_read_kb":4743,"num_write":539,"num_write_kb":1319,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":84,"num_bytes_recovered":18960,"num_keys_recovered":20,"num_objects_omap":18,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":5594,"ondisk_log_size":5594,"up":309,"acting":309,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":309,"num_osds":5,"num_per_pool_osds":5,"num_per_pool_omap_osds":5,"kb":468725760,"kb_used":3867420,"kb_used_data":39960,"kb_used_omap":125,"kb_used_meta":3827266,"kb_avail":464858340,"statfs":{"total":479975178240,"available":476014940160,"internally_reserved":0,"allocated":40919040,"data_stored":28353181,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":128662,"internal_metadata":3919120746},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":112691,"num_objects":5,"num_object_clones":0,"num_object_copies":15,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5,"num_whiteouts":0,"num_read":1338,"num_read_kb":1125,"num_write":181,"num_write_kb":121,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"4.026178"},"pg_stats":[{"pgid":"1.18","version":"79'161","reported_seq":704,"reported_epoch":660,"state":"active+clean","last_fresh":"2023-09-23T21:19:44.910014+0000","last_change":"2023-09-23T21:19:34.961897+0000","last_active":"2023-09-23T21:19:44.910014+0000","last_peered":"2023-09-23T21:19:44.910014+0000","last_clean":"2023-09-23T21:19:44.910014+0000","last_became_active":"2023-09-23T21:19:33.641493+0000","last_became_peered":"2023-09-23T21:19:33.641493+0000","last_unstale":"2023-09-23T21:19:44.910014+0000","last_undegraded":"2023-09-23T21:19:44.910014+0000","last_fullsized":"2023-09-23T21:19:44.910014+0000","mapping_epoch":653,"log_start":"0'0","ondisk_log_start":"0'0","created":479,"last_epoch_clean":654,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:19:34.961756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:19:34.961756+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:08:23.522573+0000","scrub_duration":0.002022102,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,0],"acting":[2,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1a","version":"68'1","reported_seq":252,"reported_epoch":661,"state":"active+clean","last_fresh":"2023-09-23T21:19:45.917129+0000","last_change":"2023-09-23T21:13:13.291403+0000","last_active":"2023-09-23T21:19:45.917129+0000","last_peered":"2023-09-23T21:19:45.917129+0000","last_clean":"2023-09-23T21:19:45.917129+0000","last_became_active":"2023-09-23T21:13:13.291123+0000","last_became_peered":"2023-09-23T21:13:13.291123+0000","last_unstale":"2023-09-23T21:19:45.917129+0000","last_undegraded":"2023-09-23T21:19:45.917129+0000","last_fullsized":"2023-09-23T21:19:45.917129+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":495,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:45:15.427430+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,0],"acting":[4,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":331,"reported_epoch":661,"state":"active+clean","last_fresh":"2023-09-23T21:19:45.917602+0000","last_change":"2023-09-23T21:19:30.642278+0000","last_active":"2023-09-23T21:19:45.917602+0000","last_peered":"2023-09-23T21:19:45.917602+0000","last_clean":"2023-09-23T21:19:45.917602+0000","last_became_active":"2023-09-23T21:19:30.637935+0000","last_became_peered":"2023-09-23T21:19:30.637935+0000","last_unstale":"2023-09-23T21:19:45.917602+0000","last_undegraded":"2023-09-23T21:19:45.917602+0000","last_fullsized":"2023-09-23T21:19:45.917602+0000","mapping_epoch":650,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":651,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:21:57.900400+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1b","version":"79'161","reported_seq":745,"reported_epoch":661,"state":"active+clean","last_fresh":"2023-09-23T21:19:45.917181+0000","last_change":"2023-09-23T21:19:34.862883+0000","last_active":"2023-09-23T21:19:45.917181+0000","last_peered":"2023-09-23T21:19:45.917181+0000","last_clean":"2023-09-23T21:19:45.917181+0000","last_became_active":"2023-09-23T21:19:33.639640+0000","last_became_peered":"2023-09-23T21:19:33.639640+0000","last_unstale":"2023-09-23T21:19:45.917181+0000","last_undegraded":"2023-09-23T21:19:45.917181+0000","last_fullsized":"2023-09-23T21:19:45.917181+0000","mapping_epoch":653,"log_start":"0'0","ondisk_log_start":"0'0","created":479,"last_epoch_clean":654,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:19:34.862788+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:19:34.862788+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:23:17.102526+0000","scrub_duration":0.001907228,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,2],"acting":[4,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":680,"reported_epoch":663,"state":"active+clean","last_fresh":"2023-09-23T21:19:47.934431+0000","last_change":"2023-09-23T21:16:42.448529+0000","last_active":"2023-09-23T21:19:47.934431+0000","last_peered":"2023-09-23T21:19:47.934431+0000","last_clean":"2023-09-23T21:19:47.934431+0000","last_became_active":"2023-09-23T21:16:42.447803+0000","last_became_peered":"2023-09-23T21:16:42.447803+0000","last_unstale":"2023-09-23T21:19:47.934431+0000","last_undegraded":"2023-09-23T21:19:47.934431+0000","last_fullsized":"2023-09-23T21:19:47.934431+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:41:33.121784+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":677,"reported_epoch":659,"state":"active+clean","last_fresh":"2023-09-23T21:19:42.906856+0000","last_change":"2023-09-23T21:16:42.441896+0000","last_active":"2023-09-23T21:19:42.906856+0000","last_peered":"2023-09-23T21:19:42.906856+0000","last_clean":"2023-09-23T21:19:42.906856+0000","last_became_active":"2023-09-23T21:16:42.441526+0000","last_became_peered":"2023-09-23T21:16:42.441526+0000","last_unstale":"2023-09-23T21:19:42.906856+0000","last_undegraded":"2023-09-23T21:19:42.906856+0000","last_fullsized":"2023-09-23T21:19:42.906856+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:27:36.326225+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1a","version":"79'161","reported_seq":779,"reported_epoch":661,"state":"active+clean","last_fresh":"2023-09-23T21:19:45.916875+0000","last_change":"2023-09-23T21:19:33.632804+0000","last_active":"2023-09-23T21:19:45.916875+0000","last_peered":"2023-09-23T21:19:45.916875+0000","last_clean":"2023-09-23T21:19:45.916875+0000","last_became_active":"2023-09-23T21:19:33.632406+0000","last_became_peered":"2023-09-23T21:19:33.632406+0000","last_unstale":"2023-09-23T21:19:45.916875+0000","last_undegraded":"2023-09-23T21:19:45.916875+0000","last_fullsized":"2023-09-23T21:19:45.916875+0000","mapping_epoch":653,"log_start":"0'0","ondisk_log_start":"0'0","created":479,"last_epoch_clean":654,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:15:45.158107+0000","last_deep_scrub":"79'161","last_deep_scrub_stamp":"2023-09-23T21:05:32.630292+0000","last_clean_scrub_stamp":"2023-09-23T21:15:45.158107+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:40:42.578254+0000","scrub_duration":0.001999373,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,2],"acting":[4,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":171,"reported_epoch":659,"state":"active+clean","last_fresh":"2023-09-23T21:19:42.905746+0000","last_change":"2023-09-23T21:13:30.806084+0000","last_active":"2023-09-23T21:19:42.905746+0000","last_peered":"2023-09-23T21:19:42.905746+0000","last_clean":"2023-09-23T21:19:42.905746+0000","last_became_active":"2023-09-23T21:13:30.805743+0000","last_became_peered":"2023-09-23T21:13:30.805743+0000","last_unstale":"2023-09-23T21:19:42.905746+0000","last_undegraded":"2023-09-23T21:19:42.905746+0000","last_fullsized":"2023-09-23T21:19:42.905746+0000","mapping_epoch":501,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:48:10.849281+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":545,"reported_epoch":659,"state":"active+clean","last_fresh":"2023-09-23T21:19:42.906536+0000","last_change":"2023-09-23T21:19:30.639145+0000","last_active":"2023-09-23T21:19:42.906536+0000","last_peered":"2023-09-23T21:19:42.906536+0000","last_clean":"2023-09-23T21:19:42.906536+0000","last_became_active":"2023-09-23T21:19:30.638805+0000","last_became_peered":"2023-09-23T21:19:30.638805+0000","last_unstale":"2023-09-23T21:19:42.906536+0000","last_undegraded":"2023-09-23T21:19:42.906536+0000","last_fullsized":"2023-09-23T21:19:42.906536+0000","mapping_epoch":650,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":651,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:23:01.782834+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.15","version":"79'161","reported_seq":640,"reported_epoch":661,"state":"active+clean","last_fresh":"2023-09-23T21:19:45.922497+0000","last_change":"2023-09-23T21:19:33.643960+0000","last_active":"2023-09-23T21:19:45.922497+0000","last_peered":"2023-09-23T21:19:45.922497+0000","last_clean":"2023-09-23T21:19:45.922497+0000","last_became_active":"2023-09-23T21:19:33.643626+0000","last_became_peered":"2023-09-23T21:19:33.643626+0000","last_unstale":"2023-09-23T21:19:45.922497+0000","last_undegraded":"2023-09-23T21:19:45.922497+0000","last_fullsized":"2023-09-23T21:19:45.922497+0000","mapping_epoch":653,"log_start":"0'0","ondisk_log_start":"0'0","created":479,"last_epoch_clean":654,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:18:35.622422+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:18:35.622422+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:42:48.906664+0000","scrub_duration":0.083031242000000005,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":645,"reported_epoch":659,"state":"active+clean","last_fresh":"2023-09-23T21:19:42.906663+0000","last_change":"2023-09-23T21:19:30.636889+0000","last_active":"2023-09-23T21:19:42.906663+0000","last_peered":"2023-09-23T21:19:42.906663+0000","last_clean":"2023-09-23T21:19:42.906663+0000","last_became_active":"2023-09-23T21:19:30.636520+0000","last_became_peered":"2023-09-23T21:19:30.636520+0000","last_unstale":"2023-09-23T21:19:42.906663+0000","last_undegraded":"2023-09-23T21:19:42.906663+0000","last_fullsized":"2023-09-23T21:19:42.906663+0000","mapping_epoch":650,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":651,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:36:06.289223+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":713,"reported_epoch":663,"state":"active+clean","last_fresh":"2023-09-23T21:19:47.934436+0000","last_change":"2023-09-23T21:19:30.648517+0000","last_active":"2023-09-23T21:19:47.934436+0000","last_peered":"2023-09-23T21:19:47.934436+0000","last_clean":"2023-09-23T21:19:47.934436+0000","last_became_active":"2023-09-23T21:19:30.648184+0000","last_became_peered":"2023-09-23T21:19:30.648184+0000","last_unstale":"2023-09-23T21:19:47.934436+0000","last_undegraded":"2023-09-23T21:19:47.934436+0000","last_fullsized":"2023-09-23T21:19:47.934436+0000","mapping_epoch":650,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":651,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:45:44.022190+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,2],"acting":[3,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"79'161","reported_seq":1058,"reported_epoch":660,"state":"active+clean","last_fresh":"2023-09-23T21:19:44.910847+0000","last_change":"2023-09-23T21:16:45.424773+0000","last_active":"2023-09-23T21:19:44.910847+0000","last_peered":"2023-09-23T21:19:44.910847+0000","last_clean":"2023-09-23T21:19:44.910847+0000","last_became_active":"2023-09-23T21:16:45.424458+0000","last_became_peered":"2023-09-23T21:16:45.424458+0000","last_unstale":"2023-09-23T21:19:44.910847+0000","last_undegraded":"2023-09-23T21:19:44.910847+0000","last_fullsized":"2023-09-23T21:19:44.910847+0000","mapping_epoch":567,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":568,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:25.096401+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:25.096401+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:40:17.871561+0000","scrub_duration":0.002859393,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":325,"reported_epoch":663,"state":"active+clean","last_fresh":"2023-09-23T21:19:47.935229+0000","last_change":"2023-09-23T21:19:30.642380+0000","last_active":"2023-09-23T21:19:47.935229+0000","last_peered":"2023-09-23T21:19:47.935229+0000","last_clean":"2023-09-23T21:19:47.935229+0000","last_became_active":"2023-09-23T21:19:30.642050+0000","last_became_peered":"2023-09-23T21:19:30.642050+0000","last_unstale":"2023-09-23T21:19:47.935229+0000","last_undegraded":"2023-09-23T21:19:47.935229+0000","last_fullsized":"2023-09-23T21:19:47.935229+0000","mapping_epoch":650,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":651,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:31:40.650598+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":668,"reported_epoch":663,"state":"active+clean","last_fresh":"2023-09-23T21:19:47.934421+0000","last_change":"2023-09-23T21:19:30.643841+0000","last_active":"2023-09-23T21:19:47.934421+0000","last_peered":"2023-09-23T21:19:47.934421+0000","last_clean":"2023-09-23T21:19:47.934421+0000","last_became_active":"2023-09-23T21:19:30.641826+0000","last_became_peered":"2023-09-23T21:19:30.641826+0000","last_unstale":"2023-09-23T21:19:47.934421+0000","last_undegraded":"2023-09-23T21:19:47.934421+0000","last_fullsized":"2023-09-23T21:19:47.934421+0000","mapping_epoch":650,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":651,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:36:08.403135+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.17","version":"79'161","reported_seq":824,"reported_epoch":659,"state":"active+clean","last_fresh":"2023-09-23T21:19:42.906156+0000","last_change":"2023-09-23T21:19:33.644079+0000","last_active":"2023-09-23T21:19:42.906156+0000","last_peered":"2023-09-23T21:19:42.906156+0000","last_clean":"2023-09-23T21:19:42.906156+0000","last_became_active":"2023-09-23T21:19:33.643672+0000","last_became_peered":"2023-09-23T21:19:33.643672+0000","last_unstale":"2023-09-23T21:19:42.906156+0000","last_undegraded":"2023-09-23T21:19:42.906156+0000","last_fullsized":"2023-09-23T21:19:42.906156+0000","mapping_epoch":653,"log_start":"0'0","ondisk_log_start":"0'0","created":479,"last_epoch_clean":654,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:15:50.325741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:15:50.325741+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:14:03.837900+0000","scrub_duration":0.0040500149999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":580,"reported_epoch":663,"state":"active+clean","last_fresh":"2023-09-23T21:19:47.934906+0000","last_change":"2023-09-23T21:17:02.182606+0000","last_active":"2023-09-23T21:19:47.934906+0000","last_peered":"2023-09-23T21:19:47.934906+0000","last_clean":"2023-09-23T21:19:47.934906+0000","last_became_active":"2023-09-23T21:16:42.446517+0000","last_became_peered":"2023-09-23T21:16:42.446517+0000","last_unstale":"2023-09-23T21:19:47.934906+0000","last_undegraded":"2023-09-23T21:19:47.934906+0000","last_fullsized":"2023-09-23T21:19:47.934906+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:46:08.337878+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":540,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,4],"acting":[3,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":359,"reported_epoch":660,"state":"active+clean","last_fresh":"2023-09-23T21:19:44.910770+0000","last_change":"2023-09-23T21:19:30.637412+0000","last_active":"2023-09-23T21:19:44.910770+0000","last_peered":"2023-09-23T21:19:44.910770+0000","last_clean":"2023-09-23T21:19:44.910770+0000","last_became_active":"2023-09-23T21:19:30.637099+0000","last_became_peered":"2023-09-23T21:19:30.637099+0000","last_unstale":"2023-09-23T21:19:44.910770+0000","last_undegraded":"2023-09-23T21:19:44.910770+0000","last_fullsized":"2023-09-23T21:19:44.910770+0000","mapping_epoch":650,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":651,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:59:58.577998+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,7],"acting":[2,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.16","version":"79'161","reported_seq":592,"reported_epoch":659,"state":"active+clean","last_fresh":"2023-09-23T21:19:42.906779+0000","last_change":"2023-09-23T21:15:46.326726+0000","last_active":"2023-09-23T21:19:42.906779+0000","last_peered":"2023-09-23T21:19:42.906779+0000","last_clean":"2023-09-23T21:19:42.906779+0000","last_became_active":"2023-09-23T21:13:33.770407+0000","last_became_peered":"2023-09-23T21:13:33.770407+0000","last_unstale":"2023-09-23T21:19:42.906779+0000","last_undegraded":"2023-09-23T21:19:42.906779+0000","last_fullsized":"2023-09-23T21:19:42.906779+0000","mapping_epoch":504,"log_start":"0'0","ondisk_log_start":"0'0","created":479,"last_epoch_clean":505,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:15:46.326589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:15:46.326589+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:50:23.810054+0000","scrub_duration":0.0062069059999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.14","version":"79'22","reported_seq":265,"reported_epoch":661,"state":"active+clean","last_fresh":"2023-09-23T21:19:45.917453+0000","last_change":"2023-09-23T21:17:02.198735+0000","last_active":"2023-09-23T21:19:45.917453+0000","last_peered":"2023-09-23T21:19:45.917453+0000","last_clean":"2023-09-23T21:19:45.917453+0000","last_became_active":"2023-09-23T21:16:42.492861+0000","last_became_peered":"2023-09-23T21:16:42.492861+0000","last_unstale":"2023-09-23T21:19:45.917453+0000","last_undegraded":"2023-09-23T21:19:45.917453+0000","last_fullsized":"2023-09-23T21:19:45.917453+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:00:09.949965+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":14202,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,2],"acting":[4,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":600,"reported_epoch":661,"state":"active+clean","last_fresh":"2023-09-23T21:19:45.917277+0000","last_change":"2023-09-23T21:19:30.638011+0000","last_active":"2023-09-23T21:19:45.917277+0000","last_peered":"2023-09-23T21:19:45.917277+0000","last_clean":"2023-09-23T21:19:45.917277+0000","last_became_active":"2023-09-23T21:19:30.636676+0000","last_became_peered":"2023-09-23T21:19:30.636676+0000","last_unstale":"2023-09-23T21:19:45.917277+0000","last_undegraded":"2023-09-23T21:19:45.917277+0000","last_fullsized":"2023-09-23T21:19:45.917277+0000","mapping_epoch":650,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":651,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:26:22.982323+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.11","version":"79'161","reported_seq":614,"reported_epoch":659,"state":"active+clean","last_fresh":"2023-09-23T21:19:42.905718+0000","last_change":"2023-09-23T21:19:33.646270+0000","last_active":"2023-09-23T21:19:42.905718+0000","last_peered":"2023-09-23T21:19:42.905718+0000","last_clean":"2023-09-23T21:19:42.905718+0000","last_became_active":"2023-09-23T21:19:33.645912+0000","last_became_peered":"2023-09-23T21:19:33.645912+0000","last_unstale":"2023-09-23T21:19:42.905718+0000","last_undegraded":"2023-09-23T21:19:42.905718+0000","last_fullsized":"2023-09-23T21:19:42.905718+0000","mapping_epoch":653,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":654,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:15:47.299947+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:15:47.299947+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:23:30.166866+0000","scrub_duration":0.0061353110000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":113,"reported_epoch":661,"state":"active+clean","last_fresh":"2023-09-23T21:19:45.922052+0000","last_change":"2023-09-23T21:16:42.531060+0000","last_active":"2023-09-23T21:19:45.922052+0000","last_peered":"2023-09-23T21:19:45.922052+0000","last_clean":"2023-09-23T21:19:45.922052+0000","last_became_active":"2023-09-23T21:16:42.449198+0000","last_became_peered":"2023-09-23T21:16:42.449198+0000","last_unstale":"2023-09-23T21:19:45.922052+0000","last_undegraded":"2023-09-23T21:19:45.922052+0000","last_fullsized":"2023-09-23T21:19:45.922052+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:30:23.428886+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":222,"reported_epoch":660,"state":"active+clean","last_fresh":"2023-09-23T21:19:44.910535+0000","last_change":"2023-09-23T21:16:42.491023+0000","last_active":"2023-09-23T21:19:44.910535+0000","last_peered":"2023-09-23T21:19:44.910535+0000","last_clean":"2023-09-23T21:19:44.910535+0000","last_became_active":"2023-09-23T21:16:42.489896+0000","last_became_peered":"2023-09-23T21:16:42.489896+0000","last_unstale":"2023-09-23T21:19:44.910535+0000","last_undegraded":"2023-09-23T21:19:44.910535+0000","last_fullsized":"2023-09-23T21:19:44.910535+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:58:54.036098+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,7],"acting":[2,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.10","version":"79'161","reported_seq":856,"reported_epoch":661,"state":"active+clean","last_fresh":"2023-09-23T21:19:45.916855+0000","last_change":"2023-09-23T21:19:33.639783+0000","last_active":"2023-09-23T21:19:45.916855+0000","last_peered":"2023-09-23T21:19:45.916855+0000","last_clean":"2023-09-23T21:19:45.916855+0000","last_became_active":"2023-09-23T21:19:33.639497+0000","last_became_peered":"2023-09-23T21:19:33.639497+0000","last_unstale":"2023-09-23T21:19:45.916855+0000","last_undegraded":"2023-09-23T21:19:45.916855+0000","last_fullsized":"2023-09-23T21:19:45.916855+0000","mapping_epoch":653,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":654,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:06:50.336992+0000","last_deep_scrub":"79'161","last_deep_scrub_stamp":"2023-09-23T21:06:50.336992+0000","last_clean_scrub_stamp":"2023-09-23T21:06:50.336992+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:39:29.427376+0000","scrub_duration":0.0026150930000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":620,"reported_epoch":659,"state":"active+clean","last_fresh":"2023-09-23T21:19:42.906311+0000","last_change":"2023-09-23T21:16:42.446324+0000","last_active":"2023-09-23T21:19:42.906311+0000","last_peered":"2023-09-23T21:19:42.906311+0000","last_clean":"2023-09-23T21:19:42.906311+0000","last_became_active":"2023-09-23T21:16:42.445997+0000","last_became_peered":"2023-09-23T21:16:42.445997+0000","last_unstale":"2023-09-23T21:19:42.906311+0000","last_undegraded":"2023-09-23T21:19:42.906311+0000","last_fullsized":"2023-09-23T21:19:42.906311+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:46:35.028292+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":445,"reported_epoch":663,"state":"active+clean","last_fresh":"2023-09-23T21:19:47.934760+0000","last_change":"2023-09-23T21:19:30.642090+0000","last_active":"2023-09-23T21:19:47.934760+0000","last_peered":"2023-09-23T21:19:47.934760+0000","last_clean":"2023-09-23T21:19:47.934760+0000","last_became_active":"2023-09-23T21:19:30.641761+0000","last_became_peered":"2023-09-23T21:19:30.641761+0000","last_unstale":"2023-09-23T21:19:47.934760+0000","last_undegraded":"2023-09-23T21:19:47.934760+0000","last_fullsized":"2023-09-23T21:19:47.934760+0000","mapping_epoch":650,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":651,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:20:06.738087+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.13","version":"79'161","reported_seq":743,"reported_epoch":663,"state":"active+clean","last_fresh":"2023-09-23T21:19:47.934909+0000","last_change":"2023-09-23T21:13:14.933096+0000","last_active":"2023-09-23T21:19:47.934909+0000","last_peered":"2023-09-23T21:19:47.934909+0000","last_clean":"2023-09-23T21:19:47.934909+0000","last_became_active":"2023-09-23T21:13:14.932703+0000","last_became_peered":"2023-09-23T21:13:14.932703+0000","last_unstale":"2023-09-23T21:19:47.934909+0000","last_undegraded":"2023-09-23T21:19:47.934909+0000","last_fullsized":"2023-09-23T21:19:47.934909+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:36.420362+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:36.420362+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:29:42.163911+0000","scrub_duration":0.028838444000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":673,"reported_epoch":661,"state":"active+clean","last_fresh":"2023-09-23T21:19:45.916955+0000","last_change":"2023-09-23T21:16:42.420467+0000","last_active":"2023-09-23T21:19:45.916955+0000","last_peered":"2023-09-23T21:19:45.916955+0000","last_clean":"2023-09-23T21:19:45.916955+0000","last_became_active":"2023-09-23T21:16:42.420149+0000","last_became_peered":"2023-09-23T21:16:42.420149+0000","last_unstale":"2023-09-23T21:19:45.916955+0000","last_undegraded":"2023-09-23T21:19:45.916955+0000","last_fullsized":"2023-09-23T21:19:45.916955+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:52:07.355000+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":189,"reported_epoch":659,"state":"active+clean","last_fresh":"2023-09-23T21:19:42.906447+0000","last_change":"2023-09-23T21:16:42.427663+0000","last_active":"2023-09-23T21:19:42.906447+0000","last_peered":"2023-09-23T21:19:42.906447+0000","last_clean":"2023-09-23T21:19:42.906447+0000","last_became_active":"2023-09-23T21:16:42.426961+0000","last_became_peered":"2023-09-23T21:16:42.426961+0000","last_unstale":"2023-09-23T21:19:42.906447+0000","last_undegraded":"2023-09-23T21:19:42.906447+0000","last_fullsized":"2023-09-23T21:19:42.906447+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:44:27.422007+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.12","version":"79'161","reported_seq":856,"reported_epoch":661,"state":"active+clean","last_fresh":"2023-09-23T21:19:45.922086+0000","last_change":"2023-09-23T21:19:33.639394+0000","last_active":"2023-09-23T21:19:45.922086+0000","last_peered":"2023-09-23T21:19:45.922086+0000","last_clean":"2023-09-23T21:19:45.922086+0000","last_became_active":"2023-09-23T21:19:33.639037+0000","last_became_peered":"2023-09-23T21:19:33.639037+0000","last_unstale":"2023-09-23T21:19:45.922086+0000","last_undegraded":"2023-09-23T21:19:45.922086+0000","last_fullsized":"2023-09-23T21:19:45.922086+0000","mapping_epoch":653,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":654,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:05:56.785173+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:05:56.785173+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:12:04.530912+0000","scrub_duration":0.0042114370000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,2],"acting":[7,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":200,"reported_epoch":659,"state":"active+clean","last_fresh":"2023-09-23T21:19:42.905794+0000","last_change":"2023-09-23T21:19:30.658056+0000","last_active":"2023-09-23T21:19:42.905794+0000","last_peered":"2023-09-23T21:19:42.905794+0000","last_clean":"2023-09-23T21:19:42.905794+0000","last_became_active":"2023-09-23T21:19:30.636349+0000","last_became_peered":"2023-09-23T21:19:30.636349+0000","last_unstale":"2023-09-23T21:19:42.905794+0000","last_undegraded":"2023-09-23T21:19:42.905794+0000","last_fullsized":"2023-09-23T21:19:42.905794+0000","mapping_epoch":650,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":651,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:59:18.953067+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":521,"reported_epoch":663,"state":"active+clean","last_fresh":"2023-09-23T21:19:47.934948+0000","last_change":"2023-09-23T21:16:42.518512+0000","last_active":"2023-09-23T21:19:47.934948+0000","last_peered":"2023-09-23T21:19:47.934948+0000","last_clean":"2023-09-23T21:19:47.934948+0000","last_became_active":"2023-09-23T21:16:42.517464+0000","last_became_peered":"2023-09-23T21:16:42.517464+0000","last_unstale":"2023-09-23T21:19:47.934948+0000","last_undegraded":"2023-09-23T21:19:47.934948+0000","last_fullsized":"2023-09-23T21:19:47.934948+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:47:52.391167+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.d","version":"79'161","reported_seq":703,"reported_epoch":661,"state":"active+clean","last_fresh":"2023-09-23T21:19:45.922538+0000","last_change":"2023-09-23T21:18:47.440374+0000","last_active":"2023-09-23T21:19:45.922538+0000","last_peered":"2023-09-23T21:19:45.922538+0000","last_clean":"2023-09-23T21:19:45.922538+0000","last_became_active":"2023-09-23T21:16:45.416373+0000","last_became_peered":"2023-09-23T21:16:45.416373+0000","last_unstale":"2023-09-23T21:19:45.922538+0000","last_undegraded":"2023-09-23T21:19:45.922538+0000","last_fullsized":"2023-09-23T21:19:45.922538+0000","mapping_epoch":567,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":568,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:18:47.440263+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:18:47.440263+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:11:36.676449+0000","scrub_duration":0.0023849959999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":111,"reported_epoch":661,"state":"active+clean","last_fresh":"2023-09-23T21:19:45.922711+0000","last_change":"2023-09-23T21:17:02.190320+0000","last_active":"2023-09-23T21:19:45.922711+0000","last_peered":"2023-09-23T21:19:45.922711+0000","last_clean":"2023-09-23T21:19:45.922711+0000","last_became_active":"2023-09-23T21:16:42.451380+0000","last_became_peered":"2023-09-23T21:16:42.451380+0000","last_unstale":"2023-09-23T21:19:45.922711+0000","last_undegraded":"2023-09-23T21:19:45.922711+0000","last_fullsized":"2023-09-23T21:19:45.922711+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:48:58.425821+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":132,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":307,"reported_epoch":661,"state":"active+clean","last_fresh":"2023-09-23T21:19:45.916756+0000","last_change":"2023-09-23T21:19:30.635500+0000","last_active":"2023-09-23T21:19:45.916756+0000","last_peered":"2023-09-23T21:19:45.916756+0000","last_clean":"2023-09-23T21:19:45.916756+0000","last_became_active":"2023-09-23T21:19:30.635193+0000","last_became_peered":"2023-09-23T21:19:30.635193+0000","last_unstale":"2023-09-23T21:19:45.916756+0000","last_undegraded":"2023-09-23T21:19:45.916756+0000","last_fullsized":"2023-09-23T21:19:45.916756+0000","mapping_epoch":650,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":651,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:21:34.438775+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.c","version":"79'161","reported_seq":1047,"reported_epoch":663,"state":"active+clean","last_fresh":"2023-09-23T21:19:47.935673+0000","last_change":"2023-09-23T21:15:31.992756+0000","last_active":"2023-09-23T21:19:47.935673+0000","last_peered":"2023-09-23T21:19:47.935673+0000","last_clean":"2023-09-23T21:19:47.935673+0000","last_became_active":"2023-09-23T21:13:13.401526+0000","last_became_peered":"2023-09-23T21:13:13.401526+0000","last_unstale":"2023-09-23T21:19:47.935673+0000","last_undegraded":"2023-09-23T21:19:47.935673+0000","last_fullsized":"2023-09-23T21:19:47.935673+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":532,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:15:31.992598+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:15:31.992598+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:41:14.215543+0000","scrub_duration":0.002960678,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":111,"reported_epoch":661,"state":"active+clean","last_fresh":"2023-09-23T21:19:45.922564+0000","last_change":"2023-09-23T21:17:02.215996+0000","last_active":"2023-09-23T21:19:45.922564+0000","last_peered":"2023-09-23T21:19:45.922564+0000","last_clean":"2023-09-23T21:19:45.922564+0000","last_became_active":"2023-09-23T21:16:42.451205+0000","last_became_peered":"2023-09-23T21:16:42.451205+0000","last_unstale":"2023-09-23T21:19:45.922564+0000","last_undegraded":"2023-09-23T21:19:45.922564+0000","last_fullsized":"2023-09-23T21:19:45.922564+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:51:46.876281+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":624,"reported_epoch":661,"state":"active+clean","last_fresh":"2023-09-23T21:19:45.917018+0000","last_change":"2023-09-23T21:13:15.152792+0000","last_active":"2023-09-23T21:19:45.917018+0000","last_peered":"2023-09-23T21:19:45.917018+0000","last_clean":"2023-09-23T21:19:45.917018+0000","last_became_active":"2023-09-23T21:13:14.515084+0000","last_became_peered":"2023-09-23T21:13:14.515084+0000","last_unstale":"2023-09-23T21:19:45.917018+0000","last_undegraded":"2023-09-23T21:19:45.917018+0000","last_fullsized":"2023-09-23T21:19:45.917018+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:08:16.702072+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.f","version":"79'161","reported_seq":758,"reported_epoch":661,"state":"active+clean","last_fresh":"2023-09-23T21:19:45.917449+0000","last_change":"2023-09-23T21:19:33.640767+0000","last_active":"2023-09-23T21:19:45.917449+0000","last_peered":"2023-09-23T21:19:45.917449+0000","last_clean":"2023-09-23T21:19:45.917449+0000","last_became_active":"2023-09-23T21:19:33.640477+0000","last_became_peered":"2023-09-23T21:19:33.640477+0000","last_unstale":"2023-09-23T21:19:45.917449+0000","last_undegraded":"2023-09-23T21:19:45.917449+0000","last_fullsized":"2023-09-23T21:19:45.917449+0000","mapping_epoch":653,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":654,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:27.880776+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:27.880776+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:40:30.960460+0000","scrub_duration":0.0014004899999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":25,"reported_epoch":661,"state":"active+clean","last_fresh":"2023-09-23T21:19:45.917307+0000","last_change":"2023-09-23T21:19:30.662735+0000","last_active":"2023-09-23T21:19:45.917307+0000","last_peered":"2023-09-23T21:19:45.917307+0000","last_clean":"2023-09-23T21:19:45.917307+0000","last_became_active":"2023-09-23T21:19:30.635656+0000","last_became_peered":"2023-09-23T21:19:30.635656+0000","last_unstale":"2023-09-23T21:19:45.917307+0000","last_undegraded":"2023-09-23T21:19:45.917307+0000","last_fullsized":"2023-09-23T21:19:45.917307+0000","mapping_epoch":650,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":651,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:51:48.290452+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":140,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":536,"reported_epoch":659,"state":"active+clean","last_fresh":"2023-09-23T21:19:42.905841+0000","last_change":"2023-09-23T21:19:30.639010+0000","last_active":"2023-09-23T21:19:42.905841+0000","last_peered":"2023-09-23T21:19:42.905841+0000","last_clean":"2023-09-23T21:19:42.905841+0000","last_became_active":"2023-09-23T21:19:30.638768+0000","last_became_peered":"2023-09-23T21:19:30.638768+0000","last_unstale":"2023-09-23T21:19:42.905841+0000","last_undegraded":"2023-09-23T21:19:42.905841+0000","last_fullsized":"2023-09-23T21:19:42.905841+0000","mapping_epoch":650,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":651,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:48:45.915425+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.e","version":"79'161","reported_seq":880,"reported_epoch":661,"state":"active+clean","last_fresh":"2023-09-23T21:19:45.917631+0000","last_change":"2023-09-23T21:16:45.433204+0000","last_active":"2023-09-23T21:19:45.917631+0000","last_peered":"2023-09-23T21:19:45.917631+0000","last_clean":"2023-09-23T21:19:45.917631+0000","last_became_active":"2023-09-23T21:16:45.432836+0000","last_became_peered":"2023-09-23T21:16:45.432836+0000","last_unstale":"2023-09-23T21:19:45.917631+0000","last_undegraded":"2023-09-23T21:19:45.917631+0000","last_fullsized":"2023-09-23T21:19:45.917631+0000","mapping_epoch":567,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":568,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:15:39.046962+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:15:39.046962+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:11:20.720731+0000","scrub_duration":0.0082638769999999993,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":472,"reported_epoch":663,"state":"active+clean","last_fresh":"2023-09-23T21:19:47.935994+0000","last_change":"2023-09-23T21:19:30.634873+0000","last_active":"2023-09-23T21:19:47.935994+0000","last_peered":"2023-09-23T21:19:47.935994+0000","last_clean":"2023-09-23T21:19:47.935994+0000","last_became_active":"2023-09-23T21:19:30.634549+0000","last_became_peered":"2023-09-23T21:19:30.634549+0000","last_unstale":"2023-09-23T21:19:47.935994+0000","last_undegraded":"2023-09-23T21:19:47.935994+0000","last_fullsized":"2023-09-23T21:19:47.935994+0000","mapping_epoch":650,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":651,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:51:20.192644+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,2],"acting":[3,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":634,"reported_epoch":659,"state":"active+clean","last_fresh":"2023-09-23T21:19:42.906349+0000","last_change":"2023-09-23T21:13:14.339670+0000","last_active":"2023-09-23T21:19:42.906349+0000","last_peered":"2023-09-23T21:19:42.906349+0000","last_clean":"2023-09-23T21:19:42.906349+0000","last_became_active":"2023-09-23T21:13:14.339341+0000","last_became_peered":"2023-09-23T21:13:14.339341+0000","last_unstale":"2023-09-23T21:19:42.906349+0000","last_undegraded":"2023-09-23T21:19:42.906349+0000","last_fullsized":"2023-09-23T21:19:42.906349+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:23:19.411368+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.9","version":"79'161","reported_seq":863,"reported_epoch":660,"state":"active+clean","last_fresh":"2023-09-23T21:19:44.910091+0000","last_change":"2023-09-23T21:16:45.433968+0000","last_active":"2023-09-23T21:19:44.910091+0000","last_peered":"2023-09-23T21:19:44.910091+0000","last_clean":"2023-09-23T21:19:44.910091+0000","last_became_active":"2023-09-23T21:16:45.433602+0000","last_became_peered":"2023-09-23T21:16:45.433602+0000","last_unstale":"2023-09-23T21:19:44.910091+0000","last_undegraded":"2023-09-23T21:19:44.910091+0000","last_fullsized":"2023-09-23T21:19:44.910091+0000","mapping_epoch":567,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":568,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:15:43.437369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:15:43.437369+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:16:44.538102+0000","scrub_duration":0.0024129429999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,3],"acting":[2,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":220,"reported_epoch":659,"state":"active+clean","last_fresh":"2023-09-23T21:19:42.906393+0000","last_change":"2023-09-23T21:13:30.793273+0000","last_active":"2023-09-23T21:19:42.906393+0000","last_peered":"2023-09-23T21:19:42.906393+0000","last_clean":"2023-09-23T21:19:42.906393+0000","last_became_active":"2023-09-23T21:13:30.791269+0000","last_became_peered":"2023-09-23T21:13:30.791269+0000","last_unstale":"2023-09-23T21:19:42.906393+0000","last_undegraded":"2023-09-23T21:19:42.906393+0000","last_fullsized":"2023-09-23T21:19:42.906393+0000","mapping_epoch":501,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:38:07.142290+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":255,"reported_epoch":661,"state":"active+clean","last_fresh":"2023-09-23T21:19:45.917338+0000","last_change":"2023-09-23T21:16:42.417057+0000","last_active":"2023-09-23T21:19:45.917338+0000","last_peered":"2023-09-23T21:19:45.917338+0000","last_clean":"2023-09-23T21:19:45.917338+0000","last_became_active":"2023-09-23T21:16:42.416601+0000","last_became_peered":"2023-09-23T21:16:42.416601+0000","last_unstale":"2023-09-23T21:19:45.917338+0000","last_undegraded":"2023-09-23T21:19:45.917338+0000","last_fullsized":"2023-09-23T21:19:45.917338+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:56:52.167562+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,7],"acting":[4,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.8","version":"79'161","reported_seq":812,"reported_epoch":660,"state":"active+clean","last_fresh":"2023-09-23T21:19:44.910146+0000","last_change":"2023-09-23T21:19:33.642569+0000","last_active":"2023-09-23T21:19:44.910146+0000","last_peered":"2023-09-23T21:19:44.910146+0000","last_clean":"2023-09-23T21:19:44.910146+0000","last_became_active":"2023-09-23T21:19:33.641967+0000","last_became_peered":"2023-09-23T21:19:33.641967+0000","last_unstale":"2023-09-23T21:19:44.910146+0000","last_undegraded":"2023-09-23T21:19:44.910146+0000","last_fullsized":"2023-09-23T21:19:44.910146+0000","mapping_epoch":653,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":654,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:15:36.367640+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:15:36.367640+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:52:22.742871+0000","scrub_duration":0.005826899,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":248,"reported_epoch":661,"state":"active+clean","last_fresh":"2023-09-23T21:19:45.917121+0000","last_change":"2023-09-23T21:19:30.637441+0000","last_active":"2023-09-23T21:19:45.917121+0000","last_peered":"2023-09-23T21:19:45.917121+0000","last_clean":"2023-09-23T21:19:45.917121+0000","last_became_active":"2023-09-23T21:19:30.635390+0000","last_became_peered":"2023-09-23T21:19:30.635390+0000","last_unstale":"2023-09-23T21:19:45.917121+0000","last_undegraded":"2023-09-23T21:19:45.917121+0000","last_fullsized":"2023-09-23T21:19:45.917121+0000","mapping_epoch":650,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":651,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:36:18.999002+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,7],"acting":[4,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":239,"reported_epoch":661,"state":"active+clean","last_fresh":"2023-09-23T21:19:45.917391+0000","last_change":"2023-09-23T21:13:30.772451+0000","last_active":"2023-09-23T21:19:45.917391+0000","last_peered":"2023-09-23T21:19:45.917391+0000","last_clean":"2023-09-23T21:19:45.917391+0000","last_became_active":"2023-09-23T21:13:30.772040+0000","last_became_peered":"2023-09-23T21:13:30.772040+0000","last_unstale":"2023-09-23T21:19:45.917391+0000","last_undegraded":"2023-09-23T21:19:45.917391+0000","last_fullsized":"2023-09-23T21:19:45.917391+0000","mapping_epoch":501,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:24:49.246727+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,0],"acting":[4,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.b","version":"79'161","reported_seq":694,"reported_epoch":659,"state":"active+clean","last_fresh":"2023-09-23T21:19:42.906112+0000","last_change":"2023-09-23T21:19:33.645366+0000","last_active":"2023-09-23T21:19:42.906112+0000","last_peered":"2023-09-23T21:19:42.906112+0000","last_clean":"2023-09-23T21:19:42.906112+0000","last_became_active":"2023-09-23T21:19:33.644992+0000","last_became_peered":"2023-09-23T21:19:33.644992+0000","last_unstale":"2023-09-23T21:19:42.906112+0000","last_undegraded":"2023-09-23T21:19:42.906112+0000","last_fullsized":"2023-09-23T21:19:42.906112+0000","mapping_epoch":653,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":654,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:15:36.339885+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:15:36.339885+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:23:18.575298+0000","scrub_duration":0.0068473279999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,3],"acting":[0,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":239,"reported_epoch":661,"state":"active+clean","last_fresh":"2023-09-23T21:19:45.916933+0000","last_change":"2023-09-23T21:16:42.425103+0000","last_active":"2023-09-23T21:19:45.916933+0000","last_peered":"2023-09-23T21:19:45.916933+0000","last_clean":"2023-09-23T21:19:45.916933+0000","last_became_active":"2023-09-23T21:16:42.424543+0000","last_became_peered":"2023-09-23T21:16:42.424543+0000","last_unstale":"2023-09-23T21:19:45.916933+0000","last_undegraded":"2023-09-23T21:19:45.916933+0000","last_fullsized":"2023-09-23T21:19:45.916933+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:11:28.885246+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":603,"reported_epoch":661,"state":"active+clean","last_fresh":"2023-09-23T21:19:45.917735+0000","last_change":"2023-09-23T21:19:30.635633+0000","last_active":"2023-09-23T21:19:45.917735+0000","last_peered":"2023-09-23T21:19:45.917735+0000","last_clean":"2023-09-23T21:19:45.917735+0000","last_became_active":"2023-09-23T21:19:30.635295+0000","last_became_peered":"2023-09-23T21:19:30.635295+0000","last_unstale":"2023-09-23T21:19:45.917735+0000","last_undegraded":"2023-09-23T21:19:45.917735+0000","last_fullsized":"2023-09-23T21:19:45.917735+0000","mapping_epoch":650,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":651,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:17:02.473642+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,7],"acting":[4,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.a","version":"79'161","reported_seq":838,"reported_epoch":660,"state":"active+clean","last_fresh":"2023-09-23T21:19:44.910544+0000","last_change":"2023-09-23T21:19:33.639230+0000","last_active":"2023-09-23T21:19:44.910544+0000","last_peered":"2023-09-23T21:19:44.910544+0000","last_clean":"2023-09-23T21:19:44.910544+0000","last_became_active":"2023-09-23T21:19:33.638876+0000","last_became_peered":"2023-09-23T21:19:33.638876+0000","last_unstale":"2023-09-23T21:19:44.910544+0000","last_undegraded":"2023-09-23T21:19:44.910544+0000","last_fullsized":"2023-09-23T21:19:44.910544+0000","mapping_epoch":653,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":654,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:15:46.420650+0000","last_deep_scrub":"79'161","last_deep_scrub_stamp":"2023-09-23T21:05:32.630292+0000","last_clean_scrub_stamp":"2023-09-23T21:15:46.420650+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:55:09.974123+0000","scrub_duration":0.0072898930000000004,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,0],"acting":[2,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":531,"reported_epoch":663,"state":"active+clean","last_fresh":"2023-09-23T21:19:47.934213+0000","last_change":"2023-09-23T21:16:42.447403+0000","last_active":"2023-09-23T21:19:47.934213+0000","last_peered":"2023-09-23T21:19:47.934213+0000","last_clean":"2023-09-23T21:19:47.934213+0000","last_became_active":"2023-09-23T21:16:42.447080+0000","last_became_peered":"2023-09-23T21:16:42.447080+0000","last_unstale":"2023-09-23T21:19:47.934213+0000","last_undegraded":"2023-09-23T21:19:47.934213+0000","last_fullsized":"2023-09-23T21:19:47.934213+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:15:19.446334+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,4],"acting":[3,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":662,"reported_epoch":663,"state":"active+clean","last_fresh":"2023-09-23T21:19:47.935092+0000","last_change":"2023-09-23T21:16:42.518739+0000","last_active":"2023-09-23T21:19:47.935092+0000","last_peered":"2023-09-23T21:19:47.935092+0000","last_clean":"2023-09-23T21:19:47.935092+0000","last_became_active":"2023-09-23T21:16:42.517945+0000","last_became_peered":"2023-09-23T21:16:42.517945+0000","last_unstale":"2023-09-23T21:19:47.935092+0000","last_undegraded":"2023-09-23T21:19:47.935092+0000","last_fullsized":"2023-09-23T21:19:47.935092+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:47:26.236933+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.7","version":"79'161","reported_seq":821,"reported_epoch":663,"state":"active+clean","last_fresh":"2023-09-23T21:19:47.934902+0000","last_change":"2023-09-23T21:19:33.645253+0000","last_active":"2023-09-23T21:19:47.934902+0000","last_peered":"2023-09-23T21:19:47.934902+0000","last_clean":"2023-09-23T21:19:47.934902+0000","last_became_active":"2023-09-23T21:19:33.644706+0000","last_became_peered":"2023-09-23T21:19:33.644706+0000","last_unstale":"2023-09-23T21:19:47.934902+0000","last_undegraded":"2023-09-23T21:19:47.934902+0000","last_fullsized":"2023-09-23T21:19:47.934902+0000","mapping_epoch":653,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":654,"parent":"0.0","parent_split_bits":3,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:15:51.016427+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:15:51.016427+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:01:01.623826+0000","scrub_duration":0.0017648640000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.4","version":"660'92","reported_seq":843,"reported_epoch":660,"state":"active+clean","last_fresh":"2023-09-23T21:19:44.965057+0000","last_change":"2023-09-23T21:19:42.907395+0000","last_active":"2023-09-23T21:19:44.965057+0000","last_peered":"2023-09-23T21:19:44.965057+0000","last_clean":"2023-09-23T21:19:44.965057+0000","last_became_active":"2023-09-23T21:16:42.518488+0000","last_became_peered":"2023-09-23T21:16:42.518488+0000","last_unstale":"2023-09-23T21:19:44.965057+0000","last_undegraded":"2023-09-23T21:19:44.965057+0000","last_fullsized":"2023-09-23T21:19:44.965057+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":92,"log_dups_size":0,"ondisk_log_size":92,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:42:44.144383+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00080818000000000003,"stat_sum":{"num_bytes":35,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":348,"num_read_kb":769,"num_write":99,"num_write_kb":281,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,2],"acting":[0,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[{"start":"4","length":"2"}]},{"pgid":"3.5","version":"0'0","reported_seq":155,"reported_epoch":663,"state":"active+clean","last_fresh":"2023-09-23T21:19:47.934996+0000","last_change":"2023-09-23T21:19:30.643393+0000","last_active":"2023-09-23T21:19:47.934996+0000","last_peered":"2023-09-23T21:19:47.934996+0000","last_clean":"2023-09-23T21:19:47.934996+0000","last_became_active":"2023-09-23T21:19:30.643100+0000","last_became_peered":"2023-09-23T21:19:30.643100+0000","last_unstale":"2023-09-23T21:19:47.934996+0000","last_undegraded":"2023-09-23T21:19:47.934996+0000","last_fullsized":"2023-09-23T21:19:47.934996+0000","mapping_epoch":650,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":651,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:38:20.831056+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,7],"acting":[3,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":519,"reported_epoch":661,"state":"active+clean","last_fresh":"2023-09-23T21:19:45.916969+0000","last_change":"2023-09-23T21:19:30.636897+0000","last_active":"2023-09-23T21:19:45.916969+0000","last_peered":"2023-09-23T21:19:45.916969+0000","last_clean":"2023-09-23T21:19:45.916969+0000","last_became_active":"2023-09-23T21:19:30.635812+0000","last_became_peered":"2023-09-23T21:19:30.635812+0000","last_unstale":"2023-09-23T21:19:45.916969+0000","last_undegraded":"2023-09-23T21:19:45.916969+0000","last_fullsized":"2023-09-23T21:19:45.916969+0000","mapping_epoch":650,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":651,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:17:40.371897+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,2],"acting":[4,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1","version":"79'161","reported_seq":762,"reported_epoch":659,"state":"active+clean","last_fresh":"2023-09-23T21:19:42.906026+0000","last_change":"2023-09-23T21:19:33.638597+0000","last_active":"2023-09-23T21:19:42.906026+0000","last_peered":"2023-09-23T21:19:42.906026+0000","last_clean":"2023-09-23T21:19:42.906026+0000","last_became_active":"2023-09-23T21:19:33.638177+0000","last_became_peered":"2023-09-23T21:19:33.638177+0000","last_unstale":"2023-09-23T21:19:42.906026+0000","last_undegraded":"2023-09-23T21:19:42.906026+0000","last_fullsized":"2023-09-23T21:19:42.906026+0000","mapping_epoch":653,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":654,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:15:34.345138+0000","last_deep_scrub":"79'161","last_deep_scrub_stamp":"2023-09-23T21:05:30.652189+0000","last_clean_scrub_stamp":"2023-09-23T21:15:34.345138+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:02:45.462934+0000","scrub_duration":0.0057520000000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"659'50","reported_seq":689,"reported_epoch":660,"state":"active+clean","last_fresh":"2023-09-23T21:19:44.917538+0000","last_change":"2023-09-23T21:19:42.907126+0000","last_active":"2023-09-23T21:19:44.917538+0000","last_peered":"2023-09-23T21:19:44.917538+0000","last_clean":"2023-09-23T21:19:44.917538+0000","last_became_active":"2023-09-23T21:19:30.638661+0000","last_became_peered":"2023-09-23T21:19:30.638661+0000","last_unstale":"2023-09-23T21:19:44.917538+0000","last_undegraded":"2023-09-23T21:19:44.917538+0000","last_fullsized":"2023-09-23T21:19:44.917538+0000","mapping_epoch":650,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":651,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":50,"log_dups_size":0,"ondisk_log_size":50,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:36:27.438599+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00095677999999999996,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":558,"num_read_kb":785,"num_write":30,"num_write_kb":168,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[{"start":"4","length":"2"}]},{"pgid":"3.3","version":"0'0","reported_seq":605,"reported_epoch":661,"state":"active+clean","last_fresh":"2023-09-23T21:19:45.917541+0000","last_change":"2023-09-23T21:19:30.638123+0000","last_active":"2023-09-23T21:19:45.917541+0000","last_peered":"2023-09-23T21:19:45.917541+0000","last_clean":"2023-09-23T21:19:45.917541+0000","last_became_active":"2023-09-23T21:19:30.637749+0000","last_became_peered":"2023-09-23T21:19:30.637749+0000","last_unstale":"2023-09-23T21:19:45.917541+0000","last_undegraded":"2023-09-23T21:19:45.917541+0000","last_fullsized":"2023-09-23T21:19:45.917541+0000","mapping_epoch":650,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":651,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:35:54.298068+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":501,"reported_epoch":660,"state":"active+clean","last_fresh":"2023-09-23T21:19:44.910767+0000","last_change":"2023-09-23T21:16:42.491111+0000","last_active":"2023-09-23T21:19:44.910767+0000","last_peered":"2023-09-23T21:19:44.910767+0000","last_clean":"2023-09-23T21:19:44.910767+0000","last_became_active":"2023-09-23T21:16:42.490275+0000","last_became_peered":"2023-09-23T21:16:42.490275+0000","last_unstale":"2023-09-23T21:19:44.910767+0000","last_undegraded":"2023-09-23T21:19:44.910767+0000","last_fullsized":"2023-09-23T21:19:44.910767+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:39:11.652564+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,7],"acting":[2,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.2","version":"79'161","reported_seq":976,"reported_epoch":659,"state":"active+clean","last_fresh":"2023-09-23T21:19:42.905937+0000","last_change":"2023-09-23T21:16:45.417075+0000","last_active":"2023-09-23T21:19:42.905937+0000","last_peered":"2023-09-23T21:19:42.905937+0000","last_clean":"2023-09-23T21:19:42.905937+0000","last_became_active":"2023-09-23T21:16:45.416752+0000","last_became_peered":"2023-09-23T21:16:45.416752+0000","last_unstale":"2023-09-23T21:19:42.905937+0000","last_undegraded":"2023-09-23T21:19:42.905937+0000","last_fullsized":"2023-09-23T21:19:42.905937+0000","mapping_epoch":567,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":568,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:24.802451+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:24.802451+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:01:51.129779+0000","scrub_duration":0.0090861250000000005,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,4],"acting":[0,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"661'55","reported_seq":884,"reported_epoch":661,"state":"active+clean","last_fresh":"2023-09-23T21:19:46.095164+0000","last_change":"2023-09-23T21:19:42.911743+0000","last_active":"2023-09-23T21:19:46.095164+0000","last_peered":"2023-09-23T21:19:46.095164+0000","last_clean":"2023-09-23T21:19:46.095164+0000","last_became_active":"2023-09-23T21:13:12.206650+0000","last_became_peered":"2023-09-23T21:13:12.206650+0000","last_unstale":"2023-09-23T21:19:46.095164+0000","last_undegraded":"2023-09-23T21:19:46.095164+0000","last_fullsized":"2023-09-23T21:19:46.095164+0000","mapping_epoch":493,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":55,"log_dups_size":0,"ondisk_log_size":55,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:10:01.892936+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000324808,"stat_sum":{"num_bytes":112656,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":338,"num_read_kb":616,"num_write":49,"num_write_kb":221,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[{"start":"4","length":"2"}]},{"pgid":"3.0","version":"0'0","reported_seq":664,"reported_epoch":660,"state":"active+clean","last_fresh":"2023-09-23T21:19:44.910286+0000","last_change":"2023-09-23T21:19:30.634282+0000","last_active":"2023-09-23T21:19:44.910286+0000","last_peered":"2023-09-23T21:19:44.910286+0000","last_clean":"2023-09-23T21:19:44.910286+0000","last_became_active":"2023-09-23T21:19:30.633882+0000","last_became_peered":"2023-09-23T21:19:30.633882+0000","last_unstale":"2023-09-23T21:19:44.910286+0000","last_undegraded":"2023-09-23T21:19:44.910286+0000","last_fullsized":"2023-09-23T21:19:44.910286+0000","mapping_epoch":650,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":651,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:59:27.972139+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,3],"acting":[2,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":198,"reported_epoch":659,"state":"active+clean","last_fresh":"2023-09-23T21:19:42.906585+0000","last_change":"2023-09-23T21:19:30.639918+0000","last_active":"2023-09-23T21:19:42.906585+0000","last_peered":"2023-09-23T21:19:42.906585+0000","last_clean":"2023-09-23T21:19:42.906585+0000","last_became_active":"2023-09-23T21:19:30.639573+0000","last_became_peered":"2023-09-23T21:19:30.639573+0000","last_unstale":"2023-09-23T21:19:42.906585+0000","last_undegraded":"2023-09-23T21:19:42.906585+0000","last_fullsized":"2023-09-23T21:19:42.906585+0000","mapping_epoch":650,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":651,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:11:50.254704+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.3","version":"661'78","reported_seq":688,"reported_epoch":661,"state":"active+clean","last_fresh":"2023-09-23T21:19:46.031684+0000","last_change":"2023-09-23T21:19:42.912745+0000","last_active":"2023-09-23T21:19:46.031684+0000","last_peered":"2023-09-23T21:19:46.031684+0000","last_clean":"2023-09-23T21:19:46.031684+0000","last_became_active":"2023-09-23T21:19:30.634381+0000","last_became_peered":"2023-09-23T21:19:30.634381+0000","last_unstale":"2023-09-23T21:19:46.031684+0000","last_undegraded":"2023-09-23T21:19:46.031684+0000","last_fullsized":"2023-09-23T21:19:46.031684+0000","mapping_epoch":650,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":651,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":78,"log_dups_size":0,"ondisk_log_size":78,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:13:45.243362+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00082540800000000002,"stat_sum":{"num_bytes":0,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":298,"num_read_kb":278,"num_write":76,"num_write_kb":59,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,3],"acting":[2,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[{"start":"4","length":"2"}]},{"pgid":"1.0","version":"79'161","reported_seq":879,"reported_epoch":659,"state":"active+clean","last_fresh":"2023-09-23T21:19:42.905667+0000","last_change":"2023-09-23T21:16:45.428032+0000","last_active":"2023-09-23T21:19:42.905667+0000","last_peered":"2023-09-23T21:19:42.905667+0000","last_clean":"2023-09-23T21:19:42.905667+0000","last_became_active":"2023-09-23T21:16:45.427674+0000","last_became_peered":"2023-09-23T21:16:45.427674+0000","last_unstale":"2023-09-23T21:19:42.905667+0000","last_undegraded":"2023-09-23T21:19:42.905667+0000","last_fullsized":"2023-09-23T21:19:42.905667+0000","mapping_epoch":567,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":568,"parent":"0.0","parent_split_bits":0,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:20.738273+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:20.738273+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:39:42.441711+0000","scrub_duration":0.0088336339999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,2],"acting":[0,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":466,"reported_epoch":661,"state":"active+clean","last_fresh":"2023-09-23T21:19:45.917072+0000","last_change":"2023-09-23T21:19:30.635491+0000","last_active":"2023-09-23T21:19:45.917072+0000","last_peered":"2023-09-23T21:19:45.917072+0000","last_clean":"2023-09-23T21:19:45.917072+0000","last_became_active":"2023-09-23T21:19:30.635158+0000","last_became_peered":"2023-09-23T21:19:30.635158+0000","last_unstale":"2023-09-23T21:19:45.917072+0000","last_undegraded":"2023-09-23T21:19:45.917072+0000","last_fullsized":"2023-09-23T21:19:45.917072+0000","mapping_epoch":650,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":651,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:13:44.654497+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,7],"acting":[4,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":256,"reported_epoch":661,"state":"active+clean","last_fresh":"2023-09-23T21:19:45.917482+0000","last_change":"2023-09-23T21:16:42.424717+0000","last_active":"2023-09-23T21:19:45.917482+0000","last_peered":"2023-09-23T21:19:45.917482+0000","last_clean":"2023-09-23T21:19:45.917482+0000","last_became_active":"2023-09-23T21:16:42.424376+0000","last_became_peered":"2023-09-23T21:16:42.424376+0000","last_unstale":"2023-09-23T21:19:45.917482+0000","last_undegraded":"2023-09-23T21:19:45.917482+0000","last_fullsized":"2023-09-23T21:19:45.917482+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:25:14.162321+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.3","version":"79'161","reported_seq":984,"reported_epoch":661,"state":"active+clean","last_fresh":"2023-09-23T21:19:45.916993+0000","last_change":"2023-09-23T21:19:33.643576+0000","last_active":"2023-09-23T21:19:45.916993+0000","last_peered":"2023-09-23T21:19:45.916993+0000","last_clean":"2023-09-23T21:19:45.916993+0000","last_became_active":"2023-09-23T21:19:33.643287+0000","last_became_peered":"2023-09-23T21:19:33.643287+0000","last_unstale":"2023-09-23T21:19:45.916993+0000","last_undegraded":"2023-09-23T21:19:45.916993+0000","last_fullsized":"2023-09-23T21:19:45.916993+0000","mapping_epoch":653,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":654,"parent":"0.0","parent_split_bits":3,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:05:57.832302+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:05:57.832302+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:16:10.374830+0000","scrub_duration":0.010206847999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.0","version":"659'44","reported_seq":696,"reported_epoch":659,"state":"active+clean","last_fresh":"2023-09-23T21:19:44.169656+0000","last_change":"2023-09-23T21:19:42.907121+0000","last_active":"2023-09-23T21:19:44.169656+0000","last_peered":"2023-09-23T21:19:44.169656+0000","last_clean":"2023-09-23T21:19:44.169656+0000","last_became_active":"2023-09-23T21:16:42.452829+0000","last_became_peered":"2023-09-23T21:16:42.452829+0000","last_unstale":"2023-09-23T21:19:44.169656+0000","last_undegraded":"2023-09-23T21:19:44.169656+0000","last_fullsized":"2023-09-23T21:19:44.169656+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":44,"log_dups_size":0,"ondisk_log_size":44,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:49:26.303389+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.001223561,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":326,"num_read_kb":290,"num_write":31,"num_write_kb":23,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[{"start":"4","length":"2"}]},{"pgid":"3.1","version":"0'0","reported_seq":661,"reported_epoch":659,"state":"active+clean","last_fresh":"2023-09-23T21:19:42.905858+0000","last_change":"2023-09-23T21:13:44.409212+0000","last_active":"2023-09-23T21:19:42.905858+0000","last_peered":"2023-09-23T21:19:42.905858+0000","last_clean":"2023-09-23T21:19:42.905858+0000","last_became_active":"2023-09-23T21:13:44.408958+0000","last_became_peered":"2023-09-23T21:13:44.408958+0000","last_unstale":"2023-09-23T21:19:42.905858+0000","last_undegraded":"2023-09-23T21:19:42.905858+0000","last_fullsized":"2023-09-23T21:19:42.905858+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:03:52.749964+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":189,"reported_epoch":659,"state":"active+clean","last_fresh":"2023-09-23T21:19:42.906692+0000","last_change":"2023-09-23T21:16:42.427490+0000","last_active":"2023-09-23T21:19:42.906692+0000","last_peered":"2023-09-23T21:19:42.906692+0000","last_clean":"2023-09-23T21:19:42.906692+0000","last_became_active":"2023-09-23T21:16:42.426566+0000","last_became_peered":"2023-09-23T21:16:42.426566+0000","last_unstale":"2023-09-23T21:19:42.906692+0000","last_undegraded":"2023-09-23T21:19:42.906692+0000","last_fullsized":"2023-09-23T21:19:42.906692+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:20:26.396005+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,7],"acting":[0,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.6","version":"79'161","reported_seq":592,"reported_epoch":659,"state":"active+clean","last_fresh":"2023-09-23T21:19:42.906553+0000","last_change":"2023-09-23T21:15:43.333919+0000","last_active":"2023-09-23T21:19:42.906553+0000","last_peered":"2023-09-23T21:19:42.906553+0000","last_clean":"2023-09-23T21:19:42.906553+0000","last_became_active":"2023-09-23T21:13:33.769303+0000","last_became_peered":"2023-09-23T21:13:33.769303+0000","last_unstale":"2023-09-23T21:19:42.906553+0000","last_undegraded":"2023-09-23T21:19:42.906553+0000","last_fullsized":"2023-09-23T21:19:42.906553+0000","mapping_epoch":504,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":505,"parent":"0.0","parent_split_bits":3,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:15:43.333733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:15:43.333733+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:42:29.787357+0000","scrub_duration":0.0092107530000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"662'25","reported_seq":183,"reported_epoch":662,"state":"active+clean","last_fresh":"2023-09-23T21:19:46.937048+0000","last_change":"2023-09-23T21:19:42.909904+0000","last_active":"2023-09-23T21:19:46.937048+0000","last_peered":"2023-09-23T21:19:46.937048+0000","last_clean":"2023-09-23T21:19:46.937048+0000","last_became_active":"2023-09-23T21:16:42.482847+0000","last_became_peered":"2023-09-23T21:16:42.482847+0000","last_unstale":"2023-09-23T21:19:46.937048+0000","last_undegraded":"2023-09-23T21:19:46.937048+0000","last_fullsized":"2023-09-23T21:19:46.937048+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":25,"log_dups_size":0,"ondisk_log_size":25,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:38:57.905679+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00031864300000000002,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":125,"num_read_kb":119,"num_write":28,"num_write_kb":24,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[{"start":"4","length":"2"}]},{"pgid":"3.4","version":"0'0","reported_seq":555,"reported_epoch":660,"state":"active+clean","last_fresh":"2023-09-23T21:19:44.911000+0000","last_change":"2023-09-23T21:19:30.642771+0000","last_active":"2023-09-23T21:19:44.911000+0000","last_peered":"2023-09-23T21:19:44.911000+0000","last_clean":"2023-09-23T21:19:44.911000+0000","last_became_active":"2023-09-23T21:19:30.642356+0000","last_became_peered":"2023-09-23T21:19:30.642356+0000","last_unstale":"2023-09-23T21:19:44.911000+0000","last_undegraded":"2023-09-23T21:19:44.911000+0000","last_fullsized":"2023-09-23T21:19:44.911000+0000","mapping_epoch":650,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":651,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:54:42.234332+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,4],"acting":[2,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":635,"reported_epoch":659,"state":"active+clean","last_fresh":"2023-09-23T21:19:42.906504+0000","last_change":"2023-09-23T21:19:30.639797+0000","last_active":"2023-09-23T21:19:42.906504+0000","last_peered":"2023-09-23T21:19:42.906504+0000","last_clean":"2023-09-23T21:19:42.906504+0000","last_became_active":"2023-09-23T21:19:30.639468+0000","last_became_peered":"2023-09-23T21:19:30.639468+0000","last_unstale":"2023-09-23T21:19:42.906504+0000","last_undegraded":"2023-09-23T21:19:42.906504+0000","last_fullsized":"2023-09-23T21:19:42.906504+0000","mapping_epoch":650,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":651,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:50:42.845762+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,2],"acting":[0,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.5","version":"79'161","reported_seq":719,"reported_epoch":661,"state":"active+clean","last_fresh":"2023-09-23T21:19:45.917451+0000","last_change":"2023-09-23T21:19:33.639639+0000","last_active":"2023-09-23T21:19:45.917451+0000","last_peered":"2023-09-23T21:19:45.917451+0000","last_clean":"2023-09-23T21:19:45.917451+0000","last_became_active":"2023-09-23T21:19:33.639275+0000","last_became_peered":"2023-09-23T21:19:33.639275+0000","last_unstale":"2023-09-23T21:19:45.917451+0000","last_undegraded":"2023-09-23T21:19:45.917451+0000","last_fullsized":"2023-09-23T21:19:45.917451+0000","mapping_epoch":653,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":654,"parent":"0.0","parent_split_bits":3,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:15:41.108579+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:15:41.108579+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:15:48.675079+0000","scrub_duration":0.0061511960000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,0],"acting":[4,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":229,"reported_epoch":659,"state":"active+clean","last_fresh":"2023-09-23T21:19:42.906732+0000","last_change":"2023-09-23T21:17:02.204086+0000","last_active":"2023-09-23T21:19:42.906732+0000","last_peered":"2023-09-23T21:19:42.906732+0000","last_clean":"2023-09-23T21:19:42.906732+0000","last_became_active":"2023-09-23T21:16:42.517549+0000","last_became_peered":"2023-09-23T21:16:42.517549+0000","last_unstale":"2023-09-23T21:19:42.906732+0000","last_undegraded":"2023-09-23T21:19:42.906732+0000","last_fullsized":"2023-09-23T21:19:42.906732+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:43:07.991844+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.6","version":"662'134","reported_seq":1103,"reported_epoch":662,"state":"active+clean","last_fresh":"2023-09-23T21:19:46.925769+0000","last_change":"2023-09-23T21:19:42.906318+0000","last_active":"2023-09-23T21:19:46.925769+0000","last_peered":"2023-09-23T21:19:46.925769+0000","last_clean":"2023-09-23T21:19:46.925769+0000","last_became_active":"2023-09-23T21:13:30.807222+0000","last_became_peered":"2023-09-23T21:13:30.807222+0000","last_unstale":"2023-09-23T21:19:46.925769+0000","last_undegraded":"2023-09-23T21:19:46.925769+0000","last_fullsized":"2023-09-23T21:19:46.925769+0000","mapping_epoch":501,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":134,"log_dups_size":0,"ondisk_log_size":134,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:40:31.989749+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00020826500000000001,"stat_sum":{"num_bytes":112691,"num_objects":3,"num_object_clones":0,"num_object_copies":9,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3,"num_whiteouts":0,"num_read":879,"num_read_kb":1010,"num_write":109,"num_write_kb":217,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[{"start":"4","length":"2"}]},{"pgid":"4.0","version":"0'0","reported_seq":526,"reported_epoch":664,"state":"active+clean","last_fresh":"2023-09-23T21:19:48.936180+0000","last_change":"2023-09-23T21:16:42.463411+0000","last_active":"2023-09-23T21:19:48.936180+0000","last_peered":"2023-09-23T21:19:48.936180+0000","last_clean":"2023-09-23T21:19:48.936180+0000","last_became_active":"2023-09-23T21:16:42.463020+0000","last_became_peered":"2023-09-23T21:16:42.463020+0000","last_unstale":"2023-09-23T21:19:48.936180+0000","last_undegraded":"2023-09-23T21:19:48.936180+0000","last_fullsized":"2023-09-23T21:19:48.936180+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:07:34.346129+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.4","version":"79'161","reported_seq":1088,"reported_epoch":663,"state":"active+clean","last_fresh":"2023-09-23T21:19:47.935759+0000","last_change":"2023-09-23T21:19:33.644983+0000","last_active":"2023-09-23T21:19:47.935759+0000","last_peered":"2023-09-23T21:19:47.935759+0000","last_clean":"2023-09-23T21:19:47.935759+0000","last_became_active":"2023-09-23T21:19:33.644547+0000","last_became_peered":"2023-09-23T21:19:33.644547+0000","last_unstale":"2023-09-23T21:19:47.935759+0000","last_undegraded":"2023-09-23T21:19:47.935759+0000","last_fullsized":"2023-09-23T21:19:47.935759+0000","mapping_epoch":653,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":654,"parent":"0.0","parent_split_bits":3,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:20.012764+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:20.012764+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:45:31.294744+0000","scrub_duration":0.0045251919999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,4],"acting":[3,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"660'70","reported_seq":857,"reported_epoch":661,"state":"active+clean","last_fresh":"2023-09-23T21:19:45.919075+0000","last_change":"2023-09-23T21:19:42.912790+0000","last_active":"2023-09-23T21:19:45.919075+0000","last_peered":"2023-09-23T21:19:45.919075+0000","last_clean":"2023-09-23T21:19:45.919075+0000","last_became_active":"2023-09-23T21:16:42.459413+0000","last_became_peered":"2023-09-23T21:16:42.459413+0000","last_unstale":"2023-09-23T21:19:45.919075+0000","last_undegraded":"2023-09-23T21:19:45.919075+0000","last_fullsized":"2023-09-23T21:19:45.919075+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":70,"log_dups_size":0,"ondisk_log_size":70,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:37:44.548863+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00073354900000000001,"stat_sum":{"num_bytes":35,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":377,"num_read_kb":876,"num_write":56,"num_write_kb":297,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,3],"acting":[2,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[{"start":"4","length":"2"}]},{"pgid":"3.6","version":"70'1","reported_seq":661,"reported_epoch":659,"state":"active+clean","last_fresh":"2023-09-23T21:19:42.906943+0000","last_change":"2023-09-23T21:17:02.208077+0000","last_active":"2023-09-23T21:19:42.906943+0000","last_peered":"2023-09-23T21:19:42.906943+0000","last_clean":"2023-09-23T21:19:42.906943+0000","last_became_active":"2023-09-23T21:16:42.431308+0000","last_became_peered":"2023-09-23T21:16:42.431308+0000","last_unstale":"2023-09-23T21:19:42.906943+0000","last_undegraded":"2023-09-23T21:19:42.906943+0000","last_fullsized":"2023-09-23T21:19:42.906943+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:16:49.009013+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":246,"reported_epoch":661,"state":"active+clean","last_fresh":"2023-09-23T21:19:45.916867+0000","last_change":"2023-09-23T21:19:30.636392+0000","last_active":"2023-09-23T21:19:45.916867+0000","last_peered":"2023-09-23T21:19:45.916867+0000","last_clean":"2023-09-23T21:19:45.916867+0000","last_became_active":"2023-09-23T21:19:30.635999+0000","last_became_peered":"2023-09-23T21:19:30.635999+0000","last_unstale":"2023-09-23T21:19:45.916867+0000","last_undegraded":"2023-09-23T21:19:45.916867+0000","last_fullsized":"2023-09-23T21:19:45.916867+0000","mapping_epoch":650,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":651,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:33:20.969444+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,7],"acting":[4,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.19","version":"79'161","reported_seq":868,"reported_epoch":659,"state":"active+clean","last_fresh":"2023-09-23T21:19:42.905679+0000","last_change":"2023-09-23T21:16:45.428879+0000","last_active":"2023-09-23T21:19:42.905679+0000","last_peered":"2023-09-23T21:19:42.905679+0000","last_clean":"2023-09-23T21:19:42.905679+0000","last_became_active":"2023-09-23T21:16:45.428660+0000","last_became_peered":"2023-09-23T21:16:45.428660+0000","last_unstale":"2023-09-23T21:19:42.905679+0000","last_undegraded":"2023-09-23T21:19:42.905679+0000","last_fullsized":"2023-09-23T21:19:42.905679+0000","mapping_epoch":567,"log_start":"0'0","ondisk_log_start":"0'0","created":479,"last_epoch_clean":568,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:15:48.271791+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:15:48.271791+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:25:26.361045+0000","scrub_duration":0.0057149180000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":189,"reported_epoch":659,"state":"active+clean","last_fresh":"2023-09-23T21:19:42.906451+0000","last_change":"2023-09-23T21:16:42.434830+0000","last_active":"2023-09-23T21:19:42.906451+0000","last_peered":"2023-09-23T21:19:42.906451+0000","last_clean":"2023-09-23T21:19:42.906451+0000","last_became_active":"2023-09-23T21:16:42.430289+0000","last_became_peered":"2023-09-23T21:16:42.430289+0000","last_unstale":"2023-09-23T21:19:42.906451+0000","last_undegraded":"2023-09-23T21:19:42.906451+0000","last_fullsized":"2023-09-23T21:19:42.906451+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:38:48.053766+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":166,"reported_epoch":660,"state":"active+clean","last_fresh":"2023-09-23T21:19:44.910918+0000","last_change":"2023-09-23T21:13:38.373814+0000","last_active":"2023-09-23T21:19:44.910918+0000","last_peered":"2023-09-23T21:19:44.910918+0000","last_clean":"2023-09-23T21:19:44.910918+0000","last_became_active":"2023-09-23T21:13:38.373458+0000","last_became_peered":"2023-09-23T21:13:38.373458+0000","last_unstale":"2023-09-23T21:19:44.910918+0000","last_undegraded":"2023-09-23T21:19:44.910918+0000","last_fullsized":"2023-09-23T21:19:44.910918+0000","mapping_epoch":507,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:23:08.288401+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.1e","version":"79'161","reported_seq":648,"reported_epoch":663,"state":"active+clean","last_fresh":"2023-09-23T21:19:47.935107+0000","last_change":"2023-09-23T21:16:45.420788+0000","last_active":"2023-09-23T21:19:47.935107+0000","last_peered":"2023-09-23T21:19:47.935107+0000","last_clean":"2023-09-23T21:19:47.935107+0000","last_became_active":"2023-09-23T21:16:45.420448+0000","last_became_peered":"2023-09-23T21:16:45.420448+0000","last_unstale":"2023-09-23T21:19:47.935107+0000","last_undegraded":"2023-09-23T21:19:47.935107+0000","last_fullsized":"2023-09-23T21:19:47.935107+0000","mapping_epoch":567,"log_start":"0'0","ondisk_log_start":"0'0","created":479,"last_epoch_clean":568,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:15:53.061319+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:15:53.061319+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:46:31.982439+0000","scrub_duration":0.007429878,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,4],"acting":[3,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":285,"reported_epoch":661,"state":"active+clean","last_fresh":"2023-09-23T21:19:45.917328+0000","last_change":"2023-09-23T21:19:30.660286+0000","last_active":"2023-09-23T21:19:45.917328+0000","last_peered":"2023-09-23T21:19:45.917328+0000","last_clean":"2023-09-23T21:19:45.917328+0000","last_became_active":"2023-09-23T21:19:30.631029+0000","last_became_peered":"2023-09-23T21:19:30.631029+0000","last_unstale":"2023-09-23T21:19:45.917328+0000","last_undegraded":"2023-09-23T21:19:45.917328+0000","last_fullsized":"2023-09-23T21:19:45.917328+0000","mapping_epoch":650,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":651,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:13:01.408389+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,2],"acting":[4,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":226,"reported_epoch":661,"state":"active+clean","last_fresh":"2023-09-23T21:19:45.917167+0000","last_change":"2023-09-23T21:12:10.770474+0000","last_active":"2023-09-23T21:19:45.917167+0000","last_peered":"2023-09-23T21:19:45.917167+0000","last_clean":"2023-09-23T21:19:45.917167+0000","last_became_active":"2023-09-23T21:12:10.770184+0000","last_became_peered":"2023-09-23T21:12:10.770184+0000","last_unstale":"2023-09-23T21:19:45.917167+0000","last_undegraded":"2023-09-23T21:19:45.917167+0000","last_fullsized":"2023-09-23T21:19:45.917167+0000","mapping_epoch":470,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:05:09.407151+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":465,"reported_epoch":660,"state":"active+clean","last_fresh":"2023-09-23T21:19:44.910801+0000","last_change":"2023-09-23T21:19:30.651034+0000","last_active":"2023-09-23T21:19:44.910801+0000","last_peered":"2023-09-23T21:19:44.910801+0000","last_clean":"2023-09-23T21:19:44.910801+0000","last_became_active":"2023-09-23T21:19:30.634668+0000","last_became_peered":"2023-09-23T21:19:30.634668+0000","last_unstale":"2023-09-23T21:19:44.910801+0000","last_undegraded":"2023-09-23T21:19:44.910801+0000","last_fullsized":"2023-09-23T21:19:44.910801+0000","mapping_epoch":650,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":651,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:41:32.520798+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,7],"acting":[2,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":236,"reported_epoch":661,"state":"active+clean","last_fresh":"2023-09-23T21:19:45.916984+0000","last_change":"2023-09-23T21:13:12.216602+0000","last_active":"2023-09-23T21:19:45.916984+0000","last_peered":"2023-09-23T21:19:45.916984+0000","last_clean":"2023-09-23T21:19:45.916984+0000","last_became_active":"2023-09-23T21:13:12.216204+0000","last_became_peered":"2023-09-23T21:13:12.216204+0000","last_unstale":"2023-09-23T21:19:45.916984+0000","last_undegraded":"2023-09-23T21:19:45.916984+0000","last_fullsized":"2023-09-23T21:19:45.916984+0000","mapping_epoch":493,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:23:37.043390+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1c","version":"79'161","reported_seq":1111,"reported_epoch":663,"state":"active+clean","last_fresh":"2023-09-23T21:19:47.935580+0000","last_change":"2023-09-23T21:19:33.645432+0000","last_active":"2023-09-23T21:19:47.935580+0000","last_peered":"2023-09-23T21:19:47.935580+0000","last_clean":"2023-09-23T21:19:47.935580+0000","last_became_active":"2023-09-23T21:19:33.645042+0000","last_became_peered":"2023-09-23T21:19:33.645042+0000","last_unstale":"2023-09-23T21:19:47.935580+0000","last_undegraded":"2023-09-23T21:19:47.935580+0000","last_fullsized":"2023-09-23T21:19:47.935580+0000","mapping_epoch":653,"log_start":"0'0","ondisk_log_start":"0'0","created":479,"last_epoch_clean":654,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:15:54.019828+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:15:54.019828+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:45:29.799601+0000","scrub_duration":0.0020962530000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":270,"reported_epoch":660,"state":"active+clean","last_fresh":"2023-09-23T21:19:44.910909+0000","last_change":"2023-09-23T21:16:42.424187+0000","last_active":"2023-09-23T21:19:44.910909+0000","last_peered":"2023-09-23T21:19:44.910909+0000","last_clean":"2023-09-23T21:19:44.910909+0000","last_became_active":"2023-09-23T21:16:42.423805+0000","last_became_peered":"2023-09-23T21:16:42.423805+0000","last_unstale":"2023-09-23T21:19:44.910909+0000","last_undegraded":"2023-09-23T21:19:44.910909+0000","last_fullsized":"2023-09-23T21:19:44.910909+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:55:46.051199+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,7],"acting":[2,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":330,"reported_epoch":660,"state":"active+clean","last_fresh":"2023-09-23T21:19:44.910246+0000","last_change":"2023-09-23T21:13:12.200767+0000","last_active":"2023-09-23T21:19:44.910246+0000","last_peered":"2023-09-23T21:19:44.910246+0000","last_clean":"2023-09-23T21:19:44.910246+0000","last_became_active":"2023-09-23T21:13:12.200435+0000","last_became_peered":"2023-09-23T21:13:12.200435+0000","last_unstale":"2023-09-23T21:19:44.910246+0000","last_undegraded":"2023-09-23T21:19:44.910246+0000","last_fullsized":"2023-09-23T21:19:44.910246+0000","mapping_epoch":493,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:15:44.937886+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.1d","version":"79'161","reported_seq":809,"reported_epoch":663,"state":"active+clean","last_fresh":"2023-09-23T21:19:47.934600+0000","last_change":"2023-09-23T21:19:10.941440+0000","last_active":"2023-09-23T21:19:47.934600+0000","last_peered":"2023-09-23T21:19:47.934600+0000","last_clean":"2023-09-23T21:19:47.934600+0000","last_became_active":"2023-09-23T21:15:23.801704+0000","last_became_peered":"2023-09-23T21:15:23.801704+0000","last_unstale":"2023-09-23T21:19:47.934600+0000","last_undegraded":"2023-09-23T21:19:47.934600+0000","last_fullsized":"2023-09-23T21:19:47.934600+0000","mapping_epoch":542,"log_start":"0'0","ondisk_log_start":"0'0","created":479,"last_epoch_clean":543,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:19:10.941299+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:19:10.941299+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:31:22.166514+0000","scrub_duration":0.0018618599999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,2],"acting":[3,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":203,"reported_epoch":659,"state":"active+clean","last_fresh":"2023-09-23T21:19:42.906470+0000","last_change":"2023-09-23T21:19:30.664321+0000","last_active":"2023-09-23T21:19:42.906470+0000","last_peered":"2023-09-23T21:19:42.906470+0000","last_clean":"2023-09-23T21:19:42.906470+0000","last_became_active":"2023-09-23T21:19:30.646140+0000","last_became_peered":"2023-09-23T21:19:30.646140+0000","last_unstale":"2023-09-23T21:19:42.906470+0000","last_undegraded":"2023-09-23T21:19:42.906470+0000","last_fullsized":"2023-09-23T21:19:42.906470+0000","mapping_epoch":650,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":651,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:42:27.475154+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":13,"num_bytes_recovered":3878,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":236,"reported_epoch":661,"state":"active+clean","last_fresh":"2023-09-23T21:19:45.916852+0000","last_change":"2023-09-23T21:12:10.793610+0000","last_active":"2023-09-23T21:19:45.916852+0000","last_peered":"2023-09-23T21:19:45.916852+0000","last_clean":"2023-09-23T21:19:45.916852+0000","last_became_active":"2023-09-23T21:12:10.793081+0000","last_became_peered":"2023-09-23T21:12:10.793081+0000","last_unstale":"2023-09-23T21:19:45.916852+0000","last_undegraded":"2023-09-23T21:19:45.916852+0000","last_fullsized":"2023-09-23T21:19:45.916852+0000","mapping_epoch":470,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:34:36.291550+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":84,"num_bytes_recovered":18960,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":126976,"data_stored":21974,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":14322,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":31,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2375680,"data_stored":2361472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4991,"ondisk_log_size":4991,"up":93,"acting":93,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":225436,"num_objects":12,"num_object_clones":0,"num_object_copies":36,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":12,"num_whiteouts":0,"num_read":3249,"num_read_kb":4743,"num_write":478,"num_write_kb":1290,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":139264,"data_stored":51722,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":548,"ondisk_log_size":548,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201752,"num_pgs":65,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":659828,"kb_used_data":7076,"kb_used_omap":0,"kb_used_meta":652736,"kb_avail":93085324,"statfs":{"total":95995035648,"available":95319371776,"internally_reserved":0,"allocated":7245824,"data_stored":5474340,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":668401664},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:19:23 2023","interfaces":[{"interface":"back","average":{"1min":0.77800000000000002,"5min":1.145,"15min":1.9990000000000001},"min":{"1min":0.23599999999999999,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":3.448,"5min":5.1790000000000003,"15min":37.207999999999998},"last":0.39300000000000002},{"interface":"front","average":{"1min":0.86699999999999999,"5min":1.474,"15min":1.98},"min":{"1min":0.222,"5min":0.222,"15min":0.222},"max":{"1min":3.6930000000000001,"5min":10.429,"15min":37.122},"last":0.54700000000000004}]},{"osd":1,"last update":"Sat Sep 23 21:19:18 2023","interfaces":[{"interface":"back","average":{"1min":0.80200000000000005,"5min":1.1930000000000001,"15min":1.704},"min":{"1min":0.29399999999999998,"5min":0.27600000000000002,"15min":0.26700000000000002},"max":{"1min":5.2619999999999996,"5min":6.9130000000000003,"15min":14.034000000000001},"last":0.436},{"interface":"front","average":{"1min":0.84199999999999997,"5min":1.3380000000000001,"15min":1.6930000000000001},"min":{"1min":0.36599999999999999,"5min":0.29999999999999999,"15min":0.26300000000000001},"max":{"1min":2.512,"5min":11.002000000000001,"15min":11.226000000000001},"last":0.56200000000000006}]},{"osd":2,"last update":"Sat Sep 23 21:19:39 2023","interfaces":[{"interface":"back","average":{"1min":10.231,"5min":3.9590000000000001,"15min":2.3860000000000001},"min":{"1min":0.28899999999999998,"5min":0.27200000000000002,"15min":0.26200000000000001},"max":{"1min":166.696,"5min":166.696,"15min":166.696},"last":0.377},{"interface":"front","average":{"1min":10.039999999999999,"5min":3.6440000000000001,"15min":2.3210000000000002},"min":{"1min":0.316,"5min":0.28799999999999998,"15min":0.26700000000000002},"max":{"1min":165.57499999999999,"5min":165.57499999999999,"15min":165.57499999999999},"last":0.505}]},{"osd":3,"last update":"Sat Sep 23 21:19:06 2023","interfaces":[{"interface":"back","average":{"1min":1.6220000000000001,"5min":1.6040000000000001,"15min":1.788},"min":{"1min":0.317,"5min":0.30599999999999999,"15min":0.30099999999999999},"max":{"1min":5.7830000000000004,"5min":8.6259999999999994,"15min":12.359999999999999},"last":0.58499999999999996},{"interface":"front","average":{"1min":1.504,"5min":1.8100000000000001,"15min":1.79},"min":{"1min":0.33400000000000002,"5min":0.28799999999999998,"15min":0.28699999999999998},"max":{"1min":4.6769999999999996,"5min":15.82,"15min":15.82},"last":0.41799999999999998}]},{"osd":4,"last update":"Sat Sep 23 21:19:06 2023","interfaces":[{"interface":"back","average":{"1min":0.42199999999999999,"5min":0.78500000000000003,"15min":1.1160000000000001},"min":{"1min":0.27000000000000002,"5min":0.251,"15min":0.248},"max":{"1min":0.92700000000000005,"5min":4.2130000000000001,"15min":11.297000000000001},"last":0.34399999999999997},{"interface":"front","average":{"1min":0.61299999999999999,"5min":0.91800000000000004,"15min":1.0800000000000001},"min":{"1min":0.26800000000000002,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":3.73,"5min":7.1870000000000003,"15min":12.752000000000001},"last":0.59799999999999998}]},{"osd":5,"last update":"Sat Sep 23 21:19:06 2023","interfaces":[{"interface":"back","average":{"1min":0.57199999999999995,"5min":0.91600000000000004,"15min":1.0640000000000001},"min":{"1min":0.28199999999999997,"5min":0.28199999999999997,"15min":0.26000000000000001},"max":{"1min":1.7150000000000001,"5min":6.9740000000000002,"15min":9.4789999999999992},"last":0.52100000000000002},{"interface":"front","average":{"1min":0.65900000000000003,"5min":1.004,"15min":1.157},"min":{"1min":0.32300000000000001,"5min":0.32000000000000001,"15min":0.26200000000000001},"max":{"1min":3.1349999999999998,"5min":6.7439999999999998,"15min":11.257},"last":0.60999999999999999}]},{"osd":6,"last update":"Sat Sep 23 21:19:18 2023","interfaces":[{"interface":"back","average":{"1min":0.72299999999999998,"5min":0.86899999999999999,"15min":1.0269999999999999},"min":{"1min":0.312,"5min":0.312,"15min":0.30099999999999999},"max":{"1min":3.8170000000000002,"5min":7.5339999999999998,"15min":10.941000000000001},"last":0.622},{"interface":"front","average":{"1min":0.80800000000000005,"5min":1.006,"15min":1.145},"min":{"1min":0.35099999999999998,"5min":0.35099999999999998,"15min":0.33100000000000002},"max":{"1min":3.7559999999999998,"5min":5.6760000000000002,"15min":12.715},"last":0.67000000000000004}]}]},{"osd":6,"up_from":0,"seq":704374636836,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":1,"up_from":0,"seq":1503238553768,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":0,"up_from":493,"seq":2117418877008,"num_pgs":63,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":434168,"kb_used_data":8740,"kb_used_omap":36,"kb_used_meta":425371,"kb_avail":93310984,"statfs":{"total":95995035648,"available":95550447616,"internally_reserved":0,"allocated":8949760,"data_stored":5960434,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":37336,"internal_metadata":435580456},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 21:19:30 2023","interfaces":[{"interface":"back","average":{"1min":0.73799999999999999,"5min":1.4370000000000001,"15min":0.98299999999999998},"min":{"1min":0.23000000000000001,"5min":0.214,"15min":0.214},"max":{"1min":3.8610000000000002,"5min":13.779999999999999,"15min":13.779999999999999},"last":0.30199999999999999},{"interface":"front","average":{"1min":0.98099999999999998,"5min":1.7989999999999999,"15min":1.113},"min":{"1min":0.23300000000000001,"5min":0.20799999999999999,"15min":0.20799999999999999},"max":{"1min":3.0539999999999998,"5min":11.964,"15min":11.964},"last":0.46999999999999997}]},{"osd":2,"last update":"Sat Sep 23 21:19:30 2023","interfaces":[{"interface":"back","average":{"1min":2.8300000000000001,"5min":2.2360000000000002,"15min":1.181},"min":{"1min":0.25800000000000001,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":35.68,"5min":35.68,"15min":35.68},"last":0.372},{"interface":"front","average":{"1min":2.8889999999999998,"5min":2.8450000000000002,"15min":1.593},"min":{"1min":0.28499999999999998,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":35.832999999999998,"5min":35.832999999999998,"15min":35.832999999999998},"last":0.44400000000000001}]},{"osd":3,"last update":"Sat Sep 23 21:19:30 2023","interfaces":[{"interface":"back","average":{"1min":1.891,"5min":2.8039999999999998,"15min":1.5129999999999999},"min":{"1min":0.247,"5min":0.247,"15min":0.23400000000000001},"max":{"1min":8.3089999999999993,"5min":37.079999999999998,"15min":37.079999999999998},"last":0.38600000000000001},{"interface":"front","average":{"1min":0.98299999999999998,"5min":2.1760000000000002,"15min":1.47},"min":{"1min":0.28899999999999998,"5min":0.28299999999999997,"15min":0.26200000000000001},"max":{"1min":3.3069999999999999,"5min":32.935000000000002,"15min":32.935000000000002},"last":0.41099999999999998}]},{"osd":4,"last update":"Sat Sep 23 21:19:30 2023","interfaces":[{"interface":"back","average":{"1min":0.89300000000000002,"5min":1.6990000000000001,"15min":1.2949999999999999},"min":{"1min":0.34200000000000003,"5min":0.312,"15min":0.30199999999999999},"max":{"1min":2.4209999999999998,"5min":22.780000000000001,"15min":22.780000000000001},"last":0.54500000000000004},{"interface":"front","average":{"1min":0.98499999999999999,"5min":1.496,"15min":1.1619999999999999},"min":{"1min":0.33700000000000002,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":3.8199999999999998,"5min":22.652000000000001,"15min":22.652000000000001},"last":0.433}]},{"osd":5,"last update":"Sat Sep 23 21:19:30 2023","interfaces":[{"interface":"back","average":{"1min":0.875,"5min":1.4950000000000001,"15min":1.3220000000000001},"min":{"1min":0.33200000000000002,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":3.0950000000000002,"5min":8.9329999999999998,"15min":8.9329999999999998},"last":0.52100000000000002},{"interface":"front","average":{"1min":0.91200000000000003,"5min":1.6659999999999999,"15min":1.264},"min":{"1min":0.377,"5min":0.313,"15min":0.313},"max":{"1min":2.4750000000000001,"5min":15.321999999999999,"15min":15.321999999999999},"last":0.50600000000000001}]},{"osd":6,"last update":"Sat Sep 23 21:19:30 2023","interfaces":[{"interface":"back","average":{"1min":1.042,"5min":1.464,"15min":1.2390000000000001},"min":{"1min":0.432,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":3.8999999999999999,"5min":7.2050000000000001,"15min":7.2050000000000001},"last":0.47899999999999998},{"interface":"front","average":{"1min":0.872,"5min":1.508,"15min":1.3740000000000001},"min":{"1min":0.35099999999999998,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":3.125,"5min":8.8889999999999993,"15min":8.8889999999999993},"last":0.498}]},{"osd":7,"last update":"Sat Sep 23 21:19:30 2023","interfaces":[{"interface":"back","average":{"1min":0.93799999999999994,"5min":1.77,"15min":1.3959999999999999},"min":{"1min":0.39700000000000002,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":2.4449999999999998,"5min":15.273999999999999,"15min":15.273999999999999},"last":0.46000000000000002},{"interface":"front","average":{"1min":1.0489999999999999,"5min":1.7490000000000001,"15min":1.355},"min":{"1min":0.40300000000000002,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":3.9390000000000001,"5min":33.779000000000003,"15min":33.779000000000003},"last":0.52900000000000003}]}]},{"osd":2,"up_from":93,"seq":399431958880,"num_pgs":53,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1040232,"kb_used_data":8472,"kb_used_omap":81,"kb_used_meta":1031662,"kb_avail":92704920,"statfs":{"total":95995035648,"available":94929838080,"internally_reserved":0,"allocated":8675328,"data_stored":5634290,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":83750,"internal_metadata":1056422106},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:19:26 2023","interfaces":[{"interface":"back","average":{"1min":1.7470000000000001,"5min":2.1200000000000001,"15min":1.262},"min":{"1min":0.20399999999999999,"5min":0.187,"15min":0.187},"max":{"1min":9.2799999999999994,"5min":13.25,"15min":13.25},"last":0.28399999999999997},{"interface":"front","average":{"1min":1.4650000000000001,"5min":2.0489999999999999,"15min":1.7929999999999999},"min":{"1min":0.29599999999999999,"5min":0.20200000000000001,"15min":0.20200000000000001},"max":{"1min":5.649,"5min":11.269,"15min":18.111999999999998},"last":0.25900000000000001}]},{"osd":1,"last update":"Sat Sep 23 21:19:26 2023","interfaces":[{"interface":"back","average":{"1min":2.0419999999999998,"5min":1.9750000000000001,"15min":2.2000000000000002},"min":{"1min":0.25800000000000001,"5min":0.20599999999999999,"15min":0.188},"max":{"1min":7.4939999999999998,"5min":10.535,"15min":34.396999999999998},"last":0.379},{"interface":"front","average":{"1min":1.1439999999999999,"5min":1.8600000000000001,"15min":2.1779999999999999},"min":{"1min":0.34999999999999998,"5min":0.24099999999999999,"15min":0.217},"max":{"1min":4.9560000000000004,"5min":16.274000000000001,"15min":34.462000000000003},"last":0.30499999999999999}]},{"osd":3,"last update":"Sat Sep 23 21:19:02 2023","interfaces":[{"interface":"back","average":{"1min":2.2949999999999999,"5min":2.3999999999999999,"15min":2.149},"min":{"1min":0.27300000000000002,"5min":0.248,"15min":0.20599999999999999},"max":{"1min":8.141,"5min":20.106999999999999,"15min":35.549999999999997},"last":0.37},{"interface":"front","average":{"1min":2.593,"5min":2.9249999999999998,"15min":2.4609999999999999},"min":{"1min":0.34899999999999998,"5min":0.28599999999999998,"15min":0.21299999999999999},"max":{"1min":8.2469999999999999,"5min":20.239000000000001,"15min":36.951999999999998},"last":0.42199999999999999}]},{"osd":4,"last update":"Sat Sep 23 21:19:26 2023","interfaces":[{"interface":"back","average":{"1min":0.79200000000000004,"5min":1.375,"15min":1.4039999999999999},"min":{"1min":0.30599999999999999,"5min":0.27000000000000002,"15min":0.249},"max":{"1min":3.1880000000000002,"5min":7.9770000000000003,"15min":10.266999999999999},"last":0.435},{"interface":"front","average":{"1min":1.3149999999999999,"5min":1.6259999999999999,"15min":1.645},"min":{"1min":0.375,"5min":0.30299999999999999,"15min":0.26700000000000002},"max":{"1min":7.601,"5min":10.459,"15min":12.49},"last":0.47899999999999998}]},{"osd":5,"last update":"Sat Sep 23 21:19:26 2023","interfaces":[{"interface":"back","average":{"1min":5.6959999999999997,"5min":2.407,"15min":1.827},"min":{"1min":0.36699999999999999,"5min":0.29099999999999998,"15min":0.28000000000000003},"max":{"1min":102.56100000000001,"5min":102.56100000000001,"15min":102.56100000000001},"last":0.40899999999999997},{"interface":"front","average":{"1min":6.4020000000000001,"5min":2.6299999999999999,"15min":1.954},"min":{"1min":0.46000000000000002,"5min":0.33800000000000002,"15min":0.29399999999999998},"max":{"1min":102.28700000000001,"5min":102.28700000000001,"15min":102.28700000000001},"last":0.45900000000000002}]},{"osd":6,"last update":"Sat Sep 23 21:19:26 2023","interfaces":[{"interface":"back","average":{"1min":1.9430000000000001,"5min":1.831,"15min":1.8640000000000001},"min":{"1min":0.44,"5min":0.377,"15min":0.35699999999999998},"max":{"1min":9.8680000000000003,"5min":10.335000000000001,"15min":12.518000000000001},"last":0.51800000000000002},{"interface":"front","average":{"1min":1.5429999999999999,"5min":1.756,"15min":1.9930000000000001},"min":{"1min":0.443,"5min":0.377,"15min":0.35599999999999998},"max":{"1min":8.4149999999999991,"5min":10.381,"15min":12.544},"last":0.55300000000000005}]},{"osd":7,"last update":"Sat Sep 23 21:19:26 2023","interfaces":[{"interface":"back","average":{"1min":1.0900000000000001,"5min":1.4930000000000001,"15min":1.6359999999999999},"min":{"1min":0.42199999999999999,"5min":0.31,"15min":0.29799999999999999},"max":{"1min":3.1190000000000002,"5min":7.5819999999999999,"15min":18.021999999999998},"last":0.48499999999999999},{"interface":"front","average":{"1min":0.83099999999999996,"5min":1.5509999999999999,"15min":1.6879999999999999},"min":{"1min":0.372,"5min":0.34999999999999998,"15min":0.32200000000000001},"max":{"1min":3.222,"5min":7.46,"15min":10.92},"last":0.45200000000000001}]}]},{"osd":3,"up_from":98,"seq":420906795354,"num_pgs":69,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1138324,"kb_used_data":8524,"kb_used_omap":7,"kb_used_meta":1129784,"kb_avail":92606828,"statfs":{"total":95995035648,"available":94829391872,"internally_reserved":0,"allocated":8728576,"data_stored":5733741,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7576,"internal_metadata":1156899432},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:19:18 2023","interfaces":[{"interface":"back","average":{"1min":1.3009999999999999,"5min":4.2670000000000003,"15min":1.647},"min":{"1min":0.20399999999999999,"5min":0.18099999999999999,"15min":0.18099999999999999},"max":{"1min":9.3819999999999997,"5min":147.38900000000001,"15min":147.38900000000001},"last":1.097},{"interface":"front","average":{"1min":0.93899999999999995,"5min":3.6400000000000001,"15min":1.6579999999999999},"min":{"1min":0.214,"5min":0.214,"15min":0.214},"max":{"1min":5.2110000000000003,"5min":149.989,"15min":149.989},"last":0.443}]},{"osd":1,"last update":"Sat Sep 23 21:19:18 2023","interfaces":[{"interface":"back","average":{"1min":1.4710000000000001,"5min":1.6779999999999999,"15min":2.0720000000000001},"min":{"1min":0.248,"5min":0.23699999999999999,"15min":0.20000000000000001},"max":{"1min":11.146000000000001,"5min":11.146000000000001,"15min":22.577999999999999},"last":0.95299999999999996},{"interface":"front","average":{"1min":0.79100000000000004,"5min":1.335,"15min":1.8879999999999999},"min":{"1min":0.28899999999999998,"5min":0.255,"15min":0.216},"max":{"1min":2.7799999999999998,"5min":7.6829999999999998,"15min":21.617999999999999},"last":0.73799999999999999}]},{"osd":2,"last update":"Sat Sep 23 21:19:03 2023","interfaces":[{"interface":"back","average":{"1min":0.84999999999999998,"5min":1.395,"15min":1.893},"min":{"1min":0.23799999999999999,"5min":0.23799999999999999,"15min":0.20899999999999999},"max":{"1min":3.1000000000000001,"5min":11.321,"15min":56.317999999999998},"last":0.65100000000000002},{"interface":"front","average":{"1min":3.1120000000000001,"5min":2.331,"15min":2.343},"min":{"1min":0.34899999999999998,"5min":0.23400000000000001,"15min":0.216},"max":{"1min":11.177,"5min":17.423999999999999,"15min":55.988},"last":0.51000000000000001}]},{"osd":4,"last update":"Sat Sep 23 21:19:03 2023","interfaces":[{"interface":"back","average":{"1min":1.0609999999999999,"5min":1.27,"15min":1.472},"min":{"1min":0.36899999999999999,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":5.1589999999999998,"5min":10.807,"15min":21.478999999999999},"last":0.77400000000000002},{"interface":"front","average":{"1min":2.145,"5min":1.4870000000000001,"15min":1.7549999999999999},"min":{"1min":0.39600000000000002,"5min":0.311,"15min":0.28499999999999998},"max":{"1min":11.202,"5min":11.202,"15min":22.402000000000001},"last":0.57799999999999996}]},{"osd":5,"last update":"Sat Sep 23 21:19:03 2023","interfaces":[{"interface":"back","average":{"1min":0.91400000000000003,"5min":1.3109999999999999,"15min":1.6040000000000001},"min":{"1min":0.34999999999999998,"5min":0.313,"15min":0.311},"max":{"1min":3.109,"5min":10.882999999999999,"15min":21.562000000000001},"last":0.72799999999999998},{"interface":"front","average":{"1min":0.90300000000000002,"5min":1.2470000000000001,"15min":1.679},"min":{"1min":0.30199999999999999,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":2.8010000000000002,"5min":5.7300000000000004,"15min":14.029},"last":0.66500000000000004}]},{"osd":6,"last update":"Sat Sep 23 21:19:18 2023","interfaces":[{"interface":"back","average":{"1min":1.605,"5min":1.534,"15min":1.794},"min":{"1min":0.44500000000000001,"5min":0.35899999999999999,"15min":0.34599999999999997},"max":{"1min":11.092000000000001,"5min":11.092000000000001,"15min":22.446999999999999},"last":0.61199999999999999},{"interface":"front","average":{"1min":0.80000000000000004,"5min":1.2589999999999999,"15min":1.7350000000000001},"min":{"1min":0.34300000000000003,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":2.839,"5min":7.5949999999999998,"15min":12.375999999999999},"last":0.70499999999999996}]},{"osd":7,"last update":"Sat Sep 23 21:19:03 2023","interfaces":[{"interface":"back","average":{"1min":0.91100000000000003,"5min":1.456,"15min":1.74},"min":{"1min":0.41099999999999998,"5min":0.41099999999999998,"15min":0.33900000000000002},"max":{"1min":2.4279999999999999,"5min":10.914999999999999,"15min":21.518999999999998},"last":0.69499999999999995},{"interface":"front","average":{"1min":1.2370000000000001,"5min":1.589,"15min":1.9059999999999999},"min":{"1min":0.38800000000000001,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":3.8130000000000002,"5min":8.2590000000000003,"15min":12.624000000000001},"last":0.79300000000000004}]}]},{"osd":4,"up_from":31,"seq":133143986660,"num_pgs":59,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":594868,"kb_used_data":7148,"kb_used_omap":0,"kb_used_meta":587712,"kb_avail":93150284,"statfs":{"total":95995035648,"available":95385890816,"internally_reserved":0,"allocated":7319552,"data_stored":5550376,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":601817088},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:19:24 2023","interfaces":[{"interface":"back","average":{"1min":0.73799999999999999,"5min":1.641,"15min":1.147},"min":{"1min":0.24299999999999999,"5min":0.22800000000000001,"15min":0.22500000000000001},"max":{"1min":4.4429999999999996,"5min":28.683,"15min":28.683},"last":0.307},{"interface":"front","average":{"1min":1.038,"5min":1.782,"15min":1.3500000000000001},"min":{"1min":0.26000000000000001,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":6.5490000000000004,"5min":29.155000000000001,"15min":29.155000000000001},"last":0.49199999999999999}]},{"osd":1,"last update":"Sat Sep 23 21:19:24 2023","interfaces":[{"interface":"back","average":{"1min":0.70099999999999996,"5min":1.2689999999999999,"15min":1.899},"min":{"1min":0.31,"5min":0.26900000000000002,"15min":0.221},"max":{"1min":2.964,"5min":8.1020000000000003,"15min":37.960000000000001},"last":0.42799999999999999},{"interface":"front","average":{"1min":0.75800000000000001,"5min":1.3899999999999999,"15min":1.8640000000000001},"min":{"1min":0.35799999999999998,"5min":0.30299999999999999,"15min":0.27000000000000002},"max":{"1min":2.976,"5min":13.983000000000001,"15min":37.933},"last":0.44}]},{"osd":2,"last update":"Sat Sep 23 21:19:24 2023","interfaces":[{"interface":"back","average":{"1min":1.556,"5min":1.9970000000000001,"15min":1.6890000000000001},"min":{"1min":0.29099999999999998,"5min":0.26400000000000001,"15min":0.247},"max":{"1min":13.791,"5min":17.602,"15min":17.602},"last":0.33900000000000002},{"interface":"front","average":{"1min":1.6359999999999999,"5min":1.9399999999999999,"15min":1.7649999999999999},"min":{"1min":0.28799999999999998,"5min":0.28799999999999998,"15min":0.27800000000000002},"max":{"1min":13.439,"5min":14.339,"15min":14.417999999999999},"last":0.47499999999999998}]},{"osd":3,"last update":"Sat Sep 23 21:18:55 2023","interfaces":[{"interface":"back","average":{"1min":1.3129999999999999,"5min":2.1579999999999999,"15min":1.9790000000000001},"min":{"1min":0.29499999999999998,"5min":0.27900000000000003,"15min":0.25600000000000001},"max":{"1min":6.1230000000000002,"5min":67.105999999999995,"15min":67.105999999999995},"last":0.46400000000000002},{"interface":"front","average":{"1min":1.5289999999999999,"5min":2.2719999999999998,"15min":1.966},"min":{"1min":0.40400000000000003,"5min":0.32200000000000001,"15min":0.30199999999999999},"max":{"1min":6.1319999999999997,"5min":67.046999999999997,"15min":67.046999999999997},"last":0.52300000000000002}]},{"osd":5,"last update":"Sat Sep 23 21:19:17 2023","interfaces":[{"interface":"back","average":{"1min":0.39400000000000002,"5min":0.95299999999999996,"15min":1.0700000000000001},"min":{"1min":0.26900000000000002,"5min":0.254,"15min":0.222},"max":{"1min":0.73999999999999999,"5min":14.398999999999999,"15min":14.398999999999999},"last":0.40699999999999997},{"interface":"front","average":{"1min":0.41699999999999998,"5min":0.86699999999999999,"15min":1.0589999999999999},"min":{"1min":0.29699999999999999,"5min":0.25800000000000001,"15min":0.255},"max":{"1min":0.57699999999999996,"5min":8.016,"15min":9.8979999999999997},"last":0.41599999999999998}]},{"osd":6,"last update":"Sat Sep 23 21:19:24 2023","interfaces":[{"interface":"back","average":{"1min":0.439,"5min":0.90400000000000003,"15min":1.167},"min":{"1min":0.311,"5min":0.27200000000000002,"15min":0.26800000000000002},"max":{"1min":0.62,"5min":8.7420000000000009,"15min":24.452999999999999},"last":0.39100000000000001},{"interface":"front","average":{"1min":0.55800000000000005,"5min":1.071,"15min":1.0600000000000001},"min":{"1min":0.42199999999999999,"5min":0.36399999999999999,"15min":0.32500000000000001},"max":{"1min":0.79700000000000004,"5min":14.374000000000001,"15min":14.387},"last":0.505}]},{"osd":7,"last update":"Sat Sep 23 21:18:55 2023","interfaces":[{"interface":"back","average":{"1min":0.70599999999999996,"5min":1.077,"15min":1.1659999999999999},"min":{"1min":0.33800000000000002,"5min":0.32000000000000001,"15min":0.30099999999999999},"max":{"1min":2.6419999999999999,"5min":14.422000000000001,"15min":16.890999999999998},"last":0.53900000000000003},{"interface":"front","average":{"1min":0.70999999999999996,"5min":0.89800000000000002,"15min":1.179},"min":{"1min":0.32900000000000001,"5min":0.29199999999999998,"15min":0.27400000000000002},"max":{"1min":2.6579999999999999,"5min":7.7229999999999999,"15min":11.170999999999999},"last":0.45400000000000001}]}]},{"osd":5,"up_from":0,"seq":167503725026,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":10310,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":10361,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":49152,"data_stored":20636,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":10291,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":124,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8488,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1259,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5322,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":32768,"data_stored":5504,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":4838,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T21:19:50.788 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph osd pool set .mgr pgp_num 51 2023-09-23T21:19:50.949 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:19:50 smithi026 ceph-mon[179688]: failed to encode map e665 with expected crc 2023-09-23T21:19:50.949 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:19:50 smithi026 ceph-mon[179688]: osdmap e665: 8 total, 8 up, 5 in 2023-09-23T21:19:50.949 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:19:50 smithi026 ceph-mon[179688]: failed to encode map e665 with expected crc 2023-09-23T21:19:50.949 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:19:50 smithi026 ceph-mon[179688]: from='client.47519 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T21:19:50.949 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:19:50 smithi026 ceph-mon[179688]: osdmap e666: 8 total, 8 up, 5 in 2023-09-23T21:19:50.949 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:19:50 smithi026 ceph-mon[175151]: failed to encode map e665 with expected crc 2023-09-23T21:19:50.950 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:19:50 smithi026 ceph-mon[175151]: osdmap e665: 8 total, 8 up, 5 in 2023-09-23T21:19:50.950 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:19:50 smithi026 ceph-mon[175151]: failed to encode map e665 with expected crc 2023-09-23T21:19:50.950 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:19:50 smithi026 ceph-mon[175151]: from='client.47519 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T21:19:50.950 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:19:50 smithi026 ceph-mon[175151]: osdmap e666: 8 total, 8 up, 5 in 2023-09-23T21:19:51.150 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:19:51.198 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:19:51.201 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:51.201 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:19:51.302 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:19:51.337 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:51.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:19:50 smithi070 ceph-mon[168282]: failed to encode map e665 with expected crc 2023-09-23T21:19:51.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:19:50 smithi070 ceph-mon[168282]: osdmap e665: 8 total, 8 up, 5 in 2023-09-23T21:19:51.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:19:50 smithi070 ceph-mon[168282]: failed to encode map e665 with expected crc 2023-09-23T21:19:51.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:19:50 smithi070 ceph-mon[168282]: from='client.47519 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T21:19:51.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:19:50 smithi070 ceph-mon[168282]: osdmap e666: 8 total, 8 up, 5 in 2023-09-23T21:19:51.476 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:19:51.478 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:51.479 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:19:51.580 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:19:51.611 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:51.811 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:19:51.814 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:51.815 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:19:51.914 INFO:tasks.workunit.client.0.smithi026.stderr: Removing all snapshots: 50% complete... Removing all snapshots: 100% complete... Removing all snapshots: 100% complete...done. 2023-09-23T21:19:51.916 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:19:51.926 INFO:tasks.workunit.client.0.smithi026.stderr:+ rbd remove testimg 2023-09-23T21:19:51.974 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:52.047 INFO:tasks.workunit.client.0.smithi026.stderr: Removing image: 100% complete...done. 2023-09-23T21:19:52.051 INFO:tasks.workunit.client.0.smithi026.stderr:+ rbd remove testimg_import 2023-09-23T21:19:52.118 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:19:52.121 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:19:51 smithi026 ceph-mon[179688]: failed to encode map e666 with expected crc 2023-09-23T21:19:52.121 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:19:51 smithi026 ceph-mon[179688]: failed to encode map e666 with expected crc 2023-09-23T21:19:52.121 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:19:51 smithi026 ceph-mon[179688]: failed to encode map e666 with expected crc 2023-09-23T21:19:52.121 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:19:51 smithi026 ceph-mon[179688]: pgmap v1627: 103 pgs: 103 active+clean; 802 KiB data, 3.7 GiB used, 443 GiB / 447 GiB avail; 23 KiB/s rd, 2.0 KiB/s wr, 30 op/s 2023-09-23T21:19:52.121 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:19:51 smithi026 ceph-mon[179688]: osdmap e667: 8 total, 8 up, 5 in 2023-09-23T21:19:52.122 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:52.122 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:19:52.123 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:19:51 smithi026 ceph-mon[175151]: failed to encode map e666 with expected crc 2023-09-23T21:19:52.123 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:19:51 smithi026 ceph-mon[175151]: failed to encode map e666 with expected crc 2023-09-23T21:19:52.123 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:19:51 smithi026 ceph-mon[175151]: failed to encode map e666 with expected crc 2023-09-23T21:19:52.123 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:19:51 smithi026 ceph-mon[175151]: pgmap v1627: 103 pgs: 103 active+clean; 802 KiB data, 3.7 GiB used, 443 GiB / 447 GiB avail; 23 KiB/s rd, 2.0 KiB/s wr, 30 op/s 2023-09-23T21:19:52.123 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:19:51 smithi026 ceph-mon[175151]: osdmap e667: 8 total, 8 up, 5 in 2023-09-23T21:19:52.150 INFO:tasks.workunit.client.0.smithi026.stderr: Removing image: 100% complete...done. 2023-09-23T21:19:52.154 INFO:tasks.workunit.client.0.smithi026.stderr:+ tiered=0 2023-09-23T21:19:52.154 INFO:tasks.workunit.client.0.smithi026.stderr:+ ceph osd dump 2023-09-23T21:19:52.154 INFO:tasks.workunit.client.0.smithi026.stderr:+ grep '^pool' 2023-09-23T21:19:52.154 INFO:tasks.workunit.client.0.smithi026.stderr:+ grep ''\''rbd'\''' 2023-09-23T21:19:52.154 INFO:tasks.workunit.client.0.smithi026.stderr:+ grep tier 2023-09-23T21:19:52.210 INFO:teuthology.orchestra.run.smithi026.stdout:{ 2023-09-23T21:19:52.211 INFO:teuthology.orchestra.run.smithi026.stdout: "ops": [], 2023-09-23T21:19:52.211 INFO:teuthology.orchestra.run.smithi026.stdout: "num_ops": 0 2023-09-23T21:19:52.211 INFO:teuthology.orchestra.run.smithi026.stdout:} 2023-09-23T21:19:52.223 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:19:52.249 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:19:51 smithi070 ceph-mon[168282]: failed to encode map e666 with expected crc 2023-09-23T21:19:52.249 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:19:51 smithi070 ceph-mon[168282]: failed to encode map e666 with expected crc 2023-09-23T21:19:52.249 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:19:51 smithi070 ceph-mon[168282]: failed to encode map e666 with expected crc 2023-09-23T21:19:52.249 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:19:51 smithi070 ceph-mon[168282]: pgmap v1627: 103 pgs: 103 active+clean; 802 KiB data, 3.7 GiB used, 443 GiB / 447 GiB avail; 23 KiB/s rd, 2.0 KiB/s wr, 30 op/s 2023-09-23T21:19:52.249 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:19:51 smithi070 ceph-mon[168282]: osdmap e667: 8 total, 8 up, 5 in 2023-09-23T21:19:52.251 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:52.285 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:19:52.287 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:52.288 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:19:52.389 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:19:52.421 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:52.456 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T21:19:52.459 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:52.459 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:19:52.560 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:19:52.587 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:52.680 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:19:52.722 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T21:19:52.725 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:52.725 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:19:52.741 INFO:tasks.workunit.client.0.smithi026.stderr:+ dd if=/dev/urandom bs=1M seek=1 count=1 of=/tmp/rbd_import_export_456745/sparse1 2023-09-23T21:19:52.747 INFO:tasks.workunit.client.0.smithi026.stderr:1+0 records in 2023-09-23T21:19:52.747 INFO:tasks.workunit.client.0.smithi026.stderr:1+0 records out 2023-09-23T21:19:52.747 INFO:tasks.workunit.client.0.smithi026.stderr:1048576 bytes (1.0 MB, 1.0 MiB) copied, 0.00505081 s, 208 MB/s 2023-09-23T21:19:52.747 INFO:tasks.workunit.client.0.smithi026.stderr:+ dd if=/dev/urandom bs=1M count=1 of=/tmp/rbd_import_export_456745/sparse2 2023-09-23T21:19:52.753 INFO:tasks.workunit.client.0.smithi026.stderr:1+0 records in 2023-09-23T21:19:52.753 INFO:tasks.workunit.client.0.smithi026.stderr:1+0 records out 2023-09-23T21:19:52.753 INFO:tasks.workunit.client.0.smithi026.stderr:1048576 bytes (1.0 MB, 1.0 MiB) copied, 0.00508609 s, 206 MB/s 2023-09-23T21:19:52.753 INFO:tasks.workunit.client.0.smithi026.stderr:+ truncate /tmp/rbd_import_export_456745/sparse2 -s 2M 2023-09-23T21:19:52.781 INFO:tasks.workunit.client.0.smithi026.stderr:+ rbd rm sparse1 2023-09-23T21:19:52.820 INFO:tasks.workunit.client.0.smithi026.stderr: Removing image: 0% complete...failed. 2023-09-23T21:19:52.820 INFO:tasks.workunit.client.0.smithi026.stderr:rbd: delete error: (2) No such file or directory 2023-09-23T21:19:52.823 INFO:tasks.workunit.client.0.smithi026.stderr:+ true 2023-09-23T21:19:52.823 INFO:tasks.workunit.client.0.smithi026.stderr:+ rbd import --new-format --order 20 /tmp/rbd_import_export_456745/sparse1 2023-09-23T21:19:52.826 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:19:52.859 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:52.890 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_blocked_ops 2023-09-23T21:19:52.897 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:19:52.897 INFO:tasks.workunit.client.0.smithi026.stderr: Importing image: 50% complete... Importing image: 100% complete...done. 2023-09-23T21:19:52.898 INFO:tasks.workunit.client.0.smithi026.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-09-23T21:19:52.898 INFO:tasks.workunit.client.0.smithi026.stderr:rbd: --order is deprecated, use --object-size 2023-09-23T21:19:52.900 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:52.900 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:19:52.901 INFO:tasks.workunit.client.0.smithi026.stderr:+ rbd ls -l 2023-09-23T21:19:52.902 INFO:tasks.workunit.client.0.smithi026.stderr:+ grep sparse1 2023-09-23T21:19:52.902 INFO:tasks.workunit.client.0.smithi026.stderr:+ grep -Ei '(2 MiB|2048k)' 2023-09-23T21:19:52.944 INFO:tasks.workunit.client.0.smithi026.stdout:sparse1 2 MiB 2 2023-09-23T21:19:52.944 INFO:tasks.workunit.client.0.smithi026.stderr:++ objects sparse1 2023-09-23T21:19:52.944 INFO:tasks.workunit.client.0.smithi026.stderr:++ image=sparse1 2023-09-23T21:19:52.945 INFO:tasks.workunit.client.0.smithi026.stderr:+++ rbd info sparse1 2023-09-23T21:19:52.945 INFO:tasks.workunit.client.0.smithi026.stderr:+++ grep block_name_prefix 2023-09-23T21:19:52.945 INFO:tasks.workunit.client.0.smithi026.stderr:+++ awk '{print $NF;}' 2023-09-23T21:19:52.970 INFO:teuthology.orchestra.run.smithi026.stderr:nodeep-scrub is set 2023-09-23T21:19:53.001 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:19:53.001 INFO:tasks.workunit.client.0.smithi026.stderr:++ prefix=rbd_data.9264f100950 2023-09-23T21:19:53.002 INFO:tasks.workunit.client.0.smithi026.stderr:+++ grep rbd_data.9264f100950 2023-09-23T21:19:53.002 INFO:tasks.workunit.client.0.smithi026.stderr:++++ get_image_data_pool sparse1 2023-09-23T21:19:53.002 INFO:tasks.workunit.client.0.smithi026.stderr:++++ image=sparse1 2023-09-23T21:19:53.002 INFO:tasks.workunit.client.0.smithi026.stderr:+++ sed -e 's/rbd_data.9264f100950\.//' -e 's/^0*\([0-9a-f]\)/\1/' 2023-09-23T21:19:53.004 INFO:tasks.workunit.client.0.smithi026.stderr:+++ sort -u 2023-09-23T21:19:53.004 INFO:tasks.workunit.client.0.smithi026.stderr:+++++ rbd info sparse1 2023-09-23T21:19:53.004 INFO:tasks.workunit.client.0.smithi026.stderr:+++++ grep 'data_pool: ' 2023-09-23T21:19:53.004 INFO:tasks.workunit.client.0.smithi026.stderr:+++++ awk -F: '{ print $NF }' 2023-09-23T21:19:53.032 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:53.047 INFO:tasks.workunit.client.0.smithi026.stderr:++++ data_pool= 2023-09-23T21:19:53.047 INFO:tasks.workunit.client.0.smithi026.stderr:++++ '[' -z ']' 2023-09-23T21:19:53.047 INFO:tasks.workunit.client.0.smithi026.stderr:++++ data_pool=rbd 2023-09-23T21:19:53.047 INFO:tasks.workunit.client.0.smithi026.stderr:++++ echo rbd 2023-09-23T21:19:53.047 INFO:tasks.workunit.client.0.smithi026.stderr:+++ rados ls -p rbd 2023-09-23T21:19:53.096 INFO:tasks.workunit.client.0.smithi026.stderr:++ objects=1 2023-09-23T21:19:53.096 INFO:tasks.workunit.client.0.smithi026.stderr:++ echo 1 2023-09-23T21:19:53.096 INFO:tasks.workunit.client.0.smithi026.stderr:+ '[' 0 -eq 1 -o 1 = 1 ']' 2023-09-23T21:19:53.096 INFO:tasks.workunit.client.0.smithi026.stderr:+ rbd export sparse1 /tmp/rbd_import_export_456745/sparse1.out 2023-09-23T21:19:53.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:19:52 smithi070 ceph-mon[168282]: failed to encode map e667 with expected crc 2023-09-23T21:19:53.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:19:52 smithi070 ceph-mon[168282]: failed to encode map e667 with expected crc 2023-09-23T21:19:53.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:19:52 smithi070 ceph-mon[168282]: failed to encode map e667 with expected crc 2023-09-23T21:19:53.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:19:52 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/53669174' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:19:53.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:19:52 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:19:53.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:19:52 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/2680051164' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2023-09-23T21:19:53.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:19:52 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/2680051164' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2023-09-23T21:19:53.136 INFO:tasks.workunit.client.0.smithi026.stderr: Exporting image: 50% complete... Exporting image: 100% complete...done. 2023-09-23T21:19:53.141 INFO:tasks.workunit.client.0.smithi026.stderr:+ compare_files_and_ondisk_sizes /tmp/rbd_import_export_456745/sparse1 /tmp/rbd_import_export_456745/sparse1.out 2023-09-23T21:19:53.141 INFO:tasks.workunit.client.0.smithi026.stderr:+ cmp -l /tmp/rbd_import_export_456745/sparse1 /tmp/rbd_import_export_456745/sparse1.out 2023-09-23T21:19:53.144 INFO:tasks.workunit.client.0.smithi026.stderr:++ stat /tmp/rbd_import_export_456745/sparse1 --format %b 2023-09-23T21:19:53.145 INFO:tasks.workunit.client.0.smithi026.stderr:+ origsize=2048 2023-09-23T21:19:53.145 INFO:tasks.workunit.client.0.smithi026.stderr:++ stat /tmp/rbd_import_export_456745/sparse1.out --format %b 2023-09-23T21:19:53.146 INFO:tasks.workunit.client.0.smithi026.stderr:+ exportsize=2048 2023-09-23T21:19:53.146 INFO:tasks.workunit.client.0.smithi026.stderr:+ difference=0 2023-09-23T21:19:53.147 INFO:tasks.workunit.client.0.smithi026.stderr:+ difference=0 2023-09-23T21:19:53.147 INFO:tasks.workunit.client.0.smithi026.stderr:+ test 0 -ge 0 -a 0 -lt 4096 2023-09-23T21:19:53.147 INFO:tasks.workunit.client.0.smithi026.stderr:+ rm /tmp/rbd_import_export_456745/sparse1.out 2023-09-23T21:19:53.147 INFO:tasks.workunit.client.0.smithi026.stderr:+ rbd rm sparse1 2023-09-23T21:19:53.176 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:19:53.179 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:53.180 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:19:53.248 INFO:tasks.workunit.client.0.smithi026.stderr: Removing image: 50% complete... Removing image: 100% complete...done. 2023-09-23T21:19:53.251 INFO:tasks.workunit.client.0.smithi026.stderr:+ rbd rm sparse2 2023-09-23T21:19:53.258 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:19:52 smithi026 ceph-mon[175151]: failed to encode map e667 with expected crc 2023-09-23T21:19:53.259 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:19:52 smithi026 ceph-mon[175151]: failed to encode map e667 with expected crc 2023-09-23T21:19:53.259 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:19:52 smithi026 ceph-mon[175151]: failed to encode map e667 with expected crc 2023-09-23T21:19:53.259 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:19:52 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/53669174' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:19:53.259 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:19:52 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:19:53.259 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:19:52 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/2680051164' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2023-09-23T21:19:53.259 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:19:52 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/2680051164' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2023-09-23T21:19:53.259 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:19:52 smithi026 ceph-mon[179688]: failed to encode map e667 with expected crc 2023-09-23T21:19:53.259 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:19:52 smithi026 ceph-mon[179688]: failed to encode map e667 with expected crc 2023-09-23T21:19:53.259 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:19:52 smithi026 ceph-mon[179688]: failed to encode map e667 with expected crc 2023-09-23T21:19:53.260 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:19:52 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/53669174' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:19:53.260 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:19:52 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:19:53.260 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:19:52 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/2680051164' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2023-09-23T21:19:53.260 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:19:52 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/2680051164' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2023-09-23T21:19:53.280 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:19:53.298 INFO:tasks.workunit.client.0.smithi026.stderr: Removing image: 0% complete...failed. 2023-09-23T21:19:53.298 INFO:tasks.workunit.client.0.smithi026.stderr:rbd: delete error: (2) No such file or directory 2023-09-23T21:19:53.302 INFO:tasks.workunit.client.0.smithi026.stderr:+ true 2023-09-23T21:19:53.302 INFO:tasks.workunit.client.0.smithi026.stderr:+ rbd import --new-format --order 20 /tmp/rbd_import_export_456745/sparse2 2023-09-23T21:19:53.319 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:53.358 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:19:53.362 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:53.362 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:19:53.371 INFO:tasks.workunit.client.0.smithi026.stderr: Importing image: 50% complete... Importing image: 100% complete...done. 2023-09-23T21:19:53.372 INFO:tasks.workunit.client.0.smithi026.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-09-23T21:19:53.372 INFO:tasks.workunit.client.0.smithi026.stderr:rbd: --order is deprecated, use --object-size 2023-09-23T21:19:53.375 INFO:tasks.workunit.client.0.smithi026.stderr:+ rbd ls -l 2023-09-23T21:19:53.375 INFO:tasks.workunit.client.0.smithi026.stderr:+ grep sparse2 2023-09-23T21:19:53.375 INFO:tasks.workunit.client.0.smithi026.stderr:+ grep -Ei '(2 MiB|2048k)' 2023-09-23T21:19:53.411 INFO:tasks.workunit.client.0.smithi026.stdout:sparse2 2 MiB 2 2023-09-23T21:19:53.411 INFO:tasks.workunit.client.0.smithi026.stderr:++ objects sparse2 2023-09-23T21:19:53.412 INFO:tasks.workunit.client.0.smithi026.stderr:++ image=sparse2 2023-09-23T21:19:53.412 INFO:tasks.workunit.client.0.smithi026.stderr:+++ rbd info sparse2 2023-09-23T21:19:53.412 INFO:tasks.workunit.client.0.smithi026.stderr:+++ grep block_name_prefix 2023-09-23T21:19:53.412 INFO:tasks.workunit.client.0.smithi026.stderr:+++ awk '{print $NF;}' 2023-09-23T21:19:53.447 INFO:tasks.workunit.client.0.smithi026.stderr:++ prefix=rbd_data.b9fc1b6ab219 2023-09-23T21:19:53.449 INFO:tasks.workunit.client.0.smithi026.stderr:+++ grep rbd_data.b9fc1b6ab219 2023-09-23T21:19:53.449 INFO:tasks.workunit.client.0.smithi026.stderr:++++ get_image_data_pool sparse2 2023-09-23T21:19:53.449 INFO:tasks.workunit.client.0.smithi026.stderr:++++ image=sparse2 2023-09-23T21:19:53.449 INFO:tasks.workunit.client.0.smithi026.stderr:+++ sed -e 's/rbd_data.b9fc1b6ab219\.//' -e 's/^0*\([0-9a-f]\)/\1/' 2023-09-23T21:19:53.449 INFO:tasks.workunit.client.0.smithi026.stderr:+++ sort -u 2023-09-23T21:19:53.449 INFO:tasks.workunit.client.0.smithi026.stderr:+++++ rbd info sparse2 2023-09-23T21:19:53.449 INFO:tasks.workunit.client.0.smithi026.stderr:+++++ grep 'data_pool: ' 2023-09-23T21:19:53.449 INFO:tasks.workunit.client.0.smithi026.stderr:+++++ awk -F: '{ print $NF }' 2023-09-23T21:19:53.463 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:19:53.485 INFO:tasks.workunit.client.0.smithi026.stderr:++++ data_pool= 2023-09-23T21:19:53.485 INFO:tasks.workunit.client.0.smithi026.stderr:++++ '[' -z ']' 2023-09-23T21:19:53.485 INFO:tasks.workunit.client.0.smithi026.stderr:++++ data_pool=rbd 2023-09-23T21:19:53.485 INFO:tasks.workunit.client.0.smithi026.stderr:++++ echo rbd 2023-09-23T21:19:53.485 INFO:tasks.workunit.client.0.smithi026.stderr:+++ rados ls -p rbd 2023-09-23T21:19:53.491 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:53.518 INFO:tasks.workunit.client.0.smithi026.stderr:++ objects=0 2023-09-23T21:19:53.518 INFO:tasks.workunit.client.0.smithi026.stderr:++ echo 0 2023-09-23T21:19:53.518 INFO:tasks.workunit.client.0.smithi026.stderr:+ '[' 0 -eq 1 -o 0 = 0 ']' 2023-09-23T21:19:53.518 INFO:tasks.workunit.client.0.smithi026.stderr:+ rbd export sparse2 /tmp/rbd_import_export_456745/sparse2.out 2023-09-23T21:19:53.525 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:19:53.527 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:53.528 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:19:53.553 INFO:tasks.workunit.client.0.smithi026.stderr: Exporting image: 50% complete... Exporting image: 100% complete...done. 2023-09-23T21:19:53.556 INFO:tasks.workunit.client.0.smithi026.stderr:+ compare_files_and_ondisk_sizes /tmp/rbd_import_export_456745/sparse2 /tmp/rbd_import_export_456745/sparse2.out 2023-09-23T21:19:53.556 INFO:tasks.workunit.client.0.smithi026.stderr:+ cmp -l /tmp/rbd_import_export_456745/sparse2 /tmp/rbd_import_export_456745/sparse2.out 2023-09-23T21:19:53.558 INFO:tasks.workunit.client.0.smithi026.stderr:++ stat /tmp/rbd_import_export_456745/sparse2 --format %b 2023-09-23T21:19:53.559 INFO:tasks.workunit.client.0.smithi026.stderr:+ origsize=2048 2023-09-23T21:19:53.560 INFO:tasks.workunit.client.0.smithi026.stderr:++ stat /tmp/rbd_import_export_456745/sparse2.out --format %b 2023-09-23T21:19:53.561 INFO:tasks.workunit.client.0.smithi026.stderr:+ exportsize=2048 2023-09-23T21:19:53.561 INFO:tasks.workunit.client.0.smithi026.stderr:+ difference=0 2023-09-23T21:19:53.561 INFO:tasks.workunit.client.0.smithi026.stderr:+ difference=0 2023-09-23T21:19:53.561 INFO:tasks.workunit.client.0.smithi026.stderr:+ test 0 -ge 0 -a 0 -lt 4096 2023-09-23T21:19:53.561 INFO:tasks.workunit.client.0.smithi026.stderr:+ rm /tmp/rbd_import_export_456745/sparse2.out 2023-09-23T21:19:53.562 INFO:tasks.workunit.client.0.smithi026.stderr:+ rbd rm sparse2 2023-09-23T21:19:53.629 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:19:53.646 INFO:tasks.workunit.client.0.smithi026.stderr: Removing image: 50% complete... Removing image: 100% complete...done. 2023-09-23T21:19:53.649 INFO:tasks.workunit.client.0.smithi026.stderr:+ truncate /tmp/rbd_import_export_456745/sparse1 -s 10M 2023-09-23T21:19:53.650 INFO:tasks.workunit.client.0.smithi026.stderr:+ rbd import --new-format --order 20 - sparse1 2023-09-23T21:19:53.656 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:53.688 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:19:53.690 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:53.691 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:19:53.734 INFO:tasks.workunit.client.0.smithi026.stderr: Importing image: 100% complete...done. 2023-09-23T21:19:53.735 INFO:tasks.workunit.client.0.smithi026.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-09-23T21:19:53.735 INFO:tasks.workunit.client.0.smithi026.stderr:rbd: --order is deprecated, use --object-size 2023-09-23T21:19:53.738 INFO:tasks.workunit.client.0.smithi026.stderr:+ rbd ls -l 2023-09-23T21:19:53.738 INFO:tasks.workunit.client.0.smithi026.stderr:+ grep sparse1 2023-09-23T21:19:53.738 INFO:tasks.workunit.client.0.smithi026.stderr:+ grep -Ei '(10 MiB|10240k)' 2023-09-23T21:19:53.777 INFO:tasks.workunit.client.0.smithi026.stdout:sparse1 10 MiB 2 2023-09-23T21:19:53.778 INFO:tasks.workunit.client.0.smithi026.stderr:++ objects sparse1 2023-09-23T21:19:53.778 INFO:tasks.workunit.client.0.smithi026.stderr:++ image=sparse1 2023-09-23T21:19:53.778 INFO:tasks.workunit.client.0.smithi026.stderr:+++ rbd info sparse1 2023-09-23T21:19:53.778 INFO:tasks.workunit.client.0.smithi026.stderr:+++ grep block_name_prefix 2023-09-23T21:19:53.778 INFO:tasks.workunit.client.0.smithi026.stderr:+++ awk '{print $NF;}' 2023-09-23T21:19:53.792 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:19:53.816 INFO:tasks.workunit.client.0.smithi026.stderr:++ prefix=rbd_data.92c42cb13722 2023-09-23T21:19:53.817 INFO:tasks.workunit.client.0.smithi026.stderr:+++ grep rbd_data.92c42cb13722 2023-09-23T21:19:53.817 INFO:tasks.workunit.client.0.smithi026.stderr:++++ get_image_data_pool sparse1 2023-09-23T21:19:53.817 INFO:tasks.workunit.client.0.smithi026.stderr:++++ image=sparse1 2023-09-23T21:19:53.817 INFO:tasks.workunit.client.0.smithi026.stderr:+++ sed -e 's/rbd_data.92c42cb13722\.//' -e 's/^0*\([0-9a-f]\)/\1/' 2023-09-23T21:19:53.817 INFO:tasks.workunit.client.0.smithi026.stderr:+++ sort -u 2023-09-23T21:19:53.818 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:53.818 INFO:tasks.workunit.client.0.smithi026.stderr:+++++ rbd info sparse1 2023-09-23T21:19:53.820 INFO:tasks.workunit.client.0.smithi026.stderr:+++++ grep 'data_pool: ' 2023-09-23T21:19:53.820 INFO:tasks.workunit.client.0.smithi026.stderr:+++++ awk -F: '{ print $NF }' 2023-09-23T21:19:53.849 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:19:53.851 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:53.851 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:19:53.858 INFO:tasks.workunit.client.0.smithi026.stderr:++++ data_pool= 2023-09-23T21:19:53.858 INFO:tasks.workunit.client.0.smithi026.stderr:++++ '[' -z ']' 2023-09-23T21:19:53.858 INFO:tasks.workunit.client.0.smithi026.stderr:++++ data_pool=rbd 2023-09-23T21:19:53.858 INFO:tasks.workunit.client.0.smithi026.stderr:++++ echo rbd 2023-09-23T21:19:53.858 INFO:tasks.workunit.client.0.smithi026.stderr:+++ rados ls -p rbd 2023-09-23T21:19:53.896 INFO:tasks.workunit.client.0.smithi026.stderr:++ objects=1 2023-09-23T21:19:53.896 INFO:tasks.workunit.client.0.smithi026.stderr:++ echo 1 2023-09-23T21:19:53.896 INFO:tasks.workunit.client.0.smithi026.stderr:+ '[' 0 -eq 1 -o 1 = 1 ']' 2023-09-23T21:19:53.896 INFO:tasks.workunit.client.0.smithi026.stderr:+ rbd export sparse1 /tmp/rbd_import_export_456745/sparse1.out 2023-09-23T21:19:53.910 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0: {} 2023-09-23T21:19:53.936 INFO:tasks.workunit.client.0.smithi026.stderr: Exporting image: 10% complete... Exporting image: 20% complete... Exporting image: 30% complete... Exporting image: 40% complete... Exporting image: 50% complete... Exporting image: 60% complete... Exporting image: 70% complete... Exporting image: 80% complete... Exporting image: 90% complete... Exporting image: 100% complete...done. 2023-09-23T21:19:53.940 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1: {} 2023-09-23T21:19:53.941 INFO:tasks.workunit.client.0.smithi026.stderr:+ compare_files_and_ondisk_sizes /tmp/rbd_import_export_456745/sparse1 /tmp/rbd_import_export_456745/sparse1.out 2023-09-23T21:19:53.941 INFO:tasks.workunit.client.0.smithi026.stderr:+ cmp -l /tmp/rbd_import_export_456745/sparse1 /tmp/rbd_import_export_456745/sparse1.out 2023-09-23T21:19:53.952 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:19:53.953 INFO:tasks.workunit.client.0.smithi026.stderr:++ stat /tmp/rbd_import_export_456745/sparse1 --format %b 2023-09-23T21:19:53.955 INFO:tasks.workunit.client.0.smithi026.stderr:+ origsize=2048 2023-09-23T21:19:53.955 INFO:tasks.workunit.client.0.smithi026.stderr:++ stat /tmp/rbd_import_export_456745/sparse1.out --format %b 2023-09-23T21:19:53.957 INFO:tasks.workunit.client.0.smithi026.stderr:+ exportsize=2048 2023-09-23T21:19:53.957 INFO:tasks.workunit.client.0.smithi026.stderr:+ difference=0 2023-09-23T21:19:53.957 INFO:tasks.workunit.client.0.smithi026.stderr:+ difference=0 2023-09-23T21:19:53.957 INFO:tasks.workunit.client.0.smithi026.stderr:+ test 0 -ge 0 -a 0 -lt 4096 2023-09-23T21:19:53.957 INFO:tasks.workunit.client.0.smithi026.stderr:+ rm /tmp/rbd_import_export_456745/sparse1.out 2023-09-23T21:19:53.958 INFO:tasks.workunit.client.0.smithi026.stderr:+ rbd rm sparse1 2023-09-23T21:19:53.961 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:19:53 smithi026 ceph-mon[179688]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:19:53.961 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:19:53 smithi026 ceph-mon[179688]: failed to encode map e668 with expected crc 2023-09-23T21:19:53.962 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:19:53 smithi026 ceph-mon[179688]: failed to encode map e668 with expected crc 2023-09-23T21:19:53.962 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:19:53 smithi026 ceph-mon[179688]: failed to encode map e668 with expected crc 2023-09-23T21:19:53.962 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:19:53 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T21:19:53.962 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:19:53 smithi026 ceph-mon[179688]: osdmap e668: 8 total, 8 up, 5 in 2023-09-23T21:19:53.962 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:19:53 smithi026 ceph-mon[179688]: failed to encode map e668 with expected crc 2023-09-23T21:19:53.962 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:19:53 smithi026 ceph-mon[179688]: pgmap v1630: 103 pgs: 103 active+clean; 692 KiB data, 3.7 GiB used, 443 GiB / 447 GiB avail; 118 KiB/s rd, 7.7 KiB/s wr, 159 op/s 2023-09-23T21:19:53.962 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:19:53 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/3196176160' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:19:53.967 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2: {} 2023-09-23T21:19:53.978 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:54.006 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3: {} 2023-09-23T21:19:54.010 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:19:54.013 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:54.013 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:19:54.029 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4: {} 2023-09-23T21:19:54.060 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T21:19:54.065 INFO:tasks.workunit.client.0.smithi026.stderr: Removing image: 10% complete... Removing image: 20% complete... Removing image: 30% complete... Removing image: 40% complete... Removing image: 50% complete... Removing image: 60% complete... Removing image: 70% complete... Removing image: 80% complete... Removing image: 90% complete... Removing image: 100% complete...done. 2023-09-23T21:19:54.069 INFO:tasks.workunit.client.0.smithi026.stderr:+ dd if=/dev/urandom bs=2M count=1 of=/tmp/rbd_import_export_456745/sparse2 oflag=append conv=notrunc 2023-09-23T21:19:54.081 INFO:tasks.workunit.client.0.smithi026.stderr:1+0 records in 2023-09-23T21:19:54.081 INFO:tasks.workunit.client.0.smithi026.stderr:1+0 records out 2023-09-23T21:19:54.081 INFO:tasks.workunit.client.0.smithi026.stderr:2097152 bytes (2.1 MB, 2.0 MiB) copied, 0.0103044 s, 204 MB/s 2023-09-23T21:19:54.081 INFO:tasks.workunit.client.0.smithi026.stderr:+ rbd import --new-format --order 20 - sparse2 2023-09-23T21:19:54.084 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6: {} 2023-09-23T21:19:54.103 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T21:19:54.104 INFO:teuthology.orchestra.run.smithi026.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T21:19:54.105 INFO:teuthology.orchestra.run.smithi026.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T21:19:54.105 INFO:teuthology.orchestra.run.smithi026.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T21:19:54.105 INFO:teuthology.orchestra.run.smithi026.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T21:19:54.105 INFO:teuthology.orchestra.run.smithi026.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T21:19:54.105 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T21:19:54.105 INFO:teuthology.orchestra.run.smithi026.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T21:19:54.105 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T21:19:54.114 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:19:54.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:19:53 smithi070 ceph-mon[168282]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:19:54.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:19:53 smithi070 ceph-mon[168282]: failed to encode map e668 with expected crc 2023-09-23T21:19:54.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:19:53 smithi070 ceph-mon[168282]: failed to encode map e668 with expected crc 2023-09-23T21:19:54.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:19:53 smithi070 ceph-mon[168282]: failed to encode map e668 with expected crc 2023-09-23T21:19:54.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:19:53 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T21:19:54.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:19:53 smithi070 ceph-mon[168282]: osdmap e668: 8 total, 8 up, 5 in 2023-09-23T21:19:54.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:19:53 smithi070 ceph-mon[168282]: failed to encode map e668 with expected crc 2023-09-23T21:19:54.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:19:53 smithi070 ceph-mon[168282]: pgmap v1630: 103 pgs: 103 active+clean; 692 KiB data, 3.7 GiB used, 443 GiB / 447 GiB avail; 118 KiB/s rd, 7.7 KiB/s wr, 159 op/s 2023-09-23T21:19:54.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:19:53 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/3196176160' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:19:54.129 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:19:54.140 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:54.166 INFO:tasks.workunit.client.0.smithi026.stderr: Importing image: 100% complete...done. 2023-09-23T21:19:54.167 INFO:tasks.workunit.client.0.smithi026.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-09-23T21:19:54.167 INFO:tasks.workunit.client.0.smithi026.stderr:rbd: --order is deprecated, use --object-size 2023-09-23T21:19:54.171 INFO:tasks.workunit.client.0.smithi026.stderr:+ rbd ls -l 2023-09-23T21:19:54.171 INFO:tasks.workunit.client.0.smithi026.stderr:+ grep sparse2 2023-09-23T21:19:54.171 INFO:tasks.workunit.client.0.smithi026.stderr:+ grep -Ei '(4 MiB|4096k)' 2023-09-23T21:19:54.174 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:19:54.176 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:54.177 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:19:54.219 INFO:tasks.workunit.client.0.smithi026.stdout:sparse2 4 MiB 2 2023-09-23T21:19:54.220 INFO:tasks.workunit.client.0.smithi026.stderr:++ objects sparse2 2023-09-23T21:19:54.220 INFO:tasks.workunit.client.0.smithi026.stderr:++ image=sparse2 2023-09-23T21:19:54.220 INFO:tasks.workunit.client.0.smithi026.stderr:+++ rbd info sparse2 2023-09-23T21:19:54.220 INFO:tasks.workunit.client.0.smithi026.stderr:+++ grep block_name_prefix 2023-09-23T21:19:54.221 INFO:tasks.workunit.client.0.smithi026.stderr:+++ awk '{print $NF;}' 2023-09-23T21:19:54.243 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:19:53 smithi026 ceph-mon[175151]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:19:54.243 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:19:53 smithi026 ceph-mon[175151]: failed to encode map e668 with expected crc 2023-09-23T21:19:54.243 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:19:53 smithi026 ceph-mon[175151]: failed to encode map e668 with expected crc 2023-09-23T21:19:54.243 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:19:53 smithi026 ceph-mon[175151]: failed to encode map e668 with expected crc 2023-09-23T21:19:54.243 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:19:53 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T21:19:54.243 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:19:53 smithi026 ceph-mon[175151]: osdmap e668: 8 total, 8 up, 5 in 2023-09-23T21:19:54.244 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:19:53 smithi026 ceph-mon[175151]: failed to encode map e668 with expected crc 2023-09-23T21:19:54.244 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:19:53 smithi026 ceph-mon[175151]: pgmap v1630: 103 pgs: 103 active+clean; 692 KiB data, 3.7 GiB used, 443 GiB / 447 GiB avail; 118 KiB/s rd, 7.7 KiB/s wr, 159 op/s 2023-09-23T21:19:54.244 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:19:53 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/3196176160' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:19:54.261 INFO:tasks.workunit.client.0.smithi026.stderr:++ prefix=rbd_data.ba5cd1aa9795 2023-09-23T21:19:54.262 INFO:tasks.workunit.client.0.smithi026.stderr:+++ grep rbd_data.ba5cd1aa9795 2023-09-23T21:19:54.262 INFO:tasks.workunit.client.0.smithi026.stderr:++++ get_image_data_pool sparse2 2023-09-23T21:19:54.263 INFO:tasks.workunit.client.0.smithi026.stderr:++++ image=sparse2 2023-09-23T21:19:54.263 INFO:tasks.workunit.client.0.smithi026.stderr:+++ sed -e 's/rbd_data.ba5cd1aa9795\.//' -e 's/^0*\([0-9a-f]\)/\1/' 2023-09-23T21:19:54.263 INFO:tasks.workunit.client.0.smithi026.stderr:+++ sort -u 2023-09-23T21:19:54.263 INFO:tasks.workunit.client.0.smithi026.stderr:+++++ rbd info sparse2 2023-09-23T21:19:54.263 INFO:tasks.workunit.client.0.smithi026.stderr:+++++ grep 'data_pool: ' 2023-09-23T21:19:54.263 INFO:tasks.workunit.client.0.smithi026.stderr:+++++ awk -F: '{ print $NF }' 2023-09-23T21:19:54.278 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:19:54.302 INFO:tasks.workunit.client.0.smithi026.stderr:++++ data_pool= 2023-09-23T21:19:54.302 INFO:tasks.workunit.client.0.smithi026.stderr:++++ '[' -z ']' 2023-09-23T21:19:54.302 INFO:tasks.workunit.client.0.smithi026.stderr:++++ data_pool=rbd 2023-09-23T21:19:54.302 INFO:tasks.workunit.client.0.smithi026.stderr:++++ echo rbd 2023-09-23T21:19:54.302 INFO:tasks.workunit.client.0.smithi026.stderr:+++ rados ls -p rbd 2023-09-23T21:19:54.303 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:54.334 INFO:tasks.workunit.client.0.smithi026.stderr:++ objects='0 2023-09-23T21:19:54.334 INFO:tasks.workunit.client.0.smithi026.stderr:2 2023-09-23T21:19:54.335 INFO:tasks.workunit.client.0.smithi026.stderr:3' 2023-09-23T21:19:54.335 INFO:tasks.workunit.client.0.smithi026.stderr:++ echo 0 2 3 2023-09-23T21:19:54.335 INFO:tasks.workunit.client.0.smithi026.stderr:+ '[' 0 -eq 1 -o '0 2 3' = '0 2 3' ']' 2023-09-23T21:19:54.335 INFO:tasks.workunit.client.0.smithi026.stderr:+ rbd export sparse2 /tmp/rbd_import_export_456745/sparse2.out 2023-09-23T21:19:54.336 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:19:54.338 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:54.339 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:19:54.373 INFO:tasks.workunit.client.0.smithi026.stderr: Exporting image: 25% complete... Exporting image: 50% complete... Exporting image: 75% complete... Exporting image: 100% complete...done. 2023-09-23T21:19:54.376 INFO:tasks.workunit.client.0.smithi026.stderr:+ compare_files_and_ondisk_sizes /tmp/rbd_import_export_456745/sparse2 /tmp/rbd_import_export_456745/sparse2.out 2023-09-23T21:19:54.376 INFO:tasks.workunit.client.0.smithi026.stderr:+ cmp -l /tmp/rbd_import_export_456745/sparse2 /tmp/rbd_import_export_456745/sparse2.out 2023-09-23T21:19:54.380 INFO:tasks.workunit.client.0.smithi026.stderr:++ stat /tmp/rbd_import_export_456745/sparse2 --format %b 2023-09-23T21:19:54.381 INFO:tasks.workunit.client.0.smithi026.stderr:+ origsize=6144 2023-09-23T21:19:54.381 INFO:tasks.workunit.client.0.smithi026.stderr:++ stat /tmp/rbd_import_export_456745/sparse2.out --format %b 2023-09-23T21:19:54.382 INFO:tasks.workunit.client.0.smithi026.stderr:+ exportsize=6144 2023-09-23T21:19:54.382 INFO:tasks.workunit.client.0.smithi026.stderr:+ difference=0 2023-09-23T21:19:54.382 INFO:tasks.workunit.client.0.smithi026.stderr:+ difference=0 2023-09-23T21:19:54.382 INFO:tasks.workunit.client.0.smithi026.stderr:+ test 0 -ge 0 -a 0 -lt 4096 2023-09-23T21:19:54.382 INFO:tasks.workunit.client.0.smithi026.stderr:+ rm /tmp/rbd_import_export_456745/sparse2.out 2023-09-23T21:19:54.383 INFO:tasks.workunit.client.0.smithi026.stderr:+ rbd rm sparse2 2023-09-23T21:19:54.439 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:19:54.472 INFO:tasks.workunit.client.0.smithi026.stderr: Removing image: 25% complete... Removing image: 50% complete... Removing image: 75% complete... Removing image: 100% complete...done. 2023-09-23T21:19:54.475 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:54.476 INFO:tasks.workunit.client.0.smithi026.stderr:+ echo 'partially-sparse file imports to partially-sparse image' 2023-09-23T21:19:54.476 INFO:tasks.workunit.client.0.smithi026.stdout:partially-sparse file imports to partially-sparse image 2023-09-23T21:19:54.476 INFO:tasks.workunit.client.0.smithi026.stderr:+ rbd import --new-format --order 20 /tmp/rbd_import_export_456745/sparse1 sparse 2023-09-23T21:19:54.542 INFO:tasks.workunit.client.0.smithi026.stderr: Importing image: 10% complete... Importing image: 20% complete... Importing image: 30% complete... Importing image: 40% complete... Importing image: 50% complete... Importing image: 60% complete... Importing image: 70% complete... Importing image: 80% complete... Importing image: 90% complete... Importing image: 100% complete...done. 2023-09-23T21:19:54.542 INFO:tasks.workunit.client.0.smithi026.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-09-23T21:19:54.542 INFO:tasks.workunit.client.0.smithi026.stderr:rbd: --order is deprecated, use --object-size 2023-09-23T21:19:54.546 INFO:tasks.workunit.client.0.smithi026.stderr:++ objects sparse 2023-09-23T21:19:54.546 INFO:tasks.workunit.client.0.smithi026.stderr:++ image=sparse 2023-09-23T21:19:54.546 INFO:tasks.workunit.client.0.smithi026.stderr:+++ rbd info sparse 2023-09-23T21:19:54.546 INFO:tasks.workunit.client.0.smithi026.stderr:+++ grep block_name_prefix 2023-09-23T21:19:54.546 INFO:tasks.workunit.client.0.smithi026.stderr:+++ awk '{print $NF;}' 2023-09-23T21:19:54.565 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:19:54.568 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:54.568 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:19:54.586 INFO:tasks.workunit.client.0.smithi026.stderr:++ prefix=rbd_data.931e280698bd 2023-09-23T21:19:54.587 INFO:tasks.workunit.client.0.smithi026.stderr:+++ grep rbd_data.931e280698bd 2023-09-23T21:19:54.587 INFO:tasks.workunit.client.0.smithi026.stderr:++++ get_image_data_pool sparse 2023-09-23T21:19:54.587 INFO:tasks.workunit.client.0.smithi026.stderr:+++ sed -e 's/rbd_data.931e280698bd\.//' -e 's/^0*\([0-9a-f]\)/\1/' 2023-09-23T21:19:54.587 INFO:tasks.workunit.client.0.smithi026.stderr:++++ image=sparse 2023-09-23T21:19:54.588 INFO:tasks.workunit.client.0.smithi026.stderr:+++ sort -u 2023-09-23T21:19:54.588 INFO:tasks.workunit.client.0.smithi026.stderr:+++++ rbd info sparse 2023-09-23T21:19:54.588 INFO:tasks.workunit.client.0.smithi026.stderr:+++++ awk -F: '{ print $NF }' 2023-09-23T21:19:54.588 INFO:tasks.workunit.client.0.smithi026.stderr:+++++ grep 'data_pool: ' 2023-09-23T21:19:54.627 INFO:tasks.workunit.client.0.smithi026.stderr:++++ data_pool= 2023-09-23T21:19:54.627 INFO:tasks.workunit.client.0.smithi026.stderr:++++ '[' -z ']' 2023-09-23T21:19:54.627 INFO:tasks.workunit.client.0.smithi026.stderr:++++ data_pool=rbd 2023-09-23T21:19:54.627 INFO:tasks.workunit.client.0.smithi026.stderr:++++ echo rbd 2023-09-23T21:19:54.627 INFO:tasks.workunit.client.0.smithi026.stderr:+++ rados ls -p rbd 2023-09-23T21:19:54.670 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:19:54.671 INFO:tasks.workunit.client.0.smithi026.stderr:++ objects=1 2023-09-23T21:19:54.671 INFO:tasks.workunit.client.0.smithi026.stderr:++ echo 1 2023-09-23T21:19:54.671 INFO:tasks.workunit.client.0.smithi026.stderr:+ '[' 0 -eq 1 -o 1 = 1 ']' 2023-09-23T21:19:54.671 INFO:tasks.workunit.client.0.smithi026.stderr:+ rbd rm sparse 2023-09-23T21:19:54.703 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:54.751 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:19:54.754 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:54.755 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:19:54.783 INFO:tasks.workunit.client.0.smithi026.stderr: Removing image: 10% complete... Removing image: 20% complete... Removing image: 30% complete... Removing image: 40% complete... Removing image: 50% complete... Removing image: 60% complete... Removing image: 70% complete... Removing image: 80% complete... Removing image: 90% complete... Removing image: 100% complete...done. 2023-09-23T21:19:54.787 INFO:tasks.workunit.client.0.smithi026.stdout:zeros import through stdin to sparse image 2023-09-23T21:19:54.788 INFO:tasks.workunit.client.0.smithi026.stderr:+ echo 'zeros import through stdin to sparse image' 2023-09-23T21:19:54.788 INFO:tasks.workunit.client.0.smithi026.stderr:+ dd if=/dev/zero bs=1M count=4 2023-09-23T21:19:54.788 INFO:tasks.workunit.client.0.smithi026.stderr:+ rbd import --new-format - sparse 2023-09-23T21:19:54.840 INFO:tasks.workunit.client.0.smithi026.stderr:4+0 records in 2023-09-23T21:19:54.840 INFO:tasks.workunit.client.0.smithi026.stderr:4+0 records out 2023-09-23T21:19:54.840 INFO:tasks.workunit.client.0.smithi026.stderr:4194304 bytes (4.2 MB, 4.0 MiB) copied, 0.0511851 s, 81.9 MB/s 2023-09-23T21:19:54.849 INFO:tasks.workunit.client.0.smithi026.stderr: Importing image: 100% complete...done. 2023-09-23T21:19:54.850 INFO:tasks.workunit.client.0.smithi026.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-09-23T21:19:54.853 INFO:tasks.workunit.client.0.smithi026.stderr:++ objects sparse 2023-09-23T21:19:54.853 INFO:tasks.workunit.client.0.smithi026.stderr:++ image=sparse 2023-09-23T21:19:54.853 INFO:tasks.workunit.client.0.smithi026.stderr:+++ rbd info sparse 2023-09-23T21:19:54.853 INFO:tasks.workunit.client.0.smithi026.stderr:+++ grep block_name_prefix 2023-09-23T21:19:54.853 INFO:tasks.workunit.client.0.smithi026.stderr:+++ awk '{print $NF;}' 2023-09-23T21:19:54.856 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:19:54.888 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:54.893 INFO:tasks.workunit.client.0.smithi026.stderr:++ prefix=rbd_data.baada5b727b 2023-09-23T21:19:54.894 INFO:tasks.workunit.client.0.smithi026.stderr:+++ grep rbd_data.baada5b727b 2023-09-23T21:19:54.894 INFO:tasks.workunit.client.0.smithi026.stderr:++++ get_image_data_pool sparse 2023-09-23T21:19:54.895 INFO:tasks.workunit.client.0.smithi026.stderr:++++ image=sparse 2023-09-23T21:19:54.895 INFO:tasks.workunit.client.0.smithi026.stderr:+++ sed -e 's/rbd_data.baada5b727b\.//' -e 's/^0*\([0-9a-f]\)/\1/' 2023-09-23T21:19:54.895 INFO:tasks.workunit.client.0.smithi026.stderr:+++ sort -u 2023-09-23T21:19:54.895 INFO:tasks.workunit.client.0.smithi026.stderr:+++++ awk -F: '{ print $NF }' 2023-09-23T21:19:54.895 INFO:tasks.workunit.client.0.smithi026.stderr:+++++ rbd info sparse 2023-09-23T21:19:54.896 INFO:tasks.workunit.client.0.smithi026.stderr:+++++ grep 'data_pool: ' 2023-09-23T21:19:54.901 INFO:teuthology.orchestra.run.smithi026.stderr:Error EINVAL: specified pgp_num 51 > pg_num 31 2023-09-23T21:19:54.944 INFO:tasks.workunit.client.0.smithi026.stderr:++++ data_pool= 2023-09-23T21:19:54.944 INFO:tasks.workunit.client.0.smithi026.stderr:++++ '[' -z ']' 2023-09-23T21:19:54.944 INFO:tasks.workunit.client.0.smithi026.stderr:++++ data_pool=rbd 2023-09-23T21:19:54.944 INFO:tasks.workunit.client.0.smithi026.stderr:++++ echo rbd 2023-09-23T21:19:54.944 INFO:tasks.workunit.client.0.smithi026.stderr:+++ rados ls -p rbd 2023-09-23T21:19:54.980 INFO:tasks.workunit.client.0.smithi026.stderr:++ objects= 2023-09-23T21:19:54.980 INFO:tasks.workunit.client.0.smithi026.stderr:++ echo 2023-09-23T21:19:54.980 INFO:tasks.workunit.client.0.smithi026.stderr:+ '[' 0 -eq 1 -o '' = '' ']' 2023-09-23T21:19:54.980 INFO:tasks.workunit.client.0.smithi026.stderr:+ rbd rm sparse 2023-09-23T21:19:55.074 INFO:tasks.workunit.client.0.smithi026.stderr: Removing image: 100% complete...done. 2023-09-23T21:19:55.077 INFO:tasks.workunit.client.0.smithi026.stderr:+ echo 'zeros export to sparse file' 2023-09-23T21:19:55.078 INFO:tasks.workunit.client.0.smithi026.stdout:zeros export to sparse file 2023-09-23T21:19:55.078 INFO:tasks.workunit.client.0.smithi026.stderr:+ rbd create --new-format sparse --size 4 2023-09-23T21:19:55.078 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T21:19:55.081 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:55.081 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:19:55.116 INFO:tasks.workunit.client.0.smithi026.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-09-23T21:19:55.118 INFO:tasks.workunit.client.0.smithi026.stderr:++ rbd info sparse 2023-09-23T21:19:55.119 INFO:tasks.workunit.client.0.smithi026.stderr:++ grep block_name_prefix 2023-09-23T21:19:55.119 INFO:tasks.workunit.client.0.smithi026.stderr:++ awk '{print $NF;}' 2023-09-23T21:19:55.155 INFO:tasks.workunit.client.0.smithi026.stderr:+ prefix=rbd_data.bac8c15cbd9f 2023-09-23T21:19:55.155 INFO:tasks.workunit.client.0.smithi026.stderr:+ dd if=/dev/zero bs=4M count=1 2023-09-23T21:19:55.156 INFO:tasks.workunit.client.0.smithi026.stderr:++ get_image_data_pool sparse 2023-09-23T21:19:55.156 INFO:tasks.workunit.client.0.smithi026.stderr:++ image=sparse 2023-09-23T21:19:55.156 INFO:tasks.workunit.client.0.smithi026.stderr:+++ rbd info sparse 2023-09-23T21:19:55.156 INFO:tasks.workunit.client.0.smithi026.stderr:+++ grep 'data_pool: ' 2023-09-23T21:19:55.157 INFO:tasks.workunit.client.0.smithi026.stderr:+++ awk -F: '{ print $NF }' 2023-09-23T21:19:55.182 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:19:55.192 INFO:tasks.workunit.client.0.smithi026.stderr:++ data_pool= 2023-09-23T21:19:55.192 INFO:tasks.workunit.client.0.smithi026.stderr:++ '[' -z ']' 2023-09-23T21:19:55.192 INFO:tasks.workunit.client.0.smithi026.stderr:++ data_pool=rbd 2023-09-23T21:19:55.192 INFO:tasks.workunit.client.0.smithi026.stderr:++ echo rbd 2023-09-23T21:19:55.192 INFO:tasks.workunit.client.0.smithi026.stderr:+ rados -p rbd put rbd_data.bac8c15cbd9f.000000000000 - 2023-09-23T21:19:55.208 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:55.222 INFO:tasks.workunit.client.0.smithi026.stderr:1+0 records in 2023-09-23T21:19:55.223 INFO:tasks.workunit.client.0.smithi026.stderr:1+0 records out 2023-09-23T21:19:55.223 INFO:tasks.workunit.client.0.smithi026.stderr:4194304 bytes (4.2 MB, 4.0 MiB) copied, 0.0665326 s, 63.0 MB/s 2023-09-23T21:19:55.238 INFO:tasks.workunit.client.0.smithi026.stderr:++ objects sparse 2023-09-23T21:19:55.239 INFO:tasks.workunit.client.0.smithi026.stderr:++ image=sparse 2023-09-23T21:19:55.239 INFO:tasks.workunit.client.0.smithi026.stderr:+++ rbd info sparse 2023-09-23T21:19:55.239 INFO:tasks.workunit.client.0.smithi026.stderr:+++ grep block_name_prefix 2023-09-23T21:19:55.239 INFO:tasks.workunit.client.0.smithi026.stderr:+++ awk '{print $NF;}' 2023-09-23T21:19:55.241 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T21:19:55.243 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:19:54 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/3863537974' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "51"}]: dispatch 2023-09-23T21:19:55.243 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:19:54 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "51"}]: dispatch 2023-09-23T21:19:55.244 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:55.244 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:19:55.274 INFO:tasks.workunit.client.0.smithi026.stderr:++ prefix=rbd_data.bac8c15cbd9f 2023-09-23T21:19:55.275 INFO:tasks.workunit.client.0.smithi026.stderr:+++ grep rbd_data.bac8c15cbd9f 2023-09-23T21:19:55.275 INFO:tasks.workunit.client.0.smithi026.stderr:++++ get_image_data_pool sparse 2023-09-23T21:19:55.275 INFO:tasks.workunit.client.0.smithi026.stderr:++++ image=sparse 2023-09-23T21:19:55.276 INFO:tasks.workunit.client.0.smithi026.stderr:+++ sed -e 's/rbd_data.bac8c15cbd9f\.//' -e 's/^0*\([0-9a-f]\)/\1/' 2023-09-23T21:19:55.276 INFO:tasks.workunit.client.0.smithi026.stderr:+++ sort -u 2023-09-23T21:19:55.276 INFO:tasks.workunit.client.0.smithi026.stderr:+++++ rbd info sparse 2023-09-23T21:19:55.276 INFO:tasks.workunit.client.0.smithi026.stderr:+++++ grep 'data_pool: ' 2023-09-23T21:19:55.276 INFO:tasks.workunit.client.0.smithi026.stderr:+++++ awk -F: '{ print $NF }' 2023-09-23T21:19:55.284 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:19:54 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/3863537974' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "51"}]: dispatch 2023-09-23T21:19:55.284 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:19:54 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "51"}]: dispatch 2023-09-23T21:19:55.284 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:19:54 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/3863537974' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "51"}]: dispatch 2023-09-23T21:19:55.284 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:19:54 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "51"}]: dispatch 2023-09-23T21:19:55.312 INFO:tasks.workunit.client.0.smithi026.stderr:++++ data_pool= 2023-09-23T21:19:55.312 INFO:tasks.workunit.client.0.smithi026.stderr:++++ '[' -z ']' 2023-09-23T21:19:55.312 INFO:tasks.workunit.client.0.smithi026.stderr:++++ data_pool=rbd 2023-09-23T21:19:55.313 INFO:tasks.workunit.client.0.smithi026.stderr:++++ echo rbd 2023-09-23T21:19:55.313 INFO:tasks.workunit.client.0.smithi026.stderr:+++ rados ls -p rbd 2023-09-23T21:19:55.344 INFO:tasks.workunit.client.0.smithi026.stderr:++ objects=0 2023-09-23T21:19:55.344 INFO:tasks.workunit.client.0.smithi026.stderr:++ echo 0 2023-09-23T21:19:55.344 INFO:tasks.workunit.client.0.smithi026.stderr:+ '[' 0 -eq 1 -o 0 = 0 ']' 2023-09-23T21:19:55.344 INFO:tasks.workunit.client.0.smithi026.stderr:+ rm /tmp/rbd_import_export_456745/sparse 2023-09-23T21:19:55.345 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:19:55.346 INFO:tasks.workunit.client.0.smithi026.stderr:rm: cannot remove '/tmp/rbd_import_export_456745/sparse': No such file or directory 2023-09-23T21:19:55.346 INFO:tasks.workunit.client.0.smithi026.stderr:+ true 2023-09-23T21:19:55.346 INFO:tasks.workunit.client.0.smithi026.stderr:+ rbd export sparse /tmp/rbd_import_export_456745/sparse 2023-09-23T21:19:55.371 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:55.380 INFO:tasks.workunit.client.0.smithi026.stderr: Exporting image: 100% complete...done. 2023-09-23T21:19:55.384 INFO:tasks.workunit.client.0.smithi026.stderr:++ stat /tmp/rbd_import_export_456745/sparse --format=%b 2023-09-23T21:19:55.385 INFO:tasks.workunit.client.0.smithi026.stderr:+ '[' 0 = 0 ']' 2023-09-23T21:19:55.385 INFO:tasks.workunit.client.0.smithi026.stderr:+ rbd rm sparse 2023-09-23T21:19:55.403 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T21:19:55.406 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:55.407 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:19:55.465 INFO:tasks.workunit.client.0.smithi026.stderr: Removing image: 100% complete...done. 2023-09-23T21:19:55.468 INFO:tasks.workunit.client.0.smithi026.stderr:+ rm /tmp/rbd_import_export_456745/sparse /tmp/rbd_import_export_456745/sparse1 /tmp/rbd_import_export_456745/sparse2 /tmp/rbd_import_export_456745/sparse3 2023-09-23T21:19:55.471 INFO:tasks.workunit.client.0.smithi026.stderr:rm: cannot remove '/tmp/rbd_import_export_456745/sparse3': No such file or directory 2023-09-23T21:19:55.471 INFO:tasks.workunit.client.0.smithi026.stderr:+ true 2023-09-23T21:19:55.471 INFO:tasks.workunit.client.0.smithi026.stdout:OK 2023-09-23T21:19:55.472 INFO:tasks.workunit.client.0.smithi026.stderr:+ echo OK 2023-09-23T21:19:55.472 INFO:tasks.workunit.client.0.smithi026.stderr:+ rm -rf /tmp/rbd_import_export_456745 2023-09-23T21:19:55.472 INFO:teuthology.orchestra.run:Running command with timeout 3600 2023-09-23T21:19:55.473 DEBUG:teuthology.orchestra.run.smithi026:> sudo rm -rf -- /home/ubuntu/cephtest/mnt.0/client.0/tmp 2023-09-23T21:19:55.507 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:19:55.545 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:55.546 INFO:tasks.workunit:Stopping ['rbd/import_export.sh'] on client.0... 2023-09-23T21:19:55.546 DEBUG:teuthology.orchestra.run.smithi026:> sudo rm -rf -- /home/ubuntu/cephtest/workunits.list.client.0 /home/ubuntu/cephtest/clone.client.0 2023-09-23T21:19:55.597 DEBUG:teuthology.orchestra.run:got remote process result: 22 2023-09-23T21:19:55.696 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T21:19:55.698 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:55.699 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:19:55.800 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:19:55.827 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:55.859 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:19:55.861 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:55.862 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:19:55.924 DEBUG:teuthology.parallel:result is None 2023-09-23T21:19:55.924 DEBUG:teuthology.orchestra.run.smithi026:> sudo rm -rf -- /home/ubuntu/cephtest/mnt.0/client.0 2023-09-23T21:19:55.954 INFO:tasks.workunit:Deleted dir /home/ubuntu/cephtest/mnt.0/client.0 2023-09-23T21:19:55.954 DEBUG:teuthology.orchestra.run.smithi026:> rmdir -- /home/ubuntu/cephtest/mnt.0 2023-09-23T21:19:55.959 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T21:19:55.963 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:19:55.989 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:56.013 INFO:tasks.workunit:Deleted artificial mount point /home/ubuntu/cephtest/mnt.0/client.0 2023-09-23T21:19:56.014 INFO:teuthology.task.sequential:In sequential, running task print... 2023-09-23T21:19:56.014 INFO:teuthology.task.print:**** done rbd/import_export.sh 5-workload 2023-09-23T21:19:56.014 INFO:teuthology.task.sequential:In sequential, running task workunit... 2023-09-23T21:19:56.014 INFO:tasks.workunit:Pulling workunits from ref quincy 2023-09-23T21:19:56.014 INFO:tasks.workunit:Making a separate scratch dir for every client... 2023-09-23T21:19:56.015 DEBUG:teuthology.orchestra.run.smithi026:> stat -- /home/ubuntu/cephtest/mnt.0 2023-09-23T21:19:56.020 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T21:19:56.023 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:56.023 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:19:56.073 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T21:19:56.074 INFO:teuthology.orchestra.run.smithi026.stderr:stat: cannot statx '/home/ubuntu/cephtest/mnt.0': No such file or directory 2023-09-23T21:19:56.074 DEBUG:teuthology.orchestra.run.smithi026:> mkdir -- /home/ubuntu/cephtest/mnt.0 2023-09-23T21:19:56.081 INFO:teuthology.orchestra.run.smithi026.stdout:ERROR: (22) Invalid argument 2023-09-23T21:19:56.081 INFO:teuthology.orchestra.run.smithi026.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T21:19:56.098 INFO:tasks.workunit:Created dir /home/ubuntu/cephtest/mnt.0 2023-09-23T21:19:56.099 DEBUG:teuthology.orchestra.run.smithi026:> cd -- /home/ubuntu/cephtest/mnt.0 && mkdir -- client.0 2023-09-23T21:19:56.114 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:19:55 smithi026 ceph-mon[179688]: pgmap v1631: 103 pgs: 103 active+clean; 692 KiB data, 3.7 GiB used, 443 GiB / 447 GiB avail; 90 KiB/s rd, 5.9 KiB/s wr, 120 op/s 2023-09-23T21:19:56.114 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:19:55 smithi026 ceph-mon[179688]: osdmap e669: 8 total, 8 up, 5 in 2023-09-23T21:19:56.115 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:19:55 smithi026 ceph-mon[175151]: pgmap v1631: 103 pgs: 103 active+clean; 692 KiB data, 3.7 GiB used, 443 GiB / 447 GiB avail; 90 KiB/s rd, 5.9 KiB/s wr, 120 op/s 2023-09-23T21:19:56.115 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:19:55 smithi026 ceph-mon[175151]: osdmap e669: 8 total, 8 up, 5 in 2023-09-23T21:19:56.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:19:55 smithi070 ceph-mon[168282]: pgmap v1631: 103 pgs: 103 active+clean; 692 KiB data, 3.7 GiB used, 443 GiB / 447 GiB avail; 90 KiB/s rd, 5.9 KiB/s wr, 120 op/s 2023-09-23T21:19:56.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:19:55 smithi070 ceph-mon[168282]: osdmap e669: 8 total, 8 up, 5 in 2023-09-23T21:19:56.124 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:19:56.131 INFO:tasks.thrashosds:joining thrashosds 2023-09-23T21:19:56.203 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:56.463 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID "ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T21:19:56.465 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:56.466 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:19:56.492 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_historic_ops 2023-09-23T21:19:56.794 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:19:57.300 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:19:56 smithi026 ceph-mon[175151]: failed to encode map e669 with expected crc 2023-09-23T21:19:57.300 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:19:56 smithi026 ceph-mon[179688]: failed to encode map e669 with expected crc 2023-09-23T21:19:57.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:19:56 smithi070 ceph-mon[168282]: failed to encode map e669 with expected crc 2023-09-23T21:19:57.568 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:19:58.319 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:19:57 smithi026 ceph-mon[175151]: pgmap v1633: 103 pgs: 103 active+clean; 4.6 MiB data, 3.7 GiB used, 443 GiB / 447 GiB avail; 1.3 MiB/s rd, 1.9 MiB/s wr, 459 op/s 2023-09-23T21:19:58.319 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:19:57 smithi026 ceph-mon[179688]: pgmap v1633: 103 pgs: 103 active+clean; 4.6 MiB data, 3.7 GiB used, 443 GiB / 447 GiB avail; 1.3 MiB/s rd, 1.9 MiB/s wr, 459 op/s 2023-09-23T21:19:58.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:19:57 smithi070 ceph-mon[168282]: pgmap v1633: 103 pgs: 103 active+clean; 4.6 MiB data, 3.7 GiB used, 443 GiB / 447 GiB avail; 1.3 MiB/s rd, 1.9 MiB/s wr, 459 op/s 2023-09-23T21:19:58.986 INFO:teuthology.orchestra.run.smithi026.stderr:noscrub is unset 2023-09-23T21:19:59.065 INFO:teuthology.orchestra.run.smithi026.stdout:ERROR: (22) Invalid argument 2023-09-23T21:19:59.065 INFO:teuthology.orchestra.run.smithi026.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T21:19:59.118 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:19:58 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/1329036379' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T21:19:59.118 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:19:58 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T21:19:59.118 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:19:58 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/1329036379' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T21:19:59.119 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:19:58 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T21:19:59.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:19:58 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/1329036379' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T21:19:59.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:19:58 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T21:19:59.719 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_ops_in_flight 2023-09-23T21:19:59.881 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 21:19:59 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[164029]: ::ffff:172.21.15.70 - - [23/Sep/2023:21:19:59] "GET /metrics HTTP/1.1" 200 33991 "" "Prometheus/2.43.0" 2023-09-23T21:20:00.331 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:20:00.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:19:59 smithi070 ceph-mon[168282]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:20:00.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:19:59 smithi070 ceph-mon[168282]: failed to encode map e670 with expected crc 2023-09-23T21:20:00.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:19:59 smithi070 ceph-mon[168282]: failed to encode map e670 with expected crc 2023-09-23T21:20:00.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:19:59 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T21:20:00.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:19:59 smithi070 ceph-mon[168282]: osdmap e670: 8 total, 8 up, 5 in 2023-09-23T21:20:00.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:19:59 smithi070 ceph-mon[168282]: failed to encode map e670 with expected crc 2023-09-23T21:20:00.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:19:59 smithi070 ceph-mon[168282]: pgmap v1635: 103 pgs: 103 active+clean; 4.6 MiB data, 3.7 GiB used, 443 GiB / 447 GiB avail; 1.2 MiB/s rd, 1.8 MiB/s wr, 395 op/s 2023-09-23T21:20:00.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:19:59 smithi026 ceph-mon[179688]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:20:00.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:19:59 smithi026 ceph-mon[179688]: failed to encode map e670 with expected crc 2023-09-23T21:20:00.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:19:59 smithi026 ceph-mon[179688]: failed to encode map e670 with expected crc 2023-09-23T21:20:00.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:19:59 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T21:20:00.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:19:59 smithi026 ceph-mon[179688]: osdmap e670: 8 total, 8 up, 5 in 2023-09-23T21:20:00.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:19:59 smithi026 ceph-mon[179688]: failed to encode map e670 with expected crc 2023-09-23T21:20:00.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:19:59 smithi026 ceph-mon[179688]: pgmap v1635: 103 pgs: 103 active+clean; 4.6 MiB data, 3.7 GiB used, 443 GiB / 447 GiB avail; 1.2 MiB/s rd, 1.8 MiB/s wr, 395 op/s 2023-09-23T21:20:00.383 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:19:59 smithi026 ceph-mon[175151]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:20:00.383 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:19:59 smithi026 ceph-mon[175151]: failed to encode map e670 with expected crc 2023-09-23T21:20:00.383 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:19:59 smithi026 ceph-mon[175151]: failed to encode map e670 with expected crc 2023-09-23T21:20:00.383 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:19:59 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T21:20:00.383 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:19:59 smithi026 ceph-mon[175151]: osdmap e670: 8 total, 8 up, 5 in 2023-09-23T21:20:00.383 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:19:59 smithi026 ceph-mon[175151]: failed to encode map e670 with expected crc 2023-09-23T21:20:00.384 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:19:59 smithi026 ceph-mon[175151]: pgmap v1635: 103 pgs: 103 active+clean; 4.6 MiB data, 3.7 GiB used, 443 GiB / 447 GiB avail; 1.2 MiB/s rd, 1.8 MiB/s wr, 395 op/s 2023-09-23T21:20:00.598 INFO:tasks.thrashosds.thrasher:inning osd 2023-09-23T21:20:00.598 INFO:tasks.thrashosds.thrasher:Adding osd 6 2023-09-23T21:20:00.598 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph osd in 6 2023-09-23T21:20:01.131 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:20:00 smithi026 ceph-mon[175151]: failed to encode map e670 with expected crc 2023-09-23T21:20:01.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:20:00 smithi026 ceph-mon[175151]: Health detail: HEALTH_WARN nodeep-scrub flag(s) set 2023-09-23T21:20:01.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:20:00 smithi026 ceph-mon[175151]: [WRN] OSDMAP_FLAGS: nodeep-scrub flag(s) set 2023-09-23T21:20:01.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:20:00 smithi026 ceph-mon[175151]: osdmap e671: 8 total, 8 up, 5 in 2023-09-23T21:20:01.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:20:00 smithi026 ceph-mon[179688]: failed to encode map e670 with expected crc 2023-09-23T21:20:01.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:20:00 smithi026 ceph-mon[179688]: Health detail: HEALTH_WARN nodeep-scrub flag(s) set 2023-09-23T21:20:01.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:20:00 smithi026 ceph-mon[179688]: [WRN] OSDMAP_FLAGS: nodeep-scrub flag(s) set 2023-09-23T21:20:01.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:20:00 smithi026 ceph-mon[179688]: osdmap e671: 8 total, 8 up, 5 in 2023-09-23T21:20:01.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:20:00 smithi070 ceph-mon[168282]: failed to encode map e670 with expected crc 2023-09-23T21:20:01.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:20:00 smithi070 ceph-mon[168282]: Health detail: HEALTH_WARN nodeep-scrub flag(s) set 2023-09-23T21:20:01.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:20:00 smithi070 ceph-mon[168282]: [WRN] OSDMAP_FLAGS: nodeep-scrub flag(s) set 2023-09-23T21:20:01.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:20:00 smithi070 ceph-mon[168282]: osdmap e671: 8 total, 8 up, 5 in 2023-09-23T21:20:01.592 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T21:20:01.595 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:20:01.606 INFO:teuthology.orchestra.run.smithi026.stdout:ERROR: (22) Invalid argument 2023-09-23T21:20:01.606 INFO:teuthology.orchestra.run.smithi026.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T21:20:02.131 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:20:01 smithi026 ceph-mon[175151]: failed to encode map e671 with expected crc 2023-09-23T21:20:02.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:20:01 smithi026 ceph-mon[175151]: failed to encode map e671 with expected crc 2023-09-23T21:20:02.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:20:01 smithi026 ceph-mon[175151]: failed to encode map e671 with expected crc 2023-09-23T21:20:02.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:20:01 smithi026 ceph-mon[175151]: pgmap v1637: 103 pgs: 103 active+clean; 4.6 MiB data, 3.7 GiB used, 443 GiB / 447 GiB avail; 1.3 MiB/s rd, 1.9 MiB/s wr, 412 op/s 2023-09-23T21:20:02.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:20:01 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:20:02.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:20:01 smithi026 ceph-mon[179688]: failed to encode map e671 with expected crc 2023-09-23T21:20:02.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:20:01 smithi026 ceph-mon[179688]: failed to encode map e671 with expected crc 2023-09-23T21:20:02.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:20:01 smithi026 ceph-mon[179688]: failed to encode map e671 with expected crc 2023-09-23T21:20:02.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:20:01 smithi026 ceph-mon[179688]: pgmap v1637: 103 pgs: 103 active+clean; 4.6 MiB data, 3.7 GiB used, 443 GiB / 447 GiB avail; 1.3 MiB/s rd, 1.9 MiB/s wr, 412 op/s 2023-09-23T21:20:02.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:20:01 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:20:02.190 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_blocked_ops 2023-09-23T21:20:02.259 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:20:01 smithi070 ceph-mon[168282]: failed to encode map e671 with expected crc 2023-09-23T21:20:02.259 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:20:01 smithi070 ceph-mon[168282]: failed to encode map e671 with expected crc 2023-09-23T21:20:02.260 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:20:01 smithi070 ceph-mon[168282]: failed to encode map e671 with expected crc 2023-09-23T21:20:02.260 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:20:01 smithi070 ceph-mon[168282]: pgmap v1637: 103 pgs: 103 active+clean; 4.6 MiB data, 3.7 GiB used, 443 GiB / 447 GiB avail; 1.3 MiB/s rd, 1.9 MiB/s wr, 412 op/s 2023-09-23T21:20:02.260 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:20:01 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:20:02.813 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:20:03.190 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:20:03.273 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:20:02 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T21:20:03.274 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:20:02 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:20:03.274 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:20:02 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:20:03.274 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:20:02 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T21:20:03.274 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:20:02 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:20:03.274 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:20:02 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:20:03.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:20:02 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T21:20:03.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:20:02 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:20:03.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:20:02 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:20:03.591 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:20:03 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mon-a[175127]: 2023-09-23T21:20:03.371+0000 7f1e2d487700 -1 mon.a@0(leader).osd e671 definitely_dead 0 2023-09-23T21:20:04.015 INFO:teuthology.orchestra.run.smithi026.stderr:marked in osd.6. 2023-09-23T21:20:04.131 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:20:03 smithi026 ceph-mon[175151]: pgmap v1638: 103 pgs: 103 active+clean; 4.6 MiB data, 3.7 GiB used, 443 GiB / 447 GiB avail; 734 KiB/s rd, 570 KiB/s wr, 231 op/s 2023-09-23T21:20:04.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:20:03 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/8218457' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["6"]}]: dispatch 2023-09-23T21:20:04.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:20:03 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["6"]}]: dispatch 2023-09-23T21:20:04.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:20:03 smithi026 ceph-mon[179688]: pgmap v1638: 103 pgs: 103 active+clean; 4.6 MiB data, 3.7 GiB used, 443 GiB / 447 GiB avail; 734 KiB/s rd, 570 KiB/s wr, 231 op/s 2023-09-23T21:20:04.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:20:03 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/8218457' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["6"]}]: dispatch 2023-09-23T21:20:04.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:20:03 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["6"]}]: dispatch 2023-09-23T21:20:04.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:20:03 smithi070 ceph-mon[168282]: pgmap v1638: 103 pgs: 103 active+clean; 4.6 MiB data, 3.7 GiB used, 443 GiB / 447 GiB avail; 734 KiB/s rd, 570 KiB/s wr, 231 op/s 2023-09-23T21:20:04.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:20:03 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/8218457' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["6"]}]: dispatch 2023-09-23T21:20:04.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:20:03 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["6"]}]: dispatch 2023-09-23T21:20:05.171 INFO:tasks.thrashosds.thrasher:Added osd 6 2023-09-23T21:20:05.171 INFO:tasks.thrashosds.thrasher:inning osd 2023-09-23T21:20:05.172 INFO:tasks.thrashosds.thrasher:Adding osd 1 2023-09-23T21:20:05.172 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph osd in 1 2023-09-23T21:20:05.282 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:20:05 smithi026 ceph-mon[175151]: failed to encode map e672 with expected crc 2023-09-23T21:20:05.282 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:20:05 smithi026 ceph-mon[175151]: failed to encode map e672 with expected crc 2023-09-23T21:20:05.282 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:20:05 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["6"]}]': finished 2023-09-23T21:20:05.282 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:20:05 smithi026 ceph-mon[175151]: osdmap e672: 8 total, 8 up, 6 in 2023-09-23T21:20:05.282 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:20:05 smithi026 ceph-mon[175151]: failed to encode map e672 with expected crc 2023-09-23T21:20:05.282 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:20:05 smithi026 ceph-mon[179688]: failed to encode map e672 with expected crc 2023-09-23T21:20:05.282 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:20:05 smithi026 ceph-mon[179688]: failed to encode map e672 with expected crc 2023-09-23T21:20:05.283 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:20:05 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["6"]}]': finished 2023-09-23T21:20:05.283 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:20:05 smithi026 ceph-mon[179688]: osdmap e672: 8 total, 8 up, 6 in 2023-09-23T21:20:05.283 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:20:05 smithi026 ceph-mon[179688]: failed to encode map e672 with expected crc 2023-09-23T21:20:05.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:20:05 smithi070 ceph-mon[168282]: failed to encode map e672 with expected crc 2023-09-23T21:20:05.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:20:05 smithi070 ceph-mon[168282]: failed to encode map e672 with expected crc 2023-09-23T21:20:05.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:20:05 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["6"]}]': finished 2023-09-23T21:20:05.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:20:05 smithi070 ceph-mon[168282]: osdmap e672: 8 total, 8 up, 6 in 2023-09-23T21:20:05.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:20:05 smithi070 ceph-mon[168282]: failed to encode map e672 with expected crc 2023-09-23T21:20:05.483 INFO:teuthology.orchestra.run.smithi026.stdout:ERROR: (22) Invalid argument 2023-09-23T21:20:05.483 INFO:teuthology.orchestra.run.smithi026.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T21:20:06.017 INFO:teuthology.orchestra.run.smithi026.stderr:noscrub is unset 2023-09-23T21:20:06.086 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:20:06 smithi026 ceph-mon[175151]: failed to encode map e673 with expected crc 2023-09-23T21:20:06.086 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:20:06 smithi026 ceph-mon[175151]: failed to encode map e673 with expected crc 2023-09-23T21:20:06.086 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:20:06 smithi026 ceph-mon[175151]: failed to encode map e673 with expected crc 2023-09-23T21:20:06.086 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:20:06 smithi026 ceph-mon[175151]: osdmap e673: 8 total, 8 up, 6 in 2023-09-23T21:20:06.086 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:20:06 smithi026 ceph-mon[175151]: failed to encode map e673 with expected crc 2023-09-23T21:20:06.086 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:20:06 smithi026 ceph-mon[175151]: pgmap v1641: 103 pgs: 14 peering, 89 active+clean; 4.6 MiB data, 4.0 GiB used, 532 GiB / 536 GiB avail 2023-09-23T21:20:06.086 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:20:06 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/4134761782' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T21:20:06.087 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:20:06 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/4134761782' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T21:20:06.087 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:20:06 smithi026 ceph-mon[175151]: osdmap e674: 8 total, 8 up, 6 in 2023-09-23T21:20:06.087 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:20:06 smithi026 ceph-mon[179688]: failed to encode map e673 with expected crc 2023-09-23T21:20:06.087 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:20:06 smithi026 ceph-mon[179688]: failed to encode map e673 with expected crc 2023-09-23T21:20:06.087 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:20:06 smithi026 ceph-mon[179688]: failed to encode map e673 with expected crc 2023-09-23T21:20:06.087 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:20:06 smithi026 ceph-mon[179688]: osdmap e673: 8 total, 8 up, 6 in 2023-09-23T21:20:06.087 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:20:06 smithi026 ceph-mon[179688]: failed to encode map e673 with expected crc 2023-09-23T21:20:06.087 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:20:06 smithi026 ceph-mon[179688]: pgmap v1641: 103 pgs: 14 peering, 89 active+clean; 4.6 MiB data, 4.0 GiB used, 532 GiB / 536 GiB avail 2023-09-23T21:20:06.087 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:20:06 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/4134761782' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T21:20:06.088 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:20:06 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/4134761782' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T21:20:06.088 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:20:06 smithi026 ceph-mon[179688]: osdmap e674: 8 total, 8 up, 6 in 2023-09-23T21:20:06.089 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_historic_ops 2023-09-23T21:20:06.109 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:20:06.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:20:06 smithi070 ceph-mon[168282]: failed to encode map e673 with expected crc 2023-09-23T21:20:06.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:20:06 smithi070 ceph-mon[168282]: failed to encode map e673 with expected crc 2023-09-23T21:20:06.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:20:06 smithi070 ceph-mon[168282]: failed to encode map e673 with expected crc 2023-09-23T21:20:06.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:20:06 smithi070 ceph-mon[168282]: osdmap e673: 8 total, 8 up, 6 in 2023-09-23T21:20:06.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:20:06 smithi070 ceph-mon[168282]: failed to encode map e673 with expected crc 2023-09-23T21:20:06.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:20:06 smithi070 ceph-mon[168282]: pgmap v1641: 103 pgs: 14 peering, 89 active+clean; 4.6 MiB data, 4.0 GiB used, 532 GiB / 536 GiB avail 2023-09-23T21:20:06.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:20:06 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/4134761782' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T21:20:06.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:20:06 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/4134761782' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T21:20:06.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:20:06 smithi070 ceph-mon[168282]: osdmap e674: 8 total, 8 up, 6 in 2023-09-23T21:20:06.442 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T21:20:07.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:20:07 smithi026 ceph-mon[179688]: failed to encode map e674 with expected crc 2023-09-23T21:20:07.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:20:07 smithi026 ceph-mon[179688]: failed to encode map e674 with expected crc 2023-09-23T21:20:07.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:20:07 smithi026 ceph-mon[179688]: failed to encode map e674 with expected crc 2023-09-23T21:20:07.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:20:07 smithi026 ceph-mon[179688]: failed to encode map e674 with expected crc 2023-09-23T21:20:07.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:20:07 smithi026 ceph-mon[179688]: failed to encode map e674 with expected crc 2023-09-23T21:20:07.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:20:07 smithi026 ceph-mon[179688]: failed to encode map e674 with expected crc 2023-09-23T21:20:07.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:20:07 smithi026 ceph-mon[179688]: osdmap e675: 8 total, 8 up, 6 in 2023-09-23T21:20:07.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:20:07 smithi026 ceph-mon[175151]: failed to encode map e674 with expected crc 2023-09-23T21:20:07.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:20:07 smithi026 ceph-mon[175151]: failed to encode map e674 with expected crc 2023-09-23T21:20:07.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:20:07 smithi026 ceph-mon[175151]: failed to encode map e674 with expected crc 2023-09-23T21:20:07.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:20:07 smithi026 ceph-mon[175151]: failed to encode map e674 with expected crc 2023-09-23T21:20:07.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:20:07 smithi026 ceph-mon[175151]: failed to encode map e674 with expected crc 2023-09-23T21:20:07.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:20:07 smithi026 ceph-mon[175151]: failed to encode map e674 with expected crc 2023-09-23T21:20:07.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:20:07 smithi026 ceph-mon[175151]: osdmap e675: 8 total, 8 up, 6 in 2023-09-23T21:20:07.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:20:07 smithi070 ceph-mon[168282]: failed to encode map e674 with expected crc 2023-09-23T21:20:07.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:20:07 smithi070 ceph-mon[168282]: failed to encode map e674 with expected crc 2023-09-23T21:20:07.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:20:07 smithi070 ceph-mon[168282]: failed to encode map e674 with expected crc 2023-09-23T21:20:07.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:20:07 smithi070 ceph-mon[168282]: failed to encode map e674 with expected crc 2023-09-23T21:20:07.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:20:07 smithi070 ceph-mon[168282]: failed to encode map e674 with expected crc 2023-09-23T21:20:07.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:20:07 smithi070 ceph-mon[168282]: failed to encode map e674 with expected crc 2023-09-23T21:20:07.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:20:07 smithi070 ceph-mon[168282]: osdmap e675: 8 total, 8 up, 6 in 2023-09-23T21:20:08.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:20:08 smithi026 ceph-mon[175151]: failed to encode map e674 with expected crc 2023-09-23T21:20:08.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:20:08 smithi026 ceph-mon[175151]: failed to encode map e675 with expected crc 2023-09-23T21:20:08.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:20:08 smithi026 ceph-mon[175151]: failed to encode map e675 with expected crc 2023-09-23T21:20:08.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:20:08 smithi026 ceph-mon[175151]: failed to encode map e675 with expected crc 2023-09-23T21:20:08.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:20:08 smithi026 ceph-mon[175151]: failed to encode map e675 with expected crc 2023-09-23T21:20:08.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:20:08 smithi026 ceph-mon[175151]: pgmap v1644: 103 pgs: 1 active+undersized+remapped, 7 remapped+peering, 14 peering, 3 active+remapped, 78 active+clean; 4.6 MiB data, 4.0 GiB used, 532 GiB / 536 GiB avail; 1.3 KiB/s, 0 objects/s recovering 2023-09-23T21:20:08.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:20:08 smithi026 ceph-mon[175151]: osdmap e676: 8 total, 8 up, 6 in 2023-09-23T21:20:08.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:20:08 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:20:08.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:20:08 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:20:08.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:20:08 smithi026 ceph-mon[179688]: failed to encode map e674 with expected crc 2023-09-23T21:20:08.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:20:08 smithi026 ceph-mon[179688]: failed to encode map e675 with expected crc 2023-09-23T21:20:08.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:20:08 smithi026 ceph-mon[179688]: failed to encode map e675 with expected crc 2023-09-23T21:20:08.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:20:08 smithi026 ceph-mon[179688]: failed to encode map e675 with expected crc 2023-09-23T21:20:08.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:20:08 smithi026 ceph-mon[179688]: failed to encode map e675 with expected crc 2023-09-23T21:20:08.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:20:08 smithi026 ceph-mon[179688]: pgmap v1644: 103 pgs: 1 active+undersized+remapped, 7 remapped+peering, 14 peering, 3 active+remapped, 78 active+clean; 4.6 MiB data, 4.0 GiB used, 532 GiB / 536 GiB avail; 1.3 KiB/s, 0 objects/s recovering 2023-09-23T21:20:08.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:20:08 smithi026 ceph-mon[179688]: osdmap e676: 8 total, 8 up, 6 in 2023-09-23T21:20:08.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:20:08 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:20:08.133 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:20:08 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:20:08.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:20:08 smithi070 ceph-mon[168282]: failed to encode map e674 with expected crc 2023-09-23T21:20:08.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:20:08 smithi070 ceph-mon[168282]: failed to encode map e675 with expected crc 2023-09-23T21:20:08.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:20:08 smithi070 ceph-mon[168282]: failed to encode map e675 with expected crc 2023-09-23T21:20:08.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:20:08 smithi070 ceph-mon[168282]: failed to encode map e675 with expected crc 2023-09-23T21:20:08.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:20:08 smithi070 ceph-mon[168282]: failed to encode map e675 with expected crc 2023-09-23T21:20:08.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:20:08 smithi070 ceph-mon[168282]: pgmap v1644: 103 pgs: 1 active+undersized+remapped, 7 remapped+peering, 14 peering, 3 active+remapped, 78 active+clean; 4.6 MiB data, 4.0 GiB used, 532 GiB / 536 GiB avail; 1.3 KiB/s, 0 objects/s recovering 2023-09-23T21:20:08.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:20:08 smithi070 ceph-mon[168282]: osdmap e676: 8 total, 8 up, 6 in 2023-09-23T21:20:08.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:20:08 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:20:08.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:20:08 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:20:09.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:20:09 smithi070 ceph-mon[168282]: failed to encode map e676 with expected crc 2023-09-23T21:20:09.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:20:09 smithi070 ceph-mon[168282]: failed to encode map e676 with expected crc 2023-09-23T21:20:09.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:20:09 smithi070 ceph-mon[168282]: failed to encode map e676 with expected crc 2023-09-23T21:20:09.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:20:09 smithi070 ceph-mon[168282]: failed to encode map e676 with expected crc 2023-09-23T21:20:09.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:20:09 smithi070 ceph-mon[168282]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2023-09-23T21:20:09.381 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:20:09 smithi026 ceph-mon[175151]: failed to encode map e676 with expected crc 2023-09-23T21:20:09.381 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:20:09 smithi026 ceph-mon[175151]: failed to encode map e676 with expected crc 2023-09-23T21:20:09.381 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:20:09 smithi026 ceph-mon[175151]: failed to encode map e676 with expected crc 2023-09-23T21:20:09.381 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:20:09 smithi026 ceph-mon[175151]: failed to encode map e676 with expected crc 2023-09-23T21:20:09.381 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:20:09 smithi026 ceph-mon[175151]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2023-09-23T21:20:09.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:20:09 smithi026 ceph-mon[179688]: failed to encode map e676 with expected crc 2023-09-23T21:20:09.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:20:09 smithi026 ceph-mon[179688]: failed to encode map e676 with expected crc 2023-09-23T21:20:09.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:20:09 smithi026 ceph-mon[179688]: failed to encode map e676 with expected crc 2023-09-23T21:20:09.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:20:09 smithi026 ceph-mon[179688]: failed to encode map e676 with expected crc 2023-09-23T21:20:09.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:20:09 smithi026 ceph-mon[179688]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2023-09-23T21:20:09.382 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:20:09.866 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:20:09.881 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 21:20:09 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[164029]: ::ffff:172.21.15.70 - - [23/Sep/2023:21:20:09] "GET /metrics HTTP/1.1" 200 34002 "" "Prometheus/2.43.0" 2023-09-23T21:20:10.263 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:20:10 smithi026 ceph-mon[179688]: pgmap v1646: 103 pgs: 1 active+undersized+remapped, 7 remapped+peering, 14 peering, 3 active+remapped, 78 active+clean; 4.6 MiB data, 4.0 GiB used, 532 GiB / 536 GiB avail; 1.2 KiB/s, 1 objects/s recovering 2023-09-23T21:20:10.263 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:20:10 smithi026 ceph-mon[175151]: pgmap v1646: 103 pgs: 1 active+undersized+remapped, 7 remapped+peering, 14 peering, 3 active+remapped, 78 active+clean; 4.6 MiB data, 4.0 GiB used, 532 GiB / 536 GiB avail; 1.2 KiB/s, 1 objects/s recovering 2023-09-23T21:20:10.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:20:10 smithi070 ceph-mon[168282]: pgmap v1646: 103 pgs: 1 active+undersized+remapped, 7 remapped+peering, 14 peering, 3 active+remapped, 78 active+clean; 4.6 MiB data, 4.0 GiB used, 532 GiB / 536 GiB avail; 1.2 KiB/s, 1 objects/s recovering 2023-09-23T21:20:10.599 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:20:10 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mon-a[175127]: 2023-09-23T21:20:10.432+0000 7f1e2d487700 -1 mon.a@0(leader).osd e676 definitely_dead 0 2023-09-23T21:20:11.069 INFO:teuthology.orchestra.run.smithi026.stderr:marked in osd.1. 2023-09-23T21:20:11.174 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:20:11 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/185244045' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["1"]}]: dispatch 2023-09-23T21:20:11.174 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:20:11 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["1"]}]: dispatch 2023-09-23T21:20:11.174 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:20:11 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/185244045' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["1"]}]: dispatch 2023-09-23T21:20:11.175 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:20:11 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["1"]}]: dispatch 2023-09-23T21:20:11.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:20:11 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/185244045' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["1"]}]: dispatch 2023-09-23T21:20:11.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:20:11 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["1"]}]: dispatch 2023-09-23T21:20:11.635 INFO:tasks.thrashosds.thrasher:Added osd 1 2023-09-23T21:20:11.635 INFO:tasks.thrashosds.thrasher:inning osd 2023-09-23T21:20:11.635 INFO:tasks.thrashosds.thrasher:Adding osd 5 2023-09-23T21:20:11.636 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph osd in 5 2023-09-23T21:20:12.144 INFO:teuthology.orchestra.run.smithi026.stdout:ERROR: (22) Invalid argument 2023-09-23T21:20:12.144 INFO:teuthology.orchestra.run.smithi026.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T21:20:12.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:20:12 smithi070 ceph-mon[168282]: failed to encode map e677 with expected crc 2023-09-23T21:20:12.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:20:12 smithi070 ceph-mon[168282]: failed to encode map e677 with expected crc 2023-09-23T21:20:12.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:20:12 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["1"]}]': finished 2023-09-23T21:20:12.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:20:12 smithi070 ceph-mon[168282]: osdmap e677: 8 total, 8 up, 7 in 2023-09-23T21:20:12.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:20:12 smithi070 ceph-mon[168282]: failed to encode map e677 with expected crc 2023-09-23T21:20:12.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:20:12 smithi070 ceph-mon[168282]: failed to encode map e677 with expected crc 2023-09-23T21:20:12.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:20:12 smithi070 ceph-mon[168282]: pgmap v1648: 103 pgs: 1 active+undersized+remapped, 102 active+clean; 4.6 MiB data, 4.0 GiB used, 532 GiB / 536 GiB avail; 367 KiB/s rd, 253 KiB/s wr, 95 op/s; 62 KiB/s, 8 objects/s recovering 2023-09-23T21:20:12.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:20:12 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:20:12.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:20:12 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:20:12.381 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:20:12 smithi026 ceph-mon[175151]: failed to encode map e677 with expected crc 2023-09-23T21:20:12.381 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:20:12 smithi026 ceph-mon[175151]: failed to encode map e677 with expected crc 2023-09-23T21:20:12.381 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:20:12 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["1"]}]': finished 2023-09-23T21:20:12.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:20:12 smithi026 ceph-mon[175151]: osdmap e677: 8 total, 8 up, 7 in 2023-09-23T21:20:12.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:20:12 smithi026 ceph-mon[175151]: failed to encode map e677 with expected crc 2023-09-23T21:20:12.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:20:12 smithi026 ceph-mon[175151]: failed to encode map e677 with expected crc 2023-09-23T21:20:12.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:20:12 smithi026 ceph-mon[175151]: pgmap v1648: 103 pgs: 1 active+undersized+remapped, 102 active+clean; 4.6 MiB data, 4.0 GiB used, 532 GiB / 536 GiB avail; 367 KiB/s rd, 253 KiB/s wr, 95 op/s; 62 KiB/s, 8 objects/s recovering 2023-09-23T21:20:12.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:20:12 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:20:12.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:20:12 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:20:12.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:20:12 smithi026 ceph-mon[179688]: failed to encode map e677 with expected crc 2023-09-23T21:20:12.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:20:12 smithi026 ceph-mon[179688]: failed to encode map e677 with expected crc 2023-09-23T21:20:12.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:20:12 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["1"]}]': finished 2023-09-23T21:20:12.383 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:20:12 smithi026 ceph-mon[179688]: osdmap e677: 8 total, 8 up, 7 in 2023-09-23T21:20:12.383 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:20:12 smithi026 ceph-mon[179688]: failed to encode map e677 with expected crc 2023-09-23T21:20:12.383 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:20:12 smithi026 ceph-mon[179688]: failed to encode map e677 with expected crc 2023-09-23T21:20:12.383 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:20:12 smithi026 ceph-mon[179688]: pgmap v1648: 103 pgs: 1 active+undersized+remapped, 102 active+clean; 4.6 MiB data, 4.0 GiB used, 532 GiB / 536 GiB avail; 367 KiB/s rd, 253 KiB/s wr, 95 op/s; 62 KiB/s, 8 objects/s recovering 2023-09-23T21:20:12.383 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:20:12 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:20:12.383 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:20:12 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:20:12.671 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:20:13.068 INFO:teuthology.orchestra.run.smithi026.stderr:nodeep-scrub is unset 2023-09-23T21:20:13.190 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:20:13 smithi026 ceph-mon[175151]: failed to encode map e677 with expected crc 2023-09-23T21:20:13.190 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:20:13 smithi026 ceph-mon[175151]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2023-09-23T21:20:13.190 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:20:13 smithi026 ceph-mon[175151]: failed to encode map e678 with expected crc 2023-09-23T21:20:13.190 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:20:13 smithi026 ceph-mon[175151]: failed to encode map e678 with expected crc 2023-09-23T21:20:13.190 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:20:13 smithi026 ceph-mon[175151]: failed to encode map e678 with expected crc 2023-09-23T21:20:13.190 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:20:13 smithi026 ceph-mon[175151]: osdmap e678: 8 total, 8 up, 7 in 2023-09-23T21:20:13.190 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:20:13 smithi026 ceph-mon[175151]: failed to encode map e678 with expected crc 2023-09-23T21:20:13.191 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:20:13 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/2942479045' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:20:13.191 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:20:13 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T21:20:13.191 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:20:13 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T21:20:13.191 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:20:13 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:20:13.191 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:20:13 smithi026 ceph-mon[179688]: failed to encode map e677 with expected crc 2023-09-23T21:20:13.191 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:20:13 smithi026 ceph-mon[179688]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2023-09-23T21:20:13.191 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:20:13 smithi026 ceph-mon[179688]: failed to encode map e678 with expected crc 2023-09-23T21:20:13.191 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:20:13 smithi026 ceph-mon[179688]: failed to encode map e678 with expected crc 2023-09-23T21:20:13.191 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:20:13 smithi026 ceph-mon[179688]: failed to encode map e678 with expected crc 2023-09-23T21:20:13.192 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:20:13 smithi026 ceph-mon[179688]: osdmap e678: 8 total, 8 up, 7 in 2023-09-23T21:20:13.192 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:20:13 smithi026 ceph-mon[179688]: failed to encode map e678 with expected crc 2023-09-23T21:20:13.192 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:20:13 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/2942479045' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:20:13.192 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:20:13 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T21:20:13.192 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:20:13 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T21:20:13.192 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:20:13 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:20:13.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:20:13 smithi070 ceph-mon[168282]: failed to encode map e677 with expected crc 2023-09-23T21:20:13.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:20:13 smithi070 ceph-mon[168282]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2023-09-23T21:20:13.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:20:13 smithi070 ceph-mon[168282]: failed to encode map e678 with expected crc 2023-09-23T21:20:13.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:20:13 smithi070 ceph-mon[168282]: failed to encode map e678 with expected crc 2023-09-23T21:20:13.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:20:13 smithi070 ceph-mon[168282]: failed to encode map e678 with expected crc 2023-09-23T21:20:13.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:20:13 smithi070 ceph-mon[168282]: osdmap e678: 8 total, 8 up, 7 in 2023-09-23T21:20:13.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:20:13 smithi070 ceph-mon[168282]: failed to encode map e678 with expected crc 2023-09-23T21:20:13.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:20:13 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/2942479045' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:20:13.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:20:13 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T21:20:13.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:20:13 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T21:20:13.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:20:13 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:20:14.326 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:20:14 smithi026 ceph-mon[175151]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T21:20:14.326 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:20:14 smithi026 ceph-mon[175151]: Cluster is now healthy 2023-09-23T21:20:14.327 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:20:14 smithi026 ceph-mon[175151]: failed to encode map e679 with expected crc 2023-09-23T21:20:14.327 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:20:14 smithi026 ceph-mon[175151]: failed to encode map e679 with expected crc 2023-09-23T21:20:14.327 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:20:14 smithi026 ceph-mon[175151]: failed to encode map e679 with expected crc 2023-09-23T21:20:14.327 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:20:14 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/2942479045' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T21:20:14.327 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:20:14 smithi026 ceph-mon[175151]: osdmap e679: 8 total, 8 up, 7 in 2023-09-23T21:20:14.327 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:20:14 smithi026 ceph-mon[175151]: failed to encode map e679 with expected crc 2023-09-23T21:20:14.327 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:20:14 smithi026 ceph-mon[175151]: pgmap v1651: 103 pgs: 1 active+recovering+degraded, 3 remapped+peering, 4 peering, 95 active+clean; 4.6 MiB data, 4.0 GiB used, 532 GiB / 536 GiB avail; 368 KiB/s rd, 253 KiB/s wr, 95 op/s; 1/84 objects misplaced (1.190%); 61 KiB/s, 7 objects/s recovering 2023-09-23T21:20:14.327 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:20:14 smithi026 ceph-mon[179688]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T21:20:14.327 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:20:14 smithi026 ceph-mon[179688]: Cluster is now healthy 2023-09-23T21:20:14.328 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:20:14 smithi026 ceph-mon[179688]: failed to encode map e679 with expected crc 2023-09-23T21:20:14.328 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:20:14 smithi026 ceph-mon[179688]: failed to encode map e679 with expected crc 2023-09-23T21:20:14.328 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:20:14 smithi026 ceph-mon[179688]: failed to encode map e679 with expected crc 2023-09-23T21:20:14.328 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:20:14 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/2942479045' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T21:20:14.328 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:20:14 smithi026 ceph-mon[179688]: osdmap e679: 8 total, 8 up, 7 in 2023-09-23T21:20:14.328 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:20:14 smithi026 ceph-mon[179688]: failed to encode map e679 with expected crc 2023-09-23T21:20:14.328 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:20:14 smithi026 ceph-mon[179688]: pgmap v1651: 103 pgs: 1 active+recovering+degraded, 3 remapped+peering, 4 peering, 95 active+clean; 4.6 MiB data, 4.0 GiB used, 532 GiB / 536 GiB avail; 368 KiB/s rd, 253 KiB/s wr, 95 op/s; 1/84 objects misplaced (1.190%); 61 KiB/s, 7 objects/s recovering 2023-09-23T21:20:14.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:20:14 smithi070 ceph-mon[168282]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T21:20:14.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:20:14 smithi070 ceph-mon[168282]: Cluster is now healthy 2023-09-23T21:20:14.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:20:14 smithi070 ceph-mon[168282]: failed to encode map e679 with expected crc 2023-09-23T21:20:14.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:20:14 smithi070 ceph-mon[168282]: failed to encode map e679 with expected crc 2023-09-23T21:20:14.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:20:14 smithi070 ceph-mon[168282]: failed to encode map e679 with expected crc 2023-09-23T21:20:14.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:20:14 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/2942479045' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T21:20:14.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:20:14 smithi070 ceph-mon[168282]: osdmap e679: 8 total, 8 up, 7 in 2023-09-23T21:20:14.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:20:14 smithi070 ceph-mon[168282]: failed to encode map e679 with expected crc 2023-09-23T21:20:14.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:20:14 smithi070 ceph-mon[168282]: pgmap v1651: 103 pgs: 1 active+recovering+degraded, 3 remapped+peering, 4 peering, 95 active+clean; 4.6 MiB data, 4.0 GiB used, 532 GiB / 536 GiB avail; 368 KiB/s rd, 253 KiB/s wr, 95 op/s; 1/84 objects misplaced (1.190%); 61 KiB/s, 7 objects/s recovering 2023-09-23T21:20:14.631 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:20:14 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mon-a[175127]: 2023-09-23T21:20:14.323+0000 7f1e2d487700 -1 mon.a@0(leader).osd e680 definitely_dead 0 2023-09-23T21:20:15.084 INFO:teuthology.orchestra.run.smithi026.stderr:marked in osd.5. 2023-09-23T21:20:15.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:20:15 smithi070 ceph-mon[168282]: Health check failed: Degraded data redundancy: 1 pg degraded (PG_DEGRADED) 2023-09-23T21:20:15.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:20:15 smithi070 ceph-mon[168282]: failed to encode map e680 with expected crc 2023-09-23T21:20:15.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:20:15 smithi070 ceph-mon[168282]: failed to encode map e680 with expected crc 2023-09-23T21:20:15.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:20:15 smithi070 ceph-mon[168282]: osdmap e680: 8 total, 8 up, 7 in 2023-09-23T21:20:15.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:20:15 smithi070 ceph-mon[168282]: failed to encode map e680 with expected crc 2023-09-23T21:20:15.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:20:15 smithi070 ceph-mon[168282]: failed to encode map e680 with expected crc 2023-09-23T21:20:15.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:20:15 smithi070 ceph-mon[168282]: failed to encode map e680 with expected crc 2023-09-23T21:20:15.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:20:15 smithi070 ceph-mon[168282]: failed to encode map e680 with expected crc 2023-09-23T21:20:15.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:20:15 smithi070 ceph-mon[168282]: failed to encode map e680 with expected crc 2023-09-23T21:20:15.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:20:15 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/2817764667' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["5"]}]: dispatch 2023-09-23T21:20:15.381 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:20:15 smithi026 ceph-mon[175151]: Health check failed: Degraded data redundancy: 1 pg degraded (PG_DEGRADED) 2023-09-23T21:20:15.381 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:20:15 smithi026 ceph-mon[175151]: failed to encode map e680 with expected crc 2023-09-23T21:20:15.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:20:15 smithi026 ceph-mon[175151]: failed to encode map e680 with expected crc 2023-09-23T21:20:15.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:20:15 smithi026 ceph-mon[175151]: osdmap e680: 8 total, 8 up, 7 in 2023-09-23T21:20:15.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:20:15 smithi026 ceph-mon[175151]: failed to encode map e680 with expected crc 2023-09-23T21:20:15.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:20:15 smithi026 ceph-mon[175151]: failed to encode map e680 with expected crc 2023-09-23T21:20:15.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:20:15 smithi026 ceph-mon[175151]: failed to encode map e680 with expected crc 2023-09-23T21:20:15.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:20:15 smithi026 ceph-mon[175151]: failed to encode map e680 with expected crc 2023-09-23T21:20:15.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:20:15 smithi026 ceph-mon[175151]: failed to encode map e680 with expected crc 2023-09-23T21:20:15.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:20:15 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/2817764667' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["5"]}]: dispatch 2023-09-23T21:20:15.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:20:15 smithi026 ceph-mon[179688]: Health check failed: Degraded data redundancy: 1 pg degraded (PG_DEGRADED) 2023-09-23T21:20:15.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:20:15 smithi026 ceph-mon[179688]: failed to encode map e680 with expected crc 2023-09-23T21:20:15.383 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:20:15 smithi026 ceph-mon[179688]: failed to encode map e680 with expected crc 2023-09-23T21:20:15.383 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:20:15 smithi026 ceph-mon[179688]: osdmap e680: 8 total, 8 up, 7 in 2023-09-23T21:20:15.383 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:20:15 smithi026 ceph-mon[179688]: failed to encode map e680 with expected crc 2023-09-23T21:20:15.383 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:20:15 smithi026 ceph-mon[179688]: failed to encode map e680 with expected crc 2023-09-23T21:20:15.383 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:20:15 smithi026 ceph-mon[179688]: failed to encode map e680 with expected crc 2023-09-23T21:20:15.383 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:20:15 smithi026 ceph-mon[179688]: failed to encode map e680 with expected crc 2023-09-23T21:20:15.383 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:20:15 smithi026 ceph-mon[179688]: failed to encode map e680 with expected crc 2023-09-23T21:20:15.383 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:20:15 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/2817764667' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["5"]}]: dispatch 2023-09-23T21:20:15.551 INFO:tasks.thrashosds.thrasher:Added osd 5 2023-09-23T21:20:15.551 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph -- tell 'mon.*' injectargs --mon_osd_down_out_interval=600 2023-09-23T21:20:16.181 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:20:16.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:20:16 smithi070 ceph-mon[168282]: failed to encode map e681 with expected crc 2023-09-23T21:20:16.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:20:16 smithi070 ceph-mon[168282]: failed to encode map e681 with expected crc 2023-09-23T21:20:16.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:20:16 smithi070 ceph-mon[168282]: failed to encode map e681 with expected crc 2023-09-23T21:20:16.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:20:16 smithi070 ceph-mon[168282]: failed to encode map e681 with expected crc 2023-09-23T21:20:16.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:20:16 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/2817764667' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["5"]}]': finished 2023-09-23T21:20:16.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:20:16 smithi070 ceph-mon[168282]: osdmap e681: 8 total, 8 up, 8 in 2023-09-23T21:20:16.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:20:16 smithi070 ceph-mon[168282]: failed to encode map e681 with expected crc 2023-09-23T21:20:16.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:20:16 smithi070 ceph-mon[168282]: failed to encode map e681 with expected crc 2023-09-23T21:20:16.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:20:16 smithi070 ceph-mon[168282]: failed to encode map e681 with expected crc 2023-09-23T21:20:16.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:20:16 smithi070 ceph-mon[168282]: pgmap v1654: 103 pgs: 1 active+recovering+degraded, 3 remapped+peering, 5 peering, 94 active+clean; 4.6 MiB data, 4.3 GiB used, 621 GiB / 626 GiB avail; 1/84 objects misplaced (1.190%); 1024 KiB/s, 4 objects/s recovering 2023-09-23T21:20:16.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:20:16 smithi070 ceph-mon[168282]: osdmap e682: 8 total, 8 up, 8 in 2023-09-23T21:20:16.381 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:20:16 smithi026 ceph-mon[179688]: failed to encode map e681 with expected crc 2023-09-23T21:20:16.381 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:20:16 smithi026 ceph-mon[179688]: failed to encode map e681 with expected crc 2023-09-23T21:20:16.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:20:16 smithi026 ceph-mon[179688]: failed to encode map e681 with expected crc 2023-09-23T21:20:16.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:20:16 smithi026 ceph-mon[179688]: failed to encode map e681 with expected crc 2023-09-23T21:20:16.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:20:16 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/2817764667' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["5"]}]': finished 2023-09-23T21:20:16.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:20:16 smithi026 ceph-mon[179688]: osdmap e681: 8 total, 8 up, 8 in 2023-09-23T21:20:16.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:20:16 smithi026 ceph-mon[179688]: failed to encode map e681 with expected crc 2023-09-23T21:20:16.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:20:16 smithi026 ceph-mon[179688]: failed to encode map e681 with expected crc 2023-09-23T21:20:16.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:20:16 smithi026 ceph-mon[179688]: failed to encode map e681 with expected crc 2023-09-23T21:20:16.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:20:16 smithi026 ceph-mon[179688]: pgmap v1654: 103 pgs: 1 active+recovering+degraded, 3 remapped+peering, 5 peering, 94 active+clean; 4.6 MiB data, 4.3 GiB used, 621 GiB / 626 GiB avail; 1/84 objects misplaced (1.190%); 1024 KiB/s, 4 objects/s recovering 2023-09-23T21:20:16.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:20:16 smithi026 ceph-mon[179688]: osdmap e682: 8 total, 8 up, 8 in 2023-09-23T21:20:16.383 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:20:16 smithi026 ceph-mon[175151]: failed to encode map e681 with expected crc 2023-09-23T21:20:16.383 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:20:16 smithi026 ceph-mon[175151]: failed to encode map e681 with expected crc 2023-09-23T21:20:16.383 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:20:16 smithi026 ceph-mon[175151]: failed to encode map e681 with expected crc 2023-09-23T21:20:16.383 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:20:16 smithi026 ceph-mon[175151]: failed to encode map e681 with expected crc 2023-09-23T21:20:16.383 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:20:16 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/2817764667' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["5"]}]': finished 2023-09-23T21:20:16.383 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:20:16 smithi026 ceph-mon[175151]: osdmap e681: 8 total, 8 up, 8 in 2023-09-23T21:20:16.383 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:20:16 smithi026 ceph-mon[175151]: failed to encode map e681 with expected crc 2023-09-23T21:20:16.384 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:20:16 smithi026 ceph-mon[175151]: failed to encode map e681 with expected crc 2023-09-23T21:20:16.384 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:20:16 smithi026 ceph-mon[175151]: failed to encode map e681 with expected crc 2023-09-23T21:20:16.384 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:20:16 smithi026 ceph-mon[175151]: pgmap v1654: 103 pgs: 1 active+recovering+degraded, 3 remapped+peering, 5 peering, 94 active+clean; 4.6 MiB data, 4.3 GiB used, 621 GiB / 626 GiB avail; 1/84 objects misplaced (1.190%); 1024 KiB/s, 4 objects/s recovering 2023-09-23T21:20:16.384 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:20:16 smithi026 ceph-mon[175151]: osdmap e682: 8 total, 8 up, 8 in 2023-09-23T21:20:17.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:20:17 smithi070 ceph-mon[168282]: failed to encode map e682 with expected crc 2023-09-23T21:20:17.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:20:17 smithi070 ceph-mon[168282]: failed to encode map e682 with expected crc 2023-09-23T21:20:17.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:20:17 smithi070 ceph-mon[168282]: failed to encode map e682 with expected crc 2023-09-23T21:20:17.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:20:17 smithi070 ceph-mon[168282]: failed to encode map e682 with expected crc 2023-09-23T21:20:17.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:20:17 smithi070 ceph-mon[168282]: failed to encode map e682 with expected crc 2023-09-23T21:20:17.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:20:17 smithi070 ceph-mon[168282]: failed to encode map e682 with expected crc 2023-09-23T21:20:17.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:20:17 smithi070 ceph-mon[168282]: failed to encode map e682 with expected crc 2023-09-23T21:20:17.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:20:17 smithi070 ceph-mon[168282]: failed to encode map e682 with expected crc 2023-09-23T21:20:17.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:20:17 smithi070 ceph-mon[168282]: failed to encode map e682 with expected crc 2023-09-23T21:20:17.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:20:17 smithi070 ceph-mon[168282]: failed to encode map e682 with expected crc 2023-09-23T21:20:17.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:20:17 smithi070 ceph-mon[168282]: failed to encode map e682 with expected crc 2023-09-23T21:20:17.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:20:17 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:20:17.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:20:17 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:20:17.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:20:17 smithi070 ceph-mon[168282]: osdmap e683: 8 total, 8 up, 8 in 2023-09-23T21:20:17.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:20:17 smithi026 ceph-mon[175151]: failed to encode map e682 with expected crc 2023-09-23T21:20:17.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:20:17 smithi026 ceph-mon[175151]: failed to encode map e682 with expected crc 2023-09-23T21:20:17.383 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:20:17 smithi026 ceph-mon[175151]: failed to encode map e682 with expected crc 2023-09-23T21:20:17.383 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:20:17 smithi026 ceph-mon[175151]: failed to encode map e682 with expected crc 2023-09-23T21:20:17.383 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:20:17 smithi026 ceph-mon[175151]: failed to encode map e682 with expected crc 2023-09-23T21:20:17.383 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:20:17 smithi026 ceph-mon[175151]: failed to encode map e682 with expected crc 2023-09-23T21:20:17.384 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:20:17 smithi026 ceph-mon[175151]: failed to encode map e682 with expected crc 2023-09-23T21:20:17.384 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:20:17 smithi026 ceph-mon[175151]: failed to encode map e682 with expected crc 2023-09-23T21:20:17.384 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:20:17 smithi026 ceph-mon[175151]: failed to encode map e682 with expected crc 2023-09-23T21:20:17.384 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:20:17 smithi026 ceph-mon[175151]: failed to encode map e682 with expected crc 2023-09-23T21:20:17.385 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:20:17 smithi026 ceph-mon[175151]: failed to encode map e682 with expected crc 2023-09-23T21:20:17.385 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:20:17 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:20:17.385 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:20:17 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:20:17.385 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:20:17 smithi026 ceph-mon[175151]: osdmap e683: 8 total, 8 up, 8 in 2023-09-23T21:20:17.385 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:20:17 smithi026 ceph-mon[179688]: failed to encode map e682 with expected crc 2023-09-23T21:20:17.386 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:20:17 smithi026 ceph-mon[179688]: failed to encode map e682 with expected crc 2023-09-23T21:20:17.386 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:20:17 smithi026 ceph-mon[179688]: failed to encode map e682 with expected crc 2023-09-23T21:20:17.386 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:20:17 smithi026 ceph-mon[179688]: failed to encode map e682 with expected crc 2023-09-23T21:20:17.386 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:20:17 smithi026 ceph-mon[179688]: failed to encode map e682 with expected crc 2023-09-23T21:20:17.386 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:20:17 smithi026 ceph-mon[179688]: failed to encode map e682 with expected crc 2023-09-23T21:20:17.387 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:20:17 smithi026 ceph-mon[179688]: failed to encode map e682 with expected crc 2023-09-23T21:20:17.387 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:20:17 smithi026 ceph-mon[179688]: failed to encode map e682 with expected crc 2023-09-23T21:20:17.387 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:20:17 smithi026 ceph-mon[179688]: failed to encode map e682 with expected crc 2023-09-23T21:20:17.387 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:20:17 smithi026 ceph-mon[179688]: failed to encode map e682 with expected crc 2023-09-23T21:20:17.387 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:20:17 smithi026 ceph-mon[179688]: failed to encode map e682 with expected crc 2023-09-23T21:20:17.387 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:20:17 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:20:17.388 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:20:17 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:20:17.388 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:20:17 smithi026 ceph-mon[179688]: osdmap e683: 8 total, 8 up, 8 in 2023-09-23T21:20:17.431 INFO:teuthology.orchestra.run.smithi026.stdout:mon.a: {} 2023-09-23T21:20:17.442 INFO:teuthology.orchestra.run.smithi026.stdout:mon.b: {} 2023-09-23T21:20:17.452 INFO:teuthology.orchestra.run.smithi026.stdout:mon.c: {} 2023-09-23T21:20:17.454 INFO:teuthology.orchestra.run.smithi026.stderr:mon.a: mon_osd_down_out_interval = '600' (not observed, change may require restart) 2023-09-23T21:20:17.454 INFO:teuthology.orchestra.run.smithi026.stderr:mon.b: mon_osd_down_out_interval = '600' (not observed, change may require restart) 2023-09-23T21:20:17.454 INFO:teuthology.orchestra.run.smithi026.stderr:mon.c: mon_osd_down_out_interval = '600' (not observed, change may require restart) 2023-09-23T21:20:17.844 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph osd reweight 2 1 2023-09-23T21:20:18.067 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:20:18 smithi026 ceph-mon[175151]: failed to encode map e683 with expected crc 2023-09-23T21:20:18.068 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:20:18 smithi026 ceph-mon[175151]: failed to encode map e683 with expected crc 2023-09-23T21:20:18.070 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:20:18 smithi026 ceph-mon[179688]: failed to encode map e683 with expected crc 2023-09-23T21:20:18.070 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:20:18 smithi026 ceph-mon[179688]: failed to encode map e683 with expected crc 2023-09-23T21:20:18.070 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:20:18 smithi026 ceph-mon[179688]: failed to encode map e683 with expected crc 2023-09-23T21:20:18.070 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:20:18 smithi026 ceph-mon[179688]: failed to encode map e683 with expected crc 2023-09-23T21:20:18.070 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:20:18 smithi026 ceph-mon[179688]: failed to encode map e683 with expected crc 2023-09-23T21:20:18.070 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:20:18 smithi026 ceph-mon[179688]: failed to encode map e683 with expected crc 2023-09-23T21:20:18.071 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:20:18 smithi026 ceph-mon[179688]: pgmap v1657: 103 pgs: 2 activating+undersized+remapped, 6 remapped+peering, 7 peering, 88 active+clean; 4.6 MiB data, 4.3 GiB used, 621 GiB / 626 GiB avail; 1.1 MiB/s, 5 objects/s recovering 2023-09-23T21:20:18.071 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:20:18 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/2440411264' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-09-23T21:20:18.071 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:20:18 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:20:18.361 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:20:18.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:20:18 smithi070 ceph-mon[168282]: failed to encode map e683 with expected crc 2023-09-23T21:20:18.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:20:18 smithi070 ceph-mon[168282]: failed to encode map e683 with expected crc 2023-09-23T21:20:18.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:20:18 smithi070 ceph-mon[168282]: failed to encode map e683 with expected crc 2023-09-23T21:20:18.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:20:18 smithi070 ceph-mon[168282]: failed to encode map e683 with expected crc 2023-09-23T21:20:18.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:20:18 smithi070 ceph-mon[168282]: failed to encode map e683 with expected crc 2023-09-23T21:20:18.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:20:18 smithi070 ceph-mon[168282]: failed to encode map e683 with expected crc 2023-09-23T21:20:18.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:20:18 smithi070 ceph-mon[168282]: pgmap v1657: 103 pgs: 2 activating+undersized+remapped, 6 remapped+peering, 7 peering, 88 active+clean; 4.6 MiB data, 4.3 GiB used, 621 GiB / 626 GiB avail; 1.1 MiB/s, 5 objects/s recovering 2023-09-23T21:20:18.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:20:18 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/2440411264' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-09-23T21:20:18.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:20:18 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:20:18.381 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:20:18 smithi026 ceph-mon[175151]: failed to encode map e683 with expected crc 2023-09-23T21:20:18.381 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:20:18 smithi026 ceph-mon[175151]: failed to encode map e683 with expected crc 2023-09-23T21:20:18.381 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:20:18 smithi026 ceph-mon[175151]: failed to encode map e683 with expected crc 2023-09-23T21:20:18.381 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:20:18 smithi026 ceph-mon[175151]: failed to encode map e683 with expected crc 2023-09-23T21:20:18.381 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:20:18 smithi026 ceph-mon[175151]: pgmap v1657: 103 pgs: 2 activating+undersized+remapped, 6 remapped+peering, 7 peering, 88 active+clean; 4.6 MiB data, 4.3 GiB used, 621 GiB / 626 GiB avail; 1.1 MiB/s, 5 objects/s recovering 2023-09-23T21:20:18.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:20:18 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/2440411264' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-09-23T21:20:18.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:20:18 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:20:19.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:20:19 smithi070 ceph-mon[168282]: 1.1d scrub starts 2023-09-23T21:20:19.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:20:19 smithi070 ceph-mon[168282]: 1.1d scrub ok 2023-09-23T21:20:19.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:20:19 smithi070 ceph-mon[168282]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 1 pg degraded) 2023-09-23T21:20:19.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:20:19 smithi070 ceph-mon[168282]: Cluster is now healthy 2023-09-23T21:20:19.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:20:19 smithi070 ceph-mon[168282]: failed to encode map e684 with expected crc 2023-09-23T21:20:19.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:20:19 smithi070 ceph-mon[168282]: failed to encode map e684 with expected crc 2023-09-23T21:20:19.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:20:19 smithi070 ceph-mon[168282]: failed to encode map e684 with expected crc 2023-09-23T21:20:19.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:20:19 smithi070 ceph-mon[168282]: failed to encode map e684 with expected crc 2023-09-23T21:20:19.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:20:19 smithi070 ceph-mon[168282]: failed to encode map e684 with expected crc 2023-09-23T21:20:19.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:20:19 smithi070 ceph-mon[168282]: osdmap e684: 8 total, 8 up, 8 in 2023-09-23T21:20:19.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:20:19 smithi070 ceph-mon[168282]: failed to encode map e684 with expected crc 2023-09-23T21:20:19.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:20:19 smithi070 ceph-mon[168282]: failed to encode map e684 with expected crc 2023-09-23T21:20:19.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:20:19 smithi070 ceph-mon[168282]: failed to encode map e684 with expected crc 2023-09-23T21:20:19.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:20:19 smithi070 ceph-mon[168282]: failed to encode map e684 with expected crc 2023-09-23T21:20:19.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:20:19 smithi070 ceph-mon[168282]: failed to encode map e684 with expected crc 2023-09-23T21:20:19.380 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:20:19 smithi026 ceph-mon[175151]: 1.1d scrub starts 2023-09-23T21:20:19.380 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:20:19 smithi026 ceph-mon[175151]: 1.1d scrub ok 2023-09-23T21:20:19.381 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:20:19 smithi026 ceph-mon[175151]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 1 pg degraded) 2023-09-23T21:20:19.381 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:20:19 smithi026 ceph-mon[175151]: Cluster is now healthy 2023-09-23T21:20:19.381 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:20:19 smithi026 ceph-mon[175151]: failed to encode map e684 with expected crc 2023-09-23T21:20:19.381 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:20:19 smithi026 ceph-mon[175151]: failed to encode map e684 with expected crc 2023-09-23T21:20:19.381 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:20:19 smithi026 ceph-mon[175151]: failed to encode map e684 with expected crc 2023-09-23T21:20:19.381 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:20:19 smithi026 ceph-mon[175151]: failed to encode map e684 with expected crc 2023-09-23T21:20:19.381 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:20:19 smithi026 ceph-mon[175151]: failed to encode map e684 with expected crc 2023-09-23T21:20:19.381 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:20:19 smithi026 ceph-mon[175151]: osdmap e684: 8 total, 8 up, 8 in 2023-09-23T21:20:19.381 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:20:19 smithi026 ceph-mon[175151]: failed to encode map e684 with expected crc 2023-09-23T21:20:19.381 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:20:19 smithi026 ceph-mon[175151]: failed to encode map e684 with expected crc 2023-09-23T21:20:19.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:20:19 smithi026 ceph-mon[175151]: failed to encode map e684 with expected crc 2023-09-23T21:20:19.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:20:19 smithi026 ceph-mon[175151]: failed to encode map e684 with expected crc 2023-09-23T21:20:19.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:20:19 smithi026 ceph-mon[175151]: failed to encode map e684 with expected crc 2023-09-23T21:20:19.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:20:19 smithi026 ceph-mon[179688]: 1.1d scrub starts 2023-09-23T21:20:19.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:20:19 smithi026 ceph-mon[179688]: 1.1d scrub ok 2023-09-23T21:20:19.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:20:19 smithi026 ceph-mon[179688]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 1 pg degraded) 2023-09-23T21:20:19.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:20:19 smithi026 ceph-mon[179688]: Cluster is now healthy 2023-09-23T21:20:19.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:20:19 smithi026 ceph-mon[179688]: failed to encode map e684 with expected crc 2023-09-23T21:20:19.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:20:19 smithi026 ceph-mon[179688]: failed to encode map e684 with expected crc 2023-09-23T21:20:19.383 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:20:19 smithi026 ceph-mon[179688]: failed to encode map e684 with expected crc 2023-09-23T21:20:19.383 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:20:19 smithi026 ceph-mon[179688]: failed to encode map e684 with expected crc 2023-09-23T21:20:19.383 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:20:19 smithi026 ceph-mon[179688]: failed to encode map e684 with expected crc 2023-09-23T21:20:19.383 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:20:19 smithi026 ceph-mon[179688]: osdmap e684: 8 total, 8 up, 8 in 2023-09-23T21:20:19.383 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:20:19 smithi026 ceph-mon[179688]: failed to encode map e684 with expected crc 2023-09-23T21:20:19.383 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:20:19 smithi026 ceph-mon[179688]: failed to encode map e684 with expected crc 2023-09-23T21:20:19.383 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:20:19 smithi026 ceph-mon[179688]: failed to encode map e684 with expected crc 2023-09-23T21:20:19.383 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:20:19 smithi026 ceph-mon[179688]: failed to encode map e684 with expected crc 2023-09-23T21:20:19.383 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:20:19 smithi026 ceph-mon[179688]: failed to encode map e684 with expected crc 2023-09-23T21:20:19.631 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 21:20:19 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[164029]: ::ffff:172.21.15.70 - - [23/Sep/2023:21:20:19] "GET /metrics HTTP/1.1" 200 34030 "" "Prometheus/2.43.0" 2023-09-23T21:20:20.079 INFO:teuthology.orchestra.run.smithi026.stderr:reweighted osd.2 to 1 (10000) 2023-09-23T21:20:20.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:20:20 smithi070 ceph-mon[168282]: failed to encode map e685 with expected crc 2023-09-23T21:20:20.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:20:20 smithi070 ceph-mon[168282]: failed to encode map e685 with expected crc 2023-09-23T21:20:20.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:20:20 smithi070 ceph-mon[168282]: osdmap e685: 8 total, 8 up, 8 in 2023-09-23T21:20:20.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:20:20 smithi070 ceph-mon[168282]: failed to encode map e685 with expected crc 2023-09-23T21:20:20.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:20:20 smithi070 ceph-mon[168282]: failed to encode map e685 with expected crc 2023-09-23T21:20:20.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:20:20 smithi070 ceph-mon[168282]: failed to encode map e685 with expected crc 2023-09-23T21:20:20.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:20:20 smithi070 ceph-mon[168282]: failed to encode map e685 with expected crc 2023-09-23T21:20:20.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:20:20 smithi070 ceph-mon[168282]: failed to encode map e685 with expected crc 2023-09-23T21:20:20.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:20:20 smithi070 ceph-mon[168282]: failed to encode map e685 with expected crc 2023-09-23T21:20:20.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:20:20 smithi070 ceph-mon[168282]: failed to encode map e685 with expected crc 2023-09-23T21:20:20.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:20:20 smithi070 ceph-mon[168282]: failed to encode map e685 with expected crc 2023-09-23T21:20:20.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:20:20 smithi070 ceph-mon[168282]: failed to encode map e685 with expected crc 2023-09-23T21:20:20.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:20:20 smithi070 ceph-mon[168282]: failed to encode map e685 with expected crc 2023-09-23T21:20:20.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:20:20 smithi070 ceph-mon[168282]: pgmap v1660: 103 pgs: 1 active+undersized+remapped, 2 activating+undersized+remapped, 4 remapped+peering, 9 peering, 87 active+clean; 4.6 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 1.1 MiB/s, 6 objects/s recovering 2023-09-23T21:20:20.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:20:20 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/215059305' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T21:20:20.381 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:20:20 smithi026 ceph-mon[175151]: failed to encode map e685 with expected crc 2023-09-23T21:20:20.381 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:20:20 smithi026 ceph-mon[175151]: failed to encode map e685 with expected crc 2023-09-23T21:20:20.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:20:20 smithi026 ceph-mon[175151]: osdmap e685: 8 total, 8 up, 8 in 2023-09-23T21:20:20.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:20:20 smithi026 ceph-mon[175151]: failed to encode map e685 with expected crc 2023-09-23T21:20:20.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:20:20 smithi026 ceph-mon[175151]: failed to encode map e685 with expected crc 2023-09-23T21:20:20.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:20:20 smithi026 ceph-mon[175151]: failed to encode map e685 with expected crc 2023-09-23T21:20:20.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:20:20 smithi026 ceph-mon[175151]: failed to encode map e685 with expected crc 2023-09-23T21:20:20.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:20:20 smithi026 ceph-mon[175151]: failed to encode map e685 with expected crc 2023-09-23T21:20:20.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:20:20 smithi026 ceph-mon[175151]: failed to encode map e685 with expected crc 2023-09-23T21:20:20.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:20:20 smithi026 ceph-mon[175151]: failed to encode map e685 with expected crc 2023-09-23T21:20:20.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:20:20 smithi026 ceph-mon[175151]: failed to encode map e685 with expected crc 2023-09-23T21:20:20.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:20:20 smithi026 ceph-mon[175151]: failed to encode map e685 with expected crc 2023-09-23T21:20:20.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:20:20 smithi026 ceph-mon[175151]: failed to encode map e685 with expected crc 2023-09-23T21:20:20.383 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:20:20 smithi026 ceph-mon[175151]: pgmap v1660: 103 pgs: 1 active+undersized+remapped, 2 activating+undersized+remapped, 4 remapped+peering, 9 peering, 87 active+clean; 4.6 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 1.1 MiB/s, 6 objects/s recovering 2023-09-23T21:20:20.383 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:20:20 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/215059305' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T21:20:20.383 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:20:20 smithi026 ceph-mon[179688]: failed to encode map e685 with expected crc 2023-09-23T21:20:20.383 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:20:20 smithi026 ceph-mon[179688]: failed to encode map e685 with expected crc 2023-09-23T21:20:20.383 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:20:20 smithi026 ceph-mon[179688]: osdmap e685: 8 total, 8 up, 8 in 2023-09-23T21:20:20.383 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:20:20 smithi026 ceph-mon[179688]: failed to encode map e685 with expected crc 2023-09-23T21:20:20.383 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:20:20 smithi026 ceph-mon[179688]: failed to encode map e685 with expected crc 2023-09-23T21:20:20.383 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:20:20 smithi026 ceph-mon[179688]: failed to encode map e685 with expected crc 2023-09-23T21:20:20.384 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:20:20 smithi026 ceph-mon[179688]: failed to encode map e685 with expected crc 2023-09-23T21:20:20.384 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:20:20 smithi026 ceph-mon[179688]: failed to encode map e685 with expected crc 2023-09-23T21:20:20.384 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:20:20 smithi026 ceph-mon[179688]: failed to encode map e685 with expected crc 2023-09-23T21:20:20.384 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:20:20 smithi026 ceph-mon[179688]: failed to encode map e685 with expected crc 2023-09-23T21:20:20.384 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:20:20 smithi026 ceph-mon[179688]: failed to encode map e685 with expected crc 2023-09-23T21:20:20.384 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:20:20 smithi026 ceph-mon[179688]: failed to encode map e685 with expected crc 2023-09-23T21:20:20.384 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:20:20 smithi026 ceph-mon[179688]: failed to encode map e685 with expected crc 2023-09-23T21:20:20.384 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:20:20 smithi026 ceph-mon[179688]: pgmap v1660: 103 pgs: 1 active+undersized+remapped, 2 activating+undersized+remapped, 4 remapped+peering, 9 peering, 87 active+clean; 4.6 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 1.1 MiB/s, 6 objects/s recovering 2023-09-23T21:20:20.384 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:20:20 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/215059305' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T21:20:20.520 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph osd primary-affinity 2 1 2023-09-23T21:20:21.130 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:20:21.336 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:20:21 smithi026 ceph-mon[179688]: failed to encode map e685 with expected crc 2023-09-23T21:20:21.336 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:20:21 smithi026 ceph-mon[179688]: failed to encode map e685 with expected crc 2023-09-23T21:20:21.336 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:20:21 smithi026 ceph-mon[179688]: failed to encode map e686 with expected crc 2023-09-23T21:20:21.336 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:20:21 smithi026 ceph-mon[179688]: failed to encode map e686 with expected crc 2023-09-23T21:20:21.336 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:20:21 smithi026 ceph-mon[179688]: failed to encode map e686 with expected crc 2023-09-23T21:20:21.336 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:20:21 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/215059305' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-09-23T21:20:21.337 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:20:21 smithi026 ceph-mon[179688]: osdmap e686: 8 total, 8 up, 8 in 2023-09-23T21:20:21.337 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:20:21 smithi026 ceph-mon[175151]: failed to encode map e685 with expected crc 2023-09-23T21:20:21.337 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:20:21 smithi026 ceph-mon[175151]: failed to encode map e685 with expected crc 2023-09-23T21:20:21.337 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:20:21 smithi026 ceph-mon[175151]: failed to encode map e686 with expected crc 2023-09-23T21:20:21.337 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:20:21 smithi026 ceph-mon[175151]: failed to encode map e686 with expected crc 2023-09-23T21:20:21.337 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:20:21 smithi026 ceph-mon[175151]: failed to encode map e686 with expected crc 2023-09-23T21:20:21.337 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:20:21 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/215059305' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-09-23T21:20:21.337 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:20:21 smithi026 ceph-mon[175151]: osdmap e686: 8 total, 8 up, 8 in 2023-09-23T21:20:21.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:20:21 smithi070 ceph-mon[168282]: failed to encode map e685 with expected crc 2023-09-23T21:20:21.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:20:21 smithi070 ceph-mon[168282]: failed to encode map e685 with expected crc 2023-09-23T21:20:21.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:20:21 smithi070 ceph-mon[168282]: failed to encode map e686 with expected crc 2023-09-23T21:20:21.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:20:21 smithi070 ceph-mon[168282]: failed to encode map e686 with expected crc 2023-09-23T21:20:21.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:20:21 smithi070 ceph-mon[168282]: failed to encode map e686 with expected crc 2023-09-23T21:20:21.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:20:21 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/215059305' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-09-23T21:20:21.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:20:21 smithi070 ceph-mon[168282]: osdmap e686: 8 total, 8 up, 8 in 2023-09-23T21:20:22.086 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:20:22 smithi026 ceph-mon[175151]: pgmap v1662: 103 pgs: 1 active+undersized+remapped, 2 activating+undersized+remapped, 4 remapped+peering, 2 peering, 94 active+clean; 4.6 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 949 KiB/s, 6 objects/s recovering 2023-09-23T21:20:22.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:20:22 smithi070 ceph-mon[168282]: pgmap v1662: 103 pgs: 1 active+undersized+remapped, 2 activating+undersized+remapped, 4 remapped+peering, 2 peering, 94 active+clean; 4.6 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 949 KiB/s, 6 objects/s recovering 2023-09-23T21:20:22.381 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:20:22 smithi026 ceph-mon[179688]: pgmap v1662: 103 pgs: 1 active+undersized+remapped, 2 activating+undersized+remapped, 4 remapped+peering, 2 peering, 94 active+clean; 4.6 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 949 KiB/s, 6 objects/s recovering 2023-09-23T21:20:23.098 INFO:teuthology.orchestra.run.smithi026.stderr:set osd.2 primary-affinity to 1 (8655362) 2023-09-23T21:20:23.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:20:23 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/2825977013' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T21:20:23.381 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:20:23 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/2825977013' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T21:20:23.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:20:23 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/2825977013' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T21:20:23.500 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph osd reweight 3 1 2023-09-23T21:20:24.116 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:20:24.351 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:20:24 smithi026 ceph-mon[179688]: failed to encode map e687 with expected crc 2023-09-23T21:20:24.352 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:20:24 smithi026 ceph-mon[179688]: failed to encode map e687 with expected crc 2023-09-23T21:20:24.352 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:20:24 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/2825977013' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 2, "weight": 1.0}]': finished 2023-09-23T21:20:24.352 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:20:24 smithi026 ceph-mon[179688]: osdmap e687: 8 total, 8 up, 8 in 2023-09-23T21:20:24.352 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:20:24 smithi026 ceph-mon[179688]: pgmap v1664: 103 pgs: 2 peering, 101 active+clean; 4.6 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 844 KiB/s, 6 objects/s recovering 2023-09-23T21:20:24.352 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:20:24 smithi026 ceph-mon[179688]: 1.1b scrub starts 2023-09-23T21:20:24.352 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:20:24 smithi026 ceph-mon[179688]: 1.1b scrub ok 2023-09-23T21:20:24.352 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:20:24 smithi026 ceph-mon[175151]: failed to encode map e687 with expected crc 2023-09-23T21:20:24.352 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:20:24 smithi026 ceph-mon[175151]: failed to encode map e687 with expected crc 2023-09-23T21:20:24.352 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:20:24 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/2825977013' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 2, "weight": 1.0}]': finished 2023-09-23T21:20:24.353 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:20:24 smithi026 ceph-mon[175151]: osdmap e687: 8 total, 8 up, 8 in 2023-09-23T21:20:24.353 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:20:24 smithi026 ceph-mon[175151]: pgmap v1664: 103 pgs: 2 peering, 101 active+clean; 4.6 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 844 KiB/s, 6 objects/s recovering 2023-09-23T21:20:24.353 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:20:24 smithi026 ceph-mon[175151]: 1.1b scrub starts 2023-09-23T21:20:24.353 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:20:24 smithi026 ceph-mon[175151]: 1.1b scrub ok 2023-09-23T21:20:24.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:20:24 smithi070 ceph-mon[168282]: failed to encode map e687 with expected crc 2023-09-23T21:20:24.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:20:24 smithi070 ceph-mon[168282]: failed to encode map e687 with expected crc 2023-09-23T21:20:24.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:20:24 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/2825977013' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 2, "weight": 1.0}]': finished 2023-09-23T21:20:24.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:20:24 smithi070 ceph-mon[168282]: osdmap e687: 8 total, 8 up, 8 in 2023-09-23T21:20:24.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:20:24 smithi070 ceph-mon[168282]: pgmap v1664: 103 pgs: 2 peering, 101 active+clean; 4.6 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 844 KiB/s, 6 objects/s recovering 2023-09-23T21:20:24.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:20:24 smithi070 ceph-mon[168282]: 1.1b scrub starts 2023-09-23T21:20:24.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:20:24 smithi070 ceph-mon[168282]: 1.1b scrub ok 2023-09-23T21:20:26.292 INFO:teuthology.orchestra.run.smithi026.stderr:reweighted osd.3 to 1 (10000) 2023-09-23T21:20:26.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:20:26 smithi070 ceph-mon[168282]: pgmap v1665: 103 pgs: 103 active+clean; 4.6 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 42 KiB/s, 2 objects/s recovering 2023-09-23T21:20:26.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:20:26 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/3897713344' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-09-23T21:20:26.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:20:26 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-09-23T21:20:26.631 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:20:26 smithi026 ceph-mon[179688]: pgmap v1665: 103 pgs: 103 active+clean; 4.6 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 42 KiB/s, 2 objects/s recovering 2023-09-23T21:20:26.631 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:20:26 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/3897713344' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-09-23T21:20:26.631 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:20:26 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-09-23T21:20:26.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:20:26 smithi026 ceph-mon[175151]: pgmap v1665: 103 pgs: 103 active+clean; 4.6 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 42 KiB/s, 2 objects/s recovering 2023-09-23T21:20:26.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:20:26 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/3897713344' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-09-23T21:20:26.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:20:26 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-09-23T21:20:26.768 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph osd primary-affinity 3 1 2023-09-23T21:20:27.380 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:20:27.585 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:20:27 smithi026 ceph-mon[175151]: failed to encode map e688 with expected crc 2023-09-23T21:20:27.585 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:20:27 smithi026 ceph-mon[175151]: failed to encode map e688 with expected crc 2023-09-23T21:20:27.585 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:20:27 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-09-23T21:20:27.585 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:20:27 smithi026 ceph-mon[175151]: osdmap e688: 8 total, 8 up, 8 in 2023-09-23T21:20:27.586 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:20:27 smithi026 ceph-mon[175151]: failed to encode map e688 with expected crc 2023-09-23T21:20:27.586 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:20:27 smithi026 ceph-mon[175151]: failed to encode map e688 with expected crc 2023-09-23T21:20:27.586 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:20:27 smithi026 ceph-mon[179688]: failed to encode map e688 with expected crc 2023-09-23T21:20:27.586 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:20:27 smithi026 ceph-mon[179688]: failed to encode map e688 with expected crc 2023-09-23T21:20:27.586 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:20:27 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-09-23T21:20:27.586 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:20:27 smithi026 ceph-mon[179688]: osdmap e688: 8 total, 8 up, 8 in 2023-09-23T21:20:27.586 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:20:27 smithi026 ceph-mon[179688]: failed to encode map e688 with expected crc 2023-09-23T21:20:27.586 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:20:27 smithi026 ceph-mon[179688]: failed to encode map e688 with expected crc 2023-09-23T21:20:27.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:20:27 smithi070 ceph-mon[168282]: failed to encode map e688 with expected crc 2023-09-23T21:20:27.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:20:27 smithi070 ceph-mon[168282]: failed to encode map e688 with expected crc 2023-09-23T21:20:27.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:20:27 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-09-23T21:20:27.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:20:27 smithi070 ceph-mon[168282]: osdmap e688: 8 total, 8 up, 8 in 2023-09-23T21:20:27.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:20:27 smithi070 ceph-mon[168282]: failed to encode map e688 with expected crc 2023-09-23T21:20:27.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:20:27 smithi070 ceph-mon[168282]: failed to encode map e688 with expected crc 2023-09-23T21:20:28.378 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:20:28 smithi026 ceph-mon[179688]: failed to encode map e688 with expected crc 2023-09-23T21:20:28.378 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:20:28 smithi026 ceph-mon[179688]: pgmap v1667: 103 pgs: 103 active+clean; 4.6 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 36 KiB/s, 1 objects/s recovering 2023-09-23T21:20:28.378 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:20:28 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:20:28.378 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:20:28 smithi026 ceph-mon[175151]: failed to encode map e688 with expected crc 2023-09-23T21:20:28.379 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:20:28 smithi026 ceph-mon[175151]: pgmap v1667: 103 pgs: 103 active+clean; 4.6 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 36 KiB/s, 1 objects/s recovering 2023-09-23T21:20:28.379 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:20:28 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:20:28.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:20:28 smithi070 ceph-mon[168282]: failed to encode map e688 with expected crc 2023-09-23T21:20:28.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:20:28 smithi070 ceph-mon[168282]: pgmap v1667: 103 pgs: 103 active+clean; 4.6 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 36 KiB/s, 1 objects/s recovering 2023-09-23T21:20:28.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:20:28 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:20:29.310 INFO:teuthology.orchestra.run.smithi026.stderr:set osd.3 primary-affinity to 1 (8655362) 2023-09-23T21:20:29.612 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:20:29 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/1104044376' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 3, "weight": 1.0}]: dispatch 2023-09-23T21:20:29.612 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:20:29 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 3, "weight": 1.0}]: dispatch 2023-09-23T21:20:29.612 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:20:29 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/1104044376' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 3, "weight": 1.0}]: dispatch 2023-09-23T21:20:29.612 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:20:29 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 3, "weight": 1.0}]: dispatch 2023-09-23T21:20:29.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:20:29 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/1104044376' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 3, "weight": 1.0}]: dispatch 2023-09-23T21:20:29.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:20:29 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 3, "weight": 1.0}]: dispatch 2023-09-23T21:20:29.749 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph osd reweight 4 1 2023-09-23T21:20:29.881 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 21:20:29 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[164029]: ::ffff:172.21.15.70 - - [23/Sep/2023:21:20:29] "GET /metrics HTTP/1.1" 200 34030 "" "Prometheus/2.43.0" 2023-09-23T21:20:30.364 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:20:30.576 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:20:30 smithi026 ceph-mon[175151]: pgmap v1668: 103 pgs: 103 active+clean; 4.6 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 32 KiB/s, 0 objects/s recovering 2023-09-23T21:20:30.576 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:20:30 smithi026 ceph-mon[175151]: failed to encode map e689 with expected crc 2023-09-23T21:20:30.577 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:20:30 smithi026 ceph-mon[175151]: failed to encode map e689 with expected crc 2023-09-23T21:20:30.577 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:20:30 smithi026 ceph-mon[175151]: failed to encode map e689 with expected crc 2023-09-23T21:20:30.577 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:20:30 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 3, "weight": 1.0}]': finished 2023-09-23T21:20:30.577 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:20:30 smithi026 ceph-mon[175151]: osdmap e689: 8 total, 8 up, 8 in 2023-09-23T21:20:30.577 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:20:30 smithi026 ceph-mon[179688]: pgmap v1668: 103 pgs: 103 active+clean; 4.6 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 32 KiB/s, 0 objects/s recovering 2023-09-23T21:20:30.577 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:20:30 smithi026 ceph-mon[179688]: failed to encode map e689 with expected crc 2023-09-23T21:20:30.577 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:20:30 smithi026 ceph-mon[179688]: failed to encode map e689 with expected crc 2023-09-23T21:20:30.577 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:20:30 smithi026 ceph-mon[179688]: failed to encode map e689 with expected crc 2023-09-23T21:20:30.577 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:20:30 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 3, "weight": 1.0}]': finished 2023-09-23T21:20:30.578 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:20:30 smithi026 ceph-mon[179688]: osdmap e689: 8 total, 8 up, 8 in 2023-09-23T21:20:30.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:20:30 smithi070 ceph-mon[168282]: pgmap v1668: 103 pgs: 103 active+clean; 4.6 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 32 KiB/s, 0 objects/s recovering 2023-09-23T21:20:30.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:20:30 smithi070 ceph-mon[168282]: failed to encode map e689 with expected crc 2023-09-23T21:20:30.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:20:30 smithi070 ceph-mon[168282]: failed to encode map e689 with expected crc 2023-09-23T21:20:30.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:20:30 smithi070 ceph-mon[168282]: failed to encode map e689 with expected crc 2023-09-23T21:20:30.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:20:30 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 3, "weight": 1.0}]': finished 2023-09-23T21:20:30.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:20:30 smithi070 ceph-mon[168282]: osdmap e689: 8 total, 8 up, 8 in 2023-09-23T21:20:31.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:20:31 smithi070 ceph-mon[168282]: failed to encode map e690 with expected crc 2023-09-23T21:20:31.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:20:31 smithi070 ceph-mon[168282]: failed to encode map e690 with expected crc 2023-09-23T21:20:31.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:20:31 smithi070 ceph-mon[168282]: failed to encode map e690 with expected crc 2023-09-23T21:20:31.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:20:31 smithi070 ceph-mon[168282]: osdmap e690: 8 total, 8 up, 8 in 2023-09-23T21:20:31.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:20:31 smithi070 ceph-mon[168282]: failed to encode map e690 with expected crc 2023-09-23T21:20:31.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:20:31 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:20:31.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:20:31 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:20:31.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:20:31 smithi026 ceph-mon[175151]: failed to encode map e690 with expected crc 2023-09-23T21:20:31.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:20:31 smithi026 ceph-mon[175151]: failed to encode map e690 with expected crc 2023-09-23T21:20:31.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:20:31 smithi026 ceph-mon[175151]: failed to encode map e690 with expected crc 2023-09-23T21:20:31.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:20:31 smithi026 ceph-mon[175151]: osdmap e690: 8 total, 8 up, 8 in 2023-09-23T21:20:31.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:20:31 smithi026 ceph-mon[175151]: failed to encode map e690 with expected crc 2023-09-23T21:20:31.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:20:31 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:20:31.633 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:20:31 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:20:31.633 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:20:31 smithi026 ceph-mon[179688]: failed to encode map e690 with expected crc 2023-09-23T21:20:31.633 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:20:31 smithi026 ceph-mon[179688]: failed to encode map e690 with expected crc 2023-09-23T21:20:31.633 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:20:31 smithi026 ceph-mon[179688]: failed to encode map e690 with expected crc 2023-09-23T21:20:31.633 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:20:31 smithi026 ceph-mon[179688]: osdmap e690: 8 total, 8 up, 8 in 2023-09-23T21:20:31.633 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:20:31 smithi026 ceph-mon[179688]: failed to encode map e690 with expected crc 2023-09-23T21:20:31.633 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:20:31 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:20:31.633 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:20:31 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:20:32.315 INFO:teuthology.orchestra.run.smithi026.stderr:reweighted osd.4 to 1 (10000) 2023-09-23T21:20:32.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:20:32 smithi070 ceph-mon[168282]: pgmap v1671: 103 pgs: 10 peering, 93 active+clean; 4.6 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail 2023-09-23T21:20:32.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:20:32 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/3534635706' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-09-23T21:20:32.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:20:32 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-09-23T21:20:32.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:20:32 smithi070 ceph-mon[168282]: Health check failed: Reduced data availability: 3 pgs inactive, 8 pgs peering (PG_AVAILABILITY) 2023-09-23T21:20:32.631 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:20:32 smithi026 ceph-mon[179688]: pgmap v1671: 103 pgs: 10 peering, 93 active+clean; 4.6 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail 2023-09-23T21:20:32.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:20:32 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/3534635706' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-09-23T21:20:32.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:20:32 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-09-23T21:20:32.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:20:32 smithi026 ceph-mon[179688]: Health check failed: Reduced data availability: 3 pgs inactive, 8 pgs peering (PG_AVAILABILITY) 2023-09-23T21:20:32.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:20:32 smithi026 ceph-mon[175151]: pgmap v1671: 103 pgs: 10 peering, 93 active+clean; 4.6 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail 2023-09-23T21:20:32.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:20:32 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/3534635706' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-09-23T21:20:32.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:20:32 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-09-23T21:20:32.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:20:32 smithi026 ceph-mon[175151]: Health check failed: Reduced data availability: 3 pgs inactive, 8 pgs peering (PG_AVAILABILITY) 2023-09-23T21:20:32.723 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph osd primary-affinity 4 1 2023-09-23T21:20:33.337 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:20:33.577 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:20:33 smithi026 ceph-mon[179688]: 1.9 scrub starts 2023-09-23T21:20:33.577 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:20:33 smithi026 ceph-mon[179688]: 1.9 scrub ok 2023-09-23T21:20:33.577 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:20:33 smithi026 ceph-mon[179688]: failed to encode map e691 with expected crc 2023-09-23T21:20:33.577 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:20:33 smithi026 ceph-mon[179688]: failed to encode map e691 with expected crc 2023-09-23T21:20:33.577 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:20:33 smithi026 ceph-mon[179688]: failed to encode map e691 with expected crc 2023-09-23T21:20:33.578 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:20:33 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-09-23T21:20:33.578 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:20:33 smithi026 ceph-mon[179688]: osdmap e691: 8 total, 8 up, 8 in 2023-09-23T21:20:33.578 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:20:33 smithi026 ceph-mon[175151]: 1.9 scrub starts 2023-09-23T21:20:33.578 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:20:33 smithi026 ceph-mon[175151]: 1.9 scrub ok 2023-09-23T21:20:33.578 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:20:33 smithi026 ceph-mon[175151]: failed to encode map e691 with expected crc 2023-09-23T21:20:33.578 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:20:33 smithi026 ceph-mon[175151]: failed to encode map e691 with expected crc 2023-09-23T21:20:33.578 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:20:33 smithi026 ceph-mon[175151]: failed to encode map e691 with expected crc 2023-09-23T21:20:33.578 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:20:33 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-09-23T21:20:33.578 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:20:33 smithi026 ceph-mon[175151]: osdmap e691: 8 total, 8 up, 8 in 2023-09-23T21:20:33.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:20:33 smithi070 ceph-mon[168282]: 1.9 scrub starts 2023-09-23T21:20:33.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:20:33 smithi070 ceph-mon[168282]: 1.9 scrub ok 2023-09-23T21:20:33.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:20:33 smithi070 ceph-mon[168282]: failed to encode map e691 with expected crc 2023-09-23T21:20:33.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:20:33 smithi070 ceph-mon[168282]: failed to encode map e691 with expected crc 2023-09-23T21:20:33.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:20:33 smithi070 ceph-mon[168282]: failed to encode map e691 with expected crc 2023-09-23T21:20:33.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:20:33 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-09-23T21:20:33.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:20:33 smithi070 ceph-mon[168282]: osdmap e691: 8 total, 8 up, 8 in 2023-09-23T21:20:34.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:20:34 smithi070 ceph-mon[168282]: 1.19 scrub starts 2023-09-23T21:20:34.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:20:34 smithi070 ceph-mon[168282]: 1.19 scrub ok 2023-09-23T21:20:34.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:20:34 smithi070 ceph-mon[168282]: pgmap v1673: 103 pgs: 10 peering, 93 active+clean; 4.6 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail 2023-09-23T21:20:34.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:20:34 smithi026 ceph-mon[175151]: 1.19 scrub starts 2023-09-23T21:20:34.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:20:34 smithi026 ceph-mon[175151]: 1.19 scrub ok 2023-09-23T21:20:34.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:20:34 smithi026 ceph-mon[175151]: pgmap v1673: 103 pgs: 10 peering, 93 active+clean; 4.6 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail 2023-09-23T21:20:34.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:20:34 smithi026 ceph-mon[179688]: 1.19 scrub starts 2023-09-23T21:20:34.633 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:20:34 smithi026 ceph-mon[179688]: 1.19 scrub ok 2023-09-23T21:20:34.633 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:20:34 smithi026 ceph-mon[179688]: pgmap v1673: 103 pgs: 10 peering, 93 active+clean; 4.6 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail 2023-09-23T21:20:35.327 INFO:teuthology.orchestra.run.smithi026.stderr:set osd.4 primary-affinity to 1 (8655362) 2023-09-23T21:20:35.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:20:35 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/3525044174' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 4, "weight": 1.0}]: dispatch 2023-09-23T21:20:35.633 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:20:35 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/3525044174' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 4, "weight": 1.0}]: dispatch 2023-09-23T21:20:35.633 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:20:35 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/3525044174' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 4, "weight": 1.0}]: dispatch 2023-09-23T21:20:35.691 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph osd reweight 5 1 2023-09-23T21:20:36.305 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:20:36.584 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:20:36 smithi026 ceph-mon[175151]: pgmap v1674: 103 pgs: 103 active+clean; 4.6 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail 2023-09-23T21:20:36.584 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:20:36 smithi026 ceph-mon[175151]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 3 pgs inactive, 8 pgs peering) 2023-09-23T21:20:36.584 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:20:36 smithi026 ceph-mon[175151]: Cluster is now healthy 2023-09-23T21:20:36.584 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:20:36 smithi026 ceph-mon[175151]: failed to encode map e692 with expected crc 2023-09-23T21:20:36.584 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:20:36 smithi026 ceph-mon[175151]: failed to encode map e692 with expected crc 2023-09-23T21:20:36.584 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:20:36 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/3525044174' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 4, "weight": 1.0}]': finished 2023-09-23T21:20:36.584 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:20:36 smithi026 ceph-mon[175151]: osdmap e692: 8 total, 8 up, 8 in 2023-09-23T21:20:36.585 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:20:36 smithi026 ceph-mon[179688]: pgmap v1674: 103 pgs: 103 active+clean; 4.6 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail 2023-09-23T21:20:36.585 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:20:36 smithi026 ceph-mon[179688]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 3 pgs inactive, 8 pgs peering) 2023-09-23T21:20:36.585 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:20:36 smithi026 ceph-mon[179688]: Cluster is now healthy 2023-09-23T21:20:36.585 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:20:36 smithi026 ceph-mon[179688]: failed to encode map e692 with expected crc 2023-09-23T21:20:36.585 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:20:36 smithi026 ceph-mon[179688]: failed to encode map e692 with expected crc 2023-09-23T21:20:36.585 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:20:36 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/3525044174' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 4, "weight": 1.0}]': finished 2023-09-23T21:20:36.585 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:20:36 smithi026 ceph-mon[179688]: osdmap e692: 8 total, 8 up, 8 in 2023-09-23T21:20:36.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:20:36 smithi070 ceph-mon[168282]: pgmap v1674: 103 pgs: 103 active+clean; 4.6 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail 2023-09-23T21:20:36.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:20:36 smithi070 ceph-mon[168282]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 3 pgs inactive, 8 pgs peering) 2023-09-23T21:20:36.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:20:36 smithi070 ceph-mon[168282]: Cluster is now healthy 2023-09-23T21:20:36.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:20:36 smithi070 ceph-mon[168282]: failed to encode map e692 with expected crc 2023-09-23T21:20:36.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:20:36 smithi070 ceph-mon[168282]: failed to encode map e692 with expected crc 2023-09-23T21:20:36.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:20:36 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/3525044174' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 4, "weight": 1.0}]': finished 2023-09-23T21:20:36.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:20:36 smithi070 ceph-mon[168282]: osdmap e692: 8 total, 8 up, 8 in 2023-09-23T21:20:37.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:20:37 smithi070 ceph-mon[168282]: failed to encode map e692 with expected crc 2023-09-23T21:20:37.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:20:37 smithi070 ceph-mon[168282]: failed to encode map e692 with expected crc 2023-09-23T21:20:37.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:20:37 smithi026 ceph-mon[175151]: failed to encode map e692 with expected crc 2023-09-23T21:20:37.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:20:37 smithi026 ceph-mon[175151]: failed to encode map e692 with expected crc 2023-09-23T21:20:37.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:20:37 smithi026 ceph-mon[179688]: failed to encode map e692 with expected crc 2023-09-23T21:20:37.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:20:37 smithi026 ceph-mon[179688]: failed to encode map e692 with expected crc 2023-09-23T21:20:38.334 INFO:teuthology.orchestra.run.smithi026.stderr:reweighted osd.5 to 1 (10000) 2023-09-23T21:20:38.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:20:38 smithi070 ceph-mon[168282]: pgmap v1676: 103 pgs: 103 active+clean; 4.6 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail 2023-09-23T21:20:38.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:20:38 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/687002676' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-09-23T21:20:38.631 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:20:38 smithi026 ceph-mon[179688]: pgmap v1676: 103 pgs: 103 active+clean; 4.6 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail 2023-09-23T21:20:38.631 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:20:38 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/687002676' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-09-23T21:20:38.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:20:38 smithi026 ceph-mon[175151]: pgmap v1676: 103 pgs: 103 active+clean; 4.6 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail 2023-09-23T21:20:38.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:20:38 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/687002676' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-09-23T21:20:38.672 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph osd primary-affinity 5 1 2023-09-23T21:20:39.284 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:20:39.611 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:20:39 smithi026 ceph-mon[179688]: failed to encode map e693 with expected crc 2023-09-23T21:20:39.612 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:20:39 smithi026 ceph-mon[179688]: failed to encode map e693 with expected crc 2023-09-23T21:20:39.612 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:20:39 smithi026 ceph-mon[179688]: failed to encode map e693 with expected crc 2023-09-23T21:20:39.612 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:20:39 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/687002676' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-09-23T21:20:39.612 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:20:39 smithi026 ceph-mon[179688]: osdmap e693: 8 total, 8 up, 8 in 2023-09-23T21:20:39.612 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:20:39 smithi026 ceph-mon[175151]: failed to encode map e693 with expected crc 2023-09-23T21:20:39.612 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:20:39 smithi026 ceph-mon[175151]: failed to encode map e693 with expected crc 2023-09-23T21:20:39.612 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:20:39 smithi026 ceph-mon[175151]: failed to encode map e693 with expected crc 2023-09-23T21:20:39.612 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:20:39 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/687002676' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-09-23T21:20:39.612 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:20:39 smithi026 ceph-mon[175151]: osdmap e693: 8 total, 8 up, 8 in 2023-09-23T21:20:39.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:20:39 smithi070 ceph-mon[168282]: failed to encode map e693 with expected crc 2023-09-23T21:20:39.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:20:39 smithi070 ceph-mon[168282]: failed to encode map e693 with expected crc 2023-09-23T21:20:39.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:20:39 smithi070 ceph-mon[168282]: failed to encode map e693 with expected crc 2023-09-23T21:20:39.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:20:39 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/687002676' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-09-23T21:20:39.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:20:39 smithi070 ceph-mon[168282]: osdmap e693: 8 total, 8 up, 8 in 2023-09-23T21:20:39.881 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 21:20:39 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[164029]: ::ffff:172.21.15.70 - - [23/Sep/2023:21:20:39] "GET /metrics HTTP/1.1" 200 34018 "" "Prometheus/2.43.0" 2023-09-23T21:20:40.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:20:40 smithi070 ceph-mon[168282]: pgmap v1678: 103 pgs: 103 active+clean; 4.6 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail 2023-09-23T21:20:40.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:20:40 smithi070 ceph-mon[168282]: failed to encode map e693 with expected crc 2023-09-23T21:20:40.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:20:40 smithi026 ceph-mon[175151]: pgmap v1678: 103 pgs: 103 active+clean; 4.6 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail 2023-09-23T21:20:40.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:20:40 smithi026 ceph-mon[175151]: failed to encode map e693 with expected crc 2023-09-23T21:20:40.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:20:40 smithi026 ceph-mon[179688]: pgmap v1678: 103 pgs: 103 active+clean; 4.6 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail 2023-09-23T21:20:40.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:20:40 smithi026 ceph-mon[179688]: failed to encode map e693 with expected crc 2023-09-23T21:20:41.346 INFO:teuthology.orchestra.run.smithi026.stderr:set osd.5 primary-affinity to 1 (8655362) 2023-09-23T21:20:41.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:20:41 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/3960845192' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 5, "weight": 1.0}]: dispatch 2023-09-23T21:20:41.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:20:41 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 5, "weight": 1.0}]: dispatch 2023-09-23T21:20:41.631 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:20:41 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/3960845192' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 5, "weight": 1.0}]: dispatch 2023-09-23T21:20:41.631 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:20:41 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 5, "weight": 1.0}]: dispatch 2023-09-23T21:20:41.631 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:20:41 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/3960845192' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 5, "weight": 1.0}]: dispatch 2023-09-23T21:20:41.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:20:41 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 5, "weight": 1.0}]: dispatch 2023-09-23T21:20:41.751 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph osd reweight 7 1 2023-09-23T21:20:42.364 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:20:42.612 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:20:42 smithi026 ceph-mon[179688]: pgmap v1679: 103 pgs: 103 active+clean; 4.6 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail 2023-09-23T21:20:42.612 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:20:42 smithi026 ceph-mon[179688]: failed to encode map e694 with expected crc 2023-09-23T21:20:42.613 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:20:42 smithi026 ceph-mon[179688]: failed to encode map e694 with expected crc 2023-09-23T21:20:42.613 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:20:42 smithi026 ceph-mon[179688]: failed to encode map e694 with expected crc 2023-09-23T21:20:42.613 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:20:42 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 5, "weight": 1.0}]': finished 2023-09-23T21:20:42.613 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:20:42 smithi026 ceph-mon[179688]: osdmap e694: 8 total, 8 up, 8 in 2023-09-23T21:20:42.613 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:20:42 smithi026 ceph-mon[175151]: pgmap v1679: 103 pgs: 103 active+clean; 4.6 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail 2023-09-23T21:20:42.613 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:20:42 smithi026 ceph-mon[175151]: failed to encode map e694 with expected crc 2023-09-23T21:20:42.613 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:20:42 smithi026 ceph-mon[175151]: failed to encode map e694 with expected crc 2023-09-23T21:20:42.613 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:20:42 smithi026 ceph-mon[175151]: failed to encode map e694 with expected crc 2023-09-23T21:20:42.613 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:20:42 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 5, "weight": 1.0}]': finished 2023-09-23T21:20:42.614 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:20:42 smithi026 ceph-mon[175151]: osdmap e694: 8 total, 8 up, 8 in 2023-09-23T21:20:42.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:20:42 smithi070 ceph-mon[168282]: pgmap v1679: 103 pgs: 103 active+clean; 4.6 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail 2023-09-23T21:20:42.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:20:42 smithi070 ceph-mon[168282]: failed to encode map e694 with expected crc 2023-09-23T21:20:42.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:20:42 smithi070 ceph-mon[168282]: failed to encode map e694 with expected crc 2023-09-23T21:20:42.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:20:42 smithi070 ceph-mon[168282]: failed to encode map e694 with expected crc 2023-09-23T21:20:42.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:20:42 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 5, "weight": 1.0}]': finished 2023-09-23T21:20:42.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:20:42 smithi070 ceph-mon[168282]: osdmap e694: 8 total, 8 up, 8 in 2023-09-23T21:20:43.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:20:43 smithi070 ceph-mon[168282]: failed to encode map e695 with expected crc 2023-09-23T21:20:43.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:20:43 smithi070 ceph-mon[168282]: failed to encode map e695 with expected crc 2023-09-23T21:20:43.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:20:43 smithi070 ceph-mon[168282]: failed to encode map e695 with expected crc 2023-09-23T21:20:43.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:20:43 smithi070 ceph-mon[168282]: osdmap e695: 8 total, 8 up, 8 in 2023-09-23T21:20:43.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:20:43 smithi070 ceph-mon[168282]: failed to encode map e695 with expected crc 2023-09-23T21:20:43.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:20:43 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:20:43.633 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:20:43 smithi026 ceph-mon[175151]: failed to encode map e695 with expected crc 2023-09-23T21:20:43.633 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:20:43 smithi026 ceph-mon[175151]: failed to encode map e695 with expected crc 2023-09-23T21:20:43.633 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:20:43 smithi026 ceph-mon[175151]: failed to encode map e695 with expected crc 2023-09-23T21:20:43.633 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:20:43 smithi026 ceph-mon[175151]: osdmap e695: 8 total, 8 up, 8 in 2023-09-23T21:20:43.633 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:20:43 smithi026 ceph-mon[175151]: failed to encode map e695 with expected crc 2023-09-23T21:20:43.633 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:20:43 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:20:43.633 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:20:43 smithi026 ceph-mon[179688]: failed to encode map e695 with expected crc 2023-09-23T21:20:43.633 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:20:43 smithi026 ceph-mon[179688]: failed to encode map e695 with expected crc 2023-09-23T21:20:43.633 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:20:43 smithi026 ceph-mon[179688]: failed to encode map e695 with expected crc 2023-09-23T21:20:43.634 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:20:43 smithi026 ceph-mon[179688]: osdmap e695: 8 total, 8 up, 8 in 2023-09-23T21:20:43.634 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:20:43 smithi026 ceph-mon[179688]: failed to encode map e695 with expected crc 2023-09-23T21:20:43.634 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:20:43 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:20:44.488 INFO:teuthology.orchestra.run.smithi026.stderr:reweighted osd.7 to 1 (10000) 2023-09-23T21:20:44.830 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:20:44 smithi026 ceph-mon[179688]: pgmap v1682: 103 pgs: 103 active+clean; 4.6 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail 2023-09-23T21:20:44.831 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:20:44 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/2980664265' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-09-23T21:20:44.831 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:20:44 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-09-23T21:20:44.831 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:20:44 smithi026 ceph-mon[175151]: pgmap v1682: 103 pgs: 103 active+clean; 4.6 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail 2023-09-23T21:20:44.831 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:20:44 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/2980664265' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-09-23T21:20:44.831 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:20:44 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-09-23T21:20:44.832 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph osd primary-affinity 7 1 2023-09-23T21:20:44.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:20:44 smithi070 ceph-mon[168282]: pgmap v1682: 103 pgs: 103 active+clean; 4.6 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail 2023-09-23T21:20:44.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:20:44 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/2980664265' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-09-23T21:20:44.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:20:44 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-09-23T21:20:45.447 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:20:45.740 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:20:45 smithi026 ceph-mon[179688]: failed to encode map e696 with expected crc 2023-09-23T21:20:45.740 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:20:45 smithi026 ceph-mon[179688]: failed to encode map e696 with expected crc 2023-09-23T21:20:45.740 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:20:45 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-09-23T21:20:45.740 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:20:45 smithi026 ceph-mon[179688]: osdmap e696: 8 total, 8 up, 8 in 2023-09-23T21:20:45.740 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:20:45 smithi026 ceph-mon[179688]: failed to encode map e696 with expected crc 2023-09-23T21:20:45.740 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:20:45 smithi026 ceph-mon[175151]: failed to encode map e696 with expected crc 2023-09-23T21:20:45.740 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:20:45 smithi026 ceph-mon[175151]: failed to encode map e696 with expected crc 2023-09-23T21:20:45.740 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:20:45 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-09-23T21:20:45.741 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:20:45 smithi026 ceph-mon[175151]: osdmap e696: 8 total, 8 up, 8 in 2023-09-23T21:20:45.741 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:20:45 smithi026 ceph-mon[175151]: failed to encode map e696 with expected crc 2023-09-23T21:20:45.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:20:45 smithi070 ceph-mon[168282]: failed to encode map e696 with expected crc 2023-09-23T21:20:45.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:20:45 smithi070 ceph-mon[168282]: failed to encode map e696 with expected crc 2023-09-23T21:20:45.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:20:45 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-09-23T21:20:45.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:20:45 smithi070 ceph-mon[168282]: osdmap e696: 8 total, 8 up, 8 in 2023-09-23T21:20:45.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:20:45 smithi070 ceph-mon[168282]: failed to encode map e696 with expected crc 2023-09-23T21:20:46.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:20:46 smithi070 ceph-mon[168282]: pgmap v1684: 103 pgs: 103 active+clean; 4.6 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail 2023-09-23T21:20:46.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:20:46 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:20:46.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:20:46 smithi026 ceph-mon[175151]: pgmap v1684: 103 pgs: 103 active+clean; 4.6 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail 2023-09-23T21:20:46.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:20:46 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:20:46.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:20:46 smithi026 ceph-mon[179688]: pgmap v1684: 103 pgs: 103 active+clean; 4.6 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail 2023-09-23T21:20:46.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:20:46 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:20:47.500 INFO:teuthology.orchestra.run.smithi026.stderr:set osd.7 primary-affinity to 1 (8655362) 2023-09-23T21:20:47.867 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:20:47 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/1535015209' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 7, "weight": 1.0}]: dispatch 2023-09-23T21:20:47.868 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:20:47 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/1535015209' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 7, "weight": 1.0}]: dispatch 2023-09-23T21:20:47.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:20:47 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/1535015209' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 7, "weight": 1.0}]: dispatch 2023-09-23T21:20:47.903 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph osd reweight 6 1 2023-09-23T21:20:48.517 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:20:48.758 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:20:48 smithi026 ceph-mon[175151]: pgmap v1685: 103 pgs: 103 active+clean; 4.6 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail 2023-09-23T21:20:48.759 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:20:48 smithi026 ceph-mon[175151]: failed to encode map e697 with expected crc 2023-09-23T21:20:48.759 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:20:48 smithi026 ceph-mon[175151]: failed to encode map e697 with expected crc 2023-09-23T21:20:48.759 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:20:48 smithi026 ceph-mon[175151]: failed to encode map e697 with expected crc 2023-09-23T21:20:48.759 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:20:48 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/1535015209' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 7, "weight": 1.0}]': finished 2023-09-23T21:20:48.759 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:20:48 smithi026 ceph-mon[175151]: osdmap e697: 8 total, 8 up, 8 in 2023-09-23T21:20:48.759 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:20:48 smithi026 ceph-mon[179688]: pgmap v1685: 103 pgs: 103 active+clean; 4.6 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail 2023-09-23T21:20:48.759 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:20:48 smithi026 ceph-mon[179688]: failed to encode map e697 with expected crc 2023-09-23T21:20:48.759 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:20:48 smithi026 ceph-mon[179688]: failed to encode map e697 with expected crc 2023-09-23T21:20:48.759 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:20:48 smithi026 ceph-mon[179688]: failed to encode map e697 with expected crc 2023-09-23T21:20:48.760 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:20:48 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/1535015209' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 7, "weight": 1.0}]': finished 2023-09-23T21:20:48.760 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:20:48 smithi026 ceph-mon[179688]: osdmap e697: 8 total, 8 up, 8 in 2023-09-23T21:20:48.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:20:48 smithi070 ceph-mon[168282]: pgmap v1685: 103 pgs: 103 active+clean; 4.6 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail 2023-09-23T21:20:48.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:20:48 smithi070 ceph-mon[168282]: failed to encode map e697 with expected crc 2023-09-23T21:20:48.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:20:48 smithi070 ceph-mon[168282]: failed to encode map e697 with expected crc 2023-09-23T21:20:48.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:20:48 smithi070 ceph-mon[168282]: failed to encode map e697 with expected crc 2023-09-23T21:20:48.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:20:48 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/1535015209' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 7, "weight": 1.0}]': finished 2023-09-23T21:20:48.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:20:48 smithi070 ceph-mon[168282]: osdmap e697: 8 total, 8 up, 8 in 2023-09-23T21:20:49.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:20:49 smithi070 ceph-mon[168282]: failed to encode map e698 with expected crc 2023-09-23T21:20:49.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:20:49 smithi070 ceph-mon[168282]: failed to encode map e698 with expected crc 2023-09-23T21:20:49.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:20:49 smithi070 ceph-mon[168282]: osdmap e698: 8 total, 8 up, 8 in 2023-09-23T21:20:49.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:20:49 smithi070 ceph-mon[168282]: failed to encode map e698 with expected crc 2023-09-23T21:20:49.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:20:49 smithi070 ceph-mon[168282]: failed to encode map e698 with expected crc 2023-09-23T21:20:49.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:20:49 smithi026 ceph-mon[175151]: failed to encode map e698 with expected crc 2023-09-23T21:20:49.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:20:49 smithi026 ceph-mon[175151]: failed to encode map e698 with expected crc 2023-09-23T21:20:49.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:20:49 smithi026 ceph-mon[175151]: osdmap e698: 8 total, 8 up, 8 in 2023-09-23T21:20:49.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:20:49 smithi026 ceph-mon[175151]: failed to encode map e698 with expected crc 2023-09-23T21:20:49.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:20:49 smithi026 ceph-mon[175151]: failed to encode map e698 with expected crc 2023-09-23T21:20:49.883 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 21:20:49 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[164029]: ::ffff:172.21.15.70 - - [23/Sep/2023:21:20:49] "GET /metrics HTTP/1.1" 200 34017 "" "Prometheus/2.43.0" 2023-09-23T21:20:49.883 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:20:49 smithi026 ceph-mon[179688]: failed to encode map e698 with expected crc 2023-09-23T21:20:49.883 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:20:49 smithi026 ceph-mon[179688]: failed to encode map e698 with expected crc 2023-09-23T21:20:49.883 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:20:49 smithi026 ceph-mon[179688]: osdmap e698: 8 total, 8 up, 8 in 2023-09-23T21:20:49.883 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:20:49 smithi026 ceph-mon[179688]: failed to encode map e698 with expected crc 2023-09-23T21:20:49.883 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:20:49 smithi026 ceph-mon[179688]: failed to encode map e698 with expected crc 2023-09-23T21:20:50.517 INFO:teuthology.orchestra.run.smithi026.stderr:reweighted osd.6 to 1 (10000) 2023-09-23T21:20:50.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:20:50 smithi070 ceph-mon[168282]: pgmap v1688: 103 pgs: 103 active+clean; 4.6 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail 2023-09-23T21:20:50.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:20:50 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/932167228' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-09-23T21:20:50.875 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:20:50 smithi026 ceph-mon[175151]: pgmap v1688: 103 pgs: 103 active+clean; 4.6 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail 2023-09-23T21:20:50.875 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:20:50 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/932167228' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-09-23T21:20:50.875 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:20:50 smithi026 ceph-mon[179688]: pgmap v1688: 103 pgs: 103 active+clean; 4.6 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail 2023-09-23T21:20:50.876 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:20:50 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/932167228' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-09-23T21:20:50.881 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph osd primary-affinity 6 1 2023-09-23T21:20:51.503 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:20:51.767 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:20:51 smithi026 ceph-mon[175151]: failed to encode map e699 with expected crc 2023-09-23T21:20:51.767 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:20:51 smithi026 ceph-mon[175151]: failed to encode map e699 with expected crc 2023-09-23T21:20:51.767 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:20:51 smithi026 ceph-mon[175151]: failed to encode map e699 with expected crc 2023-09-23T21:20:51.767 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:20:51 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/932167228' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-09-23T21:20:51.767 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:20:51 smithi026 ceph-mon[175151]: osdmap e699: 8 total, 8 up, 8 in 2023-09-23T21:20:51.768 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:20:51 smithi026 ceph-mon[175151]: failed to encode map e700 with expected crc 2023-09-23T21:20:51.768 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:20:51 smithi026 ceph-mon[175151]: failed to encode map e700 with expected crc 2023-09-23T21:20:51.768 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:20:51 smithi026 ceph-mon[175151]: failed to encode map e700 with expected crc 2023-09-23T21:20:51.768 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:20:51 smithi026 ceph-mon[175151]: osdmap e700: 8 total, 8 up, 8 in 2023-09-23T21:20:51.768 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:20:51 smithi026 ceph-mon[175151]: failed to encode map e700 with expected crc 2023-09-23T21:20:51.768 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:20:51 smithi026 ceph-mon[175151]: failed to encode map e700 with expected crc 2023-09-23T21:20:51.768 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:20:51 smithi026 ceph-mon[175151]: failed to encode map e700 with expected crc 2023-09-23T21:20:51.768 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:20:51 smithi026 ceph-mon[175151]: failed to encode map e700 with expected crc 2023-09-23T21:20:51.768 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:20:51 smithi026 ceph-mon[179688]: failed to encode map e699 with expected crc 2023-09-23T21:20:51.768 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:20:51 smithi026 ceph-mon[179688]: failed to encode map e699 with expected crc 2023-09-23T21:20:51.769 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:20:51 smithi026 ceph-mon[179688]: failed to encode map e699 with expected crc 2023-09-23T21:20:51.769 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:20:51 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/932167228' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-09-23T21:20:51.769 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:20:51 smithi026 ceph-mon[179688]: osdmap e699: 8 total, 8 up, 8 in 2023-09-23T21:20:51.769 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:20:51 smithi026 ceph-mon[179688]: failed to encode map e700 with expected crc 2023-09-23T21:20:51.769 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:20:51 smithi026 ceph-mon[179688]: failed to encode map e700 with expected crc 2023-09-23T21:20:51.769 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:20:51 smithi026 ceph-mon[179688]: failed to encode map e700 with expected crc 2023-09-23T21:20:51.769 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:20:51 smithi026 ceph-mon[179688]: osdmap e700: 8 total, 8 up, 8 in 2023-09-23T21:20:51.769 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:20:51 smithi026 ceph-mon[179688]: failed to encode map e700 with expected crc 2023-09-23T21:20:51.769 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:20:51 smithi026 ceph-mon[179688]: failed to encode map e700 with expected crc 2023-09-23T21:20:51.769 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:20:51 smithi026 ceph-mon[179688]: failed to encode map e700 with expected crc 2023-09-23T21:20:51.769 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:20:51 smithi026 ceph-mon[179688]: failed to encode map e700 with expected crc 2023-09-23T21:20:51.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:20:51 smithi070 ceph-mon[168282]: failed to encode map e699 with expected crc 2023-09-23T21:20:51.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:20:51 smithi070 ceph-mon[168282]: failed to encode map e699 with expected crc 2023-09-23T21:20:51.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:20:51 smithi070 ceph-mon[168282]: failed to encode map e699 with expected crc 2023-09-23T21:20:51.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:20:51 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/932167228' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-09-23T21:20:51.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:20:51 smithi070 ceph-mon[168282]: osdmap e699: 8 total, 8 up, 8 in 2023-09-23T21:20:51.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:20:51 smithi070 ceph-mon[168282]: failed to encode map e700 with expected crc 2023-09-23T21:20:51.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:20:51 smithi070 ceph-mon[168282]: failed to encode map e700 with expected crc 2023-09-23T21:20:51.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:20:51 smithi070 ceph-mon[168282]: failed to encode map e700 with expected crc 2023-09-23T21:20:51.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:20:51 smithi070 ceph-mon[168282]: osdmap e700: 8 total, 8 up, 8 in 2023-09-23T21:20:51.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:20:51 smithi070 ceph-mon[168282]: failed to encode map e700 with expected crc 2023-09-23T21:20:51.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:20:51 smithi070 ceph-mon[168282]: failed to encode map e700 with expected crc 2023-09-23T21:20:51.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:20:51 smithi070 ceph-mon[168282]: failed to encode map e700 with expected crc 2023-09-23T21:20:51.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:20:51 smithi070 ceph-mon[168282]: failed to encode map e700 with expected crc 2023-09-23T21:20:52.513 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:20:52 smithi026 ceph-mon[175151]: pgmap v1691: 103 pgs: 103 active+clean; 4.6 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail 2023-09-23T21:20:52.513 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:20:52 smithi026 ceph-mon[175151]: failed to encode map e701 with expected crc 2023-09-23T21:20:52.513 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:20:52 smithi026 ceph-mon[175151]: failed to encode map e701 with expected crc 2023-09-23T21:20:52.513 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:20:52 smithi026 ceph-mon[175151]: osdmap e701: 8 total, 8 up, 8 in 2023-09-23T21:20:52.513 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:20:52 smithi026 ceph-mon[175151]: failed to encode map e701 with expected crc 2023-09-23T21:20:52.513 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:20:52 smithi026 ceph-mon[175151]: failed to encode map e701 with expected crc 2023-09-23T21:20:52.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:20:52 smithi070 ceph-mon[168282]: pgmap v1691: 103 pgs: 103 active+clean; 4.6 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail 2023-09-23T21:20:52.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:20:52 smithi070 ceph-mon[168282]: failed to encode map e701 with expected crc 2023-09-23T21:20:52.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:20:52 smithi070 ceph-mon[168282]: failed to encode map e701 with expected crc 2023-09-23T21:20:52.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:20:52 smithi070 ceph-mon[168282]: osdmap e701: 8 total, 8 up, 8 in 2023-09-23T21:20:52.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:20:52 smithi070 ceph-mon[168282]: failed to encode map e701 with expected crc 2023-09-23T21:20:52.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:20:52 smithi070 ceph-mon[168282]: failed to encode map e701 with expected crc 2023-09-23T21:20:52.881 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:20:52 smithi026 ceph-mon[179688]: pgmap v1691: 103 pgs: 103 active+clean; 4.6 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail 2023-09-23T21:20:52.881 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:20:52 smithi026 ceph-mon[179688]: failed to encode map e701 with expected crc 2023-09-23T21:20:52.881 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:20:52 smithi026 ceph-mon[179688]: failed to encode map e701 with expected crc 2023-09-23T21:20:52.881 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:20:52 smithi026 ceph-mon[179688]: osdmap e701: 8 total, 8 up, 8 in 2023-09-23T21:20:52.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:20:52 smithi026 ceph-mon[179688]: failed to encode map e701 with expected crc 2023-09-23T21:20:52.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:20:52 smithi026 ceph-mon[179688]: failed to encode map e701 with expected crc 2023-09-23T21:20:53.938 INFO:teuthology.orchestra.run.smithi026.stderr:set osd.6 primary-affinity to 1 (8655362) 2023-09-23T21:20:54.324 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:20:53 smithi026 ceph-mon[179688]: failed to encode map e702 with expected crc 2023-09-23T21:20:54.324 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:20:53 smithi026 ceph-mon[179688]: failed to encode map e702 with expected crc 2023-09-23T21:20:54.324 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:20:53 smithi026 ceph-mon[179688]: osdmap e702: 8 total, 8 up, 8 in 2023-09-23T21:20:54.324 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:20:53 smithi026 ceph-mon[179688]: failed to encode map e702 with expected crc 2023-09-23T21:20:54.324 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:20:53 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/144012515' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 6, "weight": 1.0}]: dispatch 2023-09-23T21:20:54.324 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:20:53 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 6, "weight": 1.0}]: dispatch 2023-09-23T21:20:54.324 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:20:53 smithi026 ceph-mon[179688]: pgmap v1694: 103 pgs: 2 active+undersized+remapped, 101 active+clean; 4.6 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 1024 KiB/s, 0 objects/s recovering 2023-09-23T21:20:54.325 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:20:53 smithi026 ceph-mon[175151]: failed to encode map e702 with expected crc 2023-09-23T21:20:54.325 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:20:53 smithi026 ceph-mon[175151]: failed to encode map e702 with expected crc 2023-09-23T21:20:54.325 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:20:53 smithi026 ceph-mon[175151]: osdmap e702: 8 total, 8 up, 8 in 2023-09-23T21:20:54.325 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:20:53 smithi026 ceph-mon[175151]: failed to encode map e702 with expected crc 2023-09-23T21:20:54.325 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:20:53 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/144012515' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 6, "weight": 1.0}]: dispatch 2023-09-23T21:20:54.325 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:20:53 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 6, "weight": 1.0}]: dispatch 2023-09-23T21:20:54.325 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:20:53 smithi026 ceph-mon[175151]: pgmap v1694: 103 pgs: 2 active+undersized+remapped, 101 active+clean; 4.6 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 1024 KiB/s, 0 objects/s recovering 2023-09-23T21:20:54.366 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph osd reweight 1 1 2023-09-23T21:20:54.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:20:53 smithi070 ceph-mon[168282]: failed to encode map e702 with expected crc 2023-09-23T21:20:54.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:20:53 smithi070 ceph-mon[168282]: failed to encode map e702 with expected crc 2023-09-23T21:20:54.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:20:53 smithi070 ceph-mon[168282]: osdmap e702: 8 total, 8 up, 8 in 2023-09-23T21:20:54.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:20:53 smithi070 ceph-mon[168282]: failed to encode map e702 with expected crc 2023-09-23T21:20:54.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:20:53 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/144012515' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 6, "weight": 1.0}]: dispatch 2023-09-23T21:20:54.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:20:53 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 6, "weight": 1.0}]: dispatch 2023-09-23T21:20:54.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:20:53 smithi070 ceph-mon[168282]: pgmap v1694: 103 pgs: 2 active+undersized+remapped, 101 active+clean; 4.6 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 1024 KiB/s, 0 objects/s recovering 2023-09-23T21:20:54.975 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:20:55.188 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:20:54 smithi026 ceph-mon[175151]: failed to encode map e703 with expected crc 2023-09-23T21:20:55.188 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:20:54 smithi026 ceph-mon[175151]: failed to encode map e703 with expected crc 2023-09-23T21:20:55.188 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:20:54 smithi026 ceph-mon[175151]: failed to encode map e703 with expected crc 2023-09-23T21:20:55.188 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:20:54 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 6, "weight": 1.0}]': finished 2023-09-23T21:20:55.188 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:20:54 smithi026 ceph-mon[175151]: osdmap e703: 8 total, 8 up, 8 in 2023-09-23T21:20:55.188 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:20:54 smithi026 ceph-mon[175151]: failed to encode map e703 with expected crc 2023-09-23T21:20:55.189 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:20:54 smithi026 ceph-mon[179688]: failed to encode map e703 with expected crc 2023-09-23T21:20:55.189 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:20:54 smithi026 ceph-mon[179688]: failed to encode map e703 with expected crc 2023-09-23T21:20:55.189 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:20:54 smithi026 ceph-mon[179688]: failed to encode map e703 with expected crc 2023-09-23T21:20:55.189 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:20:54 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 6, "weight": 1.0}]': finished 2023-09-23T21:20:55.189 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:20:54 smithi026 ceph-mon[179688]: osdmap e703: 8 total, 8 up, 8 in 2023-09-23T21:20:55.189 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:20:54 smithi026 ceph-mon[179688]: failed to encode map e703 with expected crc 2023-09-23T21:20:55.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:20:54 smithi070 ceph-mon[168282]: failed to encode map e703 with expected crc 2023-09-23T21:20:55.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:20:54 smithi070 ceph-mon[168282]: failed to encode map e703 with expected crc 2023-09-23T21:20:55.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:20:54 smithi070 ceph-mon[168282]: failed to encode map e703 with expected crc 2023-09-23T21:20:55.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:20:54 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 6, "weight": 1.0}]': finished 2023-09-23T21:20:55.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:20:54 smithi070 ceph-mon[168282]: osdmap e703: 8 total, 8 up, 8 in 2023-09-23T21:20:55.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:20:54 smithi070 ceph-mon[168282]: failed to encode map e703 with expected crc 2023-09-23T21:20:56.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:20:55 smithi070 ceph-mon[168282]: pgmap v1696: 103 pgs: 1 peering, 2 active+undersized+remapped, 100 active+clean; 4.6 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 948 KiB/s, 0 objects/s recovering 2023-09-23T21:20:56.381 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:20:55 smithi026 ceph-mon[175151]: pgmap v1696: 103 pgs: 1 peering, 2 active+undersized+remapped, 100 active+clean; 4.6 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 948 KiB/s, 0 objects/s recovering 2023-09-23T21:20:56.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:20:55 smithi026 ceph-mon[179688]: pgmap v1696: 103 pgs: 1 peering, 2 active+undersized+remapped, 100 active+clean; 4.6 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 948 KiB/s, 0 objects/s recovering 2023-09-23T21:20:56.942 INFO:teuthology.orchestra.run.smithi026.stderr:reweighted osd.1 to 1 (10000) 2023-09-23T21:20:57.366 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:20:56 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/3607861292' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-09-23T21:20:57.366 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:20:56 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-09-23T21:20:57.366 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:20:56 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/3607861292' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-09-23T21:20:57.366 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:20:56 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-09-23T21:20:57.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:20:56 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/3607861292' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-09-23T21:20:57.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:20:56 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-09-23T21:20:57.438 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph osd primary-affinity 1 1 2023-09-23T21:20:58.052 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:20:58.247 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:20:57 smithi026 ceph-mon[175151]: failed to encode map e704 with expected crc 2023-09-23T21:20:58.247 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:20:57 smithi026 ceph-mon[175151]: failed to encode map e704 with expected crc 2023-09-23T21:20:58.247 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:20:57 smithi026 ceph-mon[175151]: failed to encode map e704 with expected crc 2023-09-23T21:20:58.247 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:20:57 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-09-23T21:20:58.247 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:20:57 smithi026 ceph-mon[175151]: osdmap e704: 8 total, 8 up, 8 in 2023-09-23T21:20:58.247 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:20:57 smithi026 ceph-mon[175151]: pgmap v1698: 103 pgs: 1 peering, 1 active+undersized+remapped, 101 active+clean; 4.6 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 771 KiB/s, 0 objects/s recovering 2023-09-23T21:20:58.247 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:20:57 smithi026 ceph-mon[179688]: failed to encode map e704 with expected crc 2023-09-23T21:20:58.247 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:20:57 smithi026 ceph-mon[179688]: failed to encode map e704 with expected crc 2023-09-23T21:20:58.248 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:20:57 smithi026 ceph-mon[179688]: failed to encode map e704 with expected crc 2023-09-23T21:20:58.248 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:20:57 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-09-23T21:20:58.248 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:20:57 smithi026 ceph-mon[179688]: osdmap e704: 8 total, 8 up, 8 in 2023-09-23T21:20:58.248 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:20:57 smithi026 ceph-mon[179688]: pgmap v1698: 103 pgs: 1 peering, 1 active+undersized+remapped, 101 active+clean; 4.6 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 771 KiB/s, 0 objects/s recovering 2023-09-23T21:20:58.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:20:57 smithi070 ceph-mon[168282]: failed to encode map e704 with expected crc 2023-09-23T21:20:58.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:20:57 smithi070 ceph-mon[168282]: failed to encode map e704 with expected crc 2023-09-23T21:20:58.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:20:57 smithi070 ceph-mon[168282]: failed to encode map e704 with expected crc 2023-09-23T21:20:58.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:20:57 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-09-23T21:20:58.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:20:57 smithi070 ceph-mon[168282]: osdmap e704: 8 total, 8 up, 8 in 2023-09-23T21:20:58.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:20:57 smithi070 ceph-mon[168282]: pgmap v1698: 103 pgs: 1 peering, 1 active+undersized+remapped, 101 active+clean; 4.6 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 771 KiB/s, 0 objects/s recovering 2023-09-23T21:20:59.881 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 21:20:59 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[164029]: ::ffff:172.21.15.70 - - [23/Sep/2023:21:20:59] "GET /metrics HTTP/1.1" 200 34017 "" "Prometheus/2.43.0" 2023-09-23T21:21:00.300 INFO:teuthology.orchestra.run.smithi026.stderr:set osd.1 primary-affinity to 1 (8655362) 2023-09-23T21:21:00.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:21:00 smithi070 ceph-mon[168282]: pgmap v1699: 103 pgs: 103 active+clean; 4.6 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail 2023-09-23T21:21:00.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:21:00 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/482825770' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 1, "weight": 1.0}]: dispatch 2023-09-23T21:21:00.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:21:00 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 1, "weight": 1.0}]: dispatch 2023-09-23T21:21:00.633 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:21:00 smithi026 ceph-mon[179688]: pgmap v1699: 103 pgs: 103 active+clean; 4.6 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail 2023-09-23T21:21:00.633 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:21:00 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/482825770' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 1, "weight": 1.0}]: dispatch 2023-09-23T21:21:00.633 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:21:00 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 1, "weight": 1.0}]: dispatch 2023-09-23T21:21:00.633 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:21:00 smithi026 ceph-mon[175151]: pgmap v1699: 103 pgs: 103 active+clean; 4.6 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail 2023-09-23T21:21:00.633 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:21:00 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/482825770' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 1, "weight": 1.0}]: dispatch 2023-09-23T21:21:00.634 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:21:00 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 1, "weight": 1.0}]: dispatch 2023-09-23T21:21:00.720 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph osd reweight 0 1 2023-09-23T21:21:01.334 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:21:01.556 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:21:01 smithi026 ceph-mon[175151]: failed to encode map e705 with expected crc 2023-09-23T21:21:01.557 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:21:01 smithi026 ceph-mon[175151]: failed to encode map e705 with expected crc 2023-09-23T21:21:01.557 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:21:01 smithi026 ceph-mon[175151]: failed to encode map e705 with expected crc 2023-09-23T21:21:01.557 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:21:01 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 1, "weight": 1.0}]': finished 2023-09-23T21:21:01.557 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:21:01 smithi026 ceph-mon[175151]: osdmap e705: 8 total, 8 up, 8 in 2023-09-23T21:21:01.557 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:21:01 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:21:01.557 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:21:01 smithi026 ceph-mon[179688]: failed to encode map e705 with expected crc 2023-09-23T21:21:01.557 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:21:01 smithi026 ceph-mon[179688]: failed to encode map e705 with expected crc 2023-09-23T21:21:01.557 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:21:01 smithi026 ceph-mon[179688]: failed to encode map e705 with expected crc 2023-09-23T21:21:01.557 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:21:01 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 1, "weight": 1.0}]': finished 2023-09-23T21:21:01.557 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:21:01 smithi026 ceph-mon[179688]: osdmap e705: 8 total, 8 up, 8 in 2023-09-23T21:21:01.558 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:21:01 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:21:01.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:21:01 smithi070 ceph-mon[168282]: failed to encode map e705 with expected crc 2023-09-23T21:21:01.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:21:01 smithi070 ceph-mon[168282]: failed to encode map e705 with expected crc 2023-09-23T21:21:01.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:21:01 smithi070 ceph-mon[168282]: failed to encode map e705 with expected crc 2023-09-23T21:21:01.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:21:01 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 1, "weight": 1.0}]': finished 2023-09-23T21:21:01.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:21:01 smithi070 ceph-mon[168282]: osdmap e705: 8 total, 8 up, 8 in 2023-09-23T21:21:01.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:21:01 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:21:02.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:21:02 smithi026 ceph-mon[175151]: pgmap v1701: 103 pgs: 103 active+clean; 4.6 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail 2023-09-23T21:21:02.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:21:02 smithi026 ceph-mon[179688]: pgmap v1701: 103 pgs: 103 active+clean; 4.6 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail 2023-09-23T21:21:02.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:21:02 smithi070 ceph-mon[168282]: pgmap v1701: 103 pgs: 103 active+clean; 4.6 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail 2023-09-23T21:21:03.309 INFO:teuthology.orchestra.run.smithi026.stderr:reweighted osd.0 to 1 (10000) 2023-09-23T21:21:03.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:21:03 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/4214220040' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-09-23T21:21:03.631 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:21:03 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/4214220040' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-09-23T21:21:03.631 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:21:03 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/4214220040' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-09-23T21:21:03.792 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph osd primary-affinity 0 1 2023-09-23T21:21:04.408 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:21:04.587 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:21:04 smithi026 ceph-mon[179688]: pgmap v1702: 103 pgs: 103 active+clean; 4.6 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail 2023-09-23T21:21:04.587 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:21:04 smithi026 ceph-mon[179688]: failed to encode map e706 with expected crc 2023-09-23T21:21:04.587 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:21:04 smithi026 ceph-mon[179688]: failed to encode map e706 with expected crc 2023-09-23T21:21:04.587 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:21:04 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/4214220040' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-09-23T21:21:04.587 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:21:04 smithi026 ceph-mon[179688]: osdmap e706: 8 total, 8 up, 8 in 2023-09-23T21:21:04.588 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:21:04 smithi026 ceph-mon[175151]: pgmap v1702: 103 pgs: 103 active+clean; 4.6 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail 2023-09-23T21:21:04.588 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:21:04 smithi026 ceph-mon[175151]: failed to encode map e706 with expected crc 2023-09-23T21:21:04.588 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:21:04 smithi026 ceph-mon[175151]: failed to encode map e706 with expected crc 2023-09-23T21:21:04.588 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:21:04 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/4214220040' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-09-23T21:21:04.588 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:21:04 smithi026 ceph-mon[175151]: osdmap e706: 8 total, 8 up, 8 in 2023-09-23T21:21:04.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:21:04 smithi070 ceph-mon[168282]: pgmap v1702: 103 pgs: 103 active+clean; 4.6 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail 2023-09-23T21:21:04.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:21:04 smithi070 ceph-mon[168282]: failed to encode map e706 with expected crc 2023-09-23T21:21:04.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:21:04 smithi070 ceph-mon[168282]: failed to encode map e706 with expected crc 2023-09-23T21:21:04.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:21:04 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/4214220040' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-09-23T21:21:04.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:21:04 smithi070 ceph-mon[168282]: osdmap e706: 8 total, 8 up, 8 in 2023-09-23T21:21:05.421 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:21:05 smithi026 ceph-mon[175151]: failed to encode map e706 with expected crc 2023-09-23T21:21:05.421 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:21:05 smithi026 ceph-mon[175151]: failed to encode map e706 with expected crc 2023-09-23T21:21:05.421 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:21:05 smithi026 ceph-mon[179688]: failed to encode map e706 with expected crc 2023-09-23T21:21:05.421 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:21:05 smithi026 ceph-mon[179688]: failed to encode map e706 with expected crc 2023-09-23T21:21:05.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:21:05 smithi070 ceph-mon[168282]: failed to encode map e706 with expected crc 2023-09-23T21:21:05.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:21:05 smithi070 ceph-mon[168282]: failed to encode map e706 with expected crc 2023-09-23T21:21:06.319 INFO:teuthology.orchestra.run.smithi026.stderr:set osd.0 primary-affinity to 1 (8655362) 2023-09-23T21:21:06.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:21:06 smithi070 ceph-mon[168282]: pgmap v1704: 103 pgs: 103 active+clean; 4.6 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail 2023-09-23T21:21:06.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:21:06 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/1359382395' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 0, "weight": 1.0}]: dispatch 2023-09-23T21:21:06.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:21:06 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 0, "weight": 1.0}]: dispatch 2023-09-23T21:21:06.631 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:21:06 smithi026 ceph-mon[179688]: pgmap v1704: 103 pgs: 103 active+clean; 4.6 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail 2023-09-23T21:21:06.631 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:21:06 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/1359382395' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 0, "weight": 1.0}]: dispatch 2023-09-23T21:21:06.631 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:21:06 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 0, "weight": 1.0}]: dispatch 2023-09-23T21:21:06.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:21:06 smithi026 ceph-mon[175151]: pgmap v1704: 103 pgs: 103 active+clean; 4.6 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail 2023-09-23T21:21:06.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:21:06 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/1359382395' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 0, "weight": 1.0}]: dispatch 2023-09-23T21:21:06.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:21:06 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 0, "weight": 1.0}]: dispatch 2023-09-23T21:21:06.773 INFO:tasks.thrashosds.thrasher:joining the do_sighup greenlet 2023-09-23T21:21:06.773 INFO:tasks.thrashosds.thrasher:joining the do_optrack_toggle greenlet 2023-09-23T21:21:06.774 INFO:tasks.thrashosds.thrasher:joining the do_dump_ops greenlet 2023-09-23T21:21:06.774 INFO:tasks.thrashosds.thrasher:joining the do_noscrub_toggle greenlet 2023-09-23T21:21:06.774 INFO:tasks.cephadm.ceph_manager.ceph:waiting for all up 2023-09-23T21:21:06.774 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph osd dump --format=json 2023-09-23T21:21:07.386 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:21:07.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:21:07 smithi070 ceph-mon[168282]: failed to encode map e707 with expected crc 2023-09-23T21:21:07.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:21:07 smithi070 ceph-mon[168282]: failed to encode map e707 with expected crc 2023-09-23T21:21:07.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:21:07 smithi070 ceph-mon[168282]: failed to encode map e707 with expected crc 2023-09-23T21:21:07.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:21:07 smithi070 ceph-mon[168282]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 0, "weight": 1.0}]': finished 2023-09-23T21:21:07.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:21:07 smithi070 ceph-mon[168282]: osdmap e707: 8 total, 8 up, 8 in 2023-09-23T21:21:07.631 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:21:07 smithi026 ceph-mon[179688]: failed to encode map e707 with expected crc 2023-09-23T21:21:07.631 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:21:07 smithi026 ceph-mon[179688]: failed to encode map e707 with expected crc 2023-09-23T21:21:07.631 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:21:07 smithi026 ceph-mon[179688]: failed to encode map e707 with expected crc 2023-09-23T21:21:07.631 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:21:07 smithi026 ceph-mon[179688]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 0, "weight": 1.0}]': finished 2023-09-23T21:21:07.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:21:07 smithi026 ceph-mon[179688]: osdmap e707: 8 total, 8 up, 8 in 2023-09-23T21:21:07.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:21:07 smithi026 ceph-mon[175151]: failed to encode map e707 with expected crc 2023-09-23T21:21:07.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:21:07 smithi026 ceph-mon[175151]: failed to encode map e707 with expected crc 2023-09-23T21:21:07.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:21:07 smithi026 ceph-mon[175151]: failed to encode map e707 with expected crc 2023-09-23T21:21:07.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:21:07 smithi026 ceph-mon[175151]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 0, "weight": 1.0}]': finished 2023-09-23T21:21:07.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:21:07 smithi026 ceph-mon[175151]: osdmap e707: 8 total, 8 up, 8 in 2023-09-23T21:21:08.452 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:21:08 smithi026 ceph-mon[179688]: failed to encode map e707 with expected crc 2023-09-23T21:21:08.452 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:21:08 smithi026 ceph-mon[179688]: pgmap v1706: 103 pgs: 103 active+clean; 4.6 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail 2023-09-23T21:21:08.452 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:21:08 smithi026 ceph-mon[175151]: failed to encode map e707 with expected crc 2023-09-23T21:21:08.452 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:21:08 smithi026 ceph-mon[175151]: pgmap v1706: 103 pgs: 103 active+clean; 4.6 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail 2023-09-23T21:21:08.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:21:08 smithi070 ceph-mon[168282]: failed to encode map e707 with expected crc 2023-09-23T21:21:08.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:21:08 smithi070 ceph-mon[168282]: pgmap v1706: 103 pgs: 103 active+clean; 4.6 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail 2023-09-23T21:21:08.813 INFO:teuthology.orchestra.run.smithi026.stdout: 2023-09-23T21:21:08.813 INFO:teuthology.orchestra.run.smithi026.stdout:{"epoch":707,"fsid":"b3f8e4ec-5a50-11ee-8db4-212e2dc638e7","created":"2023-09-23T20:35:58.919705+0000","modified":"2023-09-23T21:21:06.307600+0000","last_up_change":"2023-09-23T21:13:11.143516+0000","last_in_change":"2023-09-23T21:20:15.053512+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":25,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":12,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"quincy","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2023-09-23T20:38:57.022844+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":31,"pg_placement_num":31,"pg_placement_num_target":31,"pg_num_target":31,"pg_num_pending":31,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"539","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"479","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_acting":2.059999942779541,"score_stable":2.059999942779541,"optimal_score":1,"raw_score_acting":2.059999942779541,"raw_score_stable":2.059999942779541,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2023-09-23T20:41:15.182110+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"667","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":15,"snap_epoch":667,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_acting":2,"score_stable":2,"optimal_score":1,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs.foo.meta","create_time":"2023-09-23T20:41:57.679446+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"68","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"foo"}},"read_balance":{"score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":4,"pool_name":"cephfs.foo.data","create_time":"2023-09-23T20:41:58.331482+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"68","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"cephfs":{"data":"foo"}},"read_balance":{"score_acting":2.25,"score_stable":2.25,"optimal_score":1,"raw_score_acting":2.25,"raw_score_stable":2.25,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"f837feb7-060e-4965-9b48-bed9f3672d05","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":83,"last_clean_end":464,"up_from":493,"up_thru":684,"down_at":465,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6802","nonce":1823091461},{"type":"v1","addr":"172.21.15.26:6803","nonce":1823091461}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6804","nonce":1823091461},{"type":"v1","addr":"172.21.15.26:6805","nonce":1823091461}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6808","nonce":1823091461},{"type":"v1","addr":"172.21.15.26:6809","nonce":1823091461}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6806","nonce":1823091461},{"type":"v1","addr":"172.21.15.26:6807","nonce":1823091461}]},"public_addr":"172.21.15.26:6803/1823091461","cluster_addr":"172.21.15.26:6805/1823091461","heartbeat_back_addr":"172.21.15.26:6809/1823091461","heartbeat_front_addr":"172.21.15.26:6807/1823091461","state":["exists","up"]},{"osd":1,"uuid":"917f65aa-5963-4576-988a-396698b7c6a8","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":88,"last_clean_end":338,"up_from":350,"up_thru":702,"down_at":339,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6810","nonce":2971312233},{"type":"v1","addr":"172.21.15.26:6811","nonce":2971312233}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6812","nonce":2971312233},{"type":"v1","addr":"172.21.15.26:6813","nonce":2971312233}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6816","nonce":2971312233},{"type":"v1","addr":"172.21.15.26:6817","nonce":2971312233}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6814","nonce":2971312233},{"type":"v1","addr":"172.21.15.26:6815","nonce":2971312233}]},"public_addr":"172.21.15.26:6811/2971312233","cluster_addr":"172.21.15.26:6813/2971312233","heartbeat_back_addr":"172.21.15.26:6817/2971312233","heartbeat_front_addr":"172.21.15.26:6815/2971312233","state":["exists","up"]},{"osd":2,"uuid":"2e115328-76c6-478e-915c-5786903be49f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":18,"last_clean_end":89,"up_from":93,"up_thru":684,"down_at":90,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6818","nonce":38979282},{"type":"v1","addr":"172.21.15.26:6819","nonce":38979282}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6820","nonce":38979282},{"type":"v1","addr":"172.21.15.26:6821","nonce":38979282}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6824","nonce":38979282},{"type":"v1","addr":"172.21.15.26:6825","nonce":38979282}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6822","nonce":38979282},{"type":"v1","addr":"172.21.15.26:6823","nonce":38979282}]},"public_addr":"172.21.15.26:6819/38979282","cluster_addr":"172.21.15.26:6821/38979282","heartbeat_back_addr":"172.21.15.26:6825/38979282","heartbeat_front_addr":"172.21.15.26:6823/38979282","state":["exists","up"]},{"osd":3,"uuid":"c3dd2c3f-5cf5-4fb0-964c-1b8926f75c8a","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":26,"last_clean_end":94,"up_from":98,"up_thru":699,"down_at":95,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6826","nonce":2947396145},{"type":"v1","addr":"172.21.15.26:6827","nonce":2947396145}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6828","nonce":2947396145},{"type":"v1","addr":"172.21.15.26:6829","nonce":2947396145}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6832","nonce":2947396145},{"type":"v1","addr":"172.21.15.26:6833","nonce":2947396145}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6830","nonce":2947396145},{"type":"v1","addr":"172.21.15.26:6831","nonce":2947396145}]},"public_addr":"172.21.15.26:6827/2947396145","cluster_addr":"172.21.15.26:6829/2947396145","heartbeat_back_addr":"172.21.15.26:6833/2947396145","heartbeat_front_addr":"172.21.15.26:6831/2947396145","state":["exists","up"]},{"osd":4,"uuid":"a5f31996-1b38-491b-956b-5aebc72d0170","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":31,"up_thru":702,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6800","nonce":1528249138},{"type":"v1","addr":"172.21.15.70:6801","nonce":1528249138}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6802","nonce":1528249138},{"type":"v1","addr":"172.21.15.70:6803","nonce":1528249138}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6806","nonce":1528249138},{"type":"v1","addr":"172.21.15.70:6807","nonce":1528249138}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6804","nonce":1528249138},{"type":"v1","addr":"172.21.15.70:6805","nonce":1528249138}]},"public_addr":"172.21.15.70:6801/1528249138","cluster_addr":"172.21.15.70:6803/1528249138","heartbeat_back_addr":"172.21.15.70:6807/1528249138","heartbeat_front_addr":"172.21.15.70:6805/1528249138","state":["exists","up"]},{"osd":5,"uuid":"21dad198-b9db-42c6-8823-112df1ecd874","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":39,"up_thru":702,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6808","nonce":3802232906},{"type":"v1","addr":"172.21.15.70:6809","nonce":3802232906}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6810","nonce":3802232906},{"type":"v1","addr":"172.21.15.70:6811","nonce":3802232906}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6814","nonce":3802232906},{"type":"v1","addr":"172.21.15.70:6815","nonce":3802232906}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6812","nonce":3802232906},{"type":"v1","addr":"172.21.15.70:6813","nonce":3802232906}]},"public_addr":"172.21.15.70:6809/3802232906","cluster_addr":"172.21.15.70:6811/3802232906","heartbeat_back_addr":"172.21.15.70:6815/3802232906","heartbeat_front_addr":"172.21.15.70:6813/3802232906","state":["exists","up"]},{"osd":6,"uuid":"6f3be784-7149-4f13-81b5-98924e97c056","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":47,"last_clean_end":154,"up_from":164,"up_thru":699,"down_at":155,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6816","nonce":4141218814},{"type":"v1","addr":"172.21.15.70:6817","nonce":4141218814}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6818","nonce":4141218814},{"type":"v1","addr":"172.21.15.70:6819","nonce":4141218814}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6822","nonce":4141218814},{"type":"v1","addr":"172.21.15.70:6823","nonce":4141218814}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6820","nonce":4141218814},{"type":"v1","addr":"172.21.15.70:6821","nonce":4141218814}]},"public_addr":"172.21.15.70:6817/4141218814","cluster_addr":"172.21.15.70:6819/4141218814","heartbeat_back_addr":"172.21.15.70:6823/4141218814","heartbeat_front_addr":"172.21.15.70:6821/4141218814","state":["exists","up"]},{"osd":7,"uuid":"499f5f85-f4b2-40a7-ac0f-79cd22738d30","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":55,"up_thru":697,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6824","nonce":2824974165},{"type":"v1","addr":"172.21.15.70:6825","nonce":2824974165}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6826","nonce":2824974165},{"type":"v1","addr":"172.21.15.70:6827","nonce":2824974165}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6830","nonce":2824974165},{"type":"v1","addr":"172.21.15.70:6831","nonce":2824974165}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6828","nonce":2824974165},{"type":"v1","addr":"172.21.15.70:6829","nonce":2824974165}]},"public_addr":"172.21.15.70:6825/2824974165","cluster_addr":"172.21.15.70:6827/2824974165","heartbeat_back_addr":"172.21.15.70:6831/2824974165","heartbeat_front_addr":"172.21.15.70:6829/2824974165","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T20:38:10.875468+0000","dead_epoch":464},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T20:38:32.449963+0000","dead_epoch":338},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T20:38:53.633316+0000","dead_epoch":89},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T20:39:14.828940+0000","dead_epoch":94},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T20:39:34.945663+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T20:39:55.954288+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T20:40:16.847102+0000","dead_epoch":154},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T20:40:37.820404+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"3.1d","mappings":[{"from":4,"to":2}]}],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.26:0/2817523682":"2023-09-24T20:45:16.037413+0000","172.21.15.26:0/934875067":"2023-09-24T20:45:16.037413+0000","172.21.15.26:0/190503462":"2023-09-24T20:45:16.037413+0000","172.21.15.26:0/3175144810":"2023-09-24T20:36:31.159649+0000","172.21.15.26:0/188049283":"2023-09-24T20:41:03.790648+0000","172.21.15.26:0/1815190465":"2023-09-24T20:36:31.159649+0000","172.21.15.26:0/289994511":"2023-09-24T20:36:56.498559+0000","172.21.15.26:6801/91363814":"2023-09-24T20:41:03.790648+0000","172.21.15.70:0/2989870914":"2023-09-24T20:44:32.835628+0000","172.21.15.26:6800/2440434873":"2023-09-24T20:43:50.641728+0000","172.21.15.26:0/1083572999":"2023-09-24T20:41:03.790648+0000","172.21.15.70:0/3393166416":"2023-09-24T20:44:32.835628+0000","172.21.15.26:6800/952511372":"2023-09-24T20:36:56.498559+0000","172.21.15.70:0/2198692933":"2023-09-24T20:44:32.835628+0000","172.21.15.26:6801/2440434873":"2023-09-24T20:43:50.641728+0000","172.21.15.70:0/302414747":"2023-09-24T20:44:32.835628+0000","172.21.15.70:0/413499922":"2023-09-24T20:44:32.835628+0000","172.21.15.26:0/686734212":"2023-09-24T20:36:31.159649+0000","172.21.15.70:6834/1275773750":"2023-09-24T20:44:32.835628+0000","172.21.15.26:0/635571892":"2023-09-24T20:43:50.641728+0000","172.21.15.26:0/1376232363":"2023-09-24T20:43:50.641728+0000","172.21.15.26:0/394447930":"2023-09-24T20:41:03.790648+0000","172.21.15.70:6835/1275773750":"2023-09-24T20:44:32.835628+0000","172.21.15.26:0/1625326392":"2023-09-24T20:36:56.498559+0000","172.21.15.26:6801/952511372":"2023-09-24T20:36:56.498559+0000","172.21.15.26:0/2838335646":"2023-09-24T20:36:56.498559+0000","172.21.15.26:6801/4099624907":"2023-09-24T20:36:31.159649+0000","172.21.15.26:0/3408257869":"2023-09-24T20:45:16.037413+0000","172.21.15.26:6800/91363814":"2023-09-24T20:41:03.790648+0000","172.21.15.70:0/1035724255":"2023-09-24T20:44:32.835628+0000","172.21.15.26:0/4215395943":"2023-09-24T20:43:50.641728+0000","172.21.15.26:0/233963796":"2023-09-24T20:45:16.037413+0000","172.21.15.26:0/3461760126":"2023-09-24T20:41:03.790648+0000","172.21.15.26:0/237900280":"2023-09-24T20:43:50.641728+0000","172.21.15.26:6801/1446178988":"2023-09-24T20:45:16.037413+0000","172.21.15.26:0/2633606443":"2023-09-24T20:45:16.037413+0000","172.21.15.26:6800/4099624907":"2023-09-24T20:36:31.159649+0000","172.21.15.26:6800/1446178988":"2023-09-24T20:45:16.037413+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[{"pool":5,"snaps":[{"begin":2,"length":1}]},{"pool":12,"snaps":[{"begin":2,"length":1}]}],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2023-09-23T21:21:09.251 INFO:tasks.cephadm.ceph_manager.ceph:all up! 2023-09-23T21:21:09.251 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph osd dump --format=json 2023-09-23T21:21:09.612 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:21:09 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/4018762477' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-09-23T21:21:09.613 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:21:09 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/4018762477' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-09-23T21:21:09.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:21:09 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/4018762477' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-09-23T21:21:09.862 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:21:09.881 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 21:21:09 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[164029]: ::ffff:172.21.15.70 - - [23/Sep/2023:21:21:09] "GET /metrics HTTP/1.1" 200 34003 "" "Prometheus/2.43.0" 2023-09-23T21:21:10.372 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:21:10 smithi026 ceph-mon[175151]: pgmap v1707: 103 pgs: 103 active+clean; 4.6 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail 2023-09-23T21:21:10.372 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:21:10 smithi026 ceph-mon[179688]: pgmap v1707: 103 pgs: 103 active+clean; 4.6 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail 2023-09-23T21:21:10.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:21:10 smithi070 ceph-mon[168282]: pgmap v1707: 103 pgs: 103 active+clean; 4.6 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail 2023-09-23T21:21:11.255 INFO:teuthology.orchestra.run.smithi026.stdout: 2023-09-23T21:21:11.255 INFO:teuthology.orchestra.run.smithi026.stdout:{"epoch":707,"fsid":"b3f8e4ec-5a50-11ee-8db4-212e2dc638e7","created":"2023-09-23T20:35:58.919705+0000","modified":"2023-09-23T21:21:06.307600+0000","last_up_change":"2023-09-23T21:13:11.143516+0000","last_in_change":"2023-09-23T21:20:15.053512+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":25,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":12,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"quincy","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2023-09-23T20:38:57.022844+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":31,"pg_placement_num":31,"pg_placement_num_target":31,"pg_num_target":31,"pg_num_pending":31,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"539","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"479","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_acting":2.059999942779541,"score_stable":2.059999942779541,"optimal_score":1,"raw_score_acting":2.059999942779541,"raw_score_stable":2.059999942779541,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2023-09-23T20:41:15.182110+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"667","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":15,"snap_epoch":667,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_acting":2,"score_stable":2,"optimal_score":1,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs.foo.meta","create_time":"2023-09-23T20:41:57.679446+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"68","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"foo"}},"read_balance":{"score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":4,"pool_name":"cephfs.foo.data","create_time":"2023-09-23T20:41:58.331482+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"68","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"cephfs":{"data":"foo"}},"read_balance":{"score_acting":2.25,"score_stable":2.25,"optimal_score":1,"raw_score_acting":2.25,"raw_score_stable":2.25,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"f837feb7-060e-4965-9b48-bed9f3672d05","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":83,"last_clean_end":464,"up_from":493,"up_thru":684,"down_at":465,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6802","nonce":1823091461},{"type":"v1","addr":"172.21.15.26:6803","nonce":1823091461}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6804","nonce":1823091461},{"type":"v1","addr":"172.21.15.26:6805","nonce":1823091461}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6808","nonce":1823091461},{"type":"v1","addr":"172.21.15.26:6809","nonce":1823091461}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6806","nonce":1823091461},{"type":"v1","addr":"172.21.15.26:6807","nonce":1823091461}]},"public_addr":"172.21.15.26:6803/1823091461","cluster_addr":"172.21.15.26:6805/1823091461","heartbeat_back_addr":"172.21.15.26:6809/1823091461","heartbeat_front_addr":"172.21.15.26:6807/1823091461","state":["exists","up"]},{"osd":1,"uuid":"917f65aa-5963-4576-988a-396698b7c6a8","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":88,"last_clean_end":338,"up_from":350,"up_thru":702,"down_at":339,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6810","nonce":2971312233},{"type":"v1","addr":"172.21.15.26:6811","nonce":2971312233}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6812","nonce":2971312233},{"type":"v1","addr":"172.21.15.26:6813","nonce":2971312233}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6816","nonce":2971312233},{"type":"v1","addr":"172.21.15.26:6817","nonce":2971312233}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6814","nonce":2971312233},{"type":"v1","addr":"172.21.15.26:6815","nonce":2971312233}]},"public_addr":"172.21.15.26:6811/2971312233","cluster_addr":"172.21.15.26:6813/2971312233","heartbeat_back_addr":"172.21.15.26:6817/2971312233","heartbeat_front_addr":"172.21.15.26:6815/2971312233","state":["exists","up"]},{"osd":2,"uuid":"2e115328-76c6-478e-915c-5786903be49f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":18,"last_clean_end":89,"up_from":93,"up_thru":684,"down_at":90,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6818","nonce":38979282},{"type":"v1","addr":"172.21.15.26:6819","nonce":38979282}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6820","nonce":38979282},{"type":"v1","addr":"172.21.15.26:6821","nonce":38979282}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6824","nonce":38979282},{"type":"v1","addr":"172.21.15.26:6825","nonce":38979282}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6822","nonce":38979282},{"type":"v1","addr":"172.21.15.26:6823","nonce":38979282}]},"public_addr":"172.21.15.26:6819/38979282","cluster_addr":"172.21.15.26:6821/38979282","heartbeat_back_addr":"172.21.15.26:6825/38979282","heartbeat_front_addr":"172.21.15.26:6823/38979282","state":["exists","up"]},{"osd":3,"uuid":"c3dd2c3f-5cf5-4fb0-964c-1b8926f75c8a","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":26,"last_clean_end":94,"up_from":98,"up_thru":699,"down_at":95,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6826","nonce":2947396145},{"type":"v1","addr":"172.21.15.26:6827","nonce":2947396145}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6828","nonce":2947396145},{"type":"v1","addr":"172.21.15.26:6829","nonce":2947396145}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6832","nonce":2947396145},{"type":"v1","addr":"172.21.15.26:6833","nonce":2947396145}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6830","nonce":2947396145},{"type":"v1","addr":"172.21.15.26:6831","nonce":2947396145}]},"public_addr":"172.21.15.26:6827/2947396145","cluster_addr":"172.21.15.26:6829/2947396145","heartbeat_back_addr":"172.21.15.26:6833/2947396145","heartbeat_front_addr":"172.21.15.26:6831/2947396145","state":["exists","up"]},{"osd":4,"uuid":"a5f31996-1b38-491b-956b-5aebc72d0170","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":31,"up_thru":702,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6800","nonce":1528249138},{"type":"v1","addr":"172.21.15.70:6801","nonce":1528249138}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6802","nonce":1528249138},{"type":"v1","addr":"172.21.15.70:6803","nonce":1528249138}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6806","nonce":1528249138},{"type":"v1","addr":"172.21.15.70:6807","nonce":1528249138}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6804","nonce":1528249138},{"type":"v1","addr":"172.21.15.70:6805","nonce":1528249138}]},"public_addr":"172.21.15.70:6801/1528249138","cluster_addr":"172.21.15.70:6803/1528249138","heartbeat_back_addr":"172.21.15.70:6807/1528249138","heartbeat_front_addr":"172.21.15.70:6805/1528249138","state":["exists","up"]},{"osd":5,"uuid":"21dad198-b9db-42c6-8823-112df1ecd874","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":39,"up_thru":702,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6808","nonce":3802232906},{"type":"v1","addr":"172.21.15.70:6809","nonce":3802232906}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6810","nonce":3802232906},{"type":"v1","addr":"172.21.15.70:6811","nonce":3802232906}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6814","nonce":3802232906},{"type":"v1","addr":"172.21.15.70:6815","nonce":3802232906}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6812","nonce":3802232906},{"type":"v1","addr":"172.21.15.70:6813","nonce":3802232906}]},"public_addr":"172.21.15.70:6809/3802232906","cluster_addr":"172.21.15.70:6811/3802232906","heartbeat_back_addr":"172.21.15.70:6815/3802232906","heartbeat_front_addr":"172.21.15.70:6813/3802232906","state":["exists","up"]},{"osd":6,"uuid":"6f3be784-7149-4f13-81b5-98924e97c056","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":47,"last_clean_end":154,"up_from":164,"up_thru":699,"down_at":155,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6816","nonce":4141218814},{"type":"v1","addr":"172.21.15.70:6817","nonce":4141218814}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6818","nonce":4141218814},{"type":"v1","addr":"172.21.15.70:6819","nonce":4141218814}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6822","nonce":4141218814},{"type":"v1","addr":"172.21.15.70:6823","nonce":4141218814}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6820","nonce":4141218814},{"type":"v1","addr":"172.21.15.70:6821","nonce":4141218814}]},"public_addr":"172.21.15.70:6817/4141218814","cluster_addr":"172.21.15.70:6819/4141218814","heartbeat_back_addr":"172.21.15.70:6823/4141218814","heartbeat_front_addr":"172.21.15.70:6821/4141218814","state":["exists","up"]},{"osd":7,"uuid":"499f5f85-f4b2-40a7-ac0f-79cd22738d30","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":55,"up_thru":697,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6824","nonce":2824974165},{"type":"v1","addr":"172.21.15.70:6825","nonce":2824974165}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6826","nonce":2824974165},{"type":"v1","addr":"172.21.15.70:6827","nonce":2824974165}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6830","nonce":2824974165},{"type":"v1","addr":"172.21.15.70:6831","nonce":2824974165}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6828","nonce":2824974165},{"type":"v1","addr":"172.21.15.70:6829","nonce":2824974165}]},"public_addr":"172.21.15.70:6825/2824974165","cluster_addr":"172.21.15.70:6827/2824974165","heartbeat_back_addr":"172.21.15.70:6831/2824974165","heartbeat_front_addr":"172.21.15.70:6829/2824974165","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T20:38:10.875468+0000","dead_epoch":464},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T20:38:32.449963+0000","dead_epoch":338},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T20:38:53.633316+0000","dead_epoch":89},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T20:39:14.828940+0000","dead_epoch":94},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T20:39:34.945663+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T20:39:55.954288+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T20:40:16.847102+0000","dead_epoch":154},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T20:40:37.820404+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"3.1d","mappings":[{"from":4,"to":2}]}],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.26:6801/91363814":"2023-09-24T20:41:03.790648+0000","172.21.15.26:0/3175144810":"2023-09-24T20:36:31.159649+0000","172.21.15.26:0/1625326392":"2023-09-24T20:36:56.498559+0000","172.21.15.70:0/3393166416":"2023-09-24T20:44:32.835628+0000","172.21.15.26:0/1083572999":"2023-09-24T20:41:03.790648+0000","172.21.15.26:6801/2440434873":"2023-09-24T20:43:50.641728+0000","172.21.15.70:6834/1275773750":"2023-09-24T20:44:32.835628+0000","172.21.15.26:0/289994511":"2023-09-24T20:36:56.498559+0000","172.21.15.70:0/2989870914":"2023-09-24T20:44:32.835628+0000","172.21.15.26:6800/2440434873":"2023-09-24T20:43:50.641728+0000","172.21.15.70:0/302414747":"2023-09-24T20:44:32.835628+0000","172.21.15.70:0/413499922":"2023-09-24T20:44:32.835628+0000","172.21.15.26:0/4215395943":"2023-09-24T20:43:50.641728+0000","172.21.15.26:0/233963796":"2023-09-24T20:45:16.037413+0000","172.21.15.26:0/934875067":"2023-09-24T20:45:16.037413+0000","172.21.15.70:0/1035724255":"2023-09-24T20:44:32.835628+0000","172.21.15.26:0/1376232363":"2023-09-24T20:43:50.641728+0000","172.21.15.26:0/394447930":"2023-09-24T20:41:03.790648+0000","172.21.15.70:6835/1275773750":"2023-09-24T20:44:32.835628+0000","172.21.15.26:0/190503462":"2023-09-24T20:45:16.037413+0000","172.21.15.70:0/2198692933":"2023-09-24T20:44:32.835628+0000","172.21.15.26:6800/952511372":"2023-09-24T20:36:56.498559+0000","172.21.15.26:0/2633606443":"2023-09-24T20:45:16.037413+0000","172.21.15.26:0/237900280":"2023-09-24T20:43:50.641728+0000","172.21.15.26:0/3461760126":"2023-09-24T20:41:03.790648+0000","172.21.15.26:0/635571892":"2023-09-24T20:43:50.641728+0000","172.21.15.26:0/188049283":"2023-09-24T20:41:03.790648+0000","172.21.15.26:0/1815190465":"2023-09-24T20:36:31.159649+0000","172.21.15.26:0/686734212":"2023-09-24T20:36:31.159649+0000","172.21.15.26:6801/952511372":"2023-09-24T20:36:56.498559+0000","172.21.15.26:0/2838335646":"2023-09-24T20:36:56.498559+0000","172.21.15.26:6801/1446178988":"2023-09-24T20:45:16.037413+0000","172.21.15.26:6800/1446178988":"2023-09-24T20:45:16.037413+0000","172.21.15.26:6800/4099624907":"2023-09-24T20:36:31.159649+0000","172.21.15.26:6801/4099624907":"2023-09-24T20:36:31.159649+0000","172.21.15.26:0/3408257869":"2023-09-24T20:45:16.037413+0000","172.21.15.26:6800/91363814":"2023-09-24T20:41:03.790648+0000","172.21.15.26:0/2817523682":"2023-09-24T20:45:16.037413+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[{"pool":5,"snaps":[{"begin":2,"length":1}]},{"pool":12,"snaps":[{"begin":2,"length":1}]}],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2023-09-23T21:21:11.816 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph tell osd.0 flush_pg_stats 2023-09-23T21:21:11.817 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph tell osd.1 flush_pg_stats 2023-09-23T21:21:11.817 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph tell osd.2 flush_pg_stats 2023-09-23T21:21:11.818 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph tell osd.3 flush_pg_stats 2023-09-23T21:21:11.818 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph tell osd.4 flush_pg_stats 2023-09-23T21:21:11.819 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph tell osd.5 flush_pg_stats 2023-09-23T21:21:11.819 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph tell osd.6 flush_pg_stats 2023-09-23T21:21:11.819 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph tell osd.7 flush_pg_stats 2023-09-23T21:21:12.143 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:21:11 smithi026 ceph-mon[179688]: pgmap v1708: 103 pgs: 103 active+clean; 4.6 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail 2023-09-23T21:21:12.143 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:21:11 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/1850812381' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-09-23T21:21:12.153 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:21:11 smithi026 ceph-mon[175151]: pgmap v1708: 103 pgs: 103 active+clean; 4.6 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail 2023-09-23T21:21:12.154 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:21:11 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/1850812381' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-09-23T21:21:12.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:21:11 smithi070 ceph-mon[168282]: pgmap v1708: 103 pgs: 103 active+clean; 4.6 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail 2023-09-23T21:21:12.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:21:11 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/1850812381' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-09-23T21:21:12.756 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:21:12.907 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:21:12.908 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:21:13.099 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:21:13.251 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:21:13.251 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:21:13.253 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:21:13.669 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:21:13 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:21:13.669 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:21:13 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T21:21:13.669 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:21:13 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:21:13.670 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:21:13 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T21:21:13.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:21:13 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:21:13.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:21:13 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T21:21:14.331 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:21:14.774 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:21:14 smithi026 ceph-mon[179688]: pgmap v1709: 103 pgs: 103 active+clean; 4.6 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail 2023-09-23T21:21:14.775 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:21:14 smithi026 ceph-mon[175151]: pgmap v1709: 103 pgs: 103 active+clean; 4.6 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail 2023-09-23T21:21:14.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:21:14 smithi070 ceph-mon[168282]: pgmap v1709: 103 pgs: 103 active+clean; 4.6 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail 2023-09-23T21:21:16.783 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:21:16 smithi026 ceph-mon[179688]: pgmap v1710: 103 pgs: 103 active+clean; 4.6 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail 2023-09-23T21:21:16.783 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:21:16 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:21:16.784 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:21:16 smithi026 ceph-mon[175151]: pgmap v1710: 103 pgs: 103 active+clean; 4.6 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail 2023-09-23T21:21:16.784 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:21:16 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:21:16.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:21:16 smithi070 ceph-mon[168282]: pgmap v1710: 103 pgs: 103 active+clean; 4.6 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail 2023-09-23T21:21:16.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:21:16 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:21:17.496 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:21:17 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.d", "id": [1, 5]}]: dispatch 2023-09-23T21:21:17.497 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:21:17 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.12", "id": [1, 5]}]: dispatch 2023-09-23T21:21:17.497 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:21:17 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1d", "id": [1, 2]}]: dispatch 2023-09-23T21:21:17.755 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:21:17 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.d", "id": [1, 5]}]: dispatch 2023-09-23T21:21:17.755 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:21:17 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.12", "id": [1, 5]}]: dispatch 2023-09-23T21:21:17.755 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:21:17 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1d", "id": [1, 2]}]: dispatch 2023-09-23T21:21:17.864 INFO:teuthology.orchestra.run.smithi026.stdout:1503238553786 2023-09-23T21:21:17.865 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.1 2023-09-23T21:21:17.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:21:17 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.d", "id": [1, 5]}]: dispatch 2023-09-23T21:21:17.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:21:17 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.12", "id": [1, 5]}]: dispatch 2023-09-23T21:21:17.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:21:17 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1d", "id": [1, 2]}]: dispatch 2023-09-23T21:21:18.439 INFO:teuthology.orchestra.run.smithi026.stdout:420906795372 2023-09-23T21:21:18.439 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.3 2023-09-23T21:21:18.559 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:21:18 smithi026 ceph-mon[179688]: pgmap v1711: 103 pgs: 103 active+clean; 4.6 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail 2023-09-23T21:21:18.559 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:21:18 smithi026 ceph-mon[179688]: failed to encode map e708 with expected crc 2023-09-23T21:21:18.559 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:21:18 smithi026 ceph-mon[179688]: failed to encode map e708 with expected crc 2023-09-23T21:21:18.560 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:21:18 smithi026 ceph-mon[179688]: failed to encode map e708 with expected crc 2023-09-23T21:21:18.560 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:21:18 smithi026 ceph-mon[179688]: failed to encode map e708 with expected crc 2023-09-23T21:21:18.560 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:21:18 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.d", "id": [1, 5]}]': finished 2023-09-23T21:21:18.560 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:21:18 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.12", "id": [1, 5]}]': finished 2023-09-23T21:21:18.560 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:21:18 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1d", "id": [1, 2]}]': finished 2023-09-23T21:21:18.560 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:21:18 smithi026 ceph-mon[179688]: osdmap e708: 8 total, 8 up, 8 in 2023-09-23T21:21:18.560 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:21:18 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:21:18.560 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:21:18 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:21:18.560 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:21:18 smithi026 ceph-mon[175151]: pgmap v1711: 103 pgs: 103 active+clean; 4.6 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail 2023-09-23T21:21:18.560 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:21:18 smithi026 ceph-mon[175151]: failed to encode map e708 with expected crc 2023-09-23T21:21:18.561 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:21:18 smithi026 ceph-mon[175151]: failed to encode map e708 with expected crc 2023-09-23T21:21:18.561 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:21:18 smithi026 ceph-mon[175151]: failed to encode map e708 with expected crc 2023-09-23T21:21:18.561 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:21:18 smithi026 ceph-mon[175151]: failed to encode map e708 with expected crc 2023-09-23T21:21:18.561 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:21:18 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.d", "id": [1, 5]}]': finished 2023-09-23T21:21:18.561 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:21:18 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.12", "id": [1, 5]}]': finished 2023-09-23T21:21:18.561 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:21:18 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1d", "id": [1, 2]}]': finished 2023-09-23T21:21:18.561 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:21:18 smithi026 ceph-mon[175151]: osdmap e708: 8 total, 8 up, 8 in 2023-09-23T21:21:18.561 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:21:18 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:21:18.561 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:21:18 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:21:18.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:21:18 smithi070 ceph-mon[168282]: pgmap v1711: 103 pgs: 103 active+clean; 4.6 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail 2023-09-23T21:21:18.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:21:18 smithi070 ceph-mon[168282]: failed to encode map e708 with expected crc 2023-09-23T21:21:18.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:21:18 smithi070 ceph-mon[168282]: failed to encode map e708 with expected crc 2023-09-23T21:21:18.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:21:18 smithi070 ceph-mon[168282]: failed to encode map e708 with expected crc 2023-09-23T21:21:18.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:21:18 smithi070 ceph-mon[168282]: failed to encode map e708 with expected crc 2023-09-23T21:21:18.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:21:18 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.d", "id": [1, 5]}]': finished 2023-09-23T21:21:18.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:21:18 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.12", "id": [1, 5]}]': finished 2023-09-23T21:21:18.871 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:21:18 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1d", "id": [1, 2]}]': finished 2023-09-23T21:21:18.871 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:21:18 smithi070 ceph-mon[168282]: osdmap e708: 8 total, 8 up, 8 in 2023-09-23T21:21:18.871 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:21:18 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:21:18.871 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:21:18 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:21:19.847 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:21:19 smithi026 ceph-mon[175151]: failed to encode map e709 with expected crc 2023-09-23T21:21:19.847 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:21:19 smithi026 ceph-mon[175151]: failed to encode map e709 with expected crc 2023-09-23T21:21:19.847 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:21:19 smithi026 ceph-mon[175151]: failed to encode map e709 with expected crc 2023-09-23T21:21:19.847 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:21:19 smithi026 ceph-mon[175151]: osdmap e709: 8 total, 8 up, 8 in 2023-09-23T21:21:19.847 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:21:19 smithi026 ceph-mon[175151]: failed to encode map e709 with expected crc 2023-09-23T21:21:19.847 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:21:19 smithi026 ceph-mon[175151]: failed to encode map e709 with expected crc 2023-09-23T21:21:19.848 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:21:19 smithi026 ceph-mon[179688]: failed to encode map e709 with expected crc 2023-09-23T21:21:19.848 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:21:19 smithi026 ceph-mon[179688]: failed to encode map e709 with expected crc 2023-09-23T21:21:19.848 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:21:19 smithi026 ceph-mon[179688]: failed to encode map e709 with expected crc 2023-09-23T21:21:19.848 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:21:19 smithi026 ceph-mon[179688]: osdmap e709: 8 total, 8 up, 8 in 2023-09-23T21:21:19.848 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:21:19 smithi026 ceph-mon[179688]: failed to encode map e709 with expected crc 2023-09-23T21:21:19.848 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:21:19 smithi026 ceph-mon[179688]: failed to encode map e709 with expected crc 2023-09-23T21:21:19.848 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 21:21:19 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[164029]: ::ffff:172.21.15.70 - - [23/Sep/2023:21:21:19] "GET /metrics HTTP/1.1" 200 34001 "" "Prometheus/2.43.0" 2023-09-23T21:21:19.853 INFO:teuthology.orchestra.run.smithi026.stdout:236223201771 2023-09-23T21:21:19.853 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.7 2023-09-23T21:21:19.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:21:19 smithi070 ceph-mon[168282]: failed to encode map e709 with expected crc 2023-09-23T21:21:19.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:21:19 smithi070 ceph-mon[168282]: failed to encode map e709 with expected crc 2023-09-23T21:21:19.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:21:19 smithi070 ceph-mon[168282]: failed to encode map e709 with expected crc 2023-09-23T21:21:19.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:21:19 smithi070 ceph-mon[168282]: osdmap e709: 8 total, 8 up, 8 in 2023-09-23T21:21:19.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:21:19 smithi070 ceph-mon[168282]: failed to encode map e709 with expected crc 2023-09-23T21:21:19.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:21:19 smithi070 ceph-mon[168282]: failed to encode map e709 with expected crc 2023-09-23T21:21:20.267 INFO:teuthology.orchestra.run.smithi026.stdout:133143986679 2023-09-23T21:21:20.268 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.4 2023-09-23T21:21:20.368 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:21:20.372 INFO:teuthology.orchestra.run.smithi026.stdout:399431958899 2023-09-23T21:21:20.372 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.2 2023-09-23T21:21:20.761 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:21:20 smithi026 ceph-mon[175151]: pgmap v1714: 103 pgs: 103 active+clean; 4.6 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail 2023-09-23T21:21:20.761 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:21:20 smithi026 ceph-mon[179688]: pgmap v1714: 103 pgs: 103 active+clean; 4.6 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail 2023-09-23T21:21:20.869 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:21:20 smithi070 ceph-mon[168282]: pgmap v1714: 103 pgs: 103 active+clean; 4.6 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail 2023-09-23T21:21:21.144 INFO:teuthology.orchestra.run.smithi026.stdout:167503725045 2023-09-23T21:21:21.144 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.5 2023-09-23T21:21:21.349 INFO:teuthology.orchestra.run.smithi026.stdout:704374636855 2023-09-23T21:21:21.349 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.6 2023-09-23T21:21:21.785 INFO:teuthology.orchestra.run.smithi026.stdout:2117418877027 2023-09-23T21:21:21.786 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.0 2023-09-23T21:21:22.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:21:21 smithi026 ceph-mon[179688]: pgmap v1715: 103 pgs: 103 active+clean; 4.6 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail 2023-09-23T21:21:22.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:21:21 smithi026 ceph-mon[175151]: pgmap v1715: 103 pgs: 103 active+clean; 4.6 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail 2023-09-23T21:21:22.367 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:21:22.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:21:21 smithi070 ceph-mon[168282]: pgmap v1715: 103 pgs: 103 active+clean; 4.6 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail 2023-09-23T21:21:22.642 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:21:23.072 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:21:23.416 INFO:teuthology.orchestra.run.smithi026.stdout:1503238553787 2023-09-23T21:21:23.503 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:21:23.503 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:21:23.749 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:21:24.361 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:21:24.434 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:21:24 smithi026 ceph-mon[179688]: pgmap v1716: 103 pgs: 103 active+clean; 4.6 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail 2023-09-23T21:21:24.434 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:21:24 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/1922378673' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2023-09-23T21:21:24.435 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:21:24 smithi026 ceph-mon[175151]: pgmap v1716: 103 pgs: 103 active+clean; 4.6 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail 2023-09-23T21:21:24.435 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:21:24 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/1922378673' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2023-09-23T21:21:24.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:21:24 smithi070 ceph-mon[168282]: pgmap v1716: 103 pgs: 103 active+clean; 4.6 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail 2023-09-23T21:21:24.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:21:24 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/1922378673' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2023-09-23T21:21:25.997 INFO:tasks.cephadm.ceph_manager.ceph:need seq 1503238553786 got 1503238553787 for osd.1 2023-09-23T21:21:25.997 DEBUG:teuthology.parallel:result is None 2023-09-23T21:21:26.363 INFO:teuthology.orchestra.run.smithi026.stdout:399431958900 2023-09-23T21:21:26.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:21:26 smithi070 ceph-mon[168282]: pgmap v1717: 103 pgs: 103 active+clean; 4.6 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail 2023-09-23T21:21:26.625 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:21:26 smithi026 ceph-mon[179688]: pgmap v1717: 103 pgs: 103 active+clean; 4.6 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail 2023-09-23T21:21:26.625 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:21:26 smithi026 ceph-mon[175151]: pgmap v1717: 103 pgs: 103 active+clean; 4.6 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail 2023-09-23T21:21:27.187 INFO:teuthology.orchestra.run.smithi026.stdout:2117418877028 2023-09-23T21:21:27.192 INFO:teuthology.orchestra.run.smithi026.stdout:704374636856 2023-09-23T21:21:27.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:21:27 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/3699843890' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2023-09-23T21:21:27.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:21:27 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/1619325864' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2023-09-23T21:21:27.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:21:27 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:21:27.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:21:27 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:21:27.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:21:27 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/297858316' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2023-09-23T21:21:27.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:21:27 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/3699843890' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2023-09-23T21:21:27.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:21:27 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/1619325864' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2023-09-23T21:21:27.487 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:21:27 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:21:27.487 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:21:27 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:21:27.487 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:21:27 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/297858316' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2023-09-23T21:21:27.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:21:27 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/3699843890' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2023-09-23T21:21:27.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:21:27 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/1619325864' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2023-09-23T21:21:27.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:21:27 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:21:27.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:21:27 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:21:27.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:21:27 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/297858316' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2023-09-23T21:21:28.030 INFO:teuthology.orchestra.run.smithi026.stdout:167503725046 2023-09-23T21:21:28.340 INFO:tasks.cephadm.ceph_manager.ceph:need seq 399431958899 got 399431958900 for osd.2 2023-09-23T21:21:28.340 DEBUG:teuthology.parallel:result is None 2023-09-23T21:21:28.825 INFO:tasks.cephadm.ceph_manager.ceph:need seq 2117418877027 got 2117418877028 for osd.0 2023-09-23T21:21:28.825 DEBUG:teuthology.parallel:result is None 2023-09-23T21:21:28.857 INFO:tasks.cephadm.ceph_manager.ceph:need seq 704374636855 got 704374636856 for osd.6 2023-09-23T21:21:28.857 DEBUG:teuthology.parallel:result is None 2023-09-23T21:21:29.107 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:21:28 smithi026 ceph-mon[175151]: pgmap v1718: 103 pgs: 103 active+clean; 4.6 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail 2023-09-23T21:21:29.107 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:21:28 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/2593076484' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2023-09-23T21:21:29.107 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:21:28 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:21:29.108 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:21:28 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:21:29.108 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:21:28 smithi026 ceph-mon[179688]: pgmap v1718: 103 pgs: 103 active+clean; 4.6 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail 2023-09-23T21:21:29.108 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:21:28 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/2593076484' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2023-09-23T21:21:29.109 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:21:28 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:21:29.109 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:21:28 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:21:29.254 INFO:tasks.cephadm.ceph_manager.ceph:need seq 167503725045 got 167503725046 for osd.5 2023-09-23T21:21:29.254 DEBUG:teuthology.parallel:result is None 2023-09-23T21:21:29.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:21:28 smithi070 ceph-mon[168282]: pgmap v1718: 103 pgs: 103 active+clean; 4.6 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail 2023-09-23T21:21:29.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:21:28 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/2593076484' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2023-09-23T21:21:29.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:21:28 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:21:29.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:21:28 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:21:29.729 INFO:teuthology.orchestra.run.smithi026.stdout:133143986681 2023-09-23T21:21:29.881 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 21:21:29 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[164029]: ::ffff:172.21.15.70 - - [23/Sep/2023:21:21:29] "GET /metrics HTTP/1.1" 200 34001 "" "Prometheus/2.43.0" 2023-09-23T21:21:30.025 INFO:teuthology.orchestra.run.smithi026.stdout:236223201773 2023-09-23T21:21:30.110 INFO:teuthology.orchestra.run.smithi026.stdout:420906795374 2023-09-23T21:21:30.135 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:21:29 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/1371296036' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2023-09-23T21:21:30.135 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:21:29 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/1371296036' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2023-09-23T21:21:30.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:21:29 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/1371296036' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2023-09-23T21:21:30.515 INFO:tasks.cephadm.ceph_manager.ceph:need seq 133143986679 got 133143986681 for osd.4 2023-09-23T21:21:30.515 DEBUG:teuthology.parallel:result is None 2023-09-23T21:21:31.024 INFO:tasks.cephadm.ceph_manager.ceph:need seq 420906795372 got 420906795374 for osd.3 2023-09-23T21:21:31.024 DEBUG:teuthology.parallel:result is None 2023-09-23T21:21:31.250 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:21:30 smithi026 ceph-mon[175151]: pgmap v1719: 103 pgs: 103 active+clean; 4.6 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail 2023-09-23T21:21:31.251 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:21:30 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/3780665280' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2023-09-23T21:21:31.251 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:21:30 smithi026 ceph-mon[175151]: from='client.? 172.21.15.26:0/1103299016' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2023-09-23T21:21:31.251 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:21:30 smithi026 ceph-mon[179688]: pgmap v1719: 103 pgs: 103 active+clean; 4.6 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail 2023-09-23T21:21:31.251 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:21:30 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/3780665280' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2023-09-23T21:21:31.251 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:21:30 smithi026 ceph-mon[179688]: from='client.? 172.21.15.26:0/1103299016' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2023-09-23T21:21:31.336 INFO:tasks.cephadm.ceph_manager.ceph:need seq 236223201771 got 236223201773 for osd.7 2023-09-23T21:21:31.336 DEBUG:teuthology.parallel:result is None 2023-09-23T21:21:31.336 INFO:tasks.cephadm.ceph_manager.ceph:waiting for recovery to complete 2023-09-23T21:21:31.337 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T21:21:31.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:21:30 smithi070 ceph-mon[168282]: pgmap v1719: 103 pgs: 103 active+clean; 4.6 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail 2023-09-23T21:21:31.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:21:30 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/3780665280' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2023-09-23T21:21:31.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:21:30 smithi070 ceph-mon[168282]: from='client.? 172.21.15.26:0/1103299016' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2023-09-23T21:21:31.956 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:21:32.241 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:21:31 smithi026 ceph-mon[175151]: pgmap v1720: 103 pgs: 103 active+clean; 4.6 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail 2023-09-23T21:21:32.241 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:21:31 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:21:32.242 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:21:31 smithi026 ceph-mon[179688]: pgmap v1720: 103 pgs: 103 active+clean; 4.6 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail 2023-09-23T21:21:32.242 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:21:31 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:21:32.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:21:31 smithi070 ceph-mon[168282]: pgmap v1720: 103 pgs: 103 active+clean; 4.6 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail 2023-09-23T21:21:32.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:21:31 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:21:33.860 INFO:teuthology.orchestra.run.smithi026.stdout: 2023-09-23T21:21:33.862 INFO:teuthology.orchestra.run.smithi026.stderr:dumped all 2023-09-23T21:21:34.238 INFO:teuthology.orchestra.run.smithi026.stdout:{"pg_ready":true,"pg_map":{"version":1721,"stamp":"2023-09-23T21:21:33.244519+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4790195,"num_objects":28,"num_object_clones":0,"num_object_copies":84,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":28,"num_whiteouts":0,"num_read":5995,"num_read_kb":13212,"num_write":907,"num_write_kb":12725,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":162,"num_bytes_recovered":9266624,"num_keys_recovered":20,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6022,"ondisk_log_size":6022,"up":309,"acting":309,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":309,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":5144844,"kb_used_data":77388,"kb_used_omap":145,"kb_used_meta":5067246,"kb_avail":744816372,"statfs":{"total":767960285184,"available":762691964928,"internally_reserved":0,"allocated":79245312,"data_stored":59741207,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":149488,"internal_metadata":5188859920},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.003101"},"pg_stats":[{"pgid":"1.18","version":"79'161","reported_seq":775,"reported_epoch":708,"state":"active+clean","last_fresh":"2023-09-23T21:21:17.507478+0000","last_change":"2023-09-23T21:20:19.087341+0000","last_active":"2023-09-23T21:21:17.507478+0000","last_peered":"2023-09-23T21:21:17.507478+0000","last_clean":"2023-09-23T21:21:17.507478+0000","last_became_active":"2023-09-23T21:20:19.086834+0000","last_became_peered":"2023-09-23T21:20:19.086834+0000","last_unstale":"2023-09-23T21:21:17.507478+0000","last_undegraded":"2023-09-23T21:21:17.507478+0000","last_fullsized":"2023-09-23T21:21:17.507478+0000","mapping_epoch":684,"log_start":"0'0","ondisk_log_start":"0'0","created":479,"last_epoch_clean":685,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:19:34.961756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:19:34.961756+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:04:09.454020+0000","scrub_duration":0.002022102,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":525376,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1a","version":"68'1","reported_seq":309,"reported_epoch":708,"state":"active+clean","last_fresh":"2023-09-23T21:21:17.505503+0000","last_change":"2023-09-23T21:20:12.080378+0000","last_active":"2023-09-23T21:21:17.505503+0000","last_peered":"2023-09-23T21:21:17.505503+0000","last_clean":"2023-09-23T21:21:17.505503+0000","last_became_active":"2023-09-23T21:20:12.080067+0000","last_became_peered":"2023-09-23T21:20:12.080067+0000","last_unstale":"2023-09-23T21:21:17.505503+0000","last_undegraded":"2023-09-23T21:21:17.505503+0000","last_fullsized":"2023-09-23T21:21:17.505503+0000","mapping_epoch":677,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":678,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:41:56.535018+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":467,"reported_epoch":709,"state":"active+clean","last_fresh":"2023-09-23T21:21:18.513696+0000","last_change":"2023-09-23T21:21:18.513696+0000","last_active":"2023-09-23T21:21:18.513696+0000","last_peered":"2023-09-23T21:21:18.513696+0000","last_clean":"2023-09-23T21:21:18.513696+0000","last_became_active":"2023-09-23T21:21:18.513311+0000","last_became_peered":"2023-09-23T21:21:18.513311+0000","last_unstale":"2023-09-23T21:21:18.513696+0000","last_undegraded":"2023-09-23T21:21:18.513696+0000","last_fullsized":"2023-09-23T21:21:18.513696+0000","mapping_epoch":708,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":709,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:50:03.921453+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1b","version":"79'161","reported_seq":590,"reported_epoch":708,"state":"active+clean","last_fresh":"2023-09-23T21:21:17.506840+0000","last_change":"2023-09-23T21:20:23.452618+0000","last_active":"2023-09-23T21:21:17.506840+0000","last_peered":"2023-09-23T21:21:17.506840+0000","last_clean":"2023-09-23T21:21:17.506840+0000","last_became_active":"2023-09-23T21:20:19.092143+0000","last_became_peered":"2023-09-23T21:20:19.092143+0000","last_unstale":"2023-09-23T21:21:17.506840+0000","last_undegraded":"2023-09-23T21:21:17.506840+0000","last_fullsized":"2023-09-23T21:21:17.506840+0000","mapping_epoch":684,"log_start":"0'0","ondisk_log_start":"0'0","created":479,"last_epoch_clean":685,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:20:23.452497+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:20:23.452497+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:36:41.779187+0000","scrub_duration":0.0025756709999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":42,"reported_epoch":708,"state":"active+clean","last_fresh":"2023-09-23T21:21:17.502526+0000","last_change":"2023-09-23T21:20:12.115671+0000","last_active":"2023-09-23T21:21:17.502526+0000","last_peered":"2023-09-23T21:21:17.502526+0000","last_clean":"2023-09-23T21:21:17.502526+0000","last_became_active":"2023-09-23T21:20:12.115349+0000","last_became_peered":"2023-09-23T21:20:12.115349+0000","last_unstale":"2023-09-23T21:21:17.502526+0000","last_undegraded":"2023-09-23T21:21:17.502526+0000","last_fullsized":"2023-09-23T21:21:17.502526+0000","mapping_epoch":677,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":678,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:01:43.499564+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":727,"reported_epoch":708,"state":"active+clean","last_fresh":"2023-09-23T21:21:17.902200+0000","last_change":"2023-09-23T21:16:42.441896+0000","last_active":"2023-09-23T21:21:17.902200+0000","last_peered":"2023-09-23T21:21:17.902200+0000","last_clean":"2023-09-23T21:21:17.902200+0000","last_became_active":"2023-09-23T21:16:42.441526+0000","last_became_peered":"2023-09-23T21:16:42.441526+0000","last_unstale":"2023-09-23T21:21:17.902200+0000","last_undegraded":"2023-09-23T21:21:17.902200+0000","last_fullsized":"2023-09-23T21:21:17.902200+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:27:36.326225+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1a","version":"79'161","reported_seq":846,"reported_epoch":708,"state":"active+clean","last_fresh":"2023-09-23T21:21:17.505337+0000","last_change":"2023-09-23T21:20:15.080123+0000","last_active":"2023-09-23T21:21:17.505337+0000","last_peered":"2023-09-23T21:21:17.505337+0000","last_clean":"2023-09-23T21:21:17.505337+0000","last_became_active":"2023-09-23T21:20:15.079745+0000","last_became_peered":"2023-09-23T21:20:15.079745+0000","last_unstale":"2023-09-23T21:21:17.505337+0000","last_undegraded":"2023-09-23T21:21:17.505337+0000","last_fullsized":"2023-09-23T21:21:17.505337+0000","mapping_epoch":680,"log_start":"0'0","ondisk_log_start":"0'0","created":479,"last_epoch_clean":681,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:15:45.158107+0000","last_deep_scrub":"79'161","last_deep_scrub_stamp":"2023-09-23T21:05:32.630292+0000","last_clean_scrub_stamp":"2023-09-23T21:15:45.158107+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:19:56.996183+0000","scrub_duration":0.001999373,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":465,"reported_epoch":708,"state":"active+clean","last_fresh":"2023-09-23T21:21:17.505400+0000","last_change":"2023-09-23T21:20:30.312213+0000","last_active":"2023-09-23T21:21:17.505400+0000","last_peered":"2023-09-23T21:21:17.505400+0000","last_clean":"2023-09-23T21:21:17.505400+0000","last_became_active":"2023-09-23T21:20:30.311897+0000","last_became_peered":"2023-09-23T21:20:30.311897+0000","last_unstale":"2023-09-23T21:21:17.505400+0000","last_undegraded":"2023-09-23T21:21:17.505400+0000","last_fullsized":"2023-09-23T21:21:17.505400+0000","mapping_epoch":689,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":690,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:25:54.138000+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":66,"reported_epoch":708,"state":"active+clean","last_fresh":"2023-09-23T21:21:17.508706+0000","last_change":"2023-09-23T21:20:15.934532+0000","last_active":"2023-09-23T21:21:17.508706+0000","last_peered":"2023-09-23T21:21:17.508706+0000","last_clean":"2023-09-23T21:21:17.508706+0000","last_became_active":"2023-09-23T21:20:15.934179+0000","last_became_peered":"2023-09-23T21:20:15.934179+0000","last_unstale":"2023-09-23T21:21:17.508706+0000","last_undegraded":"2023-09-23T21:21:17.508706+0000","last_fullsized":"2023-09-23T21:21:17.508706+0000","mapping_epoch":681,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":682,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:52:50.286017+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.15","version":"79'161","reported_seq":719,"reported_epoch":708,"state":"active+clean","last_fresh":"2023-09-23T21:21:17.506412+0000","last_change":"2023-09-23T21:20:19.089275+0000","last_active":"2023-09-23T21:21:17.506412+0000","last_peered":"2023-09-23T21:21:17.506412+0000","last_clean":"2023-09-23T21:21:17.506412+0000","last_became_active":"2023-09-23T21:20:19.088773+0000","last_became_peered":"2023-09-23T21:20:19.088773+0000","last_unstale":"2023-09-23T21:21:17.506412+0000","last_undegraded":"2023-09-23T21:21:17.506412+0000","last_fullsized":"2023-09-23T21:21:17.506412+0000","mapping_epoch":684,"log_start":"0'0","ondisk_log_start":"0'0","created":479,"last_epoch_clean":685,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:18:35.622422+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:18:35.622422+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:55:40.821855+0000","scrub_duration":0.083031242000000005,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,1],"acting":[7,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":713,"reported_epoch":708,"state":"active+clean","last_fresh":"2023-09-23T21:21:17.901931+0000","last_change":"2023-09-23T21:20:15.934405+0000","last_active":"2023-09-23T21:21:17.901931+0000","last_peered":"2023-09-23T21:21:17.901931+0000","last_clean":"2023-09-23T21:21:17.901931+0000","last_became_active":"2023-09-23T21:20:15.934102+0000","last_became_peered":"2023-09-23T21:20:15.934102+0000","last_unstale":"2023-09-23T21:21:17.901931+0000","last_undegraded":"2023-09-23T21:21:17.901931+0000","last_fullsized":"2023-09-23T21:21:17.901931+0000","mapping_epoch":681,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":682,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:11:01.173661+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":777,"reported_epoch":709,"state":"active+clean","last_fresh":"2023-09-23T21:21:18.506184+0000","last_change":"2023-09-23T21:20:50.927053+0000","last_active":"2023-09-23T21:21:18.506184+0000","last_peered":"2023-09-23T21:21:18.506184+0000","last_clean":"2023-09-23T21:21:18.506184+0000","last_became_active":"2023-09-23T21:20:50.926718+0000","last_became_peered":"2023-09-23T21:20:50.926718+0000","last_unstale":"2023-09-23T21:21:18.506184+0000","last_undegraded":"2023-09-23T21:21:18.506184+0000","last_fullsized":"2023-09-23T21:21:18.506184+0000","mapping_epoch":699,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":700,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:06:15.619866+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"79'161","reported_seq":1107,"reported_epoch":708,"state":"active+clean","last_fresh":"2023-09-23T21:21:17.508505+0000","last_change":"2023-09-23T21:16:45.424773+0000","last_active":"2023-09-23T21:21:17.508505+0000","last_peered":"2023-09-23T21:21:17.508505+0000","last_clean":"2023-09-23T21:21:17.508505+0000","last_became_active":"2023-09-23T21:16:45.424458+0000","last_became_peered":"2023-09-23T21:16:45.424458+0000","last_unstale":"2023-09-23T21:21:17.508505+0000","last_undegraded":"2023-09-23T21:21:17.508505+0000","last_fullsized":"2023-09-23T21:21:17.508505+0000","mapping_epoch":567,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":568,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:25.096401+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:25.096401+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:40:17.871561+0000","scrub_duration":0.002859393,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":37,"reported_epoch":708,"state":"active+clean","last_fresh":"2023-09-23T21:21:17.507984+0000","last_change":"2023-09-23T21:20:15.933213+0000","last_active":"2023-09-23T21:21:17.507984+0000","last_peered":"2023-09-23T21:21:17.507984+0000","last_clean":"2023-09-23T21:21:17.507984+0000","last_became_active":"2023-09-23T21:20:15.932952+0000","last_became_peered":"2023-09-23T21:20:15.932952+0000","last_unstale":"2023-09-23T21:21:17.507984+0000","last_undegraded":"2023-09-23T21:21:17.507984+0000","last_fullsized":"2023-09-23T21:21:17.507984+0000","mapping_epoch":681,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":682,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:33:08.774578+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":723,"reported_epoch":709,"state":"active+clean","last_fresh":"2023-09-23T21:21:18.506434+0000","last_change":"2023-09-23T21:20:05.021571+0000","last_active":"2023-09-23T21:21:18.506434+0000","last_peered":"2023-09-23T21:21:18.506434+0000","last_clean":"2023-09-23T21:21:18.506434+0000","last_became_active":"2023-09-23T21:20:05.021192+0000","last_became_peered":"2023-09-23T21:20:05.021192+0000","last_unstale":"2023-09-23T21:21:18.506434+0000","last_undegraded":"2023-09-23T21:21:18.506434+0000","last_fullsized":"2023-09-23T21:21:18.506434+0000","mapping_epoch":672,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":673,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:30:55.557751+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.17","version":"79'161","reported_seq":893,"reported_epoch":708,"state":"active+clean","last_fresh":"2023-09-23T21:21:17.508181+0000","last_change":"2023-09-23T21:20:42.355872+0000","last_active":"2023-09-23T21:21:17.508181+0000","last_peered":"2023-09-23T21:21:17.508181+0000","last_clean":"2023-09-23T21:21:17.508181+0000","last_became_active":"2023-09-23T21:20:42.355550+0000","last_became_peered":"2023-09-23T21:20:42.355550+0000","last_unstale":"2023-09-23T21:21:17.508181+0000","last_undegraded":"2023-09-23T21:21:17.508181+0000","last_fullsized":"2023-09-23T21:21:17.508181+0000","mapping_epoch":694,"log_start":"0'0","ondisk_log_start":"0'0","created":479,"last_epoch_clean":695,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:15:50.325741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:15:50.325741+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:13:11.438795+0000","scrub_duration":0.0040500149999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,6],"acting":[5,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":489,"reported_epoch":708,"state":"active+clean","last_fresh":"2023-09-23T21:21:17.506132+0000","last_change":"2023-09-23T21:20:48.511983+0000","last_active":"2023-09-23T21:21:17.506132+0000","last_peered":"2023-09-23T21:21:17.506132+0000","last_clean":"2023-09-23T21:21:17.506132+0000","last_became_active":"2023-09-23T21:20:48.511621+0000","last_became_peered":"2023-09-23T21:20:48.511621+0000","last_unstale":"2023-09-23T21:21:17.506132+0000","last_undegraded":"2023-09-23T21:21:17.506132+0000","last_fullsized":"2023-09-23T21:21:17.506132+0000","mapping_epoch":697,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":698,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:46:22.322452+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":450,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":10,"reported_epoch":709,"state":"active+clean","last_fresh":"2023-09-23T21:21:18.514367+0000","last_change":"2023-09-23T21:21:18.514367+0000","last_active":"2023-09-23T21:21:18.514367+0000","last_peered":"2023-09-23T21:21:18.514367+0000","last_clean":"2023-09-23T21:21:18.514367+0000","last_became_active":"2023-09-23T21:21:18.514026+0000","last_became_peered":"2023-09-23T21:21:18.514026+0000","last_unstale":"2023-09-23T21:21:18.514367+0000","last_undegraded":"2023-09-23T21:21:18.514367+0000","last_fullsized":"2023-09-23T21:21:18.514367+0000","mapping_epoch":708,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":709,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:27:27.146957+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,2],"acting":[5,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.16","version":"79'161","reported_seq":650,"reported_epoch":708,"state":"active+clean","last_fresh":"2023-09-23T21:21:17.504146+0000","last_change":"2023-09-23T21:20:15.083112+0000","last_active":"2023-09-23T21:21:17.504146+0000","last_peered":"2023-09-23T21:21:17.504146+0000","last_clean":"2023-09-23T21:21:17.504146+0000","last_became_active":"2023-09-23T21:20:15.082692+0000","last_became_peered":"2023-09-23T21:20:15.082692+0000","last_unstale":"2023-09-23T21:21:17.504146+0000","last_undegraded":"2023-09-23T21:21:17.504146+0000","last_fullsized":"2023-09-23T21:21:17.504146+0000","mapping_epoch":680,"log_start":"0'0","ondisk_log_start":"0'0","created":479,"last_epoch_clean":681,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:15:46.326589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:15:46.326589+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:15:50.608532+0000","scrub_duration":0.0062069059999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"79'22","reported_seq":332,"reported_epoch":708,"state":"active+clean","last_fresh":"2023-09-23T21:21:17.505778+0000","last_change":"2023-09-23T21:20:05.024837+0000","last_active":"2023-09-23T21:21:17.505778+0000","last_peered":"2023-09-23T21:21:17.505778+0000","last_clean":"2023-09-23T21:21:17.505778+0000","last_became_active":"2023-09-23T21:20:05.018268+0000","last_became_peered":"2023-09-23T21:20:05.018268+0000","last_unstale":"2023-09-23T21:21:17.505778+0000","last_undegraded":"2023-09-23T21:21:17.505778+0000","last_fullsized":"2023-09-23T21:21:17.505778+0000","mapping_epoch":672,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":673,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:16:08.334109+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":18936,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":667,"reported_epoch":708,"state":"active+clean","last_fresh":"2023-09-23T21:21:17.505728+0000","last_change":"2023-09-23T21:20:12.078453+0000","last_active":"2023-09-23T21:21:17.505728+0000","last_peered":"2023-09-23T21:21:17.505728+0000","last_clean":"2023-09-23T21:21:17.505728+0000","last_became_active":"2023-09-23T21:20:12.078115+0000","last_became_peered":"2023-09-23T21:20:12.078115+0000","last_unstale":"2023-09-23T21:21:17.505728+0000","last_undegraded":"2023-09-23T21:21:17.505728+0000","last_fullsized":"2023-09-23T21:21:17.505728+0000","mapping_epoch":677,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":678,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:04:11.282079+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.11","version":"79'161","reported_seq":690,"reported_epoch":708,"state":"active+clean","last_fresh":"2023-09-23T21:21:17.901445+0000","last_change":"2023-09-23T21:20:19.087016+0000","last_active":"2023-09-23T21:21:17.901445+0000","last_peered":"2023-09-23T21:21:17.901445+0000","last_clean":"2023-09-23T21:21:17.901445+0000","last_became_active":"2023-09-23T21:20:19.086608+0000","last_became_peered":"2023-09-23T21:20:19.086608+0000","last_unstale":"2023-09-23T21:21:17.901445+0000","last_undegraded":"2023-09-23T21:21:17.901445+0000","last_fullsized":"2023-09-23T21:21:17.901445+0000","mapping_epoch":684,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":685,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:15:47.299947+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:15:47.299947+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:21:19.868572+0000","scrub_duration":0.0061353110000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":160,"reported_epoch":708,"state":"active+clean","last_fresh":"2023-09-23T21:21:17.505822+0000","last_change":"2023-09-23T21:16:42.531060+0000","last_active":"2023-09-23T21:21:17.505822+0000","last_peered":"2023-09-23T21:21:17.505822+0000","last_clean":"2023-09-23T21:21:17.505822+0000","last_became_active":"2023-09-23T21:16:42.449198+0000","last_became_peered":"2023-09-23T21:16:42.449198+0000","last_unstale":"2023-09-23T21:21:17.505822+0000","last_undegraded":"2023-09-23T21:21:17.505822+0000","last_fullsized":"2023-09-23T21:21:17.505822+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:30:23.428886+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":465,"reported_epoch":708,"state":"active+clean","last_fresh":"2023-09-23T21:21:17.506037+0000","last_change":"2023-09-23T21:20:30.314319+0000","last_active":"2023-09-23T21:21:17.506037+0000","last_peered":"2023-09-23T21:21:17.506037+0000","last_clean":"2023-09-23T21:21:17.506037+0000","last_became_active":"2023-09-23T21:20:30.313943+0000","last_became_peered":"2023-09-23T21:20:30.313943+0000","last_unstale":"2023-09-23T21:21:17.506037+0000","last_undegraded":"2023-09-23T21:21:17.506037+0000","last_fullsized":"2023-09-23T21:21:17.506037+0000","mapping_epoch":689,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":690,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:24:51.385745+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.10","version":"79'161","reported_seq":927,"reported_epoch":708,"state":"active+clean","last_fresh":"2023-09-23T21:21:17.507398+0000","last_change":"2023-09-23T21:20:53.939329+0000","last_active":"2023-09-23T21:21:17.507398+0000","last_peered":"2023-09-23T21:21:17.507398+0000","last_clean":"2023-09-23T21:21:17.507398+0000","last_became_active":"2023-09-23T21:20:53.938896+0000","last_became_peered":"2023-09-23T21:20:53.938896+0000","last_unstale":"2023-09-23T21:21:17.507398+0000","last_undegraded":"2023-09-23T21:21:17.507398+0000","last_fullsized":"2023-09-23T21:21:17.507398+0000","mapping_epoch":702,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":703,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:06:50.336992+0000","last_deep_scrub":"79'161","last_deep_scrub_stamp":"2023-09-23T21:06:50.336992+0000","last_clean_scrub_stamp":"2023-09-23T21:06:50.336992+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:11:03.277383+0000","scrub_duration":0.0026150930000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":670,"reported_epoch":708,"state":"active+clean","last_fresh":"2023-09-23T21:21:17.901763+0000","last_change":"2023-09-23T21:16:42.446324+0000","last_active":"2023-09-23T21:21:17.901763+0000","last_peered":"2023-09-23T21:21:17.901763+0000","last_clean":"2023-09-23T21:21:17.901763+0000","last_became_active":"2023-09-23T21:16:42.445997+0000","last_became_peered":"2023-09-23T21:16:42.445997+0000","last_unstale":"2023-09-23T21:21:17.901763+0000","last_undegraded":"2023-09-23T21:21:17.901763+0000","last_fullsized":"2023-09-23T21:21:17.901763+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:46:35.028292+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":499,"reported_epoch":708,"state":"active+clean","last_fresh":"2023-09-23T21:21:17.508135+0000","last_change":"2023-09-23T21:20:42.355719+0000","last_active":"2023-09-23T21:21:17.508135+0000","last_peered":"2023-09-23T21:21:17.508135+0000","last_clean":"2023-09-23T21:21:17.508135+0000","last_became_active":"2023-09-23T21:20:42.354541+0000","last_became_peered":"2023-09-23T21:20:42.354541+0000","last_unstale":"2023-09-23T21:21:17.508135+0000","last_undegraded":"2023-09-23T21:21:17.508135+0000","last_fullsized":"2023-09-23T21:21:17.508135+0000","mapping_epoch":694,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":695,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:29:24.648781+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.13","version":"79'161","reported_seq":716,"reported_epoch":708,"state":"active+clean","last_fresh":"2023-09-23T21:21:17.502856+0000","last_change":"2023-09-23T21:20:15.081573+0000","last_active":"2023-09-23T21:21:17.502856+0000","last_peered":"2023-09-23T21:21:17.502856+0000","last_clean":"2023-09-23T21:21:17.502856+0000","last_became_active":"2023-09-23T21:20:15.080863+0000","last_became_peered":"2023-09-23T21:20:15.080863+0000","last_unstale":"2023-09-23T21:21:17.502856+0000","last_undegraded":"2023-09-23T21:21:17.502856+0000","last_fullsized":"2023-09-23T21:21:17.502856+0000","mapping_epoch":680,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":681,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:36.420362+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:36.420362+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:34:39.845514+0000","scrub_duration":0.028838444000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":709,"reported_epoch":708,"state":"active+clean","last_fresh":"2023-09-23T21:21:17.506514+0000","last_change":"2023-09-23T21:20:48.507356+0000","last_active":"2023-09-23T21:21:17.506514+0000","last_peered":"2023-09-23T21:21:17.506514+0000","last_clean":"2023-09-23T21:21:17.506514+0000","last_became_active":"2023-09-23T21:20:48.507015+0000","last_became_peered":"2023-09-23T21:20:48.507015+0000","last_unstale":"2023-09-23T21:21:17.506514+0000","last_undegraded":"2023-09-23T21:21:17.506514+0000","last_fullsized":"2023-09-23T21:21:17.506514+0000","mapping_epoch":697,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":698,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:47:25.662099+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":239,"reported_epoch":708,"state":"active+clean","last_fresh":"2023-09-23T21:21:17.901911+0000","last_change":"2023-09-23T21:16:42.427663+0000","last_active":"2023-09-23T21:21:17.901911+0000","last_peered":"2023-09-23T21:21:17.901911+0000","last_clean":"2023-09-23T21:21:17.901911+0000","last_became_active":"2023-09-23T21:16:42.426961+0000","last_became_peered":"2023-09-23T21:16:42.426961+0000","last_unstale":"2023-09-23T21:21:17.901911+0000","last_undegraded":"2023-09-23T21:21:17.901911+0000","last_fullsized":"2023-09-23T21:21:17.901911+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:44:27.422007+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.12","version":"79'161","reported_seq":769,"reported_epoch":708,"state":"active+clean","last_fresh":"2023-09-23T21:21:17.507483+0000","last_change":"2023-09-23T21:20:19.090120+0000","last_active":"2023-09-23T21:21:17.507483+0000","last_peered":"2023-09-23T21:21:17.507483+0000","last_clean":"2023-09-23T21:21:17.507483+0000","last_became_active":"2023-09-23T21:20:19.089743+0000","last_became_peered":"2023-09-23T21:20:19.089743+0000","last_unstale":"2023-09-23T21:21:17.507483+0000","last_undegraded":"2023-09-23T21:21:17.507483+0000","last_fullsized":"2023-09-23T21:21:17.507483+0000","mapping_epoch":684,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":685,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:05:56.785173+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:05:56.785173+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:47:18.107527+0000","scrub_duration":0.0042114370000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":76,"reported_epoch":708,"state":"active+clean","last_fresh":"2023-09-23T21:21:17.508531+0000","last_change":"2023-09-23T21:20:15.939391+0000","last_active":"2023-09-23T21:21:17.508531+0000","last_peered":"2023-09-23T21:21:17.508531+0000","last_clean":"2023-09-23T21:21:17.508531+0000","last_became_active":"2023-09-23T21:20:15.928437+0000","last_became_peered":"2023-09-23T21:20:15.928437+0000","last_unstale":"2023-09-23T21:21:17.508531+0000","last_undegraded":"2023-09-23T21:21:17.508531+0000","last_fullsized":"2023-09-23T21:21:17.508531+0000","mapping_epoch":681,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":682,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:51:06.828186+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":575,"reported_epoch":708,"state":"active+clean","last_fresh":"2023-09-23T21:21:17.507034+0000","last_change":"2023-09-23T21:20:12.105028+0000","last_active":"2023-09-23T21:21:17.507034+0000","last_peered":"2023-09-23T21:21:17.507034+0000","last_clean":"2023-09-23T21:21:17.507034+0000","last_became_active":"2023-09-23T21:20:12.104725+0000","last_became_peered":"2023-09-23T21:20:12.104725+0000","last_unstale":"2023-09-23T21:21:17.507034+0000","last_undegraded":"2023-09-23T21:21:17.507034+0000","last_fullsized":"2023-09-23T21:21:17.507034+0000","mapping_epoch":677,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":678,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:01:01.926290+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.d","version":"79'161","reported_seq":770,"reported_epoch":708,"state":"active+clean","last_fresh":"2023-09-23T21:21:17.506492+0000","last_change":"2023-09-23T21:20:15.080132+0000","last_active":"2023-09-23T21:21:17.506492+0000","last_peered":"2023-09-23T21:21:17.506492+0000","last_clean":"2023-09-23T21:21:17.506492+0000","last_became_active":"2023-09-23T21:20:15.079810+0000","last_became_peered":"2023-09-23T21:20:15.079810+0000","last_unstale":"2023-09-23T21:21:17.506492+0000","last_undegraded":"2023-09-23T21:21:17.506492+0000","last_fullsized":"2023-09-23T21:21:17.506492+0000","mapping_epoch":680,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":681,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:18:47.440263+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:18:47.440263+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:36:22.438645+0000","scrub_duration":0.0023849959999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":158,"reported_epoch":708,"state":"active+clean","last_fresh":"2023-09-23T21:21:17.506054+0000","last_change":"2023-09-23T21:17:02.190320+0000","last_active":"2023-09-23T21:21:17.506054+0000","last_peered":"2023-09-23T21:21:17.506054+0000","last_clean":"2023-09-23T21:21:17.506054+0000","last_became_active":"2023-09-23T21:16:42.451380+0000","last_became_peered":"2023-09-23T21:16:42.451380+0000","last_unstale":"2023-09-23T21:21:17.506054+0000","last_undegraded":"2023-09-23T21:21:17.506054+0000","last_fullsized":"2023-09-23T21:21:17.506054+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:48:58.425821+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":132,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":47,"reported_epoch":708,"state":"active+clean","last_fresh":"2023-09-23T21:21:17.507655+0000","last_change":"2023-09-23T21:20:50.928140+0000","last_active":"2023-09-23T21:21:17.507655+0000","last_peered":"2023-09-23T21:21:17.507655+0000","last_clean":"2023-09-23T21:21:17.507655+0000","last_became_active":"2023-09-23T21:20:50.927735+0000","last_became_peered":"2023-09-23T21:20:50.927735+0000","last_unstale":"2023-09-23T21:21:17.507655+0000","last_undegraded":"2023-09-23T21:21:17.507655+0000","last_fullsized":"2023-09-23T21:21:17.507655+0000","mapping_epoch":699,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":700,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:29:04.097873+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.c","version":"79'161","reported_seq":1113,"reported_epoch":708,"state":"active+clean","last_fresh":"2023-09-23T21:21:17.505871+0000","last_change":"2023-09-23T21:20:08.011328+0000","last_active":"2023-09-23T21:21:17.505871+0000","last_peered":"2023-09-23T21:21:17.505871+0000","last_clean":"2023-09-23T21:21:17.505871+0000","last_became_active":"2023-09-23T21:20:08.010905+0000","last_became_peered":"2023-09-23T21:20:08.010905+0000","last_unstale":"2023-09-23T21:21:17.505871+0000","last_undegraded":"2023-09-23T21:21:17.505871+0000","last_fullsized":"2023-09-23T21:21:17.505871+0000","mapping_epoch":675,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":676,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:15:31.992598+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:15:31.992598+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:22:35.797510+0000","scrub_duration":0.002960678,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":327680,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":172,"reported_epoch":708,"state":"active+clean","last_fresh":"2023-09-23T21:21:17.506109+0000","last_change":"2023-09-23T21:20:12.104469+0000","last_active":"2023-09-23T21:21:17.506109+0000","last_peered":"2023-09-23T21:21:17.506109+0000","last_clean":"2023-09-23T21:21:17.506109+0000","last_became_active":"2023-09-23T21:20:12.086524+0000","last_became_peered":"2023-09-23T21:20:12.086524+0000","last_unstale":"2023-09-23T21:21:17.506109+0000","last_undegraded":"2023-09-23T21:21:17.506109+0000","last_fullsized":"2023-09-23T21:21:17.506109+0000","mapping_epoch":677,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":678,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:05:30.178772+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":681,"reported_epoch":708,"state":"active+clean","last_fresh":"2023-09-23T21:21:17.505453+0000","last_change":"2023-09-23T21:20:12.078362+0000","last_active":"2023-09-23T21:21:17.505453+0000","last_peered":"2023-09-23T21:21:17.505453+0000","last_clean":"2023-09-23T21:21:17.505453+0000","last_became_active":"2023-09-23T21:20:12.077958+0000","last_became_peered":"2023-09-23T21:20:12.077958+0000","last_unstale":"2023-09-23T21:21:17.505453+0000","last_undegraded":"2023-09-23T21:21:17.505453+0000","last_fullsized":"2023-09-23T21:21:17.505453+0000","mapping_epoch":677,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":678,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:10:04.783325+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.f","version":"79'161","reported_seq":800,"reported_epoch":708,"state":"active+clean","last_fresh":"2023-09-23T21:21:17.509361+0000","last_change":"2023-09-23T21:20:19.094207+0000","last_active":"2023-09-23T21:21:17.509361+0000","last_peered":"2023-09-23T21:21:17.509361+0000","last_clean":"2023-09-23T21:21:17.509361+0000","last_became_active":"2023-09-23T21:20:19.093826+0000","last_became_peered":"2023-09-23T21:20:19.093826+0000","last_unstale":"2023-09-23T21:21:17.509361+0000","last_undegraded":"2023-09-23T21:21:17.509361+0000","last_fullsized":"2023-09-23T21:21:17.509361+0000","mapping_epoch":684,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":685,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:27.880776+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:27.880776+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:31:51.006419+0000","scrub_duration":0.0014004899999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":327,"reported_epoch":708,"state":"active+clean","last_fresh":"2023-09-23T21:21:17.506511+0000","last_change":"2023-09-23T21:20:48.509999+0000","last_active":"2023-09-23T21:21:17.506511+0000","last_peered":"2023-09-23T21:21:17.506511+0000","last_clean":"2023-09-23T21:21:17.506511+0000","last_became_active":"2023-09-23T21:20:48.509613+0000","last_became_peered":"2023-09-23T21:20:48.509613+0000","last_unstale":"2023-09-23T21:21:17.506511+0000","last_undegraded":"2023-09-23T21:21:17.506511+0000","last_fullsized":"2023-09-23T21:21:17.506511+0000","mapping_epoch":697,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":698,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:47:38.322761+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":350,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":19,"reported_epoch":708,"state":"active+clean","last_fresh":"2023-09-23T21:21:17.509211+0000","last_change":"2023-09-23T21:20:50.930303+0000","last_active":"2023-09-23T21:21:17.509211+0000","last_peered":"2023-09-23T21:21:17.509211+0000","last_clean":"2023-09-23T21:21:17.509211+0000","last_became_active":"2023-09-23T21:20:50.929881+0000","last_became_peered":"2023-09-23T21:20:50.929881+0000","last_unstale":"2023-09-23T21:21:17.509211+0000","last_undegraded":"2023-09-23T21:21:17.509211+0000","last_fullsized":"2023-09-23T21:21:17.509211+0000","mapping_epoch":699,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":700,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:52:39.669108+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.e","version":"79'161","reported_seq":927,"reported_epoch":708,"state":"active+clean","last_fresh":"2023-09-23T21:21:17.505789+0000","last_change":"2023-09-23T21:16:45.433204+0000","last_active":"2023-09-23T21:21:17.505789+0000","last_peered":"2023-09-23T21:21:17.505789+0000","last_clean":"2023-09-23T21:21:17.505789+0000","last_became_active":"2023-09-23T21:16:45.432836+0000","last_became_peered":"2023-09-23T21:16:45.432836+0000","last_unstale":"2023-09-23T21:21:17.505789+0000","last_undegraded":"2023-09-23T21:21:17.505789+0000","last_fullsized":"2023-09-23T21:21:17.505789+0000","mapping_epoch":567,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":568,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:15:39.046962+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:15:39.046962+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:11:20.720731+0000","scrub_duration":0.0082638769999999993,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":535,"reported_epoch":708,"state":"active+clean","last_fresh":"2023-09-23T21:21:17.507300+0000","last_change":"2023-09-23T21:20:42.357079+0000","last_active":"2023-09-23T21:21:17.507300+0000","last_peered":"2023-09-23T21:21:17.507300+0000","last_clean":"2023-09-23T21:21:17.507300+0000","last_became_active":"2023-09-23T21:20:42.356777+0000","last_became_peered":"2023-09-23T21:20:42.356777+0000","last_unstale":"2023-09-23T21:21:17.507300+0000","last_undegraded":"2023-09-23T21:21:17.507300+0000","last_fullsized":"2023-09-23T21:21:17.507300+0000","mapping_epoch":694,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":695,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:57:20.387559+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":693,"reported_epoch":708,"state":"active+clean","last_fresh":"2023-09-23T21:21:17.901703+0000","last_change":"2023-09-23T21:20:12.080657+0000","last_active":"2023-09-23T21:21:17.901703+0000","last_peered":"2023-09-23T21:21:17.901703+0000","last_clean":"2023-09-23T21:21:17.901703+0000","last_became_active":"2023-09-23T21:20:12.080285+0000","last_became_peered":"2023-09-23T21:20:12.080285+0000","last_unstale":"2023-09-23T21:21:17.901703+0000","last_undegraded":"2023-09-23T21:21:17.901703+0000","last_fullsized":"2023-09-23T21:21:17.901703+0000","mapping_epoch":677,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":678,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:43:27.608818+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.9","version":"79'161","reported_seq":774,"reported_epoch":708,"state":"active+clean","last_fresh":"2023-09-23T21:21:17.506330+0000","last_change":"2023-09-23T21:20:48.511155+0000","last_active":"2023-09-23T21:21:17.506330+0000","last_peered":"2023-09-23T21:21:17.506330+0000","last_clean":"2023-09-23T21:21:17.506330+0000","last_became_active":"2023-09-23T21:20:48.510795+0000","last_became_peered":"2023-09-23T21:20:48.510795+0000","last_unstale":"2023-09-23T21:21:17.506330+0000","last_undegraded":"2023-09-23T21:21:17.506330+0000","last_fullsized":"2023-09-23T21:21:17.506330+0000","mapping_epoch":697,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":698,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:20:31.919517+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:20:31.919517+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:35:43.721278+0000","scrub_duration":0.0024129429999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":456,"reported_epoch":708,"state":"active+clean","last_fresh":"2023-09-23T21:21:17.506280+0000","last_change":"2023-09-23T21:20:30.314664+0000","last_active":"2023-09-23T21:21:17.506280+0000","last_peered":"2023-09-23T21:21:17.506280+0000","last_clean":"2023-09-23T21:21:17.506280+0000","last_became_active":"2023-09-23T21:20:30.314368+0000","last_became_peered":"2023-09-23T21:20:30.314368+0000","last_unstale":"2023-09-23T21:21:17.506280+0000","last_undegraded":"2023-09-23T21:21:17.506280+0000","last_fullsized":"2023-09-23T21:21:17.506280+0000","mapping_epoch":689,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":690,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:18:26.565378+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":312,"reported_epoch":708,"state":"active+clean","last_fresh":"2023-09-23T21:21:17.505608+0000","last_change":"2023-09-23T21:20:05.018378+0000","last_active":"2023-09-23T21:21:17.505608+0000","last_peered":"2023-09-23T21:21:17.505608+0000","last_clean":"2023-09-23T21:21:17.505608+0000","last_became_active":"2023-09-23T21:20:05.017706+0000","last_became_peered":"2023-09-23T21:20:05.017706+0000","last_unstale":"2023-09-23T21:21:17.505608+0000","last_undegraded":"2023-09-23T21:21:17.505608+0000","last_fullsized":"2023-09-23T21:21:17.505608+0000","mapping_epoch":672,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":673,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:35:40.095410+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.8","version":"79'161","reported_seq":863,"reported_epoch":708,"state":"active+clean","last_fresh":"2023-09-23T21:21:17.507560+0000","last_change":"2023-09-23T21:20:42.355580+0000","last_active":"2023-09-23T21:21:17.507560+0000","last_peered":"2023-09-23T21:21:17.507560+0000","last_clean":"2023-09-23T21:21:17.507560+0000","last_became_active":"2023-09-23T21:20:42.355019+0000","last_became_peered":"2023-09-23T21:20:42.355019+0000","last_unstale":"2023-09-23T21:21:17.507560+0000","last_undegraded":"2023-09-23T21:21:17.507560+0000","last_fullsized":"2023-09-23T21:21:17.507560+0000","mapping_epoch":694,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":695,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:15:36.367640+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:15:36.367640+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:32:15.572118+0000","scrub_duration":0.005826899,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":56,"reported_epoch":708,"state":"active+clean","last_fresh":"2023-09-23T21:21:17.509397+0000","last_change":"2023-09-23T21:20:12.078659+0000","last_active":"2023-09-23T21:21:17.509397+0000","last_peered":"2023-09-23T21:21:17.509397+0000","last_clean":"2023-09-23T21:21:17.509397+0000","last_became_active":"2023-09-23T21:20:12.078340+0000","last_became_peered":"2023-09-23T21:20:12.078340+0000","last_unstale":"2023-09-23T21:21:17.509397+0000","last_undegraded":"2023-09-23T21:21:17.509397+0000","last_fullsized":"2023-09-23T21:21:17.509397+0000","mapping_epoch":677,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":678,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:42:13.222106+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":316,"reported_epoch":709,"state":"active+clean","last_fresh":"2023-09-23T21:21:18.514308+0000","last_change":"2023-09-23T21:21:18.514308+0000","last_active":"2023-09-23T21:21:18.514308+0000","last_peered":"2023-09-23T21:21:18.514308+0000","last_clean":"2023-09-23T21:21:18.514308+0000","last_became_active":"2023-09-23T21:21:18.513966+0000","last_became_peered":"2023-09-23T21:21:18.513966+0000","last_unstale":"2023-09-23T21:21:18.514308+0000","last_undegraded":"2023-09-23T21:21:18.514308+0000","last_fullsized":"2023-09-23T21:21:18.514308+0000","mapping_epoch":708,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":709,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:38:09.611613+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.b","version":"79'161","reported_seq":779,"reported_epoch":708,"state":"active+clean","last_fresh":"2023-09-23T21:21:17.502961+0000","last_change":"2023-09-23T21:20:19.085592+0000","last_active":"2023-09-23T21:21:17.502961+0000","last_peered":"2023-09-23T21:21:17.502961+0000","last_clean":"2023-09-23T21:21:17.502961+0000","last_became_active":"2023-09-23T21:20:19.085161+0000","last_became_peered":"2023-09-23T21:20:19.085161+0000","last_unstale":"2023-09-23T21:21:17.502961+0000","last_undegraded":"2023-09-23T21:21:17.502961+0000","last_fullsized":"2023-09-23T21:21:17.502961+0000","mapping_epoch":684,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":685,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:15:36.339885+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:15:36.339885+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:45:27.119479+0000","scrub_duration":0.0068473279999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,5],"acting":[1,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":286,"reported_epoch":708,"state":"active+clean","last_fresh":"2023-09-23T21:21:17.506029+0000","last_change":"2023-09-23T21:16:42.425103+0000","last_active":"2023-09-23T21:21:17.506029+0000","last_peered":"2023-09-23T21:21:17.506029+0000","last_clean":"2023-09-23T21:21:17.506029+0000","last_became_active":"2023-09-23T21:16:42.424543+0000","last_became_peered":"2023-09-23T21:16:42.424543+0000","last_unstale":"2023-09-23T21:21:17.506029+0000","last_undegraded":"2023-09-23T21:21:17.506029+0000","last_fullsized":"2023-09-23T21:21:17.506029+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:11:28.885246+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":670,"reported_epoch":708,"state":"active+clean","last_fresh":"2023-09-23T21:21:17.505927+0000","last_change":"2023-09-23T21:20:50.930124+0000","last_active":"2023-09-23T21:21:17.505927+0000","last_peered":"2023-09-23T21:21:17.505927+0000","last_clean":"2023-09-23T21:21:17.505927+0000","last_became_active":"2023-09-23T21:20:50.929778+0000","last_became_peered":"2023-09-23T21:20:50.929778+0000","last_unstale":"2023-09-23T21:21:17.505927+0000","last_undegraded":"2023-09-23T21:21:17.505927+0000","last_fullsized":"2023-09-23T21:21:17.505927+0000","mapping_epoch":699,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":700,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:05:56.336810+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.a","version":"79'161","reported_seq":906,"reported_epoch":708,"state":"active+clean","last_fresh":"2023-09-23T21:21:17.507401+0000","last_change":"2023-09-23T21:20:42.355595+0000","last_active":"2023-09-23T21:21:17.507401+0000","last_peered":"2023-09-23T21:21:17.507401+0000","last_clean":"2023-09-23T21:21:17.507401+0000","last_became_active":"2023-09-23T21:20:42.355194+0000","last_became_peered":"2023-09-23T21:20:42.355194+0000","last_unstale":"2023-09-23T21:21:17.507401+0000","last_undegraded":"2023-09-23T21:21:17.507401+0000","last_fullsized":"2023-09-23T21:21:17.507401+0000","mapping_epoch":694,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":695,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:15:46.420650+0000","last_deep_scrub":"79'161","last_deep_scrub_stamp":"2023-09-23T21:05:32.630292+0000","last_clean_scrub_stamp":"2023-09-23T21:15:46.420650+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:18:43.952878+0000","scrub_duration":0.0072898930000000004,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":585,"reported_epoch":708,"state":"active+clean","last_fresh":"2023-09-23T21:21:17.505289+0000","last_change":"2023-09-23T21:20:12.104617+0000","last_active":"2023-09-23T21:21:17.505289+0000","last_peered":"2023-09-23T21:21:17.505289+0000","last_clean":"2023-09-23T21:21:17.505289+0000","last_became_active":"2023-09-23T21:20:12.104106+0000","last_became_peered":"2023-09-23T21:20:12.104106+0000","last_unstale":"2023-09-23T21:21:17.505289+0000","last_undegraded":"2023-09-23T21:21:17.505289+0000","last_fullsized":"2023-09-23T21:21:17.505289+0000","mapping_epoch":677,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":678,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:09:16.885310+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":42,"reported_epoch":708,"state":"active+clean","last_fresh":"2023-09-23T21:21:17.504641+0000","last_change":"2023-09-23T21:20:12.115012+0000","last_active":"2023-09-23T21:21:17.504641+0000","last_peered":"2023-09-23T21:21:17.504641+0000","last_clean":"2023-09-23T21:21:17.504641+0000","last_became_active":"2023-09-23T21:20:12.114632+0000","last_became_peered":"2023-09-23T21:20:12.114632+0000","last_unstale":"2023-09-23T21:21:17.504641+0000","last_undegraded":"2023-09-23T21:21:17.504641+0000","last_fullsized":"2023-09-23T21:21:17.504641+0000","mapping_epoch":677,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":678,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:41:02.454109+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.7","version":"79'161","reported_seq":842,"reported_epoch":708,"state":"active+clean","last_fresh":"2023-09-23T21:21:17.509325+0000","last_change":"2023-09-23T21:20:08.010981+0000","last_active":"2023-09-23T21:21:17.509325+0000","last_peered":"2023-09-23T21:21:17.509325+0000","last_clean":"2023-09-23T21:21:17.509325+0000","last_became_active":"2023-09-23T21:20:08.010682+0000","last_became_peered":"2023-09-23T21:20:08.010682+0000","last_unstale":"2023-09-23T21:21:17.509325+0000","last_undegraded":"2023-09-23T21:21:17.509325+0000","last_fullsized":"2023-09-23T21:21:17.509325+0000","mapping_epoch":675,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":676,"parent":"0.0","parent_split_bits":3,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:15:51.016427+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:15:51.016427+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:52:28.243159+0000","scrub_duration":0.0017648640000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.4","version":"668'149","reported_seq":1052,"reported_epoch":708,"state":"active+clean","last_fresh":"2023-09-23T21:21:17.503057+0000","last_change":"2023-09-23T21:20:15.083615+0000","last_active":"2023-09-23T21:21:17.503057+0000","last_peered":"2023-09-23T21:21:17.503057+0000","last_clean":"2023-09-23T21:21:17.503057+0000","last_became_active":"2023-09-23T21:20:15.082980+0000","last_became_peered":"2023-09-23T21:20:15.082980+0000","last_unstale":"2023-09-23T21:21:17.503057+0000","last_undegraded":"2023-09-23T21:21:17.503057+0000","last_fullsized":"2023-09-23T21:21:17.503057+0000","mapping_epoch":680,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":681,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:06:27.749409+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00068658800000000002,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":533,"num_read_kb":902,"num_write":165,"num_write_kb":306,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":13,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":37,"reported_epoch":708,"state":"active+clean","last_fresh":"2023-09-23T21:21:17.507978+0000","last_change":"2023-09-23T21:20:15.937817+0000","last_active":"2023-09-23T21:21:17.507978+0000","last_peered":"2023-09-23T21:21:17.507978+0000","last_clean":"2023-09-23T21:21:17.507978+0000","last_became_active":"2023-09-23T21:20:15.937449+0000","last_became_peered":"2023-09-23T21:20:15.937449+0000","last_unstale":"2023-09-23T21:21:17.507978+0000","last_undegraded":"2023-09-23T21:21:17.507978+0000","last_fullsized":"2023-09-23T21:21:17.507978+0000","mapping_epoch":681,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":682,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:10:39.568587+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":51,"reported_epoch":708,"state":"active+clean","last_fresh":"2023-09-23T21:21:17.502525+0000","last_change":"2023-09-23T21:20:15.931234+0000","last_active":"2023-09-23T21:21:17.502525+0000","last_peered":"2023-09-23T21:21:17.502525+0000","last_clean":"2023-09-23T21:21:17.502525+0000","last_became_active":"2023-09-23T21:20:15.930213+0000","last_became_peered":"2023-09-23T21:20:15.930213+0000","last_unstale":"2023-09-23T21:21:17.502525+0000","last_undegraded":"2023-09-23T21:21:17.502525+0000","last_fullsized":"2023-09-23T21:21:17.502525+0000","mapping_epoch":681,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":682,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:46:34.523504+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"79'161","reported_seq":813,"reported_epoch":708,"state":"active+clean","last_fresh":"2023-09-23T21:21:17.506655+0000","last_change":"2023-09-23T21:20:42.353395+0000","last_active":"2023-09-23T21:21:17.506655+0000","last_peered":"2023-09-23T21:21:17.506655+0000","last_clean":"2023-09-23T21:21:17.506655+0000","last_became_active":"2023-09-23T21:20:42.352986+0000","last_became_peered":"2023-09-23T21:20:42.352986+0000","last_unstale":"2023-09-23T21:21:17.506655+0000","last_undegraded":"2023-09-23T21:21:17.506655+0000","last_fullsized":"2023-09-23T21:21:17.506655+0000","mapping_epoch":694,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":695,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:15:34.345138+0000","last_deep_scrub":"79'161","last_deep_scrub_stamp":"2023-09-23T21:05:30.652189+0000","last_clean_scrub_stamp":"2023-09-23T21:15:34.345138+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:13:49.266544+0000","scrub_duration":0.0057520000000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"668'88","reported_seq":95,"reported_epoch":708,"state":"active+clean","last_fresh":"2023-09-23T21:21:17.507266+0000","last_change":"2023-09-23T21:20:50.953069+0000","last_active":"2023-09-23T21:21:17.507266+0000","last_peered":"2023-09-23T21:21:17.507266+0000","last_clean":"2023-09-23T21:21:17.507266+0000","last_became_active":"2023-09-23T21:20:50.930023+0000","last_became_peered":"2023-09-23T21:20:50.930023+0000","last_unstale":"2023-09-23T21:21:17.507266+0000","last_undegraded":"2023-09-23T21:21:17.507266+0000","last_fullsized":"2023-09-23T21:21:17.507266+0000","mapping_epoch":699,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":700,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":88,"log_dups_size":0,"ondisk_log_size":88,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:20:23.306383+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00049772800000000002,"stat_sum":{"num_bytes":4194323,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":875,"num_read_kb":1071,"num_write":62,"num_write_kb":4277,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":18,"num_bytes_recovered":8388646,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":662,"reported_epoch":708,"state":"active+clean","last_fresh":"2023-09-23T21:21:17.505808+0000","last_change":"2023-09-23T21:20:05.017845+0000","last_active":"2023-09-23T21:21:17.505808+0000","last_peered":"2023-09-23T21:21:17.505808+0000","last_clean":"2023-09-23T21:21:17.505808+0000","last_became_active":"2023-09-23T21:20:05.017517+0000","last_became_peered":"2023-09-23T21:20:05.017517+0000","last_unstale":"2023-09-23T21:21:17.505808+0000","last_undegraded":"2023-09-23T21:21:17.505808+0000","last_fullsized":"2023-09-23T21:21:17.505808+0000","mapping_epoch":672,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":673,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:24:33.218000+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":42,"reported_epoch":708,"state":"active+clean","last_fresh":"2023-09-23T21:21:17.503100+0000","last_change":"2023-09-23T21:20:12.107633+0000","last_active":"2023-09-23T21:21:17.503100+0000","last_peered":"2023-09-23T21:21:17.503100+0000","last_clean":"2023-09-23T21:21:17.503100+0000","last_became_active":"2023-09-23T21:20:12.107060+0000","last_became_peered":"2023-09-23T21:20:12.107060+0000","last_unstale":"2023-09-23T21:21:17.503100+0000","last_undegraded":"2023-09-23T21:21:17.503100+0000","last_fullsized":"2023-09-23T21:21:17.503100+0000","mapping_epoch":677,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":678,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:40:43.679026+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"79'161","reported_seq":888,"reported_epoch":708,"state":"active+clean","last_fresh":"2023-09-23T21:21:17.506225+0000","last_change":"2023-09-23T21:20:48.511190+0000","last_active":"2023-09-23T21:21:17.506225+0000","last_peered":"2023-09-23T21:21:17.506225+0000","last_clean":"2023-09-23T21:21:17.506225+0000","last_became_active":"2023-09-23T21:20:48.510409+0000","last_became_peered":"2023-09-23T21:20:48.510409+0000","last_unstale":"2023-09-23T21:21:17.506225+0000","last_undegraded":"2023-09-23T21:21:17.506225+0000","last_fullsized":"2023-09-23T21:21:17.506225+0000","mapping_epoch":697,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":698,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:24.802451+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:24.802451+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:13:19.915510+0000","scrub_duration":0.0090861250000000005,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"668'111","reported_seq":1241,"reported_epoch":708,"state":"active+clean","last_fresh":"2023-09-23T21:21:17.508223+0000","last_change":"2023-09-23T21:19:51.908105+0000","last_active":"2023-09-23T21:21:17.508223+0000","last_peered":"2023-09-23T21:21:17.508223+0000","last_clean":"2023-09-23T21:21:17.508223+0000","last_became_active":"2023-09-23T21:13:12.206650+0000","last_became_peered":"2023-09-23T21:13:12.206650+0000","last_unstale":"2023-09-23T21:21:17.508223+0000","last_undegraded":"2023-09-23T21:21:17.508223+0000","last_fullsized":"2023-09-23T21:21:17.508223+0000","mapping_epoch":493,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":111,"log_dups_size":0,"ondisk_log_size":111,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:10:01.892936+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0016921,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":876,"num_read_kb":3120,"num_write":86,"num_write_kb":2279,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":42,"reported_epoch":708,"state":"active+clean","last_fresh":"2023-09-23T21:21:17.502563+0000","last_change":"2023-09-23T21:20:12.094430+0000","last_active":"2023-09-23T21:21:17.502563+0000","last_peered":"2023-09-23T21:21:17.502563+0000","last_clean":"2023-09-23T21:21:17.502563+0000","last_became_active":"2023-09-23T21:20:12.088960+0000","last_became_peered":"2023-09-23T21:20:12.088960+0000","last_unstale":"2023-09-23T21:21:17.502563+0000","last_undegraded":"2023-09-23T21:21:17.502563+0000","last_fullsized":"2023-09-23T21:21:17.502563+0000","mapping_epoch":677,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":678,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:05:13.583404+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":51,"reported_epoch":708,"state":"active+clean","last_fresh":"2023-09-23T21:21:17.504722+0000","last_change":"2023-09-23T21:20:15.935747+0000","last_active":"2023-09-23T21:21:17.504722+0000","last_peered":"2023-09-23T21:21:17.504722+0000","last_clean":"2023-09-23T21:21:17.504722+0000","last_became_active":"2023-09-23T21:20:15.935401+0000","last_became_peered":"2023-09-23T21:20:15.935401+0000","last_unstale":"2023-09-23T21:21:17.504722+0000","last_undegraded":"2023-09-23T21:21:17.504722+0000","last_fullsized":"2023-09-23T21:21:17.504722+0000","mapping_epoch":681,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":682,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:44:17.963923+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.3","version":"668'166","reported_seq":1059,"reported_epoch":708,"state":"active+clean","last_fresh":"2023-09-23T21:21:17.506982+0000","last_change":"2023-09-23T21:20:42.354590+0000","last_active":"2023-09-23T21:21:17.506982+0000","last_peered":"2023-09-23T21:21:17.506982+0000","last_clean":"2023-09-23T21:21:17.506982+0000","last_became_active":"2023-09-23T21:20:42.354054+0000","last_became_peered":"2023-09-23T21:20:42.354054+0000","last_unstale":"2023-09-23T21:21:17.506982+0000","last_undegraded":"2023-09-23T21:21:17.506982+0000","last_fullsized":"2023-09-23T21:21:17.506982+0000","mapping_epoch":694,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":695,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":166,"log_dups_size":0,"ondisk_log_size":166,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:36:43.918675+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00031341300000000002,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":787,"num_read_kb":1740,"num_write":150,"num_write_kb":1123,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"79'161","reported_seq":791,"reported_epoch":708,"state":"active+clean","last_fresh":"2023-09-23T21:21:17.505813+0000","last_change":"2023-09-23T21:20:48.510993+0000","last_active":"2023-09-23T21:21:17.505813+0000","last_peered":"2023-09-23T21:21:17.505813+0000","last_clean":"2023-09-23T21:21:17.505813+0000","last_became_active":"2023-09-23T21:20:48.510554+0000","last_became_peered":"2023-09-23T21:20:48.510554+0000","last_unstale":"2023-09-23T21:21:17.505813+0000","last_undegraded":"2023-09-23T21:21:17.505813+0000","last_fullsized":"2023-09-23T21:21:17.505813+0000","mapping_epoch":697,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":698,"parent":"0.0","parent_split_bits":0,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:20.738273+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:20.738273+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:03:57.365350+0000","scrub_duration":0.0088336339999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":477,"reported_epoch":708,"state":"active+clean","last_fresh":"2023-09-23T21:21:17.505576+0000","last_change":"2023-09-23T21:20:30.373442+0000","last_active":"2023-09-23T21:21:17.505576+0000","last_peered":"2023-09-23T21:21:17.505576+0000","last_clean":"2023-09-23T21:21:17.505576+0000","last_became_active":"2023-09-23T21:20:30.372967+0000","last_became_peered":"2023-09-23T21:20:30.372967+0000","last_unstale":"2023-09-23T21:21:17.505576+0000","last_undegraded":"2023-09-23T21:21:17.505576+0000","last_fullsized":"2023-09-23T21:21:17.505576+0000","mapping_epoch":689,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":690,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:13:46.307043+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":19,"reported_epoch":708,"state":"active+clean","last_fresh":"2023-09-23T21:21:17.508665+0000","last_change":"2023-09-23T21:20:50.930557+0000","last_active":"2023-09-23T21:21:17.508665+0000","last_peered":"2023-09-23T21:21:17.508665+0000","last_clean":"2023-09-23T21:21:17.508665+0000","last_became_active":"2023-09-23T21:20:50.930264+0000","last_became_peered":"2023-09-23T21:20:50.930264+0000","last_unstale":"2023-09-23T21:21:17.508665+0000","last_undegraded":"2023-09-23T21:21:17.508665+0000","last_fullsized":"2023-09-23T21:21:17.508665+0000","mapping_epoch":699,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":700,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:41:33.282685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.3","version":"79'161","reported_seq":859,"reported_epoch":708,"state":"active+clean","last_fresh":"2023-09-23T21:21:17.502861+0000","last_change":"2023-09-23T21:20:53.938250+0000","last_active":"2023-09-23T21:21:17.502861+0000","last_peered":"2023-09-23T21:21:17.502861+0000","last_clean":"2023-09-23T21:21:17.502861+0000","last_became_active":"2023-09-23T21:20:53.937861+0000","last_became_peered":"2023-09-23T21:20:53.937861+0000","last_unstale":"2023-09-23T21:21:17.502861+0000","last_undegraded":"2023-09-23T21:21:17.502861+0000","last_fullsized":"2023-09-23T21:21:17.502861+0000","mapping_epoch":702,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":703,"parent":"0.0","parent_split_bits":3,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:05:57.832302+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:05:57.832302+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:16:41.826413+0000","scrub_duration":0.010206847999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"668'67","reported_seq":854,"reported_epoch":708,"state":"active+clean","last_fresh":"2023-09-23T21:21:17.505908+0000","last_change":"2023-09-23T21:20:48.511538+0000","last_active":"2023-09-23T21:21:17.505908+0000","last_peered":"2023-09-23T21:21:17.505908+0000","last_clean":"2023-09-23T21:21:17.505908+0000","last_became_active":"2023-09-23T21:20:48.510475+0000","last_became_peered":"2023-09-23T21:20:48.510475+0000","last_unstale":"2023-09-23T21:21:17.505908+0000","last_undegraded":"2023-09-23T21:21:17.505908+0000","last_fullsized":"2023-09-23T21:21:17.505908+0000","mapping_epoch":697,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":698,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":67,"log_dups_size":0,"ondisk_log_size":67,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:55:03.524020+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00037825000000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":538,"num_read_kb":2522,"num_write":45,"num_write_kb":3096,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":711,"reported_epoch":709,"state":"active+clean","last_fresh":"2023-09-23T21:21:18.510285+0000","last_change":"2023-09-23T21:13:44.409212+0000","last_active":"2023-09-23T21:21:18.510285+0000","last_peered":"2023-09-23T21:21:18.510285+0000","last_clean":"2023-09-23T21:21:18.510285+0000","last_became_active":"2023-09-23T21:13:44.408958+0000","last_became_peered":"2023-09-23T21:13:44.408958+0000","last_unstale":"2023-09-23T21:21:18.510285+0000","last_undegraded":"2023-09-23T21:21:18.510285+0000","last_fullsized":"2023-09-23T21:21:18.510285+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:03:52.749964+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":248,"reported_epoch":708,"state":"active+clean","last_fresh":"2023-09-23T21:21:17.902062+0000","last_change":"2023-09-23T21:20:12.079524+0000","last_active":"2023-09-23T21:21:17.902062+0000","last_peered":"2023-09-23T21:21:17.902062+0000","last_clean":"2023-09-23T21:21:17.902062+0000","last_became_active":"2023-09-23T21:20:12.079185+0000","last_became_peered":"2023-09-23T21:20:12.079185+0000","last_unstale":"2023-09-23T21:21:17.902062+0000","last_undegraded":"2023-09-23T21:21:17.902062+0000","last_fullsized":"2023-09-23T21:21:17.902062+0000","mapping_epoch":677,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":678,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:49:26.875689+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.6","version":"79'161","reported_seq":652,"reported_epoch":708,"state":"active+clean","last_fresh":"2023-09-23T21:21:17.508790+0000","last_change":"2023-09-23T21:20:08.014769+0000","last_active":"2023-09-23T21:21:17.508790+0000","last_peered":"2023-09-23T21:21:17.508790+0000","last_clean":"2023-09-23T21:21:17.508790+0000","last_became_active":"2023-09-23T21:20:08.014281+0000","last_became_peered":"2023-09-23T21:20:08.014281+0000","last_unstale":"2023-09-23T21:21:17.508790+0000","last_undegraded":"2023-09-23T21:21:17.508790+0000","last_fullsized":"2023-09-23T21:21:17.508790+0000","mapping_epoch":675,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":676,"parent":"0.0","parent_split_bits":3,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:15:43.333733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:15:43.333733+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:24:17.713184+0000","scrub_duration":0.0092107530000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.5","version":"668'105","reported_seq":550,"reported_epoch":708,"state":"active+clean","last_fresh":"2023-09-23T21:21:17.506331+0000","last_change":"2023-09-23T21:19:51.914035+0000","last_active":"2023-09-23T21:21:17.506331+0000","last_peered":"2023-09-23T21:21:17.506331+0000","last_clean":"2023-09-23T21:21:17.506331+0000","last_became_active":"2023-09-23T21:16:42.482847+0000","last_became_peered":"2023-09-23T21:16:42.482847+0000","last_unstale":"2023-09-23T21:21:17.506331+0000","last_undegraded":"2023-09-23T21:21:17.506331+0000","last_fullsized":"2023-09-23T21:21:17.506331+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":105,"log_dups_size":0,"ondisk_log_size":105,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:38:57.905679+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0010907810000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":751,"num_read_kb":635,"num_write":101,"num_write_kb":54,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":51,"reported_epoch":708,"state":"active+clean","last_fresh":"2023-09-23T21:21:17.503007+0000","last_change":"2023-09-23T21:20:15.935131+0000","last_active":"2023-09-23T21:21:17.503007+0000","last_peered":"2023-09-23T21:21:17.503007+0000","last_clean":"2023-09-23T21:21:17.503007+0000","last_became_active":"2023-09-23T21:20:15.934725+0000","last_became_peered":"2023-09-23T21:20:15.934725+0000","last_unstale":"2023-09-23T21:21:17.503007+0000","last_undegraded":"2023-09-23T21:21:17.503007+0000","last_fullsized":"2023-09-23T21:21:17.503007+0000","mapping_epoch":681,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":682,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:55:09.651551+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":703,"reported_epoch":709,"state":"active+clean","last_fresh":"2023-09-23T21:21:18.509625+0000","last_change":"2023-09-23T21:20:15.927550+0000","last_active":"2023-09-23T21:21:18.509625+0000","last_peered":"2023-09-23T21:21:18.509625+0000","last_clean":"2023-09-23T21:21:18.509625+0000","last_became_active":"2023-09-23T21:20:15.927228+0000","last_became_peered":"2023-09-23T21:20:15.927228+0000","last_unstale":"2023-09-23T21:21:18.509625+0000","last_undegraded":"2023-09-23T21:21:18.509625+0000","last_fullsized":"2023-09-23T21:21:18.509625+0000","mapping_epoch":681,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":682,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:46:22.938004+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.5","version":"79'161","reported_seq":806,"reported_epoch":708,"state":"active+clean","last_fresh":"2023-09-23T21:21:17.506727+0000","last_change":"2023-09-23T21:20:53.938526+0000","last_active":"2023-09-23T21:21:17.506727+0000","last_peered":"2023-09-23T21:21:17.506727+0000","last_clean":"2023-09-23T21:21:17.506727+0000","last_became_active":"2023-09-23T21:20:53.938023+0000","last_became_peered":"2023-09-23T21:20:53.938023+0000","last_unstale":"2023-09-23T21:21:17.506727+0000","last_undegraded":"2023-09-23T21:21:17.506727+0000","last_fullsized":"2023-09-23T21:21:17.506727+0000","mapping_epoch":702,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":703,"parent":"0.0","parent_split_bits":3,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:15:41.108579+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:15:41.108579+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:37:01.378332+0000","scrub_duration":0.0061511960000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":490,"reported_epoch":708,"state":"active+clean","last_fresh":"2023-09-23T21:21:17.507181+0000","last_change":"2023-09-23T21:20:30.314590+0000","last_active":"2023-09-23T21:21:17.507181+0000","last_peered":"2023-09-23T21:21:17.507181+0000","last_clean":"2023-09-23T21:21:17.507181+0000","last_became_active":"2023-09-23T21:20:30.314211+0000","last_became_peered":"2023-09-23T21:20:30.314211+0000","last_unstale":"2023-09-23T21:21:17.507181+0000","last_undegraded":"2023-09-23T21:21:17.507181+0000","last_fullsized":"2023-09-23T21:21:17.507181+0000","mapping_epoch":689,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":690,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:18:59.216392+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"668'199","reported_seq":1349,"reported_epoch":708,"state":"active+clean","last_fresh":"2023-09-23T21:21:17.504341+0000","last_change":"2023-09-23T21:20:15.077518+0000","last_active":"2023-09-23T21:21:17.504341+0000","last_peered":"2023-09-23T21:21:17.504341+0000","last_clean":"2023-09-23T21:21:17.504341+0000","last_became_active":"2023-09-23T21:20:15.076836+0000","last_became_peered":"2023-09-23T21:20:15.076836+0000","last_unstale":"2023-09-23T21:21:17.504341+0000","last_undegraded":"2023-09-23T21:21:17.504341+0000","last_fullsized":"2023-09-23T21:21:17.504341+0000","mapping_epoch":680,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":681,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":199,"log_dups_size":0,"ondisk_log_size":199,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:44:53.169150+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00037772100000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1225,"num_read_kb":1302,"num_write":160,"num_write_kb":238,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":14,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":571,"reported_epoch":708,"state":"active+clean","last_fresh":"2023-09-23T21:21:17.505290+0000","last_change":"2023-09-23T21:16:42.463411+0000","last_active":"2023-09-23T21:21:17.505290+0000","last_peered":"2023-09-23T21:21:17.505290+0000","last_clean":"2023-09-23T21:21:17.505290+0000","last_became_active":"2023-09-23T21:16:42.463020+0000","last_became_peered":"2023-09-23T21:16:42.463020+0000","last_unstale":"2023-09-23T21:21:17.505290+0000","last_undegraded":"2023-09-23T21:21:17.505290+0000","last_fullsized":"2023-09-23T21:21:17.505290+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:07:34.346129+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.4","version":"79'161","reported_seq":1168,"reported_epoch":708,"state":"active+clean","last_fresh":"2023-09-23T21:21:17.505954+0000","last_change":"2023-09-23T21:20:19.086061+0000","last_active":"2023-09-23T21:21:17.505954+0000","last_peered":"2023-09-23T21:21:17.505954+0000","last_clean":"2023-09-23T21:21:17.505954+0000","last_became_active":"2023-09-23T21:20:19.085679+0000","last_became_peered":"2023-09-23T21:20:19.085679+0000","last_unstale":"2023-09-23T21:21:17.505954+0000","last_undegraded":"2023-09-23T21:21:17.505954+0000","last_fullsized":"2023-09-23T21:21:17.505954+0000","mapping_epoch":684,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":685,"parent":"0.0","parent_split_bits":3,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:20.012764+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:20.012764+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:44:58.774529+0000","scrub_duration":0.0045251919999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"668'91","reported_seq":71,"reported_epoch":708,"state":"active+clean","last_fresh":"2023-09-23T21:21:17.509250+0000","last_change":"2023-09-23T21:20:05.037623+0000","last_active":"2023-09-23T21:21:17.509250+0000","last_peered":"2023-09-23T21:21:17.509250+0000","last_clean":"2023-09-23T21:21:17.509250+0000","last_became_active":"2023-09-23T21:20:05.014215+0000","last_became_peered":"2023-09-23T21:20:05.014215+0000","last_unstale":"2023-09-23T21:21:17.509250+0000","last_undegraded":"2023-09-23T21:21:17.509250+0000","last_fullsized":"2023-09-23T21:21:17.509250+0000","mapping_epoch":672,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":673,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":91,"log_dups_size":0,"ondisk_log_size":91,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:05:14.623182+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00051098199999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":410,"num_read_kb":1920,"num_write":77,"num_write_kb":1323,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"70'1","reported_seq":724,"reported_epoch":708,"state":"active+clean","last_fresh":"2023-09-23T21:21:17.902286+0000","last_change":"2023-09-23T21:20:12.101250+0000","last_active":"2023-09-23T21:21:17.902286+0000","last_peered":"2023-09-23T21:21:17.902286+0000","last_clean":"2023-09-23T21:21:17.902286+0000","last_became_active":"2023-09-23T21:20:12.079298+0000","last_became_peered":"2023-09-23T21:20:12.079298+0000","last_unstale":"2023-09-23T21:21:17.902286+0000","last_undegraded":"2023-09-23T21:21:17.902286+0000","last_fullsized":"2023-09-23T21:21:17.902286+0000","mapping_epoch":677,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":678,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:43:20.958541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":313,"reported_epoch":708,"state":"active+clean","last_fresh":"2023-09-23T21:21:17.505292+0000","last_change":"2023-09-23T21:20:15.926559+0000","last_active":"2023-09-23T21:21:17.505292+0000","last_peered":"2023-09-23T21:21:17.505292+0000","last_clean":"2023-09-23T21:21:17.505292+0000","last_became_active":"2023-09-23T21:20:15.926213+0000","last_became_peered":"2023-09-23T21:20:15.926213+0000","last_unstale":"2023-09-23T21:21:17.505292+0000","last_undegraded":"2023-09-23T21:21:17.505292+0000","last_fullsized":"2023-09-23T21:21:17.505292+0000","mapping_epoch":681,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":682,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:59:08.018105+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.19","version":"79'161","reported_seq":886,"reported_epoch":708,"state":"active+clean","last_fresh":"2023-09-23T21:21:17.506403+0000","last_change":"2023-09-23T21:20:32.918963+0000","last_active":"2023-09-23T21:21:17.506403+0000","last_peered":"2023-09-23T21:21:17.506403+0000","last_clean":"2023-09-23T21:21:17.506403+0000","last_became_active":"2023-09-23T21:20:30.311729+0000","last_became_peered":"2023-09-23T21:20:30.311729+0000","last_unstale":"2023-09-23T21:21:17.506403+0000","last_undegraded":"2023-09-23T21:21:17.506403+0000","last_fullsized":"2023-09-23T21:21:17.506403+0000","mapping_epoch":689,"log_start":"0'0","ondisk_log_start":"0'0","created":479,"last_epoch_clean":690,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:20:32.918868+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:20:32.918868+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:30:10.488726+0000","scrub_duration":0.001861538,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":239,"reported_epoch":708,"state":"active+clean","last_fresh":"2023-09-23T21:21:17.901826+0000","last_change":"2023-09-23T21:16:42.434830+0000","last_active":"2023-09-23T21:21:17.901826+0000","last_peered":"2023-09-23T21:21:17.901826+0000","last_clean":"2023-09-23T21:21:17.901826+0000","last_became_active":"2023-09-23T21:16:42.430289+0000","last_became_peered":"2023-09-23T21:16:42.430289+0000","last_unstale":"2023-09-23T21:21:17.901826+0000","last_undegraded":"2023-09-23T21:21:17.901826+0000","last_fullsized":"2023-09-23T21:21:17.901826+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:38:48.053766+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":224,"reported_epoch":708,"state":"active+clean","last_fresh":"2023-09-23T21:21:17.508589+0000","last_change":"2023-09-23T21:20:12.108763+0000","last_active":"2023-09-23T21:21:17.508589+0000","last_peered":"2023-09-23T21:21:17.508589+0000","last_clean":"2023-09-23T21:21:17.508589+0000","last_became_active":"2023-09-23T21:20:12.108295+0000","last_became_peered":"2023-09-23T21:20:12.108295+0000","last_unstale":"2023-09-23T21:21:17.508589+0000","last_undegraded":"2023-09-23T21:21:17.508589+0000","last_fullsized":"2023-09-23T21:21:17.508589+0000","mapping_epoch":677,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":678,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:44:58.483044+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.1e","version":"79'161","reported_seq":711,"reported_epoch":709,"state":"active+clean","last_fresh":"2023-09-23T21:21:18.508545+0000","last_change":"2023-09-23T21:20:08.012623+0000","last_active":"2023-09-23T21:21:18.508545+0000","last_peered":"2023-09-23T21:21:18.508545+0000","last_clean":"2023-09-23T21:21:18.508545+0000","last_became_active":"2023-09-23T21:20:08.012212+0000","last_became_peered":"2023-09-23T21:20:08.012212+0000","last_unstale":"2023-09-23T21:21:18.508545+0000","last_undegraded":"2023-09-23T21:21:18.508545+0000","last_fullsized":"2023-09-23T21:21:18.508545+0000","mapping_epoch":675,"log_start":"0'0","ondisk_log_start":"0'0","created":479,"last_epoch_clean":676,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:15:53.061319+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:15:53.061319+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:46:55.339646+0000","scrub_duration":0.007429878,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":347,"reported_epoch":708,"state":"active+clean","last_fresh":"2023-09-23T21:21:17.507149+0000","last_change":"2023-09-23T21:20:42.351206+0000","last_active":"2023-09-23T21:21:17.507149+0000","last_peered":"2023-09-23T21:21:17.507149+0000","last_clean":"2023-09-23T21:21:17.507149+0000","last_became_active":"2023-09-23T21:20:42.350828+0000","last_became_peered":"2023-09-23T21:20:42.350828+0000","last_unstale":"2023-09-23T21:21:17.507149+0000","last_undegraded":"2023-09-23T21:21:17.507149+0000","last_fullsized":"2023-09-23T21:21:17.507149+0000","mapping_epoch":694,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":695,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:37:53.392509+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":9,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":283,"reported_epoch":708,"state":"active+clean","last_fresh":"2023-09-23T21:21:17.505654+0000","last_change":"2023-09-23T21:20:12.104580+0000","last_active":"2023-09-23T21:21:17.505654+0000","last_peered":"2023-09-23T21:21:17.505654+0000","last_clean":"2023-09-23T21:21:17.505654+0000","last_became_active":"2023-09-23T21:20:12.104238+0000","last_became_peered":"2023-09-23T21:20:12.104238+0000","last_unstale":"2023-09-23T21:21:17.505654+0000","last_undegraded":"2023-09-23T21:21:17.505654+0000","last_fullsized":"2023-09-23T21:21:17.505654+0000","mapping_epoch":677,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":678,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:10:20.352583+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":541,"reported_epoch":708,"state":"active+clean","last_fresh":"2023-09-23T21:21:17.507901+0000","last_change":"2023-09-23T21:20:50.933085+0000","last_active":"2023-09-23T21:21:17.507901+0000","last_peered":"2023-09-23T21:21:17.507901+0000","last_clean":"2023-09-23T21:21:17.507901+0000","last_became_active":"2023-09-23T21:20:50.926757+0000","last_became_peered":"2023-09-23T21:20:50.926757+0000","last_unstale":"2023-09-23T21:21:17.507901+0000","last_undegraded":"2023-09-23T21:21:17.507901+0000","last_fullsized":"2023-09-23T21:21:17.507901+0000","mapping_epoch":699,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":700,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:48:16.967780+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":283,"reported_epoch":708,"state":"active+clean","last_fresh":"2023-09-23T21:21:17.505418+0000","last_change":"2023-09-23T21:13:12.216602+0000","last_active":"2023-09-23T21:21:17.505418+0000","last_peered":"2023-09-23T21:21:17.505418+0000","last_clean":"2023-09-23T21:21:17.505418+0000","last_became_active":"2023-09-23T21:13:12.216204+0000","last_became_peered":"2023-09-23T21:13:12.216204+0000","last_unstale":"2023-09-23T21:21:17.505418+0000","last_undegraded":"2023-09-23T21:21:17.505418+0000","last_fullsized":"2023-09-23T21:21:17.505418+0000","mapping_epoch":493,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:23:37.043390+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1c","version":"79'161","reported_seq":1003,"reported_epoch":708,"state":"active+clean","last_fresh":"2023-09-23T21:21:17.506201+0000","last_change":"2023-09-23T21:20:48.511669+0000","last_active":"2023-09-23T21:21:17.506201+0000","last_peered":"2023-09-23T21:21:17.506201+0000","last_clean":"2023-09-23T21:21:17.506201+0000","last_became_active":"2023-09-23T21:20:48.511243+0000","last_became_peered":"2023-09-23T21:20:48.511243+0000","last_unstale":"2023-09-23T21:21:17.506201+0000","last_undegraded":"2023-09-23T21:21:17.506201+0000","last_fullsized":"2023-09-23T21:21:17.506201+0000","mapping_epoch":697,"log_start":"0'0","ondisk_log_start":"0'0","created":479,"last_epoch_clean":698,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:15:54.019828+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:15:54.019828+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:15:41.480572+0000","scrub_duration":0.0020962530000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,1],"acting":[7,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":472,"reported_epoch":708,"state":"active+clean","last_fresh":"2023-09-23T21:21:17.507349+0000","last_change":"2023-09-23T21:20:50.929346+0000","last_active":"2023-09-23T21:21:17.507349+0000","last_peered":"2023-09-23T21:21:17.507349+0000","last_clean":"2023-09-23T21:21:17.507349+0000","last_became_active":"2023-09-23T21:20:50.928917+0000","last_became_peered":"2023-09-23T21:20:50.928917+0000","last_unstale":"2023-09-23T21:21:17.507349+0000","last_undegraded":"2023-09-23T21:21:17.507349+0000","last_fullsized":"2023-09-23T21:21:17.507349+0000","mapping_epoch":699,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":700,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:08:56.715607+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":453,"reported_epoch":708,"state":"active+clean","last_fresh":"2023-09-23T21:21:17.506539+0000","last_change":"2023-09-23T21:20:30.311794+0000","last_active":"2023-09-23T21:21:17.506539+0000","last_peered":"2023-09-23T21:21:17.506539+0000","last_clean":"2023-09-23T21:21:17.506539+0000","last_became_active":"2023-09-23T21:20:30.311484+0000","last_became_peered":"2023-09-23T21:20:30.311484+0000","last_unstale":"2023-09-23T21:21:17.506539+0000","last_undegraded":"2023-09-23T21:21:17.506539+0000","last_fullsized":"2023-09-23T21:21:17.506539+0000","mapping_epoch":689,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":690,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:47:43.155883+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1d","version":"79'161","reported_seq":722,"reported_epoch":708,"state":"active+clean","last_fresh":"2023-09-23T21:21:17.503536+0000","last_change":"2023-09-23T21:20:17.475961+0000","last_active":"2023-09-23T21:21:17.503536+0000","last_peered":"2023-09-23T21:21:17.503536+0000","last_clean":"2023-09-23T21:21:17.503536+0000","last_became_active":"2023-09-23T21:20:15.080849+0000","last_became_peered":"2023-09-23T21:20:15.080849+0000","last_unstale":"2023-09-23T21:21:17.503536+0000","last_undegraded":"2023-09-23T21:21:17.503536+0000","last_fullsized":"2023-09-23T21:21:17.503536+0000","mapping_epoch":680,"log_start":"0'0","ondisk_log_start":"0'0","created":479,"last_epoch_clean":681,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:20:17.475859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:20:17.475859+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:35:49.456854+0000","scrub_duration":0.0021360649999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":283,"reported_epoch":708,"state":"active+clean","last_fresh":"2023-09-23T21:21:17.901779+0000","last_change":"2023-09-23T21:20:15.942285+0000","last_active":"2023-09-23T21:21:17.901779+0000","last_peered":"2023-09-23T21:21:17.901779+0000","last_clean":"2023-09-23T21:21:17.901779+0000","last_became_active":"2023-09-23T21:20:15.932312+0000","last_became_peered":"2023-09-23T21:20:15.932312+0000","last_unstale":"2023-09-23T21:21:17.901779+0000","last_undegraded":"2023-09-23T21:21:17.901779+0000","last_fullsized":"2023-09-23T21:21:17.901779+0000","mapping_epoch":681,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":682,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:56:55.450387+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":17,"num_bytes_recovered":4986,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":303,"reported_epoch":708,"state":"active+clean","last_fresh":"2023-09-23T21:21:17.505222+0000","last_change":"2023-09-23T21:20:50.930893+0000","last_active":"2023-09-23T21:21:17.505222+0000","last_peered":"2023-09-23T21:21:17.505222+0000","last_clean":"2023-09-23T21:21:17.505222+0000","last_became_active":"2023-09-23T21:20:50.930601+0000","last_became_peered":"2023-09-23T21:20:50.930601+0000","last_unstale":"2023-09-23T21:21:17.505222+0000","last_undegraded":"2023-09-23T21:21:17.505222+0000","last_fullsized":"2023-09-23T21:21:17.505222+0000","mapping_epoch":699,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":700,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:00:50.010273+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":104,"num_bytes_recovered":24922,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":14322,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":31,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":853056,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4991,"ondisk_log_size":4991,"up":93,"acting":93,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5995,"num_read_kb":13212,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":55,"num_bytes_recovered":8388646,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201774,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":665700,"kb_used_data":7452,"kb_used_omap":0,"kb_used_meta":658240,"kb_avail":93079452,"statfs":{"total":95995035648,"available":95313358848,"internally_reserved":0,"allocated":7630848,"data_stored":5848948,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":674037760},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:21:28 2023","interfaces":[{"interface":"back","average":{"1min":0.56100000000000005,"5min":0.872,"15min":1.746},"min":{"1min":0.318,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":0.72099999999999997,"5min":3.968,"15min":37.207999999999998},"last":0.57399999999999995},{"interface":"front","average":{"1min":0.51700000000000002,"5min":0.96099999999999997,"15min":1.7529999999999999},"min":{"1min":0.29099999999999998,"5min":0.222,"15min":0.222},"max":{"1min":0.70499999999999996,"5min":5.2910000000000004,"15min":37.122},"last":0.60499999999999998}]},{"osd":1,"last update":"Sat Sep 23 21:21:23 2023","interfaces":[{"interface":"back","average":{"1min":0.55900000000000005,"5min":0.84099999999999997,"15min":1.486},"min":{"1min":0.32100000000000001,"5min":0.27600000000000002,"15min":0.26700000000000002},"max":{"1min":0.72999999999999998,"5min":6.9130000000000003,"15min":14.034000000000001},"last":0.64700000000000002},{"interface":"front","average":{"1min":0.64900000000000002,"5min":0.95899999999999996,"15min":1.496},"min":{"1min":0.35299999999999998,"5min":0.29999999999999999,"15min":0.26300000000000001},"max":{"1min":1.8859999999999999,"5min":5.4039999999999999,"15min":11.226000000000001},"last":0.63200000000000001}]},{"osd":2,"last update":"Sat Sep 23 21:20:41 2023","interfaces":[{"interface":"back","average":{"1min":0.58599999999999997,"5min":3.6200000000000001,"15min":2.3220000000000001},"min":{"1min":0.377,"5min":0.27200000000000002,"15min":0.26200000000000001},"max":{"1min":1.276,"5min":166.696,"15min":166.696},"last":0.501},{"interface":"front","average":{"1min":0.626,"5min":3.52,"15min":2.2730000000000001},"min":{"1min":0.46300000000000002,"5min":0.28799999999999998,"15min":0.26700000000000002},"max":{"1min":1.1479999999999999,"5min":165.57499999999999,"15min":165.57499999999999},"last":0.70599999999999996}]},{"osd":3,"last update":"Sat Sep 23 21:21:09 2023","interfaces":[{"interface":"back","average":{"1min":0.61299999999999999,"5min":1.2749999999999999,"15min":1.6539999999999999},"min":{"1min":0.42899999999999999,"5min":0.317,"15min":0.30099999999999999},"max":{"1min":0.77500000000000002,"5min":8.6259999999999994,"15min":9.1449999999999996},"last":0.59499999999999997},{"interface":"front","average":{"1min":0.622,"5min":1.234,"15min":1.7050000000000001},"min":{"1min":0.45200000000000001,"5min":0.28799999999999998,"15min":0.28699999999999998},"max":{"1min":0.83799999999999997,"5min":15.82,"15min":15.82},"last":0.68200000000000005}]},{"osd":4,"last update":"Sat Sep 23 21:21:09 2023","interfaces":[{"interface":"back","average":{"1min":0.56000000000000005,"5min":0.60099999999999998,"15min":1.113},"min":{"1min":0.33000000000000002,"5min":0.251,"15min":0.251},"max":{"1min":0.94499999999999995,"5min":3.9990000000000001,"15min":11.297000000000001},"last":0.73299999999999998},{"interface":"front","average":{"1min":0.60199999999999998,"5min":0.67000000000000004,"15min":1.0589999999999999},"min":{"1min":0.35999999999999999,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":0.92100000000000004,"5min":5.6349999999999998,"15min":12.752000000000001},"last":0.53500000000000003}]},{"osd":5,"last update":"Sat Sep 23 21:21:09 2023","interfaces":[{"interface":"back","average":{"1min":0.59699999999999998,"5min":0.73899999999999999,"15min":1.05},"min":{"1min":0.38700000000000001,"5min":0.28199999999999997,"15min":0.26000000000000001},"max":{"1min":0.80900000000000005,"5min":6.9740000000000002,"15min":9.4789999999999992},"last":0.55600000000000005},{"interface":"front","average":{"1min":0.64900000000000002,"5min":0.81699999999999995,"15min":1.133},"min":{"1min":0.41999999999999998,"5min":0.32000000000000001,"15min":0.26200000000000001},"max":{"1min":0.89700000000000002,"5min":6.7439999999999998,"15min":11.257},"last":0.66600000000000004}]},{"osd":6,"last update":"Sat Sep 23 21:21:23 2023","interfaces":[{"interface":"back","average":{"1min":0.63900000000000001,"5min":0.68100000000000005,"15min":1.0129999999999999},"min":{"1min":0.39500000000000002,"5min":0.312,"15min":0.30099999999999999},"max":{"1min":0.97199999999999998,"5min":5.7069999999999999,"15min":10.941000000000001},"last":0.72099999999999997},{"interface":"front","average":{"1min":0.69999999999999996,"5min":0.83299999999999996,"15min":1.1279999999999999},"min":{"1min":0.437,"5min":0.35099999999999998,"15min":0.33100000000000002},"max":{"1min":1,"5min":5.6760000000000002,"15min":12.715},"last":0.746}]}]},{"osd":6,"up_from":164,"seq":704374636857,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":280468,"kb_used_data":11596,"kb_used_omap":10,"kb_used_meta":268853,"kb_avail":93464684,"statfs":{"total":95995035648,"available":95707836416,"internally_reserved":0,"allocated":11874304,"data_stored":10107852,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":10328,"internal_metadata":275306408},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:21:26 2023","interfaces":[{"interface":"back","average":{"1min":0.54600000000000004,"5min":0.99299999999999999,"15min":0.93300000000000005},"min":{"1min":0.33900000000000002,"5min":0.20599999999999999,"15min":0.20599999999999999},"max":{"1min":0.82299999999999995,"5min":6.4539999999999997,"15min":6.4539999999999997},"last":0.57599999999999996},{"interface":"front","average":{"1min":0.54900000000000004,"5min":1.012,"15min":1.238},"min":{"1min":0.27600000000000002,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":0.70399999999999996,"5min":11.823,"15min":11.823},"last":0.46899999999999997}]},{"osd":1,"last update":"Sat Sep 23 21:21:19 2023","interfaces":[{"interface":"back","average":{"1min":0.54500000000000004,"5min":0.86699999999999999,"15min":1.4179999999999999},"min":{"1min":0.32400000000000001,"5min":0.28100000000000003,"15min":0.20300000000000001},"max":{"1min":0.79900000000000004,"5min":6.8490000000000002,"15min":12.904999999999999},"last":0.42299999999999999},{"interface":"front","average":{"1min":0.61599999999999999,"5min":0.78900000000000003,"15min":1.284},"min":{"1min":0.46999999999999997,"5min":0.26300000000000001,"15min":0.24099999999999999},"max":{"1min":0.85999999999999999,"5min":4.4000000000000004,"15min":8.9830000000000005},"last":0.60399999999999998}]},{"osd":2,"last update":"Sat Sep 23 21:21:26 2023","interfaces":[{"interface":"back","average":{"1min":0.56499999999999995,"5min":1.0660000000000001,"15min":1.6200000000000001},"min":{"1min":0.34000000000000002,"5min":0.27000000000000002,"15min":0.252},"max":{"1min":0.77700000000000002,"5min":7.117,"15min":23.954999999999998},"last":0.51900000000000002},{"interface":"front","average":{"1min":0.62,"5min":1.1910000000000001,"15min":1.554},"min":{"1min":0.36299999999999999,"5min":0.32000000000000001,"15min":0.251},"max":{"1min":0.82099999999999995,"5min":7.258,"15min":13.385999999999999},"last":0.62}]},{"osd":3,"last update":"Sat Sep 23 21:21:26 2023","interfaces":[{"interface":"back","average":{"1min":0.60499999999999998,"5min":1.7270000000000001,"15min":1.7909999999999999},"min":{"1min":0.40899999999999997,"5min":0.32500000000000001,"15min":0.28100000000000003},"max":{"1min":1.571,"5min":56.460000000000001,"15min":56.460000000000001},"last":0.48999999999999999},{"interface":"front","average":{"1min":0.626,"5min":1.651,"15min":1.944},"min":{"1min":0.45700000000000002,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":0.92000000000000004,"5min":56.494999999999997,"15min":56.494999999999997},"last":0.66700000000000004}]},{"osd":4,"last update":"Sat Sep 23 21:21:26 2023","interfaces":[{"interface":"back","average":{"1min":0.61299999999999999,"5min":0.65200000000000002,"15min":1.0329999999999999},"min":{"1min":0.34599999999999997,"5min":0.26500000000000001,"15min":0.22800000000000001},"max":{"1min":0.86399999999999999,"5min":4.1520000000000001,"15min":11.800000000000001},"last":0.50800000000000001},{"interface":"front","average":{"1min":0.63600000000000001,"5min":0.627,"15min":0.93700000000000006},"min":{"1min":0.31,"5min":0.27700000000000002,"15min":0.27400000000000002},"max":{"1min":1.0249999999999999,"5min":4.2530000000000001,"15min":13.920999999999999},"last":0.63900000000000001}]},{"osd":5,"last update":"Sat Sep 23 21:21:26 2023","interfaces":[{"interface":"back","average":{"1min":0.67200000000000004,"5min":0.71499999999999997,"15min":1.0680000000000001},"min":{"1min":0.43099999999999999,"5min":0.30299999999999999,"15min":0.27100000000000002},"max":{"1min":0.93799999999999994,"5min":4.2999999999999998,"15min":7.9710000000000001},"last":0.54100000000000004},{"interface":"front","average":{"1min":0.64800000000000002,"5min":0.74099999999999999,"15min":0.92700000000000005},"min":{"1min":0.36699999999999999,"5min":0.33900000000000002,"15min":0.30299999999999999},"max":{"1min":0.99099999999999999,"5min":5.2789999999999999,"15min":9.016},"last":0.69099999999999995}]},{"osd":7,"last update":"Sat Sep 23 21:21:26 2023","interfaces":[{"interface":"back","average":{"1min":0.67300000000000004,"5min":0.79400000000000004,"15min":1.8580000000000001},"min":{"1min":0.42399999999999999,"5min":0.32500000000000001,"15min":0.28199999999999997},"max":{"1min":0.89500000000000002,"5min":4.117,"15min":205.95599999999999},"last":0.55700000000000005},{"interface":"front","average":{"1min":0.65500000000000003,"5min":0.64800000000000002,"15min":1.8500000000000001},"min":{"1min":0.41599999999999998,"5min":0.32300000000000001,"15min":0.308},"max":{"1min":1.008,"5min":3.355,"15min":206.05699999999999},"last":0.78000000000000003}]}]},{"osd":1,"up_from":350,"seq":1503238553789,"num_pgs":42,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":369840,"kb_used_data":12648,"kb_used_omap":10,"kb_used_meta":357173,"kb_avail":93375312,"statfs":{"total":95995035648,"available":95616319488,"internally_reserved":0,"allocated":12951552,"data_stored":9946541,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":10498,"internal_metadata":365745918},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:20:30 2023","interfaces":[{"interface":"back","average":{"1min":0.39000000000000001,"5min":1.498,"15min":1.0429999999999999},"min":{"1min":0.23200000000000001,"5min":0.21199999999999999,"15min":0.21199999999999999},"max":{"1min":0.83399999999999996,"5min":13.323,"15min":33.429000000000002},"last":0.35199999999999998},{"interface":"front","average":{"1min":0.56799999999999995,"5min":1.1339999999999999,"15min":1.151},"min":{"1min":0.26500000000000001,"5min":0.23400000000000001,"15min":0.221},"max":{"1min":3.79,"5min":9.1890000000000001,"15min":34.432000000000002},"last":0.26100000000000001}]},{"osd":2,"last update":"Sat Sep 23 21:20:30 2023","interfaces":[{"interface":"back","average":{"1min":0.439,"5min":1.736,"15min":1.8899999999999999},"min":{"1min":0.28199999999999997,"5min":0.27700000000000002,"15min":0.19900000000000001},"max":{"1min":0.77400000000000002,"5min":13.368,"15min":34.387},"last":0.28100000000000003},{"interface":"front","average":{"1min":0.623,"5min":1.637,"15min":1.8879999999999999},"min":{"1min":0.27700000000000002,"5min":0.27700000000000002,"15min":0.214},"max":{"1min":3.8140000000000001,"5min":7.0460000000000003,"15min":38.573999999999998},"last":0.39600000000000002}]},{"osd":3,"last update":"Sat Sep 23 21:20:30 2023","interfaces":[{"interface":"back","average":{"1min":0.61699999999999999,"5min":1.337,"15min":1.8680000000000001},"min":{"1min":0.36899999999999999,"5min":0.28000000000000003,"15min":0.24099999999999999},"max":{"1min":2.8490000000000002,"5min":5.2389999999999999,"15min":17.055},"last":0.36899999999999999},{"interface":"front","average":{"1min":0.44500000000000001,"5min":1.4510000000000001,"15min":1.9470000000000001},"min":{"1min":0.318,"5min":0.25900000000000001,"15min":0.25600000000000001},"max":{"1min":0.72599999999999998,"5min":8.7590000000000003,"15min":15.237},"last":0.33200000000000002}]},{"osd":4,"last update":"Sat Sep 23 21:20:30 2023","interfaces":[{"interface":"back","average":{"1min":0.625,"5min":1.119,"15min":1.393},"min":{"1min":0.38400000000000001,"5min":0.27000000000000002,"15min":0.26300000000000001},"max":{"1min":1.577,"5min":13.250999999999999,"15min":13.477},"last":0.54300000000000004},{"interface":"front","average":{"1min":0.77600000000000002,"5min":1.069,"15min":1.5449999999999999},"min":{"1min":0.435,"5min":0.34599999999999997,"15min":0.313},"max":{"1min":4.173,"5min":5.1609999999999996,"15min":16.030000000000001},"last":0.50700000000000001}]},{"osd":5,"last update":"Sat Sep 23 21:20:30 2023","interfaces":[{"interface":"back","average":{"1min":0.56200000000000006,"5min":1.0269999999999999,"15min":1.6910000000000001},"min":{"1min":0.40999999999999998,"5min":0.36199999999999999,"15min":0.29799999999999999},"max":{"1min":0.84999999999999998,"5min":4.923,"15min":14.813000000000001},"last":0.52900000000000003},{"interface":"front","average":{"1min":0.59499999999999997,"5min":1.1539999999999999,"15min":1.585},"min":{"1min":0.41399999999999998,"5min":0.32600000000000001,"15min":0.32500000000000001},"max":{"1min":0.79600000000000004,"5min":13.182,"15min":13.182},"last":0.49399999999999999}]},{"osd":6,"last update":"Sat Sep 23 21:20:30 2023","interfaces":[{"interface":"back","average":{"1min":0.56399999999999995,"5min":1.117,"15min":1.6499999999999999},"min":{"1min":0.44700000000000001,"5min":0.34899999999999998,"15min":0.33900000000000002},"max":{"1min":0.80600000000000005,"5min":6.0010000000000003,"15min":14.859999999999999},"last":0.57299999999999995},{"interface":"front","average":{"1min":0.64400000000000002,"5min":1.244,"15min":1.591},"min":{"1min":0.48099999999999998,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":0.85999999999999999,"5min":13.287000000000001,"15min":16.494},"last":0.58399999999999996}]},{"osd":7,"last update":"Sat Sep 23 21:20:30 2023","interfaces":[{"interface":"back","average":{"1min":0.81200000000000006,"5min":1.593,"15min":1.806},"min":{"1min":0.44,"5min":0.38500000000000001,"15min":0.35299999999999998},"max":{"1min":3.948,"5min":46.856999999999999,"15min":46.856999999999999},"last":0.56000000000000005},{"interface":"front","average":{"1min":0.60399999999999998,"5min":1.732,"15min":2.0070000000000001},"min":{"1min":0.44600000000000001,"5min":0.40500000000000003,"15min":0.39700000000000002},"max":{"1min":0.88100000000000001,"5min":46.912999999999997,"15min":46.912999999999997},"last":0.59899999999999998}]}]},{"osd":0,"up_from":493,"seq":2117418877030,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":438500,"kb_used_data":8988,"kb_used_omap":36,"kb_used_meta":429467,"kb_avail":93306652,"statfs":{"total":95995035648,"available":95546011648,"internally_reserved":0,"allocated":9203712,"data_stored":6093432,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":37336,"internal_metadata":439774760},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 21:20:31 2023","interfaces":[{"interface":"back","average":{"1min":0.39200000000000002,"5min":1.1759999999999999,"15min":0.95899999999999996},"min":{"1min":0.255,"5min":0.214,"15min":0.214},"max":{"1min":0.59199999999999997,"5min":13.779999999999999,"15min":13.779999999999999},"last":0.34599999999999997},{"interface":"front","average":{"1min":0.36499999999999999,"5min":1.5600000000000001,"15min":1.0860000000000001},"min":{"1min":0.27500000000000002,"5min":0.23300000000000001,"15min":0.20799999999999999},"max":{"1min":0.54500000000000004,"5min":10.551,"15min":11.964},"last":0.40300000000000002}]},{"osd":2,"last update":"Sat Sep 23 21:20:31 2023","interfaces":[{"interface":"back","average":{"1min":0.41599999999999998,"5min":1.9870000000000001,"15min":1.165},"min":{"1min":0.29299999999999998,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":0.71699999999999997,"5min":35.68,"15min":35.68},"last":0.47099999999999997},{"interface":"front","average":{"1min":0.46500000000000002,"5min":2.6080000000000001,"15min":1.5600000000000001},"min":{"1min":0.307,"5min":0.28499999999999998,"15min":0.27900000000000003},"max":{"1min":0.84199999999999997,"5min":35.832999999999998,"15min":35.832999999999998},"last":0.42799999999999999}]},{"osd":3,"last update":"Sat Sep 23 21:20:31 2023","interfaces":[{"interface":"back","average":{"1min":0.40500000000000003,"5min":2.5070000000000001,"15min":1.482},"min":{"1min":0.28999999999999998,"5min":0.247,"15min":0.23400000000000001},"max":{"1min":0.50900000000000001,"5min":37.079999999999998,"15min":37.079999999999998},"last":0.45800000000000002},{"interface":"front","average":{"1min":0.42899999999999999,"5min":1.978,"15min":1.425},"min":{"1min":0.309,"5min":0.28899999999999998,"15min":0.26200000000000001},"max":{"1min":0.56100000000000005,"5min":32.935000000000002,"15min":32.935000000000002},"last":0.378}]},{"osd":4,"last update":"Sat Sep 23 21:20:31 2023","interfaces":[{"interface":"back","average":{"1min":0.58399999999999996,"5min":1.5600000000000001,"15min":1.2609999999999999},"min":{"1min":0.42799999999999999,"5min":0.34000000000000002,"15min":0.30199999999999999},"max":{"1min":0.73099999999999998,"5min":22.780000000000001,"15min":22.780000000000001},"last":0.70999999999999996},{"interface":"front","average":{"1min":0.59599999999999997,"5min":1.3240000000000001,"15min":1.1359999999999999},"min":{"1min":0.433,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":0.81799999999999995,"5min":22.652000000000001,"15min":22.652000000000001},"last":0.57599999999999996}]},{"osd":5,"last update":"Sat Sep 23 21:20:31 2023","interfaces":[{"interface":"back","average":{"1min":0.625,"5min":1.3380000000000001,"15min":1.2809999999999999},"min":{"1min":0.439,"5min":0.312,"15min":0.29999999999999999},"max":{"1min":0.80900000000000005,"5min":8.9329999999999998,"15min":8.9329999999999998},"last":0.745},{"interface":"front","average":{"1min":0.58099999999999996,"5min":1.5029999999999999,"15min":1.232},"min":{"1min":0.42399999999999999,"5min":0.377,"15min":0.313},"max":{"1min":0.81699999999999995,"5min":15.321999999999999,"15min":15.321999999999999},"last":0.621}]},{"osd":6,"last update":"Sat Sep 23 21:20:31 2023","interfaces":[{"interface":"back","average":{"1min":0.65400000000000003,"5min":1.226,"15min":1.2070000000000001},"min":{"1min":0.47899999999999998,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":0.876,"5min":6.9409999999999998,"15min":7.2050000000000001},"last":0.61099999999999999},{"interface":"front","average":{"1min":0.61199999999999999,"5min":1.3169999999999999,"15min":1.3280000000000001},"min":{"1min":0.48499999999999999,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":0.79600000000000004,"5min":8.8889999999999993,"15min":8.8889999999999993},"last":0.77400000000000002}]},{"osd":7,"last update":"Sat Sep 23 21:20:31 2023","interfaces":[{"interface":"back","average":{"1min":0.59499999999999997,"5min":1.4570000000000001,"15min":1.355},"min":{"1min":0.46000000000000002,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":0.79500000000000004,"5min":15.273999999999999,"15min":15.273999999999999},"last":0.58999999999999997},{"interface":"front","average":{"1min":0.68200000000000005,"5min":1.548,"15min":1.323},"min":{"1min":0.52600000000000002,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":0.89800000000000002,"5min":33.779000000000003,"15min":33.779000000000003},"last":0.75800000000000001}]}]},{"osd":2,"up_from":93,"seq":399431958902,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1044848,"kb_used_data":9000,"kb_used_omap":81,"kb_used_meta":1035758,"kb_avail":92700304,"statfs":{"total":95995035648,"available":94925111296,"internally_reserved":0,"allocated":9216000,"data_stored":6030839,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":83750,"internal_metadata":1060616410},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:20:29 2023","interfaces":[{"interface":"back","average":{"1min":0.52700000000000002,"5min":1.7809999999999999,"15min":1.2410000000000001},"min":{"1min":0.27500000000000002,"5min":0.187,"15min":0.187},"max":{"1min":3.2320000000000002,"5min":13.25,"15min":13.25},"last":0.44700000000000001},{"interface":"front","average":{"1min":0.42099999999999999,"5min":1.6000000000000001,"15min":1.71},"min":{"1min":0.25900000000000001,"5min":0.20200000000000001,"15min":0.20200000000000001},"max":{"1min":1.3580000000000001,"5min":9.8849999999999998,"15min":18.111999999999998},"last":0.29499999999999998}]},{"osd":1,"last update":"Sat Sep 23 21:20:29 2023","interfaces":[{"interface":"back","average":{"1min":0.56000000000000005,"5min":1.6759999999999999,"15min":2.0390000000000001},"min":{"1min":0.28699999999999998,"5min":0.25800000000000001,"15min":0.188},"max":{"1min":3.1920000000000002,"5min":10.535,"15min":34.396999999999998},"last":0.39700000000000002},{"interface":"front","average":{"1min":0.58999999999999997,"5min":1.532,"15min":1.972},"min":{"1min":0.30499999999999999,"5min":0.24099999999999999,"15min":0.217},"max":{"1min":3.911,"5min":8.5510000000000002,"15min":34.462000000000003},"last":0.33800000000000002}]},{"osd":3,"last update":"Sat Sep 23 21:21:10 2023","interfaces":[{"interface":"back","average":{"1min":0.36199999999999999,"5min":1.7849999999999999,"15min":1.9139999999999999},"min":{"1min":0.253,"5min":0.248,"15min":0.20599999999999999},"max":{"1min":0.496,"5min":20.106999999999999,"15min":35.549999999999997},"last":0.63100000000000001},{"interface":"front","average":{"1min":0.42199999999999999,"5min":2.1019999999999999,"15min":2.2240000000000002},"min":{"1min":0.34399999999999997,"5min":0.33700000000000002,"15min":0.21299999999999999},"max":{"1min":0.60299999999999998,"5min":20.239000000000001,"15min":36.951999999999998},"last":0.55000000000000004}]},{"osd":4,"last update":"Sat Sep 23 21:20:29 2023","interfaces":[{"interface":"back","average":{"1min":0.55800000000000005,"5min":1.171,"15min":1.365},"min":{"1min":0.39300000000000002,"5min":0.27000000000000002,"15min":0.249},"max":{"1min":0.78900000000000003,"5min":7.577,"15min":10.266999999999999},"last":0.56699999999999995},{"interface":"front","average":{"1min":0.72099999999999997,"5min":1.423,"15min":1.597},"min":{"1min":0.44400000000000001,"5min":0.30299999999999999,"15min":0.26700000000000002},"max":{"1min":3.1280000000000001,"5min":10.459,"15min":12.49},"last":0.69499999999999995}]},{"osd":5,"last update":"Sat Sep 23 21:20:29 2023","interfaces":[{"interface":"back","average":{"1min":0.55800000000000005,"5min":2.2810000000000001,"15min":1.7629999999999999},"min":{"1min":0.40899999999999997,"5min":0.29099999999999998,"15min":0.28000000000000003},"max":{"1min":0.74199999999999999,"5min":102.56100000000001,"15min":102.56100000000001},"last":0.66800000000000004},{"interface":"front","average":{"1min":0.73399999999999999,"5min":2.4409999999999998,"15min":1.8959999999999999},"min":{"1min":0.442,"5min":0.34899999999999998,"15min":0.29399999999999998},"max":{"1min":3.1629999999999998,"5min":102.28700000000001,"15min":102.28700000000001},"last":0.57799999999999996}]},{"osd":6,"last update":"Sat Sep 23 21:20:29 2023","interfaces":[{"interface":"back","average":{"1min":0.71999999999999997,"5min":1.591,"15min":1.7809999999999999},"min":{"1min":0.48999999999999999,"5min":0.377,"15min":0.35699999999999998},"max":{"1min":2.4649999999999999,"5min":10.335000000000001,"15min":12.518000000000001},"last":0.64800000000000002},{"interface":"front","average":{"1min":0.73999999999999999,"5min":1.5569999999999999,"15min":1.9099999999999999},"min":{"1min":0.48899999999999999,"5min":0.40200000000000002,"15min":0.35599999999999998},"max":{"1min":2.387,"5min":10.381,"15min":12.544},"last":0.59499999999999997}]},{"osd":7,"last update":"Sat Sep 23 21:20:29 2023","interfaces":[{"interface":"back","average":{"1min":0.66800000000000004,"5min":1.29,"15min":1.554},"min":{"1min":0.45700000000000002,"5min":0.31,"15min":0.29799999999999999},"max":{"1min":1.6870000000000001,"5min":7.5819999999999999,"15min":18.021999999999998},"last":0.61499999999999999},{"interface":"front","average":{"1min":0.67700000000000005,"5min":1.415,"15min":1.6299999999999999},"min":{"1min":0.42499999999999999,"5min":0.34999999999999998,"15min":0.32200000000000001},"max":{"1min":2.0489999999999999,"5min":7.46,"15min":10.92},"last":0.60199999999999998}]}]},{"osd":3,"up_from":98,"seq":420906795375,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1126488,"kb_used_data":8976,"kb_used_omap":7,"kb_used_meta":1117496,"kb_avail":92618664,"statfs":{"total":95995035648,"available":94841511936,"internally_reserved":0,"allocated":9191424,"data_stored":6087814,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7576,"internal_metadata":1144316520},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:21:23 2023","interfaces":[{"interface":"back","average":{"1min":0.40200000000000002,"5min":1.98,"15min":1.6779999999999999},"min":{"1min":0.25600000000000001,"5min":0.18099999999999999,"15min":0.18099999999999999},"max":{"1min":0.89300000000000002,"5min":81.408000000000001,"15min":147.38900000000001},"last":0.29899999999999999},{"interface":"front","average":{"1min":0.39500000000000002,"5min":1.532,"15min":1.6439999999999999},"min":{"1min":0.26400000000000001,"5min":0.214,"15min":0.214},"max":{"1min":0.54300000000000004,"5min":82.397000000000006,"15min":149.989},"last":0.318}]},{"osd":1,"last update":"Sat Sep 23 21:21:23 2023","interfaces":[{"interface":"back","average":{"1min":1.333,"5min":1.476,"15min":1.9199999999999999},"min":{"1min":0.252,"5min":0.23400000000000001,"15min":0.20000000000000001},"max":{"1min":19.132000000000001,"5min":19.132000000000001,"15min":22.577999999999999},"last":0.28399999999999997},{"interface":"front","average":{"1min":0.42799999999999999,"5min":0.98199999999999998,"15min":1.6719999999999999},"min":{"1min":0.30099999999999999,"5min":0.255,"15min":0.216},"max":{"1min":0.623,"5min":6.1219999999999999,"15min":21.617999999999999},"last":0.373}]},{"osd":2,"last update":"Sat Sep 23 21:21:06 2023","interfaces":[{"interface":"back","average":{"1min":0.73699999999999999,"5min":1.024,"15min":1.7629999999999999},"min":{"1min":0.28399999999999997,"5min":0.23799999999999999,"15min":0.20899999999999999},"max":{"1min":6.3170000000000002,"5min":11.321,"15min":56.317999999999998},"last":0.27000000000000002},{"interface":"front","average":{"1min":0.73599999999999999,"5min":2.0139999999999998,"15min":2.1659999999999999},"min":{"1min":0.371,"5min":0.23400000000000001,"15min":0.216},"max":{"1min":6.2910000000000004,"5min":17.423999999999999,"15min":55.988},"last":0.34300000000000003}]},{"osd":4,"last update":"Sat Sep 23 21:21:06 2023","interfaces":[{"interface":"back","average":{"1min":0.83399999999999996,"5min":0.85699999999999998,"15min":1.3819999999999999},"min":{"1min":0.42099999999999999,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":5.1580000000000004,"5min":10.475,"15min":21.478999999999999},"last":0.46200000000000002},{"interface":"front","average":{"1min":0.77000000000000002,"5min":1.28,"15min":1.597},"min":{"1min":0.40600000000000003,"5min":0.311,"15min":0.28499999999999998},"max":{"1min":4.8920000000000003,"5min":11.202,"15min":22.402000000000001},"last":0.40600000000000003}]},{"osd":5,"last update":"Sat Sep 23 21:21:06 2023","interfaces":[{"interface":"back","average":{"1min":0.83199999999999996,"5min":0.93899999999999995,"15min":1.4650000000000001},"min":{"1min":0.443,"5min":0.313,"15min":0.311},"max":{"1min":5.1959999999999997,"5min":10.516,"15min":21.562000000000001},"last":0.41799999999999998},{"interface":"front","average":{"1min":0.74099999999999999,"5min":1.016,"15min":1.522},"min":{"1min":0.44900000000000001,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":3.0899999999999999,"5min":5.7300000000000004,"15min":14.029},"last":0.54200000000000004}]},{"osd":6,"last update":"Sat Sep 23 21:21:23 2023","interfaces":[{"interface":"back","average":{"1min":1.5329999999999999,"5min":1.4119999999999999,"15min":1.762},"min":{"1min":0.44800000000000001,"5min":0.38100000000000001,"15min":0.34999999999999998},"max":{"1min":19.094999999999999,"5min":19.094999999999999,"15min":22.446999999999999},"last":0.53200000000000003},{"interface":"front","average":{"1min":0.65600000000000003,"5min":1.0229999999999999,"15min":1.575},"min":{"1min":0.46800000000000003,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":0.81399999999999995,"5min":7.5949999999999998,"15min":12.375999999999999},"last":0.56899999999999995}]},{"osd":7,"last update":"Sat Sep 23 21:21:06 2023","interfaces":[{"interface":"back","average":{"1min":0.86499999999999999,"5min":1.0149999999999999,"15min":1.5960000000000001},"min":{"1min":0.47299999999999998,"5min":0.41099999999999998,"15min":0.33900000000000002},"max":{"1min":5.1120000000000001,"5min":10.547000000000001,"15min":21.518999999999998},"last":0.44700000000000001},{"interface":"front","average":{"1min":0.78700000000000003,"5min":1.143,"15min":1.706},"min":{"1min":0.50700000000000001,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":3.145,"5min":5.5119999999999996,"15min":9.9879999999999995},"last":0.50700000000000001}]}]},{"osd":4,"up_from":31,"seq":133143986682,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":596128,"kb_used_data":7192,"kb_used_omap":0,"kb_used_meta":588928,"kb_avail":93149024,"statfs":{"total":95995035648,"available":95384600576,"internally_reserved":0,"allocated":7364608,"data_stored":5586513,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":603062272},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:21:26 2023","interfaces":[{"interface":"back","average":{"1min":0.51200000000000001,"5min":1.3320000000000001,"15min":1.103},"min":{"1min":0.39500000000000002,"5min":0.23699999999999999,"15min":0.22500000000000001},"max":{"1min":0.70899999999999996,"5min":28.683,"15min":28.683},"last":0.58599999999999997},{"interface":"front","average":{"1min":0.54200000000000004,"5min":1.647,"15min":1.2829999999999999},"min":{"1min":0.36399999999999999,"5min":0.249,"15min":0.23799999999999999},"max":{"1min":0.88200000000000001,"5min":29.155000000000001,"15min":29.155000000000001},"last":0.60499999999999998}]},{"osd":1,"last update":"Sat Sep 23 21:21:26 2023","interfaces":[{"interface":"back","average":{"1min":0.59399999999999997,"5min":1.0820000000000001,"15min":1.4670000000000001},"min":{"1min":0.40400000000000003,"5min":0.28499999999999998,"15min":0.221},"max":{"1min":0.86799999999999999,"5min":8.1020000000000003,"15min":37.960000000000001},"last":0.63900000000000001},{"interface":"front","average":{"1min":0.60799999999999998,"5min":1.034,"15min":1.4630000000000001},"min":{"1min":0.36799999999999999,"5min":0.33100000000000002,"15min":0.27000000000000002},"max":{"1min":0.92000000000000004,"5min":8.1509999999999998,"15min":37.933},"last":0.55100000000000005}]},{"osd":2,"last update":"Sat Sep 23 21:21:26 2023","interfaces":[{"interface":"back","average":{"1min":0.59099999999999997,"5min":1.5409999999999999,"15min":1.6120000000000001},"min":{"1min":0.42099999999999999,"5min":0.28399999999999997,"15min":0.247},"max":{"1min":0.77300000000000002,"5min":17.602,"15min":17.602},"last":0.624},{"interface":"front","average":{"1min":0.56499999999999995,"5min":1.29,"15min":1.7030000000000001},"min":{"1min":0.38100000000000001,"5min":0.28799999999999998,"15min":0.27800000000000002},"max":{"1min":0.80300000000000005,"5min":13.439,"15min":14.417999999999999},"last":0.78000000000000003}]},{"osd":3,"last update":"Sat Sep 23 21:21:01 2023","interfaces":[{"interface":"back","average":{"1min":0.61099999999999999,"5min":1.887,"15min":1.8220000000000001},"min":{"1min":0.37,"5min":0.29499999999999998,"15min":0.25600000000000001},"max":{"1min":0.79400000000000004,"5min":67.105999999999995,"15min":67.105999999999995},"last":0.49199999999999999},{"interface":"front","average":{"1min":0.70599999999999996,"5min":1.865,"15min":1.8819999999999999},"min":{"1min":0.46600000000000003,"5min":0.33600000000000002,"15min":0.32200000000000001},"max":{"1min":0.92100000000000004,"5min":67.046999999999997,"15min":67.046999999999997},"last":0.66700000000000004}]},{"osd":5,"last update":"Sat Sep 23 21:21:20 2023","interfaces":[{"interface":"back","average":{"1min":0.56799999999999995,"5min":0.61599999999999999,"15min":1.0369999999999999},"min":{"1min":0.39000000000000001,"5min":0.254,"15min":0.222},"max":{"1min":0.90100000000000002,"5min":2.923,"15min":14.398999999999999},"last":0.70999999999999996},{"interface":"front","average":{"1min":0.54300000000000004,"5min":0.59799999999999998,"15min":0.94199999999999995},"min":{"1min":0.35299999999999998,"5min":0.25800000000000001,"15min":0.255},"max":{"1min":0.68999999999999995,"5min":3.6000000000000001,"15min":9.8979999999999997},"last":0.51500000000000001}]},{"osd":6,"last update":"Sat Sep 23 21:21:26 2023","interfaces":[{"interface":"back","average":{"1min":0.57799999999999996,"5min":0.70099999999999996,"15min":1.0609999999999999},"min":{"1min":0.36799999999999999,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":0.82999999999999996,"5min":4.7270000000000003,"15min":24.452999999999999},"last":0.57199999999999995},{"interface":"front","average":{"1min":0.71099999999999997,"5min":0.69599999999999995,"15min":1.02},"min":{"1min":0.48799999999999999,"5min":0.36399999999999999,"15min":0.32500000000000001},"max":{"1min":0.94699999999999995,"5min":2.7069999999999999,"15min":14.387},"last":0.81599999999999995}]},{"osd":7,"last update":"Sat Sep 23 21:21:01 2023","interfaces":[{"interface":"back","average":{"1min":0.66300000000000003,"5min":0.745,"15min":1.1080000000000001},"min":{"1min":0.376,"5min":0.33000000000000002,"15min":0.30099999999999999},"max":{"1min":0.998,"5min":5.7789999999999999,"15min":16.890999999999998},"last":0.74399999999999999},{"interface":"front","average":{"1min":0.621,"5min":0.624,"15min":1.002},"min":{"1min":0.38900000000000001,"5min":0.29199999999999998,"15min":0.27400000000000002},"max":{"1min":0.86099999999999999,"5min":3.681,"15min":8.5830000000000002},"last":0.65100000000000002}]}]},{"osd":5,"up_from":39,"seq":167503725048,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":622872,"kb_used_data":11536,"kb_used_omap":0,"kb_used_meta":611328,"kb_avail":93122280,"statfs":{"total":95995035648,"available":95357214720,"internally_reserved":0,"allocated":11812864,"data_stored":10039268,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":625999872},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:21:27 2023","interfaces":[{"interface":"back","average":{"1min":0.55400000000000005,"5min":1.653,"15min":1.1000000000000001},"min":{"1min":0.38800000000000001,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":0.72499999999999998,"5min":88.272999999999996,"15min":88.272999999999996},"last":0.46100000000000002},{"interface":"front","average":{"1min":0.54100000000000004,"5min":1.8260000000000001,"15min":1.179},"min":{"1min":0.40500000000000003,"5min":0.215,"15min":0.215},"max":{"1min":0.73499999999999999,"5min":88.296999999999997,"15min":88.296999999999997},"last":0.505}]},{"osd":1,"last update":"Sat Sep 23 21:21:27 2023","interfaces":[{"interface":"back","average":{"1min":0.58999999999999997,"5min":1.1100000000000001,"15min":1.4139999999999999},"min":{"1min":0.42599999999999999,"5min":0.27900000000000003,"15min":0.23499999999999999},"max":{"1min":0.80200000000000005,"5min":6.4160000000000004,"15min":25.239999999999998},"last":0.63100000000000001},{"interface":"front","average":{"1min":0.63600000000000001,"5min":1.0069999999999999,"15min":1.5449999999999999},"min":{"1min":0.47299999999999998,"5min":0.32900000000000001,"15min":0.252},"max":{"1min":0.86699999999999999,"5min":6.3579999999999997,"15min":25.253},"last":0.60099999999999998}]},{"osd":2,"last update":"Sat Sep 23 21:20:37 2023","interfaces":[{"interface":"back","average":{"1min":0.505,"5min":1.3839999999999999,"15min":1.53},"min":{"1min":0.35799999999999998,"5min":0.27300000000000002,"15min":0.25800000000000001},"max":{"1min":0.64300000000000002,"5min":9.6560000000000006,"15min":12.634},"last":0.61299999999999999},{"interface":"front","average":{"1min":0.54000000000000004,"5min":1.4410000000000001,"15min":1.48},"min":{"1min":0.28899999999999998,"5min":0.28899999999999998,"15min":0.25700000000000001},"max":{"1min":0.69199999999999995,"5min":9.4060000000000006,"15min":9.7170000000000005},"last":0.56200000000000006}]},{"osd":3,"last update":"Sat Sep 23 21:20:59 2023","interfaces":[{"interface":"back","average":{"1min":0.66300000000000003,"5min":1.256,"15min":1.71},"min":{"1min":0.41599999999999998,"5min":0.314,"15min":0.29299999999999998},"max":{"1min":1.3009999999999999,"5min":7.3700000000000001,"15min":25.503},"last":0.65400000000000003},{"interface":"front","average":{"1min":0.66000000000000003,"5min":1.544,"15min":1.8959999999999999},"min":{"1min":0.46000000000000002,"5min":0.311,"15min":0.30599999999999999},"max":{"1min":1.6599999999999999,"5min":12.382,"15min":25.466000000000001},"last":0.67400000000000004}]},{"osd":4,"last update":"Sat Sep 23 21:21:18 2023","interfaces":[{"interface":"back","average":{"1min":0.58299999999999996,"5min":0.66700000000000004,"15min":1.0409999999999999},"min":{"1min":0.35199999999999998,"5min":0.26100000000000001,"15min":0.253},"max":{"1min":0.85799999999999998,"5min":5.0279999999999996,"15min":12.776},"last":0.70199999999999996},{"interface":"front","average":{"1min":0.60499999999999998,"5min":0.68100000000000005,"15min":1.077},"min":{"1min":0.41199999999999998,"5min":0.246,"15min":0.246},"max":{"1min":0.90000000000000002,"5min":5.149,"15min":11.973000000000001},"last":0.80700000000000005}]},{"osd":6,"last update":"Sat Sep 23 21:21:27 2023","interfaces":[{"interface":"back","average":{"1min":0.61299999999999999,"5min":0.85099999999999998,"15min":0.96199999999999997},"min":{"1min":0.35999999999999999,"5min":0.27400000000000002,"15min":0.25900000000000001},"max":{"1min":0.86899999999999999,"5min":5.8520000000000003,"15min":8.0909999999999993},"last":0.77800000000000002},{"interface":"front","average":{"1min":0.69299999999999995,"5min":0.84599999999999997,"15min":1.1319999999999999},"min":{"1min":0.40000000000000002,"5min":0.36399999999999999,"15min":0.317},"max":{"1min":0.94599999999999995,"5min":7.1470000000000002,"15min":10.999000000000001},"last":0.73799999999999999}]},{"osd":7,"last update":"Sat Sep 23 21:20:59 2023","interfaces":[{"interface":"back","average":{"1min":0.58799999999999997,"5min":0.79600000000000004,"15min":1.756},"min":{"1min":0.32900000000000001,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":0.92900000000000005,"5min":4.2629999999999999,"15min":189.405},"last":0.83899999999999997},{"interface":"front","average":{"1min":0.64500000000000002,"5min":0.80900000000000005,"15min":1.9219999999999999},"min":{"1min":0.34499999999999997,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":0.91700000000000004,"5min":5.0910000000000002,"15min":189.685},"last":0.82799999999999996}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8488,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1259,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T21:21:34.245 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T21:21:34.854 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:21:35.100 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:21:34 smithi026 ceph-mon[179688]: pgmap v1721: 103 pgs: 103 active+clean; 4.6 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail 2023-09-23T21:21:35.100 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:21:34 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:21:35.100 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:21:34 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:21:35.100 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:21:34 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T21:21:35.101 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:21:34 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T21:21:35.101 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:21:34 smithi026 ceph-mon[179688]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:21:35.101 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:21:34 smithi026 ceph-mon[179688]: from='client.38043 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T21:21:35.101 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:21:34 smithi026 ceph-mon[175151]: pgmap v1721: 103 pgs: 103 active+clean; 4.6 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail 2023-09-23T21:21:35.101 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:21:34 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:21:35.101 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:21:34 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:21:35.101 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:21:34 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T21:21:35.101 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:21:34 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T21:21:35.101 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:21:34 smithi026 ceph-mon[175151]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:21:35.101 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:21:34 smithi026 ceph-mon[175151]: from='client.38043 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T21:21:35.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:21:34 smithi070 ceph-mon[168282]: pgmap v1721: 103 pgs: 103 active+clean; 4.6 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail 2023-09-23T21:21:35.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:21:34 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:21:35.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:21:34 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:21:35.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:21:34 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T21:21:35.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:21:34 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T21:21:35.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:21:34 smithi070 ceph-mon[168282]: from='mgr.25142 172.21.15.26:0/4169426973' entity='mgr.y' 2023-09-23T21:21:35.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:21:34 smithi070 ceph-mon[168282]: from='client.38043 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T21:21:36.404 INFO:teuthology.orchestra.run.smithi026.stdout: 2023-09-23T21:21:36.406 INFO:teuthology.orchestra.run.smithi026.stderr:dumped all 2023-09-23T21:21:36.701 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:21:36 smithi026 ceph-mon[175151]: pgmap v1722: 103 pgs: 103 active+clean; 4.6 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail 2023-09-23T21:21:36.815 INFO:teuthology.orchestra.run.smithi026.stdout:{"pg_ready":true,"pg_map":{"version":1722,"stamp":"2023-09-23T21:21:35.245181+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4790195,"num_objects":28,"num_object_clones":0,"num_object_copies":84,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":28,"num_whiteouts":0,"num_read":5995,"num_read_kb":13212,"num_write":907,"num_write_kb":12725,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":162,"num_bytes_recovered":9266624,"num_keys_recovered":20,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6022,"ondisk_log_size":6022,"up":309,"acting":309,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":309,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":5144844,"kb_used_data":77388,"kb_used_omap":145,"kb_used_meta":5067246,"kb_avail":744816372,"statfs":{"total":767960285184,"available":762691964928,"internally_reserved":0,"allocated":79245312,"data_stored":59741207,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":149488,"internal_metadata":5188859920},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.003247"},"pg_stats":[{"pgid":"1.18","version":"79'161","reported_seq":775,"reported_epoch":708,"state":"active+clean","last_fresh":"2023-09-23T21:21:17.507478+0000","last_change":"2023-09-23T21:20:19.087341+0000","last_active":"2023-09-23T21:21:17.507478+0000","last_peered":"2023-09-23T21:21:17.507478+0000","last_clean":"2023-09-23T21:21:17.507478+0000","last_became_active":"2023-09-23T21:20:19.086834+0000","last_became_peered":"2023-09-23T21:20:19.086834+0000","last_unstale":"2023-09-23T21:21:17.507478+0000","last_undegraded":"2023-09-23T21:21:17.507478+0000","last_fullsized":"2023-09-23T21:21:17.507478+0000","mapping_epoch":684,"log_start":"0'0","ondisk_log_start":"0'0","created":479,"last_epoch_clean":685,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:19:34.961756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:19:34.961756+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:04:09.454020+0000","scrub_duration":0.002022102,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":525376,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1a","version":"68'1","reported_seq":309,"reported_epoch":708,"state":"active+clean","last_fresh":"2023-09-23T21:21:17.505503+0000","last_change":"2023-09-23T21:20:12.080378+0000","last_active":"2023-09-23T21:21:17.505503+0000","last_peered":"2023-09-23T21:21:17.505503+0000","last_clean":"2023-09-23T21:21:17.505503+0000","last_became_active":"2023-09-23T21:20:12.080067+0000","last_became_peered":"2023-09-23T21:20:12.080067+0000","last_unstale":"2023-09-23T21:21:17.505503+0000","last_undegraded":"2023-09-23T21:21:17.505503+0000","last_fullsized":"2023-09-23T21:21:17.505503+0000","mapping_epoch":677,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":678,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:41:56.535018+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":467,"reported_epoch":709,"state":"active+clean","last_fresh":"2023-09-23T21:21:18.513696+0000","last_change":"2023-09-23T21:21:18.513696+0000","last_active":"2023-09-23T21:21:18.513696+0000","last_peered":"2023-09-23T21:21:18.513696+0000","last_clean":"2023-09-23T21:21:18.513696+0000","last_became_active":"2023-09-23T21:21:18.513311+0000","last_became_peered":"2023-09-23T21:21:18.513311+0000","last_unstale":"2023-09-23T21:21:18.513696+0000","last_undegraded":"2023-09-23T21:21:18.513696+0000","last_fullsized":"2023-09-23T21:21:18.513696+0000","mapping_epoch":708,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":709,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:50:03.921453+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1b","version":"79'161","reported_seq":590,"reported_epoch":708,"state":"active+clean","last_fresh":"2023-09-23T21:21:17.506840+0000","last_change":"2023-09-23T21:20:23.452618+0000","last_active":"2023-09-23T21:21:17.506840+0000","last_peered":"2023-09-23T21:21:17.506840+0000","last_clean":"2023-09-23T21:21:17.506840+0000","last_became_active":"2023-09-23T21:20:19.092143+0000","last_became_peered":"2023-09-23T21:20:19.092143+0000","last_unstale":"2023-09-23T21:21:17.506840+0000","last_undegraded":"2023-09-23T21:21:17.506840+0000","last_fullsized":"2023-09-23T21:21:17.506840+0000","mapping_epoch":684,"log_start":"0'0","ondisk_log_start":"0'0","created":479,"last_epoch_clean":685,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:20:23.452497+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:20:23.452497+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:36:41.779187+0000","scrub_duration":0.0025756709999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":42,"reported_epoch":708,"state":"active+clean","last_fresh":"2023-09-23T21:21:17.502526+0000","last_change":"2023-09-23T21:20:12.115671+0000","last_active":"2023-09-23T21:21:17.502526+0000","last_peered":"2023-09-23T21:21:17.502526+0000","last_clean":"2023-09-23T21:21:17.502526+0000","last_became_active":"2023-09-23T21:20:12.115349+0000","last_became_peered":"2023-09-23T21:20:12.115349+0000","last_unstale":"2023-09-23T21:21:17.502526+0000","last_undegraded":"2023-09-23T21:21:17.502526+0000","last_fullsized":"2023-09-23T21:21:17.502526+0000","mapping_epoch":677,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":678,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:01:43.499564+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":727,"reported_epoch":708,"state":"active+clean","last_fresh":"2023-09-23T21:21:17.902200+0000","last_change":"2023-09-23T21:16:42.441896+0000","last_active":"2023-09-23T21:21:17.902200+0000","last_peered":"2023-09-23T21:21:17.902200+0000","last_clean":"2023-09-23T21:21:17.902200+0000","last_became_active":"2023-09-23T21:16:42.441526+0000","last_became_peered":"2023-09-23T21:16:42.441526+0000","last_unstale":"2023-09-23T21:21:17.902200+0000","last_undegraded":"2023-09-23T21:21:17.902200+0000","last_fullsized":"2023-09-23T21:21:17.902200+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:27:36.326225+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1a","version":"79'161","reported_seq":846,"reported_epoch":708,"state":"active+clean","last_fresh":"2023-09-23T21:21:17.505337+0000","last_change":"2023-09-23T21:20:15.080123+0000","last_active":"2023-09-23T21:21:17.505337+0000","last_peered":"2023-09-23T21:21:17.505337+0000","last_clean":"2023-09-23T21:21:17.505337+0000","last_became_active":"2023-09-23T21:20:15.079745+0000","last_became_peered":"2023-09-23T21:20:15.079745+0000","last_unstale":"2023-09-23T21:21:17.505337+0000","last_undegraded":"2023-09-23T21:21:17.505337+0000","last_fullsized":"2023-09-23T21:21:17.505337+0000","mapping_epoch":680,"log_start":"0'0","ondisk_log_start":"0'0","created":479,"last_epoch_clean":681,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:15:45.158107+0000","last_deep_scrub":"79'161","last_deep_scrub_stamp":"2023-09-23T21:05:32.630292+0000","last_clean_scrub_stamp":"2023-09-23T21:15:45.158107+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:19:56.996183+0000","scrub_duration":0.001999373,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":465,"reported_epoch":708,"state":"active+clean","last_fresh":"2023-09-23T21:21:17.505400+0000","last_change":"2023-09-23T21:20:30.312213+0000","last_active":"2023-09-23T21:21:17.505400+0000","last_peered":"2023-09-23T21:21:17.505400+0000","last_clean":"2023-09-23T21:21:17.505400+0000","last_became_active":"2023-09-23T21:20:30.311897+0000","last_became_peered":"2023-09-23T21:20:30.311897+0000","last_unstale":"2023-09-23T21:21:17.505400+0000","last_undegraded":"2023-09-23T21:21:17.505400+0000","last_fullsized":"2023-09-23T21:21:17.505400+0000","mapping_epoch":689,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":690,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:25:54.138000+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":66,"reported_epoch":708,"state":"active+clean","last_fresh":"2023-09-23T21:21:17.508706+0000","last_change":"2023-09-23T21:20:15.934532+0000","last_active":"2023-09-23T21:21:17.508706+0000","last_peered":"2023-09-23T21:21:17.508706+0000","last_clean":"2023-09-23T21:21:17.508706+0000","last_became_active":"2023-09-23T21:20:15.934179+0000","last_became_peered":"2023-09-23T21:20:15.934179+0000","last_unstale":"2023-09-23T21:21:17.508706+0000","last_undegraded":"2023-09-23T21:21:17.508706+0000","last_fullsized":"2023-09-23T21:21:17.508706+0000","mapping_epoch":681,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":682,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:52:50.286017+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.15","version":"79'161","reported_seq":719,"reported_epoch":708,"state":"active+clean","last_fresh":"2023-09-23T21:21:17.506412+0000","last_change":"2023-09-23T21:20:19.089275+0000","last_active":"2023-09-23T21:21:17.506412+0000","last_peered":"2023-09-23T21:21:17.506412+0000","last_clean":"2023-09-23T21:21:17.506412+0000","last_became_active":"2023-09-23T21:20:19.088773+0000","last_became_peered":"2023-09-23T21:20:19.088773+0000","last_unstale":"2023-09-23T21:21:17.506412+0000","last_undegraded":"2023-09-23T21:21:17.506412+0000","last_fullsized":"2023-09-23T21:21:17.506412+0000","mapping_epoch":684,"log_start":"0'0","ondisk_log_start":"0'0","created":479,"last_epoch_clean":685,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:18:35.622422+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:18:35.622422+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:55:40.821855+0000","scrub_duration":0.083031242000000005,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,1],"acting":[7,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":713,"reported_epoch":708,"state":"active+clean","last_fresh":"2023-09-23T21:21:17.901931+0000","last_change":"2023-09-23T21:20:15.934405+0000","last_active":"2023-09-23T21:21:17.901931+0000","last_peered":"2023-09-23T21:21:17.901931+0000","last_clean":"2023-09-23T21:21:17.901931+0000","last_became_active":"2023-09-23T21:20:15.934102+0000","last_became_peered":"2023-09-23T21:20:15.934102+0000","last_unstale":"2023-09-23T21:21:17.901931+0000","last_undegraded":"2023-09-23T21:21:17.901931+0000","last_fullsized":"2023-09-23T21:21:17.901931+0000","mapping_epoch":681,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":682,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:11:01.173661+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":777,"reported_epoch":709,"state":"active+clean","last_fresh":"2023-09-23T21:21:18.506184+0000","last_change":"2023-09-23T21:20:50.927053+0000","last_active":"2023-09-23T21:21:18.506184+0000","last_peered":"2023-09-23T21:21:18.506184+0000","last_clean":"2023-09-23T21:21:18.506184+0000","last_became_active":"2023-09-23T21:20:50.926718+0000","last_became_peered":"2023-09-23T21:20:50.926718+0000","last_unstale":"2023-09-23T21:21:18.506184+0000","last_undegraded":"2023-09-23T21:21:18.506184+0000","last_fullsized":"2023-09-23T21:21:18.506184+0000","mapping_epoch":699,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":700,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:06:15.619866+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"79'161","reported_seq":1107,"reported_epoch":708,"state":"active+clean","last_fresh":"2023-09-23T21:21:17.508505+0000","last_change":"2023-09-23T21:16:45.424773+0000","last_active":"2023-09-23T21:21:17.508505+0000","last_peered":"2023-09-23T21:21:17.508505+0000","last_clean":"2023-09-23T21:21:17.508505+0000","last_became_active":"2023-09-23T21:16:45.424458+0000","last_became_peered":"2023-09-23T21:16:45.424458+0000","last_unstale":"2023-09-23T21:21:17.508505+0000","last_undegraded":"2023-09-23T21:21:17.508505+0000","last_fullsized":"2023-09-23T21:21:17.508505+0000","mapping_epoch":567,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":568,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:25.096401+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:25.096401+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:40:17.871561+0000","scrub_duration":0.002859393,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":37,"reported_epoch":708,"state":"active+clean","last_fresh":"2023-09-23T21:21:17.507984+0000","last_change":"2023-09-23T21:20:15.933213+0000","last_active":"2023-09-23T21:21:17.507984+0000","last_peered":"2023-09-23T21:21:17.507984+0000","last_clean":"2023-09-23T21:21:17.507984+0000","last_became_active":"2023-09-23T21:20:15.932952+0000","last_became_peered":"2023-09-23T21:20:15.932952+0000","last_unstale":"2023-09-23T21:21:17.507984+0000","last_undegraded":"2023-09-23T21:21:17.507984+0000","last_fullsized":"2023-09-23T21:21:17.507984+0000","mapping_epoch":681,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":682,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:33:08.774578+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":723,"reported_epoch":709,"state":"active+clean","last_fresh":"2023-09-23T21:21:18.506434+0000","last_change":"2023-09-23T21:20:05.021571+0000","last_active":"2023-09-23T21:21:18.506434+0000","last_peered":"2023-09-23T21:21:18.506434+0000","last_clean":"2023-09-23T21:21:18.506434+0000","last_became_active":"2023-09-23T21:20:05.021192+0000","last_became_peered":"2023-09-23T21:20:05.021192+0000","last_unstale":"2023-09-23T21:21:18.506434+0000","last_undegraded":"2023-09-23T21:21:18.506434+0000","last_fullsized":"2023-09-23T21:21:18.506434+0000","mapping_epoch":672,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":673,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:30:55.557751+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.17","version":"79'161","reported_seq":893,"reported_epoch":708,"state":"active+clean","last_fresh":"2023-09-23T21:21:17.508181+0000","last_change":"2023-09-23T21:20:42.355872+0000","last_active":"2023-09-23T21:21:17.508181+0000","last_peered":"2023-09-23T21:21:17.508181+0000","last_clean":"2023-09-23T21:21:17.508181+0000","last_became_active":"2023-09-23T21:20:42.355550+0000","last_became_peered":"2023-09-23T21:20:42.355550+0000","last_unstale":"2023-09-23T21:21:17.508181+0000","last_undegraded":"2023-09-23T21:21:17.508181+0000","last_fullsized":"2023-09-23T21:21:17.508181+0000","mapping_epoch":694,"log_start":"0'0","ondisk_log_start":"0'0","created":479,"last_epoch_clean":695,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:15:50.325741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:15:50.325741+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:13:11.438795+0000","scrub_duration":0.0040500149999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,6],"acting":[5,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":489,"reported_epoch":708,"state":"active+clean","last_fresh":"2023-09-23T21:21:17.506132+0000","last_change":"2023-09-23T21:20:48.511983+0000","last_active":"2023-09-23T21:21:17.506132+0000","last_peered":"2023-09-23T21:21:17.506132+0000","last_clean":"2023-09-23T21:21:17.506132+0000","last_became_active":"2023-09-23T21:20:48.511621+0000","last_became_peered":"2023-09-23T21:20:48.511621+0000","last_unstale":"2023-09-23T21:21:17.506132+0000","last_undegraded":"2023-09-23T21:21:17.506132+0000","last_fullsized":"2023-09-23T21:21:17.506132+0000","mapping_epoch":697,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":698,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:46:22.322452+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":450,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":10,"reported_epoch":709,"state":"active+clean","last_fresh":"2023-09-23T21:21:18.514367+0000","last_change":"2023-09-23T21:21:18.514367+0000","last_active":"2023-09-23T21:21:18.514367+0000","last_peered":"2023-09-23T21:21:18.514367+0000","last_clean":"2023-09-23T21:21:18.514367+0000","last_became_active":"2023-09-23T21:21:18.514026+0000","last_became_peered":"2023-09-23T21:21:18.514026+0000","last_unstale":"2023-09-23T21:21:18.514367+0000","last_undegraded":"2023-09-23T21:21:18.514367+0000","last_fullsized":"2023-09-23T21:21:18.514367+0000","mapping_epoch":708,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":709,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:27:27.146957+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,2],"acting":[5,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.16","version":"79'161","reported_seq":650,"reported_epoch":708,"state":"active+clean","last_fresh":"2023-09-23T21:21:17.504146+0000","last_change":"2023-09-23T21:20:15.083112+0000","last_active":"2023-09-23T21:21:17.504146+0000","last_peered":"2023-09-23T21:21:17.504146+0000","last_clean":"2023-09-23T21:21:17.504146+0000","last_became_active":"2023-09-23T21:20:15.082692+0000","last_became_peered":"2023-09-23T21:20:15.082692+0000","last_unstale":"2023-09-23T21:21:17.504146+0000","last_undegraded":"2023-09-23T21:21:17.504146+0000","last_fullsized":"2023-09-23T21:21:17.504146+0000","mapping_epoch":680,"log_start":"0'0","ondisk_log_start":"0'0","created":479,"last_epoch_clean":681,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:15:46.326589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:15:46.326589+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:15:50.608532+0000","scrub_duration":0.0062069059999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"79'22","reported_seq":332,"reported_epoch":708,"state":"active+clean","last_fresh":"2023-09-23T21:21:17.505778+0000","last_change":"2023-09-23T21:20:05.024837+0000","last_active":"2023-09-23T21:21:17.505778+0000","last_peered":"2023-09-23T21:21:17.505778+0000","last_clean":"2023-09-23T21:21:17.505778+0000","last_became_active":"2023-09-23T21:20:05.018268+0000","last_became_peered":"2023-09-23T21:20:05.018268+0000","last_unstale":"2023-09-23T21:21:17.505778+0000","last_undegraded":"2023-09-23T21:21:17.505778+0000","last_fullsized":"2023-09-23T21:21:17.505778+0000","mapping_epoch":672,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":673,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:16:08.334109+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":18936,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":667,"reported_epoch":708,"state":"active+clean","last_fresh":"2023-09-23T21:21:17.505728+0000","last_change":"2023-09-23T21:20:12.078453+0000","last_active":"2023-09-23T21:21:17.505728+0000","last_peered":"2023-09-23T21:21:17.505728+0000","last_clean":"2023-09-23T21:21:17.505728+0000","last_became_active":"2023-09-23T21:20:12.078115+0000","last_became_peered":"2023-09-23T21:20:12.078115+0000","last_unstale":"2023-09-23T21:21:17.505728+0000","last_undegraded":"2023-09-23T21:21:17.505728+0000","last_fullsized":"2023-09-23T21:21:17.505728+0000","mapping_epoch":677,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":678,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:04:11.282079+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.11","version":"79'161","reported_seq":690,"reported_epoch":708,"state":"active+clean","last_fresh":"2023-09-23T21:21:17.901445+0000","last_change":"2023-09-23T21:20:19.087016+0000","last_active":"2023-09-23T21:21:17.901445+0000","last_peered":"2023-09-23T21:21:17.901445+0000","last_clean":"2023-09-23T21:21:17.901445+0000","last_became_active":"2023-09-23T21:20:19.086608+0000","last_became_peered":"2023-09-23T21:20:19.086608+0000","last_unstale":"2023-09-23T21:21:17.901445+0000","last_undegraded":"2023-09-23T21:21:17.901445+0000","last_fullsized":"2023-09-23T21:21:17.901445+0000","mapping_epoch":684,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":685,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:15:47.299947+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:15:47.299947+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:21:19.868572+0000","scrub_duration":0.0061353110000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":160,"reported_epoch":708,"state":"active+clean","last_fresh":"2023-09-23T21:21:17.505822+0000","last_change":"2023-09-23T21:16:42.531060+0000","last_active":"2023-09-23T21:21:17.505822+0000","last_peered":"2023-09-23T21:21:17.505822+0000","last_clean":"2023-09-23T21:21:17.505822+0000","last_became_active":"2023-09-23T21:16:42.449198+0000","last_became_peered":"2023-09-23T21:16:42.449198+0000","last_unstale":"2023-09-23T21:21:17.505822+0000","last_undegraded":"2023-09-23T21:21:17.505822+0000","last_fullsized":"2023-09-23T21:21:17.505822+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:30:23.428886+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":465,"reported_epoch":708,"state":"active+clean","last_fresh":"2023-09-23T21:21:17.506037+0000","last_change":"2023-09-23T21:20:30.314319+0000","last_active":"2023-09-23T21:21:17.506037+0000","last_peered":"2023-09-23T21:21:17.506037+0000","last_clean":"2023-09-23T21:21:17.506037+0000","last_became_active":"2023-09-23T21:20:30.313943+0000","last_became_peered":"2023-09-23T21:20:30.313943+0000","last_unstale":"2023-09-23T21:21:17.506037+0000","last_undegraded":"2023-09-23T21:21:17.506037+0000","last_fullsized":"2023-09-23T21:21:17.506037+0000","mapping_epoch":689,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":690,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:24:51.385745+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.10","version":"79'161","reported_seq":927,"reported_epoch":708,"state":"active+clean","last_fresh":"2023-09-23T21:21:17.507398+0000","last_change":"2023-09-23T21:20:53.939329+0000","last_active":"2023-09-23T21:21:17.507398+0000","last_peered":"2023-09-23T21:21:17.507398+0000","last_clean":"2023-09-23T21:21:17.507398+0000","last_became_active":"2023-09-23T21:20:53.938896+0000","last_became_peered":"2023-09-23T21:20:53.938896+0000","last_unstale":"2023-09-23T21:21:17.507398+0000","last_undegraded":"2023-09-23T21:21:17.507398+0000","last_fullsized":"2023-09-23T21:21:17.507398+0000","mapping_epoch":702,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":703,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:06:50.336992+0000","last_deep_scrub":"79'161","last_deep_scrub_stamp":"2023-09-23T21:06:50.336992+0000","last_clean_scrub_stamp":"2023-09-23T21:06:50.336992+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:11:03.277383+0000","scrub_duration":0.0026150930000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":670,"reported_epoch":708,"state":"active+clean","last_fresh":"2023-09-23T21:21:17.901763+0000","last_change":"2023-09-23T21:16:42.446324+0000","last_active":"2023-09-23T21:21:17.901763+0000","last_peered":"2023-09-23T21:21:17.901763+0000","last_clean":"2023-09-23T21:21:17.901763+0000","last_became_active":"2023-09-23T21:16:42.445997+0000","last_became_peered":"2023-09-23T21:16:42.445997+0000","last_unstale":"2023-09-23T21:21:17.901763+0000","last_undegraded":"2023-09-23T21:21:17.901763+0000","last_fullsized":"2023-09-23T21:21:17.901763+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:46:35.028292+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":499,"reported_epoch":708,"state":"active+clean","last_fresh":"2023-09-23T21:21:17.508135+0000","last_change":"2023-09-23T21:20:42.355719+0000","last_active":"2023-09-23T21:21:17.508135+0000","last_peered":"2023-09-23T21:21:17.508135+0000","last_clean":"2023-09-23T21:21:17.508135+0000","last_became_active":"2023-09-23T21:20:42.354541+0000","last_became_peered":"2023-09-23T21:20:42.354541+0000","last_unstale":"2023-09-23T21:21:17.508135+0000","last_undegraded":"2023-09-23T21:21:17.508135+0000","last_fullsized":"2023-09-23T21:21:17.508135+0000","mapping_epoch":694,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":695,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:29:24.648781+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.13","version":"79'161","reported_seq":716,"reported_epoch":708,"state":"active+clean","last_fresh":"2023-09-23T21:21:17.502856+0000","last_change":"2023-09-23T21:20:15.081573+0000","last_active":"2023-09-23T21:21:17.502856+0000","last_peered":"2023-09-23T21:21:17.502856+0000","last_clean":"2023-09-23T21:21:17.502856+0000","last_became_active":"2023-09-23T21:20:15.080863+0000","last_became_peered":"2023-09-23T21:20:15.080863+0000","last_unstale":"2023-09-23T21:21:17.502856+0000","last_undegraded":"2023-09-23T21:21:17.502856+0000","last_fullsized":"2023-09-23T21:21:17.502856+0000","mapping_epoch":680,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":681,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:36.420362+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:36.420362+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:34:39.845514+0000","scrub_duration":0.028838444000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":709,"reported_epoch":708,"state":"active+clean","last_fresh":"2023-09-23T21:21:17.506514+0000","last_change":"2023-09-23T21:20:48.507356+0000","last_active":"2023-09-23T21:21:17.506514+0000","last_peered":"2023-09-23T21:21:17.506514+0000","last_clean":"2023-09-23T21:21:17.506514+0000","last_became_active":"2023-09-23T21:20:48.507015+0000","last_became_peered":"2023-09-23T21:20:48.507015+0000","last_unstale":"2023-09-23T21:21:17.506514+0000","last_undegraded":"2023-09-23T21:21:17.506514+0000","last_fullsized":"2023-09-23T21:21:17.506514+0000","mapping_epoch":697,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":698,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:47:25.662099+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":239,"reported_epoch":708,"state":"active+clean","last_fresh":"2023-09-23T21:21:17.901911+0000","last_change":"2023-09-23T21:16:42.427663+0000","last_active":"2023-09-23T21:21:17.901911+0000","last_peered":"2023-09-23T21:21:17.901911+0000","last_clean":"2023-09-23T21:21:17.901911+0000","last_became_active":"2023-09-23T21:16:42.426961+0000","last_became_peered":"2023-09-23T21:16:42.426961+0000","last_unstale":"2023-09-23T21:21:17.901911+0000","last_undegraded":"2023-09-23T21:21:17.901911+0000","last_fullsized":"2023-09-23T21:21:17.901911+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:44:27.422007+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.12","version":"79'161","reported_seq":769,"reported_epoch":708,"state":"active+clean","last_fresh":"2023-09-23T21:21:17.507483+0000","last_change":"2023-09-23T21:20:19.090120+0000","last_active":"2023-09-23T21:21:17.507483+0000","last_peered":"2023-09-23T21:21:17.507483+0000","last_clean":"2023-09-23T21:21:17.507483+0000","last_became_active":"2023-09-23T21:20:19.089743+0000","last_became_peered":"2023-09-23T21:20:19.089743+0000","last_unstale":"2023-09-23T21:21:17.507483+0000","last_undegraded":"2023-09-23T21:21:17.507483+0000","last_fullsized":"2023-09-23T21:21:17.507483+0000","mapping_epoch":684,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":685,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:05:56.785173+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:05:56.785173+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:47:18.107527+0000","scrub_duration":0.0042114370000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":76,"reported_epoch":708,"state":"active+clean","last_fresh":"2023-09-23T21:21:17.508531+0000","last_change":"2023-09-23T21:20:15.939391+0000","last_active":"2023-09-23T21:21:17.508531+0000","last_peered":"2023-09-23T21:21:17.508531+0000","last_clean":"2023-09-23T21:21:17.508531+0000","last_became_active":"2023-09-23T21:20:15.928437+0000","last_became_peered":"2023-09-23T21:20:15.928437+0000","last_unstale":"2023-09-23T21:21:17.508531+0000","last_undegraded":"2023-09-23T21:21:17.508531+0000","last_fullsized":"2023-09-23T21:21:17.508531+0000","mapping_epoch":681,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":682,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:51:06.828186+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":575,"reported_epoch":708,"state":"active+clean","last_fresh":"2023-09-23T21:21:17.507034+0000","last_change":"2023-09-23T21:20:12.105028+0000","last_active":"2023-09-23T21:21:17.507034+0000","last_peered":"2023-09-23T21:21:17.507034+0000","last_clean":"2023-09-23T21:21:17.507034+0000","last_became_active":"2023-09-23T21:20:12.104725+0000","last_became_peered":"2023-09-23T21:20:12.104725+0000","last_unstale":"2023-09-23T21:21:17.507034+0000","last_undegraded":"2023-09-23T21:21:17.507034+0000","last_fullsized":"2023-09-23T21:21:17.507034+0000","mapping_epoch":677,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":678,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:01:01.926290+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.d","version":"79'161","reported_seq":770,"reported_epoch":708,"state":"active+clean","last_fresh":"2023-09-23T21:21:17.506492+0000","last_change":"2023-09-23T21:20:15.080132+0000","last_active":"2023-09-23T21:21:17.506492+0000","last_peered":"2023-09-23T21:21:17.506492+0000","last_clean":"2023-09-23T21:21:17.506492+0000","last_became_active":"2023-09-23T21:20:15.079810+0000","last_became_peered":"2023-09-23T21:20:15.079810+0000","last_unstale":"2023-09-23T21:21:17.506492+0000","last_undegraded":"2023-09-23T21:21:17.506492+0000","last_fullsized":"2023-09-23T21:21:17.506492+0000","mapping_epoch":680,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":681,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:18:47.440263+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:18:47.440263+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:36:22.438645+0000","scrub_duration":0.0023849959999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":158,"reported_epoch":708,"state":"active+clean","last_fresh":"2023-09-23T21:21:17.506054+0000","last_change":"2023-09-23T21:17:02.190320+0000","last_active":"2023-09-23T21:21:17.506054+0000","last_peered":"2023-09-23T21:21:17.506054+0000","last_clean":"2023-09-23T21:21:17.506054+0000","last_became_active":"2023-09-23T21:16:42.451380+0000","last_became_peered":"2023-09-23T21:16:42.451380+0000","last_unstale":"2023-09-23T21:21:17.506054+0000","last_undegraded":"2023-09-23T21:21:17.506054+0000","last_fullsized":"2023-09-23T21:21:17.506054+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:48:58.425821+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":132,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":47,"reported_epoch":708,"state":"active+clean","last_fresh":"2023-09-23T21:21:17.507655+0000","last_change":"2023-09-23T21:20:50.928140+0000","last_active":"2023-09-23T21:21:17.507655+0000","last_peered":"2023-09-23T21:21:17.507655+0000","last_clean":"2023-09-23T21:21:17.507655+0000","last_became_active":"2023-09-23T21:20:50.927735+0000","last_became_peered":"2023-09-23T21:20:50.927735+0000","last_unstale":"2023-09-23T21:21:17.507655+0000","last_undegraded":"2023-09-23T21:21:17.507655+0000","last_fullsized":"2023-09-23T21:21:17.507655+0000","mapping_epoch":699,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":700,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:29:04.097873+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.c","version":"79'161","reported_seq":1113,"reported_epoch":708,"state":"active+clean","last_fresh":"2023-09-23T21:21:17.505871+0000","last_change":"2023-09-23T21:20:08.011328+0000","last_active":"2023-09-23T21:21:17.505871+0000","last_peered":"2023-09-23T21:21:17.505871+0000","last_clean":"2023-09-23T21:21:17.505871+0000","last_became_active":"2023-09-23T21:20:08.010905+0000","last_became_peered":"2023-09-23T21:20:08.010905+0000","last_unstale":"2023-09-23T21:21:17.505871+0000","last_undegraded":"2023-09-23T21:21:17.505871+0000","last_fullsized":"2023-09-23T21:21:17.505871+0000","mapping_epoch":675,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":676,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:15:31.992598+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:15:31.992598+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:22:35.797510+0000","scrub_duration":0.002960678,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":327680,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":172,"reported_epoch":708,"state":"active+clean","last_fresh":"2023-09-23T21:21:17.506109+0000","last_change":"2023-09-23T21:20:12.104469+0000","last_active":"2023-09-23T21:21:17.506109+0000","last_peered":"2023-09-23T21:21:17.506109+0000","last_clean":"2023-09-23T21:21:17.506109+0000","last_became_active":"2023-09-23T21:20:12.086524+0000","last_became_peered":"2023-09-23T21:20:12.086524+0000","last_unstale":"2023-09-23T21:21:17.506109+0000","last_undegraded":"2023-09-23T21:21:17.506109+0000","last_fullsized":"2023-09-23T21:21:17.506109+0000","mapping_epoch":677,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":678,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:05:30.178772+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":681,"reported_epoch":708,"state":"active+clean","last_fresh":"2023-09-23T21:21:17.505453+0000","last_change":"2023-09-23T21:20:12.078362+0000","last_active":"2023-09-23T21:21:17.505453+0000","last_peered":"2023-09-23T21:21:17.505453+0000","last_clean":"2023-09-23T21:21:17.505453+0000","last_became_active":"2023-09-23T21:20:12.077958+0000","last_became_peered":"2023-09-23T21:20:12.077958+0000","last_unstale":"2023-09-23T21:21:17.505453+0000","last_undegraded":"2023-09-23T21:21:17.505453+0000","last_fullsized":"2023-09-23T21:21:17.505453+0000","mapping_epoch":677,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":678,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:10:04.783325+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.f","version":"79'161","reported_seq":800,"reported_epoch":708,"state":"active+clean","last_fresh":"2023-09-23T21:21:17.509361+0000","last_change":"2023-09-23T21:20:19.094207+0000","last_active":"2023-09-23T21:21:17.509361+0000","last_peered":"2023-09-23T21:21:17.509361+0000","last_clean":"2023-09-23T21:21:17.509361+0000","last_became_active":"2023-09-23T21:20:19.093826+0000","last_became_peered":"2023-09-23T21:20:19.093826+0000","last_unstale":"2023-09-23T21:21:17.509361+0000","last_undegraded":"2023-09-23T21:21:17.509361+0000","last_fullsized":"2023-09-23T21:21:17.509361+0000","mapping_epoch":684,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":685,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:27.880776+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:27.880776+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:31:51.006419+0000","scrub_duration":0.0014004899999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":327,"reported_epoch":708,"state":"active+clean","last_fresh":"2023-09-23T21:21:17.506511+0000","last_change":"2023-09-23T21:20:48.509999+0000","last_active":"2023-09-23T21:21:17.506511+0000","last_peered":"2023-09-23T21:21:17.506511+0000","last_clean":"2023-09-23T21:21:17.506511+0000","last_became_active":"2023-09-23T21:20:48.509613+0000","last_became_peered":"2023-09-23T21:20:48.509613+0000","last_unstale":"2023-09-23T21:21:17.506511+0000","last_undegraded":"2023-09-23T21:21:17.506511+0000","last_fullsized":"2023-09-23T21:21:17.506511+0000","mapping_epoch":697,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":698,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:47:38.322761+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":350,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":19,"reported_epoch":708,"state":"active+clean","last_fresh":"2023-09-23T21:21:17.509211+0000","last_change":"2023-09-23T21:20:50.930303+0000","last_active":"2023-09-23T21:21:17.509211+0000","last_peered":"2023-09-23T21:21:17.509211+0000","last_clean":"2023-09-23T21:21:17.509211+0000","last_became_active":"2023-09-23T21:20:50.929881+0000","last_became_peered":"2023-09-23T21:20:50.929881+0000","last_unstale":"2023-09-23T21:21:17.509211+0000","last_undegraded":"2023-09-23T21:21:17.509211+0000","last_fullsized":"2023-09-23T21:21:17.509211+0000","mapping_epoch":699,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":700,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:52:39.669108+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.e","version":"79'161","reported_seq":927,"reported_epoch":708,"state":"active+clean","last_fresh":"2023-09-23T21:21:17.505789+0000","last_change":"2023-09-23T21:16:45.433204+0000","last_active":"2023-09-23T21:21:17.505789+0000","last_peered":"2023-09-23T21:21:17.505789+0000","last_clean":"2023-09-23T21:21:17.505789+0000","last_became_active":"2023-09-23T21:16:45.432836+0000","last_became_peered":"2023-09-23T21:16:45.432836+0000","last_unstale":"2023-09-23T21:21:17.505789+0000","last_undegraded":"2023-09-23T21:21:17.505789+0000","last_fullsized":"2023-09-23T21:21:17.505789+0000","mapping_epoch":567,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":568,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:15:39.046962+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:15:39.046962+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:11:20.720731+0000","scrub_duration":0.0082638769999999993,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":535,"reported_epoch":708,"state":"active+clean","last_fresh":"2023-09-23T21:21:17.507300+0000","last_change":"2023-09-23T21:20:42.357079+0000","last_active":"2023-09-23T21:21:17.507300+0000","last_peered":"2023-09-23T21:21:17.507300+0000","last_clean":"2023-09-23T21:21:17.507300+0000","last_became_active":"2023-09-23T21:20:42.356777+0000","last_became_peered":"2023-09-23T21:20:42.356777+0000","last_unstale":"2023-09-23T21:21:17.507300+0000","last_undegraded":"2023-09-23T21:21:17.507300+0000","last_fullsized":"2023-09-23T21:21:17.507300+0000","mapping_epoch":694,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":695,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:57:20.387559+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":693,"reported_epoch":708,"state":"active+clean","last_fresh":"2023-09-23T21:21:17.901703+0000","last_change":"2023-09-23T21:20:12.080657+0000","last_active":"2023-09-23T21:21:17.901703+0000","last_peered":"2023-09-23T21:21:17.901703+0000","last_clean":"2023-09-23T21:21:17.901703+0000","last_became_active":"2023-09-23T21:20:12.080285+0000","last_became_peered":"2023-09-23T21:20:12.080285+0000","last_unstale":"2023-09-23T21:21:17.901703+0000","last_undegraded":"2023-09-23T21:21:17.901703+0000","last_fullsized":"2023-09-23T21:21:17.901703+0000","mapping_epoch":677,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":678,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:43:27.608818+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.9","version":"79'161","reported_seq":774,"reported_epoch":708,"state":"active+clean","last_fresh":"2023-09-23T21:21:17.506330+0000","last_change":"2023-09-23T21:20:48.511155+0000","last_active":"2023-09-23T21:21:17.506330+0000","last_peered":"2023-09-23T21:21:17.506330+0000","last_clean":"2023-09-23T21:21:17.506330+0000","last_became_active":"2023-09-23T21:20:48.510795+0000","last_became_peered":"2023-09-23T21:20:48.510795+0000","last_unstale":"2023-09-23T21:21:17.506330+0000","last_undegraded":"2023-09-23T21:21:17.506330+0000","last_fullsized":"2023-09-23T21:21:17.506330+0000","mapping_epoch":697,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":698,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:20:31.919517+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:20:31.919517+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:35:43.721278+0000","scrub_duration":0.0024129429999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":456,"reported_epoch":708,"state":"active+clean","last_fresh":"2023-09-23T21:21:17.506280+0000","last_change":"2023-09-23T21:20:30.314664+0000","last_active":"2023-09-23T21:21:17.506280+0000","last_peered":"2023-09-23T21:21:17.506280+0000","last_clean":"2023-09-23T21:21:17.506280+0000","last_became_active":"2023-09-23T21:20:30.314368+0000","last_became_peered":"2023-09-23T21:20:30.314368+0000","last_unstale":"2023-09-23T21:21:17.506280+0000","last_undegraded":"2023-09-23T21:21:17.506280+0000","last_fullsized":"2023-09-23T21:21:17.506280+0000","mapping_epoch":689,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":690,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:18:26.565378+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":312,"reported_epoch":708,"state":"active+clean","last_fresh":"2023-09-23T21:21:17.505608+0000","last_change":"2023-09-23T21:20:05.018378+0000","last_active":"2023-09-23T21:21:17.505608+0000","last_peered":"2023-09-23T21:21:17.505608+0000","last_clean":"2023-09-23T21:21:17.505608+0000","last_became_active":"2023-09-23T21:20:05.017706+0000","last_became_peered":"2023-09-23T21:20:05.017706+0000","last_unstale":"2023-09-23T21:21:17.505608+0000","last_undegraded":"2023-09-23T21:21:17.505608+0000","last_fullsized":"2023-09-23T21:21:17.505608+0000","mapping_epoch":672,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":673,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:35:40.095410+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.8","version":"79'161","reported_seq":863,"reported_epoch":708,"state":"active+clean","last_fresh":"2023-09-23T21:21:17.507560+0000","last_change":"2023-09-23T21:20:42.355580+0000","last_active":"2023-09-23T21:21:17.507560+0000","last_peered":"2023-09-23T21:21:17.507560+0000","last_clean":"2023-09-23T21:21:17.507560+0000","last_became_active":"2023-09-23T21:20:42.355019+0000","last_became_peered":"2023-09-23T21:20:42.355019+0000","last_unstale":"2023-09-23T21:21:17.507560+0000","last_undegraded":"2023-09-23T21:21:17.507560+0000","last_fullsized":"2023-09-23T21:21:17.507560+0000","mapping_epoch":694,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":695,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:15:36.367640+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:15:36.367640+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:32:15.572118+0000","scrub_duration":0.005826899,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":56,"reported_epoch":708,"state":"active+clean","last_fresh":"2023-09-23T21:21:17.509397+0000","last_change":"2023-09-23T21:20:12.078659+0000","last_active":"2023-09-23T21:21:17.509397+0000","last_peered":"2023-09-23T21:21:17.509397+0000","last_clean":"2023-09-23T21:21:17.509397+0000","last_became_active":"2023-09-23T21:20:12.078340+0000","last_became_peered":"2023-09-23T21:20:12.078340+0000","last_unstale":"2023-09-23T21:21:17.509397+0000","last_undegraded":"2023-09-23T21:21:17.509397+0000","last_fullsized":"2023-09-23T21:21:17.509397+0000","mapping_epoch":677,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":678,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:42:13.222106+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":316,"reported_epoch":709,"state":"active+clean","last_fresh":"2023-09-23T21:21:18.514308+0000","last_change":"2023-09-23T21:21:18.514308+0000","last_active":"2023-09-23T21:21:18.514308+0000","last_peered":"2023-09-23T21:21:18.514308+0000","last_clean":"2023-09-23T21:21:18.514308+0000","last_became_active":"2023-09-23T21:21:18.513966+0000","last_became_peered":"2023-09-23T21:21:18.513966+0000","last_unstale":"2023-09-23T21:21:18.514308+0000","last_undegraded":"2023-09-23T21:21:18.514308+0000","last_fullsized":"2023-09-23T21:21:18.514308+0000","mapping_epoch":708,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":709,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:38:09.611613+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.b","version":"79'161","reported_seq":779,"reported_epoch":708,"state":"active+clean","last_fresh":"2023-09-23T21:21:17.502961+0000","last_change":"2023-09-23T21:20:19.085592+0000","last_active":"2023-09-23T21:21:17.502961+0000","last_peered":"2023-09-23T21:21:17.502961+0000","last_clean":"2023-09-23T21:21:17.502961+0000","last_became_active":"2023-09-23T21:20:19.085161+0000","last_became_peered":"2023-09-23T21:20:19.085161+0000","last_unstale":"2023-09-23T21:21:17.502961+0000","last_undegraded":"2023-09-23T21:21:17.502961+0000","last_fullsized":"2023-09-23T21:21:17.502961+0000","mapping_epoch":684,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":685,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:15:36.339885+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:15:36.339885+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:45:27.119479+0000","scrub_duration":0.0068473279999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,5],"acting":[1,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":286,"reported_epoch":708,"state":"active+clean","last_fresh":"2023-09-23T21:21:17.506029+0000","last_change":"2023-09-23T21:16:42.425103+0000","last_active":"2023-09-23T21:21:17.506029+0000","last_peered":"2023-09-23T21:21:17.506029+0000","last_clean":"2023-09-23T21:21:17.506029+0000","last_became_active":"2023-09-23T21:16:42.424543+0000","last_became_peered":"2023-09-23T21:16:42.424543+0000","last_unstale":"2023-09-23T21:21:17.506029+0000","last_undegraded":"2023-09-23T21:21:17.506029+0000","last_fullsized":"2023-09-23T21:21:17.506029+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:11:28.885246+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":670,"reported_epoch":708,"state":"active+clean","last_fresh":"2023-09-23T21:21:17.505927+0000","last_change":"2023-09-23T21:20:50.930124+0000","last_active":"2023-09-23T21:21:17.505927+0000","last_peered":"2023-09-23T21:21:17.505927+0000","last_clean":"2023-09-23T21:21:17.505927+0000","last_became_active":"2023-09-23T21:20:50.929778+0000","last_became_peered":"2023-09-23T21:20:50.929778+0000","last_unstale":"2023-09-23T21:21:17.505927+0000","last_undegraded":"2023-09-23T21:21:17.505927+0000","last_fullsized":"2023-09-23T21:21:17.505927+0000","mapping_epoch":699,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":700,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:05:56.336810+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.a","version":"79'161","reported_seq":906,"reported_epoch":708,"state":"active+clean","last_fresh":"2023-09-23T21:21:17.507401+0000","last_change":"2023-09-23T21:20:42.355595+0000","last_active":"2023-09-23T21:21:17.507401+0000","last_peered":"2023-09-23T21:21:17.507401+0000","last_clean":"2023-09-23T21:21:17.507401+0000","last_became_active":"2023-09-23T21:20:42.355194+0000","last_became_peered":"2023-09-23T21:20:42.355194+0000","last_unstale":"2023-09-23T21:21:17.507401+0000","last_undegraded":"2023-09-23T21:21:17.507401+0000","last_fullsized":"2023-09-23T21:21:17.507401+0000","mapping_epoch":694,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":695,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:15:46.420650+0000","last_deep_scrub":"79'161","last_deep_scrub_stamp":"2023-09-23T21:05:32.630292+0000","last_clean_scrub_stamp":"2023-09-23T21:15:46.420650+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:18:43.952878+0000","scrub_duration":0.0072898930000000004,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":586,"reported_epoch":709,"state":"active+clean","last_fresh":"2023-09-23T21:21:18.506026+0000","last_change":"2023-09-23T21:20:12.104617+0000","last_active":"2023-09-23T21:21:18.506026+0000","last_peered":"2023-09-23T21:21:18.506026+0000","last_clean":"2023-09-23T21:21:18.506026+0000","last_became_active":"2023-09-23T21:20:12.104106+0000","last_became_peered":"2023-09-23T21:20:12.104106+0000","last_unstale":"2023-09-23T21:21:18.506026+0000","last_undegraded":"2023-09-23T21:21:18.506026+0000","last_fullsized":"2023-09-23T21:21:18.506026+0000","mapping_epoch":677,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":678,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:09:16.885310+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":42,"reported_epoch":708,"state":"active+clean","last_fresh":"2023-09-23T21:21:17.504641+0000","last_change":"2023-09-23T21:20:12.115012+0000","last_active":"2023-09-23T21:21:17.504641+0000","last_peered":"2023-09-23T21:21:17.504641+0000","last_clean":"2023-09-23T21:21:17.504641+0000","last_became_active":"2023-09-23T21:20:12.114632+0000","last_became_peered":"2023-09-23T21:20:12.114632+0000","last_unstale":"2023-09-23T21:21:17.504641+0000","last_undegraded":"2023-09-23T21:21:17.504641+0000","last_fullsized":"2023-09-23T21:21:17.504641+0000","mapping_epoch":677,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":678,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:41:02.454109+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.7","version":"79'161","reported_seq":842,"reported_epoch":708,"state":"active+clean","last_fresh":"2023-09-23T21:21:17.509325+0000","last_change":"2023-09-23T21:20:08.010981+0000","last_active":"2023-09-23T21:21:17.509325+0000","last_peered":"2023-09-23T21:21:17.509325+0000","last_clean":"2023-09-23T21:21:17.509325+0000","last_became_active":"2023-09-23T21:20:08.010682+0000","last_became_peered":"2023-09-23T21:20:08.010682+0000","last_unstale":"2023-09-23T21:21:17.509325+0000","last_undegraded":"2023-09-23T21:21:17.509325+0000","last_fullsized":"2023-09-23T21:21:17.509325+0000","mapping_epoch":675,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":676,"parent":"0.0","parent_split_bits":3,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:15:51.016427+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:15:51.016427+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:52:28.243159+0000","scrub_duration":0.0017648640000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.4","version":"668'149","reported_seq":1052,"reported_epoch":708,"state":"active+clean","last_fresh":"2023-09-23T21:21:17.503057+0000","last_change":"2023-09-23T21:20:15.083615+0000","last_active":"2023-09-23T21:21:17.503057+0000","last_peered":"2023-09-23T21:21:17.503057+0000","last_clean":"2023-09-23T21:21:17.503057+0000","last_became_active":"2023-09-23T21:20:15.082980+0000","last_became_peered":"2023-09-23T21:20:15.082980+0000","last_unstale":"2023-09-23T21:21:17.503057+0000","last_undegraded":"2023-09-23T21:21:17.503057+0000","last_fullsized":"2023-09-23T21:21:17.503057+0000","mapping_epoch":680,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":681,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:06:27.749409+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00068658800000000002,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":533,"num_read_kb":902,"num_write":165,"num_write_kb":306,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":13,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":37,"reported_epoch":708,"state":"active+clean","last_fresh":"2023-09-23T21:21:17.507978+0000","last_change":"2023-09-23T21:20:15.937817+0000","last_active":"2023-09-23T21:21:17.507978+0000","last_peered":"2023-09-23T21:21:17.507978+0000","last_clean":"2023-09-23T21:21:17.507978+0000","last_became_active":"2023-09-23T21:20:15.937449+0000","last_became_peered":"2023-09-23T21:20:15.937449+0000","last_unstale":"2023-09-23T21:21:17.507978+0000","last_undegraded":"2023-09-23T21:21:17.507978+0000","last_fullsized":"2023-09-23T21:21:17.507978+0000","mapping_epoch":681,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":682,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:10:39.568587+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":51,"reported_epoch":708,"state":"active+clean","last_fresh":"2023-09-23T21:21:17.502525+0000","last_change":"2023-09-23T21:20:15.931234+0000","last_active":"2023-09-23T21:21:17.502525+0000","last_peered":"2023-09-23T21:21:17.502525+0000","last_clean":"2023-09-23T21:21:17.502525+0000","last_became_active":"2023-09-23T21:20:15.930213+0000","last_became_peered":"2023-09-23T21:20:15.930213+0000","last_unstale":"2023-09-23T21:21:17.502525+0000","last_undegraded":"2023-09-23T21:21:17.502525+0000","last_fullsized":"2023-09-23T21:21:17.502525+0000","mapping_epoch":681,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":682,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:46:34.523504+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"79'161","reported_seq":813,"reported_epoch":708,"state":"active+clean","last_fresh":"2023-09-23T21:21:17.506655+0000","last_change":"2023-09-23T21:20:42.353395+0000","last_active":"2023-09-23T21:21:17.506655+0000","last_peered":"2023-09-23T21:21:17.506655+0000","last_clean":"2023-09-23T21:21:17.506655+0000","last_became_active":"2023-09-23T21:20:42.352986+0000","last_became_peered":"2023-09-23T21:20:42.352986+0000","last_unstale":"2023-09-23T21:21:17.506655+0000","last_undegraded":"2023-09-23T21:21:17.506655+0000","last_fullsized":"2023-09-23T21:21:17.506655+0000","mapping_epoch":694,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":695,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:15:34.345138+0000","last_deep_scrub":"79'161","last_deep_scrub_stamp":"2023-09-23T21:05:30.652189+0000","last_clean_scrub_stamp":"2023-09-23T21:15:34.345138+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:13:49.266544+0000","scrub_duration":0.0057520000000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"668'88","reported_seq":95,"reported_epoch":708,"state":"active+clean","last_fresh":"2023-09-23T21:21:17.507266+0000","last_change":"2023-09-23T21:20:50.953069+0000","last_active":"2023-09-23T21:21:17.507266+0000","last_peered":"2023-09-23T21:21:17.507266+0000","last_clean":"2023-09-23T21:21:17.507266+0000","last_became_active":"2023-09-23T21:20:50.930023+0000","last_became_peered":"2023-09-23T21:20:50.930023+0000","last_unstale":"2023-09-23T21:21:17.507266+0000","last_undegraded":"2023-09-23T21:21:17.507266+0000","last_fullsized":"2023-09-23T21:21:17.507266+0000","mapping_epoch":699,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":700,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":88,"log_dups_size":0,"ondisk_log_size":88,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:20:23.306383+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00049772800000000002,"stat_sum":{"num_bytes":4194323,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":875,"num_read_kb":1071,"num_write":62,"num_write_kb":4277,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":18,"num_bytes_recovered":8388646,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":662,"reported_epoch":708,"state":"active+clean","last_fresh":"2023-09-23T21:21:17.505808+0000","last_change":"2023-09-23T21:20:05.017845+0000","last_active":"2023-09-23T21:21:17.505808+0000","last_peered":"2023-09-23T21:21:17.505808+0000","last_clean":"2023-09-23T21:21:17.505808+0000","last_became_active":"2023-09-23T21:20:05.017517+0000","last_became_peered":"2023-09-23T21:20:05.017517+0000","last_unstale":"2023-09-23T21:21:17.505808+0000","last_undegraded":"2023-09-23T21:21:17.505808+0000","last_fullsized":"2023-09-23T21:21:17.505808+0000","mapping_epoch":672,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":673,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:24:33.218000+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":42,"reported_epoch":708,"state":"active+clean","last_fresh":"2023-09-23T21:21:17.503100+0000","last_change":"2023-09-23T21:20:12.107633+0000","last_active":"2023-09-23T21:21:17.503100+0000","last_peered":"2023-09-23T21:21:17.503100+0000","last_clean":"2023-09-23T21:21:17.503100+0000","last_became_active":"2023-09-23T21:20:12.107060+0000","last_became_peered":"2023-09-23T21:20:12.107060+0000","last_unstale":"2023-09-23T21:21:17.503100+0000","last_undegraded":"2023-09-23T21:21:17.503100+0000","last_fullsized":"2023-09-23T21:21:17.503100+0000","mapping_epoch":677,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":678,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:40:43.679026+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"79'161","reported_seq":888,"reported_epoch":708,"state":"active+clean","last_fresh":"2023-09-23T21:21:17.506225+0000","last_change":"2023-09-23T21:20:48.511190+0000","last_active":"2023-09-23T21:21:17.506225+0000","last_peered":"2023-09-23T21:21:17.506225+0000","last_clean":"2023-09-23T21:21:17.506225+0000","last_became_active":"2023-09-23T21:20:48.510409+0000","last_became_peered":"2023-09-23T21:20:48.510409+0000","last_unstale":"2023-09-23T21:21:17.506225+0000","last_undegraded":"2023-09-23T21:21:17.506225+0000","last_fullsized":"2023-09-23T21:21:17.506225+0000","mapping_epoch":697,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":698,"parent":"0.0","parent_split_bits":4,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:24.802451+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:24.802451+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:13:19.915510+0000","scrub_duration":0.0090861250000000005,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"668'111","reported_seq":1241,"reported_epoch":708,"state":"active+clean","last_fresh":"2023-09-23T21:21:17.508223+0000","last_change":"2023-09-23T21:19:51.908105+0000","last_active":"2023-09-23T21:21:17.508223+0000","last_peered":"2023-09-23T21:21:17.508223+0000","last_clean":"2023-09-23T21:21:17.508223+0000","last_became_active":"2023-09-23T21:13:12.206650+0000","last_became_peered":"2023-09-23T21:13:12.206650+0000","last_unstale":"2023-09-23T21:21:17.508223+0000","last_undegraded":"2023-09-23T21:21:17.508223+0000","last_fullsized":"2023-09-23T21:21:17.508223+0000","mapping_epoch":493,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":111,"log_dups_size":0,"ondisk_log_size":111,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:10:01.892936+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0016921,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":876,"num_read_kb":3120,"num_write":86,"num_write_kb":2279,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":42,"reported_epoch":708,"state":"active+clean","last_fresh":"2023-09-23T21:21:17.502563+0000","last_change":"2023-09-23T21:20:12.094430+0000","last_active":"2023-09-23T21:21:17.502563+0000","last_peered":"2023-09-23T21:21:17.502563+0000","last_clean":"2023-09-23T21:21:17.502563+0000","last_became_active":"2023-09-23T21:20:12.088960+0000","last_became_peered":"2023-09-23T21:20:12.088960+0000","last_unstale":"2023-09-23T21:21:17.502563+0000","last_undegraded":"2023-09-23T21:21:17.502563+0000","last_fullsized":"2023-09-23T21:21:17.502563+0000","mapping_epoch":677,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":678,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:05:13.583404+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":51,"reported_epoch":708,"state":"active+clean","last_fresh":"2023-09-23T21:21:17.504722+0000","last_change":"2023-09-23T21:20:15.935747+0000","last_active":"2023-09-23T21:21:17.504722+0000","last_peered":"2023-09-23T21:21:17.504722+0000","last_clean":"2023-09-23T21:21:17.504722+0000","last_became_active":"2023-09-23T21:20:15.935401+0000","last_became_peered":"2023-09-23T21:20:15.935401+0000","last_unstale":"2023-09-23T21:21:17.504722+0000","last_undegraded":"2023-09-23T21:21:17.504722+0000","last_fullsized":"2023-09-23T21:21:17.504722+0000","mapping_epoch":681,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":682,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:44:17.963923+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.3","version":"668'166","reported_seq":1059,"reported_epoch":708,"state":"active+clean","last_fresh":"2023-09-23T21:21:17.506982+0000","last_change":"2023-09-23T21:20:42.354590+0000","last_active":"2023-09-23T21:21:17.506982+0000","last_peered":"2023-09-23T21:21:17.506982+0000","last_clean":"2023-09-23T21:21:17.506982+0000","last_became_active":"2023-09-23T21:20:42.354054+0000","last_became_peered":"2023-09-23T21:20:42.354054+0000","last_unstale":"2023-09-23T21:21:17.506982+0000","last_undegraded":"2023-09-23T21:21:17.506982+0000","last_fullsized":"2023-09-23T21:21:17.506982+0000","mapping_epoch":694,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":695,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":166,"log_dups_size":0,"ondisk_log_size":166,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:36:43.918675+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00031341300000000002,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":787,"num_read_kb":1740,"num_write":150,"num_write_kb":1123,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"79'161","reported_seq":791,"reported_epoch":708,"state":"active+clean","last_fresh":"2023-09-23T21:21:17.505813+0000","last_change":"2023-09-23T21:20:48.510993+0000","last_active":"2023-09-23T21:21:17.505813+0000","last_peered":"2023-09-23T21:21:17.505813+0000","last_clean":"2023-09-23T21:21:17.505813+0000","last_became_active":"2023-09-23T21:20:48.510554+0000","last_became_peered":"2023-09-23T21:20:48.510554+0000","last_unstale":"2023-09-23T21:21:17.505813+0000","last_undegraded":"2023-09-23T21:21:17.505813+0000","last_fullsized":"2023-09-23T21:21:17.505813+0000","mapping_epoch":697,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":698,"parent":"0.0","parent_split_bits":0,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:20.738273+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:20.738273+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:03:57.365350+0000","scrub_duration":0.0088336339999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":477,"reported_epoch":708,"state":"active+clean","last_fresh":"2023-09-23T21:21:17.505576+0000","last_change":"2023-09-23T21:20:30.373442+0000","last_active":"2023-09-23T21:21:17.505576+0000","last_peered":"2023-09-23T21:21:17.505576+0000","last_clean":"2023-09-23T21:21:17.505576+0000","last_became_active":"2023-09-23T21:20:30.372967+0000","last_became_peered":"2023-09-23T21:20:30.372967+0000","last_unstale":"2023-09-23T21:21:17.505576+0000","last_undegraded":"2023-09-23T21:21:17.505576+0000","last_fullsized":"2023-09-23T21:21:17.505576+0000","mapping_epoch":689,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":690,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:13:46.307043+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":19,"reported_epoch":708,"state":"active+clean","last_fresh":"2023-09-23T21:21:17.508665+0000","last_change":"2023-09-23T21:20:50.930557+0000","last_active":"2023-09-23T21:21:17.508665+0000","last_peered":"2023-09-23T21:21:17.508665+0000","last_clean":"2023-09-23T21:21:17.508665+0000","last_became_active":"2023-09-23T21:20:50.930264+0000","last_became_peered":"2023-09-23T21:20:50.930264+0000","last_unstale":"2023-09-23T21:21:17.508665+0000","last_undegraded":"2023-09-23T21:21:17.508665+0000","last_fullsized":"2023-09-23T21:21:17.508665+0000","mapping_epoch":699,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":700,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:41:33.282685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.3","version":"79'161","reported_seq":859,"reported_epoch":708,"state":"active+clean","last_fresh":"2023-09-23T21:21:17.502861+0000","last_change":"2023-09-23T21:20:53.938250+0000","last_active":"2023-09-23T21:21:17.502861+0000","last_peered":"2023-09-23T21:21:17.502861+0000","last_clean":"2023-09-23T21:21:17.502861+0000","last_became_active":"2023-09-23T21:20:53.937861+0000","last_became_peered":"2023-09-23T21:20:53.937861+0000","last_unstale":"2023-09-23T21:21:17.502861+0000","last_undegraded":"2023-09-23T21:21:17.502861+0000","last_fullsized":"2023-09-23T21:21:17.502861+0000","mapping_epoch":702,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":703,"parent":"0.0","parent_split_bits":3,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:05:57.832302+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:05:57.832302+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:16:41.826413+0000","scrub_duration":0.010206847999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"668'67","reported_seq":854,"reported_epoch":708,"state":"active+clean","last_fresh":"2023-09-23T21:21:17.505908+0000","last_change":"2023-09-23T21:20:48.511538+0000","last_active":"2023-09-23T21:21:17.505908+0000","last_peered":"2023-09-23T21:21:17.505908+0000","last_clean":"2023-09-23T21:21:17.505908+0000","last_became_active":"2023-09-23T21:20:48.510475+0000","last_became_peered":"2023-09-23T21:20:48.510475+0000","last_unstale":"2023-09-23T21:21:17.505908+0000","last_undegraded":"2023-09-23T21:21:17.505908+0000","last_fullsized":"2023-09-23T21:21:17.505908+0000","mapping_epoch":697,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":698,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":67,"log_dups_size":0,"ondisk_log_size":67,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:55:03.524020+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00037825000000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":538,"num_read_kb":2522,"num_write":45,"num_write_kb":3096,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":711,"reported_epoch":709,"state":"active+clean","last_fresh":"2023-09-23T21:21:18.510285+0000","last_change":"2023-09-23T21:13:44.409212+0000","last_active":"2023-09-23T21:21:18.510285+0000","last_peered":"2023-09-23T21:21:18.510285+0000","last_clean":"2023-09-23T21:21:18.510285+0000","last_became_active":"2023-09-23T21:13:44.408958+0000","last_became_peered":"2023-09-23T21:13:44.408958+0000","last_unstale":"2023-09-23T21:21:18.510285+0000","last_undegraded":"2023-09-23T21:21:18.510285+0000","last_fullsized":"2023-09-23T21:21:18.510285+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:03:52.749964+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":248,"reported_epoch":708,"state":"active+clean","last_fresh":"2023-09-23T21:21:17.902062+0000","last_change":"2023-09-23T21:20:12.079524+0000","last_active":"2023-09-23T21:21:17.902062+0000","last_peered":"2023-09-23T21:21:17.902062+0000","last_clean":"2023-09-23T21:21:17.902062+0000","last_became_active":"2023-09-23T21:20:12.079185+0000","last_became_peered":"2023-09-23T21:20:12.079185+0000","last_unstale":"2023-09-23T21:21:17.902062+0000","last_undegraded":"2023-09-23T21:21:17.902062+0000","last_fullsized":"2023-09-23T21:21:17.902062+0000","mapping_epoch":677,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":678,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:49:26.875689+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.6","version":"79'161","reported_seq":652,"reported_epoch":708,"state":"active+clean","last_fresh":"2023-09-23T21:21:17.508790+0000","last_change":"2023-09-23T21:20:08.014769+0000","last_active":"2023-09-23T21:21:17.508790+0000","last_peered":"2023-09-23T21:21:17.508790+0000","last_clean":"2023-09-23T21:21:17.508790+0000","last_became_active":"2023-09-23T21:20:08.014281+0000","last_became_peered":"2023-09-23T21:20:08.014281+0000","last_unstale":"2023-09-23T21:21:17.508790+0000","last_undegraded":"2023-09-23T21:21:17.508790+0000","last_fullsized":"2023-09-23T21:21:17.508790+0000","mapping_epoch":675,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":676,"parent":"0.0","parent_split_bits":3,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:15:43.333733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:15:43.333733+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:24:17.713184+0000","scrub_duration":0.0092107530000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.5","version":"668'105","reported_seq":550,"reported_epoch":708,"state":"active+clean","last_fresh":"2023-09-23T21:21:17.506331+0000","last_change":"2023-09-23T21:19:51.914035+0000","last_active":"2023-09-23T21:21:17.506331+0000","last_peered":"2023-09-23T21:21:17.506331+0000","last_clean":"2023-09-23T21:21:17.506331+0000","last_became_active":"2023-09-23T21:16:42.482847+0000","last_became_peered":"2023-09-23T21:16:42.482847+0000","last_unstale":"2023-09-23T21:21:17.506331+0000","last_undegraded":"2023-09-23T21:21:17.506331+0000","last_fullsized":"2023-09-23T21:21:17.506331+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":105,"log_dups_size":0,"ondisk_log_size":105,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:38:57.905679+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0010907810000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":751,"num_read_kb":635,"num_write":101,"num_write_kb":54,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":51,"reported_epoch":708,"state":"active+clean","last_fresh":"2023-09-23T21:21:17.503007+0000","last_change":"2023-09-23T21:20:15.935131+0000","last_active":"2023-09-23T21:21:17.503007+0000","last_peered":"2023-09-23T21:21:17.503007+0000","last_clean":"2023-09-23T21:21:17.503007+0000","last_became_active":"2023-09-23T21:20:15.934725+0000","last_became_peered":"2023-09-23T21:20:15.934725+0000","last_unstale":"2023-09-23T21:21:17.503007+0000","last_undegraded":"2023-09-23T21:21:17.503007+0000","last_fullsized":"2023-09-23T21:21:17.503007+0000","mapping_epoch":681,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":682,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:55:09.651551+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":703,"reported_epoch":709,"state":"active+clean","last_fresh":"2023-09-23T21:21:18.509625+0000","last_change":"2023-09-23T21:20:15.927550+0000","last_active":"2023-09-23T21:21:18.509625+0000","last_peered":"2023-09-23T21:21:18.509625+0000","last_clean":"2023-09-23T21:21:18.509625+0000","last_became_active":"2023-09-23T21:20:15.927228+0000","last_became_peered":"2023-09-23T21:20:15.927228+0000","last_unstale":"2023-09-23T21:21:18.509625+0000","last_undegraded":"2023-09-23T21:21:18.509625+0000","last_fullsized":"2023-09-23T21:21:18.509625+0000","mapping_epoch":681,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":682,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:46:22.938004+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.5","version":"79'161","reported_seq":806,"reported_epoch":708,"state":"active+clean","last_fresh":"2023-09-23T21:21:17.506727+0000","last_change":"2023-09-23T21:20:53.938526+0000","last_active":"2023-09-23T21:21:17.506727+0000","last_peered":"2023-09-23T21:21:17.506727+0000","last_clean":"2023-09-23T21:21:17.506727+0000","last_became_active":"2023-09-23T21:20:53.938023+0000","last_became_peered":"2023-09-23T21:20:53.938023+0000","last_unstale":"2023-09-23T21:21:17.506727+0000","last_undegraded":"2023-09-23T21:21:17.506727+0000","last_fullsized":"2023-09-23T21:21:17.506727+0000","mapping_epoch":702,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":703,"parent":"0.0","parent_split_bits":3,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:15:41.108579+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:15:41.108579+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:37:01.378332+0000","scrub_duration":0.0061511960000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":490,"reported_epoch":708,"state":"active+clean","last_fresh":"2023-09-23T21:21:17.507181+0000","last_change":"2023-09-23T21:20:30.314590+0000","last_active":"2023-09-23T21:21:17.507181+0000","last_peered":"2023-09-23T21:21:17.507181+0000","last_clean":"2023-09-23T21:21:17.507181+0000","last_became_active":"2023-09-23T21:20:30.314211+0000","last_became_peered":"2023-09-23T21:20:30.314211+0000","last_unstale":"2023-09-23T21:21:17.507181+0000","last_undegraded":"2023-09-23T21:21:17.507181+0000","last_fullsized":"2023-09-23T21:21:17.507181+0000","mapping_epoch":689,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":690,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:18:59.216392+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"668'199","reported_seq":1349,"reported_epoch":708,"state":"active+clean","last_fresh":"2023-09-23T21:21:17.504341+0000","last_change":"2023-09-23T21:20:15.077518+0000","last_active":"2023-09-23T21:21:17.504341+0000","last_peered":"2023-09-23T21:21:17.504341+0000","last_clean":"2023-09-23T21:21:17.504341+0000","last_became_active":"2023-09-23T21:20:15.076836+0000","last_became_peered":"2023-09-23T21:20:15.076836+0000","last_unstale":"2023-09-23T21:21:17.504341+0000","last_undegraded":"2023-09-23T21:21:17.504341+0000","last_fullsized":"2023-09-23T21:21:17.504341+0000","mapping_epoch":680,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":681,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":199,"log_dups_size":0,"ondisk_log_size":199,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:44:53.169150+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00037772100000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1225,"num_read_kb":1302,"num_write":160,"num_write_kb":238,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":14,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":571,"reported_epoch":708,"state":"active+clean","last_fresh":"2023-09-23T21:21:17.505290+0000","last_change":"2023-09-23T21:16:42.463411+0000","last_active":"2023-09-23T21:21:17.505290+0000","last_peered":"2023-09-23T21:21:17.505290+0000","last_clean":"2023-09-23T21:21:17.505290+0000","last_became_active":"2023-09-23T21:16:42.463020+0000","last_became_peered":"2023-09-23T21:16:42.463020+0000","last_unstale":"2023-09-23T21:21:17.505290+0000","last_undegraded":"2023-09-23T21:21:17.505290+0000","last_fullsized":"2023-09-23T21:21:17.505290+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:07:34.346129+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.4","version":"79'161","reported_seq":1168,"reported_epoch":708,"state":"active+clean","last_fresh":"2023-09-23T21:21:17.505954+0000","last_change":"2023-09-23T21:20:19.086061+0000","last_active":"2023-09-23T21:21:17.505954+0000","last_peered":"2023-09-23T21:21:17.505954+0000","last_clean":"2023-09-23T21:21:17.505954+0000","last_became_active":"2023-09-23T21:20:19.085679+0000","last_became_peered":"2023-09-23T21:20:19.085679+0000","last_unstale":"2023-09-23T21:21:17.505954+0000","last_undegraded":"2023-09-23T21:21:17.505954+0000","last_fullsized":"2023-09-23T21:21:17.505954+0000","mapping_epoch":684,"log_start":"0'0","ondisk_log_start":"0'0","created":153,"last_epoch_clean":685,"parent":"0.0","parent_split_bits":3,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T20:57:20.012764+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T20:57:20.012764+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:44:58.774529+0000","scrub_duration":0.0045251919999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"668'91","reported_seq":71,"reported_epoch":708,"state":"active+clean","last_fresh":"2023-09-23T21:21:17.509250+0000","last_change":"2023-09-23T21:20:05.037623+0000","last_active":"2023-09-23T21:21:17.509250+0000","last_peered":"2023-09-23T21:21:17.509250+0000","last_clean":"2023-09-23T21:21:17.509250+0000","last_became_active":"2023-09-23T21:20:05.014215+0000","last_became_peered":"2023-09-23T21:20:05.014215+0000","last_unstale":"2023-09-23T21:21:17.509250+0000","last_undegraded":"2023-09-23T21:21:17.509250+0000","last_fullsized":"2023-09-23T21:21:17.509250+0000","mapping_epoch":672,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":673,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:15.541610+0000","last_clean_scrub_stamp":"2023-09-23T20:41:15.541610+0000","objects_scrubbed":0,"log_size":91,"log_dups_size":0,"ondisk_log_size":91,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:05:14.623182+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00051098199999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":410,"num_read_kb":1920,"num_write":77,"num_write_kb":1323,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"70'1","reported_seq":724,"reported_epoch":708,"state":"active+clean","last_fresh":"2023-09-23T21:21:17.902286+0000","last_change":"2023-09-23T21:20:12.101250+0000","last_active":"2023-09-23T21:21:17.902286+0000","last_peered":"2023-09-23T21:21:17.902286+0000","last_clean":"2023-09-23T21:21:17.902286+0000","last_became_active":"2023-09-23T21:20:12.079298+0000","last_became_peered":"2023-09-23T21:20:12.079298+0000","last_unstale":"2023-09-23T21:21:17.902286+0000","last_undegraded":"2023-09-23T21:21:17.902286+0000","last_fullsized":"2023-09-23T21:21:17.902286+0000","mapping_epoch":677,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":678,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:43:20.958541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":313,"reported_epoch":708,"state":"active+clean","last_fresh":"2023-09-23T21:21:17.505292+0000","last_change":"2023-09-23T21:20:15.926559+0000","last_active":"2023-09-23T21:21:17.505292+0000","last_peered":"2023-09-23T21:21:17.505292+0000","last_clean":"2023-09-23T21:21:17.505292+0000","last_became_active":"2023-09-23T21:20:15.926213+0000","last_became_peered":"2023-09-23T21:20:15.926213+0000","last_unstale":"2023-09-23T21:21:17.505292+0000","last_undegraded":"2023-09-23T21:21:17.505292+0000","last_fullsized":"2023-09-23T21:21:17.505292+0000","mapping_epoch":681,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":682,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:59:08.018105+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.19","version":"79'161","reported_seq":886,"reported_epoch":708,"state":"active+clean","last_fresh":"2023-09-23T21:21:17.506403+0000","last_change":"2023-09-23T21:20:32.918963+0000","last_active":"2023-09-23T21:21:17.506403+0000","last_peered":"2023-09-23T21:21:17.506403+0000","last_clean":"2023-09-23T21:21:17.506403+0000","last_became_active":"2023-09-23T21:20:30.311729+0000","last_became_peered":"2023-09-23T21:20:30.311729+0000","last_unstale":"2023-09-23T21:21:17.506403+0000","last_undegraded":"2023-09-23T21:21:17.506403+0000","last_fullsized":"2023-09-23T21:21:17.506403+0000","mapping_epoch":689,"log_start":"0'0","ondisk_log_start":"0'0","created":479,"last_epoch_clean":690,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:20:32.918868+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:20:32.918868+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:30:10.488726+0000","scrub_duration":0.001861538,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":239,"reported_epoch":708,"state":"active+clean","last_fresh":"2023-09-23T21:21:17.901826+0000","last_change":"2023-09-23T21:16:42.434830+0000","last_active":"2023-09-23T21:21:17.901826+0000","last_peered":"2023-09-23T21:21:17.901826+0000","last_clean":"2023-09-23T21:21:17.901826+0000","last_became_active":"2023-09-23T21:16:42.430289+0000","last_became_peered":"2023-09-23T21:16:42.430289+0000","last_unstale":"2023-09-23T21:21:17.901826+0000","last_undegraded":"2023-09-23T21:21:17.901826+0000","last_fullsized":"2023-09-23T21:21:17.901826+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:38:48.053766+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":224,"reported_epoch":708,"state":"active+clean","last_fresh":"2023-09-23T21:21:17.508589+0000","last_change":"2023-09-23T21:20:12.108763+0000","last_active":"2023-09-23T21:21:17.508589+0000","last_peered":"2023-09-23T21:21:17.508589+0000","last_clean":"2023-09-23T21:21:17.508589+0000","last_became_active":"2023-09-23T21:20:12.108295+0000","last_became_peered":"2023-09-23T21:20:12.108295+0000","last_unstale":"2023-09-23T21:21:17.508589+0000","last_undegraded":"2023-09-23T21:21:17.508589+0000","last_fullsized":"2023-09-23T21:21:17.508589+0000","mapping_epoch":677,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":678,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:44:58.483044+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.1e","version":"79'161","reported_seq":711,"reported_epoch":709,"state":"active+clean","last_fresh":"2023-09-23T21:21:18.508545+0000","last_change":"2023-09-23T21:20:08.012623+0000","last_active":"2023-09-23T21:21:18.508545+0000","last_peered":"2023-09-23T21:21:18.508545+0000","last_clean":"2023-09-23T21:21:18.508545+0000","last_became_active":"2023-09-23T21:20:08.012212+0000","last_became_peered":"2023-09-23T21:20:08.012212+0000","last_unstale":"2023-09-23T21:21:18.508545+0000","last_undegraded":"2023-09-23T21:21:18.508545+0000","last_fullsized":"2023-09-23T21:21:18.508545+0000","mapping_epoch":675,"log_start":"0'0","ondisk_log_start":"0'0","created":479,"last_epoch_clean":676,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:15:53.061319+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:15:53.061319+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:46:55.339646+0000","scrub_duration":0.007429878,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":347,"reported_epoch":708,"state":"active+clean","last_fresh":"2023-09-23T21:21:17.507149+0000","last_change":"2023-09-23T21:20:42.351206+0000","last_active":"2023-09-23T21:21:17.507149+0000","last_peered":"2023-09-23T21:21:17.507149+0000","last_clean":"2023-09-23T21:21:17.507149+0000","last_became_active":"2023-09-23T21:20:42.350828+0000","last_became_peered":"2023-09-23T21:20:42.350828+0000","last_unstale":"2023-09-23T21:21:17.507149+0000","last_undegraded":"2023-09-23T21:21:17.507149+0000","last_fullsized":"2023-09-23T21:21:17.507149+0000","mapping_epoch":694,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":695,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:37:53.392509+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":9,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":283,"reported_epoch":708,"state":"active+clean","last_fresh":"2023-09-23T21:21:17.505654+0000","last_change":"2023-09-23T21:20:12.104580+0000","last_active":"2023-09-23T21:21:17.505654+0000","last_peered":"2023-09-23T21:21:17.505654+0000","last_clean":"2023-09-23T21:21:17.505654+0000","last_became_active":"2023-09-23T21:20:12.104238+0000","last_became_peered":"2023-09-23T21:20:12.104238+0000","last_unstale":"2023-09-23T21:21:17.505654+0000","last_undegraded":"2023-09-23T21:21:17.505654+0000","last_fullsized":"2023-09-23T21:21:17.505654+0000","mapping_epoch":677,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":678,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:10:20.352583+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":541,"reported_epoch":708,"state":"active+clean","last_fresh":"2023-09-23T21:21:17.507901+0000","last_change":"2023-09-23T21:20:50.933085+0000","last_active":"2023-09-23T21:21:17.507901+0000","last_peered":"2023-09-23T21:21:17.507901+0000","last_clean":"2023-09-23T21:21:17.507901+0000","last_became_active":"2023-09-23T21:20:50.926757+0000","last_became_peered":"2023-09-23T21:20:50.926757+0000","last_unstale":"2023-09-23T21:21:17.507901+0000","last_undegraded":"2023-09-23T21:21:17.507901+0000","last_fullsized":"2023-09-23T21:21:17.507901+0000","mapping_epoch":699,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":700,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:48:16.967780+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":283,"reported_epoch":708,"state":"active+clean","last_fresh":"2023-09-23T21:21:17.505418+0000","last_change":"2023-09-23T21:13:12.216602+0000","last_active":"2023-09-23T21:21:17.505418+0000","last_peered":"2023-09-23T21:21:17.505418+0000","last_clean":"2023-09-23T21:21:17.505418+0000","last_became_active":"2023-09-23T21:13:12.216204+0000","last_became_peered":"2023-09-23T21:13:12.216204+0000","last_unstale":"2023-09-23T21:21:17.505418+0000","last_undegraded":"2023-09-23T21:21:17.505418+0000","last_fullsized":"2023-09-23T21:21:17.505418+0000","mapping_epoch":493,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:23:37.043390+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1c","version":"79'161","reported_seq":1003,"reported_epoch":708,"state":"active+clean","last_fresh":"2023-09-23T21:21:17.506201+0000","last_change":"2023-09-23T21:20:48.511669+0000","last_active":"2023-09-23T21:21:17.506201+0000","last_peered":"2023-09-23T21:21:17.506201+0000","last_clean":"2023-09-23T21:21:17.506201+0000","last_became_active":"2023-09-23T21:20:48.511243+0000","last_became_peered":"2023-09-23T21:20:48.511243+0000","last_unstale":"2023-09-23T21:21:17.506201+0000","last_undegraded":"2023-09-23T21:21:17.506201+0000","last_fullsized":"2023-09-23T21:21:17.506201+0000","mapping_epoch":697,"log_start":"0'0","ondisk_log_start":"0'0","created":479,"last_epoch_clean":698,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:15:54.019828+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:15:54.019828+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:15:41.480572+0000","scrub_duration":0.0020962530000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,1],"acting":[7,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":472,"reported_epoch":708,"state":"active+clean","last_fresh":"2023-09-23T21:21:17.507349+0000","last_change":"2023-09-23T21:20:50.929346+0000","last_active":"2023-09-23T21:21:17.507349+0000","last_peered":"2023-09-23T21:21:17.507349+0000","last_clean":"2023-09-23T21:21:17.507349+0000","last_became_active":"2023-09-23T21:20:50.928917+0000","last_became_peered":"2023-09-23T21:20:50.928917+0000","last_unstale":"2023-09-23T21:21:17.507349+0000","last_undegraded":"2023-09-23T21:21:17.507349+0000","last_fullsized":"2023-09-23T21:21:17.507349+0000","mapping_epoch":699,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":700,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:08:56.715607+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":453,"reported_epoch":708,"state":"active+clean","last_fresh":"2023-09-23T21:21:17.506539+0000","last_change":"2023-09-23T21:20:30.311794+0000","last_active":"2023-09-23T21:21:17.506539+0000","last_peered":"2023-09-23T21:21:17.506539+0000","last_clean":"2023-09-23T21:21:17.506539+0000","last_became_active":"2023-09-23T21:20:30.311484+0000","last_became_peered":"2023-09-23T21:20:30.311484+0000","last_unstale":"2023-09-23T21:21:17.506539+0000","last_undegraded":"2023-09-23T21:21:17.506539+0000","last_fullsized":"2023-09-23T21:21:17.506539+0000","mapping_epoch":689,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":690,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:47:43.155883+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1d","version":"79'161","reported_seq":722,"reported_epoch":708,"state":"active+clean","last_fresh":"2023-09-23T21:21:17.503536+0000","last_change":"2023-09-23T21:20:17.475961+0000","last_active":"2023-09-23T21:21:17.503536+0000","last_peered":"2023-09-23T21:21:17.503536+0000","last_clean":"2023-09-23T21:21:17.503536+0000","last_became_active":"2023-09-23T21:20:15.080849+0000","last_became_peered":"2023-09-23T21:20:15.080849+0000","last_unstale":"2023-09-23T21:21:17.503536+0000","last_undegraded":"2023-09-23T21:21:17.503536+0000","last_fullsized":"2023-09-23T21:21:17.503536+0000","mapping_epoch":680,"log_start":"0'0","ondisk_log_start":"0'0","created":479,"last_epoch_clean":681,"parent":"0.0","parent_split_bits":5,"last_scrub":"79'161","last_scrub_stamp":"2023-09-23T21:20:17.475859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:38:57.324224+0000","last_clean_scrub_stamp":"2023-09-23T21:20:17.475859+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:35:49.456854+0000","scrub_duration":0.0021360649999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":283,"reported_epoch":708,"state":"active+clean","last_fresh":"2023-09-23T21:21:17.901779+0000","last_change":"2023-09-23T21:20:15.942285+0000","last_active":"2023-09-23T21:21:17.901779+0000","last_peered":"2023-09-23T21:21:17.901779+0000","last_clean":"2023-09-23T21:21:17.901779+0000","last_became_active":"2023-09-23T21:20:15.932312+0000","last_became_peered":"2023-09-23T21:20:15.932312+0000","last_unstale":"2023-09-23T21:21:17.901779+0000","last_undegraded":"2023-09-23T21:21:17.901779+0000","last_fullsized":"2023-09-23T21:21:17.901779+0000","mapping_epoch":681,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":682,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:57.875777+0000","last_clean_scrub_stamp":"2023-09-23T20:41:57.875777+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:56:55.450387+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":17,"num_bytes_recovered":4986,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":303,"reported_epoch":708,"state":"active+clean","last_fresh":"2023-09-23T21:21:17.505222+0000","last_change":"2023-09-23T21:20:50.930893+0000","last_active":"2023-09-23T21:21:17.505222+0000","last_peered":"2023-09-23T21:21:17.505222+0000","last_clean":"2023-09-23T21:21:17.505222+0000","last_became_active":"2023-09-23T21:20:50.930601+0000","last_became_peered":"2023-09-23T21:20:50.930601+0000","last_unstale":"2023-09-23T21:21:17.505222+0000","last_undegraded":"2023-09-23T21:21:17.505222+0000","last_fullsized":"2023-09-23T21:21:17.505222+0000","mapping_epoch":699,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":700,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:58.880246+0000","last_clean_scrub_stamp":"2023-09-23T20:41:58.880246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:00:50.010273+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":104,"num_bytes_recovered":24922,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":14322,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":31,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":853056,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4991,"ondisk_log_size":4991,"up":93,"acting":93,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5995,"num_read_kb":13212,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":55,"num_bytes_recovered":8388646,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201774,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":665700,"kb_used_data":7452,"kb_used_omap":0,"kb_used_meta":658240,"kb_avail":93079452,"statfs":{"total":95995035648,"available":95313358848,"internally_reserved":0,"allocated":7630848,"data_stored":5848948,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":674037760},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:21:28 2023","interfaces":[{"interface":"back","average":{"1min":0.56100000000000005,"5min":0.872,"15min":1.746},"min":{"1min":0.318,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":0.72099999999999997,"5min":3.968,"15min":37.207999999999998},"last":0.57399999999999995},{"interface":"front","average":{"1min":0.51700000000000002,"5min":0.96099999999999997,"15min":1.7529999999999999},"min":{"1min":0.29099999999999998,"5min":0.222,"15min":0.222},"max":{"1min":0.70499999999999996,"5min":5.2910000000000004,"15min":37.122},"last":0.60499999999999998}]},{"osd":1,"last update":"Sat Sep 23 21:21:23 2023","interfaces":[{"interface":"back","average":{"1min":0.55900000000000005,"5min":0.84099999999999997,"15min":1.486},"min":{"1min":0.32100000000000001,"5min":0.27600000000000002,"15min":0.26700000000000002},"max":{"1min":0.72999999999999998,"5min":6.9130000000000003,"15min":14.034000000000001},"last":0.64700000000000002},{"interface":"front","average":{"1min":0.64900000000000002,"5min":0.95899999999999996,"15min":1.496},"min":{"1min":0.35299999999999998,"5min":0.29999999999999999,"15min":0.26300000000000001},"max":{"1min":1.8859999999999999,"5min":5.4039999999999999,"15min":11.226000000000001},"last":0.63200000000000001}]},{"osd":2,"last update":"Sat Sep 23 21:20:41 2023","interfaces":[{"interface":"back","average":{"1min":0.58599999999999997,"5min":3.6200000000000001,"15min":2.3220000000000001},"min":{"1min":0.377,"5min":0.27200000000000002,"15min":0.26200000000000001},"max":{"1min":1.276,"5min":166.696,"15min":166.696},"last":0.501},{"interface":"front","average":{"1min":0.626,"5min":3.52,"15min":2.2730000000000001},"min":{"1min":0.46300000000000002,"5min":0.28799999999999998,"15min":0.26700000000000002},"max":{"1min":1.1479999999999999,"5min":165.57499999999999,"15min":165.57499999999999},"last":0.70599999999999996}]},{"osd":3,"last update":"Sat Sep 23 21:21:09 2023","interfaces":[{"interface":"back","average":{"1min":0.61299999999999999,"5min":1.2749999999999999,"15min":1.6539999999999999},"min":{"1min":0.42899999999999999,"5min":0.317,"15min":0.30099999999999999},"max":{"1min":0.77500000000000002,"5min":8.6259999999999994,"15min":9.1449999999999996},"last":0.59499999999999997},{"interface":"front","average":{"1min":0.622,"5min":1.234,"15min":1.7050000000000001},"min":{"1min":0.45200000000000001,"5min":0.28799999999999998,"15min":0.28699999999999998},"max":{"1min":0.83799999999999997,"5min":15.82,"15min":15.82},"last":0.68200000000000005}]},{"osd":4,"last update":"Sat Sep 23 21:21:09 2023","interfaces":[{"interface":"back","average":{"1min":0.56000000000000005,"5min":0.60099999999999998,"15min":1.113},"min":{"1min":0.33000000000000002,"5min":0.251,"15min":0.251},"max":{"1min":0.94499999999999995,"5min":3.9990000000000001,"15min":11.297000000000001},"last":0.73299999999999998},{"interface":"front","average":{"1min":0.60199999999999998,"5min":0.67000000000000004,"15min":1.0589999999999999},"min":{"1min":0.35999999999999999,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":0.92100000000000004,"5min":5.6349999999999998,"15min":12.752000000000001},"last":0.53500000000000003}]},{"osd":5,"last update":"Sat Sep 23 21:21:09 2023","interfaces":[{"interface":"back","average":{"1min":0.59699999999999998,"5min":0.73899999999999999,"15min":1.05},"min":{"1min":0.38700000000000001,"5min":0.28199999999999997,"15min":0.26000000000000001},"max":{"1min":0.80900000000000005,"5min":6.9740000000000002,"15min":9.4789999999999992},"last":0.55600000000000005},{"interface":"front","average":{"1min":0.64900000000000002,"5min":0.81699999999999995,"15min":1.133},"min":{"1min":0.41999999999999998,"5min":0.32000000000000001,"15min":0.26200000000000001},"max":{"1min":0.89700000000000002,"5min":6.7439999999999998,"15min":11.257},"last":0.66600000000000004}]},{"osd":6,"last update":"Sat Sep 23 21:21:23 2023","interfaces":[{"interface":"back","average":{"1min":0.63900000000000001,"5min":0.68100000000000005,"15min":1.0129999999999999},"min":{"1min":0.39500000000000002,"5min":0.312,"15min":0.30099999999999999},"max":{"1min":0.97199999999999998,"5min":5.7069999999999999,"15min":10.941000000000001},"last":0.72099999999999997},{"interface":"front","average":{"1min":0.69999999999999996,"5min":0.83299999999999996,"15min":1.1279999999999999},"min":{"1min":0.437,"5min":0.35099999999999998,"15min":0.33100000000000002},"max":{"1min":1,"5min":5.6760000000000002,"15min":12.715},"last":0.746}]}]},{"osd":6,"up_from":164,"seq":704374636858,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":280468,"kb_used_data":11596,"kb_used_omap":10,"kb_used_meta":268853,"kb_avail":93464684,"statfs":{"total":95995035648,"available":95707836416,"internally_reserved":0,"allocated":11874304,"data_stored":10107852,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":10328,"internal_metadata":275306408},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:21:26 2023","interfaces":[{"interface":"back","average":{"1min":0.54600000000000004,"5min":0.99299999999999999,"15min":0.93300000000000005},"min":{"1min":0.33900000000000002,"5min":0.20599999999999999,"15min":0.20599999999999999},"max":{"1min":0.82299999999999995,"5min":6.4539999999999997,"15min":6.4539999999999997},"last":0.48399999999999999},{"interface":"front","average":{"1min":0.54900000000000004,"5min":1.012,"15min":1.238},"min":{"1min":0.27600000000000002,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":0.70399999999999996,"5min":11.823,"15min":11.823},"last":0.45000000000000001}]},{"osd":1,"last update":"Sat Sep 23 21:21:19 2023","interfaces":[{"interface":"back","average":{"1min":0.54500000000000004,"5min":0.86699999999999999,"15min":1.4179999999999999},"min":{"1min":0.32400000000000001,"5min":0.28100000000000003,"15min":0.20300000000000001},"max":{"1min":0.79900000000000004,"5min":6.8490000000000002,"15min":12.904999999999999},"last":0.32700000000000001},{"interface":"front","average":{"1min":0.61599999999999999,"5min":0.78900000000000003,"15min":1.284},"min":{"1min":0.46999999999999997,"5min":0.26300000000000001,"15min":0.24099999999999999},"max":{"1min":0.85999999999999999,"5min":4.4000000000000004,"15min":8.9830000000000005},"last":0.42599999999999999}]},{"osd":2,"last update":"Sat Sep 23 21:21:26 2023","interfaces":[{"interface":"back","average":{"1min":0.56499999999999995,"5min":1.0660000000000001,"15min":1.6200000000000001},"min":{"1min":0.34000000000000002,"5min":0.27000000000000002,"15min":0.252},"max":{"1min":0.77700000000000002,"5min":7.117,"15min":23.954999999999998},"last":0.40400000000000003},{"interface":"front","average":{"1min":0.62,"5min":1.1910000000000001,"15min":1.554},"min":{"1min":0.36299999999999999,"5min":0.32000000000000001,"15min":0.251},"max":{"1min":0.82099999999999995,"5min":7.258,"15min":13.385999999999999},"last":0.371}]},{"osd":3,"last update":"Sat Sep 23 21:21:26 2023","interfaces":[{"interface":"back","average":{"1min":0.60499999999999998,"5min":1.7270000000000001,"15min":1.7909999999999999},"min":{"1min":0.40899999999999997,"5min":0.32500000000000001,"15min":0.28100000000000003},"max":{"1min":1.571,"5min":56.460000000000001,"15min":56.460000000000001},"last":0.52900000000000003},{"interface":"front","average":{"1min":0.626,"5min":1.651,"15min":1.944},"min":{"1min":0.45700000000000002,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":0.92000000000000004,"5min":56.494999999999997,"15min":56.494999999999997},"last":0.55300000000000005}]},{"osd":4,"last update":"Sat Sep 23 21:21:26 2023","interfaces":[{"interface":"back","average":{"1min":0.61299999999999999,"5min":0.65200000000000002,"15min":1.0329999999999999},"min":{"1min":0.34599999999999997,"5min":0.26500000000000001,"15min":0.22800000000000001},"max":{"1min":0.86399999999999999,"5min":4.1520000000000001,"15min":11.800000000000001},"last":0.51600000000000001},{"interface":"front","average":{"1min":0.63600000000000001,"5min":0.627,"15min":0.93700000000000006},"min":{"1min":0.31,"5min":0.27700000000000002,"15min":0.27400000000000002},"max":{"1min":1.0249999999999999,"5min":4.2530000000000001,"15min":13.920999999999999},"last":0.59499999999999997}]},{"osd":5,"last update":"Sat Sep 23 21:21:26 2023","interfaces":[{"interface":"back","average":{"1min":0.67200000000000004,"5min":0.71499999999999997,"15min":1.0680000000000001},"min":{"1min":0.43099999999999999,"5min":0.30299999999999999,"15min":0.27100000000000002},"max":{"1min":0.93799999999999994,"5min":4.2999999999999998,"15min":7.9710000000000001},"last":0.54200000000000004},{"interface":"front","average":{"1min":0.64800000000000002,"5min":0.74099999999999999,"15min":0.92700000000000005},"min":{"1min":0.36699999999999999,"5min":0.33900000000000002,"15min":0.30299999999999999},"max":{"1min":0.99099999999999999,"5min":5.2789999999999999,"15min":9.016},"last":0.56399999999999995}]},{"osd":7,"last update":"Sat Sep 23 21:21:26 2023","interfaces":[{"interface":"back","average":{"1min":0.67300000000000004,"5min":0.79400000000000004,"15min":1.8580000000000001},"min":{"1min":0.42399999999999999,"5min":0.32500000000000001,"15min":0.28199999999999997},"max":{"1min":0.89500000000000002,"5min":4.117,"15min":205.95599999999999},"last":0.629},{"interface":"front","average":{"1min":0.65500000000000003,"5min":0.64800000000000002,"15min":1.8500000000000001},"min":{"1min":0.41599999999999998,"5min":0.32300000000000001,"15min":0.308},"max":{"1min":1.008,"5min":3.355,"15min":206.05699999999999},"last":0.57999999999999996}]}]},{"osd":1,"up_from":350,"seq":1503238553790,"num_pgs":42,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":369840,"kb_used_data":12648,"kb_used_omap":10,"kb_used_meta":357173,"kb_avail":93375312,"statfs":{"total":95995035648,"available":95616319488,"internally_reserved":0,"allocated":12951552,"data_stored":9946541,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":10498,"internal_metadata":365745918},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:21:30 2023","interfaces":[{"interface":"back","average":{"1min":0.33300000000000002,"5min":1.216,"15min":1.028},"min":{"1min":0.25700000000000001,"5min":0.21199999999999999,"15min":0.21199999999999999},"max":{"1min":0.46400000000000002,"5min":13.323,"15min":33.429000000000002},"last":0.28999999999999998},{"interface":"front","average":{"1min":0.42299999999999999,"5min":0.92000000000000004,"15min":1.1259999999999999},"min":{"1min":0.26100000000000001,"5min":0.23400000000000001,"15min":0.221},"max":{"1min":0.71499999999999997,"5min":9.1890000000000001,"15min":34.432000000000002},"last":0.42899999999999999}]},{"osd":2,"last update":"Sat Sep 23 21:21:30 2023","interfaces":[{"interface":"back","average":{"1min":0.374,"5min":1.4259999999999999,"15min":1.7589999999999999},"min":{"1min":0.27200000000000002,"5min":0.27200000000000002,"15min":0.19900000000000001},"max":{"1min":0.51800000000000002,"5min":13.368,"15min":34.387},"last":0.32500000000000001},{"interface":"front","average":{"1min":0.44,"5min":1.3660000000000001,"15min":1.7829999999999999},"min":{"1min":0.33700000000000002,"5min":0.27700000000000002,"15min":0.214},"max":{"1min":0.68700000000000006,"5min":6.9859999999999998,"15min":38.573999999999998},"last":0.47599999999999998}]},{"osd":3,"last update":"Sat Sep 23 21:21:30 2023","interfaces":[{"interface":"back","average":{"1min":0.44500000000000001,"5min":1.1279999999999999,"15min":1.7150000000000001},"min":{"1min":0.29399999999999998,"5min":0.28000000000000003,"15min":0.24099999999999999},"max":{"1min":0.60899999999999999,"5min":5.0179999999999998,"15min":17.055},"last":0.55400000000000005},{"interface":"front","average":{"1min":0.432,"5min":1.2569999999999999,"15min":1.772},"min":{"1min":0.29999999999999999,"5min":0.25900000000000001,"15min":0.25600000000000001},"max":{"1min":0.65000000000000002,"5min":8.7590000000000003,"15min":15.237},"last":0.65000000000000002}]},{"osd":4,"last update":"Sat Sep 23 21:21:30 2023","interfaces":[{"interface":"back","average":{"1min":0.63200000000000001,"5min":1.0569999999999999,"15min":1.319},"min":{"1min":0.495,"5min":0.27000000000000002,"15min":0.26300000000000001},"max":{"1min":0.90300000000000002,"5min":13.250999999999999,"15min":13.477},"last":0.499},{"interface":"front","average":{"1min":0.66400000000000003,"5min":0.95399999999999996,"15min":1.4830000000000001},"min":{"1min":0.50700000000000001,"5min":0.34599999999999997,"15min":0.313},"max":{"1min":0.995,"5min":4.173,"15min":16.030000000000001},"last":0.51600000000000001}]},{"osd":5,"last update":"Sat Sep 23 21:21:30 2023","interfaces":[{"interface":"back","average":{"1min":0.59699999999999998,"5min":0.99399999999999999,"15min":1.5580000000000001},"min":{"1min":0.44400000000000001,"5min":0.36199999999999999,"15min":0.29799999999999999},"max":{"1min":0.89000000000000001,"5min":4.923,"15min":14.813000000000001},"last":0.44500000000000001},{"interface":"front","average":{"1min":0.64200000000000002,"5min":1.083,"15min":1.4510000000000001},"min":{"1min":0.45500000000000002,"5min":0.32600000000000001,"15min":0.32500000000000001},"max":{"1min":0.95899999999999996,"5min":13.182,"15min":13.182},"last":0.53500000000000003}]},{"osd":6,"last update":"Sat Sep 23 21:21:30 2023","interfaces":[{"interface":"back","average":{"1min":0.61699999999999999,"5min":1.073,"15min":1.5349999999999999},"min":{"1min":0.40100000000000002,"5min":0.34899999999999998,"15min":0.33900000000000002},"max":{"1min":0.94999999999999996,"5min":6.0010000000000003,"15min":14.859999999999999},"last":0.40100000000000002},{"interface":"front","average":{"1min":0.63300000000000001,"5min":1.159,"15min":1.472},"min":{"1min":0.51900000000000002,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":0.878,"5min":13.287000000000001,"15min":16.494},"last":0.57699999999999996}]},{"osd":7,"last update":"Sat Sep 23 21:21:30 2023","interfaces":[{"interface":"back","average":{"1min":0.68100000000000005,"5min":1.482,"15min":1.7150000000000001},"min":{"1min":0.51700000000000002,"5min":0.38500000000000001,"15min":0.35299999999999998},"max":{"1min":0.95799999999999996,"5min":46.856999999999999,"15min":46.856999999999999},"last":0.72099999999999997},{"interface":"front","average":{"1min":0.65500000000000003,"5min":1.6970000000000001,"15min":1.875},"min":{"1min":0.53400000000000003,"5min":0.40500000000000003,"15min":0.39700000000000002},"max":{"1min":0.92100000000000004,"5min":46.912999999999997,"15min":46.912999999999997},"last":0.62}]}]},{"osd":0,"up_from":493,"seq":2117418877030,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":438500,"kb_used_data":8988,"kb_used_omap":36,"kb_used_meta":429467,"kb_avail":93306652,"statfs":{"total":95995035648,"available":95546011648,"internally_reserved":0,"allocated":9203712,"data_stored":6093432,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":37336,"internal_metadata":439774760},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 21:20:31 2023","interfaces":[{"interface":"back","average":{"1min":0.39200000000000002,"5min":1.1759999999999999,"15min":0.95899999999999996},"min":{"1min":0.255,"5min":0.214,"15min":0.214},"max":{"1min":0.59199999999999997,"5min":13.779999999999999,"15min":13.779999999999999},"last":0.34599999999999997},{"interface":"front","average":{"1min":0.36499999999999999,"5min":1.5600000000000001,"15min":1.0860000000000001},"min":{"1min":0.27500000000000002,"5min":0.23300000000000001,"15min":0.20799999999999999},"max":{"1min":0.54500000000000004,"5min":10.551,"15min":11.964},"last":0.40300000000000002}]},{"osd":2,"last update":"Sat Sep 23 21:20:31 2023","interfaces":[{"interface":"back","average":{"1min":0.41599999999999998,"5min":1.9870000000000001,"15min":1.165},"min":{"1min":0.29299999999999998,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":0.71699999999999997,"5min":35.68,"15min":35.68},"last":0.47099999999999997},{"interface":"front","average":{"1min":0.46500000000000002,"5min":2.6080000000000001,"15min":1.5600000000000001},"min":{"1min":0.307,"5min":0.28499999999999998,"15min":0.27900000000000003},"max":{"1min":0.84199999999999997,"5min":35.832999999999998,"15min":35.832999999999998},"last":0.42799999999999999}]},{"osd":3,"last update":"Sat Sep 23 21:20:31 2023","interfaces":[{"interface":"back","average":{"1min":0.40500000000000003,"5min":2.5070000000000001,"15min":1.482},"min":{"1min":0.28999999999999998,"5min":0.247,"15min":0.23400000000000001},"max":{"1min":0.50900000000000001,"5min":37.079999999999998,"15min":37.079999999999998},"last":0.45800000000000002},{"interface":"front","average":{"1min":0.42899999999999999,"5min":1.978,"15min":1.425},"min":{"1min":0.309,"5min":0.28899999999999998,"15min":0.26200000000000001},"max":{"1min":0.56100000000000005,"5min":32.935000000000002,"15min":32.935000000000002},"last":0.378}]},{"osd":4,"last update":"Sat Sep 23 21:20:31 2023","interfaces":[{"interface":"back","average":{"1min":0.58399999999999996,"5min":1.5600000000000001,"15min":1.2609999999999999},"min":{"1min":0.42799999999999999,"5min":0.34000000000000002,"15min":0.30199999999999999},"max":{"1min":0.73099999999999998,"5min":22.780000000000001,"15min":22.780000000000001},"last":0.70999999999999996},{"interface":"front","average":{"1min":0.59599999999999997,"5min":1.3240000000000001,"15min":1.1359999999999999},"min":{"1min":0.433,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":0.81799999999999995,"5min":22.652000000000001,"15min":22.652000000000001},"last":0.57599999999999996}]},{"osd":5,"last update":"Sat Sep 23 21:20:31 2023","interfaces":[{"interface":"back","average":{"1min":0.625,"5min":1.3380000000000001,"15min":1.2809999999999999},"min":{"1min":0.439,"5min":0.312,"15min":0.29999999999999999},"max":{"1min":0.80900000000000005,"5min":8.9329999999999998,"15min":8.9329999999999998},"last":0.745},{"interface":"front","average":{"1min":0.58099999999999996,"5min":1.5029999999999999,"15min":1.232},"min":{"1min":0.42399999999999999,"5min":0.377,"15min":0.313},"max":{"1min":0.81699999999999995,"5min":15.321999999999999,"15min":15.321999999999999},"last":0.621}]},{"osd":6,"last update":"Sat Sep 23 21:20:31 2023","interfaces":[{"interface":"back","average":{"1min":0.65400000000000003,"5min":1.226,"15min":1.2070000000000001},"min":{"1min":0.47899999999999998,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":0.876,"5min":6.9409999999999998,"15min":7.2050000000000001},"last":0.61099999999999999},{"interface":"front","average":{"1min":0.61199999999999999,"5min":1.3169999999999999,"15min":1.3280000000000001},"min":{"1min":0.48499999999999999,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":0.79600000000000004,"5min":8.8889999999999993,"15min":8.8889999999999993},"last":0.77400000000000002}]},{"osd":7,"last update":"Sat Sep 23 21:20:31 2023","interfaces":[{"interface":"back","average":{"1min":0.59499999999999997,"5min":1.4570000000000001,"15min":1.355},"min":{"1min":0.46000000000000002,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":0.79500000000000004,"5min":15.273999999999999,"15min":15.273999999999999},"last":0.58999999999999997},{"interface":"front","average":{"1min":0.68200000000000005,"5min":1.548,"15min":1.323},"min":{"1min":0.52600000000000002,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":0.89800000000000002,"5min":33.779000000000003,"15min":33.779000000000003},"last":0.75800000000000001}]}]},{"osd":2,"up_from":93,"seq":399431958902,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1044848,"kb_used_data":9000,"kb_used_omap":81,"kb_used_meta":1035758,"kb_avail":92700304,"statfs":{"total":95995035648,"available":94925111296,"internally_reserved":0,"allocated":9216000,"data_stored":6030839,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":83750,"internal_metadata":1060616410},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:20:29 2023","interfaces":[{"interface":"back","average":{"1min":0.52700000000000002,"5min":1.7809999999999999,"15min":1.2410000000000001},"min":{"1min":0.27500000000000002,"5min":0.187,"15min":0.187},"max":{"1min":3.2320000000000002,"5min":13.25,"15min":13.25},"last":0.44700000000000001},{"interface":"front","average":{"1min":0.42099999999999999,"5min":1.6000000000000001,"15min":1.71},"min":{"1min":0.25900000000000001,"5min":0.20200000000000001,"15min":0.20200000000000001},"max":{"1min":1.3580000000000001,"5min":9.8849999999999998,"15min":18.111999999999998},"last":0.29499999999999998}]},{"osd":1,"last update":"Sat Sep 23 21:20:29 2023","interfaces":[{"interface":"back","average":{"1min":0.56000000000000005,"5min":1.6759999999999999,"15min":2.0390000000000001},"min":{"1min":0.28699999999999998,"5min":0.25800000000000001,"15min":0.188},"max":{"1min":3.1920000000000002,"5min":10.535,"15min":34.396999999999998},"last":0.39700000000000002},{"interface":"front","average":{"1min":0.58999999999999997,"5min":1.532,"15min":1.972},"min":{"1min":0.30499999999999999,"5min":0.24099999999999999,"15min":0.217},"max":{"1min":3.911,"5min":8.5510000000000002,"15min":34.462000000000003},"last":0.33800000000000002}]},{"osd":3,"last update":"Sat Sep 23 21:21:10 2023","interfaces":[{"interface":"back","average":{"1min":0.36199999999999999,"5min":1.7849999999999999,"15min":1.9139999999999999},"min":{"1min":0.253,"5min":0.248,"15min":0.20599999999999999},"max":{"1min":0.496,"5min":20.106999999999999,"15min":35.549999999999997},"last":0.63100000000000001},{"interface":"front","average":{"1min":0.42199999999999999,"5min":2.1019999999999999,"15min":2.2240000000000002},"min":{"1min":0.34399999999999997,"5min":0.33700000000000002,"15min":0.21299999999999999},"max":{"1min":0.60299999999999998,"5min":20.239000000000001,"15min":36.951999999999998},"last":0.55000000000000004}]},{"osd":4,"last update":"Sat Sep 23 21:20:29 2023","interfaces":[{"interface":"back","average":{"1min":0.55800000000000005,"5min":1.171,"15min":1.365},"min":{"1min":0.39300000000000002,"5min":0.27000000000000002,"15min":0.249},"max":{"1min":0.78900000000000003,"5min":7.577,"15min":10.266999999999999},"last":0.56699999999999995},{"interface":"front","average":{"1min":0.72099999999999997,"5min":1.423,"15min":1.597},"min":{"1min":0.44400000000000001,"5min":0.30299999999999999,"15min":0.26700000000000002},"max":{"1min":3.1280000000000001,"5min":10.459,"15min":12.49},"last":0.69499999999999995}]},{"osd":5,"last update":"Sat Sep 23 21:20:29 2023","interfaces":[{"interface":"back","average":{"1min":0.55800000000000005,"5min":2.2810000000000001,"15min":1.7629999999999999},"min":{"1min":0.40899999999999997,"5min":0.29099999999999998,"15min":0.28000000000000003},"max":{"1min":0.74199999999999999,"5min":102.56100000000001,"15min":102.56100000000001},"last":0.66800000000000004},{"interface":"front","average":{"1min":0.73399999999999999,"5min":2.4409999999999998,"15min":1.8959999999999999},"min":{"1min":0.442,"5min":0.34899999999999998,"15min":0.29399999999999998},"max":{"1min":3.1629999999999998,"5min":102.28700000000001,"15min":102.28700000000001},"last":0.57799999999999996}]},{"osd":6,"last update":"Sat Sep 23 21:20:29 2023","interfaces":[{"interface":"back","average":{"1min":0.71999999999999997,"5min":1.591,"15min":1.7809999999999999},"min":{"1min":0.48999999999999999,"5min":0.377,"15min":0.35699999999999998},"max":{"1min":2.4649999999999999,"5min":10.335000000000001,"15min":12.518000000000001},"last":0.64800000000000002},{"interface":"front","average":{"1min":0.73999999999999999,"5min":1.5569999999999999,"15min":1.9099999999999999},"min":{"1min":0.48899999999999999,"5min":0.40200000000000002,"15min":0.35599999999999998},"max":{"1min":2.387,"5min":10.381,"15min":12.544},"last":0.59499999999999997}]},{"osd":7,"last update":"Sat Sep 23 21:20:29 2023","interfaces":[{"interface":"back","average":{"1min":0.66800000000000004,"5min":1.29,"15min":1.554},"min":{"1min":0.45700000000000002,"5min":0.31,"15min":0.29799999999999999},"max":{"1min":1.6870000000000001,"5min":7.5819999999999999,"15min":18.021999999999998},"last":0.61499999999999999},{"interface":"front","average":{"1min":0.67700000000000005,"5min":1.415,"15min":1.6299999999999999},"min":{"1min":0.42499999999999999,"5min":0.34999999999999998,"15min":0.32200000000000001},"max":{"1min":2.0489999999999999,"5min":7.46,"15min":10.92},"last":0.60199999999999998}]}]},{"osd":3,"up_from":98,"seq":420906795376,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1126488,"kb_used_data":8976,"kb_used_omap":7,"kb_used_meta":1117496,"kb_avail":92618664,"statfs":{"total":95995035648,"available":94841511936,"internally_reserved":0,"allocated":9191424,"data_stored":6087814,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7576,"internal_metadata":1144316520},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:21:23 2023","interfaces":[{"interface":"back","average":{"1min":0.40200000000000002,"5min":1.98,"15min":1.6779999999999999},"min":{"1min":0.25600000000000001,"5min":0.18099999999999999,"15min":0.18099999999999999},"max":{"1min":0.89300000000000002,"5min":81.408000000000001,"15min":147.38900000000001},"last":0.39700000000000002},{"interface":"front","average":{"1min":0.39500000000000002,"5min":1.532,"15min":1.6439999999999999},"min":{"1min":0.26400000000000001,"5min":0.214,"15min":0.214},"max":{"1min":0.54300000000000004,"5min":82.397000000000006,"15min":149.989},"last":0.34799999999999998}]},{"osd":1,"last update":"Sat Sep 23 21:21:23 2023","interfaces":[{"interface":"back","average":{"1min":1.333,"5min":1.476,"15min":1.9199999999999999},"min":{"1min":0.252,"5min":0.23400000000000001,"15min":0.20000000000000001},"max":{"1min":19.132000000000001,"5min":19.132000000000001,"15min":22.577999999999999},"last":0.33100000000000002},{"interface":"front","average":{"1min":0.42799999999999999,"5min":0.98199999999999998,"15min":1.6719999999999999},"min":{"1min":0.30099999999999999,"5min":0.255,"15min":0.216},"max":{"1min":0.623,"5min":6.1219999999999999,"15min":21.617999999999999},"last":0.443}]},{"osd":2,"last update":"Sat Sep 23 21:21:06 2023","interfaces":[{"interface":"back","average":{"1min":0.73699999999999999,"5min":1.024,"15min":1.7629999999999999},"min":{"1min":0.28399999999999997,"5min":0.23799999999999999,"15min":0.20899999999999999},"max":{"1min":6.3170000000000002,"5min":11.321,"15min":56.317999999999998},"last":0.38900000000000001},{"interface":"front","average":{"1min":0.73599999999999999,"5min":2.0139999999999998,"15min":2.1659999999999999},"min":{"1min":0.371,"5min":0.23400000000000001,"15min":0.216},"max":{"1min":6.2910000000000004,"5min":17.423999999999999,"15min":55.988},"last":0.41299999999999998}]},{"osd":4,"last update":"Sat Sep 23 21:21:06 2023","interfaces":[{"interface":"back","average":{"1min":0.83399999999999996,"5min":0.85699999999999998,"15min":1.3819999999999999},"min":{"1min":0.42099999999999999,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":5.1580000000000004,"5min":10.475,"15min":21.478999999999999},"last":0.63400000000000001},{"interface":"front","average":{"1min":0.77000000000000002,"5min":1.28,"15min":1.597},"min":{"1min":0.40600000000000003,"5min":0.311,"15min":0.28499999999999998},"max":{"1min":4.8920000000000003,"5min":11.202,"15min":22.402000000000001},"last":0.75}]},{"osd":5,"last update":"Sat Sep 23 21:21:06 2023","interfaces":[{"interface":"back","average":{"1min":0.83199999999999996,"5min":0.93899999999999995,"15min":1.4650000000000001},"min":{"1min":0.443,"5min":0.313,"15min":0.311},"max":{"1min":5.1959999999999997,"5min":10.516,"15min":21.562000000000001},"last":0.70299999999999996},{"interface":"front","average":{"1min":0.74099999999999999,"5min":1.016,"15min":1.522},"min":{"1min":0.44900000000000001,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":3.0899999999999999,"5min":5.7300000000000004,"15min":14.029},"last":0.81499999999999995}]},{"osd":6,"last update":"Sat Sep 23 21:21:23 2023","interfaces":[{"interface":"back","average":{"1min":1.5329999999999999,"5min":1.4119999999999999,"15min":1.762},"min":{"1min":0.44800000000000001,"5min":0.38100000000000001,"15min":0.34999999999999998},"max":{"1min":19.094999999999999,"5min":19.094999999999999,"15min":22.446999999999999},"last":0.64500000000000002},{"interface":"front","average":{"1min":0.65600000000000003,"5min":1.0229999999999999,"15min":1.575},"min":{"1min":0.46800000000000003,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":0.81399999999999995,"5min":7.5949999999999998,"15min":12.375999999999999},"last":0.78900000000000003}]},{"osd":7,"last update":"Sat Sep 23 21:21:06 2023","interfaces":[{"interface":"back","average":{"1min":0.86499999999999999,"5min":1.0149999999999999,"15min":1.5960000000000001},"min":{"1min":0.47299999999999998,"5min":0.41099999999999998,"15min":0.33900000000000002},"max":{"1min":5.1120000000000001,"5min":10.547000000000001,"15min":21.518999999999998},"last":0.72399999999999998},{"interface":"front","average":{"1min":0.78700000000000003,"5min":1.143,"15min":1.706},"min":{"1min":0.50700000000000001,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":3.145,"5min":5.5119999999999996,"15min":9.9879999999999995},"last":0.83699999999999997}]}]},{"osd":4,"up_from":31,"seq":133143986682,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":596128,"kb_used_data":7192,"kb_used_omap":0,"kb_used_meta":588928,"kb_avail":93149024,"statfs":{"total":95995035648,"available":95384600576,"internally_reserved":0,"allocated":7364608,"data_stored":5586513,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":603062272},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:21:26 2023","interfaces":[{"interface":"back","average":{"1min":0.51200000000000001,"5min":1.3320000000000001,"15min":1.103},"min":{"1min":0.39500000000000002,"5min":0.23699999999999999,"15min":0.22500000000000001},"max":{"1min":0.70899999999999996,"5min":28.683,"15min":28.683},"last":0.58599999999999997},{"interface":"front","average":{"1min":0.54200000000000004,"5min":1.647,"15min":1.2829999999999999},"min":{"1min":0.36399999999999999,"5min":0.249,"15min":0.23799999999999999},"max":{"1min":0.88200000000000001,"5min":29.155000000000001,"15min":29.155000000000001},"last":0.60499999999999998}]},{"osd":1,"last update":"Sat Sep 23 21:21:26 2023","interfaces":[{"interface":"back","average":{"1min":0.59399999999999997,"5min":1.0820000000000001,"15min":1.4670000000000001},"min":{"1min":0.40400000000000003,"5min":0.28499999999999998,"15min":0.221},"max":{"1min":0.86799999999999999,"5min":8.1020000000000003,"15min":37.960000000000001},"last":0.63900000000000001},{"interface":"front","average":{"1min":0.60799999999999998,"5min":1.034,"15min":1.4630000000000001},"min":{"1min":0.36799999999999999,"5min":0.33100000000000002,"15min":0.27000000000000002},"max":{"1min":0.92000000000000004,"5min":8.1509999999999998,"15min":37.933},"last":0.55100000000000005}]},{"osd":2,"last update":"Sat Sep 23 21:21:26 2023","interfaces":[{"interface":"back","average":{"1min":0.59099999999999997,"5min":1.5409999999999999,"15min":1.6120000000000001},"min":{"1min":0.42099999999999999,"5min":0.28399999999999997,"15min":0.247},"max":{"1min":0.77300000000000002,"5min":17.602,"15min":17.602},"last":0.624},{"interface":"front","average":{"1min":0.56499999999999995,"5min":1.29,"15min":1.7030000000000001},"min":{"1min":0.38100000000000001,"5min":0.28799999999999998,"15min":0.27800000000000002},"max":{"1min":0.80300000000000005,"5min":13.439,"15min":14.417999999999999},"last":0.78000000000000003}]},{"osd":3,"last update":"Sat Sep 23 21:21:01 2023","interfaces":[{"interface":"back","average":{"1min":0.61099999999999999,"5min":1.887,"15min":1.8220000000000001},"min":{"1min":0.37,"5min":0.29499999999999998,"15min":0.25600000000000001},"max":{"1min":0.79400000000000004,"5min":67.105999999999995,"15min":67.105999999999995},"last":0.49199999999999999},{"interface":"front","average":{"1min":0.70599999999999996,"5min":1.865,"15min":1.8819999999999999},"min":{"1min":0.46600000000000003,"5min":0.33600000000000002,"15min":0.32200000000000001},"max":{"1min":0.92100000000000004,"5min":67.046999999999997,"15min":67.046999999999997},"last":0.66700000000000004}]},{"osd":5,"last update":"Sat Sep 23 21:21:20 2023","interfaces":[{"interface":"back","average":{"1min":0.56799999999999995,"5min":0.61599999999999999,"15min":1.0369999999999999},"min":{"1min":0.39000000000000001,"5min":0.254,"15min":0.222},"max":{"1min":0.90100000000000002,"5min":2.923,"15min":14.398999999999999},"last":0.70999999999999996},{"interface":"front","average":{"1min":0.54300000000000004,"5min":0.59799999999999998,"15min":0.94199999999999995},"min":{"1min":0.35299999999999998,"5min":0.25800000000000001,"15min":0.255},"max":{"1min":0.68999999999999995,"5min":3.6000000000000001,"15min":9.8979999999999997},"last":0.51500000000000001}]},{"osd":6,"last update":"Sat Sep 23 21:21:26 2023","interfaces":[{"interface":"back","average":{"1min":0.57799999999999996,"5min":0.70099999999999996,"15min":1.0609999999999999},"min":{"1min":0.36799999999999999,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":0.82999999999999996,"5min":4.7270000000000003,"15min":24.452999999999999},"last":0.57199999999999995},{"interface":"front","average":{"1min":0.71099999999999997,"5min":0.69599999999999995,"15min":1.02},"min":{"1min":0.48799999999999999,"5min":0.36399999999999999,"15min":0.32500000000000001},"max":{"1min":0.94699999999999995,"5min":2.7069999999999999,"15min":14.387},"last":0.81599999999999995}]},{"osd":7,"last update":"Sat Sep 23 21:21:01 2023","interfaces":[{"interface":"back","average":{"1min":0.66300000000000003,"5min":0.745,"15min":1.1080000000000001},"min":{"1min":0.376,"5min":0.33000000000000002,"15min":0.30099999999999999},"max":{"1min":0.998,"5min":5.7789999999999999,"15min":16.890999999999998},"last":0.74399999999999999},{"interface":"front","average":{"1min":0.621,"5min":0.624,"15min":1.002},"min":{"1min":0.38900000000000001,"5min":0.29199999999999998,"15min":0.27400000000000002},"max":{"1min":0.86099999999999999,"5min":3.681,"15min":8.5830000000000002},"last":0.65100000000000002}]}]},{"osd":5,"up_from":39,"seq":167503725048,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":622872,"kb_used_data":11536,"kb_used_omap":0,"kb_used_meta":611328,"kb_avail":93122280,"statfs":{"total":95995035648,"available":95357214720,"internally_reserved":0,"allocated":11812864,"data_stored":10039268,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":625999872},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:21:27 2023","interfaces":[{"interface":"back","average":{"1min":0.55400000000000005,"5min":1.653,"15min":1.1000000000000001},"min":{"1min":0.38800000000000001,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":0.72499999999999998,"5min":88.272999999999996,"15min":88.272999999999996},"last":0.46100000000000002},{"interface":"front","average":{"1min":0.54100000000000004,"5min":1.8260000000000001,"15min":1.179},"min":{"1min":0.40500000000000003,"5min":0.215,"15min":0.215},"max":{"1min":0.73499999999999999,"5min":88.296999999999997,"15min":88.296999999999997},"last":0.505}]},{"osd":1,"last update":"Sat Sep 23 21:21:27 2023","interfaces":[{"interface":"back","average":{"1min":0.58999999999999997,"5min":1.1100000000000001,"15min":1.4139999999999999},"min":{"1min":0.42599999999999999,"5min":0.27900000000000003,"15min":0.23499999999999999},"max":{"1min":0.80200000000000005,"5min":6.4160000000000004,"15min":25.239999999999998},"last":0.63100000000000001},{"interface":"front","average":{"1min":0.63600000000000001,"5min":1.0069999999999999,"15min":1.5449999999999999},"min":{"1min":0.47299999999999998,"5min":0.32900000000000001,"15min":0.252},"max":{"1min":0.86699999999999999,"5min":6.3579999999999997,"15min":25.253},"last":0.60099999999999998}]},{"osd":2,"last update":"Sat Sep 23 21:20:37 2023","interfaces":[{"interface":"back","average":{"1min":0.505,"5min":1.3839999999999999,"15min":1.53},"min":{"1min":0.35799999999999998,"5min":0.27300000000000002,"15min":0.25800000000000001},"max":{"1min":0.64300000000000002,"5min":9.6560000000000006,"15min":12.634},"last":0.61299999999999999},{"interface":"front","average":{"1min":0.54000000000000004,"5min":1.4410000000000001,"15min":1.48},"min":{"1min":0.28899999999999998,"5min":0.28899999999999998,"15min":0.25700000000000001},"max":{"1min":0.69199999999999995,"5min":9.4060000000000006,"15min":9.7170000000000005},"last":0.56200000000000006}]},{"osd":3,"last update":"Sat Sep 23 21:20:59 2023","interfaces":[{"interface":"back","average":{"1min":0.66300000000000003,"5min":1.256,"15min":1.71},"min":{"1min":0.41599999999999998,"5min":0.314,"15min":0.29299999999999998},"max":{"1min":1.3009999999999999,"5min":7.3700000000000001,"15min":25.503},"last":0.65400000000000003},{"interface":"front","average":{"1min":0.66000000000000003,"5min":1.544,"15min":1.8959999999999999},"min":{"1min":0.46000000000000002,"5min":0.311,"15min":0.30599999999999999},"max":{"1min":1.6599999999999999,"5min":12.382,"15min":25.466000000000001},"last":0.67400000000000004}]},{"osd":4,"last update":"Sat Sep 23 21:21:18 2023","interfaces":[{"interface":"back","average":{"1min":0.58299999999999996,"5min":0.66700000000000004,"15min":1.0409999999999999},"min":{"1min":0.35199999999999998,"5min":0.26100000000000001,"15min":0.253},"max":{"1min":0.85799999999999998,"5min":5.0279999999999996,"15min":12.776},"last":0.70199999999999996},{"interface":"front","average":{"1min":0.60499999999999998,"5min":0.68100000000000005,"15min":1.077},"min":{"1min":0.41199999999999998,"5min":0.246,"15min":0.246},"max":{"1min":0.90000000000000002,"5min":5.149,"15min":11.973000000000001},"last":0.80700000000000005}]},{"osd":6,"last update":"Sat Sep 23 21:21:27 2023","interfaces":[{"interface":"back","average":{"1min":0.61299999999999999,"5min":0.85099999999999998,"15min":0.96199999999999997},"min":{"1min":0.35999999999999999,"5min":0.27400000000000002,"15min":0.25900000000000001},"max":{"1min":0.86899999999999999,"5min":5.8520000000000003,"15min":8.0909999999999993},"last":0.77800000000000002},{"interface":"front","average":{"1min":0.69299999999999995,"5min":0.84599999999999997,"15min":1.1319999999999999},"min":{"1min":0.40000000000000002,"5min":0.36399999999999999,"15min":0.317},"max":{"1min":0.94599999999999995,"5min":7.1470000000000002,"15min":10.999000000000001},"last":0.73799999999999999}]},{"osd":7,"last update":"Sat Sep 23 21:20:59 2023","interfaces":[{"interface":"back","average":{"1min":0.58799999999999997,"5min":0.79600000000000004,"15min":1.756},"min":{"1min":0.32900000000000001,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":0.92900000000000005,"5min":4.2629999999999999,"15min":189.405},"last":0.83899999999999997},{"interface":"front","average":{"1min":0.64500000000000002,"5min":0.80900000000000005,"15min":1.9219999999999999},"min":{"1min":0.34499999999999997,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":0.91700000000000004,"5min":5.0910000000000002,"15min":189.685},"last":0.82799999999999996}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8488,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1259,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T21:21:36.822 INFO:tasks.cephadm.ceph_manager.ceph:recovered! 2023-09-23T21:21:36.822 ERROR:teuthology.run_tasks:Saw exception from tasks. Traceback (most recent call last): File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/run_tasks.py", line 105, in run_tasks manager = run_one_task(taskname, ctx=ctx, config=config) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/run_tasks.py", line 83, in run_one_task return task(**kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/task/parallel.py", line 56, in task p.spawn(_run_spawned, ctx, confg, taskname) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/parallel.py", line 84, in __exit__ for result in self: File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/parallel.py", line 98, in __next__ resurrect_traceback(result) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/parallel.py", line 30, in resurrect_traceback raise exc.exc_info[1] File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/parallel.py", line 23, in capture_traceback return func(*args, **kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/task/parallel.py", line 64, in _run_spawned mgr = run_tasks.run_one_task(taskname, ctx=ctx, config=config) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/run_tasks.py", line 83, in run_one_task return task(**kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/task/sequential.py", line 47, in task mgr = run_tasks.run_one_task(taskname, ctx=ctx, config=confg) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/run_tasks.py", line 83, in run_one_task return task(**kwargs) File "/home/teuthworker/src/github.com_ceph_ceph_bb0daeff97f020f686596fca6eba20c310e26fb7/qa/tasks/workunit.py", line 119, in task assert 'client' in role AssertionError 2023-09-23T21:21:36.894 ERROR:teuthology.util.sentry: Sentry event: https://sentry.ceph.com/organizations/ceph/?query=40a38caae7284f1cac6fe0ed040e8d56 Traceback (most recent call last): File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/run_tasks.py", line 105, in run_tasks manager = run_one_task(taskname, ctx=ctx, config=config) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/run_tasks.py", line 83, in run_one_task return task(**kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/task/parallel.py", line 56, in task p.spawn(_run_spawned, ctx, confg, taskname) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/parallel.py", line 84, in __exit__ for result in self: File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/parallel.py", line 98, in __next__ resurrect_traceback(result) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/parallel.py", line 30, in resurrect_traceback raise exc.exc_info[1] File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/parallel.py", line 23, in capture_traceback return func(*args, **kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/task/parallel.py", line 64, in _run_spawned mgr = run_tasks.run_one_task(taskname, ctx=ctx, config=config) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/run_tasks.py", line 83, in run_one_task return task(**kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/task/sequential.py", line 47, in task mgr = run_tasks.run_one_task(taskname, ctx=ctx, config=confg) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/run_tasks.py", line 83, in run_one_task return task(**kwargs) File "/home/teuthworker/src/github.com_ceph_ceph_bb0daeff97f020f686596fca6eba20c310e26fb7/qa/tasks/workunit.py", line 119, in task assert 'client' in role AssertionError 2023-09-23T21:21:36.896 DEBUG:teuthology.run_tasks:Unwinding manager cephadm 2023-09-23T21:21:36.906 INFO:tasks.cephadm:Teardown begin 2023-09-23T21:21:36.907 DEBUG:teuthology.orchestra.run.smithi026:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2023-09-23T21:21:36.974 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:21:36 smithi026 ceph-mon[179688]: pgmap v1722: 103 pgs: 103 active+clean; 4.6 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail 2023-09-23T21:21:36.975 DEBUG:teuthology.orchestra.run.smithi070:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2023-09-23T21:21:37.007 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:21:36 smithi070 ceph-mon[168282]: pgmap v1722: 103 pgs: 103 active+clean; 4.6 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail 2023-09-23T21:21:37.039 INFO:tasks.cephadm:Cleaning up testdir ceph.* files... 2023-09-23T21:21:37.040 DEBUG:teuthology.orchestra.run.smithi026:> rm -f /home/ubuntu/cephtest/seed.ceph.conf /home/ubuntu/cephtest/ceph.pub 2023-09-23T21:21:37.057 DEBUG:teuthology.orchestra.run.smithi070:> rm -f /home/ubuntu/cephtest/seed.ceph.conf /home/ubuntu/cephtest/ceph.pub 2023-09-23T21:21:37.101 INFO:tasks.cephadm:Stopping all daemons... 2023-09-23T21:21:37.101 INFO:tasks.cephadm.mon.a:Stopping mon.a... 2023-09-23T21:21:37.101 DEBUG:teuthology.orchestra.run.smithi026:> sudo systemctl stop ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7@mon.a 2023-09-23T21:21:37.381 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:21:37 smithi026 systemd[1]: Stopping Ceph mon.a for b3f8e4ec-5a50-11ee-8db4-212e2dc638e7... 2023-09-23T21:21:37.791 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:21:37 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mon-a[175127]: 2023-09-23T21:21:37.458+0000 7f1e32c92700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-mon -n mon.a -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2023-09-23T21:21:37.791 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:21:37 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mon-a[175127]: 2023-09-23T21:21:37.458+0000 7f1e32c92700 -1 mon.a@0(leader) e4 *** Got Signal Terminated *** 2023-09-23T21:21:37.792 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 21:21:37 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[164029]: [23/Sep/2023:21:21:37] ENGINE Bus STOPPING 2023-09-23T21:21:37.792 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 21:21:37 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[164029]: [23/Sep/2023:21:21:37] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2023-09-23T21:21:37.792 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 21:21:37 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[164029]: [23/Sep/2023:21:21:37] ENGINE Bus STOPPED 2023-09-23T21:21:37.792 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 21:21:37 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[164029]: [23/Sep/2023:21:21:37] ENGINE Bus STARTING 2023-09-23T21:21:37.792 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 21:21:37 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[164029]: [23/Sep/2023:21:21:37] ENGINE Serving on http://:::9283 2023-09-23T21:21:37.792 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 21:21:37 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[164029]: [23/Sep/2023:21:21:37] ENGINE Bus STARTED 2023-09-23T21:21:38.010 DEBUG:teuthology.orchestra.run.smithi026:> sudo pkill -f 'journalctl -f -n 0 -u ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7@mon.a.service' 2023-09-23T21:21:38.086 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-09-23T21:21:38.086 INFO:tasks.cephadm.mon.a:Stopped mon.a 2023-09-23T21:21:38.087 INFO:tasks.cephadm.mon.b:Stopping mon.c... 2023-09-23T21:21:38.087 DEBUG:teuthology.orchestra.run.smithi026:> sudo systemctl stop ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7@mon.c 2023-09-23T21:21:38.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:21:38 smithi026 systemd[1]: Stopping Ceph mon.c for b3f8e4ec-5a50-11ee-8db4-212e2dc638e7... 2023-09-23T21:21:38.749 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:21:38 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mon-c[179643]: 2023-09-23T21:21:38.463+0000 7fafd42d6700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-mon -n mon.c -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2023-09-23T21:21:38.750 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:21:38 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mon-c[179643]: 2023-09-23T21:21:38.463+0000 7fafd42d6700 -1 mon.c@2(peon) e4 *** Got Signal Terminated *** 2023-09-23T21:21:38.750 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 21:21:38 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[164029]: [23/Sep/2023:21:21:38] ENGINE Bus STOPPING 2023-09-23T21:21:38.750 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 21:21:38 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[164029]: [23/Sep/2023:21:21:38] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2023-09-23T21:21:38.750 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 21:21:38 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[164029]: [23/Sep/2023:21:21:38] ENGINE Bus STOPPED 2023-09-23T21:21:38.751 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 21:21:38 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[164029]: [23/Sep/2023:21:21:38] ENGINE Bus STARTING 2023-09-23T21:21:38.751 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 21:21:38 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[164029]: [23/Sep/2023:21:21:38] ENGINE Serving on http://:::9283 2023-09-23T21:21:38.751 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 21:21:38 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[164029]: [23/Sep/2023:21:21:38] ENGINE Bus STARTED 2023-09-23T21:21:39.012 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:21:38 smithi026 bash[479234]: ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mon-c 2023-09-23T21:21:39.018 DEBUG:teuthology.orchestra.run.smithi026:> sudo pkill -f 'journalctl -f -n 0 -u ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7@mon.c.service' 2023-09-23T21:21:39.058 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-09-23T21:21:39.058 INFO:tasks.cephadm.mon.b:Stopped mon.c 2023-09-23T21:21:39.058 INFO:tasks.cephadm.mon.b:Stopping mon.b... 2023-09-23T21:21:39.059 DEBUG:teuthology.orchestra.run.smithi070:> sudo systemctl stop ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7@mon.b 2023-09-23T21:21:39.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:21:39 smithi070 systemd[1]: Stopping Ceph mon.b for b3f8e4ec-5a50-11ee-8db4-212e2dc638e7... 2023-09-23T21:21:39.706 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:21:39 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mon-b[168258]: 2023-09-23T21:21:39.410+0000 7f8ce3c2d700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-mon -n mon.b -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2023-09-23T21:21:39.706 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:21:39 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mon-b[168258]: 2023-09-23T21:21:39.410+0000 7f8ce3c2d700 -1 mon.b@1(peon) e4 *** Got Signal Terminated *** 2023-09-23T21:21:39.882 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 21:21:39 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[164029]: ::ffff:172.21.15.70 - - [23/Sep/2023:21:21:39] "GET /metrics HTTP/1.1" 200 34002 "" "Prometheus/2.43.0" 2023-09-23T21:21:40.265 DEBUG:teuthology.orchestra.run.smithi070:> sudo pkill -f 'journalctl -f -n 0 -u ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7@mon.b.service' 2023-09-23T21:21:40.290 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:21:40 smithi070 bash[308400]: ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mon-b 2023-09-23T21:21:40.290 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:21:40 smithi070 systemd[1]: ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7@mon.b.service: Succeeded. 2023-09-23T21:21:40.290 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:21:40 smithi070 systemd[1]: Stopped Ceph mon.b for b3f8e4ec-5a50-11ee-8db4-212e2dc638e7. 2023-09-23T21:21:40.304 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-09-23T21:21:40.305 INFO:tasks.cephadm.mon.b:Stopped mon.b 2023-09-23T21:21:40.305 INFO:tasks.cephadm.mgr.y:Stopping mgr.y... 2023-09-23T21:21:40.305 DEBUG:teuthology.orchestra.run.smithi026:> sudo systemctl stop ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7@mgr.y 2023-09-23T21:21:40.631 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 21:21:40 smithi026 systemd[1]: Stopping Ceph mgr.y for b3f8e4ec-5a50-11ee-8db4-212e2dc638e7... 2023-09-23T21:21:40.966 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 21:21:40 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[164029]: 2023-09-23T21:21:40.654+0000 7fd5da336700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-mgr -n mgr.y -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2023-09-23T21:21:40.966 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 21:21:40 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y[164029]: 2023-09-23T21:21:40.654+0000 7fd5da336700 -1 mgr handle_mgr_signal *** Got signal Terminated *** 2023-09-23T21:21:41.215 DEBUG:teuthology.orchestra.run.smithi026:> sudo pkill -f 'journalctl -f -n 0 -u ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7@mgr.y.service' 2023-09-23T21:21:41.241 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 21:21:40 smithi026 bash[479389]: ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-mgr-y 2023-09-23T21:21:41.241 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 21:21:41 smithi026 systemd[1]: ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7@mgr.y.service: Succeeded. 2023-09-23T21:21:41.241 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 21:21:41 smithi026 systemd[1]: Stopped Ceph mgr.y for b3f8e4ec-5a50-11ee-8db4-212e2dc638e7. 2023-09-23T21:21:41.254 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-09-23T21:21:41.254 INFO:tasks.cephadm.mgr.y:Stopped mgr.y 2023-09-23T21:21:41.254 INFO:tasks.cephadm.mgr.x:Stopping mgr.x... 2023-09-23T21:21:41.255 DEBUG:teuthology.orchestra.run.smithi070:> sudo systemctl stop ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7@mgr.x 2023-09-23T21:21:41.620 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 21:21:41 smithi070 systemd[1]: Stopping Ceph mgr.x for b3f8e4ec-5a50-11ee-8db4-212e2dc638e7... 2023-09-23T21:21:42.147 DEBUG:teuthology.orchestra.run.smithi070:> sudo pkill -f 'journalctl -f -n 0 -u ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7@mgr.x.service' 2023-09-23T21:21:42.183 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-09-23T21:21:42.184 INFO:tasks.cephadm.mgr.x:Stopped mgr.x 2023-09-23T21:21:42.184 INFO:tasks.cephadm.osd.0:Stopping osd.0... 2023-09-23T21:21:42.184 DEBUG:teuthology.orchestra.run.smithi026:> sudo systemctl stop ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7@osd.0 2023-09-23T21:21:42.568 INFO:journalctl@ceph.osd.0.smithi026.stdout:Sep 23 21:21:42 smithi026 systemd[1]: Stopping Ceph osd.0 for b3f8e4ec-5a50-11ee-8db4-212e2dc638e7... 2023-09-23T21:21:42.881 INFO:journalctl@ceph.osd.0.smithi026.stdout:Sep 23 21:21:42 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd-0[399811]: 2023-09-23T21:21:42.566+0000 7fe49d712700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-osd -n osd.0 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2023-09-23T21:21:42.882 INFO:journalctl@ceph.osd.0.smithi026.stdout:Sep 23 21:21:42 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd-0[399811]: 2023-09-23T21:21:42.566+0000 7fe49d712700 -1 osd.0 709 *** Got signal Terminated *** 2023-09-23T21:21:42.882 INFO:journalctl@ceph.osd.0.smithi026.stdout:Sep 23 21:21:42 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd-0[399811]: 2023-09-23T21:21:42.566+0000 7fe49d712700 -1 osd.0 709 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-09-23T21:21:47.119 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 21:21:46 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-prometheus-a[155975]: ts=2023-09-23T21:21:46.658Z caller=refresh.go:99 level=error component="discovery manager scrape" discovery=http config=node msg="Unable to refresh target groups" err="Get \"http://172.21.15.26:8765/sd/prometheus/sd-config?service=node-exporter\": dial tcp 172.21.15.26:8765: connect: connection refused" 2023-09-23T21:21:47.119 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 21:21:46 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-prometheus-a[155975]: ts=2023-09-23T21:21:46.658Z caller=refresh.go:99 level=error component="discovery manager notify" discovery=http config=config-0 msg="Unable to refresh target groups" err="Get \"http://172.21.15.26:8765/sd/prometheus/sd-config?service=alertmanager\": dial tcp 172.21.15.26:8765: connect: connection refused" 2023-09-23T21:21:47.120 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 21:21:46 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-prometheus-a[155975]: ts=2023-09-23T21:21:46.661Z caller=refresh.go:99 level=error component="discovery manager scrape" discovery=http config=ceph msg="Unable to refresh target groups" err="Get \"http://172.21.15.26:8765/sd/prometheus/sd-config?service=mgr-prometheus\": dial tcp 172.21.15.26:8765: connect: connection refused" 2023-09-23T21:21:47.120 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 21:21:46 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-prometheus-a[155975]: ts=2023-09-23T21:21:46.661Z caller=refresh.go:99 level=error component="discovery manager scrape" discovery=http config=ceph-exporter msg="Unable to refresh target groups" err="Get \"http://172.21.15.26:8765/sd/prometheus/sd-config?service=ceph-exporter\": dial tcp 172.21.15.26:8765: connect: connection refused" 2023-09-23T21:21:48.431 INFO:journalctl@ceph.osd.0.smithi026.stdout:Sep 23 21:21:48 smithi026 bash[479530]: ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd-0 2023-09-23T21:21:49.809 DEBUG:teuthology.orchestra.run.smithi026:> sudo pkill -f 'journalctl -f -n 0 -u ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7@osd.0.service' 2023-09-23T21:21:49.853 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-09-23T21:21:49.853 INFO:tasks.cephadm.osd.0:Stopped osd.0 2023-09-23T21:21:49.853 INFO:tasks.cephadm.osd.1:Stopping osd.1... 2023-09-23T21:21:49.853 DEBUG:teuthology.orchestra.run.smithi026:> sudo systemctl stop ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7@osd.1 2023-09-23T21:21:50.132 INFO:journalctl@ceph.osd.1.smithi026.stdout:Sep 23 21:21:49 smithi026 systemd[1]: Stopping Ceph osd.1 for b3f8e4ec-5a50-11ee-8db4-212e2dc638e7... 2023-09-23T21:21:50.632 INFO:journalctl@ceph.osd.1.smithi026.stdout:Sep 23 21:21:50 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd-1[340180]: 2023-09-23T21:21:50.228+0000 7f49c5449700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-osd -n osd.1 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2023-09-23T21:21:50.632 INFO:journalctl@ceph.osd.1.smithi026.stdout:Sep 23 21:21:50 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd-1[340180]: 2023-09-23T21:21:50.228+0000 7f49c5449700 -1 osd.1 709 *** Got signal Terminated *** 2023-09-23T21:21:50.632 INFO:journalctl@ceph.osd.1.smithi026.stdout:Sep 23 21:21:50 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd-1[340180]: 2023-09-23T21:21:50.228+0000 7f49c5449700 -1 osd.1 709 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-09-23T21:21:56.382 INFO:journalctl@ceph.osd.1.smithi026.stdout:Sep 23 21:21:56 smithi026 bash[479794]: ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd-1 2023-09-23T21:21:57.615 DEBUG:teuthology.orchestra.run.smithi026:> sudo pkill -f 'journalctl -f -n 0 -u ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7@osd.1.service' 2023-09-23T21:21:57.658 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-09-23T21:21:57.658 INFO:tasks.cephadm.osd.1:Stopped osd.1 2023-09-23T21:21:57.659 INFO:tasks.cephadm.osd.2:Stopping osd.2... 2023-09-23T21:21:57.659 DEBUG:teuthology.orchestra.run.smithi026:> sudo systemctl stop ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7@osd.2 2023-09-23T21:21:57.882 INFO:journalctl@ceph.osd.2.smithi026.stdout:Sep 23 21:21:57 smithi026 systemd[1]: Stopping Ceph osd.2 for b3f8e4ec-5a50-11ee-8db4-212e2dc638e7... 2023-09-23T21:21:58.381 INFO:journalctl@ceph.osd.2.smithi026.stdout:Sep 23 21:21:58 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd-2[202510]: 2023-09-23T21:21:58.056+0000 7f127a582700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-osd -n osd.2 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2023-09-23T21:21:58.382 INFO:journalctl@ceph.osd.2.smithi026.stdout:Sep 23 21:21:58 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd-2[202510]: 2023-09-23T21:21:58.056+0000 7f127a582700 -1 osd.2 709 *** Got signal Terminated *** 2023-09-23T21:21:58.382 INFO:journalctl@ceph.osd.2.smithi026.stdout:Sep 23 21:21:58 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd-2[202510]: 2023-09-23T21:21:58.056+0000 7f127a582700 -1 osd.2 709 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-09-23T21:22:03.881 INFO:journalctl@ceph.osd.2.smithi026.stdout:Sep 23 21:22:03 smithi026 bash[480076]: ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd-2 2023-09-23T21:22:05.106 DEBUG:teuthology.orchestra.run.smithi026:> sudo pkill -f 'journalctl -f -n 0 -u ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7@osd.2.service' 2023-09-23T21:22:05.148 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-09-23T21:22:05.148 INFO:tasks.cephadm.osd.2:Stopped osd.2 2023-09-23T21:22:05.149 INFO:tasks.cephadm.osd.3:Stopping osd.3... 2023-09-23T21:22:05.149 DEBUG:teuthology.orchestra.run.smithi026:> sudo systemctl stop ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7@osd.3 2023-09-23T21:22:05.382 INFO:journalctl@ceph.osd.3.smithi026.stdout:Sep 23 21:22:05 smithi026 systemd[1]: Stopping Ceph osd.3 for b3f8e4ec-5a50-11ee-8db4-212e2dc638e7... 2023-09-23T21:22:05.882 INFO:journalctl@ceph.osd.3.smithi026.stdout:Sep 23 21:22:05 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd-3[208592]: 2023-09-23T21:22:05.534+0000 7fc5249fb700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-osd -n osd.3 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2023-09-23T21:22:05.882 INFO:journalctl@ceph.osd.3.smithi026.stdout:Sep 23 21:22:05 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd-3[208592]: 2023-09-23T21:22:05.534+0000 7fc5249fb700 -1 osd.3 709 *** Got signal Terminated *** 2023-09-23T21:22:05.882 INFO:journalctl@ceph.osd.3.smithi026.stdout:Sep 23 21:22:05 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd-3[208592]: 2023-09-23T21:22:05.534+0000 7fc5249fb700 -1 osd.3 709 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-09-23T21:22:09.120 INFO:journalctl@ceph.osd.7.smithi070.stdout:Sep 23 21:22:08 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd-7[128949]: 2023-09-23T21:22:08.970+0000 7f12105aa700 -1 osd.7 709 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2023-09-23T21:21:46.720306+0000 front 2023-09-23T21:21:46.720441+0000 (oldest deadline 2023-09-23T21:22:08.420326+0000) 2023-09-23T21:22:09.120 INFO:journalctl@ceph.osd.5.smithi070.stdout:Sep 23 21:22:08 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd-5[113356]: 2023-09-23T21:22:08.748+0000 7ff1a3eca700 -1 osd.5 709 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2023-09-23T21:21:43.545440+0000 front 2023-09-23T21:21:43.545514+0000 (oldest deadline 2023-09-23T21:22:08.245621+0000) 2023-09-23T21:22:09.869 INFO:journalctl@ceph.osd.5.smithi070.stdout:Sep 23 21:22:09 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd-5[113356]: 2023-09-23T21:22:09.776+0000 7ff1a3eca700 -1 osd.5 709 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2023-09-23T21:21:43.545440+0000 front 2023-09-23T21:21:43.545514+0000 (oldest deadline 2023-09-23T21:22:08.245621+0000) 2023-09-23T21:22:09.869 INFO:journalctl@ceph.osd.6.smithi070.stdout:Sep 23 21:22:09 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd-6[193219]: 2023-09-23T21:22:09.608+0000 7f4171b44700 -1 osd.6 709 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2023-09-23T21:21:46.046262+0000 front 2023-09-23T21:21:46.046399+0000 (oldest deadline 2023-09-23T21:22:09.546231+0000) 2023-09-23T21:22:10.369 INFO:journalctl@ceph.osd.7.smithi070.stdout:Sep 23 21:22:10 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd-7[128949]: 2023-09-23T21:22:10.019+0000 7f12105aa700 -1 osd.7 709 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2023-09-23T21:21:46.720306+0000 front 2023-09-23T21:21:46.720441+0000 (oldest deadline 2023-09-23T21:22:08.420326+0000) 2023-09-23T21:22:10.869 INFO:journalctl@ceph.osd.6.smithi070.stdout:Sep 23 21:22:10 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd-6[193219]: 2023-09-23T21:22:10.590+0000 7f4171b44700 -1 osd.6 709 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2023-09-23T21:21:46.046262+0000 front 2023-09-23T21:21:46.046399+0000 (oldest deadline 2023-09-23T21:22:09.546231+0000) 2023-09-23T21:22:10.869 INFO:journalctl@ceph.osd.5.smithi070.stdout:Sep 23 21:22:10 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd-5[113356]: 2023-09-23T21:22:10.785+0000 7ff1a3eca700 -1 osd.5 709 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2023-09-23T21:21:43.545440+0000 front 2023-09-23T21:21:43.545514+0000 (oldest deadline 2023-09-23T21:22:08.245621+0000) 2023-09-23T21:22:11.369 INFO:journalctl@ceph.osd.7.smithi070.stdout:Sep 23 21:22:11 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd-7[128949]: 2023-09-23T21:22:11.046+0000 7f12105aa700 -1 osd.7 709 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2023-09-23T21:21:46.720306+0000 front 2023-09-23T21:21:46.720441+0000 (oldest deadline 2023-09-23T21:22:08.420326+0000) 2023-09-23T21:22:11.381 INFO:journalctl@ceph.osd.3.smithi026.stdout:Sep 23 21:22:11 smithi026 bash[480336]: ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd-3 2023-09-23T21:22:11.936 INFO:journalctl@ceph.osd.6.smithi070.stdout:Sep 23 21:22:11 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd-6[193219]: 2023-09-23T21:22:11.639+0000 7f4171b44700 -1 osd.6 709 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2023-09-23T21:21:46.046262+0000 front 2023-09-23T21:21:46.046399+0000 (oldest deadline 2023-09-23T21:22:09.546231+0000) 2023-09-23T21:22:11.936 INFO:journalctl@ceph.osd.5.smithi070.stdout:Sep 23 21:22:11 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd-5[113356]: 2023-09-23T21:22:11.752+0000 7ff1a3eca700 -1 osd.5 709 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2023-09-23T21:21:43.545440+0000 front 2023-09-23T21:21:43.545514+0000 (oldest deadline 2023-09-23T21:22:08.245621+0000) 2023-09-23T21:22:12.369 INFO:journalctl@ceph.osd.7.smithi070.stdout:Sep 23 21:22:12 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd-7[128949]: 2023-09-23T21:22:12.033+0000 7f12105aa700 -1 osd.7 709 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2023-09-23T21:21:46.720306+0000 front 2023-09-23T21:21:46.720441+0000 (oldest deadline 2023-09-23T21:22:08.420326+0000) 2023-09-23T21:22:12.369 INFO:journalctl@ceph.osd.4.smithi070.stdout:Sep 23 21:22:11 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd-4[106327]: 2023-09-23T21:22:11.935+0000 7f9b39365700 -1 osd.4 709 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2023-09-23T21:21:46.938479+0000 front 2023-09-23T21:21:46.938635+0000 (oldest deadline 2023-09-23T21:22:11.038432+0000) 2023-09-23T21:22:12.801 DEBUG:teuthology.orchestra.run.smithi026:> sudo pkill -f 'journalctl -f -n 0 -u ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7@osd.3.service' 2023-09-23T21:22:12.842 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-09-23T21:22:12.842 INFO:tasks.cephadm.osd.3:Stopped osd.3 2023-09-23T21:22:12.842 INFO:tasks.cephadm.osd.4:Stopping osd.4... 2023-09-23T21:22:12.842 DEBUG:teuthology.orchestra.run.smithi070:> sudo systemctl stop ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7@osd.4 2023-09-23T21:22:12.944 INFO:journalctl@ceph.osd.5.smithi070.stdout:Sep 23 21:22:12 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd-5[113356]: 2023-09-23T21:22:12.768+0000 7ff1a3eca700 -1 osd.5 709 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2023-09-23T21:21:43.545440+0000 front 2023-09-23T21:21:43.545514+0000 (oldest deadline 2023-09-23T21:22:08.245621+0000) 2023-09-23T21:22:12.944 INFO:journalctl@ceph.osd.6.smithi070.stdout:Sep 23 21:22:12 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd-6[193219]: 2023-09-23T21:22:12.658+0000 7f4171b44700 -1 osd.6 709 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2023-09-23T21:21:46.046262+0000 front 2023-09-23T21:21:46.046399+0000 (oldest deadline 2023-09-23T21:22:09.546231+0000) 2023-09-23T21:22:12.944 INFO:journalctl@ceph.osd.4.smithi070.stdout:Sep 23 21:22:12 smithi070 systemd[1]: Stopping Ceph osd.4 for b3f8e4ec-5a50-11ee-8db4-212e2dc638e7... 2023-09-23T21:22:13.223 INFO:journalctl@ceph.osd.4.smithi070.stdout:Sep 23 21:22:12 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd-4[106327]: 2023-09-23T21:22:12.943+0000 7f9b39365700 -1 osd.4 709 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2023-09-23T21:21:46.938479+0000 front 2023-09-23T21:21:46.938635+0000 (oldest deadline 2023-09-23T21:22:11.038432+0000) 2023-09-23T21:22:13.224 INFO:journalctl@ceph.osd.7.smithi070.stdout:Sep 23 21:22:13 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd-7[128949]: 2023-09-23T21:22:13.058+0000 7f12105aa700 -1 osd.7 709 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2023-09-23T21:21:46.720306+0000 front 2023-09-23T21:21:46.720441+0000 (oldest deadline 2023-09-23T21:22:08.420326+0000) 2023-09-23T21:22:13.619 INFO:journalctl@ceph.osd.4.smithi070.stdout:Sep 23 21:22:13 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd-4[106327]: 2023-09-23T21:22:13.222+0000 7f9b3bd65700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-osd -n osd.4 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2023-09-23T21:22:13.619 INFO:journalctl@ceph.osd.4.smithi070.stdout:Sep 23 21:22:13 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd-4[106327]: 2023-09-23T21:22:13.222+0000 7f9b3bd65700 -1 osd.4 709 *** Got signal Terminated *** 2023-09-23T21:22:13.619 INFO:journalctl@ceph.osd.4.smithi070.stdout:Sep 23 21:22:13 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd-4[106327]: 2023-09-23T21:22:13.222+0000 7f9b3bd65700 -1 osd.4 709 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-09-23T21:22:13.934 INFO:journalctl@ceph.osd.5.smithi070.stdout:Sep 23 21:22:13 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd-5[113356]: 2023-09-23T21:22:13.756+0000 7ff1a3eca700 -1 osd.5 709 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2023-09-23T21:21:43.545440+0000 front 2023-09-23T21:21:43.545514+0000 (oldest deadline 2023-09-23T21:22:08.245621+0000) 2023-09-23T21:22:13.934 INFO:journalctl@ceph.osd.6.smithi070.stdout:Sep 23 21:22:13 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd-6[193219]: 2023-09-23T21:22:13.661+0000 7f4171b44700 -1 osd.6 709 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2023-09-23T21:21:46.046262+0000 front 2023-09-23T21:21:46.046399+0000 (oldest deadline 2023-09-23T21:22:09.546231+0000) 2023-09-23T21:22:14.369 INFO:journalctl@ceph.osd.4.smithi070.stdout:Sep 23 21:22:13 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd-4[106327]: 2023-09-23T21:22:13.934+0000 7f9b39365700 -1 osd.4 709 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2023-09-23T21:21:46.938479+0000 front 2023-09-23T21:21:46.938635+0000 (oldest deadline 2023-09-23T21:22:11.038432+0000) 2023-09-23T21:22:14.369 INFO:journalctl@ceph.osd.7.smithi070.stdout:Sep 23 21:22:14 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd-7[128949]: 2023-09-23T21:22:14.084+0000 7f12105aa700 -1 osd.7 709 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2023-09-23T21:21:46.720306+0000 front 2023-09-23T21:21:46.720441+0000 (oldest deadline 2023-09-23T21:22:08.420326+0000) 2023-09-23T21:22:14.969 INFO:journalctl@ceph.osd.6.smithi070.stdout:Sep 23 21:22:14 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd-6[193219]: 2023-09-23T21:22:14.698+0000 7f4171b44700 -1 osd.6 709 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2023-09-23T21:21:46.046262+0000 front 2023-09-23T21:21:46.046399+0000 (oldest deadline 2023-09-23T21:22:09.546231+0000) 2023-09-23T21:22:14.969 INFO:journalctl@ceph.osd.5.smithi070.stdout:Sep 23 21:22:14 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd-5[113356]: 2023-09-23T21:22:14.737+0000 7ff1a3eca700 -1 osd.5 709 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2023-09-23T21:21:43.545440+0000 front 2023-09-23T21:21:43.545514+0000 (oldest deadline 2023-09-23T21:22:08.245621+0000) 2023-09-23T21:22:15.369 INFO:journalctl@ceph.osd.4.smithi070.stdout:Sep 23 21:22:14 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd-4[106327]: 2023-09-23T21:22:14.968+0000 7f9b39365700 -1 osd.4 709 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2023-09-23T21:21:46.938479+0000 front 2023-09-23T21:21:46.938635+0000 (oldest deadline 2023-09-23T21:22:11.038432+0000) 2023-09-23T21:22:15.369 INFO:journalctl@ceph.osd.7.smithi070.stdout:Sep 23 21:22:15 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd-7[128949]: 2023-09-23T21:22:15.083+0000 7f12105aa700 -1 osd.7 709 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2023-09-23T21:21:46.720306+0000 front 2023-09-23T21:21:46.720441+0000 (oldest deadline 2023-09-23T21:22:08.420326+0000) 2023-09-23T21:22:15.931 INFO:journalctl@ceph.osd.5.smithi070.stdout:Sep 23 21:22:15 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd-5[113356]: 2023-09-23T21:22:15.777+0000 7ff1a3eca700 -1 osd.5 709 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2023-09-23T21:21:43.545440+0000 front 2023-09-23T21:21:43.545514+0000 (oldest deadline 2023-09-23T21:22:08.245621+0000) 2023-09-23T21:22:15.931 INFO:journalctl@ceph.osd.6.smithi070.stdout:Sep 23 21:22:15 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd-6[193219]: 2023-09-23T21:22:15.662+0000 7f4171b44700 -1 osd.6 709 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2023-09-23T21:21:46.046262+0000 front 2023-09-23T21:21:46.046399+0000 (oldest deadline 2023-09-23T21:22:09.546231+0000) 2023-09-23T21:22:16.369 INFO:journalctl@ceph.osd.4.smithi070.stdout:Sep 23 21:22:15 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd-4[106327]: 2023-09-23T21:22:15.930+0000 7f9b39365700 -1 osd.4 709 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2023-09-23T21:21:46.938479+0000 front 2023-09-23T21:21:46.938635+0000 (oldest deadline 2023-09-23T21:22:11.038432+0000) 2023-09-23T21:22:16.369 INFO:journalctl@ceph.osd.7.smithi070.stdout:Sep 23 21:22:16 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd-7[128949]: 2023-09-23T21:22:16.047+0000 7f12105aa700 -1 osd.7 709 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2023-09-23T21:21:46.720306+0000 front 2023-09-23T21:21:46.720441+0000 (oldest deadline 2023-09-23T21:22:08.420326+0000) 2023-09-23T21:22:16.913 INFO:journalctl@ceph.osd.6.smithi070.stdout:Sep 23 21:22:16 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd-6[193219]: 2023-09-23T21:22:16.657+0000 7f4171b44700 -1 osd.6 709 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2023-09-23T21:21:46.046262+0000 front 2023-09-23T21:21:46.046399+0000 (oldest deadline 2023-09-23T21:22:09.546231+0000) 2023-09-23T21:22:16.913 INFO:journalctl@ceph.osd.5.smithi070.stdout:Sep 23 21:22:16 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd-5[113356]: 2023-09-23T21:22:16.799+0000 7ff1a3eca700 -1 osd.5 709 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2023-09-23T21:21:43.545440+0000 front 2023-09-23T21:21:43.545514+0000 (oldest deadline 2023-09-23T21:22:08.245621+0000) 2023-09-23T21:22:17.369 INFO:journalctl@ceph.osd.4.smithi070.stdout:Sep 23 21:22:16 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd-4[106327]: 2023-09-23T21:22:16.913+0000 7f9b39365700 -1 osd.4 709 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2023-09-23T21:21:46.938479+0000 front 2023-09-23T21:21:46.938635+0000 (oldest deadline 2023-09-23T21:22:11.038432+0000) 2023-09-23T21:22:17.369 INFO:journalctl@ceph.osd.7.smithi070.stdout:Sep 23 21:22:17 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd-7[128949]: 2023-09-23T21:22:17.034+0000 7f12105aa700 -1 osd.7 709 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2023-09-23T21:21:46.720306+0000 front 2023-09-23T21:21:46.720441+0000 (oldest deadline 2023-09-23T21:22:08.420326+0000) 2023-09-23T21:22:18.000 INFO:journalctl@ceph.osd.4.smithi070.stdout:Sep 23 21:22:17 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd-4[106327]: 2023-09-23T21:22:17.883+0000 7f9b39365700 -1 osd.4 709 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2023-09-23T21:21:46.938479+0000 front 2023-09-23T21:21:46.938635+0000 (oldest deadline 2023-09-23T21:22:11.038432+0000) 2023-09-23T21:22:18.001 INFO:journalctl@ceph.osd.6.smithi070.stdout:Sep 23 21:22:17 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd-6[193219]: 2023-09-23T21:22:17.689+0000 7f4171b44700 -1 osd.6 709 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2023-09-23T21:21:46.046262+0000 front 2023-09-23T21:21:46.046399+0000 (oldest deadline 2023-09-23T21:22:09.546231+0000) 2023-09-23T21:22:18.001 INFO:journalctl@ceph.osd.5.smithi070.stdout:Sep 23 21:22:17 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd-5[113356]: 2023-09-23T21:22:17.780+0000 7ff1a3eca700 -1 osd.5 709 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2023-09-23T21:21:43.545440+0000 front 2023-09-23T21:21:43.545514+0000 (oldest deadline 2023-09-23T21:22:08.245621+0000) 2023-09-23T21:22:18.001 INFO:journalctl@ceph.osd.5.smithi070.stdout:Sep 23 21:22:17 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd-5[113356]: 2023-09-23T21:22:17.780+0000 7ff1a3eca700 -1 osd.5 709 heartbeat_check: no reply from 172.21.15.26:6814 osd.1 since back 2023-09-23T21:21:52.147906+0000 front 2023-09-23T21:21:52.147831+0000 (oldest deadline 2023-09-23T21:22:17.447727+0000) 2023-09-23T21:22:18.369 INFO:journalctl@ceph.osd.7.smithi070.stdout:Sep 23 21:22:18 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd-7[128949]: 2023-09-23T21:22:18.000+0000 7f12105aa700 -1 osd.7 709 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2023-09-23T21:21:46.720306+0000 front 2023-09-23T21:21:46.720441+0000 (oldest deadline 2023-09-23T21:22:08.420326+0000) 2023-09-23T21:22:19.013 INFO:journalctl@ceph.osd.5.smithi070.stdout:Sep 23 21:22:18 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd-5[113356]: 2023-09-23T21:22:18.753+0000 7ff1a3eca700 -1 osd.5 709 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2023-09-23T21:21:43.545440+0000 front 2023-09-23T21:21:43.545514+0000 (oldest deadline 2023-09-23T21:22:08.245621+0000) 2023-09-23T21:22:19.013 INFO:journalctl@ceph.osd.5.smithi070.stdout:Sep 23 21:22:18 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd-5[113356]: 2023-09-23T21:22:18.753+0000 7ff1a3eca700 -1 osd.5 709 heartbeat_check: no reply from 172.21.15.26:6814 osd.1 since back 2023-09-23T21:21:52.147906+0000 front 2023-09-23T21:21:52.147831+0000 (oldest deadline 2023-09-23T21:22:17.447727+0000) 2023-09-23T21:22:19.014 INFO:journalctl@ceph.osd.6.smithi070.stdout:Sep 23 21:22:18 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd-6[193219]: 2023-09-23T21:22:18.704+0000 7f4171b44700 -1 osd.6 709 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2023-09-23T21:21:46.046262+0000 front 2023-09-23T21:21:46.046399+0000 (oldest deadline 2023-09-23T21:22:09.546231+0000) 2023-09-23T21:22:19.014 INFO:journalctl@ceph.osd.6.smithi070.stdout:Sep 23 21:22:18 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd-6[193219]: 2023-09-23T21:22:18.704+0000 7f4171b44700 -1 osd.6 709 heartbeat_check: no reply from 172.21.15.26:6814 osd.1 since back 2023-09-23T21:21:52.947781+0000 front 2023-09-23T21:21:52.947835+0000 (oldest deadline 2023-09-23T21:22:18.247705+0000) 2023-09-23T21:22:19.284 INFO:journalctl@ceph.osd.4.smithi070.stdout:Sep 23 21:22:19 smithi070 bash[308688]: ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd-4 2023-09-23T21:22:19.285 INFO:journalctl@ceph.osd.7.smithi070.stdout:Sep 23 21:22:19 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd-7[128949]: 2023-09-23T21:22:19.011+0000 7f12105aa700 -1 osd.7 709 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2023-09-23T21:21:46.720306+0000 front 2023-09-23T21:21:46.720441+0000 (oldest deadline 2023-09-23T21:22:08.420326+0000) 2023-09-23T21:22:19.285 INFO:journalctl@ceph.osd.7.smithi070.stdout:Sep 23 21:22:19 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd-7[128949]: 2023-09-23T21:22:19.011+0000 7f12105aa700 -1 osd.7 709 heartbeat_check: no reply from 172.21.15.26:6814 osd.1 since back 2023-09-23T21:21:53.721516+0000 front 2023-09-23T21:21:53.721501+0000 (oldest deadline 2023-09-23T21:22:18.421368+0000) 2023-09-23T21:22:20.030 INFO:journalctl@ceph.osd.5.smithi070.stdout:Sep 23 21:22:19 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd-5[113356]: 2023-09-23T21:22:19.772+0000 7ff1a3eca700 -1 osd.5 709 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2023-09-23T21:21:43.545440+0000 front 2023-09-23T21:21:43.545514+0000 (oldest deadline 2023-09-23T21:22:08.245621+0000) 2023-09-23T21:22:20.030 INFO:journalctl@ceph.osd.5.smithi070.stdout:Sep 23 21:22:19 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd-5[113356]: 2023-09-23T21:22:19.772+0000 7ff1a3eca700 -1 osd.5 709 heartbeat_check: no reply from 172.21.15.26:6814 osd.1 since back 2023-09-23T21:21:52.147906+0000 front 2023-09-23T21:21:52.147831+0000 (oldest deadline 2023-09-23T21:22:17.447727+0000) 2023-09-23T21:22:20.031 INFO:journalctl@ceph.osd.6.smithi070.stdout:Sep 23 21:22:19 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd-6[193219]: 2023-09-23T21:22:19.678+0000 7f4171b44700 -1 osd.6 709 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2023-09-23T21:21:46.046262+0000 front 2023-09-23T21:21:46.046399+0000 (oldest deadline 2023-09-23T21:22:09.546231+0000) 2023-09-23T21:22:20.031 INFO:journalctl@ceph.osd.6.smithi070.stdout:Sep 23 21:22:19 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd-6[193219]: 2023-09-23T21:22:19.678+0000 7f4171b44700 -1 osd.6 709 heartbeat_check: no reply from 172.21.15.26:6814 osd.1 since back 2023-09-23T21:21:52.947781+0000 front 2023-09-23T21:21:52.947835+0000 (oldest deadline 2023-09-23T21:22:18.247705+0000) 2023-09-23T21:22:20.331 INFO:journalctl@ceph.osd.7.smithi070.stdout:Sep 23 21:22:20 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd-7[128949]: 2023-09-23T21:22:20.029+0000 7f12105aa700 -1 osd.7 709 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2023-09-23T21:21:46.720306+0000 front 2023-09-23T21:21:46.720441+0000 (oldest deadline 2023-09-23T21:22:08.420326+0000) 2023-09-23T21:22:20.331 INFO:journalctl@ceph.osd.7.smithi070.stdout:Sep 23 21:22:20 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd-7[128949]: 2023-09-23T21:22:20.029+0000 7f12105aa700 -1 osd.7 709 heartbeat_check: no reply from 172.21.15.26:6814 osd.1 since back 2023-09-23T21:21:53.721516+0000 front 2023-09-23T21:21:53.721501+0000 (oldest deadline 2023-09-23T21:22:18.421368+0000) 2023-09-23T21:22:20.548 DEBUG:teuthology.orchestra.run.smithi070:> sudo pkill -f 'journalctl -f -n 0 -u ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7@osd.4.service' 2023-09-23T21:22:20.586 INFO:journalctl@ceph.osd.4.smithi070.stdout:Sep 23 21:22:20 smithi070 systemd[1]: ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7@osd.4.service: Succeeded. 2023-09-23T21:22:20.587 INFO:journalctl@ceph.osd.4.smithi070.stdout:Sep 23 21:22:20 smithi070 systemd[1]: Stopped Ceph osd.4 for b3f8e4ec-5a50-11ee-8db4-212e2dc638e7. 2023-09-23T21:22:20.601 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-09-23T21:22:20.601 INFO:tasks.cephadm.osd.4:Stopped osd.4 2023-09-23T21:22:20.601 INFO:tasks.cephadm.osd.5:Stopping osd.5... 2023-09-23T21:22:20.601 DEBUG:teuthology.orchestra.run.smithi070:> sudo systemctl stop ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7@osd.5 2023-09-23T21:22:20.870 INFO:journalctl@ceph.osd.5.smithi070.stdout:Sep 23 21:22:20 smithi070 systemd[1]: Stopping Ceph osd.5 for b3f8e4ec-5a50-11ee-8db4-212e2dc638e7... 2023-09-23T21:22:20.870 INFO:journalctl@ceph.osd.5.smithi070.stdout:Sep 23 21:22:20 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd-5[113356]: 2023-09-23T21:22:20.792+0000 7ff1a3eca700 -1 osd.5 709 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2023-09-23T21:21:43.545440+0000 front 2023-09-23T21:21:43.545514+0000 (oldest deadline 2023-09-23T21:22:08.245621+0000) 2023-09-23T21:22:20.870 INFO:journalctl@ceph.osd.5.smithi070.stdout:Sep 23 21:22:20 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd-5[113356]: 2023-09-23T21:22:20.792+0000 7ff1a3eca700 -1 osd.5 709 heartbeat_check: no reply from 172.21.15.26:6814 osd.1 since back 2023-09-23T21:21:52.147906+0000 front 2023-09-23T21:21:52.147831+0000 (oldest deadline 2023-09-23T21:22:17.447727+0000) 2023-09-23T21:22:20.870 INFO:journalctl@ceph.osd.6.smithi070.stdout:Sep 23 21:22:20 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd-6[193219]: 2023-09-23T21:22:20.714+0000 7f4171b44700 -1 osd.6 709 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2023-09-23T21:21:46.046262+0000 front 2023-09-23T21:21:46.046399+0000 (oldest deadline 2023-09-23T21:22:09.546231+0000) 2023-09-23T21:22:20.871 INFO:journalctl@ceph.osd.6.smithi070.stdout:Sep 23 21:22:20 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd-6[193219]: 2023-09-23T21:22:20.714+0000 7f4171b44700 -1 osd.6 709 heartbeat_check: no reply from 172.21.15.26:6814 osd.1 since back 2023-09-23T21:21:52.947781+0000 front 2023-09-23T21:21:52.947835+0000 (oldest deadline 2023-09-23T21:22:18.247705+0000) 2023-09-23T21:22:21.369 INFO:journalctl@ceph.osd.5.smithi070.stdout:Sep 23 21:22:20 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd-5[113356]: 2023-09-23T21:22:20.969+0000 7ff1a60c9700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-osd -n osd.5 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2023-09-23T21:22:21.369 INFO:journalctl@ceph.osd.5.smithi070.stdout:Sep 23 21:22:20 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd-5[113356]: 2023-09-23T21:22:20.969+0000 7ff1a60c9700 -1 osd.5 709 *** Got signal Terminated *** 2023-09-23T21:22:21.369 INFO:journalctl@ceph.osd.5.smithi070.stdout:Sep 23 21:22:20 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd-5[113356]: 2023-09-23T21:22:20.969+0000 7ff1a60c9700 -1 osd.5 709 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-09-23T21:22:21.370 INFO:journalctl@ceph.osd.7.smithi070.stdout:Sep 23 21:22:20 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd-7[128949]: 2023-09-23T21:22:20.986+0000 7f12105aa700 -1 osd.7 709 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2023-09-23T21:21:46.720306+0000 front 2023-09-23T21:21:46.720441+0000 (oldest deadline 2023-09-23T21:22:08.420326+0000) 2023-09-23T21:22:21.370 INFO:journalctl@ceph.osd.7.smithi070.stdout:Sep 23 21:22:20 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd-7[128949]: 2023-09-23T21:22:20.986+0000 7f12105aa700 -1 osd.7 709 heartbeat_check: no reply from 172.21.15.26:6814 osd.1 since back 2023-09-23T21:21:53.721516+0000 front 2023-09-23T21:21:53.721501+0000 (oldest deadline 2023-09-23T21:22:18.421368+0000) 2023-09-23T21:22:21.968 INFO:journalctl@ceph.osd.6.smithi070.stdout:Sep 23 21:22:21 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd-6[193219]: 2023-09-23T21:22:21.671+0000 7f4171b44700 -1 osd.6 709 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2023-09-23T21:21:46.046262+0000 front 2023-09-23T21:21:46.046399+0000 (oldest deadline 2023-09-23T21:22:09.546231+0000) 2023-09-23T21:22:21.968 INFO:journalctl@ceph.osd.6.smithi070.stdout:Sep 23 21:22:21 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd-6[193219]: 2023-09-23T21:22:21.671+0000 7f4171b44700 -1 osd.6 709 heartbeat_check: no reply from 172.21.15.26:6814 osd.1 since back 2023-09-23T21:21:52.947781+0000 front 2023-09-23T21:21:52.947835+0000 (oldest deadline 2023-09-23T21:22:18.247705+0000) 2023-09-23T21:22:21.968 INFO:journalctl@ceph.osd.5.smithi070.stdout:Sep 23 21:22:21 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd-5[113356]: 2023-09-23T21:22:21.818+0000 7ff1a3eca700 -1 osd.5 709 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2023-09-23T21:21:43.545440+0000 front 2023-09-23T21:21:43.545514+0000 (oldest deadline 2023-09-23T21:22:08.245621+0000) 2023-09-23T21:22:21.968 INFO:journalctl@ceph.osd.5.smithi070.stdout:Sep 23 21:22:21 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd-5[113356]: 2023-09-23T21:22:21.818+0000 7ff1a3eca700 -1 osd.5 709 heartbeat_check: no reply from 172.21.15.26:6814 osd.1 since back 2023-09-23T21:21:52.147906+0000 front 2023-09-23T21:21:52.147831+0000 (oldest deadline 2023-09-23T21:22:17.447727+0000) 2023-09-23T21:22:22.369 INFO:journalctl@ceph.osd.7.smithi070.stdout:Sep 23 21:22:21 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd-7[128949]: 2023-09-23T21:22:21.967+0000 7f12105aa700 -1 osd.7 709 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2023-09-23T21:21:46.720306+0000 front 2023-09-23T21:21:46.720441+0000 (oldest deadline 2023-09-23T21:22:08.420326+0000) 2023-09-23T21:22:22.369 INFO:journalctl@ceph.osd.7.smithi070.stdout:Sep 23 21:22:21 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd-7[128949]: 2023-09-23T21:22:21.967+0000 7f12105aa700 -1 osd.7 709 heartbeat_check: no reply from 172.21.15.26:6814 osd.1 since back 2023-09-23T21:21:53.721516+0000 front 2023-09-23T21:21:53.721501+0000 (oldest deadline 2023-09-23T21:22:18.421368+0000) 2023-09-23T21:22:22.943 INFO:journalctl@ceph.osd.6.smithi070.stdout:Sep 23 21:22:22 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd-6[193219]: 2023-09-23T21:22:22.646+0000 7f4171b44700 -1 osd.6 709 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2023-09-23T21:21:46.046262+0000 front 2023-09-23T21:21:46.046399+0000 (oldest deadline 2023-09-23T21:22:09.546231+0000) 2023-09-23T21:22:22.943 INFO:journalctl@ceph.osd.6.smithi070.stdout:Sep 23 21:22:22 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd-6[193219]: 2023-09-23T21:22:22.646+0000 7f4171b44700 -1 osd.6 709 heartbeat_check: no reply from 172.21.15.26:6814 osd.1 since back 2023-09-23T21:21:52.947781+0000 front 2023-09-23T21:21:52.947835+0000 (oldest deadline 2023-09-23T21:22:18.247705+0000) 2023-09-23T21:22:22.943 INFO:journalctl@ceph.osd.5.smithi070.stdout:Sep 23 21:22:22 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd-5[113356]: 2023-09-23T21:22:22.806+0000 7ff1a3eca700 -1 osd.5 709 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2023-09-23T21:21:43.545440+0000 front 2023-09-23T21:21:43.545514+0000 (oldest deadline 2023-09-23T21:22:08.245621+0000) 2023-09-23T21:22:22.943 INFO:journalctl@ceph.osd.5.smithi070.stdout:Sep 23 21:22:22 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd-5[113356]: 2023-09-23T21:22:22.806+0000 7ff1a3eca700 -1 osd.5 709 heartbeat_check: no reply from 172.21.15.26:6814 osd.1 since back 2023-09-23T21:21:52.147906+0000 front 2023-09-23T21:21:52.147831+0000 (oldest deadline 2023-09-23T21:22:17.447727+0000) 2023-09-23T21:22:23.369 INFO:journalctl@ceph.osd.7.smithi070.stdout:Sep 23 21:22:22 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd-7[128949]: 2023-09-23T21:22:22.942+0000 7f12105aa700 -1 osd.7 709 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2023-09-23T21:21:46.720306+0000 front 2023-09-23T21:21:46.720441+0000 (oldest deadline 2023-09-23T21:22:08.420326+0000) 2023-09-23T21:22:23.369 INFO:journalctl@ceph.osd.7.smithi070.stdout:Sep 23 21:22:22 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd-7[128949]: 2023-09-23T21:22:22.942+0000 7f12105aa700 -1 osd.7 709 heartbeat_check: no reply from 172.21.15.26:6814 osd.1 since back 2023-09-23T21:21:53.721516+0000 front 2023-09-23T21:21:53.721501+0000 (oldest deadline 2023-09-23T21:22:18.421368+0000) 2023-09-23T21:22:23.962 INFO:journalctl@ceph.osd.5.smithi070.stdout:Sep 23 21:22:23 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd-5[113356]: 2023-09-23T21:22:23.817+0000 7ff1a3eca700 -1 osd.5 709 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2023-09-23T21:21:43.545440+0000 front 2023-09-23T21:21:43.545514+0000 (oldest deadline 2023-09-23T21:22:08.245621+0000) 2023-09-23T21:22:23.963 INFO:journalctl@ceph.osd.5.smithi070.stdout:Sep 23 21:22:23 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd-5[113356]: 2023-09-23T21:22:23.817+0000 7ff1a3eca700 -1 osd.5 709 heartbeat_check: no reply from 172.21.15.26:6814 osd.1 since back 2023-09-23T21:21:52.147906+0000 front 2023-09-23T21:21:52.147831+0000 (oldest deadline 2023-09-23T21:22:17.447727+0000) 2023-09-23T21:22:23.963 INFO:journalctl@ceph.osd.6.smithi070.stdout:Sep 23 21:22:23 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd-6[193219]: 2023-09-23T21:22:23.649+0000 7f4171b44700 -1 osd.6 709 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2023-09-23T21:21:46.046262+0000 front 2023-09-23T21:21:46.046399+0000 (oldest deadline 2023-09-23T21:22:09.546231+0000) 2023-09-23T21:22:23.963 INFO:journalctl@ceph.osd.6.smithi070.stdout:Sep 23 21:22:23 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd-6[193219]: 2023-09-23T21:22:23.649+0000 7f4171b44700 -1 osd.6 709 heartbeat_check: no reply from 172.21.15.26:6814 osd.1 since back 2023-09-23T21:21:52.947781+0000 front 2023-09-23T21:21:52.947835+0000 (oldest deadline 2023-09-23T21:22:18.247705+0000) 2023-09-23T21:22:24.369 INFO:journalctl@ceph.osd.7.smithi070.stdout:Sep 23 21:22:23 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd-7[128949]: 2023-09-23T21:22:23.961+0000 7f12105aa700 -1 osd.7 709 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2023-09-23T21:21:46.720306+0000 front 2023-09-23T21:21:46.720441+0000 (oldest deadline 2023-09-23T21:22:08.420326+0000) 2023-09-23T21:22:24.369 INFO:journalctl@ceph.osd.7.smithi070.stdout:Sep 23 21:22:23 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd-7[128949]: 2023-09-23T21:22:23.961+0000 7f12105aa700 -1 osd.7 709 heartbeat_check: no reply from 172.21.15.26:6814 osd.1 since back 2023-09-23T21:21:53.721516+0000 front 2023-09-23T21:21:53.721501+0000 (oldest deadline 2023-09-23T21:22:18.421368+0000) 2023-09-23T21:22:24.967 INFO:journalctl@ceph.osd.5.smithi070.stdout:Sep 23 21:22:24 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd-5[113356]: 2023-09-23T21:22:24.815+0000 7ff1a3eca700 -1 osd.5 709 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2023-09-23T21:21:43.545440+0000 front 2023-09-23T21:21:43.545514+0000 (oldest deadline 2023-09-23T21:22:08.245621+0000) 2023-09-23T21:22:24.967 INFO:journalctl@ceph.osd.5.smithi070.stdout:Sep 23 21:22:24 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd-5[113356]: 2023-09-23T21:22:24.815+0000 7ff1a3eca700 -1 osd.5 709 heartbeat_check: no reply from 172.21.15.26:6814 osd.1 since back 2023-09-23T21:21:52.147906+0000 front 2023-09-23T21:21:52.147831+0000 (oldest deadline 2023-09-23T21:22:17.447727+0000) 2023-09-23T21:22:24.968 INFO:journalctl@ceph.osd.6.smithi070.stdout:Sep 23 21:22:24 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd-6[193219]: 2023-09-23T21:22:24.696+0000 7f4171b44700 -1 osd.6 709 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2023-09-23T21:21:46.046262+0000 front 2023-09-23T21:21:46.046399+0000 (oldest deadline 2023-09-23T21:22:09.546231+0000) 2023-09-23T21:22:24.968 INFO:journalctl@ceph.osd.6.smithi070.stdout:Sep 23 21:22:24 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd-6[193219]: 2023-09-23T21:22:24.696+0000 7f4171b44700 -1 osd.6 709 heartbeat_check: no reply from 172.21.15.26:6814 osd.1 since back 2023-09-23T21:21:52.947781+0000 front 2023-09-23T21:21:52.947835+0000 (oldest deadline 2023-09-23T21:22:18.247705+0000) 2023-09-23T21:22:25.369 INFO:journalctl@ceph.osd.7.smithi070.stdout:Sep 23 21:22:24 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd-7[128949]: 2023-09-23T21:22:24.967+0000 7f12105aa700 -1 osd.7 709 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2023-09-23T21:21:46.720306+0000 front 2023-09-23T21:21:46.720441+0000 (oldest deadline 2023-09-23T21:22:08.420326+0000) 2023-09-23T21:22:25.369 INFO:journalctl@ceph.osd.7.smithi070.stdout:Sep 23 21:22:24 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd-7[128949]: 2023-09-23T21:22:24.967+0000 7f12105aa700 -1 osd.7 709 heartbeat_check: no reply from 172.21.15.26:6814 osd.1 since back 2023-09-23T21:21:53.721516+0000 front 2023-09-23T21:21:53.721501+0000 (oldest deadline 2023-09-23T21:22:18.421368+0000) 2023-09-23T21:22:25.986 INFO:journalctl@ceph.osd.6.smithi070.stdout:Sep 23 21:22:25 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd-6[193219]: 2023-09-23T21:22:25.659+0000 7f4171b44700 -1 osd.6 709 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2023-09-23T21:21:46.046262+0000 front 2023-09-23T21:21:46.046399+0000 (oldest deadline 2023-09-23T21:22:09.546231+0000) 2023-09-23T21:22:25.986 INFO:journalctl@ceph.osd.6.smithi070.stdout:Sep 23 21:22:25 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd-6[193219]: 2023-09-23T21:22:25.659+0000 7f4171b44700 -1 osd.6 709 heartbeat_check: no reply from 172.21.15.26:6814 osd.1 since back 2023-09-23T21:21:52.947781+0000 front 2023-09-23T21:21:52.947835+0000 (oldest deadline 2023-09-23T21:22:18.247705+0000) 2023-09-23T21:22:25.986 INFO:journalctl@ceph.osd.5.smithi070.stdout:Sep 23 21:22:25 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd-5[113356]: 2023-09-23T21:22:25.843+0000 7ff1a3eca700 -1 osd.5 709 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2023-09-23T21:21:43.545440+0000 front 2023-09-23T21:21:43.545514+0000 (oldest deadline 2023-09-23T21:22:08.245621+0000) 2023-09-23T21:22:25.987 INFO:journalctl@ceph.osd.5.smithi070.stdout:Sep 23 21:22:25 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd-5[113356]: 2023-09-23T21:22:25.843+0000 7ff1a3eca700 -1 osd.5 709 heartbeat_check: no reply from 172.21.15.26:6814 osd.1 since back 2023-09-23T21:21:52.147906+0000 front 2023-09-23T21:21:52.147831+0000 (oldest deadline 2023-09-23T21:22:17.447727+0000) 2023-09-23T21:22:26.369 INFO:journalctl@ceph.osd.7.smithi070.stdout:Sep 23 21:22:25 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd-7[128949]: 2023-09-23T21:22:25.985+0000 7f12105aa700 -1 osd.7 709 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2023-09-23T21:21:46.720306+0000 front 2023-09-23T21:21:46.720441+0000 (oldest deadline 2023-09-23T21:22:08.420326+0000) 2023-09-23T21:22:26.369 INFO:journalctl@ceph.osd.7.smithi070.stdout:Sep 23 21:22:25 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd-7[128949]: 2023-09-23T21:22:25.985+0000 7f12105aa700 -1 osd.7 709 heartbeat_check: no reply from 172.21.15.26:6814 osd.1 since back 2023-09-23T21:21:53.721516+0000 front 2023-09-23T21:21:53.721501+0000 (oldest deadline 2023-09-23T21:22:18.421368+0000) 2023-09-23T21:22:26.915 INFO:journalctl@ceph.osd.5.smithi070.stdout:Sep 23 21:22:26 smithi070 bash[308951]: ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd-5 2023-09-23T21:22:26.916 INFO:journalctl@ceph.osd.6.smithi070.stdout:Sep 23 21:22:26 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd-6[193219]: 2023-09-23T21:22:26.643+0000 7f4171b44700 -1 osd.6 709 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2023-09-23T21:21:46.046262+0000 front 2023-09-23T21:21:46.046399+0000 (oldest deadline 2023-09-23T21:22:09.546231+0000) 2023-09-23T21:22:26.916 INFO:journalctl@ceph.osd.6.smithi070.stdout:Sep 23 21:22:26 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd-6[193219]: 2023-09-23T21:22:26.643+0000 7f4171b44700 -1 osd.6 709 heartbeat_check: no reply from 172.21.15.26:6814 osd.1 since back 2023-09-23T21:21:52.947781+0000 front 2023-09-23T21:21:52.947835+0000 (oldest deadline 2023-09-23T21:22:18.247705+0000) 2023-09-23T21:22:27.369 INFO:journalctl@ceph.osd.7.smithi070.stdout:Sep 23 21:22:27 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd-7[128949]: 2023-09-23T21:22:27.001+0000 7f12105aa700 -1 osd.7 709 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2023-09-23T21:21:46.720306+0000 front 2023-09-23T21:21:46.720441+0000 (oldest deadline 2023-09-23T21:22:08.420326+0000) 2023-09-23T21:22:27.369 INFO:journalctl@ceph.osd.7.smithi070.stdout:Sep 23 21:22:27 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd-7[128949]: 2023-09-23T21:22:27.001+0000 7f12105aa700 -1 osd.7 709 heartbeat_check: no reply from 172.21.15.26:6814 osd.1 since back 2023-09-23T21:21:53.721516+0000 front 2023-09-23T21:21:53.721501+0000 (oldest deadline 2023-09-23T21:22:18.421368+0000) 2023-09-23T21:22:27.870 INFO:journalctl@ceph.osd.6.smithi070.stdout:Sep 23 21:22:27 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd-6[193219]: 2023-09-23T21:22:27.660+0000 7f4171b44700 -1 osd.6 709 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2023-09-23T21:21:46.046262+0000 front 2023-09-23T21:21:46.046399+0000 (oldest deadline 2023-09-23T21:22:09.546231+0000) 2023-09-23T21:22:27.870 INFO:journalctl@ceph.osd.6.smithi070.stdout:Sep 23 21:22:27 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd-6[193219]: 2023-09-23T21:22:27.660+0000 7f4171b44700 -1 osd.6 709 heartbeat_check: no reply from 172.21.15.26:6814 osd.1 since back 2023-09-23T21:21:52.947781+0000 front 2023-09-23T21:21:52.947835+0000 (oldest deadline 2023-09-23T21:22:18.247705+0000) 2023-09-23T21:22:28.268 INFO:journalctl@ceph.osd.7.smithi070.stdout:Sep 23 21:22:27 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd-7[128949]: 2023-09-23T21:22:27.998+0000 7f12105aa700 -1 osd.7 709 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2023-09-23T21:21:46.720306+0000 front 2023-09-23T21:21:46.720441+0000 (oldest deadline 2023-09-23T21:22:08.420326+0000) 2023-09-23T21:22:28.268 INFO:journalctl@ceph.osd.7.smithi070.stdout:Sep 23 21:22:27 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd-7[128949]: 2023-09-23T21:22:27.998+0000 7f12105aa700 -1 osd.7 709 heartbeat_check: no reply from 172.21.15.26:6814 osd.1 since back 2023-09-23T21:21:53.721516+0000 front 2023-09-23T21:21:53.721501+0000 (oldest deadline 2023-09-23T21:22:18.421368+0000) 2023-09-23T21:22:28.268 INFO:journalctl@ceph.osd.7.smithi070.stdout:Sep 23 21:22:27 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd-7[128949]: 2023-09-23T21:22:27.998+0000 7f12105aa700 -1 osd.7 709 heartbeat_check: no reply from 172.21.15.26:6822 osd.2 since back 2023-09-23T21:22:02.522110+0000 front 2023-09-23T21:22:02.522145+0000 (oldest deadline 2023-09-23T21:22:27.822275+0000) 2023-09-23T21:22:28.275 DEBUG:teuthology.orchestra.run.smithi070:> sudo pkill -f 'journalctl -f -n 0 -u ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7@osd.5.service' 2023-09-23T21:22:28.316 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-09-23T21:22:28.317 INFO:tasks.cephadm.osd.5:Stopped osd.5 2023-09-23T21:22:28.317 INFO:tasks.cephadm.osd.6:Stopping osd.6... 2023-09-23T21:22:28.317 DEBUG:teuthology.orchestra.run.smithi070:> sudo systemctl stop ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7@osd.6 2023-09-23T21:22:28.619 INFO:journalctl@ceph.osd.6.smithi070.stdout:Sep 23 21:22:28 smithi070 systemd[1]: Stopping Ceph osd.6 for b3f8e4ec-5a50-11ee-8db4-212e2dc638e7... 2023-09-23T21:22:28.998 INFO:journalctl@ceph.osd.6.smithi070.stdout:Sep 23 21:22:28 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd-6[193219]: 2023-09-23T21:22:28.625+0000 7f4171b44700 -1 osd.6 709 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2023-09-23T21:21:46.046262+0000 front 2023-09-23T21:21:46.046399+0000 (oldest deadline 2023-09-23T21:22:09.546231+0000) 2023-09-23T21:22:28.998 INFO:journalctl@ceph.osd.6.smithi070.stdout:Sep 23 21:22:28 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd-6[193219]: 2023-09-23T21:22:28.625+0000 7f4171b44700 -1 osd.6 709 heartbeat_check: no reply from 172.21.15.26:6814 osd.1 since back 2023-09-23T21:21:52.947781+0000 front 2023-09-23T21:21:52.947835+0000 (oldest deadline 2023-09-23T21:22:18.247705+0000) 2023-09-23T21:22:28.998 INFO:journalctl@ceph.osd.6.smithi070.stdout:Sep 23 21:22:28 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd-6[193219]: 2023-09-23T21:22:28.695+0000 7f4174544700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-osd -n osd.6 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2023-09-23T21:22:28.998 INFO:journalctl@ceph.osd.6.smithi070.stdout:Sep 23 21:22:28 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd-6[193219]: 2023-09-23T21:22:28.695+0000 7f4174544700 -1 osd.6 709 *** Got signal Terminated *** 2023-09-23T21:22:28.999 INFO:journalctl@ceph.osd.6.smithi070.stdout:Sep 23 21:22:28 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd-6[193219]: 2023-09-23T21:22:28.695+0000 7f4174544700 -1 osd.6 709 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-09-23T21:22:29.369 INFO:journalctl@ceph.osd.7.smithi070.stdout:Sep 23 21:22:28 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd-7[128949]: 2023-09-23T21:22:28.998+0000 7f12105aa700 -1 osd.7 709 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2023-09-23T21:21:46.720306+0000 front 2023-09-23T21:21:46.720441+0000 (oldest deadline 2023-09-23T21:22:08.420326+0000) 2023-09-23T21:22:29.369 INFO:journalctl@ceph.osd.7.smithi070.stdout:Sep 23 21:22:28 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd-7[128949]: 2023-09-23T21:22:28.998+0000 7f12105aa700 -1 osd.7 709 heartbeat_check: no reply from 172.21.15.26:6814 osd.1 since back 2023-09-23T21:21:53.721516+0000 front 2023-09-23T21:21:53.721501+0000 (oldest deadline 2023-09-23T21:22:18.421368+0000) 2023-09-23T21:22:29.369 INFO:journalctl@ceph.osd.7.smithi070.stdout:Sep 23 21:22:28 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd-7[128949]: 2023-09-23T21:22:28.998+0000 7f12105aa700 -1 osd.7 709 heartbeat_check: no reply from 172.21.15.26:6822 osd.2 since back 2023-09-23T21:22:02.522110+0000 front 2023-09-23T21:22:02.522145+0000 (oldest deadline 2023-09-23T21:22:27.822275+0000) 2023-09-23T21:22:29.869 INFO:journalctl@ceph.osd.6.smithi070.stdout:Sep 23 21:22:29 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd-6[193219]: 2023-09-23T21:22:29.603+0000 7f4171b44700 -1 osd.6 709 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2023-09-23T21:21:46.046262+0000 front 2023-09-23T21:21:46.046399+0000 (oldest deadline 2023-09-23T21:22:09.546231+0000) 2023-09-23T21:22:29.869 INFO:journalctl@ceph.osd.6.smithi070.stdout:Sep 23 21:22:29 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd-6[193219]: 2023-09-23T21:22:29.603+0000 7f4171b44700 -1 osd.6 709 heartbeat_check: no reply from 172.21.15.26:6814 osd.1 since back 2023-09-23T21:21:52.947781+0000 front 2023-09-23T21:21:52.947835+0000 (oldest deadline 2023-09-23T21:22:18.247705+0000) 2023-09-23T21:22:29.869 INFO:journalctl@ceph.osd.6.smithi070.stdout:Sep 23 21:22:29 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd-6[193219]: 2023-09-23T21:22:29.603+0000 7f4171b44700 -1 osd.6 709 heartbeat_check: no reply from 172.21.15.26:6822 osd.2 since back 2023-09-23T21:22:02.849197+0000 front 2023-09-23T21:22:02.849150+0000 (oldest deadline 2023-09-23T21:22:28.749106+0000) 2023-09-23T21:22:30.369 INFO:journalctl@ceph.osd.7.smithi070.stdout:Sep 23 21:22:29 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd-7[128949]: 2023-09-23T21:22:29.951+0000 7f12105aa700 -1 osd.7 709 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2023-09-23T21:21:46.720306+0000 front 2023-09-23T21:21:46.720441+0000 (oldest deadline 2023-09-23T21:22:08.420326+0000) 2023-09-23T21:22:30.369 INFO:journalctl@ceph.osd.7.smithi070.stdout:Sep 23 21:22:29 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd-7[128949]: 2023-09-23T21:22:29.951+0000 7f12105aa700 -1 osd.7 709 heartbeat_check: no reply from 172.21.15.26:6814 osd.1 since back 2023-09-23T21:21:53.721516+0000 front 2023-09-23T21:21:53.721501+0000 (oldest deadline 2023-09-23T21:22:18.421368+0000) 2023-09-23T21:22:30.369 INFO:journalctl@ceph.osd.7.smithi070.stdout:Sep 23 21:22:29 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd-7[128949]: 2023-09-23T21:22:29.951+0000 7f12105aa700 -1 osd.7 709 heartbeat_check: no reply from 172.21.15.26:6822 osd.2 since back 2023-09-23T21:22:02.522110+0000 front 2023-09-23T21:22:02.522145+0000 (oldest deadline 2023-09-23T21:22:27.822275+0000) 2023-09-23T21:22:30.869 INFO:journalctl@ceph.osd.6.smithi070.stdout:Sep 23 21:22:30 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd-6[193219]: 2023-09-23T21:22:30.562+0000 7f4171b44700 -1 osd.6 709 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2023-09-23T21:21:46.046262+0000 front 2023-09-23T21:21:46.046399+0000 (oldest deadline 2023-09-23T21:22:09.546231+0000) 2023-09-23T21:22:30.869 INFO:journalctl@ceph.osd.6.smithi070.stdout:Sep 23 21:22:30 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd-6[193219]: 2023-09-23T21:22:30.562+0000 7f4171b44700 -1 osd.6 709 heartbeat_check: no reply from 172.21.15.26:6814 osd.1 since back 2023-09-23T21:21:52.947781+0000 front 2023-09-23T21:21:52.947835+0000 (oldest deadline 2023-09-23T21:22:18.247705+0000) 2023-09-23T21:22:30.869 INFO:journalctl@ceph.osd.6.smithi070.stdout:Sep 23 21:22:30 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd-6[193219]: 2023-09-23T21:22:30.562+0000 7f4171b44700 -1 osd.6 709 heartbeat_check: no reply from 172.21.15.26:6822 osd.2 since back 2023-09-23T21:22:02.849197+0000 front 2023-09-23T21:22:02.849150+0000 (oldest deadline 2023-09-23T21:22:28.749106+0000) 2023-09-23T21:22:31.369 INFO:journalctl@ceph.osd.7.smithi070.stdout:Sep 23 21:22:30 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd-7[128949]: 2023-09-23T21:22:30.979+0000 7f12105aa700 -1 osd.7 709 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2023-09-23T21:21:46.720306+0000 front 2023-09-23T21:21:46.720441+0000 (oldest deadline 2023-09-23T21:22:08.420326+0000) 2023-09-23T21:22:31.369 INFO:journalctl@ceph.osd.7.smithi070.stdout:Sep 23 21:22:30 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd-7[128949]: 2023-09-23T21:22:30.979+0000 7f12105aa700 -1 osd.7 709 heartbeat_check: no reply from 172.21.15.26:6814 osd.1 since back 2023-09-23T21:21:53.721516+0000 front 2023-09-23T21:21:53.721501+0000 (oldest deadline 2023-09-23T21:22:18.421368+0000) 2023-09-23T21:22:31.369 INFO:journalctl@ceph.osd.7.smithi070.stdout:Sep 23 21:22:30 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd-7[128949]: 2023-09-23T21:22:30.979+0000 7f12105aa700 -1 osd.7 709 heartbeat_check: no reply from 172.21.15.26:6822 osd.2 since back 2023-09-23T21:22:02.522110+0000 front 2023-09-23T21:22:02.522145+0000 (oldest deadline 2023-09-23T21:22:27.822275+0000) 2023-09-23T21:22:31.869 INFO:journalctl@ceph.osd.6.smithi070.stdout:Sep 23 21:22:31 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd-6[193219]: 2023-09-23T21:22:31.570+0000 7f4171b44700 -1 osd.6 709 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2023-09-23T21:21:46.046262+0000 front 2023-09-23T21:21:46.046399+0000 (oldest deadline 2023-09-23T21:22:09.546231+0000) 2023-09-23T21:22:31.869 INFO:journalctl@ceph.osd.6.smithi070.stdout:Sep 23 21:22:31 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd-6[193219]: 2023-09-23T21:22:31.570+0000 7f4171b44700 -1 osd.6 709 heartbeat_check: no reply from 172.21.15.26:6814 osd.1 since back 2023-09-23T21:21:52.947781+0000 front 2023-09-23T21:21:52.947835+0000 (oldest deadline 2023-09-23T21:22:18.247705+0000) 2023-09-23T21:22:31.869 INFO:journalctl@ceph.osd.6.smithi070.stdout:Sep 23 21:22:31 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd-6[193219]: 2023-09-23T21:22:31.570+0000 7f4171b44700 -1 osd.6 709 heartbeat_check: no reply from 172.21.15.26:6822 osd.2 since back 2023-09-23T21:22:02.849197+0000 front 2023-09-23T21:22:02.849150+0000 (oldest deadline 2023-09-23T21:22:28.749106+0000) 2023-09-23T21:22:32.369 INFO:journalctl@ceph.osd.7.smithi070.stdout:Sep 23 21:22:31 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd-7[128949]: 2023-09-23T21:22:31.990+0000 7f12105aa700 -1 osd.7 709 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2023-09-23T21:21:46.720306+0000 front 2023-09-23T21:21:46.720441+0000 (oldest deadline 2023-09-23T21:22:08.420326+0000) 2023-09-23T21:22:32.369 INFO:journalctl@ceph.osd.7.smithi070.stdout:Sep 23 21:22:31 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd-7[128949]: 2023-09-23T21:22:31.990+0000 7f12105aa700 -1 osd.7 709 heartbeat_check: no reply from 172.21.15.26:6814 osd.1 since back 2023-09-23T21:21:53.721516+0000 front 2023-09-23T21:21:53.721501+0000 (oldest deadline 2023-09-23T21:22:18.421368+0000) 2023-09-23T21:22:32.370 INFO:journalctl@ceph.osd.7.smithi070.stdout:Sep 23 21:22:31 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd-7[128949]: 2023-09-23T21:22:31.990+0000 7f12105aa700 -1 osd.7 709 heartbeat_check: no reply from 172.21.15.26:6822 osd.2 since back 2023-09-23T21:22:02.522110+0000 front 2023-09-23T21:22:02.522145+0000 (oldest deadline 2023-09-23T21:22:27.822275+0000) 2023-09-23T21:22:32.370 INFO:journalctl@ceph.osd.7.smithi070.stdout:Sep 23 21:22:31 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd-7[128949]: 2023-09-23T21:22:31.990+0000 7f12105aa700 -1 osd.7 709 heartbeat_check: no reply from 172.21.15.26:6830 osd.3 since back 2023-09-23T21:22:07.822900+0000 front 2023-09-23T21:22:07.822976+0000 (oldest deadline 2023-09-23T21:22:31.322875+0000) 2023-09-23T21:22:32.869 INFO:journalctl@ceph.osd.6.smithi070.stdout:Sep 23 21:22:32 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd-6[193219]: 2023-09-23T21:22:32.583+0000 7f4171b44700 -1 osd.6 709 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2023-09-23T21:21:46.046262+0000 front 2023-09-23T21:21:46.046399+0000 (oldest deadline 2023-09-23T21:22:09.546231+0000) 2023-09-23T21:22:32.869 INFO:journalctl@ceph.osd.6.smithi070.stdout:Sep 23 21:22:32 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd-6[193219]: 2023-09-23T21:22:32.583+0000 7f4171b44700 -1 osd.6 709 heartbeat_check: no reply from 172.21.15.26:6814 osd.1 since back 2023-09-23T21:21:52.947781+0000 front 2023-09-23T21:21:52.947835+0000 (oldest deadline 2023-09-23T21:22:18.247705+0000) 2023-09-23T21:22:32.869 INFO:journalctl@ceph.osd.6.smithi070.stdout:Sep 23 21:22:32 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd-6[193219]: 2023-09-23T21:22:32.583+0000 7f4171b44700 -1 osd.6 709 heartbeat_check: no reply from 172.21.15.26:6822 osd.2 since back 2023-09-23T21:22:02.849197+0000 front 2023-09-23T21:22:02.849150+0000 (oldest deadline 2023-09-23T21:22:28.749106+0000) 2023-09-23T21:22:33.370 INFO:journalctl@ceph.osd.7.smithi070.stdout:Sep 23 21:22:33 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd-7[128949]: 2023-09-23T21:22:33.008+0000 7f12105aa700 -1 osd.7 709 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2023-09-23T21:21:46.720306+0000 front 2023-09-23T21:21:46.720441+0000 (oldest deadline 2023-09-23T21:22:08.420326+0000) 2023-09-23T21:22:33.371 INFO:journalctl@ceph.osd.7.smithi070.stdout:Sep 23 21:22:33 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd-7[128949]: 2023-09-23T21:22:33.008+0000 7f12105aa700 -1 osd.7 709 heartbeat_check: no reply from 172.21.15.26:6814 osd.1 since back 2023-09-23T21:21:53.721516+0000 front 2023-09-23T21:21:53.721501+0000 (oldest deadline 2023-09-23T21:22:18.421368+0000) 2023-09-23T21:22:33.371 INFO:journalctl@ceph.osd.7.smithi070.stdout:Sep 23 21:22:33 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd-7[128949]: 2023-09-23T21:22:33.008+0000 7f12105aa700 -1 osd.7 709 heartbeat_check: no reply from 172.21.15.26:6822 osd.2 since back 2023-09-23T21:22:02.522110+0000 front 2023-09-23T21:22:02.522145+0000 (oldest deadline 2023-09-23T21:22:27.822275+0000) 2023-09-23T21:22:33.371 INFO:journalctl@ceph.osd.7.smithi070.stdout:Sep 23 21:22:33 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd-7[128949]: 2023-09-23T21:22:33.008+0000 7f12105aa700 -1 osd.7 709 heartbeat_check: no reply from 172.21.15.26:6830 osd.3 since back 2023-09-23T21:22:07.822900+0000 front 2023-09-23T21:22:07.822976+0000 (oldest deadline 2023-09-23T21:22:31.322875+0000) 2023-09-23T21:22:33.971 INFO:journalctl@ceph.osd.6.smithi070.stdout:Sep 23 21:22:33 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd-6[193219]: 2023-09-23T21:22:33.620+0000 7f4171b44700 -1 osd.6 709 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2023-09-23T21:21:46.046262+0000 front 2023-09-23T21:21:46.046399+0000 (oldest deadline 2023-09-23T21:22:09.546231+0000) 2023-09-23T21:22:33.971 INFO:journalctl@ceph.osd.6.smithi070.stdout:Sep 23 21:22:33 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd-6[193219]: 2023-09-23T21:22:33.620+0000 7f4171b44700 -1 osd.6 709 heartbeat_check: no reply from 172.21.15.26:6814 osd.1 since back 2023-09-23T21:21:52.947781+0000 front 2023-09-23T21:21:52.947835+0000 (oldest deadline 2023-09-23T21:22:18.247705+0000) 2023-09-23T21:22:33.971 INFO:journalctl@ceph.osd.6.smithi070.stdout:Sep 23 21:22:33 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd-6[193219]: 2023-09-23T21:22:33.620+0000 7f4171b44700 -1 osd.6 709 heartbeat_check: no reply from 172.21.15.26:6822 osd.2 since back 2023-09-23T21:22:02.849197+0000 front 2023-09-23T21:22:02.849150+0000 (oldest deadline 2023-09-23T21:22:28.749106+0000) 2023-09-23T21:22:34.369 INFO:journalctl@ceph.osd.7.smithi070.stdout:Sep 23 21:22:33 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd-7[128949]: 2023-09-23T21:22:33.970+0000 7f12105aa700 -1 osd.7 709 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2023-09-23T21:21:46.720306+0000 front 2023-09-23T21:21:46.720441+0000 (oldest deadline 2023-09-23T21:22:08.420326+0000) 2023-09-23T21:22:34.369 INFO:journalctl@ceph.osd.7.smithi070.stdout:Sep 23 21:22:33 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd-7[128949]: 2023-09-23T21:22:33.970+0000 7f12105aa700 -1 osd.7 709 heartbeat_check: no reply from 172.21.15.26:6814 osd.1 since back 2023-09-23T21:21:53.721516+0000 front 2023-09-23T21:21:53.721501+0000 (oldest deadline 2023-09-23T21:22:18.421368+0000) 2023-09-23T21:22:34.369 INFO:journalctl@ceph.osd.7.smithi070.stdout:Sep 23 21:22:33 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd-7[128949]: 2023-09-23T21:22:33.970+0000 7f12105aa700 -1 osd.7 709 heartbeat_check: no reply from 172.21.15.26:6822 osd.2 since back 2023-09-23T21:22:02.522110+0000 front 2023-09-23T21:22:02.522145+0000 (oldest deadline 2023-09-23T21:22:27.822275+0000) 2023-09-23T21:22:34.370 INFO:journalctl@ceph.osd.7.smithi070.stdout:Sep 23 21:22:33 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd-7[128949]: 2023-09-23T21:22:33.970+0000 7f12105aa700 -1 osd.7 709 heartbeat_check: no reply from 172.21.15.26:6830 osd.3 since back 2023-09-23T21:22:07.822900+0000 front 2023-09-23T21:22:07.822976+0000 (oldest deadline 2023-09-23T21:22:31.322875+0000) 2023-09-23T21:22:34.680 INFO:journalctl@ceph.osd.6.smithi070.stdout:Sep 23 21:22:34 smithi070 bash[309230]: ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd-6 2023-09-23T21:22:35.274 INFO:journalctl@ceph.osd.7.smithi070.stdout:Sep 23 21:22:34 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd-7[128949]: 2023-09-23T21:22:34.988+0000 7f12105aa700 -1 osd.7 709 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2023-09-23T21:21:46.720306+0000 front 2023-09-23T21:21:46.720441+0000 (oldest deadline 2023-09-23T21:22:08.420326+0000) 2023-09-23T21:22:35.274 INFO:journalctl@ceph.osd.7.smithi070.stdout:Sep 23 21:22:34 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd-7[128949]: 2023-09-23T21:22:34.988+0000 7f12105aa700 -1 osd.7 709 heartbeat_check: no reply from 172.21.15.26:6814 osd.1 since back 2023-09-23T21:21:53.721516+0000 front 2023-09-23T21:21:53.721501+0000 (oldest deadline 2023-09-23T21:22:18.421368+0000) 2023-09-23T21:22:35.275 INFO:journalctl@ceph.osd.7.smithi070.stdout:Sep 23 21:22:34 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd-7[128949]: 2023-09-23T21:22:34.988+0000 7f12105aa700 -1 osd.7 709 heartbeat_check: no reply from 172.21.15.26:6822 osd.2 since back 2023-09-23T21:22:02.522110+0000 front 2023-09-23T21:22:02.522145+0000 (oldest deadline 2023-09-23T21:22:27.822275+0000) 2023-09-23T21:22:35.275 INFO:journalctl@ceph.osd.7.smithi070.stdout:Sep 23 21:22:34 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd-7[128949]: 2023-09-23T21:22:34.988+0000 7f12105aa700 -1 osd.7 709 heartbeat_check: no reply from 172.21.15.26:6830 osd.3 since back 2023-09-23T21:22:07.822900+0000 front 2023-09-23T21:22:07.822976+0000 (oldest deadline 2023-09-23T21:22:31.322875+0000) 2023-09-23T21:22:35.942 DEBUG:teuthology.orchestra.run.smithi070:> sudo pkill -f 'journalctl -f -n 0 -u ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7@osd.6.service' 2023-09-23T21:22:35.981 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-09-23T21:22:35.982 INFO:tasks.cephadm.osd.6:Stopped osd.6 2023-09-23T21:22:35.982 INFO:tasks.cephadm.osd.7:Stopping osd.7... 2023-09-23T21:22:35.982 DEBUG:teuthology.orchestra.run.smithi070:> sudo systemctl stop ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7@osd.7 2023-09-23T21:22:36.369 INFO:journalctl@ceph.osd.7.smithi070.stdout:Sep 23 21:22:35 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd-7[128949]: 2023-09-23T21:22:35.971+0000 7f12105aa700 -1 osd.7 709 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2023-09-23T21:21:46.720306+0000 front 2023-09-23T21:21:46.720441+0000 (oldest deadline 2023-09-23T21:22:08.420326+0000) 2023-09-23T21:22:36.369 INFO:journalctl@ceph.osd.7.smithi070.stdout:Sep 23 21:22:35 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd-7[128949]: 2023-09-23T21:22:35.971+0000 7f12105aa700 -1 osd.7 709 heartbeat_check: no reply from 172.21.15.26:6814 osd.1 since back 2023-09-23T21:21:53.721516+0000 front 2023-09-23T21:21:53.721501+0000 (oldest deadline 2023-09-23T21:22:18.421368+0000) 2023-09-23T21:22:36.369 INFO:journalctl@ceph.osd.7.smithi070.stdout:Sep 23 21:22:35 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd-7[128949]: 2023-09-23T21:22:35.971+0000 7f12105aa700 -1 osd.7 709 heartbeat_check: no reply from 172.21.15.26:6822 osd.2 since back 2023-09-23T21:22:02.522110+0000 front 2023-09-23T21:22:02.522145+0000 (oldest deadline 2023-09-23T21:22:27.822275+0000) 2023-09-23T21:22:36.370 INFO:journalctl@ceph.osd.7.smithi070.stdout:Sep 23 21:22:35 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd-7[128949]: 2023-09-23T21:22:35.971+0000 7f12105aa700 -1 osd.7 709 heartbeat_check: no reply from 172.21.15.26:6830 osd.3 since back 2023-09-23T21:22:07.822900+0000 front 2023-09-23T21:22:07.822976+0000 (oldest deadline 2023-09-23T21:22:31.322875+0000) 2023-09-23T21:22:36.370 INFO:journalctl@ceph.osd.7.smithi070.stdout:Sep 23 21:22:36 smithi070 systemd[1]: Stopping Ceph osd.7 for b3f8e4ec-5a50-11ee-8db4-212e2dc638e7... 2023-09-23T21:22:36.869 INFO:journalctl@ceph.osd.7.smithi070.stdout:Sep 23 21:22:36 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd-7[128949]: 2023-09-23T21:22:36.370+0000 7f1216fb2700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-osd -n osd.7 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2023-09-23T21:22:36.869 INFO:journalctl@ceph.osd.7.smithi070.stdout:Sep 23 21:22:36 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd-7[128949]: 2023-09-23T21:22:36.370+0000 7f1216fb2700 -1 osd.7 709 *** Got signal Terminated *** 2023-09-23T21:22:36.869 INFO:journalctl@ceph.osd.7.smithi070.stdout:Sep 23 21:22:36 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd-7[128949]: 2023-09-23T21:22:36.370+0000 7f1216fb2700 -1 osd.7 709 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-09-23T21:22:37.369 INFO:journalctl@ceph.osd.7.smithi070.stdout:Sep 23 21:22:36 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd-7[128949]: 2023-09-23T21:22:36.996+0000 7f12105aa700 -1 osd.7 709 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2023-09-23T21:21:46.720306+0000 front 2023-09-23T21:21:46.720441+0000 (oldest deadline 2023-09-23T21:22:08.420326+0000) 2023-09-23T21:22:37.369 INFO:journalctl@ceph.osd.7.smithi070.stdout:Sep 23 21:22:36 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd-7[128949]: 2023-09-23T21:22:36.996+0000 7f12105aa700 -1 osd.7 709 heartbeat_check: no reply from 172.21.15.26:6814 osd.1 since back 2023-09-23T21:21:53.721516+0000 front 2023-09-23T21:21:53.721501+0000 (oldest deadline 2023-09-23T21:22:18.421368+0000) 2023-09-23T21:22:37.369 INFO:journalctl@ceph.osd.7.smithi070.stdout:Sep 23 21:22:36 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd-7[128949]: 2023-09-23T21:22:36.996+0000 7f12105aa700 -1 osd.7 709 heartbeat_check: no reply from 172.21.15.26:6822 osd.2 since back 2023-09-23T21:22:02.522110+0000 front 2023-09-23T21:22:02.522145+0000 (oldest deadline 2023-09-23T21:22:27.822275+0000) 2023-09-23T21:22:37.369 INFO:journalctl@ceph.osd.7.smithi070.stdout:Sep 23 21:22:36 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd-7[128949]: 2023-09-23T21:22:36.996+0000 7f12105aa700 -1 osd.7 709 heartbeat_check: no reply from 172.21.15.26:6830 osd.3 since back 2023-09-23T21:22:07.822900+0000 front 2023-09-23T21:22:07.822976+0000 (oldest deadline 2023-09-23T21:22:31.322875+0000) 2023-09-23T21:22:38.369 INFO:journalctl@ceph.osd.7.smithi070.stdout:Sep 23 21:22:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd-7[128949]: 2023-09-23T21:22:37.999+0000 7f12105aa700 -1 osd.7 709 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2023-09-23T21:21:46.720306+0000 front 2023-09-23T21:21:46.720441+0000 (oldest deadline 2023-09-23T21:22:08.420326+0000) 2023-09-23T21:22:38.370 INFO:journalctl@ceph.osd.7.smithi070.stdout:Sep 23 21:22:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd-7[128949]: 2023-09-23T21:22:37.999+0000 7f12105aa700 -1 osd.7 709 heartbeat_check: no reply from 172.21.15.26:6814 osd.1 since back 2023-09-23T21:21:53.721516+0000 front 2023-09-23T21:21:53.721501+0000 (oldest deadline 2023-09-23T21:22:18.421368+0000) 2023-09-23T21:22:38.370 INFO:journalctl@ceph.osd.7.smithi070.stdout:Sep 23 21:22:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd-7[128949]: 2023-09-23T21:22:37.999+0000 7f12105aa700 -1 osd.7 709 heartbeat_check: no reply from 172.21.15.26:6822 osd.2 since back 2023-09-23T21:22:02.522110+0000 front 2023-09-23T21:22:02.522145+0000 (oldest deadline 2023-09-23T21:22:27.822275+0000) 2023-09-23T21:22:38.370 INFO:journalctl@ceph.osd.7.smithi070.stdout:Sep 23 21:22:37 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd-7[128949]: 2023-09-23T21:22:37.999+0000 7f12105aa700 -1 osd.7 709 heartbeat_check: no reply from 172.21.15.26:6830 osd.3 since back 2023-09-23T21:22:07.822900+0000 front 2023-09-23T21:22:07.822976+0000 (oldest deadline 2023-09-23T21:22:31.322875+0000) 2023-09-23T21:22:39.369 INFO:journalctl@ceph.osd.7.smithi070.stdout:Sep 23 21:22:38 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd-7[128949]: 2023-09-23T21:22:38.973+0000 7f12105aa700 -1 osd.7 709 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2023-09-23T21:21:46.720306+0000 front 2023-09-23T21:21:46.720441+0000 (oldest deadline 2023-09-23T21:22:08.420326+0000) 2023-09-23T21:22:39.369 INFO:journalctl@ceph.osd.7.smithi070.stdout:Sep 23 21:22:38 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd-7[128949]: 2023-09-23T21:22:38.973+0000 7f12105aa700 -1 osd.7 709 heartbeat_check: no reply from 172.21.15.26:6814 osd.1 since back 2023-09-23T21:21:53.721516+0000 front 2023-09-23T21:21:53.721501+0000 (oldest deadline 2023-09-23T21:22:18.421368+0000) 2023-09-23T21:22:39.369 INFO:journalctl@ceph.osd.7.smithi070.stdout:Sep 23 21:22:38 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd-7[128949]: 2023-09-23T21:22:38.973+0000 7f12105aa700 -1 osd.7 709 heartbeat_check: no reply from 172.21.15.26:6822 osd.2 since back 2023-09-23T21:22:02.522110+0000 front 2023-09-23T21:22:02.522145+0000 (oldest deadline 2023-09-23T21:22:27.822275+0000) 2023-09-23T21:22:39.370 INFO:journalctl@ceph.osd.7.smithi070.stdout:Sep 23 21:22:38 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd-7[128949]: 2023-09-23T21:22:38.973+0000 7f12105aa700 -1 osd.7 709 heartbeat_check: no reply from 172.21.15.26:6830 osd.3 since back 2023-09-23T21:22:07.822900+0000 front 2023-09-23T21:22:07.822976+0000 (oldest deadline 2023-09-23T21:22:31.322875+0000) 2023-09-23T21:22:40.369 INFO:journalctl@ceph.osd.7.smithi070.stdout:Sep 23 21:22:39 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd-7[128949]: 2023-09-23T21:22:39.926+0000 7f12105aa700 -1 osd.7 709 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2023-09-23T21:21:46.720306+0000 front 2023-09-23T21:21:46.720441+0000 (oldest deadline 2023-09-23T21:22:08.420326+0000) 2023-09-23T21:22:40.369 INFO:journalctl@ceph.osd.7.smithi070.stdout:Sep 23 21:22:39 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd-7[128949]: 2023-09-23T21:22:39.926+0000 7f12105aa700 -1 osd.7 709 heartbeat_check: no reply from 172.21.15.26:6814 osd.1 since back 2023-09-23T21:21:53.721516+0000 front 2023-09-23T21:21:53.721501+0000 (oldest deadline 2023-09-23T21:22:18.421368+0000) 2023-09-23T21:22:40.370 INFO:journalctl@ceph.osd.7.smithi070.stdout:Sep 23 21:22:39 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd-7[128949]: 2023-09-23T21:22:39.926+0000 7f12105aa700 -1 osd.7 709 heartbeat_check: no reply from 172.21.15.26:6822 osd.2 since back 2023-09-23T21:22:02.522110+0000 front 2023-09-23T21:22:02.522145+0000 (oldest deadline 2023-09-23T21:22:27.822275+0000) 2023-09-23T21:22:40.370 INFO:journalctl@ceph.osd.7.smithi070.stdout:Sep 23 21:22:39 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd-7[128949]: 2023-09-23T21:22:39.926+0000 7f12105aa700 -1 osd.7 709 heartbeat_check: no reply from 172.21.15.26:6830 osd.3 since back 2023-09-23T21:22:07.822900+0000 front 2023-09-23T21:22:07.822976+0000 (oldest deadline 2023-09-23T21:22:31.322875+0000) 2023-09-23T21:22:40.370 INFO:journalctl@ceph.osd.7.smithi070.stdout:Sep 23 21:22:39 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd-7[128949]: 2023-09-23T21:22:39.926+0000 7f12105aa700 -1 osd.7 709 heartbeat_check: no reply from 172.21.15.70:6804 osd.4 since back 2023-09-23T21:22:17.223795+0000 front 2023-09-23T21:22:17.223776+0000 (oldest deadline 2023-09-23T21:22:39.523718+0000) 2023-09-23T21:22:41.370 INFO:journalctl@ceph.osd.7.smithi070.stdout:Sep 23 21:22:40 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd-7[128949]: 2023-09-23T21:22:40.928+0000 7f12105aa700 -1 osd.7 709 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2023-09-23T21:21:46.720306+0000 front 2023-09-23T21:21:46.720441+0000 (oldest deadline 2023-09-23T21:22:08.420326+0000) 2023-09-23T21:22:41.370 INFO:journalctl@ceph.osd.7.smithi070.stdout:Sep 23 21:22:40 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd-7[128949]: 2023-09-23T21:22:40.928+0000 7f12105aa700 -1 osd.7 709 heartbeat_check: no reply from 172.21.15.26:6814 osd.1 since back 2023-09-23T21:21:53.721516+0000 front 2023-09-23T21:21:53.721501+0000 (oldest deadline 2023-09-23T21:22:18.421368+0000) 2023-09-23T21:22:41.370 INFO:journalctl@ceph.osd.7.smithi070.stdout:Sep 23 21:22:40 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd-7[128949]: 2023-09-23T21:22:40.928+0000 7f12105aa700 -1 osd.7 709 heartbeat_check: no reply from 172.21.15.26:6822 osd.2 since back 2023-09-23T21:22:02.522110+0000 front 2023-09-23T21:22:02.522145+0000 (oldest deadline 2023-09-23T21:22:27.822275+0000) 2023-09-23T21:22:41.371 INFO:journalctl@ceph.osd.7.smithi070.stdout:Sep 23 21:22:40 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd-7[128949]: 2023-09-23T21:22:40.928+0000 7f12105aa700 -1 osd.7 709 heartbeat_check: no reply from 172.21.15.26:6830 osd.3 since back 2023-09-23T21:22:07.822900+0000 front 2023-09-23T21:22:07.822976+0000 (oldest deadline 2023-09-23T21:22:31.322875+0000) 2023-09-23T21:22:41.371 INFO:journalctl@ceph.osd.7.smithi070.stdout:Sep 23 21:22:40 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd-7[128949]: 2023-09-23T21:22:40.928+0000 7f12105aa700 -1 osd.7 709 heartbeat_check: no reply from 172.21.15.70:6804 osd.4 since back 2023-09-23T21:22:17.223795+0000 front 2023-09-23T21:22:17.223776+0000 (oldest deadline 2023-09-23T21:22:39.523718+0000) 2023-09-23T21:22:42.620 INFO:journalctl@ceph.osd.7.smithi070.stdout:Sep 23 21:22:42 smithi070 bash[309495]: ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-osd-7 2023-09-23T21:22:43.626 DEBUG:teuthology.orchestra.run.smithi070:> sudo pkill -f 'journalctl -f -n 0 -u ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7@osd.7.service' 2023-09-23T21:22:43.665 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-09-23T21:22:43.666 INFO:tasks.cephadm.osd.7:Stopped osd.7 2023-09-23T21:22:43.666 INFO:tasks.cephadm.prometheus.a:Stopping prometheus.a... 2023-09-23T21:22:43.666 DEBUG:teuthology.orchestra.run.smithi070:> sudo systemctl stop ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7@prometheus.a 2023-09-23T21:22:44.052 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 21:22:43 smithi070 systemd[1]: Stopping Ceph prometheus.a for b3f8e4ec-5a50-11ee-8db4-212e2dc638e7... 2023-09-23T21:22:44.369 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 21:22:44 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-prometheus-a[155975]: ts=2023-09-23T21:22:44.051Z caller=main.go:840 level=warn msg="Received SIGTERM, exiting gracefully..." 2023-09-23T21:22:44.370 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 21:22:44 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-prometheus-a[155975]: ts=2023-09-23T21:22:44.052Z caller=main.go:864 level=info msg="Stopping scrape discovery manager..." 2023-09-23T21:22:44.370 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 21:22:44 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-prometheus-a[155975]: ts=2023-09-23T21:22:44.052Z caller=main.go:878 level=info msg="Stopping notify discovery manager..." 2023-09-23T21:22:44.370 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 21:22:44 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-prometheus-a[155975]: ts=2023-09-23T21:22:44.052Z caller=manager.go:988 level=info component="rule manager" msg="Stopping rule manager..." 2023-09-23T21:22:44.370 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 21:22:44 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-prometheus-a[155975]: ts=2023-09-23T21:22:44.052Z caller=main.go:860 level=info msg="Scrape discovery manager stopped" 2023-09-23T21:22:44.370 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 21:22:44 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-prometheus-a[155975]: ts=2023-09-23T21:22:44.052Z caller=manager.go:998 level=info component="rule manager" msg="Rule manager stopped" 2023-09-23T21:22:44.370 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 21:22:44 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-prometheus-a[155975]: ts=2023-09-23T21:22:44.052Z caller=main.go:915 level=info msg="Stopping scrape manager..." 2023-09-23T21:22:44.371 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 21:22:44 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-prometheus-a[155975]: ts=2023-09-23T21:22:44.052Z caller=main.go:874 level=info msg="Notify discovery manager stopped" 2023-09-23T21:22:44.371 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 21:22:44 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-prometheus-a[155975]: ts=2023-09-23T21:22:44.052Z caller=main.go:907 level=info msg="Scrape manager stopped" 2023-09-23T21:22:44.371 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 21:22:44 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-prometheus-a[155975]: ts=2023-09-23T21:22:44.053Z caller=notifier.go:606 level=info component=notifier msg="Stopping notification manager..." 2023-09-23T21:22:44.371 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 21:22:44 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-prometheus-a[155975]: ts=2023-09-23T21:22:44.053Z caller=main.go:1135 level=info msg="Notifier manager stopped" 2023-09-23T21:22:44.371 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 21:22:44 smithi070 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-prometheus-a[155975]: ts=2023-09-23T21:22:44.053Z caller=main.go:1147 level=info msg="See you next time!" 2023-09-23T21:22:44.371 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 21:22:44 smithi070 bash[309762]: ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-prometheus-a 2023-09-23T21:22:44.581 DEBUG:teuthology.orchestra.run.smithi070:> sudo pkill -f 'journalctl -f -n 0 -u ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7@prometheus.a.service' 2023-09-23T21:22:44.615 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-09-23T21:22:44.615 INFO:tasks.cephadm.prometheus.a:Stopped prometheus.a 2023-09-23T21:22:44.616 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 --force --keep-logs 2023-09-23T21:22:46.382 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Sep 23 21:22:46 smithi026 systemd[1]: Stopping Ceph alertmanager.a for b3f8e4ec-5a50-11ee-8db4-212e2dc638e7... 2023-09-23T21:22:46.873 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Sep 23 21:22:46 smithi026 ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-alertmanager-a[163035]: ts=2023-09-23T21:22:46.449Z caller=main.go:583 level=info msg="Received SIGTERM, exiting gracefully..." 2023-09-23T21:22:47.131 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Sep 23 21:22:46 smithi026 bash[480823]: ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-alertmanager-a 2023-09-23T21:22:47.132 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Sep 23 21:22:47 smithi026 systemd[1]: ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7@alertmanager.a.service: Succeeded. 2023-09-23T21:22:47.132 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Sep 23 21:22:47 smithi026 systemd[1]: Stopped Ceph alertmanager.a for b3f8e4ec-5a50-11ee-8db4-212e2dc638e7. 2023-09-23T21:22:47.623 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 23 21:22:47 smithi026 systemd[1]: Stopping Ceph node-exporter.a for b3f8e4ec-5a50-11ee-8db4-212e2dc638e7... 2023-09-23T21:22:47.881 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 23 21:22:47 smithi026 bash[480946]: ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-node-exporter-a 2023-09-23T21:22:48.163 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 23 21:22:47 smithi026 systemd[1]: ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7@node-exporter.a.service: Main process exited, code=exited, status=143/n/a 2023-09-23T21:22:48.164 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 23 21:22:48 smithi026 systemd[1]: ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7@node-exporter.a.service: Failed with result 'exit-code'. 2023-09-23T21:22:48.164 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 23 21:22:48 smithi026 systemd[1]: Stopped Ceph node-exporter.a for b3f8e4ec-5a50-11ee-8db4-212e2dc638e7. 2023-09-23T21:22:55.811 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 --force --keep-logs 2023-09-23T21:22:57.370 INFO:journalctl@ceph.node-exporter.b.smithi070.stdout:Sep 23 21:22:57 smithi070 systemd[1]: Stopping Ceph node-exporter.b for b3f8e4ec-5a50-11ee-8db4-212e2dc638e7... 2023-09-23T21:22:57.869 INFO:journalctl@ceph.node-exporter.b.smithi070.stdout:Sep 23 21:22:57 smithi070 bash[310129]: ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-node-exporter-b 2023-09-23T21:22:57.869 INFO:journalctl@ceph.node-exporter.b.smithi070.stdout:Sep 23 21:22:57 smithi070 systemd[1]: ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7@node-exporter.b.service: Main process exited, code=exited, status=143/n/a 2023-09-23T21:22:58.369 INFO:journalctl@ceph.node-exporter.b.smithi070.stdout:Sep 23 21:22:57 smithi070 systemd[1]: ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7@node-exporter.b.service: Failed with result 'exit-code'. 2023-09-23T21:22:58.369 INFO:journalctl@ceph.node-exporter.b.smithi070.stdout:Sep 23 21:22:57 smithi070 systemd[1]: Stopped Ceph node-exporter.b for b3f8e4ec-5a50-11ee-8db4-212e2dc638e7. 2023-09-23T21:22:58.370 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 21:22:58 smithi070 systemd[1]: Stopping Ceph grafana.a for b3f8e4ec-5a50-11ee-8db4-212e2dc638e7... 2023-09-23T21:22:58.692 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 21:22:58 smithi070 bash[310256]: ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7-grafana-a 2023-09-23T21:22:58.980 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 21:22:58 smithi070 systemd[1]: ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7@grafana.a.service: Main process exited, code=exited, status=143/n/a 2023-09-23T21:22:58.980 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 21:22:58 smithi070 systemd[1]: ceph-b3f8e4ec-5a50-11ee-8db4-212e2dc638e7@grafana.a.service: Failed with result 'exit-code'. 2023-09-23T21:22:58.980 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 21:22:58 smithi070 systemd[1]: Stopped Ceph grafana.a for b3f8e4ec-5a50-11ee-8db4-212e2dc638e7. 2023-09-23T21:23:02.358 DEBUG:teuthology.orchestra.run.smithi026:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2023-09-23T21:23:02.397 DEBUG:teuthology.orchestra.run.smithi070:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2023-09-23T21:23:02.427 INFO:tasks.cephadm:Archiving crash dumps... 2023-09-23T21:23:02.429 DEBUG:teuthology.misc:Transferring archived files from smithi026:/var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/crash to /home/teuthworker/archive/teuthology-2023-09-23_14:23:10-upgrade:quincy-x-reef-distro-default-smithi/7401835/remote/smithi026/crash 2023-09-23T21:23:02.430 DEBUG:teuthology.orchestra.run.smithi026:> sudo tar c -f - -C /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/crash -- . 2023-09-23T21:23:02.493 INFO:teuthology.orchestra.run.smithi026.stderr:tar: /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/crash: Cannot open: No such file or directory 2023-09-23T21:23:02.493 INFO:teuthology.orchestra.run.smithi026.stderr:tar: Error is not recoverable: exiting now 2023-09-23T21:23:02.495 DEBUG:teuthology.misc:Transferring archived files from smithi070:/var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/crash to /home/teuthworker/archive/teuthology-2023-09-23_14:23:10-upgrade:quincy-x-reef-distro-default-smithi/7401835/remote/smithi070/crash 2023-09-23T21:23:02.495 DEBUG:teuthology.orchestra.run.smithi070:> sudo tar c -f - -C /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/crash -- . 2023-09-23T21:23:02.552 INFO:teuthology.orchestra.run.smithi070.stderr:tar: /var/lib/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/crash: Cannot open: No such file or directory 2023-09-23T21:23:02.553 INFO:teuthology.orchestra.run.smithi070.stderr:tar: Error is not recoverable: exiting now 2023-09-23T21:23:02.554 INFO:tasks.cephadm:Checking cluster log for badness... 2023-09-23T21:23:02.554 DEBUG:teuthology.orchestra.run.smithi026:> sudo egrep '\[ERR\]|\[WRN\]|\[SEC\]' /var/log/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/ceph.log | egrep -v '\(MDS_ALL_DOWN\)' | egrep -v '\(MDS_UP_LESS_THAN_MAX\)' | head -n 1 2023-09-23T21:23:02.589 INFO:teuthology.orchestra.run.smithi026.stderr:grep: /var/log/ceph/b3f8e4ec-5a50-11ee-8db4-212e2dc638e7/ceph.log: No such file or directory 2023-09-23T21:23:02.591 INFO:tasks.cephadm:Compressing logs... 2023-09-23T21:23:02.592 DEBUG:teuthology.orchestra.run.smithi026:> sudo find /var/log/ceph /var/log/rbd-target-api -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2023-09-23T21:23:02.633 DEBUG:teuthology.orchestra.run.smithi070:> sudo find /var/log/ceph /var/log/rbd-target-api -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2023-09-23T21:23:02.665 INFO:teuthology.orchestra.run.smithi070.stderr:find: ‘/var/log/rbd-target-api’: No such file or directory 2023-09-23T21:23:02.685 INFO:teuthology.orchestra.run.smithi026.stderr:find: ‘/var/log/rbd-target-api’: No such file or directory 2023-09-23T21:33:42.460 INFO:tasks.cephadm:Archiving logs... 2023-09-23T21:33:42.486 DEBUG:teuthology.misc:Transferring archived files from smithi026:/var/log/ceph to /home/teuthworker/archive/teuthology-2023-09-23_14:23:10-upgrade:quincy-x-reef-distro-default-smithi/7401835/remote/smithi026/log 2023-09-23T21:33:42.487 DEBUG:teuthology.orchestra.run.smithi026:> sudo tar c -f - -C /var/log/ceph -- . 2023-09-23T21:34:57.291 DEBUG:teuthology.misc:Transferring archived files from smithi070:/var/log/ceph to /home/teuthworker/archive/teuthology-2023-09-23_14:23:10-upgrade:quincy-x-reef-distro-default-smithi/7401835/remote/smithi070/log 2023-09-23T21:34:57.292 DEBUG:teuthology.orchestra.run.smithi070:> sudo tar c -f - -C /var/log/ceph -- . 2023-09-23T21:35:34.048 INFO:tasks.cephadm:Removing cluster... 2023-09-23T21:35:34.049 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 --force 2023-09-23T21:35:35.644 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid b3f8e4ec-5a50-11ee-8db4-212e2dc638e7 --force 2023-09-23T21:35:36.766 INFO:tasks.cephadm:Removing cephadm ... 2023-09-23T21:35:36.766 DEBUG:teuthology.orchestra.run.smithi026:> rm -rf /home/ubuntu/cephtest/cephadm 2023-09-23T21:35:36.789 DEBUG:teuthology.orchestra.run.smithi070:> rm -rf /home/ubuntu/cephtest/cephadm 2023-09-23T21:35:36.822 INFO:tasks.cephadm:Teardown complete 2023-09-23T21:35:36.822 DEBUG:teuthology.run_tasks:Unwinding manager install 2023-09-23T21:35:36.834 INFO:teuthology.task.install.util:Removing shipped files: /home/ubuntu/cephtest/valgrind.supp /usr/bin/daemon-helper /usr/bin/adjust-ulimits /usr/bin/stdin-killer... 2023-09-23T21:35:36.834 DEBUG:teuthology.orchestra.run.smithi026:> sudo rm -f -- /home/ubuntu/cephtest/valgrind.supp /usr/bin/daemon-helper /usr/bin/adjust-ulimits /usr/bin/stdin-killer 2023-09-23T21:35:36.837 DEBUG:teuthology.orchestra.run.smithi070:> sudo rm -f -- /home/ubuntu/cephtest/valgrind.supp /usr/bin/daemon-helper /usr/bin/adjust-ulimits /usr/bin/stdin-killer 2023-09-23T21:35:36.942 DEBUG:teuthology.run_tasks:Unwinding manager clock 2023-09-23T21:35:36.952 INFO:teuthology.task.clock:Checking final clock skew... 2023-09-23T21:35:36.953 DEBUG:teuthology.orchestra.run.smithi026:> PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2023-09-23T21:35:36.955 DEBUG:teuthology.orchestra.run.smithi070:> PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2023-09-23T21:35:36.971 INFO:teuthology.orchestra.run.smithi070.stderr:bash: ntpq: command not found 2023-09-23T21:35:36.973 INFO:teuthology.orchestra.run.smithi026.stderr:bash: ntpq: command not found 2023-09-23T21:35:36.999 INFO:teuthology.orchestra.run.smithi026.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2023-09-23T21:35:37.000 INFO:teuthology.orchestra.run.smithi026.stdout:=============================================================================== 2023-09-23T21:35:37.000 INFO:teuthology.orchestra.run.smithi026.stdout:^+ hv01.front.sepia.ceph.com 2 6 377 6 -1114us[-1134us] +/- 81ms 2023-09-23T21:35:37.000 INFO:teuthology.orchestra.run.smithi026.stdout:^* hv02.front.sepia.ceph.com 3 6 377 4 -84us[ -104us] +/- 73ms 2023-09-23T21:35:37.000 INFO:teuthology.orchestra.run.smithi026.stdout:^+ hv03.front.sepia.ceph.com 3 6 377 4 +1071us[+1050us] +/- 72ms 2023-09-23T21:35:37.000 INFO:teuthology.orchestra.run.smithi026.stdout:^? hv04.front.sepia.ceph.com 0 10 0 - +0ns[ +0ns] +/- 0ns 2023-09-23T21:35:37.011 INFO:teuthology.orchestra.run.smithi070.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2023-09-23T21:35:37.011 INFO:teuthology.orchestra.run.smithi070.stdout:=============================================================================== 2023-09-23T21:35:37.011 INFO:teuthology.orchestra.run.smithi070.stdout:^+ hv01.front.sepia.ceph.com 2 6 377 11 -1118us[-1102us] +/- 81ms 2023-09-23T21:35:37.012 INFO:teuthology.orchestra.run.smithi070.stdout:^* hv02.front.sepia.ceph.com 3 6 377 10 -76us[ -60us] +/- 73ms 2023-09-23T21:35:37.012 INFO:teuthology.orchestra.run.smithi070.stdout:^+ hv03.front.sepia.ceph.com 3 6 377 17 +1068us[+1084us] +/- 71ms 2023-09-23T21:35:37.012 INFO:teuthology.orchestra.run.smithi070.stdout:^? hv04.front.sepia.ceph.com 0 10 0 - +0ns[ +0ns] +/- 0ns 2023-09-23T21:35:37.012 DEBUG:teuthology.run_tasks:Unwinding manager ansible.cephlab 2023-09-23T21:35:37.021 INFO:teuthology.task.ansible:Skipping ansible cleanup... 2023-09-23T21:35:37.022 DEBUG:teuthology.run_tasks:Unwinding manager selinux 2023-09-23T21:35:37.031 DEBUG:teuthology.orchestra.run.smithi026:> mkdir /home/ubuntu/cephtest/archive/audit && sudo cp /var/log/audit/audit.log /home/ubuntu/cephtest/archive/audit && sudo chown $USER /home/ubuntu/cephtest/archive/audit/audit.log && gzip /home/ubuntu/cephtest/archive/audit/audit.log 2023-09-23T21:35:37.175 DEBUG:teuthology.orchestra.run.smithi070:> mkdir /home/ubuntu/cephtest/archive/audit && sudo cp /var/log/audit/audit.log /home/ubuntu/cephtest/archive/audit && sudo chown $USER /home/ubuntu/cephtest/archive/audit/audit.log && gzip /home/ubuntu/cephtest/archive/audit/audit.log 2023-09-23T21:35:37.284 DEBUG:teuthology.orchestra.run.smithi026:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 2023-09-23T21:35:37.320 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T21:35:37.320 DEBUG:teuthology.orchestra.run.smithi070:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 2023-09-23T21:35:37.350 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T21:35:37.351 DEBUG:teuthology.run_tasks:Unwinding manager pcp 2023-09-23T21:35:37.361 DEBUG:teuthology.run_tasks:Unwinding manager internal.timer 2023-09-23T21:35:37.371 INFO:teuthology.task.internal:Duration was 4428.635946 seconds 2023-09-23T21:35:37.371 DEBUG:teuthology.run_tasks:Unwinding manager internal.syslog 2023-09-23T21:35:37.381 INFO:teuthology.task.internal.syslog:Shutting down syslog monitoring... 2023-09-23T21:35:37.381 DEBUG:teuthology.orchestra.run.smithi026:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart 2023-09-23T21:35:37.383 DEBUG:teuthology.orchestra.run.smithi070:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart 2023-09-23T21:35:37.448 INFO:teuthology.orchestra.run.smithi026.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2023-09-23T21:35:37.490 INFO:teuthology.orchestra.run.smithi070.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2023-09-23T21:35:37.913 INFO:teuthology.task.internal.syslog:Checking logs for errors... 2023-09-23T21:35:37.914 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi026.front.sepia.ceph.com 2023-09-23T21:35:37.914 DEBUG:teuthology.orchestra.run.smithi026:> egrep --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | egrep -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | egrep -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 2023-09-23T21:35:37.939 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi070.front.sepia.ceph.com 2023-09-23T21:35:37.939 DEBUG:teuthology.orchestra.run.smithi070:> egrep --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | egrep -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | egrep -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 2023-09-23T21:35:37.973 INFO:teuthology.task.internal.syslog:Compressing syslogs... 2023-09-23T21:35:37.973 DEBUG:teuthology.orchestra.run.smithi026:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2023-09-23T21:35:37.981 DEBUG:teuthology.orchestra.run.smithi070:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2023-09-23T21:35:38.163 INFO:teuthology.task.internal.syslog:Gathering journactl -b0... 2023-09-23T21:35:38.163 DEBUG:teuthology.orchestra.run.smithi026:> sudo journalctl -b0 | gzip -9 > /home/ubuntu/cephtest/archive/syslog/journalctl-b0.gz 2023-09-23T21:35:38.165 DEBUG:teuthology.orchestra.run.smithi070:> sudo journalctl -b0 | gzip -9 > /home/ubuntu/cephtest/archive/syslog/journalctl-b0.gz 2023-09-23T21:35:38.549 DEBUG:teuthology.run_tasks:Unwinding manager internal.sudo 2023-09-23T21:35:38.561 INFO:teuthology.task.internal:Restoring /etc/sudoers... 2023-09-23T21:35:38.561 DEBUG:teuthology.orchestra.run.smithi026:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers 2023-09-23T21:35:38.602 DEBUG:teuthology.orchestra.run.smithi070:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers 2023-09-23T21:35:38.667 DEBUG:teuthology.run_tasks:Unwinding manager internal.coredump 2023-09-23T21:35:38.678 DEBUG:teuthology.orchestra.run.smithi026:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump 2023-09-23T21:35:38.680 DEBUG:teuthology.orchestra.run.smithi070:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump 2023-09-23T21:35:38.710 INFO:teuthology.orchestra.run.smithi026.stdout:kernel.core_pattern = core 2023-09-23T21:35:38.710 INFO:teuthology.orchestra.run.smithi070.stdout:kernel.core_pattern = core 2023-09-23T21:35:38.732 DEBUG:teuthology.orchestra.run.smithi026:> test -e /home/ubuntu/cephtest/archive/coredump 2023-09-23T21:35:38.781 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T21:35:38.781 DEBUG:teuthology.orchestra.run.smithi070:> test -e /home/ubuntu/cephtest/archive/coredump 2023-09-23T21:35:38.797 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T21:35:38.798 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive 2023-09-23T21:35:38.809 INFO:teuthology.task.internal:Transferring archived files... 2023-09-23T21:35:38.811 DEBUG:teuthology.misc:Transferring archived files from smithi026:/home/ubuntu/cephtest/archive to /home/teuthworker/archive/teuthology-2023-09-23_14:23:10-upgrade:quincy-x-reef-distro-default-smithi/7401835/remote/smithi026 2023-09-23T21:35:38.811 DEBUG:teuthology.orchestra.run.smithi026:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- . 2023-09-23T21:35:38.885 DEBUG:teuthology.misc:Transferring archived files from smithi070:/home/ubuntu/cephtest/archive to /home/teuthworker/archive/teuthology-2023-09-23_14:23:10-upgrade:quincy-x-reef-distro-default-smithi/7401835/remote/smithi070 2023-09-23T21:35:38.885 DEBUG:teuthology.orchestra.run.smithi070:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- . 2023-09-23T21:35:38.935 INFO:teuthology.task.internal:Removing archive directory... 2023-09-23T21:35:38.935 DEBUG:teuthology.orchestra.run.smithi026:> rm -rf -- /home/ubuntu/cephtest/archive 2023-09-23T21:35:38.938 DEBUG:teuthology.orchestra.run.smithi070:> rm -rf -- /home/ubuntu/cephtest/archive 2023-09-23T21:35:38.981 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive_upload 2023-09-23T21:35:38.998 INFO:teuthology.task.internal:Not uploading archives. 2023-09-23T21:35:38.998 DEBUG:teuthology.run_tasks:Unwinding manager internal.base 2023-09-23T21:35:39.009 INFO:teuthology.task.internal:Tidying up after the test... 2023-09-23T21:35:39.009 DEBUG:teuthology.orchestra.run.smithi026:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest 2023-09-23T21:35:39.011 DEBUG:teuthology.orchestra.run.smithi070:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest 2023-09-23T21:35:39.026 INFO:teuthology.orchestra.run.smithi026.stdout: 262307 4 drwxr-xr-x 3 ubuntu ubuntu 4096 Sep 23 21:35 /home/ubuntu/cephtest 2023-09-23T21:35:39.026 INFO:teuthology.orchestra.run.smithi026.stdout: 4325377 4 drwxrwxr-x 3 ubuntu ubuntu 4096 Sep 23 21:19 /home/ubuntu/cephtest/mnt.0 2023-09-23T21:35:39.026 INFO:teuthology.orchestra.run.smithi026.stdout: 4325378 4 drwxrwxr-x 2 ubuntu ubuntu 4096 Sep 23 21:19 /home/ubuntu/cephtest/mnt.0/client.0 2023-09-23T21:35:39.038 INFO:teuthology.orchestra.run.smithi070.stdout: 262320 4 drwxr-xr-x 2 ubuntu ubuntu 4096 Sep 23 21:35 /home/ubuntu/cephtest 2023-09-23T21:35:39.048 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T21:35:39.048 INFO:teuthology.orchestra.run.smithi026.stderr:rmdir: failed to remove '/home/ubuntu/cephtest': Directory not empty 2023-09-23T21:35:39.049 ERROR:teuthology.run_tasks:Manager failed: internal.base Traceback (most recent call last): File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/run_tasks.py", line 154, in run_tasks suppress = manager.__exit__(*exc_info) File "/usr/lib/python3.8/contextlib.py", line 120, in __exit__ next(self.gen) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/task/internal/__init__.py", line 53, in base run.wait( File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/orchestra/run.py", line 479, in wait proc.wait() File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/orchestra/run.py", line 161, in wait self._raise_for_status() File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/orchestra/run.py", line 181, in _raise_for_status raise CommandFailedError( teuthology.exceptions.CommandFailedError: Command failed on smithi026 with status 1: 'find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest' 2023-09-23T21:35:39.055 DEBUG:teuthology.run_tasks:Unwinding manager kernel 2023-09-23T21:35:39.067 DEBUG:teuthology.run_tasks:Unwinding manager console_log 2023-09-23T21:35:39.078 DEBUG:teuthology.run_tasks:Exception was not quenched, exiting: CommandFailedError: Command failed on smithi026 with status 1: 'find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest' 2023-09-23T21:35:39.078 INFO:teuthology.nuke:Checking targets against current locks 2023-09-23T21:35:39.108 DEBUG:teuthology.nuke:shortname: smithi026 2023-09-23T21:35:39.108 INFO:teuthology.task.internal.check_lock:Checking locks... 2023-09-23T21:35:39.131 DEBUG:teuthology.nuke:shortname: smithi070 2023-09-23T21:35:39.132 INFO:teuthology.task.internal.check_lock:Checking locks... 2023-09-23T21:35:39.136 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi026.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2023-09-23_14:23:10-upgrade:quincy-x-reef-distro-default-smithi/7401835', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'rhel', 'os_version': '8.6', 'arch': 'x86_64', 'locked': True, 'locked_since': '2023-09-23 20:17:05.031747', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBLVvN3+LmzPZ4ixiwyCkS3KZLZ9R6MOPm+kPtaJbl8fExpzbk8TudXCcJWHBL7DGPWFvye2jT6sei9eiX0EUQYs='} 2023-09-23T21:35:39.153 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi070.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2023-09-23_14:23:10-upgrade:quincy-x-reef-distro-default-smithi/7401835', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'rhel', 'os_version': '8.6', 'arch': 'x86_64', 'locked': True, 'locked_since': '2023-09-23 20:17:05.034040', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBLVvN3+LmzPZ4ixiwyCkS3KZLZ9R6MOPm+kPtaJbl8fExpzbk8TudXCcJWHBL7DGPWFvye2jT6sei9eiX0EUQYs='} 2023-09-23T21:35:39.186 INFO:teuthology.orchestra.console.smithi026:Power off 2023-09-23T21:35:39.186 DEBUG:teuthology.orchestra.console.smithi026:pexpect command: ipmitool -H smithi026.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power off 2023-09-23T21:35:39.201 INFO:teuthology.orchestra.console.smithi070:Power off 2023-09-23T21:35:39.202 DEBUG:teuthology.orchestra.console.smithi070:pexpect command: ipmitool -H smithi070.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power off 2023-09-23T21:35:39.210 DEBUG:teuthology.orchestra.console.smithi026:power off output: Chassis Power Control: Down/Off 2023-09-23T21:35:39.210 DEBUG:teuthology.orchestra.console.smithi026:pexpect command: ipmitool -H smithi026.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2023-09-23T21:35:39.223 DEBUG:teuthology.orchestra.console.smithi070:power off output: Chassis Power Control: Down/Off 2023-09-23T21:35:39.224 DEBUG:teuthology.orchestra.console.smithi070:pexpect command: ipmitool -H smithi070.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2023-09-23T21:35:39.239 DEBUG:teuthology.orchestra.console.smithi026:check power output: Chassis Power is on 2023-09-23T21:35:39.245 DEBUG:teuthology.orchestra.console.smithi070:check power output: Chassis Power is on 2023-09-23T21:35:43.241 DEBUG:teuthology.orchestra.console.smithi026:pexpect command: ipmitool -H smithi026.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2023-09-23T21:35:43.249 DEBUG:teuthology.orchestra.console.smithi070:pexpect command: ipmitool -H smithi070.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2023-09-23T21:35:43.351 DEBUG:teuthology.orchestra.console.smithi026:check power output: Chassis Power is on 2023-09-23T21:35:43.359 DEBUG:teuthology.orchestra.console.smithi070:check power output: Chassis Power is on 2023-09-23T21:35:47.354 DEBUG:teuthology.orchestra.console.smithi026:pexpect command: ipmitool -H smithi026.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2023-09-23T21:35:47.362 DEBUG:teuthology.orchestra.console.smithi070:pexpect command: ipmitool -H smithi070.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2023-09-23T21:35:47.464 DEBUG:teuthology.orchestra.console.smithi026:check power output: Chassis Power is off 2023-09-23T21:35:47.471 DEBUG:teuthology.orchestra.console.smithi070:check power output: Chassis Power is off 2023-09-23T21:35:47.566 INFO:teuthology.orchestra.console.smithi026:Power off completed 2023-09-23T21:35:47.572 INFO:teuthology.orchestra.console.smithi070:Power off completed 2023-09-23T21:35:47.676 INFO:teuthology.run:Summary data: description: upgrade:quincy-x/stress-split/{0-distro/rhel_8.6_container_tools_rhel8 0-roles 1-start 2-first-half-tasks/rbd-cls 3-stress-tasks/{radosbench rbd-cls rbd-import-export rbd_api readwrite snaps-few-objects} 4-second-half-tasks/radosbench mon_election/connectivity} duration: 4428.635946035385 failure_reason: '' flavor: default owner: scheduled_teuthology@teuthology sentry_event: https://sentry.ceph.com/organizations/ceph/?query=40a38caae7284f1cac6fe0ed040e8d56 status: fail success: false 2023-09-23T21:35:47.677 DEBUG:teuthology.report:Pushing job info to https://paddles.front.sepia.ceph.com/ 2023-09-23T21:35:47.750 INFO:teuthology.run:FAIL